diff --git a/.ci/compute_projects.py b/.ci/compute_projects.py
index 2dc5629328457..8567552fa25a6 100644
--- a/.ci/compute_projects.py
+++ b/.ci/compute_projects.py
@@ -49,8 +49,7 @@
         "flang",
     },
     "lld": {"bolt", "cross-project-tests"},
-    # TODO(issues/132795): LLDB should be enabled on clang changes.
-    "clang": {"clang-tools-extra", "cross-project-tests"},
+    "clang": {"clang-tools-extra", "cross-project-tests", "lldb"},
     "mlir": {"flang"},
     # Test everything if ci scripts are changed.
     ".ci": {
@@ -80,6 +79,7 @@
     "clang": {"compiler-rt"},
     "clang-tools-extra": {"libc"},
     "libc": {"libc"},
+    "compiler-rt": {"compiler-rt"},
     ".ci": {"compiler-rt", "libc"},
 }
 DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {
@@ -100,6 +100,9 @@
     "libc",  # No Windows Support.
     "lldb",  # TODO(issues/132800): Needs environment setup.
     "bolt",  # No Windows Support.
+    "libcxx",
+    "libcxxabi",
+    "libunwind",
 }
 
 # These are projects that we should test if the project itself is changed but
@@ -118,6 +121,9 @@
     "lldb",
     "openmp",
     "polly",
+    "libcxx",
+    "libcxxabi",
+    "libunwind",
 }
 
 PROJECT_CHECK_TARGETS = {
diff --git a/.ci/compute_projects_test.py b/.ci/compute_projects_test.py
index bd761198300a8..7d780b51ca5d1 100644
--- a/.ci/compute_projects_test.py
+++ b/.ci/compute_projects_test.py
@@ -45,16 +45,14 @@ def test_llvm_windows(self):
             env_variables["project_check_targets"],
             "check-clang check-clang-tools check-lld check-llvm check-mlir check-polly",
         )
-        self.assertEqual(
-            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
-        )
+        self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(
             env_variables["runtimes_check_targets"],
             "",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets_needs_reconfig"],
-            "check-cxx check-cxxabi check-unwind",
+            "",
         )
 
     def test_llvm_mac(self):
@@ -69,16 +67,14 @@ def test_llvm_mac(self):
             env_variables["project_check_targets"],
             "check-clang check-clang-tools check-lld check-llvm check-mlir",
         )
-        self.assertEqual(
-            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
-        )
+        self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(
             env_variables["runtimes_check_targets"],
             "",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets_needs_reconfig"],
-            "check-cxx check-cxxabi check-unwind",
+            "",
         )
 
     def test_clang(self):
@@ -87,11 +83,11 @@ def test_clang(self):
         )
         self.assertEqual(
             env_variables["projects_to_build"],
-            "clang;clang-tools-extra;lld;llvm",
+            "clang;clang-tools-extra;lld;lldb;llvm",
         )
         self.assertEqual(
             env_variables["project_check_targets"],
-            "check-clang check-clang-tools",
+            "check-clang check-clang-tools check-lldb",
         )
         self.assertEqual(
             env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"
@@ -119,30 +115,54 @@ def test_clang_windows(self):
         self.assertEqual(
             env_variables["project_check_targets"], "check-clang check-clang-tools"
         )
-        self.assertEqual(
-            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
-        )
+        self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(
             env_variables["runtimes_check_targets"],
             "",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets_needs_reconfig"],
-            "check-cxx check-cxxabi check-unwind",
+            "",
         )
         self.assertEqual(env_variables["enable_cir"], "OFF")
 
+    def test_compiler_rt(self):
+        env_variables = compute_projects.get_env_variables(
+            ["compiler-rt/lib/asan/asan_allocator.cpp"], "Linux"
+        )
+        self.assertEqual(
+            env_variables["projects_to_build"],
+            "clang;lld",
+        )
+        self.assertEqual(
+            env_variables["project_check_targets"],
+            "",
+        )
+        self.assertEqual(env_variables["runtimes_to_build"], "compiler-rt")
+        self.assertEqual(
+            env_variables["runtimes_check_targets"],
+            "check-compiler-rt",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["enable_cir"],
+            "OFF",
+        )
+
     def test_cir(self):
         env_variables = compute_projects.get_env_variables(
             ["clang/lib/CIR/CMakeLists.txt"], "Linux"
         )
         self.assertEqual(
             env_variables["projects_to_build"],
-            "clang;clang-tools-extra;lld;llvm;mlir",
+            "clang;clang-tools-extra;lld;lldb;llvm;mlir",
         )
         self.assertEqual(
             env_variables["project_check_targets"],
-            "check-clang check-clang-cir check-clang-tools",
+            "check-clang check-clang-cir check-clang-tools check-lldb",
         )
         self.assertEqual(
             env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"
@@ -298,18 +318,15 @@ def test_windows_ci(self):
         )
         self.assertEqual(
             env_variables["runtimes_to_build"],
-            "libcxx;libcxxabi;libunwind",
+            "",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
             "",
         )
-        # TODO(boomanaiden154): We should not be emitting these on Windows.
-        # It does not currently impact anything because we do not build the
-        # runtimes on Windows though.
         self.assertEqual(
             env_variables["runtimes_check_targets_needs_reconfig"],
-            "check-cxx check-cxxabi check-unwind",
+            "",
         )
 
     def test_lldb(self):
diff --git a/.ci/generate_test_report_github.py b/.ci/generate_test_report_github.py
index be43c4b4f693f..7242264723cbf 100644
--- a/.ci/generate_test_report_github.py
+++ b/.ci/generate_test_report_github.py
@@ -4,20 +4,25 @@
 """Script to generate a build report for Github."""
 
 import argparse
+import platform
 
 import generate_test_report_lib
 
+PLATFORM_TITLES = {
+    "Windows": ":window: Windows x64 Test Results",
+    "Linux": ":penguin: Linux x64 Test Results",
+}
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
+    parser.add_argument("return_code", help="The build's return code.", type=int)
     parser.add_argument(
-        "title", help="Title of the test report, without Markdown formatting."
+        "build_test_logs", help="Paths to JUnit report files and ninja logs.", nargs="*"
     )
-    parser.add_argument("return_code", help="The build's return code.", type=int)
-    parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*")
     args = parser.parse_args()
 
     report = generate_test_report_lib.generate_report_from_files(
-        args.title, args.return_code, args.junit_files
+        PLATFORM_TITLES[platform.system()], args.return_code, args.build_test_logs
     )
 
     print(report)
diff --git a/.ci/generate_test_report_lib.py b/.ci/generate_test_report_lib.py
index 25d810f1c6d17..d868c08ab69ef 100644
--- a/.ci/generate_test_report_lib.py
+++ b/.ci/generate_test_report_lib.py
@@ -12,6 +12,84 @@
     "https://github.com/llvm/llvm-project/issues and add the "
     "`infrastructure` label."
 )
+# The maximum number of lines to pull from a ninja failure.
+NINJA_LOG_SIZE_THRESHOLD = 500
+
+
+def _parse_ninja_log(ninja_log: list[str]) -> list[tuple[str, str]]:
+    """Parses an individual ninja log."""
+    failures = []
+    index = 0
+    while index < len(ninja_log):
+        while index < len(ninja_log) and not ninja_log[index].startswith("FAILED:"):
+            index += 1
+        if index == len(ninja_log):
+            # We hit the end of the log without finding a build failure, go to
+            # the next log.
+            return failures
+        # We are trying to parse cases like the following:
+        #
+        # [4/5] test/4.stamp
+        # FAILED: touch test/4.stamp
+        # touch test/4.stamp
+        #
+        # index will point to the line that starts with Failed:. The progress
+        # indicator is the line before this ([4/5] test/4.stamp) and contains a pretty
+        # printed version of the target being built (test/4.stamp). We use this line
+        # and remove the progress information to get a succinct name for the target.
+        failing_action = ninja_log[index - 1].split("] ")[1]
+        failure_log = []
+        while (
+            index < len(ninja_log)
+            and not ninja_log[index].startswith("[")
+            and not ninja_log[index].startswith("ninja: build stopped:")
+            and len(failure_log) < NINJA_LOG_SIZE_THRESHOLD
+        ):
+            failure_log.append(ninja_log[index])
+            index += 1
+        failures.append((failing_action, "\n".join(failure_log)))
+    return failures
+
+
+def find_failure_in_ninja_logs(ninja_logs: list[list[str]]) -> list[tuple[str, str]]:
+    """Extracts failure messages from ninja output.
+
+    This function takes stdout/stderr from ninja in the form of a list of files
+    represented as a list of lines. This function then returns tuples containing
+    the name of the target and the error message.
+
+    Args:
+      ninja_logs: A list of files in the form of a list of lines representing the log
+        files captured from ninja.
+
+    Returns:
+      A list of tuples. The first string is the name of the target that failed. The
+      second string is the error message.
+    """
+    failures = []
+    for ninja_log in ninja_logs:
+        log_failures = _parse_ninja_log(ninja_log)
+        failures.extend(log_failures)
+    return failures
+
+
+def _format_ninja_failures(ninja_failures: list[tuple[str, str]]) -> list[str]:
+    """Formats ninja failures into summary views for the report."""
+    output = []
+    for build_failure in ninja_failures:
+        failed_action, failure_message = build_failure
+        output.extend(
+            [
+                "<details>",
+                f"<summary>{failed_action}</summary>",
+                "",
+                "```",
+                failure_message,
+                "```",
+                "</details>",
+            ]
+        )
+    return output
 
 
 # Set size_limit to limit the byte size of the report. The default is 1MB as this
@@ -24,6 +102,7 @@ def generate_report(
     title,
     return_code,
     junit_objects,
+    ninja_logs: list[list[str]],
     size_limit=1024 * 1024,
     list_failures=True,
 ):
@@ -61,15 +140,34 @@ def generate_report(
                 ]
             )
         else:
-            report.extend(
-                [
-                    "The build failed before running any tests.",
-                    "",
-                    SEE_BUILD_FILE_STR,
-                    "",
-                    UNRELATED_FAILURES_STR,
-                ]
-            )
+            ninja_failures = find_failure_in_ninja_logs(ninja_logs)
+            if not ninja_failures:
+                report.extend(
+                    [
+                        "The build failed before running any tests. Detailed "
+                        "information about the build failure could not be "
+                        "automatically obtained.",
+                        "",
+                        SEE_BUILD_FILE_STR,
+                        "",
+                        UNRELATED_FAILURES_STR,
+                    ]
+                )
+            else:
+                report.extend(
+                    [
+                        "The build failed before running any tests. Click on a "
+                        "failure below to see the details.",
+                        "",
+                    ]
+                )
+                report.extend(_format_ninja_failures(ninja_failures))
+                report.extend(
+                    [
+                        "",
+                        UNRELATED_FAILURES_STR,
+                    ]
+                )
         return "\n".join(report)
 
     tests_passed = tests_run - tests_skipped - tests_failed
@@ -114,14 +212,28 @@ def plural(num_tests):
     elif return_code != 0:
         # No tests failed but the build was in a failed state. Bring this to the user's
         # attention.
-        report.extend(
-            [
-                "",
-                "All tests passed but another part of the build **failed**.",
-                "",
-                SEE_BUILD_FILE_STR,
-            ]
-        )
+        ninja_failures = find_failure_in_ninja_logs(ninja_logs)
+        if not ninja_failures:
+            report.extend(
+                [
+                    "",
+                    "All tests passed but another part of the build **failed**. "
+                    "Information about the build failure could not be automatically "
+                    "obtained.",
+                    "",
+                    SEE_BUILD_FILE_STR,
+                ]
+            )
+        else:
+            report.extend(
+                [
+                    "",
+                    "All tests passed but another part of the build **failed**. Click on "
+                    "a failure below to see the details.",
+                    "",
+                ]
+            )
+            report.extend(_format_ninja_failures(ninja_failures))
 
     if failures or return_code != 0:
         report.extend(["", UNRELATED_FAILURES_STR])
@@ -139,9 +251,19 @@ def plural(num_tests):
     return report
 
 
-def generate_report_from_files(title, return_code, junit_files):
+def generate_report_from_files(title, return_code, build_log_files):
+    junit_files = [
+        junit_file for junit_file in build_log_files if junit_file.endswith(".xml")
+    ]
+    ninja_log_files = [
+        ninja_log for ninja_log in build_log_files if ninja_log.endswith(".log")
+    ]
+    ninja_logs = []
+    for ninja_log_file in ninja_log_files:
+        with open(ninja_log_file, "r") as ninja_log_file_handle:
+            ninja_logs.append(
+                [log_line.strip() for log_line in ninja_log_file_handle.readlines()]
+            )
     return generate_report(
-        title,
-        return_code,
-        [JUnitXml.fromfile(p) for p in junit_files],
+        title, return_code, [JUnitXml.fromfile(p) for p in junit_files], ninja_logs
     )
diff --git a/.ci/generate_test_report_lib_test.py b/.ci/generate_test_report_lib_test.py
index eda76ead19b9d..466a8234776dc 100644
--- a/.ci/generate_test_report_lib_test.py
+++ b/.ci/generate_test_report_lib_test.py
@@ -8,6 +8,8 @@
 import unittest
 from io import StringIO
 from textwrap import dedent
+import tempfile
+import os
 
 from junitparser import JUnitXml
 
@@ -19,9 +21,114 @@ def junit_from_xml(xml):
 
 
 class TestReports(unittest.TestCase):
+    def test_find_failure_ninja_logs(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/5] test/1.stamp",
+                    "[2/5] test/2.stamp",
+                    "[3/5] test/3.stamp",
+                    "[4/5] test/4.stamp",
+                    "FAILED: touch test/4.stamp",
+                    "Wow! This system is really broken!",
+                    "[5/5] test/5.stamp",
+                ],
+            ]
+        )
+        self.assertEqual(len(failures), 1)
+        self.assertEqual(
+            failures[0],
+            (
+                "test/4.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/4.stamp
+                    Wow! This system is really broken!"""
+                ),
+            ),
+        )
+
+    def test_no_failure_ninja_log(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/3] test/1.stamp",
+                    "[2/3] test/2.stamp",
+                    "[3/3] test/3.stamp",
+                ]
+            ]
+        )
+        self.assertEqual(failures, [])
+
+    def test_ninja_log_end(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/3] test/1.stamp",
+                    "[2/3] test/2.stamp",
+                    "[3/3] test/3.stamp",
+                    "FAILED: touch test/3.stamp",
+                    "Wow! This system is really broken!",
+                    "ninja: build stopped: subcommand failed.",
+                ]
+            ]
+        )
+        self.assertEqual(len(failures), 1)
+        self.assertEqual(
+            failures[0],
+            (
+                "test/3.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/3.stamp
+                    Wow! This system is really broken!"""
+                ),
+            ),
+        )
+
+    def test_ninja_log_multiple_failures(self):
+        failures = generate_test_report_lib.find_failure_in_ninja_logs(
+            [
+                [
+                    "[1/5] test/1.stamp",
+                    "[2/5] test/2.stamp",
+                    "FAILED: touch test/2.stamp",
+                    "Wow! This system is really broken!",
+                    "[3/5] test/3.stamp",
+                    "[4/5] test/4.stamp",
+                    "FAILED: touch test/4.stamp",
+                    "Wow! This system is maybe broken!",
+                    "[5/5] test/5.stamp",
+                ]
+            ]
+        )
+        self.assertEqual(len(failures), 2)
+        self.assertEqual(
+            failures[0],
+            (
+                "test/2.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/2.stamp
+                    Wow! This system is really broken!"""
+                ),
+            ),
+        )
+        self.assertEqual(
+            failures[1],
+            (
+                "test/4.stamp",
+                dedent(
+                    """\
+                    FAILED: touch test/4.stamp
+                    Wow! This system is maybe broken!"""
+                ),
+            ),
+        )
+
     def test_title_only(self):
         self.assertEqual(
-            generate_test_report_lib.generate_report("Foo", 0, []),
+            generate_test_report_lib.generate_report("Foo", 0, [], []),
             dedent(
                 """\
                 # Foo
@@ -32,12 +139,12 @@ def test_title_only(self):
 
     def test_title_only_failure(self):
         self.assertEqual(
-            generate_test_report_lib.generate_report("Foo", 1, []),
+            generate_test_report_lib.generate_report("Foo", 1, [], []),
             dedent(
                 """\
             # Foo
 
-            The build failed before running any tests.
+            The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
 
             Download the build's log file to see the details.
 
@@ -45,6 +152,45 @@ def test_title_only_failure(self):
             ),
         )
 
+    def test_title_only_failure_ninja_log(self):
+        self.assertEqual(
+            generate_test_report_lib.generate_report(
+                "Foo",
+                1,
+                [],
+                [
+                    [
+                        "[1/5] test/1.stamp",
+                        "[2/5] test/2.stamp",
+                        "[3/5] test/3.stamp",
+                        "[4/5] test/4.stamp",
+                        "FAILED: test/4.stamp",
+                        "touch test/4.stamp",
+                        "Wow! Risk!",
+                        "[5/5] test/5.stamp",
+                    ]
+                ],
+            ),
+            dedent(
+                """\
+            # Foo
+
+            The build failed before running any tests. Click on a failure below to see the details.
+
+            <details>
+            <summary>test/4.stamp</summary>
+
+            ```
+            FAILED: test/4.stamp
+            touch test/4.stamp
+            Wow! Risk!
+            ```
+            </details>
+            
+            If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+            ),
+        )
+
     def test_no_tests_in_testsuite(self):
         self.assertEqual(
             generate_test_report_lib.generate_report(
@@ -62,12 +208,13 @@ def test_no_tests_in_testsuite(self):
                         )
                     )
                 ],
+                [],
             ),
             dedent(
                 """\
                 # Foo
 
-                The build failed before running any tests.
+                The build failed before running any tests. Detailed information about the build failure could not be automatically obtained.
 
                 Download the build's log file to see the details.
 
@@ -93,6 +240,7 @@ def test_no_failures(self):
                         )
                     )
                 ],
+                [],
             ),
             (
                 dedent(
@@ -122,6 +270,7 @@ def test_no_failures_build_failed(self):
                         )
                     )
                 ],
+                [],
             ),
             (
                 dedent(
@@ -130,7 +279,7 @@ def test_no_failures_build_failed(self):
 
               * 1 test passed
 
-              All tests passed but another part of the build **failed**.
+              All tests passed but another part of the build **failed**. Information about the build failure could not be automatically obtained.
 
               Download the build's log file to see the details.
               
@@ -139,6 +288,155 @@ def test_no_failures_build_failed(self):
             ),
         )
 
+    def test_no_failures_build_failed_ninja_log(self):
+        self.assertEqual(
+            generate_test_report_lib.generate_report(
+                "Foo",
+                1,
+                [
+                    junit_from_xml(
+                        dedent(
+                            """\
+          <?xml version="1.0" encoding="UTF-8"?>
+          <testsuites time="0.00">
+          <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+          <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+          </testsuite>
+          </testsuites>"""
+                        )
+                    )
+                ],
+                [
+                    [
+                        "[1/5] test/1.stamp",
+                        "[2/5] test/2.stamp",
+                        "[3/5] test/3.stamp",
+                        "[4/5] test/4.stamp",
+                        "FAILED: test/4.stamp",
+                        "touch test/4.stamp",
+                        "Wow! Close To You!",
+                        "[5/5] test/5.stamp",
+                    ]
+                ],
+            ),
+            (
+                dedent(
+                    """\
+                    # Foo
+
+                    * 1 test passed
+
+                    All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+                    <details>
+                    <summary>test/4.stamp</summary>
+
+                    ```
+                    FAILED: test/4.stamp
+                    touch test/4.stamp
+                    Wow! Close To You!
+                    ```
+                    </details>
+
+                    If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+                )
+            ),
+        )
+
+    def test_no_failures_multiple_build_failed_ninja_log(self):
+        test = generate_test_report_lib.generate_report(
+            "Foo",
+            1,
+            [
+                junit_from_xml(
+                    dedent(
+                        """\
+          <?xml version="1.0" encoding="UTF-8"?>
+          <testsuites time="0.00">
+          <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+          <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+          </testsuite>
+          </testsuites>"""
+                    )
+                )
+            ],
+            [
+                [
+                    "[1/5] test/1.stamp",
+                    "[2/5] test/2.stamp",
+                    "FAILED: touch test/2.stamp",
+                    "Wow! Be Kind!",
+                    "[3/5] test/3.stamp",
+                    "[4/5] test/4.stamp",
+                    "FAILED: touch test/4.stamp",
+                    "Wow! I Dare You!",
+                    "[5/5] test/5.stamp",
+                ]
+            ],
+        )
+        print(test)
+        self.assertEqual(
+            generate_test_report_lib.generate_report(
+                "Foo",
+                1,
+                [
+                    junit_from_xml(
+                        dedent(
+                            """\
+          <?xml version="1.0" encoding="UTF-8"?>
+          <testsuites time="0.00">
+          <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+          <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+          </testsuite>
+          </testsuites>"""
+                        )
+                    )
+                ],
+                [
+                    [
+                        "[1/5] test/1.stamp",
+                        "[2/5] test/2.stamp",
+                        "FAILED: touch test/2.stamp",
+                        "Wow! Be Kind!",
+                        "[3/5] test/3.stamp",
+                        "[4/5] test/4.stamp",
+                        "FAILED: touch test/4.stamp",
+                        "Wow! I Dare You!",
+                        "[5/5] test/5.stamp",
+                    ]
+                ],
+            ),
+            (
+                dedent(
+                    """\
+                    # Foo
+
+                    * 1 test passed
+
+                    All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+                    <details>
+                    <summary>test/2.stamp</summary>
+
+                    ```
+                    FAILED: touch test/2.stamp
+                    Wow! Be Kind!
+                    ```
+                    </details>
+                    <details>
+                    <summary>test/4.stamp</summary>
+
+                    ```
+                    FAILED: touch test/4.stamp
+                    Wow! I Dare You!
+                    ```
+                    </details>
+
+                    If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+                )
+            ),
+        )
+
     def test_report_single_file_single_testsuite(self):
         self.assertEqual(
             generate_test_report_lib.generate_report(
@@ -166,6 +464,7 @@ def test_report_single_file_single_testsuite(self):
                         )
                     )
                 ],
+                [],
             ),
             (
                 dedent(
@@ -261,6 +560,7 @@ def test_report_single_file_multiple_testsuites(self):
                         )
                     )
                 ],
+                [],
             ),
             self.MULTI_SUITE_OUTPUT,
         )
@@ -302,6 +602,7 @@ def test_report_multiple_files_multiple_testsuites(self):
                         )
                     ),
                 ],
+                [],
             ),
             self.MULTI_SUITE_OUTPUT,
         )
@@ -326,6 +627,7 @@ def test_report_dont_list_failures(self):
                         )
                     )
                 ],
+                [],
                 list_failures=False,
             ),
             (
@@ -362,6 +664,7 @@ def test_report_dont_list_failures_link_to_log(self):
                         )
                     )
                 ],
+                [],
                 list_failures=False,
             ),
             (
@@ -401,6 +704,7 @@ def test_report_size_limit(self):
                         )
                     )
                 ],
+                [],
                 size_limit=512,
             ),
             (
@@ -416,3 +720,59 @@ def test_report_size_limit(self):
                 )
             ),
         )
+
+    def test_generate_report_end_to_end(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            junit_xml_file = os.path.join(temp_dir, "junit.xml")
+            with open(junit_xml_file, "w") as junit_xml_handle:
+                junit_xml_handle.write(
+                    dedent(
+                        """\
+                        <?xml version="1.0" encoding="UTF-8"?>
+                        <testsuites time="0.00">
+                        <testsuite name="Passed" tests="1" failures="0" skipped="0" time="0.00">
+                        <testcase classname="Bar/test_1" name="test_1" time="0.00"/>
+                        </testsuite>
+                        </testsuites>"""
+                    )
+                )
+            ninja_log_file = os.path.join(temp_dir, "ninja.log")
+            with open(ninja_log_file, "w") as ninja_log_handle:
+                ninja_log_handle.write(
+                    dedent(
+                        """\
+                        [1/5] test/1.stamp
+                        [2/5] test/2.stamp
+                        [3/5] test/3.stamp
+                        [4/5] test/4.stamp
+                        FAILED: test/4.stamp
+                        touch test/4.stamp
+                        Wow! That's so True!
+                        [5/5] test/5.stamp"""
+                    )
+                )
+            self.assertEqual(
+                generate_test_report_lib.generate_report_from_files(
+                    "Foo", 1, [junit_xml_file, ninja_log_file]
+                ),
+                dedent(
+                    """\
+                    # Foo
+
+                    * 1 test passed
+
+                    All tests passed but another part of the build **failed**. Click on a failure below to see the details.
+
+                    <details>
+                    <summary>test/4.stamp</summary>
+
+                    ```
+                    FAILED: test/4.stamp
+                    touch test/4.stamp
+                    Wow! That's so True!
+                    ```
+                    </details>
+
+                    If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label."""
+                ),
+            )
diff --git a/.ci/metrics/metrics.py b/.ci/metrics/metrics.py
index 26fdeef1913ab..0896b193b6b03 100644
--- a/.ci/metrics/metrics.py
+++ b/.ci/metrics/metrics.py
@@ -28,7 +28,10 @@
 # Lists the Github workflows we want to track. Maps the Github job name to
 # the metric name prefix in grafana.
 # This metric name is also used as a key in the job->name map.
-GITHUB_WORKFLOW_TO_TRACK = {"CI Checks": "github_llvm_premerge_checks"}
+GITHUB_WORKFLOW_TO_TRACK = {
+    "CI Checks": "github_llvm_premerge_checks",
+    "Build and Test libc++": "github_libcxx_premerge_checks",
+}
 
 # Lists the Github jobs to track for a given workflow. The key is the stable
 # name (metric name) of the workflow (see GITHUB_WORKFLOW_TO_TRACK).
@@ -38,7 +41,12 @@
     "github_llvm_premerge_checks": {
         "Build and Test Linux": "premerge_linux",
         "Build and Test Windows": "premerge_windows",
-    }
+    },
+    "github_libcxx_premerge_checks": {
+        "stage1": "premerge_libcxx_stage1",
+        "stage2": "premerge_libcxx_stage2",
+        "stage3": "premerge_libcxx_stage3",
+    },
 }
 
 # The number of workflows to pull when sampling Github workflows.
@@ -62,13 +70,14 @@
 # by trial and error).
 GRAFANA_METRIC_MAX_AGE_MN = 120
 
-
 @dataclass
 class JobMetrics:
     job_name: str
     queue_time: int
     run_time: int
     status: int
+    created_at_ns: int
+    started_at_ns: int
     completed_at_ns: int
     workflow_id: int
     workflow_name: str
@@ -81,6 +90,159 @@ class GaugeMetric:
     time_ns: int
 
 
+@dataclass
+class AggregateMetric:
+    aggregate_name: str
+    aggregate_queue_time: int
+    aggregate_run_time: int
+    aggregate_status: int
+    completed_at_ns: int
+    workflow_id: int
+
+
+def _construct_aggregate(ag_name: str, job_list: list[JobMetrics]) -> AggregateMetric:
+    """Create a libc++ AggregateMetric from a list of libc++ JobMetrics
+
+    How aggregates are computed:
+    queue time: Time from when first job in group is created until last job in
+                group has started.
+    run time: Time from when first job in group starts running until last job
+              in group finishes running.
+    status: logical 'and' of all the job statuses in the group.
+
+    Args:
+      ag_name: The name for this particular AggregateMetric
+      job_list: This list of JobMetrics to be combined into the AggregateMetric.
+        The input list should contain all (and only!) the libc++ JobMetrics
+        for a particular stage and a particular workflow_id.
+
+    Returns:
+      Returns the AggregateMetric constructed from the inputs.
+    """
+
+    # Initialize the aggregate values
+    earliest_create = job_list[0].created_at_ns
+    earliest_start = job_list[0].started_at_ns
+    earliest_complete = job_list[0].completed_at_ns
+    latest_start = job_list[0].started_at_ns
+    latest_complete = job_list[0].completed_at_ns
+    ag_status = job_list[0].status
+    ag_workflow_id = job_list[0].workflow_id
+
+    # Go through rest of jobs for this workflow id, if any, updating stats
+    for job in job_list[1:]:
+        # Update the status
+        ag_status = ag_status and job.status
+        # Get the earliest & latest times
+        if job.created_at_ns < earliest_create:
+            earliest_create = job.created_at_ns
+        if job.completed_at_ns < earliest_complete:
+            earliest_complete = job.completed_at_ns
+        if job.started_at_ns > latest_start:
+            latest_start = job.started_at_ns
+        if job.started_at_ns < earliest_start:
+            earliest_start = job.started_at_ns
+        if job.completed_at_ns > latest_complete:
+            latest_complete = job.completed_at_ns
+
+    # Compute aggregate run time (in seconds, not ns)
+    ag_run_time = (latest_complete - earliest_start) / 1000000000
+    # Compute aggregate queue time (in seconds, not ns)
+    ag_queue_time = (latest_start - earliest_create) / 1000000000
+    # Append the aggregate metrics to the workflow metrics list.
+    return AggregateMetric(
+        ag_name, ag_queue_time, ag_run_time, ag_status, latest_complete, ag_workflow_id
+    )
+
+
+def create_and_append_libcxx_aggregates(workflow_metrics: list[JobMetrics]):
+    """Find libc++ JobMetric entries and create aggregate metrics for them.
+
+    Sort the libc++ JobMetric entries by workflow id, and for each workflow
+    id group them by stages. Call _construct_aggregate to reate an aggregate
+    metric for each stage for each unique workflow id. Append each aggregate
+    metric to the input workflow_metrics list.
+
+     Args:
+      workflow_metrics: A list of JobMetrics entries collected so far.
+    """
+    # Separate the jobs by workflow_id. Only look at JobMetrics entries.
+    aggregate_data = dict()
+    for job in workflow_metrics:
+        # Only want to look at JobMetrics
+        if not isinstance(job, JobMetrics):
+            continue
+        # Only want libc++ jobs.
+        if job.workflow_name != "Build and Test libc++":
+            continue
+        if job.workflow_id not in aggregate_data.keys():
+            aggregate_data[job.workflow_id] = [job]
+        else:
+            aggregate_data[job.workflow_id].append(job)
+
+    # Go through each aggregate_data list (workflow id) and find all the
+    # needed data
+    for ag_workflow_id in aggregate_data:
+        job_list = aggregate_data[ag_workflow_id]
+        stage1_jobs = list()
+        stage2_jobs = list()
+        stage3_jobs = list()
+        # sort jobs into stage1, stage2, & stage3.
+        for job in job_list:
+            if job.job_name.find("stage1") > 0:
+                stage1_jobs.append(job)
+            elif job.job_name.find("stage2") > 0:
+                stage2_jobs.append(job)
+            elif job.job_name.find("stage3") > 0:
+                stage3_jobs.append(job)
+
+        if len(stage1_jobs) > 0:
+            aggregate = _construct_aggregate(
+                "github_libcxx_premerge_checks_stage1_aggregate", stage1_jobs
+            )
+            workflow_metrics.append(aggregate)
+        if len(stage2_jobs) > 0:
+            aggregate = _construct_aggregate(
+                "github_libcxx_premerge_checks_stage2_aggregate", stage2_jobs
+            )
+            workflow_metrics.append(aggregate)
+        if len(stage3_jobs) > 0:
+            aggregate = _construct_aggregate(
+                "github_libcxx_premerge_checks_stage3_aggregate", stage3_jobs
+            )
+            workflow_metrics.append(aggregate)
+
+
+def clean_up_libcxx_job_name(old_name: str) -> str:
+    """Convert libcxx job names to generically legal strings.
+
+    Args:
+      old_name: A string with the full name of the libc++ test that was run.
+
+    Returns:
+      Returns the input string with characters that might not be acceptable
+        in some indentifier strings replaced with safer characters.
+
+    Take a name like 'stage1 (generic-cxx03, clang-22, clang++-22)'
+    and convert it to 'stage1_generic_cxx03__clang_22__clangxx_22'.
+    (Remove parentheses; replace commas, hyphens and spaces with
+    underscores; replace '+' with 'x'.)
+    """
+    # Names should have exactly one set of parentheses, so break on that. If
+    # they don't have any parentheses, then don't update them at all.
+    if old_name.find("(") == -1:
+        return old_name
+    stage, remainder = old_name.split("(")
+    stage = stage.strip()
+    if remainder[-1] == ")":
+        remainder = remainder[:-1]
+    remainder = remainder.replace("-", "_")
+    remainder = remainder.replace(",", "_")
+    remainder = remainder.replace(" ", "_")
+    remainder = remainder.replace("+", "x")
+    new_name = stage + "_" + remainder
+    return new_name
+
 def github_get_metrics(
     github_repo: github.Repository, last_workflows_seen_as_completed: set[int]
 ) -> tuple[list[JobMetrics], int]:
@@ -146,6 +308,10 @@ def github_get_metrics(
         if task.name not in GITHUB_WORKFLOW_TO_TRACK:
             continue
 
+        libcxx_testing = False
+        if task.name == "Build and Test libc++":
+            libcxx_testing = True
+
         if task.status == "completed":
             workflow_seen_as_completed.add(task.id)
 
@@ -155,11 +321,19 @@ def github_get_metrics(
 
         name_prefix = GITHUB_WORKFLOW_TO_TRACK[task.name]
         for job in task.jobs():
+            if libcxx_testing:
+                # We're not running macos or windows libc++ tests on our
+                # infrastructure.
+                if job.name.find("macos") != -1 or job.name.find("windows") != -1:
+                    continue
             # This job is not interesting to us.
-            if job.name not in GITHUB_JOB_TO_TRACK[name_prefix]:
+            elif job.name not in GITHUB_JOB_TO_TRACK[name_prefix]:
                 continue
 
-            name_suffix = GITHUB_JOB_TO_TRACK[name_prefix][job.name]
+            if libcxx_testing:
+                name_suffix = clean_up_libcxx_job_name(job.name)
+            else:
+                name_suffix = GITHUB_JOB_TO_TRACK[name_prefix][job.name]
             metric_name = name_prefix + "_" + name_suffix
 
             if task.status != "completed":
@@ -208,8 +382,13 @@ def github_get_metrics(
                 continue
 
             logging.info(f"Adding a job metric for job {job.id} in workflow {task.id}")
-            # The timestamp associated with the event is expected by Grafana to be
-            # in nanoseconds.
+            # The completed_at_ns timestamp associated with the event is
+            # expected by Grafana to be in nanoseconds. Because we do math using
+            # all three times (when creating libc++ aggregates), we need them
+            # all to be in nanoseconds, even though created_at and started_at
+            # are not returned to Grafana.
+            created_at_ns = int(created_at.timestamp()) * 10**9
+            started_at_ns = int(started_at.timestamp()) * 10**9
             completed_at_ns = int(completed_at.timestamp()) * 10**9
             workflow_metrics.append(
                 JobMetrics(
@@ -217,12 +396,18 @@ def github_get_metrics(
                     queue_time.seconds,
                     run_time.seconds,
                     job_result,
+                    created_at_ns,
+                    started_at_ns,
                     completed_at_ns,
                     task.id,
                     task.name,
                 )
             )
 
+    # Finished collecting the JobMetrics for all jobs; now create the
+    # aggregates for any libc++ jobs.
+    create_and_append_libcxx_aggregates(workflow_metrics)
+
     for name, value in queued_count.items():
         workflow_metrics.append(
             GaugeMetric(f"workflow_queue_size_{name}", value, time.time_ns())
@@ -278,6 +463,11 @@ def upload_metrics(workflow_metrics, metrics_userid, api_key):
             metrics_batch.append(
                 f"{name} queue_time={workflow_metric.queue_time},run_time={workflow_metric.run_time},status={workflow_metric.status} {workflow_metric.completed_at_ns}"
             )
+        elif isinstance(workflow_metric, AggregateMetric):
+            name = workflow_metric.aggregate_name.lower().replace(" ", "_")
+            metrics_batch.append(
+                f"{name} queue_time={workflow_metric.aggregate_queue_time},run_time={workflow_metric.aggregate_run_time},status={workflow_metric.aggregate_status} {workflow_metric.completed_at_ns}"
+            )
         else:
             raise ValueError(
                 f"Unsupported object type {type(workflow_metric)}: {str(workflow_metric)}"
diff --git a/.ci/metrics/metrics_test.py b/.ci/metrics/metrics_test.py
index 259e55f817939..406b68ae4d4f1 100644
--- a/.ci/metrics/metrics_test.py
+++ b/.ci/metrics/metrics_test.py
@@ -36,7 +36,9 @@ def test_upload_gauge_metric(self):
     def test_upload_job_metric(self):
         """Test that we can upload a job metric correctly."""
         test_metrics = [
-            metrics.JobMetrics("test_job", 5, 10, 1, 1000, 7, "test_workflow")
+            metrics.JobMetrics(
+                "test_job", 5, 10, 1, 1000, 1000, 1000, 7, "test_workflow"
+            )
         ]
         return_value = requests.Response()
         return_value.status_code = 204
@@ -49,6 +51,22 @@ def test_upload_job_metric(self):
                 "test_job queue_time=5,run_time=10,status=1 1000",
             )
 
+    def test_upload_aggregate_metric(self):
+        """Test that we can upload an aggregate metric correctly."""
+        test_metrics = [
+            metrics.AggregateMetric("stage1_aggregate", 211, 1124, 1, 1200, 9)
+        ]
+        return_value = requests.Response()
+        return_value.status_code = 204
+        with unittest.mock.patch(
+            "requests.post", return_value=return_value
+        ) as post_mock:
+            metrics.upload_metrics(test_metrics, "test_userid", "test_aoi_key")
+            self.assertEqual(
+                post_mock.call_args.kwargs["data"],
+                "stage1_aggregate queue_time=211,run_time=1124,status=1 1200",
+            )
+
     def test_upload_unknown_metric(self):
         """Test we report an error if we encounter an unknown metric type."""
 
@@ -70,6 +88,326 @@ def test_bad_response_code(self):
         with unittest.mock.patch("requests.post", return_value=return_value) as _:
             metrics.upload_metrics(test_metrics, "test_userid", "test_api_key")
 
+    def test_create_and_append_aggregate_metric_1_stage(self):
+        """Test the creation of a single AggregateMetric"""
+        test_metrics = [
+            metrics.JobMetrics(
+                "libcxx_stage1_test1",
+                8,
+                388,
+                1,
+                created_at_ns=1755697953000000000,
+                started_at_ns=1755697961000000000,
+                completed_at_ns=1755698349000000000,
+                workflow_id=3,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage1_test2",
+                107,
+                357,
+                1,
+                created_at_ns=1755697953000000000,
+                started_at_ns=1755698060000000000,
+                completed_at_ns=1755698417000000000,
+                workflow_id=3,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage1_test3",
+                8,
+                824,
+                1,
+                created_at_ns=1755697953000000000,
+                started_at_ns=1755697961000000000,
+                completed_at_ns=1755698785000000000,
+                workflow_id=3,
+                workflow_name="Build and Test libc++",
+            ),
+        ]
+        metrics.create_and_append_libcxx_aggregates(test_metrics)
+        self.assertEqual(len(test_metrics), 4)
+        self.assertTrue(isinstance(test_metrics[-1], metrics.AggregateMetric))
+        aggregate = test_metrics[-1]
+        self.assertEqual(
+            aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"
+        )
+        self.assertEqual(aggregate.aggregate_queue_time, 107)
+        self.assertEqual(aggregate.aggregate_run_time, 824)
+        self.assertEqual(aggregate.aggregate_status, 1)
+        self.assertEqual(aggregate.completed_at_ns, 1755698785000000000)
+        self.assertEqual(aggregate.workflow_id, 3)
+
+    def test_create_and_append_aggregate_metric_multiple_workflow_ids(self):
+        """Test creation of AggregateMetric for same stage with diff workflow ids."""
+        test_metrics = [
+            metrics.JobMetrics(
+                "libcxx_stage1_test1",
+                8,
+                388,
+                1,
+                created_at_ns=1755697953000000000,
+                started_at_ns=1755697961000000000,
+                completed_at_ns=1755698349000000000,
+                workflow_id=3,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage1_test2",
+                107,
+                357,
+                0,
+                created_at_ns=1755697953000000000,
+                started_at_ns=1755698060000000000,
+                completed_at_ns=1755698417000000000,
+                workflow_id=3,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage1_test3",
+                8,
+                824,
+                1,
+                created_at_ns=1755697953000000000,
+                started_at_ns=1755697961000000000,
+                completed_at_ns=1755698785000000000,
+                workflow_id=25,
+                workflow_name="Build and Test libc++",
+            ),
+        ]
+        metrics.create_and_append_libcxx_aggregates(test_metrics)
+        self.assertEqual(len(test_metrics), 5)
+        self.assertTrue(isinstance(test_metrics[3], metrics.AggregateMetric))
+        self.assertTrue(isinstance(test_metrics[4], metrics.AggregateMetric))
+        aggregate = test_metrics[3]
+        self.assertEqual(
+            aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"
+        )
+        self.assertEqual(aggregate.aggregate_queue_time, 107)
+        self.assertEqual(aggregate.aggregate_run_time, 456)
+        self.assertEqual(aggregate.aggregate_status, 0)
+        self.assertEqual(aggregate.completed_at_ns, 1755698417000000000)
+        self.assertEqual(aggregate.workflow_id, 3)
+
+        aggregate = test_metrics[4]
+        self.assertEqual(
+            aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"
+        )
+        self.assertEqual(aggregate.aggregate_queue_time, 8)
+        self.assertEqual(aggregate.aggregate_run_time, 824)
+        self.assertEqual(aggregate.aggregate_status, 1)
+        self.assertEqual(aggregate.completed_at_ns, 1755698785000000000)
+        self.assertEqual(aggregate.workflow_id, 25)
+
+    def test_create_and_append_aggregate_metric_3_stages(self):
+        """Test the creation of AggregateMetric for each of 3 stages."""
+        test_metrics = [
+            metrics.JobMetrics(
+                "libcxx_stage1_test1",
+                124,
+                1454,
+                1,
+                created_at_ns=1755696929000000000,
+                started_at_ns=1755697053000000000,
+                completed_at_ns=1755698507000000000,
+                workflow_id=17,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage1_test2",
+                129,
+                827,
+                1,
+                created_at_ns=1755696929000000000,
+                started_at_ns=1755697058000000000,
+                completed_at_ns=1755697885000000000,
+                workflow_id=17,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage2_test1",
+                6,
+                580,
+                1,
+                created_at_ns=1755698507000000000,
+                started_at_ns=1755698513000000000,
+                completed_at_ns=1755699093000000000,
+                workflow_id=17,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage2_test2",
+                7,
+                473,
+                1,
+                created_at_ns=1755698507000000000,
+                started_at_ns=1755698514000000000,
+                completed_at_ns=1755698987000000000,
+                workflow_id=17,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage2_test3",
+                7,
+                820,
+                1,
+                created_at_ns=1755698507000000000,
+                started_at_ns=1755698514000000000,
+                completed_at_ns=1755699334000000000,
+                workflow_id=17,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage3_test1",
+                7,
+                919,
+                1,
+                created_at_ns=1755709005000000000,
+                started_at_ns=1755709012000000000,
+                completed_at_ns=1755709931000000000,
+                workflow_id=17,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage3_test2",
+                141,
+                834,
+                1,
+                created_at_ns=1755709005000000000,
+                started_at_ns=1755709146000000000,
+                completed_at_ns=1755709980000000000,
+                workflow_id=17,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage3_test3",
+                131,
+                370,
+                1,
+                created_at_ns=1755709005000000000,
+                started_at_ns=1755709136000000000,
+                completed_at_ns=1755709514000000000,
+                workflow_id=17,
+                workflow_name="Build and Test libc++",
+            ),
+        ]
+        metrics.create_and_append_libcxx_aggregates(test_metrics)
+        self.assertEqual(len(test_metrics), 11)
+        self.assertTrue(isinstance(test_metrics[8], metrics.AggregateMetric))
+        self.assertTrue(isinstance(test_metrics[9], metrics.AggregateMetric))
+        self.assertTrue(isinstance(test_metrics[10], metrics.AggregateMetric))
+        aggregate = test_metrics[8]
+        self.assertEqual(
+            aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"
+        )
+        self.assertEqual(aggregate.aggregate_queue_time, 129)
+        self.assertEqual(aggregate.aggregate_run_time, 1454)
+        self.assertEqual(aggregate.aggregate_status, 1)
+        self.assertEqual(aggregate.completed_at_ns, 1755698507000000000)
+        self.assertEqual(aggregate.workflow_id, 17)
+
+        aggregate = test_metrics[9]
+        self.assertEqual(
+            aggregate.aggregate_name, "github_libcxx_premerge_checks_stage2_aggregate"
+        )
+        self.assertEqual(aggregate.aggregate_queue_time, 7)
+        self.assertEqual(aggregate.aggregate_run_time, 821)
+        self.assertEqual(aggregate.aggregate_status, 1)
+        self.assertEqual(aggregate.completed_at_ns, 1755699334000000000)
+        self.assertEqual(aggregate.workflow_id, 17)
+
+        aggregate = test_metrics[10]
+        self.assertEqual(
+            aggregate.aggregate_name, "github_libcxx_premerge_checks_stage3_aggregate"
+        )
+        self.assertEqual(aggregate.aggregate_queue_time, 141)
+        self.assertEqual(aggregate.aggregate_run_time, 968)
+        self.assertEqual(aggregate.aggregate_status, 1)
+        self.assertEqual(aggregate.completed_at_ns, 1755709980000000000)
+        self.assertEqual(aggregate.workflow_id, 17)
+
+    def test_create_and_append_aggregate_metric_mixed_job_types(self):
+        """Test the creation of AggregateMetric with non-lib++ jobs thrown in."""
+        test_metrics = [
+            metrics.JobMetrics(
+                "ci_test1", 5, 10, 1, 1000, 1200, 1400, 5, "premerge_test"
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage1_test1",
+                8,
+                388,
+                1,
+                created_at_ns=1755697953000000000,
+                started_at_ns=1755697961000000000,
+                completed_at_ns=1755698349000000000,
+                workflow_id=3,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "ci_test2", 3, 20, 1, 2000, 2200, 2400, 37, "premerge_test"
+            ),
+            metrics.JobMetrics(
+                "libcxx_stage1_test2",
+                107,
+                357,
+                0,
+                created_at_ns=1755697953000000000,
+                started_at_ns=1755698060000000000,
+                completed_at_ns=1755698417000000000,
+                workflow_id=3,
+                workflow_name="Build and Test libc++",
+            ),
+            metrics.JobMetrics(
+                "ci_test3", 7, 35, 1, 3000, 3200, 3400, 85, "premerge_test"
+            ),
+        ]
+        metrics.create_and_append_libcxx_aggregates(test_metrics)
+        self.assertEqual(len(test_metrics), 6)
+        self.assertTrue(isinstance(test_metrics[5], metrics.AggregateMetric))
+        aggregate = test_metrics[5]
+        self.assertEqual(
+            aggregate.aggregate_name, "github_libcxx_premerge_checks_stage1_aggregate"
+        )
+        self.assertEqual(aggregate.aggregate_queue_time, 107)
+        self.assertEqual(aggregate.aggregate_run_time, 456)
+        self.assertEqual(aggregate.aggregate_status, 0)
+        self.assertEqual(aggregate.completed_at_ns, 1755698417000000000)
+        self.assertEqual(aggregate.workflow_id, 3)
+
+    def test_create_and_append_aggregate_metric_no_libcxx_jobs(self):
+        """Test the creation of AggregateMetric with no libc++ jobs.
+
+        In this case, no AggregateMetric should be created, but no
+        errors or complaints should be raised.
+        """
+        test_metrics = [
+            metrics.JobMetrics(
+                "ci_test1", 5, 10, 1, 1000, 1200, 1400, 5, "premerge_test"
+            ),
+            metrics.JobMetrics(
+                "ci_test2", 3, 20, 1, 2000, 2200, 2400, 37, "premerge_test"
+            ),
+            metrics.JobMetrics(
+                "ci_test3", 7, 35, 1, 3000, 3200, 3400, 85, "premerge_test"
+            ),
+        ]
+        metrics.create_and_append_libcxx_aggregates(test_metrics)
+        self.assertEqual(len(test_metrics), 3)
+
+    def test_clean_up_libcxx_job_name(self):
+        """Test that we correctly update (or not) libcxx job names."""
+        stage1_name = "stage1 (test1, C++-test2, my-c++-test-25)"
+        stage2_name = "stage2 (generic-cxx26, clang-21, clang++21)"
+        stage3_name = "stage3 (generic-cxx26, libcxx-next-runners, junk)"
+        bad_name = "this is a bad name"
+        out_name1 = metrics.clean_up_libcxx_job_name(stage1_name)
+        self.assertEqual(out_name1, "stage1_test1__Cxx_test2__my_cxx_test_25")
+        out_name2 = metrics.clean_up_libcxx_job_name(stage2_name)
+        self.assertEqual(out_name2, "stage2_generic_cxx26__clang_21__clangxx21")
+        out_name3 = metrics.clean_up_libcxx_job_name(stage3_name)
+        self.assertEqual(out_name3, "stage3_generic_cxx26__libcxx_next_runners__junk")
+        out_name4 = metrics.clean_up_libcxx_job_name(bad_name)
+        self.assertEqual(out_name4, bad_name)
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 6db24d894eb73..3ca4d05f4c891 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -13,36 +13,15 @@
 # run only the relevant tests.
 #
 
-set -ex
-set -o pipefail
+source .ci/utils.sh
 
-MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
-BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
 INSTALL_DIR="${BUILD_DIR}/install"
-rm -rf "${BUILD_DIR}"
-
-sccache --zero-stats
 
 mkdir -p artifacts/reproducers
 
-# Make sure any clang reproducers will end up as artifacts.
+# Make sure any clang reproducers will end up as artifacts
 export CLANG_CRASH_DIAGNOSTICS_DIR=`realpath artifacts/reproducers`
 
-function at-exit {
-  retcode=$?
-
-  sccache --show-stats > artifacts/sccache_stats.txt
-  cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
-  cp "${BUILD_DIR}"/test-results.*.xml artifacts/ || :
-
-  # If building fails there will be no results files.
-  shopt -s nullglob
-  
-  python3 "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":penguin: Linux x64 Test Results" \
-    $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
-}
-trap at-exit EXIT
-
 projects="${1}"
 targets="${2}"
 runtimes="${3}"
@@ -50,9 +29,9 @@ runtime_targets="${4}"
 runtime_targets_needs_reconfig="${5}"
 enable_cir="${6}"
 
-lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
+lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests --succinct"
 
-echo "::group::cmake"
+start-group "CMake"
 export PIP_BREAK_SYSTEM_PACKAGES=1
 pip install -q -r "${MONOREPO_ROOT}"/.ci/all_requirements.txt
 
@@ -81,51 +60,44 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D MLIR_ENABLE_BINDINGS_PYTHON=ON \
       -D LLDB_ENABLE_PYTHON=ON \
       -D LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS=ON \
-      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
+      -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+      -D CMAKE_EXE_LINKER_FLAGS="-no-pie"
 
-echo "::endgroup::"
-echo "::group::ninja"
+start-group "ninja"
 
 # Targets are not escaped as they are passed as separate arguments.
-ninja -C "${BUILD_DIR}" -k 0 ${targets}
-
-echo "::endgroup::"
+ninja -C "${BUILD_DIR}" -k 0 ${targets} |& tee ninja.log
 
 if [[ "${runtime_targets}" != "" ]]; then
-  echo "::group::ninja runtimes"
-
-  ninja -C "${BUILD_DIR}" ${runtime_targets}
+  start-group "ninja Runtimes"
 
-  echo "::endgroup::"
+  ninja -C "${BUILD_DIR}" ${runtime_targets} |& tee ninja_runtimes.log
 fi
 
 # Compiling runtimes with just-built Clang and running their tests
 # as an additional testing for Clang.
 if [[ "${runtime_targets_needs_reconfig}" != "" ]]; then
-  echo "::group::cmake runtimes C++26"
+  start-group "CMake Runtimes C++26"
 
   cmake \
     -D LIBCXX_TEST_PARAMS="std=c++26" \
     -D LIBCXXABI_TEST_PARAMS="std=c++26" \
     "${BUILD_DIR}"
 
-  echo "::endgroup::"
-  echo "::group::ninja runtimes C++26"
+  start-group "ninja Runtimes C++26"
 
-  ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
+  ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig} \
+    |& tee ninja_runtimes_needs_reconfig1.log
 
-  echo "::endgroup::"
-  echo "::group::cmake runtimes clang modules"
+  start-group "CMake Runtimes Clang Modules"
 
   cmake \
     -D LIBCXX_TEST_PARAMS="enable_modules=clang" \
     -D LIBCXXABI_TEST_PARAMS="enable_modules=clang" \
     "${BUILD_DIR}"
 
-  echo "::endgroup::"
-  echo "::group::ninja runtimes clang modules"
-
-  ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
+  start-group "ninja Runtimes Clang Modules"
 
-  echo "::endgroup::"
+  ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig} \
+    |& tee ninja_runtimes_needs_reconfig2.log
 fi
diff --git a/.ci/monolithic-windows.sh b/.ci/monolithic-windows.sh
index 50a741677d734..0f3a1994a0497 100755
--- a/.ci/monolithic-windows.sh
+++ b/.ci/monolithic-windows.sh
@@ -13,35 +13,12 @@
 # run only the relevant tests.
 #
 
-set -ex
-set -o pipefail
-
-MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
-BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
-
-rm -rf "${BUILD_DIR}"
-
-sccache --zero-stats
-function at-exit {
-  retcode=$?
-
-  mkdir -p artifacts
-  sccache --show-stats >> artifacts/sccache_stats.txt
-  cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
-  cp "${BUILD_DIR}"/test-results.*.xml artifacts/ || :
-
-  # If building fails there will be no results files.
-  shopt -s nullglob
-    
-  python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py ":window: Windows x64 Test Results" \
-    $retcode "${BUILD_DIR}"/test-results.*.xml >> $GITHUB_STEP_SUMMARY
-}
-trap at-exit EXIT
+source .ci/utils.sh
 
 projects="${1}"
 targets="${2}"
 
-echo "::group::cmake"
+start-group "CMake"
 pip install -q -r "${MONOREPO_ROOT}"/.ci/all_requirements.txt
 
 export CC=cl
@@ -62,7 +39,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D LLVM_ENABLE_ASSERTIONS=ON \
       -D LLVM_BUILD_EXAMPLES=ON \
       -D COMPILER_RT_BUILD_LIBFUZZER=OFF \
-      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests" \
+      -D LLVM_LIT_ARGS="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests --succinct" \
       -D COMPILER_RT_BUILD_ORC=OFF \
       -D CMAKE_C_COMPILER_LAUNCHER=sccache \
       -D CMAKE_CXX_COMPILER_LAUNCHER=sccache \
@@ -71,10 +48,7 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO"
 
-echo "::endgroup::"
-echo "::group::ninja"
+start-group "ninja"
 
 # Targets are not escaped as they are passed as separate arguments.
-ninja -C "${BUILD_DIR}" -k 0 ${targets}
-
-echo "::endgroup"
+ninja -C "${BUILD_DIR}" -k 0 ${targets} |& tee ninja.log
diff --git a/.ci/utils.sh b/.ci/utils.sh
new file mode 100644
index 0000000000000..2a3d2426b630a
--- /dev/null
+++ b/.ci/utils.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#===----------------------------------------------------------------------===##
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===----------------------------------------------------------------------===##
+
+# This script performs some setup and contains some utilities used for in the
+# monolithic-linux.sh and monolithic-windows.sh scripts.
+
+set -ex
+set -o pipefail
+
+MONOREPO_ROOT="${MONOREPO_ROOT:="$(git rev-parse --show-toplevel)"}"
+BUILD_DIR="${BUILD_DIR:=${MONOREPO_ROOT}/build}"
+
+rm -rf "${BUILD_DIR}"
+
+sccache --zero-stats
+
+function at-exit {
+  retcode=$?
+
+  mkdir -p artifacts
+  sccache --show-stats
+  sccache --show-stats >> artifacts/sccache_stats.txt
+  cp "${BUILD_DIR}"/.ninja_log artifacts/.ninja_log
+  cp "${MONOREPO_ROOT}"/*.log artifacts/ || :
+  cp "${BUILD_DIR}"/test-results.*.xml artifacts/ || :
+
+  # If building fails there will be no results files.
+  shopt -s nullglob
+
+  if [[ "$GITHUB_STEP_SUMMARY" != "" ]]; then
+    python "${MONOREPO_ROOT}"/.ci/generate_test_report_github.py \
+      $retcode "${BUILD_DIR}"/test-results.*.xml "${MONOREPO_ROOT}"/ninja*.log \
+      >> $GITHUB_STEP_SUMMARY
+  fi
+}
+trap at-exit EXIT
+
+function start-group {
+  groupname=$1
+  if [[ "$GITHUB_ACTIONS" != "" ]]; then
+    echo "::endgroup"
+    echo "::group::$groupname"
+  elif [[ "$POSTCOMMIT_CI" != "" ]]; then
+    echo "@@@$STEP@@@"
+  else
+    echo "Starting $groupname"
+  fi
+}
diff --git a/.github/new-prs-labeler.yml b/.github/new-prs-labeler.yml
index 8e0fa8d42d735..dab3db2616f59 100644
--- a/.github/new-prs-labeler.yml
+++ b/.github/new-prs-labeler.yml
@@ -90,9 +90,6 @@ LTO:
   - llvm/lib/Transforms/*/FunctionImport*
   - llvm/tools/gold/**
 
-mc:
-  - llvm/*/MC/**
-
 clang:driver:
   - clang/*/Driver/**
 
@@ -621,6 +618,12 @@ llvm:adt:
 llvm:support:
   - llvm/**/Support/**
 
+# Skip llvm/test/MC and llvm/unittests/MC, which includes target-specific directories.
+llvm:mc:
+  - llvm/include/llvm/MC/**
+  - llvm/lib/MC/**
+  - llvm/tools/llvm-mc/**
+
 llvm:transforms:
   - llvm/lib/Transforms/**
   - llvm/include/llvm/Transforms/**
diff --git a/.github/workflows/bazel-checks.yml b/.github/workflows/bazel-checks.yml
new file mode 100644
index 0000000000000..65d51649dd9e7
--- /dev/null
+++ b/.github/workflows/bazel-checks.yml
@@ -0,0 +1,32 @@
+name: Bazel Checks
+
+permissions:
+  contents: read
+
+on:
+  push:
+    paths:
+      - '.github/workflows/bazel-checks.yml'
+      - 'utils/bazel/**'
+    branches:
+      - main
+  pull_request:
+    paths:
+      - '.github/workflows/bazel-checks.yml'
+      - 'utils/bazel/**'
+
+jobs:
+  buildifier:
+    name: "Buildifier"
+    runs-on: ubuntu-24.04
+    if: github.repository == 'llvm/llvm-project'
+    steps:
+      - name: Fetch LLVM sources
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Setup Buildifier
+        run: |
+          sudo curl -L https://github.com/bazelbuild/buildtools/releases/download/v8.2.1/buildifier-linux-amd64 -o /usr/bin/buildifier
+          sudo chmod +x /usr/bin/buildifier
+      - name: Run Buildifier
+        run: |
+          buildifier --mode=check $(find ./utils/bazel -name *BUILD*)
diff --git a/.github/workflows/build-ci-container-windows.yml b/.github/workflows/build-ci-container-windows.yml
index f76c69f29fb30..55a269c001c2b 100644
--- a/.github/workflows/build-ci-container-windows.yml
+++ b/.github/workflows/build-ci-container-windows.yml
@@ -25,7 +25,7 @@ jobs:
       container-filename: ${{ steps.vars.outputs.container-filename }}
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci-windows
       - name: Write Variables
diff --git a/.github/workflows/build-ci-container.yml b/.github/workflows/build-ci-container.yml
index 7f01264af8534..3e91c49a51d19 100644
--- a/.github/workflows/build-ci-container.yml
+++ b/.github/workflows/build-ci-container.yml
@@ -30,7 +30,7 @@ jobs:
             runs-on: depot-ubuntu-24.04-arm-16
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: .github/workflows/containers/github-action-ci/
       # podman is not installed by default on the ARM64 images.
diff --git a/.github/workflows/build-metrics-container.yml b/.github/workflows/build-metrics-container.yml
index af4d599f76417..265fd73cc0bb7 100644
--- a/.github/workflows/build-metrics-container.yml
+++ b/.github/workflows/build-metrics-container.yml
@@ -27,7 +27,7 @@ jobs:
       container-filename: ${{ steps.vars.outputs.container-filename }}
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: .ci/metrics/
       - name: Write Variables
diff --git a/.github/workflows/check-ci.yml b/.github/workflows/check-ci.yml
index befea2093f908..7e8c15696e344 100644
--- a/.github/workflows/check-ci.yml
+++ b/.github/workflows/check-ci.yml
@@ -5,6 +5,8 @@ permissions:
 
 on:
   push:
+    branches:
+     - main
     paths:
      - '.ci/**'
      - '.github/workflows/check-ci.yml'
@@ -20,7 +22,7 @@ jobs:
     if: github.repository == 'llvm/llvm-project'
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: .ci
       - name: Setup Python
diff --git a/.github/workflows/ci-post-commit-analyzer.yml b/.github/workflows/ci-post-commit-analyzer.yml
index b8074859d23a2..7d37b900d7909 100644
--- a/.github/workflows/ci-post-commit-analyzer.yml
+++ b/.github/workflows/ci-post-commit-analyzer.yml
@@ -41,7 +41,7 @@ jobs:
       LLVM_VERSION: 18
     steps:
       - name: Checkout Source
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Setup ccache
         uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
diff --git a/.github/workflows/commit-access-greeter.yml b/.github/workflows/commit-access-greeter.yml
index a5fbbbb94e222..f31cd015642e2 100644
--- a/.github/workflows/commit-access-greeter.yml
+++ b/.github/workflows/commit-access-greeter.yml
@@ -18,7 +18,7 @@ jobs:
       github.event.label.name == 'infra:commit-access-request'
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: llvm/utils/git/
 
diff --git a/.github/workflows/commit-access-review.yml b/.github/workflows/commit-access-review.yml
index d401a137737c4..a7be81b0e2da5 100644
--- a/.github/workflows/commit-access-review.yml
+++ b/.github/workflows/commit-access-review.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       
       - name: Install dependencies
         run: |
diff --git a/.github/workflows/containers/github-action-ci-windows/Dockerfile b/.github/workflows/containers/github-action-ci-windows/Dockerfile
index c06fcc0580756..640d34da02532 100644
--- a/.github/workflows/containers/github-action-ci-windows/Dockerfile
+++ b/.github/workflows/containers/github-action-ci-windows/Dockerfile
@@ -90,7 +90,7 @@ RUN powershell -Command \
 RUN git config --system core.longpaths true & \
     git config --global core.autocrlf false
 
-ARG RUNNER_VERSION=2.327.1
+ARG RUNNER_VERSION=2.328.0
 ENV RUNNER_VERSION=$RUNNER_VERSION
 
 RUN powershell -Command \
diff --git a/.github/workflows/containers/github-action-ci/Dockerfile b/.github/workflows/containers/github-action-ci/Dockerfile
index 0444f617e3711..8a888f3a411c0 100644
--- a/.github/workflows/containers/github-action-ci/Dockerfile
+++ b/.github/workflows/containers/github-action-ci/Dockerfile
@@ -59,9 +59,12 @@ RUN apt-get update && \
     sudo \
     # These are needed by the premerge pipeline. Pip is used to install
     # dependent python packages. File and tzdata are used for tests.
+    # Having a symlink from python to python3 enables code sharing between
+    # the Linux and Windows pipelines.
     python3-pip \
     file \
-    tzdata && \
+    tzdata \
+    python-is-python3 && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -78,6 +81,8 @@ RUN curl -L 'https://github.com/mozilla/sccache/releases/download/v0.10.0/sccach
 
 ENV LLVM_SYSROOT=$LLVM_SYSROOT
 ENV PATH=${LLVM_SYSROOT}/bin:${PATH}
+ENV CC=clang
+ENV CXX=clang++
 
 # Create a new user to avoid test failures related to a lack of expected
 # permissions issues in some tests. Set the user id to 1001 as that is the
@@ -94,7 +99,7 @@ WORKDIR /home/gha
 
 FROM ci-container as ci-container-agent
 
-ENV GITHUB_RUNNER_VERSION=2.327.1
+ENV GITHUB_RUNNER_VERSION=2.328.0
 
 RUN mkdir actions-runner && \
     cd actions-runner && \
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 3970271e4adbd..b627803f61b27 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -55,7 +55,7 @@ jobs:
     if: github.repository == 'llvm/llvm-project'
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 2
       - name: Get subprojects that have doc changes
diff --git a/.github/workflows/email-check.yaml b/.github/workflows/email-check.yaml
index 904ad718f97dd..9390fba4d4e3b 100644
--- a/.github/workflows/email-check.yaml
+++ b/.github/workflows/email-check.yaml
@@ -14,7 +14,7 @@ jobs:
     if: github.repository == 'llvm/llvm-project'
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
diff --git a/.github/workflows/hlsl-test-all.yaml b/.github/workflows/hlsl-test-all.yaml
index b6530fe11b840..72cbbe2b7dded 100644
--- a/.github/workflows/hlsl-test-all.yaml
+++ b/.github/workflows/hlsl-test-all.yaml
@@ -29,25 +29,25 @@ jobs:
     runs-on: ${{ inputs.SKU }}
     steps:
       - name: Checkout DXC
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           repository: Microsoft/DirectXShaderCompiler
           ref: main
           path: DXC
           submodules: true
       - name: Checkout LLVM
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           ref: ${{ inputs.LLVM-branch }}
           path: llvm-project
       - name: Checkout OffloadTest
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           repository: llvm/offload-test-suite
           ref: main
           path: OffloadTest
       - name: Checkout Golden Images
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           repository: llvm/offload-golden-images
           ref: main
diff --git a/.github/workflows/issue-release-workflow.yml b/.github/workflows/issue-release-workflow.yml
index efd045990d013..7fd0280b2eedf 100644
--- a/.github/workflows/issue-release-workflow.yml
+++ b/.github/workflows/issue-release-workflow.yml
@@ -42,7 +42,7 @@ jobs:
       contains(github.event.action == 'opened' && github.event.issue.body || github.event.comment.body, '/cherry-pick')
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           repository: llvm/llvm-project
           # GitHub stores the token used for checkout and uses it for pushes
diff --git a/.github/workflows/issue-subscriber.yml b/.github/workflows/issue-subscriber.yml
index de1c45c944960..afcd17c757b39 100644
--- a/.github/workflows/issue-subscriber.yml
+++ b/.github/workflows/issue-subscriber.yml
@@ -14,7 +14,7 @@ jobs:
     if: github.repository == 'llvm/llvm-project'
     steps:
       - name: Checkout Automation Script
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: llvm/utils/git/
           ref: main
diff --git a/.github/workflows/issue-write.yml b/.github/workflows/issue-write.yml
index a2c4f58d6febe..3036582a64a58 100644
--- a/.github/workflows/issue-write.yml
+++ b/.github/workflows/issue-write.yml
@@ -25,7 +25,7 @@ jobs:
       )
     steps:
       - name: Fetch Sources
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: |
             .github/workflows/unprivileged-download-artifact/action.yml
diff --git a/.github/workflows/libc-fullbuild-tests.yml b/.github/workflows/libc-fullbuild-tests.yml
index 9ba77ff8bb845..8967cd0949c11 100644
--- a/.github/workflows/libc-fullbuild-tests.yml
+++ b/.github/workflows/libc-fullbuild-tests.yml
@@ -52,7 +52,7 @@ jobs:
           # - c_compiler: gcc
           #   cpp_compiler: g++
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
     
     # Libc's build is relatively small comparing with other components of LLVM.
     # A fresh fullbuild takes about 190MiB of uncompressed disk space, which can
diff --git a/.github/workflows/libc-overlay-tests.yml b/.github/workflows/libc-overlay-tests.yml
index e3dc4166aa26c..7154946ac5c3d 100644
--- a/.github/workflows/libc-overlay-tests.yml
+++ b/.github/workflows/libc-overlay-tests.yml
@@ -41,7 +41,7 @@ jobs:
               cpp_compiler: clang++
     
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
     
     # Libc's build is relatively small comparing with other components of LLVM.
     # A fresh linux overlay takes about 180MiB of uncompressed disk space, which can
diff --git a/.github/workflows/libclang-abi-tests.yml b/.github/workflows/libclang-abi-tests.yml
index 4d47c07f42205..3836cc56a7c22 100644
--- a/.github/workflows/libclang-abi-tests.yml
+++ b/.github/workflows/libclang-abi-tests.yml
@@ -38,7 +38,7 @@ jobs:
       LLVM_VERSION_PATCH: ${{ steps.version.outputs.patch }}
     steps:
       - name: Checkout source
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 250
 
diff --git a/.github/workflows/libclang-python-tests.yml b/.github/workflows/libclang-python-tests.yml
index 50ef4acf2feb1..e168928325561 100644
--- a/.github/workflows/libclang-python-tests.yml
+++ b/.github/workflows/libclang-python-tests.yml
@@ -4,7 +4,6 @@ permissions:
   contents: read
 
 on:
-  workflow_dispatch:
   push:
     branches:
       - 'main'
@@ -13,29 +12,46 @@ on:
       - 'clang/tools/libclang/**'
       - 'clang/CMakeList.txt'
       - '.github/workflows/libclang-python-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
   pull_request:
     paths:
       - 'clang/bindings/python/**'
       - 'clang/tools/libclang/**'
       - 'clang/CMakeList.txt'
       - '.github/workflows/libclang-python-tests.yml'
-      - '.github/workflows/llvm-project-tests.yml'
 
 jobs:
   check-clang-python:
     # Build libclang and then run the libclang Python binding's unit tests.
+    # There is an issue running on "windows-2019".
+    # See https://github.com/llvm/llvm-project/issues/76601#issuecomment-1873049082.
     name: Build and run Python unit tests
     if: github.repository == 'llvm/llvm-project'
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.8", "3.13"]
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      build_target: check-clang-python
-      projects: clang
-      # There is an issue running on "windows-2019".
-      # See https://github.com/llvm/llvm-project/issues/76601#issuecomment-1873049082.
-      os_list: '["ubuntu-24.04"]'
-      python_version: ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Setup Python
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
+        with:
+          max-size: 2G
+          key: spirv-ubuntu-24.04
+          variant: sccache
+      - name: Build and Test
+        run: |
+          mkdir build
+          cmake -GNinja \
+            -S llvm \
+            -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DLLVM_ENABLE_ASSERTIONS=ON \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+            -DLLVM_ENABLE_PROJECTS=clang
+          ninja -C build check-clang-python
diff --git a/.github/workflows/libcxx-build-and-test.yaml b/.github/workflows/libcxx-build-and-test.yaml
index 4b29c9296bf9d..2e6ff7f91b6fc 100644
--- a/.github/workflows/libcxx-build-and-test.yaml
+++ b/.github/workflows/libcxx-build-and-test.yaml
@@ -36,7 +36,7 @@ concurrency:
 jobs:
   stage1:
     if: github.repository_owner == 'llvm'
-    runs-on: llvm-premerge-libcxx-runners
+    runs-on: llvm-premerge-libcxx-next-runners
     continue-on-error: false
     strategy:
       fail-fast: false
@@ -47,14 +47,14 @@ jobs:
           'generic-cxx26',
           'generic-modules'
         ]
-        cc: [  'clang-21' ]
-        cxx: [ 'clang++-21' ]
+        cc: [  'clang-22' ]
+        cxx: [ 'clang++-22' ]
         include:
           - config: 'generic-gcc'
             cc: 'gcc-15'
             cxx: 'g++-15'
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - name: ${{ matrix.config }}.${{ matrix.cxx }}
         run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
@@ -73,7 +73,7 @@ jobs:
             **/crash_diagnostics/*
   stage2:
     if: github.repository_owner == 'llvm'
-    runs-on: llvm-premerge-libcxx-runners
+    runs-on: llvm-premerge-libcxx-next-runners
     needs: [ stage1 ]
     continue-on-error: false
     strategy:
@@ -86,20 +86,20 @@ jobs:
           'generic-cxx20',
           'generic-cxx23'
         ]
-        cc: [ 'clang-21' ]
-        cxx: [ 'clang++-21' ]
+        cc: [ 'clang-22' ]
+        cxx: [ 'clang++-22' ]
         include:
           - config: 'generic-gcc-cxx11'
             cc: 'gcc-15'
             cxx: 'g++-15'
+          - config: 'generic-cxx26'
+            cc: 'clang-21'
+            cxx: 'clang++-21'
           - config: 'generic-cxx26'
             cc: 'clang-20'
             cxx: 'clang++-20'
-          - config: 'generic-cxx26'
-            cc: 'clang-19'
-            cxx: 'clang++-19'
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - name: ${{ matrix.config }}
         run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
@@ -148,27 +148,27 @@ jobs:
           'generic-static',
           'bootstrapping-build'
         ]
-        machine: [ 'llvm-premerge-libcxx-runners' ]
+        machine: [ 'llvm-premerge-libcxx-next-runners' ]
         include:
         - config: 'generic-cxx26'
-          machine: llvm-premerge-libcxx-runners
+          machine: llvm-premerge-libcxx-next-runners
         - config: 'generic-asan'
-          machine: llvm-premerge-libcxx-runners
+          machine: llvm-premerge-libcxx-next-runners
         - config: 'generic-tsan'
-          machine: llvm-premerge-libcxx-runners
+          machine: llvm-premerge-libcxx-next-runners
         - config: 'generic-ubsan'
-          machine: llvm-premerge-libcxx-runners
+          machine: llvm-premerge-libcxx-next-runners
         # Use a larger machine for MSAN to avoid timeout and memory allocation issues.
         - config: 'generic-msan'
-          machine: llvm-premerge-libcxx-runners
+          machine: llvm-premerge-libcxx-next-runners
     runs-on: ${{ matrix.machine }}
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - name: ${{ matrix.config }}
         run: libcxx/utils/ci/run-buildbot ${{ matrix.config }}
         env:
-          CC: clang-21
-          CXX: clang++-21
+          CC: clang-22
+          CXX: clang++-22
       - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
         if: always()
         with:
@@ -211,7 +211,7 @@ jobs:
           os: macos-15
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - uses: maxim-lobanov/setup-xcode@60606e260d2fc5762a71e64e74b2174e8ea3c8bd # v1.6.0
         with:
           # https://github.com/actions/runner-images/blob/main/images/macos/macos-15-Readme.md
@@ -252,7 +252,7 @@ jobs:
         - { config: mingw-dll-i686, mingw: true }
         - { config: mingw-incomplete-sysroot, mingw: true }
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - name: Install dependencies
         run: |
           choco install -y ninja
diff --git a/.github/workflows/libcxx-build-containers.yml b/.github/workflows/libcxx-build-containers.yml
index 43c446a2c02ce..c87ee8e3be250 100644
--- a/.github/workflows/libcxx-build-containers.yml
+++ b/.github/workflows/libcxx-build-containers.yml
@@ -30,7 +30,7 @@ jobs:
       packages: write
 
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Build the Linux builder image
       working-directory: libcxx/utils/ci
diff --git a/.github/workflows/libcxx-check-generated-files.yml b/.github/workflows/libcxx-check-generated-files.yml
index 0226edd7aa17a..f338bd6952779 100644
--- a/.github/workflows/libcxx-check-generated-files.yml
+++ b/.github/workflows/libcxx-check-generated-files.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Install dependencies
         uses: aminya/setup-cpp@17c11551771948abc5752bbf3183482567c7caf0 # v1.1.1
diff --git a/.github/workflows/llvm-project-tests.yml b/.github/workflows/llvm-project-tests.yml
deleted file mode 100644
index d40ed5babb459..0000000000000
--- a/.github/workflows/llvm-project-tests.yml
+++ /dev/null
@@ -1,149 +0,0 @@
-name: LLVM Project Tests
-
-permissions:
-  contents: read
-
-on:
-  workflow_dispatch:
-    inputs:
-      build_target:
-        required: false
-      projects:
-        required: false
-      extra_cmake_args:
-        required: false
-      os_list:
-        required: false
-        default: '["ubuntu-24.04", "windows-2019", "macOS-13"]'
-      python_version:
-        required: false
-        type: string
-        default: '3.11'
-  workflow_call:
-    inputs:
-      build_target:
-        required: false
-        type: string
-        default: "all"
-
-      projects:
-        required: true
-        type: string
-
-      extra_cmake_args:
-        required: false
-        type: string
-
-      os_list:
-        required: false
-        type: string
-        # Use windows-2019 due to:
-        # https://developercommunity.visualstudio.com/t/Prev-Issue---with-__assume-isnan-/1597317
-        default: '["ubuntu-24.04", "windows-2019", "macOS-13"]'
-
-      python_version:
-        required: false
-        type: string
-        default: '3.11'
-
-concurrency:
-  # Skip intermediate builds: always.
-  # Cancel intermediate builds: only if it is a pull request build.
-  # If the group name here is the same as the group name in the workflow that includes
-  # this one, then the action will try to wait on itself and get stuck.
-  group: llvm-project-${{ github.workflow }}-${{ inputs.projects }}-${{ inputs.python_version }}${{ github.ref }}
-  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
-
-jobs:
-  lit-tests:
-    name: Lit Tests
-    runs-on: ${{ matrix.os }}
-    container:
-      image: ${{(startsWith(matrix.os, 'ubuntu') && 'ghcr.io/llvm/ci-ubuntu-24.04:latest') || null}}
-      volumes:
-        - /mnt/:/mnt/
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ${{ fromJSON(inputs.os_list) }}
-    steps:
-      - name: Setup Windows
-        if: startsWith(matrix.os, 'windows')
-        uses: llvm/actions/setup-windows@main
-        with:
-          arch: amd64
-      # On Windows, starting with win19/20220814.1, cmake choose the 32-bit
-      # python3.10.6 libraries instead of the 64-bit libraries when building
-      # lldb.  Using this setup-python action to make 3.10 the default
-      # python fixes this.
-      - name: Setup Python
-        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
-        with:
-          python-version: ${{ inputs.python_version }}
-      - name: Install Ninja
-        if: runner.os != 'Linux'
-        uses: llvm/actions/install-ninja@main
-      # actions/checkout deletes any existing files in the new git directory,
-      # so this needs to either run before ccache-action or it has to use
-      # clean: false.
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          fetch-depth: 250
-      - name: Setup ccache
-        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
-        with:
-          # A full build of llvm, clang, lld, and lldb takes about 250MB
-          # of ccache space. There's not much reason to have more than this,
-          # because we usually won't need to save cache entries from older
-          # builds.  Also, there is an overall 10GB cache limit, and each
-          # run creates a new cache entry so we want to ensure that we have
-          # enough cache space for all the tests to run at once and still
-          # fit under the 10 GB limit.
-          # Default to 2G to workaround: https://github.com/hendrikmuhs/ccache-action/issues/174
-          max-size: 2G
-          key: ${{ matrix.os }}
-          variant: sccache
-      - name: Build and Test
-        env:
-          # Workaround for https://github.com/actions/virtual-environments/issues/5900.
-          # This should be a no-op for non-mac OSes
-          PKG_CONFIG_PATH: /usr/local/Homebrew/Library/Homebrew/os/mac/pkgconfig//12
-        shell: bash
-        id: build-llvm
-        run: |
-          if [ "${{ runner.os }}" == "Linux" ]; then
-            builddir="/mnt/build/"
-            sudo mkdir -p $builddir
-            sudo chown gha $builddir
-            extra_cmake_args="-DCMAKE_CXX_COMPILER=clang++ -DCMAKE_C_COMPILER=clang"
-          else
-            builddir="$(pwd)"/build
-          fi
-          if [ "${{ runner.os }}" == "macOS" ]; then
-            # Workaround test failure on some lld tests on MacOS
-            # https://github.com/llvm/llvm-project/issues/81967
-            extra_cmake_args="-DLLVM_DISABLE_ASSEMBLY_FILES=ON"
-          fi
-          echo "llvm-builddir=$builddir" >> "$GITHUB_OUTPUT"
-          cmake -G Ninja \
-                -B "$builddir" \
-                -S llvm \
-                -DLLVM_ENABLE_PROJECTS="${{ inputs.projects }}" \
-                -DCMAKE_BUILD_TYPE=Release \
-                -DLLVM_ENABLE_ASSERTIONS=ON \
-                -DLLDB_INCLUDE_TESTS=OFF \
-                -DLIBCLC_TARGETS_TO_BUILD="amdgcn--;amdgcn--amdhsa;r600--;nvptx--;nvptx64--;nvptx--nvidiacl;nvptx64--nvidiacl" \
-                -DCMAKE_C_COMPILER_LAUNCHER=sccache \
-                -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
-                $extra_cmake_args \
-                ${{ inputs.extra_cmake_args }}
-          ninja -C "$builddir" '${{ inputs.build_target }}'
-
-      - name: Build and Test libclc
-        if: "!startsWith(matrix.os, 'windows') && contains(inputs.projects, 'libclc')"
-        env:
-          LLVM_BUILDDIR: ${{ steps.build-llvm.outputs.llvm-builddir }}
-        run: |
-          # The libclc tests don't have a generated check target so all we can
-          # do is build it.
-          ninja -C "$LLVM_BUILDDIR"
diff --git a/.github/workflows/llvm-project-workflow-tests.yml b/.github/workflows/llvm-project-workflow-tests.yml
deleted file mode 100644
index a2539b279be0a..0000000000000
--- a/.github/workflows/llvm-project-workflow-tests.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-# This workflow will test the llvm-project-tests workflow in PRs
-# targetting the main branch.  Since this workflow doesn't normally
-# run on main PRs, we need some way to test it to ensure new updates
-# don't break it.
-
-name: LLVM Workflow Test
-
-permissions:
-  contents: read
-
-on:
-  pull_request:
-    branches:
-      - 'main'
-    paths:
-      - '.github/workflows/llvm-project-tests.yml'
-      - '.github/workflows/llvm-project-workflow-tests.yml'
-
-concurrency:
-  # Skip intermediate builds: always.
-  # Cancel intermediate builds: only if it is a pull request build.
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
-
-jobs:
-  llvm-test:
-    if: github.repository_owner == 'llvm'
-    name: Build and Test
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      build_target: check-all
-      projects: clang;lld;libclc;lldb
diff --git a/.github/workflows/llvm-tests.yml b/.github/workflows/llvm-tests.yml
index a9bd8db462cf7..52b486e7e62fc 100644
--- a/.github/workflows/llvm-tests.yml
+++ b/.github/workflows/llvm-tests.yml
@@ -38,7 +38,7 @@ jobs:
       LLVM_VERSION_PATCH: ${{ steps.version.outputs.patch }}
     steps:
       - name: Checkout source
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 250
 
diff --git a/.github/workflows/merged-prs.yml b/.github/workflows/merged-prs.yml
index c771736389802..107bbc51b5314 100644
--- a/.github/workflows/merged-prs.yml
+++ b/.github/workflows/merged-prs.yml
@@ -21,7 +21,7 @@ jobs:
       (github.event.pull_request.merged == true)
     steps:
       - name: Checkout Automation Script
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: llvm/utils/git/
           ref: main
diff --git a/.github/workflows/mlir-spirv-tests.yml b/.github/workflows/mlir-spirv-tests.yml
new file mode 100644
index 0000000000000..78952ccad2642
--- /dev/null
+++ b/.github/workflows/mlir-spirv-tests.yml
@@ -0,0 +1,51 @@
+name: MLIR SPIR-V Tests
+
+permissions:
+  contents: read
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'mlir/include/mlir/Dialect/SPIRV/**'
+      - 'mlir/lib/Dialect/SPIRV/**'
+      - 'mlir/include/mlir/Target/SPIRV/**'
+      - 'mlir/lib/Target/SPIRV/**'
+      - 'mlir/test/Target/SPIRV/**'
+      - '.github/workflows/mlir-spirv-tests.yml'
+
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  check_spirv:
+    if: github.repository_owner == 'llvm'
+    name: Test MLIR SPIR-V
+    runs-on: ubuntu-24.04
+    container:
+      image: ghcr.io/llvm/ci-ubuntu-24.04:latest
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
+        with:
+          max-size: 2G
+          key: spirv-mlir-ubuntu-24.04
+          variant: sccache
+      - name: Build and Test
+        run: |
+          mkdir build
+          cmake -GNinja \
+            -S llvm \
+            -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DLLVM_ENABLE_ASSERTIONS=ON \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+            -DLLVM_TARGETS_TO_BUILD="host" \
+            -DLLVM_INCLUDE_SPIRV_TOOLS_TESTS=ON \
+            -DLLVM_ENABLE_PROJECTS=mlir
+          ninja -C build check-mlir
diff --git a/.github/workflows/new-prs.yml b/.github/workflows/new-prs.yml
index 935598e410dbb..e1f2e754c1a3d 100644
--- a/.github/workflows/new-prs.yml
+++ b/.github/workflows/new-prs.yml
@@ -35,7 +35,7 @@ jobs:
       (github.event.pull_request.author_association != 'OWNER')
     steps:
       - name: Checkout Automation Script
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: llvm/utils/git/
           ref: main
diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 70bcaafbd0cf3..5540555ae05ed 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -19,7 +19,7 @@ jobs:
     if: github.repository == 'llvm/llvm-project'
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 2
 
@@ -35,7 +35,7 @@ jobs:
       # We need to pull the script from the main branch, so that we ensure
       # we get the latest version of this script.
       - name: Fetch code formatting utils
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           repository: ${{ github.repository }}
           ref: ${{ github.base_ref }}
@@ -70,8 +70,6 @@ jobs:
       - name: Run code formatter
         env:
           GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
-          START_REV: ${{ github.event.pull_request.base.sha }}
-          END_REV: ${{ github.event.pull_request.head.sha }}
           CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
         # Create an empty comments file so the pr-write job doesn't fail.
         run: |
diff --git a/.github/workflows/pr-request-release-note.yml b/.github/workflows/pr-request-release-note.yml
index 57425e04ec2f4..f0197d71d6aa9 100644
--- a/.github/workflows/pr-request-release-note.yml
+++ b/.github/workflows/pr-request-release-note.yml
@@ -19,7 +19,7 @@ jobs:
       # We need to pull the script from the main branch, so that we ensure
       # we get the latest version of this script.
       - name: Checkout Scripts
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: |
             llvm/utils/git/requirements.txt
diff --git a/.github/workflows/pr-subscriber.yml b/.github/workflows/pr-subscriber.yml
index f558da8a8fe0e..23c7a679185ee 100644
--- a/.github/workflows/pr-subscriber.yml
+++ b/.github/workflows/pr-subscriber.yml
@@ -14,7 +14,7 @@ jobs:
     if: github.repository == 'llvm/llvm-project'
     steps:
       - name: Checkout Automation Script
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: llvm/utils/git/
           ref: main
diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml
index d0518fa6879e2..9d925517a7211 100644
--- a/.github/workflows/premerge.yaml
+++ b/.github/workflows/premerge.yaml
@@ -31,13 +31,10 @@ jobs:
     runs-on: llvm-premerge-linux-runners
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 2
       - name: Build and Test
-        # Mark the job as a success even if the step fails so that people do
-        # not get notified while the new premerge pipeline is in an
-        # experimental state.
         run: |
           git config --global --add safe.directory '*'
 
@@ -72,6 +69,11 @@ jobs:
 
           ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}" "${enable_cir}"
       - name: Upload Artifacts
+        # In some cases, Github will fail to upload the artifact. We want to
+        # continue anyways as a failed artifact upload is an infra failure, not
+        # a checks failure.
+        # https://github.com/actions/upload-artifact/issues/569
+        continue-on-error: true
         if: '!cancelled()'
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
@@ -91,7 +93,7 @@ jobs:
         shell: bash
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 2
       - name: Compute Projects
@@ -109,9 +111,6 @@ jobs:
           echo "windows-projects=${projects_to_build}" >> $GITHUB_OUTPUT
           echo "windows-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT
       - name: Build and Test
-        # Mark the job as a success even if the step fails so that people do
-        # not get notified while the new premerge pipeline is in an
-        # experimental state.
         if: ${{ steps.vars.outputs.windows-projects != '' }}
         shell: cmd
         run: |
@@ -120,6 +119,11 @@ jobs:
           # these environment variables.
           bash -c "export SCCACHE_GCS_BUCKET=$CACHE_GCS_BUCKET; export SCCACHE_GCS_RW_MODE=READ_WRITE; export SCCACHE_IDLE_TIMEOUT=0; sccache --start-server; .ci/monolithic-windows.sh \"${{ steps.vars.outputs.windows-projects }}\" \"${{ steps.vars.outputs.windows-check-targets }}\""
       - name: Upload Artifacts
+        # In some cases, Github will fail to upload the artifact. We want to
+        # continue anyways as a failed artifact upload is an infra failure, not
+        # a checks failure.
+        # https://github.com/actions/upload-artifact/issues/569
+        continue-on-error: true
         if: '!cancelled()'
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
@@ -138,7 +142,7 @@ jobs:
       (github.event_name != 'pull_request' || github.event.action != 'closed')
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 2
       - name: Setup ccache
diff --git a/.github/workflows/release-asset-audit.yml b/.github/workflows/release-asset-audit.yml
index 7a1f232ae7335..6546540a1b547 100644
--- a/.github/workflows/release-asset-audit.yml
+++ b/.github/workflows/release-asset-audit.yml
@@ -23,7 +23,7 @@ jobs:
     if: github.repository == 'llvm/llvm-project'
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: |
             .github/workflows/release-asset-audit.py
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index c113b42dc8ed4..116bdfb3929d3 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -73,7 +73,7 @@ jobs:
         python-version: '3.12'
 
     - name: Checkout LLVM
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Install Dependencies
       shell: bash
@@ -195,7 +195,7 @@ jobs:
     steps:
 
     - name: Checkout Actions
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       with:
         ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
         sparse-checkout: |
@@ -216,7 +216,7 @@ jobs:
       run: mv workflows  ../workflows-main
 
     - name: Checkout LLVM
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       with:
         ref: ${{ needs.prepare.outputs.ref }}
 
@@ -286,7 +286,7 @@ jobs:
 
     steps:
     - name: Checkout Release Scripts
-      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       with:
         sparse-checkout: |
           llvm/utils/release/github-upload-release.py
@@ -338,7 +338,7 @@ jobs:
     runs-on: ${{ needs.prepare.outputs.test-runs-on }}
     steps:
     - name: Checkout Actions
-      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       with:
         ref: ${{ (github.event_name == 'pull_request' && github.sha) || 'main' }}
         sparse-checkout: |
diff --git a/.github/workflows/release-documentation.yml b/.github/workflows/release-documentation.yml
index 5a0aa063d32ac..712ff1831170e 100644
--- a/.github/workflows/release-documentation.yml
+++ b/.github/workflows/release-documentation.yml
@@ -34,7 +34,7 @@ jobs:
       upload: ${{ inputs.upload && !contains(inputs.release-version, 'rc') }}
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Setup Python env
         uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
@@ -66,7 +66,7 @@ jobs:
 
       - name: Clone www-releases
         if: env.upload
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           repository: ${{ github.repository_owner }}/www-releases
           ref: main
diff --git a/.github/workflows/release-doxygen.yml b/.github/workflows/release-doxygen.yml
index d47c4337c07b2..17c677413f744 100644
--- a/.github/workflows/release-doxygen.yml
+++ b/.github/workflows/release-doxygen.yml
@@ -40,7 +40,7 @@ jobs:
       upload: ${{ inputs.upload && !contains(inputs.release-version, 'rc') }}
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Setup Python env
         uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
diff --git a/.github/workflows/release-lit.yml b/.github/workflows/release-lit.yml
index 9adeffb74d52a..60ec64462bc31 100644
--- a/.github/workflows/release-lit.yml
+++ b/.github/workflows/release-lit.yml
@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           ref: "llvmorg-${{ inputs.release-version }}"
 
diff --git a/.github/workflows/release-sources.yml b/.github/workflows/release-sources.yml
index 99438918b56f0..14cc4c4e9b94f 100644
--- a/.github/workflows/release-sources.yml
+++ b/.github/workflows/release-sources.yml
@@ -71,7 +71,7 @@ jobs:
       attestations: write
     steps:
       - name: Checkout LLVM
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           ref: ${{ needs.inputs.outputs.ref }}
           fetch-tags: true
diff --git a/.github/workflows/release-tasks.yml b/.github/workflows/release-tasks.yml
index c9ae7e1ce97c3..a184996968cdd 100644
--- a/.github/workflows/release-tasks.yml
+++ b/.github/workflows/release-tasks.yml
@@ -38,7 +38,7 @@ jobs:
           sudo apt-get install python3-github
 
       - name: Checkout LLVM
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Create Release
         env:
@@ -129,7 +129,7 @@ jobs:
           sudo apt-get install python3-github
 
       - name: Checkout LLVM
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           sparse-checkout: llvm/utils/release/github-upload-release.py
           sparse-checkout-cone-mode: false
diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
index 6cc80fb316c67..40db5504294ef 100644
--- a/.github/workflows/scorecard.yml
+++ b/.github/workflows/scorecard.yml
@@ -31,7 +31,7 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           persist-credentials: false
 
diff --git a/.github/workflows/spirv-tests.yml b/.github/workflows/spirv-tests.yml
index f15ca1cb64ba5..8708fb06d9eb8 100644
--- a/.github/workflows/spirv-tests.yml
+++ b/.github/workflows/spirv-tests.yml
@@ -4,7 +4,6 @@ permissions:
   contents: read
 
 on:
-  workflow_dispatch:
   pull_request:
     paths:
       - 'llvm/lib/Target/SPIRV/**'
@@ -21,9 +20,27 @@ jobs:
   check_spirv:
     if: github.repository_owner == 'llvm'
     name: Test SPIR-V
-    uses: ./.github/workflows/llvm-project-tests.yml
-    with:
-      build_target: check-llvm-codegen-spirv
-      projects:
-      extra_cmake_args: '-DLLVM_TARGETS_TO_BUILD="SPIRV" -DLLVM_INCLUDE_SPIRV_TOOLS_TESTS=ON'
-      os_list: '["ubuntu-24.04"]'
+    runs-on: ubuntu-24.04
+    container:
+      image: ghcr.io/llvm/ci-ubuntu-24.04:latest
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@a1209f81afb8c005c13b4296c32e363431bffea5 # v1.2.17
+        with:
+          max-size: 2G
+          key: spirv-ubuntu-24.04
+          variant: sccache
+      - name: Build and Test
+        run: |
+          mkdir build
+          cmake -GNinja \
+            -S llvm \
+            -B build \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DLLVM_ENABLE_ASSERTIONS=ON \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
+            -DLLVM_TARGETS_TO_BUILD="SPIRV" \
+            -DLLVM_INCLUDE_SPIRV_TOOLS_TESTS=ON
+          ninja -C build check-llvm-codegen-spirv
diff --git a/.github/workflows/version-check.yml b/.github/workflows/version-check.yml
index a0a598094376f..7e451880f4cfa 100644
--- a/.github/workflows/version-check.yml
+++ b/.github/workflows/version-check.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - name: Fetch LLVM sources
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
 
diff --git a/.gitignore b/.gitignore
index a84268a7f6863..860b8ea12abd4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -52,6 +52,11 @@ autoconf/autom4te.cache
 # CLion project configuration
 /.idea
 /cmake-build*
+# Coding assistants' stuff
+/CLAUDE.md
+/.claude/
+/GEMINI.md
+/.gemini/
 
 #==============================================================================#
 # Directories to ignore (do not add trailing '/'s, they skip symlinks).
diff --git a/bolt/docs/CommandLineArgumentReference.md b/bolt/docs/CommandLineArgumentReference.md
index f3881c9a640a9..d65cf39e16b29 100644
--- a/bolt/docs/CommandLineArgumentReference.md
+++ b/bolt/docs/CommandLineArgumentReference.md
@@ -138,6 +138,12 @@
   Dump function CFGs to graphviz format after each stage;enable '-print-loops'
   for color-coded blocks
 
+- `--dump-dot-func=<func1,func2,func3...>`
+
+  Dump function CFGs to graphviz format for specified functions only;
+  takes function name patterns (regex supported). Note: C++ function names
+  must be passed using their mangled names
+
 - `--dump-linux-exceptions`
 
   Dump Linux kernel exception table
diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h
index ae580520b9110..b59926cc75571 100644
--- a/bolt/include/bolt/Core/BinaryFunction.h
+++ b/bolt/include/bolt/Core/BinaryFunction.h
@@ -1196,11 +1196,6 @@ class BinaryFunction {
     return getSecondaryEntryPointSymbol(BB.getLabel());
   }
 
-  /// Remove a label from the secondary entry point map.
-  void removeSymbolFromSecondaryEntryPointMap(const MCSymbol *Label) {
-    SecondaryEntryPoints.erase(Label);
-  }
-
   /// Return true if the basic block is an entry point into the function
   /// (either primary or secondary).
   bool isEntryPoint(const BinaryBasicBlock &BB) const {
diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h
index f902a8c43cd1d..ae04891e791f9 100644
--- a/bolt/include/bolt/Core/MCPlusBuilder.h
+++ b/bolt/include/bolt/Core/MCPlusBuilder.h
@@ -718,6 +718,20 @@ class MCPlusBuilder {
     return false;
   }
 
+  /// Returns true if Inst is a trap instruction.
+  ///
+  /// Tests if Inst is an instruction that immediately causes an abnormal
+  /// program termination, for example when a security violation is detected
+  /// by a compiler-inserted check.
+  ///
+  /// @note An implementation of this method should likely return false for
+  /// calls to library functions like abort(), as it is possible that the
+  /// execution state is partially attacker-controlled at this point.
+  virtual bool isTrap(const MCInst &Inst) const {
+    llvm_unreachable("not implemented");
+    return false;
+  }
+
   virtual bool isBreakpoint(const MCInst &Inst) const {
     llvm_unreachable("not implemented");
     return false;
@@ -740,6 +754,10 @@ class MCPlusBuilder {
     return false;
   }
 
+  /// Return true if the hlt instruction under the x86, otherwise, default to
+  /// false.
+  virtual bool isX86HLT(const MCInst &Inst) const { return false; }
+
   /// Return the width, in bytes, of the memory access performed by \p Inst, if
   /// this is a pop instruction. Return zero otherwise.
   virtual int getPopSize(const MCInst &Inst) const {
diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h
index cb1b87f8d0d65..db0f6903185b7 100644
--- a/bolt/include/bolt/Profile/DataAggregator.h
+++ b/bolt/include/bolt/Profile/DataAggregator.h
@@ -502,9 +502,6 @@ class DataAggregator : public DataReader {
   /// entries).
   void imputeFallThroughs();
 
-  /// Register profiled functions for lite mode.
-  void registerProfiledFunctions();
-
   /// Debugging dump methods
   void dump() const;
   void dump(const PerfBranchSample &Sample) const;
diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h
index 91d62a78de390..19dcce8205ebc 100644
--- a/bolt/include/bolt/Rewrite/RewriteInstance.h
+++ b/bolt/include/bolt/Rewrite/RewriteInstance.h
@@ -241,7 +241,7 @@ class RewriteInstance {
 
   /// Adjust function sizes and set proper maximum size values after the whole
   /// symbol table has been processed.
-  void adjustFunctionBoundaries();
+  void adjustFunctionBoundaries(DenseMap<uint64_t, MarkerSymType> &MarkerSyms);
 
   /// Make .eh_frame section relocatable.
   void relocateEHFrameSection();
diff --git a/bolt/include/bolt/Utils/CommandLineOpts.h b/bolt/include/bolt/Utils/CommandLineOpts.h
index a75b6bf720ec4..859d6f3bf6774 100644
--- a/bolt/include/bolt/Utils/CommandLineOpts.h
+++ b/bolt/include/bolt/Utils/CommandLineOpts.h
@@ -15,6 +15,12 @@
 
 #include "llvm/Support/CommandLine.h"
 
+namespace llvm {
+namespace bolt {
+class BinaryFunction;
+}
+} // namespace llvm
+
 namespace opts {
 
 enum HeatmapModeKind {
@@ -100,6 +106,9 @@ extern llvm::cl::opt<unsigned> Verbosity;
 /// Return true if we should process all functions in the binary.
 bool processAllFunctions();
 
+/// Return true if we should dump dot graphs for the given function.
+bool shouldDumpDot(const llvm::bolt::BinaryFunction &Function);
+
 enum GadgetScannerKind { GS_PACRET, GS_PAUTH, GS_ALL };
 
 extern llvm::cl::bits<GadgetScannerKind> GadgetScannersToRun;
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 84f1853469709..da59a188c6b60 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -2517,7 +2517,7 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
   // Clean-up the effect of the code emission.
   for (const MCSymbol &Symbol : Assembler.symbols()) {
     MCSymbol *MutableSymbol = const_cast<MCSymbol *>(&Symbol);
-    MutableSymbol->setUndefined();
+    MutableSymbol->setFragment(nullptr);
     MutableSymbol->setIsRegistered(false);
   }
 
diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp
index eec68ff5a5fce..8f494f105fbba 100644
--- a/bolt/lib/Core/BinaryFunction.cpp
+++ b/bolt/lib/Core/BinaryFunction.cpp
@@ -1915,13 +1915,9 @@ void BinaryFunction::postProcessEntryPoints() {
       continue;
 
     // If we have grabbed a wrong code label which actually points to some
-    // constant island inside the function, ignore this label and remove it
-    // from the secondary entry point map.
-    if (isStartOfConstantIsland(Offset)) {
-      BC.SymbolToFunctionMap.erase(Label);
-      removeSymbolFromSecondaryEntryPointMap(Label);
+    // constant island inside the function, ignore this label.
+    if (isStartOfConstantIsland(Offset))
       continue;
-    }
 
     BC.errs() << "BOLT-WARNING: reference in the middle of instruction "
                  "detected in function "
diff --git a/bolt/lib/Core/MCPlusBuilder.cpp b/bolt/lib/Core/MCPlusBuilder.cpp
index fa8f4d1df308b..7f962e14ea115 100644
--- a/bolt/lib/Core/MCPlusBuilder.cpp
+++ b/bolt/lib/Core/MCPlusBuilder.cpp
@@ -30,6 +30,11 @@ using namespace bolt;
 using namespace MCPlus;
 
 namespace opts {
+cl::opt<bool>
+    TerminalHLT("terminal-x86-hlt",
+                cl::desc("Assume that execution stops at x86 HLT instruction"),
+                cl::init(true), cl::Hidden, cl::cat(BoltCategory));
+
 cl::opt<bool>
     TerminalTrap("terminal-trap",
                  cl::desc("Assume that execution stops at trap instruction"),
@@ -132,8 +137,13 @@ bool MCPlusBuilder::equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,
 }
 
 bool MCPlusBuilder::isTerminator(const MCInst &Inst) const {
-  return Analysis->isTerminator(Inst) ||
-         (opts::TerminalTrap && Info->get(Inst.getOpcode()).isTrap());
+  if (isX86HLT(Inst))
+    return opts::TerminalHLT;
+
+  if (Info->get(Inst.getOpcode()).isTrap())
+    return opts::TerminalTrap;
+
+  return Analysis->isTerminator(Inst);
 }
 
 void MCPlusBuilder::setTailCall(MCInst &Inst) const {
diff --git a/bolt/lib/Passes/BinaryPasses.cpp b/bolt/lib/Passes/BinaryPasses.cpp
index 5d44e1a1a4902..d7f02b9470030 100644
--- a/bolt/lib/Passes/BinaryPasses.cpp
+++ b/bolt/lib/Passes/BinaryPasses.cpp
@@ -662,7 +662,7 @@ Error CleanMCState::runOnFunctions(BinaryContext &BC) {
     if (S->isDefined()) {
       LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Symbol \"" << S->getName()
                         << "\" is already defined\n");
-      const_cast<MCSymbol *>(S)->setUndefined();
+      const_cast<MCSymbol *>(S)->setFragment(nullptr);
     }
     if (S->isRegistered()) {
       LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Symbol \"" << S->getName()
diff --git a/bolt/lib/Passes/FrameOptimizer.cpp b/bolt/lib/Passes/FrameOptimizer.cpp
index 81d4d9367f58c..b0b7207feac01 100644
--- a/bolt/lib/Passes/FrameOptimizer.cpp
+++ b/bolt/lib/Passes/FrameOptimizer.cpp
@@ -224,6 +224,11 @@ Error FrameOptimizerPass::runOnFunctions(BinaryContext &BC) {
   if (opts::FrameOptimization == FOP_NONE)
     return Error::success();
 
+  if (!BC.isX86()) {
+    BC.errs() << "BOLT-ERROR: " << getName() << " is supported only on X86\n";
+    exit(1);
+  }
+
   std::unique_ptr<BinaryFunctionCallGraph> CG;
   std::unique_ptr<FrameAnalysis> FA;
   std::unique_ptr<RegAnalysis> RA;
diff --git a/bolt/lib/Passes/PAuthGadgetScanner.cpp b/bolt/lib/Passes/PAuthGadgetScanner.cpp
index f928dd49edb25..65c84ebc8c4f4 100644
--- a/bolt/lib/Passes/PAuthGadgetScanner.cpp
+++ b/bolt/lib/Passes/PAuthGadgetScanner.cpp
@@ -1078,6 +1078,15 @@ class DstSafetyAnalysis {
       dbgs() << ")\n";
     });
 
+    // If this instruction terminates the program immediately, no
+    // authentication oracles are possible past this point.
+    if (BC.MIB->isTrap(Point)) {
+      LLVM_DEBUG({ traceInst(BC, "Trap instruction found", Point); });
+      DstState Next(NumRegs, RegsToTrackInstsFor.getNumTrackedRegisters());
+      Next.CannotEscapeUnchecked.set();
+      return Next;
+    }
+
     // If this instruction is reachable by the analysis, a non-empty state will
     // be propagated to it sooner or later. Until then, skip computeNext().
     if (Cur.empty()) {
@@ -1185,8 +1194,8 @@ class DataflowDstSafetyAnalysis
     //
     // A basic block without any successors, on the other hand, can be
     // pessimistically initialized to everything-is-unsafe: this will naturally
-    // handle both return and tail call instructions and is harmless for
-    // internal indirect branch instructions (such as computed gotos).
+    // handle return, trap and tail call instructions. At the same time, it is
+    // harmless for internal indirect branch instructions, like computed gotos.
     if (BB.succ_empty())
       return createUnsafeState();
 
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index c13fa6dbe582b..3604fdd3a94b4 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -581,26 +581,6 @@ void DataAggregator::imputeFallThroughs() {
     outs() << "BOLT-INFO: imputed " << InferredTraces << " traces\n";
 }
 
-void DataAggregator::registerProfiledFunctions() {
-  DenseSet<uint64_t> Addrs;
-  for (const auto &Trace : llvm::make_first_range(Traces)) {
-    if (Trace.Branch != Trace::FT_ONLY &&
-        Trace.Branch != Trace::FT_EXTERNAL_ORIGIN)
-      Addrs.insert(Trace.Branch);
-    Addrs.insert(Trace.From);
-  }
-
-  for (const auto [PC, _] : BasicSamples)
-    Addrs.insert(PC);
-
-  for (const PerfMemSample &MemSample : MemSamples)
-    Addrs.insert(MemSample.PC);
-
-  for (const uint64_t Addr : Addrs)
-    if (BinaryFunction *Func = getBinaryFunctionContainingAddress(Addr))
-      Func->setHasProfileAvailable();
-}
-
 Error DataAggregator::preprocessProfile(BinaryContext &BC) {
   this->BC = &BC;
 
@@ -623,7 +603,6 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) {
       exit(0);
   }
 
-  registerProfiledFunctions();
   return Error::success();
 }
 
@@ -1368,6 +1347,10 @@ std::error_code DataAggregator::parseAggregatedLBREntry() {
   }
 
   const uint64_t FromOffset = Addr[0]->Offset;
+  BinaryFunction *FromFunc = getBinaryFunctionContainingAddress(FromOffset);
+  if (FromFunc)
+    FromFunc->setHasProfileAvailable();
+
   int64_t Count = Counters[0];
   int64_t Mispreds = Counters[1];
 
@@ -1378,6 +1361,11 @@ std::error_code DataAggregator::parseAggregatedLBREntry() {
     return std::error_code();
   }
 
+  const uint64_t ToOffset = Addr[1]->Offset;
+  BinaryFunction *ToFunc = getBinaryFunctionContainingAddress(ToOffset);
+  if (ToFunc)
+    ToFunc->setHasProfileAvailable();
+
   /// For fall-through types, adjust locations to match Trace container.
   if (Type == FT || Type == FT_EXTERNAL_ORIGIN || Type == FT_EXTERNAL_RETURN) {
     Addr[2] = Location(Addr[1]->Offset); // Trace To
@@ -1625,6 +1613,9 @@ std::error_code DataAggregator::parseBranchEvents() {
   Traces.reserve(TraceMap.size());
   for (const auto &[Trace, Info] : TraceMap) {
     Traces.emplace_back(Trace, Info);
+    for (const uint64_t Addr : {Trace.Branch, Trace.From})
+      if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Addr))
+        BF->setHasProfileAvailable();
   }
   clear(TraceMap);
 
@@ -1685,6 +1676,9 @@ std::error_code DataAggregator::parseBasicEvents() {
       continue;
     ++NumTotalSamples;
 
+    if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Sample->PC))
+      BF->setHasProfileAvailable();
+
     ++BasicSamples[Sample->PC];
     EventNames.insert(Sample->EventName);
   }
@@ -1722,6 +1716,9 @@ std::error_code DataAggregator::parseMemEvents() {
     if (std::error_code EC = Sample.getError())
       return EC;
 
+    if (BinaryFunction *BF = getBinaryFunctionContainingAddress(Sample->PC))
+      BF->setHasProfileAvailable();
+
     MemSamples.emplace_back(std::move(Sample.get()));
   }
 
diff --git a/bolt/lib/Rewrite/BinaryPassManager.cpp b/bolt/lib/Rewrite/BinaryPassManager.cpp
index 996d2e972599d..0ddb73f828878 100644
--- a/bolt/lib/Rewrite/BinaryPassManager.cpp
+++ b/bolt/lib/Rewrite/BinaryPassManager.cpp
@@ -52,6 +52,7 @@ namespace opts {
 extern cl::opt<bool> PrintAll;
 extern cl::opt<bool> PrintDynoStats;
 extern cl::opt<bool> DumpDotAll;
+extern bool shouldDumpDot(const bolt::BinaryFunction &Function);
 extern cl::opt<std::string> AsmDump;
 extern cl::opt<bolt::PLTCall::OptType> PLT;
 extern cl::opt<bolt::IdenticalCodeFolding::ICFLevel, false,
@@ -340,7 +341,7 @@ Error BinaryFunctionPassManager::runPasses() {
 
       Function.print(BC.outs(), Message);
 
-      if (opts::DumpDotAll)
+      if (opts::shouldDumpDot(Function))
         Function.dumpGraphForPass(PassIdName);
     }
   }
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index fe4a23cc01382..4f5a75b770ce6 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -84,6 +84,7 @@ extern cl::opt<bool> KeepNops;
 extern cl::opt<bool> Lite;
 extern cl::list<std::string> ReorderData;
 extern cl::opt<bolt::ReorderFunctions::ReorderType> ReorderFunctions;
+extern cl::opt<bool> TerminalHLT;
 extern cl::opt<bool> TerminalTrap;
 extern cl::opt<bool> TimeBuild;
 extern cl::opt<bool> TimeRewrite;
@@ -114,6 +115,35 @@ cl::opt<bool> DumpDotAll(
              "enable '-print-loops' for color-coded blocks"),
     cl::Hidden, cl::cat(BoltCategory));
 
+cl::list<std::string> DumpDotFunc(
+    "dump-dot-func", cl::CommaSeparated,
+    cl::desc(
+        "dump function CFGs to graphviz format for specified functions only;"
+        "takes function name patterns (regex supported)"),
+    cl::value_desc("func1,func2,func3,..."), cl::Hidden, cl::cat(BoltCategory));
+
+bool shouldDumpDot(const bolt::BinaryFunction &Function) {
+  // If dump-dot-all is enabled, dump all functions
+  if (DumpDotAll)
+    return !Function.isIgnored();
+
+  // If no specific functions specified in dump-dot-func, don't dump any
+  if (DumpDotFunc.empty())
+    return false;
+
+  if (Function.isIgnored())
+    return false;
+
+  // Check if function matches any of the specified patterns
+  for (const std::string &Name : DumpDotFunc) {
+    if (Function.hasNameRegex(Name)) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
 static cl::list<std::string>
 ForceFunctionNames("funcs",
   cl::CommaSeparated,
@@ -880,14 +910,9 @@ void RewriteInstance::discoverFileObjects() {
   // code section (see IHI0056B). $d identifies data contents.
   // Compilers usually merge multiple data objects in a single $d-$x interval,
   // but we need every data object to be marked with $d. Because of that we
-  // create a vector of MarkerSyms with all locations of data objects.
+  // keep track of marker symbols with all locations of data objects.
 
-  struct MarkerSym {
-    uint64_t Address;
-    MarkerSymType Type;
-  };
-
-  std::vector<MarkerSym> SortedMarkerSymbols;
+  DenseMap<uint64_t, MarkerSymType> MarkerSymbols;
   auto addExtraDataMarkerPerSymbol = [&]() {
     bool IsData = false;
     uint64_t LastAddr = 0;
@@ -911,14 +936,14 @@ void RewriteInstance::discoverFileObjects() {
       }
 
       if (MarkerType != MarkerSymType::NONE) {
-        SortedMarkerSymbols.push_back(MarkerSym{SymInfo.Address, MarkerType});
+        MarkerSymbols[SymInfo.Address] = MarkerType;
         LastAddr = SymInfo.Address;
         IsData = MarkerType == MarkerSymType::DATA;
         continue;
       }
 
       if (IsData) {
-        SortedMarkerSymbols.push_back({SymInfo.Address, MarkerSymType::DATA});
+        MarkerSymbols[SymInfo.Address] = MarkerSymType::DATA;
         LastAddr = SymInfo.Address;
       }
     }
@@ -1283,27 +1308,24 @@ void RewriteInstance::discoverFileObjects() {
   BC->setHasSymbolsWithFileName(FileSymbols.size());
 
   // Now that all the functions were created - adjust their boundaries.
-  adjustFunctionBoundaries();
+  adjustFunctionBoundaries(MarkerSymbols);
 
   // Annotate functions with code/data markers in AArch64
-  for (auto ISym = SortedMarkerSymbols.begin();
-       ISym != SortedMarkerSymbols.end(); ++ISym) {
-
-    auto *BF =
-        BC->getBinaryFunctionContainingAddress(ISym->Address, true, true);
+  for (auto &[Address, Type] : MarkerSymbols) {
+    auto *BF = BC->getBinaryFunctionContainingAddress(Address, true, true);
 
     if (!BF) {
       // Stray marker
       continue;
     }
-    const auto EntryOffset = ISym->Address - BF->getAddress();
-    if (ISym->Type == MarkerSymType::CODE) {
+    const auto EntryOffset = Address - BF->getAddress();
+    if (Type == MarkerSymType::CODE) {
       BF->markCodeAtOffset(EntryOffset);
       continue;
     }
-    if (ISym->Type == MarkerSymType::DATA) {
+    if (Type == MarkerSymType::DATA) {
       BF->markDataAtOffset(EntryOffset);
-      BC->AddressToConstantIslandMap[ISym->Address] = BF;
+      BC->AddressToConstantIslandMap[Address] = BF;
       continue;
     }
     llvm_unreachable("Unknown marker");
@@ -1832,7 +1854,8 @@ void RewriteInstance::disassemblePLT() {
   }
 }
 
-void RewriteInstance::adjustFunctionBoundaries() {
+void RewriteInstance::adjustFunctionBoundaries(
+    DenseMap<uint64_t, MarkerSymType> &MarkerSyms) {
   for (auto BFI = BC->getBinaryFunctions().begin(),
             BFE = BC->getBinaryFunctions().end();
        BFI != BFE; ++BFI) {
@@ -1870,12 +1893,15 @@ void RewriteInstance::adjustFunctionBoundaries() {
         continue;
       }
 
-      // This is potentially another entry point into the function.
-      uint64_t EntryOffset = NextSymRefI->first - Function.getAddress();
-      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding entry point to function "
-                        << Function << " at offset 0x"
-                        << Twine::utohexstr(EntryOffset) << '\n');
-      Function.addEntryPointAtOffset(EntryOffset);
+      auto It = MarkerSyms.find(NextSymRefI->first);
+      if (It == MarkerSyms.end() || It->second != MarkerSymType::DATA) {
+        // This is potentially another entry point into the function.
+        uint64_t EntryOffset = NextSymRefI->first - Function.getAddress();
+        LLVM_DEBUG(dbgs() << "BOLT-DEBUG: adding entry point to function "
+                          << Function << " at offset 0x"
+                          << Twine::utohexstr(EntryOffset) << '\n');
+        Function.addEntryPointAtOffset(EntryOffset);
+      }
 
       ++NextSymRefI;
     }
@@ -2177,7 +2203,9 @@ void RewriteInstance::adjustCommandLineOptions() {
     if (!opts::KeepNops.getNumOccurrences())
       opts::KeepNops = true;
 
-    // Linux kernel may resume execution after a trap instruction in some cases.
+    // Linux kernel may resume execution after a trap or x86 HLT instruction.
+    if (!opts::TerminalHLT.getNumOccurrences())
+      opts::TerminalHLT = false;
     if (!opts::TerminalTrap.getNumOccurrences())
       opts::TerminalTrap = false;
   }
@@ -3570,7 +3598,7 @@ void RewriteInstance::postProcessFunctions() {
     if (opts::PrintAll || opts::PrintCFG)
       Function.print(BC->outs(), "after building cfg");
 
-    if (opts::DumpDotAll)
+    if (opts::shouldDumpDot(Function))
       Function.dumpGraphForPass("00_build-cfg");
 
     if (opts::PrintLoopInfo) {
diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index 973261765f951..72f95cea6fa1d 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -382,10 +382,9 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     // the list of successors of this basic block as appropriate.
 
     // Any of the above code sequences assume the fall-through basic block
-    // is a dead-end BRK instruction (any immediate operand is accepted).
+    // is a dead-end trap instruction.
     const BinaryBasicBlock *BreakBB = BB.getFallthrough();
-    if (!BreakBB || BreakBB->empty() ||
-        BreakBB->front().getOpcode() != AArch64::BRK)
+    if (!BreakBB || BreakBB->empty() || !isTrap(BreakBB->front()))
       return std::nullopt;
 
     // Iterate over the instructions of BB in reverse order, matching opcodes
@@ -1744,6 +1743,34 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     Inst.addOperand(MCOperand::createImm(0));
   }
 
+  bool isTrap(const MCInst &Inst) const override {
+    if (Inst.getOpcode() != AArch64::BRK)
+      return false;
+    // Only match the immediate values that are likely to indicate this BRK
+    // instruction is emitted to terminate the program immediately and not to
+    // be handled by a SIGTRAP handler, for example.
+    switch (Inst.getOperand(0).getImm()) {
+    case 0xc470:
+    case 0xc471:
+    case 0xc472:
+    case 0xc473:
+      // Explicit Pointer Authentication check failed, see
+      // AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue().
+      return true;
+    case 0x1:
+      // __builtin_trap(), as emitted by Clang.
+      return true;
+    case 0x3e8: // decimal 1000
+      // __builtin_trap(), as emitted by GCC.
+      return true;
+    default:
+      // Some constants may indicate intentionally recoverable break-points.
+      // This is the case at least for 0xf000, which is used by
+      // __builtin_debugtrap() supported by Clang.
+      return false;
+    }
+  }
+
   bool isStorePair(const MCInst &Inst) const {
     const unsigned opcode = Inst.getOpcode();
 
diff --git a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
index a60c1a6bf156e..1842509dcc5e0 100644
--- a/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
+++ b/bolt/lib/Target/X86/X86MCPlusBuilder.cpp
@@ -223,6 +223,10 @@ class X86MCPlusBuilder : public MCPlusBuilder {
     return Inst.getOpcode() == X86::ENDBR32 || Inst.getOpcode() == X86::ENDBR64;
   }
 
+  bool isX86HLT(const MCInst &Inst) const override {
+    return Inst.getOpcode() == X86::HLT;
+  }
+
   int getPopSize(const MCInst &Inst) const override {
     switch (Inst.getOpcode()) {
     case X86::POP16r:
diff --git a/bolt/test/AArch64/data-marker-invalidates-extra-entrypoint.s b/bolt/test/AArch64/data-marker-invalidates-extra-entrypoint.s
new file mode 100644
index 0000000000000..3bcbcbba8a385
--- /dev/null
+++ b/bolt/test/AArch64/data-marker-invalidates-extra-entrypoint.s
@@ -0,0 +1,38 @@
+# This test is to ensure that we query data marker symbols to avoid
+# misidentifying constant data island symbol as extra entry point.
+
+# RUN: %clang %cflags %s -o %t.so -Wl,-q -Wl,--init=_bar -Wl,--fini=_bar
+# RUN: llvm-bolt %t.so -o %t.instr.so
+
+  .text
+  .global _start
+  .type _start, %function
+_start:
+  ret
+
+  .text
+  .global _foo
+  .type _foo, %function
+_foo:
+  cbz x1, _foo_2
+_foo_1:
+  add x1, x2, x0
+  b _foo
+_foo_2:
+  ret
+
+# None of these constant island symbols should be identified as extra entry
+# point for function `_foo'.
+  .align 4
+_const1: .short 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80
+_const2: .short 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0
+_const3: .short 0x04, 0x08, 0x0c, 0x20, 0x60, 0x80, 0xa0, 0xc0
+
+  .text
+  .global _bar
+  .type _bar, %function
+_bar:
+  ret
+
+  # Dummy relocation to force relocation mode
+  .reloc 0, R_AARCH64_NONE
diff --git a/bolt/test/AArch64/unsupported-passes.test b/bolt/test/AArch64/unsupported-passes.test
new file mode 100644
index 0000000000000..886fc1c574dcf
--- /dev/null
+++ b/bolt/test/AArch64/unsupported-passes.test
@@ -0,0 +1,8 @@
+// Checks that non-fully supported passes on AArch64 are handled appropriately.
+
+// REQUIRES: system-linux,asserts,target=aarch64{{.*}}
+
+RUN: %clang %cflags %p/../Inputs/hello.c -o %t -Wl,-q
+RUN: not llvm-bolt %t -o %t.bolt --frame-opt=all 2>&1 | FileCheck %s
+
+CHECK: BOLT-ERROR: frame-optimizer is supported only on X86
diff --git a/bolt/test/Inputs/multi-func.cpp b/bolt/test/Inputs/multi-func.cpp
new file mode 100644
index 0000000000000..61c968fc27f98
--- /dev/null
+++ b/bolt/test/Inputs/multi-func.cpp
@@ -0,0 +1,24 @@
+#include <iostream>
+
+// Multiple functions to test selective dumping
+int add(int a, int b) { return a + b; }
+
+int multiply(int a, int b) { return a * b; }
+
+int main_helper() {
+  std::cout << "Helper function" << std::endl;
+  return 42;
+}
+
+int main_secondary() { return add(5, 3); }
+
+void other_function() { std::cout << "Other function" << std::endl; }
+
+int main() {
+  int result = add(10, 20);
+  result = multiply(result, 2);
+  main_helper();
+  main_secondary();
+  other_function();
+  return result;
+}
diff --git a/bolt/test/X86/hlt-terminator.s b/bolt/test/X86/hlt-terminator.s
new file mode 100644
index 0000000000000..3f67182fdf432
--- /dev/null
+++ b/bolt/test/X86/hlt-terminator.s
@@ -0,0 +1,24 @@
+## Check that HLT instruction is handled differently depending on the flags.
+## It's a terminator in the user-level code, but the execution can resume in
+## ring 0.
+
+# RUN: %clang %cflags %s -static -o %t.exe -nostdlib
+# RUN: llvm-bolt %t.exe --print-cfg --print-only=main --terminal-x86-hlt=0 \
+# RUN:   -o %t.ring0 2>&1 | FileCheck %s --check-prefix=CHECK-RING0
+# RUN: llvm-bolt %t.exe --print-cfg --print-only=main \
+# RUN:   -o %t.ring3 2>&1 | FileCheck %s --check-prefix=CHECK-RING3
+# RUN: llvm-objdump -d %t.ring0 --print-imm-hex | FileCheck %s --check-prefix=CHECK-BIN
+
+# CHECK-RING0: BB Count    : 1
+# CHECK-RING3: BB Count    : 2
+
+# CHECK-BIN: <main>:
+# CHECK-BIN-NEXT: f4                            hlt
+# CHECK-BIN-NEXT: c3                            retq
+
+.global main
+  .type main, %function
+main:
+        hlt
+        retq
+.size main, .-main
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-address-checks.s b/bolt/test/binary-analysis/AArch64/gs-pauth-address-checks.s
index 3f982ddaf6e38..74f276197923f 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pauth-address-checks.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-address-checks.s
@@ -31,7 +31,7 @@ resign_xpaci_good:
         xpaci   x16
         cmp     x0, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -46,7 +46,7 @@ resign_xpacd_good:
         xpacd   x16
         cmp     x0, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc473
 1:
         pacda   x0, x2
         ret
@@ -117,7 +117,7 @@ resign_xpaci_unrelated_auth_and_check:
         xpaci   x16
         cmp     x0, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x10, x2
         ret
@@ -139,7 +139,7 @@ resign_xpaci_wrong_pattern_1:
         xpaci   x16
         cmp     x0, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -157,7 +157,7 @@ resign_xpaci_wrong_pattern_2:
         xpaci   x0        // x0 instead of x16
         cmp     x0, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -174,7 +174,7 @@ resign_xpaci_wrong_pattern_3:
         xpaci   x16
         cmp     x16, x16  // x16 instead of x0
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -191,7 +191,7 @@ resign_xpaci_wrong_pattern_4:
         xpaci   x16
         cmp     x0, x0    // x0 instead of x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -208,7 +208,7 @@ resign_xpaci_wrong_pattern_5:
         mov     x16, x16  // replace xpaci with a no-op instruction
         cmp     x0, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -228,7 +228,7 @@ resign_xpaclri_good:
         xpaclri
         cmp     x30, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x30, x2
 
@@ -246,7 +246,7 @@ xpaclri_check_keeps_lr_safe:
         xpaclri         // clobbers LR
         cmp     x30, x16
         b.eq    1f
-        brk     0x1234    // marks LR as trusted and safe-to-dereference
+        brk     0xc471    // marks LR as trusted and safe-to-dereference
 1:
         ret             // not reporting non-protected return
         .size xpaclri_check_keeps_lr_safe, .-xpaclri_check_keeps_lr_safe
@@ -265,7 +265,7 @@ xpaclri_check_requires_safe_lr:
         xpaclri
         cmp     x30, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         ret
         .size xpaclri_check_requires_safe_lr, .-xpaclri_check_requires_safe_lr
@@ -283,7 +283,7 @@ resign_xpaclri_wrong_reg:
         xpaclri         // ... but xpaclri still operates on x30
         cmp     x20, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x20, x2
 
@@ -303,7 +303,7 @@ resign_checked_not_authenticated:
         xpaci   x16
         cmp     x0, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -323,7 +323,7 @@ resign_checked_before_authenticated:
         xpaci   x16
         cmp     x0, x16
         b.eq    1f
-        brk     0x1234
+        brk     0xc471
 1:
         autib   x0, x1
         pacia   x0, x2
@@ -339,7 +339,7 @@ resign_high_bits_tbz_good:
         autib   x0, x1
         eor     x16, x0, x0, lsl #1
         tbz     x16, #62, 1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -378,7 +378,7 @@ resign_high_bits_tbz_wrong_bit:
         autib   x0, x1
         eor     x16, x0, x0, lsl #1
         tbz     x16, #63, 1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -393,7 +393,7 @@ resign_high_bits_tbz_wrong_shift_amount:
         autib   x0, x1
         eor     x16, x0, x0, lsl #2
         tbz     x16, #62, 1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -408,7 +408,7 @@ resign_high_bits_tbz_wrong_shift_type:
         autib   x0, x1
         eor     x16, x0, x0, lsr #1
         tbz     x16, #62, 1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -423,7 +423,7 @@ resign_high_bits_tbz_wrong_pattern_1:
         autib   x0, x1
         eor     x16, x0, x0, lsl #1
         tbz     x17, #62, 1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -438,7 +438,7 @@ resign_high_bits_tbz_wrong_pattern_2:
         autib   x0, x1
         eor     x16, x10, x0, lsl #1
         tbz     x16, #62, 1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -453,7 +453,7 @@ resign_high_bits_tbz_wrong_pattern_3:
         autib   x0, x1
         eor     x16, x0, x10, lsl #1
         tbz     x16, #62, 1f
-        brk     0x1234
+        brk     0xc471
 1:
         pacia   x0, x2
         ret
@@ -648,7 +648,7 @@ many_checked_regs:
         xpacd   x16       // ...
         cmp     x2, x16   // ...
         b.eq    2f        // end of basic block
-        brk     0x1234
+        brk     0xc473
 2:
         pacdza  x0
         pacdza  x1
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s b/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s
index c314bc7cfe5a3..f44ba21b9d484 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s
@@ -79,7 +79,7 @@ good_explicit_check:
         autia   x0, x1
         eor     x16, x0, x0, lsl #1
         tbz     x16, #62, 1f
-        brk     0x1234
+        brk     0xc470
 1:
         ret
         .size good_explicit_check, .-good_explicit_check
@@ -373,7 +373,7 @@ good_explicit_check_multi_bb:
 1:
         eor     x16, x0, x0, lsl #1
         tbz     x16, #62, 2f
-        brk     0x1234
+        brk     0xc470
 2:
         cbz     x1, 3f
         nop
@@ -685,8 +685,7 @@ good_address_arith_nocfg:
         .globl  good_explicit_check_unrelated_reg
         .type   good_explicit_check_unrelated_reg,@function
 good_explicit_check_unrelated_reg:
-// CHECK-LABEL: GS-PAUTH: authentication oracle found in function good_explicit_check_unrelated_reg, basic block {{[^,]+}}, at address
-        // FIXME: The below instruction is not an authentication oracle
+// CHECK-NOT: good_explicit_check_unrelated_reg
         autia   x2, x3    // One of possible execution paths after this instruction
                           // ends at BRK below, thus BRK used as a trap instruction
                           // should formally "check everything" not to introduce
@@ -694,7 +693,7 @@ good_explicit_check_unrelated_reg:
         autia   x0, x1
         eor     x16, x0, x0, lsl #1
         tbz     x16, #62, 1f
-        brk     0x1234
+        brk     0xc470
 1:
         ldr     x4, [x2]  // Right before this instruction X2 is checked - this
                           // should be propagated to the basic block ending with
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-signing-oracles.s b/bolt/test/binary-analysis/AArch64/gs-pauth-signing-oracles.s
index 3a4d383ec5bc6..4d4bb7b0fb251 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pauth-signing-oracles.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-signing-oracles.s
@@ -57,7 +57,7 @@ good_sign_auted_checked_brk:
         autda   x0, x2
         eor     x16, x0, x0, lsl #1
         tbz     x16, #62, 1f
-        brk     0x1234
+        brk     0xc472
 1:
         pacda   x0, x1
         ret
@@ -351,7 +351,7 @@ good_sign_auted_checked_brk_multi_bb:
 1:
         eor     x16, x0, x0, lsl #1
         tbz     x16, #62, 2f
-        brk     0x1234
+        brk     0xc472
 2:
         cbz     x4, 3f
         nop
@@ -705,7 +705,7 @@ good_resign_with_increment_brk:
         add     x0, x0, #8
         eor     x16, x0, x0, lsl #1
         tbz     x16, #62, 1f
-        brk     0x1234
+        brk     0xc472
 1:
         mov     x2, x0
         pacda   x2, x1
diff --git a/bolt/test/binary-analysis/AArch64/trap-instructions.s b/bolt/test/binary-analysis/AArch64/trap-instructions.s
new file mode 100644
index 0000000000000..7810b2d3c3626
--- /dev/null
+++ b/bolt/test/binary-analysis/AArch64/trap-instructions.s
@@ -0,0 +1,213 @@
+// RUN: %clang %cflags -march=armv8.3-a %s -o %t.exe -Wl,--emit-relocs
+// RUN: llvm-bolt-binary-analysis --scanners=pauth %t.exe 2>&1 | FileCheck %s
+
+// Test what instructions can be used to terminate the program abnormally
+// on security violation.
+//
+// All test cases have the same structure:
+//
+//      cbz     x0, 1f    // [a], ensures [c] is never reported as unreachable
+//      autia   x2, x3
+//      cbz     x1, 2f    // [b]
+//      [instruction under test]
+// 1:
+//      ret               // [c]
+// 2:
+//      ldr     x0, [x2]
+//      ret
+//
+// This is to handle three possible cases: the instruction under test may be
+// considered by BOLT as
+// * trapping (and thus no-return): after being authenticated, x2 is ether
+//   checked by LDR (if [b] is taken) or the program is terminated
+//   immediately without leaking x2 (if [b] falls through to the trapping
+//   instruction under test). Nothing is reported.
+// * non-trapping, but no-return (such as calling abort()): x2 is leaked if [b]
+//   falls through. Authentication oracle is reported.
+// * non-trapping and falling-through (i.e. a regular instruction):
+//   x2 is leaked by [c]. Authentication oracle is reported.
+
+        .text
+
+        .globl  brk_key_ia
+        .type   brk_key_ia,@function
+brk_key_ia:
+// CHECK-NOT: brk_key_ia
+        cbz     x0, 1f
+        autia   x2, x3
+        cbz     x1, 2f
+        brk     0xc470
+1:
+        ret
+2:
+        ldr     x0, [x2]
+        ret
+        .size   brk_key_ia, .-brk_key_ia
+
+        .globl  brk_key_ib
+        .type   brk_key_ib,@function
+brk_key_ib:
+// CHECK-NOT: brk_key_ib
+        cbz     x0, 1f
+        autia   x2, x3
+        cbz     x1, 2f
+        brk     0xc471
+1:
+        ret
+2:
+        ldr     x0, [x2]
+        ret
+        .size   brk_key_ib, .-brk_key_ib
+
+        .globl  brk_key_da
+        .type   brk_key_da,@function
+brk_key_da:
+// CHECK-NOT: brk_key_da
+        cbz     x0, 1f
+        autia   x2, x3
+        cbz     x1, 2f
+        brk     0xc472
+1:
+        ret
+2:
+        ldr     x0, [x2]
+        ret
+        .size   brk_key_da, .-brk_key_da
+
+        .globl  brk_key_db
+        .type   brk_key_db,@function
+brk_key_db:
+// CHECK-NOT: brk_key_db
+        cbz     x0, 1f
+        autia   x2, x3
+        cbz     x1, 2f
+        brk     0xc473
+1:
+        ret
+2:
+        ldr     x0, [x2]
+        ret
+        .size   brk_key_db, .-brk_key_db
+
+// The immediate operand of BRK instruction may indicate whether the instruction
+// is intended to be a non-recoverable trap: for example, for this code
+//
+//     int test_trap(void) {
+//       __builtin_trap();
+//       return 42;
+//     }
+//     int test_debugtrap(void) {
+//       __builtin_debugtrap();
+//       return 42;
+//     }
+//
+// Clang produces the following assembly:
+//
+//     test_trap:
+//             brk     #0x1
+//     test_debugtrap:
+//             brk     #0xf000
+//             mov     w0, #42
+//             ret
+//
+// In GCC, __builtin_trap() uses "brk 0x3e8" (i.e. decimal 1000) and
+// __builtin_debugtrap() is not supported.
+//
+// At the time of writing these test cases, any BRK instruction is considered
+// no-return by BOLT, thus it ends its basic block and prevents falling through
+// to the next BB.
+// FIXME: Make BOLT handle __builtin_debugtrap() properly from the CFG point
+//        of view.
+
+        .globl  brk_gcc_builtin_trap
+        .type   brk_gcc_builtin_trap,@function
+brk_gcc_builtin_trap:
+// CHECK-NOT: brk_gcc_builtin_trap
+        cbz     x0, 1f
+        autia   x2, x3
+        cbz     x1, 2f
+        brk     0x3e8     // __builtin_trap()
+1:
+        ret
+2:
+        ldr     x0, [x2]
+        ret
+        .size   brk_gcc_builtin_trap, .-brk_gcc_builtin_trap
+
+        .globl  brk_clang_builtin_trap
+        .type   brk_clang_builtin_trap,@function
+brk_clang_builtin_trap:
+// CHECK-NOT: brk_clang_builtin_trap
+        cbz     x0, 1f
+        autia   x2, x3
+        cbz     x1, 2f
+        brk     0x1       // __builtin_trap()
+1:
+        ret
+2:
+        ldr     x0, [x2]
+        ret
+        .size   brk_clang_builtin_trap, .-brk_clang_builtin_trap
+
+        .globl  brk_clang_builtin_debugtrap
+        .type   brk_clang_builtin_debugtrap,@function
+brk_clang_builtin_debugtrap:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function brk_clang_builtin_debugtrap, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x2, x3
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        cbz     x0, 1f
+        autia   x2, x3
+        cbz     x1, 2f
+        brk     0xf000    // __builtin_debugtrap()
+1:
+        ret
+2:
+        ldr     x0, [x2]
+        ret
+        .size   brk_clang_builtin_debugtrap, .-brk_clang_builtin_debugtrap
+
+// Conservatively assume BRK with an unknown immediate operand as not suitable
+// for terminating the program on security violation.
+        .globl  brk_unknown_imm
+        .type   brk_unknown_imm,@function
+brk_unknown_imm:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function brk_unknown_imm, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x2, x3
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        cbz     x0, 1f
+        autia   x2, x3
+        cbz     x1, 2f
+        brk     0x3572
+1:
+        ret
+2:
+        ldr     x0, [x2]
+        ret
+        .size   brk_unknown_imm, .-brk_unknown_imm
+
+// Conservatively assume calling the abort() function may be an unsafe way to
+// terminate the program, as there is some amount of instructions that would
+// be executed when the program state is already tampered with.
+        .globl  call_abort_fn
+        .type   call_abort_fn,@function
+call_abort_fn:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function call_abort_fn, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x2, x3
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        cbz     x0, 1f
+        autia   x2, x3
+        cbz     x1, 2f
+        b       abort    // a no-return tail call to abort()
+1:
+        ret
+2:
+        ldr     x0, [x2]
+        ret
+        .size   call_abort_fn, .-call_abort_fn
+
+        .globl  main
+        .type   main,@function
+main:
+        mov     x0, 0
+        ret
+        .size   main, .-main
diff --git a/bolt/test/dump-dot-func.test b/bolt/test/dump-dot-func.test
new file mode 100644
index 0000000000000..510713dde6167
--- /dev/null
+++ b/bolt/test/dump-dot-func.test
@@ -0,0 +1,52 @@
+# Test the --dump-dot-func option with multiple functions 
+# (includes tests for both mangled/unmangled names)
+
+RUN: %clang++ %p/Inputs/multi-func.cpp -o %t.exe -Wl,-q
+
+# Test 1: --dump-dot-func with specific function name (mangled)
+RUN: llvm-bolt %t.exe -o %t.bolt1 --dump-dot-func=_Z3addii -v=1 2>&1 | FileCheck %s --check-prefix=ADD
+
+# Test 2: --dump-dot-func with regex pattern (main.*)
+RUN: llvm-bolt %t.exe -o %t.bolt2 --dump-dot-func="main.*" -v=1 2>&1 | FileCheck %s --check-prefix=MAIN-REGEX
+
+# Test 3: --dump-dot-func with multiple specific functions (mangled names)
+RUN: llvm-bolt %t.exe -o %t.bolt3 --dump-dot-func=_Z3addii,_Z8multiplyii -v=1 2>&1 | FileCheck %s --check-prefix=MULTI
+
+# Test 4: No option specified should create no dot files
+RUN: llvm-bolt %t.exe -o %t.bolt4 2>&1 | FileCheck %s --check-prefix=NONE
+
+# Test 5: --dump-dot-func with non-existent function
+RUN: llvm-bolt %t.exe -o %t.bolt5 --dump-dot-func=nonexistent -v=1 2>&1 | FileCheck %s --check-prefix=NONEXISTENT
+
+# Test 6: Backward compatibility - --dump-dot-all should still work
+RUN: llvm-bolt %t.exe -o %t.bolt6 --dump-dot-all -v=1 2>&1 | FileCheck %s --check-prefix=ALL
+
+# Test 7: Test with unmangled function name (main function)
+RUN: llvm-bolt %t.exe -o %t.bolt7 --dump-dot-func=main -v=1 2>&1 | FileCheck %s --check-prefix=MAIN-UNMANGLED
+
+# Check that specific functions are dumped
+ADD: BOLT-INFO: dumping CFG to _Z3addii-00_build-cfg.dot
+ADD-NOT: BOLT-INFO: dumping CFG to main-00_build-cfg.dot
+ADD-NOT: BOLT-INFO: dumping CFG to _Z8multiplyii-00_build-cfg.dot
+ADD-NOT: BOLT-INFO: dumping CFG to _Z11main_helperv-00_build-cfg.dot
+
+MAIN-REGEX-DAG: BOLT-INFO: dumping CFG to main-00_build-cfg.dot
+MAIN-REGEX-NOT: BOLT-INFO: dumping CFG to _Z3addii-00_build-cfg.dot
+MAIN-REGEX-NOT: BOLT-INFO: dumping CFG to _Z8multiplyii-00_build-cfg.dot
+
+MULTI-DAG: BOLT-INFO: dumping CFG to _Z3addii-00_build-cfg.dot
+MULTI-DAG: BOLT-INFO: dumping CFG to _Z8multiplyii-00_build-cfg.dot
+MULTI-NOT: BOLT-INFO: dumping CFG to main-00_build-cfg.dot
+MULTI-NOT: BOLT-INFO: dumping CFG to _Z11main_helperv-00_build-cfg.dot
+
+# Should be no dumping messages when no option is specified
+NONE-NOT: BOLT-INFO: dumping CFG
+
+# Should be no dumping messages for non-existent function
+NONEXISTENT-NOT: BOLT-INFO: dumping CFG
+
+ALL: BOLT-INFO: dumping CFG to main-00_build-cfg.dot
+
+MAIN-UNMANGLED: BOLT-INFO: dumping CFG to main-00_build-cfg.dot
+MAIN-UNMANGLED-NOT: BOLT-INFO: dumping CFG to _Z3addii-00_build-cfg.dot
+MAIN-UNMANGLED-NOT: BOLT-INFO: dumping CFG to _Z8multiplyii-00_build-cfg.dot
\ No newline at end of file
diff --git a/bolt/unittests/Core/CMakeLists.txt b/bolt/unittests/Core/CMakeLists.txt
index 54e8ea10cda12..f10b0d9472067 100644
--- a/bolt/unittests/Core/CMakeLists.txt
+++ b/bolt/unittests/Core/CMakeLists.txt
@@ -11,6 +11,11 @@ add_bolt_unittest(CoreTests
   MemoryMaps.cpp
   DynoStats.cpp
 
+  # FIXME CoreTests uses `llvm::detail::TakeError(llvm::Error)`, but linking
+  #       to LLVMTestingSupport introduces a transitive dependency on the
+  #       dynamic LLVM library when LLVM_LINK_LLVM_DYLIB is ON.
+  ${LLVM_MAIN_SRC_DIR}/lib/Testing/Support/Error.cpp
+
   DISABLE_LLVM_LINK_LLVM_DYLIB
   )
 
@@ -20,7 +25,6 @@ target_link_libraries(CoreTests
   LLVMBOLTRewrite
   LLVMBOLTProfile
   LLVMBOLTUtils
-  LLVMTestingSupport
   )
 
 foreach (tgt ${BOLT_TARGETS_TO_BUILD})
diff --git a/bolt/unittests/Profile/CMakeLists.txt b/bolt/unittests/Profile/CMakeLists.txt
index ce01c6c4b949e..7b3cbd2cad724 100644
--- a/bolt/unittests/Profile/CMakeLists.txt
+++ b/bolt/unittests/Profile/CMakeLists.txt
@@ -16,7 +16,6 @@ target_link_libraries(ProfileTests
   LLVMBOLTCore
   LLVMBOLTProfile
   LLVMTargetParser
-  LLVMTestingSupport
   )
 
 foreach (tgt ${BOLT_TARGETS_TO_BUILD})
diff --git a/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp b/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp
index 3e367ab1a5558..471ca45fb5a53 100644
--- a/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp
+++ b/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp
@@ -31,24 +31,9 @@ llvm::SmallVector<llvm::StringRef, 4> splitSymbolName(llvm::StringRef Name) {
   return Splitted;
 }
 
-SourceLocation startLocationForType(TypeLoc TLoc) {
-  // For elaborated types (e.g. `struct a::A`) we want the portion after the
-  // `struct` but including the namespace qualifier, `a::`.
-  if (TLoc.getTypeLocClass() == TypeLoc::Elaborated) {
-    NestedNameSpecifierLoc NestedNameSpecifier =
-        TLoc.castAs<ElaboratedTypeLoc>().getQualifierLoc();
-    if (NestedNameSpecifier.getNestedNameSpecifier())
-      return NestedNameSpecifier.getBeginLoc();
-    TLoc = TLoc.getNextTypeLoc();
-  }
-  return TLoc.getBeginLoc();
-}
-
 SourceLocation endLocationForType(TypeLoc TLoc) {
-  // Dig past any namespace or keyword qualifications.
-  while (TLoc.getTypeLocClass() == TypeLoc::Elaborated ||
-         TLoc.getTypeLocClass() == TypeLoc::Qualified)
-    TLoc = TLoc.getNextTypeLoc();
+  if (auto QTL = TLoc.getAs<QualifiedTypeLoc>())
+    TLoc = QTL.getUnqualifiedLoc();
 
   // The location for template specializations (e.g. Foo<int>) includes the
   // templated types in its location range.  We want to restrict this to just
@@ -550,8 +535,8 @@ void ChangeNamespaceTool::run(
                  Result.Nodes.getNodeAs<NestedNameSpecifierLoc>(
                      "nested_specifier_loc")) {
     SourceLocation Start = Specifier->getBeginLoc();
-    SourceLocation End = endLocationForType(Specifier->getTypeLoc());
-    fixTypeLoc(Result, Start, End, Specifier->getTypeLoc());
+    SourceLocation End = endLocationForType(Specifier->castAsTypeLoc());
+    fixTypeLoc(Result, Start, End, Specifier->castAsTypeLoc());
   } else if (const auto *BaseInitializer =
                  Result.Nodes.getNodeAs<CXXCtorInitializer>(
                      "base_initializer")) {
@@ -562,19 +547,16 @@ void ChangeNamespaceTool::run(
     // filtered by matchers in some cases, e.g. the type is templated. We should
     // handle the record type qualifier instead.
     TypeLoc Loc = *TLoc;
-    while (Loc.getTypeLocClass() == TypeLoc::Qualified)
-      Loc = Loc.getNextTypeLoc();
-    if (Loc.getTypeLocClass() == TypeLoc::Elaborated) {
-      NestedNameSpecifierLoc NestedNameSpecifier =
-          Loc.castAs<ElaboratedTypeLoc>().getQualifierLoc();
-      // FIXME: avoid changing injected class names.
-      if (auto *NNS = NestedNameSpecifier.getNestedNameSpecifier()) {
-        const Type *SpecifierType = NNS->getAsType();
-        if (SpecifierType && SpecifierType->isRecordType())
-          return;
-      }
-    }
-    fixTypeLoc(Result, startLocationForType(Loc), endLocationForType(Loc), Loc);
+    if (auto QTL = Loc.getAs<QualifiedTypeLoc>())
+      Loc = QTL.getUnqualifiedLoc();
+    // FIXME: avoid changing injected class names.
+    if (NestedNameSpecifier NestedNameSpecifier =
+            Loc.getPrefix().getNestedNameSpecifier();
+        NestedNameSpecifier.getKind() == NestedNameSpecifier::Kind::Type &&
+        NestedNameSpecifier.getAsType()->isRecordType())
+      return;
+    fixTypeLoc(Result, Loc.getNonElaboratedBeginLoc(), endLocationForType(Loc),
+               Loc);
   } else if (const auto *VarRef =
                  Result.Nodes.getNodeAs<DeclRefExpr>("var_ref")) {
     const auto *Var = Result.Nodes.getNodeAs<VarDecl>("var_decl");
@@ -588,10 +570,9 @@ void ChangeNamespaceTool::run(
   } else if (const auto *EnumConstRef =
                  Result.Nodes.getNodeAs<DeclRefExpr>("enum_const_ref")) {
     // Do not rename the reference if it is already scoped by the EnumDecl name.
-    if (EnumConstRef->hasQualifier() &&
-        EnumConstRef->getQualifier()->getKind() ==
-            NestedNameSpecifier::SpecifierKind::TypeSpec &&
-        EnumConstRef->getQualifier()->getAsType()->isEnumeralType())
+    if (NestedNameSpecifier Qualifier = EnumConstRef->getQualifier();
+        Qualifier.getKind() == NestedNameSpecifier::Kind::Type &&
+        Qualifier.getAsType()->isEnumeralType())
       return;
     const auto *EnumConstDecl =
         Result.Nodes.getNodeAs<EnumConstantDecl>("enum_const_decl");
diff --git a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
index a64cb5ea26a79..1ab40aacbfe09 100644
--- a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
@@ -144,17 +144,22 @@ Error MustacheHTMLGenerator::generateDocs(
     } else
       return JSONGenerator.takeError();
   }
+  SmallString<128> JSONPath;
+  sys::path::native(RootDir.str() + "/json", JSONPath);
 
   StringMap<json::Value> JSONFileMap;
   {
     llvm::TimeTraceScope TS("Iterate JSON files");
     std::error_code EC;
-    sys::fs::directory_iterator JSONIter(RootDir, EC);
+    sys::fs::directory_iterator JSONIter(JSONPath, EC);
     std::vector<json::Value> JSONFiles;
     JSONFiles.reserve(Infos.size());
     if (EC)
       return createStringError("Failed to create directory iterator.");
 
+    SmallString<128> HTMLDirPath(RootDir.str() + "/html/");
+    if (auto EC = sys::fs::create_directories(HTMLDirPath))
+      return createFileError(HTMLDirPath, EC);
     while (JSONIter != sys::fs::directory_iterator()) {
       if (EC)
         return createFileError("Failed to iterate: " + JSONIter->path(), EC);
@@ -177,14 +182,15 @@ Error MustacheHTMLGenerator::generateDocs(
         return Parsed.takeError();
 
       std::error_code FileErr;
-      SmallString<16> HTMLPath(Path.begin(), Path.end());
-      sys::path::replace_extension(HTMLPath, "html");
-      raw_fd_ostream InfoOS(HTMLPath, FileErr, sys::fs::OF_None);
+      SmallString<128> HTMLFilePath(HTMLDirPath);
+      sys::path::append(HTMLFilePath, sys::path::filename(Path));
+      sys::path::replace_extension(HTMLFilePath, "html");
+      raw_fd_ostream InfoOS(HTMLFilePath, FileErr, sys::fs::OF_None);
       if (FileErr)
         return createFileOpenError(Path, FileErr);
 
-      if (Error Err = generateDocForJSON(*Parsed, sys::path::stem(HTMLPath),
-                                         HTMLPath, InfoOS, CDCtx))
+      if (Error Err = generateDocForJSON(*Parsed, sys::path::stem(HTMLFilePath),
+                                         HTMLFilePath, InfoOS, CDCtx))
         return Err;
       JSONIter.increment(EC);
     }
diff --git a/clang-tools-extra/clang-doc/JSONGenerator.cpp b/clang-tools-extra/clang-doc/JSONGenerator.cpp
index 599b381cea60d..26794a5e34d02 100644
--- a/clang-tools-extra/clang-doc/JSONGenerator.cpp
+++ b/clang-tools-extra/clang-doc/JSONGenerator.cpp
@@ -600,7 +600,9 @@ Error JSONGenerator::generateDocs(
     Info *Info = Group.getValue().get();
 
     SmallString<128> Path;
-    sys::path::native(RootDir, Path);
+    auto RootDirStr = RootDir.str() + "/json";
+    StringRef JSONDir = StringRef(RootDirStr);
+    sys::path::native(JSONDir, Path);
     if (!CreatedDirs.contains(Path)) {
       if (std::error_code Err = sys::fs::create_directories(Path);
           Err != std::error_code())
diff --git a/clang-tools-extra/clang-doc/Serialize.cpp b/clang-tools-extra/clang-doc/Serialize.cpp
index de73f68b09386..bcab4f1b8a729 100644
--- a/clang-tools-extra/clang-doc/Serialize.cpp
+++ b/clang-tools-extra/clang-doc/Serialize.cpp
@@ -902,8 +902,8 @@ parseBases(RecordInfo &I, const CXXRecordDecl *D, bool IsFileInRootDir,
     return;
   for (const CXXBaseSpecifier &B : D->bases()) {
     if (const RecordType *Ty = B.getType()->getAs<RecordType>()) {
-      if (const CXXRecordDecl *Base =
-              cast_or_null<CXXRecordDecl>(Ty->getDecl()->getDefinition())) {
+      if (const CXXRecordDecl *Base = cast_or_null<CXXRecordDecl>(
+              Ty->getOriginalDecl()->getDefinition())) {
         // Initialized without USR and name, this will be set in the following
         // if-else stmt.
         BaseRecordInfo BI(
diff --git a/clang-tools-extra/clang-include-fixer/find-all-symbols/FindAllSymbols.cpp b/clang-tools-extra/clang-include-fixer/find-all-symbols/FindAllSymbols.cpp
index bb48883f88815..1f30d27c0a54f 100644
--- a/clang-tools-extra/clang-include-fixer/find-all-symbols/FindAllSymbols.cpp
+++ b/clang-tools-extra/clang-include-fixer/find-all-symbols/FindAllSymbols.cpp
@@ -216,8 +216,7 @@ void FindAllSymbols::registerMatchers(MatchFinder *MatchFinder) {
   // Uses of most types: just look at what the typeLoc refers to.
   MatchFinder->addMatcher(
       typeLoc(isExpansionInMainFile(),
-              loc(qualType(allOf(unless(elaboratedType()),
-                                 hasDeclaration(Types.bind("use")))))),
+              loc(qualType(hasDeclaration(Types.bind("use"))))),
       this);
   // Uses of typedefs: these are often transparent to hasDeclaration, so we need
   // to handle them explicitly.
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index 4ae2864d310d0..2064c7826da0c 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -424,6 +424,10 @@ ClangTidyASTConsumerFactory::createASTConsumer(
     FinderOptions.CheckProfiling.emplace(Profiling->Records);
   }
 
+  // Avoid processing system headers, unless the user explicitly requests it
+  if (!Context.getOptions().SystemHeaders.value_or(false))
+    FinderOptions.IgnoreSystemHeaders = true;
+
   std::unique_ptr<ast_matchers::MatchFinder> Finder(
       new ast_matchers::MatchFinder(std::move(FinderOptions)));
 
@@ -540,7 +544,7 @@ runClangTidy(clang::tidy::ClangTidyContext &Context,
              ArrayRef<std::string> InputFiles,
              llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> BaseFS,
              bool ApplyAnyFix, bool EnableCheckProfile,
-             llvm::StringRef StoreCheckProfile) {
+             llvm::StringRef StoreCheckProfile, bool Quiet) {
   ClangTool Tool(Compilations, InputFiles,
                  std::make_shared<PCHContainerOperations>(), BaseFS);
 
@@ -577,8 +581,9 @@ runClangTidy(clang::tidy::ClangTidyContext &Context,
   class ActionFactory : public FrontendActionFactory {
   public:
     ActionFactory(ClangTidyContext &Context,
-                  IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> BaseFS)
-        : ConsumerFactory(Context, std::move(BaseFS)) {}
+                  IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> BaseFS,
+                  bool Quiet)
+        : ConsumerFactory(Context, std::move(BaseFS)), Quiet(Quiet) {}
     std::unique_ptr<FrontendAction> create() override {
       return std::make_unique<Action>(&ConsumerFactory);
     }
@@ -589,6 +594,8 @@ runClangTidy(clang::tidy::ClangTidyContext &Context,
                        DiagnosticConsumer *DiagConsumer) override {
       // Explicitly ask to define __clang_analyzer__ macro.
       Invocation->getPreprocessorOpts().SetUpStaticAnalyzer = true;
+      if (Quiet)
+        Invocation->getDiagnosticOpts().ShowCarets = false;
       return FrontendActionFactory::runInvocation(
           Invocation, Files, PCHContainerOps, DiagConsumer);
     }
@@ -607,9 +614,10 @@ runClangTidy(clang::tidy::ClangTidyContext &Context,
     };
 
     ClangTidyASTConsumerFactory ConsumerFactory;
+    bool Quiet;
   };
 
-  ActionFactory Factory(Context, std::move(BaseFS));
+  ActionFactory Factory(Context, std::move(BaseFS), Quiet);
   Tool.run(&Factory);
   return DiagConsumer.take();
 }
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.h b/clang-tools-extra/clang-tidy/ClangTidy.h
index 454261bbd6840..d37d68ec0a5b9 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.h
+++ b/clang-tools-extra/clang-tidy/ClangTidy.h
@@ -94,7 +94,8 @@ runClangTidy(clang::tidy::ClangTidyContext &Context,
              ArrayRef<std::string> InputFiles,
              llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> BaseFS,
              bool ApplyAnyFix, bool EnableCheckProfile = false,
-             llvm::StringRef StoreCheckProfile = StringRef());
+             llvm::StringRef StoreCheckProfile = StringRef(),
+             bool Quiet = false);
 
 /// Controls what kind of fixes clang-tidy is allowed to apply.
 enum FixBehaviour {
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index f9d75978d0ea8..fac6e0418d163 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -533,7 +533,8 @@ void ClangTidyDiagnosticConsumer::forwardDiagnostic(const Diagnostic &Info) {
       Builder << reinterpret_cast<const NamedDecl *>(Info.getRawArg(Index));
       break;
     case clang::DiagnosticsEngine::ak_nestednamespec:
-      Builder << reinterpret_cast<NestedNameSpecifier *>(Info.getRawArg(Index));
+      Builder << NestedNameSpecifier::getFromVoidPointer(
+          reinterpret_cast<void *>(Info.getRawArg(Index)));
       break;
     case clang::DiagnosticsEngine::ak_declcontext:
       Builder << reinterpret_cast<DeclContext *>(Info.getRawArg(Index));
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.h
index 7f6b652058cfd..ac71f34fed180 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationAdditionCheck.h
@@ -22,6 +22,9 @@ class DurationAdditionCheck : public ClangTidyCheck {
 public:
   DurationAdditionCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.h
index d759e1d8bb789..65ab7a38eb289 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationComparisonCheck.h
@@ -22,6 +22,9 @@ class DurationComparisonCheck : public ClangTidyCheck {
 public:
   DurationComparisonCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.h
index fea9f703bae65..a898ba0483966 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationConversionCastCheck.h
@@ -22,6 +22,9 @@ class DurationConversionCastCheck : public ClangTidyCheck {
 public:
   DurationConversionCastCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.h
index 6394b3f2b4675..e7c3985a7fd92 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationFactoryFloatCheck.h
@@ -24,6 +24,9 @@ class DurationFactoryFloatCheck : public ClangTidyCheck {
 public:
   DurationFactoryFloatCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.h
index 40ffb30b4769c..f5f088c49897d 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationFactoryScaleCheck.h
@@ -24,6 +24,9 @@ class DurationFactoryScaleCheck : public ClangTidyCheck {
 public:
   DurationFactoryScaleCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.h
index 17f7853adabd6..c865f2f842a0d 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationSubtractionCheck.h
@@ -22,6 +22,9 @@ class DurationSubtractionCheck : public ClangTidyCheck {
 public:
   DurationSubtractionCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.h b/clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.h
index aa25c5f43e0f0..fc9cf23459425 100644
--- a/clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/DurationUnnecessaryConversionCheck.h
@@ -22,6 +22,9 @@ class DurationUnnecessaryConversionCheck : public ClangTidyCheck {
 public:
   DurationUnnecessaryConversionCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.h b/clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.h
index 0c64708758b46..bf22977e9d0df 100644
--- a/clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/TimeComparisonCheck.h
@@ -22,6 +22,9 @@ class TimeComparisonCheck : public ClangTidyCheck {
 public:
   TimeComparisonCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h
index c947f6bca7f31..9e2ec1c8def20 100644
--- a/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h
+++ b/clang-tools-extra/clang-tidy/abseil/TimeSubtractionCheck.h
@@ -22,6 +22,9 @@ class TimeSubtractionCheck : public ClangTidyCheck {
 public:
   TimeSubtractionCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
diff --git a/clang-tools-extra/clang-tidy/add_new_check.py b/clang-tools-extra/clang-tidy/add_new_check.py
index e366f10053535..2b51a1dc40ebc 100755
--- a/clang-tools-extra/clang-tidy/add_new_check.py
+++ b/clang-tools-extra/clang-tidy/add_new_check.py
@@ -89,13 +89,9 @@ def write_header(
             + check_name_camel.upper()
             + "_H"
         )
-        f.write("//===--- ")
-        f.write(os.path.basename(filename))
-        f.write(" - clang-tidy ")
-        f.write("-" * max(0, 42 - len(os.path.basename(filename))))
-        f.write("*- C++ -*-===//")
         f.write(
             """
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -145,13 +141,9 @@ def write_implementation(
     filename = os.path.join(module_path, check_name_camel) + ".cpp"
     print("Creating %s..." % filename)
     with io.open(filename, "w", encoding="utf8", newline="\n") as f:
-        f.write("//===--- ")
-        f.write(os.path.basename(filename))
-        f.write(" - clang-tidy ")
-        f.write("-" * max(0, 51 - len(os.path.basename(filename))))
-        f.write("-===//")
         f.write(
             """
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp
index 6565fa3f7c85b..0625468d9da88 100644
--- a/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/CrtpConstructorAccessibilityCheck.cpp
@@ -43,7 +43,8 @@ static bool isDerivedClassBefriended(const CXXRecordDecl *CRTP,
       return false;
     }
 
-    return FriendType->getType()->getAsCXXRecordDecl() == Derived;
+    return declaresSameEntity(FriendType->getType()->getAsCXXRecordDecl(),
+                              Derived);
   });
 }
 
@@ -55,7 +56,8 @@ getDerivedParameter(const ClassTemplateSpecializationDecl *CRTP,
       CRTP->getTemplateArgs().asArray(), [&](const TemplateArgument &Arg) {
         ++Idx;
         return Arg.getKind() == TemplateArgument::Type &&
-               Arg.getAsType()->getAsCXXRecordDecl() == Derived;
+               declaresSameEntity(Arg.getAsType()->getAsCXXRecordDecl(),
+                                  Derived);
       });
 
   return AnyOf ? CRTP->getSpecializedTemplate()
diff --git a/clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.h b/clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.h
index 3044304892029..981e9b571a618 100644
--- a/clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/DanglingHandleCheck.h
@@ -13,14 +13,16 @@
 
 namespace clang::tidy::bugprone {
 
-/// Detect dangling references in value handlers like
-/// std::experimental::string_view.
+/// Detect dangling references in value handlers like std::string_view.
 ///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/bugprone/dangling-handle.html
 class DanglingHandleCheck : public ClangTidyCheck {
 public:
   DanglingHandleCheck(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
diff --git a/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
index a179d4bf66b4d..7f1eeef8ea0fd 100644
--- a/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
@@ -577,7 +577,7 @@ approximateImplicitConversion(const TheCheck &Check, QualType LType,
                               ImplicitConversionModellingMode ImplicitMode);
 
 static inline bool isUselessSugar(const Type *T) {
-  return isa<AttributedType, DecayedType, ElaboratedType, ParenType>(T);
+  return isa<AttributedType, DecayedType, ParenType>(T);
 }
 
 namespace {
@@ -1040,7 +1040,9 @@ approximateStandardConversionSequence(const TheCheck &Check, QualType From,
   const auto *ToRecord = To->getAsCXXRecordDecl();
   if (isDerivedToBase(FromRecord, ToRecord)) {
     LLVM_DEBUG(llvm::dbgs() << "--- approximateStdConv. Derived To Base.\n");
-    WorkType = QualType{ToRecord->getTypeForDecl(), FastQualifiersToApply};
+    WorkType = QualType{
+        ToRecord->getASTContext().getCanonicalTagType(ToRecord)->getTypePtr(),
+        FastQualifiersToApply};
   }
 
   if (Ctx.getLangOpts().CPlusPlus17 && FromPtr && ToPtr) {
@@ -1072,9 +1074,9 @@ approximateStandardConversionSequence(const TheCheck &Check, QualType From,
     WorkType = To;
   }
 
-  if (WorkType == To) {
+  if (Ctx.hasSameType(WorkType, To)) {
     LLVM_DEBUG(llvm::dbgs() << "<<< approximateStdConv. Reached 'To' type.\n");
-    return {WorkType};
+    return {Ctx.getCommonSugaredType(WorkType, To)};
   }
 
   LLVM_DEBUG(llvm::dbgs() << "<<< approximateStdConv. Did not reach 'To'.\n");
@@ -1219,7 +1221,7 @@ tryConversionOperators(const TheCheck &Check, const CXXRecordDecl *RD,
 
   if (std::optional<UserDefinedConversionSelector::PreparedConversion>
           SelectedConversion = ConversionSet()) {
-    QualType RecordType{RD->getTypeForDecl(), 0};
+    CanQualType RecordType = RD->getASTContext().getCanonicalTagType(RD);
 
     ConversionSequence Result{RecordType, ToType};
     // The conversion from the operator call's return type to ToType was
@@ -1270,7 +1272,7 @@ tryConvertingConstructors(const TheCheck &Check, QualType FromType,
 
   if (std::optional<UserDefinedConversionSelector::PreparedConversion>
           SelectedConversion = ConversionSet()) {
-    QualType RecordType{RD->getTypeForDecl(), 0};
+    CanQualType RecordType = RD->getASTContext().getCanonicalTagType(RD);
 
     ConversionSequence Result{FromType, RecordType};
     Result.AfterFirstStandard = SelectedConversion->Seq.AfterFirstStandard;
@@ -1573,6 +1575,10 @@ template <typename T, std::size_t N = SmallDataStructureSize>
 using ParamToSmallSetMap =
     llvm::DenseMap<const ParmVarDecl *, llvm::SmallSet<T, N>>;
 
+template <typename T, std::size_t N = SmallDataStructureSize>
+using ParamToSmallPtrSetMap =
+    llvm::DenseMap<const ParmVarDecl *, llvm::SmallPtrSet<T, N>>;
+
 /// Returns whether the sets mapped to the two elements in the map have at
 /// least one element in common.
 template <typename MapTy, typename ElemTy>
@@ -1697,7 +1703,7 @@ class PassedToSameFunction {
 /// Implements the heuristic that marks two parameters related if the same
 /// member is accessed (referred to) inside the current function's body.
 class AccessedSameMemberOf {
-  ParamToSmallSetMap<const Decl *> AccessedMembers;
+  ParamToSmallPtrSetMap<const Decl *> AccessedMembers;
 
 public:
   void setup(const FunctionDecl *FD) {
diff --git a/clang-tools-extra/clang-tidy/bugprone/FoldInitTypeCheck.h b/clang-tools-extra/clang-tidy/bugprone/FoldInitTypeCheck.h
index af8c1e50ef86c..435c440ddd29f 100644
--- a/clang-tools-extra/clang-tidy/bugprone/FoldInitTypeCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/FoldInitTypeCheck.h
@@ -26,6 +26,9 @@ class FoldInitTypeCheck : public ClangTidyCheck {
 public:
   FoldInitTypeCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/ForwardDeclarationNamespaceCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ForwardDeclarationNamespaceCheck.cpp
index 75ef628436738..070ed04efffc4 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ForwardDeclarationNamespaceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ForwardDeclarationNamespaceCheck.cpp
@@ -69,10 +69,9 @@ void ForwardDeclarationNamespaceCheck::check(
     //      struct B { friend A; };
     //    \endcode
     // `A` will not be marked as "referenced" in the AST.
-    if (const TypeSourceInfo *Tsi = Decl->getFriendType()) {
-      QualType Desugared = Tsi->getType().getDesugaredType(*Result.Context);
-      FriendTypes.insert(Desugared.getTypePtr());
-    }
+    if (const TypeSourceInfo *Tsi = Decl->getFriendType())
+      FriendTypes.insert(
+          Tsi->getType()->getCanonicalTypeUnqualified().getTypePtr());
   }
 }
 
@@ -119,7 +118,9 @@ void ForwardDeclarationNamespaceCheck::onEndOfTranslationUnit() {
       if (CurDecl->hasDefinition() || CurDecl->isReferenced()) {
         continue; // Skip forward declarations that are used/referenced.
       }
-      if (FriendTypes.contains(CurDecl->getTypeForDecl())) {
+      if (FriendTypes.contains(CurDecl->getASTContext()
+                                   .getCanonicalTagType(CurDecl)
+                                   ->getTypePtr())) {
         continue; // Skip forward declarations referenced as friend.
       }
       if (CurDecl->getLocation().isMacroID() ||
diff --git a/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
index 00e8f7e514368..10b747e17e2ad 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ForwardingReferenceOverloadCheck.cpp
@@ -33,21 +33,17 @@ AST_MATCHER(QualType, isEnableIf) {
     BaseType = BaseType->getPointeeType().getTypePtr();
   }
   // Case: type parameter dependent (enable_if<is_integral<T>>).
-  if (const auto *Dependent = BaseType->getAs<DependentNameType>()) {
-    BaseType = Dependent->getQualifier()->getAsType();
-  }
+  if (const auto *Dependent = BaseType->getAs<DependentNameType>())
+    BaseType = Dependent->getQualifier().getAsType();
   if (!BaseType)
     return false;
   if (CheckTemplate(BaseType->getAs<TemplateSpecializationType>()))
     return true; // Case: enable_if_t< >.
-  if (const auto *Elaborated = BaseType->getAs<ElaboratedType>()) {
-    if (const auto *Q = Elaborated->getQualifier())
-      if (const auto *Qualifier = Q->getAsType()) {
-        if (CheckTemplate(Qualifier->getAs<TemplateSpecializationType>())) {
-          return true; // Case: enable_if< >::type.
-        }
-      }
-  }
+  if (const auto *TT = BaseType->getAs<TypedefType>())
+    if (NestedNameSpecifier Q = TT->getQualifier();
+        Q.getKind() == NestedNameSpecifier::Kind::Type)
+      if (CheckTemplate(Q.getAsType()->getAs<TemplateSpecializationType>()))
+        return true; // Case: enable_if< >::type.
   return false;
 }
 AST_MATCHER_P(TemplateTypeParmDecl, hasDefaultArgument,
diff --git a/clang-tools-extra/clang-tidy/bugprone/IncorrectEnableIfCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/IncorrectEnableIfCheck.cpp
index 75f1107904fce..07cd90d64c2a4 100644
--- a/clang-tools-extra/clang-tidy/bugprone/IncorrectEnableIfCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/IncorrectEnableIfCheck.cpp
@@ -32,13 +32,10 @@ AST_MATCHER_P(TemplateTypeParmDecl, hasUnnamedDefaultArgument,
 void IncorrectEnableIfCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       templateTypeParmDecl(
-          hasUnnamedDefaultArgument(
-              elaboratedTypeLoc(
-                  hasNamedTypeLoc(templateSpecializationTypeLoc(
-                                      loc(qualType(hasDeclaration(namedDecl(
-                                          hasName("::std::enable_if"))))))
-                                      .bind("enable_if_specialization")))
-                  .bind("elaborated")))
+          hasUnnamedDefaultArgument(templateSpecializationTypeLoc(
+                                        loc(qualType(hasDeclaration(namedDecl(
+                                            hasName("::std::enable_if"))))))
+                                        .bind("enable_if_specialization")))
           .bind("enable_if"),
       this);
 }
@@ -46,13 +43,11 @@ void IncorrectEnableIfCheck::registerMatchers(MatchFinder *Finder) {
 void IncorrectEnableIfCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *EnableIf =
       Result.Nodes.getNodeAs<TemplateTypeParmDecl>("enable_if");
-  const auto *ElaboratedLoc =
-      Result.Nodes.getNodeAs<ElaboratedTypeLoc>("elaborated");
   const auto *EnableIfSpecializationLoc =
       Result.Nodes.getNodeAs<TemplateSpecializationTypeLoc>(
           "enable_if_specialization");
 
-  if (!EnableIf || !ElaboratedLoc || !EnableIfSpecializationLoc)
+  if (!EnableIf || !EnableIfSpecializationLoc)
     return;
 
   const SourceManager &SM = *Result.SourceManager;
@@ -62,8 +57,10 @@ void IncorrectEnableIfCheck::check(const MatchFinder::MatchResult &Result) {
   auto Diag = diag(EnableIf->getBeginLoc(),
                    "incorrect std::enable_if usage detected; use "
                    "'typename std::enable_if<...>::type'");
+  // FIXME: This should handle the enable_if specialization already having an
+  // elaborated keyword.
   if (!getLangOpts().CPlusPlus20) {
-    Diag << FixItHint::CreateInsertion(ElaboratedLoc->getBeginLoc(),
+    Diag << FixItHint::CreateInsertion(EnableIfSpecializationLoc->getBeginLoc(),
                                        "typename ");
   }
   Diag << FixItHint::CreateInsertion(RAngleLoc.getLocWithOffset(1), "::type");
diff --git a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
index 4b495e3877000..cda9c4e7a6e58 100644
--- a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
@@ -188,7 +188,7 @@ static bool isKnownToHaveValue(const Expr &Cond, const ASTContext &Ctx,
 /// \return true iff all `CallExprs` visited have callees; false otherwise
 ///         indicating there is an unresolved indirect call.
 static bool populateCallees(const Stmt *StmtNode,
-                            llvm::SmallSet<const Decl *, 16> &Callees) {
+                            llvm::SmallPtrSet<const Decl *, 16> &Callees) {
   if (const auto *Call = dyn_cast<CallExpr>(StmtNode)) {
     const Decl *Callee = Call->getDirectCallee();
 
@@ -212,7 +212,7 @@ static bool populateCallees(const Stmt *StmtNode,
 /// returns true iff `SCC` contains `Func` and its' function set overlaps with
 /// `Callees`
 static bool overlap(ArrayRef<CallGraphNode *> SCC,
-                    const llvm::SmallSet<const Decl *, 16> &Callees,
+                    const llvm::SmallPtrSet<const Decl *, 16> &Callees,
                     const Decl *Func) {
   bool ContainsFunc = false, Overlap = false;
 
@@ -264,7 +264,7 @@ static bool hasRecursionOverStaticLoopCondVariables(const Expr *Cond,
   if (!hasStaticLocalVariable(Cond))
     return false;
 
-  llvm::SmallSet<const Decl *, 16> CalleesInLoop;
+  llvm::SmallPtrSet<const Decl *, 16> CalleesInLoop;
 
   if (!populateCallees(LoopStmt, CalleesInLoop)) {
     // If there are unresolved indirect calls, we assume there could
diff --git a/clang-tools-extra/clang-tidy/bugprone/InvalidEnumDefaultInitializationCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/InvalidEnumDefaultInitializationCheck.cpp
index f903e631e0be0..7d92ef301aec3 100644
--- a/clang-tools-extra/clang-tidy/bugprone/InvalidEnumDefaultInitializationCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/InvalidEnumDefaultInitializationCheck.cpp
@@ -67,15 +67,15 @@ class FindEnumMember : public TypeVisitor<FindEnumMember, bool> {
     return Visit(T->getElementType().getTypePtr());
   }
   bool VisitEnumType(const EnumType *T) {
-    if (isCompleteAndHasNoZeroValue(T->getDecl())) {
+    if (isCompleteAndHasNoZeroValue(T->getOriginalDecl())) {
       FoundEnum = T;
       return true;
     }
     return false;
   }
   bool VisitRecordType(const RecordType *T) {
-    const RecordDecl *RD = T->getDecl();
-    if (RD->isUnion())
+    const RecordDecl *RD = T->getOriginalDecl()->getDefinition();
+    if (!RD || RD->isUnion())
       return false;
     auto VisitField = [this](const FieldDecl *F) {
       return Visit(F->getType().getTypePtr());
@@ -125,7 +125,7 @@ void InvalidEnumDefaultInitializationCheck::check(
     if (!Finder.Visit(InitList->getArrayFiller()->getType().getTypePtr()))
       return;
     InitExpr = InitList;
-    Enum = Finder.FoundEnum->getDecl();
+    Enum = Finder.FoundEnum->getOriginalDecl();
   }
 
   if (!InitExpr || !Enum)
diff --git a/clang-tools-extra/clang-tidy/bugprone/LambdaFunctionNameCheck.h b/clang-tools-extra/clang-tidy/bugprone/LambdaFunctionNameCheck.h
index dab64f74aa6ca..04ba3596167e3 100644
--- a/clang-tools-extra/clang-tidy/bugprone/LambdaFunctionNameCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/LambdaFunctionNameCheck.h
@@ -32,6 +32,9 @@ class LambdaFunctionNameCheck : public ClangTidyCheck {
   using SourceRangeSet = std::set<SourceRange, SourceRangeLessThan>;
 
   LambdaFunctionNameCheck(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus11;
+  }
 
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
diff --git a/clang-tools-extra/clang-tidy/bugprone/MoveForwardingReferenceCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MoveForwardingReferenceCheck.cpp
index bfa2ab51a6d03..5dc988d6662df 100644
--- a/clang-tools-extra/clang-tidy/bugprone/MoveForwardingReferenceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/MoveForwardingReferenceCheck.cpp
@@ -39,24 +39,31 @@ static void replaceMoveWithForward(const UnresolvedLookupExpr *Callee,
     // std::move(). This will hopefully prevent erroneous replacements if the
     // code does unusual things (e.g. create an alias for std::move() in
     // another namespace).
-    NestedNameSpecifier *NNS = Callee->getQualifier();
-    if (!NNS) {
+    NestedNameSpecifier NNS = Callee->getQualifier();
+    switch (NNS.getKind()) {
+    case NestedNameSpecifier::Kind::Null:
       // Called as "move" (i.e. presumably the code had a "using std::move;").
       // We still conservatively put a "std::" in front of the forward because
       // we don't know whether the code also had a "using std::forward;".
       Diag << FixItHint::CreateReplacement(CallRange, "std::" + ForwardName);
-    } else if (const NamespaceBaseDecl *Namespace = NNS->getAsNamespace()) {
+      break;
+    case NestedNameSpecifier::Kind::Namespace: {
+      auto [Namespace, Prefix] = NNS.getAsNamespaceAndPrefix();
       if (Namespace->getName() == "std") {
-        if (!NNS->getPrefix()) {
+        if (!Prefix) {
           // Called as "std::move".
           Diag << FixItHint::CreateReplacement(CallRange,
                                                "std::" + ForwardName);
-        } else if (NNS->getPrefix()->getKind() == NestedNameSpecifier::Global) {
+        } else if (Prefix.getKind() == NestedNameSpecifier::Kind::Global) {
           // Called as "::std::move".
           Diag << FixItHint::CreateReplacement(CallRange,
                                                "::std::" + ForwardName);
         }
       }
+      break;
+    }
+    default:
+      return;
     }
   }
 }
diff --git a/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp
index b68888cb5b928..6344b4bb6271e 100644
--- a/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/MultipleNewInOneExpressionCheck.cpp
@@ -15,14 +15,12 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::bugprone {
 
-namespace {
-
 // Determine if the result of an expression is "stored" in some way.
 // It is true if the value is stored into a variable or used as initialization
 // or passed to a function or constructor.
 // For this use case compound assignments are not counted as a "store" (the 'E'
 // expression should have pointer type).
-bool isExprValueStored(const Expr *E, ASTContext &C) {
+static bool isExprValueStored(const Expr *E, ASTContext &C) {
   E = E->IgnoreParenCasts();
   // Get first non-paren, non-cast parent.
   ParentMapContext &PMap = C.getParentMapContext();
@@ -49,6 +47,8 @@ bool isExprValueStored(const Expr *E, ASTContext &C) {
   return isa<CallExpr, CXXConstructExpr>(ParentE);
 }
 
+namespace {
+
 AST_MATCHER_P(CXXTryStmt, hasHandlerFor,
               ast_matchers::internal::Matcher<QualType>, InnerMatcher) {
   for (unsigned NH = Node.getNumHandlers(), I = 0; I < NH; ++I) {
diff --git a/clang-tools-extra/clang-tidy/bugprone/NarrowingConversionsCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/NarrowingConversionsCheck.cpp
index 53782231b6421..249c77ca0c432 100644
--- a/clang-tools-extra/clang-tidy/bugprone/NarrowingConversionsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/NarrowingConversionsCheck.cpp
@@ -555,15 +555,22 @@ bool NarrowingConversionsCheck::handleConditionalOperator(
     // We have an expression like so: `output = cond ? lhs : rhs`
     // From the point of view of narrowing conversion we treat it as two
     // expressions `output = lhs` and `output = rhs`.
-    handleBinaryOperator(Context, CO->getLHS()->getExprLoc(), Lhs,
-                         *CO->getLHS());
-    handleBinaryOperator(Context, CO->getRHS()->getExprLoc(), Lhs,
-                         *CO->getRHS());
+    handleConditionalOperatorArgument(Context, Lhs, CO->getLHS());
+    handleConditionalOperatorArgument(Context, Lhs, CO->getRHS());
     return true;
   }
   return false;
 }
 
+void NarrowingConversionsCheck::handleConditionalOperatorArgument(
+    const ASTContext &Context, const Expr &Lhs, const Expr *Arg) {
+  if (const auto *ICE = llvm::dyn_cast<ImplicitCastExpr>(Arg))
+    if (!Arg->getIntegerConstantExpr(Context))
+      Arg = ICE->getSubExpr();
+
+  handleBinaryOperator(Context, Arg->getExprLoc(), Lhs, *Arg);
+}
+
 void NarrowingConversionsCheck::handleImplicitCast(
     const ASTContext &Context, const ImplicitCastExpr &Cast) {
   if (Cast.getExprLoc().isMacroID())
diff --git a/clang-tools-extra/clang-tidy/bugprone/NarrowingConversionsCheck.h b/clang-tools-extra/clang-tidy/bugprone/NarrowingConversionsCheck.h
index 20403f920b925..116a8cba8d321 100644
--- a/clang-tools-extra/clang-tidy/bugprone/NarrowingConversionsCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/NarrowingConversionsCheck.h
@@ -85,6 +85,8 @@ class NarrowingConversionsCheck : public ClangTidyCheck {
   bool handleConditionalOperator(const ASTContext &Context, const Expr &Lhs,
                                  const Expr &Rhs);
 
+  void handleConditionalOperatorArgument(const ASTContext &Context,
+                                         const Expr &Lhs, const Expr *Arg);
   void handleImplicitCast(const ASTContext &Context,
                           const ImplicitCastExpr &Cast);
 
diff --git a/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.h b/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.h
index b044219948988..888d29fc937bd 100644
--- a/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/OptionalValueConversionCheck.h
@@ -23,6 +23,9 @@ namespace clang::tidy::bugprone {
 class OptionalValueConversionCheck : public ClangTidyCheck {
 public:
   OptionalValueConversionCheck(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
diff --git a/clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.h b/clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.h
index f552965e1876b..293069fd24665 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/ParentVirtualCallCheck.h
@@ -21,6 +21,9 @@ class ParentVirtualCallCheck : public ClangTidyCheck {
 public:
   ParentVirtualCallCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofContainerCheck.h b/clang-tools-extra/clang-tidy/bugprone/SizeofContainerCheck.h
index 7227c3d106f43..f50ce99c6d4c0 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SizeofContainerCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/SizeofContainerCheck.h
@@ -22,6 +22,9 @@ class SizeofContainerCheck : public ClangTidyCheck {
 public:
   SizeofContainerCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
index 88e048e65d4e8..8da6227e172cd 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SizeofExpressionCheck.cpp
@@ -425,7 +425,7 @@ void SizeofExpressionCheck::check(const MatchFinder::MatchResult &Result) {
            "suspicious usage of 'sizeof(array)/sizeof(...)';"
            " denominator differs from the size of array elements")
           << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
-    } else if (NumTy && DenomTy && NumTy == DenomTy &&
+    } else if (NumTy && DenomTy && Ctx.hasSameType(NumTy, DenomTy) &&
                !NumTy->isDependentType()) {
       // Dependent type should not be compared.
       diag(E->getOperatorLoc(),
@@ -434,7 +434,7 @@ void SizeofExpressionCheck::check(const MatchFinder::MatchResult &Result) {
           << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
     } else if (!WarnOnSizeOfPointer) {
       // When 'WarnOnSizeOfPointer' is enabled, these messages become redundant:
-      if (PointedTy && DenomTy && PointedTy == DenomTy) {
+      if (PointedTy && DenomTy && Ctx.hasSameType(PointedTy, DenomTy)) {
         diag(E->getOperatorLoc(),
              "suspicious usage of 'sizeof(...)/sizeof(...)'; size of pointer "
              "is divided by size of pointed type")
@@ -463,7 +463,8 @@ void SizeofExpressionCheck::check(const MatchFinder::MatchResult &Result) {
     const auto *SizeOfExpr =
         Result.Nodes.getNodeAs<UnaryExprOrTypeTraitExpr>("sizeof-ptr-mul-expr");
 
-    if ((LPtrTy == RPtrTy) && (LPtrTy == SizeofArgTy)) {
+    if (Ctx.hasSameType(LPtrTy, RPtrTy) &&
+        Ctx.hasSameType(LPtrTy, SizeofArgTy)) {
       diag(SizeOfExpr->getBeginLoc(), "suspicious usage of 'sizeof(...)' in "
                                       "pointer arithmetic")
           << SizeOfExpr->getSourceRange() << E->getOperatorLoc()
@@ -477,7 +478,8 @@ void SizeofExpressionCheck::check(const MatchFinder::MatchResult &Result) {
     const auto *SizeOfExpr =
         Result.Nodes.getNodeAs<UnaryExprOrTypeTraitExpr>("sizeof-ptr-div-expr");
 
-    if ((LPtrTy == RPtrTy) && (LPtrTy == SizeofArgTy)) {
+    if (Ctx.hasSameType(LPtrTy, RPtrTy) &&
+        Ctx.hasSameType(LPtrTy, SizeofArgTy)) {
       diag(SizeOfExpr->getBeginLoc(), "suspicious usage of 'sizeof(...)' in "
                                       "pointer arithmetic")
           << SizeOfExpr->getSourceRange() << E->getOperatorLoc()
diff --git a/clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.cpp
index 81c38d07a0c7e..5b1b28dbfbadd 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SuspiciousMissingCommaCheck.cpp
@@ -14,10 +14,8 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::bugprone {
 
-namespace {
-
-bool isConcatenatedLiteralsOnPurpose(ASTContext *Ctx,
-                                     const StringLiteral *Lit) {
+static bool isConcatenatedLiteralsOnPurpose(ASTContext *Ctx,
+                                            const StringLiteral *Lit) {
   // String literals surrounded by parentheses are assumed to be on purpose.
   //    i.e.:  const char* Array[] = { ("a" "b" "c"), "d", [...] };
 
@@ -58,6 +56,8 @@ bool isConcatenatedLiteralsOnPurpose(ASTContext *Ctx,
   return false;
 }
 
+namespace {
+
 AST_MATCHER_P(StringLiteral, isConcatenatedLiteral, unsigned,
               MaxConcatenatedTokens) {
   return Node.getNumConcatenated() > 1 &&
diff --git a/clang-tools-extra/clang-tidy/cert/StrToNumCheck.cpp b/clang-tools-extra/clang-tidy/cert/StrToNumCheck.cpp
index 3b59d2357fe29..95536bb1cfdb2 100644
--- a/clang-tools-extra/clang-tidy/cert/StrToNumCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cert/StrToNumCheck.cpp
@@ -46,7 +46,9 @@ enum class ConversionKind {
   ToLongDouble
 };
 
-ConversionKind classifyConversionFunc(const FunctionDecl *FD) {
+} // namespace
+
+static ConversionKind classifyConversionFunc(const FunctionDecl *FD) {
   return llvm::StringSwitch<ConversionKind>(FD->getName())
       .Cases("atoi", "atol", ConversionKind::ToInt)
       .Case("atoll", ConversionKind::ToLongInt)
@@ -54,8 +56,8 @@ ConversionKind classifyConversionFunc(const FunctionDecl *FD) {
       .Default(ConversionKind::None);
 }
 
-ConversionKind classifyFormatString(StringRef Fmt, const LangOptions &LO,
-                                    const TargetInfo &TI) {
+static ConversionKind classifyFormatString(StringRef Fmt, const LangOptions &LO,
+                                           const TargetInfo &TI) {
   // Scan the format string for the first problematic format specifier, then
   // report that as the conversion type. This will miss additional conversion
   // specifiers, but that is acceptable behavior.
@@ -128,7 +130,7 @@ ConversionKind classifyFormatString(StringRef Fmt, const LangOptions &LO,
   return H.get();
 }
 
-StringRef classifyConversionType(ConversionKind K) {
+static StringRef classifyConversionType(ConversionKind K) {
   switch (K) {
   case ConversionKind::None:
     llvm_unreachable("Unexpected conversion kind");
@@ -148,7 +150,7 @@ StringRef classifyConversionType(ConversionKind K) {
   llvm_unreachable("Unknown conversion kind");
 }
 
-StringRef classifyReplacement(ConversionKind K) {
+static StringRef classifyReplacement(ConversionKind K) {
   switch (K) {
   case ConversionKind::None:
     llvm_unreachable("Unexpected conversion kind");
@@ -173,7 +175,6 @@ StringRef classifyReplacement(ConversionKind K) {
   }
   llvm_unreachable("Unknown conversion kind");
 }
-} // unnamed namespace
 
 void StrToNumCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *Call = Result.Nodes.getNodeAs<CallExpr>("expr");
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt b/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
index 2fb4d7f1d7349..0abb000991859 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
@@ -21,6 +21,7 @@ add_clang_library(clangTidyCppCoreGuidelinesModule STATIC
   OwningMemoryCheck.cpp
   PreferMemberInitializerCheck.cpp
   ProBoundsArrayToPointerDecayCheck.cpp
+  ProBoundsAvoidUncheckedContainerAccess.cpp
   ProBoundsConstantArrayIndexCheck.cpp
   ProBoundsPointerArithmeticCheck.cpp
   ProTypeConstCastCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
index 4b3b7bf963fdc..cc1ae156eef3e 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
@@ -36,6 +36,7 @@
 #include "OwningMemoryCheck.h"
 #include "PreferMemberInitializerCheck.h"
 #include "ProBoundsArrayToPointerDecayCheck.h"
+#include "ProBoundsAvoidUncheckedContainerAccess.h"
 #include "ProBoundsConstantArrayIndexCheck.h"
 #include "ProBoundsPointerArithmeticCheck.h"
 #include "ProTypeConstCastCheck.h"
@@ -107,6 +108,8 @@ class CppCoreGuidelinesModule : public ClangTidyModule {
         "cppcoreguidelines-prefer-member-initializer");
     CheckFactories.registerCheck<ProBoundsArrayToPointerDecayCheck>(
         "cppcoreguidelines-pro-bounds-array-to-pointer-decay");
+    CheckFactories.registerCheck<ProBoundsAvoidUncheckedContainerAccess>(
+        "cppcoreguidelines-pro-bounds-avoid-unchecked-container-access");
     CheckFactories.registerCheck<ProBoundsConstantArrayIndexCheck>(
         "cppcoreguidelines-pro-bounds-constant-array-index");
     CheckFactories.registerCheck<ProBoundsPointerArithmeticCheck>(
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/NoSuspendWithLockCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/NoSuspendWithLockCheck.cpp
index ca293178c78b4..29470b1f725fb 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/NoSuspendWithLockCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/NoSuspendWithLockCheck.cpp
@@ -23,9 +23,9 @@ void NoSuspendWithLockCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 }
 
 void NoSuspendWithLockCheck::registerMatchers(MatchFinder *Finder) {
-  auto LockType = elaboratedType(namesType(templateSpecializationType(
+  auto LockType = templateSpecializationType(
       hasDeclaration(namedDecl(matchers::matchesAnyListedName(
-          utils::options::parseStringList(LockGuards)))))));
+          utils::options::parseStringList(LockGuards)))));
 
   StatementMatcher Lock =
       declStmt(has(varDecl(hasType(LockType)).bind("lock-decl")))
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp
index 593a4f85d1309..79cd4bbcc9a60 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/PreferMemberInitializerCheck.cpp
@@ -191,6 +191,9 @@ void PreferMemberInitializerCheck::check(
     if (!AssignmentToMember)
       continue;
     const FieldDecl *Field = AssignmentToMember->Field;
+    // Skip if the field is inherited from a base class.
+    if (Field->getParent() != Class)
+      continue;
     const Expr *InitValue = AssignmentToMember->Init;
     updateAssignmentLevel(Field, InitValue, Ctor, AssignedFields);
     if (!canAdvanceAssignment(AssignedFields[Field]))
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccess.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccess.cpp
new file mode 100644
index 0000000000000..35f432efa88ca
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccess.cpp
@@ -0,0 +1,262 @@
+//===--- ProBoundsAvoidUncheckedContainerAccess.cpp - clang-tidy ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ProBoundsAvoidUncheckedContainerAccess.h"
+#include "../utils/Matchers.h"
+#include "../utils/OptionsUtils.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "llvm/ADT/StringRef.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::cppcoreguidelines {
+
+static constexpr llvm::StringRef DefaultExclusionStr =
+    "::std::map;::std::unordered_map;::std::flat_map";
+
+ProBoundsAvoidUncheckedContainerAccess::ProBoundsAvoidUncheckedContainerAccess(
+    StringRef Name, ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      ExcludedClasses(utils::options::parseStringList(
+          Options.get("ExcludeClasses", DefaultExclusionStr))),
+      FixMode(Options.get("FixMode", None)),
+      FixFunction(Options.get("FixFunction", "gsl::at")),
+      FixFunctionEmptyArgs(Options.get("FixFunctionEmptyArgs", FixFunction)) {}
+
+void ProBoundsAvoidUncheckedContainerAccess::storeOptions(
+    ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "ExcludeClasses",
+                utils::options::serializeStringList(ExcludedClasses));
+  Options.store(Opts, "FixMode", FixMode);
+  Options.store(Opts, "FixFunction", FixFunction);
+  Options.store(Opts, "FixFunctionEmptyArgs", FixFunctionEmptyArgs);
+}
+
+// TODO: if at() is defined in another class in the class hierarchy of the class
+// that defines the operator[] we matched on, findAlternative() will not detect
+// it.
+static const CXXMethodDecl *
+findAlternativeAt(const CXXMethodDecl *MatchedOperator) {
+  const CXXRecordDecl *Parent = MatchedOperator->getParent();
+  const QualType SubscriptThisObjType =
+      MatchedOperator->getFunctionObjectParameterReferenceType();
+
+  for (const CXXMethodDecl *Method : Parent->methods()) {
+    // Require 'Method' to be as accessible as 'MatchedOperator' or more
+    if (MatchedOperator->getAccess() < Method->getAccess())
+      continue;
+
+    if (MatchedOperator->isConst() != Method->isConst())
+      continue;
+
+    const QualType AtThisObjType =
+        Method->getFunctionObjectParameterReferenceType();
+    if (SubscriptThisObjType != AtThisObjType)
+      continue;
+
+    if (!Method->getNameInfo().getName().isIdentifier() ||
+        Method->getName() != "at")
+      continue;
+
+    const bool SameReturnType =
+        Method->getReturnType() == MatchedOperator->getReturnType();
+    if (!SameReturnType)
+      continue;
+
+    const bool SameNumberOfArguments =
+        Method->getNumParams() == MatchedOperator->getNumParams();
+    if (!SameNumberOfArguments)
+      continue;
+
+    for (unsigned ArgInd = 0; ArgInd < Method->getNumParams(); ArgInd++) {
+      const bool SameArgType =
+          Method->parameters()[ArgInd]->getOriginalType() ==
+          MatchedOperator->parameters()[ArgInd]->getOriginalType();
+      if (!SameArgType)
+        continue;
+    }
+
+    return Method;
+  }
+  return nullptr;
+}
+
+void ProBoundsAvoidUncheckedContainerAccess::registerMatchers(
+    MatchFinder *Finder) {
+  Finder->addMatcher(
+      mapAnyOf(cxxOperatorCallExpr, cxxMemberCallExpr)
+          .with(callee(
+              cxxMethodDecl(
+                  hasOverloadedOperatorName("[]"),
+                  anyOf(parameterCountIs(0), parameterCountIs(1)),
+                  unless(matchers::matchesAnyListedName(ExcludedClasses)))
+                  .bind("operator")))
+          .bind("caller"),
+      this);
+}
+
+void ProBoundsAvoidUncheckedContainerAccess::check(
+    const MatchFinder::MatchResult &Result) {
+
+  const auto *MatchedExpr = Result.Nodes.getNodeAs<CallExpr>("caller");
+
+  if (FixMode == None) {
+    diag(MatchedExpr->getCallee()->getBeginLoc(),
+         "possibly unsafe 'operator[]', consider bounds-safe alternatives")
+        << MatchedExpr->getCallee()->getSourceRange();
+    return;
+  }
+
+  if (const auto *OCE = dyn_cast<CXXOperatorCallExpr>(MatchedExpr)) {
+    // Case: a[i]
+    const auto LeftBracket = SourceRange(OCE->getCallee()->getBeginLoc(),
+                                         OCE->getCallee()->getBeginLoc());
+    const auto RightBracket =
+        SourceRange(OCE->getOperatorLoc(), OCE->getOperatorLoc());
+
+    if (FixMode == At) {
+      // Case: a[i] => a.at(i)
+      const auto *MatchedOperator =
+          Result.Nodes.getNodeAs<CXXMethodDecl>("operator");
+      const CXXMethodDecl *Alternative = findAlternativeAt(MatchedOperator);
+
+      if (!Alternative) {
+        diag(MatchedExpr->getCallee()->getBeginLoc(),
+             "possibly unsafe 'operator[]', consider "
+             "bounds-safe alternatives")
+            << MatchedExpr->getCallee()->getSourceRange();
+        return;
+      }
+
+      diag(MatchedExpr->getCallee()->getBeginLoc(),
+           "possibly unsafe 'operator[]', consider "
+           "bounds-safe alternative 'at()'")
+          << MatchedExpr->getCallee()->getSourceRange()
+          << FixItHint::CreateReplacement(LeftBracket, ".at(")
+          << FixItHint::CreateReplacement(RightBracket, ")");
+
+      diag(Alternative->getBeginLoc(), "viable 'at()' is defined here",
+           DiagnosticIDs::Note)
+          << Alternative->getNameInfo().getSourceRange();
+
+    } else if (FixMode == Function) {
+      // Case: a[i] => f(a, i)
+      //
+      // Since C++23, the subscript operator may also be called without an
+      // argument, which makes the following distinction necessary
+      const bool EmptySubscript =
+          MatchedExpr->getDirectCallee()->getNumParams() == 0;
+
+      if (EmptySubscript) {
+        auto D = diag(MatchedExpr->getCallee()->getBeginLoc(),
+                      "possibly unsafe 'operator[]'%select{, use safe "
+                      "function '%1() instead|}0")
+                 << FixFunctionEmptyArgs.empty() << FixFunctionEmptyArgs.str()
+                 << MatchedExpr->getCallee()->getSourceRange();
+        if (!FixFunctionEmptyArgs.empty()) {
+          D << FixItHint::CreateInsertion(OCE->getArg(0)->getBeginLoc(),
+                                          FixFunctionEmptyArgs.str() + "(")
+            << FixItHint::CreateRemoval(LeftBracket)
+            << FixItHint::CreateReplacement(RightBracket, ")");
+        }
+      } else {
+        diag(MatchedExpr->getCallee()->getBeginLoc(),
+             "possibly unsafe 'operator[]', use safe function '%0()' instead")
+            << FixFunction.str() << MatchedExpr->getCallee()->getSourceRange()
+            << FixItHint::CreateInsertion(OCE->getArg(0)->getBeginLoc(),
+                                          FixFunction.str() + "(")
+            << FixItHint::CreateReplacement(LeftBracket, ", ")
+            << FixItHint::CreateReplacement(RightBracket, ")");
+      }
+    }
+  } else if (const auto *MCE = dyn_cast<CXXMemberCallExpr>(MatchedExpr)) {
+    // Case: a.operator[](i) or a->operator[](i)
+    const auto *Callee = dyn_cast<MemberExpr>(MCE->getCallee());
+
+    if (FixMode == At) {
+      // Cases: a.operator[](i) => a.at(i) and a->operator[](i) => a->at(i)
+
+      const auto *MatchedOperator =
+          Result.Nodes.getNodeAs<CXXMethodDecl>("operator");
+
+      const CXXMethodDecl *Alternative = findAlternativeAt(MatchedOperator);
+      if (!Alternative) {
+        diag(Callee->getBeginLoc(), "possibly unsafe 'operator[]', consider "
+                                    "bounds-safe alternative 'at()'")
+            << Callee->getSourceRange();
+        return;
+      }
+      diag(MatchedExpr->getCallee()->getBeginLoc(),
+           "possibly unsafe 'operator[]', consider "
+           "bounds-safe alternative 'at()'")
+          << FixItHint::CreateReplacement(
+                 SourceRange(Callee->getMemberLoc(), Callee->getEndLoc()),
+                 "at");
+
+      diag(Alternative->getBeginLoc(), "viable 'at()' defined here",
+           DiagnosticIDs::Note)
+          << Alternative->getNameInfo().getSourceRange();
+
+    } else if (FixMode == Function) {
+      // Cases: a.operator[](i) => f(a, i) and a->operator[](i) => f(*a, i)
+      const auto *Callee = dyn_cast<MemberExpr>(MCE->getCallee());
+
+      const bool EmptySubscript =
+          MCE->getMethodDecl()->getNumNonObjectParams() == 0;
+
+      std::string BeginInsertion =
+          (EmptySubscript ? FixFunctionEmptyArgs.str() : FixFunction.str()) +
+          "(";
+
+      if (Callee->isArrow())
+        BeginInsertion += "*";
+
+      // Since C++23, the subscript operator may also be called without an
+      // argument, which makes the following distinction necessary
+      if (EmptySubscript) {
+        auto D = diag(MatchedExpr->getCallee()->getBeginLoc(),
+                      "possibly unsafe 'operator[]'%select{, use safe "
+                      "function '%1()' instead|}0")
+                 << FixFunctionEmptyArgs.empty() << FixFunctionEmptyArgs.str()
+                 << Callee->getSourceRange();
+
+        if (!FixFunctionEmptyArgs.empty()) {
+          D << FixItHint::CreateInsertion(MatchedExpr->getBeginLoc(),
+                                          BeginInsertion)
+            << FixItHint::CreateRemoval(
+                   SourceRange(Callee->getOperatorLoc(),
+                               MCE->getRParenLoc().getLocWithOffset(-1)));
+        }
+      } else {
+        diag(Callee->getBeginLoc(),
+             "possibly unsafe 'operator[]', use safe function '%0()' instead")
+            << FixFunction.str() << Callee->getSourceRange()
+            << FixItHint::CreateInsertion(MatchedExpr->getBeginLoc(),
+                                          BeginInsertion)
+            << FixItHint::CreateReplacement(
+                   SourceRange(
+                       Callee->getOperatorLoc(),
+                       MCE->getArg(0)->getBeginLoc().getLocWithOffset(-1)),
+                   ", ");
+      }
+    }
+  }
+}
+
+} // namespace clang::tidy::cppcoreguidelines
+
+namespace clang::tidy {
+using P = cppcoreguidelines::ProBoundsAvoidUncheckedContainerAccess;
+
+llvm::ArrayRef<std::pair<P::FixModes, StringRef>>
+OptionEnumMapping<P::FixModes>::getEnumMapping() {
+  static constexpr std::pair<P::FixModes, StringRef> Mapping[] = {
+      {P::None, "none"}, {P::At, "at"}, {P::Function, "function"}};
+  return {Mapping};
+}
+} // namespace clang::tidy
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccess.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccess.h
new file mode 100644
index 0000000000000..cfd52d69c0f58
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsAvoidUncheckedContainerAccess.h
@@ -0,0 +1,56 @@
+//===--- ProBoundsAvoidUncheckedContainerAccess.h - clang-tidy --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_BOUNDS_AVOID_UNCHECKED_CONTAINER_ACCESS_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_BOUNDS_AVOID_UNCHECKED_CONTAINER_ACCESS_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang::tidy::cppcoreguidelines {
+
+/// Flags calls to operator[] in STL containers and suggests replacing it with
+/// safe alternatives.
+///
+/// See
+/// https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#slcon3-avoid-bounds-errors
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/cppcoreguidelines/pro-bounds-avoid-unchecked-container-access.html
+class ProBoundsAvoidUncheckedContainerAccess : public ClangTidyCheck {
+public:
+  ProBoundsAvoidUncheckedContainerAccess(StringRef Name,
+                                         ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+
+  enum FixModes { None, At, Function };
+
+private:
+  // A list of class names that are excluded from the warning
+  std::vector<llvm::StringRef> ExcludedClasses;
+  // Setting which fix to suggest
+  FixModes FixMode;
+  llvm::StringRef FixFunction;
+  llvm::StringRef FixFunctionEmptyArgs;
+};
+} // namespace clang::tidy::cppcoreguidelines
+
+namespace clang::tidy {
+template <>
+struct OptionEnumMapping<
+    cppcoreguidelines::ProBoundsAvoidUncheckedContainerAccess::FixModes> {
+  static ArrayRef<std::pair<
+      cppcoreguidelines::ProBoundsAvoidUncheckedContainerAccess::FixModes,
+      StringRef>>
+  getEnumMapping();
+};
+} // namespace clang::tidy
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_PRO_BOUNDS_AVOID_UNCHECKED_CONTAINER_ACCESS_H
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
index 9ac7b9e057e35..51995c5f64ef6 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.cpp
@@ -14,6 +14,18 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::cppcoreguidelines {
 
+ProBoundsPointerArithmeticCheck::ProBoundsPointerArithmeticCheck(
+    StringRef Name, ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      AllowIncrementDecrementOperators(
+          Options.get("AllowIncrementDecrementOperators", false)) {}
+
+void ProBoundsPointerArithmeticCheck::storeOptions(
+    ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "AllowIncrementDecrementOperators",
+                AllowIncrementDecrementOperators);
+}
+
 void ProBoundsPointerArithmeticCheck::registerMatchers(MatchFinder *Finder) {
   const auto AllPointerTypes =
       anyOf(hasType(hasUnqualifiedDesugaredType(pointerType())),
@@ -30,13 +42,14 @@ void ProBoundsPointerArithmeticCheck::registerMatchers(MatchFinder *Finder) {
       this);
 
   // Flag all operators ++, -- that result in a pointer
-  Finder->addMatcher(
-      unaryOperator(hasAnyOperatorName("++", "--"),
-                    hasType(hasUnqualifiedDesugaredType(pointerType())),
-                    unless(hasUnaryOperand(
-                        ignoringImpCasts(declRefExpr(to(isImplicit()))))))
-          .bind("expr"),
-      this);
+  if (!AllowIncrementDecrementOperators)
+    Finder->addMatcher(
+        unaryOperator(hasAnyOperatorName("++", "--"),
+                      hasType(hasUnqualifiedDesugaredType(pointerType())),
+                      unless(hasUnaryOperand(
+                          ignoringImpCasts(declRefExpr(to(isImplicit()))))))
+            .bind("expr"),
+        this);
 
   // Array subscript on a pointer (not an array) is also pointer arithmetic
   Finder->addMatcher(
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
index 3466c72a769e9..785f754055fb8 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProBoundsPointerArithmeticCheck.h
@@ -21,13 +21,16 @@ namespace clang::tidy::cppcoreguidelines {
 /// http://clang.llvm.org/extra/clang-tidy/checks/cppcoreguidelines/pro-bounds-pointer-arithmetic.html
 class ProBoundsPointerArithmeticCheck : public ClangTidyCheck {
 public:
-  ProBoundsPointerArithmeticCheck(StringRef Name, ClangTidyContext *Context)
-      : ClangTidyCheck(Name, Context) {}
+  ProBoundsPointerArithmeticCheck(StringRef Name, ClangTidyContext *Context);
   bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
     return LangOpts.CPlusPlus;
   }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+
+private:
+  const bool AllowIncrementDecrementOperators;
 };
 
 } // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp
index b413b12cd37ab..40607597297b5 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/ProTypeMemberInitCheck.cpp
@@ -190,7 +190,7 @@ struct InitializerInsertion {
 // Convenience utility to get a RecordDecl from a QualType.
 const RecordDecl *getCanonicalRecordDecl(const QualType &Type) {
   if (const auto *RT = Type.getCanonicalType()->getAs<RecordType>())
-    return RT->getDecl();
+    return RT->getOriginalDecl();
   return nullptr;
 }
 
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.cpp
index 76754394de760..40fd15c08f0a1 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.cpp
@@ -92,7 +92,7 @@ void SlicingCheck::diagnoseSlicedOverriddenMethods(
   for (const auto &Base : DerivedDecl.bases()) {
     if (const auto *BaseRecordType = Base.getType()->getAs<RecordType>()) {
       if (const auto *BaseRecord = cast_or_null<CXXRecordDecl>(
-              BaseRecordType->getDecl()->getDefinition()))
+              BaseRecordType->getOriginalDecl()->getDefinition()))
         diagnoseSlicedOverriddenMethods(Call, *BaseRecord, BaseDecl);
     }
   }
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h
index 02bfaf1205f40..317547f0a9c87 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/SlicingCheck.h
@@ -22,6 +22,9 @@ class SlicingCheck : public ClangTidyCheck {
 public:
   SlicingCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 
diff --git a/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsCallsCheck.h b/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsCallsCheck.h
index 49d7820d04410..120dc90b2cbc0 100644
--- a/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsCallsCheck.h
+++ b/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsCallsCheck.h
@@ -21,6 +21,9 @@ class DefaultArgumentsCallsCheck : public ClangTidyCheck {
 public:
   DefaultArgumentsCallsCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.h b/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.h
index d03d0b7a10f05..da73fa4064cbd 100644
--- a/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.h
+++ b/clang-tools-extra/clang-tidy/fuchsia/DefaultArgumentsDeclarationsCheck.h
@@ -21,6 +21,9 @@ class DefaultArgumentsDeclarationsCheck : public ClangTidyCheck {
 public:
   DefaultArgumentsDeclarationsCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp b/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp
index 80ff97a762134..0302a5ad4957c 100644
--- a/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/fuchsia/MultipleInheritanceCheck.cpp
@@ -74,7 +74,7 @@ bool MultipleInheritanceCheck::isInterface(const CXXRecordDecl *Node) {
     const auto *Ty = I.getType()->getAs<RecordType>();
     if (!Ty)
       continue;
-    const RecordDecl *D = Ty->getDecl()->getDefinition();
+    const RecordDecl *D = Ty->getOriginalDecl()->getDefinition();
     if (!D)
       continue;
     const auto *Base = cast<CXXRecordDecl>(D);
@@ -106,7 +106,8 @@ void MultipleInheritanceCheck::check(const MatchFinder::MatchResult &Result) {
       const auto *Ty = I.getType()->getAs<RecordType>();
       if (!Ty)
         continue;
-      const auto *Base = cast<CXXRecordDecl>(Ty->getDecl()->getDefinition());
+      const auto *Base =
+          cast<CXXRecordDecl>(Ty->getOriginalDecl()->getDefinition());
       if (!isInterface(Base))
         NumConcrete++;
     }
@@ -117,7 +118,8 @@ void MultipleInheritanceCheck::check(const MatchFinder::MatchResult &Result) {
       const auto *Ty = V.getType()->getAs<RecordType>();
       if (!Ty)
         continue;
-      const auto *Base = cast<CXXRecordDecl>(Ty->getDecl()->getDefinition());
+      const auto *Base =
+          cast<CXXRecordDecl>(Ty->getOriginalDecl()->getDefinition());
       if (!isInterface(Base))
         NumConcrete++;
     }
diff --git a/clang-tools-extra/clang-tidy/fuchsia/OverloadedOperatorCheck.h b/clang-tools-extra/clang-tidy/fuchsia/OverloadedOperatorCheck.h
index 007f410a6eac6..d26349d6e9afc 100644
--- a/clang-tools-extra/clang-tidy/fuchsia/OverloadedOperatorCheck.h
+++ b/clang-tools-extra/clang-tidy/fuchsia/OverloadedOperatorCheck.h
@@ -21,6 +21,9 @@ class OverloadedOperatorCheck : public ClangTidyCheck {
 public:
   OverloadedOperatorCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.h b/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.h
index 426d89d046a63..1bdf19f9146fb 100644
--- a/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.h
+++ b/clang-tools-extra/clang-tidy/fuchsia/VirtualInheritanceCheck.h
@@ -21,6 +21,9 @@ class VirtualInheritanceCheck : public ClangTidyCheck {
 public:
   VirtualInheritanceCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp b/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp
index e076b39b5d978..14e11eb0bc697 100644
--- a/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp
@@ -89,6 +89,30 @@ static StringRef getDestTypeString(const SourceManager &SM,
                               SM, LangOpts);
 }
 
+static bool sameTypeAsWritten(QualType X, QualType Y) {
+  if (X.getCanonicalType() != Y.getCanonicalType())
+    return false;
+
+  auto TC = X->getTypeClass();
+  if (TC != Y->getTypeClass())
+    return false;
+
+  switch (TC) {
+  case Type::Typedef:
+    return declaresSameEntity(cast<TypedefType>(X)->getDecl(),
+                              cast<TypedefType>(Y)->getDecl());
+  case Type::Pointer:
+    return sameTypeAsWritten(cast<PointerType>(X)->getPointeeType(),
+                             cast<PointerType>(Y)->getPointeeType());
+  case Type::RValueReference:
+  case Type::LValueReference:
+    return sameTypeAsWritten(cast<ReferenceType>(X)->getPointeeType(),
+                             cast<ReferenceType>(Y)->getPointeeType());
+  default:
+    return true;
+  }
+}
+
 void AvoidCStyleCastsCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *CastExpr = Result.Nodes.getNodeAs<ExplicitCastExpr>("cast");
 
@@ -128,12 +152,7 @@ void AvoidCStyleCastsCheck::check(const MatchFinder::MatchResult &Result) {
     // case of overloaded functions, so detection of redundant casts is trickier
     // in this case. Don't emit "redundant cast" warnings for function
     // pointer/reference types.
-    QualType Src = SourceTypeAsWritten, Dst = DestTypeAsWritten;
-    if (const auto *ElTy = dyn_cast<ElaboratedType>(Src))
-      Src = ElTy->getNamedType();
-    if (const auto *ElTy = dyn_cast<ElaboratedType>(Dst))
-      Dst = ElTy->getNamedType();
-    if (Src == Dst) {
+    if (sameTypeAsWritten(SourceTypeAsWritten, DestTypeAsWritten)) {
       diag(CastExpr->getBeginLoc(), "redundant cast to the same type")
           << FixItHint::CreateRemoval(ReplaceRange);
       return;
diff --git a/clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.h b/clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.h
index baec40e440cbd..b53e6c45913d5 100644
--- a/clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.h
+++ b/clang-tools-extra/clang-tidy/google/AvoidUnderscoreInGoogletestNameCheck.h
@@ -21,6 +21,9 @@ namespace clang::tidy::google::readability {
 class AvoidUnderscoreInGoogletestNameCheck : public ClangTidyCheck {
 public:
   using ClangTidyCheck::ClangTidyCheck;
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
 
   void registerPPCallbacks(const SourceManager &SM, Preprocessor *PP,
                            Preprocessor *ModuleExpanderPP) override;
diff --git a/clang-tools-extra/clang-tidy/google/DefaultArgumentsCheck.h b/clang-tools-extra/clang-tidy/google/DefaultArgumentsCheck.h
index a8f0b0112fb94..49d95a5acd35c 100644
--- a/clang-tools-extra/clang-tidy/google/DefaultArgumentsCheck.h
+++ b/clang-tools-extra/clang-tidy/google/DefaultArgumentsCheck.h
@@ -23,6 +23,9 @@ class DefaultArgumentsCheck : public ClangTidyCheck {
 public:
   DefaultArgumentsCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.cpp b/clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.cpp
index 3deea0620514b..68233ec6bd441 100644
--- a/clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/ExplicitConstructorCheck.cpp
@@ -72,7 +72,7 @@ static bool isStdInitializerList(QualType Type) {
   }
   if (const auto *RT = Type->getAs<RecordType>()) {
     if (const auto *Specialization =
-            dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl()))
+            dyn_cast<ClassTemplateSpecializationDecl>(RT->getOriginalDecl()))
       return declIsStdInitializerList(Specialization->getSpecializedTemplate());
   }
   return false;
diff --git a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
index 805dcaf3ce402..274b8afa98bd6 100644
--- a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
@@ -257,8 +257,13 @@ getAliasNameRange(const MatchFinder::MatchResult &Result) {
     return CharSourceRange::getTokenRange(
         Using->getNameInfo().getSourceRange());
   }
-  return CharSourceRange::getTokenRange(
-      Result.Nodes.getNodeAs<TypeLoc>("typeloc")->getSourceRange());
+  TypeLoc TL = *Result.Nodes.getNodeAs<TypeLoc>("typeloc");
+  if (auto QTL = TL.getAs<QualifiedTypeLoc>())
+    TL = QTL.getUnqualifiedLoc();
+
+  if (auto TTL = TL.getAs<TypedefTypeLoc>())
+    return CharSourceRange::getTokenRange(TTL.getNameLoc());
+  return CharSourceRange::getTokenRange(TL.castAs<UsingTypeLoc>().getNameLoc());
 }
 
 void UpgradeGoogletestCaseCheck::check(const MatchFinder::MatchResult &Result) {
diff --git a/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h
index 48354c34a8581..39c45fea9aae4 100644
--- a/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h
+++ b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h
@@ -21,6 +21,9 @@ namespace clang::tidy::hicpp {
 class IgnoredRemoveResultCheck : public bugprone::UnusedReturnValueCheck {
 public:
   IgnoredRemoveResultCheck(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
 };
 
diff --git a/clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.h b/clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.h
index 1099ab0cd0e44..07e018a6fc969 100644
--- a/clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.h
+++ b/clang-tools-extra/clang-tidy/llvm/PreferRegisterOverUnsignedCheck.h
@@ -23,6 +23,9 @@ class PreferRegisterOverUnsignedCheck : public ClangTidyCheck {
 public:
   PreferRegisterOverUnsignedCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.h b/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.h
index e1f25e530289a..b4550ecb226bf 100644
--- a/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.h
+++ b/clang-tools-extra/clang-tidy/llvm/TwineLocalCheck.h
@@ -19,6 +19,9 @@ class TwineLocalCheck : public ClangTidyCheck {
 public:
   TwineLocalCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
index fd7affd22a463..2cfee5fd10713 100644
--- a/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/misc/CMakeLists.txt
@@ -32,6 +32,7 @@ add_clang_library(clangTidyMiscModule STATIC
   NoRecursionCheck.cpp
   NonCopyableObjects.cpp
   NonPrivateMemberVariablesInClassesCheck.cpp
+  OverrideWithDifferentVisibilityCheck.cpp
   RedundantExpressionCheck.cpp
   StaticAssertCheck.cpp
   ThrowByValueCatchByReferenceCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp b/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp
index 697398a54332d..b32507d66cbac 100644
--- a/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ConstCorrectnessCheck.cpp
@@ -98,11 +98,12 @@ void ConstCorrectnessCheck::registerMatchers(MatchFinder *Finder) {
       hasType(referenceType(pointee(hasCanonicalType(templateTypeParmType())))),
       hasType(referenceType(pointee(substTemplateTypeParmType()))));
 
-  const auto AllowedType = hasType(qualType(anyOf(
-      hasDeclaration(namedDecl(matchers::matchesAnyListedName(AllowedTypes))),
-      references(namedDecl(matchers::matchesAnyListedName(AllowedTypes))),
-      pointerType(pointee(hasDeclaration(
-          namedDecl(matchers::matchesAnyListedName(AllowedTypes))))))));
+  auto AllowedTypeDecl = namedDecl(
+      anyOf(matchers::matchesAnyListedName(AllowedTypes), usingShadowDecl()));
+
+  const auto AllowedType = hasType(qualType(
+      anyOf(hasDeclaration(AllowedTypeDecl), references(AllowedTypeDecl),
+            pointerType(pointee(hasDeclaration(AllowedTypeDecl))))));
 
   const auto AutoTemplateType = varDecl(
       anyOf(hasType(autoType()), hasType(referenceType(pointee(autoType()))),
diff --git a/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp b/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
index 6ddebcbc0e152..f675ca70deb9d 100644
--- a/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/misc/MiscTidyModule.cpp
@@ -22,6 +22,7 @@
 #include "NoRecursionCheck.h"
 #include "NonCopyableObjects.h"
 #include "NonPrivateMemberVariablesInClassesCheck.h"
+#include "OverrideWithDifferentVisibilityCheck.h"
 #include "RedundantExpressionCheck.h"
 #include "StaticAssertCheck.h"
 #include "ThrowByValueCatchByReferenceCheck.h"
@@ -81,6 +82,8 @@ class MiscModule : public ClangTidyModule {
         "misc-use-anonymous-namespace");
     CheckFactories.registerCheck<UseInternalLinkageCheck>(
         "misc-use-internal-linkage");
+    CheckFactories.registerCheck<OverrideWithDifferentVisibilityCheck>(
+        "misc-override-with-different-visibility");
   }
 };
 
diff --git a/clang-tools-extra/clang-tidy/misc/MisplacedConstCheck.cpp b/clang-tools-extra/clang-tidy/misc/MisplacedConstCheck.cpp
index 0cdd48c13b2a6..bb64a5618620c 100644
--- a/clang-tools-extra/clang-tidy/misc/MisplacedConstCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/MisplacedConstCheck.cpp
@@ -19,13 +19,13 @@ void MisplacedConstCheck::registerMatchers(MatchFinder *Finder) {
       pointee(anyOf(isConstQualified(), ignoringParens(functionType()))))));
 
   Finder->addMatcher(
-      valueDecl(hasType(qualType(
-                    isConstQualified(),
-                    elaboratedType(namesType(typedefType(hasDeclaration(
-                        anyOf(typedefDecl(NonConstAndNonFunctionPointerType)
-                                  .bind("typedef"),
-                              typeAliasDecl(NonConstAndNonFunctionPointerType)
-                                  .bind("typeAlias")))))))))
+      valueDecl(
+          hasType(qualType(isConstQualified(),
+                           typedefType(hasDeclaration(anyOf(
+                               typedefDecl(NonConstAndNonFunctionPointerType)
+                                   .bind("typedef"),
+                               typeAliasDecl(NonConstAndNonFunctionPointerType)
+                                   .bind("typeAlias")))))))
           .bind("decl"),
       this);
 }
diff --git a/clang-tools-extra/clang-tidy/misc/NewDeleteOverloadsCheck.cpp b/clang-tools-extra/clang-tidy/misc/NewDeleteOverloadsCheck.cpp
index 40808aaf7c3da..2837f40bc49b8 100644
--- a/clang-tools-extra/clang-tidy/misc/NewDeleteOverloadsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/NewDeleteOverloadsCheck.cpp
@@ -59,7 +59,9 @@ AST_MATCHER(FunctionDecl, isPlacementOverload) {
   return true;
 }
 
-OverloadedOperatorKind getCorrespondingOverload(const FunctionDecl *FD) {
+} // namespace
+
+static OverloadedOperatorKind getCorrespondingOverload(const FunctionDecl *FD) {
   switch (FD->getOverloadedOperator()) {
   default:
     break;
@@ -75,7 +77,7 @@ OverloadedOperatorKind getCorrespondingOverload(const FunctionDecl *FD) {
   llvm_unreachable("Not an overloaded allocation operator");
 }
 
-const char *getOperatorName(OverloadedOperatorKind K) {
+static const char *getOperatorName(OverloadedOperatorKind K) {
   switch (K) {
   default:
     break;
@@ -91,13 +93,14 @@ const char *getOperatorName(OverloadedOperatorKind K) {
   llvm_unreachable("Not an overloaded allocation operator");
 }
 
-bool areCorrespondingOverloads(const FunctionDecl *LHS,
-                               const FunctionDecl *RHS) {
+static bool areCorrespondingOverloads(const FunctionDecl *LHS,
+                                      const FunctionDecl *RHS) {
   return RHS->getOverloadedOperator() == getCorrespondingOverload(LHS);
 }
 
-bool hasCorrespondingOverloadInBaseClass(const CXXMethodDecl *MD,
-                                         const CXXRecordDecl *RD = nullptr) {
+static bool
+hasCorrespondingOverloadInBaseClass(const CXXMethodDecl *MD,
+                                    const CXXRecordDecl *RD = nullptr) {
   if (RD) {
     // Check the methods in the given class and accessible to derived classes.
     for (const auto *BMD : RD->methods())
@@ -124,8 +127,6 @@ bool hasCorrespondingOverloadInBaseClass(const CXXMethodDecl *MD,
   return false;
 }
 
-} // anonymous namespace
-
 void NewDeleteOverloadsCheck::registerMatchers(MatchFinder *Finder) {
   // Match all operator new and operator delete overloads (including the array
   // forms). Do not match implicit operators, placement operators, or
diff --git a/clang-tools-extra/clang-tidy/misc/OverrideWithDifferentVisibilityCheck.cpp b/clang-tools-extra/clang-tidy/misc/OverrideWithDifferentVisibilityCheck.cpp
new file mode 100644
index 0000000000000..12f78affe463e
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/misc/OverrideWithDifferentVisibilityCheck.cpp
@@ -0,0 +1,150 @@
+//===--- OverrideWithDifferentVisibilityCheck.cpp - clang-tidy ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "OverrideWithDifferentVisibilityCheck.h"
+#include "../utils/Matchers.h"
+#include "../utils/OptionsUtils.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+
+using namespace clang::ast_matchers;
+using namespace clang;
+
+namespace {
+
+AST_MATCHER(NamedDecl, isOperatorDecl) {
+  DeclarationName::NameKind const NK = Node.getDeclName().getNameKind();
+  return NK != DeclarationName::Identifier &&
+         NK != DeclarationName::CXXConstructorName &&
+         NK != DeclarationName::CXXDestructorName;
+}
+
+} // namespace
+
+namespace clang::tidy {
+
+template <>
+struct OptionEnumMapping<
+    misc::OverrideWithDifferentVisibilityCheck::ChangeKind> {
+  static llvm::ArrayRef<std::pair<
+      misc::OverrideWithDifferentVisibilityCheck::ChangeKind, StringRef>>
+  getEnumMapping() {
+    static constexpr std::pair<
+        misc::OverrideWithDifferentVisibilityCheck::ChangeKind, StringRef>
+        Mapping[] = {
+            {misc::OverrideWithDifferentVisibilityCheck::ChangeKind::Any,
+             "any"},
+            {misc::OverrideWithDifferentVisibilityCheck::ChangeKind::Widening,
+             "widening"},
+            {misc::OverrideWithDifferentVisibilityCheck::ChangeKind::Narrowing,
+             "narrowing"},
+        };
+    return {Mapping};
+  }
+};
+
+namespace misc {
+
+OverrideWithDifferentVisibilityCheck::OverrideWithDifferentVisibilityCheck(
+    StringRef Name, ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      DetectVisibilityChange(
+          Options.get("DisallowedVisibilityChange", ChangeKind::Any)),
+      CheckDestructors(Options.get("CheckDestructors", false)),
+      CheckOperators(Options.get("CheckOperators", false)),
+      IgnoredFunctions(utils::options::parseStringList(
+          Options.get("IgnoredFunctions", ""))) {}
+
+void OverrideWithDifferentVisibilityCheck::storeOptions(
+    ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "DisallowedVisibilityChange", DetectVisibilityChange);
+  Options.store(Opts, "CheckDestructors", CheckDestructors);
+  Options.store(Opts, "CheckOperators", CheckOperators);
+  Options.store(Opts, "IgnoredFunctions",
+                utils::options::serializeStringList(IgnoredFunctions));
+}
+
+void OverrideWithDifferentVisibilityCheck::registerMatchers(
+    MatchFinder *Finder) {
+  const auto IgnoredDecl =
+      namedDecl(matchers::matchesAnyListedName(IgnoredFunctions));
+  const auto FilterDestructors =
+      CheckDestructors ? decl() : decl(unless(cxxDestructorDecl()));
+  const auto FilterOperators =
+      CheckOperators ? namedDecl() : namedDecl(unless(isOperatorDecl()));
+  Finder->addMatcher(
+      cxxMethodDecl(
+          isVirtual(), FilterDestructors, FilterOperators,
+          ofClass(
+              cxxRecordDecl(unless(isExpansionInSystemHeader())).bind("class")),
+          forEachOverridden(cxxMethodDecl(ofClass(cxxRecordDecl().bind("base")),
+                                          unless(IgnoredDecl))
+                                .bind("base_func")))
+          .bind("func"),
+      this);
+}
+
+void OverrideWithDifferentVisibilityCheck::check(
+    const MatchFinder::MatchResult &Result) {
+  const auto *const MatchedFunction =
+      Result.Nodes.getNodeAs<FunctionDecl>("func");
+  if (!MatchedFunction->isCanonicalDecl())
+    return;
+
+  const auto *const ParentClass =
+      Result.Nodes.getNodeAs<CXXRecordDecl>("class");
+  const auto *const BaseClass = Result.Nodes.getNodeAs<CXXRecordDecl>("base");
+  CXXBasePaths Paths;
+  if (!ParentClass->isDerivedFrom(BaseClass, Paths))
+    return;
+
+  const auto *const OverriddenFunction =
+      Result.Nodes.getNodeAs<FunctionDecl>("base_func");
+  AccessSpecifier const ActualAccess = MatchedFunction->getAccess();
+  AccessSpecifier OverriddenAccess = OverriddenFunction->getAccess();
+
+  const CXXBaseSpecifier *InheritanceWithStrictVisibility = nullptr;
+  for (const CXXBasePath &Path : Paths) {
+    for (const CXXBasePathElement &Elem : Path) {
+      if (Elem.Base->getAccessSpecifier() > OverriddenAccess) {
+        OverriddenAccess = Elem.Base->getAccessSpecifier();
+        InheritanceWithStrictVisibility = Elem.Base;
+      }
+    }
+  }
+
+  if (ActualAccess != OverriddenAccess) {
+    if (DetectVisibilityChange == ChangeKind::Widening &&
+        ActualAccess > OverriddenAccess)
+      return;
+    if (DetectVisibilityChange == ChangeKind::Narrowing &&
+        ActualAccess < OverriddenAccess)
+      return;
+
+    if (InheritanceWithStrictVisibility) {
+      diag(MatchedFunction->getLocation(),
+           "visibility of function %0 is changed from %1 (through %1 "
+           "inheritance of class %2) to %3")
+          << MatchedFunction << OverriddenAccess
+          << InheritanceWithStrictVisibility->getType() << ActualAccess;
+      diag(InheritanceWithStrictVisibility->getBeginLoc(),
+           "%0 is inherited as %1 here", DiagnosticIDs::Note)
+          << InheritanceWithStrictVisibility->getType() << OverriddenAccess;
+    } else {
+      diag(MatchedFunction->getLocation(),
+           "visibility of function %0 is changed from %1 in class %2 to %3")
+          << MatchedFunction << OverriddenAccess << BaseClass << ActualAccess;
+    }
+    diag(OverriddenFunction->getLocation(), "function declared here as %0",
+         DiagnosticIDs::Note)
+        << OverriddenFunction->getAccess();
+  }
+}
+
+} // namespace misc
+
+} // namespace clang::tidy
diff --git a/clang-tools-extra/clang-tidy/misc/OverrideWithDifferentVisibilityCheck.h b/clang-tools-extra/clang-tidy/misc/OverrideWithDifferentVisibilityCheck.h
new file mode 100644
index 0000000000000..1f5222d99196b
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/misc/OverrideWithDifferentVisibilityCheck.h
@@ -0,0 +1,43 @@
+//===--- OverrideWithDifferentVisibilityCheck.h - clang-tidy --*- C++ -*---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_OVERRIDEWITHDIFFERENTVISIBILITYCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_OVERRIDEWITHDIFFERENTVISIBILITYCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang::tidy::misc {
+
+/// Finds virtual function overrides with different visibility than the function
+/// in the base class.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/misc/override-with-different-visibility.html
+class OverrideWithDifferentVisibilityCheck : public ClangTidyCheck {
+public:
+  enum class ChangeKind { Any, Widening, Narrowing };
+
+  OverrideWithDifferentVisibilityCheck(StringRef Name,
+                                       ClangTidyContext *Context);
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
+
+private:
+  ChangeKind DetectVisibilityChange;
+  bool CheckDestructors;
+  bool CheckOperators;
+  std::vector<llvm::StringRef> IgnoredFunctions;
+};
+
+} // namespace clang::tidy::misc
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_MISC_OVERRIDEWITHDIFFERENTVISIBILITYCHECK_H
diff --git a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
index 9b2af2a8ca7d8..107eda2e98f27 100644
--- a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
@@ -45,14 +45,6 @@ static bool incrementWithoutOverflow(const APSInt &Value, APSInt &Result) {
   return Value < Result;
 }
 
-static bool areEquivalentNameSpecifier(const NestedNameSpecifier *Left,
-                                       const NestedNameSpecifier *Right) {
-  llvm::FoldingSetNodeID LeftID, RightID;
-  Left->Profile(LeftID);
-  Right->Profile(RightID);
-  return LeftID == RightID;
-}
-
 static bool areEquivalentExpr(const Expr *Left, const Expr *Right) {
   if (!Left || !Right)
     return !Left && !Right;
@@ -104,9 +96,8 @@ static bool areEquivalentExpr(const Expr *Left, const Expr *Right) {
     if (cast<DependentScopeDeclRefExpr>(Left)->getDeclName() !=
         cast<DependentScopeDeclRefExpr>(Right)->getDeclName())
       return false;
-    return areEquivalentNameSpecifier(
-        cast<DependentScopeDeclRefExpr>(Left)->getQualifier(),
-        cast<DependentScopeDeclRefExpr>(Right)->getQualifier());
+    return cast<DependentScopeDeclRefExpr>(Left)->getQualifier() ==
+           cast<DependentScopeDeclRefExpr>(Right)->getQualifier();
   case Stmt::DeclRefExprClass:
     return cast<DeclRefExpr>(Left)->getDecl() ==
            cast<DeclRefExpr>(Right)->getDecl();
diff --git a/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp b/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
index 3fdaf9239f6af..8200239b982a0 100644
--- a/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/UnconventionalAssignOperatorCheck.cpp
@@ -29,11 +29,13 @@ void UnconventionalAssignOperatorCheck::registerMatchers(
   const auto HasGoodReturnType =
       cxxMethodDecl(returns(hasCanonicalType(lValueReferenceType(pointee(
           unless(isConstQualified()),
-          anyOf(autoType(), hasDeclaration(equalsBoundNode("class"))))))));
+          anyOf(autoType(),
+                hasDeclaration(declaresSameEntityAsBoundNode("class"))))))));
 
   const auto IsSelf = qualType(hasCanonicalType(
-      anyOf(hasDeclaration(equalsBoundNode("class")),
-            referenceType(pointee(hasDeclaration(equalsBoundNode("class")))))));
+      anyOf(hasDeclaration(declaresSameEntityAsBoundNode("class")),
+            referenceType(pointee(
+                hasDeclaration(declaresSameEntityAsBoundNode("class")))))));
   const auto IsAssign =
       cxxMethodDecl(unless(anyOf(isDeleted(), isPrivate(), isImplicit())),
                     hasName("operator="), ofClass(recordDecl().bind("class")))
diff --git a/clang-tools-extra/clang-tidy/misc/UnusedAliasDeclsCheck.cpp b/clang-tools-extra/clang-tidy/misc/UnusedAliasDeclsCheck.cpp
index 86992cd8a141b..4fa679aa8dd88 100644
--- a/clang-tools-extra/clang-tidy/misc/UnusedAliasDeclsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/UnusedAliasDeclsCheck.cpp
@@ -35,12 +35,12 @@ void UnusedAliasDeclsCheck::check(const MatchFinder::MatchResult &Result) {
   }
 
   if (const auto *NestedName =
-          Result.Nodes.getNodeAs<NestedNameSpecifier>("nns")) {
-    if (const auto *AliasDecl = dyn_cast_if_present<NamespaceAliasDecl>(
-            NestedName->getAsNamespace())) {
+          Result.Nodes.getNodeAs<NestedNameSpecifier>("nns");
+      NestedName &&
+      NestedName->getKind() == NestedNameSpecifier::Kind::Namespace)
+    if (const auto *AliasDecl = dyn_cast<NamespaceAliasDecl>(
+            NestedName->getAsNamespaceAndPrefix().Namespace))
       FoundDecls[AliasDecl] = CharSourceRange();
-    }
-  }
 }
 
 void UnusedAliasDeclsCheck::onEndOfTranslationUnit() {
diff --git a/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.cpp b/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.cpp
index d5c5fa3364d63..8211a0ec6a5e1 100644
--- a/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/UnusedUsingDeclsCheck.cpp
@@ -71,11 +71,7 @@ void UnusedUsingDeclsCheck::registerMatchers(MatchFinder *Finder) {
                          templateArgument().bind("used")))),
                      this);
   Finder->addMatcher(userDefinedLiteral().bind("used"), this);
-  Finder->addMatcher(
-      loc(elaboratedType(unless(hasQualifier(nestedNameSpecifier())),
-                         hasUnqualifiedDesugaredType(
-                             type(asTagDecl(tagDecl().bind("used")))))),
-      this);
+  Finder->addMatcher(loc(asTagDecl(tagDecl().bind("used"))), this);
   // Cases where we can identify the UsingShadowDecl directly, rather than
   // just its target.
   // FIXME: cover more cases in this way, as the AST supports it.
@@ -136,7 +132,7 @@ void UnusedUsingDeclsCheck::check(const MatchFinder::MatchResult &Result) {
     }
     if (const auto *ECD = dyn_cast<EnumConstantDecl>(Used)) {
       if (const auto *ET = ECD->getType()->getAs<EnumType>())
-        removeFromFoundDecls(ET->getDecl());
+        removeFromFoundDecls(ET->getOriginalDecl());
     }
   };
   // We rely on the fact that the clang AST is walked in order, usages are only
diff --git a/clang-tools-extra/clang-tidy/modernize/DeprecatedIosBaseAliasesCheck.cpp b/clang-tools-extra/clang-tidy/modernize/DeprecatedIosBaseAliasesCheck.cpp
index f22f48d831608..2aca61021166d 100644
--- a/clang-tools-extra/clang-tidy/modernize/DeprecatedIosBaseAliasesCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/DeprecatedIosBaseAliasesCheck.cpp
@@ -29,8 +29,7 @@ static std::optional<const char *> getReplacementType(StringRef Type) {
 
 void DeprecatedIosBaseAliasesCheck::registerMatchers(MatchFinder *Finder) {
   auto IoStateDecl = typedefDecl(hasAnyName(DeprecatedTypes)).bind("TypeDecl");
-  auto IoStateType =
-      qualType(hasDeclaration(IoStateDecl), unless(elaboratedType()));
+  auto IoStateType = typedefType(hasDeclaration(IoStateDecl));
 
   Finder->addMatcher(typeLoc(loc(IoStateType)).bind("TypeLoc"), this);
 }
@@ -43,12 +42,14 @@ void DeprecatedIosBaseAliasesCheck::check(
   StringRef TypeName = Typedef->getName();
   auto Replacement = getReplacementType(TypeName);
 
-  const auto *TL = Result.Nodes.getNodeAs<TypeLoc>("TypeLoc");
-  SourceLocation IoStateLoc = TL->getBeginLoc();
+  TypeLoc TL = *Result.Nodes.getNodeAs<TypeLoc>("TypeLoc");
+  if (auto QTL = TL.getAs<QualifiedTypeLoc>())
+    TL = QTL.getUnqualifiedLoc();
 
+  SourceLocation IoStateLoc = TL.castAs<TypedefTypeLoc>().getNameLoc();
   // Do not generate fixits for matches depending on template arguments and
   // macro expansions.
-  bool Fix = Replacement && !TL->getType()->isDependentType();
+  bool Fix = Replacement && !TL.getType()->isDependentType();
   if (IoStateLoc.isMacroID()) {
     IoStateLoc = SM.getSpellingLoc(IoStateLoc);
     Fix = false;
diff --git a/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp
index c2db858f72e32..118e96a6f34ae 100644
--- a/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/MacroToEnumCheck.cpp
@@ -395,16 +395,12 @@ void MacroToEnumCallbacks::Endif(SourceLocation Loc, SourceLocation IfLoc) {
   --CurrentFile->ConditionScopes;
 }
 
-namespace {
-
 template <size_t N>
-bool textEquals(const char (&Needle)[N], const char *HayStack) {
+static bool textEquals(const char (&Needle)[N], const char *HayStack) {
   return StringRef{HayStack, N - 1} == Needle;
 }
 
-template <size_t N> size_t len(const char (&)[N]) { return N - 1; }
-
-} // namespace
+template <size_t N> static size_t len(const char (&)[N]) { return N - 1; }
 
 void MacroToEnumCallbacks::PragmaDirective(SourceLocation Loc,
                                            PragmaIntroducerKind Introducer) {
diff --git a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
index deef3586628c6..cea48ce6f4564 100644
--- a/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/MakeSmartPtrCheck.cpp
@@ -16,14 +16,13 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::modernize {
 
-namespace {
+static constexpr char ConstructorCall[] = "constructorCall";
+static constexpr char ResetCall[] = "resetCall";
+static constexpr char NewExpression[] = "newExpression";
 
-constexpr char ConstructorCall[] = "constructorCall";
-constexpr char ResetCall[] = "resetCall";
-constexpr char NewExpression[] = "newExpression";
-
-std::string getNewExprName(const CXXNewExpr *NewExpr, const SourceManager &SM,
-                           const LangOptions &Lang) {
+static std::string getNewExprName(const CXXNewExpr *NewExpr,
+                                  const SourceManager &SM,
+                                  const LangOptions &Lang) {
   StringRef WrittenName = Lexer::getSourceText(
       CharSourceRange::getTokenRange(
           NewExpr->getAllocatedTypeSourceInfo()->getTypeLoc().getSourceRange()),
@@ -34,8 +33,6 @@ std::string getNewExprName(const CXXNewExpr *NewExpr, const SourceManager &SM,
   return WrittenName.str();
 }
 
-} // namespace
-
 const char MakeSmartPtrCheck::PointerType[] = "pointerType";
 
 MakeSmartPtrCheck::MakeSmartPtrCheck(StringRef Name, ClangTidyContext *Context,
diff --git a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
index 1e271dfa768ce..a54d0721a5b7d 100644
--- a/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/PassByValueCheck.cpp
@@ -77,8 +77,7 @@ AST_MATCHER_P(CXXRecordDecl, isMoveConstructibleInBoundCXXRecordDecl, StringRef,
 
 static TypeMatcher notTemplateSpecConstRefType() {
   return lValueReferenceType(
-      pointee(unless(elaboratedType(namesType(templateSpecializationType()))),
-              isConstQualified()));
+      pointee(unless(templateSpecializationType()), isConstQualified()));
 }
 
 static TypeMatcher nonConstValueType() {
diff --git a/clang-tools-extra/clang-tidy/modernize/RawStringLiteralCheck.cpp b/clang-tools-extra/clang-tidy/modernize/RawStringLiteralCheck.cpp
index 24674a407cb36..0c9e909fea7f9 100644
--- a/clang-tools-extra/clang-tidy/modernize/RawStringLiteralCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/RawStringLiteralCheck.cpp
@@ -19,9 +19,7 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::modernize {
 
-namespace {
-
-bool containsEscapes(StringRef HayStack, StringRef Escapes) {
+static bool containsEscapes(StringRef HayStack, StringRef Escapes) {
   size_t BackSlash = HayStack.find('\\');
   if (BackSlash == StringRef::npos)
     return false;
@@ -35,16 +33,16 @@ bool containsEscapes(StringRef HayStack, StringRef Escapes) {
   return true;
 }
 
-bool isRawStringLiteral(StringRef Text) {
+static bool isRawStringLiteral(StringRef Text) {
   // Already a raw string literal if R comes before ".
   const size_t QuotePos = Text.find('"');
   assert(QuotePos != StringRef::npos);
   return (QuotePos > 0) && (Text[QuotePos - 1] == 'R');
 }
 
-bool containsEscapedCharacters(const MatchFinder::MatchResult &Result,
-                               const StringLiteral *Literal,
-                               const CharsBitSet &DisallowedChars) {
+static bool containsEscapedCharacters(const MatchFinder::MatchResult &Result,
+                                      const StringLiteral *Literal,
+                                      const CharsBitSet &DisallowedChars) {
   // FIXME: Handle L"", u8"", u"" and U"" literals.
   if (!Literal->isOrdinary())
     return false;
@@ -64,14 +62,12 @@ bool containsEscapedCharacters(const MatchFinder::MatchResult &Result,
   return containsEscapes(Text, R"('\"?x01)");
 }
 
-bool containsDelimiter(StringRef Bytes, const std::string &Delimiter) {
+static bool containsDelimiter(StringRef Bytes, const std::string &Delimiter) {
   return Bytes.find(Delimiter.empty()
                         ? std::string(R"lit()")lit")
                         : (")" + Delimiter + R"(")")) != StringRef::npos;
 }
 
-} // namespace
-
 RawStringLiteralCheck::RawStringLiteralCheck(StringRef Name,
                                              ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
index 1ad31d315dc2a..f2142b810a126 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceAutoPtrCheck.cpp
@@ -48,7 +48,7 @@ void ReplaceAutoPtrCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
 
 void ReplaceAutoPtrCheck::registerMatchers(MatchFinder *Finder) {
   auto AutoPtrDecl = recordDecl(hasName("auto_ptr"), isInStdNamespace());
-  auto AutoPtrType = qualType(hasDeclaration(AutoPtrDecl));
+  auto AutoPtrType = hasCanonicalType(recordType(hasDeclaration(AutoPtrDecl)));
 
   //   std::auto_ptr<int> a;
   //        ^~~~~~~~~~~~~
@@ -58,11 +58,7 @@ void ReplaceAutoPtrCheck::registerMatchers(MatchFinder *Finder) {
   //
   //   std::auto_ptr<int> fn(std::auto_ptr<int>);
   //        ^~~~~~~~~~~~~         ^~~~~~~~~~~~~
-  Finder->addMatcher(typeLoc(loc(qualType(AutoPtrType,
-                                          // Skip elaboratedType() as the named
-                                          // type will match soon thereafter.
-                                          unless(elaboratedType()))))
-                         .bind(AutoPtrTokenId),
+  Finder->addMatcher(typeLoc(loc(qualType(AutoPtrType))).bind(AutoPtrTokenId),
                      this);
 
   //   using std::auto_ptr;
@@ -118,10 +114,13 @@ void ReplaceAutoPtrCheck::check(const MatchFinder::MatchResult &Result) {
   }
 
   SourceLocation AutoPtrLoc;
-  if (const auto *TL = Result.Nodes.getNodeAs<TypeLoc>(AutoPtrTokenId)) {
+  if (const auto *PTL = Result.Nodes.getNodeAs<TypeLoc>(AutoPtrTokenId)) {
+    auto TL = *PTL;
+    if (auto QTL = TL.getAs<QualifiedTypeLoc>())
+      TL = QTL.getUnqualifiedLoc();
     //   std::auto_ptr<int> i;
     //        ^
-    if (auto Loc = TL->getAs<TemplateSpecializationTypeLoc>())
+    if (auto Loc = TL.getAs<TemplateSpecializationTypeLoc>())
       AutoPtrLoc = Loc.getTemplateNameLoc();
   } else if (const auto *D =
                  Result.Nodes.getNodeAs<UsingDecl>(AutoPtrTokenId)) {
diff --git a/clang-tools-extra/clang-tidy/modernize/TypeTraitsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/TypeTraitsCheck.cpp
index ff0b3213cb58f..472128201acc2 100644
--- a/clang-tools-extra/clang-tidy/modernize/TypeTraitsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/TypeTraitsCheck.cpp
@@ -168,19 +168,6 @@ static DeclarationName getName(const DeclRefExpr &D) {
   return D.getDecl()->getDeclName();
 }
 
-static bool isNamedType(const ElaboratedTypeLoc &ETL) {
-  if (const auto *TFT =
-          ETL.getNamedTypeLoc().getTypePtr()->getAs<TypedefType>()) {
-    const TypedefNameDecl *Decl = TFT->getDecl();
-    return Decl->getDeclName().isIdentifier() && Decl->getName() == "type";
-  }
-  return false;
-}
-
-static bool isNamedType(const DependentNameTypeLoc &DTL) {
-  return DTL.getTypePtr()->getIdentifier()->getName() == "type";
-}
-
 namespace {
 AST_POLYMORPHIC_MATCHER(isValue, AST_POLYMORPHIC_SUPPORTED_TYPES(
                                      DeclRefExpr, DependentScopeDeclRefExpr)) {
@@ -188,25 +175,20 @@ AST_POLYMORPHIC_MATCHER(isValue, AST_POLYMORPHIC_SUPPORTED_TYPES(
   return Ident && Ident->isStr("value");
 }
 
-AST_POLYMORPHIC_MATCHER(isType,
-                        AST_POLYMORPHIC_SUPPORTED_TYPES(ElaboratedTypeLoc,
-                                                        DependentNameTypeLoc)) {
-  return Node.getBeginLoc().isValid() && isNamedType(Node);
+AST_MATCHER(TypeLoc, isType) {
+  if (auto TL = Node.getAs<TypedefTypeLoc>()) {
+    const auto *TD = TL.getDecl();
+    return TD->getDeclName().isIdentifier() && TD->getName() == "type";
+  }
+  if (auto TL = Node.getAs<DependentNameTypeLoc>())
+    return TL.getTypePtr()->getIdentifier()->getName() == "type";
+  return false;
 }
 } // namespace
 
 static constexpr char Bind[] = "";
 
 void TypeTraitsCheck::registerMatchers(MatchFinder *Finder) {
-  const ast_matchers::internal::VariadicDynCastAllOfMatcher<
-      Stmt,
-      DependentScopeDeclRefExpr>
-      dependentScopeDeclRefExpr; // NOLINT(readability-identifier-naming)
-  const ast_matchers::internal::VariadicDynCastAllOfMatcher<
-      TypeLoc,
-      DependentNameTypeLoc>
-      dependentNameTypeLoc; // NOLINT(readability-identifier-naming)
-
   // Only register matchers for trait<...>::value in c++17 mode.
   if (getLangOpts().CPlusPlus17) {
     Finder->addMatcher(mapAnyOf(declRefExpr, dependentScopeDeclRefExpr)
@@ -214,10 +196,7 @@ void TypeTraitsCheck::registerMatchers(MatchFinder *Finder) {
                            .bind(Bind),
                        this);
   }
-  Finder->addMatcher(mapAnyOf(elaboratedTypeLoc, dependentNameTypeLoc)
-                         .with(isType())
-                         .bind(Bind),
-                     this);
+  Finder->addMatcher(typeLoc(isType()).bind(Bind), this);
 }
 
 static bool isNamedDeclInStdTraitsSet(const NamedDecl *ND,
@@ -226,14 +205,11 @@ static bool isNamedDeclInStdTraitsSet(const NamedDecl *ND,
          Set.contains(ND->getName());
 }
 
-static bool checkTemplatedDecl(const NestedNameSpecifier *NNS,
+static bool checkTemplatedDecl(NestedNameSpecifier NNS,
                                const llvm::StringSet<> &Set) {
-  if (!NNS)
+  if (NNS.getKind() != NestedNameSpecifier::Kind::Type)
     return false;
-  const Type *NNST = NNS->getAsType();
-  if (!NNST)
-    return false;
-  const auto *TST = NNST->getAs<TemplateSpecializationType>();
+  const auto *TST = NNS.getAsType()->getAs<TemplateSpecializationType>();
   if (!TST)
     return false;
   if (const TemplateDecl *TD = TST->getTemplateName().getAsTemplateDecl()) {
@@ -250,8 +226,8 @@ void TypeTraitsCheck::check(const MatchFinder::MatchResult &Result) {
   auto EmitValueWarning = [this, &Result](const NestedNameSpecifierLoc &QualLoc,
                                           SourceLocation EndLoc) {
     SourceLocation TemplateNameEndLoc;
-    if (auto TSTL = QualLoc.getTypeLoc().getAs<TemplateSpecializationTypeLoc>();
-        !TSTL.isNull())
+    if (auto TSTL =
+            QualLoc.getAsTypeLoc().getAs<TemplateSpecializationTypeLoc>())
       TemplateNameEndLoc = Lexer::getLocForEndOfToken(
           TSTL.getTemplateNameLoc(), 0, *Result.SourceManager,
           Result.Context->getLangOpts());
@@ -274,8 +250,8 @@ void TypeTraitsCheck::check(const MatchFinder::MatchResult &Result) {
                                          SourceLocation EndLoc,
                                          SourceLocation TypenameLoc) {
     SourceLocation TemplateNameEndLoc;
-    if (auto TSTL = QualLoc.getTypeLoc().getAs<TemplateSpecializationTypeLoc>();
-        !TSTL.isNull())
+    if (auto TSTL =
+            QualLoc.getAsTypeLoc().getAs<TemplateSpecializationTypeLoc>())
       TemplateNameEndLoc = Lexer::getLocForEndOfToken(
           TSTL.getTemplateNameLoc(), 0, *Result.SourceManager,
           Result.Context->getLangOpts());
@@ -301,23 +277,21 @@ void TypeTraitsCheck::check(const MatchFinder::MatchResult &Result) {
     if (!DRE->hasQualifier())
       return;
     if (const auto *CTSD = dyn_cast_if_present<ClassTemplateSpecializationDecl>(
-            DRE->getQualifier()->getAsRecordDecl())) {
+            DRE->getQualifier().getAsRecordDecl())) {
       if (isNamedDeclInStdTraitsSet(CTSD, ValueTraits))
         EmitValueWarning(DRE->getQualifierLoc(), DRE->getEndLoc());
     }
     return;
   }
 
-  if (const auto *ETL = Result.Nodes.getNodeAs<ElaboratedTypeLoc>(Bind)) {
-    const NestedNameSpecifierLoc QualLoc = ETL->getQualifierLoc();
-    const auto *NNS = QualLoc.getNestedNameSpecifier();
-    if (!NNS)
-      return;
+  if (const auto *TL = Result.Nodes.getNodeAs<TypedefTypeLoc>(Bind)) {
+    const NestedNameSpecifierLoc QualLoc = TL->getQualifierLoc();
+    NestedNameSpecifier NNS = QualLoc.getNestedNameSpecifier();
     if (const auto *CTSD = dyn_cast_if_present<ClassTemplateSpecializationDecl>(
-            NNS->getAsRecordDecl())) {
+            NNS.getAsRecordDecl())) {
       if (isNamedDeclInStdTraitsSet(CTSD, TypeTraits))
-        EmitTypeWarning(ETL->getQualifierLoc(), ETL->getEndLoc(),
-                        ETL->getElaboratedKeywordLoc());
+        EmitTypeWarning(TL->getQualifierLoc(), TL->getEndLoc(),
+                        TL->getElaboratedKeywordLoc());
     }
     return;
   }
diff --git a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp
index f4b63087b7234..b601620633cee 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseAutoCheck.cpp
@@ -186,16 +186,14 @@ TypeMatcher nestedIterator() {
 /// declarations and which name standard iterators for standard containers.
 TypeMatcher iteratorFromUsingDeclaration() {
   auto HasIteratorDecl = hasDeclaration(namedDecl(hasStdIteratorName()));
-  // Types resulting from using declarations are represented by elaboratedType.
-  return elaboratedType(
-      // Unwrap the nested name specifier to test for one of the standard
-      // containers.
-      hasQualifier(specifiesType(templateSpecializationType(hasDeclaration(
-          namedDecl(hasStdContainerName(), isInStdNamespace()))))),
-      // the named type is what comes after the final '::' in the type. It
-      // should name one of the standard iterator names.
-      namesType(
-          anyOf(typedefType(HasIteratorDecl), recordType(HasIteratorDecl))));
+  // Unwrap the nested name specifier to test for one of the standard
+  // containers.
+  auto Qualifier = hasQualifier(specifiesType(templateSpecializationType(
+      hasDeclaration(namedDecl(hasStdContainerName(), isInStdNamespace())))));
+  // the named type is what comes after the final '::' in the type. It should
+  // name one of the standard iterator names.
+  return anyOf(typedefType(HasIteratorDecl, Qualifier),
+               recordType(HasIteratorDecl, Qualifier));
 }
 
 /// This matcher returns declaration statements that contain variable
diff --git a/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
index e9b96c4016af6..c4a64be537a44 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseConstraintsCheck.cpp
@@ -8,6 +8,7 @@
 
 #include "UseConstraintsCheck.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/DeclTemplate.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Lex/Lexer.h"
 
@@ -60,9 +61,11 @@ matchEnableIfSpecializationImplTypename(TypeLoc TheType) {
          Keyword != ElaboratedTypeKeyword::None)) {
       return std::nullopt;
     }
-    TheType = Dep.getQualifierLoc().getTypeLoc();
+    TheType = Dep.getQualifierLoc().getAsTypeLoc();
     if (TheType.isNull())
       return std::nullopt;
+  } else {
+    return std::nullopt;
   }
 
   if (const auto SpecializationLoc =
@@ -78,6 +81,13 @@ matchEnableIfSpecializationImplTypename(TypeLoc TheType) {
     if (!TD || TD->getName() != "enable_if")
       return std::nullopt;
 
+    assert(!TD->getTemplateParameters()->empty() &&
+           "found template with no template parameters?");
+    const auto *FirstParam = dyn_cast<NonTypeTemplateParmDecl>(
+        TD->getTemplateParameters()->getParam(0));
+    if (!FirstParam || !FirstParam->getType()->isBooleanType())
+      return std::nullopt;
+
     int NumArgs = SpecializationLoc.getNumArgs();
     if (NumArgs != 1 && NumArgs != 2)
       return std::nullopt;
@@ -89,9 +99,6 @@ matchEnableIfSpecializationImplTypename(TypeLoc TheType) {
 
 static std::optional<TemplateSpecializationTypeLoc>
 matchEnableIfSpecializationImplTrait(TypeLoc TheType) {
-  if (const auto Elaborated = TheType.getAs<ElaboratedTypeLoc>())
-    TheType = Elaborated.getNamedTypeLoc();
-
   if (const auto SpecializationLoc =
           TheType.getAs<TemplateSpecializationTypeLoc>()) {
 
@@ -108,6 +115,13 @@ matchEnableIfSpecializationImplTrait(TypeLoc TheType) {
     if (!Specialization->isTypeAlias())
       return std::nullopt;
 
+    assert(!TD->getTemplateParameters()->empty() &&
+           "found template with no template parameters?");
+    const auto *FirstParam = dyn_cast<NonTypeTemplateParmDecl>(
+        TD->getTemplateParameters()->getParam(0));
+    if (!FirstParam || !FirstParam->getType()->isBooleanType())
+      return std::nullopt;
+
     if (const auto *AliasedType =
             dyn_cast<DependentNameType>(Specialization->getAliasedType())) {
       ElaboratedTypeKeyword Keyword = AliasedType->getKeyword();
diff --git a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
index aaf24eaa33c1b..ee49d8a7cb0b0 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
@@ -164,10 +164,10 @@ void UseEmplaceCheck::registerMatchers(MatchFinder *Finder) {
   auto CallEmplacy = cxxMemberCallExpr(
       hasDeclaration(
           functionDecl(hasAnyNameIgnoringTemplates(EmplacyFunctions))),
-      on(hasTypeOrPointeeType(hasCanonicalType(hasDeclaration(
-          has(typedefNameDecl(hasName("value_type"),
-                              hasType(type(hasUnqualifiedDesugaredType(
-                                  recordType().bind("value_type")))))))))));
+      on(hasTypeOrPointeeType(
+          hasCanonicalType(hasDeclaration(has(typedefNameDecl(
+              hasName("value_type"),
+              hasType(hasCanonicalType(recordType().bind("value_type"))))))))));
 
   // We can't replace push_backs of smart pointer because
   // if emplacement fails (f.e. bad_alloc in vector) we will have leak of
@@ -241,17 +241,16 @@ void UseEmplaceCheck::registerMatchers(MatchFinder *Finder) {
 
   auto HasConstructExprWithValueTypeType =
       has(ignoringImplicit(cxxConstructExpr(
-          SoughtConstructExpr, hasType(type(hasUnqualifiedDesugaredType(
-                                   type(equalsBoundNode("value_type"))))))));
-
-  auto HasBracedInitListWithValueTypeType =
-      anyOf(allOf(HasConstructInitListExpr,
-                  has(initListExpr(hasType(type(hasUnqualifiedDesugaredType(
-                      type(equalsBoundNode("value_type")))))))),
-            has(cxxBindTemporaryExpr(
-                HasConstructInitListExpr,
-                has(initListExpr(hasType(type(hasUnqualifiedDesugaredType(
-                    type(equalsBoundNode("value_type"))))))))));
+          SoughtConstructExpr,
+          hasType(hasCanonicalType(type(equalsBoundNode("value_type")))))));
+
+  auto HasBracedInitListWithValueTypeType = anyOf(
+      allOf(HasConstructInitListExpr,
+            has(initListExpr(hasType(
+                hasCanonicalType(type(equalsBoundNode("value_type"))))))),
+      has(cxxBindTemporaryExpr(HasConstructInitListExpr,
+                               has(initListExpr(hasType(hasCanonicalType(
+                                   type(equalsBoundNode("value_type")))))))));
 
   auto HasConstructExprWithValueTypeTypeAsLastArgument = hasLastArgument(
       materializeTemporaryExpr(
@@ -289,19 +288,17 @@ void UseEmplaceCheck::registerMatchers(MatchFinder *Finder) {
       this);
 
   Finder->addMatcher(
-      traverse(
-          TK_AsIs,
-          cxxMemberCallExpr(
-              CallEmplacy,
-              on(hasType(cxxRecordDecl(has(typedefNameDecl(
-                  hasName("value_type"),
-                  hasType(type(
-                      hasUnqualifiedDesugaredType(recordType(hasDeclaration(
-                          cxxRecordDecl(hasAnyName(SmallVector<StringRef, 2>(
-                              TupleTypes.begin(), TupleTypes.end()))))))))))))),
-              has(MakeTuple), hasSameNumArgsAsDeclNumParams(),
-              unless(isInTemplateInstantiation()))
-              .bind("emplacy_call")),
+      traverse(TK_AsIs,
+               cxxMemberCallExpr(
+                   CallEmplacy,
+                   on(hasType(cxxRecordDecl(has(typedefNameDecl(
+                       hasName("value_type"),
+                       hasType(hasCanonicalType(recordType(hasDeclaration(
+                           cxxRecordDecl(hasAnyName(SmallVector<StringRef, 2>(
+                               TupleTypes.begin(), TupleTypes.end())))))))))))),
+                   has(MakeTuple), hasSameNumArgsAsDeclNumParams(),
+                   unless(isInTemplateInstantiation()))
+                   .bind("emplacy_call")),
       this);
 }
 
diff --git a/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp
index 52e9a9f8d49e0..5310f2fd25381 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseScopedLockCheck.cpp
@@ -29,7 +29,7 @@ static bool isLockGuardDecl(const NamedDecl *Decl) {
 
 static bool isLockGuard(const QualType &Type) {
   if (const auto *Record = Type->getAs<RecordType>())
-    if (const RecordDecl *Decl = Record->getDecl())
+    if (const RecordDecl *Decl = Record->getOriginalDecl())
       return isLockGuardDecl(Decl);
 
   if (const auto *TemplateSpecType = Type->getAs<TemplateSpecializationType>())
@@ -89,17 +89,6 @@ findLocksInCompoundStmt(const CompoundStmt *Block,
   return LockGuardGroups;
 }
 
-static TemplateSpecializationTypeLoc
-getTemplateLockGuardTypeLoc(const TypeSourceInfo *SourceInfo) {
-  const TypeLoc Loc = SourceInfo->getTypeLoc();
-
-  const auto ElaboratedLoc = Loc.getAs<ElaboratedTypeLoc>();
-  if (!ElaboratedLoc)
-    return {};
-
-  return ElaboratedLoc.getNamedTypeLoc().getAs<TemplateSpecializationTypeLoc>();
-}
-
 // Find the exact source range of the 'lock_guard' token
 static SourceRange getLockGuardRange(const TypeSourceInfo *SourceInfo) {
   const TypeLoc LockGuardTypeLoc = SourceInfo->getTypeLoc();
@@ -110,7 +99,7 @@ static SourceRange getLockGuardRange(const TypeSourceInfo *SourceInfo) {
 // Find the exact source range of the 'lock_guard' name token
 static SourceRange getLockGuardNameRange(const TypeSourceInfo *SourceInfo) {
   const TemplateSpecializationTypeLoc TemplateLoc =
-      getTemplateLockGuardTypeLoc(SourceInfo);
+      SourceInfo->getTypeLoc().getAs<TemplateSpecializationTypeLoc>();
   if (!TemplateLoc)
     return {};
 
@@ -136,11 +125,11 @@ void UseScopedLockCheck::registerMatchers(MatchFinder *Finder) {
   const auto LockGuardClassDecl =
       namedDecl(hasName("lock_guard"), isInStdNamespace());
 
-  const auto LockGuardType = qualType(anyOf(
-      hasUnqualifiedDesugaredType(
-          recordType(hasDeclaration(LockGuardClassDecl))),
-      elaboratedType(namesType(hasUnqualifiedDesugaredType(
-          templateSpecializationType(hasDeclaration(LockGuardClassDecl)))))));
+  const auto LockGuardType =
+      qualType(anyOf(hasUnqualifiedDesugaredType(
+                         recordType(hasDeclaration(LockGuardClassDecl))),
+                     hasUnqualifiedDesugaredType(templateSpecializationType(
+                         hasDeclaration(LockGuardClassDecl)))));
 
   const auto LockVarDecl = varDecl(hasType(LockGuardType));
 
@@ -165,18 +154,16 @@ void UseScopedLockCheck::registerMatchers(MatchFinder *Finder) {
   if (WarnOnUsingAndTypedef) {
     // Match 'typedef std::lock_guard<std::mutex> Lock'
     Finder->addMatcher(typedefDecl(unless(isExpansionInSystemHeader()),
-                                   hasUnderlyingType(LockGuardType))
+                                   hasType(hasUnderlyingType(LockGuardType)))
                            .bind("lock-guard-typedef"),
                        this);
 
     // Match 'using Lock = std::lock_guard<std::mutex>'
-    Finder->addMatcher(
-        typeAliasDecl(
-            unless(isExpansionInSystemHeader()),
-            hasType(elaboratedType(namesType(templateSpecializationType(
-                hasDeclaration(LockGuardClassDecl))))))
-            .bind("lock-guard-using-alias"),
-        this);
+    Finder->addMatcher(typeAliasDecl(unless(isExpansionInSystemHeader()),
+                                     hasType(templateSpecializationType(
+                                         hasDeclaration(LockGuardClassDecl))))
+                           .bind("lock-guard-using-alias"),
+                       this);
 
     // Match 'using std::lock_guard'
     Finder->addMatcher(
@@ -288,8 +275,8 @@ void UseScopedLockCheck::diagOnSourceInfo(
     const ast_matchers::MatchFinder::MatchResult &Result) {
   const TypeLoc TL = LockGuardSourceInfo->getTypeLoc();
 
-  if (const auto ElaboratedTL = TL.getAs<ElaboratedTypeLoc>()) {
-    auto Diag = diag(ElaboratedTL.getBeginLoc(), UseScopedLockMessage);
+  if (const auto TTL = TL.getAs<TemplateSpecializationTypeLoc>()) {
+    auto Diag = diag(TTL.getBeginLoc(), UseScopedLockMessage);
 
     const SourceRange LockGuardRange =
         getLockGuardNameRange(LockGuardSourceInfo);
diff --git a/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp
index ced4825f79a99..82f64096cbec1 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseTrailingReturnTypeCheck.cpp
@@ -64,66 +64,65 @@ struct UnqualNameVisitor : public RecursiveASTVisitor<UnqualNameVisitor> {
     return false;
   }
 
-  bool TraverseTypeLoc(TypeLoc TL, bool Elaborated = false) {
+  bool TraverseTypeLoc(TypeLoc TL, bool TraverseQualifier = true) {
     if (TL.isNull())
       return true;
 
-    if (!Elaborated) {
-      switch (TL.getTypeLocClass()) {
-      case TypeLoc::Record:
-        if (visitUnqualName(
-                TL.getAs<RecordTypeLoc>().getTypePtr()->getDecl()->getName()))
-          return false;
+    switch (TL.getTypeLocClass()) {
+    case TypeLoc::InjectedClassName:
+    case TypeLoc::Record:
+    case TypeLoc::Enum: {
+      auto TTL = TL.getAs<TagTypeLoc>();
+      const auto *T = TTL.getTypePtr();
+      if (T->getKeyword() != ElaboratedTypeKeyword::None ||
+          TTL.getQualifierLoc())
         break;
-      case TypeLoc::Enum:
-        if (visitUnqualName(
-                TL.getAs<EnumTypeLoc>().getTypePtr()->getDecl()->getName()))
-          return false;
-        break;
-      case TypeLoc::TemplateSpecialization:
-        if (visitUnqualName(TL.getAs<TemplateSpecializationTypeLoc>()
-                                .getTypePtr()
-                                ->getTemplateName()
-                                .getAsTemplateDecl()
-                                ->getName()))
-          return false;
-        break;
-      case TypeLoc::Typedef:
-        if (visitUnqualName(
-                TL.getAs<TypedefTypeLoc>().getTypePtr()->getDecl()->getName()))
-          return false;
+      if (visitUnqualName(T->getOriginalDecl()->getName()))
+        return false;
+      break;
+    }
+    case TypeLoc::TemplateSpecialization: {
+      auto TTL = TL.getAs<TemplateSpecializationTypeLoc>();
+      const auto *T = TTL.getTypePtr();
+      if (T->getKeyword() != ElaboratedTypeKeyword::None ||
+          TTL.getQualifierLoc())
         break;
-      case TypeLoc::Using:
-        if (visitUnqualName(TL.getAs<UsingTypeLoc>()
-                                .getTypePtr()
-                                ->getFoundDecl()
-                                ->getName()))
-          return false;
+      if (visitUnqualName(T->getTemplateName().getAsTemplateDecl()->getName()))
+        return false;
+      break;
+    }
+    case TypeLoc::Typedef: {
+      auto TTL = TL.getAs<TypedefTypeLoc>();
+      const auto *T = TTL.getTypePtr();
+      if (T->getKeyword() != ElaboratedTypeKeyword::None ||
+          TTL.getQualifierLoc())
         break;
-      default:
+      if (visitUnqualName(T->getDecl()->getName()))
+        return false;
+      break;
+    }
+    case TypeLoc::Using: {
+      auto TTL = TL.getAs<UsingTypeLoc>();
+      const auto *T = TTL.getTypePtr();
+      if (T->getKeyword() != ElaboratedTypeKeyword::None ||
+          TTL.getQualifierLoc())
         break;
-      }
+      if (visitUnqualName(T->getDecl()->getName()))
+        return false;
+      break;
+    }
+    default:
+      break;
     }
 
-    return RecursiveASTVisitor<UnqualNameVisitor>::TraverseTypeLoc(TL);
+    return RecursiveASTVisitor<UnqualNameVisitor>::TraverseTypeLoc(
+        TL, TraverseQualifier);
   }
 
   // Replace the base method in order to call our own
   // TraverseTypeLoc().
-  bool TraverseQualifiedTypeLoc(QualifiedTypeLoc TL) {
-    return TraverseTypeLoc(TL.getUnqualifiedLoc());
-  }
-
-  // Replace the base version to inform TraverseTypeLoc that the type is
-  // elaborated.
-  bool TraverseElaboratedTypeLoc(ElaboratedTypeLoc TL) {
-    if (TL.getQualifierLoc() &&
-        !TraverseNestedNameSpecifierLoc(TL.getQualifierLoc()))
-      return false;
-    const auto *T = TL.getTypePtr();
-    return TraverseTypeLoc(TL.getNamedTypeLoc(),
-                           T->getKeyword() != ElaboratedTypeKeyword::None ||
-                               T->getQualifier());
+  bool TraverseQualifiedTypeLoc(QualifiedTypeLoc TL, bool TraverseQualifier) {
+    return TraverseTypeLoc(TL.getUnqualifiedLoc(), TraverseQualifier);
   }
 
   bool VisitDeclRefExpr(DeclRefExpr *S) {
diff --git a/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp
index a053c07f95ce2..2373a26fe48b4 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseTransparentFunctorsCheck.cpp
@@ -37,15 +37,13 @@ void UseTransparentFunctorsCheck::registerMatchers(MatchFinder *Finder) {
 
   // Non-transparent functor mentioned as a template parameter. FIXIT.
   Finder->addMatcher(
-      loc(qualType(
-              unless(elaboratedType()),
-              hasDeclaration(classTemplateSpecializationDecl(
-                  unless(hasAnyTemplateArgument(templateArgument(refersToType(
-                      qualType(pointsTo(qualType(isAnyCharacter()))))))),
-                  hasAnyTemplateArgument(
-                      templateArgument(refersToType(qualType(hasDeclaration(
-                                           TransparentFunctors))))
-                          .bind("Functor"))))))
+      loc(qualType(hasDeclaration(classTemplateSpecializationDecl(
+              unless(hasAnyTemplateArgument(templateArgument(refersToType(
+                  qualType(pointsTo(qualType(isAnyCharacter()))))))),
+              hasAnyTemplateArgument(
+                  templateArgument(refersToType(qualType(
+                                       hasDeclaration(TransparentFunctors))))
+                      .bind("Functor"))))))
           .bind("FunctorParentLoc"),
       this);
 
diff --git a/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp b/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
index bd9bdd1701975..8e4ed41c5f501 100644
--- a/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
+++ b/clang-tools-extra/clang-tidy/objc/NSInvocationArgumentLifetimeCheck.cpp
@@ -29,12 +29,13 @@
 using namespace clang::ast_matchers;
 
 namespace clang::tidy::objc {
-namespace {
 
 static constexpr StringRef WeakText = "__weak";
 static constexpr StringRef StrongText = "__strong";
 static constexpr StringRef UnsafeUnretainedText = "__unsafe_unretained";
 
+namespace {
+
 /// Matches ObjCIvarRefExpr, DeclRefExpr, or MemberExpr that reference
 /// Objective-C object (or block) variables or fields whose object lifetimes
 /// are not __unsafe_unretained.
@@ -49,6 +50,8 @@ AST_POLYMORPHIC_MATCHER(isObjCManagedLifetime,
          QT.getQualifiers().getObjCLifetime() > Qualifiers::OCL_ExplicitNone;
 }
 
+} // namespace
+
 static std::optional<FixItHint>
 fixItHintReplacementForOwnershipString(StringRef Text, CharSourceRange Range,
                                        StringRef Ownership) {
@@ -93,8 +96,6 @@ fixItHintForVarDecl(const VarDecl *VD, const SourceManager &SM,
   return FixItHint::CreateInsertion(Range.getBegin(), "__unsafe_unretained ");
 }
 
-} // namespace
-
 void NSInvocationArgumentLifetimeCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       traverse(
diff --git a/clang-tools-extra/clang-tidy/objc/PropertyDeclarationCheck.cpp b/clang-tools-extra/clang-tidy/objc/PropertyDeclarationCheck.cpp
index ffbdb025848d7..01ee4d518b97c 100644
--- a/clang-tools-extra/clang-tidy/objc/PropertyDeclarationCheck.cpp
+++ b/clang-tools-extra/clang-tidy/objc/PropertyDeclarationCheck.cpp
@@ -27,11 +27,14 @@ enum NamingStyle {
   CategoryProperty = 2,
 };
 
+} // namespace
+
 /// For now we will only fix 'CamelCase' or 'abc_CamelCase' property to
 /// 'camelCase' or 'abc_camelCase'. For other cases the users need to
 /// come up with a proper name by their own.
 /// FIXME: provide fix for snake_case to snakeCase
-FixItHint generateFixItHint(const ObjCPropertyDecl *Decl, NamingStyle Style) {
+static FixItHint generateFixItHint(const ObjCPropertyDecl *Decl,
+                                   NamingStyle Style) {
   auto Name = Decl->getName();
   auto NewName = Decl->getName().str();
   size_t Index = 0;
@@ -50,7 +53,7 @@ FixItHint generateFixItHint(const ObjCPropertyDecl *Decl, NamingStyle Style) {
   return {};
 }
 
-std::string validPropertyNameRegex(bool UsedInMatcher) {
+static std::string validPropertyNameRegex(bool UsedInMatcher) {
   // Allow any of these names:
   // foo
   // fooBar
@@ -72,13 +75,13 @@ std::string validPropertyNameRegex(bool UsedInMatcher) {
   return StartMatcher + "([a-z]|[A-Z][A-Z0-9])[a-z0-9A-Z]*$";
 }
 
-bool hasCategoryPropertyPrefix(llvm::StringRef PropertyName) {
+static bool hasCategoryPropertyPrefix(llvm::StringRef PropertyName) {
   auto RegexExp =
       llvm::Regex("^[a-zA-Z][a-zA-Z0-9]*_[a-zA-Z0-9][a-zA-Z0-9_]+$");
   return RegexExp.match(PropertyName);
 }
 
-bool prefixedPropertyNameValid(llvm::StringRef PropertyName) {
+static bool prefixedPropertyNameValid(llvm::StringRef PropertyName) {
   size_t Start = PropertyName.find_first_of('_');
   assert(Start != llvm::StringRef::npos && Start + 1 < PropertyName.size());
   auto Prefix = PropertyName.substr(0, Start);
@@ -88,7 +91,6 @@ bool prefixedPropertyNameValid(llvm::StringRef PropertyName) {
   auto RegexExp = llvm::Regex(llvm::StringRef(validPropertyNameRegex(false)));
   return RegexExp.match(PropertyName.substr(Start + 1));
 }
-} // namespace
 
 void PropertyDeclarationCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(objcPropertyDecl(
diff --git a/clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.cpp b/clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.cpp
index 7022e9d784fa7..1c018999432e3 100644
--- a/clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/NoAutomaticMoveCheck.cpp
@@ -42,11 +42,11 @@ void NoAutomaticMoveCheck::registerMatchers(MatchFinder *Finder) {
   // A matcher for a `DstT::DstT(const Src&)` where DstT also has a
   // `DstT::DstT(Src&&)`.
   const auto LValueRefCtor = cxxConstructorDecl(
-      hasParameter(0,
-                   hasType(lValueReferenceType(pointee(type().bind("SrcT"))))),
+      hasParameter(0, hasType(hasCanonicalType(
+                          lValueReferenceType(pointee(type().bind("SrcT")))))),
       ofClass(cxxRecordDecl(hasMethod(cxxConstructorDecl(
-          hasParameter(0, hasType(rValueReferenceType(
-                              pointee(type(equalsBoundNode("SrcT")))))))))));
+          hasParameter(0, hasType(hasCanonicalType(rValueReferenceType(
+                              pointee(type(equalsBoundNode("SrcT"))))))))))));
 
   // A matcher for `DstT::DstT(const Src&&)`, which typically comes from an
   // instantiation of `template <typename U> DstT::DstT(U&&)`.
diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
index 120f7fb749493..c413090b3a0a4 100644
--- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
+++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp
@@ -17,7 +17,6 @@
 #include <optional>
 
 namespace clang::tidy::performance {
-namespace {
 
 using namespace ::clang::ast_matchers;
 using llvm::StringRef;
@@ -30,8 +29,8 @@ static constexpr StringRef MethodDeclId = "methodDecl";
 static constexpr StringRef FunctionDeclId = "functionDecl";
 static constexpr StringRef OldVarDeclId = "oldVarDecl";
 
-void recordFixes(const VarDecl &Var, ASTContext &Context,
-                 DiagnosticBuilder &Diagnostic) {
+static void recordFixes(const VarDecl &Var, ASTContext &Context,
+                        DiagnosticBuilder &Diagnostic) {
   Diagnostic << utils::fixit::changeVarDeclToReference(Var, Context);
   if (!Var.getType().isLocalConstQualified()) {
     if (std::optional<FixItHint> Fix = utils::fixit::addQualifierToVarDecl(
@@ -40,8 +39,8 @@ void recordFixes(const VarDecl &Var, ASTContext &Context,
   }
 }
 
-std::optional<SourceLocation> firstLocAfterNewLine(SourceLocation Loc,
-                                                   SourceManager &SM) {
+static std::optional<SourceLocation> firstLocAfterNewLine(SourceLocation Loc,
+                                                          SourceManager &SM) {
   bool Invalid = false;
   const char *TextAfter = SM.getCharacterData(Loc, &Invalid);
   if (Invalid) {
@@ -51,8 +50,8 @@ std::optional<SourceLocation> firstLocAfterNewLine(SourceLocation Loc,
   return Loc.getLocWithOffset(TextAfter[Offset] == '\0' ? Offset : Offset + 1);
 }
 
-void recordRemoval(const DeclStmt &Stmt, ASTContext &Context,
-                   DiagnosticBuilder &Diagnostic) {
+static void recordRemoval(const DeclStmt &Stmt, ASTContext &Context,
+                          DiagnosticBuilder &Diagnostic) {
   auto &SM = Context.getSourceManager();
   // Attempt to remove trailing comments as well.
   auto Tok = utils::lexer::findNextTokenSkippingComments(Stmt.getEndLoc(), SM,
@@ -74,6 +73,8 @@ void recordRemoval(const DeclStmt &Stmt, ASTContext &Context,
   }
 }
 
+namespace {
+
 AST_MATCHER_FUNCTION_P(StatementMatcher,
                        isRefReturningMethodCallWithConstOverloads,
                        std::vector<StringRef>, ExcludedContainerTypes) {
@@ -130,6 +131,8 @@ AST_MATCHER_FUNCTION_P(StatementMatcher, initializerReturnsReferenceToConst,
                                            hasUnaryOperand(OldVarDeclRef)))));
 }
 
+} // namespace
+
 // This checks that the variable itself is only used as const, and also makes
 // sure that it does not reference another variable that could be modified in
 // the BlockStmt. It does this by checking the following:
@@ -180,13 +183,13 @@ static bool isInitializingVariableImmutable(
   return false;
 }
 
-bool isVariableUnused(const VarDecl &Var, const Stmt &BlockStmt,
-                      ASTContext &Context) {
+static bool isVariableUnused(const VarDecl &Var, const Stmt &BlockStmt,
+                             ASTContext &Context) {
   return allDeclRefExprs(Var, BlockStmt, Context).empty();
 }
 
-const SubstTemplateTypeParmType *getSubstitutedType(const QualType &Type,
-                                                    ASTContext &Context) {
+static const SubstTemplateTypeParmType *
+getSubstitutedType(const QualType &Type, ASTContext &Context) {
   auto Matches = match(
       qualType(anyOf(substTemplateTypeParmType().bind("subst"),
                      hasDescendant(substTemplateTypeParmType().bind("subst")))),
@@ -194,9 +197,9 @@ const SubstTemplateTypeParmType *getSubstitutedType(const QualType &Type,
   return selectFirst<SubstTemplateTypeParmType>("subst", Matches);
 }
 
-bool differentReplacedTemplateParams(const QualType &VarType,
-                                     const QualType &InitializerType,
-                                     ASTContext &Context) {
+static bool differentReplacedTemplateParams(const QualType &VarType,
+                                            const QualType &InitializerType,
+                                            ASTContext &Context) {
   if (const SubstTemplateTypeParmType *VarTmplType =
           getSubstitutedType(VarType, Context)) {
     if (const SubstTemplateTypeParmType *InitializerTmplType =
@@ -212,8 +215,8 @@ bool differentReplacedTemplateParams(const QualType &VarType,
   return false;
 }
 
-QualType constructorArgumentType(const VarDecl *OldVar,
-                                 const BoundNodes &Nodes) {
+static QualType constructorArgumentType(const VarDecl *OldVar,
+                                        const BoundNodes &Nodes) {
   if (OldVar) {
     return OldVar->getType();
   }
@@ -224,8 +227,6 @@ QualType constructorArgumentType(const VarDecl *OldVar,
   return MethodDecl->getReturnType();
 }
 
-} // namespace
-
 UnnecessaryCopyInitialization::UnnecessaryCopyInitialization(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
index fbd2ba67805d9..c1aa52bacf99f 100644
--- a/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryValueParamCheck.cpp
@@ -21,16 +21,14 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::performance {
 
-namespace {
-
-std::string paramNameOrIndex(StringRef Name, size_t Index) {
+static std::string paramNameOrIndex(StringRef Name, size_t Index) {
   return (Name.empty() ? llvm::Twine('#') + llvm::Twine(Index + 1)
                        : llvm::Twine('\'') + Name + llvm::Twine('\''))
       .str();
 }
 
-bool hasLoopStmtAncestor(const DeclRefExpr &DeclRef, const Decl &Decl,
-                         ASTContext &Context) {
+static bool hasLoopStmtAncestor(const DeclRefExpr &DeclRef, const Decl &Decl,
+                                ASTContext &Context) {
   auto Matches = match(
       traverse(TK_AsIs,
                decl(forEachDescendant(declRefExpr(
@@ -41,8 +39,6 @@ bool hasLoopStmtAncestor(const DeclRefExpr &DeclRef, const Decl &Decl,
   return Matches.empty();
 }
 
-} // namespace
-
 UnnecessaryValueParamCheck::UnnecessaryValueParamCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
diff --git a/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.cpp b/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.cpp
index 3b4d65be7dfa1..5a3c9a4203eb9 100644
--- a/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.cpp
+++ b/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.cpp
@@ -15,10 +15,11 @@ namespace clang::tidy::portability {
 
 void StdAllocatorConstCheck::registerMatchers(MatchFinder *Finder) {
   // Match std::allocator<const T>.
-  auto AllocatorConst =
+  auto AllocatorConst = qualType(hasCanonicalType(
       recordType(hasDeclaration(classTemplateSpecializationDecl(
           hasName("::std::allocator"),
-          hasTemplateArgument(0, refersToType(qualType(isConstQualified()))))));
+          hasTemplateArgument(0,
+                              refersToType(qualType(isConstQualified()))))))));
 
   auto HasContainerName =
       hasAnyName("::std::vector", "::std::deque", "::std::list",
@@ -31,8 +32,10 @@ void StdAllocatorConstCheck::registerMatchers(MatchFinder *Finder) {
   // aren't caught.
   Finder->addMatcher(
       typeLoc(
-          templateSpecializationTypeLoc(),
-          loc(hasUnqualifiedDesugaredType(anyOf(
+          anyOf(templateSpecializationTypeLoc(),
+                qualifiedTypeLoc(
+                    hasUnqualifiedLoc(templateSpecializationTypeLoc()))),
+          loc(qualType(anyOf(
               recordType(hasDeclaration(classTemplateSpecializationDecl(
                   HasContainerName,
                   anyOf(
diff --git a/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.h b/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.h
index d22d1aafa00af..87702af91bdb6 100644
--- a/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.h
+++ b/clang-tools-extra/clang-tidy/portability/StdAllocatorConstCheck.h
@@ -23,6 +23,9 @@ class StdAllocatorConstCheck : public ClangTidyCheck {
 public:
   StdAllocatorConstCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
 
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
diff --git a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
index ce736a8d16023..c4dc319f23c38 100644
--- a/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ContainerSizeEmptyCheck.cpp
@@ -238,9 +238,12 @@ void ContainerSizeEmptyCheck::check(const MatchFinder::MatchResult &Result) {
           ? MemberCallObject
           : (Pointee ? Pointee : Result.Nodes.getNodeAs<Expr>("STLObject"));
   FixItHint Hint;
-  std::string ReplacementText = std::string(
-      Lexer::getSourceText(CharSourceRange::getTokenRange(E->getSourceRange()),
-                           *Result.SourceManager, getLangOpts()));
+  std::string ReplacementText =
+      E->isImplicitCXXThis()
+          ? ""
+          : std::string(Lexer::getSourceText(
+                CharSourceRange::getTokenRange(E->getSourceRange()),
+                *Result.SourceManager, getLangOpts()));
   const auto *OpCallExpr = dyn_cast<CXXOperatorCallExpr>(E);
   if (isBinaryOrTernary(E) || isa<UnaryOperator>(E) ||
       (OpCallExpr && (OpCallExpr->getOperator() == OO_Star))) {
@@ -251,6 +254,8 @@ void ContainerSizeEmptyCheck::check(const MatchFinder::MatchResult &Result) {
     // This can happen if the object is a smart pointer. Don't add anything
     // because a '->' is already there (PR#51776), just call the method.
     ReplacementText += "empty()";
+  } else if (E->isImplicitCXXThis()) {
+    ReplacementText += "empty()";
   } else if (E->getType()->isPointerType())
     ReplacementText += "->empty()";
   else
diff --git a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.h b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.h
index 79b0d39bed41b..1b12fec972998 100644
--- a/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.h
+++ b/clang-tools-extra/clang-tidy/readability/ConvertMemberFunctionsToStatic.h
@@ -23,6 +23,9 @@ class ConvertMemberFunctionsToStatic : public ClangTidyCheck {
 public:
   ConvertMemberFunctionsToStatic(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
diff --git a/clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.h b/clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.h
index e9e7c942d1d2c..6e746d803d3ee 100644
--- a/clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/DeleteNullPointerCheck.h
@@ -22,6 +22,9 @@ class DeleteNullPointerCheck : public ClangTidyCheck {
 public:
   DeleteNullPointerCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   std::optional<TraversalKind> getCheckTraversalKind() const override {
diff --git a/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp
index b4a157c153bb9..9eef5c4db2d01 100644
--- a/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/EnumInitialValueCheck.cpp
@@ -122,15 +122,15 @@ AST_MATCHER(EnumDecl, hasSequentialInitialValues) {
   return !AllEnumeratorsArePowersOfTwo;
 }
 
-std::string getName(const EnumDecl *Decl) {
+} // namespace
+
+static std::string getName(const EnumDecl *Decl) {
   if (!Decl->getDeclName())
     return "<unnamed>";
 
   return Decl->getQualifiedNameAsString();
 }
 
-} // namespace
-
 EnumInitialValueCheck::EnumInitialValueCheck(StringRef Name,
                                              ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
diff --git a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp
index e1fb42b8210e2..2f59aaa86b157 100644
--- a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp
@@ -144,6 +144,8 @@ struct CognitiveComplexity final {
   void account(SourceLocation Loc, unsigned short Nesting, Criteria C);
 };
 
+} // namespace
+
 // All the possible messages that can be output. The choice of the message
 // to use is based of the combination of the CognitiveComplexity::Criteria.
 // It would be nice to have it in CognitiveComplexity struct, but then it is
@@ -163,23 +165,27 @@ static const std::array<const StringRef, 4> Msgs = {{
 }};
 
 // Criteria is a bitset, thus a few helpers are needed.
-CognitiveComplexity::Criteria operator|(CognitiveComplexity::Criteria LHS,
-                                        CognitiveComplexity::Criteria RHS) {
+static CognitiveComplexity::Criteria
+operator|(CognitiveComplexity::Criteria LHS,
+          CognitiveComplexity::Criteria RHS) {
   return static_cast<CognitiveComplexity::Criteria>(llvm::to_underlying(LHS) |
                                                     llvm::to_underlying(RHS));
 }
-CognitiveComplexity::Criteria operator&(CognitiveComplexity::Criteria LHS,
-                                        CognitiveComplexity::Criteria RHS) {
+static CognitiveComplexity::Criteria
+operator&(CognitiveComplexity::Criteria LHS,
+          CognitiveComplexity::Criteria RHS) {
   return static_cast<CognitiveComplexity::Criteria>(llvm::to_underlying(LHS) &
                                                     llvm::to_underlying(RHS));
 }
-CognitiveComplexity::Criteria &operator|=(CognitiveComplexity::Criteria &LHS,
-                                          CognitiveComplexity::Criteria RHS) {
+static CognitiveComplexity::Criteria &
+operator|=(CognitiveComplexity::Criteria &LHS,
+           CognitiveComplexity::Criteria RHS) {
   LHS = operator|(LHS, RHS);
   return LHS;
 }
-CognitiveComplexity::Criteria &operator&=(CognitiveComplexity::Criteria &LHS,
-                                          CognitiveComplexity::Criteria RHS) {
+static CognitiveComplexity::Criteria &
+operator&=(CognitiveComplexity::Criteria &LHS,
+           CognitiveComplexity::Criteria RHS) {
   LHS = operator&(LHS, RHS);
   return LHS;
 }
@@ -199,6 +205,8 @@ void CognitiveComplexity::account(SourceLocation Loc, unsigned short Nesting,
   Total += Increase;
 }
 
+namespace {
+
 class FunctionASTVisitor final
     : public RecursiveASTVisitor<FunctionASTVisitor> {
   using Base = RecursiveASTVisitor<FunctionASTVisitor>;
diff --git a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
index 5f19706e16866..6b10e6b206a31 100644
--- a/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/ImplicitBoolConversionCheck.cpp
@@ -41,9 +41,11 @@ AST_MATCHER(Stmt, isNULLMacroExpansion) {
   return isNULLMacroExpansion(&Node, Finder->getASTContext());
 }
 
-StringRef getZeroLiteralToCompareWithForType(CastKind CastExprKind,
-                                             QualType Type,
-                                             ASTContext &Context) {
+} // namespace
+
+static StringRef getZeroLiteralToCompareWithForType(CastKind CastExprKind,
+                                                    QualType Type,
+                                                    ASTContext &Context) {
   switch (CastExprKind) {
   case CK_IntegralToBoolean:
     return Type->isUnsignedIntegerType() ? "0u" : "0";
@@ -62,15 +64,15 @@ StringRef getZeroLiteralToCompareWithForType(CastKind CastExprKind,
   }
 }
 
-bool isUnaryLogicalNotOperator(const Stmt *Statement) {
+static bool isUnaryLogicalNotOperator(const Stmt *Statement) {
   const auto *UnaryOperatorExpr = dyn_cast<UnaryOperator>(Statement);
   return UnaryOperatorExpr && UnaryOperatorExpr->getOpcode() == UO_LNot;
 }
 
-void fixGenericExprCastToBool(DiagnosticBuilder &Diag,
-                              const ImplicitCastExpr *Cast, const Stmt *Parent,
-                              ASTContext &Context,
-                              bool UseUpperCaseLiteralSuffix) {
+static void fixGenericExprCastToBool(DiagnosticBuilder &Diag,
+                                     const ImplicitCastExpr *Cast,
+                                     const Stmt *Parent, ASTContext &Context,
+                                     bool UseUpperCaseLiteralSuffix) {
   // In case of expressions like (! integer), we should remove the redundant not
   // operator and use inverted comparison (integer == 0).
   bool InvertComparison =
@@ -133,8 +135,8 @@ void fixGenericExprCastToBool(DiagnosticBuilder &Diag,
   Diag << FixItHint::CreateInsertion(EndLoc, EndLocInsertion);
 }
 
-StringRef getEquivalentBoolLiteralForExpr(const Expr *Expression,
-                                          ASTContext &Context) {
+static StringRef getEquivalentBoolLiteralForExpr(const Expr *Expression,
+                                                 ASTContext &Context) {
   if (isNULLMacroExpansion(Expression, Context)) {
     return "false";
   }
@@ -161,7 +163,7 @@ StringRef getEquivalentBoolLiteralForExpr(const Expr *Expression,
   return {};
 }
 
-bool needsSpacePrefix(SourceLocation Loc, ASTContext &Context) {
+static bool needsSpacePrefix(SourceLocation Loc, ASTContext &Context) {
   SourceRange PrefixRange(Loc.getLocWithOffset(-1), Loc);
   StringRef SpaceBeforeStmtStr = Lexer::getSourceText(
       CharSourceRange::getCharRange(PrefixRange), Context.getSourceManager(),
@@ -173,9 +175,10 @@ bool needsSpacePrefix(SourceLocation Loc, ASTContext &Context) {
   return !AllowedCharacters.contains(SpaceBeforeStmtStr.back());
 }
 
-void fixGenericExprCastFromBool(DiagnosticBuilder &Diag,
-                                const ImplicitCastExpr *Cast,
-                                ASTContext &Context, StringRef OtherType) {
+static void fixGenericExprCastFromBool(DiagnosticBuilder &Diag,
+                                       const ImplicitCastExpr *Cast,
+                                       ASTContext &Context,
+                                       StringRef OtherType) {
   if (!Context.getLangOpts().CPlusPlus) {
     Diag << FixItHint::CreateInsertion(Cast->getBeginLoc(),
                                        (Twine("(") + OtherType + ")").str());
@@ -200,8 +203,9 @@ void fixGenericExprCastFromBool(DiagnosticBuilder &Diag,
   }
 }
 
-StringRef getEquivalentForBoolLiteral(const CXXBoolLiteralExpr *BoolLiteral,
-                                      QualType DestType, ASTContext &Context) {
+static StringRef
+getEquivalentForBoolLiteral(const CXXBoolLiteralExpr *BoolLiteral,
+                            QualType DestType, ASTContext &Context) {
   // Prior to C++11, false literal could be implicitly converted to pointer.
   if (!Context.getLangOpts().CPlusPlus11 &&
       (DestType->isPointerType() || DestType->isMemberPointerType()) &&
@@ -222,8 +226,8 @@ StringRef getEquivalentForBoolLiteral(const CXXBoolLiteralExpr *BoolLiteral,
   return BoolLiteral->getValue() ? "1" : "0";
 }
 
-bool isCastAllowedInCondition(const ImplicitCastExpr *Cast,
-                              ASTContext &Context) {
+static bool isCastAllowedInCondition(const ImplicitCastExpr *Cast,
+                                     ASTContext &Context) {
   std::queue<const Stmt *> Q;
   Q.push(Cast);
 
@@ -251,8 +255,6 @@ bool isCastAllowedInCondition(const ImplicitCastExpr *Cast,
   return false;
 }
 
-} // anonymous namespace
-
 ImplicitBoolConversionCheck::ImplicitBoolConversionCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
diff --git a/clang-tools-extra/clang-tidy/readability/QualifiedAutoCheck.cpp b/clang-tools-extra/clang-tidy/readability/QualifiedAutoCheck.cpp
index 561f067d471d1..44a784bc9f21a 100644
--- a/clang-tools-extra/clang-tidy/readability/QualifiedAutoCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/QualifiedAutoCheck.cpp
@@ -28,8 +28,11 @@ AST_MATCHER_P(QualType, hasUnqualifiedType,
 
 enum class Qualifier { Const, Volatile, Restrict };
 
-std::optional<Token> findQualToken(const VarDecl *Decl, Qualifier Qual,
-                                   const MatchFinder::MatchResult &Result) {
+} // namespace
+
+static std::optional<Token>
+findQualToken(const VarDecl *Decl, Qualifier Qual,
+              const MatchFinder::MatchResult &Result) {
   // Since either of the locs can be in a macro, use `makeFileCharRange` to be
   // sure that we have a consistent `CharSourceRange`, located entirely in the
   // source file.
@@ -58,7 +61,7 @@ std::optional<Token> findQualToken(const VarDecl *Decl, Qualifier Qual,
                                           *Result.SourceManager);
 }
 
-std::optional<SourceRange>
+static std::optional<SourceRange>
 getTypeSpecifierLocation(const VarDecl *Var,
                          const MatchFinder::MatchResult &Result) {
   SourceRange TypeSpecifier(
@@ -73,8 +76,8 @@ getTypeSpecifierLocation(const VarDecl *Var,
   return TypeSpecifier;
 }
 
-std::optional<SourceRange> mergeReplacementRange(SourceRange &TypeSpecifier,
-                                                 const Token &ConstToken) {
+static std::optional<SourceRange>
+mergeReplacementRange(SourceRange &TypeSpecifier, const Token &ConstToken) {
   if (TypeSpecifier.getBegin().getLocWithOffset(-1) == ConstToken.getEndLoc()) {
     TypeSpecifier.setBegin(ConstToken.getLocation());
     return std::nullopt;
@@ -86,21 +89,19 @@ std::optional<SourceRange> mergeReplacementRange(SourceRange &TypeSpecifier,
   return SourceRange(ConstToken.getLocation(), ConstToken.getEndLoc());
 }
 
-bool isPointerConst(QualType QType) {
+static bool isPointerConst(QualType QType) {
   QualType Pointee = QType->getPointeeType();
   assert(!Pointee.isNull() && "can't have a null Pointee");
   return Pointee.isConstQualified();
 }
 
-bool isAutoPointerConst(QualType QType) {
+static bool isAutoPointerConst(QualType QType) {
   QualType Pointee =
       cast<AutoType>(QType->getPointeeType().getTypePtr())->desugar();
   assert(!Pointee.isNull() && "can't have a null Pointee");
   return Pointee.isConstQualified();
 }
 
-} // namespace
-
 QualifiedAutoCheck::QualifiedAutoCheck(StringRef Name,
                                        ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
diff --git a/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.cpp b/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.cpp
index 70016a372b8bd..d93077cc6884e 100644
--- a/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/RedundantControlFlowCheck.cpp
@@ -14,19 +14,18 @@ using namespace clang::ast_matchers;
 
 namespace clang::tidy::readability {
 
-namespace {
+static const char *const RedundantReturnDiag =
+    "redundant return statement at the end "
+    "of a function with a void return type";
+static const char *const RedundantContinueDiag =
+    "redundant continue statement at the "
+    "end of loop statement";
 
-const char *const RedundantReturnDiag = "redundant return statement at the end "
-                                        "of a function with a void return type";
-const char *const RedundantContinueDiag = "redundant continue statement at the "
-                                          "end of loop statement";
-
-bool isLocationInMacroExpansion(const SourceManager &SM, SourceLocation Loc) {
+static bool isLocationInMacroExpansion(const SourceManager &SM,
+                                       SourceLocation Loc) {
   return SM.isMacroBodyExpansion(Loc) || SM.isMacroArgExpansion(Loc);
 }
 
-} // namespace
-
 void RedundantControlFlowCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
       functionDecl(isDefinition(), returns(voidType()),
diff --git a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
index fffb136e5a332..a7b3c4a1f7cf9 100644
--- a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
@@ -19,19 +19,25 @@ namespace {
 AST_MATCHER(CXXMethodDecl, isStatic) { return Node.isStatic(); }
 } // namespace
 
-static unsigned getNameSpecifierNestingLevel(const QualType &QType) {
-  if (const auto *ElType = QType->getAs<ElaboratedType>()) {
-    if (const NestedNameSpecifier *NestedSpecifiers = ElType->getQualifier()) {
-      unsigned NameSpecifierNestingLevel = 1;
-      do {
-        NameSpecifierNestingLevel++;
-        NestedSpecifiers = NestedSpecifiers->getPrefix();
-      } while (NestedSpecifiers);
-
+static unsigned getNameSpecifierNestingLevel(QualType QType) {
+  unsigned NameSpecifierNestingLevel = 1;
+  for (NestedNameSpecifier Qualifier = QType->getPrefix(); /**/;
+       ++NameSpecifierNestingLevel) {
+    switch (Qualifier.getKind()) {
+    case NestedNameSpecifier::Kind::Null:
       return NameSpecifierNestingLevel;
+    case NestedNameSpecifier::Kind::Global:
+    case NestedNameSpecifier::Kind::MicrosoftSuper:
+      return NameSpecifierNestingLevel + 1;
+    case NestedNameSpecifier::Kind::Namespace:
+      Qualifier = Qualifier.getAsNamespaceAndPrefix().Prefix;
+      continue;
+    case NestedNameSpecifier::Kind::Type:
+      Qualifier = Qualifier.getAsType()->getPrefix();
+      continue;
     }
+    llvm_unreachable("unhandled nested name specifier kind");
   }
-  return 0;
 }
 
 void StaticAccessedThroughInstanceCheck::storeOptions(
diff --git a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.h b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.h
index 4044f13769aa5..9869855c17d6b 100644
--- a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.h
@@ -24,6 +24,9 @@ class StaticAccessedThroughInstanceCheck : public ClangTidyCheck {
       : ClangTidyCheck(Name, Context),
         NameSpecifierNestingThreshold(
             Options.get("NameSpecifierNestingThreshold", 3U)) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
 
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
diff --git a/clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.cpp b/clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.cpp
index 5a04029e4a6fa..447c2437666cf 100644
--- a/clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/SuspiciousCallArgumentCheck.cpp
@@ -414,9 +414,9 @@ static bool areTypesCompatible(QualType ArgType, QualType ParamType,
   // Arithmetic types are interconvertible, except scoped enums.
   if (ParamType->isArithmeticType() && ArgType->isArithmeticType()) {
     if ((ParamType->isEnumeralType() &&
-         ParamType->castAs<EnumType>()->getDecl()->isScoped()) ||
+         ParamType->castAs<EnumType>()->getOriginalDecl()->isScoped()) ||
         (ArgType->isEnumeralType() &&
-         ArgType->castAs<EnumType>()->getDecl()->isScoped()))
+         ArgType->castAs<EnumType>()->getOriginalDecl()->isScoped()))
       return false;
 
     return true;
diff --git a/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.h b/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.h
index 4dfcf36e4789f..2768955109d26 100644
--- a/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/UniqueptrDeleteReleaseCheck.h
@@ -21,6 +21,9 @@ namespace clang::tidy::readability {
 class UniqueptrDeleteReleaseCheck : public ClangTidyCheck {
 public:
   UniqueptrDeleteReleaseCheck(StringRef Name, ClangTidyContext *Context);
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
diff --git a/clang-tools-extra/clang-tidy/readability/UseStdMinMaxCheck.cpp b/clang-tools-extra/clang-tidy/readability/UseStdMinMaxCheck.cpp
index 6f6b8a853a91e..718467ed02f0a 100644
--- a/clang-tools-extra/clang-tidy/readability/UseStdMinMaxCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/UseStdMinMaxCheck.cpp
@@ -67,11 +67,7 @@ static QualType getNonTemplateAlias(QualType QT) {
       if (!TT->getDecl()->getDescribedTemplate() &&
           !TT->getDecl()->getDeclContext()->isDependentContext())
         return QT;
-      QT = TT->getDecl()->getUnderlyingType();
-    }
-    // cast to elaborated type
-    else if (const ElaboratedType *ET = dyn_cast<ElaboratedType>(QT)) {
-      QT = ET->getNamedType();
+      QT = TT->desugar();
     } else {
       break;
     }
diff --git a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
index 97dfd0fc943b4..bef3b938b5afd 100644
--- a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
+++ b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
@@ -162,14 +162,19 @@ in .clang-tidy file, if any.
                                    cl::init(false), cl::cat(ClangTidyCategory));
 
 static cl::opt<std::string> LineFilter("line-filter", desc(R"(
-List of files with line ranges to filter the
-warnings. Can be used together with
--header-filter. The format of the list is a
-JSON array of objects:
+List of files and line ranges to output diagnostics from.
+The range is inclusive on both ends. Can be used together
+with -header-filter. The format of the list is a JSON
+array of objects. For example:
+
   [
     {"name":"file1.cpp","lines":[[1,3],[5,7]]},
     {"name":"file2.h"}
   ]
+
+This will output diagnostics from 'file1.cpp' only for
+the line ranges [1,3] and [5,7], as well as all from the
+entire 'file2.h'.
 )"),
                                        cl::init(""),
                                        cl::cat(ClangTidyCategory));
@@ -712,7 +717,7 @@ int clangTidyMain(int argc, const char **argv) {
                            EnableModuleHeadersParsing);
   std::vector<ClangTidyError> Errors =
       runClangTidy(Context, OptionsParser->getCompilations(), PathList, BaseFS,
-                   FixNotes, EnableCheckProfile, ProfilePrefix);
+                   FixNotes, EnableCheckProfile, ProfilePrefix, Quiet);
   bool FoundErrors = llvm::any_of(Errors, [](const ClangTidyError &E) {
     return E.DiagLevel == ClangTidyError::Error;
   });
diff --git a/clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py b/clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
index 7cd21afd70f7e..d7899e0a18d0c 100755
--- a/clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
+++ b/clang-tools-extra/clang-tidy/tool/clang-tidy-diff.py
@@ -28,6 +28,7 @@
 import json
 import multiprocessing
 import os
+import queue
 import re
 import shutil
 import subprocess
@@ -42,13 +43,6 @@
 except ImportError:
     yaml = None
 
-is_py2 = sys.version[0] == "2"
-
-if is_py2:
-    import Queue as queue
-else:
-    import queue as queue
-
 
 def run_tidy(task_queue, lock, timeout, failed_files):
     watchdog = None
diff --git a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
index 80f1766b0bf10..670e0a2c7678a 100755
--- a/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
+++ b/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py
@@ -483,7 +483,7 @@ async def main() -> None:
     parser.add_argument(
         "-line-filter",
         default=None,
-        help="List of files with line ranges to filter the warnings.",
+        help="List of files and line ranges to output diagnostics from.",
     )
     if yaml:
         parser.add_argument(
diff --git a/clang-tools-extra/clang-tidy/utils/ExceptionSpecAnalyzer.cpp b/clang-tools-extra/clang-tidy/utils/ExceptionSpecAnalyzer.cpp
index 0637d0eff688c..aa6aefcf0c493 100644
--- a/clang-tools-extra/clang-tidy/utils/ExceptionSpecAnalyzer.cpp
+++ b/clang-tools-extra/clang-tidy/utils/ExceptionSpecAnalyzer.cpp
@@ -66,7 +66,8 @@ ExceptionSpecAnalyzer::analyzeBase(const CXXBaseSpecifier &Base,
   if (!RecType)
     return State::Unknown;
 
-  const auto *BaseClass = cast<CXXRecordDecl>(RecType->getDecl());
+  const auto *BaseClass =
+      cast<CXXRecordDecl>(RecType->getOriginalDecl())->getDefinitionOrSelf();
 
   return analyzeRecord(BaseClass, Kind);
 }
diff --git a/clang-tools-extra/clang-tidy/utils/FormatStringConverter.cpp b/clang-tools-extra/clang-tidy/utils/FormatStringConverter.cpp
index e1c1bee97f6d4..0df8e913100fc 100644
--- a/clang-tools-extra/clang-tidy/utils/FormatStringConverter.cpp
+++ b/clang-tools-extra/clang-tidy/utils/FormatStringConverter.cpp
@@ -461,8 +461,9 @@ bool FormatStringConverter::emitIntegerArgument(
     // the signedness based on the format string, so we need to do the
     // same.
     if (const auto *ET = ArgType->getAs<EnumType>()) {
-      if (const std::optional<std::string> MaybeCastType =
-              castTypeForArgument(ArgKind, ET->getDecl()->getIntegerType()))
+      if (const std::optional<std::string> MaybeCastType = castTypeForArgument(
+              ArgKind,
+              ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType()))
         ArgFixes.emplace_back(
             ArgIndex, (Twine("static_cast<") + *MaybeCastType + ">(").str());
       else
diff --git a/clang-tools-extra/clang-tidy/utils/Matchers.cpp b/clang-tools-extra/clang-tidy/utils/Matchers.cpp
index 4974a9cdb9f4b..bd7b03eb39ad7 100644
--- a/clang-tools-extra/clang-tidy/utils/Matchers.cpp
+++ b/clang-tools-extra/clang-tidy/utils/Matchers.cpp
@@ -34,7 +34,7 @@ bool MatchesAnyListedTypeNameMatcher::matches(
   PrintingPolicy PrintingPolicyWithSuppressedTag(
       Finder->getASTContext().getLangOpts());
   PrintingPolicyWithSuppressedTag.PrintAsCanonical = CanonicalTypes;
-  PrintingPolicyWithSuppressedTag.SuppressElaboration = true;
+  PrintingPolicyWithSuppressedTag.FullyQualifiedName = true;
   PrintingPolicyWithSuppressedTag.SuppressScope = false;
   PrintingPolicyWithSuppressedTag.SuppressTagKeyword = true;
   PrintingPolicyWithSuppressedTag.SuppressUnwrittenScope = true;
diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
index eaa04fef08c43..90539eaabbe03 100644
--- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
@@ -194,6 +194,8 @@ class RenamerClangTidyCheckPPCallbacks : public PPCallbacks {
       return;
     if (SM.isWrittenInCommandLineFile(MacroNameTok.getLocation()))
       return;
+    if (SM.isInSystemHeader(MacroNameTok.getLocation()))
+      return;
     Check->checkMacro(MacroNameTok, Info, SM);
   }
 
@@ -281,9 +283,10 @@ class RenamerClangTidyVisitor
   }
 
   bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc Loc) {
-    if (const NestedNameSpecifier *Spec = Loc.getNestedNameSpecifier()) {
+    if (NestedNameSpecifier Spec = Loc.getNestedNameSpecifier();
+        Spec.getKind() == NestedNameSpecifier::Kind::Namespace) {
       if (const auto *Decl =
-              dyn_cast_if_present<NamespaceDecl>(Spec->getAsNamespace()))
+              dyn_cast<NamespaceDecl>(Spec.getAsNamespaceAndPrefix().Namespace))
         Check->addUsage(Decl, Loc.getLocalSourceRange(), SM);
     }
 
@@ -323,48 +326,34 @@ class RenamerClangTidyVisitor
   }
 
   bool VisitTypedefTypeLoc(const TypedefTypeLoc &Loc) {
-    Check->addUsage(Loc.getTypedefNameDecl(), Loc.getSourceRange(), SM);
+    Check->addUsage(Loc.getDecl(), Loc.getNameLoc(), SM);
     return true;
   }
 
   bool VisitTagTypeLoc(const TagTypeLoc &Loc) {
-    Check->addUsage(Loc.getDecl(), Loc.getSourceRange(), SM);
-    return true;
-  }
-
-  bool VisitInjectedClassNameTypeLoc(const InjectedClassNameTypeLoc &Loc) {
-    Check->addUsage(Loc.getDecl(), Loc.getSourceRange(), SM);
+    Check->addUsage(Loc.getOriginalDecl(), Loc.getNameLoc(), SM);
     return true;
   }
 
   bool VisitUnresolvedUsingTypeLoc(const UnresolvedUsingTypeLoc &Loc) {
-    Check->addUsage(Loc.getDecl(), Loc.getSourceRange(), SM);
+    Check->addUsage(Loc.getDecl(), Loc.getNameLoc(), SM);
     return true;
   }
 
   bool VisitTemplateTypeParmTypeLoc(const TemplateTypeParmTypeLoc &Loc) {
-    Check->addUsage(Loc.getDecl(), Loc.getSourceRange(), SM);
+    Check->addUsage(Loc.getDecl(), Loc.getNameLoc(), SM);
     return true;
   }
 
   bool
   VisitTemplateSpecializationTypeLoc(const TemplateSpecializationTypeLoc &Loc) {
     const TemplateDecl *Decl =
-        Loc.getTypePtr()->getTemplateName().getAsTemplateDecl();
+        Loc.getTypePtr()->getTemplateName().getAsTemplateDecl(
+            /*IgnoreDeduced=*/true);
 
-    SourceRange Range(Loc.getTemplateNameLoc(), Loc.getTemplateNameLoc());
-    if (const auto *ClassDecl = dyn_cast<TemplateDecl>(Decl)) {
+    if (const auto *ClassDecl = dyn_cast<TemplateDecl>(Decl))
       if (const NamedDecl *TemplDecl = ClassDecl->getTemplatedDecl())
-        Check->addUsage(TemplDecl, Range, SM);
-    }
-
-    return true;
-  }
-
-  bool VisitDependentTemplateSpecializationTypeLoc(
-      const DependentTemplateSpecializationTypeLoc &Loc) {
-    if (const TagDecl *Decl = Loc.getTypePtr()->getAsTagDecl())
-      Check->addUsage(Decl, Loc.getSourceRange(), SM);
+        Check->addUsage(TemplDecl, Loc.getTemplateNameLoc(), SM);
 
     return true;
   }
diff --git a/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp b/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp
index 44db0c2aed607..5518afd32e7f0 100644
--- a/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp
+++ b/clang-tools-extra/clang-tidy/utils/TypeTraits.cpp
@@ -13,16 +13,14 @@
 
 namespace clang::tidy::utils::type_traits {
 
-namespace {
-
-bool classHasTrivialCopyAndDestroy(QualType Type) {
+static bool classHasTrivialCopyAndDestroy(QualType Type) {
   auto *Record = Type->getAsCXXRecordDecl();
   return Record && Record->hasDefinition() &&
          !Record->hasNonTrivialCopyConstructor() &&
          !Record->hasNonTrivialDestructor();
 }
 
-bool hasDeletedCopyConstructor(QualType Type) {
+static bool hasDeletedCopyConstructor(QualType Type) {
   auto *Record = Type->getAsCXXRecordDecl();
   if (!Record || !Record->hasDefinition())
     return false;
@@ -33,8 +31,6 @@ bool hasDeletedCopyConstructor(QualType Type) {
   return false;
 }
 
-} // namespace
-
 std::optional<bool> isExpensiveToCopy(QualType Type,
                                       const ASTContext &Context) {
   if (Type->isDependentType() || Type->isIncompleteType())
@@ -124,7 +120,8 @@ bool isTriviallyDefaultConstructible(QualType Type, const ASTContext &Context) {
     return true;
 
   if (const auto *RT = CanonicalType->getAs<RecordType>()) {
-    return recordIsTriviallyDefaultConstructible(*RT->getDecl(), Context);
+    return recordIsTriviallyDefaultConstructible(
+        *RT->getOriginalDecl()->getDefinitionOrSelf(), Context);
   }
 
   // No other types can match.
diff --git a/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.h b/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.h
index e1f1f06ab3735..b2d5ab61fb0dc 100644
--- a/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.h
+++ b/clang-tools-extra/clang-tidy/zircon/TemporaryObjectsCheck.h
@@ -24,6 +24,9 @@ class TemporaryObjectsCheck : public ClangTidyCheck {
   TemporaryObjectsCheck(StringRef Name, ClangTidyContext *Context)
       : ClangTidyCheck(Name, Context),
         Names(utils::options::parseStringList(Options.get("Names", ""))) {}
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus;
+  }
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
diff --git a/clang-tools-extra/clangd/AST.cpp b/clang-tools-extra/clangd/AST.cpp
index f2631e5abb6a3..2f46ecc92576c 100644
--- a/clang-tools-extra/clangd/AST.cpp
+++ b/clang-tools-extra/clangd/AST.cpp
@@ -102,54 +102,78 @@ getUsingNamespaceDirectives(const DeclContext *DestContext,
 // ancestor is redundant, therefore we stop at lowest common ancestor.
 // In addition to that stops early whenever IsVisible returns true. This can be
 // used to implement support for "using namespace" decls.
-std::string
-getQualification(ASTContext &Context, const DeclContext *DestContext,
-                 const DeclContext *SourceContext,
-                 llvm::function_ref<bool(NestedNameSpecifier *)> IsVisible) {
-  std::vector<const NestedNameSpecifier *> Parents;
-  bool ReachedNS = false;
+std::string getQualification(ASTContext &Context,
+                             const DeclContext *DestContext,
+                             const DeclContext *SourceContext,
+                             llvm::function_ref<bool(const Decl *)> IsVisible) {
+  std::vector<const Decl *> Parents;
+  [[maybe_unused]] bool ReachedNS = false;
   for (const DeclContext *CurContext = SourceContext; CurContext;
        CurContext = CurContext->getLookupParent()) {
     // Stop once we reach a common ancestor.
     if (CurContext->Encloses(DestContext))
       break;
 
-    NestedNameSpecifier *NNS = nullptr;
+    const Decl *CurD;
     if (auto *TD = llvm::dyn_cast<TagDecl>(CurContext)) {
       // There can't be any more tag parents after hitting a namespace.
       assert(!ReachedNS);
-      (void)ReachedNS;
-      NNS = NestedNameSpecifier::Create(Context, nullptr, TD->getTypeForDecl());
+      CurD = TD;
     } else if (auto *NSD = llvm::dyn_cast<NamespaceDecl>(CurContext)) {
       ReachedNS = true;
-      NNS = NestedNameSpecifier::Create(Context, nullptr, NSD);
       // Anonymous and inline namespace names are not spelled while qualifying
       // a name, so skip those.
       if (NSD->isAnonymousNamespace() || NSD->isInlineNamespace())
         continue;
+      CurD = NSD;
     } else {
       // Other types of contexts cannot be spelled in code, just skip over
       // them.
       continue;
     }
     // Stop if this namespace is already visible at DestContext.
-    if (IsVisible(NNS))
+    if (IsVisible(CurD))
       break;
 
-    Parents.push_back(NNS);
+    Parents.push_back(CurD);
+  }
+
+  // Go over the declarations in reverse order, since we stored inner-most
+  // parent first.
+  NestedNameSpecifier Qualifier = std::nullopt;
+  bool IsFirst = true;
+  for (const auto *CurD : llvm::reverse(Parents)) {
+    if (auto *TD = llvm::dyn_cast<TagDecl>(CurD)) {
+      QualType T;
+      if (const auto *RD = dyn_cast<CXXRecordDecl>(TD);
+          ClassTemplateDecl *CTD = RD->getDescribedClassTemplate()) {
+        ArrayRef<TemplateArgument> Args;
+        if (const auto *SD = dyn_cast<ClassTemplateSpecializationDecl>(RD))
+          Args = SD->getTemplateArgs().asArray();
+        else
+          Args = CTD->getTemplateParameters()->getInjectedTemplateArgs(Context);
+        T = Context.getTemplateSpecializationType(
+            ElaboratedTypeKeyword::None,
+            Context.getQualifiedTemplateName(
+                Qualifier, /*TemplateKeyword=*/!IsFirst, TemplateName(CTD)),
+            Args, /*CanonicalArgs=*/{}, Context.getCanonicalTagType(RD));
+      } else {
+        T = Context.getTagType(ElaboratedTypeKeyword::None, Qualifier, TD,
+                               /*OwnsTag=*/false);
+      }
+      Qualifier = NestedNameSpecifier(T.getTypePtr());
+    } else {
+      Qualifier =
+          NestedNameSpecifier(Context, cast<NamespaceDecl>(CurD), Qualifier);
+    }
+    IsFirst = false;
   }
+  if (!Qualifier)
+    return "";
 
-  // Go over name-specifiers in reverse order to create necessary qualification,
-  // since we stored inner-most parent first.
   std::string Result;
   llvm::raw_string_ostream OS(Result);
-  for (const auto *Parent : llvm::reverse(Parents)) {
-    if (Parent != *Parents.rbegin() && Parent->isDependent() &&
-        Parent->getAsRecordDecl() &&
-        Parent->getAsRecordDecl()->getDescribedClassTemplate())
-      OS << "template ";
-    Parent->print(OS, Context.getPrintingPolicy());
-  }
+  Qualifier.print(OS, Context.getPrintingPolicy());
   return OS.str();
 }
 
@@ -187,6 +211,7 @@ std::string printQualifiedName(const NamedDecl &ND) {
   // include them, but at query time it's hard to find all the inline
   // namespaces to query: the preamble doesn't have a dedicated list.
   Policy.SuppressUnwrittenScope = true;
+  Policy.SuppressScope = true;
   // (unnamed struct), not (unnamed struct at /path/to/foo.cc:42:1).
   // In clangd, context is usually available and paths are mostly noise.
   Policy.AnonymousTagLocations = false;
@@ -213,8 +238,7 @@ std::string printUsingNamespaceName(const ASTContext &Ctx,
   std::string Name;
   llvm::raw_string_ostream Out(Name);
 
-  if (auto *Qual = D.getQualifier())
-    Qual->print(Out, PP);
+  D.getQualifier().print(Out, PP);
   D.getNominatedNamespaceAsWritten()->printName(Out);
   return Out.str();
 }
@@ -229,8 +253,7 @@ std::string printName(const ASTContext &Ctx, const NamedDecl &ND) {
   // Handle 'using namespace'. They all have the same name - <using-directive>.
   if (auto *UD = llvm::dyn_cast<UsingDirectiveDecl>(&ND)) {
     Out << "using namespace ";
-    if (auto *Qual = UD->getQualifier())
-      Qual->print(Out, PP);
+    UD->getQualifier().print(Out, PP);
     UD->getNominatedNamespaceAsWritten()->printName(Out);
     return Out.str();
   }
@@ -250,8 +273,7 @@ std::string printName(const ASTContext &Ctx, const NamedDecl &ND) {
   }
 
   // Print nested name qualifier if it was written in the source code.
-  if (auto *Qualifier = getQualifierLoc(ND).getNestedNameSpecifier())
-    Qualifier->print(Out, PP);
+  getQualifierLoc(ND).getNestedNameSpecifier().print(Out, PP);
   // Print the name itself.
   ND.getDeclName().print(Out, PP);
   // Print template arguments.
@@ -391,12 +413,13 @@ preferredIncludeDirective(llvm::StringRef FileName, const LangOptions &LangOpts,
 }
 
 std::string printType(const QualType QT, const DeclContext &CurContext,
-                      const llvm::StringRef Placeholder) {
+                      const llvm::StringRef Placeholder, bool FullyQualify) {
   std::string Result;
   llvm::raw_string_ostream OS(Result);
   PrintingPolicy PP(CurContext.getParentASTContext().getPrintingPolicy());
   PP.SuppressTagKeyword = true;
   PP.SuppressUnwrittenScope = true;
+  PP.FullyQualifiedName = FullyQualify;
 
   class PrintCB : public PrintingCallbacks {
   public:
@@ -439,6 +462,7 @@ QualType declaredType(const TypeDecl *D) {
   if (const auto *CTSD = llvm::dyn_cast<ClassTemplateSpecializationDecl>(D))
     if (const auto *Args = CTSD->getTemplateArgsAsWritten())
       return Context.getTemplateSpecializationType(
+          ElaboratedTypeKeyword::None,
           TemplateName(CTSD->getSpecializedTemplate()), Args->arguments(),
           /*CanonicalArgs=*/{});
   return Context.getTypeDeclType(D);
@@ -664,13 +688,10 @@ std::string getQualification(ASTContext &Context,
   auto VisibleNamespaceDecls =
       getUsingNamespaceDirectives(DestContext, InsertionPoint);
   return getQualification(
-      Context, DestContext, ND->getDeclContext(),
-      [&](NestedNameSpecifier *NNS) {
-        const NamespaceDecl *NS =
-            dyn_cast_if_present<NamespaceDecl>(NNS->getAsNamespace());
-        if (!NS)
+      Context, DestContext, ND->getDeclContext(), [&](const Decl *D) {
+        if (D->getKind() != Decl::Namespace)
           return false;
-        NS = NS->getCanonicalDecl();
+        const auto *NS = cast<NamespaceDecl>(D)->getCanonicalDecl();
         return llvm::any_of(VisibleNamespaceDecls,
                             [NS](const NamespaceDecl *NSD) {
                               return NSD->getCanonicalDecl() == NS;
@@ -687,12 +708,11 @@ std::string getQualification(ASTContext &Context,
     (void)NS;
   }
   return getQualification(
-      Context, DestContext, ND->getDeclContext(),
-      [&](NestedNameSpecifier *NNS) {
+      Context, DestContext, ND->getDeclContext(), [&](const Decl *D) {
         return llvm::any_of(VisibleNamespaces, [&](llvm::StringRef Namespace) {
           std::string NS;
           llvm::raw_string_ostream OS(NS);
-          NNS->print(OS, Context.getPrintingPolicy());
+          D->print(OS, Context.getPrintingPolicy());
           return OS.str() == Namespace;
         });
       });
@@ -965,7 +985,7 @@ resolveForwardingParameters(const FunctionDecl *D, unsigned MaxDepth) {
     // Recurse on pack parameters
     size_t Depth = 0;
     const FunctionDecl *CurrentFunction = D;
-    llvm::SmallSet<const FunctionTemplateDecl *, 4> SeenTemplates;
+    llvm::SmallPtrSet<const FunctionTemplateDecl *, 4> SeenTemplates;
     if (const auto *Template = D->getPrimaryTemplate()) {
       SeenTemplates.insert(Template);
     }
diff --git a/clang-tools-extra/clangd/AST.h b/clang-tools-extra/clangd/AST.h
index fb0722d697cd0..1538d12172593 100644
--- a/clang-tools-extra/clangd/AST.h
+++ b/clang-tools-extra/clangd/AST.h
@@ -135,7 +135,8 @@ preferredIncludeDirective(llvm::StringRef FileName, const LangOptions &LangOpts,
 /// Returns a QualType as string. The result doesn't contain unwritten scopes
 /// like anonymous/inline namespace.
 std::string printType(const QualType QT, const DeclContext &CurContext,
-                      llvm::StringRef Placeholder = "");
+                      llvm::StringRef Placeholder = "",
+                      bool FullyQualify = false);
 
 /// Indicates if \p D is a template instantiation implicitly generated by the
 /// compiler, e.g.
diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index a1e9da41b4b32..fb3f05329be21 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -6,7 +6,7 @@ add_subdirectory(support)
 
 # Configure the Features.inc file.
 if (NOT DEFINED CLANGD_BUILD_XPC)
-  if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  if("${CMAKE_SYSTEM_NAME}" MATCHES "Darwin")
     set(CLANGD_BUILD_XPC_DEFAULT ON)
   else ()
     set(CLANGD_BUILD_XPC_DEFAULT OFF)
@@ -108,6 +108,7 @@ add_clang_library(clangDaemon STATIC
   SemanticHighlighting.cpp
   SemanticSelection.cpp
   SourceCode.cpp
+  SymbolDocumentation.cpp
   SystemIncludeExtractor.cpp
   TidyProvider.cpp
   TUScheduler.cpp
@@ -192,7 +193,7 @@ if(CLANGD_TIDY_CHECKS)
 endif()
 
 add_subdirectory(refactor/tweaks)
-if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
+if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
   # FIXME: Make fuzzer not use linux-specific APIs, build it everywhere.
   add_subdirectory(fuzzer)
 endif()
diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp
index 9c17b4ca9b706..16559e4a35155 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -1466,19 +1466,15 @@ bool allowIndex(CodeCompletionContext &CC) {
   auto Scope = CC.getCXXScopeSpecifier();
   if (!Scope)
     return true;
-  NestedNameSpecifier *NameSpec = (*Scope)->getScopeRep();
-  if (!NameSpec)
-    return true;
   // We only query the index when qualifier is a namespace.
   // If it's a class, we rely solely on sema completions.
-  switch (NameSpec->getKind()) {
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Namespace:
+  switch ((*Scope)->getScopeRep().getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::Namespace:
     return true;
-  case NestedNameSpecifier::Super:
-  case NestedNameSpecifier::TypeSpec:
-  // Unresolved inside a template.
-  case NestedNameSpecifier::Identifier:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+  case NestedNameSpecifier::Kind::Type:
     return false;
   }
   llvm_unreachable("invalid NestedNameSpecifier kind");
@@ -2434,6 +2430,9 @@ CompletionItem CodeCompletion::render(const CodeCompleteOptions &Opts) const {
 }
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const CodeCompletion &C) {
+  OS << "Signature: " << "\"" << C.Signature << "\", "
+     << "SnippetSuffix: " << "\"" << C.SnippetSuffix << "\""
+     << ", Rendered:";
   // For now just lean on CompletionItem.
   return OS << C.render(CodeCompleteOptions());
 }
diff --git a/clang-tools-extra/clangd/CodeCompletionStrings.cpp b/clang-tools-extra/clangd/CodeCompletionStrings.cpp
index 9b4442b0bb76f..d6579640cb0fb 100644
--- a/clang-tools-extra/clangd/CodeCompletionStrings.cpp
+++ b/clang-tools-extra/clangd/CodeCompletionStrings.cpp
@@ -7,13 +7,18 @@
 //===----------------------------------------------------------------------===//
 
 #include "CodeCompletionStrings.h"
+#include "Config.h"
+#include "SymbolDocumentation.h"
 #include "clang-c/Index.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Comment.h"
+#include "clang/AST/Decl.h"
 #include "clang/AST/RawCommentList.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/JSON.h"
+#include "llvm/Support/raw_ostream.h"
 #include <limits>
 #include <utility>
 
@@ -100,16 +105,51 @@ std::string getDeclComment(const ASTContext &Ctx, const NamedDecl &Decl) {
     // the comments for namespaces.
     return "";
   }
-  const RawComment *RC = getCompletionComment(Ctx, &Decl);
-  if (!RC)
-    return "";
-  // Sanity check that the comment does not come from the PCH. We choose to not
-  // write them into PCH, because they are racy and slow to load.
-  assert(!Ctx.getSourceManager().isLoadedSourceLocation(RC->getBeginLoc()));
-  std::string Doc =
-      RC->getFormattedText(Ctx.getSourceManager(), Ctx.getDiagnostics());
-  if (!looksLikeDocComment(Doc))
-    return "";
+
+  const RawComment *RC = nullptr;
+  const Config &Cfg = Config::current();
+
+  std::string Doc;
+
+  if (Cfg.Documentation.CommentFormat == Config::CommentFormatPolicy::Doxygen &&
+      isa<ParmVarDecl>(Decl)) {
+    // Parameters are documented in their declaration context (function or
+    // template function).
+    const NamedDecl *ND = dyn_cast<NamedDecl>(Decl.getDeclContext());
+    if (!ND)
+      return "";
+
+    RC = getCompletionComment(Ctx, ND);
+    if (!RC)
+      return "";
+
+    // Sanity check that the comment does not come from the PCH. We choose to
+    // not write them into PCH, because they are racy and slow to load.
+    assert(!Ctx.getSourceManager().isLoadedSourceLocation(RC->getBeginLoc()));
+
+    comments::FullComment *FC = RC->parse(Ctx, /*PP=*/nullptr, ND);
+    if (!FC)
+      return "";
+
+    SymbolDocCommentVisitor V(FC, Ctx.getLangOpts().CommentOpts);
+    std::string RawDoc;
+    llvm::raw_string_ostream OS(RawDoc);
+
+    V.parameterDocToString(dyn_cast<ParmVarDecl>(&Decl)->getName(), OS);
+
+    Doc = StringRef(RawDoc).trim().str();
+  } else {
+    RC = getCompletionComment(Ctx, &Decl);
+    if (!RC)
+      return "";
+    // Sanity check that the comment does not come from the PCH. We choose to
+    // not write them into PCH, because they are racy and slow to load.
+    assert(!Ctx.getSourceManager().isLoadedSourceLocation(RC->getBeginLoc()));
+    Doc = RC->getFormattedText(Ctx.getSourceManager(), Ctx.getDiagnostics());
+    if (!looksLikeDocComment(Doc))
+      return "";
+  }
+
   // Clang requires source to be UTF-8, but doesn't enforce this in comments.
   if (!llvm::json::isUTF8(Doc))
     Doc = llvm::json::fixUTF8(Doc);
diff --git a/clang-tools-extra/clangd/ConfigFragment.h b/clang-tools-extra/clangd/ConfigFragment.h
index a6a7cd53fb9bf..0f11f37e14698 100644
--- a/clang-tools-extra/clangd/ConfigFragment.h
+++ b/clang-tools-extra/clangd/ConfigFragment.h
@@ -315,7 +315,7 @@ struct Fragment {
     /// AngledHeaders (i.e. a header matches a regex in both QuotedHeaders and
     /// AngledHeaders), system headers use <> and non-system headers use "".
     /// These can match any suffix of the header file in question.
-    /// Matching is performed against the header text, not its absolute path
+    /// Matching is performed against the absolute path of the header
     /// within the project.
     std::vector<Located<std::string>> QuotedHeaders;
     /// List of regexes for headers that should always be included with a
@@ -323,7 +323,7 @@ struct Fragment {
     /// AngledHeaders (i.e. a header matches a regex in both QuotedHeaders and
     /// AngledHeaders), system headers use <> and non-system headers use "".
     /// These can match any suffix of the header file in question.
-    /// Matching is performed against the header text, not its absolute path
+    /// Matching is performed against the absolute path of the header
     /// within the project.
     std::vector<Located<std::string>> AngledHeaders;
   };
diff --git a/clang-tools-extra/clangd/DumpAST.cpp b/clang-tools-extra/clangd/DumpAST.cpp
index c6075e75e9a6b..9a8d41d870929 100644
--- a/clang-tools-extra/clangd/DumpAST.cpp
+++ b/clang-tools-extra/clangd/DumpAST.cpp
@@ -147,17 +147,17 @@ class DumpVisitor : public RecursiveASTVisitor<DumpVisitor> {
     }
     llvm_unreachable("Unhandled ArgKind enum");
   }
-  std::string getKind(const NestedNameSpecifierLoc &NNSL) {
-    assert(NNSL.getNestedNameSpecifier());
-    switch (NNSL.getNestedNameSpecifier()->getKind()) {
+  std::string getKind(NestedNameSpecifierLoc NNSL) {
+    switch (NNSL.getNestedNameSpecifier().getKind()) {
+    case NestedNameSpecifier::Kind::Null:
+      llvm_unreachable("unexpected null nested name specifier");
 #define NNS_KIND(X)                                                            \
-  case NestedNameSpecifier::X:                                                 \
+  case NestedNameSpecifier::Kind::X:                                           \
     return #X
-      NNS_KIND(Identifier);
       NNS_KIND(Namespace);
-      NNS_KIND(TypeSpec);
+      NNS_KIND(Type);
       NNS_KIND(Global);
-      NNS_KIND(Super);
+      NNS_KIND(MicrosoftSuper);
 #undef NNS_KIND
     }
     llvm_unreachable("Unhandled SpecifierKind enum");
@@ -261,7 +261,7 @@ class DumpVisitor : public RecursiveASTVisitor<DumpVisitor> {
       return TL.getType().getLocalQualifiers().getAsString(
           Ctx.getPrintingPolicy());
     if (const auto *TT = dyn_cast<TagType>(TL.getTypePtr()))
-      return getDetail(TT->getDecl());
+      return getDetail(TT->getOriginalDecl());
     if (const auto *DT = dyn_cast<DeducedType>(TL.getTypePtr()))
       if (DT->isDeduced())
         return DT->getDeducedType().getAsString(Ctx.getPrintingPolicy());
@@ -273,16 +273,11 @@ class DumpVisitor : public RecursiveASTVisitor<DumpVisitor> {
       return getDetail(TT->getDecl());
     return "";
   }
-  std::string getDetail(const NestedNameSpecifierLoc &NNSL) {
-    const auto &NNS = *NNSL.getNestedNameSpecifier();
-    switch (NNS.getKind()) {
-    case NestedNameSpecifier::Identifier:
-      return NNS.getAsIdentifier()->getName().str() + "::";
-    case NestedNameSpecifier::Namespace:
-      return NNS.getAsNamespace()->getNameAsString() + "::";
-    default:
+  std::string getDetail(NestedNameSpecifierLoc NNSL) {
+    NestedNameSpecifier NNS = NNSL.getNestedNameSpecifier();
+    if (NNS.getKind() != NestedNameSpecifier::Kind::Namespace)
       return "";
-    }
+    return NNS.getAsNamespaceAndPrefix().Namespace->getNameAsString() + "::";
   }
   std::string getDetail(const CXXCtorInitializer *CCI) {
     if (FieldDecl *FD = CCI->getAnyMember())
@@ -346,8 +341,10 @@ class DumpVisitor : public RecursiveASTVisitor<DumpVisitor> {
     return !D || isInjectedClassName(D) ||
            traverseNode("declaration", D, [&] { Base::TraverseDecl(D); });
   }
-  bool TraverseTypeLoc(TypeLoc TL) {
-    return !TL || traverseNode("type", TL, [&] { Base::TraverseTypeLoc(TL); });
+  bool TraverseTypeLoc(TypeLoc TL, bool TraverseQualifier = true) {
+    return !TL || traverseNode("type", TL, [&] {
+      Base::TraverseTypeLoc(TL, TraverseQualifier);
+    });
   }
   bool TraverseTemplateName(const TemplateName &TN) {
     return traverseNode("template name", TN,
@@ -389,11 +386,11 @@ class DumpVisitor : public RecursiveASTVisitor<DumpVisitor> {
   // This means we'd never see 'int' in 'const int'! Work around that here.
   // (The reason for the behavior is to avoid traversing the nested Type twice,
   // but we ignore TraverseType anyway).
-  bool TraverseQualifiedTypeLoc(QualifiedTypeLoc QTL) {
+  bool TraverseQualifiedTypeLoc(QualifiedTypeLoc QTL, bool TraverseQualifier) {
     return TraverseTypeLoc(QTL.getUnqualifiedLoc());
   }
   // Uninteresting parts of the AST that don't have locations within them.
-  bool TraverseNestedNameSpecifier(NestedNameSpecifier *) { return true; }
+  bool TraverseNestedNameSpecifier(NestedNameSpecifier) { return true; }
   bool TraverseType(QualType) { return true; }
 
   // OpaqueValueExpr blocks traversal, we must explicitly traverse it.
@@ -420,7 +417,7 @@ ASTNode dumpAST(const DynTypedNode &N, const syntax::TokenBuffer &Tokens,
     V.TraverseNestedNameSpecifierLoc(
         *const_cast<NestedNameSpecifierLoc *>(NNSL));
   else if (const auto *NNS = N.get<NestedNameSpecifier>())
-    V.TraverseNestedNameSpecifier(const_cast<NestedNameSpecifier *>(NNS));
+    V.TraverseNestedNameSpecifier(*NNS);
   else if (const auto *TL = N.get<TypeLoc>())
     V.TraverseTypeLoc(*const_cast<TypeLoc *>(TL));
   else if (const auto *QT = N.get<QualType>())
diff --git a/clang-tools-extra/clangd/FindTarget.cpp b/clang-tools-extra/clangd/FindTarget.cpp
index b1089577ba819..32018d1bf3a84 100644
--- a/clang-tools-extra/clangd/FindTarget.cpp
+++ b/clang-tools-extra/clangd/FindTarget.cpp
@@ -366,19 +366,11 @@ struct TargetFinder {
       Visitor(TargetFinder &Outer, RelSet Flags) : Outer(Outer), Flags(Flags) {}
 
       void VisitTagType(const TagType *TT) {
-        Outer.add(TT->getAsTagDecl(), Flags);
-      }
-
-      void VisitElaboratedType(const ElaboratedType *ET) {
-        Outer.add(ET->desugar(), Flags);
+        Outer.add(cast<TagType>(TT)->getOriginalDecl(), Flags);
       }
 
       void VisitUsingType(const UsingType *ET) {
-        Outer.add(ET->getFoundDecl(), Flags);
-      }
-
-      void VisitInjectedClassNameType(const InjectedClassNameType *ICNT) {
-        Outer.add(ICNT->getDecl(), Flags);
+        Outer.add(ET->getDecl(), Flags);
       }
 
       void VisitDecltypeType(const DecltypeType *DTT) {
@@ -483,30 +475,27 @@ struct TargetFinder {
     Visitor(*this, Flags).Visit(T.getTypePtr());
   }
 
-  void add(const NestedNameSpecifier *NNS, RelSet Flags) {
+  void add(NestedNameSpecifier NNS, RelSet Flags) {
     if (!NNS)
       return;
-    debug(*NNS, Flags);
-    switch (NNS->getKind()) {
-    case NestedNameSpecifier::Namespace:
-      add(NNS->getAsNamespace(), Flags);
-      return;
-    case NestedNameSpecifier::Identifier:
-      if (Resolver) {
-        add(Resolver->resolveNestedNameSpecifierToType(NNS), Flags);
-      }
+    debug(NNS, Flags);
+    switch (NNS.getKind()) {
+    case NestedNameSpecifier::Kind::Namespace:
+      add(NNS.getAsNamespaceAndPrefix().Namespace, Flags);
       return;
-    case NestedNameSpecifier::TypeSpec:
-      add(QualType(NNS->getAsType(), 0), Flags);
+    case NestedNameSpecifier::Kind::Type:
+      add(QualType(NNS.getAsType(), 0), Flags);
       return;
-    case NestedNameSpecifier::Global:
+    case NestedNameSpecifier::Kind::Global:
       // This should be TUDecl, but we can't get a pointer to it!
       return;
-    case NestedNameSpecifier::Super:
-      add(NNS->getAsRecordDecl(), Flags);
+    case NestedNameSpecifier::Kind::MicrosoftSuper:
+      add(NNS.getAsMicrosoftSuper(), Flags);
       return;
+    case NestedNameSpecifier::Kind::Null:
+      llvm_unreachable("unexpected null nested name specifier");
     }
-    llvm_unreachable("unhandled NestedNameSpecifier::SpecifierKind");
+    llvm_unreachable("unhandled NestedNameSpecifier::Kind");
   }
 
   void add(const CXXCtorInitializer *CCI, RelSet Flags) {
@@ -555,7 +544,7 @@ allTargetDecls(const DynTypedNode &N, const HeuristicResolver *Resolver) {
   else if (const NestedNameSpecifierLoc *NNSL = N.get<NestedNameSpecifierLoc>())
     Finder.add(NNSL->getNestedNameSpecifier(), Flags);
   else if (const NestedNameSpecifier *NNS = N.get<NestedNameSpecifier>())
-    Finder.add(NNS, Flags);
+    Finder.add(*NNS, Flags);
   else if (const TypeLoc *TL = N.get<TypeLoc>())
     Finder.add(TL->getType(), Flags);
   else if (const QualType *QT = N.get<QualType>())
@@ -861,32 +850,25 @@ refInTypeLoc(TypeLoc L, const HeuristicResolver *Resolver) {
     const HeuristicResolver *Resolver;
     llvm::SmallVector<ReferenceLoc> Refs;
 
-    void VisitElaboratedTypeLoc(ElaboratedTypeLoc L) {
-      // We only know about qualifier, rest if filled by inner locations.
-      size_t InitialSize = Refs.size();
-      Visit(L.getNamedTypeLoc().getUnqualifiedLoc());
-      size_t NewSize = Refs.size();
-      // Add qualifier for the newly-added refs.
-      for (unsigned I = InitialSize; I < NewSize; ++I) {
-        ReferenceLoc *Ref = &Refs[I];
-        // Fill in the qualifier.
-        assert(!Ref->Qualifier.hasQualifier() && "qualifier already set");
-        Ref->Qualifier = L.getQualifierLoc();
-      }
+    void VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc L) {
+      Refs.push_back(ReferenceLoc{L.getQualifierLoc(),
+                                  L.getLocalSourceRange().getBegin(),
+                                  /*IsDecl=*/false,
+                                  {L.getDecl()}});
     }
 
     void VisitUsingTypeLoc(UsingTypeLoc L) {
-      Refs.push_back(ReferenceLoc{NestedNameSpecifierLoc(),
+      Refs.push_back(ReferenceLoc{L.getQualifierLoc(),
                                   L.getLocalSourceRange().getBegin(),
                                   /*IsDecl=*/false,
-                                  {L.getFoundDecl()}});
+                                  {L.getDecl()}});
     }
 
     void VisitTagTypeLoc(TagTypeLoc L) {
-      Refs.push_back(ReferenceLoc{NestedNameSpecifierLoc(),
+      Refs.push_back(ReferenceLoc{L.getQualifierLoc(),
                                   L.getNameLoc(),
                                   /*IsDecl=*/false,
-                                  {L.getDecl()}});
+                                  {L.getOriginalDecl()}});
     }
 
     void VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc L) {
@@ -906,25 +888,18 @@ refInTypeLoc(TypeLoc L, const HeuristicResolver *Resolver) {
       //    2. 'vector<int>' with mask 'Underlying'.
       //  we want to return only #1 in this case.
       Refs.push_back(ReferenceLoc{
-          NestedNameSpecifierLoc(), L.getTemplateNameLoc(), /*IsDecl=*/false,
+          L.getQualifierLoc(), L.getTemplateNameLoc(), /*IsDecl=*/false,
           explicitReferenceTargets(DynTypedNode::create(L.getType()),
                                    DeclRelation::Alias, Resolver)});
     }
     void VisitDeducedTemplateSpecializationTypeLoc(
         DeducedTemplateSpecializationTypeLoc L) {
       Refs.push_back(ReferenceLoc{
-          NestedNameSpecifierLoc(), L.getNameLoc(), /*IsDecl=*/false,
+          L.getQualifierLoc(), L.getNameLoc(), /*IsDecl=*/false,
           explicitReferenceTargets(DynTypedNode::create(L.getType()),
                                    DeclRelation::Alias, Resolver)});
     }
 
-    void VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc TL) {
-      Refs.push_back(ReferenceLoc{NestedNameSpecifierLoc(),
-                                  TL.getNameLoc(),
-                                  /*IsDecl=*/false,
-                                  {TL.getDecl()}});
-    }
-
     void VisitDependentTemplateSpecializationTypeLoc(
         DependentTemplateSpecializationTypeLoc L) {
       Refs.push_back(
@@ -943,12 +918,12 @@ refInTypeLoc(TypeLoc L, const HeuristicResolver *Resolver) {
     }
 
     void VisitTypedefTypeLoc(TypedefTypeLoc L) {
-      if (shouldSkipTypedef(L.getTypedefNameDecl()))
+      if (shouldSkipTypedef(L.getDecl()))
         return;
-      Refs.push_back(ReferenceLoc{NestedNameSpecifierLoc(),
+      Refs.push_back(ReferenceLoc{L.getQualifierLoc(),
                                   L.getNameLoc(),
                                   /*IsDecl=*/false,
-                                  {L.getTypedefNameDecl()}});
+                                  {L.getDecl()}});
     }
 
     void VisitObjCInterfaceTypeLoc(ObjCInterfaceTypeLoc L) {
@@ -980,17 +955,6 @@ class ExplicitReferenceCollector
     return true;
   }
 
-  bool TraverseElaboratedTypeLoc(ElaboratedTypeLoc L) {
-    // ElaboratedTypeLoc will reports information for its inner type loc.
-    // Otherwise we loose information about inner types loc's qualifier.
-    TypeLoc Inner = L.getNamedTypeLoc().getUnqualifiedLoc();
-    if (L.getBeginLoc() == Inner.getBeginLoc())
-      return RecursiveASTVisitor::TraverseTypeLoc(Inner);
-    else
-      TypeLocsToSkip.insert(Inner.getBeginLoc());
-    return RecursiveASTVisitor::TraverseElaboratedTypeLoc(L);
-  }
-
   bool VisitStmt(Stmt *S) {
     visitNode(DynTypedNode::create(*S));
     return true;
@@ -1051,7 +1015,7 @@ class ExplicitReferenceCollector
       return true;
     visitNode(DynTypedNode::create(L));
     // Inner type is missing information about its qualifier, skip it.
-    if (auto TL = L.getTypeLoc())
+    if (auto TL = L.getAsTypeLoc())
       TypeLocsToSkip.insert(TL.getBeginLoc());
     return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(L);
   }
@@ -1092,12 +1056,21 @@ class ExplicitReferenceCollector
     if (auto *S = N.get<Stmt>())
       return refInStmt(S, Resolver);
     if (auto *NNSL = N.get<NestedNameSpecifierLoc>()) {
-      // (!) 'DeclRelation::Alias' ensures we do not loose namespace aliases.
-      return {ReferenceLoc{
-          NNSL->getPrefix(), NNSL->getLocalBeginLoc(), false,
-          explicitReferenceTargets(
-              DynTypedNode::create(*NNSL->getNestedNameSpecifier()),
-              DeclRelation::Alias, Resolver)}};
+      // (!) 'DeclRelation::Alias' ensures we do not lose namespace aliases.
+      NestedNameSpecifierLoc Qualifier;
+      SourceLocation NameLoc;
+      if (auto TL = NNSL->getAsTypeLoc()) {
+        Qualifier = TL.getPrefix();
+        NameLoc = TL.getNonPrefixBeginLoc();
+      } else {
+        Qualifier = NNSL->getAsNamespaceAndPrefix().Prefix;
+        NameLoc = NNSL->getLocalBeginLoc();
+      }
+      return {
+          ReferenceLoc{Qualifier, NameLoc, false,
+                       explicitReferenceTargets(
+                           DynTypedNode::create(NNSL->getNestedNameSpecifier()),
+                           DeclRelation::Alias, Resolver)}};
     }
     if (const TypeLoc *TL = N.get<TypeLoc>())
       return refInTypeLoc(*TL, Resolver);
@@ -1210,8 +1183,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, ReferenceLoc R) {
   OS << "}";
   if (R.Qualifier) {
     OS << ", qualifier = '";
-    R.Qualifier.getNestedNameSpecifier()->print(OS,
-                                                PrintingPolicy(LangOptions()));
+    R.Qualifier.getNestedNameSpecifier().print(OS,
+                                               PrintingPolicy(LangOptions()));
     OS << "'";
   }
   if (R.IsDecl)
diff --git a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
index 7c0eb9651feaa..c6afd0bc07cbd 100644
--- a/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
+++ b/clang-tools-extra/clangd/GlobalCompilationDatabase.cpp
@@ -833,6 +833,10 @@ bool OverlayCDB::setCompileCommand(PathRef File,
 std::unique_ptr<ProjectModules>
 OverlayCDB::getProjectModules(PathRef File) const {
   auto MDB = DelegatingCDB::getProjectModules(File);
+  if (!MDB) {
+    log("Failed to get compilation Database for {0}", File);
+    return {};
+  }
   MDB->setCommandMangler([&Mangler = Mangler](tooling::CompileCommand &Command,
                                               PathRef CommandPath) {
     Mangler(Command, CommandPath);
diff --git a/clang-tools-extra/clangd/Headers.cpp b/clang-tools-extra/clangd/Headers.cpp
index 87fd261b906e6..b9d67cc6a1602 100644
--- a/clang-tools-extra/clangd/Headers.cpp
+++ b/clang-tools-extra/clangd/Headers.cpp
@@ -304,16 +304,17 @@ IncludeInserter::calculateIncludePath(const HeaderFile &InsertedHeader,
   // FIXME: should we allow (some limited number of) "../header.h"?
   if (llvm::sys::path::is_absolute(Suggested))
     return std::nullopt;
+  auto HeaderPath = llvm::sys::path::convert_to_slash(InsertedHeader.File);
   bool IsAngled = false;
   for (auto &Filter : AngledHeaders) {
-    if (Filter(Suggested)) {
+    if (Filter(HeaderPath)) {
       IsAngled = true;
       break;
     }
   }
   bool IsQuoted = false;
   for (auto &Filter : QuotedHeaders) {
-    if (Filter(Suggested)) {
+    if (Filter(HeaderPath)) {
       IsQuoted = true;
       break;
     }
@@ -324,7 +325,7 @@ IncludeInserter::calculateIncludePath(const HeaderFile &InsertedHeader,
     if (IsAngled && IsQuoted) {
       elog("Header '{0}' matches both quoted and angled regexes, default will "
            "be used.",
-           Suggested);
+           HeaderPath);
     }
     IsAngled = IsAngledByDefault;
   }
diff --git a/clang-tools-extra/clangd/Hover.cpp b/clang-tools-extra/clangd/Hover.cpp
index 1e0718d673260..a7cf45c632827 100644
--- a/clang-tools-extra/clangd/Hover.cpp
+++ b/clang-tools-extra/clangd/Hover.cpp
@@ -18,6 +18,7 @@
 #include "Protocol.h"
 #include "Selection.h"
 #include "SourceCode.h"
+#include "SymbolDocumentation.h"
 #include "clang-include-cleaner/Analysis.h"
 #include "clang-include-cleaner/IncludeSpeller.h"
 #include "clang-include-cleaner/Types.h"
@@ -41,6 +42,7 @@
 #include "clang/AST/Type.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/LLVM.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/Specifiers.h"
@@ -170,13 +172,14 @@ HoverInfo::PrintedType printType(QualType QT, ASTContext &ASTCtx,
     QT = QT->castAs<DecltypeType>()->getUnderlyingType();
   HoverInfo::PrintedType Result;
   llvm::raw_string_ostream OS(Result.Type);
-  // Special case: if the outer type is a tag type without qualifiers, then
-  // include the tag for extra clarity.
-  // This isn't very idiomatic, so don't attempt it for complex cases, including
-  // pointers/references, template specializations, etc.
+  // Special case: if the outer type is a canonical tag type, then include the
+  // tag for extra clarity. This isn't very idiomatic, so don't attempt it for
+  // complex cases, including pointers/references, template specializations,
+  // etc.
   if (!QT.isNull() && !QT.hasQualifiers() && PP.SuppressTagKeyword) {
-    if (auto *TT = llvm::dyn_cast<TagType>(QT.getTypePtr()))
-      OS << TT->getDecl()->getKindName() << " ";
+    if (auto *TT = llvm::dyn_cast<TagType>(QT.getTypePtr());
+        TT && TT->isCanonicalUnqualified())
+      OS << TT->getOriginalDecl()->getKindName() << " ";
   }
   QT.print(OS, PP);
 
@@ -452,7 +455,7 @@ std::optional<std::string> printExprValue(const Expr *E,
     // Compare to int64_t to avoid bit-width match requirements.
     int64_t Val = Constant.Val.getInt().getExtValue();
     for (const EnumConstantDecl *ECD :
-         T->castAs<EnumType>()->getDecl()->enumerators())
+         T->castAs<EnumType>()->getOriginalDecl()->enumerators())
       if (ECD->getInitVal() == Val)
         return llvm::formatv("{0} ({1})", ECD->getNameAsString(),
                              printHex(Constant.Val.getInt()))
@@ -627,6 +630,9 @@ HoverInfo getHoverContents(const NamedDecl *D, const PrintingPolicy &PP,
   HI.Name = printName(Ctx, *D);
   const auto *CommentD = getDeclForComment(D);
   HI.Documentation = getDeclComment(Ctx, *CommentD);
+  // save the language options to be able to create the comment::CommandTraits
+  // to parse the documentation
+  HI.CommentOpts = D->getASTContext().getLangOpts().CommentOpts;
   enhanceFromIndex(HI, *CommentD, Index);
   if (HI.Documentation.empty())
     HI.Documentation = synthesizeDocumentation(D);
@@ -967,10 +973,11 @@ void addLayoutInfo(const NamedDecl &ND, HoverInfo &HI) {
 
   const auto &Ctx = ND.getASTContext();
   if (auto *RD = llvm::dyn_cast<RecordDecl>(&ND)) {
-    if (auto Size = Ctx.getTypeSizeInCharsIfKnown(RD->getTypeForDecl()))
+    CanQualType RT = Ctx.getCanonicalTagType(RD);
+    if (auto Size = Ctx.getTypeSizeInCharsIfKnown(RT))
       HI.Size = Size->getQuantity() * 8;
     if (!RD->isDependentType() && RD->isCompleteDefinition())
-      HI.Align = Ctx.getTypeAlign(RD->getTypeForDecl());
+      HI.Align = Ctx.getTypeAlign(RT);
     return;
   }
 
@@ -1388,9 +1395,100 @@ static std::string formatOffset(uint64_t OffsetInBits) {
   return Offset;
 }
 
-markup::Document HoverInfo::present() const {
-  markup::Document Output;
+void HoverInfo::calleeArgInfoToMarkupParagraph(markup::Paragraph &P) const {
+  assert(CallPassType);
+  std::string Buffer;
+  llvm::raw_string_ostream OS(Buffer);
+  OS << "Passed ";
+  if (CallPassType->PassBy != HoverInfo::PassType::Value) {
+    OS << "by ";
+    if (CallPassType->PassBy == HoverInfo::PassType::ConstRef)
+      OS << "const ";
+    OS << "reference ";
+  }
+  if (CalleeArgInfo->Name)
+    OS << "as " << CalleeArgInfo->Name;
+  else if (CallPassType->PassBy == HoverInfo::PassType::Value)
+    OS << "by value";
+  if (CallPassType->Converted && CalleeArgInfo->Type)
+    OS << " (converted to " << CalleeArgInfo->Type->Type << ")";
+  P.appendText(OS.str());
+}
+
+void HoverInfo::usedSymbolNamesToMarkup(markup::Document &Output) const {
+  markup::Paragraph &P = Output.addParagraph();
+  P.appendText("provides ");
+
+  const std::vector<std::string>::size_type SymbolNamesLimit = 5;
+  auto Front = llvm::ArrayRef(UsedSymbolNames).take_front(SymbolNamesLimit);
+
+  llvm::interleave(
+      Front, [&](llvm::StringRef Sym) { P.appendCode(Sym); },
+      [&] { P.appendText(", "); });
+  if (UsedSymbolNames.size() > Front.size()) {
+    P.appendText(" and ");
+    P.appendText(std::to_string(UsedSymbolNames.size() - Front.size()));
+    P.appendText(" more");
+  }
+}
+
+void HoverInfo::providerToMarkupParagraph(markup::Document &Output) const {
+  markup::Paragraph &DI = Output.addParagraph();
+  DI.appendText("provided by");
+  DI.appendSpace();
+  DI.appendCode(Provider);
+}
+
+void HoverInfo::definitionScopeToMarkup(markup::Document &Output) const {
+  std::string Buffer;
+
+  // Append scope comment, dropping trailing "::".
+  // Note that we don't print anything for global namespace, to not annoy
+  // non-c++ projects or projects that are not making use of namespaces.
+  if (!LocalScope.empty()) {
+    // Container name, e.g. class, method, function.
+    // We might want to propagate some info about container type to print
+    // function foo, class X, method X::bar, etc.
+    Buffer += "// In " + llvm::StringRef(LocalScope).rtrim(':').str() + '\n';
+  } else if (NamespaceScope && !NamespaceScope->empty()) {
+    Buffer += "// In namespace " +
+              llvm::StringRef(*NamespaceScope).rtrim(':').str() + '\n';
+  }
+
+  if (!AccessSpecifier.empty()) {
+    Buffer += AccessSpecifier + ": ";
+  }
 
+  Buffer += Definition;
+
+  Output.addCodeBlock(Buffer, DefinitionLanguage);
+}
+
+void HoverInfo::valueToMarkupParagraph(markup::Paragraph &P) const {
+  P.appendText("Value = ");
+  P.appendCode(*Value);
+}
+
+void HoverInfo::offsetToMarkupParagraph(markup::Paragraph &P) const {
+  P.appendText("Offset: " + formatOffset(*Offset));
+}
+
+void HoverInfo::sizeToMarkupParagraph(markup::Paragraph &P) const {
+  P.appendText("Size: " + formatSize(*Size));
+  if (Padding && *Padding != 0) {
+    P.appendText(llvm::formatv(" (+{0} padding)", formatSize(*Padding)).str());
+  }
+  if (Align)
+    P.appendText(", alignment " + formatSize(*Align));
+}
+
+markup::Document HoverInfo::presentDoxygen() const {
+  // NOTE: this function is currently almost identical to presentDefault().
+  // This is to have a minimal change when introducing the doxygen parser.
+  // This function will be changed when rearranging the output for doxygen
+  // parsed documentation.
+
+  markup::Document Output;
   // Header contains a text of the form:
   // variable `var`
   //
@@ -1407,14 +1505,99 @@ markup::Document HoverInfo::present() const {
   if (Kind != index::SymbolKind::Unknown)
     Header.appendText(index::getSymbolKindString(Kind)).appendSpace();
   assert(!Name.empty() && "hover triggered on a nameless symbol");
+
   Header.appendCode(Name);
 
   if (!Provider.empty()) {
-    markup::Paragraph &DI = Output.addParagraph();
-    DI.appendText("provided by");
-    DI.appendSpace();
-    DI.appendCode(Provider);
+    providerToMarkupParagraph(Output);
+  }
+
+  // Put a linebreak after header to increase readability.
+  Output.addRuler();
+  // Print Types on their own lines to reduce chances of getting line-wrapped by
+  // editor, as they might be long.
+  if (ReturnType) {
+    // For functions we display signature in a list form, e.g.:
+    // → `x`
+    // Parameters:
+    // - `bool param1`
+    // - `int param2 = 5`
+    Output.addParagraph().appendText("→ ").appendCode(
+        llvm::to_string(*ReturnType));
+  }
+
+  SymbolDocCommentVisitor SymbolDoc(Documentation, CommentOpts);
+
+  if (Parameters && !Parameters->empty()) {
+    Output.addParagraph().appendText("Parameters:");
+    markup::BulletList &L = Output.addBulletList();
+    for (const auto &Param : *Parameters) {
+      markup::Paragraph &P = L.addItem().addParagraph();
+      P.appendCode(llvm::to_string(Param));
+
+      if (SymbolDoc.isParameterDocumented(llvm::to_string(Param.Name))) {
+        P.appendText(" -");
+        SymbolDoc.parameterDocToMarkup(llvm::to_string(Param.Name), P);
+      }
+    }
+  }
+  // Don't print Type after Parameters or ReturnType as this will just duplicate
+  // the information
+  if (Type && !ReturnType && !Parameters)
+    Output.addParagraph().appendText("Type: ").appendCode(
+        llvm::to_string(*Type));
+
+  if (Value) {
+    valueToMarkupParagraph(Output.addParagraph());
+  }
+
+  if (Offset)
+    offsetToMarkupParagraph(Output.addParagraph());
+  if (Size) {
+    sizeToMarkupParagraph(Output.addParagraph());
+  }
+
+  if (CalleeArgInfo) {
+    calleeArgInfoToMarkupParagraph(Output.addParagraph());
+  }
+
+  SymbolDoc.docToMarkup(Output);
+
+  if (!Definition.empty()) {
     Output.addRuler();
+    definitionScopeToMarkup(Output);
+  }
+
+  if (!UsedSymbolNames.empty()) {
+    Output.addRuler();
+    usedSymbolNamesToMarkup(Output);
+  }
+
+  return Output;
+}
+
+markup::Document HoverInfo::presentDefault() const {
+  markup::Document Output;
+  // Header contains a text of the form:
+  // variable `var`
+  //
+  // class `X`
+  //
+  // function `foo`
+  //
+  // expression
+  //
+  // Note that we are making use of a level-3 heading because VSCode renders
+  // level 1 and 2 headers in a huge font, see
+  // https://github.com/microsoft/vscode/issues/88417 for details.
+  markup::Paragraph &Header = Output.addHeading(3);
+  if (Kind != index::SymbolKind::Unknown)
+    Header.appendText(index::getSymbolKindString(Kind)).appendSpace();
+  assert(!Name.empty() && "hover triggered on a nameless symbol");
+  Header.appendCode(Name);
+
+  if (!Provider.empty()) {
+    providerToMarkupParagraph(Output);
   }
 
   // Put a linebreak after header to increase readability.
@@ -1445,41 +1628,17 @@ markup::Document HoverInfo::present() const {
         llvm::to_string(*Type));
 
   if (Value) {
-    markup::Paragraph &P = Output.addParagraph();
-    P.appendText("Value = ");
-    P.appendCode(*Value);
+    valueToMarkupParagraph(Output.addParagraph());
   }
 
   if (Offset)
-    Output.addParagraph().appendText("Offset: " + formatOffset(*Offset));
+    offsetToMarkupParagraph(Output.addParagraph());
   if (Size) {
-    auto &P = Output.addParagraph().appendText("Size: " + formatSize(*Size));
-    if (Padding && *Padding != 0) {
-      P.appendText(
-          llvm::formatv(" (+{0} padding)", formatSize(*Padding)).str());
-    }
-    if (Align)
-      P.appendText(", alignment " + formatSize(*Align));
+    sizeToMarkupParagraph(Output.addParagraph());
   }
 
   if (CalleeArgInfo) {
-    assert(CallPassType);
-    std::string Buffer;
-    llvm::raw_string_ostream OS(Buffer);
-    OS << "Passed ";
-    if (CallPassType->PassBy != HoverInfo::PassType::Value) {
-      OS << "by ";
-      if (CallPassType->PassBy == HoverInfo::PassType::ConstRef)
-        OS << "const ";
-      OS << "reference ";
-    }
-    if (CalleeArgInfo->Name)
-      OS << "as " << CalleeArgInfo->Name;
-    else if (CallPassType->PassBy == HoverInfo::PassType::Value)
-      OS << "by value";
-    if (CallPassType->Converted && CalleeArgInfo->Type)
-      OS << " (converted to " << CalleeArgInfo->Type->Type << ")";
-    Output.addParagraph().appendText(OS.str());
+    calleeArgInfoToMarkupParagraph(Output.addParagraph());
   }
 
   if (!Documentation.empty())
@@ -1487,49 +1646,12 @@ markup::Document HoverInfo::present() const {
 
   if (!Definition.empty()) {
     Output.addRuler();
-    std::string Buffer;
-
-    if (!Definition.empty()) {
-      // Append scope comment, dropping trailing "::".
-      // Note that we don't print anything for global namespace, to not annoy
-      // non-c++ projects or projects that are not making use of namespaces.
-      if (!LocalScope.empty()) {
-        // Container name, e.g. class, method, function.
-        // We might want to propagate some info about container type to print
-        // function foo, class X, method X::bar, etc.
-        Buffer +=
-            "// In " + llvm::StringRef(LocalScope).rtrim(':').str() + '\n';
-      } else if (NamespaceScope && !NamespaceScope->empty()) {
-        Buffer += "// In namespace " +
-                  llvm::StringRef(*NamespaceScope).rtrim(':').str() + '\n';
-      }
-
-      if (!AccessSpecifier.empty()) {
-        Buffer += AccessSpecifier + ": ";
-      }
-
-      Buffer += Definition;
-    }
-
-    Output.addCodeBlock(Buffer, DefinitionLanguage);
+    definitionScopeToMarkup(Output);
   }
 
   if (!UsedSymbolNames.empty()) {
     Output.addRuler();
-    markup::Paragraph &P = Output.addParagraph();
-    P.appendText("provides ");
-
-    const std::vector<std::string>::size_type SymbolNamesLimit = 5;
-    auto Front = llvm::ArrayRef(UsedSymbolNames).take_front(SymbolNamesLimit);
-
-    llvm::interleave(
-        Front, [&](llvm::StringRef Sym) { P.appendCode(Sym); },
-        [&] { P.appendText(", "); });
-    if (UsedSymbolNames.size() > Front.size()) {
-      P.appendText(" and ");
-      P.appendText(std::to_string(UsedSymbolNames.size() - Front.size()));
-      P.appendText(" more");
-    }
+    usedSymbolNamesToMarkup(Output);
   }
 
   return Output;
@@ -1538,21 +1660,19 @@ markup::Document HoverInfo::present() const {
 std::string HoverInfo::present(MarkupKind Kind) const {
   if (Kind == MarkupKind::Markdown) {
     const Config &Cfg = Config::current();
-    if ((Cfg.Documentation.CommentFormat ==
-         Config::CommentFormatPolicy::Markdown) ||
-        (Cfg.Documentation.CommentFormat ==
-         Config::CommentFormatPolicy::Doxygen))
-      // If the user prefers Markdown, we use the present() method to generate
-      // the Markdown output.
-      return present().asMarkdown();
+    if (Cfg.Documentation.CommentFormat ==
+        Config::CommentFormatPolicy::Markdown)
+      return presentDefault().asMarkdown();
+    if (Cfg.Documentation.CommentFormat == Config::CommentFormatPolicy::Doxygen)
+      return presentDoxygen().asMarkdown();
     if (Cfg.Documentation.CommentFormat ==
         Config::CommentFormatPolicy::PlainText)
       // If the user prefers plain text, we use the present() method to generate
       // the plain text output.
-      return present().asEscapedMarkdown();
+      return presentDefault().asEscapedMarkdown();
   }
 
-  return present().asPlainText();
+  return presentDefault().asPlainText();
 }
 
 // If the backtick at `Offset` starts a probable quoted range, return the range
diff --git a/clang-tools-extra/clangd/Hover.h b/clang-tools-extra/clangd/Hover.h
index 2f65431bd72de..614180a7b9846 100644
--- a/clang-tools-extra/clangd/Hover.h
+++ b/clang-tools-extra/clangd/Hover.h
@@ -74,6 +74,8 @@ struct HoverInfo {
   std::optional<Range> SymRange;
   index::SymbolKind Kind = index::SymbolKind::Unknown;
   std::string Documentation;
+  // required to create a comments::CommandTraits object without the ASTContext
+  CommentOptions CommentOpts;
   /// Source code containing the definition of the symbol.
   std::string Definition;
   const char *DefinitionLanguage = "cpp";
@@ -118,10 +120,23 @@ struct HoverInfo {
   // alphabetical order.
   std::vector<std::string> UsedSymbolNames;
 
-  /// Produce a user-readable information.
-  markup::Document present() const;
-
+  /// Produce a user-readable information based on the specified markup kind.
   std::string present(MarkupKind Kind) const;
+
+private:
+  void usedSymbolNamesToMarkup(markup::Document &Output) const;
+  void providerToMarkupParagraph(markup::Document &Output) const;
+  void definitionScopeToMarkup(markup::Document &Output) const;
+  void calleeArgInfoToMarkupParagraph(markup::Paragraph &P) const;
+  void valueToMarkupParagraph(markup::Paragraph &P) const;
+  void offsetToMarkupParagraph(markup::Paragraph &P) const;
+  void sizeToMarkupParagraph(markup::Paragraph &P) const;
+
+  /// Parse and render the hover information as Doxygen documentation.
+  markup::Document presentDoxygen() const;
+
+  /// Render the hover information as a default documentation.
+  markup::Document presentDefault() const;
 };
 
 inline bool operator==(const HoverInfo::PrintedType &LHS,
diff --git a/clang-tools-extra/clangd/IncludeFixer.cpp b/clang-tools-extra/clangd/IncludeFixer.cpp
index 50bc2bd7ccb94..c27d960cd963b 100644
--- a/clang-tools-extra/clangd/IncludeFixer.cpp
+++ b/clang-tools-extra/clangd/IncludeFixer.cpp
@@ -173,7 +173,7 @@ std::vector<Fix> IncludeFixer::fix(DiagnosticsEngine::Level DiagLevel,
           // `enum x : int;' is not formally an incomplete type.
           // We may need a full definition anyway.
           if (auto * ET = llvm::dyn_cast<EnumType>(T))
-            if (!ET->getDecl()->getDefinition())
+            if (!ET->getOriginalDecl()->getDefinition())
               return fixIncompleteType(*T);
         }
       }
@@ -400,35 +400,35 @@ std::optional<CheapUnresolvedName> extractUnresolvedNameCheaply(
   CheapUnresolvedName Result;
   Result.Name = Unresolved.getAsString();
   if (SS && SS->isNotEmpty()) { // "::" or "ns::"
-    if (auto *Nested = SS->getScopeRep()) {
-      if (Nested->getKind() == NestedNameSpecifier::Global) {
-        Result.ResolvedScope = "";
-      } else if (const NamespaceBaseDecl *NSB = Nested->getAsNamespace()) {
-        if (const auto *NS = dyn_cast<NamespaceDecl>(NSB)) {
-          std::string SpecifiedNS = printNamespaceScope(*NS);
-          std::optional<std::string> Spelling = getSpelledSpecifier(*SS, SM);
-
-          // Check the specifier spelled in the source.
-          // If the resolved scope doesn't end with the spelled scope, the
-          // resolved scope may come from a sema typo correction. For example,
-          // sema assumes that "clangd::" is a typo of "clang::" and uses
-          // "clang::" as the specified scope in:
-          //     namespace clang { clangd::X; }
-          // In this case, we use the "typo" specifier as extra scope instead
-          // of using the scope assumed by sema.
-          if (!Spelling || llvm::StringRef(SpecifiedNS).ends_with(*Spelling)) {
-            Result.ResolvedScope = std::move(SpecifiedNS);
-          } else {
-            Result.UnresolvedScope = std::move(*Spelling);
-          }
+    NestedNameSpecifier Nested = SS->getScopeRep();
+    if (Nested.getKind() == NestedNameSpecifier::Kind::Global) {
+      Result.ResolvedScope = "";
+    } else if (Nested.getKind() == NestedNameSpecifier::Kind::Namespace) {
+      const NamespaceBaseDecl *NSB = Nested.getAsNamespaceAndPrefix().Namespace;
+      if (const auto *NS = dyn_cast<NamespaceDecl>(NSB)) {
+        std::string SpecifiedNS = printNamespaceScope(*NS);
+        std::optional<std::string> Spelling = getSpelledSpecifier(*SS, SM);
+
+        // Check the specifier spelled in the source.
+        // If the resolved scope doesn't end with the spelled scope, the
+        // resolved scope may come from a sema typo correction. For example,
+        // sema assumes that "clangd::" is a typo of "clang::" and uses
+        // "clang::" as the specified scope in:
+        //     namespace clang { clangd::X; }
+        // In this case, we use the "typo" specifier as extra scope instead
+        // of using the scope assumed by sema.
+        if (!Spelling || llvm::StringRef(SpecifiedNS).ends_with(*Spelling)) {
+          Result.ResolvedScope = std::move(SpecifiedNS);
         } else {
-          Result.ResolvedScope = printNamespaceScope(*cast<NamespaceAliasDecl>(NSB)->getNamespace());
+          Result.UnresolvedScope = std::move(*Spelling);
         }
       } else {
-        // We don't fix symbols in scopes that are not top-level e.g. class
-        // members, as we don't collect includes for them.
-        return std::nullopt;
+        Result.ResolvedScope = printNamespaceScope(*cast<NamespaceAliasDecl>(NSB)->getNamespace());
       }
+    } else {
+      // We don't fix symbols in scopes that are not top-level e.g. class
+      // members, as we don't collect includes for them.
+      return std::nullopt;
     }
   }
 
diff --git a/clang-tools-extra/clangd/InlayHints.cpp b/clang-tools-extra/clangd/InlayHints.cpp
index 197c62c40dcf0..cd479e1b7c9bc 100644
--- a/clang-tools-extra/clangd/InlayHints.cpp
+++ b/clang-tools-extra/clangd/InlayHints.cpp
@@ -55,18 +55,24 @@ void stripLeadingUnderscores(StringRef &Name) { Name = Name.ltrim('_'); }
 
 // getDeclForType() returns the decl responsible for Type's spelling.
 // This is the inverse of ASTContext::getTypeDeclType().
-template <typename Ty, typename = decltype(((Ty *)nullptr)->getDecl())>
-const NamedDecl *getDeclForTypeImpl(const Ty *T) {
-  return T->getDecl();
-}
-const NamedDecl *getDeclForTypeImpl(const void *T) { return nullptr; }
 const NamedDecl *getDeclForType(const Type *T) {
   switch (T->getTypeClass()) {
-#define ABSTRACT_TYPE(TY, BASE)
-#define TYPE(TY, BASE)                                                         \
-  case Type::TY:                                                               \
-    return getDeclForTypeImpl(llvm::cast<TY##Type>(T));
-#include "clang/AST/TypeNodes.inc"
+  case Type::Enum:
+  case Type::Record:
+  case Type::InjectedClassName:
+    return cast<TagType>(T)->getOriginalDecl();
+  case Type::TemplateSpecialization:
+    return cast<TemplateSpecializationType>(T)
+        ->getTemplateName()
+        .getAsTemplateDecl(/*IgnoreDeduced=*/true);
+  case Type::Typedef:
+    return cast<TypedefType>(T)->getDecl();
+  case Type::UnresolvedUsing:
+    return cast<UnresolvedUsingType>(T)->getDecl();
+  case Type::Using:
+    return cast<UsingType>(T)->getDecl();
+  default:
+    return nullptr;
   }
   llvm_unreachable("Unknown TypeClass enum");
 }
@@ -81,8 +87,6 @@ llvm::StringRef getSimpleName(const NamedDecl &D) {
   return getSimpleName(D.getDeclName());
 }
 llvm::StringRef getSimpleName(QualType T) {
-  if (const auto *ET = llvm::dyn_cast<ElaboratedType>(T))
-    return getSimpleName(ET->getNamedType());
   if (const auto *BT = llvm::dyn_cast<BuiltinType>(T)) {
     PrintingPolicy PP(LangOptions{});
     PP.adjustForCPlusPlus();
diff --git a/clang-tools-extra/clangd/Selection.cpp b/clang-tools-extra/clangd/Selection.cpp
index 277cb8769a1b1..06165dfbbcdd2 100644
--- a/clang-tools-extra/clangd/Selection.cpp
+++ b/clang-tools-extra/clangd/Selection.cpp
@@ -62,7 +62,8 @@ void recordMetrics(const SelectionTree &S, const LangOptions &Lang) {
 }
 
 // Return the range covering a node and all its children.
-SourceRange getSourceRange(const DynTypedNode &N) {
+SourceRange getSourceRange(const DynTypedNode &N,
+                           bool IncludeQualifier = false) {
   // MemberExprs to implicitly access anonymous fields should not claim any
   // tokens for themselves. Given:
   //   struct A { struct { int b; }; };
@@ -80,7 +81,7 @@ SourceRange getSourceRange(const DynTypedNode &N) {
                  ? getSourceRange(DynTypedNode::create(*ME->getBase()))
                  : SourceRange();
   }
-  return N.getSourceRange();
+  return N.getSourceRange(IncludeQualifier);
 }
 
 // An IntervalSet maintains a set of disjoint subranges of an array.
@@ -643,8 +644,9 @@ class SelectionVisitor : public RecursiveASTVisitor<SelectionVisitor> {
     }
     return traverseNode(X, [&] { return Base::TraverseDecl(X); });
   }
-  bool TraverseTypeLoc(TypeLoc X) {
-    return traverseNode(&X, [&] { return Base::TraverseTypeLoc(X); });
+  bool TraverseTypeLoc(TypeLoc X, bool TraverseQualifier = true) {
+    return traverseNode(
+        &X, [&] { return Base::TraverseTypeLoc(X, TraverseQualifier); });
   }
   bool TraverseTemplateArgumentLoc(const TemplateArgumentLoc &X) {
     return traverseNode(&X,
@@ -690,7 +692,8 @@ class SelectionVisitor : public RecursiveASTVisitor<SelectionVisitor> {
   // This means we'd never see 'int' in 'const int'! Work around that here.
   // (The reason for the behavior is to avoid traversing the nested Type twice,
   // but we ignore TraverseType anyway).
-  bool TraverseQualifiedTypeLoc(QualifiedTypeLoc QX) {
+  bool TraverseQualifiedTypeLoc(QualifiedTypeLoc QX,
+                                bool TraverseQualifier = true) {
     return traverseNode<TypeLoc>(
         &QX, [&] { return TraverseTypeLoc(QX.getUnqualifiedLoc()); });
   }
@@ -698,7 +701,7 @@ class SelectionVisitor : public RecursiveASTVisitor<SelectionVisitor> {
     return traverseNode(&PL, [&] { return Base::TraverseObjCProtocolLoc(PL); });
   }
   // Uninteresting parts of the AST that don't have locations within them.
-  bool TraverseNestedNameSpecifier(NestedNameSpecifier *) { return true; }
+  bool TraverseNestedNameSpecifier(NestedNameSpecifier) { return true; }
   bool TraverseType(QualType) { return true; }
 
   // The DeclStmt for the loop variable claims to cover the whole range
@@ -798,7 +801,7 @@ class SelectionVisitor : public RecursiveASTVisitor<SelectionVisitor> {
   // An optimization for a common case: nodes outside macro expansions that
   // don't intersect the selection may be recursively skipped.
   bool canSafelySkipNode(const DynTypedNode &N) {
-    SourceRange S = getSourceRange(N);
+    SourceRange S = getSourceRange(N, /*IncludeQualifier=*/true);
     if (auto *TL = N.get<TypeLoc>()) {
       // FIXME: TypeLoc::getBeginLoc()/getEndLoc() are pretty fragile
       // heuristics. We should consider only pruning critical TypeLoc nodes, to
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index e6d5cf7053694..2b151b1274428 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -1127,21 +1127,6 @@ class CollectExtraHighlightings
     return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
   }
 
-  // findExplicitReferences will walk nested-name-specifiers and
-  // find anything that can be resolved to a Decl. However, non-leaf
-  // components of nested-name-specifiers which are dependent names
-  // (kind "Identifier") cannot be resolved to a decl, so we visit
-  // them here.
-  bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc Q) {
-    if (NestedNameSpecifier *NNS = Q.getNestedNameSpecifier()) {
-      if (NNS->getKind() == NestedNameSpecifier::Identifier)
-        H.addToken(Q.getLocalBeginLoc(), HighlightingKind::Type)
-            .addModifier(HighlightingModifier::DependentName)
-            .addModifier(HighlightingModifier::ClassScope);
-    }
-    return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(Q);
-  }
-
 private:
   HighlightingsBuilder &H;
 };
diff --git a/clang-tools-extra/clangd/SymbolDocumentation.cpp b/clang-tools-extra/clangd/SymbolDocumentation.cpp
new file mode 100644
index 0000000000000..dea637b9100da
--- /dev/null
+++ b/clang-tools-extra/clangd/SymbolDocumentation.cpp
@@ -0,0 +1,297 @@
+//===--- SymbolDocumentation.cpp ==-------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolDocumentation.h"
+
+#include "support/Markup.h"
+#include "clang/AST/Comment.h"
+#include "clang/AST/CommentCommandTraits.h"
+#include "clang/AST/CommentVisitor.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+std::string commandMarkerAsString(comments::CommandMarkerKind CommandMarker) {
+  switch (CommandMarker) {
+  case comments::CommandMarkerKind::CMK_At:
+    return "@";
+  case comments::CommandMarkerKind::CMK_Backslash:
+    return "\\";
+  }
+  llvm_unreachable("Unknown command marker kind");
+}
+
+void commandToMarkup(markup::Paragraph &Out, StringRef Command,
+                     comments::CommandMarkerKind CommandMarker,
+                     StringRef Args) {
+  Out.appendBoldText(commandMarkerAsString(CommandMarker) + Command.str());
+  if (!Args.empty()) {
+    Out.appendSpace();
+    Out.appendEmphasizedText(Args.str());
+  }
+}
+} // namespace
+
+class ParagraphToMarkupDocument
+    : public comments::ConstCommentVisitor<ParagraphToMarkupDocument> {
+public:
+  ParagraphToMarkupDocument(markup::Paragraph &Out,
+                            const comments::CommandTraits &Traits)
+      : Out(Out), Traits(Traits) {}
+
+  void visitParagraphComment(const comments::ParagraphComment *C) {
+    if (!C)
+      return;
+
+    for (const auto *Child = C->child_begin(); Child != C->child_end();
+         ++Child) {
+      visit(*Child);
+    }
+  }
+
+  void visitTextComment(const comments::TextComment *C) {
+    // Always trim leading space after a newline.
+    StringRef Text = C->getText();
+    if (LastChunkEndsWithNewline && C->getText().starts_with(' '))
+      Text = Text.drop_front();
+
+    LastChunkEndsWithNewline = C->hasTrailingNewline();
+    Out.appendText(Text.str() + (LastChunkEndsWithNewline ? "\n" : ""));
+  }
+
+  void visitInlineCommandComment(const comments::InlineCommandComment *C) {
+
+    if (C->getNumArgs() > 0) {
+      std::string ArgText;
+      for (unsigned I = 0; I < C->getNumArgs(); ++I) {
+        if (!ArgText.empty())
+          ArgText += " ";
+        ArgText += C->getArgText(I);
+      }
+
+      switch (C->getRenderKind()) {
+      case comments::InlineCommandRenderKind::Monospaced:
+        Out.appendCode(ArgText);
+        break;
+      case comments::InlineCommandRenderKind::Bold:
+        Out.appendBoldText(ArgText);
+        break;
+      case comments::InlineCommandRenderKind::Emphasized:
+        Out.appendEmphasizedText(ArgText);
+        break;
+      default:
+        commandToMarkup(Out, C->getCommandName(Traits), C->getCommandMarker(),
+                        ArgText);
+        break;
+      }
+    } else {
+      if (C->getCommandName(Traits) == "n") {
+        // \n is a special case, it is used to create a new line.
+        Out.appendText("  \n");
+        LastChunkEndsWithNewline = true;
+        return;
+      }
+
+      commandToMarkup(Out, C->getCommandName(Traits), C->getCommandMarker(),
+                      "");
+    }
+  }
+
+  void visitHTMLStartTagComment(const comments::HTMLStartTagComment *STC) {
+    std::string TagText = "<" + STC->getTagName().str();
+
+    for (unsigned I = 0; I < STC->getNumAttrs(); ++I) {
+      const comments::HTMLStartTagComment::Attribute &Attr = STC->getAttr(I);
+      TagText += " " + Attr.Name.str() + "=\"" + Attr.Value.str() + "\"";
+    }
+
+    if (STC->isSelfClosing())
+      TagText += " /";
+    TagText += ">";
+
+    LastChunkEndsWithNewline = STC->hasTrailingNewline();
+    Out.appendText(TagText + (LastChunkEndsWithNewline ? "\n" : ""));
+  }
+
+  void visitHTMLEndTagComment(const comments::HTMLEndTagComment *ETC) {
+    LastChunkEndsWithNewline = ETC->hasTrailingNewline();
+    Out.appendText("</" + ETC->getTagName().str() + ">" +
+                   (LastChunkEndsWithNewline ? "\n" : ""));
+  }
+
+private:
+  markup::Paragraph &Out;
+  const comments::CommandTraits &Traits;
+
+  /// If true, the next leading space after a new line is trimmed.
+  bool LastChunkEndsWithNewline = false;
+};
+
+class ParagraphToString
+    : public comments::ConstCommentVisitor<ParagraphToString> {
+public:
+  ParagraphToString(llvm::raw_string_ostream &Out,
+                    const comments::CommandTraits &Traits)
+      : Out(Out), Traits(Traits) {}
+
+  void visitParagraphComment(const comments::ParagraphComment *C) {
+    if (!C)
+      return;
+
+    for (const auto *Child = C->child_begin(); Child != C->child_end();
+         ++Child) {
+      visit(*Child);
+    }
+  }
+
+  void visitTextComment(const comments::TextComment *C) { Out << C->getText(); }
+
+  void visitInlineCommandComment(const comments::InlineCommandComment *C) {
+    Out << commandMarkerAsString(C->getCommandMarker());
+    Out << C->getCommandName(Traits);
+    if (C->getNumArgs() > 0) {
+      for (unsigned I = 0; I < C->getNumArgs(); ++I)
+        Out << " " << C->getArgText(I);
+    }
+    Out << " ";
+  }
+
+  void visitHTMLStartTagComment(const comments::HTMLStartTagComment *STC) {
+    Out << "<" << STC->getTagName().str();
+
+    for (unsigned I = 0; I < STC->getNumAttrs(); ++I) {
+      const comments::HTMLStartTagComment::Attribute &Attr = STC->getAttr(I);
+      Out << " " << Attr.Name.str();
+      if (!Attr.Value.str().empty())
+        Out << "=\"" << Attr.Value.str() << "\"";
+    }
+
+    if (STC->isSelfClosing())
+      Out << " /";
+    Out << ">";
+
+    Out << (STC->hasTrailingNewline() ? "\n" : "");
+  }
+
+  void visitHTMLEndTagComment(const comments::HTMLEndTagComment *ETC) {
+    Out << "</" << ETC->getTagName().str() << ">"
+        << (ETC->hasTrailingNewline() ? "\n" : "");
+  }
+
+private:
+  llvm::raw_string_ostream &Out;
+  const comments::CommandTraits &Traits;
+};
+
+class BlockCommentToMarkupDocument
+    : public comments::ConstCommentVisitor<BlockCommentToMarkupDocument> {
+public:
+  BlockCommentToMarkupDocument(markup::Document &Out,
+                               const comments::CommandTraits &Traits)
+      : Out(Out), Traits(Traits) {}
+
+  void visitBlockCommandComment(const comments::BlockCommandComment *B) {
+
+    switch (B->getCommandID()) {
+    case comments::CommandTraits::KCI_arg:
+    case comments::CommandTraits::KCI_li:
+      // \li and \arg are special cases, they are used to create a list item.
+      // In markdown it is a bullet list.
+      ParagraphToMarkupDocument(Out.addBulletList().addItem().addParagraph(),
+                                Traits)
+          .visit(B->getParagraph());
+      break;
+    default: {
+      // Some commands have arguments, like \throws.
+      // The arguments are not part of the paragraph.
+      // We need reconstruct them here.
+      std::string ArgText;
+      for (unsigned I = 0; I < B->getNumArgs(); ++I) {
+        if (!ArgText.empty())
+          ArgText += " ";
+        ArgText += B->getArgText(I);
+      }
+      auto &P = Out.addParagraph();
+      commandToMarkup(P, B->getCommandName(Traits), B->getCommandMarker(),
+                      ArgText);
+      if (B->getParagraph() && !B->getParagraph()->isWhitespace()) {
+        // For commands with arguments, the paragraph starts after the first
+        // space. Therefore we need to append a space manually in this case.
+        if (!ArgText.empty())
+          P.appendSpace();
+        ParagraphToMarkupDocument(P, Traits).visit(B->getParagraph());
+      }
+    }
+    }
+  }
+
+  void visitVerbatimBlockComment(const comments::VerbatimBlockComment *VB) {
+    commandToMarkup(Out.addParagraph(), VB->getCommandName(Traits),
+                    VB->getCommandMarker(), "");
+
+    std::string VerbatimText;
+
+    for (const auto *LI = VB->child_begin(); LI != VB->child_end(); ++LI) {
+      if (const auto *Line = cast<comments::VerbatimBlockLineComment>(*LI)) {
+        VerbatimText += Line->getText().str() + "\n";
+      }
+    }
+
+    Out.addCodeBlock(VerbatimText, "");
+
+    commandToMarkup(Out.addParagraph(), VB->getCloseName(),
+                    VB->getCommandMarker(), "");
+  }
+
+  void visitVerbatimLineComment(const comments::VerbatimLineComment *VL) {
+    auto &P = Out.addParagraph();
+    commandToMarkup(P, VL->getCommandName(Traits), VL->getCommandMarker(), "");
+    P.appendSpace().appendCode(VL->getText().str(), true).appendSpace();
+  }
+
+private:
+  markup::Document &Out;
+  const comments::CommandTraits &Traits;
+  StringRef CommentEscapeMarker;
+};
+
+void SymbolDocCommentVisitor::parameterDocToMarkup(StringRef ParamName,
+                                                   markup::Paragraph &Out) {
+  if (ParamName.empty())
+    return;
+
+  if (const auto *P = Parameters.lookup(ParamName)) {
+    ParagraphToMarkupDocument(Out, Traits).visit(P->getParagraph());
+  }
+}
+
+void SymbolDocCommentVisitor::parameterDocToString(
+    StringRef ParamName, llvm::raw_string_ostream &Out) {
+  if (ParamName.empty())
+    return;
+
+  if (const auto *P = Parameters.lookup(ParamName)) {
+    ParagraphToString(Out, Traits).visit(P->getParagraph());
+  }
+}
+
+void SymbolDocCommentVisitor::docToMarkup(markup::Document &Out) {
+  for (unsigned I = 0; I < CommentPartIndex; ++I) {
+    if (const auto *BC = BlockCommands.lookup(I)) {
+      BlockCommentToMarkupDocument(Out, Traits).visit(BC);
+    } else if (const auto *P = FreeParagraphs.lookup(I)) {
+      ParagraphToMarkupDocument(Out.addParagraph(), Traits).visit(P);
+    }
+  }
+}
+
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/SymbolDocumentation.h b/clang-tools-extra/clangd/SymbolDocumentation.h
new file mode 100644
index 0000000000000..b5120ba04e8f1
--- /dev/null
+++ b/clang-tools-extra/clangd/SymbolDocumentation.h
@@ -0,0 +1,155 @@
+//===--- SymbolDocumentation.h ==---------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Class to parse doxygen comments into a flat structure for consumption
+// in e.g. Hover and Code Completion
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H
+
+#include "support/Markup.h"
+#include "clang/AST/Comment.h"
+#include "clang/AST/CommentLexer.h"
+#include "clang/AST/CommentParser.h"
+#include "clang/AST/CommentSema.h"
+#include "clang/AST/CommentVisitor.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/Support/raw_ostream.h"
+#include <string>
+
+namespace clang {
+namespace clangd {
+
+class SymbolDocCommentVisitor
+    : public comments::ConstCommentVisitor<SymbolDocCommentVisitor> {
+public:
+  SymbolDocCommentVisitor(comments::FullComment *FC,
+                          const CommentOptions &CommentOpts)
+      : Traits(Allocator, CommentOpts), Allocator() {
+    if (!FC)
+      return;
+
+    for (auto *Block : FC->getBlocks()) {
+      visit(Block);
+    }
+  }
+
+  SymbolDocCommentVisitor(llvm::StringRef Documentation,
+                          const CommentOptions &CommentOpts)
+      : Traits(Allocator, CommentOpts), Allocator() {
+
+    if (Documentation.empty())
+      return;
+
+    CommentWithMarkers.reserve(Documentation.size() +
+                               Documentation.count('\n') * 3);
+
+    // The comment lexer expects doxygen markers, so add them back.
+    // We need to use the /// style doxygen markers because the comment could
+    // contain the closing the closing tag "*/" of a C Style "/** */" comment
+    // which would break the parsing if we would just enclose the comment text
+    // with "/** */".
+    CommentWithMarkers = "///";
+    bool NewLine = true;
+    for (char C : Documentation) {
+      if (C == '\n') {
+        CommentWithMarkers += "\n///";
+        NewLine = true;
+      } else {
+        if (NewLine && (C == '<')) {
+          // A comment line starting with '///<' is treated as a doxygen
+          // comment. Therefore add a space to separate the '<' from the comment
+          // marker. This allows to parse html tags at the beginning of a line
+          // and the escape marker prevents adding the artificial space in the
+          // markup documentation. The extra space will not be rendered, since
+          // we render it as markdown.
+          CommentWithMarkers += ' ';
+        }
+        CommentWithMarkers += C;
+        NewLine = false;
+      }
+    }
+    SourceManagerForFile SourceMgrForFile("mock_file.cpp", CommentWithMarkers);
+
+    SourceManager &SourceMgr = SourceMgrForFile.get();
+    // The doxygen Sema requires a Diagostics consumer, since it reports
+    // warnings e.g. when parameters are not documented correctly. These
+    // warnings are not relevant for us, so we can ignore them.
+    SourceMgr.getDiagnostics().setClient(new IgnoringDiagConsumer);
+
+    comments::Sema S(Allocator, SourceMgr, SourceMgr.getDiagnostics(), Traits,
+                     /*PP=*/nullptr);
+    comments::Lexer L(Allocator, SourceMgr.getDiagnostics(), Traits,
+                      SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID()),
+                      CommentWithMarkers.data(),
+                      CommentWithMarkers.data() + CommentWithMarkers.size());
+    comments::Parser P(L, S, Allocator, SourceMgr, SourceMgr.getDiagnostics(),
+                       Traits);
+    comments::FullComment *FC = P.parseFullComment();
+
+    if (!FC)
+      return;
+
+    for (auto *Block : FC->getBlocks()) {
+      visit(Block);
+    }
+  }
+
+  bool isParameterDocumented(StringRef ParamName) const {
+    return Parameters.contains(ParamName);
+  }
+
+  void parameterDocToMarkup(StringRef ParamName, markup::Paragraph &Out);
+
+  void parameterDocToString(StringRef ParamName, llvm::raw_string_ostream &Out);
+
+  void docToMarkup(markup::Document &Out);
+
+  void visitBlockCommandComment(const comments::BlockCommandComment *B) {
+    BlockCommands[CommentPartIndex] = B;
+    CommentPartIndex++;
+  }
+
+  void visitParagraphComment(const comments::ParagraphComment *P) {
+    FreeParagraphs[CommentPartIndex] = P;
+    CommentPartIndex++;
+  }
+
+  void visitParamCommandComment(const comments::ParamCommandComment *P) {
+    Parameters[P->getParamNameAsWritten()] = P;
+  }
+
+private:
+  comments::CommandTraits Traits;
+  llvm::BumpPtrAllocator Allocator;
+  std::string CommentWithMarkers;
+
+  /// Index to keep track of the order of the comments.
+  /// We want to rearange some commands like \\param.
+  /// This index allows us to keep the order of the other comment parts.
+  unsigned CommentPartIndex = 0;
+
+  /// Parsed paragaph(s) of the "param" comamnd(s)
+  llvm::SmallDenseMap<StringRef, const comments::ParamCommandComment *>
+      Parameters;
+
+  /// All the block commands.
+  llvm::SmallDenseMap<unsigned, const comments::BlockCommandComment *>
+      BlockCommands;
+
+  /// All "free" text paragraphs.
+  llvm::SmallDenseMap<unsigned, const comments::ParagraphComment *>
+      FreeParagraphs;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_SYMBOLDOCUMENTATION_H
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index 83a8b7289aec3..a253a630a48cc 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -1876,7 +1876,7 @@ static void fillSubTypes(const SymbolID &ID,
   });
 }
 
-using RecursionProtectionSet = llvm::SmallSet<const CXXRecordDecl *, 4>;
+using RecursionProtectionSet = llvm::SmallPtrSet<const CXXRecordDecl *, 4>;
 
 // Extracts parents from AST and populates the type hierarchy item.
 static void fillSuperTypes(const CXXRecordDecl &CXXRD, llvm::StringRef TUPath,
@@ -1965,7 +1965,8 @@ std::vector<const CXXRecordDecl *> findRecordTypeAt(ParsedAST &AST,
 
 // Return the type most associated with an AST node.
 // This isn't precisely defined: we want "go to type" to do something useful.
-static QualType typeForNode(const SelectionTree::Node *N) {
+static QualType typeForNode(const ASTContext &Ctx,
+                            const SelectionTree::Node *N) {
   // If we're looking at a namespace qualifier, walk up to what it's qualifying.
   // (If we're pointing at a *class* inside a NNS, N will be a TypeLoc).
   while (N && N->ASTNode.get<NestedNameSpecifierLoc>())
@@ -1999,10 +2000,13 @@ static QualType typeForNode(const SelectionTree::Node *N) {
 
   if (const Decl *D = N->ASTNode.get<Decl>()) {
     struct Visitor : ConstDeclVisitor<Visitor, QualType> {
+      const ASTContext &Ctx;
+      Visitor(const ASTContext &Ctx) : Ctx(Ctx) {}
+
       QualType VisitValueDecl(const ValueDecl *D) { return D->getType(); }
       // Declaration of a type => that type.
       QualType VisitTypeDecl(const TypeDecl *D) {
-        return QualType(D->getTypeForDecl(), 0);
+        return Ctx.getTypeDeclType(D);
       }
       // Exception: alias declaration => the underlying type, not the alias.
       QualType VisitTypedefNameDecl(const TypedefNameDecl *D) {
@@ -2012,7 +2016,7 @@ static QualType typeForNode(const SelectionTree::Node *N) {
       QualType VisitTemplateDecl(const TemplateDecl *D) {
         return Visit(D->getTemplatedDecl());
       }
-    } V;
+    } V(Ctx);
     return V.Visit(D);
   }
 
@@ -2156,7 +2160,8 @@ std::vector<LocatedSymbol> findType(ParsedAST &AST, Position Pos,
     // unique_ptr<unique_ptr<T>>. Let's *not* remove them, because it gives you some
     // information about the type you may have not known before
     // (since unique_ptr<unique_ptr<T>> != unique_ptr<T>).
-    for (const QualType& Type : unwrapFindType(typeForNode(N), AST.getHeuristicResolver()))
+    for (const QualType &Type : unwrapFindType(
+             typeForNode(AST.getASTContext(), N), AST.getHeuristicResolver()))
       llvm::copy(locateSymbolForType(AST, Type, Index),
                  std::back_inserter(LocatedSymbols));
 
diff --git a/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp b/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
index 67fc451a6a1a1..f65c74fdbc9ee 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
@@ -115,13 +115,6 @@ class UsingFinder : public RecursiveASTVisitor<UsingFinder> {
   const SourceManager &SM;
 };
 
-bool isFullyQualified(const NestedNameSpecifier *NNS) {
-  if (!NNS)
-    return false;
-  return NNS->getKind() == NestedNameSpecifier::Global ||
-         isFullyQualified(NNS->getPrefix());
-}
-
 struct InsertionPointData {
   // Location to insert the "using" statement. If invalid then the statement
   // should not be inserted at all (it already exists).
@@ -167,18 +160,20 @@ findInsertionPoint(const Tweak::Selection &Inputs,
   for (auto &U : Usings) {
     // Only "upgrade" to fully qualified is all relevant using decls are fully
     // qualified. Otherwise trust what the user typed.
-    if (!isFullyQualified(U->getQualifier()))
+    if (!U->getQualifier().isFullyQualified())
       AlwaysFullyQualify = false;
 
     if (SM.isBeforeInTranslationUnit(Inputs.Cursor, U->getUsingLoc()))
       // "Usings" is sorted, so we're done.
       break;
-    if (const auto *Namespace = dyn_cast_if_present<NamespaceDecl>(
-            U->getQualifier()->getAsNamespace())) {
+    if (NestedNameSpecifier Qualifier = U->getQualifier();
+        Qualifier.getKind() == NestedNameSpecifier::Kind::Namespace) {
+      const auto *Namespace =
+          U->getQualifier().getAsNamespaceAndPrefix().Namespace;
       if (Namespace->getCanonicalDecl() ==
               QualifierToRemove.getNestedNameSpecifier()
-                  ->getAsNamespace()
-                  ->getCanonicalDecl() &&
+                  .getAsNamespaceAndPrefix()
+                  .Namespace->getCanonicalDecl() &&
           U->getName() == Name) {
         return InsertionPointData();
       }
@@ -232,8 +227,9 @@ findInsertionPoint(const Tweak::Selection &Inputs,
 }
 
 bool isNamespaceForbidden(const Tweak::Selection &Inputs,
-                          const NestedNameSpecifier &Namespace) {
-  const auto *NS = dyn_cast<NamespaceDecl>(Namespace.getAsNamespace());
+                          NestedNameSpecifier Namespace) {
+  const auto *NS =
+      dyn_cast<NamespaceDecl>(Namespace.getAsNamespaceAndPrefix().Namespace);
   if (!NS)
     return true;
   std::string NamespaceStr = printNamespaceScope(*NS);
@@ -247,11 +243,11 @@ bool isNamespaceForbidden(const Tweak::Selection &Inputs,
   return false;
 }
 
-std::string getNNSLAsString(NestedNameSpecifierLoc &NNSL,
+std::string getNNSLAsString(NestedNameSpecifierLoc NNSL,
                             const PrintingPolicy &Policy) {
   std::string Out;
   llvm::raw_string_ostream OutStream(Out);
-  NNSL.getNestedNameSpecifier()->print(OutStream, Policy);
+  NNSL.getNestedNameSpecifier().print(OutStream, Policy);
   return OutStream.str();
 }
 
@@ -276,16 +272,15 @@ bool AddUsing::prepare(const Selection &Inputs) {
       continue;
     }
     if (auto *T = Node->ASTNode.get<TypeLoc>()) {
-      if (T->getAs<ElaboratedTypeLoc>()) {
+      // Find the outermost TypeLoc.
+      if (Node->Parent->ASTNode.get<NestedNameSpecifierLoc>())
+        continue;
+      if (isa<TagType, TemplateSpecializationType, TypedefType, UsingType,
+              UnresolvedUsingType>(T->getTypePtr()))
         break;
-      }
-      if (Node->Parent->ASTNode.get<TypeLoc>() ||
-          Node->Parent->ASTNode.get<NestedNameSpecifierLoc>()) {
-        // Node is TypeLoc, but it's parent is either TypeLoc or
-        // NestedNameSpecifier. In both cases, we want to go up, to find
-        // the outermost TypeLoc.
+      // Find the outermost TypeLoc.
+      if (Node->Parent->ASTNode.get<TypeLoc>())
         continue;
-      }
     }
     break;
   }
@@ -307,32 +302,70 @@ bool AddUsing::prepare(const Selection &Inputs) {
       MustInsertAfterLoc = D->getDecl()->getBeginLoc();
     }
   } else if (auto *T = Node->ASTNode.get<TypeLoc>()) {
-    if (auto E = T->getAs<ElaboratedTypeLoc>()) {
-      QualifierToRemove = E.getQualifierLoc();
-
-      SpelledNameRange = E.getSourceRange();
-      if (auto T = E.getNamedTypeLoc().getAs<TemplateSpecializationTypeLoc>()) {
-        // Remove the template arguments from the name.
-        SpelledNameRange.setEnd(T.getLAngleLoc().getLocWithOffset(-1));
-      }
-
-      if (const auto *ET = E.getTypePtr()) {
-        if (const auto *TDT =
-                dyn_cast<TypedefType>(ET->getNamedType().getTypePtr())) {
-          MustInsertAfterLoc = TDT->getDecl()->getBeginLoc();
-        } else if (auto *TD = ET->getAsTagDecl()) {
-          MustInsertAfterLoc = TD->getBeginLoc();
-        }
-      }
+    switch (T->getTypeLocClass()) {
+    case TypeLoc::TemplateSpecialization: {
+      auto TL = T->castAs<TemplateSpecializationTypeLoc>();
+      QualifierToRemove = TL.getQualifierLoc();
+      if (!QualifierToRemove)
+        break;
+      SpelledNameRange = TL.getTemplateNameLoc();
+      if (auto *TD = TL.getTypePtr()->getTemplateName().getAsTemplateDecl(
+              /*IgnoreDeduced=*/true))
+        MustInsertAfterLoc = TD->getBeginLoc();
+      break;
+    }
+    case TypeLoc::Enum:
+    case TypeLoc::Record:
+    case TypeLoc::InjectedClassName: {
+      auto TL = T->castAs<TagTypeLoc>();
+      QualifierToRemove = TL.getQualifierLoc();
+      if (!QualifierToRemove)
+        break;
+      SpelledNameRange = TL.getNameLoc();
+      MustInsertAfterLoc = TL.getOriginalDecl()->getBeginLoc();
+      break;
+    }
+    case TypeLoc::Typedef: {
+      auto TL = T->castAs<TypedefTypeLoc>();
+      QualifierToRemove = TL.getQualifierLoc();
+      if (!QualifierToRemove)
+        break;
+      SpelledNameRange = TL.getNameLoc();
+      MustInsertAfterLoc = TL.getDecl()->getBeginLoc();
+      break;
+    }
+    case TypeLoc::UnresolvedUsing: {
+      auto TL = T->castAs<UnresolvedUsingTypeLoc>();
+      QualifierToRemove = TL.getQualifierLoc();
+      if (!QualifierToRemove)
+        break;
+      SpelledNameRange = TL.getNameLoc();
+      MustInsertAfterLoc = TL.getDecl()->getBeginLoc();
+      break;
+    }
+    case TypeLoc::Using: {
+      auto TL = T->castAs<UsingTypeLoc>();
+      QualifierToRemove = TL.getQualifierLoc();
+      if (!QualifierToRemove)
+        break;
+      SpelledNameRange = TL.getNameLoc();
+      MustInsertAfterLoc = TL.getDecl()->getBeginLoc();
+      break;
+    }
+    default:
+      break;
     }
+    if (QualifierToRemove)
+      SpelledNameRange.setBegin(QualifierToRemove.getBeginLoc());
   }
   if (!QualifierToRemove ||
       // FIXME: This only supports removing qualifiers that are made up of just
       // namespace names. If qualifier contains a type, we could take the
       // longest namespace prefix and remove that.
-      !QualifierToRemove.getNestedNameSpecifier()->getAsNamespace() ||
+      QualifierToRemove.getNestedNameSpecifier().getKind() !=
+          NestedNameSpecifier::Kind::Namespace ||
       // Respect user config.
-      isNamespaceForbidden(Inputs, *QualifierToRemove.getNestedNameSpecifier()))
+      isNamespaceForbidden(Inputs, QualifierToRemove.getNestedNameSpecifier()))
     return false;
   // Macros are difficult. We only want to offer code action when what's spelled
   // under the cursor is a namespace qualifier. If it's a macro that expands to
@@ -384,7 +417,7 @@ Expected<Tweak::Effect> AddUsing::apply(const Selection &Inputs) {
     llvm::raw_string_ostream UsingTextStream(UsingText);
     UsingTextStream << "using ";
     if (InsertionPoint->AlwaysFullyQualify &&
-        !isFullyQualified(QualifierToRemove.getNestedNameSpecifier()))
+        !QualifierToRemove.getNestedNameSpecifier().isFullyQualified())
       UsingTextStream << "::";
     UsingTextStream << QualifierToSpell << SpelledName << ";"
                     << InsertionPoint->Suffix;
diff --git a/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp b/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
index cd07cbf73635c..bc9a790232507 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
@@ -181,7 +181,7 @@ struct ExtractionZone {
   bool requiresHoisting(const SourceManager &SM,
                         const HeuristicResolver *Resolver) const {
     // First find all the declarations that happened inside extraction zone.
-    llvm::SmallSet<const Decl *, 1> DeclsInExtZone;
+    llvm::SmallPtrSet<const Decl *, 1> DeclsInExtZone;
     for (auto *RootStmt : RootStmts) {
       findExplicitReferences(
           RootStmt,
@@ -362,7 +362,7 @@ struct NewFunction {
   SourceLocation DefinitionPoint;
   std::optional<SourceLocation> ForwardDeclarationPoint;
   const CXXRecordDecl *EnclosingClass = nullptr;
-  const NestedNameSpecifier *DefinitionQualifier = nullptr;
+  NestedNameSpecifier DefinitionQualifier = std::nullopt;
   const DeclContext *SemanticDC = nullptr;
   const DeclContext *SyntacticDC = nullptr;
   const DeclContext *ForwardDeclarationSyntacticDC = nullptr;
@@ -455,13 +455,12 @@ std::string NewFunction::renderQualifiers() const {
 }
 
 std::string NewFunction::renderDeclarationName(FunctionDeclKind K) const {
-  if (DefinitionQualifier == nullptr || K != OutOfLineDefinition) {
+  if (!DefinitionQualifier || K != OutOfLineDefinition)
     return Name;
-  }
 
   std::string QualifierName;
   llvm::raw_string_ostream Oss(QualifierName);
-  DefinitionQualifier->print(Oss, *LangOpts);
+  DefinitionQualifier.print(Oss, *LangOpts);
   return llvm::formatv("{0}{1}", QualifierName, Name);
 }
 
diff --git a/clang-tools-extra/clangd/refactor/tweaks/PopulateSwitch.cpp b/clang-tools-extra/clangd/refactor/tweaks/PopulateSwitch.cpp
index 43cfc769f7f71..7e616968c6046 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/PopulateSwitch.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/PopulateSwitch.cpp
@@ -116,7 +116,7 @@ bool PopulateSwitch::prepare(const Selection &Sel) {
   EnumT = Cond->getType().getCanonicalType()->getAsAdjusted<EnumType>();
   if (!EnumT)
     return false;
-  EnumD = EnumT->getDecl();
+  EnumD = EnumT->getOriginalDecl();
   if (!EnumD || EnumD->isDependentType())
     return false;
 
diff --git a/clang-tools-extra/clangd/support/Markup.cpp b/clang-tools-extra/clangd/support/Markup.cpp
index a13083026f26b..89bdc656d440f 100644
--- a/clang-tools-extra/clangd/support/Markup.cpp
+++ b/clang-tools-extra/clangd/support/Markup.cpp
@@ -363,7 +363,12 @@ class CodeBlock : public Block {
   void renderMarkdown(llvm::raw_ostream &OS) const override {
     std::string Marker = getMarkerForCodeBlock(Contents);
     // No need to pad from previous blocks, as they should end with a new line.
-    OS << Marker << Language << '\n' << Contents << '\n' << Marker << '\n';
+    OS << Marker << Language << '\n' << Contents;
+    if (!Contents.empty() && Contents.back() != '\n')
+      OS << '\n';
+    // Always end with an empty line to separate code blocks from following
+    // paragraphs.
+    OS << Marker << "\n\n";
   }
 
   void renderPlainText(llvm::raw_ostream &OS) const override {
diff --git a/clang-tools-extra/clangd/test/modules_no_cdb.test b/clang-tools-extra/clangd/test/modules_no_cdb.test
new file mode 100644
index 0000000000000..8f92be2c7b3f3
--- /dev/null
+++ b/clang-tools-extra/clangd/test/modules_no_cdb.test
@@ -0,0 +1,66 @@
+# A smoke test to check that clangd works without compilation database
+#
+# Windows have different escaping modes.
+# FIXME: We should add one for windows.
+# UNSUPPORTED: system-windows
+#
+# RUN: rm -fr %t
+# RUN: mkdir -p %t
+# RUN: split-file %s %t
+#
+# RUN: sed -e "s|DIR|%/t|g" %t/definition.jsonrpc.tmpl > %t/definition.jsonrpc
+#
+# RUN: clangd -experimental-modules-support -lit-test < %t/definition.jsonrpc \
+# RUN:      | FileCheck -strict-whitespace %t/definition.jsonrpc
+
+#--- A.h
+void printA();
+
+#--- Use.cpp
+#include "A.h"
+void foo() {
+    print
+}
+
+#--- definition.jsonrpc.tmpl
+{
+  "jsonrpc": "2.0",
+  "id": 0,
+  "method": "initialize",
+  "params": {
+    "processId": 123,
+    "rootPath": "clangd",
+    "capabilities": {
+      "textDocument": {
+        "completion": {
+          "completionItem": {
+            "snippetSupport": true
+          }
+        }
+      }
+    },
+    "trace": "off"
+  }
+}
+---
+{
+  "jsonrpc": "2.0",
+  "method": "textDocument/didOpen",
+  "params": {
+    "textDocument": {
+      "uri": "file://DIR/Use.cpp",
+      "languageId": "cpp",
+      "version": 1,
+      "text": "#include \"A.h\"\nvoid foo() {\n    print\n}\n"
+    }
+  }
+}
+
+# CHECK: "message"{{.*}}printA{{.*}}(fix available)
+
+---
+{"jsonrpc":"2.0","id":1,"method":"textDocument/completion","params":{"textDocument":{"uri":"file://DIR/Use.cpp"},"context":{"triggerKind":1},"position":{"line":2,"character":6}}}
+---
+{"jsonrpc":"2.0","id":2,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang-tools-extra/clangd/unittests/ASTTests.cpp b/clang-tools-extra/clangd/unittests/ASTTests.cpp
index d0bc3c4d7db98..76d46bad82224 100644
--- a/clang-tools-extra/clangd/unittests/ASTTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ASTTests.cpp
@@ -421,7 +421,7 @@ TEST(ClangdAST, GetQualification) {
       {
           R"cpp(
             namespace ns1 { namespace ns2 { void Foo(); } }
-            void insert(); // ns2::Foo
+            void insert(); // ns1::ns2::Foo
             namespace ns1 {
               void insert(); // ns2::Foo
               namespace ns2 {
@@ -429,7 +429,7 @@ TEST(ClangdAST, GetQualification) {
               }
             }
           )cpp",
-          {"ns2::", "ns2::", ""},
+          {"ns1::ns2::", "ns2::", ""},
           {"ns1::"},
       },
       {
@@ -531,7 +531,8 @@ TEST(ClangdAST, PrintType) {
     ASSERT_EQ(InsertionPoints.size(), Case.Types.size());
     for (size_t I = 0, E = InsertionPoints.size(); I != E; ++I) {
       const auto *DC = InsertionPoints[I];
-      EXPECT_EQ(printType(AST.getASTContext().getTypeDeclType(TargetDecl), *DC),
+      EXPECT_EQ(printType(AST.getASTContext().getTypeDeclType(TargetDecl), *DC,
+                          /*Placeholder=*/"", /*FullyQualify=*/true),
                 Case.Types[I]);
     }
   }
diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index d425070c7f3b7..9656eeaeb37ce 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -92,6 +92,7 @@ add_unittest(ClangdUnitTests ClangdTests
   SourceCodeTests.cpp
   StdLibTests.cpp
   SymbolCollectorTests.cpp
+  SymbolDocumentationTests.cpp
   SymbolInfoTests.cpp
   SyncAPI.cpp
   TUSchedulerTests.cpp
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 61bd6318b46cf..7640569128172 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -938,7 +938,7 @@ TEST(CompletionTest, IncludeInsertionRespectsQuotedAngledConfig) {
   {
     Config C;
     C.Style.AngledHeaders.push_back(
-        [](auto header) { return header == "bar.h"; });
+        [](auto header) { return header.contains("bar.h"); });
     WithContextValue WithCfg(Config::Key, std::move(C));
     Results = completions(TU, Test.point(), {Sym});
     EXPECT_THAT(Results.Completions,
@@ -947,7 +947,7 @@ TEST(CompletionTest, IncludeInsertionRespectsQuotedAngledConfig) {
   {
     Config C;
     C.Style.QuotedHeaders.push_back(
-        [](auto header) { return header == "bar.h"; });
+        [](auto header) { return header.contains("bar.h"); });
     WithContextValue WithCfg(Config::Key, std::move(C));
     Results = completions(TU, Test.point(), {Sym});
     EXPECT_THAT(Results.Completions,
@@ -4473,6 +4473,198 @@ TEST(CompletionTest, SkipExplicitObjectParameter) {
                                   snippetSuffix(""))));
   }
 }
+
+TEST(CompletionTest, MemberAccessInExplicitObjMemfn) {
+  Annotations Code(R"cpp(
+    struct A {
+      int member {};
+      int memberFnA(int a);
+      int memberFnA(this A&, float a);
+
+      void foo(this A& self) {
+        // Should not offer any members here, since 
+        // it needs to be referenced through `self`.
+        mem$c1^;
+        // should offer all results
+        self.mem$c2^;
+
+        [&]() {
+          // should not offer any results
+          mem$c3^;
+        }();
+      }
+    };
+  )cpp");
+
+  auto TU = TestTU::withCode(Code.code());
+  TU.ExtraArgs = {"-std=c++23"};
+
+  auto Preamble = TU.preamble();
+  ASSERT_TRUE(Preamble);
+
+  CodeCompleteOptions Opts{};
+
+  MockFS FS;
+  auto Inputs = TU.inputs(FS);
+
+  {
+    auto Result = codeComplete(testPath(TU.Filename), Code.point("c1"),
+                               Preamble.get(), Inputs, Opts);
+
+    EXPECT_THAT(Result.Completions, ElementsAre());
+  }
+  {
+    auto Result = codeComplete(testPath(TU.Filename), Code.point("c2"),
+                               Preamble.get(), Inputs, Opts);
+
+    EXPECT_THAT(
+        Result.Completions,
+        UnorderedElementsAre(named("member"),
+                             AllOf(named("memberFnA"), signature("(int a)"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("memberFnA"), signature("(float a)"),
+                                   snippetSuffix("(${1:float a})"))));
+  }
+  {
+    auto Result = codeComplete(testPath(TU.Filename), Code.point("c3"),
+                               Preamble.get(), Inputs, Opts);
+
+    EXPECT_THAT(Result.Completions, ElementsAre());
+  }
+}
+
+TEST(CompletionTest, ListExplicitObjectOverloads) {
+  Annotations Code(R"cpp(
+    struct S {
+      void foo1(int a);
+      void foo2(int a) const;
+      void foo2(this const S& self, float a);
+      void foo3(this const S& self, int a);
+      void foo4(this S& self, int a);
+    };
+
+    void S::foo1(int a) {
+      this->$c1^;
+    }
+
+    void S::foo2(int a) const {
+      this->$c2^;
+    }
+
+    void S::foo3(this const S& self, int a) {
+      self.$c3^;
+    }
+
+    void S::foo4(this S& self, int a) {
+      self.$c4^;
+    }
+
+    void test1(S s) {
+      s.$c5^;
+    }
+
+    void test2(const S s) {
+      s.$c6^;
+    }
+  )cpp");
+
+  auto TU = TestTU::withCode(Code.code());
+  TU.ExtraArgs = {"-std=c++23"};
+
+  auto Preamble = TU.preamble();
+  ASSERT_TRUE(Preamble);
+
+  CodeCompleteOptions Opts{};
+
+  MockFS FS;
+  auto Inputs = TU.inputs(FS);
+
+  {
+    auto Result = codeComplete(testPath(TU.Filename), Code.point("c1"),
+                               Preamble.get(), Inputs, Opts);
+    EXPECT_THAT(
+        Result.Completions,
+        UnorderedElementsAre(AllOf(named("foo1"), signature("(int a)"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo2"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo2"), signature("(float a) const"),
+                                   snippetSuffix("(${1:float a})")),
+                             AllOf(named("foo3"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo4"), signature("(int a)"),
+                                   snippetSuffix("(${1:int a})"))));
+  }
+  {
+    auto Result = codeComplete(testPath(TU.Filename), Code.point("c2"),
+                               Preamble.get(), Inputs, Opts);
+    EXPECT_THAT(
+        Result.Completions,
+        UnorderedElementsAre(AllOf(named("foo2"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo2"), signature("(float a) const"),
+                                   snippetSuffix("(${1:float a})")),
+                             AllOf(named("foo3"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})"))));
+  }
+  {
+    auto Result = codeComplete(testPath(TU.Filename), Code.point("c3"),
+                               Preamble.get(), Inputs, Opts);
+    EXPECT_THAT(
+        Result.Completions,
+        UnorderedElementsAre(AllOf(named("foo2"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo2"), signature("(float a) const"),
+                                   snippetSuffix("(${1:float a})")),
+                             AllOf(named("foo3"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})"))));
+  }
+  {
+    auto Result = codeComplete(testPath(TU.Filename), Code.point("c4"),
+                               Preamble.get(), Inputs, Opts);
+    EXPECT_THAT(
+        Result.Completions,
+        UnorderedElementsAre(AllOf(named("foo1"), signature("(int a)"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo2"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo2"), signature("(float a) const"),
+                                   snippetSuffix("(${1:float a})")),
+                             AllOf(named("foo3"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo4"), signature("(int a)"),
+                                   snippetSuffix("(${1:int a})"))));
+  }
+  {
+    auto Result = codeComplete(testPath(TU.Filename), Code.point("c5"),
+                               Preamble.get(), Inputs, Opts);
+    EXPECT_THAT(
+        Result.Completions,
+        UnorderedElementsAre(AllOf(named("foo1"), signature("(int a)"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo2"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo2"), signature("(float a) const"),
+                                   snippetSuffix("(${1:float a})")),
+                             AllOf(named("foo3"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo4"), signature("(int a)"),
+                                   snippetSuffix("(${1:int a})"))));
+  }
+  {
+    auto Result = codeComplete(testPath(TU.Filename), Code.point("c6"),
+                               Preamble.get(), Inputs, Opts);
+    EXPECT_THAT(
+        Result.Completions,
+        UnorderedElementsAre(AllOf(named("foo2"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})")),
+                             AllOf(named("foo2"), signature("(float a) const"),
+                                   snippetSuffix("(${1:float a})")),
+                             AllOf(named("foo3"), signature("(int a) const"),
+                                   snippetSuffix("(${1:int a})"))));
+  }
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/DumpASTTests.cpp b/clang-tools-extra/clangd/unittests/DumpASTTests.cpp
index cb2c17ad4ef0d..5c857d0b8ae3e 100644
--- a/clang-tools-extra/clangd/unittests/DumpASTTests.cpp
+++ b/clang-tools-extra/clangd/unittests/DumpASTTests.cpp
@@ -72,15 +72,14 @@ declaration: Namespace - root
       expression: BinaryOperator - +
         expression: ImplicitCast - LValueToRValue
           expression: DeclRef - x
-            specifier: TypeSpec
+            specifier: Type
               type: Record - S
         expression: ImplicitCast - LValueToRValue
           expression: Member - x
             expression: CXXBindTemporary
               expression: CXXTemporaryObject - S
-                type: Elaborated
+                type: Record - S
                   specifier: Namespace - root::
-                  type: Record - S
       )"},
       {R"cpp(
 namespace root {
@@ -104,14 +103,13 @@ declaration: Namespace - root
     expression: BinaryOperator - +
       expression: ImplicitCast - LValueToRValue
         expression: DeclRef - x
-          specifier: TypeSpec
+          specifier: Type
             type: Record - S
       expression: ImplicitCast - LValueToRValue
         expression: Member - x
           expression: CXXTemporaryObject - S
-            type: Elaborated
+            type: Record - S
               specifier: Namespace - root::
-              type: Record - S
       )"},
       {R"cpp(
 namespace root {
@@ -138,7 +136,7 @@ declaration: Namespace - root
                   type: Builtin - unsigned int
         statement: Return
           expression: DependentScopeDeclRef - value
-            specifier: TypeSpec
+            specifier: Type
               type: TemplateTypeParm - T
       )"},
       {R"cpp(
@@ -154,8 +152,7 @@ declaration: Var - root
         expression: DeclRef - operator+
       expression: MaterializeTemporary - lvalue
         expression: CXXTemporaryObject - Foo
-          type: Elaborated
-            type: Record - Foo
+          type: Record - Foo
       expression: IntegerLiteral - 42
       )"},
       {R"cpp(
diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
index 4d77f9d690ca0..f369e1b0341e8 100644
--- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
@@ -731,6 +731,12 @@ TEST_F(TargetDeclTest, BuiltinTemplates) {
     using type_pack_element = [[__type_pack_element]]<N, Pack...>;
   )cpp";
   EXPECT_DECLS("TemplateSpecializationTypeLoc", );
+
+  Code = R"cpp(
+    template <template <class...> class Templ, class... Types>
+    using dedup_types = Templ<[[__builtin_dedup_pack]]<Types...>...>;
+  )cpp";
+  EXPECT_DECLS("TemplateSpecializationTypeLoc", );
 }
 
 TEST_F(TargetDeclTest, MemberOfTemplate) {
@@ -992,7 +998,7 @@ TEST_F(TargetDeclTest, DependentTypes) {
       )cpp";
   EXPECT_DECLS("DependentNameTypeLoc", "struct B");
 
-  // Heuristic resolution of dependent type name which doesn't get a TypeLoc
+  // Heuristic resolution of dependent type name within a NestedNameSpecifierLoc
   Code = R"cpp(
         template <typename>
         struct A { struct B { struct C {}; }; };
@@ -1000,7 +1006,7 @@ TEST_F(TargetDeclTest, DependentTypes) {
         template <typename T>
         void foo(typename A<T>::[[B]]::C);
       )cpp";
-  EXPECT_DECLS("NestedNameSpecifierLoc", "struct B");
+  EXPECT_DECLS("DependentNameTypeLoc", "struct B");
 
   // Heuristic resolution of dependent type name whose qualifier is also
   // dependent
diff --git a/clang-tools-extra/clangd/unittests/HeadersTests.cpp b/clang-tools-extra/clangd/unittests/HeadersTests.cpp
index 751383e3b4650..440582e14239a 100644
--- a/clang-tools-extra/clangd/unittests/HeadersTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HeadersTests.cpp
@@ -344,6 +344,17 @@ TEST_F(HeadersTest, ShortenIncludesInSearchPathBracketed) {
   EXPECT_EQ(calculate(BarHeader), "<sub/bar.h>");
 }
 
+TEST_F(HeadersTest, ShortenIncludesInSearchPathBracketedFilterByFullPath) {
+  // The filter receives the full path of the header, so it is able to filter by
+  // the parent directory, even if it is part of the include search path
+  AngledHeaders.push_back([](auto Path) {
+    llvm::Regex Pattern("sub/.*");
+    return Pattern.match(Path);
+  });
+  std::string BarHeader = testPath("sub/bar.h");
+  EXPECT_EQ(calculate(BarHeader), "<bar.h>");
+}
+
 TEST_F(HeadersTest, ShortenedIncludeNotInSearchPath) {
   std::string BarHeader =
       llvm::sys::path::convert_to_slash(testPath("sub-2/bar.h"));
diff --git a/clang-tools-extra/clangd/unittests/HoverTests.cpp b/clang-tools-extra/clangd/unittests/HoverTests.cpp
index 12d260db7ea11..6407349d9af9b 100644
--- a/clang-tools-extra/clangd/unittests/HoverTests.cpp
+++ b/clang-tools-extra/clangd/unittests/HoverTests.cpp
@@ -2894,7 +2894,7 @@ TEST(Hover, All) {
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "this";
-            HI.Definition = "const Foo<T> *";
+            HI.Definition = "const ns::Foo<T> *";
           }},
       {
           R"cpp(// this expr for specialization class
@@ -2910,7 +2910,7 @@ TEST(Hover, All) {
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "this";
-            HI.Definition = "Foo<int> *";
+            HI.Definition = "ns::Foo<int> *";
           }},
       {
           R"cpp(// this expr for partial specialization struct
@@ -2926,7 +2926,7 @@ TEST(Hover, All) {
           )cpp",
           [](HoverInfo &HI) {
             HI.Name = "this";
-            HI.Definition = "const Foo<int, F> *";
+            HI.Definition = "const ns::Foo<int, F> *";
           }},
       {
           R"cpp(
@@ -3046,8 +3046,8 @@ TEST(Hover, All) {
          HI.Kind = index::SymbolKind::Function;
          HI.NamespaceScope = "";
          HI.Definition = "MyRect foobar()";
-         HI.Type = {"MyRect ()", "MyRect ()"};
-         HI.ReturnType = {"MyRect", "MyRect"};
+         HI.Type = {"MyRect ()", "struct MyRect ()"};
+         HI.ReturnType = {"MyRect", "struct MyRect"};
          HI.Parameters.emplace();
        }},
       {R"cpp(
@@ -3762,6 +3762,127 @@ provides Foo, Bar, Baz, Foobar, Qux and 1 more)"}};
   }
 }
 
+TEST(Hover, PresentDocumentation) {
+  struct {
+    const std::function<void(HoverInfo &)> Builder;
+    llvm::StringRef ExpectedRender;
+  } Cases[] = {
+      {[](HoverInfo &HI) {
+         HI.Kind = index::SymbolKind::Function;
+         HI.Documentation = "@brief brief doc\n\n"
+                            "longer doc";
+         HI.Definition = "void foo()";
+         HI.Name = "foo";
+       },
+       R"(### function `foo`
+
+---
+**@brief** brief doc
+
+longer doc
+
+---
+```cpp
+void foo()
+```)"},
+      {[](HoverInfo &HI) {
+         HI.Kind = index::SymbolKind::Function;
+         HI.Documentation = "@brief brief doc\n\n"
+                            "longer doc";
+         HI.Definition = "int foo()";
+         HI.ReturnType = "int";
+         HI.Name = "foo";
+       },
+       R"(### function `foo`
+
+---
+→ `int`
+
+**@brief** brief doc
+
+longer doc
+
+---
+```cpp
+int foo()
+```)"},
+      {[](HoverInfo &HI) {
+         HI.Kind = index::SymbolKind::Function;
+         HI.Documentation = "@brief brief doc\n\n"
+                            "longer doc\n@param a this is a param\n@return it "
+                            "returns something";
+         HI.Definition = "int foo(int a)";
+         HI.ReturnType = "int";
+         HI.Name = "foo";
+         HI.Parameters.emplace();
+         HI.Parameters->emplace_back();
+         HI.Parameters->back().Type = "int";
+         HI.Parameters->back().Name = "a";
+       },
+       R"(### function `foo`
+
+---
+→ `int`
+
+Parameters:
+
+- `int a` - this is a param
+
+**@brief** brief doc
+
+longer doc
+
+**@return** it returns something
+
+---
+```cpp
+int foo(int a)
+```)"},
+      {[](HoverInfo &HI) {
+         HI.Kind = index::SymbolKind::Function;
+         HI.Documentation = "@brief brief doc\n\n"
+                            "longer doc\n@param a this is a param\n@param b "
+                            "does not exist\n@return it returns something";
+         HI.Definition = "int foo(int a)";
+         HI.ReturnType = "int";
+         HI.Name = "foo";
+         HI.Parameters.emplace();
+         HI.Parameters->emplace_back();
+         HI.Parameters->back().Type = "int";
+         HI.Parameters->back().Name = "a";
+       },
+       R"(### function `foo`
+
+---
+→ `int`
+
+Parameters:
+
+- `int a` - this is a param
+
+**@brief** brief doc
+
+longer doc
+
+**@return** it returns something
+
+---
+```cpp
+int foo(int a)
+```)"},
+  };
+
+  for (const auto &C : Cases) {
+    HoverInfo HI;
+    C.Builder(HI);
+    Config Cfg;
+    Cfg.Hover.ShowAKA = true;
+    Cfg.Documentation.CommentFormat = Config::CommentFormatPolicy::Doxygen;
+    WithContextValue WithCfg(Config::Key, std::move(Cfg));
+    EXPECT_EQ(HI.present(MarkupKind::Markdown), C.ExpectedRender);
+  }
+}
+
 TEST(Hover, ParseDocumentation) {
   struct Case {
     llvm::StringRef Documentation;
@@ -4339,6 +4460,149 @@ constexpr u64 pow_with_mod(u64 a, u64 b, u64 p) {
   EXPECT_TRUE(H->Value);
   EXPECT_TRUE(H->Type);
 }
+
+TEST(Hover, FunctionParameters) {
+  struct {
+    const char *const Code;
+    const std::function<void(HoverInfo &)> ExpectedBuilder;
+    std::string ExpectedRender;
+  } Cases[] = {
+      {R"cpp(/// Function doc
+      void foo(int [[^a]]);
+    )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "a";
+         HI.Kind = index::SymbolKind::Parameter;
+         HI.NamespaceScope = "";
+         HI.LocalScope = "foo::";
+         HI.Type = "int";
+         HI.Definition = "int a";
+         HI.Documentation = "";
+       },
+       "### param `a`\n\n---\nType: `int`\n\n---\n```cpp\n// In foo\nint "
+       "a\n```"},
+      {R"cpp(/// Function doc
+      /// @param a this is doc for a
+      void foo(int [[^a]]);
+    )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "a";
+         HI.Kind = index::SymbolKind::Parameter;
+         HI.NamespaceScope = "";
+         HI.LocalScope = "foo::";
+         HI.Type = "int";
+         HI.Definition = "int a";
+         HI.Documentation = "this is doc for a";
+       },
+       "### param `a`\n\n---\nType: `int`\n\nthis is doc for "
+       "a\n\n---\n```cpp\n// In foo\nint a\n```"},
+      {R"cpp(/// Function doc
+      /// @param b this is doc for b
+      void foo(int [[^a]], int b);
+    )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "a";
+         HI.Kind = index::SymbolKind::Parameter;
+         HI.NamespaceScope = "";
+         HI.LocalScope = "foo::";
+         HI.Type = "int";
+         HI.Definition = "int a";
+         HI.Documentation = "";
+       },
+       "### param `a`\n\n---\nType: `int`\n\n---\n```cpp\n// In foo\nint "
+       "a\n```"},
+      {R"cpp(/// Function doc
+      /// @param b this is doc for \p b
+      void foo(int a, int [[^b]]);
+    )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "b";
+         HI.Kind = index::SymbolKind::Parameter;
+         HI.NamespaceScope = "";
+         HI.LocalScope = "foo::";
+         HI.Type = "int";
+         HI.Definition = "int b";
+         HI.Documentation = "this is doc for \\p b";
+       },
+       "### param `b`\n\n---\nType: `int`\n\nthis is doc for "
+       "`b`\n\n---\n```cpp\n// In foo\nint b\n```"},
+      {R"cpp(/// Function doc
+      /// @param b this is doc for \p b
+      template <typename T>
+      void foo(T a, T [[^b]]);
+    )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "b";
+         HI.Kind = index::SymbolKind::Parameter;
+         HI.NamespaceScope = "";
+         HI.LocalScope = "foo::";
+         HI.Type = "T";
+         HI.Definition = "T b";
+         HI.Documentation = "this is doc for \\p b";
+       },
+       "### param `b`\n\n---\nType: `T`\n\nthis is doc for "
+       "`b`\n\n---\n```cpp\n// In foo\nT b\n```"},
+      {R"cpp(/// Function doc
+      /// @param b this is <b>doc</b> <html-tag attribute/> <another-html-tag attribute="value">for</another-html-tag> \p b
+      void foo(int a, int [[^b]]);
+    )cpp",
+       [](HoverInfo &HI) {
+         HI.Name = "b";
+         HI.Kind = index::SymbolKind::Parameter;
+         HI.NamespaceScope = "";
+         HI.LocalScope = "foo::";
+         HI.Type = "int";
+         HI.Definition = "int b";
+         HI.Documentation =
+             "this is <b>doc</b> <html-tag attribute/> <another-html-tag "
+             "attribute=\"value\">for</another-html-tag> \\p b";
+       },
+       "### param `b`\n\n---\nType: `int`\n\nthis is \\<b>doc\\</b> "
+       "\\<html-tag attribute/> \\<another-html-tag "
+       "attribute=\"value\">for\\</another-html-tag> "
+       "`b`\n\n---\n```cpp\n// In foo\nint b\n```"},
+  };
+
+  // Create a tiny index, so tests above can verify documentation is fetched.
+  Symbol IndexSym = func("indexSymbol");
+  IndexSym.Documentation = "comment from index";
+  SymbolSlab::Builder Symbols;
+  Symbols.insert(IndexSym);
+  auto Index =
+      MemIndex::build(std::move(Symbols).build(), RefSlab(), RelationSlab());
+
+  for (const auto &Case : Cases) {
+    SCOPED_TRACE(Case.Code);
+
+    Annotations T(Case.Code);
+    TestTU TU = TestTU::withCode(T.code());
+    auto AST = TU.build();
+    Config Cfg;
+    Cfg.Hover.ShowAKA = true;
+    Cfg.Documentation.CommentFormat = Config::CommentFormatPolicy::Doxygen;
+    WithContextValue WithCfg(Config::Key, std::move(Cfg));
+    auto H = getHover(AST, T.point(), format::getLLVMStyle(), Index.get());
+    ASSERT_TRUE(H);
+    HoverInfo Expected;
+    Expected.SymRange = T.range();
+    Case.ExpectedBuilder(Expected);
+
+    EXPECT_EQ(H->present(MarkupKind::Markdown), Case.ExpectedRender);
+    EXPECT_EQ(H->NamespaceScope, Expected.NamespaceScope);
+    EXPECT_EQ(H->LocalScope, Expected.LocalScope);
+    EXPECT_EQ(H->Name, Expected.Name);
+    EXPECT_EQ(H->Kind, Expected.Kind);
+    EXPECT_EQ(H->Documentation, Expected.Documentation);
+    EXPECT_EQ(H->Definition, Expected.Definition);
+    EXPECT_EQ(H->Type, Expected.Type);
+    EXPECT_EQ(H->ReturnType, Expected.ReturnType);
+    EXPECT_EQ(H->Parameters, Expected.Parameters);
+    EXPECT_EQ(H->TemplateParameters, Expected.TemplateParameters);
+    EXPECT_EQ(H->SymRange, Expected.SymRange);
+    EXPECT_EQ(H->Value, Expected.Value);
+  }
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
index e0cd955bb1c9a..99e728c40063d 100644
--- a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
+++ b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
@@ -1295,14 +1295,7 @@ TEST(TypeHints, NoQualifiers) {
       }
     }
   )cpp",
-                  ExpectedHint{": S1", "x"},
-                  // FIXME: We want to suppress scope specifiers
-                  //        here because we are into the whole
-                  //        brevity thing, but the ElaboratedType
-                  //        printer does not honor the SuppressScope
-                  //        flag by design, so we need to extend the
-                  //        PrintingPolicy to support this use case.
-                  ExpectedHint{": S2::Inner<int>", "y"});
+                  ExpectedHint{": S1", "x"}, ExpectedHint{": Inner<int>", "y"});
 }
 
 TEST(TypeHints, Lambda) {
diff --git a/clang-tools-extra/clangd/unittests/QualityTests.cpp b/clang-tools-extra/clangd/unittests/QualityTests.cpp
index 619ea32115357..4954659a45e02 100644
--- a/clang-tools-extra/clangd/unittests/QualityTests.cpp
+++ b/clang-tools-extra/clangd/unittests/QualityTests.cpp
@@ -121,7 +121,9 @@ TEST(QualityTests, SymbolRelevanceSignalExtraction) {
 
   SymbolRelevanceSignals Relevance;
   Relevance.merge(CodeCompletionResult(&findDecl(AST, "deprecated"),
-                                       /*Priority=*/42, nullptr, false,
+                                       /*Priority=*/42,
+                                       /*Qualifier=*/std::nullopt,
+                                       /*QualifierIsInformative=*/false,
                                        /*Accessible=*/false));
   EXPECT_EQ(Relevance.NameMatch, SymbolRelevanceSignals().NameMatch);
   EXPECT_TRUE(Relevance.Forbidden);
@@ -487,13 +489,15 @@ TEST(QualityTests, ItemWithFixItsRankedDown) {
   auto AST = Header.build();
 
   SymbolRelevanceSignals RelevanceWithFixIt;
-  RelevanceWithFixIt.merge(CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr,
-                                                false, true, {FixItHint{}}));
+  RelevanceWithFixIt.merge(CodeCompletionResult(
+      &findDecl(AST, "x"), /*Priority=*/0, /*Qualifier=*/std::nullopt,
+      /*QualifierIsInformative=*/false, /*Accessible=*/true, {FixItHint{}}));
   EXPECT_TRUE(RelevanceWithFixIt.NeedsFixIts);
 
   SymbolRelevanceSignals RelevanceWithoutFixIt;
-  RelevanceWithoutFixIt.merge(
-      CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr, false, true, {}));
+  RelevanceWithoutFixIt.merge(CodeCompletionResult(
+      &findDecl(AST, "x"), /*Priority=*/0, /*Qualifier=*/std::nullopt,
+      /*QualifierIsInformative=*/false, /*Accessible=*/true, {}));
   EXPECT_FALSE(RelevanceWithoutFixIt.NeedsFixIts);
 
   EXPECT_LT(RelevanceWithFixIt.evaluateHeuristics(),
diff --git a/clang-tools-extra/clangd/unittests/SelectionTests.cpp b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
index aaaf758e72236..3df19d8fc174d 100644
--- a/clang-tools-extra/clangd/unittests/SelectionTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
@@ -104,9 +104,9 @@ TEST(SelectionTest, CommonAncestor) {
       {
           R"cpp(
             template <typename T>
-            int x = [[T::^U::]]ccc();
+            int x = T::[[^U]]::ccc();
           )cpp",
-          "NestedNameSpecifierLoc",
+          "DependentNameTypeLoc",
       },
       {
           R"cpp(
diff --git a/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp b/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp
new file mode 100644
index 0000000000000..69eb13b2142d2
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/SymbolDocumentationTests.cpp
@@ -0,0 +1,161 @@
+//===-- SymbolDocumentationTests.cpp --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "SymbolDocumentation.h"
+
+#include "support/Markup.h"
+#include "clang/Basic/CommentOptions.h"
+#include "llvm/ADT/StringRef.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+
+TEST(SymbolDocumentation, Parse) {
+
+  CommentOptions CommentOpts;
+
+  struct Case {
+    llvm::StringRef Documentation;
+    llvm::StringRef ExpectedRenderEscapedMarkdown;
+    llvm::StringRef ExpectedRenderMarkdown;
+    llvm::StringRef ExpectedRenderPlainText;
+  } Cases[] = {
+      {
+          "foo bar",
+          "foo bar",
+          "foo bar",
+          "foo bar",
+      },
+      {
+          "foo\nbar\n",
+          "foo\nbar",
+          "foo\nbar",
+          "foo bar",
+      },
+      {
+          "foo\n\nbar\n",
+          "foo\n\nbar",
+          "foo\n\nbar",
+          "foo\n\nbar",
+      },
+      {
+          "foo \\p bar baz",
+          "foo `bar` baz",
+          "foo `bar` baz",
+          "foo bar baz",
+      },
+      {
+          "foo \\e bar baz",
+          "foo \\*bar\\* baz",
+          "foo *bar* baz",
+          "foo *bar* baz",
+      },
+      {
+          "foo \\b bar baz",
+          "foo \\*\\*bar\\*\\* baz",
+          "foo **bar** baz",
+          "foo **bar** baz",
+      },
+      {
+          "foo \\ref bar baz",
+          "foo \\*\\*\\\\ref\\*\\* \\*bar\\* baz",
+          "foo **\\ref** *bar* baz",
+          "foo **\\ref** *bar* baz",
+      },
+      {
+          "foo @ref bar baz",
+          "foo \\*\\*@ref\\*\\* \\*bar\\* baz",
+          "foo **@ref** *bar* baz",
+          "foo **@ref** *bar* baz",
+      },
+      {
+          "\\brief this is a \\n\nbrief description",
+          "\\*\\*\\\\brief\\*\\* this is a   \nbrief description",
+          "**\\brief** this is a   \nbrief description",
+          "**\\brief** this is a\nbrief description",
+      },
+      {
+          "\\throw exception foo",
+          "\\*\\*\\\\throw\\*\\* \\*exception\\* foo",
+          "**\\throw** *exception* foo",
+          "**\\throw** *exception* foo",
+      },
+      {
+          "\\brief this is a brief description\n\n\\li item 1\n\\li item "
+          "2\n\\arg item 3",
+          "\\*\\*\\\\brief\\*\\* this is a brief description\n\n- item 1\n\n- "
+          "item "
+          "2\n\n- "
+          "item 3",
+          "**\\brief** this is a brief description\n\n- item 1\n\n- item "
+          "2\n\n- "
+          "item 3",
+          "**\\brief** this is a brief description\n\n- item 1\n\n- item "
+          "2\n\n- "
+          "item 3",
+      },
+      {
+          "\\defgroup mygroup this is a group\nthis is not a group description",
+          "\\*\\*@defgroup\\*\\* `mygroup this is a group`\n\nthis is not a "
+          "group "
+          "description",
+          "**@defgroup** `mygroup this is a group`\n\nthis is not a group "
+          "description",
+          "**@defgroup** `mygroup this is a group`\n\nthis is not a group "
+          "description",
+      },
+      {
+          "\\verbatim\nthis is a\nverbatim block containing\nsome verbatim "
+          "text\n\\endverbatim",
+          "\\*\\*@verbatim\\*\\*\n\n```\nthis is a\nverbatim block "
+          "containing\nsome "
+          "verbatim text\n```\n\n\\*\\*@endverbatim\\*\\*",
+          "**@verbatim**\n\n```\nthis is a\nverbatim block containing\nsome "
+          "verbatim text\n```\n\n**@endverbatim**",
+          "**@verbatim**\n\nthis is a\nverbatim block containing\nsome "
+          "verbatim text\n\n**@endverbatim**",
+      },
+      {
+          "@param foo this is a parameter\n@param bar this is another "
+          "parameter",
+          "",
+          "",
+          "",
+      },
+      {
+          "@brief brief docs\n\n@param foo this is a parameter\n\nMore "
+          "description\ndocumentation",
+          "\\*\\*@brief\\*\\* brief docs\n\nMore description\ndocumentation",
+          "**@brief** brief docs\n\nMore description\ndocumentation",
+          "**@brief** brief docs\n\nMore description documentation",
+      },
+      {
+          "<b>this is a bold text</b>\nnormal text\n<i>this is an italic "
+          "text</i>\n<code>this is a code block</code>",
+          "\\<b>this is a bold text\\</b>\nnormal text\n\\<i>this is an italic "
+          "text\\</i>\n\\<code>this is a code block\\</code>",
+          "\\<b>this is a bold text\\</b>\nnormal text\n\\<i>this is an italic "
+          "text\\</i>\n\\<code>this is a code block\\</code>",
+          "<b>this is a bold text</b> normal text <i>this is an italic "
+          "text</i> <code>this is a code block</code>",
+      },
+  };
+  for (const auto &C : Cases) {
+    markup::Document Doc;
+    SymbolDocCommentVisitor SymbolDoc(C.Documentation, CommentOpts);
+
+    SymbolDoc.docToMarkup(Doc);
+
+    EXPECT_EQ(Doc.asPlainText(), C.ExpectedRenderPlainText);
+    EXPECT_EQ(Doc.asMarkdown(), C.ExpectedRenderMarkdown);
+    EXPECT_EQ(Doc.asEscapedMarkdown(), C.ExpectedRenderEscapedMarkdown);
+  }
+}
+
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/support/MarkupTests.cpp b/clang-tools-extra/clangd/unittests/support/MarkupTests.cpp
index 482f230fb86fe..5f91f31557176 100644
--- a/clang-tools-extra/clangd/unittests/support/MarkupTests.cpp
+++ b/clang-tools-extra/clangd/unittests/support/MarkupTests.cpp
@@ -463,6 +463,7 @@ TEST(Document, Separators) {
 ```cpp
 test
 ```
+
 bar)md";
   EXPECT_EQ(D.asEscapedMarkdown(), ExpectedMarkdown);
   EXPECT_EQ(D.asMarkdown(), ExpectedMarkdown);
@@ -559,6 +560,7 @@ foo
   bar
   baz
 ```
+
 ```cpp
 foo
 ```)md";
@@ -571,6 +573,12 @@ foo
 
 foo)pt";
   EXPECT_EQ(D.asPlainText(), ExpectedPlainText);
+
+  Document D2;
+  D2.addCodeBlock("");
+  EXPECT_EQ(D2.asEscapedMarkdown(), "```cpp\n```");
+  EXPECT_EQ(D2.asMarkdown(), "```cpp\n```");
+  EXPECT_EQ(D2.asPlainText(), "");
 }
 
 TEST(BulletList, Render) {
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 2dc5c73073cf8..4fee4f93908da 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -1,3 +1,6 @@
+.. If you want to modify sections/contents permanently, you should modify both
+   ReleaseNotes.rst and ReleaseNotesTemplate.txt.
+
 ====================================================
 Extra Clang Tools |release| |ReleaseNotesTitle|
 ====================================================
@@ -101,6 +104,14 @@ Improvements to clang-query
 Improvements to clang-tidy
 --------------------------
 
+- The :program:`check_clang_tidy.py` tool now recognizes the ``-std`` argument
+  when run over C files. If ``-std`` is not specified, it defaults to
+  ``c99-or-later``.
+
+- :program:`clang-tidy` no longer attemps to analyze code from system headers
+  by default, greatly improving performance. This behavior is disabled if the
+  `SystemHeaders` option is enabled.
+
 - The :program:`run-clang-tidy.py` and :program:`clang-tidy-diff.py` scripts
   now run checks in parallel by default using all available hardware threads.
   Both scripts display the number of threads being used in their output.
@@ -109,6 +120,12 @@ Improvements to clang-tidy
   `enable-check-profile` to enable per-check timing profiles and print a
   report based on all analyzed files.
 
+- Improved documentation of the `-line-filter` command-line flag of
+  :program:`clang-tidy` and :program:`run-clang-tidy.py`.
+
+- Improved :program:`clang-tidy` option `-quiet` by suppressing diagnostic
+  count messages.
+
 New checks
 ^^^^^^^^^^
 
@@ -118,12 +135,25 @@ New checks
   Detects default initialization (to 0) of variables with ``enum`` type where
   the enum has no enumerator with value of 0.
 
+- New :doc:`cppcoreguidelines-pro-bounds-avoid-unchecked-container-access
+  <clang-tidy/checks/cppcoreguidelines/pro-bounds-avoid-unchecked-container-access>`
+  check.
+
+  Finds calls to ``operator[]`` in STL containers and suggests replacing them
+  with safe alternatives.
+
 - New :doc:`llvm-mlir-op-builder
   <clang-tidy/checks/llvm/use-new-mlir-op-builder>` check.
 
   Checks for uses of MLIR's old/to be deprecated ``OpBuilder::create<T>`` form
   and suggests using ``T::create`` instead.
 
+- New :doc:`misc-override-with-different-visibility
+  <clang-tidy/checks/misc/override-with-different-visibility>` check.
+
+  Finds virtual function overrides with different visibility than the function
+  in the base class.
+
 New check aliases
 ^^^^^^^^^^^^^^^^^
 
@@ -134,9 +164,13 @@ Changes in existing checks
   <clang-tidy/checks/bugprone/infinite-loop>` check by adding detection for
   variables introduced by structured bindings.
 
+- Improved :doc:`bugprone-narrowing-conversions
+  <clang-tidy/checks/bugprone/narrowing-conversions>` check by fixing
+  false positive from analysis of a conditional expression in C.
+
 - Improved :doc:`bugprone-reserved-identifier
   <clang-tidy/checks/bugprone/reserved-identifier>` check by ignoring
-  declarations in system headers.
+  declarations and macros in system headers.
 
 - Improved :doc:`bugprone-signed-char-misuse
   <clang-tidy/checks/bugprone/signed-char-misuse>` check by fixing
@@ -153,9 +187,23 @@ Changes in existing checks
   an additional matcher that generalizes the copy-and-swap idiom pattern
   detection.
 
+- Improved :doc:`cppcoreguidelines-prefer-member-initializer
+  <clang-tidy/checks/cppcoreguidelines/prefer-member-initializer>` check to
+  avoid false positives on inherited members in class templates.
+
+- Improved :doc:`cppcoreguidelines-pro-bounds-pointer-arithmetic
+  <clang-tidy/checks/cppcoreguidelines/pro-bounds-pointer-arithmetic>` check
+  adding an option to allow pointer arithmetic via prefix/postfix increment or
+  decrement operators.
+
 - Improved :doc:`misc-header-include-cycle
   <clang-tidy/checks/misc/header-include-cycle>` check performance.
 
+- Improved :doc:`modernize-use-constraints
+  <clang-tidy/checks/modernize/use-constraints>` check by fixing a crash on
+  uses of non-standard ``enable_if`` with a signature different from
+  ``std::enable_if`` (such as ``boost::enable_if``).
+
 - Improved :doc:`modernize-use-designated-initializers
   <clang-tidy/checks/modernize/use-designated-initializers>` check to
   suggest using designated initializers for aliased aggregate types.
@@ -182,9 +230,14 @@ Changes in existing checks
   <clang-tidy/checks/portability/template-virtual-member-function>` check to
   avoid false positives on pure virtual member functions.
 
+- Improved :doc:`readability-container-size-empty
+  <clang-tidy/checks/readability/container-size-empty>` check by correctly
+  generating fix-it hints when size method is called from implicit ``this``.
+
 - Improved :doc:`readability-identifier-naming
   <clang-tidy/checks/readability/identifier-naming>` check by ignoring
-  declarations in system headers.
+  declarations and macros in system headers. The documentation is also improved
+  to differentiate the general options from the specific ones.
 
 - Improved :doc:`readability-qualified-auto
   <clang-tidy/checks/readability/qualified-auto>` check by adding the option
diff --git a/clang-tools-extra/docs/ReleaseNotesTemplate.txt b/clang-tools-extra/docs/ReleaseNotesTemplate.txt
new file mode 100644
index 0000000000000..b17799b3b557d
--- /dev/null
+++ b/clang-tools-extra/docs/ReleaseNotesTemplate.txt
@@ -0,0 +1,122 @@
+.. If you want to modify sections/contents permanently, you should modify both
+   ReleaseNotes.rst and ReleaseNotesTemplate.txt.
+
+====================================================
+Extra Clang Tools |release| |ReleaseNotesTitle|
+====================================================
+
+.. contents::
+   :local:
+   :depth: 3
+
+Written by the `LLVM Team <https://llvm.org/>`_
+
+.. only:: PreRelease
+
+  .. warning::
+     These are in-progress notes for the upcoming Extra Clang Tools |version| release.
+     Release notes for previous releases can be found on
+     `the Download Page <https://releases.llvm.org/download.html>`_.
+
+Introduction
+============
+
+This document contains the release notes for the Extra Clang Tools, part of the
+Clang release |release|. Here we describe the status of the Extra Clang Tools in
+some detail, including major improvements from the previous release and new
+feature work. All LLVM releases may be downloaded from the `LLVM releases web
+site <https://llvm.org/releases/>`_.
+
+For more information about Clang or LLVM, including information about
+the latest release, please see the `Clang Web Site <https://clang.llvm.org>`_ or
+the `LLVM Web Site <https://llvm.org>`_.
+
+Note that if you are reading this file from a Git checkout or the
+main Clang web page, this document applies to the *next* release, not
+the current one. To see the release notes for a specific release, please
+see the `releases page <https://llvm.org/releases/>`_.
+
+What's New in Extra Clang Tools |release|?
+==========================================
+
+Some of the major new features and improvements to Extra Clang Tools are listed
+here. Generic improvements to Extra Clang Tools as a whole or to its underlying
+infrastructure are described first, followed by tool-specific sections.
+
+Major New Features
+------------------
+
+Improvements to clangd
+----------------------
+
+Inlay hints
+^^^^^^^^^^^
+
+Diagnostics
+^^^^^^^^^^^
+
+Semantic Highlighting
+^^^^^^^^^^^^^^^^^^^^^
+
+Compile flags
+^^^^^^^^^^^^^
+
+Hover
+^^^^^
+
+Code completion
+^^^^^^^^^^^^^^^
+
+Code actions
+^^^^^^^^^^^^
+
+Signature help
+^^^^^^^^^^^^^^
+
+Cross-references
+^^^^^^^^^^^^^^^^
+
+Objective-C
+^^^^^^^^^^^
+
+Miscellaneous
+^^^^^^^^^^^^^
+
+Improvements to clang-doc
+-------------------------
+
+Improvements to clang-query
+---------------------------
+
+Improvements to clang-tidy
+--------------------------
+
+New checks
+^^^^^^^^^^
+
+New check aliases
+^^^^^^^^^^^^^^^^^
+
+Changes in existing checks
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Removed checks
+^^^^^^^^^^^^^^
+
+Miscellaneous
+^^^^^^^^^^^^^
+
+Improvements to include-fixer
+-----------------------------
+
+Improvements to clang-include-fixer
+-----------------------------------
+
+Improvements to modularize
+--------------------------
+
+Improvements to pp-trace
+------------------------
+
+Clang-tidy Visual Studio plugin
+-------------------------------
diff --git a/clang-tools-extra/docs/clang-change-namespace.rst b/clang-tools-extra/docs/clang-change-namespace.rst
new file mode 100644
index 0000000000000..1eab83f5069b6
--- /dev/null
+++ b/clang-tools-extra/docs/clang-change-namespace.rst
@@ -0,0 +1,314 @@
+======================
+Clang-Change-Namespace
+======================
+
+.. contents::
+
+.. toctree::
+  :maxdepth: 1
+
+:program:`clang-change-namespace` can be used to change the surrounding
+namespaces of class/function definitions.
+
+Classes/functions in the moved namespace will have new namespaces while
+references to symbols (e.g. types, functions) which are not defined in the
+changed namespace will be correctly qualified by prepending namespace specifiers
+before them. This will try to add shortest namespace specifiers possible.
+
+When a symbol reference needs to be fully-qualified, this adds a `::` prefix to
+the namespace specifiers unless the new namespace is the global namespace. For
+classes, only classes that are declared/defined in the given namespace in
+specified files will be moved: forward declarations will remain in the old
+namespace. The will be demonstrated in the next example.
+
+Example usage
+-------------
+
+For example, consider this `test.cc` example here with the forward declared
+class `FWD` and the defined class `A`, both in the namespace `a`.
+
+.. code-block:: c++
+
+  namespace a {
+  class FWD;
+  class A {
+    FWD *fwd;
+  };
+  } // namespace a
+
+And now let's change the namespace `a` to `x`.
+
+.. code-block:: console
+
+  clang-change-namespace \
+    --old_namespace "a" \
+    --new_namespace "x" \
+    --file_pattern "test.cc" \
+    --i \
+    test.cc
+
+Note that in the code below there's still the forward decalred class `FWD` that
+stayed in the namespace `a`. It wasn't moved to the new namespace because it
+wasn't defined/declared here in `a` but only forward declared.
+
+.. code-block:: c++
+
+  namespace a {
+  class FWD;
+  } // namespace a
+  namespace x {
+
+  class A {
+    a::FWD *fwd;
+  };
+  } // namespace x
+
+
+Another example
+---------------
+
+Consider this `test.cc` file:
+
+.. code-block:: c++
+
+  namespace na {
+  class X {};
+  namespace nb {
+  class Y {
+    X x;
+  };
+  } // namespace nb
+  } // namespace na
+
+To move the definition of class `Y` from namespace `na::nb` to `x::y`, run:
+
+.. code-block:: console
+
+  clang-change-namespace \
+    --old_namespace "na::nb" \
+    --new_namespace "x::y" \
+    --file_pattern "test.cc" \
+    --i \
+    test.cc
+
+This will overwrite `test.cc` to look like this:
+
+.. code-block:: c++
+
+  namespace na {
+  class X {};
+
+  } // namespace na
+  namespace x {
+  namespace y {
+  class Y {
+    na::X x;
+  };
+  } // namespace y
+  } // namespace x
+
+Note, that we've successfully moved the class `Y` from namespace `na::nb` to
+namespace `x::y`.
+
+Caveats
+=======
+
+Content already exists in new namespace
+---------------------------------------
+
+Consider this `test.cc` example that defines two `class A` one inside the
+namespace `a` and one in namespace `b`:
+
+.. code-block:: c++
+
+  namespace a {
+  class A {
+      int classAFromWithinNamespace_a;
+  };
+  } // namespace a
+
+  namespace b {
+  class A {
+      int classAFromWithinNamespace_b;
+  };
+  } //namespace b
+
+Let's move everything from namespace `a` to namespace `b`:
+
+.. code-block:: console
+
+  clang-change-namespace \
+    --old_namespace "a" \
+    --new_namespace "b" \
+    --file_pattern test.cc \
+    test.cc
+
+As expected we now have to definitions of `class A` inside the namespace `b`:
+
+.. code-block:: c++
+
+  namespace b {
+  class A {
+    int classAFromWithinNamespace_a;
+  };
+  } // namespace b
+
+  namespace b {
+  class A {
+      int classAFromWithinNamespace_b;
+  };
+  } //namespace b
+
+The re-factoring looks correct but the code will not compile due to the name
+duplication. It is not up to the tool to ensure compilability in that sense.
+But one has to be aware of that.
+
+Inline namespace doesn't work
+-----------------------------
+
+Consider this usage of two versions of implementations for a `greet` function:
+
+.. code-block:: c++
+
+  #include <cstdio>
+
+  namespace Greeter {
+  inline namespace Version1 {
+    const char* greet() { return "Hello from version 1!"; }
+  } // namespace Version1
+  namespace Version2 {
+    const char* greet() { return "Hello from version 2!"; }
+  } // namespace Version2
+  } // namespace Greeter
+
+  int main(int argc, char* argv[]) {
+    printf("%s\n", Greeter::greet());
+    return 0;
+  }
+
+Note, that currently `Greeter::greet()` will result in a call to
+`Greeter::Version1::greet()` because that's the inlined namespace.
+
+Let's say you want to move one and make `Version2` the default now and remove
+the `inline` from the `Version1`. First let's try to turn `namespace Version2`
+into `inline namespace Version2`:
+
+.. code-block:: console
+
+  clang-change-namespace \
+    --old_namespace "Greeter::Version2" \
+    --new_namespace "inline Version2" \
+    --file_pattern main.cc main.cc
+
+But this will put the `inline` keyword in the wrong place resulting in:
+
+.. code-block:: c++
+
+  #include <cstdio>
+
+  namespace Greeter {
+  inline namespace Version1 {
+          const char* greet() { return "Hello from version 1!"; }
+  } // namespace Version1
+
+  } // namespace Greeter
+  namespace inline Greeter {
+  namespace Version2 {
+  const char *greet() { return "Hello from version 2!"; }
+  } // namespace Version2
+  } // namespace inline Greeter
+
+  int main(int argc, char* argv[]) {
+          printf("%s\n", Greeter::greet());
+          return 0;
+  }
+
+One cannot use `:program:`clang-change-namespace` to inline a namespace.
+
+Symbol references not updated
+-----------------------------
+
+Consider this `test.cc` file:
+
+.. code-block:: c++
+
+  namespace old {
+  struct foo {};
+  }  // namespace old
+
+  namespace b {
+  old::foo g_foo;
+  }  // namespace b
+
+Notice that namespace `b` defines a global variable of type `old::foo`. If we
+now change the name of the `old` namespace to `modern`, the reference will not
+be updated:
+
+.. code-block:: console
+
+  clang-change-namespace \
+    --old_namespace "old" \
+    --new_namespace "modern" \
+    --file_pattern test.cc \
+    test.cc
+
+.. code-block:: c++
+
+  namespace modern {
+  struct foo {};
+  } // namespace modern
+
+  namespace b {
+  old::foo g_foo;
+  }  // namespace b
+
+`g_foo` is still of the no longer existing type `old::foo` while instead it
+should use `modern::foo`.
+
+Only symbol references in the moved namespace are updated, not outside of it.
+
+
+:program:`clang-change-namespace` Command Line Options
+======================================================
+
+.. option:: --allowed_file=<string>
+
+  A file containing regexes of symbol names that are not expected to be updated
+  when changing namespaces around them.
+
+.. option:: --dump_result
+
+  Dump new file contents in YAML, if specified.
+
+.. option:: --extra-arg=<string>
+
+  Additional argument to append to the compiler command line
+
+.. option:: --extra-arg-before=<string>
+
+  Additional argument to prepend to the compiler command line
+
+.. option:: --file_pattern=<string>
+
+  Only rename namespaces in files that match the given regular expression
+  pattern.
+
+.. option:: -i
+
+  Inplace edit <file>s, if specified.
+
+.. option:: --new_namespace=<string>
+
+  New namespace. Use `""` when you target the global namespace.
+
+.. option:: --old_namespace=<string>
+
+  Old namespace.
+
+.. option:: -p <string>
+
+  Build path
+
+.. option:: --style=<string>
+
+  The style name used for reformatting.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/pro-bounds-avoid-unchecked-container-access.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/pro-bounds-avoid-unchecked-container-access.rst
new file mode 100644
index 0000000000000..556d90213b216
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/pro-bounds-avoid-unchecked-container-access.rst
@@ -0,0 +1,64 @@
+.. title:: clang-tidy - cppcoreguidelines-pro-bounds-avoid-unchecked-container-access
+
+cppcoreguidelines-pro-bounds-avoid-unchecked-container-access
+=============================================================
+
+Finds calls to ``operator[]`` in STL containers and suggests replacing them
+with safe alternatives.
+Safe alternatives include STL ``at`` or GSL ``at`` functions, ``begin()`` or
+``end()`` functions, ``range-for`` loops, ``std::span``, or an appropriate
+function from ``<algorithms>``.
+
+For example, both
+
+.. code-block:: c++
+
+  std::vector<int> a;
+  int b = a[4];
+
+and
+
+.. code-block:: c++
+
+  std::unique_ptr<vector> a;
+  int b = a[0];
+
+will generate a warning.
+
+STL containers for which ``operator[]`` is well-defined for all inputs are excluded
+from this check (e.g.: ``std::map::operator[]``).
+
+This check enforces part of the `SL.con.3
+<https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#slcon3-avoid-bounds-errors>`
+guideline and is part of the `Bounds Safety (Bounds 4)
+<https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Pro-bounds-arrayindex>`
+profile from the C++ Core Guidelines.
+
+Options
+-------
+
+.. option:: ExcludeClasses
+
+    Semicolon-delimited list of class names for overwriting the default
+    exclusion list. The default is:
+    `::std::map;::std::unordered_map;::std::flat_map`.
+    
+.. option:: FixMode
+
+    Determines what fixes are suggested. Either `none`, `at` (use 
+    ``a.at(index)`` if a fitting function exists) or `function` (use a 
+    function ``f(a, index)``). The default is `none`.
+
+.. option:: FixFunction
+
+    The function to use in the `function` mode. For C++23 and beyond, the
+    passed function must support the empty subscript operator, i.e., the case
+    where ``a[]`` becomes ``f(a)``. :option:`FixFunctionEmptyArgs` can be
+    used to override the suggested function in that case. The default is `gsl::at`. 
+
+.. option:: FixFunctionEmptyArgs
+
+    The function to use in the `function` mode for the empty subscript operator
+    case in C++23 and beyond only. If no fixes should be made for empty
+    subscript operators, pass an empty string. In that case, only the warnings
+    will be printed. The default is the value of :option:`FixFunction`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/pro-bounds-pointer-arithmetic.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/pro-bounds-pointer-arithmetic.rst
index 12a8f60184fe5..a3f13714e809c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/pro-bounds-pointer-arithmetic.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/pro-bounds-pointer-arithmetic.rst
@@ -13,3 +13,11 @@ arrays of data.
 This rule is part of the `Bounds safety (Bounds 1)
 <https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Pro-bounds-arithmetic>`_
 profile from the C++ Core Guidelines.
+
+Options
+-------
+
+.. option:: AllowIncrementDecrementOperators
+
+   When enabled, the check will allow using the prefix/postfix increment or
+   decrement operators on pointers. Default is ``false``.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index b6444eb3c9aec..5e3ffc4f8aca3 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -201,6 +201,7 @@ Clang-Tidy Checks
    :doc:`cppcoreguidelines-owning-memory <cppcoreguidelines/owning-memory>`,
    :doc:`cppcoreguidelines-prefer-member-initializer <cppcoreguidelines/prefer-member-initializer>`, "Yes"
    :doc:`cppcoreguidelines-pro-bounds-array-to-pointer-decay <cppcoreguidelines/pro-bounds-array-to-pointer-decay>`,
+   :doc:`cppcoreguidelines-pro-bounds-avoid-unchecked-container-access <cppcoreguidelines/pro-bounds-avoid-unchecked-container-access>`, "Yes"
    :doc:`cppcoreguidelines-pro-bounds-constant-array-index <cppcoreguidelines/pro-bounds-constant-array-index>`, "Yes"
    :doc:`cppcoreguidelines-pro-bounds-pointer-arithmetic <cppcoreguidelines/pro-bounds-pointer-arithmetic>`,
    :doc:`cppcoreguidelines-pro-type-const-cast <cppcoreguidelines/pro-type-const-cast>`,
@@ -271,6 +272,7 @@ Clang-Tidy Checks
    :doc:`misc-no-recursion <misc/no-recursion>`,
    :doc:`misc-non-copyable-objects <misc/non-copyable-objects>`,
    :doc:`misc-non-private-member-variables-in-classes <misc/non-private-member-variables-in-classes>`,
+   :doc:`misc-override-with-different-visibility <misc/override-with-different-visibility>`,
    :doc:`misc-redundant-expression <misc/redundant-expression>`, "Yes"
    :doc:`misc-static-assert <misc/static-assert>`, "Yes"
    :doc:`misc-throw-by-value-catch-by-reference <misc/throw-by-value-catch-by-reference>`,
diff --git a/clang-tools-extra/docs/clang-tidy/checks/misc/override-with-different-visibility.rst b/clang-tools-extra/docs/clang-tidy/checks/misc/override-with-different-visibility.rst
new file mode 100644
index 0000000000000..310bfe2b01080
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/misc/override-with-different-visibility.rst
@@ -0,0 +1,87 @@
+.. title:: clang-tidy - misc-override-with-different-visibility
+
+misc-override-with-different-visibility
+=======================================
+
+Finds virtual function overrides with different visibility than the function
+in the base class. This includes for example if a virtual function declared as
+``private`` is overridden and declared as ``public`` in a subclass. The detected
+change is the modification of visibility resulting from keywords ``public``,
+``protected``, ``private`` at overridden virtual functions. The check applies to
+any normal virtual function and optionally to destructors or operators. Use of
+the ``using`` keyword is not considered as visibility change by this check.
+
+
+.. code-block:: c++
+
+  class A {
+  public:
+    virtual void f_pub();
+  private:
+    virtual void f_priv();
+  };
+  
+  class B: public A {
+  public:
+    void f_priv(); // warning: changed visibility from private to public
+  private:
+    void f_pub(); // warning: changed visibility from public to private
+  };
+
+  class C: private A {
+    // no warning: f_pub becomes private in this case but this is from the
+    // private inheritance
+  };
+
+  class D: private A {
+  public:
+    void f_pub(); // warning: changed visibility from private to public
+                  // 'f_pub' would have private access but is forced to be
+                  // public
+  };
+
+If the visibility is changed in this way, it can indicate bad design or
+programming error.
+
+If a virtual function is private in a subclass but public in the base class, it
+can still be accessed from a pointer to the subclass if the pointer is converted
+to the base type. Probably private inheritance can be used instead.
+
+A protected virtual function that is made public in a subclass may have valid
+use cases but similar (not exactly same) effect can be achieved with the
+``using`` keyword.
+
+Options
+-------
+
+.. option:: DisallowedVisibilityChange
+
+  Controls what kind of change to the visibility will be detected by the check.
+  Possible values are `any`, `widening`, `narrowing`. For example the
+  `widening` option will produce warning only if the visibility is changed
+  from more restrictive (``private``) to less restrictive (``public``).
+  Default value is `any`.
+
+.. option:: CheckDestructors
+
+  If `true`, the check does apply to destructors too. Otherwise destructors
+  are ignored by the check.
+  Default value is `false`.
+
+.. option:: CheckOperators
+
+  If `true`, the check does apply to overloaded C++ operators (as virtual
+  member functions) too. This includes other special member functions (like
+  conversions) too. This option is probably useful only in rare cases because
+  operators and conversions are not often virtual functions.
+  Default value is `false`.
+
+.. option:: IgnoredFunctions
+
+  This option can be used to ignore the check at specific functions.
+  To configure this option, a semicolon-separated list of function names
+  should be provided. The list can contain regular expressions, in this way it
+  is possible to select all functions of a specific class (like `MyClass::.*`)
+  or a specific function of any class (like `my_function` or
+  `::.*::my_function`). The function names are matched at the base class.
+  Default value is empty string.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst b/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
index 8a54687a25704..0e031a15dea90 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/readability/identifier-naming.rst
@@ -43,14 +43,21 @@ The options and their corresponding values are:
  - ``LowerCase`` - example: ``int i_Variable``
  - ``CamelCase`` - example: ``int IVariable``
 
-Options
--------
+Options summary
+---------------
 
-The following options are described below:
+The available options are summarized below:
+
+**General options**
 
- - :option:`AbstractClassCase`, :option:`AbstractClassPrefix`, :option:`AbstractClassSuffix`, :option:`AbstractClassIgnoredRegexp`, :option:`AbstractClassHungarianPrefix`
  - :option:`AggressiveDependentMemberLookup`
  - :option:`CheckAnonFieldInParent`
+ - :option:`GetConfigPerFile`
+ - :option:`IgnoreMainLikeFunctions`
+
+**Specific options**
+
+ - :option:`AbstractClassCase`, :option:`AbstractClassPrefix`, :option:`AbstractClassSuffix`, :option:`AbstractClassIgnoredRegexp`, :option:`AbstractClassHungarianPrefix`
  - :option:`ClassCase`, :option:`ClassPrefix`, :option:`ClassSuffix`, :option:`ClassIgnoredRegexp`, :option:`ClassHungarianPrefix`
  - :option:`ClassConstantCase`, :option:`ClassConstantPrefix`, :option:`ClassConstantSuffix`, :option:`ClassConstantIgnoredRegexp`, :option:`ClassConstantHungarianPrefix`
  - :option:`ClassMemberCase`, :option:`ClassMemberPrefix`, :option:`ClassMemberSuffix`, :option:`ClassMemberIgnoredRegexp`, :option:`ClassMemberHungarianPrefix`
@@ -66,13 +73,11 @@ The following options are described below:
  - :option:`EnumCase`, :option:`EnumPrefix`, :option:`EnumSuffix`, :option:`EnumIgnoredRegexp`
  - :option:`EnumConstantCase`, :option:`EnumConstantPrefix`, :option:`EnumConstantSuffix`, :option:`EnumConstantIgnoredRegexp`, :option:`EnumConstantHungarianPrefix`
  - :option:`FunctionCase`, :option:`FunctionPrefix`, :option:`FunctionSuffix`, :option:`FunctionIgnoredRegexp`
- - :option:`GetConfigPerFile`
  - :option:`GlobalConstantCase`, :option:`GlobalConstantPrefix`, :option:`GlobalConstantSuffix`, :option:`GlobalConstantIgnoredRegexp`, :option:`GlobalConstantHungarianPrefix`
  - :option:`GlobalConstantPointerCase`, :option:`GlobalConstantPointerPrefix`, :option:`GlobalConstantPointerSuffix`, :option:`GlobalConstantPointerIgnoredRegexp`, :option:`GlobalConstantPointerHungarianPrefix`
  - :option:`GlobalFunctionCase`, :option:`GlobalFunctionPrefix`, :option:`GlobalFunctionSuffix`, :option:`GlobalFunctionIgnoredRegexp`
  - :option:`GlobalPointerCase`, :option:`GlobalPointerPrefix`, :option:`GlobalPointerSuffix`, :option:`GlobalPointerIgnoredRegexp`, :option:`GlobalPointerHungarianPrefix`
  - :option:`GlobalVariableCase`, :option:`GlobalVariablePrefix`, :option:`GlobalVariableSuffix`, :option:`GlobalVariableIgnoredRegexp`, :option:`GlobalVariableHungarianPrefix`
- - :option:`IgnoreMainLikeFunctions`
  - :option:`InlineNamespaceCase`, :option:`InlineNamespacePrefix`, :option:`InlineNamespaceSuffix`, :option:`InlineNamespaceIgnoredRegexp`
  - :option:`LocalConstantCase`, :option:`LocalConstantPrefix`, :option:`LocalConstantSuffix`, :option:`LocalConstantIgnoredRegexp`, :option:`LocalConstantHungarianPrefix`
  - :option:`LocalConstantPointerCase`, :option:`LocalConstantPointerPrefix`, :option:`LocalConstantPointerSuffix`, :option:`LocalConstantPointerIgnoredRegexp`, :option:`LocalConstantPointerHungarianPrefix`
@@ -105,6 +110,12 @@ The following options are described below:
  - :option:`VariableCase`, :option:`VariablePrefix`, :option:`VariableSuffix`, :option:`VariableIgnoredRegexp`, :option:`VariableHungarianPrefix`
  - :option:`VirtualMethodCase`, :option:`VirtualMethodPrefix`, :option:`VirtualMethodSuffix`, :option:`VirtualMethodIgnoredRegexp`
 
+
+Options description
+-------------------
+
+A detailed description of each option is presented below:
+
 .. option:: AbstractClassCase
 
     When defined, the check will ensure abstract class names conform to the
diff --git a/clang-tools-extra/docs/clang-tidy/index.rst b/clang-tools-extra/docs/clang-tidy/index.rst
index b7a366e874130..3abd6639c3025 100644
--- a/clang-tools-extra/docs/clang-tidy/index.rst
+++ b/clang-tools-extra/docs/clang-tidy/index.rst
@@ -111,6 +111,13 @@ Diagnostics which have a corresponding warning option, are named
 ``-Wliteral-conversion`` will be reported with check name
 ``clang-diagnostic-literal-conversion``.
 
+Clang compiler errors (such as syntax errors, semantic errors, or other failures
+that prevent Clang from compiling the code) are reported with the check name
+``clang-diagnostic-error``. These represent fundamental compilation failures that
+must be fixed before :program:`clang-tidy` can perform its analysis. Unlike other
+diagnostics, ``clang-diagnostic-error`` cannot be disabled, as :program:`clang-tidy`
+requires valid code to function.
+
 The ``-fix`` flag instructs :program:`clang-tidy` to fix found errors if
 supported by corresponding checks.
 
@@ -213,14 +220,19 @@ An overview of all the command-line options:
                                        Can be used together with -line-filter.
                                        This option overrides the 'HeaderFilterRegex'
                                        option in .clang-tidy file, if any.
-    --line-filter=<string>           - List of files with line ranges to filter the
-                                       warnings. Can be used together with
-                                       -header-filter. The format of the list is a
-                                       JSON array of objects:
+    --line-filter=<string>           - List of files and line ranges to output diagnostics from.
+                                       The range is inclusive on both ends. Can be used together
+                                       with -header-filter. The format of the list is a JSON
+                                       array of objects. For example:
+
                                          [
                                            {"name":"file1.cpp","lines":[[1,3],[5,7]]},
                                            {"name":"file2.h"}
                                          ]
+
+                                       This will output diagnostics from 'file1.cpp' only for
+                                       the line ranges [1,3] and [5,7], as well as all from the
+                                       entire 'file2.h'.
     --list-checks                    - List all enabled checks and exit. Use with
                                        -checks=* to list all available checks.
     --load=<pluginfilename>          - Load the specified plugin
@@ -331,6 +343,107 @@ An overview of all the command-line options:
         some-check.SomeOption: 'some value'
       ...
 
+Clang-Tidy Automation
+=====================
+
+:program:`clang-tidy` can analyze multiple source files by specifying them on
+the command line. For larger projects, automation scripts provide additional
+functionality like parallel execution and integration with version control
+systems.
+
+Running Clang-Tidy in Parallel
+-------------------------------
+
+:program:`clang-tidy` can process multiple files sequentially, but for projects
+with many source files, the :program:`run-clang-tidy.py` script provides
+parallel execution to significantly reduce analysis time. This script is
+included with clang-tidy and runs :program:`clang-tidy` over all files in a
+compilation database or a specified path concurrently.
+
+The script requires a compilation database (``compile_commands.json``) which
+can be generated by build systems like CMake (using
+``-DCMAKE_EXPORT_COMPILE_COMMANDS=ON``) or by tools like `Bear`_.
+
+The script supports most of the same options as :program:`clang-tidy` itself,
+including ``-checks=``, ``-fix``, ``-header-filter=``, and configuration
+options. Run ``run-clang-tidy.py --help`` for a complete list of available
+options.
+
+Example invocations:
+
+.. code-block:: console
+
+  # Run clang-tidy on all files in the compilation database in parallel
+  $ run-clang-tidy.py -p=build/
+
+  # Run with specific checks and apply fixes
+  $ run-clang-tidy.py -p=build/ -fix -checks=-*,readability-*
+
+  # Run on specific files/directories with header filtering
+  $ run-clang-tidy.py -p=build/ -header-filter=src/ src/
+
+  # Run with parallel execution (uses all CPU cores by default)
+  $ run-clang-tidy.py -p=build/ -j 4
+
+Running Clang-Tidy on Diff
+---------------------------
+
+The :program:`clang-tidy-diff.py` script allows you to run
+:program:`clang-tidy` on the lines that have been modified in your working
+directory or in a specific diff. Importantly, :program:`clang-tidy-diff.py` only reports
+diagnostics for changed lines; :program:`clang-tidy` still analyzes the entire
+file and filters out unchanged lines after analysis, so this does not improve
+performance. This is particularly useful for code reviews and continuous
+integration, as it focuses analysis on the changed code rather than the entire
+codebase.
+
+The script can work with various diff sources:
+
+* Git working directory changes
+* Output from ``git diff``
+* Output from ``svn diff``
+* Patch files
+
+Example invocations:
+
+.. code-block:: console
+
+  # Run clang-tidy on all changes in the working directory
+  $ git diff -U0 --no-color HEAD^ | clang-tidy-diff.py -p1
+
+  # Run with specific checks and apply fixes
+  $ git diff -U0 --no-color HEAD^ | clang-tidy-diff.py -p1 -fix \
+    -checks=-*,readability-*
+
+  # Run on staged changes
+  $ git diff -U0 --no-color --cached | clang-tidy-diff.py -p1
+
+  # Run on changes between two commits
+  $ git diff -U0 --no-color HEAD~2 HEAD | clang-tidy-diff.py -p1
+
+  # Run on a patch file
+  $ clang-tidy-diff.py -p1 < changes.patch
+
+The ``-p1`` option tells the script to strip one level of path prefix from
+the diff, which is typically needed for Git diffs. The script supports most of
+the same options as :program:`clang-tidy` itself, including ``-checks=``,
+``-fix``, ``-header-filter=``, and configuration options.
+
+While :program:`clang-tidy-diff.py` is useful for focusing on recent changes,
+relying solely on it may lead to incomplete analysis. Since the script only
+reports warnings from the modified lines, it may miss issues that are caused
+by the changes but manifest elsewhere in the code. For example, changes that
+only add lines to a function may cause it to violate size limits (e.g.,
+`readability-function-size <checks/readability/function-size.html>`_), but the
+diagnostic will be reported at the function declaration, which may not be in
+the diff and thus filtered out. Modifications to header files may also affect
+many implementation files, but only warnings in the modified header lines will
+be reported.
+
+For comprehensive analysis, especially before merging significant changes,
+consider running :program:`clang-tidy` on the entire affected files or the
+whole project using :program:`run-clang-tidy.py`.
+
 .. _clang-tidy-nolint:
 
 Suppressing Undesired Diagnostics
@@ -453,5 +566,6 @@ example, ``NOLINTBEGIN(check-name)`` can be paired with
 :program:`clang-tidy` will generate a ``clang-tidy-nolint`` error diagnostic if
 any ``NOLINTBEGIN``/``NOLINTEND`` comment violates these requirements.
 
+.. _Bear: https://github.com/rizsotto/Bear
 .. _LibTooling: https://clang.llvm.org/docs/LibTooling.html
 .. _How To Setup Tooling For LLVM: https://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
diff --git a/clang-tools-extra/docs/index.rst b/clang-tools-extra/docs/index.rst
index 9f7324fcf7419..3f3a99d1b70c6 100644
--- a/clang-tools-extra/docs/index.rst
+++ b/clang-tools-extra/docs/index.rst
@@ -17,6 +17,7 @@ Contents
 
    clang-tidy/index
    clang-include-fixer
+   clang-change-namespace
    modularize
    pp-trace
    clangd <https://clangd.llvm.org/>
diff --git a/clang-tools-extra/include-cleaner/include/clang-include-cleaner/Types.h b/clang-tools-extra/include-cleaner/include/clang-include-cleaner/Types.h
index 2888e25226755..057b92c147047 100644
--- a/clang-tools-extra/include-cleaner/include/clang-include-cleaner/Types.h
+++ b/clang-tools-extra/include-cleaner/include/clang-include-cleaner/Types.h
@@ -136,7 +136,7 @@ struct Header {
   }
   StringRef verbatim() const { return std::get<Verbatim>(Storage); }
 
-  /// For phiscal files, either absolute path or path relative to the execution
+  /// For physical files, either absolute path or path relative to the execution
   /// root. Otherwise just the spelling without surrounding quotes/brackets.
   llvm::StringRef resolvedPath() const;
 
diff --git a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
index 49cc13606f4c2..0cbf9a080a3ce 100644
--- a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
+++ b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
@@ -60,14 +60,10 @@ class ASTWalker : public RecursiveASTVisitor<ASTWalker> {
   NamedDecl *getMemberProvider(QualType Base) {
     if (Base->isPointerType())
       return getMemberProvider(Base->getPointeeType());
-    // Unwrap the sugar ElaboratedType.
-    if (const auto *ElTy = dyn_cast<ElaboratedType>(Base))
-      return getMemberProvider(ElTy->getNamedType());
-
     if (const auto *TT = dyn_cast<TypedefType>(Base))
       return TT->getDecl();
     if (const auto *UT = dyn_cast<UsingType>(Base))
-      return UT->getFoundDecl();
+      return UT->getDecl();
     // A heuristic: to resolve a template type to **only** its template name.
     // We're only using this method for the base type of MemberExpr, in general
     // the template provides the member, and the critical case `unique_ptr<Foo>`
@@ -135,16 +131,14 @@ class ASTWalker : public RecursiveASTVisitor<ASTWalker> {
   }
 
   bool qualifierIsNamespaceOrNone(DeclRefExpr *DRE) {
-    const auto *Qual = DRE->getQualifier();
-    if (!Qual)
-      return true;
-    switch (Qual->getKind()) {
-    case NestedNameSpecifier::Namespace:
-    case NestedNameSpecifier::Global:
+    NestedNameSpecifier Qual = DRE->getQualifier();
+    switch (Qual.getKind()) {
+    case NestedNameSpecifier::Kind::Null:
+    case NestedNameSpecifier::Kind::Namespace:
+    case NestedNameSpecifier::Kind::Global:
       return true;
-    case NestedNameSpecifier::TypeSpec:
-    case NestedNameSpecifier::Super:
-    case NestedNameSpecifier::Identifier:
+    case NestedNameSpecifier::Kind::Type:
+    case NestedNameSpecifier::Kind::MicrosoftSuper:
       return false;
     }
     llvm_unreachable("Unknown value for NestedNameSpecifierKind");
@@ -341,17 +335,17 @@ class ASTWalker : public RecursiveASTVisitor<ASTWalker> {
   }
 
   bool VisitUsingTypeLoc(UsingTypeLoc TL) {
-    reportType(TL.getNameLoc(), TL.getFoundDecl());
+    reportType(TL.getNameLoc(), TL.getDecl());
     return true;
   }
 
   bool VisitTagTypeLoc(TagTypeLoc TTL) {
-    reportType(TTL.getNameLoc(), TTL.getDecl());
+    reportType(TTL.getNameLoc(), TTL.getOriginalDecl());
     return true;
   }
 
   bool VisitTypedefTypeLoc(TypedefTypeLoc TTL) {
-    reportType(TTL.getNameLoc(), TTL.getTypedefNameDecl());
+    reportType(TTL.getNameLoc(), TTL.getDecl());
     return true;
   }
 
diff --git a/clang-tools-extra/test/clang-doc/basic-project.mustache.test b/clang-tools-extra/test/clang-doc/basic-project.mustache.test
index e2d9da60183fa..55099517101f2 100644
--- a/clang-tools-extra/test/clang-doc/basic-project.mustache.test
+++ b/clang-tools-extra/test/clang-doc/basic-project.mustache.test
@@ -2,10 +2,10 @@
 // RUN: sed 's|$test_dir|%/S|g' %S/Inputs/basic-project/database_template.json > %t/build/compile_commands.json
 
 // RUN: clang-doc --format=mustache --output=%t/docs --executor=all-TUs %t/build/compile_commands.json
-// RUN: FileCheck %s -input-file=%t/docs/_ZTV5Shape.html -check-prefix=HTML-SHAPE
-// RUN: FileCheck %s -input-file=%t/docs/_ZTV10Calculator.html -check-prefix=HTML-CALC
-// RUN: FileCheck %s -input-file=%t/docs/_ZTV9Rectangle.html -check-prefix=HTML-RECTANGLE
-// RUN: FileCheck %s -input-file=%t/docs/_ZTV6Circle.html -check-prefix=HTML-CIRCLE
+// RUN: FileCheck %s -input-file=%t/docs/html/_ZTV5Shape.html -check-prefix=HTML-SHAPE
+// RUN: FileCheck %s -input-file=%t/docs/html/_ZTV10Calculator.html -check-prefix=HTML-CALC
+// RUN: FileCheck %s -input-file=%t/docs/html/_ZTV9Rectangle.html -check-prefix=HTML-RECTANGLE
+// RUN: FileCheck %s -input-file=%t/docs/html/_ZTV6Circle.html -check-prefix=HTML-CIRCLE
 
 HTML-SHAPE: <html lang="en-US">
 HTML-SHAPE: <head>
diff --git a/clang-tools-extra/test/clang-doc/json/class-requires.cpp b/clang-tools-extra/test/clang-doc/json/class-requires.cpp
index bf6c889849a70..513961723990e 100644
--- a/clang-tools-extra/test/clang-doc/json/class-requires.cpp
+++ b/clang-tools-extra/test/clang-doc/json/class-requires.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/_ZTV7MyClass.json
+// RUN: FileCheck %s < %t/json/_ZTV7MyClass.json
 
 template<typename T>
 concept Addable = requires(T a, T b) {
diff --git a/clang-tools-extra/test/clang-doc/json/class-specialization.cpp b/clang-tools-extra/test/clang-doc/json/class-specialization.cpp
index e9259edad5cb8..d3ad6957e7851 100644
--- a/clang-tools-extra/test/clang-doc/json/class-specialization.cpp
+++ b/clang-tools-extra/test/clang-doc/json/class-specialization.cpp
@@ -1,7 +1,7 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/_ZTV7MyClass.json --check-prefix=BASE
-// RUN: FileCheck %s < %t/_ZTV7MyClassIiE.json --check-prefix=SPECIALIZATION
+// RUN: FileCheck %s < %t/json/_ZTV7MyClass.json --check-prefix=BASE
+// RUN: FileCheck %s < %t/json/_ZTV7MyClassIiE.json --check-prefix=SPECIALIZATION
 
 template<typename T> struct MyClass {};
 
diff --git a/clang-tools-extra/test/clang-doc/json/class-template.cpp b/clang-tools-extra/test/clang-doc/json/class-template.cpp
index 149248c772055..5ef78f54854dd 100644
--- a/clang-tools-extra/test/clang-doc/json/class-template.cpp
+++ b/clang-tools-extra/test/clang-doc/json/class-template.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/_ZTV7MyClass.json
+// RUN: FileCheck %s < %t/json/_ZTV7MyClass.json
 
 template<typename T> struct MyClass {
   T MemberTemplate;
diff --git a/clang-tools-extra/test/clang-doc/json/class.cpp b/clang-tools-extra/test/clang-doc/json/class.cpp
index 79b8fed0a0188..20a9f218b3d79 100644
--- a/clang-tools-extra/test/clang-doc/json/class.cpp
+++ b/clang-tools-extra/test/clang-doc/json/class.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/_ZTV7MyClass.json
+// RUN: FileCheck %s < %t/json/_ZTV7MyClass.json
 
 struct Foo;
 
diff --git a/clang-tools-extra/test/clang-doc/json/compound-constraints.cpp b/clang-tools-extra/test/clang-doc/json/compound-constraints.cpp
index bb2b4ca770fc0..1a73a0ddb722f 100644
--- a/clang-tools-extra/test/clang-doc/json/compound-constraints.cpp
+++ b/clang-tools-extra/test/clang-doc/json/compound-constraints.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/index.json
+// RUN: FileCheck %s < %t/json/index.json
 
 template<typename T> concept Incrementable = requires (T a) {
   a++;
diff --git a/clang-tools-extra/test/clang-doc/json/concept.cpp b/clang-tools-extra/test/clang-doc/json/concept.cpp
index 4c810244ca41b..e96ec14d7dde4 100644
--- a/clang-tools-extra/test/clang-doc/json/concept.cpp
+++ b/clang-tools-extra/test/clang-doc/json/concept.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/index.json
+// RUN: FileCheck %s < %t/json/index.json
 
 // Requires that T suports post and pre-incrementing.
 template<typename T>
diff --git a/clang-tools-extra/test/clang-doc/json/function-requires.cpp b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
index 59ed39ee61fda..94271467cba63 100644
--- a/clang-tools-extra/test/clang-doc/json/function-requires.cpp
+++ b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/index.json
+// RUN: FileCheck %s < %t/json/index.json
 
 template<typename T>
 concept Incrementable = requires(T x) {
diff --git a/clang-tools-extra/test/clang-doc/json/function-specifiers.cpp b/clang-tools-extra/test/clang-doc/json/function-specifiers.cpp
index b194e3371bf76..faaccb7d4f63f 100644
--- a/clang-tools-extra/test/clang-doc/json/function-specifiers.cpp
+++ b/clang-tools-extra/test/clang-doc/json/function-specifiers.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/index.json
+// RUN: FileCheck %s < %t/json/index.json
 
 static void myFunction() {}
 
diff --git a/clang-tools-extra/test/clang-doc/json/method-template.cpp b/clang-tools-extra/test/clang-doc/json/method-template.cpp
index 14232d00e277a..87977f891a223 100644
--- a/clang-tools-extra/test/clang-doc/json/method-template.cpp
+++ b/clang-tools-extra/test/clang-doc/json/method-template.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/_ZTV7MyClass.json
+// RUN: FileCheck %s < %t/json/_ZTV7MyClass.json
 
 struct MyClass {
   template<class T> T methodTemplate(T param) {
diff --git a/clang-tools-extra/test/clang-doc/json/namespace.cpp b/clang-tools-extra/test/clang-doc/json/namespace.cpp
index 4b6b38869f714..dcf83236bae28 100644
--- a/clang-tools-extra/test/clang-doc/json/namespace.cpp
+++ b/clang-tools-extra/test/clang-doc/json/namespace.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/index.json
+// RUN: FileCheck %s < %t/json/index.json
 
 class MyClass {};
 
diff --git a/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp b/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp
index 255e540bd6c7c..b19afc1885104 100644
--- a/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp
+++ b/clang-tools-extra/test/clang-doc/json/nested-namespace.cpp
@@ -1,7 +1,7 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --output=%t --format=json --executor=standalone %s
-// RUN: FileCheck %s < %t/nested.json --check-prefix=NESTED
-// RUN: FileCheck %s < %t/inner.json --check-prefix=INNER
+// RUN: FileCheck %s < %t/json/nested.json --check-prefix=NESTED
+// RUN: FileCheck %s < %t/json/inner.json --check-prefix=INNER
 
 namespace nested {
   int Global;
diff --git a/clang-tools-extra/test/clang-doc/mustache-index.cpp b/clang-tools-extra/test/clang-doc/mustache-index.cpp
index 910233b943666..f9aad193799b3 100644
--- a/clang-tools-extra/test/clang-doc/mustache-index.cpp
+++ b/clang-tools-extra/test/clang-doc/mustache-index.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --format=mustache --output=%t --executor=standalone %s 
-// RUN: FileCheck %s < %t/index.html
+// RUN: FileCheck %s < %t/html/index.html
 
 enum Color {
   RED,
diff --git a/clang-tools-extra/test/clang-doc/mustache-separate-namespace.cpp b/clang-tools-extra/test/clang-doc/mustache-separate-namespace.cpp
index 7d7d108e63873..a73a5ab6a843b 100644
--- a/clang-tools-extra/test/clang-doc/mustache-separate-namespace.cpp
+++ b/clang-tools-extra/test/clang-doc/mustache-separate-namespace.cpp
@@ -1,6 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --format=mustache --output=%t --executor=standalone %s 
-// RUN: FileCheck %s < %t/MyNamespace.html
+// RUN: FileCheck %s < %t/html/MyNamespace.html
 
 namespace MyNamespace {
   class Foo;
diff --git a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
index 774bc970ef284..d225258c5d75d 100755
--- a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
+++ b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
@@ -135,8 +135,7 @@ def __init__(self, args: argparse.Namespace, extra_args: List[str]) -> None:
                 "-fblocks",
             ] + self.clang_extra_args
 
-        if extension in [".cpp", ".hpp", ".mm"]:
-            self.clang_extra_args.append("-std=" + self.std)
+        self.clang_extra_args.append("-std=" + self.std)
 
         # Tests should not rely on STL being available, and instead provide mock
         # implementations of relevant APIs.
@@ -374,7 +373,7 @@ def parse_arguments() -> Tuple[argparse.Namespace, List[str]]:
     parser.add_argument(
         "-std",
         type=csv,
-        default=["c++11-or-later"],
+        default=None,
         help="Passed to clang. Special -or-later values are expanded.",
     )
     parser.add_argument(
@@ -382,7 +381,15 @@ def parse_arguments() -> Tuple[argparse.Namespace, List[str]]:
         action="store_true",
         help="allow partial line matches for fixes",
     )
-    return parser.parse_known_args()
+
+    args, extra_args = parser.parse_known_args()
+    if args.std is None:
+        _, extension = os.path.splitext(args.assume_filename or args.input_file_name)
+        args.std = [
+            "c++11-or-later" if extension in [".cpp", ".hpp", ".mm"] else "c99-or-later"
+        ]
+
+    return (args, extra_args)
 
 
 def main() -> None:
diff --git a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/system-other.h b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/system-other.h
index 28c26b73ba6b4..011c869d2b9a7 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/system-other.h
+++ b/clang-tools-extra/test/clang-tidy/checkers/Inputs/Headers/system-other.h
@@ -11,6 +11,8 @@
 
 // Special system calls.
 
+#if __STDC_VERSION__ < 202311L
 void other_call();
+#endif
 
 #endif // _SYSTEM_OTHER_H_
diff --git a/clang-tools-extra/test/clang-tidy/checkers/android/comparison-in-temp-failure-retry-custom-macro.c b/clang-tools-extra/test/clang-tidy/checkers/android/comparison-in-temp-failure-retry-custom-macro.c
index 56c382c87f1f3..3f60860e1f425 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/android/comparison-in-temp-failure-retry-custom-macro.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/android/comparison-in-temp-failure-retry-custom-macro.c
@@ -2,7 +2,7 @@
 
 #define MY_TEMP_FAILURE_RETRY(x) \
   ({                             \
-    typeof(x) __z;               \
+    __typeof__(x) __z;           \
     do                           \
       __z = (x);                 \
     while (__z == -1);           \
@@ -11,7 +11,7 @@
 
 #define MY_OTHER_TEMP_FAILURE_RETRY(x) \
   ({                                   \
-    typeof(x) __z;                     \
+    __typeof__(x) __z;                 \
     do                                 \
       __z = (x);                       \
     while (__z == -1);                 \
diff --git a/clang-tools-extra/test/clang-tidy/checkers/android/comparison-in-temp-failure-retry.c b/clang-tools-extra/test/clang-tidy/checkers/android/comparison-in-temp-failure-retry.c
index 461fb4ad52542..1c71ebb75604e 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/android/comparison-in-temp-failure-retry.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/android/comparison-in-temp-failure-retry.c
@@ -2,7 +2,7 @@
 
 #define TEMP_FAILURE_RETRY(x)                                                  \
   ({                                                                           \
-    typeof(x) __z;                                                             \
+    __typeof__(x) __z;                                                         \
     do                                                                         \
       __z = (x);                                                               \
     while (__z == -1);                                                         \
@@ -130,7 +130,7 @@ void obscured_temp_failure_retry(void) {
 #undef TEMP_FAILURE_RETRY
 #define IMPL(x)                                                                \
   ({                                                                           \
-    typeof(x) __z;                                                             \
+    __typeof__(x) __z;                                                         \
     do                                                                         \
       __z = (x);                                                               \
     while (__z == -1);                                                         \
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/branch-clone-macro-crash.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/branch-clone-macro-crash.c
index a4cb7347f88c6..0f4389c885e8f 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/branch-clone-macro-crash.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/branch-clone-macro-crash.c
@@ -1,11 +1,11 @@
 // RUN: %check_clang_tidy %s bugprone-branch-clone %t
 int x = 0;
 int y = 1;
-#define a(b, c) \
-  typeof(b) d;  \
-  if (b)        \
-    d = b;      \
-  else if (c)   \
+#define a(b, c)    \
+  __typeof__(b) d; \
+  if (b)           \
+    d = b;         \
+  else if (c)      \
     d = b;
 
 void f(void) {
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/copy-constructor-init.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/copy-constructor-init.cpp
index b42d3fcd2b62b..35314d12b59bf 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/copy-constructor-init.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/copy-constructor-init.cpp
@@ -165,7 +165,6 @@ FROMMACRO
 class X15 : public CopyableAlias2 {
   X15(const X15 &other) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: calling a base constructor
-  // CHECK-FIXES: X15(const X15 &other)  : Copyable5(other) {}
 };
 
 class X16 : public NonCopyable {
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/easily-swappable-parameters-relatedness.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/easily-swappable-parameters-relatedness.c
index 7231361a2e080..45752de36a90c 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/easily-swappable-parameters-relatedness.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/easily-swappable-parameters-relatedness.c
@@ -1,4 +1,15 @@
-// RUN: %check_clang_tidy %s bugprone-easily-swappable-parameters %t \
+// RUN: %check_clang_tidy -std=c99,c11,c17 -check-suffixes=,BEFORE-C23 %s bugprone-easily-swappable-parameters %t \
+// RUN:   -config='{CheckOptions: { \
+// RUN:     bugprone-easily-swappable-parameters.MinimumLength: 2, \
+// RUN:     bugprone-easily-swappable-parameters.IgnoredParameterNames: "", \
+// RUN:     bugprone-easily-swappable-parameters.IgnoredParameterTypeSuffixes: "", \
+// RUN:     bugprone-easily-swappable-parameters.QualifiersMix: 0, \
+// RUN:     bugprone-easily-swappable-parameters.ModelImplicitConversions: 0, \
+// RUN:     bugprone-easily-swappable-parameters.SuppressParametersUsedTogether: 1, \
+// RUN:     bugprone-easily-swappable-parameters.NamePrefixSuffixSilenceDissimilarityTreshold: 0 \
+// RUN:  }}' -- -Wno-strict-prototypes -x c
+//
+// RUN: %check_clang_tidy -std=c23-or-later %s bugprone-easily-swappable-parameters %t \
 // RUN:   -config='{CheckOptions: { \
 // RUN:     bugprone-easily-swappable-parameters.MinimumLength: 2, \
 // RUN:     bugprone-easily-swappable-parameters.IgnoredParameterNames: "", \
@@ -9,7 +20,6 @@
 // RUN:     bugprone-easily-swappable-parameters.NamePrefixSuffixSilenceDissimilarityTreshold: 0 \
 // RUN:  }}' -- -Wno-strict-prototypes -x c
 
-int myprint();
 int add(int X, int Y);
 
 void notRelated(int A, int B) {}
@@ -19,13 +29,18 @@ void notRelated(int A, int B) {}
 
 int addedTogether(int A, int B) { return add(A, B); } // NO-WARN: Passed to same function.
 
+#if __STDC_VERSION__ < 202311L
+int myprint();
 void passedToSameKNRFunction(int A, int B) {
   myprint("foo", A);
   myprint("bar", B);
 }
-// CHECK-MESSAGES: :[[@LINE-4]]:30: warning: 2 adjacent parameters of 'passedToSameKNRFunction' of similar type ('int')
-// CHECK-MESSAGES: :[[@LINE-5]]:34: note: the first parameter in the range is 'A'
-// CHECK-MESSAGES: :[[@LINE-6]]:41: note: the last parameter in the range is 'B'
-// This is actually a false positive: the "passed to same function" heuristic
+// CHECK-MESSAGES-BEFORE-C23: :[[@LINE-4]]:30: warning: 2 adjacent parameters of 'passedToSameKNRFunction' of similar type ('int')
+// CHECK-MESSAGES-BEFORE-C23: :[[@LINE-5]]:34: note: the first parameter in the range is 'A'
+// CHECK-MESSAGES-BEFORE-C23: :[[@LINE-6]]:41: note: the last parameter in the range is 'B'
+// FIXME: This is actually a false positive: the "passed to same function" heuristic
 // can't map the parameter index 1 to A and B because myprint() has no
 // parameters.
+// If you fix this, you should be able to combine the `%check_clang_tidy` invocations
+// in this file into one and remove the `-std` and `-check-suffixes` arguments.
+#endif
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/easily-swappable-parameters.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/easily-swappable-parameters.c
index b6c9bf4ad0c15..be44cfc889ed0 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/easily-swappable-parameters.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/easily-swappable-parameters.c
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s bugprone-easily-swappable-parameters %t \
+// RUN: %check_clang_tidy -std=c99,c11,c17 -check-suffixes=,BEFORE-23 %s bugprone-easily-swappable-parameters %t \
 // RUN:   -config='{CheckOptions: { \
 // RUN:     bugprone-easily-swappable-parameters.MinimumLength: 2, \
 // RUN:     bugprone-easily-swappable-parameters.IgnoredParameterNames: "", \
@@ -7,7 +7,18 @@
 // RUN:     bugprone-easily-swappable-parameters.ModelImplicitConversions: 0, \
 // RUN:     bugprone-easily-swappable-parameters.SuppressParametersUsedTogether: 0, \
 // RUN:     bugprone-easily-swappable-parameters.NamePrefixSuffixSilenceDissimilarityTreshold: 0 \
-// RUN:  }}' -- -Wno-strict-prototypes -x c
+// RUN:  }}' -- -Wno-strict-prototypes
+//
+// RUN: %check_clang_tidy -std=c23-or-later %s bugprone-easily-swappable-parameters %t \
+// RUN:   -config='{CheckOptions: { \
+// RUN:     bugprone-easily-swappable-parameters.MinimumLength: 2, \
+// RUN:     bugprone-easily-swappable-parameters.IgnoredParameterNames: "", \
+// RUN:     bugprone-easily-swappable-parameters.IgnoredParameterTypeSuffixes: "bool;MyBool;struct U;MAKE_LOGICAL_TYPE(int)", \
+// RUN:     bugprone-easily-swappable-parameters.QualifiersMix: 0, \
+// RUN:     bugprone-easily-swappable-parameters.ModelImplicitConversions: 0, \
+// RUN:     bugprone-easily-swappable-parameters.SuppressParametersUsedTogether: 0, \
+// RUN:     bugprone-easily-swappable-parameters.NamePrefixSuffixSilenceDissimilarityTreshold: 0 \
+// RUN:  }}' -- -Wno-strict-prototypes
 
 #define bool _Bool
 #define true 1
@@ -45,8 +56,11 @@ void pointerConversion(int *IP, long *LP) {}
 
 void testVariadicsCall() {
   int IVal = 1;
+
+#if __STDC_VERSION__ < 202311L
   decl(IVal); // NO-WARN: Particular calls to "variadics" are like template
               // instantiations, and we do not model them.
+#endif
 
   variadic(IVal);          // NO-WARN.
   variadic(IVal, 2, 3, 4); // NO-WARN.
@@ -64,13 +78,15 @@ void taggedTypes2(struct S SVar1, struct S SVar2) {}
 
 void wrappers(struct { int I; } I1, struct { int I; } I2) {} // NO-WARN: Distinct anonymous types.
 
+#if __STDC_VERSION__ < 202311L
 void knr(I, J)
   int I;
   int J;
 {}
-// CHECK-MESSAGES: :[[@LINE-3]]:3: warning: 2 adjacent parameters of 'knr' of similar type ('int')
-// CHECK-MESSAGES: :[[@LINE-4]]:7: note: the first parameter in the range is 'I'
-// CHECK-MESSAGES: :[[@LINE-4]]:7: note: the last parameter in the range is 'J'
+// CHECK-MESSAGES-BEFORE-23: :[[@LINE-3]]:3: warning: 2 adjacent parameters of 'knr' of similar type ('int')
+// CHECK-MESSAGES-BEFORE-23: :[[@LINE-4]]:7: note: the first parameter in the range is 'I'
+// CHECK-MESSAGES-BEFORE-23: :[[@LINE-4]]:7: note: the last parameter in the range is 'J'
+#endif
 
 void boolAsWritten(bool B1, bool B2) {} // NO-WARN: The type name is ignored.
 // Note that "bool" is a macro that expands to "_Bool" internally, but it is
@@ -145,7 +161,7 @@ void thisIsGettingRidiculous(MAKE_PRIMITIVE_WRAPPER(int) I1,
 void macroMagic3(MAKE_LOGICAL_TYPE(char) B1, MAKE_LOGICAL_TYPE(long) B2) {}
 // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: 2 adjacent parameters of 'macroMagic3' of similar type ('bool')
 // CHECK-MESSAGES: :[[@LINE-4]]:30: note: expanded from macro 'MAKE_LOGICAL_TYPE'
-// CHECK-MESSAGES: :[[@LINE-136]]:14: note: expanded from macro 'bool'
+// CHECK-MESSAGES: :[[@LINE-141]]:14: note: expanded from macro 'bool'
 // CHECK-MESSAGES: :[[@LINE-4]]:42: note: the first parameter in the range is 'B1'
 // CHECK-MESSAGES: :[[@LINE-5]]:70: note: the last parameter in the range is 'B2'
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/narrowing-conversions-conditional-expressions.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/narrowing-conversions-conditional-expressions.c
new file mode 100644
index 0000000000000..ce26b789112da
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/narrowing-conversions-conditional-expressions.c
@@ -0,0 +1,22 @@
+// RUN: %check_clang_tidy %s bugprone-narrowing-conversions %t -- \
+// RUN: -- -target x86_64-unknown-linux
+
+char test_char(int cond, char c) {
+	char ret = cond > 0 ? ':' : c;
+	return ret;
+}
+
+short test_short(int cond, short s) {
+	short ret = cond > 0 ? ':' : s;
+	return ret;
+}
+
+int test_int(int cond, int i) {
+	int ret = cond > 0 ? ':' : i;
+	return ret;
+}
+
+void test(int cond, int i) {
+  char x = cond > 0 ? ':' : i;
+  // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: narrowing conversion from 'int' to signed type 'char' is implementation-defined [bugprone-narrowing-conversions]
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/narrowing-conversions-conditional-expressions.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/narrowing-conversions-conditional-expressions.cpp
new file mode 100644
index 0000000000000..ce26b789112da
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/narrowing-conversions-conditional-expressions.cpp
@@ -0,0 +1,22 @@
+// RUN: %check_clang_tidy %s bugprone-narrowing-conversions %t -- \
+// RUN: -- -target x86_64-unknown-linux
+
+char test_char(int cond, char c) {
+	char ret = cond > 0 ? ':' : c;
+	return ret;
+}
+
+short test_short(int cond, short s) {
+	short ret = cond > 0 ? ':' : s;
+	return ret;
+}
+
+int test_int(int cond, int i) {
+	int ret = cond > 0 ? ':' : i;
+	return ret;
+}
+
+void test(int cond, int i) {
+  char x = cond > 0 ? ':' : i;
+  // CHECK-MESSAGES: :[[@LINE-1]]:29: warning: narrowing conversion from 'int' to signed type 'char' is implementation-defined [bugprone-narrowing-conversions]
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-in-initialization-strlen.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-in-initialization-strlen.c
index a383958fbb906..b241d683b0cdc 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-in-initialization-strlen.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-in-initialization-strlen.c
@@ -1,5 +1,5 @@
 // RUN: %check_clang_tidy --match-partial-fixes %s bugprone-not-null-terminated-result %t -- \
-// RUN: -- -std=c11 -I %S/Inputs/not-null-terminated-result
+// RUN: -- -I %S/Inputs/not-null-terminated-result
 
 #include "not-null-terminated-result-c.h"
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-memcpy-before-safe.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-memcpy-before-safe.c
index 434cfcc0ca9d5..ea7e183b92531 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-memcpy-before-safe.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-memcpy-before-safe.c
@@ -1,7 +1,7 @@
 // RUN: %check_clang_tidy %s bugprone-not-null-terminated-result %t -- \
 // RUN: -config="{CheckOptions: \
 // RUN: {bugprone-not-null-terminated-result.WantToUseSafeFunction: true}}" \
-// RUN: -- -std=c11 -I %S/Inputs/not-null-terminated-result
+// RUN: -- -I %S/Inputs/not-null-terminated-result
 
 #include "not-null-terminated-result-c.h"
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-memcpy-safe.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-memcpy-safe.c
index 5a5e35ef733f6..ae430878daf21 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-memcpy-safe.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-memcpy-safe.c
@@ -1,5 +1,5 @@
 // RUN: %check_clang_tidy %s bugprone-not-null-terminated-result %t -- \
-// RUN: -- -std=c11 -I %S/Inputs/not-null-terminated-result
+// RUN: -- -I %S/Inputs/not-null-terminated-result
 
 #include "not-null-terminated-result-c.h"
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-stdc-want-lib-ext1-not-a-literal.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-stdc-want-lib-ext1-not-a-literal.c
index 0628d0ca53300..94806714efc86 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-stdc-want-lib-ext1-not-a-literal.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-stdc-want-lib-ext1-not-a-literal.c
@@ -1,5 +1,5 @@
 // RUN: %check_clang_tidy %s bugprone-not-null-terminated-result %t -- \
-// RUN: -- -std=c11 -I %S/Inputs/not-null-terminated-result
+// RUN: -- -I %S/Inputs/not-null-terminated-result
 
 #include "not-null-terminated-result-c.h"
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-strlen.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-strlen.c
index 4970af83bf4b6..366c1698e4f2d 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-strlen.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-strlen.c
@@ -1,5 +1,5 @@
 // RUN: %check_clang_tidy --match-partial-fixes %s bugprone-not-null-terminated-result %t -- \
-// RUN: -- -std=c11 -I %S/Inputs/not-null-terminated-result
+// RUN: -- -I %S/Inputs/not-null-terminated-result
 
 // FIXME: Something wrong with the APInt un/signed conversion on Windows:
 // in 'strncmp(str6, "string", 7);' it tries to inject '4294967302' as length.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-undef-stdc-want-lib-ext1.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-undef-stdc-want-lib-ext1.c
index 2704dc1ee513b..bb5961f3b421e 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-undef-stdc-want-lib-ext1.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/not-null-terminated-result-undef-stdc-want-lib-ext1.c
@@ -1,5 +1,5 @@
 // RUN: %check_clang_tidy %s bugprone-not-null-terminated-result %t -- \
-// RUN: -- -std=c11 -I %S/Inputs/not-null-terminated-result
+// RUN: -- -I %S/Inputs/not-null-terminated-result
 
 #include "not-null-terminated-result-c.h"
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/signal-handler.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/signal-handler.c
index c7daec0870f49..a8f6f3bba20b9 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/signal-handler.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/signal-handler.c
@@ -1,4 +1,5 @@
-// RUN: %check_clang_tidy %s bugprone-signal-handler %t -- -- -isystem %clang_tidy_headers
+// RUN: %check_clang_tidy -std=c99,c11,c17 -check-suffixes=,BEFORE-23 %s bugprone-signal-handler %t -- -- -isystem %clang_tidy_headers
+// RUN: %check_clang_tidy -std=c23-or-later %s bugprone-signal-handler %t -- -- -isystem %clang_tidy_headers
 
 #include "signal.h"
 #include "stdlib.h"
@@ -174,8 +175,10 @@ void test_other(void) {
   signal(SIGINT, handler_signal);
 
   signal(SIGINT, _Exit);
+#if __STDC_VERSION__ < 202311L
   signal(SIGINT, other_call);
-  // CHECK-NOTES: :[[@LINE-1]]:18: warning: standard function 'other_call' may not be asynchronous-safe; using it as a signal handler may be dangerous [bugprone-signal-handler]
+  // CHECK-NOTES-BEFORE-23: :[[@LINE-1]]:18: warning: standard function 'other_call' may not be asynchronous-safe; using it as a signal handler may be dangerous [bugprone-signal-handler]
+#endif
   signal(SIGINT, f_extern_handler);
   // CHECK-NOTES: :[[@LINE-1]]:18: warning: cannot verify that external function 'f_extern_handler' is asynchronous-safe; using it as a signal handler may be dangerous [bugprone-signal-handler]
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/signed-char-misuse-c23.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/signed-char-misuse-c23.c
index ad6e31d873e5a..cd665118c4270 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/signed-char-misuse-c23.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/signed-char-misuse-c23.c
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s bugprone-signed-char-misuse %t -- -- -std=c23
+// RUN: %check_clang_tidy -std=c23-or-later %s bugprone-signed-char-misuse %t
 
 ///////////////////////////////////////////////////////////////////
 /// Test cases correctly caught by the check.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.c
index b898071a56613..871715b22cc56 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/sizeof-expression.c
@@ -1,5 +1,5 @@
-// RUN: %check_clang_tidy %s bugprone-sizeof-expression %t -- --
-// RUN: %check_clang_tidy %s bugprone-sizeof-expression %t -- -- -x c++
+// RUN: %check_clang_tidy %s bugprone-sizeof-expression %t
+// RUN: %check_clang_tidy -std=c++11-or-later %s bugprone-sizeof-expression %t -- -- -x c++
 
 #ifdef __cplusplus
 #define STRKWD
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions.c b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions.c
index 0409dd6bfcaa3..452e2295cb599 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unsafe-functions.c
@@ -5,14 +5,14 @@
 // parsing and preprocessor state will not have that case.
 // UNSUPPORTED: target={{.*-(ps4|ps5)}}
 //
-// RUN: %check_clang_tidy -check-suffix=WITH-ANNEX-K            %s bugprone-unsafe-functions %t -- -- -D__STDC_LIB_EXT1__=1 -D__STDC_WANT_LIB_EXT1__=1
-// RUN: %check_clang_tidy -check-suffix=WITHOUT-ANNEX-K         %s bugprone-unsafe-functions %t -- -- -U__STDC_LIB_EXT1__   -U__STDC_WANT_LIB_EXT1__
-// RUN: %check_clang_tidy -check-suffix=WITHOUT-ANNEX-K         %s bugprone-unsafe-functions %t -- -- -D__STDC_LIB_EXT1__=1 -U__STDC_WANT_LIB_EXT1__
-// RUN: %check_clang_tidy -check-suffix=WITHOUT-ANNEX-K         %s bugprone-unsafe-functions %t -- -- -U__STDC_LIB_EXT1__   -D__STDC_WANT_LIB_EXT1__=1
-// RUN: %check_clang_tidy -check-suffix=WITH-ANNEX-K-CERT-ONLY  %s bugprone-unsafe-functions %t -- \
+// RUN: %check_clang_tidy -std=c11-or-later -check-suffix=WITH-ANNEX-K %s bugprone-unsafe-functions %t -- -- -D__STDC_LIB_EXT1__=1 -D__STDC_WANT_LIB_EXT1__=1
+// RUN: %check_clang_tidy -check-suffix=WITHOUT-ANNEX-K %s bugprone-unsafe-functions %t -- -- -U__STDC_LIB_EXT1__   -U__STDC_WANT_LIB_EXT1__
+// RUN: %check_clang_tidy -check-suffix=WITHOUT-ANNEX-K %s bugprone-unsafe-functions %t -- -- -D__STDC_LIB_EXT1__=1 -U__STDC_WANT_LIB_EXT1__
+// RUN: %check_clang_tidy -check-suffix=WITHOUT-ANNEX-K %s bugprone-unsafe-functions %t -- -- -U__STDC_LIB_EXT1__   -D__STDC_WANT_LIB_EXT1__=1
+// RUN: %check_clang_tidy -std=c11-or-later -check-suffix=WITH-ANNEX-K-CERT-ONLY  %s bugprone-unsafe-functions %t -- \
 // RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.ReportMoreUnsafeFunctions: false}}" \
 // RUN:                                                                                            -- -D__STDC_LIB_EXT1__=1 -D__STDC_WANT_LIB_EXT1__=1
-// RUN: %check_clang_tidy -check-suffix=WITH-NONE-ENABLED       %s bugprone-unsafe-functions %t --\
+// RUN: %check_clang_tidy -check-suffix=WITH-NONE-ENABLED %s bugprone-unsafe-functions %t --\
 // RUN:   -config="{CheckOptions: {bugprone-unsafe-functions.ReportDefaultFunctions: false}}" \
 // RUN:                                                                                            -- -D__STDC_LIB_EXT1__=1 -D__STDC_WANT_LIB_EXT1__=1
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-local-non-trivial-variable.cpp b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-local-non-trivial-variable.cpp
index 721c55b1fb538..5dd23871b21b3 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-local-non-trivial-variable.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/bugprone/unused-local-non-trivial-variable.cpp
@@ -69,14 +69,14 @@ template<typename T>
 T qux(T Generic) {
     async::Future<Units> PendingA = acquireUnits();
     auto PendingB = acquireUnits();
-    // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: unused local variable 'PendingB' of type 'a::Future<Units>' (aka 'Future<Units>') [bugprone-unused-local-non-trivial-variable]
+    // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: unused local variable 'PendingB' of type 'a::Future<Units>' (aka 'async::Future<Units>') [bugprone-unused-local-non-trivial-variable]
     async::Future<Units> MustBeUsed;
     // CHECK-MESSAGES: :[[@LINE-1]]:26: warning: unused local variable 'MustBeUsed' of type 'async::Future<Units>' [bugprone-unused-local-non-trivial-variable]
     PendingA.get();
     async::Future<T> TemplateType;
     // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: unused local variable 'TemplateType' of type 'async::Future<T>' [bugprone-unused-local-non-trivial-variable]
     a::Future<T> AliasTemplateType;
-    // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: unused local variable 'AliasTemplateType' of type 'a::Future<T>' (aka 'Future<T>') [bugprone-unused-local-non-trivial-variable]
+    // CHECK-MESSAGES: :[[@LINE-1]]:18: warning: unused local variable 'AliasTemplateType' of type 'a::Future<T>' (aka 'async::Future<T>') [bugprone-unused-local-non-trivial-variable]
     [[maybe_unused]] async::Future<Units> MaybeUnused;
     return Generic;
 }
@@ -86,7 +86,7 @@ async::Future<int> Global;
 int bar(int Num) {
     a::Future<Units> PendingA = acquireUnits();
     a::Future<Units> PendingB = acquireUnits(); // not used at all, unused variable not fired because of destructor side effect
-    // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: unused local variable 'PendingB' of type 'a::Future<Units>' (aka 'Future<Units>') [bugprone-unused-local-non-trivial-variable]
+    // CHECK-MESSAGES: :[[@LINE-1]]:22: warning: unused local variable 'PendingB' of type 'a::Future<Units>' (aka 'async::Future<Units>') [bugprone-unused-local-non-trivial-variable]
     auto Num2 = PendingA.get();
     auto Num3 = qux(Num);
     async::Ptr<a::Future<Units>> Shared = async::Ptr<a::Future<Units>>(acquireUnits());
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp
index 7d6164946fc3d..e8d7db17f3c60 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/prefer-member-initializer.cpp
@@ -650,3 +650,16 @@ struct InitFromBindingDecl {
   }
 };
 } // namespace GH82970
+
+struct A {
+  int m;
+};
+
+struct B : A {
+  B() { m = 0; }
+};
+
+template <class T>
+struct C : A {
+  C() { m = 0; }
+};
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-avoid-unchecked-container-access.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-avoid-unchecked-container-access.cpp
new file mode 100644
index 0000000000000..30d03bd70edf6
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-avoid-unchecked-container-access.cpp
@@ -0,0 +1,341 @@
+// RUN: rm -rf %t && mkdir %t
+// RUN: split-file %s %t
+
+
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17,c++20 -check-suffix=DEFAULT %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- -- -I%t
+
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17,c++20 -check-suffix=AT %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: at}}' -- -I%t
+
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17,c++20 -check-suffix=FUNC %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: function, \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixFunction: "f"}}' -- -I%t
+
+
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17,c++20 -check-suffix=DEFAULT-NO-EXCL %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-no-excl.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.ExcludeClasses: ""}}' -- -I%t
+
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17,c++20 -check-suffix=AT-NO-EXCL %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-no-excl.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.ExcludeClasses: "", \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: at}}' -- -I%t
+
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17,c++20 -check-suffix=FUNC-NO-EXCL %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-no-excl.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.ExcludeClasses: "", \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: function, \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixFunction: "f"}}' -- -I%t
+
+
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17,c++20 -check-suffix=DEFAULT-EXCL %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-excl.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.ExcludeClasses: "ExcludedClass1;ExcludedClass2"}}' -- -I%t
+
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17,c++20 -check-suffix=AT-EXCL %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-excl.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.ExcludeClasses: "ExcludedClass1;ExcludedClass2", \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: at}}' -- -I%t
+
+// RUN: %check_clang_tidy -std=c++11,c++14,c++17,c++20 -check-suffix=FUNC-EXCL %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-excl.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.ExcludeClasses: "ExcludedClass1;ExcludedClass2", \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: function, \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixFunction: "f"}}' -- -I%t
+
+
+// RUN: %check_clang_tidy -std=c++23 -check-suffixes=DEFAULT-CXX-23 %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-cxx-23.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- -- -I%t -DCXX_23=1
+
+// RUN: %check_clang_tidy -std=c++23 -check-suffixes=AT-CXX-23 %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-cxx-23.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: at}}' -- -I%t -DCXX_23=1 
+
+// RUN: %check_clang_tidy -std=c++23 -check-suffixes=FUNC-CXX-23 %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-cxx-23.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: function, \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixFunction: "f"}}' -- -I%t -DCXX_23=1 
+
+// RUN: %check_clang_tidy -std=c++23 -check-suffixes=FUNC-EMPTY-ARGS-CXX-23 %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-cxx-23.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: function, \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixFunction: "f", cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixFunctionEmptyArgs: "g", }}' -- -I%t -DCXX_23=1
+
+// RUN: %check_clang_tidy -std=c++23 -check-suffix=FUNC-EMPTY-ARGS-EMPTY-CXX-23 %t/cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-cxx-23.cpp \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access %t -- \
+// RUN: -config='{CheckOptions: {cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixMode: function, \
+// RUN: cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.FixFunctionEmptyArgs: "", }}' -- -I%t -DCXX_23=1
+
+//--- cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.h
+
+namespace std {
+  template<typename T, unsigned size>
+  struct array {
+    T operator[](unsigned i) {
+      return T{1};
+    }
+    T at(unsigned i) {
+      return T{1};
+    }
+    T at() {
+      return T{1};
+    }
+  };
+
+  template<typename T, typename V>
+  struct map {
+    T operator[](unsigned i) {
+      return T{1};
+    }
+    T at(unsigned i) {
+      return T{1};
+    }
+  };
+
+  template<typename T>
+  struct unique_ptr {
+    T operator[](unsigned i) {
+      return T{1};
+    }
+  };
+
+  template<typename T>
+  struct span {
+    T operator[](unsigned i) {
+      return T{1};
+    }
+  };
+} // namespace std
+
+namespace json {
+  template<typename T>
+  struct node{
+    T operator[](unsigned i) {
+      return T{1};
+    }
+  };
+} // namespace json
+
+//--- cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.cpp
+
+#include "cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.h"
+
+struct SubClass : std::array<int, 3> {};
+
+template<class T> int f(T, unsigned){ return 0;}
+template<class T> int f(T){ return 0;}
+
+std::array<int, 3> a;
+
+auto b = a[0];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto b = a.at(0);
+// CHECK-FIXES-FUNC: auto b = f(a, 0);
+
+auto c = a[1+1];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto c = a.at(1+1);
+// CHECK-FIXES-FUNC: auto c = f(a, 1+1);
+
+constexpr int Index = 1;
+
+auto d = a[Index];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto d = a.at(Index);
+// CHECK-FIXES-FUNC: auto d = f(a, Index);
+
+int e(int Ind) {
+  return a[Ind];
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+  // CHECK-FIXES-AT: return a.at(Ind);
+  // CHECK-FIXES-FUNC: return f(a, Ind);
+}
+
+auto fa = (&a)->operator[](1);
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto fa = (&a)->at(1);
+// CHECK-FIXES-FUNC: auto fa = f(*(&a), 1);
+
+auto fd = a.operator[](1);
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto fd = a.at(1);
+// CHECK-FIXES-FUNC: auto fd = f(a, 1);
+
+
+
+auto g = a.at(0);
+
+std::unique_ptr<int> p;
+auto q = p[0];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto q = p[0];
+// CHECK-FIXES-FUNC: auto q = f(p, 0);
+
+std::span<int> s;
+auto t = s[0];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto t = s[0];
+// CHECK-FIXES-FUNC: auto t = f(s, 0);
+
+json::node<int> n;
+auto m = n[0];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto m = n[0];
+// CHECK-FIXES-FUNC: auto m = f(n, 0);
+
+SubClass Sub;
+auto r = Sub[0];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:13: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto r = Sub.at(0);
+// CHECK-FIXES-FUNC: auto r = f(Sub, 0);
+
+typedef std::array<int, 3> ar;
+ar BehindDef;
+auto u = BehindDef[0];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:19: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto u = BehindDef.at(0);
+// CHECK-FIXES-FUNC: auto u = f(BehindDef, 0);
+
+template<typename T> int TestTemplate(T t){
+  return t[0];
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:10: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+
+}
+
+
+auto v = TestTemplate<>(a);
+auto w = TestTemplate<>(p);
+
+#define SUBSCRIPT_BEHIND_MACRO(x) a[x]
+#define ARG_BEHIND_MACRO 0
+#define OBJECT_BEHIND_MACRO a
+
+auto m1 = SUBSCRIPT_BEHIND_MACRO(0);
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+
+auto m2 = a[ARG_BEHIND_MACRO];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:12: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto m2 = a.at(ARG_BEHIND_MACRO);
+// CHECK-FIXES-FUNC: auto m2 = f(a, ARG_BEHIND_MACRO);
+
+auto m3 = OBJECT_BEHIND_MACRO[0];
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:30: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto m3 = OBJECT_BEHIND_MACRO.at(0);
+// CHECK-FIXES-FUNC: auto m3 = f(OBJECT_BEHIND_MACRO, 0);
+
+// Check that spacing does not invalidate the fixes 
+std::array<int , 3> longname;
+
+auto z1 = longname   [    0    ]  ;
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:22: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto z1 = longname   .at(    0    )  ;
+// CHECK-FIXES-FUNC: auto z1 = f(longname   ,     0    )  ;
+auto z2 = longname   . operator[]   ( 0 );
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto z2 = longname   . at   ( 0 );
+// CHECK-FIXES-FUNC: auto z2 = f(longname   ,  0 );
+auto z3 = (&longname)   -> operator[]   ( 0 );
+// CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:11: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT: auto z3 = (&longname)   -> at   ( 0 );
+// CHECK-FIXES-FUNC: auto z3 = f(*(&longname)   ,  0 );
+
+
+//--- cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-no-excl.cpp
+
+#include "cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.h"
+
+class ExcludedClass1 {
+  public:
+    int operator[](unsigned i) {
+      return 1;
+    }
+    int at(unsigned i) {
+      return 1;
+    }
+};
+
+class ExcludedClass2 {
+  public:
+    int operator[](unsigned i) {
+      return 1;
+    }
+    int at(unsigned i) {
+      return 1;
+    }
+};
+
+ExcludedClass1 E1;
+auto x1 = E1[0];
+// CHECK-MESSAGES-DEFAULT-NO-EXCL: :[[@LINE-1]]:13: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT-NO-EXCL: auto x1 = E1.at(0);
+// CHECK-FIXES-FUNC-NO-EXCL: auto x1 = f(E1, 0);
+
+ExcludedClass2 E2;
+auto x2 = E2[0];
+// CHECK-MESSAGES-DEFAULT-NO-EXCL: :[[@LINE-1]]:13: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT-NO-EXCL: auto x2 = E2.at(0);
+// CHECK-FIXES-FUNC-NO-EXCL: auto x2 = f(E2, 0);
+
+std::map<int,int> TestMapNoExcl;
+auto y = TestMapNoExcl[0];
+// CHECK-MESSAGES-DEFAULT-NO-EXCL: :[[@LINE-1]]:23: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT-NO-EXCL: auto y = TestMapNoExcl.at(0);
+// CHECK-FIXES-FUNC-NO-EXCL: auto y = f(TestMapNoExcl, 0);
+
+
+//--- cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-excl.cpp
+
+#include "cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.h"
+
+std::map<int,int> TestMapExcl;
+auto y = TestMapExcl[0];
+// CHECK-MESSAGES-DEFAULT-EXCL: :[[@LINE-1]]:21: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT-EXCL: auto y = TestMapExcl.at(0);   
+// CHECK-FIXES-FUNC-EXCL: auto y = f(TestMapExcl, 0); 
+
+
+//--- cppcoreguidelines-pro-bounds-avoid-unchecked-container-access-cxx-23.cpp
+#ifdef CXX_23
+#include "cppcoreguidelines-pro-bounds-avoid-unchecked-container-access.h"
+
+namespace std {
+  template<typename T, unsigned size>
+  struct array_cxx_23 {
+    T operator[]() {
+      return T{1};
+    }
+    T at() {
+      return T{1};
+    }
+  };
+};
+
+std::array_cxx_23<int, 3> a;
+
+auto b23 = a[];
+// CHECK-MESSAGES-DEFAULT-CXX-23: :[[@LINE-1]]:13: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT-CXX-23: auto b23 = a.at();
+// CHECK-FIXES-FUNC-CXX-23: auto b23 = f(a);
+// CHECK-FIXES-FUNC-EMPTY-ARGS-CXX-23: auto b23 = g(a);
+// CHECK-MESSAGES-FUNC-EMPTY-ARGS-EMPTY-CXX-23: :[[@LINE-5]]:13: warning: possibly unsafe 'operator[]' [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access] 
+// CHECK-MESSAGES-FUNC-EMPTY-ARGS-EMPTY-CXX-23-NOT: :[[@LINE-6]]:{{.*}}: note: FIX-IT applied suggested code changes 
+
+auto fa23 = (&a)->operator[]();
+// CHECK-MESSAGES-DEFAULT-CXX-23: :[[@LINE-1]]:13: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT-CXX-23: auto fa23 = (&a)->at();
+// CHECK-FIXES-FUNC-CXX-23: auto fa23 = f(*(&a));
+// CHECK-FIXES-FUNC-EMPTY-ARGS-CXX-23: auto fa23 = g(*(&a));
+// CHECK-MESSAGES-FUNC-EMPTY-ARGS-EMPTY-CXX-23: :[[@LINE-5]]:13: warning: possibly unsafe 'operator[]' [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access] 
+// CHECK-MESSAGES-FUNC-EMPTY-ARGS-EMPTY-CXX-23-NOT: :[[@LINE-6]]:{{.*}}: note: FIX-IT applied suggested code changes 
+
+auto fd23 = a.operator[]();
+// CHECK-MESSAGES-DEFAULT-CXX-23: :[[@LINE-1]]:13: warning: possibly unsafe 'operator[]', consider bounds-safe alternatives [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access]
+// CHECK-FIXES-AT-CXX-23: auto fd23 = a.at();
+// CHECK-FIXES-FUNC-CXX-23: auto fd23 = f(a);
+// CHECK-FIXES-FUNC-EMPTY-ARGS-CXX-23: auto fd23 = g(a);
+// CHECK-MESSAGES-FUNC-EMPTY-ARGS-EMPTY-CXX-23: :[[@LINE-5]]:13: warning: possibly unsafe 'operator[]' [cppcoreguidelines-pro-bounds-avoid-unchecked-container-access] 
+// CHECK-MESSAGES-FUNC-EMPTY-ARGS-EMPTY-CXX-23-NOT: :[[@LINE-6]]:{{.*}}: note: FIX-IT applied suggested code changes 
+#endif
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp
index aed6080471e1f..fa81c135a1803 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/pro-bounds-pointer-arithmetic.cpp
@@ -1,4 +1,6 @@
-// RUN: %check_clang_tidy %s cppcoreguidelines-pro-bounds-pointer-arithmetic %t
+// RUN: %check_clang_tidy %s cppcoreguidelines-pro-bounds-pointer-arithmetic -check-suffixes=,DEFAULT  %t
+// RUN: %check_clang_tidy %s cppcoreguidelines-pro-bounds-pointer-arithmetic %t -- \
+// RUN:   -config="{CheckOptions: {cppcoreguidelines-pro-bounds-pointer-arithmetic.AllowIncrementDecrementOperators: true}}" --
 
 enum E {
   ENUM_LITERAL = 1
@@ -42,14 +44,14 @@ void fail() {
   // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not use pointer arithmetic
 
   p++;
-  // CHECK-MESSAGES: :[[@LINE-1]]:4: warning: do not use pointer arithmetic
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:4: warning: do not use pointer arithmetic
   ++p;
-  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: do not use pointer arithmetic
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:3: warning: do not use pointer arithmetic
 
   p--;
-  // CHECK-MESSAGES: :[[@LINE-1]]:4: warning: do not use pointer arithmetic
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:4: warning: do not use pointer arithmetic
   --p;
-  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: do not use pointer arithmetic
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:3: warning: do not use pointer arithmetic
 
   i = p[1];
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not use pointer arithmetic
@@ -57,7 +59,7 @@ void fail() {
   p = ip + 1;
   // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: do not use pointer arithmetic
   ip++;
-  // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: do not use pointer arithmetic
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:5: warning: do not use pointer arithmetic
   i = ip[1];
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not use pointer arithmetic
 }
@@ -72,7 +74,7 @@ void template_fail() {
   q = p - 1;
   // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: do not use pointer arithmetic
   p++;
-  // CHECK-MESSAGES: :[[@LINE-1]]:4: warning: do not use pointer arithmetic
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:4: warning: do not use pointer arithmetic
   i = p[1];
   // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: do not use pointer arithmetic
 }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/google/objc-function-naming.m b/clang-tools-extra/test/clang-tidy/checkers/google/objc-function-naming.m
index 4c204c7468256..13d00a86b3c72 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/google/objc-function-naming.m
+++ b/clang-tools-extra/test/clang-tidy/checkers/google/objc-function-naming.m
@@ -8,7 +8,7 @@ static void TestImplicitFunctionDeclaration(int a) {
   printf("%d", a);
 }
 
-typedef _Bool bool;
+#define bool _Bool
 
 static bool ispositive(int a) { return a > 0; }
 // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: static function named 'ispositive'
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/override-with-different-visibility/test-system-header.h b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/override-with-different-visibility/test-system-header.h
new file mode 100644
index 0000000000000..e64e1924a1708
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/Inputs/override-with-different-visibility/test-system-header.h
@@ -0,0 +1,14 @@
+#pragma clang system_header
+
+namespace sys {
+
+struct Base {
+  virtual void publicF();
+};
+
+struct Derived: public Base {
+private:
+  void publicF() override;
+};
+
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/override-with-different-visibility-ignore.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/override-with-different-visibility-ignore.cpp
new file mode 100644
index 0000000000000..934cfb7bc7082
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/override-with-different-visibility-ignore.cpp
@@ -0,0 +1,60 @@
+// RUN: %check_clang_tidy %s misc-override-with-different-visibility %t -- \
+// RUN:   -config="{CheckOptions: {misc-override-with-different-visibility.IgnoredFunctions: 'IgnoreAlways::.*;::a::IgnoreSelected::.*;IgnoreFunctions::f1;ignored_f'}}"
+
+class IgnoreAlways {
+  virtual void f();
+};
+
+class IgnoreSelected {
+  virtual void f();
+};
+
+namespace a {
+class IgnoreAlways {
+  virtual void f();
+};
+class IgnoreSelected {
+  virtual void f();
+};
+}
+
+namespace ignore_always {
+class Test1: public IgnoreAlways {
+public:
+  void f();
+  void ignored_f(int);
+};
+class Test2: public a::IgnoreAlways {
+public:
+  void f();
+};
+}
+
+namespace ignore_selected {
+class Test1: public IgnoreSelected {
+public:
+  void f();
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'f'
+  // CHECK-MESSAGES: :9:16: note: function declared here
+  void ignored_f(int);
+};
+class Test2: public a::IgnoreSelected {
+public:
+  void f();
+};
+}
+
+class IgnoreFunctions {
+  virtual void f1();
+  virtual void f2();
+  virtual void ignored_f();
+};
+
+class IgnoreFunctionsTest: public IgnoreFunctions {
+public:
+  void f1();
+  void f2();
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'f2'
+  // CHECK-MESSAGES: :[[@LINE-9]]:16: note: function declared here
+  void ignored_f();
+};
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/override-with-different-visibility-options.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/override-with-different-visibility-options.cpp
new file mode 100644
index 0000000000000..0a363ddee3806
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/override-with-different-visibility-options.cpp
@@ -0,0 +1,75 @@
+// RUN: %check_clang_tidy -check-suffixes=DTORS,WIDENING,NARROWING %s misc-override-with-different-visibility %t -- \
+// RUN:   -config="{CheckOptions: {misc-override-with-different-visibility.CheckDestructors: true}}"
+
+// RUN: %check_clang_tidy -check-suffixes=OPS,WIDENING,NARROWING %s misc-override-with-different-visibility %t -- \
+// RUN:   -config="{CheckOptions: {misc-override-with-different-visibility.CheckOperators: true}}"
+
+// RUN: %check_clang_tidy -check-suffixes=WIDENING %s misc-override-with-different-visibility %t -- \
+// RUN:   -config="{CheckOptions: {misc-override-with-different-visibility.DisallowedVisibilityChange: 'widening'}}"
+
+// RUN: %check_clang_tidy -check-suffixes=NARROWING %s misc-override-with-different-visibility %t -- \
+// RUN:   -config="{CheckOptions: {misc-override-with-different-visibility.DisallowedVisibilityChange: 'narrowing'}}"
+
+namespace test_change {
+
+class A {
+protected:
+  virtual void f1();
+  virtual void f2();
+};
+
+class B: public A {
+public:
+  void f1();
+  // CHECK-MESSAGES-WIDENING: :[[@LINE-1]]:8: warning: visibility of function 'f1'
+  // CHECK-MESSAGES-WIDENING: :[[@LINE-8]]:16: note: function declared here
+private:
+  void f2();
+  // CHECK-MESSAGES-NARROWING: :[[@LINE-1]]:8: warning: visibility of function 'f2'
+  // CHECK-MESSAGES-NARROWING: :[[@LINE-11]]:16: note: function declared here
+};
+
+}
+
+namespace test_destructor {
+
+class A {
+public:
+  virtual ~A();
+};
+
+class B: public A {
+protected:
+  ~B();
+  // CHECK-MESSAGES-DTORS: :[[@LINE-1]]:3: warning: visibility of function '~B'
+  // CHECK-MESSAGES-DTORS: :[[@LINE-7]]:11: note: function declared here
+};
+
+}
+
+namespace test_operator {
+
+class A {
+  virtual A& operator=(const A&);
+  virtual A& operator++();
+  virtual int operator()(int);
+  virtual operator double() const;
+};
+
+class B: public A {
+protected:
+  A& operator=(const A&);
+  // CHECK-MESSAGES-OPS: :[[@LINE-1]]:6: warning: visibility of function 'operator='
+  // CHECK-MESSAGES-OPS: :[[@LINE-10]]:14: note: function declared here
+  A& operator++();
+  // CHECK-MESSAGES-OPS: :[[@LINE-1]]:6: warning: visibility of function 'operator++'
+  // CHECK-MESSAGES-OPS: :[[@LINE-12]]:14: note: function declared here
+  int operator()(int);
+  // CHECK-MESSAGES-OPS: :[[@LINE-1]]:7: warning: visibility of function 'operator()'
+  // CHECK-MESSAGES-OPS: :[[@LINE-14]]:15: note: function declared here
+  operator double() const;
+  // CHECK-MESSAGES-OPS: :[[@LINE-1]]:3: warning: visibility of function 'operator double'
+  // CHECK-MESSAGES-OPS: :[[@LINE-16]]:11: note: function declared here
+};
+
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/override-with-different-visibility.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/override-with-different-visibility.cpp
new file mode 100644
index 0000000000000..fd541a44dc25f
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/override-with-different-visibility.cpp
@@ -0,0 +1,289 @@
+// RUN: %check_clang_tidy %s misc-override-with-different-visibility %t -- -config="{CheckOptions: {misc-override-with-different-visibility.CheckDestructors: true,misc-override-with-different-visibility.CheckOperators: true}}" -- -I %S/Inputs/override-with-different-visibility
+#include <test-system-header.h>
+class A {
+public:
+  virtual void pub_foo1() {}
+  virtual void pub_foo2() {}
+  virtual void pub_foo3() {}
+protected:
+  virtual void prot_foo1();
+  virtual void prot_foo2();
+  virtual void prot_foo3();
+private:
+  virtual void priv_foo1() {}
+  virtual void priv_foo2() {}
+  virtual void priv_foo3() {}
+};
+
+void A::prot_foo1() {}
+void A::prot_foo2() {}
+void A::prot_foo3() {}
+
+namespace test1 {
+
+class B: public A {
+public:
+  void pub_foo1() override {}
+  void prot_foo1() override {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'prot_foo1' is changed from protected in class 'A' to public [misc-override-with-different-visibility]
+  // CHECK-MESSAGES: :9:16: note: function declared here as protected
+  void priv_foo1() override {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'priv_foo1' is changed from private in class 'A' to public [misc-override-with-different-visibility]
+  // CHECK-MESSAGES: :13:16: note: function declared here as private
+protected:
+  void pub_foo2() override {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'pub_foo2' is changed from public in class 'A' to protected [misc-override-with-different-visibility]
+  // CHECK-MESSAGES: :6:16: note: function declared here as public
+  void prot_foo2() override {}
+  void priv_foo2() override {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'priv_foo2' is changed from private in class 'A' to protected [misc-override-with-different-visibility]
+  // CHECK-MESSAGES: :14:16: note: function declared here as private
+private:
+  void pub_foo3() override {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'pub_foo3' is changed from public in class 'A' to private [misc-override-with-different-visibility]
+  // CHECK-MESSAGES: :7:16: note: function declared here as public
+  void prot_foo3() override {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'prot_foo3' is changed from protected in class 'A' to private [misc-override-with-different-visibility]
+  // CHECK-MESSAGES: :11:16: note: function declared here as protected
+  void priv_foo3() override {}
+};
+
+class C: public B {
+public:
+  void pub_foo1() override;
+protected:
+  void prot_foo1() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'prot_foo1' is changed from public in class 'B' to protected [misc-override-with-different-visibility]
+  // CHECK-MESSAGES: :27:8: note: function declared here as public
+private:
+  void priv_foo1() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'priv_foo1' is changed from public in class 'B' to private [misc-override-with-different-visibility]
+  // CHECK-MESSAGES: :30:8: note: function declared here as public
+};
+
+void C::prot_foo1() {}
+void C::priv_foo1() {}
+
+}
+
+namespace test2 {
+
+class B: public A {
+public:
+  void pub_foo1() override;
+protected:
+  void prot_foo1() override;
+private:
+  void priv_foo1() override;
+};
+
+class C: public B {
+public:
+  void pub_foo1() override;
+  void prot_foo1() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'prot_foo1' is changed from protected in class 'B' to public
+  // CHECK-MESSAGES: :75:8: note: function declared here as protected
+  void priv_foo1() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'priv_foo1' is changed from private in class 'B' to public
+  // CHECK-MESSAGES: :77:8: note: function declared here as private
+
+  void pub_foo2() override;
+  void prot_foo2() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'prot_foo2' is changed from protected in class 'A' to public
+  // CHECK-MESSAGES: :10:16: note: function declared here as protected
+  void priv_foo2() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'priv_foo2' is changed from private in class 'A' to public
+  // CHECK-MESSAGES: :14:16: note: function declared here as private
+};
+
+}
+
+namespace test3 {
+
+class B: private A {
+public:
+  void pub_foo1() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'pub_foo1' is changed from private (through private inheritance of class 'A') to public
+  // CHECK-MESSAGES: :103:10: note: 'A' is inherited as private here
+  // CHECK-MESSAGES: :5:16: note: function declared here as public
+protected:
+  void prot_foo1() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'prot_foo1' is changed from private (through private inheritance of class 'A') to protected
+  // CHECK-MESSAGES: :103:10: note: 'A' is inherited as private here
+  // CHECK-MESSAGES: :9:16: note: function declared here as protected
+private:
+  void priv_foo1() override;
+
+public:
+  void prot_foo2() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'prot_foo2' is changed from private (through private inheritance of class 'A') to public
+  // CHECK-MESSAGES: :103:10: note: 'A' is inherited as private here
+  // CHECK-MESSAGES: :10:16: note: function declared here as protected
+  void priv_foo2() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'priv_foo2' is changed from private in class 'A' to public
+  // CHECK-MESSAGES: :14:16: note: function declared here as private
+
+private:
+  void pub_foo3() override;
+  void prot_foo3() override;
+};
+
+class C: private A {
+};
+
+class D: public C {
+public:
+  void pub_foo1() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'pub_foo1' is changed from private (through private inheritance of class 'A') to public
+  // CHECK-MESSAGES: :131:10: note: 'A' is inherited as private here
+  // CHECK-MESSAGES: :5:16: note: function declared here as public
+};
+
+
+}
+
+namespace test4 {
+
+struct Base1 {
+public:
+  virtual void foo1();
+private:
+  virtual void foo2();
+};
+
+struct Base2 {
+public:
+  virtual void foo2();
+private:
+  virtual void foo1();
+};
+
+struct A : public Base1, public Base2 {
+protected:
+  void foo1() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'foo1' is changed from private in class 'Base2' to protected
+  // CHECK-MESSAGES: :158:16: note: function declared here as private
+  // CHECK-MESSAGES: :[[@LINE-3]]:8: warning: visibility of function 'foo1' is changed from public in class 'Base1' to protected
+  // CHECK-MESSAGES: :149:16: note: function declared here as public
+private:
+  void foo2() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'foo2' is changed from public in class 'Base2' to private
+  // CHECK-MESSAGES: :156:16: note: function declared here as public
+};
+
+}
+
+namespace test5 {
+
+struct B1: virtual public A {};
+struct B2: virtual private A {};
+struct B: public B1, public B2 {
+public:
+  void pub_foo1() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'pub_foo1' is changed from private (through private inheritance of class 'A') to public
+  // CHECK-MESSAGES: :179:12: note: 'A' is inherited as private here
+  // CHECK-MESSAGES: :5:16: note: function declared here as public
+};
+
+}
+
+namespace test_using {
+
+class A {
+private:
+  A(int);
+protected:
+  virtual void f();
+};
+
+class B: public A {
+public:
+  using A::A;
+  using A::f;
+};
+
+}
+
+namespace test_template {
+
+template <typename T>
+class A {
+protected:
+  virtual T foo();
+};
+
+template <typename T>
+class B: public A<T> {
+private:
+  T foo() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: visibility of function 'foo' is changed from protected in class 'A<int>' to private
+  // CHECK-MESSAGES: :[[@LINE-8]]:13: note: function declared here as protected
+};
+
+template <typename T>
+class C: private A<T> {
+public:
+  T foo() override;
+  // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: visibility of function 'foo' is changed from private (through private inheritance of class 'A<int>') to public
+  // CHECK-MESSAGES: :[[@LINE-4]]:10: note: 'A<int>' is inherited as private here
+  // CHECK-MESSAGES: :[[@LINE-17]]:13: note: function declared here as protected
+};
+
+B<int> fB() {
+  return B<int>{};
+}
+
+C<int> fC() {
+  return C<int>{};
+}
+
+}
+
+namespace test_system_header {
+
+struct SysDerived: public sys::Base {
+private:
+  void publicF();
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: visibility of function 'publicF' is changed from public in class 'Base' to private
+};
+
+}
+
+namespace test_destructor {
+
+class A {
+public:
+  virtual ~A();
+};
+
+class B: public A {
+protected:
+  ~B();
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: visibility of function '~B'
+  // CHECK-MESSAGES: :[[@LINE-7]]:11: note: function declared here
+};
+
+}
+
+namespace test_operator {
+
+class A {
+  virtual int operator()(int);
+  virtual A& operator++();
+  virtual operator double() const;
+};
+
+class B: public A {
+protected:
+  int operator()(int);
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: visibility of function 'operator()'
+  // CHECK-MESSAGES: :[[@LINE-9]]:15: note: function declared here
+  A& operator++();
+  // CHECK-MESSAGES: :[[@LINE-1]]:6: warning: visibility of function 'operator++'
+  // CHECK-MESSAGES: :[[@LINE-11]]:14: note: function declared here
+  operator double() const;
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: visibility of function 'operator double'
+  // CHECK-MESSAGES: :[[@LINE-13]]:11: note: function declared here
+};
+
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.c b/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.c
index 77262c219df03..b83a09362da04 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/static-assert.c
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s misc-static-assert %t -- -- -std=c11
+// RUN: %check_clang_tidy -std=c11-or-later %s misc-static-assert %t
 // RUN: clang-tidy %s -checks=-*,misc-static-assert -- -std=c99 | count 0
 
 void abort(void) {}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp
index 28b53ae4af63d..d7a5797cc2844 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/unconventional-assign-operator.cpp
@@ -176,3 +176,11 @@ struct TemplateAssignment {
   }
 };
 }
+
+namespace GH153770 {
+  struct A;
+  struct A {
+    A() = default;
+    A& operator=(const A&) = default;
+  };
+} // namespace GH153770
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/unused-parameters.c b/clang-tools-extra/test/clang-tidy/checkers/misc/unused-parameters.c
index bde93ea219a57..e2f501c708f01 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/unused-parameters.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/unused-parameters.c
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s misc-unused-parameters %t -- -- -Wno-strict-prototypes -xc
+// RUN: %check_clang_tidy %s misc-unused-parameters %t -- -- -Wno-strict-prototypes
 
 // Basic removal
 // =============
@@ -6,14 +6,18 @@ void a(int i) {;}
 // CHECK-MESSAGES: :[[@LINE-1]]:12: warning: parameter 'i' is unused [misc-unused-parameters]
 // CHECK-FIXES: {{^}}void a(int i) {;}{{$}}
 
-static void b(); // In C, forward declarations can leave out parameters.
+#if __STDC_VERSION__ < 202311L
+static void b(); // In C before C23, forward declarations can leave out parameters.
+#endif
 static void b(int i) {;}
 // CHECK-MESSAGES: :[[@LINE-1]]:19: warning: parameter 'i' is unused [misc-unused-parameters]
 // CHECK-FIXES: {{^}}static void b() {;}{{$}}
 
 // Unchanged cases
 // ===============
+#if __STDC_VERSION__ < 202311L
 void h(i, c, d) int i; char *c, *d; {} // Don't mess with K&R style
+#endif
 
 // Do not warn on naked functions.
 __attribute__((naked)) void nakedFunction(int a, int b) { ; }
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/type-traits-GH153649.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/type-traits-GH153649.cpp
new file mode 100644
index 0000000000000..142eb5847ae10
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/type-traits-GH153649.cpp
@@ -0,0 +1,15 @@
+// RUN: %check_clang_tidy -std=c++20 %s modernize-type-traits %t
+
+namespace std {
+template <class> struct tuple_size {
+  static const int value = 1;
+};
+template <int, class> struct tuple_element {
+  using type = int;
+};
+}
+
+struct A {};
+template <int> int get(const A&);
+
+auto [a] = A();
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-constraints.cpp b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-constraints.cpp
index 3bcd5cd74024e..90131c3d86920 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-constraints.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-constraints.cpp
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy -std=c++20 %s modernize-use-constraints %t -- -- -fno-delayed-template-parsing
+// RUN: %check_clang_tidy -std=c++20-or-later %s modernize-use-constraints %t -- -- -fno-delayed-template-parsing
 
 // NOLINTBEGIN
 namespace std {
@@ -756,3 +756,69 @@ abs(const number<T, ExpressionTemplates> &v) {
 }
 
 }
+
+template <typename T>
+struct some_type_trait {
+  static constexpr bool value = true;
+};
+
+// Fix-its are offered even for a non-standard enable_if.
+namespace nonstd {
+
+template <bool Condition, typename T = void>
+struct enable_if : std::enable_if<Condition, T> {};
+
+template <bool Condition, typename T = void>
+using enable_if_t = typename enable_if<Condition, T>::type;
+
+}
+
+template <typename T>
+typename nonstd::enable_if<some_type_trait<T>::value, void>::type nonstd_enable_if() {}
+// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use C++20 requires constraints instead of enable_if [modernize-use-constraints]
+// CHECK-FIXES: {{^}}void nonstd_enable_if() requires some_type_trait<T>::value {}{{$}}
+
+template <typename T>
+nonstd::enable_if_t<some_type_trait<T>::value, void> nonstd_enable_if_t() {}
+// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use C++20 requires constraints instead of enable_if [modernize-use-constraints]
+// CHECK-FIXES: {{^}}void nonstd_enable_if_t() requires some_type_trait<T>::value {}{{$}}
+
+template <>
+nonstd::enable_if_t<some_type_trait<int>::value, void> nonstd_enable_if_t<int>() {}
+// FIXME - Support non-dependent enable_ifs.
+
+template <typename T>
+typename nonstd::enable_if<some_type_trait<T>::value>::type nonstd_enable_if_one_param() {}
+// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use C++20 requires constraints instead of enable_if [modernize-use-constraints]
+// CHECK-FIXES: {{^}}void nonstd_enable_if_one_param() requires some_type_trait<T>::value {}{{$}}
+
+template <typename T>
+nonstd::enable_if_t<some_type_trait<T>::value> nonstd_enable_if_t_one_param() {}
+// CHECK-MESSAGES: :[[@LINE-1]]:1: warning: use C++20 requires constraints instead of enable_if [modernize-use-constraints]
+// CHECK-FIXES: {{^}}void nonstd_enable_if_t_one_param() requires some_type_trait<T>::value {}{{$}}
+
+// No fix-its are offered for an enable_if with a different signature from the standard one.
+namespace boost {
+
+template <typename Condition, typename T = void>
+struct enable_if : std::enable_if<Condition::value, T> {};
+
+template <typename Condition, typename T = void>
+using enable_if_t = typename enable_if<Condition, T>::type;
+
+}
+
+template <typename T>
+typename boost::enable_if<some_type_trait<T>, void>::type boost_enable_if() {}
+
+template <typename T>
+boost::enable_if_t<some_type_trait<T>, void> boost_enable_if_t() {}
+
+template <>
+boost::enable_if_t<some_type_trait<int>, void> boost_enable_if_t<int>() {}
+
+template <typename T>
+typename boost::enable_if<some_type_trait<T>>::type boost_enable_if_one_param() {}
+
+template <typename T>
+boost::enable_if_t<some_type_trait<T>> boost_enable_if_t_one_param() {}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-nullptr-c23.c b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-nullptr-c23.c
index 6fb879b91e41c..d9b7ec2f7d190 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/modernize/use-nullptr-c23.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/modernize/use-nullptr-c23.c
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s modernize-use-nullptr %t -- -- -std=c23
+// RUN: %check_clang_tidy -std=c23-or-later %s modernize-use-nullptr %t
 
 #define NULL 0
 
diff --git a/clang-tools-extra/test/clang-tidy/checkers/portability/std-allocator-const.cpp b/clang-tools-extra/test/clang-tidy/checkers/portability/std-allocator-const.cpp
index 732cf5d34aca9..a38594aa94cbb 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/portability/std-allocator-const.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/portability/std-allocator-const.cpp
@@ -43,25 +43,25 @@ template <class T>
 class allocator {};
 
 void simple(const std::vector<const char> &v, std::deque<const short> *d) {
-  // CHECK-MESSAGES: [[#@LINE-1]]:24: warning: container using std::allocator<const T> is a deprecated libc++ extension; remove const for compatibility with other standard libraries
-  // CHECK-MESSAGES: [[#@LINE-2]]:52: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:19: warning: container using std::allocator<const T> is a deprecated libc++ extension; remove const for compatibility with other standard libraries
+  // CHECK-MESSAGES: [[#@LINE-2]]:47: warning: container
   std::list<const long> l;
-  // CHECK-MESSAGES: [[#@LINE-1]]:8: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:3: warning: container
 
   std::multiset<int *const> ms;
-  // CHECK-MESSAGES: [[#@LINE-1]]:8: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:3: warning: container
   std::set<const std::hash<int>> s;
-  // CHECK-MESSAGES: [[#@LINE-1]]:8: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:3: warning: container
   std::unordered_multiset<int *const> ums;
-  // CHECK-MESSAGES: [[#@LINE-1]]:8: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:3: warning: container
   std::unordered_set<const int> us;
-  // CHECK-MESSAGES: [[#@LINE-1]]:8: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:3: warning: container
 
   absl::flat_hash_set<const int> fhs;
-  // CHECK-MESSAGES: [[#@LINE-1]]:9: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:3: warning: container
 
   using my_vector = std::vector<const int>;
-  // CHECK-MESSAGES: [[#@LINE-1]]:26: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:21: warning: container
   my_vector v1;
   using my_vector2 = my_vector;
 
@@ -76,7 +76,7 @@ void simple(const std::vector<const char> &v, std::deque<const short> *d) {
 template <class T>
 void temp1() {
   std::vector<const T> v;
-  // CHECK-MESSAGES: [[#@LINE-1]]:8: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:3: warning: container
 
   std::vector<T> neg1;
   std::forward_list<const T> neg2;
@@ -87,7 +87,7 @@ template <class T>
 void temp2() {
   // Match std::vector<const dependent> for the uninstantiated temp2.
   std::vector<const T> v;
-  // CHECK-MESSAGES: [[#@LINE-1]]:8: warning: container
+  // CHECK-MESSAGES: [[#@LINE-1]]:3: warning: container
 
   std::vector<T> neg1;
   std::forward_list<const T> neg2;
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/bitint-no-crash.c b/clang-tools-extra/test/clang-tidy/checkers/readability/bitint-no-crash.c
index f8660924cbef5..01e458e1b8d8b 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/bitint-no-crash.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/bitint-no-crash.c
@@ -1,6 +1,5 @@
-// RUN: %check_clang_tidy %s readability-magic-numbers %t --
+// RUN: %check_clang_tidy -std=c23-or-later %s readability-magic-numbers %t
 
 // Don't crash
 
 _BitInt(128) A = 4533629751480627964421wb;
-// CHECK-MESSAGES: warning
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp b/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp
index 2fd0b2224cb1c..b1e68672a3a9a 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/container-size-empty.cpp
@@ -895,3 +895,16 @@ namespace PR94454 {
   int operator""_ci() { return 0; }
   auto eq = 0_ci == 0;
 }
+
+namespace GH152387 {
+
+class foo : public std::string{
+  void doit() {
+    if (!size()) {
+      // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: the 'empty' method should be used to check for emptiness instead of 'size'
+      // CHECK-FIXES: if (empty()) {
+    }
+  }
+};
+
+}
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-standard-types.h b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-standard-types.h
index e1036483ee8f6..51fa9c3021348 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-standard-types.h
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/identifier-naming-standard-types.h
@@ -50,8 +50,8 @@ typedef void*               FILE;       // NOLINT
 #define NULL                (0)         // NOLINT
 
 #ifndef __cplusplus
-typedef _Bool               bool;       // NOLINT
 typedef __WCHAR_TYPE__      wchar_t;    // NOLINT
+#define bool                _Bool       // NOLINT
 #define true                1           // NOLINT
 #define false               0           // NOLINT
 #endif
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.c b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.c
index 11ff7dd816a44..5a4627aa49f1a 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/implicit-bool-conversion.c
@@ -1,8 +1,8 @@
-// RUN: %check_clang_tidy --match-partial-fixes %s readability-implicit-bool-conversion %t -- -- -std=c23
-// RUN: %check_clang_tidy -check-suffix=UPPER-CASE %s readability-implicit-bool-conversion %t -- \
+// RUN: %check_clang_tidy -std=c23-or-later --match-partial-fixes %s readability-implicit-bool-conversion %t
+// RUN: %check_clang_tidy -std=c23-or-later -check-suffix=UPPER-CASE %s readability-implicit-bool-conversion %t -- \
 // RUN:     -config='{CheckOptions: { \
 // RUN:         readability-implicit-bool-conversion.UseUpperCaseLiteralSuffix: true \
-// RUN:     }}' -- -std=c23
+// RUN:     }}'
 
 #undef NULL
 #define NULL 0L
diff --git a/clang-tools-extra/test/clang-tidy/checkers/readability/non-const-parameter.c b/clang-tools-extra/test/clang-tidy/checkers/readability/non-const-parameter.c
index db50467f3dd94..e254215f5f5d6 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/readability/non-const-parameter.c
+++ b/clang-tools-extra/test/clang-tidy/checkers/readability/non-const-parameter.c
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s readability-non-const-parameter %t
+// RUN: %check_clang_tidy -std=c99,c11,c17 %s readability-non-const-parameter %t
 
 static int f();
 
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/diagnostic.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/diagnostic.cpp
index 610d1dadfdb56..a9d140bc9318e 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/diagnostic.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/diagnostic.cpp
@@ -15,7 +15,7 @@
 // use it after failing to parse commands from the command line:
 //
 // RUN: mkdir -p %t.dir/diagnostics/
-// RUN: echo '[{"directory": "%/t.dir/diagnostics/","command": "clang++ -fan-option-from-compilation-database -c %/T/diagnostics/input.cpp", "file": "%/T/diagnostics/input.cpp"}]' > %t.dir/diagnostics/compile_commands.json
+// RUN: echo '[{"directory": "%/t.dir/diagnostics/","command": "clang++ -fan-option-from-compilation-database -c %/t.dir/diagnostics/input.cpp", "file": "%/t.dir/diagnostics/input.cpp"}]' > %t.dir/diagnostics/compile_commands.json
 // RUN: cat %s > %t.dir/diagnostics/input.cpp
 // RUN: not clang-tidy -checks='-*,modernize-use-override' %t.dir/diagnostics/nonexistent.cpp -- 2>&1 | FileCheck -check-prefix=CHECK1 -implicit-check-not='{{warning:|error:}}' %s
 // RUN: not clang-tidy -checks='-*,clang-diagnostic-*,google-explicit-constructor' %t.dir/diagnostics/input.cpp -- -fan-unknown-option 2>&1 | FileCheck -check-prefix=CHECK2 -implicit-check-not='{{warning:|error:}}' %s
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/export-relpath.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/export-relpath.cpp
index 5fd7303173e1c..051f008ad3f3a 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/export-relpath.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/export-relpath.cpp
@@ -2,7 +2,7 @@
 // RUN: rm -rf %t.dir/clang-tidy/export-relpath
 // RUN: mkdir -p %t.dir/clang-tidy/export-relpath/subdir
 // RUN: cp %s %t.dir/clang-tidy/export-relpath/subdir/source.cpp
-// RUN: echo '[{ "directory": "%/t.dir/clang-tidy/export-relpath/subdir", "command": "clang++ source.cpp", "file": "%/T/clang-tidy/export-relpath/subdir/source.cpp"}]' > %t.dir/clang-tidy/export-relpath/subdir/compile_commands.json
+// RUN: echo '[{ "directory": "%/t.dir/clang-tidy/export-relpath/subdir", "command": "clang++ source.cpp", "file": "%/t.dir/clang-tidy/export-relpath/subdir/source.cpp"}]' > %t.dir/clang-tidy/export-relpath/subdir/compile_commands.json
 //
 // Check that running clang-tidy in './subdir' and storing results
 // in './fixes.yaml' works as expected.
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter-symlinks.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter-symlinks.cpp
index 7efa7d070f69f..58f3b23cb1dbf 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter-symlinks.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter-symlinks.cpp
@@ -10,7 +10,7 @@
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header_alias\.h' %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER_ALIAS %s
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header_alias\.h' -quiet %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER_ALIAS %s
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header\.h' %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER %s
-// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header\.h' -quiet %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header\.h' -quiet %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER --allow-empty %s
 
 // Check that `-header-filter` operates on the same file paths as paths in
 // diagnostics printed by ClangTidy.
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp
index 448ef9ddf166c..d9ec1049963b0 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/file-filter.cpp
@@ -66,19 +66,14 @@ class A { A(int); };
 // CHECK4-NOT: warning:
 // CHECK4-QUIET-NOT: warning:
 
-// CHECK: Suppressed 3 warnings (3 in non-user code)
 // CHECK: Use -header-filter=.* to display errors from all non-system headers.
 // CHECK-QUIET-NOT: Suppressed
-// CHECK2: Suppressed 1 warnings (1 in non-user code)
-// CHECK2: Use -header-filter=.* {{.*}}
 // CHECK2-QUIET-NOT: Suppressed
-// CHECK3: Suppressed 2 warnings (2 in non-user code)
 // CHECK3: Use -header-filter=.* {{.*}}
 // CHECK3-QUIET-NOT: Suppressed
 // CHECK4-NOT: Suppressed {{.*}} warnings
 // CHECK4-NOT: Use -header-filter=.* {{.*}}
 // CHECK4-QUIET-NOT: Suppressed
-// CHECK6: Suppressed 2 warnings (2 in non-user code)
 // CHECK6: Use -header-filter=.* {{.*}}
 
 int x = 123;
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/quiet-flag.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/quiet-flag.cpp
new file mode 100644
index 0000000000000..0ed6d017a934f
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/quiet-flag.cpp
@@ -0,0 +1,26 @@
+// This test ensures that the --quiet flag only suppresses the "X warnings generated" 
+// message while keeping all diagnostic information including caret indicators (^).
+
+// RUN: clang-tidy -checks=-*,readability-magic-numbers,clang-diagnostic-sign-compare %s -- \
+// RUN:   -Wsign-compare 2>&1 | FileCheck %s --check-prefix=CHECK-NORMAL
+// RUN: clang-tidy -checks=-*,readability-magic-numbers,clang-diagnostic-sign-compare -quiet %s -- \
+// RUN:   -Wsign-compare 2>&1 | FileCheck %s --check-prefix=CHECK-QUIET
+
+// CHECK-NORMAL: 2 warnings generated
+// CHECK-NORMAL-DAG: warning: 42 is a magic number
+// CHECK-NORMAL-DAG: {{[ ]*\^}}
+// CHECK-NORMAL-DAG: warning: comparison of integers of different signs
+// CHECK-NORMAL-DAG: {{[ ]*~ \^ ~}}
+
+// CHECK-QUIET-NOT: {{[0-9]+}} warning{{s?}} generated
+// CHECK-QUIET-DAG: warning: 42 is a magic number
+// CHECK-QUIET-DAG: {{[ ]*\^}}
+// CHECK-QUIET-DAG: warning: comparison of integers of different signs
+// CHECK-QUIET-DAG: {{[ ]*~ \^ ~}}
+
+int main() {
+  const int CONST_VAL = 10;
+  int x = 42; // trigger 'readability-magic-numbers' with caret: ^
+  unsigned int y = CONST_VAL;
+  return x < y; // trigger 'clang-diagnostic-sign-compare' with caret: ^
+}
diff --git a/clang-tools-extra/test/clang-tidy/infrastructure/system-headers.cpp b/clang-tools-extra/test/clang-tidy/infrastructure/system-headers.cpp
index 9fa990b6aac8c..a25480e9aa39c 100644
--- a/clang-tools-extra/test/clang-tidy/infrastructure/system-headers.cpp
+++ b/clang-tools-extra/test/clang-tidy/infrastructure/system-headers.cpp
@@ -11,9 +11,9 @@
 // RUN: clang-tidy -help | FileCheck -check-prefix=CHECK-OPT-PRESENT %s
 
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -system-headers=true %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-SYSTEM-HEADERS %s
-// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -system-headers=false %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-NO-SYSTEM-HEADERS %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -system-headers=false %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-NO-SYSTEM-HEADERS --allow-empty %s
 // RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -config='SystemHeaders: true' %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-SYSTEM-HEADERS %s
-// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -config='SystemHeaders: false' %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-NO-SYSTEM-HEADERS %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='.*' -config='SystemHeaders: false' %s -- -isystem %S/Inputs/system-headers 2>&1 | FileCheck -check-prefix=CHECK-NO-SYSTEM-HEADERS --allow-empty %s
 
 #include <system_header.h>
 // CHECK-SYSTEM-HEADERS: system_header.h:1:13: warning: single-argument constructors must be marked explicit
diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 824674309d262..812ad2cd2dc13 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1907,6 +1907,15 @@ def linkage(self) -> LinkageKind:
 
         return LinkageKind.from_id(self._linkage)
 
+    @property
+    @cursor_null_guard
+    def language(self) -> LanguageKind:
+        """Determine the "language" of the entity referred to by a given cursor."""
+        if not hasattr(self, "_language"):
+            self._language = conf.lib.clang_getCursorLanguage(self)
+
+        return LanguageKind.from_id(self._language)
+
     @property
     @cursor_null_guard
     def tls_kind(self) -> TLSKind:
@@ -2584,6 +2593,17 @@ class LinkageKind(BaseEnumeration):
     EXTERNAL = 4
 
 
+class LanguageKind(BaseEnumeration):
+    """
+    Describe the "language" of the entity referred to by a cursor.
+    """
+
+    INVALID = 0
+    C = 1
+    OBJ_C = 2
+    C_PLUS_PLUS = 3
+
+
 class TLSKind(BaseEnumeration):
     """Describes the kind of thread-local storage (TLS) of a cursor."""
 
@@ -4084,6 +4104,7 @@ def set_property(self, property, value):
     ("clang_getCursorDisplayName", [Cursor], _CXString),
     ("clang_getCursorExceptionSpecificationType", [Cursor], c_int),
     ("clang_getCursorExtent", [Cursor], SourceRange),
+    ("clang_getCursorLanguage", [Cursor], c_int),
     ("clang_getCursorLexicalParent", [Cursor], Cursor),
     ("clang_getCursorLinkage", [Cursor], c_int),
     ("clang_getCursorLocation", [Cursor], SourceLocation),
diff --git a/clang/bindings/python/tests/CMakeLists.txt b/clang/bindings/python/tests/CMakeLists.txt
index a0ddabc21bb41..d9a6bbf452bd6 100644
--- a/clang/bindings/python/tests/CMakeLists.txt
+++ b/clang/bindings/python/tests/CMakeLists.txt
@@ -35,7 +35,7 @@ if(WIN32)
 endif()
 
 # The Python FFI interface is broken on AIX: https://bugs.python.org/issue38628.
-if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(RUN_PYTHON_TESTS FALSE)
 endif()
 
diff --git a/clang/bindings/python/tests/cindex/test_cursor_language.py b/clang/bindings/python/tests/cindex/test_cursor_language.py
new file mode 100644
index 0000000000000..de07a7bdeef40
--- /dev/null
+++ b/clang/bindings/python/tests/cindex/test_cursor_language.py
@@ -0,0 +1,27 @@
+import os
+
+from clang.cindex import Config, LanguageKind
+
+if "CLANG_LIBRARY_PATH" in os.environ:
+    Config.set_library_path(os.environ["CLANG_LIBRARY_PATH"])
+
+import unittest
+
+from .util import get_cursor, get_tu
+
+
+class TestCursorLanguage(unittest.TestCase):
+    def test_c(self):
+        tu = get_tu("int a;", lang="c")
+        main_func = get_cursor(tu.cursor, "a")
+        self.assertEqual(main_func.language, LanguageKind.C)
+
+    def test_c(self):
+        tu = get_tu("class Cls {};", lang="cpp")
+        main_func = get_cursor(tu.cursor, "Cls")
+        self.assertEqual(main_func.language, LanguageKind.C_PLUS_PLUS)
+
+    def test_obj_c(self):
+        tu = get_tu("@interface If : NSObject", lang="objc")
+        main_func = get_cursor(tu.cursor, "If")
+        self.assertEqual(main_func.language, LanguageKind.OBJ_C)
diff --git a/clang/bindings/python/tests/cindex/test_enums.py b/clang/bindings/python/tests/cindex/test_enums.py
index 9e7f44fcf7867..48452fd4f82d0 100644
--- a/clang/bindings/python/tests/cindex/test_enums.py
+++ b/clang/bindings/python/tests/cindex/test_enums.py
@@ -6,6 +6,7 @@
     BinaryOperator,
     CursorKind,
     ExceptionSpecificationKind,
+    LanguageKind,
     LinkageKind,
     RefQualifierKind,
     StorageClass,
@@ -26,6 +27,7 @@ class TestEnums(unittest.TestCase):
         AccessSpecifier,
         TypeKind,
         RefQualifierKind,
+        LanguageKind,
         LinkageKind,
         TLSKind,
         StorageClass,
diff --git a/clang/bindings/python/tests/cindex/test_type.py b/clang/bindings/python/tests/cindex/test_type.py
index 34081bb88f3dd..cc101beca8cc5 100644
--- a/clang/bindings/python/tests/cindex/test_type.py
+++ b/clang/bindings/python/tests/cindex/test_type.py
@@ -63,7 +63,7 @@ def test_a_struct(self):
         self.assertIsNotNone(fields[1].translation_unit)
         self.assertEqual(fields[1].spelling, "b")
         self.assertFalse(fields[1].type.is_const_qualified())
-        self.assertEqual(fields[1].type.kind, TypeKind.ELABORATED)
+        self.assertEqual(fields[1].type.kind, TypeKind.TYPEDEF)
         self.assertEqual(fields[1].type.get_canonical().kind, TypeKind.INT)
         self.assertEqual(fields[1].type.get_declaration().spelling, "I")
         self.assertEqual(fields[1].type.get_typedef_name(), "I")
diff --git a/clang/docs/APINotes.rst b/clang/docs/APINotes.rst
index e5ec154237b0d..dec4b186ff72f 100644
--- a/clang/docs/APINotes.rst
+++ b/clang/docs/APINotes.rst
@@ -206,6 +206,17 @@ declaration kind), all of which are optional:
     - Name: tzdb
       SwiftCopyable: false
 
+  A non-copyable type can have a "destroy" operation, specified with
+  `SwiftDestroyOp`, which will be invoked on the instance when it is no
+  longer in use to free up resources.
+
+  ::
+
+    Tags:
+    - Name: WGPUAdapterInfo
+      SwiftCopyable: false
+      SwiftDestroyOp: wgpuAdapterInfoFreeMembers
+
 :SwiftConformsTo:
 
   Allows annotating a C++ class as conforming to a Swift protocol. Equivalent
diff --git a/clang/docs/AutomaticReferenceCounting.rst b/clang/docs/AutomaticReferenceCounting.rst
index bcac73215c9d3..80bbd25121148 100644
--- a/clang/docs/AutomaticReferenceCounting.rst
+++ b/clang/docs/AutomaticReferenceCounting.rst
@@ -740,7 +740,7 @@ following rules apply:
 
 * If the qualifier is so applied at a position in the declaration
   where the next-innermost declarator is a function declarator, and
-  there is an block declarator within that function declarator, then
+  there is a block declarator within that function declarator, then
   the qualifier applies instead to that block declarator and this rule
   is considered afresh beginning from the new position.
 
@@ -924,7 +924,7 @@ not support ``__weak`` references.
 
 A class may indicate that it does not support weak references by providing the
 ``objc_arc_weak_reference_unavailable`` attribute on the class's interface declaration.  A
-retainable object pointer type is **weak-unavailable** if
+retainable object pointer type is **weak-unavailable** if it
 is a pointer to an (optionally protocol-qualified) Objective-C class ``T`` where
 ``T`` or one of its superclasses has the ``objc_arc_weak_reference_unavailable``
 attribute.  A program is ill-formed if it applies the ``__weak`` ownership
@@ -1129,7 +1129,7 @@ be the same for identical code.
   C and C++ while still automatically managing memory.  While it is
   usually simpler and more idiomatic to use Objective-C objects for
   secondary data structures, doing so can introduce extra allocation
-  and message-send overhead, which can cause to unacceptable
+  and message-send overhead, which can cause unacceptable
   performance.  Using structs can resolve some of this tension.
 
   ``__autoreleasing`` is forbidden because it is treacherous to rely
@@ -1446,7 +1446,7 @@ ill-formed.
 Template arguments
 ^^^^^^^^^^^^^^^^^^
 
-If a template argument for a template type parameter is an retainable object
+If a template argument for a template type parameter is a retainable object
 owner type that does not have an explicit ownership qualifier, it is adjusted
 to have ``__strong`` qualification.  This adjustment occurs regardless of
 whether the template argument was deduced or explicitly specified.
@@ -2064,7 +2064,7 @@ You can test if your compiler has support for ``objc_externally_retained`` with
 ``self``
 --------
 
-The ``self`` parameter variable of an non-init Objective-C method is considered
+The ``self`` parameter variable of a non-init Objective-C method is considered
 :ref:`externally-retained <arc.misc.externally_retained>` by the implementation.
 It is undefined behavior, or at least dangerous, to cause an object to be
 deallocated during a message send to that object.  In an init method, ``self``
@@ -2334,7 +2334,7 @@ emit, which are described in the remainder of this section.
 
   * the machine code to do so is significantly smaller,
   * it is much easier to recognize the C functions in the ARC optimizer, and
-  * a sufficient sophisticated runtime may be able to avoid the message send in
+  * a sufficiently sophisticated runtime may be able to avoid the message send in
     common cases.
 
   Several other of these functions are "fused" operations which can be
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 02986a94a656c..3ac9e3795cae7 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -126,6 +126,9 @@ clang-format is turned off or back on.
   // clang-format on
   void formatted_code_again;
 
+In addition, the ``OneLineFormatOffRegex`` option gives you a concise way to
+disable formatting for all of the lines that match the regular expression.
+
 
 Configuring Style in Code
 =========================
@@ -6483,13 +6486,51 @@ the configuration (without a prefix: ``Auto``).
 .. _SpaceInEmptyBlock:
 
 **SpaceInEmptyBlock** (``Boolean``) :versionbadge:`clang-format 10` :ref:`¶ <SpaceInEmptyBlock>`
-  If ``true``, spaces will be inserted into ``{}``.
+  This option is **deprecated**. See ``Block`` of ``SpaceInEmptyBraces``.
+
+.. _SpaceInEmptyBraces:
+
+**SpaceInEmptyBraces** (``SpaceInEmptyBracesStyle``) :versionbadge:`clang-format 22` :ref:`¶ <SpaceInEmptyBraces>`
+  Specifies when to insert a space in empty braces.
+
+  .. note::
+
+   This option doesn't apply to initializer braces if
+   ``Cpp11BracedListStyle`` is set to ``true``.
+
+  Possible values:
+
+  * ``SIEB_Always`` (in configuration: ``Always``)
+    Always insert a space in empty braces.
+
+    .. code-block:: c++
+
+       void f() { }
+       class Unit { };
+       auto a = [] { };
+       int x{ };
+
+  * ``SIEB_Block`` (in configuration: ``Block``)
+    Only insert a space in empty blocks.
+
+    .. code-block:: c++
+
+       void f() { }
+       class Unit { };
+       auto a = [] { };
+       int x{};
+
+  * ``SIEB_Never`` (in configuration: ``Never``)
+    Never insert a space in empty braces.
+
+    .. code-block:: c++
+
+       void f() {}
+       class Unit {};
+       auto a = [] {};
+       int x{};
 
-  .. code-block:: c++
 
-     true:                                false:
-     void f() { }                   vs.   void f() {}
-     while (true) { }                     while (true) {}
 
 .. _SpaceInEmptyParentheses:
 
diff --git a/clang/docs/ClangLinkerWrapper.rst b/clang/docs/ClangLinkerWrapper.rst
index e69cdba434c93..eb38d2b8fb5ee 100644
--- a/clang/docs/ClangLinkerWrapper.rst
+++ b/clang/docs/ClangLinkerWrapper.rst
@@ -60,6 +60,10 @@ only for the linker wrapper will be forwarded to the wrapped linker job.
     --v                    Display the version number and exit
     --                     The separator for the wrapped linker arguments
 
+The linker wrapper will generate the appropriate runtime calls to register the
+generated device binary with the offloading runtime. To do this step manually we
+provide the ``llvm-offload-wrapper`` utility.
+
 Relocatable Linking
 ===================
 
diff --git a/clang/docs/ClangOffloadBundler.rst b/clang/docs/ClangOffloadBundler.rst
index 62cf1642a03a3..5570dbb08ab9a 100644
--- a/clang/docs/ClangOffloadBundler.rst
+++ b/clang/docs/ClangOffloadBundler.rst
@@ -525,15 +525,15 @@ The compressed offload bundle begins with a header followed by the compressed bi
     This is a unique identifier to distinguish compressed offload bundles. The value is the string 'CCOB' (Compressed Clang Offload Bundle).
 
 - **Version Number (16-bit unsigned int)**:
-    This denotes the version of the compressed offload bundle format. The current version is `2`.
+    This denotes the version of the compressed offload bundle format. The current version is `3`.
 
 - **Compression Method (16-bit unsigned int)**:
     This field indicates the compression method used. The value corresponds to either `zlib` or `zstd`, represented as a 16-bit unsigned integer cast from the LLVM compression enumeration.
 
-- **Total File Size (32-bit unsigned int)**:
+- **Total File Size (unsigned int, 32-bit in v2, 64-bit in v3)**:
     This is the total size (in bytes) of the file, including the header. Available in version 2 and above.
 
-- **Uncompressed Binary Size (32-bit unsigned int)**:
+- **Uncompressed Binary Size (unsigned int, 32-bit in v2, 64-bit in v3)**:
     This is the size (in bytes) of the binary data before it was compressed.
 
 - **Hash (64-bit unsigned int)**:
@@ -542,4 +542,4 @@ The compressed offload bundle begins with a header followed by the compressed bi
 - **Compressed Data**:
     The actual compressed binary data follows the header. Its size can be inferred from the total size of the file minus the header size.
 
-    > **Note**: Version 3 of the format is under development. It uses 64-bit fields for Total File Size and Uncompressed Binary Size to support files larger than 4GB. To experiment with version 3, set the environment variable `COMPRESSED_BUNDLE_FORMAT_VERSION=3`. This support is experimental and not recommended for production use.
+    > **Note**: Version 3 is now the default format. For backward compatibility with older HIP runtimes that support version 2 only, set the environment variable `COMPRESSED_BUNDLE_FORMAT_VERSION=2`.
diff --git a/clang/docs/DebuggingCoroutines.rst b/clang/docs/DebuggingCoroutines.rst
index 8702152747c65..9eaf8d4028adf 100644
--- a/clang/docs/DebuggingCoroutines.rst
+++ b/clang/docs/DebuggingCoroutines.rst
@@ -9,14 +9,14 @@ Introduction
 ============
 
 Coroutines in C++ were introduced in C++20, and the user experience for
-debugging them can still be challenging. This document guides you how to most
+debugging them can still be challenging. This document guides you on how to most
 efficiently debug coroutines and how to navigate existing shortcomings in
 debuggers and compilers.
 
 Coroutines are generally used either as generators or for asynchronous
 programming. In this document, we will discuss both use cases. Even if you are
 using coroutines for asynchronous programming, you should still read the
-generators section, as it will introduce foundational debugging techniques also
+generators section, as it introduces foundational debugging techniques also
 applicable to the debugging of asynchronous programs.
 
 Both compilers (clang, gcc, ...) and debuggers (lldb, gdb, ...) are
@@ -34,15 +34,15 @@ scripting. This guide comes with a basic GDB script for coroutine debugging.
 This guide will first showcase the more polished, bleeding-edge experience, but
 will also show you how to debug coroutines with older toolchains. In general,
 the older your toolchain, the deeper you will have to dive into the
-implementation details of coroutines (such as their ABI). The further down in
-this document you go, the more low-level, technical the content will become. If
+implementation details of coroutines (such as their ABI). The further down you go in
+this document, the more low-level, technical the content will become. If
 you are on an up-to-date toolchain, you will hopefully be able to stop reading
 earlier.
 
 Debugging generators
 ====================
 
-One of the two major use cases for coroutines in C++ are generators, i.e.,
+One of the two major use cases for coroutines in C++ is generators, i.e.,
 functions which can produce values via ``co_yield``. Values are produced
 lazily, on-demand. For this purpose, every time a new value is requested, the
 coroutine gets resumed. As soon as it reaches a ``co_yield`` and thereby
@@ -141,7 +141,7 @@ a regular function.
 
 Note the two additional variables ``__promise`` and ``__coro_frame``. Those
 show the internal state of the coroutine. They are not relevant for our
-generator example, but will be relevant for asynchronous programming described
+generator example but will be relevant for asynchronous programming described
 in the next section.
 
 Stepping out of a coroutine
@@ -174,7 +174,7 @@ Inspecting a suspended coroutine
 --------------------------------
 
 The ``print10Elements`` function receives an opaque ``generator`` type. Let's
-assume we are suspended at the ``++gen;`` line, and want to inspect the
+assume we are suspended at the ``++gen;`` line and want to inspect the
 generator and its internal state.
 
 To do so, we can simply look into the ``gen.hdl`` variable. LLDB comes with a
@@ -188,7 +188,7 @@ We can see two function pointers ``resume`` and ``destroy``. These pointers
 point to the resume / destroy functions. By inspecting those function pointers,
 we can see that our ``generator`` is actually backed by our ``fibonacci``
 coroutine. When using VS Code + lldb-dap, you can Cmd+Click on the function
-address (``0x555...`` in the screenshot) to directly jump to the function
+address (``0x555...`` in the screenshot) to jump directly to the function
 definition backing your coroutine handle.
 
 Next, we see the ``promise``. In our case, this reveals the current value of
@@ -247,12 +247,12 @@ the line number of the current suspension point in the promise:
   };
 
 This stores the return address of ``await_suspend`` within the promise.
-Thereby, we can read it back from the promise of a suspended coroutine, and map
+Thereby, we can read it back from the promise of a suspended coroutine and map
 it to an exact source code location. For a complete example, see the ``task``
 type used below for asynchronous programming.
 
 Alternatively, we can modify the C++ code to store the line number in the
-promise type. We can use a ``std::source_location`` to get the line number of
+promise type. We can use ``std::source_location`` to get the line number of
 the await and store it inside the ``promise_type``. In the debugger, we can
 then read the line number from the promise of the suspended coroutine.
 
@@ -270,7 +270,7 @@ then read the line number from the promise of the suspended coroutine.
   };
 
 The downside of both approaches is that they come at the price of additional
-runtime cost. In particular the second approach increases binary size, since it
+runtime cost. In particular, the second approach increases binary size, since it
 requires additional ``std::source_location`` objects, and those source
 locations are not stripped by split-dwarf. Whether the first approach is worth
 the additional runtime cost is a trade-off you need to make yourself.
@@ -285,7 +285,7 @@ provide custom debugging support, so in addition to this guide, you might want
 to check out their documentation.
 
 When using coroutines for asynchronous programming, your library usually
-provides you some ``task`` type. This type usually looks similar to this:
+provides you with some ``task`` type. This type usually looks similar to this:
 
 .. code-block:: c++
 
@@ -479,7 +479,7 @@ One such solution is to store the list of in-flight coroutines in a collection:
   };
 
 With this in place, it is possible to inspect ``inflight_coroutines`` from the
-debugger, and rely on LLDB's ``std::coroutine_handle`` pretty-printer to
+debugger and rely on LLDB's ``std::coroutine_handle`` pretty-printer to
 inspect the coroutines.
 
 This technique will track *all* coroutines, also the ones which are currently
@@ -498,8 +498,8 @@ LLDB before 21.0 did not yet show the ``__coro_frame`` inside
 ``coroutine_handle``. To inspect the coroutine frame, you had to use the
 approach described in the :ref:`devirtualization` section.
 
-LLDB before 18.0 was hiding the ``__promise`` and ``__coro_frame``
-variable by default. The variables are still present, but they need to be
+LLDB before 18.0 hid the ``__promise`` and ``__coro_frame``
+variables by default. The variables are still present, but they need to be
 explicitly added to the "watch" pane in VS Code or requested via
 ``print __promise`` and ``print __coro_frame`` from the debugger console.
 
@@ -511,9 +511,9 @@ section.
 Toolchain Implementation Details
 ================================
 
-This section covers the ABI, as well as additional compiler-specific behavior.
+This section covers the ABI as well as additional compiler-specific behavior.
 The ABI is followed by all compilers, on all major systems, including Windows,
-Linux and macOS. Different compilers emit different debug information, though.
+Linux, and macOS. Different compilers emit different debug information, though.
 
 Ramp, resume and destroy functions
 ----------------------------------
@@ -595,7 +595,7 @@ functions as their first two members. As such, we can read the function
 pointers from the coroutine frame and then obtain the function's name from its
 address.
 
-The promise is guaranteed to be at a 16 byte offset from the coroutine frame.
+The promise is guaranteed to be at a 16-byte offset from the coroutine frame.
 If we have a coroutine handle at address 0x416eb0, we can hence reinterpret-cast
 the promise as follows:
 
@@ -607,8 +607,8 @@ Implementation in clang / LLVM
 ------------------------------
 
 The C++ Coroutines feature in the Clang compiler is implemented in two parts of
-the compiler. Semantic analysis is performed in Clang, and Coroutine
-construction and optimization takes place in the LLVM middle-end.
+the compiler. Semantic analysis is performed in Clang, and coroutine
+construction and optimization take place in the LLVM middle-end.
 
 For each coroutine function, the frontend generates a single corresponding
 LLVM-IR function. This function uses special ``llvm.coro.suspend`` intrinsics
@@ -622,7 +622,7 @@ points into the coroutine frame. Most of the heavy lifting to preserve debugging
 information is done in this pass. This pass needs to rewrite all variable
 locations to point into the coroutine frame.
 
-Afterwards, a couple of additional optimizations are applied, before code
+Afterwards, a couple of additional optimizations are applied before code
 gets emitted, but none of them are really interesting regarding debugging
 information.
 
@@ -636,8 +636,8 @@ However, this is not possible for coroutine frames because the frames are
 constructed in the LLVM middle-end.
 
 To mitigate this problem, the LLVM middle end attempts to generate some debug
-information, which is unfortunately incomplete, since much of the language
-specific information is missing in the middle end.
+information, which is unfortunately incomplete, since much of the
+language-specific information is missing in the middle end.
 
 .. _devirtualization:
 
@@ -655,7 +655,7 @@ There are two possible approaches to do so:
    We can lookup their types and thereby get the types of promise
    and coroutine frame.
 
-In gdb, one can use the following approach to devirtualize coroutine type,
+In gdb, one can use the following approach to devirtualize a coroutine type,
 assuming we have a ``std::coroutine_handle`` is at address 0x418eb0:
 
 ::
@@ -679,7 +679,7 @@ LLDB comes with devirtualization support out of the box, as part of the
 pretty-printer for ``std::coroutine_handle``. Internally, this pretty-printer
 uses the second approach. We look up the types in the destroy function and not
 the resume function because the resume function pointer will be set to a
-nullptr as soon as a coroutine reaches its final suspension point. If we used
+``nullptr`` as soon as a coroutine reaches its final suspension point. If we used
 the resume function, devirtualization would hence fail for all coroutines that
 have reached their final suspension point.
 
@@ -687,10 +687,10 @@ Interpreting the coroutine frame in optimized builds
 ----------------------------------------------------
 
 The ``__coro_frame`` variable usually refers to the coroutine frame of an
-*in-flight* coroutine. This means, the coroutine is currently executing.
+*in-flight* coroutine. This means the coroutine is currently executing.
 However, the compiler only guarantees the coroutine frame to be in a consistent
 state while the coroutine is suspended. As such, the variables inside the
-``__coro_frame`` variable might be outdated, in particular when optimizations
+``__coro_frame`` variable might be outdated, particularly when optimizations
 are enabled.
 
 Furthermore, when optimizations are enabled, the compiler will layout the
@@ -731,7 +731,7 @@ despite ``a`` being frequently incremented.
 
 While this might be surprising, this is a result of the optimizer recognizing
 that it can eliminate most of the load/store operations.
-The above code gets optimized to the equivalent of:
+The above code is optimized to the equivalent of:
 
 .. code-block:: c++
 
@@ -1180,5 +1180,5 @@ The authors of the Folly libraries wrote a blog post series on how they debug co
 * `Async stack traces in folly: Improving debugging in the developer lifecycle <https://developers.facebook.com/blog/post/2021/10/21/async-stack-traces-folly-improving-debugging-developer-lifecycle/>`_
 
 Besides some topics also covered here (stack traces from the debugger), Folly's blog post series also covers
-more additional topics, such as capturing async stack traces in performance profiles via eBPF filters
+additional topics, such as capturing async stack traces in performance profiles via eBPF filters
 and printing async stack traces on crashes.
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index b5bb198ca637a..3c6c97bb1fa10 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -635,11 +635,12 @@ C and C++.  For example:
     return v;
   }
 
+
 Boolean vectors are a Clang extension of the ext vector type.  Boolean vectors
 are intended, though not guaranteed, to map to vector mask registers.  The size
 parameter of a boolean vector type is the number of bits in the vector.  The
 boolean vector is dense and each bit in the boolean vector is one vector
-element.
+element. Query for this feature with ``__has_feature(ext_vector_type_boolean)``.
 
 The semantics of boolean vectors borrows from C bit-fields with the following
 differences:
@@ -657,6 +658,16 @@ The size and alignment are both the number of bits rounded up to the next power
 of two, but the alignment is at most the maximum vector alignment of the
 target.
 
+A boolean vector can be used in a ternary `?:` operator to select vector
+elements of a different type.
+
+.. code-block:: c++
+
+  typedef int int4 __attribute__((ext_vector_type(4)));
+  typedef bool bool4 __attribute__((ext_vector_type(4)));
+
+  int4 blend(bool4 cond, int4 a, int4 b) { return cond ? a : b; }
+
 
 Vector Literals
 ---------------
@@ -757,9 +768,12 @@ elementwise to the input.
 
 Unless specified otherwise operation(±0) = ±0 and operation(±infinity) = ±infinity
 
-The integer elementwise intrinsics, including ``__builtin_elementwise_popcount``,
+The elementwise intrinsics ``__builtin_elementwise_popcount``,
 ``__builtin_elementwise_bitreverse``, ``__builtin_elementwise_add_sat``,
-``__builtin_elementwise_sub_sat`` can be called in a ``constexpr`` context.
+``__builtin_elementwise_sub_sat``, ``__builtin_elementwise_max``,
+``__builtin_elementwise_min``, ``__builtin_elementwise_abs``,
+``__builtin_elementwise_ctlz``, ``__builtin_elementwise_cttz``, and
+``__builtin_elementwise_fma`` can be called in a ``constexpr`` context.
 
 No implicit promotion of integer types takes place. The mixing of integer types
 of different sizes and signs is forbidden in binary and ternary builtins.
@@ -859,6 +873,23 @@ of different sizes and signs is forbidden in binary and ternary builtins.
                                                 semantics, see `LangRef
                                                 <http://llvm.org/docs/LangRef.html#llvm-min-intrinsics-comparation>`_
                                                 for the comparison.
+T __builtin_elementwise_fshl(T x, T y, T z)     perform a funnel shift left. Concatenate x and y (x is the most        integer types
+                                                significant bits of the wide value), the combined value is shifted
+                                                left by z, and the most significant bits are extracted to produce
+                                                a result that is the same size as the original arguments.
+
+T __builtin_elementwise_fshr(T x, T y, T z)     perform a funnel shift right. Concatenate x and y (x is the most       integer types
+                                                significant bits of the wide value), the combined value is shifted
+                                                right by z, and the least significant bits are extracted to produce
+                                                a result that is the same size as the original arguments.
+ T __builtin_elementwise_ctlz(T x[, T y])       return the number of leading 0 bits in the first argument. If          integer types
+                                                the first argument is 0 and an optional second argument is provided,
+                                                the second argument is returned. It is undefined behaviour if the
+                                                first argument is 0 and no second argument is provided.
+ T __builtin_elementwise_cttz(T x[, T y])       return the number of trailing 0 bits in the first argument. If         integer types
+                                                the first argument is 0 and an optional second argument is provided,
+                                                the second argument is returned. It is undefined behaviour if the
+                                                first argument is 0 and no second argument is provided.
 ============================================== ====================================================================== =========================================
 
 
@@ -911,6 +942,24 @@ Let ``VT`` be a vector type and ``ET`` the element type of ``VT``.
                                          for the comparison.
 ======================================= ====================================================================== ==================================
 
+*Masked Builtins*
+
+Each builtin accesses memory according to a provided boolean mask. These are
+provided as ``__builtin_masked_load`` and ``__builtin_masked_store``. The first
+argument is always boolean mask vector.
+
+Example:
+
+.. code-block:: c++
+
+    using v8b = bool [[clang::ext_vector_type(8)]];
+    using v8i = int [[clang::ext_vector_type(8)]];
+
+    v8i load(v8b m, v8i *p) { return __builtin_masked_load(m, p); }
+
+    void store(v8b m, v8i v, v8i *p) { __builtin_masked_store(m, v, p); }
+
+
 Matrix Types
 ============
 
@@ -1704,11 +1753,41 @@ Variadic Friends                              __cpp_variadic_friend            C
 Trivial Relocatability                        __cpp_trivial_relocatability     C++26         C++03
 --------------------------------------------- -------------------------------- ------------- -------------
 Designated initializers (N494)                                                 C99           C89
+``_Complex`` (N693)                                                            C99           C89, C++
+``_Bool`` (N815)                                                               C99           C89
+Variable-length arrays (N683)                                                  C99           C89, C++
+Flexible array members                                                         C99           C89, C++
+static and type quals in arrays                                                C99           C89
+``long long`` (N601)                                                           C99           C89
+Hexadecimal floating constants (N308)                                          C99           C89
+Compound literals (N716)                                                       C99           C89, C++
+``//`` comments (N644)                                                         C99           C89
+Mixed declarations and code (N740)                                             C99           C89
+init-statement in for (N740)                                                   C99           C89
+Variadic macros (N707)                                                         C99           C89
+Empty macro arguments (N570)                                                   C99           C89
+Trailing comma in enum declaration                                             C99           C89
+Implicit ``return 0`` in ``main``                                              C99           C89
+``__func__`` (N611)                                                            C99           C89
+``_Generic`` (N1441)                                                           C11           C89, C++
+``_Static_assert`` (N1330)                                                     C11           C89, C++
+``_Atomic`` (N1485)                                                            C11           C89, C++
+``_Thread_local`` (N1364)                                                      C11           C89, C++
 Array & element qualification (N2607)                                          C23           C89
 Attributes (N2335)                                                             C23           C89
 ``#embed`` (N3017)                                                             C23           C89, C++
+Enum with fixed underlying type (N3030)                                        C23           C89
+``#warning`` (N2686)                                                           C23           C89
+``_BitInt`` (N3035)                                                            C23           C89, C++
+Binary literals (N2549)                                                        C23           C89
+Unnamed parameters in a function definition                                    C23           C89
+Free positioning of labels (N2508)                                             C23           C89
+``#elifdef`` (N2645)                                                           C23           C89
+``__has_include`` (N2799)                                                      C23           C89
 Octal literals prefixed with ``0o`` or ``0O``                                  C2y           C89, C++
 ``_Countof`` (N3369, N3469)                                                    C2y           C89
+``_Generic`` with a type operand (N3260)                                       C2y           C89, C++
+``++``/``--`` on ``_Complex`` value (N3259)                                    C2y           C89, C++
 ============================================= ================================ ============= =============
 
 Builtin type aliases
@@ -1751,6 +1830,37 @@ __make_integer_seq
 
 This alias returns ``IntSeq`` instantiated with ``IntSeqT = T``and ``Ints`` being the pack ``0, ..., N - 1``.
 
+__builtin_dedup_pack
+--------------------
+
+.. code-block:: c++
+
+  template <class... Ts>
+  using __builtin_dedup_pack = ...;
+
+This alias takes a template parameter pack ``Ts`` and produces a new unexpanded pack containing the unique types
+from ``Ts``, with the order of the first occurrence of each type preserved.
+It is useful in template metaprogramming to normalize type lists.
+
+The resulting pack can be expanded in contexts like template argument lists or base specifiers.
+
+**Example of Use**:
+
+.. code-block:: c++
+
+  template <typename...> struct TypeList;
+
+  // The resulting type is TypeList<int, double, char>
+  template <typename ...ExtraTypes>
+  using MyTypeList = TypeList<__builtin_dedup_pack<int, double, int, char, double, ExtraTypes...>...>;
+
+**Limitations**:
+
+* This builtin can only be used inside a template.
+* The resulting pack is currently only supported for expansion in template argument lists and base specifiers.
+* This builtin cannot be assigned to a template template parameter.
+
+
 Type Trait Primitives
 =====================
 
@@ -4330,7 +4440,7 @@ fall into one of the specified floating-point classes.
 
   if (__builtin_isfpclass(x, 448)) {
      // `x` is positive finite value
-	 ...
+         ...
   }
 
 **Description**:
diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 58cd10ad4d8fa..2b510747bea90 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -191,7 +191,7 @@ implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | device                       | teams construct on the host device                           | :good:`done`             | r371553                                                               |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| device                       | support non-contiguous array sections for target update      | :good:`done`             |                                                                       |
+| device                       | support non-contiguous array sections for target update      | :good:`done`             | https://github.com/llvm/llvm-project/pull/144635                      |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | device                       | pointer attachment                                           | :good:`done`             |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
@@ -256,7 +256,7 @@ implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | device                       | device-specific environment variables                        | :none:`unclaimed`        |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| device                       | omp_target_is_accessible routine                             | :none:`unclaimed`        |                                                                       |
+| device                       | omp_target_is_accessible routine                             | :part:`In Progress`      | https://github.com/llvm/llvm-project/pull/138294                      |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | device                       | omp_get_mapped_ptr routine                                   | :good:`done`             | D141545                                                               |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
@@ -338,7 +338,7 @@ implementation.
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | OMPT                         | new 'emi' callbacks for external monitoring interfaces       | :good:`done`             |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
-| OMPT                         | device tracing interface                                     | :none:`unclaimed`        |                                                                       |
+| OMPT                         | device tracing interface                                     | :none:`in progress`      | jplehr                                                                |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
 | task                         | 'strict' modifier for taskloop construct                     | :none:`unclaimed`        |                                                                       |
 +------------------------------+--------------------------------------------------------------+--------------------------+-----------------------------------------------------------------------+
@@ -376,7 +376,7 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop stripe transformation                                  | :good:`done`              | https://github.com/llvm/llvm-project/pull/119891                                                     |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| work distribute construct                                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| workdistribute construct                                    |                           | :none:`in progress`       | @skc7, @mjklemm                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | task_iteration                                              | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
@@ -449,7 +449,7 @@ implementation.
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Clarifications to Fortran map semantics                     | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| default clause at target construct                          | :part:`In Progress`         | :none:`unclaimed`         |                                                                        |
+| default clause at target construct                          | :part:`In Progress`       | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | ref count update use_device_{ptr, addr}                     | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
@@ -470,12 +470,18 @@ implementation.
 | need_device_addr modifier for adjust_args clause            | :part:`partial`           | :none:`unclaimed`         | Parsing/Sema: https://github.com/llvm/llvm-project/pull/143442           |
 |                                                             |                           |                           |               https://github.com/llvm/llvm-project/pull/149586           |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| Prescriptive num_threads                                    | :part:`In Progress`       | :none:`unclaimed`         |                                                                          |
+| Prescriptive num_threads                                    | :part:`In Progress`       | :none:`unclaimed`         | ro-i                                                                     |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| Message and severity clauses                                | :part:`In Progress`       | :none:`unclaimed`         |                                                                          |
+| Message and severity clauses                                | :part:`In Progress`       | :none:`unclaimed`         | ro-i                                                                     |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | Local clause on declare target                              | :part:`In Progress`       | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
+| groupprivate directive                                      | :part:`In Progress`       | :part:`In Progress`       | Flang: kparzysz, mjklemm                                                 |
++-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
+| variable-category on default clause                         | :part:`In Progress`       | :none:`unclaimed`         |                                                                          |
++-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
+| Changes to omp_target_is_accessible                         | :part:`In Progress`       | :part:`In Progress`       |                                                                          |
++-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 
 OpenMP Extensions
 =================
diff --git a/clang/docs/PointerAuthentication.rst b/clang/docs/PointerAuthentication.rst
index 913291c954447..96eb498bc48b6 100644
--- a/clang/docs/PointerAuthentication.rst
+++ b/clang/docs/PointerAuthentication.rst
@@ -47,16 +47,16 @@ This document serves four purposes:
 - It documents several language extensions that are useful on targets using
   pointer authentication.
 
-- It will eventually present a theory of operation for the security mitigation,
-  describing the basic requirements for correctness, various weaknesses in the
-  mechanism, and ways in which programmers can strengthen its protections
-  (including recommendations for language implementors).
+- It presents a theory of operation for the security mitigation, describing the
+  basic requirements for correctness, various weaknesses in the mechanism, and
+  ways in which programmers can strengthen its protections (including
+  recommendations for language implementors).
 
-- It will eventually document the language ABIs currently used for C, C++,
-  Objective-C, and Swift on arm64e, although these are not yet stable on any
-  target.
+- It documents the stable ABI of the C, C++, and Objective-C languages on arm64e
+  platforms.
 
-Basic Concepts
+
+Basic concepts
 --------------
 
 The simple address of an object or function is a **raw pointer**.  A raw
@@ -125,7 +125,7 @@ independently for I and D keys.)
   interfaces or as primitives in a compiler IR because they expose raw
   pointers.  Raw pointers require special attention in the language
   implementation to avoid the accidental creation of exploitable code
-  sequences.
+  sequences; see the section on `Attackable code sequences`_.
 
 The following details are all implementation-defined:
 
@@ -167,10 +167,15 @@ a cryptographic signature, other implementations may be possible.  See
     signing key, and stores it in the high bits as the signature. ``auth``
     removes the signature, computes the same hash, and compares the result with
     the stored signature.  ``strip`` removes the signature without
-    authenticating it.  While ``aut*`` instructions do not themselves trap on
-    failure in Armv8.3 PAuth, they do with the later optional FPAC extension.
-    An implementation can also choose to emulate this trapping behavior by
-    emitting additional instructions around ``aut*``.
+    authenticating it.  The ``aut`` instructions in the baseline Armv8.3 PAuth
+    feature do not guarantee to trap on authentication failure; instead, they
+    simply corrupt the pointer so that later uses will likely trap. Unless the
+    "later use" follows immediately and cannot be recovered from (e.g. with a
+    signal handler), this does not provide adequate protection against
+    `authentication oracles`_, so implementations must emit additional
+    instructions to force an immediate trap. This is unnecessary if the
+    processor provides the optional ``FPAC`` extension, which guarantees an
+    immediate trap.
 
   - ``sign_generic`` corresponds to the ``pacga`` instruction, which takes two
     64-bit values and produces a 64-bit cryptographic hash. Implementations of
@@ -234,7 +239,7 @@ implementation-defined.
 
 .. _Signing schemas:
 
-Signing Schemas
+Signing schemas
 ~~~~~~~~~~~~~~~
 
 Correct use of pointer authentication requires the signing code and the
@@ -255,33 +260,172 @@ signing schema breaks down even more simply:
 It is important that the signing schema be independently derived at all signing
 and authentication sites.  Preferably, the schema should be hard-coded
 everywhere it is needed, but at the very least, it must not be derived by
-inspecting information stored along with the pointer.
+inspecting information stored along with the pointer.  See the section on
+`Attacks on pointer authentication`_ for more information.
 
-Language Features
------------------
 
-There is currently one main pointer authentication language feature:
+Language features
+-----------------
 
-- The language provides the ``<ptrauth.h>`` intrinsic interface for manually
-  signing and authenticating pointers in code.  These can be used in
+There are three levels of the pointer authentication language feature:
+
+- The language implementation automatically signs and authenticates function
+  pointers (and certain data pointers) across a variety of standard situations,
+  including return addresses, function pointers, and C++ virtual functions. The
+  intent is for all pointers to code in program memory to be signed in some way
+  and for all branches to code in program text to authenticate those
+  signatures. In addition to the code pointers themselves, we also use pointer
+  authentication to protect data values that directly or indirectly influence
+  control flow or program integrity, or can provide attackers with some other
+  powerful program compromise.
+
+- The language also provides extensions to override the default rules used by
+  the language implementation.  For example, the ``__ptrauth`` type qualifier
+  can be used to change how pointers or pointer sized integers are signed when
+  they are stored in a particular variable or field; this provides much stronger
+  protection than is guaranteed by the default rules for C function and data
+  pointers.
+
+- Finally, the language provides the ``<ptrauth.h>`` intrinsic interface for
+  manually signing and authenticating pointers in code.  These can be used in
   circumstances where very specific behavior is required.
 
+Language implementation
+~~~~~~~~~~~~~~~~~~~~~~~
+
+For the most part, pointer authentication is an unobserved detail of the
+implementation of the programming language.  Any element of the language
+implementation that would perform an indirect branch to a pointer is implicitly
+altered so that the pointer is signed when first constructed and authenticated
+when the branch is performed.  This includes:
+
+- indirect-call features in the programming language, such as C function
+  pointers, C++ virtual functions, C++ member function pointers, the "blocks"
+  C extension, and so on;
+
+- returning from a function, no matter how it is called; and
+
+- indirect calls introduced by the implementation, such as branches through the
+  global offset table (GOT) used to implement direct calls to functions defined
+  outside of the current shared object.
+
+For more information about this, see the `Language ABI`_ section.
+
+However, some aspects of the implementation are observable by the programmer or
+otherwise require special notice.
+
+C data pointers
+^^^^^^^^^^^^^^^
+
+The current implementation in Clang does not sign pointers to ordinary data by
+default. For a partial explanation of the reasoning behind this, see the
+`Theory of Operation`_ section.
+
+A specific data pointer which is more security-sensitive than most can be
+signed using the `__ptrauth qualifier`_ or using the ``<ptrauth.h>``
+intrinsics.
+
+C function pointers
+^^^^^^^^^^^^^^^^^^^
+
+The C standard imposes restrictions on the representation and semantics of
+function pointer types which make it difficult to achieve satisfactory
+signature diversity in the default language rules.  See `Attacks on pointer
+authentication`_ for more information about signature diversity.  Programmers
+should strongly consider using the ``__ptrauth`` qualifier to improve the
+protections for important function pointers, such as the components of of
+a hand-rolled "v-table"; see the section on the `__ptrauth qualifier`_ for
+details.
+
+The value of a pointer to a C function includes a signature, even when the
+value is cast to a non-function-pointer type like ``void*`` or ``intptr_t``. On
+implementations that use high bits to store the signature, this means that
+relational comparisons and hashes will vary according to the exact signature
+value, which is likely to change between executions of a program.  In some
+implementations, it may also vary based on the exact function pointer type.
+
+Null pointers
+^^^^^^^^^^^^^
+
+In principle, an implementation could derive the signed null pointer value
+simply by applying the standard signing algorithm to the raw null pointer
+value. However, for likely signing algorithms, this would mean that the signed
+null pointer value would no longer be statically known, which would have many
+negative consequences.  For one, it would become substantially more expensive
+to emit null pointer values or to perform null-pointer checks.  For another,
+the pervasive (even if technically unportable) assumption that null pointers
+are bitwise zero would be invalidated, making it substantially more difficult
+to adopt pointer authentication, as well as weakening common optimizations for
+zero-initialized memory such as the use of ``.bzz`` sections.  Therefore it is
+beneficial to treat null pointers specially by giving them their usual
+representation.  On AArch64, this requires additional code when working with
+possibly-null pointers, such as when copying a pointer field that has been
+signed with address diversity.
+
+While this representation of nulls is the safest option for the general case,
+there are some situations in which a null pointer may have important semantic
+or security impact. For that purpose Clang has the concept of a pointer
+authentication schema that signs and authenticates null values.
+
+Return addresses
+^^^^^^^^^^^^^^^^
+
+The current implementation in Clang implicitly signs the return addresses in
+function calls.  While the value of the return address is technically an
+implementation detail of a function, there are some important libraries and
+development tools which rely on manually walking the chain of stack frames.
+These tools must be updated to correctly account for pointer authentication,
+either by stripping signatures (if security is not important for the tool, e.g.
+if it is capturing a stack trace during a crash) or properly authenticating
+them.  More information about how these values are signed is available in the
+`Language ABI`_ section.
+
+C++ virtual functions
+^^^^^^^^^^^^^^^^^^^^^
+
+The current implementation in Clang signs virtual function pointers with
+a discriminator derived from the full signature of the overridden method,
+including the method name and parameter types.  It is possible to write C++
+code that relies on v-table layout remaining constant despite changes to
+a method signature; for example, a parameter might be a ``typedef`` that
+resolves to a different type based on a build setting.  Such code violates
+C++'s One Definition Rule (ODR), but that violation is not normally detected;
+however, pointer authentication will detect it.
 
-Language Extensions
+Language extensions
 ~~~~~~~~~~~~~~~~~~~
 
-Feature Testing
+Feature testing
 ^^^^^^^^^^^^^^^
 
 Whether the current target uses pointer authentication can be tested for with
 a number of different tests.
 
-- ``__has_feature(ptrauth_intrinsics)`` is true if ``<ptrauth.h>`` provides its
-  normal interface.  This may be true even on targets where pointer
-  authentication is not enabled by default.
+- ``__PTRAUTH__`` macro is defined if ``<ptrauth.h>`` provides its normal
+  interface. This implies support for the pointer authentication intrinsics
+  and the ``__ptrauth`` qualifier.
 
-__ptrauth Qualifier
-^^^^^^^^^^^^^^^^^^^
+- ``__has_feature(ptrauth_returns)`` is true if the target uses pointer
+  authentication to protect return addresses.
+
+- ``__has_feature(ptrauth_calls)`` is true if the target uses pointer
+  authentication to protect indirect branches.  On arm64e this implies
+  ``__has_feature(ptrauth_returns)``, ``__has_feature(ptrauth_intrinsics)``,
+  and the ``__PTRAUTH__`` macro.
+
+- For backwards compatibility purposes ``__has_feature(ptrauth_intrinsics)``
+  and ``__has_feature(ptrauth_qualifier)`` are available on arm64e targets.
+  These features are synonymous with each other, and are equivalent to testing
+  for the ``__PTRAUTH__`` macro definition. Use of these features should be
+  restricted to cases where backwards compatibility is required, and should be
+  paired with ``defined(__PTRAUTH__)``.
+
+
+Clang provides several other tests only for historical purposes; for current
+purposes they are all equivalent to ``ptrauth_calls``.
+
+``__ptrauth`` qualifier
+^^^^^^^^^^^^^^^^^^^^^^^
 
 ``__ptrauth(key, address, discriminator)`` is an extended type
 qualifier which causes so-qualified objects to hold pointers or pointer sized
@@ -293,6 +437,11 @@ type, either to a function or to an object, or a pointer sized integer.  It
 currently cannot be an Objective-C pointer type, a C++ reference type, or a
 block pointer type; these restrictions may be lifted in the future.
 
+The current implementation in Clang is known to not provide adequate safety
+guarantees against the creation of `signing oracles`_ when assigning data
+pointers to ``__ptrauth``-qualified gl-values.  See the section on `safe
+derivation`_ for more information.
+
 The qualifier's operands are as follows:
 
 - ``key`` - an expression evaluating to a key value from ``<ptrauth.h>``; must
@@ -327,6 +476,57 @@ a discriminator determined as follows:
   is ``ptrauth_blend_discriminator(&x, discriminator)``; see
   `ptrauth_blend_discriminator`_.
 
+Non-triviality from address diversity
++++++++++++++++++++++++++++++++++++++
+
+Address diversity must impose additional restrictions in order to allow the
+implementation to correctly copy values.  In C++, a type qualified with address
+diversity is treated like a class type with non-trivial copy/move constructors
+and assignment operators, with the usual effect on containing classes and
+unions.  C does not have a standard concept of non-triviality, and so we must
+describe the basic rules here, with the intention of imitating the emergent
+rules of C++:
+
+- A type may be **non-trivial to copy**.
+
+- A type may also be **illegal to copy**. Types that are illegal to copy are
+  always non-trivial to copy.
+
+- A type may also be **address-sensitive**. This includes types that use self
+  referencing pointers, data protected by address diversified pointer
+  authentication, or other similar concepts.
+
+- A type qualified with a ``ptrauth`` qualifier or implicit authentication
+  schema that requires address diversity is non-trivial to copy and
+  address-sensitive.
+
+- An array type is illegal to copy, non-trivial to copy, or address-sensitive
+  if its element type is illegal to copy, non-trivial to copy, or
+  address-sensitive, respectively.
+
+- A struct type is illegal to copy, non-trivial to copy, or address-sensitive
+  if it has a field whose type is illegal to copy, non-trivial to copy, or
+  address-sensitive, respectively.
+
+- A union type is both illegal and non-trivial to copy if it has a field whose
+  type is non-trivial or illegal to copy.
+
+- A union type is address-sensitive if it has a field whose type is
+  address-sensitive.
+
+- A program is ill-formed if it uses a type that is illegal to copy as
+  a function parameter, argument, or return type.
+
+- A program is ill-formed if an expression requires a type to be copied that is
+  illegal to copy.
+
+- Otherwise, copying a type that is non-trivial to copy correctly copies its
+  subobjects.
+
+- Types that are address-sensitive must always be passed and returned
+  indirectly. Thus, changing the address-sensitivity of a type may be
+  ABI-breaking even if its size and alignment do not change.
+
 ``<ptrauth.h>``
 ~~~~~~~~~~~~~~~
 
@@ -433,7 +633,7 @@ Produce a signed pointer for the given raw pointer without applying any
 authentication or extra treatment.  This operation is not required to have the
 same behavior on a null pointer that the language implementation would.
 
-This is a treacherous operation that can easily result in signing oracles.
+This is a treacherous operation that can easily result in `signing oracles`_.
 Programs should use it seldom and carefully.
 
 ``ptrauth_auth_and_resign``
@@ -454,7 +654,29 @@ a null pointer that the language implementation would.
 The code sequence produced for this operation must not be directly attackable.
 However, if the discriminator values are not constant integers, their
 computations may still be attackable.  In the future, Clang should be enhanced
-to guaranteed non-attackability if these expressions are safely-derived.
+to guaranteed non-attackability if these expressions are
+:ref:`safely-derived<Safe derivation>`.
+
+``ptrauth_auth_function``
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. code-block:: c
+
+  ptrauth_auth_function(pointer, key, discriminator)
+
+Authenticate that ``pointer`` is signed with ``key`` and ``discriminator`` and
+re-sign it to the standard schema for a function pointer of its type.
+
+``pointer`` must have function pointer type.  The result will have the same
+type as ``pointer``.  This operation is not required to have the same behavior
+on a null pointer that the language implementation would.
+
+This operation makes the same attackability guarantees as
+``ptrauth_auth_and_resign``.
+
+If this operation appears syntactically as the function operand of a call,
+Clang guarantees that the call will directly authenticate the function value
+using the given schema rather than re-signing to the standard schema.
 
 ``ptrauth_auth_data``
 ^^^^^^^^^^^^^^^^^^^^^
@@ -500,12 +722,921 @@ type.  Implementations are not required to make all bits of the result equally
 significant; in particular, some implementations are known to not leave
 meaningful data in the low bits.
 
+Standard ``__ptrauth`` qualifiers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``<ptrauth.h>`` additionally provides several macros which expand to
+``__ptrauth`` qualifiers for common ABI situations.
+
+For convenience, these macros expand to nothing when pointer authentication is
+disabled.
+
+These macros can be found in the header; some details of these macros may be
+unstable or implementation-specific.
+
+
+Theory of operation
+-------------------
+
+The threat model of pointer authentication is as follows:
+
+- The attacker has the ability to read and write to a certain range of
+  addresses, possibly the entire address space.  However, they are constrained
+  by the normal rules of the process: for example, they cannot write to memory
+  that is mapped read-only, and if they access unmapped memory it will trigger
+  a trap.
+
+- The attacker has no ability to add arbitrary executable code to the program.
+  For example, the program does not include malicious code to begin with, and
+  the attacker cannot alter existing instructions, load a malicious shared
+  library, or remap writable pages as executable.  If the attacker wants to get
+  the process to perform a specific sequence of actions, they must somehow
+  subvert the normal control flow of the process.
+
+In both of the above paragraphs, it is merely assumed that the attacker's
+*current* capabilities are restricted; that is, their current exploit does not
+directly give them the power to do these things.  The attacker's immediate goal
+may well be to leverage their exploit to gain these capabilities, e.g. to load
+a malicious dynamic library into the process, even though the process does not
+directly contain code to do so.
+
+Note that any bug that fits the above threat model can be immediately exploited
+as a denial-of-service attack by simply performing an illegal access and
+crashing the program.  Pointer authentication cannot protect against this.
+While denial-of-service attacks are unfortunate, they are also unquestionably
+the best possible result of a bug this severe. Therefore, pointer authentication
+enthusiastically embraces the idea of halting the program on a pointer
+authentication failure rather than continuing in a possibly-compromised state.
+
+Pointer authentication is a form of control-flow integrity (CFI) enforcement.
+The basic security hypothesis behind CFI enforcement is that many bugs can only
+be usefully exploited (other than as a denial-of-service) by leveraging them to
+subvert the control flow of the program.  If this is true, then by inhibiting or
+limiting that subversion, it may be possible to largely mitigate the security
+consequences of those bugs by rendering them impractical (or, ideally,
+impossible) to exploit.
+
+Every indirect branch in a program has a purpose.  Using human intelligence, a
+programmer can describe where a particular branch *should* go according to this
+purpose: a ``return`` in ``printf`` should return to the call site, a particular
+call in ``qsort`` should call the comparator that was passed in as an argument,
+and so on.  But for CFI to enforce that every branch in a program goes where it
+*should* in this sense would require CFI to perfectly enforce every semantic
+rule of the program's abstract machine; that is, it would require making the
+programming environment perfectly sound.  That is out of scope.  Instead, the
+goal of CFI is merely to catch attempts to make a branch go somewhere that its
+obviously *shouldn't* for its purpose: for example, to stop a call from
+branching into the middle of a function rather than its beginning.  As the
+information available to CFI gets better about the purpose of the branch, CFI
+can enforce tighter and tighter restrictions on where the branch is permitted to
+go.  Still, ultimately CFI cannot make the program sound.  This may help explain
+why pointer authentication makes some of the choices it does: for example, to
+sign and authenticate mostly code pointers rather than every pointer in the
+program.  Preventing attackers from redirecting branches is both particularly
+important and particularly approachable as a goal.  Detecting corruption more
+broadly is infeasible with these techniques, and the attempt would have far
+higher cost.
+
+Attacks on pointer authentication
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Pointer authentication works as follows.  Every indirect branch in a program has
+a purpose.  For every purpose, the implementation chooses a
+:ref:`signing schema<Signing schemas>`.  At some place where a pointer is known
+to be correct for its purpose, it is signed according to the purpose's schema.
+At every place where the pointer is needed for its purpose, it is authenticated
+according to the purpose's schema.  If that authentication fails, the program is
+halted.
+
+There are a variety of ways to attack this.
+
+Attacks of interest to programmers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These attacks arise from weaknesses in the default protections offered by
+pointer authentication.  They can be addressed by using attributes or intrinsics
+to opt in to stronger protection.
+
+Substitution attacks
+++++++++++++++++++++
+
+An attacker can simply overwrite a pointer intended for one purpose with a
+pointer intended for another purpose if both purposes use the same signing
+schema and that schema does not use address diversity.
+
+The most common source of this weakness is when code relies on using the default
+language rules for C function pointers.  The current implementation uses the
+exact same signing schema for all C function pointers, even for functions of
+substantially different type.  While efforts are ongoing to improve constant
+diversity for C function pointers of different type, there are necessary limits
+to this.  The C standard requires function pointers to be copyable with
+``memcpy``, which means that function pointers can never use address diversity.
+Furthermore, even if a function pointer can only be replaced with another
+function of the exact same type, that can still be useful to an attacker, as in
+the following example of a hand-rolled "v-table":
+
+.. code-block:: c
+
+  struct ObjectOperations {
+    void (*retain)(Object *);
+    void (*release)(Object *);
+    void (*deallocate)(Object *);
+    void (*logStatus)(Object *);
+  };
+
+The weakness in this design is that by lacking any context specific
+discriminator, this means an attacker can substitute any of these fields with
+any other function pointer signed with the default schema. Similarly the lack of
+address diversity allows an attacker to replace the functions in one type's
+"v-table" with those of another. This can be mitigated by overriding the default
+authentication schema with a more specific signing schema for each purpose.  For
+instance, in this example, the ``__ptrauth`` qualifier can be used with a
+different constant discriminator for each field.  Since there's no particular
+reason it's important for this v-table to be copyable with ``memcpy``, the
+functions can also be signed with address diversity:
+
+.. code-block:: c
+
+  #if defined(__PTRAUTH__)
+  #define objectOperation(discriminator) \
+    __ptrauth(ptrauth_key_function_pointer, 1, discriminator)
+  #else
+  #define objectOperation(discriminator)
+  #endif
+
+  struct ObjectOperations {
+    void (*objectOperation(0xf017) retain)(Object *);
+    void (*objectOperation(0x2639) release)(Object *);
+    void (*objectOperation(0x8bb0) deallocate)(Object *);
+    void (*objectOperation(0xc5d4) logStatus)(Object *);
+  };
+
+This weakness can also sometimes be mitigated by simply keeping the signed
+pointer in constant memory, but this is less effective than using better signing
+diversity.
+
+.. _Access path attacks:
+
+Access path attacks
++++++++++++++++++++
+
+If a signed pointer is often accessed indirectly (that is, by first loading the
+address of the object where the signed pointer is stored), an attacker can
+affect uses of it by overwriting the intermediate pointer in the access path.
+
+The most common scenario exhibiting this weakness is an object with a pointer to
+a "v-table" (a structure holding many function pointers). An attacker does not
+need to replace a signed function pointer in the v-table if they can instead
+simply replace the v-table pointer in the object with their own pointer ---
+perhaps to memory where they've constructed their own v-table, or to existing
+memory that coincidentally happens to contain a signed pointer at the right
+offset that's been signed with the right signing schema.
+
+This attack arises because data pointers are not signed by default. It works
+even if the signed pointer uses address diversity: address diversity merely
+means that each pointer is signed with its own storage address,
+which (by design) is invariant to changes in the accessing pointer.
+
+Using sufficiently diverse signing schemas within the v-table can provide
+reasonably strong mitigation against this weakness.  Always use address and type
+diversity in v-tables to prevent attackers from assembling their own v-table.
+Avoid re-using constant discriminators to prevent attackers from replacing a
+v-table pointer with a pointer to totally unrelated memory that just happens to
+contain an similarly-signed pointer, or reused memory containing a different
+type.
+
+Further mitigation can be attained by signing pointers to v-tables. Any
+signature at all should prevent attackers from forging v-table pointers; they
+will need to somehow harvest an existing signed pointer from elsewhere in
+memory.  Using a meaningful constant discriminator will force this to be
+harvested from an object with similar structure (e.g. a different implementation
+of the same interface).  Using address diversity will prevent such harvesting
+entirely.  However, care must be taken when sourcing the v-table pointer
+originally; do not blindly sign a pointer that is not
+:ref:`safely derived<Safe derivation>`.
+
+.. _Signing oracles:
+
+Signing oracles
++++++++++++++++
+
+A signing oracle is a bit of code which can be exploited by an attacker to sign
+an arbitrary pointer in a way that can later be recovered.  Such oracles can be
+used by attackers to forge signatures matching the oracle's signing schema,
+which is likely to cause a total compromise of pointer authentication's
+effectiveness.
+
+This attack only affects ordinary programmers if they are using certain
+treacherous patterns of code.  Currently this includes:
+
+- all uses of the ``__ptrauth_sign_unauthenticated`` intrinsic and
+- assigning values to ``__ptrauth``-qualified l-values.
+
+Care must be taken in these situations to ensure that the pointer being signed
+has been :ref:`safely derived<Safe derivation>` or is otherwise not possible to
+attack.  (In some cases, this may be challenging without compiler support.)
+
+A diagnostic will be added in the future for implicitly dangerous patterns of
+code, such as assigning a non-safely-derived values to a
+``__ptrauth``-qualified l-value.
+
+.. _Authentication oracles:
+
+Authentication oracles
+++++++++++++++++++++++
+
+An authentication oracle is a bit of code which can be exploited by an attacker
+to leak whether a signed pointer is validly signed without halting the program
+if it isn't.  Such oracles can be used to forge signatures matching the oracle's
+signing schema if the attacker can repeatedly invoke the oracle for different
+candidate signed pointers. This is likely to cause a total compromise of pointer
+authentication's effectiveness.
+
+There should be no way for an ordinary programmer to create an authentication
+oracle using the current set of operations. However, implementation flaws in the
+past have occasionally given rise to authentication oracles due to a failure to
+immediately trap on authentication failure.
+
+The likelihood of creating an authentication oracle is why there is currently no
+intrinsic which queries whether a signed pointer is validly signed.
+
+
+Attacks of interest to implementors
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These attacks are not inherent to the model; they arise from mistakes in either
+implementing or using the `sign` and `auth` operations. Avoiding these mistakes
+requires careful work throughout the system.
+
+Failure to trap on authentication failure
++++++++++++++++++++++++++++++++++++++++++
+
+Any failure to halt the program on an authentication failure is likely to be
+exploitable by attackers to create an
+:ref:`authentication oracle<Authentication oracles>`.
+
+There are several different ways to introduce this problem:
+
+- The implementation might try to halt the program in some way that can be
+  intercepted.
+
+  For example, the Armv8.3 ``aut`` instructions do not directly trap on
+  authentication failure on processors that lack the ``FPAC`` extension.
+  Instead, they corrupt their results to be invalid pointers, with the idea that
+  subsequent uses of those pointers will trigger traps as bad memory accesses.
+  However, most kernels do not immediately halt programs that trap due to bad
+  memory accesses; instead, they notify the process to give it an opportunity to
+  recover. If this happens with an ``auth`` failure, the attacker may be able to
+  exploit the recovery path in a way that creates an oracle. Kernels must
+  provide a way for a process to trap unrecoverably, and this should cover all
+  ``FPAC`` traps. Compilers must ensure that ``auth`` failures trigger an
+  unrecoverable trap, ideally by taking advantage of ``FPAC``, but if necessary
+  by emitting extra instructions.
+
+- A compiler might use an intermediate representation (IR) for ``sign`` and
+  ``auth`` operations that cannot make adequate correctness guarantees.
+
+  For example, suppose that an IR uses ARMv8.3-like semantics for ``auth``: the
+  operation merely corrupts its result on failure instead of promising to trap.
+  A frontend might emit patterns of IR that always follow an ``auth`` with a
+  memory access, thinking that this ensures correctness. But if the IR can be
+  transformed to insert code between the ``auth`` and the access, or if the
+  ``auth`` can be speculated, then this potentially creates an oracle.  It is
+  better for ``auth`` to semantically guarantee to trap, potentially requiring
+  an explicit check in the generated code. An ARMv8.3-like target can avoid this
+  explicit check in the common case by recognizing the pattern of an ``auth``
+  followed immediately by an access.
+
+Attackable code sequences
++++++++++++++++++++++++++
+
+If code that is part of a pointer authentication operation is interleaved with
+code that may itself be vulnerable to attacks, an attacker may be able to use
+this to create a :ref:`signing<Signing oracles>` or
+:ref:`authentication<Authentication oracles>` oracle.
+
+For example, suppose that the compiler is generating a call to a function and
+passing two arguments: a signed constant pointer and a value derived from a
+call.  In ARMv8.3, this code might look like so:
+
+.. code-block:: asm
+
+  adr x19, _callback.        ; compute &_callback
+  paciza x19                 ; sign it with a constant discriminator of 0
+  blr _argGenerator          ; call _argGenerator() (returns in x0)
+  mov x1, x0                 ; move call result to second arg register
+  mov x0, x19                ; move signed &_callback to first arg register
+  blr _function              ; call _function
+
+This code is correct, as would be a sequencing that does *both* the ``adr`` and
+the ``paciza`` after the call to ``_argGenerator``.  But a sequence that
+computes the address of ``_callback`` but leaves it as a raw pointer in a
+register during the call to ``_argGenerator`` would be vulnerable:
+
+.. code-block:: asm
+
+  adr x19, _callback.        ; compute &_callback
+  blr _argGenerator          ; call _argGenerator() (returns in x0)
+  mov x1, x0                 ; move call result to second arg register
+  paciza x19                 ; sign &_callback
+  mov x0, x19                ; move signed &_callback to first arg register
+  blr _function              ; call _function
+
+If ``_argGenerator`` spills ``x19`` (a callee-save register), and if the
+attacker can perform a write during this call, then the attacker can overwrite
+the spill slot with an arbitrary pointer that will eventually be unconditionally
+signed after the function returns.  This would be a signing oracle.
+
+The implementation can avoid this by obeying two basic rules:
+
+- The compiler's intermediate representations (IR) should not provide operations
+  that expose intermediate raw pointers.  This may require providing extra
+  operations that perform useful combinations of operations.
+
+  For example, there should be an "atomic" auth-and-resign operation that should
+  be used instead of emitting an ``auth`` operation whose result is fed into a
+  ``sign``.
+
+  Similarly, if a pointer should be authenticated as part of doing a memory
+  access or a call, then the access or call should be decorated with enough
+  information to perform the authentication; there should not be a separate
+  ``auth`` whose result is used as the pointer operand for the access or call.
+  (In LLVM IR, we do this for calls, but not yet for loads or stores.)
+
+  "Operations" includes things like materializing a signed value to a known
+  function or global variable.  The compiler must be able to recognize and emit
+  this as a unified operation, rather than potentially splitting it up as in
+  the example above.
+
+- The compiler backend should not be too aggressive about scheduling
+  instructions that are part of a pointer authentication operation. This may
+  require custom code-generation of these operations in some cases.
+
+Register clobbering
++++++++++++++++++++
+
+As a refinement of the section on `Attackable code sequences`_, if the attacker
+has the ability to modify arbitrary *register* state at arbitrary points in the
+program, then special care must be taken.
+
+For example, ARMv8.3 might materialize a signed function pointer like so:
+
+.. code-block:: asm
+
+  adr x0, _callback.        ; compute &_callback
+  paciza x0                 ; sign it with a constant discriminator of 0
+
+If an attacker has the ability to overwrite ``x0`` between these two
+instructions, this code sequence is vulnerable to becoming a signing oracle.
+
+For the most part, this sort of attack is not possible: it is a basic element of
+the design of modern computation that register state is private and inviolable.
+However, in systems that support asynchronous interrupts, this property requires
+the cooperation of the interrupt-handling code. If that code saves register
+state to memory, and that memory can be overwritten by an attacker, then
+essentially the attack can overwrite arbitrary register state at an arbitrary
+point.  This could be a concern if the threat model includes attacks on the
+kernel or if the program uses user-space preemptive multitasking.
+
+(Readers might object that an attacker cannot rely on asynchronous interrupts
+triggering at an exact instruction boundary.  In fact, researchers have had some
+success in doing exactly that.  Even ignoring that, though, we should aim to
+protect against lucky attackers just as much as good ones.)
+
+To protect against this, saved register state must be at least partially signed
+(using something like `ptrauth_sign_generic_data`_).  This is required for
+correctness anyway because saved thread states include security-critical
+registers such as SP, FP, PC, and LR (where applicable).  Ideally, this
+signature would cover all the registers, but since saving and restoring
+registers can be very performance-sensitive, that may not be acceptable. It is
+sufficient to set aside a small number of scratch registers that will be
+guaranteed to be preserved correctly; the compiler can then be careful to only
+store critical values like intermediate raw pointers in those registers.
+
+``setjmp`` and ``longjmp`` should sign and authenticate the core registers (SP,
+FP, PC, and LR), but they do not need to worry about intermediate values because
+``setjmp`` can only be called synchronously, and the compiler should never
+schedule pointer-authentication operations interleaved with arbitrary calls.
+
+.. _Relative addresses:
+
+Attacks on relative addressing
+++++++++++++++++++++++++++++++
+
+Relative addressing is a technique used to compress and reduce the load-time
+cost of infrequently-used global data.  The pointer authentication system is
+unlikely to support signing or authenticating a relative address, and in most
+cases it would defeat the point to do so: it would take additional storage
+space, and applying the signature would take extra work at load time.
+
+Relative addressing is not precluded by the use of pointer authentication, but
+it does take extra considerations to make it secure:
+
+- Relative addresses must only be stored in read-only memory.  A writable
+  relative address can be overwritten to point nearly anywhere, making it
+  inherently insecure; this danger can only be compensated for with techniques
+  for protecting arbitrary data like `ptrauth_sign_generic_data`_.
+
+- Relative addresses must only be accessed through signed pointers with adequate
+  diversity.  If an attacker can perform an `access path attack` to replace the
+  pointer through which the relative address is accessed, they can easily cause
+  the relative address to point wherever they want.
+
+Signature forging
++++++++++++++++++
+
+If an attacker can exactly reproduce the behavior of the signing algorithm, and
+they know all the correct inputs to it, then they can perfectly forge a
+signature on an arbitrary pointer.
+
+There are three components to avoiding this mistake:
+
+- The abstract signing algorithm should be good: it should not have glaring
+  flaws which would allow attackers to predict its result with better than
+  random accuracy without knowing all the inputs (like the key values).
+
+- The key values should be kept secret.  If at all possible, they should never
+  be stored in accessible memory, or perhaps only stored encrypted.
+
+- Contexts that are meant to be independently protected should use different
+  key values.  For example, the kernel should not use the same keys as user
+  processes.  Different user processes should also use different keys from each
+  other as much as possible, although this may pose its own technical
+  challenges.
+
+Remapping
++++++++++
+
+If an attacker can change the memory protections on certain pages of the
+program's memory, that can substantially weaken the protections afforded by
+pointer authentication.
+
+- If an attacker can inject their own executable code, they can also certainly
+  inject code that can be used as a :ref:`signing oracle<Signing Oracles>`.
+  The same is true if they can write to the instruction stream.
+
+- If an attacker can remap read-only program data sections to be writable, then
+  any use of :ref:`relative addresses` in global data becomes insecure.
+
+- On platforms that use them, if an attacker can remap the memory containing
+  the `global offset tables`_ as writable, then any unsigned pointers in those
+  tables are insecure.
+
+Remapping memory in this way often requires the attacker to have already
+substantively subverted the control flow of the process.  Nonetheless, if the
+operating system has a mechanism for mapping pages in a way that cannot be
+remapped, this should be used wherever possible.
+
+.. _Safe Derivation:
+
+Safe derivation
+~~~~~~~~~~~~~~~
+
+Whether a data pointer is stored, even briefly, as a raw pointer can affect the
+security-correctness of a program.  (Function pointers are never implicitly
+stored as raw pointers; raw pointers to functions can only be produced with the
+``<ptrauth.h>`` intrinsics.)  Repeated re-signing can also impact performance.
+Clang makes a modest set of guarantees in this area:
+
+- An expression of pointer type is said to be **safely derived** if:
+
+  - it takes the address of a global variable or function, or
+
+  - it is a load from a gl-value of ``__ptrauth``-qualified type, or
+
+  - it is a load from read-only memory that has been initialized from a safely
+    derived source, such as the `data const` section of a binary or library.
+
+- If a value that is safely derived is assigned to a ``__ptrauth``-qualified
+  object, including by initialization, then the value will be directly signed as
+  appropriate for the target qualifier and will not be stored as a raw pointer.
+
+- If the function expression of a call is a gl-value of ``__ptrauth``-qualified
+  type, then the call will be authenticated directly according to the source
+  qualifier and will not be resigned to the default rule for a function pointer
+  of its type.
+
+These guarantees are known to be inadequate for data pointer security. In
+particular, Clang should be enhanced to make the following guarantees:
+
+- A pointer should additionally be considered safely derived if it is:
+
+  - the address of a gl-value that is safely derived,
+
+  - the result of pointer arithmetic on a pointer that is safely derived (with
+    some restrictions on the integer operand),
+
+  - the result of a comma operator where the second operand is safely derived,
+
+  - the result of a conditional operator where the selected operand is safely
+    derived, or
+
+  - the result of loading from a safely derived gl-value.
+
+- A gl-value should be considered safely derived if it is:
+
+  - a dereference of a safely derived pointer,
+
+  - a member access into a safely derived gl-value, or
+
+  - a reference to a variable.
+
+- An access to a safely derived gl-value should be guaranteed to not allow
+  replacement of any of the safely-derived component values at any point in the
+  access.  "Access" should include loading a function pointer.
+
+- Assignments should include pointer-arithmetic operators like ``+=``.
+
+Making these guarantees will require further work, including significant new
+support in LLVM IR.
+
+Furthermore, Clang should implement a warning when assigning a data pointer that
+is not safely derived to a ``__ptrauth``-qualified gl-value.
+
+
+Language ABI
+------------
+
+This section describes the pointer-authentication ABI currently implemented in
+Clang for the Apple arm64e target.  As other targets adopt pointer
+authentication, this section should be generalized to express their ABIs as
+well.
+
+Key assignments
+~~~~~~~~~~~~~~~
+
+ARMv8.3 provides four abstract signing keys: ``IA``, ``IB``, ``DA``, and ``DB``.
+The architecture designates ``IA`` and ``IB`` for signing code pointers and
+``DA`` and ``DB`` for signing data pointers; this is reinforced by two
+properties:
+
+- The ISA provides instructions that perform combined auth+call and auth+load
+  operations; these instructions can only use the ``I`` keys and ``D`` keys,
+  respectively.
+
+- AArch64's TBI feature can be separately enabled for code pointers (controlling
+  whether indirect-branch instructions ignore those bits) and data pointers
+  (controlling whether memory-access instructions) ignore those bits. If TBI is
+  enabled for a kind of pointer, the sign and auth operations preserve the TBI
+  bits when signing with an associated keys (at the cost of shrinking the number
+  of available signing bits by 8).
+
+arm64e then further subdivides the keys as follows:
+
+- The ``A`` keys are used for primarily "global" purposes like signing v-tables
+  and function pointers.  These keys are sometimes called *process-independent*
+  or *cross-process* because on existing OSes they are not changed when changing
+  processes, although this is not a platform guarantee.
+
+- The ``B`` keys are used for primarily "local" purposes like signing return
+  addresses.  These keys are sometimes called *process-specific* because they
+  are typically different between processes. However, they are in fact shared
+  across processes in one situation: systems which provide ``fork`` cannot
+  change these keys in the child process; they can only be changed during
+  ``exec``.
+
+Implementation-defined algorithms and quantities
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The cryptographic hash algorithm used to compute signatures in ARMv8.3 is a
+private detail of the hardware implementation.
+
+arm64e restricts constant discriminators (used in ``__ptrauth`` and
+``ptrauth_blend_discriminator``) to the range from 0 to 65535, inclusive.  A 0
+discriminator generally signifies that no blending is required; see the
+documentation for ``ptrauth_blend_discriminator``.  This range is somewhat
+narrow but has two advantages:
+
+- The AArch64 ISA allows an arbitrary 16-bit immediate to be written over the
+  top 16 bits of a register in a single instruction:
+
+  .. code-block:: asm
+
+    movk xN, #0x4849, LSL 48
+
+  This is ideal for the discriminator blending operation because it adds minimal
+  code-size overhead and avoids overwriting any interesting bits from the
+  pointer.  Blending in a wider constant discriminator would either clobber
+  interesting bits (e.g. if it was loaded with ``movk xN, #0x4c4f, LSL 32``) or
+  require significantly more code (e.g. if the discriminator was loaded with a
+  ``mov+bfi`` sequence).
+
+- It is possible to pack a 16-bit discriminator into loader metadata with
+  minimal compromises, whereas a wider discriminator would require extra
+  metadata storage and therefore significantly impact load times.
+
+The string hash used by ``ptrauth_string_discriminator`` is a 64-bit SipHash-2-4
+using the constant seed ``b5d4c9eb79104a796fec8b1b428781d4`` (big-endian), with
+the result reduced by modulo to the range of non-zero discriminators (i.e.
+``(rawHash % 65535) + 1``).
+
+Return addresses
+~~~~~~~~~~~~~~~~
+
+The kernel must ensure that attackers cannot replace LR due to an asynchronous
+exception; see `Register clobbering`_.  If this is done by generally protecting
+LR, then functions which don't spill LR to the stack can avoid signing it
+entirely.  Otherwise, the return address must be signed; on arm64e it is signed
+with the ``IB`` key using the stack pointer on entry as the discriminator.
+
+Protecting return addresses is of such particular importance that the ``IB`` key
+is almost entirely reserved for this purpose.
+
+Global offset tables
+~~~~~~~~~~~~~~~~~~~~
+
+The global offset table (GOT) is not part of the language ABI, but it is a
+common implementation technique for dynamic linking which deserves special
+discussion here.
+
+Whenever possible, signed pointers should be materialized directly in code
+rather than via the GOT, e.g. using an ``adrp+add+pac`` sequence on ARMv8.3.
+This decreases the amount of work necessary at load time to initialize the GOT,
+but more importantly, it defines away the potential for several attacks:
+
+- Attackers cannot change instructions, so there is no way to cause this code
+  sequence to materialize a different pointer, whereas an access via the GOT
+  always has *at minimum* a probabilistic chance to be the target of successful
+  `substitution attacks`_.
+
+- The GOT is a dense pool of fixed pointers at a fixed offset relative to code;
+  attackers can search this pool for useful pointers that can be used in
+  `substitution attacks`_, whereas pointers that are only materialized directly
+  are not so easily available.
+
+- Similarly, attackers can use `access path attacks`_ to replace a pointer to a
+  signed pointer with a pointer to the GOT if the signing schema used within the
+  GOT happens to be the same as the original pointer.  This kind of collision
+  becomes much less likely to be useful the fewer pointers are in the GOT in the
+  first place.
+
+If this can be done for a symbol, then the compiler need only ensure that it
+materializes the signed pointer using registers that are safe against
+`register clobbering`_.
+
+However, many symbols can only be accessed via the GOT, e.g. because they
+resolve to definitions outside of the current image.  In this case, care must
+be taken to ensure that using the GOT does not introduce weaknesses.
+
+- If the entire GOT can be mapped read-only after loading, then no signing is
+  required within the GOT.  In fact, not signing pointers in the GOT is
+  preferable in this case because it makes the GOT useless for the harvesting
+  and access-path attacks above.  Storing raw pointers in this way is usually
+  extremely unsafe, but for the special case of an immutable GOT entry it's fine
+  because the GOT is always accessed via an address that is directly
+  materialized in code and thus provably unattackable.  (But see `Remapping`_.)
+
+- Otherwise, GOT entries which are used for producing a signed pointer constant
+  must be signed.  The signing schema used in the GOT need not match the target
+  signing schema for the signed constant.  To counteract the threats of
+  substitution attacks, it's best if GOT entries can be signed with address
+  diversity.  Using a good constant discriminator as well (perhaps derived from
+  the symbol name) can make it less useful to use a pointer to the GOT as the
+  replacement in an :ref:`access path attack<Access path attacks>`.
+
+In either case, the compiler must ensure that materializing the address of a GOT
+entry as part of producing a signed pointer constant is not vulnerable to
+`register clobbering`_.  If the linker also generates code for this, e.g. for
+call stubs, this generated code must take the same precautions.
+
+Dynamic symbol lookup
+~~~~~~~~~~~~~~~~~~~~~
+
+On platforms that support dynamically loading or resolving symbols it is
+necessary for them to define the pointer authentication semantics of the APIs
+provided to perform such lookups. While the platform may choose to reply
+unsigned pointers from such function and rely on the caller performing the
+initial signing, doing so creates the opportunity for caller side errors that
+create :ref:`signing oracles<Signing Oracles>`.
+
+On arm64e the `dlsym` function is used to resolve a symbol at runtime. If the
+resolved symbol is a function or other code pointer the returned pointer is
+signed using the default function signing schema described in
+:ref:`C function pointers<C function abi>`. If the resolved symbol is not a code pointer it is
+returned as an unsigned pointer.
+
+.. _C function abi:
+
+C function pointers
+~~~~~~~~~~~~~~~~~~~
+
+On arm64e, C function pointers are currently signed with the ``IA`` key without
+address diversity and with a constant discriminator of 0.
+
+The C and C++ standards do not permit C function pointers to be signed with
+address diversity by default: in C++ terms, function pointer types are required
+to be trivially copyable, which means they must be copyable with ``memcpy``.
+
+The use of a uniform constant discriminator greatly simplifies the adoption of
+arm64e, but it is a significant weakness in the mitigation because it allows any
+C function pointer to be replaced with another. Clang supports
+`-fptrauth-function-pointer-type-discrimination`, which enables a variant ABI
+that uses type discrimination for function pointers. When generating the type
+based discriminator for a function type all primitive integer types are
+considered equivalent due to the prevalence of mismatching integer parameter
+types in real world code. Type discrimination of function pointers is
+ABI-incompatible with the standard arm64e ABI, but it can be used in constrained
+contexts such as embedded systems or in code that does not require function
+pointer interoperation with the standard ABI (e.g. because it does not pass
+function pointers back and forth, or only does so through
+``__ptrauth``-qualified l-values).
+
+C++ virtual tables
+~~~~~~~~~~~~~~~~~~
+
+By default the pointer to a C++ virtual table is currently signed with the
+``DA`` key, address diversity, and a constant discriminator equal to the string
+hash (see `ptrauth_string_discriminator`_) of the mangled v-table identifier
+of the primary base class for the v-table. To support existing code or ABI
+constraints it is possible to use the `ptrauth_vtable_pointer` attribute to
+override the schema used for the v-table pointer of the base type of
+polymorphic class hierarchy. This attribute permits the configuration of the
+key, address diversity mode, and any extra constant discriminator to be used.
+
+Virtual functions in a C++ virtual table are signed with the ``IA`` key, address
+diversity, and a constant discriminator equal to the string hash (see
+`ptrauth_string_discriminator`_) of the mangled name of the function which
+originally gave rise to the v-table slot.
+
+C++ dynamic_cast
+~~~~~~~~~~~~~~~~
+
+C++'s ``dynamic_cast`` presents a difficulty relative to other polymorphic
+languages that have a
+`top type <https://en.wikipedia.org/wiki/Any_type>` as the use of declaration
+diversity for v-table pointers results in distinct signing schemas for each
+isolated type hierarchy. As a result it is not possible for the Itanium ABI
+defined ``__dynamic_cast`` entry point to directly authenticate the v-table
+pointer of the provided object.
+
+The current implementation uses a forced authentication of the subject object's
+v-table prior to invoking ``__dynamic_cast`` to partially verify that the
+object's vtable is valid. The ``__dynamic_cast`` implementation currently relies
+on this caller side check to limit the substitutability of the v-table pointer
+with an incorrect or invalid v-table. The subsequent implementation of the
+dynamic cast algorithm is built on pointer auth protected ``type_info`` objects.
+
+In future a richer solution may be developed to support vending the correct
+authentication schema directly to the ``dynamic_cast`` implementation.
+
+C++ std::type_info v-table pointers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The v-table pointer of the ``std::type_info`` type is signed with the ``DA`` key
+and no additional diversity.
+
+C++ member function pointers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A member function pointer is signed with the ``IA`` key, no address diversity,
+and a constant discriminator equal to the string hash
+(see `ptrauth_string_discriminator`_) of the member pointer type.  Address
+diversity is not permitted by C++ for member function pointers because they must
+be trivially-copyable types.
+
+The Itanium C++ ABI specifies that member function pointers to virtual functions
+simply store an offset to the correct v-table slot.  This ABI cannot be used
+securely with pointer authentication because there is no safe place to store the
+constant discriminator for the target v-table slot: if it's stored with the
+offset, an attacker can simply overwrite it with the right discriminator for the
+offset.  Even if the programmer never uses pointers to virtual functions, the
+existence of this code path makes all member function pointer dereferences
+insecure.
+
+arm64e changes this ABI so that virtual function pointers are stored using
+dispatch thunks with vague linkage.  Because arm64e supports interoperation with
+``arm64`` code when pointer authentication is disabled, an arm64e member
+function pointer dereference still recognizes the virtual-function
+representation but uses an bogus discriminator on that path that should always
+trap if pointer authentication is enabled dynamically.
+
+The use of dispatch thunks means that ``==`` on member function pointers is no
+longer reliable for virtual functions, but this is acceptable because the
+standard makes no guarantees about it in the first place.
+
+The use of dispatch thunks also is required to support declaration specific
+authentication schemas for v-table pointers.
+
+C++ mangling
+~~~~~~~~~~~~
+
+When the ``__ptrauth`` qualifier appears in a C++ mangled name,
+it is mangled as a vendor qualifier with the signature
+``U9__ptrauthILj<key>ELb<addressDiscriminated>ELj<extraDiscriminator>EE``.
+
+e.g. ``int * __ptrauth(1, 0, 1234)`` will be mangled as
+``U9__ptrauthILj1ELb0ELj1234EE``.
+
+If the vtable pointer authentication scheme of a polymorphic class is overridden
+we mangle the override information with the vendor qualifier
+``__vtptrauth(int key, bool addressDiscriminated, unsigned extraDiscriminator)``,
+where the extra discriminator is the explicit value the specified discrimination
+mode evalutes to.
+
+Blocks
+~~~~~~
+
+Block pointers are data pointers which must interoperate with the ObjC `id` type
+and therefore cannot be signed themselves. As blocks conform to the ObjC `id`
+type, they contain an ``isa`` pointer signed as described
+:ref:`below<Objc isa and super>`.
+
+The invocation pointer in a block is signed with the ``IA`` key using address
+diversity and a constant dicriminator of 0.  Using a uniform discriminator is
+seen as a weakness to be potentially improved, but this is tricky due to the
+subtype polymorphism directly permitted for blocks.
+
+Block descriptors and ``__block`` variables can contain pointers to functions
+that can be used to copy or destroy the object.  These functions are signed with
+the ``IA`` key, address diversity, and a constant discriminator of 0.  The
+structure of block descriptors is under consideration for improvement.
+
+Objective-C runtime
+~~~~~~~~~~~~~~~~~~~
+
+In addition to the compile time ABI design, the Objective-C runtime provides
+additional protection to methods and other metadata that have been loaded into
+the Objective-C method cache; this protection is private to the runtime.
+
+Objective-C methods
+~~~~~~~~~~~~~~~~~~~
+
+Objective-C method lists sign methods with the ``IA`` key using address
+diversity and a constant discriminator of 0.  Using a uniform constant
+discriminator is believed to be acceptable because these tables are only
+accessed internally to the Objective-C runtime.
+
+Objective-C class method list pointer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The method list pointer in Objective-C classes are signed with the ``DA`` key
+using address diversity, and a constant discriminator of 0xC310.
+
+Objective-C class read-only data pointer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The read-only data pointer in Objective-C classes are signed with the ``DA`` key
+using address diversity, and a constant discriminator of 0x61F8.
+
+.. _Objc isa and super:
+
+Objective-C ``isa`` and ``super`` pointers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+An Objective-C object's ``isa`` and ``super`` pointers are both signed with
+the ``DA`` key using address diversity and constant discriminators of 0x6AE1
+and 0x25DA respectively.
+
+Objective-C ``SEL`` pointers
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+By default, the type of an Objective-C instance variable of type ``SEL``, when
+the qualifiers do not include an explicit ``__ptrauth`` qualifier, is adjusted
+to be qualified with ``__ptrauth(ptrauth_key_asdb, 1, 0x57C2)``.
+
+This provides a measure of implicit at-rest protection to  Objective-C classes
+that store selectors, as in the common target-action design pattern. This
+prevents attackers from overriding the selector to invoke an arbitrary different
+method, which is a major attack vector in Objective-C. Since ``SEL`` values are
+not normally passed around as signed pointers, there is a
+:ref:`signing oracle<Signing Oracles>` associated with the initialization of the
+ivar, but the use of address and constant diversity limit the risks.
+
+The implicit qualifier means that the type of the ivar does not match its
+declaration, which can cause type errors if the address of the ivar is taken:
+
+.. code-block:: ObjC
+
+  @interface A : NSObject {
+    SEL _s;
+  }
+  @end
+
+  void f(SEL *);
+
+  @implementation A
+  -(void)g
+  {
+     f(&_s);
+  }
+  @end
+
+To fix such an mismatch the schema macro from `<ptrauth.h>`:
+
+.. code-block:: ObjC
+
+  #include <ptrauth.h>
+
+  void f(SEL __ptrauth_objc_sel*);
 
+or less safely, and introducing the possibility of an
+:ref:`signing or authentication oracle<Signing oracles>`, an unauthencaticated
+temporary may be used as intermediate storage.
 
-Alternative Implementations
+Alternative implementations
 ---------------------------
 
-Signature Storage
+Signature storage
 ~~~~~~~~~~~~~~~~~
 
 It is not critical for the security of pointer authentication that the
@@ -536,7 +1667,7 @@ Storing the signature in the high bits, as Armv8.3 does, has several trade-offs:
   return signed pointers.  This means that clients of these APIs will not
   require insecure code in order to correctly receive a function pointer.
 
-Hashing vs. Encrypting Pointers
+Hashing vs. encrypting pointers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Armv8.3 implements ``sign`` by computing a cryptographic hash and storing that
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 2a95d1e2736a3..6a73754b73a00 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -1,3 +1,6 @@
+.. If you want to modify sections/contents permanently, you should modify both
+   ReleaseNotes.rst and ReleaseNotesTemplate.txt.
+
 ===========================================
 Clang |release| |ReleaseNotesTitle|
 ===========================================
@@ -34,6 +37,22 @@ latest release, please see the `Clang Web Site <https://clang.llvm.org>`_ or the
 Potentially Breaking Changes
 ============================
 
+- Clang will now emit a warning if the auto-detected GCC installation
+  directory (i.e. the one with the largest version number) does not
+  contain libstdc++ include directories although a "complete" GCC
+  installation directory containing the include directories is
+  available. It is planned to change the auto-detection to prefer the
+  "complete" directory in the future.  The warning will disappear if
+  the libstdc++ include directories are either installed or removed
+  for all GCC installation directories considered by the
+  auto-detection; see the output of ``clang -v`` for a list of those
+  directories. If the GCC installations cannot be modified and
+  maintaining the current choice of the auto-detection is desired, the
+  GCC installation directory can be selected explicitly using the
+  ``--gcc-install-dir`` command line argument. This will silence the
+  warning. It can also be disabled using the
+  ``-Wno-gcc-install-dir-libstdcxx`` command line flag.
+
 C/C++ Language Potentially Breaking Changes
 -------------------------------------------
 
@@ -70,12 +89,19 @@ ABI Changes in This Version
 
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
+- How nested name specifiers are dumped and printed changes, keeping track of clang AST changes.
 
 Clang Frontend Potentially Breaking Changes
 -------------------------------------------
+- Members of anonymous unions/structs are now injected as ``IndirectFieldDecl``
+  into the enclosing record even if their names conflict with other names in the
+  scope. These ``IndirectFieldDecl`` are marked invalid.
 
 Clang Python Bindings Potentially Breaking Changes
 --------------------------------------------------
+- TypeKind ``ELABORATED`` is not used anymore, per clang AST changes removing
+  ElaboratedTypes. The value becomes unused, and all the existing users should
+  expect the former underlying type to be reported instead.
 
 What's New in Clang |release|?
 ==============================
@@ -109,21 +135,59 @@ C23 Feature Support
 
 Non-comprehensive list of changes in this release
 -------------------------------------------------
+- Added ``__builtin_elementwise_fshl`` and ``__builtin_elementwise_fshr``.
+
+- ``__builtin_elementwise_abs`` can now be used in constant expression.
+
 - Added ``__builtin_elementwise_minnumnum`` and ``__builtin_elementwise_maxnumnum``.
 
-- Trapping UBSan (e.g. ``-fsanitize-trap=undefined``) now emits a string describing the reason for 
-  trapping into the generated debug info. This feature allows debuggers (e.g. LLDB) to display 
-  the reason for trapping if the trap is reached. The string is currently encoded in the debug 
-  info as an artificial frame that claims to be inlined at the trap location. The function used 
-  for the artificial frame is an artificial function whose name encodes the reason for trapping. 
-  The encoding used is currently the same as ``__builtin_verbose_trap`` but might change in the future. 
-  This feature is enabled by default but can be disabled by compiling with 
+- Trapping UBSan (e.g. ``-fsanitize-trap=undefined``) now emits a string describing the reason for
+  trapping into the generated debug info. This feature allows debuggers (e.g. LLDB) to display
+  the reason for trapping if the trap is reached. The string is currently encoded in the debug
+  info as an artificial frame that claims to be inlined at the trap location. The function used
+  for the artificial frame is an artificial function whose name encodes the reason for trapping.
+  The encoding used is currently the same as ``__builtin_verbose_trap`` but might change in the future.
+  This feature is enabled by default but can be disabled by compiling with
   ``-fno-sanitize-annotate-debug-info-traps``.
 
+- ``__builtin_elementwise_max`` and ``__builtin_elementwise_min`` functions for integer types can
+  now be used in constant expressions.
+
+- A vector of booleans is now a valid condition for the ternary ``?:`` operator.
+  This binds to a simple vector select operation.
+
+- Added ``__builtin_masked_load`` and ``__builtin_masked_store`` for conditional
+  memory loads from vectors. Binds to the LLVM intrinsic of the same name.
+
+- Use of ``__has_feature`` to detect the ``ptrauth_qualifier`` and ``ptrauth_intrinsics``
+  features has been deprecated, and is restricted to the arm64e target only. The
+  correct method to check for these features is to test for the ``__PTRAUTH__``
+  macro.
+
+- Added a new builtin, ``__builtin_dedup_pack``, to remove duplicate types from a parameter pack.
+  This feature is particularly useful in template metaprogramming for normalizing type lists.
+  The builtin produces a new, unexpanded parameter pack that can be used in contexts like template
+  argument lists or base specifiers.
+
+  .. code-block:: c++
+
+    template <typename...> struct TypeList;
+
+    // The resulting type is TypeList<int, double, char>
+    using MyTypeList = TypeList<__builtin_dedup_pack<int, double, int, char, double>...>;
+
+  Currently, the use of ``__builtin_dedup_pack`` is limited to template arguments and base
+  specifiers, it also must be used within a template context.
+
+
 New Compiler Flags
 ------------------
 - New option ``-fno-sanitize-annotate-debug-info-traps`` added to disable emitting trap reasons into the debug info when compiling with trapping UBSan (e.g. ``-fsanitize-trap=undefined``).
 
+Lanai Support
+^^^^^^^^^^^^^^
+- The option ``-mcmodel={small,medium,large}`` is supported again.
+
 Deprecated Compiler Flags
 -------------------------
 
@@ -143,8 +207,37 @@ Improvements to Clang's diagnostics
   Moved the warning for a missing (though implied) attribute on a redeclaration into this group.
   Added a new warning in this group for the case where the attribute is missing/implicit on
   an override of a virtual method.
-- Fixed fix-it hint for fold expressions. Clang now correctly places the suggested right 
+- Fixed fix-it hint for fold expressions. Clang now correctly places the suggested right
   parenthesis when diagnosing malformed fold expressions. (#GH151787)
+- Added fix-it hint for when scoped enumerations require explicit conversions for binary operations. (#GH24265)
+
+- Fixed an issue where emitted format-signedness diagnostics were not associated with an appropriate
+  diagnostic id. Besides being incorrect from an API standpoint, this was user visible, e.g.:
+  "format specifies type 'unsigned int' but the argument has type 'int' [-Wformat]"
+  "signedness of format specifier 'u' is incompatible with 'c' [-Wformat]"
+  This was misleading, because even though -Wformat is required in order to emit the diagnostics,
+  the warning flag the user needs to concerned with here is -Wformat-signedness, which is also
+  required and is not enabled by default. With the change you'll now see:
+  "format specifies type 'unsigned int' but the argument has type 'int', which differs in signedness [-Wformat-signedness]"
+  "signedness of format specifier 'u' is incompatible with 'c' [-Wformat-signedness]"
+  and the API-visible diagnostic id will be appropriate.
+  
+- Fixed false positives in ``-Waddress-of-packed-member`` diagnostics when
+  potential misaligned members get processed before they can get discarded.
+  (#GH144729)
+
+- Fixed false positive in ``-Wmissing-noreturn`` diagnostic when it was requiring the usage of
+  ``[[noreturn]]`` on lambdas before C++23 (#GH154493).
+
+- Clang now diagnoses the use of ``#`` and ``##`` preprocessor tokens in
+  attribute argument lists in C++ when ``-pedantic`` is enabled. The operators
+  can be used in macro replacement lists with the usual preprocessor semantics,
+  however, non-preprocessor use of tokens now triggers a pedantic warning in C++.
+  Compilation in C mode is unchanged, and still permits these tokens to be used. (#GH147217)
+
+- Clang now diagnoses misplaced array bounds on declarators for template
+  specializations in th same way as it already did for other declarators.
+  (#GH147333)
 
 Improvements to Clang's time-trace
 ----------------------------------
@@ -161,6 +254,12 @@ Bug Fixes in This Version
   targets that treat ``_Float16``/``__fp16`` as native scalar types. Previously
   the warning was silently lost because the operands differed only by an implicit
   cast chain. (#GH149967).
+- Fixed a crash with incompatible pointer to integer conversions in designated
+  initializers involving string literals. (#GH154046)
+- Clang now emits a frontend error when a function marked with the `flatten` attribute
+  calls another function that requires target features not enabled in the caller. This
+  prevents a fatal error in the backend.
+- Fixed scope of typedefs present inside a template class. (#GH91451)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -170,8 +269,11 @@ Bug Fixes to Compiler Builtins
 Bug Fixes to Attribute Support
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- ``[[nodiscard]]`` is now respected on Objective-C and Objective-C++ methods.
-  (#GH141504)
+- ``[[nodiscard]]`` is now respected on Objective-C and Objective-C++ methods
+  (#GH141504) and on types returned from indirect calls (#GH142453).
+- Fixes some late parsed attributes, when applied to function definitions, not being parsed
+  in function try blocks, and some situations where parsing of the function body
+  is skipped, such as error recovery and code completion. (#GH153551)
 - Using ``[[gnu::cleanup(some_func)]]`` where some_func is annotated with
   ``[[gnu::error("some error")]]`` now correctly triggers an error. (#GH146520)
 
@@ -180,11 +282,26 @@ Bug Fixes to C++ Support
 - Diagnose binding a reference to ``*nullptr`` during constant evaluation. (#GH48665)
 - Suppress ``-Wdeprecated-declarations`` in implicitly generated functions. (#GH147293)
 - Fix a crash when deleting a pointer to an incomplete array (#GH150359).
+- Fixed a mismatched lambda scope bug when propagating up ``consteval`` within nested lambdas. (#GH145776)
 - Fix an assertion failure when expression in assumption attribute
   (``[[assume(expr)]]``) creates temporary objects.
+- Fix the dynamic_cast to final class optimization to correctly handle
+  casts that are guaranteed to fail (#GH137518).
+- Fix bug rejecting partial specialization of variable templates with auto NTTPs (#GH118190).
+- Fix a crash if errors "member of anonymous [...] redeclares" and
+  "intializing multiple members of union" coincide (#GH149985).
+- Fix a crash when using ``explicit(bool)`` in pre-C++11 language modes. (#GH152729)
+- Fix the parsing of variadic member functions when the ellipis immediately follows a default argument.(#GH153445)
+- Fixed a bug that caused ``this`` captured by value in a lambda with a dependent explicit object parameter to not be
+  instantiated properly. (#GH154054)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
+- Fix incorrect name qualifiers applied to alias CTAD. (#GH136624)
+- Fixed ElaboratedTypes appearing within NestedNameSpecifier, which was not a
+  legal representation. This is fixed because ElaboratedTypes don't exist anymore. (#GH43179) (#GH68670) (#GH92757)
+- Fix unrecognized html tag causing undesirable comment lexing (#GH152944)
+- Fix comment lexing of special command names (#GH152943)
 
 Miscellaneous Bug Fixes
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -206,6 +323,13 @@ NVPTX Support
 
 X86 Support
 ^^^^^^^^^^^
+- More SSE, AVX and AVX512 intrinsics, including initializers and general
+  arithmetic can now be used in C++ constant expressions.
+- Some SSE, AVX and AVX512 intrinsics have been converted to wrap
+  generic __builtin intrinsics.
+- NOTE: Please avoid use of the __builtin_ia32_* intrinsics - these are not 
+  guaranteed to exist in future releases, or match behaviour with previous
+  releases of clang or other compilers.
 
 Arm and AArch64 Support
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -225,6 +349,9 @@ RISC-V Support
 - Add support for `__attribute__((interrupt("rnmi")))` to be used with the `Smrnmi` extension.
   With this the `Smrnmi` extension is fully supported.
 
+- Add `-march=unset` to clear any previous `-march=` value. This ISA string will
+  be computed from `-mcpu` or the platform default.
+
 CUDA/HIP Language Changes
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -254,11 +381,20 @@ Fixed Point Support in Clang
 
 AST Matchers
 ------------
+- Removed elaboratedType matchers, and related nested name specifier changes,
+  following the corresponding changes in the clang AST.
 - Ensure ``hasBitWidth`` doesn't crash on bit widths that are dependent on template
   parameters.
 
+- Add a boolean member ``IgnoreSystemHeaders`` to ``MatchFinderOptions``. This
+  allows it to ignore nodes in system headers when traversing the AST.
+
+- ``hasConditionVariableStatement`` now supports ``for`` loop, ``while`` loop
+  and ``switch`` statements.
+
 clang-format
 ------------
+- Add ``SpaceInEmptyBraces`` option and set it to ``Always`` for WebKit style.
 
 libclang
 --------
@@ -278,8 +414,9 @@ New features
 
 Crash and bug fixes
 ^^^^^^^^^^^^^^^^^^^
-- Fixed a crash in the static analyzer that when the expression in an 
+- Fixed a crash in the static analyzer that when the expression in an
   ``[[assume(expr)]]`` attribute was enclosed in parentheses.  (#GH151529)
+- Fixed a crash when parsing ``#embed`` parameters with unmatched closing brackets. (#GH152829)
 
 Improvements
 ^^^^^^^^^^^^
@@ -294,12 +431,14 @@ Sanitizers
 
 Python Binding Changes
 ----------------------
+- Exposed `clang_getCursorLanguage` via `Cursor.language`.
 
 OpenMP Support
 --------------
 - Added parsing and semantic analysis support for the ``need_device_addr``
   modifier in the ``adjust_args`` clause.
 - Allow array length to be omitted in array section subscript expression.
+- Fixed non-contiguous strided update in the ``omp target update`` directive with the ``from`` clause.
 
 Improvements
 ^^^^^^^^^^^^
diff --git a/clang/docs/ReleaseNotesTemplate.txt b/clang/docs/ReleaseNotesTemplate.txt
new file mode 100644
index 0000000000000..cc04deb45bb53
--- /dev/null
+++ b/clang/docs/ReleaseNotesTemplate.txt
@@ -0,0 +1,246 @@
+.. If you want to modify sections/contents permanently, you should modify both
+   ReleaseNotes.rst and ReleaseNotesTemplate.txt.
+
+===========================================
+Clang |release| |ReleaseNotesTitle|
+===========================================
+
+.. contents::
+   :local:
+   :depth: 2
+
+Written by the `LLVM Team <https://llvm.org/>`_
+
+.. only:: PreRelease
+
+  .. warning::
+     These are in-progress notes for the upcoming Clang |version| release.
+     Release notes for previous releases can be found on
+     `the Releases Page <https://llvm.org/releases/>`_.
+
+Introduction
+============
+
+This document contains the release notes for the Clang C/C++/Objective-C
+frontend, part of the LLVM Compiler Infrastructure, release |release|. Here we
+describe the status of Clang in some detail, including major
+improvements from the previous release and new feature work. For the
+general LLVM release notes, see `the LLVM
+documentation <https://llvm.org/docs/ReleaseNotes.html>`_. For the libc++ release notes,
+see `this page <https://libcxx.llvm.org/ReleaseNotes.html>`_. All LLVM releases
+may be downloaded from the `LLVM releases web site <https://llvm.org/releases/>`_.
+
+For more information about Clang or LLVM, including information about the
+latest release, please see the `Clang Web Site <https://clang.llvm.org>`_ or the
+`LLVM Web Site <https://llvm.org>`_.
+
+Potentially Breaking Changes
+============================
+
+C/C++ Language Potentially Breaking Changes
+-------------------------------------------
+
+C++ Specific Potentially Breaking Changes
+-----------------------------------------
+
+ABI Changes in This Version
+---------------------------
+
+AST Dumping Potentially Breaking Changes
+----------------------------------------
+
+Clang Frontend Potentially Breaking Changes
+-------------------------------------------
+
+Clang Python Bindings Potentially Breaking Changes
+--------------------------------------------------
+
+What's New in Clang |release|?
+==============================
+
+C++ Language Changes
+--------------------
+
+C++2c Feature Support
+^^^^^^^^^^^^^^^^^^^^^
+
+C++23 Feature Support
+^^^^^^^^^^^^^^^^^^^^^
+
+C++20 Feature Support
+^^^^^^^^^^^^^^^^^^^^^
+
+C++17 Feature Support
+^^^^^^^^^^^^^^^^^^^^^
+
+Resolutions to C++ Defect Reports
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+C Language Changes
+------------------
+
+C2y Feature Support
+^^^^^^^^^^^^^^^^^^^
+
+C23 Feature Support
+^^^^^^^^^^^^^^^^^^^
+
+Non-comprehensive list of changes in this release
+-------------------------------------------------
+
+New Compiler Flags
+------------------
+
+Deprecated Compiler Flags
+-------------------------
+
+Modified Compiler Flags
+-----------------------
+
+Removed Compiler Flags
+----------------------
+
+Attribute Changes in Clang
+--------------------------
+
+Improvements to Clang's diagnostics
+-----------------------------------
+
+Improvements to Clang's time-trace
+----------------------------------
+
+Improvements to Coverage Mapping
+--------------------------------
+
+Bug Fixes in This Version
+-------------------------
+
+Bug Fixes to Compiler Builtins
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Bug Fixes to Attribute Support
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Bug Fixes to C++ Support
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Bug Fixes to AST Handling
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Miscellaneous Bug Fixes
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Miscellaneous Clang Crashes Fixed
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+OpenACC Specific Changes
+------------------------
+
+Target Specific Changes
+-----------------------
+
+AMDGPU Support
+^^^^^^^^^^^^^^
+
+NVPTX Support
+^^^^^^^^^^^^^^
+
+X86 Support
+^^^^^^^^^^^
+
+Arm and AArch64 Support
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Android Support
+^^^^^^^^^^^^^^^
+
+Windows Support
+^^^^^^^^^^^^^^^
+
+LoongArch Support
+^^^^^^^^^^^^^^^^^
+
+RISC-V Support
+^^^^^^^^^^^^^^
+
+CUDA/HIP Language Changes
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+CUDA Support
+^^^^^^^^^^^^
+
+AIX Support
+^^^^^^^^^^^
+
+NetBSD Support
+^^^^^^^^^^^^^^
+
+WebAssembly Support
+^^^^^^^^^^^^^^^^^^^
+
+AVR Support
+^^^^^^^^^^^
+
+DWARF Support in Clang
+----------------------
+
+Floating Point Support in Clang
+-------------------------------
+
+Fixed Point Support in Clang
+----------------------------
+
+AST Matchers
+------------
+
+clang-format
+------------
+
+libclang
+--------
+
+Code Completion
+---------------
+
+Static Analyzer
+---------------
+
+New features
+^^^^^^^^^^^^
+
+Crash and bug fixes
+^^^^^^^^^^^^^^^^^^^
+
+Improvements
+^^^^^^^^^^^^
+
+Moved checkers
+^^^^^^^^^^^^^^
+
+.. _release-notes-sanitizers:
+
+Sanitizers
+----------
+
+Python Binding Changes
+----------------------
+
+OpenMP Support
+--------------
+
+Improvements
+^^^^^^^^^^^^
+
+Additional Information
+======================
+
+A wide variety of additional information is available on the `Clang web
+page <https://clang.llvm.org/>`_. The web page contains versions of the
+API documentation which are up-to-date with the Git version of
+the source code. You can access versions of these documents specific to
+this release by going into the "``clang/docs/``" directory in the Clang
+tree.
+
+If you have any questions or comments about Clang, please feel free to
+contact us on the `Discourse forums (Clang Frontend category)
+<https://discourse.llvm.org/c/clang/6>`_.
diff --git a/clang/docs/SourceBasedCodeCoverage.rst b/clang/docs/SourceBasedCodeCoverage.rst
index 3e8642479a56d..2f114070a8fb2 100644
--- a/clang/docs/SourceBasedCodeCoverage.rst
+++ b/clang/docs/SourceBasedCodeCoverage.rst
@@ -66,17 +66,17 @@ supported. Uninstrumented code simply won't be accounted for in reports.
 
 To compile code with Modified Condition/Decision Coverage (MC/DC) enabled,
 pass ``-fcoverage-mcdc`` in addition to the clang options specified above.
-MC/DC is an advanced form of code coverage most applicable in the embedded
+MC/DC is an advanced form of code coverage most applicable to the embedded
 space.
 
 Running the instrumented program
 ================================
 
-The next step is to run the instrumented program. When the program exits it
+The next step is to run the instrumented program. When the program exits, it
 will write a **raw profile** to the path specified by the ``LLVM_PROFILE_FILE``
 environment variable. If that variable does not exist, the profile is written
 to ``default.profraw`` in the current directory of the program. If
-``LLVM_PROFILE_FILE`` contains a path to a non-existent directory, the missing
+``LLVM_PROFILE_FILE`` specifies a path to a non-existent directory, the missing
 directory structure will be created.  Additionally, the following special
 **pattern strings** are rewritten:
 
@@ -97,7 +97,7 @@ directory structure will be created.  Additionally, the following special
 * "%b" expands out to the binary ID (build ID). It can be used with "%Nm" to
   avoid binary signature collisions. To use it, the program should be compiled
   with the build ID linker option (``--build-id`` for GNU ld or LLD,
-  ``/build-id`` for lld-link on Windows). Linux, Windows and AIX are supported.
+  ``/build-id`` for lld-link on Windows). Linux, Windows, and AIX are supported.
 
 * "%c" expands out to nothing, but enables a mode in which profile counter
   updates are continuously synced to a file. This means that if the
@@ -128,7 +128,7 @@ and set bias to the offset between the original and the new counter location,
 at which point every subsequent counter access will be to the new location,
 which allows updating profile directly akin to the continuous mode.
 
-The advantage of this approach is that doesn't require any special OS support.
+The advantage of this approach is that it doesn't require any special OS support.
 The disadvantage is the extra overhead due to additional instructions required
 for each counter access (overhead both in terms of binary size and performance)
 plus duplication of counters (i.e. one copy in the binary itself and another
@@ -137,7 +137,7 @@ other platforms by passing the ``-runtime-counter-relocation`` option to the
 backend during compilation.
 
 For a program such as the `Lit <https://llvm.org/docs/CommandGuide/lit.html>`_
-testing tool which invokes other programs, it may be necessary to set
+testing tool, which invokes other programs, it may be necessary to set
 ``LLVM_PROFILE_FILE`` for each invocation. The pattern strings "%p" or "%Nm"
 may help to avoid corruption due to concurrency. Note that "%p" is also a Lit
 token and needs to be escaped as "%%p".
@@ -149,7 +149,7 @@ token and needs to be escaped as "%%p".
 Creating coverage reports
 =========================
 
-Raw profiles have to be **indexed** before they can be used to generate
+Raw profiles must be **indexed** before they can be used to generate
 coverage reports. This is done using the "merge" tool in ``llvm-profdata``
 (which can combine multiple raw profiles and index them at the same time):
 
@@ -240,13 +240,13 @@ line-oriented report, try:
     TOTAL                   13                 0   100.00%           3                 0   100.00%          13                 0   100.00%           12                  2    83.33%
 
 The ``llvm-cov`` tool supports specifying a custom demangler, writing out
-reports in a directory structure, and generating html reports. For the full
+reports in a directory structure, and generating HTML reports. For the full
 list of options, please refer to the `command guide
 <https://llvm.org/docs/CommandGuide/llvm-cov.html>`_.
 
 A few final notes:
 
-* The ``-sparse`` flag is optional but can result in dramatically smaller
+* The ``-sparse`` flag is optional but can produce dramatically smaller
   indexed profiles. This option should not be used if the indexed profile will
   be reused for PGO.
 
@@ -255,7 +255,7 @@ A few final notes:
   information directly into an existing raw profile on disk. The details are
   out of scope.
 
-* The ``llvm-profdata`` tool can be used to merge together multiple raw or
+* The ``llvm-profdata`` tool can be used to merge multiple raw or
   indexed profiles. To combine profiling data from multiple runs of a program,
   try e.g:
 
@@ -299,7 +299,7 @@ There are six statistics tracked in a coverage summary:
   source code that may each evaluate to either "true" or "false".  These
   conditions may comprise larger boolean expressions linked by boolean logical
   operators. For example, "x = (y == 2) || (z < 10)" is a boolean expression
-  that is comprised of two individual conditions, each of which evaluates to
+  comprised of two individual conditions, each of which evaluates to
   either true or false, producing four total branch outcomes.
 
 * Modified Condition/Decision Coverage (MC/DC) is the percentage of individual
@@ -316,7 +316,7 @@ There are six statistics tracked in a coverage summary:
   ``-show-mcdc-summary`` option as long as code was also compiled using the
   clang option ``-fcoverage-mcdc``.
 
-  * Boolean expressions that are only comprised of one condition (and therefore
+  * Boolean expressions comprised of only one condition (and therefore
     have no logical operators) are not included in MC/DC analysis and are
     trivially deducible using branch coverage.
 
@@ -366,7 +366,7 @@ By default the compiler runtime uses a static initializer to determine the
 profile output path and to register a writer function. To collect profiles
 without using static initializers, do this manually:
 
-* Export a ``int __llvm_profile_runtime`` symbol from each instrumented shared
+* Export an ``int __llvm_profile_runtime`` symbol from each instrumented shared
   library and executable. When the linker finds a definition of this symbol, it
   knows to skip loading the object which contains the profiling runtime's
   static initializer.
@@ -380,7 +380,7 @@ without using static initializers, do this manually:
   to ``__llvm_profile_write_file``.
 
 * Forward-declare ``int __llvm_profile_write_file(void)`` and call it to write
-  out a profile. This function returns 0 when it succeeds, and a non-zero value
+  out a profile. This function returns 0 on success, and a non-zero value
   otherwise. Calling this function multiple times appends profile data to an
   existing on-disk raw profile.
 
@@ -418,7 +418,7 @@ Collecting coverage reports for the llvm project
 ================================================
 
 To prepare a coverage report for llvm (and any of its sub-projects), add
-``-DLLVM_BUILD_INSTRUMENTED_COVERAGE=On`` to the cmake configuration. Raw
+``-DLLVM_BUILD_INSTRUMENTED_COVERAGE=On`` to the CMake configuration. Raw
 profiles will be written to ``$BUILD_DIR/profiles/``. To prepare an html
 report, run ``llvm/utils/prepare-code-coverage-artifact.py``.
 
@@ -429,7 +429,7 @@ To specify an alternate directory for raw profiles, use
 Drawbacks and limitations
 =========================
 
-* Prior to version 2.26, the GNU binutils BFD linker is not able link programs
+* Prior to version 2.26, the GNU binutils BFD linker cannot link programs
   compiled with ``-fcoverage-mapping`` in its ``--gc-sections`` mode.  Possible
   workarounds include disabling ``--gc-sections``, upgrading to a newer version
   of BFD, or using the Gold linker.
diff --git a/clang/docs/StandardCPlusPlusModules.rst b/clang/docs/StandardCPlusPlusModules.rst
index 31d0a5e769378..7155ad6cff83f 100644
--- a/clang/docs/StandardCPlusPlusModules.rst
+++ b/clang/docs/StandardCPlusPlusModules.rst
@@ -226,8 +226,8 @@ one-phase compilation model is simpler for build systems to implement while the
 two-phase compilation has the potential to compile faster due to higher
 parallelism. As an example, if there are two module units ``A`` and ``B``, and
 ``B`` depends on ``A``, the one-phase compilation model needs to compile them
-serially, whereas the two-phase compilation model is able to be compiled as
-soon as ``A.pcm`` is available, and thus can be compiled simultaneously as the
+serially, whereas the two-phase compilation model can be compiled as
+soon as ``A.pcm`` is available, and thus can be compiled simultaneously with the
 ``A.pcm`` to ``A.o`` compilation step.
 
 File name requirements
@@ -391,7 +391,7 @@ And the compilation processes for module units are like:
 As the diagrams show, we need to compile the BMI from module units to object
 files and then link the object files. (However, this cannot be done for the BMI
 from header units. See the section on :ref:`header units <header-units>` for
-more details.
+more details.)
 
 BMIs cannot be shipped in an archive to create a module library. Instead, the
 BMIs(``*.pcm``) are compiled into object files(``*.o``) and those object files
@@ -403,7 +403,7 @@ clang-cl
 ``clang-cl`` supports the same options as ``clang++`` for modules as detailed above;
 there is no need to prefix these options with ``/clang:``. Note that ``cl.exe``
 `options to emit/consume IFC files <https://devblogs.microsoft.com/cppblog/using-cpp-modules-in-msvc-from-the-command-line-part-1/>` are *not* supported.
-The resultant precompiled modules are also not compatible for use with ``cl.exe``.
+The resulting precompiled modules are also not compatible for use with ``cl.exe``.
 
 We recommend that build system authors use the above-mentioned ``clang++`` options  with ``clang-cl`` to build modules.
 
@@ -411,7 +411,7 @@ Consistency Requirements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 Modules can be viewed as a kind of cache to speed up compilation. Thus, like
-other caching techniques, it is important to maintain cache consistency which
+other caching techniques, it is important to maintain cache consistency, which
 is why Clang does very strict checking for consistency.
 
 Options consistency
@@ -472,8 +472,8 @@ To overcome these requirements and simplify cases like distributed builds and sa
 builds, users can use the ``-fmodules-embed-all-files`` flag to embed all input files
 into the BMI so that Clang does not need to open the corresponding file on disk.
 
-When the ``-fmodules-embed-all-files`` flag are enabled, Clang explicitly emits the source
-code into the BMI file, the contents of the BMI file contain a sufficiently verbose
+When the ``-fmodules-embed-all-files`` flag is enabled, Clang explicitly emits the source
+code into the BMI file; the BMI file contains a sufficiently verbose
 representation to reproduce the original source file.
 
 .. [1] Input files: The source files which took part in the compilation of the BMI.
@@ -578,7 +578,7 @@ handle the dynamic initialization of non-inline variables in the module unit.
 The importable module unit has to emit the initializer even if there is no
 dynamic initialization; otherwise, the importer may call a nonexistent
 function. The initializer function emits calls to imported modules first
-followed by calls to all to of the dynamic initializers in the current module
+followed by calls to all of the dynamic initializers in the current module
 unit.
 
 Translation units that explicitly or implicitly import a named module must call
@@ -689,9 +689,9 @@ ensure it is reachable, e.g. ``using N::g;``.
 
 As of Clang 22.x, the Reduced BMI is enabled by default. You may still want to
 use Full BMI with ``-fno-modules-reduced-bmi`` in the following case:
-1. Your build system uses two-phase compilation but it haven't adjusted the
+1. Your build system uses two-phase compilation, but it hasn't adjusted the
 implementation for reduced BMI.
-2. You meet a regression with Reduced BMI that you cannot work around. Please
+2. You encounter a regression with Reduced BMI that you cannot work around. Please
 report an issue for this case.
 
 Experimental Non-Cascading Changes
@@ -699,7 +699,7 @@ Experimental Non-Cascading Changes
 
 This section is primarily for build system vendors. For end compiler users,
 if you don't want to read it all, this is helpful to reduce recompilations.
-We encourage build system vendors and end users try this out and bring feedback.
+We encourage build system vendors and end users to try this out and bring feedback.
 
 Before Clang 19, a change in BMI of any (transitive) dependency would cause the
 outputs of the BMI to change. Starting with Clang 19, changes to non-direct
@@ -786,7 +786,7 @@ We encourage build systems to add an experimental mode that
 reuses the cached BMI when **direct** dependencies did not change,
 even if **transitive** dependencies did change.
 
-Given there are potential compiler bugs, we recommend that build systems
+Given that there are potential compiler bugs, we recommend that build systems
 support this feature as a configurable option so that users
 can go back to the transitive change mode safely at any time.
 
@@ -813,7 +813,7 @@ With reduced BMI, non-cascading changes can be more powerful. For example,
   $ md5sum B.pcm
   6c2bd452ca32ab418bf35cd141b060b9  B.pcm
 
-And let's change the implementation for ``A.cppm`` into:
+And let's change the implementation for ``A.cppm`` to:
 
 .. code-block:: c++
 
@@ -830,7 +830,7 @@ and recompile the example:
   $ md5sum B.pcm
   6c2bd452ca32ab418bf35cd141b060b9  B.pcm
 
-We should find the contents of ``B.pcm`` remains the same. In this case, the build system is
+We should find the contents of ``B.pcm`` remain the same. In this case, the build system is
 allowed to skip recompilations of TUs which solely and directly depend on module ``B``.
 
 This only happens with a reduced BMI. With reduced BMIs, we won't record the function body
@@ -845,7 +845,7 @@ Reduce duplications
 
 While it is valid to have duplicated declarations in the global module fragments
 of different module units, it is not free for Clang to deal with the duplicated
-declarations. A translation unit will compile more slowly if there is a lot of
+declarations. A translation unit will compile more slowly if there are a lot of
 duplicated declarations between the translation unit and modules it imports.
 For example:
 
@@ -937,7 +937,7 @@ possible. However, it may be a breaking change for existing code or libraries
 to switch to modules. As a result, many existing libraries need to provide
 both headers and module interfaces for a while to not break existing users.
 
-This section suggests some suggestions on how to ease the transition process
+This section provides some suggestions on how to ease the transition process
 for existing libraries. **Note that this information is only intended as
 guidance, rather than as requirements to use modules in Clang.** It presumes
 the project is starting with no module-based dependencies.
@@ -1140,7 +1140,7 @@ module unit which is internal to the module itself.
 Providing a header to skip parsing redundant headers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Many redeclarations shared between translation units causes Clang to have
+Many redeclarations shared between translation units cause Clang to have
 slower compile-time performance. Further, there are known issues with
 `include after import <https://github.com/llvm/llvm-project/issues/61465>`_.
 Even when that issue is resolved, users may still get slower compilation speed
@@ -1408,8 +1408,8 @@ P1857R3 is implemented. This is tracked by
 Until then, it is recommended not to mix macros with module declarations.
 
 
-In consistent filename suffix requirement for importable module units
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Inconsistent filename suffix requirement for importable module units
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Currently, Clang requires the file name of an ``importable module unit`` to
 have ``.cppm`` (or ``.ccm``, ``.cxxm``, ``.c++m``) as the file extension.
@@ -1484,7 +1484,7 @@ How to build projects using header units
 .. warning::
 
    The support for header units, including related command line options, is
-   experimental. There are still many unanswered question about how tools
+   experimental. There are still many unanswered questions about how tools
    should interact with header units. The details described here may change in
    the future.
 
@@ -1881,7 +1881,7 @@ Individual command line options can be specified after ``--``.
 options. Note that the path to the compiler executable needs to be specified
 explicitly instead of using ``clang++`` directly.
 
-Users may want the scanner to get the transitional dependency information for
+Users may want the scanner to get the transitive dependency information for
 headers. Otherwise, the project has to be scanned twice, once for headers and
 once for modules. To address this, ``clang-scan-deps`` will recognize the
 specified preprocessor options in the given command line and generate the
@@ -1912,7 +1912,7 @@ Possible Issues: Failed to find system headers
 
 If encountering an error like ``fatal error: 'stddef.h' file not found``,
 the specified ``<path-to-compiler-executable>/clang++`` probably refers to a
-symlink instead a real binary. There are four potential solutions to the
+symlink instead of a real binary. There are four potential solutions to the
 problem:
 
 1. Point the specified compiler executable to the real binary instead of the
diff --git a/clang/docs/ThreadSafetyAnalysis.rst b/clang/docs/ThreadSafetyAnalysis.rst
index 4fc7ff28e9931..853a8fae4a907 100644
--- a/clang/docs/ThreadSafetyAnalysis.rst
+++ b/clang/docs/ThreadSafetyAnalysis.rst
@@ -825,13 +825,6 @@ doesn't know that munl.mu == mutex.  The SCOPED_CAPABILITY attribute handles
 aliasing for MutexLocker, but does so only for that particular pattern.
 
 
-ACQUIRED_BEFORE(...) and ACQUIRED_AFTER(...) support is still experimental.
----------------------------------------------------------------------------
-
-ACQUIRED_BEFORE(...) and ACQUIRED_AFTER(...) are currently being developed under
-the ``-Wthread-safety-beta`` flag.
-
-
 .. _mutexheader:
 
 mutex.h
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index c7039290ec6d5..0e85c8109fd5e 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -2635,7 +2635,7 @@ violates the strict aliasing rules. For example:
 
 Strict aliasing can be explicitly enabled with ``-fstrict-aliasing`` and
 disabled with ``-fno-strict-aliasing``. ``clang-cl`` defaults to
-``-fno-strict-aliasing``; see . Otherwise, Clang defaults to ``-fstrict-aliasing``.
+``-fno-strict-aliasing``. Otherwise, Clang defaults to ``-fstrict-aliasing``.
 
 C and C++ specify slightly different rules for strict aliasing. To improve
 language interoperability, Clang allows two types to alias if either language
diff --git a/clang/include/clang/APINotes/Types.h b/clang/include/clang/APINotes/Types.h
index 0f2e49613b514..71625715bda19 100644
--- a/clang/include/clang/APINotes/Types.h
+++ b/clang/include/clang/APINotes/Types.h
@@ -141,6 +141,9 @@ class CommonTypeInfo : public CommonEntityInfo {
   /// The NS error domain for this type.
   std::optional<std::string> NSErrorDomain;
 
+  /// The Swift protocol that this type should be automatically conformed to.
+  std::optional<std::string> SwiftConformance;
+
 public:
   CommonTypeInfo() {}
 
@@ -165,6 +168,14 @@ class CommonTypeInfo : public CommonEntityInfo {
                            : std::nullopt;
   }
 
+  std::optional<std::string> getSwiftConformance() const {
+    return SwiftConformance;
+  }
+
+  void setSwiftConformance(std::optional<std::string> conformance) {
+    SwiftConformance = conformance;
+  }
+
   friend bool operator==(const CommonTypeInfo &, const CommonTypeInfo &);
 
   CommonTypeInfo &operator|=(const CommonTypeInfo &RHS) {
@@ -175,6 +186,8 @@ class CommonTypeInfo : public CommonEntityInfo {
       setSwiftBridge(RHS.getSwiftBridge());
     if (!NSErrorDomain)
       setNSErrorDomain(RHS.getNSErrorDomain());
+    if (SwiftConformance)
+      setSwiftConformance(RHS.getSwiftConformance());
 
     return *this;
   }
@@ -185,7 +198,8 @@ class CommonTypeInfo : public CommonEntityInfo {
 inline bool operator==(const CommonTypeInfo &LHS, const CommonTypeInfo &RHS) {
   return static_cast<const CommonEntityInfo &>(LHS) == RHS &&
          LHS.SwiftBridge == RHS.SwiftBridge &&
-         LHS.NSErrorDomain == RHS.NSErrorDomain;
+         LHS.NSErrorDomain == RHS.NSErrorDomain &&
+         LHS.SwiftConformance == RHS.SwiftConformance;
 }
 
 inline bool operator!=(const CommonTypeInfo &LHS, const CommonTypeInfo &RHS) {
@@ -737,11 +751,9 @@ class TagInfo : public CommonTypeInfo {
   std::optional<std::string> SwiftImportAs;
   std::optional<std::string> SwiftRetainOp;
   std::optional<std::string> SwiftReleaseOp;
+  std::optional<std::string> SwiftDestroyOp;
   std::optional<std::string> SwiftDefaultOwnership;
 
-  /// The Swift protocol that this type should be automatically conformed to.
-  std::optional<std::string> SwiftConformance;
-
   std::optional<EnumExtensibilityKind> EnumExtensibility;
 
   TagInfo()
@@ -787,12 +799,11 @@ class TagInfo : public CommonTypeInfo {
       SwiftRetainOp = RHS.SwiftRetainOp;
     if (!SwiftReleaseOp)
       SwiftReleaseOp = RHS.SwiftReleaseOp;
+    if (!SwiftDestroyOp)
+      SwiftDestroyOp = RHS.SwiftDestroyOp;
     if (!SwiftDefaultOwnership)
       SwiftDefaultOwnership = RHS.SwiftDefaultOwnership;
 
-    if (!SwiftConformance)
-      SwiftConformance = RHS.SwiftConformance;
-
     if (!HasFlagEnum)
       setFlagEnum(RHS.isFlagEnum());
 
@@ -818,8 +829,8 @@ inline bool operator==(const TagInfo &LHS, const TagInfo &RHS) {
          LHS.SwiftImportAs == RHS.SwiftImportAs &&
          LHS.SwiftRetainOp == RHS.SwiftRetainOp &&
          LHS.SwiftReleaseOp == RHS.SwiftReleaseOp &&
+         LHS.SwiftDestroyOp == RHS.SwiftDestroyOp &&
          LHS.SwiftDefaultOwnership == RHS.SwiftDefaultOwnership &&
-         LHS.SwiftConformance == RHS.SwiftConformance &&
          LHS.isFlagEnum() == RHS.isFlagEnum() &&
          LHS.isSwiftCopyable() == RHS.isSwiftCopyable() &&
          LHS.isSwiftEscapable() == RHS.isSwiftEscapable() &&
diff --git a/clang/include/clang/AST/APValue.h b/clang/include/clang/AST/APValue.h
index 9999a30c51ade..cb942ea865e2d 100644
--- a/clang/include/clang/AST/APValue.h
+++ b/clang/include/clang/AST/APValue.h
@@ -143,7 +143,7 @@ class APValue {
     AddrLabelDiff
   };
 
-  class LValueBase {
+  class alignas(uint64_t) LValueBase {
     typedef llvm::PointerUnion<const ValueDecl *, const Expr *, TypeInfoLValue,
                                DynamicAllocLValue>
         PtrTy;
diff --git a/clang/include/clang/AST/ASTConcept.h b/clang/include/clang/AST/ASTConcept.h
index 7ccac4481b14c..72da0059744f2 100644
--- a/clang/include/clang/AST/ASTConcept.h
+++ b/clang/include/clang/AST/ASTConcept.h
@@ -15,7 +15,7 @@
 #define LLVM_CLANG_AST_ASTCONCEPT_H
 
 #include "clang/AST/DeclarationName.h"
-#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/NestedNameSpecifierBase.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Basic/UnsignedOrNone.h"
@@ -177,12 +177,7 @@ class ConceptReference {
 
   SourceLocation getLocation() const { return getConceptNameLoc(); }
 
-  SourceLocation getBeginLoc() const LLVM_READONLY {
-    // Note that if the qualifier is null the template KW must also be null.
-    if (auto QualifierLoc = getNestedNameSpecifierLoc())
-      return QualifierLoc.getBeginLoc();
-    return getConceptNameInfo().getBeginLoc();
-  }
+  SourceLocation getBeginLoc() const LLVM_READONLY;
 
   SourceLocation getEndLoc() const LLVM_READONLY {
     return getTemplateArgsAsWritten() &&
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 2b7ba41e870ec..07f4935e06555 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -230,13 +230,16 @@ class ASTContext : public RefCountedBase<ASTContext> {
     SubstTemplateTypeParmTypes;
   mutable llvm::FoldingSet<SubstTemplateTypeParmPackType>
     SubstTemplateTypeParmPackTypes;
+  mutable llvm::FoldingSet<SubstBuiltinTemplatePackType>
+      SubstBuiltinTemplatePackTypes;
   mutable llvm::ContextualFoldingSet<TemplateSpecializationType, ASTContext&>
     TemplateSpecializationTypes;
   mutable llvm::FoldingSet<ParenType> ParenTypes{GeneralTypesLog2InitSize};
+  mutable llvm::FoldingSet<TagTypeFoldingSetPlaceholder> TagTypes;
+  mutable llvm::FoldingSet<FoldingSetPlaceholder<UnresolvedUsingType>>
+      UnresolvedUsingTypes;
   mutable llvm::FoldingSet<UsingType> UsingTypes;
-  mutable llvm::FoldingSet<TypedefType> TypedefTypes;
-  mutable llvm::FoldingSet<ElaboratedType> ElaboratedTypes{
-      GeneralTypesLog2InitSize};
+  mutable llvm::FoldingSet<FoldingSetPlaceholder<TypedefType>> TypedefTypes;
   mutable llvm::FoldingSet<DependentNameType> DependentNameTypes;
   mutable llvm::DenseMap<llvm::FoldingSetNodeID,
                          DependentTemplateSpecializationType *>
@@ -282,11 +285,11 @@ class ASTContext : public RefCountedBase<ASTContext> {
                      llvm::to_underlying(PredefinedSugarType::Kind::Last) + 1>
       PredefinedSugarTypes{};
 
-  /// The set of nested name specifiers.
+  /// Internal storage for NestedNameSpecifiers.
   ///
   /// This set is managed by the NestedNameSpecifier class.
-  mutable llvm::FoldingSet<NestedNameSpecifier> NestedNameSpecifiers;
-  mutable NestedNameSpecifier *GlobalNestedNameSpecifier = nullptr;
+  mutable llvm::FoldingSet<NamespaceAndPrefixStorage>
+      NamespaceAndPrefixStorages;
 
   /// A cache mapping from RecordDecls to ASTRecordLayouts.
   ///
@@ -1386,8 +1389,6 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// Return a type with extended qualifiers.
   QualType getExtQualType(const Type *Base, Qualifiers Quals) const;
 
-  QualType getTypeDeclTypeSlow(const TypeDecl *Decl) const;
-
   QualType getPipeType(QualType T, bool ReadOnly) const;
 
 public:
@@ -1630,7 +1631,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
 
   /// Return the uniqued reference to the type for a member pointer to
   /// the specified type in the specified nested name.
-  QualType getMemberPointerType(QualType T, NestedNameSpecifier *Qualifier,
+  QualType getMemberPointerType(QualType T, NestedNameSpecifier Qualifier,
                                 const CXXRecordDecl *Cls) const;
 
   /// Return a non-unique reference to the type for a variable array of
@@ -1767,34 +1768,53 @@ class ASTContext : public RefCountedBase<ASTContext> {
                       bool IsCanon = false) const;
 
 public:
+  QualType getTypeDeclType(ElaboratedTypeKeyword Keyword,
+                           NestedNameSpecifier Qualifier,
+                           const TypeDecl *Decl) const;
+
   /// Return the unique reference to the type for the specified type
   /// declaration.
-  QualType getTypeDeclType(const TypeDecl *Decl,
-                           const TypeDecl *PrevDecl = nullptr) const {
-    assert(Decl && "Passed null for Decl param");
-    if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
-
-    if (PrevDecl) {
-      assert(PrevDecl->TypeForDecl && "previous decl has no TypeForDecl");
-      Decl->TypeForDecl = PrevDecl->TypeForDecl;
-      return QualType(PrevDecl->TypeForDecl, 0);
-    }
+  QualType getTypeDeclType(const TypeDecl *Decl) const;
 
-    return getTypeDeclTypeSlow(Decl);
-  }
+  /// Use the normal 'getFooBarType' constructors to obtain these types.
+  QualType getTypeDeclType(const TagDecl *) const = delete;
+  QualType getTypeDeclType(const TypedefDecl *) const = delete;
+  QualType getTypeDeclType(const TypeAliasDecl *) const = delete;
+  QualType getTypeDeclType(const UnresolvedUsingTypenameDecl *) const = delete;
+
+  CanQualType getCanonicalTypeDeclType(const TypeDecl *TD) const;
 
-  QualType getUsingType(const UsingShadowDecl *Found,
-                        QualType Underlying) const;
+  QualType getUsingType(ElaboratedTypeKeyword Keyword,
+                        NestedNameSpecifier Qualifier, const UsingShadowDecl *D,
+                        QualType UnderlyingType = QualType()) const;
 
   /// Return the unique reference to the type for the specified
   /// typedef-name decl.
-  QualType getTypedefType(const TypedefNameDecl *Decl,
-                          QualType Underlying = QualType()) const;
+  /// FIXME: TypeMatchesDeclOrNone is a workaround for a serialization issue:
+  /// The decl underlying type might still not be available.
+  QualType getTypedefType(
+      ElaboratedTypeKeyword Keyword, NestedNameSpecifier Qualifier,
+      const TypedefNameDecl *Decl, QualType UnderlyingType = QualType(),
+      std::optional<bool> TypeMatchesDeclOrNone = std::nullopt) const;
+
+  CanQualType getCanonicalTagType(const TagDecl *TD) const;
+  QualType getTagType(ElaboratedTypeKeyword Keyword,
+                      NestedNameSpecifier Qualifier, const TagDecl *TD,
+                      bool OwnsTag) const;
 
-  QualType getRecordType(const RecordDecl *Decl) const;
+private:
+  UnresolvedUsingType *getUnresolvedUsingTypeInternal(
+      ElaboratedTypeKeyword Keyword, NestedNameSpecifier Qualifier,
+      const UnresolvedUsingTypenameDecl *D, void *InsertPos,
+      const Type *CanonicalType) const;
 
-  QualType getEnumType(const EnumDecl *Decl) const;
+  TagType *getTagTypeInternal(ElaboratedTypeKeyword Keyword,
+                              NestedNameSpecifier Qualifier, const TagDecl *Tag,
+                              bool OwnsTag, bool IsInjected,
+                              const Type *CanonicalType,
+                              bool WithFoldingSetNode) const;
 
+public:
   /// Compute BestType and BestPromotionType for an enum based on the highest
   /// number of negative and positive bits of its elements.
   /// Returns true if enum width is too large.
@@ -1843,10 +1863,11 @@ class ASTContext : public RefCountedBase<ASTContext> {
     return MembersRepresentableByInt;
   }
 
-  QualType
-  getUnresolvedUsingType(const UnresolvedUsingTypenameDecl *Decl) const;
-
-  QualType getInjectedClassNameType(CXXRecordDecl *Decl, QualType TST) const;
+  CanQualType
+  getCanonicalUnresolvedUsingType(const UnresolvedUsingTypenameDecl *D) const;
+  QualType getUnresolvedUsingType(ElaboratedTypeKeyword Keyword,
+                                  NestedNameSpecifier Qualifier,
+                                  const UnresolvedUsingTypenameDecl *D) const;
 
   QualType getAttributedType(attr::Kind attrKind, QualType modifiedType,
                              QualType equivalentType,
@@ -1876,6 +1897,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
   QualType getSubstTemplateTypeParmPackType(Decl *AssociatedDecl,
                                             unsigned Index, bool Final,
                                             const TemplateArgument &ArgPack);
+  QualType getSubstBuiltinTemplatePack(const TemplateArgument &ArgPack);
 
   QualType
   getTemplateTypeParmType(unsigned Depth, unsigned Index,
@@ -1886,18 +1908,20 @@ class ASTContext : public RefCountedBase<ASTContext> {
       TemplateName T, ArrayRef<TemplateArgument> CanonicalArgs) const;
 
   QualType
-  getTemplateSpecializationType(TemplateName T,
+  getTemplateSpecializationType(ElaboratedTypeKeyword Keyword, TemplateName T,
                                 ArrayRef<TemplateArgument> SpecifiedArgs,
                                 ArrayRef<TemplateArgument> CanonicalArgs,
                                 QualType Underlying = QualType()) const;
 
   QualType
-  getTemplateSpecializationType(TemplateName T,
+  getTemplateSpecializationType(ElaboratedTypeKeyword Keyword, TemplateName T,
                                 ArrayRef<TemplateArgumentLoc> SpecifiedArgs,
                                 ArrayRef<TemplateArgument> CanonicalArgs,
                                 QualType Canon = QualType()) const;
 
   TypeSourceInfo *getTemplateSpecializationTypeInfo(
+      ElaboratedTypeKeyword Keyword, SourceLocation ElaboratedKeywordLoc,
+      NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKeywordLoc,
       TemplateName T, SourceLocation TLoc,
       const TemplateArgumentListInfo &SpecifiedArgs,
       ArrayRef<TemplateArgument> CanonicalArgs,
@@ -1908,11 +1932,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   QualType getMacroQualifiedType(QualType UnderlyingTy,
                                  const IdentifierInfo *MacroII) const;
 
-  QualType getElaboratedType(ElaboratedTypeKeyword Keyword,
-                             NestedNameSpecifier *NNS, QualType NamedType,
-                             TagDecl *OwnedTagDecl = nullptr) const;
   QualType getDependentNameType(ElaboratedTypeKeyword Keyword,
-                                NestedNameSpecifier *NNS,
+                                NestedNameSpecifier NNS,
                                 const IdentifierInfo *Name) const;
 
   QualType getDependentTemplateSpecializationType(
@@ -1999,21 +2020,17 @@ class ASTContext : public RefCountedBase<ASTContext> {
   QualType getUnconstrainedType(QualType T) const;
 
   /// C++17 deduced class template specialization type.
-  QualType getDeducedTemplateSpecializationType(TemplateName Template,
+  QualType getDeducedTemplateSpecializationType(ElaboratedTypeKeyword Keyword,
+                                                TemplateName Template,
                                                 QualType DeducedType,
                                                 bool IsDependent) const;
 
 private:
-  QualType getDeducedTemplateSpecializationTypeInternal(TemplateName Template,
-                                                        QualType DeducedType,
-                                                        bool IsDependent,
-                                                        QualType Canon) const;
+  QualType getDeducedTemplateSpecializationTypeInternal(
+      ElaboratedTypeKeyword Keyword, TemplateName Template,
+      QualType DeducedType, bool IsDependent, QualType Canon) const;
 
 public:
-  /// Return the unique reference to the type for the specified TagDecl
-  /// (struct/union/class/enum) decl.
-  QualType getTagDeclType(const TagDecl *Decl) const;
-
   /// Return the unique type for "size_t" (C99 7.17), defined in
   /// <stddef.h>.
   ///
@@ -2089,7 +2106,9 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// if it hasn't yet been built.
   QualType getRawCFConstantStringType() const {
     if (CFConstantStringTypeDecl)
-      return getTypedefType(CFConstantStringTypeDecl);
+      return getTypedefType(ElaboratedTypeKeyword::None,
+                            /*Qualifier=*/std::nullopt,
+                            CFConstantStringTypeDecl);
     return QualType();
   }
   void setCFConstantStringType(QualType T);
@@ -2186,10 +2205,11 @@ class ASTContext : public RefCountedBase<ASTContext> {
   }
 #include "clang/Basic/BuiltinTemplates.inc"
 
-  /// Retrieve the Objective-C "instancetype" type, if already known;
-  /// otherwise, returns a NULL type;
+  /// Retrieve the Objective-C "instancetype" type.
   QualType getObjCInstanceType() {
-    return getTypeDeclType(getObjCInstanceTypeDecl());
+    return getTypedefType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt,
+                          getObjCInstanceTypeDecl());
   }
 
   /// Retrieve the typedef declaration corresponding to the Objective-C
@@ -2202,7 +2222,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// Retrieve the C FILE type.
   QualType getFILEType() const {
     if (FILEDecl)
-      return getTypeDeclType(FILEDecl);
+      return getTypeDeclType(ElaboratedTypeKeyword::None,
+                             /*Qualifier=*/std::nullopt, FILEDecl);
     return QualType();
   }
 
@@ -2214,7 +2235,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// Retrieve the C jmp_buf type.
   QualType getjmp_bufType() const {
     if (jmp_bufDecl)
-      return getTypeDeclType(jmp_bufDecl);
+      return getTypeDeclType(ElaboratedTypeKeyword::None,
+                             /*Qualifier=*/std::nullopt, jmp_bufDecl);
     return QualType();
   }
 
@@ -2226,7 +2248,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// Retrieve the C sigjmp_buf type.
   QualType getsigjmp_bufType() const {
     if (sigjmp_bufDecl)
-      return getTypeDeclType(sigjmp_bufDecl);
+      return getTypeDeclType(ElaboratedTypeKeyword::None,
+                             /*Qualifier=*/std::nullopt, sigjmp_bufDecl);
     return QualType();
   }
 
@@ -2238,12 +2261,13 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// Retrieve the C ucontext_t type.
   QualType getucontext_tType() const {
     if (ucontext_tDecl)
-      return getTypeDeclType(ucontext_tDecl);
+      return getTypeDeclType(ElaboratedTypeKeyword::None,
+                             /*Qualifier=*/std::nullopt, ucontext_tDecl);
     return QualType();
   }
 
   /// The result type of logical operations, '<', '>', '!=', etc.
-  QualType getLogicalOperationType() const {
+  CanQualType getLogicalOperationType() const {
     return getLangOpts().CPlusPlus ? BoolTy : IntTy;
   }
 
@@ -2308,7 +2332,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// This is set up lazily, by Sema.  \c id is always a (typedef for a)
   /// pointer type, a pointer to a struct.
   QualType getObjCIdType() const {
-    return getTypeDeclType(getObjCIdDecl());
+    return getTypedefType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt, getObjCIdDecl());
   }
 
   /// Retrieve the typedef corresponding to the predefined 'SEL' type
@@ -2318,7 +2343,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// Retrieve the type that corresponds to the predefined Objective-C
   /// 'SEL' type.
   QualType getObjCSelType() const {
-    return getTypeDeclType(getObjCSelDecl());
+    return getTypedefType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt, getObjCSelDecl());
   }
 
   PointerAuthQualifier getObjCMemberSelTypePtrAuth();
@@ -2332,7 +2358,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// This is set up lazily, by Sema.  \c Class is always a (typedef for a)
   /// pointer type, a pointer to a struct.
   QualType getObjCClassType() const {
-    return getTypeDeclType(getObjCClassDecl());
+    return getTypedefType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt, getObjCClassDecl());
   }
 
   /// Retrieve the Objective-C class declaration corresponding to
@@ -2351,7 +2378,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
 
   /// type of 'BOOL' type.
   QualType getBOOLType() const {
-    return getTypeDeclType(getBOOLDecl());
+    return getTypedefType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt, getBOOLDecl());
   }
 
   /// Retrieve the type of the Objective-C \c Protocol class.
@@ -2365,7 +2393,8 @@ class ASTContext : public RefCountedBase<ASTContext> {
 
   /// Retrieve the type of the \c __builtin_va_list type.
   QualType getBuiltinVaListType() const {
-    return getTypeDeclType(getBuiltinVaListDecl());
+    return getTypedefType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt, getBuiltinVaListDecl());
   }
 
   /// Retrieve the C type declaration corresponding to the predefined
@@ -2379,16 +2408,17 @@ class ASTContext : public RefCountedBase<ASTContext> {
 
   /// Retrieve the type of the \c __builtin_ms_va_list type.
   QualType getBuiltinMSVaListType() const {
-    return getTypeDeclType(getBuiltinMSVaListDecl());
+    return getTypedefType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt, getBuiltinMSVaListDecl());
   }
 
   /// Retrieve the implicitly-predeclared 'struct _GUID' declaration.
   TagDecl *getMSGuidTagDecl() const { return MSGuidTagDecl; }
 
   /// Retrieve the implicitly-predeclared 'struct _GUID' type.
-  QualType getMSGuidType() const {
+  CanQualType getMSGuidType() const {
     assert(MSGuidTagDecl && "asked for GUID type but MS extensions disabled");
-    return getTagDeclType(MSGuidTagDecl);
+    return getCanonicalTagType(MSGuidTagDecl);
   }
 
   /// Retrieve the implicitly-predeclared 'struct type_info' declaration.
@@ -2477,7 +2507,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
                                          UnresolvedSetIterator End) const;
   TemplateName getAssumedTemplateName(DeclarationName Name) const;
 
-  TemplateName getQualifiedTemplateName(NestedNameSpecifier *NNS,
+  TemplateName getQualifiedTemplateName(NestedNameSpecifier Qualifier,
                                         bool TemplateKeyword,
                                         TemplateName Template) const;
   TemplateName
@@ -2919,32 +2949,6 @@ class ASTContext : public RefCountedBase<ASTContext> {
   /// Determine if two types are similar, ignoring only CVR qualifiers.
   bool hasCvrSimilarType(QualType T1, QualType T2);
 
-  /// Retrieves the "canonical" nested name specifier for a
-  /// given nested name specifier.
-  ///
-  /// The canonical nested name specifier is a nested name specifier
-  /// that uniquely identifies a type or namespace within the type
-  /// system. For example, given:
-  ///
-  /// \code
-  /// namespace N {
-  ///   struct S {
-  ///     template<typename T> struct X { typename T* type; };
-  ///   };
-  /// }
-  ///
-  /// template<typename T> struct Y {
-  ///   typename N::S::X<T>::type member;
-  /// };
-  /// \endcode
-  ///
-  /// Here, the nested-name-specifier for N::S::X<T>:: will be
-  /// S::X<template-param-0-0>, since 'S' and 'X' are uniquely defined
-  /// by declarations in the type system and the canonical type for
-  /// the template type parameter 'T' is template-param-0-0.
-  NestedNameSpecifier *
-  getCanonicalNestedNameSpecifier(NestedNameSpecifier *NNS) const;
-
   /// Retrieves the default calling convention for the current context.
   ///
   /// The context's default calling convention may differ from the current
@@ -3158,7 +3162,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
   mergeExceptionSpecs(FunctionProtoType::ExceptionSpecInfo ESI1,
                       FunctionProtoType::ExceptionSpecInfo ESI2,
                       SmallVectorImpl<QualType> &ExceptionTypeStorage,
-                      bool AcceptDependent);
+                      bool AcceptDependent) const;
 
   // For two "same" types, return a type which has
   // the common sugar between them. If Unqualified is true,
@@ -3166,7 +3170,7 @@ class ASTContext : public RefCountedBase<ASTContext> {
   // The result will drop the qualifiers which do not occur
   // in both types.
   QualType getCommonSugaredType(QualType X, QualType Y,
-                                bool Unqualified = false);
+                                bool Unqualified = false) const;
 
 private:
   // Helper for integer ordering
@@ -3184,23 +3188,11 @@ class ASTContext : public RefCountedBase<ASTContext> {
   bool propertyTypesAreCompatible(QualType, QualType);
   bool typesAreBlockPointerCompatible(QualType, QualType);
 
-  bool isObjCIdType(QualType T) const {
-    if (const auto *ET = dyn_cast<ElaboratedType>(T))
-      T = ET->getNamedType();
-    return T == getObjCIdType();
-  }
+  bool isObjCIdType(QualType T) const { return T == getObjCIdType(); }
 
-  bool isObjCClassType(QualType T) const {
-    if (const auto *ET = dyn_cast<ElaboratedType>(T))
-      T = ET->getNamedType();
-    return T == getObjCClassType();
-  }
+  bool isObjCClassType(QualType T) const { return T == getObjCClassType(); }
 
-  bool isObjCSelType(QualType T) const {
-    if (const auto *ET = dyn_cast<ElaboratedType>(T))
-      T = ET->getNamedType();
-    return T == getObjCSelType();
-  }
+  bool isObjCSelType(QualType T) const { return T == getObjCSelType(); }
 
   bool ObjCQualifiedIdTypesAreCompatible(const ObjCObjectPointerType *LHS,
                                          const ObjCObjectPointerType *RHS,
diff --git a/clang/include/clang/AST/ASTImporter.h b/clang/include/clang/AST/ASTImporter.h
index c40b92666a2ff..4a0ca45b785a9 100644
--- a/clang/include/clang/AST/ASTImporter.h
+++ b/clang/include/clang/AST/ASTImporter.h
@@ -404,7 +404,7 @@ class TypeSourceInfo;
     ///
     /// \returns The equivalent nested-name-specifier in the "to"
     /// context, or the import error.
-    llvm::Expected<NestedNameSpecifier *> Import(NestedNameSpecifier *FromNNS);
+    llvm::Expected<NestedNameSpecifier> Import(NestedNameSpecifier FromNNS);
 
     /// Import the given nested-name-specifier-loc from the "from"
     /// context into the "to" context.
diff --git a/clang/include/clang/AST/ASTNodeTraverser.h b/clang/include/clang/AST/ASTNodeTraverser.h
index 8ebabb2bde10d..d9dc8290b0e49 100644
--- a/clang/include/clang/AST/ASTNodeTraverser.h
+++ b/clang/include/clang/AST/ASTNodeTraverser.h
@@ -394,12 +394,14 @@ class ASTNodeTraverser
   }
   void VisitMemberPointerType(const MemberPointerType *T) {
     // FIXME: Provide a NestedNameSpecifier visitor.
-    NestedNameSpecifier *Qualifier = T->getQualifier();
-    if (NestedNameSpecifier::SpecifierKind K = Qualifier->getKind();
-        K == NestedNameSpecifier::TypeSpec)
-      Visit(Qualifier->getAsType());
+    NestedNameSpecifier Qualifier = T->getQualifier();
+    if (NestedNameSpecifier::Kind K = Qualifier.getKind();
+        K == NestedNameSpecifier::Kind::Type)
+      Visit(Qualifier.getAsType());
     if (T->isSugared())
-      Visit(T->getMostRecentCXXRecordDecl()->getTypeForDecl());
+      Visit(cast<MemberPointerType>(T->getCanonicalTypeUnqualified())
+                ->getQualifier()
+                .getAsType());
     Visit(T->getPointeeType());
   }
   void VisitArrayType(const ArrayType *T) { Visit(T->getElementType()); }
@@ -510,7 +512,7 @@ class ASTNodeTraverser
   }
   void VisitMemberPointerTypeLoc(MemberPointerTypeLoc TL) {
     // FIXME: Provide NestedNamespecifierLoc visitor.
-    Visit(TL.getQualifierLoc().getTypeLoc());
+    Visit(TL.getQualifierLoc().castAsTypeLoc());
   }
   void VisitVariableArrayTypeLoc(VariableArrayTypeLoc TL) {
     Visit(TL.getSizeExpr());
@@ -772,17 +774,16 @@ class ASTNodeTraverser
   }
 
   void VisitUsingShadowDecl(const UsingShadowDecl *D) {
-    if (auto *TD = dyn_cast<TypeDecl>(D->getUnderlyingDecl()))
-      Visit(TD->getTypeForDecl());
+    Visit(D->getTargetDecl());
   }
 
   void VisitFriendDecl(const FriendDecl *D) {
     if (D->getFriendType()) {
       // Traverse any CXXRecordDecl owned by this type, since
       // it will not be in the parent context:
-      if (auto *ET = D->getFriendType()->getType()->getAs<ElaboratedType>())
-        if (auto *TD = ET->getOwnedTagDecl())
-          Visit(TD);
+      if (auto *TT = D->getFriendType()->getType()->getAs<TagType>())
+        if (TT->isTagOwned())
+          Visit(TT->getOriginalDecl());
     } else {
       Visit(D->getFriendDecl());
     }
diff --git a/clang/include/clang/AST/ASTTypeTraits.h b/clang/include/clang/AST/ASTTypeTraits.h
index 3988a15971db5..6f40705fb7436 100644
--- a/clang/include/clang/AST/ASTTypeTraits.h
+++ b/clang/include/clang/AST/ASTTypeTraits.h
@@ -307,7 +307,7 @@ class DynTypedNode {
 
   /// For nodes which represent textual entities in the source code,
   /// return their SourceRange.  For all other nodes, return SourceRange().
-  SourceRange getSourceRange() const;
+  SourceRange getSourceRange(bool IncludeQualifier = false) const;
 
   /// @{
   /// Imposes an order on \c DynTypedNode.
@@ -336,9 +336,9 @@ class DynTypedNode {
             NodeKind)) {
       auto NNSLA = getUnchecked<NestedNameSpecifierLoc>();
       auto NNSLB = Other.getUnchecked<NestedNameSpecifierLoc>();
-      return std::make_pair(NNSLA.getNestedNameSpecifier(),
+      return std::make_pair(NNSLA.getNestedNameSpecifier().getAsVoidPointer(),
                             NNSLA.getOpaqueData()) <
-             std::make_pair(NNSLB.getNestedNameSpecifier(),
+             std::make_pair(NNSLB.getNestedNameSpecifier().getAsVoidPointer(),
                             NNSLB.getOpaqueData());
     }
 
@@ -393,8 +393,9 @@ class DynTypedNode {
       if (ASTNodeKind::getFromNodeKind<NestedNameSpecifierLoc>().isSame(
               Val.NodeKind)) {
         auto NNSL = Val.getUnchecked<NestedNameSpecifierLoc>();
-        return llvm::hash_combine(NNSL.getNestedNameSpecifier(),
-                                  NNSL.getOpaqueData());
+        return llvm::hash_combine(
+            NNSL.getNestedNameSpecifier().getAsVoidPointer(),
+            NNSL.getOpaqueData());
       }
 
       assert(Val.getMemoizationData());
@@ -539,8 +540,8 @@ struct DynTypedNode::BaseConverter<
     : public DynCastPtrConverter<T, Attr> {};
 
 template <>
-struct DynTypedNode::BaseConverter<
-    NestedNameSpecifier, void> : public PtrConverter<NestedNameSpecifier> {};
+struct DynTypedNode::BaseConverter<NestedNameSpecifier, void>
+    : public ValueConverter<NestedNameSpecifier> {};
 
 template <>
 struct DynTypedNode::BaseConverter<
diff --git a/clang/include/clang/AST/AbstractBasicReader.h b/clang/include/clang/AST/AbstractBasicReader.h
index 0a2db9e205c7c..26052b8086cf7 100644
--- a/clang/include/clang/AST/AbstractBasicReader.h
+++ b/clang/include/clang/AST/AbstractBasicReader.h
@@ -197,7 +197,7 @@ class DataStreamBasicReader : public BasicReaderBase<Impl> {
         unsigned int_ = asImpl().readUInt32();
         Decl *decl = asImpl().template readDeclAs<Decl>();
         if (auto *recordDecl = dyn_cast<CXXRecordDecl>(decl))
-          elemTy = getASTContext().getRecordType(recordDecl);
+          elemTy = getASTContext().getCanonicalTagType(recordDecl);
         else
           elemTy = cast<ValueDecl>(decl)->getType();
         path.push_back(
@@ -252,39 +252,34 @@ class DataStreamBasicReader : public BasicReaderBase<Impl> {
     return EffectConditionExpr{asImpl().readExprRef()};
   }
 
-  NestedNameSpecifier *readNestedNameSpecifier() {
+  NestedNameSpecifier readNestedNameSpecifier() {
     auto &ctx = getASTContext();
 
     // We build this up iteratively.
-    NestedNameSpecifier *cur = nullptr;
+    NestedNameSpecifier cur = std::nullopt;
 
     uint32_t depth = asImpl().readUInt32();
     for (uint32_t i = 0; i != depth; ++i) {
       auto kind = asImpl().readNestedNameSpecifierKind();
       switch (kind) {
-      case NestedNameSpecifier::Identifier:
-        cur = NestedNameSpecifier::Create(ctx, cur,
-                                          asImpl().readIdentifier());
+      case NestedNameSpecifier::Kind::Namespace:
+        cur =
+            NestedNameSpecifier(ctx, asImpl().readNamespaceBaseDeclRef(), cur);
         continue;
-
-      case NestedNameSpecifier::Namespace:
-        cur = NestedNameSpecifier::Create(ctx, cur,
-                                          asImpl().readNamespaceBaseDeclRef());
+      case NestedNameSpecifier::Kind::Type:
+        assert(!cur);
+        cur = NestedNameSpecifier(asImpl().readQualType().getTypePtr());
         continue;
-
-      case NestedNameSpecifier::TypeSpec:
-        cur = NestedNameSpecifier::Create(ctx, cur,
-                                          asImpl().readQualType().getTypePtr());
+      case NestedNameSpecifier::Kind::Global:
+        assert(!cur);
+        cur = NestedNameSpecifier::getGlobal();
         continue;
-
-      case NestedNameSpecifier::Global:
-        cur = NestedNameSpecifier::GlobalSpecifier(ctx);
-        continue;
-
-      case NestedNameSpecifier::Super:
-        cur = NestedNameSpecifier::SuperSpecifier(ctx,
-                                            asImpl().readCXXRecordDeclRef());
+      case NestedNameSpecifier::Kind::MicrosoftSuper:
+        assert(!cur);
+        cur = NestedNameSpecifier(asImpl().readCXXRecordDeclRef());
         continue;
+      case NestedNameSpecifier::Kind::Null:
+        llvm_unreachable("unexpected null nested name specifier");
       }
       llvm_unreachable("bad nested name specifier kind");
     }
diff --git a/clang/include/clang/AST/AbstractBasicWriter.h b/clang/include/clang/AST/AbstractBasicWriter.h
index c105bbbe45c92..d41e655986ef9 100644
--- a/clang/include/clang/AST/AbstractBasicWriter.h
+++ b/clang/include/clang/AST/AbstractBasicWriter.h
@@ -181,7 +181,7 @@ class DataStreamBasicWriter : public BasicWriterBase<Impl> {
         const Decl *baseOrMember = elem.getAsBaseOrMember().getPointer();
         if (const auto *recordDecl = dyn_cast<CXXRecordDecl>(baseOrMember)) {
           asImpl().writeDeclRef(recordDecl);
-          elemTy = ctx.getRecordType(recordDecl);
+          elemTy = ctx.getCanonicalTagType(recordDecl);
         } else {
           const auto *valueDecl = cast<ValueDecl>(baseOrMember);
           asImpl().writeDeclRef(valueDecl);
@@ -229,42 +229,43 @@ class DataStreamBasicWriter : public BasicWriterBase<Impl> {
     asImpl().writeExprRef(CE.getCondition());
   }
 
-  void writeNestedNameSpecifier(NestedNameSpecifier *NNS) {
+  void writeNestedNameSpecifier(NestedNameSpecifier NNS) {
     // Nested name specifiers usually aren't too long. I think that 8 would
     // typically accommodate the vast majority.
-    SmallVector<NestedNameSpecifier *, 8> nestedNames;
+    SmallVector<NestedNameSpecifier, 8> nestedNames;
 
     // Push each of the NNS's onto a stack for serialization in reverse order.
     while (NNS) {
       nestedNames.push_back(NNS);
-      NNS = NNS->getPrefix();
+      NNS = NNS.getKind() == NestedNameSpecifier::Kind::Namespace
+                ? NNS.getAsNamespaceAndPrefix().Prefix
+                : std::nullopt;
     }
 
     asImpl().writeUInt32(nestedNames.size());
     while (!nestedNames.empty()) {
       NNS = nestedNames.pop_back_val();
-      NestedNameSpecifier::SpecifierKind kind = NNS->getKind();
+      NestedNameSpecifier::Kind kind = NNS.getKind();
       asImpl().writeNestedNameSpecifierKind(kind);
       switch (kind) {
-      case NestedNameSpecifier::Identifier:
-        asImpl().writeIdentifier(NNS->getAsIdentifier());
+      case NestedNameSpecifier::Kind::Namespace:
+        asImpl().writeNamespaceBaseDeclRef(
+            NNS.getAsNamespaceAndPrefix().Namespace);
         continue;
-
-      case NestedNameSpecifier::Namespace:
-        asImpl().writeNamespaceBaseDeclRef(NNS->getAsNamespace());
-        continue;
-
-      case NestedNameSpecifier::TypeSpec:
-        asImpl().writeQualType(QualType(NNS->getAsType(), 0));
+      case NestedNameSpecifier::Kind::Type:
+        asImpl().writeQualType(QualType(NNS.getAsType(), 0));
         continue;
 
-      case NestedNameSpecifier::Global:
+      case NestedNameSpecifier::Kind::Global:
         // Don't need to write an associated value.
         continue;
 
-      case NestedNameSpecifier::Super:
-        asImpl().writeDeclRef(NNS->getAsRecordDecl());
+      case NestedNameSpecifier::Kind::MicrosoftSuper:
+        asImpl().writeDeclRef(NNS.getAsMicrosoftSuper());
         continue;
+
+      case NestedNameSpecifier::Kind::Null:
+        llvm_unreachable("unexpected null nested name specifier");
       }
       llvm_unreachable("bad nested name specifier kind");
     }
diff --git a/clang/include/clang/AST/CXXInheritance.h b/clang/include/clang/AST/CXXInheritance.h
index bbef01843e0b0..e89326081a180 100644
--- a/clang/include/clang/AST/CXXInheritance.h
+++ b/clang/include/clang/AST/CXXInheritance.h
@@ -359,7 +359,7 @@ class CXXFinalOverriderMap
 
 /// A set of all the primary bases for a class.
 class CXXIndirectPrimaryBaseSet
-  : public llvm::SmallSet<const CXXRecordDecl*, 32> {};
+    : public llvm::SmallPtrSet<const CXXRecordDecl *, 32> {};
 
 inline bool
 inheritanceModelHasVBPtrOffsetField(MSInheritanceModel Inheritance) {
diff --git a/clang/include/clang/AST/CanonicalType.h b/clang/include/clang/AST/CanonicalType.h
index 35db68971e029..b5a4e94e1330a 100644
--- a/clang/include/clang/AST/CanonicalType.h
+++ b/clang/include/clang/AST/CanonicalType.h
@@ -453,7 +453,7 @@ template<>
 struct CanProxyAdaptor<MemberPointerType>
   : public CanProxyBase<MemberPointerType> {
   LLVM_CLANG_CANPROXY_TYPE_ACCESSOR(getPointeeType)
-  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(NestedNameSpecifier *, getQualifier)
+  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(NestedNameSpecifier, getQualifier)
   LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(const CXXRecordDecl *,
                                       getMostRecentCXXRecordDecl)
 };
@@ -551,21 +551,18 @@ struct CanProxyAdaptor<UnaryTransformType>
 
 template<>
 struct CanProxyAdaptor<TagType> : public CanProxyBase<TagType> {
-  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(TagDecl *, getDecl)
-  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, isBeingDefined)
+  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(TagDecl *, getOriginalDecl)
 };
 
 template<>
 struct CanProxyAdaptor<RecordType> : public CanProxyBase<RecordType> {
-  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(RecordDecl *, getDecl)
-  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, isBeingDefined)
+  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(RecordDecl *, getOriginalDecl)
   LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, hasConstFields)
 };
 
 template<>
 struct CanProxyAdaptor<EnumType> : public CanProxyBase<EnumType> {
-  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(EnumDecl *, getDecl)
-  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(bool, isBeingDefined)
+  LLVM_CLANG_CANPROXY_SIMPLE_ACCESSOR(EnumDecl *, getOriginalDecl)
 };
 
 template<>
diff --git a/clang/include/clang/AST/Comment.h b/clang/include/clang/AST/Comment.h
index dd9906727293f..5ba95c8291d38 100644
--- a/clang/include/clang/AST/Comment.h
+++ b/clang/include/clang/AST/Comment.h
@@ -19,6 +19,7 @@
 #include "clang/Basic/SourceLocation.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 
 namespace clang {
 class Decl;
@@ -119,6 +120,11 @@ class Comment {
 
     LLVM_PREFERRED_TYPE(CommandTraits::KnownCommandIDs)
     unsigned CommandID : CommandInfo::NumCommandIDBits;
+
+    /// Describes the syntax that was used in a documentation command.
+    /// Contains values from CommandMarkerKind enum.
+    LLVM_PREFERRED_TYPE(CommandMarkerKind)
+    unsigned CommandMarker : 1;
   };
   enum { NumInlineCommandCommentBits = NumInlineContentCommentBits + 3 +
                                        CommandInfo::NumCommandIDBits };
@@ -347,6 +353,16 @@ class InlineCommandComment : public InlineContentComment {
     InlineCommandCommentBits.RenderKind = llvm::to_underlying(RK);
     InlineCommandCommentBits.CommandID = CommandID;
   }
+  InlineCommandComment(SourceLocation LocBegin, SourceLocation LocEnd,
+                       unsigned CommandID, InlineCommandRenderKind RK,
+                       CommandMarkerKind CommandMarker, ArrayRef<Argument> Args)
+      : InlineContentComment(CommentKind::InlineCommandComment, LocBegin,
+                             LocEnd),
+        Args(Args) {
+    InlineCommandCommentBits.RenderKind = llvm::to_underlying(RK);
+    InlineCommandCommentBits.CommandID = CommandID;
+    InlineCommandCommentBits.CommandMarker = llvm::to_underlying(CommandMarker);
+  }
 
   static bool classof(const Comment *C) {
     return C->getCommentKind() == CommentKind::InlineCommandComment;
@@ -384,6 +400,11 @@ class InlineCommandComment : public InlineContentComment {
   SourceRange getArgRange(unsigned Idx) const {
     return Args[Idx].Range;
   }
+
+  CommandMarkerKind getCommandMarker() const {
+    return static_cast<CommandMarkerKind>(
+        InlineCommandCommentBits.CommandMarker);
+  }
 };
 
 /// Abstract class for opening and closing HTML tags.  HTML tags are always
diff --git a/clang/include/clang/AST/CommentHTMLTags.td b/clang/include/clang/AST/CommentHTMLTags.td
index a1ce8c6da96c0..9b89bc0c811fc 100644
--- a/clang/include/clang/AST/CommentHTMLTags.td
+++ b/clang/include/clang/AST/CommentHTMLTags.td
@@ -51,6 +51,11 @@ def Col     : Tag<"col"> { let EndTagForbidden = 1; }
 def Tr      : Tag<"tr"> { let EndTagOptional = 1; }
 def Th      : Tag<"th"> { let EndTagOptional = 1; }
 def Td      : Tag<"td"> { let EndTagOptional = 1; }
+def Summary : Tag<"summary">;
+def Details : Tag<"details">;
+def Mark    : Tag<"mark">;
+def Figure  : Tag<"figure">;
+def FigCaption : Tag<"figcaption">;
 
 // Define a list of attributes that are not safe to pass through to HTML
 // output if the input is untrusted.
diff --git a/clang/include/clang/AST/CommentSema.h b/clang/include/clang/AST/CommentSema.h
index 916d7945329c5..3169e2b0d86b9 100644
--- a/clang/include/clang/AST/CommentSema.h
+++ b/clang/include/clang/AST/CommentSema.h
@@ -131,6 +131,7 @@ class Sema {
   InlineCommandComment *actOnInlineCommand(SourceLocation CommandLocBegin,
                                            SourceLocation CommandLocEnd,
                                            unsigned CommandID,
+                                           CommandMarkerKind CommandMarker,
                                            ArrayRef<Comment::Argument> Args);
 
   InlineContentComment *actOnUnknownCommand(SourceLocation LocBegin,
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index 08fe1f881503b..bebbde3661a33 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -20,7 +20,7 @@
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclarationName.h"
 #include "clang/AST/ExternalASTSource.h"
-#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/NestedNameSpecifierBase.h"
 #include "clang/AST/Redeclarable.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/AddressSpaces.h"
@@ -833,9 +833,9 @@ class DeclaratorDecl : public ValueDecl {
 
   /// Retrieve the nested-name-specifier that qualifies the name of this
   /// declaration, if it was present in the source.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return hasExtInfo() ? getExtInfo()->QualifierLoc.getNestedNameSpecifier()
-                        : nullptr;
+                        : std::nullopt;
   }
 
   /// Retrieve the nested-name-specifier (with source-location
@@ -3526,10 +3526,16 @@ class TypeDecl : public NamedDecl {
 public:
   // Low-level accessor. If you just want the type defined by this node,
   // check out ASTContext::getTypeDeclType or one of
-  // ASTContext::getTypedefType, ASTContext::getRecordType, etc. if you
+  // ASTContext::getTypedefType, ASTContext::getTagType, etc. if you
   // already know the specific kind of node this is.
-  const Type *getTypeForDecl() const { return TypeForDecl; }
-  void setTypeForDecl(const Type *TD) { TypeForDecl = TD; }
+  const Type *getTypeForDecl() const {
+    assert(!isa<TagDecl>(this));
+    return TypeForDecl;
+  }
+  void setTypeForDecl(const Type *TD) {
+    assert(!isa<TagDecl>(this));
+    TypeForDecl = TD;
+  }
 
   SourceLocation getBeginLoc() const LLVM_READONLY { return LocStart; }
   void setLocStart(SourceLocation L) { LocStart = L; }
@@ -3635,6 +3641,10 @@ class TypedefNameDecl : public TypeDecl, public Redeclarable<TypedefNameDecl> {
     return isTransparentTagSlow();
   }
 
+  // These types are created lazily, use the ASTContext methods to obtain them.
+  const Type *getTypeForDecl() const = delete;
+  void setTypeForDecl(const Type *TD) = delete;
+
   // Implement isa/cast/dyncast/etc.
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) {
@@ -3754,14 +3764,6 @@ class TagDecl : public TypeDecl,
   /// True if this decl is currently being defined.
   void setBeingDefined(bool V = true) { TagDeclBits.IsBeingDefined = V; }
 
-  /// Indicates whether it is possible for declarations of this kind
-  /// to have an out-of-date definition.
-  ///
-  /// This option is only enabled when modules are enabled.
-  void setMayHaveOutOfDateDef(bool V = true) {
-    TagDeclBits.MayHaveOutOfDateDef = V;
-  }
-
 public:
   friend class ASTDeclReader;
   friend class ASTDeclWriter;
@@ -3842,12 +3844,6 @@ class TagDecl : public TypeDecl,
     TagDeclBits.IsFreeStanding = isFreeStanding;
   }
 
-  /// Indicates whether it is possible for declarations of this kind
-  /// to have an out-of-date definition.
-  ///
-  /// This option is only enabled when modules are enabled.
-  bool mayHaveOutOfDateDef() const { return TagDeclBits.MayHaveOutOfDateDef; }
-
   /// Whether this declaration declares a type that is
   /// dependent, i.e., a type that somehow depends on template
   /// parameters.
@@ -3888,6 +3884,19 @@ class TagDecl : public TypeDecl,
   ///  the struct/union/class/enum.
   TagDecl *getDefinition() const;
 
+  TagDecl *getDefinitionOrSelf() const {
+    if (TagDecl *Def = getDefinition())
+      return Def;
+    return const_cast<TagDecl *>(this);
+  }
+
+  /// Determines whether this entity is in the process of being defined.
+  bool isEntityBeingDefined() const {
+    if (const TagDecl *Def = getDefinition())
+      return Def->isBeingDefined();
+    return false;
+  }
+
   StringRef getKindName() const {
     return TypeWithKeyword::getTagTypeKindName(getTagKind());
   }
@@ -3934,9 +3943,9 @@ class TagDecl : public TypeDecl,
 
   /// Retrieve the nested-name-specifier that qualifies the name of this
   /// declaration, if it was present in the source.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return hasExtInfo() ? getExtInfo()->QualifierLoc.getNestedNameSpecifier()
-                        : nullptr;
+                        : std::nullopt;
   }
 
   /// Retrieve the nested-name-specifier (with source-location
@@ -3958,6 +3967,10 @@ class TagDecl : public TypeDecl,
     return getExtInfo()->TemplParamLists[i];
   }
 
+  // These types are created lazily, use the ASTContext methods to obtain them.
+  const Type *getTypeForDecl() const = delete;
+  void setTypeForDecl(const Type *TD) = delete;
+
   using TypeDecl::printName;
   void printName(raw_ostream &OS, const PrintingPolicy &Policy) const override;
 
@@ -4087,6 +4100,10 @@ class EnumDecl : public TagDecl {
     return cast_or_null<EnumDecl>(TagDecl::getDefinition());
   }
 
+  EnumDecl *getDefinitionOrSelf() const {
+    return cast_or_null<EnumDecl>(TagDecl::getDefinitionOrSelf());
+  }
+
   static EnumDecl *Create(ASTContext &C, DeclContext *DC,
                           SourceLocation StartLoc, SourceLocation IdLoc,
                           IdentifierInfo *Id, EnumDecl *PrevDecl,
@@ -4469,6 +4486,10 @@ class RecordDecl : public TagDecl {
     return cast_or_null<RecordDecl>(TagDecl::getDefinition());
   }
 
+  RecordDecl *getDefinitionOrSelf() const {
+    return cast_or_null<RecordDecl>(TagDecl::getDefinitionOrSelf());
+  }
+
   /// Returns whether this record is a union, or contains (at any nesting level)
   /// a union member. This is used by CMSE to warn about possible information
   /// leaks.
@@ -5299,6 +5320,8 @@ void Redeclarable<decl_type>::setPreviousDecl(decl_type *PrevDecl) {
 /// We use this function to break a cycle between the inline definitions in
 /// Type.h and Decl.h.
 inline bool IsEnumDeclComplete(EnumDecl *ED) {
+  if (const auto *Def = ED->getDefinition())
+    return Def->isComplete();
   return ED->isComplete();
 }
 
diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index dd67ebc9873ff..c6326a8ba506d 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -410,9 +410,6 @@ class alignas(8) Decl {
 
   virtual ~Decl();
 
-  /// Update a potentially out-of-date declaration.
-  void updateOutOfDate(IdentifierInfo &II) const;
-
   Linkage getCachedLinkage() const {
     return static_cast<Linkage>(CacheValidAndLinkage);
   }
@@ -625,6 +622,12 @@ class alignas(8) Decl {
 
   void setReferenced(bool R = true) { Referenced = R; }
 
+  /// When doing manipulations which might change the computed linkage,
+  /// such as changing the DeclContext after the declaration has already been
+  /// used, invalidating the cache will make sure its linkage will be
+  /// recomputed.
+  void invalidateCachedLinkage() { setCachedLinkage(Linkage::Invalid); }
+
   /// Whether this declaration is a top-level declaration (function,
   /// global variable, etc.) that is lexically inside an objc container
   /// definition.
@@ -1564,13 +1567,6 @@ class DeclContext {
     LLVM_PREFERRED_TYPE(bool)
     uint64_t IsFreeStanding : 1;
 
-    /// Indicates whether it is possible for declarations of this kind
-    /// to have an out-of-date definition.
-    ///
-    /// This option is only enabled when modules are enabled.
-    LLVM_PREFERRED_TYPE(bool)
-    uint64_t MayHaveOutOfDateDef : 1;
-
     /// Has the full definition of this type been required by a use somewhere in
     /// the TU.
     LLVM_PREFERRED_TYPE(bool)
diff --git a/clang/include/clang/AST/DeclCXX.h b/clang/include/clang/AST/DeclCXX.h
index 33ae3d604020b..1d2ef0f4f2319 100644
--- a/clang/include/clang/AST/DeclCXX.h
+++ b/clang/include/clang/AST/DeclCXX.h
@@ -545,20 +545,6 @@ class CXXRecordDecl : public RecordDecl {
     return const_cast<CXXRecordDecl*>(this)->getMostRecentDecl();
   }
 
-  CXXRecordDecl *getMostRecentNonInjectedDecl() {
-    CXXRecordDecl *Recent = getMostRecentDecl();
-    while (Recent->isInjectedClassName()) {
-      // FIXME: Does injected class name need to be in the redeclarations chain?
-      assert(Recent->getPreviousDecl());
-      Recent = Recent->getPreviousDecl();
-    }
-    return Recent;
-  }
-
-  const CXXRecordDecl *getMostRecentNonInjectedDecl() const {
-    return const_cast<CXXRecordDecl*>(this)->getMostRecentNonInjectedDecl();
-  }
-
   CXXRecordDecl *getDefinition() const {
     // We only need an update if we don't already know which
     // declaration is the definition.
@@ -566,13 +552,18 @@ class CXXRecordDecl : public RecordDecl {
     return DD ? DD->Definition : nullptr;
   }
 
+  CXXRecordDecl *getDefinitionOrSelf() const {
+    if (auto *Def = getDefinition())
+      return Def;
+    return const_cast<CXXRecordDecl *>(this);
+  }
+
   bool hasDefinition() const { return DefinitionData || dataPtr(); }
 
   static CXXRecordDecl *Create(const ASTContext &C, TagKind TK, DeclContext *DC,
                                SourceLocation StartLoc, SourceLocation IdLoc,
                                IdentifierInfo *Id,
-                               CXXRecordDecl *PrevDecl = nullptr,
-                               bool DelayTypeCreation = false);
+                               CXXRecordDecl *PrevDecl = nullptr);
   static CXXRecordDecl *CreateLambda(const ASTContext &C, DeclContext *DC,
                                      TypeSourceInfo *Info, SourceLocation Loc,
                                      unsigned DependencyKind, bool IsGeneric,
@@ -1903,6 +1894,20 @@ class CXXRecordDecl : public RecordDecl {
   /// \endcode
   bool isInjectedClassName() const;
 
+  /// Determines whether this declaration has is canonically of an injected
+  /// class type. These are non-instantiated class template patterns, which can
+  /// be used from within the class template itself. For example:
+  ///
+  /// \code
+  /// template<class T> struct C {
+  ///   C *t; // Here `C *` is a pointer to an injected class type.
+  /// };
+  /// \endcode
+  bool hasInjectedClassType() const;
+
+  CanQualType
+  getCanonicalTemplateSpecializationType(const ASTContext &Ctx) const;
+
   // Determine whether this type is an Interface Like type for
   // __interface inheritance purposes.
   bool isInterfaceLike() const;
@@ -3131,7 +3136,7 @@ class UsingDirectiveDecl : public NamedDecl {
 
   /// Retrieve the nested-name-specifier that qualifies the
   /// name of the namespace.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return QualifierLoc.getNestedNameSpecifier();
   }
 
@@ -3252,7 +3257,7 @@ class NamespaceAliasDecl : public NamespaceBaseDecl,
 
   /// Retrieve the nested-name-specifier that qualifies the
   /// name of the namespace.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return QualifierLoc.getNestedNameSpecifier();
   }
 
@@ -3614,7 +3619,7 @@ class UsingDecl : public BaseUsingDecl, public Mergeable<UsingDecl> {
   NestedNameSpecifierLoc getQualifierLoc() const { return QualifierLoc; }
 
   /// Retrieve the nested-name-specifier that qualifies the name.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return QualifierLoc.getNestedNameSpecifier();
   }
 
@@ -3804,13 +3809,11 @@ class UsingEnumDecl : public BaseUsingDecl, public Mergeable<UsingEnumDecl> {
   /// The source location of the 'enum' keyword.
   SourceLocation getEnumLoc() const { return EnumLocation; }
   void setEnumLoc(SourceLocation L) { EnumLocation = L; }
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return getQualifierLoc().getNestedNameSpecifier();
   }
   NestedNameSpecifierLoc getQualifierLoc() const {
-    if (auto ETL = EnumType->getTypeLoc().getAs<ElaboratedTypeLoc>())
-      return ETL.getQualifierLoc();
-    return NestedNameSpecifierLoc();
+    return getEnumTypeLoc().getPrefix();
   }
   // Returns the "qualifier::Name" part as a TypeLoc.
   TypeLoc getEnumTypeLoc() const {
@@ -3970,7 +3973,7 @@ class UnresolvedUsingValueDecl : public ValueDecl,
   NestedNameSpecifierLoc getQualifierLoc() const { return QualifierLoc; }
 
   /// Retrieve the nested-name-specifier that qualifies the name.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return QualifierLoc.getNestedNameSpecifier();
   }
 
@@ -4060,7 +4063,7 @@ class UnresolvedUsingTypenameDecl
   NestedNameSpecifierLoc getQualifierLoc() const { return QualifierLoc; }
 
   /// Retrieve the nested-name-specifier that qualifies the name.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return QualifierLoc.getNestedNameSpecifier();
   }
 
diff --git a/clang/include/clang/AST/DeclObjC.h b/clang/include/clang/AST/DeclObjC.h
index 9014d76f8433b..2541edba83855 100644
--- a/clang/include/clang/AST/DeclObjC.h
+++ b/clang/include/clang/AST/DeclObjC.h
@@ -643,6 +643,9 @@ class ObjCTypeParamDecl : public TypedefNameDecl {
   /// from the explicitly-specified bound.
   SourceLocation getColonLoc() const { return ColonLoc; }
 
+  using TypeDecl::getTypeForDecl;
+  using TypeDecl::setTypeForDecl;
+
   // Implement isa/cast/dyncast/etc.
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K == ObjCTypeParam; }
diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 32de203f2d831..bba72365089f9 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -1796,7 +1796,10 @@ class BuiltinTemplateDecl : public TemplateDecl {
   }
 
   BuiltinTemplateKind getBuiltinTemplateKind() const { return BTK; }
+
+  bool isPackProducingBuiltinTemplate() const;
 };
+bool isPackProducingBuiltinTemplateName(TemplateName N);
 
 /// Provides information about an explicit instantiation of a variable or class
 /// template.
@@ -1898,14 +1901,14 @@ class ClassTemplateSpecializationDecl : public CXXRecordDecl,
   void getNameForDiagnostic(raw_ostream &OS, const PrintingPolicy &Policy,
                             bool Qualified) const override;
 
-  // FIXME: This is broken. CXXRecordDecl::getMostRecentDecl() returns a
-  // different "most recent" declaration from this function for the same
-  // declaration, because we don't override getMostRecentDeclImpl(). But
-  // it's not clear that we should override that, because the most recent
-  // declaration as a CXXRecordDecl sometimes is the injected-class-name.
   ClassTemplateSpecializationDecl *getMostRecentDecl() {
     return cast<ClassTemplateSpecializationDecl>(
-        getMostRecentNonInjectedDecl());
+        CXXRecordDecl::getMostRecentDecl());
+  }
+
+  ClassTemplateSpecializationDecl *getDefinitionOrSelf() const {
+    return cast<ClassTemplateSpecializationDecl>(
+        CXXRecordDecl::getDefinitionOrSelf());
   }
 
   /// Retrieve the template that this specialization specializes.
@@ -2123,10 +2126,13 @@ class ClassTemplatePartialSpecializationDecl
   llvm::PointerIntPair<ClassTemplatePartialSpecializationDecl *, 1, bool>
       InstantiatedFromMember;
 
+  mutable CanQualType CanonInjectedTST;
+
   ClassTemplatePartialSpecializationDecl(
       ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
       SourceLocation IdLoc, TemplateParameterList *Params,
       ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
+      CanQualType CanonInjectedTST,
       ClassTemplatePartialSpecializationDecl *PrevDecl);
 
   ClassTemplatePartialSpecializationDecl(ASTContext &C)
@@ -2143,7 +2149,7 @@ class ClassTemplatePartialSpecializationDecl
   Create(ASTContext &Context, TagKind TK, DeclContext *DC,
          SourceLocation StartLoc, SourceLocation IdLoc,
          TemplateParameterList *Params, ClassTemplateDecl *SpecializedTemplate,
-         ArrayRef<TemplateArgument> Args, QualType CanonInjectedType,
+         ArrayRef<TemplateArgument> Args, CanQualType CanonInjectedTST,
          ClassTemplatePartialSpecializationDecl *PrevDecl);
 
   static ClassTemplatePartialSpecializationDecl *
@@ -2160,12 +2166,6 @@ class ClassTemplatePartialSpecializationDecl
     return TemplateParams;
   }
 
-  /// Get the template argument list of the template parameter list.
-  ArrayRef<TemplateArgument>
-  getInjectedTemplateArgs(const ASTContext &Context) const {
-    return getTemplateParameters()->getInjectedTemplateArgs(Context);
-  }
-
   /// \brief All associated constraints of this partial specialization,
   /// including the requires clause and any constraints derived from
   /// constrained-parameters.
@@ -2247,14 +2247,10 @@ class ClassTemplatePartialSpecializationDecl
     return First->InstantiatedFromMember.setInt(true);
   }
 
-  /// Retrieves the injected specialization type for this partial
-  /// specialization.  This is not the same as the type-decl-type for
-  /// this partial specialization, which is an InjectedClassNameType.
-  QualType getInjectedSpecializationType() const {
-    assert(getTypeForDecl() && "partial specialization has no type set!");
-    return cast<InjectedClassNameType>(getTypeForDecl())
-             ->getInjectedSpecializationType();
-  }
+  /// Retrieves the canonical injected specialization type for this partial
+  /// specialization.
+  CanQualType
+  getCanonicalInjectedSpecializationType(const ASTContext &Ctx) const;
 
   SourceRange getSourceRange() const override LLVM_READONLY;
 
@@ -2289,8 +2285,8 @@ class ClassTemplateDecl : public RedeclarableTemplateDecl {
     llvm::FoldingSetVector<ClassTemplatePartialSpecializationDecl>
       PartialSpecializations;
 
-    /// The injected-class-name type for this class template.
-    QualType InjectedClassNameType;
+    /// The Injected Template Specialization Type for this declaration.
+    CanQualType CanonInjectedTST;
 
     Common() = default;
   };
@@ -2427,7 +2423,7 @@ class ClassTemplateDecl : public RedeclarableTemplateDecl {
   findPartialSpecInstantiatedFromMember(
                                      ClassTemplatePartialSpecializationDecl *D);
 
-  /// Retrieve the template specialization type of the
+  /// Retrieve the canonical template specialization type of the
   /// injected-class-name for this class template.
   ///
   /// The injected-class-name for a class template \c X is \c
@@ -2441,7 +2437,8 @@ class ClassTemplateDecl : public RedeclarableTemplateDecl {
   ///   typedef array this_type; // "array" is equivalent to "array<T, N>"
   /// };
   /// \endcode
-  QualType getInjectedClassNameSpecialization();
+  CanQualType
+  getCanonicalInjectedSpecializationType(const ASTContext &Ctx) const;
 
   using spec_iterator = SpecIterator<ClassTemplateSpecializationDecl>;
   using spec_range = llvm::iterator_range<spec_iterator>;
diff --git a/clang/include/clang/AST/DependenceFlags.h b/clang/include/clang/AST/DependenceFlags.h
index bdcaabc143cc4..c4395259f0758 100644
--- a/clang/include/clang/AST/DependenceFlags.h
+++ b/clang/include/clang/AST/DependenceFlags.h
@@ -293,7 +293,7 @@ inline TypeDependence toSemanticDependence(TypeDependence D) {
 }
 
 inline NestedNameSpecifierDependence
-toNestedNameSpecifierDependendence(TypeDependence D) {
+toNestedNameSpecifierDependence(TypeDependence D) {
   return Dependence(D).nestedNameSpecifier();
 }
 
diff --git a/clang/include/clang/AST/DynamicRecursiveASTVisitor.h b/clang/include/clang/AST/DynamicRecursiveASTVisitor.h
index 703cca22777ad..7b5bdca318348 100644
--- a/clang/include/clang/AST/DynamicRecursiveASTVisitor.h
+++ b/clang/include/clang/AST/DynamicRecursiveASTVisitor.h
@@ -134,8 +134,7 @@ template <bool IsConst> class DynamicRecursiveASTVisitorBase {
   /// Recursively visit a C++ nested-name-specifier.
   ///
   /// \returns false if the visitation was terminated early, true otherwise.
-  virtual bool
-  TraverseNestedNameSpecifier(MaybeConst<NestedNameSpecifier> *NNS);
+  virtual bool TraverseNestedNameSpecifier(NestedNameSpecifier NNS);
 
   /// Recursively visit a C++ nested-name-specifier with location
   /// information.
@@ -181,14 +180,14 @@ template <bool IsConst> class DynamicRecursiveASTVisitorBase {
   ///
   /// \returns false if the visitation was terminated early, true
   /// otherwise (including when the argument is a Null type).
-  virtual bool TraverseType(QualType T);
+  virtual bool TraverseType(QualType T, bool TraverseQualifier = true);
 
   /// Recursively visit a type with location, by dispatching to
   /// Traverse*TypeLoc() based on the argument type's getTypeClass() property.
   ///
   /// \returns false if the visitation was terminated early, true
   /// otherwise (including when the argument is a Null type location).
-  virtual bool TraverseTypeLoc(TypeLoc TL);
+  virtual bool TraverseTypeLoc(TypeLoc TL, bool TraverseQualifier = true);
 
   /// Recursively visit an Objective-C protocol reference with location
   /// information.
@@ -273,7 +272,8 @@ template <bool IsConst> class DynamicRecursiveASTVisitorBase {
 #define ABSTRACT_TYPE(CLASS, BASE)
 #define TYPE(CLASS, BASE)                                                      \
   bool WalkUpFrom##CLASS##Type(MaybeConst<CLASS##Type> *T);                    \
-  virtual bool Traverse##CLASS##Type(MaybeConst<CLASS##Type> *T);
+  virtual bool Traverse##CLASS##Type(MaybeConst<CLASS##Type> *T,               \
+                                     bool TraverseQualifier = true);
 #include "clang/AST/TypeNodes.inc"
 
 #define TYPE(CLASS, BASE)                                                      \
@@ -283,7 +283,8 @@ template <bool IsConst> class DynamicRecursiveASTVisitorBase {
   // TypeLocs.
 #define ABSTRACT_TYPELOC(CLASS, BASE)
 #define TYPELOC(CLASS, BASE)                                                   \
-  virtual bool Traverse##CLASS##TypeLoc(CLASS##TypeLoc TL);
+  virtual bool Traverse##CLASS##TypeLoc(CLASS##TypeLoc TL,                     \
+                                        bool TraverseQualifier);
 #include "clang/AST/TypeLocNodes.def"
 
 #define TYPELOC(CLASS, BASE)                                                   \
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 237b3b2cf7444..77d0912d1f41b 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -1369,7 +1369,7 @@ class DeclRefExpr final
 
   /// If the name was qualified, retrieves the nested-name-specifier
   /// that precedes the name. Otherwise, returns NULL.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return getQualifierLoc().getNestedNameSpecifier();
   }
 
@@ -3398,7 +3398,7 @@ class MemberExpr final
   /// If the member name was qualified, retrieves the
   /// nested-name-specifier that precedes the member name. Otherwise, returns
   /// NULL.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return getQualifierLoc().getNestedNameSpecifier();
   }
 
@@ -3548,6 +3548,7 @@ class CompoundLiteralExpr : public Expr {
                       QualType T, ExprValueKind VK, Expr *init, bool fileScope)
       : Expr(CompoundLiteralExprClass, T, VK, OK_Ordinary),
         LParenLoc(lparenloc), TInfoAndScope(tinfo, fileScope), Init(init) {
+    assert(Init && "Init is a nullptr");
     setDependence(computeDependence(this));
   }
 
@@ -3577,19 +3578,11 @@ class CompoundLiteralExpr : public Expr {
   APValue &getStaticValue() const;
 
   SourceLocation getBeginLoc() const LLVM_READONLY {
-    // FIXME: Init should never be null.
-    if (!Init)
-      return SourceLocation();
     if (LParenLoc.isInvalid())
       return Init->getBeginLoc();
     return LParenLoc;
   }
-  SourceLocation getEndLoc() const LLVM_READONLY {
-    // FIXME: Init should never be null.
-    if (!Init)
-      return SourceLocation();
-    return Init->getEndLoc();
-  }
+  SourceLocation getEndLoc() const LLVM_READONLY { return Init->getEndLoc(); }
 
   static bool classof(const Stmt *T) {
     return T->getStmtClass() == CompoundLiteralExprClass;
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index 7a26934de6259..9fedb230ce397 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -1712,6 +1712,19 @@ class CXXConstructExpr : public Expr {
     CXXConstructExprBits.IsImmediateEscalating = Set;
   }
 
+  /// Returns the WarnUnusedResultAttr that is declared on the callee
+  /// or its return type declaration, together with a NamedDecl that
+  /// refers to the declaration the attribute is attached to.
+  std::pair<const NamedDecl *, const WarnUnusedResultAttr *>
+  getUnusedResultAttr(const ASTContext &Ctx) const {
+    return getUnusedResultAttrImpl(getConstructor(), getType());
+  }
+
+  /// Returns true if this call expression should warn on unused results.
+  bool hasUnusedResultAttr(const ASTContext &Ctx) const {
+    return getUnusedResultAttr(Ctx).second != nullptr;
+  }
+
   SourceLocation getBeginLoc() const LLVM_READONLY;
   SourceLocation getEndLoc() const LLVM_READONLY;
   SourceRange getParenOrBraceRange() const { return ParenOrBraceRange; }
@@ -2781,7 +2794,7 @@ class CXXPseudoDestructorExpr : public Expr {
   /// If the member name was qualified, retrieves the
   /// nested-name-specifier that precedes the member name. Otherwise, returns
   /// null.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return QualifierLoc.getNestedNameSpecifier();
   }
 
@@ -3222,7 +3235,7 @@ class OverloadExpr : public Expr {
   SourceLocation getNameLoc() const { return NameInfo.getLoc(); }
 
   /// Fetches the nested-name qualifier, if one was given.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return QualifierLoc.getNestedNameSpecifier();
   }
 
@@ -3540,7 +3553,7 @@ class DependentScopeDeclRefExpr final
 
   /// Retrieve the nested-name-specifier that qualifies this
   /// declaration.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return QualifierLoc.getNestedNameSpecifier();
   }
 
@@ -3955,7 +3968,7 @@ class CXXDependentScopeMemberExpr final
   }
 
   /// Retrieve the nested-name-specifier that qualifies the member name.
-  NestedNameSpecifier *getQualifier() const {
+  NestedNameSpecifier getQualifier() const {
     return QualifierLoc.getNestedNameSpecifier();
   }
 
diff --git a/clang/include/clang/AST/JSONNodeDumper.h b/clang/include/clang/AST/JSONNodeDumper.h
index 570662b58ccf0..8640780206dba 100644
--- a/clang/include/clang/AST/JSONNodeDumper.h
+++ b/clang/include/clang/AST/JSONNodeDumper.h
@@ -240,7 +240,6 @@ class JSONNodeDumper
   void VisitInjectedClassNameType(const InjectedClassNameType *ICNT);
   void VisitObjCInterfaceType(const ObjCInterfaceType *OIT);
   void VisitPackExpansionType(const PackExpansionType *PET);
-  void VisitElaboratedType(const ElaboratedType *ET);
   void VisitMacroQualifiedType(const MacroQualifiedType *MQT);
   void VisitMemberPointerType(const MemberPointerType *MPT);
 
diff --git a/clang/include/clang/AST/NestedNameSpecifier.h b/clang/include/clang/AST/NestedNameSpecifier.h
index 1614f9d7c94e4..f198a8bca9078 100644
--- a/clang/include/clang/AST/NestedNameSpecifier.h
+++ b/clang/include/clang/AST/NestedNameSpecifier.h
@@ -6,507 +6,266 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file defines the NestedNameSpecifier class, which represents
-//  a C++ nested-name-specifier.
+//  This file completes the definition of the NestedNameSpecifier class.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_CLANG_AST_NESTEDNAMESPECIFIER_H
 #define LLVM_CLANG_AST_NESTEDNAMESPECIFIER_H
 
-#include "clang/AST/DependenceFlags.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/SourceLocation.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/NestedNameSpecifierBase.h"
+#include "clang/AST/Type.h"
+#include "clang/AST/TypeLoc.h"
 #include "llvm/ADT/DenseMapInfo.h"
-#include "llvm/ADT/FoldingSet.h"
-#include "llvm/ADT/PointerIntPair.h"
-#include "llvm/Support/Compiler.h"
-#include <cstdint>
-#include <cstdlib>
-#include <utility>
 
 namespace clang {
 
-class ASTContext;
-class CXXRecordDecl;
-class IdentifierInfo;
-class LangOptions;
-class NamespaceBaseDecl;
-struct PrintingPolicy;
-class Type;
-class TypeLoc;
-
-/// Represents a C++ nested name specifier, such as
-/// "\::std::vector<int>::".
-///
-/// C++ nested name specifiers are the prefixes to qualified
-/// names. For example, "foo::" in "foo::x" is a nested name
-/// specifier. Nested name specifiers are made up of a sequence of
-/// specifiers, each of which can be a namespace, type, identifier
-/// (for dependent names), decltype specifier, or the global specifier ('::').
-/// The last two specifiers can only appear at the start of a
-/// nested-namespace-specifier.
-class NestedNameSpecifier : public llvm::FoldingSetNode {
-  /// Enumeration describing
-  enum StoredSpecifierKind {
-    StoredIdentifier = 0,
-    StoredDecl = 1,
-    StoredTypeSpec = 2
-  };
-
-  /// The nested name specifier that precedes this nested name
-  /// specifier.
-  ///
-  /// The pointer is the nested-name-specifier that precedes this
-  /// one. The integer stores one of the first four values of type
-  /// SpecifierKind.
-  llvm::PointerIntPair<NestedNameSpecifier *, 2, StoredSpecifierKind> Prefix;
-
-  /// The last component in the nested name specifier, which
-  /// can be an identifier, a declaration, or a type.
-  ///
-  /// When the pointer is NULL, this specifier represents the global
-  /// specifier '::'. Otherwise, the pointer is one of
-  /// IdentifierInfo*, Namespace*, or Type*, depending on the kind of
-  /// specifier as encoded within the prefix.
-  void* Specifier = nullptr;
-
-public:
-  /// The kind of specifier that completes this nested name
-  /// specifier.
-  enum SpecifierKind {
-    /// An identifier, stored as an IdentifierInfo*.
-    Identifier,
-
-    /// A namespace-like entity, stored as a NamespaceBaseDecl*.
-    Namespace,
-
-    /// A type, stored as a Type*.
-    TypeSpec,
-
-    /// The global specifier '::'. There is no stored value.
-    Global,
-
-    /// Microsoft's '__super' specifier, stored as a CXXRecordDecl* of
-    /// the class it appeared in.
-    Super
-  };
-
-private:
-  /// Builds the global specifier.
-  NestedNameSpecifier() : Prefix(nullptr, StoredIdentifier) {}
-
-  /// Copy constructor used internally to clone nested name
-  /// specifiers.
-  NestedNameSpecifier(const NestedNameSpecifier &Other) = default;
-
-  /// Either find or insert the given nested name specifier
-  /// mockup in the given context.
-  static NestedNameSpecifier *FindOrInsert(const ASTContext &Context,
-                                           const NestedNameSpecifier &Mockup);
-
-public:
-  NestedNameSpecifier &operator=(const NestedNameSpecifier &) = delete;
-
-  /// Builds a specifier combining a prefix and an identifier.
-  ///
-  /// The prefix must be dependent, since nested name specifiers
-  /// referencing an identifier are only permitted when the identifier
-  /// cannot be resolved.
-  static NestedNameSpecifier *Create(const ASTContext &Context,
-                                     NestedNameSpecifier *Prefix,
-                                     const IdentifierInfo *II);
-
-  /// Builds a nested name specifier that names a namespace or namespace alias.
-  static NestedNameSpecifier *Create(const ASTContext &Context,
-                                     NestedNameSpecifier *Prefix,
-                                     const NamespaceBaseDecl *NS);
-
-  /// Builds a nested name specifier that names a type.
-  static NestedNameSpecifier *
-  Create(const ASTContext &Context, NestedNameSpecifier *Prefix, const Type *T);
-
-  /// Builds a specifier that consists of just an identifier.
-  ///
-  /// The nested-name-specifier is assumed to be dependent, but has no
-  /// prefix because the prefix is implied by something outside of the
-  /// nested name specifier, e.g., in "x->Base::f", the "x" has a dependent
-  /// type.
-  static NestedNameSpecifier *Create(const ASTContext &Context,
-                                     const IdentifierInfo *II);
-
-  /// Returns the nested name specifier representing the global
-  /// scope.
-  static NestedNameSpecifier *GlobalSpecifier(const ASTContext &Context);
-
-  /// Returns the nested name specifier representing the __super scope
-  /// for the given CXXRecordDecl.
-  static NestedNameSpecifier *SuperSpecifier(const ASTContext &Context,
-                                             CXXRecordDecl *RD);
-
-  /// Return the prefix of this nested name specifier.
-  ///
-  /// The prefix contains all of the parts of the nested name
-  /// specifier that precede this current specifier. For example, for a
-  /// nested name specifier that represents "foo::bar::", the current
-  /// specifier will contain "bar::" and the prefix will contain
-  /// "foo::".
-  NestedNameSpecifier *getPrefix() const { return Prefix.getPointer(); }
-
-  /// Determine what kind of nested name specifier is stored.
-  SpecifierKind getKind() const;
-
-  /// Retrieve the identifier stored in this nested name
-  /// specifier.
-  IdentifierInfo *getAsIdentifier() const {
-    if (Prefix.getInt() == StoredIdentifier)
-      return (IdentifierInfo *)Specifier;
-
-    return nullptr;
+auto NestedNameSpecifier::getKind() const -> Kind {
+  if (!isStoredKind()) {
+    switch (getFlagKind()) {
+    case FlagKind::Null:
+      return Kind::Null;
+    case FlagKind::Global:
+      return Kind::Global;
+    case FlagKind::Invalid:
+      llvm_unreachable("use of invalid NestedNameSpecifier");
+    }
+    llvm_unreachable("unhandled FlagKind");
   }
+  switch (auto [K, Ptr] = getStored(); K) {
+  case StoredKind::Type:
+    return Kind::Type;
+  case StoredKind::NamespaceWithGlobal:
+  case StoredKind::NamespaceWithNamespace:
+    return Kind::Namespace;
+  case StoredKind::NamespaceOrSuper:
+    switch (static_cast<const Decl *>(Ptr)->getKind()) {
+    case Decl::Namespace:
+    case Decl::NamespaceAlias:
+      return Kind::Namespace;
+    case Decl::CXXRecord:
+    case Decl::ClassTemplateSpecialization:
+    case Decl::ClassTemplatePartialSpecialization:
+      return Kind::MicrosoftSuper;
+    default:
+      llvm_unreachable("unexpected decl kind");
+    }
+  }
+  llvm_unreachable("unknown StoredKind");
+}
 
-  /// Retrieve the namespace or namespace alias stored in this nested name
-  /// specifier.
-  NamespaceBaseDecl *getAsNamespace() const;
-
-  /// Retrieve the record declaration stored in this nested name
-  /// specifier.
-  CXXRecordDecl *getAsRecordDecl() const;
-
-  /// Retrieve the type stored in this nested name specifier.
-  const Type *getAsType() const {
-    if (Prefix.getInt() == StoredTypeSpec)
-      return (const Type *)Specifier;
+NestedNameSpecifier::NestedNameSpecifier(const Type *T)
+    : NestedNameSpecifier({StoredKind::Type, T}) {
+  assert(getKind() == Kind::Type);
+}
 
-    return nullptr;
+auto NestedNameSpecifier::MakeNamespacePtrKind(
+    const ASTContext &Ctx, const NamespaceBaseDecl *Namespace,
+    NestedNameSpecifier Prefix) -> PtrKind {
+  switch (Prefix.getKind()) {
+  case Kind::Null:
+    return {StoredKind::NamespaceOrSuper, Namespace};
+  case Kind::Global:
+    return {StoredKind::NamespaceWithGlobal, Namespace};
+  case Kind::Namespace:
+    return {StoredKind::NamespaceWithNamespace,
+            MakeNamespaceAndPrefixStorage(Ctx, Namespace, Prefix)};
+  case Kind::MicrosoftSuper:
+  case Kind::Type:
+    llvm_unreachable("invalid prefix for namespace");
   }
+  llvm_unreachable("unhandled kind");
+}
 
-  /// Fully translate this nested name specifier to a type.
-  /// Unlike getAsType, this will convert this entire nested
-  /// name specifier chain into its equivalent type.
-  const Type *translateToType(const ASTContext &Context) const;
+/// Builds a nested name specifier that names a namespace.
+NestedNameSpecifier::NestedNameSpecifier(const ASTContext &Ctx,
+                                         const NamespaceBaseDecl *Namespace,
+                                         NestedNameSpecifier Prefix)
+    : NestedNameSpecifier(MakeNamespacePtrKind(Ctx, Namespace, Prefix)) {
+  assert(getKind() == Kind::Namespace);
+}
 
-  NestedNameSpecifierDependence getDependence() const;
+/// Builds a nested name specifier that names a class through microsoft's
+/// __super specifier.
+NestedNameSpecifier::NestedNameSpecifier(CXXRecordDecl *RD)
+    : NestedNameSpecifier({StoredKind::NamespaceOrSuper, RD}) {
+  assert(getKind() == Kind::MicrosoftSuper);
+}
 
-  /// Whether this nested name specifier refers to a dependent
-  /// type or not.
-  bool isDependent() const;
+CXXRecordDecl *NestedNameSpecifier::getAsRecordDecl() const {
+  switch (getKind()) {
+  case Kind::MicrosoftSuper:
+    return getAsMicrosoftSuper();
+  case Kind::Type:
+    return getAsType()->getAsCXXRecordDecl();
+  case Kind::Global:
+  case Kind::Namespace:
+  case Kind::Null:
+    return nullptr;
+  }
+  llvm_unreachable("Invalid NNS Kind!");
+}
 
-  /// Whether this nested name specifier involves a template
-  /// parameter.
-  bool isInstantiationDependent() const;
+NestedNameSpecifier NestedNameSpecifier::getCanonical() const {
+  switch (getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    // These are canonical and unique.
+    return *this;
+  case NestedNameSpecifier::Kind::Namespace: {
+    // A namespace is canonical; build a nested-name-specifier with
+    // this namespace and no prefix.
+    const NamespaceBaseDecl *ND = getAsNamespaceAndPrefix().Namespace;
+    return NestedNameSpecifier(
+        {StoredKind::NamespaceOrSuper, ND->getNamespace()->getCanonicalDecl()});
+  }
+  case NestedNameSpecifier::Kind::Type:
+    return NestedNameSpecifier(
+        getAsType()->getCanonicalTypeInternal().getTypePtr());
+  }
+  llvm_unreachable("unhandled kind");
+}
 
-  /// Whether this nested-name-specifier contains an unexpanded
-  /// parameter pack (for C++11 variadic templates).
-  bool containsUnexpandedParameterPack() const;
+bool NestedNameSpecifier::isCanonical() const {
+  return *this == getCanonical();
+}
 
-  /// Whether this nested name specifier contains an error.
-  bool containsErrors() const;
+TypeLoc NestedNameSpecifierLoc::castAsTypeLoc() const {
+  return TypeLoc(Qualifier.getAsType(), LoadPointer(/*Offset=*/0));
+}
 
-  /// Print this nested name specifier to the given output stream. If
-  /// `ResolveTemplateArguments` is true, we'll print actual types, e.g.
-  /// `ns::SomeTemplate<int, MyClass>` instead of
-  /// `ns::SomeTemplate<Container::value_type, T>`.
-  void print(raw_ostream &OS, const PrintingPolicy &Policy,
-             bool ResolveTemplateArguments = false,
-             bool PrintFinalScopeResOp = true) const;
+TypeLoc NestedNameSpecifierLoc::getAsTypeLoc() const {
+  if (Qualifier.getKind() != NestedNameSpecifier::Kind::Type)
+    return TypeLoc();
+  return castAsTypeLoc();
+}
 
-  void Profile(llvm::FoldingSetNodeID &ID) const {
-    ID.AddPointer(Prefix.getOpaqueValue());
-    ID.AddPointer(Specifier);
+unsigned
+NestedNameSpecifierLoc::getLocalDataLength(NestedNameSpecifier Qualifier) {
+  // Location of the trailing '::'.
+  unsigned Length = sizeof(SourceLocation::UIntTy);
+
+  switch (Qualifier.getKind()) {
+  case NestedNameSpecifier::Kind::Global:
+    // Nothing more to add.
+    break;
+
+  case NestedNameSpecifier::Kind::Namespace:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    // The location of the identifier or namespace name.
+    Length += sizeof(SourceLocation::UIntTy);
+    break;
+
+  case NestedNameSpecifier::Kind::Type:
+    // The "void*" that points at the TypeLoc data.
+    // Note: the 'template' keyword is part of the TypeLoc.
+    Length += sizeof(void *);
+    break;
+
+  case NestedNameSpecifier::Kind::Null:
+    llvm_unreachable("Expected a non-NULL qualifier");
   }
 
-  /// Dump the nested name specifier to standard output to aid
-  /// in debugging.
-  void dump(const LangOptions &LO) const;
-  void dump() const;
-  void dump(llvm::raw_ostream &OS) const;
-  void dump(llvm::raw_ostream &OS, const LangOptions &LO) const;
-};
+  return Length;
+}
 
-/// A C++ nested-name-specifier augmented with source location
-/// information.
-class NestedNameSpecifierLoc {
-  NestedNameSpecifier *Qualifier = nullptr;
-  void *Data = nullptr;
-
-  /// Determines the data length for the last component in the
-  /// given nested-name-specifier.
-  static unsigned getLocalDataLength(NestedNameSpecifier *Qualifier);
-
-  /// Determines the data length for the entire
-  /// nested-name-specifier.
-  static unsigned getDataLength(NestedNameSpecifier *Qualifier);
-
-public:
-  /// Construct an empty nested-name-specifier.
-  NestedNameSpecifierLoc() = default;
-
-  /// Construct a nested-name-specifier with source location information
-  /// from
-  NestedNameSpecifierLoc(NestedNameSpecifier *Qualifier, void *Data)
-      : Qualifier(Qualifier), Data(Data) {}
-
-  /// Evaluates true when this nested-name-specifier location is
-  /// non-empty.
-  explicit operator bool() const { return Qualifier; }
-
-  /// Evaluates true when this nested-name-specifier location is
-  /// non-empty.
-  bool hasQualifier() const { return Qualifier; }
-
-  /// Retrieve the nested-name-specifier to which this instance
-  /// refers.
-  NestedNameSpecifier *getNestedNameSpecifier() const {
-    return Qualifier;
-  }
+NamespaceAndPrefixLoc NestedNameSpecifierLoc::castAsNamespaceAndPrefix() const {
+  auto [Namespace, Prefix] = Qualifier.getAsNamespaceAndPrefix();
+  return {Namespace, NestedNameSpecifierLoc(Prefix, Data)};
+}
 
-  /// Retrieve the opaque pointer that refers to source-location data.
-  void *getOpaqueData() const { return Data; }
-
-  /// Retrieve the source range covering the entirety of this
-  /// nested-name-specifier.
-  ///
-  /// For example, if this instance refers to a nested-name-specifier
-  /// \c \::std::vector<int>::, the returned source range would cover
-  /// from the initial '::' to the last '::'.
-  SourceRange getSourceRange() const LLVM_READONLY {
-    return SourceRange(getBeginLoc(), getEndLoc());
-  }
+NamespaceAndPrefixLoc NestedNameSpecifierLoc::getAsNamespaceAndPrefix() const {
+  if (Qualifier.getKind() != NestedNameSpecifier::Kind::Namespace)
+    return {};
+  return castAsNamespaceAndPrefix();
+}
 
-  /// Retrieve the source range covering just the last part of
-  /// this nested-name-specifier, not including the prefix.
-  ///
-  /// For example, if this instance refers to a nested-name-specifier
-  /// \c \::std::vector<int>::, the returned source range would cover
-  /// from "vector" to the last '::'.
-  SourceRange getLocalSourceRange() const;
-
-  /// Retrieve the location of the beginning of this
-  /// nested-name-specifier.
-  SourceLocation getBeginLoc() const {
-    if (!Qualifier)
-      return SourceLocation();
-
-    NestedNameSpecifierLoc First = *this;
-    while (NestedNameSpecifierLoc Prefix = First.getPrefix())
-      First = Prefix;
-    return First.getLocalSourceRange().getBegin();
+unsigned NestedNameSpecifierLoc::getDataLength(NestedNameSpecifier Qualifier) {
+  unsigned Length = 0;
+  for (; Qualifier; Qualifier = Qualifier.getAsNamespaceAndPrefix().Prefix) {
+    Length += getLocalDataLength(Qualifier);
+    if (Qualifier.getKind() != NestedNameSpecifier::Kind::Namespace)
+      break;
   }
+  return Length;
+}
 
-  /// Retrieve the location of the end of this
-  /// nested-name-specifier.
-  SourceLocation getEndLoc() const { return getLocalSourceRange().getEnd(); }
+unsigned NestedNameSpecifierLoc::getDataLength() const {
+  return getDataLength(Qualifier);
+}
 
-  /// Retrieve the location of the beginning of this
-  /// component of the nested-name-specifier.
-  SourceLocation getLocalBeginLoc() const {
-    return getLocalSourceRange().getBegin();
+SourceRange NestedNameSpecifierLoc::getLocalSourceRange() const {
+  switch (auto Kind = Qualifier.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+    return SourceRange();
+  case NestedNameSpecifier::Kind::Global:
+    return LoadSourceLocation(/*Offset=*/0);
+  case NestedNameSpecifier::Kind::Namespace:
+  case NestedNameSpecifier::Kind::MicrosoftSuper: {
+    unsigned Offset =
+        Kind == NestedNameSpecifier::Kind::Namespace
+            ? getDataLength(Qualifier.getAsNamespaceAndPrefix().Prefix)
+            : 0;
+    return SourceRange(
+        LoadSourceLocation(Offset),
+        LoadSourceLocation(Offset + sizeof(SourceLocation::UIntTy)));
   }
-
-  /// Retrieve the location of the end of this component of the
-  /// nested-name-specifier.
-  SourceLocation getLocalEndLoc() const {
-    return getLocalSourceRange().getEnd();
+  case NestedNameSpecifier::Kind::Type: {
+    // The "void*" that points at the TypeLoc data.
+    // Note: the 'template' keyword is part of the TypeLoc.
+    void *TypeData = LoadPointer(/*Offset=*/0);
+    TypeLoc TL(Qualifier.getAsType(), TypeData);
+    return SourceRange(TL.getBeginLoc(), LoadSourceLocation(sizeof(void *)));
   }
-
-  /// Return the prefix of this nested-name-specifier.
-  ///
-  /// For example, if this instance refers to a nested-name-specifier
-  /// \c \::std::vector<int>::, the prefix is \c \::std::. Note that the
-  /// returned prefix may be empty, if this is the first component of
-  /// the nested-name-specifier.
-  NestedNameSpecifierLoc getPrefix() const {
-    if (!Qualifier)
-      return *this;
-
-    return NestedNameSpecifierLoc(Qualifier->getPrefix(), Data);
   }
 
-  /// For a nested-name-specifier that refers to a type,
-  /// retrieve the type with source-location information.
-  TypeLoc getTypeLoc() const;
+  llvm_unreachable("Invalid NNS Kind!");
+}
 
-  /// Determines the data length for the entire
-  /// nested-name-specifier.
-  unsigned getDataLength() const { return getDataLength(Qualifier); }
+SourceRange NestedNameSpecifierLoc::getSourceRange() const {
+  return SourceRange(getBeginLoc(), getEndLoc());
+}
 
-  friend bool operator==(NestedNameSpecifierLoc X,
-                         NestedNameSpecifierLoc Y) {
-    return X.Qualifier == Y.Qualifier && X.Data == Y.Data;
-  }
+SourceLocation NestedNameSpecifierLoc::getEndLoc() const {
+  return getLocalSourceRange().getEnd();
+}
 
-  friend bool operator!=(NestedNameSpecifierLoc X,
-                         NestedNameSpecifierLoc Y) {
-    return !(X == Y);
-  }
-};
+/// Retrieve the location of the beginning of this
+/// component of the nested-name-specifier.
+SourceLocation NestedNameSpecifierLoc::getLocalBeginLoc() const {
+  return getLocalSourceRange().getBegin();
+}
 
-/// Class that aids in the construction of nested-name-specifiers along
-/// with source-location information for all of the components of the
+/// Retrieve the location of the end of this component of the
 /// nested-name-specifier.
-class NestedNameSpecifierLocBuilder {
-  /// The current representation of the nested-name-specifier we're
-  /// building.
-  NestedNameSpecifier *Representation = nullptr;
-
-  /// Buffer used to store source-location information for the
-  /// nested-name-specifier.
-  ///
-  /// Note that we explicitly manage the buffer (rather than using a
-  /// SmallVector) because \c Declarator expects it to be possible to memcpy()
-  /// a \c CXXScopeSpec, and CXXScopeSpec uses a NestedNameSpecifierLocBuilder.
-  char *Buffer = nullptr;
-
-  /// The size of the buffer used to store source-location information
-  /// for the nested-name-specifier.
-  unsigned BufferSize = 0;
-
-  /// The capacity of the buffer used to store source-location
-  /// information for the nested-name-specifier.
-  unsigned BufferCapacity = 0;
-
-public:
-  NestedNameSpecifierLocBuilder() = default;
-  NestedNameSpecifierLocBuilder(const NestedNameSpecifierLocBuilder &Other);
-
-  NestedNameSpecifierLocBuilder &
-  operator=(const NestedNameSpecifierLocBuilder &Other);
-
-  ~NestedNameSpecifierLocBuilder() {
-    if (BufferCapacity)
-      free(Buffer);
-  }
+SourceLocation NestedNameSpecifierLoc::getLocalEndLoc() const {
+  return getLocalSourceRange().getEnd();
+}
 
-  /// Retrieve the representation of the nested-name-specifier.
-  NestedNameSpecifier *getRepresentation() const { return Representation; }
-
-  /// Extend the current nested-name-specifier by another
-  /// nested-name-specifier component of the form 'type::'.
-  ///
-  /// \param Context The AST context in which this nested-name-specifier
-  /// resides.
-  ///
-  /// \param TL The TypeLoc that describes the type preceding the '::'.
-  ///
-  /// \param ColonColonLoc The location of the trailing '::'.
-  void Extend(ASTContext &Context, TypeLoc TL, SourceLocation ColonColonLoc);
-
-  /// Extend the current nested-name-specifier by another
-  /// nested-name-specifier component of the form 'identifier::'.
-  ///
-  /// \param Context The AST context in which this nested-name-specifier
-  /// resides.
-  ///
-  /// \param Identifier The identifier.
-  ///
-  /// \param IdentifierLoc The location of the identifier.
-  ///
-  /// \param ColonColonLoc The location of the trailing '::'.
-  void Extend(ASTContext &Context, IdentifierInfo *Identifier,
-              SourceLocation IdentifierLoc, SourceLocation ColonColonLoc);
-
-  /// Extend the current nested-name-specifier by another
-  /// nested-name-specifier component of the form 'namespace::'.
-  ///
-  /// \param Context The AST context in which this nested-name-specifier
-  /// resides.
-  ///
-  /// \param Namespace The namespace or namespace alias.
-  ///
-  /// \param NamespaceLoc The location of the namespace name or the namespace
-  //  alias.
-  ///
-  /// \param ColonColonLoc The location of the trailing '::'.
-  void Extend(ASTContext &Context, NamespaceBaseDecl *Namespace,
-              SourceLocation NamespaceLoc, SourceLocation ColonColonLoc);
-
-  /// Turn this (empty) nested-name-specifier into the global
-  /// nested-name-specifier '::'.
-  void MakeGlobal(ASTContext &Context, SourceLocation ColonColonLoc);
-
-  /// Turns this (empty) nested-name-specifier into '__super'
-  /// nested-name-specifier.
-  ///
-  /// \param Context The AST context in which this nested-name-specifier
-  /// resides.
-  ///
-  /// \param RD The declaration of the class in which nested-name-specifier
-  /// appeared.
-  ///
-  /// \param SuperLoc The location of the '__super' keyword.
-  /// name.
-  ///
-  /// \param ColonColonLoc The location of the trailing '::'.
-  void MakeSuper(ASTContext &Context, CXXRecordDecl *RD,
-                 SourceLocation SuperLoc, SourceLocation ColonColonLoc);
-
-  /// Make a new nested-name-specifier from incomplete source-location
-  /// information.
-  ///
-  /// This routine should be used very, very rarely, in cases where we
-  /// need to synthesize a nested-name-specifier. Most code should instead use
-  /// \c Adopt() with a proper \c NestedNameSpecifierLoc.
-  void MakeTrivial(ASTContext &Context, NestedNameSpecifier *Qualifier,
-                   SourceRange R);
-
-  /// Adopt an existing nested-name-specifier (with source-range
-  /// information).
-  void Adopt(NestedNameSpecifierLoc Other);
-
-  /// Retrieve the source range covered by this nested-name-specifier.
-  SourceRange getSourceRange() const LLVM_READONLY {
-    return NestedNameSpecifierLoc(Representation, Buffer).getSourceRange();
-  }
+SourceRange NestedNameSpecifierLocBuilder::getSourceRange() const {
+  return NestedNameSpecifierLoc(Representation, Buffer).getSourceRange();
+}
 
-  /// Retrieve a nested-name-specifier with location information,
-  /// copied into the given AST context.
-  ///
-  /// \param Context The context into which this nested-name-specifier will be
-  /// copied.
-  NestedNameSpecifierLoc getWithLocInContext(ASTContext &Context) const;
-
-  /// Retrieve a nested-name-specifier with location
-  /// information based on the information in this builder.
-  ///
-  /// This loc will contain references to the builder's internal data and may
-  /// be invalidated by any change to the builder.
-  NestedNameSpecifierLoc getTemporary() const {
-    return NestedNameSpecifierLoc(Representation, Buffer);
-  }
+} // namespace clang
+
+namespace llvm {
+
+template <> struct DenseMapInfo<clang::NestedNameSpecifier> {
+  static clang::NestedNameSpecifier getEmptyKey() { return std::nullopt; }
 
-  /// Clear out this builder, and prepare it to build another
-  /// nested-name-specifier with source-location information.
-  void Clear() {
-    Representation = nullptr;
-    BufferSize = 0;
+  static clang::NestedNameSpecifier getTombstoneKey() {
+    return clang::NestedNameSpecifier::getInvalid();
   }
 
-  /// Retrieve the underlying buffer.
-  ///
-  /// \returns A pair containing a pointer to the buffer of source-location
-  /// data and the size of the source-location data that resides in that
-  /// buffer.
-  std::pair<char *, unsigned> getBuffer() const {
-    return std::make_pair(Buffer, BufferSize);
+  static unsigned getHashValue(const clang::NestedNameSpecifier &V) {
+    return hash_combine(V.getAsVoidPointer());
   }
 };
 
-/// Insertion operator for diagnostics.  This allows sending
-/// NestedNameSpecifiers into a diagnostic with <<.
-inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
-                                             NestedNameSpecifier *NNS) {
-  DB.AddTaggedVal(reinterpret_cast<uint64_t>(NNS),
-                  DiagnosticsEngine::ak_nestednamespec);
-  return DB;
-}
-
-} // namespace clang
-
-namespace llvm {
-
 template <> struct DenseMapInfo<clang::NestedNameSpecifierLoc> {
-  using FirstInfo = DenseMapInfo<clang::NestedNameSpecifier *>;
+  using FirstInfo = DenseMapInfo<clang::NestedNameSpecifier>;
   using SecondInfo = DenseMapInfo<void *>;
 
   static clang::NestedNameSpecifierLoc getEmptyKey() {
diff --git a/clang/include/clang/AST/NestedNameSpecifierBase.h b/clang/include/clang/AST/NestedNameSpecifierBase.h
new file mode 100644
index 0000000000000..73c60ba695419
--- /dev/null
+++ b/clang/include/clang/AST/NestedNameSpecifierBase.h
@@ -0,0 +1,586 @@
+//===- NestedNameSpecifier.h - C++ nested name specifiers -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the NestedNameSpecifier class, which represents
+//  a C++ nested-name-specifier.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_AST_NESTEDNAMESPECIFIERBASE_H
+#define LLVM_CLANG_AST_NESTEDNAMESPECIFIERBASE_H
+
+#include "clang/AST/DependenceFlags.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/Support/Compiler.h"
+#include "llvm/Support/PointerLikeTypeTraits.h"
+#include <cstdint>
+#include <cstdlib>
+#include <utility>
+
+namespace clang {
+
+class ASTContext;
+class CXXRecordDecl;
+class NamedDecl;
+class IdentifierInfo;
+class LangOptions;
+class NamespaceBaseDecl;
+struct PrintingPolicy;
+class Type;
+class TypeLoc;
+
+struct NamespaceAndPrefix;
+struct alignas(8) NamespaceAndPrefixStorage;
+
+/// Represents a C++ nested name specifier, such as
+/// "\::std::vector<int>::".
+///
+/// C++ nested name specifiers are the prefixes to qualified
+/// names. For example, "foo::" in "foo::x" is a nested name
+/// specifier. Nested name specifiers are made up of a sequence of
+/// specifiers, each of which can be a namespace, type, decltype specifier, or
+/// the global specifier ('::'). The last two specifiers can only appear at the
+/// start of a nested-namespace-specifier.
+class NestedNameSpecifier {
+  enum class FlagKind { Null, Global, Invalid };
+  enum class StoredKind {
+    Type,
+    NamespaceOrSuper,
+    NamespaceWithGlobal,
+    NamespaceWithNamespace
+  };
+  static constexpr uintptr_t FlagBits = 2, FlagMask = (1u << FlagBits) - 1u,
+                             FlagOffset = 1, PtrOffset = FlagBits + FlagOffset,
+                             PtrMask = (1u << PtrOffset) - 1u;
+
+  uintptr_t StoredOrFlag;
+
+  explicit NestedNameSpecifier(uintptr_t StoredOrFlag)
+      : StoredOrFlag(StoredOrFlag) {}
+  struct PtrKind {
+    StoredKind SK;
+    const void *Ptr;
+  };
+  explicit NestedNameSpecifier(PtrKind PK)
+      : StoredOrFlag(uintptr_t(PK.Ptr) | (uintptr_t(PK.SK) << FlagOffset)) {
+    assert(PK.Ptr != nullptr);
+    assert((uintptr_t(PK.Ptr) & ((1u << PtrOffset) - 1u)) == 0);
+    assert((uintptr_t(PK.Ptr) >> PtrOffset) != 0);
+  }
+
+  explicit constexpr NestedNameSpecifier(FlagKind K)
+      : StoredOrFlag(uintptr_t(K) << FlagOffset) {}
+
+  bool isStoredKind() const { return (StoredOrFlag >> PtrOffset) != 0; }
+
+  std::pair<StoredKind, const void *> getStored() const {
+    assert(isStoredKind());
+    return {StoredKind(StoredOrFlag >> FlagOffset & FlagMask),
+            reinterpret_cast<const void *>(StoredOrFlag & ~PtrMask)};
+  }
+
+  FlagKind getFlagKind() const {
+    assert(!isStoredKind());
+    return FlagKind(StoredOrFlag >> FlagOffset);
+  }
+
+  static const NamespaceAndPrefixStorage *
+  MakeNamespaceAndPrefixStorage(const ASTContext &Ctx,
+                                const NamespaceBaseDecl *Namespace,
+                                NestedNameSpecifier Prefix);
+  static inline PtrKind MakeNamespacePtrKind(const ASTContext &Ctx,
+                                             const NamespaceBaseDecl *Namespace,
+                                             NestedNameSpecifier Prefix);
+
+public:
+  static constexpr NestedNameSpecifier getInvalid() {
+    return NestedNameSpecifier(FlagKind::Invalid);
+  }
+
+  static constexpr NestedNameSpecifier getGlobal() {
+    return NestedNameSpecifier(FlagKind::Global);
+  }
+
+  NestedNameSpecifier() : NestedNameSpecifier(FlagKind::Invalid) {}
+
+  /// The kind of specifier that completes this nested name
+  /// specifier.
+  enum class Kind {
+    /// Empty.
+    Null,
+
+    /// The global specifier '::'. There is no stored value.
+    Global,
+
+    /// A type, stored as a Type*.
+    Type,
+
+    /// A namespace-like entity, stored as a NamespaceBaseDecl*.
+    Namespace,
+
+    /// Microsoft's '__super' specifier, stored as a CXXRecordDecl* of
+    /// the class it appeared in.
+    MicrosoftSuper,
+  };
+
+  inline Kind getKind() const;
+
+  NestedNameSpecifier(std::nullopt_t) : StoredOrFlag(0) {}
+
+  explicit inline NestedNameSpecifier(const Type *T);
+
+  /// Builds a nested name specifier that names a namespace.
+  inline NestedNameSpecifier(const ASTContext &Ctx,
+                             const NamespaceBaseDecl *Namespace,
+                             NestedNameSpecifier Prefix);
+
+  /// Builds a nested name specifier that names a class through microsoft's
+  /// __super specifier.
+  explicit inline NestedNameSpecifier(CXXRecordDecl *RD);
+
+  explicit operator bool() const { return StoredOrFlag != 0; }
+
+  void *getAsVoidPointer() const {
+    return reinterpret_cast<void *>(StoredOrFlag);
+  }
+  static NestedNameSpecifier getFromVoidPointer(const void *Ptr) {
+    return NestedNameSpecifier(reinterpret_cast<uintptr_t>(Ptr));
+  }
+
+  const Type *getAsType() const {
+    auto [Kind, Ptr] = getStored();
+    assert(Kind == StoredKind::Type);
+    assert(Ptr != nullptr);
+    return static_cast<const Type *>(Ptr);
+  }
+
+  inline NamespaceAndPrefix getAsNamespaceAndPrefix() const;
+
+  CXXRecordDecl *getAsMicrosoftSuper() const {
+    auto [Kind, Ptr] = getStored();
+    assert(Kind == StoredKind::NamespaceOrSuper);
+    assert(Ptr != nullptr);
+    return static_cast<CXXRecordDecl *>(const_cast<void *>(Ptr));
+  }
+
+  /// Retrieve the record declaration stored in this nested name
+  /// specifier, or null.
+  inline CXXRecordDecl *getAsRecordDecl() const;
+
+  friend bool operator==(NestedNameSpecifier LHS, NestedNameSpecifier RHS) {
+    return LHS.StoredOrFlag == RHS.StoredOrFlag;
+  }
+  friend bool operator!=(NestedNameSpecifier LHS, NestedNameSpecifier RHS) {
+    return LHS.StoredOrFlag != RHS.StoredOrFlag;
+  }
+
+  /// Retrieves the "canonical" nested name specifier for a
+  /// given nested name specifier.
+  ///
+  /// The canonical nested name specifier is a nested name specifier
+  /// that uniquely identifies a type or namespace within the type
+  /// system. For example, given:
+  ///
+  /// \code
+  /// namespace N {
+  ///   struct S {
+  ///     template<typename T> struct X { typename T* type; };
+  ///   };
+  /// }
+  ///
+  /// template<typename T> struct Y {
+  ///   typename N::S::X<T>::type member;
+  /// };
+  /// \endcode
+  ///
+  /// Here, the nested-name-specifier for N::S::X<T>:: will be
+  /// S::X<template-param-0-0>, since 'S' and 'X' are uniquely defined
+  /// by declarations in the type system and the canonical type for
+  /// the template type parameter 'T' is template-param-0-0.
+  inline NestedNameSpecifier getCanonical() const;
+
+  /// Whether this nested name specifier is canonical.
+  inline bool isCanonical() const;
+
+  /// Whether this nested name specifier starts with a '::'.
+  bool isFullyQualified() const;
+
+  NestedNameSpecifierDependence getDependence() const;
+
+  /// Whether this nested name specifier refers to a dependent
+  /// type or not.
+  bool isDependent() const {
+    return getDependence() & NestedNameSpecifierDependence::Dependent;
+  }
+
+  /// Whether this nested name specifier involves a template
+  /// parameter.
+  bool isInstantiationDependent() const {
+    return getDependence() & NestedNameSpecifierDependence::Instantiation;
+  }
+
+  /// Whether this nested-name-specifier contains an unexpanded
+  /// parameter pack (for C++11 variadic templates).
+  bool containsUnexpandedParameterPack() const {
+    return getDependence() & NestedNameSpecifierDependence::UnexpandedPack;
+  }
+
+  /// Whether this nested name specifier contains an error.
+  bool containsErrors() const {
+    return getDependence() & NestedNameSpecifierDependence::Error;
+  }
+
+  /// Print this nested name specifier to the given output stream. If
+  /// `ResolveTemplateArguments` is true, we'll print actual types, e.g.
+  /// `ns::SomeTemplate<int, MyClass>` instead of
+  /// `ns::SomeTemplate<Container::value_type, T>`.
+  void print(raw_ostream &OS, const PrintingPolicy &Policy,
+             bool ResolveTemplateArguments = false,
+             bool PrintFinalScopeResOp = true) const;
+
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    ID.AddInteger(StoredOrFlag);
+  }
+
+  /// Dump the nested name specifier to aid in debugging.
+  void dump(llvm::raw_ostream *OS = nullptr,
+            const LangOptions *LO = nullptr) const;
+  void dump(const LangOptions &LO) const;
+  void dump(llvm::raw_ostream &OS) const;
+  void dump(llvm::raw_ostream &OS, const LangOptions &LO) const;
+
+  static constexpr auto NumLowBitsAvailable = FlagOffset;
+};
+
+struct NamespaceAndPrefix {
+  const NamespaceBaseDecl *Namespace;
+  NestedNameSpecifier Prefix;
+};
+
+struct alignas(8) NamespaceAndPrefixStorage : NamespaceAndPrefix,
+                                              llvm::FoldingSetNode {
+  NamespaceAndPrefixStorage(const NamespaceBaseDecl *Namespace,
+                            NestedNameSpecifier Prefix)
+      : NamespaceAndPrefix{Namespace, Prefix} {}
+  void Profile(llvm::FoldingSetNodeID &ID) { Profile(ID, Namespace, Prefix); }
+  static void Profile(llvm::FoldingSetNodeID &ID,
+                      const NamespaceBaseDecl *Namespace,
+                      NestedNameSpecifier Prefix) {
+    ID.AddPointer(Namespace);
+    Prefix.Profile(ID);
+  }
+};
+
+NamespaceAndPrefix NestedNameSpecifier::getAsNamespaceAndPrefix() const {
+  auto [Kind, Ptr] = getStored();
+  switch (Kind) {
+  case StoredKind::NamespaceOrSuper:
+  case StoredKind::NamespaceWithGlobal:
+    return {static_cast<const NamespaceBaseDecl *>(Ptr),
+            Kind == StoredKind::NamespaceWithGlobal
+                ? NestedNameSpecifier::getGlobal()
+                : std::nullopt};
+  case StoredKind::NamespaceWithNamespace:
+    return *static_cast<const NamespaceAndPrefixStorage *>(Ptr);
+  case StoredKind::Type:;
+  }
+  llvm_unreachable("unexpected stored kind");
+}
+
+struct NamespaceAndPrefixLoc;
+
+/// A C++ nested-name-specifier augmented with source location
+/// information.
+class NestedNameSpecifierLoc {
+  NestedNameSpecifier Qualifier = std::nullopt;
+  void *Data = nullptr;
+
+  /// Load a (possibly unaligned) source location from a given address
+  /// and offset.
+  SourceLocation LoadSourceLocation(unsigned Offset) const {
+    SourceLocation::UIntTy Raw;
+    memcpy(&Raw, static_cast<char *>(Data) + Offset, sizeof(Raw));
+    return SourceLocation::getFromRawEncoding(Raw);
+  }
+
+  /// Load a (possibly unaligned) pointer from a given address and
+  /// offset.
+  void *LoadPointer(unsigned Offset) const {
+    void *Result;
+    memcpy(&Result, static_cast<char *>(Data) + Offset, sizeof(void *));
+    return Result;
+  }
+
+  /// Determines the data length for the last component in the
+  /// given nested-name-specifier.
+  static inline unsigned getLocalDataLength(NestedNameSpecifier Qualifier);
+
+  /// Determines the data length for the entire
+  /// nested-name-specifier.
+  static inline unsigned getDataLength(NestedNameSpecifier Qualifier);
+
+public:
+  /// Construct an empty nested-name-specifier.
+  NestedNameSpecifierLoc() = default;
+
+  /// Construct a nested-name-specifier with source location information
+  /// from
+  NestedNameSpecifierLoc(NestedNameSpecifier Qualifier, void *Data)
+      : Qualifier(Qualifier), Data(Data) {}
+
+  /// Evaluates true when this nested-name-specifier location is
+  /// non-empty.
+  explicit operator bool() const { return bool(Qualifier); }
+
+  /// Evaluates true when this nested-name-specifier location is
+  /// non-empty.
+  bool hasQualifier() const { return bool(Qualifier); }
+
+  /// Retrieve the nested-name-specifier to which this instance
+  /// refers.
+  NestedNameSpecifier getNestedNameSpecifier() const { return Qualifier; }
+
+  /// Retrieve the opaque pointer that refers to source-location data.
+  void *getOpaqueData() const { return Data; }
+
+  /// Retrieve the source range covering the entirety of this
+  /// nested-name-specifier.
+  ///
+  /// For example, if this instance refers to a nested-name-specifier
+  /// \c \::std::vector<int>::, the returned source range would cover
+  /// from the initial '::' to the last '::'.
+  inline SourceRange getSourceRange() const LLVM_READONLY;
+
+  /// Retrieve the source range covering just the last part of
+  /// this nested-name-specifier, not including the prefix.
+  ///
+  /// For example, if this instance refers to a nested-name-specifier
+  /// \c \::std::vector<int>::, the returned source range would cover
+  /// from "vector" to the last '::'.
+  inline SourceRange getLocalSourceRange() const;
+
+  /// Retrieve the location of the beginning of this
+  /// nested-name-specifier.
+  SourceLocation getBeginLoc() const;
+
+  /// Retrieve the location of the end of this
+  /// nested-name-specifier.
+  inline SourceLocation getEndLoc() const;
+
+  /// Retrieve the location of the beginning of this
+  /// component of the nested-name-specifier.
+  inline SourceLocation getLocalBeginLoc() const;
+
+  /// Retrieve the location of the end of this component of the
+  /// nested-name-specifier.
+  inline SourceLocation getLocalEndLoc() const;
+
+  /// For a nested-name-specifier that refers to a namespace,
+  /// retrieve the namespace and its prefix.
+  ///
+  /// For example, if this instance refers to a nested-name-specifier
+  /// \c \::std::chrono::, the prefix is \c \::std::. Note that the
+  /// returned prefix may be empty, if this is the first component of
+  /// the nested-name-specifier.
+  inline NamespaceAndPrefixLoc castAsNamespaceAndPrefix() const;
+  inline NamespaceAndPrefixLoc getAsNamespaceAndPrefix() const;
+
+  /// For a nested-name-specifier that refers to a type,
+  /// retrieve the type with source-location information.
+  inline TypeLoc castAsTypeLoc() const;
+  inline TypeLoc getAsTypeLoc() const;
+
+  /// Determines the data length for the entire
+  /// nested-name-specifier.
+  inline unsigned getDataLength() const;
+
+  friend bool operator==(NestedNameSpecifierLoc X, NestedNameSpecifierLoc Y) {
+    return X.Qualifier == Y.Qualifier && X.Data == Y.Data;
+  }
+
+  friend bool operator!=(NestedNameSpecifierLoc X, NestedNameSpecifierLoc Y) {
+    return !(X == Y);
+  }
+};
+
+struct NamespaceAndPrefixLoc {
+  const NamespaceBaseDecl *Namespace = nullptr;
+  NestedNameSpecifierLoc Prefix;
+
+  explicit operator bool() const { return Namespace != nullptr; }
+};
+
+/// Class that aids in the construction of nested-name-specifiers along
+/// with source-location information for all of the components of the
+/// nested-name-specifier.
+class NestedNameSpecifierLocBuilder {
+  /// The current representation of the nested-name-specifier we're
+  /// building.
+  NestedNameSpecifier Representation = std::nullopt;
+
+  /// Buffer used to store source-location information for the
+  /// nested-name-specifier.
+  ///
+  /// Note that we explicitly manage the buffer (rather than using a
+  /// SmallVector) because \c Declarator expects it to be possible to memcpy()
+  /// a \c CXXScopeSpec, and CXXScopeSpec uses a NestedNameSpecifierLocBuilder.
+  char *Buffer = nullptr;
+
+  /// The size of the buffer used to store source-location information
+  /// for the nested-name-specifier.
+  unsigned BufferSize = 0;
+
+  /// The capacity of the buffer used to store source-location
+  /// information for the nested-name-specifier.
+  unsigned BufferCapacity = 0;
+
+  void PushTrivial(ASTContext &Context, NestedNameSpecifier Qualifier,
+                   SourceRange R);
+
+public:
+  NestedNameSpecifierLocBuilder() = default;
+  NestedNameSpecifierLocBuilder(const NestedNameSpecifierLocBuilder &Other);
+
+  NestedNameSpecifierLocBuilder &
+  operator=(const NestedNameSpecifierLocBuilder &Other);
+
+  ~NestedNameSpecifierLocBuilder() {
+    if (BufferCapacity)
+      free(Buffer);
+  }
+
+  /// Retrieve the representation of the nested-name-specifier.
+  NestedNameSpecifier getRepresentation() const { return Representation; }
+
+  /// Make a nested-name-specifier of the form 'type::'.
+  ///
+  /// \param Context The AST context in which this nested-name-specifier
+  /// resides.
+  ///
+  /// \param TL The TypeLoc that describes the type preceding the '::'.
+  ///
+  /// \param ColonColonLoc The location of the trailing '::'.
+  void Make(ASTContext &Context, TypeLoc TL, SourceLocation ColonColonLoc);
+
+  /// Extend the current nested-name-specifier by another
+  /// nested-name-specifier component of the form 'namespace::'.
+  ///
+  /// \param Context The AST context in which this nested-name-specifier
+  /// resides.
+  ///
+  /// \param Namespace The namespace.
+  ///
+  /// \param NamespaceLoc The location of the namespace name.
+  ///
+  /// \param ColonColonLoc The location of the trailing '::'.
+  void Extend(ASTContext &Context, const NamespaceBaseDecl *Namespace,
+              SourceLocation NamespaceLoc, SourceLocation ColonColonLoc);
+
+  /// Turn this (empty) nested-name-specifier into the global
+  /// nested-name-specifier '::'.
+  void MakeGlobal(ASTContext &Context, SourceLocation ColonColonLoc);
+
+  /// Turns this (empty) nested-name-specifier into '__super'
+  /// nested-name-specifier.
+  ///
+  /// \param Context The AST context in which this nested-name-specifier
+  /// resides.
+  ///
+  /// \param RD The declaration of the class in which nested-name-specifier
+  /// appeared.
+  ///
+  /// \param SuperLoc The location of the '__super' keyword.
+  /// name.
+  ///
+  /// \param ColonColonLoc The location of the trailing '::'.
+  void MakeMicrosoftSuper(ASTContext &Context, CXXRecordDecl *RD,
+                          SourceLocation SuperLoc,
+                          SourceLocation ColonColonLoc);
+
+  /// Make a new nested-name-specifier from incomplete source-location
+  /// information.
+  ///
+  /// This routine should be used very, very rarely, in cases where we
+  /// need to synthesize a nested-name-specifier. Most code should instead use
+  /// \c Adopt() with a proper \c NestedNameSpecifierLoc.
+  void MakeTrivial(ASTContext &Context, NestedNameSpecifier Qualifier,
+                   SourceRange R) {
+    Representation = Qualifier;
+    BufferSize = 0;
+    PushTrivial(Context, Qualifier, R);
+  }
+
+  /// Adopt an existing nested-name-specifier (with source-range
+  /// information).
+  void Adopt(NestedNameSpecifierLoc Other);
+
+  /// Retrieve the source range covered by this nested-name-specifier.
+  inline SourceRange getSourceRange() const LLVM_READONLY;
+
+  /// Retrieve a nested-name-specifier with location information,
+  /// copied into the given AST context.
+  ///
+  /// \param Context The context into which this nested-name-specifier will be
+  /// copied.
+  NestedNameSpecifierLoc getWithLocInContext(ASTContext &Context) const;
+
+  /// Retrieve a nested-name-specifier with location
+  /// information based on the information in this builder.
+  ///
+  /// This loc will contain references to the builder's internal data and may
+  /// be invalidated by any change to the builder.
+  NestedNameSpecifierLoc getTemporary() const {
+    return NestedNameSpecifierLoc(Representation, Buffer);
+  }
+
+  /// Clear out this builder, and prepare it to build another
+  /// nested-name-specifier with source-location information.
+  void Clear() {
+    Representation = std::nullopt;
+    BufferSize = 0;
+  }
+
+  /// Retrieve the underlying buffer.
+  ///
+  /// \returns A pair containing a pointer to the buffer of source-location
+  /// data and the size of the source-location data that resides in that
+  /// buffer.
+  std::pair<char *, unsigned> getBuffer() const {
+    return std::make_pair(Buffer, BufferSize);
+  }
+};
+
+/// Insertion operator for diagnostics.  This allows sending
+/// NestedNameSpecifiers into a diagnostic with <<.
+inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &DB,
+                                             NestedNameSpecifier NNS) {
+  DB.AddTaggedVal(reinterpret_cast<uintptr_t>(NNS.getAsVoidPointer()),
+                  DiagnosticsEngine::ak_nestednamespec);
+  return DB;
+}
+
+} // namespace clang
+
+namespace llvm {
+
+template <> struct PointerLikeTypeTraits<clang::NestedNameSpecifier> {
+  static void *getAsVoidPointer(clang::NestedNameSpecifier P) {
+    return P.getAsVoidPointer();
+  }
+  static clang::NestedNameSpecifier getFromVoidPointer(const void *P) {
+    return clang::NestedNameSpecifier::getFromVoidPointer(P);
+  }
+  static constexpr int NumLowBitsAvailable =
+      clang::NestedNameSpecifier::NumLowBitsAvailable;
+};
+
+} // namespace llvm
+
+#endif // LLVM_CLANG_AST_NESTEDNAMESPECIFIERBASE_H
diff --git a/clang/include/clang/AST/ODRHash.h b/clang/include/clang/AST/ODRHash.h
index 11f917a1a4634..ae3fab697786a 100644
--- a/clang/include/clang/AST/ODRHash.h
+++ b/clang/include/clang/AST/ODRHash.h
@@ -93,7 +93,7 @@ class ODRHash {
   void AddQualType(QualType T);
   void AddStmt(const Stmt *S);
   void AddIdentifierInfo(const IdentifierInfo *II);
-  void AddNestedNameSpecifier(const NestedNameSpecifier *NNS);
+  void AddNestedNameSpecifier(NestedNameSpecifier NNS);
   void AddDependentTemplateName(const DependentTemplateStorage &Name);
   void AddTemplateName(TemplateName Name);
   void AddDeclarationNameInfo(DeclarationNameInfo NameInfo,
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 8c848a10d40ad..2f4aba1cdcd90 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -876,21 +876,32 @@ class OpenACCPrivateClause final
   }
 };
 
+// A 'pair' to stand in for the recipe.  RecipeDecl is the main declaration, and
+// InitFromTemporary is the 'temp' declaration we put in to be 'copied from'.
+struct OpenACCFirstPrivateRecipe {
+  VarDecl *RecipeDecl, *InitFromTemporary;
+  OpenACCFirstPrivateRecipe(VarDecl *R, VarDecl *T)
+      : RecipeDecl(R), InitFromTemporary(T) {}
+  OpenACCFirstPrivateRecipe(std::pair<VarDecl *, VarDecl *> p)
+      : RecipeDecl(p.first), InitFromTemporary(p.second) {}
+};
+
 class OpenACCFirstPrivateClause final
     : public OpenACCClauseWithVarList,
       private llvm::TrailingObjects<OpenACCFirstPrivateClause, Expr *,
-                                    VarDecl *> {
+                                    OpenACCFirstPrivateRecipe> {
   friend TrailingObjects;
 
   OpenACCFirstPrivateClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
                             ArrayRef<Expr *> VarList,
-                            ArrayRef<VarDecl *> InitRecipes,
+                            ArrayRef<OpenACCFirstPrivateRecipe> InitRecipes,
                             SourceLocation EndLoc)
       : OpenACCClauseWithVarList(OpenACCClauseKind::FirstPrivate, BeginLoc,
                                  LParenLoc, EndLoc) {
     assert(VarList.size() == InitRecipes.size());
     setExprs(getTrailingObjects<Expr *>(VarList.size()), VarList);
-    llvm::uninitialized_copy(InitRecipes, getTrailingObjects<VarDecl *>());
+    llvm::uninitialized_copy(InitRecipes,
+                             getTrailingObjects<OpenACCFirstPrivateRecipe>());
   }
 
 public:
@@ -900,19 +911,20 @@ class OpenACCFirstPrivateClause final
 
   // Gets a list of 'made up' `VarDecl` objects that can be used by codegen to
   // ensure that we properly initialize each of these variables.
-  ArrayRef<VarDecl *> getInitRecipes() {
-    return ArrayRef<VarDecl *>{getTrailingObjects<VarDecl *>(),
-                               getExprs().size()};
+  ArrayRef<OpenACCFirstPrivateRecipe> getInitRecipes() {
+    return ArrayRef<OpenACCFirstPrivateRecipe>{
+        getTrailingObjects<OpenACCFirstPrivateRecipe>(), getExprs().size()};
   }
 
-  ArrayRef<VarDecl *> getInitRecipes() const {
-    return ArrayRef<VarDecl *>{getTrailingObjects<VarDecl *>(),
-                               getExprs().size()};
+  ArrayRef<OpenACCFirstPrivateRecipe> getInitRecipes() const {
+    return ArrayRef<OpenACCFirstPrivateRecipe>{
+        getTrailingObjects<OpenACCFirstPrivateRecipe>(), getExprs().size()};
   }
 
   static OpenACCFirstPrivateClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
-         ArrayRef<Expr *> VarList, ArrayRef<VarDecl *> InitRecipes,
+         ArrayRef<Expr *> VarList,
+         ArrayRef<OpenACCFirstPrivateRecipe> InitRecipes,
          SourceLocation EndLoc);
 
   size_t numTrailingObjects(OverloadToken<Expr *>) const {
@@ -1238,19 +1250,32 @@ class OpenACCCreateClause final
          SourceLocation EndLoc);
 };
 
+// A structure to stand in for the recipe on a reduction.  RecipeDecl is the
+// 'main' declaration used for initializaiton, which is fixed. 
+struct OpenACCReductionRecipe {
+  VarDecl *RecipeDecl;
+  // TODO: OpenACC: this should eventually have the operations here too.
+};
+
 class OpenACCReductionClause final
     : public OpenACCClauseWithVarList,
-      private llvm::TrailingObjects<OpenACCReductionClause, Expr *> {
+      private llvm::TrailingObjects<OpenACCReductionClause, Expr *,
+                                    OpenACCReductionRecipe> {
   friend TrailingObjects;
   OpenACCReductionOperator Op;
 
   OpenACCReductionClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
                          OpenACCReductionOperator Operator,
-                         ArrayRef<Expr *> VarList, SourceLocation EndLoc)
+                         ArrayRef<Expr *> VarList,
+                         ArrayRef<OpenACCReductionRecipe> Recipes,
+                         SourceLocation EndLoc)
       : OpenACCClauseWithVarList(OpenACCClauseKind::Reduction, BeginLoc,
                                  LParenLoc, EndLoc),
         Op(Operator) {
-    setExprs(getTrailingObjects(VarList.size()), VarList);
+          assert(VarList.size() == Recipes.size());
+    setExprs(getTrailingObjects<Expr *>(VarList.size()), VarList);
+    llvm::uninitialized_copy(Recipes, getTrailingObjects<
+                             OpenACCReductionRecipe > ());
   }
 
 public:
@@ -1258,12 +1283,26 @@ class OpenACCReductionClause final
     return C->getClauseKind() == OpenACCClauseKind::Reduction;
   }
 
+  ArrayRef<OpenACCReductionRecipe> getRecipes() {
+    return ArrayRef<OpenACCReductionRecipe>{
+        getTrailingObjects<OpenACCReductionRecipe>(), getExprs().size()};
+  }
+
+  ArrayRef<OpenACCReductionRecipe> getRecipes() const {
+    return ArrayRef<OpenACCReductionRecipe>{
+        getTrailingObjects<OpenACCReductionRecipe>(), getExprs().size()};
+  }
+
   static OpenACCReductionClause *
   Create(const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
          OpenACCReductionOperator Operator, ArrayRef<Expr *> VarList,
-         SourceLocation EndLoc);
+         ArrayRef<OpenACCReductionRecipe> Recipes, SourceLocation EndLoc);
 
   OpenACCReductionOperator getReductionOp() const { return Op; }
+
+  size_t numTrailingObjects(OverloadToken<Expr *>) const {
+    return getExprs().size();
+  }
 };
 
 class OpenACCLinkClause final
diff --git a/clang/include/clang/AST/PrettyPrinter.h b/clang/include/clang/AST/PrettyPrinter.h
index 875769cd02bc0..fd995a653d167 100644
--- a/clang/include/clang/AST/PrettyPrinter.h
+++ b/clang/include/clang/AST/PrettyPrinter.h
@@ -63,9 +63,9 @@ struct PrintingPolicy {
         SuppressTagKeyword(LO.CPlusPlus), IncludeTagDefinition(false),
         SuppressScope(false), SuppressUnwrittenScope(false),
         SuppressInlineNamespace(SuppressInlineNamespaceMode::Redundant),
-        SuppressElaboration(false), SuppressInitializers(false),
-        ConstantArraySizeAsWritten(false), AnonymousTagLocations(true),
-        SuppressStrongLifetime(false), SuppressLifetimeQualifiers(false),
+        SuppressInitializers(false), ConstantArraySizeAsWritten(false),
+        AnonymousTagLocations(true), SuppressStrongLifetime(false),
+        SuppressLifetimeQualifiers(false),
         SuppressTemplateArgsInCXXConstructors(false),
         SuppressDefaultTemplateArgs(true), Bool(LO.Bool),
         Nullptr(LO.CPlusPlus11 || LO.C23), NullptrTypeInNamespace(LO.CPlusPlus),
@@ -150,11 +150,6 @@ struct PrintingPolicy {
   LLVM_PREFERRED_TYPE(SuppressInlineNamespaceMode)
   unsigned SuppressInlineNamespace : 2;
 
-  /// Ignore qualifiers and tag keywords as specified by elaborated type sugar,
-  /// instead letting the underlying type print as normal.
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned SuppressElaboration : 1;
-
   /// Suppress printing of variable initializers.
   ///
   /// This flag is used when printing the loop variable in a for-range
diff --git a/clang/include/clang/AST/PropertiesBase.td b/clang/include/clang/AST/PropertiesBase.td
index 0438e4dfbafac..5b10127526e4e 100644
--- a/clang/include/clang/AST/PropertiesBase.td
+++ b/clang/include/clang/AST/PropertiesBase.td
@@ -127,9 +127,8 @@ def LValuePathSerializationHelper :
     PropertyType<"APValue::LValuePathSerializationHelper"> {
   let BufferElementTypes = [ LValuePathEntry ];
 }
-def NestedNameSpecifier : PropertyType<"NestedNameSpecifier *">;
-def NestedNameSpecifierKind :
-  EnumPropertyType<"NestedNameSpecifier::SpecifierKind">;
+def NestedNameSpecifier : PropertyType<"NestedNameSpecifier">;
+def NestedNameSpecifierKind : EnumPropertyType<"NestedNameSpecifier::Kind">;
 def OverloadedOperatorKind : EnumPropertyType;
 def Qualifiers : PropertyType;
 def QualType : DefaultValuePropertyType;
diff --git a/clang/include/clang/AST/QualTypeNames.h b/clang/include/clang/AST/QualTypeNames.h
index daa86cda2d992..9f5cf045d1d4c 100644
--- a/clang/include/clang/AST/QualTypeNames.h
+++ b/clang/include/clang/AST/QualTypeNames.h
@@ -87,6 +87,16 @@ std::string getFullyQualifiedName(QualType QT, const ASTContext &Ctx,
 /// specifier "::" should be prepended or not.
 QualType getFullyQualifiedType(QualType QT, const ASTContext &Ctx,
                                bool WithGlobalNsPrefix = false);
+
+/// Get the fully qualified name for the declared context of a declaration.
+///
+/// \param[in] Ctx - the ASTContext to be used.
+/// \param[in] Decl - the declaration for which to get the fully qualified name.
+/// \param[in] WithGlobalNsPrefix - If true, then the global namespace
+/// specifier "::" will be prepended to the fully qualified name.
+NestedNameSpecifier
+getFullyQualifiedDeclaredContext(const ASTContext &Ctx, const Decl *Decl,
+                                 bool WithGlobalNsPrefix = false);
 } // end namespace TypeName
 } // end namespace clang
 #endif // LLVM_CLANG_AST_QUALTYPENAMES_H
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 62991d986e675..02581c8e73299 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -216,14 +216,14 @@ template <typename Derived> class RecursiveASTVisitor {
   ///
   /// \returns false if the visitation was terminated early, true
   /// otherwise (including when the argument is a Null type).
-  bool TraverseType(QualType T);
+  bool TraverseType(QualType T, bool TraverseQualifier = true);
 
   /// Recursively visit a type with location, by dispatching to
   /// Traverse*TypeLoc() based on the argument type's getTypeClass() property.
   ///
   /// \returns false if the visitation was terminated early, true
   /// otherwise (including when the argument is a Null type location).
-  bool TraverseTypeLoc(TypeLoc TL);
+  bool TraverseTypeLoc(TypeLoc TL, bool TraverseQualifier = true);
 
   /// Recursively visit an attribute, by dispatching to
   /// Traverse*Attr() based on the argument's dynamic type.
@@ -242,7 +242,7 @@ template <typename Derived> class RecursiveASTVisitor {
   /// Recursively visit a C++ nested-name-specifier.
   ///
   /// \returns false if the visitation was terminated early, true otherwise.
-  bool TraverseNestedNameSpecifier(NestedNameSpecifier *NNS);
+  bool TraverseNestedNameSpecifier(NestedNameSpecifier NNS);
 
   /// Recursively visit a C++ nested-name-specifier with location
   /// information.
@@ -389,7 +389,8 @@ template <typename Derived> class RecursiveASTVisitor {
 
 // Declare Traverse*() for all concrete Type classes.
 #define ABSTRACT_TYPE(CLASS, BASE)
-#define TYPE(CLASS, BASE) bool Traverse##CLASS##Type(CLASS##Type *T);
+#define TYPE(CLASS, BASE)                                                      \
+  bool Traverse##CLASS##Type(CLASS##Type *T, bool TraverseQualifier);
 #include "clang/AST/TypeNodes.inc"
   // The above header #undefs ABSTRACT_TYPE and TYPE upon exit.
 
@@ -410,7 +411,8 @@ template <typename Derived> class RecursiveASTVisitor {
 
 // Declare Traverse*() for all concrete TypeLoc classes.
 #define ABSTRACT_TYPELOC(CLASS, BASE)
-#define TYPELOC(CLASS, BASE) bool Traverse##CLASS##TypeLoc(CLASS##TypeLoc TL);
+#define TYPELOC(CLASS, BASE)                                                   \
+  bool Traverse##CLASS##TypeLoc(CLASS##TypeLoc TL, bool TraverseQualifier);
 #include "clang/AST/TypeLocNodes.def"
   // The above header #undefs ABSTRACT_TYPELOC and TYPELOC upon exit.
 
@@ -490,6 +492,8 @@ template <typename Derived> class RecursiveASTVisitor {
   bool TraverseTemplateArgumentLocsHelper(const TemplateArgumentLoc *TAL,
                                           unsigned Count);
   bool TraverseArrayTypeLocHelper(ArrayTypeLoc TL);
+  bool TraverseSubstPackTypeHelper(SubstPackType *T);
+  bool TraverseSubstPackTypeLocHelper(SubstPackTypeLoc TL);
   bool TraverseRecordHelper(RecordDecl *D);
   bool TraverseCXXRecordHelper(CXXRecordDecl *D);
   bool TraverseDeclaratorHelper(DeclaratorDecl *D);
@@ -499,6 +503,8 @@ template <typename Derived> class RecursiveASTVisitor {
   bool TraverseOMPExecutableDirective(OMPExecutableDirective *S);
   bool TraverseOMPLoopDirective(OMPLoopDirective *S);
   bool TraverseOMPClause(OMPClause *C);
+  bool TraverseTagType(TagType *T, bool TraverseQualifier);
+  bool TraverseTagTypeLoc(TagTypeLoc TL, bool TraverseQualifier);
 #define GEN_CLANG_CLAUSE_CLASS
 #define CLAUSE_CLASS(Enum, Str, Class) bool Visit##Class(Class *C);
 #include "llvm/Frontend/OpenMP/OMP.inc"
@@ -698,7 +704,8 @@ RecursiveASTVisitor<Derived>::TraverseStmt(Stmt *S, DataRecursionQueue *Queue) {
 }
 
 template <typename Derived>
-bool RecursiveASTVisitor<Derived>::TraverseType(QualType T) {
+bool RecursiveASTVisitor<Derived>::TraverseType(QualType T,
+                                                bool TraverseQualifier) {
   if (T.isNull())
     return true;
 
@@ -707,7 +714,8 @@ bool RecursiveASTVisitor<Derived>::TraverseType(QualType T) {
 #define TYPE(CLASS, BASE)                                                      \
   case Type::CLASS:                                                            \
     return getDerived().Traverse##CLASS##Type(                                 \
-        static_cast<CLASS##Type *>(const_cast<Type *>(T.getTypePtr())));
+        static_cast<CLASS##Type *>(const_cast<Type *>(T.getTypePtr())),        \
+        TraverseQualifier);
 #include "clang/AST/TypeNodes.inc"
   }
 
@@ -715,7 +723,8 @@ bool RecursiveASTVisitor<Derived>::TraverseType(QualType T) {
 }
 
 template <typename Derived>
-bool RecursiveASTVisitor<Derived>::TraverseTypeLoc(TypeLoc TL) {
+bool RecursiveASTVisitor<Derived>::TraverseTypeLoc(TypeLoc TL,
+                                                   bool TraverseQualifier) {
   if (TL.isNull())
     return true;
 
@@ -723,7 +732,8 @@ bool RecursiveASTVisitor<Derived>::TraverseTypeLoc(TypeLoc TL) {
 #define ABSTRACT_TYPELOC(CLASS, BASE)
 #define TYPELOC(CLASS, BASE)                                                   \
   case TypeLoc::CLASS:                                                         \
-    return getDerived().Traverse##CLASS##TypeLoc(TL.castAs<CLASS##TypeLoc>());
+    return getDerived().Traverse##CLASS##TypeLoc(TL.castAs<CLASS##TypeLoc>(),  \
+                                                 TraverseQualifier);
 #include "clang/AST/TypeLocNodes.def"
   }
 
@@ -779,46 +789,43 @@ bool RecursiveASTVisitor<Derived>::TraverseDecl(Decl *D) {
 
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::TraverseNestedNameSpecifier(
-    NestedNameSpecifier *NNS) {
-  if (!NNS)
+    NestedNameSpecifier NNS) {
+  switch (NNS.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     return true;
-
-  if (NNS->getPrefix())
-    TRY_TO(TraverseNestedNameSpecifier(NNS->getPrefix()));
-
-  switch (NNS->getKind()) {
-  case NestedNameSpecifier::Identifier:
-  case NestedNameSpecifier::Namespace:
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Super:
+  case NestedNameSpecifier::Kind::Namespace:
+    TRY_TO(TraverseNestedNameSpecifier(NNS.getAsNamespaceAndPrefix().Prefix));
+    return true;
+  case NestedNameSpecifier::Kind::Type: {
+    auto *T = const_cast<Type *>(NNS.getAsType());
+    TRY_TO(TraverseNestedNameSpecifier(T->getPrefix()));
+    TRY_TO(TraverseType(QualType(T, 0), /*TraverseQualifier=*/false));
     return true;
-
-  case NestedNameSpecifier::TypeSpec:
-    TRY_TO(TraverseType(QualType(NNS->getAsType(), 0)));
   }
-
-  return true;
+  }
+  llvm_unreachable("unhandled kind");
 }
 
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::TraverseNestedNameSpecifierLoc(
     NestedNameSpecifierLoc NNS) {
-  if (!NNS)
+  switch (NNS.getNestedNameSpecifier().getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     return true;
-
-  if (NestedNameSpecifierLoc Prefix = NNS.getPrefix())
-    TRY_TO(TraverseNestedNameSpecifierLoc(Prefix));
-
-  switch (NNS.getNestedNameSpecifier()->getKind()) {
-  case NestedNameSpecifier::Identifier:
-  case NestedNameSpecifier::Namespace:
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Super:
+  case NestedNameSpecifier::Kind::Namespace:
+    TRY_TO(
+        TraverseNestedNameSpecifierLoc(NNS.castAsNamespaceAndPrefix().Prefix));
     return true;
-
-  case NestedNameSpecifier::TypeSpec:
-    TRY_TO(TraverseTypeLoc(NNS.getTypeLoc()));
-    break;
+  case NestedNameSpecifier::Kind::Type: {
+    TypeLoc TL = NNS.castAsTypeLoc();
+    TRY_TO(TraverseNestedNameSpecifierLoc(TL.getPrefix()));
+    TRY_TO(TraverseTypeLoc(TL, /*TraverseQualifier=*/false));
+    return true;
+  }
   }
 
   return true;
@@ -975,10 +982,13 @@ RecursiveASTVisitor<Derived>::TraverseLambdaCapture(LambdaExpr *LE,
 // This macro makes available a variable T, the passed-in type.
 #define DEF_TRAVERSE_TYPE(TYPE, CODE)                                          \
   template <typename Derived>                                                  \
-  bool RecursiveASTVisitor<Derived>::Traverse##TYPE(TYPE *T) {                 \
+  bool RecursiveASTVisitor<Derived>::Traverse##TYPE(TYPE *T,                   \
+                                                    bool TraverseQualifier) {  \
     if (!getDerived().shouldTraversePostOrder())                               \
       TRY_TO(WalkUpFrom##TYPE(T));                                             \
-    { CODE; }                                                                  \
+    {                                                                          \
+      CODE;                                                                    \
+    }                                                                          \
     if (getDerived().shouldTraversePostOrder())                                \
       TRY_TO(WalkUpFrom##TYPE(T));                                             \
     return true;                                                               \
@@ -1000,10 +1010,11 @@ DEF_TRAVERSE_TYPE(RValueReferenceType,
                   { TRY_TO(TraverseType(T->getPointeeType())); })
 
 DEF_TRAVERSE_TYPE(MemberPointerType, {
-  TRY_TO(TraverseNestedNameSpecifier(T->getQualifier()));
-  if (T->isSugared())
-    TRY_TO(TraverseType(
-        QualType(T->getMostRecentCXXRecordDecl()->getTypeForDecl(), 0)));
+  NestedNameSpecifier Qualifier =
+      T->isSugared() ? cast<MemberPointerType>(T->getCanonicalTypeUnqualified())
+                           ->getQualifier()
+                     : T->getQualifier();
+  TRY_TO(TraverseNestedNameSpecifier(Qualifier));
   TRY_TO(TraverseType(T->getPointeeType()));
 })
 
@@ -1087,9 +1098,18 @@ DEF_TRAVERSE_TYPE(FunctionProtoType, {
     TRY_TO(TraverseStmt(NE));
 })
 
-DEF_TRAVERSE_TYPE(UsingType, {})
-DEF_TRAVERSE_TYPE(UnresolvedUsingType, {})
-DEF_TRAVERSE_TYPE(TypedefType, {})
+DEF_TRAVERSE_TYPE(UsingType, {
+  if (TraverseQualifier)
+    TRY_TO(TraverseNestedNameSpecifier(T->getQualifier()));
+})
+DEF_TRAVERSE_TYPE(UnresolvedUsingType, {
+  if (TraverseQualifier)
+    TRY_TO(TraverseNestedNameSpecifier(T->getQualifier()));
+})
+DEF_TRAVERSE_TYPE(TypedefType, {
+  if (TraverseQualifier)
+    TRY_TO(TraverseNestedNameSpecifier(T->getQualifier()));
+})
 
 DEF_TRAVERSE_TYPE(TypeOfExprType,
                   { TRY_TO(TraverseStmt(T->getUnderlyingExpr())); })
@@ -1115,27 +1135,15 @@ DEF_TRAVERSE_TYPE(AutoType, {
     TRY_TO(TraverseTemplateArguments(T->getTypeConstraintArguments()));
   }
 })
-DEF_TRAVERSE_TYPE(DeducedTemplateSpecializationType, {
-  TRY_TO(TraverseTemplateName(T->getTemplateName()));
-  TRY_TO(TraverseType(T->getDeducedType()));
-})
 
-DEF_TRAVERSE_TYPE(RecordType, {})
-DEF_TRAVERSE_TYPE(EnumType, {})
 DEF_TRAVERSE_TYPE(TemplateTypeParmType, {})
 DEF_TRAVERSE_TYPE(SubstTemplateTypeParmType, {
   TRY_TO(TraverseType(T->getReplacementType()));
 })
-DEF_TRAVERSE_TYPE(SubstTemplateTypeParmPackType, {
-  TRY_TO(TraverseTemplateArgument(T->getArgumentPack()));
-})
-
-DEF_TRAVERSE_TYPE(TemplateSpecializationType, {
-  TRY_TO(TraverseTemplateName(T->getTemplateName()));
-  TRY_TO(TraverseTemplateArguments(T->template_arguments()));
-})
-
-DEF_TRAVERSE_TYPE(InjectedClassNameType, {})
+DEF_TRAVERSE_TYPE(SubstTemplateTypeParmPackType,
+                  { TRY_TO(TraverseSubstPackTypeHelper(T)); })
+DEF_TRAVERSE_TYPE(SubstBuiltinTemplatePackType,
+                  { TRY_TO(TraverseSubstPackTypeHelper(T)); })
 
 DEF_TRAVERSE_TYPE(AttributedType,
                   { TRY_TO(TraverseType(T->getModifiedType())); })
@@ -1165,22 +1173,54 @@ DEF_TRAVERSE_TYPE(ParenType, { TRY_TO(TraverseType(T->getInnerType())); })
 DEF_TRAVERSE_TYPE(MacroQualifiedType,
                   { TRY_TO(TraverseType(T->getUnderlyingType())); })
 
-DEF_TRAVERSE_TYPE(ElaboratedType, {
-  if (T->getQualifier()) {
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::TraverseTagType(TagType *T,
+                                                   bool TraverseQualifier) {
+  if (TraverseQualifier)
     TRY_TO(TraverseNestedNameSpecifier(T->getQualifier()));
-  }
-  TRY_TO(TraverseType(T->getNamedType()));
-})
+  return true;
+}
 
-DEF_TRAVERSE_TYPE(DependentNameType,
-                  { TRY_TO(TraverseNestedNameSpecifier(T->getQualifier())); })
+DEF_TRAVERSE_TYPE(EnumType, { TRY_TO(TraverseTagType(T, TraverseQualifier)); })
+DEF_TRAVERSE_TYPE(RecordType,
+                  { TRY_TO(TraverseTagType(T, TraverseQualifier)); })
+DEF_TRAVERSE_TYPE(InjectedClassNameType,
+                  { TRY_TO(TraverseTagType(T, TraverseQualifier)); })
+
+DEF_TRAVERSE_TYPE(DependentNameType, {
+  if (TraverseQualifier)
+    TRY_TO(TraverseNestedNameSpecifier(T->getQualifier()));
+})
 
 DEF_TRAVERSE_TYPE(DependentTemplateSpecializationType, {
   const DependentTemplateStorage &S = T->getDependentTemplateName();
-  TRY_TO(TraverseNestedNameSpecifier(S.getQualifier()));
+  if (TraverseQualifier)
+    TRY_TO(TraverseNestedNameSpecifier(S.getQualifier()));
+  TRY_TO(TraverseTemplateArguments(T->template_arguments()));
+})
+
+DEF_TRAVERSE_TYPE(TemplateSpecializationType, {
+  if (TraverseQualifier) {
+    TRY_TO(TraverseTemplateName(T->getTemplateName()));
+  } else {
+    // FIXME: Try to preserve the rest of the template name.
+    TRY_TO(TraverseTemplateName(TemplateName(
+        T->getTemplateName().getAsTemplateDecl(/*IgnoreDeduced=*/true))));
+  }
   TRY_TO(TraverseTemplateArguments(T->template_arguments()));
 })
 
+DEF_TRAVERSE_TYPE(DeducedTemplateSpecializationType, {
+  if (TraverseQualifier) {
+    TRY_TO(TraverseTemplateName(T->getTemplateName()));
+  } else {
+    // FIXME: Try to preserve the rest of the template name.
+    TRY_TO(TraverseTemplateName(TemplateName(
+        T->getTemplateName().getAsTemplateDecl(/*IgnoreDeduced=*/true))));
+  }
+  TRY_TO(TraverseType(T->getDeducedType()));
+})
+
 DEF_TRAVERSE_TYPE(PackExpansionType, { TRY_TO(TraverseType(T->getPattern())); })
 
 DEF_TRAVERSE_TYPE(ObjCTypeParamType, {})
@@ -1221,13 +1261,16 @@ DEF_TRAVERSE_TYPE(PredefinedSugarType, {})
 // continue to work.
 #define DEF_TRAVERSE_TYPELOC(TYPE, CODE)                                       \
   template <typename Derived>                                                  \
-  bool RecursiveASTVisitor<Derived>::Traverse##TYPE##Loc(TYPE##Loc TL) {       \
+  bool RecursiveASTVisitor<Derived>::Traverse##TYPE##Loc(                      \
+      TYPE##Loc TL, bool TraverseQualifier) {                                  \
     if (!getDerived().shouldTraversePostOrder()) {                             \
       TRY_TO(WalkUpFrom##TYPE##Loc(TL));                                       \
       if (getDerived().shouldWalkTypesOfTypeLocs())                            \
         TRY_TO(WalkUpFrom##TYPE(const_cast<TYPE *>(TL.getTypePtr())));         \
     }                                                                          \
-    { CODE; }                                                                  \
+    {                                                                          \
+      CODE;                                                                    \
+    }                                                                          \
     if (getDerived().shouldTraversePostOrder()) {                              \
       TRY_TO(WalkUpFrom##TYPE##Loc(TL));                                       \
       if (getDerived().shouldWalkTypesOfTypeLocs())                            \
@@ -1237,8 +1280,10 @@ DEF_TRAVERSE_TYPE(PredefinedSugarType, {})
   }
 
 template <typename Derived>
-bool
-RecursiveASTVisitor<Derived>::TraverseQualifiedTypeLoc(QualifiedTypeLoc TL) {
+bool RecursiveASTVisitor<Derived>::TraverseQualifiedTypeLoc(
+    QualifiedTypeLoc TL, bool TraverseQualifier) {
+  assert(TraverseQualifier &&
+         "Qualifiers should never occur within NestedNameSpecifiers");
   // Move this over to the 'main' typeloc tree.  Note that this is a
   // move -- we pretend that we were really looking at the unqualified
   // typeloc all along -- rather than a recursion, so we don't follow
@@ -1391,9 +1436,21 @@ DEF_TRAVERSE_TYPELOC(FunctionProtoType, {
     TRY_TO(TraverseStmt(NE));
 })
 
-DEF_TRAVERSE_TYPELOC(UsingType, {})
-DEF_TRAVERSE_TYPELOC(UnresolvedUsingType, {})
-DEF_TRAVERSE_TYPELOC(TypedefType, {})
+DEF_TRAVERSE_TYPELOC(UsingType, {
+  if (NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+      TraverseQualifier && QualifierLoc)
+    TRY_TO(TraverseNestedNameSpecifierLoc(QualifierLoc));
+})
+DEF_TRAVERSE_TYPELOC(UnresolvedUsingType, {
+  if (NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+      TraverseQualifier && QualifierLoc)
+    TRY_TO(TraverseNestedNameSpecifierLoc(QualifierLoc));
+})
+DEF_TRAVERSE_TYPELOC(TypedefType, {
+  if (NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+      TraverseQualifier && QualifierLoc)
+    TRY_TO(TraverseNestedNameSpecifierLoc(QualifierLoc));
+})
 
 DEF_TRAVERSE_TYPELOC(TypeOfExprType,
                      { TRY_TO(TraverseStmt(TL.getUnderlyingExpr())); })
@@ -1423,30 +1480,30 @@ DEF_TRAVERSE_TYPELOC(AutoType, {
   }
 })
 
-DEF_TRAVERSE_TYPELOC(DeducedTemplateSpecializationType, {
-  TRY_TO(TraverseTemplateName(TL.getTypePtr()->getTemplateName()));
-  TRY_TO(TraverseType(TL.getTypePtr()->getDeducedType()));
-})
-
-DEF_TRAVERSE_TYPELOC(RecordType, {})
-DEF_TRAVERSE_TYPELOC(EnumType, {})
 DEF_TRAVERSE_TYPELOC(TemplateTypeParmType, {})
 DEF_TRAVERSE_TYPELOC(SubstTemplateTypeParmType, {
   TRY_TO(TraverseType(TL.getTypePtr()->getReplacementType()));
 })
-DEF_TRAVERSE_TYPELOC(SubstTemplateTypeParmPackType, {
+
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::TraverseSubstPackTypeLocHelper(
+    SubstPackTypeLoc TL) {
   TRY_TO(TraverseTemplateArgument(TL.getTypePtr()->getArgumentPack()));
-})
+  return true;
+}
 
-// FIXME: use the loc for the template name?
-DEF_TRAVERSE_TYPELOC(TemplateSpecializationType, {
-  TRY_TO(TraverseTemplateName(TL.getTypePtr()->getTemplateName()));
-  for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I) {
-    TRY_TO(TraverseTemplateArgumentLoc(TL.getArgLoc(I)));
-  }
-})
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::TraverseSubstPackTypeHelper(
+    SubstPackType *T) {
+  TRY_TO(TraverseTemplateArgument(T->getArgumentPack()));
+  return true;
+}
+
+DEF_TRAVERSE_TYPELOC(SubstTemplateTypeParmPackType,
+                     { TRY_TO(TraverseSubstPackTypeLocHelper(TL)); })
 
-DEF_TRAVERSE_TYPELOC(InjectedClassNameType, {})
+DEF_TRAVERSE_TYPELOC(SubstBuiltinTemplatePackType,
+                     { TRY_TO(TraverseSubstPackTypeLocHelper(TL)); })
 
 DEF_TRAVERSE_TYPELOC(ParenType, { TRY_TO(TraverseTypeLoc(TL.getInnerLoc())); })
 
@@ -1468,27 +1525,63 @@ DEF_TRAVERSE_TYPELOC(HLSLAttributedResourceType,
 DEF_TRAVERSE_TYPELOC(HLSLInlineSpirvType,
                      { TRY_TO(TraverseType(TL.getType())); })
 
-DEF_TRAVERSE_TYPELOC(ElaboratedType, {
-  if (TL.getQualifierLoc()) {
-    TRY_TO(TraverseNestedNameSpecifierLoc(TL.getQualifierLoc()));
-  }
-  TRY_TO(TraverseTypeLoc(TL.getNamedTypeLoc()));
-})
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::TraverseTagTypeLoc(TagTypeLoc TL,
+                                                      bool TraverseQualifier) {
+  if (NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+      TraverseQualifier && QualifierLoc)
+    TRY_TO(TraverseNestedNameSpecifierLoc(QualifierLoc));
+  return true;
+}
+
+DEF_TRAVERSE_TYPELOC(EnumType,
+                     { TRY_TO(TraverseTagTypeLoc(TL, TraverseQualifier)); })
+DEF_TRAVERSE_TYPELOC(RecordType,
+                     { TRY_TO(TraverseTagTypeLoc(TL, TraverseQualifier)); })
+DEF_TRAVERSE_TYPELOC(InjectedClassNameType,
+                     { TRY_TO(TraverseTagTypeLoc(TL, TraverseQualifier)); })
 
 DEF_TRAVERSE_TYPELOC(DependentNameType, {
-  TRY_TO(TraverseNestedNameSpecifierLoc(TL.getQualifierLoc()));
+  if (TraverseQualifier)
+    TRY_TO(TraverseNestedNameSpecifierLoc(TL.getQualifierLoc()));
 })
 
 DEF_TRAVERSE_TYPELOC(DependentTemplateSpecializationType, {
-  if (TL.getQualifierLoc()) {
+  if (TraverseQualifier)
     TRY_TO(TraverseNestedNameSpecifierLoc(TL.getQualifierLoc()));
+
+  for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I) {
+    TRY_TO(TraverseTemplateArgumentLoc(TL.getArgLoc(I)));
   }
+})
+
+DEF_TRAVERSE_TYPELOC(TemplateSpecializationType, {
+  if (TraverseQualifier)
+    TRY_TO(TraverseNestedNameSpecifierLoc(TL.getQualifierLoc()));
+
+  // FIXME: Try to preserve the rest of the template name.
+  TRY_TO(TraverseTemplateName(
+      TemplateName(TL.getTypePtr()->getTemplateName().getAsTemplateDecl(
+          /*IgnoreDeduced=*/true))));
 
   for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I) {
     TRY_TO(TraverseTemplateArgumentLoc(TL.getArgLoc(I)));
   }
 })
 
+DEF_TRAVERSE_TYPELOC(DeducedTemplateSpecializationType, {
+  if (TraverseQualifier)
+    TRY_TO(TraverseNestedNameSpecifierLoc(TL.getQualifierLoc()));
+
+  const auto *T = TL.getTypePtr();
+  // FIXME: Try to preserve the rest of the template name.
+  TRY_TO(
+      TraverseTemplateName(TemplateName(T->getTemplateName().getAsTemplateDecl(
+          /*IgnoreDeduced=*/true))));
+
+  TRY_TO(TraverseType(T->getDeducedType()));
+})
+
 DEF_TRAVERSE_TYPELOC(PackExpansionType,
                      { TRY_TO(TraverseTypeLoc(TL.getPatternLoc())); })
 
@@ -1631,8 +1724,9 @@ DEF_TRAVERSE_DECL(FriendDecl, {
     TRY_TO(TraverseTypeLoc(D->getFriendType()->getTypeLoc()));
     // Traverse any CXXRecordDecl owned by this type, since
     // it will not be in the parent context:
-    if (auto *ET = D->getFriendType()->getType()->getAs<ElaboratedType>())
-      TRY_TO(TraverseDecl(ET->getOwnedTagDecl()));
+    if (auto *TT = D->getFriendType()->getType()->getAs<TagType>();
+        TT && TT->isTagOwned())
+      TRY_TO(TraverseDecl(TT->getOriginalDecl()));
   } else {
     TRY_TO(TraverseDecl(D->getFriendDecl()));
   }
diff --git a/clang/include/clang/AST/TemplateBase.h b/clang/include/clang/AST/TemplateBase.h
index eb384eae3faa7..69481e85d87fa 100644
--- a/clang/include/clang/AST/TemplateBase.h
+++ b/clang/include/clang/AST/TemplateBase.h
@@ -15,7 +15,7 @@
 #define LLVM_CLANG_AST_TEMPLATEBASE_H
 
 #include "clang/AST/DependenceFlags.h"
-#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/NestedNameSpecifierBase.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/LLVM.h"
@@ -478,31 +478,25 @@ class TemplateArgument {
 
 /// Location information for a TemplateArgument.
 struct TemplateArgumentLocInfo {
-private:
   struct TemplateTemplateArgLocInfo {
-    // FIXME: We'd like to just use the qualifier in the TemplateName,
-    // but template arguments get canonicalized too quickly.
-    NestedNameSpecifier *Qualifier;
     void *QualifierLocData;
+    SourceLocation TemplateKwLoc;
     SourceLocation TemplateNameLoc;
     SourceLocation EllipsisLoc;
   };
 
-  llvm::PointerUnion<TemplateTemplateArgLocInfo *, Expr *, TypeSourceInfo *>
-      Pointer;
-
   TemplateTemplateArgLocInfo *getTemplate() const {
     return cast<TemplateTemplateArgLocInfo *>(Pointer);
   }
 
-public:
   TemplateArgumentLocInfo() {}
   TemplateArgumentLocInfo(TypeSourceInfo *Declarator) { Pointer = Declarator; }
 
   TemplateArgumentLocInfo(Expr *E) { Pointer = E; }
   // Ctx is used for allocation -- this case is unusually large and also rare,
   // so we store the payload out-of-line.
-  TemplateArgumentLocInfo(ASTContext &Ctx, NestedNameSpecifierLoc QualifierLoc,
+  TemplateArgumentLocInfo(ASTContext &Ctx, SourceLocation TemplateKwLoc,
+                          NestedNameSpecifierLoc QualifierLoc,
                           SourceLocation TemplateNameLoc,
                           SourceLocation EllipsisLoc);
 
@@ -512,10 +506,8 @@ struct TemplateArgumentLocInfo {
 
   Expr *getAsExpr() const { return cast<Expr *>(Pointer); }
 
-  NestedNameSpecifierLoc getTemplateQualifierLoc() const {
-    const auto *Template = getTemplate();
-    return NestedNameSpecifierLoc(Template->Qualifier,
-                                  Template->QualifierLocData);
+  SourceLocation getTemplateKwLoc() const {
+    return getTemplate()->TemplateKwLoc;
   }
 
   SourceLocation getTemplateNameLoc() const {
@@ -525,6 +517,10 @@ struct TemplateArgumentLocInfo {
   SourceLocation getTemplateEllipsisLoc() const {
     return getTemplate()->EllipsisLoc;
   }
+
+private:
+  llvm::PointerUnion<TemplateTemplateArgLocInfo *, Expr *, TypeSourceInfo *>
+      Pointer;
 };
 
 /// Location wrapper for a TemplateArgument.  TemplateArgument is to
@@ -558,14 +554,10 @@ class TemplateArgumentLoc {
   }
 
   TemplateArgumentLoc(ASTContext &Ctx, const TemplateArgument &Argument,
+                      SourceLocation TemplateKWLoc,
                       NestedNameSpecifierLoc QualifierLoc,
                       SourceLocation TemplateNameLoc,
-                      SourceLocation EllipsisLoc = SourceLocation())
-      : Argument(Argument),
-        LocInfo(Ctx, QualifierLoc, TemplateNameLoc, EllipsisLoc) {
-    assert(Argument.getKind() == TemplateArgument::Template ||
-           Argument.getKind() == TemplateArgument::TemplateExpansion);
-  }
+                      SourceLocation EllipsisLoc = SourceLocation());
 
   /// - Fetches the primary location of the argument.
   SourceLocation getLocation() const {
@@ -614,13 +606,15 @@ class TemplateArgumentLoc {
     return LocInfo.getAsExpr();
   }
 
-  NestedNameSpecifierLoc getTemplateQualifierLoc() const {
+  SourceLocation getTemplateKWLoc() const {
     if (Argument.getKind() != TemplateArgument::Template &&
         Argument.getKind() != TemplateArgument::TemplateExpansion)
-      return NestedNameSpecifierLoc();
-    return LocInfo.getTemplateQualifierLoc();
+      return SourceLocation();
+    return LocInfo.getTemplateKwLoc();
   }
 
+  NestedNameSpecifierLoc getTemplateQualifierLoc() const;
+
   SourceLocation getTemplateNameLoc() const {
     if (Argument.getKind() != TemplateArgument::Template &&
         Argument.getKind() != TemplateArgument::TemplateExpansion)
diff --git a/clang/include/clang/AST/TemplateName.h b/clang/include/clang/AST/TemplateName.h
index 63949f898f6a2..37ea401a0045a 100644
--- a/clang/include/clang/AST/TemplateName.h
+++ b/clang/include/clang/AST/TemplateName.h
@@ -14,7 +14,7 @@
 #define LLVM_CLANG_AST_TEMPLATENAME_H
 
 #include "clang/AST/DependenceFlags.h"
-#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/NestedNameSpecifierBase.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/OperatorKinds.h"
 #include "clang/Basic/UnsignedOrNone.h"
@@ -335,10 +335,18 @@ class TemplateName {
   /// structure, if any.
   QualifiedTemplateName *getAsQualifiedTemplateName() const;
 
+  /// Retrieve the underlying qualified template name,
+  /// looking through underlying nodes.
+  QualifiedTemplateName *getAsAdjustedQualifiedTemplateName() const;
+
   /// Retrieve the underlying dependent template name
   /// structure, if any.
   DependentTemplateName *getAsDependentTemplateName() const;
 
+  // Retrieve the qualifier stored in either a underlying DependentTemplateName
+  // or QualifiedTemplateName.
+  NestedNameSpecifier getQualifier() const;
+
   /// Retrieve the using shadow declaration through which the underlying
   /// template declaration is introduced, if any.
   UsingShadowDecl *getAsUsingShadowDecl() const;
@@ -503,7 +511,7 @@ class QualifiedTemplateName : public llvm::FoldingSetNode {
   /// "template" keyword is always redundant in this case (otherwise,
   /// the template name would be a dependent name and we would express
   /// this name with DependentTemplateName).
-  llvm::PointerIntPair<NestedNameSpecifier *, 1> Qualifier;
+  llvm::PointerIntPair<NestedNameSpecifier, 1, bool> Qualifier;
 
   /// The underlying template name, it is either
   ///  1) a Template -- a template declaration that this qualified name refers
@@ -512,7 +520,7 @@ class QualifiedTemplateName : public llvm::FoldingSetNode {
   ///     using-shadow declaration.
   TemplateName UnderlyingTemplate;
 
-  QualifiedTemplateName(NestedNameSpecifier *NNS, bool TemplateKeyword,
+  QualifiedTemplateName(NestedNameSpecifier NNS, bool TemplateKeyword,
                         TemplateName Template)
       : Qualifier(NNS, TemplateKeyword ? 1 : 0), UnderlyingTemplate(Template) {
     assert(UnderlyingTemplate.getKind() == TemplateName::Template ||
@@ -521,7 +529,7 @@ class QualifiedTemplateName : public llvm::FoldingSetNode {
 
 public:
   /// Return the nested name specifier that qualifies this name.
-  NestedNameSpecifier *getQualifier() const { return Qualifier.getPointer(); }
+  NestedNameSpecifier getQualifier() const { return Qualifier.getPointer(); }
 
   /// Whether the template name was prefixed by the "template"
   /// keyword.
@@ -534,9 +542,9 @@ class QualifiedTemplateName : public llvm::FoldingSetNode {
     Profile(ID, getQualifier(), hasTemplateKeyword(), UnderlyingTemplate);
   }
 
-  static void Profile(llvm::FoldingSetNodeID &ID, NestedNameSpecifier *NNS,
+  static void Profile(llvm::FoldingSetNodeID &ID, NestedNameSpecifier NNS,
                       bool TemplateKeyword, TemplateName TN) {
-    ID.AddPointer(NNS);
+    NNS.Profile(ID);
     ID.AddBoolean(TemplateKeyword);
     ID.AddPointer(TN.getAsVoidPointer());
   }
@@ -585,18 +593,18 @@ class DependentTemplateStorage {
   ///
   /// The bit stored in this qualifier describes whether the \c Name field
   /// was preceeded by a template keyword.
-  llvm::PointerIntPair<NestedNameSpecifier *, 1, bool> Qualifier;
+  llvm::PointerIntPair<NestedNameSpecifier, 1, bool> Qualifier;
 
   /// The dependent template name.
   IdentifierOrOverloadedOperator Name;
 
 public:
-  DependentTemplateStorage(NestedNameSpecifier *Qualifier,
+  DependentTemplateStorage(NestedNameSpecifier Qualifier,
                            IdentifierOrOverloadedOperator Name,
                            bool HasTemplateKeyword);
 
   /// Return the nested name specifier that qualifies this name.
-  NestedNameSpecifier *getQualifier() const { return Qualifier.getPointer(); }
+  NestedNameSpecifier getQualifier() const { return Qualifier.getPointer(); }
 
   IdentifierOrOverloadedOperator getName() const { return Name; }
 
@@ -609,10 +617,10 @@ class DependentTemplateStorage {
     Profile(ID, getQualifier(), getName(), hasTemplateKeyword());
   }
 
-  static void Profile(llvm::FoldingSetNodeID &ID, NestedNameSpecifier *NNS,
+  static void Profile(llvm::FoldingSetNodeID &ID, NestedNameSpecifier NNS,
                       IdentifierOrOverloadedOperator Name,
                       bool HasTemplateKeyword) {
-    ID.AddPointer(NNS);
+    NNS.Profile(ID);
     ID.AddBoolean(HasTemplateKeyword);
     Name.Profile(ID);
   }
diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h
index 1917a8ac29f05..6d2795111685a 100644
--- a/clang/include/clang/AST/TextNodeDumper.h
+++ b/clang/include/clang/AST/TextNodeDumper.h
@@ -211,7 +211,7 @@ class TextNodeDumper
   void dumpAccessSpecifier(AccessSpecifier AS);
   void dumpCleanupObject(const ExprWithCleanups::CleanupObject &C);
   void dumpTemplateSpecializationKind(TemplateSpecializationKind TSK);
-  void dumpNestedNameSpecifier(const NestedNameSpecifier *NNS);
+  void dumpNestedNameSpecifier(NestedNameSpecifier NNS);
   void dumpConceptReference(const ConceptReference *R);
   void dumpTemplateArgument(const TemplateArgument &TA);
   void dumpBareTemplateName(TemplateName TN);
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 12dce309127e5..adf5cb0462154 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -18,7 +18,7 @@
 #define LLVM_CLANG_AST_TYPE_H
 
 #include "clang/AST/DependenceFlags.h"
-#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/NestedNameSpecifierBase.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/AttrKinds.h"
@@ -133,6 +133,7 @@ struct PrintingPolicy;
 class RecordDecl;
 class Stmt;
 class TagDecl;
+class ClassTemplateDecl;
 class TemplateArgument;
 class TemplateArgumentListInfo;
 class TemplateArgumentLoc;
@@ -2044,8 +2045,8 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
     unsigned InnerRef : 1;
   };
 
-  class TypeWithKeywordBitfields {
-    friend class TypeWithKeyword;
+  class KeywordWrapperBitfields {
+    template <class> friend class KeywordWrapper;
 
     LLVM_PREFERRED_TYPE(TypeBitfields)
     unsigned : NumTypeBits;
@@ -2057,15 +2058,23 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
 
   enum { NumTypeWithKeywordBits = NumTypeBits + 8 };
 
-  class ElaboratedTypeBitfields {
-    friend class ElaboratedType;
+  class TagTypeBitfields {
+    friend class TagType;
 
-    LLVM_PREFERRED_TYPE(TypeWithKeywordBitfields)
+    LLVM_PREFERRED_TYPE(KeywordWrapperBitfields)
     unsigned : NumTypeWithKeywordBits;
 
-    /// Whether the ElaboratedType has a trailing OwnedTagDecl.
+    /// Whether the TagType has a trailing Qualifier.
     LLVM_PREFERRED_TYPE(bool)
-    unsigned HasOwnedTagDecl : 1;
+    unsigned HasQualifier : 1;
+
+    /// Whether the TagType owns the Tag.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned OwnsTag : 1;
+
+    /// Whether the TagType was created from an injected name.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned IsInjected : 1;
   };
 
   class VectorTypeBitfields {
@@ -2124,22 +2133,37 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
     unsigned Kind : 1;
   };
 
+  class UnresolvedUsingBitfields {
+    friend class UnresolvedUsingType;
+
+    LLVM_PREFERRED_TYPE(KeywordWrapperBitfields)
+    unsigned : NumTypeWithKeywordBits;
+
+    /// True if there is a non-null qualifier.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned hasQualifier : 1;
+  };
+
   class UsingBitfields {
     friend class UsingType;
 
-    LLVM_PREFERRED_TYPE(TypeBitfields)
-    unsigned : NumTypeBits;
+    LLVM_PREFERRED_TYPE(KeywordWrapperBitfields)
+    unsigned : NumTypeWithKeywordBits;
 
-    /// True if the underlying type is different from the declared one.
+    /// True if there is a non-null qualifier.
     LLVM_PREFERRED_TYPE(bool)
-    unsigned hasTypeDifferentFromDecl : 1;
+    unsigned hasQualifier : 1;
   };
 
   class TypedefBitfields {
     friend class TypedefType;
 
-    LLVM_PREFERRED_TYPE(TypeBitfields)
-    unsigned : NumTypeBits;
+    LLVM_PREFERRED_TYPE(KeywordWrapperBitfields)
+    unsigned : NumTypeWithKeywordBits;
+
+    /// True if there is a non-null qualifier.
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned hasQualifier : 1;
 
     /// True if the underlying type is different from the declared one.
     LLVM_PREFERRED_TYPE(bool)
@@ -2186,27 +2210,31 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
     unsigned PackIndex : 15;
   };
 
-  class SubstTemplateTypeParmPackTypeBitfields {
+  class SubstPackTypeBitfields {
+    friend class SubstPackType;
     friend class SubstTemplateTypeParmPackType;
 
     LLVM_PREFERRED_TYPE(TypeBitfields)
     unsigned : NumTypeBits;
 
-    // The index of the template parameter this substitution represents.
-    unsigned Index : 16;
-
     /// The number of template arguments in \c Arguments, which is
     /// expected to be able to hold at least 1024 according to [implimits].
     /// However as this limit is somewhat easy to hit with template
     /// metaprogramming we'd prefer to keep it as large as possible.
     unsigned NumArgs : 16;
+
+    // The index of the template parameter this substitution represents.
+    // Only used by SubstTemplateTypeParmPackType. We keep it in the same
+    // class to avoid dealing with complexities of bitfields that go over
+    // the size of `unsigned`.
+    unsigned SubstTemplTypeParmPackIndex : 16;
   };
 
   class TemplateSpecializationTypeBitfields {
     friend class TemplateSpecializationType;
 
-    LLVM_PREFERRED_TYPE(TypeBitfields)
-    unsigned : NumTypeBits;
+    LLVM_PREFERRED_TYPE(KeywordWrapperBitfields)
+    unsigned : NumTypeWithKeywordBits;
 
     /// Whether this template specialization type is a substituted type alias.
     LLVM_PREFERRED_TYPE(bool)
@@ -2225,7 +2253,7 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
   class DependentTemplateSpecializationTypeBitfields {
     friend class DependentTemplateSpecializationType;
 
-    LLVM_PREFERRED_TYPE(TypeWithKeywordBitfields)
+    LLVM_PREFERRED_TYPE(KeywordWrapperBitfields)
     unsigned : NumTypeWithKeywordBits;
 
     /// The number of template arguments named in this class template
@@ -2305,17 +2333,18 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
     AutoTypeBitfields AutoTypeBits;
     TypeOfBitfields TypeOfBits;
     TypedefBitfields TypedefBits;
+    UnresolvedUsingBitfields UnresolvedUsingBits;
     UsingBitfields UsingBits;
     BuiltinTypeBitfields BuiltinTypeBits;
     FunctionTypeBitfields FunctionTypeBits;
     ObjCObjectTypeBitfields ObjCObjectTypeBits;
     ReferenceTypeBitfields ReferenceTypeBits;
-    TypeWithKeywordBitfields TypeWithKeywordBits;
-    ElaboratedTypeBitfields ElaboratedTypeBits;
+    KeywordWrapperBitfields KeywordWrapperBits;
+    TagTypeBitfields TagTypeBits;
     VectorTypeBitfields VectorTypeBits;
     TemplateTypeParmTypeBitfields TemplateTypeParmTypeBits;
     SubstTemplateTypeParmTypeBitfields SubstTemplateTypeParmTypeBits;
-    SubstTemplateTypeParmPackTypeBitfields SubstTemplateTypeParmPackTypeBits;
+    SubstPackTypeBitfields SubstPackTypeBits;
     TemplateSpecializationTypeBitfields TemplateSpecializationTypeBits;
     DependentTemplateSpecializationTypeBitfields
       DependentTemplateSpecializationTypeBits;
@@ -2724,6 +2753,7 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
   bool isHLSLAttributedResourceType() const;
   bool isHLSLInlineSpirvType() const;
   bool isHLSLResourceRecord() const;
+  bool isHLSLResourceRecordArray() const;
   bool isHLSLIntangibleType()
       const; // Any HLSL intangible type (builtin, array, class)
 
@@ -2931,6 +2961,11 @@ class alignas(TypeAlignment) Type : public ExtQualsTypeCommonBase {
   /// qualifiers from the outermost type.
   const ArrayType *castAsArrayTypeUnsafe() const;
 
+  /// If this type represents a qualified-id, this returns its nested name
+  /// specifier. For example, for the qualified-id "foo::bar::baz", this returns
+  /// "foo::bar". Returns null if this type represents an unqualified-id.
+  NestedNameSpecifier getPrefix() const;
+
   /// Determine whether this type had the specified attribute applied to it
   /// (looking through top-level type sugar).
   bool hasAttr(attr::Kind AK) const;
@@ -3608,12 +3643,12 @@ class MemberPointerType : public Type, public llvm::FoldingSetNode {
 
   /// The class of which the pointee is a member. Must ultimately be a
   /// CXXRecordType, but could be a typedef or a template parameter too.
-  NestedNameSpecifier *Qualifier;
+  NestedNameSpecifier Qualifier;
 
-  MemberPointerType(QualType Pointee, NestedNameSpecifier *Qualifier,
+  MemberPointerType(QualType Pointee, NestedNameSpecifier Qualifier,
                     QualType CanonicalPtr)
       : Type(MemberPointer, CanonicalPtr,
-             (toTypeDependence(Qualifier->getDependence()) &
+             (toTypeDependence(Qualifier.getDependence()) &
               ~TypeDependence::VariablyModified) |
                  Pointee->getDependence()),
         PointeeType(Pointee), Qualifier(Qualifier) {}
@@ -3633,7 +3668,7 @@ class MemberPointerType : public Type, public llvm::FoldingSetNode {
     return !PointeeType->isFunctionProtoType();
   }
 
-  NestedNameSpecifier *getQualifier() const { return Qualifier; }
+  NestedNameSpecifier getQualifier() const { return Qualifier; }
   /// Note: this can trigger extra deserialization when external AST sources are
   /// used. Prefer `getCXXRecordDecl()` unless you really need the most recent
   /// decl.
@@ -3652,7 +3687,7 @@ class MemberPointerType : public Type, public llvm::FoldingSetNode {
   }
 
   static void Profile(llvm::FoldingSetNodeID &ID, QualType Pointee,
-                      const NestedNameSpecifier *Qualifier,
+                      const NestedNameSpecifier Qualifier,
                       const CXXRecordDecl *Cls);
 
   static bool classof(const Type *T) {
@@ -4656,6 +4691,9 @@ class FunctionType : public Type {
     /// [implimits] 8 bits would be enough here.
     unsigned NumExceptionType : 10;
 
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned HasExtraAttributeInfo : 1;
+
     LLVM_PREFERRED_TYPE(bool)
     unsigned HasArmTypeAttributes : 1;
 
@@ -4664,14 +4702,26 @@ class FunctionType : public Type {
     unsigned NumFunctionEffects : 4;
 
     FunctionTypeExtraBitfields()
-        : NumExceptionType(0), HasArmTypeAttributes(false),
-          EffectsHaveConditions(false), NumFunctionEffects(0) {}
+        : NumExceptionType(0), HasExtraAttributeInfo(false),
+          HasArmTypeAttributes(false), EffectsHaveConditions(false),
+          NumFunctionEffects(0) {}
+  };
+
+  /// A holder for extra information from attributes which aren't part of an
+  /// \p AttributedType.
+  struct alignas(void *) FunctionTypeExtraAttributeInfo {
+    /// A CFI "salt" that differentiates functions with the same prototype.
+    StringRef CFISalt;
+
+    operator bool() const { return !CFISalt.empty(); }
+
+    void Profile(llvm::FoldingSetNodeID &ID) const { ID.AddString(CFISalt); }
   };
 
   /// The AArch64 SME ACLE (Arm C/C++ Language Extensions) define a number
   /// of function type attributes that can be set on function types, including
   /// function pointers.
-  enum AArch64SMETypeAttributes : unsigned {
+  enum AArch64SMETypeAttributes : uint16_t {
     SME_NormalFunction = 0,
     SME_PStateSMEnabledMask = 1 << 0,
     SME_PStateSMCompatibleMask = 1 << 1,
@@ -4701,11 +4751,11 @@ class FunctionType : public Type {
   };
 
   static ArmStateValue getArmZAState(unsigned AttrBits) {
-    return (ArmStateValue)((AttrBits & SME_ZAMask) >> SME_ZAShift);
+    return static_cast<ArmStateValue>((AttrBits & SME_ZAMask) >> SME_ZAShift);
   }
 
   static ArmStateValue getArmZT0State(unsigned AttrBits) {
-    return (ArmStateValue)((AttrBits & SME_ZT0Mask) >> SME_ZT0Shift);
+    return static_cast<ArmStateValue>((AttrBits & SME_ZT0Mask) >> SME_ZT0Shift);
   }
 
   /// A holder for Arm type attributes as described in the Arm C/C++
@@ -4714,6 +4764,7 @@ class FunctionType : public Type {
   struct alignas(void *) FunctionTypeArmAttributes {
     /// Any AArch64 SME ACLE type attributes that need to be propagated
     /// on declarations and function pointers.
+    LLVM_PREFERRED_TYPE(AArch64SMETypeAttributes)
     unsigned AArch64SMEAttributes : 9;
 
     FunctionTypeArmAttributes() : AArch64SMEAttributes(SME_NormalFunction) {}
@@ -5195,6 +5246,7 @@ class FunctionProtoType final
       private llvm::TrailingObjects<
           FunctionProtoType, QualType, SourceLocation,
           FunctionType::FunctionTypeExtraBitfields,
+          FunctionType::FunctionTypeExtraAttributeInfo,
           FunctionType::FunctionTypeArmAttributes, FunctionType::ExceptionType,
           Expr *, FunctionDecl *, FunctionType::ExtParameterInfo, Qualifiers,
           FunctionEffect, EffectConditionExpr> {
@@ -5284,19 +5336,22 @@ class FunctionProtoType final
   /// the various bits of extra information about a function prototype.
   struct ExtProtoInfo {
     FunctionType::ExtInfo ExtInfo;
+    Qualifiers TypeQuals;
+    RefQualifierKind RefQualifier = RQ_None;
+    ExceptionSpecInfo ExceptionSpec;
+    const ExtParameterInfo *ExtParameterInfos = nullptr;
+    SourceLocation EllipsisLoc;
+    FunctionEffectsRef FunctionEffects;
+    FunctionTypeExtraAttributeInfo ExtraAttributeInfo;
+
     LLVM_PREFERRED_TYPE(bool)
     unsigned Variadic : 1;
     LLVM_PREFERRED_TYPE(bool)
     unsigned HasTrailingReturn : 1;
     LLVM_PREFERRED_TYPE(bool)
     unsigned CFIUncheckedCallee : 1;
+    LLVM_PREFERRED_TYPE(AArch64SMETypeAttributes)
     unsigned AArch64SMEAttributes : 9;
-    Qualifiers TypeQuals;
-    RefQualifierKind RefQualifier = RQ_None;
-    ExceptionSpecInfo ExceptionSpec;
-    const ExtParameterInfo *ExtParameterInfos = nullptr;
-    SourceLocation EllipsisLoc;
-    FunctionEffectsRef FunctionEffects;
 
     ExtProtoInfo()
         : Variadic(false), HasTrailingReturn(false), CFIUncheckedCallee(false),
@@ -5321,6 +5376,7 @@ class FunctionProtoType final
     bool requiresFunctionProtoTypeExtraBitfields() const {
       return ExceptionSpec.Type == EST_Dynamic ||
              requiresFunctionProtoTypeArmAttributes() ||
+             requiresFunctionProtoTypeExtraAttributeInfo() ||
              !FunctionEffects.empty();
     }
 
@@ -5328,6 +5384,10 @@ class FunctionProtoType final
       return AArch64SMEAttributes != SME_NormalFunction;
     }
 
+    bool requiresFunctionProtoTypeExtraAttributeInfo() const {
+      return static_cast<bool>(ExtraAttributeInfo);
+    }
+
     void setArmSMEAttribute(AArch64SMETypeAttributes Kind, bool Enable = true) {
       if (Enable)
         AArch64SMEAttributes |= Kind;
@@ -5353,6 +5413,11 @@ class FunctionProtoType final
     return hasExtraBitfields();
   }
 
+  unsigned
+  numTrailingObjects(OverloadToken<FunctionTypeExtraAttributeInfo>) const {
+    return hasExtraAttributeInfo();
+  }
+
   unsigned numTrailingObjects(OverloadToken<ExceptionType>) const {
     return getExceptionSpecSize().NumExceptionType;
   }
@@ -5445,6 +5510,12 @@ class FunctionProtoType final
 
   }
 
+  bool hasExtraAttributeInfo() const {
+    return FunctionTypeBits.HasExtraBitfields &&
+           getTrailingObjects<FunctionTypeExtraBitfields>()
+               ->HasExtraAttributeInfo;
+  }
+
   bool hasArmTypeAttributes() const {
     return FunctionTypeBits.HasExtraBitfields &&
            getTrailingObjects<FunctionTypeExtraBitfields>()
@@ -5478,6 +5549,7 @@ class FunctionProtoType final
     EPI.TypeQuals = getMethodQuals();
     EPI.RefQualifier = getRefQualifier();
     EPI.ExtParameterInfos = getExtParameterInfosOrNull();
+    EPI.ExtraAttributeInfo = getExtraAttributeInfo();
     EPI.AArch64SMEAttributes = getAArch64SMEAttributes();
     EPI.FunctionEffects = getFunctionEffects();
     return EPI;
@@ -5665,6 +5737,13 @@ class FunctionProtoType final
     return getTrailingObjects<ExtParameterInfo>();
   }
 
+  /// Return the extra attribute information.
+  FunctionTypeExtraAttributeInfo getExtraAttributeInfo() const {
+    if (hasExtraAttributeInfo())
+      return *getTrailingObjects<FunctionTypeExtraAttributeInfo>();
+    return FunctionTypeExtraAttributeInfo();
+  }
+
   /// Return a bitmask describing the SME attributes on the function type, see
   /// AArch64SMETypeAttributes for their values.
   unsigned getAArch64SMEAttributes() const {
@@ -5767,84 +5846,254 @@ class FunctionProtoType final
                       bool Canonical);
 };
 
+/// The elaboration keyword that precedes a qualified type name or
+/// introduces an elaborated-type-specifier.
+enum class ElaboratedTypeKeyword {
+  /// The "struct" keyword introduces the elaborated-type-specifier.
+  Struct,
+
+  /// The "__interface" keyword introduces the elaborated-type-specifier.
+  Interface,
+
+  /// The "union" keyword introduces the elaborated-type-specifier.
+  Union,
+
+  /// The "class" keyword introduces the elaborated-type-specifier.
+  Class,
+
+  /// The "enum" keyword introduces the elaborated-type-specifier.
+  Enum,
+
+  /// The "typename" keyword precedes the qualified type name, e.g.,
+  /// \c typename T::type.
+  Typename,
+
+  /// No keyword precedes the qualified type name.
+  None
+};
+
+/// The kind of a tag type.
+enum class TagTypeKind {
+  /// The "struct" keyword.
+  Struct,
+
+  /// The "__interface" keyword.
+  Interface,
+
+  /// The "union" keyword.
+  Union,
+
+  /// The "class" keyword.
+  Class,
+
+  /// The "enum" keyword.
+  Enum
+};
+
+/// Provides a few static helpers for converting and printing
+/// elaborated type keyword and tag type kind enumerations.
+struct KeywordHelpers {
+  /// Converts a type specifier (DeclSpec::TST) into an elaborated type keyword.
+  static ElaboratedTypeKeyword getKeywordForTypeSpec(unsigned TypeSpec);
+
+  /// Converts a type specifier (DeclSpec::TST) into a tag type kind.
+  /// It is an error to provide a type specifier which *isn't* a tag kind here.
+  static TagTypeKind getTagTypeKindForTypeSpec(unsigned TypeSpec);
+
+  /// Converts a TagTypeKind into an elaborated type keyword.
+  static ElaboratedTypeKeyword getKeywordForTagTypeKind(TagTypeKind Tag);
+
+  /// Converts an elaborated type keyword into a TagTypeKind.
+  /// It is an error to provide an elaborated type keyword
+  /// which *isn't* a tag kind here.
+  static TagTypeKind getTagTypeKindForKeyword(ElaboratedTypeKeyword Keyword);
+
+  static bool KeywordIsTagTypeKind(ElaboratedTypeKeyword Keyword);
+
+  static StringRef getKeywordName(ElaboratedTypeKeyword Keyword);
+
+  static StringRef getTagTypeKindName(TagTypeKind Kind) {
+    return getKeywordName(getKeywordForTagTypeKind(Kind));
+  }
+};
+
+template <class T> class KeywordWrapper : public T, public KeywordHelpers {
+protected:
+  template <class... As>
+  KeywordWrapper(ElaboratedTypeKeyword Keyword, As &&...as)
+      : T(std::forward<As>(as)...) {
+    this->KeywordWrapperBits.Keyword = llvm::to_underlying(Keyword);
+  }
+
+public:
+  ElaboratedTypeKeyword getKeyword() const {
+    return static_cast<ElaboratedTypeKeyword>(this->KeywordWrapperBits.Keyword);
+  }
+
+  class CannotCastToThisType {};
+  static CannotCastToThisType classof(const T *);
+};
+
+/// A helper class for Type nodes having an ElaboratedTypeKeyword.
+/// The keyword in stored in the free bits of the base class.
+class TypeWithKeyword : public KeywordWrapper<Type> {
+protected:
+  TypeWithKeyword(ElaboratedTypeKeyword Keyword, TypeClass tc,
+                  QualType Canonical, TypeDependence Dependence)
+      : KeywordWrapper(Keyword, tc, Canonical, Dependence) {}
+};
+
+template <class T> struct FoldingSetPlaceholder : llvm::FoldingSetNode {
+  void Profile(llvm::FoldingSetNodeID &ID) { getType()->Profile(ID); }
+
+  inline const T *getType() const {
+    constexpr unsigned long Offset =
+        llvm::alignTo(sizeof(T), alignof(FoldingSetPlaceholder));
+    const auto *Addr = reinterpret_cast<const T *>(
+        reinterpret_cast<const char *>(this) - Offset);
+    assert(llvm::isAddrAligned(llvm::Align(alignof(T)), Addr));
+    return Addr;
+  }
+};
+
 /// Represents the dependent type named by a dependently-scoped
 /// typename using declaration, e.g.
 ///   using typename Base<T>::foo;
 ///
 /// Template instantiation turns these into the underlying type.
-class UnresolvedUsingType : public Type {
+class UnresolvedUsingType final
+    : public TypeWithKeyword,
+      private llvm::TrailingObjects<UnresolvedUsingType,
+                                    FoldingSetPlaceholder<UnresolvedUsingType>,
+                                    NestedNameSpecifier> {
   friend class ASTContext; // ASTContext creates these.
+  friend TrailingObjects;
 
   UnresolvedUsingTypenameDecl *Decl;
 
-  UnresolvedUsingType(const UnresolvedUsingTypenameDecl *D)
-      : Type(UnresolvedUsing, QualType(),
-             TypeDependence::DependentInstantiation),
-        Decl(const_cast<UnresolvedUsingTypenameDecl *>(D)) {}
+  unsigned numTrailingObjects(
+      OverloadToken<FoldingSetPlaceholder<UnresolvedUsingType>>) const {
+    assert(UnresolvedUsingBits.hasQualifier ||
+           getKeyword() != ElaboratedTypeKeyword::None);
+    return 1;
+  }
+
+  FoldingSetPlaceholder<UnresolvedUsingType> *getFoldingSetPlaceholder() {
+    assert(numTrailingObjects(
+               OverloadToken<FoldingSetPlaceholder<UnresolvedUsingType>>{}) ==
+           1);
+    return getTrailingObjects<FoldingSetPlaceholder<UnresolvedUsingType>>();
+  }
+
+  UnresolvedUsingType(ElaboratedTypeKeyword Keyword,
+                      NestedNameSpecifier Qualifier,
+                      const UnresolvedUsingTypenameDecl *D,
+                      const Type *CanonicalType);
 
 public:
+  NestedNameSpecifier getQualifier() const {
+    return UnresolvedUsingBits.hasQualifier
+               ? *getTrailingObjects<NestedNameSpecifier>()
+               : std::nullopt;
+  }
+
   UnresolvedUsingTypenameDecl *getDecl() const { return Decl; }
 
   bool isSugared() const { return false; }
   QualType desugar() const { return QualType(this, 0); }
 
-  static bool classof(const Type *T) {
-    return T->getTypeClass() == UnresolvedUsing;
+  static void Profile(llvm::FoldingSetNodeID &ID, ElaboratedTypeKeyword Keyword,
+                      NestedNameSpecifier Qualifier,
+                      const UnresolvedUsingTypenameDecl *D) {
+    static_assert(llvm::to_underlying(ElaboratedTypeKeyword::None) <= 7);
+    ID.AddInteger(uintptr_t(D) | llvm::to_underlying(Keyword));
+    if (Qualifier)
+      Qualifier.Profile(ID);
   }
 
-  void Profile(llvm::FoldingSetNodeID &ID) {
-    return Profile(ID, Decl);
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    Profile(ID, getKeyword(), getQualifier(), getDecl());
   }
 
-  static void Profile(llvm::FoldingSetNodeID &ID,
-                      UnresolvedUsingTypenameDecl *D) {
-    ID.AddPointer(D);
+  static bool classof(const Type *T) {
+    return T->getTypeClass() == UnresolvedUsing;
   }
 };
 
-class UsingType final : public Type,
+class UsingType final : public TypeWithKeyword,
                         public llvm::FoldingSetNode,
-                        private llvm::TrailingObjects<UsingType, QualType> {
-  UsingShadowDecl *Found;
+                        llvm::TrailingObjects<UsingType, NestedNameSpecifier> {
+  UsingShadowDecl *D;
+  QualType UnderlyingType;
+
   friend class ASTContext; // ASTContext creates these.
   friend TrailingObjects;
 
-  UsingType(const UsingShadowDecl *Found, QualType Underlying, QualType Canon);
+  UsingType(ElaboratedTypeKeyword Keyword, NestedNameSpecifier Qualifier,
+            const UsingShadowDecl *D, QualType UnderlyingType);
 
 public:
-  UsingShadowDecl *getFoundDecl() const { return Found; }
-  QualType getUnderlyingType() const;
-
-  bool isSugared() const { return true; }
+  NestedNameSpecifier getQualifier() const {
+    return UsingBits.hasQualifier ? *getTrailingObjects() : std::nullopt;
+  }
 
-  // This always has the 'same' type as declared, but not necessarily identical.
-  QualType desugar() const { return getUnderlyingType(); }
+  UsingShadowDecl *getDecl() const { return D; }
 
-  // Internal helper, for debugging purposes.
-  bool typeMatchesDecl() const { return !UsingBits.hasTypeDifferentFromDecl; }
+  QualType desugar() const { return UnderlyingType; }
+  bool isSugared() const { return true; }
 
-  void Profile(llvm::FoldingSetNodeID &ID) {
-    Profile(ID, Found, getUnderlyingType());
+  static void Profile(llvm::FoldingSetNodeID &ID, ElaboratedTypeKeyword Keyword,
+                      NestedNameSpecifier Qualifier, const UsingShadowDecl *D,
+                      QualType UnderlyingType) {
+    static_assert(llvm::to_underlying(ElaboratedTypeKeyword::None) <= 7);
+    ID.AddInteger(uintptr_t(D) | llvm::to_underlying(Keyword));
+    UnderlyingType.Profile(ID);
+    if (Qualifier)
+      Qualifier.Profile(ID);
   }
-  static void Profile(llvm::FoldingSetNodeID &ID, const UsingShadowDecl *Found,
-                      QualType Underlying) {
-    ID.AddPointer(Found);
-    Underlying.Profile(ID);
+
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    Profile(ID, getKeyword(), getQualifier(), D, desugar());
   }
   static bool classof(const Type *T) { return T->getTypeClass() == Using; }
 };
 
-class TypedefType final : public Type,
-                          public llvm::FoldingSetNode,
-                          private llvm::TrailingObjects<TypedefType, QualType> {
+class TypedefType final
+    : public TypeWithKeyword,
+      private llvm::TrailingObjects<TypedefType,
+                                    FoldingSetPlaceholder<TypedefType>,
+                                    NestedNameSpecifier, QualType> {
   TypedefNameDecl *Decl;
   friend class ASTContext; // ASTContext creates these.
   friend TrailingObjects;
 
-  TypedefType(TypeClass tc, const TypedefNameDecl *D, QualType UnderlyingType,
-              bool HasTypeDifferentFromDecl);
+  unsigned
+  numTrailingObjects(OverloadToken<FoldingSetPlaceholder<TypedefType>>) const {
+    assert(TypedefBits.hasQualifier || TypedefBits.hasTypeDifferentFromDecl ||
+           getKeyword() != ElaboratedTypeKeyword::None);
+    return 1;
+  }
+
+  unsigned numTrailingObjects(OverloadToken<NestedNameSpecifier>) const {
+    return TypedefBits.hasQualifier;
+  }
+
+  TypedefType(TypeClass TC, ElaboratedTypeKeyword Keyword,
+              NestedNameSpecifier Qualifier, const TypedefNameDecl *D,
+              QualType UnderlyingType, bool HasTypeDifferentFromDecl);
+
+  FoldingSetPlaceholder<TypedefType> *getFoldingSetPlaceholder() {
+    assert(numTrailingObjects(
+               OverloadToken<FoldingSetPlaceholder<TypedefType>>{}) == 1);
+    return getTrailingObjects<FoldingSetPlaceholder<TypedefType>>();
+  }
 
 public:
+  NestedNameSpecifier getQualifier() const {
+    return TypedefBits.hasQualifier ? *getTrailingObjects<NestedNameSpecifier>()
+                                    : std::nullopt;
+  }
+
   TypedefNameDecl *getDecl() const { return Decl; }
 
   bool isSugared() const { return true; }
@@ -5855,16 +6104,25 @@ class TypedefType final : public Type,
   // Internal helper, for debugging purposes.
   bool typeMatchesDecl() const { return !TypedefBits.hasTypeDifferentFromDecl; }
 
-  void Profile(llvm::FoldingSetNodeID &ID) {
-    Profile(ID, Decl, typeMatchesDecl() ? QualType() : desugar());
-  }
-  static void Profile(llvm::FoldingSetNodeID &ID, const TypedefNameDecl *Decl,
-                      QualType Underlying) {
-    ID.AddPointer(Decl);
+  static void Profile(llvm::FoldingSetNodeID &ID, ElaboratedTypeKeyword Keyword,
+                      NestedNameSpecifier Qualifier,
+                      const TypedefNameDecl *Decl, QualType Underlying) {
+
+    ID.AddInteger(uintptr_t(Decl) | (Keyword != ElaboratedTypeKeyword::None) |
+                  (!Qualifier << 1));
+    if (Keyword != ElaboratedTypeKeyword::None)
+      ID.AddInteger(llvm::to_underlying(Keyword));
+    if (Qualifier)
+      Qualifier.Profile(ID);
     if (!Underlying.isNull())
       Underlying.Profile(ID);
   }
 
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    Profile(ID, getKeyword(), getQualifier(), getDecl(),
+            typeMatchesDecl() ? QualType() : desugar());
+  }
+
   static bool classof(const Type *T) { return T->getTypeClass() == Typedef; }
 };
 
@@ -6131,71 +6389,155 @@ class UnaryTransformType : public Type, public llvm::FoldingSetNode {
   }
 };
 
-class TagType : public Type {
-  friend class ASTReader;
-  template <class T> friend class serialization::AbstractTypeReader;
+class TagType : public TypeWithKeyword {
+  friend class ASTContext; // ASTContext creates these.
 
   /// Stores the TagDecl associated with this type. The decl may point to any
   /// TagDecl that declares the entity.
   TagDecl *decl;
 
+  void *getTrailingPointer() const;
+  NestedNameSpecifier &getTrailingQualifier() const;
+
 protected:
-  TagType(TypeClass TC, const TagDecl *D, QualType can);
+  TagType(TypeClass TC, ElaboratedTypeKeyword Keyword,
+          NestedNameSpecifier Qualifier, const TagDecl *TD, bool OwnsTag,
+          bool IsInjected, const Type *CanonicalType);
 
 public:
-  TagDecl *getDecl() const;
+  // FIXME: Temporarily renamed from `getDecl` in order to facilitate
+  // rebasing, due to change in behaviour. This should be renamed back
+  // to `getDecl` once the change is settled.
+  TagDecl *getOriginalDecl() const { return decl; }
+
+  NestedNameSpecifier getQualifier() const;
+
+  /// Does the TagType own this declaration of the Tag?
+  bool isTagOwned() const { return TagTypeBits.OwnsTag; }
+
+  bool isInjected() const { return TagTypeBits.IsInjected; }
 
-  /// Determines whether this type is in the process of being defined.
-  bool isBeingDefined() const;
+  ClassTemplateDecl *getTemplateDecl() const;
+  TemplateName getTemplateName(const ASTContext &Ctx) const;
+  ArrayRef<TemplateArgument> getTemplateArgs(const ASTContext &Ctx) const;
+
+  bool isSugared() const { return false; }
+  QualType desugar() const { return getCanonicalTypeInternal(); }
 
   static bool classof(const Type *T) {
-    return T->getTypeClass() == Enum || T->getTypeClass() == Record;
+    return T->getTypeClass() == Enum || T->getTypeClass() == Record ||
+           T->getTypeClass() == InjectedClassName;
+  }
+};
+
+struct TagTypeFoldingSetPlaceholder : public llvm::FoldingSetNode {
+  static constexpr size_t getOffset() {
+    return alignof(TagType) -
+           (sizeof(TagTypeFoldingSetPlaceholder) % alignof(TagType));
+  }
+
+  static void Profile(llvm::FoldingSetNodeID &ID, ElaboratedTypeKeyword Keyword,
+                      NestedNameSpecifier Qualifier, const TagDecl *Tag,
+                      bool OwnsTag, bool IsInjected) {
+    ID.AddInteger(uintptr_t(Tag) | OwnsTag | (IsInjected << 1) |
+                  ((Keyword != ElaboratedTypeKeyword::None) << 2));
+    if (Keyword != ElaboratedTypeKeyword::None)
+      ID.AddInteger(llvm::to_underlying(Keyword));
+    if (Qualifier)
+      Qualifier.Profile(ID);
+  }
+
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    const TagType *T = getTagType();
+    Profile(ID, T->getKeyword(), T->getQualifier(), T->getOriginalDecl(),
+            T->isTagOwned(), T->isInjected());
+  }
+
+  TagType *getTagType() {
+    return reinterpret_cast<TagType *>(reinterpret_cast<char *>(this + 1) +
+                                       getOffset());
+  }
+  const TagType *getTagType() const {
+    return const_cast<TagTypeFoldingSetPlaceholder *>(this)->getTagType();
+  }
+  static TagTypeFoldingSetPlaceholder *fromTagType(TagType *T) {
+    return reinterpret_cast<TagTypeFoldingSetPlaceholder *>(
+               reinterpret_cast<char *>(T) - getOffset()) -
+           1;
   }
 };
 
 /// A helper class that allows the use of isa/cast/dyncast
 /// to detect TagType objects of structs/unions/classes.
-class RecordType : public TagType {
-protected:
-  friend class ASTContext; // ASTContext creates these.
-
-  explicit RecordType(const RecordDecl *D)
-      : TagType(Record, reinterpret_cast<const TagDecl*>(D), QualType()) {}
-  explicit RecordType(TypeClass TC, RecordDecl *D)
-      : TagType(TC, reinterpret_cast<const TagDecl*>(D), QualType()) {}
+class RecordType final : public TagType {
+  using TagType::TagType;
 
 public:
-  RecordDecl *getDecl() const {
-    return reinterpret_cast<RecordDecl*>(TagType::getDecl());
+  // FIXME: Temporarily renamed from `getDecl` in order to facilitate
+  // rebasing, due to change in behaviour. This should be renamed back
+  // to `getDecl` once the change is settled.
+  RecordDecl *getOriginalDecl() const {
+    return reinterpret_cast<RecordDecl *>(TagType::getOriginalDecl());
   }
 
   /// Recursively check all fields in the record for const-ness. If any field
   /// is declared const, return true. Otherwise, return false.
   bool hasConstFields() const;
 
-  bool isSugared() const { return false; }
-  QualType desugar() const { return QualType(this, 0); }
-
   static bool classof(const Type *T) { return T->getTypeClass() == Record; }
 };
 
 /// A helper class that allows the use of isa/cast/dyncast
 /// to detect TagType objects of enums.
-class EnumType : public TagType {
+class EnumType final : public TagType {
+  using TagType::TagType;
+
+public:
+  // FIXME: Temporarily renamed from `getDecl` in order to facilitate
+  // rebasing, due to change in behaviour. This should be renamed back
+  // to `getDecl` once the change is settled.
+  EnumDecl *getOriginalDecl() const {
+    return reinterpret_cast<EnumDecl *>(TagType::getOriginalDecl());
+  }
+
+  static bool classof(const Type *T) { return T->getTypeClass() == Enum; }
+};
+
+/// The injected class name of a C++ class template or class
+/// template partial specialization.  Used to record that a type was
+/// spelled with a bare identifier rather than as a template-id; the
+/// equivalent for non-templated classes is just RecordType.
+///
+/// Injected class name types are always dependent.  Template
+/// instantiation turns these into RecordTypes.
+///
+/// Injected class name types are always canonical.  This works
+/// because it is impossible to compare an injected class name type
+/// with the corresponding non-injected template type, for the same
+/// reason that it is impossible to directly compare template
+/// parameters from different dependent contexts: injected class name
+/// types can only occur within the scope of a particular templated
+/// declaration, and within that scope every template specialization
+/// will canonicalize to the injected class name (when appropriate
+/// according to the rules of the language).
+class InjectedClassNameType final : public TagType {
   friend class ASTContext; // ASTContext creates these.
 
-  explicit EnumType(const EnumDecl *D)
-      : TagType(Enum, reinterpret_cast<const TagDecl*>(D), QualType()) {}
+  InjectedClassNameType(ElaboratedTypeKeyword Keyword,
+                        NestedNameSpecifier Qualifier, const TagDecl *TD,
+                        bool IsInjected, const Type *CanonicalType);
 
 public:
-  EnumDecl *getDecl() const {
-    return reinterpret_cast<EnumDecl*>(TagType::getDecl());
+  // FIXME: Temporarily renamed from `getDecl` in order to facilitate
+  // rebasing, due to change in behaviour. This should be renamed back
+  // to `getDecl` once the change is settled.
+  CXXRecordDecl *getOriginalDecl() const {
+    return reinterpret_cast<CXXRecordDecl *>(TagType::getOriginalDecl());
   }
 
-  bool isSugared() const { return false; }
-  QualType desugar() const { return QualType(this, 0); }
-
-  static bool classof(const Type *T) { return T->getTypeClass() == Enum; }
+  static bool classof(const Type *T) {
+    return T->getTypeClass() == InjectedClassName;
+  }
 };
 
 /// An attributed type is a type to which a type attribute has been applied.
@@ -6654,6 +6996,56 @@ class SubstTemplateTypeParmType final
   }
 };
 
+/// Represents the result of substituting a set of types as a template argument
+/// that needs to be expanded later.
+///
+/// These types are always dependent and produced depending on the situations:
+/// - SubstTemplateTypeParmPack is an expansion that had to be delayed,
+/// - SubstBuiltinTemplatePackType is an expansion from a builtin.
+class SubstPackType : public Type, public llvm::FoldingSetNode {
+  friend class ASTContext;
+
+  /// A pointer to the set of template arguments that this
+  /// parameter pack is instantiated with.
+  const TemplateArgument *Arguments;
+
+protected:
+  SubstPackType(TypeClass Derived, QualType Canon,
+                const TemplateArgument &ArgPack);
+
+public:
+  unsigned getNumArgs() const { return SubstPackTypeBits.NumArgs; }
+
+  TemplateArgument getArgumentPack() const;
+
+  void Profile(llvm::FoldingSetNodeID &ID);
+  static void Profile(llvm::FoldingSetNodeID &ID,
+                      const TemplateArgument &ArgPack);
+
+  static bool classof(const Type *T) {
+    return T->getTypeClass() == SubstTemplateTypeParmPack ||
+           T->getTypeClass() == SubstBuiltinTemplatePack;
+  }
+};
+
+/// Represents the result of substituting a builtin template as a pack.
+class SubstBuiltinTemplatePackType : public SubstPackType {
+  friend class ASTContext;
+
+  SubstBuiltinTemplatePackType(QualType Canon, const TemplateArgument &ArgPack);
+
+public:
+  bool isSugared() const { return false; }
+  QualType desugar() const { return QualType(this, 0); }
+
+  /// Mark that we reuse the Profile. We do not introduce new fields.
+  using SubstPackType::Profile;
+
+  static bool classof(const Type *T) {
+    return T->getTypeClass() == SubstBuiltinTemplatePack;
+  }
+};
+
 /// Represents the result of substituting a set of types for a template
 /// type parameter pack.
 ///
@@ -6666,7 +7058,7 @@ class SubstTemplateTypeParmType final
 /// that pack expansion (e.g., when all template parameters have corresponding
 /// arguments), this type will be replaced with the \c SubstTemplateTypeParmType
 /// at the current pack substitution index.
-class SubstTemplateTypeParmPackType : public Type, public llvm::FoldingSetNode {
+class SubstTemplateTypeParmPackType : public SubstPackType {
   friend class ASTContext;
 
   /// A pointer to the set of template arguments that this
@@ -6692,21 +7084,17 @@ class SubstTemplateTypeParmPackType : public Type, public llvm::FoldingSetNode {
 
   /// Returns the index of the replaced parameter in the associated declaration.
   /// This should match the result of `getReplacedParameter()->getIndex()`.
-  unsigned getIndex() const { return SubstTemplateTypeParmPackTypeBits.Index; }
+  unsigned getIndex() const {
+    return SubstPackTypeBits.SubstTemplTypeParmPackIndex;
+  }
 
   // This substitution will be Final, which means the substitution will be fully
   // sugared: it doesn't need to be resugared later.
   bool getFinal() const;
 
-  unsigned getNumArgs() const {
-    return SubstTemplateTypeParmPackTypeBits.NumArgs;
-  }
-
   bool isSugared() const { return false; }
   QualType desugar() const { return QualType(this, 0); }
 
-  TemplateArgument getArgumentPack() const;
-
   void Profile(llvm::FoldingSetNodeID &ID);
   static void Profile(llvm::FoldingSetNodeID &ID, const Decl *AssociatedDecl,
                       unsigned Index, bool Final,
@@ -6806,34 +7194,38 @@ class AutoType : public DeducedType {
 };
 
 /// Represents a C++17 deduced template specialization type.
-class DeducedTemplateSpecializationType : public DeducedType,
+class DeducedTemplateSpecializationType : public KeywordWrapper<DeducedType>,
                                           public llvm::FoldingSetNode {
   friend class ASTContext; // ASTContext creates these
 
   /// The name of the template whose arguments will be deduced.
   TemplateName Template;
 
-  DeducedTemplateSpecializationType(TemplateName Template,
+  DeducedTemplateSpecializationType(ElaboratedTypeKeyword Keyword,
+                                    TemplateName Template,
                                     QualType DeducedAsType,
                                     bool IsDeducedAsDependent, QualType Canon)
-      : DeducedType(DeducedTemplateSpecialization, DeducedAsType,
-                    toTypeDependence(Template.getDependence()) |
-                        (IsDeducedAsDependent
-                             ? TypeDependence::DependentInstantiation
-                             : TypeDependence::None),
-                    Canon),
+      : KeywordWrapper(Keyword, DeducedTemplateSpecialization, DeducedAsType,
+                       toTypeDependence(Template.getDependence()) |
+                           (IsDeducedAsDependent
+                                ? TypeDependence::DependentInstantiation
+                                : TypeDependence::None),
+                       Canon),
         Template(Template) {}
 
 public:
   /// Retrieve the name of the template that we are deducing.
-  TemplateName getTemplateName() const { return Template;}
+  TemplateName getTemplateName() const { return Template; }
 
   void Profile(llvm::FoldingSetNodeID &ID) const {
-    Profile(ID, getTemplateName(), getDeducedType(), isDependentType());
+    Profile(ID, getKeyword(), getTemplateName(), getDeducedType(),
+            isDependentType());
   }
 
-  static void Profile(llvm::FoldingSetNodeID &ID, TemplateName Template,
-                      QualType Deduced, bool IsDependent) {
+  static void Profile(llvm::FoldingSetNodeID &ID, ElaboratedTypeKeyword Keyword,
+                      TemplateName Template, QualType Deduced,
+                      bool IsDependent) {
+    ID.AddInteger(llvm::to_underlying(Keyword));
     Template.Profile(ID);
     Deduced.Profile(ID);
     ID.AddBoolean(IsDependent || Template.isDependent());
@@ -6864,7 +7256,8 @@ class DeducedTemplateSpecializationType : public DeducedType,
 /// TemplateArguments, followed by a QualType representing the
 /// non-canonical aliased type when the template is a type alias
 /// template.
-class TemplateSpecializationType : public Type, public llvm::FoldingSetNode {
+class TemplateSpecializationType : public TypeWithKeyword,
+                                   public llvm::FoldingSetNode {
   friend class ASTContext; // ASTContext creates these
 
   /// The name of the template being specialized.  This is
@@ -6876,8 +7269,8 @@ class TemplateSpecializationType : public Type, public llvm::FoldingSetNode {
   /// replacement must, recursively, be one of these).
   TemplateName Template;
 
-  TemplateSpecializationType(TemplateName T, bool IsAlias,
-                             ArrayRef<TemplateArgument> Args,
+  TemplateSpecializationType(ElaboratedTypeKeyword Keyword, TemplateName T,
+                             bool IsAlias, ArrayRef<TemplateArgument> Args,
                              QualType Underlying);
 
 public:
@@ -6936,9 +7329,7 @@ class TemplateSpecializationType : public Type, public llvm::FoldingSetNode {
             TemplateSpecializationTypeBits.NumArgs};
   }
 
-  bool isSugared() const {
-    return !isDependentType() || isCurrentInstantiation() || isTypeAlias();
-  }
+  bool isSugared() const;
 
   QualType desugar() const {
     return isTypeAlias() ? getAliasedType() : getCanonicalTypeInternal();
@@ -6978,241 +7369,6 @@ bool isSubstitutedDefaultArgument(ASTContext &Ctx, TemplateArgument Arg,
                                   ArrayRef<TemplateArgument> Args,
                                   unsigned Depth);
 
-/// The injected class name of a C++ class template or class
-/// template partial specialization.  Used to record that a type was
-/// spelled with a bare identifier rather than as a template-id; the
-/// equivalent for non-templated classes is just RecordType.
-///
-/// Injected class name types are always dependent.  Template
-/// instantiation turns these into RecordTypes.
-///
-/// Injected class name types are always canonical.  This works
-/// because it is impossible to compare an injected class name type
-/// with the corresponding non-injected template type, for the same
-/// reason that it is impossible to directly compare template
-/// parameters from different dependent contexts: injected class name
-/// types can only occur within the scope of a particular templated
-/// declaration, and within that scope every template specialization
-/// will canonicalize to the injected class name (when appropriate
-/// according to the rules of the language).
-class InjectedClassNameType : public Type {
-  friend class ASTContext; // ASTContext creates these.
-  friend class ASTNodeImporter;
-  friend class ASTReader; // FIXME: ASTContext::getInjectedClassNameType is not
-                          // currently suitable for AST reading, too much
-                          // interdependencies.
-  template <class T> friend class serialization::AbstractTypeReader;
-
-  CXXRecordDecl *Decl;
-
-  /// The template specialization which this type represents.
-  /// For example, in
-  ///   template <class T> class A { ... };
-  /// this is A<T>, whereas in
-  ///   template <class X, class Y> class A<B<X,Y> > { ... };
-  /// this is A<B<X,Y> >.
-  ///
-  /// It is always unqualified, always a template specialization type,
-  /// and always dependent.
-  QualType InjectedType;
-
-  InjectedClassNameType(CXXRecordDecl *D, QualType TST)
-      : Type(InjectedClassName, QualType(),
-             TypeDependence::DependentInstantiation),
-        Decl(D), InjectedType(TST) {
-    assert(isa<TemplateSpecializationType>(TST));
-    assert(!TST.hasQualifiers());
-    assert(TST->isDependentType());
-  }
-
-public:
-  QualType getInjectedSpecializationType() const { return InjectedType; }
-
-  const TemplateSpecializationType *getInjectedTST() const {
-    return cast<TemplateSpecializationType>(InjectedType.getTypePtr());
-  }
-
-  TemplateName getTemplateName() const {
-    return getInjectedTST()->getTemplateName();
-  }
-
-  CXXRecordDecl *getDecl() const;
-
-  bool isSugared() const { return false; }
-  QualType desugar() const { return QualType(this, 0); }
-
-  static bool classof(const Type *T) {
-    return T->getTypeClass() == InjectedClassName;
-  }
-};
-
-/// The elaboration keyword that precedes a qualified type name or
-/// introduces an elaborated-type-specifier.
-enum class ElaboratedTypeKeyword {
-  /// The "struct" keyword introduces the elaborated-type-specifier.
-  Struct,
-
-  /// The "__interface" keyword introduces the elaborated-type-specifier.
-  Interface,
-
-  /// The "union" keyword introduces the elaborated-type-specifier.
-  Union,
-
-  /// The "class" keyword introduces the elaborated-type-specifier.
-  Class,
-
-  /// The "enum" keyword introduces the elaborated-type-specifier.
-  Enum,
-
-  /// The "typename" keyword precedes the qualified type name, e.g.,
-  /// \c typename T::type.
-  Typename,
-
-  /// No keyword precedes the qualified type name.
-  None
-};
-
-/// The kind of a tag type.
-enum class TagTypeKind {
-  /// The "struct" keyword.
-  Struct,
-
-  /// The "__interface" keyword.
-  Interface,
-
-  /// The "union" keyword.
-  Union,
-
-  /// The "class" keyword.
-  Class,
-
-  /// The "enum" keyword.
-  Enum
-};
-
-/// A helper class for Type nodes having an ElaboratedTypeKeyword.
-/// The keyword in stored in the free bits of the base class.
-/// Also provides a few static helpers for converting and printing
-/// elaborated type keyword and tag type kind enumerations.
-class TypeWithKeyword : public Type {
-protected:
-  TypeWithKeyword(ElaboratedTypeKeyword Keyword, TypeClass tc,
-                  QualType Canonical, TypeDependence Dependence)
-      : Type(tc, Canonical, Dependence) {
-    TypeWithKeywordBits.Keyword = llvm::to_underlying(Keyword);
-  }
-
-public:
-  ElaboratedTypeKeyword getKeyword() const {
-    return static_cast<ElaboratedTypeKeyword>(TypeWithKeywordBits.Keyword);
-  }
-
-  /// Converts a type specifier (DeclSpec::TST) into an elaborated type keyword.
-  static ElaboratedTypeKeyword getKeywordForTypeSpec(unsigned TypeSpec);
-
-  /// Converts a type specifier (DeclSpec::TST) into a tag type kind.
-  /// It is an error to provide a type specifier which *isn't* a tag kind here.
-  static TagTypeKind getTagTypeKindForTypeSpec(unsigned TypeSpec);
-
-  /// Converts a TagTypeKind into an elaborated type keyword.
-  static ElaboratedTypeKeyword getKeywordForTagTypeKind(TagTypeKind Tag);
-
-  /// Converts an elaborated type keyword into a TagTypeKind.
-  /// It is an error to provide an elaborated type keyword
-  /// which *isn't* a tag kind here.
-  static TagTypeKind getTagTypeKindForKeyword(ElaboratedTypeKeyword Keyword);
-
-  static bool KeywordIsTagTypeKind(ElaboratedTypeKeyword Keyword);
-
-  static StringRef getKeywordName(ElaboratedTypeKeyword Keyword);
-
-  static StringRef getTagTypeKindName(TagTypeKind Kind) {
-    return getKeywordName(getKeywordForTagTypeKind(Kind));
-  }
-
-  class CannotCastToThisType {};
-  static CannotCastToThisType classof(const Type *);
-};
-
-/// Represents a type that was referred to using an elaborated type
-/// keyword, e.g., struct S, or via a qualified name, e.g., N::M::type,
-/// or both.
-///
-/// This type is used to keep track of a type name as written in the
-/// source code, including tag keywords and any nested-name-specifiers.
-/// The type itself is always "sugar", used to express what was written
-/// in the source code but containing no additional semantic information.
-class ElaboratedType final
-    : public TypeWithKeyword,
-      public llvm::FoldingSetNode,
-      private llvm::TrailingObjects<ElaboratedType, TagDecl *> {
-  friend class ASTContext; // ASTContext creates these
-  friend TrailingObjects;
-
-  /// The nested name specifier containing the qualifier.
-  NestedNameSpecifier *NNS;
-
-  /// The type that this qualified name refers to.
-  QualType NamedType;
-
-  /// The (re)declaration of this tag type owned by this occurrence is stored
-  /// as a trailing object if there is one. Use getOwnedTagDecl to obtain
-  /// it, or obtain a null pointer if there is none.
-
-  ElaboratedType(ElaboratedTypeKeyword Keyword, NestedNameSpecifier *NNS,
-                 QualType NamedType, QualType CanonType, TagDecl *OwnedTagDecl)
-      : TypeWithKeyword(Keyword, Elaborated, CanonType,
-                        // Any semantic dependence on the qualifier will have
-                        // been incorporated into NamedType. We still need to
-                        // track syntactic (instantiation / error / pack)
-                        // dependence on the qualifier.
-                        NamedType->getDependence() |
-                            (NNS ? toSyntacticDependence(
-                                       toTypeDependence(NNS->getDependence()))
-                                 : TypeDependence::None)),
-        NNS(NNS), NamedType(NamedType) {
-    ElaboratedTypeBits.HasOwnedTagDecl = false;
-    if (OwnedTagDecl) {
-      ElaboratedTypeBits.HasOwnedTagDecl = true;
-      *getTrailingObjects() = OwnedTagDecl;
-    }
-  }
-
-public:
-  /// Retrieve the qualification on this type.
-  NestedNameSpecifier *getQualifier() const { return NNS; }
-
-  /// Retrieve the type named by the qualified-id.
-  QualType getNamedType() const { return NamedType; }
-
-  /// Remove a single level of sugar.
-  QualType desugar() const { return getNamedType(); }
-
-  /// Returns whether this type directly provides sugar.
-  bool isSugared() const { return true; }
-
-  /// Return the (re)declaration of this type owned by this occurrence of this
-  /// type, or nullptr if there is none.
-  TagDecl *getOwnedTagDecl() const {
-    return ElaboratedTypeBits.HasOwnedTagDecl ? *getTrailingObjects() : nullptr;
-  }
-
-  void Profile(llvm::FoldingSetNodeID &ID) {
-    Profile(ID, getKeyword(), NNS, NamedType, getOwnedTagDecl());
-  }
-
-  static void Profile(llvm::FoldingSetNodeID &ID, ElaboratedTypeKeyword Keyword,
-                      NestedNameSpecifier *NNS, QualType NamedType,
-                      TagDecl *OwnedTagDecl) {
-    ID.AddInteger(llvm::to_underlying(Keyword));
-    ID.AddPointer(NNS);
-    NamedType.Profile(ID);
-    ID.AddPointer(OwnedTagDecl);
-  }
-
-  static bool classof(const Type *T) { return T->getTypeClass() == Elaborated; }
-};
-
 /// Represents a qualified type name for which the type name is
 /// dependent.
 ///
@@ -7229,24 +7385,24 @@ class DependentNameType : public TypeWithKeyword, public llvm::FoldingSetNode {
   friend class ASTContext; // ASTContext creates these
 
   /// The nested name specifier containing the qualifier.
-  NestedNameSpecifier *NNS;
+  NestedNameSpecifier NNS;
 
   /// The type that this typename specifier refers to.
   const IdentifierInfo *Name;
 
-  DependentNameType(ElaboratedTypeKeyword Keyword, NestedNameSpecifier *NNS,
+  DependentNameType(ElaboratedTypeKeyword Keyword, NestedNameSpecifier NNS,
                     const IdentifierInfo *Name, QualType CanonType)
       : TypeWithKeyword(Keyword, DependentName, CanonType,
                         TypeDependence::DependentInstantiation |
-                            toTypeDependence(NNS->getDependence())),
+                            (NNS ? toTypeDependence(NNS.getDependence())
+                                 : TypeDependence::Dependent)),
         NNS(NNS), Name(Name) {
-    assert(NNS);
     assert(Name);
   }
 
 public:
   /// Retrieve the qualification on this type.
-  NestedNameSpecifier *getQualifier() const { return NNS; }
+  NestedNameSpecifier getQualifier() const { return NNS; }
 
   /// Retrieve the identifier that terminates this type name.
   /// For example, "type" in "typename T::type".
@@ -7262,9 +7418,9 @@ class DependentNameType : public TypeWithKeyword, public llvm::FoldingSetNode {
   }
 
   static void Profile(llvm::FoldingSetNodeID &ID, ElaboratedTypeKeyword Keyword,
-                      NestedNameSpecifier *NNS, const IdentifierInfo *Name) {
+                      NestedNameSpecifier NNS, const IdentifierInfo *Name) {
     ID.AddInteger(llvm::to_underlying(Keyword));
-    ID.AddPointer(NNS);
+    NNS.Profile(ID);
     ID.AddPointer(Name);
   }
 
@@ -8815,8 +8971,8 @@ inline bool Type::isIntegerType() const {
   if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType)) {
     // Incomplete enum types are not treated as integer types.
     // FIXME: In C++, enum types are never integer types.
-    return IsEnumDeclComplete(ET->getDecl()) &&
-      !IsEnumDeclScoped(ET->getDecl());
+    return IsEnumDeclComplete(ET->getOriginalDecl()) &&
+           !IsEnumDeclScoped(ET->getOriginalDecl());
   }
   return isBitIntType();
 }
@@ -8874,7 +9030,7 @@ inline bool Type::isScalarType() const {
   if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType))
     // Enums are scalar types, but only if they are defined.  Incomplete enums
     // are not treated as scalar types.
-    return IsEnumDeclComplete(ET->getDecl());
+    return IsEnumDeclComplete(ET->getOriginalDecl());
   return isa<PointerType>(CanonicalType) ||
          isa<BlockPointerType>(CanonicalType) ||
          isa<MemberPointerType>(CanonicalType) ||
@@ -8890,7 +9046,7 @@ inline bool Type::isIntegralOrEnumerationType() const {
   // Check for a complete enum type; incomplete enum types are not properly an
   // enumeration type in the sense required here.
   if (const auto *ET = dyn_cast<EnumType>(CanonicalType))
-    return IsEnumDeclComplete(ET->getDecl());
+    return IsEnumDeclComplete(ET->getOriginalDecl());
 
   return isBitIntType();
 }
@@ -9026,8 +9182,6 @@ template <typename T> const T *Type::getAsAdjusted() const {
       Ty = A->getWrappedType().getTypePtr();
     else if (const auto *A = dyn_cast<HLSLAttributedResourceType>(Ty))
       Ty = A->getWrappedType().getTypePtr();
-    else if (const auto *E = dyn_cast<ElaboratedType>(Ty))
-      Ty = E->desugar().getTypePtr();
     else if (const auto *P = dyn_cast<ParenType>(Ty))
       Ty = P->desugar().getTypePtr();
     else if (const auto *A = dyn_cast<AdjustedType>(Ty))
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 52ef7ac54145e..00835f1490eda 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -16,7 +16,7 @@
 
 #include "clang/AST/ASTConcept.h"
 #include "clang/AST/DeclarationName.h"
-#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/NestedNameSpecifierBase.h"
 #include "clang/AST/TemplateBase.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/LLVM.h"
@@ -193,6 +193,21 @@ class TypeLoc {
   /// Get the SourceLocation of the template keyword (if any).
   SourceLocation getTemplateKeywordLoc() const;
 
+  /// If this type represents a qualified-id, this returns it's nested name
+  /// specifier. For example, for the qualified-id "foo::bar::baz", this returns
+  /// "foo::bar". Returns null if this type represents an unqualified-id.
+  NestedNameSpecifierLoc getPrefix() const;
+
+  /// This returns the position of the type after any elaboration, such as the
+  /// 'struct' keyword, and name qualifiers. This will the 'template' keyword if
+  /// present, or the name location otherwise.
+  SourceLocation getNonPrefixBeginLoc() const;
+
+  /// This returns the position of the type after any elaboration, such as the
+  /// 'struct' keyword. This may be the position of the name qualifiers,
+  /// 'template' keyword, or the name location otherwise.
+  SourceLocation getNonElaboratedBeginLoc() const;
+
   /// Initializes this to state that every location in this
   /// type is the given location.
   ///
@@ -679,62 +694,164 @@ class BuiltinTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc,
   }
 };
 
-/// Wrapper for source info for types used via transparent aliases.
-class UsingTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
-                                                      UsingTypeLoc, UsingType> {
-public:
-  QualType getUnderlyingType() const {
-    return getTypePtr()->getUnderlyingType();
+struct ElaboratedNameLocInfo {
+  SourceLocation NameLoc;
+  SourceLocation ElaboratedKeywordLoc;
+
+  ElaboratedNameLocInfo() = default;
+  ElaboratedNameLocInfo(SourceLocation ElaboratedKeywordLoc,
+                        NestedNameSpecifierLoc QualifierLoc,
+                        SourceLocation NameLoc)
+      : NameLoc(NameLoc), ElaboratedKeywordLoc(ElaboratedKeywordLoc),
+        QualifierData(QualifierLoc.getOpaqueData()) {}
+  ElaboratedNameLocInfo(ASTContext &Context, ElaboratedTypeKeyword Keyword,
+                        NestedNameSpecifier Qualifier, SourceLocation Loc)
+      : NameLoc(Loc),
+        ElaboratedKeywordLoc(
+            Keyword != ElaboratedTypeKeyword::None ? Loc : SourceLocation()),
+        QualifierData(getTrivialQualifierData(Context, Qualifier, Loc)) {}
+
+  NestedNameSpecifierLoc getQualifierLoc(NestedNameSpecifier Qualifier) const {
+    assert(!Qualifier == !QualifierData);
+    return NestedNameSpecifierLoc(Qualifier, QualifierData);
+  }
+
+  SourceRange getLocalSourceRange(NestedNameSpecifier Qualifier) const {
+    SourceLocation BeginLoc = ElaboratedKeywordLoc;
+    if (NestedNameSpecifierLoc QualifierLoc = getQualifierLoc(Qualifier);
+        BeginLoc.isInvalid() && Qualifier)
+      BeginLoc = QualifierLoc.getBeginLoc();
+    if (BeginLoc.isInvalid())
+      BeginLoc = NameLoc;
+    return SourceRange(BeginLoc, NameLoc);
   }
-  UsingShadowDecl *getFoundDecl() const { return getTypePtr()->getFoundDecl(); }
-};
 
-/// Wrapper for source info for typedefs.
-class TypedefTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
-                                                        TypedefTypeLoc,
-                                                        TypedefType> {
-public:
-  TypedefNameDecl *getTypedefNameDecl() const {
-    return getTypePtr()->getDecl();
+private:
+  void *QualifierData;
+
+  static void *getTrivialQualifierData(ASTContext &Context,
+                                       NestedNameSpecifier Qualifier,
+                                       SourceLocation Loc) {
+    if (!Qualifier)
+      return nullptr;
+    NestedNameSpecifierLocBuilder Builder;
+    Builder.MakeTrivial(Context, Qualifier, Loc);
+    return Builder.getWithLocInContext(Context).getOpaqueData();
   }
 };
 
-/// Wrapper for source info for injected class names of class
-/// templates.
-class InjectedClassNameTypeLoc :
-    public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
-                                     InjectedClassNameTypeLoc,
-                                     InjectedClassNameType> {
+template <class TL, class T>
+class ElaboratedNameTypeLoc
+    : public ConcreteTypeLoc<UnqualTypeLoc, TL, T, ElaboratedNameLocInfo> {
 public:
-  CXXRecordDecl *getDecl() const {
-    return getTypePtr()->getDecl();
+  auto *getDecl() const { return this->getTypePtr()->getDecl(); }
+
+  void set(SourceLocation ElaboratedKeywordLoc,
+           NestedNameSpecifierLoc QualifierLoc, SourceLocation NameLoc) {
+    assert(QualifierLoc.getNestedNameSpecifier() ==
+           this->getTypePtr()->getQualifier());
+    *this->getLocalData() =
+        ElaboratedNameLocInfo(ElaboratedKeywordLoc, QualifierLoc, NameLoc);
+  }
+
+  SourceLocation getElaboratedKeywordLoc() const {
+    return this->getLocalData()->ElaboratedKeywordLoc;
+  }
+
+  NestedNameSpecifierLoc getQualifierLoc() const {
+    return this->getLocalData()->getQualifierLoc(
+        this->getTypePtr()->getQualifier());
+  }
+
+  SourceLocation getNameLoc() const { return this->getLocalData()->NameLoc; }
+
+  SourceRange getLocalSourceRange() const {
+    return this->getLocalData()->getLocalSourceRange(
+        this->getTypePtr()->getQualifier());
+  }
+
+  void initializeLocal(ASTContext &Context, SourceLocation Loc) {
+    const auto *Ptr = this->getTypePtr();
+    *this->getLocalData() = ElaboratedNameLocInfo(Context, Ptr->getKeyword(),
+                                                  Ptr->getQualifier(), Loc);
   }
 };
 
+/// Wrapper for source info for typedefs.
+class TypedefTypeLoc
+    : public ElaboratedNameTypeLoc<TypedefTypeLoc, TypedefType> {};
+
 /// Wrapper for source info for unresolved typename using decls.
-class UnresolvedUsingTypeLoc :
-    public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
-                                     UnresolvedUsingTypeLoc,
-                                     UnresolvedUsingType> {
-public:
-  UnresolvedUsingTypenameDecl *getDecl() const {
-    return getTypePtr()->getDecl();
-  }
+class UnresolvedUsingTypeLoc
+    : public ElaboratedNameTypeLoc<UnresolvedUsingTypeLoc,
+                                   UnresolvedUsingType> {};
+
+/// Wrapper for source info for types used via transparent aliases.
+class UsingTypeLoc : public ElaboratedNameTypeLoc<UsingTypeLoc, UsingType> {};
+
+struct TagTypeLocInfo {
+  SourceLocation NameLoc;
+  SourceLocation ElaboratedKWLoc;
+  void *QualifierData;
 };
 
-/// Wrapper for source info for tag types.  Note that this only
-/// records source info for the name itself; a type written 'struct foo'
-/// should be represented as an ElaboratedTypeLoc.  We currently
-/// only do that when C++ is enabled because of the expense of
-/// creating an ElaboratedType node for so many type references in C.
-class TagTypeLoc : public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
-                                                    TagTypeLoc,
-                                                    TagType> {
+class TagTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc, TagTypeLoc, TagType,
+                                          TagTypeLocInfo> {
 public:
-  TagDecl *getDecl() const { return getTypePtr()->getDecl(); }
+  TagDecl *getOriginalDecl() const { return getTypePtr()->getOriginalDecl(); }
 
   /// True if the tag was defined in this type specifier.
   bool isDefinition() const;
+
+  SourceLocation getElaboratedKeywordLoc() const {
+    return getLocalData()->ElaboratedKWLoc;
+  }
+
+  void setElaboratedKeywordLoc(SourceLocation Loc) {
+    getLocalData()->ElaboratedKWLoc = Loc;
+  }
+
+  NestedNameSpecifierLoc getQualifierLoc() const {
+    NestedNameSpecifier Qualifier = getTypePtr()->getQualifier();
+    void *QualifierData = getLocalData()->QualifierData;
+    assert(!Qualifier == !QualifierData);
+    return NestedNameSpecifierLoc(Qualifier, QualifierData);
+  }
+
+  void setQualifierLoc(NestedNameSpecifierLoc QualifierLoc) {
+    assert(QualifierLoc.getNestedNameSpecifier() ==
+           getTypePtr()->getQualifier());
+    getLocalData()->QualifierData = QualifierLoc.getOpaqueData();
+  }
+
+  SourceLocation getNameLoc() const { return getLocalData()->NameLoc; }
+
+  void setNameLoc(SourceLocation Loc) { getLocalData()->NameLoc = Loc; }
+
+  SourceRange getLocalSourceRange() const {
+    SourceLocation BeginLoc = getElaboratedKeywordLoc();
+    if (NestedNameSpecifierLoc Qualifier = getQualifierLoc();
+        BeginLoc.isInvalid() && Qualifier)
+      BeginLoc = Qualifier.getBeginLoc();
+    if (BeginLoc.isInvalid())
+      BeginLoc = getNameLoc();
+    return SourceRange(BeginLoc, getNameLoc());
+  }
+
+  void initializeLocal(ASTContext &Context, SourceLocation Loc) {
+    setElaboratedKeywordLoc(getTypePtr()->getKeyword() !=
+                                    ElaboratedTypeKeyword::None
+                                ? Loc
+                                : SourceLocation());
+    if (NestedNameSpecifier Qualifier = getTypePtr()->getQualifier()) {
+      NestedNameSpecifierLocBuilder Builder;
+      Builder.MakeTrivial(Context, Qualifier, Loc);
+      setQualifierLoc(Builder.getWithLocInContext(Context));
+    } else {
+      getLocalData()->QualifierData = nullptr;
+    }
+    setNameLoc(Loc);
+  }
 };
 
 /// Wrapper for source info for record types.
@@ -742,7 +859,9 @@ class RecordTypeLoc : public InheritingConcreteTypeLoc<TagTypeLoc,
                                                        RecordTypeLoc,
                                                        RecordType> {
 public:
-  RecordDecl *getDecl() const { return getTypePtr()->getDecl(); }
+  RecordDecl *getOriginalDecl() const {
+    return getTypePtr()->getOriginalDecl();
+  }
 };
 
 /// Wrapper for source info for enum types.
@@ -750,7 +869,18 @@ class EnumTypeLoc : public InheritingConcreteTypeLoc<TagTypeLoc,
                                                      EnumTypeLoc,
                                                      EnumType> {
 public:
-  EnumDecl *getDecl() const { return getTypePtr()->getDecl(); }
+  EnumDecl *getOriginalDecl() const { return getTypePtr()->getOriginalDecl(); }
+};
+
+/// Wrapper for source info for injected class names of class
+/// templates.
+class InjectedClassNameTypeLoc
+    : public InheritingConcreteTypeLoc<TagTypeLoc, InjectedClassNameTypeLoc,
+                                       InjectedClassNameType> {
+public:
+  CXXRecordDecl *getOriginalDecl() const {
+    return getTypePtr()->getOriginalDecl();
+  }
 };
 
 /// Wrapper for template type parameters.
@@ -859,12 +989,22 @@ class SubstTemplateTypeParmTypeLoc :
                                      SubstTemplateTypeParmType> {
 };
 
-  /// Wrapper for substituted template type parameters.
-class SubstTemplateTypeParmPackTypeLoc :
-    public InheritingConcreteTypeLoc<TypeSpecTypeLoc,
-                                     SubstTemplateTypeParmPackTypeLoc,
-                                     SubstTemplateTypeParmPackType> {
-};
+/// Abstract type representing delayed type pack expansions.
+class SubstPackTypeLoc
+    : public InheritingConcreteTypeLoc<TypeSpecTypeLoc, SubstPackTypeLoc,
+                                       SubstPackType> {};
+
+/// Wrapper for substituted template type parameters.
+class SubstTemplateTypeParmPackTypeLoc
+    : public InheritingConcreteTypeLoc<SubstPackTypeLoc,
+                                       SubstTemplateTypeParmPackTypeLoc,
+                                       SubstTemplateTypeParmPackType> {};
+
+/// Wrapper for substituted template type parameters.
+class SubstBuiltinTemplatePackTypeLoc
+    : public InheritingConcreteTypeLoc<SubstPackTypeLoc,
+                                       SubstBuiltinTemplatePackTypeLoc,
+                                       SubstBuiltinTemplatePackType> {};
 
 struct AttributedLocInfo {
   const Attr *TypeAttr;
@@ -1405,7 +1545,7 @@ class MemberPointerTypeLoc : public PointerLikeTypeLoc<MemberPointerTypeLoc,
 
   void initializeLocal(ASTContext &Context, SourceLocation Loc) {
     setSigilLoc(Loc);
-    if (auto *Qualifier = getTypePtr()->getQualifier()) {
+    if (NestedNameSpecifier Qualifier = getTypePtr()->getQualifier()) {
       NestedNameSpecifierLocBuilder Builder;
       Builder.MakeTrivial(Context, Qualifier, Loc);
       setQualifierLoc(Builder.getWithLocInContext(Context));
@@ -1701,9 +1841,11 @@ struct TemplateNameLocInfo {
 };
 
 struct TemplateSpecializationLocInfo : TemplateNameLocInfo {
+  SourceRange SR;
+  SourceLocation ElaboratedKWLoc;
   SourceLocation TemplateKWLoc;
   SourceLocation LAngleLoc;
-  SourceLocation RAngleLoc;
+  void *QualifierData;
 };
 
 class TemplateSpecializationTypeLoc :
@@ -1712,54 +1854,53 @@ class TemplateSpecializationTypeLoc :
                            TemplateSpecializationType,
                            TemplateSpecializationLocInfo> {
 public:
-  SourceLocation getTemplateKeywordLoc() const {
-    return getLocalData()->TemplateKWLoc;
-  }
+  void set(SourceLocation ElaboratedKeywordLoc,
+           NestedNameSpecifierLoc QualifierLoc,
+           SourceLocation TemplateKeywordLoc, SourceLocation NameLoc,
+           SourceLocation LAngleLoc, SourceLocation RAngleLoc);
 
-  void setTemplateKeywordLoc(SourceLocation Loc) {
-    getLocalData()->TemplateKWLoc = Loc;
-  }
+  void set(SourceLocation ElaboratedKeywordLoc,
+           NestedNameSpecifierLoc QualifierLoc,
+           SourceLocation TemplateKeywordLoc, SourceLocation NameLoc,
+           const TemplateArgumentListInfo &TAL);
 
-  SourceLocation getLAngleLoc() const {
-    return getLocalData()->LAngleLoc;
+  SourceLocation getElaboratedKeywordLoc() const {
+    return getLocalData()->ElaboratedKWLoc;
   }
 
-  void setLAngleLoc(SourceLocation Loc) {
-    getLocalData()->LAngleLoc = Loc;
-  }
+  NestedNameSpecifierLoc getQualifierLoc() const {
+    if (!getLocalData()->QualifierData)
+      return NestedNameSpecifierLoc();
 
-  SourceLocation getRAngleLoc() const {
-    return getLocalData()->RAngleLoc;
+    auto *QTN =
+        getTypePtr()->getTemplateName().getAsAdjustedQualifiedTemplateName();
+    assert(QTN && "missing qualification");
+    return NestedNameSpecifierLoc(QTN->getQualifier(),
+                                  getLocalData()->QualifierData);
   }
 
-  void setRAngleLoc(SourceLocation Loc) {
-    getLocalData()->RAngleLoc = Loc;
+  SourceLocation getTemplateKeywordLoc() const {
+    return getLocalData()->TemplateKWLoc;
   }
 
+  SourceLocation getTemplateNameLoc() const { return getLocalData()->NameLoc; }
+
+  SourceLocation getLAngleLoc() const { return getLocalData()->LAngleLoc; }
+
   unsigned getNumArgs() const {
     return getTypePtr()->template_arguments().size();
   }
 
-  void setArgLocInfo(unsigned i, TemplateArgumentLocInfo AI) {
-    getArgInfos()[i] = AI;
-  }
-
-  TemplateArgumentLocInfo getArgLocInfo(unsigned i) const {
-    return getArgInfos()[i];
+  MutableArrayRef<TemplateArgumentLocInfo> getArgLocInfos() {
+    return {getArgInfos(), getNumArgs()};
   }
 
   TemplateArgumentLoc getArgLoc(unsigned i) const {
     return TemplateArgumentLoc(getTypePtr()->template_arguments()[i],
-                               getArgLocInfo(i));
-  }
-
-  SourceLocation getTemplateNameLoc() const {
-    return getLocalData()->NameLoc;
+                               getArgInfos()[i]);
   }
 
-  void setTemplateNameLoc(SourceLocation Loc) {
-    getLocalData()->NameLoc = Loc;
-  }
+  SourceLocation getRAngleLoc() const { return getLocalData()->SR.getEnd(); }
 
   /// - Copy the location information from the given info.
   void copy(TemplateSpecializationTypeLoc Loc) {
@@ -1773,21 +1914,9 @@ class TemplateSpecializationTypeLoc :
     memcpy(Data, Loc.Data, size);
   }
 
-  SourceRange getLocalSourceRange() const {
-    if (getTemplateKeywordLoc().isValid())
-      return SourceRange(getTemplateKeywordLoc(), getRAngleLoc());
-    else
-      return SourceRange(getTemplateNameLoc(), getRAngleLoc());
-  }
+  SourceRange getLocalSourceRange() const { return getLocalData()->SR; }
 
-  void initializeLocal(ASTContext &Context, SourceLocation Loc) {
-    setTemplateKeywordLoc(SourceLocation());
-    setTemplateNameLoc(Loc);
-    setLAngleLoc(Loc);
-    setRAngleLoc(Loc);
-    initializeArgLocs(Context, getTypePtr()->template_arguments(),
-                      getArgInfos(), Loc);
-  }
+  void initializeLocal(ASTContext &Context, SourceLocation Loc);
 
   static void initializeArgLocs(ASTContext &Context,
                                 ArrayRef<TemplateArgument> Args,
@@ -2346,99 +2475,77 @@ class AutoTypeLoc
   void initializeLocal(ASTContext &Context, SourceLocation Loc);
 };
 
-class DeducedTemplateSpecializationTypeLoc
-    : public InheritingConcreteTypeLoc<DeducedTypeLoc,
-                                       DeducedTemplateSpecializationTypeLoc,
-                                       DeducedTemplateSpecializationType> {
-public:
-  SourceLocation getTemplateNameLoc() const {
-    return getNameLoc();
-  }
-
-  void setTemplateNameLoc(SourceLocation Loc) {
-    setNameLoc(Loc);
-  }
-};
-
-struct ElaboratedLocInfo {
+struct DeducedTemplateSpecializationLocInfo : TypeSpecLocInfo {
   SourceLocation ElaboratedKWLoc;
-
   /// Data associated with the nested-name-specifier location.
   void *QualifierData;
 };
 
-class ElaboratedTypeLoc : public ConcreteTypeLoc<UnqualTypeLoc,
-                                                 ElaboratedTypeLoc,
-                                                 ElaboratedType,
-                                                 ElaboratedLocInfo> {
+class DeducedTemplateSpecializationTypeLoc
+    : public ConcreteTypeLoc<DeducedTypeLoc,
+                             DeducedTemplateSpecializationTypeLoc,
+                             DeducedTemplateSpecializationType,
+                             DeducedTemplateSpecializationLocInfo> {
 public:
   SourceLocation getElaboratedKeywordLoc() const {
-    return !isEmpty() ? getLocalData()->ElaboratedKWLoc : SourceLocation();
+    return getLocalData()->ElaboratedKWLoc;
   }
 
   void setElaboratedKeywordLoc(SourceLocation Loc) {
-    if (isEmpty()) {
-      assert(Loc.isInvalid());
-      return;
-    }
     getLocalData()->ElaboratedKWLoc = Loc;
   }
 
+  SourceLocation getTemplateNameLoc() const { return getNameLoc(); }
+
+  void setTemplateNameLoc(SourceLocation Loc) { setNameLoc(Loc); }
+
   NestedNameSpecifierLoc getQualifierLoc() const {
-    return !isEmpty() ? NestedNameSpecifierLoc(getTypePtr()->getQualifier(),
-                                               getLocalData()->QualifierData)
-                      : NestedNameSpecifierLoc();
+    void *Data = getLocalData()->QualifierData;
+    if (!Data)
+      return NestedNameSpecifierLoc();
+    NestedNameSpecifier Qualifier = getTypePtr()
+                                        ->getTemplateName()
+                                        .getAsAdjustedQualifiedTemplateName()
+                                        ->getQualifier();
+    return NestedNameSpecifierLoc(Qualifier, Data);
   }
 
   void setQualifierLoc(NestedNameSpecifierLoc QualifierLoc) {
-    assert(QualifierLoc.getNestedNameSpecifier() ==
-               getTypePtr()->getQualifier() &&
-           "Inconsistent nested-name-specifier pointer");
-    if (isEmpty()) {
-      assert(!QualifierLoc.hasQualifier());
+    if (!QualifierLoc) {
+      // Even if we have a nested-name-specifier in the dependent
+      // template specialization type, we won't record the nested-name-specifier
+      // location information when this type-source location information is
+      // part of a nested-name-specifier.
+      getLocalData()->QualifierData = nullptr;
       return;
     }
+
+    assert(QualifierLoc.getNestedNameSpecifier() ==
+               getTypePtr()
+                   ->getTemplateName()
+                   .getAsAdjustedQualifiedTemplateName()
+                   ->getQualifier() &&
+           "Inconsistent nested-name-specifier pointer");
     getLocalData()->QualifierData = QualifierLoc.getOpaqueData();
   }
 
   SourceRange getLocalSourceRange() const {
-    if (getElaboratedKeywordLoc().isValid())
-      if (getQualifierLoc())
-        return SourceRange(getElaboratedKeywordLoc(),
-                           getQualifierLoc().getEndLoc());
-      else
-        return SourceRange(getElaboratedKeywordLoc());
-    else
-      return getQualifierLoc().getSourceRange();
+    SourceLocation BeginLoc = getElaboratedKeywordLoc();
+    if (BeginLoc.isInvalid())
+      BeginLoc = getQualifierLoc().getBeginLoc();
+    if (BeginLoc.isInvalid())
+      BeginLoc = getNameLoc();
+    return {BeginLoc, getNameLoc()};
   }
 
   void initializeLocal(ASTContext &Context, SourceLocation Loc);
+};
 
-  TypeLoc getNamedTypeLoc() const { return getInnerTypeLoc(); }
-
-  QualType getInnerType() const { return getTypePtr()->getNamedType(); }
-
-  bool isEmpty() const {
-    return getTypePtr()->getKeyword() == ElaboratedTypeKeyword::None &&
-           !getTypePtr()->getQualifier();
-  }
-
-  unsigned getLocalDataAlignment() const {
-    // FIXME: We want to return 1 here in the empty case, but
-    // there are bugs in how alignment is handled in TypeLocs
-    // that prevent this from working.
-    return ConcreteTypeLoc::getLocalDataAlignment();
-  }
-
-  unsigned getLocalDataSize() const {
-    return !isEmpty() ? ConcreteTypeLoc::getLocalDataSize() : 0;
-  }
+struct ElaboratedLocInfo {
+  SourceLocation ElaboratedKWLoc;
 
-  void copy(ElaboratedTypeLoc Loc) {
-    unsigned size = getFullDataSize();
-    assert(size == Loc.getFullDataSize());
-    memcpy(Data, Loc.Data, size);
-  }
+  /// Data associated with the nested-name-specifier location.
+  void *QualifierData;
 };
 
 // This is exactly the structure of an ElaboratedTypeLoc whose inner
@@ -2749,8 +2856,6 @@ inline T TypeLoc::getAsAdjusted() const {
       Cur = ATL.getWrappedLoc();
     else if (auto ATL = Cur.getAs<HLSLAttributedResourceTypeLoc>())
       Cur = ATL.getWrappedLoc();
-    else if (auto ETL = Cur.getAs<ElaboratedTypeLoc>())
-      Cur = ETL.getNamedTypeLoc();
     else if (auto ATL = Cur.getAs<AdjustedTypeLoc>())
       Cur = ATL.getOriginalLoc();
     else if (auto MQL = Cur.getAs<MacroQualifiedTypeLoc>())
diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 3373e963038f1..185a968217f97 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -379,38 +379,59 @@ let Class = AtomicType in {
 }
 
 let Class = UnresolvedUsingType in {
-  def : Property<"declaration", DeclRef> {
-    let Read = [{ node->getDecl() }];
+  def : Property<"IsCanonical", Bool> {
+    let Read = [{ node->isCanonicalUnqualified() }];
   }
-
+  def : Property<"Keyword", ElaboratedTypeKeyword> {
+    let Conditional = [{ !IsCanonical }];
+    let Read = [{ node->getKeyword() }];
+  }
+  def : Property<"Qualifier", NestedNameSpecifier> {
+    let Conditional = [{ !IsCanonical }];
+    let Read = [{ node->getQualifier() }];
+  }
+  def : Property<"D", DeclRef> { let Read = [{ node->getDecl() }]; }
   def : Creator<[{
-    return ctx.getUnresolvedUsingType(cast<UnresolvedUsingTypenameDecl>(declaration));
+    auto *UD = cast<UnresolvedUsingTypenameDecl>(D);
+    return IsCanonical ? ctx.getCanonicalUnresolvedUsingType(UD) : ctx.getUnresolvedUsingType(*Keyword, *Qualifier, UD);
   }]>;
 }
 
 let Class = UsingType in {
-  def : Property<"foundDeclaration", UsingShadowDeclRef> {
-    let Read = [{ node->getFoundDecl() }];
+  def : Property<"Keyword", ElaboratedTypeKeyword> {
+    let Read = [{ node->getKeyword() }];
   }
-  def : Property<"underlyingType", QualType> {
-    let Read = [{ node->getUnderlyingType() }];
+  def : Property<"Qualifier", NestedNameSpecifier> {
+    let Read = [{ node->getQualifier() }];
+  }
+  def : Property<"D", UsingShadowDeclRef> { let Read = [{ node->getDecl() }]; }
+  def : Property<"UnderlyingType", QualType> {
+    let Read = [{ node->desugar() }];
   }
-
   def : Creator<[{
-    return ctx.getUsingType(foundDeclaration, underlyingType);
+    return ctx.getUsingType(Keyword, Qualifier, D, UnderlyingType);
   }]>;
 }
 
 let Class = TypedefType in {
+  def : Property<"Keyword", ElaboratedTypeKeyword> {
+    let Read = [{ node->getKeyword() }];
+  }
+  def : Property<"Qualifier", NestedNameSpecifier> {
+    let Read = [{ node->getQualifier() }];
+  }
   def : Property<"declaration", DeclRef> {
     let Read = [{ node->getDecl() }];
   }
-  def : Property<"underlyingType", QualType> {
+  def : Property<"UnderlyingType", QualType> {
     let Read = [{ node->desugar() }];
   }
+  def : Property<"TypeMatchesDecl", Bool> {
+    let Read = [{ node->typeMatchesDecl() }];
+  }
 
   def : Creator<[{
-    return ctx.getTypedefType(cast<TypedefNameDecl>(declaration), underlyingType);
+    return ctx.getTypedefType(Keyword, Qualifier, cast<TypedefNameDecl>(declaration), UnderlyingType, TypeMatchesDecl);
   }]>;
 }
 
@@ -520,6 +541,9 @@ let Class = AutoType in {
 }
 
 let Class = DeducedTemplateSpecializationType in {
+  def : Property<"keyword", ElaboratedTypeKeyword> {
+    let Read = [{ node->getKeyword() }];
+  }
   def : Property<"templateName", Optional<TemplateName>> {
     let Read = [{ makeOptionalFromNullable(node->getTemplateName()) }];
   }
@@ -533,97 +557,42 @@ let Class = DeducedTemplateSpecializationType in {
   }
 
   def : Creator<[{
-    return ctx.getDeducedTemplateSpecializationType(
+    return ctx.getDeducedTemplateSpecializationType(keyword,
                                      makeNullableFromOptional(templateName),
                                      deducedType, dependent);
   }]>;
 }
 
 let Class = TagType in {
-  def : Property<"dependent", Bool> {
-    let Read = [{ node->isDependentType() }];
+  def : Property<"IsCanonical", Bool> {
+    let Read = [{ node->isCanonicalUnqualified() }];
   }
-  def : Property<"declaration", DeclRef> {
-    // We don't know which declaration was originally referenced here, and we
-    // cannot reference a declaration that follows the use (because that can
-    // introduce deserialization cycles), so conservatively generate a
-    // reference to the first declaration.
-    // FIXME: If this is a reference to a class template specialization, that
-    // can still introduce a deserialization cycle.
-    let Read = [{ node->getDecl()->getCanonicalDecl() }];
+  def : Property<"Keyword", ElaboratedTypeKeyword> {
+    let Conditional = [{ !IsCanonical }];
+    let Read = [{ node->getKeyword() }];
   }
+  def : Property<"Qualifier", NestedNameSpecifier> {
+    let Conditional = [{ !IsCanonical }];
+    let Read = [{ node->getQualifier() }];
+  }
+  def : Property<"TD", TagDeclRef> { let Read = [{ node->getOriginalDecl() }]; }
 }
 
 let Class = EnumType in {
+  def : Property<"OwnsTag", Bool> { let Read = [{ node->isTagOwned() }]; }
   def : Creator<[{
-    QualType result = ctx.getEnumType(cast<EnumDecl>(declaration));
-    if (dependent)
-      const_cast<Type *>(result.getTypePtr())
-          ->addDependence(TypeDependence::DependentInstantiation);
-    return result;
+    return IsCanonical ? ctx.getCanonicalTagType(TD) : ctx.getTagType(*Keyword, *Qualifier, TD, OwnsTag);
   }]>;
 }
-
 let Class = RecordType in {
+  def : Property<"OwnsTag", Bool> { let Read = [{ node->isTagOwned() }]; }
   def : Creator<[{
-    auto record = cast<RecordDecl>(declaration);
-    QualType result = ctx.getRecordType(record);
-    if (dependent)
-      const_cast<Type *>(result.getTypePtr())
-          ->addDependence(TypeDependence::DependentInstantiation);
-    return result;
-  }]>;
-}
-
-let Class = ElaboratedType in {
-  def : Property<"keyword", ElaboratedTypeKeyword> {
-    let Read = [{ node->getKeyword() }];
-  }
-  def : Property<"qualifier", NestedNameSpecifier> {
-    let Read = [{ node->getQualifier() }];
-  }
-  def : Property<"namedType", QualType> {
-    let Read = [{ node->getNamedType() }];
-  }
-  def : Property<"ownedTag", Optional<TagDeclRef>> {
-    let Read = [{ makeOptionalFromPointer(
-                    const_cast<const TagDecl *>(node->getOwnedTagDecl())) }];
-  }
-
-  def : Creator<[{
-    return ctx.getElaboratedType(keyword, qualifier, namedType,
-                                 makePointerFromOptional(ownedTag));
+    return IsCanonical ? ctx.getCanonicalTagType(TD) : ctx.getTagType(*Keyword, *Qualifier, TD, OwnsTag);
   }]>;
 }
-
 let Class = InjectedClassNameType in {
-  def : Property<"declaration", DeclRef> {
-    // FIXME: drilling down to the canonical declaration is what the
-    // existing serialization code was doing, but it's not clear why.
-    let Read = [{ node->getDecl()->getCanonicalDecl() }];
-  }
-  def : Property<"injectedSpecializationType", QualType> {
-    let Read = [{ node->getInjectedSpecializationType() }];
-  }
-
   def : Creator<[{
-    // FIXME: ASTContext::getInjectedClassNameType is not currently suitable
-    // for AST reading, too much interdependencies.
-    const Type *T = nullptr;
-    auto typeDecl = cast<CXXRecordDecl>(declaration);
-    for (auto *DI = typeDecl; DI; DI = DI->getPreviousDecl()) {
-      if (const Type *existing = DI->getTypeForDecl()) {
-        T = existing;
-        break;
-      }
-    }
-    if (!T) {
-      T = new (ctx, TypeAlignment)
-            InjectedClassNameType(typeDecl, injectedSpecializationType);
-      for (auto *DI = typeDecl; DI; DI = DI->getPreviousDecl())
-        DI->setTypeForDecl(T);
-    }
-    return QualType(T, 0);
+    return IsCanonical ? ctx.getCanonicalTagType(TD) : ctx.getTagType(*Keyword, *Qualifier, TD, /*OwnsTag=*/false);
   }]>;
 }
 
@@ -741,6 +710,9 @@ let Class = DependentAddressSpaceType in {
 }
 
 let Class = TemplateSpecializationType in {
+  def : Property<"keyword", ElaboratedTypeKeyword> {
+    let Read = [{ node->getKeyword() }];
+  }
   def : Property<"templateName", TemplateName> {
     let Read = [{ node->getTemplateName() }];
   }
@@ -753,7 +725,7 @@ let Class = TemplateSpecializationType in {
   }
 
   def : Creator<[{
-    return ctx.getTemplateSpecializationType(templateName, args, {}, UnderlyingType);
+    return ctx.getTemplateSpecializationType(keyword, templateName, args, {}, UnderlyingType);
   }]>;
 }
 
@@ -848,6 +820,12 @@ let Class = PackExpansionType in {
   }]>;
 }
 
+let Class = SubstPackType in {
+  def : Property<"replacementPack", TemplateArgument> {
+    let Read = [{ node->getArgumentPack() }];
+  }
+}
+
 let Class = SubstTemplateTypeParmPackType in {
   def : Property<"associatedDecl", DeclRef> {
     let Read = [{ node->getAssociatedDecl() }];
@@ -855,12 +833,7 @@ let Class = SubstTemplateTypeParmPackType in {
   def : Property<"Index", UInt32> {
     let Read = [{ node->getIndex() }];
   }
-  def : Property<"Final", Bool> {
-    let Read = [{ node->getFinal() }];
-  }
-  def : Property<"replacementPack", TemplateArgument> {
-    let Read = [{ node->getArgumentPack() }];
-  }
+  def : Property<"Final", Bool> { let Read = [{ node->getFinal() }]; }
 
   def : Creator<[{
     return ctx.getSubstTemplateTypeParmPackType(
@@ -868,6 +841,12 @@ let Class = SubstTemplateTypeParmPackType in {
   }]>;
 }
 
+let Class = SubstBuiltinTemplatePackType in {
+  def : Creator<[{
+    return ctx.getSubstBuiltinTemplatePack(replacementPack);
+  }]>;
+}
+
 let Class = BuiltinType in {
   def : Property<"kind", BuiltinTypeKind> {
     let Read = [{ node->getKind() }];
diff --git a/clang/include/clang/ASTMatchers/ASTMatchFinder.h b/clang/include/clang/ASTMatchers/ASTMatchFinder.h
index 73cbcf1f25025..2d36e8c4fae1c 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchFinder.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchFinder.h
@@ -135,10 +135,15 @@ class MatchFinder {
       llvm::StringMap<llvm::TimeRecord> &Records;
     };
 
+    MatchFinderOptions() {}
+
     /// Enables per-check timers.
     ///
     /// It prints a report after match.
     std::optional<Profiling> CheckProfiling;
+
+    /// Avoids matching declarations in system headers.
+    bool IgnoreSystemHeaders{false};
   };
 
   MatchFinder(MatchFinderOptions Options = MatchFinderOptions());
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index 08c898f7758ec..4520c4af38c8d 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -222,6 +222,19 @@ extern const internal::VariadicDynCastAllOfMatcher<Decl, TypedefNameDecl>
 extern const internal::VariadicDynCastAllOfMatcher<Decl, TypeAliasDecl>
     typeAliasDecl;
 
+/// \brief Matches shadow declarations introduced into a scope by a
+///        (resolved) using declaration.
+///
+/// Given
+/// \code
+///   namespace n { int f; }
+///   namespace declToImport { using n::f; }
+/// \endcode
+/// usingShadowDecl()
+///   matches \code f \endcode
+extern const internal::VariadicDynCastAllOfMatcher<Decl, UsingShadowDecl>
+    usingShadowDecl;
+
 /// Matches type alias template declarations.
 ///
 /// typeAliasTemplateDecl() matches
@@ -3740,7 +3753,7 @@ extern const internal::VariadicOperatorMatcherFunc<1, 1> unless;
 ///   Matcher<MemberExpr>, Matcher<QualType>, Matcher<RecordType>,
 ///   Matcher<TagType>, Matcher<TemplateSpecializationType>,
 ///   Matcher<TemplateTypeParmType>, Matcher<TypedefType>,
-///   Matcher<UnresolvedUsingType>
+///   Matcher<UnresolvedUsingType>, Matcher<UsingType>
 inline internal::PolymorphicMatcher<
     internal::HasDeclarationMatcher,
     void(internal::HasDeclarationSupportedTypes), internal::Matcher<Decl>>
@@ -4375,7 +4388,13 @@ AST_POLYMORPHIC_MATCHER_P(throughUsingDecl,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(DeclRefExpr,
                                                           UsingType),
                           internal::Matcher<UsingShadowDecl>, Inner) {
-  const NamedDecl *FoundDecl = Node.getFoundDecl();
+  const NamedDecl *FoundDecl;
+  if constexpr (std::is_same_v<NodeType, UsingType>) {
+    FoundDecl = Node.getDecl();
+  } else {
+    static_assert(std::is_same_v<NodeType, DeclRefExpr>);
+    FoundDecl = Node.getFoundDecl();
+  }
   if (const UsingShadowDecl *UsingDecl = dyn_cast<UsingShadowDecl>(FoundDecl))
     return Inner.matches(*UsingDecl, Finder, Builder);
   return false;
@@ -5642,8 +5661,8 @@ AST_POLYMORPHIC_MATCHER_P(hasInitStatement,
   return Init != nullptr && InnerMatcher.matches(*Init, Finder, Builder);
 }
 
-/// Matches the condition expression of an if statement, for loop,
-/// switch statement or conditional operator.
+/// Matches the condition expression of an if statement, for loop, while loop,
+/// do-while loop, switch statement or conditional operator.
 ///
 /// Example matches true (matcher = hasCondition(cxxBoolLiteral(equals(true))))
 /// \code
@@ -5720,16 +5739,29 @@ AST_POLYMORPHIC_MATCHER_P(equalsBoundNode,
   return Builder->removeBindings(Predicate);
 }
 
-/// Matches the condition variable statement in an if statement.
+/// Matches a declaration if it declares the same entity as the node previously
+/// bound to \p ID.
+AST_MATCHER_P(Decl, declaresSameEntityAsBoundNode, std::string, ID) {
+  return Builder->removeBindings([&](const internal::BoundNodesMap &Nodes) {
+    return !clang::declaresSameEntity(&Node, Nodes.getNodeAs<Decl>(ID));
+  });
+}
+
+/// Matches the condition variable statement in an if statement, for loop,
+/// while loop or switch statement.
 ///
 /// Given
 /// \code
 ///   if (A* a = GetAPointer()) {}
+///   for (; A* a = GetAPointer(); ) {}
 /// \endcode
 /// hasConditionVariableStatement(...)
-///   matches 'A* a = GetAPointer()'.
-AST_MATCHER_P(IfStmt, hasConditionVariableStatement,
-              internal::Matcher<DeclStmt>, InnerMatcher) {
+///   matches both 'A* a = GetAPointer()'.
+AST_POLYMORPHIC_MATCHER_P(hasConditionVariableStatement,
+                          AST_POLYMORPHIC_SUPPORTED_TYPES(IfStmt, ForStmt,
+                                                          WhileStmt,
+                                                          SwitchStmt),
+                          internal::Matcher<DeclStmt>, InnerMatcher) {
   const DeclStmt* const DeclarationStatement =
     Node.getConditionVariableDeclStmt();
   return DeclarationStatement != nullptr &&
@@ -7004,37 +7036,6 @@ AST_POLYMORPHIC_MATCHER_P2(
          InnerMatcher.matches(Args[Index], Finder, Builder);
 }
 
-/// Matches C or C++ elaborated `TypeLoc`s.
-///
-/// Given
-/// \code
-///   struct s {};
-///   struct s ss;
-/// \endcode
-/// elaboratedTypeLoc()
-///   matches the `TypeLoc` of the variable declaration of `ss`.
-extern const internal::VariadicDynCastAllOfMatcher<TypeLoc, ElaboratedTypeLoc>
-    elaboratedTypeLoc;
-
-/// Matches elaborated `TypeLoc`s that have a named `TypeLoc` matching
-/// `InnerMatcher`.
-///
-/// Given
-/// \code
-///   template <typename T>
-///   class C {};
-///   class C<int> c;
-///
-///   class D {};
-///   class D d;
-/// \endcode
-/// elaboratedTypeLoc(hasNamedTypeLoc(templateSpecializationTypeLoc()));
-///   matches the `TypeLoc` of the variable declaration of `c`, but not `d`.
-AST_MATCHER_P(ElaboratedTypeLoc, hasNamedTypeLoc, internal::Matcher<TypeLoc>,
-              InnerMatcher) {
-  return InnerMatcher.matches(Node.getNamedTypeLoc(), Finder, Builder);
-}
-
 /// Matches type \c bool.
 ///
 /// Given
@@ -7301,7 +7302,7 @@ extern const AstTypeMatcher<DecltypeType> decltypeType;
 AST_TYPE_TRAVERSE_MATCHER(hasDeducedType, getDeducedType,
                           AST_POLYMORPHIC_SUPPORTED_TYPES(AutoType));
 
-/// Matches \c DecltypeType or \c UsingType nodes to find the underlying type.
+/// Matches \c QualType nodes to find the underlying type.
 ///
 /// Given
 /// \code
@@ -7311,10 +7312,13 @@ AST_TYPE_TRAVERSE_MATCHER(hasDeducedType, getDeducedType,
 /// decltypeType(hasUnderlyingType(isInteger()))
 ///   matches the type of "a"
 ///
-/// Usable as: Matcher<DecltypeType>, Matcher<UsingType>
-AST_TYPE_TRAVERSE_MATCHER(hasUnderlyingType, getUnderlyingType,
-                          AST_POLYMORPHIC_SUPPORTED_TYPES(DecltypeType,
-                                                          UsingType));
+/// Usable as: Matcher<QualType>
+AST_MATCHER_P(Type, hasUnderlyingType, internal::Matcher<QualType>, Inner) {
+  QualType QT = Node.getLocallyUnqualifiedSingleStepDesugaredType();
+  if (QT == QualType(&Node, 0))
+    return false;
+  return Inner.matches(QT, Finder, Builder);
+}
 
 /// Matches \c FunctionType nodes.
 ///
@@ -7593,27 +7597,7 @@ extern const AstTypeMatcher<RecordType> recordType;
 /// and \c c.
 extern const AstTypeMatcher<TagType> tagType;
 
-/// Matches types specified with an elaborated type keyword or with a
-/// qualified name.
-///
-/// Given
-/// \code
-///   namespace N {
-///     namespace M {
-///       class D {};
-///     }
-///   }
-///   class C {};
-///
-///   class C c;
-///   N::M::D d;
-/// \endcode
-///
-/// \c elaboratedType() matches the type of the variable declarations of both
-/// \c c and \c d.
-extern const AstTypeMatcher<ElaboratedType> elaboratedType;
-
-/// Matches ElaboratedTypes whose qualifier, a NestedNameSpecifier,
+/// Matches Types whose qualifier, a NestedNameSpecifier,
 /// matches \c InnerMatcher if the qualifier exists.
 ///
 /// Given
@@ -7628,34 +7612,14 @@ extern const AstTypeMatcher<ElaboratedType> elaboratedType;
 ///
 /// \c elaboratedType(hasQualifier(hasPrefix(specifiesNamespace(hasName("N"))))
 /// matches the type of the variable declaration of \c d.
-AST_MATCHER_P(ElaboratedType, hasQualifier,
-              internal::Matcher<NestedNameSpecifier>, InnerMatcher) {
-  if (const NestedNameSpecifier *Qualifier = Node.getQualifier())
-    return InnerMatcher.matches(*Qualifier, Finder, Builder);
+AST_MATCHER_P(Type, hasQualifier, internal::Matcher<NestedNameSpecifier>,
+              InnerMatcher) {
+  if (NestedNameSpecifier Qualifier = Node.getPrefix())
+    return InnerMatcher.matches(Qualifier, Finder, Builder);
 
   return false;
 }
 
-/// Matches ElaboratedTypes whose named type matches \c InnerMatcher.
-///
-/// Given
-/// \code
-///   namespace N {
-///     namespace M {
-///       class D {};
-///     }
-///   }
-///   N::M::D d;
-/// \endcode
-///
-/// \c elaboratedType(namesType(recordType(
-/// hasDeclaration(namedDecl(hasName("D")))))) matches the type of the variable
-/// declaration of \c d.
-AST_MATCHER_P(ElaboratedType, namesType, internal::Matcher<QualType>,
-              InnerMatcher) {
-  return InnerMatcher.matches(Node.getNamedType(), Finder, Builder);
-}
-
 /// Matches types specified through a using declaration.
 ///
 /// Given
@@ -7824,7 +7788,7 @@ AST_MATCHER_FUNCTION_P_OVERLOAD(
 ///   matches "A::"
 AST_MATCHER_P(NestedNameSpecifier, specifiesType,
               internal::Matcher<QualType>, InnerMatcher) {
-  if (!Node.getAsType())
+  if (Node.getKind() != NestedNameSpecifier::Kind::Type)
     return false;
   return InnerMatcher.matches(QualType(Node.getAsType(), 0), Finder, Builder);
 }
@@ -7842,8 +7806,12 @@ AST_MATCHER_P(NestedNameSpecifier, specifiesType,
 ///   matches "A::"
 AST_MATCHER_P(NestedNameSpecifierLoc, specifiesTypeLoc,
               internal::Matcher<TypeLoc>, InnerMatcher) {
-  return Node && Node.getNestedNameSpecifier()->getAsType() &&
-         InnerMatcher.matches(Node.getTypeLoc(), Finder, Builder);
+  if (!Node)
+    return false;
+  TypeLoc TL = Node.getAsTypeLoc();
+  if (!TL)
+    return false;
+  return InnerMatcher.matches(TL, Finder, Builder);
 }
 
 /// Matches on the prefix of a \c NestedNameSpecifier.
@@ -7858,10 +7826,21 @@ AST_MATCHER_P(NestedNameSpecifierLoc, specifiesTypeLoc,
 AST_MATCHER_P_OVERLOAD(NestedNameSpecifier, hasPrefix,
                        internal::Matcher<NestedNameSpecifier>, InnerMatcher,
                        0) {
-  const NestedNameSpecifier *NextNode = Node.getPrefix();
+  NestedNameSpecifier NextNode = std::nullopt;
+  switch (Node.getKind()) {
+  case NestedNameSpecifier::Kind::Namespace:
+    NextNode = Node.getAsNamespaceAndPrefix().Prefix;
+    break;
+  case NestedNameSpecifier::Kind::Type:
+    NextNode = Node.getAsType()->getPrefix();
+    break;
+  default:
+    break;
+  }
+
   if (!NextNode)
     return false;
-  return InnerMatcher.matches(*NextNode, Finder, Builder);
+  return InnerMatcher.matches(NextNode, Finder, Builder);
 }
 
 /// Matches on the prefix of a \c NestedNameSpecifierLoc.
@@ -7876,7 +7855,12 @@ AST_MATCHER_P_OVERLOAD(NestedNameSpecifier, hasPrefix,
 AST_MATCHER_P_OVERLOAD(NestedNameSpecifierLoc, hasPrefix,
                        internal::Matcher<NestedNameSpecifierLoc>, InnerMatcher,
                        1) {
-  NestedNameSpecifierLoc NextNode = Node.getPrefix();
+  NestedNameSpecifierLoc NextNode;
+  if (TypeLoc TL = Node.getAsTypeLoc())
+    NextNode = TL.getPrefix();
+  else
+    NextNode = Node.getAsNamespaceAndPrefix().Prefix;
+
   if (!NextNode)
     return false;
   return InnerMatcher.matches(NextNode, Finder, Builder);
@@ -7894,9 +7878,13 @@ AST_MATCHER_P_OVERLOAD(NestedNameSpecifierLoc, hasPrefix,
 ///   matches "ns::"
 AST_MATCHER_P(NestedNameSpecifier, specifiesNamespace,
               internal::Matcher<NamespaceDecl>, InnerMatcher) {
-  if (auto *NS = dyn_cast_if_present<NamespaceDecl>(Node.getAsNamespace()))
-    return InnerMatcher.matches(*NS, Finder, Builder);
-  return false;
+  if (Node.getKind() != NestedNameSpecifier::Kind::Namespace)
+    return false;
+  const auto *Namespace =
+      dyn_cast<NamespaceDecl>(Node.getAsNamespaceAndPrefix().Namespace);
+  if (!Namespace)
+    return false;
+  return InnerMatcher.matches(*Namespace, Finder, Builder);
 }
 
 /// Matches attributes.
diff --git a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
index 5df2294792552..1ab6f11a23e12 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchersInternal.h
@@ -1017,10 +1017,7 @@ class HasDeclarationMatcher : public MatcherInterface<T> {
     // First, for any types that have a declaration, extract the declaration and
     // match on it.
     if (const auto *S = dyn_cast<TagType>(&Node)) {
-      return matchesDecl(S->getDecl(), Finder, Builder);
-    }
-    if (const auto *S = dyn_cast<InjectedClassNameType>(&Node)) {
-      return matchesDecl(S->getDecl(), Finder, Builder);
+      return matchesDecl(S->getOriginalDecl(), Finder, Builder);
     }
     if (const auto *S = dyn_cast<TemplateTypeParmType>(&Node)) {
       return matchesDecl(S->getDecl(), Finder, Builder);
@@ -1031,6 +1028,9 @@ class HasDeclarationMatcher : public MatcherInterface<T> {
     if (const auto *S = dyn_cast<UnresolvedUsingType>(&Node)) {
       return matchesDecl(S->getDecl(), Finder, Builder);
     }
+    if (const auto *S = dyn_cast<UsingType>(&Node)) {
+      return matchesDecl(S->getDecl(), Finder, Builder);
+    }
     if (const auto *S = dyn_cast<ObjCObjectType>(&Node)) {
       return matchesDecl(S->getInterface(), Finder, Builder);
     }
@@ -1066,12 +1066,6 @@ class HasDeclarationMatcher : public MatcherInterface<T> {
                          Builder);
     }
 
-    // FIXME: We desugar elaborated types. This makes the assumption that users
-    // do never want to match on whether a type is elaborated - there are
-    // arguments for both sides; for now, continue desugaring.
-    if (const auto *S = dyn_cast<ElaboratedType>(&Node)) {
-      return matchesSpecialized(S->desugar(), Finder, Builder);
-    }
     // Similarly types found via using declarations.
     // These are *usually* meaningless sugar, and this matches the historical
     // behavior prior to the introduction of UsingType.
@@ -1211,8 +1205,8 @@ using AdaptativeDefaultToTypes =
 /// All types that are supported by HasDeclarationMatcher above.
 using HasDeclarationSupportedTypes =
     TypeList<CallExpr, CXXConstructExpr, CXXNewExpr, DeclRefExpr, EnumType,
-             ElaboratedType, InjectedClassNameType, LabelStmt, AddrLabelExpr,
-             MemberExpr, QualType, RecordType, TagType,
+             InjectedClassNameType, LabelStmt, AddrLabelExpr, MemberExpr,
+             QualType, RecordType, TagType, UsingType,
              TemplateSpecializationType, TemplateTypeParmType, TypedefType,
              UnresolvedUsingType, ObjCIvarRefExpr, ObjCInterfaceDecl>;
 
@@ -1789,7 +1783,7 @@ class LocMatcher : public MatcherInterface<TLoc> {
 
 private:
   static DynTypedNode extract(const NestedNameSpecifierLoc &Loc) {
-    return DynTypedNode::create(*Loc.getNestedNameSpecifier());
+    return DynTypedNode::create(Loc.getNestedNameSpecifier());
   }
 };
 
diff --git a/clang/include/clang/Analysis/Analyses/LifetimeSafety.h b/clang/include/clang/Analysis/Analyses/LifetimeSafety.h
index 1c00558d32f63..7e1bfc903083e 100644
--- a/clang/include/clang/Analysis/Analyses/LifetimeSafety.h
+++ b/clang/include/clang/Analysis/Analyses/LifetimeSafety.h
@@ -19,14 +19,35 @@
 #define LLVM_CLANG_ANALYSIS_ANALYSES_LIFETIMESAFETY_H
 #include "clang/Analysis/AnalysisDeclContext.h"
 #include "clang/Analysis/CFG.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/ImmutableMap.h"
 #include "llvm/ADT/ImmutableSet.h"
 #include "llvm/ADT/StringMap.h"
 #include <memory>
 
 namespace clang::lifetimes {
 
+/// Enum to track the confidence level of a potential error.
+enum class Confidence {
+  None,
+  Maybe,   // Reported as a potential error (-Wlifetime-safety-strict)
+  Definite // Reported as a definite error (-Wlifetime-safety-permissive)
+};
+
+class LifetimeSafetyReporter {
+public:
+  LifetimeSafetyReporter() = default;
+  virtual ~LifetimeSafetyReporter() = default;
+
+  virtual void reportUseAfterFree(const Expr *IssueExpr, const Expr *UseExpr,
+                                  SourceLocation FreeLoc,
+                                  Confidence Confidence) {}
+};
+
 /// The main entry point for the analysis.
-void runLifetimeSafetyAnalysis(AnalysisDeclContext &AC);
+void runLifetimeSafetyAnalysis(AnalysisDeclContext &AC,
+                               LifetimeSafetyReporter *Reporter);
 
 namespace internal {
 // Forward declarations of internal types.
@@ -53,6 +74,7 @@ template <typename Tag> struct ID {
     IDBuilder.AddInteger(Value);
   }
 };
+
 template <typename Tag>
 inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, ID<Tag> ID) {
   return OS << ID.Value;
@@ -78,7 +100,8 @@ using ProgramPoint = const Fact *;
 /// encapsulates the various dataflow analyses.
 class LifetimeSafetyAnalysis {
 public:
-  LifetimeSafetyAnalysis(AnalysisDeclContext &AC);
+  LifetimeSafetyAnalysis(AnalysisDeclContext &AC,
+                         LifetimeSafetyReporter *Reporter);
   ~LifetimeSafetyAnalysis();
 
   void run();
@@ -87,7 +110,7 @@ class LifetimeSafetyAnalysis {
   LoanSet getLoansAtPoint(OriginID OID, ProgramPoint PP) const;
 
   /// Returns the set of loans that have expired at a specific program point.
-  LoanSet getExpiredLoansAtPoint(ProgramPoint PP) const;
+  std::vector<LoanID> getExpiredLoansAtPoint(ProgramPoint PP) const;
 
   /// Finds the OriginID for a given declaration.
   /// Returns a null optional if not found.
@@ -110,6 +133,7 @@ class LifetimeSafetyAnalysis {
 
 private:
   AnalysisDeclContext &AC;
+  LifetimeSafetyReporter *Reporter;
   std::unique_ptr<LifetimeFactory> Factory;
   std::unique_ptr<FactManager> FactMgr;
   std::unique_ptr<LoanPropagationAnalysis> LoanPropagation;
@@ -118,4 +142,25 @@ class LifetimeSafetyAnalysis {
 } // namespace internal
 } // namespace clang::lifetimes
 
+namespace llvm {
+template <typename Tag>
+struct DenseMapInfo<clang::lifetimes::internal::ID<Tag>> {
+  using ID = clang::lifetimes::internal::ID<Tag>;
+
+  static inline ID getEmptyKey() {
+    return {DenseMapInfo<uint32_t>::getEmptyKey()};
+  }
+
+  static inline ID getTombstoneKey() {
+    return {DenseMapInfo<uint32_t>::getTombstoneKey()};
+  }
+
+  static unsigned getHashValue(const ID &Val) {
+    return DenseMapInfo<uint32_t>::getHashValue(Val.Value);
+  }
+
+  static bool isEqual(const ID &LHS, const ID &RHS) { return LHS == RHS; }
+};
+} // namespace llvm
+
 #endif // LLVM_CLANG_ANALYSIS_ANALYSES_LIFETIMESAFETY_H
diff --git a/clang/include/clang/Analysis/FlowSensitive/ASTOps.h b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
index 8c7ee86d15c06..a404b06cd62ca 100644
--- a/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
+++ b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
@@ -112,8 +112,14 @@ class AnalysisASTVisitor : public DynamicRecursiveASTVisitor {
   // fields that are only used in these.
   // Note: The operand of the `noexcept` operator is an unevaluated operand, but
   // nevertheless it appears in the Clang CFG, so we don't exclude it here.
-  bool TraverseDecltypeTypeLoc(DecltypeTypeLoc) override { return true; }
-  bool TraverseTypeOfExprTypeLoc(TypeOfExprTypeLoc) override { return true; }
+  bool TraverseDecltypeTypeLoc(DecltypeTypeLoc,
+                               bool TraverseQualifier) override {
+    return true;
+  }
+  bool TraverseTypeOfExprTypeLoc(TypeOfExprTypeLoc,
+                                 bool TraverseQualifier) override {
+    return true;
+  }
   bool TraverseCXXTypeidExpr(CXXTypeidExpr *TIE) override {
     if (TIE->isPotentiallyEvaluated())
       return DynamicRecursiveASTVisitor::TraverseCXXTypeidExpr(TIE);
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
index 5be4a1145f40d..11042e865c4e6 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
@@ -42,6 +42,18 @@ struct ContextSensitiveOptions {
   unsigned Depth = 2;
 };
 
+/// A simple representation of essential elements of the logical context used in
+/// environments. Designed for import/export for applications requiring
+/// serialization support.
+struct SimpleLogicalContext {
+  // Global invariant that applies for all definitions in the context.
+  const Formula *Invariant;
+  // Flow-condition tokens in the context.
+  llvm::DenseMap<Atom, const Formula *> TokenDefs;
+  // Dependencies between flow-condition definitions.
+  llvm::DenseMap<Atom, llvm::DenseSet<Atom>> TokenDeps;
+};
+
 /// Owns objects that encompass the state of a program and stores context that
 /// is used during dataflow analysis.
 class DataflowAnalysisContext {
@@ -140,6 +152,15 @@ class DataflowAnalysisContext {
   /// Adds `Constraint` to the flow condition identified by `Token`.
   void addFlowConditionConstraint(Atom Token, const Formula &Constraint);
 
+  /// Adds `Deps` to the dependencies of the flow condition identified by
+  /// `Token`. Intended for use in deserializing contexts. The formula alone
+  /// doesn't have enough information to indicate its deps.
+  void addFlowConditionDeps(Atom Token, const llvm::DenseSet<Atom> &Deps) {
+    // Avoid creating an entry for `Token` with an empty set.
+    if (!Deps.empty())
+      FlowConditionDeps[Token].insert(Deps.begin(), Deps.end());
+  }
+
   /// Creates a new flow condition with the same constraints as the flow
   /// condition identified by `Token` and returns its token.
   Atom forkFlowCondition(Atom Token);
@@ -207,6 +228,14 @@ class DataflowAnalysisContext {
     return {};
   }
 
+  /// Export the logical-context portions of `AC`, limited to the given target
+  /// flow-condition tokens.
+  SimpleLogicalContext
+  exportLogicalContext(llvm::DenseSet<dataflow::Atom> TargetTokens) const;
+
+  /// Initializes this context's "logical" components with `LC`.
+  void initLogicalContext(SimpleLogicalContext LC);
+
 private:
   friend class Environment;
 
@@ -228,6 +257,11 @@ class DataflowAnalysisContext {
   DataflowAnalysisContext(Solver &S, std::unique_ptr<Solver> &&OwnedSolver,
                           Options Opts);
 
+  /// Computes the transitive closure of dependencies of (flow-condition)
+  /// `Tokens`. That is, the set of flow-condition tokens reachable from
+  /// `Tokens` in the dependency graph.
+  llvm::DenseSet<Atom> collectDependencies(llvm::DenseSet<Atom> Tokens) const;
+
   // Extends the set of modeled field declarations.
   void addModeledFields(const FieldSet &Fields);
 
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
index 097ff2bdfe7ad..076714462bb2a 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
@@ -157,10 +157,18 @@ class Environment {
   };
 
   /// Creates an environment that uses `DACtx` to store objects that encompass
-  /// the state of a program.
+  /// the state of a program. `FlowConditionToken` sets the flow condition
+  /// associated with the environment. Generally, new environments should be
+  /// initialized with a fresh token, by using one of the other
+  /// constructors. This constructor is for specialized use, including
+  /// deserialization and delegation from other constructors.
+  Environment(DataflowAnalysisContext &DACtx, Atom FlowConditionToken)
+      : DACtx(&DACtx), FlowConditionToken(FlowConditionToken) {}
+
+  /// Creates an environment that uses `DACtx` to store objects that encompass
+  /// the state of a program. Populates a fresh atom as flow condition token.
   explicit Environment(DataflowAnalysisContext &DACtx)
-      : DACtx(&DACtx),
-        FlowConditionToken(DACtx.arena().makeFlowConditionToken()) {}
+      : Environment(DACtx, DACtx.arena().makeFlowConditionToken()) {}
 
   /// Creates an environment that uses `DACtx` to store objects that encompass
   /// the state of a program, with `S` as the statement to analyze.
diff --git a/clang/include/clang/Analysis/FlowSensitive/Formula.h b/clang/include/clang/Analysis/FlowSensitive/Formula.h
index 0e6352403a832..3959bc98619b9 100644
--- a/clang/include/clang/Analysis/FlowSensitive/Formula.h
+++ b/clang/include/clang/Analysis/FlowSensitive/Formula.h
@@ -85,21 +85,17 @@ class alignas(const Formula *) Formula {
   }
 
   using AtomNames = llvm::DenseMap<Atom, std::string>;
-  // Produce a stable human-readable representation of this formula.
-  // For example: (V3 | !(V1 & V2))
-  // If AtomNames is provided, these override the default V0, V1... names.
+  /// Produces a stable human-readable representation of this formula.
+  /// For example: (V3 | !(V1 & V2))
+  /// If AtomNames is provided, these override the default V0, V1... names.
   void print(llvm::raw_ostream &OS, const AtomNames * = nullptr) const;
 
-  // Allocate Formulas using Arena rather than calling this function directly.
+  /// Allocates Formulas using Arena rather than calling this function directly.
   static const Formula &create(llvm::BumpPtrAllocator &Alloc, Kind K,
                                ArrayRef<const Formula *> Operands,
                                unsigned Value = 0);
 
-private:
-  Formula() = default;
-  Formula(const Formula &) = delete;
-  Formula &operator=(const Formula &) = delete;
-
+  /// Count of operands (sub-formulas) associated with Formulas of kind `K`.
   static unsigned numOperands(Kind K) {
     switch (K) {
     case AtomRef:
@@ -116,6 +112,11 @@ class alignas(const Formula *) Formula {
     llvm_unreachable("Unhandled Formula::Kind enum");
   }
 
+private:
+  Formula() = default;
+  Formula(const Formula &) = delete;
+  Formula &operator=(const Formula &) = delete;
+
   Kind FormulaKind;
   // Some kinds of formula have scalar values, e.g. AtomRef's atom number.
   unsigned Value;
diff --git a/clang/include/clang/Analysis/FlowSensitive/FormulaSerialization.h b/clang/include/clang/Analysis/FlowSensitive/FormulaSerialization.h
new file mode 100644
index 0000000000000..119f93e5d73f6
--- /dev/null
+++ b/clang/include/clang/Analysis/FlowSensitive/FormulaSerialization.h
@@ -0,0 +1,40 @@
+//=== FormulaSerialization.h - Formula De/Serialization support -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_FORMULA_SERIALIZATION_H
+#define LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_FORMULA_SERIALIZATION_H
+
+#include "clang/Analysis/FlowSensitive/Arena.h"
+#include "clang/Analysis/FlowSensitive/Formula.h"
+#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <string>
+
+namespace clang::dataflow {
+
+/// Prints `F` to `OS` in a compact format, optimized for easy parsing
+/// (deserialization) rather than human use.
+void serializeFormula(const Formula &F, llvm::raw_ostream &OS);
+
+/// Parses `Str` to build a serialized Formula.
+/// @returns error on parse failure or if parsing does not fully consume `Str`.
+/// @param A used to construct the formula components.
+/// @param AtomMap maps serialized Atom identifiers (unsigned ints) to Atoms.
+///        This map is provided by the caller to enable consistency across
+///        multiple formulas in a single file.
+llvm::Expected<const Formula *>
+parseFormula(llvm::StringRef Str, Arena &A,
+             llvm::DenseMap<unsigned, Atom> &AtomMap);
+
+} // namespace clang::dataflow
+#endif
diff --git a/clang/include/clang/Analysis/FlowSensitive/StorageLocation.h b/clang/include/clang/Analysis/FlowSensitive/StorageLocation.h
index 8fcc6a44027a0..534b9a017d8f0 100644
--- a/clang/include/clang/Analysis/FlowSensitive/StorageLocation.h
+++ b/clang/include/clang/Analysis/FlowSensitive/StorageLocation.h
@@ -17,6 +17,7 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/Type.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Debug.h"
 #include <cassert>
 
@@ -152,6 +153,11 @@ class RecordStorageLocation final : public StorageLocation {
     return {SyntheticFields.begin(), SyntheticFields.end()};
   }
 
+  /// Add a synthetic field, if none by that name is already present.
+  void addSyntheticField(llvm::StringRef Name, StorageLocation &Loc) {
+    SyntheticFields.insert({Name, &Loc});
+  }
+
   /// Changes the child storage location for a field `D` of reference type.
   /// All other fields cannot change their storage location and always retain
   /// the storage location passed to the `RecordStorageLocation` constructor.
@@ -164,6 +170,11 @@ class RecordStorageLocation final : public StorageLocation {
     Children[&D] = Loc;
   }
 
+  /// Add a child storage location for a field `D`, if not already present.
+  void addChild(const ValueDecl &D, StorageLocation *Loc) {
+    Children.insert({&D, Loc});
+  }
+
   llvm::iterator_range<FieldToLoc::const_iterator> children() const {
     return {Children.begin(), Children.end()};
   }
diff --git a/clang/include/clang/Basic/ABIVersions.def b/clang/include/clang/Basic/ABIVersions.def
new file mode 100644
index 0000000000000..f6524bc3bafb9
--- /dev/null
+++ b/clang/include/clang/Basic/ABIVersions.def
@@ -0,0 +1,135 @@
+//===--- ABIVersions.def - Clang ABI Versions Database ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file enumerates Clang ABI versions.
+//
+//===----------------------------------------------------------------------===//
+//
+/// @file ABIVersions.def
+///
+/// In this file, each of the Clang ABI Versions is enumerated
+/// ABI_VER_MAJOR_MINOR, ABI_VER_MAJOR, or ABI_VER_LATEST macro.
+///
+/// ABI_VER_MAJOR is used when the minor version is 0 or can be omitted.
+///
+/// The first argument of ABI_VER_MAJOR_MINOR and ABI_VER_MAJOR is the major
+/// version.
+///
+/// The second argument of ABI_VER_MAJOR_MINOR is the minor version.
+///
+/// The first argument of ABI_VER_LATEST is an identifier `Latest`.
+
+#if defined(ABI_VER_MAJOR_MINOR) != defined(ABI_VER_MAJOR) ||                  \
+    defined(ABI_VER_MAJOR) != defined(ABI_VER_LATEST)
+#  error ABI_VER_MAJOR_MINOR, ABI_VER_MAJOR and ABI_VER_LATEST should be defined simultaneously
+#endif
+
+#ifndef ABI_VER_MAJOR_MINOR
+#  define ABI_VER_MAJOR_MINOR(Major, Minor)
+#endif
+
+#ifndef ABI_VER_MAJOR
+#  define ABI_VER_MAJOR(Major)
+#endif
+
+#ifndef ABI_VER_LATEST
+#  define ABI_VER_LATEST(Latest)
+#endif
+
+/// Attempt to be ABI-compatible with code generated by Clang 3.8.x
+/// (SVN r257626). This causes <1 x long long> to be passed in an integer
+/// register instead of an SSE register on x64_64.
+ABI_VER_MAJOR_MINOR(3, 8)
+
+/// Attempt to be ABI-compatible with code generated by Clang 4.0.x
+/// (SVN r291814). This causes move operations to be ignored when determining
+/// whether a class type can be passed or returned directly.
+ABI_VER_MAJOR(4)
+
+/// Attempt to be ABI-compatible with code generated by Clang 6.0.x
+/// (SVN r321711). This causes determination of whether a type is
+/// standard-layout to ignore collisions between empty base classes and between
+/// base classes and member subobjects, which affects whether we reuse base
+/// class tail padding in some ABIs.
+ABI_VER_MAJOR(6)
+
+/// Attempt to be ABI-compatible with code generated by Clang 7.0.x
+/// (SVN r338536). This causes alignof (C++) and _Alignof (C11) to be compatible
+/// with __alignof (i.e., return the preferred alignment) rather than returning
+/// the required alignment.
+ABI_VER_MAJOR(7)
+
+/// Attempt to be ABI-compatible with code generated by Clang 9.0.x
+/// (SVN r351319). This causes vectors of __int128 to be passed in memory
+/// instead of passing in multiple scalar registers on x86_64 on Linux and
+/// NetBSD.
+ABI_VER_MAJOR(9)
+
+/// Attempt to be ABI-compatible with code generated by Clang 11.0.x
+/// (git 2e10b7a39b93). This causes clang to pass unions with a 256-bit vector
+/// member on the stack instead of using registers, to not properly mangle
+/// substitutions for template names in some cases, and to mangle declaration
+/// template arguments without a cast to the parameter type even when that can
+/// lead to mangling collisions.
+ABI_VER_MAJOR(11)
+
+/// Attempt to be ABI-compatible with code generated by Clang 12.0.x
+/// (git 8e464dd76bef). This causes clang to mangle lambdas within global-scope
+/// inline variables incorrectly.
+ABI_VER_MAJOR(12)
+
+/// Attempt to be ABI-compatible with code generated by Clang 14.0.x.
+/// This causes clang to:
+///   - mangle dependent nested names incorrectly.
+///   - make trivial only those defaulted copy constructors with a
+///     parameter-type-list equivalent to the parameter-type-list of an implicit
+///     declaration.
+ABI_VER_MAJOR(14)
+
+/// Attempt to be ABI-compatible with code generated by Clang 15.0.x.
+/// This causes clang to:
+///   - Reverse the implementation for CWG692, CWG1395 and CWG1432.
+///   - pack non-POD members of packed structs.
+///   - consider classes with defaulted special member functions non-pod.
+ABI_VER_MAJOR(15)
+
+/// Attempt to be ABI-compatible with code generated by Clang 17.0.x.
+/// This causes clang to revert some fixes to its implementation of the Itanium
+/// name mangling scheme, with the consequence that overloaded function
+/// templates are mangled the same if they differ only by:
+///   - constraints
+///   - whether a non-type template parameter has a deduced type
+///   - the parameter list of a template template parameter
+ABI_VER_MAJOR(17)
+
+/// Attempt to be ABI-compatible with code generated by Clang 18.0.x.
+/// This causes clang to revert some fixes to the mangling of lambdas in the
+/// initializers of members of local classes.
+ABI_VER_MAJOR(18)
+
+/// Attempt to be ABI-compatible with code generated by Clang 19.0.x.
+/// This causes clang to:
+///   - Incorrectly mangle the 'base type' substitutions of the CXX construction
+///     vtable because it hasn't added 'type' as a substitution.
+///   - Skip mangling enclosing class templates of member-like friend function
+///     templates.
+///   - Ignore empty struct arguments in C++ mode for ARM, instead of passing
+///     them as if they had a size of 1 byte.
+ABI_VER_MAJOR(19)
+
+/// Attempt to be ABI-compatible with code generated by Clang 20.0.x.
+/// This causes clang to:
+///   - Incorrectly return C++ records in AVX registers on x86_64.
+ABI_VER_MAJOR(20)
+
+/// Conform to the underlying platform's C and C++ ABIs as closely as we can.
+ABI_VER_LATEST(Latest)
+
+#undef ABI_VER_MAJOR_MINOR
+#undef ABI_VER_MAJOR
+#undef ABI_VER_LATEST
diff --git a/clang/include/clang/Basic/AllDiagnostics.h b/clang/include/clang/Basic/AllDiagnostics.h
index e64634cc138f7..3b782732c1507 100644
--- a/clang/include/clang/Basic/AllDiagnostics.h
+++ b/clang/include/clang/Basic/AllDiagnostics.h
@@ -23,20 +23,20 @@
 #include "clang/Basic/DiagnosticInstallAPI.h"
 #include "clang/Basic/DiagnosticLex.h"
 #include "clang/Basic/DiagnosticParse.h"
+#include "clang/Basic/DiagnosticRefactoring.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/DiagnosticSerialization.h"
-#include "clang/Basic/DiagnosticRefactoring.h"
 
 namespace clang {
-template <size_t SizeOfStr, typename FieldType>
-class StringSizerHelper {
+template <size_t SizeOfStr, typename FieldType> class StringSizerHelper {
   static_assert(SizeOfStr <= FieldType(~0U), "Field too small!");
+
 public:
   enum { Size = SizeOfStr };
 };
 } // end namespace clang
 
-#define STR_SIZE(str, fieldTy) clang::StringSizerHelper<sizeof(str)-1, \
-                                                        fieldTy>::Size
+#define STR_SIZE(str, fieldTy)                                                 \
+  clang::StringSizerHelper<sizeof(str) - 1, fieldTy>::Size
 
 #endif
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 30efb9f39e4f4..29364c5903d31 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -1632,6 +1632,13 @@ def DeviceKernel : DeclOrTypeAttr {
 }];
 }
 
+def SYCLExternal : InheritableAttr {
+  let Spellings = [CXX11<"clang", "sycl_external">];
+  let Subjects = SubjectList<[Function], ErrorDiag>;
+  let LangOpts = [SYCLHost, SYCLDevice];
+  let Documentation = [SYCLExternalDocs];
+}
+
 def SYCLKernelEntryPoint : InheritableAttr {
   let Spellings = [CXX11<"clang", "sycl_kernel_entry_point">];
   let Args = [
@@ -3922,6 +3929,14 @@ def CFICanonicalJumpTable : InheritableAttr {
   let SimpleHandler = 1;
 }
 
+def CFISalt : TypeAttr {
+  let Spellings = [Clang<"cfi_salt">];
+  let Args = [StringArgument<"Salt">];
+  let Subjects = SubjectList<[FunctionLike], ErrorDiag>;
+  let Documentation = [CFISaltDocs];
+  let LangOpts = [COnly];
+}
+
 // C/C++ Thread safety attributes (e.g. for deadlock, data race checking)
 // Not all of these attributes will be given a [[]] spelling. The attributes
 // which require access to function parameter names cannot use the [[]] spelling
@@ -4932,6 +4947,7 @@ def HLSLResourceBinding: InheritableAttr {
         return SpaceNumber;
       }
       void setImplicitBindingOrderID(uint32_t Value) {
+        assert(!hasImplicitBindingOrderID() && "attribute already has implicit binding order id");
         ImplicitBindingOrderID = Value;
       }
       bool hasImplicitBindingOrderID() const {
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 2b095ab975202..b9405c5fc86c1 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -476,6 +476,47 @@ The SYCL kernel in the previous code sample meets these expectations.
   }];
 }
 
+def SYCLExternalDocs : Documentation {
+  let Category = DocCatFunction;
+  let Heading = "sycl_external";
+  let Content = [{
+The ``sycl_external`` attribute indicates that a function defined in another
+translation unit may be called by a device function defined in the current
+translation unit or, if defined in the current translation unit, the function
+may be called by device functions defined in other translation units.
+The attribute is intended for use in the implementation of the ``SYCL_EXTERNAL``
+macro as specified in section 5.10.1, "SYCL functions and member functions
+linkage", of the SYCL 2020 specification.
+
+The attribute only appertains to functions and only those that meet the
+following requirements:
+
+* Has external linkage
+* Is not explicitly defined as deleted (the function may be an explicitly
+  defaulted function that is defined as deleted)
+
+The attribute shall be present on the first declaration of a function and
+may optionally be present on subsequent declarations.
+
+When compiling for a SYCL device target that does not support the generic
+address space, the function shall not specify a raw pointer or reference type
+as the return type or as a parameter type.
+See section 5.10, "SYCL offline linking", of the SYCL 2020 specification.
+The following examples demonstrate the use of this attribute:
+
+.. code-block:: c++
+
+  [[clang::sycl_external]] void Foo(); // Ok.
+
+  [[clang::sycl_external]] void Bar() { /* ... */ } // Ok.
+
+  [[clang::sycl_external]] extern void Baz(); // Ok.
+
+  [[clang::sycl_external]] static void Quux() { /* ... */ } // error:  Quux() has internal linkage.
+
+  }];
+}
+
 def SYCLKernelEntryPointDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
@@ -3646,6 +3687,99 @@ make the function's CFI jump table canonical. See :ref:`the CFI documentation
   }];
 }
 
+def CFISaltDocs : Documentation {
+  let Category = DocCatFunction;
+  let Heading = "cfi_salt";
+  let Label = "langext-cfi_salt";
+  let Content = [{
+The ``cfi_salt`` attribute specifies a string literal that is used as a salt
+for Control-Flow Integrity (CFI) checks to distinguish between functions with
+the same type signature. This attribute can be applied to function declarations,
+function definitions, and function pointer typedefs.
+
+The attribute prevents function pointers from being replaced with pointers to
+functions that have a compatible type, which can be a CFI bypass vector.
+
+**Syntax:**
+
+* GNU-style: ``__attribute__((cfi_salt("<salt_string>")))``
+* C++11-style: ``[[clang::cfi_salt("<salt_string>")]]``
+
+**Usage:**
+
+The attribute takes a single string literal argument that serves as the salt.
+Functions or function types with different salt values will have different CFI
+hashes, even if they have identical type signatures.
+
+**Motivation:**
+
+In large codebases like the Linux kernel, there are often hundreds of functions
+with identical type signatures that are called indirectly:
+
+.. code-block::
+
+  1662 functions with void (*)(void)
+  1179 functions with int (*)(void)
+   ...
+
+By salting the CFI hashes, you can make CFI more robust by ensuring that
+functions intended for different purposes have distinct CFI identities.
+
+**Type Compatibility:**
+
+* Functions with different salt values are considered to have incompatible types
+* Function pointers with different salt values cannot be assigned to each other
+* All declarations of the same function must use the same salt value
+
+**Example:**
+
+.. code-block:: c
+
+  // Header file - define convenience macros
+  #define __cfi_salt(s) __attribute__((cfi_salt(s)))
+  
+  // Typedef for regular function pointers
+  typedef int (*fptr_t)(void);
+  
+  // Typedef for salted function pointers  
+  typedef int (*fptr_salted_t)(void) __cfi_salt("pepper");
+  
+  struct widget_ops {
+    fptr_t init;          // Regular CFI
+    fptr_salted_t exec;   // Salted CFI
+    fptr_t cleanup;       // Regular CFI
+  };
+  
+  // Function implementations
+  static int widget_init(void) { return 0; }
+  static int widget_exec(void) __cfi_salt("pepper") { return 1; }
+  static int widget_cleanup(void) { return 0; }
+  
+  static struct widget_ops ops = {
+    .init = widget_init,      // OK - compatible types
+    .exec = widget_exec,      // OK - both use "pepper" salt
+    .cleanup = widget_cleanup // OK - compatible types
+  };
+  
+  // Using C++11 attribute syntax
+  void secure_callback(void) [[clang::cfi_salt("secure")]];
+  
+  // This would cause a compilation error:
+  // fptr_t bad_ptr = widget_exec;  // Error: incompatible types
+
+**Notes:**
+
+* The salt string can contain non-NULL ASCII characters, including spaces and
+  quotes
+* This attribute only applies to function types; using it on non-function 
+  types will generate a warning
+* All declarations and definitions of the same function must use identical 
+  salt values
+* The attribute affects type compatibility during compilation and CFI hash 
+  generation during code generation
+  }];
+}
+
 def DocCatTypeSafety : DocumentationCategory<"Type Safety Checking"> {
   let Content = [{
 Clang supports additional attributes to enable checking type safety properties
diff --git a/clang/include/clang/Basic/BuiltinTemplates.td b/clang/include/clang/Basic/BuiltinTemplates.td
index 5b9672b395955..504405acbdc78 100644
--- a/clang/include/clang/Basic/BuiltinTemplates.td
+++ b/clang/include/clang/Basic/BuiltinTemplates.td
@@ -62,3 +62,7 @@ def __builtin_common_type : CPlusPlusBuiltinTemplate<
 //           typename ...Operands>
 def __hlsl_spirv_type : HLSLBuiltinTemplate<
 [Uint32T, Uint32T, Uint32T, Class<"Operands", /*is_variadic=*/1>]>;
+
+// template <class ...Args>
+def __builtin_dedup_pack
+    : CPlusPlusBuiltinTemplate<[Class<"Args", /*is_variadic=*/1>]>;
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index c81714e9b009d..6d21c620bfc80 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -1232,6 +1232,18 @@ def ConvertVector : Builtin {
   let Prototype = "void(...)";
 }
 
+def MaskedLoad : Builtin {
+  let Spellings = ["__builtin_masked_load"];
+  let Attributes = [NoThrow, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
+def MaskedStore : Builtin {
+  let Spellings = ["__builtin_masked_store"];
+  let Attributes = [NoThrow, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
 def AllocaUninitialized : Builtin {
   let Spellings = ["__builtin_alloca_uninitialized"];
   let Attributes = [FunctionWithBuiltinPrefix, NoThrow];
@@ -1264,7 +1276,7 @@ def NondetermenisticValue : Builtin {
 
 def ElementwiseAbs : Builtin {
   let Spellings = ["__builtin_elementwise_abs"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
 
@@ -1300,13 +1312,13 @@ def ElementwiseBitreverse : Builtin {
 
 def ElementwiseMax : Builtin {
   let Spellings = ["__builtin_elementwise_max"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
 
 def ElementwiseMin : Builtin {
   let Spellings = ["__builtin_elementwise_min"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
 
@@ -1498,7 +1510,7 @@ def ElementwiseCopysign : Builtin {
 
 def ElementwiseFma : Builtin {
   let Spellings = ["__builtin_elementwise_fma"];
-  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
   let Prototype = "void(...)";
 }
 
@@ -1514,6 +1526,30 @@ def ElementwiseSubSat : Builtin {
   let Prototype = "void(...)";
 }
 
+def ElementwiseFshl : Builtin {
+  let Spellings = ["__builtin_elementwise_fshl"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
+def ElementwiseFshr : Builtin {
+  let Spellings = ["__builtin_elementwise_fshr"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking];
+  let Prototype = "void(...)";
+}
+
+def ElementwiseCtlz : Builtin {
+  let Spellings = ["__builtin_elementwise_ctlz"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
+  let Prototype = "void(...)";
+}
+
+def ElementwiseCttz : Builtin {
+  let Spellings = ["__builtin_elementwise_cttz"];
+  let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
+  let Prototype = "void(...)";
+}
+
 def ReduceMax : Builtin {
   let Spellings = ["__builtin_reduce_max"];
   let Attributes = [NoThrow, Const, CustomTypeChecking, Constexpr];
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index b16d4a22207a7..f8f55772db8fe 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -503,6 +503,9 @@ TARGET_BUILTIN(__builtin_amdgcn_s_barrier_signal, "vIi", "n", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_s_barrier_signal_var, "vv*i", "n", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_s_barrier_wait, "vIs", "n", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_s_barrier_signal_isfirst, "bIi", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_init, "vv*i", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_join, "vv*", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_leave, "vIs", "n", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_s_get_barrier_state, "Uii", "n", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_s_get_named_barrier_state, "Uiv*", "n", "gfx12-insts")
 TARGET_BUILTIN(__builtin_amdgcn_s_prefetch_data, "vvC*Ui", "nc", "gfx12-insts")
diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index e7d6741eb695e..79df84abd74f0 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -580,6 +580,8 @@ TARGET_BUILTIN(__builtin_ppc_bcdsub_p, "iiV16UcV16Uc", "",
                "isa-v207-instructions")
 
 // P9 Binary-coded decimal (BCD) builtins.                                                
+TARGET_BUILTIN(__builtin_ppc_bcdcopysign, "V16UcV16UcV16Uc", "", "power9-vector")
+TARGET_BUILTIN(__builtin_ppc_bcdsetsign, "V16UcV16UcUc", "t", "power9-vector")
 TARGET_BUILTIN(__builtin_ppc_national2packed, "V16UcV16UcUc", "t", "power9-vector")
 TARGET_BUILTIN(__builtin_ppc_packed2national, "V16UcV16Uc", "", "power9-vector")
 TARGET_BUILTIN(__builtin_ppc_packed2zoned, "V16UcV16UcUc", "t", "power9-vector")
diff --git a/clang/include/clang/Basic/BuiltinsX86.td b/clang/include/clang/Basic/BuiltinsX86.td
index a4acc72fdc37d..527acd9ef086e 100644
--- a/clang/include/clang/Basic/BuiltinsX86.td
+++ b/clang/include/clang/Basic/BuiltinsX86.td
@@ -93,13 +93,11 @@ let Attributes = [Const, NoThrow, RequiredVectorWidth<128>] in {
   }
 
   let Features = "sse2" in {
-    def pmulhw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
     def pavgb128 : X86Builtin<"_Vector<16, char>(_Vector<16, char>, _Vector<16, char>)">;
     def pavgw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
     def packsswb128 : X86Builtin<"_Vector<16, char>(_Vector<8, short>, _Vector<8, short>)">;
     def packssdw128 : X86Builtin<"_Vector<8, short>(_Vector<4, int>, _Vector<4, int>)">;
     def packuswb128 : X86Builtin<"_Vector<16, char>(_Vector<8, short>, _Vector<8, short>)">;
-    def pmulhuw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
     def vec_ext_v2di : X86Builtin<"long long int(_Vector<2, long long int>, _Constant int)">;
     def vec_ext_v4si : X86Builtin<"int(_Vector<4, int>, _Constant int)">;
     def vec_ext_v4sf : X86Builtin<"float(_Vector<4, float>, _Constant int)">;
@@ -107,6 +105,11 @@ let Attributes = [Const, NoThrow, RequiredVectorWidth<128>] in {
     def vec_set_v8hi : X86Builtin<"_Vector<8, short>(_Vector<8, short>, short, _Constant int)">;
   }
 
+  let Features = "sse2", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
+    def pmulhw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
+    def pmulhuw128 : X86Builtin<"_Vector<8, unsigned short>(_Vector<8, unsigned short>, _Vector<8, unsigned short>)">;
+  }
+
   let Features = "sse3" in {
     foreach Op = ["addsub", "hadd", "hsub"] in {
       def Op#ps : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>)">;
@@ -265,7 +268,6 @@ let Header = "emmintrin.h", Attributes = [NoThrow, RequireDeclaration] in {
 }
 
 let Features = "sse2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def pmuludq128 : X86Builtin<"_Vector<2, long long int>(_Vector<4, int>, _Vector<4, int>)">;
   def psraw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
   def psrad128 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
   def psrlw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
@@ -287,6 +289,10 @@ let Features = "sse2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] i
   def psrldqi128_byteshift : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Constant int)">;
 }
 
+let Features = "sse2", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
+  def pmuludq128 : X86Builtin<"_Vector<2, long long int>(_Vector<4, int>, _Vector<4, int>)">;
+}
+
 let Features = "sse3", Attributes = [NoThrow] in {
   def monitor : X86Builtin<"void(void const *, unsigned int, unsigned int)">;
   def mwait : X86Builtin<"void(unsigned int, unsigned int)">;
@@ -309,7 +315,6 @@ let Features = "sse4.1", Attributes = [NoThrow, Const, RequiredVectorWidth<128>]
   def blendvpd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Vector<2, double>)">;
   def blendvps : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>, _Vector<4, float>)">;
   def packusdw128 : X86Builtin<"_Vector<8, short>(_Vector<4, int>, _Vector<4, int>)">;
-  def pmuldq128 : X86Builtin<"_Vector<2, long long int>(_Vector<4, int>, _Vector<4, int>)">;
   def roundps : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Constant int)">;
   def roundss : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>, _Constant int)">;
   def roundsd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Constant int)">;
@@ -326,6 +331,10 @@ let Features = "sse4.1", Attributes = [NoThrow, Const, RequiredVectorWidth<128>]
   def vec_set_v4si : X86Builtin<"_Vector<4, int>(_Vector<4, int>, int, _Constant int)">;
 }
 
+let Features = "sse4.1", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
+  def pmuldq128 : X86Builtin<"_Vector<2, long long int>(_Vector<4, int>, _Vector<4, int>)">;
+}
+
 let Features = "sse4.2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
   def pcmpistrm128 : X86Builtin<"_Vector<16, char>(_Vector<16, char>, _Vector<16, char>, _Constant char)">;
   def pcmpistri128 : X86Builtin<"int(_Vector<16, char>, _Vector<16, char>, _Constant char)">;
@@ -577,11 +586,7 @@ let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] i
   def pmaddubsw256 : X86Builtin<"_Vector<16, short>(_Vector<32, char>, _Vector<32, char>)">;
   def pmaddwd256 : X86Builtin<"_Vector<8, int>(_Vector<16, short>, _Vector<16, short>)">;
   def pmovmskb256 : X86Builtin<"int(_Vector<32, char>)">;
-  def pmuldq256 : X86Builtin<"_Vector<4, long long int>(_Vector<8, int>, _Vector<8, int>)">;
   def pmulhrsw256 : X86Builtin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>)">;
-  def pmulhuw256 : X86Builtin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>)">;
-  def pmulhw256 : X86Builtin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>)">;
-  def pmuludq256 : X86Builtin<"_Vector<4, long long int>(_Vector<8, int>, _Vector<8, int>)">;
   def psadbw256 : X86Builtin<"_Vector<4, long long int>(_Vector<32, char>, _Vector<32, char>)">;
   def pshufb256 : X86Builtin<"_Vector<32, char>(_Vector<32, char>, _Vector<32, char>)">;
   def pshufd256 : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Constant int)">;
@@ -619,6 +624,28 @@ let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] i
   def insert128i256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<2, long long int>, _Constant int)">;
 }
 
+let Features = "avx2", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
+  def pmuldq256 : X86Builtin<"_Vector<4, long long int>(_Vector<8, int>, _Vector<8, int>)">;
+  def pmuludq256 : X86Builtin<"_Vector<4, long long int>(_Vector<8, int>, _Vector<8, int>)">;
+
+  def pmulhuw256 : X86Builtin<"_Vector<16, unsigned short>(_Vector<16, unsigned short>, _Vector<16, unsigned short>)">;
+  def pmulhw256 : X86Builtin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>)">;
+
+  def psllv8si : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>)">;
+  def psrav8si : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>)">;
+  def psrlv8si : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>)">;
+  def psllv4di : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>)">;
+  def psrlv4di : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>)">;
+}
+
+let Features = "avx2", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
+  def psllv4si : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
+  def psrav4si : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
+  def psrlv4si : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
+  def psllv2di : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>)">;
+  def psrlv2di : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>)">;
+}
+
 let Features = "avx2", Attributes = [NoThrow, RequiredVectorWidth<256>] in {
   def maskloadd256 : X86Builtin<"_Vector<8, int>(_Vector<8, int const *>, _Vector<8, int>)">;
   def maskloadq256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int const *>, _Vector<4, long long int>)">;
@@ -639,46 +666,6 @@ let Features = "avx2", Attributes = [NoThrow, RequiredVectorWidth<128>] in {
   def maskstoreq : X86Builtin<"void(_Vector<2, long long int *>, _Vector<2, long long int>, _Vector<2, long long int>)">;
 }
 
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def psllv8si : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>)">;
-}
-
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def psllv4si : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
-}
-
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def psllv4di : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>)">;
-}
-
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def psllv2di : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>)">;
-}
-
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def psrav8si : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>)">;
-}
-
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def psrav4si : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
-}
-
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def psrlv8si : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>)">;
-}
-
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def psrlv4si : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
-}
-
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def psrlv4di : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>)">;
-}
-
-let Features = "avx2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def psrlv2di : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>)">;
-}
-
 let Features = "avx2", Attributes = [NoThrow, RequiredVectorWidth<128>] in {
   def gatherd_pd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, double const *, _Vector<4, int>, _Vector<2, double>, _Constant char)">;
 }
@@ -751,14 +738,6 @@ let Features = "f16c", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] i
   def vcvtps2ph256 : X86Builtin<"_Vector<8, short>(_Vector<8, float>, _Constant int)">;
 }
 
-let Features = "f16c", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vcvtph2ps : X86Builtin<"_Vector<4, float>(_Vector<8, short>)">;
-}
-
-let Features = "f16c", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vcvtph2ps256 : X86Builtin<"_Vector<8, float>(_Vector<8, short>)">;
-}
-
 let Features = "rdrnd", Attributes = [NoThrow] in {
   def rdrand16_step : X86Builtin<"unsigned int(unsigned short *)">;
   def rdrand32_step : X86Builtin<"unsigned int(unsigned int *)">;
@@ -878,11 +857,6 @@ let Features = "sha", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in
   def sha256msg2 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
 }
 
-let Features = "fma|fma4", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vfmaddps : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>, _Vector<4, float>)">;
-  def vfmaddpd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Vector<2, double>)">;
-}
-
 let Features = "fma", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
   def vfmaddss3 : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>, _Vector<4, float>)">;
   def vfmaddsd3 : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Vector<2, double>)">;
@@ -898,9 +872,8 @@ let Features = "fma|fma4", Attributes = [NoThrow, Const, RequiredVectorWidth<128
   def vfmaddsubpd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Vector<2, double>)">;
 }
 
-let Features = "fma|fma4", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vfmaddps256 : X86Builtin<"_Vector<8, float>(_Vector<8, float>, _Vector<8, float>, _Vector<8, float>)">;
-  def vfmaddpd256 : X86Builtin<"_Vector<4, double>(_Vector<4, double>, _Vector<4, double>, _Vector<4, double>)">;
+let Features = "fma|fma4",
+    Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
   def vfmaddsubps256 : X86Builtin<"_Vector<8, float>(_Vector<8, float>, _Vector<8, float>, _Vector<8, float>)">;
   def vfmaddsubpd256 : X86Builtin<"_Vector<4, double>(_Vector<4, double>, _Vector<4, double>, _Vector<4, double>)">;
 }
@@ -953,10 +926,6 @@ let Features = "xop", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in
   def vphsubwd : X86Builtin<"_Vector<4, int>(_Vector<8, short>)">;
   def vphsubdq : X86Builtin<"_Vector<2, long long int>(_Vector<4, int>)">;
   def vpperm : X86Builtin<"_Vector<16, char>(_Vector<16, char>, _Vector<16, char>, _Vector<16, char>)">;
-  def vprotb : X86Builtin<"_Vector<16, char>(_Vector<16, char>, _Vector<16, char>)">;
-  def vprotw : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>)">;
-  def vprotd : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
-  def vprotq : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>)">;
   def vprotbi : X86Builtin<"_Vector<16, char>(_Vector<16, char>, _Constant char)">;
   def vprotwi : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Constant char)">;
   def vprotdi : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Constant char)">;
@@ -1090,6 +1059,9 @@ let Features = "avx512f,evex512", Attributes = [NoThrow, Const, RequiredVectorWi
   def cvtpd2ps512_mask : X86Builtin<"_Vector<8, float>(_Vector<8, double>, _Vector<8, float>, unsigned char, _Constant int)">;
   def vcvtps2ph512_mask : X86Builtin<"_Vector<16, short>(_Vector<16, float>, _Constant int, _Vector<16, short>, unsigned short)">;
   def vcvtph2ps512_mask : X86Builtin<"_Vector<16, float>(_Vector<16, short>, _Vector<16, float>, unsigned short, _Constant int)">;
+}
+
+let Features = "avx512f,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
   def pmuldq512 : X86Builtin<"_Vector<8, long long int>(_Vector<16, int>, _Vector<16, int>)">;
   def pmuludq512 : X86Builtin<"_Vector<8, long long int>(_Vector<16, int>, _Vector<16, int>)">;
 }
@@ -1429,7 +1401,10 @@ let Features = "avx512bitalg,evex512", Attributes = [NoThrow, Const, RequiredVec
 
 let Features = "avx512bw,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
   def pmulhrsw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Vector<32, short>)">;
-  def pmulhuw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Vector<32, short>)">;
+}
+
+let Features = "avx512bw,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
+  def pmulhuw512 : X86Builtin<"_Vector<32, unsigned short>(_Vector<32, unsigned short>, _Vector<32, unsigned short>)">;
   def pmulhw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Vector<32, short>)">;
 }
 
@@ -1887,78 +1862,6 @@ let Features = "avx512vbmi2,evex512", Attributes = [NoThrow, Const, RequiredVect
   def vpshldw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Vector<32, short>, _Constant int)">;
 }
 
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vpshldvd128 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>, _Vector<4, int>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vpshldvd256 : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>, _Vector<8, int>)">;
-}
-
-let Features = "avx512vbmi2,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
-  def vpshldvd512 : X86Builtin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vpshldvq128 : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>, _Vector<2, long long int>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vpshldvq256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>, _Vector<4, long long int>)">;
-}
-
-let Features = "avx512vbmi2,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
-  def vpshldvq512 : X86Builtin<"_Vector<8, long long int>(_Vector<8, long long int>, _Vector<8, long long int>, _Vector<8, long long int>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vpshldvw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>, _Vector<8, short>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vpshldvw256 : X86Builtin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>, _Vector<16, short>)">;
-}
-
-let Features = "avx512vbmi2,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
-  def vpshldvw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Vector<32, short>, _Vector<32, short>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vpshrdvd128 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>, _Vector<4, int>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vpshrdvd256 : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>, _Vector<8, int>)">;
-}
-
-let Features = "avx512vbmi2,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
-  def vpshrdvd512 : X86Builtin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>, _Vector<16, int>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vpshrdvq128 : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>, _Vector<2, long long int>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vpshrdvq256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>, _Vector<4, long long int>)">;
-}
-
-let Features = "avx512vbmi2,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
-  def vpshrdvq512 : X86Builtin<"_Vector<8, long long int>(_Vector<8, long long int>, _Vector<8, long long int>, _Vector<8, long long int>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vpshrdvw128 : X86Builtin<"_Vector<8, short>(_Vector<8, short>, _Vector<8, short>, _Vector<8, short>)">;
-}
-
-let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vpshrdvw256 : X86Builtin<"_Vector<16, short>(_Vector<16, short>, _Vector<16, short>, _Vector<16, short>)">;
-}
-
-let Features = "avx512vbmi2,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
-  def vpshrdvw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Vector<32, short>, _Vector<32, short>)">;
-}
-
 let Features = "avx512vl,avx512vbmi2", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
   def vpshrdd128 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>, _Constant int)">;
 }
@@ -2170,28 +2073,10 @@ let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256
 }
 
 let Features = "avx512f,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
-  def prolvd512 : X86Builtin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
-  def prolvq512 : X86Builtin<"_Vector<8, long long int>(_Vector<8, long long int>, _Vector<8, long long int>)">;
   def prord512 : X86Builtin<"_Vector<16, int>(_Vector<16, int>, _Constant int)">;
   def prorq512 : X86Builtin<"_Vector<8, long long int>(_Vector<8, long long int>, _Constant int)">;
 }
 
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def prolvd128 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
-}
-
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def prolvd256 : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>)">;
-}
-
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def prolvq128 : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>)">;
-}
-
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def prolvq256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>)">;
-}
-
 let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
   def prord128 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Constant int)">;
 }
@@ -2208,27 +2093,6 @@ let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256
   def prorq256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Constant int)">;
 }
 
-let Features = "avx512f,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
-  def prorvd512 : X86Builtin<"_Vector<16, int>(_Vector<16, int>, _Vector<16, int>)">;
-  def prorvq512 : X86Builtin<"_Vector<8, long long int>(_Vector<8, long long int>, _Vector<8, long long int>)">;
-}
-
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def prorvd128 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
-}
-
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def prorvd256 : X86Builtin<"_Vector<8, int>(_Vector<8, int>, _Vector<8, int>)">;
-}
-
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def prorvq128 : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>)">;
-}
-
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def prorvq256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>)">;
-}
-
 let Features = "avx512bw,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
   def pshufhw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Constant int)">;
   def pshuflw512 : X86Builtin<"_Vector<32, short>(_Vector<32, short>, _Constant int)">;
@@ -4140,14 +4004,6 @@ let Features = "avx512fp16,evex512", Attributes = [NoThrow, Const, RequiredVecto
   def vcvtps2phx512_mask : X86Builtin<"_Vector<16, _Float16>(_Vector<16, float>, _Vector<16, _Float16>, unsigned short, _Constant int)">;
 }
 
-let Features = "avx512fp16,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vfmaddph : X86Builtin<"_Vector<8, _Float16>(_Vector<8, _Float16>, _Vector<8, _Float16>, _Vector<8, _Float16>)">;
-}
-
-let Features = "avx512fp16,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vfmaddph256 : X86Builtin<"_Vector<16, _Float16>(_Vector<16, _Float16>, _Vector<16, _Float16>, _Vector<16, _Float16>)">;
-}
-
 let Features = "avx512fp16,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
   def vfmaddph512_mask : X86Builtin<"_Vector<32, _Float16>(_Vector<32, _Float16>, _Vector<32, _Float16>, _Vector<32, _Float16>, unsigned int, _Constant int)">;
   def vfmaddph512_mask3 : X86Builtin<"_Vector<32, _Float16>(_Vector<32, _Float16>, _Vector<32, _Float16>, _Vector<32, _Float16>, unsigned int, _Constant int)">;
@@ -4246,99 +4102,99 @@ let Features = "avx512fp16,evex512", Attributes = [NoThrow, Const, RequiredVecto
   def vfcmulcph512_mask : X86Builtin<"_Vector<16, float>(_Vector<16, float>, _Vector<16, float>, _Vector<16, float>, unsigned short, _Constant int)">;
 }
 
-let Features = "avx512bw,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
+let Features = "avx512bw,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
   def selectb_128 : X86Builtin<"_Vector<16, char>(unsigned short, _Vector<16, char>, _Vector<16, char>)">;
 }
 
-let Features = "avx512bw,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
+let Features = "avx512bw,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
   def selectb_256 : X86Builtin<"_Vector<32, char>(unsigned int, _Vector<32, char>, _Vector<32, char>)">;
 }
 
-let Features = "avx512bw,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
+let Features = "avx512bw,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
   def selectb_512 : X86Builtin<"_Vector<64, char>(unsigned long long int, _Vector<64, char>, _Vector<64, char>)">;
 }
 
-let Features = "avx512bw,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
+let Features = "avx512bw,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
   def selectw_128 : X86Builtin<"_Vector<8, short>(unsigned char, _Vector<8, short>, _Vector<8, short>)">;
 }
 
-let Features = "avx512bw,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
+let Features = "avx512bw,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
   def selectw_256 : X86Builtin<"_Vector<16, short>(unsigned short, _Vector<16, short>, _Vector<16, short>)">;
 }
 
-let Features = "avx512bw,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
+let Features = "avx512bw,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
   def selectw_512 : X86Builtin<"_Vector<32, short>(unsigned int, _Vector<32, short>, _Vector<32, short>)">;
 }
 
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
+let Features = "avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
   def selectd_128 : X86Builtin<"_Vector<4, int>(unsigned char, _Vector<4, int>, _Vector<4, int>)">;
 }
 
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
+let Features = "avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
   def selectd_256 : X86Builtin<"_Vector<8, int>(unsigned char, _Vector<8, int>, _Vector<8, int>)">;
 }
 
-let Features = "avx512f,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
+let Features = "avx512f,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
   def selectd_512 : X86Builtin<"_Vector<16, int>(unsigned short, _Vector<16, int>, _Vector<16, int>)">;
 }
 
-let Features = "avx512fp16,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
+let Features = "avx512fp16,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
   def selectph_128 : X86Builtin<"_Vector<8, _Float16>(unsigned char, _Vector<8, _Float16>, _Vector<8, _Float16>)">;
 }
 
-let Features = "avx512fp16,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
+let Features = "avx512fp16,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
   def selectph_256 : X86Builtin<"_Vector<16, _Float16>(unsigned short, _Vector<16, _Float16>, _Vector<16, _Float16>)">;
 }
 
-let Features = "avx512fp16,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
+let Features = "avx512fp16,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
   def selectph_512 : X86Builtin<"_Vector<32, _Float16>(unsigned int, _Vector<32, _Float16>, _Vector<32, _Float16>)">;
 }
 
-let Features = "avx512bf16,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
+let Features = "avx512bf16,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
   def selectpbf_128 : X86Builtin<"_Vector<8, __bf16>(unsigned char, _Vector<8, __bf16>, _Vector<8, __bf16>)">;
 }
 
-let Features = "avx512bf16,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
+let Features = "avx512bf16,avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
   def selectpbf_256 : X86Builtin<"_Vector<16, __bf16>(unsigned short, _Vector<16, __bf16>, _Vector<16, __bf16>)">;
 }
 
-let Features = "avx512bf16,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
+let Features = "avx512bf16,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
   def selectpbf_512 : X86Builtin<"_Vector<32, __bf16>(unsigned int, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
 }
 
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
+let Features = "avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
   def selectq_128 : X86Builtin<"_Vector<2, long long int>(unsigned char, _Vector<2, long long int>, _Vector<2, long long int>)">;
 }
 
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
+let Features = "avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
   def selectq_256 : X86Builtin<"_Vector<4, long long int>(unsigned char, _Vector<4, long long int>, _Vector<4, long long int>)">;
 }
 
-let Features = "avx512f,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
+let Features = "avx512f,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
   def selectq_512 : X86Builtin<"_Vector<8, long long int>(unsigned char, _Vector<8, long long int>, _Vector<8, long long int>)">;
 }
 
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
+let Features = "avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
   def selectps_128 : X86Builtin<"_Vector<4, float>(unsigned char, _Vector<4, float>, _Vector<4, float>)">;
 }
 
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
+let Features = "avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
   def selectps_256 : X86Builtin<"_Vector<8, float>(unsigned char, _Vector<8, float>, _Vector<8, float>)">;
 }
 
-let Features = "avx512f,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
+let Features = "avx512f,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
   def selectps_512 : X86Builtin<"_Vector<16, float>(unsigned short, _Vector<16, float>, _Vector<16, float>)">;
 }
 
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
+let Features = "avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
   def selectpd_128 : X86Builtin<"_Vector<2, double>(unsigned char, _Vector<2, double>, _Vector<2, double>)">;
 }
 
-let Features = "avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
+let Features = "avx512vl", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
   def selectpd_256 : X86Builtin<"_Vector<4, double>(unsigned char, _Vector<4, double>, _Vector<4, double>)">;
 }
 
-let Features = "avx512f,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
+let Features = "avx512f,evex512", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
   def selectpd_512 : X86Builtin<"_Vector<8, double>(unsigned char, _Vector<8, double>, _Vector<8, double>)">;
 }
 
@@ -5373,13 +5229,4 @@ let Features = "avx10.2-256", Attributes = [NoThrow, Const, RequiredVectorWidth<
 
 let Features = "avx10.2-512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
   def vsqrtbf16512 : X86Builtin<"_Vector<32, __bf16>(_Vector<32, __bf16>)">;
-  def vfmaddbf16512 : X86Builtin<"_Vector<32, __bf16>(_Vector<32, __bf16>, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
-}
-
-let Features = "avx10.2-256", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
-  def vfmaddbf16256 : X86Builtin<"_Vector<16, __bf16>(_Vector<16, __bf16>, _Vector<16, __bf16>, _Vector<16, __bf16>)">;
-}
-
-let Features = "avx10.2-256", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
-  def vfmaddbf16128 : X86Builtin<"_Vector<8, __bf16>(_Vector<8, __bf16>, _Vector<8, __bf16>, _Vector<8, __bf16>)">;
 }
diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td
index 46d04b031798c..a63bd80b89657 100644
--- a/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -400,6 +400,9 @@ def note_constexpr_non_const_vectorelements : Note<
   "cannot determine number of elements for sizeless vectors in a constant expression">;
 def note_constexpr_assumption_failed : Note<
   "assumption evaluated to false">;
+def note_constexpr_countzeroes_zero : Note<
+  "evaluation of %select{__builtin_elementwise_ctlz|__builtin_elementwise_cttz}0 "
+  "with a zero value is undefined">;
 def err_experimental_clang_interp_failed : Error<
   "the experimental clang interpreter failed to evaluate an expression">;
 
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 0f17f4aa761ea..b8c7c6e8d6909 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -581,6 +581,13 @@ def err_drv_reduced_module_output_overrided : Warning<
   "please consider use '-fmodule-output=' to specify the output file for reduced BMI explicitly">,
   InGroup<DiagGroup<"reduced-bmi-output-overrided">>;
 
+def remark_found_cxx20_module_usage : Remark<
+  "found C++20 module usage in file '%0'">,
+  InGroup<ModulesDriver>;
+def remark_performing_driver_managed_module_build : Remark<
+  "performing driver managed module build">,
+  InGroup<ModulesDriver>;
+
 def warn_drv_delayed_template_parsing_after_cxx20 : Warning<
   "-fdelayed-template-parsing is deprecated after C++20">,
   InGroup<DiagGroup<"delayed-template-parsing-in-cxx20">>;
@@ -878,4 +885,9 @@ def warn_drv_openacc_without_cir
     : Warning<"OpenACC directives will result in no runtime behavior; use "
               "-fclangir to enable runtime effect">,
       InGroup<SourceUsesOpenACC>;
+
+def warn_drv_gcc_install_dir_libstdcxx : Warning<
+    "future releases of the clang compiler will prefer GCC installations "
+    "containing libstdc++ include directories; '%0' would be chosen over '%1'">,
+    InGroup<DiagGroup<"gcc-install-dir-libstdcxx">>;
 }
diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index 8a8db27490f06..b7e27d8f94c6e 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -287,6 +287,10 @@ def err_function_needs_feature : Error<
   "always_inline function %1 requires target feature '%2', but would "
   "be inlined into function %0 that is compiled without support for '%2'">;
 
+def err_flatten_function_needs_feature : Error<
+  "flatten function %0 calls %1 which requires target feature '%2', but the "
+  "caller is compiled without support for '%2'">;
+
 let CategoryName = "Codegen ABI Check" in {
 def err_function_always_inline_attribute_mismatch : Error<
   "always_inline function %1 and its caller %0 have mismatching %2 attributes">;
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index ccb18aa37447e..0c994e0b5ca4d 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -533,7 +533,14 @@ def Dangling : DiagGroup<"dangling", [DanglingAssignment,
                                       DanglingGsl,
                                       ReturnStackAddress]>;
 
-def LifetimeSafety : DiagGroup<"experimental-lifetime-safety">;
+def LifetimeSafetyPermissive : DiagGroup<"experimental-lifetime-safety-permissive">;
+def LifetimeSafetyStrict : DiagGroup<"experimental-lifetime-safety-strict">;
+def LifetimeSafety : DiagGroup<"experimental-lifetime-safety",
+                               [LifetimeSafetyPermissive, LifetimeSafetyStrict]> {
+  code Documentation = [{
+    Experimental warnings to detect use-after-free and related temporal safety bugs based on lifetime safety analysis.
+  }];
+}
 
 def DistributedObjectModifiers : DiagGroup<"distributed-object-modifiers">;
 def DllexportExplicitInstantiationDecl : DiagGroup<"dllexport-explicit-instantiation-decl">;
@@ -628,6 +635,7 @@ def ModuleConflict : DiagGroup<"module-conflict">;
 def ModuleFileExtension : DiagGroup<"module-file-extension">;
 def ModuleIncludeDirectiveTranslation : DiagGroup<"module-include-translation">;
 def ModuleMap : DiagGroup<"module-map">;
+def ModulesDriver : DiagGroup<"modules-driver">;
 def RoundTripCC1Args : DiagGroup<"round-trip-cc1-args">;
 def NewlineEOF : DiagGroup<"newline-eof">;
 def Nullability : DiagGroup<"nullability">;
@@ -644,6 +652,7 @@ def NonNull : DiagGroup<"nonnull">;
 def NonPODVarargs : DiagGroup<"non-pod-varargs">;
 def ClassVarargs : DiagGroup<"class-varargs", [NonPODVarargs]>;
 def : DiagGroup<"nonportable-cfstrings">;
+def NonPortableSYCL : DiagGroup<"nonportable-sycl">;
 def NonVirtualDtor : DiagGroup<"non-virtual-dtor">;
 def GNUNullPointerArithmetic : DiagGroup<"gnu-null-pointer-arithmetic">;
 def NullPointerArithmetic
diff --git a/clang/include/clang/Basic/DiagnosticIDs.h b/clang/include/clang/Basic/DiagnosticIDs.h
index b21a3b6232fc0..17fecd346f03e 100644
--- a/clang/include/clang/Basic/DiagnosticIDs.h
+++ b/clang/include/clang/Basic/DiagnosticIDs.h
@@ -23,76 +23,78 @@
 #include <vector>
 
 namespace clang {
-  class DiagnosticsEngine;
-  class DiagnosticBuilder;
-  class LangOptions;
-  class SourceLocation;
-
-  // Import the diagnostic enums themselves.
-  namespace diag {
-    enum class Group;
-
-    // Size of each of the diagnostic categories.
-    enum {
-      DIAG_SIZE_COMMON        =  300,
-      DIAG_SIZE_DRIVER        =  400,
-      DIAG_SIZE_FRONTEND      =  200,
-      DIAG_SIZE_SERIALIZATION =  120,
-      DIAG_SIZE_LEX           =  500,
-      DIAG_SIZE_PARSE         =  800,
-      DIAG_SIZE_AST           =  300,
-      DIAG_SIZE_COMMENT       =  100,
-      DIAG_SIZE_CROSSTU       =  100,
-      DIAG_SIZE_SEMA          = 5000,
-      DIAG_SIZE_ANALYSIS      =  100,
-      DIAG_SIZE_REFACTORING   = 1000,
-      DIAG_SIZE_INSTALLAPI    =  100,
-    };
-    // Start position for diagnostics.
-    enum {
-      DIAG_START_COMMON        =                          0,
-      DIAG_START_DRIVER        = DIAG_START_COMMON        + static_cast<int>(DIAG_SIZE_COMMON),
-      DIAG_START_FRONTEND      = DIAG_START_DRIVER        + static_cast<int>(DIAG_SIZE_DRIVER),
-      DIAG_START_SERIALIZATION = DIAG_START_FRONTEND      + static_cast<int>(DIAG_SIZE_FRONTEND),
-      DIAG_START_LEX           = DIAG_START_SERIALIZATION + static_cast<int>(DIAG_SIZE_SERIALIZATION),
-      DIAG_START_PARSE         = DIAG_START_LEX           + static_cast<int>(DIAG_SIZE_LEX),
-      DIAG_START_AST           = DIAG_START_PARSE         + static_cast<int>(DIAG_SIZE_PARSE),
-      DIAG_START_COMMENT       = DIAG_START_AST           + static_cast<int>(DIAG_SIZE_AST),
-      DIAG_START_CROSSTU       = DIAG_START_COMMENT       + static_cast<int>(DIAG_SIZE_COMMENT),
-      DIAG_START_SEMA          = DIAG_START_CROSSTU       + static_cast<int>(DIAG_SIZE_CROSSTU),
-      DIAG_START_ANALYSIS      = DIAG_START_SEMA          + static_cast<int>(DIAG_SIZE_SEMA),
-      DIAG_START_REFACTORING   = DIAG_START_ANALYSIS      + static_cast<int>(DIAG_SIZE_ANALYSIS),
-      DIAG_START_INSTALLAPI    = DIAG_START_REFACTORING   + static_cast<int>(DIAG_SIZE_REFACTORING),
-      DIAG_UPPER_LIMIT         = DIAG_START_INSTALLAPI    + static_cast<int>(DIAG_SIZE_INSTALLAPI)
-    };
-
-    class CustomDiagInfo;
-
-    /// All of the diagnostics that can be emitted by the frontend.
-    typedef unsigned kind;
-
-    /// Enum values that allow the client to map NOTEs, WARNINGs, and EXTENSIONs
-    /// to either Ignore (nothing), Remark (emit a remark), Warning
-    /// (emit a warning) or Error (emit as an error).  It allows clients to
-    /// map ERRORs to Error or Fatal (stop emitting diagnostics after this one).
-    enum class Severity : uint8_t {
-      // NOTE: 0 means "uncomputed".
-      Ignored = 1, ///< Do not present this diagnostic, ignore it.
-      Remark = 2,  ///< Present this diagnostic as a remark.
-      Warning = 3, ///< Present this diagnostic as a warning.
-      Error = 4,   ///< Present this diagnostic as an error.
-      Fatal = 5    ///< Present this diagnostic as a fatal error.
-    };
-
-    /// Flavors of diagnostics we can emit. Used to filter for a particular
-    /// kind of diagnostic (for instance, for -W/-R flags).
-    enum class Flavor {
-      WarningOrError, ///< A diagnostic that indicates a problem or potential
-                      ///< problem. Can be made fatal by -Werror.
-      Remark          ///< A diagnostic that indicates normal progress through
-                      ///< compilation.
-    };
-  } // end namespace diag
+class DiagnosticsEngine;
+class DiagnosticBuilder;
+class LangOptions;
+class SourceLocation;
+
+// Import the diagnostic enums themselves.
+namespace diag {
+enum class Group;
+
+// Size of each of the diagnostic categories.
+enum {
+  DIAG_SIZE_COMMON = 300,
+  DIAG_SIZE_DRIVER = 400,
+  DIAG_SIZE_FRONTEND = 200,
+  DIAG_SIZE_SERIALIZATION = 120,
+  DIAG_SIZE_LEX = 500,
+  DIAG_SIZE_PARSE = 800,
+  DIAG_SIZE_AST = 300,
+  DIAG_SIZE_COMMENT = 100,
+  DIAG_SIZE_CROSSTU = 100,
+  DIAG_SIZE_SEMA = 5000,
+  DIAG_SIZE_ANALYSIS = 100,
+  DIAG_SIZE_REFACTORING = 1000,
+  DIAG_SIZE_INSTALLAPI = 100,
+};
+// Start position for diagnostics.
+// clang-format off
+enum {
+  DIAG_START_COMMON        =                          0,
+  DIAG_START_DRIVER        = DIAG_START_COMMON        + static_cast<int>(DIAG_SIZE_COMMON),
+  DIAG_START_FRONTEND      = DIAG_START_DRIVER        + static_cast<int>(DIAG_SIZE_DRIVER),
+  DIAG_START_SERIALIZATION = DIAG_START_FRONTEND      + static_cast<int>(DIAG_SIZE_FRONTEND),
+  DIAG_START_LEX           = DIAG_START_SERIALIZATION + static_cast<int>(DIAG_SIZE_SERIALIZATION),
+  DIAG_START_PARSE         = DIAG_START_LEX           + static_cast<int>(DIAG_SIZE_LEX),
+  DIAG_START_AST           = DIAG_START_PARSE         + static_cast<int>(DIAG_SIZE_PARSE),
+  DIAG_START_COMMENT       = DIAG_START_AST           + static_cast<int>(DIAG_SIZE_AST),
+  DIAG_START_CROSSTU       = DIAG_START_COMMENT       + static_cast<int>(DIAG_SIZE_COMMENT),
+  DIAG_START_SEMA          = DIAG_START_CROSSTU       + static_cast<int>(DIAG_SIZE_CROSSTU),
+  DIAG_START_ANALYSIS      = DIAG_START_SEMA          + static_cast<int>(DIAG_SIZE_SEMA),
+  DIAG_START_REFACTORING   = DIAG_START_ANALYSIS      + static_cast<int>(DIAG_SIZE_ANALYSIS),
+  DIAG_START_INSTALLAPI    = DIAG_START_REFACTORING   + static_cast<int>(DIAG_SIZE_REFACTORING),
+  DIAG_UPPER_LIMIT         = DIAG_START_INSTALLAPI    + static_cast<int>(DIAG_SIZE_INSTALLAPI)
+};
+// clang-format on
+
+class CustomDiagInfo;
+
+/// All of the diagnostics that can be emitted by the frontend.
+typedef unsigned kind;
+
+/// Enum values that allow the client to map NOTEs, WARNINGs, and EXTENSIONs
+/// to either Ignore (nothing), Remark (emit a remark), Warning
+/// (emit a warning) or Error (emit as an error).  It allows clients to
+/// map ERRORs to Error or Fatal (stop emitting diagnostics after this one).
+enum class Severity : uint8_t {
+  // NOTE: 0 means "uncomputed".
+  Ignored = 1, ///< Do not present this diagnostic, ignore it.
+  Remark = 2,  ///< Present this diagnostic as a remark.
+  Warning = 3, ///< Present this diagnostic as a warning.
+  Error = 4,   ///< Present this diagnostic as an error.
+  Fatal = 5    ///< Present this diagnostic as a fatal error.
+};
+
+/// Flavors of diagnostics we can emit. Used to filter for a particular
+/// kind of diagnostic (for instance, for -W/-R flags).
+enum class Flavor {
+  WarningOrError, ///< A diagnostic that indicates a problem or potential
+                  ///< problem. Can be made fatal by -Werror.
+  Remark          ///< A diagnostic that indicates normal progress through
+                  ///< compilation.
+};
+} // end namespace diag
 } // end namespace clang
 
 // This has to be included *after* the DIAG_START_ enums above are defined.
@@ -173,7 +175,8 @@ class DiagnosticMapping {
 
 /// Used for handling and querying diagnostic IDs.
 ///
-/// Can be used and shared by multiple Diagnostics for multiple translation units.
+/// Can be used and shared by multiple Diagnostics for multiple translation
+/// units.
 class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
 public:
   /// The level of the diagnostic, after it has been through mapping.
@@ -498,6 +501,6 @@ class DiagnosticIDs : public RefCountedBase<DiagnosticIDs> {
   friend class DiagnosticsEngine;
 };
 
-}  // end namespace clang
+} // end namespace clang
 
 #endif
diff --git a/clang/include/clang/Basic/DiagnosticLexKinds.td b/clang/include/clang/Basic/DiagnosticLexKinds.td
index c7fe6e1db6d1f..c03c4033cd3a6 100644
--- a/clang/include/clang/Basic/DiagnosticLexKinds.td
+++ b/clang/include/clang/Basic/DiagnosticLexKinds.td
@@ -912,6 +912,11 @@ def err_mmap_nested_submodule_id : Error<
   "qualified module name can only be used to define modules at the top level">;
 def err_mmap_expected_feature : Error<"expected a feature name">;
 def err_mmap_expected_attribute : Error<"expected an attribute name">;
+def warn_mmap_link_redeclaration : Warning<"redeclaration of link library '%0'">,
+  InGroup<DiagGroup<"module-link-redeclaration">>, DefaultError;
+def note_mmap_prev_link_declaration : Note<"previously declared here">;
+def err_mmap_submodule_link_decl
+    : Error<"link declaration is not allowed in submodules">;
 def warn_mmap_unknown_attribute : Warning<"unknown attribute '%0'">,
   InGroup<IgnoredAttributes>;
 def warn_mmap_mismatched_private_submodule : Warning<
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 0042afccba2c8..3a6a9e582c7ca 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -830,6 +830,9 @@ def err_ms_property_expected_comma_or_rparen : Error<
   "expected ',' or ')' at end of property accessor list">;
 def err_ms_property_initializer : Error<
   "property declaration cannot have a default member initializer">;
+def ext_invalid_attribute_argument
+    : Extension<"'%0' is not allowed in an attribute argument list">,
+    InGroup<DiagGroup<"attribute-preprocessor-tokens">>;
 
 def err_assume_attr_expects_cond_expr : Error<
   "use of this expression in an %0 attribute requires parentheses">;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index f903b7f4dacd0..4c98e0f8b04ed 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -1772,7 +1772,8 @@ def note_unsatisfied_trait
            "%Replaceable{replaceable}|"
            "%TriviallyCopyable{trivially copyable}|"
            "%Empty{empty}|"
-           "%StandardLayout{standard-layout}"
+           "%StandardLayout{standard-layout}|"
+           "%Final{final}"
            "}1">;
 
 def note_unsatisfied_trait_reason
@@ -1815,7 +1816,9 @@ def note_unsatisfied_trait_reason
            "%sub{select_special_member_kind}1}|"
            "%FunctionType{is a function type}|"
            "%CVVoidType{is a cv void type}|"
-           "%IncompleteArrayType{is an incomplete array type}"
+           "%IncompleteArrayType{is an incomplete array type}|"
+           "%NotClassOrUnion{is not a class or union type}|"
+           "%NotMarkedFinal{is not marked 'final'}"
            "}0">;
 
 def warn_consteval_if_always_true : Warning<
@@ -4404,6 +4407,11 @@ def warn_impcast_different_enum_types : Warning<
 def warn_impcast_int_to_enum : Warning<
   "implicit conversion from %0 to enumeration type %1 is invalid in C++">,
   InGroup<ImplicitIntToEnumCast>, DefaultIgnore;
+
+def note_no_implicit_conversion_for_scoped_enum
+    : Note<"no implicit conversion for scoped enum; consider casting to "
+           "underlying type">;
+
 def warn_impcast_bool_to_null_pointer : Warning<
     "initialization of pointer of type %0 to null from a constant boolean "
     "expression">, InGroup<BoolConversion>;
@@ -6074,6 +6082,13 @@ def warn_cxx23_pack_indexing : Warning<
 def err_pack_outside_template : Error<
   "pack declaration outside of template">;
 
+def err_builtin_pack_outside_template
+    : Error<"%0 cannot be used outside of template">;
+
+def err_unsupported_builtin_template_pack_expansion
+    : Error<"expansions of %0 are not supported here. Only expansions in "
+            "template arguments and class bases are supported">;
+
 def err_fold_expression_packs_both_sides : Error<
   "binary fold expression has unexpanded parameter packs in both operands">;
 def err_fold_expression_empty : Error<
@@ -10423,9 +10438,10 @@ def warn_format_conversion_argument_type_mismatch : Warning<
 def warn_format_conversion_argument_type_mismatch_pedantic : Extension<
   warn_format_conversion_argument_type_mismatch.Summary>,
   InGroup<FormatPedantic>;
-def warn_format_conversion_argument_type_mismatch_signedness : Warning<
-  warn_format_conversion_argument_type_mismatch.Summary>,
-  InGroup<FormatSignedness>, DefaultIgnore;
+def warn_format_conversion_argument_type_mismatch_signedness: Warning<
+      "format specifies type %0 but the argument has %select{type|underlying "
+      "type}2 %1, which differs in signedness" >
+    , InGroup<FormatSignedness>, DefaultIgnore;
 def warn_format_conversion_argument_type_mismatch_confusion : Warning<
   warn_format_conversion_argument_type_mismatch.Summary>,
   InGroup<FormatTypeConfusion>, DefaultIgnore;
@@ -10537,8 +10553,10 @@ def warn_format_cmp_sensitivity_mismatch : Warning<
   "it should be %select{unspecified|private|public|sensitive}1">, InGroup<Format>;
 def warn_format_cmp_specifier_mismatch : Warning<
   "format specifier '%0' is incompatible with '%1'">, InGroup<Format>;
-def warn_format_cmp_specifier_sign_mismatch : Warning<
-  "signedness of format specifier '%0' is incompatible with '%1'">, InGroup<Format>;
+def warn_format_cmp_specifier_sign_mismatch
+    : Warning<"signedness of format specifier '%0' is incompatible with '%1'">,
+      InGroup<FormatSignedness>,
+      DefaultIgnore;
 def warn_format_cmp_specifier_mismatch_pedantic : Extension<
   warn_format_cmp_specifier_sign_mismatch.Summary>, InGroup<FormatPedantic>;
 def note_format_cmp_with : Note<
@@ -10668,9 +10686,15 @@ def warn_dangling_reference_captured_by_unknown : Warning<
    "object whose reference is captured will be destroyed at the end of "
    "the full-expression">, InGroup<DanglingCapture>;
 
-def warn_experimental_lifetime_safety_dummy_warning : Warning<
-   "todo: remove this warning after we have atleast one warning based on the lifetime analysis">, 
-   InGroup<LifetimeSafety>, DefaultIgnore;
+// Diagnostics based on the Lifetime safety analysis.
+def warn_lifetime_safety_loan_expires_permissive : Warning<
+   "object whose reference is captured does not live long enough">, 
+   InGroup<LifetimeSafetyPermissive>, DefaultIgnore;
+def warn_lifetime_safety_loan_expires_strict : Warning<
+   "object whose reference is captured may not live long enough">,
+   InGroup<LifetimeSafetyStrict>, DefaultIgnore;
+def note_lifetime_safety_used_here : Note<"later used here">;
+def note_lifetime_safety_destroyed_here : Note<"destroyed here">;
 
 // For non-floating point, expressions of the form x == x or x != x
 // should result in a warning, since these always evaluate to a constant.
@@ -10985,10 +11009,15 @@ def err_block_on_vm : Error<
 def err_sizeless_nonlocal : Error<
   "non-local variable with sizeless type %0">;
 
+def err_vec_masked_load_store_ptr : Error<
+ "%ordinal0 argument must be a %1">;
+def err_vec_masked_load_store_size : Error<
+ "all arguments to %0 must have the same number of elements (was %1 and %2)">;
+
 def err_vec_builtin_non_vector : Error<
  "%select{first two|all}1 arguments to %0 must be vectors">;
 def err_vec_builtin_incompatible_vector : Error<
-  "%select{first two|all}1 arguments to %0 must have the same type">;
+  "%select{first two|all|last two}1 arguments to %0 must have the same type">;
 def err_vsx_builtin_nonconstant_argument : Error<
   "argument %0 to %1 must be a 2-bit unsigned literal (i.e. 0, 1, 2 or 3)">;
 
@@ -12850,7 +12879,7 @@ def err_builtin_invalid_arg_type: Error<
   "%plural{0:|: }1"
   // Second component: integer-like types
   "%select{|integer|signed integer|unsigned integer|'int'|"
-  "pointer to a valid matrix element}2"
+  "pointer to a valid matrix element|boolean}2"
   // A space after a non-empty second component
   "%plural{0:|: }2"
   // An 'or' if non-empty second and third components are combined
@@ -12942,6 +12971,17 @@ def err_sycl_special_type_num_init_method : Error<
   "types with 'sycl_special_class' attribute must have one and only one '__init' "
   "method defined">;
 
+// SYCL external attribute diagnostics
+def err_sycl_external_invalid_linkage : Error<
+  "%0 can only be applied to functions with external linkage">;
+def err_sycl_external_invalid_main : Error<
+  "%0 cannot be applied to the 'main' function">;
+def err_sycl_external_invalid_deleted_function : Error<
+  "%0 cannot be applied to an explicitly deleted function">;
+def warn_sycl_external_missing_on_first_decl : Warning<
+  "%0 attribute does not appear on the first declaration">,
+  InGroup<NonPortableSYCL>;
+
 // SYCL kernel entry point diagnostics
 def err_sycl_entry_point_invalid : Error<
   "the %0 attribute cannot be applied to a"
@@ -12956,7 +12996,7 @@ def err_sycl_kernel_name_conflict : Error<
   "the %0 kernel name argument conflicts with a previous declaration">;
 def warn_sycl_kernel_name_not_a_class_type : Warning<
   "%0 is not a valid SYCL kernel name type; a non-union class type is required">,
-  InGroup<DiagGroup<"nonportable-sycl">>, DefaultError;
+  InGroup<NonPortableSYCL>, DefaultError;
 def warn_sycl_entry_point_redundant_declaration : Warning<
   "redundant %0 attribute">, InGroup<RedundantAttribute>;
 def err_sycl_entry_point_after_definition : Error<
@@ -13234,9 +13274,9 @@ def err_wasm_builtin_arg_must_match_table_element_type : Error <
   "%ordinal0 argument must match the element type of the WebAssembly table in the %ordinal1 argument">;
 def err_wasm_builtin_arg_must_be_integer_type : Error <
   "%ordinal0 argument must be an integer">;
-def err_wasm_builtin_test_fp_sig_cannot_include_reference_type
-    : Error<"not supported for "
-            "function pointers with a reference type %select{return "
+def err_wasm_builtin_test_fp_sig_cannot_include_struct_or_union
+    : Error<"not supported with the multivalue ABI for "
+            "function pointers with a struct/union as %select{return "
             "value|parameter}0">;
 
 // OpenACC diagnostics.
@@ -13359,16 +13399,23 @@ def err_acc_reduction_num_gangs_conflict
             "appear on a '%2' construct "
             "with a '%3' clause%select{ with more than 1 argument|}0">;
 def err_acc_reduction_type
-    : Error<"OpenACC 'reduction' variable must be of scalar type, aggregate, "
-            "sub-array, or a composite of scalar types;%select{| sub-array "
-            "base}1 type is %0">;
-def err_acc_reduction_composite_type
-    : Error<"OpenACC 'reduction' variable must be a composite of scalar types; "
-            "%1 %select{is not a class or struct|is incomplete|is not an "
-            "aggregate}0">;
-def err_acc_reduction_composite_member_type :Error<
-    "OpenACC 'reduction' composite variable must not have non-scalar field">;
-def note_acc_reduction_composite_member_loc : Note<"invalid field is here">;
+    : Error<"invalid type %0 used in OpenACC 'reduction' variable reference; "
+            "type is %enum_select<OACCReductionTy>{%NotScalar{not a scalar "
+            "value, or array of scalars, or composite of "
+            "scalars}|%MemberNotScalar{not a scalar value}|%NotAgg{not an "
+            "aggregate}|%NotComplete{not a complete type}|%NotClassStruct{not "
+            "a class or struct}}1">;
+def note_acc_reduction_array
+    : Note<"used as element type of "
+           "%enum_select<OACCReductionArray>{%Section{sub-array"
+           "}|%Subscript{array}|%ArrayTy{array}}0 type %1">;
+def note_acc_reduction_member_of_composite
+    : Note<"used as field '%0' of composite '%1'">;
+def note_acc_reduction_type_summary
+    : Note<"OpenACC 'reduction' variable reference must be a scalar variable "
+           "or a "
+           "composite of scalars, or an array, sub-array, or element of scalar "
+           "types">;
 def err_acc_loop_not_for_loop
     : Error<"OpenACC '%0' construct can only be applied to a 'for' loop">;
 def note_acc_construct_here : Note<"'%0' construct is here">;
@@ -13529,8 +13576,13 @@ def err_acc_invalid_modifier
 def err_acc_invalid_default_type
     : Error<"invalid value %0 in '%1' clause; valid values are %2">;
 def err_acc_device_type_multiple_archs
-    : Error<"OpenACC 'device_type' clause on a 'set' construct only permits "
+    : Error<"OpenACC 'device_type' clause on a '%0' construct only permits "
             "one architecture">;
+def warn_acc_var_referenced_non_const_array
+    : Warning<"variable of array type %0 referenced in OpenACC '%1' clause "
+              "does not have constant bounds; initialization will happen after "
+              "decay to pointer">,
+      InGroup<DiagGroup<"openacc-var-non-const-array">>;
 def warn_acc_var_referenced_lacks_op
     : Warning<"variable of type %0 referenced in OpenACC '%1' clause does not "
               "have a %enum_select<AccVarReferencedReason>{%DefCtor{default "
diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def
index 72f23614aef11..0e91b42a132c1 100644
--- a/clang/include/clang/Basic/Features.def
+++ b/clang/include/clang/Basic/Features.def
@@ -128,6 +128,7 @@ FEATURE(attribute_overloadable, true)
 FEATURE(attribute_unavailable_with_message, true)
 FEATURE(attribute_unused_on_fields, true)
 FEATURE(attribute_diagnose_if_objc, true)
+FEATURE(ext_vector_type_boolean, true)
 FEATURE(blocks, LangOpts.Blocks)
 FEATURE(c_thread_safety_attributes, true)
 FEATURE(cxx_exceptions, LangOpts.CXXExceptions)
@@ -147,14 +148,17 @@ FEATURE(type_sanitizer, LangOpts.Sanitize.has(SanitizerKind::Type))
 FEATURE(thread_sanitizer, LangOpts.Sanitize.has(SanitizerKind::Thread))
 FEATURE(dataflow_sanitizer, LangOpts.Sanitize.has(SanitizerKind::DataFlow))
 FEATURE(scudo, LangOpts.Sanitize.hasOneOf(SanitizerKind::Scudo))
-FEATURE(ptrauth_intrinsics, LangOpts.PointerAuthIntrinsics)
-EXTENSION(ptrauth_qualifier, LangOpts.PointerAuthIntrinsics)
+FEATURE(ptrauth_intrinsics, LangOpts.PointerAuthIntrinsics &&
+                            PP.getTargetInfo().getTriple().isOSDarwin())
+FEATURE(ptrauth_qualifier, LangOpts.PointerAuthIntrinsics &&
+                           PP.getTargetInfo().getTriple().isOSDarwin())
 FEATURE(ptrauth_calls, LangOpts.PointerAuthCalls)
 FEATURE(ptrauth_returns, LangOpts.PointerAuthReturns)
 FEATURE(ptrauth_vtable_pointer_address_discrimination, LangOpts.PointerAuthVTPtrAddressDiscrimination)
 FEATURE(ptrauth_vtable_pointer_type_discrimination, LangOpts.PointerAuthVTPtrTypeDiscrimination)
 FEATURE(ptrauth_type_info_vtable_pointer_discrimination, LangOpts.PointerAuthTypeInfoVTPtrDiscrimination)
 FEATURE(ptrauth_member_function_pointer_type_discrimination, LangOpts.PointerAuthCalls)
+FEATURE(ptrauth_signed_block_descriptors, LangOpts.PointerAuthBlockDescriptorPointers)
 FEATURE(ptrauth_function_pointer_type_discrimination, LangOpts.PointerAuthFunctionTypeDiscrimination)
 FEATURE(ptrauth_indirect_gotos, LangOpts.PointerAuthIndirectGotos)
 FEATURE(ptrauth_init_fini, LangOpts.PointerAuthInitFini)
@@ -163,7 +167,7 @@ FEATURE(ptrauth_elf_got, LangOpts.PointerAuthELFGOT)
 
 FEATURE(ptrauth_objc_isa, LangOpts.PointerAuthObjcIsa)
 FEATURE(ptrauth_objc_interface_sel, LangOpts.PointerAuthObjcInterfaceSel)
-FEATURE(ptrauth_objc_signable_class, true)
+FEATURE(ptrauth_objc_signable_class, LangOpts.PointerAuthIntrinsics)
 FEATURE(ptrauth_objc_method_list_pointer, LangOpts.PointerAuthCalls)
 
 EXTENSION(swiftcc,
@@ -303,6 +307,14 @@ FEATURE(is_trivially_assignable, LangOpts.CPlusPlus)
 FEATURE(is_trivially_constructible, LangOpts.CPlusPlus)
 FEATURE(is_trivially_copyable, LangOpts.CPlusPlus)
 FEATURE(is_union, LangOpts.CPlusPlus)
+FEATURE(cfi_sanitizer, LangOpts.Sanitize.hasOneOf(SanitizerKind::CFI))
+FEATURE(cfi_cast_strict_sanitizer, LangOpts.Sanitize.has(SanitizerKind::CFICastStrict))
+FEATURE(cfi_derived_cast_sanitizer, LangOpts.Sanitize.has(SanitizerKind::CFIDerivedCast))
+FEATURE(cfi_icall_sanitizer, LangOpts.Sanitize.has(SanitizerKind::CFIICall))
+FEATURE(cfi_mfcall_sanitizer, LangOpts.Sanitize.has(SanitizerKind::CFIMFCall))
+FEATURE(cfi_unrelated_cast_sanitizer, LangOpts.Sanitize.has(SanitizerKind::CFIUnrelatedCast))
+FEATURE(cfi_nvcall_sanitizer, LangOpts.Sanitize.has(SanitizerKind::CFINVCall))
+FEATURE(cfi_vcall_sanitizer, LangOpts.Sanitize.has(SanitizerKind::CFIVCall))
 FEATURE(kcfi, LangOpts.Sanitize.has(SanitizerKind::KCFI))
 FEATURE(kcfi_arity, LangOpts.Sanitize.has(SanitizerKind::KCFI))
 FEATURE(modules, LangOpts.Modules)
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 08d98a77e0252..f094ba112988f 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -136,6 +136,8 @@ LANGOPT(PointerAuthObjcInterfaceSel, 1, 0, NotCompatible, "authentication of SEL
 LANGOPT(PointerAuthObjcInterfaceSelKey, 16, 0, NotCompatible, "authentication key for SEL fields of ObjC interfaces")
 LANGOPT(PointerAuthObjcClassROPointers, 1, 0, Benign, "class_ro_t pointer authentication")
 
+LANGOPT(PointerAuthBlockDescriptorPointers, 1, 0, NotCompatible, "enable signed block descriptors")
+
 LANGOPT(DoubleSquareBracketAttributes, 1, 0, NotCompatible, "'[[]]' attributes extension for all language standard modes")
 LANGOPT(ExperimentalLateParseAttributes, 1, 0, NotCompatible, "experimental late parsing of attributes")
 
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 0407897359b5e..569584bcc2297 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -186,95 +186,10 @@ class LangOptionsBase {
 
   /// Clang versions with different platform ABI conformance.
   enum class ClangABI {
-    /// Attempt to be ABI-compatible with code generated by Clang 3.8.x
-    /// (SVN r257626). This causes <1 x long long> to be passed in an
-    /// integer register instead of an SSE register on x64_64.
-    Ver3_8,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 4.0.x
-    /// (SVN r291814). This causes move operations to be ignored when
-    /// determining whether a class type can be passed or returned directly.
-    Ver4,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 6.0.x
-    /// (SVN r321711). This causes determination of whether a type is
-    /// standard-layout to ignore collisions between empty base classes
-    /// and between base classes and member subobjects, which affects
-    /// whether we reuse base class tail padding in some ABIs.
-    Ver6,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 7.0.x
-    /// (SVN r338536). This causes alignof (C++) and _Alignof (C11) to be
-    /// compatible with __alignof (i.e., return the preferred alignment)
-    /// rather than returning the required alignment.
-    Ver7,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 9.0.x
-    /// (SVN r351319). This causes vectors of __int128 to be passed in memory
-    /// instead of passing in multiple scalar registers on x86_64 on Linux and
-    /// NetBSD.
-    Ver9,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 11.0.x
-    /// (git 2e10b7a39b93). This causes clang to pass unions with a 256-bit
-    /// vector member on the stack instead of using registers, to not properly
-    /// mangle substitutions for template names in some cases, and to mangle
-    /// declaration template arguments without a cast to the parameter type
-    /// even when that can lead to mangling collisions.
-    Ver11,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 12.0.x
-    /// (git 8e464dd76bef). This causes clang to mangle lambdas within
-    /// global-scope inline variables incorrectly.
-    Ver12,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 14.0.x.
-    /// This causes clang to:
-    ///   - mangle dependent nested names incorrectly.
-    ///   - make trivial only those defaulted copy constructors with a
-    ///     parameter-type-list equivalent to the parameter-type-list of an
-    ///     implicit declaration.
-    Ver14,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 15.0.x.
-    /// This causes clang to:
-    ///   - Reverse the implementation for DR692, DR1395 and DR1432.
-    ///   - pack non-POD members of packed structs.
-    ///   - consider classes with defaulted special member functions non-pod.
-    Ver15,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 17.0.x.
-    /// This causes clang to revert some fixes to its implementation of the
-    /// Itanium name mangling scheme, with the consequence that overloaded
-    /// function templates are mangled the same if they differ only by:
-    ///   - constraints
-    ///   - whether a non-type template parameter has a deduced type
-    ///   - the parameter list of a template template parameter
-    Ver17,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 18.0.x.
-    /// This causes clang to revert some fixes to the mangling of lambdas
-    /// in the initializers of members of local classes.
-    Ver18,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 19.0.x.
-    /// This causes clang to:
-    ///   - Incorrectly mangle the 'base type' substitutions of the CXX
-    ///   construction vtable because it hasn't added 'type' as a substitution.
-    ///   - Skip mangling enclosing class templates of member-like friend
-    ///   function templates.
-    ///   - Ignore empty struct arguments in C++ mode for ARM, instead of
-    ///   passing them as if they had a size of 1 byte.
-    Ver19,
-
-    /// Attempt to be ABI-compatible with code generated by Clang 20.0.x.
-    /// This causes clang to:
-    ///   - Incorrectly return C++ records in AVX registers on x86_64.
-    Ver20,
-
-    /// Conform to the underlying platform's C and C++ ABIs as closely
-    /// as we can.
-    Latest
+#define ABI_VER_MAJOR_MINOR(Major, Minor) Ver##Major##_##Minor,
+#define ABI_VER_MAJOR(Major) Ver##Major,
+#define ABI_VER_LATEST(Latest) Latest
+#include "clang/Basic/ABIVersions.def"
   };
 
   enum class CoreFoundationABI {
diff --git a/clang/include/clang/Basic/PointerAuthOptions.h b/clang/include/clang/Basic/PointerAuthOptions.h
index fb6dddf3ae9ce..2b920250721fc 100644
--- a/clang/include/clang/Basic/PointerAuthOptions.h
+++ b/clang/include/clang/Basic/PointerAuthOptions.h
@@ -23,6 +23,10 @@
 
 namespace clang {
 
+/// Constant discriminator to be used with block descriptor pointers. The value
+/// is ptrauth_string_discriminator("block_descriptor")
+constexpr uint16_t BlockDescriptorConstantDiscriminator = 0xC0BB;
+
 /// Constant discriminator to be used with function pointers in .init_array and
 /// .fini_array. The value is ptrauth_string_discriminator("init_fini")
 constexpr uint16_t InitFiniPointerConstantDiscriminator = 0xD9D4;
@@ -223,6 +227,18 @@ struct PointerAuthOptions {
   /// The ABI for function addresses in .init_array and .fini_array
   PointerAuthSchema InitFiniPointers;
 
+  /// The ABI for block invocation function pointers.
+  PointerAuthSchema BlockInvocationFunctionPointers;
+
+  /// The ABI for block object copy/destroy function pointers.
+  PointerAuthSchema BlockHelperFunctionPointers;
+
+  /// The ABI for __block variable copy/destroy function pointers.
+  PointerAuthSchema BlockByrefHelperFunctionPointers;
+
+  /// The ABI for pointers to block descriptors.
+  PointerAuthSchema BlockDescriptorPointers;
+
   /// The ABI for Objective-C method lists.
   PointerAuthSchema ObjCMethodListFunctionPointers;
 
diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h
index ce4677e540226..25b68622656fa 100644
--- a/clang/include/clang/Basic/TargetInfo.h
+++ b/clang/include/clang/Basic/TargetInfo.h
@@ -233,8 +233,9 @@ class TargetInfo : public TransferrableTargetInfo,
   bool TLSSupported;
   bool VLASupported;
   bool NoAsmVariants;  // True if {|} are normal characters.
-  bool HasLegalHalfType; // True if the backend supports operations on the half
-                         // LLVM IR type.
+  bool HasFastHalfType;    // True if the backend has native half float support,
+                           // and performing calculations in float instead does
+                           // not have a performance advantage.
   bool HalfArgsAndReturns; // OpenCL 6.1.1.1, NEON (IEEE 754-2008 half) type.
   bool HasFloat128;
   bool HasFloat16;
@@ -700,8 +701,9 @@ class TargetInfo : public TransferrableTargetInfo,
     return 128;
   }
 
-  /// Determine whether _Float16 is supported on this target.
-  virtual bool hasLegalHalfType() const { return HasLegalHalfType; }
+  /// Determine whether the target has fast native support for operations
+  /// on half types.
+  virtual bool hasFastHalfType() const { return HasFastHalfType; }
 
   /// Whether half args and returns are supported.
   virtual bool allowHalfArgsAndReturns() const { return HalfArgsAndReturns; }
diff --git a/clang/include/clang/Basic/TokenKinds.h b/clang/include/clang/Basic/TokenKinds.h
index 1b133dde89587..d84f3598cbf33 100644
--- a/clang/include/clang/Basic/TokenKinds.h
+++ b/clang/include/clang/Basic/TokenKinds.h
@@ -95,10 +95,20 @@ inline bool isStringLiteral(TokenKind K) {
 /// Return true if this is a "literal" kind, like a numeric
 /// constant, string, etc.
 inline bool isLiteral(TokenKind K) {
-  return K == tok::numeric_constant || K == tok::char_constant ||
-         K == tok::wide_char_constant || K == tok::utf8_char_constant ||
-         K == tok::utf16_char_constant || K == tok::utf32_char_constant ||
-         isStringLiteral(K) || K == tok::header_name || K == tok::binary_data;
+  const bool isInLiteralRange =
+      K >= tok::numeric_constant && K <= tok::utf32_string_literal;
+
+#if !NDEBUG
+  const bool isLiteralExplicit =
+      K == tok::numeric_constant || K == tok::char_constant ||
+      K == tok::wide_char_constant || K == tok::utf8_char_constant ||
+      K == tok::utf16_char_constant || K == tok::utf32_char_constant ||
+      isStringLiteral(K) || K == tok::header_name || K == tok::binary_data;
+  assert(isInLiteralRange == isLiteralExplicit &&
+         "TokenKind literals should be contiguous");
+#endif
+
+  return isInLiteralRange;
 }
 
 /// Return true if this is any of tok::annot_* kinds.
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
index 971ce541d4831..c8d45dec78816 100644
--- a/clang/include/clang/Basic/TypeNodes.td
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -90,19 +90,20 @@ def UnaryTransformType : TypeNode<Type>, NeverCanonicalUnlessDependent;
 def TagType : TypeNode<Type, 1>;
 def RecordType : TypeNode<TagType>, LeafType;
 def EnumType : TypeNode<TagType>, LeafType;
-def ElaboratedType : TypeNode<Type>, NeverCanonical;
+def InjectedClassNameType : TypeNode<TagType>, AlwaysDependent, LeafType;
 def AttributedType : TypeNode<Type>, NeverCanonical;
 def BTFTagAttributedType : TypeNode<Type>, NeverCanonical;
 def HLSLAttributedResourceType : TypeNode<Type>;
 def HLSLInlineSpirvType : TypeNode<Type>;
 def TemplateTypeParmType : TypeNode<Type>, AlwaysDependent, LeafType;
 def SubstTemplateTypeParmType : TypeNode<Type>, NeverCanonical;
-def SubstTemplateTypeParmPackType : TypeNode<Type>, AlwaysDependent;
+def SubstPackType : TypeNode<Type, 1>;
+def SubstTemplateTypeParmPackType : TypeNode<SubstPackType>, AlwaysDependent;
+def SubstBuiltinTemplatePackType : TypeNode<SubstPackType>, AlwaysDependent;
 def TemplateSpecializationType : TypeNode<Type>, NeverCanonicalUnlessDependent;
 def DeducedType : TypeNode<Type, 1>;
 def AutoType : TypeNode<DeducedType>;
 def DeducedTemplateSpecializationType : TypeNode<DeducedType>;
-def InjectedClassNameType : TypeNode<Type>, AlwaysDependent, LeafType;
 def DependentNameType : TypeNode<Type>, AlwaysDependent;
 def DependentTemplateSpecializationType : TypeNode<Type>, AlwaysDependent;
 def PackExpansionType : TypeNode<Type>, AlwaysDependent;
diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index c491eb0f7e460..a4eb92e76968c 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -21,23 +21,21 @@ let SVETargetGuard = InvalidMode in {
 // Loads
 
 multiclass ZALoad<string n_suffix, string t, string i_prefix, list<ImmCheck> ch> {
-  let SMETargetGuard = "sme" in {
-    def NAME # _H : MInst<"svld1_hor_" # n_suffix, "vimPQ", t,
-                          [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
-                          MemEltTyDefault, i_prefix # "_horiz", ch>;
-
-    def NAME # _H_VNUM : MInst<"svld1_hor_vnum_" # n_suffix, "vimPQl", t,
-                               [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
-                               MemEltTyDefault, i_prefix # "_horiz", ch>;
-
-    def NAME # _V : MInst<"svld1_ver_" # n_suffix, "vimPQ", t,
-                          [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
-                          MemEltTyDefault, i_prefix # "_vert", ch>;
-
-    def NAME # _V_VNUM : MInst<"svld1_ver_vnum_" # n_suffix, "vimPQl", t,
-                               [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
-                               MemEltTyDefault, i_prefix # "_vert", ch>;
-  }
+  def NAME # _H : MInst<"svld1_hor_" # n_suffix, "vimPQ", t,
+                        [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
+                        MemEltTyDefault, i_prefix # "_horiz", ch>;
+
+  def NAME # _H_VNUM : MInst<"svld1_hor_vnum_" # n_suffix, "vimPQl", t,
+                             [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
+                             MemEltTyDefault, i_prefix # "_horiz", ch>;
+
+  def NAME # _V : MInst<"svld1_ver_" # n_suffix, "vimPQ", t,
+                        [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
+                        MemEltTyDefault, i_prefix # "_vert", ch>;
+
+  def NAME # _V_VNUM : MInst<"svld1_ver_vnum_" # n_suffix, "vimPQl", t,
+                             [IsLoad, IsOverloadNone, IsStreaming, IsInOutZA],
+                             MemEltTyDefault, i_prefix # "_vert", ch>;
 }
 
 defm SVLD1_ZA8 : ZALoad<"za8", "c", "aarch64_sme_ld1b", [ImmCheck<0, ImmCheck0_0>]>;
@@ -46,7 +44,6 @@ defm SVLD1_ZA32 : ZALoad<"za32", "i", "aarch64_sme_ld1w", [ImmCheck<0, ImmCheck0
 defm SVLD1_ZA64 : ZALoad<"za64", "l", "aarch64_sme_ld1d", [ImmCheck<0, ImmCheck0_7>]>;
 defm SVLD1_ZA128 : ZALoad<"za128", "q", "aarch64_sme_ld1q", [ImmCheck<0, ImmCheck0_15>]>;
 
-let SMETargetGuard = "sme" in {
 def SVLDR_VNUM_ZA : MInst<"svldr_vnum_za", "vmQl", "",
                           [IsOverloadNone, IsStreamingCompatible, IsInOutZA],
                           MemEltTyDefault, "aarch64_sme_ldr">;
@@ -54,29 +51,26 @@ def SVLDR_VNUM_ZA : MInst<"svldr_vnum_za", "vmQl", "",
 def SVLDR_ZA : MInst<"svldr_za", "vmQ", "",
                           [IsOverloadNone, IsStreamingCompatible, IsInOutZA],
                           MemEltTyDefault, "aarch64_sme_ldr", []>;
-}
 
 ////////////////////////////////////////////////////////////////////////////////
 // Stores
 
 multiclass ZAStore<string n_suffix, string t, string i_prefix, list<ImmCheck> ch> {
-  let SMETargetGuard = "sme" in {
-    def NAME # _H : MInst<"svst1_hor_" # n_suffix, "vimP%", t,
-                          [IsStore, IsOverloadNone, IsStreaming, IsInZA],
-                          MemEltTyDefault, i_prefix # "_horiz", ch>;
-
-    def NAME # _H_VNUM : MInst<"svst1_hor_vnum_" # n_suffix, "vimP%l", t,
-                               [IsStore, IsOverloadNone, IsStreaming, IsInZA],
-                               MemEltTyDefault, i_prefix # "_horiz", ch>;
-
-    def NAME # _V : MInst<"svst1_ver_" # n_suffix, "vimP%", t,
-                          [IsStore, IsOverloadNone, IsStreaming, IsInZA],
-                          MemEltTyDefault, i_prefix # "_vert", ch>;
-
-    def NAME # _V_VNUM : MInst<"svst1_ver_vnum_" # n_suffix, "vimP%l", t,
-                               [IsStore, IsOverloadNone, IsStreaming, IsInZA],
-                               MemEltTyDefault, i_prefix # "_vert", ch>;
-  }
+  def NAME # _H : MInst<"svst1_hor_" # n_suffix, "vimP%", t,
+                        [IsStore, IsOverloadNone, IsStreaming, IsInZA],
+                        MemEltTyDefault, i_prefix # "_horiz", ch>;
+
+  def NAME # _H_VNUM : MInst<"svst1_hor_vnum_" # n_suffix, "vimP%l", t,
+                             [IsStore, IsOverloadNone, IsStreaming, IsInZA],
+                             MemEltTyDefault, i_prefix # "_horiz", ch>;
+
+  def NAME # _V : MInst<"svst1_ver_" # n_suffix, "vimP%", t,
+                        [IsStore, IsOverloadNone, IsStreaming, IsInZA],
+                        MemEltTyDefault, i_prefix # "_vert", ch>;
+
+  def NAME # _V_VNUM : MInst<"svst1_ver_vnum_" # n_suffix, "vimP%l", t,
+                             [IsStore, IsOverloadNone, IsStreaming, IsInZA],
+                             MemEltTyDefault, i_prefix # "_vert", ch>;
 }
 
 defm SVST1_ZA8 : ZAStore<"za8", "c", "aarch64_sme_st1b", [ImmCheck<0, ImmCheck0_0>]>;
@@ -85,7 +79,6 @@ defm SVST1_ZA32 : ZAStore<"za32", "i", "aarch64_sme_st1w", [ImmCheck<0, ImmCheck
 defm SVST1_ZA64 : ZAStore<"za64", "l", "aarch64_sme_st1d", [ImmCheck<0, ImmCheck0_7>]>;
 defm SVST1_ZA128 : ZAStore<"za128", "q", "aarch64_sme_st1q", [ImmCheck<0, ImmCheck0_15>]>;
 
-let SMETargetGuard = "sme" in {
 def SVSTR_VNUM_ZA : MInst<"svstr_vnum_za", "vm%l", "",
                           [IsOverloadNone, IsStreamingCompatible, IsInZA],
                           MemEltTyDefault, "aarch64_sme_str">;
@@ -93,21 +86,18 @@ def SVSTR_VNUM_ZA : MInst<"svstr_vnum_za", "vm%l", "",
 def SVSTR_ZA : MInst<"svstr_za", "vm%", "",
                       [IsOverloadNone, IsStreamingCompatible, IsInZA],
                       MemEltTyDefault, "aarch64_sme_str", []>;
-}
 
 ////////////////////////////////////////////////////////////////////////////////
 // Read horizontal/vertical ZA slices
 
 multiclass ZARead<string n_suffix, string t, string i_prefix, list<ImmCheck> ch> {
-  let SMETargetGuard = "sme" in {
-    def NAME # _H : SInst<"svread_hor_" # n_suffix # "[_{d}]", "ddPim", t,
-                          MergeOp1, i_prefix # "_horiz",
-                          [IsReadZA, IsStreaming, IsInZA], ch>;
-
-    def NAME # _V : SInst<"svread_ver_" # n_suffix # "[_{d}]", "ddPim", t,
-                          MergeOp1, i_prefix # "_vert",
-                          [IsReadZA, IsStreaming, IsInZA], ch>;
-  }
+  def NAME # _H : SInst<"svread_hor_" # n_suffix # "[_{d}]", "ddPim", t,
+                        MergeOp1, i_prefix # "_horiz",
+                        [IsReadZA, IsStreaming, IsInZA], ch>;
+
+  def NAME # _V : SInst<"svread_ver_" # n_suffix # "[_{d}]", "ddPim", t,
+                        MergeOp1, i_prefix # "_vert",
+                        [IsReadZA, IsStreaming, IsInZA], ch>;
 }
 
 defm SVREAD_ZA8 : ZARead<"za8", "cUcm", "aarch64_sme_read", [ImmCheck<2, ImmCheck0_0>]>;
@@ -120,15 +110,13 @@ defm SVREAD_ZA128 : ZARead<"za128", "csilUcUsUiUlmhbfd", "aarch64_sme_readq", [I
 // Write horizontal/vertical ZA slices
 
 multiclass ZAWrite<string n_suffix, string t, string i_prefix, list<ImmCheck> ch> {
-  let SMETargetGuard = "sme" in {
-    def NAME # _H : SInst<"svwrite_hor_" # n_suffix # "[_{d}]", "vimPd", t,
-                          MergeOp1, i_prefix # "_horiz",
-                          [IsWriteZA, IsStreaming, IsInOutZA], ch>;
-
-    def NAME # _V : SInst<"svwrite_ver_" # n_suffix # "[_{d}]", "vimPd", t,
-                          MergeOp1, i_prefix # "_vert",
-                          [IsWriteZA, IsStreaming, IsInOutZA], ch>;
-  }
+  def NAME # _H : SInst<"svwrite_hor_" # n_suffix # "[_{d}]", "vimPd", t,
+                        MergeOp1, i_prefix # "_horiz",
+                        [IsWriteZA, IsStreaming, IsInOutZA], ch>;
+
+  def NAME # _V : SInst<"svwrite_ver_" # n_suffix # "[_{d}]", "vimPd", t,
+                        MergeOp1, i_prefix # "_vert",
+                        [IsWriteZA, IsStreaming, IsInOutZA], ch>;
 }
 
 defm SVWRITE_ZA8 : ZAWrite<"za8", "cUcm", "aarch64_sme_write", [ImmCheck<0, ImmCheck0_0>]>;
@@ -140,13 +128,11 @@ defm SVWRITE_ZA128 : ZAWrite<"za128", "csilUcUsUiUlmhbfd", "aarch64_sme_writeq",
 ////////////////////////////////////////////////////////////////////////////////
 // SME - Zero
 
-let SMETargetGuard = "sme" in {
-  def SVZERO_MASK_ZA : SInst<"svzero_mask_za", "vi", "", MergeNone, "aarch64_sme_zero",
-                             [IsOverloadNone, IsStreamingCompatible, IsInOutZA],
-                             [ImmCheck<0, ImmCheck0_255>]>;
-  def SVZERO_ZA      : SInst<"svzero_za", "vv", "", MergeNone, "aarch64_sme_zero",
-                             [IsOverloadNone, IsStreamingCompatible, IsOutZA]>;
-}
+def SVZERO_MASK_ZA : SInst<"svzero_mask_za", "vi", "", MergeNone, "aarch64_sme_zero",
+                           [IsOverloadNone, IsStreamingCompatible, IsInOutZA],
+                           [ImmCheck<0, ImmCheck0_255>]>;
+def SVZERO_ZA      : SInst<"svzero_za", "vv", "", MergeNone, "aarch64_sme_zero",
+                           [IsOverloadNone, IsStreamingCompatible, IsOutZA]>;
 
 let SMETargetGuard = "sme2p1" in {
   def SVZERO_ZA64_VG1x2 : SInst<"svzero_za64_vg1x2", "vm", "", MergeNone, "aarch64_sme_zero_za64_vg1x2",
@@ -171,11 +157,9 @@ let SMETargetGuard = "sme2p1" in {
 // SME - Counting elements in a streaming vector
 
 multiclass ZACount<string n_suffix> {
-  let SMETargetGuard = "sme" in {
-    def NAME : SInst<"sv" # n_suffix, "nv", "", MergeNone,
-                      "aarch64_sme_" # n_suffix,
-                      [IsOverloadNone, IsStreamingCompatible]>;
-  }
+  def NAME : SInst<"sv" # n_suffix, "nv", "", MergeNone,
+                    "aarch64_sme_" # n_suffix,
+                    [IsOverloadNone, IsStreamingCompatible]>;
 }
 
 defm SVCNTSB : ZACount<"cntsb">;
@@ -187,11 +171,9 @@ defm SVCNTSD : ZACount<"cntsd">;
 // SME - ADDHA/ADDVA
 
 multiclass ZAAdd<string n_suffix> {
-  let SMETargetGuard = "sme" in {
-    def NAME # _ZA32: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPd", "iUi", MergeOp1,
-                      "aarch64_sme_" # n_suffix, [IsStreaming, IsInOutZA],
-                      [ImmCheck<0, ImmCheck0_3>]>;
-  }
+  def NAME # _ZA32: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPd", "iUi", MergeOp1,
+                    "aarch64_sme_" # n_suffix, [IsStreaming, IsInOutZA],
+                    [ImmCheck<0, ImmCheck0_3>]>;
 
   let SMETargetGuard = "sme-i16i64" in {
     def NAME # _ZA64: SInst<"sv" # n_suffix # "_za64[_{d}]", "viPPd", "lUl", MergeOp1,
@@ -207,13 +189,11 @@ defm SVADDVA : ZAAdd<"addva">;
 // SME - SMOPA, SMOPS, UMOPA, UMOPS
 
 multiclass ZAIntOuterProd<string n_suffix1, string n_suffix2> {
-  let SMETargetGuard = "sme" in {
-    def NAME # _ZA32_B: SInst<"sv" # n_suffix2 # "_za32[_{d}]",
-                              "viPPdd", !cond(!eq(n_suffix1, "s") : "", true: "U") # "c",
-                              MergeOp1, "aarch64_sme_" # n_suffix1 # n_suffix2 # "_wide",
-                              [IsStreaming, IsInOutZA],
-                              [ImmCheck<0, ImmCheck0_3>]>;
-  }
+  def NAME # _ZA32_B: SInst<"sv" # n_suffix2 # "_za32[_{d}]",
+                            "viPPdd", !cond(!eq(n_suffix1, "s") : "", true: "U") # "c",
+                            MergeOp1, "aarch64_sme_" # n_suffix1 # n_suffix2 # "_wide",
+                            [IsStreaming, IsInOutZA],
+                            [ImmCheck<0, ImmCheck0_3>]>;
 
   let SMETargetGuard = "sme-i16i64" in {
     def NAME # _ZA64_H: SInst<"sv" # n_suffix2 # "_za64[_{d}]",
@@ -233,14 +213,12 @@ defm SVUMOPS : ZAIntOuterProd<"u", "mops">;
 // SME - SUMOPA, SUMOPS, USMOPA, USMOPS
 
 multiclass ZAIntOuterProdMixedSigns<string n_suffix1, string n_suffix2> {
-  let SMETargetGuard = "sme" in {
-    def NAME # _ZA32_B: SInst<"sv" # n_suffix1 # n_suffix2 # "_za32[_{d}]",
-                              "viPPd" # !cond(!eq(n_suffix1, "su") : "u", true: "x"),
-                              !cond(!eq(n_suffix1, "su") : "", true: "U") # "c",
-                              MergeOp1, "aarch64_sme_" # n_suffix1 # n_suffix2 # "_wide",
-                              [IsStreaming, IsInOutZA],
-                              [ImmCheck<0, ImmCheck0_3>]>;
-  }
+  def NAME # _ZA32_B: SInst<"sv" # n_suffix1 # n_suffix2 # "_za32[_{d}]",
+                            "viPPd" # !cond(!eq(n_suffix1, "su") : "u", true: "x"),
+                            !cond(!eq(n_suffix1, "su") : "", true: "U") # "c",
+                            MergeOp1, "aarch64_sme_" # n_suffix1 # n_suffix2 # "_wide",
+                            [IsStreaming, IsInOutZA],
+                            [ImmCheck<0, ImmCheck0_3>]>;
 
   let SMETargetGuard = "sme-i16i64" in {
     def NAME # _ZA64_H: SInst<"sv" # n_suffix1 # n_suffix2 # "_za64[_{d}]",
@@ -261,22 +239,20 @@ defm SVUSMOPS : ZAIntOuterProdMixedSigns<"us", "mops">;
 // SME - FMOPA, FMOPS
 
 multiclass ZAFPOuterProd<string n_suffix> {
-  let SMETargetGuard = "sme" in {
-    def NAME # _ZA32_B: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPdd", "h",
-                              MergeOp1, "aarch64_sme_" # n_suffix # "_wide",
-                              [IsStreaming, IsInOutZA],
-                              [ImmCheck<0, ImmCheck0_3>]>;
+  def NAME # _ZA32_B: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPdd", "h",
+                            MergeOp1, "aarch64_sme_" # n_suffix # "_wide",
+                            [IsStreaming, IsInOutZA],
+                            [ImmCheck<0, ImmCheck0_3>]>;
 
-    def NAME # _ZA32_H: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPdd", "b",
-                              MergeOp1, "aarch64_sme_" # n_suffix # "_wide",
-                              [IsStreaming, IsInOutZA],
-                              [ImmCheck<0, ImmCheck0_3>]>;
+  def NAME # _ZA32_H: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPdd", "b",
+                            MergeOp1, "aarch64_sme_" # n_suffix # "_wide",
+                            [IsStreaming, IsInOutZA],
+                            [ImmCheck<0, ImmCheck0_3>]>;
 
-    def NAME # _ZA32_S: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPdd", "f",
-                              MergeOp1, "aarch64_sme_" # n_suffix,
-                              [IsStreaming, IsInOutZA],
-                              [ImmCheck<0, ImmCheck0_3>]>;
-  }
+  def NAME # _ZA32_S: SInst<"sv" # n_suffix # "_za32[_{d}]", "viPPdd", "f",
+                            MergeOp1, "aarch64_sme_" # n_suffix,
+                            [IsStreaming, IsInOutZA],
+                            [ImmCheck<0, ImmCheck0_3>]>;
 
   let SMETargetGuard = "sme-f64f64" in {
     def NAME # _ZA64_D: SInst<"sv" # n_suffix # "_za64[_{d}]", "viPPdd", "d",
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 07786c6c8a7ee..9ba07d8795909 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -36,7 +36,7 @@ def SVLD1UH_VNUM : MInst<"svld1uh_vnum_{d}", "dPXl", "ilUiUl",            [IsLoa
 def SVLD1SW_VNUM : MInst<"svld1sw_vnum_{d}", "dPUl", "lUl",               [IsLoad, VerifyRuntimeMode],               MemEltTyInt32,   "aarch64_sve_ld1">;
 def SVLD1UW_VNUM : MInst<"svld1uw_vnum_{d}", "dPYl", "lUl",               [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt32,   "aarch64_sve_ld1">;
 
-let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
+let SMETargetGuard = InvalidMode in {
 // Load one vector (vector base)
 def SVLD1_GATHER_BASES_U   : MInst<"svld1_gather[_{2}base]_{d}",   "dPu", "ilUiUlfd", [IsGatherLoad],               MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">;
 def SVLD1SB_GATHER_BASES_U : MInst<"svld1sb_gather[_{2}base]_{d}", "dPu", "ilUiUl",   [IsGatherLoad],               MemEltTyInt8,    "aarch64_sve_ld1_gather_scalar_offset">;
@@ -134,7 +134,7 @@ def SVLDFF1SW_VNUM : MInst<"svldff1sw_vnum_{d}", "dPUl", "lUl",               [I
 def SVLDFF1UW_VNUM : MInst<"svldff1uw_vnum_{d}", "dPYl", "lUl",               [IsLoad, IsZExtReturn], MemEltTyInt32,   "aarch64_sve_ldff1">;
 }
 
-let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
+let SMETargetGuard = InvalidMode in {
 // First-faulting load one vector (vector base)
 def SVLDFF1_GATHER_BASES_U   : MInst<"svldff1_gather[_{2}base]_{d}",   "dPu", "ilUiUlfd", [IsGatherLoad],               MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">;
 def SVLDFF1SB_GATHER_BASES_U : MInst<"svldff1sb_gather[_{2}base]_{d}", "dPu", "ilUiUl",   [IsGatherLoad],               MemEltTyInt8,    "aarch64_sve_ldff1_gather_scalar_offset">;
@@ -251,15 +251,15 @@ def SVLD3_VNUM : SInst<"svld3_vnum[_{2}]", "3Pcl", "csilUcUsUiUlhfdbm", MergeNon
 def SVLD4_VNUM : SInst<"svld4_vnum[_{2}]", "4Pcl", "csilUcUsUiUlhfdbm", MergeNone, "aarch64_sve_ld4_sret", [IsStructLoad, VerifyRuntimeMode]>;
 
 // Load one octoword and replicate (scalar base)
-let SVETargetGuard = "sve,f64mm", SMETargetGuard = InvalidMode in {
+let SVETargetGuard = "f64mm", SMETargetGuard = InvalidMode in {
   def SVLD1RO : SInst<"svld1ro[_{2}]", "dPc", "csilUcUsUiUlhfdbm", MergeNone, "aarch64_sve_ld1ro">;
 }
 
-let SVETargetGuard = "sve,bf16", SMETargetGuard = InvalidMode in {
+let SVETargetGuard = "bf16", SMETargetGuard = InvalidMode in {
   def SVBFMMLA       : SInst<"svbfmmla[_{0}]",       "MMdd",  "b", MergeNone, "aarch64_sve_bfmmla",       [IsOverloadNone]>;
 }
 
-let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
+let SVETargetGuard = "bf16", SMETargetGuard = "bf16" in {
   def SVBFDOT        : SInst<"svbfdot[_{0}]",        "MMdd",  "b", MergeNone, "aarch64_sve_bfdot",           [IsOverloadNone, VerifyRuntimeMode]>;
   def SVBFMLALB      : SInst<"svbfmlalb[_{0}]",      "MMdd",  "b", MergeNone, "aarch64_sve_bfmlalb",         [IsOverloadNone, VerifyRuntimeMode]>;
   def SVBFMLALT      : SInst<"svbfmlalt[_{0}]",      "MMdd",  "b", MergeNone, "aarch64_sve_bfmlalt",         [IsOverloadNone, VerifyRuntimeMode]>;
@@ -326,7 +326,7 @@ def SVST1H_VNUM_U : MInst<"svst1h_vnum[_{d}]", "vPFld", "UiUl",              [Is
 def SVST1W_VNUM_S : MInst<"svst1w_vnum[_{d}]", "vPCld", "l",                 [IsStore, VerifyRuntimeMode], MemEltTyInt32,   "aarch64_sve_st1">;
 def SVST1W_VNUM_U : MInst<"svst1w_vnum[_{d}]", "vPGld", "Ul",                [IsStore, VerifyRuntimeMode], MemEltTyInt32,   "aarch64_sve_st1">;
 
-let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
+let SMETargetGuard = InvalidMode in {
 // Store one vector (vector base)
 def SVST1_SCATTER_BASES_U     : MInst<"svst1_scatter[_{2}base_{d}]",  "vPud",  "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">;
 def SVST1B_SCATTER_BASES_U    : MInst<"svst1b_scatter[_{2}base_{d}]", "vPud",  "ilUiUl",   [IsScatterStore], MemEltTyInt8,    "aarch64_sve_st1_scatter_scalar_offset">;
@@ -464,7 +464,7 @@ def SVPRFH_VNUM : MInst<"svprfh_vnum", "vPQlJ", "s", [IsPrefetch, VerifyRuntimeM
 def SVPRFW_VNUM : MInst<"svprfw_vnum", "vPQlJ", "i", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_prf">;
 def SVPRFD_VNUM : MInst<"svprfd_vnum", "vPQlJ", "l", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt64, "aarch64_sve_prf">;
 
-let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
+let SMETargetGuard = InvalidMode in {
 // Prefetch (Vector bases)
 def SVPRFB_GATHER_BASES : MInst<"svprfb_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt8,  "aarch64_sve_prfb_gather_scalar_offset">;
 def SVPRFH_GATHER_BASES : MInst<"svprfh_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_scalar_offset">;
@@ -502,7 +502,7 @@ def SVPRFD_GATHER_BASES_OFFSET : MInst<"svprfd_gather[_{2}base]_index",  "vPdlJ"
 ////////////////////////////////////////////////////////////////////////////////
 // Address calculations
 
-let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
+let SMETargetGuard = InvalidMode in {
 def SVADRB : SInst<"svadrb[_{0}base]_[{2}]offset", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrb">;
 def SVADRH : SInst<"svadrh[_{0}base]_[{2}]index",  "uud", "ilUiUl", MergeNone, "aarch64_sve_adrh">;
 def SVADRW : SInst<"svadrw[_{0}base]_[{2}]index",  "uud", "ilUiUl", MergeNone, "aarch64_sve_adrw">;
@@ -778,11 +778,11 @@ defm SVRINTX : SInstZPZ<"svrintx", "hfd", "aarch64_sve_frintx">;
 defm SVRINTZ : SInstZPZ<"svrintz", "hfd", "aarch64_sve_frintz">;
 defm SVSQRT  : SInstZPZ<"svsqrt",  "hfd", "aarch64_sve_fsqrt">;
 
-let SVETargetGuard = "sve", SMETargetGuard = "sme2,ssve-fexpa" in {
+let SMETargetGuard = "sme2,ssve-fexpa" in {
 def SVEXPA  : SInst<"svexpa[_{d}]",  "du",   "hfd", MergeNone, "aarch64_sve_fexpa_x", [VerifyRuntimeMode]>;
 }
 
-let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
+let SMETargetGuard = InvalidMode in {
 def SVTMAD  : SInst<"svtmad[_{d}]",  "dddi", "hfd", MergeNone, "aarch64_sve_ftmad_x", [], [ImmCheck<2, ImmCheck0_7>]>;
 def SVTSMUL : SInst<"svtsmul[_{d}]", "ddu",  "hfd", MergeNone, "aarch64_sve_ftsmul_x">;
 def SVTSSEL : SInst<"svtssel[_{d}]", "ddu",  "hfd", MergeNone, "aarch64_sve_ftssel_x">;
@@ -825,7 +825,7 @@ def SVRSQRTS : SInst<"svrsqrts[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frs
 ////////////////////////////////////////////////////////////////////////////////
 // Floating-point reductions
 
-let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
+let SMETargetGuard = InvalidMode in {
 def SVFADDA : SInst<"svadda[_{d}]",   "sPsd", "hfd", MergeNone, "aarch64_sve_fadda">;
 }
 
@@ -946,14 +946,14 @@ defm SVFCVT_F32_F64   : SInstCvtMXZ<"svcvt_f32[_f64]", "MMPd", "MPd", "d", "aarc
 defm SVFCVT_F64_F16   : SInstCvtMXZ<"svcvt_f64[_f16]", "ddPO", "dPO", "d", "aarch64_sve_fcvt_f64f16">;
 defm SVFCVT_F64_F32   : SInstCvtMXZ<"svcvt_f64[_f32]", "ddPM", "dPM", "d", "aarch64_sve_fcvt_f64f32">;
 
-let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in {
+let SVETargetGuard = "bf16", SMETargetGuard = "bf16" in {
 defm SVCVT_BF16_F32    : SInstCvtMXZ<"svcvt_bf16[_f32]", "$$Pd", "$Pd", "f", "aarch64_sve_fcvt_bf16f32_v2">;
 
 def SVCVTNT_BF16_F32   : SInst<"svcvtnt_bf16[_f32]", "$$Pd", "f", MergeOp1, "aarch64_sve_fcvtnt_bf16f32_v2", [IsOverloadNone, VerifyRuntimeMode]>;
 //  SVCVTNT_X_BF16_F32 : Implemented as macro by SveEmitter.cpp
 }
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 defm SVCVTLT_F32_F16 : SInstCvtMX<"svcvtlt_f32[_f16]",  "ddPh", "dPh", "f", "aarch64_sve_fcvtlt_f32f16">;
 defm SVCVTLT_F64_F32 : SInstCvtMX<"svcvtlt_f64[_f32]",  "ddPh", "dPh", "d", "aarch64_sve_fcvtlt_f64f32">;
 
@@ -980,8 +980,8 @@ defm SVCLASTA_N : SVEPerm<"svclasta[_n_{d}]", "sPsd", "aarch64_sve_clasta_n">;
 defm SVCLASTB   : SVEPerm<"svclastb[_{d}]",   "dPdd", "aarch64_sve_clastb">;
 defm SVCLASTB_N : SVEPerm<"svclastb[_n_{d}]", "sPsd", "aarch64_sve_clastb_n">;
 
-let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
-def SVCOMPACT : SInst<"svcompact[_{d}]",   "dPd",  "ilUiUlfd", MergeNone, "aarch64_sve_compact">;
+let SMETargetGuard = "sme2p2" in {
+def SVCOMPACT : SInst<"svcompact[_{d}]", "dPd",  "ilUiUlfd", MergeNone, "aarch64_sve_compact", [VerifyRuntimeMode]>;
 }
 
 // Note: svdup_lane is implemented using the intrinsic for TBL to represent a
@@ -1088,7 +1088,7 @@ def SVPTEST_LAST  : SInst<"svptest_last",  "sPP", "Pc", MergeNone, "aarch64_sve_
 ////////////////////////////////////////////////////////////////////////////////
 // FFR manipulation
 
-let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in {
+let SMETargetGuard = InvalidMode in {
 def SVRDFFR   : SInst<"svrdffr",   "Pv", "Pc", MergeNone, "", [IsOverloadNone]>;
 def SVRDFFR_Z : SInst<"svrdffr_z", "PP", "Pc", MergeNone, "", [IsOverloadNone]>;
 def SVSETFFR  : SInst<"svsetffr",  "vv", "",   MergeNone, "", [IsOverloadNone]>;
@@ -1173,13 +1173,13 @@ def SVQINCP_N_S64 : SInst<"svqincp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "
 def SVQINCP_N_U32 : SInst<"svqincp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n32", [VerifyRuntimeMode]>;
 def SVQINCP_N_U64 : SInst<"svqincp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n64", [VerifyRuntimeMode]>;
 
-let SVETargetGuard = "sve,i8mm", SMETargetGuard = InvalidMode in {
+let SVETargetGuard = "i8mm", SMETargetGuard = InvalidMode in {
 def SVMLLA_S32   : SInst<"svmmla[_s32]",   "ddqq","i",  MergeNone, "aarch64_sve_smmla">;
 def SVMLLA_U32   : SInst<"svmmla[_u32]",   "ddqq","Ui", MergeNone, "aarch64_sve_ummla">;
 def SVUSMLLA_S32 : SInst<"svusmmla[_s32]", "ddbq","i",  MergeNone, "aarch64_sve_usmmla">;
 }
 
-let SVETargetGuard = "sve,i8mm", SMETargetGuard = "sme,i8mm"in {
+let SVETargetGuard = "i8mm", SMETargetGuard = "i8mm"in {
 def SVUSDOT_S   : SInst<"svusdot[_s32]",   "ddbq", "i", MergeNone, "aarch64_sve_usdot", [VerifyRuntimeMode]>;
 def SVUSDOT_N_S : SInst<"svusdot[_n_s32]", "ddbr", "i", MergeNone, "aarch64_sve_usdot", [VerifyRuntimeMode]>;
 def SVSUDOT_S   : SInst<"svsudot[_s32]",   "ddqb", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT, VerifyRuntimeMode]>;
@@ -1189,11 +1189,11 @@ def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]", "ddbqi",  "i", MergeNone, "aarc
 def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]", "ddqbi",  "i", MergeNone, "aarch64_sve_sudot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
 }
 
-let SVETargetGuard = "sve,f32mm", SMETargetGuard = InvalidMode in {
+let SVETargetGuard = "f32mm", SMETargetGuard = InvalidMode in {
 def SVMLLA_F32 : SInst<"svmmla[_f32]", "dddd","f", MergeNone, "aarch64_sve_fmmla">;
 }
 
-let SVETargetGuard = "sve,f64mm", SMETargetGuard = InvalidMode in {
+let SVETargetGuard = "f64mm", SMETargetGuard = InvalidMode in {
 def SVMLLA_F64 : SInst<"svmmla[_f64]",  "dddd", "d", MergeNone, "aarch64_sve_fmmla">;
 
 def SVTRN1Q : SInst<"svtrn1q[_{d}]", "ddd", "csilUcUsUiUlhfdb", MergeNone, "aarch64_sve_trn1q">;
@@ -1243,7 +1243,7 @@ let SVETargetGuard = "sve2p1|sme2", SMETargetGuard = "sve2p1|sme2" in {
 
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 WhileGE/GT
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
 def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
 def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>;
@@ -1268,7 +1268,7 @@ let SVETargetGuard = "sve2p1|sme2", SMETargetGuard = "sve2p1|sme2" in {
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Uniform DSP operations
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 defm SVQADD_S  : SInstZPZZ<"svqadd",  "csli",     "aarch64_sve_sqadd",  "aarch64_sve_sqadd">;
 defm SVQADD_U  : SInstZPZZ<"svqadd",  "UcUsUiUl", "aarch64_sve_uqadd",  "aarch64_sve_uqadd">;
 defm SVHADD_S  : SInstZPZZ<"svhadd",  "csli",     "aarch64_sve_shadd",  "aarch64_sve_shadd">;
@@ -1303,7 +1303,7 @@ multiclass SInstZPZxZ<string name, string types, string pat_v, string pat_n, str
   def _N_Z : SInst<name # "[_n_{d}]", pat_n, types, MergeZero, intrinsic, flags>;
 }
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 defm SVQRSHL_S : SInstZPZxZ<"svqrshl", "csil",     "dPdx", "dPdK", "aarch64_sve_sqrshl", [VerifyRuntimeMode]>;
 defm SVQRSHL_U : SInstZPZxZ<"svqrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqrshl", [VerifyRuntimeMode]>;
 defm SVQSHL_S  : SInstZPZxZ<"svqshl",  "csil",     "dPdx", "dPdK", "aarch64_sve_sqshl", [VerifyRuntimeMode]>;
@@ -1357,7 +1357,7 @@ multiclass SInstPairwise<string name, string types, string intrinsic, list<FlagT
   def _X   : SInst<name # "[_{d}]", "dPdd", types, MergeAny, intrinsic, flags>;
 }
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 defm SVADDP   : SInstPairwise<"svaddp",   "csliUcUsUiUl", "aarch64_sve_addp", [VerifyRuntimeMode]>;
 defm SVADDP_F : SInstPairwise<"svaddp",   "hfd",          "aarch64_sve_faddp", [VerifyRuntimeMode]>;
 defm SVMAXNMP : SInstPairwise<"svmaxnmp", "hfd",          "aarch64_sve_fmaxnmp", [VerifyRuntimeMode]>;
@@ -1373,7 +1373,7 @@ defm SVMINP_U : SInstPairwise<"svminp",   "UcUsUiUl",     "aarch64_sve_uminp", [
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Widening pairwise arithmetic
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVADALP_S_M : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeOp1,  "aarch64_sve_sadalp", [VerifyRuntimeMode]>;
 def SVADALP_S_X : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeAny,  "aarch64_sve_sadalp", [VerifyRuntimeMode]>;
 def SVADALP_S_Z : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeZero, "aarch64_sve_sadalp", [VerifyRuntimeMode]>;
@@ -1387,7 +1387,7 @@ def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_s
 // SVE2 - Bitwise ternary logical instructions
 //
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVBCAX  : SInst<"svbcax[_{d}]",  "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax", [VerifyRuntimeMode]>;
 def SVBSL   : SInst<"svbsl[_{d}]",   "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl", [VerifyRuntimeMode]>;
 def SVBSL1N : SInst<"svbsl1n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n", [VerifyRuntimeMode]>;
@@ -1407,7 +1407,7 @@ def SVXAR_N   : SInst<"svxar[_n_{d}]",   "dddi", "csilUcUsUiUl", MergeNone, "aar
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Large integer arithmetic
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVADCLB : SInst<"svadclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclb", [VerifyRuntimeMode]>;
 def SVADCLT : SInst<"svadclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclt", [VerifyRuntimeMode]>;
 def SVSBCLB : SInst<"svsbclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclb", [VerifyRuntimeMode]>;
@@ -1422,7 +1422,7 @@ def SVSBCLT_N : SInst<"svsbclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Multiplication by indexed elements
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVMLA_LANE_2 : SInst<"svmla_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mla_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
 def SVMLS_LANE_2 : SInst<"svmls_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mls_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
 def SVMUL_LANE_2 : SInst<"svmul_lane[_{d}]", "dddi",  "silUsUiUl", MergeNone, "aarch64_sve_mul_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>;
@@ -1430,7 +1430,7 @@ def SVMUL_LANE_2 : SInst<"svmul_lane[_{d}]", "dddi",  "silUsUiUl", MergeNone, "a
 
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Uniform complex integer arithmetic
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVCADD             : SInst<"svcadd[_{d}]",          "dddi",   "csilUcUsUiUl", MergeNone, "aarch64_sve_cadd_x",           [VerifyRuntimeMode], [ImmCheck<2, ImmCheckComplexRot90_270>]>;
 def SVSQCADD           : SInst<"svqcadd[_{d}]",         "dddi",   "csil",         MergeNone, "aarch64_sve_sqcadd_x",         [VerifyRuntimeMode], [ImmCheck<2, ImmCheckComplexRot90_270>]>;
 def SVCMLA             : SInst<"svcmla[_{d}]",          "ddddi",  "csilUcUsUiUl", MergeNone, "aarch64_sve_cmla_x",           [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRotAll90>]>;
@@ -1457,7 +1457,7 @@ multiclass SInstWideDSPWide<string name, string types, string intrinsic> {
   def _N : SInst<name # "[_n_{d}]", "ddR", types, MergeNone, intrinsic, [VerifyRuntimeMode]>;
 }
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 defm SVABALB_S : SInstWideDSPAcc<"svabalb",   "sil",    "aarch64_sve_sabalb">;
 defm SVABALB_U : SInstWideDSPAcc<"svabalb",   "UsUiUl", "aarch64_sve_uabalb">;
 defm SVABALT_S : SInstWideDSPAcc<"svabalt",   "sil",    "aarch64_sve_sabalt">;
@@ -1536,7 +1536,7 @@ def SVQDMULLT_LANE : SInst<"svqdmullt_lane[_{d}]", "dhhi",  "il",   MergeNone, "
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Narrowing DSP operations
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVADDHNB  : SInst<"svaddhnb[_{d}]",     "hdd",  "silUsUiUl", MergeNone, "aarch64_sve_addhnb", [VerifyRuntimeMode]>;
 def SVADDHNT  : SInst<"svaddhnt[_{d}]",     "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnt", [VerifyRuntimeMode]>;
 def SVRADDHNB : SInst<"svraddhnb[_{d}]",    "hdd",  "silUsUiUl", MergeNone, "aarch64_sve_raddhnb", [VerifyRuntimeMode]>;
@@ -1576,7 +1576,7 @@ def SVQRSHRNT_U : SInst<"svqrshrnt[_n_{d}]",  "hhdi", "UsUiUl",    MergeNone, "a
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Unary narrowing operations
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVQXTNB_S  : SInst<"svqxtnb[_{d}]",  "hd",  "sil",    MergeNone, "aarch64_sve_sqxtnb", [VerifyRuntimeMode]>;
 def SVQXTNB_U  : SInst<"svqxtnb[_{d}]",  "hd",  "UsUiUl", MergeNone, "aarch64_sve_uqxtnb", [VerifyRuntimeMode]>;
 def SVQXTUNB_S : SInst<"svqxtunb[_{d}]", "ed",  "sil",    MergeNone, "aarch64_sve_sqxtunb", [VerifyRuntimeMode]>;
@@ -1589,7 +1589,7 @@ def SVQXTUNT_S : SInst<"svqxtunt[_{d}]", "eed", "sil",    MergeNone, "aarch64_sv
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Widening complex integer arithmetic
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 defm SVADDLBT : SInstWideDSPLong<"svaddlbt", "sil", "aarch64_sve_saddlbt">;
 defm SVSUBLBT : SInstWideDSPLong<"svsublbt", "sil", "aarch64_sve_ssublbt">;
 defm SVSUBLTB : SInstWideDSPLong<"svsubltb", "sil", "aarch64_sve_ssubltb">;
@@ -1723,7 +1723,7 @@ def SVSTNT1W_SCATTER_INDEX_S : MInst<"svstnt1w_scatter[_{2}base]_index[_{d}]", "
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Polynomial arithmetic
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVEORBT         : SInst<"sveorbt[_{d}]",         "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [VerifyRuntimeMode]>;
 def SVEORBT_N       : SInst<"sveorbt[_n_{d}]",       "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [VerifyRuntimeMode]>;
 def SVEORTB         : SInst<"sveortb[_{d}]",         "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb", [VerifyRuntimeMode]>;
@@ -1744,7 +1744,7 @@ def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda",  "UcUi", MergeNone,
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Complex integer dot product
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVCDOT      : SInst<"svcdot[_{d}]",      "ddqqi",  "il",   MergeNone, "aarch64_sve_cdot",      [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRotAll90>]>;
 def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il",   MergeNone, "aarch64_sve_cdot_lane", [VerifyRuntimeMode], [ImmCheck<4, ImmCheckComplexRotAll90>, ImmCheck<3, ImmCheckLaneIndexDot, 2>]>;
 }
@@ -1752,7 +1752,7 @@ def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il",   MergeNone, "aarch
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Floating-point widening multiply-accumulate
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVMLALB_F      : SInst<"svmlalb[_{d}]",      "ddhh",  "f", MergeNone, "aarch64_sve_fmlalb", [VerifyRuntimeMode]>;
 def SVMLALB_F_N    : SInst<"svmlalb[_n_{d}]",    "ddhR",  "f", MergeNone, "aarch64_sve_fmlalb", [VerifyRuntimeMode]>;
 def SVMLALB_F_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>;
@@ -1770,7 +1770,7 @@ def SVMLSLT_F_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Floating-point integer binary logarithm
 
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVLOGB_M : SInst<"svlogb[_{d}]", "xxPd", "hfd", MergeOp1,     "aarch64_sve_flogb", [VerifyRuntimeMode]>;
 def SVLOGB_X : SInst<"svlogb[_{d}]", "xPd",  "hfd", MergeAnyExp,  "aarch64_sve_flogb", [VerifyRuntimeMode]>;
 def SVLOGB_Z : SInst<"svlogb[_{d}]", "xPd",  "hfd", MergeZeroExp, "aarch64_sve_flogb", [VerifyRuntimeMode]>;
@@ -1794,7 +1794,7 @@ def SVNMATCH : SInst<"svnmatch[_{d}]", "PPdd", "csUcUs", MergeNone, "aarch64_sve
 
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Contiguous conflict detection
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVWHILERW_B : SInst<"svwhilerw[_{1}]", "Pcc", "cUc",  MergeNone, "aarch64_sve_whilerw_b", [IsOverloadWhileRW, VerifyRuntimeMode]>;
 def SVWHILERW_H : SInst<"svwhilerw[_{1}]", "Pcc", "sUshb", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW, VerifyRuntimeMode]>;
 def SVWHILERW_S : SInst<"svwhilerw[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilerw_s", [IsOverloadWhileRW, VerifyRuntimeMode]>;
@@ -1808,7 +1808,7 @@ def SVWHILEWR_D : SInst<"svwhilewr[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sv
 
 ////////////////////////////////////////////////////////////////////////////////
 // SVE2 - Extended table lookup/permute
-let SVETargetGuard = "sve2", SMETargetGuard = "sme" in {
+let SVETargetGuard = "sve2" in {
 def SVTBL2 : SInst<"svtbl2[_{d}]", "d2u",  "csilUcUsUiUlhfdb", MergeNone, "", [VerifyRuntimeMode]>;
 def SVTBX  : SInst<"svtbx[_{d}]",  "dddu", "csilUcUsUiUlhfdb", MergeNone, "aarch64_sve_tbx", [VerifyRuntimeMode]>;
 }
@@ -1850,7 +1850,7 @@ def SVSM4E    : SInst<"svsm4e[_{d}]",    "ddd", "Ui", MergeNone, "aarch64_sve_sm
 def SVSM4EKEY : SInst<"svsm4ekey[_{d}]", "ddd", "Ui", MergeNone, "aarch64_sve_sm4ekey", [IsOverloadNone]>;
 }
 
-let SVETargetGuard = "sve2,sve-bitperm", SMETargetGuard = "sme,ssve-bitperm" in {
+let SVETargetGuard = "sve2,sve-bitperm", SMETargetGuard = "ssve-bitperm" in {
 def SVBDEP   : SInst<"svbdep[_{d}]",   "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bdep_x", [VerifyRuntimeMode]>;
 def SVBDEP_N : SInst<"svbdep[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bdep_x", [VerifyRuntimeMode]>;
 def SVBEXT   : SInst<"svbext[_{d}]",   "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bext_x", [VerifyRuntimeMode]>;
@@ -1859,7 +1859,7 @@ def SVBGRP   : SInst<"svbgrp[_{d}]",   "ddd", "UcUsUiUl", MergeNone, "aarch64_sv
 def SVBGRP_N : SInst<"svbgrp[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x", [VerifyRuntimeMode]>;
 }
 
-let SVETargetGuard = "sve2p1|sme", SMETargetGuard = "sve2p1|sme" in {
+let SVETargetGuard = "sve2p1|sme" in {
 def SVPSEL_B : SInst<"svpsel_lane_b8",  "PPPm", "Pc", MergeNone, "", [VerifyRuntimeMode], []>;
 def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [VerifyRuntimeMode], []>;
 def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [VerifyRuntimeMode], []>;
@@ -1965,7 +1965,7 @@ def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "f",  MergeNone, "a
 def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [VerifyRuntimeMode], []>;
 }
 
-let SVETargetGuard = "sve2p1|sme", SMETargetGuard = "sve2p1|sme" in {
+let SVETargetGuard = "sve2p1|sme" in {
 def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil",     MergeNone, "aarch64_sve_sclamp", [VerifyRuntimeMode], []>;
 def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [VerifyRuntimeMode], []>;
 
@@ -2340,7 +2340,7 @@ let SVETargetGuard = "sve2,fp8", SMETargetGuard = "sme2,fp8" in {
   def SVFCVTNT : SInst<"svcvtnt_mf8[_f32_x2]", "~~2>", "f", MergeNone, "aarch64_sve_fp8_cvtnt", [VerifyRuntimeMode]>;
 }
 
-let SVETargetGuard = "sve2,fp8dot2", SMETargetGuard ="sme,ssve-fp8dot2" in {
+let SVETargetGuard = "sve2,fp8dot2", SMETargetGuard ="ssve-fp8dot2" in {
   // 8-bit floating-point dot product to half-precision (vectors)
   def SVFDOT_2WAY   :  SInst<"svdot[_f16_mf8]",   "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode]>;
   def SVFDOT_N_2WAY :  SInst<"svdot[_n_f16_mf8]", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode]>;
@@ -2349,7 +2349,7 @@ let SVETargetGuard = "sve2,fp8dot2", SMETargetGuard ="sme,ssve-fp8dot2" in {
   def SVFDOT_LANE_2WAY :  SInst<"svdot_lane[_f16_mf8]", "dd~~i>", "h", MergeNone, "aarch64_sve_fp8_fdot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>;
 }
 
-let SVETargetGuard = "sve2,fp8dot4", SMETargetGuard ="sme,ssve-fp8dot4" in {
+let SVETargetGuard = "sve2,fp8dot4", SMETargetGuard ="ssve-fp8dot4" in {
   // 8-bit floating-point dot product to single-precision (vectors)
   def SVFDOT_4WAY   : SInst<"svdot[_f32_mf8]",   "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode]>;
   def SVFDOT_N_4WAY : SInst<"svdot[_n_f32_mf8]", "dd~!>", "f", MergeNone, "aarch64_sve_fp8_fdot", [VerifyRuntimeMode]>;
@@ -2358,7 +2358,7 @@ let SVETargetGuard = "sve2,fp8dot4", SMETargetGuard ="sme,ssve-fp8dot4" in {
   def SVFDOT_LANE_4WAY :  SInst<"svdot_lane[_f32_mf8]", "dd~~i>", "f", MergeNone, "aarch64_sve_fp8_fdot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>;
 }
 
-let SVETargetGuard = "sve2,fp8fma", SMETargetGuard = "sme,ssve-fp8fma" in {
+let SVETargetGuard = "sve2,fp8fma", SMETargetGuard = "ssve-fp8fma" in {
   // 8-bit floating-point multiply-add long to half-precision (bottom)
   def SVFMLALB   : SInst<"svmlalb[_f16_mf8]",   "dd~~>", "h", MergeNone, "aarch64_sve_fp8_fmlalb", [VerifyRuntimeMode]>;
   def SVFMLALB_N : SInst<"svmlalb[_n_f16_mf8]", "dd~!>", "h", MergeNone, "aarch64_sve_fp8_fmlalb", [VerifyRuntimeMode]>;
diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td
index 275bb2b9924dd..c1de2bfe4243d 100644
--- a/clang/include/clang/Basic/riscv_vector.td
+++ b/clang/include/clang/Basic/riscv_vector.td
@@ -721,8 +721,6 @@ multiclass RVVUnitStridedSegLoadTuple<string op> {
             NF = nf,
             ManualCodegen = [{
     {
-      SmallVector<llvm::Value*, 6> Operands;
-
       bool NoPassthru =
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) |
         (!IsMasked && (PolicyAttrs & RVV_VTA));
@@ -733,24 +731,18 @@ multiclass RVVUnitStridedSegLoadTuple<string op> {
       else
         IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops.back()->getType()};
 
-      if (NoPassthru) { // Push poison into passthru
-        Operands.push_back(llvm::PoisonValue::get(ResultType));
-      } else { // Push intrinsics operands into passthru
-        llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0];
-        Operands.push_back(PassthruOperand);
-      }
-
-      Operands.push_back(Ops[Offset]); // Ptr
       if (IsMasked)
-        Operands.push_back(Ops[0]);
-      Operands.push_back(Ops[Offset + 1]); // VL
+        std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+      if (NoPassthru)
+        Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+
       if (IsMasked)
-        Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
+        Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
 
       llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
 
-      llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
+      llvm::Value *LoadValue = Builder.CreateCall(F, Ops, "");
       if (ReturnValue.isNull())
         return LoadValue;
       else
@@ -787,26 +779,24 @@ multiclass RVVUnitStridedSegStoreTuple<string op> {
     {
       // Masked
       // Builtin: (mask, ptr, v_tuple, vl)
-      // Intrinsic: (tuple, ptr, mask, vl)
+      // Intrinsic: (tuple, ptr, mask, vl, SegInstSEW)
       // Unmasked
       // Builtin: (ptr, v_tuple, vl)
-      // Intrinsic: (tuple, ptr, vl)
-      unsigned Offset = IsMasked ? 1 : 0;
+      // Intrinsic: (tuple, ptr, vl, SegInstSEW)
 
-      SmallVector<llvm::Value*, 5> Operands;
-      Operands.push_back(Ops[Offset + 1]); // tuple
-      Operands.push_back(Ops[Offset]); // Ptr
       if (IsMasked)
-        Operands.push_back(Ops[0]);
-      Operands.push_back(Ops[Offset + 2]); // VL
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
+        std::swap(Ops[0], Ops[2]);
+      else
+        std::swap(Ops[0], Ops[1]);
+
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
 
       if (IsMasked)
-        IntrinsicTypes = {Operands[0]->getType(), Ops[Offset]->getType(), Ops[0]->getType(), Operands.back()->getType()};
+        IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(), Ops[3]->getType()};
       else
-        IntrinsicTypes = {Operands[0]->getType(), Ops[Offset]->getType(), Operands.back()->getType()};
-      llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-      return Builder.CreateCall(F, Operands, "");
+        IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType()};
+
+      break;
    }
       }] in {
         defvar T = "(Tuple:" # nf # ")";
@@ -836,8 +826,6 @@ multiclass RVVUnitStridedSegLoadFFTuple<string op> {
             NF = nf,
             ManualCodegen = [{
     {
-      SmallVector<llvm::Value*, 6> Operands;
-
       bool NoPassthru =
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) |
         (!IsMasked && (PolicyAttrs & RVV_VTA));
@@ -848,24 +836,21 @@ multiclass RVVUnitStridedSegLoadFFTuple<string op> {
       else
         IntrinsicTypes = {ResultType, Ops.back()->getType(), Ops[Offset]->getType()};
 
-      if (NoPassthru) { // Push poison into passthru
-        Operands.push_back(llvm::PoisonValue::get(ResultType));
-      } else { // Push intrinsics operands into passthru
-        llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0];
-        Operands.push_back(PassthruOperand);
-      }
-
-      Operands.push_back(Ops[Offset]); // Ptr
       if (IsMasked)
-        Operands.push_back(Ops[0]);
-      Operands.push_back(Ops[Offset + 2]); // vl
+        std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+      if (NoPassthru)
+        Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+
       if (IsMasked)
-        Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
+        Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
+
+      Value *NewVL = Ops[2];
+      Ops.erase(Ops.begin() + 2);
 
       llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
 
-      llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
+      llvm::Value *LoadValue = Builder.CreateCall(F, Ops, "");
       // Get alignment from the new vl operand
       clang::CharUnits Align =
           CGM.getNaturalPointeeTypeAlignment(E->getArg(Offset + 1)->getType());
@@ -874,7 +859,7 @@ multiclass RVVUnitStridedSegLoadFFTuple<string op> {
 
       // Store new_vl
       llvm::Value *V = Builder.CreateExtractValue(LoadValue, 1);
-      Builder.CreateStore(V, Address(Ops[Offset + 1], V->getType(), Align));
+      Builder.CreateStore(V, Address(NewVL, V->getType(), Align));
 
       if (ReturnValue.isNull())
         return ReturnTuple;
@@ -909,8 +894,6 @@ multiclass RVVStridedSegLoadTuple<string op> {
             NF = nf,
             ManualCodegen = [{
     {
-      SmallVector<llvm::Value*, 7> Operands;
-
       bool NoPassthru =
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) |
         (!IsMasked && (PolicyAttrs & RVV_VTA));
@@ -921,24 +904,17 @@ multiclass RVVStridedSegLoadTuple<string op> {
       else
         IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops.back()->getType()};
 
-      if (NoPassthru) { // Push poison into passthru
-        Operands.push_back(llvm::PoisonValue::get(ResultType));
-      } else { // Push intrinsics operands into passthru
-        llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0];
-        Operands.push_back(PassthruOperand);
-      }
-
-      Operands.push_back(Ops[Offset]); // Ptr
-      Operands.push_back(Ops[Offset + 1]); // Stride
       if (IsMasked)
-        Operands.push_back(Ops[0]);
-      Operands.push_back(Ops[Offset + 2]); // VL
+        std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+      if (NoPassthru)
+        Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+
       if (IsMasked)
-        Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
+        Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
 
       llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-      llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
+      llvm::Value *LoadValue = Builder.CreateCall(F, Ops, "");
 
       if (ReturnValue.isNull())
         return LoadValue;
@@ -977,27 +953,23 @@ multiclass RVVStridedSegStoreTuple<string op> {
     {
       // Masked
       // Builtin: (mask, ptr, stride, v_tuple, vl)
-      // Intrinsic: (tuple, ptr, stride, mask, vl)
+      // Intrinsic: (tuple, ptr, stride, mask, vl, SegInstSEW)
       // Unmasked
       // Builtin: (ptr, stride, v_tuple, vl)
-      // Intrinsic: (tuple, ptr, stride, vl)
-      unsigned Offset = IsMasked ? 1 : 0;
+      // Intrinsic: (tuple, ptr, stride, vl, SegInstSEW)
 
-      SmallVector<llvm::Value*, 6> Operands;
-      Operands.push_back(Ops[Offset + 2]); // tuple
-      Operands.push_back(Ops[Offset]); // Ptr
-      Operands.push_back(Ops[Offset + 1]); // Stride
       if (IsMasked)
-        Operands.push_back(Ops[0]);
-      Operands.push_back(Ops[Offset + 3]); // VL
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
+        std::swap(Ops[0], Ops[3]);
+      else
+        std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3);
+
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
 
       if (IsMasked)
-        IntrinsicTypes = {Operands[0]->getType(), Operands[1]->getType(), Operands.back()->getType(), Ops[0]->getType()};
+        IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[4]->getType(), Ops[3]->getType()};
       else
-        IntrinsicTypes = {Operands[0]->getType(), Operands[1]->getType(), Operands.back()->getType()};
-      llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-      return Builder.CreateCall(F, Operands, "");
+        IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[3]->getType()};
+      break;
     }
             }] in {
         defvar T = "(Tuple:" # nf # ")";
@@ -1022,40 +994,30 @@ multiclass RVVIndexedSegLoadTuple<string op> {
             NF = nf,
             ManualCodegen = [{
     {
-      SmallVector<llvm::Value*, 7> Operands;
-
       bool NoPassthru =
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) |
         (!IsMasked && (PolicyAttrs & RVV_VTA));
-      unsigned Offset = IsMasked ? NoPassthru ? 1 : 2 : NoPassthru ? 0 : 1;
 
-      if (NoPassthru) { // Push poison into passthru
-        Operands.push_back(llvm::PoisonValue::get(ResultType));
-      } else { // Push intrinsics operands into passthru
-        llvm::Value *PassthruOperand = IsMasked ? Ops[1] : Ops[0];
-        Operands.push_back(PassthruOperand);
-      }
-
-      Operands.push_back(Ops[Offset]); // Ptr
-      Operands.push_back(Ops[Offset + 1]); // Idx
       if (IsMasked)
-        Operands.push_back(Ops[0]);
-      Operands.push_back(Ops[Offset + 2]); // VL
+        std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);
+      if (NoPassthru)
+        Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+
       if (IsMasked)
-        Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
+        Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
 
       if (IsMasked)
-        IntrinsicTypes = {ResultType, Ops[Offset]->getType(),
-                          Ops[Offset + 1]->getType(),
-                          Ops[0]->getType(),
-                          Ops.back()->getType()};
+        IntrinsicTypes = {ResultType, Ops[1]->getType(),
+                          Ops[2]->getType(),
+                          Ops[3]->getType(),
+                          Ops[4]->getType()};
       else
-        IntrinsicTypes = {ResultType, Ops[Offset]->getType(),
-                          Ops[Offset + 1]->getType(),
-                          Ops.back()->getType()};
+        IntrinsicTypes = {ResultType, Ops[1]->getType(),
+                          Ops[2]->getType(),
+                          Ops[3]->getType()};
       llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-      llvm::Value *LoadValue = Builder.CreateCall(F, Operands, "");
+      llvm::Value *LoadValue = Builder.CreateCall(F, Ops, "");
 
       if (ReturnValue.isNull())
         return LoadValue;
@@ -1090,30 +1052,25 @@ multiclass RVVIndexedSegStoreTuple<string op> {
     {
       // Masked
       // Builtin: (mask, ptr, index, v_tuple, vl)
-      // Intrinsic: (tuple, ptr, index, mask, vl)
+      // Intrinsic: (tuple, ptr, index, mask, vl, SegInstSEW)
       // Unmasked
       // Builtin: (ptr, index, v_tuple, vl)
-      // Intrinsic: (tuple, ptr, index, vl)
-      unsigned Offset = IsMasked ? 1 : 0;
+      // Intrinsic: (tuple, ptr, index, vl, SegInstSEW)
 
-      SmallVector<llvm::Value*, 6> Operands;
-      Operands.push_back(Ops[Offset + 2]); // tuple
-      Operands.push_back(Ops[Offset]); // Ptr
-      Operands.push_back(Ops[Offset + 1]); // Idx
       if (IsMasked)
-        Operands.push_back(Ops[0]);
-      Operands.push_back(Ops[Offset + 3]); // VL
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
+        std::swap(Ops[0], Ops[3]);
+      else
+        std::rotate(Ops.begin(), Ops.begin() + 2, Ops.begin() + 3);
+
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), SegInstSEW));
 
       if (IsMasked)
-        IntrinsicTypes = {Operands[0]->getType(), Ops[Offset]->getType(), Ops[Offset + 1]->getType(),
-                          Ops[0]->getType(),
-                          Operands.back()->getType()};
+        IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(),
+                          Ops[3]->getType(), Ops[4]->getType()};
       else
-        IntrinsicTypes = {Operands[0]->getType(), Ops[Offset]->getType(), Ops[Offset + 1]->getType(),
-                          Operands.back()->getType()};
-      llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-      return Builder.CreateCall(F, Operands, "");
+        IntrinsicTypes = {Ops[0]->getType(), Ops[1]->getType(), Ops[2]->getType(),
+                          Ops[3]->getType()};
+      break;
     }
             }] in {
           defvar T = "(Tuple:" # nf # ")";
@@ -1359,33 +1316,21 @@ let ManualCodegen = [{
     // Unmasked: (passthru, op0, op1, round_mode, vl)
     // Masked:   (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy)
 
-    SmallVector<llvm::Value*, 7> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
-    unsigned Offset = IsMasked ?
-        (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0);
-
-    if (!HasMaskedOff)
-      Operands.push_back(llvm::PoisonValue::get(ResultType));
-    else
-      Operands.push_back(Ops[IsMasked ? 1 : 0]);
-
-    Operands.push_back(Ops[Offset]); // op0
-    Operands.push_back(Ops[Offset + 1]); // op1
 
     if (IsMasked)
-      Operands.push_back(Ops[0]); // mask
+      std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
 
-    Operands.push_back(Ops[Offset + 2]); // vxrm
-    Operands.push_back(Ops[Offset + 3]); // vl
+    if (!HasMaskedOff)
+      Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
 
     if (IsMasked)
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
 
-    IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(), Ops.back()->getType()};
-    llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-    return Builder.CreateCall(F, Operands, "");
+    IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()};
+    break;
   }
 }] in {
   // 12.2. Vector Single-Width Averaging Add and Subtract
@@ -1408,34 +1353,22 @@ let ManualCodegen = [{
     // Unmasked: (passthru, op0, op1, round_mode, vl)
     // Masked:   (passthru, vector_in, vector_in/scalar_in, mask, vxrm, vl, policy)
 
-    SmallVector<llvm::Value*, 7> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
-    unsigned Offset = IsMasked ?
-        (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0);
-
-    if (!HasMaskedOff)
-      Operands.push_back(llvm::PoisonValue::get(ResultType));
-    else
-      Operands.push_back(Ops[IsMasked ? 1 : 0]);
-
-    Operands.push_back(Ops[Offset]); // op0
-    Operands.push_back(Ops[Offset + 1]); // op1
 
     if (IsMasked)
-      Operands.push_back(Ops[0]); // mask
+      std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
 
-    Operands.push_back(Ops[Offset + 2]); // vxrm
-    Operands.push_back(Ops[Offset + 3]); // vl
+    if (!HasMaskedOff)
+      Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
 
     if (IsMasked)
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
 
-    IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(),
+    IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType(),
                       Ops.back()->getType()};
-    llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-    return Builder.CreateCall(F, Operands, "");
+    break;
   }
 }] in {
   // 12.5. Vector Narrowing Fixed-Point Clip Instructions
@@ -1463,7 +1396,6 @@ let ManualCodegen = [{
     // Unmasked: (passthru, op0, op1, round_mode, vl)
     // Masked:   (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy)
 
-    SmallVector<llvm::Value*, 7> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
@@ -1471,35 +1403,20 @@ let ManualCodegen = [{
       (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) :
       (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4);
 
-    unsigned Offset = IsMasked ?
-        (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0);
-
-    if (!HasMaskedOff)
-      Operands.push_back(llvm::PoisonValue::get(ResultType));
-    else
-      Operands.push_back(Ops[IsMasked ? 1 : 0]);
-
-    Operands.push_back(Ops[Offset]); // op0
-    Operands.push_back(Ops[Offset + 1]); // op1
+    if (!HasRoundModeOp)
+      Ops.insert(Ops.end() - 1, ConstantInt::get(Ops.back()->getType(), 7)); // frm
 
     if (IsMasked)
-      Operands.push_back(Ops[0]); // mask
-
-    if (HasRoundModeOp) {
-      Operands.push_back(Ops[Offset + 2]); // frm
-      Operands.push_back(Ops[Offset + 3]); // vl
-    } else {
-      Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm
-      Operands.push_back(Ops[Offset + 2]); // vl
-    }
+      std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+
+    if (!HasMaskedOff)
+      Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
 
     if (IsMasked)
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
 
-    IntrinsicTypes = {ResultType, Ops[Offset + 1]->getType(),
-                      Operands.back()->getType()};
-    llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-    return Builder.CreateCall(F, Operands, "");
+    IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops.back()->getType()};
+    break;
   }
 }] in {
   let HasFRMRoundModeOp = true in {
@@ -1540,7 +1457,6 @@ let ManualCodegen = [{
     // Unmasked: (passthru, op0, op1, round_mode, vl)
     // Masked:   (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy)
 
-    SmallVector<llvm::Value*, 7> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
@@ -1548,35 +1464,21 @@ let ManualCodegen = [{
       (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) :
       (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4);
 
-    unsigned Offset = IsMasked ?
-        (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0);
-
-    if (!HasMaskedOff)
-      Operands.push_back(llvm::PoisonValue::get(ResultType));
-    else
-      Operands.push_back(Ops[IsMasked ? 1 : 0]);
-
-    Operands.push_back(Ops[Offset]); // op0
-    Operands.push_back(Ops[Offset + 1]); // op1
+    if (!HasRoundModeOp)
+      Ops.insert(Ops.end() - 1, ConstantInt::get(Ops.back()->getType(), 7)); // frm
 
     if (IsMasked)
-      Operands.push_back(Ops[0]); // mask
-
-    if (HasRoundModeOp) {
-      Operands.push_back(Ops[Offset + 2]); // frm
-      Operands.push_back(Ops[Offset + 3]); // vl
-    } else {
-      Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm
-      Operands.push_back(Ops[Offset + 2]); // vl
-    }
+      std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+
+    if (!HasMaskedOff)
+      Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
 
     if (IsMasked)
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
 
-    IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(),
+    IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType(),
                       Ops.back()->getType()};
-    llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-    return Builder.CreateCall(F, Operands, "");
+    break;
   }
 }] in {
   let HasFRMRoundModeOp = true in {
@@ -1619,38 +1521,23 @@ let UnMaskedPolicyScheme = HasPolicyOperand in {
 let ManualCodegen = [{
   {
     // LLVM intrinsic
-    // Unmasked: (passthru, op0, op1, round_mode, vl)
-    // Masked:   (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy)
+    // Unmasked: (vector_in, vector_in/scalar_in, vector_in, round_mode, vl, policy)
+    // Masked:   (vector_in, vector_in/scalar_in, vector_in, mask, frm, vl, policy)
 
-    SmallVector<llvm::Value*, 7> Operands;
     bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5;
 
-    unsigned Offset = IsMasked ? 2 : 1;
-
-    Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough
-
-    Operands.push_back(Ops[Offset]); // op0
-    Operands.push_back(Ops[Offset + 1]); // op1
+    if (!HasRoundModeOp)
+      Ops.insert(Ops.end() - 1, ConstantInt::get(Ops.back()->getType(), 7)); // frm
 
     if (IsMasked)
-      Operands.push_back(Ops[0]); // mask
-
-    if (HasRoundModeOp) {
-      Operands.push_back(Ops[Offset + 2]); // frm
-      Operands.push_back(Ops[Offset + 3]); // vl
-    } else {
-      Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm
-      Operands.push_back(Ops[Offset + 2]); // vl
-    }
-
-    Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
 
-    IntrinsicTypes = {ResultType, Ops[Offset]->getType(),
-                      Operands.back()->getType()};
+    Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
 
-    llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
+    IntrinsicTypes = {ResultType, Ops[1]->getType(),
+                      Ops.back()->getType()};
 
-    return Builder.CreateCall(F, Operands, "");
+    break;
   }
 }] in {
   let HasFRMRoundModeOp = 1 in {
@@ -1678,38 +1565,23 @@ let ManualCodegen = [{
 let ManualCodegen = [{
   {
     // LLVM intrinsic
-    // Unmasked: (passthru, op0, op1, round_mode, vl)
-    // Masked:   (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy)
+    // Unmasked: (vector_in, vector_in/scalar_in, vector_in, round_mode, vl, policy)
+    // Masked:   (vector_in, vector_in/scalar_in, vector_in, mask, frm, vl, policy)
 
-    SmallVector<llvm::Value*, 7> Operands;
     bool HasRoundModeOp = IsMasked ? Ops.size() == 6 : Ops.size() == 5;
 
-    unsigned Offset = IsMasked ? 2 : 1;
-
-    Operands.push_back(Ops[IsMasked ? 1 : 0]); // passthrough
-
-    Operands.push_back(Ops[Offset]); // op0
-    Operands.push_back(Ops[Offset + 1]); // op1
+    if (!HasRoundModeOp)
+      Ops.insert(Ops.end() - 1, ConstantInt::get(Ops.back()->getType(), 7)); // frm
 
     if (IsMasked)
-      Operands.push_back(Ops[0]); // mask
-
-    if (HasRoundModeOp) {
-      Operands.push_back(Ops[Offset + 2]); // frm
-      Operands.push_back(Ops[Offset + 3]); // vl
-    } else {
-      Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm
-      Operands.push_back(Ops[Offset + 2]); // vl
-    }
-
-    Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      std::rotate(Ops.begin(), Ops.begin() + 1, Ops.begin() + 4);
 
-    IntrinsicTypes = {ResultType, Ops[Offset]->getType(), Ops[Offset + 1]->getType(),
-                      Operands.back()->getType()};
+    Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
 
-    llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
+    IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops[2]->getType(),
+                      Ops.back()->getType()};
 
-    return Builder.CreateCall(F, Operands, "");
+    break;
   }
 }] in {
   let HasFRMRoundModeOp = 1 in {
@@ -1751,7 +1623,6 @@ let ManualCodegen = [{
     // Unmasked: (passthru, op0, round_mode, vl)
     // Masked:   (passthru, op0, mask, frm, vl, policy)
 
-    SmallVector<llvm::Value*, 6> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
@@ -1759,33 +1630,20 @@ let ManualCodegen = [{
       (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) :
       (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3);
 
-    unsigned Offset = IsMasked ?
-        (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0);
-
-    if (!HasMaskedOff)
-      Operands.push_back(llvm::PoisonValue::get(ResultType));
-    else
-      Operands.push_back(Ops[IsMasked ? 1 : 0]);
-
-    Operands.push_back(Ops[Offset]); // op0
+    if (!HasRoundModeOp)
+      Ops.insert(Ops.end() - 1, ConstantInt::get(Ops.back()->getType(), 7)); // frm
 
     if (IsMasked)
-      Operands.push_back(Ops[0]); // mask
-
-    if (HasRoundModeOp) {
-      Operands.push_back(Ops[Offset + 1]); // frm
-      Operands.push_back(Ops[Offset + 2]); // vl
-    } else {
-      Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm
-      Operands.push_back(Ops[Offset + 1]); // vl
-    }
+      std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+
+    if (!HasMaskedOff)
+      Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
 
     if (IsMasked)
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
 
-    IntrinsicTypes = {ResultType, Operands.back()->getType()};
-    llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-    return Builder.CreateCall(F, Operands, "");
+    IntrinsicTypes = {ResultType, Ops.back()->getType()};
+    break;
   }
 }] in {
   let HasFRMRoundModeOp = 1 in {
@@ -1950,7 +1808,6 @@ let ManualCodegen = [{
     // LLVM intrinsic
     // Unmasked: (passthru, op0, frm, vl)
     // Masked:   (passthru, op0, mask, frm, vl, policy)
-    SmallVector<llvm::Value*, 6> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
@@ -1958,34 +1815,20 @@ let ManualCodegen = [{
       (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4) :
       (HasMaskedOff ? Ops.size() == 4 : Ops.size() == 3);
 
-    unsigned Offset = IsMasked ?
-        (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0);
-
-    if (!HasMaskedOff)
-      Operands.push_back(llvm::PoisonValue::get(ResultType));
-    else
-      Operands.push_back(Ops[IsMasked ? 1 : 0]);
-
-    Operands.push_back(Ops[Offset]); // op0
+    if (!HasRoundModeOp)
+      Ops.insert(Ops.end() - 1, ConstantInt::get(Ops.back()->getType(), 7)); // frm
 
     if (IsMasked)
-      Operands.push_back(Ops[0]); // mask
-
-    if (HasRoundModeOp) {
-      Operands.push_back(Ops[Offset + 1]); // frm
-      Operands.push_back(Ops[Offset + 2]); // vl
-    } else {
-      Operands.push_back(ConstantInt::get(Ops[Offset + 1]->getType(), 7)); // frm
-      Operands.push_back(Ops[Offset + 1]); // vl
-    }
+      std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
+
+    if (!HasMaskedOff)
+      Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
 
     if (IsMasked)
-      Operands.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
+      Ops.push_back(ConstantInt::get(Ops.back()->getType(), PolicyAttrs));
 
-    IntrinsicTypes = {ResultType, Ops[Offset]->getType(),
-                      Operands.back()->getType()};
-    llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-    return Builder.CreateCall(F, Operands, "");
+    IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops.back()->getType()};
+    break;
   }
 }] in {
   let HasFRMRoundModeOp = 1 in {
@@ -2155,7 +1998,6 @@ let ManualCodegen = [{
     // Unmasked: (passthru, op0, op1, round_mode, vl)
     // Masked:   (passthru, vector_in, vector_in/scalar_in, mask, frm, vl, policy)
 
-    SmallVector<llvm::Value*, 6> Operands;
     bool HasMaskedOff = !(
         (IsMasked && (PolicyAttrs & RVV_VTA) && (PolicyAttrs & RVV_VMA)) ||
         (!IsMasked && PolicyAttrs & RVV_VTA));
@@ -2163,32 +2005,17 @@ let ManualCodegen = [{
       (HasMaskedOff ? Ops.size() == 6 : Ops.size() == 5) :
       (HasMaskedOff ? Ops.size() == 5 : Ops.size() == 4);
 
-    unsigned Offset = IsMasked ?
-        (HasMaskedOff ? 2 : 1) : (HasMaskedOff ? 1 : 0);
-
-    if (!HasMaskedOff)
-      Operands.push_back(llvm::PoisonValue::get(ResultType));
-    else
-      Operands.push_back(Ops[IsMasked ? 1 : 0]);
-
-    Operands.push_back(Ops[Offset]); // op0
-    Operands.push_back(Ops[Offset + 1]); // op1
+    if (!HasRoundModeOp)
+      Ops.insert(Ops.end() - 1, ConstantInt::get(Ops.back()->getType(), 7)); // frm
 
     if (IsMasked)
-      Operands.push_back(Ops[0]); // mask
-
-    if (HasRoundModeOp) {
-      Operands.push_back(Ops[Offset + 2]); // frm
-      Operands.push_back(Ops[Offset + 3]); // vl
-    } else {
-      Operands.push_back(ConstantInt::get(Ops[Offset + 2]->getType(), 7)); // frm
-      Operands.push_back(Ops[Offset + 2]); // vl
-    }
+      std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);
 
-    IntrinsicTypes = {ResultType, Ops[Offset]->getType(),
-                      Ops.back()->getType()};
-    llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes);
-    return Builder.CreateCall(F, Operands, "");
+    if (!HasMaskedOff)
+      Ops.insert(Ops.begin(), llvm::PoisonValue::get(ResultType));
+
+    IntrinsicTypes = {ResultType, Ops[1]->getType(), Ops.back()->getType()};
+    break;
   }
 }] in {
   let HasFRMRoundModeOp = 1 in {
diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index b26e5581014d5..d29e5687d2544 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -63,11 +63,11 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value getConstAPInt(mlir::Location loc, mlir::Type typ,
                             const llvm::APInt &val) {
-    return create<cir::ConstantOp>(loc, cir::IntAttr::get(typ, val));
+    return cir::ConstantOp::create(*this, loc, cir::IntAttr::get(typ, val));
   }
 
   cir::ConstantOp getConstant(mlir::Location loc, mlir::TypedAttr attr) {
-    return create<cir::ConstantOp>(loc, attr);
+    return cir::ConstantOp::create(*this, loc, attr);
   }
 
   cir::ConstantOp getConstantInt(mlir::Location loc, mlir::Type ty,
@@ -119,7 +119,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   }
 
   cir::ConstantOp getBool(bool state, mlir::Location loc) {
-    return create<cir::ConstantOp>(loc, getCIRBoolAttr(state));
+    return cir::ConstantOp::create(*this, loc, getCIRBoolAttr(state));
   }
   cir::ConstantOp getFalse(mlir::Location loc) { return getBool(false, loc); }
   cir::ConstantOp getTrue(mlir::Location loc) { return getBool(true, loc); }
@@ -144,21 +144,37 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   mlir::Value createComplexCreate(mlir::Location loc, mlir::Value real,
                                   mlir::Value imag) {
     auto resultComplexTy = cir::ComplexType::get(real.getType());
-    return create<cir::ComplexCreateOp>(loc, resultComplexTy, real, imag);
+    return cir::ComplexCreateOp::create(*this, loc, resultComplexTy, real,
+                                        imag);
   }
 
   mlir::Value createComplexReal(mlir::Location loc, mlir::Value operand) {
     auto operandTy = mlir::cast<cir::ComplexType>(operand.getType());
-    return create<cir::ComplexRealOp>(loc, operandTy.getElementType(), operand);
+    return cir::ComplexRealOp::create(*this, loc, operandTy.getElementType(),
+                                      operand);
   }
 
   mlir::Value createComplexImag(mlir::Location loc, mlir::Value operand) {
     auto operandTy = mlir::cast<cir::ComplexType>(operand.getType());
-    return create<cir::ComplexImagOp>(loc, operandTy.getElementType(), operand);
+    return cir::ComplexImagOp::create(*this, loc, operandTy.getElementType(),
+                                      operand);
+  }
+
+  cir::LoadOp createLoad(mlir::Location loc, mlir::Value ptr,
+                         uint64_t alignment = 0) {
+    mlir::IntegerAttr alignmentAttr = getAlignmentAttr(alignment);
+    assert(!cir::MissingFeatures::opLoadStoreVolatile());
+    return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false,
+                               alignmentAttr, cir::MemOrderAttr{});
+  }
+
+  mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr,
+                                uint64_t alignment) {
+    return createLoad(loc, ptr, alignment);
   }
 
   mlir::Value createNot(mlir::Value value) {
-    return create<cir::UnaryOp>(value.getLoc(), value.getType(),
+    return cir::UnaryOp::create(*this, value.getLoc(), value.getType(),
                                 cir::UnaryOpKind::Not, value);
   }
 
@@ -167,7 +183,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
       mlir::Location loc,
       llvm::function_ref<void(mlir::OpBuilder &, mlir::Location)> condBuilder,
       llvm::function_ref<void(mlir::OpBuilder &, mlir::Location)> bodyBuilder) {
-    return create<cir::DoWhileOp>(loc, condBuilder, bodyBuilder);
+    return cir::DoWhileOp::create(*this, loc, condBuilder, bodyBuilder);
   }
 
   /// Create a while operation.
@@ -175,7 +191,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
       mlir::Location loc,
       llvm::function_ref<void(mlir::OpBuilder &, mlir::Location)> condBuilder,
       llvm::function_ref<void(mlir::OpBuilder &, mlir::Location)> bodyBuilder) {
-    return create<cir::WhileOp>(loc, condBuilder, bodyBuilder);
+    return cir::WhileOp::create(*this, loc, condBuilder, bodyBuilder);
   }
 
   /// Create a for operation.
@@ -184,22 +200,23 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
       llvm::function_ref<void(mlir::OpBuilder &, mlir::Location)> condBuilder,
       llvm::function_ref<void(mlir::OpBuilder &, mlir::Location)> bodyBuilder,
       llvm::function_ref<void(mlir::OpBuilder &, mlir::Location)> stepBuilder) {
-    return create<cir::ForOp>(loc, condBuilder, bodyBuilder, stepBuilder);
+    return cir::ForOp::create(*this, loc, condBuilder, bodyBuilder,
+                              stepBuilder);
   }
 
   /// Create a break operation.
   cir::BreakOp createBreak(mlir::Location loc) {
-    return create<cir::BreakOp>(loc);
+    return cir::BreakOp::create(*this, loc);
   }
 
   /// Create a continue operation.
   cir::ContinueOp createContinue(mlir::Location loc) {
-    return create<cir::ContinueOp>(loc);
+    return cir::ContinueOp::create(*this, loc);
   }
 
   mlir::Value createUnaryOp(mlir::Location loc, cir::UnaryOpKind kind,
                             mlir::Value operand) {
-    return create<cir::UnaryOp>(loc, kind, operand);
+    return cir::UnaryOp::create(*this, loc, kind, operand);
   }
 
   mlir::TypedAttr getConstPtrAttr(mlir::Type type, int64_t value) {
@@ -209,13 +226,21 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   mlir::Value createAlloca(mlir::Location loc, cir::PointerType addrType,
                            mlir::Type type, llvm::StringRef name,
                            mlir::IntegerAttr alignment) {
-    return create<cir::AllocaOp>(loc, addrType, type, name, alignment);
+    return cir::AllocaOp::create(*this, loc, addrType, type, name, alignment);
+  }
+
+  /// Get constant address of a global variable as an MLIR attribute.
+  cir::GlobalViewAttr getGlobalViewAttr(cir::PointerType type,
+                                        cir::GlobalOp globalOp,
+                                        mlir::ArrayAttr indices = {}) {
+    auto symbol = mlir::FlatSymbolRefAttr::get(globalOp.getSymNameAttr());
+    return cir::GlobalViewAttr::get(type, symbol, indices);
   }
 
   mlir::Value createGetGlobal(mlir::Location loc, cir::GlobalOp global) {
     assert(!cir::MissingFeatures::addressSpace());
-    return create<cir::GetGlobalOp>(loc, getPointerTo(global.getSymType()),
-                                    global.getSymName());
+    return cir::GetGlobalOp::create(
+        *this, loc, getPointerTo(global.getSymType()), global.getSymName());
   }
 
   mlir::Value createGetGlobal(cir::GlobalOp global) {
@@ -223,36 +248,39 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   }
 
   cir::StoreOp createStore(mlir::Location loc, mlir::Value val, mlir::Value dst,
-                           mlir::IntegerAttr align = {}) {
-    return create<cir::StoreOp>(loc, val, dst, align);
+                           bool isVolatile = false,
+                           mlir::IntegerAttr align = {},
+                           cir::MemOrderAttr order = {}) {
+    return cir::StoreOp::create(*this, loc, val, dst, align, order);
   }
 
   [[nodiscard]] cir::GlobalOp createGlobal(mlir::ModuleOp mlirModule,
                                            mlir::Location loc,
                                            mlir::StringRef name,
-                                           mlir::Type type,
+                                           mlir::Type type, bool isConstant,
                                            cir::GlobalLinkageKind linkage) {
     mlir::OpBuilder::InsertionGuard guard(*this);
     setInsertionPointToStart(mlirModule.getBody());
-    return create<cir::GlobalOp>(loc, name, type, linkage);
+    return cir::GlobalOp::create(*this, loc, name, type, isConstant, linkage);
   }
 
   cir::GetMemberOp createGetMember(mlir::Location loc, mlir::Type resultTy,
                                    mlir::Value base, llvm::StringRef name,
                                    unsigned index) {
-    return create<cir::GetMemberOp>(loc, resultTy, base, name, index);
+    return cir::GetMemberOp::create(*this, loc, resultTy, base, name, index);
   }
 
   mlir::Value createDummyValue(mlir::Location loc, mlir::Type type,
                                clang::CharUnits alignment) {
     mlir::IntegerAttr alignmentAttr = getAlignmentAttr(alignment);
     auto addr = createAlloca(loc, getPointerTo(type), type, {}, alignmentAttr);
-    return create<cir::LoadOp>(loc, addr, /*isDeref=*/false, alignmentAttr);
+    return cir::LoadOp::create(*this, loc, addr, /*isDeref=*/false,
+                               alignmentAttr, /*mem_order=*/{});
   }
 
   cir::PtrStrideOp createPtrStride(mlir::Location loc, mlir::Value base,
                                    mlir::Value stride) {
-    return create<cir::PtrStrideOp>(loc, base.getType(), base, stride);
+    return cir::PtrStrideOp::create(*this, loc, base.getType(), base, stride);
   }
 
   //===--------------------------------------------------------------------===//
@@ -262,7 +290,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   cir::CallOp createCallOp(mlir::Location loc, mlir::SymbolRefAttr callee,
                            mlir::Type returnType, mlir::ValueRange operands,
                            llvm::ArrayRef<mlir::NamedAttribute> attrs = {}) {
-    auto op = create<cir::CallOp>(loc, callee, returnType, operands);
+    auto op = cir::CallOp::create(*this, loc, callee, returnType, operands);
     op->setAttrs(attrs);
     return op;
   }
@@ -293,7 +321,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
                          mlir::Value src, mlir::Type newTy) {
     if (newTy == src.getType())
       return src;
-    return create<cir::CastOp>(loc, newTy, kind, src);
+    return cir::CastOp::create(*this, loc, newTy, kind, src);
   }
 
   mlir::Value createCast(cir::CastKind kind, mlir::Value src,
@@ -343,7 +371,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value createBinop(mlir::Location loc, mlir::Value lhs,
                           cir::BinOpKind kind, mlir::Value rhs) {
-    return create<cir::BinOp>(loc, lhs.getType(), kind, lhs, rhs);
+    return cir::BinOp::create(*this, loc, lhs.getType(), kind, lhs, rhs);
   }
 
   mlir::Value createLowBitsSet(mlir::Location loc, unsigned size,
@@ -365,8 +393,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
                            mlir::Value trueValue, mlir::Value falseValue) {
     assert(trueValue.getType() == falseValue.getType() &&
            "trueValue and falseValue should have the same type");
-    return create<cir::SelectOp>(loc, trueValue.getType(), condition, trueValue,
-                                 falseValue);
+    return cir::SelectOp::create(*this, loc, trueValue.getType(), condition,
+                                 trueValue, falseValue);
   }
 
   mlir::Value createLogicalAnd(mlir::Location loc, mlir::Value lhs,
@@ -381,8 +409,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value createMul(mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
                         OverflowBehavior ob = OverflowBehavior::None) {
-    auto op =
-        create<cir::BinOp>(loc, lhs.getType(), cir::BinOpKind::Mul, lhs, rhs);
+    auto op = cir::BinOp::create(*this, loc, lhs.getType(), cir::BinOpKind::Mul,
+                                 lhs, rhs);
     op.setNoUnsignedWrap(
         llvm::to_underlying(ob & OverflowBehavior::NoUnsignedWrap));
     op.setNoSignedWrap(
@@ -400,8 +428,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value createSub(mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
                         OverflowBehavior ob = OverflowBehavior::Saturated) {
-    auto op =
-        create<cir::BinOp>(loc, lhs.getType(), cir::BinOpKind::Sub, lhs, rhs);
+    auto op = cir::BinOp::create(*this, loc, lhs.getType(), cir::BinOpKind::Sub,
+                                 lhs, rhs);
     op.setNoUnsignedWrap(
         llvm::to_underlying(ob & OverflowBehavior::NoUnsignedWrap));
     op.setNoSignedWrap(
@@ -422,8 +450,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value createAdd(mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
                         OverflowBehavior ob = OverflowBehavior::None) {
-    auto op =
-        create<cir::BinOp>(loc, lhs.getType(), cir::BinOpKind::Add, lhs, rhs);
+    auto op = cir::BinOp::create(*this, loc, lhs.getType(), cir::BinOpKind::Add,
+                                 lhs, rhs);
     op.setNoUnsignedWrap(
         llvm::to_underlying(ob & OverflowBehavior::NoUnsignedWrap));
     op.setNoSignedWrap(
@@ -444,7 +472,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   cir::CmpOp createCompare(mlir::Location loc, cir::CmpOpKind kind,
                            mlir::Value lhs, mlir::Value rhs) {
-    return create<cir::CmpOp>(loc, getBoolTy(), kind, lhs, rhs);
+    return cir::CmpOp::create(*this, loc, getBoolTy(), kind, lhs, rhs);
   }
 
   mlir::Value createIsNaN(mlir::Location loc, mlir::Value operand) {
@@ -453,7 +481,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value createShift(mlir::Location loc, mlir::Value lhs, mlir::Value rhs,
                           bool isShiftLeft) {
-    return create<cir::ShiftOp>(loc, lhs.getType(), lhs, rhs, isShiftLeft);
+    return cir::ShiftOp::create(*this, loc, lhs.getType(), lhs, rhs,
+                                isShiftLeft);
   }
 
   mlir::Value createShift(mlir::Location loc, mlir::Value lhs,
@@ -496,8 +525,7 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   static OpBuilder::InsertPoint getBestAllocaInsertPoint(mlir::Block *block) {
     auto last =
         std::find_if(block->rbegin(), block->rend(), [](mlir::Operation &op) {
-          // TODO: Add LabelOp missing feature here
-          return mlir::isa<cir::AllocaOp>(&op);
+          return mlir::isa<cir::AllocaOp, cir::LabelOp>(&op);
         });
 
     if (last != block->rend())
@@ -532,12 +560,12 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   /// Create a loop condition.
   cir::ConditionOp createCondition(mlir::Value condition) {
-    return create<cir::ConditionOp>(condition.getLoc(), condition);
+    return cir::ConditionOp::create(*this, condition.getLoc(), condition);
   }
 
   /// Create a yield operation.
   cir::YieldOp createYield(mlir::Location loc, mlir::ValueRange value = {}) {
-    return create<cir::YieldOp>(loc, value);
+    return cir::YieldOp::create(*this, loc, value);
   }
 };
 
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
index 588fb0d74a509..16b818f851e1c 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
@@ -50,6 +50,45 @@ class CIR_UnitAttr<string name, string attrMnemonic, list<Trait> traits = []>
   let isOptional = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// SourceLanguageAttr
+//===----------------------------------------------------------------------===//
+
+// TODO: Add cases for other languages that Clang supports.
+
+def CIR_SourceLanguage : CIR_I32EnumAttr<"SourceLanguage", "source language", [
+  I32EnumAttrCase<"C", 1, "c">,
+  I32EnumAttrCase<"CXX", 2, "cxx">
+]> {
+  // The enum attr class is defined in `CIR_SourceLanguageAttr` below,
+  // so that it can define extra class methods.
+  let genSpecializedAttr = 0;
+}
+
+def CIR_SourceLanguageAttr : CIR_EnumAttr<CIR_SourceLanguage, "lang"> {
+
+  let summary = "Module source language";
+  let description = [{
+    Represents the source language used to generate the module.
+
+    Example:
+    ```
+    // Module compiled from C.
+    module attributes {cir.lang = cir.lang<c>} {}
+    // Module compiled from C++.
+    module attributes {cir.lang = cir.lang<cxx>} {}
+    ```
+
+    Module source language attribute name is `cir.lang` is defined by
+    `getSourceLanguageAttrName` method in CIRDialect class.
+  }];
+
+  let extraClassDeclaration = [{
+    bool isC() const { return getValue() == SourceLanguage::C; }
+    bool isCXX() const { return getValue() == SourceLanguage::CXX; }
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // OptInfoAttr
 //===----------------------------------------------------------------------===//
@@ -341,6 +380,44 @@ def CIR_ConstVectorAttr : CIR_Attr<"ConstVector", "const_vector", [
   let genVerifyDecl = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// ConstRecordAttr
+//===----------------------------------------------------------------------===//
+
+def CIR_ConstRecordAttr : CIR_Attr<"ConstRecord", "const_record", [
+  TypedAttrInterface
+]> {
+  let summary = "Represents a constant record";
+  let description = [{
+    Effectively supports "struct-like" constants. It's must be built from
+    an `mlir::ArrayAttr` instance where each element is a typed attribute
+    (`mlir::TypedAttribute`).
+
+    Example:
+    ```
+    cir.global external @rgb2 = #cir.const_record<{0 : i8,
+                                                   5 : i64, #cir.null : !cir.ptr<i8>
+                                                  }> : !cir.record<"", i8, i64, !cir.ptr<i8>>
+    ```
+  }];
+
+  let parameters = (ins AttributeSelfTypeParameter<"">:$type,
+                        "mlir::ArrayAttr":$members);
+
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "cir::RecordType":$type,
+                                        "mlir::ArrayAttr":$members), [{
+      return $_get(type.getContext(), type, members);
+    }]>
+  ];
+
+  let assemblyFormat = [{
+    `<` custom<RecordMembers>($members) `>`
+  }];
+
+  let genVerifyDecl = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // ConstPtrAttr
 //===----------------------------------------------------------------------===//
@@ -370,6 +447,160 @@ def CIR_ConstPtrAttr : CIR_Attr<"ConstPtr", "ptr", [TypedAttrInterface]> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// GlobalViewAttr
+//===----------------------------------------------------------------------===//
+
+def CIR_GlobalViewAttr : CIR_Attr<"GlobalView", "global_view", [
+  TypedAttrInterface
+]> {
+  let summary = "Provides constant access to a global address";
+  let description = [{
+    Get constant address of global `symbol` and optionally apply offsets to
+    access existing subelements. It provides a way to access globals from other
+    global and always produces a pointer.
+
+    The type of the input symbol can be different from `#cir.global_view`
+    output type, since a given view of the global might require a static
+    cast for initializing other globals.
+
+    A list of indices can be optionally passed and each element subsequently
+    indexes underlying types. For `symbol` types like `!cir.array`
+    and `!cir.record`, it leads to the constant address of sub-elements, while
+    for `!cir.ptr`, an offset is applied. The first index is relative to the
+    original symbol type, not the produced one.
+
+    The result type of this attribute may be an integer type. In such a case,
+    the pointer to the referenced global is casted to an integer and this
+    attribute represents the casted result.
+
+    Example:
+
+    ```
+      cir.global external @s = @".str2": !cir.ptr<i8>
+      cir.global external @x = #cir.global_view<@s> : !cir.ptr<i8>
+      cir.global external @s_addr = #cir.global_view<@s> : !s64i
+
+      cir.global external @rgb = #cir.const_array<[0 : i8, -23 : i8, 33 : i8]
+                                                   : !cir.array<i8 x 3>>
+      cir.global external @elt_ptr = #cir.global_view<@rgb, [1]> : !cir.ptr<i8>
+    ```
+
+    Note, that unlike LLVM IR's gep instruction, CIR doesn't add the leading
+    zero index when it's known to be constant zero, e.g. for pointers, i.e. we
+    use indexes exactly to access sub elements or for the offset. The leading
+    zero index is added later in the lowering.
+
+    Example:
+    ```
+    struct A {
+      int a;
+    };
+
+    struct B:  virtual A {
+      int b;
+    };
+    ```
+    VTT for B in CIR:
+    ```
+    cir.global linkonce_odr @_ZTT1B = #cir.const_array<[
+              #cir.global_view<@_ZTV1B, [0 : i32, 3 : i32]> : !cir.ptr<!u8i>]>
+                   : !cir.array<!cir.ptr<!u8i> x 1>
+    ```
+    VTT for B in LLVM IR:
+    ```
+    @_ZTT1B = linkonce_odr global [1 x ptr] [ptr getelementptr inbounds
+              ({ [3 x ptr] }, ptr @_ZTV1B, i32 0, i32 0, i32 3)], align 8
+    ```
+  }];
+
+  let parameters = (ins AttributeSelfTypeParameter<"">:$type,
+                        "mlir::FlatSymbolRefAttr":$symbol,
+                        OptionalParameter<"mlir::ArrayAttr">:$indices);
+
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "mlir::Type":$type,
+                                        "mlir::FlatSymbolRefAttr":$symbol,
+                                        CArg<"mlir::ArrayAttr", "{}">:$indices), [{
+      return $_get(type.getContext(), type, symbol, indices);
+    }]>
+  ];
+
+  // let genVerifyDecl = 1;
+  let assemblyFormat = [{
+    `<`
+      $symbol
+      (`,` $indices^)?
+    `>`
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// VTableAttr
+//===----------------------------------------------------------------------===//
+
+def CIR_VTableAttr : CIR_Attr<"VTable", "vtable", [TypedAttrInterface]> {
+  let summary = "Represents a C++ vtable";
+  let description = [{
+    Wraps a #cir.const_record containing one or more vtable arrays.
+
+    In most cases, the anonymous record type wrapped by this attribute will
+    contain a single array corresponding to the vtable for one class. However,
+    in the case of multiple inheritence, the anonymous structure may contain
+    multiple arrays, each of which is a vtable.
+
+    Example 1 (single vtable):
+    ```mlir
+    cir.global linkonce_odr @_ZTV6Mother =
+      #cir.vtable<{
+        #cir.const_array<[
+          #cir.ptr<null> : !cir.ptr<!u8i>,
+          #cir.global_view<@_ZTI6Mother> : !cir.ptr<!u8i>,
+          #cir.global_view<@_ZN6Mother9MotherFooEv> : !cir.ptr<!u8i>,
+          #cir.global_view<@_ZN6Mother10MotherFoo2Ev> : !cir.ptr<!u8i>
+        ]> : !cir.array<!cir.ptr<!u8i> x 4>
+      }> : !rec_anon_struct1
+    ```
+
+    Example 2 (multiple vtables):
+    ```mlir
+    cir.global linkonce_odr @_ZTV5Child =
+      #cir.vtable<{
+        #cir.const_array<[
+          #cir.ptr<null> : !cir.ptr<!u8i>,
+          #cir.global_view<@_ZTI5Child> : !cir.ptr<!u8i>,
+          #cir.global_view<@_ZN5Child9MotherFooEv> : !cir.ptr<!u8i>,
+          #cir.global_view<@_ZN6Mother10MotherFoo2Ev> : !cir.ptr<!u8i>
+        ]> : !cir.array<!cir.ptr<!u8i> x 4>,
+        #cir.const_array<[
+          #cir.ptr<-8 : i64> : !cir.ptr<!u8i>,
+          #cir.global_view<@_ZTI5Child> : !cir.ptr<!u8i>,
+          #cir.global_view<@_ZN6Father9FatherFooEv> : !cir.ptr<!u8i>
+        ]> : !cir.array<!cir.ptr<!u8i> x 3>
+      }> : !rec_anon_struct2
+    ```
+  }];
+
+  // `data` is a const record with one element, containing an array of
+  // vtable information.
+  let parameters = (ins
+    AttributeSelfTypeParameter<"">:$type,
+    "mlir::ArrayAttr":$data
+  );
+
+  let builders = [
+    AttrBuilderWithInferredContext<(ins "mlir::Type":$type,
+                                        "mlir::ArrayAttr":$data), [{
+      return $_get(type.getContext(), type, data);
+    }]>
+  ];
+
+  let genVerifyDecl = 1;
+  let assemblyFormat = [{
+    `<` custom<RecordMembers>($data) `>`
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // ConstComplexAttr
 //===----------------------------------------------------------------------===//
@@ -516,4 +747,36 @@ def CIR_BitfieldInfoAttr : CIR_Attr<"BitfieldInfo", "bitfield_info"> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// AddressPointAttr
+//===----------------------------------------------------------------------===//
+
+def CIR_AddressPointAttr : CIR_Attr<"AddressPoint", "address_point"> {
+  let summary = "Address point attribute";
+
+  let description = [{
+    Attribute specifying the address point within a C++ virtual table (vtable).
+
+    The `index` (vtable index) parameter identifies which vtable to use within a
+    vtable group, while the `offset` (address point index) specifies the offset
+    within that vtable where the address begins.
+
+    Example:
+    ```mlir
+    cir.global linkonce_odr @_ZTV1B = ...
+    ...
+    %3 = cir.vtable.address_point(@_ZTV1B,
+                                  address_point = <index = 0, offset = 2>)
+                                 : !cir.vptr
+    ```
+  }];
+
+  let parameters = (ins "int32_t":$index,
+                        "int32_t":$offset);
+
+  let assemblyFormat = [{
+    `<` struct($index, $offset) `>`
+  }];
+}
+
 #endif // CLANG_CIR_DIALECT_IR_CIRATTRS_TD
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h b/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h
index 8ef565d6afd34..ecc681ee310e3 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h
+++ b/clang/include/clang/CIR/Dialect/IR/CIRDataLayout.h
@@ -34,6 +34,41 @@ class CIRDataLayout {
   void reset(mlir::DataLayoutSpecInterface spec);
 
   bool isBigEndian() const { return bigEndian; }
+
+  /// Internal helper method that returns requested alignment for type.
+  llvm::Align getAlignment(mlir::Type ty, bool abiOrPref) const;
+
+  llvm::Align getABITypeAlign(mlir::Type ty) const {
+    return getAlignment(ty, true);
+  }
+
+  /// Returns the maximum number of bytes that may be overwritten by
+  /// storing the specified type.
+  ///
+  /// If Ty is a scalable vector type, the scalable property will be set and
+  /// the runtime size will be a positive integer multiple of the base size.
+  ///
+  /// For example, returns 5 for i36 and 10 for x86_fp80.
+  llvm::TypeSize getTypeStoreSize(mlir::Type ty) const {
+    llvm::TypeSize baseSize = getTypeSizeInBits(ty);
+    return {llvm::divideCeil(baseSize.getKnownMinValue(), 8),
+            baseSize.isScalable()};
+  }
+
+  /// Returns the offset in bytes between successive objects of the
+  /// specified type, including alignment padding.
+  ///
+  /// If Ty is a scalable vector type, the scalable property will be set and
+  /// the runtime size will be a positive integer multiple of the base size.
+  ///
+  /// This is the amount that alloca reserves for this type. For example,
+  /// returns 12 or 16 for x86_fp80, depending on alignment.
+  llvm::TypeSize getTypeAllocSize(mlir::Type ty) const {
+    // Round up to the next alignment boundary.
+    return llvm::alignTo(getTypeStoreSize(ty), getABITypeAlign(ty).value());
+  }
+
+  llvm::TypeSize getTypeSizeInBits(mlir::Type ty) const;
 };
 
 } // namespace cir
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRDialect.td b/clang/include/clang/CIR/Dialect/IR/CIRDialect.td
index 3fdbf65573b36..15d5fa0b4753e 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRDialect.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRDialect.td
@@ -35,11 +35,13 @@ def CIR_Dialect : Dialect {
   let hasConstantMaterializer = 1;
 
   let extraClassDeclaration = [{
+    static llvm::StringRef getSourceLanguageAttrName() { return "cir.lang"; }
     static llvm::StringRef getTripleAttrName() { return "cir.triple"; }
     static llvm::StringRef getOptInfoAttrName() { return "cir.opt_info"; }
     static llvm::StringRef getCalleeAttrName() { return "callee"; }
     static llvm::StringRef getNoThrowAttrName() { return "nothrow"; }
     static llvm::StringRef getSideEffectAttrName() { return "side_effect"; }
+    static llvm::StringRef getModuleLevelAsmAttrName() { return "cir.module_asm"; }
 
     void registerAttributes();
     void registerTypes();
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 72841a1c08441..8ab602000cc29 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -299,6 +299,20 @@ def CIR_ConstantOp : CIR_Op<"const", [
   let hasFolder = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// C/C++ memory order definitions
+//===----------------------------------------------------------------------===//
+
+def CIR_MemOrder : CIR_I32EnumAttr<
+  "MemOrder", "Memory order according to C++11 memory model", [
+    I32EnumAttrCase<"Relaxed", 0, "relaxed">,
+    I32EnumAttrCase<"Consume", 1, "consume">,
+    I32EnumAttrCase<"Acquire", 2, "acquire">,
+    I32EnumAttrCase<"Release", 3, "release">,
+    I32EnumAttrCase<"AcquireRelease", 4, "acq_rel">,
+    I32EnumAttrCase<"SequentiallyConsistent", 5, "seq_cst">
+]>;
+
 //===----------------------------------------------------------------------===//
 // AllocaOp
 //===----------------------------------------------------------------------===//
@@ -408,13 +422,14 @@ def CIR_LoadOp : CIR_Op<"load", [
   let arguments = (ins Arg<CIR_PointerType, "the address to load from",
                            [MemRead]>:$addr,
                        UnitAttr:$isDeref,
-                       OptionalAttr<I64Attr>:$alignment
-                       );
+                       OptionalAttr<I64Attr>:$alignment,
+                       OptionalAttr<CIR_MemOrder>:$mem_order);
   let results = (outs CIR_AnyType:$result);
 
   let assemblyFormat = [{
     (`deref` $isDeref^)?
     (`align` `(` $alignment^ `)`)?
+    (`atomic` `(` $mem_order^ `)`)?
     $addr `:` qualified(type($addr)) `,` type($result) attr-dict
   }];
 
@@ -451,10 +466,12 @@ def CIR_StoreOp : CIR_Op<"store", [
   let arguments = (ins CIR_AnyType:$value,
                        Arg<CIR_PointerType, "the address to store the value",
                            [MemWrite]>:$addr,
-                           OptionalAttr<I64Attr>:$alignment);
+                       OptionalAttr<I64Attr>:$alignment,
+                       OptionalAttr<CIR_MemOrder>:$mem_order);
 
   let assemblyFormat = [{
     (`align` `(` $alignment^ `)`)?
+    (`atomic` `(` $mem_order^ `)`)?
     $value `,` $addr attr-dict `:` type($value) `,` qualified(type($addr))
   }];
 
@@ -1060,6 +1077,77 @@ def CIR_BrOp : CIR_Op<"br",[
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// GotoOp
+//===----------------------------------------------------------------------===//
+
+def CIR_GotoOp : CIR_Op<"goto", [Terminator]> {
+  let description = [{
+
+  Transfers control to the specified `label`. This requires a corresponding
+  `cir.label` to exist and is used by to represent source level `goto`s
+  that jump across region boundaries. Alternatively, `cir.br` is used to
+  construct goto's that don't violate such boundaries.
+
+  `cir.goto` is completely symbolic (i.e. it "jumps" on a label that isn't
+  yet materialized) and should be taken into account by passes and analysis
+  when deciding if it's safe to make some assumptions about a given region
+  or basic block.
+
+  Example:
+  ```C++
+    int test(int x) {
+      if (x)
+        goto label;
+      {
+        x = 10;
+    label:
+        return x;
+      }
+    }
+  ```
+
+  ```mlir
+    cir.scope {  // REGION #1
+      %2 = cir.load %0 : !cir.ptr<!s32i>, !s32i
+      %3 = cir.cast(int_to_bool, %2 : !s32i), !cir.bool
+      cir.if %3 {
+        cir.goto "label"
+      }
+      }
+      cir.scope {  // REGION #2
+        %2 = cir.const #cir.int<10> : !s32i
+        cir.store %2, %0 : !s32i, !cir.ptr<!s32i>
+        cir.br ^bb1
+      ^bb1:  // pred: ^bb0
+        cir.label "label"
+        %3 = cir.load %0 : !cir.ptr<!s32i>, !s32i
+        cir.store %3, %1 : !s32i, !cir.ptr<!s32i>
+        %4 = cir.load %1 : !cir.ptr<!s32i>, !s32i
+        cir.return %4 : !s32i
+      }
+      cir.unreachable
+  ```
+  }];
+  let arguments = (ins StrAttr:$label);
+  let assemblyFormat = [{ $label attr-dict }];
+}
+
+//===----------------------------------------------------------------------===//
+// LabelOp
+//===----------------------------------------------------------------------===//
+
+// The LabelOp has AlwaysSpeculatable trait in order to not to be swept
+// by canonicalizer
+def CIR_LabelOp : CIR_Op<"label", [AlwaysSpeculatable]> {
+  let description = [{
+    An identifier which may be referred by cir.goto operation
+  }];
+  let arguments = (ins StrAttr:$label);
+  let assemblyFormat = [{ $label attr-dict }];
+  let hasVerifier = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // UnaryOp
 //===----------------------------------------------------------------------===//
@@ -1631,12 +1719,14 @@ def CIR_GlobalOp : CIR_Op<"global", [
                        CIR_GlobalLinkageKind:$linkage,
                        OptionalAttr<AnyAttr>:$initial_value,
                        UnitAttr:$comdat,
+                       UnitAttr:$constant,
                        UnitAttr:$dso_local,
                        OptionalAttr<I64Attr>:$alignment);
 
   let assemblyFormat = [{
     ($sym_visibility^)?
     (`` $global_visibility^)?
+    (`constant` $constant^)?
     $linkage
     (`comdat` $comdat^)?
     (`dso_local` $dso_local^)?
@@ -1655,6 +1745,7 @@ def CIR_GlobalOp : CIR_Op<"global", [
   let builders = [OpBuilder<(ins
     "llvm::StringRef":$sym_name,
     "mlir::Type":$sym_type,
+    CArg<"bool", "false">:$isConstant,
     // CIR defaults to external linkage.
     CArg<"cir::GlobalLinkageKind",
     "cir::GlobalLinkageKind::ExternalLinkage">:$linkage)>];
@@ -1691,6 +1782,130 @@ def CIR_GetGlobalOp : CIR_Op<"get_global", [
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// VTableAddrPointOp
+//===----------------------------------------------------------------------===//
+
+def CIR_VTableAddrPointOp : CIR_Op<"vtable.address_point", [
+  Pure, DeclareOpInterfaceMethods<SymbolUserOpInterface>
+]> {
+  let summary = "Get the vtable (global variable) address point";
+  let description = [{
+    The `vtable.address_point` operation retrieves the "effective" address
+    (address point) of a C++ virtual table. An object internal `__vptr`
+    gets initializated on top of the value returned by this operation.
+
+    `address_point.index` (vtable index) provides the appropriate vtable within
+    the vtable group (as specified by Itanium ABI), and `address_point.offset`
+    (address point index) the actual address point within that vtable.
+
+    The return type is always `!cir.vptr`.
+
+    Example:
+    ```mlir
+    cir.global linkonce_odr @_ZTV1B = ...
+    ...
+    %3 = cir.vtable.address_point(@_ZTV1B,
+              address_point = <index = 0, offset = 2>) : !cir.vptr
+    ```
+  }];
+
+  let arguments = (ins
+    FlatSymbolRefAttr:$name,
+    CIR_AddressPointAttr:$address_point
+  );
+
+  let results = (outs Res<CIR_VPtrType, "", []>:$addr);
+
+  let assemblyFormat = [{
+    `(`
+      $name `,` `address_point` `=` $address_point
+    `)`
+    `:` qualified(type($addr)) attr-dict
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// VTableGetVPtr
+//===----------------------------------------------------------------------===//
+
+def CIR_VTableGetVPtrOp : CIR_Op<"vtable.get_vptr", [Pure]> {
+  let summary = "Get a the address of the vtable pointer for an object";
+  let description = [{
+    The `vtable.get_vptr` operation retrieves the address of the vptr for a
+    C++ object. This operation requires that the object pointer points to
+    the start of a complete object. (TODO: Describe how we get that).
+    The vptr will always be at offset zero in the object, but this operation
+    is more explicit about what is being retrieved than a direct bitcast.
+
+    The return type is always `!cir.ptr<!cir.vptr>`.
+
+    Example:
+    ```mlir
+    %2 = cir.load %0 : !cir.ptr<!cir.ptr<!rec_C>>, !cir.ptr<!rec_C>
+    %3 = cir.vtable.get_vptr %2 : !cir.ptr<!rec_C> -> !cir.ptr<!cir.vptr>
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<CIR_PointerType, "the vptr address", [MemRead]>:$src
+  );
+
+  let results = (outs CIR_PtrToVPtr:$result);
+
+  let assemblyFormat = [{
+      $src `:` qualified(type($src)) `->` qualified(type($result)) attr-dict
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// VTableGetVirtualFnAddrOp
+//===----------------------------------------------------------------------===//
+
+def CIR_VTableGetVirtualFnAddrOp : CIR_Op<"vtable.get_virtual_fn_addr", [
+  Pure
+]> {
+  let summary = "Get a the address of a virtual function pointer";
+  let description = [{
+    The `vtable.get_virtual_fn_addr` operation retrieves the address of a
+    virtual function pointer from an object's vtable (__vptr).
+    This is an abstraction to perform the basic pointer arithmetic to get
+    the address of the virtual function pointer, which can then be loaded and
+    called.
+
+    The `vptr` operand must be a `!cir.ptr<!cir.vptr>` value, which would
+    have been returned by a previous call to `cir.vatble.get_vptr`. The
+    `index` operand is an index of the virtual function in the vtable.
+
+    The return type is a pointer-to-pointer to the function type.
+
+    Example:
+    ```mlir
+    %2 = cir.load %0 : !cir.ptr<!cir.ptr<!rec_C>>, !cir.ptr<!rec_C>
+    %3 = cir.vtable.get_vptr %2 : !cir.ptr<!rec_C> -> !cir.ptr<!cir.vptr>
+    %4 = cir.load %3 : !cir.ptr<!cir.vptr>, !cir.vptr
+    %5 = cir.vtable.get_virtual_fn_addr %4[2] : !cir.vptr
+                  -> !cir.ptr<!cir.ptr<!cir.func<(!cir.ptr<!rec_C>) -> !s32i>>>
+    %6 = cir.load align(8) %5 : !cir.ptr<!cir.ptr<!cir.func<(!cir.ptr<!rec_C>)
+                                                                 -> !s32i>>>,
+                                !cir.ptr<!cir.func<(!cir.ptr<!rec_C>) -> !s32i>>
+    %7 = cir.call %6(%2) : (!cir.ptr<!cir.func<(!cir.ptr<!rec_C>) -> !s32i>>,
+                            !cir.ptr<!rec_C>) -> !s32i
+    ```
+  }];
+
+  let arguments = (ins
+    Arg<CIR_VPtrType, "vptr", [MemRead]>:$vptr,
+    I64Attr:$index);
+
+  let results = (outs CIR_PointerType:$result);
+
+  let assemblyFormat = [{
+    $vptr `[` $index `]` attr-dict
+    `:` qualified(type($vptr)) `->` qualified(type($result))
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // SetBitfieldOp
 //===----------------------------------------------------------------------===//
@@ -2152,6 +2367,68 @@ def CIR_CallOp : CIR_CallOpBase<"call", [NoRegionArguments]> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// ReturnAddrOp and FrameAddrOp
+//===----------------------------------------------------------------------===//
+
+class CIR_FuncAddrBuiltinOp<string mnemonic> : CIR_Op<mnemonic, []> {
+  let arguments = (ins CIR_UInt32:$level);
+  let results = (outs CIR_VoidPtrType:$result);
+  let assemblyFormat = [{
+    `(` $level `)` attr-dict
+  }];
+}
+
+def CIR_ReturnAddrOp : CIR_FuncAddrBuiltinOp<"return_address"> {
+  let summary =
+      "The return address of the current function, or of one of its callers";
+
+  let description = [{
+    Represents a call to builtin function ` __builtin_return_address` in CIR.
+    This builtin function returns the return address of the current function,
+    or of one of its callers.
+
+    The `level` argument is number of frames to scan up the call stack.
+    For instance, value of 0 yields the return address of the current function,
+    value of 1 yields the return address of the caller of the current function,
+    and so forth.
+
+    Examples:
+
+    ```mlir
+    %p = return_address(%level) -> !cir.ptr<!void>
+    ```
+  }];
+}
+
+def CIR_FrameAddrOp : CIR_FuncAddrBuiltinOp<"frame_address"> {
+  let summary =
+      "The frame address of the current function, or of one of its callers";
+
+  let description = [{
+    Represents a call to builtin function ` __builtin_frame_address` in CIR.
+    This builtin function returns the frame address of the current function,
+    or of one of its callers. The frame is the area on the stack that holds
+    local variables and saved registers. The frame address is normally the
+    address of the first word pushed on to the stack by the function.
+    However, the exact definition depends upon the processor and the calling
+    convention. If the processor has a dedicated frame pointer register, and
+    the function has a frame, then __builtin_frame_address returns the value of
+    the frame pointer register.
+
+    The `level` argument is number of frames to scan up the call stack.
+    For instance, value of 0 yields the frame address of the current function,
+    value of 1 yields the frame address of the caller of the current function,
+    and so forth.
+
+    Examples:
+
+    ```mlir
+    %p = frame_address(%level) -> !cir.ptr<!void>
+    ```
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // StackSaveOp & StackRestoreOp
 //===----------------------------------------------------------------------===//
@@ -2196,6 +2473,87 @@ def CIR_StackRestoreOp : CIR_Op<"stackrestore"> {
   let assemblyFormat = "$ptr attr-dict `:` qualified(type($ptr))";
 }
 
+//===----------------------------------------------------------------------===//
+// InlineAsmOp
+//===----------------------------------------------------------------------===//
+
+def CIR_AsmFlavor : CIR_I32EnumAttr<"AsmFlavor", "ATT or Intel",
+                                    [I32EnumAttrCase<"x86_att", 0>,
+                                     I32EnumAttrCase<"x86_intel", 1>]>;
+
+def CIR_InlineAsmOp : CIR_Op<"asm", [RecursiveMemoryEffects]> {
+  let description = [{
+    The `cir.asm` operation represents C/C++ asm inline.
+
+    CIR constraints strings follow the same rules that are established for
+    the C level assembler constraints with several differences caused by
+    clang::AsmStmt processing.
+
+    Thus, numbers that appears in the constraint string may also refer to:
+    - the output variable index referenced by the input operands.
+    - the index of early-clobber operand
+
+    Operand attributes are a storage, where each element corresponds to the
+    operand with the same index. The first index relates to the operation
+    result (if any).
+    The operands themselves are stored as VariadicOfVariadic in the following
+    order: output, input and then in/out operands. When several output operands
+    are present, the result type may be represented as an anonymous record type.
+
+    Example:
+    ```C++
+    __asm__("foo" : : : );
+    __asm__("bar $42 %[val]" : [val] "=r" (x), "+&r"(x));
+    __asm__("baz $42 %[val]" : [val] "=r" (x), "+&r"(x) : "[val]"(y));
+    ```
+
+    ```mlir
+    !rec_22anon2E022 = !cir.record<struct "anon.0" {!cir.int<s, 32>, !cir.int<s, 32>}>
+    !rec_22anon2E122 = !cir.record<struct "anon.1" {!cir.int<s, 32>, !cir.int<s, 32>}>
+    ...
+    %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init]
+    %1 = cir.alloca !s32i, !cir.ptr<!s32i>, ["y", init]
+    ...
+    %2 = cir.load %0 : !cir.ptr<!s32i>, !s32i
+    %3 = cir.load %1 : !cir.ptr<!s32i>, !s32i
+
+    cir.asm(x86_att,
+      out = [],
+      in = [],
+      in_out = [],
+      {"foo" "~{dirflag},~{fpsr},~{flags}"}) side_effects
+
+    cir.asm(x86_att,
+      out = [],
+      in = [],
+      in_out = [%2 : !s32i],
+      {"bar $$42 $0" "=r,=&r,1,~{dirflag},~{fpsr},~{flags}"}) -> !rec_22anon2E022
+
+    cir.asm(x86_att,
+      out = [],
+      in = [%3 : !s32i],
+      in_out = [%2 : !s32i],
+      {"baz $$42 $0" "=r,=&r,0,1,~{dirflag},~{fpsr},~{flags}"}) -> !rec_22anon2E122
+    ```
+  }];
+
+  let results = (outs Optional<CIR_AnyType>:$res);
+
+  let arguments =
+      (ins VariadicOfVariadic<AnyType, "operands_segments">:$asm_operands,
+          StrAttr:$asm_string, StrAttr:$constraints, UnitAttr:$side_effects,
+          CIR_AsmFlavor:$asm_flavor, ArrayAttr:$operand_attrs,
+          DenseI32ArrayAttr:$operands_segments);
+
+  let builders = [OpBuilder<(ins
+      "llvm::ArrayRef<mlir::ValueRange>":$asmOperands,
+      "llvm::StringRef":$asmString, "llvm::StringRef":$constraints,
+      "bool":$sideEffects, "AsmFlavor":$asmFlavor,
+      "llvm::ArrayRef<mlir::Attribute>":$operandAttrs)>];
+
+  let hasCustomAssemblyFormat = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // UnreachableOp
 //===----------------------------------------------------------------------===//
@@ -2827,7 +3185,7 @@ def CIR_ComplexSubOp : CIR_Op<"complex.sub", [
 }
 
 //===----------------------------------------------------------------------===//
-// ComplexMulOp
+// ComplexMulOp & ComplexDivOp
 //===----------------------------------------------------------------------===//
 
 def CIR_ComplexRangeKind : CIR_I32EnumAttr<
@@ -2846,12 +3204,13 @@ def CIR_ComplexMulOp : CIR_Op<"complex.mul", [
     The `cir.complex.mul` operation takes two complex numbers and returns
     their product.
 
-    Range is used to select the implementation used when the operation
-    is lowered to the LLVM dialect. For multiplication, 'improved',
-    'promoted', and 'basic' are all handled equivalently, producing the
-    algebraic formula with no special handling for NaN value. If 'full' is
-    used, a runtime-library function is called if one of the intermediate
-    calculations produced a NaN value.
+    For complex types with floating-point components, the `range` attribute
+    specifies the algorithm to be used when the operation is lowered to
+    the LLVM dialect. For multiplication, 'improved', 'promoted', and 'basic'
+    are all handled equivalently, producing the algebraic formula with no
+    special handling for NaN value. If 'full' is used, a runtime-library
+    function is called if one of the intermediate calculations produced
+    a NaN value.
 
     Example:
 
@@ -2874,6 +3233,48 @@ def CIR_ComplexMulOp : CIR_Op<"complex.mul", [
   }];
 }
 
+def CIR_ComplexDivOp : CIR_Op<"complex.div", [
+  Pure, SameOperandsAndResultType
+]> {
+  let summary = "Complex division";
+  let description = [{
+    The `cir.complex.div` operation takes two complex numbers and returns
+    their quotient.
+
+    For complex types with floating-point components, the `range` attribute
+    specifies the algorithm to be used when the operation is lowered to
+    the LLVM dialect. For division, 'improved' produces Smith's algorithms for
+    Complex division with no additional handling for NaN values. If 'promoted'
+    is used, the values are promoted to a higher precision type, if possible,
+    and the calculation is performed using the algebraic formula, with
+    no additional handling for NaN values. We fall back on Smith's algorithm
+    when the target doesn't support a higher precision type. If 'full' is used,
+    a runtime-library function is called if one of the intermediate
+    calculations produced a NaN value. and for 'basic' algebraic formula with
+    no additional handling for the NaN value will be used. For integers types
+    `range` attribute will be ignored.
+
+    Example:
+
+    ```mlir
+    %2 = cir.complex.div %0, %1 range(basic) : !cir.complex<!cir.float>
+    %2 = cir.complex.div %0, %1 range(full) : !cir.complex<!cir.float>
+    ```
+  }];
+
+  let arguments = (ins
+    CIR_ComplexType:$lhs,
+    CIR_ComplexType:$rhs,
+    CIR_ComplexRangeKind:$range
+  );
+
+  let results = (outs CIR_ComplexType:$result);
+
+  let assemblyFormat = [{
+    $lhs `,` $rhs `range` `(` $range `)` `:` qualified(type($result)) attr-dict
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // Bit Manipulation Operations
 //===----------------------------------------------------------------------===//
@@ -3143,6 +3544,56 @@ def CIR_AssumeOp : CIR_Op<"assume"> {
   }];
 }
 
+def CIR_AssumeAlignedOp : CIR_Op<"assume_aligned", [
+  Pure, AllTypesMatch<["pointer", "result"]>
+]> {
+  let summary = "Tell the optimizer that a pointer is aligned";
+  let description = [{
+    The `cir.assume_aligned` operation takes two or three arguments. The first
+    argument `pointer` gives the pointer value whose alignment is to be assumed,
+    and the second argument `align` is an integer attribute that gives the
+    assumed alignment.
+
+    The `offset` argument is optional. If given, it represents misalignment
+    offset. When it's present, this operation tells the optimizer that the
+    pointer is always misaligned to the alignment by `offset` bytes, a.k.a. the
+    pointer yielded by `(char *)pointer - offset` is aligned to the specified
+    alignment. Note that the `offset` argument is an SSA value rather than an
+    attribute, which means that you could pass a dynamically determined value
+    as the mialignment offset.
+
+    The result of this operation has the same value as the `pointer` argument,
+    but it additionally carries any alignment information indicated by this
+    operation.
+
+    This operation corresponds to the `__builtin_assume_aligned` builtin
+    function.
+
+    Example:
+
+    ```mlir
+    // Assume that %0 is a CIR pointer value of type !cir.ptr<!s32i>
+    %1 = cir.assume_aligned %0 alignment 16 : !cir.ptr<!s32i>
+
+    // With a misalignment offset of 4 bytes:
+    %2 = cir.const #cir.int<4> : !u64i
+    %3 = cir.assume_aligned %0 alignment 16 [offset %2 : !u64i] : !cir.ptr<!s32i>
+    ```
+  }];
+
+  let arguments = (ins CIR_PointerType:$pointer,
+                       I64Attr:$alignment,
+                       Optional<CIR_IntType>:$offset);
+  let results = (outs CIR_PointerType:$result);
+
+  let assemblyFormat = [{
+    $pointer
+    `alignment` $alignment
+    (`[` `offset` $offset^ `:` type($offset) `]`)?
+    `:` qualified(type($pointer)) attr-dict
+  }];
+}
+
 def CIR_AssumeSepStorageOp : CIR_Op<"assume_separate_storage", [
   SameTypeOperands
 ]> {
@@ -3202,4 +3653,151 @@ def CIR_ExpectOp : CIR_Op<"expect", [
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// Floating Point Ops
+//===----------------------------------------------------------------------===//
+
+class CIR_UnaryFPToFPBuiltinOp<string mnemonic, string llvmOpName>
+    : CIR_Op<mnemonic, [Pure, SameOperandsAndResultType]>
+{
+  let arguments = (ins CIR_AnyFloatOrVecOfFloatType:$src);
+  let results = (outs CIR_AnyFloatOrVecOfFloatType:$result);
+
+  let assemblyFormat = "$src `:` type($src) attr-dict";
+
+  let llvmOp = llvmOpName;
+}
+
+def CIR_FAbsOp : CIR_UnaryFPToFPBuiltinOp<"fabs", "FAbsOp"> {
+  let summary = "Computes the floating-point absolute value";
+  let description = [{
+    `cir.fabs` computes the absolute value of a floating-point operand
+    and returns a result of the same type, ignoring floating-point
+    exceptions. It does not set `errno`.
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// Variadic Operations
+//===----------------------------------------------------------------------===//
+
+def CIR_VAStartOp : CIR_Op<"va_start"> {
+  let summary = "Starts a variable argument list";
+  let description = [{
+    The cir.va_start operation models the C/C++ va_start macro by
+    initializing a variable argument list at the given va_list storage
+    location.
+
+    The first operand must be a pointer to the target's `va_list`
+    representation. This operation has no results and produces its effect by
+    mutating the storage referenced by the pointer operand. The second operand
+    must be an integer value that contains the expected number of arguments in
+    that list.
+
+    Each `cir.va_start` must be paired with a corresponding `cir.va_end`
+    on the same logical `va_list` object along all control-flow paths. After
+    `cir.va_end`, the `va_list` must not be accessed unless reinitialized
+    with another `cir.va_start`.
+
+    Lowering maps this to the LLVM intrinsic `llvm.va_start`, passing the
+    appropriately decayed pointer to the underlying `va_list` storage.
+
+    Example:
+
+    ```mlir
+    // %args : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>
+    %p = cir.cast(array_to_ptrdecay, %args
+          : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>),
+        !cir.ptr<!rec___va_list_tag>
+    %count = cir.load %0 : !cir.ptr<!s32i>, !s32i
+    cir.va_start %p %count : !cir.ptr<!rec___va_list_tag>, !s32i
+    ```
+  }];
+  let arguments = (ins
+    CIR_PointerType:$arg_list,
+    CIR_AnyFundamentalIntType:$count
+  );
+
+  let assemblyFormat = [{
+    $arg_list $count attr-dict `:` type(operands)
+  }];
+}
+
+def CIR_VAEndOp : CIR_Op<"va_end"> {
+  let summary = "Ends a variable argument list";
+  let description = [{
+    The `cir.va_end` operation models the C/C++ va_end macro by finalizing
+    and cleaning up a variable argument list previously initialized with
+    `cir.va_start`.
+
+    The operand must be a pointer to the target's `va_list` representation.
+    This operation has no results and produces its effect by mutating the
+    storage referenced by the pointer operand.
+
+    `cir.va_end` must only be called after a matching `cir.va_start` on the
+    same `va_list` along all control-flow paths. After `cir.va_end`, the
+    `va_list` is invalid and must not be accessed unless reinitialized.
+
+    Lowering typically maps this to the LLVM intrinsic `llvm.va_end`,
+    passing the appropriately decayed pointer to the underlying `va_list`
+    storage.
+
+    Example:
+    ```mlir
+    // %args : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>
+    %p = cir.cast(array_to_ptrdecay, %args
+          : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>),
+        !cir.ptr<!rec___va_list_tag>
+    cir.va_end %p : !cir.ptr<!rec___va_list_tag>
+    ```
+  }];
+
+  let arguments = (ins CIR_PointerType:$arg_list);
+
+  let assemblyFormat = [{
+    $arg_list attr-dict `:` type(operands)
+  }];
+}
+
+def CIR_VAArgOp : CIR_Op<"va_arg"> {
+  let summary = "Fetches next variadic element as a given type";
+  let description = [{
+    The `cir.va_arg` operation models the C/C++ `va_arg` macro by reading the
+    next argument from an active variable argument list and producing it as a
+    value of a specified result type.
+
+    The operand must be a pointer to the target's `va_list` representation.
+    The operation advances the `va_list` state as a side effect and returns
+    the fetched value as the result, whose type is chosen by the user of the
+    operation.
+
+    A `cir.va_arg` must only be used on a `va_list` that has been initialized
+    with `cir.va.start` and not yet finalized by `cir.va.end`. The semantics
+    (including alignment and promotion rules) follow the platform ABI; the
+    frontend is responsible for providing a `va_list` pointer that matches the
+    target representation.
+
+    Example:
+    ```mlir
+    // %args : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>
+    %p = cir.cast(array_to_ptrdecay, %args
+            : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>),
+          !cir.ptr<!rec___va_list_tag>
+    cir.va.start %p : !cir.ptr<!rec___va_list_tag>
+
+    // Fetch an `int` from the vararg list.
+    %v = cir.va_arg %p : (!cir.ptr<!rec___va_list_tag>) -> !s32i
+
+    cir.va.end %p : !cir.ptr<!rec___va_list_tag>
+    ```
+  }];
+
+  let arguments = (ins CIR_PointerType:$arg_list);
+  let results = (outs CIR_AnyType:$result);
+
+  let assemblyFormat = [{
+    $arg_list attr-dict `:` functional-type(operands, $result)
+  }];
+}
+
 #endif // CLANG_CIR_DIALECT_IR_CIROPS_TD
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROpsEnums.h b/clang/include/clang/CIR/Dialect/IR/CIROpsEnums.h
index fead5725d183d..17fddaee871b3 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROpsEnums.h
+++ b/clang/include/clang/CIR/Dialect/IR/CIROpsEnums.h
@@ -113,6 +113,18 @@ LLVM_ATTRIBUTE_UNUSED static bool isValidLinkage(GlobalLinkageKind gl) {
          isLinkOnceLinkage(gl);
 }
 
+bool operator<(cir::MemOrder, cir::MemOrder) = delete;
+bool operator>(cir::MemOrder, cir::MemOrder) = delete;
+bool operator<=(cir::MemOrder, cir::MemOrder) = delete;
+bool operator>=(cir::MemOrder, cir::MemOrder) = delete;
+
+// Validate an integral value which isn't known to fit within the enum's range
+// is a valid AtomicOrderingCABI.
+template <typename Int> inline bool isValidCIRAtomicOrderingCABI(Int value) {
+  return static_cast<Int>(cir::MemOrder::Relaxed) <= value &&
+         value <= static_cast<Int>(cir::MemOrder::SequentiallyConsistent);
+}
+
 } // namespace cir
 
 #endif // CLANG_CIR_DIALECT_IR_CIROPSENUMS_H
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypeConstraints.td b/clang/include/clang/CIR/Dialect/IR/CIRTypeConstraints.td
index d7d55dfbc0654..82f6e1d33043e 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRTypeConstraints.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypeConstraints.td
@@ -289,6 +289,14 @@ def CIR_AnyFloatOrVecOfFloatType
     let cppFunctionName = "isFPOrVectorOfFPType";
 }
 
+//===----------------------------------------------------------------------===//
+// VPtr type predicates
+//===----------------------------------------------------------------------===//
+
+def CIR_AnyVPtrType : CIR_TypeBase<"::cir::VPtrType", "vptr type">;
+
+def CIR_PtrToVPtr : CIR_PtrToType<CIR_AnyVPtrType>;
+
 //===----------------------------------------------------------------------===//
 // Scalar Type predicates
 //===----------------------------------------------------------------------===//
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
index a258df79a6184..312d0a9422673 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRTypes.td
@@ -296,10 +296,10 @@ def CIR_VPtrType : CIR_Type<"VPtr", "vptr", [
     access to the vptr.
 
     This type will be the element type of the 'vptr' member of structures that
-    require a vtable pointer. A pointer to this type is returned by the
-    `cir.vtable.address_point` and `cir.vtable.get_vptr` operations, and this
-    pointer may be passed to the `cir.vtable.get_virtual_fn_addr` operation to
-    get the address of a virtual function pointer.
+    require a vtable pointer. The `cir.vtable.address_point` operation returns
+    this type. The `cir.vtable.get_vptr` operations returns a pointer to this
+    type. This pointer may be passed to the `cir.vtable.get_virtual_fn_addr`
+    operation to get the address of a virtual function pointer.
 
     The pointer may also be cast to other pointer types in order to perform
     pointer arithmetic based on information encoded in the AST layout to get
diff --git a/clang/include/clang/CIR/Dialect/Passes.h b/clang/include/clang/CIR/Dialect/Passes.h
index 7a202b1e04ef9..32c3e27d07dfb 100644
--- a/clang/include/clang/CIR/Dialect/Passes.h
+++ b/clang/include/clang/CIR/Dialect/Passes.h
@@ -26,6 +26,7 @@ std::unique_ptr<Pass> createCIRSimplifyPass();
 std::unique_ptr<Pass> createHoistAllocasPass();
 std::unique_ptr<Pass> createLoweringPreparePass();
 std::unique_ptr<Pass> createLoweringPreparePass(clang::ASTContext *astCtx);
+std::unique_ptr<Pass> createGotoSolverPass();
 
 void populateCIRPreLoweringPasses(mlir::OpPassManager &pm);
 
diff --git a/clang/include/clang/CIR/Dialect/Passes.td b/clang/include/clang/CIR/Dialect/Passes.td
index 7d5ec2ffed39d..0f5783945f8ae 100644
--- a/clang/include/clang/CIR/Dialect/Passes.td
+++ b/clang/include/clang/CIR/Dialect/Passes.td
@@ -72,6 +72,16 @@ def CIRFlattenCFG : Pass<"cir-flatten-cfg"> {
   let dependentDialects = ["cir::CIRDialect"];
 }
 
+def GotoSolver : Pass<"cir-goto-solver"> {
+  let summary = "Replaces goto operations with branches";
+  let description = [{
+    This pass transforms CIR and replaces goto-s with branch
+    operations to the proper blocks.
+  }];
+  let constructor = "mlir::createGotoSolverPass()";
+  let dependentDialects = ["cir::CIRDialect"];
+}
+
 def LoweringPrepare : Pass<"cir-lowering-prepare"> {
   let summary = "Lower to more fine-grained CIR operations before lowering to "
     "other dialects";
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 27dd181f2fb37..d7a2e49ec162a 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -27,9 +27,6 @@ struct MissingFeatures {
   // Address space related
   static bool addressSpace() { return false; }
 
-  // CIRGenFunction implementation details
-  static bool cgfSymbolTable() { return false; }
-
   // Unhandled global/linkage information.
   static bool opGlobalThreadLocal() { return false; }
   static bool opGlobalConstant() { return false; }
@@ -52,7 +49,6 @@ struct MissingFeatures {
   static bool opLoadEmitScalarRangeCheck() { return false; }
   static bool opLoadBooleanRepresentation() { return false; }
   static bool opLoadStoreTbaa() { return false; }
-  static bool opLoadStoreMemOrder() { return false; }
   static bool opLoadStoreVolatile() { return false; }
   static bool opLoadStoreAtomic() { return false; }
   static bool opLoadStoreObjC() { return false; }
@@ -87,7 +83,6 @@ struct MissingFeatures {
   static bool setFunctionAttributes() { return false; }
 
   // CallOp handling
-  static bool opCallPseudoDtor() { return false; }
   static bool opCallAggregateArgs() { return false; }
   static bool opCallPaddingArgs() { return false; }
   static bool opCallABIExtendArg() { return false; }
@@ -99,7 +94,6 @@ struct MissingFeatures {
   static bool opCallArgEvaluationOrder() { return false; }
   static bool opCallCallConv() { return false; }
   static bool opCallMustTail() { return false; }
-  static bool opCallVirtual() { return false; }
   static bool opCallInAlloca() { return false; }
   static bool opCallAttrs() { return false; }
   static bool opCallSurroundingTry() { return false; }
@@ -162,6 +156,15 @@ struct MissingFeatures {
   static bool addressIsKnownNonNull() { return false; }
   static bool addressPointerAuthInfo() { return false; }
 
+  // Atomic
+  static bool atomicExpr() { return false; }
+  static bool atomicInfo() { return false; }
+  static bool atomicInfoGetAtomicPointer() { return false; }
+  static bool atomicInfoGetAtomicAddress() { return false; }
+  static bool atomicUseLibCall() { return false; }
+  static bool atomicScope() { return false; }
+  static bool atomicSyncScopeID() { return false; }
+
   // Misc
   static bool abiArgInfo() { return false; }
   static bool addHeapAllocSiteMetadata() { return false; }
@@ -173,7 +176,12 @@ struct MissingFeatures {
   static bool aggValueSlotVolatile() { return false; }
   static bool alignCXXRecordDecl() { return false; }
   static bool armComputeVolatileBitfields() { return false; }
+  static bool asmGoto() { return false; }
+  static bool asmInputOperands() { return false; }
   static bool asmLabelAttr() { return false; }
+  static bool asmMemoryEffects() { return false; }
+  static bool asmOutputOperands() { return false; }
+  static bool asmUnwindClobber() { return false; }
   static bool assignMemcpyizer() { return false; }
   static bool astVarDeclInterface() { return false; }
   static bool attributeBuiltin() { return false; }
@@ -192,13 +200,20 @@ struct MissingFeatures {
   static bool constantFoldSwitchStatement() { return false; }
   static bool constructABIArgDirectExtend() { return false; }
   static bool coverageMapping() { return false; }
+  static bool createInvariantGroup() { return false; }
   static bool createProfileWeightsForLoop() { return false; }
   static bool ctorMemcpyizer() { return false; }
   static bool cudaSupport() { return false; }
   static bool cxxRecordStaticMembers() { return false; }
+  static bool dataLayoutTypeIsSized() { return false; }
   static bool dataLayoutTypeAllocSize() { return false; }
+  static bool dataLayoutTypeStoreSize() { return false; }
   static bool deferredCXXGlobalInit() { return false; }
+  static bool devirtualizeMemberFunction() { return false; }
   static bool ehCleanupFlags() { return false; }
+  static bool ehCleanupScope() { return false; }
+  static bool ehCleanupScopeRequiresEHCleanup() { return false; }
+  static bool ehCleanupBranchFixups() { return false; }
   static bool ehstackBranches() { return false; }
   static bool emitCheckedInBoundsGEP() { return false; }
   static bool emitCondLikelihoodViaExpectIntrinsic() { return false; }
@@ -206,12 +221,16 @@ struct MissingFeatures {
   static bool emitLValueAlignmentAssumption() { return false; }
   static bool emitNullabilityCheck() { return false; }
   static bool emitTypeCheck() { return false; }
+  static bool emitTypeMetadataCodeForVCall() { return false; }
   static bool fastMathFlags() { return false; }
   static bool fpConstraints() { return false; }
   static bool generateDebugInfo() { return false; }
+  static bool globalViewIndices() { return false; }
+  static bool globalViewIntLowering() { return false; }
   static bool hip() { return false; }
   static bool implicitConstructorArgs() { return false; }
   static bool incrementProfileCounter() { return false; }
+  static bool innermostEHScope() { return false; }
   static bool insertBuiltinUnpredictable() { return false; }
   static bool instrumentation() { return false; }
   static bool intrinsics() { return false; }
@@ -231,8 +250,8 @@ struct MissingFeatures {
   static bool objCBlocks() { return false; }
   static bool objCGC() { return false; }
   static bool objCLifetime() { return false; }
+  static bool openCL() { return false; }
   static bool openMP() { return false; }
-  static bool opGlobalViewAttr() { return false; }
   static bool opTBAA() { return false; }
   static bool peepholeProtection() { return false; }
   static bool pgoUse() { return false; }
@@ -245,6 +264,7 @@ struct MissingFeatures {
   static bool setNonGC() { return false; }
   static bool setObjCGCLValueClass() { return false; }
   static bool setTargetAttributes() { return false; }
+  static bool sourceLanguageCases() { return false; }
   static bool stackBase() { return false; }
   static bool stackSaveOp() { return false; }
   static bool targetCIRGenInfoArch() { return false; }
@@ -258,7 +278,10 @@ struct MissingFeatures {
   static bool appleKext() { return false; }
   static bool dtorCleanups() { return false; }
   static bool vtableInitialization() { return false; }
+  static bool vtableEmitMetadata() { return false; }
+  static bool vtableRelativeLayout() { return false; }
   static bool msvcBuiltins() { return false; }
+  static bool vaArgABILowering() { return false; }
   static bool vlas() { return false; }
 
   // Missing types
diff --git a/clang/include/clang/CodeGen/CGFunctionInfo.h b/clang/include/clang/CodeGen/CGFunctionInfo.h
index 50be51769f1a8..713b52a4cc2b8 100644
--- a/clang/include/clang/CodeGen/CGFunctionInfo.h
+++ b/clang/include/clang/CodeGen/CGFunctionInfo.h
@@ -77,6 +77,11 @@ class ABIArgInfo {
     /// Array elements in the type are assumed to be padding and skipped.
     CoerceAndExpand,
 
+    /// TargetSpecific - Some argument types are passed as target specific types
+    /// such as RISC-V's tuple type, these need to be handled in the target
+    /// hook.
+    TargetSpecific,
+
     /// InAlloca - Pass the argument directly using the LLVM inalloca attribute.
     /// This is similar to indirect with byval, except it only applies to
     /// arguments stored in memory and forbids any implicit copies.  When
@@ -120,7 +125,7 @@ class ABIArgInfo {
 
   bool canHavePaddingType() const {
     return isDirect() || isExtend() || isIndirect() || isIndirectAliased() ||
-           isExpand();
+           isExpand() || isTargetSpecific();
   }
   void setPaddingType(llvm::Type *T) {
     assert(canHavePaddingType());
@@ -291,6 +296,20 @@ class ABIArgInfo {
     return AI;
   }
 
+  static ABIArgInfo getTargetSpecific(llvm::Type *T = nullptr,
+                                      unsigned Offset = 0,
+                                      llvm::Type *Padding = nullptr,
+                                      bool CanBeFlattened = true,
+                                      unsigned Align = 0) {
+    auto AI = ABIArgInfo(TargetSpecific);
+    AI.setCoerceToType(T);
+    AI.setPaddingType(Padding);
+    AI.setDirectOffset(Offset);
+    AI.setDirectAlign(Align);
+    AI.setCanBeFlattened(CanBeFlattened);
+    return AI;
+  }
+
   static bool isPaddingForCoerceAndExpand(llvm::Type *eltType) {
     return eltType->isArrayTy() &&
            eltType->getArrayElementType()->isIntegerTy(8);
@@ -305,27 +324,33 @@ class ABIArgInfo {
   bool isIndirectAliased() const { return TheKind == IndirectAliased; }
   bool isExpand() const { return TheKind == Expand; }
   bool isCoerceAndExpand() const { return TheKind == CoerceAndExpand; }
+  bool isTargetSpecific() const { return TheKind == TargetSpecific; }
 
   bool canHaveCoerceToType() const {
-    return isDirect() || isExtend() || isCoerceAndExpand();
+    return isDirect() || isExtend() || isCoerceAndExpand() ||
+           isTargetSpecific();
   }
 
   // Direct/Extend accessors
   unsigned getDirectOffset() const {
-    assert((isDirect() || isExtend()) && "Not a direct or extend kind");
+    assert((isDirect() || isExtend() || isTargetSpecific()) &&
+           "Not a direct or extend or target specific kind");
     return DirectAttr.Offset;
   }
   void setDirectOffset(unsigned Offset) {
-    assert((isDirect() || isExtend()) && "Not a direct or extend kind");
+    assert((isDirect() || isExtend() || isTargetSpecific()) &&
+           "Not a direct or extend or target specific kind");
     DirectAttr.Offset = Offset;
   }
 
   unsigned getDirectAlign() const {
-    assert((isDirect() || isExtend()) && "Not a direct or extend kind");
+    assert((isDirect() || isExtend() || isTargetSpecific()) &&
+           "Not a direct or extend or target specific kind");
     return DirectAttr.Align;
   }
   void setDirectAlign(unsigned Align) {
-    assert((isDirect() || isExtend()) && "Not a direct or extend kind");
+    assert((isDirect() || isExtend() || isTargetSpecific()) &&
+           "Not a direct or extend or target specific kind");
     DirectAttr.Align = Align;
   }
 
@@ -394,12 +419,14 @@ class ABIArgInfo {
   }
 
   bool getInReg() const {
-    assert((isDirect() || isExtend() || isIndirect()) && "Invalid kind!");
+    assert((isDirect() || isExtend() || isIndirect() || isTargetSpecific()) &&
+           "Invalid kind!");
     return InReg;
   }
 
   void setInReg(bool IR) {
-    assert((isDirect() || isExtend() || isIndirect()) && "Invalid kind!");
+    assert((isDirect() || isExtend() || isIndirect() || isTargetSpecific()) &&
+           "Invalid kind!");
     InReg = IR;
   }
 
@@ -481,12 +508,12 @@ class ABIArgInfo {
   }
 
   bool getCanBeFlattened() const {
-    assert(isDirect() && "Invalid kind!");
+    assert((isDirect() || isTargetSpecific()) && "Invalid kind!");
     return CanBeFlattened;
   }
 
   void setCanBeFlattened(bool Flatten) {
-    assert(isDirect() && "Invalid kind!");
+    assert((isDirect() || isTargetSpecific()) && "Invalid kind!");
     CanBeFlattened = Flatten;
   }
 
diff --git a/clang/include/clang/CodeGen/CodeGenABITypes.h b/clang/include/clang/CodeGen/CodeGenABITypes.h
index 836fdd75477c7..16f39b991a273 100644
--- a/clang/include/clang/CodeGen/CodeGenABITypes.h
+++ b/clang/include/clang/CodeGen/CodeGenABITypes.h
@@ -32,6 +32,7 @@
 namespace llvm {
 class AttrBuilder;
 class Constant;
+class ConstantInt;
 class Function;
 class FunctionType;
 class Type;
@@ -126,6 +127,12 @@ uint16_t getPointerAuthDeclDiscriminator(CodeGenModule &CGM, GlobalDecl GD);
 uint16_t getPointerAuthTypeDiscriminator(CodeGenModule &CGM,
                                          QualType FunctionType);
 
+/// Return a signed constant pointer.
+llvm::Constant *getConstantSignedPointer(CodeGenModule &CGM,
+                                         llvm::Constant *Pointer, unsigned Key,
+                                         llvm::Constant *StorageAddress,
+                                         llvm::ConstantInt *OtherDiscriminator);
+
 /// Given the language and code-generation options that Clang was configured
 /// with, set the default LLVM IR attributes for a function definition.
 /// The attributes set here are mostly global target-configuration and
diff --git a/clang/include/clang/Driver/CommonArgs.h b/clang/include/clang/Driver/CommonArgs.h
index 9802962d107c9..1464ce4e1b31b 100644
--- a/clang/include/clang/Driver/CommonArgs.h
+++ b/clang/include/clang/Driver/CommonArgs.h
@@ -85,6 +85,8 @@ const char *RelocationModelName(llvm::Reloc::Model Model);
 std::tuple<llvm::Reloc::Model, unsigned, bool>
 ParsePICArgs(const ToolChain &ToolChain, const llvm::opt::ArgList &Args);
 
+bool getStaticPIE(const llvm::opt::ArgList &Args, const ToolChain &TC);
+
 unsigned ParseFunctionAlignment(const ToolChain &TC,
                                 const llvm::opt::ArgList &Args);
 
diff --git a/clang/include/clang/Driver/Driver.h b/clang/include/clang/Driver/Driver.h
index 4d32552b7f85f..b9b187ada8add 100644
--- a/clang/include/clang/Driver/Driver.h
+++ b/clang/include/clang/Driver/Driver.h
@@ -512,6 +512,9 @@ class Driver {
 
   /// BuildActions - Construct the list of actions to perform for the
   /// given arguments, which are only done for a single architecture.
+  /// If the compilation is an explicit module build, delegates to
+  /// BuildDriverManagedModuleBuildActions. Otherwise, BuildDefaultActions is
+  /// used.
   ///
   /// \param C - The compilation that is being built.
   /// \param Args - The input arguments.
@@ -796,6 +799,35 @@ class Driver {
   /// compilation based on which -f(no-)?lto(=.*)? option occurs last.
   void setLTOMode(const llvm::opt::ArgList &Args);
 
+  /// BuildDefaultActions - Constructs the list of actions to perform
+  /// for the provided arguments, which are only done for a single architecture.
+  ///
+  /// \param C - The compilation that is being built.
+  /// \param Args - The input arguments.
+  /// \param Actions - The list to store the resulting actions onto.
+  void BuildDefaultActions(Compilation &C, llvm::opt::DerivedArgList &Args,
+                           const InputList &Inputs, ActionList &Actions) const;
+
+  /// BuildDriverManagedModuleBuildActions - Performs a dependency
+  /// scan and constructs the list of actions to perform for dependency order
+  /// and the provided arguments. This is only done for a single a architecture.
+  ///
+  /// \param C - The compilation that is being built.
+  /// \param Args - The input arguments.
+  /// \param Actions - The list to store the resulting actions onto.
+  void BuildDriverManagedModuleBuildActions(Compilation &C,
+                                            llvm::opt::DerivedArgList &Args,
+                                            const InputList &Inputs,
+                                            ActionList &Actions) const;
+
+  /// Scans the leading lines of the C++ source inputs to detect C++20 module
+  /// usage.
+  ///
+  /// \returns True if module usage is detected, false otherwise, or an error on
+  /// read failure.
+  llvm::ErrorOr<bool>
+  ScanInputsForCXX20ModulesUsage(const InputList &Inputs) const;
+
   /// Retrieves a ToolChain for a particular \p Target triple.
   ///
   /// Will cache ToolChains for the life of the driver object, and create them
diff --git a/clang/include/clang/Driver/OffloadBundler.h b/clang/include/clang/Driver/OffloadBundler.h
index 667156a524b79..e7306ce3cc9ab 100644
--- a/clang/include/clang/Driver/OffloadBundler.h
+++ b/clang/include/clang/Driver/OffloadBundler.h
@@ -120,7 +120,7 @@ class CompressedOffloadBundle {
     static llvm::Expected<CompressedBundleHeader> tryParse(llvm::StringRef);
   };
 
-  static inline const uint16_t DefaultVersion = 2;
+  static inline const uint16_t DefaultVersion = 3;
 
   static llvm::Expected<std::unique_ptr<llvm::MemoryBuffer>>
   compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input,
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 6aab43c9ed57f..6a46fec1701f3 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3296,6 +3296,13 @@ defm modules_reduced_bmi : BoolOption<"f", "modules-reduced-bmi",
   PosFlag<SetTrue, [], [ClangOption, CC1Option],
           "Generate the reduced BMI">>;
 
+def fmodules_driver : Flag<["-"], "fmodules-driver">,
+  Group<f_Group>, Visibility<[ClangOption]>,
+  HelpText<"Enable support for driver managed module builds (experimental)">;
+def fno_modules_driver : Flag<["-"], "fno-modules-driver">,
+  Group<f_Group>, Visibility<[ClangOption]>,
+  HelpText<"Disable support for driver managed module builds (experimental)">;
+
 def experimental_modules_reduced_bmi : Flag<["-"], "fexperimental-modules-reduced-bmi">,
   Group<f_Group>, Visibility<[ClangOption, CC1Option]>, Alias<fmodules_reduced_bmi>;
 
@@ -3731,14 +3738,20 @@ def fopenmp_relocatable_target : Flag<["-"], "fopenmp-relocatable-target">,
 def fnoopenmp_relocatable_target : Flag<["-"], "fnoopenmp-relocatable-target">,
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>,
   Visibility<[ClangOption, CC1Option]>;
-def fopenmp_simd : Flag<["-"], "fopenmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>,
-  HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
+def fopenmp_simd : Flag<["-"], "fopenmp-simd">,
+                   Group<f_Group>,
+                   Flags<[NoArgumentUnused]>,
+                   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+                   HelpText<"Emit OpenMP code only for SIMD-based constructs.">;
 def fopenmp_enable_irbuilder : Flag<["-"], "fopenmp-enable-irbuilder">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Use the experimental OpenMP-IR-Builder codegen path.">;
-def fno_openmp_simd : Flag<["-"], "fno-openmp-simd">, Group<f_Group>,
-  Flags<[NoArgumentUnused]>, Visibility<[ClangOption, CC1Option]>;
+def fno_openmp_simd
+    : Flag<["-"], "fno-openmp-simd">,
+      Group<f_Group>,
+      Flags<[NoArgumentUnused]>,
+      Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
+      HelpText<"Do not emit code for any OpenMP constructs.">;
 def fopenmp_cuda_mode : Flag<["-"], "fopenmp-cuda-mode">, Group<f_Group>,
   Flags<[NoArgumentUnused, HelpHidden]>, Visibility<[ClangOption, CC1Option]>;
 def fno_openmp_cuda_mode : Flag<["-"], "fno-openmp-cuda-mode">, Group<f_Group>,
@@ -4540,6 +4553,7 @@ defm aarch64_jump_table_hardening: OptInCC1FFlag<"aarch64-jump-table-hardening",
 defm ptrauth_objc_isa : OptInCC1FFlag<"ptrauth-objc-isa", "Enable signing and authentication of Objective-C object's 'isa' field">;
 defm ptrauth_objc_interface_sel : OptInCC1FFlag<"ptrauth-objc-interface-sel", "Enable signing and authentication of Objective-C object's 'SEL' fields">;
 defm ptrauth_objc_class_ro : OptInCC1FFlag<"ptrauth-objc-class-ro", "Enable signing and authentication for ObjC class_ro pointers">;
+defm ptrauth_block_descriptor_pointers : OptInCC1FFlag<"ptrauth-block-descriptor-pointers", "Enable signing and authentication of block descriptors">;
 }
 
 def fenable_matrix : Flag<["-"], "fenable-matrix">, Group<f_Group>,
@@ -5589,7 +5603,8 @@ def mno_outline_atomics : Flag<["-"], "mno-outline-atomics">, Group<f_clang_Grou
   HelpText<"Don't generate local calls to out-of-line atomic operations">;
 def mno_implicit_float : Flag<["-"], "mno-implicit-float">, Group<m_Group>,
   HelpText<"Don't generate implicit floating point or vector instructions">;
-def mimplicit_float : Flag<["-"], "mimplicit-float">, Group<m_Group>;
+def mimplicit_float : Flag<["-"], "mimplicit-float">, Group<m_Group>,
+  HelpText<"Generate implicit floating point or vector instructions">;
 def mrecip : Flag<["-"], "mrecip">, Group<m_Group>,
   Visibility<[ClangOption, CC1Option, FlangOption, FC1Option]>,
   HelpText<"Equivalent to '-mrecip=all'">;
@@ -6805,10 +6820,10 @@ def mapx_features_EQ : CommaJoined<["-"], "mapx-features=">, Group<m_x86_Feature
 def mno_apx_features_EQ : CommaJoined<["-"], "mno-apx-features=">, Group<m_x86_Features_Group>,
     HelpText<"Disable features of APX">, Values<"egpr,push2pop2,ppx,ndd,ccmp,nf,cf,zu">, Visibility<[ClangOption, CLOption, FlangOption]>;
 def mapxf : Flag<["-"], "mapxf">, Alias<mapx_features_EQ>,
-            AliasArgs<["egpr","push2pop2","ppx","ndd","ccmp","nf","cf","zu"]>,
+            AliasArgs<["egpr","push2pop2","ppx","ndd","ccmp","nf","zu"]>,
             Group<m_x86_Features_Group>;
 def mno_apxf : Flag<["-"], "mno-apxf">, Alias<mno_apx_features_EQ>,
-               AliasArgs<["egpr","push2pop2","ppx","ndd","ccmp","nf","cf","zu"]>,
+               AliasArgs<["egpr","push2pop2","ppx","ndd","ccmp","nf","zu"]>,
                Group<m_x86_Features_Group>;
 def mapx_inline_asm_use_gpr32 : Flag<["-"], "mapx-inline-asm-use-gpr32">, Group<m_Group>,
                                 HelpText<"Enable use of GPR32 in inline assembly for APX">;
@@ -6979,7 +6994,6 @@ def static_libgfortran : Flag<["-"], "static-libgfortran">, Group<gfortran_Group
 // "f" options with values for gfortran.
 def fblas_matmul_limit_EQ : Joined<["-"], "fblas-matmul-limit=">, Group<gfortran_Group>;
 def fcheck_EQ : Joined<["-"], "fcheck=">, Group<gfortran_Group>;
-def fcoarray_EQ : Joined<["-"], "fcoarray=">, Group<gfortran_Group>;
 def ffpe_trap_EQ : Joined<["-"], "ffpe-trap=">, Group<gfortran_Group>;
 def ffree_line_length_VALUE : Joined<["-"], "ffree-line-length-">, Group<gfortran_Group>;
 def finit_character_EQ : Joined<["-"], "finit-character=">, Group<gfortran_Group>;
@@ -8687,6 +8701,15 @@ def fopenmp_host_ir_file_path : Separate<["-"], "fopenmp-host-ir-file-path">,
 
 } // let Visibility = [CC1Option, FC1Option]
 
+//===----------------------------------------------------------------------===//
+// Coarray Options
+//===----------------------------------------------------------------------===//
+
+def fcoarray : Flag<["-"], "fcoarray">,
+               Group<f_Group>,
+               Visibility<[FlangOption, FC1Option]>,
+               HelpText<"Enable Coarray features">;
+
 //===----------------------------------------------------------------------===//
 // SYCL Options
 //===----------------------------------------------------------------------===//
@@ -9445,8 +9468,12 @@ def fspv_target_env_EQ : Joined<["-"], "fspv-target-env=">, Group<dxc_Group>,
 def fspv_extension_EQ
     : Joined<["-"], "fspv-extension=">,
       Group<dxc_Group>,
-      HelpText<"Specify the available SPIR-V extensions. If this option is not "
-               "specified, then all extensions are available.">;
+      HelpText<
+          "Specify the available SPIR-V extensions. If this option is not "
+          "specified, then all extensions are available. If KHR is specified, "
+          "then all KHR extensions will be available. If DXC is specifided, "
+          "then all extensions implemented by the DirectX Shader compiler will "
+          "be available. This option is useful for moving from DXC to Clang.">;
 def fvk_use_dx_layout
     : DXCFlag<"fvk-use-dx-layout">,
       HelpText<"Use DirectX memory layout for Vulkan resources.">;
diff --git a/clang/include/clang/Driver/ToolChain.h b/clang/include/clang/Driver/ToolChain.h
index 243056360370f..1425714d34110 100644
--- a/clang/include/clang/Driver/ToolChain.h
+++ b/clang/include/clang/Driver/ToolChain.h
@@ -224,9 +224,6 @@ class ToolChain {
   static void addSystemFrameworkInclude(const llvm::opt::ArgList &DriverArgs,
                                         llvm::opt::ArgStringList &CC1Args,
                                         const Twine &Path);
-  static void addSystemInclude(const llvm::opt::ArgList &DriverArgs,
-                               llvm::opt::ArgStringList &CC1Args,
-                               const Twine &Path);
   static void addExternCSystemInclude(const llvm::opt::ArgList &DriverArgs,
                                       llvm::opt::ArgStringList &CC1Args,
                                       const Twine &Path);
@@ -246,6 +243,9 @@ class ToolChain {
   ///@}
 
 public:
+  static void addSystemInclude(const llvm::opt::ArgList &DriverArgs,
+                               llvm::opt::ArgStringList &CC1Args,
+                               const Twine &Path);
   virtual ~ToolChain();
 
   // Accessors
diff --git a/clang/include/clang/ExtractAPI/DeclarationFragments.h b/clang/include/clang/ExtractAPI/DeclarationFragments.h
index 4ac744459031e..4859225ef4ced 100644
--- a/clang/include/clang/ExtractAPI/DeclarationFragments.h
+++ b/clang/include/clang/ExtractAPI/DeclarationFragments.h
@@ -440,9 +440,8 @@ class DeclarationFragmentsBuilder {
                                                   DeclarationFragments &);
 
   /// Build DeclarationFragments for a NestedNameSpecifier.
-  static DeclarationFragments getFragmentsForNNS(const NestedNameSpecifier *,
-                                                 ASTContext &,
-                                                 DeclarationFragments &);
+  static DeclarationFragments
+  getFragmentsForNNS(NestedNameSpecifier, ASTContext &, DeclarationFragments &);
 
   /// Build DeclarationFragments for Qualifiers.
   static DeclarationFragments getFragmentsForQualifiers(const Qualifiers quals);
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 31582a40de866..5dfdb23594610 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -4813,14 +4813,45 @@ struct FormatStyle {
   /// \version 7
   bool SpaceBeforeRangeBasedForLoopColon;
 
-  /// If ``true``, spaces will be inserted into ``{}``.
-  /// \code
-  ///    true:                                false:
-  ///    void f() { }                   vs.   void f() {}
-  ///    while (true) { }                     while (true) {}
-  /// \endcode
+  /// This option is **deprecated**. See ``Block`` of ``SpaceInEmptyBraces``.
   /// \version 10
-  bool SpaceInEmptyBlock;
+  // bool SpaceInEmptyBlock;
+
+  /// Style of when to insert a space in empty braces.
+  enum SpaceInEmptyBracesStyle : int8_t {
+    /// Always insert a space in empty braces.
+    /// \code
+    ///    void f() { }
+    ///    class Unit { };
+    ///    auto a = [] { };
+    ///    int x{ };
+    /// \endcode
+    SIEB_Always,
+    /// Only insert a space in empty blocks.
+    /// \code
+    ///    void f() { }
+    ///    class Unit { };
+    ///    auto a = [] { };
+    ///    int x{};
+    /// \endcode
+    SIEB_Block,
+    /// Never insert a space in empty braces.
+    /// \code
+    ///    void f() {}
+    ///    class Unit {};
+    ///    auto a = [] {};
+    ///    int x{};
+    /// \endcode
+    SIEB_Never
+  };
+
+  /// Specifies when to insert a space in empty braces.
+  /// \note
+  ///  This option doesn't apply to initializer braces if
+  ///  ``Cpp11BracedListStyle`` is set to ``true``.
+  /// \endnote
+  /// \version 22
+  SpaceInEmptyBracesStyle SpaceInEmptyBraces;
 
   /// If ``true``, spaces may be inserted into ``()``.
   /// This option is **deprecated**. See ``InEmptyParentheses`` of
@@ -5494,7 +5525,7 @@ struct FormatStyle {
            SpaceBeforeRangeBasedForLoopColon ==
                R.SpaceBeforeRangeBasedForLoopColon &&
            SpaceBeforeSquareBrackets == R.SpaceBeforeSquareBrackets &&
-           SpaceInEmptyBlock == R.SpaceInEmptyBlock &&
+           SpaceInEmptyBraces == R.SpaceInEmptyBraces &&
            SpacesBeforeTrailingComments == R.SpacesBeforeTrailingComments &&
            SpacesInAngles == R.SpacesInAngles &&
            SpacesInContainerLiterals == R.SpacesInContainerLiterals &&
diff --git a/clang/include/clang/Lex/DependencyDirectivesScanner.h b/clang/include/clang/Lex/DependencyDirectivesScanner.h
index f9fec3998ca53..c0b742d652a03 100644
--- a/clang/include/clang/Lex/DependencyDirectivesScanner.h
+++ b/clang/include/clang/Lex/DependencyDirectivesScanner.h
@@ -135,6 +135,13 @@ void printDependencyDirectivesAsSource(
     ArrayRef<dependency_directives_scan::Directive> Directives,
     llvm::raw_ostream &OS);
 
+/// Scan an input source buffer for C++20 named module usage.
+///
+/// \param Source The input source buffer.
+///
+/// \returns true if any C++20 named modules related directive was found.
+bool scanInputForCXX20ModulesUsage(StringRef Source);
+
 /// Functor that returns the dependency directives for a given file.
 class DependencyDirectivesGetter {
 public:
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index 06971ff87ab96..423f2ffe2f852 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -143,9 +143,6 @@ class Lexer : public PreprocessorLexer {
   /// True if this is the first time we're lexing the input file.
   bool IsFirstTimeLexingFile;
 
-  /// True if current lexing token is the first pp-token.
-  bool IsFirstPPToken;
-
   // NewLinePtr - A pointer to new line character '\n' being lexed. For '\r\n',
   // it also points to '\n.'
   const char *NewLinePtr;
diff --git a/clang/include/clang/Lex/NoTrivialPPDirectiveTracer.h b/clang/include/clang/Lex/NoTrivialPPDirectiveTracer.h
new file mode 100644
index 0000000000000..9ab3c6a528a1a
--- /dev/null
+++ b/clang/include/clang/Lex/NoTrivialPPDirectiveTracer.h
@@ -0,0 +1,310 @@
+//===--- NoTrivialPPDirectiveTracer.h ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the NoTrivialPPDirectiveTracer interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEX_NO_TRIVIAL_PPDIRECTIVE_TRACER_H
+#define LLVM_CLANG_LEX_NO_TRIVIAL_PPDIRECTIVE_TRACER_H
+
+#include "clang/Lex/PPCallbacks.h"
+
+namespace clang {
+class Preprocessor;
+
+/// Consider the following code:
+///
+/// # 1 __FILE__ 1 3
+/// export module a;
+///
+/// According to the wording in
+/// [P1857R3](https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p1857r3.html):
+///
+///   A module directive may only appear as the first preprocessing tokens in a
+///   file (excluding the global module fragment.)
+///
+/// and the wording in
+/// [[cpp.pre]](https://eel.is/c++draft/cpp.pre#nt:module-file):
+///   module-file:
+///     pp-global-module-fragment[opt] pp-module group[opt]
+///     pp-private-module-fragment[opt]
+///
+/// `#` is the first pp-token in the translation unit, and it was rejected by
+/// clang, but they really should be exempted from this rule. The goal is to not
+/// allow any preprocessor conditionals or most state changes, but these don't
+/// fit that.
+///
+/// State change would mean most semantically observable preprocessor state,
+/// particularly anything that is order dependent. Global flags like being a
+/// system header/module shouldn't matter.
+///
+/// We should exempt a brunch of directives, even though it violates the current
+/// standard wording.
+///
+/// This class used to trace 'no-trivial' pp-directives in main file, which may
+/// change the preprocessing state.
+///
+/// FIXME: Once the wording of the standard is revised, we need to follow the
+/// wording of the standard. Currently this is just a workaround
+class NoTrivialPPDirectiveTracer : public PPCallbacks {
+  Preprocessor &PP;
+
+  /// Whether preprocessing main file. We only focus on the main file.
+  bool InMainFile = true;
+
+  /// Whether one or more conditional, include or other 'no-trivial'
+  /// pp-directives has seen before.
+  bool SeenNoTrivialPPDirective = false;
+
+  void setSeenNoTrivialPPDirective();
+
+public:
+  NoTrivialPPDirectiveTracer(Preprocessor &P) : PP(P) {}
+
+  bool hasSeenNoTrivialPPDirective() const;
+
+  /// Callback invoked whenever the \p Lexer moves to a different file for
+  /// lexing. Unlike \p FileChanged line number directives and other related
+  /// pragmas do not trigger callbacks to \p LexedFileChanged.
+  ///
+  /// \param FID The \p FileID that the \p Lexer moved to.
+  ///
+  /// \param Reason Whether the \p Lexer entered a new file or exited one.
+  ///
+  /// \param FileType The \p CharacteristicKind of the file the \p Lexer moved
+  /// to.
+  ///
+  /// \param PrevFID The \p FileID the \p Lexer was using before the change.
+  ///
+  /// \param Loc The location where the \p Lexer entered a new file from or the
+  /// location that the \p Lexer moved into after exiting a file.
+  void LexedFileChanged(FileID FID, LexedFileChangeReason Reason,
+                        SrcMgr::CharacteristicKind FileType, FileID PrevFID,
+                        SourceLocation Loc) override;
+
+  /// Callback invoked whenever an embed directive has been processed,
+  /// regardless of whether the embed will actually find a file.
+  ///
+  /// \param HashLoc The location of the '#' that starts the embed directive.
+  ///
+  /// \param FileName The name of the file being included, as written in the
+  /// source code.
+  ///
+  /// \param IsAngled Whether the file name was enclosed in angle brackets;
+  /// otherwise, it was enclosed in quotes.
+  ///
+  /// \param File The actual file that may be included by this embed directive.
+  ///
+  /// \param Params The parameters used by the directive.
+  void EmbedDirective(SourceLocation HashLoc, StringRef FileName, bool IsAngled,
+                      OptionalFileEntryRef File,
+                      const LexEmbedParametersResult &Params) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Callback invoked whenever an inclusion directive of
+  /// any kind (\c \#include, \c \#import, etc.) has been processed, regardless
+  /// of whether the inclusion will actually result in an inclusion.
+  ///
+  /// \param HashLoc The location of the '#' that starts the inclusion
+  /// directive.
+  ///
+  /// \param IncludeTok The token that indicates the kind of inclusion
+  /// directive, e.g., 'include' or 'import'.
+  ///
+  /// \param FileName The name of the file being included, as written in the
+  /// source code.
+  ///
+  /// \param IsAngled Whether the file name was enclosed in angle brackets;
+  /// otherwise, it was enclosed in quotes.
+  ///
+  /// \param FilenameRange The character range of the quotes or angle brackets
+  /// for the written file name.
+  ///
+  /// \param File The actual file that may be included by this inclusion
+  /// directive.
+  ///
+  /// \param SearchPath Contains the search path which was used to find the file
+  /// in the file system. If the file was found via an absolute include path,
+  /// SearchPath will be empty. For framework includes, the SearchPath and
+  /// RelativePath will be split up. For example, if an include of "Some/Some.h"
+  /// is found via the framework path
+  /// "path/to/Frameworks/Some.framework/Headers/Some.h", SearchPath will be
+  /// "path/to/Frameworks/Some.framework/Headers" and RelativePath will be
+  /// "Some.h".
+  ///
+  /// \param RelativePath The path relative to SearchPath, at which the include
+  /// file was found. This is equal to FileName except for framework includes.
+  ///
+  /// \param SuggestedModule The module suggested for this header, if any.
+  ///
+  /// \param ModuleImported Whether this include was translated into import of
+  /// \p SuggestedModule.
+  ///
+  /// \param FileType The characteristic kind, indicates whether a file or
+  /// directory holds normal user code, system code, or system code which is
+  /// implicitly 'extern "C"' in C++ mode.
+  ///
+  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
+                          StringRef FileName, bool IsAngled,
+                          CharSourceRange FilenameRange,
+                          OptionalFileEntryRef File, StringRef SearchPath,
+                          StringRef RelativePath, const Module *SuggestedModule,
+                          bool ModuleImported,
+                          SrcMgr::CharacteristicKind FileType) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Callback invoked whenever there was an explicit module-import
+  /// syntax.
+  ///
+  /// \param ImportLoc The location of import directive token.
+  ///
+  /// \param Path The identifiers (and their locations) of the module
+  /// "path", e.g., "std.vector" would be split into "std" and "vector".
+  ///
+  /// \param Imported The imported module; can be null if importing failed.
+  ///
+  void moduleImport(SourceLocation ImportLoc, ModuleIdPath Path,
+                    const Module *Imported) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Callback invoked when the end of the main file is reached.
+  ///
+  /// No subsequent callbacks will be made.
+  void EndOfMainFile() override { setSeenNoTrivialPPDirective(); }
+
+  /// Callback invoked when start reading any pragma directive.
+  void PragmaDirective(SourceLocation Loc,
+                       PragmaIntroducerKind Introducer) override {}
+
+  /// Called by Preprocessor::HandleMacroExpandedIdentifier when a
+  /// macro invocation is found.
+  void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
+                    SourceRange Range, const MacroArgs *Args) override;
+
+  /// Hook called whenever a macro definition is seen.
+  void MacroDefined(const Token &MacroNameTok,
+                    const MacroDirective *MD) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever a macro \#undef is seen.
+  /// \param MacroNameTok The active Token
+  /// \param MD A MacroDefinition for the named macro.
+  /// \param Undef New MacroDirective if the macro was defined, null otherwise.
+  ///
+  /// MD is released immediately following this callback.
+  void MacroUndefined(const Token &MacroNameTok, const MacroDefinition &MD,
+                      const MacroDirective *Undef) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever the 'defined' operator is seen.
+  /// \param MD The MacroDirective if the name was a macro, null otherwise.
+  void Defined(const Token &MacroNameTok, const MacroDefinition &MD,
+               SourceRange Range) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever an \#if is seen.
+  /// \param Loc the source location of the directive.
+  /// \param ConditionRange The SourceRange of the expression being tested.
+  /// \param ConditionValue The evaluated value of the condition.
+  ///
+  // FIXME: better to pass in a list (or tree!) of Tokens.
+  void If(SourceLocation Loc, SourceRange ConditionRange,
+          ConditionValueKind ConditionValue) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever an \#elif is seen.
+  /// \param Loc the source location of the directive.
+  /// \param ConditionRange The SourceRange of the expression being tested.
+  /// \param ConditionValue The evaluated value of the condition.
+  /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive.
+  // FIXME: better to pass in a list (or tree!) of Tokens.
+  void Elif(SourceLocation Loc, SourceRange ConditionRange,
+            ConditionValueKind ConditionValue, SourceLocation IfLoc) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever an \#ifdef is seen.
+  /// \param Loc the source location of the directive.
+  /// \param MacroNameTok Information on the token being tested.
+  /// \param MD The MacroDefinition if the name was a macro, null otherwise.
+  void Ifdef(SourceLocation Loc, const Token &MacroNameTok,
+             const MacroDefinition &MD) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever an \#elifdef branch is taken.
+  /// \param Loc the source location of the directive.
+  /// \param MacroNameTok Information on the token being tested.
+  /// \param MD The MacroDefinition if the name was a macro, null otherwise.
+  void Elifdef(SourceLocation Loc, const Token &MacroNameTok,
+               const MacroDefinition &MD) override {
+    setSeenNoTrivialPPDirective();
+  }
+  /// Hook called whenever an \#elifdef is skipped.
+  /// \param Loc the source location of the directive.
+  /// \param ConditionRange The SourceRange of the expression being tested.
+  /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive.
+  // FIXME: better to pass in a list (or tree!) of Tokens.
+  void Elifdef(SourceLocation Loc, SourceRange ConditionRange,
+               SourceLocation IfLoc) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever an \#ifndef is seen.
+  /// \param Loc the source location of the directive.
+  /// \param MacroNameTok Information on the token being tested.
+  /// \param MD The MacroDefiniton if the name was a macro, null otherwise.
+  void Ifndef(SourceLocation Loc, const Token &MacroNameTok,
+              const MacroDefinition &MD) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever an \#elifndef branch is taken.
+  /// \param Loc the source location of the directive.
+  /// \param MacroNameTok Information on the token being tested.
+  /// \param MD The MacroDefinition if the name was a macro, null otherwise.
+  void Elifndef(SourceLocation Loc, const Token &MacroNameTok,
+                const MacroDefinition &MD) override {
+    setSeenNoTrivialPPDirective();
+  }
+  /// Hook called whenever an \#elifndef is skipped.
+  /// \param Loc the source location of the directive.
+  /// \param ConditionRange The SourceRange of the expression being tested.
+  /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive.
+  // FIXME: better to pass in a list (or tree!) of Tokens.
+  void Elifndef(SourceLocation Loc, SourceRange ConditionRange,
+                SourceLocation IfLoc) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever an \#else is seen.
+  /// \param Loc the source location of the directive.
+  /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive.
+  void Else(SourceLocation Loc, SourceLocation IfLoc) override {
+    setSeenNoTrivialPPDirective();
+  }
+
+  /// Hook called whenever an \#endif is seen.
+  /// \param Loc the source location of the directive.
+  /// \param IfLoc the source location of the \#if/\#ifdef/\#ifndef directive.
+  void Endif(SourceLocation Loc, SourceLocation IfLoc) override {
+    setSeenNoTrivialPPDirective();
+  }
+};
+
+} // namespace clang
+
+#endif // LLVM_CLANG_LEX_NO_TRIVIAL_PPDIRECTIVE_TRACER_H
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 71b0f8eab3bfa..39754847a93e4 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -82,6 +82,7 @@ class PreprocessorLexer;
 class PreprocessorOptions;
 class ScratchBuffer;
 class TargetInfo;
+class NoTrivialPPDirectiveTracer;
 
 namespace Builtin {
 class Context;
@@ -353,6 +354,11 @@ class Preprocessor {
   /// First pp-token source location in current translation unit.
   SourceLocation FirstPPTokenLoc;
 
+  /// A preprocessor directive tracer to trace whether the preprocessing
+  /// state changed. These changes would mean most semantically observable
+  /// preprocessor state, particularly anything that is order dependent.
+  NoTrivialPPDirectiveTracer *DirTracer = nullptr;
+
   /// A position within a C++20 import-seq.
   class StdCXXImportSeq {
   public:
@@ -609,6 +615,8 @@ class Preprocessor {
       return State == NamedModuleImplementation && !getName().contains(':');
     }
 
+    bool isNotAModuleDecl() const { return State == NotAModuleDecl; }
+
     StringRef getName() const {
       assert(isNamedModule() && "Can't get name from a non named module");
       return Name;
@@ -3091,6 +3099,10 @@ class Preprocessor {
   bool setDeserializedSafeBufferOptOutMap(
       const SmallVectorImpl<SourceLocation> &SrcLocSeqs);
 
+  /// Whether we've seen pp-directives which may have changed the preprocessing
+  /// state.
+  bool hasSeenNoTrivialPPDirective() const;
+
 private:
   /// Helper functions to forward lexing to the actual lexer. They all share the
   /// same signature.
diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index fc43e72593b94..d9dc5a562d802 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -86,12 +86,12 @@ class Token {
                                 // macro stringizing or charizing operator.
     CommaAfterElided = 0x200, // The comma following this token was elided (MS).
     IsEditorPlaceholder = 0x400, // This identifier is a placeholder.
-
-    IsReinjected = 0x800,  // A phase 4 token that was produced before and
-                           // re-added, e.g. via EnterTokenStream. Annotation
-                           // tokens are *not* reinjected.
-    FirstPPToken = 0x1000, // This token is the first pp token in the
-                           // translation unit.
+    IsReinjected = 0x800,        // A phase 4 token that was produced before and
+                          // re-added, e.g. via EnterTokenStream. Annotation
+                          // tokens are *not* reinjected.
+    HasSeenNoTrivialPPDirective =
+        0x1000, // Whether we've seen any 'no-trivial' pp-directives before
+                // current position.
   };
 
   tok::TokenKind getKind() const { return Kind; }
@@ -321,8 +321,9 @@ class Token {
   /// lexer uses identifier tokens to represent placeholders.
   bool isEditorPlaceholder() const { return getFlag(IsEditorPlaceholder); }
 
-  /// Returns true if this token is the first pp-token.
-  bool isFirstPPToken() const { return getFlag(FirstPPToken); }
+  bool hasSeenNoTrivialPPDirective() const {
+    return getFlag(HasSeenNoTrivialPPDirective);
+  }
 };
 
 /// Information about the conditional stack (\#if directives)
diff --git a/clang/include/clang/Sema/CodeCompleteConsumer.h b/clang/include/clang/Sema/CodeCompleteConsumer.h
index 2dd27593778c7..c26f4e33d289c 100644
--- a/clang/include/clang/Sema/CodeCompleteConsumer.h
+++ b/clang/include/clang/Sema/CodeCompleteConsumer.h
@@ -162,7 +162,8 @@ SimplifiedTypeClass getSimplifiedTypeClass(CanQualType T);
 
 /// Determine the type that this declaration will have if it is used
 /// as a type or in an expression.
-QualType getDeclUsageType(ASTContext &C, const NamedDecl *ND);
+QualType getDeclUsageType(ASTContext &C, NestedNameSpecifier Qualifier,
+                          const NamedDecl *ND);
 
 /// Determine the priority to be given to a macro code completion result
 /// with the given name.
@@ -867,7 +868,7 @@ class CodeCompletionResult {
   /// If the result should have a nested-name-specifier, this is it.
   /// When \c QualifierIsInformative, the nested-name-specifier is
   /// informative rather than required.
-  NestedNameSpecifier *Qualifier = nullptr;
+  NestedNameSpecifier Qualifier = std::nullopt;
 
   /// If this Decl was unshadowed by using declaration, this can store a
   /// pointer to the UsingShadowDecl which was used in the unshadowing process.
@@ -882,7 +883,7 @@ class CodeCompletionResult {
 
   /// Build a result that refers to a declaration.
   CodeCompletionResult(const NamedDecl *Declaration, unsigned Priority,
-                       NestedNameSpecifier *Qualifier = nullptr,
+                       NestedNameSpecifier Qualifier = std::nullopt,
                        bool QualifierIsInformative = false,
                        bool Accessible = true,
                        std::vector<FixItHint> FixIts = std::vector<FixItHint>())
diff --git a/clang/include/clang/Sema/DeclSpec.h b/clang/include/clang/Sema/DeclSpec.h
index e5680813e74de..c1a99a1fddc80 100644
--- a/clang/include/clang/Sema/DeclSpec.h
+++ b/clang/include/clang/Sema/DeclSpec.h
@@ -91,12 +91,11 @@ class CXXScopeSpec {
   }
 
   /// Retrieve the representation of the nested-name-specifier.
-  NestedNameSpecifier *getScopeRep() const {
+  NestedNameSpecifier getScopeRep() const {
     return Builder.getRepresentation();
   }
 
-  /// Extend the current nested-name-specifier by another
-  /// nested-name-specifier component of the form 'type::'.
+  /// Make a nested-name-specifier of the form 'type::'.
   ///
   /// \param Context The AST context in which this nested-name-specifier
   /// resides.
@@ -106,21 +105,7 @@ class CXXScopeSpec {
   /// \param TL The TypeLoc that describes the type preceding the '::'.
   ///
   /// \param ColonColonLoc The location of the trailing '::'.
-  void Extend(ASTContext &Context, TypeLoc TL, SourceLocation ColonColonLoc);
-
-  /// Extend the current nested-name-specifier by another
-  /// nested-name-specifier component of the form 'identifier::'.
-  ///
-  /// \param Context The AST context in which this nested-name-specifier
-  /// resides.
-  ///
-  /// \param Identifier The identifier.
-  ///
-  /// \param IdentifierLoc The location of the identifier.
-  ///
-  /// \param ColonColonLoc The location of the trailing '::'.
-  void Extend(ASTContext &Context, IdentifierInfo *Identifier,
-              SourceLocation IdentifierLoc, SourceLocation ColonColonLoc);
+  void Make(ASTContext &Context, TypeLoc TL, SourceLocation ColonColonLoc);
 
   /// Extend the current nested-name-specifier by another
   /// nested-name-specifier component of the form 'namespace::'.
@@ -154,8 +139,9 @@ class CXXScopeSpec {
   /// name.
   ///
   /// \param ColonColonLoc The location of the trailing '::'.
-  void MakeSuper(ASTContext &Context, CXXRecordDecl *RD,
-                 SourceLocation SuperLoc, SourceLocation ColonColonLoc);
+  void MakeMicrosoftSuper(ASTContext &Context, CXXRecordDecl *RD,
+                          SourceLocation SuperLoc,
+                          SourceLocation ColonColonLoc);
 
   /// Make a new nested-name-specifier from incomplete source-location
   /// information.
@@ -163,7 +149,7 @@ class CXXScopeSpec {
   /// FIXME: This routine should be used very, very rarely, in cases where we
   /// need to synthesize a nested-name-specifier. Most code should instead use
   /// \c Adopt() with a proper \c NestedNameSpecifierLoc.
-  void MakeTrivial(ASTContext &Context, NestedNameSpecifier *Qualifier,
+  void MakeTrivial(ASTContext &Context, NestedNameSpecifier Qualifier,
                    SourceRange R);
 
   /// Adopt an existing nested-name-specifier (with source-range
@@ -189,14 +175,14 @@ class CXXScopeSpec {
   SourceLocation getLastQualifierNameLoc() const;
 
   /// No scope specifier.
-  bool isEmpty() const { return Range.isInvalid() && getScopeRep() == nullptr; }
+  bool isEmpty() const { return Range.isInvalid() && !getScopeRep(); }
   /// A scope specifier is present, but may be valid or invalid.
   bool isNotEmpty() const { return !isEmpty(); }
 
   /// An error occurred during parsing of the scope specifier.
-  bool isInvalid() const { return Range.isValid() && getScopeRep() == nullptr; }
+  bool isInvalid() const { return Range.isValid() && !getScopeRep(); }
   /// A scope specifier is present, and it refers to a real scope.
-  bool isValid() const { return getScopeRep() != nullptr; }
+  bool isValid() const { return bool(getScopeRep()); }
 
   /// Indicate that this nested-name-specifier is invalid.
   void SetInvalid(SourceRange R) {
@@ -209,7 +195,7 @@ class CXXScopeSpec {
 
   /// Deprecated.  Some call sites intend isNotEmpty() while others intend
   /// isValid().
-  bool isSet() const { return getScopeRep() != nullptr; }
+  bool isSet() const { return bool(getScopeRep()); }
 
   void clear() {
     Range = SourceRange();
diff --git a/clang/include/clang/Sema/HeuristicResolver.h b/clang/include/clang/Sema/HeuristicResolver.h
index e193c0bc14cd9..71588bee92d16 100644
--- a/clang/include/clang/Sema/HeuristicResolver.h
+++ b/clang/include/clang/Sema/HeuristicResolver.h
@@ -67,8 +67,7 @@ class HeuristicResolver {
   // Try to heuristically resolve a dependent nested name specifier
   // to the type it likely denotes. Note that *dependent* name specifiers always
   // denote types, not namespaces.
-  QualType
-  resolveNestedNameSpecifierToType(const NestedNameSpecifier *NNS) const;
+  QualType resolveNestedNameSpecifierToType(NestedNameSpecifier NNS) const;
 
   // Perform an imprecise lookup of a dependent name in `RD`.
   // This function does not follow strict semantic rules and should be used
diff --git a/clang/include/clang/Sema/ParsedTemplate.h b/clang/include/clang/Sema/ParsedTemplate.h
index 3a8050f9a0a3d..4a3df781b5a58 100644
--- a/clang/include/clang/Sema/ParsedTemplate.h
+++ b/clang/include/clang/Sema/ParsedTemplate.h
@@ -48,8 +48,8 @@ namespace clang {
     ///
     /// \param Arg the template type argument or non-type template argument.
     /// \param Loc the location of the type.
-    ParsedTemplateArgument(KindType Kind, void *Arg, SourceLocation Loc)
-      : Kind(Kind), Arg(Arg), Loc(Loc) { }
+    ParsedTemplateArgument(KindType Kind, void *Arg, SourceLocation NameLoc)
+        : Kind(Kind), Arg(Arg), NameLoc(NameLoc) {}
 
     /// Create a template template argument.
     ///
@@ -60,11 +60,11 @@ namespace clang {
     /// argument refers.
     ///
     /// \param TemplateLoc the location of the template name.
-    ParsedTemplateArgument(const CXXScopeSpec &SS,
-                           ParsedTemplateTy Template,
-                           SourceLocation TemplateLoc)
-      : Kind(ParsedTemplateArgument::Template),
-        Arg(Template.getAsOpaquePtr()), SS(SS), Loc(TemplateLoc) {}
+    ParsedTemplateArgument(SourceLocation TemplateKwLoc, const CXXScopeSpec &SS,
+                           ParsedTemplateTy Template, SourceLocation NameLoc)
+        : Kind(ParsedTemplateArgument::Template),
+          Arg(Template.getAsOpaquePtr()), SS(SS), TemplateKwLoc(TemplateKwLoc),
+          NameLoc(NameLoc) {}
 
     /// Determine whether the given template argument is invalid.
     bool isInvalid() const { return Arg == nullptr; }
@@ -91,7 +91,10 @@ namespace clang {
     }
 
     /// Retrieve the location of the template argument.
-    SourceLocation getLocation() const { return Loc; }
+    SourceLocation getTemplateKwLoc() const { return TemplateKwLoc; }
+
+    /// Retrieve the location of the template argument.
+    SourceLocation getNameLoc() const { return NameLoc; }
 
     /// Retrieve the nested-name-specifier that precedes the template
     /// name in a template template argument.
@@ -128,8 +131,11 @@ namespace clang {
     /// argument.
     CXXScopeSpec SS;
 
-    /// the location of the template argument.
-    SourceLocation Loc;
+    /// the location of the template keyword.
+    SourceLocation TemplateKwLoc;
+
+    /// the location of the template name.
+    SourceLocation NameLoc;
 
     /// The ellipsis location that can accompany a template template
     /// argument (turning it into a template template argument expansion).
diff --git a/clang/include/clang/Sema/ScopeInfo.h b/clang/include/clang/Sema/ScopeInfo.h
index 94b247a689c2d..4f4d38c961140 100644
--- a/clang/include/clang/Sema/ScopeInfo.h
+++ b/clang/include/clang/Sema/ScopeInfo.h
@@ -933,7 +933,7 @@ class LambdaScopeInfo final :
   ///  to local variables that are usable as constant expressions and
   ///  do not involve an odr-use (they may still need to be captured
   ///  if the enclosing full-expression is instantiation dependent).
-  llvm::SmallSet<Expr *, 8> NonODRUsedCapturingExprs;
+  llvm::SmallPtrSet<Expr *, 8> NonODRUsedCapturingExprs;
 
   /// A map of explicit capture indices to their introducer source ranges.
   llvm::DenseMap<unsigned, SourceRange> ExplicitCaptureRanges;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 5211373367677..a78f5635f44da 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -228,7 +228,9 @@ void threadSafetyCleanup(BeforeSet *Cache);
 
 // FIXME: No way to easily map from TemplateTypeParmTypes to
 // TemplateTypeParmDecls, so we have this horrible PointerUnion.
-typedef std::pair<llvm::PointerUnion<const TemplateTypeParmType *, NamedDecl *>,
+typedef std::pair<llvm::PointerUnion<const TemplateTypeParmType *, NamedDecl *,
+                                     const TemplateSpecializationType *,
+                                     const SubstBuiltinTemplatePackType *>,
                   SourceLocation>
     UnexpandedParameterPack;
 
@@ -2659,9 +2661,9 @@ class Sema final : public SemaBase {
   /// identifies the magic value.
   typedef std::pair<const IdentifierInfo *, uint64_t> TypeTagMagicValue;
 
-  /// Diagnoses the current set of gathered accesses. This typically
-  /// happens at full expression level. The set is cleared after emitting the
-  /// diagnostics.
+  /// Diagnoses the current set of gathered accesses. This happens at the end of
+  /// each expression evaluation context. Diagnostics are emitted only for
+  /// accesses gathered in the current evaluation context.
   void DiagnoseMisalignedMembers();
 
   /// This function checks if the expression is in the sef of potentially
@@ -3117,9 +3119,6 @@ class Sema final : public SemaBase {
 
     bool operator==(const MisalignedMember &m) { return this->E == m.E; }
   };
-  /// Small set of gathered accesses to potentially misaligned members
-  /// due to the packed attribute.
-  SmallVector<MisalignedMember, 4> MisalignedMembers;
 
   /// Adds an expression to the set of gathered misaligned members.
   void AddPotentialMisalignedMembers(Expr *E, RecordDecl *RD, ValueDecl *MD,
@@ -3225,7 +3224,7 @@ class Sema final : public SemaBase {
   /// current instantiation (C++0x [temp.dep.type]p1).
   ///
   /// \param NNS a dependent nested name specifier.
-  CXXRecordDecl *getCurrentInstantiationOf(NestedNameSpecifier *NNS);
+  CXXRecordDecl *getCurrentInstantiationOf(NestedNameSpecifier NNS);
 
   /// The parser has parsed a global nested-name-specifier '::'.
   ///
@@ -3262,7 +3261,7 @@ class Sema final : public SemaBase {
   /// (e.g., Base::), perform name lookup for that identifier as a
   /// nested-name-specifier within the given scope, and return the result of
   /// that name lookup.
-  NamedDecl *FindFirstQualifierInScope(Scope *S, NestedNameSpecifier *NNS);
+  NamedDecl *FindFirstQualifierInScope(Scope *S, NestedNameSpecifier NNS);
 
   /// Keeps information about an identifier in a nested-name-spec.
   ///
@@ -3581,8 +3580,8 @@ class Sema final : public SemaBase {
   /// Returns the TypeDeclType for the given type declaration,
   /// as ASTContext::getTypeDeclType would, but
   /// performs the required semantic checks for name lookup of said entity.
-  QualType getTypeDeclType(DeclContext *LookupCtx, DiagCtorKind DCK,
-                           TypeDecl *TD, SourceLocation NameLoc);
+  void checkTypeDeclType(DeclContext *LookupCtx, DiagCtorKind DCK, TypeDecl *TD,
+                         SourceLocation NameLoc);
 
   /// If the identifier refers to a type name within this scope,
   /// return the declaration of that type.
@@ -4179,8 +4178,15 @@ class Sema final : public SemaBase {
   /// return statement in the scope of a variable has the same NRVO candidate,
   /// that candidate is an NRVO variable.
   void computeNRVO(Stmt *Body, sema::FunctionScopeInfo *Scope);
-  Decl *ActOnFinishFunctionBody(Decl *Decl, Stmt *Body);
-  Decl *ActOnFinishFunctionBody(Decl *Decl, Stmt *Body, bool IsInstantiation);
+
+  /// Performs semantic analysis at the end of a function body.
+  ///
+  /// \param RetainFunctionScopeInfo If \c true, the client is responsible for
+  /// releasing the associated \p FunctionScopeInfo. This is useful when
+  /// building e.g. LambdaExprs.
+  Decl *ActOnFinishFunctionBody(Decl *Decl, Stmt *Body,
+                                bool IsInstantiation = false,
+                                bool RetainFunctionScopeInfo = false);
   Decl *ActOnSkippedFunctionBody(Decl *Decl);
   void ActOnFinishInlineFunctionDef(FunctionDecl *D);
 
@@ -6765,6 +6771,10 @@ class Sema final : public SemaBase {
     /// InLifetimeExtendingContext is true.
     SmallVector<MaterializeTemporaryExpr *, 8> ForRangeLifetimeExtendTemps;
 
+    /// Small set of gathered accesses to potentially misaligned members
+    /// due to the packed attribute.
+    SmallVector<MisalignedMember, 4> MisalignedMembers;
+
     /// \brief Describes whether we are in an expression constext which we have
     /// to handle differently.
     enum ExpressionKind {
@@ -6873,23 +6883,23 @@ class Sema final : public SemaBase {
     assert(!ExprEvalContexts.empty() &&
            "Must be in an expression evaluation context");
     return ExprEvalContexts.back();
-  };
+  }
 
   ExpressionEvaluationContextRecord &currentEvaluationContext() {
     assert(!ExprEvalContexts.empty() &&
            "Must be in an expression evaluation context");
     return ExprEvalContexts.back();
-  };
+  }
 
   ExpressionEvaluationContextRecord &parentEvaluationContext() {
     assert(ExprEvalContexts.size() >= 2 &&
            "Must be in an expression evaluation context");
     return ExprEvalContexts[ExprEvalContexts.size() - 2];
-  };
+  }
 
   const ExpressionEvaluationContextRecord &parentEvaluationContext() const {
     return const_cast<Sema *>(this)->parentEvaluationContext();
-  };
+  }
 
   bool isAttrContext() const {
     return ExprEvalContexts.back().ExprContext ==
@@ -7618,7 +7628,7 @@ class Sema final : public SemaBase {
   /// "real" base class is checked as appropriate when checking the access of
   /// the member name.
   ExprResult PerformObjectMemberConversion(Expr *From,
-                                           NestedNameSpecifier *Qualifier,
+                                           NestedNameSpecifier Qualifier,
                                            NamedDecl *FoundDecl,
                                            NamedDecl *Member);
 
@@ -8055,8 +8065,8 @@ class Sema final : public SemaBase {
                                         ExprResult &RHS);
 
   QualType CheckMultiplyDivideOperands( // C99 6.5.5
-      ExprResult &LHS, ExprResult &RHS, SourceLocation Loc, bool IsCompAssign,
-      bool IsDivide);
+      ExprResult &LHS, ExprResult &RHS, SourceLocation Loc,
+      BinaryOperatorKind Opc);
   QualType CheckRemainderOperands( // C99 6.5.5
       ExprResult &LHS, ExprResult &RHS, SourceLocation Loc,
       bool IsCompAssign = false);
@@ -8065,7 +8075,7 @@ class Sema final : public SemaBase {
       BinaryOperatorKind Opc, QualType *CompLHSTy = nullptr);
   QualType CheckSubtractionOperands( // C99 6.5.6
       ExprResult &LHS, ExprResult &RHS, SourceLocation Loc,
-      QualType *CompLHSTy = nullptr);
+      BinaryOperatorKind Opc, QualType *CompLHSTy = nullptr);
   QualType CheckShiftOperands( // C99 6.5.7
       ExprResult &LHS, ExprResult &RHS, SourceLocation Loc,
       BinaryOperatorKind Opc, bool IsCompAssign = false);
@@ -9139,8 +9149,7 @@ class Sema final : public SemaBase {
 
   /// Complete a lambda-expression having processed and attached the
   /// lambda body.
-  ExprResult BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
-                             sema::LambdaScopeInfo *LSI);
+  ExprResult BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc);
 
   /// Get the return type to use for a lambda's conversion function(s) to
   /// function pointer type, given the type of the call operator.
@@ -9836,7 +9845,7 @@ class Sema final : public SemaBase {
                                  SourceLocation ModuleLoc, ModuleDeclKind MDK,
                                  ModuleIdPath Path, ModuleIdPath Partition,
                                  ModuleImportState &ImportState,
-                                 bool IntroducerIsFirstPPToken);
+                                 bool SeenNoTrivialPPDirective);
 
   /// The parser has processed a global-module-fragment declaration that begins
   /// the definition of the global module fragment of the current module unit.
@@ -10210,7 +10219,7 @@ class Sema final : public SemaBase {
   ExprResult InitializeExplicitObjectArgument(Sema &S, Expr *Obj,
                                               FunctionDecl *Fun);
   ExprResult PerformImplicitObjectArgumentInitialization(
-      Expr *From, NestedNameSpecifier *Qualifier, NamedDecl *FoundDecl,
+      Expr *From, NestedNameSpecifier Qualifier, NamedDecl *FoundDecl,
       CXXMethodDecl *Method);
 
   /// PerformContextuallyConvertToBool - Perform a contextual conversion
@@ -11618,13 +11627,16 @@ class Sema final : public SemaBase {
 
   void NoteAllFoundTemplates(TemplateName Name);
 
-  QualType CheckTemplateIdType(TemplateName Template,
+  QualType CheckTemplateIdType(ElaboratedTypeKeyword Keyword,
+                               TemplateName Template,
                                SourceLocation TemplateLoc,
                                TemplateArgumentListInfo &TemplateArgs);
 
   TypeResult
-  ActOnTemplateIdType(Scope *S, CXXScopeSpec &SS, SourceLocation TemplateKWLoc,
-                      TemplateTy Template, const IdentifierInfo *TemplateII,
+  ActOnTemplateIdType(Scope *S, ElaboratedTypeKeyword ElaboratedKeyword,
+                      SourceLocation ElaboratedKeywordLoc, CXXScopeSpec &SS,
+                      SourceLocation TemplateKWLoc, TemplateTy Template,
+                      const IdentifierInfo *TemplateII,
                       SourceLocation TemplateIILoc, SourceLocation LAngleLoc,
                       ASTTemplateArgsPtr TemplateArgs, SourceLocation RAngleLoc,
                       bool IsCtorOrDtorName = false, bool IsClassName = false,
@@ -11858,8 +11870,8 @@ class Sema final : public SemaBase {
   /// argument, substitute into that default template argument and
   /// return the corresponding template argument.
   TemplateArgumentLoc SubstDefaultTemplateArgumentIfAvailable(
-      TemplateDecl *Template, SourceLocation TemplateLoc,
-      SourceLocation RAngleLoc, Decl *Param,
+      TemplateDecl *Template, SourceLocation TemplateKWLoc,
+      SourceLocation TemplateNameLoc, SourceLocation RAngleLoc, Decl *Param,
       ArrayRef<TemplateArgument> SugaredConverted,
       ArrayRef<TemplateArgument> CanonicalConverted, bool &HasDefaultArg);
 
@@ -13487,8 +13499,6 @@ class Sema final : public SemaBase {
     ~ArgPackSubstIndexRAII() { Self.ArgPackSubstIndex = OldSubstIndex; }
   };
 
-  friend class ArgumentPackSubstitutionRAII;
-
   void pushCodeSynthesisContext(CodeSynthesisContext Ctx);
   void popCodeSynthesisContext();
 
@@ -13762,8 +13772,9 @@ class Sema final : public SemaBase {
   SubstDeclarationNameInfo(const DeclarationNameInfo &NameInfo,
                            const MultiLevelTemplateArgumentList &TemplateArgs);
   TemplateName
-  SubstTemplateName(NestedNameSpecifierLoc QualifierLoc, TemplateName Name,
-                    SourceLocation Loc,
+  SubstTemplateName(SourceLocation TemplateKWLoc,
+                    NestedNameSpecifierLoc &QualifierLoc, TemplateName Name,
+                    SourceLocation NameLoc,
                     const MultiLevelTemplateArgumentList &TemplateArgs);
 
   bool SubstTypeConstraint(TemplateTypeParmDecl *Inst, const TypeConstraint *TC,
@@ -14417,6 +14428,15 @@ class Sema final : public SemaBase {
   static void collectUnexpandedParameterPacks(
       Expr *E, SmallVectorImpl<UnexpandedParameterPack> &Unexpanded);
 
+  /// Invoked when parsing a template argument.
+  ///
+  /// \param Arg the template argument, which may already be invalid.
+  ///
+  /// If it is followed by ellipsis, this function is called before
+  /// `ActOnPackExpansion`.
+  ParsedTemplateArgument
+  ActOnTemplateTemplateArgument(const ParsedTemplateArgument &Arg);
+
   /// Invoked when parsing a template argument followed by an
   /// ellipsis, which creates a pack expansion.
   ///
@@ -14504,7 +14524,8 @@ class Sema final : public SemaBase {
   bool CheckParameterPacksForExpansion(
       SourceLocation EllipsisLoc, SourceRange PatternRange,
       ArrayRef<UnexpandedParameterPack> Unexpanded,
-      const MultiLevelTemplateArgumentList &TemplateArgs, bool &ShouldExpand,
+      const MultiLevelTemplateArgumentList &TemplateArgs,
+      bool FailOnPackProducingTemplates, bool &ShouldExpand,
       bool &RetainExpansion, UnsignedOrNone &NumExpansions);
 
   /// Determine the number of arguments in the given pack expansion
@@ -15179,14 +15200,6 @@ class Sema final : public SemaBase {
     return RequireCompleteExprType(E, CompleteTypeKind::Default, Diagnoser);
   }
 
-  /// Retrieve a version of the type 'T' that is elaborated by Keyword,
-  /// qualified by the nested-name-specifier contained in SS, and that is
-  /// (re)declared by OwnedTagDecl, which is nullptr if this is not a
-  /// (re)declaration.
-  QualType getElaboratedType(ElaboratedTypeKeyword Keyword,
-                             const CXXScopeSpec &SS, QualType T,
-                             TagDecl *OwnedTagDecl = nullptr);
-
   // Returns the underlying type of a decltype with the given expression.
   QualType getDecltypeForExpr(Expr *E);
 
@@ -15331,6 +15344,16 @@ class Sema final : public SemaBase {
     NamedDecl *Hidden;
     return hasVisibleDefinition(const_cast<NamedDecl *>(D), &Hidden);
   }
+  /// Determine if \p D has a definition which allows we redefine it in current
+  /// TU. \p Suggested is the definition that should be made visible to expose
+  /// the definition.
+  bool isRedefinitionAllowedFor(NamedDecl *D, NamedDecl **Suggested,
+                                bool &Visible);
+  bool isRedefinitionAllowedFor(const NamedDecl *D, bool &Visible) {
+    NamedDecl *Hidden;
+    return isRedefinitionAllowedFor(const_cast<NamedDecl *>(D), &Hidden,
+                                    Visible);
+  }
 
   /// Determine if \p D has a reachable definition. If not, suggest a
   /// declaration that should be made reachable to expose the definition.
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index 085c9ed9f3ebd..016456f241eed 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -229,10 +229,17 @@ class SemaHLSL : public SemaBase {
 
   void diagnoseAvailabilityViolations(TranslationUnitDecl *TU);
 
-  bool initGlobalResourceDecl(VarDecl *VD);
   uint32_t getNextImplicitBindingOrderID() {
     return ImplicitBindingNextOrderID++;
   }
+
+  bool initGlobalResourceDecl(VarDecl *VD);
+  bool initGlobalResourceArrayDecl(VarDecl *VD);
+  void createResourceRecordCtorArgs(const Type *ResourceTy, StringRef VarName,
+                                    HLSLResourceBindingAttr *RBA,
+                                    HLSLVkBindingAttr *VkBinding,
+                                    uint32_t ArrayIndex,
+                                    llvm::SmallVectorImpl<Expr *> &Args);
 };
 
 } // namespace clang
diff --git a/clang/include/clang/Sema/SemaInternal.h b/clang/include/clang/Sema/SemaInternal.h
index 42c9469e44e53..8f6041b5f00e4 100644
--- a/clang/include/clang/Sema/SemaInternal.h
+++ b/clang/include/clang/Sema/SemaInternal.h
@@ -71,12 +71,17 @@ inline std::pair<unsigned, unsigned> getDepthAndIndex(const NamedDecl *ND) {
 }
 
 /// Retrieve the depth and index of an unexpanded parameter pack.
-inline std::pair<unsigned, unsigned>
+/// Returns nullopt when the unexpanded packs do not correspond to template
+/// parameters, e.g. __builtin_dedup_types.
+inline std::optional<std::pair<unsigned, unsigned>>
 getDepthAndIndex(UnexpandedParameterPack UPP) {
   if (const auto *TTP = dyn_cast<const TemplateTypeParmType *>(UPP.first))
     return std::make_pair(TTP->getDepth(), TTP->getIndex());
-
-  return getDepthAndIndex(cast<NamedDecl *>(UPP.first));
+  if (isa<NamedDecl *>(UPP.first))
+    return getDepthAndIndex(cast<NamedDecl *>(UPP.first));
+  assert((isa<const TemplateSpecializationType *,
+              const SubstBuiltinTemplatePackType *>(UPP.first)));
+  return std::nullopt;
 }
 
 class TypoCorrectionConsumer : public VisibleDeclConsumer {
@@ -209,7 +214,7 @@ class TypoCorrectionConsumer : public VisibleDeclConsumer {
   class NamespaceSpecifierSet {
     struct SpecifierInfo {
       DeclContext* DeclCtx;
-      NestedNameSpecifier* NameSpecifier;
+      NestedNameSpecifier NameSpecifier;
       unsigned EditDistance;
     };
 
@@ -229,9 +234,9 @@ class TypoCorrectionConsumer : public VisibleDeclConsumer {
     static DeclContextList buildContextChain(DeclContext *Start);
 
     unsigned buildNestedNameSpecifier(DeclContextList &DeclChain,
-                                      NestedNameSpecifier *&NNS);
+                                      NestedNameSpecifier &NNS);
 
-   public:
+  public:
     NamespaceSpecifierSet(ASTContext &Context, DeclContext *CurContext,
                           CXXScopeSpec *CurScopeSpec);
 
@@ -276,7 +281,7 @@ class TypoCorrectionConsumer : public VisibleDeclConsumer {
   };
 
   void addName(StringRef Name, NamedDecl *ND,
-               NestedNameSpecifier *NNS = nullptr, bool isKeyword = false);
+               NestedNameSpecifier NNS = std::nullopt, bool isKeyword = false);
 
   /// Find any visible decls for the given typo correction candidate.
   /// If none are found, it to the set of candidates for which qualified lookups
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 964749c2fd627..e7b62aafc060b 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -240,7 +240,11 @@ class SemaOpenACC : public SemaBase {
   // Creates a VarDecl with a proper default init for the purposes of a
   // `private`/'firstprivate'/'reduction' clause, so it can be used to generate
   // a recipe later.
-  VarDecl *CreateInitRecipe(OpenACCClauseKind CK, const Expr *VarExpr);
+  //  The first entry is the recipe itself, the second is any required
+  //  'temporary' created for the init (in the case of a copy), such as with
+  //  firstprivate.
+  std::pair<VarDecl *, VarDecl *> CreateInitRecipe(OpenACCClauseKind CK,
+                                                   const Expr *VarExpr);
 
 public:
   ComputeConstructInfo &getActiveComputeConstructInfo() {
@@ -943,12 +947,12 @@ class SemaOpenACC : public SemaBase {
                   ArrayRef<Expr *> IntExprs, SourceLocation EndLoc);
   // Does the checking for a 'reduction ' clause that needs to be done in
   // dependent and not dependent cases.
-  OpenACCClause *
-  CheckReductionClause(ArrayRef<const OpenACCClause *> ExistingClauses,
-                       OpenACCDirectiveKind DirectiveKind,
-                       SourceLocation BeginLoc, SourceLocation LParenLoc,
-                       OpenACCReductionOperator ReductionOp,
-                       ArrayRef<Expr *> Vars, SourceLocation EndLoc);
+  OpenACCClause *CheckReductionClause(
+      ArrayRef<const OpenACCClause *> ExistingClauses,
+      OpenACCDirectiveKind DirectiveKind, SourceLocation BeginLoc,
+      SourceLocation LParenLoc, OpenACCReductionOperator ReductionOp,
+      ArrayRef<Expr *> Vars, ArrayRef<OpenACCReductionRecipe> Recipes,
+      SourceLocation EndLoc);
 
   ExprResult BuildOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc);
   ExprResult ActOnOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc);
diff --git a/clang/include/clang/Sema/SemaSYCL.h b/clang/include/clang/Sema/SemaSYCL.h
index b47b2f155ef93..7ae556da2bec1 100644
--- a/clang/include/clang/Sema/SemaSYCL.h
+++ b/clang/include/clang/Sema/SemaSYCL.h
@@ -64,6 +64,7 @@ class SemaSYCL : public SemaBase {
   void handleKernelAttr(Decl *D, const ParsedAttr &AL);
   void handleKernelEntryPointAttr(Decl *D, const ParsedAttr &AL);
 
+  void CheckSYCLExternalFunctionDecl(FunctionDecl *FD);
   void CheckSYCLEntryPointFunctionDecl(FunctionDecl *FD);
   StmtResult BuildSYCLKernelCallStmt(FunctionDecl *FD, CompoundStmt *Body);
 };
diff --git a/clang/include/clang/Sema/SemaWasm.h b/clang/include/clang/Sema/SemaWasm.h
index 8c0639fd7e76f..f82590755d183 100644
--- a/clang/include/clang/Sema/SemaWasm.h
+++ b/clang/include/clang/Sema/SemaWasm.h
@@ -37,7 +37,8 @@ class SemaWasm : public SemaBase {
   bool BuiltinWasmTableGrow(CallExpr *TheCall);
   bool BuiltinWasmTableFill(CallExpr *TheCall);
   bool BuiltinWasmTableCopy(CallExpr *TheCall);
-  bool BuiltinWasmTestFunctionPointerSignature(CallExpr *TheCall);
+  bool BuiltinWasmTestFunctionPointerSignature(const TargetInfo &TI,
+                                               CallExpr *TheCall);
 
   WebAssemblyImportNameAttr *
   mergeImportNameAttr(Decl *D, const WebAssemblyImportNameAttr &AL);
diff --git a/clang/include/clang/Sema/TypoCorrection.h b/clang/include/clang/Sema/TypoCorrection.h
index 09de164297e7b..1d780c45efd55 100644
--- a/clang/include/clang/Sema/TypoCorrection.h
+++ b/clang/include/clang/Sema/TypoCorrection.h
@@ -57,15 +57,15 @@ class TypoCorrection {
   static const unsigned CallbackDistanceWeight = 150U;
 
   TypoCorrection(const DeclarationName &Name, NamedDecl *NameDecl,
-                 NestedNameSpecifier *NNS = nullptr, unsigned CharDistance = 0,
-                 unsigned QualifierDistance = 0)
+                 NestedNameSpecifier NNS = std::nullopt,
+                 unsigned CharDistance = 0, unsigned QualifierDistance = 0)
       : CorrectionName(Name), CorrectionNameSpec(NNS),
         CharDistance(CharDistance), QualifierDistance(QualifierDistance) {
     if (NameDecl)
       CorrectionDecls.push_back(NameDecl);
   }
 
-  TypoCorrection(NamedDecl *Name, NestedNameSpecifier *NNS = nullptr,
+  TypoCorrection(NamedDecl *Name, NestedNameSpecifier NNS = std::nullopt,
                  unsigned CharDistance = 0)
       : CorrectionName(Name->getDeclName()), CorrectionNameSpec(NNS),
         CharDistance(CharDistance) {
@@ -73,7 +73,7 @@ class TypoCorrection {
       CorrectionDecls.push_back(Name);
   }
 
-  TypoCorrection(DeclarationName Name, NestedNameSpecifier *NNS = nullptr,
+  TypoCorrection(DeclarationName Name, NestedNameSpecifier NNS = std::nullopt,
                  unsigned CharDistance = 0)
       : CorrectionName(Name), CorrectionNameSpec(NNS),
         CharDistance(CharDistance) {}
@@ -88,13 +88,13 @@ class TypoCorrection {
   }
 
   /// Gets the NestedNameSpecifier needed to use the typo correction
-  NestedNameSpecifier *getCorrectionSpecifier() const {
+  NestedNameSpecifier getCorrectionSpecifier() const {
     return CorrectionNameSpec;
   }
 
-  void setCorrectionSpecifier(NestedNameSpecifier *NNS) {
+  void setCorrectionSpecifier(NestedNameSpecifier NNS) {
     CorrectionNameSpec = NNS;
-    ForceSpecifierReplacement = (NNS != nullptr);
+    ForceSpecifierReplacement = !!NNS;
   }
 
   void WillReplaceSpecifier(bool ForceReplacement) {
@@ -264,7 +264,7 @@ class TypoCorrection {
 
   // Results.
   DeclarationName CorrectionName;
-  NestedNameSpecifier *CorrectionNameSpec = nullptr;
+  NestedNameSpecifier CorrectionNameSpec = std::nullopt;
   SmallVector<NamedDecl *, 1> CorrectionDecls;
   unsigned CharDistance = 0;
   unsigned QualifierDistance = 0;
@@ -282,8 +282,9 @@ class CorrectionCandidateCallback {
 public:
   static const unsigned InvalidDistance = TypoCorrection::InvalidDistance;
 
-  explicit CorrectionCandidateCallback(const IdentifierInfo *Typo = nullptr,
-                                       NestedNameSpecifier *TypoNNS = nullptr)
+  explicit CorrectionCandidateCallback(
+      const IdentifierInfo *Typo = nullptr,
+      NestedNameSpecifier TypoNNS = std::nullopt)
       : Typo(Typo), TypoNNS(TypoNNS) {}
 
   virtual ~CorrectionCandidateCallback() = default;
@@ -320,7 +321,7 @@ class CorrectionCandidateCallback {
   virtual std::unique_ptr<CorrectionCandidateCallback> clone() = 0;
 
   void setTypoName(const IdentifierInfo *II) { Typo = II; }
-  void setTypoNNS(NestedNameSpecifier *NNS) { TypoNNS = NNS; }
+  void setTypoNNS(NestedNameSpecifier NNS) { TypoNNS = NNS; }
 
   // Flags for context-dependent keywords. WantFunctionLikeCasts is only
   // used/meaningful when WantCXXNamedCasts is false.
@@ -346,13 +347,13 @@ class CorrectionCandidateCallback {
   }
 
   const IdentifierInfo *Typo;
-  NestedNameSpecifier *TypoNNS;
+  NestedNameSpecifier TypoNNS;
 };
 
 class DefaultFilterCCC final : public CorrectionCandidateCallback {
 public:
   explicit DefaultFilterCCC(const IdentifierInfo *Typo = nullptr,
-                            NestedNameSpecifier *TypoNNS = nullptr)
+                            NestedNameSpecifier TypoNNS = std::nullopt)
       : CorrectionCandidateCallback(Typo, TypoNNS) {}
 
   std::unique_ptr<CorrectionCandidateCallback> clone() override {
@@ -366,7 +367,7 @@ template <class C>
 class DeclFilterCCC final : public CorrectionCandidateCallback {
 public:
   explicit DeclFilterCCC(const IdentifierInfo *Typo = nullptr,
-                         NestedNameSpecifier *TypoNNS = nullptr)
+                         NestedNameSpecifier TypoNNS = std::nullopt)
       : CorrectionCandidateCallback(Typo, TypoNNS) {}
 
   bool ValidateCandidate(const TypoCorrection &candidate) override {
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 7ce626cedae74..3ede0925676e8 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -526,6 +526,9 @@ class ASTReader
   /// A timer used to track the time spent deserializing.
   std::unique_ptr<llvm::Timer> ReadTimer;
 
+  // A TimeRegion used to start and stop ReadTimer via RAII.
+  std::optional<llvm::TimeRegion> ReadTimeRegion;
+
   /// The location where the module file will be considered as
   /// imported from. For non-module AST types it should be invalid.
   SourceLocation CurrentImportLoc;
diff --git a/clang/include/clang/Serialization/ASTRecordReader.h b/clang/include/clang/Serialization/ASTRecordReader.h
index 1472497ff5e7e..aed1b7d309001 100644
--- a/clang/include/clang/Serialization/ASTRecordReader.h
+++ b/clang/include/clang/Serialization/ASTRecordReader.h
@@ -223,7 +223,7 @@ class ASTRecordReader
   void readQualifierInfo(QualifierInfo &Info);
 
   /// Return a nested name specifier, advancing Idx.
-  // NestedNameSpecifier *readNestedNameSpecifier(); (inherited)
+  // NestedNameSpecifier readNestedNameSpecifier(); (inherited)
 
   NestedNameSpecifierLoc readNestedNameSpecifierLoc();
 
diff --git a/clang/include/clang/Serialization/ASTRecordWriter.h b/clang/include/clang/Serialization/ASTRecordWriter.h
index ee005ec287708..9849ea6b395ab 100644
--- a/clang/include/clang/Serialization/ASTRecordWriter.h
+++ b/clang/include/clang/Serialization/ASTRecordWriter.h
@@ -247,8 +247,7 @@ class ASTRecordWriter
   void AddTypeLoc(TypeLoc TL);
 
   /// Emits a template argument location info.
-  void AddTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
-                                  const TemplateArgumentLocInfo &Arg);
+  void AddTemplateArgumentLocInfo(const TemplateArgumentLoc &Arg);
 
   /// Emits a template argument location.
   void AddTemplateArgumentLoc(const TemplateArgumentLoc &Arg);
@@ -280,7 +279,7 @@ class ASTRecordWriter
   void AddQualifierInfo(const QualifierInfo &Info);
 
   /// Emit a nested name specifier.
-  void AddNestedNameSpecifier(NestedNameSpecifier *NNS) {
+  void AddNestedNameSpecifier(NestedNameSpecifier NNS) {
     writeNestedNameSpecifier(NNS);
   }
 
diff --git a/clang/include/clang/Serialization/TypeBitCodes.def b/clang/include/clang/Serialization/TypeBitCodes.def
index 613eb6af2005a..bea15254922c1 100644
--- a/clang/include/clang/Serialization/TypeBitCodes.def
+++ b/clang/include/clang/Serialization/TypeBitCodes.def
@@ -32,7 +32,6 @@ TYPE_BIT_CODE(Enum, ENUM, 20)
 TYPE_BIT_CODE(ObjCInterface, OBJC_INTERFACE, 21)
 TYPE_BIT_CODE(ObjCObjectPointer, OBJC_OBJECT_POINTER, 22)
 TYPE_BIT_CODE(Decltype, DECLTYPE, 23)
-TYPE_BIT_CODE(Elaborated, ELABORATED, 24)
 TYPE_BIT_CODE(SubstTemplateTypeParm, SUBST_TEMPLATE_TYPE_PARM, 25)
 TYPE_BIT_CODE(UnresolvedUsing, UNRESOLVED_USING, 26)
 TYPE_BIT_CODE(InjectedClassName, INJECTED_CLASS_NAME, 27)
@@ -70,5 +69,6 @@ TYPE_BIT_CODE(ArrayParameter, ARRAY_PARAMETER, 58)
 TYPE_BIT_CODE(HLSLAttributedResource, HLSLRESOURCE_ATTRIBUTED, 59)
 TYPE_BIT_CODE(HLSLInlineSpirv, HLSL_INLINE_SPIRV, 60)
 TYPE_BIT_CODE(PredefinedSugar, PREDEFINED_SUGAR, 61)
+TYPE_BIT_CODE(SubstBuiltinTemplatePack, SUBST_BUILTIN_TEMPLATE_PACK, 62)
 
 #undef TYPE_BIT_CODE
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
index 8696fce39add1..f6a023368f3d2 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
@@ -320,7 +320,7 @@ class PathSensitiveBugReport : public BugReport {
 
   /// A set of location contexts that correspoind to call sites which should be
   /// considered "interesting".
-  llvm::SmallSet<const LocationContext *, 2> InterestingLocationContexts;
+  llvm::SmallPtrSet<const LocationContext *, 2> InterestingLocationContexts;
 
   /// A set of custom visitors which generate "event" diagnostics at
   /// interesting points in the path.
@@ -348,7 +348,7 @@ class PathSensitiveBugReport : public BugReport {
   llvm::SmallSet<InvalidationRecord, 4> Invalidations;
 
   /// Conditions we're already tracking.
-  llvm::SmallSet<const ExplodedNode *, 4> TrackedConditions;
+  llvm::SmallPtrSet<const ExplodedNode *, 4> TrackedConditions;
 
   /// Reports with different uniqueing locations are considered to be different
   /// for the purposes of deduplication.
@@ -623,10 +623,12 @@ class BugReporter {
   ASTContext &getContext() { return D.getASTContext(); }
 
   const SourceManager &getSourceManager() { return D.getSourceManager(); }
+  const SourceManager &getSourceManager() const { return D.getSourceManager(); }
 
   const AnalyzerOptions &getAnalyzerOptions() { return D.getAnalyzerOptions(); }
 
   Preprocessor &getPreprocessor() { return D.getPreprocessor(); }
+  const Preprocessor &getPreprocessor() const { return D.getPreprocessor(); }
 
   /// Get the top-level entry point for the issue to be reported.
   const Decl *getAnalysisEntryPoint() const { return AnalysisEntryPoint; }
diff --git a/clang/include/clang/StaticAnalyzer/Core/Checker.h b/clang/include/clang/StaticAnalyzer/Core/Checker.h
index 31cc095c29bfe..d9a7c00254a00 100644
--- a/clang/include/clang/StaticAnalyzer/Core/Checker.h
+++ b/clang/include/clang/StaticAnalyzer/Core/Checker.h
@@ -209,8 +209,8 @@ class Location {
 class Bind {
   template <typename CHECKER>
   static void _checkBind(void *checker, SVal location, SVal val, const Stmt *S,
-                         CheckerContext &C) {
-    ((const CHECKER *)checker)->checkBind(location, val, S, C);
+                         bool AtDeclInit, CheckerContext &C) {
+    ((const CHECKER *)checker)->checkBind(location, val, S, AtDeclInit, C);
   }
 
 public:
diff --git a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
index c8e6f1265a3db..bf33ce616d954 100644
--- a/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
+++ b/clang/include/clang/StaticAnalyzer/Core/CheckerManager.h
@@ -338,10 +338,9 @@ class CheckerManager {
                               ExprEngine &Eng);
 
   /// Run checkers for binding of a value to a location.
-  void runCheckersForBind(ExplodedNodeSet &Dst,
-                          const ExplodedNodeSet &Src,
-                          SVal location, SVal val,
-                          const Stmt *S, ExprEngine &Eng,
+  void runCheckersForBind(ExplodedNodeSet &Dst, const ExplodedNodeSet &Src,
+                          SVal location, SVal val, const Stmt *S,
+                          bool AtDeclInit, ExprEngine &Eng,
                           const ProgramPoint &PP);
 
   /// Run checkers after taking a control flow edge.
@@ -499,8 +498,8 @@ class CheckerManager {
   using CheckLocationFunc = CheckerFn<void(SVal location, bool isLoad,
                                            const Stmt *S, CheckerContext &)>;
 
-  using CheckBindFunc =
-      CheckerFn<void(SVal location, SVal val, const Stmt *S, CheckerContext &)>;
+  using CheckBindFunc = CheckerFn<void(SVal location, SVal val, const Stmt *S,
+                                       bool AtDeclInit, CheckerContext &)>;
 
   using CheckBlockEntranceFunc =
       CheckerFn<void(const BlockEntrance &, CheckerContext &)>;
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
index f20b0031c1528..cf035a999af8a 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h
@@ -58,25 +58,34 @@ class CheckerContext {
   AnalysisManager &getAnalysisManager() {
     return Eng.getAnalysisManager();
   }
+  const AnalysisManager &getAnalysisManager() const {
+    return Eng.getAnalysisManager();
+  }
 
   ConstraintManager &getConstraintManager() {
     return Eng.getConstraintManager();
   }
+  const ConstraintManager &getConstraintManager() const {
+    return Eng.getConstraintManager();
+  }
 
   StoreManager &getStoreManager() {
     return Eng.getStoreManager();
   }
+  const StoreManager &getStoreManager() const { return Eng.getStoreManager(); }
 
   /// Returns the previous node in the exploded graph, which includes
   /// the state of the program before the checker ran. Note, checkers should
   /// not retain the node in their state since the nodes might get invalidated.
   ExplodedNode *getPredecessor() { return Pred; }
+  const ExplodedNode *getPredecessor() const { return Pred; }
   const ProgramPoint getLocation() const { return Location; }
   const ProgramStateRef &getState() const { return Pred->getState(); }
 
   /// Check if the checker changed the state of the execution; ex: added
   /// a new transition or a bug report.
   bool isDifferent() { return Changed; }
+  bool isDifferent() const { return Changed; }
 
   /// Returns the number of times the current block has been visited
   /// along the analyzed path.
@@ -108,24 +117,38 @@ class CheckerContext {
   BugReporter &getBugReporter() {
     return Eng.getBugReporter();
   }
+  const BugReporter &getBugReporter() const { return Eng.getBugReporter(); }
 
   const SourceManager &getSourceManager() {
     return getBugReporter().getSourceManager();
   }
+  const SourceManager &getSourceManager() const {
+    return getBugReporter().getSourceManager();
+  }
 
   Preprocessor &getPreprocessor() { return getBugReporter().getPreprocessor(); }
+  const Preprocessor &getPreprocessor() const {
+    return getBugReporter().getPreprocessor();
+  }
 
   SValBuilder &getSValBuilder() {
     return Eng.getSValBuilder();
   }
+  const SValBuilder &getSValBuilder() const { return Eng.getSValBuilder(); }
 
   SymbolManager &getSymbolManager() {
     return getSValBuilder().getSymbolManager();
   }
+  const SymbolManager &getSymbolManager() const {
+    return getSValBuilder().getSymbolManager();
+  }
 
   ProgramStateManager &getStateManager() {
     return Eng.getStateManager();
   }
+  const ProgramStateManager &getStateManager() const {
+    return Eng.getStateManager();
+  }
 
   AnalysisDeclContext *getCurrentAnalysisDeclContext() const {
     return Pred->getLocationContext()->getAnalysisDeclContext();
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
index fbb34340a5c67..d184986cda15d 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h
@@ -196,6 +196,7 @@ class ExprEngine {
   ASTContext &getContext() const { return AMgr.getASTContext(); }
 
   AnalysisManager &getAnalysisManager() { return AMgr; }
+  const AnalysisManager &getAnalysisManager() const { return AMgr; }
 
   AnalysisDeclContextManager &getAnalysisDeclContextManager() {
     return AMgr.getAnalysisDeclContextManager();
@@ -206,8 +207,10 @@ class ExprEngine {
   }
 
   SValBuilder &getSValBuilder() { return svalBuilder; }
+  const SValBuilder &getSValBuilder() const { return svalBuilder; }
 
   BugReporter &getBugReporter() { return BR; }
+  const BugReporter &getBugReporter() const { return BR; }
 
   cross_tu::CrossTranslationUnitContext *
   getCrossTranslationUnitContext() {
@@ -416,12 +419,19 @@ class ExprEngine {
                  unsigned int Space, bool IsDot) const;
 
   ProgramStateManager &getStateManager() { return StateMgr; }
+  const ProgramStateManager &getStateManager() const { return StateMgr; }
 
   StoreManager &getStoreManager() { return StateMgr.getStoreManager(); }
+  const StoreManager &getStoreManager() const {
+    return StateMgr.getStoreManager();
+  }
 
   ConstraintManager &getConstraintManager() {
     return StateMgr.getConstraintManager();
   }
+  const ConstraintManager &getConstraintManager() const {
+    return StateMgr.getConstraintManager();
+  }
 
   // FIXME: Remove when we migrate over to just using SValBuilder.
   BasicValueFactory &getBasicVals() {
@@ -429,6 +439,7 @@ class ExprEngine {
   }
 
   SymbolManager &getSymbolManager() { return SymMgr; }
+  const SymbolManager &getSymbolManager() const { return SymMgr; }
   MemRegionManager &getRegionManager() { return MRMgr; }
 
   DataTag::Factory &getDataTags() { return Engine.getDataTags(); }
@@ -660,7 +671,7 @@ class ExprEngine {
   /// evalBind - Handle the semantics of binding a value to a specific location.
   ///  This method is used by evalStore, VisitDeclStmt, and others.
   void evalBind(ExplodedNodeSet &Dst, const Stmt *StoreE, ExplodedNode *Pred,
-                SVal location, SVal Val, bool atDeclInit = false,
+                SVal location, SVal Val, bool AtDeclInit = false,
                 const ProgramPoint *PP = nullptr);
 
   ProgramStateRef
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
index 52714535e7907..12487a33a485e 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h
@@ -572,7 +572,11 @@ class ProgramStateManager {
   CallEventManager &getCallEventManager() { return *CallEventMgr; }
 
   StoreManager &getStoreManager() { return *StoreMgr; }
+  const StoreManager &getStoreManager() const { return *StoreMgr; }
   ConstraintManager &getConstraintManager() { return *ConstraintMgr; }
+  const ConstraintManager &getConstraintManager() const {
+    return *ConstraintMgr;
+  }
   ExprEngine &getOwningEngine() { return *Eng; }
 
   ProgramStateRef
diff --git a/clang/include/clang/Tooling/Refactoring/Lookup.h b/clang/include/clang/Tooling/Refactoring/Lookup.h
index dcb40b7eee66c..fe0df8656bce1 100644
--- a/clang/include/clang/Tooling/Refactoring/Lookup.h
+++ b/clang/include/clang/Tooling/Refactoring/Lookup.h
@@ -38,8 +38,7 @@ namespace tooling {
 /// \param ReplacementString The replacement nested name. Must be fully
 ///                          qualified including a leading "::".
 /// \returns The new name to be inserted in place of the current nested name.
-std::string replaceNestedName(const NestedNameSpecifier *Use,
-                              SourceLocation UseLoc,
+std::string replaceNestedName(NestedNameSpecifier Use, SourceLocation UseLoc,
                               const DeclContext *UseContext,
                               const NamedDecl *FromDecl,
                               StringRef ReplacementString);
diff --git a/clang/include/clang/Tooling/Refactoring/RecursiveSymbolVisitor.h b/clang/include/clang/Tooling/Refactoring/RecursiveSymbolVisitor.h
index 271232e66626e..319569fd5ab33 100644
--- a/clang/include/clang/Tooling/Refactoring/RecursiveSymbolVisitor.h
+++ b/clang/include/clang/Tooling/Refactoring/RecursiveSymbolVisitor.h
@@ -108,19 +108,21 @@ class RecursiveSymbolVisitor
   bool VisitTypedefTypeLoc(TypedefTypeLoc TL) {
     const SourceLocation TypeEndLoc =
         Lexer::getLocForEndOfToken(TL.getBeginLoc(), 0, SM, LangOpts);
-    return visit(TL.getTypedefNameDecl(), TL.getBeginLoc(), TypeEndLoc);
+    return visit(TL.getDecl(), TL.getBeginLoc(), TypeEndLoc);
   }
 
-  bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS) {
+  bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc QualifierLoc) {
     // The base visitor will visit NNSL prefixes, so we should only look at
     // the current NNS.
-    if (NNS) {
-      const auto *ND = dyn_cast_if_present<NamespaceDecl>(
-          NNS.getNestedNameSpecifier()->getAsNamespace());
-      if (!visit(ND, NNS.getLocalBeginLoc(), NNS.getLocalEndLoc()))
+    if (NestedNameSpecifier Qualifier = QualifierLoc.getNestedNameSpecifier();
+        Qualifier.getKind() == NestedNameSpecifier::Kind::Namespace) {
+      const auto *ND = dyn_cast<NamespaceDecl>(
+          Qualifier.getAsNamespaceAndPrefix().Namespace);
+      if (!visit(ND, QualifierLoc.getLocalBeginLoc(),
+                 QualifierLoc.getLocalEndLoc()))
         return false;
     }
-    return BaseType::TraverseNestedNameSpecifierLoc(NNS);
+    return BaseType::TraverseNestedNameSpecifierLoc(QualifierLoc);
   }
 
   bool VisitDesignatedInitExpr(const DesignatedInitExpr *E) {
diff --git a/clang/lib/APINotes/APINotesFormat.h b/clang/lib/APINotes/APINotesFormat.h
index bb0c276e74964..69d180e7b3eb5 100644
--- a/clang/lib/APINotes/APINotesFormat.h
+++ b/clang/lib/APINotes/APINotesFormat.h
@@ -24,7 +24,7 @@ const uint16_t VERSION_MAJOR = 0;
 /// API notes file minor version number.
 ///
 /// When the format changes IN ANY WAY, this number should be incremented.
-const uint16_t VERSION_MINOR = 35; // SwiftDefaultOwnership
+const uint16_t VERSION_MINOR = 37; // SwiftDestroyOp
 
 const uint8_t kSwiftConforms = 1;
 const uint8_t kSwiftDoesNotConform = 2;
diff --git a/clang/lib/APINotes/APINotesReader.cpp b/clang/lib/APINotes/APINotesReader.cpp
index 7cc4df2a99369..573356f97ff73 100644
--- a/clang/lib/APINotes/APINotesReader.cpp
+++ b/clang/lib/APINotes/APINotesReader.cpp
@@ -134,6 +134,13 @@ void ReadCommonTypeInfo(const uint8_t *&Data, CommonTypeInfo &Info) {
         reinterpret_cast<const char *>(Data), ErrorDomainLength - 1)));
     Data += ErrorDomainLength - 1;
   }
+
+  if (unsigned ConformanceLength =
+          endian::readNext<uint16_t, llvm::endianness::little>(Data)) {
+    Info.setSwiftConformance(std::string(reinterpret_cast<const char *>(Data),
+                                         ConformanceLength - 1));
+    Data += ConformanceLength - 1;
+  }
 }
 
 /// Used to deserialize the on-disk identifier table.
@@ -629,11 +636,12 @@ class TagTableInfo
           reinterpret_cast<const char *>(Data), DefaultOwnershipLength - 1);
       Data += DefaultOwnershipLength - 1;
     }
-    if (unsigned ConformanceLength =
-            endian::readNext<uint16_t, llvm::endianness::little>(Data)) {
-      Info.SwiftConformance = std::string(reinterpret_cast<const char *>(Data),
-                                          ConformanceLength - 1);
-      Data += ConformanceLength - 1;
+    unsigned DestroyOpLength =
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
+    if (DestroyOpLength > 0) {
+      Info.SwiftDestroyOp = std::string(reinterpret_cast<const char *>(Data),
+                                        DestroyOpLength - 1);
+      Data += DestroyOpLength - 1;
     }
 
     ReadCommonTypeInfo(Data, Info);
diff --git a/clang/lib/APINotes/APINotesWriter.cpp b/clang/lib/APINotes/APINotesWriter.cpp
index ffc5473988735..cf88d118d0979 100644
--- a/clang/lib/APINotes/APINotesWriter.cpp
+++ b/clang/lib/APINotes/APINotesWriter.cpp
@@ -536,7 +536,8 @@ unsigned getCommonEntityInfoSize(const CommonEntityInfo &CEI) {
 // in on-disk hash tables.
 unsigned getCommonTypeInfoSize(const CommonTypeInfo &CTI) {
   return 2 + (CTI.getSwiftBridge() ? CTI.getSwiftBridge()->size() : 0) + 2 +
-         (CTI.getNSErrorDomain() ? CTI.getNSErrorDomain()->size() : 0) +
+         (CTI.getNSErrorDomain() ? CTI.getNSErrorDomain()->size() : 0) + 2 +
+         (CTI.getSwiftConformance() ? CTI.getSwiftConformance()->size() : 0) +
          getCommonEntityInfoSize(CTI);
 }
 
@@ -557,6 +558,12 @@ void emitCommonTypeInfo(raw_ostream &OS, const CommonTypeInfo &CTI) {
   } else {
     writer.write<uint16_t>(0);
   }
+  if (auto conformance = CTI.getSwiftConformance()) {
+    writer.write<uint16_t>(conformance->size() + 1);
+    OS.write(conformance->c_str(), conformance->size());
+  } else {
+    writer.write<uint16_t>(0);
+  }
 }
 
 /// Used to serialize the on-disk Objective-C property table.
@@ -1273,8 +1280,8 @@ class TagTableInfo : public CommonTypeTableInfo<TagTableInfo, TagInfo> {
     return 2 + (TI.SwiftImportAs ? TI.SwiftImportAs->size() : 0) +
            2 + (TI.SwiftRetainOp ? TI.SwiftRetainOp->size() : 0) +
            2 + (TI.SwiftReleaseOp ? TI.SwiftReleaseOp->size() : 0) +
+           2 + (TI.SwiftDestroyOp ? TI.SwiftDestroyOp->size() : 0) +
            2 + (TI.SwiftDefaultOwnership ? TI.SwiftDefaultOwnership->size() : 0) +
-           2 + (TI.SwiftConformance ? TI.SwiftConformance->size() : 0) +
            3 + getCommonTypeInfoSize(TI);
     // clang-format on
   }
@@ -1328,9 +1335,9 @@ class TagTableInfo : public CommonTypeTableInfo<TagTableInfo, TagInfo> {
     } else {
       writer.write<uint16_t>(0);
     }
-    if (auto Conformance = TI.SwiftConformance) {
-      writer.write<uint16_t>(Conformance->size() + 1);
-      OS.write(Conformance->c_str(), Conformance->size());
+    if (auto DestroyOp = TI.SwiftDestroyOp) {
+      writer.write<uint16_t>(DestroyOp->size() + 1);
+      OS.write(DestroyOp->c_str(), DestroyOp->size());
     } else {
       writer.write<uint16_t>(0);
     }
diff --git a/clang/lib/APINotes/APINotesYAMLCompiler.cpp b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
index 803410c54c646..a91a1eea03d81 100644
--- a/clang/lib/APINotes/APINotesYAMLCompiler.cpp
+++ b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
@@ -251,6 +251,7 @@ struct Class {
   std::optional<StringRef> NSErrorDomain;
   std::optional<bool> SwiftImportAsNonGeneric;
   std::optional<bool> SwiftObjCMembers;
+  std::optional<std::string> SwiftConformance;
   MethodsSeq Methods;
   PropertiesSeq Properties;
 };
@@ -275,6 +276,7 @@ template <> struct MappingTraits<Class> {
     IO.mapOptional("NSErrorDomain", C.NSErrorDomain);
     IO.mapOptional("SwiftImportAsNonGeneric", C.SwiftImportAsNonGeneric);
     IO.mapOptional("SwiftObjCMembers", C.SwiftObjCMembers);
+    IO.mapOptional("SwiftConformsTo", C.SwiftConformance);
     IO.mapOptional("Methods", C.Methods);
     IO.mapOptional("Properties", C.Properties);
   }
@@ -460,6 +462,7 @@ struct Tag {
   std::optional<std::string> SwiftImportAs;
   std::optional<std::string> SwiftRetainOp;
   std::optional<std::string> SwiftReleaseOp;
+  std::optional<std::string> SwiftDestroyOp;
   std::optional<std::string> SwiftDefaultOwnership;
   std::optional<std::string> SwiftConformance;
   std::optional<EnumExtensibilityKind> EnumExtensibility;
@@ -501,6 +504,7 @@ template <> struct MappingTraits<Tag> {
     IO.mapOptional("SwiftImportAs", T.SwiftImportAs);
     IO.mapOptional("SwiftReleaseOp", T.SwiftReleaseOp);
     IO.mapOptional("SwiftRetainOp", T.SwiftRetainOp);
+    IO.mapOptional("SwiftDestroyOp", T.SwiftDestroyOp);
     IO.mapOptional("SwiftDefaultOwnership", T.SwiftDefaultOwnership);
     IO.mapOptional("SwiftConformsTo", T.SwiftConformance);
     IO.mapOptional("EnumExtensibility", T.EnumExtensibility);
@@ -525,6 +529,7 @@ struct Typedef {
   std::optional<StringRef> SwiftBridge;
   std::optional<StringRef> NSErrorDomain;
   std::optional<SwiftNewTypeKind> SwiftType;
+  std::optional<std::string> SwiftConformance;
 };
 
 typedef std::vector<Typedef> TypedefsSeq;
@@ -553,6 +558,7 @@ template <> struct MappingTraits<Typedef> {
     IO.mapOptional("SwiftBridge", T.SwiftBridge);
     IO.mapOptional("NSErrorDomain", T.NSErrorDomain);
     IO.mapOptional("SwiftWrapper", T.SwiftType);
+    IO.mapOptional("SwiftConformsTo", T.SwiftConformance);
   }
 };
 } // namespace yaml
@@ -802,6 +808,8 @@ class YAMLConverter {
     if (Common.SwiftBridge)
       Info.setSwiftBridge(std::string(*Common.SwiftBridge));
     Info.setNSErrorDomain(Common.NSErrorDomain);
+    if (auto conformance = Common.SwiftConformance)
+      Info.setSwiftConformance(conformance);
   }
 
   // Translate from Method into ObjCMethodInfo and write it out.
@@ -990,8 +998,8 @@ class YAMLConverter {
       TI.SwiftRetainOp = T.SwiftRetainOp;
     if (T.SwiftReleaseOp)
       TI.SwiftReleaseOp = T.SwiftReleaseOp;
-    if (T.SwiftConformance)
-      TI.SwiftConformance = T.SwiftConformance;
+    if (T.SwiftDestroyOp)
+      TI.SwiftDestroyOp = T.SwiftDestroyOp;
     if (T.SwiftDefaultOwnership)
       TI.SwiftDefaultOwnership = T.SwiftDefaultOwnership;
 
diff --git a/clang/lib/AST/APValue.cpp b/clang/lib/AST/APValue.cpp
index ee3dc84479fd9..2d62209bbc28c 100644
--- a/clang/lib/AST/APValue.cpp
+++ b/clang/lib/AST/APValue.cpp
@@ -902,8 +902,9 @@ void APValue::printPretty(raw_ostream &Out, const PrintingPolicy &Policy,
   }
   case APValue::Struct: {
     Out << '{';
-    const RecordDecl *RD = Ty->castAs<RecordType>()->getDecl();
     bool First = true;
+    const RecordDecl *RD =
+        Ty->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
     if (unsigned N = getStructNumBases()) {
       const CXXRecordDecl *CD = cast<CXXRecordDecl>(RD);
       CXXRecordDecl::base_class_const_iterator BI = CD->bases_begin();
diff --git a/clang/lib/AST/ASTConcept.cpp b/clang/lib/AST/ASTConcept.cpp
index 2243ac035115f..d658890e076c2 100644
--- a/clang/lib/AST/ASTConcept.cpp
+++ b/clang/lib/AST/ASTConcept.cpp
@@ -14,6 +14,7 @@
 #include "clang/AST/ASTConcept.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ExprConcepts.h"
+#include "clang/AST/NestedNameSpecifier.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "llvm/ADT/StringExtras.h"
 
@@ -92,10 +93,16 @@ ConceptReference::Create(const ASTContext &C, NestedNameSpecifierLoc NNS,
                                   FoundDecl, NamedConcept, ArgsAsWritten);
 }
 
+SourceLocation ConceptReference::getBeginLoc() const {
+  // Note that if the qualifier is null the template KW must also be null.
+  if (auto QualifierLoc = getNestedNameSpecifierLoc())
+    return QualifierLoc.getBeginLoc();
+  return getConceptNameInfo().getBeginLoc();
+}
+
 void ConceptReference::print(llvm::raw_ostream &OS,
                              const PrintingPolicy &Policy) const {
-  if (NestedNameSpec)
-    NestedNameSpec.getNestedNameSpecifier()->print(OS, Policy);
+  NestedNameSpec.getNestedNameSpecifier().print(OS, Policy);
   ConceptName.printName(OS, Policy);
   if (hasExplicitTemplateArgs()) {
     OS << "<";
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 3a16111dd5f7d..bbb957067c4c8 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -654,7 +654,7 @@ comments::FullComment *ASTContext::getCommentForDecl(
       // does not have one of its own.
       QualType QT = TD->getUnderlyingType();
       if (const auto *TT = QT->getAs<TagType>())
-        if (const Decl *TD = TT->getDecl())
+        if (const Decl *TD = TT->getOriginalDecl())
           if (comments::FullComment *FC = getCommentForDecl(TD, PP))
             return cloneFullComment(FC, D);
     }
@@ -1933,10 +1933,12 @@ TypeInfoChars ASTContext::getTypeInfoDataSizeInChars(QualType T) const {
   // of a base-class subobject.  We decide whether that's possible
   // during class layout, so here we can just trust the layout results.
   if (getLangOpts().CPlusPlus) {
-    if (const auto *RT = T->getAs<RecordType>();
-        RT && !RT->getDecl()->isInvalidDecl()) {
-      const ASTRecordLayout &layout = getASTRecordLayout(RT->getDecl());
-      Info.Width = layout.getDataSize();
+    if (const auto *RT = T->getAs<RecordType>()) {
+      const auto *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
+      if (!RD->isInvalidDecl()) {
+        const ASTRecordLayout &layout = getASTRecordLayout(RD);
+        Info.Width = layout.getDataSize();
+      }
     }
   }
 
@@ -2003,8 +2005,9 @@ bool ASTContext::isPromotableIntegerType(QualType T) const {
   // Enumerated types are promotable to their compatible integer types
   // (C99 6.3.1.1) a.k.a. its underlying type (C++ [conv.prom]p2).
   if (const auto *ET = T->getAs<EnumType>()) {
-    if (T->isDependentType() || ET->getDecl()->getPromotionType().isNull() ||
-        ET->getDecl()->isScoped())
+    const EnumDecl *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+    if (T->isDependentType() || ED->getPromotionType().isNull() ||
+        ED->isScoped())
       return false;
 
     return true;
@@ -2040,8 +2043,8 @@ unsigned ASTContext::getTypeAlignIfKnown(QualType T,
       return Align;
 
   // Otherwise, see if the declaration of the type had an attribute.
-  if (const auto *TT = T->getAs<TagType>())
-    return TT->getDecl()->getMaxAlignment();
+  if (const auto *TD = T->getAsTagDecl())
+    return TD->getMaxAlignment();
 
   return 0;
 }
@@ -2472,15 +2475,16 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
   case Type::Record:
   case Type::Enum: {
     const auto *TT = cast<TagType>(T);
+    const TagDecl *TD = TT->getOriginalDecl()->getDefinitionOrSelf();
 
-    if (TT->getDecl()->isInvalidDecl()) {
+    if (TD->isInvalidDecl()) {
       Width = 8;
       Align = 8;
       break;
     }
 
-    if (const auto *ET = dyn_cast<EnumType>(TT)) {
-      const EnumDecl *ED = ET->getDecl();
+    if (isa<EnumType>(TT)) {
+      const EnumDecl *ED = cast<EnumDecl>(TD);
       TypeInfo Info =
           getTypeInfo(ED->getIntegerType()->getUnqualifiedDesugaredType());
       if (unsigned AttrAlign = ED->getMaxAlignment()) {
@@ -2490,8 +2494,7 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
       return Info;
     }
 
-    const auto *RT = cast<RecordType>(TT);
-    const RecordDecl *RD = RT->getDecl();
+    const auto *RD = cast<RecordDecl>(TD);
     const ASTRecordLayout &Layout = getASTRecordLayout(RD);
     Width = toBits(Layout.getSize());
     Align = toBits(Layout.getAlignment());
@@ -2543,9 +2546,6 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
     break;
   }
 
-  case Type::Elaborated:
-    return getTypeInfo(cast<ElaboratedType>(T)->getNamedType().getTypePtr());
-
   case Type::Attributed:
     return getTypeInfo(
                   cast<AttributedType>(T)->getEquivalentType().getTypePtr());
@@ -2619,8 +2619,7 @@ unsigned ASTContext::getTypeUnadjustedAlign(const Type *T) const {
 
   unsigned UnadjustedAlign;
   if (const auto *RT = T->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
-    const ASTRecordLayout &Layout = getASTRecordLayout(RD);
+    const ASTRecordLayout &Layout = getASTRecordLayout(RT->getOriginalDecl());
     UnadjustedAlign = toBits(Layout.getUnadjustedAlignment());
   } else if (const auto *ObjCI = T->getAs<ObjCInterfaceType>()) {
     const ASTRecordLayout &Layout = getASTObjCInterfaceLayout(ObjCI->getDecl());
@@ -2696,7 +2695,7 @@ unsigned ASTContext::getPreferredTypeAlign(const Type *T) const {
     return ABIAlign;
 
   if (const auto *RT = T->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
 
     // When used as part of a typedef, or together with a 'packed' attribute,
     // the 'aligned' attribute can be used to decrease alignment. Note that the
@@ -2719,7 +2718,10 @@ unsigned ASTContext::getPreferredTypeAlign(const Type *T) const {
   if (const auto *CT = T->getAs<ComplexType>())
     T = CT->getElementType().getTypePtr();
   if (const auto *ET = T->getAs<EnumType>())
-    T = ET->getDecl()->getIntegerType().getTypePtr();
+    T = ET->getOriginalDecl()
+            ->getDefinitionOrSelf()
+            ->getIntegerType()
+            .getTypePtr();
   if (T->isSpecificBuiltinType(BuiltinType::Double) ||
       T->isSpecificBuiltinType(BuiltinType::LongLong) ||
       T->isSpecificBuiltinType(BuiltinType::ULongLong) ||
@@ -2851,7 +2853,8 @@ static bool unionHasUniqueObjectRepresentations(const ASTContext &Context,
                                                 const RecordDecl *RD,
                                                 bool CheckIfTriviallyCopyable) {
   assert(RD->isUnion() && "Must be union type");
-  CharUnits UnionSize = Context.getTypeSizeInChars(RD->getTypeForDecl());
+  CharUnits UnionSize =
+      Context.getTypeSizeInChars(Context.getCanonicalTagType(RD));
 
   for (const auto *Field : RD->fields()) {
     if (!Context.hasUniqueObjectRepresentations(Field->getType(),
@@ -3045,7 +3048,8 @@ bool ASTContext::hasUniqueObjectRepresentations(
     return !ABI->getMemberPointerInfo(MPT).HasPadding;
 
   if (Ty->isRecordType()) {
-    const RecordDecl *Record = Ty->castAs<RecordType>()->getDecl();
+    const RecordDecl *Record =
+        Ty->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
 
     if (Record->isInvalidDecl())
       return false;
@@ -3418,7 +3422,10 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
     //   type, or an unsigned integer type.
     //
     // So we have to treat enum types as integers.
-    QualType UnderlyingType = cast<EnumType>(T)->getDecl()->getIntegerType();
+    QualType UnderlyingType = cast<EnumType>(T)
+                                  ->getOriginalDecl()
+                                  ->getDefinitionOrSelf()
+                                  ->getIntegerType();
     return encodeTypeForFunctionPointerAuth(
         Ctx, OS, UnderlyingType.isNull() ? Ctx.IntTy : UnderlyingType);
   }
@@ -3456,7 +3463,7 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
     OS << "M";
     const auto *MPT = T->castAs<MemberPointerType>();
     encodeTypeForFunctionPointerAuth(
-        Ctx, OS, QualType(MPT->getQualifier()->getAsType(), 0));
+        Ctx, OS, QualType(MPT->getQualifier().getAsType(), 0));
     encodeTypeForFunctionPointerAuth(Ctx, OS, MPT->getPointeeType());
     return;
   }
@@ -3562,7 +3569,8 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
     llvm_unreachable("should never get here");
   }
   case Type::Record: {
-    const RecordDecl *RD = T->castAs<RecordType>()->getDecl();
+    const RecordDecl *RD =
+        T->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
     const IdentifierInfo *II = RD->getIdentifier();
 
     // In C++, an immediate typedef of an anonymous struct or union
@@ -3740,12 +3748,6 @@ ASTContext::adjustType(QualType Orig,
                                    adjustType(BTFT->getWrappedType(), Adjust));
   }
 
-  case Type::Elaborated: {
-    const auto *ET = cast<ElaboratedType>(Orig);
-    return getElaboratedType(ET->getKeyword(), ET->getQualifier(),
-                             adjustType(ET->getNamedType(), Adjust));
-  }
-
   case Type::Paren:
     return getParenType(
         adjustType(cast<ParenType>(Orig)->getInnerType(), Adjust));
@@ -4163,14 +4165,13 @@ QualType ASTContext::getRValueReferenceType(QualType T) const {
 }
 
 QualType ASTContext::getMemberPointerType(QualType T,
-                                          NestedNameSpecifier *Qualifier,
+                                          NestedNameSpecifier Qualifier,
                                           const CXXRecordDecl *Cls) const {
   if (!Qualifier) {
     assert(Cls && "At least one of Qualifier or Cls must be provided");
-    Qualifier = NestedNameSpecifier::Create(*this, /*Prefix=*/nullptr,
-                                            getTypeDeclType(Cls).getTypePtr());
+    Qualifier = NestedNameSpecifier(getCanonicalTagType(Cls).getTypePtr());
   } else if (!Cls) {
-    Cls = Qualifier->getAsRecordDecl();
+    Cls = Qualifier.getAsRecordDecl();
   }
   // Unique pointers, to guarantee there is only one pointer of a particular
   // structure.
@@ -4182,12 +4183,11 @@ QualType ASTContext::getMemberPointerType(QualType T,
       MemberPointerTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(PT, 0);
 
-  NestedNameSpecifier *CanonicalQualifier = [&] {
+  NestedNameSpecifier CanonicalQualifier = [&] {
     if (!Cls)
-      return getCanonicalNestedNameSpecifier(Qualifier);
-    NestedNameSpecifier *R = NestedNameSpecifier::Create(
-        *this, /*Prefix=*/nullptr, Cls->getCanonicalDecl()->getTypeForDecl());
-    assert(R == getCanonicalNestedNameSpecifier(R));
+      return Qualifier.getCanonical();
+    NestedNameSpecifier R(getCanonicalTagType(Cls).getTypePtr());
+    assert(R.isCanonical());
     return R;
   }();
   // If the pointee or class type isn't canonical, this won't be a canonical
@@ -4306,6 +4306,7 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
   case Type::DependentTemplateSpecialization:
   case Type::TemplateTypeParm:
   case Type::SubstTemplateTypeParmPack:
+  case Type::SubstBuiltinTemplatePack:
   case Type::Auto:
   case Type::DeducedTemplateSpecialization:
   case Type::PackExpansion:
@@ -5128,10 +5129,12 @@ QualType ASTContext::getFunctionTypeInternal(
       EPI.ExceptionSpec.Type, EPI.ExceptionSpec.Exceptions.size());
   size_t Size = FunctionProtoType::totalSizeToAlloc<
       QualType, SourceLocation, FunctionType::FunctionTypeExtraBitfields,
+      FunctionType::FunctionTypeExtraAttributeInfo,
       FunctionType::FunctionTypeArmAttributes, FunctionType::ExceptionType,
       Expr *, FunctionDecl *, FunctionProtoType::ExtParameterInfo, Qualifiers,
       FunctionEffect, EffectConditionExpr>(
       NumArgs, EPI.Variadic, EPI.requiresFunctionProtoTypeExtraBitfields(),
+      EPI.requiresFunctionProtoTypeExtraAttributeInfo(),
       EPI.requiresFunctionProtoTypeArmAttributes(), ESH.NumExceptionType,
       ESH.NumExprPtr, ESH.NumFunctionDeclPtr,
       EPI.ExtParameterInfos ? NumArgs : 0,
@@ -5245,7 +5248,6 @@ ASTContext::getPredefinedSugarType(PredefinedSugarType::Kind KD) const {
     }
     llvm_unreachable("unexpected kind");
   };
-
   auto *New = new (*this, alignof(PredefinedSugarType))
       PredefinedSugarType(KD, &Idents.get(PredefinedSugarType::getName(KD)),
                           getCanonicalType(*this, static_cast<Kind>(KD)));
@@ -5254,153 +5256,297 @@ ASTContext::getPredefinedSugarType(PredefinedSugarType::Kind KD) const {
   return QualType(New, 0);
 }
 
-#ifndef NDEBUG
-static bool NeedsInjectedClassNameType(const RecordDecl *D) {
-  if (!isa<CXXRecordDecl>(D)) return false;
-  const auto *RD = cast<CXXRecordDecl>(D);
-  if (isa<ClassTemplatePartialSpecializationDecl>(RD))
-    return true;
-  if (RD->getDescribedClassTemplate() &&
-      !isa<ClassTemplateSpecializationDecl>(RD))
-    return true;
-  return false;
-}
-#endif
+QualType ASTContext::getTypeDeclType(ElaboratedTypeKeyword Keyword,
+                                     NestedNameSpecifier Qualifier,
+                                     const TypeDecl *Decl) const {
+  if (auto *Tag = dyn_cast<TagDecl>(Decl))
+    return getTagType(Keyword, Qualifier, Tag,
+                      /*OwnsTag=*/false);
+  if (auto *Typedef = dyn_cast<TypedefNameDecl>(Decl))
+    return getTypedefType(Keyword, Qualifier, Typedef);
+  if (auto *UD = dyn_cast<UnresolvedUsingTypenameDecl>(Decl))
+    return getUnresolvedUsingType(Keyword, Qualifier, UD);
 
-/// getInjectedClassNameType - Return the unique reference to the
-/// injected class name type for the specified templated declaration.
-QualType ASTContext::getInjectedClassNameType(CXXRecordDecl *Decl,
-                                              QualType TST) const {
-  assert(NeedsInjectedClassNameType(Decl));
-  if (Decl->TypeForDecl) {
-    assert(isa<InjectedClassNameType>(Decl->TypeForDecl));
-  } else if (CXXRecordDecl *PrevDecl = Decl->getPreviousDecl()) {
-    assert(PrevDecl->TypeForDecl && "previous declaration has no type");
-    Decl->TypeForDecl = PrevDecl->TypeForDecl;
-    assert(isa<InjectedClassNameType>(Decl->TypeForDecl));
-  } else {
-    Type *newType = new (*this, alignof(InjectedClassNameType))
-        InjectedClassNameType(Decl, TST);
-    Decl->TypeForDecl = newType;
-    Types.push_back(newType);
-  }
+  assert(Keyword == ElaboratedTypeKeyword::None);
+  assert(!Qualifier);
   return QualType(Decl->TypeForDecl, 0);
 }
 
-/// getTypeDeclType - Return the unique reference to the type for the
-/// specified type declaration.
-QualType ASTContext::getTypeDeclTypeSlow(const TypeDecl *Decl) const {
-  assert(Decl && "Passed null for Decl param");
-  assert(!Decl->TypeForDecl && "TypeForDecl present in slow case");
-
-  if (const auto *Typedef = dyn_cast<TypedefNameDecl>(Decl))
-    return getTypedefType(Typedef);
-
-  assert(!isa<TemplateTypeParmDecl>(Decl) &&
-         "Template type parameter types are always available.");
-
-  if (const auto *Record = dyn_cast<RecordDecl>(Decl)) {
-    assert(Record->isFirstDecl() && "struct/union has previous declaration");
-    assert(!NeedsInjectedClassNameType(Record));
-    return getRecordType(Record);
-  } else if (const auto *Enum = dyn_cast<EnumDecl>(Decl)) {
-    assert(Enum->isFirstDecl() && "enum has previous declaration");
-    return getEnumType(Enum);
-  } else if (const auto *Using = dyn_cast<UnresolvedUsingTypenameDecl>(Decl)) {
-    return getUnresolvedUsingType(Using);
-  } else
-    llvm_unreachable("TypeDecl without a type?");
-
+CanQualType ASTContext::getCanonicalTypeDeclType(const TypeDecl *TD) const {
+  if (auto *Tag = dyn_cast<TagDecl>(TD))
+    return getCanonicalTagType(Tag);
+  if (auto *TN = dyn_cast<TypedefNameDecl>(TD))
+    return getCanonicalType(TN->getUnderlyingType());
+  if (const auto *UD = dyn_cast<UnresolvedUsingTypenameDecl>(TD))
+    return getCanonicalUnresolvedUsingType(UD);
+  assert(TD->TypeForDecl);
+  return TD->TypeForDecl->getCanonicalTypeUnqualified();
+}
+
+QualType ASTContext::getTypeDeclType(const TypeDecl *Decl) const {
+  if (const auto *TD = dyn_cast<TagDecl>(Decl))
+    return getCanonicalTagType(TD);
+  if (const auto *TD = dyn_cast<TypedefNameDecl>(Decl);
+      isa_and_nonnull<TypedefDecl, TypeAliasDecl>(TD))
+    return getTypedefType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt, TD);
+  if (const auto *Using = dyn_cast<UnresolvedUsingTypenameDecl>(Decl))
+    return getCanonicalUnresolvedUsingType(Using);
+
+  assert(Decl->TypeForDecl);
   return QualType(Decl->TypeForDecl, 0);
 }
 
 /// getTypedefType - Return the unique reference to the type for the
 /// specified typedef name decl.
-QualType ASTContext::getTypedefType(const TypedefNameDecl *Decl,
-                                    QualType Underlying) const {
-  if (!Decl->TypeForDecl) {
-    if (Underlying.isNull())
-      Underlying = Decl->getUnderlyingType();
-    auto *NewType = new (*this, alignof(TypedefType)) TypedefType(
-        Type::Typedef, Decl, Underlying, /*HasTypeDifferentFromDecl=*/false);
-    Decl->TypeForDecl = NewType;
+QualType
+ASTContext::getTypedefType(ElaboratedTypeKeyword Keyword,
+                           NestedNameSpecifier Qualifier,
+                           const TypedefNameDecl *Decl, QualType UnderlyingType,
+                           std::optional<bool> TypeMatchesDeclOrNone) const {
+  if (!TypeMatchesDeclOrNone) {
+    QualType DeclUnderlyingType = Decl->getUnderlyingType();
+    assert(!DeclUnderlyingType.isNull());
+    if (UnderlyingType.isNull())
+      UnderlyingType = DeclUnderlyingType;
+    else
+      assert(hasSameType(UnderlyingType, DeclUnderlyingType));
+    TypeMatchesDeclOrNone = UnderlyingType == DeclUnderlyingType;
+  } else {
+    // FIXME: This is a workaround for a serialization cycle: assume the decl
+    // underlying type is not available; don't touch it.
+    assert(!UnderlyingType.isNull());
+  }
+
+  if (Keyword == ElaboratedTypeKeyword::None && !Qualifier &&
+      *TypeMatchesDeclOrNone) {
+    if (Decl->TypeForDecl)
+      return QualType(Decl->TypeForDecl, 0);
+
+    auto *NewType = new (*this, alignof(TypedefType))
+        TypedefType(Type::Typedef, Keyword, Qualifier, Decl, UnderlyingType,
+                    !*TypeMatchesDeclOrNone);
+
     Types.push_back(NewType);
+    Decl->TypeForDecl = NewType;
     return QualType(NewType, 0);
   }
-  if (Underlying.isNull() || Decl->getUnderlyingType() == Underlying)
-    return QualType(Decl->TypeForDecl, 0);
-  assert(hasSameType(Decl->getUnderlyingType(), Underlying));
 
   llvm::FoldingSetNodeID ID;
-  TypedefType::Profile(ID, Decl, Underlying);
+  TypedefType::Profile(ID, Keyword, Qualifier, Decl, UnderlyingType);
 
   void *InsertPos = nullptr;
-  if (TypedefType *T = TypedefTypes.FindNodeOrInsertPos(ID, InsertPos)) {
-    assert(!T->typeMatchesDecl() &&
-           "non-divergent case should be handled with TypeDecl");
-    return QualType(T, 0);
-  }
+  if (FoldingSetPlaceholder<TypedefType> *Placeholder =
+          TypedefTypes.FindNodeOrInsertPos(ID, InsertPos))
+    return QualType(Placeholder->getType(), 0);
 
-  void *Mem = Allocate(TypedefType::totalSizeToAlloc<QualType>(true),
-                       alignof(TypedefType));
-  auto *NewType = new (Mem) TypedefType(Type::Typedef, Decl, Underlying,
-                                        /*HasTypeDifferentFromDecl=*/true);
-  TypedefTypes.InsertNode(NewType, InsertPos);
+  void *Mem =
+      Allocate(TypedefType::totalSizeToAlloc<FoldingSetPlaceholder<TypedefType>,
+                                             NestedNameSpecifier, QualType>(
+                   1, !!Qualifier, !*TypeMatchesDeclOrNone),
+               alignof(TypedefType));
+  auto *NewType =
+      new (Mem) TypedefType(Type::Typedef, Keyword, Qualifier, Decl,
+                            UnderlyingType, !*TypeMatchesDeclOrNone);
+  auto *Placeholder = new (NewType->getFoldingSetPlaceholder())
+      FoldingSetPlaceholder<TypedefType>();
+  TypedefTypes.InsertNode(Placeholder, InsertPos);
   Types.push_back(NewType);
   return QualType(NewType, 0);
 }
 
-QualType ASTContext::getUsingType(const UsingShadowDecl *Found,
-                                  QualType Underlying) const {
+QualType ASTContext::getUsingType(ElaboratedTypeKeyword Keyword,
+                                  NestedNameSpecifier Qualifier,
+                                  const UsingShadowDecl *D,
+                                  QualType UnderlyingType) const {
+  // FIXME: This is expensive to compute every time!
+  if (UnderlyingType.isNull()) {
+    const auto *UD = cast<UsingDecl>(D->getIntroducer());
+    UnderlyingType =
+        getTypeDeclType(UD->hasTypename() ? ElaboratedTypeKeyword::Typename
+                                          : ElaboratedTypeKeyword::None,
+                        UD->getQualifier(), cast<TypeDecl>(D->getTargetDecl()));
+  }
+
   llvm::FoldingSetNodeID ID;
-  UsingType::Profile(ID, Found, Underlying);
+  UsingType::Profile(ID, Keyword, Qualifier, D, UnderlyingType);
 
   void *InsertPos = nullptr;
-  if (UsingType *T = UsingTypes.FindNodeOrInsertPos(ID, InsertPos))
+  if (const UsingType *T = UsingTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(T, 0);
 
-  const Type *TypeForDecl =
-      cast<TypeDecl>(Found->getTargetDecl())->getTypeForDecl();
+  assert(!UnderlyingType.hasLocalQualifiers());
 
-  assert(!Underlying.hasLocalQualifiers());
-  QualType Canon = Underlying->getCanonicalTypeInternal();
-  assert(TypeForDecl->getCanonicalTypeInternal() == Canon);
+  assert(
+      hasSameType(getCanonicalTypeDeclType(cast<TypeDecl>(D->getTargetDecl())),
+                  UnderlyingType));
 
-  if (Underlying.getTypePtr() == TypeForDecl)
-    Underlying = QualType();
   void *Mem =
-      Allocate(UsingType::totalSizeToAlloc<QualType>(!Underlying.isNull()),
+      Allocate(UsingType::totalSizeToAlloc<NestedNameSpecifier>(!!Qualifier),
                alignof(UsingType));
-  UsingType *NewType = new (Mem) UsingType(Found, Underlying, Canon);
-  Types.push_back(NewType);
-  UsingTypes.InsertNode(NewType, InsertPos);
-  return QualType(NewType, 0);
+  UsingType *T = new (Mem) UsingType(Keyword, Qualifier, D, UnderlyingType);
+  Types.push_back(T);
+  UsingTypes.InsertNode(T, InsertPos);
+  return QualType(T, 0);
 }
 
-QualType ASTContext::getRecordType(const RecordDecl *Decl) const {
-  if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
+TagType *ASTContext::getTagTypeInternal(ElaboratedTypeKeyword Keyword,
+                                        NestedNameSpecifier Qualifier,
+                                        const TagDecl *TD, bool OwnsTag,
+                                        bool IsInjected,
+                                        const Type *CanonicalType,
+                                        bool WithFoldingSetNode) const {
+  auto [TC, Size] = [&] {
+    switch (TD->getDeclKind()) {
+    case Decl::Enum:
+      static_assert(alignof(EnumType) == alignof(TagType));
+      return std::make_tuple(Type::Enum, sizeof(EnumType));
+    case Decl::ClassTemplatePartialSpecialization:
+    case Decl::ClassTemplateSpecialization:
+    case Decl::CXXRecord:
+      static_assert(alignof(RecordType) == alignof(TagType));
+      static_assert(alignof(InjectedClassNameType) == alignof(TagType));
+      if (cast<CXXRecordDecl>(TD)->hasInjectedClassType())
+        return std::make_tuple(Type::InjectedClassName,
+                               sizeof(InjectedClassNameType));
+      [[fallthrough]];
+    case Decl::Record:
+      return std::make_tuple(Type::Record, sizeof(RecordType));
+    default:
+      llvm_unreachable("unexpected decl kind");
+    }
+  }();
 
-  if (const RecordDecl *PrevDecl = Decl->getPreviousDecl())
-    if (PrevDecl->TypeForDecl)
-      return QualType(Decl->TypeForDecl = PrevDecl->TypeForDecl, 0);
+  if (Qualifier) {
+    static_assert(alignof(NestedNameSpecifier) <= alignof(TagType));
+    Size = llvm::alignTo(Size, alignof(NestedNameSpecifier)) +
+           sizeof(NestedNameSpecifier);
+  }
+  void *Mem;
+  if (WithFoldingSetNode) {
+    // FIXME: It would be more profitable to tail allocate the folding set node
+    // from the type, instead of the other way around, due to the greater
+    // alignment requirements of the type. But this makes it harder to deal with
+    // the different type node sizes. This would require either uniquing from
+    // different folding sets, or having the folding setaccept a
+    // contextual parameter which is not fixed at construction.
+    Mem = Allocate(
+        sizeof(TagTypeFoldingSetPlaceholder) +
+            TagTypeFoldingSetPlaceholder::getOffset() + Size,
+        std::max(alignof(TagTypeFoldingSetPlaceholder), alignof(TagType)));
+    auto *T = new (Mem) TagTypeFoldingSetPlaceholder();
+    Mem = T->getTagType();
+  } else {
+    Mem = Allocate(Size, alignof(TagType));
+  }
+
+  auto *T = [&, TC = TC]() -> TagType * {
+    switch (TC) {
+    case Type::Enum: {
+      assert(isa<EnumDecl>(TD));
+      auto *T = new (Mem) EnumType(TC, Keyword, Qualifier, TD, OwnsTag,
+                                   IsInjected, CanonicalType);
+      assert(reinterpret_cast<void *>(T) ==
+                 reinterpret_cast<void *>(static_cast<TagType *>(T)) &&
+             "TagType must be the first base of EnumType");
+      return T;
+    }
+    case Type::Record: {
+      assert(isa<RecordDecl>(TD));
+      auto *T = new (Mem) RecordType(TC, Keyword, Qualifier, TD, OwnsTag,
+                                     IsInjected, CanonicalType);
+      assert(reinterpret_cast<void *>(T) ==
+                 reinterpret_cast<void *>(static_cast<TagType *>(T)) &&
+             "TagType must be the first base of RecordType");
+      return T;
+    }
+    case Type::InjectedClassName: {
+      auto *T = new (Mem) InjectedClassNameType(Keyword, Qualifier, TD,
+                                                IsInjected, CanonicalType);
+      assert(reinterpret_cast<void *>(T) ==
+                 reinterpret_cast<void *>(static_cast<TagType *>(T)) &&
+             "TagType must be the first base of InjectedClassNameType");
+      return T;
+    }
+    default:
+      llvm_unreachable("unexpected type class");
+    }
+  }();
+  assert(T->getKeyword() == Keyword);
+  assert(T->getQualifier() == Qualifier);
+  assert(T->getOriginalDecl() == TD);
+  assert(T->isInjected() == IsInjected);
+  assert(T->isTagOwned() == OwnsTag);
+  assert((T->isCanonicalUnqualified()
+              ? QualType()
+              : T->getCanonicalTypeInternal()) == QualType(CanonicalType, 0));
+  Types.push_back(T);
+  return T;
+}
 
-  auto *newType = new (*this, alignof(RecordType)) RecordType(Decl);
-  Decl->TypeForDecl = newType;
-  Types.push_back(newType);
-  return QualType(newType, 0);
+static bool getNonInjectedClassName(const TagDecl *&TD) {
+  if (const auto *RD = dyn_cast<CXXRecordDecl>(TD);
+      RD && RD->isInjectedClassName()) {
+    TD = cast<TagDecl>(RD->getDeclContext());
+    return true;
+  }
+  return false;
 }
 
-QualType ASTContext::getEnumType(const EnumDecl *Decl) const {
-  if (Decl->TypeForDecl) return QualType(Decl->TypeForDecl, 0);
+CanQualType ASTContext::getCanonicalTagType(const TagDecl *TD) const {
+  ::getNonInjectedClassName(TD);
+  TD = TD->getCanonicalDecl();
+  if (TD->TypeForDecl)
+    return TD->TypeForDecl->getCanonicalTypeUnqualified();
+
+  const Type *CanonicalType = getTagTypeInternal(
+      ElaboratedTypeKeyword::None,
+      /*Qualifier=*/std::nullopt, TD,
+      /*OwnsTag=*/false, /*IsInjected=*/false, /*CanonicalType=*/nullptr,
+      /*WithFoldingSetNode=*/false);
+  TD->TypeForDecl = CanonicalType;
+  return CanQualType::CreateUnsafe(QualType(CanonicalType, 0));
+}
+
+QualType ASTContext::getTagType(ElaboratedTypeKeyword Keyword,
+                                NestedNameSpecifier Qualifier,
+                                const TagDecl *TD, bool OwnsTag) const {
+  ElaboratedTypeKeyword PreferredKeyword =
+      getLangOpts().CPlusPlus
+          ? ElaboratedTypeKeyword::None
+          : KeywordHelpers::getKeywordForTagTypeKind(TD->getTagKind());
+
+  if (Keyword == PreferredKeyword && !Qualifier && !OwnsTag) {
+    if (const Type *T = TD->TypeForDecl; T && !T->isCanonicalUnqualified())
+      return QualType(T, 0);
+
+    bool IsInjected = ::getNonInjectedClassName(TD);
+    const Type *CanonicalType = getCanonicalTagType(TD).getTypePtr();
+    const Type *T =
+        getTagTypeInternal(Keyword,
+                           /*Qualifier=*/std::nullopt, TD,
+                           /*OwnsTag=*/false, IsInjected, CanonicalType,
+                           /*WithFoldingSetNode=*/false);
+    TD->TypeForDecl = T;
+    return QualType(T, 0);
+  }
 
-  if (const EnumDecl *PrevDecl = Decl->getPreviousDecl())
-    if (PrevDecl->TypeForDecl)
-      return QualType(Decl->TypeForDecl = PrevDecl->TypeForDecl, 0);
+  bool IsInjected = ::getNonInjectedClassName(TD);
 
-  auto *newType = new (*this, alignof(EnumType)) EnumType(Decl);
-  Decl->TypeForDecl = newType;
-  Types.push_back(newType);
-  return QualType(newType, 0);
+  llvm::FoldingSetNodeID ID;
+  TagTypeFoldingSetPlaceholder::Profile(ID, Keyword, Qualifier, TD, OwnsTag,
+                                        IsInjected);
+
+  void *InsertPos = nullptr;
+  if (TagTypeFoldingSetPlaceholder *T =
+          TagTypes.FindNodeOrInsertPos(ID, InsertPos))
+    return QualType(T->getTagType(), 0);
+
+  const Type *CanonicalType = getCanonicalTagType(TD).getTypePtr();
+  TagType *T = getTagTypeInternal(Keyword, Qualifier, TD, OwnsTag, IsInjected,
+                                  CanonicalType, /*WithFoldingSetNode=*/true);
+  TagTypes.InsertNode(TagTypeFoldingSetPlaceholder::fromTagType(T), InsertPos);
+  return QualType(T, 0);
 }
 
 bool ASTContext::computeBestEnumTypes(bool IsPacked, unsigned NumNegativeBits,
@@ -5495,21 +5641,69 @@ bool ASTContext::isRepresentableIntegerValue(llvm::APSInt &Value, QualType T) {
   return Value.getSignificantBits() <= BitWidth;
 }
 
-QualType ASTContext::getUnresolvedUsingType(
-    const UnresolvedUsingTypenameDecl *Decl) const {
-  if (Decl->TypeForDecl)
-    return QualType(Decl->TypeForDecl, 0);
+UnresolvedUsingType *ASTContext::getUnresolvedUsingTypeInternal(
+    ElaboratedTypeKeyword Keyword, NestedNameSpecifier Qualifier,
+    const UnresolvedUsingTypenameDecl *D, void *InsertPos,
+    const Type *CanonicalType) const {
+  void *Mem = Allocate(
+      UnresolvedUsingType::totalSizeToAlloc<
+          FoldingSetPlaceholder<UnresolvedUsingType>, NestedNameSpecifier>(
+          !!InsertPos, !!Qualifier),
+      alignof(UnresolvedUsingType));
+  auto *T = new (Mem) UnresolvedUsingType(Keyword, Qualifier, D, CanonicalType);
+  if (InsertPos) {
+    auto *Placeholder = new (T->getFoldingSetPlaceholder())
+        FoldingSetPlaceholder<TypedefType>();
+    TypedefTypes.InsertNode(Placeholder, InsertPos);
+  }
+  Types.push_back(T);
+  return T;
+}
 
-  if (const UnresolvedUsingTypenameDecl *CanonicalDecl =
-          Decl->getCanonicalDecl())
-    if (CanonicalDecl->TypeForDecl)
-      return QualType(Decl->TypeForDecl = CanonicalDecl->TypeForDecl, 0);
+CanQualType ASTContext::getCanonicalUnresolvedUsingType(
+    const UnresolvedUsingTypenameDecl *D) const {
+  D = D->getCanonicalDecl();
+  if (D->TypeForDecl)
+    return D->TypeForDecl->getCanonicalTypeUnqualified();
 
-  Type *newType =
-      new (*this, alignof(UnresolvedUsingType)) UnresolvedUsingType(Decl);
-  Decl->TypeForDecl = newType;
-  Types.push_back(newType);
-  return QualType(newType, 0);
+  const Type *CanonicalType = getUnresolvedUsingTypeInternal(
+      ElaboratedTypeKeyword::None,
+      /*Qualifier=*/std::nullopt, D,
+      /*InsertPos=*/nullptr, /*CanonicalType=*/nullptr);
+  D->TypeForDecl = CanonicalType;
+  return CanQualType::CreateUnsafe(QualType(CanonicalType, 0));
+}
+
+QualType
+ASTContext::getUnresolvedUsingType(ElaboratedTypeKeyword Keyword,
+                                   NestedNameSpecifier Qualifier,
+                                   const UnresolvedUsingTypenameDecl *D) const {
+  if (Keyword == ElaboratedTypeKeyword::None && !Qualifier) {
+    if (const Type *T = D->TypeForDecl; T && !T->isCanonicalUnqualified())
+      return QualType(T, 0);
+
+    const Type *CanonicalType = getCanonicalUnresolvedUsingType(D).getTypePtr();
+    const Type *T =
+        getUnresolvedUsingTypeInternal(ElaboratedTypeKeyword::None,
+                                       /*Qualifier=*/std::nullopt, D,
+                                       /*InsertPos=*/nullptr, CanonicalType);
+    D->TypeForDecl = T;
+    return QualType(T, 0);
+  }
+
+  llvm::FoldingSetNodeID ID;
+  UnresolvedUsingType::Profile(ID, Keyword, Qualifier, D);
+
+  void *InsertPos = nullptr;
+  if (FoldingSetPlaceholder<UnresolvedUsingType> *Placeholder =
+          UnresolvedUsingTypes.FindNodeOrInsertPos(ID, InsertPos))
+    return QualType(Placeholder->getType(), 0);
+  assert(InsertPos);
+
+  const Type *CanonicalType = getCanonicalUnresolvedUsingType(D).getTypePtr();
+  const Type *T = getUnresolvedUsingTypeInternal(Keyword, Qualifier, D,
+                                                 InsertPos, CanonicalType);
+  return QualType(T, 0);
 }
 
 QualType ASTContext::getAttributedType(attr::Kind attrKind,
@@ -5655,7 +5849,6 @@ QualType ASTContext::getSubstTemplateTypeParmType(QualType Replacement,
   return QualType(SubstParm, 0);
 }
 
-/// Retrieve a
 QualType
 ASTContext::getSubstTemplateTypeParmPackType(Decl *AssociatedDecl,
                                              unsigned Index, bool Final,
@@ -5694,6 +5887,34 @@ ASTContext::getSubstTemplateTypeParmPackType(Decl *AssociatedDecl,
   return QualType(SubstParm, 0);
 }
 
+QualType
+ASTContext::getSubstBuiltinTemplatePack(const TemplateArgument &ArgPack) {
+  assert(llvm::all_of(ArgPack.pack_elements(),
+                      [](const auto &P) {
+                        return P.getKind() == TemplateArgument::Type;
+                      }) &&
+         "Pack contains a non-type");
+
+  llvm::FoldingSetNodeID ID;
+  SubstBuiltinTemplatePackType::Profile(ID, ArgPack);
+
+  void *InsertPos = nullptr;
+  if (auto *T =
+          SubstBuiltinTemplatePackTypes.FindNodeOrInsertPos(ID, InsertPos))
+    return QualType(T, 0);
+
+  QualType Canon;
+  TemplateArgument CanonArgPack = getCanonicalTemplateArgument(ArgPack);
+  if (!CanonArgPack.structurallyEquals(ArgPack))
+    Canon = getSubstBuiltinTemplatePack(CanonArgPack);
+
+  auto *PackType = new (*this, alignof(SubstBuiltinTemplatePackType))
+      SubstBuiltinTemplatePackType(Canon, ArgPack);
+  Types.push_back(PackType);
+  SubstBuiltinTemplatePackTypes.InsertNode(PackType, InsertPos);
+  return QualType(PackType, 0);
+}
+
 /// Retrieve the template type parameter type for a template
 /// parameter or parameter pack with the given depth, index, and (optionally)
 /// name.
@@ -5729,34 +5950,32 @@ QualType ASTContext::getTemplateTypeParmType(unsigned Depth, unsigned Index,
 }
 
 TypeSourceInfo *ASTContext::getTemplateSpecializationTypeInfo(
+    ElaboratedTypeKeyword Keyword, SourceLocation ElaboratedKeywordLoc,
+    NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateKeywordLoc,
     TemplateName Name, SourceLocation NameLoc,
     const TemplateArgumentListInfo &SpecifiedArgs,
     ArrayRef<TemplateArgument> CanonicalArgs, QualType Underlying) const {
-  QualType TST = getTemplateSpecializationType(Name, SpecifiedArgs.arguments(),
-                                               CanonicalArgs, Underlying);
+  QualType TST = getTemplateSpecializationType(
+      Keyword, Name, SpecifiedArgs.arguments(), CanonicalArgs, Underlying);
 
   TypeSourceInfo *DI = CreateTypeSourceInfo(TST);
-  TemplateSpecializationTypeLoc TL =
-      DI->getTypeLoc().castAs<TemplateSpecializationTypeLoc>();
-  TL.setTemplateKeywordLoc(SourceLocation());
-  TL.setTemplateNameLoc(NameLoc);
-  TL.setLAngleLoc(SpecifiedArgs.getLAngleLoc());
-  TL.setRAngleLoc(SpecifiedArgs.getRAngleLoc());
-  for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
-    TL.setArgLocInfo(i, SpecifiedArgs[i].getLocInfo());
+  DI->getTypeLoc().castAs<TemplateSpecializationTypeLoc>().set(
+      ElaboratedKeywordLoc, QualifierLoc, TemplateKeywordLoc, NameLoc,
+      SpecifiedArgs);
   return DI;
 }
 
 QualType ASTContext::getTemplateSpecializationType(
-    TemplateName Template, ArrayRef<TemplateArgumentLoc> SpecifiedArgs,
+    ElaboratedTypeKeyword Keyword, TemplateName Template,
+    ArrayRef<TemplateArgumentLoc> SpecifiedArgs,
     ArrayRef<TemplateArgument> CanonicalArgs, QualType Underlying) const {
   SmallVector<TemplateArgument, 4> SpecifiedArgVec;
   SpecifiedArgVec.reserve(SpecifiedArgs.size());
   for (const TemplateArgumentLoc &Arg : SpecifiedArgs)
     SpecifiedArgVec.push_back(Arg.getArgument());
 
-  return getTemplateSpecializationType(Template, SpecifiedArgVec, CanonicalArgs,
-                                       Underlying);
+  return getTemplateSpecializationType(Keyword, Template, SpecifiedArgVec,
+                                       CanonicalArgs, Underlying);
 }
 
 [[maybe_unused]] static bool
@@ -5787,7 +6006,8 @@ QualType ASTContext::getCanonicalTemplateSpecializationType(
                            sizeof(TemplateArgument) * Args.size(),
                        alignof(TemplateSpecializationType));
   auto *Spec = new (Mem)
-      TemplateSpecializationType(Template, /*IsAlias=*/false, Args, QualType());
+      TemplateSpecializationType(ElaboratedTypeKeyword::None, Template,
+                                 /*IsAlias=*/false, Args, QualType());
   assert(Spec->isDependentType() &&
          "canonical template specialization must be dependent");
   Types.push_back(Spec);
@@ -5796,7 +6016,8 @@ QualType ASTContext::getCanonicalTemplateSpecializationType(
 }
 
 QualType ASTContext::getTemplateSpecializationType(
-    TemplateName Template, ArrayRef<TemplateArgument> SpecifiedArgs,
+    ElaboratedTypeKeyword Keyword, TemplateName Template,
+    ArrayRef<TemplateArgument> SpecifiedArgs,
     ArrayRef<TemplateArgument> CanonicalArgs, QualType Underlying) const {
   assert(!Template.getUnderlying().getAsDependentTemplateName() &&
          "No dependent template names here!");
@@ -5806,7 +6027,8 @@ QualType ASTContext::getTemplateSpecializationType(
   if (Underlying.isNull()) {
     TemplateName CanonTemplate =
         getCanonicalTemplateName(Template, /*IgnoreDeduced=*/true);
-    bool NonCanonical = Template != CanonTemplate;
+    bool NonCanonical =
+        Template != CanonTemplate || Keyword != ElaboratedTypeKeyword::None;
     SmallVector<TemplateArgument, 4> CanonArgsVec;
     if (CanonicalArgs.empty()) {
       CanonArgsVec = SmallVector<TemplateArgument, 4>(SpecifiedArgs);
@@ -5837,42 +6059,12 @@ QualType ASTContext::getTemplateSpecializationType(
                            sizeof(TemplateArgument) * SpecifiedArgs.size() +
                            (IsTypeAlias ? sizeof(QualType) : 0),
                        alignof(TemplateSpecializationType));
-  auto *Spec = new (Mem) TemplateSpecializationType(Template, IsTypeAlias,
-                                                    SpecifiedArgs, Underlying);
+  auto *Spec = new (Mem) TemplateSpecializationType(
+      Keyword, Template, IsTypeAlias, SpecifiedArgs, Underlying);
   Types.push_back(Spec);
   return QualType(Spec, 0);
 }
 
-QualType ASTContext::getElaboratedType(ElaboratedTypeKeyword Keyword,
-                                       NestedNameSpecifier *NNS,
-                                       QualType NamedType,
-                                       TagDecl *OwnedTagDecl) const {
-  llvm::FoldingSetNodeID ID;
-  ElaboratedType::Profile(ID, Keyword, NNS, NamedType, OwnedTagDecl);
-
-  void *InsertPos = nullptr;
-  ElaboratedType *T = ElaboratedTypes.FindNodeOrInsertPos(ID, InsertPos);
-  if (T)
-    return QualType(T, 0);
-
-  QualType Canon = NamedType;
-  if (!Canon.isCanonical()) {
-    Canon = getCanonicalType(NamedType);
-    ElaboratedType *CheckT = ElaboratedTypes.FindNodeOrInsertPos(ID, InsertPos);
-    assert(!CheckT && "Elaborated canonical type broken");
-    (void)CheckT;
-  }
-
-  void *Mem =
-      Allocate(ElaboratedType::totalSizeToAlloc<TagDecl *>(!!OwnedTagDecl),
-               alignof(ElaboratedType));
-  T = new (Mem) ElaboratedType(Keyword, NNS, NamedType, Canon, OwnedTagDecl);
-
-  Types.push_back(T);
-  ElaboratedTypes.InsertNode(T, InsertPos);
-  return QualType(T, 0);
-}
-
 QualType
 ASTContext::getParenType(QualType InnerType) const {
   llvm::FoldingSetNodeID ID;
@@ -5935,7 +6127,7 @@ getCanonicalElaboratedTypeKeyword(ElaboratedTypeKeyword Keyword) {
 }
 
 QualType ASTContext::getDependentNameType(ElaboratedTypeKeyword Keyword,
-                                          NestedNameSpecifier *NNS,
+                                          NestedNameSpecifier NNS,
                                           const IdentifierInfo *Name) const {
   llvm::FoldingSetNodeID ID;
   DependentNameType::Profile(ID, Keyword, NNS, Name);
@@ -5947,7 +6139,7 @@ QualType ASTContext::getDependentNameType(ElaboratedTypeKeyword Keyword,
 
   ElaboratedTypeKeyword CanonKeyword =
       getCanonicalElaboratedTypeKeyword(Keyword);
-  NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
+  NestedNameSpecifier CanonNNS = NNS.getCanonical();
 
   QualType Canon;
   if (CanonKeyword != Keyword || CanonNNS != NNS) {
@@ -5985,13 +6177,13 @@ QualType ASTContext::getDependentTemplateSpecializationType(
       T_iter != DependentTemplateSpecializationTypes.end())
     return QualType(T_iter->getSecond(), 0);
 
-  NestedNameSpecifier *NNS = Name.getQualifier();
+  NestedNameSpecifier NNS = Name.getQualifier();
 
   QualType Canon;
   if (!IsCanonical) {
     ElaboratedTypeKeyword CanonKeyword =
         getCanonicalElaboratedTypeKeyword(Keyword);
-    NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
+    NestedNameSpecifier CanonNNS = NNS.getCanonical();
     bool AnyNonCanonArgs = false;
     auto CanonArgs =
         ::getCanonicalTemplateArguments(*this, Args, AnyNonCanonArgs);
@@ -6006,7 +6198,7 @@ QualType ASTContext::getDependentTemplateSpecializationType(
   } else {
     assert(Keyword == getCanonicalElaboratedTypeKeyword(Keyword));
     assert(Name.hasTemplateKeyword());
-    assert(NNS == getCanonicalNestedNameSpecifier(NNS));
+    assert(NNS.isCanonical());
 #ifndef NDEBUG
     for (const auto &Arg : Args)
       assert(Arg.structurallyEquals(getCanonicalTemplateArgument(Arg)));
@@ -6062,7 +6254,8 @@ TemplateArgument ASTContext::getInjectedTemplateArg(NamedDecl *Param) const {
   } else {
     auto *TTP = cast<TemplateTemplateParmDecl>(Param);
     TemplateName Name = getQualifiedTemplateName(
-        nullptr, /*TemplateKeyword=*/false, TemplateName(TTP));
+        /*Qualifier=*/std::nullopt, /*TemplateKeyword=*/false,
+        TemplateName(TTP));
     if (TTP->isParameterPack())
       Arg = TemplateArgument(Name, /*NumExpansions=*/std::nullopt);
     else
@@ -6334,11 +6527,11 @@ void ASTContext::adjustObjCTypeParamBoundType(const ObjCTypeParamDecl *Orig,
                                               ObjCTypeParamDecl *New) const {
   New->setTypeSourceInfo(getTrivialTypeSourceInfo(Orig->getUnderlyingType()));
   // Update TypeForDecl after updating TypeSourceInfo.
-  auto NewTypeParamTy = cast<ObjCTypeParamType>(New->getTypeForDecl());
+  auto *NewTypeParamTy = cast<ObjCTypeParamType>(New->TypeForDecl);
   SmallVector<ObjCProtocolDecl *, 8> protocols;
   protocols.append(NewTypeParamTy->qual_begin(), NewTypeParamTy->qual_end());
   QualType UpdatedTy = getObjCTypeParamType(New, protocols);
-  New->setTypeForDecl(UpdatedTy.getTypePtr());
+  New->TypeForDecl = UpdatedTy.getTypePtr();
 }
 
 /// ObjCObjectAdoptsQTypeProtocols - Checks that protocols in IC's
@@ -6737,20 +6930,20 @@ QualType ASTContext::getUnconstrainedType(QualType T) const {
 }
 
 QualType ASTContext::getDeducedTemplateSpecializationTypeInternal(
-    TemplateName Template, QualType DeducedType, bool IsDependent,
-    QualType Canon) const {
+    ElaboratedTypeKeyword Keyword, TemplateName Template, QualType DeducedType,
+    bool IsDependent, QualType Canon) const {
   // Look in the folding set for an existing type.
   void *InsertPos = nullptr;
   llvm::FoldingSetNodeID ID;
-  DeducedTemplateSpecializationType::Profile(ID, Template, DeducedType,
+  DeducedTemplateSpecializationType::Profile(ID, Keyword, Template, DeducedType,
                                              IsDependent);
   if (DeducedTemplateSpecializationType *DTST =
           DeducedTemplateSpecializationTypes.FindNodeOrInsertPos(ID, InsertPos))
     return QualType(DTST, 0);
 
   auto *DTST = new (*this, alignof(DeducedTemplateSpecializationType))
-      DeducedTemplateSpecializationType(Template, DeducedType, IsDependent,
-                                        Canon);
+      DeducedTemplateSpecializationType(Keyword, Template, DeducedType,
+                                        IsDependent, Canon);
 
 #ifndef NDEBUG
   llvm::FoldingSetNodeID TempID;
@@ -6766,14 +6959,20 @@ QualType ASTContext::getDeducedTemplateSpecializationTypeInternal(
 /// which has been deduced to the given type, or to the canonical undeduced
 /// such type, or the canonical deduced-but-dependent such type.
 QualType ASTContext::getDeducedTemplateSpecializationType(
-    TemplateName Template, QualType DeducedType, bool IsDependent) const {
-  QualType Canon = DeducedType.isNull()
-                       ? getDeducedTemplateSpecializationTypeInternal(
-                             getCanonicalTemplateName(Template), QualType(),
-                             IsDependent, QualType())
-                       : DeducedType.getCanonicalType();
-  return getDeducedTemplateSpecializationTypeInternal(Template, DeducedType,
-                                                      IsDependent, Canon);
+    ElaboratedTypeKeyword Keyword, TemplateName Template, QualType DeducedType,
+    bool IsDependent) const {
+  // FIXME: This could save an extra hash table lookup if it handled all the
+  // parameters already being canonical.
+  // FIXME: Can this be formed from a DependentTemplateName, such that the
+  // keyword should be part of the canonical type?
+  QualType Canon =
+      DeducedType.isNull()
+          ? getDeducedTemplateSpecializationTypeInternal(
+                ElaboratedTypeKeyword::None, getCanonicalTemplateName(Template),
+                QualType(), IsDependent, QualType())
+          : DeducedType.getCanonicalType();
+  return getDeducedTemplateSpecializationTypeInternal(
+      Keyword, Template, DeducedType, IsDependent, Canon);
 }
 
 /// getAtomicType - Return the uniqued reference to the atomic type for
@@ -6823,15 +7022,6 @@ QualType ASTContext::getAutoRRefDeductType() const {
   return AutoRRefDeductTy;
 }
 
-/// getTagDeclType - Return the unique reference to the type for the
-/// specified TagDecl (struct/union/class/enum) decl.
-QualType ASTContext::getTagDeclType(const TagDecl *Decl) const {
-  assert(Decl);
-  // FIXME: What is the design on getTagDeclType when it requires casting
-  // away const?  mutable?
-  return getTypeDeclType(const_cast<TagDecl*>(Decl));
-}
-
 /// getSizeType - Return the unique type for "size_t" (C99 7.17), the result
 /// of the sizeof operator (C99 6.5.3.4p4). The value is target dependent and
 /// needs to agree with the definition in <stddef.h>.
@@ -7053,8 +7243,8 @@ bool ASTContext::UnwrapSimilarTypes(QualType &T1, QualType &T2,
         *RD2 = T2MPType->getMostRecentCXXRecordDecl();
         RD1 != RD2 && RD1->getCanonicalDecl() != RD2->getCanonicalDecl())
       return false;
-    if (getCanonicalNestedNameSpecifier(T1MPType->getQualifier()) !=
-        getCanonicalNestedNameSpecifier(T2MPType->getQualifier()))
+    if (T1MPType->getQualifier().getCanonical() !=
+        T2MPType->getQualifier().getCanonical())
       return false;
     T1 = T1MPType->getPointeeType();
     T2 = T2MPType->getPointeeType();
@@ -7211,9 +7401,8 @@ TemplateName ASTContext::getCanonicalTemplateName(TemplateName Name,
   case TemplateName::DependentTemplate: {
     DependentTemplateName *DTN = Name.getAsDependentTemplateName();
     assert(DTN && "Non-dependent template names must refer to template decls.");
-    NestedNameSpecifier *Qualifier = DTN->getQualifier();
-    NestedNameSpecifier *CanonQualifier =
-        getCanonicalNestedNameSpecifier(Qualifier);
+    NestedNameSpecifier Qualifier = DTN->getQualifier();
+    NestedNameSpecifier CanonQualifier = Qualifier.getCanonical();
     if (Qualifier != CanonQualifier || !DTN->hasTemplateKeyword())
       return getDependentTemplateName({CanonQualifier, DTN->getName(),
                                        /*HasTemplateKeyword=*/true});
@@ -7430,38 +7619,40 @@ bool ASTContext::isSameDefaultTemplateArgument(const NamedDecl *X,
   return hasSameTemplateName(TAX.getAsTemplate(), TAY.getAsTemplate());
 }
 
-static bool isSameQualifier(const NestedNameSpecifier *X,
-                            const NestedNameSpecifier *Y) {
-  if (X->getKind() != Y->getKind())
+static bool isSameQualifier(const NestedNameSpecifier X,
+                            const NestedNameSpecifier Y) {
+  if (X == Y)
+    return true;
+  if (!X || !Y)
+    return false;
+
+  auto Kind = X.getKind();
+  if (Kind != Y.getKind())
     return false;
 
   // FIXME: For namespaces and types, we're permitted to check that the entity
   // is named via the same tokens. We should probably do so.
-  switch (X->getKind()) {
-  case NestedNameSpecifier::Identifier:
-    if (X->getAsIdentifier() != Y->getAsIdentifier())
-      return false;
-    break;
-  case NestedNameSpecifier::Namespace:
-    if (!declaresSameEntity(X->getAsNamespace(), Y->getAsNamespace()))
+  switch (Kind) {
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [NamespaceX, PrefixX] = X.getAsNamespaceAndPrefix();
+    auto [NamespaceY, PrefixY] = Y.getAsNamespaceAndPrefix();
+    if (!declaresSameEntity(NamespaceX->getNamespace(),
+                            NamespaceY->getNamespace()))
       return false;
-    break;
-  case NestedNameSpecifier::TypeSpec:
-    if (X->getAsType()->getCanonicalTypeInternal() !=
-        Y->getAsType()->getCanonicalTypeInternal())
+    return isSameQualifier(PrefixX, PrefixY);
+  }
+  case NestedNameSpecifier::Kind::Type: {
+    const auto *TX = X.getAsType(), *TY = Y.getAsType();
+    if (TX->getCanonicalTypeInternal() != TY->getCanonicalTypeInternal())
       return false;
-    break;
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Super:
+    return isSameQualifier(TX->getPrefix(), TY->getPrefix());
+  }
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     return true;
   }
-
-  // Recurse into earlier portion of NNS, if any.
-  auto *PX = X->getPrefix();
-  auto *PY = Y->getPrefix();
-  if (PX && PY)
-    return isSameQualifier(PX, PY);
-  return !PX && !PY;
+  llvm_unreachable("unhandled qualifier kind");
 }
 
 static bool hasSameCudaAttrs(const FunctionDecl *A, const FunctionDecl *B) {
@@ -7854,63 +8045,6 @@ bool ASTContext::isSameTemplateArgument(const TemplateArgument &Arg1,
   llvm_unreachable("Unhandled template argument kind");
 }
 
-NestedNameSpecifier *
-ASTContext::getCanonicalNestedNameSpecifier(NestedNameSpecifier *NNS) const {
-  if (!NNS)
-    return nullptr;
-
-  switch (NNS->getKind()) {
-  case NestedNameSpecifier::Identifier:
-    // Canonicalize the prefix but keep the identifier the same.
-    return NestedNameSpecifier::Create(*this,
-                         getCanonicalNestedNameSpecifier(NNS->getPrefix()),
-                                       NNS->getAsIdentifier());
-
-  case NestedNameSpecifier::Namespace:
-    // A namespace is canonical; build a nested-name-specifier with
-    // this namespace and no prefix.
-    return NestedNameSpecifier::Create(
-        *this, nullptr, NNS->getAsNamespace()->getNamespace()->getFirstDecl());
-
-  // The difference between TypeSpec and TypeSpecWithTemplate is that the
-  // latter will have the 'template' keyword when printed.
-  case NestedNameSpecifier::TypeSpec: {
-    const Type *T = getCanonicalType(NNS->getAsType());
-
-    // If we have some kind of dependent-named type (e.g., "typename T::type"),
-    // break it apart into its prefix and identifier, then reconsititute those
-    // as the canonical nested-name-specifier. This is required to canonicalize
-    // a dependent nested-name-specifier involving typedefs of dependent-name
-    // types, e.g.,
-    //   typedef typename T::type T1;
-    //   typedef typename T1::type T2;
-    if (const auto *DNT = T->getAs<DependentNameType>())
-      return NestedNameSpecifier::Create(*this, DNT->getQualifier(),
-                                         DNT->getIdentifier());
-    if (const auto *DTST = T->getAs<DependentTemplateSpecializationType>()) {
-      const DependentTemplateStorage &DTN = DTST->getDependentTemplateName();
-      QualType NewT = getDependentTemplateSpecializationType(
-          ElaboratedTypeKeyword::None,
-          {/*NNS=*/nullptr, DTN.getName(), /*HasTemplateKeyword=*/true},
-          DTST->template_arguments(), /*IsCanonical=*/true);
-      assert(NewT.isCanonical());
-      NestedNameSpecifier *Prefix = DTN.getQualifier();
-      if (!Prefix)
-        Prefix = getCanonicalNestedNameSpecifier(NNS->getPrefix());
-      return NestedNameSpecifier::Create(*this, Prefix, NewT.getTypePtr());
-    }
-    return NestedNameSpecifier::Create(*this, nullptr, T);
-  }
-
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Super:
-    // The global specifier and __super specifer are canonical and unique.
-    return NNS;
-  }
-
-  llvm_unreachable("Invalid NestedNameSpecifier::Kind!");
-}
-
 const ArrayType *ASTContext::getAsArrayType(QualType T) const {
   // Handle the non-qualified case efficiently.
   if (!T.hasLocalQualifiers()) {
@@ -8229,7 +8363,7 @@ QualType ASTContext::getPromotedIntegerType(QualType Promotable) const {
   assert(!Promotable.isNull());
   assert(isPromotableIntegerType(Promotable));
   if (const auto *ET = Promotable->getAs<EnumType>())
-    return ET->getDecl()->getPromotionType();
+    return ET->getOriginalDecl()->getDefinitionOrSelf()->getPromotionType();
 
   if (const auto *BT = Promotable->getAs<BuiltinType>()) {
     // C++ [conv.prom]: A prvalue of type char16_t, char32_t, or wchar_t
@@ -8288,8 +8422,9 @@ Qualifiers::ObjCLifetime ASTContext::getInnerObjCOwnership(QualType T) const {
 static const Type *getIntegerTypeForEnum(const EnumType *ET) {
   // Incomplete enum types are not treated as integer types.
   // FIXME: In C++, enum types are never integer types.
-  if (ET->getDecl()->isComplete() && !ET->getDecl()->isScoped())
-    return ET->getDecl()->getIntegerType().getTypePtr();
+  const EnumDecl *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+  if (ED->isComplete() && !ED->isScoped())
+    return ED->getIntegerType().getTypePtr();
   return nullptr;
 }
 
@@ -8418,7 +8553,7 @@ TypedefDecl *ASTContext::getCFConstantStringDecl() const {
   CFConstantStringTagDecl->completeDefinition();
   // This type is designed to be compatible with NSConstantString, but cannot
   // use the same name, since NSConstantString is an interface.
-  auto tagType = getTagDeclType(CFConstantStringTagDecl);
+  CanQualType tagType = getCanonicalTagType(CFConstantStringTagDecl);
   CFConstantStringTypeDecl =
       buildImplicitTypedef(tagType, "__NSConstantString");
 
@@ -8433,14 +8568,15 @@ RecordDecl *ASTContext::getCFConstantStringTagDecl() const {
 
 // getCFConstantStringType - Return the type used for constant CFStrings.
 QualType ASTContext::getCFConstantStringType() const {
-  return getTypedefType(getCFConstantStringDecl());
+  return getTypedefType(ElaboratedTypeKeyword::None, /*Qualifier=*/std::nullopt,
+                        getCFConstantStringDecl());
 }
 
 QualType ASTContext::getObjCSuperType() const {
   if (ObjCSuperType.isNull()) {
     RecordDecl *ObjCSuperTypeDecl = buildImplicitRecord("objc_super");
     getTranslationUnitDecl()->addDecl(ObjCSuperTypeDecl);
-    ObjCSuperType = getTagDeclType(ObjCSuperTypeDecl);
+    ObjCSuperType = getCanonicalTagType(ObjCSuperTypeDecl);
   }
   return ObjCSuperType;
 }
@@ -8449,12 +8585,12 @@ void ASTContext::setCFConstantStringType(QualType T) {
   const auto *TD = T->castAs<TypedefType>();
   CFConstantStringTypeDecl = cast<TypedefDecl>(TD->getDecl());
   const auto *TagType = TD->castAs<RecordType>();
-  CFConstantStringTagDecl = TagType->getDecl();
+  CFConstantStringTagDecl = TagType->getOriginalDecl()->getDefinitionOrSelf();
 }
 
 QualType ASTContext::getBlockDescriptorType() const {
   if (BlockDescriptorType)
-    return getTagDeclType(BlockDescriptorType);
+    return getCanonicalTagType(BlockDescriptorType);
 
   RecordDecl *RD;
   // FIXME: Needs the FlagAppleBlock bit.
@@ -8484,12 +8620,12 @@ QualType ASTContext::getBlockDescriptorType() const {
 
   BlockDescriptorType = RD;
 
-  return getTagDeclType(BlockDescriptorType);
+  return getCanonicalTagType(BlockDescriptorType);
 }
 
 QualType ASTContext::getBlockDescriptorExtendedType() const {
   if (BlockDescriptorExtendedType)
-    return getTagDeclType(BlockDescriptorExtendedType);
+    return getCanonicalTagType(BlockDescriptorExtendedType);
 
   RecordDecl *RD;
   // FIXME: Needs the FlagAppleBlock bit.
@@ -8523,7 +8659,7 @@ QualType ASTContext::getBlockDescriptorExtendedType() const {
   RD->completeDefinition();
 
   BlockDescriptorExtendedType = RD;
-  return getTagDeclType(BlockDescriptorExtendedType);
+  return getCanonicalTagType(BlockDescriptorExtendedType);
 }
 
 OpenCLTypeKind ASTContext::getOpenCLTypeKind(const Type *T) const {
@@ -9161,7 +9297,7 @@ static char getObjCEncodingForPrimitiveType(const ASTContext *C,
 }
 
 static char ObjCEncodingForEnumType(const ASTContext *C, const EnumType *ET) {
-  EnumDecl *Enum = ET->getDecl();
+  EnumDecl *Enum = ET->getOriginalDecl()->getDefinitionOrSelf();
 
   // The encoding of an non-fixed enum type is always 'i', regardless of size.
   if (!Enum->isFixed())
@@ -9330,13 +9466,14 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
         return;
       }
     } else if (const auto *RTy = PointeeTy->getAs<RecordType>()) {
+      const IdentifierInfo *II = RTy->getOriginalDecl()->getIdentifier();
       // GCC binary compat: Need to convert "struct objc_class *" to "#".
-      if (RTy->getDecl()->getIdentifier() == &Idents.get("objc_class")) {
+      if (II == &Idents.get("objc_class")) {
         S += '#';
         return;
       }
       // GCC binary compat: Need to convert "struct objc_object *" to "@".
-      if (RTy->getDecl()->getIdentifier() == &Idents.get("objc_object")) {
+      if (II == &Idents.get("objc_object")) {
         S += '@';
         return;
       }
@@ -9401,7 +9538,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
     return;
 
   case Type::Record: {
-    RecordDecl *RDecl = cast<RecordType>(CT)->getDecl();
+    RecordDecl *RDecl = cast<RecordType>(CT)->getOriginalDecl();
     S += RDecl->isUnion() ? '(' : '{';
     // Anonymous structures print as '?'
     if (const IdentifierInfo *II = RDecl->getIdentifier()) {
@@ -9900,7 +10037,7 @@ CreateAArch64ABIBuiltinVaListDecl(const ASTContext *Context) {
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
-  QualType VaListTagType = Context->getRecordType(VaListTagDecl);
+  CanQualType VaListTagType = Context->getCanonicalTagType(VaListTagDecl);
 
   // } __builtin_va_list;
   return Context->buildImplicitTypedef(VaListTagType, "__builtin_va_list");
@@ -9952,14 +10089,15 @@ static TypedefDecl *CreatePowerABIBuiltinVaListDecl(const ASTContext *Context) {
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
-  QualType VaListTagType = Context->getRecordType(VaListTagDecl);
+  CanQualType VaListTagType = Context->getCanonicalTagType(VaListTagDecl);
 
   // } __va_list_tag;
   TypedefDecl *VaListTagTypedefDecl =
       Context->buildImplicitTypedef(VaListTagType, "__va_list_tag");
 
   QualType VaListTagTypedefType =
-    Context->getTypedefType(VaListTagTypedefDecl);
+      Context->getTypedefType(ElaboratedTypeKeyword::None,
+                              /*Qualifier=*/std::nullopt, VaListTagTypedefDecl);
 
   // typedef __va_list_tag __builtin_va_list[1];
   llvm::APInt Size(Context->getTypeSize(Context->getSizeType()), 1);
@@ -10011,7 +10149,7 @@ CreateX86_64ABIBuiltinVaListDecl(const ASTContext *Context) {
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
-  QualType VaListTagType = Context->getRecordType(VaListTagDecl);
+  CanQualType VaListTagType = Context->getCanonicalTagType(VaListTagDecl);
 
   // };
 
@@ -10059,7 +10197,7 @@ CreateAAPCSABIBuiltinVaListDecl(const ASTContext *Context) {
   Context->VaListTagDecl = VaListDecl;
 
   // typedef struct __va_list __builtin_va_list;
-  QualType T = Context->getRecordType(VaListDecl);
+  CanQualType T = Context->getCanonicalTagType(VaListDecl);
   return Context->buildImplicitTypedef(T, "__builtin_va_list");
 }
 
@@ -10106,7 +10244,7 @@ CreateSystemZBuiltinVaListDecl(const ASTContext *Context) {
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
-  QualType VaListTagType = Context->getRecordType(VaListTagDecl);
+  CanQualType VaListTagType = Context->getCanonicalTagType(VaListTagDecl);
 
   // };
 
@@ -10153,13 +10291,15 @@ static TypedefDecl *CreateHexagonBuiltinVaListDecl(const ASTContext *Context) {
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
-  QualType VaListTagType = Context->getRecordType(VaListTagDecl);
+  CanQualType VaListTagType = Context->getCanonicalTagType(VaListTagDecl);
 
   // } __va_list_tag;
   TypedefDecl *VaListTagTypedefDecl =
       Context->buildImplicitTypedef(VaListTagType, "__va_list_tag");
 
-  QualType VaListTagTypedefType = Context->getTypedefType(VaListTagTypedefDecl);
+  QualType VaListTagTypedefType =
+      Context->getTypedefType(ElaboratedTypeKeyword::None,
+                              /*Qualifier=*/std::nullopt, VaListTagTypedefDecl);
 
   // typedef __va_list_tag __builtin_va_list[1];
   llvm::APInt Size(Context->getTypeSize(Context->getSizeType()), 1);
@@ -10197,7 +10337,7 @@ CreateXtensaABIBuiltinVaListDecl(const ASTContext *Context) {
   }
   VaListTagDecl->completeDefinition();
   Context->VaListTagDecl = VaListTagDecl;
-  QualType VaListTagType = Context->getRecordType(VaListTagDecl);
+  CanQualType VaListTagType = Context->getCanonicalTagType(VaListTagDecl);
 
   // } __va_list_tag;
   TypedefDecl *VaListTagTypedefDecl =
@@ -10311,7 +10451,7 @@ TemplateName ASTContext::getAssumedTemplateName(DeclarationName Name) const {
 
 /// Retrieve the template name that represents a qualified
 /// template name such as \c std::vector.
-TemplateName ASTContext::getQualifiedTemplateName(NestedNameSpecifier *NNS,
+TemplateName ASTContext::getQualifiedTemplateName(NestedNameSpecifier Qualifier,
                                                   bool TemplateKeyword,
                                                   TemplateName Template) const {
   assert(Template.getKind() == TemplateName::Template ||
@@ -10319,14 +10459,14 @@ TemplateName ASTContext::getQualifiedTemplateName(NestedNameSpecifier *NNS,
 
   // FIXME: Canonicalization?
   llvm::FoldingSetNodeID ID;
-  QualifiedTemplateName::Profile(ID, NNS, TemplateKeyword, Template);
+  QualifiedTemplateName::Profile(ID, Qualifier, TemplateKeyword, Template);
 
   void *InsertPos = nullptr;
   QualifiedTemplateName *QTN =
-    QualifiedTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
+      QualifiedTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
   if (!QTN) {
     QTN = new (*this, alignof(QualifiedTemplateName))
-        QualifiedTemplateName(NNS, TemplateKeyword, Template);
+        QualifiedTemplateName(Qualifier, TemplateKeyword, Template);
     QualifiedTemplateNames.InsertNode(QTN, InsertPos);
   }
 
@@ -11296,7 +11436,7 @@ QualType ASTContext::mergeTransparentUnionType(QualType T, QualType SubType,
                                                bool OfBlockPointer,
                                                bool Unqualified) {
   if (const RecordType *UT = T->getAsUnionType()) {
-    RecordDecl *UD = UT->getDecl();
+    RecordDecl *UD = UT->getOriginalDecl()->getMostRecentDecl();
     if (UD->hasAttr<TransparentUnionAttr>()) {
       for (const auto *I : UD->fields()) {
         QualType ET = I->getType().getUnqualifiedType();
@@ -11442,6 +11582,11 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
     if (lproto->getMethodQuals() != rproto->getMethodQuals())
       return {};
 
+    // Function protos with different 'cfi_salt' values aren't compatible.
+    if (lproto->getExtraAttributeInfo().CFISalt !=
+        rproto->getExtraAttributeInfo().CFISalt)
+      return {};
+
     // Function effects are handled similarly to noreturn, see above.
     FunctionEffectsRef LHSFX = lproto->getFunctionEffects();
     FunctionEffectsRef RHSFX = rproto->getFunctionEffects();
@@ -11528,7 +11673,8 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
       // Look at the converted type of enum types, since that is the type used
       // to pass enum values.
       if (const auto *Enum = paramTy->getAs<EnumType>()) {
-        paramTy = Enum->getDecl()->getIntegerType();
+        paramTy =
+            Enum->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
         if (paramTy.isNull())
           return {};
       }
@@ -11560,7 +11706,8 @@ static QualType mergeEnumWithInteger(ASTContext &Context, const EnumType *ET,
   // a signed integer type, or an unsigned integer type.
   // Compatibility is based on the underlying type, not the promotion
   // type.
-  QualType underlyingType = ET->getDecl()->getIntegerType();
+  QualType underlyingType =
+      ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
   if (underlyingType.isNull())
     return {};
   if (Context.hasSameType(underlyingType, other))
@@ -12119,7 +12266,7 @@ QualType ASTContext::mergeObjCGCQualifiers(QualType LHS, QualType RHS) {
 
 unsigned ASTContext::getIntWidth(QualType T) const {
   if (const auto *ET = T->getAs<EnumType>())
-    T = ET->getDecl()->getIntegerType();
+    T = ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
   if (T->isBooleanType())
     return 1;
   if (const auto *EIT = T->getAs<BitIntType>())
@@ -12145,7 +12292,7 @@ QualType ASTContext::getCorrespondingUnsignedType(QualType T) const {
   // For enums, get the underlying integer type of the enum, and let the general
   // integer type signchanging code handle it.
   if (const auto *ETy = T->getAs<EnumType>())
-    T = ETy->getDecl()->getIntegerType();
+    T = ETy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   switch (T->castAs<BuiltinType>()->getKind()) {
   case BuiltinType::Char_U:
@@ -12219,7 +12366,7 @@ QualType ASTContext::getCorrespondingSignedType(QualType T) const {
   // For enums, get the underlying integer type of the enum, and let the general
   // integer type signchanging code handle it.
   if (const auto *ETy = T->getAs<EnumType>())
-    T = ETy->getDecl()->getIntegerType();
+    T = ETy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   switch (T->castAs<BuiltinType>()->getKind()) {
   case BuiltinType::Char_S:
@@ -12980,6 +13127,14 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
   if (D->hasAttr<WeakRefAttr>())
     return false;
 
+  // SYCL device compilation requires that functions defined with the
+  // sycl_kernel_entry_point or sycl_external attributes be emitted. All
+  // other entities are emitted only if they are used by a function
+  // defined with one of those attributes.
+  if (LangOpts.SYCLIsDevice)
+    return isa<FunctionDecl>(D) && (D->hasAttr<SYCLKernelEntryPointAttr>() ||
+                                    D->hasAttr<SYCLExternalAttr>());
+
   // Aliases and used decls are required.
   if (D->hasAttr<AliasAttr>() || D->hasAttr<UsedAttr>())
     return true;
@@ -12989,15 +13144,6 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
     if (!FD->doesThisDeclarationHaveABody())
       return FD->doesDeclarationForceExternallyVisibleDefinition();
 
-    // Function definitions with the sycl_kernel_entry_point attribute are
-    // required during device compilation so that SYCL kernel caller offload
-    // entry points are emitted.
-    if (LangOpts.SYCLIsDevice && FD->hasAttr<SYCLKernelEntryPointAttr>())
-      return true;
-
-    // FIXME: Functions declared with SYCL_EXTERNAL are required during
-    // device compilation.
-
     // Constructors and destructors are required.
     if (FD->hasAttr<ConstructorAttr>() || FD->hasAttr<DestructorAttr>())
       return true;
@@ -13573,7 +13719,7 @@ static T *getCommonDeclChecked(T *X, T *Y) {
                                const_cast<Decl *>(cast<Decl>(Y))));
 }
 
-static TemplateName getCommonTemplateName(ASTContext &Ctx, TemplateName X,
+static TemplateName getCommonTemplateName(const ASTContext &Ctx, TemplateName X,
                                           TemplateName Y,
                                           bool IgnoreDeduced = false) {
   if (X.getAsVoidPointer() == Y.getAsVoidPointer())
@@ -13588,7 +13734,7 @@ static TemplateName getCommonTemplateName(ASTContext &Ctx, TemplateName X,
   return CX;
 }
 
-static TemplateName getCommonTemplateNameChecked(ASTContext &Ctx,
+static TemplateName getCommonTemplateNameChecked(const ASTContext &Ctx,
                                                  TemplateName X, TemplateName Y,
                                                  bool IgnoreDeduced) {
   TemplateName R = getCommonTemplateName(Ctx, X, Y, IgnoreDeduced);
@@ -13596,7 +13742,7 @@ static TemplateName getCommonTemplateNameChecked(ASTContext &Ctx,
   return R;
 }
 
-static auto getCommonTypes(ASTContext &Ctx, ArrayRef<QualType> Xs,
+static auto getCommonTypes(const ASTContext &Ctx, ArrayRef<QualType> Xs,
                            ArrayRef<QualType> Ys, bool Unqualified = false) {
   assert(Xs.size() == Ys.size());
   SmallVector<QualType, 8> Rs(Xs.size());
@@ -13611,7 +13757,7 @@ static SourceLocation getCommonAttrLoc(const T *X, const T *Y) {
                                                       : SourceLocation();
 }
 
-static TemplateArgument getCommonTemplateArgument(ASTContext &Ctx,
+static TemplateArgument getCommonTemplateArgument(const ASTContext &Ctx,
                                                   const TemplateArgument &X,
                                                   const TemplateArgument &Y) {
   if (X.getKind() != Y.getKind())
@@ -13657,7 +13803,7 @@ static TemplateArgument getCommonTemplateArgument(ASTContext &Ctx,
   }
 }
 
-static bool getCommonTemplateArguments(ASTContext &Ctx,
+static bool getCommonTemplateArguments(const ASTContext &Ctx,
                                        SmallVectorImpl<TemplateArgument> &R,
                                        ArrayRef<TemplateArgument> Xs,
                                        ArrayRef<TemplateArgument> Ys) {
@@ -13672,7 +13818,7 @@ static bool getCommonTemplateArguments(ASTContext &Ctx,
   return false;
 }
 
-static auto getCommonTemplateArguments(ASTContext &Ctx,
+static auto getCommonTemplateArguments(const ASTContext &Ctx,
                                        ArrayRef<TemplateArgument> Xs,
                                        ArrayRef<TemplateArgument> Ys) {
   SmallVector<TemplateArgument, 8> R;
@@ -13683,163 +13829,101 @@ static auto getCommonTemplateArguments(ASTContext &Ctx,
 }
 
 template <class T>
-static ElaboratedTypeKeyword getCommonTypeKeyword(const T *X, const T *Y) {
-  return X->getKeyword() == Y->getKeyword() ? X->getKeyword()
-                                            : ElaboratedTypeKeyword::None;
+static ElaboratedTypeKeyword getCommonTypeKeyword(const T *X, const T *Y,
+                                                  bool IsSame) {
+  ElaboratedTypeKeyword KX = X->getKeyword(), KY = Y->getKeyword();
+  if (KX == KY)
+    return KX;
+  KX = getCanonicalElaboratedTypeKeyword(KX);
+  assert(!IsSame || KX == getCanonicalElaboratedTypeKeyword(KY));
+  return KX;
 }
 
 /// Returns a NestedNameSpecifier which has only the common sugar
 /// present in both NNS1 and NNS2.
-static NestedNameSpecifier *getCommonNNS(ASTContext &Ctx,
-                                         NestedNameSpecifier *NNS1,
-                                         NestedNameSpecifier *NNS2,
-                                         bool IsSame) {
+static NestedNameSpecifier getCommonNNS(const ASTContext &Ctx,
+                                        NestedNameSpecifier NNS1,
+                                        NestedNameSpecifier NNS2, bool IsSame) {
   // If they are identical, all sugar is common.
   if (NNS1 == NNS2)
     return NNS1;
 
-  // IsSame implies both NNSes are equivalent.
-  NestedNameSpecifier *Canon = Ctx.getCanonicalNestedNameSpecifier(NNS1);
-  if (Canon != Ctx.getCanonicalNestedNameSpecifier(NNS2)) {
+  // IsSame implies both Qualifiers are equivalent.
+  NestedNameSpecifier Canon = NNS1.getCanonical();
+  if (Canon != NNS2.getCanonical()) {
     assert(!IsSame && "Should be the same NestedNameSpecifier");
     // If they are not the same, there is nothing to unify.
-    // FIXME: It would be useful here if we could represent a canonically
-    // empty NNS, which is not identical to an empty-as-written NNS.
-    return nullptr;
+    return std::nullopt;
   }
 
-  NestedNameSpecifier *R = nullptr;
-  NestedNameSpecifier::SpecifierKind K1 = NNS1->getKind(), K2 = NNS2->getKind();
-  switch (K1) {
-  case NestedNameSpecifier::SpecifierKind::Identifier: {
-    assert(K2 == NestedNameSpecifier::SpecifierKind::Identifier);
-    IdentifierInfo *II = NNS1->getAsIdentifier();
-    assert(II == NNS2->getAsIdentifier());
-    // For an identifier, the prefixes are significant, so they must be the
-    // same.
-    NestedNameSpecifier *P = ::getCommonNNS(Ctx, NNS1->getPrefix(),
-                                            NNS2->getPrefix(), /*IsSame=*/true);
-    R = NestedNameSpecifier::Create(Ctx, P, II);
-    break;
-  }
-  case NestedNameSpecifier::SpecifierKind::Namespace: {
-    assert(K2 == NestedNameSpecifier::SpecifierKind::Namespace);
-    // The prefixes for namespaces are not significant, its declaration
-    // identifies it uniquely.
-    NestedNameSpecifier *P =
-        ::getCommonNNS(Ctx, NNS1->getPrefix(), NNS2->getPrefix(),
-                       /*IsSame=*/false);
-    NamespaceBaseDecl *Namespace1 = NNS1->getAsNamespace(),
-                      *Namespace2 = NNS2->getAsNamespace();
+  NestedNameSpecifier R = std::nullopt;
+  NestedNameSpecifier::Kind Kind = NNS1.getKind();
+  assert(Kind == NNS2.getKind());
+  switch (Kind) {
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [Namespace1, Prefix1] = NNS1.getAsNamespaceAndPrefix();
+    auto [Namespace2, Prefix2] = NNS2.getAsNamespaceAndPrefix();
     auto Kind = Namespace1->getKind();
     if (Kind != Namespace2->getKind() ||
         (Kind == Decl::NamespaceAlias &&
          !declaresSameEntity(Namespace1, Namespace2))) {
-      R = NestedNameSpecifier::Create(
-          Ctx, P,
+      R = NestedNameSpecifier(
+          Ctx,
           ::getCommonDeclChecked(Namespace1->getNamespace(),
-                                 Namespace2->getNamespace()));
+                                 Namespace2->getNamespace()),
+          /*Prefix=*/std::nullopt);
       break;
     }
-    R = NestedNameSpecifier::Create(
-        Ctx, P, ::getCommonDeclChecked(Namespace1, Namespace2));
+    // The prefixes for namespaces are not significant, its declaration
+    // identifies it uniquely.
+    NestedNameSpecifier Prefix = ::getCommonNNS(Ctx, Prefix1, Prefix2,
+                                                /*IsSame=*/false);
+    R = NestedNameSpecifier(Ctx, ::getCommonDeclChecked(Namespace1, Namespace2),
+                            Prefix);
     break;
   }
-  case NestedNameSpecifier::SpecifierKind::TypeSpec: {
-    // FIXME: See comment below, on Super case.
-    if (K2 == NestedNameSpecifier::SpecifierKind::Super)
-      return Ctx.getCanonicalNestedNameSpecifier(NNS1);
-
-    assert(K2 == NestedNameSpecifier::SpecifierKind::TypeSpec);
-
-    const Type *T1 = NNS1->getAsType(), *T2 = NNS2->getAsType();
-    if (T1 == T2) {
-      // If the types are indentical, then only the prefixes differ.
-      // A well-formed NNS never has these types, as they have
-      // special normalized forms.
-      assert((!isa<DependentNameType, ElaboratedType>(T1)));
-      // Only for a DependentTemplateSpecializationType the prefix
-      // is actually significant. A DependentName, which would be another
-      // plausible case, cannot occur here, as explained above.
-      bool IsSame = isa<DependentTemplateSpecializationType>(T1);
-      NestedNameSpecifier *P =
-          ::getCommonNNS(Ctx, NNS1->getPrefix(), NNS2->getPrefix(), IsSame);
-      R = NestedNameSpecifier::Create(Ctx, P, T1);
-      break;
-    }
-    // TODO: Try to salvage the original prefix.
-    // If getCommonSugaredType removed any top level sugar, the original prefix
-    // is not applicable anymore.
+  case NestedNameSpecifier::Kind::Type: {
+    const Type *T1 = NNS1.getAsType(), *T2 = NNS2.getAsType();
     const Type *T = Ctx.getCommonSugaredType(QualType(T1, 0), QualType(T2, 0),
                                              /*Unqualified=*/true)
                         .getTypePtr();
-
-    // A NestedNameSpecifier has special normalization rules for certain types.
-    switch (T->getTypeClass()) {
-    case Type::Elaborated: {
-      // An ElaboratedType is stripped off, it's Qualifier becomes the prefix.
-      auto *ET = cast<ElaboratedType>(T);
-      R = NestedNameSpecifier::Create(Ctx, ET->getQualifier(),
-                                      ET->getNamedType().getTypePtr());
-      break;
-    }
-    case Type::DependentName: {
-      // A DependentName is turned into an Identifier NNS.
-      auto *DN = cast<DependentNameType>(T);
-      R = NestedNameSpecifier::Create(Ctx, DN->getQualifier(),
-                                      DN->getIdentifier());
-      break;
-    }
-    case Type::DependentTemplateSpecialization: {
-      // A DependentTemplateSpecializationType loses it's Qualifier, which
-      // is turned into the prefix.
-      auto *DTST = cast<DependentTemplateSpecializationType>(T);
-      const DependentTemplateStorage &DTN = DTST->getDependentTemplateName();
-      DependentTemplateStorage NewDTN(/*Qualifier=*/nullptr, DTN.getName(),
-                                      DTN.hasTemplateKeyword());
-      T = Ctx.getDependentTemplateSpecializationType(DTST->getKeyword(), NewDTN,
-                                                     DTST->template_arguments())
-              .getTypePtr();
-      R = NestedNameSpecifier::Create(Ctx, DTN.getQualifier(), T);
-      break;
-    }
-    default:
-      R = NestedNameSpecifier::Create(Ctx, /*Prefix=*/nullptr, T);
-      break;
-    }
+    R = NestedNameSpecifier(T);
     break;
   }
-  case NestedNameSpecifier::SpecifierKind::Super:
+  case NestedNameSpecifier::Kind::MicrosoftSuper: {
     // FIXME: Can __super even be used with data members?
     // If it's only usable in functions, we will never see it here,
     // unless we save the qualifiers used in function types.
     // In that case, it might be possible NNS2 is a type,
     // in which case we should degrade the result to
     // a CXXRecordType.
-    return Ctx.getCanonicalNestedNameSpecifier(NNS1);
-  case NestedNameSpecifier::SpecifierKind::Global:
-    // The global NNS is a singleton.
-    assert(K2 == NestedNameSpecifier::SpecifierKind::Global &&
-           "Global NNS cannot be equivalent to any other kind");
-    llvm_unreachable("Global NestedNameSpecifiers did not compare equal");
-  }
-  assert(Ctx.getCanonicalNestedNameSpecifier(R) == Canon);
+    R = NestedNameSpecifier(getCommonDeclChecked(NNS1.getAsMicrosoftSuper(),
+                                                 NNS2.getAsMicrosoftSuper()));
+    break;
+  }
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+    // These are singletons.
+    llvm_unreachable("singletons did not compare equal");
+  }
+  assert(R.getCanonical() == Canon);
   return R;
 }
 
 template <class T>
-static NestedNameSpecifier *getCommonQualifier(ASTContext &Ctx, const T *X,
-                                               const T *Y, bool IsSame) {
+static NestedNameSpecifier getCommonQualifier(const ASTContext &Ctx, const T *X,
+                                              const T *Y, bool IsSame) {
   return ::getCommonNNS(Ctx, X->getQualifier(), Y->getQualifier(), IsSame);
 }
 
 template <class T>
-static QualType getCommonElementType(ASTContext &Ctx, const T *X, const T *Y) {
+static QualType getCommonElementType(const ASTContext &Ctx, const T *X,
+                                     const T *Y) {
   return Ctx.getCommonSugaredType(X->getElementType(), Y->getElementType());
 }
 
 template <class T>
-static QualType getCommonArrayElementType(ASTContext &Ctx, const T *X,
+static QualType getCommonArrayElementType(const ASTContext &Ctx, const T *X,
                                           Qualifiers &QX, const T *Y,
                                           Qualifiers &QY) {
   QualType EX = X->getElementType(), EY = Y->getElementType();
@@ -13856,11 +13940,13 @@ static QualType getCommonArrayElementType(ASTContext &Ctx, const T *X,
 }
 
 template <class T>
-static QualType getCommonPointeeType(ASTContext &Ctx, const T *X, const T *Y) {
+static QualType getCommonPointeeType(const ASTContext &Ctx, const T *X,
+                                     const T *Y) {
   return Ctx.getCommonSugaredType(X->getPointeeType(), Y->getPointeeType());
 }
 
-template <class T> static auto *getCommonSizeExpr(ASTContext &Ctx, T *X, T *Y) {
+template <class T>
+static auto *getCommonSizeExpr(const ASTContext &Ctx, T *X, T *Y) {
   assert(Ctx.hasSameExpr(X->getSizeExpr(), Y->getSizeExpr()));
   return X->getSizeExpr();
 }
@@ -13880,8 +13966,9 @@ static auto getCommonIndexTypeCVRQualifiers(const ArrayType *X,
 // each type (in a canonical sense) only once, in the order they appear
 // from X to Y. If they occur in both X and Y, the result will contain
 // the common sugared type between them.
-static void mergeTypeLists(ASTContext &Ctx, SmallVectorImpl<QualType> &Out,
-                           ArrayRef<QualType> X, ArrayRef<QualType> Y) {
+static void mergeTypeLists(const ASTContext &Ctx,
+                           SmallVectorImpl<QualType> &Out, ArrayRef<QualType> X,
+                           ArrayRef<QualType> Y) {
   llvm::DenseMap<QualType, unsigned> Found;
   for (auto Ts : {X, Y}) {
     for (QualType T : Ts) {
@@ -13900,7 +13987,7 @@ FunctionProtoType::ExceptionSpecInfo
 ASTContext::mergeExceptionSpecs(FunctionProtoType::ExceptionSpecInfo ESI1,
                                 FunctionProtoType::ExceptionSpecInfo ESI2,
                                 SmallVectorImpl<QualType> &ExceptionTypeStorage,
-                                bool AcceptDependent) {
+                                bool AcceptDependent) const {
   ExceptionSpecificationType EST1 = ESI1.Type, EST2 = ESI2.Type;
 
   // If either of them can throw anything, that is the result.
@@ -13964,7 +14051,7 @@ ASTContext::mergeExceptionSpecs(FunctionProtoType::ExceptionSpecInfo ESI1,
   llvm_unreachable("invalid ExceptionSpecificationType");
 }
 
-static QualType getCommonNonSugarTypeNode(ASTContext &Ctx, const Type *X,
+static QualType getCommonNonSugarTypeNode(const ASTContext &Ctx, const Type *X,
                                           Qualifiers &QX, const Type *Y,
                                           Qualifiers &QY) {
   Type::TypeClass TC = X->getTypeClass();
@@ -13982,11 +14069,10 @@ static QualType getCommonNonSugarTypeNode(ASTContext &Ctx, const Type *X,
     SUGAR_FREE_TYPE(Builtin)
     SUGAR_FREE_TYPE(DeducedTemplateSpecialization)
     SUGAR_FREE_TYPE(DependentBitInt)
-    SUGAR_FREE_TYPE(Enum)
     SUGAR_FREE_TYPE(BitInt)
     SUGAR_FREE_TYPE(ObjCInterface)
-    SUGAR_FREE_TYPE(Record)
     SUGAR_FREE_TYPE(SubstTemplateTypeParmPack)
+    SUGAR_FREE_TYPE(SubstBuiltinTemplatePack)
     SUGAR_FREE_TYPE(UnresolvedUsing)
     SUGAR_FREE_TYPE(HLSLAttributedResource)
     SUGAR_FREE_TYPE(HLSLInlineSpirv)
@@ -14203,13 +14289,15 @@ static QualType getCommonNonSugarTypeNode(ASTContext &Ctx, const Type *X,
         getCommonElementType(Ctx, VX, VY), getCommonSizeExpr(Ctx, VX, VY),
         getCommonAttrLoc(VX, VY), VX->getVectorKind());
   }
+  case Type::Enum:
+  case Type::Record:
   case Type::InjectedClassName: {
-    const auto *IX = cast<InjectedClassNameType>(X),
-               *IY = cast<InjectedClassNameType>(Y);
-    return Ctx.getInjectedClassNameType(
-        getCommonDeclChecked(IX->getDecl(), IY->getDecl()),
-        Ctx.getCommonSugaredType(IX->getInjectedSpecializationType(),
-                                 IY->getInjectedSpecializationType()));
+    const auto *TX = cast<TagType>(X), *TY = cast<TagType>(Y);
+    return Ctx.getTagType(
+        ::getCommonTypeKeyword(TX, TY, /*IsSame=*/false),
+        ::getCommonQualifier(Ctx, TX, TY, /*IsSame=*/false),
+        ::getCommonDeclChecked(TX->getOriginalDecl(), TY->getOriginalDecl()),
+        /*OwnedTag=*/false);
   }
   case Type::TemplateSpecialization: {
     const auto *TX = cast<TemplateSpecializationType>(X),
@@ -14217,6 +14305,7 @@ static QualType getCommonNonSugarTypeNode(ASTContext &Ctx, const Type *X,
     auto As = getCommonTemplateArguments(Ctx, TX->template_arguments(),
                                          TY->template_arguments());
     return Ctx.getTemplateSpecializationType(
+        getCommonTypeKeyword(TX, TY, /*IsSame=*/false),
         ::getCommonTemplateNameChecked(Ctx, TX->getTemplateName(),
                                        TY->getTemplateName(),
                                        /*IgnoreDeduced=*/true),
@@ -14244,7 +14333,7 @@ static QualType getCommonNonSugarTypeNode(ASTContext &Ctx, const Type *X,
                *NY = cast<DependentNameType>(Y);
     assert(NX->getIdentifier() == NY->getIdentifier());
     return Ctx.getDependentNameType(
-        getCommonTypeKeyword(NX, NY),
+        getCommonTypeKeyword(NX, NY, /*IsSame=*/true),
         getCommonQualifier(Ctx, NX, NY, /*IsSame=*/true), NX->getIdentifier());
   }
   case Type::DependentTemplateSpecialization: {
@@ -14260,7 +14349,7 @@ static QualType getCommonNonSugarTypeNode(ASTContext &Ctx, const Type *X,
                      /*IsSame=*/true),
         SX.getName(), SX.hasTemplateKeyword() || SY.hasTemplateKeyword());
     return Ctx.getDependentTemplateSpecializationType(
-        getCommonTypeKeyword(TX, TY), Name, As);
+        getCommonTypeKeyword(TX, TY, /*IsSame=*/true), Name, As);
   }
   case Type::UnaryTransform: {
     const auto *TX = cast<UnaryTransformType>(X),
@@ -14301,7 +14390,7 @@ static QualType getCommonNonSugarTypeNode(ASTContext &Ctx, const Type *X,
   llvm_unreachable("Unknown Type Class");
 }
 
-static QualType getCommonSugarTypeNode(ASTContext &Ctx, const Type *X,
+static QualType getCommonSugarTypeNode(const ASTContext &Ctx, const Type *X,
                                        const Type *Y,
                                        SplitQualType Underlying) {
   Type::TypeClass TC = X->getTypeClass();
@@ -14413,15 +14502,6 @@ static QualType getCommonSugarTypeNode(ASTContext &Ctx, const Type *X,
   case Type::DeducedTemplateSpecialization:
     // FIXME: Try to merge these.
     return QualType();
-
-  case Type::Elaborated: {
-    const auto *EX = cast<ElaboratedType>(X), *EY = cast<ElaboratedType>(Y);
-    return Ctx.getElaboratedType(
-        ::getCommonTypeKeyword(EX, EY),
-        ::getCommonQualifier(Ctx, EX, EY, /*IsSame=*/false),
-        Ctx.getQualifiedType(Underlying),
-        ::getCommonDecl(EX->getOwnedTagDecl(), EY->getOwnedTagDecl()));
-  }
   case Type::MacroQualified: {
     const auto *MX = cast<MacroQualifiedType>(X),
                *MY = cast<MacroQualifiedType>(Y);
@@ -14465,16 +14545,19 @@ static QualType getCommonSugarTypeNode(ASTContext &Ctx, const Type *X,
     if (getCommonTemplateArguments(Ctx, As, TX->template_arguments(),
                                    TY->template_arguments()))
       return QualType();
-    return Ctx.getTemplateSpecializationType(CTN, As,
-                                             /*CanonicalArgs=*/{},
-                                             Ctx.getQualifiedType(Underlying));
+    return Ctx.getTemplateSpecializationType(
+        getCommonTypeKeyword(TX, TY, /*IsSame=*/false), CTN, As,
+        /*CanonicalArgs=*/{}, Ctx.getQualifiedType(Underlying));
   }
   case Type::Typedef: {
     const auto *TX = cast<TypedefType>(X), *TY = cast<TypedefType>(Y);
     const TypedefNameDecl *CD = ::getCommonDecl(TX->getDecl(), TY->getDecl());
     if (!CD)
       return QualType();
-    return Ctx.getTypedefType(CD, Ctx.getQualifiedType(Underlying));
+    return Ctx.getTypedefType(
+        ::getCommonTypeKeyword(TX, TY, /*IsSame=*/false),
+        ::getCommonQualifier(Ctx, TX, TY, /*IsSame=*/false), CD,
+        Ctx.getQualifiedType(Underlying));
   }
   case Type::TypeOf: {
     // The common sugar between two typeof expressions, where one is
@@ -14505,11 +14588,12 @@ static QualType getCommonSugarTypeNode(ASTContext &Ctx, const Type *X,
   }
   case Type::Using: {
     const auto *UX = cast<UsingType>(X), *UY = cast<UsingType>(Y);
-    const UsingShadowDecl *CD =
-        ::getCommonDecl(UX->getFoundDecl(), UY->getFoundDecl());
+    const UsingShadowDecl *CD = ::getCommonDecl(UX->getDecl(), UY->getDecl());
     if (!CD)
       return QualType();
-    return Ctx.getUsingType(CD, Ctx.getQualifiedType(Underlying));
+    return Ctx.getUsingType(::getCommonTypeKeyword(UX, UY, /*IsSame=*/false),
+                            ::getCommonQualifier(Ctx, UX, UY, /*IsSame=*/false),
+                            CD, Ctx.getQualifiedType(Underlying));
   }
   case Type::MemberPointer: {
     const auto *PX = cast<MemberPointerType>(X),
@@ -14568,7 +14652,7 @@ static auto unwrapSugar(SplitQualType &T, Qualifiers &QTotal) {
 }
 
 QualType ASTContext::getCommonSugaredType(QualType X, QualType Y,
-                                          bool Unqualified) {
+                                          bool Unqualified) const {
   assert(Unqualified ? hasSameUnqualifiedType(X, Y) : hasSameType(X, Y));
   if (X == Y)
     return X;
diff --git a/clang/lib/AST/ASTDiagnostic.cpp b/clang/lib/AST/ASTDiagnostic.cpp
index 2ef0c31ec1bd9..d7fd411ab464c 100644
--- a/clang/lib/AST/ASTDiagnostic.cpp
+++ b/clang/lib/AST/ASTDiagnostic.cpp
@@ -36,11 +36,6 @@ QualType clang::desugarForDiagnostic(ASTContext &Context, QualType QT,
   while (true) {
     const Type *Ty = QC.strip(QT);
 
-    // Don't aka just because we saw an elaborated type...
-    if (const ElaboratedType *ET = dyn_cast<ElaboratedType>(Ty)) {
-      QT = ET->desugar();
-      continue;
-    }
     // ... or a using type ...
     if (const UsingType *UT = dyn_cast<UsingType>(Ty)) {
       QT = UT->desugar();
@@ -130,7 +125,8 @@ QualType clang::desugarForDiagnostic(ASTContext &Context, QualType QT,
         if (DesugarArgument) {
           ShouldAKA = true;
           QT = Context.getTemplateSpecializationType(
-              TST->getTemplateName(), Args, /*CanonicalArgs=*/{}, QT);
+              TST->getKeyword(), TST->getTemplateName(), Args,
+              /*CanonicalArgs=*/{}, QT);
         }
         break;
       }
@@ -200,7 +196,8 @@ break; \
     // Don't desugar through the primary typedef of an anonymous type.
     if (const TagType *UTT = Underlying->getAs<TagType>())
       if (const TypedefType *QTT = dyn_cast<TypedefType>(QT))
-        if (UTT->getDecl()->getTypedefNameForAnonDecl() == QTT->getDecl())
+        if (UTT->getOriginalDecl()->getTypedefNameForAnonDecl() ==
+            QTT->getDecl())
           break;
 
     // Record that we actually looked through an opaque type here.
@@ -461,13 +458,12 @@ void clang::FormatASTNodeDiagnosticArgument(
       ND->getNameForDiagnostic(OS, Context.getPrintingPolicy(), Qualified);
       break;
     }
-    case DiagnosticsEngine::ak_nestednamespec: {
-      NestedNameSpecifier *NNS = reinterpret_cast<NestedNameSpecifier*>(Val);
-      NNS->print(OS, Context.getPrintingPolicy(),
+    case DiagnosticsEngine::ak_nestednamespec:
+      NestedNameSpecifier::getFromVoidPointer(reinterpret_cast<void *>(Val))
+          .print(OS, Context.getPrintingPolicy(),
                  /*ResolveTemplateArguments=*/false,
                  /*PrintFinalScopeResOp=*/false);
       break;
-    }
     case DiagnosticsEngine::ak_declcontext: {
       DeclContext *DC = reinterpret_cast<DeclContext *> (Val);
       assert(DC && "Should never have a null declaration context");
@@ -484,9 +480,8 @@ void clang::FormatASTNodeDiagnosticArgument(
       } else if (isLambdaCallOperator(DC)) {
         OS << "lambda expression";
       } else if (TypeDecl *Type = dyn_cast<TypeDecl>(DC)) {
-        OS << ConvertTypeToDiagnosticString(Context,
-                                            Context.getTypeDeclType(Type),
-                                            PrevArgs, QualTypeVals);
+        OS << ConvertTypeToDiagnosticString(
+            Context, Context.getTypeDeclType(Type), PrevArgs, QualTypeVals);
       } else {
         assert(isa<NamedDecl>(DC) && "Expected a NamedDecl");
         NamedDecl *ND = cast<NamedDecl>(DC);
@@ -1158,12 +1153,13 @@ class TemplateDiff {
       return nullptr;
 
     const ClassTemplateSpecializationDecl *CTSD =
-        dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl());
+        dyn_cast<ClassTemplateSpecializationDecl>(RT->getOriginalDecl());
 
     if (!CTSD)
       return nullptr;
 
     Ty = Context.getTemplateSpecializationType(
+        ElaboratedTypeKeyword::None,
         TemplateName(CTSD->getSpecializedTemplate()),
         CTSD->getTemplateArgs().asArray(), /*CanonicalArgs=*/{},
         Ty.getLocalUnqualifiedType().getCanonicalType());
@@ -1743,25 +1739,10 @@ class TemplateDiff {
 
     std::string FromTypeStr = FromType.isNull() ? "(no argument)"
                                                 : FromType.getAsString(Policy);
-    std::string ToTypeStr = ToType.isNull() ? "(no argument)"
-                                            : ToType.getAsString(Policy);
-    // Print without ElaboratedType sugar if it is better.
+    std::string ToTypeStr =
+        ToType.isNull() ? "(no argument)" : ToType.getAsString(Policy);
     // TODO: merge this with other aka printing above.
     if (FromTypeStr == ToTypeStr) {
-      const auto *FromElTy = dyn_cast<ElaboratedType>(FromType),
-                 *ToElTy = dyn_cast<ElaboratedType>(ToType);
-      if (FromElTy || ToElTy) {
-        std::string FromNamedTypeStr =
-            FromElTy ? FromElTy->getNamedType().getAsString(Policy)
-                     : FromTypeStr;
-        std::string ToNamedTypeStr =
-            ToElTy ? ToElTy->getNamedType().getAsString(Policy) : ToTypeStr;
-        if (FromNamedTypeStr != ToNamedTypeStr) {
-          FromTypeStr = FromNamedTypeStr;
-          ToTypeStr = ToNamedTypeStr;
-          goto PrintTypes;
-        }
-      }
       // Switch to canonical typename if it is better.
       std::string FromCanTypeStr =
           FromType.getCanonicalType().getAsString(Policy);
@@ -1772,8 +1753,8 @@ class TemplateDiff {
       }
     }
 
-  PrintTypes:
-    if (PrintTree) OS << '[';
+    if (PrintTree)
+      OS << '[';
     OS << (FromDefault ? "(default) " : "");
     Bold();
     OS << FromTypeStr;
diff --git a/clang/lib/AST/ASTDumper.cpp b/clang/lib/AST/ASTDumper.cpp
index 5e4487e0687d3..8e651a0a68629 100644
--- a/clang/lib/AST/ASTDumper.cpp
+++ b/clang/lib/AST/ASTDumper.cpp
@@ -117,7 +117,9 @@ void ASTDumper::dumpTemplateDeclSpecialization(const SpecializationDecl *D,
     // FIXME: The redecls() range sometimes has elements of a less-specific
     // type. (In particular, ClassTemplateSpecializationDecl::redecls() gives
     // us TagDecls, and should give CXXRecordDecls).
-    auto *Redecl = cast<SpecializationDecl>(RedeclWithBadType);
+    auto *Redecl = dyn_cast<SpecializationDecl>(RedeclWithBadType);
+    if (!Redecl)
+      continue;
     switch (Redecl->getTemplateSpecializationKind()) {
     case TSK_ExplicitInstantiationDeclaration:
     case TSK_ExplicitInstantiationDefinition:
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 8e2927bdc8d6f..6299efaf6bbfc 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -949,8 +949,10 @@ ASTNodeImporter::import(const TemplateArgumentLoc &TALoc) {
     else
       return TSIOrErr.takeError();
   } else {
-    auto ToTemplateQualifierLocOrErr =
-        import(FromInfo.getTemplateQualifierLoc());
+    auto ToTemplateKWLocOrErr = import(FromInfo.getTemplateKwLoc());
+    if (!ToTemplateKWLocOrErr)
+      return ToTemplateKWLocOrErr.takeError();
+    auto ToTemplateQualifierLocOrErr = import(TALoc.getTemplateQualifierLoc());
     if (!ToTemplateQualifierLocOrErr)
       return ToTemplateQualifierLocOrErr.takeError();
     auto ToTemplateNameLocOrErr = import(FromInfo.getTemplateNameLoc());
@@ -961,8 +963,9 @@ ASTNodeImporter::import(const TemplateArgumentLoc &TALoc) {
     if (!ToTemplateEllipsisLocOrErr)
       return ToTemplateEllipsisLocOrErr.takeError();
     ToInfo = TemplateArgumentLocInfo(
-        Importer.getToContext(), *ToTemplateQualifierLocOrErr,
-        *ToTemplateNameLocOrErr, *ToTemplateEllipsisLocOrErr);
+        Importer.getToContext(), *ToTemplateKWLocOrErr,
+        *ToTemplateQualifierLocOrErr, *ToTemplateNameLocOrErr,
+        *ToTemplateEllipsisLocOrErr);
   }
 
   return TemplateArgumentLoc(Arg, ToInfo);
@@ -1597,13 +1600,15 @@ ASTNodeImporter::VisitFunctionProtoType(const FunctionProtoType *T) {
 ExpectedType ASTNodeImporter::VisitUnresolvedUsingType(
     const UnresolvedUsingType *T) {
   Error Err = Error::success();
-  auto ToD = importChecked(Err, T->getDecl());
-  auto ToPrevD = importChecked(Err, T->getDecl()->getPreviousDecl());
+  auto ToQualifier = importChecked(Err, T->getQualifier());
+  auto *ToD = importChecked(Err, T->getDecl());
   if (Err)
     return std::move(Err);
 
-  return Importer.getToContext().getTypeDeclType(
-      ToD, cast_or_null<TypeDecl>(ToPrevD));
+  if (T->isCanonicalUnqualified())
+    return Importer.getToContext().getCanonicalUnresolvedUsingType(ToD);
+  return Importer.getToContext().getUnresolvedUsingType(T->getKeyword(),
+                                                        ToQualifier, ToD);
 }
 
 ExpectedType ASTNodeImporter::VisitParenType(const ParenType *T) {
@@ -1631,15 +1636,17 @@ ExpectedType ASTNodeImporter::VisitTypedefType(const TypedefType *T) {
   if (!ToDeclOrErr)
     return ToDeclOrErr.takeError();
 
-  TypedefNameDecl *ToDecl = *ToDeclOrErr;
-  if (ToDecl->getTypeForDecl())
-    return QualType(ToDecl->getTypeForDecl(), 0);
+  auto ToQualifierOrErr = import(T->getQualifier());
+  if (!ToQualifierOrErr)
+    return ToQualifierOrErr.takeError();
 
-  ExpectedType ToUnderlyingTypeOrErr = import(T->desugar());
+  ExpectedType ToUnderlyingTypeOrErr =
+      T->typeMatchesDecl() ? QualType() : import(T->desugar());
   if (!ToUnderlyingTypeOrErr)
     return ToUnderlyingTypeOrErr.takeError();
 
-  return Importer.getToContext().getTypedefType(ToDecl, *ToUnderlyingTypeOrErr);
+  return Importer.getToContext().getTypedefType(
+      T->getKeyword(), *ToQualifierOrErr, *ToDeclOrErr, *ToUnderlyingTypeOrErr);
 }
 
 ExpectedType ASTNodeImporter::VisitTypeOfExprType(const TypeOfExprType *T) {
@@ -1658,14 +1665,14 @@ ExpectedType ASTNodeImporter::VisitTypeOfType(const TypeOfType *T) {
 }
 
 ExpectedType ASTNodeImporter::VisitUsingType(const UsingType *T) {
-  Expected<UsingShadowDecl *> FoundOrErr = import(T->getFoundDecl());
-  if (!FoundOrErr)
-    return FoundOrErr.takeError();
-  Expected<QualType> UnderlyingOrErr = import(T->getUnderlyingType());
-  if (!UnderlyingOrErr)
-    return UnderlyingOrErr.takeError();
-
-  return Importer.getToContext().getUsingType(*FoundOrErr, *UnderlyingOrErr);
+  Error Err = Error::success();
+  auto ToQualifier = importChecked(Err, T->getQualifier());
+  auto *ToD = importChecked(Err, T->getDecl());
+  QualType ToT = importChecked(Err, T->desugar());
+  if (Err)
+    return std::move(Err);
+  return Importer.getToContext().getUsingType(T->getKeyword(), ToQualifier, ToD,
+                                              ToT);
 }
 
 ExpectedType ASTNodeImporter::VisitDecltypeType(const DecltypeType *T) {
@@ -1728,36 +1735,37 @@ ExpectedType ASTNodeImporter::VisitDeducedTemplateSpecializationType(
     return ToDeducedTypeOrErr.takeError();
 
   return Importer.getToContext().getDeducedTemplateSpecializationType(
-      *ToTemplateNameOrErr, *ToDeducedTypeOrErr, T->isDependentType());
+      T->getKeyword(), *ToTemplateNameOrErr, *ToDeducedTypeOrErr,
+      T->isDependentType());
 }
 
-ExpectedType ASTNodeImporter::VisitInjectedClassNameType(
-    const InjectedClassNameType *T) {
-  Expected<CXXRecordDecl *> ToDeclOrErr = import(T->getDecl());
+ExpectedType ASTNodeImporter::VisitTagType(const TagType *T) {
+  Expected<TagDecl *> ToDeclOrErr = import(T->getOriginalDecl());
   if (!ToDeclOrErr)
     return ToDeclOrErr.takeError();
 
-  // The InjectedClassNameType is created in VisitRecordDecl when the
-  // T->getDecl() is imported. Here we can return the existing type.
-  const Type *Ty = (*ToDeclOrErr)->getTypeForDecl();
-  assert(isa_and_nonnull<InjectedClassNameType>(Ty));
-  return QualType(Ty, 0);
-}
+  if (T->isCanonicalUnqualified())
+    return Importer.getToContext().getCanonicalTagType(*ToDeclOrErr);
 
-ExpectedType ASTNodeImporter::VisitRecordType(const RecordType *T) {
-  Expected<RecordDecl *> ToDeclOrErr = import(T->getDecl());
-  if (!ToDeclOrErr)
-    return ToDeclOrErr.takeError();
+  auto ToQualifierOrErr = import(T->getQualifier());
+  if (!ToQualifierOrErr)
+    return ToQualifierOrErr.takeError();
 
-  return Importer.getToContext().getTagDeclType(*ToDeclOrErr);
+  return Importer.getToContext().getTagType(T->getKeyword(), *ToQualifierOrErr,
+                                            *ToDeclOrErr, T->isTagOwned());
 }
 
 ExpectedType ASTNodeImporter::VisitEnumType(const EnumType *T) {
-  Expected<EnumDecl *> ToDeclOrErr = import(T->getDecl());
-  if (!ToDeclOrErr)
-    return ToDeclOrErr.takeError();
+  return VisitTagType(T);
+}
 
-  return Importer.getToContext().getTagDeclType(*ToDeclOrErr);
+ExpectedType ASTNodeImporter::VisitRecordType(const RecordType *T) {
+  return VisitTagType(T);
+}
+
+ExpectedType
+ASTNodeImporter::VisitInjectedClassNameType(const InjectedClassNameType *T) {
+  return VisitTagType(T);
 }
 
 ExpectedType ASTNodeImporter::VisitAttributedType(const AttributedType *T) {
@@ -1834,6 +1842,14 @@ ExpectedType ASTNodeImporter::VisitSubstTemplateTypeParmPackType(
       *ReplacedOrErr, T->getIndex(), T->getFinal(), *ToArgumentPack);
 }
 
+ExpectedType ASTNodeImporter::VisitSubstBuiltinTemplatePackType(
+    const SubstBuiltinTemplatePackType *T) {
+  Expected<TemplateArgument> ToArgumentPack = import(T->getArgumentPack());
+  if (!ToArgumentPack)
+    return ToArgumentPack.takeError();
+  return Importer.getToContext().getSubstBuiltinTemplatePack(*ToArgumentPack);
+}
+
 ExpectedType ASTNodeImporter::VisitTemplateSpecializationType(
                                        const TemplateSpecializationType *T) {
   auto ToTemplateOrErr = import(T->getTemplateName());
@@ -1850,27 +1866,8 @@ ExpectedType ASTNodeImporter::VisitTemplateSpecializationType(
   if (!ToUnderlyingOrErr)
     return ToUnderlyingOrErr.takeError();
   return Importer.getToContext().getTemplateSpecializationType(
-      *ToTemplateOrErr, ToTemplateArgs, {}, *ToUnderlyingOrErr);
-}
-
-ExpectedType ASTNodeImporter::VisitElaboratedType(const ElaboratedType *T) {
-  // Note: the qualifier in an ElaboratedType is optional.
-  auto ToQualifierOrErr = import(T->getQualifier());
-  if (!ToQualifierOrErr)
-    return ToQualifierOrErr.takeError();
-
-  ExpectedType ToNamedTypeOrErr = import(T->getNamedType());
-  if (!ToNamedTypeOrErr)
-    return ToNamedTypeOrErr.takeError();
-
-  Expected<TagDecl *> ToOwnedTagDeclOrErr = import(T->getOwnedTagDecl());
-  if (!ToOwnedTagDeclOrErr)
-    return ToOwnedTagDeclOrErr.takeError();
-
-  return Importer.getToContext().getElaboratedType(T->getKeyword(),
-                                                   *ToQualifierOrErr,
-                                                   *ToNamedTypeOrErr,
-                                                   *ToOwnedTagDeclOrErr);
+      T->getKeyword(), *ToTemplateOrErr, ToTemplateArgs, {},
+      *ToUnderlyingOrErr);
 }
 
 ExpectedType
@@ -2168,7 +2165,7 @@ Error ASTNodeImporter::ImportDeclParts(
       const Type *LeafT =
           getLeafPointeeType(P->getType().getCanonicalType().getTypePtr());
       auto *RT = dyn_cast<RecordType>(LeafT);
-      if (RT && RT->getDecl() == D) {
+      if (RT && RT->getOriginalDecl() == D) {
         Importer.FromDiag(D->getLocation(), diag::err_unsupported_ast_node)
             << D->getDeclKindName();
         return make_error<ASTImportError>(ASTImportError::UnsupportedConstruct);
@@ -2421,8 +2418,8 @@ Error ASTNodeImporter::ImportFieldDeclDefinition(const FieldDecl *From,
     const RecordType *RecordTo = ToType->getAs<RecordType>();
 
     if (RecordFrom && RecordTo) {
-      FromRecordDecl = RecordFrom->getDecl();
-      ToRecordDecl = RecordTo->getDecl();
+      FromRecordDecl = RecordFrom->getOriginalDecl();
+      ToRecordDecl = RecordTo->getOriginalDecl();
     }
   }
 
@@ -2643,7 +2640,7 @@ Error ASTNodeImporter::ImportDefinition(
     return Err;
 
   ExpectedType ToTypeOrErr =
-      import(Importer.getFromContext().getTypeDeclType(From));
+      import(QualType(Importer.getFromContext().getCanonicalTagType(From)));
   if (!ToTypeOrErr)
     return ToTypeOrErr.takeError();
 
@@ -3218,7 +3215,7 @@ ExpectedDecl ASTNodeImporter::VisitEnumDecl(EnumDecl *D) {
 
       if (auto *Typedef = dyn_cast<TypedefNameDecl>(FoundDecl)) {
         if (const auto *Tag = Typedef->getUnderlyingType()->getAs<TagType>())
-          FoundDecl = Tag->getDecl();
+          FoundDecl = Tag->getOriginalDecl();
       }
 
       if (auto *FoundEnum = dyn_cast<EnumDecl>(FoundDecl)) {
@@ -3349,7 +3346,7 @@ ExpectedDecl ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
       Decl *Found = FoundDecl;
       if (auto *Typedef = dyn_cast<TypedefNameDecl>(Found)) {
         if (const auto *Tag = Typedef->getUnderlyingType()->getAs<TagType>())
-          Found = Tag->getDecl();
+          Found = Tag->getOriginalDecl();
       }
 
       if (auto *FoundRecord = dyn_cast<RecordDecl>(Found)) {
@@ -3428,17 +3425,6 @@ ExpectedDecl ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
         return CDeclOrErr.takeError();
       Numbering.ContextDecl = *CDeclOrErr;
       D2CXX->setLambdaNumbering(Numbering);
-   } else if (DCXX->isInjectedClassName()) {
-      // We have to be careful to do a similar dance to the one in
-      // Sema::ActOnStartCXXMemberDeclarations
-      const bool DelayTypeCreation = true;
-      if (GetImportedOrCreateDecl(
-              D2CXX, D, Importer.getToContext(), D->getTagKind(), DC,
-              *BeginLocOrErr, Loc, Name.getAsIdentifierInfo(),
-              cast_or_null<CXXRecordDecl>(PrevDecl), DelayTypeCreation))
-        return D2CXX;
-      Importer.getToContext().getTypeDeclType(
-          D2CXX, dyn_cast<CXXRecordDecl>(DC));
     } else {
       if (GetImportedOrCreateDecl(D2CXX, D, Importer.getToContext(),
                                   D->getTagKind(), DC, *BeginLocOrErr, Loc,
@@ -3458,51 +3444,6 @@ ExpectedDecl ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
       if (Error Err = importInto(ToDescribed, FromDescribed))
         return std::move(Err);
       D2CXX->setDescribedClassTemplate(ToDescribed);
-      if (!DCXX->isInjectedClassName() && !IsFriendTemplate) {
-        // In a record describing a template the type should be an
-        // InjectedClassNameType (see Sema::CheckClassTemplate). Update the
-        // previously set type to the correct value here (ToDescribed is not
-        // available at record create).
-        CXXRecordDecl *Injected = nullptr;
-        for (NamedDecl *Found : D2CXX->noload_lookup(Name)) {
-          auto *Record = dyn_cast<CXXRecordDecl>(Found);
-          if (Record && Record->isInjectedClassName()) {
-            Injected = Record;
-            break;
-          }
-        }
-        // Create an injected type for the whole redecl chain.
-        // The chain may contain an already existing injected type at the start,
-        // if yes this should be reused. We must ensure that only one type
-        // object exists for the injected type (including the injected record
-        // declaration), ASTContext does not check it.
-        SmallVector<Decl *, 2> Redecls =
-            getCanonicalForwardRedeclChain(D2CXX);
-        const Type *FrontTy =
-            cast<CXXRecordDecl>(Redecls.front())->getTypeForDecl();
-        QualType InjSpec;
-        if (auto *InjTy = FrontTy->getAs<InjectedClassNameType>())
-          InjSpec = InjTy->getInjectedSpecializationType();
-        else
-          InjSpec = ToDescribed->getInjectedClassNameSpecialization();
-        for (auto *R : Redecls) {
-          auto *RI = cast<CXXRecordDecl>(R);
-          if (R != Redecls.front() ||
-              !isa<InjectedClassNameType>(RI->getTypeForDecl()))
-            RI->setTypeForDecl(nullptr);
-          // This function tries to get the injected type from getTypeForDecl,
-          // then from the previous declaration if possible. If not, it creates
-          // a new type.
-          Importer.getToContext().getInjectedClassNameType(RI, InjSpec);
-        }
-        // Set the new type for the injected decl too.
-        if (Injected) {
-          Injected->setTypeForDecl(nullptr);
-          // This function will copy the injected type from D2CXX into Injected.
-          // The injected decl does not have a previous decl to copy from.
-          Importer.getToContext().getTypeDeclType(Injected, D2CXX);
-        }
-      }
     } else if (MemberSpecializationInfo *MemberInfo =
                    DCXX->getMemberSpecializationInfo()) {
         TemplateSpecializationKind SK =
@@ -3826,11 +3767,12 @@ class IsTypeDeclaredInsideVisitor
   }
 
   std::optional<bool> VisitTagType(const TagType *T) {
-    if (auto *Spec = dyn_cast<ClassTemplateSpecializationDecl>(T->getDecl()))
+    if (auto *Spec =
+            dyn_cast<ClassTemplateSpecializationDecl>(T->getOriginalDecl()))
       for (const auto &Arg : Spec->getTemplateArgs().asArray())
         if (checkTemplateArgument(Arg))
           return true;
-    return isAncestorDeclContextOf(ParentDC, T->getDecl());
+    return isAncestorDeclContextOf(ParentDC, T->getOriginalDecl());
   }
 
   std::optional<bool> VisitPointerType(const PointerType *T) {
@@ -3842,17 +3784,11 @@ class IsTypeDeclaredInsideVisitor
   }
 
   std::optional<bool> VisitTypedefType(const TypedefType *T) {
-    const TypedefNameDecl *TD = T->getDecl();
-    assert(TD);
-    return isAncestorDeclContextOf(ParentDC, TD);
+    return isAncestorDeclContextOf(ParentDC, T->getDecl());
   }
 
   std::optional<bool> VisitUsingType(const UsingType *T) {
-    if (T->getFoundDecl() &&
-        isAncestorDeclContextOf(ParentDC, T->getFoundDecl()))
-      return true;
-
-    return {};
+    return isAncestorDeclContextOf(ParentDC, T->getDecl());
   }
 
   std::optional<bool>
@@ -6521,16 +6457,10 @@ ExpectedDecl ASTNodeImporter::VisitClassTemplateSpecializationDecl(
   // Create the specialization.
   ClassTemplateSpecializationDecl *D2 = nullptr;
   if (PartialSpec) {
-    QualType CanonInjType;
-    if (Error Err = importInto(
-        CanonInjType, PartialSpec->getInjectedSpecializationType()))
-      return std::move(Err);
-    CanonInjType = CanonInjType.getCanonicalType();
-
     if (GetImportedOrCreateDecl<ClassTemplatePartialSpecializationDecl>(
             D2, D, Importer.getToContext(), D->getTagKind(), DC, *BeginLocOrErr,
             *IdLocOrErr, ToTPList, ClassTemplate, ArrayRef(TemplateArgs),
-            CanonInjType,
+            /*CanonInjectedTST=*/CanQualType(),
             cast_or_null<ClassTemplatePartialSpecializationDecl>(PrevDecl)))
       return D2;
 
@@ -10050,46 +9980,34 @@ Expected<Stmt *> ASTImporter::Import(Stmt *FromS) {
   return ToSOrErr;
 }
 
-Expected<NestedNameSpecifier *>
-ASTImporter::Import(NestedNameSpecifier *FromNNS) {
-  if (!FromNNS)
-    return nullptr;
-
-  NestedNameSpecifier *Prefix = nullptr;
-  if (Error Err = importInto(Prefix, FromNNS->getPrefix()))
-    return std::move(Err);
-
-  switch (FromNNS->getKind()) {
-  case NestedNameSpecifier::Identifier:
-    assert(FromNNS->getAsIdentifier() && "NNS should contain identifier.");
-    return NestedNameSpecifier::Create(ToContext, Prefix,
-                                       Import(FromNNS->getAsIdentifier()));
-
-  case NestedNameSpecifier::Namespace:
-    if (ExpectedDecl NSOrErr = Import(FromNNS->getAsNamespace())) {
-      return NestedNameSpecifier::Create(ToContext, Prefix,
-                                         cast<NamespaceBaseDecl>(*NSOrErr));
-    } else
+Expected<NestedNameSpecifier> ASTImporter::Import(NestedNameSpecifier FromNNS) {
+  switch (FromNNS.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+    return FromNNS;
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [Namespace, Prefix] = FromNNS.getAsNamespaceAndPrefix();
+    auto NSOrErr = Import(Namespace);
+    if (!NSOrErr)
       return NSOrErr.takeError();
-
-  case NestedNameSpecifier::Global:
-    return NestedNameSpecifier::GlobalSpecifier(ToContext);
-
-  case NestedNameSpecifier::Super:
-    if (ExpectedDecl RDOrErr = Import(FromNNS->getAsRecordDecl()))
-      return NestedNameSpecifier::SuperSpecifier(ToContext,
-                                                 cast<CXXRecordDecl>(*RDOrErr));
+    auto PrefixOrErr = Import(Prefix);
+    if (!PrefixOrErr)
+      return PrefixOrErr.takeError();
+    return NestedNameSpecifier(ToContext, cast<NamespaceBaseDecl>(*NSOrErr),
+                               *PrefixOrErr);
+  }
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    if (ExpectedDecl RDOrErr = Import(FromNNS.getAsMicrosoftSuper()))
+      return NestedNameSpecifier(cast<CXXRecordDecl>(*RDOrErr));
     else
       return RDOrErr.takeError();
-
-  case NestedNameSpecifier::TypeSpec:
-    if (ExpectedTypePtr TyOrErr = Import(FromNNS->getAsType())) {
-      return NestedNameSpecifier::Create(ToContext, Prefix, *TyOrErr);
+  case NestedNameSpecifier::Kind::Type:
+    if (ExpectedTypePtr TyOrErr = Import(FromNNS.getAsType())) {
+      return NestedNameSpecifier(*TyOrErr);
     } else {
       return TyOrErr.takeError();
     }
   }
-
   llvm_unreachable("Invalid nested name specifier kind");
 }
 
@@ -10103,64 +10021,62 @@ ASTImporter::Import(NestedNameSpecifierLoc FromNNS) {
   // serialization in reverse order.
   while (NNS) {
     NestedNames.push_back(NNS);
-    NNS = NNS.getPrefix();
+    NNS = NNS.getAsNamespaceAndPrefix().Prefix;
   }
 
   NestedNameSpecifierLocBuilder Builder;
 
   while (!NestedNames.empty()) {
     NNS = NestedNames.pop_back_val();
-    NestedNameSpecifier *Spec = nullptr;
+    NestedNameSpecifier Spec = std::nullopt;
     if (Error Err = importInto(Spec, NNS.getNestedNameSpecifier()))
       return std::move(Err);
 
-    NestedNameSpecifier::SpecifierKind Kind = Spec->getKind();
+    NestedNameSpecifier::Kind Kind = Spec.getKind();
 
     SourceLocation ToLocalBeginLoc, ToLocalEndLoc;
-    if (Kind != NestedNameSpecifier::Super) {
+    if (Kind != NestedNameSpecifier::Kind::MicrosoftSuper) {
       if (Error Err = importInto(ToLocalBeginLoc, NNS.getLocalBeginLoc()))
         return std::move(Err);
 
-      if (Kind != NestedNameSpecifier::Global)
+      if (Kind != NestedNameSpecifier::Kind::Global)
         if (Error Err = importInto(ToLocalEndLoc, NNS.getLocalEndLoc()))
           return std::move(Err);
     }
 
     switch (Kind) {
-    case NestedNameSpecifier::Identifier:
-      Builder.Extend(getToContext(), Spec->getAsIdentifier(), ToLocalBeginLoc,
-                     ToLocalEndLoc);
-      break;
-
-    case NestedNameSpecifier::Namespace:
-      Builder.Extend(getToContext(), Spec->getAsNamespace(), ToLocalBeginLoc,
-                     ToLocalEndLoc);
+    case NestedNameSpecifier::Kind::Namespace:
+      Builder.Extend(getToContext(), Spec.getAsNamespaceAndPrefix().Namespace,
+                     ToLocalBeginLoc, ToLocalEndLoc);
       break;
 
-    case NestedNameSpecifier::TypeSpec: {
+    case NestedNameSpecifier::Kind::Type: {
       SourceLocation ToTLoc;
-      if (Error Err = importInto(ToTLoc, NNS.getTypeLoc().getBeginLoc()))
+      if (Error Err = importInto(ToTLoc, NNS.castAsTypeLoc().getBeginLoc()))
         return std::move(Err);
       TypeSourceInfo *TSI = getToContext().getTrivialTypeSourceInfo(
-          QualType(Spec->getAsType(), 0), ToTLoc);
-      Builder.Extend(getToContext(), TSI->getTypeLoc(), ToLocalEndLoc);
+          QualType(Spec.getAsType(), 0), ToTLoc);
+      Builder.Make(getToContext(), TSI->getTypeLoc(), ToLocalEndLoc);
       break;
     }
 
-    case NestedNameSpecifier::Global:
+    case NestedNameSpecifier::Kind::Global:
       Builder.MakeGlobal(getToContext(), ToLocalBeginLoc);
       break;
 
-    case NestedNameSpecifier::Super: {
+    case NestedNameSpecifier::Kind::MicrosoftSuper: {
       auto ToSourceRangeOrErr = Import(NNS.getSourceRange());
       if (!ToSourceRangeOrErr)
         return ToSourceRangeOrErr.takeError();
 
-      Builder.MakeSuper(getToContext(), Spec->getAsRecordDecl(),
-                        ToSourceRangeOrErr->getBegin(),
-                        ToSourceRangeOrErr->getEnd());
+      Builder.MakeMicrosoftSuper(getToContext(), Spec.getAsMicrosoftSuper(),
+                                 ToSourceRangeOrErr->getBegin(),
+                                 ToSourceRangeOrErr->getEnd());
+      break;
+    }
+    case NestedNameSpecifier::Kind::Null:
+      llvm_unreachable("unexpected null nested name specifier");
     }
-  }
   }
 
   return Builder.getWithLocInContext(getToContext());
@@ -10740,7 +10656,7 @@ ASTNodeImporter::ImportAPValue(const APValue &FromValue) {
           if (Err)
             return std::move(Err);
           if (auto *RD = dyn_cast<CXXRecordDecl>(FromDecl))
-            FromElemTy = Importer.FromContext.getRecordType(RD);
+            FromElemTy = Importer.FromContext.getCanonicalTagType(RD);
           else
             FromElemTy = cast<ValueDecl>(FromDecl)->getType();
           ToPath[LoopIdx] = APValue::LValuePathEntry(APValue::BaseOrMemberType(
diff --git a/clang/lib/AST/ASTImporterLookupTable.cpp b/clang/lib/AST/ASTImporterLookupTable.cpp
index 4ed3198d7ea62..29c3af1703b4f 100644
--- a/clang/lib/AST/ASTImporterLookupTable.cpp
+++ b/clang/lib/AST/ASTImporterLookupTable.cpp
@@ -49,8 +49,6 @@ struct Builder : RecursiveASTVisitor<Builder> {
   bool VisitFriendDecl(FriendDecl *D) {
     if (D->getFriendType()) {
       QualType Ty = D->getFriendType()->getType();
-      if (isa<ElaboratedType>(Ty))
-        Ty = cast<ElaboratedType>(Ty)->getNamedType();
       // A FriendDecl with a dependent type (e.g. ClassTemplateSpecialization)
       // always has that decl as child node.
       // However, there are non-dependent cases which does not have the
@@ -64,13 +62,15 @@ struct Builder : RecursiveASTVisitor<Builder> {
                      dyn_cast<SubstTemplateTypeParmType>(Ty)) {
           if (SubstTy->getAsCXXRecordDecl())
             LT.add(SubstTy->getAsCXXRecordDecl());
-        } else if (isa<TypedefType>(Ty)) {
-          // We do not put friend typedefs to the lookup table because
-          // ASTImporter does not organize typedefs into redecl chains.
-        } else if (isa<UsingType>(Ty)) {
-          // Similar to TypedefType, not putting into lookup table.
         } else {
-          llvm_unreachable("Unhandled type of friend class");
+          if (isa<TypedefType>(Ty)) {
+            // We do not put friend typedefs to the lookup table because
+            // ASTImporter does not organize typedefs into redecl chains.
+          } else if (isa<UsingType>(Ty)) {
+            // Similar to TypedefType, not putting into lookup table.
+          } else {
+            llvm_unreachable("Unhandled type of friend class");
+          }
         }
       }
     }
diff --git a/clang/lib/AST/ASTStructuralEquivalence.cpp b/clang/lib/AST/ASTStructuralEquivalence.cpp
index 0e2023f86c19d..6d83de384ee10 100644
--- a/clang/lib/AST/ASTStructuralEquivalence.cpp
+++ b/clang/lib/AST/ASTStructuralEquivalence.cpp
@@ -110,8 +110,8 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      const TemplateArgumentLoc &Arg1,
                                      const TemplateArgumentLoc &Arg2);
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
-                                     NestedNameSpecifier *NNS1,
-                                     NestedNameSpecifier *NNS2);
+                                     NestedNameSpecifier NNS1,
+                                     NestedNameSpecifier NNS2);
 static bool IsStructurallyEquivalent(const IdentifierInfo *Name1,
                                      const IdentifierInfo *Name2);
 
@@ -579,35 +579,30 @@ static bool IsStructurallyEquivalent(const IdentifierInfo *Name1,
 
 /// Determine whether two nested-name-specifiers are equivalent.
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
-                                     NestedNameSpecifier *NNS1,
-                                     NestedNameSpecifier *NNS2) {
-  if (NNS1->getKind() != NNS2->getKind())
-    return false;
-
-  NestedNameSpecifier *Prefix1 = NNS1->getPrefix(),
-                      *Prefix2 = NNS2->getPrefix();
-  if ((bool)Prefix1 != (bool)Prefix2)
-    return false;
-
-  if (Prefix1)
-    if (!IsStructurallyEquivalent(Context, Prefix1, Prefix2))
-      return false;
-
-  switch (NNS1->getKind()) {
-  case NestedNameSpecifier::Identifier:
-    return IsStructurallyEquivalent(NNS1->getAsIdentifier(),
-                                    NNS2->getAsIdentifier());
-  case NestedNameSpecifier::Namespace:
-    return IsStructurallyEquivalent(Context, NNS1->getAsNamespace(),
-                                    NNS2->getAsNamespace());
-  case NestedNameSpecifier::TypeSpec:
-    return IsStructurallyEquivalent(Context, QualType(NNS1->getAsType(), 0),
-                                    QualType(NNS2->getAsType(), 0));
-  case NestedNameSpecifier::Global:
+                                     NestedNameSpecifier NNS1,
+                                     NestedNameSpecifier NNS2) {
+  auto Kind = NNS1.getKind();
+  if (Kind != NNS2.getKind())
+    return false;
+  switch (Kind) {
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
     return true;
-  case NestedNameSpecifier::Super:
-    return IsStructurallyEquivalent(Context, NNS1->getAsRecordDecl(),
-                                    NNS2->getAsRecordDecl());
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [Namespace1, Prefix1] = NNS1.getAsNamespaceAndPrefix();
+    auto [Namespace2, Prefix2] = NNS2.getAsNamespaceAndPrefix();
+    if (!IsStructurallyEquivalent(Context,
+                                  const_cast<NamespaceBaseDecl *>(Namespace1),
+                                  const_cast<NamespaceBaseDecl *>(Namespace2)))
+      return false;
+    return IsStructurallyEquivalent(Context, Prefix1, Prefix2);
+  }
+  case NestedNameSpecifier::Kind::Type:
+    return IsStructurallyEquivalent(Context, QualType(NNS1.getAsType(), 0),
+                                    QualType(NNS2.getAsType(), 0));
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    return IsStructurallyEquivalent(Context, NNS1.getAsMicrosoftSuper(),
+                                    NNS2.getAsMicrosoftSuper());
   }
   return false;
 }
@@ -615,9 +610,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      const DependentTemplateStorage &S1,
                                      const DependentTemplateStorage &S2) {
-  if (NestedNameSpecifier *NNS1 = S1.getQualifier(), *NNS2 = S2.getQualifier();
-      !NNS1 != !NNS2 ||
-      (NNS1 && !IsStructurallyEquivalent(Context, NNS1, NNS2)))
+  if (!IsStructurallyEquivalent(Context, S1.getQualifier(), S2.getQualifier()))
     return false;
 
   IdentifierOrOverloadedOperator IO1 = S1.getName(), IO2 = S2.getName();
@@ -885,10 +878,10 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       // Treat the enumeration as its underlying type and use the builtin type
       // class comparison.
       if (T1->getTypeClass() == Type::Enum) {
-        T1 = T1->getAs<EnumType>()->getDecl()->getIntegerType();
+        T1 = T1->getAs<EnumType>()->getOriginalDecl()->getIntegerType();
         assert(T2->isBuiltinType() && !T1.isNull()); // Sanity check
       } else if (T2->getTypeClass() == Type::Enum) {
-        T2 = T2->getAs<EnumType>()->getDecl()->getIntegerType();
+        T2 = T2->getAs<EnumType>()->getOriginalDecl()->getIntegerType();
         assert(T1->isBuiltinType() && !T2.isNull()); // Sanity check
       }
       TC = Type::Builtin;
@@ -1206,23 +1199,35 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       return false;
     break;
 
-  case Type::Using:
-    if (!IsStructurallyEquivalent(Context, cast<UsingType>(T1)->getFoundDecl(),
-                                  cast<UsingType>(T2)->getFoundDecl()))
+  case Type::Using: {
+    auto *U1 = cast<UsingType>(T1), *U2 = cast<UsingType>(T2);
+    if (U1->getKeyword() != U2->getKeyword())
       return false;
-    if (!IsStructurallyEquivalent(Context,
-                                  cast<UsingType>(T1)->getUnderlyingType(),
-                                  cast<UsingType>(T2)->getUnderlyingType()))
+    if (!IsStructurallyEquivalent(Context, U1->getQualifier(),
+                                  U2->getQualifier()))
+      return false;
+    if (!IsStructurallyEquivalent(Context, U1->getDecl(), U2->getDecl()))
+      return false;
+    if (!IsStructurallyEquivalent(Context, U1->desugar(), U2->desugar()))
       return false;
     break;
-
-  case Type::Typedef:
-    if (!IsStructurallyEquivalent(Context, cast<TypedefType>(T1)->getDecl(),
-                                  cast<TypedefType>(T2)->getDecl()) ||
-        !IsStructurallyEquivalent(Context, cast<TypedefType>(T1)->desugar(),
-                                  cast<TypedefType>(T2)->desugar()))
+  }
+  case Type::Typedef: {
+    auto *U1 = cast<TypedefType>(T1), *U2 = cast<TypedefType>(T2);
+    if (U1->getKeyword() != U2->getKeyword())
+      return false;
+    if (!IsStructurallyEquivalent(Context, U1->getQualifier(),
+                                  U2->getQualifier()))
+      return false;
+    if (!IsStructurallyEquivalent(Context, U1->getDecl(), U2->getDecl()))
+      return false;
+    if (U1->typeMatchesDecl() != U2->typeMatchesDecl())
+      return false;
+    if (!U1->typeMatchesDecl() &&
+        !IsStructurallyEquivalent(Context, U1->desugar(), U2->desugar()))
       return false;
     break;
+  }
 
   case Type::TypeOfExpr:
     if (!IsStructurallyEquivalent(
@@ -1286,10 +1291,20 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
 
   case Type::Record:
   case Type::Enum:
-    if (!IsStructurallyEquivalent(Context, cast<TagType>(T1)->getDecl(),
-                                  cast<TagType>(T2)->getDecl()))
+  case Type::InjectedClassName: {
+    const auto *TT1 = cast<TagType>(T1), *TT2 = cast<TagType>(T2);
+    if (TT1->getKeyword() != TT2->getKeyword())
+      return false;
+    if (TT1->isTagOwned() != TT2->isTagOwned())
+      return false;
+    if (!IsStructurallyEquivalent(Context, TT1->getQualifier(),
+                                  TT2->getQualifier()))
+      return false;
+    if (!IsStructurallyEquivalent(Context, TT1->getOriginalDecl(),
+                                  TT2->getOriginalDecl()))
       return false;
     break;
+  }
 
   case Type::TemplateTypeParm: {
     const auto *Parm1 = cast<TemplateTypeParmType>(T1);
@@ -1322,6 +1337,14 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     break;
   }
 
+  case Type::SubstBuiltinTemplatePack: {
+    const auto *Subst1 = cast<SubstBuiltinTemplatePackType>(T1);
+    const auto *Subst2 = cast<SubstBuiltinTemplatePackType>(T2);
+    if (!IsStructurallyEquivalent(Context, Subst1->getArgumentPack(),
+                                  Subst2->getArgumentPack()))
+      return false;
+    break;
+  }
   case Type::SubstTemplateTypeParmPack: {
     const auto *Subst1 = cast<SubstTemplateTypeParmPackType>(T1);
     const auto *Subst2 = cast<SubstTemplateTypeParmPackType>(T2);
@@ -1348,33 +1371,6 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     break;
   }
 
-  case Type::Elaborated: {
-    const auto *Elab1 = cast<ElaboratedType>(T1);
-    const auto *Elab2 = cast<ElaboratedType>(T2);
-    // CHECKME: what if a keyword is ElaboratedTypeKeyword::None or
-    // ElaboratedTypeKeyword::Typename
-    // ?
-    if (Elab1->getKeyword() != Elab2->getKeyword())
-      return false;
-    if (!IsStructurallyEquivalent(Context, Elab1->getQualifier(),
-                                  Elab2->getQualifier()))
-      return false;
-    if (!IsStructurallyEquivalent(Context, Elab1->getNamedType(),
-                                  Elab2->getNamedType()))
-      return false;
-    break;
-  }
-
-  case Type::InjectedClassName: {
-    const auto *Inj1 = cast<InjectedClassNameType>(T1);
-    const auto *Inj2 = cast<InjectedClassNameType>(T2);
-    if (!IsStructurallyEquivalent(Context,
-                                  Inj1->getInjectedSpecializationType(),
-                                  Inj2->getInjectedSpecializationType()))
-      return false;
-    break;
-  }
-
   case Type::DependentName: {
     const auto *Typename1 = cast<DependentNameType>(T1);
     const auto *Typename2 = cast<DependentNameType>(T2);
@@ -1549,8 +1545,8 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
   // types
   if (Field1->isAnonymousStructOrUnion() &&
       Field2->isAnonymousStructOrUnion()) {
-    RecordDecl *D1 = Field1->getType()->castAs<RecordType>()->getDecl();
-    RecordDecl *D2 = Field2->getType()->castAs<RecordType>()->getDecl();
+    RecordDecl *D1 = Field1->getType()->castAs<RecordType>()->getOriginalDecl();
+    RecordDecl *D2 = Field2->getType()->castAs<RecordType>()->getOriginalDecl();
     return IsStructurallyEquivalent(Context, D1, D2);
   }
 
@@ -1628,7 +1624,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      FieldDecl *Field1, FieldDecl *Field2) {
   const auto *Owner2 = cast<RecordDecl>(Field2->getDeclContext());
   return IsStructurallyEquivalent(Context, Field1, Field2,
-                                  Context.ToCtx.getTypeDeclType(Owner2));
+                                  Context.ToCtx.getCanonicalTagType(Owner2));
 }
 
 /// Determine structural equivalence of two methods.
@@ -1801,7 +1797,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     if (Context.Complain) {
       Context.Diag2(D2->getLocation(), Context.getApplicableDiagnostic(
                                            diag::err_odr_tag_type_inconsistent))
-          << Context.ToCtx.getTypeDeclType(D2)
+          << Context.ToCtx.getCanonicalTagType(D2)
           << (&Context.FromCtx != &Context.ToCtx);
       Context.Diag1(D1->getLocation(), diag::note_odr_tag_kind_here)
           << D1->getDeclName() << (unsigned)D1->getTagKind();
@@ -1903,7 +1899,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
           Context.Diag2(D2->getLocation(),
                         Context.getApplicableDiagnostic(
                             diag::err_odr_tag_type_inconsistent))
-              << Context.ToCtx.getTypeDeclType(D2)
+              << Context.ToCtx.getCanonicalTagType(D2)
               << (&Context.FromCtx != &Context.ToCtx);
           Context.Diag2(D2->getLocation(), diag::note_odr_number_of_bases)
               << D2CXX->getNumBases();
@@ -1924,7 +1920,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
             Context.Diag2(D2->getLocation(),
                           Context.getApplicableDiagnostic(
                               diag::err_odr_tag_type_inconsistent))
-                << Context.ToCtx.getTypeDeclType(D2)
+                << Context.ToCtx.getCanonicalTagType(D2)
                 << (&Context.FromCtx != &Context.ToCtx);
             Context.Diag2(Base2->getBeginLoc(), diag::note_odr_base)
                 << Base2->getType() << Base2->getSourceRange();
@@ -1940,7 +1936,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
             Context.Diag2(D2->getLocation(),
                           Context.getApplicableDiagnostic(
                               diag::err_odr_tag_type_inconsistent))
-                << Context.ToCtx.getTypeDeclType(D2)
+                << Context.ToCtx.getCanonicalTagType(D2)
                 << (&Context.FromCtx != &Context.ToCtx);
             Context.Diag2(Base2->getBeginLoc(), diag::note_odr_virtual_base)
                 << Base2->isVirtual() << Base2->getSourceRange();
@@ -1962,7 +1958,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
             Context.Diag2(D2->getLocation(),
                           Context.getApplicableDiagnostic(
                               diag::err_odr_tag_type_inconsistent))
-                << Context.ToCtx.getTypeDeclType(D2CXX)
+                << Context.ToCtx.getCanonicalTagType(D2CXX)
                 << (&Context.FromCtx != &Context.ToCtx);
             Context.Diag1((*Friend1)->getFriendLoc(), diag::note_odr_friend);
             Context.Diag2(D2->getLocation(), diag::note_odr_missing_friend);
@@ -1975,7 +1971,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
             Context.Diag2(D2->getLocation(),
                           Context.getApplicableDiagnostic(
                               diag::err_odr_tag_type_inconsistent))
-                << Context.ToCtx.getTypeDeclType(D2CXX)
+                << Context.ToCtx.getCanonicalTagType(D2CXX)
                 << (&Context.FromCtx != &Context.ToCtx);
             Context.Diag1((*Friend1)->getFriendLoc(), diag::note_odr_friend);
             Context.Diag2((*Friend2)->getFriendLoc(), diag::note_odr_friend);
@@ -1989,7 +1985,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
           Context.Diag2(D2->getLocation(),
                         Context.getApplicableDiagnostic(
                             diag::err_odr_tag_type_inconsistent))
-              << Context.ToCtx.getTypeDeclType(D2)
+              << Context.ToCtx.getCanonicalTagType(D2)
               << (&Context.FromCtx != &Context.ToCtx);
           Context.Diag2((*Friend2)->getFriendLoc(), diag::note_odr_friend);
           Context.Diag1(D1->getLocation(), diag::note_odr_missing_friend);
@@ -2001,7 +1997,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
         Context.Diag2(D2->getLocation(),
                       Context.getApplicableDiagnostic(
                           diag::err_odr_tag_type_inconsistent))
-            << Context.ToCtx.getTypeDeclType(D2)
+            << Context.ToCtx.getCanonicalTagType(D2)
             << (&Context.FromCtx != &Context.ToCtx);
         const CXXBaseSpecifier *Base1 = D1CXX->bases_begin();
         Context.Diag1(Base1->getBeginLoc(), diag::note_odr_base)
@@ -2013,7 +2009,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
   }
 
   // Check the fields for consistency.
-  QualType D2Type = Context.ToCtx.getTypeDeclType(D2);
+  CanQualType D2Type = Context.ToCtx.getCanonicalTagType(D2);
   RecordDecl::field_iterator Field2 = D2->field_begin(),
                              Field2End = D2->field_end();
   for (RecordDecl::field_iterator Field1 = D1->field_begin(),
@@ -2024,7 +2020,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
         Context.Diag2(D2->getLocation(),
                       Context.getApplicableDiagnostic(
                           diag::err_odr_tag_type_inconsistent))
-            << Context.ToCtx.getTypeDeclType(D2)
+            << Context.ToCtx.getCanonicalTagType(D2)
             << (&Context.FromCtx != &Context.ToCtx);
         Context.Diag1(Field1->getLocation(), diag::note_odr_field)
             << Field1->getDeclName() << Field1->getType();
@@ -2041,7 +2037,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     if (Context.Complain) {
       Context.Diag2(D2->getLocation(), Context.getApplicableDiagnostic(
                                            diag::err_odr_tag_type_inconsistent))
-          << Context.ToCtx.getTypeDeclType(D2)
+          << Context.ToCtx.getCanonicalTagType(D2)
           << (&Context.FromCtx != &Context.ToCtx);
       Context.Diag2(Field2->getLocation(), diag::note_odr_field)
           << Field2->getDeclName() << Field2->getType();
@@ -2101,7 +2097,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
         Context.Diag2(D2->getLocation(),
                       Context.getApplicableDiagnostic(
                           diag::err_odr_tag_type_inconsistent))
-            << Context.ToCtx.getTypeDeclType(D2)
+            << Context.ToCtx.getCanonicalTagType(D2)
             << (&Context.FromCtx != &Context.ToCtx);
         Context.Diag1(D1->getLocation(),
                       D1->isFixed()
@@ -2124,7 +2120,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
           Context.Diag2(D2->getLocation(),
                         Context.getApplicableDiagnostic(
                             diag::err_odr_tag_type_inconsistent))
-              << Context.ToCtx.getTypeDeclType(D2)
+              << Context.ToCtx.getCanonicalTagType(D2)
               << (&Context.FromCtx != &Context.ToCtx);
           Context.Diag2(D2->getLocation(),
                         diag::note_odr_incompatible_fixed_underlying_type)
@@ -2162,7 +2158,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
         Context.Diag2(D2->getLocation(),
                       Context.getApplicableDiagnostic(
                           diag::err_odr_tag_type_inconsistent))
-            << Context.ToCtx.getTypeDeclType(D2)
+            << Context.ToCtx.getCanonicalTagType(D2)
             << (&Context.FromCtx != &Context.ToCtx);
         Context.Diag1((*EC1)->getLocation(), diag::note_odr_enumerator)
             << (*EC1)->getDeclName() << toString((*EC1)->getInitVal(), 10);
@@ -2180,7 +2176,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
         Context.Diag2(D2->getLocation(),
                       Context.getApplicableDiagnostic(
                           diag::err_odr_tag_type_inconsistent))
-            << Context.ToCtx.getTypeDeclType(D2)
+            << Context.ToCtx.getCanonicalTagType(D2)
             << (&Context.FromCtx != &Context.ToCtx);
         Context.Diag2((*EC2)->getLocation(), diag::note_odr_enumerator)
             << (*EC2)->getDeclName() << toString((*EC2)->getInitVal(), 10);
@@ -2198,7 +2194,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     if (Context.Complain) {
       Context.Diag2(D2->getLocation(), Context.getApplicableDiagnostic(
                                            diag::err_odr_tag_type_inconsistent))
-          << Context.ToCtx.getTypeDeclType(D2)
+          << Context.ToCtx.getCanonicalTagType(D2)
           << (&Context.FromCtx != &Context.ToCtx);
       Context.Diag2((*EC2)->getLocation(), diag::note_odr_enumerator)
           << (*EC2)->getDeclName() << toString((*EC2)->getInitVal(), 10);
@@ -2595,7 +2591,7 @@ DiagnosticBuilder StructuralEquivalenceContext::Diag2(SourceLocation Loc,
 UnsignedOrNone
 StructuralEquivalenceContext::findUntaggedStructOrUnionIndex(RecordDecl *Anon) {
   ASTContext &Context = Anon->getASTContext();
-  QualType AnonTy = Context.getRecordType(Anon);
+  CanQualType AnonTy = Context.getCanonicalTagType(Anon);
 
   const auto *Owner = dyn_cast<RecordDecl>(Anon->getDeclContext());
   if (!Owner)
@@ -2617,12 +2613,8 @@ StructuralEquivalenceContext::findUntaggedStructOrUnionIndex(RecordDecl *Anon) {
     // If the field looks like this:
     // struct { ... } A;
     QualType FieldType = F->getType();
-    // In case of nested structs.
-    while (const auto *ElabType = dyn_cast<ElaboratedType>(FieldType))
-      FieldType = ElabType->getNamedType();
-
     if (const auto *RecType = dyn_cast<RecordType>(FieldType)) {
-      const RecordDecl *RecDecl = RecType->getDecl();
+      const RecordDecl *RecDecl = RecType->getOriginalDecl();
       if (RecDecl->getDeclContext() == Owner && !RecDecl->getIdentifier()) {
         if (Context.hasSameType(FieldType, AnonTy))
           break;
diff --git a/clang/lib/AST/ASTTypeTraits.cpp b/clang/lib/AST/ASTTypeTraits.cpp
index 99916f523aa95..d2f7fdbbad04d 100644
--- a/clang/lib/AST/ASTTypeTraits.cpp
+++ b/clang/lib/AST/ASTTypeTraits.cpp
@@ -194,8 +194,8 @@ void DynTypedNode::print(llvm::raw_ostream &OS,
   else if (const NestedNameSpecifier *NNS = get<NestedNameSpecifier>())
     NNS->print(OS, PP);
   else if (const NestedNameSpecifierLoc *NNSL = get<NestedNameSpecifierLoc>()) {
-    if (const NestedNameSpecifier *NNS = NNSL->getNestedNameSpecifier())
-      NNS->print(OS, PP);
+    if (NestedNameSpecifier NNS = NNSL->getNestedNameSpecifier())
+      NNS.print(OS, PP);
     else
       OS << "(empty NestedNameSpecifierLoc)";
   } else if (const QualType *QT = get<QualType>())
@@ -234,13 +234,39 @@ void DynTypedNode::dump(llvm::raw_ostream &OS,
     OS << "Unable to dump values of type " << NodeKind.asStringRef() << "\n";
 }
 
-SourceRange DynTypedNode::getSourceRange() const {
+SourceRange DynTypedNode::getSourceRange(bool IncludeQualifier) const {
   if (const CXXCtorInitializer *CCI = get<CXXCtorInitializer>())
     return CCI->getSourceRange();
   if (const NestedNameSpecifierLoc *NNSL = get<NestedNameSpecifierLoc>())
     return NNSL->getSourceRange();
-  if (const TypeLoc *TL = get<TypeLoc>())
-    return TL->getSourceRange();
+  if (const TypeLoc *TL = get<TypeLoc>()) {
+    if (IncludeQualifier)
+      return TL->getSourceRange();
+    switch (TL->getTypeLocClass()) {
+    case TypeLoc::DependentName:
+      return TL->castAs<DependentNameTypeLoc>().getNameLoc();
+    case TypeLoc::TemplateSpecialization: {
+      auto T = TL->castAs<TemplateSpecializationTypeLoc>();
+      return SourceRange(T.getTemplateNameLoc(), T.getEndLoc());
+    }
+    case TypeLoc::DependentTemplateSpecialization: {
+      auto T = TL->castAs<DependentTemplateSpecializationTypeLoc>();
+      return SourceRange(T.getTemplateNameLoc(), T.getEndLoc());
+    }
+    case TypeLoc::Enum:
+    case TypeLoc::Record:
+    case TypeLoc::InjectedClassName:
+      return TL->castAs<TagTypeLoc>().getNameLoc();
+    case TypeLoc::Typedef:
+      return TL->castAs<TypedefTypeLoc>().getNameLoc();
+    case TypeLoc::UnresolvedUsing:
+      return TL->castAs<UnresolvedUsingTypeLoc>().getNameLoc();
+    case TypeLoc::Using:
+      return TL->castAs<UsingTypeLoc>().getNameLoc();
+    default:
+      return TL->getSourceRange();
+    }
+  }
   if (const Decl *D = get<Decl>())
     return D->getSourceRange();
   if (const Stmt *S = get<Stmt>())
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index cc99efa01b83e..c36cb644282d8 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -60,16 +60,18 @@ template <class Emitter> class OptionScope final {
 public:
   /// Root constructor, compiling or discarding primitives.
   OptionScope(Compiler<Emitter> *Ctx, bool NewDiscardResult,
-              bool NewInitializing)
+              bool NewInitializing, bool NewToLValue)
       : Ctx(Ctx), OldDiscardResult(Ctx->DiscardResult),
-        OldInitializing(Ctx->Initializing) {
+        OldInitializing(Ctx->Initializing), OldToLValue(Ctx->ToLValue) {
     Ctx->DiscardResult = NewDiscardResult;
     Ctx->Initializing = NewInitializing;
+    Ctx->ToLValue = NewToLValue;
   }
 
   ~OptionScope() {
     Ctx->DiscardResult = OldDiscardResult;
     Ctx->Initializing = OldInitializing;
+    Ctx->ToLValue = OldToLValue;
   }
 
 private:
@@ -78,6 +80,7 @@ template <class Emitter> class OptionScope final {
   /// Old discard flag to restore.
   bool OldDiscardResult;
   bool OldInitializing;
+  bool OldToLValue;
 };
 
 template <class Emitter>
@@ -222,6 +225,9 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
   switch (CE->getCastKind()) {
   case CK_LValueToRValue: {
+    if (ToLValue && CE->getType()->isPointerType())
+      return this->delegate(SubExpr);
+
     if (SubExpr->getType().isVolatileQualified())
       return this->emitInvalidCast(CastKind::Volatile, /*Fatal=*/true, CE);
 
@@ -250,7 +256,7 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
 
     // Prepare storage for the result.
     if (!Initializing && !SubExprT) {
-      std::optional<unsigned> LocalIndex = allocateLocal(SubExpr);
+      UnsignedOrNone LocalIndex = allocateLocal(SubExpr);
       if (!LocalIndex)
         return false;
       if (!this->emitGetPtrLocal(*LocalIndex, CE))
@@ -553,9 +559,10 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     // Possibly diagnose casts to enum types if the target type does not
     // have a fixed size.
     if (Ctx.getLangOpts().CPlusPlus && CE->getType()->isEnumeralType()) {
-      if (const auto *ET = CE->getType().getCanonicalType()->castAs<EnumType>();
-          !ET->getDecl()->isFixed()) {
-        if (!this->emitCheckEnumValue(*FromT, ET->getDecl(), CE))
+      const auto *ET = CE->getType().getCanonicalType()->castAs<EnumType>();
+      const auto *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+      if (!ED->isFixed()) {
+        if (!this->emitCheckEnumValue(*FromT, ED, CE))
           return false;
       }
     }
@@ -602,7 +609,7 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     // We're creating a complex value here, so we need to
     // allocate storage for it.
     if (!Initializing) {
-      std::optional<unsigned> LocalIndex = allocateTemporary(CE);
+      UnsignedOrNone LocalIndex = allocateTemporary(CE);
       if (!LocalIndex)
         return false;
       if (!this->emitGetPtrLocal(*LocalIndex, CE))
@@ -626,7 +633,7 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     assert(CE->getType()->isAnyComplexType());
     assert(SubExpr->getType()->isAnyComplexType());
     if (!Initializing) {
-      std::optional<unsigned> LocalIndex = allocateLocal(CE);
+      UnsignedOrNone LocalIndex = allocateLocal(CE);
       if (!LocalIndex)
         return false;
       if (!this->emitGetPtrLocal(*LocalIndex, CE))
@@ -666,12 +673,12 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   }
 
   case CK_VectorSplat: {
-    assert(!classify(CE->getType()));
-    assert(classify(SubExpr->getType()));
+    assert(!canClassify(CE->getType()));
+    assert(canClassify(SubExpr->getType()));
     assert(CE->getType()->isVectorType());
 
     if (!Initializing) {
-      std::optional<unsigned> LocalIndex = allocateLocal(CE);
+      UnsignedOrNone LocalIndex = allocateLocal(CE);
       if (!LocalIndex)
         return false;
       if (!this->emitGetPtrLocal(*LocalIndex, CE))
@@ -715,7 +722,7 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
     assert(CE->getType()->isVectorType());
 
     if (!Initializing) {
-      std::optional<unsigned> LocalIndex = allocateTemporary(CE);
+      UnsignedOrNone LocalIndex = allocateTemporary(CE);
       if (!LocalIndex)
         return false;
       if (!this->emitGetPtrLocal(*LocalIndex, CE))
@@ -803,7 +810,7 @@ bool Compiler<Emitter>::VisitImaginaryLiteral(const ImaginaryLiteral *E) {
     return true;
 
   if (!Initializing) {
-    std::optional<unsigned> LocalIndex = allocateTemporary(E);
+    UnsignedOrNone LocalIndex = allocateTemporary(E);
     if (!LocalIndex)
       return false;
     if (!this->emitGetPtrLocal(*LocalIndex, E))
@@ -904,7 +911,7 @@ bool Compiler<Emitter>::VisitBinaryOperator(const BinaryOperator *BO) {
 
     // We need a temporary variable holding our return value.
     if (!Initializing) {
-      std::optional<unsigned> ResultIndex = this->allocateLocal(BO);
+      UnsignedOrNone ResultIndex = this->allocateLocal(BO);
       if (!this->emitGetPtrLocal(*ResultIndex, BO))
         return false;
     }
@@ -1144,7 +1151,7 @@ template <class Emitter>
 bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
   // Prepare storage for result.
   if (!Initializing) {
-    std::optional<unsigned> LocalIndex = allocateTemporary(E);
+    UnsignedOrNone LocalIndex = allocateTemporary(E);
     if (!LocalIndex)
       return false;
     if (!this->emitGetPtrLocal(*LocalIndex, E))
@@ -1203,7 +1210,7 @@ bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
 
     if (!LHSIsComplex) {
       // This is using the RHS type for the fake-complex LHS.
-      std::optional<unsigned> LocalIndex = allocateTemporary(RHS);
+      UnsignedOrNone LocalIndex = allocateTemporary(RHS);
       if (!LocalIndex)
         return false;
       LHSOffset = *LocalIndex;
@@ -1378,7 +1385,7 @@ bool Compiler<Emitter>::VisitVectorBinOp(const BinaryOperator *E) {
 
   // Prepare storage for result.
   if (!Initializing && !E->isCompoundAssignmentOp()) {
-    std::optional<unsigned> LocalIndex = allocateTemporary(E);
+    UnsignedOrNone LocalIndex = allocateTemporary(E);
     if (!LocalIndex)
       return false;
     if (!this->emitGetPtrLocal(*LocalIndex, E))
@@ -1931,15 +1938,19 @@ bool Compiler<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
               dyn_cast<EmbedExpr>(Init->IgnoreParenImpCasts())) {
         PrimType TargetT = classifyPrim(Init->getType());
 
-        auto Eval = [&](const Expr *Init, unsigned ElemIndex) {
-          PrimType InitT = classifyPrim(Init->getType());
-          if (!this->visit(Init))
-            return false;
-          if (InitT != TargetT) {
-            if (!this->emitCast(InitT, TargetT, E))
+        auto Eval = [&](const IntegerLiteral *IL, unsigned ElemIndex) {
+          if (TargetT == PT_Float) {
+            if (!this->emitConst(IL->getValue(), classifyPrim(IL), Init))
+              return false;
+            const auto *Sem = &Ctx.getFloatSemantics(CAT->getElementType());
+            if (!this->emitCastIntegralFloating(classifyPrim(IL), Sem,
+                                                getFPOptions(E), E))
+              return false;
+          } else {
+            if (!this->emitConst(IL->getValue(), TargetT, Init))
               return false;
           }
-          return this->emitInitElem(TargetT, ElemIndex, Init);
+          return this->emitInitElem(TargetT, ElemIndex, IL);
         };
         if (!EmbedS->doForEachDataElement(Eval, ElementIndex))
           return false;
@@ -2061,24 +2072,36 @@ bool Compiler<Emitter>::visitArrayElemInit(unsigned ElemIndex, const Expr *Init,
 template <class Emitter>
 bool Compiler<Emitter>::visitCallArgs(ArrayRef<const Expr *> Args,
                                       const FunctionDecl *FuncDecl,
-                                      bool Activate) {
+                                      bool Activate, bool IsOperatorCall) {
   assert(VarScope->getKind() == ScopeKind::Call);
-  bool HasNonNullAttr = false;
   llvm::BitVector NonNullArgs;
-  if (FuncDecl && FuncDecl->hasAttr<NonNullAttr>()) {
-    HasNonNullAttr = true;
+  if (FuncDecl && FuncDecl->hasAttr<NonNullAttr>())
     NonNullArgs = collectNonNullArgs(FuncDecl, Args);
-  }
+
+  bool ExplicitMemberFn = false;
+  if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(FuncDecl))
+    ExplicitMemberFn = MD->isExplicitObjectMemberFunction();
 
   unsigned ArgIndex = 0;
   for (const Expr *Arg : Args) {
-    if (OptPrimType T = classify(Arg)) {
+    if (canClassify(Arg)) {
       if (!this->visit(Arg))
         return false;
     } else {
 
-      std::optional<unsigned> LocalIndex = allocateLocal(
-          Arg, Arg->getType(), /*ExtendingDecl=*/nullptr, ScopeKind::Call);
+      DeclTy Source = Arg;
+      if (FuncDecl) {
+        // Try to use the parameter declaration instead of the argument
+        // expression as a source.
+        unsigned DeclIndex = ArgIndex - IsOperatorCall + ExplicitMemberFn;
+        if (DeclIndex < FuncDecl->getNumParams())
+          Source = FuncDecl->getParamDecl(ArgIndex - IsOperatorCall +
+                                          ExplicitMemberFn);
+      }
+
+      UnsignedOrNone LocalIndex =
+          allocateLocal(std::move(Source), Arg->getType(),
+                        /*ExtendingDecl=*/nullptr, ScopeKind::Call);
       if (!LocalIndex)
         return false;
 
@@ -2094,7 +2117,7 @@ bool Compiler<Emitter>::visitCallArgs(ArrayRef<const Expr *> Args,
         return false;
     }
 
-    if (HasNonNullAttr && NonNullArgs[ArgIndex]) {
+    if (!NonNullArgs.empty() && NonNullArgs[ArgIndex]) {
       PrimType ArgT = classify(Arg).value_or(PT_Ptr);
       if (ArgT == PT_Ptr) {
         if (!this->emitCheckNonNullArg(ArgT, Arg))
@@ -2919,7 +2942,7 @@ bool Compiler<Emitter>::VisitMaterializeTemporaryExpr(
       return false;
 
     const Expr *Inner = E->getSubExpr()->skipRValueSubobjectAdjustments();
-    if (std::optional<unsigned> LocalIndex =
+    if (UnsignedOrNone LocalIndex =
             allocateLocal(E, Inner->getType(), E->getExtendingDecl())) {
       InitLinkScope<Emitter> ILS(this, InitLink::Temp(*LocalIndex));
       if (!this->emitGetPtrLocal(*LocalIndex, E))
@@ -2966,20 +2989,25 @@ bool Compiler<Emitter>::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
     if (T && !E->isLValue())
       return this->delegate(Init);
 
-    if (std::optional<unsigned> GlobalIndex = P.createGlobal(E)) {
-      if (!this->emitGetPtrGlobal(*GlobalIndex, E))
-        return false;
+    std::optional<unsigned> GlobalIndex = P.createGlobal(E);
+    if (!GlobalIndex)
+      return false;
 
-      if (T) {
-        if (!this->visit(Init))
-          return false;
-        return this->emitInitGlobal(*T, *GlobalIndex, E);
-      }
+    if (!this->emitGetPtrGlobal(*GlobalIndex, E))
+      return false;
+
+    // Since this is a global variable, we might've already seen,
+    // don't do it again.
+    if (P.isGlobalInitialized(*GlobalIndex))
+      return true;
 
-      return this->visitInitializer(Init) && this->emitFinishInit(E);
+    if (T) {
+      if (!this->visit(Init))
+        return false;
+      return this->emitInitGlobal(*T, *GlobalIndex, E);
     }
 
-    return false;
+    return this->visitInitializer(Init) && this->emitFinishInit(E);
   }
 
   // Otherwise, use a local variable.
@@ -2991,7 +3019,7 @@ bool Compiler<Emitter>::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
   unsigned LocalIndex;
   if (T)
     LocalIndex = this->allocateLocalPrimitive(Init, *T, /*IsConst=*/false);
-  else if (std::optional<unsigned> MaybeIndex = this->allocateLocal(Init))
+  else if (UnsignedOrNone MaybeIndex = this->allocateLocal(Init))
     LocalIndex = *MaybeIndex;
   else
     return false;
@@ -3157,25 +3185,18 @@ bool Compiler<Emitter>::VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) {
 template <class Emitter>
 bool Compiler<Emitter>::VisitCXXConstructExpr(const CXXConstructExpr *E) {
   QualType T = E->getType();
-  assert(!classify(T));
+  assert(!canClassify(T));
 
   if (T->isRecordType()) {
     const CXXConstructorDecl *Ctor = E->getConstructor();
 
-    // Trivial copy/move constructor. Avoid copy.
-    if (Ctor->isDefaulted() && Ctor->isCopyOrMoveConstructor() &&
-        Ctor->isTrivial() &&
-        E->getArg(0)->isTemporaryObject(Ctx.getASTContext(),
-                                        T->getAsCXXRecordDecl()))
-      return this->visitInitializer(E->getArg(0));
-
     // If we're discarding a construct expression, we still need
     // to allocate a variable and call the constructor and destructor.
     if (DiscardResult) {
       if (Ctor->isTrivial())
         return true;
       assert(!Initializing);
-      std::optional<unsigned> LocalIndex = allocateLocal(E);
+      UnsignedOrNone LocalIndex = allocateLocal(E);
 
       if (!LocalIndex)
         return false;
@@ -3184,6 +3205,13 @@ bool Compiler<Emitter>::VisitCXXConstructExpr(const CXXConstructExpr *E) {
         return false;
     }
 
+    // Trivial copy/move constructor. Avoid copy.
+    if (Ctor->isDefaulted() && Ctor->isCopyOrMoveConstructor() &&
+        Ctor->isTrivial() &&
+        E->getArg(0)->isTemporaryObject(Ctx.getASTContext(),
+                                        T->getAsCXXRecordDecl()))
+      return this->visitInitializer(E->getArg(0));
+
     // Zero initialization.
     if (E->requiresZeroInitialization()) {
       const Record *R = getRecord(E->getType());
@@ -3378,7 +3406,7 @@ bool Compiler<Emitter>::VisitCXXScalarValueInitExpr(
 
   if (const auto *CT = Ty->getAs<ComplexType>()) {
     if (!Initializing) {
-      std::optional<unsigned> LocalIndex = allocateLocal(E);
+      UnsignedOrNone LocalIndex = allocateLocal(E);
       if (!LocalIndex)
         return false;
       if (!this->emitGetPtrLocal(*LocalIndex, E))
@@ -3401,7 +3429,7 @@ bool Compiler<Emitter>::VisitCXXScalarValueInitExpr(
   if (const auto *VT = Ty->getAs<VectorType>()) {
     // FIXME: Code duplication with the _Complex case above.
     if (!Initializing) {
-      std::optional<unsigned> LocalIndex = allocateLocal(E);
+      UnsignedOrNone LocalIndex = allocateLocal(E);
       if (!LocalIndex)
         return false;
       if (!this->emitGetPtrLocal(*LocalIndex, E))
@@ -4027,8 +4055,7 @@ bool Compiler<Emitter>::VisitExtVectorElementExpr(
 
   // Now the vector variable for the return value.
   if (!Initializing) {
-    std::optional<unsigned> ResultIndex;
-    ResultIndex = allocateLocal(E);
+    UnsignedOrNone ResultIndex = allocateLocal(E);
     if (!ResultIndex)
       return false;
     if (!this->emitGetPtrLocal(*ResultIndex, E))
@@ -4089,8 +4116,7 @@ bool Compiler<Emitter>::VisitCXXStdInitializerListExpr(
 
   PrimType SecondFieldT = classifyPrim(R->getField(1u)->Decl->getType());
   if (isIntegralType(SecondFieldT)) {
-    if (!this->emitConst(static_cast<APSInt>(ArrayType->getSize()),
-                         SecondFieldT, E))
+    if (!this->emitConst(ArrayType->getSize(), SecondFieldT, E))
       return false;
     return this->emitInitField(SecondFieldT, R->getField(1u)->Offset, E);
   }
@@ -4100,7 +4126,7 @@ bool Compiler<Emitter>::VisitCXXStdInitializerListExpr(
     return false;
   if (!this->emitExpandPtr(E))
     return false;
-  if (!this->emitConst(static_cast<APSInt>(ArrayType->getSize()), PT_Uint64, E))
+  if (!this->emitConst(ArrayType->getSize(), PT_Uint64, E))
     return false;
   if (!this->emitArrayElemPtrPop(PT_Uint64, E))
     return false;
@@ -4132,13 +4158,13 @@ bool Compiler<Emitter>::VisitStmtExpr(const StmtExpr *E) {
 
 template <class Emitter> bool Compiler<Emitter>::discard(const Expr *E) {
   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/true,
-                             /*NewInitializing=*/false);
+                             /*NewInitializing=*/false, /*ToLValue=*/false);
   return this->Visit(E);
 }
 
 template <class Emitter> bool Compiler<Emitter>::delegate(const Expr *E) {
   // We're basically doing:
-  // OptionScope<Emitter> Scope(this, DicardResult, Initializing);
+  // OptionScope<Emitter> Scope(this, DicardResult, Initializing, ToLValue);
   // but that's unnecessary of course.
   return this->Visit(E);
 }
@@ -4152,8 +4178,8 @@ template <class Emitter> bool Compiler<Emitter>::visit(const Expr *E) {
 
   // Create local variable to hold the return value.
   if (!E->isGLValue() && !E->getType()->isAnyComplexType() &&
-      !classify(E->getType())) {
-    std::optional<unsigned> LocalIndex = allocateLocal(E);
+      !canClassify(E->getType())) {
+    UnsignedOrNone LocalIndex = allocateLocal(E);
     if (!LocalIndex)
       return false;
 
@@ -4166,16 +4192,22 @@ template <class Emitter> bool Compiler<Emitter>::visit(const Expr *E) {
   //  Otherwise,we have a primitive return value, produce the value directly
   //  and push it on the stack.
   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
-                             /*NewInitializing=*/false);
+                             /*NewInitializing=*/false, /*ToLValue=*/ToLValue);
   return this->Visit(E);
 }
 
 template <class Emitter>
 bool Compiler<Emitter>::visitInitializer(const Expr *E) {
-  assert(!classify(E->getType()));
+  assert(!canClassify(E->getType()));
+
+  OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
+                             /*NewInitializing=*/true, /*ToLValue=*/false);
+  return this->Visit(E);
+}
 
+template <class Emitter> bool Compiler<Emitter>::visitAsLValue(const Expr *E) {
   OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
-                             /*NewInitializing=*/true);
+                             /*NewInitializing=*/false, /*ToLValue=*/true);
   return this->Visit(E);
 }
 
@@ -4379,7 +4411,7 @@ bool Compiler<Emitter>::visitZeroArrayInitializer(QualType T, const Expr *E) {
 template <class Emitter>
 bool Compiler<Emitter>::visitAssignment(const Expr *LHS, const Expr *RHS,
                                         const Expr *E) {
-  if (!classify(E->getType()))
+  if (!canClassify(E->getType()))
     return false;
 
   if (!this->visit(RHS))
@@ -4472,12 +4504,18 @@ bool Compiler<Emitter>::emitConst(T Value, const Expr *E) {
 template <class Emitter>
 bool Compiler<Emitter>::emitConst(const APSInt &Value, PrimType Ty,
                                   const Expr *E) {
+  return this->emitConst(static_cast<const APInt &>(Value), Ty, E);
+}
+
+template <class Emitter>
+bool Compiler<Emitter>::emitConst(const APInt &Value, PrimType Ty,
+                                  const Expr *E) {
   if (Ty == PT_IntAPS)
     return this->emitConstIntAPS(Value, E);
   if (Ty == PT_IntAP)
     return this->emitConstIntAP(Value, E);
 
-  if (Value.isSigned())
+  if (isSignedType(Ty))
     return this->emitConst(Value.getSExtValue(), Ty, E);
   return this->emitConst(Value.getZExtValue(), Ty, E);
 }
@@ -4491,14 +4529,6 @@ template <class Emitter>
 unsigned Compiler<Emitter>::allocateLocalPrimitive(
     DeclTy &&Src, PrimType Ty, bool IsConst, const ValueDecl *ExtendingDecl,
     ScopeKind SC, bool IsConstexprUnknown) {
-  // Make sure we don't accidentally register the same decl twice.
-  if (const auto *VD =
-          dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
-    assert(!P.getGlobal(VD));
-    assert(!Locals.contains(VD));
-    (void)VD;
-  }
-
   // FIXME: There are cases where Src.is<Expr*>() is wrong, e.g.
   //   (int){12} in C. Consider using Expr::isTemporaryObject() instead
   //   or isa<MaterializeTemporaryExpr>().
@@ -4516,23 +4546,15 @@ unsigned Compiler<Emitter>::allocateLocalPrimitive(
 }
 
 template <class Emitter>
-std::optional<unsigned>
-Compiler<Emitter>::allocateLocal(DeclTy &&Src, QualType Ty,
-                                 const ValueDecl *ExtendingDecl, ScopeKind SC,
-                                 bool IsConstexprUnknown) {
-  // Make sure we don't accidentally register the same decl twice.
-  if ([[maybe_unused]] const auto *VD =
-          dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
-    assert(!P.getGlobal(VD));
-    assert(!Locals.contains(VD));
-  }
-
+UnsignedOrNone Compiler<Emitter>::allocateLocal(DeclTy &&Src, QualType Ty,
+                                                const ValueDecl *ExtendingDecl,
+                                                ScopeKind SC,
+                                                bool IsConstexprUnknown) {
   const ValueDecl *Key = nullptr;
   const Expr *Init = nullptr;
   bool IsTemporary = false;
   if (auto *VD = dyn_cast_if_present<ValueDecl>(Src.dyn_cast<const Decl *>())) {
     Key = VD;
-    Ty = VD->getType();
 
     if (const auto *VarD = dyn_cast<VarDecl>(VD))
       Init = VarD->getInit();
@@ -4561,7 +4583,7 @@ Compiler<Emitter>::allocateLocal(DeclTy &&Src, QualType Ty,
 }
 
 template <class Emitter>
-std::optional<unsigned> Compiler<Emitter>::allocateTemporary(const Expr *E) {
+UnsignedOrNone Compiler<Emitter>::allocateTemporary(const Expr *E) {
   QualType Ty = E->getType();
   assert(!Ty->isRecordType());
 
@@ -4592,7 +4614,7 @@ const RecordType *Compiler<Emitter>::getRecordTy(QualType Ty) {
 
 template <class Emitter> Record *Compiler<Emitter>::getRecord(QualType Ty) {
   if (const auto *RecordTy = getRecordTy(Ty))
-    return getRecord(RecordTy->getDecl());
+    return getRecord(RecordTy->getOriginalDecl()->getDefinitionOrSelf());
   return nullptr;
 }
 
@@ -4640,7 +4662,7 @@ bool Compiler<Emitter>::visitExpr(const Expr *E, bool DestroyToplevelScope) {
   // Expressions with a composite return type.
   // For us, that means everything we don't
   // have a PrimType for.
-  if (std::optional<unsigned> LocalOffset = this->allocateLocal(E)) {
+  if (UnsignedOrNone LocalOffset = this->allocateLocal(E)) {
     InitLinkScope<Emitter> ILS(this, InitLink::Temp(*LocalOffset));
     if (!this->emitGetPtrLocal(*LocalOffset, E))
       return false;
@@ -4837,7 +4859,7 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
         return this->emitSetLocal(*VarT, Offset, VD);
     }
   } else {
-    if (std::optional<unsigned> Offset = this->allocateLocal(
+    if (UnsignedOrNone Offset = this->allocateLocal(
             VD, VD->getType(), nullptr, ScopeKind::Block, IsConstexprUnknown)) {
       if (!Init)
         return true;
@@ -4952,7 +4974,6 @@ bool Compiler<Emitter>::visitAPValueInitializer(const APValue &Val,
 template <class Emitter>
 bool Compiler<Emitter>::VisitBuiltinCallExpr(const CallExpr *E,
                                              unsigned BuiltinID) {
-
   if (BuiltinID == Builtin::BI__builtin_constant_p) {
     // Void argument is always invalid and harder to handle later.
     if (E->getArg(0)->getType()->isVoidType()) {
@@ -4990,18 +5011,38 @@ bool Compiler<Emitter>::VisitBuiltinCallExpr(const CallExpr *E,
 
   // Non-primitive return type. Prepare storage.
   if (!Initializing && !ReturnT && !ReturnType->isVoidType()) {
-    std::optional<unsigned> LocalIndex = allocateLocal(E);
+    UnsignedOrNone LocalIndex = allocateLocal(E);
     if (!LocalIndex)
       return false;
     if (!this->emitGetPtrLocal(*LocalIndex, E))
       return false;
   }
 
-  if (!Context::isUnevaluatedBuiltin(BuiltinID)) {
-    // Put arguments on the stack.
-    for (const auto *Arg : E->arguments()) {
-      if (!this->visit(Arg))
+  // Prepare function arguments including special cases.
+  switch (BuiltinID) {
+  case Builtin::BI__builtin_object_size:
+  case Builtin::BI__builtin_dynamic_object_size: {
+    assert(E->getNumArgs() == 2);
+    const Expr *Arg0 = E->getArg(0);
+    if (Arg0->isGLValue()) {
+      if (!this->visit(Arg0))
         return false;
+
+    } else {
+      if (!this->visitAsLValue(Arg0))
+        return false;
+    }
+    if (!this->visit(E->getArg(1)))
+      return false;
+
+  } break;
+  default:
+    if (!Context::isUnevaluatedBuiltin(BuiltinID)) {
+      // Put arguments on the stack.
+      for (const auto *Arg : E->arguments()) {
+        if (!this->visit(Arg))
+          return false;
+      }
     }
   }
 
@@ -5066,7 +5107,7 @@ bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
       // If we need to discard the return value but the function returns its
       // value via an RVO pointer, we need to create one such pointer just
       // for this call.
-      if (std::optional<unsigned> LocalIndex = allocateLocal(E)) {
+      if (UnsignedOrNone LocalIndex = allocateLocal(E)) {
         if (!this->emitGetPtrLocal(*LocalIndex, E))
           return false;
       }
@@ -5074,7 +5115,7 @@ bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
       // We need the result. Prepare a pointer to return or
       // dup the current one.
       if (!Initializing) {
-        if (std::optional<unsigned> LocalIndex = allocateLocal(E)) {
+        if (UnsignedOrNone LocalIndex = allocateLocal(E)) {
           if (!this->emitGetPtrLocal(*LocalIndex, E))
             return false;
         }
@@ -5110,7 +5151,7 @@ bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
   }
 
   bool Devirtualized = false;
-  std::optional<unsigned> CalleeOffset;
+  UnsignedOrNone CalleeOffset = std::nullopt;
   // Add the (optional, implicit) This pointer.
   if (const auto *MC = dyn_cast<CXXMemberCallExpr>(E)) {
     if (!FuncDecl && classifyPrim(E->getCallee()) == PT_MemberPtr) {
@@ -5154,7 +5195,8 @@ bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
     if (!this->emitCheckPseudoDtor(E))
       return false;
     const Expr *Base = PD->getBase();
-    if (!Base->isGLValue())
+    // E.g. `using T = int; 0.~T();`.
+    if (OptPrimType BaseT = classify(Base); !BaseT || BaseT != PT_Ptr)
       return this->discard(Base);
     if (!this->visit(Base))
       return false;
@@ -5169,7 +5211,8 @@ bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
       return false;
   }
 
-  if (!this->visitCallArgs(Args, FuncDecl, IsAssignmentOperatorCall))
+  if (!this->visitCallArgs(Args, FuncDecl, IsAssignmentOperatorCall,
+                           isa<CXXOperatorCallExpr>(E)))
     return false;
 
   // Undo the argument reversal we did earlier.
@@ -6516,7 +6559,7 @@ bool Compiler<Emitter>::VisitComplexUnaryOperator(const UnaryOperator *E) {
   OptPrimType ResT = classify(E);
   auto prepareResult = [=]() -> bool {
     if (!ResT && !Initializing) {
-      std::optional<unsigned> LocalIndex = allocateLocal(SubExpr);
+      UnsignedOrNone LocalIndex = allocateLocal(SubExpr);
       if (!LocalIndex)
         return false;
       return this->emitGetPtrLocal(*LocalIndex, E);
@@ -6634,7 +6677,7 @@ bool Compiler<Emitter>::VisitVectorUnaryOperator(const UnaryOperator *E) {
     return this->delegate(SubExpr);
 
   if (!Initializing) {
-    std::optional<unsigned> LocalIndex = allocateLocal(SubExpr);
+    UnsignedOrNone LocalIndex = allocateLocal(SubExpr);
     if (!LocalIndex)
       return false;
     if (!this->emitGetPtrLocal(*LocalIndex, E))
@@ -6752,6 +6795,22 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
   // value.
   bool IsReference = D->getType()->isReferenceType();
 
+  // Function parameters.
+  // Note that it's important to check them first since we might have a local
+  // variable created for a ParmVarDecl as well.
+  if (const auto *PVD = dyn_cast<ParmVarDecl>(D)) {
+    if (Ctx.getLangOpts().CPlusPlus && !Ctx.getLangOpts().CPlusPlus11 &&
+        !D->getType()->isIntegralOrEnumerationType()) {
+      return this->emitInvalidDeclRef(cast<DeclRefExpr>(E),
+                                      /*InitializerFailed=*/false, E);
+    }
+    if (auto It = this->Params.find(PVD); It != this->Params.end()) {
+      if (IsReference || !It->second.IsPtr)
+        return this->emitGetParam(classifyPrim(E), It->second.Offset, E);
+
+      return this->emitGetPtrParam(It->second.Offset, E);
+    }
+  }
   // Local variables.
   if (auto It = Locals.find(D); It != Locals.end()) {
     const unsigned Offset = It->second.Offset;
@@ -6769,20 +6828,6 @@ bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
 
     return this->emitGetPtrGlobal(*GlobalIndex, E);
   }
-  // Function parameters.
-  if (const auto *PVD = dyn_cast<ParmVarDecl>(D)) {
-    if (Ctx.getLangOpts().CPlusPlus && !Ctx.getLangOpts().CPlusPlus11 &&
-        !D->getType()->isIntegralOrEnumerationType()) {
-      return this->emitInvalidDeclRef(cast<DeclRefExpr>(E),
-                                      /*InitializerFailed=*/false, E);
-    }
-    if (auto It = this->Params.find(PVD); It != this->Params.end()) {
-      if (IsReference || !It->second.IsPtr)
-        return this->emitGetParam(classifyPrim(E), It->second.Offset, E);
-
-      return this->emitGetPtrParam(It->second.Offset, E);
-    }
-  }
 
   // In case we need to re-visit a declaration.
   auto revisit = [&](const VarDecl *VD) -> bool {
@@ -7142,10 +7187,6 @@ bool Compiler<Emitter>::emitDestruction(const Descriptor *Desc,
   assert(!Desc->isPrimitive());
   assert(!Desc->isPrimitiveArray());
 
-  // Can happen if the decl is invalid.
-  if (Desc->isDummy())
-    return true;
-
   // Arrays.
   if (Desc->isArray()) {
     const Descriptor *ElemDesc = Desc->ElemDesc;
@@ -7240,7 +7281,7 @@ bool Compiler<Emitter>::emitBuiltinBitCast(const CastExpr *E) {
 
   // Prepare storage for the result in case we discard.
   if (DiscardResult && !Initializing && !ToT) {
-    std::optional<unsigned> LocalIndex = allocateLocal(E);
+    UnsignedOrNone LocalIndex = allocateLocal(E);
     if (!LocalIndex)
       return false;
     if (!this->emitGetPtrLocal(*LocalIndex, E))
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index ee8327d8fee1e..475faee4d3fde 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -256,6 +256,8 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
 
   OptPrimType classify(const Expr *E) const { return Ctx.classify(E); }
   OptPrimType classify(QualType Ty) const { return Ctx.classify(Ty); }
+  bool canClassify(const Expr *E) const { return Ctx.canClassify(E); }
+  bool canClassify(QualType T) const { return Ctx.canClassify(T); }
 
   /// Classifies a known primitive type.
   PrimType classifyPrim(QualType Ty) const {
@@ -280,6 +282,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   /// been created. visitInitializer() then relies on a pointer to this
   /// variable being on top of the stack.
   bool visitInitializer(const Expr *E);
+  bool visitAsLValue(const Expr *E);
   /// Evaluates an expression for side effects and discards the result.
   bool discard(const Expr *E);
   /// Just pass evaluation on to \p E. This leaves all the parsing flags
@@ -304,7 +307,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   bool visitArrayElemInit(unsigned ElemIndex, const Expr *Init,
                           OptPrimType InitT);
   bool visitCallArgs(ArrayRef<const Expr *> Args, const FunctionDecl *FuncDecl,
-                     bool Activate);
+                     bool Activate, bool IsOperatorCall);
 
   /// Creates a local primitive value.
   unsigned allocateLocalPrimitive(DeclTy &&Decl, PrimType Ty, bool IsConst,
@@ -313,11 +316,11 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
                                   bool IsConstexprUnknown = false);
 
   /// Allocates a space storing a local given its type.
-  std::optional<unsigned>
-  allocateLocal(DeclTy &&Decl, QualType Ty = QualType(),
-                const ValueDecl *ExtendingDecl = nullptr,
-                ScopeKind = ScopeKind::Block, bool IsConstexprUnknown = false);
-  std::optional<unsigned> allocateTemporary(const Expr *E);
+  UnsignedOrNone allocateLocal(DeclTy &&Decl, QualType Ty = QualType(),
+                               const ValueDecl *ExtendingDecl = nullptr,
+                               ScopeKind = ScopeKind::Block,
+                               bool IsConstexprUnknown = false);
+  UnsignedOrNone allocateTemporary(const Expr *E);
 
 private:
   friend class VariableScope<Emitter>;
@@ -344,9 +347,10 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
 
   /// Emits an APSInt constant.
   bool emitConst(const llvm::APSInt &Value, PrimType Ty, const Expr *E);
+  bool emitConst(const llvm::APInt &Value, PrimType Ty, const Expr *E);
   bool emitConst(const llvm::APSInt &Value, const Expr *E);
   bool emitConst(const llvm::APInt &Value, const Expr *E) {
-    return emitConst(static_cast<llvm::APSInt>(Value), E);
+    return emitConst(Value, classifyPrim(E), E);
   }
 
   /// Emits an integer constant.
@@ -424,6 +428,7 @@ class Compiler : public ConstStmtVisitor<Compiler<Emitter>, bool>,
   bool DiscardResult = false;
 
   bool InStmtExpr = false;
+  bool ToLValue = false;
 
   /// Flag inidicating if we're initializing an already created
   /// variable. This is set in visitInitializer().
@@ -563,7 +568,7 @@ template <class Emitter> class LocalScope : public VariableScope<Emitter> {
 
   void addLocal(const Scope::Local &Local) override {
     if (!Idx) {
-      Idx = this->Ctx->Descriptors.size();
+      Idx = static_cast<unsigned>(this->Ctx->Descriptors.size());
       this->Ctx->Descriptors.emplace_back();
       this->Ctx->emitInitScope(*Idx, {});
     }
@@ -611,7 +616,7 @@ template <class Emitter> class LocalScope : public VariableScope<Emitter> {
   }
 
   /// Index of the scope in the chain.
-  std::optional<unsigned> Idx;
+  UnsignedOrNone Idx = std::nullopt;
 };
 
 /// Scope for storage declared in a compound statement.
diff --git a/clang/lib/AST/ByteCode/Context.cpp b/clang/lib/AST/ByteCode/Context.cpp
index f7f528c4e6484..36eb7607e70bf 100644
--- a/clang/lib/AST/ByteCode/Context.cpp
+++ b/clang/lib/AST/ByteCode/Context.cpp
@@ -365,7 +365,7 @@ OptPrimType Context::classify(QualType T) const {
   }
 
   if (const auto *ET = T->getAs<EnumType>()) {
-    const auto *D = ET->getDecl();
+    const auto *D = ET->getOriginalDecl()->getDefinitionOrSelf();
     if (!D->isComplete())
       return std::nullopt;
     return classify(D->getIntegerType());
@@ -398,17 +398,11 @@ const llvm::fltSemantics &Context::getFloatSemantics(QualType T) const {
 }
 
 bool Context::Run(State &Parent, const Function *Func) {
-
-  {
-    InterpState State(Parent, *P, Stk, *this, Func);
-    if (Interpret(State)) {
-      assert(Stk.empty());
-      return true;
-    }
-    // State gets destroyed here, so the Stk.clear() below doesn't accidentally
-    // remove values the State's destructor might access.
+  InterpState State(Parent, *P, Stk, *this, Func);
+  if (Interpret(State)) {
+    assert(Stk.empty());
+    return true;
   }
-
   Stk.clear();
   return false;
 }
@@ -501,7 +495,7 @@ const Function *Context::getOrCreateFunction(const FunctionDecl *FuncDecl) {
   // elsewhere in the code.
   QualType Ty = FuncDecl->getReturnType();
   bool HasRVO = false;
-  if (!Ty->isVoidType() && !classify(Ty)) {
+  if (!Ty->isVoidType() && !canClassify(Ty)) {
     HasRVO = true;
     ParamTypes.push_back(PT_Ptr);
     ParamOffsets.push_back(ParamOffset);
diff --git a/clang/lib/AST/ByteCode/Context.h b/clang/lib/AST/ByteCode/Context.h
index 1c084acbe916b..fa98498dbe8fa 100644
--- a/clang/lib/AST/ByteCode/Context.h
+++ b/clang/lib/AST/ByteCode/Context.h
@@ -30,7 +30,7 @@ namespace interp {
 class Function;
 class Program;
 class State;
-enum PrimType : unsigned;
+enum PrimType : uint8_t;
 
 struct ParamOffset {
   unsigned Offset;
@@ -93,6 +93,25 @@ class Context final {
     return classify(E->getType());
   }
 
+  bool canClassify(QualType T) {
+    if (const auto *BT = dyn_cast<BuiltinType>(T)) {
+      if (BT->isInteger() || BT->isFloatingPoint())
+        return true;
+      if (BT->getKind() == BuiltinType::Bool)
+        return true;
+    }
+
+    if (T->isArrayType() || T->isRecordType() || T->isAnyComplexType() ||
+        T->isVectorType())
+      return false;
+    return classify(T) != std::nullopt;
+  }
+  bool canClassify(const Expr *E) {
+    if (E->isGLValue())
+      return true;
+    return canClassify(E->getType());
+  }
+
   const CXXMethodDecl *
   getOverridingFunction(const CXXRecordDecl *DynamicDecl,
                         const CXXRecordDecl *StaticDecl,
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 7403e90ce53db..9ecc7b673cf28 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -153,28 +153,6 @@ static void dtorArrayDesc(Block *B, std::byte *Ptr, const Descriptor *D) {
   }
 }
 
-static void moveArrayDesc(Block *B, std::byte *Src, std::byte *Dst,
-                          const Descriptor *D) {
-  const unsigned NumElems = D->getNumElems();
-  const unsigned ElemSize =
-      D->ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
-
-  unsigned ElemOffset = 0;
-  for (unsigned I = 0; I < NumElems; ++I, ElemOffset += ElemSize) {
-    auto *SrcPtr = Src + ElemOffset;
-    auto *DstPtr = Dst + ElemOffset;
-
-    auto *SrcDesc = reinterpret_cast<InlineDescriptor *>(SrcPtr);
-    auto *SrcElemLoc = reinterpret_cast<std::byte *>(SrcDesc + 1);
-    auto *DstDesc = reinterpret_cast<InlineDescriptor *>(DstPtr);
-    auto *DstElemLoc = reinterpret_cast<std::byte *>(DstDesc + 1);
-
-    *DstDesc = *SrcDesc;
-    if (auto Fn = D->ElemDesc->MoveFn)
-      Fn(B, SrcElemLoc, DstElemLoc, D->ElemDesc);
-  }
-}
-
 static void initField(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable,
                       bool IsVolatile, bool IsActive, bool IsUnionField,
                       bool InUnion, const Descriptor *D, unsigned FieldOffset) {
@@ -268,45 +246,6 @@ static void dtorRecord(Block *B, std::byte *Ptr, const Descriptor *D) {
     destroyBase(B, Ptr, F.Desc, F.Offset);
 }
 
-static void moveRecord(Block *B, std::byte *Src, std::byte *Dst,
-                       const Descriptor *D) {
-  assert(D);
-  assert(D->ElemRecord);
-
-  // FIXME: Code duplication.
-  for (const auto &F : D->ElemRecord->fields()) {
-    auto FieldOffset = F.Offset;
-    const auto *SrcDesc =
-        reinterpret_cast<const InlineDescriptor *>(Src + FieldOffset) - 1;
-    auto *DestDesc =
-        reinterpret_cast<InlineDescriptor *>(Dst + FieldOffset) - 1;
-    std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor));
-
-    if (auto Fn = F.Desc->MoveFn)
-      Fn(B, Src + FieldOffset, Dst + FieldOffset, F.Desc);
-  }
-
-  for (const auto &Base : D->ElemRecord->bases()) {
-    auto BaseOffset = Base.Offset;
-    const auto *SrcDesc =
-        reinterpret_cast<const InlineDescriptor *>(Src + BaseOffset) - 1;
-    auto *DestDesc = reinterpret_cast<InlineDescriptor *>(Dst + BaseOffset) - 1;
-    std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor));
-
-    if (auto Fn = Base.Desc->MoveFn)
-      Fn(B, Src + BaseOffset, Dst + BaseOffset, Base.Desc);
-  }
-
-  for (const auto &VBase : D->ElemRecord->virtual_bases()) {
-    auto VBaseOffset = VBase.Offset;
-    const auto *SrcDesc =
-        reinterpret_cast<const InlineDescriptor *>(Src + VBaseOffset) - 1;
-    auto *DestDesc =
-        reinterpret_cast<InlineDescriptor *>(Dst + VBaseOffset) - 1;
-    std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor));
-  }
-}
-
 static BlockCtorFn getCtorPrim(PrimType Type) {
   // Floating types are special. They are primitives, but need their
   // constructor called.
@@ -337,18 +276,6 @@ static BlockDtorFn getDtorPrim(PrimType Type) {
   COMPOSITE_TYPE_SWITCH(Type, return dtorTy<T>, return nullptr);
 }
 
-static BlockMoveFn getMovePrim(PrimType Type) {
-  if (Type == PT_Float)
-    return moveTy<PrimConv<PT_Float>::T>;
-  if (Type == PT_IntAP)
-    return moveTy<PrimConv<PT_IntAP>::T>;
-  if (Type == PT_IntAPS)
-    return moveTy<PrimConv<PT_IntAPS>::T>;
-  if (Type == PT_MemberPtr)
-    return moveTy<PrimConv<PT_MemberPtr>::T>;
-  COMPOSITE_TYPE_SWITCH(Type, return moveTy<T>, return nullptr);
-}
-
 static BlockCtorFn getCtorArrayPrim(PrimType Type) {
   TYPE_SWITCH(Type, return ctorArrayTy<T>);
   llvm_unreachable("unknown Expr");
@@ -359,11 +286,6 @@ static BlockDtorFn getDtorArrayPrim(PrimType Type) {
   llvm_unreachable("unknown Expr");
 }
 
-static BlockMoveFn getMoveArrayPrim(PrimType Type) {
-  TYPE_SWITCH(Type, return moveArrayTy<T>);
-  llvm_unreachable("unknown Expr");
-}
-
 /// Primitives.
 Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type,
                        MetadataSize MD, bool IsConst, bool IsTemporary,
@@ -372,7 +294,7 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type,
       MDSize(MD.value_or(0)), AllocSize(align(Size + MDSize)), PrimT(Type),
       IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary),
       IsVolatile(IsVolatile), CtorFn(getCtorPrim(Type)),
-      DtorFn(getDtorPrim(Type)), MoveFn(getMovePrim(Type)) {
+      DtorFn(getDtorPrim(Type)) {
   assert(AllocSize >= Size);
   assert(Source && "Missing source");
 }
@@ -386,7 +308,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
       AllocSize(align(MDSize) + align(Size) + sizeof(InitMapPtr)), PrimT(Type),
       IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary),
       IsArray(true), CtorFn(getCtorArrayPrim(Type)),
-      DtorFn(getDtorArrayPrim(Type)), MoveFn(getMoveArrayPrim(Type)) {
+      DtorFn(getDtorArrayPrim(Type)) {
   assert(Source && "Missing source");
   assert(NumElems <= (MaxArrayElemBytes / ElemSize));
 }
@@ -399,7 +321,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
       AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), PrimT(Type),
       IsConst(IsConst), IsMutable(false), IsTemporary(IsTemporary),
       IsArray(true), CtorFn(getCtorArrayPrim(Type)),
-      DtorFn(getDtorArrayPrim(Type)), MoveFn(getMoveArrayPrim(Type)) {
+      DtorFn(getDtorArrayPrim(Type)) {
   assert(Source && "Missing source");
 }
 
@@ -414,7 +336,7 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy,
       AllocSize(std::max<size_t>(alignof(void *), Size) + MDSize),
       ElemDesc(Elem), IsConst(IsConst), IsMutable(IsMutable),
       IsTemporary(IsTemporary), IsArray(true), CtorFn(ctorArrayDesc),
-      DtorFn(dtorArrayDesc), MoveFn(moveArrayDesc) {
+      DtorFn(dtorArrayDesc) {
   assert(Source && "Missing source");
 }
 
@@ -425,7 +347,7 @@ Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD,
       Size(UnknownSizeMark), MDSize(MD.value_or(0)),
       AllocSize(MDSize + alignof(void *)), ElemDesc(Elem), IsConst(true),
       IsMutable(false), IsTemporary(IsTemporary), IsArray(true),
-      CtorFn(ctorArrayDesc), DtorFn(dtorArrayDesc), MoveFn(moveArrayDesc) {
+      CtorFn(ctorArrayDesc), DtorFn(dtorArrayDesc) {
   assert(Source && "Missing source");
 }
 
@@ -437,7 +359,7 @@ Descriptor::Descriptor(const DeclTy &D, const Record *R, MetadataSize MD,
       Size(ElemSize), MDSize(MD.value_or(0)), AllocSize(Size + MDSize),
       ElemRecord(R), IsConst(IsConst), IsMutable(IsMutable),
       IsTemporary(IsTemporary), IsVolatile(IsVolatile), CtorFn(ctorRecord),
-      DtorFn(dtorRecord), MoveFn(moveRecord) {
+      DtorFn(dtorRecord) {
   assert(Source && "Missing source");
 }
 
@@ -445,7 +367,7 @@ Descriptor::Descriptor(const DeclTy &D, const Record *R, MetadataSize MD,
 Descriptor::Descriptor(const DeclTy &D, MetadataSize MD)
     : Source(D), ElemSize(1), Size(1), MDSize(MD.value_or(0)),
       AllocSize(MDSize), ElemRecord(nullptr), IsConst(true), IsMutable(false),
-      IsTemporary(false), IsDummy(true) {
+      IsTemporary(false) {
   assert(Source && "Missing source");
 }
 
@@ -455,12 +377,14 @@ QualType Descriptor::getType() const {
   if (const auto *D = asValueDecl())
     return D->getType();
   if (const auto *T = dyn_cast_if_present<TypeDecl>(asDecl()))
-    return QualType(T->getTypeForDecl(), 0);
+    return T->getASTContext().getTypeDeclType(T);
 
   // The Source sometimes has a different type than the once
   // we really save. Try to consult the Record first.
-  if (isRecord())
-    return QualType(ElemRecord->getDecl()->getTypeForDecl(), 0);
+  if (isRecord()) {
+    const RecordDecl *RD = ElemRecord->getDecl();
+    return RD->getASTContext().getCanonicalTagType(RD);
+  }
   if (const auto *E = asExpr())
     return E->getType();
   llvm_unreachable("Invalid descriptor type");
@@ -531,7 +455,7 @@ SourceInfo Descriptor::getLoc() const {
 }
 
 bool Descriptor::hasTrivialDtor() const {
-  if (isPrimitive() || isPrimitiveArray() || isDummy())
+  if (isPrimitive() || isPrimitiveArray())
     return true;
 
   if (isRecord()) {
@@ -540,17 +464,16 @@ bool Descriptor::hasTrivialDtor() const {
     return !Dtor || Dtor->isTrivial();
   }
 
+  if (!ElemDesc)
+    return true;
   // Composite arrays.
-  assert(ElemDesc);
   return ElemDesc->hasTrivialDtor();
 }
 
 bool Descriptor::isUnion() const { return isRecord() && ElemRecord->isUnion(); }
 
 InitMap::InitMap(unsigned N)
-    : UninitFields(N), Data(std::make_unique<T[]>(numFields(N))) {
-  std::fill_n(data(), numFields(N), 0);
-}
+    : UninitFields(N), Data(std::make_unique<T[]>(numFields(N))) {}
 
 bool InitMap::initializeElement(unsigned I) {
   unsigned Bucket = I / PER_FIELD;
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index 4c925f6f0af1e..90dc2b4aa3111 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -24,7 +24,7 @@ class Record;
 class SourceInfo;
 struct InitMap;
 struct Descriptor;
-enum PrimType : unsigned;
+enum PrimType : uint8_t;
 
 using DeclTy = llvm::PointerUnion<const Decl *, const Expr *>;
 using InitMapPtr = std::optional<std::pair<bool, std::shared_ptr<InitMap>>>;
@@ -41,14 +41,6 @@ using BlockCtorFn = void (*)(Block *Storage, std::byte *FieldPtr, bool IsConst,
 using BlockDtorFn = void (*)(Block *Storage, std::byte *FieldPtr,
                              const Descriptor *FieldDesc);
 
-/// Invoked when a block with pointers referencing it goes out of scope. Such
-/// blocks are persisted: the move function copies all inline descriptors and
-/// non-trivial fields, as existing pointers might need to reference those
-/// descriptors. Data is not copied since it cannot be legally read.
-using BlockMoveFn = void (*)(Block *Storage, std::byte *SrcFieldPtr,
-                             std::byte *DstFieldPtr,
-                             const Descriptor *FieldDesc);
-
 enum class GlobalInitState {
   Initialized,
   NoInitializer,
@@ -174,14 +166,11 @@ struct Descriptor final {
   const bool IsVolatile = false;
   /// Flag indicating if the block is an array.
   const bool IsArray = false;
-  /// Flag indicating if this is a dummy descriptor.
-  bool IsDummy = false;
   bool IsConstexprUnknown = false;
 
   /// Storage management methods.
   const BlockCtorFn CtorFn = nullptr;
   const BlockDtorFn DtorFn = nullptr;
-  const BlockMoveFn MoveFn = nullptr;
 
   /// Allocates a descriptor for a primitive.
   Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type,
@@ -212,9 +201,6 @@ struct Descriptor final {
   /// Allocates a dummy descriptor.
   Descriptor(const DeclTy &D, MetadataSize MD = std::nullopt);
 
-  /// Make this descriptor a dummy descriptor.
-  void makeDummy() { IsDummy = true; }
-
   QualType getType() const;
   QualType getElemQualType() const;
   QualType getDataType(const ASTContext &Ctx) const;
@@ -282,8 +268,6 @@ struct Descriptor final {
   bool isRecord() const { return !IsArray && ElemRecord; }
   /// Checks if the descriptor is of a union.
   bool isUnion() const;
-  /// Checks if this is a dummy descriptor.
-  bool isDummy() const { return IsDummy; }
 
   /// Whether variables of this descriptor need their destructor called or not.
   bool hasTrivialDtor() const;
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index 5049a6545eef6..58e6d1a722afa 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -338,7 +338,7 @@ LLVM_DUMP_METHOD void Program::dump(llvm::raw_ostream &OS) const {
     }
 
     OS << "\n";
-    if (GP.isInitialized() && Desc->isPrimitive() && !Desc->isDummy()) {
+    if (GP.isInitialized() && Desc->isPrimitive() && !G->block()->isDummy()) {
       OS << "   ";
       {
         ColorScope SC(OS, true, {llvm::raw_ostream::BRIGHT_CYAN, false});
@@ -394,8 +394,6 @@ LLVM_DUMP_METHOD void Descriptor::dump(llvm::raw_ostream &OS) const {
   else if (isUnknownSizeArray())
     OS << " unknown-size-array";
 
-  if (isDummy())
-    OS << " dummy";
   if (IsConstexprUnknown)
     OS << " constexpr-unknown";
 }
@@ -541,12 +539,13 @@ LLVM_DUMP_METHOD void Block::dump(llvm::raw_ostream &OS) const {
   else
     OS << "-\n";
   OS << "  Pointers: " << NPointers << "\n";
-  OS << "  Dead: " << IsDead << "\n";
+  OS << "  Dead: " << isDead() << "\n";
   OS << "  Static: " << IsStatic << "\n";
-  OS << "  Extern: " << IsExtern << "\n";
+  OS << "  Extern: " << isExtern() << "\n";
   OS << "  Initialized: " << IsInitialized << "\n";
-  OS << "  Weak: " << IsWeak << "\n";
-  OS << "  Dynamic: " << IsDynamic << "\n";
+  OS << "  Weak: " << isWeak() << "\n";
+  OS << "  Dummy: " << isDummy() << '\n';
+  OS << "  Dynamic: " << isDynamic() << "\n";
 }
 
 LLVM_DUMP_METHOD void EvaluationResult::dump() const {
diff --git a/clang/lib/AST/ByteCode/DynamicAllocator.cpp b/clang/lib/AST/ByteCode/DynamicAllocator.cpp
index 9b8b664df6afd..4fedac667b389 100644
--- a/clang/lib/AST/ByteCode/DynamicAllocator.cpp
+++ b/clang/lib/AST/ByteCode/DynamicAllocator.cpp
@@ -13,25 +13,6 @@
 using namespace clang;
 using namespace clang::interp;
 
-// FIXME: There is a peculiar problem with the way we track pointers
-// to blocks and the way we allocate dynamic memory.
-//
-// When we have code like this:
-// while (true) {
-//   char *buffer = new char[1024];
-//   delete[] buffer;
-// }
-//
-// We have a local variable 'buffer' pointing to the heap allocated memory.
-// When deallocating the memory via delete[], that local variable still
-// points to the memory, which means we will create a DeadBlock for it and move
-// it over to that block, essentially duplicating the allocation. Moving
-// the data is also slow.
-//
-// However, when we actually try to access the allocation after it has been
-// freed, we need the block to still exist (alive or dead) so we can tell
-// that it's a dynamic allocation.
-
 DynamicAllocator::~DynamicAllocator() { cleanup(); }
 
 void DynamicAllocator::cleanup() {
@@ -42,12 +23,15 @@ void DynamicAllocator::cleanup() {
   for (auto &Iter : AllocationSites) {
     auto &AllocSite = Iter.second;
     for (auto &Alloc : AllocSite.Allocations) {
-      Block *B = reinterpret_cast<Block *>(Alloc.Memory.get());
+      Block *B = Alloc.block();
+      assert(!B->isDead());
+      assert(B->isInitialized());
       B->invokeDtor();
+
       if (B->hasPointers()) {
         while (B->Pointers) {
           Pointer *Next = B->Pointers->asBlockPointer().Next;
-          B->Pointers->PointeeStorage.BS.Pointee = nullptr;
+          B->Pointers->BS.Pointee = nullptr;
           B->Pointers = Next;
         }
         B->Pointers = nullptr;
@@ -89,6 +73,12 @@ Block *DynamicAllocator::allocate(const Descriptor *D, unsigned EvalID,
   assert(D);
   assert(D->asExpr());
 
+  // Garbage collection. Remove all dead allocations that don't have pointers to
+  // them anymore.
+  llvm::erase_if(DeadAllocations, [](Allocation &Alloc) -> bool {
+    return !Alloc.block()->hasPointers();
+  });
+
   auto Memory =
       std::make_unique<std::byte[]>(sizeof(Block) + D->getAllocSize());
   auto *B = new (Memory.get()) Block(EvalID, D, /*isStatic=*/false);
@@ -111,13 +101,17 @@ Block *DynamicAllocator::allocate(const Descriptor *D, unsigned EvalID,
     ID->LifeState =
         AllocForm == Form::Operator ? Lifetime::Ended : Lifetime::Started;
 
-  B->IsDynamic = true;
-
-  if (auto It = AllocationSites.find(D->asExpr()); It != AllocationSites.end())
+  if (auto It = AllocationSites.find(D->asExpr());
+      It != AllocationSites.end()) {
     It->second.Allocations.emplace_back(std::move(Memory));
-  else
+    B->setDynAllocId(It->second.NumAllocs);
+    ++It->second.NumAllocs;
+  } else {
     AllocationSites.insert(
         {D->asExpr(), AllocationSite(std::move(Memory), AllocForm)});
+    B->setDynAllocId(0);
+  }
+  assert(B->isDynamic());
   return B;
 }
 
@@ -128,23 +122,39 @@ bool DynamicAllocator::deallocate(const Expr *Source,
     return false;
 
   auto &Site = It->second;
-  assert(Site.size() > 0);
+  assert(!Site.empty());
 
   // Find the Block to delete.
-  auto AllocIt = llvm::find_if(Site.Allocations, [&](const Allocation &A) {
-    const Block *B = reinterpret_cast<const Block *>(A.Memory.get());
-    return BlockToDelete == B;
+  auto *AllocIt = llvm::find_if(Site.Allocations, [&](const Allocation &A) {
+    return BlockToDelete == A.block();
   });
 
   assert(AllocIt != Site.Allocations.end());
 
-  Block *B = reinterpret_cast<Block *>(AllocIt->Memory.get());
+  Block *B = AllocIt->block();
+  assert(B->isInitialized());
+  assert(!B->isDead());
   B->invokeDtor();
 
-  S.deallocate(B);
-  Site.Allocations.erase(AllocIt);
+  // Almost all our dynamic allocations have a pointer pointing to them
+  // when we deallocate them, since otherwise we can't call delete() at all.
+  // This means that we would usually need to create DeadBlocks for all of them.
+  // To work around that, we instead mark them as dead without moving the data
+  // over to a DeadBlock and simply keep the block in a separate DeadAllocations
+  // list.
+  if (B->hasPointers()) {
+    B->AccessFlags |= Block::DeadFlag;
+    DeadAllocations.push_back(std::move(*AllocIt));
+    Site.Allocations.erase(AllocIt);
+
+    if (Site.size() == 0)
+      AllocationSites.erase(It);
+    return true;
+  }
 
-  if (Site.size() == 0)
+  // Get rid of the allocation altogether.
+  Site.Allocations.erase(AllocIt);
+  if (Site.empty())
     AllocationSites.erase(It);
 
   return true;
diff --git a/clang/lib/AST/ByteCode/DynamicAllocator.h b/clang/lib/AST/ByteCode/DynamicAllocator.h
index cff09bf4f6a6e..ab1058bdf9f22 100644
--- a/clang/lib/AST/ByteCode/DynamicAllocator.h
+++ b/clang/lib/AST/ByteCode/DynamicAllocator.h
@@ -43,18 +43,22 @@ class DynamicAllocator final {
     std::unique_ptr<std::byte[]> Memory;
     Allocation(std::unique_ptr<std::byte[]> Memory)
         : Memory(std::move(Memory)) {}
+    Block *block() const { return reinterpret_cast<Block *>(Memory.get()); }
   };
 
   struct AllocationSite {
     llvm::SmallVector<Allocation> Allocations;
+    unsigned NumAllocs = 0;
     Form AllocForm;
 
     AllocationSite(std::unique_ptr<std::byte[]> Memory, Form AllocForm)
         : AllocForm(AllocForm) {
       Allocations.push_back({std::move(Memory)});
+      ++NumAllocs;
     }
 
     size_t size() const { return Allocations.size(); }
+    bool empty() const { return Allocations.empty(); }
   };
 
 public:
@@ -65,8 +69,6 @@ class DynamicAllocator final {
 
   void cleanup();
 
-  unsigned getNumAllocations() const { return AllocationSites.size(); }
-
   /// Allocate ONE element of the given descriptor.
   Block *allocate(const Descriptor *D, unsigned EvalID, Form AllocForm);
   /// Allocate \p NumElements primitive elements of the given type.
@@ -96,8 +98,13 @@ class DynamicAllocator final {
     return llvm::make_range(AllocationSites.begin(), AllocationSites.end());
   }
 
+  bool hasAllocations() const { return !AllocationSites.empty(); }
+
 private:
   llvm::DenseMap<const Expr *, AllocationSite> AllocationSites;
+  // Allocations that have already been deallocated but had pointers
+  // to them.
+  llvm::SmallVector<Allocation> DeadAllocations;
 
   using PoolAllocTy = llvm::BumpPtrAllocator;
   PoolAllocTy DescAllocator;
diff --git a/clang/lib/AST/ByteCode/EvalEmitter.cpp b/clang/lib/AST/ByteCode/EvalEmitter.cpp
index 976b7c051bd3a..1ebadae811bdf 100644
--- a/clang/lib/AST/ByteCode/EvalEmitter.cpp
+++ b/clang/lib/AST/ByteCode/EvalEmitter.cpp
@@ -98,10 +98,7 @@ bool EvalEmitter::interpretCall(const FunctionDecl *FD, const Expr *E) {
     this->Params.insert({PD, {0, false}});
   }
 
-  if (!this->visit(E))
-    return false;
-  PrimType T = Ctx.classify(E).value_or(PT_Ptr);
-  return this->emitPop(T, E);
+  return this->visitExpr(E, /*DestroyToplevelScope=*/false);
 }
 
 void EvalEmitter::emitLabel(LabelTy Label) { CurrentLabel = Label; }
@@ -216,10 +213,8 @@ template <> bool EvalEmitter::emitRet<PT_Ptr>(const SourceInfo &Info) {
     if (!Ptr.isZero() && !Ptr.isDereferencable())
       return false;
 
-    if (S.getLangOpts().CPlusPlus11 && Ptr.isBlockPointer() &&
-        !CheckFinalLoad(S, OpPC, Ptr)) {
+    if (!Ptr.isZero() && !CheckFinalLoad(S, OpPC, Ptr))
       return false;
-    }
 
     // Never allow reading from a non-const pointer, unless the memory
     // has been created in this evaluation.
@@ -292,7 +287,7 @@ bool EvalEmitter::emitGetLocal(uint32_t I, const SourceInfo &Info) {
 
   Block *B = getLocal(I);
 
-  if (!CheckLocalLoad(S, OpPC, Pointer(B)))
+  if (!CheckLocalLoad(S, OpPC, B))
     return false;
 
   S.Stk.push<T>(*reinterpret_cast<T *>(B->data()));
diff --git a/clang/lib/AST/ByteCode/Function.h b/clang/lib/AST/ByteCode/Function.h
index 92363b62c85d4..af429b7849e88 100644
--- a/clang/lib/AST/ByteCode/Function.h
+++ b/clang/lib/AST/ByteCode/Function.h
@@ -28,7 +28,7 @@ namespace interp {
 class Program;
 class ByteCodeEmitter;
 class Pointer;
-enum PrimType : uint32_t;
+enum PrimType : uint8_t;
 
 /// Describes a scope block.
 ///
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index eb4e4800d44ae..796c24e9071ec 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -211,25 +211,26 @@ static void diagnoseNonConstVariable(InterpState &S, CodePtr OpPC,
   S.Note(VD->getLocation(), diag::note_declared_at);
 }
 
-static bool CheckTemporary(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
+static bool CheckTemporary(InterpState &S, CodePtr OpPC, const Block *B,
                            AccessKinds AK) {
-  if (auto ID = Ptr.getDeclID()) {
-    if (!Ptr.isStaticTemporary())
+  if (B->getDeclID()) {
+    if (!(B->isStatic() && B->isTemporary()))
       return true;
 
     const auto *MTE = dyn_cast_if_present<MaterializeTemporaryExpr>(
-        Ptr.getDeclDesc()->asExpr());
+        B->getDescriptor()->asExpr());
     if (!MTE)
       return true;
 
     // FIXME(perf): Since we do this check on every Load from a static
     // temporary, it might make sense to cache the value of the
     // isUsableInConstantExpressions call.
-    if (!MTE->isUsableInConstantExpressions(S.getASTContext()) &&
-        Ptr.block()->getEvalID() != S.Ctx.getEvalID()) {
+    if (B->getEvalID() != S.Ctx.getEvalID() &&
+        !MTE->isUsableInConstantExpressions(S.getASTContext())) {
       const SourceInfo &E = S.Current->getSource(OpPC);
       S.FFDiag(E, diag::note_constexpr_access_static_temporary, 1) << AK;
-      S.Note(Ptr.getDeclLoc(), diag::note_constexpr_temporary_here);
+      S.Note(B->getDescriptor()->getLocation(),
+             diag::note_constexpr_temporary_here);
       return false;
     }
   }
@@ -517,7 +518,7 @@ bool CheckNull(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
 bool CheckRange(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                 AccessKinds AK) {
-  if (!Ptr.isOnePastEnd())
+  if (!Ptr.isOnePastEnd() && !Ptr.isZeroSizeArray())
     return true;
   if (S.getLangOpts().CPlusPlus) {
     const SourceInfo &Loc = S.Current->getSource(OpPC);
@@ -529,7 +530,7 @@ bool CheckRange(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
 bool CheckRange(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                 CheckSubobjectKind CSK) {
-  if (!Ptr.isElementPastEnd())
+  if (!Ptr.isElementPastEnd() && !Ptr.isZeroSizeArray())
     return true;
   const SourceInfo &Loc = S.Current->getSource(OpPC);
   S.FFDiag(Loc, diag::note_constexpr_past_end_subobject)
@@ -658,17 +659,19 @@ static bool CheckVolatile(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
   return false;
 }
 
-bool CheckInitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
-                      AccessKinds AK) {
+bool DiagnoseUninitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
+                           AccessKinds AK) {
   assert(Ptr.isLive());
+  assert(!Ptr.isInitialized());
+  return DiagnoseUninitialized(S, OpPC, Ptr.isExtern(), Ptr.getDeclDesc(), AK);
+}
 
-  if (Ptr.isInitialized())
-    return true;
-
-  if (Ptr.isExtern() && S.checkingPotentialConstantExpression())
+bool DiagnoseUninitialized(InterpState &S, CodePtr OpPC, bool Extern,
+                           const Descriptor *Desc, AccessKinds AK) {
+  if (Extern && S.checkingPotentialConstantExpression())
     return false;
 
-  if (const auto *VD = Ptr.getDeclDesc()->asVarDecl();
+  if (const auto *VD = Desc->asVarDecl();
       VD && (VD->isConstexpr() || VD->hasGlobalStorage())) {
 
     if (VD == S.EvaluatingDecl &&
@@ -703,9 +706,9 @@ bool CheckInitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
   return false;
 }
 
-static bool CheckLifetime(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
+static bool CheckLifetime(InterpState &S, CodePtr OpPC, Lifetime LT,
                           AccessKinds AK) {
-  if (Ptr.getLifetime() == Lifetime::Started)
+  if (LT == Lifetime::Started)
     return true;
 
   if (!S.checkingPotentialConstantExpression()) {
@@ -715,11 +718,11 @@ static bool CheckLifetime(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
   return false;
 }
 
-static bool CheckWeak(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
-  if (!Ptr.isWeak())
+static bool CheckWeak(InterpState &S, CodePtr OpPC, const Block *B) {
+  if (!B->isWeak())
     return true;
 
-  const auto *VD = Ptr.getDeclDesc()->asVarDecl();
+  const auto *VD = B->getDescriptor()->asVarDecl();
   assert(VD);
   S.FFDiag(S.Current->getLocation(OpPC), diag::note_constexpr_var_init_weak)
       << VD;
@@ -732,57 +735,100 @@ static bool CheckWeak(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
 // ones removed that are impossible on primitive global values.
 // For example, since those can't be members of structs, they also can't
 // be mutable.
-bool CheckGlobalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
-  if (!CheckExtern(S, OpPC, Ptr))
-    return false;
-  if (!CheckConstant(S, OpPC, Ptr))
-    return false;
-  if (!CheckDummy(S, OpPC, Ptr, AK_Read))
-    return false;
-  if (!CheckInitialized(S, OpPC, Ptr, AK_Read))
-    return false;
-  if (!CheckTemporary(S, OpPC, Ptr, AK_Read))
+bool CheckGlobalLoad(InterpState &S, CodePtr OpPC, const Block *B) {
+  const auto &Desc =
+      *reinterpret_cast<const GlobalInlineDescriptor *>(B->rawData());
+  if (!B->isAccessible()) {
+    if (!CheckExtern(S, OpPC, Pointer(const_cast<Block *>(B))))
+      return false;
+    if (!CheckDummy(S, OpPC, B, AK_Read))
+      return false;
+    return CheckWeak(S, OpPC, B);
+  }
+
+  if (!CheckConstant(S, OpPC, B->getDescriptor()))
     return false;
-  if (!CheckWeak(S, OpPC, Ptr))
+  if (Desc.InitState != GlobalInitState::Initialized)
+    return DiagnoseUninitialized(S, OpPC, B->isExtern(), B->getDescriptor(),
+                                 AK_Read);
+  if (!CheckTemporary(S, OpPC, B, AK_Read))
     return false;
-  if (!CheckVolatile(S, OpPC, Ptr, AK_Read))
+  if (B->getDescriptor()->IsVolatile) {
+    if (!S.getLangOpts().CPlusPlus)
+      return Invalid(S, OpPC);
+
+    const ValueDecl *D = B->getDescriptor()->asValueDecl();
+    S.FFDiag(S.Current->getLocation(OpPC),
+             diag::note_constexpr_access_volatile_obj, 1)
+        << AK_Read << 1 << D;
+    S.Note(D->getLocation(), diag::note_constexpr_volatile_here) << 1;
     return false;
+  }
   return true;
 }
 
 // Similarly, for local loads.
-bool CheckLocalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
-  if (!CheckLifetime(S, OpPC, Ptr, AK_Read))
-    return false;
-  if (!CheckInitialized(S, OpPC, Ptr, AK_Read))
-    return false;
-  if (!CheckVolatile(S, OpPC, Ptr, AK_Read))
+bool CheckLocalLoad(InterpState &S, CodePtr OpPC, const Block *B) {
+  assert(!B->isExtern());
+  const auto &Desc = *reinterpret_cast<const InlineDescriptor *>(B->rawData());
+  if (!CheckLifetime(S, OpPC, Desc.LifeState, AK_Read))
+    return false;
+  if (!Desc.IsInitialized)
+    return DiagnoseUninitialized(S, OpPC, /*Extern=*/false, B->getDescriptor(),
+                                 AK_Read);
+  if (B->getDescriptor()->IsVolatile) {
+    if (!S.getLangOpts().CPlusPlus)
+      return Invalid(S, OpPC);
+
+    const ValueDecl *D = B->getDescriptor()->asValueDecl();
+    S.FFDiag(S.Current->getLocation(OpPC),
+             diag::note_constexpr_access_volatile_obj, 1)
+        << AK_Read << 1 << D;
+    S.Note(D->getLocation(), diag::note_constexpr_volatile_here) << 1;
     return false;
+  }
   return true;
 }
 
 bool CheckLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                AccessKinds AK) {
-  if (!CheckLive(S, OpPC, Ptr, AK))
-    return false;
-  if (!CheckExtern(S, OpPC, Ptr))
+  if (!Ptr.isBlockPointer()) {
+    if (Ptr.isZero()) {
+      const auto &Src = S.Current->getSource(OpPC);
+
+      if (Ptr.isField())
+        S.FFDiag(Src, diag::note_constexpr_null_subobject) << CSK_Field;
+      else
+        S.FFDiag(Src, diag::note_constexpr_access_null) << AK;
+    }
     return false;
+  }
+
+  // Block pointers are the only ones we can actually read from.
+  if (!Ptr.block()->isAccessible()) {
+    if (!CheckLive(S, OpPC, Ptr, AK))
+      return false;
+    if (!CheckExtern(S, OpPC, Ptr))
+      return false;
+    if (!CheckDummy(S, OpPC, Ptr.block(), AK))
+      return false;
+    if (!CheckWeak(S, OpPC, Ptr.block()))
+      return false;
+  }
+
   if (!CheckConstant(S, OpPC, Ptr))
     return false;
-  if (!CheckDummy(S, OpPC, Ptr, AK))
-    return false;
   if (!CheckRange(S, OpPC, Ptr, AK))
     return false;
   if (!CheckActive(S, OpPC, Ptr, AK))
     return false;
-  if (!CheckLifetime(S, OpPC, Ptr, AK))
+  if (!CheckLifetime(S, OpPC, Ptr.getLifetime(), AK))
     return false;
-  if (!CheckInitialized(S, OpPC, Ptr, AK))
-    return false;
-  if (!CheckTemporary(S, OpPC, Ptr, AK))
-    return false;
-  if (!CheckWeak(S, OpPC, Ptr))
+  if (!Ptr.isInitialized())
+    return DiagnoseUninitialized(S, OpPC, Ptr, AK);
+  if (!CheckTemporary(S, OpPC, Ptr.block(), AK))
     return false;
+
   if (!CheckMutable(S, OpPC, Ptr))
     return false;
   if (!CheckVolatile(S, OpPC, Ptr, AK))
@@ -793,26 +839,30 @@ bool CheckLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 /// This is not used by any of the opcodes directly. It's used by
 /// EvalEmitter to do the final lvalue-to-rvalue conversion.
 bool CheckFinalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
-  if (!CheckLive(S, OpPC, Ptr, AK_Read))
+  assert(!Ptr.isZero());
+  if (!Ptr.isBlockPointer())
     return false;
+
+  if (!Ptr.block()->isAccessible()) {
+    if (!CheckLive(S, OpPC, Ptr, AK_Read))
+      return false;
+    if (!CheckExtern(S, OpPC, Ptr))
+      return false;
+    if (!CheckDummy(S, OpPC, Ptr.block(), AK_Read))
+      return false;
+    return CheckWeak(S, OpPC, Ptr.block());
+  }
+
   if (!CheckConstant(S, OpPC, Ptr))
     return false;
 
-  if (!CheckDummy(S, OpPC, Ptr, AK_Read))
-    return false;
-  if (!CheckExtern(S, OpPC, Ptr))
-    return false;
-  if (!CheckRange(S, OpPC, Ptr, AK_Read))
-    return false;
   if (!CheckActive(S, OpPC, Ptr, AK_Read))
     return false;
-  if (!CheckLifetime(S, OpPC, Ptr, AK_Read))
-    return false;
-  if (!CheckInitialized(S, OpPC, Ptr, AK_Read))
-    return false;
-  if (!CheckTemporary(S, OpPC, Ptr, AK_Read))
+  if (!CheckLifetime(S, OpPC, Ptr.getLifetime(), AK_Read))
     return false;
-  if (!CheckWeak(S, OpPC, Ptr))
+  if (!Ptr.isInitialized())
+    return DiagnoseUninitialized(S, OpPC, Ptr, AK_Read);
+  if (!CheckTemporary(S, OpPC, Ptr.block(), AK_Read))
     return false;
   if (!CheckMutable(S, OpPC, Ptr))
     return false;
@@ -820,13 +870,17 @@ bool CheckFinalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
 }
 
 bool CheckStore(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
-  if (!CheckLive(S, OpPC, Ptr, AK_Assign))
-    return false;
-  if (!CheckDummy(S, OpPC, Ptr, AK_Assign))
-    return false;
-  if (!CheckLifetime(S, OpPC, Ptr, AK_Assign))
+  if (!Ptr.isBlockPointer())
     return false;
-  if (!CheckExtern(S, OpPC, Ptr))
+
+  if (!Ptr.block()->isAccessible()) {
+    if (!CheckLive(S, OpPC, Ptr, AK_Assign))
+      return false;
+    if (!CheckExtern(S, OpPC, Ptr))
+      return false;
+    return CheckDummy(S, OpPC, Ptr.block(), AK_Assign);
+  }
+  if (!CheckLifetime(S, OpPC, Ptr.getLifetime(), AK_Assign))
     return false;
   if (!CheckRange(S, OpPC, Ptr, AK_Assign))
     return false;
@@ -1098,13 +1152,11 @@ bool CheckDeclRef(InterpState &S, CodePtr OpPC, const DeclRefExpr *DR) {
   return diagnoseUnknownDecl(S, OpPC, D);
 }
 
-bool CheckDummy(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
-                AccessKinds AK) {
-  if (!Ptr.isDummy())
+bool CheckDummy(InterpState &S, CodePtr OpPC, const Block *B, AccessKinds AK) {
+  if (!B->isDummy())
     return true;
 
-  const Descriptor *Desc = Ptr.getDeclDesc();
-  const ValueDecl *D = Desc->asValueDecl();
+  const ValueDecl *D = B->getDescriptor()->asValueDecl();
   if (!D)
     return false;
 
@@ -1251,7 +1303,7 @@ bool Free(InterpState &S, CodePtr OpPC, bool DeleteIsArrayForm,
       return false;
     }
 
-    if (!Ptr.isRoot() || Ptr.isOnePastEnd() ||
+    if (!Ptr.isRoot() || (Ptr.isOnePastEnd() && !Ptr.isZeroSizeArray()) ||
         (Ptr.isArrayElement() && Ptr.getIndex() != 0)) {
       const SourceInfo &Loc = S.Current->getSource(OpPC);
       S.FFDiag(Loc, diag::note_constexpr_delete_subobject)
@@ -1426,7 +1478,7 @@ static bool checkConstructor(InterpState &S, CodePtr OpPC, const Function *Func,
 bool CheckDestructor(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
   if (!CheckLive(S, OpPC, Ptr, AK_Destroy))
     return false;
-  if (!CheckTemporary(S, OpPC, Ptr, AK_Destroy))
+  if (!CheckTemporary(S, OpPC, Ptr.block(), AK_Destroy))
     return false;
   if (!CheckRange(S, OpPC, Ptr, AK_Destroy))
     return false;
@@ -1620,8 +1672,17 @@ bool CallVirt(InterpState &S, CodePtr OpPC, const Function *Func,
 
   const auto *StaticDecl = cast<CXXRecordDecl>(Func->getParentDecl());
   const auto *InitialFunction = cast<CXXMethodDecl>(Callee);
-  const CXXMethodDecl *Overrider = S.getContext().getOverridingFunction(
-      DynamicDecl, StaticDecl, InitialFunction);
+  const CXXMethodDecl *Overrider;
+
+  if (StaticDecl != DynamicDecl) {
+    if (!DynamicDecl->isDerivedFrom(StaticDecl))
+      return false;
+    Overrider = S.getContext().getOverridingFunction(DynamicDecl, StaticDecl,
+                                                     InitialFunction);
+
+  } else {
+    Overrider = InitialFunction;
+  }
 
   if (Overrider != InitialFunction) {
     // DR1872: An instantiated virtual constexpr function can't be called in a
@@ -1749,7 +1810,7 @@ static void startLifetimeRecurse(const Pointer &Ptr) {
 
 bool StartLifetime(InterpState &S, CodePtr OpPC) {
   const auto &Ptr = S.Stk.peek<Pointer>();
-  if (!CheckDummy(S, OpPC, Ptr, AK_Destroy))
+  if (Ptr.isBlockPointer() && !CheckDummy(S, OpPC, Ptr.block(), AK_Destroy))
     return false;
   startLifetimeRecurse(Ptr.narrow());
   return true;
@@ -1780,8 +1841,13 @@ static void endLifetimeRecurse(const Pointer &Ptr) {
 /// Ends the lifetime of the peek'd pointer.
 bool EndLifetime(InterpState &S, CodePtr OpPC) {
   const auto &Ptr = S.Stk.peek<Pointer>();
-  if (!CheckDummy(S, OpPC, Ptr, AK_Destroy))
+  if (Ptr.isBlockPointer() && !CheckDummy(S, OpPC, Ptr.block(), AK_Destroy))
     return false;
+
+  // FIXME: We need per-element lifetime information for primitive arrays.
+  if (Ptr.isArrayElement())
+    return true;
+
   endLifetimeRecurse(Ptr.narrow());
   return true;
 }
@@ -1789,8 +1855,13 @@ bool EndLifetime(InterpState &S, CodePtr OpPC) {
 /// Ends the lifetime of the pop'd pointer.
 bool EndLifetimePop(InterpState &S, CodePtr OpPC) {
   const auto &Ptr = S.Stk.pop<Pointer>();
-  if (!CheckDummy(S, OpPC, Ptr, AK_Destroy))
+  if (Ptr.isBlockPointer() && !CheckDummy(S, OpPC, Ptr.block(), AK_Destroy))
     return false;
+
+  // FIXME: We need per-element lifetime information for primitive arrays.
+  if (Ptr.isArrayElement())
+    return true;
+
   endLifetimeRecurse(Ptr.narrow());
   return true;
 }
@@ -1802,26 +1873,32 @@ bool CheckNewTypeMismatch(InterpState &S, CodePtr OpPC, const Expr *E,
   if (Ptr.inUnion() && Ptr.getBase().getRecord()->isUnion())
     Ptr.activate();
 
+  if (!Ptr.isBlockPointer())
+    return false;
+
   // Similar to CheckStore(), but with the additional CheckTemporary() call and
   // the AccessKinds are different.
-  if (!CheckTemporary(S, OpPC, Ptr, AK_Construct))
-    return false;
-  if (!CheckLive(S, OpPC, Ptr, AK_Construct))
-    return false;
-  if (!CheckDummy(S, OpPC, Ptr, AK_Construct))
+
+  if (!Ptr.block()->isAccessible()) {
+    if (!CheckExtern(S, OpPC, Ptr))
+      return false;
+    if (!CheckLive(S, OpPC, Ptr, AK_Construct))
+      return false;
+    return CheckDummy(S, OpPC, Ptr.block(), AK_Construct);
+  }
+  if (!CheckTemporary(S, OpPC, Ptr.block(), AK_Construct))
     return false;
 
   // CheckLifetime for this and all base pointers.
   for (Pointer P = Ptr;;) {
-    if (!CheckLifetime(S, OpPC, P, AK_Construct))
+    if (!CheckLifetime(S, OpPC, P.getLifetime(), AK_Construct))
       return false;
 
     if (P.isRoot())
       break;
     P = P.getBase();
   }
-  if (!CheckExtern(S, OpPC, Ptr))
-    return false;
+
   if (!CheckRange(S, OpPC, Ptr, AK_Construct))
     return false;
   if (!CheckGlobal(S, OpPC, Ptr))
@@ -2011,7 +2088,7 @@ bool GetTypeidPtr(InterpState &S, CodePtr OpPC, const Type *TypeInfoType) {
     return false;
 
   // Pick the most-derived type.
-  const Type *T = P.getDeclPtr().getType().getTypePtr();
+  CanQualType T = P.getDeclPtr().getType()->getCanonicalTypeUnqualified();
   // ... unless we're currently constructing this object.
   // FIXME: We have a similar check to this in more places.
   if (S.Current->getFunction()) {
@@ -2019,14 +2096,14 @@ bool GetTypeidPtr(InterpState &S, CodePtr OpPC, const Type *TypeInfoType) {
       if (const Function *Func = Frame->getFunction();
           Func && (Func->isConstructor() || Func->isDestructor()) &&
           P.block() == Frame->getThis().block()) {
-        T = Func->getParentDecl()->getTypeForDecl();
+        T = S.getContext().getASTContext().getCanonicalTagType(
+            Func->getParentDecl());
         break;
       }
     }
   }
 
-  S.Stk.push<Pointer>(T->getCanonicalTypeUnqualified().getTypePtr(),
-                      TypeInfoType);
+  S.Stk.push<Pointer>(T->getTypePtr(), TypeInfoType);
   return true;
 }
 
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 8a28106c0583a..92e60b6b88e6a 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -51,8 +51,7 @@ bool CheckLive(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                AccessKinds AK);
 
 /// Checks if a pointer is a dummy pointer.
-bool CheckDummy(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
-                AccessKinds AK);
+bool CheckDummy(InterpState &S, CodePtr OpPC, const Block *B, AccessKinds AK);
 
 /// Checks if a pointer is null.
 bool CheckNull(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
@@ -89,11 +88,14 @@ bool CheckLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                AccessKinds AK = AK_Read);
 bool CheckFinalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr);
 
-bool CheckInitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
-                      AccessKinds AK);
+bool DiagnoseUninitialized(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
+                           AccessKinds AK);
+bool DiagnoseUninitialized(InterpState &S, CodePtr OpPC, bool Extern,
+                           const Descriptor *Desc, AccessKinds AK);
+
 /// Checks a direct load of a primitive value from a global or local variable.
-bool CheckGlobalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr);
-bool CheckLocalLoad(InterpState &S, CodePtr OpPC, const Pointer &Ptr);
+bool CheckGlobalLoad(InterpState &S, CodePtr OpPC, const Block *B);
+bool CheckLocalLoad(InterpState &S, CodePtr OpPC, const Block *B);
 
 /// Checks if a value can be stored in a block.
 bool CheckStore(InterpState &S, CodePtr OpPC, const Pointer &Ptr);
@@ -1351,10 +1353,10 @@ inline bool ConstFloat(InterpState &S, CodePtr OpPC, const Floating &F) {
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool GetLocal(InterpState &S, CodePtr OpPC, uint32_t I) {
-  const Pointer &Ptr = S.Current->getLocalPointer(I);
-  if (!CheckLocalLoad(S, OpPC, Ptr))
+  const Block *B = S.Current->getLocalBlock(I);
+  if (!CheckLocalLoad(S, OpPC, B))
     return false;
-  S.Stk.push<T>(Ptr.deref<T>());
+  S.Stk.push<T>(B->deref<T>());
   return true;
 }
 
@@ -1465,22 +1467,26 @@ bool SetThisField(InterpState &S, CodePtr OpPC, uint32_t I) {
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool GetGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
-  const Pointer &Ptr = S.P.getPtrGlobal(I);
+  const Block *B = S.P.getGlobal(I);
 
-  if (!CheckGlobalLoad(S, OpPC, Ptr))
+  if (!CheckGlobalLoad(S, OpPC, B))
     return false;
 
-  S.Stk.push<T>(Ptr.deref<T>());
+  S.Stk.push<T>(B->deref<T>());
   return true;
 }
 
 /// Same as GetGlobal, but without the checks.
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool GetGlobalUnchecked(InterpState &S, CodePtr OpPC, uint32_t I) {
-  const Pointer &Ptr = S.P.getPtrGlobal(I);
-  if (!CheckInitialized(S, OpPC, Ptr, AK_Read))
-    return false;
-  S.Stk.push<T>(Ptr.deref<T>());
+  const Block *B = S.P.getGlobal(I);
+  const auto &Desc =
+      *reinterpret_cast<const GlobalInlineDescriptor *>(B->rawData());
+  if (Desc.InitState != GlobalInitState::Initialized)
+    return DiagnoseUninitialized(S, OpPC, B->isExtern(), B->getDescriptor(),
+                                 AK_Read);
+
+  S.Stk.push<T>(B->deref<T>());
   return true;
 }
 
@@ -1634,6 +1640,9 @@ bool InitField(InterpState &S, CodePtr OpPC, uint32_t I) {
   const Pointer &Ptr = S.Stk.peek<Pointer>();
   if (!CheckRange(S, OpPC, Ptr, CSK_Field))
     return false;
+  if (!CheckArray(S, OpPC, Ptr))
+    return false;
+
   const Pointer &Field = Ptr.atField(I);
   Field.deref<T>() = Value;
   Field.initialize();
@@ -1646,6 +1655,9 @@ bool InitFieldActivate(InterpState &S, CodePtr OpPC, uint32_t I) {
   const Pointer &Ptr = S.Stk.peek<Pointer>();
   if (!CheckRange(S, OpPC, Ptr, CSK_Field))
     return false;
+  if (!CheckArray(S, OpPC, Ptr))
+    return false;
+
   const Pointer &Field = Ptr.atField(I);
   Field.deref<T>() = Value;
   Field.activate();
@@ -1657,7 +1669,13 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool InitBitField(InterpState &S, CodePtr OpPC, const Record::Field *F) {
   assert(F->isBitField());
   const T &Value = S.Stk.pop<T>();
-  const Pointer &Field = S.Stk.peek<Pointer>().atField(F->Offset);
+  const Pointer &Ptr = S.Stk.peek<Pointer>();
+  if (!CheckRange(S, OpPC, Ptr, CSK_Field))
+    return false;
+  if (!CheckArray(S, OpPC, Ptr))
+    return false;
+
+  const Pointer &Field = Ptr.atField(F->Offset);
 
   if constexpr (needsAlloc<T>()) {
     T Result = S.allocAP<T>(Value.bitWidth());
@@ -1683,7 +1701,13 @@ bool InitBitFieldActivate(InterpState &S, CodePtr OpPC,
                           const Record::Field *F) {
   assert(F->isBitField());
   const T &Value = S.Stk.pop<T>();
-  const Pointer &Field = S.Stk.peek<Pointer>().atField(F->Offset);
+  const Pointer &Ptr = S.Stk.peek<Pointer>();
+  if (!CheckRange(S, OpPC, Ptr, CSK_Field))
+    return false;
+  if (!CheckArray(S, OpPC, Ptr))
+    return false;
+
+  const Pointer &Field = Ptr.atField(F->Offset);
 
   if constexpr (needsAlloc<T>()) {
     T Result = S.allocAP<T>(Value.bitWidth());
@@ -1764,10 +1788,7 @@ inline bool GetPtrDerivedPop(InterpState &S, CodePtr OpPC, uint32_t Off,
   const Record *TargetRecord = Ptr.atFieldSub(Off).getRecord();
   assert(TargetRecord);
 
-  if (TargetRecord->getDecl()
-          ->getTypeForDecl()
-          ->getAsCXXRecordDecl()
-          ->getCanonicalDecl() !=
+  if (TargetRecord->getDecl()->getCanonicalDecl() !=
       TargetType->getAsCXXRecordDecl()->getCanonicalDecl()) {
     QualType MostDerivedType = Ptr.getDeclDesc()->getType();
     S.CCEDiag(S.Current->getSource(OpPC), diag::note_constexpr_invalid_downcast)
@@ -1785,6 +1806,8 @@ inline bool GetPtrBase(InterpState &S, CodePtr OpPC, uint32_t Off) {
     return false;
 
   if (!Ptr.isBlockPointer()) {
+    if (!Ptr.isIntegralPointer())
+      return false;
     S.Stk.push<Pointer>(Ptr.asIntPointer().baseCast(S.getASTContext(), Off));
     return true;
   }
@@ -1806,6 +1829,8 @@ inline bool GetPtrBasePop(InterpState &S, CodePtr OpPC, uint32_t Off,
     return false;
 
   if (!Ptr.isBlockPointer()) {
+    if (!Ptr.isIntegralPointer())
+      return false;
     S.Stk.push<Pointer>(Ptr.asIntPointer().baseCast(S.getASTContext(), Off));
     return true;
   }
@@ -2351,8 +2376,8 @@ static inline bool IncDecPtrHelper(InterpState &S, CodePtr OpPC,
 static inline bool IncPtr(InterpState &S, CodePtr OpPC) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
 
-  if (!CheckInitialized(S, OpPC, Ptr, AK_Increment))
-    return false;
+  if (!Ptr.isInitialized())
+    return DiagnoseUninitialized(S, OpPC, Ptr, AK_Increment);
 
   return IncDecPtrHelper<ArithOp::Add>(S, OpPC, Ptr);
 }
@@ -2360,8 +2385,8 @@ static inline bool IncPtr(InterpState &S, CodePtr OpPC) {
 static inline bool DecPtr(InterpState &S, CodePtr OpPC) {
   const Pointer &Ptr = S.Stk.pop<Pointer>();
 
-  if (!CheckInitialized(S, OpPC, Ptr, AK_Decrement))
-    return false;
+  if (!Ptr.isInitialized())
+    return DiagnoseUninitialized(S, OpPC, Ptr, AK_Decrement);
 
   return IncDecPtrHelper<ArithOp::Sub>(S, OpPC, Ptr);
 }
@@ -2434,9 +2459,17 @@ inline bool Destroy(InterpState &S, CodePtr OpPC, uint32_t I) {
     const Pointer &Ptr = S.Current->getLocalPointer(Local.Offset);
 
     if (Ptr.getLifetime() == Lifetime::Ended) {
-      auto *D = cast<NamedDecl>(Ptr.getFieldDesc()->asDecl());
-      S.FFDiag(D->getLocation(), diag::note_constexpr_destroy_out_of_lifetime)
-          << D->getNameAsString();
+      // Try to use the declaration for better diagnostics
+      if (const Decl *D = Ptr.getDeclDesc()->asDecl()) {
+        auto *ND = cast<NamedDecl>(D);
+        S.FFDiag(ND->getLocation(),
+                 diag::note_constexpr_destroy_out_of_lifetime)
+            << ND->getNameAsString();
+      } else {
+        S.FFDiag(Ptr.getDeclDesc()->getLocation(),
+                 diag::note_constexpr_destroy_out_of_lifetime)
+            << Ptr.toDiagnosticString(S.getASTContext());
+      }
       return false;
     }
   }
@@ -3155,8 +3188,10 @@ inline bool ArrayDecay(InterpState &S, CodePtr OpPC) {
     return true;
   }
 
-  if (!CheckRange(S, OpPC, Ptr, CSK_ArrayToPointer))
-    return false;
+  if (!Ptr.isZeroSizeArray()) {
+    if (!CheckRange(S, OpPC, Ptr, CSK_ArrayToPointer))
+      return false;
+  }
 
   if (Ptr.isRoot() || !Ptr.isUnknownSizeArray()) {
     S.Stk.push<Pointer>(Ptr.atIndex(0));
@@ -3195,6 +3230,9 @@ inline bool GetMemberPtr(InterpState &S, CodePtr OpPC, const ValueDecl *D) {
 inline bool GetMemberPtrBase(InterpState &S, CodePtr OpPC) {
   const auto &MP = S.Stk.pop<MemberPointer>();
 
+  if (!MP.isBaseCastPossible())
+    return false;
+
   S.Stk.push<Pointer>(MP.getBase());
   return true;
 }
diff --git a/clang/lib/AST/ByteCode/InterpBlock.cpp b/clang/lib/AST/ByteCode/InterpBlock.cpp
index 963b54ec50cff..ac6f01f3cdca1 100644
--- a/clang/lib/AST/ByteCode/InterpBlock.cpp
+++ b/clang/lib/AST/ByteCode/InterpBlock.cpp
@@ -18,18 +18,14 @@ using namespace clang::interp;
 
 void Block::addPointer(Pointer *P) {
   assert(P);
-  if (IsStatic) {
-    assert(!Pointers);
-    return;
-  }
 
 #ifndef NDEBUG
   assert(!hasPointer(P));
 #endif
   if (Pointers)
-    Pointers->PointeeStorage.BS.Prev = P;
-  P->PointeeStorage.BS.Next = Pointers;
-  P->PointeeStorage.BS.Prev = nullptr;
+    Pointers->BS.Prev = P;
+  P->BS.Next = Pointers;
+  P->BS.Prev = nullptr;
   Pointers = P;
 #ifndef NDEBUG
   assert(hasPointer(P));
@@ -39,32 +35,28 @@ void Block::addPointer(Pointer *P) {
 void Block::removePointer(Pointer *P) {
   assert(P->isBlockPointer());
   assert(P);
-  if (IsStatic) {
-    assert(!Pointers);
-    return;
-  }
 
 #ifndef NDEBUG
   assert(hasPointer(P));
 #endif
 
-  BlockPointer &BP = P->PointeeStorage.BS;
+  BlockPointer &BP = P->BS;
 
   if (Pointers == P)
     Pointers = BP.Next;
 
   if (BP.Prev)
-    BP.Prev->PointeeStorage.BS.Next = BP.Next;
+    BP.Prev->BS.Next = BP.Next;
   if (BP.Next)
-    BP.Next->PointeeStorage.BS.Prev = BP.Prev;
-  P->PointeeStorage.BS.Pointee = nullptr;
+    BP.Next->BS.Prev = BP.Prev;
+  P->BS.Pointee = nullptr;
 #ifndef NDEBUG
   assert(!hasPointer(P));
 #endif
 }
 
 void Block::cleanup() {
-  if (Pointers == nullptr && IsDead)
+  if (Pointers == nullptr && !isDynamic() && isDead())
     (reinterpret_cast<DeadBlock *>(this + 1) - 1)->free();
 }
 
@@ -74,21 +66,17 @@ void Block::replacePointer(Pointer *Old, Pointer *New) {
   assert(New);
   assert(New->isBlockPointer());
   assert(Old != New);
-  if (IsStatic) {
-    assert(!Pointers);
-    return;
-  }
 #ifndef NDEBUG
   assert(hasPointer(Old));
 #endif
 
-  BlockPointer &OldBP = Old->PointeeStorage.BS;
-  BlockPointer &NewBP = New->PointeeStorage.BS;
+  BlockPointer &OldBP = Old->BS;
+  BlockPointer &NewBP = New->BS;
 
   if (OldBP.Prev)
-    OldBP.Prev->PointeeStorage.BS.Next = New;
+    OldBP.Prev->BS.Next = New;
   if (OldBP.Next)
-    OldBP.Next->PointeeStorage.BS.Prev = New;
+    OldBP.Next->BS.Prev = New;
   NewBP.Prev = OldBP.Prev;
   NewBP.Next = OldBP.Next;
   if (Pointers == Old)
@@ -113,8 +101,8 @@ bool Block::hasPointer(const Pointer *P) const {
 #endif
 
 DeadBlock::DeadBlock(DeadBlock *&Root, Block *Blk)
-    : Root(Root), B(~0u, Blk->Desc, Blk->IsStatic, Blk->IsExtern, Blk->IsWeak,
-                    /*isDead=*/true) {
+    : Root(Root), B(~0u, Blk->Desc, Blk->isExtern(), Blk->IsStatic,
+                    Blk->isWeak(), Blk->isDummy(), /*IsDead=*/true) {
   // Add the block to the chain of dead blocks.
   if (Root)
     Root->Prev = this;
@@ -123,18 +111,17 @@ DeadBlock::DeadBlock(DeadBlock *&Root, Block *Blk)
   Prev = nullptr;
   Root = this;
 
-  B.IsDynamic = Blk->IsDynamic;
+  B.DynAllocId = Blk->DynAllocId;
 
   // Transfer pointers.
   B.Pointers = Blk->Pointers;
   for (Pointer *P = Blk->Pointers; P; P = P->asBlockPointer().Next)
-    P->PointeeStorage.BS.Pointee = &B;
+    P->BS.Pointee = &B;
   Blk->Pointers = nullptr;
 }
 
 void DeadBlock::free() {
-  if (B.IsInitialized)
-    B.invokeDtor();
+  assert(!B.isInitialized());
 
   if (Prev)
     Prev->Next = Next;
diff --git a/clang/lib/AST/ByteCode/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h
index 51622238e275c..ea9f44c38842e 100644
--- a/clang/lib/AST/ByteCode/InterpBlock.h
+++ b/clang/lib/AST/ByteCode/InterpBlock.h
@@ -22,7 +22,7 @@ class Block;
 class DeadBlock;
 class InterpState;
 class Pointer;
-enum PrimType : unsigned;
+enum PrimType : uint8_t;
 
 /// A memory block, either on the stack or in the heap.
 ///
@@ -42,21 +42,31 @@ enum PrimType : unsigned;
 /// the data size and the metadata size.
 ///
 class Block final {
+private:
+  static constexpr uint8_t ExternFlag = 1 << 0;
+  static constexpr uint8_t DeadFlag = 1 << 1;
+  static constexpr uint8_t WeakFlag = 1 << 2;
+  static constexpr uint8_t DummyFlag = 1 << 3;
+
 public:
   /// Creates a new block.
-  Block(unsigned EvalID, const std::optional<unsigned> &DeclID,
-        const Descriptor *Desc, bool IsStatic = false, bool IsExtern = false,
-        bool IsWeak = false)
-      : EvalID(EvalID), DeclID(DeclID), IsStatic(IsStatic), IsExtern(IsExtern),
-        IsDynamic(false), IsWeak(IsWeak), Desc(Desc) {
+  Block(unsigned EvalID, UnsignedOrNone DeclID, const Descriptor *Desc,
+        bool IsStatic = false, bool IsExtern = false, bool IsWeak = false,
+        bool IsDummy = false)
+      : Desc(Desc), DeclID(DeclID), EvalID(EvalID), IsStatic(IsStatic) {
     assert(Desc);
+    AccessFlags |= (ExternFlag * IsExtern);
+    AccessFlags |= (WeakFlag * IsWeak);
+    AccessFlags |= (DummyFlag * IsDummy);
   }
 
   Block(unsigned EvalID, const Descriptor *Desc, bool IsStatic = false,
-        bool IsExtern = false, bool IsWeak = false)
-      : EvalID(EvalID), DeclID((unsigned)-1), IsStatic(IsStatic),
-        IsExtern(IsExtern), IsDynamic(false), IsWeak(IsWeak), Desc(Desc) {
+        bool IsExtern = false, bool IsWeak = false, bool IsDummy = false)
+      : Desc(Desc), EvalID(EvalID), IsStatic(IsStatic) {
     assert(Desc);
+    AccessFlags |= (ExternFlag * IsExtern);
+    AccessFlags |= (WeakFlag * IsWeak);
+    AccessFlags |= (DummyFlag * IsDummy);
   }
 
   /// Returns the block's descriptor.
@@ -64,17 +74,19 @@ class Block final {
   /// Checks if the block has any live pointers.
   bool hasPointers() const { return Pointers; }
   /// Checks if the block is extern.
-  bool isExtern() const { return IsExtern; }
+  bool isExtern() const { return AccessFlags & ExternFlag; }
   /// Checks if the block has static storage duration.
   bool isStatic() const { return IsStatic; }
   /// Checks if the block is temporary.
   bool isTemporary() const { return Desc->IsTemporary; }
-  bool isWeak() const { return IsWeak; }
-  bool isDynamic() const { return IsDynamic; }
+  bool isWeak() const { return AccessFlags & WeakFlag; }
+  bool isDynamic() const { return (DynAllocId != std::nullopt); }
+  bool isDummy() const { return AccessFlags & DummyFlag; }
+  bool isDead() const { return AccessFlags & DeadFlag; }
   /// Returns the size of the block.
   unsigned getSize() const { return Desc->getAllocSize(); }
   /// Returns the declaration ID.
-  std::optional<unsigned> getDeclID() const { return DeclID; }
+  UnsignedOrNone getDeclID() const { return DeclID; }
   /// Returns whether the data of this block has been initialized via
   /// invoking the Ctor func.
   bool isInitialized() const { return IsInitialized; }
@@ -103,6 +115,10 @@ class Block final {
     return reinterpret_cast<const std::byte *>(this) + sizeof(Block);
   }
 
+  template <typename T> T deref() const {
+    return *reinterpret_cast<const T *>(data());
+  }
+
   /// Invokes the constructor.
   void invokeCtor() {
     assert(!IsInitialized);
@@ -126,19 +142,28 @@ class Block final {
   void dump() const { dump(llvm::errs()); }
   void dump(llvm::raw_ostream &OS) const;
 
+  bool isAccessible() const { return AccessFlags == 0; }
+
 private:
   friend class Pointer;
   friend class DeadBlock;
   friend class InterpState;
   friend class DynamicAllocator;
+  friend class Program;
 
   Block(unsigned EvalID, const Descriptor *Desc, bool IsExtern, bool IsStatic,
-        bool IsWeak, bool IsDead)
-      : EvalID(EvalID), IsStatic(IsStatic), IsExtern(IsExtern), IsDead(true),
-        IsDynamic(false), IsWeak(IsWeak), Desc(Desc) {
+        bool IsWeak, bool IsDummy, bool IsDead)
+      : Desc(Desc), EvalID(EvalID), IsStatic(IsStatic) {
     assert(Desc);
+    AccessFlags |= (ExternFlag * IsExtern);
+    AccessFlags |= (DeadFlag * IsDead);
+    AccessFlags |= (WeakFlag * IsWeak);
+    AccessFlags |= (DummyFlag * IsDummy);
   }
 
+  /// To be called by DynamicAllocator.
+  void setDynAllocId(unsigned ID) { DynAllocId = ID; }
+
   /// Deletes a dead block at the end of its lifetime.
   void cleanup();
 
@@ -150,27 +175,22 @@ class Block final {
   bool hasPointer(const Pointer *P) const;
 #endif
 
-  const unsigned EvalID = ~0u;
+  /// Pointer to the stack slot descriptor.
+  const Descriptor *Desc;
   /// Start of the chain of pointers.
   Pointer *Pointers = nullptr;
   /// Unique identifier of the declaration.
-  std::optional<unsigned> DeclID;
+  UnsignedOrNone DeclID = std::nullopt;
+  const unsigned EvalID = ~0u;
   /// Flag indicating if the block has static storage duration.
   bool IsStatic = false;
-  /// Flag indicating if the block is an extern.
-  bool IsExtern = false;
-  /// Flag indicating if the pointer is dead. This is only ever
-  /// set once, when converting the Block to a DeadBlock.
-  bool IsDead = false;
   /// Flag indicating if the block contents have been initialized
   /// via invokeCtor.
   bool IsInitialized = false;
-  /// Flag indicating if this block has been allocated via dynamic
-  /// memory allocation (e.g. malloc).
-  bool IsDynamic = false;
-  bool IsWeak = false;
-  /// Pointer to the stack slot descriptor.
-  const Descriptor *Desc;
+  /// Allocation ID for this dynamic allocation, if it is one.
+  UnsignedOrNone DynAllocId = std::nullopt;
+  /// AccessFlags containing IsExtern, IsDead, IsWeak, and IsDummy bits.
+  uint8_t AccessFlags = 0;
 };
 
 /// Descriptor for a dead block.
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index f908d02c0412d..2e28814abfddb 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -205,6 +205,8 @@ static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
 
   if (A.isDummy() || B.isDummy())
     return false;
+  if (!A.isBlockPointer() || !B.isBlockPointer())
+    return false;
 
   bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
                 ID == Builtin::BI__builtin_wcscmp ||
@@ -212,7 +214,10 @@ static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
   assert(A.getFieldDesc()->isPrimitiveArray());
   assert(B.getFieldDesc()->isPrimitiveArray());
 
-  assert(getElemType(A).getTypePtr() == getElemType(B).getTypePtr());
+  // Different element types shouldn't happen, but with casts they can.
+  if (!S.getASTContext().hasSameUnqualifiedType(getElemType(A), getElemType(B)))
+    return false;
+
   PrimType ElemT = *S.getContext().classify(getElemType(A));
 
   auto returnResult = [&](int V) -> bool {
@@ -276,7 +281,7 @@ static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
   if (!CheckLive(S, OpPC, StrPtr, AK_Read))
     return false;
 
-  if (!CheckDummy(S, OpPC, StrPtr, AK_Read))
+  if (!CheckDummy(S, OpPC, StrPtr.block(), AK_Read))
     return false;
 
   assert(StrPtr.getFieldDesc()->isPrimitiveArray());
@@ -459,12 +464,13 @@ static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
                                   const InterpFrame *Frame, bool CheckSign,
                                   const CallExpr *Call) {
   const Floating &Arg = S.Stk.pop<Floating>();
-  bool IsInf = Arg.isInf();
+  APFloat F = Arg.getAPFloat();
+  bool IsInf = F.isInfinity();
 
   if (CheckSign)
-    pushInteger(S, IsInf ? (Arg.isNegative() ? -1 : 1) : 0, Call->getType());
+    pushInteger(S, IsInf ? (F.isNegative() ? -1 : 1) : 0, Call->getType());
   else
-    pushInteger(S, Arg.isInf(), Call->getType());
+    pushInteger(S, IsInf, Call->getType());
   return true;
 }
 
@@ -597,6 +603,17 @@ static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+static inline Floating abs(InterpState &S, const Floating &In) {
+  if (!In.isNegative())
+    return In;
+
+  Floating Output = S.allocFloat(In.getSemantics());
+  APFloat New = In.getAPFloat();
+  New.changeSign();
+  Output.copy(New);
+  return Output;
+}
+
 // The C standard says "fabs raises no floating-point exceptions,
 // even if x is a signaling NaN. The returned value is independent of
 // the current rounding direction mode."  Therefore constant folding can
@@ -605,16 +622,7 @@ static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame) {
   const Floating &Val = S.Stk.pop<Floating>();
-  APFloat F = Val.getAPFloat();
-  if (!F.isNegative()) {
-    S.Stk.push<Floating>(Val);
-    return true;
-  }
-
-  Floating Result = S.allocFloat(Val.getSemantics());
-  F.changeSign();
-  Result.copy(F);
-  S.Stk.push<Floating>(Result);
+  S.Stk.push<Floating>(abs(S, Val));
   return true;
 }
 
@@ -1544,8 +1552,7 @@ static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
   // Composite arrays
   if (IsArray) {
     const Descriptor *Desc =
-        S.P.createDescriptor(NewCall, ElemType.getTypePtr(),
-                             IsArray ? std::nullopt : Descriptor::InlineDescMD);
+        S.P.createDescriptor(NewCall, ElemType.getTypePtr(), std::nullopt);
     Block *B =
         Allocator.allocate(Desc, NumElems.getZExtValue(), S.Ctx.getEvalID(),
                            DynamicAllocator::Form::Operator);
@@ -1558,9 +1565,8 @@ static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
   QualType AllocType = S.getASTContext().getConstantArrayType(
       ElemType, NumElems, nullptr, ArraySizeModifier::Normal, 0);
 
-  const Descriptor *Desc =
-      S.P.createDescriptor(NewCall, AllocType.getTypePtr(),
-                           IsArray ? std::nullopt : Descriptor::InlineDescMD);
+  const Descriptor *Desc = S.P.createDescriptor(NewCall, AllocType.getTypePtr(),
+                                                Descriptor::InlineDescMD);
   Block *B = Allocator.allocate(Desc, S.getContext().getEvalID(),
                                 DynamicAllocator::Form::Operator);
   assert(B);
@@ -1687,6 +1693,57 @@ static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,
+                                            const InterpFrame *Frame,
+                                            const CallExpr *Call,
+                                            unsigned BuiltinID) {
+  assert(Call->getNumArgs() == 1);
+  QualType Ty = Call->getArg(0)->getType();
+  if (Ty->isIntegerType()) {
+    PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
+    APSInt Val = popToAPSInt(S.Stk, ArgT);
+
+    pushInteger(S, Val.abs(), Call->getType());
+    return true;
+  }
+
+  if (Ty->isFloatingType()) {
+    Floating Val = S.Stk.pop<Floating>();
+    Floating Result = abs(S, Val);
+    S.Stk.push<Floating>(Result);
+    return true;
+  }
+
+  // Otherwise, the argument must be a vector.
+  assert(Call->getArg(0)->getType()->isVectorType());
+  const Pointer &Arg = S.Stk.pop<Pointer>();
+  assert(Arg.getFieldDesc()->isPrimitiveArray());
+  const Pointer &Dst = S.Stk.peek<Pointer>();
+  assert(Dst.getFieldDesc()->isPrimitiveArray());
+  assert(Arg.getFieldDesc()->getNumElems() ==
+         Dst.getFieldDesc()->getNumElems());
+
+  QualType ElemType = Arg.getFieldDesc()->getElemQualType();
+  PrimType ElemT = *S.getContext().classify(ElemType);
+  unsigned NumElems = Arg.getNumElems();
+  // we can either have a vector of integer or a vector of floating point
+  for (unsigned I = 0; I != NumElems; ++I) {
+    if (ElemType->isIntegerType()) {
+      INT_TYPE_SWITCH_NO_BOOL(ElemT, {
+        Dst.elem<T>(I) = T::from(static_cast<T>(
+            APSInt(Arg.elem<T>(I).toAPSInt().abs(),
+                   ElemType->isUnsignedIntegerOrEnumerationType())));
+      });
+    } else {
+      Floating Val = Arg.elem<Floating>(I);
+      Dst.elem<Floating>(I) = abs(S, Val);
+    }
+  }
+  Dst.initializeAllElements();
+
+  return true;
+}
+
 /// Can be called with an integer or vector as the first and only parameter.
 static bool interp__builtin_elementwise_popcount(InterpState &S, CodePtr OpPC,
                                                  const InterpFrame *Frame,
@@ -1733,6 +1790,94 @@ static bool interp__builtin_elementwise_popcount(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+/// Can be called with an integer or vector as the first and only parameter.
+static bool interp__builtin_elementwise_countzeroes(InterpState &S,
+                                                    CodePtr OpPC,
+                                                    const InterpFrame *Frame,
+                                                    const CallExpr *Call,
+                                                    unsigned BuiltinID) {
+  const bool HasZeroArg = Call->getNumArgs() == 2;
+  const bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_cttz;
+  assert(Call->getNumArgs() == 1 || HasZeroArg);
+  if (Call->getArg(0)->getType()->isIntegerType()) {
+    PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
+    APSInt Val = popToAPSInt(S.Stk, ArgT);
+    std::optional<APSInt> ZeroVal;
+    if (HasZeroArg) {
+      ZeroVal = Val;
+      Val = popToAPSInt(S.Stk, ArgT);
+    }
+
+    if (Val.isZero()) {
+      if (ZeroVal) {
+        pushInteger(S, *ZeroVal, Call->getType());
+        return true;
+      }
+      // If we haven't been provided the second argument, the result is
+      // undefined
+      S.FFDiag(S.Current->getSource(OpPC),
+               diag::note_constexpr_countzeroes_zero)
+          << /*IsTrailing=*/IsCTTZ;
+      return false;
+    }
+
+    if (BuiltinID == Builtin::BI__builtin_elementwise_ctlz) {
+      pushInteger(S, Val.countLeadingZeros(), Call->getType());
+    } else {
+      pushInteger(S, Val.countTrailingZeros(), Call->getType());
+    }
+    return true;
+  }
+  // Otherwise, the argument must be a vector.
+  const ASTContext &ASTCtx = S.getASTContext();
+  Pointer ZeroArg;
+  if (HasZeroArg) {
+    assert(Call->getArg(1)->getType()->isVectorType() &&
+           ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
+                                         Call->getArg(1)->getType()));
+    (void)ASTCtx;
+    ZeroArg = S.Stk.pop<Pointer>();
+    assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
+  }
+  assert(Call->getArg(0)->getType()->isVectorType());
+  const Pointer &Arg = S.Stk.pop<Pointer>();
+  assert(Arg.getFieldDesc()->isPrimitiveArray());
+  const Pointer &Dst = S.Stk.peek<Pointer>();
+  assert(Dst.getFieldDesc()->isPrimitiveArray());
+  assert(Arg.getFieldDesc()->getNumElems() ==
+         Dst.getFieldDesc()->getNumElems());
+
+  QualType ElemType = Arg.getFieldDesc()->getElemQualType();
+  PrimType ElemT = *S.getContext().classify(ElemType);
+  unsigned NumElems = Arg.getNumElems();
+
+  // FIXME: Reading from uninitialized vector elements?
+  for (unsigned I = 0; I != NumElems; ++I) {
+    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
+      APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
+      if (EltVal.isZero()) {
+        if (HasZeroArg) {
+          Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
+        } else {
+          // If we haven't been provided the second argument, the result is
+          // undefined
+          S.FFDiag(S.Current->getSource(OpPC),
+                   diag::note_constexpr_countzeroes_zero)
+              << /*IsTrailing=*/IsCTTZ;
+          return false;
+        }
+      } else if (IsCTTZ) {
+        Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
+      } else {
+        Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
+      }
+      Dst.atIndex(I).initialize();
+    });
+  }
+
+  return true;
+}
+
 static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
                                    const InterpFrame *Frame,
                                    const CallExpr *Call, unsigned ID) {
@@ -1784,7 +1929,27 @@ static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
   if (DestPtr.isDummy() || SrcPtr.isDummy())
     return false;
 
+  if (DestPtr.getType()->isIncompleteType()) {
+    S.FFDiag(S.Current->getSource(OpPC),
+             diag::note_constexpr_memcpy_incomplete_type)
+        << Move << DestPtr.getType();
+    return false;
+  }
+  if (SrcPtr.getType()->isIncompleteType()) {
+    S.FFDiag(S.Current->getSource(OpPC),
+             diag::note_constexpr_memcpy_incomplete_type)
+        << Move << SrcPtr.getType();
+    return false;
+  }
+
   QualType DestElemType = getElemType(DestPtr);
+  if (DestElemType->isIncompleteType()) {
+    S.FFDiag(S.Current->getSource(OpPC),
+             diag::note_constexpr_memcpy_incomplete_type)
+        << Move << DestElemType;
+    return false;
+  }
+
   size_t RemainingDestElems;
   if (DestPtr.getFieldDesc()->isArray()) {
     RemainingDestElems = DestPtr.isUnknownSizeArray()
@@ -1826,16 +1991,6 @@ static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
     return false;
   }
 
-  if (DestElemType->isIncompleteType() ||
-      DestPtr.getType()->isIncompleteType()) {
-    QualType DiagType =
-        DestElemType->isIncompleteType() ? DestElemType : DestPtr.getType();
-    S.FFDiag(S.Current->getSource(OpPC),
-             diag::note_constexpr_memcpy_incomplete_type)
-        << Move << DiagType;
-    return false;
-  }
-
   if (!DestElemType.isTriviallyCopyableType(ASTCtx)) {
     S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_nontrivial)
         << Move << DestElemType;
@@ -2024,8 +2179,13 @@ static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
     return true;
   }
 
-  if (Ptr.isDummy())
+  if (Ptr.isDummy()) {
+    if (Ptr.getType()->isIncompleteType())
+      S.FFDiag(S.Current->getSource(OpPC),
+               diag::note_constexpr_ltor_incomplete_type)
+          << Ptr.getType();
     return false;
+  }
 
   // Null is only okay if the given size is 0.
   if (Ptr.isZero()) {
@@ -2103,29 +2263,32 @@ static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
   return true;
 }
 
-static unsigned computeFullDescSize(const ASTContext &ASTCtx,
-                                    const Descriptor *Desc) {
-
+static std::optional<unsigned> computeFullDescSize(const ASTContext &ASTCtx,
+                                                   const Descriptor *Desc) {
   if (Desc->isPrimitive())
     return ASTCtx.getTypeSizeInChars(Desc->getType()).getQuantity();
-
   if (Desc->isArray())
     return ASTCtx.getTypeSizeInChars(Desc->getElemQualType()).getQuantity() *
            Desc->getNumElems();
+  if (Desc->isRecord()) {
+    // Can't use Descriptor::getType() as that may return a pointer type. Look
+    // at the decl directly.
+    return ASTCtx
+        .getTypeSizeInChars(
+            ASTCtx.getCanonicalTagType(Desc->ElemRecord->getDecl()))
+        .getQuantity();
+  }
 
-  if (Desc->isRecord())
-    return ASTCtx.getTypeSizeInChars(Desc->getType()).getQuantity();
-
-  llvm_unreachable("Unhandled descriptor type");
-  return 0;
+  return std::nullopt;
 }
 
+/// Compute the byte offset of \p Ptr in the full declaration.
 static unsigned computePointerOffset(const ASTContext &ASTCtx,
                                      const Pointer &Ptr) {
   unsigned Result = 0;
 
   Pointer P = Ptr;
-  while (P.isArrayElement() || P.isField()) {
+  while (P.isField() || P.isArrayElement()) {
     P = P.expand();
     const Descriptor *D = P.getFieldDesc();
 
@@ -2138,7 +2301,6 @@ static unsigned computePointerOffset(const ASTContext &ASTCtx,
         Result += ElemSize * P.getIndex();
       P = P.expand().getArray();
     } else if (P.isBaseClass()) {
-
       const auto *RD = cast<CXXRecordDecl>(D->asDecl());
       bool IsVirtual = Ptr.isVirtualBaseClass();
       P = P.getBase();
@@ -2167,30 +2329,136 @@ static unsigned computePointerOffset(const ASTContext &ASTCtx,
   return Result;
 }
 
+/// Does Ptr point to the last subobject?
+static bool pointsToLastObject(const Pointer &Ptr) {
+  Pointer P = Ptr;
+  while (!P.isRoot()) {
+
+    if (P.isArrayElement()) {
+      P = P.expand().getArray();
+      continue;
+    }
+    if (P.isBaseClass()) {
+      if (P.getRecord()->getNumFields() > 0)
+        return false;
+      P = P.getBase();
+      continue;
+    }
+
+    Pointer Base = P.getBase();
+    if (const Record *R = Base.getRecord()) {
+      assert(P.getField());
+      if (P.getField()->getFieldIndex() != R->getNumFields() - 1)
+        return false;
+    }
+    P = Base;
+  }
+
+  return true;
+}
+
+/// Does Ptr point to the last object AND to a flexible array member?
+static bool isUserWritingOffTheEnd(const ASTContext &Ctx, const Pointer &Ptr) {
+  auto isFlexibleArrayMember = [&](const Descriptor *FieldDesc) {
+    using FAMKind = LangOptions::StrictFlexArraysLevelKind;
+    FAMKind StrictFlexArraysLevel =
+        Ctx.getLangOpts().getStrictFlexArraysLevel();
+
+    if (StrictFlexArraysLevel == FAMKind::Default)
+      return true;
+
+    unsigned NumElems = FieldDesc->getNumElems();
+    if (NumElems == 0 && StrictFlexArraysLevel != FAMKind::IncompleteOnly)
+      return true;
+
+    if (NumElems == 1 && StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete)
+      return true;
+    return false;
+  };
+
+  const Descriptor *FieldDesc = Ptr.getFieldDesc();
+  if (!FieldDesc->isArray())
+    return false;
+
+  return Ptr.isDummy() && pointsToLastObject(Ptr) &&
+         isFlexibleArrayMember(FieldDesc);
+}
+
 static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
                                         const InterpFrame *Frame,
                                         const CallExpr *Call) {
+  const ASTContext &ASTCtx = S.getASTContext();
   PrimType KindT = *S.getContext().classify(Call->getArg(1));
-  [[maybe_unused]] unsigned Kind = popToAPSInt(S.Stk, KindT).getZExtValue();
-
+  // From the GCC docs:
+  // Kind is an integer constant from 0 to 3. If the least significant bit is
+  // clear, objects are whole variables. If it is set, a closest surrounding
+  // subobject is considered the object a pointer points to. The second bit
+  // determines if maximum or minimum of remaining bytes is computed.
+  unsigned Kind = popToAPSInt(S.Stk, KindT).getZExtValue();
   assert(Kind <= 3 && "unexpected kind");
-
+  bool UseFieldDesc = (Kind & 1u);
+  bool ReportMinimum = (Kind & 2u);
   const Pointer &Ptr = S.Stk.pop<Pointer>();
 
-  if (Ptr.isZero())
+  if (Call->getArg(0)->HasSideEffects(ASTCtx)) {
+    // "If there are any side effects in them, it returns (size_t) -1
+    // for type 0 or 1 and (size_t) 0 for type 2 or 3."
+    pushInteger(S, Kind <= 1 ? -1 : 0, Call->getType());
+    return true;
+  }
+
+  if (Ptr.isZero() || !Ptr.isBlockPointer())
     return false;
 
-  const Descriptor *DeclDesc = Ptr.getDeclDesc();
-  if (!DeclDesc)
+  // We can't load through pointers.
+  if (Ptr.isDummy() && Ptr.getType()->isPointerType())
     return false;
 
-  const ASTContext &ASTCtx = S.getASTContext();
+  bool DetermineForCompleteObject = Ptr.getFieldDesc() == Ptr.getDeclDesc();
+  const Descriptor *DeclDesc = Ptr.getDeclDesc();
+  assert(DeclDesc);
+
+  if (!UseFieldDesc || DetermineForCompleteObject) {
+    // Lower bound, so we can't fall back to this.
+    if (ReportMinimum && !DetermineForCompleteObject)
+      return false;
+
+    // Can't read beyond the pointer decl desc.
+    if (!UseFieldDesc && !ReportMinimum && DeclDesc->getType()->isPointerType())
+      return false;
+  } else {
+    if (isUserWritingOffTheEnd(ASTCtx, Ptr.expand())) {
+      // If we cannot determine the size of the initial allocation, then we
+      // can't given an accurate upper-bound. However, we are still able to give
+      // conservative lower-bounds for Type=3.
+      if (Kind == 1)
+        return false;
+    }
+  }
+
+  const Descriptor *Desc = UseFieldDesc ? Ptr.getFieldDesc() : DeclDesc;
+  assert(Desc);
+
+  std::optional<unsigned> FullSize = computeFullDescSize(ASTCtx, Desc);
+  if (!FullSize)
+    return false;
 
-  unsigned ByteOffset = computePointerOffset(ASTCtx, Ptr);
-  unsigned FullSize = computeFullDescSize(ASTCtx, DeclDesc);
+  unsigned ByteOffset;
+  if (UseFieldDesc) {
+    if (Ptr.isBaseClass())
+      ByteOffset = computePointerOffset(ASTCtx, Ptr.getBase()) -
+                   computePointerOffset(ASTCtx, Ptr);
+    else
+      ByteOffset =
+          computePointerOffset(ASTCtx, Ptr) -
+          computePointerOffset(ASTCtx, Ptr.expand().atIndex(0).narrow());
+  } else
+    ByteOffset = computePointerOffset(ASTCtx, Ptr);
 
-  pushInteger(S, FullSize - ByteOffset, Call->getType());
+  assert(ByteOffset <= *FullSize);
+  unsigned Result = *FullSize - ByteOffset;
 
+  pushInteger(S, Result, Call->getType());
   return true;
 }
 
@@ -2232,7 +2500,7 @@ static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
       return false;
     if (!CheckMutable(S, OpPC, Ptr))
       return false;
-    if (!CheckDummy(S, OpPC, Ptr, AK_Read))
+    if (!CheckDummy(S, OpPC, Ptr.block(), AK_Read))
       return false;
   }
 
@@ -2253,7 +2521,6 @@ static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_elementwise_sat(InterpState &S, CodePtr OpPC,
                                             const CallExpr *Call,
                                             unsigned BuiltinID) {
-  Call->dumpColor();
   assert(Call->getNumArgs() == 2);
 
   // Single integer case.
@@ -2272,7 +2539,7 @@ static bool interp__builtin_elementwise_sat(InterpState &S, CodePtr OpPC,
       llvm_unreachable("Wrong builtin ID");
     }
 
-    pushInteger(S, APSInt(Result, !LHS.isSigned()), Call->getType());
+    pushInteger(S, APSInt(std::move(Result), !LHS.isSigned()), Call->getType());
     return true;
   }
 
@@ -2300,14 +2567,103 @@ static bool interp__builtin_elementwise_sat(InterpState &S, CodePtr OpPC,
     });
 
     APSInt Result;
-    if (BuiltinID == Builtin::BI__builtin_elementwise_add_sat) {
+    switch (BuiltinID) {
+    case Builtin::BI__builtin_elementwise_add_sat:
       Result = APSInt(Elem1.isSigned() ? Elem1.sadd_sat(Elem2)
                                        : Elem1.uadd_sat(Elem2),
                       Call->getType()->isUnsignedIntegerOrEnumerationType());
-    } else if (BuiltinID == Builtin::BI__builtin_elementwise_sub_sat) {
+      break;
+    case Builtin::BI__builtin_elementwise_sub_sat:
       Result = APSInt(Elem1.isSigned() ? Elem1.ssub_sat(Elem2)
                                        : Elem1.usub_sat(Elem2),
                       Call->getType()->isUnsignedIntegerOrEnumerationType());
+      break;
+    case clang::X86::BI__builtin_ia32_pmulhuw128:
+    case clang::X86::BI__builtin_ia32_pmulhuw256:
+    case clang::X86::BI__builtin_ia32_pmulhuw512:
+      Result = APSInt(llvm::APIntOps::mulhu(Elem1, Elem2),
+                      /*isUnsigned=*/true);
+      break;
+    case clang::X86::BI__builtin_ia32_pmulhw128:
+    case clang::X86::BI__builtin_ia32_pmulhw256:
+    case clang::X86::BI__builtin_ia32_pmulhw512:
+      Result = APSInt(llvm::APIntOps::mulhs(Elem1, Elem2),
+                      /*isUnsigned=*/false);
+      break;
+    default:
+      llvm_unreachable("Wrong builtin ID");
+    }
+
+    INT_TYPE_SWITCH_NO_BOOL(ElemT,
+                            { Dst.elem<T>(I) = static_cast<T>(Result); });
+  }
+  Dst.initializeAllElements();
+
+  return true;
+}
+
+static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,
+                                               const CallExpr *Call,
+                                               unsigned BuiltinID) {
+  assert(Call->getNumArgs() == 2);
+
+  QualType Arg0Type = Call->getArg(0)->getType();
+
+  // TODO: Support floating-point types.
+  if (!(Arg0Type->isIntegerType() ||
+        (Arg0Type->isVectorType() &&
+         Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
+    return false;
+
+  if (!Arg0Type->isVectorType()) {
+    assert(!Call->getArg(1)->getType()->isVectorType());
+    APSInt RHS = popToAPSInt(
+        S.Stk, *S.getContext().classify(Call->getArg(1)->getType()));
+    APSInt LHS = popToAPSInt(
+        S.Stk, *S.getContext().classify(Call->getArg(0)->getType()));
+    APInt Result;
+    if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
+      Result = std::max(LHS, RHS);
+    } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
+      Result = std::min(LHS, RHS);
+    } else {
+      llvm_unreachable("Wrong builtin ID");
+    }
+
+    pushInteger(S, APSInt(Result, !LHS.isSigned()), Call->getType());
+    return true;
+  }
+
+  // Vector case.
+  assert(Call->getArg(0)->getType()->isVectorType() &&
+         Call->getArg(1)->getType()->isVectorType());
+  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
+  assert(VT->getElementType() ==
+         Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
+  assert(VT->getNumElements() ==
+         Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
+  assert(VT->getElementType()->isIntegralOrEnumerationType());
+
+  const Pointer &RHS = S.Stk.pop<Pointer>();
+  const Pointer &LHS = S.Stk.pop<Pointer>();
+  const Pointer &Dst = S.Stk.peek<Pointer>();
+  PrimType ElemT = *S.getContext().classify(VT->getElementType());
+  unsigned NumElems = VT->getNumElements();
+  for (unsigned I = 0; I != NumElems; ++I) {
+    APSInt Elem1;
+    APSInt Elem2;
+    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
+      Elem1 = LHS.elem<T>(I).toAPSInt();
+      Elem2 = RHS.elem<T>(I).toAPSInt();
+    });
+
+    APSInt Result;
+    if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
+      Result = APSInt(std::max(Elem1, Elem2),
+                      Call->getType()->isUnsignedIntegerOrEnumerationType());
+    } else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
+      Result = APSInt(std::min(Elem1, Elem2),
+                      Call->getType()->isUnsignedIntegerOrEnumerationType());
     } else {
       llvm_unreachable("Wrong builtin ID");
     }
@@ -2320,6 +2676,147 @@ static bool interp__builtin_elementwise_sat(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+static bool interp__builtin_ia32_pmul(InterpState &S, CodePtr OpPC,
+                                      const CallExpr *Call,
+                                      unsigned BuiltinID) {
+  assert(Call->getArg(0)->getType()->isVectorType() &&
+         Call->getArg(1)->getType()->isVectorType());
+  const Pointer &RHS = S.Stk.pop<Pointer>();
+  const Pointer &LHS = S.Stk.pop<Pointer>();
+  const Pointer &Dst = S.Stk.peek<Pointer>();
+
+  const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
+  PrimType ElemT = *S.getContext().classify(VT->getElementType());
+  unsigned SourceLen = VT->getNumElements();
+
+  PrimType DstElemT = *S.getContext().classify(
+      Call->getType()->castAs<VectorType>()->getElementType());
+  unsigned DstElem = 0;
+  for (unsigned I = 0; I != SourceLen; I += 2) {
+    APSInt Elem1;
+    APSInt Elem2;
+    INT_TYPE_SWITCH_NO_BOOL(ElemT, {
+      Elem1 = LHS.elem<T>(I).toAPSInt();
+      Elem2 = RHS.elem<T>(I).toAPSInt();
+    });
+
+    APSInt Result;
+    switch (BuiltinID) {
+    case clang::X86::BI__builtin_ia32_pmuludq128:
+    case clang::X86::BI__builtin_ia32_pmuludq256:
+    case clang::X86::BI__builtin_ia32_pmuludq512:
+      Result = APSInt(llvm::APIntOps::muluExtended(Elem1, Elem2),
+                      /*IsUnsigned=*/true);
+      break;
+    case clang::X86::BI__builtin_ia32_pmuldq128:
+    case clang::X86::BI__builtin_ia32_pmuldq256:
+    case clang::X86::BI__builtin_ia32_pmuldq512:
+      Result = APSInt(llvm::APIntOps::mulsExtended(Elem1, Elem2),
+                      /*IsUnsigned=*/false);
+      break;
+    }
+    INT_TYPE_SWITCH_NO_BOOL(DstElemT,
+                            { Dst.elem<T>(DstElem) = static_cast<T>(Result); });
+    ++DstElem;
+  }
+
+  Dst.initializeAllElements();
+  return true;
+}
+
+static bool interp__builtin_elementwise_fma(InterpState &S, CodePtr OpPC,
+                                            const CallExpr *Call) {
+  assert(Call->getNumArgs() == 3);
+
+  FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
+  llvm::RoundingMode RM = getRoundingMode(FPO);
+  const QualType Arg1Type = Call->getArg(0)->getType();
+  const QualType Arg2Type = Call->getArg(1)->getType();
+  const QualType Arg3Type = Call->getArg(2)->getType();
+
+  // Non-vector floating point types.
+  if (!Arg1Type->isVectorType()) {
+    assert(!Arg2Type->isVectorType());
+    assert(!Arg3Type->isVectorType());
+    (void)Arg2Type;
+    (void)Arg3Type;
+
+    const Floating &Z = S.Stk.pop<Floating>();
+    const Floating &Y = S.Stk.pop<Floating>();
+    const Floating &X = S.Stk.pop<Floating>();
+    APFloat F = X.getAPFloat();
+    F.fusedMultiplyAdd(Y.getAPFloat(), Z.getAPFloat(), RM);
+    Floating Result = S.allocFloat(X.getSemantics());
+    Result.copy(F);
+    S.Stk.push<Floating>(Result);
+    return true;
+  }
+
+  // Vector type.
+  assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
+         Arg3Type->isVectorType());
+
+  const VectorType *VecT = Arg1Type->castAs<VectorType>();
+  const QualType ElemT = VecT->getElementType();
+  unsigned NumElems = VecT->getNumElements();
+
+  assert(ElemT == Arg2Type->castAs<VectorType>()->getElementType() &&
+         ElemT == Arg3Type->castAs<VectorType>()->getElementType());
+  assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
+         NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
+  assert(ElemT->isRealFloatingType());
+  (void)ElemT;
+
+  const Pointer &VZ = S.Stk.pop<Pointer>();
+  const Pointer &VY = S.Stk.pop<Pointer>();
+  const Pointer &VX = S.Stk.pop<Pointer>();
+  const Pointer &Dst = S.Stk.peek<Pointer>();
+  for (unsigned I = 0; I != NumElems; ++I) {
+    using T = PrimConv<PT_Float>::T;
+    APFloat X = VX.elem<T>(I).getAPFloat();
+    APFloat Y = VY.elem<T>(I).getAPFloat();
+    APFloat Z = VZ.elem<T>(I).getAPFloat();
+    (void)X.fusedMultiplyAdd(Y, Z, RM);
+    Dst.elem<Floating>(I) = Floating(X);
+  }
+  Dst.initializeAllElements();
+  return true;
+}
+
+/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
+static bool interp__builtin_select(InterpState &S, CodePtr OpPC,
+                                   const CallExpr *Call) {
+  const Pointer &RHS = S.Stk.pop<Pointer>();
+  const Pointer &LHS = S.Stk.pop<Pointer>();
+  PrimType MaskT = *S.getContext().classify(Call->getArg(0));
+  APSInt Mask = popToAPSInt(S.Stk, MaskT);
+  const Pointer &Dst = S.Stk.peek<Pointer>();
+
+  assert(LHS.getNumElems() == RHS.getNumElems());
+  assert(LHS.getNumElems() == Dst.getNumElems());
+  unsigned NumElems = LHS.getNumElems();
+  PrimType ElemT = LHS.getFieldDesc()->getPrimType();
+  PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
+
+  for (unsigned I = 0; I != NumElems; ++I) {
+    if (ElemT == PT_Float) {
+      assert(DstElemT == PT_Float);
+      Dst.elem<Floating>(I) =
+          Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
+    } else {
+      APSInt Elem;
+      INT_TYPE_SWITCH(ElemT, {
+        Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
+      });
+      INT_TYPE_SWITCH_NO_BOOL(DstElemT,
+                              { Dst.elem<T>(I) = static_cast<T>(Elem); });
+    }
+  }
+  Dst.initializeAllElements();
+
+  return true;
+}
+
 bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
                       uint32_t BuiltinID) {
   if (!S.getASTContext().BuiltinInfo.isConstantEvaluated(BuiltinID))
@@ -2596,6 +3093,11 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
   case Builtin::BI__builtin_ctzg:
     return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
 
+  case Builtin::BI__builtin_elementwise_ctlz:
+  case Builtin::BI__builtin_elementwise_cttz:
+    return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,
+                                                   BuiltinID);
+
   case Builtin::BI__builtin_bswap16:
   case Builtin::BI__builtin_bswap32:
   case Builtin::BI__builtin_bswap64:
@@ -2687,6 +3189,9 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
     return interp__builtin_elementwise_popcount(S, OpPC, Frame, Call,
                                                 BuiltinID);
 
+  case Builtin::BI__builtin_elementwise_abs:
+    return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);
+
   case Builtin::BI__builtin_memcpy:
   case Builtin::BImemcpy:
   case Builtin::BI__builtin_wmemcpy:
@@ -2725,8 +3230,55 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
 
   case Builtin::BI__builtin_elementwise_add_sat:
   case Builtin::BI__builtin_elementwise_sub_sat:
+  case clang::X86::BI__builtin_ia32_pmulhuw128:
+  case clang::X86::BI__builtin_ia32_pmulhuw256:
+  case clang::X86::BI__builtin_ia32_pmulhuw512:
+  case clang::X86::BI__builtin_ia32_pmulhw128:
+  case clang::X86::BI__builtin_ia32_pmulhw256:
+  case clang::X86::BI__builtin_ia32_pmulhw512:
     return interp__builtin_elementwise_sat(S, OpPC, Call, BuiltinID);
 
+  case Builtin::BI__builtin_elementwise_max:
+  case Builtin::BI__builtin_elementwise_min:
+    return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);
+
+  case clang::X86::BI__builtin_ia32_pmuldq128:
+  case clang::X86::BI__builtin_ia32_pmuldq256:
+  case clang::X86::BI__builtin_ia32_pmuldq512:
+  case clang::X86::BI__builtin_ia32_pmuludq128:
+  case clang::X86::BI__builtin_ia32_pmuludq256:
+  case clang::X86::BI__builtin_ia32_pmuludq512:
+    return interp__builtin_ia32_pmul(S, OpPC, Call, BuiltinID);
+
+  case Builtin::BI__builtin_elementwise_fma:
+    return interp__builtin_elementwise_fma(S, OpPC, Call);
+
+  case X86::BI__builtin_ia32_selectb_128:
+  case X86::BI__builtin_ia32_selectb_256:
+  case X86::BI__builtin_ia32_selectb_512:
+  case X86::BI__builtin_ia32_selectw_128:
+  case X86::BI__builtin_ia32_selectw_256:
+  case X86::BI__builtin_ia32_selectw_512:
+  case X86::BI__builtin_ia32_selectd_128:
+  case X86::BI__builtin_ia32_selectd_256:
+  case X86::BI__builtin_ia32_selectd_512:
+  case X86::BI__builtin_ia32_selectq_128:
+  case X86::BI__builtin_ia32_selectq_256:
+  case X86::BI__builtin_ia32_selectq_512:
+  case X86::BI__builtin_ia32_selectph_128:
+  case X86::BI__builtin_ia32_selectph_256:
+  case X86::BI__builtin_ia32_selectph_512:
+  case X86::BI__builtin_ia32_selectpbf_128:
+  case X86::BI__builtin_ia32_selectpbf_256:
+  case X86::BI__builtin_ia32_selectpbf_512:
+  case X86::BI__builtin_ia32_selectps_128:
+  case X86::BI__builtin_ia32_selectps_256:
+  case X86::BI__builtin_ia32_selectps_512:
+  case X86::BI__builtin_ia32_selectpd_128:
+  case X86::BI__builtin_ia32_selectpd_256:
+  case X86::BI__builtin_ia32_selectpd_512:
+    return interp__builtin_select(S, OpPC, Call);
+
   default:
     S.FFDiag(S.Current->getLocation(OpPC),
              diag::note_invalid_subexpr_in_const_expr)
@@ -2754,7 +3306,7 @@ bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
       const RecordType *RT = CurrentType->getAs<RecordType>();
       if (!RT)
         return false;
-      const RecordDecl *RD = RT->getDecl();
+      const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       if (RD->isInvalidDecl())
         return false;
       const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(RD);
@@ -2787,7 +3339,7 @@ bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
       const RecordType *RT = CurrentType->getAs<RecordType>();
       if (!RT)
         return false;
-      const RecordDecl *RD = RT->getDecl();
+      const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       if (RD->isInvalidDecl())
         return false;
       const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(RD);
@@ -2799,7 +3351,8 @@ bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
         return false;
 
       // Add the offset to the base.
-      Result += RL.getBaseClassOffset(cast<CXXRecordDecl>(BaseRT->getDecl()));
+      Result += RL.getBaseClassOffset(cast<CXXRecordDecl>(
+          BaseRT->getOriginalDecl()->getDefinitionOrSelf()));
       break;
     }
     case OffsetOfNode::Identifier:
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index 93421927d2ea9..18400b10c47b0 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -169,7 +169,7 @@ void InterpFrame::describe(llvm::raw_ostream &OS) const {
     } else if (const auto *M = dyn_cast<CXXMethodDecl>(F)) {
       print(OS, This, S.getASTContext(),
             S.getASTContext().getLValueReferenceType(
-                S.getASTContext().getRecordType(M->getParent())));
+                S.getASTContext().getCanonicalTagType(M->getParent())));
       OS << ".";
     }
   }
@@ -231,6 +231,10 @@ Pointer InterpFrame::getLocalPointer(unsigned Offset) const {
   return Pointer(localBlock(Offset));
 }
 
+Block *InterpFrame::getLocalBlock(unsigned Offset) const {
+  return localBlock(Offset);
+}
+
 Pointer InterpFrame::getParamPointer(unsigned Off) {
   // Return the block if it was created previously.
   if (auto Pt = Params.find(Off); Pt != Params.end())
diff --git a/clang/lib/AST/ByteCode/InterpFrame.h b/clang/lib/AST/ByteCode/InterpFrame.h
index cfebe936cd468..4be53911b615e 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.h
+++ b/clang/lib/AST/ByteCode/InterpFrame.h
@@ -86,6 +86,7 @@ class InterpFrame final : public Frame {
 
   /// Returns a pointer to a local variables.
   Pointer getLocalPointer(unsigned Offset) const;
+  Block *getLocalBlock(unsigned Offset) const;
 
   /// Returns the value of an argument.
   template <typename T> const T &getParam(unsigned Offset) const {
diff --git a/clang/lib/AST/ByteCode/InterpStack.cpp b/clang/lib/AST/ByteCode/InterpStack.cpp
index 6b748d62b83bd..7920378f365f9 100644
--- a/clang/lib/AST/ByteCode/InterpStack.cpp
+++ b/clang/lib/AST/ByteCode/InterpStack.cpp
@@ -26,33 +26,33 @@ InterpStack::~InterpStack() {
     std::free(Chunk);
   Chunk = nullptr;
   StackSize = 0;
-#ifndef NDEBUG
   ItemTypes.clear();
-#endif
 }
 
 // We keep the last chunk around to reuse.
 void InterpStack::clear() {
-  if (!Chunk)
-    return;
-
-  if (Chunk->Next)
-    std::free(Chunk->Next);
-
-  assert(Chunk);
-  StackSize = 0;
-#ifndef NDEBUG
-  ItemTypes.clear();
-#endif
+  for (PrimType Item : llvm::reverse(ItemTypes)) {
+    TYPE_SWITCH(Item, { this->discard<T>(); });
+  }
+  assert(ItemTypes.empty());
+  assert(empty());
 }
 
 void InterpStack::clearTo(size_t NewSize) {
-  assert(NewSize <= size());
-  size_t ToShrink = size() - NewSize;
-  if (ToShrink == 0)
+  if (NewSize == 0)
+    return clear();
+  if (NewSize == size())
     return;
 
-  shrink(ToShrink);
+  assert(NewSize <= size());
+  for (PrimType Item : llvm::reverse(ItemTypes)) {
+    TYPE_SWITCH(Item, { this->discard<T>(); });
+
+    if (size() == NewSize)
+      break;
+  }
+
+  // Note: discard() above already removed the types from ItemTypes.
   assert(size() == NewSize);
 }
 
@@ -105,25 +105,9 @@ void InterpStack::shrink(size_t Size) {
 
   Chunk->End -= Size;
   StackSize -= Size;
-
-#ifndef NDEBUG
-  size_t TypesSize = 0;
-  for (PrimType T : ItemTypes)
-    TYPE_SWITCH(T, { TypesSize += aligned_size<T>(); });
-
-  size_t StackSize = size();
-  while (TypesSize > StackSize) {
-    TYPE_SWITCH(ItemTypes.back(), {
-      TypesSize -= aligned_size<T>();
-      ItemTypes.pop_back();
-    });
-  }
-  assert(TypesSize == StackSize);
-#endif
 }
 
 void InterpStack::dump() const {
-#ifndef NDEBUG
   llvm::errs() << "Items: " << ItemTypes.size() << ". Size: " << size() << '\n';
   if (ItemTypes.empty())
     return;
@@ -133,11 +117,11 @@ void InterpStack::dump() const {
 
   // The type of the item on the top of the stack is inserted to the back
   // of the vector, so the iteration has to happen backwards.
-  for (auto TyIt = ItemTypes.rbegin(); TyIt != ItemTypes.rend(); ++TyIt) {
-    Offset += align(primSize(*TyIt));
+  for (PrimType Item : llvm::reverse(ItemTypes)) {
+    Offset += align(primSize(Item));
 
     llvm::errs() << Index << '/' << Offset << ": ";
-    TYPE_SWITCH(*TyIt, {
+    TYPE_SWITCH(Item, {
       const T &V = peek<T>(Offset);
       llvm::errs() << V;
     });
@@ -145,5 +129,4 @@ void InterpStack::dump() const {
 
     ++Index;
   }
-#endif
 }
diff --git a/clang/lib/AST/ByteCode/InterpStack.h b/clang/lib/AST/ByteCode/InterpStack.h
index 580494eb2347c..b0f9f6e225682 100644
--- a/clang/lib/AST/ByteCode/InterpStack.h
+++ b/clang/lib/AST/ByteCode/InterpStack.h
@@ -17,7 +17,6 @@
 #include "IntegralAP.h"
 #include "MemberPointer.h"
 #include "PrimType.h"
-#include <vector>
 
 namespace clang {
 namespace interp {
@@ -33,18 +32,14 @@ class InterpStack final {
   /// Constructs a value in place on the top of the stack.
   template <typename T, typename... Tys> void push(Tys &&...Args) {
     new (grow(aligned_size<T>())) T(std::forward<Tys>(Args)...);
-#ifndef NDEBUG
     ItemTypes.push_back(toPrimType<T>());
-#endif
   }
 
   /// Returns the value from the top of the stack and removes it.
   template <typename T> T pop() {
-#ifndef NDEBUG
     assert(!ItemTypes.empty());
     assert(ItemTypes.back() == toPrimType<T>());
     ItemTypes.pop_back();
-#endif
     T *Ptr = &peekInternal<T>();
     T Value = std::move(*Ptr);
     shrink(aligned_size<T>());
@@ -53,22 +48,20 @@ class InterpStack final {
 
   /// Discards the top value from the stack.
   template <typename T> void discard() {
-#ifndef NDEBUG
     assert(!ItemTypes.empty());
     assert(ItemTypes.back() == toPrimType<T>());
     ItemTypes.pop_back();
-#endif
     T *Ptr = &peekInternal<T>();
-    Ptr->~T();
+    if constexpr (!std::is_trivially_destructible_v<T>) {
+      Ptr->~T();
+    }
     shrink(aligned_size<T>());
   }
 
   /// Returns a reference to the value on the top of the stack.
   template <typename T> T &peek() const {
-#ifndef NDEBUG
     assert(!ItemTypes.empty());
     assert(ItemTypes.back() == toPrimType<T>());
-#endif
     return peekInternal<T>();
   }
 
@@ -83,7 +76,7 @@ class InterpStack final {
   /// Returns the size of the stack in bytes.
   size_t size() const { return StackSize; }
 
-  /// Clears the stack without calling any destructors.
+  /// Clears the stack.
   void clear();
   void clearTo(size_t NewSize);
 
@@ -146,9 +139,11 @@ class InterpStack final {
   /// Total size of the stack.
   size_t StackSize = 0;
 
-#ifndef NDEBUG
-  /// vector recording the type of data we pushed into the stack.
-  std::vector<PrimType> ItemTypes;
+  /// SmallVector recording the type of data we pushed into the stack.
+  /// We don't usually need this during normal code interpretation but
+  /// when aborting, we need type information to call the destructors
+  /// for what's left on the stack.
+  llvm::SmallVector<PrimType> ItemTypes;
 
   template <typename T> static constexpr PrimType toPrimType() {
     if constexpr (std::is_same_v<T, Pointer>)
@@ -192,7 +187,6 @@ class InterpStack final {
 
     llvm_unreachable("unknown type push()'ed into InterpStack");
   }
-#endif
 };
 
 } // namespace interp
diff --git a/clang/lib/AST/ByteCode/InterpState.cpp b/clang/lib/AST/ByteCode/InterpState.cpp
index a06b125b3dace..f89967759ff9b 100644
--- a/clang/lib/AST/ByteCode/InterpState.cpp
+++ b/clang/lib/AST/ByteCode/InterpState.cpp
@@ -45,6 +45,12 @@ InterpState::~InterpState() {
 
   while (DeadBlocks) {
     DeadBlock *Next = DeadBlocks->Next;
+
+    // There might be a pointer in a global structure pointing to the dead
+    // block.
+    for (Pointer *P = DeadBlocks->B.Pointers; P; P = P->asBlockPointer().Next)
+      DeadBlocks->B.removePointer(P);
+
     std::free(DeadBlocks);
     DeadBlocks = Next;
   }
@@ -53,12 +59,6 @@ InterpState::~InterpState() {
 void InterpState::cleanup() {
   // As a last resort, make sure all pointers still pointing to a dead block
   // don't point to it anymore.
-  for (DeadBlock *DB = DeadBlocks; DB; DB = DB->Next) {
-    for (Pointer *P = DB->B.Pointers; P; P = P->asBlockPointer().Next) {
-      P->PointeeStorage.BS.Pointee = nullptr;
-    }
-  }
-
   Alloc.cleanup();
 }
 
@@ -76,8 +76,9 @@ bool InterpState::reportOverflow(const Expr *E, const llvm::APSInt &Value) {
 
 void InterpState::deallocate(Block *B) {
   assert(B);
-  const Descriptor *Desc = B->getDescriptor();
-  assert(Desc);
+  assert(!B->isDynamic());
+  assert(!B->isStatic());
+  assert(!B->isDead());
 
   // The block might have a pointer saved in a field in its data
   // that points to the block itself. We call the dtor first,
@@ -87,6 +88,7 @@ void InterpState::deallocate(Block *B) {
   if (B->IsInitialized)
     B->invokeDtor();
 
+  assert(!B->isInitialized());
   if (B->hasPointers()) {
     size_t Size = B->getSize();
     // Allocate a new block, transferring over pointers.
@@ -95,24 +97,20 @@ void InterpState::deallocate(Block *B) {
     auto *D = new (Memory) DeadBlock(DeadBlocks, B);
     // Since the block doesn't hold any actual data anymore, we can just
     // memcpy() everything over.
-    std::memcpy(D->rawData(), B->rawData(), Desc->getAllocSize());
-    D->B.IsInitialized = B->IsInitialized;
-
-    // We moved the contents over to the DeadBlock.
-    B->IsInitialized = false;
+    std::memcpy(D->rawData(), B->rawData(), Size);
+    D->B.IsInitialized = false;
   }
 }
 
 bool InterpState::maybeDiagnoseDanglingAllocations() {
-  bool NoAllocationsLeft = (Alloc.getNumAllocations() == 0);
+  bool NoAllocationsLeft = !Alloc.hasAllocations();
 
   if (!checkingPotentialConstantExpression()) {
-    for (const auto &It : Alloc.allocation_sites()) {
-      assert(It.second.size() > 0);
+    for (const auto &[Source, Site] : Alloc.allocation_sites()) {
+      assert(!Site.empty());
 
-      const Expr *Source = It.first;
       CCEDiag(Source->getExprLoc(), diag::note_constexpr_memory_leak)
-          << (It.second.size() - 1) << Source->getSourceRange();
+          << (Site.size() - 1) << Source->getSourceRange();
     }
   }
   // Keep evaluating before C++20, since the CXXNewExpr wasn't valid there
diff --git a/clang/lib/AST/ByteCode/MemberPointer.h b/clang/lib/AST/ByteCode/MemberPointer.h
index b17ce256e75e2..8dd75cad092c0 100644
--- a/clang/lib/AST/ByteCode/MemberPointer.h
+++ b/clang/lib/AST/ByteCode/MemberPointer.h
@@ -51,6 +51,12 @@ class MemberPointer final {
 
   FunctionPointer toFunctionPointer(const Context &Ctx) const;
 
+  bool isBaseCastPossible() const {
+    if (PtrOffset < 0)
+      return true;
+    return static_cast<uint64_t>(PtrOffset) <= Base.getByteOffset();
+  }
+
   Pointer getBase() const {
     if (PtrOffset < 0)
       return Base.atField(-PtrOffset);
diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index dec2088f1a423..38856ad256a63 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -30,39 +30,62 @@ Pointer::Pointer(Block *Pointee)
 Pointer::Pointer(Block *Pointee, uint64_t BaseAndOffset)
     : Pointer(Pointee, BaseAndOffset, BaseAndOffset) {}
 
-Pointer::Pointer(const Pointer &P)
-    : Offset(P.Offset), StorageKind(P.StorageKind),
-      PointeeStorage(P.PointeeStorage) {
-
-  if (isBlockPointer() && PointeeStorage.BS.Pointee)
-    PointeeStorage.BS.Pointee->addPointer(this);
-}
-
 Pointer::Pointer(Block *Pointee, unsigned Base, uint64_t Offset)
     : Offset(Offset), StorageKind(Storage::Block) {
   assert((Base == RootPtrMark || Base % alignof(void *) == 0) && "wrong base");
 
-  PointeeStorage.BS = {Pointee, Base, nullptr, nullptr};
+  BS = {Pointee, Base, nullptr, nullptr};
 
   if (Pointee)
     Pointee->addPointer(this);
 }
 
-Pointer::Pointer(Pointer &&P)
-    : Offset(P.Offset), StorageKind(P.StorageKind),
-      PointeeStorage(P.PointeeStorage) {
+Pointer::Pointer(const Pointer &P)
+    : Offset(P.Offset), StorageKind(P.StorageKind) {
+  switch (StorageKind) {
+  case Storage::Int:
+    Int = P.Int;
+    break;
+  case Storage::Block:
+    BS = P.BS;
+    if (BS.Pointee)
+      BS.Pointee->addPointer(this);
+    break;
+  case Storage::Fn:
+    Fn = P.Fn;
+    break;
+  case Storage::Typeid:
+    Typeid = P.Typeid;
+    break;
+  }
+}
 
-  if (StorageKind == Storage::Block && PointeeStorage.BS.Pointee)
-    PointeeStorage.BS.Pointee->replacePointer(&P, this);
+Pointer::Pointer(Pointer &&P) : Offset(P.Offset), StorageKind(P.StorageKind) {
+  switch (StorageKind) {
+  case Storage::Int:
+    Int = P.Int;
+    break;
+  case Storage::Block:
+    BS = P.BS;
+    if (BS.Pointee)
+      BS.Pointee->replacePointer(&P, this);
+    break;
+  case Storage::Fn:
+    Fn = P.Fn;
+    break;
+  case Storage::Typeid:
+    Typeid = P.Typeid;
+    break;
+  }
 }
 
 Pointer::~Pointer() {
   if (!isBlockPointer())
     return;
 
-  if (Block *Pointee = PointeeStorage.BS.Pointee) {
+  if (Block *Pointee = BS.Pointee) {
     Pointee->removePointer(this);
-    PointeeStorage.BS.Pointee = nullptr;
+    BS.Pointee = nullptr;
     Pointee->cleanup();
   }
 }
@@ -73,13 +96,13 @@ Pointer &Pointer::operator=(const Pointer &P) {
   if (isBlockPointer()) {
     if (P.isBlockPointer() && this->block() == P.block()) {
       Offset = P.Offset;
-      PointeeStorage.BS.Base = P.PointeeStorage.BS.Base;
+      BS.Base = P.BS.Base;
       return *this;
     }
 
-    if (Block *Pointee = PointeeStorage.BS.Pointee) {
+    if (Block *Pointee = BS.Pointee) {
       Pointee->removePointer(this);
-      PointeeStorage.BS.Pointee = nullptr;
+      BS.Pointee = nullptr;
       Pointee->cleanup();
     }
   }
@@ -88,16 +111,16 @@ Pointer &Pointer::operator=(const Pointer &P) {
   Offset = P.Offset;
 
   if (P.isBlockPointer()) {
-    PointeeStorage.BS = P.PointeeStorage.BS;
+    BS = P.BS;
 
-    if (PointeeStorage.BS.Pointee)
-      PointeeStorage.BS.Pointee->addPointer(this);
+    if (BS.Pointee)
+      BS.Pointee->addPointer(this);
   } else if (P.isIntegralPointer()) {
-    PointeeStorage.Int = P.PointeeStorage.Int;
+    Int = P.Int;
   } else if (P.isFunctionPointer()) {
-    PointeeStorage.Fn = P.PointeeStorage.Fn;
+    Fn = P.Fn;
   } else if (P.isTypeidPointer()) {
-    PointeeStorage.Typeid = P.PointeeStorage.Typeid;
+    Typeid = P.Typeid;
   } else {
     assert(false && "Unhandled storage kind");
   }
@@ -110,13 +133,13 @@ Pointer &Pointer::operator=(Pointer &&P) {
   if (isBlockPointer()) {
     if (P.isBlockPointer() && this->block() == P.block()) {
       Offset = P.Offset;
-      PointeeStorage.BS.Base = P.PointeeStorage.BS.Base;
+      BS.Base = P.BS.Base;
       return *this;
     }
 
-    if (Block *Pointee = PointeeStorage.BS.Pointee) {
+    if (Block *Pointee = BS.Pointee) {
       Pointee->removePointer(this);
-      PointeeStorage.BS.Pointee = nullptr;
+      BS.Pointee = nullptr;
       Pointee->cleanup();
     }
   }
@@ -125,16 +148,16 @@ Pointer &Pointer::operator=(Pointer &&P) {
   Offset = P.Offset;
 
   if (P.isBlockPointer()) {
-    PointeeStorage.BS = P.PointeeStorage.BS;
+    BS = P.BS;
 
-    if (PointeeStorage.BS.Pointee)
-      PointeeStorage.BS.Pointee->addPointer(this);
+    if (BS.Pointee)
+      BS.Pointee->addPointer(this);
   } else if (P.isIntegralPointer()) {
-    PointeeStorage.Int = P.PointeeStorage.Int;
+    Int = P.Int;
   } else if (P.isFunctionPointer()) {
-    PointeeStorage.Fn = P.PointeeStorage.Fn;
+    Fn = P.Fn;
   } else if (P.isTypeidPointer()) {
-    PointeeStorage.Typeid = P.PointeeStorage.Typeid;
+    Typeid = P.Typeid;
   } else {
     assert(false && "Unhandled storage kind");
   }
@@ -163,12 +186,11 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   }
 
   if (isTypeidPointer()) {
-    TypeInfoLValue TypeInfo(PointeeStorage.Typeid.TypePtr);
-    return APValue(
-        APValue::LValueBase::getTypeInfo(
-            TypeInfo, QualType(PointeeStorage.Typeid.TypeInfoType, 0)),
-        CharUnits::Zero(), {},
-        /*OnePastTheEnd=*/false, /*IsNull=*/false);
+    TypeInfoLValue TypeInfo(Typeid.TypePtr);
+    return APValue(APValue::LValueBase::getTypeInfo(
+                       TypeInfo, QualType(Typeid.TypeInfoType, 0)),
+                   CharUnits::Zero(), {},
+                   /*OnePastTheEnd=*/false, /*IsNull=*/false);
   }
 
   // Build the lvalue base from the block.
@@ -179,10 +201,7 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
   else if (const auto *E = Desc->asExpr()) {
     if (block()->isDynamic()) {
       QualType AllocatedType = getDeclPtr().getFieldDesc()->getDataType(ASTCtx);
-      // FIXME: Suboptimal counting of dynamic allocations. Move this to Context
-      // or InterpState?
-      static int ReportedDynamicAllocs = 0;
-      DynamicAllocLValue DA(ReportedDynamicAllocs++);
+      DynamicAllocLValue DA(*block()->DynAllocId);
       Base = APValue::LValueBase::getDynamicAlloc(DA, AllocatedType);
     } else {
       Base = E;
@@ -212,7 +231,7 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
     UsePath = false;
 
   // Build the path into the object.
-  bool OnePastEnd = isOnePastEnd();
+  bool OnePastEnd = isOnePastEnd() && !isZeroSizeArray();
   Pointer Ptr = *this;
   while (Ptr.isField() || Ptr.isArrayElement()) {
 
@@ -259,10 +278,10 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
       Ptr = Ptr.getArray();
     } else {
       const Descriptor *Desc = Ptr.getFieldDesc();
-      bool IsVirtual = false;
 
       // Create a path entry for the field.
       if (const auto *BaseOrMember = Desc->asDecl()) {
+        bool IsVirtual = false;
         if (const auto *FD = dyn_cast<FieldDecl>(BaseOrMember)) {
           Ptr = Ptr.getBase();
           Offset += getFieldOffset(FD);
@@ -303,13 +322,13 @@ APValue Pointer::toAPValue(const ASTContext &ASTCtx) const {
 void Pointer::print(llvm::raw_ostream &OS) const {
   switch (StorageKind) {
   case Storage::Block: {
-    const Block *B = PointeeStorage.BS.Pointee;
+    const Block *B = BS.Pointee;
     OS << "(Block) " << B << " {";
 
     if (isRoot())
-      OS << "rootptr(" << PointeeStorage.BS.Base << "), ";
+      OS << "rootptr(" << BS.Base << "), ";
     else
-      OS << PointeeStorage.BS.Base << ", ";
+      OS << BS.Base << ", ";
 
     if (isElementPastEnd())
       OS << "pastend, ";
@@ -324,8 +343,7 @@ void Pointer::print(llvm::raw_ostream &OS) const {
   } break;
   case Storage::Int:
     OS << "(Int) {";
-    OS << PointeeStorage.Int.Value << " + " << Offset << ", "
-       << PointeeStorage.Int.Desc;
+    OS << Int.Value << " + " << Offset << ", " << Int.Desc;
     OS << "}";
     break;
   case Storage::Fn:
@@ -378,6 +396,8 @@ size_t Pointer::computeOffsetForComparison() const {
     }
 
     if (const Record *R = P.getBase().getRecord(); R && R->isUnion()) {
+      if (P.isOnePastEnd())
+        ++Result;
       // Direct child of a union - all have offset 0.
       P = P.getBase();
       continue;
@@ -413,18 +433,17 @@ bool Pointer::isInitialized() const {
   if (!isBlockPointer())
     return true;
 
-  if (isRoot() && PointeeStorage.BS.Base == sizeof(GlobalInlineDescriptor)) {
+  if (isRoot() && BS.Base == sizeof(GlobalInlineDescriptor)) {
     const GlobalInlineDescriptor &GD =
         *reinterpret_cast<const GlobalInlineDescriptor *>(block()->rawData());
     return GD.InitState == GlobalInitState::Initialized;
   }
 
-  assert(PointeeStorage.BS.Pointee &&
-         "Cannot check if null pointer was initialized");
+  assert(BS.Pointee && "Cannot check if null pointer was initialized");
   const Descriptor *Desc = getFieldDesc();
   assert(Desc);
   if (Desc->isPrimitiveArray()) {
-    if (isStatic() && PointeeStorage.BS.Base == 0)
+    if (isStatic() && BS.Base == 0)
       return true;
 
     InitMapPtr &IM = getInitMap();
@@ -449,9 +468,9 @@ void Pointer::initialize() const {
   if (!isBlockPointer())
     return;
 
-  assert(PointeeStorage.BS.Pointee && "Cannot initialize null pointer");
+  assert(BS.Pointee && "Cannot initialize null pointer");
 
-  if (isRoot() && PointeeStorage.BS.Base == sizeof(GlobalInlineDescriptor)) {
+  if (isRoot() && BS.Base == sizeof(GlobalInlineDescriptor)) {
     GlobalInlineDescriptor &GD = *reinterpret_cast<GlobalInlineDescriptor *>(
         asBlockPointer().Pointee->rawData());
     GD.InitState = GlobalInitState::Initialized;
@@ -462,7 +481,7 @@ void Pointer::initialize() const {
   assert(Desc);
   if (Desc->isPrimitiveArray()) {
     // Primitive global arrays don't have an initmap.
-    if (isStatic() && PointeeStorage.BS.Base == 0)
+    if (isStatic() && BS.Base == 0)
       return;
 
     // Nothing to do for these.
@@ -488,8 +507,7 @@ void Pointer::initialize() const {
   }
 
   // Field has its bit in an inline descriptor.
-  assert(PointeeStorage.BS.Base != 0 &&
-         "Only composite fields can be initialised");
+  assert(BS.Base != 0 && "Only composite fields can be initialised");
   getInlineDesc()->IsInitialized = true;
 }
 
@@ -508,10 +526,9 @@ void Pointer::initializeAllElements() const {
 
 void Pointer::activate() const {
   // Field has its bit in an inline descriptor.
-  assert(PointeeStorage.BS.Base != 0 &&
-         "Only composite fields can be activated");
+  assert(BS.Base != 0 && "Only composite fields can be activated");
 
-  if (isRoot() && PointeeStorage.BS.Base == sizeof(GlobalInlineDescriptor))
+  if (isRoot() && BS.Base == sizeof(GlobalInlineDescriptor))
     return;
   if (!getInlineDesc()->InUnion)
     return;
@@ -593,8 +610,7 @@ bool Pointer::pointToSameBlock(const Pointer &A, const Pointer &B) {
 }
 
 bool Pointer::hasSameArray(const Pointer &A, const Pointer &B) {
-  return hasSameBase(A, B) &&
-         A.PointeeStorage.BS.Base == B.PointeeStorage.BS.Base &&
+  return hasSameBase(A, B) && A.BS.Base == B.BS.Base &&
          A.getFieldDesc()->IsArray;
 }
 
@@ -689,7 +705,7 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
       assert(Record && "Missing record descriptor");
 
       bool Ok = true;
-      if (RT->getDecl()->isUnion()) {
+      if (RT->getOriginalDecl()->isUnion()) {
         const FieldDecl *ActiveField = nullptr;
         APValue Value;
         for (const auto &F : Record->fields()) {
@@ -728,14 +744,15 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
 
         for (unsigned I = 0; I < NB; ++I) {
           const Record::Base *BD = Record->getBase(I);
-          QualType BaseTy = Ctx.getASTContext().getRecordType(BD->Decl);
+          QualType BaseTy = Ctx.getASTContext().getCanonicalTagType(BD->Decl);
           const Pointer &BP = Ptr.atField(BD->Offset);
           Ok &= Composite(BaseTy, BP, R.getStructBase(I));
         }
 
         for (unsigned I = 0; I < NV; ++I) {
           const Record::Base *VD = Record->getVirtualBase(I);
-          QualType VirtBaseTy = Ctx.getASTContext().getRecordType(VD->Decl);
+          QualType VirtBaseTy =
+              Ctx.getASTContext().getCanonicalTagType(VD->Decl);
           const Pointer &VP = Ptr.atField(VD->Offset);
           Ok &= Composite(VirtBaseTy, VP, R.getStructBase(NB + I));
         }
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index 5bafc5b108ee9..66e85f93a124e 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -28,8 +28,6 @@ class Block;
 class DeadBlock;
 class Pointer;
 class Context;
-template <unsigned A, bool B> class Integral;
-enum PrimType : unsigned;
 
 class Pointer;
 inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Pointer &P);
@@ -95,31 +93,21 @@ class Pointer {
   static constexpr unsigned RootPtrMark = ~0u;
 
 public:
-  Pointer() {
-    StorageKind = Storage::Int;
-    PointeeStorage.Int.Value = 0;
-    PointeeStorage.Int.Desc = nullptr;
-  }
-  Pointer(IntPointer &&IntPtr) : StorageKind(Storage::Int) {
-    PointeeStorage.Int = std::move(IntPtr);
-  }
+  Pointer() : StorageKind(Storage::Int), Int{nullptr, 0} {}
+  Pointer(IntPointer &&IntPtr)
+      : StorageKind(Storage::Int), Int(std::move(IntPtr)) {}
   Pointer(Block *B);
   Pointer(Block *B, uint64_t BaseAndOffset);
   Pointer(const Pointer &P);
   Pointer(Pointer &&P);
   Pointer(uint64_t Address, const Descriptor *Desc, uint64_t Offset = 0)
-      : Offset(Offset), StorageKind(Storage::Int) {
-    PointeeStorage.Int.Value = Address;
-    PointeeStorage.Int.Desc = Desc;
-  }
+      : Offset(Offset), StorageKind(Storage::Int), Int{Desc, Address} {}
   Pointer(const Function *F, uint64_t Offset = 0)
-      : Offset(Offset), StorageKind(Storage::Fn) {
-    PointeeStorage.Fn = FunctionPointer(F);
-  }
+      : Offset(Offset), StorageKind(Storage::Fn), Fn(F) {}
   Pointer(const Type *TypePtr, const Type *TypeInfoType, uint64_t Offset = 0)
       : Offset(Offset), StorageKind(Storage::Typeid) {
-    PointeeStorage.Typeid.TypePtr = TypePtr;
-    PointeeStorage.Typeid.TypeInfoType = TypeInfoType;
+    Typeid.TypePtr = TypePtr;
+    Typeid.TypeInfoType = TypeInfoType;
   }
   Pointer(Block *Pointee, unsigned Base, uint64_t Offset);
   ~Pointer();
@@ -282,7 +270,7 @@ class Pointer {
   bool isLive() const {
     if (!isBlockPointer())
       return true;
-    return asBlockPointer().Pointee && !asBlockPointer().Pointee->IsDead;
+    return asBlockPointer().Pointee && !asBlockPointer().Pointee->isDead();
   }
   /// Checks if the item is a field in an object.
   bool isField() const {
@@ -350,7 +338,9 @@ class Pointer {
   /// Returns the type of the innermost field.
   QualType getType() const {
     if (isTypeidPointer())
-      return QualType(PointeeStorage.Typeid.TypeInfoType, 0);
+      return QualType(Typeid.TypeInfoType, 0);
+    if (isFunctionPointer())
+      return asFunctionPointer().getFunction()->getDecl()->getType();
 
     if (inPrimitiveArray() && Offset != asBlockPointer().Base) {
       // Unfortunately, complex and vector types are not array types in clang,
@@ -468,19 +458,19 @@ class Pointer {
 
   [[nodiscard]] const BlockPointer &asBlockPointer() const {
     assert(isBlockPointer());
-    return PointeeStorage.BS;
+    return BS;
   }
   [[nodiscard]] const IntPointer &asIntPointer() const {
     assert(isIntegralPointer());
-    return PointeeStorage.Int;
+    return Int;
   }
   [[nodiscard]] const FunctionPointer &asFunctionPointer() const {
     assert(isFunctionPointer());
-    return PointeeStorage.Fn;
+    return Fn;
   }
   [[nodiscard]] const TypeidPointer &asTypeidPointer() const {
     assert(isTypeidPointer());
-    return PointeeStorage.Typeid;
+    return Typeid;
   }
 
   bool isBlockPointer() const { return StorageKind == Storage::Block; }
@@ -568,10 +558,9 @@ class Pointer {
     if (!isBlockPointer())
       return false;
 
-    if (!asBlockPointer().Pointee)
-      return false;
-
-    return getDeclDesc()->isDummy();
+    if (const Block *Pointee = asBlockPointer().Pointee)
+      return Pointee->isDummy();
+    return false;
   }
 
   /// Checks if an object or a subfield is mutable.
@@ -594,7 +583,7 @@ class Pointer {
   }
 
   /// Returns the declaration ID.
-  std::optional<unsigned> getDeclID() const {
+  UnsignedOrNone getDeclID() const {
     if (isBlockPointer()) {
       assert(asBlockPointer().Pointee);
       return asBlockPointer().Pointee->getDeclID();
@@ -658,7 +647,7 @@ class Pointer {
     if (isUnknownSizeArray())
       return false;
 
-    return isPastEnd() || (getSize() == getOffset() && !isZeroSizeArray());
+    return isPastEnd() || (getSize() == getOffset());
   }
 
   /// Checks if the pointer points past the end of the object.
@@ -666,7 +655,7 @@ class Pointer {
     if (isIntegralPointer())
       return false;
 
-    return !isZero() && Offset > PointeeStorage.BS.Pointee->getSize();
+    return !isZero() && Offset > BS.Pointee->getSize();
   }
 
   /// Checks if the pointer is an out-of-bounds element pointer.
@@ -805,6 +794,7 @@ class Pointer {
   friend class InterpState;
   friend struct InitMap;
   friend class DynamicAllocator;
+  friend class Program;
 
   /// Returns the embedded descriptor preceding a field.
   InlineDescriptor *getInlineDesc() const {
@@ -842,7 +832,7 @@ class Pointer {
     BlockPointer BS;
     FunctionPointer Fn;
     TypeidPointer Typeid;
-  } PointeeStorage;
+  };
 };
 
 inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Pointer &P) {
diff --git a/clang/lib/AST/ByteCode/PrimType.h b/clang/lib/AST/ByteCode/PrimType.h
index 724da93ca1ef6..6e3a49cadcbf1 100644
--- a/clang/lib/AST/ByteCode/PrimType.h
+++ b/clang/lib/AST/ByteCode/PrimType.h
@@ -31,7 +31,7 @@ template <bool Signed> class IntegralAP;
 template <unsigned Bits, bool Signed> class Integral;
 
 /// Enumeration of the primitive types of the VM.
-enum PrimType : unsigned {
+enum PrimType : uint8_t {
   PT_Sint8 = 0,
   PT_Uint8 = 1,
   PT_Sint16 = 2,
@@ -51,14 +51,15 @@ enum PrimType : unsigned {
 
 // Like std::optional<PrimType>, but only sizeof(PrimType).
 class OptPrimType final {
-  unsigned V = ~0u;
+  static constexpr uint8_t None = 0xFF;
+  uint8_t V = None;
 
 public:
   OptPrimType() = default;
   OptPrimType(std::nullopt_t) {}
   OptPrimType(PrimType T) : V(static_cast<unsigned>(T)) {}
 
-  explicit constexpr operator bool() const { return V != ~0u; }
+  explicit constexpr operator bool() const { return V != None; }
   PrimType operator*() const {
     assert(operator bool());
     return static_cast<PrimType>(V);
@@ -85,6 +86,19 @@ inline constexpr bool isPtrType(PrimType T) {
   return T == PT_Ptr || T == PT_MemberPtr;
 }
 
+inline constexpr bool isSignedType(PrimType T) {
+  switch (T) {
+  case PT_Sint8:
+  case PT_Sint16:
+  case PT_Sint32:
+  case PT_Sint64:
+    return true;
+  default:
+    return false;
+  }
+  return false;
+}
+
 enum class CastKind : uint8_t {
   Reinterpret,
   Volatile,
diff --git a/clang/lib/AST/ByteCode/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp
index 4daa4ab0d02f9..5d72044af969e 100644
--- a/clang/lib/AST/ByteCode/Program.cpp
+++ b/clang/lib/AST/ByteCode/Program.cpp
@@ -164,8 +164,8 @@ unsigned Program::getOrCreateDummy(const DeclTy &D) {
     const auto *VD = cast<ValueDecl>(cast<const Decl *>(D));
     IsWeak = VD->isWeak();
     QT = VD->getType();
-    if (const auto *RT = QT->getAs<ReferenceType>())
-      QT = RT->getPointeeType();
+    if (QT->isPointerOrReferenceType())
+      QT = QT->getPointeeType();
   }
   assert(!QT.isNull());
 
@@ -180,17 +180,15 @@ unsigned Program::getOrCreateDummy(const DeclTy &D) {
     Desc = allocateDescriptor(D);
 
   assert(Desc);
-  Desc->makeDummy();
-
-  assert(Desc->isDummy());
 
   // Allocate a block for storage.
   unsigned I = Globals.size();
 
   auto *G = new (Allocator, Desc->getAllocSize())
       Global(Ctx.getEvalID(), getCurrentDecl(), Desc, /*IsStatic=*/true,
-             /*IsExtern=*/false, IsWeak);
+             /*IsExtern=*/false, IsWeak, /*IsDummy=*/true);
   G->block()->invokeCtor();
+  assert(G->block()->isDummy());
 
   Globals.push_back(G);
   DummyVariables[D.getOpaqueValue()] = I;
@@ -215,19 +213,31 @@ std::optional<unsigned> Program::createGlobal(const ValueDecl *VD,
 
   // Register all previous declarations as well. For extern blocks, just replace
   // the index with the new variable.
-  if (auto Idx =
-          createGlobal(VD, VD->getType(), IsStatic, IsExtern, IsWeak, Init)) {
-    for (const Decl *P = VD; P; P = P->getPreviousDecl()) {
-      unsigned &PIdx = GlobalIndices[P];
-      if (P != VD) {
-        if (Globals[PIdx]->block()->isExtern())
-          Globals[PIdx] = Globals[*Idx];
+  std::optional<unsigned> Idx =
+      createGlobal(VD, VD->getType(), IsStatic, IsExtern, IsWeak, Init);
+  if (!Idx)
+    return std::nullopt;
+
+  Global *NewGlobal = Globals[*Idx];
+  for (const Decl *Redecl : VD->redecls()) {
+    unsigned &PIdx = GlobalIndices[Redecl];
+    if (Redecl != VD) {
+      if (Block *RedeclBlock = Globals[PIdx]->block();
+          RedeclBlock->isExtern()) {
+        Globals[PIdx] = NewGlobal;
+        // All pointers pointing to the previous extern decl now point to the
+        // new decl.
+        for (Pointer *Ptr = RedeclBlock->Pointers; Ptr; Ptr = Ptr->BS.Next) {
+          RedeclBlock->removePointer(Ptr);
+          Ptr->BS.Pointee = NewGlobal->block();
+          NewGlobal->block()->addPointer(Ptr);
+        }
       }
-      PIdx = *Idx;
     }
-    return *Idx;
+    PIdx = *Idx;
   }
-  return std::nullopt;
+
+  return *Idx;
 }
 
 std::optional<unsigned> Program::createGlobal(const Expr *E) {
@@ -266,7 +276,7 @@ std::optional<unsigned> Program::createGlobal(const DeclTy &D, QualType Ty,
       Ctx.getEvalID(), getCurrentDecl(), Desc, IsStatic, IsExtern, IsWeak);
   G->block()->invokeCtor();
 
-  // Initialize InlineDescriptor fields.
+  // Initialize GlobalInlineDescriptor fields.
   auto *GD = new (G->block()->rawData()) GlobalInlineDescriptor();
   if (!Init)
     GD->InitState = GlobalInitState::NoInitializer;
@@ -325,7 +335,7 @@ Record *Program::getOrCreateRecord(const RecordDecl *RD) {
       const auto *RT = Spec.getType()->getAs<RecordType>();
       if (!RT)
         return nullptr;
-      const RecordDecl *BD = RT->getDecl();
+      const RecordDecl *BD = RT->getOriginalDecl()->getDefinitionOrSelf();
       const Record *BR = getOrCreateRecord(BD);
 
       const Descriptor *Desc = GetBaseDesc(BD, BR);
@@ -342,7 +352,7 @@ Record *Program::getOrCreateRecord(const RecordDecl *RD) {
       if (!RT)
         return nullptr;
 
-      const RecordDecl *BD = RT->getDecl();
+      const RecordDecl *BD = RT->getOriginalDecl()->getDefinitionOrSelf();
       const Record *BR = getOrCreateRecord(BD);
 
       const Descriptor *Desc = GetBaseDesc(BD, BR);
@@ -399,7 +409,8 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
 
   // Classes and structures.
   if (const auto *RT = Ty->getAs<RecordType>()) {
-    if (const auto *Record = getOrCreateRecord(RT->getDecl()))
+    if (const auto *Record =
+            getOrCreateRecord(RT->getOriginalDecl()->getDefinitionOrSelf()))
       return allocateDescriptor(D, Record, MDSize, IsConst, IsTemporary,
                                 IsMutable, IsVolatile);
     return allocateDescriptor(D, MDSize);
diff --git a/clang/lib/AST/ByteCode/Program.h b/clang/lib/AST/ByteCode/Program.h
index 207ceef91da43..90b48ee5b669b 100644
--- a/clang/lib/AST/ByteCode/Program.h
+++ b/clang/lib/AST/ByteCode/Program.h
@@ -73,6 +73,10 @@ class Program final {
     return Globals[Idx]->block();
   }
 
+  bool isGlobalInitialized(unsigned Index) const {
+    return getPtrGlobal(Index).isInitialized();
+  }
+
   /// Finds a global's index.
   std::optional<unsigned> getGlobal(const ValueDecl *VD);
   std::optional<unsigned> getGlobal(const Expr *E);
@@ -152,7 +156,7 @@ class Program final {
   };
 
   /// Returns the current declaration ID.
-  std::optional<unsigned> getCurrentDecl() const {
+  UnsignedOrNone getCurrentDecl() const {
     if (CurrentDeclaration == NoDeclaration)
       return std::nullopt;
     return CurrentDeclaration;
@@ -172,9 +176,6 @@ class Program final {
   /// List of anonymous functions.
   std::vector<std::unique_ptr<Function>> AnonFuncs;
 
-  /// Function relocation locations.
-  llvm::DenseMap<const FunctionDecl *, std::vector<unsigned>> Relocs;
-
   /// Native pointers referenced by bytecode.
   std::vector<const void *> NativePointers;
   /// Cached native pointer indices.
diff --git a/clang/lib/AST/ByteCode/Record.cpp b/clang/lib/AST/ByteCode/Record.cpp
index 1d4ac7103cb76..a7934ccb4e55e 100644
--- a/clang/lib/AST/ByteCode/Record.cpp
+++ b/clang/lib/AST/ByteCode/Record.cpp
@@ -51,7 +51,7 @@ const Record::Base *Record::getBase(const RecordDecl *FD) const {
 
 const Record::Base *Record::getBase(QualType T) const {
   if (auto *RT = T->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     return BaseMap.lookup(RD);
   }
   return nullptr;
diff --git a/clang/lib/AST/CXXInheritance.cpp b/clang/lib/AST/CXXInheritance.cpp
index f0376168e43a2..e4b77edc063dc 100644
--- a/clang/lib/AST/CXXInheritance.cpp
+++ b/clang/lib/AST/CXXInheritance.cpp
@@ -132,8 +132,8 @@ bool CXXRecordDecl::forallBases(ForallBasesCallback BaseMatches) const {
       if (!Ty)
         return false;
 
-      CXXRecordDecl *Base =
-          cast_if_present<CXXRecordDecl>(Ty->getDecl()->getDefinition());
+      CXXRecordDecl *Base = cast_if_present<CXXRecordDecl>(
+          Ty->getOriginalDecl()->getDefinition());
       if (!Base ||
           (Base->isDependentContext() &&
            !Base->isCurrentInstantiation(Record))) {
@@ -256,7 +256,8 @@ bool CXXBasePaths::lookupInBases(ASTContext &Context,
             BaseSpec.getType()->getAs<TemplateSpecializationType>();
         if (!TST) {
           if (auto *RT = BaseSpec.getType()->getAs<RecordType>())
-            BaseRecord = cast<CXXRecordDecl>(RT->getDecl());
+            BaseRecord = cast<CXXRecordDecl>(RT->getOriginalDecl())
+                             ->getDefinitionOrSelf();
         } else {
           TemplateName TN = TST->getTemplateName();
           if (auto *TD =
@@ -336,7 +337,8 @@ bool CXXRecordDecl::lookupInBases(BaseMatchesCallback BaseMatches,
 
       CXXRecordDecl *VBase = nullptr;
       if (const RecordType *Record = PE.Base->getType()->getAs<RecordType>())
-        VBase = cast<CXXRecordDecl>(Record->getDecl());
+        VBase = cast<CXXRecordDecl>(Record->getOriginalDecl())
+                    ->getDefinitionOrSelf();
       if (!VBase)
         break;
 
@@ -348,7 +350,8 @@ bool CXXRecordDecl::lookupInBases(BaseMatchesCallback BaseMatches,
         CXXRecordDecl *HidingClass = nullptr;
         if (const RecordType *Record =
                 HidingP.back().Base->getType()->getAs<RecordType>())
-          HidingClass = cast<CXXRecordDecl>(Record->getDecl());
+          HidingClass = cast<CXXRecordDecl>(Record->getOriginalDecl())
+                            ->getDefinitionOrSelf();
         if (!HidingClass)
           break;
 
@@ -468,7 +471,8 @@ void FinalOverriderCollector::Collect(const CXXRecordDecl *RD,
 
   for (const auto &Base : RD->bases()) {
     if (const RecordType *RT = Base.getType()->getAs<RecordType>()) {
-      const CXXRecordDecl *BaseDecl = cast<CXXRecordDecl>(RT->getDecl());
+      const CXXRecordDecl *BaseDecl =
+          cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
       if (!BaseDecl->isPolymorphic())
         continue;
 
diff --git a/clang/lib/AST/Comment.cpp b/clang/lib/AST/Comment.cpp
index cd73d2709dc65..37e21c340c316 100644
--- a/clang/lib/AST/Comment.cpp
+++ b/clang/lib/AST/Comment.cpp
@@ -147,8 +147,6 @@ static TypeLoc lookThroughTypedefOrTypeAliasLocs(TypeLoc &SrcTL) {
     return BlockPointerTL.getPointeeLoc().getUnqualifiedLoc();
   if (MemberPointerTypeLoc MemberPointerTL = TL.getAs<MemberPointerTypeLoc>())
     return MemberPointerTL.getPointeeLoc().getUnqualifiedLoc();
-  if (ElaboratedTypeLoc ETL = TL.getAs<ElaboratedTypeLoc>())
-    return ETL.getNamedTypeLoc();
 
   return TL;
 }
diff --git a/clang/lib/AST/CommentLexer.cpp b/clang/lib/AST/CommentLexer.cpp
index e19c2327aebdc..a0903d0903dd8 100644
--- a/clang/lib/AST/CommentLexer.cpp
+++ b/clang/lib/AST/CommentLexer.cpp
@@ -214,7 +214,7 @@ bool isCommandNameStartCharacter(char C) {
 }
 
 bool isCommandNameCharacter(char C) {
-  return isAlphanumeric(C);
+  return isAsciiIdentifierContinue(C, false);
 }
 
 const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index e61846d241915..2e5821a8e4436 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/CommentParser.h"
+#include "clang/AST/Comment.h"
 #include "clang/AST/CommentCommandTraits.h"
 #include "clang/AST/CommentSema.h"
 #include "clang/Basic/CharInfo.h"
@@ -569,6 +570,8 @@ BlockCommandComment *Parser::parseBlockCommand() {
 
 InlineCommandComment *Parser::parseInlineCommand() {
   assert(Tok.is(tok::backslash_command) || Tok.is(tok::at_command));
+  CommandMarkerKind CMK =
+      Tok.is(tok::backslash_command) ? CMK_Backslash : CMK_At;
   const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
 
   const Token CommandTok = Tok;
@@ -580,7 +583,7 @@ InlineCommandComment *Parser::parseInlineCommand() {
 
   InlineCommandComment *IC = S.actOnInlineCommand(
       CommandTok.getLocation(), CommandTok.getEndLocation(),
-      CommandTok.getCommandID(), Args);
+      CommandTok.getCommandID(), CMK, Args);
 
   if (Args.size() < Info->NumArgs) {
     Diag(CommandTok.getEndLocation().getLocWithOffset(1),
diff --git a/clang/lib/AST/CommentSema.cpp b/clang/lib/AST/CommentSema.cpp
index 88520d7940e34..649fba94a2b76 100644
--- a/clang/lib/AST/CommentSema.cpp
+++ b/clang/lib/AST/CommentSema.cpp
@@ -363,12 +363,13 @@ void Sema::actOnTParamCommandFinish(TParamCommandComment *Command,
 InlineCommandComment *
 Sema::actOnInlineCommand(SourceLocation CommandLocBegin,
                          SourceLocation CommandLocEnd, unsigned CommandID,
+                         CommandMarkerKind CommandMarker,
                          ArrayRef<Comment::Argument> Args) {
   StringRef CommandName = Traits.getCommandInfo(CommandID)->Name;
 
-  return new (Allocator)
-      InlineCommandComment(CommandLocBegin, CommandLocEnd, CommandID,
-                           getInlineCommandRenderKind(CommandName), Args);
+  return new (Allocator) InlineCommandComment(
+      CommandLocBegin, CommandLocEnd, CommandID,
+      getInlineCommandRenderKind(CommandName), CommandMarker, Args);
 }
 
 InlineContentComment *Sema::actOnUnknownCommand(SourceLocation LocBegin,
@@ -905,17 +906,9 @@ bool Sema::isClassOrStructOrTagTypedefDecl() {
   if (isClassOrStructDeclImpl(ThisDeclInfo->CurrentDecl))
     return true;
 
-  if (auto *ThisTypedefDecl = dyn_cast<TypedefDecl>(ThisDeclInfo->CurrentDecl)) {
-    auto UnderlyingType = ThisTypedefDecl->getUnderlyingType();
-    if (auto ThisElaboratedType = dyn_cast<ElaboratedType>(UnderlyingType)) {
-      auto DesugaredType = ThisElaboratedType->desugar();
-      if (auto *DesugaredTypePtr = DesugaredType.getTypePtrOrNull()) {
-        if (auto *ThisRecordType = dyn_cast<RecordType>(DesugaredTypePtr)) {
-          return isClassOrStructDeclImpl(ThisRecordType->getAsRecordDecl());
-        }
-      }
-    }
-  }
+  if (auto *ThisTypedefDecl = dyn_cast<TypedefDecl>(ThisDeclInfo->CurrentDecl))
+    if (auto *D = ThisTypedefDecl->getUnderlyingType()->getAsRecordDecl())
+      return isClassOrStructDeclImpl(D);
 
   return false;
 }
diff --git a/clang/lib/AST/ComparisonCategories.cpp b/clang/lib/AST/ComparisonCategories.cpp
index 28244104d6636..0c7a7f4eacbbf 100644
--- a/clang/lib/AST/ComparisonCategories.cpp
+++ b/clang/lib/AST/ComparisonCategories.cpp
@@ -166,7 +166,7 @@ const ComparisonCategoryInfo &ComparisonCategories::getInfoForType(QualType Ty)
 
 QualType ComparisonCategoryInfo::getType() const {
   assert(Record);
-  return QualType(Record->getTypeForDecl(), 0);
+  return Record->getASTContext().getCanonicalTagType(Record);
 }
 
 StringRef ComparisonCategories::getCategoryString(ComparisonCategoryType Kind) {
diff --git a/clang/lib/AST/ComputeDependence.cpp b/clang/lib/AST/ComputeDependence.cpp
index 87334d98413e6..e0cf0deb12bd2 100644
--- a/clang/lib/AST/ComputeDependence.cpp
+++ b/clang/lib/AST/ComputeDependence.cpp
@@ -500,9 +500,8 @@ ExprDependence clang::computeDependence(OMPIteratorExpr *E) {
 ExprDependence clang::computeDependence(DeclRefExpr *E, const ASTContext &Ctx) {
   auto Deps = ExprDependence::None;
 
-  if (auto *NNS = E->getQualifier())
-    Deps |= toExprDependence(NNS->getDependence() &
-                             ~NestedNameSpecifierDependence::Dependent);
+  Deps |= toExprDependence(E->getQualifier().getDependence() &
+                           ~NestedNameSpecifierDependence::Dependent);
 
   if (auto *FirstArg = E->getTemplateArgs()) {
     unsigned NumArgs = E->getNumTemplateArgs();
@@ -673,9 +672,8 @@ ExprDependence clang::computeDependence(MemberExpr *E) {
   auto D = E->getBase()->getDependence();
   D |= getDependenceInExpr(E->getMemberNameInfo());
 
-  if (auto *NNS = E->getQualifier())
-    D |= toExprDependence(NNS->getDependence() &
-                          ~NestedNameSpecifierDependence::Dependent);
+  D |= toExprDependence(E->getQualifier().getDependence() &
+                        ~NestedNameSpecifierDependence::Dependent);
 
   for (const auto &A : E->template_arguments())
     D |= toExprDependence(A.getArgument().getDependence());
@@ -783,9 +781,8 @@ ExprDependence clang::computeDependence(CXXPseudoDestructorExpr *E) {
   if (auto *ST = E->getScopeTypeInfo())
     D |= turnTypeToValueDependence(
         toExprDependenceAsWritten(ST->getType()->getDependence()));
-  if (auto *Q = E->getQualifier())
-    D |= toExprDependence(Q->getDependence() &
-                          ~NestedNameSpecifierDependence::Dependent);
+  D |= toExprDependence(E->getQualifier().getDependence() &
+                        ~NestedNameSpecifierDependence::Dependent);
   return D;
 }
 
@@ -801,9 +798,8 @@ clang::computeDependence(OverloadExpr *E, bool KnownDependent,
   if (KnownContainsUnexpandedParameterPack)
     Deps |= ExprDependence::UnexpandedPack;
   Deps |= getDependenceInExpr(E->getNameInfo());
-  if (auto *Q = E->getQualifier())
-    Deps |= toExprDependence(Q->getDependence() &
-                             ~NestedNameSpecifierDependence::Dependent);
+  Deps |= toExprDependence(E->getQualifier().getDependence() &
+                           ~NestedNameSpecifierDependence::Dependent);
   for (auto *D : E->decls()) {
     if (D->getDeclContext()->isDependentContext() ||
         isa<UnresolvedUsingValueDecl>(D) || isa<TemplateTemplateParmDecl>(D))
@@ -820,8 +816,7 @@ clang::computeDependence(OverloadExpr *E, bool KnownDependent,
 ExprDependence clang::computeDependence(DependentScopeDeclRefExpr *E) {
   auto D = ExprDependence::TypeValue;
   D |= getDependenceInExpr(E->getNameInfo());
-  if (auto *Q = E->getQualifier())
-    D |= toExprDependence(Q->getDependence());
+  D |= toExprDependence(E->getQualifier().getDependence());
   for (const auto &A : E->template_arguments())
     D |= toExprDependence(A.getArgument().getDependence());
   return D;
@@ -872,8 +867,7 @@ ExprDependence clang::computeDependence(CXXDependentScopeMemberExpr *E) {
   auto D = ExprDependence::TypeValueInstantiation;
   if (!E->isImplicitAccess())
     D |= E->getBase()->getDependence();
-  if (auto *Q = E->getQualifier())
-    D |= toExprDependence(Q->getDependence());
+  D |= toExprDependence(E->getQualifier().getDependence());
   D |= getDependenceInExpr(E->getMemberNameInfo());
   for (const auto &A : E->template_arguments())
     D |= toExprDependence(A.getArgument().getDependence());
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 5471f31fb10e6..4507f415ce606 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -1604,17 +1604,20 @@ LinkageInfo LinkageComputer::getLVForDecl(const NamedDecl *D,
   // We have just computed the linkage for this decl. By induction we know
   // that all other computed linkages match, check that the one we just
   // computed also does.
-  NamedDecl *Old = nullptr;
-  for (auto *I : D->redecls()) {
-    auto *T = cast<NamedDecl>(I);
-    if (T == D)
+  // We can't assume the redecl chain is well formed at this point,
+  // so keep track of already visited declarations.
+  for (llvm::SmallPtrSet<const Decl *, 4> AlreadyVisited{D}; /**/; /**/) {
+    D = cast<NamedDecl>(const_cast<NamedDecl *>(D)->getNextRedeclarationImpl());
+    if (!AlreadyVisited.insert(D).second)
+      break;
+    if (D->isInvalidDecl())
       continue;
-    if (!T->isInvalidDecl() && T->hasCachedLinkage()) {
-      Old = T;
+    if (auto OldLinkage = D->getCachedLinkage();
+        OldLinkage != Linkage::Invalid) {
+      assert(LV.getLinkage() == OldLinkage);
       break;
     }
   }
-  assert(!Old || Old->getCachedLinkage() == D->getCachedLinkage());
 #endif
 
   return LV;
@@ -1693,9 +1696,9 @@ void NamedDecl::printQualifiedName(raw_ostream &OS,
     return;
   }
   printNestedNameSpecifier(OS, P);
-  if (getDeclName())
-    OS << *this;
-  else {
+  if (getDeclName()) {
+    printName(OS, P);
+  } else {
     // Give the printName override a chance to pick a different name before we
     // fall back to "(anonymous)".
     SmallString<64> NameBuffer;
@@ -1883,18 +1886,13 @@ bool NamedDecl::declarationReplaces(const NamedDecl *OldD,
 
   // Using declarations can be replaced if they import the same name from the
   // same context.
-  if (const auto *UD = dyn_cast<UsingDecl>(this)) {
-    ASTContext &Context = getASTContext();
-    return Context.getCanonicalNestedNameSpecifier(UD->getQualifier()) ==
-           Context.getCanonicalNestedNameSpecifier(
-               cast<UsingDecl>(OldD)->getQualifier());
-  }
-  if (const auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(this)) {
-    ASTContext &Context = getASTContext();
-    return Context.getCanonicalNestedNameSpecifier(UUVD->getQualifier()) ==
-           Context.getCanonicalNestedNameSpecifier(
-                        cast<UnresolvedUsingValueDecl>(OldD)->getQualifier());
-  }
+  if (const auto *UD = dyn_cast<UsingDecl>(this))
+    return UD->getQualifier().getCanonical() ==
+
+           cast<UsingDecl>(OldD)->getQualifier().getCanonical();
+  if (const auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(this))
+    return UUVD->getQualifier().getCanonical() ==
+           cast<UnresolvedUsingValueDecl>(OldD)->getQualifier().getCanonical();
 
   if (isRedeclarable(getKind())) {
     if (getCanonicalDecl() != OldD->getCanonicalDecl())
@@ -2864,7 +2862,8 @@ VarDecl::needsDestruction(const ASTContext &Ctx) const {
 bool VarDecl::hasFlexibleArrayInit(const ASTContext &Ctx) const {
   assert(hasInit() && "Expect initializer to check for flexible array init");
   auto *Ty = getType()->getAs<RecordType>();
-  if (!Ty || !Ty->getDecl()->hasFlexibleArrayMember())
+  if (!Ty ||
+      !Ty->getOriginalDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
     return false;
   auto *List = dyn_cast<InitListExpr>(getInit()->IgnoreParens());
   if (!List)
@@ -2879,7 +2878,10 @@ bool VarDecl::hasFlexibleArrayInit(const ASTContext &Ctx) const {
 CharUnits VarDecl::getFlexibleArrayInitChars(const ASTContext &Ctx) const {
   assert(hasInit() && "Expect initializer to check for flexible array init");
   auto *Ty = getType()->getAs<RecordType>();
-  if (!Ty || !Ty->getDecl()->hasFlexibleArrayMember())
+  if (!Ty)
+    return CharUnits::Zero();
+  const RecordDecl *RD = Ty->getOriginalDecl()->getDefinitionOrSelf();
+  if (!Ty || !RD->hasFlexibleArrayMember())
     return CharUnits::Zero();
   auto *List = dyn_cast<InitListExpr>(getInit()->IgnoreParens());
   if (!List || List->getNumInits() == 0)
@@ -2889,7 +2891,7 @@ CharUnits VarDecl::getFlexibleArrayInitChars(const ASTContext &Ctx) const {
   if (!InitTy)
     return CharUnits::Zero();
   CharUnits FlexibleArraySize = Ctx.getTypeSizeInChars(InitTy);
-  const ASTRecordLayout &RL = Ctx.getASTRecordLayout(Ty->getDecl());
+  const ASTRecordLayout &RL = Ctx.getASTRecordLayout(RD);
   CharUnits FlexibleArrayOffset =
       Ctx.toCharUnitsFromBits(RL.getFieldOffset(RL.getFieldCount() - 1));
   if (FlexibleArrayOffset + FlexibleArraySize < RL.getSize())
@@ -2991,7 +2993,10 @@ bool ParmVarDecl::isDestroyedInCallee() const {
   // FIXME: isParamDestroyedInCallee() should probably imply
   // isDestructedType()
   const auto *RT = getType()->getAs<RecordType>();
-  if (RT && RT->getDecl()->isParamDestroyedInCallee() &&
+  if (RT &&
+      RT->getOriginalDecl()
+          ->getDefinitionOrSelf()
+          ->isParamDestroyedInCallee() &&
       getType().isDestructedType())
     return true;
 
@@ -3502,7 +3507,7 @@ bool FunctionDecl::isUsableAsGlobalAllocationFunctionInConstantEvaluation(
     while (const auto *TD = T->getAs<TypedefType>())
       T = TD->getDecl()->getUnderlyingType();
     const IdentifierInfo *II =
-        T->castAs<EnumType>()->getDecl()->getIdentifier();
+        T->castAs<EnumType>()->getOriginalDecl()->getIdentifier();
     if (II && II->isStr("__hot_cold_t"))
       Consume();
   }
@@ -4653,7 +4658,7 @@ bool FieldDecl::isAnonymousStructOrUnion() const {
     return false;
 
   if (const auto *Record = getType()->getAs<RecordType>())
-    return Record->getDecl()->isAnonymousStructOrUnion();
+    return Record->getOriginalDecl()->isAnonymousStructOrUnion();
 
   return false;
 }
@@ -4713,7 +4718,7 @@ bool FieldDecl::isZeroSize(const ASTContext &Ctx) const {
   const auto *RT = getType()->getAs<RecordType>();
   if (!RT)
     return false;
-  const RecordDecl *RD = RT->getDecl()->getDefinition();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinition();
   if (!RD) {
     assert(isInvalidDecl() && "valid field has incomplete type");
     return false;
@@ -4836,10 +4841,6 @@ TagDecl *TagDecl::getCanonicalDecl() { return getFirstDecl(); }
 
 void TagDecl::setTypedefNameForAnonDecl(TypedefNameDecl *TDD) {
   TypedefNameDeclOrQualifier = TDD;
-  if (const Type *T = getTypeForDecl()) {
-    (void)T;
-    assert(T->isLinkageValid());
-  }
   assert(isLinkageValid());
 }
 
@@ -4867,25 +4868,16 @@ void TagDecl::completeDefinition() {
 }
 
 TagDecl *TagDecl::getDefinition() const {
-  if (isCompleteDefinition())
+  if (isCompleteDefinition() || isBeingDefined())
     return const_cast<TagDecl *>(this);
 
-  // If it's possible for us to have an out-of-date definition, check now.
-  if (mayHaveOutOfDateDef()) {
-    if (IdentifierInfo *II = getIdentifier()) {
-      if (II->isOutOfDate()) {
-        updateOutOfDate(*II);
-      }
-    }
-  }
-
   if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(this))
     return CXXRD->getDefinition();
 
-  for (auto *R : redecls())
-    if (R->isCompleteDefinition())
+  for (TagDecl *R :
+       redecl_range(redecl_iterator(getNextRedeclaration()), redecl_iterator()))
+    if (R->isCompleteDefinition() || R->isBeingDefined())
       return R;
-
   return nullptr;
 }
 
@@ -4919,7 +4911,7 @@ void TagDecl::printName(raw_ostream &OS, const PrintingPolicy &Policy) const {
     // is already printed as part of the type.
     PrintingPolicy Copy(Policy);
     Copy.SuppressScope = true;
-    getASTContext().getTagDeclType(this).print(OS, Copy);
+    QualType(getASTContext().getCanonicalTagType(this)).print(OS, Copy);
     return;
   }
   // Otherwise, do the normal printing.
@@ -4963,19 +4955,13 @@ EnumDecl *EnumDecl::Create(ASTContext &C, DeclContext *DC,
                            IdentifierInfo *Id,
                            EnumDecl *PrevDecl, bool IsScoped,
                            bool IsScopedUsingClassTag, bool IsFixed) {
-  auto *Enum = new (C, DC) EnumDecl(C, DC, StartLoc, IdLoc, Id, PrevDecl,
-                                    IsScoped, IsScopedUsingClassTag, IsFixed);
-  Enum->setMayHaveOutOfDateDef(C.getLangOpts().Modules);
-  C.getTypeDeclType(Enum, PrevDecl);
-  return Enum;
+  return new (C, DC) EnumDecl(C, DC, StartLoc, IdLoc, Id, PrevDecl, IsScoped,
+                              IsScopedUsingClassTag, IsFixed);
 }
 
 EnumDecl *EnumDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID) {
-  EnumDecl *Enum =
-      new (C, ID) EnumDecl(C, nullptr, SourceLocation(), SourceLocation(),
-                           nullptr, nullptr, false, false, false);
-  Enum->setMayHaveOutOfDateDef(C.getLangOpts().Modules);
-  return Enum;
+  return new (C, ID) EnumDecl(C, nullptr, SourceLocation(), SourceLocation(),
+                              nullptr, nullptr, false, false, false);
 }
 
 SourceRange EnumDecl::getIntegerTypeRange() const {
@@ -5035,7 +5021,7 @@ EnumDecl *EnumDecl::getTemplateInstantiationPattern() const {
       EnumDecl *ED = getInstantiatedFromMemberEnum();
       while (auto *NewED = ED->getInstantiatedFromMemberEnum())
         ED = NewED;
-      return getDefinitionOrSelf(ED);
+      return ::getDefinitionOrSelf(ED);
     }
   }
 
@@ -5125,21 +5111,15 @@ RecordDecl::RecordDecl(Kind DK, TagKind TK, const ASTContext &C,
 RecordDecl *RecordDecl::Create(const ASTContext &C, TagKind TK, DeclContext *DC,
                                SourceLocation StartLoc, SourceLocation IdLoc,
                                IdentifierInfo *Id, RecordDecl* PrevDecl) {
-  RecordDecl *R = new (C, DC) RecordDecl(Record, TK, C, DC,
-                                         StartLoc, IdLoc, Id, PrevDecl);
-  R->setMayHaveOutOfDateDef(C.getLangOpts().Modules);
-
-  C.getTypeDeclType(R, PrevDecl);
-  return R;
+  return new (C, DC)
+      RecordDecl(Record, TK, C, DC, StartLoc, IdLoc, Id, PrevDecl);
 }
 
 RecordDecl *RecordDecl::CreateDeserialized(const ASTContext &C,
                                            GlobalDeclID ID) {
-  RecordDecl *R = new (C, ID)
+  return new (C, ID)
       RecordDecl(Record, TagTypeKind::Struct, C, nullptr, SourceLocation(),
                  SourceLocation(), nullptr, nullptr);
-  R->setMayHaveOutOfDateDef(C.getLangOpts().Modules);
-  return R;
 }
 
 bool RecordDecl::isLambda() const {
@@ -5163,7 +5143,7 @@ bool RecordDecl::isOrContainsUnion() const {
   if (const RecordDecl *Def = getDefinition()) {
     for (const FieldDecl *FD : Def->fields()) {
       const RecordType *RT = FD->getType()->getAs<RecordType>();
-      if (RT && RT->getDecl()->isOrContainsUnion())
+      if (RT && RT->getOriginalDecl()->isOrContainsUnion())
         return true;
     }
   }
@@ -5295,8 +5275,9 @@ const FieldDecl *RecordDecl::findFirstNamedDataMember() const {
       return I;
 
     if (const auto *RT = I->getType()->getAs<RecordType>())
-      if (const FieldDecl *NamedDataMember =
-              RT->getDecl()->findFirstNamedDataMember())
+      if (const FieldDecl *NamedDataMember = RT->getOriginalDecl()
+                                                 ->getDefinitionOrSelf()
+                                                 ->findFirstNamedDataMember())
         return NamedDataMember;
   }
 
@@ -5658,14 +5639,14 @@ void TypedefNameDecl::anchor() {}
 
 TagDecl *TypedefNameDecl::getAnonDeclWithTypedefName(bool AnyRedecl) const {
   if (auto *TT = getTypeSourceInfo()->getType()->getAs<TagType>()) {
-    auto *OwningTypedef = TT->getDecl()->getTypedefNameForAnonDecl();
+    auto *OwningTypedef = TT->getOriginalDecl()->getTypedefNameForAnonDecl();
     auto *ThisTypedef = this;
     if (AnyRedecl && OwningTypedef) {
       OwningTypedef = OwningTypedef->getCanonicalDecl();
       ThisTypedef = ThisTypedef->getCanonicalDecl();
     }
     if (OwningTypedef == ThisTypedef)
-      return TT->getDecl();
+      return TT->getOriginalDecl()->getDefinitionOrSelf();
   }
 
   return nullptr;
@@ -5674,7 +5655,7 @@ TagDecl *TypedefNameDecl::getAnonDeclWithTypedefName(bool AnyRedecl) const {
 bool TypedefNameDecl::isTransparentTagSlow() const {
   auto determineIsTransparent = [&]() {
     if (auto *TT = getUnderlyingType()->getAs<TagType>()) {
-      if (auto *TD = TT->getDecl()) {
+      if (auto *TD = TT->getOriginalDecl()) {
         if (TD->getName() != getName())
           return false;
         SourceLocation TTLoc = getLocation();
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 13c46fdbff96a..680a4d74171a9 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -58,10 +58,6 @@ using namespace clang;
 #define ABSTRACT_DECL(DECL)
 #include "clang/AST/DeclNodes.inc"
 
-void Decl::updateOutOfDate(IdentifierInfo &II) const {
-  getASTContext().getExternalSource()->updateOutOfDateIdentifier(II);
-}
-
 #define DECL(DERIVED, BASE)                                                    \
   static_assert(alignof(Decl) >= alignof(DERIVED##Decl),                       \
                 "Alignment sufficient after objects prepended to " #DERIVED);
@@ -489,8 +485,7 @@ bool Decl::isFlexibleArrayMemberLike(
 
       // Look through typedefs.
       if (TypedefTypeLoc TTL = TL.getAsAdjusted<TypedefTypeLoc>()) {
-        const TypedefNameDecl *TDL = TTL.getTypedefNameDecl();
-        TInfo = TDL->getTypeSourceInfo();
+        TInfo = TTL.getDecl()->getTypeSourceInfo();
         continue;
       }
 
@@ -1512,30 +1507,19 @@ DeclContext *DeclContext::getPrimaryContext() {
   case Decl::ObjCCategoryImpl:
     return this;
 
-  default:
-    if (getDeclKind() >= Decl::firstTag && getDeclKind() <= Decl::lastTag) {
-      // If this is a tag type that has a definition or is currently
-      // being defined, that definition is our primary context.
-      auto *Tag = cast<TagDecl>(this);
-
-      if (TagDecl *Def = Tag->getDefinition())
-        return Def;
-
-      if (const auto *TagTy = dyn_cast<TagType>(Tag->getTypeForDecl())) {
-        // Note, TagType::getDecl returns the (partial) definition one exists.
-        TagDecl *PossiblePartialDef = TagTy->getDecl();
-        if (PossiblePartialDef->isBeingDefined())
-          return PossiblePartialDef;
-      } else {
-        assert(isa<InjectedClassNameType>(Tag->getTypeForDecl()));
-      }
-
-      return Tag;
-    }
+  // If this is a tag type that has a definition or is currently
+  // being defined, that definition is our primary context.
+  case Decl::ClassTemplatePartialSpecialization:
+  case Decl::ClassTemplateSpecialization:
+  case Decl::CXXRecord:
+    return cast<CXXRecordDecl>(this)->getDefinitionOrSelf();
+  case Decl::Record:
+  case Decl::Enum:
+    return cast<TagDecl>(this)->getDefinitionOrSelf();
 
+  default:
     assert(getDeclKind() >= Decl::firstFunction &&
-           getDeclKind() <= Decl::lastFunction &&
-          "Unknown DeclContext kind");
+           getDeclKind() <= Decl::lastFunction && "Unknown DeclContext kind");
     return this;
   }
 }
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 037a28c4255f4..50b1a1d000090 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -132,16 +132,9 @@ CXXRecordDecl::CXXRecordDecl(Kind K, TagKind TK, const ASTContext &C,
 CXXRecordDecl *CXXRecordDecl::Create(const ASTContext &C, TagKind TK,
                                      DeclContext *DC, SourceLocation StartLoc,
                                      SourceLocation IdLoc, IdentifierInfo *Id,
-                                     CXXRecordDecl *PrevDecl,
-                                     bool DelayTypeCreation) {
-  auto *R = new (C, DC) CXXRecordDecl(CXXRecord, TK, C, DC, StartLoc, IdLoc, Id,
-                                      PrevDecl);
-  R->setMayHaveOutOfDateDef(C.getLangOpts().Modules);
-
-  // FIXME: DelayTypeCreation seems like such a hack
-  if (!DelayTypeCreation)
-    C.getTypeDeclType(R, PrevDecl);
-  return R;
+                                     CXXRecordDecl *PrevDecl) {
+  return new (C, DC)
+      CXXRecordDecl(CXXRecord, TK, C, DC, StartLoc, IdLoc, Id, PrevDecl);
 }
 
 CXXRecordDecl *
@@ -154,10 +147,7 @@ CXXRecordDecl::CreateLambda(const ASTContext &C, DeclContext *DC,
   R->setBeingDefined(true);
   R->DefinitionData = new (C) struct LambdaDefinitionData(
       R, Info, DependencyKind, IsGeneric, CaptureDefault);
-  R->setMayHaveOutOfDateDef(false);
   R->setImplicit(true);
-
-  C.getTypeDeclType(R, /*PrevDecl=*/nullptr);
   return R;
 }
 
@@ -166,7 +156,6 @@ CXXRecordDecl *CXXRecordDecl::CreateDeserialized(const ASTContext &C,
   auto *R = new (C, ID)
       CXXRecordDecl(CXXRecord, TagTypeKind::Struct, C, nullptr,
                     SourceLocation(), SourceLocation(), nullptr, nullptr);
-  R->setMayHaveOutOfDateDef(false);
   return R;
 }
 
@@ -178,7 +167,7 @@ static bool hasRepeatedBaseClass(const CXXRecordDecl *StartRD) {
   SmallVector<const CXXRecordDecl*, 8> WorkList = {StartRD};
   while (!WorkList.empty()) {
     const CXXRecordDecl *RD = WorkList.pop_back_val();
-    if (RD->getTypeForDecl()->isDependentType())
+    if (RD->isDependentType())
       continue;
     for (const CXXBaseSpecifier &BaseSpec : RD->bases()) {
       if (const CXXRecordDecl *B = BaseSpec.getType()->getAsCXXRecordDecl()) {
@@ -228,7 +217,8 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
     if (BaseType->isDependentType())
       continue;
     auto *BaseClassDecl =
-        cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     // C++2a [class]p7:
     //   A standard-layout class is a class that:
@@ -1218,7 +1208,7 @@ void CXXRecordDecl::addedMember(Decl *D) {
     bool IsZeroSize = Field->isZeroSize(Context);
 
     if (const auto *RecordTy = T->getAs<RecordType>()) {
-      auto *FieldRec = cast<CXXRecordDecl>(RecordTy->getDecl());
+      auto *FieldRec = cast<CXXRecordDecl>(RecordTy->getOriginalDecl());
       if (FieldRec->getDefinition()) {
         addedClassSubobject(FieldRec);
 
@@ -1925,7 +1915,8 @@ static void CollectVisibleConversions(
       = CXXRecordDecl::MergeAccess(Access, I.getAccessSpecifier());
     bool BaseInVirtual = InVirtual || I.isVirtual();
 
-    auto *Base = cast<CXXRecordDecl>(RT->getDecl());
+    auto *Base =
+        cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
     CollectVisibleConversions(Context, Base, BaseInVirtual, BaseAccess,
                               *HiddenTypes, Output, VOutput, HiddenVBaseCs);
   }
@@ -1963,9 +1954,11 @@ static void CollectVisibleConversions(ASTContext &Context,
     const auto *RT = I.getType()->getAs<RecordType>();
     if (!RT) continue;
 
-    CollectVisibleConversions(Context, cast<CXXRecordDecl>(RT->getDecl()),
-                              I.isVirtual(), I.getAccessSpecifier(),
-                              HiddenTypes, Output, VBaseCs, HiddenVBaseCs);
+    CollectVisibleConversions(
+        Context,
+        cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf(),
+        I.isVirtual(), I.getAccessSpecifier(), HiddenTypes, Output, VBaseCs,
+        HiddenVBaseCs);
   }
 
   // Add any unhidden conversions provided by virtual bases.
@@ -2125,11 +2118,10 @@ const CXXRecordDecl *CXXRecordDecl::getTemplateInstantiationPattern() const {
 
 CXXDestructorDecl *CXXRecordDecl::getDestructor() const {
   ASTContext &Context = getASTContext();
-  QualType ClassType = Context.getTypeDeclType(this);
+  CanQualType ClassType = Context.getCanonicalTagType(this);
 
-  DeclarationName Name
-    = Context.DeclarationNames.getCXXDestructorName(
-                                          Context.getCanonicalType(ClassType));
+  DeclarationName Name =
+      Context.DeclarationNames.getCXXDestructorName(ClassType);
 
   DeclContext::lookup_result R = lookup(Name);
 
@@ -2159,6 +2151,29 @@ bool CXXRecordDecl::isInjectedClassName() const {
   return false;
 }
 
+bool CXXRecordDecl::hasInjectedClassType() const {
+  switch (getDeclKind()) {
+  case Decl::ClassTemplatePartialSpecialization:
+    return true;
+  case Decl::ClassTemplateSpecialization:
+    return false;
+  case Decl::CXXRecord:
+    return getDescribedClassTemplate() != nullptr;
+  default:
+    llvm_unreachable("unexpected decl kind");
+  }
+}
+
+CanQualType CXXRecordDecl::getCanonicalTemplateSpecializationType(
+    const ASTContext &Ctx) const {
+  if (auto *RD = dyn_cast<ClassTemplatePartialSpecializationDecl>(this))
+    return RD->getCanonicalInjectedSpecializationType(Ctx);
+  if (const ClassTemplateDecl *TD = getDescribedClassTemplate();
+      TD && !isa<ClassTemplateSpecializationDecl>(this))
+    return TD->getCanonicalInjectedSpecializationType(Ctx);
+  return CanQualType();
+}
+
 static bool isDeclContextInNamespace(const DeclContext *DC) {
   while (!DC->isTranslationUnit()) {
     if (DC->isNamespace())
@@ -2272,7 +2287,7 @@ void CXXRecordDecl::completeDefinition(CXXFinalOverriderMap *FinalOverriders) {
         Context.getDiagnostics().Report(
             AT->getLocation(),
             diag::warn_cxx20_compat_requires_explicit_init_non_aggregate)
-            << AT << FD << Context.getRecordType(this);
+            << AT << FD << Context.getCanonicalTagType(this);
     }
   }
 
@@ -2284,7 +2299,7 @@ void CXXRecordDecl::completeDefinition(CXXFinalOverriderMap *FinalOverriders) {
       if (const auto *AT = FD->getAttr<ExplicitInitAttr>())
         Context.getDiagnostics().Report(AT->getLocation(),
                                         diag::warn_attribute_needs_aggregate)
-            << AT << Context.getRecordType(this);
+            << AT << Context.getCanonicalTagType(this);
     }
     setHasUninitializedExplicitInitFields(false);
   }
@@ -2296,8 +2311,8 @@ bool CXXRecordDecl::mayBeAbstract() const {
     return false;
 
   for (const auto &B : bases()) {
-    const auto *BaseDecl =
-        cast<CXXRecordDecl>(B.getType()->castAs<RecordType>()->getDecl());
+    const auto *BaseDecl = cast<CXXRecordDecl>(
+        B.getType()->castAs<RecordType>()->getOriginalDecl());
     if (BaseDecl->isAbstract())
       return true;
   }
@@ -2460,7 +2475,8 @@ CXXMethodDecl::getCorrespondingMethodInClass(const CXXRecordDecl *RD,
     const RecordType *RT = I.getType()->getAs<RecordType>();
     if (!RT)
       continue;
-    const auto *Base = cast<CXXRecordDecl>(RT->getDecl());
+    const auto *Base =
+        cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
     if (CXXMethodDecl *D = this->getCorrespondingMethodInClass(Base))
       AddFinalOverrider(D);
   }
@@ -2712,8 +2728,7 @@ bool CXXMethodDecl::isCopyAssignmentOperator() const {
     ParamType = Ref->getPointeeType();
 
   ASTContext &Context = getASTContext();
-  QualType ClassType
-    = Context.getCanonicalType(Context.getTypeDeclType(getParent()));
+  CanQualType ClassType = Context.getCanonicalTagType(getParent());
   return Context.hasSameUnqualifiedType(ClassType, ParamType);
 }
 
@@ -2733,8 +2748,7 @@ bool CXXMethodDecl::isMoveAssignmentOperator() const {
   ParamType = ParamType->getPointeeType();
 
   ASTContext &Context = getASTContext();
-  QualType ClassType
-    = Context.getCanonicalType(Context.getTypeDeclType(getParent()));
+  CanQualType ClassType = Context.getCanonicalTagType(getParent());
   return Context.hasSameUnqualifiedType(ClassType, ParamType);
 }
 
@@ -2769,7 +2783,7 @@ CXXMethodDecl::overridden_methods() const {
 
 static QualType getThisObjectType(ASTContext &C, const FunctionProtoType *FPT,
                                   const CXXRecordDecl *Decl) {
-  QualType ClassTy = C.getTypeDeclType(Decl);
+  CanQualType ClassTy = C.getCanonicalTagType(Decl);
   return C.getQualifiedType(ClassTy, FPT->getMethodQuals());
 }
 
@@ -3027,11 +3041,9 @@ bool CXXConstructorDecl::isCopyOrMoveConstructor(unsigned &TypeQuals) const {
   // Is it a reference to our class type?
   ASTContext &Context = getASTContext();
 
-  CanQualType PointeeType
-    = Context.getCanonicalType(ParamRefType->getPointeeType());
-  CanQualType ClassTy
-    = Context.getCanonicalType(Context.getTagDeclType(getParent()));
-  if (PointeeType.getUnqualifiedType() != ClassTy)
+  QualType PointeeType = ParamRefType->getPointeeType();
+  CanQualType ClassTy = Context.getCanonicalTagType(getParent());
+  if (!Context.hasSameUnqualifiedType(PointeeType, ClassTy))
     return false;
 
   // FIXME: other qualifiers?
@@ -3066,15 +3078,11 @@ bool CXXConstructorDecl::isSpecializationCopyingObject() const {
   const ParmVarDecl *Param = getParamDecl(0);
 
   ASTContext &Context = getASTContext();
-  CanQualType ParamType = Context.getCanonicalType(Param->getType());
+  CanQualType ParamType = Param->getType()->getCanonicalTypeUnqualified();
 
   // Is it the same as our class type?
-  CanQualType ClassTy
-    = Context.getCanonicalType(Context.getTagDeclType(getParent()));
-  if (ParamType.getUnqualifiedType() != ClassTy)
-    return false;
-
-  return true;
+  CanQualType ClassTy = Context.getCanonicalTagType(getParent());
+  return ParamType == ClassTy;
 }
 
 void CXXDestructorDecl::anchor() {}
@@ -3371,7 +3379,7 @@ ConstructorUsingShadowDecl::CreateDeserialized(ASTContext &C, GlobalDeclID ID) {
 }
 
 CXXRecordDecl *ConstructorUsingShadowDecl::getNominatedBaseClass() const {
-  return getIntroducer()->getQualifier()->getAsRecordDecl();
+  return getIntroducer()->getQualifier().getAsRecordDecl();
 }
 
 void BaseUsingDecl::anchor() {}
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index f4265dd07bd46..196057f7b45a4 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -202,8 +202,7 @@ void Decl::printGroup(Decl** Begin, unsigned NumDecls,
   }
 
   Decl** End = Begin + NumDecls;
-  TagDecl* TD = dyn_cast<TagDecl>(*Begin);
-  if (TD)
+  if (isa<TagDecl>(*Begin))
     ++Begin;
 
   PrintingPolicy SubPolicy(Policy);
@@ -211,13 +210,9 @@ void Decl::printGroup(Decl** Begin, unsigned NumDecls,
   bool isFirst = true;
   for ( ; Begin != End; ++Begin) {
     if (isFirst) {
-      if(TD)
-        SubPolicy.IncludeTagDefinition = true;
-      SubPolicy.SuppressSpecifiers = false;
       isFirst = false;
     } else {
-      if (!isFirst) Out << ", ";
-      SubPolicy.IncludeTagDefinition = false;
+      Out << ", ";
       SubPolicy.SuppressSpecifiers = true;
     }
 
@@ -487,10 +482,12 @@ void DeclPrinter::VisitDeclContext(DeclContext *DC, bool Indent) {
     QualType CurDeclType = getDeclType(*D);
     if (!Decls.empty() && !CurDeclType.isNull()) {
       QualType BaseType = GetBaseType(CurDeclType);
-      if (!BaseType.isNull() && isa<ElaboratedType>(BaseType) &&
-          cast<ElaboratedType>(BaseType)->getOwnedTagDecl() == Decls[0]) {
-        Decls.push_back(*D);
-        continue;
+      if (const auto *TT = dyn_cast_or_null<TagType>(BaseType);
+          TT && TT->isTagOwned()) {
+        if (TT->getOriginalDecl() == Decls[0]) {
+          Decls.push_back(*D);
+          continue;
+        }
       }
     }
 
@@ -662,16 +659,6 @@ static void printExplicitSpecifier(ExplicitSpecifier ES, llvm::raw_ostream &Out,
   Out << Proto;
 }
 
-static void MaybePrintTagKeywordIfSupressingScopes(PrintingPolicy &Policy,
-                                                   QualType T,
-                                                   llvm::raw_ostream &Out) {
-  StringRef prefix = T->isClassType()       ? "class "
-                     : T->isStructureType() ? "struct "
-                     : T->isUnionType()     ? "union "
-                                            : "";
-  Out << prefix;
-}
-
 void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
   if (!D->getDescribedFunctionTemplate() &&
       !D->isFunctionTemplateSpecialization()) {
@@ -721,11 +708,8 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
     Proto += D->getQualifiedNameAsString();
   } else {
     llvm::raw_string_ostream OS(Proto);
-    if (!Policy.SuppressScope) {
-      if (const NestedNameSpecifier *NS = D->getQualifier()) {
-        NS->print(OS, Policy);
-      }
-    }
+    if (!Policy.SuppressScope)
+      D->getQualifier().print(OS, Policy);
     D->getNameInfo().printName(OS, Policy);
   }
 
@@ -833,10 +817,6 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
         Out << Proto << " -> ";
         Proto.clear();
       }
-      if (!Policy.SuppressTagKeyword && Policy.SuppressScope &&
-          !Policy.SuppressUnwrittenScope)
-        MaybePrintTagKeywordIfSupressingScopes(Policy, AFT->getReturnType(),
-                                               Out);
       AFT->getReturnType().print(Out, Policy, Proto);
       Proto.clear();
     }
@@ -995,10 +975,6 @@ void DeclPrinter::VisitVarDecl(VarDecl *D) {
     }
   }
 
-  if (!Policy.SuppressTagKeyword && Policy.SuppressScope &&
-      !Policy.SuppressUnwrittenScope)
-    MaybePrintTagKeywordIfSupressingScopes(Policy, T, Out);
-
   printDeclType(T, (isa<ParmVarDecl>(D) && Policy.CleanUglifiedParameters &&
                     D->getIdentifier())
                        ? D->getIdentifier()->deuglifiedName()
@@ -1028,7 +1004,6 @@ void DeclPrinter::VisitVarDecl(VarDecl *D) {
       }
       PrintingPolicy SubPolicy(Policy);
       SubPolicy.SuppressSpecifiers = false;
-      SubPolicy.IncludeTagDefinition = false;
       Init->printPretty(Out, nullptr, SubPolicy, Indentation, "\n", &Context);
       if ((D->getInitStyle() == VarDecl::CallInit) && !isa<ParenListExpr>(Init))
         Out << ")";
@@ -1086,15 +1061,13 @@ void DeclPrinter::VisitNamespaceDecl(NamespaceDecl *D) {
 
 void DeclPrinter::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
   Out << "using namespace ";
-  if (D->getQualifier())
-    D->getQualifier()->print(Out, Policy);
+  D->getQualifier().print(Out, Policy);
   Out << *D->getNominatedNamespaceAsWritten();
 }
 
 void DeclPrinter::VisitNamespaceAliasDecl(NamespaceAliasDecl *D) {
   Out << "namespace " << *D << " = ";
-  if (D->getQualifier())
-    D->getQualifier()->print(Out, Policy);
+  D->getQualifier().print(Out, Policy);
   Out << *D->getAliasedNamespace();
 }
 
@@ -1115,8 +1088,7 @@ void DeclPrinter::VisitCXXRecordDecl(CXXRecordDecl *D) {
     Out << ' ';
 
   if (D->getIdentifier()) {
-    if (auto *NNS = D->getQualifier())
-      NNS->print(Out, Policy);
+    D->getQualifier().print(Out, Policy);
     Out << *D;
 
     if (auto *S = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
@@ -1746,7 +1718,7 @@ void DeclPrinter::VisitUsingDecl(UsingDecl *D) {
     Out << "using ";
   if (D->hasTypename())
     Out << "typename ";
-  D->getQualifier()->print(Out, Policy);
+  D->getQualifier().print(Out, Policy);
 
   // Use the correct record name when the using declaration is used for
   // inheriting constructors.
@@ -1768,14 +1740,14 @@ void DeclPrinter::VisitUsingEnumDecl(UsingEnumDecl *D) {
 void
 DeclPrinter::VisitUnresolvedUsingTypenameDecl(UnresolvedUsingTypenameDecl *D) {
   Out << "using typename ";
-  D->getQualifier()->print(Out, Policy);
+  D->getQualifier().print(Out, Policy);
   Out << D->getDeclName();
 }
 
 void DeclPrinter::VisitUnresolvedUsingValueDecl(UnresolvedUsingValueDecl *D) {
   if (!D->isAccessDeclaration())
     Out << "using ";
-  D->getQualifier()->print(Out, Policy);
+  D->getQualifier().print(Out, Policy);
   Out << D->getDeclName();
 }
 
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index bc4a299e67f7c..b91b4670c63a3 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -307,8 +307,9 @@ bool TemplateDecl::hasAssociatedConstraints() const {
 bool TemplateDecl::isTypeAlias() const {
   switch (getKind()) {
   case TemplateDecl::TypeAliasTemplate:
-  case TemplateDecl::BuiltinTemplate:
     return true;
+  case TemplateDecl::BuiltinTemplate:
+    return !cast<BuiltinTemplateDecl>(this)->isPackProducingBuiltinTemplate();
   default:
     return false;
   };
@@ -632,7 +633,8 @@ ClassTemplateDecl::findPartialSpecialization(QualType T) {
   ASTContext &Context = getASTContext();
   for (ClassTemplatePartialSpecializationDecl &P :
        getPartialSpecializations()) {
-    if (Context.hasSameType(P.getInjectedSpecializationType(), T))
+    if (Context.hasSameType(P.getCanonicalInjectedSpecializationType(Context),
+                            T))
       return P.getMostRecentDecl();
   }
 
@@ -651,28 +653,20 @@ ClassTemplateDecl::findPartialSpecInstantiatedFromMember(
   return nullptr;
 }
 
-QualType
-ClassTemplateDecl::getInjectedClassNameSpecialization() {
+CanQualType ClassTemplateDecl::getCanonicalInjectedSpecializationType(
+    const ASTContext &Ctx) const {
   Common *CommonPtr = getCommonPtr();
-  if (!CommonPtr->InjectedClassNameType.isNull())
-    return CommonPtr->InjectedClassNameType;
-
-  // C++0x [temp.dep.type]p2:
-  //  The template argument list of a primary template is a template argument
-  //  list in which the nth template argument has the value of the nth template
-  //  parameter of the class template. If the nth template parameter is a
-  //  template parameter pack (14.5.3), the nth template argument is a pack
-  //  expansion (14.5.3) whose pattern is the name of the template parameter
-  //  pack.
-  ASTContext &Context = getASTContext();
-  TemplateName Name = Context.getQualifiedTemplateName(
-      /*NNS=*/nullptr, /*TemplateKeyword=*/false, TemplateName(this));
-  auto TemplateArgs = getTemplateParameters()->getInjectedTemplateArgs(Context);
-  CommonPtr->InjectedClassNameType =
-      Context.getTemplateSpecializationType(Name,
-                                            /*SpecifiedArgs=*/TemplateArgs,
-                                            /*CanonicalArgs=*/{});
-  return CommonPtr->InjectedClassNameType;
+
+  if (CommonPtr->CanonInjectedTST.isNull()) {
+    SmallVector<TemplateArgument> CanonicalArgs(
+        getTemplateParameters()->getInjectedTemplateArgs(Ctx));
+    Ctx.canonicalizeTemplateArguments(CanonicalArgs);
+    CommonPtr->CanonInjectedTST =
+        CanQualType::CreateUnsafe(Ctx.getCanonicalTemplateSpecializationType(
+            TemplateName(const_cast<ClassTemplateDecl *>(getCanonicalDecl())),
+            CanonicalArgs));
+  }
+  return CommonPtr->CanonInjectedTST;
 }
 
 //===----------------------------------------------------------------------===//
@@ -998,7 +992,6 @@ ClassTemplateSpecializationDecl *ClassTemplateSpecializationDecl::Create(
   auto *Result = new (Context, DC) ClassTemplateSpecializationDecl(
       Context, ClassTemplateSpecialization, TK, DC, StartLoc, IdLoc,
       SpecializedTemplate, Args, StrictPackMatch, PrevDecl);
-  Result->setMayHaveOutOfDateDef(false);
 
   // If the template decl is incomplete, copy the external lexical storage from
   // the base template. This allows instantiations of incomplete types to
@@ -1008,17 +1001,14 @@ ClassTemplateSpecializationDecl *ClassTemplateSpecializationDecl::Create(
     Result->setHasExternalLexicalStorage(
       SpecializedTemplate->getTemplatedDecl()->hasExternalLexicalStorage());
 
-  Context.getTypeDeclType(Result, PrevDecl);
   return Result;
 }
 
 ClassTemplateSpecializationDecl *
 ClassTemplateSpecializationDecl::CreateDeserialized(ASTContext &C,
                                                     GlobalDeclID ID) {
-  auto *Result =
-    new (C, ID) ClassTemplateSpecializationDecl(C, ClassTemplateSpecialization);
-  Result->setMayHaveOutOfDateDef(false);
-  return Result;
+  return new (C, ID)
+      ClassTemplateSpecializationDecl(C, ClassTemplateSpecialization);
 }
 
 void ClassTemplateSpecializationDecl::getNameForDiagnostic(
@@ -1180,13 +1170,15 @@ ClassTemplatePartialSpecializationDecl::ClassTemplatePartialSpecializationDecl(
     ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
     SourceLocation IdLoc, TemplateParameterList *Params,
     ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
+    CanQualType CanonInjectedTST,
     ClassTemplatePartialSpecializationDecl *PrevDecl)
     : ClassTemplateSpecializationDecl(
           Context, ClassTemplatePartialSpecialization, TK, DC, StartLoc, IdLoc,
           // Tracking StrictPackMatch for Partial
           // Specializations is not needed.
           SpecializedTemplate, Args, /*StrictPackMatch=*/false, PrevDecl),
-      TemplateParams(Params), InstantiatedFromMember(nullptr, false) {
+      TemplateParams(Params), InstantiatedFromMember(nullptr, false),
+      CanonInjectedTST(CanonInjectedTST) {
   if (AdoptTemplateParameterList(Params, this))
     setInvalidDecl();
 }
@@ -1196,24 +1188,31 @@ ClassTemplatePartialSpecializationDecl::Create(
     ASTContext &Context, TagKind TK, DeclContext *DC, SourceLocation StartLoc,
     SourceLocation IdLoc, TemplateParameterList *Params,
     ClassTemplateDecl *SpecializedTemplate, ArrayRef<TemplateArgument> Args,
-    QualType CanonInjectedType,
+    CanQualType CanonInjectedTST,
     ClassTemplatePartialSpecializationDecl *PrevDecl) {
   auto *Result = new (Context, DC) ClassTemplatePartialSpecializationDecl(
       Context, TK, DC, StartLoc, IdLoc, Params, SpecializedTemplate, Args,
-      PrevDecl);
+      CanonInjectedTST, PrevDecl);
   Result->setSpecializationKind(TSK_ExplicitSpecialization);
-  Result->setMayHaveOutOfDateDef(false);
-
-  Context.getInjectedClassNameType(Result, CanonInjectedType);
   return Result;
 }
 
 ClassTemplatePartialSpecializationDecl *
 ClassTemplatePartialSpecializationDecl::CreateDeserialized(ASTContext &C,
                                                            GlobalDeclID ID) {
-  auto *Result = new (C, ID) ClassTemplatePartialSpecializationDecl(C);
-  Result->setMayHaveOutOfDateDef(false);
-  return Result;
+  return new (C, ID) ClassTemplatePartialSpecializationDecl(C);
+}
+
+CanQualType
+ClassTemplatePartialSpecializationDecl::getCanonicalInjectedSpecializationType(
+    const ASTContext &Ctx) const {
+  if (CanonInjectedTST.isNull()) {
+    CanonInjectedTST =
+        CanQualType::CreateUnsafe(Ctx.getCanonicalTemplateSpecializationType(
+            TemplateName(getSpecializedTemplate()->getCanonicalDecl()),
+            getTemplateArgs().asArray()));
+  }
+  return CanonInjectedTST;
 }
 
 SourceRange ClassTemplatePartialSpecializationDecl::getSourceRange() const {
@@ -1600,6 +1599,16 @@ BuiltinTemplateDecl::BuiltinTemplateDecl(const ASTContext &C, DeclContext *DC,
                    createBuiltinTemplateParameterList(C, DC, BTK)),
       BTK(BTK) {}
 
+bool BuiltinTemplateDecl::isPackProducingBuiltinTemplate() const {
+  return getBuiltinTemplateKind() == clang::BTK__builtin_dedup_pack;
+}
+
+bool clang::isPackProducingBuiltinTemplateName(TemplateName N) {
+  auto *T = dyn_cast_or_null<BuiltinTemplateDecl>(
+      N.getAsTemplateDecl(/*IgnoreDeduced=*/true));
+  return T && T->isPackProducingBuiltinTemplate();
+}
+
 TemplateParamObjectDecl *TemplateParamObjectDecl::Create(const ASTContext &C,
                                                          QualType T,
                                                          const APValue &V) {
diff --git a/clang/lib/AST/DeclarationName.cpp b/clang/lib/AST/DeclarationName.cpp
index ae5fcf6e86adf..6c7b995d57567 100644
--- a/clang/lib/AST/DeclarationName.cpp
+++ b/clang/lib/AST/DeclarationName.cpp
@@ -115,12 +115,12 @@ static void printCXXConstructorDestructorName(QualType ClassType,
   Policy.adjustForCPlusPlus();
 
   if (const RecordType *ClassRec = ClassType->getAs<RecordType>()) {
-    ClassRec->getDecl()->printName(OS, Policy);
+    ClassRec->getOriginalDecl()->printName(OS, Policy);
     return;
   }
   if (Policy.SuppressTemplateArgsInCXXConstructors) {
     if (auto *InjTy = ClassType->getAs<InjectedClassNameType>()) {
-      InjTy->getDecl()->printName(OS, Policy);
+      InjTy->getOriginalDecl()->printName(OS, Policy);
       return;
     }
   }
@@ -184,7 +184,7 @@ void DeclarationName::print(raw_ostream &OS,
     OS << "operator ";
     QualType Type = getCXXNameType();
     if (const RecordType *Rec = Type->getAs<RecordType>()) {
-      OS << *Rec->getDecl();
+      OS << *Rec->getOriginalDecl();
       return;
     }
     // We know we're printing C++ here, ensure we print 'bool' properly.
diff --git a/clang/lib/AST/DynamicRecursiveASTVisitor.cpp b/clang/lib/AST/DynamicRecursiveASTVisitor.cpp
index b478e7a39ea18..8821cd332e918 100644
--- a/clang/lib/AST/DynamicRecursiveASTVisitor.cpp
+++ b/clang/lib/AST/DynamicRecursiveASTVisitor.cpp
@@ -115,8 +115,12 @@ template <bool Const> struct Impl : RecursiveASTVisitor<Impl<Const>> {
   bool TraverseAST(ASTContext &AST) { return Visitor.TraverseAST(AST); }
   bool TraverseAttr(Attr *At) { return Visitor.TraverseAttr(At); }
   bool TraverseDecl(Decl *D) { return Visitor.TraverseDecl(D); }
-  bool TraverseType(QualType T) { return Visitor.TraverseType(T); }
-  bool TraverseTypeLoc(TypeLoc TL) { return Visitor.TraverseTypeLoc(TL); }
+  bool TraverseType(QualType T, bool TraverseQualifier = true) {
+    return Visitor.TraverseType(T, TraverseQualifier);
+  }
+  bool TraverseTypeLoc(TypeLoc TL, bool TraverseQualifier = true) {
+    return Visitor.TraverseTypeLoc(TL, TraverseQualifier);
+  }
   bool TraverseStmt(Stmt *S) { return Visitor.TraverseStmt(S); }
 
   bool TraverseConstructorInitializer(CXXCtorInitializer *Init) {
@@ -172,7 +176,7 @@ template <bool Const> struct Impl : RecursiveASTVisitor<Impl<Const>> {
     return Visitor.TraverseLambdaCapture(LE, C, Init);
   }
 
-  bool TraverseNestedNameSpecifier(NestedNameSpecifier *NNS) {
+  bool TraverseNestedNameSpecifier(NestedNameSpecifier NNS) {
     return Visitor.TraverseNestedNameSpecifier(NNS);
   }
 
@@ -241,8 +245,8 @@ template <bool Const> struct Impl : RecursiveASTVisitor<Impl<Const>> {
   // Types.
 #define ABSTRACT_TYPE(CLASS, BASE)
 #define TYPE(CLASS, BASE)                                                      \
-  bool Traverse##CLASS##Type(CLASS##Type *T) {                                 \
-    return Visitor.Traverse##CLASS##Type(T);                                   \
+  bool Traverse##CLASS##Type(CLASS##Type *T, bool TraverseQualifier) {         \
+    return Visitor.Traverse##CLASS##Type(T, TraverseQualifier);                \
   }
 #include "clang/AST/TypeNodes.inc"
 
@@ -255,8 +259,8 @@ template <bool Const> struct Impl : RecursiveASTVisitor<Impl<Const>> {
   // TypeLocs.
 #define ABSTRACT_TYPELOC(CLASS, BASE)
 #define TYPELOC(CLASS, BASE)                                                   \
-  bool Traverse##CLASS##TypeLoc(CLASS##TypeLoc TL) {                           \
-    return Visitor.Traverse##CLASS##TypeLoc(TL);                               \
+  bool Traverse##CLASS##TypeLoc(CLASS##TypeLoc TL, bool TraverseQualifier) {   \
+    return Visitor.Traverse##CLASS##TypeLoc(TL, TraverseQualifier);            \
   }
 #include "clang/AST/TypeLocNodes.def"
 
@@ -297,7 +301,6 @@ FORWARD_TO_BASE(TraverseAttr, Attr, *)
 FORWARD_TO_BASE(TraverseConstructorInitializer, CXXCtorInitializer, *)
 FORWARD_TO_BASE(TraverseDecl, Decl, *)
 FORWARD_TO_BASE(TraverseStmt, Stmt, *)
-FORWARD_TO_BASE(TraverseNestedNameSpecifier, NestedNameSpecifier, *)
 FORWARD_TO_BASE(TraverseTemplateInstantiations, ClassTemplateDecl, *)
 FORWARD_TO_BASE(TraverseTemplateInstantiations, VarTemplateDecl, *)
 FORWARD_TO_BASE(TraverseTemplateInstantiations, FunctionTemplateDecl, *)
@@ -314,8 +317,22 @@ FORWARD_TO_BASE_EXACT(TraverseTemplateArgument, const TemplateArgument &)
 FORWARD_TO_BASE_EXACT(TraverseTemplateArguments, ArrayRef<TemplateArgument>)
 FORWARD_TO_BASE_EXACT(TraverseTemplateArgumentLoc, const TemplateArgumentLoc &)
 FORWARD_TO_BASE_EXACT(TraverseTemplateName, TemplateName)
-FORWARD_TO_BASE_EXACT(TraverseType, QualType)
-FORWARD_TO_BASE_EXACT(TraverseTypeLoc, TypeLoc)
+FORWARD_TO_BASE_EXACT(TraverseNestedNameSpecifier, NestedNameSpecifier)
+
+template <bool Const>
+bool DynamicRecursiveASTVisitorBase<Const>::TraverseType(
+    QualType T, bool TraverseQualifier) {
+  return Impl<Const>(*this).RecursiveASTVisitor<Impl<Const>>::TraverseType(
+      T, TraverseQualifier);
+}
+
+template <bool Const>
+bool DynamicRecursiveASTVisitorBase<Const>::TraverseTypeLoc(
+    TypeLoc TL, bool TraverseQualifier) {
+  return Impl<Const>(*this).RecursiveASTVisitor<Impl<Const>>::TraverseTypeLoc(
+      TL, TraverseQualifier);
+}
+
 FORWARD_TO_BASE_EXACT(TraverseTypeConstraint, const TypeConstraint *)
 FORWARD_TO_BASE_EXACT(TraverseObjCProtocolLoc, ObjCProtocolLoc)
 FORWARD_TO_BASE_EXACT(TraverseNestedNameSpecifierLoc, NestedNameSpecifierLoc)
@@ -354,13 +371,25 @@ bool DynamicRecursiveASTVisitorBase<Const>::dataTraverseNode(
 // Declare Traverse*() and friends for all concrete Type classes.
 #define ABSTRACT_TYPE(CLASS, BASE)
 #define TYPE(CLASS, BASE)                                                      \
-  FORWARD_TO_BASE(Traverse##CLASS##Type, CLASS##Type, *)                       \
+  template <bool Const>                                                        \
+  bool DynamicRecursiveASTVisitorBase<Const>::Traverse##CLASS##Type(           \
+      MaybeConst<CLASS##Type> *T, bool TraverseQualifier) {                    \
+    return Impl<Const>(*this)                                                  \
+        .RecursiveASTVisitor<Impl<Const>>::Traverse##CLASS##Type(              \
+            const_cast<CLASS##Type *>(T), TraverseQualifier);                  \
+  }                                                                            \
   FORWARD_TO_BASE(WalkUpFrom##CLASS##Type, CLASS##Type, *)
 #include "clang/AST/TypeNodes.inc"
 
 #define ABSTRACT_TYPELOC(CLASS, BASE)
 #define TYPELOC(CLASS, BASE)                                                   \
-  FORWARD_TO_BASE_EXACT(Traverse##CLASS##TypeLoc, CLASS##TypeLoc)
+  template <bool Const>                                                        \
+  bool DynamicRecursiveASTVisitorBase<Const>::Traverse##CLASS##TypeLoc(        \
+      CLASS##TypeLoc TL, bool TraverseQualifier) {                             \
+    return Impl<Const>(*this)                                                  \
+        .RecursiveASTVisitor<Impl<Const>>::Traverse##CLASS##TypeLoc(           \
+            TL, TraverseQualifier);                                            \
+  }
 #include "clang/AST/TypeLocNodes.def"
 
 #define TYPELOC(CLASS, BASE)                                                   \
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index cd9672d32a013..340de6d4be934 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -75,8 +75,7 @@ const CXXRecordDecl *Expr::getBestDynamicClassType() const {
     return nullptr;
 
   const RecordType *Ty = DerivedType->castAs<RecordType>();
-  Decl *D = Ty->getDecl();
-  return cast<CXXRecordDecl>(D);
+  return cast<CXXRecordDecl>(Ty->getOriginalDecl())->getDefinitionOrSelf();
 }
 
 const Expr *Expr::skipRValueSubobjectAdjustments(
@@ -92,7 +91,9 @@ const Expr *Expr::skipRValueSubobjectAdjustments(
           E->getType()->isRecordType()) {
         E = CE->getSubExpr();
         const auto *Derived =
-            cast<CXXRecordDecl>(E->getType()->castAs<RecordType>()->getDecl());
+            cast<CXXRecordDecl>(
+                E->getType()->castAs<RecordType>()->getOriginalDecl())
+                ->getDefinitionOrSelf();
         Adjustments.push_back(SubobjectAdjustment(CE, Derived));
         continue;
       }
@@ -268,7 +269,7 @@ QualType Expr::getEnumCoercedType(const ASTContext &Ctx) const {
   if (const auto *ECD = getEnumConstantDecl()) {
     const auto *ED = cast<EnumDecl>(ECD->getDeclContext());
     if (ED->isCompleteDefinition())
-      return Ctx.getTypeDeclType(ED);
+      return Ctx.getCanonicalTagType(ED);
   }
   return getType();
 }
@@ -2031,7 +2032,8 @@ CXXBaseSpecifier **CastExpr::path_buffer() {
 
 const FieldDecl *CastExpr::getTargetFieldForToUnionCast(QualType unionType,
                                                         QualType opType) {
-  auto RD = unionType->castAs<RecordType>()->getDecl();
+  auto RD =
+      unionType->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
   return getTargetFieldForToUnionCast(RD, opType);
 }
 
@@ -2774,23 +2776,22 @@ bool Expr::isUnusedResultAWarning(const Expr *&WarnE, SourceLocation &Loc,
   case UserDefinedLiteralClass: {
     // If this is a direct call, get the callee.
     const CallExpr *CE = cast<CallExpr>(this);
-    if (const Decl *FD = CE->getCalleeDecl()) {
-      // If the callee has attribute pure, const, or warn_unused_result, warn
-      // about it. void foo() { strlen("bar"); } should warn.
-      //
-      // Note: If new cases are added here, DiagnoseUnusedExprResult should be
-      // updated to match for QoI.
-      if (CE->hasUnusedResultAttr(Ctx) ||
-          FD->hasAttr<PureAttr>() || FD->hasAttr<ConstAttr>()) {
-        WarnE = this;
-        Loc = CE->getCallee()->getBeginLoc();
-        R1 = CE->getCallee()->getSourceRange();
-
-        if (unsigned NumArgs = CE->getNumArgs())
-          R2 = SourceRange(CE->getArg(0)->getBeginLoc(),
-                           CE->getArg(NumArgs - 1)->getEndLoc());
-        return true;
-      }
+    // If the callee has attribute pure, const, or warn_unused_result, warn
+    // about it. void foo() { strlen("bar"); } should warn.
+    // Note: If new cases are added here, DiagnoseUnusedExprResult should be
+    // updated to match for QoI.
+    const Decl *FD = CE->getCalleeDecl();
+    bool PureOrConst =
+        FD && (FD->hasAttr<PureAttr>() || FD->hasAttr<ConstAttr>());
+    if (CE->hasUnusedResultAttr(Ctx) || PureOrConst) {
+      WarnE = this;
+      Loc = getBeginLoc();
+      R1 = getSourceRange();
+
+      if (unsigned NumArgs = CE->getNumArgs())
+        R2 = SourceRange(CE->getArg(0)->getBeginLoc(),
+                         CE->getArg(NumArgs - 1)->getEndLoc());
+      return true;
     }
     return false;
   }
@@ -2803,32 +2804,20 @@ bool Expr::isUnusedResultAWarning(const Expr *&WarnE, SourceLocation &Loc,
 
   case CXXTemporaryObjectExprClass:
   case CXXConstructExprClass: {
-    if (const CXXRecordDecl *Type = getType()->getAsCXXRecordDecl()) {
-      const auto *WarnURAttr = Type->getAttr<WarnUnusedResultAttr>();
-      if (Type->hasAttr<WarnUnusedAttr>() ||
-          (WarnURAttr && WarnURAttr->IsCXX11NoDiscard())) {
-        WarnE = this;
-        Loc = getBeginLoc();
-        R1 = getSourceRange();
-        return true;
-      }
-    }
-
     const auto *CE = cast<CXXConstructExpr>(this);
-    if (const CXXConstructorDecl *Ctor = CE->getConstructor()) {
-      const auto *WarnURAttr = Ctor->getAttr<WarnUnusedResultAttr>();
-      if (WarnURAttr && WarnURAttr->IsCXX11NoDiscard()) {
-        WarnE = this;
-        Loc = getBeginLoc();
-        R1 = getSourceRange();
-
-        if (unsigned NumArgs = CE->getNumArgs())
-          R2 = SourceRange(CE->getArg(0)->getBeginLoc(),
-                           CE->getArg(NumArgs - 1)->getEndLoc());
-        return true;
-      }
-    }
+    const CXXRecordDecl *Type = getType()->getAsCXXRecordDecl();
 
+    if ((Type && Type->hasAttr<WarnUnusedAttr>()) ||
+        CE->hasUnusedResultAttr(Ctx)) {
+      WarnE = this;
+      Loc = getBeginLoc();
+      R1 = getSourceRange();
+
+      if (unsigned NumArgs = CE->getNumArgs())
+        R2 = SourceRange(CE->getArg(0)->getBeginLoc(),
+                         CE->getArg(NumArgs - 1)->getEndLoc());
+      return true;
+    }
     return false;
   }
 
@@ -3221,7 +3210,7 @@ static const Expr *skipTemporaryBindingsNoOpCastsAndParens(const Expr *E) {
 /// isTemporaryObject - Determines if this expression produces a
 /// temporary of the given class type.
 bool Expr::isTemporaryObject(ASTContext &C, const CXXRecordDecl *TempTy) const {
-  if (!C.hasSameUnqualifiedType(getType(), C.getTypeDeclType(TempTy)))
+  if (!C.hasSameUnqualifiedType(getType(), C.getCanonicalTagType(TempTy)))
     return false;
 
   const Expr *E = skipTemporaryBindingsNoOpCastsAndParens(this);
@@ -3407,7 +3396,10 @@ bool Expr::isConstantInitializer(ASTContext &Ctx, bool IsForRef,
 
     if (ILE->getType()->isRecordType()) {
       unsigned ElementNo = 0;
-      RecordDecl *RD = ILE->getType()->castAs<RecordType>()->getDecl();
+      RecordDecl *RD = ILE->getType()
+                           ->castAs<RecordType>()
+                           ->getOriginalDecl()
+                           ->getDefinitionOrSelf();
 
       // In C++17, bases were added to the list of members used by aggregate
       // initialization.
@@ -4050,8 +4042,10 @@ Expr::isNullPointerConstant(ASTContext &Ctx,
     return NPCK_CXX11_nullptr;
 
   if (const RecordType *UT = getType()->getAsUnionType())
-    if (!Ctx.getLangOpts().CPlusPlus11 &&
-        UT && UT->getDecl()->hasAttr<TransparentUnionAttr>())
+    if (!Ctx.getLangOpts().CPlusPlus11 && UT &&
+        UT->getOriginalDecl()
+            ->getMostRecentDecl()
+            ->hasAttr<TransparentUnionAttr>())
       if (const CompoundLiteralExpr *CLE = dyn_cast<CompoundLiteralExpr>(this)){
         const Expr *InitExpr = CLE->getInitializer();
         if (const InitListExpr *ILE = dyn_cast<InitListExpr>(InitExpr))
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index a099e97d86c50..97ae4a07f32aa 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -1319,7 +1319,7 @@ LambdaExpr *LambdaExpr::Create(const ASTContext &Context, CXXRecordDecl *Class,
                                bool ContainsUnexpandedParameterPack) {
   // Determine the type of the expression (i.e., the type of the
   // function object we're creating).
-  QualType T = Context.getTypeDeclType(Class);
+  CanQualType T = Context.getCanonicalTagType(Class);
 
   unsigned Size = totalSizeToAlloc<Stmt *>(CaptureInits.size() + 1);
   void *Mem = Context.Allocate(Size);
@@ -1687,10 +1687,9 @@ CXXRecordDecl *UnresolvedMemberExpr::getNamingClass() {
   // It can't be dependent: after all, we were actually able to do the
   // lookup.
   CXXRecordDecl *Record = nullptr;
-  auto *NNS = getQualifier();
-  if (NNS && NNS->getKind() != NestedNameSpecifier::Super) {
-    const Type *T = getQualifier()->getAsType();
-    assert(T && "qualifier in member expression does not name type");
+  if (NestedNameSpecifier Qualifier = getQualifier();
+      Qualifier.getKind() == NestedNameSpecifier::Kind::Type) {
+    const Type *T = getQualifier().getAsType();
     Record = T->getAsCXXRecordDecl();
     assert(Record && "qualifier in member expression does not name record");
   }
diff --git a/clang/lib/AST/ExprConcepts.cpp b/clang/lib/AST/ExprConcepts.cpp
index ac0e566fe6e72..36f910da49bfb 100644
--- a/clang/lib/AST/ExprConcepts.cpp
+++ b/clang/lib/AST/ExprConcepts.cpp
@@ -41,10 +41,10 @@ ConceptSpecializationExpr::ConceptSpecializationExpr(
   assert(!Loc->getNestedNameSpecifierLoc() ||
          (!Loc->getNestedNameSpecifierLoc()
                .getNestedNameSpecifier()
-               ->isInstantiationDependent() &&
+               .isInstantiationDependent() &&
           !Loc->getNestedNameSpecifierLoc()
                .getNestedNameSpecifier()
-               ->containsUnexpandedParameterPack()));
+               .containsUnexpandedParameterPack()));
   assert((!isValueDependent() || isInstantiationDependent()) &&
          "should not be value-dependent");
 }
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 34af9ccd2e96e..a71cb8b0143be 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -401,7 +401,7 @@ namespace {
       assert(!Invalid && "invalid designator has no subobject type");
       return MostDerivedPathLength == Entries.size()
                  ? MostDerivedType
-                 : Ctx.getRecordType(getAsBaseClass(Entries.back()));
+                 : Ctx.getCanonicalTagType(getAsBaseClass(Entries.back()));
     }
 
     /// Update this designator to refer to the first element within this array.
@@ -2623,7 +2623,8 @@ static bool CheckEvaluationResult(CheckEvaluationResultKind CERK,
         Value.getUnionValue(), Kind, Value.getUnionField(), CheckedTemps);
   }
   if (Value.isStruct()) {
-    RecordDecl *RD = Type->castAs<RecordType>()->getDecl();
+    RecordDecl *RD =
+        Type->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
     if (const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(RD)) {
       unsigned BaseIndex = 0;
       for (const CXXBaseSpecifier &BS : CD->bases()) {
@@ -4109,7 +4110,7 @@ findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj,
       }
 
       // Next subobject is a class, struct or union field.
-      RecordDecl *RD = ObjType->castAs<RecordType>()->getDecl();
+      RecordDecl *RD = ObjType->castAs<RecordType>()->getOriginalDecl();
       if (RD->isUnion()) {
         const FieldDecl *UnionField = O->getUnionField();
         if (!UnionField ||
@@ -4144,7 +4145,7 @@ findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj,
       const CXXRecordDecl *Base = getAsBaseClass(Sub.Entries[I]);
       O = &O->getStructBase(getBaseIndex(Derived, Base));
 
-      ObjType = getSubobjectType(ObjType, Info.Ctx.getRecordType(Base));
+      ObjType = getSubobjectType(ObjType, Info.Ctx.getCanonicalTagType(Base));
     }
   }
 }
@@ -6363,7 +6364,7 @@ static bool HandleDynamicCast(EvalInfo &Info, const ExplicitCastExpr *E,
 
   const CXXRecordDecl *C = E->getTypeAsWritten()->getPointeeCXXRecordDecl();
   assert(C && "dynamic_cast target is not void pointer nor class");
-  CanQualType CQT = Info.Ctx.getCanonicalType(Info.Ctx.getRecordType(C));
+  CanQualType CQT = Info.Ctx.getCanonicalTagType(C);
 
   auto RuntimeCheckFailed = [&] (CXXBasePaths *Paths) {
     // C++ [expr.dynamic.cast]p9:
@@ -6389,7 +6390,7 @@ static bool HandleDynamicCast(EvalInfo &Info, const ExplicitCastExpr *E,
     }
     Info.FFDiag(E, diag::note_constexpr_dynamic_cast_to_reference_failed)
         << DiagKind << Ptr.Designator.getType(Info.Ctx)
-        << Info.Ctx.getRecordType(DynType->Type)
+        << Info.Ctx.getCanonicalTagType(DynType->Type)
         << E->getType().getUnqualifiedType();
     return false;
   };
@@ -6886,8 +6887,8 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
             // FIXME: This immediately starts the lifetime of all members of
             // an anonymous struct. It would be preferable to strictly start
             // member lifetime in initialization order.
-            Success &=
-                handleDefaultInitValue(Info.Ctx.getRecordType(CD), *Value);
+            Success &= handleDefaultInitValue(Info.Ctx.getCanonicalTagType(CD),
+                                              *Value);
         }
         // Store Subobject as its parent before updating it for the last element
         // in the chain.
@@ -7794,7 +7795,8 @@ class BufferToAPValueConverter {
   }
 
   std::optional<APValue> visit(const EnumType *Ty, CharUnits Offset) {
-    QualType RepresentationType = Ty->getDecl()->getIntegerType();
+    QualType RepresentationType =
+        Ty->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
     assert(!RepresentationType.isNull() &&
            "enum forward decl should be caught by Sema");
     const auto *AsBuiltin =
@@ -8528,7 +8530,7 @@ class ExprEvaluatorBase
     if (auto *DD = dyn_cast<CXXDestructorDecl>(FD)) {
       assert(This && "no 'this' pointer for destructor call");
       return HandleDestruction(Info, E, *This,
-                               Info.Ctx.getRecordType(DD->getParent())) &&
+                               Info.Ctx.getCanonicalTagType(DD->getParent())) &&
              CallScope.destroy();
     }
 
@@ -8589,8 +8591,10 @@ class ExprEvaluatorBase
     const FieldDecl *FD = dyn_cast<FieldDecl>(E->getMemberDecl());
     if (!FD) return Error(E);
     assert(!FD->getType()->isReferenceType() && "prvalue reference?");
-    assert(BaseTy->castAs<RecordType>()->getDecl()->getCanonicalDecl() ==
-           FD->getParent()->getCanonicalDecl() && "record / field mismatch");
+    assert(
+        BaseTy->castAs<RecordType>()->getOriginalDecl()->getCanonicalDecl() ==
+            FD->getParent()->getCanonicalDecl() &&
+        "record / field mismatch");
 
     // Note: there is no lvalue base here. But this case should only ever
     // happen in C or in C++98, where we cannot be evaluating a constexpr
@@ -8817,8 +8821,10 @@ class LValueExprEvaluatorBase
 
     const ValueDecl *MD = E->getMemberDecl();
     if (const FieldDecl *FD = dyn_cast<FieldDecl>(E->getMemberDecl())) {
-      assert(BaseTy->castAs<RecordType>()->getDecl()->getCanonicalDecl() ==
-             FD->getParent()->getCanonicalDecl() && "record / field mismatch");
+      assert(
+          BaseTy->castAs<RecordType>()->getOriginalDecl()->getCanonicalDecl() ==
+              FD->getParent()->getCanonicalDecl() &&
+          "record / field mismatch");
       (void)BaseTy;
       if (!HandleLValueMember(this->Info, E, Result, FD))
         return false;
@@ -9257,8 +9263,8 @@ bool LValueExprEvaluator::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
     if (!DynType)
       return false;
 
-    TypeInfo =
-        TypeInfoLValue(Info.Ctx.getRecordType(DynType->Type).getTypePtr());
+    TypeInfo = TypeInfoLValue(
+        Info.Ctx.getCanonicalTagType(DynType->Type).getTypePtr());
   }
 
   return Success(APValue::LValueBase::getTypeInfo(TypeInfo, E->getType()));
@@ -9860,11 +9866,15 @@ bool PointerExprEvaluator::VisitCastExpr(const CastExpr *E) {
     if (Value.isInt()) {
       unsigned Size = Info.Ctx.getTypeSize(E->getType());
       uint64_t N = Value.getInt().extOrTrunc(Size).getZExtValue();
-      Result.Base = (Expr*)nullptr;
-      Result.InvalidBase = false;
-      Result.Offset = CharUnits::fromQuantity(N);
-      Result.Designator.setInvalid();
-      Result.IsNullPtr = false;
+      if (N == Info.Ctx.getTargetNullPointerValue(E->getType())) {
+        Result.setNull(Info.Ctx, E->getType());
+      } else {
+        Result.Base = (Expr *)nullptr;
+        Result.InvalidBase = false;
+        Result.Offset = CharUnits::fromQuantity(N);
+        Result.Designator.setInvalid();
+        Result.IsNullPtr = false;
+      }
       return true;
     } else {
       // In rare instances, the value isn't an lvalue.
@@ -10814,7 +10824,8 @@ static bool HandleClassZeroInitialization(EvalInfo &Info, const Expr *E,
 }
 
 bool RecordExprEvaluator::ZeroInitialization(const Expr *E, QualType T) {
-  const RecordDecl *RD = T->castAs<RecordType>()->getDecl();
+  const RecordDecl *RD =
+      T->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
   if (RD->isInvalidDecl()) return false;
   if (RD->isUnion()) {
     // C++11 [dcl.init]p5: If T is a (possibly cv-qualified) union type, the
@@ -10883,8 +10894,10 @@ bool RecordExprEvaluator::VisitInitListExpr(const InitListExpr *E) {
 
 bool RecordExprEvaluator::VisitCXXParenListOrInitListExpr(
     const Expr *ExprToVisit, ArrayRef<Expr *> Args) {
-  const RecordDecl *RD =
-      ExprToVisit->getType()->castAs<RecordType>()->getDecl();
+  const RecordDecl *RD = ExprToVisit->getType()
+                             ->castAs<RecordType>()
+                             ->getOriginalDecl()
+                             ->getDefinitionOrSelf();
   if (RD->isInvalidDecl()) return false;
   const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(RD);
   auto *CXXRD = dyn_cast<CXXRecordDecl>(RD);
@@ -11032,10 +11045,6 @@ bool RecordExprEvaluator::VisitCXXConstructExpr(const CXXConstructExpr *E,
 
   bool ZeroInit = E->requiresZeroInitialization();
   if (CheckTrivialDefaultConstructor(Info, E->getExprLoc(), FD, ZeroInit)) {
-    // If we've already performed zero-initialization, we're already done.
-    if (Result.hasValue())
-      return true;
-
     if (ZeroInit)
       return ZeroInitialization(E, T);
 
@@ -11112,7 +11121,10 @@ bool RecordExprEvaluator::VisitCXXStdInitializerListExpr(
   Result = APValue(APValue::UninitStruct(), 0, 2);
   Array.moveInto(Result.getStructField(0));
 
-  RecordDecl *Record = E->getType()->castAs<RecordType>()->getDecl();
+  RecordDecl *Record = E->getType()
+                           ->castAs<RecordType>()
+                           ->getOriginalDecl()
+                           ->getDefinitionOrSelf();
   RecordDecl::field_iterator Field = Record->field_begin();
   assert(Field != Record->field_end() &&
          Info.Ctx.hasSameType(Field->getType()->getPointeeType(),
@@ -11623,14 +11635,54 @@ bool VectorExprEvaluator::VisitCallExpr(const CallExpr *E) {
 
     return Success(APValue(ResultElements.data(), ResultElements.size()), E);
   }
+  case Builtin::BI__builtin_elementwise_abs: {
+    APValue Source;
+    if (!EvaluateAsRValue(Info, E->getArg(0), Source))
+      return false;
+
+    QualType DestEltTy = E->getType()->castAs<VectorType>()->getElementType();
+    unsigned SourceLen = Source.getVectorLength();
+    SmallVector<APValue, 4> ResultElements;
+    ResultElements.reserve(SourceLen);
+
+    for (unsigned EltNum = 0; EltNum < SourceLen; ++EltNum) {
+      APValue CurrentEle = Source.getVectorElt(EltNum);
+      APValue Val = DestEltTy->isFloatingType()
+                        ? APValue(llvm::abs(CurrentEle.getFloat()))
+                        : APValue(APSInt(
+                              CurrentEle.getInt().abs(),
+                              DestEltTy->isUnsignedIntegerOrEnumerationType()));
+      ResultElements.push_back(Val);
+    }
+
+    return Success(APValue(ResultElements.data(), ResultElements.size()), E);
+  }
+
   case Builtin::BI__builtin_elementwise_add_sat:
-  case Builtin::BI__builtin_elementwise_sub_sat: {
+  case Builtin::BI__builtin_elementwise_sub_sat:
+  case clang::X86::BI__builtin_ia32_pmulhuw128:
+  case clang::X86::BI__builtin_ia32_pmulhuw256:
+  case clang::X86::BI__builtin_ia32_pmulhuw512:
+  case clang::X86::BI__builtin_ia32_pmulhw128:
+  case clang::X86::BI__builtin_ia32_pmulhw256:
+  case clang::X86::BI__builtin_ia32_pmulhw512:
+  case clang::X86::BI__builtin_ia32_psllv2di:
+  case clang::X86::BI__builtin_ia32_psllv4di:
+  case clang::X86::BI__builtin_ia32_psllv4si:
+  case clang::X86::BI__builtin_ia32_psllv8si:
+  case clang::X86::BI__builtin_ia32_psrav4si:
+  case clang::X86::BI__builtin_ia32_psrav8si:
+  case clang::X86::BI__builtin_ia32_psrlv2di:
+  case clang::X86::BI__builtin_ia32_psrlv4di:
+  case clang::X86::BI__builtin_ia32_psrlv4si:
+  case clang::X86::BI__builtin_ia32_psrlv8si:{
     APValue SourceLHS, SourceRHS;
     if (!EvaluateAsRValue(Info, E->getArg(0), SourceLHS) ||
         !EvaluateAsRValue(Info, E->getArg(1), SourceRHS))
       return false;
 
     QualType DestEltTy = E->getType()->castAs<VectorType>()->getElementType();
+    bool DestUnsigned = DestEltTy->isUnsignedIntegerOrEnumerationType();
     unsigned SourceLen = SourceLHS.getVectorLength();
     SmallVector<APValue, 4> ResultElements;
     ResultElements.reserve(SourceLen);
@@ -11642,11 +11694,220 @@ bool VectorExprEvaluator::VisitCallExpr(const CallExpr *E) {
       case Builtin::BI__builtin_elementwise_add_sat:
         ResultElements.push_back(APValue(
             APSInt(LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS),
-                   DestEltTy->isUnsignedIntegerOrEnumerationType())));
+                   DestUnsigned)));
         break;
       case Builtin::BI__builtin_elementwise_sub_sat:
         ResultElements.push_back(APValue(
             APSInt(LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS),
+                   DestUnsigned)));
+        break;
+      case clang::X86::BI__builtin_ia32_pmulhuw128:
+      case clang::X86::BI__builtin_ia32_pmulhuw256:
+      case clang::X86::BI__builtin_ia32_pmulhuw512:
+        ResultElements.push_back(APValue(APSInt(llvm::APIntOps::mulhu(LHS, RHS),
+                                                /*isUnsigned=*/true)));
+        break;
+      case clang::X86::BI__builtin_ia32_pmulhw128:
+      case clang::X86::BI__builtin_ia32_pmulhw256:
+      case clang::X86::BI__builtin_ia32_pmulhw512:
+        ResultElements.push_back(APValue(APSInt(llvm::APIntOps::mulhs(LHS, RHS),
+                                                /*isUnsigned=*/false)));
+        break;
+      case clang::X86::BI__builtin_ia32_psllv2di:
+      case clang::X86::BI__builtin_ia32_psllv4di:
+      case clang::X86::BI__builtin_ia32_psllv4si:
+      case clang::X86::BI__builtin_ia32_psllv8si:
+        if (RHS.uge(RHS.getBitWidth())) {
+          ResultElements.push_back(
+              APValue(APSInt(APInt::getZero(RHS.getBitWidth()), DestUnsigned)));
+          break;
+        }
+        ResultElements.push_back(
+            APValue(APSInt(LHS.shl(RHS.getZExtValue()), DestUnsigned)));
+        break;
+      case clang::X86::BI__builtin_ia32_psrav4si:
+      case clang::X86::BI__builtin_ia32_psrav8si:
+        if (RHS.uge(RHS.getBitWidth())) {
+          ResultElements.push_back(
+              APValue(APSInt(LHS.ashr(RHS.getBitWidth() - 1), DestUnsigned)));
+          break;
+        }
+        ResultElements.push_back(
+            APValue(APSInt(LHS.ashr(RHS.getZExtValue()), DestUnsigned)));
+        break;
+      case clang::X86::BI__builtin_ia32_psrlv2di:
+      case clang::X86::BI__builtin_ia32_psrlv4di:
+      case clang::X86::BI__builtin_ia32_psrlv4si:
+      case clang::X86::BI__builtin_ia32_psrlv8si:
+        if (RHS.uge(RHS.getBitWidth())) {
+          ResultElements.push_back(
+              APValue(APSInt(APInt::getZero(RHS.getBitWidth()), DestUnsigned)));
+          break;
+        }
+        ResultElements.push_back(
+            APValue(APSInt(LHS.lshr(RHS.getZExtValue()), DestUnsigned)));
+        break;
+      }
+    }
+
+    return Success(APValue(ResultElements.data(), ResultElements.size()), E);
+  }
+  case clang::X86::BI__builtin_ia32_pmuldq128:
+  case clang::X86::BI__builtin_ia32_pmuldq256:
+  case clang::X86::BI__builtin_ia32_pmuldq512:
+  case clang::X86::BI__builtin_ia32_pmuludq128:
+  case clang::X86::BI__builtin_ia32_pmuludq256:
+  case clang::X86::BI__builtin_ia32_pmuludq512: {
+    APValue SourceLHS, SourceRHS;
+    if (!EvaluateAsRValue(Info, E->getArg(0), SourceLHS) ||
+        !EvaluateAsRValue(Info, E->getArg(1), SourceRHS))
+      return false;
+
+    unsigned SourceLen = SourceLHS.getVectorLength();
+    SmallVector<APValue, 4> ResultElements;
+    ResultElements.reserve(SourceLen / 2);
+
+    for (unsigned EltNum = 0; EltNum < SourceLen; EltNum += 2) {
+      APSInt LHS = SourceLHS.getVectorElt(EltNum).getInt();
+      APSInt RHS = SourceRHS.getVectorElt(EltNum).getInt();
+
+      switch (E->getBuiltinCallee()) {
+      case clang::X86::BI__builtin_ia32_pmuludq128:
+      case clang::X86::BI__builtin_ia32_pmuludq256:
+      case clang::X86::BI__builtin_ia32_pmuludq512:
+        ResultElements.push_back(
+            APValue(APSInt(llvm::APIntOps::muluExtended(LHS, RHS), true)));
+        break;
+      case clang::X86::BI__builtin_ia32_pmuldq128:
+      case clang::X86::BI__builtin_ia32_pmuldq256:
+      case clang::X86::BI__builtin_ia32_pmuldq512:
+        ResultElements.push_back(
+            APValue(APSInt(llvm::APIntOps::mulsExtended(LHS, RHS), false)));
+        break;
+      }
+    }
+
+    return Success(APValue(ResultElements.data(), ResultElements.size()), E);
+  }
+  case Builtin::BI__builtin_elementwise_max:
+  case Builtin::BI__builtin_elementwise_min: {
+    APValue SourceLHS, SourceRHS;
+    if (!EvaluateAsRValue(Info, E->getArg(0), SourceLHS) ||
+        !EvaluateAsRValue(Info, E->getArg(1), SourceRHS))
+      return false;
+
+    QualType DestEltTy = E->getType()->castAs<VectorType>()->getElementType();
+
+    if (!DestEltTy->isIntegerType())
+      return false;
+
+    unsigned SourceLen = SourceLHS.getVectorLength();
+    SmallVector<APValue, 4> ResultElements;
+    ResultElements.reserve(SourceLen);
+
+    for (unsigned EltNum = 0; EltNum < SourceLen; ++EltNum) {
+      APSInt LHS = SourceLHS.getVectorElt(EltNum).getInt();
+      APSInt RHS = SourceRHS.getVectorElt(EltNum).getInt();
+      switch (E->getBuiltinCallee()) {
+      case Builtin::BI__builtin_elementwise_max:
+        ResultElements.push_back(
+            APValue(APSInt(std::max(LHS, RHS),
+                           DestEltTy->isUnsignedIntegerOrEnumerationType())));
+        break;
+      case Builtin::BI__builtin_elementwise_min:
+        ResultElements.push_back(
+            APValue(APSInt(std::min(LHS, RHS),
+                           DestEltTy->isUnsignedIntegerOrEnumerationType())));
+        break;
+      }
+    }
+
+    return Success(APValue(ResultElements.data(), ResultElements.size()), E);
+  }
+  case X86::BI__builtin_ia32_selectb_128:
+  case X86::BI__builtin_ia32_selectb_256:
+  case X86::BI__builtin_ia32_selectb_512:
+  case X86::BI__builtin_ia32_selectw_128:
+  case X86::BI__builtin_ia32_selectw_256:
+  case X86::BI__builtin_ia32_selectw_512:
+  case X86::BI__builtin_ia32_selectd_128:
+  case X86::BI__builtin_ia32_selectd_256:
+  case X86::BI__builtin_ia32_selectd_512:
+  case X86::BI__builtin_ia32_selectq_128:
+  case X86::BI__builtin_ia32_selectq_256:
+  case X86::BI__builtin_ia32_selectq_512:
+  case X86::BI__builtin_ia32_selectph_128:
+  case X86::BI__builtin_ia32_selectph_256:
+  case X86::BI__builtin_ia32_selectph_512:
+  case X86::BI__builtin_ia32_selectpbf_128:
+  case X86::BI__builtin_ia32_selectpbf_256:
+  case X86::BI__builtin_ia32_selectpbf_512:
+  case X86::BI__builtin_ia32_selectps_128:
+  case X86::BI__builtin_ia32_selectps_256:
+  case X86::BI__builtin_ia32_selectps_512:
+  case X86::BI__builtin_ia32_selectpd_128:
+  case X86::BI__builtin_ia32_selectpd_256:
+  case X86::BI__builtin_ia32_selectpd_512: {
+    // AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
+    APValue SourceMask, SourceLHS, SourceRHS;
+    if (!EvaluateAsRValue(Info, E->getArg(0), SourceMask) ||
+        !EvaluateAsRValue(Info, E->getArg(1), SourceLHS) ||
+        !EvaluateAsRValue(Info, E->getArg(2), SourceRHS))
+      return false;
+
+    APSInt Mask = SourceMask.getInt();
+    unsigned SourceLen = SourceLHS.getVectorLength();
+    SmallVector<APValue, 4> ResultElements;
+    ResultElements.reserve(SourceLen);
+
+    for (unsigned EltNum = 0; EltNum < SourceLen; ++EltNum) {
+      const APValue &LHS = SourceLHS.getVectorElt(EltNum);
+      const APValue &RHS = SourceRHS.getVectorElt(EltNum);
+      ResultElements.push_back(Mask[EltNum] ? LHS : RHS);
+    }
+
+    return Success(APValue(ResultElements.data(), ResultElements.size()), E);
+  }
+  case Builtin::BI__builtin_elementwise_ctlz:
+  case Builtin::BI__builtin_elementwise_cttz: {
+    APValue SourceLHS;
+    std::optional<APValue> Fallback;
+    if (!EvaluateAsRValue(Info, E->getArg(0), SourceLHS))
+      return false;
+    if (E->getNumArgs() > 1) {
+      APValue FallbackTmp;
+      if (!EvaluateAsRValue(Info, E->getArg(1), FallbackTmp))
+        return false;
+      Fallback = FallbackTmp;
+    }
+
+    QualType DestEltTy = E->getType()->castAs<VectorType>()->getElementType();
+    unsigned SourceLen = SourceLHS.getVectorLength();
+    SmallVector<APValue, 4> ResultElements;
+    ResultElements.reserve(SourceLen);
+
+    for (unsigned EltNum = 0; EltNum < SourceLen; ++EltNum) {
+      APSInt LHS = SourceLHS.getVectorElt(EltNum).getInt();
+      if (!LHS) {
+        // Without a fallback, a zero element is undefined
+        if (!Fallback) {
+          Info.FFDiag(E, diag::note_constexpr_countzeroes_zero)
+              << /*IsTrailing=*/(E->getBuiltinCallee() ==
+                                 Builtin::BI__builtin_elementwise_cttz);
+          return false;
+        }
+        ResultElements.push_back(Fallback->getVectorElt(EltNum));
+        continue;
+      }
+      switch (E->getBuiltinCallee()) {
+      case Builtin::BI__builtin_elementwise_ctlz:
+        ResultElements.push_back(APValue(
+            APSInt(APInt(Info.Ctx.getIntWidth(DestEltTy), LHS.countl_zero()),
+                   DestEltTy->isUnsignedIntegerOrEnumerationType())));
+        break;
+      case Builtin::BI__builtin_elementwise_cttz:
+        ResultElements.push_back(APValue(
+            APSInt(APInt(Info.Ctx.getIntWidth(DestEltTy), LHS.countr_zero()),
                    DestEltTy->isUnsignedIntegerOrEnumerationType())));
         break;
       }
@@ -11654,6 +11915,28 @@ bool VectorExprEvaluator::VisitCallExpr(const CallExpr *E) {
 
     return Success(APValue(ResultElements.data(), ResultElements.size()), E);
   }
+
+  case Builtin::BI__builtin_elementwise_fma: {
+    APValue SourceX, SourceY, SourceZ;
+    if (!EvaluateAsRValue(Info, E->getArg(0), SourceX) ||
+        !EvaluateAsRValue(Info, E->getArg(1), SourceY) ||
+        !EvaluateAsRValue(Info, E->getArg(2), SourceZ))
+      return false;
+
+    unsigned SourceLen = SourceX.getVectorLength();
+    SmallVector<APValue> ResultElements;
+    ResultElements.reserve(SourceLen);
+    llvm::RoundingMode RM = getActiveRoundingMode(getEvalInfo(), E);
+    for (unsigned EltNum = 0; EltNum < SourceLen; ++EltNum) {
+      const APFloat &X = SourceX.getVectorElt(EltNum).getFloat();
+      const APFloat &Y = SourceY.getVectorElt(EltNum).getFloat();
+      const APFloat &Z = SourceZ.getVectorElt(EltNum).getFloat();
+      APFloat Result(X);
+      (void)Result.fusedMultiplyAdd(Y, Z, RM);
+      ResultElements.push_back(APValue(Result));
+    }
+    return Success(APValue(ResultElements.data(), ResultElements.size()), E);
+  }
   }
 }
 
@@ -12885,7 +13168,10 @@ static bool convertUnsignedAPIntToCharUnits(const llvm::APInt &Int,
 static void addFlexibleArrayMemberInitSize(EvalInfo &Info, const QualType &T,
                                            const LValue &LV, CharUnits &Size) {
   if (!T.isNull() && T->isStructureType() &&
-      T->getAsStructureType()->getDecl()->hasFlexibleArrayMember())
+      T->getAsStructureType()
+          ->getOriginalDecl()
+          ->getDefinitionOrSelf()
+          ->hasFlexibleArrayMember())
     if (const auto *V = LV.getLValueBase().dyn_cast<const ValueDecl *>())
       if (const auto *VD = dyn_cast<VarDecl>(V))
         if (VD->hasInit())
@@ -13206,6 +13492,7 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
   case Builtin::BI__builtin_clzll:
   case Builtin::BI__builtin_clzs:
   case Builtin::BI__builtin_clzg:
+  case Builtin::BI__builtin_elementwise_ctlz:
   case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
   case Builtin::BI__lzcnt:
   case Builtin::BI__lzcnt64: {
@@ -13214,7 +13501,9 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
       return false;
 
     std::optional<APSInt> Fallback;
-    if (BuiltinOp == Builtin::BI__builtin_clzg && E->getNumArgs() > 1) {
+    if ((BuiltinOp == Builtin::BI__builtin_clzg ||
+         BuiltinOp == Builtin::BI__builtin_elementwise_ctlz) &&
+        E->getNumArgs() > 1) {
       APSInt FallbackTemp;
       if (!EvaluateInteger(E->getArg(1), FallbackTemp, Info))
         return false;
@@ -13232,6 +13521,11 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
                              BuiltinOp != Builtin::BI__lzcnt &&
                              BuiltinOp != Builtin::BI__lzcnt64;
 
+      if (BuiltinOp == Builtin::BI__builtin_elementwise_ctlz) {
+        Info.FFDiag(E, diag::note_constexpr_countzeroes_zero)
+            << /*IsTrailing=*/false;
+      }
+
       if (ZeroIsUndefined)
         return Error(E);
     }
@@ -13286,13 +13580,16 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
   case Builtin::BI__builtin_ctzl:
   case Builtin::BI__builtin_ctzll:
   case Builtin::BI__builtin_ctzs:
-  case Builtin::BI__builtin_ctzg: {
+  case Builtin::BI__builtin_ctzg:
+  case Builtin::BI__builtin_elementwise_cttz: {
     APSInt Val;
     if (!EvaluateInteger(E->getArg(0), Val, Info))
       return false;
 
     std::optional<APSInt> Fallback;
-    if (BuiltinOp == Builtin::BI__builtin_ctzg && E->getNumArgs() > 1) {
+    if ((BuiltinOp == Builtin::BI__builtin_ctzg ||
+         BuiltinOp == Builtin::BI__builtin_elementwise_cttz) &&
+        E->getNumArgs() > 1) {
       APSInt FallbackTemp;
       if (!EvaluateInteger(E->getArg(1), FallbackTemp, Info))
         return false;
@@ -13303,6 +13600,10 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
       if (Fallback)
         return Success(*Fallback, E);
 
+      if (BuiltinOp == Builtin::BI__builtin_elementwise_cttz) {
+        Info.FFDiag(E, diag::note_constexpr_countzeroes_zero)
+            << /*IsTrailing=*/true;
+      }
       return Error(E);
     }
 
@@ -13315,6 +13616,14 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
     return Success(Operand, E);
   }
 
+  case Builtin::BI__builtin_elementwise_abs: {
+    APSInt Val;
+    if (!EvaluateInteger(E->getArg(0), Val, Info))
+      return false;
+
+    return Success(Val.abs(), E);
+  }
+
   case Builtin::BI__builtin_expect:
   case Builtin::BI__builtin_expect_with_probability:
     return Visit(E->getArg(0));
@@ -13548,7 +13857,24 @@ bool IntExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
     APInt Result = LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
     return Success(APSInt(Result, !LHS.isSigned()), E);
   }
+  case Builtin::BI__builtin_elementwise_max: {
+    APSInt LHS, RHS;
+    if (!EvaluateInteger(E->getArg(0), LHS, Info) ||
+        !EvaluateInteger(E->getArg(1), RHS, Info))
+      return false;
+
+    APInt Result = std::max(LHS, RHS);
+    return Success(APSInt(Result, !LHS.isSigned()), E);
+  }
+  case Builtin::BI__builtin_elementwise_min: {
+    APSInt LHS, RHS;
+    if (!EvaluateInteger(E->getArg(0), LHS, Info) ||
+        !EvaluateInteger(E->getArg(1), RHS, Info))
+      return false;
 
+    APInt Result = std::min(LHS, RHS);
+    return Success(APSInt(Result, !LHS.isSigned()), E);
+  }
   case Builtin::BIstrlen:
   case Builtin::BIwcslen:
     // A call to strlen is not a constant expression.
@@ -15132,7 +15458,7 @@ bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *OOE) {
       const RecordType *RT = CurrentType->getAs<RecordType>();
       if (!RT)
         return Error(OOE);
-      RecordDecl *RD = RT->getDecl();
+      RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       if (RD->isInvalidDecl()) return false;
       const ASTRecordLayout &RL = Info.Ctx.getASTRecordLayout(RD);
       unsigned i = MemberDecl->getFieldIndex();
@@ -15154,7 +15480,7 @@ bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *OOE) {
       const RecordType *RT = CurrentType->getAs<RecordType>();
       if (!RT)
         return Error(OOE);
-      RecordDecl *RD = RT->getDecl();
+      RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       if (RD->isInvalidDecl()) return false;
       const ASTRecordLayout &RL = Info.Ctx.getASTRecordLayout(RD);
 
@@ -15165,7 +15491,8 @@ bool IntExprEvaluator::VisitOffsetOfExpr(const OffsetOfExpr *OOE) {
         return Error(OOE);
 
       // Add the offset to the base.
-      Result += RL.getBaseClassOffset(cast<CXXRecordDecl>(BaseRT->getDecl()));
+      Result += RL.getBaseClassOffset(cast<CXXRecordDecl>(
+          BaseRT->getOriginalDecl()->getDefinitionOrSelf()));
       break;
     }
     }
@@ -15344,7 +15671,7 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
 
     if (Info.Ctx.getLangOpts().CPlusPlus && DestType->isEnumeralType()) {
       const EnumType *ET = dyn_cast<EnumType>(DestType.getCanonicalType());
-      const EnumDecl *ED = ET->getDecl();
+      const EnumDecl *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
       // Check that the value is within the range of the enumeration values.
       //
       // This corressponds to [expr.static.cast]p10 which says:
@@ -15788,6 +16115,7 @@ bool FloatExprEvaluator::VisitCallExpr(const CallExpr *E) {
       return Error(E);
     return true;
 
+  case Builtin::BI__builtin_elementwise_abs:
   case Builtin::BI__builtin_fabs:
   case Builtin::BI__builtin_fabsf:
   case Builtin::BI__builtin_fabsl:
@@ -15874,6 +16202,21 @@ bool FloatExprEvaluator::VisitCallExpr(const CallExpr *E) {
     Result = minimumnum(Result, RHS);
     return true;
   }
+
+  case Builtin::BI__builtin_elementwise_fma: {
+    if (!E->getArg(0)->isPRValue() || !E->getArg(1)->isPRValue() ||
+        !E->getArg(2)->isPRValue()) {
+      return false;
+    }
+    APFloat SourceY(0.), SourceZ(0.);
+    if (!EvaluateFloat(E->getArg(0), Result, Info) ||
+        !EvaluateFloat(E->getArg(1), SourceY, Info) ||
+        !EvaluateFloat(E->getArg(2), SourceZ, Info))
+      return false;
+    llvm::RoundingMode RM = getActiveRoundingMode(getEvalInfo(), E);
+    (void)Result.fusedMultiplyAdd(SourceY, SourceZ, RM);
+    return true;
+  }
   }
 }
 
@@ -18006,7 +18349,8 @@ bool Expr::isPotentialConstantExpr(const FunctionDecl *FD,
   // Fabricate an arbitrary expression on the stack and pretend that it
   // is a temporary being used as the 'this' pointer.
   LValue This;
-  ImplicitValueInitExpr VIE(RD ? Info.Ctx.getRecordType(RD) : Info.Ctx.IntTy);
+  ImplicitValueInitExpr VIE(RD ? Info.Ctx.getCanonicalTagType(RD)
+                               : Info.Ctx.IntTy);
   This.set({&VIE, Info.CurrentCall->Index});
 
   ArrayRef<const Expr*> Args;
diff --git a/clang/lib/AST/FormatString.cpp b/clang/lib/AST/FormatString.cpp
index 112b756d2be1a..502a3e6b145e3 100644
--- a/clang/lib/AST/FormatString.cpp
+++ b/clang/lib/AST/FormatString.cpp
@@ -417,10 +417,11 @@ ArgType::matchesType(ASTContext &C, QualType argTy) const {
         // If the enum is incomplete we know nothing about the underlying type.
         // Assume that it's 'int'. Do not use the underlying type for a scoped
         // enumeration.
-        if (!ETy->getDecl()->isComplete())
+        const EnumDecl *ED = ETy->getOriginalDecl()->getDefinitionOrSelf();
+        if (!ED->isComplete())
           return NoMatch;
         if (ETy->isUnscopedEnumerationType())
-          argTy = ETy->getDecl()->getIntegerType();
+          argTy = ED->getIntegerType();
       }
 
       if (const auto *BT = argTy->getAs<BuiltinType>()) {
@@ -466,10 +467,11 @@ ArgType::matchesType(ASTContext &C, QualType argTy) const {
         // If the enum is incomplete we know nothing about the underlying type.
         // Assume that it's 'int'. Do not use the underlying type for a scoped
         // enumeration as that needs an exact match.
-        if (!ETy->getDecl()->isComplete())
+        const EnumDecl *ED = ETy->getOriginalDecl()->getDefinitionOrSelf();
+        if (!ED->isComplete())
           argTy = C.IntTy;
         else if (ETy->isUnscopedEnumerationType())
-          argTy = ETy->getDecl()->getIntegerType();
+          argTy = ED->getIntegerType();
       }
 
       if (argTy->isSaturatedFixedPointType())
diff --git a/clang/lib/AST/InheritViz.cpp b/clang/lib/AST/InheritViz.cpp
index 1dafed837a356..c03492c64b161 100644
--- a/clang/lib/AST/InheritViz.cpp
+++ b/clang/lib/AST/InheritViz.cpp
@@ -89,8 +89,8 @@ void InheritanceHierarchyWriter::WriteNode(QualType Type, bool FromVirtual) {
   Out << " \"];\n";
 
   // Display the base classes.
-  const auto *Decl =
-      static_cast<const CXXRecordDecl *>(Type->castAs<RecordType>()->getDecl());
+  const auto *Decl = static_cast<const CXXRecordDecl *>(
+      Type->castAs<RecordType>()->getOriginalDecl());
   for (const auto &Base : Decl->bases()) {
     QualType CanonBaseType = Context.getCanonicalType(Base.getType());
 
@@ -133,7 +133,7 @@ InheritanceHierarchyWriter::WriteNodeReference(QualType Type,
 /// viewInheritance - Display the inheritance hierarchy of this C++
 /// class using GraphViz.
 void CXXRecordDecl::viewInheritance(ASTContext& Context) const {
-  QualType Self = Context.getTypeDeclType(this);
+  QualType Self = Context.getCanonicalTagType(this);
 
   int FD;
   SmallString<128> Filename;
diff --git a/clang/lib/AST/ItaniumCXXABI.cpp b/clang/lib/AST/ItaniumCXXABI.cpp
index 6ceedd657fe7e..43a8bcd9443ff 100644
--- a/clang/lib/AST/ItaniumCXXABI.cpp
+++ b/clang/lib/AST/ItaniumCXXABI.cpp
@@ -42,10 +42,10 @@ namespace {
 ///
 /// Returns the name of anonymous union VarDecl or nullptr if it is not found.
 static const IdentifierInfo *findAnonymousUnionVarDeclName(const VarDecl& VD) {
-  const RecordType *RT = VD.getType()->getAs<RecordType>();
-  assert(RT && "type of VarDecl is expected to be RecordType.");
-  assert(RT->getDecl()->isUnion() && "RecordType is expected to be a union.");
-  if (const FieldDecl *FD = RT->getDecl()->findFirstNamedDataMember()) {
+  const auto *RT = VD.getType()->castAs<RecordType>();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
+  assert(RD->isUnion() && "RecordType is expected to be a union.");
+  if (const FieldDecl *FD = RD->findFirstNamedDataMember()) {
     return FD->getIdentifier();
   }
 
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index 5233648d8f9c8..112678fb2714a 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -463,9 +463,7 @@ class CXXNameMangler {
   void mangleVendorType(StringRef Name);
 
 private:
-
   bool mangleSubstitution(const NamedDecl *ND);
-  bool mangleSubstitution(NestedNameSpecifier *NNS);
   bool mangleSubstitution(QualType T);
   bool mangleSubstitution(TemplateName Template);
   bool mangleSubstitution(uintptr_t Ptr);
@@ -479,21 +477,15 @@ class CXXNameMangler {
 
     addSubstitution(reinterpret_cast<uintptr_t>(ND));
   }
-  void addSubstitution(NestedNameSpecifier *NNS) {
-    NNS = Context.getASTContext().getCanonicalNestedNameSpecifier(NNS);
-
-    addSubstitution(reinterpret_cast<uintptr_t>(NNS));
-  }
   void addSubstitution(QualType T);
   void addSubstitution(TemplateName Template);
   void addSubstitution(uintptr_t Ptr);
   // Destructive copy substitutions from other mangler.
   void extendSubstitutions(CXXNameMangler* Other);
 
-  void mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
+  void mangleUnresolvedPrefix(NestedNameSpecifier Qualifier,
                               bool recursive = false);
-  void mangleUnresolvedName(NestedNameSpecifier *qualifier,
-                            DeclarationName name,
+  void mangleUnresolvedName(NestedNameSpecifier Qualifier, DeclarationName name,
                             const TemplateArgumentLoc *TemplateArgs,
                             unsigned NumTemplateArgs,
                             unsigned KnownArity = UnknownArity);
@@ -542,7 +534,7 @@ class CXXNameMangler {
   void mangleNestedNameWithClosurePrefix(GlobalDecl GD,
                                          const NamedDecl *PrefixND,
                                          const AbiTagList *AdditionalAbiTags);
-  void manglePrefix(NestedNameSpecifier *qualifier);
+  void manglePrefix(NestedNameSpecifier Qualifier);
   void manglePrefix(const DeclContext *DC, bool NoFunction=false);
   void manglePrefix(QualType type);
   void mangleTemplatePrefix(GlobalDecl GD, bool NoFunction=false);
@@ -588,12 +580,10 @@ class CXXNameMangler {
 
   void mangleMemberExprBase(const Expr *base, bool isArrow);
   void mangleMemberExpr(const Expr *base, bool isArrow,
-                        NestedNameSpecifier *qualifier,
-                        NamedDecl *firstQualifierLookup,
-                        DeclarationName name,
+                        NestedNameSpecifier Qualifier,
+                        NamedDecl *firstQualifierLookup, DeclarationName name,
                         const TemplateArgumentLoc *TemplateArgs,
-                        unsigned NumTemplateArgs,
-                        unsigned knownArity);
+                        unsigned NumTemplateArgs, unsigned knownArity);
   void mangleCastExpression(const Expr *E, StringRef CastEncoding);
   void mangleInitListElements(const InitListExpr *InitList);
   void mangleRequirement(SourceLocation RequiresExprLoc,
@@ -1334,6 +1324,21 @@ void CXXNameMangler::manglePrefix(QualType type) {
       mangleTemplateArgs(Template, DTST->template_arguments());
       addSubstitution(QualType(DTST, 0));
     }
+  } else if (const auto *DNT = type->getAs<DependentNameType>()) {
+    // Clang 14 and before did not consider this substitutable.
+    bool Clang14Compat = isCompatibleWith(LangOptions::ClangABI::Ver14);
+    if (!Clang14Compat && mangleSubstitution(QualType(DNT, 0)))
+      return;
+
+    // Member expressions can have these without prefixes, but that
+    // should end up in mangleUnresolvedPrefix instead.
+    assert(DNT->getQualifier());
+    manglePrefix(DNT->getQualifier());
+
+    mangleSourceName(DNT->getIdentifier());
+
+    if (!Clang14Compat)
+      addSubstitution(QualType(DNT, 0));
   } else {
     // We use the QualType mangle type variant here because it handles
     // substitutions.
@@ -1345,7 +1350,7 @@ void CXXNameMangler::manglePrefix(QualType type) {
 ///
 /// \param recursive - true if this is being called recursively,
 ///   i.e. if there is more prefix "to the right".
-void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
+void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier Qualifier,
                                             bool recursive) {
 
   // x, ::x
@@ -1362,8 +1367,11 @@ void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
   // <unresolved-name> ::= [gs] sr <unresolved-qualifier-level>+ E
   //                       <base-unresolved-name>
 
-  switch (qualifier->getKind()) {
-  case NestedNameSpecifier::Global:
+  switch (Qualifier.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+    llvm_unreachable("unexpected null nested name specifier");
+
+  case NestedNameSpecifier::Kind::Global:
     Out << "gs";
 
     // We want an 'sr' unless this is the entire NNS.
@@ -1373,27 +1381,29 @@ void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
     // We never want an 'E' here.
     return;
 
-  case NestedNameSpecifier::Super:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     llvm_unreachable("Can't mangle __super specifier");
 
-  case NestedNameSpecifier::Namespace:
-    if (qualifier->getPrefix())
-      mangleUnresolvedPrefix(qualifier->getPrefix(),
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [Namespace, Prefix] = Qualifier.getAsNamespaceAndPrefix();
+    if (Prefix)
+      mangleUnresolvedPrefix(Prefix,
                              /*recursive*/ true);
     else
       Out << "sr";
-    mangleSourceNameWithAbiTags(qualifier->getAsNamespace());
+    mangleSourceNameWithAbiTags(Namespace);
     break;
+  }
 
-  case NestedNameSpecifier::TypeSpec: {
-    const Type *type = qualifier->getAsType();
+  case NestedNameSpecifier::Kind::Type: {
+    const Type *type = Qualifier.getAsType();
 
     // We only want to use an unresolved-type encoding if this is one of:
     //   - a decltype
     //   - a template type parameter
     //   - a template template parameter with arguments
     // In all of these cases, we should have no prefix.
-    if (NestedNameSpecifier *Prefix = qualifier->getPrefix()) {
+    if (NestedNameSpecifier Prefix = type->getPrefix()) {
       mangleUnresolvedPrefix(Prefix,
                              /*recursive=*/true);
     } else {
@@ -1406,18 +1416,6 @@ void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
 
     break;
   }
-
-  case NestedNameSpecifier::Identifier:
-    // Member expressions can have these without prefixes.
-    if (qualifier->getPrefix())
-      mangleUnresolvedPrefix(qualifier->getPrefix(),
-                             /*recursive*/ true);
-    else
-      Out << "sr";
-
-    mangleSourceName(qualifier->getAsIdentifier());
-    // An Identifier has no type information, so we can't emit abi tags for it.
-    break;
   }
 
   // If this was the innermost part of the NNS, and we fell out to
@@ -1429,10 +1427,11 @@ void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
 /// Mangle an unresolved-name, which is generally used for names which
 /// weren't resolved to specific entities.
 void CXXNameMangler::mangleUnresolvedName(
-    NestedNameSpecifier *qualifier, DeclarationName name,
+    NestedNameSpecifier Qualifier, DeclarationName name,
     const TemplateArgumentLoc *TemplateArgs, unsigned NumTemplateArgs,
     unsigned knownArity) {
-  if (qualifier) mangleUnresolvedPrefix(qualifier);
+  if (Qualifier)
+    mangleUnresolvedPrefix(Qualifier);
   switch (name.getNameKind()) {
     // <base-unresolved-name> ::= <simple-id>
     case DeclarationName::Identifier:
@@ -1581,7 +1580,10 @@ void CXXNameMangler::mangleUnqualifiedName(
 
     if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
       // We must have an anonymous union or struct declaration.
-      const RecordDecl *RD = VD->getType()->castAs<RecordType>()->getDecl();
+      const RecordDecl *RD = VD->getType()
+                                 ->castAs<RecordType>()
+                                 ->getOriginalDecl()
+                                 ->getDefinitionOrSelf();
 
       // Itanium C++ ABI 5.1.2:
       //
@@ -2167,49 +2169,22 @@ void CXXNameMangler::mangleLambdaSig(const CXXRecordDecl *Lambda) {
                          Lambda->getLambdaStaticInvoker());
 }
 
-void CXXNameMangler::manglePrefix(NestedNameSpecifier *qualifier) {
-  switch (qualifier->getKind()) {
-  case NestedNameSpecifier::Global:
+void CXXNameMangler::manglePrefix(NestedNameSpecifier Qualifier) {
+  switch (Qualifier.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
     // nothing
     return;
 
-  case NestedNameSpecifier::Super:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     llvm_unreachable("Can't mangle __super specifier");
 
-  case NestedNameSpecifier::Namespace:
-    mangleName(qualifier->getAsNamespace()->getNamespace());
-    return;
-
-  case NestedNameSpecifier::TypeSpec:
-    if (NestedNameSpecifier *Prefix = qualifier->getPrefix()) {
-      const auto *DTST =
-          cast<DependentTemplateSpecializationType>(qualifier->getAsType());
-      QualType NewT = getASTContext().getDependentTemplateSpecializationType(
-          DTST->getKeyword(),
-          {Prefix, DTST->getDependentTemplateName().getName(),
-           /*HasTemplateKeyword=*/true},
-          DTST->template_arguments(), /*IsCanonical=*/true);
-      manglePrefix(NewT);
-      return;
-    }
-    manglePrefix(QualType(qualifier->getAsType(), 0));
+  case NestedNameSpecifier::Kind::Namespace:
+    mangleName(Qualifier.getAsNamespaceAndPrefix().Namespace->getNamespace());
     return;
 
-  case NestedNameSpecifier::Identifier:
-    // Clang 14 and before did not consider this substitutable.
-    bool Clang14Compat = isCompatibleWith(LangOptions::ClangABI::Ver14);
-    if (!Clang14Compat && mangleSubstitution(qualifier))
-      return;
-
-    // Member expressions can have these without prefixes, but that
-    // should end up in mangleUnresolvedPrefix instead.
-    assert(qualifier->getPrefix());
-    manglePrefix(qualifier->getPrefix());
-
-    mangleSourceName(qualifier->getAsIdentifier());
-
-    if (!Clang14Compat)
-      addSubstitution(qualifier);
+  case NestedNameSpecifier::Kind::Type:
+    manglePrefix(QualType(Qualifier.getAsType(), 0));
     return;
   }
 
@@ -2269,8 +2244,7 @@ void CXXNameMangler::mangleTemplatePrefix(TemplateName Template) {
   if (!Clang11Compat && mangleSubstitution(Template))
     return;
 
-  if (NestedNameSpecifier *Qualifier = Dependent->getQualifier())
-    manglePrefix(Qualifier);
+  manglePrefix(Dependent->getQualifier());
 
   if (Clang11Compat && mangleSubstitution(Template))
     return;
@@ -2470,6 +2444,13 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
   case Type::CountAttributed:
     llvm_unreachable("type is illegal as a nested name specifier");
 
+  case Type::SubstBuiltinTemplatePack:
+    // FIXME: not clear how to mangle this!
+    // template <class T...> class A {
+    //   template <class U...> void foo(__builtin_dedup_pack<T...>(*)(U) x...);
+    // };
+    Out << "_SUBSTBUILTINPACK_";
+    break;
   case Type::SubstTemplateTypeParmPack:
     // FIXME: not clear how to mangle this!
     // template <class T...> class A {
@@ -2525,7 +2506,8 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
 
   case Type::Enum:
   case Type::Record:
-    mangleSourceNameWithAbiTags(cast<TagType>(Ty)->getDecl());
+    mangleSourceNameWithAbiTags(
+        cast<TagType>(Ty)->getOriginalDecl()->getDefinitionOrSelf());
     break;
 
   case Type::TemplateSpecialization: {
@@ -2586,8 +2568,9 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
   }
 
   case Type::InjectedClassName:
-    mangleSourceNameWithAbiTags(
-        cast<InjectedClassNameType>(Ty)->getDecl());
+    mangleSourceNameWithAbiTags(cast<InjectedClassNameType>(Ty)
+                                    ->getOriginalDecl()
+                                    ->getDefinitionOrSelf());
     break;
 
   case Type::DependentName:
@@ -2608,9 +2591,6 @@ bool CXXNameMangler::mangleUnresolvedTypeOrSimpleId(QualType Ty,
   case Type::Using:
     return mangleUnresolvedTypeOrSimpleId(cast<UsingType>(Ty)->desugar(),
                                           Prefix);
-  case Type::Elaborated:
-    return mangleUnresolvedTypeOrSimpleId(
-        cast<ElaboratedType>(Ty)->getNamedType(), Prefix);
   }
 
   return false;
@@ -3838,7 +3818,7 @@ void CXXNameMangler::mangleType(const RecordType *T) {
   mangleType(static_cast<const TagType*>(T));
 }
 void CXXNameMangler::mangleType(const TagType *T) {
-  mangleName(T->getDecl());
+  mangleName(T->getOriginalDecl()->getDefinitionOrSelf());
 }
 
 // <type>       ::= <array-type>
@@ -3875,16 +3855,10 @@ void CXXNameMangler::mangleType(const IncompleteArrayType *T) {
 // <pointer-to-member-type> ::= M <class type> <member type>
 void CXXNameMangler::mangleType(const MemberPointerType *T) {
   Out << 'M';
-  if (auto *RD = T->getMostRecentCXXRecordDecl()) {
+  if (auto *RD = T->getMostRecentCXXRecordDecl())
     mangleCXXRecordDecl(RD);
-  } else {
-    NestedNameSpecifier *NNS = T->getQualifier();
-    if (auto *II = NNS->getAsIdentifier())
-      mangleType(getASTContext().getDependentNameType(
-          ElaboratedTypeKeyword::None, NNS->getPrefix(), II));
-    else
-      manglePrefix(NNS);
-  }
+  else
+    mangleType(QualType(T->getQualifier().getAsType(), 0));
   QualType PointeeType = T->getPointeeType();
   if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(PointeeType)) {
     mangleType(FPT);
@@ -3924,6 +3898,14 @@ void CXXNameMangler::mangleType(const SubstTemplateTypeParmPackType *T) {
   Out << "_SUBSTPACK_";
 }
 
+void CXXNameMangler::mangleType(const SubstBuiltinTemplatePackType *T) {
+  // FIXME: not clear how to mangle this!
+  // template <class T...> class A {
+  //   template <class U...> void foo(__builtin_dedup_pack<T...>(*)(U) x...);
+  // };
+  Out << "_SUBSTBUILTINPACK_";
+}
+
 // <type> ::= P <type>   # pointer-to
 void CXXNameMangler::mangleType(const PointerType *T) {
   Out << 'P';
@@ -4471,7 +4453,8 @@ void CXXNameMangler::mangleType(const InjectedClassNameType *T) {
   // Mangle injected class name types as if the user had written the
   // specialization out fully.  It may not actually be possible to see
   // this mangling, though.
-  mangleType(T->getInjectedSpecializationType());
+  mangleType(T->getOriginalDecl()->getCanonicalTemplateSpecializationType(
+      getASTContext()));
 }
 
 void CXXNameMangler::mangleType(const TemplateSpecializationType *T) {
@@ -4747,7 +4730,7 @@ void CXXNameMangler::mangleIntegerLiteral(QualType T,
 void CXXNameMangler::mangleMemberExprBase(const Expr *Base, bool IsArrow) {
   // Ignore member expressions involving anonymous unions.
   while (const auto *RT = Base->getType()->getAs<RecordType>()) {
-    if (!RT->getDecl()->isAnonymousStructOrUnion())
+    if (!RT->getOriginalDecl()->isAnonymousStructOrUnion())
       break;
     const auto *ME = dyn_cast<MemberExpr>(Base);
     if (!ME)
@@ -4768,9 +4751,8 @@ void CXXNameMangler::mangleMemberExprBase(const Expr *Base, bool IsArrow) {
 }
 
 /// Mangles a member expression.
-void CXXNameMangler::mangleMemberExpr(const Expr *base,
-                                      bool isArrow,
-                                      NestedNameSpecifier *qualifier,
+void CXXNameMangler::mangleMemberExpr(const Expr *base, bool isArrow,
+                                      NestedNameSpecifier Qualifier,
                                       NamedDecl *firstQualifierLookup,
                                       DeclarationName member,
                                       const TemplateArgumentLoc *TemplateArgs,
@@ -4780,7 +4762,7 @@ void CXXNameMangler::mangleMemberExpr(const Expr *base,
   //              ::= pt <expression> <unresolved-name>
   if (base)
     mangleMemberExprBase(base, isArrow);
-  mangleUnresolvedName(qualifier, member, TemplateArgs, NumTemplateArgs, arity);
+  mangleUnresolvedName(Qualifier, member, TemplateArgs, NumTemplateArgs, arity);
 }
 
 /// Look at the callee of the given call expression and determine if
@@ -5230,7 +5212,7 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
     const auto *PDE = cast<CXXPseudoDestructorExpr>(E);
     if (const Expr *Base = PDE->getBase())
       mangleMemberExprBase(Base, PDE->isArrow());
-    NestedNameSpecifier *Qualifier = PDE->getQualifier();
+    NestedNameSpecifier Qualifier = PDE->getQualifier();
     if (TypeSourceInfo *ScopeInfo = PDE->getScopeTypeInfo()) {
       if (Qualifier) {
         mangleUnresolvedPrefix(Qualifier,
@@ -5855,7 +5837,8 @@ void CXXNameMangler::mangleExpression(const Expr *E, unsigned Arity,
     // externally-visible declaration, so there's no standard mangling for
     // this, but mangling as a literal of the closure type seems reasonable.
     Out << "L";
-    mangleType(Context.getASTContext().getRecordType(cast<LambdaExpr>(E)->getLambdaClass()));
+    mangleType(Context.getASTContext().getCanonicalTagType(
+        cast<LambdaExpr>(E)->getLambdaClass()));
     Out << "E";
     break;
   }
@@ -6528,7 +6511,7 @@ static QualType getLValueType(ASTContext &Ctx, const APValue &LV) {
                  dyn_cast<FieldDecl>(E.getAsBaseOrMember().getPointer()))
       T = FD->getType();
     else
-      T = Ctx.getRecordType(
+      T = Ctx.getCanonicalTagType(
           cast<CXXRecordDecl>(E.getAsBaseOrMember().getPointer()));
   }
   return T;
@@ -6895,7 +6878,7 @@ void CXXNameMangler::mangleValueInTemplateArg(QualType T, const APValue &V,
             }
             TypeSoFar = FD->getType();
           } else {
-            TypeSoFar = Ctx.getRecordType(cast<CXXRecordDecl>(D));
+            TypeSoFar = Ctx.getCanonicalTagType(cast<CXXRecordDecl>(D));
           }
         }
       }
@@ -7005,14 +6988,6 @@ bool CXXNameMangler::mangleSubstitution(const NamedDecl *ND) {
   return mangleSubstitution(reinterpret_cast<uintptr_t>(ND));
 }
 
-bool CXXNameMangler::mangleSubstitution(NestedNameSpecifier *NNS) {
-  assert(NNS->getKind() == NestedNameSpecifier::Identifier &&
-         "mangleSubstitution(NestedNameSpecifier *) is only used for "
-         "identifier nested name specifiers.");
-  NNS = Context.getASTContext().getCanonicalNestedNameSpecifier(NNS);
-  return mangleSubstitution(reinterpret_cast<uintptr_t>(NNS));
-}
-
 /// Determine whether the given type has any qualifiers that are relevant for
 /// substitutions.
 static bool hasMangledSubstitutionQualifiers(QualType T) {
@@ -7023,7 +6998,7 @@ static bool hasMangledSubstitutionQualifiers(QualType T) {
 bool CXXNameMangler::mangleSubstitution(QualType T) {
   if (!hasMangledSubstitutionQualifiers(T)) {
     if (const RecordType *RT = T->getAs<RecordType>())
-      return mangleSubstitution(RT->getDecl());
+      return mangleSubstitution(RT->getOriginalDecl()->getDefinitionOrSelf());
   }
 
   uintptr_t TypePtr = reinterpret_cast<uintptr_t>(T.getAsOpaquePtr());
@@ -7064,7 +7039,7 @@ bool CXXNameMangler::isSpecializedAs(QualType S, llvm::StringRef Name,
     return false;
 
   const ClassTemplateSpecializationDecl *SD =
-    dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl());
+      dyn_cast<ClassTemplateSpecializationDecl>(RT->getOriginalDecl());
   if (!SD || !SD->getIdentifier()->isStr(Name))
     return false;
 
@@ -7194,7 +7169,7 @@ bool CXXNameMangler::mangleStandardSubstitution(const NamedDecl *ND) {
 void CXXNameMangler::addSubstitution(QualType T) {
   if (!hasMangledSubstitutionQualifiers(T)) {
     if (const RecordType *RT = T->getAs<RecordType>()) {
-      addSubstitution(RT->getDecl());
+      addSubstitution(RT->getOriginalDecl()->getDefinitionOrSelf());
       return;
     }
   }
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index 64ddb1e739347..b3f12a1cce2ec 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -395,8 +395,8 @@ llvm::json::Array JSONNodeDumper::createCastPath(const CastExpr *C) {
 
   for (auto I = C->path_begin(), E = C->path_end(); I != E; ++I) {
     const CXXBaseSpecifier *Base = *I;
-    const auto *RD =
-        cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
+    const auto *RD = cast<CXXRecordDecl>(
+        Base->getType()->castAs<RecordType>()->getOriginalDecl());
 
     llvm::json::Object Val{{"name", RD->getName()}};
     if (Base->isVirtual())
@@ -606,9 +606,8 @@ void JSONNodeDumper::VisitTypedefType(const TypedefType *TT) {
 }
 
 void JSONNodeDumper::VisitUsingType(const UsingType *TT) {
-  JOS.attribute("decl", createBareDeclRef(TT->getFoundDecl()));
-  if (!TT->typeMatchesDecl())
-    JOS.attribute("type", createQualType(TT->desugar()));
+  JOS.attribute("decl", createBareDeclRef(TT->getDecl()));
+  JOS.attribute("type", createQualType(TT->desugar()));
 }
 
 void JSONNodeDumper::VisitFunctionType(const FunctionType *T) {
@@ -759,7 +758,15 @@ void JSONNodeDumper::VisitUnaryTransformType(const UnaryTransformType *UTT) {
 }
 
 void JSONNodeDumper::VisitTagType(const TagType *TT) {
-  JOS.attribute("decl", createBareDeclRef(TT->getDecl()));
+  if (NestedNameSpecifier Qualifier = TT->getQualifier()) {
+    std::string Str;
+    llvm::raw_string_ostream OS(Str);
+    Qualifier.print(OS, PrintPolicy, /*ResolveTemplateArguments=*/true);
+    JOS.attribute("qualifier", Str);
+  }
+  JOS.attribute("decl", createBareDeclRef(TT->getOriginalDecl()));
+  if (TT->isTagOwned())
+    JOS.attribute("isTagOwned", true);
 }
 
 void JSONNodeDumper::VisitTemplateTypeParmType(
@@ -809,7 +816,7 @@ void JSONNodeDumper::VisitTemplateSpecializationType(
 
 void JSONNodeDumper::VisitInjectedClassNameType(
     const InjectedClassNameType *ICNT) {
-  JOS.attribute("decl", createBareDeclRef(ICNT->getDecl()));
+  JOS.attribute("decl", createBareDeclRef(ICNT->getOriginalDecl()));
 }
 
 void JSONNodeDumper::VisitObjCInterfaceType(const ObjCInterfaceType *OIT) {
@@ -821,17 +828,6 @@ void JSONNodeDumper::VisitPackExpansionType(const PackExpansionType *PET) {
     JOS.attribute("numExpansions", *N);
 }
 
-void JSONNodeDumper::VisitElaboratedType(const ElaboratedType *ET) {
-  if (const NestedNameSpecifier *NNS = ET->getQualifier()) {
-    std::string Str;
-    llvm::raw_string_ostream OS(Str);
-    NNS->print(OS, PrintPolicy, /*ResolveTemplateArgs*/ true);
-    JOS.attribute("qualifier", Str);
-  }
-  if (const TagDecl *TD = ET->getOwnedTagDecl())
-    JOS.attribute("ownedTagDecl", createBareDeclRef(TD));
-}
-
 void JSONNodeDumper::VisitMacroQualifiedType(const MacroQualifiedType *MQT) {
   JOS.attribute("macroName", MQT->getMacroIdentifier()->getName());
 }
@@ -902,9 +898,9 @@ void JSONNodeDumper::VisitNamespaceAliasDecl(const NamespaceAliasDecl *NAD) {
 
 void JSONNodeDumper::VisitUsingDecl(const UsingDecl *UD) {
   std::string Name;
-  if (const NestedNameSpecifier *NNS = UD->getQualifier()) {
+  if (NestedNameSpecifier Qualifier = UD->getQualifier()) {
     llvm::raw_string_ostream SOS(Name);
-    NNS->print(SOS, UD->getASTContext().getPrintingPolicy());
+    Qualifier.print(SOS, UD->getASTContext().getPrintingPolicy());
   }
   Name += UD->getNameAsString();
   JOS.attribute("name", Name);
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index e6ea0ada2e9ec..fc79ab1de24ff 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -1805,17 +1805,16 @@ void MicrosoftCXXNameMangler::mangleTemplateArg(const TemplateDecl *TD,
   case TemplateArgument::Declaration: {
     const NamedDecl *ND = TA.getAsDecl();
     if (isa<FieldDecl>(ND) || isa<IndirectFieldDecl>(ND)) {
-      mangleMemberDataPointer(cast<CXXRecordDecl>(ND->getDeclContext())
-                                  ->getMostRecentNonInjectedDecl(),
-                              cast<ValueDecl>(ND),
-                              cast<NonTypeTemplateParmDecl>(Parm),
-                              TA.getParamTypeForDecl());
+      mangleMemberDataPointer(
+          cast<CXXRecordDecl>(ND->getDeclContext())->getMostRecentDecl(),
+          cast<ValueDecl>(ND), cast<NonTypeTemplateParmDecl>(Parm),
+          TA.getParamTypeForDecl());
     } else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND)) {
       const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
       if (MD && MD->isInstance()) {
-        mangleMemberFunctionPointer(
-            MD->getParent()->getMostRecentNonInjectedDecl(), MD,
-            cast<NonTypeTemplateParmDecl>(Parm), TA.getParamTypeForDecl());
+        mangleMemberFunctionPointer(MD->getParent()->getMostRecentDecl(), MD,
+                                    cast<NonTypeTemplateParmDecl>(Parm),
+                                    TA.getParamTypeForDecl());
       } else {
         mangleFunctionPointer(FD, cast<NonTypeTemplateParmDecl>(Parm),
                               TA.getParamTypeForDecl());
@@ -2021,7 +2020,7 @@ void MicrosoftCXXNameMangler::mangleTemplateArgValue(QualType T,
             if (RD->isAnonymousStructOrUnion())
               continue;
         } else {
-          ET = getASTContext().getRecordType(cast<CXXRecordDecl>(D));
+          ET = getASTContext().getCanonicalTagType(cast<CXXRecordDecl>(D));
           // Bug in MSVC: fully qualified name of base class should be used for
           // mangling to prevent collisions e.g. on base classes with same names
           // in different namespaces.
@@ -3247,11 +3246,11 @@ void MicrosoftCXXNameMangler::mangleTagTypeKind(TagTypeKind TTK) {
 }
 void MicrosoftCXXNameMangler::mangleType(const EnumType *T, Qualifiers,
                                          SourceRange) {
-  mangleType(cast<TagType>(T)->getDecl());
+  mangleType(cast<TagType>(T)->getOriginalDecl()->getDefinitionOrSelf());
 }
 void MicrosoftCXXNameMangler::mangleType(const RecordType *T, Qualifiers,
                                          SourceRange) {
-  mangleType(cast<TagType>(T)->getDecl());
+  mangleType(cast<TagType>(T)->getOriginalDecl()->getDefinitionOrSelf());
 }
 void MicrosoftCXXNameMangler::mangleType(const TagDecl *TD) {
   mangleTagTypeKind(TD->getTagKind());
@@ -3384,6 +3383,11 @@ void MicrosoftCXXNameMangler::mangleType(const SubstTemplateTypeParmPackType *T,
   Error(Range.getBegin(), "substituted parameter pack") << Range;
 }
 
+void MicrosoftCXXNameMangler::mangleType(const SubstBuiltinTemplatePackType *T,
+                                         Qualifiers, SourceRange Range) {
+  Error(Range.getBegin(), "substituted builtin template pack") << Range;
+}
+
 // <type> ::= <pointer-type>
 // <pointer-type> ::= E? <pointer-cvr-qualifiers> <cvr-qualifiers> <type>
 //                       # the E is required for 64-bit non-static pointers
diff --git a/clang/lib/AST/NestedNameSpecifier.cpp b/clang/lib/AST/NestedNameSpecifier.cpp
index 56f74b92412d2..c6af91f5c0083 100644
--- a/clang/lib/AST/NestedNameSpecifier.cpp
+++ b/clang/lib/AST/NestedNameSpecifier.cpp
@@ -15,7 +15,6 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
-#include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DependenceFlags.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/TemplateName.h"
@@ -35,250 +34,67 @@
 
 using namespace clang;
 
-NestedNameSpecifier *
-NestedNameSpecifier::FindOrInsert(const ASTContext &Context,
-                                  const NestedNameSpecifier &Mockup) {
+const NamespaceAndPrefixStorage *
+NestedNameSpecifier::MakeNamespaceAndPrefixStorage(
+    const ASTContext &Ctx, const NamespaceBaseDecl *Namespace,
+    NestedNameSpecifier Prefix) {
   llvm::FoldingSetNodeID ID;
-  Mockup.Profile(ID);
+  NamespaceAndPrefixStorage::Profile(ID, Namespace, Prefix);
 
   void *InsertPos = nullptr;
-  NestedNameSpecifier *NNS
-    = Context.NestedNameSpecifiers.FindNodeOrInsertPos(ID, InsertPos);
-  if (!NNS) {
-    NNS =
-        new (Context, alignof(NestedNameSpecifier)) NestedNameSpecifier(Mockup);
-    Context.NestedNameSpecifiers.InsertNode(NNS, InsertPos);
+  NamespaceAndPrefixStorage *S =
+      Ctx.NamespaceAndPrefixStorages.FindNodeOrInsertPos(ID, InsertPos);
+  if (!S) {
+    S = new (Ctx, alignof(NamespaceAndPrefixStorage))
+        NamespaceAndPrefixStorage(Namespace, Prefix);
+    Ctx.NamespaceAndPrefixStorages.InsertNode(S, InsertPos);
   }
-
-  return NNS;
-}
-
-NestedNameSpecifier *NestedNameSpecifier::Create(const ASTContext &Context,
-                                                 NestedNameSpecifier *Prefix,
-                                                 const IdentifierInfo *II) {
-  assert(II && "Identifier cannot be NULL");
-  assert((!Prefix || Prefix->isDependent()) && "Prefix must be dependent");
-
-  NestedNameSpecifier Mockup;
-  Mockup.Prefix.setPointer(Prefix);
-  Mockup.Prefix.setInt(StoredIdentifier);
-  Mockup.Specifier = const_cast<IdentifierInfo *>(II);
-  return FindOrInsert(Context, Mockup);
-}
-
-NestedNameSpecifier *NestedNameSpecifier::Create(const ASTContext &Context,
-                                                 NestedNameSpecifier *Prefix,
-                                                 const NamespaceBaseDecl *NS) {
-  assert(NS && "Namespace cannot be NULL");
-  assert((!Prefix ||
-          (Prefix->getAsType() == nullptr &&
-           Prefix->getAsIdentifier() == nullptr)) &&
-         "Broken nested name specifier");
-  NestedNameSpecifier Mockup;
-  Mockup.Prefix.setPointer(Prefix);
-  Mockup.Prefix.setInt(StoredDecl);
-  Mockup.Specifier = const_cast<NamespaceBaseDecl *>(NS);
-  return FindOrInsert(Context, Mockup);
-}
-
-NestedNameSpecifier *NestedNameSpecifier::Create(const ASTContext &Context,
-                                                 NestedNameSpecifier *Prefix,
-                                                 const Type *T) {
-  assert(T && "Type cannot be NULL");
-  NestedNameSpecifier Mockup;
-  Mockup.Prefix.setPointer(Prefix);
-  Mockup.Prefix.setInt(StoredTypeSpec);
-  Mockup.Specifier = const_cast<Type*>(T);
-  return FindOrInsert(Context, Mockup);
-}
-
-NestedNameSpecifier *NestedNameSpecifier::Create(const ASTContext &Context,
-                                                 const IdentifierInfo *II) {
-  assert(II && "Identifier cannot be NULL");
-  NestedNameSpecifier Mockup;
-  Mockup.Prefix.setPointer(nullptr);
-  Mockup.Prefix.setInt(StoredIdentifier);
-  Mockup.Specifier = const_cast<IdentifierInfo *>(II);
-  return FindOrInsert(Context, Mockup);
-}
-
-NestedNameSpecifier *
-NestedNameSpecifier::GlobalSpecifier(const ASTContext &Context) {
-  if (!Context.GlobalNestedNameSpecifier)
-    Context.GlobalNestedNameSpecifier =
-        new (Context, alignof(NestedNameSpecifier)) NestedNameSpecifier();
-  return Context.GlobalNestedNameSpecifier;
-}
-
-NestedNameSpecifier *
-NestedNameSpecifier::SuperSpecifier(const ASTContext &Context,
-                                    CXXRecordDecl *RD) {
-  NestedNameSpecifier Mockup;
-  Mockup.Prefix.setPointer(nullptr);
-  Mockup.Prefix.setInt(StoredDecl);
-  Mockup.Specifier = RD;
-  return FindOrInsert(Context, Mockup);
+  return S;
 }
 
-NestedNameSpecifier::SpecifierKind NestedNameSpecifier::getKind() const {
-  if (!Specifier)
-    return Global;
-
-  switch (Prefix.getInt()) {
-  case StoredIdentifier:
-    return Identifier;
-
-  case StoredDecl: {
-    NamedDecl *ND = static_cast<NamedDecl *>(Specifier);
-    return isa<CXXRecordDecl>(ND) ? Super : Namespace;
-  }
-
-  case StoredTypeSpec:
-    return TypeSpec;
-  }
-
-  llvm_unreachable("Invalid NNS Kind!");
-}
-
-/// Retrieve the namespace or namespace alias stored in this nested name
-/// specifier.
-NamespaceBaseDecl *NestedNameSpecifier::getAsNamespace() const {
-  if (Prefix.getInt() == StoredDecl)
-    return dyn_cast<NamespaceBaseDecl>(static_cast<NamedDecl *>(Specifier));
-
-  return nullptr;
-}
-
-/// Retrieve the record declaration stored in this nested name specifier.
-CXXRecordDecl *NestedNameSpecifier::getAsRecordDecl() const {
-  switch (Prefix.getInt()) {
-  case StoredIdentifier:
-    return nullptr;
-
-  case StoredDecl:
-    return dyn_cast<CXXRecordDecl>(static_cast<NamedDecl *>(Specifier));
-
-  case StoredTypeSpec:
-    return getAsType()->getAsCXXRecordDecl();
+bool NestedNameSpecifier::isFullyQualified() const {
+  switch (getKind()) {
+  case NestedNameSpecifier::Kind::Global:
+    return true;
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    return false;
+  case NestedNameSpecifier::Kind::Namespace:
+    return getAsNamespaceAndPrefix().Prefix.isFullyQualified();
+  case NestedNameSpecifier::Kind::Type:
+    return getAsType()->getPrefix().isFullyQualified();
   }
-
   llvm_unreachable("Invalid NNS Kind!");
 }
 
 NestedNameSpecifierDependence NestedNameSpecifier::getDependence() const {
   switch (getKind()) {
-  case Identifier: {
-    // Identifier specifiers always represent dependent types
-    auto F = NestedNameSpecifierDependence::Dependent |
-             NestedNameSpecifierDependence::Instantiation;
-    // Prefix can contain unexpanded template parameters.
-    if (getPrefix())
-      return F | getPrefix()->getDependence();
-    return F;
-  }
-
-  case Namespace:
-  case Global:
-    return NestedNameSpecifierDependence::None;
-
-  case Super: {
-    CXXRecordDecl *RD = static_cast<CXXRecordDecl *>(Specifier);
-    for (const auto &Base : RD->bases())
-      if (Base.getType()->isDependentType())
-        // FIXME: must also be instantiation-dependent.
-        return NestedNameSpecifierDependence::Dependent;
+  case Kind::Null:
+  case Kind::Global:
+  case Kind::Namespace:
     return NestedNameSpecifierDependence::None;
+  case Kind::MicrosoftSuper: {
+    CXXRecordDecl *RD = getAsMicrosoftSuper();
+    return RD->isDependentContext()
+               ? NestedNameSpecifierDependence::DependentInstantiation |
+                     NestedNameSpecifierDependence::Dependent
+               : NestedNameSpecifierDependence::None;
   }
-
-  case TypeSpec: {
-    NestedNameSpecifierDependence Dep =
-        toNestedNameSpecifierDependendence(getAsType()->getDependence());
-    if (NestedNameSpecifier *Prefix = getPrefix())
-      Dep |=
-          Prefix->getDependence() & ~NestedNameSpecifierDependence::Dependent;
-    return Dep;
-  }
+  case Kind::Type:
+    return toNestedNameSpecifierDependence(getAsType()->getDependence());
   }
   llvm_unreachable("Invalid NNS Kind!");
 }
 
-bool NestedNameSpecifier::isDependent() const {
-  return getDependence() & NestedNameSpecifierDependence::Dependent;
-}
-
-bool NestedNameSpecifier::isInstantiationDependent() const {
-  return getDependence() & NestedNameSpecifierDependence::Instantiation;
-}
-
-bool NestedNameSpecifier::containsUnexpandedParameterPack() const {
-  return getDependence() & NestedNameSpecifierDependence::UnexpandedPack;
-}
-
-bool NestedNameSpecifier::containsErrors() const {
-  return getDependence() & NestedNameSpecifierDependence::Error;
-}
-
-const Type *
-NestedNameSpecifier::translateToType(const ASTContext &Context) const {
-  NestedNameSpecifier *Prefix = getPrefix();
-  switch (getKind()) {
-  case SpecifierKind::Identifier:
-    return Context
-        .getDependentNameType(ElaboratedTypeKeyword::None, Prefix,
-                              getAsIdentifier())
-        .getTypePtr();
-  case SpecifierKind::TypeSpec: {
-    const Type *T = getAsType();
-    switch (T->getTypeClass()) {
-    case Type::DependentTemplateSpecialization: {
-      const auto *DT = cast<DependentTemplateSpecializationType>(T);
-      const DependentTemplateStorage &DTN = DT->getDependentTemplateName();
-      return Context
-          .getDependentTemplateSpecializationType(
-              ElaboratedTypeKeyword::None,
-              {Prefix, DTN.getName(), DTN.hasTemplateKeyword()},
-              DT->template_arguments())
-          .getTypePtr();
-    }
-    case Type::Record:
-    case Type::TemplateSpecialization:
-    case Type::Using:
-    case Type::Enum:
-    case Type::Typedef:
-    case Type::UnresolvedUsing:
-      return Context
-          .getElaboratedType(ElaboratedTypeKeyword::None, Prefix,
-                             QualType(T, 0))
-          .getTypePtr();
-    default:
-      assert(Prefix == nullptr && "unexpected type with elaboration");
-      return T;
-    }
-  }
-  case SpecifierKind::Global:
-  case SpecifierKind::Namespace:
-  case SpecifierKind::Super:
-    // These are not representable as types.
-    return nullptr;
-  }
-  llvm_unreachable("Unhandled SpecifierKind enum");
-}
-
 /// Print this nested name specifier to the given output
 /// stream.
 void NestedNameSpecifier::print(raw_ostream &OS, const PrintingPolicy &Policy,
                                 bool ResolveTemplateArguments,
                                 bool PrintFinalScopeResOp) const {
-  if (getPrefix())
-    getPrefix()->print(OS, Policy);
-
   switch (getKind()) {
-  case Identifier:
-    OS << getAsIdentifier()->getName();
-    break;
-
-  case Namespace: {
-    NamespaceBaseDecl *Namespace = getAsNamespace();
+  case Kind::Namespace: {
+    auto [Namespace, Prefix] = getAsNamespaceAndPrefix();
+    Prefix.print(OS, Policy);
     if (const auto *NS = dyn_cast<NamespaceDecl>(Namespace)) {
       assert(!NS->isAnonymousNamespace());
       OS << NS->getName();
@@ -287,134 +103,49 @@ void NestedNameSpecifier::print(raw_ostream &OS, const PrintingPolicy &Policy,
     }
     break;
   }
-
-  case Global:
+  case Kind::Global:
     OS << "::";
     return;
-
-  case Super:
+  case Kind::MicrosoftSuper:
     OS << "__super";
     break;
-
-  case TypeSpec: {
+  case Kind::Type: {
     PrintingPolicy InnerPolicy(Policy);
-    InnerPolicy.SuppressScope = true;
     InnerPolicy.SuppressTagKeyword = true;
     QualType(getAsType(), 0).print(OS, InnerPolicy);
     break;
   }
+  case Kind::Null:
+    return;
   }
-
   if (PrintFinalScopeResOp)
     OS << "::";
 }
 
-LLVM_DUMP_METHOD void NestedNameSpecifier::dump(const LangOptions &LO) const {
-  dump(llvm::errs(), LO);
+LLVM_DUMP_METHOD void NestedNameSpecifier::dump(llvm::raw_ostream *OS,
+                                                const LangOptions *LO) const {
+  print(OS ? *OS : llvm::errs(), LO ? *LO : LangOptions());
 }
 
-LLVM_DUMP_METHOD void NestedNameSpecifier::dump() const { dump(llvm::errs()); }
-
+LLVM_DUMP_METHOD void NestedNameSpecifier::dump(const LangOptions &LO) const {
+  dump(/*OS=*/nullptr, &LO);
+}
 LLVM_DUMP_METHOD void NestedNameSpecifier::dump(llvm::raw_ostream &OS) const {
-  LangOptions LO;
-  dump(OS, LO);
+  dump(&OS);
 }
-
 LLVM_DUMP_METHOD void NestedNameSpecifier::dump(llvm::raw_ostream &OS,
                                                 const LangOptions &LO) const {
-  print(OS, PrintingPolicy(LO));
-}
-
-unsigned
-NestedNameSpecifierLoc::getLocalDataLength(NestedNameSpecifier *Qualifier) {
-  assert(Qualifier && "Expected a non-NULL qualifier");
-
-  // Location of the trailing '::'.
-  unsigned Length = sizeof(SourceLocation::UIntTy);
-
-  switch (Qualifier->getKind()) {
-  case NestedNameSpecifier::Global:
-    // Nothing more to add.
-    break;
-
-  case NestedNameSpecifier::Identifier:
-  case NestedNameSpecifier::Namespace:
-  case NestedNameSpecifier::Super:
-    // The location of the identifier or namespace name.
-    Length += sizeof(SourceLocation::UIntTy);
-    break;
-
-  case NestedNameSpecifier::TypeSpec:
-    // The "void*" that points at the TypeLoc data.
-    // Note: the 'template' keyword is part of the TypeLoc.
-    Length += sizeof(void *);
-    break;
-  }
-
-  return Length;
-}
-
-unsigned
-NestedNameSpecifierLoc::getDataLength(NestedNameSpecifier *Qualifier) {
-  unsigned Length = 0;
-  for (; Qualifier; Qualifier = Qualifier->getPrefix())
-    Length += getLocalDataLength(Qualifier);
-  return Length;
-}
-
-/// Load a (possibly unaligned) source location from a given address
-/// and offset.
-static SourceLocation LoadSourceLocation(void *Data, unsigned Offset) {
-  SourceLocation::UIntTy Raw;
-  memcpy(&Raw, static_cast<char *>(Data) + Offset, sizeof(Raw));
-  return SourceLocation::getFromRawEncoding(Raw);
+  dump(&OS, &LO);
 }
 
-/// Load a (possibly unaligned) pointer from a given address and
-/// offset.
-static void *LoadPointer(void *Data, unsigned Offset) {
-  void *Result;
-  memcpy(&Result, static_cast<char *>(Data) + Offset, sizeof(void*));
-  return Result;
-}
-
-SourceRange NestedNameSpecifierLoc::getLocalSourceRange() const {
+SourceLocation NestedNameSpecifierLoc::getBeginLoc() const {
   if (!Qualifier)
-    return SourceRange();
-
-  unsigned Offset = getDataLength(Qualifier->getPrefix());
-  switch (Qualifier->getKind()) {
-  case NestedNameSpecifier::Global:
-    return LoadSourceLocation(Data, Offset);
-
-  case NestedNameSpecifier::Identifier:
-  case NestedNameSpecifier::Namespace:
-  case NestedNameSpecifier::Super:
-    return SourceRange(
-        LoadSourceLocation(Data, Offset),
-        LoadSourceLocation(Data, Offset + sizeof(SourceLocation::UIntTy)));
-
-  case NestedNameSpecifier::TypeSpec: {
-    // The "void*" that points at the TypeLoc data.
-    // Note: the 'template' keyword is part of the TypeLoc.
-    void *TypeData = LoadPointer(Data, Offset);
-    TypeLoc TL(Qualifier->getAsType(), TypeData);
-    return SourceRange(TL.getBeginLoc(),
-                       LoadSourceLocation(Data, Offset + sizeof(void*)));
-  }
-  }
+    return SourceLocation();
 
-  llvm_unreachable("Invalid NNS Kind!");
-}
-
-TypeLoc NestedNameSpecifierLoc::getTypeLoc() const {
-  if (Qualifier->getKind() != NestedNameSpecifier::TypeSpec)
-    return TypeLoc();
-
-  // The "void*" that points at the TypeLoc data.
-  unsigned Offset = getDataLength(Qualifier->getPrefix());
-  void *TypeData = LoadPointer(Data, Offset);
-  return TypeLoc(Qualifier->getAsType(), TypeData);
+  NestedNameSpecifierLoc First = *this;
+  while (NestedNameSpecifierLoc Prefix = First.getAsNamespaceAndPrefix().Prefix)
+    First = Prefix;
+  return First.getLocalSourceRange().getBegin();
 }
 
 static void Append(char *Start, char *End, char *&Buffer, unsigned &BufferSize,
@@ -516,10 +247,10 @@ operator=(const NestedNameSpecifierLocBuilder &Other) {
   return *this;
 }
 
-void NestedNameSpecifierLocBuilder::Extend(ASTContext &Context, TypeLoc TL,
-                                           SourceLocation ColonColonLoc) {
-  Representation =
-      NestedNameSpecifier::Create(Context, Representation, TL.getTypePtr());
+void NestedNameSpecifierLocBuilder::Make(ASTContext &Context, TypeLoc TL,
+                                         SourceLocation ColonColonLoc) {
+  assert(!Representation);
+  Representation = NestedNameSpecifier(TL.getTypePtr());
 
   // Push source-location info into the buffer.
   SavePointer(TL.getOpaqueData(), Buffer, BufferSize, BufferCapacity);
@@ -527,23 +258,10 @@ void NestedNameSpecifierLocBuilder::Extend(ASTContext &Context, TypeLoc TL,
 }
 
 void NestedNameSpecifierLocBuilder::Extend(ASTContext &Context,
-                                           IdentifierInfo *Identifier,
-                                           SourceLocation IdentifierLoc,
-                                           SourceLocation ColonColonLoc) {
-  Representation = NestedNameSpecifier::Create(Context, Representation,
-                                               Identifier);
-
-  // Push source-location info into the buffer.
-  SaveSourceLocation(IdentifierLoc, Buffer, BufferSize, BufferCapacity);
-  SaveSourceLocation(ColonColonLoc, Buffer, BufferSize, BufferCapacity);
-}
-
-void NestedNameSpecifierLocBuilder::Extend(ASTContext &Context,
-                                           NamespaceBaseDecl *Namespace,
+                                           const NamespaceBaseDecl *Namespace,
                                            SourceLocation NamespaceLoc,
                                            SourceLocation ColonColonLoc) {
-  Representation = NestedNameSpecifier::Create(Context, Representation,
-                                               Namespace);
+  Representation = NestedNameSpecifier(Context, Namespace, Representation);
 
   // Push source-location info into the buffer.
   SaveSourceLocation(NamespaceLoc, Buffer, BufferSize, BufferCapacity);
@@ -553,60 +271,48 @@ void NestedNameSpecifierLocBuilder::Extend(ASTContext &Context,
 void NestedNameSpecifierLocBuilder::MakeGlobal(ASTContext &Context,
                                                SourceLocation ColonColonLoc) {
   assert(!Representation && "Already have a nested-name-specifier!?");
-  Representation = NestedNameSpecifier::GlobalSpecifier(Context);
+  Representation = NestedNameSpecifier::getGlobal();
 
   // Push source-location info into the buffer.
   SaveSourceLocation(ColonColonLoc, Buffer, BufferSize, BufferCapacity);
 }
 
-void NestedNameSpecifierLocBuilder::MakeSuper(ASTContext &Context,
-                                              CXXRecordDecl *RD,
-                                              SourceLocation SuperLoc,
-                                              SourceLocation ColonColonLoc) {
-  Representation = NestedNameSpecifier::SuperSpecifier(Context, RD);
+void NestedNameSpecifierLocBuilder::MakeMicrosoftSuper(
+    ASTContext &Context, CXXRecordDecl *RD, SourceLocation SuperLoc,
+    SourceLocation ColonColonLoc) {
+  Representation = NestedNameSpecifier(RD);
 
   // Push source-location info into the buffer.
   SaveSourceLocation(SuperLoc, Buffer, BufferSize, BufferCapacity);
   SaveSourceLocation(ColonColonLoc, Buffer, BufferSize, BufferCapacity);
 }
 
-void NestedNameSpecifierLocBuilder::MakeTrivial(ASTContext &Context,
-                                                NestedNameSpecifier *Qualifier,
+void NestedNameSpecifierLocBuilder::PushTrivial(ASTContext &Context,
+                                                NestedNameSpecifier Qualifier,
                                                 SourceRange R) {
-  Representation = Qualifier;
-
   // Construct bogus (but well-formed) source information for the
   // nested-name-specifier.
-  BufferSize = 0;
-  SmallVector<NestedNameSpecifier *, 4> Stack;
-  for (NestedNameSpecifier *NNS = Qualifier; NNS; NNS = NNS->getPrefix())
-    Stack.push_back(NNS);
-  while (!Stack.empty()) {
-    NestedNameSpecifier *NNS = Stack.pop_back_val();
-    switch (NNS->getKind()) {
-      case NestedNameSpecifier::Identifier:
-      case NestedNameSpecifier::Namespace:
-        SaveSourceLocation(R.getBegin(), Buffer, BufferSize, BufferCapacity);
-        break;
-
-      case NestedNameSpecifier::TypeSpec: {
-        TypeSourceInfo *TSInfo
-        = Context.getTrivialTypeSourceInfo(QualType(NNS->getAsType(), 0),
-                                           R.getBegin());
-        SavePointer(TSInfo->getTypeLoc().getOpaqueData(), Buffer, BufferSize,
-                    BufferCapacity);
-        break;
-      }
-
-      case NestedNameSpecifier::Global:
-      case NestedNameSpecifier::Super:
-        break;
-    }
-
-    // Save the location of the '::'.
-    SaveSourceLocation(Stack.empty()? R.getEnd() : R.getBegin(),
-                       Buffer, BufferSize, BufferCapacity);
+  switch (Qualifier.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+    return;
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [_1, Prefix] = Qualifier.getAsNamespaceAndPrefix();
+    PushTrivial(Context, Prefix, R.getBegin());
+    SaveSourceLocation(R.getBegin(), Buffer, BufferSize, BufferCapacity);
+    break;
+  }
+  case NestedNameSpecifier::Kind::Type: {
+    TypeSourceInfo *TSInfo = Context.getTrivialTypeSourceInfo(
+        QualType(Qualifier.getAsType(), 0), R.getBegin());
+    SavePointer(TSInfo->getTypeLoc().getOpaqueData(), Buffer, BufferSize,
+                BufferCapacity);
+    break;
+  }
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    break;
   }
+  SaveSourceLocation(R.getEnd(), Buffer, BufferSize, BufferCapacity);
 }
 
 void NestedNameSpecifierLocBuilder::Adopt(NestedNameSpecifierLoc Other) {
@@ -614,7 +320,7 @@ void NestedNameSpecifierLocBuilder::Adopt(NestedNameSpecifierLoc Other) {
     free(Buffer);
 
   if (!Other) {
-    Representation = nullptr;
+    Representation = std::nullopt;
     BufferSize = 0;
     return;
   }
diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp
index bd87d4418484b..fb95f58092c49 100644
--- a/clang/lib/AST/ODRHash.cpp
+++ b/clang/lib/AST/ODRHash.cpp
@@ -111,34 +111,28 @@ void ODRHash::AddDeclarationNameInfoImpl(DeclarationNameInfo NameInfo) {
   }
 }
 
-void ODRHash::AddNestedNameSpecifier(const NestedNameSpecifier *NNS) {
-  assert(NNS && "Expecting non-null pointer.");
-  const auto *Prefix = NNS->getPrefix();
-  AddBoolean(Prefix);
-  if (Prefix) {
-    AddNestedNameSpecifier(Prefix);
-  }
-  auto Kind = NNS->getKind();
-  ID.AddInteger(Kind);
+void ODRHash::AddNestedNameSpecifier(NestedNameSpecifier NNS) {
+  auto Kind = NNS.getKind();
+  ID.AddInteger(llvm::to_underlying(Kind));
   switch (Kind) {
-  case NestedNameSpecifier::Identifier:
-    AddIdentifierInfo(NNS->getAsIdentifier());
-    break;
-  case NestedNameSpecifier::Namespace:
-    AddDecl(NNS->getAsNamespace());
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [Namespace, Prefix] = NNS.getAsNamespaceAndPrefix();
+    AddDecl(Namespace);
+    AddNestedNameSpecifier(Prefix);
     break;
-  case NestedNameSpecifier::TypeSpec:
-    AddType(NNS->getAsType());
+  }
+  case NestedNameSpecifier::Kind::Type:
+    AddType(NNS.getAsType());
     break;
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Super:
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     break;
   }
 }
 
 void ODRHash::AddDependentTemplateName(const DependentTemplateStorage &Name) {
-  if (NestedNameSpecifier *NNS = Name.getQualifier())
-    AddNestedNameSpecifier(NNS);
+  AddNestedNameSpecifier(Name.getQualifier());
   if (IdentifierOrOverloadedOperator IO = Name.getName();
       const IdentifierInfo *II = IO.getIdentifier())
     AddIdentifierInfo(II);
@@ -156,8 +150,7 @@ void ODRHash::AddTemplateName(TemplateName Name) {
     break;
   case TemplateName::QualifiedTemplate: {
     QualifiedTemplateName *QTN = Name.getAsQualifiedTemplateName();
-    if (NestedNameSpecifier *NNS = QTN->getQualifier())
-      AddNestedNameSpecifier(NNS);
+    AddNestedNameSpecifier(QTN->getQualifier());
     AddBoolean(QTN->hasTemplateKeyword());
     AddTemplateName(QTN->getUnderlyingTemplate());
     break;
@@ -889,11 +882,8 @@ class ODRTypeVisitor : public TypeVisitor<ODRTypeVisitor> {
     }
   }
 
-  void AddNestedNameSpecifier(const NestedNameSpecifier *NNS) {
-    Hash.AddBoolean(NNS);
-    if (NNS) {
-      Hash.AddNestedNameSpecifier(NNS);
-    }
+  void AddNestedNameSpecifier(NestedNameSpecifier NNS) {
+    Hash.AddNestedNameSpecifier(NNS);
   }
 
   void AddIdentifierInfo(const IdentifierInfo *II) {
@@ -907,52 +897,33 @@ class ODRTypeVisitor : public TypeVisitor<ODRTypeVisitor> {
     ID.AddInteger(Quals.getAsOpaqueValue());
   }
 
-  // Return the RecordType if the typedef only strips away a keyword.
-  // Otherwise, return the original type.
-  static const Type *RemoveTypedef(const Type *T) {
+  // Handle typedefs which only strip away a keyword.
+  bool handleTypedef(const Type *T) {
     const auto *TypedefT = dyn_cast<TypedefType>(T);
-    if (!TypedefT) {
-      return T;
-    }
-
-    const TypedefNameDecl *D = TypedefT->getDecl();
-    QualType UnderlyingType = D->getUnderlyingType();
-
-    if (UnderlyingType.hasLocalQualifiers()) {
-      return T;
-    }
-
-    const auto *ElaboratedT = dyn_cast<ElaboratedType>(UnderlyingType);
-    if (!ElaboratedT) {
-      return T;
-    }
+    if (!TypedefT)
+      return false;
 
-    if (ElaboratedT->getQualifier() != nullptr) {
-      return T;
-    }
+    QualType UnderlyingType = TypedefT->desugar();
 
-    QualType NamedType = ElaboratedT->getNamedType();
-    if (NamedType.hasLocalQualifiers()) {
-      return T;
-    }
+    if (UnderlyingType.hasLocalQualifiers())
+      return false;
 
-    const auto *RecordT = dyn_cast<RecordType>(NamedType);
-    if (!RecordT) {
-      return T;
-    }
+    const auto *TagT = dyn_cast<TagType>(UnderlyingType);
+    if (!TagT || TagT->getQualifier())
+      return false;
 
-    const IdentifierInfo *TypedefII = TypedefT->getDecl()->getIdentifier();
-    const IdentifierInfo *RecordII = RecordT->getDecl()->getIdentifier();
-    if (!TypedefII || !RecordII ||
-        TypedefII->getName() != RecordII->getName()) {
-      return T;
-    }
+    if (TypedefT->getDecl()->getIdentifier() !=
+        TagT->getOriginalDecl()->getIdentifier())
+      return false;
 
-    return RecordT;
+    ID.AddInteger(TagT->getTypeClass());
+    VisitTagType(TagT, /*ElaboratedOverride=*/TypedefT);
+    return true;
   }
 
   void Visit(const Type *T) {
-    T = RemoveTypedef(T);
+    if (handleTypedef(T))
+      return;
     ID.AddInteger(T->getTypeClass());
     Inherited::Visit(T);
   }
@@ -1088,7 +1059,7 @@ class ODRTypeVisitor : public TypeVisitor<ODRTypeVisitor> {
   }
 
   void VisitInjectedClassNameType(const InjectedClassNameType *T) {
-    AddDecl(T->getDecl());
+    AddDecl(T->getOriginalDecl()->getDefinitionOrSelf());
     VisitType(T);
   }
 
@@ -1186,14 +1157,17 @@ class ODRTypeVisitor : public TypeVisitor<ODRTypeVisitor> {
     VisitType(T);
   }
 
-  void VisitTagType(const TagType *T) {
-    AddDecl(T->getDecl());
+  void VisitTagType(const TagType *T,
+                    const TypedefType *ElaboratedOverride = nullptr) {
+    ID.AddInteger(llvm::to_underlying(
+        ElaboratedOverride ? ElaboratedTypeKeyword::None : T->getKeyword()));
+    AddNestedNameSpecifier(ElaboratedOverride
+                               ? ElaboratedOverride->getQualifier()
+                               : T->getQualifier());
+    AddDecl(T->getOriginalDecl()->getDefinitionOrSelf());
     VisitType(T);
   }
 
-  void VisitRecordType(const RecordType *T) { VisitTagType(T); }
-  void VisitEnumType(const EnumType *T) { VisitTagType(T); }
-
   void VisitTemplateSpecializationType(const TemplateSpecializationType *T) {
     ID.AddInteger(T->template_arguments().size());
     for (const auto &TA : T->template_arguments()) {
@@ -1211,6 +1185,8 @@ class ODRTypeVisitor : public TypeVisitor<ODRTypeVisitor> {
   }
 
   void VisitTypedefType(const TypedefType *T) {
+    ID.AddInteger(llvm::to_underlying(T->getKeyword()));
+    AddNestedNameSpecifier(T->getQualifier());
     AddDecl(T->getDecl());
     VisitType(T);
   }
@@ -1247,12 +1223,6 @@ class ODRTypeVisitor : public TypeVisitor<ODRTypeVisitor> {
     VisitTypeWithKeyword(T);
   }
 
-  void VisitElaboratedType(const ElaboratedType *T) {
-    AddNestedNameSpecifier(T->getQualifier());
-    AddQualType(T->getNamedType());
-    VisitTypeWithKeyword(T);
-  }
-
   void VisitUnaryTransformType(const UnaryTransformType *T) {
     AddQualType(T->getUnderlyingType());
     AddQualType(T->getBaseType());
@@ -1330,7 +1300,7 @@ void ODRHash::AddStructuralValue(const APValue &Value) {
             TypeSoFar = FD->getType();
           } else {
             TypeSoFar =
-                D->getASTContext().getRecordType(cast<CXXRecordDecl>(D));
+                D->getASTContext().getCanonicalTagType(cast<CXXRecordDecl>(D));
           }
         }
       }
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index f7a98bd92190b..9a9ede467331e 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -329,10 +329,11 @@ OpenACCPrivateClause::Create(const ASTContext &C, SourceLocation BeginLoc,
 
 OpenACCFirstPrivateClause *OpenACCFirstPrivateClause::Create(
     const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
-    ArrayRef<Expr *> VarList, ArrayRef<VarDecl *> InitRecipes,
+    ArrayRef<Expr *> VarList, ArrayRef<OpenACCFirstPrivateRecipe> InitRecipes,
     SourceLocation EndLoc) {
-  void *Mem =
-      C.Allocate(OpenACCFirstPrivateClause::totalSizeToAlloc<Expr *, VarDecl *>(
+  void *Mem = C.Allocate(
+      OpenACCFirstPrivateClause::totalSizeToAlloc<Expr *,
+                                                  OpenACCFirstPrivateRecipe>(
           VarList.size(), InitRecipes.size()));
   return new (Mem) OpenACCFirstPrivateClause(BeginLoc, LParenLoc, VarList,
                                              InitRecipes, EndLoc);
@@ -505,11 +506,13 @@ OpenACCDeviceTypeClause *OpenACCDeviceTypeClause::Create(
 OpenACCReductionClause *OpenACCReductionClause::Create(
     const ASTContext &C, SourceLocation BeginLoc, SourceLocation LParenLoc,
     OpenACCReductionOperator Operator, ArrayRef<Expr *> VarList,
+    ArrayRef<OpenACCReductionRecipe> Recipes,
     SourceLocation EndLoc) {
   void *Mem = C.Allocate(
-      OpenACCReductionClause::totalSizeToAlloc<Expr *>(VarList.size()));
-  return new (Mem)
-      OpenACCReductionClause(BeginLoc, LParenLoc, Operator, VarList, EndLoc);
+      OpenACCReductionClause::totalSizeToAlloc<Expr *, OpenACCReductionRecipe>(
+          VarList.size(), Recipes.size()));
+  return new (Mem) OpenACCReductionClause(BeginLoc, LParenLoc, Operator,
+                                          VarList, Recipes, EndLoc);
 }
 
 OpenACCAutoClause *OpenACCAutoClause::Create(const ASTContext &C,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index de8b5996818de..588b0dcc6d7b8 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -2350,17 +2350,16 @@ void OMPClausePrinter::VisitOMPReductionClause(OMPReductionClause *Node) {
     if (Node->getModifierLoc().isValid())
       OS << getOpenMPSimpleClauseTypeName(OMPC_reduction, Node->getModifier())
          << ", ";
-    NestedNameSpecifier *QualifierLoc =
+    NestedNameSpecifier Qualifier =
         Node->getQualifierLoc().getNestedNameSpecifier();
     OverloadedOperatorKind OOK =
         Node->getNameInfo().getName().getCXXOverloadedOperator();
-    if (QualifierLoc == nullptr && OOK != OO_None) {
+    if (!Qualifier && OOK != OO_None) {
       // Print reduction identifier in C format
       OS << getOperatorSpelling(OOK);
     } else {
       // Use C++ format
-      if (QualifierLoc != nullptr)
-        QualifierLoc->print(OS, Policy);
+      Qualifier.print(OS, Policy);
       OS << Node->getNameInfo();
     }
     OS << ":";
@@ -2373,17 +2372,16 @@ void OMPClausePrinter::VisitOMPTaskReductionClause(
     OMPTaskReductionClause *Node) {
   if (!Node->varlist_empty()) {
     OS << "task_reduction(";
-    NestedNameSpecifier *QualifierLoc =
+    NestedNameSpecifier Qualifier =
         Node->getQualifierLoc().getNestedNameSpecifier();
     OverloadedOperatorKind OOK =
         Node->getNameInfo().getName().getCXXOverloadedOperator();
-    if (QualifierLoc == nullptr && OOK != OO_None) {
+    if (!Qualifier && OOK != OO_None) {
       // Print reduction identifier in C format
       OS << getOperatorSpelling(OOK);
     } else {
       // Use C++ format
-      if (QualifierLoc != nullptr)
-        QualifierLoc->print(OS, Policy);
+      Qualifier.print(OS, Policy);
       OS << Node->getNameInfo();
     }
     OS << ":";
@@ -2395,17 +2393,16 @@ void OMPClausePrinter::VisitOMPTaskReductionClause(
 void OMPClausePrinter::VisitOMPInReductionClause(OMPInReductionClause *Node) {
   if (!Node->varlist_empty()) {
     OS << "in_reduction(";
-    NestedNameSpecifier *QualifierLoc =
+    NestedNameSpecifier Qualifier =
         Node->getQualifierLoc().getNestedNameSpecifier();
     OverloadedOperatorKind OOK =
         Node->getNameInfo().getName().getCXXOverloadedOperator();
-    if (QualifierLoc == nullptr && OOK != OO_None) {
+    if (!Qualifier && OOK != OO_None) {
       // Print reduction identifier in C format
       OS << getOperatorSpelling(OOK);
     } else {
       // Use C++ format
-      if (QualifierLoc != nullptr)
-        QualifierLoc->print(OS, Policy);
+      Qualifier.print(OS, Policy);
       OS << Node->getNameInfo();
     }
     OS << ":";
@@ -2508,10 +2505,9 @@ template <typename T>
 static void PrintMapper(raw_ostream &OS, T *Node,
                         const PrintingPolicy &Policy) {
   OS << '(';
-  NestedNameSpecifier *MapperNNS =
+  NestedNameSpecifier MapperNNS =
       Node->getMapperQualifierLoc().getNestedNameSpecifier();
-  if (MapperNNS)
-    MapperNNS->print(OS, Policy);
+  MapperNNS.print(OS, Policy);
   OS << Node->getMapperIdInfo() << ')';
 }
 
diff --git a/clang/lib/AST/ParentMapContext.cpp b/clang/lib/AST/ParentMapContext.cpp
index 68dfe4d5d22cb..acc011cb2faa4 100644
--- a/clang/lib/AST/ParentMapContext.cpp
+++ b/clang/lib/AST/ParentMapContext.cpp
@@ -438,10 +438,12 @@ class ParentMapContext::ParentMap::ASTVisitor
         DeclNode, DeclNode, [&] { return VisitorBase::TraverseDecl(DeclNode); },
         &Map.PointerParents);
   }
-  bool TraverseTypeLoc(TypeLoc TypeLocNode) {
+  bool TraverseTypeLoc(TypeLoc TypeLocNode, bool TraverseQualifier = true) {
     return TraverseNode(
         TypeLocNode, DynTypedNode::create(TypeLocNode),
-        [&] { return VisitorBase::TraverseTypeLoc(TypeLocNode); },
+        [&] {
+          return VisitorBase::TraverseTypeLoc(TypeLocNode, TraverseQualifier);
+        },
         &Map.OtherParents);
   }
   bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNSLocNode) {
diff --git a/clang/lib/AST/PrintfFormatString.cpp b/clang/lib/AST/PrintfFormatString.cpp
index bcd44f0a85eed..687160c6116be 100644
--- a/clang/lib/AST/PrintfFormatString.cpp
+++ b/clang/lib/AST/PrintfFormatString.cpp
@@ -794,7 +794,7 @@ bool PrintfSpecifier::fixType(QualType QT, const LangOptions &LangOpt,
 
   // If it's an enum, get its underlying type.
   if (const EnumType *ETy = QT->getAs<EnumType>())
-    QT = ETy->getDecl()->getIntegerType();
+    QT = ETy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   const BuiltinType *BT = QT->getAs<BuiltinType>();
   if (!BT) {
diff --git a/clang/lib/AST/QualTypeNames.cpp b/clang/lib/AST/QualTypeNames.cpp
index 9731b3a1a3e3b..ee7fec3372fcf 100644
--- a/clang/lib/AST/QualTypeNames.cpp
+++ b/clang/lib/AST/QualTypeNames.cpp
@@ -24,10 +24,9 @@ namespace TypeName {
 /// is requested.
 /// \param[in] WithGlobalNsPrefix - Indicate whether the global namespace
 /// specifier "::" should be prepended or not.
-static NestedNameSpecifier *createNestedNameSpecifier(
-    const ASTContext &Ctx,
-    const NamespaceDecl *Namesp,
-    bool WithGlobalNsPrefix);
+static NestedNameSpecifier
+createNestedNameSpecifier(const ASTContext &Ctx, const NamespaceDecl *Namesp,
+                          bool WithGlobalNsPrefix);
 
 /// Create a NestedNameSpecifier for TagDecl and its enclosing
 /// scopes.
@@ -39,22 +38,24 @@ static NestedNameSpecifier *createNestedNameSpecifier(
 /// qualified names.
 /// \param[in] WithGlobalNsPrefix - Indicate whether the global namespace
 /// specifier "::" should be prepended or not.
-static NestedNameSpecifier *createNestedNameSpecifier(
-    const ASTContext &Ctx, const TypeDecl *TD,
-    bool FullyQualify, bool WithGlobalNsPrefix);
+static NestedNameSpecifier createNestedNameSpecifier(const ASTContext &Ctx,
+                                                     const TypeDecl *TD,
+                                                     bool FullyQualify,
+                                                     bool WithGlobalNsPrefix);
 
-static NestedNameSpecifier *createNestedNameSpecifierForScopeOf(
-    const ASTContext &Ctx, const Decl *decl,
-    bool FullyQualified, bool WithGlobalNsPrefix);
+static NestedNameSpecifier
+createNestedNameSpecifierForScopeOf(const ASTContext &Ctx, const Decl *decl,
+                                    bool FullyQualified,
+                                    bool WithGlobalNsPrefix);
 
-static NestedNameSpecifier *getFullyQualifiedNestedNameSpecifier(
-    const ASTContext &Ctx, NestedNameSpecifier *scope, bool WithGlobalNsPrefix);
+static NestedNameSpecifier getFullyQualifiedNestedNameSpecifier(
+    const ASTContext &Ctx, NestedNameSpecifier NNS, bool WithGlobalNsPrefix);
 
 static bool getFullyQualifiedTemplateName(const ASTContext &Ctx,
                                           TemplateName &TName,
                                           bool WithGlobalNsPrefix) {
   bool Changed = false;
-  NestedNameSpecifier *NNS = nullptr;
+  NestedNameSpecifier NNS = std::nullopt;
 
   TemplateDecl *ArgTDecl = TName.getAsTemplateDecl();
   // ArgTDecl won't be NULL because we asserted that this isn't a
@@ -65,13 +66,13 @@ static bool getFullyQualifiedTemplateName(const ASTContext &Ctx,
   if (QTName &&
       !QTName->hasTemplateKeyword() &&
       (NNS = QTName->getQualifier())) {
-    NestedNameSpecifier *QNNS = getFullyQualifiedNestedNameSpecifier(
-        Ctx, NNS, WithGlobalNsPrefix);
+    NestedNameSpecifier QNNS =
+        getFullyQualifiedNestedNameSpecifier(Ctx, NNS, WithGlobalNsPrefix);
     if (QNNS != NNS) {
       Changed = true;
       NNS = QNNS;
     } else {
-      NNS = nullptr;
+      NNS = std::nullopt;
     }
   } else {
     NNS = createNestedNameSpecifierForScopeOf(
@@ -116,76 +117,81 @@ static bool getFullyQualifiedTemplateArgument(const ASTContext &Ctx,
 }
 
 static const Type *getFullyQualifiedTemplateType(const ASTContext &Ctx,
-                                                 const Type *TypePtr,
+                                                 const TagType *TSTRecord,
+                                                 ElaboratedTypeKeyword Keyword,
+                                                 NestedNameSpecifier Qualifier,
                                                  bool WithGlobalNsPrefix) {
-  // DependentTemplateTypes exist within template declarations and
-  // definitions. Therefore we shouldn't encounter them at the end of
-  // a translation unit. If we do, the caller has made an error.
-  assert(!isa<DependentTemplateSpecializationType>(TypePtr));
-  // In case of template specializations, iterate over the arguments
-  // and fully qualify them as well.
-  if (const auto *TST = dyn_cast<const TemplateSpecializationType>(TypePtr)) {
-    bool MightHaveChanged = false;
-    SmallVector<TemplateArgument, 4> FQArgs;
-    // Cheap to copy and potentially modified by
-    // getFullyQualifedTemplateArgument.
-    for (TemplateArgument Arg : TST->template_arguments()) {
-      MightHaveChanged |= getFullyQualifiedTemplateArgument(
-          Ctx, Arg, WithGlobalNsPrefix);
-      FQArgs.push_back(Arg);
-    }
+  // We are asked to fully qualify and we have a Record Type,
+  // which can point to a template instantiation with no sugar in any of
+  // its template argument, however we still need to fully qualify them.
+
+  const auto *TD = TSTRecord->getOriginalDecl();
+  const auto *TSTDecl = dyn_cast<ClassTemplateSpecializationDecl>(TD);
+  if (!TSTDecl)
+    return Ctx.getTagType(Keyword, Qualifier, TD, /*OwnsTag=*/false)
+        .getTypePtr();
+
+  const TemplateArgumentList &TemplateArgs = TSTDecl->getTemplateArgs();
+
+  bool MightHaveChanged = false;
+  SmallVector<TemplateArgument, 4> FQArgs;
+  for (unsigned int I = 0, E = TemplateArgs.size(); I != E; ++I) {
+    // cheap to copy and potentially modified by
+    // getFullyQualifedTemplateArgument
+    TemplateArgument Arg(TemplateArgs[I]);
+    MightHaveChanged |=
+        getFullyQualifiedTemplateArgument(Ctx, Arg, WithGlobalNsPrefix);
+    FQArgs.push_back(Arg);
+  }
 
-    // If a fully qualified arg is different from the unqualified arg,
-    // allocate new type in the AST.
-    if (MightHaveChanged) {
-      QualType QT = Ctx.getTemplateSpecializationType(
-          TST->getTemplateName(), FQArgs,
-          /*CanonicalArgs=*/{}, TST->desugar());
-      // getTemplateSpecializationType returns a fully qualified
-      // version of the specialization itself, so no need to qualify
-      // it.
-      return QT.getTypePtr();
-    }
-  } else if (const auto *TSTRecord = dyn_cast<const RecordType>(TypePtr)) {
-    // We are asked to fully qualify and we have a Record Type,
-    // which can point to a template instantiation with no sugar in any of
-    // its template argument, however we still need to fully qualify them.
-
-    if (const auto *TSTDecl =
-        dyn_cast<ClassTemplateSpecializationDecl>(TSTRecord->getDecl())) {
-      const TemplateArgumentList &TemplateArgs = TSTDecl->getTemplateArgs();
-
-      bool MightHaveChanged = false;
-      SmallVector<TemplateArgument, 4> FQArgs;
-      for (unsigned int I = 0, E = TemplateArgs.size(); I != E; ++I) {
-        // cheap to copy and potentially modified by
-        // getFullyQualifedTemplateArgument
-        TemplateArgument Arg(TemplateArgs[I]);
-        MightHaveChanged |= getFullyQualifiedTemplateArgument(
-            Ctx, Arg, WithGlobalNsPrefix);
-        FQArgs.push_back(Arg);
-      }
+  if (!MightHaveChanged)
+    return Ctx.getTagType(Keyword, Qualifier, TD, /*OwnsTag=*/false)
+        .getTypePtr();
+  // If a fully qualified arg is different from the unqualified arg,
+  // allocate new type in the AST.
+  TemplateName TN = Ctx.getQualifiedTemplateName(
+      Qualifier, /*TemplateKeyword=*/false,
+      TemplateName(TSTDecl->getSpecializedTemplate()));
+  QualType QT = Ctx.getTemplateSpecializationType(
+      Keyword, TN, FQArgs,
+      /*CanonicalArgs=*/{}, TSTRecord->getCanonicalTypeInternal());
+  // getTemplateSpecializationType returns a fully qualified
+  // version of the specialization itself, so no need to qualify
+  // it.
+  return QT.getTypePtr();
+}
 
-      // If a fully qualified arg is different from the unqualified arg,
-      // allocate new type in the AST.
-      if (MightHaveChanged) {
-        TemplateName TN(TSTDecl->getSpecializedTemplate());
-        QualType QT = Ctx.getTemplateSpecializationType(
-            TN, FQArgs,
-            /*CanonicalArgs=*/{}, TSTRecord->getCanonicalTypeInternal());
-        // getTemplateSpecializationType returns a fully qualified
-        // version of the specialization itself, so no need to qualify
-        // it.
-        return QT.getTypePtr();
-      }
-    }
+static const Type *
+getFullyQualifiedTemplateType(const ASTContext &Ctx,
+                              const TemplateSpecializationType *TST,
+                              bool WithGlobalNsPrefix) {
+  TemplateName TName = TST->getTemplateName();
+  bool MightHaveChanged =
+      getFullyQualifiedTemplateName(Ctx, TName, WithGlobalNsPrefix);
+  SmallVector<TemplateArgument, 4> FQArgs;
+  // Cheap to copy and potentially modified by
+  // getFullyQualifedTemplateArgument.
+  for (TemplateArgument Arg : TST->template_arguments()) {
+    MightHaveChanged |=
+        getFullyQualifiedTemplateArgument(Ctx, Arg, WithGlobalNsPrefix);
+    FQArgs.push_back(Arg);
   }
-  return TypePtr;
+
+  if (!MightHaveChanged)
+    return TST;
+
+  QualType NewQT =
+      Ctx.getTemplateSpecializationType(TST->getKeyword(), TName, FQArgs,
+                                        /*CanonicalArgs=*/{}, TST->desugar());
+  // getTemplateSpecializationType returns a fully qualified
+  // version of the specialization itself, so no need to qualify
+  // it.
+  return NewQT.getTypePtr();
 }
 
-static NestedNameSpecifier *createOuterNNS(const ASTContext &Ctx, const Decl *D,
-                                           bool FullyQualify,
-                                           bool WithGlobalNsPrefix) {
+static NestedNameSpecifier createOuterNNS(const ASTContext &Ctx, const Decl *D,
+                                          bool FullyQualify,
+                                          bool WithGlobalNsPrefix) {
   const DeclContext *DC = D->getDeclContext();
   if (const auto *NS = dyn_cast<NamespaceDecl>(DC)) {
     while (NS && NS->isInline()) {
@@ -195,71 +201,63 @@ static NestedNameSpecifier *createOuterNNS(const ASTContext &Ctx, const Decl *D,
     if (NS && NS->getDeclName()) {
       return createNestedNameSpecifier(Ctx, NS, WithGlobalNsPrefix);
     }
-    return nullptr;  // no starting '::', no anonymous
-  } else if (const auto *TD = dyn_cast<TagDecl>(DC)) {
-    return createNestedNameSpecifier(Ctx, TD, FullyQualify, WithGlobalNsPrefix);
-  } else if (const auto *TDD = dyn_cast<TypedefNameDecl>(DC)) {
-    return createNestedNameSpecifier(
-        Ctx, TDD, FullyQualify, WithGlobalNsPrefix);
-  } else if (WithGlobalNsPrefix && DC->isTranslationUnit()) {
-    return NestedNameSpecifier::GlobalSpecifier(Ctx);
+    return std::nullopt; // no starting '::', no anonymous
   }
-  return nullptr;  // no starting '::' if |WithGlobalNsPrefix| is false
+  if (const auto *TD = dyn_cast<TagDecl>(DC))
+    return createNestedNameSpecifier(Ctx, TD, FullyQualify, WithGlobalNsPrefix);
+  if (const auto *TDD = dyn_cast<TypedefNameDecl>(DC))
+    return createNestedNameSpecifier(Ctx, TDD, FullyQualify,
+                                     WithGlobalNsPrefix);
+  if (WithGlobalNsPrefix && DC->isTranslationUnit())
+    return NestedNameSpecifier::getGlobal();
+  return std::nullopt; // no starting '::' if |WithGlobalNsPrefix| is false
 }
 
 /// Return a fully qualified version of this name specifier.
-static NestedNameSpecifier *getFullyQualifiedNestedNameSpecifier(
-    const ASTContext &Ctx, NestedNameSpecifier *Scope,
-    bool WithGlobalNsPrefix) {
-  switch (Scope->getKind()) {
-    case NestedNameSpecifier::Global:
-    case NestedNameSpecifier::Super:
-      // Already fully qualified
-      return Scope;
-    case NestedNameSpecifier::Namespace:
-      return TypeName::createNestedNameSpecifier(
-          Ctx, Scope->getAsNamespace()->getNamespace(), WithGlobalNsPrefix);
-    case NestedNameSpecifier::Identifier:
-      // A function or some other construct that makes it un-namable
-      // at the end of the TU. Skip the current component of the name,
-      // but use the name of it's prefix.
-      return getFullyQualifiedNestedNameSpecifier(
-          Ctx, Scope->getPrefix(), WithGlobalNsPrefix);
-    case NestedNameSpecifier::TypeSpec: {
-      const Type *Type = Scope->getAsType();
-      // Find decl context.
-      const TagDecl *TD = nullptr;
-      if (const TagType *TagDeclType = Type->getAs<TagType>()) {
-        TD = TagDeclType->getDecl();
-      } else {
-        TD = Type->getAsCXXRecordDecl();
-      }
-      if (TD) {
-        return TypeName::createNestedNameSpecifier(Ctx, TD,
-                                                   true /*FullyQualified*/,
-                                                   WithGlobalNsPrefix);
-      } else if (const auto *TDD = dyn_cast<TypedefType>(Type)) {
-        return TypeName::createNestedNameSpecifier(Ctx, TDD->getDecl(),
-                                                   true /*FullyQualified*/,
-                                                   WithGlobalNsPrefix);
-      }
+static NestedNameSpecifier getFullyQualifiedNestedNameSpecifier(
+    const ASTContext &Ctx, NestedNameSpecifier Scope, bool WithGlobalNsPrefix) {
+  switch (Scope.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+    llvm_unreachable("can't fully qualify the empty nested name specifier");
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    // Already fully qualified
+    return Scope;
+  case NestedNameSpecifier::Kind::Namespace:
+    return TypeName::createNestedNameSpecifier(
+        Ctx, Scope.getAsNamespaceAndPrefix().Namespace->getNamespace(),
+        WithGlobalNsPrefix);
+  case NestedNameSpecifier::Kind::Type: {
+    const Type *Type = Scope.getAsType();
+    // Find decl context.
+    const TypeDecl *TD;
+    if (const TagType *TagDeclType = Type->getAs<TagType>())
+      TD = TagDeclType->getOriginalDecl();
+    else if (const auto *D = dyn_cast<TypedefType>(Type))
+      TD = D->getDecl();
+    else
       return Scope;
-    }
+    return TypeName::createNestedNameSpecifier(Ctx, TD, /*FullyQualify=*/true,
+                                               WithGlobalNsPrefix);
+  }
   }
   llvm_unreachable("bad NNS kind");
 }
 
 /// Create a nested name specifier for the declaring context of
 /// the type.
-static NestedNameSpecifier *createNestedNameSpecifierForScopeOf(
-    const ASTContext &Ctx, const Decl *Decl,
-    bool FullyQualified, bool WithGlobalNsPrefix) {
+static NestedNameSpecifier
+createNestedNameSpecifierForScopeOf(const ASTContext &Ctx, const Decl *Decl,
+                                    bool FullyQualified,
+                                    bool WithGlobalNsPrefix) {
   assert(Decl);
 
   const DeclContext *DC = Decl->getDeclContext()->getRedeclContext();
   const auto *Outer = dyn_cast<NamedDecl>(DC);
   const auto *OuterNS = dyn_cast<NamespaceDecl>(DC);
-  if (Outer && !(OuterNS && OuterNS->isAnonymousNamespace())) {
+  if (OuterNS && OuterNS->isAnonymousNamespace())
+    OuterNS = dyn_cast<NamespaceDecl>(OuterNS->getParent());
+  if (Outer) {
     if (const auto *CxxDecl = dyn_cast<CXXRecordDecl>(DC)) {
       if (ClassTemplateDecl *ClassTempl =
               CxxDecl->getDescribedClassTemplate()) {
@@ -288,76 +286,80 @@ static NestedNameSpecifier *createNestedNameSpecifierForScopeOf(
           Ctx, TD, FullyQualified, WithGlobalNsPrefix);
     } else if (isa<TranslationUnitDecl>(Outer)) {
       // Context is the TU. Nothing needs to be done.
-      return nullptr;
+      return std::nullopt;
     } else {
       // Decl's context was neither the TU, a namespace, nor a
       // TagDecl, which means it is a type local to a scope, and not
       // accessible at the end of the TU.
-      return nullptr;
+      return std::nullopt;
     }
   } else if (WithGlobalNsPrefix && DC->isTranslationUnit()) {
-    return NestedNameSpecifier::GlobalSpecifier(Ctx);
+    return NestedNameSpecifier::getGlobal();
   }
-  return nullptr;
+  return std::nullopt;
 }
 
 /// Create a nested name specifier for the declaring context of
 /// the type.
-static NestedNameSpecifier *createNestedNameSpecifierForScopeOf(
-    const ASTContext &Ctx, const Type *TypePtr,
-    bool FullyQualified, bool WithGlobalNsPrefix) {
-  if (!TypePtr) return nullptr;
+static NestedNameSpecifier
+createNestedNameSpecifierForScopeOf(const ASTContext &Ctx, const Type *TypePtr,
+                                    bool FullyQualified,
+                                    bool WithGlobalNsPrefix) {
+  if (!TypePtr)
+    return std::nullopt;
 
   Decl *Decl = nullptr;
   // There are probably other cases ...
   if (const auto *TDT = dyn_cast<TypedefType>(TypePtr)) {
     Decl = TDT->getDecl();
   } else if (const auto *TagDeclType = dyn_cast<TagType>(TypePtr)) {
-    Decl = TagDeclType->getDecl();
+    Decl = TagDeclType->getOriginalDecl();
   } else if (const auto *TST = dyn_cast<TemplateSpecializationType>(TypePtr)) {
     Decl = TST->getTemplateName().getAsTemplateDecl();
   } else {
     Decl = TypePtr->getAsCXXRecordDecl();
   }
 
-  if (!Decl) return nullptr;
+  if (!Decl)
+    return std::nullopt;
 
   return createNestedNameSpecifierForScopeOf(
       Ctx, Decl, FullyQualified, WithGlobalNsPrefix);
 }
 
-NestedNameSpecifier *createNestedNameSpecifier(const ASTContext &Ctx,
-                                               const NamespaceDecl *Namespace,
-                                               bool WithGlobalNsPrefix) {
+static NestedNameSpecifier
+createNestedNameSpecifier(const ASTContext &Ctx, const NamespaceDecl *Namespace,
+                          bool WithGlobalNsPrefix) {
   while (Namespace && Namespace->isInline()) {
     // Ignore inline namespace;
     Namespace = dyn_cast<NamespaceDecl>(Namespace->getDeclContext());
   }
-  if (!Namespace) return nullptr;
+  if (!Namespace)
+    return std::nullopt;
 
-  bool FullyQualified = true;  // doesn't matter, DeclContexts are namespaces
-  return NestedNameSpecifier::Create(
-      Ctx,
-      createOuterNNS(Ctx, Namespace, FullyQualified, WithGlobalNsPrefix),
-      Namespace);
+  bool FullyQualify = true; // doesn't matter, DeclContexts are namespaces
+  return NestedNameSpecifier(
+      Ctx, Namespace,
+      createOuterNNS(Ctx, Namespace, FullyQualify, WithGlobalNsPrefix));
 }
 
-NestedNameSpecifier *createNestedNameSpecifier(const ASTContext &Ctx,
-                                               const TypeDecl *TD,
-                                               bool FullyQualify,
-                                               bool WithGlobalNsPrefix) {
-  const Type *TypePtr = TD->getTypeForDecl();
-  if (isa<const TemplateSpecializationType>(TypePtr) ||
-      isa<const RecordType>(TypePtr)) {
+NestedNameSpecifier createNestedNameSpecifier(const ASTContext &Ctx,
+                                              const TypeDecl *TD,
+                                              bool FullyQualify,
+                                              bool WithGlobalNsPrefix) {
+  const Type *TypePtr = Ctx.getTypeDeclType(TD).getTypePtr();
+  if (auto *RD = dyn_cast<TagType>(TypePtr)) {
     // We are asked to fully qualify and we have a Record Type (which
     // may point to a template specialization) or Template
     // Specialization Type. We need to fully qualify their arguments.
-
-    TypePtr = getFullyQualifiedTemplateType(Ctx, TypePtr, WithGlobalNsPrefix);
+    TypePtr = getFullyQualifiedTemplateType(
+        Ctx, RD, ElaboratedTypeKeyword::None,
+        createOuterNNS(Ctx, TD, FullyQualify, WithGlobalNsPrefix),
+        WithGlobalNsPrefix);
+  } else if (auto *TST = dyn_cast<TemplateSpecializationType>(TypePtr)) {
+    TypePtr = getFullyQualifiedTemplateType(Ctx, TST, WithGlobalNsPrefix);
   }
-
-  return NestedNameSpecifier::Create(
-      Ctx, createOuterNNS(Ctx, TD, FullyQualify, WithGlobalNsPrefix), TypePtr);
+  return NestedNameSpecifier(TypePtr);
 }
 
 /// Return the fully qualified type, including fully-qualified
@@ -381,7 +383,7 @@ QualType getFullyQualifiedType(QualType QT, const ASTContext &Ctx,
     Qualifiers Quals = QT.getQualifiers();
     // Fully qualify the pointee and class types.
     QT = getFullyQualifiedType(QT->getPointeeType(), Ctx, WithGlobalNsPrefix);
-    NestedNameSpecifier *Qualifier = getFullyQualifiedNestedNameSpecifier(
+    NestedNameSpecifier Qualifier = getFullyQualifiedNestedNameSpecifier(
         Ctx, MPT->getQualifier(), WithGlobalNsPrefix);
     QT = Ctx.getMemberPointerType(QT, Qualifier,
                                   MPT->getMostRecentCXXRecordDecl());
@@ -434,45 +436,48 @@ QualType getFullyQualifiedType(QualType QT, const ASTContext &Ctx,
     QT = Ctx.getQualifiedType(QT, Quals);
   }
 
-  NestedNameSpecifier *Prefix = nullptr;
+  if (const auto *TST =
+          dyn_cast<const TemplateSpecializationType>(QT.getTypePtr())) {
+
+    const Type *T = getFullyQualifiedTemplateType(Ctx, TST, WithGlobalNsPrefix);
+    if (T == TST)
+      return QT;
+    return Ctx.getQualifiedType(T, QT.getQualifiers());
+  }
+
   // Local qualifiers are attached to the QualType outside of the
   // elaborated type.  Retrieve them before descending into the
   // elaborated type.
   Qualifiers PrefixQualifiers = QT.getLocalQualifiers();
   QT = QualType(QT.getTypePtr(), 0);
-  ElaboratedTypeKeyword Keyword = ElaboratedTypeKeyword::None;
-  if (const auto *ETypeInput = dyn_cast<ElaboratedType>(QT.getTypePtr())) {
-    QT = ETypeInput->getNamedType();
-    assert(!QT.hasLocalQualifiers());
-    Keyword = ETypeInput->getKeyword();
-  }
 
   // We don't consider the alias introduced by `using a::X` as a new type.
   // The qualified name is still a::X.
   if (const auto *UT = QT->getAs<UsingType>()) {
-    QT = Ctx.getQualifiedType(UT->getUnderlyingType(), PrefixQualifiers);
+    QT = Ctx.getQualifiedType(UT->desugar(), PrefixQualifiers);
     return getFullyQualifiedType(QT, Ctx, WithGlobalNsPrefix);
   }
 
   // Create a nested name specifier if needed.
-  Prefix = createNestedNameSpecifierForScopeOf(Ctx, QT.getTypePtr(),
-                                               true /*FullyQualified*/,
-                                               WithGlobalNsPrefix);
+  NestedNameSpecifier Prefix = createNestedNameSpecifierForScopeOf(
+      Ctx, QT.getTypePtr(), true /*FullyQualified*/, WithGlobalNsPrefix);
 
   // In case of template specializations iterate over the arguments and
   // fully qualify them as well.
-  if (isa<const TemplateSpecializationType>(QT.getTypePtr()) ||
-      isa<const RecordType>(QT.getTypePtr())) {
+  if (const auto *TT = dyn_cast<TagType>(QT.getTypePtr())) {
     // We are asked to fully qualify and we have a Record Type (which
     // may point to a template specialization) or Template
     // Specialization Type. We need to fully qualify their arguments.
 
     const Type *TypePtr = getFullyQualifiedTemplateType(
-        Ctx, QT.getTypePtr(), WithGlobalNsPrefix);
+        Ctx, TT, TT->getKeyword(), Prefix, WithGlobalNsPrefix);
     QT = QualType(TypePtr, 0);
-  }
-  if (Prefix || Keyword != ElaboratedTypeKeyword::None) {
-    QT = Ctx.getElaboratedType(Keyword, Prefix, QT);
+  } else if (const auto *TT = dyn_cast<TypedefType>(QT.getTypePtr())) {
+    QT = Ctx.getTypedefType(
+        TT->getKeyword(), Prefix, TT->getDecl(),
+        getFullyQualifiedType(TT->desugar(), Ctx, WithGlobalNsPrefix));
+  } else {
+    assert(!Prefix && "Unhandled type node");
   }
   QT = Ctx.getQualifiedType(QT, PrefixQualifiers);
   return QT;
@@ -486,5 +491,12 @@ std::string getFullyQualifiedName(QualType QT,
   return FQQT.getAsString(Policy);
 }
 
+NestedNameSpecifier getFullyQualifiedDeclaredContext(const ASTContext &Ctx,
+                                                     const Decl *Decl,
+                                                     bool WithGlobalNsPrefix) {
+  return createNestedNameSpecifierForScopeOf(Ctx, Decl, /*FullyQualified=*/true,
+                                             WithGlobalNsPrefix);
+}
+
 }  // end namespace TypeName
 }  // end namespace clang
diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp
index 760b2fcdc0681..f1f21f426f944 100644
--- a/clang/lib/AST/RecordLayoutBuilder.cpp
+++ b/clang/lib/AST/RecordLayoutBuilder.cpp
@@ -2012,8 +2012,7 @@ void ItaniumRecordLayoutBuilder::LayoutField(const FieldDecl *D,
     } else if (const BuiltinType *BTy = BaseTy->getAs<BuiltinType>()) {
       performBuiltinTypeAlignmentUpgrade(BTy);
     } else if (const RecordType *RT = BaseTy->getAs<RecordType>()) {
-      const RecordDecl *RD = RT->getDecl();
-      assert(RD && "Expected non-null RecordDecl.");
+      const RecordDecl *RD = RT->getOriginalDecl();
       const ASTRecordLayout &FieldRecord = Context.getASTRecordLayout(RD);
       PreferredAlign = FieldRecord.getPreferredAlignment();
     }
@@ -2128,7 +2127,8 @@ void ItaniumRecordLayoutBuilder::LayoutField(const FieldDecl *D,
           // TODO: Takes no account the alignment of the outer struct
           if (FieldOffset % OriginalFieldAlign != 0)
             Diag(D->getLocation(), diag::warn_unaligned_access)
-                << Context.getTypeDeclType(RD) << D->getName() << D->getType();
+                << Context.getCanonicalTagType(RD) << D->getName()
+                << D->getType();
         }
   }
 
@@ -2193,8 +2193,7 @@ void ItaniumRecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
         InBits = false;
       }
       Diag(RD->getLocation(), diag::warn_padded_struct_size)
-          << Context.getTypeDeclType(RD)
-          << PadSize
+          << Context.getCanonicalTagType(RD) << PadSize
           << (InBits ? 1 : 0); // (byte|bit)
     }
 
@@ -2212,7 +2211,7 @@ void ItaniumRecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
          Context.getLangOpts().getClangABICompat() <=
              LangOptions::ClangABI::Ver15))
       Diag(D->getLocation(), diag::warn_unnecessary_packed)
-          << Context.getTypeDeclType(RD);
+          << Context.getCanonicalTagType(RD);
   }
 }
 
@@ -2306,7 +2305,7 @@ static void CheckFieldPadding(const ASTContext &Context, bool IsUnion,
       Context.getDiagnostics().Report(D->getLocation(),
                                       Diagnostic)
           << getPaddingDiagFromTagKind(D->getParent()->getTagKind())
-          << Context.getTypeDeclType(D->getParent()) << PadSize
+          << Context.getCanonicalTagType(D->getParent()) << PadSize
           << (InBits ? 1 : 0) // (byte|bit)
           << D->getIdentifier();
     } else {
@@ -2315,7 +2314,7 @@ static void CheckFieldPadding(const ASTContext &Context, bool IsUnion,
       Context.getDiagnostics().Report(D->getLocation(),
                                       Diagnostic)
           << getPaddingDiagFromTagKind(D->getParent()->getTagKind())
-          << Context.getTypeDeclType(D->getParent()) << PadSize
+          << Context.getCanonicalTagType(D->getParent()) << PadSize
           << (InBits ? 1 : 0); // (byte|bit)
     }
   }
@@ -2714,7 +2713,7 @@ MicrosoftRecordLayoutBuilder::getAdjustedElementInfo(
   else {
     if (auto RT =
             FD->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
-      auto const &Layout = Context.getASTRecordLayout(RT->getDecl());
+      auto const &Layout = Context.getASTRecordLayout(RT->getOriginalDecl());
       EndsWithZeroSizedObject = Layout.endsWithZeroSizedObject();
       FieldRequiredAlignment = std::max(FieldRequiredAlignment,
                                         Layout.getRequiredAlignment());
@@ -3273,7 +3272,7 @@ void MicrosoftRecordLayoutBuilder::finalizeLayout(const RecordDecl *RD) {
 
     Context.getDiagnostics().Report(RD->getLocation(),
                                     diag::warn_padded_struct_size)
-        << Context.getTypeDeclType(RD) << PadSize
+        << Context.getCanonicalTagType(RD) << PadSize
         << (InBits ? 1 : 0); // (byte|bit)
   }
 }
@@ -3631,7 +3630,7 @@ static void DumpRecordLayout(raw_ostream &OS, const RecordDecl *RD,
   auto CXXRD = dyn_cast<CXXRecordDecl>(RD);
 
   PrintOffset(OS, Offset, IndentLevel);
-  OS << C.getTypeDeclType(const_cast<RecordDecl *>(RD));
+  OS << C.getCanonicalTagType(const_cast<RecordDecl *>(RD));
   if (Description)
     OS << ' ' << Description;
   if (CXXRD && CXXRD->isEmpty())
@@ -3697,8 +3696,8 @@ static void DumpRecordLayout(raw_ostream &OS, const RecordDecl *RD,
 
     // Recursively dump fields of record type.
     if (auto RT = Field->getType()->getAs<RecordType>()) {
-      DumpRecordLayout(OS, RT->getDecl(), C, FieldOffset, IndentLevel,
-                       Field->getName().data(),
+      DumpRecordLayout(OS, RT->getOriginalDecl()->getDefinitionOrSelf(), C,
+                       FieldOffset, IndentLevel, Field->getName().data(),
                        /*PrintSizeInfo=*/false,
                        /*IncludeVirtualBases=*/true);
       continue;
@@ -3781,7 +3780,7 @@ void ASTContext::DumpRecordLayout(const RecordDecl *RD, raw_ostream &OS,
   // in libFrontend.
 
   const ASTRecordLayout &Info = getASTRecordLayout(RD);
-  OS << "Type: " << getTypeDeclType(RD) << "\n";
+  OS << "Type: " << getCanonicalTagType(RD) << "\n";
   OS << "\nLayout: ";
   OS << "<ASTRecordLayout\n";
   OS << "  Size:" << toBits(Info.getSize()) << "\n";
diff --git a/clang/lib/AST/ScanfFormatString.cpp b/clang/lib/AST/ScanfFormatString.cpp
index 1227edd47d13d..31c001d025fea 100644
--- a/clang/lib/AST/ScanfFormatString.cpp
+++ b/clang/lib/AST/ScanfFormatString.cpp
@@ -432,9 +432,10 @@ bool ScanfSpecifier::fixType(QualType QT, QualType RawQT,
   // If it's an enum, get its underlying type.
   if (const EnumType *ETy = PT->getAs<EnumType>()) {
     // Don't try to fix incomplete enums.
-    if (!ETy->getDecl()->isComplete())
+    const EnumDecl *ED = ETy->getOriginalDecl()->getDefinitionOrSelf();
+    if (!ED->isComplete())
       return false;
-    PT = ETy->getDecl()->getIntegerType();
+    PT = ED->getIntegerType();
   }
 
   const BuiltinType *BT = PT->getAs<BuiltinType>();
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 6ba5ec89964a9..afccba8778fd2 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -454,10 +454,7 @@ void StmtPrinter::VisitMSDependentExistsStmt(MSDependentExistsStmt *Node) {
   else
     OS << "__if_not_exists (";
 
-  if (NestedNameSpecifier *Qualifier
-        = Node->getQualifierLoc().getNestedNameSpecifier())
-    Qualifier->print(OS, Policy);
-
+  Node->getQualifierLoc().getNestedNameSpecifier().print(OS, Policy);
   OS << Node->getNameInfo() << ") ";
 
   PrintRawCompoundStmt(Node->getSubStmt());
@@ -1309,8 +1306,7 @@ void StmtPrinter::VisitDeclRefExpr(DeclRefExpr *Node) {
     TPOD->printAsExpr(OS, Policy);
     return;
   }
-  if (NestedNameSpecifier *Qualifier = Node->getQualifier())
-    Qualifier->print(OS, Policy);
+  Node->getQualifier().print(OS, Policy);
   if (Node->hasTemplateKeyword())
     OS << "template ";
 
@@ -1359,8 +1355,7 @@ void StmtPrinter::VisitDeclRefExpr(DeclRefExpr *Node) {
 
 void StmtPrinter::VisitDependentScopeDeclRefExpr(
                                            DependentScopeDeclRefExpr *Node) {
-  if (NestedNameSpecifier *Qualifier = Node->getQualifier())
-    Qualifier->print(OS, Policy);
+  Node->getQualifier().print(OS, Policy);
   if (Node->hasTemplateKeyword())
     OS << "template ";
   OS << Node->getNameInfo();
@@ -1369,8 +1364,7 @@ void StmtPrinter::VisitDependentScopeDeclRefExpr(
 }
 
 void StmtPrinter::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *Node) {
-  if (Node->getQualifier())
-    Node->getQualifier()->print(OS, Policy);
+  Node->getQualifier().print(OS, Policy);
   if (Node->hasTemplateKeyword())
     OS << "template ";
   OS << Node->getNameInfo();
@@ -1778,8 +1772,7 @@ void StmtPrinter::VisitMemberExpr(MemberExpr *Node) {
     if (FD->isAnonymousStructOrUnion())
       return;
 
-  if (NestedNameSpecifier *Qualifier = Node->getQualifier())
-    Qualifier->print(OS, Policy);
+  Node->getQualifier().print(OS, Policy);
   if (Node->hasTemplateKeyword())
     OS << "template ";
   OS << Node->getMemberNameInfo();
@@ -2177,9 +2170,7 @@ void StmtPrinter::VisitMSPropertyRefExpr(MSPropertyRefExpr *Node) {
     OS << "->";
   else
     OS << ".";
-  if (NestedNameSpecifier *Qualifier =
-      Node->getQualifierLoc().getNestedNameSpecifier())
-    Qualifier->print(OS, Policy);
+  Node->getQualifierLoc().getNestedNameSpecifier().print(OS, Policy);
   OS << Node->getPropertyDecl()->getDeclName();
 }
 
@@ -2509,8 +2500,7 @@ void StmtPrinter::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
     OS << "->";
   else
     OS << '.';
-  if (E->getQualifier())
-    E->getQualifier()->print(OS, Policy);
+  E->getQualifier().print(OS, Policy);
   OS << "~";
 
   if (const IdentifierInfo *II = E->getDestroyedTypeIdentifier())
@@ -2572,8 +2562,7 @@ void StmtPrinter::VisitCXXDependentScopeMemberExpr(
     PrintExpr(Node->getBase());
     OS << (Node->isArrow() ? "->" : ".");
   }
-  if (NestedNameSpecifier *Qualifier = Node->getQualifier())
-    Qualifier->print(OS, Policy);
+  Node->getQualifier().print(OS, Policy);
   if (Node->hasTemplateKeyword())
     OS << "template ";
   OS << Node->getMemberNameInfo();
@@ -2586,8 +2575,7 @@ void StmtPrinter::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *Node) {
     PrintExpr(Node->getBase());
     OS << (Node->isArrow() ? "->" : ".");
   }
-  if (NestedNameSpecifier *Qualifier = Node->getQualifier())
-    Qualifier->print(OS, Policy);
+  Node->getQualifier().print(OS, Policy);
   if (Node->hasTemplateKeyword())
     OS << "template ";
   OS << Node->getMemberNameInfo();
@@ -2678,8 +2666,7 @@ void StmtPrinter::VisitCXXParenListInitExpr(CXXParenListInitExpr *Node) {
 
 void StmtPrinter::VisitConceptSpecializationExpr(ConceptSpecializationExpr *E) {
   NestedNameSpecifierLoc NNS = E->getNestedNameSpecifierLoc();
-  if (NNS)
-    NNS.getNestedNameSpecifier()->print(OS, Policy);
+  NNS.getNestedNameSpecifier().print(OS, Policy);
   if (E->getTemplateKWLoc().isValid())
     OS << "template ";
   OS << E->getFoundDecl()->getName();
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 4c36f2473ce25..2035fa7635f2a 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -65,7 +65,7 @@ namespace {
 
     /// Visit a nested-name-specifier that occurs within an expression
     /// or statement.
-    virtual void VisitNestedNameSpecifier(NestedNameSpecifier *NNS) = 0;
+    virtual void VisitNestedNameSpecifier(NestedNameSpecifier NNS) = 0;
 
     /// Visit a template name that occurs within an expression or
     /// statement.
@@ -167,10 +167,10 @@ namespace {
       ID.AddPointer(II);
     }
 
-    void VisitNestedNameSpecifier(NestedNameSpecifier *NNS) override {
+    void VisitNestedNameSpecifier(NestedNameSpecifier NNS) override {
       if (Canonical)
-        NNS = Context.getCanonicalNestedNameSpecifier(NNS);
-      ID.AddPointer(NNS);
+        NNS = NNS.getCanonical();
+      NNS.Profile(ID);
     }
 
     void VisitTemplateName(TemplateName Name) override {
@@ -226,11 +226,10 @@ namespace {
     void VisitTemplateName(TemplateName Name) override {
       Hash.AddTemplateName(Name);
     }
-    void VisitNestedNameSpecifier(NestedNameSpecifier *NNS) override {
-      ID.AddBoolean(NNS);
-      if (NNS) {
+    void VisitNestedNameSpecifier(NestedNameSpecifier NNS) override {
+      ID.AddBoolean(bool(NNS));
+      if (NNS)
         Hash.AddNestedNameSpecifier(NNS);
-      }
     }
   };
 }
@@ -441,37 +440,37 @@ class OMPClauseProfiler : public ConstOMPClauseVisitor<OMPClauseProfiler> {
 #define GEN_CLANG_CLAUSE_CLASS
 #define CLAUSE_CLASS(Enum, Str, Class) void Visit##Class(const Class *C);
 #include "llvm/Frontend/OpenMP/OMP.inc"
-  void VistOMPClauseWithPreInit(const OMPClauseWithPreInit *C);
-  void VistOMPClauseWithPostUpdate(const OMPClauseWithPostUpdate *C);
+  void VisitOMPClauseWithPreInit(const OMPClauseWithPreInit *C);
+  void VisitOMPClauseWithPostUpdate(const OMPClauseWithPostUpdate *C);
 };
 
-void OMPClauseProfiler::VistOMPClauseWithPreInit(
+void OMPClauseProfiler::VisitOMPClauseWithPreInit(
     const OMPClauseWithPreInit *C) {
   if (auto *S = C->getPreInitStmt())
     Profiler->VisitStmt(S);
 }
 
-void OMPClauseProfiler::VistOMPClauseWithPostUpdate(
+void OMPClauseProfiler::VisitOMPClauseWithPostUpdate(
     const OMPClauseWithPostUpdate *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (auto *E = C->getPostUpdateExpr())
     Profiler->VisitStmt(E);
 }
 
 void OMPClauseProfiler::VisitOMPIfClause(const OMPIfClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (C->getCondition())
     Profiler->VisitStmt(C->getCondition());
 }
 
 void OMPClauseProfiler::VisitOMPFinalClause(const OMPFinalClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (C->getCondition())
     Profiler->VisitStmt(C->getCondition());
 }
 
 void OMPClauseProfiler::VisitOMPNumThreadsClause(const OMPNumThreadsClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (C->getNumThreads())
     Profiler->VisitStmt(C->getNumThreads());
 }
@@ -527,13 +526,13 @@ void OMPClauseProfiler::VisitOMPDetachClause(const OMPDetachClause *C) {
 }
 
 void OMPClauseProfiler::VisitOMPNovariantsClause(const OMPNovariantsClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (C->getCondition())
     Profiler->VisitStmt(C->getCondition());
 }
 
 void OMPClauseProfiler::VisitOMPNocontextClause(const OMPNocontextClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (C->getCondition())
     Profiler->VisitStmt(C->getCondition());
 }
@@ -569,7 +568,7 @@ void OMPClauseProfiler::VisitOMPMessageClause(const OMPMessageClause *C) {
 }
 
 void OMPClauseProfiler::VisitOMPScheduleClause(const OMPScheduleClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (auto *S = C->getChunkSize())
     Profiler->VisitStmt(S);
 }
@@ -647,7 +646,7 @@ void OMPClauseProfiler::VisitOMPDestroyClause(const OMPDestroyClause *C) {
 }
 
 void OMPClauseProfiler::VisitOMPFilterClause(const OMPFilterClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (C->getThreadID())
     Profiler->VisitStmt(C->getThreadID());
 }
@@ -670,7 +669,7 @@ void OMPClauseProfiler::VisitOMPPrivateClause(const OMPPrivateClause *C) {
 void
 OMPClauseProfiler::VisitOMPFirstprivateClause(const OMPFirstprivateClause *C) {
   VisitOMPClauseList(C);
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   for (auto *E : C->private_copies()) {
     if (E)
       Profiler->VisitStmt(E);
@@ -683,7 +682,7 @@ OMPClauseProfiler::VisitOMPFirstprivateClause(const OMPFirstprivateClause *C) {
 void
 OMPClauseProfiler::VisitOMPLastprivateClause(const OMPLastprivateClause *C) {
   VisitOMPClauseList(C);
-  VistOMPClauseWithPostUpdate(C);
+  VisitOMPClauseWithPostUpdate(C);
   for (auto *E : C->source_exprs()) {
     if (E)
       Profiler->VisitStmt(E);
@@ -706,7 +705,7 @@ void OMPClauseProfiler::VisitOMPReductionClause(
       C->getQualifierLoc().getNestedNameSpecifier());
   Profiler->VisitName(C->getNameInfo().getName());
   VisitOMPClauseList(C);
-  VistOMPClauseWithPostUpdate(C);
+  VisitOMPClauseWithPostUpdate(C);
   for (auto *E : C->privates()) {
     if (E)
       Profiler->VisitStmt(E);
@@ -744,7 +743,7 @@ void OMPClauseProfiler::VisitOMPTaskReductionClause(
       C->getQualifierLoc().getNestedNameSpecifier());
   Profiler->VisitName(C->getNameInfo().getName());
   VisitOMPClauseList(C);
-  VistOMPClauseWithPostUpdate(C);
+  VisitOMPClauseWithPostUpdate(C);
   for (auto *E : C->privates()) {
     if (E)
       Profiler->VisitStmt(E);
@@ -768,7 +767,7 @@ void OMPClauseProfiler::VisitOMPInReductionClause(
       C->getQualifierLoc().getNestedNameSpecifier());
   Profiler->VisitName(C->getNameInfo().getName());
   VisitOMPClauseList(C);
-  VistOMPClauseWithPostUpdate(C);
+  VisitOMPClauseWithPostUpdate(C);
   for (auto *E : C->privates()) {
     if (E)
       Profiler->VisitStmt(E);
@@ -792,7 +791,7 @@ void OMPClauseProfiler::VisitOMPInReductionClause(
 }
 void OMPClauseProfiler::VisitOMPLinearClause(const OMPLinearClause *C) {
   VisitOMPClauseList(C);
-  VistOMPClauseWithPostUpdate(C);
+  VisitOMPClauseWithPostUpdate(C);
   for (auto *E : C->privates()) {
     if (E)
       Profiler->VisitStmt(E);
@@ -874,25 +873,25 @@ void OMPClauseProfiler::VisitOMPAllocateClause(const OMPAllocateClause *C) {
 }
 void OMPClauseProfiler::VisitOMPNumTeamsClause(const OMPNumTeamsClause *C) {
   VisitOMPClauseList(C);
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
 }
 void OMPClauseProfiler::VisitOMPThreadLimitClause(
     const OMPThreadLimitClause *C) {
   VisitOMPClauseList(C);
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
 }
 void OMPClauseProfiler::VisitOMPPriorityClause(const OMPPriorityClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (C->getPriority())
     Profiler->VisitStmt(C->getPriority());
 }
 void OMPClauseProfiler::VisitOMPGrainsizeClause(const OMPGrainsizeClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (C->getGrainsize())
     Profiler->VisitStmt(C->getGrainsize());
 }
 void OMPClauseProfiler::VisitOMPNumTasksClause(const OMPNumTasksClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (C->getNumTasks())
     Profiler->VisitStmt(C->getNumTasks());
 }
@@ -953,7 +952,7 @@ void OMPClauseProfiler::VisitOMPOrderClause(const OMPOrderClause *C) {}
 void OMPClauseProfiler::VisitOMPBindClause(const OMPBindClause *C) {}
 void OMPClauseProfiler::VisitOMPXDynCGroupMemClause(
     const OMPXDynCGroupMemClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (Expr *Size = C->getSize())
     Profiler->VisitStmt(Size);
 }
@@ -1230,7 +1229,7 @@ void StmtProfiler::VisitOMPDistributeDirective(
 
 void OMPClauseProfiler::VisitOMPDistScheduleClause(
     const OMPDistScheduleClause *C) {
-  VistOMPClauseWithPreInit(C);
+  VisitOMPClauseWithPreInit(C);
   if (auto *S = C->getChunkSize())
     Profiler->VisitStmt(S);
 }
@@ -2645,8 +2644,10 @@ void OpenACCClauseProfiler::VisitFirstPrivateClause(
     const OpenACCFirstPrivateClause &Clause) {
   VisitClauseWithVarList(Clause);
 
-  for (auto *VD : Clause.getInitRecipes())
-    Profiler.VisitDecl(VD);
+  for (auto &Recipe : Clause.getInitRecipes()) {
+    Profiler.VisitDecl(Recipe.RecipeDecl);
+    Profiler.VisitDecl(Recipe.InitFromTemporary);
+  }
 }
 
 void OpenACCClauseProfiler::VisitAttachClause(
@@ -2747,6 +2748,14 @@ void OpenACCClauseProfiler::VisitGangClause(const OpenACCGangClause &Clause) {
 void OpenACCClauseProfiler::VisitReductionClause(
     const OpenACCReductionClause &Clause) {
   VisitClauseWithVarList(Clause);
+
+  for (auto &Recipe : Clause.getRecipes()) {
+    Profiler.VisitDecl(Recipe.RecipeDecl);
+    // TODO: OpenACC: Make sure we remember to update this when we figure out
+    // what we're adding for the operation recipe, in the meantime, a static
+    // assert will make sure we don't add something.
+    static_assert(sizeof(OpenACCReductionRecipe) == sizeof(int *));
+  }
 }
 
 void OpenACCClauseProfiler::VisitBindClause(const OpenACCBindClause &Clause) {
diff --git a/clang/lib/AST/TemplateBase.cpp b/clang/lib/AST/TemplateBase.cpp
index 7a0f74051d880..76050ceeb35a7 100644
--- a/clang/lib/AST/TemplateBase.cpp
+++ b/clang/lib/AST/TemplateBase.cpp
@@ -57,7 +57,7 @@ static void printIntegral(const TemplateArgument &TemplArg, raw_ostream &Out,
 
   if (Policy.UseEnumerators) {
     if (const EnumType *ET = T->getAs<EnumType>()) {
-      for (const EnumConstantDecl *ECD : ET->getDecl()->enumerators()) {
+      for (const EnumConstantDecl *ECD : ET->getOriginalDecl()->enumerators()) {
         // In Sema::CheckTemplateArugment, enum template arguments value are
         // extended to the size of the integer underlying the enum type.  This
         // may create a size difference between the enum value and template
@@ -596,6 +596,29 @@ void TemplateArgument::print(const PrintingPolicy &Policy, raw_ostream &Out,
 // TemplateArgumentLoc Implementation
 //===----------------------------------------------------------------------===//
 
+TemplateArgumentLoc::TemplateArgumentLoc(ASTContext &Ctx,
+                                         const TemplateArgument &Argument,
+                                         SourceLocation TemplateKWLoc,
+                                         NestedNameSpecifierLoc QualifierLoc,
+                                         SourceLocation TemplateNameLoc,
+                                         SourceLocation EllipsisLoc)
+    : Argument(Argument),
+      LocInfo(Ctx, TemplateKWLoc, QualifierLoc, TemplateNameLoc, EllipsisLoc) {
+  assert(Argument.getKind() == TemplateArgument::Template ||
+         Argument.getKind() == TemplateArgument::TemplateExpansion);
+  assert(QualifierLoc.getNestedNameSpecifier() ==
+         Argument.getAsTemplateOrTemplatePattern().getQualifier());
+}
+
+NestedNameSpecifierLoc TemplateArgumentLoc::getTemplateQualifierLoc() const {
+  if (Argument.getKind() != TemplateArgument::Template &&
+      Argument.getKind() != TemplateArgument::TemplateExpansion)
+    return NestedNameSpecifierLoc();
+  return NestedNameSpecifierLoc(
+      Argument.getAsTemplateOrTemplatePattern().getQualifier(),
+      LocInfo.getTemplate()->QualifierLocData);
+}
+
 SourceRange TemplateArgumentLoc::getSourceRange() const {
   switch (Argument.getKind()) {
   case TemplateArgument::Expression:
@@ -702,10 +725,11 @@ const StreamingDiagnostic &clang::operator<<(const StreamingDiagnostic &DB,
 }
 
 clang::TemplateArgumentLocInfo::TemplateArgumentLocInfo(
-    ASTContext &Ctx, NestedNameSpecifierLoc QualifierLoc,
-    SourceLocation TemplateNameLoc, SourceLocation EllipsisLoc) {
+    ASTContext &Ctx, SourceLocation TemplateKWLoc,
+    NestedNameSpecifierLoc QualifierLoc, SourceLocation TemplateNameLoc,
+    SourceLocation EllipsisLoc) {
   TemplateTemplateArgLocInfo *Template = new (Ctx) TemplateTemplateArgLocInfo;
-  Template->Qualifier = QualifierLoc.getNestedNameSpecifier();
+  Template->TemplateKwLoc = TemplateKWLoc;
   Template->QualifierLocData = QualifierLoc.getOpaqueData();
   Template->TemplateNameLoc = TemplateNameLoc;
   Template->EllipsisLoc = EllipsisLoc;
diff --git a/clang/lib/AST/TemplateName.cpp b/clang/lib/AST/TemplateName.cpp
index 5b7abc4d038a9..c171516c38c10 100644
--- a/clang/lib/AST/TemplateName.cpp
+++ b/clang/lib/AST/TemplateName.cpp
@@ -289,10 +289,30 @@ QualifiedTemplateName *TemplateName::getAsQualifiedTemplateName() const {
   return dyn_cast_if_present<QualifiedTemplateName *>(Storage);
 }
 
+QualifiedTemplateName *
+TemplateName::getAsAdjustedQualifiedTemplateName() const {
+  for (std::optional<TemplateName> Cur = *this; Cur;
+       Cur = Cur->desugar(/*IgnoreDeduced=*/true))
+    if (QualifiedTemplateName *N = Cur->getAsQualifiedTemplateName())
+      return N;
+  return nullptr;
+}
+
 DependentTemplateName *TemplateName::getAsDependentTemplateName() const {
   return Storage.dyn_cast<DependentTemplateName *>();
 }
 
+NestedNameSpecifier TemplateName::getQualifier() const {
+  for (std::optional<TemplateName> Cur = *this; Cur;
+       Cur = Cur->desugar(/*IgnoreDeduced=*/true)) {
+    if (DependentTemplateName *N = Cur->getAsDependentTemplateName())
+      return N->getQualifier();
+    if (QualifiedTemplateName *N = Cur->getAsQualifiedTemplateName())
+      return N->getQualifier();
+  }
+  return std::nullopt;
+}
+
 UsingShadowDecl *TemplateName::getAsUsingShadowDecl() const {
   if (Decl *D = Storage.dyn_cast<Decl *>())
     if (UsingShadowDecl *USD = dyn_cast<UsingShadowDecl>(D))
@@ -303,24 +323,21 @@ UsingShadowDecl *TemplateName::getAsUsingShadowDecl() const {
 }
 
 DependentTemplateStorage::DependentTemplateStorage(
-    NestedNameSpecifier *Qualifier, IdentifierOrOverloadedOperator Name,
+    NestedNameSpecifier Qualifier, IdentifierOrOverloadedOperator Name,
     bool HasTemplateKeyword)
     : Qualifier(Qualifier, HasTemplateKeyword), Name(Name) {
-  assert((!Qualifier || Qualifier->isDependent()) &&
+  assert((!Qualifier || Qualifier.isDependent()) &&
          "Qualifier must be dependent");
 }
 
 TemplateNameDependence DependentTemplateStorage::getDependence() const {
-  auto D = TemplateNameDependence::DependentInstantiation;
-  if (NestedNameSpecifier *Qualifier = getQualifier())
-    D |= toTemplateNameDependence(Qualifier->getDependence());
-  return D;
+  return toTemplateNameDependence(getQualifier().getDependence()) |
+         TemplateNameDependence::DependentInstantiation;
 }
 
 void DependentTemplateStorage::print(raw_ostream &OS,
                                      const PrintingPolicy &Policy) const {
-  if (NestedNameSpecifier *NNS = getQualifier())
-    NNS->print(OS, Policy);
+  getQualifier().print(OS, Policy);
 
   if (hasTemplateKeyword())
     OS << "template ";
@@ -363,16 +380,13 @@ TemplateNameDependence TemplateName::getDependence() const {
   case NameKind::QualifiedTemplate: {
     QualifiedTemplateName *S = getAsQualifiedTemplateName();
     TemplateNameDependence D = S->getUnderlyingTemplate().getDependence();
-    if (NestedNameSpecifier *NNS = S->getQualifier())
-      D |= toTemplateNameDependence(NNS->getDependence());
+    D |= toTemplateNameDependence(S->getQualifier().getDependence());
     return D;
   }
   case NameKind::DependentTemplate: {
     DependentTemplateName *S = getAsDependentTemplateName();
-    auto D = TemplateNameDependence::DependentInstantiation;
-    if (NestedNameSpecifier *Qualifier = S->getQualifier())
-      D |= toTemplateNameDependence(Qualifier->getDependence());
-    return D;
+    return toTemplateNameDependence(S->getQualifier().getDependence()) |
+           TemplateNameDependence::DependentInstantiation;
   }
   case NameKind::SubstTemplateTemplateParm: {
     auto *S = getAsSubstTemplateTemplateParm();
@@ -434,18 +448,20 @@ void TemplateName::print(raw_ostream &OS, const PrintingPolicy &Policy,
       Template = cast<TemplateDecl>(Template->getCanonicalDecl());
     if (handleAnonymousTTP(Template, OS))
       return;
-    if (Qual == Qualified::None)
+    if (Qual == Qualified::None || Policy.SuppressScope) {
       OS << *Template;
-    else
-      Template->printQualifiedName(OS, Policy);
+    } else {
+      PrintingPolicy NestedNamePolicy = Policy;
+      NestedNamePolicy.SuppressUnwrittenScope = true;
+      Template->printQualifiedName(OS, NestedNamePolicy);
+    }
   } else if (QualifiedTemplateName *QTN = getAsQualifiedTemplateName()) {
     if (Policy.PrintAsCanonical) {
       QTN->getUnderlyingTemplate().print(OS, Policy, Qual);
       return;
     }
-    if (NestedNameSpecifier *NNS = QTN->getQualifier();
-        Qual != Qualified::None && NNS)
-      NNS->print(OS, Policy);
+    if (Qual != Qualified::None)
+      QTN->getQualifier().print(OS, Policy);
     if (QTN->hasTemplateKeyword())
       OS << "template ";
 
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 6b524cfcd2d71..085616049373e 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -1037,35 +1037,34 @@ void clang::TextNodeDumper::dumpTemplateSpecializationKind(
   }
 }
 
-void clang::TextNodeDumper::dumpNestedNameSpecifier(const NestedNameSpecifier *NNS) {
+void clang::TextNodeDumper::dumpNestedNameSpecifier(NestedNameSpecifier NNS) {
   if (!NNS)
     return;
 
   AddChild([=] {
     OS << "NestedNameSpecifier";
 
-    switch (NNS->getKind()) {
-    case NestedNameSpecifier::Identifier:
-      OS << " Identifier";
-      OS << " '" << NNS->getAsIdentifier()->getName() << "'";
-      break;
-    case NestedNameSpecifier::Namespace:
+    switch (NNS.getKind()) {
+    case NestedNameSpecifier::Kind::Namespace: {
+      auto [Namespace, Prefix] = NNS.getAsNamespaceAndPrefix();
       OS << " "; // "Namespace" is printed as the decl kind.
-      dumpBareDeclRef(NNS->getAsNamespace());
+      dumpBareDeclRef(Namespace);
+      dumpNestedNameSpecifier(Prefix);
       break;
-    case NestedNameSpecifier::TypeSpec:
+    }
+    case NestedNameSpecifier::Kind::Type:
       OS << " TypeSpec";
-      dumpType(QualType(NNS->getAsType(), 0));
+      dumpType(QualType(NNS.getAsType(), 0));
       break;
-    case NestedNameSpecifier::Global:
+    case NestedNameSpecifier::Kind::Global:
       OS << " Global";
       break;
-    case NestedNameSpecifier::Super:
+    case NestedNameSpecifier::Kind::MicrosoftSuper:
       OS << " Super";
       break;
+    case NestedNameSpecifier::Kind::Null:
+      llvm_unreachable("unexpected null nested name specifier");
     }
-
-    dumpNestedNameSpecifier(NNS->getPrefix());
   });
 }
 
@@ -1401,8 +1400,8 @@ static void dumpBasePath(raw_ostream &OS, const CastExpr *Node) {
     if (!First)
       OS << " -> ";
 
-    const auto *RD =
-        cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
+    const auto *RD = cast<CXXRecordDecl>(
+        Base->getType()->castAs<RecordType>()->getOriginalDecl());
 
     if (Base->isVirtual())
       OS << "virtual ";
@@ -2112,19 +2111,32 @@ void TextNodeDumper::VisitFunctionProtoType(const FunctionProtoType *T) {
 }
 
 void TextNodeDumper::VisitUnresolvedUsingType(const UnresolvedUsingType *T) {
+  if (ElaboratedTypeKeyword K = T->getKeyword();
+      K != ElaboratedTypeKeyword::None)
+    OS << ' ' << TypeWithKeyword::getKeywordName(K);
+  dumpNestedNameSpecifier(T->getQualifier());
   dumpDeclRef(T->getDecl());
 }
 
 void TextNodeDumper::VisitUsingType(const UsingType *T) {
-  dumpDeclRef(T->getFoundDecl());
-  if (!T->typeMatchesDecl())
-    OS << " divergent";
+  if (ElaboratedTypeKeyword K = T->getKeyword();
+      K != ElaboratedTypeKeyword::None)
+    OS << ' ' << TypeWithKeyword::getKeywordName(K);
+  dumpNestedNameSpecifier(T->getQualifier());
+  dumpDeclRef(T->getDecl());
+  dumpType(T->desugar());
 }
 
 void TextNodeDumper::VisitTypedefType(const TypedefType *T) {
+  if (ElaboratedTypeKeyword K = T->getKeyword();
+      K != ElaboratedTypeKeyword::None)
+    OS << ' ' << TypeWithKeyword::getKeywordName(K);
+  dumpNestedNameSpecifier(T->getQualifier());
   dumpDeclRef(T->getDecl());
-  if (!T->typeMatchesDecl())
+  if (!T->typeMatchesDecl()) {
     OS << " divergent";
+    dumpType(T->desugar());
+  }
 }
 
 void TextNodeDumper::VisitUnaryTransformType(const UnaryTransformType *T) {
@@ -2138,7 +2150,17 @@ void TextNodeDumper::VisitUnaryTransformType(const UnaryTransformType *T) {
 }
 
 void TextNodeDumper::VisitTagType(const TagType *T) {
-  dumpDeclRef(T->getDecl());
+  if (T->isCanonicalUnqualified())
+    OS << " canonical";
+  if (T->isTagOwned())
+    OS << " owns_tag";
+  if (T->isInjected())
+    OS << " injected";
+  if (ElaboratedTypeKeyword K = T->getKeyword();
+      K != ElaboratedTypeKeyword::None)
+    OS << ' ' << TypeWithKeyword::getKeywordName(K);
+  dumpNestedNameSpecifier(T->getQualifier());
+  dumpDeclRef(T->getOriginalDecl());
 }
 
 void TextNodeDumper::VisitTemplateTypeParmType(const TemplateTypeParmType *T) {
@@ -2182,12 +2204,15 @@ void TextNodeDumper::VisitTemplateSpecializationType(
     const TemplateSpecializationType *T) {
   if (T->isTypeAlias())
     OS << " alias";
+  if (ElaboratedTypeKeyword K = T->getKeyword();
+      K != ElaboratedTypeKeyword::None)
+    OS << ' ' << TypeWithKeyword::getKeywordName(K);
   dumpTemplateName(T->getTemplateName(), "name");
 }
 
 void TextNodeDumper::VisitInjectedClassNameType(
     const InjectedClassNameType *T) {
-  dumpDeclRef(T->getDecl());
+  dumpDeclRef(T->getOriginalDecl());
 }
 
 void TextNodeDumper::VisitObjCInterfaceType(const ObjCInterfaceType *T) {
@@ -2778,8 +2803,7 @@ void TextNodeDumper::VisitTemplateTemplateParmDecl(
 
 void TextNodeDumper::VisitUsingDecl(const UsingDecl *D) {
   OS << ' ';
-  if (D->getQualifier())
-    D->getQualifier()->print(OS, D->getASTContext().getPrintingPolicy());
+  D->getQualifier().print(OS, D->getASTContext().getPrintingPolicy());
   OS << D->getDeclName();
   dumpNestedNameSpecifier(D->getQualifier());
 }
@@ -2792,16 +2816,14 @@ void TextNodeDumper::VisitUsingEnumDecl(const UsingEnumDecl *D) {
 void TextNodeDumper::VisitUnresolvedUsingTypenameDecl(
     const UnresolvedUsingTypenameDecl *D) {
   OS << ' ';
-  if (D->getQualifier())
-    D->getQualifier()->print(OS, D->getASTContext().getPrintingPolicy());
+  D->getQualifier().print(OS, D->getASTContext().getPrintingPolicy());
   OS << D->getDeclName();
 }
 
 void TextNodeDumper::VisitUnresolvedUsingValueDecl(
     const UnresolvedUsingValueDecl *D) {
   OS << ' ';
-  if (D->getQualifier())
-    D->getQualifier()->print(OS, D->getASTContext().getPrintingPolicy());
+  D->getQualifier().print(OS, D->getASTContext().getPrintingPolicy());
   OS << D->getDeclName();
   dumpType(D->getType());
 }
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 141edc881d683..a70bc5424009c 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -109,12 +109,14 @@ bool Qualifiers::isTargetAddressSpaceSupersetOf(LangAS A, LangAS B,
 const IdentifierInfo *QualType::getBaseTypeIdentifier() const {
   const Type *ty = getTypePtr();
   NamedDecl *ND = nullptr;
+  if (const auto *DNT = ty->getAs<DependentNameType>())
+    return DNT->getIdentifier();
   if (ty->isPointerOrReferenceType())
     return ty->getPointeeType().getBaseTypeIdentifier();
-  else if (ty->isRecordType())
-    ND = ty->castAs<RecordType>()->getDecl();
+  if (ty->isRecordType())
+    ND = ty->castAs<RecordType>()->getOriginalDecl();
   else if (ty->isEnumeralType())
-    ND = ty->castAs<EnumType>()->getDecl();
+    ND = ty->castAs<EnumType>()->getOriginalDecl();
   else if (ty->getTypeClass() == Type::Typedef)
     ND = ty->castAs<TypedefType>()->getDecl();
   else if (ty->isArrayType())
@@ -671,13 +673,13 @@ const Type *Type::getUnqualifiedDesugaredType() const {
 
 bool Type::isClassType() const {
   if (const auto *RT = getAs<RecordType>())
-    return RT->getDecl()->isClass();
+    return RT->getOriginalDecl()->isClass();
   return false;
 }
 
 bool Type::isStructureType() const {
   if (const auto *RT = getAs<RecordType>())
-    return RT->getDecl()->isStruct();
+    return RT->getOriginalDecl()->isStruct();
   return false;
 }
 
@@ -685,7 +687,7 @@ bool Type::isStructureTypeWithFlexibleArrayMember() const {
   const auto *RT = getAs<RecordType>();
   if (!RT)
     return false;
-  const auto *Decl = RT->getDecl();
+  const auto *Decl = RT->getOriginalDecl()->getDefinitionOrSelf();
   if (!Decl->isStruct())
     return false;
   return Decl->hasFlexibleArrayMember();
@@ -693,19 +695,21 @@ bool Type::isStructureTypeWithFlexibleArrayMember() const {
 
 bool Type::isObjCBoxableRecordType() const {
   if (const auto *RT = getAs<RecordType>())
-    return RT->getDecl()->hasAttr<ObjCBoxableAttr>();
+    return RT->getOriginalDecl()
+        ->getDefinitionOrSelf()
+        ->hasAttr<ObjCBoxableAttr>();
   return false;
 }
 
 bool Type::isInterfaceType() const {
   if (const auto *RT = getAs<RecordType>())
-    return RT->getDecl()->isInterface();
+    return RT->getOriginalDecl()->isInterface();
   return false;
 }
 
 bool Type::isStructureOrClassType() const {
   if (const auto *RT = getAs<RecordType>()) {
-    RecordDecl *RD = RT->getDecl();
+    RecordDecl *RD = RT->getOriginalDecl();
     return RD->isStruct() || RD->isClass() || RD->isInterface();
   }
   return false;
@@ -719,7 +723,7 @@ bool Type::isVoidPointerType() const {
 
 bool Type::isUnionType() const {
   if (const auto *RT = getAs<RecordType>())
-    return RT->getDecl()->isUnion();
+    return RT->getOriginalDecl()->isUnion();
   return false;
 }
 
@@ -736,7 +740,7 @@ bool Type::isComplexIntegerType() const {
 
 bool Type::isScopedEnumeralType() const {
   if (const auto *ET = getAs<EnumType>())
-    return ET->getDecl()->isScoped();
+    return ET->getOriginalDecl()->isScoped();
   return false;
 }
 
@@ -770,13 +774,13 @@ QualType Type::getPointeeType() const {
 const RecordType *Type::getAsStructureType() const {
   // If this is directly a structure type, return it.
   if (const auto *RT = dyn_cast<RecordType>(this)) {
-    if (RT->getDecl()->isStruct())
+    if (RT->getOriginalDecl()->isStruct())
       return RT;
   }
 
   // If the canonical form of this type isn't the right kind, reject it.
   if (const auto *RT = dyn_cast<RecordType>(CanonicalType)) {
-    if (!RT->getDecl()->isStruct())
+    if (!RT->getOriginalDecl()->isStruct())
       return nullptr;
 
     // If this is a typedef for a structure type, strip the typedef off without
@@ -789,13 +793,13 @@ const RecordType *Type::getAsStructureType() const {
 const RecordType *Type::getAsUnionType() const {
   // If this is directly a union type, return it.
   if (const auto *RT = dyn_cast<RecordType>(this)) {
-    if (RT->getDecl()->isUnion())
+    if (RT->getOriginalDecl()->isUnion())
       return RT;
   }
 
   // If the canonical form of this type isn't the right kind, reject it.
   if (const auto *RT = dyn_cast<RecordType>(CanonicalType)) {
-    if (!RT->getDecl()->isUnion())
+    if (!RT->getOriginalDecl()->isUnion())
       return nullptr;
 
     // If this is a typedef for a union type, strip the typedef off without
@@ -1272,9 +1276,6 @@ struct SimpleTransformVisitor : public TypeVisitor<Derived, QualType> {
   TRIVIAL_TYPE_CLASS(Record)
   TRIVIAL_TYPE_CLASS(Enum)
 
-  // FIXME: Non-trivial to implement, but important for C++
-  SUGARED_TYPE_CLASS(Elaborated)
-
   QualType VisitAttributedType(const AttributedType *T) {
     QualType modifiedType = recurse(T->getModifiedType());
     if (modifiedType.isNull())
@@ -1626,7 +1627,7 @@ bool QualType::UseExcessPrecision(const ASTContext &Ctx) {
     switch (BT->getKind()) {
     case BuiltinType::Kind::Float16: {
       const TargetInfo &TI = Ctx.getTargetInfo();
-      if (TI.hasFloat16Type() && !TI.hasLegalHalfType() &&
+      if (TI.hasFloat16Type() && !TI.hasFastHalfType() &&
           Ctx.getLangOpts().getFloat16ExcessPrecision() !=
               Ctx.getLangOpts().ExcessPrecisionKind::FPP_None)
         return true;
@@ -1921,25 +1922,32 @@ const CXXRecordDecl *Type::getPointeeCXXRecordDecl() const {
     return nullptr;
 
   if (const auto *RT = PointeeType->getAs<RecordType>())
-    return dyn_cast<CXXRecordDecl>(RT->getDecl());
+    return dyn_cast<CXXRecordDecl>(
+        RT->getOriginalDecl()->getDefinitionOrSelf());
 
   return nullptr;
 }
 
 CXXRecordDecl *Type::getAsCXXRecordDecl() const {
-  return dyn_cast_or_null<CXXRecordDecl>(getAsTagDecl());
+  const auto *TT = dyn_cast<TagType>(CanonicalType);
+  if (!isa_and_present<RecordType, InjectedClassNameType>(TT))
+    return nullptr;
+  auto *TD = TT->getOriginalDecl();
+  if (!isa<InjectedClassNameType>(TT) && !isa<CXXRecordDecl>(TD))
+    return nullptr;
+  return cast<CXXRecordDecl>(TD)->getDefinitionOrSelf();
 }
 
 RecordDecl *Type::getAsRecordDecl() const {
-  return dyn_cast_or_null<RecordDecl>(getAsTagDecl());
+  const auto *TT = dyn_cast<TagType>(CanonicalType);
+  if (!isa_and_present<RecordType, InjectedClassNameType>(TT))
+    return nullptr;
+  return cast<RecordDecl>(TT->getOriginalDecl())->getDefinitionOrSelf();
 }
 
 TagDecl *Type::getAsTagDecl() const {
-  if (const auto *TT = getAs<TagType>())
-    return TT->getDecl();
-  if (const auto *Injected = getAs<InjectedClassNameType>())
-    return Injected->getDecl();
-
+  if (const auto *TT = dyn_cast<TagType>(CanonicalType))
+    return TT->getOriginalDecl()->getDefinitionOrSelf();
   return nullptr;
 }
 
@@ -1951,6 +1959,35 @@ Type::getAsNonAliasTemplateSpecializationType() const {
   return TST;
 }
 
+NestedNameSpecifier Type::getPrefix() const {
+  switch (getTypeClass()) {
+  case Type::DependentName:
+    return cast<DependentNameType>(this)->getQualifier();
+  case Type::TemplateSpecialization: {
+    QualifiedTemplateName *S = cast<TemplateSpecializationType>(this)
+                                   ->getTemplateName()
+                                   .getAsAdjustedQualifiedTemplateName();
+    return S ? S->getQualifier() : std::nullopt;
+  }
+  case Type::DependentTemplateSpecialization:
+    return cast<DependentTemplateSpecializationType>(this)
+        ->getDependentTemplateName()
+        .getQualifier();
+  case Type::Enum:
+  case Type::Record:
+  case Type::InjectedClassName:
+    return cast<TagType>(this)->getQualifier();
+  case Type::Typedef:
+    return cast<TypedefType>(this)->getQualifier();
+  case Type::UnresolvedUsing:
+    return cast<UnresolvedUsingType>(this)->getQualifier();
+  case Type::Using:
+    return cast<UsingType>(this)->getQualifier();
+  default:
+    return std::nullopt;
+  }
+}
+
 bool Type::hasAttr(attr::Kind AK) const {
   const Type *Cur = this;
   while (const auto *AT = Cur->getAs<AttributedType>()) {
@@ -1989,10 +2026,6 @@ class GetContainedDeducedTypeVisitor
     return Visit(T->getReplacementType());
   }
 
-  Type *VisitElaboratedType(const ElaboratedType *T) {
-    return Visit(T->getNamedType());
-  }
-
   Type *VisitPointerType(const PointerType *T) {
     return Visit(T->getPointeeType());
   }
@@ -2114,7 +2147,7 @@ bool Type::isIntegralType(const ASTContext &Ctx) const {
   // Complete enum types are integral in C.
   if (!Ctx.getLangOpts().CPlusPlus)
     if (const auto *ET = dyn_cast<EnumType>(CanonicalType))
-      return ET->getDecl()->isComplete();
+      return IsEnumDeclComplete(ET->getOriginalDecl());
 
   return isBitIntType();
 }
@@ -2131,7 +2164,7 @@ bool Type::isIntegralOrUnscopedEnumerationType() const {
 
 bool Type::isUnscopedEnumerationType() const {
   if (const auto *ET = dyn_cast<EnumType>(CanonicalType))
-    return !ET->getDecl()->isScoped();
+    return !ET->getOriginalDecl()->isScoped();
 
   return false;
 }
@@ -2216,8 +2249,10 @@ bool Type::isSignedIntegerType() const {
   if (const EnumType *ET = dyn_cast<EnumType>(CanonicalType)) {
     // Incomplete enum types are not treated as integer types.
     // FIXME: In C++, enum types are never integer types.
-    if (ET->getDecl()->isComplete() && !ET->getDecl()->isScoped())
-      return ET->getDecl()->getIntegerType()->isSignedIntegerType();
+    const auto *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+    if (!ED->isComplete() || ED->isScoped())
+      return false;
+    return ED->getIntegerType()->isSignedIntegerType();
   }
 
   if (const auto *IT = dyn_cast<BitIntType>(CanonicalType))
@@ -2232,9 +2267,12 @@ bool Type::isSignedIntegerOrEnumerationType() const {
   if (const auto *BT = dyn_cast<BuiltinType>(CanonicalType))
     return BT->isSignedInteger();
 
-  if (const auto *ET = dyn_cast<EnumType>(CanonicalType);
-      ET && ET->getDecl()->isComplete())
-    return ET->getDecl()->getIntegerType()->isSignedIntegerType();
+  if (const auto *ET = dyn_cast<EnumType>(CanonicalType)) {
+    const auto *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+    if (!ED->isComplete())
+      return false;
+    return ED->getIntegerType()->isSignedIntegerType();
+  }
 
   if (const auto *IT = dyn_cast<BitIntType>(CanonicalType))
     return IT->isSigned();
@@ -2261,8 +2299,10 @@ bool Type::isUnsignedIntegerType() const {
   if (const auto *ET = dyn_cast<EnumType>(CanonicalType)) {
     // Incomplete enum types are not treated as integer types.
     // FIXME: In C++, enum types are never integer types.
-    if (ET->getDecl()->isComplete() && !ET->getDecl()->isScoped())
-      return ET->getDecl()->getIntegerType()->isUnsignedIntegerType();
+    const auto *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+    if (!ED->isComplete() || ED->isScoped())
+      return false;
+    return ED->getIntegerType()->isUnsignedIntegerType();
   }
 
   if (const auto *IT = dyn_cast<BitIntType>(CanonicalType))
@@ -2277,9 +2317,12 @@ bool Type::isUnsignedIntegerOrEnumerationType() const {
   if (const auto *BT = dyn_cast<BuiltinType>(CanonicalType))
     return BT->isUnsignedInteger();
 
-  if (const auto *ET = dyn_cast<EnumType>(CanonicalType);
-      ET && ET->getDecl()->isComplete())
-    return ET->getDecl()->getIntegerType()->isUnsignedIntegerType();
+  if (const auto *ET = dyn_cast<EnumType>(CanonicalType)) {
+    const auto *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+    if (!ED->isComplete())
+      return false;
+    return ED->getIntegerType()->isUnsignedIntegerType();
+  }
 
   if (const auto *IT = dyn_cast<BitIntType>(CanonicalType))
     return IT->isUnsigned();
@@ -2328,8 +2371,10 @@ bool Type::isRealType() const {
   if (const auto *BT = dyn_cast<BuiltinType>(CanonicalType))
     return BT->getKind() >= BuiltinType::Bool &&
            BT->getKind() <= BuiltinType::Ibm128;
-  if (const auto *ET = dyn_cast<EnumType>(CanonicalType))
-    return ET->getDecl()->isComplete() && !ET->getDecl()->isScoped();
+  if (const auto *ET = dyn_cast<EnumType>(CanonicalType)) {
+    const auto *ED = ET->getOriginalDecl();
+    return !ED->isScoped() && ED->getDefinitionOrSelf()->isComplete();
+  }
   return isBitIntType();
 }
 
@@ -2337,14 +2382,16 @@ bool Type::isArithmeticType() const {
   if (const auto *BT = dyn_cast<BuiltinType>(CanonicalType))
     return BT->getKind() >= BuiltinType::Bool &&
            BT->getKind() <= BuiltinType::Ibm128;
-  if (const auto *ET = dyn_cast<EnumType>(CanonicalType))
+  if (const auto *ET = dyn_cast<EnumType>(CanonicalType)) {
     // GCC allows forward declaration of enum types (forbid by C99 6.7.2.3p2).
     // If a body isn't seen by the time we get here, return false.
     //
     // C++0x: Enumerations are not arithmetic types. For now, just return
     // false for scoped enumerations since that will disable any
     // unwanted implicit conversions.
-    return !ET->getDecl()->isScoped() && ET->getDecl()->isComplete();
+    const auto *ED = ET->getOriginalDecl();
+    return !ED->isScoped() && ED->getDefinitionOrSelf()->isComplete();
+  }
   return isa<ComplexType>(CanonicalType) || isBitIntType();
 }
 
@@ -2352,8 +2399,8 @@ bool Type::hasBooleanRepresentation() const {
   if (const auto *VT = dyn_cast<VectorType>(CanonicalType))
     return VT->getElementType()->isBooleanType();
   if (const auto *ET = dyn_cast<EnumType>(CanonicalType)) {
-    return ET->getDecl()->isComplete() &&
-           ET->getDecl()->getIntegerType()->isBooleanType();
+    const auto *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+    return ED->isComplete() && ED->getIntegerType()->isBooleanType();
   }
   if (const auto *IT = dyn_cast<BitIntType>(CanonicalType))
     return IT->getNumBits() == 1;
@@ -2385,7 +2432,10 @@ Type::ScalarTypeKind Type::getScalarTypeKind() const {
   } else if (isa<MemberPointerType>(T)) {
     return STK_MemberPointer;
   } else if (isa<EnumType>(T)) {
-    assert(cast<EnumType>(T)->getDecl()->isComplete());
+    assert(cast<EnumType>(T)
+               ->getOriginalDecl()
+               ->getDefinitionOrSelf()
+               ->isComplete());
     return STK_Integral;
   } else if (const auto *CT = dyn_cast<ComplexType>(T)) {
     if (CT->getElementType()->isRealFloatingType())
@@ -2409,7 +2459,8 @@ Type::ScalarTypeKind Type::getScalarTypeKind() const {
 /// includes union types.
 bool Type::isAggregateType() const {
   if (const auto *Record = dyn_cast<RecordType>(CanonicalType)) {
-    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(Record->getDecl()))
+    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(
+            Record->getOriginalDecl()->getDefinitionOrSelf()))
       return ClassDecl->isAggregate();
 
     return true;
@@ -2443,7 +2494,8 @@ bool Type::isIncompleteType(NamedDecl **Def) const {
     // be completed.
     return isVoidType();
   case Enum: {
-    EnumDecl *EnumD = cast<EnumType>(CanonicalType)->getDecl();
+    EnumDecl *EnumD =
+        cast<EnumType>(CanonicalType)->getOriginalDecl()->getDefinitionOrSelf();
     if (Def)
       *Def = EnumD;
     return !EnumD->isComplete();
@@ -2451,13 +2503,17 @@ bool Type::isIncompleteType(NamedDecl **Def) const {
   case Record: {
     // A tagged type (struct/union/enum/class) is incomplete if the decl is a
     // forward declaration, but not a full definition (C99 6.2.5p22).
-    RecordDecl *Rec = cast<RecordType>(CanonicalType)->getDecl();
+    RecordDecl *Rec = cast<RecordType>(CanonicalType)
+                          ->getOriginalDecl()
+                          ->getDefinitionOrSelf();
     if (Def)
       *Def = Rec;
     return !Rec->isCompleteDefinition();
   }
   case InjectedClassName: {
-    CXXRecordDecl *Rec = cast<InjectedClassNameType>(CanonicalType)->getDecl();
+    CXXRecordDecl *Rec = cast<InjectedClassNameType>(CanonicalType)
+                             ->getOriginalDecl()
+                             ->getDefinitionOrSelf();
     if (!Rec->isBeingDefined())
       return false;
     if (Def)
@@ -2739,9 +2795,9 @@ bool QualType::isCXX98PODType(const ASTContext &Context) const {
     return true;
 
   case Type::Record:
-    if (const auto *ClassDecl =
-            dyn_cast<CXXRecordDecl>(cast<RecordType>(CanonicalType)->getDecl()))
-      return ClassDecl->isPOD();
+    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(
+            cast<RecordType>(CanonicalType)->getOriginalDecl()))
+      return ClassDecl->getDefinitionOrSelf()->isPOD();
 
     // C struct/union is POD.
     return true;
@@ -2782,7 +2838,8 @@ bool QualType::isTrivialType(const ASTContext &Context) const {
   if (CanonicalType->isScalarType() || CanonicalType->isVectorType())
     return true;
   if (const auto *RT = CanonicalType->getAs<RecordType>()) {
-    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
+    if (const auto *ClassDecl =
+            dyn_cast<CXXRecordDecl>(RT->getOriginalDecl())) {
       // C++20 [class]p6:
       //   A trivial class is a class that is trivially copyable, and
       //     has one or more eligible default constructors such that each is
@@ -2841,14 +2898,17 @@ static bool isTriviallyCopyableTypeImpl(const QualType &type,
     return true;
 
   if (const auto *RT = CanonicalType->getAs<RecordType>()) {
-    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
+    if (const auto *ClassDecl =
+            dyn_cast<CXXRecordDecl>(RT->getOriginalDecl())) {
       if (IsCopyConstructible) {
         return ClassDecl->isTriviallyCopyConstructible();
       } else {
         return ClassDecl->isTriviallyCopyable();
       }
     }
-    return !RT->getDecl()->isNonTrivialToPrimitiveCopy();
+    return !RT->getOriginalDecl()
+                ->getDefinitionOrSelf()
+                ->isNonTrivialToPrimitiveCopy();
   }
   // No other types can match.
   return false;
@@ -2938,7 +2998,9 @@ QualType::PrimitiveDefaultInitializeKind
 QualType::isNonTrivialToPrimitiveDefaultInitialize() const {
   if (const auto *RT =
           getTypePtr()->getBaseElementTypeUnsafe()->getAs<RecordType>())
-    if (RT->getDecl()->isNonTrivialToPrimitiveDefaultInitialize())
+    if (RT->getOriginalDecl()
+            ->getDefinitionOrSelf()
+            ->isNonTrivialToPrimitiveDefaultInitialize())
       return PDIK_Struct;
 
   switch (getQualifiers().getObjCLifetime()) {
@@ -2954,7 +3016,9 @@ QualType::isNonTrivialToPrimitiveDefaultInitialize() const {
 QualType::PrimitiveCopyKind QualType::isNonTrivialToPrimitiveCopy() const {
   if (const auto *RT =
           getTypePtr()->getBaseElementTypeUnsafe()->getAs<RecordType>())
-    if (RT->getDecl()->isNonTrivialToPrimitiveCopy())
+    if (RT->getOriginalDecl()
+            ->getDefinitionOrSelf()
+            ->isNonTrivialToPrimitiveCopy())
       return PCK_Struct;
 
   Qualifiers Qs = getQualifiers();
@@ -3022,8 +3086,8 @@ bool Type::isLiteralType(const ASTContext &Ctx) const {
     //    -- all non-static data members and base classes of literal types
     //
     // We resolve DR1361 by ignoring the second bullet.
-    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
-      return ClassDecl->isLiteral();
+    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl()))
+      return ClassDecl->getDefinitionOrSelf()->isLiteral();
 
     return true;
   }
@@ -3076,8 +3140,8 @@ bool Type::isStandardLayoutType() const {
   if (BaseTy->isScalarType() || BaseTy->isVectorType())
     return true;
   if (const auto *RT = BaseTy->getAs<RecordType>()) {
-    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
-      if (!ClassDecl->isStandardLayout())
+    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl()))
+      if (!ClassDecl->getDefinitionOrSelf()->isStandardLayout())
         return false;
 
     // Default to 'true' for non-C++ class types.
@@ -3119,7 +3183,9 @@ bool QualType::isCXX11PODType(const ASTContext &Context) const {
   if (BaseTy->isScalarType() || BaseTy->isVectorType())
     return true;
   if (const auto *RT = BaseTy->getAs<RecordType>()) {
-    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
+    if (const auto *ClassDecl =
+            dyn_cast<CXXRecordDecl>(RT->getOriginalDecl())) {
+      ClassDecl = ClassDecl->getDefinitionOrSelf();
       // C++11 [class]p10:
       //   A POD struct is a non-union class that is both a trivial class [...]
       if (!ClassDecl->isTrivial())
@@ -3159,8 +3225,9 @@ bool Type::isNothrowT() const {
 
 bool Type::isAlignValT() const {
   if (const auto *ET = getAs<EnumType>()) {
-    IdentifierInfo *II = ET->getDecl()->getIdentifier();
-    if (II && II->isStr("align_val_t") && ET->getDecl()->isInStdNamespace())
+    const auto *ED = ET->getOriginalDecl();
+    IdentifierInfo *II = ED->getIdentifier();
+    if (II && II->isStr("align_val_t") && ED->isInStdNamespace())
       return true;
   }
   return false;
@@ -3168,8 +3235,9 @@ bool Type::isAlignValT() const {
 
 bool Type::isStdByteType() const {
   if (const auto *ET = getAs<EnumType>()) {
-    IdentifierInfo *II = ET->getDecl()->getIdentifier();
-    if (II && II->isStr("byte") && ET->getDecl()->isInStdNamespace())
+    const auto *ED = ET->getOriginalDecl();
+    IdentifierInfo *II = ED->getIdentifier();
+    if (II && II->isStr("byte") && ED->isInStdNamespace())
       return true;
   }
   return false;
@@ -3188,7 +3256,6 @@ bool Type::isSpecifierType() const {
   case TemplateTypeParm:
   case SubstTemplateTypeParm:
   case TemplateSpecialization:
-  case Elaborated:
   case DependentName:
   case DependentTemplateSpecialization:
   case ObjCInterface:
@@ -3199,8 +3266,7 @@ bool Type::isSpecifierType() const {
   }
 }
 
-ElaboratedTypeKeyword
-TypeWithKeyword::getKeywordForTypeSpec(unsigned TypeSpec) {
+ElaboratedTypeKeyword KeywordHelpers::getKeywordForTypeSpec(unsigned TypeSpec) {
   switch (TypeSpec) {
   default:
     return ElaboratedTypeKeyword::None;
@@ -3219,7 +3285,7 @@ TypeWithKeyword::getKeywordForTypeSpec(unsigned TypeSpec) {
   }
 }
 
-TagTypeKind TypeWithKeyword::getTagTypeKindForTypeSpec(unsigned TypeSpec) {
+TagTypeKind KeywordHelpers::getTagTypeKindForTypeSpec(unsigned TypeSpec) {
   switch (TypeSpec) {
   case TST_class:
     return TagTypeKind::Class;
@@ -3237,7 +3303,7 @@ TagTypeKind TypeWithKeyword::getTagTypeKindForTypeSpec(unsigned TypeSpec) {
 }
 
 ElaboratedTypeKeyword
-TypeWithKeyword::getKeywordForTagTypeKind(TagTypeKind Kind) {
+KeywordHelpers::getKeywordForTagTypeKind(TagTypeKind Kind) {
   switch (Kind) {
   case TagTypeKind::Class:
     return ElaboratedTypeKeyword::Class;
@@ -3254,7 +3320,7 @@ TypeWithKeyword::getKeywordForTagTypeKind(TagTypeKind Kind) {
 }
 
 TagTypeKind
-TypeWithKeyword::getTagTypeKindForKeyword(ElaboratedTypeKeyword Keyword) {
+KeywordHelpers::getTagTypeKindForKeyword(ElaboratedTypeKeyword Keyword) {
   switch (Keyword) {
   case ElaboratedTypeKeyword::Class:
     return TagTypeKind::Class;
@@ -3273,7 +3339,7 @@ TypeWithKeyword::getTagTypeKindForKeyword(ElaboratedTypeKeyword Keyword) {
   llvm_unreachable("Unknown elaborated type keyword.");
 }
 
-bool TypeWithKeyword::KeywordIsTagTypeKind(ElaboratedTypeKeyword Keyword) {
+bool KeywordHelpers::KeywordIsTagTypeKind(ElaboratedTypeKeyword Keyword) {
   switch (Keyword) {
   case ElaboratedTypeKeyword::None:
   case ElaboratedTypeKeyword::Typename:
@@ -3288,7 +3354,7 @@ bool TypeWithKeyword::KeywordIsTagTypeKind(ElaboratedTypeKeyword Keyword) {
   llvm_unreachable("Unknown elaborated type keyword.");
 }
 
-StringRef TypeWithKeyword::getKeywordName(ElaboratedTypeKeyword Keyword) {
+StringRef KeywordHelpers::getKeywordName(ElaboratedTypeKeyword Keyword) {
   switch (Keyword) {
   case ElaboratedTypeKeyword::None:
     return {};
@@ -3338,13 +3404,21 @@ void DependentTemplateSpecializationType::Profile(
 
 bool Type::isElaboratedTypeSpecifier() const {
   ElaboratedTypeKeyword Keyword;
-  if (const auto *Elab = dyn_cast<ElaboratedType>(this))
-    Keyword = Elab->getKeyword();
+  if (const auto *TST = dyn_cast<TemplateSpecializationType>(this))
+    Keyword = TST->getKeyword();
   else if (const auto *DepName = dyn_cast<DependentNameType>(this))
     Keyword = DepName->getKeyword();
   else if (const auto *DepTST =
                dyn_cast<DependentTemplateSpecializationType>(this))
     Keyword = DepTST->getKeyword();
+  else if (const auto *T = dyn_cast<TagType>(this))
+    Keyword = T->getKeyword();
+  else if (const auto *T = dyn_cast<TypedefType>(this))
+    Keyword = T->getKeyword();
+  else if (const auto *T = dyn_cast<UnresolvedUsingType>(this))
+    Keyword = T->getKeyword();
+  else if (const auto *T = dyn_cast<UsingType>(this))
+    Keyword = T->getKeyword();
   else
     return false;
 
@@ -3677,6 +3751,16 @@ FunctionProtoType::FunctionProtoType(QualType result, ArrayRef<QualType> params,
     FunctionTypeBits.HasExtraBitfields = false;
   }
 
+  // Propagate any extra attribute information.
+  if (epi.requiresFunctionProtoTypeExtraAttributeInfo()) {
+    auto &ExtraAttrInfo = *getTrailingObjects<FunctionTypeExtraAttributeInfo>();
+    ExtraAttrInfo.CFISalt = epi.ExtraAttributeInfo.CFISalt;
+
+    // Also set the bit in FunctionTypeExtraBitfields.
+    auto &ExtraBits = *getTrailingObjects<FunctionTypeExtraBitfields>();
+    ExtraBits.HasExtraAttributeInfo = true;
+  }
+
   if (epi.requiresFunctionProtoTypeArmAttributes()) {
     auto &ArmTypeAttrs = *getTrailingObjects<FunctionTypeArmAttributes>();
     ArmTypeAttrs = FunctionTypeArmAttributes();
@@ -3894,7 +3978,8 @@ void FunctionProtoType::Profile(llvm::FoldingSetNodeID &ID, QualType Result,
   // This is followed by the ext info:
   //      int
   // Finally we have a trailing return type flag (bool)
-  // combined with AArch64 SME Attributes, to save space:
+  // combined with AArch64 SME Attributes and extra attribute info, to save
+  // space:
   //      int
   // combined with any FunctionEffects
   //
@@ -3929,6 +4014,7 @@ void FunctionProtoType::Profile(llvm::FoldingSetNodeID &ID, QualType Result,
   }
 
   epi.ExtInfo.Profile(ID);
+  epi.ExtraAttributeInfo.Profile(ID);
 
   unsigned EffectCount = epi.FunctionEffects.size();
   bool HasConds = !epi.FunctionEffects.Conditions.empty();
@@ -4011,34 +4097,53 @@ StringRef CountAttributedType::getAttributeName(bool WithMacroPrefix) const {
 #undef ENUMERATE_ATTRS
 }
 
-TypedefType::TypedefType(TypeClass tc, const TypedefNameDecl *D,
-                         QualType UnderlyingType, bool HasTypeDifferentFromDecl)
-    : Type(tc, UnderlyingType.getCanonicalType(),
-           toSemanticDependence(UnderlyingType->getDependence())),
+TypedefType::TypedefType(TypeClass TC, ElaboratedTypeKeyword Keyword,
+                         NestedNameSpecifier Qualifier,
+                         const TypedefNameDecl *D, QualType UnderlyingType,
+                         bool HasTypeDifferentFromDecl)
+    : TypeWithKeyword(
+          Keyword, TC, UnderlyingType.getCanonicalType(),
+          toSemanticDependence(UnderlyingType->getDependence()) |
+              (Qualifier
+                   ? toTypeDependence(Qualifier.getDependence() &
+                                      ~NestedNameSpecifierDependence::Dependent)
+                   : TypeDependence{})),
       Decl(const_cast<TypedefNameDecl *>(D)) {
-  TypedefBits.hasTypeDifferentFromDecl = HasTypeDifferentFromDecl;
-  if (!typeMatchesDecl())
-    *getTrailingObjects() = UnderlyingType;
+  if ((TypedefBits.hasQualifier = !!Qualifier))
+    *getTrailingObjects<NestedNameSpecifier>() = Qualifier;
+  if ((TypedefBits.hasTypeDifferentFromDecl = HasTypeDifferentFromDecl))
+    *getTrailingObjects<QualType>() = UnderlyingType;
 }
 
 QualType TypedefType::desugar() const {
-  return typeMatchesDecl() ? Decl->getUnderlyingType() : *getTrailingObjects();
-}
-
-UsingType::UsingType(const UsingShadowDecl *Found, QualType Underlying,
-                     QualType Canon)
-    : Type(Using, Canon, toSemanticDependence(Canon->getDependence())),
-      Found(const_cast<UsingShadowDecl *>(Found)) {
-  UsingBits.hasTypeDifferentFromDecl = !Underlying.isNull();
-  if (!typeMatchesDecl())
-    *getTrailingObjects() = Underlying;
-}
-
-QualType UsingType::getUnderlyingType() const {
-  return typeMatchesDecl()
-             ? QualType(
-                   cast<TypeDecl>(Found->getTargetDecl())->getTypeForDecl(), 0)
-             : *getTrailingObjects();
+  return typeMatchesDecl() ? Decl->getUnderlyingType()
+                           : *getTrailingObjects<QualType>();
+}
+
+UnresolvedUsingType::UnresolvedUsingType(ElaboratedTypeKeyword Keyword,
+                                         NestedNameSpecifier Qualifier,
+                                         const UnresolvedUsingTypenameDecl *D,
+                                         const Type *CanonicalType)
+    : TypeWithKeyword(
+          Keyword, UnresolvedUsing, QualType(CanonicalType, 0),
+          TypeDependence::DependentInstantiation |
+              (Qualifier
+                   ? toTypeDependence(Qualifier.getDependence() &
+                                      ~NestedNameSpecifierDependence::Dependent)
+                   : TypeDependence{})),
+      Decl(const_cast<UnresolvedUsingTypenameDecl *>(D)) {
+  if ((UnresolvedUsingBits.hasQualifier = !!Qualifier))
+    *getTrailingObjects<NestedNameSpecifier>() = Qualifier;
+}
+
+UsingType::UsingType(ElaboratedTypeKeyword Keyword,
+                     NestedNameSpecifier Qualifier, const UsingShadowDecl *D,
+                     QualType UnderlyingType)
+    : TypeWithKeyword(Keyword, Using, UnderlyingType.getCanonicalType(),
+                      toSemanticDependence(UnderlyingType->getDependence())),
+      D(const_cast<UsingShadowDecl *>(D)), UnderlyingType(UnderlyingType) {
+  if ((UsingBits.hasQualifier = !!Qualifier))
+    *getTrailingObjects() = Qualifier;
 }
 
 QualType MacroQualifiedType::desugar() const { return getUnderlyingType(); }
@@ -4212,24 +4317,79 @@ UnaryTransformType::UnaryTransformType(QualType BaseType,
     : Type(UnaryTransform, CanonicalType, BaseType->getDependence()),
       BaseType(BaseType), UnderlyingType(UnderlyingType), UKind(UKind) {}
 
-TagType::TagType(TypeClass TC, const TagDecl *D, QualType can)
-    : Type(TC, can,
-           D->isDependentType() ? TypeDependence::DependentInstantiation
-                                : TypeDependence::None),
-      decl(const_cast<TagDecl *>(D)) {}
-
-static TagDecl *getInterestingTagDecl(TagDecl *decl) {
-  for (auto *I : decl->redecls()) {
-    if (I->isCompleteDefinition() || I->isBeingDefined())
-      return I;
+TagType::TagType(TypeClass TC, ElaboratedTypeKeyword Keyword,
+                 NestedNameSpecifier Qualifier, const TagDecl *Tag,
+                 bool OwnsTag, bool ISInjected, const Type *CanonicalType)
+    : TypeWithKeyword(
+          Keyword, TC, QualType(CanonicalType, 0),
+          (Tag->isDependentType() ? TypeDependence::DependentInstantiation
+                                  : TypeDependence::None) |
+              (Qualifier
+                   ? toTypeDependence(Qualifier.getDependence() &
+                                      ~NestedNameSpecifierDependence::Dependent)
+                   : TypeDependence{})),
+      decl(const_cast<TagDecl *>(Tag)) {
+  if ((TagTypeBits.HasQualifier = !!Qualifier))
+    getTrailingQualifier() = Qualifier;
+  TagTypeBits.OwnsTag = !!OwnsTag;
+  TagTypeBits.IsInjected = ISInjected;
+}
+
+void *TagType::getTrailingPointer() const {
+  switch (getTypeClass()) {
+  case Type::Enum:
+    return const_cast<EnumType *>(cast<EnumType>(this) + 1);
+  case Type::Record:
+    return const_cast<RecordType *>(cast<RecordType>(this) + 1);
+  case Type::InjectedClassName:
+    return const_cast<InjectedClassNameType *>(
+        cast<InjectedClassNameType>(this) + 1);
+  default:
+    llvm_unreachable("unexpected type class");
   }
-  // If there's no definition (not even in progress), return what we have.
-  return decl;
 }
 
-TagDecl *TagType::getDecl() const { return getInterestingTagDecl(decl); }
+NestedNameSpecifier &TagType::getTrailingQualifier() const {
+  assert(TagTypeBits.HasQualifier);
+  return *reinterpret_cast<NestedNameSpecifier *>(llvm::alignAddr(
+      getTrailingPointer(), llvm::Align::Of<NestedNameSpecifier *>()));
+}
+
+NestedNameSpecifier TagType::getQualifier() const {
+  return TagTypeBits.HasQualifier ? getTrailingQualifier() : std::nullopt;
+}
+
+ClassTemplateDecl *TagType::getTemplateDecl() const {
+  auto *Decl = dyn_cast<CXXRecordDecl>(decl);
+  if (!Decl)
+    return nullptr;
+  if (auto *RD = dyn_cast<ClassTemplateSpecializationDecl>(Decl))
+    return RD->getSpecializedTemplate();
+  return Decl->getDescribedClassTemplate();
+}
+
+TemplateName TagType::getTemplateName(const ASTContext &Ctx) const {
+  auto *TD = getTemplateDecl();
+  if (!TD)
+    return TemplateName();
+  if (isCanonicalUnqualified())
+    return TemplateName(TD);
+  return Ctx.getQualifiedTemplateName(getQualifier(), /*TemplateKeyword=*/false,
+                                      TemplateName(TD));
+}
 
-bool TagType::isBeingDefined() const { return getDecl()->isBeingDefined(); }
+ArrayRef<TemplateArgument>
+TagType::getTemplateArgs(const ASTContext &Ctx) const {
+  auto *Decl = dyn_cast<CXXRecordDecl>(decl);
+  if (!Decl)
+    return {};
+
+  if (auto *RD = dyn_cast<ClassTemplateSpecializationDecl>(Decl))
+    return RD->getTemplateArgs().asArray();
+  if (ClassTemplateDecl *TD = Decl->getDescribedClassTemplate())
+    return TD->getTemplateParameters()->getInjectedTemplateArgs(Ctx);
+  return {};
+}
 
 bool RecordType::hasConstFields() const {
   std::vector<const RecordType *> RecordTypeList;
@@ -4237,8 +4397,10 @@ bool RecordType::hasConstFields() const {
   unsigned NextToCheckIndex = 0;
 
   while (RecordTypeList.size() > NextToCheckIndex) {
-    for (FieldDecl *FD :
-         RecordTypeList[NextToCheckIndex]->getDecl()->fields()) {
+    for (FieldDecl *FD : RecordTypeList[NextToCheckIndex]
+                             ->getOriginalDecl()
+                             ->getDefinitionOrSelf()
+                             ->fields()) {
       QualType FieldTy = FD->getType();
       if (FieldTy.isConstQualified())
         return true;
@@ -4253,6 +4415,13 @@ bool RecordType::hasConstFields() const {
   return false;
 }
 
+InjectedClassNameType::InjectedClassNameType(ElaboratedTypeKeyword Keyword,
+                                             NestedNameSpecifier Qualifier,
+                                             const TagDecl *TD, bool IsInjected,
+                                             const Type *CanonicalType)
+    : TagType(TypeClass::InjectedClassName, Keyword, Qualifier, TD,
+              /*OwnsTag=*/false, IsInjected, CanonicalType) {}
+
 AttributedType::AttributedType(QualType canon, const Attr *attr,
                                QualType modified, QualType equivalent)
     : AttributedType(canon, attr->getKind(), attr, modified, equivalent) {}
@@ -4340,10 +4509,6 @@ bool AttributedType::isCallingConv() const {
   llvm_unreachable("invalid attr kind");
 }
 
-CXXRecordDecl *InjectedClassNameType::getDecl() const {
-  return cast<CXXRecordDecl>(getInterestingTagDecl(Decl));
-}
-
 IdentifierInfo *TemplateTypeParmType::getIdentifier() const {
   return isCanonicalUnqualified() ? nullptr : getDecl()->getIdentifier();
 }
@@ -4393,17 +4558,45 @@ void SubstTemplateTypeParmType::Profile(llvm::FoldingSetNodeID &ID,
   ID.AddBoolean(Final);
 }
 
+SubstPackType::SubstPackType(TypeClass Derived, QualType Canon,
+                             const TemplateArgument &ArgPack)
+    : Type(Derived, Canon,
+           TypeDependence::DependentInstantiation |
+               TypeDependence::UnexpandedPack),
+      Arguments(ArgPack.pack_begin()) {
+  assert(llvm::all_of(
+             ArgPack.pack_elements(),
+             [](auto &P) { return P.getKind() == TemplateArgument::Type; }) &&
+         "non-type argument to SubstPackType?");
+  SubstPackTypeBits.NumArgs = ArgPack.pack_size();
+}
+
+TemplateArgument SubstPackType::getArgumentPack() const {
+  return TemplateArgument(llvm::ArrayRef(Arguments, getNumArgs()));
+}
+
+void SubstPackType::Profile(llvm::FoldingSetNodeID &ID) {
+  Profile(ID, getArgumentPack());
+}
+
+void SubstPackType::Profile(llvm::FoldingSetNodeID &ID,
+                            const TemplateArgument &ArgPack) {
+  ID.AddInteger(ArgPack.pack_size());
+  for (const auto &P : ArgPack.pack_elements())
+    ID.AddPointer(P.getAsType().getAsOpaquePtr());
+}
+
 SubstTemplateTypeParmPackType::SubstTemplateTypeParmPackType(
     QualType Canon, Decl *AssociatedDecl, unsigned Index, bool Final,
     const TemplateArgument &ArgPack)
-    : Type(SubstTemplateTypeParmPack, Canon,
-           TypeDependence::DependentInstantiation |
-               TypeDependence::UnexpandedPack),
-      Arguments(ArgPack.pack_begin()),
+    : SubstPackType(SubstTemplateTypeParmPack, Canon, ArgPack),
       AssociatedDeclAndFinal(AssociatedDecl, Final) {
-  SubstTemplateTypeParmPackTypeBits.Index = Index;
-  SubstTemplateTypeParmPackTypeBits.NumArgs = ArgPack.pack_size();
   assert(AssociatedDecl != nullptr);
+
+  SubstPackTypeBits.SubstTemplTypeParmPackIndex = Index;
+  assert(getNumArgs() == ArgPack.pack_size() &&
+         "Parent bitfields in SubstPackType were overwritten."
+         "Check NumSubstPackTypeBits.");
 }
 
 Decl *SubstTemplateTypeParmPackType::getAssociatedDecl() const {
@@ -4423,10 +4616,6 @@ IdentifierInfo *SubstTemplateTypeParmPackType::getIdentifier() const {
   return getReplacedParameter()->getIdentifier();
 }
 
-TemplateArgument SubstTemplateTypeParmPackType::getArgumentPack() const {
-  return TemplateArgument(llvm::ArrayRef(Arguments, getNumArgs()));
-}
-
 void SubstTemplateTypeParmPackType::Profile(llvm::FoldingSetNodeID &ID) {
   Profile(ID, getAssociatedDecl(), getIndex(), getFinal(), getArgumentPack());
 }
@@ -4438,11 +4627,13 @@ void SubstTemplateTypeParmPackType::Profile(llvm::FoldingSetNodeID &ID,
   ID.AddPointer(AssociatedDecl);
   ID.AddInteger(Index);
   ID.AddBoolean(Final);
-  ID.AddInteger(ArgPack.pack_size());
-  for (const auto &P : ArgPack.pack_elements())
-    ID.AddPointer(P.getAsType().getAsOpaquePtr());
+  SubstPackType::Profile(ID, ArgPack);
 }
 
+SubstBuiltinTemplatePackType::SubstBuiltinTemplatePackType(
+    QualType Canon, const TemplateArgument &ArgPack)
+    : SubstPackType(SubstBuiltinTemplatePack, Canon, ArgPack) {}
+
 bool TemplateSpecializationType::anyDependentTemplateArguments(
     const TemplateArgumentListInfo &Args,
     ArrayRef<TemplateArgument> Converted) {
@@ -4466,17 +4657,28 @@ bool TemplateSpecializationType::anyInstantiationDependentTemplateArguments(
   return false;
 }
 
+static TypeDependence
+getTemplateSpecializationTypeDependence(QualType Underlying, TemplateName T) {
+  TypeDependence D = Underlying.isNull()
+                         ? TypeDependence::DependentInstantiation
+                         : toSemanticDependence(Underlying->getDependence());
+  D |= toTypeDependence(T.getDependence()) & TypeDependence::UnexpandedPack;
+  if (isPackProducingBuiltinTemplateName(T)) {
+    if (Underlying.isNull()) // Dependent, will produce a pack on substitution.
+      D |= TypeDependence::UnexpandedPack;
+    else
+      D |= (Underlying->getDependence() & TypeDependence::UnexpandedPack);
+  }
+  return D;
+}
+
 TemplateSpecializationType::TemplateSpecializationType(
-    TemplateName T, bool IsAlias, ArrayRef<TemplateArgument> Args,
-    QualType Underlying)
-    : Type(TemplateSpecialization,
-           Underlying.isNull() ? QualType(this, 0)
-                               : Underlying.getCanonicalType(),
-           (Underlying.isNull()
-                ? TypeDependence::DependentInstantiation
-                : toSemanticDependence(Underlying->getDependence())) |
-               (toTypeDependence(T.getDependence()) &
-                TypeDependence::UnexpandedPack)),
+    ElaboratedTypeKeyword Keyword, TemplateName T, bool IsAlias,
+    ArrayRef<TemplateArgument> Args, QualType Underlying)
+    : TypeWithKeyword(Keyword, TemplateSpecialization,
+                      Underlying.isNull() ? QualType(this, 0)
+                                          : Underlying.getCanonicalType(),
+                      getTemplateSpecializationTypeDependence(Underlying, T)),
       Template(T) {
   TemplateSpecializationTypeBits.NumArgs = Args.size();
   TemplateSpecializationTypeBits.TypeAlias = IsAlias;
@@ -4522,6 +4724,12 @@ QualType TemplateSpecializationType::getAliasedType() const {
   return *reinterpret_cast<const QualType *>(template_arguments().end());
 }
 
+bool clang::TemplateSpecializationType::isSugared() const {
+  return !isDependentType() || isCurrentInstantiation() || isTypeAlias() ||
+         (isPackProducingBuiltinTemplateName(Template) &&
+          isa<SubstBuiltinTemplatePackType>(*getCanonicalTypeInternal()));
+}
+
 void TemplateSpecializationType::Profile(llvm::FoldingSetNodeID &ID,
                                          const ASTContext &Ctx) {
   Profile(ID, Template, template_arguments(),
@@ -4699,7 +4907,8 @@ static CachedProperties computeCachedProperties(const Type *T) {
 
   case Type::Record:
   case Type::Enum: {
-    const TagDecl *Tag = cast<TagType>(T)->getDecl();
+    const TagDecl *Tag =
+        cast<TagType>(T)->getOriginalDecl()->getDefinitionOrSelf();
 
     // C++ [basic.link]p8:
     //     - it is a class or enumeration type that is named (or has a name
@@ -4726,12 +4935,9 @@ static CachedProperties computeCachedProperties(const Type *T) {
   case Type::MemberPointer: {
     const auto *MPT = cast<MemberPointerType>(T);
     CachedProperties Cls = [&] {
-      if (auto *RD = MPT->getMostRecentCXXRecordDecl())
-        return Cache::get(QualType(RD->getTypeForDecl(), 0));
-      if (const Type *T = MPT->getQualifier()->getAsType())
-        return Cache::get(T);
-      // Treat as a dependent type.
-      return CachedProperties(Linkage::External, false);
+      if (MPT->isSugared())
+        MPT = cast<MemberPointerType>(MPT->getCanonicalTypeInternal());
+      return Cache::get(MPT->getQualifier().getAsType());
     }();
     return merge(Cls, Cache::get(MPT->getPointeeType()));
   }
@@ -4811,7 +5017,8 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
 
   case Type::Record:
   case Type::Enum:
-    return getDeclLinkageAndVisibility(cast<TagType>(T)->getDecl());
+    return getDeclLinkageAndVisibility(
+        cast<TagType>(T)->getOriginalDecl()->getDefinitionOrSelf());
 
   case Type::Complex:
     return computeTypeLinkageInfo(cast<ComplexType>(T)->getElementType());
@@ -4827,8 +5034,8 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
     LinkageInfo LV;
     if (auto *D = MPT->getMostRecentCXXRecordDecl()) {
       LV.merge(getDeclLinkageAndVisibility(D));
-    } else if (auto *Ty = MPT->getQualifier()->getAsType()) {
-      LV.merge(computeTypeLinkageInfo(Ty));
+    } else {
+      LV.merge(computeTypeLinkageInfo(MPT->getQualifier().getAsType()));
     }
     LV.merge(computeTypeLinkageInfo(MPT->getPointeeType()));
     return LV;
@@ -4938,6 +5145,7 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
   case Type::UnaryTransform:
   case Type::TemplateTypeParm:
   case Type::SubstTemplateTypeParmPack:
+  case Type::SubstBuiltinTemplatePack:
   case Type::DependentName:
   case Type::DependentTemplateSpecialization:
   case Type::Auto:
@@ -5014,7 +5222,7 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
     llvm_unreachable("unknown builtin type");
 
   case Type::Record: {
-    const RecordDecl *RD = cast<RecordType>(type)->getDecl();
+    const RecordDecl *RD = cast<RecordType>(type)->getOriginalDecl();
     // For template specializations, look only at primary template attributes.
     // This is a consistent regardless of whether the instantiation is known.
     if (const auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(RD))
@@ -5212,14 +5420,18 @@ bool Type::isCARCBridgableType() const {
 /// Check if the specified type is the CUDA device builtin surface type.
 bool Type::isCUDADeviceBuiltinSurfaceType() const {
   if (const auto *RT = getAs<RecordType>())
-    return RT->getDecl()->hasAttr<CUDADeviceBuiltinSurfaceTypeAttr>();
+    return RT->getOriginalDecl()
+        ->getMostRecentDecl()
+        ->hasAttr<CUDADeviceBuiltinSurfaceTypeAttr>();
   return false;
 }
 
 /// Check if the specified type is the CUDA device builtin texture type.
 bool Type::isCUDADeviceBuiltinTextureType() const {
   if (const auto *RT = getAs<RecordType>())
-    return RT->getDecl()->hasAttr<CUDADeviceBuiltinTextureTypeAttr>();
+    return RT->getOriginalDecl()
+        ->getMostRecentDecl()
+        ->hasAttr<CUDADeviceBuiltinTextureTypeAttr>();
   return false;
 }
 
@@ -5246,6 +5458,15 @@ bool Type::isHLSLResourceRecord() const {
   return HLSLAttributedResourceType::findHandleTypeOnResource(this) != nullptr;
 }
 
+bool Type::isHLSLResourceRecordArray() const {
+  const Type *Ty = getUnqualifiedDesugaredType();
+  if (!Ty->isArrayType())
+    return false;
+  while (isa<ConstantArrayType>(Ty))
+    Ty = Ty->getArrayElementTypeNoTypeQual();
+  return Ty->isHLSLResourceRecord();
+}
+
 bool Type::isHLSLIntangibleType() const {
   const Type *Ty = getUnqualifiedDesugaredType();
 
@@ -5283,7 +5504,7 @@ QualType::DestructionKind QualType::isDestructedTypeImpl(QualType type) {
   }
 
   if (const auto *RT = type->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl();
     if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
       /// Check if this is a C++ object with a non-trivial destructor.
       if (CXXRD->hasDefinition() && !CXXRD->hasTrivialDestructor())
@@ -5291,7 +5512,7 @@ QualType::DestructionKind QualType::isDestructedTypeImpl(QualType type) {
     } else {
       /// Check if this is a C struct that is non-trivial to destroy or an array
       /// that contains such a struct.
-      if (RD->isNonTrivialToPrimitiveDestroy())
+      if (RD->getDefinitionOrSelf()->isNonTrivialToPrimitiveDestroy())
         return DK_nontrivial_c_struct;
     }
   }
@@ -5301,16 +5522,16 @@ QualType::DestructionKind QualType::isDestructedTypeImpl(QualType type) {
 
 bool MemberPointerType::isSugared() const {
   CXXRecordDecl *D1 = getMostRecentCXXRecordDecl(),
-                *D2 = getQualifier()->getAsRecordDecl();
+                *D2 = getQualifier().getAsRecordDecl();
   assert(!D1 == !D2);
   return D1 != D2 && D1->getCanonicalDecl() != D2->getCanonicalDecl();
 }
 
 void MemberPointerType::Profile(llvm::FoldingSetNodeID &ID, QualType Pointee,
-                                const NestedNameSpecifier *Qualifier,
+                                const NestedNameSpecifier Qualifier,
                                 const CXXRecordDecl *Cls) {
   ID.AddPointer(Pointee.getAsOpaquePtr());
-  ID.AddPointer(Qualifier);
+  Qualifier.Profile(ID);
   if (Cls)
     ID.AddPointer(Cls->getCanonicalDecl());
 }
@@ -5318,14 +5539,14 @@ void MemberPointerType::Profile(llvm::FoldingSetNodeID &ID, QualType Pointee,
 CXXRecordDecl *MemberPointerType::getCXXRecordDecl() const {
   return dyn_cast<MemberPointerType>(getCanonicalTypeInternal())
       ->getQualifier()
-      ->getAsRecordDecl();
+      .getAsRecordDecl();
 }
 
 CXXRecordDecl *MemberPointerType::getMostRecentCXXRecordDecl() const {
   auto *RD = getCXXRecordDecl();
   if (!RD)
     return nullptr;
-  return RD->getMostRecentNonInjectedDecl();
+  return RD->getMostRecentDecl();
 }
 
 void clang::FixedPointValueToString(SmallVectorImpl<char> &Str,
diff --git a/clang/lib/AST/TypeLoc.cpp b/clang/lib/AST/TypeLoc.cpp
index 5c45c596538f8..fbe8772924465 100644
--- a/clang/lib/AST/TypeLoc.cpp
+++ b/clang/lib/AST/TypeLoc.cpp
@@ -195,15 +195,6 @@ SourceLocation TypeLoc::getBeginLoc() const {
   TypeLoc LeftMost = Cur;
   while (true) {
     switch (Cur.getTypeLocClass()) {
-    case Elaborated:
-      if (Cur.getLocalSourceRange().getBegin().isValid()) {
-        LeftMost = Cur;
-        break;
-      }
-      Cur = Cur.getNextTypeLoc();
-      if (Cur.isNull())
-        break;
-      continue;
     case FunctionProto:
       if (Cur.castAs<FunctionProtoTypeLoc>().getTypePtr()
               ->hasTrailingReturn()) {
@@ -275,7 +266,6 @@ SourceLocation TypeLoc::getEndLoc() const {
         Last = Cur;
       break;
     case Qualified:
-    case Elaborated:
       break;
     }
     Cur = Cur.getNextTypeLoc();
@@ -313,9 +303,8 @@ bool TypeSpecTypeLoc::isKind(const TypeLoc &TL) {
 }
 
 bool TagTypeLoc::isDefinition() const {
-  TagDecl *D = getDecl();
-  return D->isCompleteDefinition() &&
-         (D->getIdentifier() == nullptr || D->getLocation() == getNameLoc());
+  return getTypePtr()->isTagOwned() &&
+         getOriginalDecl()->isCompleteDefinition();
 }
 
 // Reimplemented to account for GNU/C++ extension
@@ -482,6 +471,134 @@ TypeLoc TypeLoc::findExplicitQualifierLoc() const {
   return {};
 }
 
+NestedNameSpecifierLoc TypeLoc::getPrefix() const {
+  switch (getTypeLocClass()) {
+  case TypeLoc::DependentName:
+    return castAs<DependentNameTypeLoc>().getQualifierLoc();
+  case TypeLoc::TemplateSpecialization:
+    return castAs<TemplateSpecializationTypeLoc>().getQualifierLoc();
+  case TypeLoc::DependentTemplateSpecialization:
+    return castAs<DependentTemplateSpecializationTypeLoc>().getQualifierLoc();
+  case TypeLoc::DeducedTemplateSpecialization:
+    return castAs<DeducedTemplateSpecializationTypeLoc>().getQualifierLoc();
+  case TypeLoc::Enum:
+  case TypeLoc::Record:
+  case TypeLoc::InjectedClassName:
+    return castAs<TagTypeLoc>().getQualifierLoc();
+  case TypeLoc::Typedef:
+    return castAs<TypedefTypeLoc>().getQualifierLoc();
+  case TypeLoc::UnresolvedUsing:
+    return castAs<UnresolvedUsingTypeLoc>().getQualifierLoc();
+  case TypeLoc::Using:
+    return castAs<UsingTypeLoc>().getQualifierLoc();
+  default:
+    return NestedNameSpecifierLoc();
+  }
+}
+
+SourceLocation TypeLoc::getNonPrefixBeginLoc() const {
+  switch (getTypeLocClass()) {
+  case TypeLoc::TemplateSpecialization: {
+    auto TL = castAs<TemplateSpecializationTypeLoc>();
+    SourceLocation Loc = TL.getTemplateKeywordLoc();
+    if (!Loc.isValid())
+      Loc = TL.getTemplateNameLoc();
+    return Loc;
+  }
+  case TypeLoc::DependentTemplateSpecialization: {
+    auto TL = castAs<DependentTemplateSpecializationTypeLoc>();
+    SourceLocation Loc = TL.getTemplateKeywordLoc();
+    if (!Loc.isValid())
+      Loc = TL.getTemplateNameLoc();
+    return Loc;
+  }
+  case TypeLoc::DeducedTemplateSpecialization: {
+    auto TL = castAs<DeducedTemplateSpecializationTypeLoc>();
+    SourceLocation Loc = TL.getTemplateKeywordLoc();
+    if (!Loc.isValid())
+      Loc = TL.getTemplateNameLoc();
+    return Loc;
+  }
+  case TypeLoc::DependentName:
+    return castAs<DependentNameTypeLoc>().getNameLoc();
+  case TypeLoc::Enum:
+  case TypeLoc::Record:
+  case TypeLoc::InjectedClassName:
+    return castAs<TagTypeLoc>().getNameLoc();
+  case TypeLoc::Typedef:
+    return castAs<TypedefTypeLoc>().getNameLoc();
+  case TypeLoc::UnresolvedUsing:
+    return castAs<UnresolvedUsingTypeLoc>().getNameLoc();
+  case TypeLoc::Using:
+    return castAs<UsingTypeLoc>().getNameLoc();
+  default:
+    return getBeginLoc();
+  }
+}
+
+SourceLocation TypeLoc::getNonElaboratedBeginLoc() const {
+  // For elaborated types (e.g. `struct a::A`) we want the portion after the
+  // `struct` but including the namespace qualifier, `a::`.
+  switch (getTypeLocClass()) {
+  case TypeLoc::Qualified:
+    return castAs<QualifiedTypeLoc>()
+        .getUnqualifiedLoc()
+        .getNonElaboratedBeginLoc();
+  case TypeLoc::TemplateSpecialization: {
+    auto T = castAs<TemplateSpecializationTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = T.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return T.getTemplateNameLoc();
+  }
+  case TypeLoc::DependentTemplateSpecialization: {
+    auto T = castAs<DependentTemplateSpecializationTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = T.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return T.getTemplateNameLoc();
+  }
+  case TypeLoc::DeducedTemplateSpecialization: {
+    auto T = castAs<DeducedTemplateSpecializationTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = T.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return T.getTemplateNameLoc();
+  }
+  case TypeLoc::DependentName: {
+    auto T = castAs<DependentNameTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = T.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return T.getNameLoc();
+  }
+  case TypeLoc::Enum:
+  case TypeLoc::Record:
+  case TypeLoc::InjectedClassName: {
+    auto T = castAs<TagTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = T.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return T.getNameLoc();
+  }
+  case TypeLoc::Typedef: {
+    auto T = castAs<TypedefTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = T.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return T.getNameLoc();
+  }
+  case TypeLoc::UnresolvedUsing: {
+    auto T = castAs<UnresolvedUsingTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = T.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return T.getNameLoc();
+  }
+  case TypeLoc::Using: {
+    auto T = castAs<UsingTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = T.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return T.getNameLoc();
+  }
+  default:
+    return getBeginLoc();
+  }
+}
+
 void ObjCTypeParamTypeLoc::initializeLocal(ASTContext &Context,
                                            SourceLocation Loc) {
   setNameLoc(Loc);
@@ -555,9 +672,9 @@ static void initializeElaboratedKeyword(TL T, SourceLocation Loc) {
                                 : SourceLocation());
 }
 
-static NestedNameSpecifierLoc
-initializeQualifier(ASTContext &Context, NestedNameSpecifier *Qualifier,
-                    SourceLocation Loc) {
+static NestedNameSpecifierLoc initializeQualifier(ASTContext &Context,
+                                                  NestedNameSpecifier Qualifier,
+                                                  SourceLocation Loc) {
   if (!Qualifier)
     return NestedNameSpecifierLoc();
   NestedNameSpecifierLocBuilder Builder;
@@ -565,15 +682,6 @@ initializeQualifier(ASTContext &Context, NestedNameSpecifier *Qualifier,
   return Builder.getWithLocInContext(Context);
 }
 
-void ElaboratedTypeLoc::initializeLocal(ASTContext &Context,
-                                        SourceLocation Loc) {
-  if (isEmpty())
-    return;
-  initializeElaboratedKeyword(*this, Loc);
-  setQualifierLoc(
-      initializeQualifier(Context, getTypePtr()->getQualifier(), Loc));
-}
-
 void DependentNameTypeLoc::initializeLocal(ASTContext &Context,
                                            SourceLocation Loc) {
   initializeElaboratedKeyword(*this, Loc);
@@ -596,6 +704,78 @@ DependentTemplateSpecializationTypeLoc::initializeLocal(ASTContext &Context,
       Context, getTypePtr()->template_arguments(), getArgInfos(), Loc);
 }
 
+void TemplateSpecializationTypeLoc::set(SourceLocation ElaboratedKeywordLoc,
+                                        NestedNameSpecifierLoc QualifierLoc,
+                                        SourceLocation TemplateKeywordLoc,
+                                        SourceLocation NameLoc,
+                                        SourceLocation LAngleLoc,
+                                        SourceLocation RAngleLoc) {
+  TemplateSpecializationLocInfo &Data = *getLocalData();
+
+  Data.ElaboratedKWLoc = ElaboratedKeywordLoc;
+  SourceLocation BeginLoc = ElaboratedKeywordLoc;
+
+  getLocalData()->QualifierData = QualifierLoc.getOpaqueData();
+
+  assert(QualifierLoc.getNestedNameSpecifier() ==
+         getTypePtr()->getTemplateName().getQualifier());
+  Data.QualifierData = QualifierLoc ? QualifierLoc.getOpaqueData() : nullptr;
+  if (QualifierLoc && !BeginLoc.isValid())
+    BeginLoc = QualifierLoc.getBeginLoc();
+
+  Data.TemplateKWLoc = TemplateKeywordLoc;
+  if (!BeginLoc.isValid())
+    BeginLoc = TemplateKeywordLoc;
+
+  Data.NameLoc = NameLoc;
+  if (!BeginLoc.isValid())
+    BeginLoc = NameLoc;
+
+  Data.LAngleLoc = LAngleLoc;
+  Data.SR = SourceRange(BeginLoc, RAngleLoc);
+}
+
+void TemplateSpecializationTypeLoc::set(SourceLocation ElaboratedKeywordLoc,
+                                        NestedNameSpecifierLoc QualifierLoc,
+                                        SourceLocation TemplateKeywordLoc,
+                                        SourceLocation NameLoc,
+                                        const TemplateArgumentListInfo &TAL) {
+  set(ElaboratedKeywordLoc, QualifierLoc, TemplateKeywordLoc, NameLoc,
+      TAL.getLAngleLoc(), TAL.getRAngleLoc());
+  MutableArrayRef<TemplateArgumentLocInfo> ArgInfos = getArgLocInfos();
+  assert(TAL.size() == ArgInfos.size());
+  for (unsigned I = 0, N = TAL.size(); I != N; ++I)
+    ArgInfos[I] = TAL[I].getLocInfo();
+}
+
+void TemplateSpecializationTypeLoc::initializeLocal(ASTContext &Context,
+                                                    SourceLocation Loc) {
+  QualifiedTemplateName *Name =
+      getTypePtr()->getTemplateName().getAsAdjustedQualifiedTemplateName();
+
+  SourceLocation ElaboratedKeywordLoc =
+      getTypePtr()->getKeyword() != ElaboratedTypeKeyword::None
+          ? Loc
+          : SourceLocation();
+
+  NestedNameSpecifierLoc QualifierLoc;
+  if (NestedNameSpecifier Qualifier =
+          Name ? Name->getQualifier() : std::nullopt) {
+    NestedNameSpecifierLocBuilder Builder;
+    Builder.MakeTrivial(Context, Qualifier, Loc);
+    QualifierLoc = Builder.getWithLocInContext(Context);
+  }
+
+  TemplateArgumentListInfo TAL(Loc, Loc);
+  set(ElaboratedKeywordLoc, QualifierLoc,
+      /*TemplateKeywordLoc=*/Name && Name->hasTemplateKeyword()
+          ? Loc
+          : SourceLocation(),
+      /*NameLoc=*/Loc, /*LAngleLoc=*/Loc, /*RAngleLoc=*/Loc);
+  initializeArgLocs(Context, getTypePtr()->template_arguments(), getArgInfos(),
+                    Loc);
+}
+
 void TemplateSpecializationTypeLoc::initializeArgLocs(
     ASTContext &Context, ArrayRef<TemplateArgument> Args,
     TemplateArgumentLocInfo *ArgInfos, SourceLocation Loc) {
@@ -631,7 +811,7 @@ void TemplateSpecializationTypeLoc::initializeArgLocs(
         Builder.MakeTrivial(Context, QTN->getQualifier(), Loc);
 
       ArgInfos[i] = TemplateArgumentLocInfo(
-          Context, Builder.getWithLocInContext(Context), Loc,
+          Context, Loc, Builder.getWithLocInContext(Context), Loc,
           Args[i].getKind() == TemplateArgument::Template ? SourceLocation()
                                                           : Loc);
       break;
@@ -680,6 +860,14 @@ void AutoTypeLoc::initializeLocal(ASTContext &Context, SourceLocation Loc) {
   }
 }
 
+void DeducedTemplateSpecializationTypeLoc::initializeLocal(ASTContext &Context,
+                                                           SourceLocation Loc) {
+  initializeElaboratedKeyword(*this, Loc);
+  setQualifierLoc(initializeQualifier(
+      Context, getTypePtr()->getTemplateName().getQualifier(), Loc));
+  setTemplateNameLoc(Loc);
+}
+
 namespace {
 
   class GetContainedAutoTypeLocVisitor :
@@ -693,10 +881,6 @@ namespace {
 
     // Only these types can contain the desired 'auto' type.
 
-    TypeLoc VisitElaboratedTypeLoc(ElaboratedTypeLoc T) {
-      return Visit(T.getNamedTypeLoc());
-    }
-
     TypeLoc VisitQualifiedTypeLoc(QualifiedTypeLoc T) {
       return Visit(T.getUnqualifiedLoc());
     }
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index deb453fe6ee75..52ff0d5b5771b 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -133,7 +133,7 @@ class TypePrinter {
   void printAfter(QualType T, raw_ostream &OS);
   void AppendScope(DeclContext *DC, raw_ostream &OS,
                    DeclarationName NameInScope);
-  void printTag(TagDecl *T, raw_ostream &OS);
+  void printTagType(const TagType *T, raw_ostream &OS);
   void printFunctionAfter(const FunctionType::ExtInfo &Info, raw_ostream &OS);
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define TYPE(CLASS, PARENT)                                                    \
@@ -230,9 +230,9 @@ bool TypePrinter::canPrefixQualifiers(const Type *T,
     case Type::UnaryTransform:
     case Type::Record:
     case Type::Enum:
-    case Type::Elaborated:
     case Type::TemplateTypeParm:
     case Type::SubstTemplateTypeParmPack:
+    case Type::SubstBuiltinTemplatePack:
     case Type::DeducedTemplateSpecialization:
     case Type::TemplateSpecialization:
     case Type::InjectedClassName:
@@ -504,11 +504,7 @@ void TypePrinter::printMemberPointerBefore(const MemberPointerType *T,
   // FIXME: this should include vectors, but vectors use attributes I guess.
   if (isa<ArrayType>(T->getPointeeType()))
     OS << '(';
-
-  PrintingPolicy InnerPolicy(Policy);
-  InnerPolicy.IncludeTagDefinition = false;
-  T->getQualifier()->print(OS, InnerPolicy);
-
+  T->getQualifier().print(OS, Policy);
   OS << "*";
 }
 
@@ -1211,29 +1207,50 @@ void TypePrinter::printTypeSpec(NamedDecl *D, raw_ostream &OS) {
 
 void TypePrinter::printUnresolvedUsingBefore(const UnresolvedUsingType *T,
                                              raw_ostream &OS) {
-  printTypeSpec(T->getDecl(), OS);
+  OS << TypeWithKeyword::getKeywordName(T->getKeyword());
+  if (T->getKeyword() != ElaboratedTypeKeyword::None)
+    OS << ' ';
+  auto *D = T->getDecl();
+  if (Policy.FullyQualifiedName || T->isCanonicalUnqualified()) {
+    AppendScope(D->getDeclContext(), OS, D->getDeclName());
+  } else {
+    T->getQualifier().print(OS, Policy);
+  }
+  OS << D->getIdentifier()->getName();
+  spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printUnresolvedUsingAfter(const UnresolvedUsingType *T,
                                             raw_ostream &OS) {}
 
 void TypePrinter::printUsingBefore(const UsingType *T, raw_ostream &OS) {
-  // After `namespace b { using a::X }`, is the type X within B a::X or b::X?
-  //
-  // - b::X is more formally correct given the UsingType model
-  // - b::X makes sense if "re-exporting" a symbol in a new namespace
-  // - a::X makes sense if "importing" a symbol for convenience
-  //
-  // The "importing" use seems much more common, so we print a::X.
-  // This could be a policy option, but the right choice seems to rest more
-  // with the intent of the code than the caller.
-  printTypeSpec(T->getFoundDecl()->getUnderlyingDecl(), OS);
+  OS << TypeWithKeyword::getKeywordName(T->getKeyword());
+  if (T->getKeyword() != ElaboratedTypeKeyword::None)
+    OS << ' ';
+  auto *D = T->getDecl();
+  if (Policy.FullyQualifiedName) {
+    AppendScope(D->getDeclContext(), OS, D->getDeclName());
+  } else {
+    T->getQualifier().print(OS, Policy);
+  }
+  OS << D->getIdentifier()->getName();
+  spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printUsingAfter(const UsingType *T, raw_ostream &OS) {}
 
 void TypePrinter::printTypedefBefore(const TypedefType *T, raw_ostream &OS) {
-  printTypeSpec(T->getDecl(), OS);
+  OS << TypeWithKeyword::getKeywordName(T->getKeyword());
+  if (T->getKeyword() != ElaboratedTypeKeyword::None)
+    OS << ' ';
+  auto *D = T->getDecl();
+  if (Policy.FullyQualifiedName) {
+    AppendScope(D->getDeclContext(), OS, D->getDeclName());
+  } else {
+    T->getQualifier().print(OS, Policy);
+  }
+  OS << D->getIdentifier()->getName();
+  spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printMacroQualifiedBefore(const MacroQualifiedType *T,
@@ -1354,14 +1371,53 @@ void TypePrinter::printAutoAfter(const AutoType *T, raw_ostream &OS) {
 
 void TypePrinter::printDeducedTemplateSpecializationBefore(
     const DeducedTemplateSpecializationType *T, raw_ostream &OS) {
-  // If the type has been deduced, print the deduced type.
+  if (ElaboratedTypeKeyword Keyword = T->getKeyword();
+      T->getKeyword() != ElaboratedTypeKeyword::None)
+    OS << KeywordHelpers::getKeywordName(Keyword) << ' ';
+
+  TemplateName Name = T->getTemplateName();
+
+  // If the type has been deduced, print the template arguments, as if this was
+  // printing the deduced type, but including elaboration and template name
+  // qualification.
+  // FIXME: There should probably be a policy which controls this.
+  // We would probably want to do this on diagnostics, but not on -ast-print.
+  ArrayRef<TemplateArgument> Args;
+  TemplateDecl *DeducedTD = nullptr;
   if (!T->getDeducedType().isNull()) {
-    printBefore(T->getDeducedType(), OS);
-  } else {
+    if (const auto *TST =
+            dyn_cast<TemplateSpecializationType>(T->getDeducedType())) {
+      DeducedTD = TST->getTemplateName().getAsTemplateDecl(
+          /*IgnoreDeduced=*/true);
+      Args = TST->template_arguments();
+    } else {
+      // Should only get here for canonical types.
+      const auto *CD = cast<ClassTemplateSpecializationDecl>(
+          cast<RecordType>(T->getDeducedType())->getOriginalDecl());
+      DeducedTD = CD->getSpecializedTemplate();
+      Args = CD->getTemplateArgs().asArray();
+    }
+
+    // FIXME: Workaround for alias template CTAD not producing guides which
+    // include the alias template specialization type.
+    // Purposefully disregard qualification when building this TemplateName;
+    // any qualification we might have, might not make sense in the
+    // context this was deduced.
+    if (!declaresSameEntity(DeducedTD, Name.getAsTemplateDecl(
+                                           /*IgnoreDeduced=*/true)))
+      Name = TemplateName(DeducedTD);
+  }
+
+  {
     IncludeStrongLifetimeRAII Strong(Policy);
-    T->getTemplateName().print(OS, Policy);
-    spaceBeforePlaceHolder(OS);
+    Name.print(OS, Policy);
   }
+  if (DeducedTD) {
+    printTemplateArgumentList(OS, Args, Policy,
+                              DeducedTD->getTemplateParameters());
+  }
+
+  spaceBeforePlaceHolder(OS);
 }
 
 void TypePrinter::printDeducedTemplateSpecializationAfter(
@@ -1480,30 +1536,37 @@ void TypePrinter::AppendScope(DeclContext *DC, raw_ostream &OS,
   }
 }
 
-void TypePrinter::printTag(TagDecl *D, raw_ostream &OS) {
-  if (Policy.IncludeTagDefinition) {
-    PrintingPolicy SubPolicy = Policy;
-    SubPolicy.IncludeTagDefinition = false;
-    D->print(OS, SubPolicy, Indentation);
+void TypePrinter::printTagType(const TagType *T, raw_ostream &OS) {
+  TagDecl *D = T->getOriginalDecl();
+
+  if (Policy.IncludeTagDefinition && T->isTagOwned()) {
+    D->print(OS, Policy, Indentation);
     spaceBeforePlaceHolder(OS);
     return;
   }
 
   bool HasKindDecoration = false;
 
-  // We don't print tags unless this is an elaborated type.
-  // In C, we just assume every RecordType is an elaborated type.
-  if (!Policy.SuppressTagKeyword && !D->getTypedefNameForAnonDecl()) {
-    HasKindDecoration = true;
-    OS << D->getKindName();
-    OS << ' ';
+  if (T->isCanonicalUnqualified()) {
+    if (!Policy.SuppressTagKeyword && !D->getTypedefNameForAnonDecl()) {
+      HasKindDecoration = true;
+      OS << D->getKindName();
+      OS << ' ';
+    }
+  } else {
+    OS << TypeWithKeyword::getKeywordName(T->getKeyword());
+    if (T->getKeyword() != ElaboratedTypeKeyword::None)
+      OS << ' ';
   }
 
-  // Compute the full nested-name-specifier for this type.
-  // In C, this will always be empty except when the type
-  // being printed is anonymous within other Record.
-  if (!Policy.SuppressScope)
+  if (!Policy.FullyQualifiedName && !T->isCanonicalUnqualified()) {
+    T->getQualifier().print(OS, Policy);
+  } else if (!Policy.SuppressScope) {
+    // Compute the full nested-name-specifier for this type.
+    // In C, this will always be empty except when the type
+    // being printed is anonymous within other Record.
     AppendScope(D->getDeclContext(), OS, D->getDeclName());
+  }
 
   if (const IdentifierInfo *II = D->getIdentifier())
     OS << II->getName();
@@ -1578,9 +1641,11 @@ void TypePrinter::printTag(TagDecl *D, raw_ostream &OS) {
 void TypePrinter::printRecordBefore(const RecordType *T, raw_ostream &OS) {
   // Print the preferred name if we have one for this type.
   if (Policy.UsePreferredNames) {
-    for (const auto *PNA : T->getDecl()->specific_attrs<PreferredNameAttr>()) {
+    for (const auto *PNA : T->getOriginalDecl()
+                               ->getMostRecentDecl()
+                               ->specific_attrs<PreferredNameAttr>()) {
       if (!declaresSameEntity(PNA->getTypedefType()->getAsCXXRecordDecl(),
-                              T->getDecl()))
+                              T->getOriginalDecl()))
         continue;
       // Find the outermost typedef or alias template.
       QualType T = PNA->getTypedefType();
@@ -1594,17 +1659,44 @@ void TypePrinter::printRecordBefore(const RecordType *T, raw_ostream &OS) {
     }
   }
 
-  printTag(T->getDecl(), OS);
+  printTagType(T, OS);
 }
 
 void TypePrinter::printRecordAfter(const RecordType *T, raw_ostream &OS) {}
 
 void TypePrinter::printEnumBefore(const EnumType *T, raw_ostream &OS) {
-  printTag(T->getDecl(), OS);
+  printTagType(T, OS);
 }
 
 void TypePrinter::printEnumAfter(const EnumType *T, raw_ostream &OS) {}
 
+void TypePrinter::printInjectedClassNameBefore(const InjectedClassNameType *T,
+                                               raw_ostream &OS) {
+  const ASTContext &Ctx = T->getOriginalDecl()->getASTContext();
+  IncludeStrongLifetimeRAII Strong(Policy);
+  T->getTemplateName(Ctx).print(OS, Policy);
+  if (Policy.PrintInjectedClassNameWithArguments) {
+    auto *Decl = T->getOriginalDecl();
+    // FIXME: Use T->getTemplateArgs(Ctx) when that supports as-written
+    // arguments.
+    if (auto *RD = dyn_cast<ClassTemplateSpecializationDecl>(Decl)) {
+      printTemplateArgumentList(OS, RD->getTemplateArgsAsWritten()->arguments(),
+                                Policy,
+                                T->getTemplateDecl()->getTemplateParameters());
+    } else {
+      ClassTemplateDecl *TD = Decl->getDescribedClassTemplate();
+      assert(TD);
+      printTemplateArgumentList(
+          OS, TD->getTemplateParameters()->getInjectedTemplateArgs(Ctx), Policy,
+          T->getTemplateDecl()->getTemplateParameters());
+    }
+  }
+  spaceBeforePlaceHolder(OS);
+}
+
+void TypePrinter::printInjectedClassNameAfter(const InjectedClassNameType *T,
+                                              raw_ostream &OS) {}
+
 void TypePrinter::printTemplateTypeParmBefore(const TemplateTypeParmType *T,
                                               raw_ostream &OS) {
   TemplateTypeParmDecl *D = T->getDecl();
@@ -1640,6 +1732,15 @@ void TypePrinter::printSubstTemplateTypeParmAfter(
   printAfter(T->getReplacementType(), OS);
 }
 
+void TypePrinter::printSubstBuiltinTemplatePackBefore(
+    const SubstBuiltinTemplatePackType *T, raw_ostream &OS) {
+  IncludeStrongLifetimeRAII Strong(Policy);
+  OS << "type-pack";
+}
+
+void TypePrinter::printSubstBuiltinTemplatePackAfter(
+    const SubstBuiltinTemplatePackType *T, raw_ostream &OS) {}
+
 void TypePrinter::printSubstTemplateTypeParmPackBefore(
                                         const SubstTemplateTypeParmPackType *T,
                                         raw_ostream &OS) {
@@ -1671,6 +1772,10 @@ void TypePrinter::printTemplateId(const TemplateSpecializationType *T,
                                   raw_ostream &OS, bool FullyQualify) {
   IncludeStrongLifetimeRAII Strong(Policy);
 
+  if (ElaboratedTypeKeyword K = T->getKeyword();
+      K != ElaboratedTypeKeyword::None)
+    OS << TypeWithKeyword::getKeywordName(K) << ' ';
+
   TemplateDecl *TD =
       T->getTemplateName().getAsTemplateDecl(/*IgnoreDeduced=*/true);
   // FIXME: Null TD never exercised in test suite.
@@ -1680,7 +1785,10 @@ void TypePrinter::printTemplateId(const TemplateSpecializationType *T,
 
     OS << TD->getName();
   } else {
-    T->getTemplateName().print(OS, Policy, TemplateName::Qualified::None);
+    T->getTemplateName().print(OS, Policy,
+                               !Policy.SuppressScope
+                                   ? TemplateName::Qualified::AsWritten
+                                   : TemplateName::Qualified::None);
   }
 
   DefaultTemplateArgsPolicyRAII TemplateArgs(Policy);
@@ -1699,77 +1807,6 @@ void TypePrinter::printTemplateSpecializationAfter(
                                             const TemplateSpecializationType *T,
                                             raw_ostream &OS) {}
 
-void TypePrinter::printInjectedClassNameBefore(const InjectedClassNameType *T,
-                                               raw_ostream &OS) {
-  if (Policy.PrintInjectedClassNameWithArguments)
-    return printTemplateSpecializationBefore(T->getInjectedTST(), OS);
-
-  IncludeStrongLifetimeRAII Strong(Policy);
-  T->getTemplateName().print(OS, Policy);
-  spaceBeforePlaceHolder(OS);
-}
-
-void TypePrinter::printInjectedClassNameAfter(const InjectedClassNameType *T,
-                                               raw_ostream &OS) {}
-
-void TypePrinter::printElaboratedBefore(const ElaboratedType *T,
-                                        raw_ostream &OS) {
-  if (Policy.IncludeTagDefinition && T->getOwnedTagDecl()) {
-    TagDecl *OwnedTagDecl = T->getOwnedTagDecl();
-    assert(OwnedTagDecl->getTypeForDecl() == T->getNamedType().getTypePtr() &&
-           "OwnedTagDecl expected to be a declaration for the type");
-    PrintingPolicy SubPolicy = Policy;
-    SubPolicy.IncludeTagDefinition = false;
-    OwnedTagDecl->print(OS, SubPolicy, Indentation);
-    spaceBeforePlaceHolder(OS);
-    return;
-  }
-
-  if (Policy.SuppressElaboration) {
-    printBefore(T->getNamedType(), OS);
-    return;
-  }
-
-  // The tag definition will take care of these.
-  if (!Policy.IncludeTagDefinition)
-  {
-    OS << TypeWithKeyword::getKeywordName(T->getKeyword());
-    if (T->getKeyword() != ElaboratedTypeKeyword::None)
-      OS << " ";
-    NestedNameSpecifier *Qualifier = T->getQualifier();
-    if (!Policy.SuppressTagKeyword && Policy.SuppressScope &&
-        !Policy.SuppressUnwrittenScope) {
-      bool OldTagKeyword = Policy.SuppressTagKeyword;
-      bool OldSupressScope = Policy.SuppressScope;
-      Policy.SuppressTagKeyword = true;
-      Policy.SuppressScope = false;
-      printBefore(T->getNamedType(), OS);
-      Policy.SuppressTagKeyword = OldTagKeyword;
-      Policy.SuppressScope = OldSupressScope;
-      return;
-    }
-    if (Qualifier)
-      Qualifier->print(OS, Policy);
-  }
-
-  ElaboratedTypePolicyRAII PolicyRAII(Policy);
-  printBefore(T->getNamedType(), OS);
-}
-
-void TypePrinter::printElaboratedAfter(const ElaboratedType *T,
-                                        raw_ostream &OS) {
-  if (Policy.IncludeTagDefinition && T->getOwnedTagDecl())
-    return;
-
-  if (Policy.SuppressElaboration) {
-    printAfter(T->getNamedType(), OS);
-    return;
-  }
-
-  ElaboratedTypePolicyRAII PolicyRAII(Policy);
-  printAfter(T->getNamedType(), OS);
-}
-
 void TypePrinter::printParenBefore(const ParenType *T, raw_ostream &OS) {
   if (!HasEmptyPlaceHolder && !isa<FunctionType>(T->getInnerType())) {
     printBefore(T->getInnerType(), OS);
@@ -1791,9 +1828,7 @@ void TypePrinter::printDependentNameBefore(const DependentNameType *T,
   OS << TypeWithKeyword::getKeywordName(T->getKeyword());
   if (T->getKeyword() != ElaboratedTypeKeyword::None)
     OS << " ";
-
-  T->getQualifier()->print(OS, Policy);
-
+  T->getQualifier().print(OS, Policy);
   OS << T->getIdentifier()->getName();
   spaceBeforePlaceHolder(OS);
 }
@@ -2129,6 +2164,9 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
   case attr::ExtVectorType:
     OS << "ext_vector_type";
     break;
+  case attr::CFISalt:
+    OS << "cfi_salt(\"" << cast<CFISaltAttr>(T->getAttr())->getSalt() << "\")";
+    break;
   }
   OS << "))";
 }
diff --git a/clang/lib/AST/VTTBuilder.cpp b/clang/lib/AST/VTTBuilder.cpp
index de011848a721e..85101aee97e66 100644
--- a/clang/lib/AST/VTTBuilder.cpp
+++ b/clang/lib/AST/VTTBuilder.cpp
@@ -64,7 +64,9 @@ void VTTBuilder::LayoutSecondaryVTTs(BaseSubobject Base) {
         continue;
 
     const auto *BaseDecl =
-        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(
+            I.getType()->castAs<RecordType>()->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     const ASTRecordLayout &Layout = Ctx.getASTRecordLayout(RD);
     CharUnits BaseOffset = Base.getBaseOffset() +
@@ -90,7 +92,9 @@ VTTBuilder::LayoutSecondaryVirtualPointers(BaseSubobject Base,
 
   for (const auto &I : RD->bases()) {
     const auto *BaseDecl =
-        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(
+            I.getType()->castAs<RecordType>()->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     // Itanium C++ ABI 2.6.2:
     //   Secondary virtual pointers are present for all bases with either
@@ -154,7 +158,9 @@ void VTTBuilder::LayoutVirtualVTTs(const CXXRecordDecl *RD,
                                    VisitedVirtualBasesSetTy &VBases) {
   for (const auto &I : RD->bases()) {
     const auto *BaseDecl =
-        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(
+            I.getType()->castAs<RecordType>()->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     // Check if this is a virtual base.
     if (I.isVirtual()) {
diff --git a/clang/lib/AST/VTableBuilder.cpp b/clang/lib/AST/VTableBuilder.cpp
index 0001745a6ff22..6cec526ba8443 100644
--- a/clang/lib/AST/VTableBuilder.cpp
+++ b/clang/lib/AST/VTableBuilder.cpp
@@ -313,10 +313,12 @@ ComputeReturnAdjustmentBaseOffset(ASTContext &Context,
   }
 
   const CXXRecordDecl *DerivedRD =
-    cast<CXXRecordDecl>(cast<RecordType>(CanDerivedReturnType)->getDecl());
+      cast<CXXRecordDecl>(
+          cast<RecordType>(CanDerivedReturnType)->getOriginalDecl())
+          ->getDefinitionOrSelf();
 
-  const CXXRecordDecl *BaseRD =
-    cast<CXXRecordDecl>(cast<RecordType>(CanBaseReturnType)->getDecl());
+  const CXXRecordDecl *BaseRD = cast<CXXRecordDecl>(
+      cast<RecordType>(CanBaseReturnType)->getOriginalDecl());
 
   return ComputeBaseOffset(Context, BaseRD, DerivedRD);
 }
diff --git a/clang/lib/ASTMatchers/ASTMatchFinder.cpp b/clang/lib/ASTMatchers/ASTMatchFinder.cpp
index 6d0ba0b7907a1..e8a0004c2e187 100644
--- a/clang/lib/ASTMatchers/ASTMatchFinder.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchFinder.cpp
@@ -122,15 +122,15 @@ class MatchChildASTVisitor
     else if (const Stmt *S = DynNode.get<Stmt>())
       traverse(*S);
     else if (const NestedNameSpecifier *NNS =
-             DynNode.get<NestedNameSpecifier>())
+                 DynNode.get<NestedNameSpecifier>())
       traverse(*NNS);
     else if (const NestedNameSpecifierLoc *NNSLoc =
              DynNode.get<NestedNameSpecifierLoc>())
       traverse(*NNSLoc);
     else if (const QualType *Q = DynNode.get<QualType>())
-      traverse(*Q);
+      traverse(*Q, /*TraverseQualifier=*/true);
     else if (const TypeLoc *T = DynNode.get<TypeLoc>())
-      traverse(*T);
+      traverse(*T, /*TraverseQualifier=*/true);
     else if (const auto *C = DynNode.get<CXXCtorInitializer>())
       traverse(*C);
     else if (const TemplateArgumentLoc *TALoc =
@@ -194,7 +194,7 @@ class MatchChildASTVisitor
   }
   // We assume that the QualType and the contained type are on the same
   // hierarchy level. Thus, we try to match either of them.
-  bool TraverseType(QualType TypeNode) {
+  bool TraverseType(QualType TypeNode, bool TraverseQualifier = true) {
     if (TypeNode.isNull())
       return true;
     ScopedIncrement ScopedDepth(&CurrentDepth);
@@ -202,11 +202,11 @@ class MatchChildASTVisitor
     if (!match(*TypeNode))
       return false;
     // The QualType is matched inside traverse.
-    return traverse(TypeNode);
+    return traverse(TypeNode, TraverseQualifier);
   }
   // We assume that the TypeLoc, contained QualType and contained Type all are
   // on the same hierarchy level. Thus, we try to match all of them.
-  bool TraverseTypeLoc(TypeLoc TypeLocNode) {
+  bool TraverseTypeLoc(TypeLoc TypeLocNode, bool TraverseQualifier = true) {
     if (TypeLocNode.isNull())
       return true;
     ScopedIncrement ScopedDepth(&CurrentDepth);
@@ -217,17 +217,17 @@ class MatchChildASTVisitor
     if (!match(TypeLocNode.getType()))
       return false;
     // The TypeLoc is matched inside traverse.
-    return traverse(TypeLocNode);
+    return traverse(TypeLocNode, TraverseQualifier);
   }
-  bool TraverseNestedNameSpecifier(NestedNameSpecifier *NNS) {
+  bool TraverseNestedNameSpecifier(NestedNameSpecifier NNS) {
     ScopedIncrement ScopedDepth(&CurrentDepth);
-    return (NNS == nullptr) || traverse(*NNS);
+    return !NNS || traverse(NNS);
   }
   bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS) {
     if (!NNS)
       return true;
     ScopedIncrement ScopedDepth(&CurrentDepth);
-    if (!match(*NNS.getNestedNameSpecifier()))
+    if (!match(NNS.getNestedNameSpecifier()))
       return false;
     return traverse(NNS);
   }
@@ -340,15 +340,14 @@ class MatchChildASTVisitor
   bool baseTraverse(const Stmt &StmtNode) {
     return VisitorBase::TraverseStmt(const_cast<Stmt*>(&StmtNode));
   }
-  bool baseTraverse(QualType TypeNode) {
-    return VisitorBase::TraverseType(TypeNode);
+  bool baseTraverse(QualType TypeNode, bool TraverseQualifier) {
+    return VisitorBase::TraverseType(TypeNode, TraverseQualifier);
   }
-  bool baseTraverse(TypeLoc TypeLocNode) {
-    return VisitorBase::TraverseTypeLoc(TypeLocNode);
+  bool baseTraverse(TypeLoc TypeLocNode, bool TraverseQualifier) {
+    return VisitorBase::TraverseTypeLoc(TypeLocNode, TraverseQualifier);
   }
-  bool baseTraverse(const NestedNameSpecifier &NNS) {
-    return VisitorBase::TraverseNestedNameSpecifier(
-        const_cast<NestedNameSpecifier*>(&NNS));
+  bool baseTraverse(NestedNameSpecifier NNS) {
+    return VisitorBase::TraverseNestedNameSpecifier(NNS);
   }
   bool baseTraverse(NestedNameSpecifierLoc NNS) {
     return VisitorBase::TraverseNestedNameSpecifierLoc(NNS);
@@ -396,13 +395,13 @@ class MatchChildASTVisitor
 
   // Traverses the subtree rooted at 'Node'; returns true if the
   // traversal should continue after this function returns.
-  template <typename T>
-  bool traverse(const T &Node) {
+  template <typename T, class... Args>
+  bool traverse(const T &Node, Args &&...args) {
     static_assert(IsBaseType<T>::value,
                   "traverse can only be instantiated with base type");
     if (!match(Node))
       return false;
-    return baseTraverse(Node);
+    return baseTraverse(Node, std::forward<Args>(args)...);
   }
 
   const DynTypedMatcher *const Matcher;
@@ -501,9 +500,9 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
 
   bool TraverseDecl(Decl *DeclNode);
   bool TraverseStmt(Stmt *StmtNode, DataRecursionQueue *Queue = nullptr);
-  bool TraverseType(QualType TypeNode);
-  bool TraverseTypeLoc(TypeLoc TypeNode);
-  bool TraverseNestedNameSpecifier(NestedNameSpecifier *NNS);
+  bool TraverseType(QualType TypeNode, bool TraverseQualifier = true);
+  bool TraverseTypeLoc(TypeLoc TypeNode, bool TraverseQualifier = true);
+  bool TraverseNestedNameSpecifier(NestedNameSpecifier NNS);
   bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS);
   bool TraverseConstructorInitializer(CXXCtorInitializer *CtorInit);
   bool TraverseTemplateArgumentLoc(TemplateArgumentLoc TAL);
@@ -577,13 +576,13 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
 
         const auto *T = Proto.getTypePtr();
         for (const auto &E : T->exceptions())
-          TraverseType(E);
+          TraverseType(E, /*TraverseQualifier=*/true);
 
         if (Expr *NE = T->getNoexceptExpr())
           TraverseStmt(NE, Queue);
 
         if (LE->hasExplicitResultType())
-          TraverseTypeLoc(Proto.getReturnLoc());
+          TraverseTypeLoc(Proto.getReturnLoc(), /*TraverseQualifier=*/true);
         TraverseStmt(
             const_cast<Expr *>(LE->getTrailingRequiresClause().ConstraintExpr));
       }
@@ -1289,33 +1288,42 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
     if (Aliases == TypeAliases.end())
       return false;
 
-    if (const auto *ElaboratedTypeNode =
-            llvm::dyn_cast<ElaboratedType>(TypeNode)) {
-      if (ElaboratedTypeNode->isSugared() && Aliases->second.size() > 1) {
-        const auto &DesugaredTypeName =
-            ElaboratedTypeNode->desugar().getAsString();
+    auto matches = [&](const TypedefNameDecl *Alias) {
+      BoundNodesTreeBuilder Result(*Builder);
+      if (Matcher.matches(*Alias, this, &Result)) {
+        *Builder = std::move(Result);
+        return true;
+      }
+      return false;
+    };
 
-        for (const TypedefNameDecl *Alias : Aliases->second) {
-          if (Alias->getName() != DesugaredTypeName) {
-            continue;
-          }
+    if (const auto *T = TypeNode->getAs<TypedefType>()) {
+      const auto *TD = T->getDecl()->getCanonicalDecl();
 
-          BoundNodesTreeBuilder Result(*Builder);
-          if (Matcher.matches(*Alias, this, &Result)) {
-            *Builder = std::move(Result);
-            return true;
-          }
+      // Prioritize exact matches.
+      SmallVector<const TypedefNameDecl *, 8> NonExactMatches;
+      for (const TypedefNameDecl *Alias : Aliases->second) {
+        if (!declaresSameEntity(TD, Alias)) {
+          NonExactMatches.push_back(Alias);
+          continue;
         }
+        if (matches(Alias))
+          return true;
       }
-    }
 
-    for (const TypedefNameDecl *Alias : Aliases->second) {
-      BoundNodesTreeBuilder Result(*Builder);
-      if (Matcher.matches(*Alias, this, &Result)) {
-        *Builder = std::move(Result);
-        return true;
+      for (const TypedefNameDecl *Alias : NonExactMatches) {
+        BoundNodesTreeBuilder Result(*Builder);
+        if (Matcher.matches(*Alias, this, &Result)) {
+          *Builder = std::move(Result);
+          return true;
+        }
       }
+      return false;
     }
+
+    for (const TypedefNameDecl *Alias : Aliases->second)
+      if (matches(Alias))
+        return true;
     return false;
   }
 
@@ -1336,6 +1344,41 @@ class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
     return false;
   }
 
+  template <typename T> static SourceLocation getNodeLocation(const T &Node) {
+    return Node.getBeginLoc();
+  }
+
+  static SourceLocation getNodeLocation(const CXXCtorInitializer &Node) {
+    return Node.getSourceLocation();
+  }
+
+  static SourceLocation getNodeLocation(const TemplateArgumentLoc &Node) {
+    return Node.getLocation();
+  }
+
+  static SourceLocation getNodeLocation(const Attr &Node) {
+    return Node.getLocation();
+  }
+
+  bool isInSystemHeader(SourceLocation Loc) {
+    const SourceManager &SM = getASTContext().getSourceManager();
+    return SM.isInSystemHeader(Loc);
+  }
+
+  template <typename T> bool shouldSkipNode(T &Node) {
+    if (Options.IgnoreSystemHeaders && isInSystemHeader(getNodeLocation(Node)))
+      return true;
+    return false;
+  }
+
+  template <typename T> bool shouldSkipNode(T *Node) {
+    return (Node == nullptr) || shouldSkipNode(*Node);
+  }
+
+  bool shouldSkipNode(QualType &) { return false; }
+
+  bool shouldSkipNode(NestedNameSpecifier &) { return false; }
+
   /// Bucket to record map.
   ///
   /// Used to get the appropriate bucket for each matcher.
@@ -1465,9 +1508,8 @@ bool MatchASTVisitor::objcClassIsDerivedFrom(
 }
 
 bool MatchASTVisitor::TraverseDecl(Decl *DeclNode) {
-  if (!DeclNode) {
+  if (shouldSkipNode(DeclNode))
     return true;
-  }
 
   bool ScopedTraversal =
       TraversingASTNodeNotSpelledInSource || DeclNode->isImplicit();
@@ -1495,9 +1537,9 @@ bool MatchASTVisitor::TraverseDecl(Decl *DeclNode) {
 }
 
 bool MatchASTVisitor::TraverseStmt(Stmt *StmtNode, DataRecursionQueue *Queue) {
-  if (!StmtNode) {
+  if (shouldSkipNode(StmtNode))
     return true;
-  }
+
   bool ScopedTraversal = TraversingASTNodeNotSpelledInSource ||
                          TraversingASTChildrenNotSpelledInSource;
 
@@ -1506,12 +1548,19 @@ bool MatchASTVisitor::TraverseStmt(Stmt *StmtNode, DataRecursionQueue *Queue) {
   return RecursiveASTVisitor<MatchASTVisitor>::TraverseStmt(StmtNode, Queue);
 }
 
-bool MatchASTVisitor::TraverseType(QualType TypeNode) {
+bool MatchASTVisitor::TraverseType(QualType TypeNode, bool TraverseQualifier) {
+  if (shouldSkipNode(TypeNode))
+    return true;
+
   match(TypeNode);
-  return RecursiveASTVisitor<MatchASTVisitor>::TraverseType(TypeNode);
+  return RecursiveASTVisitor<MatchASTVisitor>::TraverseType(TypeNode,
+                                                            TraverseQualifier);
 }
 
-bool MatchASTVisitor::TraverseTypeLoc(TypeLoc TypeLocNode) {
+bool MatchASTVisitor::TraverseTypeLoc(TypeLoc TypeLocNode,
+                                      bool TraverseQualifier) {
+  if (shouldSkipNode(TypeLocNode))
+    return true;
   // The RecursiveASTVisitor only visits types if they're not within TypeLocs.
   // We still want to find those types via matchers, so we match them here. Note
   // that the TypeLocs are structurally a shadow-hierarchy to the expressed
@@ -1519,11 +1568,15 @@ bool MatchASTVisitor::TraverseTypeLoc(TypeLoc TypeLocNode) {
   // each TypeLoc.
   match(TypeLocNode);
   match(TypeLocNode.getType());
-  return RecursiveASTVisitor<MatchASTVisitor>::TraverseTypeLoc(TypeLocNode);
+  return RecursiveASTVisitor<MatchASTVisitor>::TraverseTypeLoc(
+      TypeLocNode, TraverseQualifier);
 }
 
-bool MatchASTVisitor::TraverseNestedNameSpecifier(NestedNameSpecifier *NNS) {
-  match(*NNS);
+bool MatchASTVisitor::TraverseNestedNameSpecifier(NestedNameSpecifier NNS) {
+  if (shouldSkipNode(NNS))
+    return true;
+
+  match(NNS);
   return RecursiveASTVisitor<MatchASTVisitor>::TraverseNestedNameSpecifier(NNS);
 }
 
@@ -1532,19 +1585,22 @@ bool MatchASTVisitor::TraverseNestedNameSpecifierLoc(
   if (!NNS)
     return true;
 
+  if (shouldSkipNode(NNS))
+    return true;
+
   match(NNS);
 
   // We only match the nested name specifier here (as opposed to traversing it)
   // because the traversal is already done in the parallel "Loc"-hierarchy.
   if (NNS.hasQualifier())
-    match(*NNS.getNestedNameSpecifier());
+    match(NNS.getNestedNameSpecifier());
   return
       RecursiveASTVisitor<MatchASTVisitor>::TraverseNestedNameSpecifierLoc(NNS);
 }
 
 bool MatchASTVisitor::TraverseConstructorInitializer(
     CXXCtorInitializer *CtorInit) {
-  if (!CtorInit)
+  if (shouldSkipNode(CtorInit))
     return true;
 
   bool ScopedTraversal = TraversingASTNodeNotSpelledInSource ||
@@ -1562,11 +1618,17 @@ bool MatchASTVisitor::TraverseConstructorInitializer(
 }
 
 bool MatchASTVisitor::TraverseTemplateArgumentLoc(TemplateArgumentLoc Loc) {
+  if (shouldSkipNode(Loc))
+    return true;
+
   match(Loc);
   return RecursiveASTVisitor<MatchASTVisitor>::TraverseTemplateArgumentLoc(Loc);
 }
 
 bool MatchASTVisitor::TraverseAttr(Attr *AttrNode) {
+  if (shouldSkipNode(AttrNode))
+    return true;
+
   match(*AttrNode);
   return RecursiveASTVisitor<MatchASTVisitor>::TraverseAttr(AttrNode);
 }
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 80dc888811657..653b3810cb68b 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -755,6 +755,8 @@ const internal::VariadicDynCastAllOfMatcher<Decl, TypedefDecl> typedefDecl;
 const internal::VariadicDynCastAllOfMatcher<Decl, TypedefNameDecl>
     typedefNameDecl;
 const internal::VariadicDynCastAllOfMatcher<Decl, TypeAliasDecl> typeAliasDecl;
+const internal::VariadicDynCastAllOfMatcher<Decl, UsingShadowDecl>
+    usingShadowDecl;
 const internal::VariadicDynCastAllOfMatcher<Decl, TypeAliasTemplateDecl>
     typeAliasTemplateDecl;
 const internal::VariadicAllOfMatcher<Decl> decl;
@@ -808,8 +810,6 @@ const internal::VariadicDynCastAllOfMatcher<TypeLoc, ReferenceTypeLoc>
 const internal::VariadicDynCastAllOfMatcher<TypeLoc,
                                             TemplateSpecializationTypeLoc>
     templateSpecializationTypeLoc;
-const internal::VariadicDynCastAllOfMatcher<TypeLoc, ElaboratedTypeLoc>
-    elaboratedTypeLoc;
 
 const internal::VariadicDynCastAllOfMatcher<Stmt, UnaryExprOrTypeTraitExpr>
     unaryExprOrTypeTraitExpr;
@@ -1103,7 +1103,6 @@ const AstTypeMatcher<TemplateSpecializationType> templateSpecializationType;
 const AstTypeMatcher<UnaryTransformType> unaryTransformType;
 const AstTypeMatcher<RecordType> recordType;
 const AstTypeMatcher<TagType> tagType;
-const AstTypeMatcher<ElaboratedType> elaboratedType;
 const AstTypeMatcher<UsingType> usingType;
 const AstTypeMatcher<SubstTemplateTypeParmType> substTemplateTypeParmType;
 const AstTypeMatcher<TemplateTypeParmType> templateTypeParmType;
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index 562df715e08ae..48a7b91969aef 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -235,13 +235,12 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(designatorCountIs);
   REGISTER_MATCHER(doStmt);
   REGISTER_MATCHER(eachOf);
-  REGISTER_MATCHER(elaboratedType);
-  REGISTER_MATCHER(elaboratedTypeLoc);
   REGISTER_MATCHER(usingType);
   REGISTER_MATCHER(enumConstantDecl);
   REGISTER_MATCHER(enumDecl);
   REGISTER_MATCHER(enumType);
   REGISTER_MATCHER(equalsBoundNode);
+  REGISTER_MATCHER(declaresSameEntityAsBoundNode);
   REGISTER_MATCHER(equalsIntegralValue);
   REGISTER_MATCHER(explicitCastExpr);
   REGISTER_MATCHER(exportDecl);
@@ -341,7 +340,6 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(hasMemberName);
   REGISTER_MATCHER(hasMethod);
   REGISTER_MATCHER(hasName);
-  REGISTER_MATCHER(hasNamedTypeLoc);
   REGISTER_MATCHER(hasNullSelector);
   REGISTER_MATCHER(hasObjectExpression);
   REGISTER_MATCHER(hasOperands);
@@ -503,7 +501,6 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(memberHasSameNameAsBoundNode);
   REGISTER_MATCHER(memberPointerType);
   REGISTER_MATCHER(namedDecl);
-  REGISTER_MATCHER(namesType);
   REGISTER_MATCHER(namespaceAliasDecl);
   REGISTER_MATCHER(namespaceDecl);
   REGISTER_MATCHER(nestedNameSpecifier);
@@ -593,6 +590,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(typeLoc);
   REGISTER_MATCHER(typedefDecl);
   REGISTER_MATCHER(typedefNameDecl);
+  REGISTER_MATCHER(usingShadowDecl);
   REGISTER_MATCHER(typedefType);
   REGISTER_MATCHER(unaryExprOrTypeTraitExpr);
   REGISTER_MATCHER(unaryOperator);
diff --git a/clang/lib/Analysis/FlowSensitive/CMakeLists.txt b/clang/lib/Analysis/FlowSensitive/CMakeLists.txt
index 0c30df8b4b194..97e09c9bce95f 100644
--- a/clang/lib/Analysis/FlowSensitive/CMakeLists.txt
+++ b/clang/lib/Analysis/FlowSensitive/CMakeLists.txt
@@ -6,6 +6,7 @@ add_clang_library(clangAnalysisFlowSensitive
   DataflowAnalysisContext.cpp
   DataflowEnvironment.cpp
   Formula.cpp
+  FormulaSerialization.cpp
   HTMLLogger.cpp
   Logger.cpp
   RecordOps.cpp
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
index 6421ad3883d10..4196d6821c184 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
@@ -208,6 +208,24 @@ bool DataflowAnalysisContext::equivalentFormulas(const Formula &Val1,
   return isUnsatisfiable(std::move(Constraints));
 }
 
+llvm::DenseSet<Atom> DataflowAnalysisContext::collectDependencies(
+    llvm::DenseSet<Atom> Tokens) const {
+  // Use a worklist algorithm, with `Remaining` holding the worklist and
+  // `Tokens` tracking which atoms have already been added to the worklist.
+  std::vector<Atom> Remaining(Tokens.begin(), Tokens.end());
+  while (!Remaining.empty()) {
+    Atom CurrentToken = Remaining.back();
+    Remaining.pop_back();
+    if (auto DepsIt = FlowConditionDeps.find(CurrentToken);
+        DepsIt != FlowConditionDeps.end())
+      for (Atom A : DepsIt->second)
+        if (Tokens.insert(A).second)
+          Remaining.push_back(A);
+  }
+
+  return Tokens;
+}
+
 void DataflowAnalysisContext::addTransitiveFlowConditionConstraints(
     Atom Token, llvm::SetVector<const Formula *> &Constraints) {
   llvm::DenseSet<Atom> AddedTokens;
@@ -224,6 +242,8 @@ void DataflowAnalysisContext::addTransitiveFlowConditionConstraints(
 
     auto ConstraintsIt = FlowConditionConstraints.find(Token);
     if (ConstraintsIt == FlowConditionConstraints.end()) {
+      // The flow condition is unconstrained. Just add the atom directly, which
+      // is equivalent to asserting it is true.
       Constraints.insert(&arena().makeAtomRef(Token));
     } else {
       // Bind flow condition token via `iff` to its set of constraints:
@@ -239,6 +259,65 @@ void DataflowAnalysisContext::addTransitiveFlowConditionConstraints(
   }
 }
 
+static void getReferencedAtoms(const Formula &F,
+                               llvm::DenseSet<dataflow::Atom> &Refs) {
+  switch (F.kind()) {
+  case Formula::AtomRef:
+    Refs.insert(F.getAtom());
+    break;
+  case Formula::Literal:
+    break;
+  case Formula::Not:
+    getReferencedAtoms(*F.operands()[0], Refs);
+    break;
+  case Formula::And:
+  case Formula::Or:
+  case Formula::Implies:
+  case Formula::Equal:
+    ArrayRef<const Formula *> Operands = F.operands();
+    getReferencedAtoms(*Operands[0], Refs);
+    getReferencedAtoms(*Operands[1], Refs);
+    break;
+  }
+}
+
+SimpleLogicalContext DataflowAnalysisContext::exportLogicalContext(
+    llvm::DenseSet<dataflow::Atom> TargetTokens) const {
+  SimpleLogicalContext LC;
+
+  // Copy `Invariant` even if it is null, to initialize the field.
+  LC.Invariant = Invariant;
+  if (Invariant != nullptr)
+    getReferencedAtoms(*Invariant, TargetTokens);
+
+  llvm::DenseSet<dataflow::Atom> Dependencies =
+      collectDependencies(std::move(TargetTokens));
+
+  for (dataflow::Atom Token : Dependencies) {
+    // Only process the token if it is constrained. Unconstrained tokens don't
+    // have dependencies.
+    const Formula *Constraints = FlowConditionConstraints.lookup(Token);
+    if (Constraints == nullptr)
+      continue;
+    LC.TokenDefs[Token] = Constraints;
+
+    if (auto DepsIt = FlowConditionDeps.find(Token);
+        DepsIt != FlowConditionDeps.end())
+      LC.TokenDeps[Token] = DepsIt->second;
+  }
+
+  return LC;
+}
+
+void DataflowAnalysisContext::initLogicalContext(SimpleLogicalContext LC) {
+  Invariant = LC.Invariant;
+  FlowConditionConstraints = std::move(LC.TokenDefs);
+  // TODO: The dependencies in `LC.TokenDeps` can be reconstructed from
+  // `LC.TokenDefs`. Give the caller the option to reconstruct, rather than
+  // providing them directly, to save caller space (memory/disk).
+  FlowConditionDeps = std::move(LC.TokenDeps);
+}
+
 static void printAtomList(const llvm::SmallVector<Atom> &Atoms,
                           llvm::raw_ostream &OS) {
   OS << "(";
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index 256ea18284189..f14cb43e47dd4 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -532,9 +532,11 @@ void Environment::initialize() {
           } else if (auto *FieldBeingInitialized =
                          dyn_cast<FieldDecl>(Parent->getLambdaContextDecl())) {
             // This is in a field initializer, rather than a method.
+            const RecordDecl *RD = FieldBeingInitialized->getParent();
+            const ASTContext &Ctx = RD->getASTContext();
+            CanQualType T = Ctx.getCanonicalTagType(RD);
             setThisPointeeStorageLocation(
-                cast<RecordStorageLocation>(createObject(QualType(
-                    FieldBeingInitialized->getParent()->getTypeForDecl(), 0))));
+                cast<RecordStorageLocation>(createObject(T)));
           } else {
             assert(false && "Unexpected this-capturing lambda context.");
           }
diff --git a/clang/lib/Analysis/FlowSensitive/FormulaSerialization.cpp b/clang/lib/Analysis/FlowSensitive/FormulaSerialization.cpp
new file mode 100644
index 0000000000000..df15a1d6eaadb
--- /dev/null
+++ b/clang/lib/Analysis/FlowSensitive/FormulaSerialization.cpp
@@ -0,0 +1,153 @@
+//===- FormulaSerialization.cpp ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Analysis/FlowSensitive/FormulaSerialization.h"
+#include "clang/Analysis/FlowSensitive/Arena.h"
+#include "clang/Analysis/FlowSensitive/Formula.h"
+#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <cassert>
+
+namespace clang::dataflow {
+
+// Returns the leading indicator of operation formulas. `AtomRef` and `Literal`
+// are handled differently.
+static char compactSigil(Formula::Kind K) {
+  switch (K) {
+  case Formula::AtomRef:
+  case Formula::Literal:
+    // No sigil.
+    return '\0';
+  case Formula::Not:
+    return '!';
+  case Formula::And:
+    return '&';
+  case Formula::Or:
+    return '|';
+  case Formula::Implies:
+    return '>';
+  case Formula::Equal:
+    return '=';
+  }
+  llvm_unreachable("unhandled formula kind");
+}
+
+void serializeFormula(const Formula &F, llvm::raw_ostream &OS) {
+  switch (Formula::numOperands(F.kind())) {
+  case 0:
+    switch (F.kind()) {
+    case Formula::AtomRef:
+      OS << F.getAtom();
+      break;
+    case Formula::Literal:
+      OS << (F.literal() ? 'T' : 'F');
+      break;
+    default:
+      llvm_unreachable("unhandled formula kind");
+    }
+    break;
+  case 1:
+    OS << compactSigil(F.kind());
+    serializeFormula(*F.operands()[0], OS);
+    break;
+  case 2:
+    OS << compactSigil(F.kind());
+    serializeFormula(*F.operands()[0], OS);
+    serializeFormula(*F.operands()[1], OS);
+    break;
+  default:
+    llvm_unreachable("unhandled formula arity");
+  }
+}
+
+static llvm::Expected<const Formula *>
+parsePrefix(llvm::StringRef &Str, Arena &A,
+            llvm::DenseMap<unsigned, Atom> &AtomMap) {
+  if (Str.empty())
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "unexpected end of input");
+
+  char Prefix = Str[0];
+  Str = Str.drop_front();
+
+  switch (Prefix) {
+  case 'T':
+    return &A.makeLiteral(true);
+  case 'F':
+    return &A.makeLiteral(false);
+  case 'V': {
+    unsigned AtomID;
+    if (Str.consumeInteger(10, AtomID))
+      return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                     "expected atom id");
+    auto [It, Inserted] = AtomMap.try_emplace(AtomID, Atom());
+    if (Inserted)
+      It->second = A.makeAtom();
+    return &A.makeAtomRef(It->second);
+  }
+  case '!': {
+    auto OperandOrErr = parsePrefix(Str, A, AtomMap);
+    if (!OperandOrErr)
+      return OperandOrErr.takeError();
+    return &A.makeNot(**OperandOrErr);
+  }
+  case '&':
+  case '|':
+  case '>':
+  case '=': {
+    auto LeftOrErr = parsePrefix(Str, A, AtomMap);
+    if (!LeftOrErr)
+      return LeftOrErr.takeError();
+
+    auto RightOrErr = parsePrefix(Str, A, AtomMap);
+    if (!RightOrErr)
+      return RightOrErr.takeError();
+
+    const Formula &LHS = **LeftOrErr;
+    const Formula &RHS = **RightOrErr;
+
+    switch (Prefix) {
+    case '&':
+      return &A.makeAnd(LHS, RHS);
+    case '|':
+      return &A.makeOr(LHS, RHS);
+    case '>':
+      return &A.makeImplies(LHS, RHS);
+    case '=':
+      return &A.makeEquals(LHS, RHS);
+    default:
+      llvm_unreachable("unexpected binary op");
+    }
+  }
+  default:
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "unexpected prefix character: %c", Prefix);
+  }
+}
+
+llvm::Expected<const Formula *>
+parseFormula(llvm::StringRef Str, Arena &A,
+             llvm::DenseMap<unsigned, Atom> &AtomMap) {
+  size_t OriginalSize = Str.size();
+  llvm::Expected<const Formula *> F = parsePrefix(Str, A, AtomMap);
+  if (!F)
+    return F.takeError();
+  if (!Str.empty())
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   ("unexpected suffix of length: " +
+                                    llvm::Twine(Str.size() - OriginalSize))
+                                       .str());
+  return F;
+}
+
+} // namespace clang::dataflow
diff --git a/clang/lib/Analysis/FlowSensitive/Transfer.cpp b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
index 86a816e2e406c..23a6de45e18b1 100644
--- a/clang/lib/Analysis/FlowSensitive/Transfer.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
@@ -20,14 +20,17 @@
 #include "clang/AST/OperationKinds.h"
 #include "clang/AST/Stmt.h"
 #include "clang/AST/StmtVisitor.h"
+#include "clang/AST/Type.h"
 #include "clang/Analysis/FlowSensitive/ASTOps.h"
 #include "clang/Analysis/FlowSensitive/AdornedCFG.h"
 #include "clang/Analysis/FlowSensitive/DataflowAnalysisContext.h"
 #include "clang/Analysis/FlowSensitive/DataflowEnvironment.h"
 #include "clang/Analysis/FlowSensitive/NoopAnalysis.h"
 #include "clang/Analysis/FlowSensitive/RecordOps.h"
+#include "clang/Analysis/FlowSensitive/StorageLocation.h"
 #include "clang/Analysis/FlowSensitive/Value.h"
 #include "clang/Basic/Builtins.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/OperatorKinds.h"
 #include "llvm/Support/Casting.h"
 #include <assert.h>
@@ -287,7 +290,7 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
     }
   }
 
-  void VisitImplicitCastExpr(const ImplicitCastExpr *S) {
+  void VisitCastExpr(const CastExpr *S) {
     const Expr *SubExpr = S->getSubExpr();
     assert(SubExpr != nullptr);
 
@@ -317,6 +320,60 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
       break;
     }
 
+    case CK_BaseToDerived: {
+      // This is a cast of (single-layer) pointer or reference to a record type.
+      // We should now model the fields for the derived type.
+
+      // Get the RecordStorageLocation for the record object underneath.
+      RecordStorageLocation *Loc = nullptr;
+      if (S->getType()->isPointerType()) {
+        auto *PV = Env.get<PointerValue>(*SubExpr);
+        assert(PV != nullptr);
+        if (PV == nullptr)
+          break;
+        Loc = cast<RecordStorageLocation>(&PV->getPointeeLoc());
+      } else {
+        assert(S->getType()->isRecordType());
+        if (SubExpr->isGLValue()) {
+          Loc = Env.get<RecordStorageLocation>(*SubExpr);
+        } else {
+          Loc = &Env.getResultObjectLocation(*SubExpr);
+        }
+      }
+      if (!Loc) {
+        // Nowhere to add children or propagate from, so we're done.
+        break;
+      }
+
+      // Get the derived record type underneath the reference or pointer.
+      QualType Derived = S->getType().getNonReferenceType();
+      if (Derived->isPointerType()) {
+        Derived = Derived->getPointeeType();
+      }
+
+      // Add children to the storage location for fields (including synthetic
+      // fields) of the derived type and initialize their values.
+      for (const FieldDecl *Field :
+           Env.getDataflowAnalysisContext().getModeledFields(Derived)) {
+        assert(Field != nullptr);
+        QualType FieldType = Field->getType();
+        if (FieldType->isReferenceType()) {
+          Loc->addChild(*Field, nullptr);
+        } else {
+          Loc->addChild(*Field, &Env.createStorageLocation(FieldType));
+        }
+
+        for (const auto &Entry :
+             Env.getDataflowAnalysisContext().getSyntheticFields(Derived)) {
+          Loc->addSyntheticField(Entry.getKey(),
+                                 Env.createStorageLocation(Entry.getValue()));
+        }
+      }
+      Env.initializeFieldsWithValues(*Loc, Derived);
+
+      // Fall through to propagate SubExpr's StorageLocation to the CastExpr.
+      [[fallthrough]];
+    }
     case CK_IntegralCast:
       // FIXME: This cast creates a new integral value from the
       // subexpression. But, because we don't model integers, we don't
@@ -324,10 +381,9 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
       // modeling is added, then update this code to create a fresh location and
       // value.
     case CK_UncheckedDerivedToBase:
+    case CK_DerivedToBase:
     case CK_ConstructorConversion:
     case CK_UserDefinedConversion:
-      // FIXME: Add tests that excercise CK_UncheckedDerivedToBase,
-      // CK_ConstructorConversion, and CK_UserDefinedConversion.
     case CK_NoOp: {
       // FIXME: Consider making `Environment::getStorageLocation` skip noop
       // expressions (this and other similar expressions in the file) instead
@@ -684,15 +740,6 @@ class TransferVisitor : public ConstStmtVisitor<TransferVisitor> {
     propagateValue(*SubExpr, *S, Env);
   }
 
-  void VisitCXXStaticCastExpr(const CXXStaticCastExpr *S) {
-    if (S->getCastKind() == CK_NoOp) {
-      const Expr *SubExpr = S->getSubExpr();
-      assert(SubExpr != nullptr);
-
-      propagateValueOrStorageLocation(*SubExpr, *S, Env);
-    }
-  }
-
   void VisitConditionalOperator(const ConditionalOperator *S) {
     const Environment *TrueEnv = StmtToEnv.getEnvironment(*S->getTrueExpr());
     const Environment *FalseEnv = StmtToEnv.getEnvironment(*S->getFalseExpr());
diff --git a/clang/lib/Analysis/LifetimeSafety.cpp b/clang/lib/Analysis/LifetimeSafety.cpp
index f39998cca56fe..c2e6dd74d0758 100644
--- a/clang/lib/Analysis/LifetimeSafety.cpp
+++ b/clang/lib/Analysis/LifetimeSafety.cpp
@@ -45,10 +45,11 @@ struct Loan {
   /// is represented as empty LoanSet
   LoanID ID;
   AccessPath Path;
-  SourceLocation IssueLoc;
+  /// The expression that creates the loan, e.g., &x.
+  const Expr *IssueExpr;
 
-  Loan(LoanID id, AccessPath path, SourceLocation loc)
-      : ID(id), Path(path), IssueLoc(loc) {}
+  Loan(LoanID id, AccessPath path, const Expr *IssueExpr)
+      : ID(id), Path(path), IssueExpr(IssueExpr) {}
 };
 
 /// An Origin is a symbolic identifier that represents the set of possible
@@ -82,8 +83,8 @@ class LoanManager {
 public:
   LoanManager() = default;
 
-  Loan &addLoan(AccessPath Path, SourceLocation Loc) {
-    AllLoans.emplace_back(getNextLoanID(), Path, Loc);
+  Loan &addLoan(AccessPath Path, const Expr *IssueExpr) {
+    AllLoans.emplace_back(getNextLoanID(), Path, IssueExpr);
     return AllLoans.back();
   }
 
@@ -174,6 +175,18 @@ class OriginManager {
     return NewID;
   }
 
+  void dump(OriginID OID, llvm::raw_ostream &OS) const {
+    OS << OID << " (";
+    Origin O = getOrigin(OID);
+    if (const ValueDecl *VD = O.getDecl())
+      OS << "Decl: " << VD->getNameAsString();
+    else if (const Expr *E = O.getExpr())
+      OS << "Expr: " << E->getStmtClassName();
+    else
+      OS << "Unknown";
+    OS << ")";
+  }
+
 private:
   OriginID getNextOriginID() { return NextOriginID++; }
 
@@ -199,6 +212,8 @@ class Fact {
     AssignOrigin,
     /// An origin escapes the function by flowing into the return value.
     ReturnOfOrigin,
+    /// An origin is used (eg. dereferencing a pointer).
+    Use,
     /// A marker for a specific point in the code, for testing.
     TestPoint,
   };
@@ -219,7 +234,7 @@ class Fact {
     return nullptr;
   }
 
-  virtual void dump(llvm::raw_ostream &OS) const {
+  virtual void dump(llvm::raw_ostream &OS, const OriginManager &) const {
     OS << "Fact (Kind: " << static_cast<int>(K) << ")\n";
   }
 };
@@ -234,21 +249,27 @@ class IssueFact : public Fact {
   IssueFact(LoanID LID, OriginID OID) : Fact(Kind::Issue), LID(LID), OID(OID) {}
   LoanID getLoanID() const { return LID; }
   OriginID getOriginID() const { return OID; }
-  void dump(llvm::raw_ostream &OS) const override {
-    OS << "Issue (LoanID: " << getLoanID() << ", OriginID: " << getOriginID()
-       << ")\n";
+  void dump(llvm::raw_ostream &OS, const OriginManager &OM) const override {
+    OS << "Issue (LoanID: " << getLoanID() << ", ToOrigin: ";
+    OM.dump(getOriginID(), OS);
+    OS << ")\n";
   }
 };
 
 class ExpireFact : public Fact {
   LoanID LID;
+  SourceLocation ExpiryLoc;
 
 public:
   static bool classof(const Fact *F) { return F->getKind() == Kind::Expire; }
 
-  ExpireFact(LoanID LID) : Fact(Kind::Expire), LID(LID) {}
+  ExpireFact(LoanID LID, SourceLocation ExpiryLoc)
+      : Fact(Kind::Expire), LID(LID), ExpiryLoc(ExpiryLoc) {}
+
   LoanID getLoanID() const { return LID; }
-  void dump(llvm::raw_ostream &OS) const override {
+  SourceLocation getExpiryLoc() const { return ExpiryLoc; }
+
+  void dump(llvm::raw_ostream &OS, const OriginManager &OM) const override {
     OS << "Expire (LoanID: " << getLoanID() << ")\n";
   }
 };
@@ -266,9 +287,12 @@ class AssignOriginFact : public Fact {
       : Fact(Kind::AssignOrigin), OIDDest(OIDDest), OIDSrc(OIDSrc) {}
   OriginID getDestOriginID() const { return OIDDest; }
   OriginID getSrcOriginID() const { return OIDSrc; }
-  void dump(llvm::raw_ostream &OS) const override {
-    OS << "AssignOrigin (DestID: " << getDestOriginID()
-       << ", SrcID: " << getSrcOriginID() << ")\n";
+  void dump(llvm::raw_ostream &OS, const OriginManager &OM) const override {
+    OS << "AssignOrigin (Dest: ";
+    OM.dump(getDestOriginID(), OS);
+    OS << ", Src: ";
+    OM.dump(getSrcOriginID(), OS);
+    OS << ")\n";
   }
 };
 
@@ -282,8 +306,30 @@ class ReturnOfOriginFact : public Fact {
 
   ReturnOfOriginFact(OriginID OID) : Fact(Kind::ReturnOfOrigin), OID(OID) {}
   OriginID getReturnedOriginID() const { return OID; }
-  void dump(llvm::raw_ostream &OS) const override {
-    OS << "ReturnOfOrigin (OriginID: " << getReturnedOriginID() << ")\n";
+  void dump(llvm::raw_ostream &OS, const OriginManager &OM) const override {
+    OS << "ReturnOfOrigin (";
+    OM.dump(getReturnedOriginID(), OS);
+    OS << ")\n";
+  }
+};
+
+class UseFact : public Fact {
+  OriginID UsedOrigin;
+  const Expr *UseExpr;
+
+public:
+  static bool classof(const Fact *F) { return F->getKind() == Kind::Use; }
+
+  UseFact(OriginID UsedOrigin, const Expr *UseExpr)
+      : Fact(Kind::Use), UsedOrigin(UsedOrigin), UseExpr(UseExpr) {}
+
+  OriginID getUsedOrigin() const { return UsedOrigin; }
+  const Expr *getUseExpr() const { return UseExpr; }
+
+  void dump(llvm::raw_ostream &OS, const OriginManager &OM) const override {
+    OS << "Use (";
+    OM.dump(getUsedOrigin(), OS);
+    OS << ")\n";
   }
 };
 
@@ -300,7 +346,7 @@ class TestPointFact : public Fact {
 
   StringRef getAnnotation() const { return Annotation; }
 
-  void dump(llvm::raw_ostream &OS) const override {
+  void dump(llvm::raw_ostream &OS, const OriginManager &) const override {
     OS << "TestPoint (Annotation: \"" << getAnnotation() << "\")\n";
   }
 };
@@ -339,7 +385,7 @@ class FactManager {
       if (It != BlockToFactsMap.end()) {
         for (const Fact *F : It->second) {
           llvm::dbgs() << "    ";
-          F->dump(llvm::dbgs());
+          F->dump(llvm::dbgs(), OriginMgr);
         }
       }
       llvm::dbgs() << "  End of Block\n";
@@ -417,13 +463,17 @@ class FactGenerator : public ConstStmtVisitor<FactGenerator> {
           if (VD->hasLocalStorage()) {
             OriginID OID = FactMgr.getOriginMgr().getOrCreate(*UO);
             AccessPath AddrOfLocalVarPath(VD);
-            const Loan &L = FactMgr.getLoanMgr().addLoan(AddrOfLocalVarPath,
-                                                         UO->getOperatorLoc());
+            const Loan &L =
+                FactMgr.getLoanMgr().addLoan(AddrOfLocalVarPath, UO);
             CurrentBlockFacts.push_back(
                 FactMgr.createFact<IssueFact>(L.ID, OID));
           }
         }
       }
+    } else if (UO->getOpcode() == UO_Deref) {
+      // This is a pointer use, like '*p'.
+      OriginID OID = FactMgr.getOriginMgr().get(*UO->getSubExpr());
+      CurrentBlockFacts.push_back(FactMgr.createFact<UseFact>(OID, UO));
     }
   }
 
@@ -492,7 +542,8 @@ class FactGenerator : public ConstStmtVisitor<FactGenerator> {
       // Check if the loan is for a stack variable and if that variable
       // is the one being destructed.
       if (LoanPath.D == DestructedVD)
-        CurrentBlockFacts.push_back(FactMgr.createFact<ExpireFact>(L.ID));
+        CurrentBlockFacts.push_back(FactMgr.createFact<ExpireFact>(
+            L.ID, DtorOpt.getTriggerStmt()->getEndLoc()));
     }
   }
 
@@ -618,6 +669,7 @@ class DataflowAnalysis {
     }
   }
 
+protected:
   Lattice getState(ProgramPoint P) const { return PerPointStates.lookup(P); }
 
   Lattice getInState(const CFGBlock *B) const { return InStates.lookup(B); }
@@ -665,6 +717,8 @@ class DataflowAnalysis {
       return D->transfer(In, *F->getAs<AssignOriginFact>());
     case Fact::Kind::ReturnOfOrigin:
       return D->transfer(In, *F->getAs<ReturnOfOriginFact>());
+    case Fact::Kind::Use:
+      return D->transfer(In, *F->getAs<UseFact>());
     case Fact::Kind::TestPoint:
       return D->transfer(In, *F->getAs<TestPointFact>());
     }
@@ -676,6 +730,7 @@ class DataflowAnalysis {
   Lattice transfer(Lattice In, const ExpireFact &) { return In; }
   Lattice transfer(Lattice In, const AssignOriginFact &) { return In; }
   Lattice transfer(Lattice In, const ReturnOfOriginFact &) { return In; }
+  Lattice transfer(Lattice In, const UseFact &) { return In; }
   Lattice transfer(Lattice In, const TestPointFact &) { return In; }
 };
 
@@ -693,6 +748,20 @@ static llvm::ImmutableSet<T> join(llvm::ImmutableSet<T> A,
   return A;
 }
 
+/// Checks if set A is a subset of set B.
+template <typename T>
+static bool isSubsetOf(const llvm::ImmutableSet<T> &A,
+                       const llvm::ImmutableSet<T> &B) {
+  // Empty set is a subset of all sets.
+  if (A.isEmpty())
+    return true;
+
+  for (const T &Elem : A)
+    if (!B.contains(Elem))
+      return false;
+  return true;
+}
+
 /// Computes the key-wise union of two ImmutableMaps.
 // TODO(opt): This key-wise join is a performance bottleneck. A more
 // efficient merge could be implemented using a Patricia Trie or HAMT
@@ -700,7 +769,7 @@ static llvm::ImmutableSet<T> join(llvm::ImmutableSet<T> A,
 template <typename K, typename V, typename Joiner>
 static llvm::ImmutableMap<K, V>
 join(llvm::ImmutableMap<K, V> A, llvm::ImmutableMap<K, V> B,
-     typename llvm::ImmutableMap<K, V>::Factory &F, Joiner joinValues) {
+     typename llvm::ImmutableMap<K, V>::Factory &F, Joiner JoinValues) {
   if (A.getHeight() < B.getHeight())
     std::swap(A, B);
 
@@ -710,7 +779,7 @@ join(llvm::ImmutableMap<K, V> A, llvm::ImmutableMap<K, V> B,
     const K &Key = Entry.first;
     const V &ValB = Entry.second;
     if (const V *ValA = A.lookup(Key))
-      A = F.add(A, Key, joinValues(*ValA, ValB));
+      A = F.add(A, Key, JoinValues(*ValA, ValB));
     else
       A = F.add(A, Key, ValB);
   }
@@ -723,17 +792,14 @@ join(llvm::ImmutableMap<K, V> A, llvm::ImmutableMap<K, V> B,
 // ========================================================================= //
 
 using OriginLoanMap = llvm::ImmutableMap<OriginID, LoanSet>;
+using ExpiredLoanMap = llvm::ImmutableMap<LoanID, const ExpireFact *>;
 
 /// An object to hold the factories for immutable collections, ensuring
 /// that all created states share the same underlying memory management.
 struct LifetimeFactory {
   OriginLoanMap::Factory OriginMapFactory;
   LoanSet::Factory LoanSetFactory;
-
-  /// Creates a singleton set containing only the given loan ID.
-  LoanSet createLoanSet(LoanID LID) {
-    return LoanSetFactory.add(LoanSetFactory.getEmptySet(), LID);
-  }
+  ExpiredLoanMap::Factory ExpiredLoanMapFactory;
 };
 
 /// Represents the dataflow lattice for loan propagation.
@@ -774,13 +840,15 @@ struct LoanPropagationLattice {
 class LoanPropagationAnalysis
     : public DataflowAnalysis<LoanPropagationAnalysis, LoanPropagationLattice,
                               Direction::Forward> {
-
-  LifetimeFactory &Factory;
+  OriginLoanMap::Factory &OriginLoanMapFactory;
+  LoanSet::Factory &LoanSetFactory;
 
 public:
   LoanPropagationAnalysis(const CFG &C, AnalysisDeclContext &AC, FactManager &F,
-                          LifetimeFactory &Factory)
-      : DataflowAnalysis(C, AC, F), Factory(Factory) {}
+                          LifetimeFactory &LFactory)
+      : DataflowAnalysis(C, AC, F),
+        OriginLoanMapFactory(LFactory.OriginMapFactory),
+        LoanSetFactory(LFactory.LoanSetFactory) {}
 
   using Base::transfer;
 
@@ -792,9 +860,9 @@ class LoanPropagationAnalysis
   // TODO(opt): Keep the state small by removing origins which become dead.
   Lattice join(Lattice A, Lattice B) {
     OriginLoanMap JoinedOrigins =
-        utils::join(A.Origins, B.Origins, Factory.OriginMapFactory,
-                    [this](LoanSet S1, LoanSet S2) {
-                      return utils::join(S1, S2, Factory.LoanSetFactory);
+        utils::join(A.Origins, B.Origins, OriginLoanMapFactory,
+                    [&](LoanSet S1, LoanSet S2) {
+                      return utils::join(S1, S2, LoanSetFactory);
                     });
     return Lattice(JoinedOrigins);
   }
@@ -803,8 +871,9 @@ class LoanPropagationAnalysis
   Lattice transfer(Lattice In, const IssueFact &F) {
     OriginID OID = F.getOriginID();
     LoanID LID = F.getLoanID();
-    return LoanPropagationLattice(Factory.OriginMapFactory.add(
-        In.Origins, OID, Factory.createLoanSet(LID)));
+    return LoanPropagationLattice(OriginLoanMapFactory.add(
+        In.Origins, OID,
+        LoanSetFactory.add(LoanSetFactory.getEmptySet(), LID)));
   }
 
   /// The destination origin's loan set is replaced by the source's.
@@ -814,7 +883,7 @@ class LoanPropagationAnalysis
     OriginID SrcOID = F.getSrcOriginID();
     LoanSet SrcLoans = getLoans(In, SrcOID);
     return LoanPropagationLattice(
-        Factory.OriginMapFactory.add(In.Origins, DestOID, SrcLoans));
+        OriginLoanMapFactory.add(In.Origins, DestOID, SrcLoans));
   }
 
   LoanSet getLoans(OriginID OID, ProgramPoint P) {
@@ -825,7 +894,7 @@ class LoanPropagationAnalysis
   LoanSet getLoans(Lattice L, OriginID OID) {
     if (auto *Loans = L.Origins.lookup(OID))
       return *Loans;
-    return Factory.LoanSetFactory.getEmptySet();
+    return LoanSetFactory.getEmptySet();
   }
 };
 
@@ -835,10 +904,11 @@ class LoanPropagationAnalysis
 
 /// The dataflow lattice for tracking the set of expired loans.
 struct ExpiredLattice {
-  LoanSet Expired;
+  /// Map from an expired `LoanID` to the `ExpireFact` that made it expire.
+  ExpiredLoanMap Expired;
 
   ExpiredLattice() : Expired(nullptr) {};
-  explicit ExpiredLattice(LoanSet S) : Expired(S) {}
+  explicit ExpiredLattice(ExpiredLoanMap M) : Expired(M) {}
 
   bool operator==(const ExpiredLattice &Other) const {
     return Expired == Other.Expired;
@@ -851,8 +921,8 @@ struct ExpiredLattice {
     OS << "ExpiredLattice State:\n";
     if (Expired.isEmpty())
       OS << "  <empty>\n";
-    for (const LoanID &LID : Expired)
-      OS << "  Loan " << LID << " is expired\n";
+    for (const auto &[ID, _] : Expired)
+      OS << "  Loan " << ID << " is expired\n";
   }
 };
 
@@ -861,26 +931,31 @@ class ExpiredLoansAnalysis
     : public DataflowAnalysis<ExpiredLoansAnalysis, ExpiredLattice,
                               Direction::Forward> {
 
-  LoanSet::Factory &Factory;
+  ExpiredLoanMap::Factory &Factory;
 
 public:
   ExpiredLoansAnalysis(const CFG &C, AnalysisDeclContext &AC, FactManager &F,
                        LifetimeFactory &Factory)
-      : DataflowAnalysis(C, AC, F), Factory(Factory.LoanSetFactory) {}
+      : DataflowAnalysis(C, AC, F), Factory(Factory.ExpiredLoanMapFactory) {}
 
   using Base::transfer;
 
   StringRef getAnalysisName() const { return "ExpiredLoans"; }
 
-  Lattice getInitialState() { return Lattice(Factory.getEmptySet()); }
+  Lattice getInitialState() { return Lattice(Factory.getEmptyMap()); }
 
-  /// Merges two lattices by taking the union of the expired loan sets.
-  Lattice join(Lattice L1, Lattice L2) const {
-    return Lattice(utils::join(L1.Expired, L2.Expired, Factory));
+  /// Merges two lattices by taking the union of the two expired loans.
+  Lattice join(Lattice L1, Lattice L2) {
+    return Lattice(
+        utils::join(L1.Expired, L2.Expired, Factory,
+                    // Take the last expiry fact to make this hermetic.
+                    [](const ExpireFact *F1, const ExpireFact *F2) {
+                      return F1->getExpiryLoc() > F2->getExpiryLoc() ? F1 : F2;
+                    }));
   }
 
   Lattice transfer(Lattice In, const ExpireFact &F) {
-    return Lattice(Factory.add(In.Expired, F.getLoanID()));
+    return Lattice(Factory.add(In.Expired, F.getLoanID(), &F));
   }
 
   // Removes the loan from the set of expired loans.
@@ -912,15 +987,116 @@ class ExpiredLoansAnalysis
   Lattice transfer(Lattice In, const IssueFact &F) {
     return Lattice(Factory.remove(In.Expired, F.getLoanID()));
   }
+
+  ExpiredLoanMap getExpiredLoans(ProgramPoint P) { return getState(P).Expired; }
 };
 
 // ========================================================================= //
-//  TODO:
-// - Modify loan expiry analysis to answer `bool isExpired(Loan L, Point P)`
-// - Modify origin liveness analysis to answer `bool isLive(Origin O, Point P)`
-// - Using the above three to perform the final error reporting.
+//                       Lifetime checker and Error reporter
 // ========================================================================= //
 
+/// Struct to store the complete context for a potential lifetime violation.
+struct PendingWarning {
+  SourceLocation ExpiryLoc; // Where the loan expired.
+  const Expr *UseExpr;      // Where the origin holding this loan was used.
+  Confidence ConfidenceLevel;
+};
+
+class LifetimeChecker {
+private:
+  llvm::DenseMap<LoanID, PendingWarning> FinalWarningsMap;
+  LoanPropagationAnalysis &LoanPropagation;
+  ExpiredLoansAnalysis &ExpiredLoans;
+  FactManager &FactMgr;
+  AnalysisDeclContext &ADC;
+  LifetimeSafetyReporter *Reporter;
+
+public:
+  LifetimeChecker(LoanPropagationAnalysis &LPA, ExpiredLoansAnalysis &ELA,
+                  FactManager &FM, AnalysisDeclContext &ADC,
+                  LifetimeSafetyReporter *Reporter)
+      : LoanPropagation(LPA), ExpiredLoans(ELA), FactMgr(FM), ADC(ADC),
+        Reporter(Reporter) {}
+
+  void run() {
+    llvm::TimeTraceScope TimeProfile("LifetimeChecker");
+    for (const CFGBlock *B : *ADC.getAnalysis<PostOrderCFGView>())
+      for (const Fact *F : FactMgr.getFacts(B))
+        if (const auto *UF = F->getAs<UseFact>())
+          checkUse(UF);
+    issuePendingWarnings();
+  }
+
+  /// Checks for use-after-free errors for a given use of an Origin.
+  ///
+  /// This method is called for each 'UseFact' identified in the control flow
+  /// graph. It determines if the loans held by the used origin have expired
+  /// at the point of use.
+  void checkUse(const UseFact *UF) {
+
+    OriginID O = UF->getUsedOrigin();
+
+    // Get the set of loans that the origin might hold at this program point.
+    LoanSet HeldLoans = LoanPropagation.getLoans(O, UF);
+
+    // Get the set of all loans that have expired at this program point.
+    ExpiredLoanMap AllExpiredLoans = ExpiredLoans.getExpiredLoans(UF);
+
+    // If the pointer holds no loans or no loans have expired, there's nothing
+    // to check.
+    if (HeldLoans.isEmpty() || AllExpiredLoans.isEmpty())
+      return;
+
+    // Identify loans that which have expired but are held by the pointer. Using
+    // them is a use-after-free.
+    llvm::SmallVector<LoanID> DefaultedLoans;
+    // A definite UaF error occurs if all loans the origin might hold have
+    // expired.
+    bool IsDefiniteError = true;
+    for (LoanID L : HeldLoans) {
+      if (AllExpiredLoans.contains(L))
+        DefaultedLoans.push_back(L);
+      else
+        // If at least one loan is not expired, this use is not a definite UaF.
+        IsDefiniteError = false;
+    }
+    // If there are no defaulted loans, the use is safe.
+    if (DefaultedLoans.empty())
+      return;
+
+    // Determine the confidence level of the error (definite or maybe).
+    Confidence CurrentConfidence =
+        IsDefiniteError ? Confidence::Definite : Confidence::Maybe;
+
+    // For each expired loan, create a pending warning.
+    for (LoanID DefaultedLoan : DefaultedLoans) {
+      // If we already have a warning for this loan with a higher or equal
+      // confidence, skip this one.
+      if (FinalWarningsMap.count(DefaultedLoan) &&
+          CurrentConfidence <= FinalWarningsMap[DefaultedLoan].ConfidenceLevel)
+        continue;
+
+      auto *EF = AllExpiredLoans.lookup(DefaultedLoan);
+      assert(EF && "Could not find ExpireFact for an expired loan.");
+
+      FinalWarningsMap[DefaultedLoan] = {/*ExpiryLoc=*/(*EF)->getExpiryLoc(),
+                                         /*UseExpr=*/UF->getUseExpr(),
+                                         /*ConfidenceLevel=*/CurrentConfidence};
+    }
+  }
+
+  void issuePendingWarnings() {
+    if (!Reporter)
+      return;
+    for (const auto &[LID, Warning] : FinalWarningsMap) {
+      const Loan &L = FactMgr.getLoanMgr().getLoan(LID);
+      const Expr *IssueExpr = L.IssueExpr;
+      Reporter->reportUseAfterFree(IssueExpr, Warning.UseExpr,
+                                   Warning.ExpiryLoc, Warning.ConfidenceLevel);
+    }
+  }
+};
+
 // ========================================================================= //
 //                  LifetimeSafetyAnalysis Class Implementation
 // ========================================================================= //
@@ -928,8 +1104,9 @@ class ExpiredLoansAnalysis
 // We need this here for unique_ptr with forward declared class.
 LifetimeSafetyAnalysis::~LifetimeSafetyAnalysis() = default;
 
-LifetimeSafetyAnalysis::LifetimeSafetyAnalysis(AnalysisDeclContext &AC)
-    : AC(AC), Factory(std::make_unique<LifetimeFactory>()),
+LifetimeSafetyAnalysis::LifetimeSafetyAnalysis(AnalysisDeclContext &AC,
+                                               LifetimeSafetyReporter *Reporter)
+    : AC(AC), Reporter(Reporter), Factory(std::make_unique<LifetimeFactory>()),
       FactMgr(std::make_unique<FactManager>()) {}
 
 void LifetimeSafetyAnalysis::run() {
@@ -952,6 +1129,8 @@ void LifetimeSafetyAnalysis::run() {
   ///    blocks; only Decls are visible.  Therefore, loans in a block that
   ///    never reach an Origin associated with a Decl can be safely dropped by
   ///    the analysis.
+  /// 3. Collapse ExpireFacts belonging to same source location into a single
+  ///    Fact.
   LoanPropagation =
       std::make_unique<LoanPropagationAnalysis>(Cfg, AC, *FactMgr, *Factory);
   LoanPropagation->run();
@@ -959,6 +1138,10 @@ void LifetimeSafetyAnalysis::run() {
   ExpiredLoans =
       std::make_unique<ExpiredLoansAnalysis>(Cfg, AC, *FactMgr, *Factory);
   ExpiredLoans->run();
+
+  LifetimeChecker Checker(*LoanPropagation, *ExpiredLoans, *FactMgr, AC,
+                          Reporter);
+  Checker.run();
 }
 
 LoanSet LifetimeSafetyAnalysis::getLoansAtPoint(OriginID OID,
@@ -967,9 +1150,13 @@ LoanSet LifetimeSafetyAnalysis::getLoansAtPoint(OriginID OID,
   return LoanPropagation->getLoans(OID, PP);
 }
 
-LoanSet LifetimeSafetyAnalysis::getExpiredLoansAtPoint(ProgramPoint PP) const {
+std::vector<LoanID>
+LifetimeSafetyAnalysis::getExpiredLoansAtPoint(ProgramPoint PP) const {
   assert(ExpiredLoans && "ExpiredLoansAnalysis has not been run.");
-  return ExpiredLoans->getState(PP).Expired;
+  std::vector<LoanID> Result;
+  for (const auto &pair : ExpiredLoans->getExpiredLoans(PP))
+    Result.push_back(pair.first);
+  return Result;
 }
 
 std::optional<OriginID>
@@ -1009,8 +1196,9 @@ llvm::StringMap<ProgramPoint> LifetimeSafetyAnalysis::getTestPoints() const {
 }
 } // namespace internal
 
-void runLifetimeSafetyAnalysis(AnalysisDeclContext &AC) {
-  internal::LifetimeSafetyAnalysis Analysis(AC);
+void runLifetimeSafetyAnalysis(AnalysisDeclContext &AC,
+                               LifetimeSafetyReporter *Reporter) {
+  internal::LifetimeSafetyAnalysis Analysis(AC, Reporter);
   Analysis.run();
 }
 } // namespace clang::lifetimes
diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp
index c9fd9cc6bd855..131170df9976e 100644
--- a/clang/lib/Analysis/ThreadSafety.cpp
+++ b/clang/lib/Analysis/ThreadSafety.cpp
@@ -1392,8 +1392,7 @@ void ThreadSafetyAnalyzer::addLock(FactSet &FSet, const FactEntry *Entry,
   }
 
   // Check before/after constraints
-  if (Handler.issueBetaWarnings() &&
-      !Entry->asserted() && !Entry->declared()) {
+  if (!Entry->asserted() && !Entry->declared()) {
     GlobalBeforeSet->checkBeforeAfter(Entry->valueDecl(), FSet, *this,
                                       Entry->loc(), Entry->getKind());
   }
@@ -1929,7 +1928,9 @@ void BuildLockset::handleCall(const Expr *Exp, const NamedDecl *D,
       assert(inserted.second && "Are we visiting the same expression again?");
       if (isa<CXXConstructExpr>(Exp))
         Self = Placeholder;
-      if (TagT->getDecl()->hasAttr<ScopedLockableAttr>())
+      if (TagT->getOriginalDecl()
+              ->getMostRecentDecl()
+              ->hasAttr<ScopedLockableAttr>())
         Scp = CapabilityExpr(Placeholder, Exp->getType(), /*Neg=*/false);
     }
 
diff --git a/clang/lib/Analysis/ThreadSafetyCommon.cpp b/clang/lib/Analysis/ThreadSafetyCommon.cpp
index ddbd0a9ca904b..f560dd8ae1dd1 100644
--- a/clang/lib/Analysis/ThreadSafetyCommon.cpp
+++ b/clang/lib/Analysis/ThreadSafetyCommon.cpp
@@ -84,8 +84,8 @@ static std::pair<StringRef, bool> classifyCapability(QualType QT) {
   // which it is. The type should either be a record or a typedef, or a pointer
   // or reference thereof.
   if (const auto *RT = QT->getAs<RecordType>()) {
-    if (const auto *RD = RT->getDecl())
-      return classifyCapability(*RD);
+    if (const auto *RD = RT->getOriginalDecl())
+      return classifyCapability(*RD->getDefinitionOrSelf());
   } else if (const auto *TT = QT->getAs<TypedefType>()) {
     if (const auto *TD = TT->getDecl())
       return classifyCapability(*TD);
diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp
index 40dff7eaf06bc..1d7b8722103aa 100644
--- a/clang/lib/Analysis/UnsafeBufferUsage.cpp
+++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp
@@ -182,18 +182,22 @@ class MatchDescendantVisitor : public DynamicRecursiveASTVisitor {
     return DynamicRecursiveASTVisitor::TraverseUnaryExprOrTypeTraitExpr(Node);
   }
 
-  bool TraverseTypeOfExprTypeLoc(TypeOfExprTypeLoc Node) override {
+  bool TraverseTypeOfExprTypeLoc(TypeOfExprTypeLoc Node,
+                                 bool TraverseQualifier) override {
     // Unevaluated context.
     if (ignoreUnevaluatedContext)
       return true;
-    return DynamicRecursiveASTVisitor::TraverseTypeOfExprTypeLoc(Node);
+    return DynamicRecursiveASTVisitor::TraverseTypeOfExprTypeLoc(
+        Node, TraverseQualifier);
   }
 
-  bool TraverseDecltypeTypeLoc(DecltypeTypeLoc Node) override {
+  bool TraverseDecltypeTypeLoc(DecltypeTypeLoc Node,
+                               bool TraverseQualifier) override {
     // Unevaluated context.
     if (ignoreUnevaluatedContext)
       return true;
-    return DynamicRecursiveASTVisitor::TraverseDecltypeTypeLoc(Node);
+    return DynamicRecursiveASTVisitor::TraverseDecltypeTypeLoc(
+        Node, TraverseQualifier);
   }
 
   bool TraverseCXXNoexceptExpr(CXXNoexceptExpr *Node) override {
@@ -1986,6 +1990,14 @@ class UnsafeLibcFunctionCallGadget : public WarningGadget {
     const auto *FD = dyn_cast<FunctionDecl>(CE->getDirectCallee());
     if (!FD)
       return false;
+
+    bool IsGlobalAndNotInAnyNamespace =
+        FD->isGlobal() && !FD->getEnclosingNamespaceContext()->isNamespace();
+
+    // A libc function must either be in the std:: namespace or a global
+    // function that is not in any namespace:
+    if (!FD->isInStdNamespace() && !IsGlobalAndNotInAnyNamespace)
+      return false;
     auto isSingleStringLiteralArg = false;
     if (CE->getNumArgs() == 1) {
       isSingleStringLiteralArg =
@@ -2244,7 +2256,7 @@ namespace {
 // declarations to its uses and make sure we've covered all uses with our
 // analysis before we try to fix the declaration.
 class DeclUseTracker {
-  using UseSetTy = llvm::SmallSet<const DeclRefExpr *, 16>;
+  using UseSetTy = llvm::SmallPtrSet<const DeclRefExpr *, 16>;
   using DefMapTy = llvm::DenseMap<const VarDecl *, const DeclStmt *>;
 
   // Allocate on the heap for easier move.
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 343c26e17236d..d8ec837f0f7b9 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -1171,14 +1171,14 @@ unsigned SourceManager::getColumnNumber(FileID FID, unsigned FilePos,
         if (Buf[FilePos - 1] == '\r' || Buf[FilePos - 1] == '\n')
           --FilePos;
       }
-      return FilePos - LineStart + 1;
+      return (FilePos - LineStart) + 1;
     }
   }
 
   unsigned LineStart = FilePos;
   while (LineStart && Buf[LineStart-1] != '\n' && Buf[LineStart-1] != '\r')
     --LineStart;
-  return FilePos-LineStart+1;
+  return (FilePos - LineStart) + 1;
 }
 
 // isInvalid - Return the result of calling loc.isInvalid(), and
diff --git a/clang/lib/Basic/TargetInfo.cpp b/clang/lib/Basic/TargetInfo.cpp
index 21fc084d19772..2fbf1ee39b789 100644
--- a/clang/lib/Basic/TargetInfo.cpp
+++ b/clang/lib/Basic/TargetInfo.cpp
@@ -62,7 +62,7 @@ TargetInfo::TargetInfo(const llvm::Triple &T) : Triple(T) {
   TLSSupported = true;
   VLASupported = true;
   NoAsmVariants = false;
-  HasLegalHalfType = false;
+  HasFastHalfType = false;
   HalfArgsAndReturns = false;
   HasFloat128 = false;
   HasIbm128 = false;
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 2b023e5fdb7d8..9e03a0846ffba 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -142,7 +142,7 @@ AArch64TargetInfo::AArch64TargetInfo(const llvm::Triple &Triple,
   AddrSpaceMap = &ARM64AddrSpaceMap;
 
   // All AArch64 implementations support ARMv8 FP, which makes half a legal type.
-  HasLegalHalfType = true;
+  HasFastHalfType = true;
   HalfArgsAndReturns = true;
   HasFloat16 = true;
   HasStrictFP = true;
diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp
index 52cbdbc3719d3..639e735202f26 100644
--- a/clang/lib/Basic/Targets/AMDGPU.cpp
+++ b/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -251,7 +251,7 @@ AMDGPUTargetInfo::AMDGPUTargetInfo(const llvm::Triple &Triple,
     BFloat16Format = &llvm::APFloat::BFloat();
   }
 
-  HasLegalHalfType = true;
+  HasFastHalfType = true;
   HasFloat16 = true;
   WavefrontSize = (GPUFeatures & llvm::AMDGPU::FEATURE_WAVE32) ? 32 : 64;
 
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 75fdf38e2104f..3de17d2c829f1 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -585,13 +585,13 @@ bool ARMTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
     } else if (Feature == "+fp16") {
       HW_FP |= HW_FP_HP;
     } else if (Feature == "+fullfp16") {
-      HasLegalHalfType = true;
+      HasFastHalfType = true;
     } else if (Feature == "+dotprod") {
       DotProd = true;
     } else if (Feature == "+mve") {
       MVE |= MVE_INT;
     } else if (Feature == "+mve.fp") {
-      HasLegalHalfType = true;
+      HasFastHalfType = true;
       FPU |= FPARMV8;
       MVE |= MVE_INT | MVE_FP;
       HW_FP |= HW_FP_SP | HW_FP_HP;
@@ -1014,11 +1014,11 @@ void ARMTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__ARM_FP_FAST", "1");
 
   // Armv8.2-A FP16 vector intrinsic
-  if ((FPU & NeonFPU) && HasLegalHalfType)
+  if ((FPU & NeonFPU) && HasFastHalfType)
     Builder.defineMacro("__ARM_FEATURE_FP16_VECTOR_ARITHMETIC", "1");
 
   // Armv8.2-A FP16 scalar intrinsics
-  if (HasLegalHalfType)
+  if (HasFastHalfType)
     Builder.defineMacro("__ARM_FEATURE_FP16_SCALAR_ARITHMETIC", "1");
 
   // Armv8.2-A dot product intrinsics
diff --git a/clang/lib/Basic/Targets/AVR.h b/clang/lib/Basic/Targets/AVR.h
index 75c969fd59dc9..b666778659603 100644
--- a/clang/lib/Basic/Targets/AVR.h
+++ b/clang/lib/Basic/Targets/AVR.h
@@ -57,7 +57,7 @@ class LLVM_LIBRARY_VISIBILITY AVRTargetInfo : public TargetInfo {
     Int16Type = SignedInt;
     Char32Type = UnsignedLong;
     SigAtomicType = SignedChar;
-    resetDataLayout("e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8-a:8");
+    resetDataLayout("e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8:16-a:8");
   }
 
   void getTargetDefines(const LangOptions &Opts,
diff --git a/clang/lib/Basic/Targets/DirectX.h b/clang/lib/Basic/Targets/DirectX.h
index 17240cf358902..bd13c9ee0fd05 100644
--- a/clang/lib/Basic/Targets/DirectX.h
+++ b/clang/lib/Basic/Targets/DirectX.h
@@ -59,7 +59,7 @@ class LLVM_LIBRARY_VISIBILITY DirectXTargetInfo : public TargetInfo {
     VLASupported = false;
     AddrSpaceMap = &DirectXAddrSpaceMap;
     UseAddrSpaceMapMangling = true;
-    HasLegalHalfType = true;
+    HasFastHalfType = true;
     HasFloat16 = true;
     NoAsmVariants = true;
     PlatformMinVersion = Triple.getOSVersion();
diff --git a/clang/lib/Basic/Targets/Hexagon.cpp b/clang/lib/Basic/Targets/Hexagon.cpp
index 06dcac03baa5b..cea64f9860036 100644
--- a/clang/lib/Basic/Targets/Hexagon.cpp
+++ b/clang/lib/Basic/Targets/Hexagon.cpp
@@ -149,7 +149,7 @@ bool HexagonTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasAudio = true;
   }
   if (CPU.compare("hexagonv68") >= 0) {
-    HasLegalHalfType = true;
+    HasFastHalfType = true;
     HasFloat16 = true;
   }
   return true;
diff --git a/clang/lib/Basic/Targets/LoongArch.cpp b/clang/lib/Basic/Targets/LoongArch.cpp
index f6915df1520b7..8e29bb745734b 100644
--- a/clang/lib/Basic/Targets/LoongArch.cpp
+++ b/clang/lib/Basic/Targets/LoongArch.cpp
@@ -461,6 +461,8 @@ LoongArchTargetInfo::parseTargetAttr(StringRef Features) const {
 
     case AttrFeatureKind::Feature:
       Ret.Features.push_back("+" + Value.str());
+      if (Value == "lasx")
+        Ret.Features.push_back("+lsx");
       break;
     }
   }
diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp
index 79995ccc21b27..5cf2dc187b836 100644
--- a/clang/lib/Basic/Targets/NVPTX.cpp
+++ b/clang/lib/Basic/Targets/NVPTX.cpp
@@ -65,7 +65,7 @@ NVPTXTargetInfo::NVPTXTargetInfo(const llvm::Triple &Triple,
   GPU = OffloadArch::UNUSED;
 
   // PTX supports f16 as a fundamental type.
-  HasLegalHalfType = true;
+  HasFastHalfType = true;
   HasFloat16 = true;
 
   if (TargetPointerWidth == 32)
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index 94b018a0751d1..a733f6e97b3a4 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -174,6 +174,9 @@ class LLVM_LIBRARY_VISIBILITY DragonFlyBSDTargetInfo
     DefineStd(Builder, "unix", Opts);
     if (this->HasFloat128)
       Builder.defineMacro("__FLOAT128__");
+
+    if (Opts.C11)
+      Builder.defineMacro("__STDC_NO_THREADS__");
   }
 
 public:
diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index ef18354525fac..a6e1ad10568bb 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -89,6 +89,8 @@ bool PPCTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
 }
 
 static void defineXLCompatMacros(MacroBuilder &Builder) {
+  Builder.defineMacro("__builtin_bcdcopysign", "__builtin_ppc_bcdcopysign");
+  Builder.defineMacro("__builtin_bcdsetsign", "__builtin_ppc_bcdsetsign");
   Builder.defineMacro("__builtin_national2packed",
                       "__builtin_ppc_national2packed");
   Builder.defineMacro("__builtin_packed2national",
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index a6a5ec4b325bc..04da4e637af51 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -427,7 +427,7 @@ bool RISCVTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
     ABI = ISAInfo->computeDefaultABI().str();
 
   if (ISAInfo->hasExtension("zfh") || ISAInfo->hasExtension("zhinx"))
-    HasLegalHalfType = true;
+    HasFastHalfType = true;
 
   FastScalarUnalignedAccess =
       llvm::is_contained(Features, "+unaligned-scalar-mem");
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 9d0ced2afdbc7..fb15b77065543 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -106,7 +106,7 @@ class LLVM_LIBRARY_VISIBILITY BaseSPIRTargetInfo : public TargetInfo {
     LongWidth = LongAlign = 64;
     AddrSpaceMap = &SPIRDefIsPrivMap;
     UseAddrSpaceMapMangling = true;
-    HasLegalHalfType = true;
+    HasFastHalfType = true;
     HasFloat16 = true;
     // Define available target features
     // These must be defined in sorted order!
@@ -427,7 +427,7 @@ class LLVM_LIBRARY_VISIBILITY SPIRV64AMDGCNTargetInfo final
     BFloat16Width = BFloat16Align = 16;
     BFloat16Format = &llvm::APFloat::BFloat();
 
-    HasLegalHalfType = true;
+    HasFastHalfType = true;
     HasFloat16 = true;
     HalfArgsAndReturns = true;
 
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
index 7f7dcf815bd8f..dc2185e1b45ca 100644
--- a/clang/lib/Basic/Targets/SystemZ.h
+++ b/clang/lib/Basic/Targets/SystemZ.h
@@ -104,7 +104,7 @@ class LLVM_LIBRARY_VISIBILITY SystemZTargetInfo : public TargetInfo {
     // -ffloat16-excess-precision=none is given, no conversions will be made
     // and instead the backend will promote each half operation to float
     // individually.
-    HasLegalHalfType = false;
+    HasFastHalfType = false;
 
     HasStrictFP = true;
   }
diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp
index 24ecec24d2a4a..f9424cb68ad7e 100644
--- a/clang/lib/Basic/Targets/X86.cpp
+++ b/clang/lib/Basic/Targets/X86.cpp
@@ -348,7 +348,7 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
       HasAVX512BF16 = true;
     } else if (Feature == "+avx512fp16") {
       HasAVX512FP16 = true;
-      HasLegalHalfType = true;
+      HasFastHalfType = true;
     } else if (Feature == "+avx512dq") {
       HasAVX512DQ = true;
     } else if (Feature == "+avx512bitalg") {
@@ -1029,8 +1029,7 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro("__CF__");
   if (HasZU)
     Builder.defineMacro("__ZU__");
-  if (HasEGPR && HasPush2Pop2 && HasPPX && HasNDD && HasCCMP && HasNF &&
-      HasCF && HasZU)
+  if (HasEGPR && HasPush2Pop2 && HasPPX && HasNDD && HasCCMP && HasNF && HasZU)
     Builder.defineMacro("__APX_F__");
   if (HasEGPR && HasInlineAsmUseGPR32)
     Builder.defineMacro("__APX_INLINE_ASM_USE_GPR32__");
diff --git a/clang/lib/CIR/CodeGen/Address.h b/clang/lib/CIR/CodeGen/Address.h
index 6c927e9eda9cc..a851d06321cc1 100644
--- a/clang/lib/CIR/CodeGen/Address.h
+++ b/clang/lib/CIR/CodeGen/Address.h
@@ -68,6 +68,12 @@ class Address {
     return pointerAndKnownNonNull.getPointer() != nullptr;
   }
 
+  /// Return address with different pointer, but same element type and
+  /// alignment.
+  Address withPointer(mlir::Value newPtr) const {
+    return Address(newPtr, getElementType(), getAlignment());
+  }
+
   /// Return address with different element type, a bitcast pointer, and
   /// the same alignment.
   Address withElementType(CIRGenBuilderTy &builder, mlir::Type ElemTy) const;
diff --git a/clang/lib/CIR/CodeGen/CIRGenAsm.cpp b/clang/lib/CIR/CodeGen/CIRGenAsm.cpp
new file mode 100644
index 0000000000000..17dffb3515d2a
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenAsm.cpp
@@ -0,0 +1,136 @@
+//===--- CIRGenAsm.cpp - Inline Assembly Support for CIR CodeGen ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code to emit inline assembly.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenFunction.h"
+#include "clang/CIR/MissingFeatures.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+using namespace cir;
+
+static AsmFlavor inferFlavor(const CIRGenModule &cgm, const AsmStmt &s) {
+  AsmFlavor gnuAsmFlavor =
+      cgm.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT
+          ? AsmFlavor::x86_att
+          : AsmFlavor::x86_intel;
+
+  return isa<MSAsmStmt>(&s) ? AsmFlavor::x86_intel : gnuAsmFlavor;
+}
+
+static void collectClobbers(const CIRGenFunction &cgf, const AsmStmt &s,
+                            std::string &constraints, bool &hasUnwindClobber,
+                            bool &readOnly, bool readNone) {
+
+  hasUnwindClobber = false;
+  const CIRGenModule &cgm = cgf.getCIRGenModule();
+
+  // Clobbers
+  for (unsigned i = 0, e = s.getNumClobbers(); i != e; i++) {
+    std::string clobber = s.getClobber(i);
+    if (clobber == "memory") {
+      readOnly = readNone = false;
+    } else if (clobber == "unwind") {
+      hasUnwindClobber = true;
+      continue;
+    } else if (clobber != "cc") {
+      clobber = cgf.getTarget().getNormalizedGCCRegisterName(clobber);
+      if (cgm.getCodeGenOpts().StackClashProtector &&
+          cgf.getTarget().isSPRegName(clobber))
+        cgm.getDiags().Report(s.getAsmLoc(),
+                              diag::warn_stack_clash_protection_inline_asm);
+    }
+
+    if (isa<MSAsmStmt>(&s)) {
+      if (clobber == "eax" || clobber == "edx") {
+        if (constraints.find("=&A") != std::string::npos)
+          continue;
+        std::string::size_type position1 =
+            constraints.find("={" + clobber + "}");
+        if (position1 != std::string::npos) {
+          constraints.insert(position1 + 1, "&");
+          continue;
+        }
+        std::string::size_type position2 = constraints.find("=A");
+        if (position2 != std::string::npos) {
+          constraints.insert(position2 + 1, "&");
+          continue;
+        }
+      }
+    }
+    if (!constraints.empty())
+      constraints += ',';
+
+    constraints += "~{";
+    constraints += clobber;
+    constraints += '}';
+  }
+
+  // Add machine specific clobbers
+  std::string_view machineClobbers = cgf.getTarget().getClobbers();
+  if (!machineClobbers.empty()) {
+    if (!constraints.empty())
+      constraints += ',';
+    constraints += machineClobbers;
+  }
+}
+
+mlir::LogicalResult CIRGenFunction::emitAsmStmt(const AsmStmt &s) {
+  // Assemble the final asm string.
+  std::string asmString = s.generateAsmString(getContext());
+
+  bool isGCCAsmGoto = false;
+
+  std::string constraints;
+  std::vector<mlir::Value> outArgs;
+  std::vector<mlir::Value> inArgs;
+  std::vector<mlir::Value> inOutArgs;
+
+  // An inline asm can be marked readonly if it meets the following conditions:
+  //  - it doesn't have any sideeffects
+  //  - it doesn't clobber memory
+  //  - it doesn't return a value by-reference
+  // It can be marked readnone if it doesn't have any input memory constraints
+  // in addition to meeting the conditions listed above.
+  bool readOnly = true, readNone = true;
+
+  if (s.getNumInputs() != 0 || s.getNumOutputs() != 0) {
+    assert(!cir::MissingFeatures::asmInputOperands());
+    assert(!cir::MissingFeatures::asmOutputOperands());
+    cgm.errorNYI(s.getAsmLoc(), "asm with operands");
+  }
+
+  bool hasUnwindClobber = false;
+  collectClobbers(*this, s, constraints, hasUnwindClobber, readOnly, readNone);
+
+  std::array<mlir::ValueRange, 3> operands = {outArgs, inArgs, inOutArgs};
+
+  mlir::Type resultType;
+
+  bool hasSideEffect = s.isVolatile() || s.getNumOutputs() == 0;
+
+  cir::InlineAsmOp ia = builder.create<cir::InlineAsmOp>(
+      getLoc(s.getAsmLoc()), resultType, operands, asmString, constraints,
+      hasSideEffect, inferFlavor(cgm, s), mlir::ArrayAttr());
+
+  if (isGCCAsmGoto) {
+    assert(!cir::MissingFeatures::asmGoto());
+  } else if (hasUnwindClobber) {
+    assert(!cir::MissingFeatures::asmUnwindClobber());
+  } else {
+    assert(!cir::MissingFeatures::asmMemoryEffects());
+  }
+
+  llvm::SmallVector<mlir::Attribute> operandAttrs;
+  ia.setOperandAttrsAttr(builder.getArrayAttr(operandAttrs));
+
+  return mlir::success();
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
new file mode 100644
index 0000000000000..80aacb946e7c7
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
@@ -0,0 +1,556 @@
+//===--- CIRGenAtomic.cpp - Emit CIR for atomic operations ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the code for emitting atomic operations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenFunction.h"
+#include "clang/CIR/MissingFeatures.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+using namespace cir;
+
+namespace {
+class AtomicInfo {
+  CIRGenFunction &cgf;
+  QualType atomicTy;
+  QualType valueTy;
+  uint64_t atomicSizeInBits = 0;
+  uint64_t valueSizeInBits = 0;
+  CharUnits atomicAlign;
+  CharUnits valueAlign;
+  TypeEvaluationKind evaluationKind = cir::TEK_Scalar;
+  LValue lvalue;
+  mlir::Location loc;
+
+public:
+  AtomicInfo(CIRGenFunction &cgf, LValue &lvalue, mlir::Location loc)
+      : cgf(cgf), loc(loc) {
+    assert(!lvalue.isGlobalReg());
+    ASTContext &ctx = cgf.getContext();
+    if (lvalue.isSimple()) {
+      atomicTy = lvalue.getType();
+      if (auto *ty = atomicTy->getAs<AtomicType>())
+        valueTy = ty->getValueType();
+      else
+        valueTy = atomicTy;
+      evaluationKind = cgf.getEvaluationKind(valueTy);
+
+      TypeInfo valueTypeInfo = ctx.getTypeInfo(valueTy);
+      TypeInfo atomicTypeInfo = ctx.getTypeInfo(atomicTy);
+      uint64_t valueAlignInBits = valueTypeInfo.Align;
+      uint64_t atomicAlignInBits = atomicTypeInfo.Align;
+      valueSizeInBits = valueTypeInfo.Width;
+      atomicSizeInBits = atomicTypeInfo.Width;
+      assert(valueSizeInBits <= atomicSizeInBits);
+      assert(valueAlignInBits <= atomicAlignInBits);
+
+      atomicAlign = ctx.toCharUnitsFromBits(atomicAlignInBits);
+      valueAlign = ctx.toCharUnitsFromBits(valueAlignInBits);
+      if (lvalue.getAlignment().isZero())
+        lvalue.setAlignment(atomicAlign);
+
+      this->lvalue = lvalue;
+    } else {
+      assert(!cir::MissingFeatures::atomicInfo());
+      cgf.cgm.errorNYI(loc, "AtomicInfo: non-simple lvalue");
+    }
+
+    assert(!cir::MissingFeatures::atomicUseLibCall());
+  }
+
+  QualType getValueType() const { return valueTy; }
+  CharUnits getAtomicAlignment() const { return atomicAlign; }
+  TypeEvaluationKind getEvaluationKind() const { return evaluationKind; }
+  mlir::Value getAtomicPointer() const {
+    if (lvalue.isSimple())
+      return lvalue.getPointer();
+    assert(!cir::MissingFeatures::atomicInfoGetAtomicPointer());
+    return nullptr;
+  }
+  Address getAtomicAddress() const {
+    mlir::Type elemTy;
+    if (lvalue.isSimple()) {
+      elemTy = lvalue.getAddress().getElementType();
+    } else {
+      assert(!cir::MissingFeatures::atomicInfoGetAtomicAddress());
+      cgf.cgm.errorNYI(loc, "AtomicInfo::getAtomicAddress: non-simple lvalue");
+    }
+    return Address(getAtomicPointer(), elemTy, getAtomicAlignment());
+  }
+
+  /// Is the atomic size larger than the underlying value type?
+  ///
+  /// Note that the absence of padding does not mean that atomic
+  /// objects are completely interchangeable with non-atomic
+  /// objects: we might have promoted the alignment of a type
+  /// without making it bigger.
+  bool hasPadding() const { return (valueSizeInBits != atomicSizeInBits); }
+
+  bool emitMemSetZeroIfNecessary() const;
+
+  /// Cast the given pointer to an integer pointer suitable for atomic
+  /// operations on the source.
+  Address castToAtomicIntPointer(Address addr) const;
+
+  /// If addr is compatible with the iN that will be used for an atomic
+  /// operation, bitcast it. Otherwise, create a temporary that is suitable and
+  /// copy the value across.
+  Address convertToAtomicIntPointer(Address addr) const;
+
+  /// Copy an atomic r-value into atomic-layout memory.
+  void emitCopyIntoMemory(RValue rvalue) const;
+
+  /// Project an l-value down to the value field.
+  LValue projectValue() const {
+    assert(lvalue.isSimple());
+    Address addr = getAtomicAddress();
+    if (hasPadding()) {
+      cgf.cgm.errorNYI(loc, "AtomicInfo::projectValue: padding");
+    }
+
+    assert(!cir::MissingFeatures::opTBAA());
+    return LValue::makeAddr(addr, getValueType(), lvalue.getBaseInfo());
+  }
+
+  /// Creates temp alloca for intermediate operations on atomic value.
+  Address createTempAlloca() const;
+
+private:
+  bool requiresMemSetZero(mlir::Type ty) const;
+};
+} // namespace
+
+// This function emits any expression (scalar, complex, or aggregate)
+// into a temporary alloca.
+static Address emitValToTemp(CIRGenFunction &cgf, Expr *e) {
+  Address declPtr = cgf.createMemTemp(
+      e->getType(), cgf.getLoc(e->getSourceRange()), ".atomictmp");
+  cgf.emitAnyExprToMem(e, declPtr, e->getType().getQualifiers(),
+                       /*Init*/ true);
+  return declPtr;
+}
+
+/// Does a store of the given IR type modify the full expected width?
+static bool isFullSizeType(CIRGenModule &cgm, mlir::Type ty,
+                           uint64_t expectedSize) {
+  return cgm.getDataLayout().getTypeStoreSize(ty) * 8 == expectedSize;
+}
+
+/// Does the atomic type require memsetting to zero before initialization?
+///
+/// The IR type is provided as a way of making certain queries faster.
+bool AtomicInfo::requiresMemSetZero(mlir::Type ty) const {
+  // If the atomic type has size padding, we definitely need a memset.
+  if (hasPadding())
+    return true;
+
+  // Otherwise, do some simple heuristics to try to avoid it:
+  switch (getEvaluationKind()) {
+  // For scalars and complexes, check whether the store size of the
+  // type uses the full size.
+  case cir::TEK_Scalar:
+    return !isFullSizeType(cgf.cgm, ty, atomicSizeInBits);
+  case cir::TEK_Complex:
+    cgf.cgm.errorNYI(loc, "AtomicInfo::requiresMemSetZero: complex type");
+    return false;
+
+  // Padding in structs has an undefined bit pattern.  User beware.
+  case cir::TEK_Aggregate:
+    return false;
+  }
+  llvm_unreachable("bad evaluation kind");
+}
+
+Address AtomicInfo::convertToAtomicIntPointer(Address addr) const {
+  mlir::Type ty = addr.getElementType();
+  uint64_t sourceSizeInBits = cgf.cgm.getDataLayout().getTypeSizeInBits(ty);
+  if (sourceSizeInBits != atomicSizeInBits) {
+    cgf.cgm.errorNYI(
+        loc,
+        "AtomicInfo::convertToAtomicIntPointer: convert through temp alloca");
+  }
+
+  return castToAtomicIntPointer(addr);
+}
+
+Address AtomicInfo::createTempAlloca() const {
+  Address tempAlloca = cgf.createMemTemp(
+      (lvalue.isBitField() && valueSizeInBits > atomicSizeInBits) ? valueTy
+                                                                  : atomicTy,
+      getAtomicAlignment(), loc, "atomic-temp");
+
+  // Cast to pointer to value type for bitfields.
+  if (lvalue.isBitField()) {
+    cgf.cgm.errorNYI(loc, "AtomicInfo::createTempAlloca: bitfield lvalue");
+  }
+
+  return tempAlloca;
+}
+
+Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
+  auto intTy = mlir::dyn_cast<cir::IntType>(addr.getElementType());
+  // Don't bother with int casts if the integer size is the same.
+  if (intTy && intTy.getWidth() == atomicSizeInBits)
+    return addr;
+  auto ty = cgf.getBuilder().getUIntNTy(atomicSizeInBits);
+  return addr.withElementType(cgf.getBuilder(), ty);
+}
+
+bool AtomicInfo::emitMemSetZeroIfNecessary() const {
+  assert(lvalue.isSimple());
+  Address addr = lvalue.getAddress();
+  if (!requiresMemSetZero(addr.getElementType()))
+    return false;
+
+  cgf.cgm.errorNYI(loc,
+                   "AtomicInfo::emitMemSetZeroIfNecessary: emit memset zero");
+  return false;
+}
+
+/// Copy an r-value into memory as part of storing to an atomic type.
+/// This needs to create a bit-pattern suitable for atomic operations.
+void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
+  assert(lvalue.isSimple());
+
+  // If we have an r-value, the rvalue should be of the atomic type,
+  // which means that the caller is responsible for having zeroed
+  // any padding.  Just do an aggregate copy of that type.
+  if (rvalue.isAggregate()) {
+    cgf.cgm.errorNYI("copying aggregate into atomic lvalue");
+    return;
+  }
+
+  // Okay, otherwise we're copying stuff.
+
+  // Zero out the buffer if necessary.
+  emitMemSetZeroIfNecessary();
+
+  // Drill past the padding if present.
+  LValue tempLValue = projectValue();
+
+  // Okay, store the rvalue in.
+  if (rvalue.isScalar()) {
+    cgf.emitStoreOfScalar(rvalue.getValue(), tempLValue, /*isInit=*/true);
+  } else {
+    cgf.cgm.errorNYI("copying complex into atomic lvalue");
+  }
+}
+
+static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
+                         Address ptr, Address val1, uint64_t size,
+                         cir::MemOrder order) {
+  std::unique_ptr<AtomicScopeModel> scopeModel = expr->getScopeModel();
+  if (scopeModel) {
+    assert(!cir::MissingFeatures::atomicScope());
+    cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: atomic scope");
+    return;
+  }
+
+  assert(!cir::MissingFeatures::atomicSyncScopeID());
+
+  CIRGenBuilderTy &builder = cgf.getBuilder();
+  mlir::Location loc = cgf.getLoc(expr->getSourceRange());
+  auto orderAttr = cir::MemOrderAttr::get(builder.getContext(), order);
+
+  switch (expr->getOp()) {
+  case AtomicExpr::AO__c11_atomic_init:
+    llvm_unreachable("already handled!");
+
+  case AtomicExpr::AO__c11_atomic_load:
+  case AtomicExpr::AO__atomic_load: {
+    cir::LoadOp load =
+        builder.createLoad(loc, ptr, /*isVolatile=*/expr->isVolatile());
+
+    assert(!cir::MissingFeatures::atomicSyncScopeID());
+
+    load->setAttr("mem_order", orderAttr);
+
+    builder.createStore(loc, load->getResult(0), dest);
+    return;
+  }
+
+  case AtomicExpr::AO__c11_atomic_store:
+  case AtomicExpr::AO__atomic_store: {
+    cir::LoadOp loadVal1 = builder.createLoad(loc, val1);
+
+    assert(!cir::MissingFeatures::atomicSyncScopeID());
+
+    builder.createStore(loc, loadVal1, ptr, expr->isVolatile(),
+                        /*align=*/mlir::IntegerAttr{}, orderAttr);
+    return;
+  }
+
+  case AtomicExpr::AO__opencl_atomic_init:
+
+  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
+  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
+  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
+
+  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
+  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
+  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
+
+  case AtomicExpr::AO__atomic_compare_exchange:
+  case AtomicExpr::AO__atomic_compare_exchange_n:
+  case AtomicExpr::AO__scoped_atomic_compare_exchange:
+  case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
+
+  case AtomicExpr::AO__opencl_atomic_load:
+  case AtomicExpr::AO__hip_atomic_load:
+  case AtomicExpr::AO__atomic_load_n:
+  case AtomicExpr::AO__scoped_atomic_load_n:
+  case AtomicExpr::AO__scoped_atomic_load:
+
+  case AtomicExpr::AO__opencl_atomic_store:
+  case AtomicExpr::AO__hip_atomic_store:
+  case AtomicExpr::AO__atomic_store_n:
+  case AtomicExpr::AO__scoped_atomic_store:
+  case AtomicExpr::AO__scoped_atomic_store_n:
+
+  case AtomicExpr::AO__c11_atomic_exchange:
+  case AtomicExpr::AO__hip_atomic_exchange:
+  case AtomicExpr::AO__opencl_atomic_exchange:
+  case AtomicExpr::AO__atomic_exchange_n:
+  case AtomicExpr::AO__atomic_exchange:
+  case AtomicExpr::AO__scoped_atomic_exchange_n:
+  case AtomicExpr::AO__scoped_atomic_exchange:
+
+  case AtomicExpr::AO__atomic_add_fetch:
+  case AtomicExpr::AO__scoped_atomic_add_fetch:
+
+  case AtomicExpr::AO__c11_atomic_fetch_add:
+  case AtomicExpr::AO__hip_atomic_fetch_add:
+  case AtomicExpr::AO__opencl_atomic_fetch_add:
+  case AtomicExpr::AO__atomic_fetch_add:
+  case AtomicExpr::AO__scoped_atomic_fetch_add:
+
+  case AtomicExpr::AO__atomic_sub_fetch:
+  case AtomicExpr::AO__scoped_atomic_sub_fetch:
+
+  case AtomicExpr::AO__c11_atomic_fetch_sub:
+  case AtomicExpr::AO__hip_atomic_fetch_sub:
+  case AtomicExpr::AO__opencl_atomic_fetch_sub:
+  case AtomicExpr::AO__atomic_fetch_sub:
+  case AtomicExpr::AO__scoped_atomic_fetch_sub:
+
+  case AtomicExpr::AO__atomic_min_fetch:
+  case AtomicExpr::AO__scoped_atomic_min_fetch:
+
+  case AtomicExpr::AO__c11_atomic_fetch_min:
+  case AtomicExpr::AO__hip_atomic_fetch_min:
+  case AtomicExpr::AO__opencl_atomic_fetch_min:
+  case AtomicExpr::AO__atomic_fetch_min:
+  case AtomicExpr::AO__scoped_atomic_fetch_min:
+
+  case AtomicExpr::AO__atomic_max_fetch:
+  case AtomicExpr::AO__scoped_atomic_max_fetch:
+
+  case AtomicExpr::AO__c11_atomic_fetch_max:
+  case AtomicExpr::AO__hip_atomic_fetch_max:
+  case AtomicExpr::AO__opencl_atomic_fetch_max:
+  case AtomicExpr::AO__atomic_fetch_max:
+  case AtomicExpr::AO__scoped_atomic_fetch_max:
+
+  case AtomicExpr::AO__atomic_and_fetch:
+  case AtomicExpr::AO__scoped_atomic_and_fetch:
+
+  case AtomicExpr::AO__c11_atomic_fetch_and:
+  case AtomicExpr::AO__hip_atomic_fetch_and:
+  case AtomicExpr::AO__opencl_atomic_fetch_and:
+  case AtomicExpr::AO__atomic_fetch_and:
+  case AtomicExpr::AO__scoped_atomic_fetch_and:
+
+  case AtomicExpr::AO__atomic_or_fetch:
+  case AtomicExpr::AO__scoped_atomic_or_fetch:
+
+  case AtomicExpr::AO__c11_atomic_fetch_or:
+  case AtomicExpr::AO__hip_atomic_fetch_or:
+  case AtomicExpr::AO__opencl_atomic_fetch_or:
+  case AtomicExpr::AO__atomic_fetch_or:
+  case AtomicExpr::AO__scoped_atomic_fetch_or:
+
+  case AtomicExpr::AO__atomic_xor_fetch:
+  case AtomicExpr::AO__scoped_atomic_xor_fetch:
+
+  case AtomicExpr::AO__c11_atomic_fetch_xor:
+  case AtomicExpr::AO__hip_atomic_fetch_xor:
+  case AtomicExpr::AO__opencl_atomic_fetch_xor:
+  case AtomicExpr::AO__atomic_fetch_xor:
+  case AtomicExpr::AO__scoped_atomic_fetch_xor:
+
+  case AtomicExpr::AO__atomic_nand_fetch:
+  case AtomicExpr::AO__scoped_atomic_nand_fetch:
+
+  case AtomicExpr::AO__c11_atomic_fetch_nand:
+  case AtomicExpr::AO__atomic_fetch_nand:
+  case AtomicExpr::AO__scoped_atomic_fetch_nand:
+
+  case AtomicExpr::AO__atomic_test_and_set:
+
+  case AtomicExpr::AO__atomic_clear:
+    cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: expr op NYI");
+    break;
+  }
+}
+
+static bool isMemOrderValid(uint64_t order, bool isStore, bool isLoad) {
+  if (!cir::isValidCIRAtomicOrderingCABI(order))
+    return false;
+  auto memOrder = static_cast<cir::MemOrder>(order);
+  if (isStore)
+    return memOrder != cir::MemOrder::Consume &&
+           memOrder != cir::MemOrder::Acquire &&
+           memOrder != cir::MemOrder::AcquireRelease;
+  if (isLoad)
+    return memOrder != cir::MemOrder::Release &&
+           memOrder != cir::MemOrder::AcquireRelease;
+  return true;
+}
+
+RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
+  QualType atomicTy = e->getPtr()->getType()->getPointeeType();
+  QualType memTy = atomicTy;
+  if (const auto *ty = atomicTy->getAs<AtomicType>())
+    memTy = ty->getValueType();
+
+  Address val1 = Address::invalid();
+  Address dest = Address::invalid();
+  Address ptr = emitPointerWithAlignment(e->getPtr());
+
+  assert(!cir::MissingFeatures::openCL());
+  if (e->getOp() == AtomicExpr::AO__c11_atomic_init) {
+    LValue lvalue = makeAddrLValue(ptr, atomicTy);
+    emitAtomicInit(e->getVal1(), lvalue);
+    return RValue::get(nullptr);
+  }
+
+  TypeInfoChars typeInfo = getContext().getTypeInfoInChars(atomicTy);
+  uint64_t size = typeInfo.Width.getQuantity();
+
+  Expr::EvalResult orderConst;
+  mlir::Value order;
+  if (!e->getOrder()->EvaluateAsInt(orderConst, getContext()))
+    order = emitScalarExpr(e->getOrder());
+
+  bool shouldCastToIntPtrTy = true;
+
+  switch (e->getOp()) {
+  default:
+    cgm.errorNYI(e->getSourceRange(), "atomic op NYI");
+    return RValue::get(nullptr);
+
+  case AtomicExpr::AO__c11_atomic_init:
+    llvm_unreachable("already handled above with emitAtomicInit");
+
+  case AtomicExpr::AO__c11_atomic_load:
+    break;
+
+  case AtomicExpr::AO__atomic_load:
+    dest = emitPointerWithAlignment(e->getVal1());
+    break;
+
+  case AtomicExpr::AO__atomic_store:
+    val1 = emitPointerWithAlignment(e->getVal1());
+    break;
+
+  case AtomicExpr::AO__c11_atomic_store:
+    val1 = emitValToTemp(*this, e->getVal1());
+    break;
+  }
+
+  QualType resultTy = e->getType().getUnqualifiedType();
+
+  // The inlined atomics only function on iN types, where N is a power of 2. We
+  // need to make sure (via temporaries if necessary) that all incoming values
+  // are compatible.
+  LValue atomicValue = makeAddrLValue(ptr, atomicTy);
+  AtomicInfo atomics(*this, atomicValue, getLoc(e->getSourceRange()));
+
+  if (shouldCastToIntPtrTy) {
+    ptr = atomics.castToAtomicIntPointer(ptr);
+    if (val1.isValid())
+      val1 = atomics.convertToAtomicIntPointer(val1);
+  }
+  if (dest.isValid()) {
+    if (shouldCastToIntPtrTy)
+      dest = atomics.castToAtomicIntPointer(dest);
+  } else if (!resultTy->isVoidType()) {
+    dest = atomics.createTempAlloca();
+    if (shouldCastToIntPtrTy)
+      dest = atomics.castToAtomicIntPointer(dest);
+  }
+
+  bool powerOf2Size = (size & (size - 1)) == 0;
+  bool useLibCall = !powerOf2Size || (size > 16);
+
+  // For atomics larger than 16 bytes, emit a libcall from the frontend. This
+  // avoids the overhead of dealing with excessively-large value types in IR.
+  // Non-power-of-2 values also lower to libcall here, as they are not currently
+  // permitted in IR instructions (although that constraint could be relaxed in
+  // the future). For other cases where a libcall is required on a given
+  // platform, we let the backend handle it (this includes handling for all of
+  // the size-optimized libcall variants, which are only valid up to 16 bytes.)
+  //
+  // See: https://llvm.org/docs/Atomics.html#libcalls-atomic
+  if (useLibCall) {
+    assert(!cir::MissingFeatures::atomicUseLibCall());
+    cgm.errorNYI(e->getSourceRange(), "emitAtomicExpr: emit atomic lib call");
+    return RValue::get(nullptr);
+  }
+
+  bool isStore = e->getOp() == AtomicExpr::AO__c11_atomic_store ||
+                 e->getOp() == AtomicExpr::AO__atomic_store;
+  bool isLoad = e->getOp() == AtomicExpr::AO__c11_atomic_load ||
+                e->getOp() == AtomicExpr::AO__atomic_load;
+
+  if (!order) {
+    // We have evaluated the memory order as an integer constant in orderConst.
+    // We should not ever get to a case where the ordering isn't a valid CABI
+    // value, but it's hard to enforce that in general.
+    uint64_t ord = orderConst.Val.getInt().getZExtValue();
+    if (isMemOrderValid(ord, isStore, isLoad))
+      emitAtomicOp(*this, e, dest, ptr, val1, size,
+                   static_cast<cir::MemOrder>(ord));
+  } else {
+    assert(!cir::MissingFeatures::atomicExpr());
+    cgm.errorNYI(e->getSourceRange(), "emitAtomicExpr: dynamic memory order");
+    return RValue::get(nullptr);
+  }
+
+  if (resultTy->isVoidType())
+    return RValue::get(nullptr);
+
+  return convertTempToRValue(
+      dest.withElementType(builder, convertTypeForMem(resultTy)), resultTy,
+      e->getExprLoc());
+}
+
+void CIRGenFunction::emitAtomicInit(Expr *init, LValue dest) {
+  AtomicInfo atomics(*this, dest, getLoc(init->getSourceRange()));
+
+  switch (atomics.getEvaluationKind()) {
+  case cir::TEK_Scalar: {
+    mlir::Value value = emitScalarExpr(init);
+    atomics.emitCopyIntoMemory(RValue::get(value));
+    return;
+  }
+
+  case cir::TEK_Complex:
+    cgm.errorNYI(init->getSourceRange(), "emitAtomicInit: complex type");
+    return;
+
+  case cir::TEK_Aggregate:
+    cgm.errorNYI(init->getSourceRange(), "emitAtomicInit: aggregate type");
+    return;
+  }
+
+  llvm_unreachable("bad evaluation kind");
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
index 4a5a1dd53a05a..755c76c89a645 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CIRGenBuilder.h"
+#include "llvm/ADT/TypeSwitch.h"
 
 using namespace clang::CIRGen;
 
@@ -66,6 +67,69 @@ clang::CIRGen::CIRGenBuilderTy::getConstFP(mlir::Location loc, mlir::Type t,
   return create<cir::ConstantOp>(loc, cir::FPAttr::get(t, fpVal));
 }
 
+void CIRGenBuilderTy::computeGlobalViewIndicesFromFlatOffset(
+    int64_t offset, mlir::Type ty, cir::CIRDataLayout layout,
+    llvm::SmallVectorImpl<int64_t> &indices) {
+  if (!offset)
+    return;
+
+  auto getIndexAndNewOffset =
+      [](int64_t offset, int64_t eltSize) -> std::pair<int64_t, int64_t> {
+    int64_t divRet = offset / eltSize;
+    if (divRet < 0)
+      divRet -= 1; // make sure offset is positive
+    int64_t modRet = offset - (divRet * eltSize);
+    return {divRet, modRet};
+  };
+
+  mlir::Type subType =
+      llvm::TypeSwitch<mlir::Type, mlir::Type>(ty)
+          .Case<cir::ArrayType>([&](auto arrayTy) {
+            int64_t eltSize = layout.getTypeAllocSize(arrayTy.getElementType());
+            const auto [index, newOffset] =
+                getIndexAndNewOffset(offset, eltSize);
+            indices.push_back(index);
+            offset = newOffset;
+            return arrayTy.getElementType();
+          })
+          .Case<cir::RecordType>([&](auto recordTy) {
+            ArrayRef<mlir::Type> elts = recordTy.getMembers();
+            int64_t pos = 0;
+            for (size_t i = 0; i < elts.size(); ++i) {
+              int64_t eltSize =
+                  (int64_t)layout.getTypeAllocSize(elts[i]).getFixedValue();
+              unsigned alignMask = layout.getABITypeAlign(elts[i]).value() - 1;
+              if (recordTy.getPacked())
+                alignMask = 0;
+              // Union's fields have the same offset, so no need to change pos
+              // here, we just need to find eltSize that is greater then the
+              // required offset. The same is true for the similar union type
+              // check below
+              if (!recordTy.isUnion())
+                pos = (pos + alignMask) & ~alignMask;
+              assert(offset >= 0);
+              if (offset < pos + eltSize) {
+                indices.push_back(i);
+                offset -= pos;
+                return elts[i];
+              }
+              // No need to update pos here, see the comment above.
+              if (!recordTy.isUnion())
+                pos += eltSize;
+            }
+            llvm_unreachable("offset was not found within the record");
+          })
+          .Default([](mlir::Type otherTy) {
+            llvm_unreachable("unexpected type");
+            return otherTy; // Even though this is unreachable, we need to
+                            // return a type to satisfy the return type of the
+                            // lambda.
+          });
+
+  assert(subType);
+  computeGlobalViewIndicesFromFlatOffset(offset, subType, layout, indices);
+}
+
 // This can't be defined in Address.h because that file is included by
 // CIRGenBuilder.h
 Address Address::withElementType(CIRGenBuilderTy &builder,
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
index ff8e12190c972..d5cb6d416bc9b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h
+++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h
@@ -12,6 +12,7 @@
 #include "Address.h"
 #include "CIRGenRecordLayout.h"
 #include "CIRGenTypeCache.h"
+#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
 #include "clang/CIR/Interfaces/CIRTypeInterfaces.h"
 #include "clang/CIR/MissingFeatures.h"
 
@@ -59,6 +60,23 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
         trailingZerosNum);
   }
 
+  cir::ConstRecordAttr getAnonConstRecord(mlir::ArrayAttr arrayAttr,
+                                          bool packed = false,
+                                          bool padded = false,
+                                          mlir::Type ty = {}) {
+    llvm::SmallVector<mlir::Type, 4> members;
+    for (auto &f : arrayAttr) {
+      auto ta = mlir::cast<mlir::TypedAttr>(f);
+      members.push_back(ta.getType());
+    }
+
+    if (!ty)
+      ty = getAnonRecordTy(members, packed, padded);
+
+    auto sTy = mlir::cast<cir::RecordType>(ty);
+    return cir::ConstRecordAttr::get(sTy, arrayAttr);
+  }
+
   std::string getUniqueAnonRecordName() { return getUniqueRecordName("anon"); }
 
   std::string getUniqueRecordName(const std::string &baseName) {
@@ -83,6 +101,10 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     llvm_unreachable("Unsupported format for long double");
   }
 
+  mlir::Type getPtrToVPtrType() {
+    return getPointerTo(cir::VPtrType::get(getContext()));
+  }
+
   /// Get a CIR record kind from a AST declaration tag.
   cir::RecordType::RecordKind getRecordKind(const clang::TagTypeKind kind) {
     switch (kind) {
@@ -244,6 +266,17 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   }
   bool isInt(mlir::Type i) { return mlir::isa<cir::IntType>(i); }
 
+  // Fetch the type representing a pointer to unsigned int8 values.
+  cir::PointerType getUInt8PtrTy() { return typeCache.UInt8PtrTy; }
+
+  /// Get a CIR anonymous record type.
+  cir::RecordType getAnonRecordTy(llvm::ArrayRef<mlir::Type> members,
+                                  bool packed = false, bool padded = false) {
+    assert(!cir::MissingFeatures::astRecordDeclAttr());
+    auto kind = cir::RecordType::RecordKind::Struct;
+    return getType<cir::RecordType>(members, packed, padded, kind);
+  }
+
   //
   // Constant creation helpers
   // -------------------------
@@ -251,11 +284,14 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   cir::ConstantOp getSInt32(int32_t c, mlir::Location loc) {
     return getConstantInt(loc, getSInt32Ty(), c);
   }
+  cir::ConstantOp getUInt32(uint32_t c, mlir::Location loc) {
+    return getConstantInt(loc, getUInt32Ty(), c);
+  }
 
   // Creates constant nullptr for pointer type ty.
   cir::ConstantOp getNullPtr(mlir::Type ty, mlir::Location loc) {
     assert(!cir::MissingFeatures::targetCodeGenInfoGetNullPointer());
-    return create<cir::ConstantOp>(loc, getConstPtrAttr(ty, 0));
+    return cir::ConstantOp::create(*this, loc, getConstPtrAttr(ty, 0));
   }
 
   mlir::Value createNeg(mlir::Value value) {
@@ -264,7 +300,7 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
       // Source is a unsigned integer: first cast it to signed.
       if (intTy.isUnsigned())
         value = createIntCast(value, getSIntNTy(intTy.getWidth()));
-      return create<cir::UnaryOp>(value.getLoc(), value.getType(),
+      return cir::UnaryOp::create(*this, value.getLoc(), value.getType(),
                                   cir::UnaryOpKind::Minus, value);
     }
 
@@ -276,8 +312,8 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   mlir::Value createFloatingCast(mlir::Value v, mlir::Type destType) {
     assert(!cir::MissingFeatures::fpConstraints());
 
-    return create<cir::CastOp>(v.getLoc(), destType, cir::CastKind::floating,
-                               v);
+    return cir::CastOp::create(*this, v.getLoc(), destType,
+                               cir::CastKind::floating, v);
   }
 
   mlir::Value createFSub(mlir::Location loc, mlir::Value lhs, mlir::Value rhs) {
@@ -285,7 +321,7 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     assert(!cir::MissingFeatures::fpConstraints());
     assert(!cir::MissingFeatures::fastMathFlags());
 
-    return create<cir::BinOp>(loc, cir::BinOpKind::Sub, lhs, rhs);
+    return cir::BinOp::create(*this, loc, cir::BinOpKind::Sub, lhs, rhs);
   }
 
   mlir::Value createFAdd(mlir::Location loc, mlir::Value lhs, mlir::Value rhs) {
@@ -293,21 +329,21 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     assert(!cir::MissingFeatures::fpConstraints());
     assert(!cir::MissingFeatures::fastMathFlags());
 
-    return create<cir::BinOp>(loc, cir::BinOpKind::Add, lhs, rhs);
+    return cir::BinOp::create(*this, loc, cir::BinOpKind::Add, lhs, rhs);
   }
   mlir::Value createFMul(mlir::Location loc, mlir::Value lhs, mlir::Value rhs) {
     assert(!cir::MissingFeatures::metaDataNode());
     assert(!cir::MissingFeatures::fpConstraints());
     assert(!cir::MissingFeatures::fastMathFlags());
 
-    return create<cir::BinOp>(loc, cir::BinOpKind::Mul, lhs, rhs);
+    return cir::BinOp::create(*this, loc, cir::BinOpKind::Mul, lhs, rhs);
   }
   mlir::Value createFDiv(mlir::Location loc, mlir::Value lhs, mlir::Value rhs) {
     assert(!cir::MissingFeatures::metaDataNode());
     assert(!cir::MissingFeatures::fpConstraints());
     assert(!cir::MissingFeatures::fastMathFlags());
 
-    return create<cir::BinOp>(loc, cir::BinOpKind::Div, lhs, rhs);
+    return cir::BinOp::create(*this, loc, cir::BinOpKind::Div, lhs, rhs);
   }
 
   Address createBaseClassAddr(mlir::Location loc, Address addr,
@@ -317,8 +353,9 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
       return addr;
 
     auto ptrTy = getPointerTo(destType);
-    auto baseAddr = create<cir::BaseClassAddrOp>(
-        loc, ptrTy, addr.getPointer(), mlir::APInt(64, offset), assumeNotNull);
+    auto baseAddr =
+        cir::BaseClassAddrOp::create(*this, loc, ptrTy, addr.getPointer(),
+                                     mlir::APInt(64, offset), assumeNotNull);
     return Address(baseAddr, destType, addr.getAlignment());
   }
 
@@ -337,15 +374,19 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   cir::LoadOp createLoad(mlir::Location loc, Address addr,
                          bool isVolatile = false) {
     mlir::IntegerAttr align = getAlignmentAttr(addr.getAlignment());
-    return create<cir::LoadOp>(loc, addr.getPointer(), /*isDeref=*/false,
-                               align);
+    return cir::LoadOp::create(*this, loc, addr.getPointer(), /*isDeref=*/false,
+                               /*alignment=*/align,
+                               /*mem_order=*/cir::MemOrderAttr{});
   }
 
   cir::StoreOp createStore(mlir::Location loc, mlir::Value val, Address dst,
-                           mlir::IntegerAttr align = {}) {
+                           bool isVolatile = false,
+                           mlir::IntegerAttr align = {},
+                           cir::MemOrderAttr order = {}) {
     if (!align)
       align = getAlignmentAttr(dst.getAlignment());
-    return CIRBaseBuilderTy::createStore(loc, val, dst.getPointer(), align);
+    return CIRBaseBuilderTy::createStore(loc, val, dst.getPointer(), isVolatile,
+                                         align, order);
   }
 
   /// Create a cir.complex.real_ptr operation that derives a pointer to the real
@@ -353,8 +394,8 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   mlir::Value createComplexRealPtr(mlir::Location loc, mlir::Value value) {
     auto srcPtrTy = mlir::cast<cir::PointerType>(value.getType());
     auto srcComplexTy = mlir::cast<cir::ComplexType>(srcPtrTy.getPointee());
-    return create<cir::ComplexRealPtrOp>(
-        loc, getPointerTo(srcComplexTy.getElementType()), value);
+    return cir::ComplexRealPtrOp::create(
+        *this, loc, getPointerTo(srcComplexTy.getElementType()), value);
   }
 
   Address createComplexRealPtr(mlir::Location loc, Address addr) {
@@ -368,8 +409,8 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   mlir::Value createComplexImagPtr(mlir::Location loc, mlir::Value value) {
     auto srcPtrTy = mlir::cast<cir::PointerType>(value.getType());
     auto srcComplexTy = mlir::cast<cir::ComplexType>(srcPtrTy.getPointee());
-    return create<cir::ComplexImagPtrOp>(
-        loc, getPointerTo(srcComplexTy.getElementType()), value);
+    return cir::ComplexImagPtrOp::create(
+        *this, loc, getPointerTo(srcComplexTy.getElementType()), value);
   }
 
   Address createComplexImagPtr(mlir::Location loc, Address addr) {
@@ -390,12 +431,20 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   mlir::Value maybeBuildArrayDecay(mlir::Location loc, mlir::Value arrayPtr,
                                    mlir::Type eltTy);
 
+  // Convert byte offset to sequence of high-level indices suitable for
+  // GlobalViewAttr. Ideally we shouldn't deal with low-level offsets at all
+  // but currently some parts of Clang AST, which we don't want to touch just
+  // yet, return them.
+  void computeGlobalViewIndicesFromFlatOffset(
+      int64_t offset, mlir::Type ty, cir::CIRDataLayout layout,
+      llvm::SmallVectorImpl<int64_t> &indices);
+
   /// Creates a versioned global variable. If the symbol is already taken, an ID
   /// will be appended to the symbol. The returned global must always be queried
   /// for its name so it can be referenced correctly.
   [[nodiscard]] cir::GlobalOp
   createVersionedGlobal(mlir::ModuleOp module, mlir::Location loc,
-                        mlir::StringRef name, mlir::Type type,
+                        mlir::StringRef name, mlir::Type type, bool isConstant,
                         cir::GlobalLinkageKind linkage) {
     // Create a unique name if the given name is already taken.
     std::string uniqueName;
@@ -404,7 +453,7 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
     else
       uniqueName = name.str();
 
-    return createGlobal(module, loc, uniqueName, type, linkage);
+    return createGlobal(module, loc, uniqueName, type, isConstant, linkage);
   }
 
   mlir::Value createSetBitfield(mlir::Location loc, mlir::Type resultType,
@@ -419,9 +468,9 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
         useVolatile ? cir::IntType::get(storageType.getContext(),
                                         info.volatileStorageSize, info.isSigned)
                     : storageType;
-    return create<cir::SetBitfieldOp>(
-        loc, resultType, dstAddr.getPointer(), storageType, src, info.name,
-        info.size, offset, info.isSigned, isLvalueVolatile,
+    return cir::SetBitfieldOp::create(
+        *this, loc, resultType, dstAddr.getPointer(), storageType, src,
+        info.name, info.size, offset, info.isSigned, isLvalueVolatile,
         dstAddr.getAlignment().getAsAlign().value());
   }
 
@@ -437,7 +486,7 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
         useVolatile ? cir::IntType::get(storageType.getContext(),
                                         info.volatileStorageSize, info.isSigned)
                     : storageType;
-    return create<cir::GetBitfieldOp>(loc, resultType, addr.getPointer(),
+    return cir::GetBitfieldOp::create(*this, loc, resultType, addr.getPointer(),
                                       storageType, info.name, info.size, offset,
                                       info.isSigned, isLvalueVolatile,
                                       addr.getAlignment().getAsAlign().value());
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 7767bf481c795..b6a6299667308 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -72,6 +72,19 @@ RValue CIRGenFunction::emitRotate(const CallExpr *e, bool isRotateLeft) {
   return RValue::get(r);
 }
 
+template <class Operation>
+static RValue emitUnaryMaybeConstrainedFPBuiltin(CIRGenFunction &cgf,
+                                                 const CallExpr &e) {
+  mlir::Value arg = cgf.emitScalarExpr(e.getArg(0));
+
+  assert(!cir::MissingFeatures::cgFPOptionsRAII());
+  assert(!cir::MissingFeatures::fpConstraints());
+
+  auto call =
+      Operation::create(cgf.getBuilder(), arg.getLoc(), arg.getType(), arg);
+  return RValue::get(call->getResult(0));
+}
+
 RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
                                        const CallExpr *e,
                                        ReturnValueSlot returnValue) {
@@ -112,6 +125,32 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
   default:
     break;
 
+  // C stdarg builtins.
+  case Builtin::BI__builtin_stdarg_start:
+  case Builtin::BI__builtin_va_start:
+  case Builtin::BI__va_start: {
+    mlir::Value vaList = builtinID == Builtin::BI__va_start
+                             ? emitScalarExpr(e->getArg(0))
+                             : emitVAListRef(e->getArg(0)).getPointer();
+    mlir::Value count = emitScalarExpr(e->getArg(1));
+    emitVAStart(vaList, count);
+    return {};
+  }
+
+  case Builtin::BI__builtin_va_end:
+    emitVAEnd(emitVAListRef(e->getArg(0)).getPointer());
+    return {};
+
+  case Builtin::BIfabs:
+  case Builtin::BIfabsf:
+  case Builtin::BIfabsl:
+  case Builtin::BI__builtin_fabs:
+  case Builtin::BI__builtin_fabsf:
+  case Builtin::BI__builtin_fabsf16:
+  case Builtin::BI__builtin_fabsl:
+  case Builtin::BI__builtin_fabsf128:
+    return emitUnaryMaybeConstrainedFPBuiltin<cir::FAbsOp>(*this, *e);
+
   case Builtin::BI__assume:
   case Builtin::BI__builtin_assume: {
     if (e->getArg(0)->HasSideEffects(getContext()))
@@ -129,6 +168,24 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
     return RValue::get(nullptr);
   }
 
+  case Builtin::BI__builtin_assume_aligned: {
+    const Expr *ptrExpr = e->getArg(0);
+    mlir::Value ptrValue = emitScalarExpr(ptrExpr);
+    mlir::Value offsetValue =
+        (e->getNumArgs() > 2) ? emitScalarExpr(e->getArg(2)) : nullptr;
+
+    std::optional<llvm::APSInt> alignment =
+        e->getArg(1)->getIntegerConstantExpr(getContext());
+    assert(alignment.has_value() &&
+           "the second argument to __builtin_assume_aligned must be an "
+           "integral constant expression");
+
+    mlir::Value result =
+        emitAlignmentAssumption(ptrValue, ptrExpr, ptrExpr->getExprLoc(),
+                                alignment->getSExtValue(), offsetValue);
+    return RValue::get(result);
+  }
+
   case Builtin::BI__builtin_complex: {
     mlir::Value real = emitScalarExpr(e->getArg(0));
     mlir::Value imag = emitScalarExpr(e->getArg(1));
@@ -271,6 +328,20 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
   case Builtin::BI__builtin_rotateright64:
     return emitRotate(e, /*isRotateLeft=*/false);
 
+  case Builtin::BI__builtin_return_address:
+  case Builtin::BI__builtin_frame_address: {
+    mlir::Location loc = getLoc(e->getExprLoc());
+    llvm::APSInt level = e->getArg(0)->EvaluateKnownConstInt(getContext());
+    if (builtinID == Builtin::BI__builtin_return_address) {
+      return RValue::get(cir::ReturnAddrOp::create(
+          builder, loc,
+          builder.getConstAPInt(loc, builder.getUInt32Ty(), level)));
+    }
+    return RValue::get(cir::FrameAddrOp::create(
+        builder, loc,
+        builder.getConstAPInt(loc, builder.getUInt32Ty(), level)));
+  }
+
   case Builtin::BI__builtin_trap:
     emitTrap(loc, /*createNewBlock=*/true);
     return RValue::get(nullptr);
@@ -320,3 +391,25 @@ mlir::Value CIRGenFunction::emitCheckedArgForAssume(const Expr *e) {
                "emitCheckedArgForAssume: sanitizers are NYI");
   return {};
 }
+
+void CIRGenFunction::emitVAStart(mlir::Value vaList, mlir::Value count) {
+  // LLVM codegen casts to *i8, no real gain on doing this for CIRGen this
+  // early, defer to LLVM lowering.
+  cir::VAStartOp::create(builder, vaList.getLoc(), vaList, count);
+}
+
+void CIRGenFunction::emitVAEnd(mlir::Value vaList) {
+  cir::VAEndOp::create(builder, vaList.getLoc(), vaList);
+}
+
+// FIXME(cir): This completely abstracts away the ABI with a generic CIR Op. By
+// default this lowers to llvm.va_arg which is incomplete and not ABI-compliant
+// on most targets so cir.va_arg will need some ABI handling in LoweringPrepare
+mlir::Value CIRGenFunction::emitVAArg(VAArgExpr *ve) {
+  assert(!cir::MissingFeatures::msabi());
+  assert(!cir::MissingFeatures::vlas());
+  mlir::Location loc = cgm.getLoc(ve->getExprLoc());
+  mlir::Type type = convertType(ve->getType());
+  mlir::Value vaList = emitVAListRef(ve->getSubExpr()).getPointer();
+  return cir::VAArgOp::create(builder, loc, type, vaList);
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenCXXABI.h b/clang/lib/CIR/CodeGen/CIRGenCXXABI.h
index 5929568505ef2..b5f2e1a067274 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCXXABI.h
+++ b/clang/lib/CIR/CodeGen/CIRGenCXXABI.h
@@ -63,6 +63,16 @@ class CIRGenCXXABI {
   /// parameter.
   virtual bool needsVTTParameter(clang::GlobalDecl gd) { return false; }
 
+  /// Perform ABI-specific "this" argument adjustment required prior to
+  /// a call of a virtual function.
+  /// The "VirtualCall" argument is true iff the call itself is virtual.
+  virtual Address adjustThisArgumentForVirtualFunctionCall(CIRGenFunction &cgf,
+                                                           clang::GlobalDecl gd,
+                                                           Address thisPtr,
+                                                           bool virtualCall) {
+    return thisPtr;
+  }
+
   /// Build a parameter variable suitable for 'this'.
   void buildThisParam(CIRGenFunction &cgf, FunctionArgList &params);
 
@@ -80,6 +90,15 @@ class CIRGenCXXABI {
                                   bool forVirtualBase, bool delegating,
                                   Address thisAddr, QualType thisTy) = 0;
 
+  /// Checks if ABI requires extra virtual offset for vtable field.
+  virtual bool
+  isVirtualOffsetNeededForVTableField(CIRGenFunction &cgf,
+                                      CIRGenFunction::VPtr vptr) = 0;
+
+  /// Emits the VTable definitions required for the given record type.
+  virtual void emitVTableDefinitions(CIRGenVTables &cgvt,
+                                     const CXXRecordDecl *rd) = 0;
+
   /// Returns true if the given destructor type should be emitted as a linkonce
   /// delegating thunk, regardless of whether the dtor is defined in this TU or
   /// not.
@@ -90,6 +109,33 @@ class CIRGenCXXABI {
   getCXXDestructorLinkage(GVALinkage linkage, const CXXDestructorDecl *dtor,
                           CXXDtorType dt) const;
 
+  /// Get the address of the vtable for the given record decl which should be
+  /// used for the vptr at the given offset in RD.
+  virtual cir::GlobalOp getAddrOfVTable(const CXXRecordDecl *rd,
+                                        CharUnits vptrOffset) = 0;
+
+  /// Build a virtual function pointer in the ABI-specific way.
+  virtual CIRGenCallee getVirtualFunctionPointer(CIRGenFunction &cgf,
+                                                 clang::GlobalDecl gd,
+                                                 Address thisAddr,
+                                                 mlir::Type ty,
+                                                 SourceLocation loc) = 0;
+
+  /// Get the address point of the vtable for the given base subobject.
+  virtual mlir::Value
+  getVTableAddressPoint(BaseSubobject base,
+                        const CXXRecordDecl *vtableClass) = 0;
+
+  /// Get the address point of the vtable for the given base subobject while
+  /// building a constructor or a destructor.
+  virtual mlir::Value getVTableAddressPointInStructor(
+      CIRGenFunction &cgf, const CXXRecordDecl *vtableClass, BaseSubobject base,
+      const CXXRecordDecl *nearestVBase) = 0;
+
+  /// Checks if ABI requires to initialize vptrs for given dynamic class.
+  virtual bool
+  doStructorsInitializeVPtrs(const clang::CXXRecordDecl *vtableClass) = 0;
+
   /// Returns true if the given constructor or destructor is one of the kinds
   /// that the ABI says returns 'this' (only applies when called non-virtually
   /// for destructors).
diff --git a/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp
index 67d8988a5fbbd..c9e4ed92d16bb 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCXXExpr.cpp
@@ -75,15 +75,14 @@ static MemberCallInfo commonBuildCXXMemberOrOperatorCall(
 
 RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
     const CallExpr *ce, const CXXMethodDecl *md, ReturnValueSlot returnValue,
-    bool hasQualifier, NestedNameSpecifier *qualifier, bool isArrow,
+    bool hasQualifier, NestedNameSpecifier qualifier, bool isArrow,
     const Expr *base) {
   assert(isa<CXXMemberCallExpr>(ce) || isa<CXXOperatorCallExpr>(ce));
 
-  if (md->isVirtual()) {
-    cgm.errorNYI(ce->getSourceRange(),
-                 "emitCXXMemberOrOperatorMemberCallExpr: virtual call");
-    return RValue::get(nullptr);
-  }
+  // Compute the object pointer.
+  bool canUseVirtualCall = md->isVirtual() && !hasQualifier;
+  const CXXMethodDecl *devirtualizedMethod = nullptr;
+  assert(!cir::MissingFeatures::devirtualizeMemberFunction());
 
   // Note on trivial assignment
   // --------------------------
@@ -127,7 +126,8 @@ RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
     return RValue::get(nullptr);
 
   // Compute the function type we're calling
-  const CXXMethodDecl *calleeDecl = md;
+  const CXXMethodDecl *calleeDecl =
+      devirtualizedMethod ? devirtualizedMethod : md;
   const CIRGenFunctionInfo *fInfo = nullptr;
   if (isa<CXXDestructorDecl>(calleeDecl)) {
     cgm.errorNYI(ce->getSourceRange(),
@@ -137,25 +137,46 @@ RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
 
   fInfo = &cgm.getTypes().arrangeCXXMethodDeclaration(calleeDecl);
 
-  mlir::Type ty = cgm.getTypes().getFunctionType(*fInfo);
+  cir::FuncType ty = cgm.getTypes().getFunctionType(*fInfo);
 
   assert(!cir::MissingFeatures::sanitizers());
   assert(!cir::MissingFeatures::emitTypeCheck());
 
+  // C++ [class.virtual]p12:
+  //   Explicit qualification with the scope operator (5.1) suppresses the
+  //   virtual call mechanism.
+  //
+  // We also don't emit a virtual call if the base expression has a record type
+  // because then we know what the type is.
+  bool useVirtualCall = canUseVirtualCall && !devirtualizedMethod;
+
   if (isa<CXXDestructorDecl>(calleeDecl)) {
     cgm.errorNYI(ce->getSourceRange(),
                  "emitCXXMemberOrOperatorMemberCallExpr: destructor call");
     return RValue::get(nullptr);
   }
 
-  assert(!cir::MissingFeatures::sanitizers());
-  if (getLangOpts().AppleKext) {
-    cgm.errorNYI(ce->getSourceRange(),
-                 "emitCXXMemberOrOperatorMemberCallExpr: AppleKext");
-    return RValue::get(nullptr);
+  CIRGenCallee callee;
+  if (useVirtualCall) {
+    callee = CIRGenCallee::forVirtual(ce, md, thisPtr.getAddress(), ty);
+  } else {
+    assert(!cir::MissingFeatures::sanitizers());
+    if (getLangOpts().AppleKext) {
+      cgm.errorNYI(ce->getSourceRange(),
+                   "emitCXXMemberOrOperatorMemberCallExpr: AppleKext");
+      return RValue::get(nullptr);
+    }
+
+    callee = CIRGenCallee::forDirect(cgm.getAddrOfFunction(calleeDecl, ty),
+                                     GlobalDecl(calleeDecl));
+  }
+
+  if (md->isVirtual()) {
+    Address newThisAddr =
+        cgm.getCXXABI().adjustThisArgumentForVirtualFunctionCall(
+            *this, calleeDecl, thisPtr.getAddress(), useVirtualCall);
+    thisPtr.setAddress(newThisAddr);
   }
-  CIRGenCallee callee =
-      CIRGenCallee::forDirect(cgm.getAddrOfFunction(md, ty), GlobalDecl(md));
 
   return emitCXXMemberOrOperatorCall(
       calleeDecl, callee, returnValue, thisPtr.getPointer(),
@@ -169,7 +190,7 @@ CIRGenFunction::emitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *e,
   assert(md->isInstance() &&
          "Trying to emit a member call expr on a static method!");
   return emitCXXMemberOrOperatorMemberCallExpr(
-      e, md, returnValue, /*HasQualifier=*/false, /*Qualifier=*/nullptr,
+      e, md, returnValue, /*HasQualifier=*/false, /*Qualifier=*/std::nullopt,
       /*IsArrow=*/false, e->getArg(0));
 }
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.cpp b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
index fc208fffe216f..8a15e5f96aea2 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
@@ -42,21 +42,41 @@ CIRGenFunctionInfo::create(CanQualType resultType,
   return fi;
 }
 
-cir::FuncType CIRGenTypes::getFunctionType(const CIRGenFunctionInfo &fi) {
-  mlir::Type resultType = convertType(fi.getReturnType());
+cir::FuncType CIRGenTypes::getFunctionType(GlobalDecl gd) {
+  const CIRGenFunctionInfo &fi = arrangeGlobalDeclaration(gd);
+  return getFunctionType(fi);
+}
+
+cir::FuncType CIRGenTypes::getFunctionType(const CIRGenFunctionInfo &info) {
+  mlir::Type resultType = convertType(info.getReturnType());
   SmallVector<mlir::Type, 8> argTypes;
-  argTypes.reserve(fi.getNumRequiredArgs());
+  argTypes.reserve(info.getNumRequiredArgs());
 
-  for (const CanQualType &argType : fi.requiredArguments())
+  for (const CanQualType &argType : info.requiredArguments())
     argTypes.push_back(convertType(argType));
 
   return cir::FuncType::get(argTypes,
                             (resultType ? resultType : builder.getVoidTy()),
-                            fi.isVariadic());
+                            info.isVariadic());
+}
+
+cir::FuncType CIRGenTypes::getFunctionTypeForVTable(GlobalDecl gd) {
+  const CXXMethodDecl *md = cast<CXXMethodDecl>(gd.getDecl());
+  const FunctionProtoType *fpt = md->getType()->getAs<FunctionProtoType>();
+
+  if (!isFuncTypeConvertible(fpt))
+    cgm.errorNYI("getFunctionTypeForVTable: non-convertible function type");
+
+  return getFunctionType(gd);
 }
 
 CIRGenCallee CIRGenCallee::prepareConcreteCallee(CIRGenFunction &cgf) const {
-  assert(!cir::MissingFeatures::opCallVirtual());
+  if (isVirtual()) {
+    const CallExpr *ce = getVirtualCallExpr();
+    return cgf.cgm.getCXXABI().getVirtualFunctionPointer(
+        cgf, getVirtualMethodDecl(), getThisAddress(), getVirtualFunctionType(),
+        ce ? ce->getBeginLoc() : SourceLocation());
+  }
   return *this;
 }
 
@@ -203,9 +223,9 @@ CIRGenTypes::arrangeCXXStructorDeclaration(GlobalDecl gd) {
 /// when calling a method pointer.
 CanQualType CIRGenTypes::deriveThisType(const CXXRecordDecl *rd,
                                         const CXXMethodDecl *md) {
-  QualType recTy;
+  CanQualType recTy;
   if (rd) {
-    recTy = getASTContext().getTagDeclType(rd)->getCanonicalTypeInternal();
+    recTy = getASTContext().getCanonicalTagType(rd);
   } else {
     // This can happen with the MS ABI. It shouldn't need anything more than
     // setting recTy to VoidTy here, but we're flagging it for now because we
@@ -215,9 +235,9 @@ CanQualType CIRGenTypes::deriveThisType(const CXXRecordDecl *rd,
   }
 
   if (md)
-    recTy = getASTContext().getAddrSpaceQualType(
-        recTy, md->getMethodQualifiers().getAddressSpace());
-  return getASTContext().getPointerType(CanQualType::CreateUnsafe(recTy));
+    recTy = CanQualType::CreateUnsafe(getASTContext().getAddrSpaceQualType(
+        recTy, md->getMethodQualifiers().getAddressSpace()));
+  return getASTContext().getPointerType(recTy);
 }
 
 /// Arrange the CIR function layout for a value of the given function type, on
@@ -267,7 +287,10 @@ void CIRGenFunction::emitDelegateCallArg(CallArgList &args,
   // Deactivate the cleanup for the callee-destructed param that was pushed.
   assert(!cir::MissingFeatures::thunks());
   if (type->isRecordType() &&
-      type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
+      type->castAs<RecordType>()
+          ->getOriginalDecl()
+          ->getDefinitionOrSelf()
+          ->isParamDestroyedInCallee() &&
       param->needsDestruction(getContext())) {
     cgm.errorNYI(param->getSourceRange(),
                  "emitDelegateCallArg: callee-destructed param");
@@ -667,8 +690,10 @@ void CIRGenFunction::emitCallArg(CallArgList &args, const clang::Expr *e,
   // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
   // However, we still have to push an EH-only cleanup in case we unwind before
   // we make it to the call.
-  if (argType->isRecordType() &&
-      argType->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
+  if (argType->isRecordType() && argType->castAs<RecordType>()
+                                     ->getOriginalDecl()
+                                     ->getDefinitionOrSelf()
+                                     ->isParamDestroyedInCallee()) {
     assert(!cir::MissingFeatures::msabi());
     cgm.errorNYI(e->getSourceRange(), "emitCallArg: msabi is NYI");
   }
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.h b/clang/lib/CIR/CodeGen/CIRGenCall.h
index 28576a15068fa..81cbb854f3b7d 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.h
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.h
@@ -46,20 +46,33 @@ class CIRGenCallee {
   enum class SpecialKind : uintptr_t {
     Invalid,
     Builtin,
+    PseudoDestructor,
+    Virtual,
 
-    Last = Builtin,
+    Last = Virtual
   };
 
   struct BuiltinInfoStorage {
     const clang::FunctionDecl *decl;
     unsigned id;
   };
+  struct PseudoDestructorInfoStorage {
+    const clang::CXXPseudoDestructorExpr *expr;
+  };
+  struct VirtualInfoStorage {
+    const clang::CallExpr *ce;
+    clang::GlobalDecl md;
+    Address addr;
+    cir::FuncType fTy;
+  };
 
   SpecialKind kindOrFunctionPtr;
 
   union {
     CIRGenCalleeInfo abstractInfo;
     BuiltinInfoStorage builtinInfo;
+    PseudoDestructorInfoStorage pseudoDestructorInfo;
+    VirtualInfoStorage virtualInfo;
   };
 
   explicit CIRGenCallee(SpecialKind kind) : kindOrFunctionPtr(kind) {}
@@ -98,6 +111,22 @@ class CIRGenCallee {
     return result;
   }
 
+  static CIRGenCallee
+  forPseudoDestructor(const clang::CXXPseudoDestructorExpr *expr) {
+    CIRGenCallee result(SpecialKind::PseudoDestructor);
+    result.pseudoDestructorInfo.expr = expr;
+    return result;
+  }
+
+  bool isPseudoDestructor() const {
+    return kindOrFunctionPtr == SpecialKind::PseudoDestructor;
+  }
+
+  const CXXPseudoDestructorExpr *getPseudoDestructorExpr() const {
+    assert(isPseudoDestructor());
+    return pseudoDestructorInfo.expr;
+  }
+
   bool isOrdinary() const {
     return uintptr_t(kindOrFunctionPtr) > uintptr_t(SpecialKind::Last);
   }
@@ -107,7 +136,8 @@ class CIRGenCallee {
   CIRGenCallee prepareConcreteCallee(CIRGenFunction &cgf) const;
 
   CIRGenCalleeInfo getAbstractInfo() const {
-    assert(!cir::MissingFeatures::opCallVirtual());
+    if (isVirtual())
+      return virtualInfo.md;
     assert(isOrdinary());
     return abstractInfo;
   }
@@ -117,6 +147,39 @@ class CIRGenCallee {
     return reinterpret_cast<mlir::Operation *>(kindOrFunctionPtr);
   }
 
+  bool isVirtual() const { return kindOrFunctionPtr == SpecialKind::Virtual; }
+
+  static CIRGenCallee forVirtual(const clang::CallExpr *ce,
+                                 clang::GlobalDecl md, Address addr,
+                                 cir::FuncType fTy) {
+    CIRGenCallee result(SpecialKind::Virtual);
+    result.virtualInfo.ce = ce;
+    result.virtualInfo.md = md;
+    result.virtualInfo.addr = addr;
+    result.virtualInfo.fTy = fTy;
+    return result;
+  }
+
+  const clang::CallExpr *getVirtualCallExpr() const {
+    assert(isVirtual());
+    return virtualInfo.ce;
+  }
+
+  clang::GlobalDecl getVirtualMethodDecl() const {
+    assert(isVirtual());
+    return virtualInfo.md;
+  }
+
+  Address getThisAddress() const {
+    assert(isVirtual());
+    return virtualInfo.addr;
+  }
+
+  cir::FuncType getVirtualFunctionType() const {
+    assert(isVirtual());
+    return virtualInfo.fTy;
+  }
+
   void setFunctionPointer(mlir::Operation *functionPtr) {
     assert(isOrdinary());
     kindOrFunctionPtr = SpecialKind(reinterpret_cast<uintptr_t>(functionPtr));
diff --git a/clang/lib/CIR/CodeGen/CIRGenClass.cpp b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
index 72b9d177e4c63..3e5dc22426d8e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenClass.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenClass.cpp
@@ -86,7 +86,7 @@ static void emitMemberInitializer(CIRGenFunction &cgf,
   QualType fieldType = field->getType();
 
   mlir::Value thisPtr = cgf.loadCXXThis();
-  QualType recordTy = cgf.getContext().getTypeDeclType(classDecl);
+  CanQualType recordTy = cgf.getContext().getCanonicalTagType(classDecl);
 
   // If a base constructor is being emitted, create an LValue that has the
   // non-virtual alignment.
@@ -121,7 +121,8 @@ static void emitMemberInitializer(CIRGenFunction &cgf,
 static bool isInitializerOfDynamicClass(const CXXCtorInitializer *baseInit) {
   const Type *baseType = baseInit->getBaseClass();
   const auto *baseClassDecl =
-      cast<CXXRecordDecl>(baseType->castAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(baseType->castAs<RecordType>()->getOriginalDecl())
+          ->getDefinitionOrSelf();
   return baseClassDecl->isDynamicClass();
 }
 
@@ -161,7 +162,8 @@ void CIRGenFunction::emitBaseInitializer(mlir::Location loc,
 
   const Type *baseType = baseInit->getBaseClass();
   const auto *baseClassDecl =
-      cast<CXXRecordDecl>(baseType->castAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(baseType->castAs<RecordType>()->getOriginalDecl())
+          ->getDefinitionOrSelf();
 
   bool isBaseVirtual = baseInit->isBaseVirtual();
 
@@ -197,10 +199,6 @@ void CIRGenFunction::emitCtorPrologue(const CXXConstructorDecl *cd,
     return;
   }
 
-  // If there are no member initializers, we can just return.
-  if (cd->getNumCtorInitializers() == 0)
-    return;
-
   const CXXRecordDecl *classDecl = cd->getParent();
 
   // This code doesn't use range-based iteration because we may need to emit
@@ -225,7 +223,8 @@ void CIRGenFunction::emitCtorPrologue(const CXXConstructorDecl *cd,
   }
 
   const mlir::Value oldThisValue = cxxThisValue;
-  if (!constructVBases && (*b)->isBaseInitializer() && (*b)->isBaseVirtual()) {
+  if (!constructVBases && b != e && (*b)->isBaseInitializer() &&
+      (*b)->isBaseVirtual()) {
     cgm.errorNYI(cd->getSourceRange(),
                  "emitCtorPrologue: virtual base initializer");
     return;
@@ -247,11 +246,7 @@ void CIRGenFunction::emitCtorPrologue(const CXXConstructorDecl *cd,
 
   cxxThisValue = oldThisValue;
 
-  if (classDecl->isDynamicClass()) {
-    cgm.errorNYI(cd->getSourceRange(),
-                 "emitCtorPrologue: initialize vtable pointers");
-    return;
-  }
+  initializeVTablePointers(getLoc(cd->getBeginLoc()), classDecl);
 
   // Finally, initialize class members.
   FieldConstructionScope fcs(*this, loadCXXThisAddress());
@@ -269,6 +264,96 @@ void CIRGenFunction::emitCtorPrologue(const CXXConstructorDecl *cd,
   }
 }
 
+void CIRGenFunction::initializeVTablePointer(mlir::Location loc,
+                                             const VPtr &vptr) {
+  // Compute the address point.
+  mlir::Value vtableAddressPoint =
+      cgm.getCXXABI().getVTableAddressPointInStructor(
+          *this, vptr.vtableClass, vptr.base, vptr.nearestVBase);
+
+  if (!vtableAddressPoint)
+    return;
+
+  // Compute where to store the address point.
+  mlir::Value virtualOffset{};
+  CharUnits nonVirtualOffset = CharUnits::Zero();
+
+  mlir::Type baseValueTy;
+  if (cgm.getCXXABI().isVirtualOffsetNeededForVTableField(*this, vptr)) {
+    cgm.errorNYI(loc, "initializeVTablePointer: virtual offset for vtable");
+  } else {
+    // We can just use the base offset in the complete class.
+    nonVirtualOffset = vptr.base.getBaseOffset();
+    baseValueTy =
+        convertType(getContext().getCanonicalTagType(vptr.base.getBase()));
+  }
+
+  // Apply the offsets.
+  Address classAddr = loadCXXThisAddress();
+  if (!nonVirtualOffset.isZero() || virtualOffset) {
+    cgm.errorNYI(loc,
+                 "initializeVTablePointer: non-virtual and virtual offset");
+  }
+
+  // Finally, store the address point. Use the same CIR types as the field.
+  //
+  // vtable field is derived from `this` pointer, therefore they should be in
+  // the same addr space.
+  assert(!cir::MissingFeatures::addressSpace());
+  auto vtablePtr = cir::VTableGetVPtrOp::create(
+      builder, loc, builder.getPtrToVPtrType(), classAddr.getPointer());
+  Address vtableField = Address(vtablePtr, classAddr.getAlignment());
+  builder.createStore(loc, vtableAddressPoint, vtableField);
+  assert(!cir::MissingFeatures::opTBAA());
+  assert(!cir::MissingFeatures::createInvariantGroup());
+}
+
+void CIRGenFunction::initializeVTablePointers(mlir::Location loc,
+                                              const CXXRecordDecl *rd) {
+  // Ignore classes without a vtable.
+  if (!rd->isDynamicClass())
+    return;
+
+  // Initialize the vtable pointers for this class and all of its bases.
+  if (cgm.getCXXABI().doStructorsInitializeVPtrs(rd))
+    for (const auto &vptr : getVTablePointers(rd))
+      initializeVTablePointer(loc, vptr);
+
+  if (rd->getNumVBases())
+    cgm.errorNYI(loc, "initializeVTablePointers: virtual base");
+}
+
+CIRGenFunction::VPtrsVector
+CIRGenFunction::getVTablePointers(const CXXRecordDecl *vtableClass) {
+  CIRGenFunction::VPtrsVector vptrsResult;
+  getVTablePointers(BaseSubobject(vtableClass, CharUnits::Zero()),
+                    /*NearestVBase=*/nullptr,
+                    /*OffsetFromNearestVBase=*/CharUnits::Zero(),
+                    /*BaseIsNonVirtualPrimaryBase=*/false, vtableClass,
+                    vptrsResult);
+  return vptrsResult;
+}
+
+void CIRGenFunction::getVTablePointers(BaseSubobject base,
+                                       const CXXRecordDecl *nearestVBase,
+                                       CharUnits offsetFromNearestVBase,
+                                       bool baseIsNonVirtualPrimaryBase,
+                                       const CXXRecordDecl *vtableClass,
+                                       VPtrsVector &vptrs) {
+  // If this base is a non-virtual primary base the address point has already
+  // been set.
+  if (!baseIsNonVirtualPrimaryBase) {
+    // Initialize the vtable pointer for this base.
+    VPtr vptr = {base, nearestVBase, offsetFromNearestVBase, vtableClass};
+    vptrs.push_back(vptr);
+  }
+
+  const CXXRecordDecl *rd = base.getBase();
+
+  if (rd->getNumBases())
+    cgm.errorNYI(rd->getSourceRange(), "getVTablePointers: traverse bases");
+}
+
 Address CIRGenFunction::loadCXXThisAddress() {
   assert(curFuncDecl && "loading 'this' without a func declaration?");
   assert(isa<CXXMethodDecl>(curFuncDecl));
@@ -377,7 +462,7 @@ void CIRGenFunction::emitCXXAggrConstructorCall(
   //
   // Note that these are complete objects and so we don't need to
   // use the non-virtual size or alignment.
-  QualType type = getContext().getTypeDeclType(ctor->getParent());
+  CanQualType type = getContext().getCanonicalTagType(ctor->getParent());
   CharUnits eltAlignment = arrayBase.getAlignment().alignmentOfArrayElement(
       getContext().getTypeSizeInChars(type));
 
@@ -484,7 +569,8 @@ void CIRGenFunction::emitImplicitAssignmentOperatorBody(FunctionArgList &args) {
 void CIRGenFunction::destroyCXXObject(CIRGenFunction &cgf, Address addr,
                                       QualType type) {
   const RecordType *rtype = type->castAs<RecordType>();
-  const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
+  const CXXRecordDecl *record =
+      cast<CXXRecordDecl>(rtype->getOriginalDecl())->getDefinitionOrSelf();
   const CXXDestructorDecl *dtor = record->getDestructor();
   // TODO(cir): Unlike traditional codegen, CIRGen should actually emit trivial
   // dtors which shall be removed on later CIR passes. However, only remove this
@@ -571,6 +657,37 @@ Address CIRGenFunction::getAddressOfBaseClass(
   return value;
 }
 
+// TODO(cir): this can be shared with LLVM codegen.
+bool CIRGenFunction::shouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *rd) {
+  assert(!cir::MissingFeatures::hiddenVisibility());
+  if (!cgm.getCodeGenOpts().WholeProgramVTables)
+    return false;
+
+  if (cgm.getCodeGenOpts().VirtualFunctionElimination)
+    return true;
+
+  assert(!cir::MissingFeatures::sanitizers());
+
+  return false;
+}
+
+mlir::Value CIRGenFunction::getVTablePtr(mlir::Location loc, Address thisAddr,
+                                         const CXXRecordDecl *rd) {
+  auto vtablePtr = cir::VTableGetVPtrOp::create(
+      builder, loc, builder.getPtrToVPtrType(), thisAddr.getPointer());
+  Address vtablePtrAddr = Address(vtablePtr, thisAddr.getAlignment());
+
+  auto vtable = builder.createLoad(loc, vtablePtrAddr);
+  assert(!cir::MissingFeatures::opTBAA());
+
+  if (cgm.getCodeGenOpts().OptimizationLevel > 0 &&
+      cgm.getCodeGenOpts().StrictVTablePointers) {
+    assert(!cir::MissingFeatures::createInvariantGroup());
+  }
+
+  return vtable;
+}
+
 void CIRGenFunction::emitCXXConstructorCall(const clang::CXXConstructorDecl *d,
                                             clang::CXXCtorType type,
                                             bool forVirtualBase,
diff --git a/clang/lib/CIR/CodeGen/CIRGenCleanup.cpp b/clang/lib/CIR/CodeGen/CIRGenCleanup.cpp
index be21ce9c4a18c..4d4d10be40024 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCleanup.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCleanup.cpp
@@ -16,6 +16,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "CIRGenCleanup.h"
 #include "CIRGenFunction.h"
 
 #include "clang/CIR/MissingFeatures.h"
@@ -33,37 +34,147 @@ using namespace clang::CIRGen;
 
 void EHScopeStack::Cleanup::anchor() {}
 
-static mlir::Block *getCurCleanupBlock(CIRGenFunction &cgf) {
-  mlir::OpBuilder::InsertionGuard guard(cgf.getBuilder());
-  mlir::Block *cleanup =
-      cgf.curLexScope->getOrCreateCleanupBlock(cgf.getBuilder());
-  return cleanup;
+/// Push an entry of the given size onto this protected-scope stack.
+char *EHScopeStack::allocate(size_t size) {
+  size = llvm::alignTo(size, ScopeStackAlignment);
+  if (!startOfBuffer) {
+    unsigned capacity = llvm::PowerOf2Ceil(std::max(size, 1024ul));
+    startOfBuffer = std::make_unique<char[]>(capacity);
+    startOfData = endOfBuffer = startOfBuffer.get() + capacity;
+  } else if (static_cast<size_t>(startOfData - startOfBuffer.get()) < size) {
+    unsigned currentCapacity = endOfBuffer - startOfBuffer.get();
+    unsigned usedCapacity =
+        currentCapacity - (startOfData - startOfBuffer.get());
+    unsigned requiredCapacity = usedCapacity + size;
+    // We know from the 'else if' condition that requiredCapacity is greater
+    // than currentCapacity.
+    unsigned newCapacity = llvm::PowerOf2Ceil(requiredCapacity);
+
+    std::unique_ptr<char[]> newStartOfBuffer =
+        std::make_unique<char[]>(newCapacity);
+    char *newEndOfBuffer = newStartOfBuffer.get() + newCapacity;
+    char *newStartOfData = newEndOfBuffer - usedCapacity;
+    memcpy(newStartOfData, startOfData, usedCapacity);
+    startOfBuffer.swap(newStartOfBuffer);
+    endOfBuffer = newEndOfBuffer;
+    startOfData = newStartOfData;
+  }
+
+  assert(startOfBuffer.get() + size <= startOfData);
+  startOfData -= size;
+  return startOfData;
+}
+
+void EHScopeStack::deallocate(size_t size) {
+  startOfData += llvm::alignTo(size, ScopeStackAlignment);
+}
+
+void *EHScopeStack::pushCleanup(CleanupKind kind, size_t size) {
+  char *buffer = allocate(EHCleanupScope::getSizeForCleanupSize(size));
+  bool isEHCleanup = kind & EHCleanup;
+  bool isLifetimeMarker = kind & LifetimeMarker;
+
+  assert(!cir::MissingFeatures::innermostEHScope());
+
+  EHCleanupScope *scope = new (buffer) EHCleanupScope(size);
+
+  if (isLifetimeMarker)
+    cgf->cgm.errorNYI("push lifetime marker cleanup");
+
+  // With Windows -EHa, Invoke llvm.seh.scope.begin() for EHCleanup
+  if (cgf->getLangOpts().EHAsynch && isEHCleanup && !isLifetimeMarker &&
+      cgf->getTarget().getCXXABI().isMicrosoft())
+    cgf->cgm.errorNYI("push seh cleanup");
+
+  return scope->getCleanupBuffer();
+}
+
+void EHScopeStack::popCleanup() {
+  assert(!empty() && "popping exception stack when not empty");
+
+  assert(isa<EHCleanupScope>(*begin()));
+  EHCleanupScope &cleanup = cast<EHCleanupScope>(*begin());
+  deallocate(cleanup.getAllocatedSize());
+
+  // Destroy the cleanup.
+  cleanup.destroy();
+
+  assert(!cir::MissingFeatures::ehCleanupBranchFixups());
+}
+
+static void emitCleanup(CIRGenFunction &cgf, EHScopeStack::Cleanup *cleanup) {
+  // Ask the cleanup to emit itself.
+  assert(cgf.haveInsertPoint() && "expected insertion point");
+  assert(!cir::MissingFeatures::ehCleanupFlags());
+  cleanup->emit(cgf);
+  assert(cgf.haveInsertPoint() && "cleanup ended with no insertion point?");
 }
 
 /// Pops a cleanup block. If the block includes a normal cleanup, the
 /// current insertion point is threaded through the cleanup, as are
 /// any branch fixups on the cleanup.
 void CIRGenFunction::popCleanupBlock() {
-  assert(!ehStack.cleanupStack.empty() && "cleanup stack is empty!");
-  mlir::OpBuilder::InsertionGuard guard(builder);
-  std::unique_ptr<EHScopeStack::Cleanup> cleanup =
-      ehStack.cleanupStack.pop_back_val();
+  assert(!ehStack.empty() && "cleanup stack is empty!");
+  assert(isa<EHCleanupScope>(*ehStack.begin()) && "top not a cleanup!");
+  EHCleanupScope &scope = cast<EHCleanupScope>(*ehStack.begin());
+
+  // Remember activation information.
+  bool isActive = scope.isActive();
+
+  assert(!cir::MissingFeatures::ehCleanupBranchFixups());
+
+  // - whether there's a fallthrough
+  mlir::Block *fallthroughSource = builder.getInsertionBlock();
+  bool hasFallthrough = fallthroughSource != nullptr && isActive;
+
+  bool requiresNormalCleanup = scope.isNormalCleanup() && hasFallthrough;
+
+  // If we don't need the cleanup at all, we're done.
+  assert(!cir::MissingFeatures::ehCleanupScopeRequiresEHCleanup());
+  if (!requiresNormalCleanup) {
+    ehStack.popCleanup();
+    return;
+  }
+
+  // Copy the cleanup emission data out.  This uses either a stack
+  // array or malloc'd memory, depending on the size, which is
+  // behavior that SmallVector would provide, if we could use it
+  // here. Unfortunately, if you ask for a SmallVector<char>, the
+  // alignment isn't sufficient.
+  auto *cleanupSource = reinterpret_cast<char *>(scope.getCleanupBuffer());
+  alignas(EHScopeStack::ScopeStackAlignment) char
+      cleanupBufferStack[8 * sizeof(void *)];
+  std::unique_ptr<char[]> cleanupBufferHeap;
+  size_t cleanupSize = scope.getCleanupSize();
+  EHScopeStack::Cleanup *cleanup;
+
+  // This is necessary because we are going to deallocate the cleanup
+  // (in popCleanup) before we emit it.
+  if (cleanupSize <= sizeof(cleanupBufferStack)) {
+    memcpy(cleanupBufferStack, cleanupSource, cleanupSize);
+    cleanup = reinterpret_cast<EHScopeStack::Cleanup *>(cleanupBufferStack);
+  } else {
+    cleanupBufferHeap.reset(new char[cleanupSize]);
+    memcpy(cleanupBufferHeap.get(), cleanupSource, cleanupSize);
+    cleanup =
+        reinterpret_cast<EHScopeStack::Cleanup *>(cleanupBufferHeap.get());
+  }
 
   assert(!cir::MissingFeatures::ehCleanupFlags());
-  mlir::Block *cleanupEntry = getCurCleanupBlock(*this);
-  builder.setInsertionPointToEnd(cleanupEntry);
-  cleanup->emit(*this);
+
+  ehStack.popCleanup();
+  scope.markEmitted();
+  emitCleanup(*this, cleanup);
 }
 
 /// Pops cleanup blocks until the given savepoint is reached.
-void CIRGenFunction::popCleanupBlocks(size_t oldCleanupStackDepth) {
+void CIRGenFunction::popCleanupBlocks(
+    EHScopeStack::stable_iterator oldCleanupStackDepth) {
   assert(!cir::MissingFeatures::ehstackBranches());
 
-  assert(ehStack.getStackDepth() >= oldCleanupStackDepth);
-
   // Pop cleanup blocks until we reach the base stack depth for the
   // current scope.
-  while (ehStack.getStackDepth() > oldCleanupStackDepth) {
+  while (ehStack.stable_begin() != oldCleanupStackDepth) {
     popCleanupBlock();
   }
 }
diff --git a/clang/lib/CIR/CodeGen/CIRGenCleanup.h b/clang/lib/CIR/CodeGen/CIRGenCleanup.h
new file mode 100644
index 0000000000000..a4ec8ccebbd3b
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenCleanup.h
@@ -0,0 +1,142 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// These classes support the generation of CIR for cleanups, initially based
+// on LLVM IR cleanup handling, but ought to change as CIR evolves.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_LIB_CIR_CODEGEN_CIRGENCLEANUP_H
+#define CLANG_LIB_CIR_CODEGEN_CIRGENCLEANUP_H
+
+#include "Address.h"
+#include "EHScopeStack.h"
+#include "mlir/IR/Value.h"
+
+namespace clang::CIRGen {
+
+/// A protected scope for zero-cost EH handling.
+class EHScope {
+  class CommonBitFields {
+    friend class EHScope;
+    unsigned kind : 3;
+  };
+  enum { NumCommonBits = 3 };
+
+protected:
+  class CleanupBitFields {
+    friend class EHCleanupScope;
+    unsigned : NumCommonBits;
+
+    /// Whether this cleanup needs to be run along normal edges.
+    unsigned isNormalCleanup : 1;
+
+    /// Whether this cleanup needs to be run along exception edges.
+    unsigned isEHCleanup : 1;
+
+    /// Whether this cleanup is currently active.
+    unsigned isActive : 1;
+
+    /// Whether this cleanup is a lifetime marker
+    unsigned isLifetimeMarker : 1;
+
+    /// Whether the normal cleanup should test the activation flag.
+    unsigned testFlagInNormalCleanup : 1;
+
+    /// Whether the EH cleanup should test the activation flag.
+    unsigned testFlagInEHCleanup : 1;
+
+    /// The amount of extra storage needed by the Cleanup.
+    /// Always a multiple of the scope-stack alignment.
+    unsigned cleanupSize : 12;
+  };
+
+  union {
+    CommonBitFields commonBits;
+    CleanupBitFields cleanupBits;
+  };
+
+public:
+  enum Kind { Cleanup, Catch, Terminate, Filter };
+
+  EHScope(Kind kind) { commonBits.kind = kind; }
+
+  Kind getKind() const { return static_cast<Kind>(commonBits.kind); }
+};
+
+/// A cleanup scope which generates the cleanup blocks lazily.
+class alignas(EHScopeStack::ScopeStackAlignment) EHCleanupScope
+    : public EHScope {
+public:
+  /// Gets the size required for a lazy cleanup scope with the given
+  /// cleanup-data requirements.
+  static size_t getSizeForCleanupSize(size_t size) {
+    return sizeof(EHCleanupScope) + size;
+  }
+
+  size_t getAllocatedSize() const {
+    return sizeof(EHCleanupScope) + cleanupBits.cleanupSize;
+  }
+
+  EHCleanupScope(unsigned cleanupSize) : EHScope(EHScope::Cleanup) {
+    // TODO(cir): When exception handling is upstreamed, isNormalCleanup and
+    // isEHCleanup will be arguments to the constructor.
+    cleanupBits.isNormalCleanup = true;
+    cleanupBits.isEHCleanup = false;
+    cleanupBits.isActive = true;
+    cleanupBits.isLifetimeMarker = false;
+    cleanupBits.testFlagInNormalCleanup = false;
+    cleanupBits.testFlagInEHCleanup = false;
+    cleanupBits.cleanupSize = cleanupSize;
+
+    assert(cleanupBits.cleanupSize == cleanupSize && "cleanup size overflow");
+  }
+
+  void destroy() {}
+  // Objects of EHCleanupScope are not destructed. Use destroy().
+  ~EHCleanupScope() = delete;
+
+  bool isNormalCleanup() const { return cleanupBits.isNormalCleanup; }
+
+  bool isActive() const { return cleanupBits.isActive; }
+
+  size_t getCleanupSize() const { return cleanupBits.cleanupSize; }
+  void *getCleanupBuffer() { return this + 1; }
+
+  EHScopeStack::Cleanup *getCleanup() {
+    return reinterpret_cast<EHScopeStack::Cleanup *>(getCleanupBuffer());
+  }
+
+  static bool classof(const EHScope *scope) {
+    return (scope->getKind() == Cleanup);
+  }
+
+  void markEmitted() {}
+};
+
+/// A non-stable pointer into the scope stack.
+class EHScopeStack::iterator {
+  char *ptr = nullptr;
+
+  friend class EHScopeStack;
+  explicit iterator(char *ptr) : ptr(ptr) {}
+
+public:
+  iterator() = default;
+
+  EHScope *get() const { return reinterpret_cast<EHScope *>(ptr); }
+
+  EHScope &operator*() const { return *get(); }
+};
+
+inline EHScopeStack::iterator EHScopeStack::begin() const {
+  return iterator(startOfData);
+}
+
+} // namespace clang::CIRGen
+#endif // CLANG_LIB_CIR_CODEGEN_CIRGENCLEANUP_H
diff --git a/clang/lib/CIR/CodeGen/CIRGenConstantEmitter.h b/clang/lib/CIR/CodeGen/CIRGenConstantEmitter.h
index d6dac50bb1263..d455f6e283406 100644
--- a/clang/lib/CIR/CodeGen/CIRGenConstantEmitter.h
+++ b/clang/lib/CIR/CodeGen/CIRGenConstantEmitter.h
@@ -80,7 +80,7 @@ class ConstantEmitter {
   //     initializer or to propagate to another context; for example,
   //     side effects, or emitting an initialization that requires a
   //     reference to its current location.
-  mlir::Attribute emitForMemory(mlir::Attribute c, QualType t);
+  mlir::Attribute emitForMemory(mlir::Attribute c, QualType destType);
 
   /// Try to emit the initializer of the given declaration as an abstract
   /// constant.
@@ -90,8 +90,9 @@ class ConstantEmitter {
   /// asserting that it succeeded.  This is only safe to do when the
   /// expression is known to be a constant expression with either a fairly
   /// simple type or a known simple form.
+  mlir::Attribute emitAbstract(const Expr *e, QualType destType);
   mlir::Attribute emitAbstract(SourceLocation loc, const APValue &value,
-                               QualType t);
+                               QualType destType);
 
   mlir::Attribute tryEmitConstantExpr(const ConstantExpr *ce);
 
@@ -101,6 +102,7 @@ class ConstantEmitter {
 
   mlir::Attribute tryEmitPrivateForVarInit(const VarDecl &d);
 
+  mlir::TypedAttr tryEmitPrivate(const Expr *e, QualType destType);
   mlir::Attribute tryEmitPrivate(const APValue &value, QualType destType);
   mlir::Attribute tryEmitPrivateForMemory(const APValue &value, QualType t);
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
index 78d375cc02ee0..7cc024fd5596c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenDecl.cpp
@@ -154,15 +154,19 @@ void CIRGenFunction::emitAutoVarInit(
     initializeWhatIsTechnicallyUninitialized(addr);
     LValue lv = makeAddrLValue(addr, type, AlignmentSource::Decl);
     emitExprAsInit(init, &d, lv);
-    // In case lv has uses it means we indeed initialized something
-    // out of it while trying to build the expression, mark it as such.
-    mlir::Value val = lv.getAddress().getPointer();
-    assert(val && "Should have an address");
-    auto allocaOp = val.getDefiningOp<cir::AllocaOp>();
-    assert(allocaOp && "Address should come straight out of the alloca");
-
-    if (!allocaOp.use_empty())
-      allocaOp.setInitAttr(mlir::UnitAttr::get(&getMLIRContext()));
+
+    if (!emission.wasEmittedAsOffloadClause()) {
+      // In case lv has uses it means we indeed initialized something
+      // out of it while trying to build the expression, mark it as such.
+      mlir::Value val = lv.getAddress().getPointer();
+      assert(val && "Should have an address");
+      auto allocaOp = val.getDefiningOp<cir::AllocaOp>();
+      assert(allocaOp && "Address should come straight out of the alloca");
+
+      if (!allocaOp.use_empty())
+        allocaOp.setInitAttr(mlir::UnitAttr::get(&getMLIRContext()));
+    }
+
     return;
   }
 
@@ -293,7 +297,7 @@ CIRGenModule::getOrCreateStaticVarDecl(const VarDecl &d,
   mlir::Attribute init = builder.getZeroInitAttr(convertType(ty));
 
   cir::GlobalOp gv = builder.createVersionedGlobal(
-      getModule(), getLoc(d.getLocation()), name, lty, linkage);
+      getModule(), getLoc(d.getLocation()), name, lty, false, linkage);
   // TODO(cir): infer visibility from linkage in global op builder.
   gv.setVisibility(getMLIRVisibilityFromCIRLinkage(linkage));
   gv.setInitialValueAttr(init);
@@ -667,6 +671,12 @@ struct DestroyObject final : EHScopeStack::Cleanup {
   void emit(CIRGenFunction &cgf) override {
     cgf.emitDestroy(addr, type, destroyer);
   }
+
+  // This is a placeholder until EHCleanupScope is implemented.
+  size_t getSize() const override {
+    assert(!cir::MissingFeatures::ehCleanupScope());
+    return sizeof(DestroyObject);
+  }
 };
 } // namespace
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
index a0ff08ea3de3a..2b74b27100402 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExpr.cpp
@@ -73,28 +73,58 @@ Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
 
   // Casts:
   if (auto const *ce = dyn_cast<CastExpr>(expr)) {
-    if (isa<ExplicitCastExpr>(ce)) {
-      cgm.errorNYI(expr->getSourceRange(),
-                   "emitPointerWithAlignment: explicit cast");
-      return Address::invalid();
-    }
+    if (const auto *ece = dyn_cast<ExplicitCastExpr>(ce))
+      cgm.emitExplicitCastExprType(ece);
 
     switch (ce->getCastKind()) {
     // Non-converting casts (but not C's implicit conversion from void*).
     case CK_BitCast:
     case CK_NoOp:
     case CK_AddressSpaceConversion: {
-      cgm.errorNYI(expr->getSourceRange(),
-                   "emitPointerWithAlignment: noop cast");
-      return Address::invalid();
-    } break;
+      if (const auto *ptrTy =
+              ce->getSubExpr()->getType()->getAs<PointerType>()) {
+        if (ptrTy->getPointeeType()->isVoidType())
+          break;
+
+        LValueBaseInfo innerBaseInfo;
+        assert(!cir::MissingFeatures::opTBAA());
+        Address addr =
+            emitPointerWithAlignment(ce->getSubExpr(), &innerBaseInfo);
+        if (baseInfo)
+          *baseInfo = innerBaseInfo;
+
+        if (isa<ExplicitCastExpr>(ce)) {
+          LValueBaseInfo targetTypeBaseInfo;
+
+          const QualType pointeeType = expr->getType()->getPointeeType();
+          const CharUnits align =
+              cgm.getNaturalTypeAlignment(pointeeType, &targetTypeBaseInfo);
+
+          // If the source l-value is opaque, honor the alignment of the
+          // casted-to type.
+          if (innerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
+            if (baseInfo)
+              baseInfo->mergeForCast(targetTypeBaseInfo);
+            addr = Address(addr.getPointer(), addr.getElementType(), align);
+          }
+        }
+
+        assert(!cir::MissingFeatures::sanitizers());
+
+        const mlir::Type eltTy =
+            convertTypeForMem(expr->getType()->getPointeeType());
+        addr = getBuilder().createElementBitCast(getLoc(expr->getSourceRange()),
+                                                 addr, eltTy);
+        assert(!cir::MissingFeatures::addressSpace());
+
+        return addr;
+      }
+      break;
+    }
 
     // Array-to-pointer decay. TODO(cir): BaseInfo and TBAAInfo.
-    case CK_ArrayToPointerDecay: {
-      cgm.errorNYI(expr->getSourceRange(),
-                   "emitPointerWithAlignment: array-to-pointer decay");
-      return Address::invalid();
-    }
+    case CK_ArrayToPointerDecay:
+      return emitArrayToPointerDecay(ce->getSubExpr(), baseInfo);
 
     case CK_UncheckedDerivedToBase:
     case CK_DerivedToBase: {
@@ -184,8 +214,11 @@ Address CIRGenFunction::emitPointerWithAlignment(const Expr *expr,
   if (const UnaryOperator *uo = dyn_cast<UnaryOperator>(expr)) {
     // TODO(cir): maybe we should use cir.unary for pointers here instead.
     if (uo->getOpcode() == UO_AddrOf) {
-      cgm.errorNYI(expr->getSourceRange(), "emitPointerWithAlignment: unary &");
-      return Address::invalid();
+      LValue lv = emitLValue(uo->getSubExpr());
+      if (baseInfo)
+        *baseInfo = lv.getBaseInfo();
+      assert(!cir::MissingFeatures::opTBAA());
+      return lv.getAddress();
     }
   }
 
@@ -369,9 +402,10 @@ Address CIRGenFunction::getAddrOfBitFieldStorage(LValue base,
                                                  unsigned index) {
   mlir::Location loc = getLoc(field->getLocation());
   cir::PointerType fieldPtr = cir::PointerType::get(fieldType);
-  cir::GetMemberOp sea = getBuilder().createGetMember(
-      loc, fieldPtr, base.getPointer(), field->getName(), index);
   auto rec = cast<cir::RecordType>(base.getAddress().getElementType());
+  cir::GetMemberOp sea = getBuilder().createGetMember(
+      loc, fieldPtr, base.getPointer(), field->getName(),
+      rec.isUnion() ? field->getFieldIndex() : index);
   CharUnits offset = CharUnits::fromQuantity(
       rec.getElementOffset(cgm.getDataLayout().layout, index));
   return Address(sea, base.getAlignment().alignmentAtOffset(offset));
@@ -550,6 +584,37 @@ RValue CIRGenFunction::emitLoadOfLValue(LValue lv, SourceLocation loc) {
   return RValue::get(nullptr);
 }
 
+static cir::FuncOp emitFunctionDeclPointer(CIRGenModule &cgm, GlobalDecl gd) {
+  assert(!cir::MissingFeatures::weakRefReference());
+  return cgm.getAddrOfFunction(gd);
+}
+
+static LValue emitFunctionDeclLValue(CIRGenFunction &cgf, const Expr *e,
+                                     GlobalDecl gd) {
+  const FunctionDecl *fd = cast<FunctionDecl>(gd.getDecl());
+  cir::FuncOp funcOp = emitFunctionDeclPointer(cgf.cgm, gd);
+  mlir::Location loc = cgf.getLoc(e->getSourceRange());
+  CharUnits align = cgf.getContext().getDeclAlign(fd);
+
+  assert(!cir::MissingFeatures::sanitizers());
+
+  mlir::Type fnTy = funcOp.getFunctionType();
+  mlir::Type ptrTy = cir::PointerType::get(fnTy);
+  mlir::Value addr = cgf.getBuilder().create<cir::GetGlobalOp>(
+      loc, ptrTy, funcOp.getSymName());
+
+  if (funcOp.getFunctionType() != cgf.convertType(fd->getType())) {
+    fnTy = cgf.convertType(fd->getType());
+    ptrTy = cir::PointerType::get(fnTy);
+
+    addr = cir::CastOp::create(cgf.getBuilder(), addr.getLoc(), ptrTy,
+                               cir::CastKind::bitcast, addr);
+  }
+
+  return cgf.makeAddrLValue(Address(addr, fnTy, align), e->getType(),
+                            AlignmentSource::Decl);
+}
+
 LValue CIRGenFunction::emitDeclRefLValue(const DeclRefExpr *e) {
   const NamedDecl *nd = e->getDecl();
   QualType ty = e->getType();
@@ -588,6 +653,12 @@ LValue CIRGenFunction::emitDeclRefLValue(const DeclRefExpr *e) {
             ? emitLoadOfReferenceLValue(addr, getLoc(e->getSourceRange()),
                                         vd->getType(), AlignmentSource::Decl)
             : makeAddrLValue(addr, ty, AlignmentSource::Decl);
+
+    // Statics are defined as globals, so they are not include in the function's
+    // symbol table.
+    assert((vd->isStaticLocal() || symbolTable.count(vd)) &&
+           "non-static locals should be already mapped");
+
     return lv;
   }
 
@@ -600,6 +671,16 @@ LValue CIRGenFunction::emitDeclRefLValue(const DeclRefExpr *e) {
     return emitLValue(bd->getBinding());
   }
 
+  if (const auto *fd = dyn_cast<FunctionDecl>(nd)) {
+    LValue lv = emitFunctionDeclLValue(*this, e, fd);
+
+    // Emit debuginfo for the function declaration if the target wants to.
+    if (getContext().getTargetInfo().allowDebugInfoForExternalRef())
+      assert(!cir::MissingFeatures::generateDebugInfo());
+
+    return lv;
+  }
+
   cgm.errorNYI(e->getSourceRange(), "emitDeclRefLValue: unhandled decl type");
   return LValue();
 }
@@ -1013,7 +1094,9 @@ LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
   case CK_DerivedToBase: {
     const auto *derivedClassTy =
         e->getSubExpr()->getType()->castAs<clang::RecordType>();
-    auto *derivedClassDecl = cast<CXXRecordDecl>(derivedClassTy->getDecl());
+    auto *derivedClassDecl =
+        cast<CXXRecordDecl>(derivedClassTy->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     LValue lv = emitLValue(e->getSubExpr());
     Address thisAddr = lv.getAddress();
@@ -1037,10 +1120,22 @@ LValue CIRGenFunction::emitCastLValue(const CastExpr *e) {
   llvm_unreachable("Invalid cast kind");
 }
 
+static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CIRGenFunction &cgf,
+                                                        const MemberExpr *me) {
+  if (auto *vd = dyn_cast<VarDecl>(me->getMemberDecl())) {
+    // Try to emit static variable member expressions as DREs.
+    return DeclRefExpr::Create(
+        cgf.getContext(), NestedNameSpecifierLoc(), SourceLocation(), vd,
+        /*RefersToEnclosingVariableOrCapture=*/false, me->getExprLoc(),
+        me->getType(), me->getValueKind(), nullptr, nullptr, me->isNonOdrUse());
+  }
+  return nullptr;
+}
+
 LValue CIRGenFunction::emitMemberExpr(const MemberExpr *e) {
-  if (isa<VarDecl>(e->getMemberDecl())) {
-    cgm.errorNYI(e->getSourceRange(), "emitMemberExpr: VarDecl");
-    return LValue();
+  if (DeclRefExpr *dre = tryToConvertMemberExprToDeclRefExpr(*this, e)) {
+    emitIgnoredExpr(e->getBase());
+    return emitDeclRefLValue(dre);
   }
 
   Expr *baseExpr = e->getBase();
@@ -1162,26 +1257,33 @@ static void pushTemporaryCleanup(CIRGenFunction &cgf,
     return;
   }
 
-  CXXDestructorDecl *referenceTemporaryDtor = nullptr;
-  if (const clang::RecordType *rt = e->getType()
-                                        ->getBaseElementTypeUnsafe()
-                                        ->getAs<clang::RecordType>()) {
-    // Get the destructor for the reference temporary.
-    auto *classDecl = cast<CXXRecordDecl>(rt->getDecl());
-    if (!classDecl->hasTrivialDestructor())
-      referenceTemporaryDtor = classDecl->getDestructor();
-  }
-
-  if (!referenceTemporaryDtor)
+  const QualType::DestructionKind dk = e->getType().isDestructedType();
+  if (dk == QualType::DK_none)
     return;
 
-  // Call the destructor for the temporary.
   switch (m->getStorageDuration()) {
   case SD_Static:
-  case SD_Thread:
-    cgf.cgm.errorNYI(e->getSourceRange(),
-                     "pushTemporaryCleanup: static/thread storage duration");
-    return;
+  case SD_Thread: {
+    CXXDestructorDecl *referenceTemporaryDtor = nullptr;
+    if (const clang::RecordType *rt = e->getType()
+                                          ->getBaseElementTypeUnsafe()
+                                          ->getAs<clang::RecordType>()) {
+      // Get the destructor for the reference temporary.
+      if (const auto *classDecl = dyn_cast<CXXRecordDecl>(
+              rt->getOriginalDecl()->getDefinitionOrSelf())) {
+        if (!classDecl->hasTrivialDestructor())
+          referenceTemporaryDtor =
+              classDecl->getDefinitionOrSelf()->getDestructor();
+      }
+    }
+
+    if (!referenceTemporaryDtor)
+      return;
+
+    cgf.cgm.errorNYI(e->getSourceRange(), "pushTemporaryCleanup: static/thread "
+                                          "storage duration with destructors");
+    break;
+  }
 
   case SD_FullExpression:
     cgf.pushDestroy(NormalAndEHCleanup, referenceTemporary, e->getType(),
@@ -1373,11 +1475,6 @@ RValue CIRGenFunction::emitAnyExpr(const Expr *e, AggValueSlot aggSlot) {
   llvm_unreachable("bad evaluation kind");
 }
 
-static cir::FuncOp emitFunctionDeclPointer(CIRGenModule &cgm, GlobalDecl gd) {
-  assert(!cir::MissingFeatures::weakRefReference());
-  return cgm.getAddrOfFunction(gd);
-}
-
 // Detect the unusual situation where an inline version is shadowed by a
 // non-inline version. In that case we should pick the external one
 // everywhere. That's GCC behavior too.
@@ -1540,10 +1637,10 @@ CIRGenCallee CIRGenFunction::emitCallee(const clang::Expr *e) {
     cgm.errorNYI(e->getSourceRange(),
                  "emitCallee: call to member function is NYI");
     return {};
+  } else if (auto *pde = dyn_cast<CXXPseudoDestructorExpr>(e)) {
+    return CIRGenCallee::forPseudoDestructor(pde);
   }
 
-  assert(!cir::MissingFeatures::opCallPseudoDtor());
-
   // Otherwise, we have an indirect reference.
   mlir::Value calleePtr;
   QualType functionType;
@@ -1595,10 +1692,8 @@ RValue CIRGenFunction::emitCallExpr(const clang::CallExpr *e,
     return emitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(), e,
                            returnValue);
 
-  if (isa<CXXPseudoDestructorExpr>(e->getCallee())) {
-    cgm.errorNYI(e->getSourceRange(), "call to pseudo destructor");
-  }
-  assert(!cir::MissingFeatures::opCallPseudoDtor());
+  if (callee.isPseudoDestructor())
+    return emitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr());
 
   return emitCall(e->getCallee()->getType(), callee, e, returnValue);
 }
@@ -1615,7 +1710,9 @@ void CIRGenFunction::emitIgnoredExpr(const Expr *e) {
   emitLValue(e);
 }
 
-Address CIRGenFunction::emitArrayToPointerDecay(const Expr *e) {
+Address CIRGenFunction::emitArrayToPointerDecay(const Expr *e,
+                                                LValueBaseInfo *baseInfo) {
+  assert(!cir::MissingFeatures::opTBAA());
   assert(e->getType()->isArrayType() &&
          "Array to pointer decay must have array source type!");
 
@@ -1831,7 +1928,7 @@ RValue CIRGenFunction::emitCXXMemberCallExpr(const CXXMemberCallExpr *ce,
   }
 
   bool hasQualifier = me->hasQualifier();
-  NestedNameSpecifier *qualifier = hasQualifier ? me->getQualifier() : nullptr;
+  NestedNameSpecifier qualifier = me->getQualifier();
   bool isArrow = me->isArrow();
   const Expr *base = me->getBase();
 
@@ -2052,8 +2149,8 @@ cir::AllocaOp CIRGenFunction::createTempAlloca(mlir::Type ty,
 ///
 /// For named members of enums, this is the only way they are emitted.
 CIRGenFunction::ConstantEmission
-CIRGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
-  ValueDecl *value = refExpr->getDecl();
+CIRGenFunction::tryEmitAsConstant(const DeclRefExpr *refExpr) {
+  const ValueDecl *value = refExpr->getDecl();
 
   // There is a lot more to do here, but for now only EnumConstantDecl is
   // supported.
@@ -2086,6 +2183,13 @@ CIRGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
   return ConstantEmission::forValue(cstToEmit);
 }
 
+CIRGenFunction::ConstantEmission
+CIRGenFunction::tryEmitAsConstant(const MemberExpr *me) {
+  if (DeclRefExpr *dre = tryToConvertMemberExprToDeclRefExpr(*this, me))
+    return tryEmitAsConstant(dre);
+  return ConstantEmission();
+}
+
 mlir::Value CIRGenFunction::emitScalarConstant(
     const CIRGenFunction::ConstantEmission &constant, Expr *e) {
   assert(constant && "not a constant");
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp b/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
index 51aab95e02e66..2a3c98c458256 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprAggregate.cpp
@@ -69,6 +69,12 @@ class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
   void Visit(Expr *e) { StmtVisitor<AggExprEmitter>::Visit(e); }
 
   void VisitCallExpr(const CallExpr *e);
+  void VisitStmtExpr(const StmtExpr *e) {
+    CIRGenFunction::StmtExprEvaluation eval(cgf);
+    Address retAlloca =
+        cgf.createMemTemp(e->getType(), cgf.getLoc(e->getSourceRange()));
+    (void)cgf.emitCompoundStmt(*e->getSubStmt(), &retAlloca, dest);
+  }
 
   void VisitDeclRefExpr(DeclRefExpr *e) { emitAggLoadOfLValue(e); }
 
@@ -124,8 +130,8 @@ void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
   const QualType elementType =
       cgf.getContext().getAsArrayType(arrayQTy)->getElementType();
 
-  if (elementType.isDestructedType()) {
-    cgf.cgm.errorNYI(loc, "dtorKind NYI");
+  if (elementType.isDestructedType() && cgf.cgm.getLangOpts().Exceptions) {
+    cgf.cgm.errorNYI(loc, "initialized array requires destruction");
     return;
   }
 
@@ -135,9 +141,9 @@ void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
   const cir::PointerType cirElementPtrType =
       builder.getPointerTo(cirElementType);
 
-  auto begin = builder.create<cir::CastOp>(loc, cirElementPtrType,
-                                           cir::CastKind::array_to_ptrdecay,
-                                           destPtr.getPointer());
+  auto begin = cir::CastOp::create(builder, loc, cirElementPtrType,
+                                   cir::CastKind::array_to_ptrdecay,
+                                   destPtr.getPointer());
 
   const CharUnits elementSize =
       cgf.getContext().getTypeSizeInChars(elementType);
@@ -182,8 +188,8 @@ void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
     // Advance to the start of the rest of the array.
     if (numInitElements) {
       one = builder.getConstantInt(loc, cgf.PtrDiffTy, 1);
-      element = builder.create<cir::PtrStrideOp>(loc, cirElementPtrType,
-                                                 element, one);
+      element = cir::PtrStrideOp::create(builder, loc, cirElementPtrType,
+                                         element, one);
     }
 
     // Allocate the temporary variable
@@ -193,25 +199,52 @@ void AggExprEmitter::emitArrayInit(Address destPtr, cir::ArrayType arrayTy,
     LValue tmpLV = cgf.makeAddrLValue(tmpAddr, elementPtrType);
     cgf.emitStoreThroughLValue(RValue::get(element), tmpLV);
 
-    // TODO(CIR): Replace this part later with cir::DoWhileOp
-    for (unsigned i = numInitElements; i != numArrayElements; ++i) {
-      cir::LoadOp currentElement = builder.createLoad(loc, tmpAddr);
-
-      // Emit the actual filler expression.
-      const LValue elementLV = cgf.makeAddrLValue(
-          Address(currentElement, cirElementType, elementAlign), elementType);
-
-      if (arrayFiller)
-        emitInitializationToLValue(arrayFiller, elementLV);
-      else
-        emitNullInitializationToLValue(loc, elementLV);
-
-      // Advance pointer and store them to temporary variable
-      one = builder.getConstantInt(loc, cgf.PtrDiffTy, 1);
-      cir::PtrStrideOp nextElement =
-          builder.createPtrStride(loc, currentElement, one);
-      cgf.emitStoreThroughLValue(RValue::get(nextElement), tmpLV);
-    }
+    // Compute the end of array
+    cir::ConstantOp numArrayElementsConst = builder.getConstInt(
+        loc, mlir::cast<cir::IntType>(cgf.PtrDiffTy), numArrayElements);
+    mlir::Value end = cir::PtrStrideOp::create(builder, loc, cirElementPtrType,
+                                               begin, numArrayElementsConst);
+
+    builder.createDoWhile(
+        loc,
+        /*condBuilder=*/
+        [&](mlir::OpBuilder &b, mlir::Location loc) {
+          cir::LoadOp currentElement = builder.createLoad(loc, tmpAddr);
+          mlir::Type boolTy = cgf.convertType(cgf.getContext().BoolTy);
+          cir::CmpOp cmp = cir::CmpOp::create(
+              builder, loc, boolTy, cir::CmpOpKind::ne, currentElement, end);
+          builder.createCondition(cmp);
+        },
+        /*bodyBuilder=*/
+        [&](mlir::OpBuilder &b, mlir::Location loc) {
+          cir::LoadOp currentElement = builder.createLoad(loc, tmpAddr);
+
+          assert(!cir::MissingFeatures::requiresCleanups());
+
+          // Emit the actual filler expression.
+          LValue elementLV = cgf.makeAddrLValue(
+              Address(currentElement, cirElementType, elementAlign),
+              elementType);
+          if (arrayFiller)
+            emitInitializationToLValue(arrayFiller, elementLV);
+          else
+            emitNullInitializationToLValue(loc, elementLV);
+
+          // Tell the EH cleanup that we finished with the last element.
+          if (cgf.cgm.getLangOpts().Exceptions) {
+            cgf.cgm.errorNYI(loc, "update destructed array element for EH");
+            return;
+          }
+
+          // Advance pointer and store them to temporary variable
+          cir::ConstantOp one = builder.getConstInt(
+              loc, mlir::cast<cir::IntType>(cgf.PtrDiffTy), 1);
+          auto nextElement = cir::PtrStrideOp::create(
+              builder, loc, cirElementPtrType, currentElement, one);
+          cgf.emitStoreThroughLValue(RValue::get(nextElement), tmpLV);
+
+          builder.createYield(loc);
+        });
   }
 }
 
@@ -376,7 +409,10 @@ void AggExprEmitter::visitCXXParenListOrInitListExpr(
   // the disadvantage is that the generated code is more difficult for
   // the optimizer, especially with bitfields.
   unsigned numInitElements = args.size();
-  RecordDecl *record = e->getType()->castAs<RecordType>()->getDecl();
+  RecordDecl *record = e->getType()
+                           ->castAs<RecordType>()
+                           ->getOriginalDecl()
+                           ->getDefinitionOrSelf();
 
   // We'll need to enter cleanup scopes in case any of the element
   // initializers throws an exception.
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprCXX.cpp b/clang/lib/CIR/CodeGen/CIRGenExprCXX.cpp
new file mode 100644
index 0000000000000..a320508cd0461
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenExprCXX.cpp
@@ -0,0 +1,36 @@
+//===--- CIRGenExprCXX.cpp - Emit CIR Code for C++ expressions ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code dealing with code generation of C++ expressions
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenFunction.h"
+#include "clang/AST/ExprCXX.h"
+
+using namespace clang;
+using namespace clang::CIRGen;
+
+RValue CIRGenFunction::emitCXXPseudoDestructorExpr(
+    const CXXPseudoDestructorExpr *expr) {
+  QualType destroyedType = expr->getDestroyedType();
+  if (destroyedType.hasStrongOrWeakObjCLifetime()) {
+    assert(!cir::MissingFeatures::objCLifetime());
+    cgm.errorNYI(expr->getExprLoc(),
+                 "emitCXXPseudoDestructorExpr: Objective-C lifetime is NYI");
+  } else {
+    // C++ [expr.pseudo]p1:
+    //   The result shall only be used as the operand for the function call
+    //   operator (), and the result of such a call has type void. The only
+    //   effect is the evaluation of the postfix-expression before the dot or
+    //   arrow.
+    emitIgnoredExpr(expr->getBase());
+  }
+
+  return RValue::get(nullptr);
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index 3aa170eff8606..bb1b55f2d16f4 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -62,6 +62,14 @@ class ComplexExprEmitter : public StmtVisitor<ComplexExprEmitter, mlir::Value> {
   mlir::Value VisitImplicitCastExpr(ImplicitCastExpr *e);
   mlir::Value VisitInitListExpr(InitListExpr *e);
 
+  mlir::Value VisitMemberExpr(MemberExpr *me) {
+    if (CIRGenFunction::ConstantEmission constant = cgf.tryEmitAsConstant(me)) {
+      cgf.emitIgnoredExpr(me->getBase());
+      return emitConstant(constant, me);
+    }
+    return emitLoadOfLValue(me);
+  }
+
   mlir::Value VisitCompoundLiteralExpr(CompoundLiteralExpr *e) {
     return emitLoadOfLValue(e);
   }
@@ -116,20 +124,23 @@ class ComplexExprEmitter : public StmtVisitor<ComplexExprEmitter, mlir::Value> {
 
   mlir::Value emitPromotedComplexOperand(const Expr *e, QualType promotionTy);
 
+  LValue emitCompoundAssignLValue(
+      const CompoundAssignOperator *e,
+      mlir::Value (ComplexExprEmitter::*func)(const BinOpInfo &),
+      RValue &value);
+
+  mlir::Value emitCompoundAssign(
+      const CompoundAssignOperator *e,
+      mlir::Value (ComplexExprEmitter::*func)(const BinOpInfo &));
+
   mlir::Value emitBinAdd(const BinOpInfo &op);
   mlir::Value emitBinSub(const BinOpInfo &op);
   mlir::Value emitBinMul(const BinOpInfo &op);
+  mlir::Value emitBinDiv(const BinOpInfo &op);
 
   QualType getPromotionType(QualType ty, bool isDivOpCode = false) {
     if (auto *complexTy = ty->getAs<ComplexType>()) {
       QualType elementTy = complexTy->getElementType();
-      if (isDivOpCode && elementTy->isFloatingType() &&
-          cgf.getLangOpts().getComplexRange() ==
-              LangOptions::ComplexRangeKind::CX_Promoted) {
-        cgf.cgm.errorNYI("HigherPrecisionTypeForComplexArithmetic");
-        return QualType();
-      }
-
       if (elementTy.UseExcessPrecision(cgf.getContext()))
         return cgf.getContext().getComplexType(cgf.getContext().FloatTy);
     }
@@ -145,14 +156,28 @@ class ComplexExprEmitter : public StmtVisitor<ComplexExprEmitter, mlir::Value> {
         e->getType(), e->getOpcode() == BinaryOperatorKind::BO_Div);           \
     mlir::Value result = emitBin##OP(emitBinOps(e, promotionTy));              \
     if (!promotionTy.isNull())                                                 \
-      cgf.cgm.errorNYI("Binop emitUnPromotedValue");                           \
+      result = cgf.emitUnPromotedValue(result, e->getType());                  \
     return result;                                                             \
   }
 
   HANDLEBINOP(Add)
   HANDLEBINOP(Sub)
   HANDLEBINOP(Mul)
+  HANDLEBINOP(Div)
 #undef HANDLEBINOP
+
+  // Compound assignments.
+  mlir::Value VisitBinAddAssign(const CompoundAssignOperator *e) {
+    return emitCompoundAssign(e, &ComplexExprEmitter::emitBinAdd);
+  }
+
+  mlir::Value VisitBinSubAssign(const CompoundAssignOperator *e) {
+    return emitCompoundAssign(e, &ComplexExprEmitter::emitBinSub);
+  }
+
+  mlir::Value VisitBinMulAssign(const CompoundAssignOperator *e) {
+    return emitCompoundAssign(e, &ComplexExprEmitter::emitBinMul);
+  }
 };
 } // namespace
 
@@ -602,7 +627,7 @@ mlir::Value ComplexExprEmitter::emitPromoted(const Expr *e,
 
   mlir::Value result = Visit(const_cast<Expr *>(e));
   if (!promotionTy.isNull())
-    cgf.cgm.errorNYI("emitPromoted emitPromotedValue");
+    return cgf.emitPromotedValue(result, promotionTy);
 
   return result;
 }
@@ -616,8 +641,12 @@ ComplexExprEmitter::emitPromotedComplexOperand(const Expr *e,
     return Visit(const_cast<Expr *>(e));
   }
 
-  cgf.cgm.errorNYI("emitPromotedComplexOperand non-complex type");
-  return {};
+  if (!promotionTy.isNull()) {
+    QualType complexElementTy =
+        promotionTy->castAs<ComplexType>()->getElementType();
+    return cgf.emitPromotedScalarExpr(e, complexElementTy);
+  }
+  return cgf.emitScalarExpr(e);
 }
 
 ComplexExprEmitter::BinOpInfo
@@ -630,16 +659,162 @@ ComplexExprEmitter::emitBinOps(const BinaryOperator *e, QualType promotionTy) {
   return binOpInfo;
 }
 
+LValue ComplexExprEmitter::emitCompoundAssignLValue(
+    const CompoundAssignOperator *e,
+    mlir::Value (ComplexExprEmitter::*func)(const BinOpInfo &), RValue &value) {
+  QualType lhsTy = e->getLHS()->getType();
+  QualType rhsTy = e->getRHS()->getType();
+  SourceLocation exprLoc = e->getExprLoc();
+  mlir::Location loc = cgf.getLoc(exprLoc);
+
+  if (lhsTy->getAs<AtomicType>()) {
+    cgf.cgm.errorNYI("emitCompoundAssignLValue AtmoicType");
+    return {};
+  }
+
+  BinOpInfo opInfo{loc};
+  opInfo.fpFeatures = e->getFPFeaturesInEffect(cgf.getLangOpts());
+
+  assert(!cir::MissingFeatures::cgFPOptionsRAII());
+
+  // Load the RHS and LHS operands.
+  // __block variables need to have the rhs evaluated first, plus this should
+  // improve codegen a little.
+  QualType promotionTypeCR = getPromotionType(e->getComputationResultType());
+  opInfo.ty = promotionTypeCR.isNull() ? e->getComputationResultType()
+                                       : promotionTypeCR;
+
+  QualType complexElementTy =
+      opInfo.ty->castAs<ComplexType>()->getElementType();
+  QualType promotionTypeRHS = getPromotionType(rhsTy);
+
+  // The RHS should have been converted to the computation type.
+  if (e->getRHS()->getType()->isRealFloatingType()) {
+    if (!promotionTypeRHS.isNull()) {
+      opInfo.rhs = cgf.emitPromotedScalarExpr(e->getRHS(), promotionTypeRHS);
+    } else {
+      assert(cgf.getContext().hasSameUnqualifiedType(complexElementTy, rhsTy));
+      opInfo.rhs = cgf.emitScalarExpr(e->getRHS());
+    }
+  } else {
+    if (!promotionTypeRHS.isNull()) {
+      opInfo.rhs = cgf.emitPromotedComplexExpr(e->getRHS(), promotionTypeRHS);
+    } else {
+      assert(cgf.getContext().hasSameUnqualifiedType(opInfo.ty, rhsTy));
+      opInfo.rhs = Visit(e->getRHS());
+    }
+  }
+
+  LValue lhs = cgf.emitLValue(e->getLHS());
+
+  // Load from the l-value and convert it.
+  QualType promotionTypeLHS = getPromotionType(e->getComputationLHSType());
+  if (lhsTy->isAnyComplexType()) {
+    mlir::Value lhsValue = emitLoadOfLValue(lhs, exprLoc);
+    QualType destTy = promotionTypeLHS.isNull() ? opInfo.ty : promotionTypeLHS;
+    opInfo.lhs = emitComplexToComplexCast(lhsValue, lhsTy, destTy, exprLoc);
+  } else {
+    mlir::Value lhsVal = cgf.emitLoadOfScalar(lhs, exprLoc);
+    // For floating point real operands we can directly pass the scalar form
+    // to the binary operator emission and potentially get more efficient code.
+    if (lhsTy->isRealFloatingType()) {
+      QualType promotedComplexElementTy;
+      if (!promotionTypeLHS.isNull()) {
+        promotedComplexElementTy =
+            cast<ComplexType>(promotionTypeLHS)->getElementType();
+        if (!cgf.getContext().hasSameUnqualifiedType(promotedComplexElementTy,
+                                                     promotionTypeLHS))
+          lhsVal = cgf.emitScalarConversion(lhsVal, lhsTy,
+                                            promotedComplexElementTy, exprLoc);
+      } else {
+        if (!cgf.getContext().hasSameUnqualifiedType(complexElementTy, lhsTy))
+          lhsVal = cgf.emitScalarConversion(lhsVal, lhsTy, complexElementTy,
+                                            exprLoc);
+      }
+      opInfo.lhs = lhsVal;
+    } else {
+      opInfo.lhs = emitScalarToComplexCast(lhsVal, lhsTy, opInfo.ty, exprLoc);
+    }
+  }
+
+  // Expand the binary operator.
+  mlir::Value result = (this->*func)(opInfo);
+
+  // Truncate the result and store it into the LHS lvalue.
+  if (lhsTy->isAnyComplexType()) {
+    mlir::Value resultValue =
+        emitComplexToComplexCast(result, opInfo.ty, lhsTy, exprLoc);
+    emitStoreOfComplex(loc, resultValue, lhs, /*isInit*/ false);
+    value = RValue::getComplex(resultValue);
+  } else {
+    mlir::Value resultValue =
+        cgf.emitComplexToScalarConversion(result, opInfo.ty, lhsTy, exprLoc);
+    cgf.emitStoreOfScalar(resultValue, lhs, /*isInit*/ false);
+    value = RValue::get(resultValue);
+  }
+
+  return lhs;
+}
+
+mlir::Value ComplexExprEmitter::emitCompoundAssign(
+    const CompoundAssignOperator *e,
+    mlir::Value (ComplexExprEmitter::*func)(const BinOpInfo &)) {
+  RValue val;
+  LValue lv = emitCompoundAssignLValue(e, func, val);
+
+  // The result of an assignment in C is the assigned r-value.
+  if (!cgf.getLangOpts().CPlusPlus)
+    return val.getComplexValue();
+
+  // If the lvalue is non-volatile, return the computed value of the assignment.
+  if (!lv.isVolatileQualified())
+    return val.getComplexValue();
+
+  return emitLoadOfLValue(lv, e->getExprLoc());
+}
+
 mlir::Value ComplexExprEmitter::emitBinAdd(const BinOpInfo &op) {
   assert(!cir::MissingFeatures::fastMathFlags());
   assert(!cir::MissingFeatures::cgFPOptionsRAII());
-  return builder.create<cir::ComplexAddOp>(op.loc, op.lhs, op.rhs);
+
+  if (mlir::isa<cir::ComplexType>(op.lhs.getType()) &&
+      mlir::isa<cir::ComplexType>(op.rhs.getType()))
+    return builder.create<cir::ComplexAddOp>(op.loc, op.lhs, op.rhs);
+
+  if (mlir::isa<cir::ComplexType>(op.lhs.getType())) {
+    mlir::Value real = builder.createComplexReal(op.loc, op.lhs);
+    mlir::Value imag = builder.createComplexImag(op.loc, op.lhs);
+    mlir::Value newReal = builder.createAdd(op.loc, real, op.rhs);
+    return builder.createComplexCreate(op.loc, newReal, imag);
+  }
+
+  assert(mlir::isa<cir::ComplexType>(op.rhs.getType()));
+  mlir::Value real = builder.createComplexReal(op.loc, op.rhs);
+  mlir::Value imag = builder.createComplexImag(op.loc, op.rhs);
+  mlir::Value newReal = builder.createAdd(op.loc, op.lhs, real);
+  return builder.createComplexCreate(op.loc, newReal, imag);
 }
 
 mlir::Value ComplexExprEmitter::emitBinSub(const BinOpInfo &op) {
   assert(!cir::MissingFeatures::fastMathFlags());
   assert(!cir::MissingFeatures::cgFPOptionsRAII());
-  return builder.create<cir::ComplexSubOp>(op.loc, op.lhs, op.rhs);
+
+  if (mlir::isa<cir::ComplexType>(op.lhs.getType()) &&
+      mlir::isa<cir::ComplexType>(op.rhs.getType()))
+    return builder.create<cir::ComplexSubOp>(op.loc, op.lhs, op.rhs);
+
+  if (mlir::isa<cir::ComplexType>(op.lhs.getType())) {
+    mlir::Value real = builder.createComplexReal(op.loc, op.lhs);
+    mlir::Value imag = builder.createComplexImag(op.loc, op.lhs);
+    mlir::Value newReal = builder.createSub(op.loc, real, op.rhs);
+    return builder.createComplexCreate(op.loc, newReal, imag);
+  }
+
+  assert(mlir::isa<cir::ComplexType>(op.rhs.getType()));
+  mlir::Value real = builder.createComplexReal(op.loc, op.rhs);
+  mlir::Value imag = builder.createComplexImag(op.loc, op.rhs);
+  mlir::Value newReal = builder.createSub(op.loc, op.lhs, real);
+  return builder.createComplexCreate(op.loc, newReal, imag);
 }
 
 static cir::ComplexRangeKind
@@ -654,7 +829,7 @@ getComplexRangeAttr(LangOptions::ComplexRangeKind range) {
   case LangOptions::CX_Basic:
     return cir::ComplexRangeKind::Basic;
   case LangOptions::CX_None:
-    // The default value for ComplexRangeKind is Full is no option is selected
+    // The default value for ComplexRangeKind is Full if no option is selected
     return cir::ComplexRangeKind::Full;
   }
 }
@@ -662,9 +837,44 @@ getComplexRangeAttr(LangOptions::ComplexRangeKind range) {
 mlir::Value ComplexExprEmitter::emitBinMul(const BinOpInfo &op) {
   assert(!cir::MissingFeatures::fastMathFlags());
   assert(!cir::MissingFeatures::cgFPOptionsRAII());
-  cir::ComplexRangeKind rangeKind =
-      getComplexRangeAttr(op.fpFeatures.getComplexRange());
-  return builder.create<cir::ComplexMulOp>(op.loc, op.lhs, op.rhs, rangeKind);
+
+  if (mlir::isa<cir::ComplexType>(op.lhs.getType()) &&
+      mlir::isa<cir::ComplexType>(op.rhs.getType())) {
+    cir::ComplexRangeKind rangeKind =
+        getComplexRangeAttr(op.fpFeatures.getComplexRange());
+    return builder.create<cir::ComplexMulOp>(op.loc, op.lhs, op.rhs, rangeKind);
+  }
+
+  if (mlir::isa<cir::ComplexType>(op.lhs.getType())) {
+    mlir::Value real = builder.createComplexReal(op.loc, op.lhs);
+    mlir::Value imag = builder.createComplexImag(op.loc, op.lhs);
+    mlir::Value newReal = builder.createMul(op.loc, real, op.rhs);
+    mlir::Value newImag = builder.createMul(op.loc, imag, op.rhs);
+    return builder.createComplexCreate(op.loc, newReal, newImag);
+  }
+
+  assert(mlir::isa<cir::ComplexType>(op.rhs.getType()));
+  mlir::Value real = builder.createComplexReal(op.loc, op.rhs);
+  mlir::Value imag = builder.createComplexImag(op.loc, op.rhs);
+  mlir::Value newReal = builder.createMul(op.loc, op.lhs, real);
+  mlir::Value newImag = builder.createMul(op.loc, op.lhs, imag);
+  return builder.createComplexCreate(op.loc, newReal, newImag);
+}
+
+mlir::Value ComplexExprEmitter::emitBinDiv(const BinOpInfo &op) {
+  assert(!cir::MissingFeatures::fastMathFlags());
+  assert(!cir::MissingFeatures::cgFPOptionsRAII());
+
+  if (mlir::isa<cir::ComplexType>(op.lhs.getType()) &&
+      mlir::isa<cir::ComplexType>(op.rhs.getType())) {
+    cir::ComplexRangeKind rangeKind =
+        getComplexRangeAttr(op.fpFeatures.getComplexRange());
+    return cir::ComplexDivOp::create(builder, op.loc, op.lhs, op.rhs,
+                                     rangeKind);
+  }
+
+  cgf.cgm.errorNYI("ComplexExprEmitter::emitBinDiv between Complex & Scalar");
+  return {};
 }
 
 LValue CIRGenFunction::emitComplexAssignmentLValue(const BinaryOperator *e) {
@@ -685,6 +895,31 @@ mlir::Value CIRGenFunction::emitComplexExpr(const Expr *e) {
   return ComplexExprEmitter(*this).Visit(const_cast<Expr *>(e));
 }
 
+using CompoundFunc =
+    mlir::Value (ComplexExprEmitter::*)(const ComplexExprEmitter::BinOpInfo &);
+
+static CompoundFunc getComplexOp(BinaryOperatorKind op) {
+  switch (op) {
+  case BO_MulAssign:
+    return &ComplexExprEmitter::emitBinMul;
+  case BO_DivAssign:
+    llvm_unreachable("getComplexOp: BO_DivAssign");
+  case BO_SubAssign:
+    return &ComplexExprEmitter::emitBinSub;
+  case BO_AddAssign:
+    return &ComplexExprEmitter::emitBinAdd;
+  default:
+    llvm_unreachable("unexpected complex compound assignment");
+  }
+}
+
+LValue CIRGenFunction::emitComplexCompoundAssignmentLValue(
+    const CompoundAssignOperator *e) {
+  CompoundFunc op = getComplexOp(e->getOpcode());
+  RValue val;
+  return ComplexExprEmitter(*this).emitCompoundAssignLValue(e, op, val);
+}
+
 mlir::Value CIRGenFunction::emitComplexPrePostIncDec(const UnaryOperator *e,
                                                      LValue lv,
                                                      cir::UnaryOpKind op,
@@ -729,3 +964,28 @@ mlir::Value CIRGenFunction::emitPromotedComplexExpr(const Expr *e,
                                                     QualType promotionType) {
   return ComplexExprEmitter(*this).emitPromoted(e, promotionType);
 }
+
+mlir::Value CIRGenFunction::emitPromotedValue(mlir::Value result,
+                                              QualType promotionType) {
+  assert(!mlir::cast<cir::ComplexType>(result.getType()).isIntegerComplex() &&
+         "integral complex will never be promoted");
+  return builder.createCast(cir::CastKind::float_complex, result,
+                            convertType(promotionType));
+}
+
+mlir::Value CIRGenFunction::emitUnPromotedValue(mlir::Value result,
+                                                QualType unPromotionType) {
+  assert(!mlir::cast<cir::ComplexType>(result.getType()).isIntegerComplex() &&
+         "integral complex will never be promoted");
+  return builder.createCast(cir::CastKind::float_complex, result,
+                            convertType(unPromotionType));
+}
+
+LValue CIRGenFunction::emitScalarCompoundAssignWithComplex(
+    const CompoundAssignOperator *e, mlir::Value &result) {
+  CompoundFunc op = getComplexOp(e->getOpcode());
+  RValue value;
+  LValue ret = ComplexExprEmitter(*this).emitCompoundAssignLValue(e, op, value);
+  result = value.getValue();
+  return ret;
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
index 5b3bf85cefbb0..ced3c2273e865 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprConstant.cpp
@@ -285,7 +285,7 @@ emitArrayConstant(CIRGenModule &cgm, mlir::Type desiredType,
                   mlir::Type commonElementType, unsigned arrayBound,
                   SmallVectorImpl<mlir::TypedAttr> &elements,
                   mlir::TypedAttr filler) {
-  const CIRGenBuilderTy &builder = cgm.getBuilder();
+  CIRGenBuilderTy &builder = cgm.getBuilder();
 
   unsigned nonzeroLength = arrayBound;
   if (elements.size() < nonzeroLength && builder.isNullValue(filler))
@@ -306,6 +306,33 @@ emitArrayConstant(CIRGenModule &cgm, mlir::Type desiredType,
   if (trailingZeroes >= 8) {
     assert(elements.size() >= nonzeroLength &&
            "missing initializer for non-zero element");
+
+    if (commonElementType && nonzeroLength >= 8) {
+      // If all the elements had the same type up to the trailing zeroes and
+      // there are eight or more nonzero elements, emit a struct of two arrays
+      // (the nonzero data and the zeroinitializer).
+      SmallVector<mlir::Attribute, 4> eles;
+      eles.reserve(nonzeroLength);
+      for (const auto &element : elements)
+        eles.push_back(element);
+      auto initial = cir::ConstArrayAttr::get(
+          cir::ArrayType::get(commonElementType, nonzeroLength),
+          mlir::ArrayAttr::get(builder.getContext(), eles));
+      elements.resize(2);
+      elements[0] = initial;
+    } else {
+      // Otherwise, emit a struct with individual elements for each nonzero
+      // initializer, followed by a zeroinitializer array filler.
+      elements.resize(nonzeroLength + 1);
+    }
+
+    mlir::Type fillerType =
+        commonElementType
+            ? commonElementType
+            : mlir::cast<cir::ArrayType>(desiredType).getElementType();
+    fillerType = cir::ArrayType::get(fillerType, trailingZeroes);
+    elements.back() = cir::ZeroAttr::get(fillerType);
+    commonElementType = nullptr;
   } else if (elements.size() != arrayBound) {
     elements.resize(arrayBound, filler);
 
@@ -325,8 +352,13 @@ emitArrayConstant(CIRGenModule &cgm, mlir::Type desiredType,
         mlir::ArrayAttr::get(builder.getContext(), eles));
   }
 
-  cgm.errorNYI("array with different type elements");
-  return {};
+  SmallVector<mlir::Attribute, 4> eles;
+  eles.reserve(elements.size());
+  for (auto const &element : elements)
+    eles.push_back(element);
+
+  auto arrAttr = mlir::ArrayAttr::get(builder.getContext(), eles);
+  return builder.getAnonConstRecord(arrAttr, /*isPacked=*/true);
 }
 
 //===----------------------------------------------------------------------===//
@@ -340,7 +372,11 @@ struct ConstantLValue {
   llvm::PointerUnion<mlir::Value, mlir::Attribute> value;
   bool hasOffsetApplied;
 
-  ConstantLValue(std::nullptr_t) : value(nullptr), hasOffsetApplied(false) {}
+  /*implicit*/ ConstantLValue(std::nullptr_t)
+      : value(nullptr), hasOffsetApplied(false) {}
+  /*implicit*/ ConstantLValue(cir::GlobalViewAttr address)
+      : value(address), hasOffsetApplied(false) {}
+
   ConstantLValue() : value(nullptr), hasOffsetApplied(false) {}
 };
 
@@ -380,6 +416,43 @@ class ConstantLValueEmitter
   ConstantLValue VisitCXXTypeidExpr(const CXXTypeidExpr *e);
   ConstantLValue
   VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *e);
+
+  /// Return GEP-like value offset
+  mlir::ArrayAttr getOffset(mlir::Type ty) {
+    int64_t offset = value.getLValueOffset().getQuantity();
+    cir::CIRDataLayout layout(cgm.getModule());
+    SmallVector<int64_t, 3> idxVec;
+    cgm.getBuilder().computeGlobalViewIndicesFromFlatOffset(offset, ty, layout,
+                                                            idxVec);
+
+    llvm::SmallVector<mlir::Attribute, 3> indices;
+    for (int64_t i : idxVec) {
+      mlir::IntegerAttr intAttr = cgm.getBuilder().getI32IntegerAttr(i);
+      indices.push_back(intAttr);
+    }
+
+    if (indices.empty())
+      return {};
+    return cgm.getBuilder().getArrayAttr(indices);
+  }
+
+  /// Apply the value offset to the given constant.
+  ConstantLValue applyOffset(ConstantLValue &c) {
+    // Handle attribute constant LValues.
+    if (auto attr = mlir::dyn_cast<mlir::Attribute>(c.value)) {
+      if (auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(attr)) {
+        auto baseTy = mlir::cast<cir::PointerType>(gv.getType()).getPointee();
+        mlir::Type destTy = cgm.getTypes().convertTypeForMem(destType);
+        assert(!gv.getIndices() && "Global view is already indexed");
+        return cir::GlobalViewAttr::get(destTy, gv.getSymbol(),
+                                        getOffset(baseTy));
+      }
+      llvm_unreachable("Unsupported attribute type to offset");
+    }
+
+    cgm.errorNYI("ConstantLValue: non-attribute offset");
+    return {};
+  }
 };
 
 } // namespace
@@ -411,10 +484,8 @@ mlir::Attribute ConstantLValueEmitter::tryEmit() {
     return {};
 
   // Apply the offset if necessary and not already done.
-  if (!result.hasOffsetApplied) {
-    cgm.errorNYI("ConstantLValueEmitter: apply offset");
-    return {};
-  }
+  if (!result.hasOffsetApplied)
+    value = applyOffset(result).value;
 
   // Convert to the appropriate type; this could be an lvalue for
   // an integer. FIXME: performAddrSpaceCast
@@ -453,15 +524,35 @@ ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) {
     }
 
     if (auto *fd = dyn_cast<FunctionDecl>(d)) {
-      cgm.errorNYI(fd->getSourceRange(),
-                   "ConstantLValueEmitter: function decl");
-      return {};
+      cir::FuncOp fop = cgm.getAddrOfFunction(fd);
+      CIRGenBuilderTy &builder = cgm.getBuilder();
+      mlir::MLIRContext *mlirContext = builder.getContext();
+      return cir::GlobalViewAttr::get(
+          builder.getPointerTo(fop.getFunctionType()),
+          mlir::FlatSymbolRefAttr::get(mlirContext, fop.getSymNameAttr()));
     }
 
     if (auto *vd = dyn_cast<VarDecl>(d)) {
-      cgm.errorNYI(vd->getSourceRange(), "ConstantLValueEmitter: var decl");
-      return {};
+      // We can never refer to a variable with local storage.
+      if (!vd->hasLocalStorage()) {
+        if (vd->isFileVarDecl() || vd->hasExternalStorage())
+          return cgm.getAddrOfGlobalVarAttr(vd);
+
+        if (vd->isLocalVarDecl()) {
+          cgm.errorNYI(vd->getSourceRange(),
+                       "ConstantLValueEmitter: local var decl");
+          return {};
+        }
+      }
     }
+
+    // Classic codegen handles MSGuidDecl,UnnamedGlobalConstantDecl, and
+    // TemplateParamObjectDecl, but it can also fall through from VarDecl,
+    // in which case it silently returns nullptr. For now, let's emit an
+    // error to see what cases we need to handle.
+    cgm.errorNYI(d->getSourceRange(),
+                 "ConstantLValueEmitter: unhandled value decl");
+    return {};
   }
 
   // Handle typeid(T).
@@ -487,8 +578,7 @@ ConstantLValueEmitter::VisitCompoundLiteralExpr(const CompoundLiteralExpr *e) {
 
 ConstantLValue
 ConstantLValueEmitter::VisitStringLiteral(const StringLiteral *e) {
-  cgm.errorNYI(e->getSourceRange(), "ConstantLValueEmitter: string literal");
-  return {};
+  return cgm.getAddrOfConstantStringFromLiteral(e);
 }
 
 ConstantLValue
@@ -591,7 +681,8 @@ mlir::Attribute ConstantEmitter::tryEmitPrivateForVarInit(const VarDecl &d) {
         // be a problem for the near future.
         if (cd->isTrivial() && cd->isDefaultConstructor()) {
           const auto *cxxrd =
-              cast<CXXRecordDecl>(ty->getAs<RecordType>()->getDecl());
+              cast<CXXRecordDecl>(ty->getAs<RecordType>()->getOriginalDecl())
+                  ->getDefinitionOrSelf();
           if (cxxrd->getNumBases() != 0) {
             // There may not be anything additional to do here, but this will
             // force us to pause and test this path when it is supported.
@@ -650,6 +741,16 @@ mlir::Attribute ConstantEmitter::tryEmitPrivateForMemory(const APValue &value,
   return (c ? emitForMemory(c, destType) : nullptr);
 }
 
+mlir::Attribute ConstantEmitter::emitAbstract(const Expr *e,
+                                              QualType destType) {
+  AbstractStateRAII state{*this, true};
+  mlir::Attribute c = mlir::cast<mlir::Attribute>(tryEmitPrivate(e, destType));
+  if (!c)
+    cgm.errorNYI(e->getSourceRange(),
+                 "emitAbstract failed, emit null constaant");
+  return c;
+}
+
 mlir::Attribute ConstantEmitter::emitAbstract(SourceLocation loc,
                                               const APValue &value,
                                               QualType destType) {
@@ -671,6 +772,32 @@ mlir::Attribute ConstantEmitter::emitForMemory(mlir::Attribute c,
   return c;
 }
 
+mlir::TypedAttr ConstantEmitter::tryEmitPrivate(const Expr *e,
+                                                QualType destType) {
+  assert(!destType->isVoidType() && "can't emit a void constant");
+
+  if (mlir::Attribute c =
+          ConstExprEmitter(*this).Visit(const_cast<Expr *>(e), destType))
+    return llvm::dyn_cast<mlir::TypedAttr>(c);
+
+  Expr::EvalResult result;
+
+  bool success = false;
+
+  if (destType->isReferenceType())
+    success = e->EvaluateAsLValue(result, cgm.getASTContext());
+  else
+    success =
+        e->EvaluateAsRValue(result, cgm.getASTContext(), inConstantContext);
+
+  if (success && !result.hasSideEffects()) {
+    mlir::Attribute c = tryEmitPrivate(result.Val, destType);
+    return llvm::dyn_cast<mlir::TypedAttr>(c);
+  }
+
+  return nullptr;
+}
+
 mlir::Attribute ConstantEmitter::tryEmitPrivate(const APValue &value,
                                                 QualType destType) {
   auto &builder = cgm.getBuilder();
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
index 32c1c1ada3c53..46934e7155adf 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprScalar.cpp
@@ -185,6 +185,21 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
   mlir::Value VisitCastExpr(CastExpr *e);
   mlir::Value VisitCallExpr(const CallExpr *e);
 
+  mlir::Value VisitStmtExpr(StmtExpr *e) {
+    CIRGenFunction::StmtExprEvaluation eval(cgf);
+    if (e->getType()->isVoidType()) {
+      (void)cgf.emitCompoundStmt(*e->getSubStmt());
+      return {};
+    }
+
+    Address retAlloca =
+        cgf.createMemTemp(e->getType(), cgf.getLoc(e->getSourceRange()));
+    (void)cgf.emitCompoundStmt(*e->getSubStmt(), &retAlloca);
+
+    return cgf.emitLoadOfScalar(cgf.makeAddrLValue(retAlloca, e->getType()),
+                                e->getExprLoc());
+  }
+
   mlir::Value VisitArraySubscriptExpr(ArraySubscriptExpr *e) {
     if (e->getBase()->getType()->isVectorType()) {
       assert(!cir::MissingFeatures::scalableVectors());
@@ -384,6 +399,17 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
     return Visit(e->getReplacement());
   }
 
+  mlir::Value VisitVAArgExpr(VAArgExpr *ve) {
+    QualType ty = ve->getType();
+
+    if (ty->isVariablyModifiedType()) {
+      cgf.cgm.errorNYI(ve->getSourceRange(),
+                       "variably modified types in varargs");
+    }
+
+    return cgf.emitVAArg(ve);
+  }
+
   mlir::Value VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *e);
   mlir::Value
   VisitAbstractConditionalOperator(const AbstractConditionalOperator *e);
@@ -1060,20 +1086,22 @@ class ScalarExprEmitter : public StmtVisitor<ScalarExprEmitter, mlir::Value> {
 
     return maybePromoteBoolResult(resOp.getResult(), resTy);
   }
+
+  mlir::Value VisitAtomicExpr(AtomicExpr *e) {
+    return cgf.emitAtomicExpr(e).getValue();
+  }
 };
 
 LValue ScalarExprEmitter::emitCompoundAssignLValue(
     const CompoundAssignOperator *e,
     mlir::Value (ScalarExprEmitter::*func)(const BinOpInfo &),
     mlir::Value &result) {
+  if (e->getComputationResultType()->isAnyComplexType())
+    return cgf.emitScalarCompoundAssignWithComplex(e, result);
+
   QualType lhsTy = e->getLHS()->getType();
   BinOpInfo opInfo;
 
-  if (e->getComputationResultType()->isAnyComplexType()) {
-    cgf.cgm.errorNYI(result.getLoc(), "complex lvalue assign");
-    return LValue();
-  }
-
   // Emit the RHS first.  __block variables need to have the rhs evaluated
   // first, plus this should improve codegen a little.
 
@@ -1877,6 +1905,8 @@ mlir::Value ScalarExprEmitter::VisitCastExpr(CastExpr *ce) {
         cgf.getLoc(subExpr->getSourceRange()), cgf.convertType(destTy),
         Visit(subExpr));
   }
+  case CK_FunctionToPointerDecay:
+    return cgf.emitLValue(subExpr).getPointer();
 
   default:
     cgf.getCIRGenModule().errorNYI(subExpr->getSourceRange(),
@@ -1955,6 +1985,29 @@ mlir::Value CIRGenFunction::emitScalarConversion(mlir::Value src,
       .emitScalarConversion(src, srcTy, dstTy, loc);
 }
 
+mlir::Value CIRGenFunction::emitComplexToScalarConversion(mlir::Value src,
+                                                          QualType srcTy,
+                                                          QualType dstTy,
+                                                          SourceLocation loc) {
+  assert(srcTy->isAnyComplexType() && hasScalarEvaluationKind(dstTy) &&
+         "Invalid complex -> scalar conversion");
+
+  QualType complexElemTy = srcTy->castAs<ComplexType>()->getElementType();
+  if (dstTy->isBooleanType()) {
+    auto kind = complexElemTy->isFloatingType()
+                    ? cir::CastKind::float_complex_to_bool
+                    : cir::CastKind::int_complex_to_bool;
+    return builder.createCast(getLoc(loc), kind, src, convertType(dstTy));
+  }
+
+  auto kind = complexElemTy->isFloatingType()
+                  ? cir::CastKind::float_complex_to_real
+                  : cir::CastKind::int_complex_to_real;
+  mlir::Value real =
+      builder.createCast(getLoc(loc), kind, src, convertType(complexElemTy));
+  return emitScalarConversion(real, complexElemTy, dstTy, loc);
+}
+
 mlir::Value ScalarExprEmitter::VisitUnaryLNot(const UnaryOperator *e) {
   // Perform vector logical not on comparison with zero vector.
   if (e->getType()->isVectorType() &&
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
index eb05c93d52891..706c14ae962a8 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.cpp
@@ -28,8 +28,6 @@ CIRGenFunction::CIRGenFunction(CIRGenModule &cgm, CIRGenBuilderTy &builder,
                                bool suppressNewContext)
     : CIRGenTypeCache(cgm), cgm{cgm}, builder(builder) {
   ehStack.setCGF(this);
-  currentCleanupStackDepth = 0;
-  assert(ehStack.getStackDepth() == 0);
 }
 
 CIRGenFunction::~CIRGenFunction() {}
@@ -217,7 +215,7 @@ void CIRGenFunction::declare(mlir::Value addrVal, const Decl *var, QualType ty,
                              mlir::Location loc, CharUnits alignment,
                              bool isParam) {
   assert(isa<NamedDecl>(var) && "Needs a named decl");
-  assert(!cir::MissingFeatures::cgfSymbolTable());
+  assert(!symbolTable.count(var) && "not supposed to be available just yet");
 
   auto allocaOp = addrVal.getDefiningOp<cir::AllocaOp>();
   assert(allocaOp && "expected cir::AllocaOp");
@@ -226,6 +224,8 @@ void CIRGenFunction::declare(mlir::Value addrVal, const Decl *var, QualType ty,
     allocaOp.setInitAttr(mlir::UnitAttr::get(&getMLIRContext()));
   if (ty->isReferenceType() || ty.isConstQualified())
     allocaOp.setConstantAttr(mlir::UnitAttr::get(&getMLIRContext()));
+
+  symbolTable.insert(var, allocaOp);
 }
 
 void CIRGenFunction::LexicalScope::cleanup() {
@@ -358,7 +358,8 @@ static bool mayDropFunctionReturn(const ASTContext &astContext,
   // destructor or a non-trivially copyable type.
   if (const RecordType *recordType =
           returnType.getCanonicalType()->getAs<RecordType>()) {
-    if (const auto *classDecl = dyn_cast<CXXRecordDecl>(recordType->getDecl()))
+    if (const auto *classDecl = dyn_cast<CXXRecordDecl>(
+            recordType->getOriginalDecl()->getDefinitionOrSelf()))
       return classDecl->hasTrivialDestructor();
   }
   return returnType.isTriviallyCopyableType(astContext);
@@ -409,6 +410,8 @@ void CIRGenFunction::startFunction(GlobalDecl gd, QualType returnType,
   const auto *fd = dyn_cast_or_null<FunctionDecl>(d);
   curFuncDecl = d->getNonClosureContext();
 
+  prologueCleanupDepth = ehStack.stable_begin();
+
   mlir::Block *entryBB = &fn.getBlocks().front();
   builder.setInsertionPointToStart(entryBB);
 
@@ -475,22 +478,22 @@ void CIRGenFunction::finishFunction(SourceLocation endLoc) {
   // important to do this before we enter the return block or return
   // edges will be *really* confused.
   // TODO(cir): Use prologueCleanupDepth here.
-  bool hasCleanups = ehStack.getStackDepth() != currentCleanupStackDepth;
+  bool hasCleanups = ehStack.stable_begin() != prologueCleanupDepth;
   if (hasCleanups) {
     assert(!cir::MissingFeatures::generateDebugInfo());
     // FIXME(cir): should we clearInsertionPoint? breaks many testcases
-    popCleanupBlocks(currentCleanupStackDepth);
+    popCleanupBlocks(prologueCleanupDepth);
   }
 }
 
 mlir::LogicalResult CIRGenFunction::emitFunctionBody(const clang::Stmt *body) {
-  auto result = mlir::LogicalResult::success();
+  // We start with function level scope for variables.
+  SymTableScopeTy varScope(symbolTable);
+
   if (const CompoundStmt *block = dyn_cast<CompoundStmt>(body))
-    emitCompoundStmtWithoutScope(*block);
-  else
-    result = emitStmt(body, /*useCurrentScope=*/true);
+    return emitCompoundStmtWithoutScope(*block);
 
-  return result;
+  return emitStmt(body, /*useCurrentScope=*/true);
 }
 
 static void eraseEmptyAndUnusedBlocks(cir::FuncOp func) {
@@ -530,6 +533,8 @@ cir::FuncOp CIRGenFunction::generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
   FunctionArgList args;
   QualType retTy = buildFunctionArgList(gd, args);
 
+  // Create a scope in the symbol table to hold variable declarations.
+  SymTableScopeTy varScope(symbolTable);
   {
     LexicalScope lexScope(*this, fusedLoc, entryBB);
 
@@ -553,7 +558,6 @@ cir::FuncOp CIRGenFunction::generateCode(clang::GlobalDecl gd, cir::FuncOp fn,
       emitImplicitAssignmentOperatorBody(args);
     } else if (body) {
       if (mlir::failed(emitFunctionBody(body))) {
-        fn.erase();
         return nullptr;
       }
     } else {
@@ -785,9 +789,8 @@ LValue CIRGenFunction::emitLValue(const Expr *e) {
     }
     if (!ty->isAnyComplexType())
       return emitCompoundAssignmentLValue(cast<CompoundAssignOperator>(e));
-    cgm.errorNYI(e->getSourceRange(),
-                 "CompoundAssignOperator with ComplexType");
-    return LValue();
+
+    return emitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(e));
   }
   case Expr::CallExprClass:
   case Expr::CXXMemberCallExprClass:
@@ -828,7 +831,9 @@ void CIRGenFunction::emitNullInitialization(mlir::Location loc, Address destPtr,
   // Ignore empty classes in C++.
   if (getLangOpts().CPlusPlus) {
     if (const RecordType *rt = ty->getAs<RecordType>()) {
-      if (cast<CXXRecordDecl>(rt->getDecl())->isEmpty())
+      if (cast<CXXRecordDecl>(rt->getOriginalDecl())
+              ->getDefinitionOrSelf()
+              ->isEmpty())
         return;
     }
   }
@@ -931,6 +936,23 @@ CIRGenFunction::emitArrayLength(const clang::ArrayType *origArrayType,
   return builder.getConstInt(*currSrcLoc, SizeTy, countFromCLAs);
 }
 
+mlir::Value CIRGenFunction::emitAlignmentAssumption(
+    mlir::Value ptrValue, QualType ty, SourceLocation loc,
+    SourceLocation assumptionLoc, int64_t alignment, mlir::Value offsetValue) {
+  assert(!cir::MissingFeatures::sanitizers());
+  return cir::AssumeAlignedOp::create(builder, getLoc(assumptionLoc), ptrValue,
+                                      alignment, offsetValue);
+}
+
+mlir::Value CIRGenFunction::emitAlignmentAssumption(
+    mlir::Value ptrValue, const Expr *expr, SourceLocation assumptionLoc,
+    int64_t alignment, mlir::Value offsetValue) {
+  QualType ty = expr->getType();
+  SourceLocation loc = expr->getExprLoc();
+  return emitAlignmentAssumption(ptrValue, ty, loc, assumptionLoc, alignment,
+                                 offsetValue);
+}
+
 // TODO(cir): Most of this function can be shared between CIRGen
 // and traditional LLVM codegen
 void CIRGenFunction::emitVariablyModifiedType(QualType type) {
@@ -979,10 +1001,6 @@ void CIRGenFunction::emitVariablyModifiedType(QualType type) {
     case Type::BitInt:
       llvm_unreachable("type class is never variably-modified!");
 
-    case Type::Elaborated:
-      type = cast<clang::ElaboratedType>(ty)->getNamedType();
-      break;
-
     case Type::Adjusted:
       type = cast<clang::AdjustedType>(ty)->getAdjustedType();
       break;
@@ -1058,4 +1076,10 @@ void CIRGenFunction::emitVariablyModifiedType(QualType type) {
   } while (type->isVariablyModifiedType());
 }
 
+Address CIRGenFunction::emitVAListRef(const Expr *e) {
+  if (getContext().getBuiltinVaListType()->isArrayType())
+    return emitPointerWithAlignment(e);
+  return emitLValue(e).getAddress();
+}
+
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 3d92545a4ae3c..064b3c15a310b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -23,6 +23,7 @@
 #include "Address.h"
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/BaseSubobject.h"
 #include "clang/AST/CharUnits.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/Stmt.h"
@@ -30,6 +31,7 @@
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
 #include "clang/CIR/MissingFeatures.h"
 #include "clang/CIR/TypeEvaluationKind.h"
+#include "llvm/ADT/ScopedHashTable.h"
 
 namespace {
 class ScalarExprEmitter;
@@ -102,6 +104,14 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// Sanitizers enabled for this function.
   clang::SanitizerSet sanOpts;
 
+  /// The symbol table maps a variable name to a value in the current scope.
+  /// Entering a function creates a new scope, and the function arguments are
+  /// added to the mapping. When the processing of a function is terminated,
+  /// the scope is destroyed and the mappings created in this scope are
+  /// dropped.
+  using SymTableTy = llvm::ScopedHashTable<const clang::Decl *, mlir::Value>;
+  SymTableTy symbolTable;
+
   /// Whether or not a Microsoft-style asm block has been processed within
   /// this fuction. These can potentially set the return value.
   bool sawAsmBlock = false;
@@ -324,6 +334,9 @@ class CIRGenFunction : public CIRGenTypeCache {
     ~SourceLocRAIIObject() { restore(); }
   };
 
+  using SymTableScopeTy =
+      llvm::ScopedHashTableScope<const clang::Decl *, mlir::Value>;
+
   /// Hold counters for incrementally naming temporaries
   unsigned counterRefTmp = 0;
   unsigned counterAggTmp = 0;
@@ -337,6 +350,15 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   const clang::LangOptions &getLangOpts() const { return cgm.getLangOpts(); }
 
+  /// True if an insertion point is defined. If not, this indicates that the
+  /// current code being emitted is unreachable.
+  /// FIXME(cir): we need to inspect this and perhaps use a cleaner mechanism
+  /// since we don't yet force null insertion point to designate behavior (like
+  /// LLVM's codegen does) and we probably shouldn't.
+  bool haveInsertPoint() const {
+    return builder.getInsertionBlock() != nullptr;
+  }
+
   // Wrapper for function prototype sources. Wraps either a FunctionProtoType or
   // an ObjCMethodDecl.
   struct PrototypeWrapper {
@@ -430,7 +452,8 @@ class CIRGenFunction : public CIRGenTypeCache {
     }
   };
 
-  ConstantEmission tryEmitAsConstant(DeclRefExpr *refExpr);
+  ConstantEmission tryEmitAsConstant(const DeclRefExpr *refExpr);
+  ConstantEmission tryEmitAsConstant(const MemberExpr *me);
 
   struct AutoVarEmission {
     const clang::VarDecl *Variable;
@@ -448,6 +471,10 @@ class CIRGenFunction : public CIRGenTypeCache {
     /// escaping block.
     bool IsEscapingByRef = false;
 
+    /// True if the variable was emitted as an offload recipe, and thus doesn't
+    /// have the same sort of alloca initialization.
+    bool EmittedAsOffload = false;
+
     mlir::Value NRVOFlag{};
 
     struct Invalid {};
@@ -460,11 +487,18 @@ class CIRGenFunction : public CIRGenTypeCache {
 
     bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
 
+    bool wasEmittedAsOffloadClause() const { return EmittedAsOffload; }
+
     /// Returns the raw, allocated address, which is not necessarily
     /// the address of the object itself. It is casted to default
     /// address space for address space agnostic languages.
     Address getAllocatedAddress() const { return Addr; }
 
+    // Changes the stored address for the emission.  This function should only
+    // be used in extreme cases, and isn't required to model normal AST
+    // initialization/variables.
+    void setAllocatedAddress(Address A) { Addr = A; }
+
     /// Returns the address of the object within this declaration.
     /// Note that this does not chase the forwarding pointer for
     /// __block decls.
@@ -489,9 +523,42 @@ class CIRGenFunction : public CIRGenTypeCache {
   void setAddrOfLocalVar(const clang::VarDecl *vd, Address addr) {
     assert(!localDeclMap.count(vd) && "Decl already exists in LocalDeclMap!");
     localDeclMap.insert({vd, addr});
-    // TODO: Add symbol table support
+
+    // Add to the symbol table if not there already.
+    if (symbolTable.count(vd))
+      return;
+    symbolTable.insert(vd, addr.getPointer());
   }
 
+  // A class to allow reverting changes to a var-decl's registration to the
+  // localDeclMap. This is used in cases where things are being inserted into
+  // the variable list but don't follow normal lookup/search rules, like in
+  // OpenACC recipe generation.
+  class DeclMapRevertingRAII {
+    CIRGenFunction &cgf;
+    const VarDecl *vd;
+    bool shouldDelete = false;
+    Address oldAddr = Address::invalid();
+
+  public:
+    DeclMapRevertingRAII(CIRGenFunction &cgf, const VarDecl *vd)
+        : cgf(cgf), vd(vd) {
+      auto mapItr = cgf.localDeclMap.find(vd);
+
+      if (mapItr != cgf.localDeclMap.end())
+        oldAddr = mapItr->second;
+      else
+        shouldDelete = true;
+    }
+
+    ~DeclMapRevertingRAII() {
+      if (shouldDelete)
+        cgf.localDeclMap.erase(vd);
+      else
+        cgf.localDeclMap.insert_or_assign(vd, oldAddr);
+    }
+  };
+
   bool shouldNullCheckClassCastValue(const CastExpr *ce);
 
   RValue convertTempToRValue(Address addr, clang::QualType type,
@@ -500,6 +567,30 @@ class CIRGenFunction : public CIRGenTypeCache {
   static bool
   isConstructorDelegationValid(const clang::CXXConstructorDecl *ctor);
 
+  struct VPtr {
+    clang::BaseSubobject base;
+    const clang::CXXRecordDecl *nearestVBase;
+    clang::CharUnits offsetFromNearestVBase;
+    const clang::CXXRecordDecl *vtableClass;
+  };
+
+  using VPtrsVector = llvm::SmallVector<VPtr, 4>;
+  VPtrsVector getVTablePointers(const clang::CXXRecordDecl *vtableClass);
+  void getVTablePointers(clang::BaseSubobject base,
+                         const clang::CXXRecordDecl *nearestVBase,
+                         clang::CharUnits offsetFromNearestVBase,
+                         bool baseIsNonVirtualPrimaryBase,
+                         const clang::CXXRecordDecl *vtableClass,
+                         VPtrsVector &vptrs);
+  /// Return the Value of the vtable pointer member pointed to by thisAddr.
+  mlir::Value getVTablePtr(mlir::Location loc, Address thisAddr,
+                           const clang::CXXRecordDecl *vtableClass);
+
+  /// Returns whether we should perform a type checked load when loading a
+  /// virtual function for virtual calls to members of RD. This is generally
+  /// true when both vcall CFI and whole-program-vtables are enabled.
+  bool shouldEmitVTableTypeCheckedLoad(const CXXRecordDecl *rd);
+
   /// A scope within which we are constructing the fields of an object which
   /// might use a CXXDefaultInitExpr. This stashes away a 'this' value to use if
   /// we need to evaluate the CXXDefaultInitExpr within the evaluation.
@@ -548,6 +639,10 @@ class CIRGenFunction : public CIRGenTypeCache {
     return LValue::makeAddr(addr, ty, baseInfo);
   }
 
+  void initializeVTablePointers(mlir::Location loc,
+                                const clang::CXXRecordDecl *rd);
+  void initializeVTablePointer(mlir::Location loc, const VPtr &vptr);
+
   /// Return the address of a local variable.
   Address getAddrOfLocalVar(const clang::VarDecl *vd) {
     auto it = localDeclMap.find(vd);
@@ -601,9 +696,13 @@ class CIRGenFunction : public CIRGenTypeCache {
                      FunctionArgList args, clang::SourceLocation loc,
                      clang::SourceLocation startLoc);
 
+  /// The cleanup depth enclosing all the cleanups associated with the
+  /// parameters.
+  EHScopeStack::stable_iterator prologueCleanupDepth;
+
   /// Takes the old cleanup stack size and emits the cleanup blocks
   /// that have been added.
-  void popCleanupBlocks(size_t oldCleanupStackDepth);
+  void popCleanupBlocks(EHScopeStack::stable_iterator oldCleanupStackDepth);
   void popCleanupBlock();
 
   /// Push a cleanup to be run at the end of the current full-expression.  Safe
@@ -622,7 +721,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// Enters a new scope for capturing cleanups, all of which
   /// will be executed once the scope is exited.
   class RunCleanupsScope {
-    size_t cleanupStackDepth, oldCleanupStackDepth;
+    EHScopeStack::stable_iterator cleanupStackDepth, oldCleanupStackDepth;
 
   protected:
     bool performCleanup;
@@ -638,7 +737,7 @@ class CIRGenFunction : public CIRGenTypeCache {
     /// Enter a new cleanup scope.
     explicit RunCleanupsScope(CIRGenFunction &cgf)
         : performCleanup(true), cgf(cgf) {
-      cleanupStackDepth = cgf.ehStack.getStackDepth();
+      cleanupStackDepth = cgf.ehStack.stable_begin();
       oldCleanupStackDepth = cgf.currentCleanupStackDepth;
       cgf.currentCleanupStackDepth = cleanupStackDepth;
     }
@@ -663,7 +762,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   };
 
   // Cleanup stack depth of the RunCleanupsScope that was pushed most recently.
-  size_t currentCleanupStackDepth;
+  EHScopeStack::stable_iterator currentCleanupStackDepth = ehStack.stable_end();
 
 public:
   /// Represents a scope, including function bodies, compound statements, and
@@ -825,6 +924,18 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// ----------------------
   /// CIR emit functions
   /// ----------------------
+public:
+  mlir::Value emitAlignmentAssumption(mlir::Value ptrValue, QualType ty,
+                                      SourceLocation loc,
+                                      SourceLocation assumptionLoc,
+                                      int64_t alignment,
+                                      mlir::Value offsetValue = nullptr);
+
+  mlir::Value emitAlignmentAssumption(mlir::Value ptrValue, const Expr *expr,
+                                      SourceLocation assumptionLoc,
+                                      int64_t alignment,
+                                      mlir::Value offsetValue = nullptr);
+
 private:
   void emitAndUpdateRetAlloca(clang::QualType type, mlir::Location loc,
                               clang::CharUnits alignment);
@@ -875,7 +986,13 @@ class CIRGenFunction : public CIRGenTypeCache {
                               QualType &baseType, Address &addr);
   LValue emitArraySubscriptExpr(const clang::ArraySubscriptExpr *e);
 
-  Address emitArrayToPointerDecay(const Expr *array);
+  Address emitArrayToPointerDecay(const Expr *e,
+                                  LValueBaseInfo *baseInfo = nullptr);
+
+  mlir::LogicalResult emitAsmStmt(const clang::AsmStmt &s);
+
+  RValue emitAtomicExpr(AtomicExpr *e);
+  void emitAtomicInit(Expr *init, LValue dest);
 
   AutoVarEmission emitAutoVarAlloca(const clang::VarDecl &d,
                                     mlir::OpBuilder::InsertPoint ip = {});
@@ -886,6 +1003,11 @@ class CIRGenFunction : public CIRGenTypeCache {
   void emitAutoVarDecl(const clang::VarDecl &d);
 
   void emitAutoVarCleanups(const AutoVarEmission &emission);
+  /// Emit the initializer for an allocated variable.  If this call is not
+  /// associated with the call to emitAutoVarAlloca (as the address of the
+  /// emission is not directly an alloca), the allocatedSeparately parameter can
+  /// be used to suppress the assertions.  However, this should only be used in
+  /// extreme cases, as it doesn't properly reflect the language/AST.
   void emitAutoVarInit(const AutoVarEmission &emission);
   void emitAutoVarTypeCleanup(const AutoVarEmission &emission,
                               clang::QualType::DestructionKind dtorKind);
@@ -944,6 +1066,11 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// sanitizer is enabled, a runtime check is also emitted.
   mlir::Value emitCheckedArgForAssume(const Expr *e);
 
+  /// Emit a conversion from the specified complex type to the specified
+  /// destination type, where the destination type is an LLVM scalar type.
+  mlir::Value emitComplexToScalarConversion(mlir::Value src, QualType srcTy,
+                                            QualType dstTy, SourceLocation loc);
+
   LValue emitCompoundAssignmentLValue(const clang::CompoundAssignOperator *e);
   LValue emitCompoundLiteralLValue(const CompoundLiteralExpr *e);
 
@@ -1002,7 +1129,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   RValue emitCXXMemberOrOperatorMemberCallExpr(
       const clang::CallExpr *ce, const clang::CXXMethodDecl *md,
       ReturnValueSlot returnValue, bool hasQualifier,
-      clang::NestedNameSpecifier *qualifier, bool isArrow,
+      clang::NestedNameSpecifier qualifier, bool isArrow,
       const clang::Expr *base);
 
   mlir::Value emitCXXNewExpr(const CXXNewExpr *e);
@@ -1011,6 +1138,8 @@ class CIRGenFunction : public CIRGenTypeCache {
                                        const CXXMethodDecl *md,
                                        ReturnValueSlot returnValue);
 
+  RValue emitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *expr);
+
   void emitCtorPrologue(const clang::CXXConstructorDecl *ctor,
                         clang::CXXCtorType ctorType, FunctionArgList &args);
 
@@ -1038,6 +1167,8 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::LogicalResult emitFunctionBody(const clang::Stmt *body);
 
+  mlir::LogicalResult emitGotoStmt(const clang::GotoStmt &s);
+
   void emitImplicitAssignmentOperatorBody(FunctionArgList &args);
 
   void emitInitializerForField(clang::FieldDecl *field, LValue lhs,
@@ -1047,6 +1178,8 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::Value emitPromotedScalarExpr(const Expr *e, QualType promotionType);
 
+  mlir::Value emitPromotedValue(mlir::Value result, QualType promotionType);
+
   /// Emit the computation of the specified expression of scalar type.
   mlir::Value emitScalarExpr(const clang::Expr *e);
 
@@ -1076,10 +1209,18 @@ class CIRGenFunction : public CIRGenTypeCache {
                                        cir::UnaryOpKind op, bool isPre);
 
   LValue emitComplexAssignmentLValue(const BinaryOperator *e);
+  LValue emitComplexCompoundAssignmentLValue(const CompoundAssignOperator *e);
+  LValue emitScalarCompoundAssignWithComplex(const CompoundAssignOperator *e,
+                                             mlir::Value &result);
 
-  void emitCompoundStmt(const clang::CompoundStmt &s);
+  mlir::LogicalResult
+  emitCompoundStmt(const clang::CompoundStmt &s, Address *lastValue = nullptr,
+                   AggValueSlot slot = AggValueSlot::ignored());
 
-  void emitCompoundStmtWithoutScope(const clang::CompoundStmt &s);
+  mlir::LogicalResult
+  emitCompoundStmtWithoutScope(const clang::CompoundStmt &s,
+                               Address *lastValue = nullptr,
+                               AggValueSlot slot = AggValueSlot::ignored());
 
   void emitDecl(const clang::Decl &d, bool evaluateConditionDecl = false);
   mlir::LogicalResult emitDeclStmt(const clang::DeclStmt &s);
@@ -1117,6 +1258,9 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   mlir::Value emitOpOnBoolExpr(mlir::Location loc, const clang::Expr *cond);
 
+  mlir::LogicalResult emitLabel(const clang::LabelDecl &d);
+  mlir::LogicalResult emitLabelStmt(const clang::LabelStmt &s);
+
   mlir::LogicalResult emitIfStmt(const clang::IfStmt &s);
 
   /// Emit code to compute the specified expression,
@@ -1173,7 +1317,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   /// reasonable to just ignore the returned alignment when it isn't from an
   /// explicit source.
   Address emitPointerWithAlignment(const clang::Expr *expr,
-                                   LValueBaseInfo *baseInfo);
+                                   LValueBaseInfo *baseInfo = nullptr);
 
   /// Emits a reference binding to the passed in expression.
   RValue emitReferenceBindingToExpr(const Expr *e);
@@ -1235,6 +1379,8 @@ class CIRGenFunction : public CIRGenTypeCache {
 
   LValue emitUnaryOpLValue(const clang::UnaryOperator *e);
 
+  mlir::Value emitUnPromotedValue(mlir::Value result, QualType unPromotionType);
+
   /// Emit a reached-unreachable diagnostic if \p loc is valid and runtime
   /// checking is enabled. Otherwise, just emit an unreachable instruction.
   /// \p createNewBlock indicates whether to create a new block for the IR
@@ -1302,7 +1448,7 @@ class CIRGenFunction : public CIRGenTypeCache {
       mlir::OpBuilder::InsertionGuard guard(builder);
       builder.restoreInsertionPoint(outermostConditional->getInsertPoint());
       builder.createStore(
-          value.getLoc(), value, addr,
+          value.getLoc(), value, addr, /*isVolatile=*/false,
           mlir::IntegerAttr::get(
               mlir::IntegerType::get(value.getContext(), 64),
               (uint64_t)addr.getAlignment().getAsAlign().value()));
@@ -1313,6 +1459,27 @@ class CIRGenFunction : public CIRGenTypeCache {
   // we know if a temporary should be destroyed conditionally.
   ConditionalEvaluation *outermostConditional = nullptr;
 
+  /// An RAII object to record that we're evaluating a statement
+  /// expression.
+  class StmtExprEvaluation {
+    CIRGenFunction &cgf;
+
+    /// We have to save the outermost conditional: cleanups in a
+    /// statement expression aren't conditional just because the
+    /// StmtExpr is.
+    ConditionalEvaluation *savedOutermostConditional;
+
+  public:
+    StmtExprEvaluation(CIRGenFunction &cgf)
+        : cgf(cgf), savedOutermostConditional(cgf.outermostConditional) {
+      cgf.outermostConditional = nullptr;
+    }
+
+    ~StmtExprEvaluation() {
+      cgf.outermostConditional = savedOutermostConditional;
+    }
+  };
+
   template <typename FuncTy>
   ConditionalInfo emitConditionalBlocks(const AbstractConditionalOperator *e,
                                         const FuncTy &branchGenFunc);
@@ -1321,6 +1488,35 @@ class CIRGenFunction : public CIRGenTypeCache {
                                     const clang::Stmt *thenS,
                                     const clang::Stmt *elseS);
 
+  /// Build a "reference" to a va_list; this is either the address or the value
+  /// of the expression, depending on how va_list is defined.
+  Address emitVAListRef(const Expr *e);
+
+  /// Emits the start of a CIR variable-argument operation (`cir.va_start`)
+  ///
+  /// \param vaList A reference to the \c va_list as emitted by either
+  /// \c emitVAListRef or \c emitMSVAListRef.
+  ///
+  /// \param count The number of arguments in \c vaList
+  void emitVAStart(mlir::Value vaList, mlir::Value count);
+
+  /// Emits the end of a CIR variable-argument operation (`cir.va_start`)
+  ///
+  /// \param vaList A reference to the \c va_list as emitted by either
+  /// \c emitVAListRef or \c emitMSVAListRef.
+  void emitVAEnd(mlir::Value vaList);
+
+  /// Generate code to get an argument from the passed in pointer
+  /// and update it accordingly.
+  ///
+  /// \param ve The \c VAArgExpr for which to generate code.
+  ///
+  /// \param vaListAddr Receives a reference to the \c va_list as emitted by
+  /// either \c emitVAListRef or \c emitMSVAListRef.
+  ///
+  /// \returns SSA value with the argument.
+  mlir::Value emitVAArg(VAArgExpr *ve);
+
   /// ----------------------
   /// CIR build helpers
   /// -----------------
diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
index e5e4c68a1bfa7..aaf7dc767d888 100644
--- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp
@@ -22,6 +22,7 @@
 
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/GlobalDecl.h"
+#include "clang/AST/VTableBuilder.h"
 #include "clang/CIR/MissingFeatures.h"
 #include "llvm/Support/ErrorHandling.h"
 
@@ -31,6 +32,10 @@ using namespace clang::CIRGen;
 namespace {
 
 class CIRGenItaniumCXXABI : public CIRGenCXXABI {
+protected:
+  /// All the vtables which have been defined.
+  llvm::DenseMap<const CXXRecordDecl *, cir::GlobalOp> vtables;
+
 public:
   CIRGenItaniumCXXABI(CIRGenModule &cgm) : CIRGenCXXABI(cgm) {
     assert(!cir::MissingFeatures::cxxabiUseARMMethodPtrABI());
@@ -58,6 +63,30 @@ class CIRGenItaniumCXXABI : public CIRGenCXXABI {
     // emitted with external linkage or as linkonce if they are inline and used.
     return false;
   }
+
+  bool isVirtualOffsetNeededForVTableField(CIRGenFunction &cgf,
+                                           CIRGenFunction::VPtr vptr) override;
+
+  cir::GlobalOp getAddrOfVTable(const CXXRecordDecl *rd,
+                                CharUnits vptrOffset) override;
+  CIRGenCallee getVirtualFunctionPointer(CIRGenFunction &cgf,
+                                         clang::GlobalDecl gd, Address thisAddr,
+                                         mlir::Type ty,
+                                         SourceLocation loc) override;
+
+  mlir::Value getVTableAddressPoint(BaseSubobject base,
+                                    const CXXRecordDecl *vtableClass) override;
+
+  mlir::Value getVTableAddressPointInStructor(
+      CIRGenFunction &cgf, const clang::CXXRecordDecl *vtableClass,
+      clang::BaseSubobject base,
+      const clang::CXXRecordDecl *nearestVBase) override;
+  void emitVTableDefinitions(CIRGenVTables &cgvt,
+                             const CXXRecordDecl *rd) override;
+
+  bool doStructorsInitializeVPtrs(const CXXRecordDecl *vtableClass) override {
+    return true;
+  }
 };
 
 } // namespace
@@ -243,6 +272,67 @@ bool CIRGenItaniumCXXABI::needsVTTParameter(GlobalDecl gd) {
   return false;
 }
 
+void CIRGenItaniumCXXABI::emitVTableDefinitions(CIRGenVTables &cgvt,
+                                                const CXXRecordDecl *rd) {
+  cir::GlobalOp vtable = getAddrOfVTable(rd, CharUnits());
+  if (vtable.hasInitializer())
+    return;
+
+  ItaniumVTableContext &vtContext = cgm.getItaniumVTableContext();
+  const VTableLayout &vtLayout = vtContext.getVTableLayout(rd);
+  cir::GlobalLinkageKind linkage = cgm.getVTableLinkage(rd);
+  mlir::Attribute rtti =
+      cgm.getAddrOfRTTIDescriptor(cgm.getLoc(rd->getBeginLoc()),
+                                  cgm.getASTContext().getCanonicalTagType(rd));
+
+  // Classic codegen uses ConstantInitBuilder here, which is a very general
+  // and feature-rich class to generate initializers for global values.
+  // For now, this is using a simpler approach to create the initializer in CIR.
+  cgvt.createVTableInitializer(vtable, vtLayout, rtti,
+                               cir::isLocalLinkage(linkage));
+
+  // Set the correct linkage.
+  vtable.setLinkage(linkage);
+
+  if (cgm.supportsCOMDAT() && cir::isWeakForLinker(linkage))
+    vtable.setComdat(true);
+
+  // Set the right visibility.
+  cgm.setGVProperties(vtable, rd);
+
+  // If this is the magic class __cxxabiv1::__fundamental_type_info,
+  // we will emit the typeinfo for the fundamental types. This is the
+  // same behaviour as GCC.
+  const DeclContext *DC = rd->getDeclContext();
+  if (rd->getIdentifier() &&
+      rd->getIdentifier()->isStr("__fundamental_type_info") &&
+      isa<NamespaceDecl>(DC) && cast<NamespaceDecl>(DC)->getIdentifier() &&
+      cast<NamespaceDecl>(DC)->getIdentifier()->isStr("__cxxabiv1") &&
+      DC->getParent()->isTranslationUnit()) {
+    cgm.errorNYI(rd->getSourceRange(),
+                 "emitVTableDefinitions: __fundamental_type_info");
+  }
+
+  auto vtableAsGlobalValue = dyn_cast<cir::CIRGlobalValueInterface>(*vtable);
+  assert(vtableAsGlobalValue && "VTable must support CIRGlobalValueInterface");
+  // Always emit type metadata on non-available_externally definitions, and on
+  // available_externally definitions if we are performing whole program
+  // devirtualization. For WPD we need the type metadata on all vtable
+  // definitions to ensure we associate derived classes with base classes
+  // defined in headers but with a strong definition only in a shared
+  // library.
+  assert(!cir::MissingFeatures::vtableEmitMetadata());
+  if (cgm.getCodeGenOpts().WholeProgramVTables) {
+    cgm.errorNYI(rd->getSourceRange(),
+                 "emitVTableDefinitions: WholeProgramVTables");
+  }
+
+  assert(!cir::MissingFeatures::vtableRelativeLayout());
+  if (vtContext.isRelativeLayout()) {
+    cgm.errorNYI(rd->getSourceRange(), "vtableRelativeLayout");
+  }
+}
+
 void CIRGenItaniumCXXABI::emitDestructorCall(
     CIRGenFunction &cgf, const CXXDestructorDecl *dd, CXXDtorType type,
     bool forVirtualBase, bool delegating, Address thisAddr, QualType thisTy) {
@@ -278,3 +368,135 @@ CIRGenCXXABI *clang::CIRGen::CreateCIRGenItaniumCXXABI(CIRGenModule &cgm) {
     llvm_unreachable("bad or NYI ABI kind");
   }
 }
+
+cir::GlobalOp CIRGenItaniumCXXABI::getAddrOfVTable(const CXXRecordDecl *rd,
+                                                   CharUnits vptrOffset) {
+  assert(vptrOffset.isZero() && "Itanium ABI only supports zero vptr offsets");
+  cir::GlobalOp &vtable = vtables[rd];
+  if (vtable)
+    return vtable;
+
+  // Queue up this vtable for possible deferred emission.
+  assert(!cir::MissingFeatures::deferredVtables());
+
+  SmallString<256> name;
+  llvm::raw_svector_ostream out(name);
+  getMangleContext().mangleCXXVTable(rd, out);
+
+  const VTableLayout &vtLayout =
+      cgm.getItaniumVTableContext().getVTableLayout(rd);
+  mlir::Type vtableType = cgm.getVTables().getVTableType(vtLayout);
+
+  // Use pointer alignment for the vtable. Otherwise we would align them based
+  // on the size of the initializer which doesn't make sense as only single
+  // values are read.
+  unsigned ptrAlign = cgm.getItaniumVTableContext().isRelativeLayout()
+                          ? 32
+                          : cgm.getTarget().getPointerAlign(LangAS::Default);
+
+  vtable = cgm.createOrReplaceCXXRuntimeVariable(
+      cgm.getLoc(rd->getSourceRange()), name, vtableType,
+      cir::GlobalLinkageKind::ExternalLinkage,
+      cgm.getASTContext().toCharUnitsFromBits(ptrAlign));
+  // LLVM codegen handles unnamedAddr
+  assert(!cir::MissingFeatures::opGlobalUnnamedAddr());
+
+  // In MS C++ if you have a class with virtual functions in which you are using
+  // selective member import/export, then all virtual functions must be exported
+  // unless they are inline, otherwise a link error will result. To match this
+  // behavior, for such classes, we dllimport the vtable if it is defined
+  // externally and all the non-inline virtual methods are marked dllimport, and
+  // we dllexport the vtable if it is defined in this TU and all the non-inline
+  // virtual methods are marked dllexport.
+  if (cgm.getTarget().hasPS4DLLImportExport())
+    cgm.errorNYI(rd->getSourceRange(),
+                 "getAddrOfVTable: PS4 DLL import/export");
+
+  cgm.setGVProperties(vtable, rd);
+  return vtable;
+}
+
+CIRGenCallee CIRGenItaniumCXXABI::getVirtualFunctionPointer(
+    CIRGenFunction &cgf, clang::GlobalDecl gd, Address thisAddr, mlir::Type ty,
+    SourceLocation srcLoc) {
+  CIRGenBuilderTy &builder = cgm.getBuilder();
+  mlir::Location loc = cgf.getLoc(srcLoc);
+  cir::PointerType tyPtr = builder.getPointerTo(ty);
+  auto *methodDecl = cast<CXXMethodDecl>(gd.getDecl());
+  mlir::Value vtable = cgf.getVTablePtr(loc, thisAddr, methodDecl->getParent());
+
+  uint64_t vtableIndex = cgm.getItaniumVTableContext().getMethodVTableIndex(gd);
+  mlir::Value vfunc{};
+  if (cgf.shouldEmitVTableTypeCheckedLoad(methodDecl->getParent())) {
+    cgm.errorNYI(loc, "getVirtualFunctionPointer: emitVTableTypeCheckedLoad");
+  } else {
+    assert(!cir::MissingFeatures::emitTypeMetadataCodeForVCall());
+
+    mlir::Value vfuncLoad;
+    if (cgm.getItaniumVTableContext().isRelativeLayout()) {
+      assert(!cir::MissingFeatures::vtableRelativeLayout());
+      cgm.errorNYI(loc, "getVirtualFunctionPointer: isRelativeLayout");
+    } else {
+      auto vtableSlotPtr = cir::VTableGetVirtualFnAddrOp::create(
+          builder, loc, builder.getPointerTo(tyPtr), vtable, vtableIndex);
+      vfuncLoad = builder.createAlignedLoad(
+          loc, vtableSlotPtr, cgf.getPointerAlign().getQuantity());
+    }
+
+    // Add !invariant.load md to virtual function load to indicate that
+    // function didn't change inside vtable.
+    // It's safe to add it without -fstrict-vtable-pointers, but it would not
+    // help in devirtualization because it will only matter if we will have 2
+    // the same virtual function loads from the same vtable load, which won't
+    // happen without enabled devirtualization with -fstrict-vtable-pointers.
+    if (cgm.getCodeGenOpts().OptimizationLevel > 0 &&
+        cgm.getCodeGenOpts().StrictVTablePointers) {
+      cgm.errorNYI(loc, "getVirtualFunctionPointer: strictVTablePointers");
+    }
+    vfunc = vfuncLoad;
+  }
+
+  CIRGenCallee callee(gd, vfunc.getDefiningOp());
+  return callee;
+}
+
+mlir::Value
+CIRGenItaniumCXXABI::getVTableAddressPoint(BaseSubobject base,
+                                           const CXXRecordDecl *vtableClass) {
+  cir::GlobalOp vtable = getAddrOfVTable(vtableClass, CharUnits());
+
+  // Find the appropriate vtable within the vtable group, and the address point
+  // within that vtable.
+  VTableLayout::AddressPointLocation addressPoint =
+      cgm.getItaniumVTableContext()
+          .getVTableLayout(vtableClass)
+          .getAddressPoint(base);
+
+  mlir::OpBuilder &builder = cgm.getBuilder();
+  auto vtablePtrTy = cir::VPtrType::get(builder.getContext());
+
+  return builder.create<cir::VTableAddrPointOp>(
+      cgm.getLoc(vtableClass->getSourceRange()), vtablePtrTy,
+      mlir::FlatSymbolRefAttr::get(vtable.getSymNameAttr()),
+      cir::AddressPointAttr::get(cgm.getBuilder().getContext(),
+                                 addressPoint.VTableIndex,
+                                 addressPoint.AddressPointIndex));
+}
+
+mlir::Value CIRGenItaniumCXXABI::getVTableAddressPointInStructor(
+    CIRGenFunction &cgf, const clang::CXXRecordDecl *vtableClass,
+    clang::BaseSubobject base, const clang::CXXRecordDecl *nearestVBase) {
+
+  if (base.getBase()->getNumVBases()) {
+    cgm.errorNYI(cgf.curFuncDecl->getLocation(),
+                 "getVTableAddressPointInStructor: virtual base");
+  }
+  return getVTableAddressPoint(base, vtableClass);
+}
+
+bool CIRGenItaniumCXXABI::isVirtualOffsetNeededForVTableField(
+    CIRGenFunction &cgf, CIRGenFunction::VPtr vptr) {
+  if (vptr.nearestVBase == nullptr)
+    return false;
+  return needsVTTParameter(cgf.curGD);
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.cpp b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
index b143682e435d2..43554519ebd4d 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.cpp
@@ -64,7 +64,7 @@ CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
       langOpts(astContext.getLangOpts()), codeGenOpts(cgo),
       theModule{mlir::ModuleOp::create(mlir::UnknownLoc::get(&mlirContext))},
       diags(diags), target(astContext.getTargetInfo()),
-      abi(createCXXABI(*this)), genTypes(*this) {
+      abi(createCXXABI(*this)), genTypes(*this), vtables(*this) {
 
   // Initialize cached types
   VoidTy = cir::VoidType::get(&getMLIRContext());
@@ -75,6 +75,7 @@ CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
   SInt64Ty = cir::IntType::get(&getMLIRContext(), 64, /*isSigned=*/true);
   SInt128Ty = cir::IntType::get(&getMLIRContext(), 128, /*isSigned=*/true);
   UInt8Ty = cir::IntType::get(&getMLIRContext(), 8, /*isSigned=*/false);
+  UInt8PtrTy = cir::PointerType::get(UInt8Ty);
   UInt16Ty = cir::IntType::get(&getMLIRContext(), 16, /*isSigned=*/false);
   UInt32Ty = cir::IntType::get(&getMLIRContext(), 32, /*isSigned=*/false);
   UInt64Ty = cir::IntType::get(&getMLIRContext(), 64, /*isSigned=*/false);
@@ -102,6 +103,11 @@ CIRGenModule::CIRGenModule(mlir::MLIRContext &mlirContext,
   PtrDiffTy =
       cir::IntType::get(&getMLIRContext(), sizeTypeSize, /*isSigned=*/true);
 
+  std::optional<cir::SourceLanguage> sourceLanguage = getCIRSourceLanguage();
+  if (sourceLanguage)
+    theModule->setAttr(
+        cir::CIRDialect::getSourceLanguageAttrName(),
+        cir::SourceLanguageAttr::get(&mlirContext, *sourceLanguage));
   theModule->setAttr(cir::CIRDialect::getTripleAttrName(),
                      builder.getStringAttr(getTriple().str()));
 
@@ -437,13 +443,27 @@ void CIRGenModule::emitGlobalFunctionDefinition(clang::GlobalDecl gd,
     errorNYI(funcDecl->getSourceRange(), "deferredAnnotations");
 }
 
+void CIRGenModule::handleCXXStaticMemberVarInstantiation(VarDecl *vd) {
+  VarDecl::DefinitionKind dk = vd->isThisDeclarationADefinition();
+  if (dk == VarDecl::Definition && vd->hasAttr<DLLImportAttr>())
+    return;
+
+  TemplateSpecializationKind tsk = vd->getTemplateSpecializationKind();
+  // If we have a definition, this might be a deferred decl. If the
+  // instantiation is explicit, make sure we emit it at the end.
+  if (vd->getDefinition() && tsk == TSK_ExplicitInstantiationDefinition)
+    getAddrOfGlobalVar(vd);
+
+  emitTopLevelDecl(vd);
+}
+
 mlir::Operation *CIRGenModule::getGlobalValue(StringRef name) {
   return mlir::SymbolTable::lookupSymbolIn(theModule, name);
 }
 
 cir::GlobalOp CIRGenModule::createGlobalOp(CIRGenModule &cgm,
                                            mlir::Location loc, StringRef name,
-                                           mlir::Type t,
+                                           mlir::Type t, bool isConstant,
                                            mlir::Operation *insertPoint) {
   cir::GlobalOp g;
   CIRGenBuilderTy &builder = cgm.getBuilder();
@@ -464,7 +484,7 @@ cir::GlobalOp CIRGenModule::createGlobalOp(CIRGenModule &cgm,
         builder.setInsertionPointToStart(cgm.getModule().getBody());
     }
 
-    g = builder.create<cir::GlobalOp>(loc, name, t);
+    g = builder.create<cir::GlobalOp>(loc, name, t, isConstant);
     if (!insertPoint)
       cgm.lastGlobalOp = g;
 
@@ -495,6 +515,24 @@ void CIRGenModule::setNonAliasAttributes(GlobalDecl gd, mlir::Operation *op) {
   assert(!cir::MissingFeatures::setTargetAttributes());
 }
 
+std::optional<cir::SourceLanguage> CIRGenModule::getCIRSourceLanguage() const {
+  using ClangStd = clang::LangStandard;
+  using CIRLang = cir::SourceLanguage;
+  auto opts = getLangOpts();
+
+  if (opts.CPlusPlus)
+    return CIRLang::CXX;
+  if (opts.C99 || opts.C11 || opts.C17 || opts.C23 || opts.C2y ||
+      opts.LangStd == ClangStd::lang_c89 ||
+      opts.LangStd == ClangStd::lang_gnu89)
+    return CIRLang::C;
+
+  // TODO(cir): support remaining source languages.
+  assert(!cir::MissingFeatures::sourceLanguageCases());
+  errorNYI("CIR does not yet support the given source language");
+  return std::nullopt;
+}
+
 static void setLinkageForGV(cir::GlobalOp &gv, const NamedDecl *nd) {
   // Set linkage and visibility in case we never see a definition.
   LinkageInfo lv = nd->getLinkageAndVisibility();
@@ -566,7 +604,7 @@ CIRGenModule::getOrCreateCIRGlobal(StringRef mangledName, mlir::Type ty,
   // mlir::SymbolTable::Visibility::Public is the default, no need to explicitly
   // mark it as such.
   cir::GlobalOp gv =
-      CIRGenModule::createGlobalOp(*this, loc, mangledName, ty,
+      CIRGenModule::createGlobalOp(*this, loc, mangledName, ty, false,
                                    /*insertPoint=*/entry.getOperation());
 
   // This is the first use or definition of a mangled name.  If there is a
@@ -654,6 +692,16 @@ mlir::Value CIRGenModule::getAddrOfGlobalVar(const VarDecl *d, mlir::Type ty,
                                           g.getSymName());
 }
 
+cir::GlobalViewAttr CIRGenModule::getAddrOfGlobalVarAttr(const VarDecl *d) {
+  assert(d->hasGlobalStorage() && "Not a global variable");
+  mlir::Type ty = getTypes().convertTypeForMem(d->getType());
+
+  cir::GlobalOp globalOp = getOrCreateCIRGlobal(d, ty, NotForDefinition);
+  assert(!cir::MissingFeatures::addressSpace());
+  cir::PointerType ptrTy = builder.getPointerTo(globalOp.getSymType());
+  return builder.getGlobalViewAttr(ptrTy, globalOp);
+}
+
 void CIRGenModule::emitGlobalVarDefinition(const clang::VarDecl *vd,
                                            bool isTentative) {
   if (getLangOpts().OpenCL || getLangOpts().OpenMPIsTargetDevice) {
@@ -800,7 +848,7 @@ void CIRGenModule::emitGlobalDefinition(clang::GlobalDecl gd,
         emitGlobalFunctionDefinition(gd, op);
 
       if (method->isVirtual())
-        errorNYI(method->getSourceRange(), "virtual member function");
+        getVTables().emitThunks(gd);
 
       return;
     }
@@ -946,6 +994,39 @@ void CIRGenModule::applyReplacements() {
   }
 }
 
+cir::GlobalOp CIRGenModule::createOrReplaceCXXRuntimeVariable(
+    mlir::Location loc, StringRef name, mlir::Type ty,
+    cir::GlobalLinkageKind linkage, clang::CharUnits alignment) {
+  auto gv = mlir::dyn_cast_or_null<cir::GlobalOp>(
+      mlir::SymbolTable::lookupSymbolIn(theModule, name));
+
+  if (gv) {
+    // There should be handling added here to check the type as assert that
+    // gv was a declaration if the type doesn't match and handling below
+    // to replace the variable if it was a declaration.
+    errorNYI(loc, "createOrReplaceCXXRuntimeVariable: already exists");
+    return gv;
+  }
+
+  // Create a new variable.
+  gv = createGlobalOp(*this, loc, name, ty);
+
+  // Set up extra information and add to the module
+  gv.setLinkageAttr(
+      cir::GlobalLinkageKindAttr::get(&getMLIRContext(), linkage));
+  mlir::SymbolTable::setSymbolVisibility(gv,
+                                         CIRGenModule::getMLIRVisibility(gv));
+
+  if (supportsCOMDAT() && cir::isWeakForLinker(linkage) &&
+      !gv.hasAvailableExternallyLinkage()) {
+    gv.setComdat(true);
+  }
+
+  gv.setAlignmentAttr(getSize(alignment));
+  setDSOLocal(static_cast<mlir::Operation *>(gv));
+  return gv;
+}
+
 // TODO(CIR): this could be a common method between LLVM codegen.
 static bool isVarDeclStrongDefinition(const ASTContext &astContext,
                                       CIRGenModule &cgm, const VarDecl *vd,
@@ -997,7 +1078,7 @@ static bool isVarDeclStrongDefinition(const ASTContext &astContext,
       return true;
 
     if (const auto *rt = varType->getAs<RecordType>()) {
-      const RecordDecl *rd = rt->getDecl();
+      const RecordDecl *rd = rt->getOriginalDecl()->getDefinitionOrSelf();
       for (const FieldDecl *fd : rd->fields()) {
         if (fd->isBitField())
           continue;
@@ -1181,8 +1262,8 @@ generateStringLiteral(mlir::Location loc, mlir::TypedAttr c,
 
   // Create a global variable for this string
   // FIXME(cir): check for insertion point in module level.
-  cir::GlobalOp gv =
-      CIRGenModule::createGlobalOp(cgm, loc, globalName, c.getType());
+  cir::GlobalOp gv = CIRGenModule::createGlobalOp(
+      cgm, loc, globalName, c.getType(), !cgm.getLangOpts().WritableStrings);
 
   // Set up extra information and add to the module
   gv.setAlignmentAttr(cgm.getSize(alignment));
@@ -1260,6 +1341,19 @@ cir::GlobalOp CIRGenModule::getGlobalForStringLiteral(const StringLiteral *s,
   return gv;
 }
 
+/// Return a pointer to a constant array for the given string literal.
+cir::GlobalViewAttr
+CIRGenModule::getAddrOfConstantStringFromLiteral(const StringLiteral *s,
+                                                 StringRef name) {
+  cir::GlobalOp gv = getGlobalForStringLiteral(s, name);
+  auto arrayTy = mlir::dyn_cast<cir::ArrayType>(gv.getSymType());
+  assert(arrayTy && "String literal must be array");
+  assert(!cir::MissingFeatures::addressSpace());
+  cir::PointerType ptrTy = getBuilder().getPointerTo(arrayTy.getElementType());
+
+  return builder.getGlobalViewAttr(ptrTy, gv);
+}
+
 void CIRGenModule::emitExplicitCastExprType(const ExplicitCastExpr *e,
                                             CIRGenFunction *cgf) {
   if (cgf && e->getType()->isVariablyModifiedType())
@@ -1307,7 +1401,8 @@ void CIRGenModule::emitTopLevelDecl(Decl *decl) {
   }
 
   case Decl::Var:
-  case Decl::Decomposition: {
+  case Decl::Decomposition:
+  case Decl::VarTemplateSpecialization: {
     auto *vd = cast<VarDecl>(decl);
     if (isa<DecompositionDecl>(decl)) {
       errorNYI(decl->getSourceRange(), "global variable decompositions");
@@ -1342,6 +1437,8 @@ void CIRGenModule::emitTopLevelDecl(Decl *decl) {
   case Decl::StaticAssert:
   case Decl::TypeAliasTemplate:
   case Decl::UsingShadow:
+  case Decl::VarTemplate:
+  case Decl::VarTemplatePartialSpecialization:
     break;
 
   case Decl::CXXConstructor:
@@ -1362,6 +1459,21 @@ void CIRGenModule::emitTopLevelDecl(Decl *decl) {
     assert(!cir::MissingFeatures::generateDebugInfo());
     assert(!cir::MissingFeatures::cxxRecordStaticMembers());
     break;
+
+  case Decl::FileScopeAsm:
+    // File-scope asm is ignored during device-side CUDA compilation.
+    if (langOpts.CUDA && langOpts.CUDAIsDevice)
+      break;
+    // File-scope asm is ignored during device-side OpenMP compilation.
+    if (langOpts.OpenMPIsTargetDevice)
+      break;
+    // File-scope asm is ignored during device-side SYCL compilation.
+    if (langOpts.SYCLIsDevice)
+      break;
+    auto *file_asm = cast<FileScopeAsmDecl>(decl);
+    std::string line = file_asm->getAsmString();
+    globalScopeAsm.push_back(builder.getStringAttr(line));
+    break;
   }
 }
 
@@ -1922,6 +2034,15 @@ CIRGenModule::createCIRFunction(mlir::Location loc, StringRef name,
   return func;
 }
 
+mlir::SymbolTable::Visibility
+CIRGenModule::getMLIRVisibility(cir::GlobalOp op) {
+  // MLIR doesn't accept public symbols declarations (only
+  // definitions).
+  if (op.isDeclaration())
+    return mlir::SymbolTable::Visibility::Private;
+  return getMLIRVisibilityFromCIRLinkage(op.getLinkage());
+}
+
 mlir::SymbolTable::Visibility
 CIRGenModule::getMLIRVisibilityFromCIRLinkage(cir::GlobalLinkageKind glk) {
   switch (glk) {
@@ -1975,6 +2096,9 @@ void CIRGenModule::release() {
   emitDeferred();
   applyReplacements();
 
+  theModule->setAttr(cir::CIRDialect::getModuleLevelAsmAttrName(),
+                     builder.getArrayAttr(globalScopeAsm));
+
   // There's a lot of code that is not implemented yet.
   assert(!cir::MissingFeatures::cgmRelease());
 }
@@ -2030,6 +2154,18 @@ bool CIRGenModule::verifyModule() const {
   return mlir::verify(theModule).succeeded();
 }
 
+mlir::Attribute CIRGenModule::getAddrOfRTTIDescriptor(mlir::Location loc,
+                                                      QualType ty, bool forEh) {
+  // Return a bogus pointer if RTTI is disabled, unless it's for EH.
+  // FIXME: should we even be calling this method if RTTI is disabled
+  // and it's not for EH?
+  if (!shouldEmitRTTI(forEh))
+    return builder.getConstNullPtrAttr(builder.getUInt8PtrTy());
+
+  errorNYI(loc, "getAddrOfRTTIDescriptor");
+  return mlir::Attribute();
+}
+
 // TODO(cir): this can be shared with LLVM codegen.
 CharUnits CIRGenModule::computeNonVirtualBaseClassOffset(
     const CXXRecordDecl *derivedClass,
@@ -2045,8 +2181,10 @@ CharUnits CIRGenModule::computeNonVirtualBaseClassOffset(
     // Get the layout.
     const ASTRecordLayout &layout = astContext.getASTRecordLayout(rd);
 
-    const auto *baseDecl = cast<CXXRecordDecl>(
-        base->getType()->castAs<clang::RecordType>()->getDecl());
+    const auto *baseDecl =
+        cast<CXXRecordDecl>(
+            base->getType()->castAs<clang::RecordType>()->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     // Add the offset.
     offset += layout.getBaseClassOffset(baseDecl);
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index 5d07d38012318..4f5c7f898af8c 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -17,6 +17,7 @@
 #include "CIRGenCall.h"
 #include "CIRGenTypeCache.h"
 #include "CIRGenTypes.h"
+#include "CIRGenVTables.h"
 #include "CIRGenValue.h"
 
 #include "clang/AST/CharUnits.h"
@@ -86,16 +87,22 @@ class CIRGenModule : public CIRGenTypeCache {
 
   CIRGenTypes genTypes;
 
+  /// Holds information about C++ vtables.
+  CIRGenVTables vtables;
+
   /// Per-function codegen information. Updated everytime emitCIR is called
   /// for FunctionDecls's.
   CIRGenFunction *curCGF = nullptr;
 
+  llvm::SmallVector<mlir::Attribute> globalScopeAsm;
+
 public:
   mlir::ModuleOp getModule() const { return theModule; }
   CIRGenBuilderTy &getBuilder() { return builder; }
   clang::ASTContext &getASTContext() const { return astContext; }
   const clang::TargetInfo &getTarget() const { return target; }
   const clang::CodeGenOptions &getCodeGenOpts() const { return codeGenOpts; }
+  clang::DiagnosticsEngine &getDiags() const { return diags; }
   CIRGenTypes &getTypes() { return genTypes; }
   const clang::LangOptions &getLangOpts() const { return langOpts; }
 
@@ -114,6 +121,9 @@ class CIRGenModule : public CIRGenTypeCache {
 
   mlir::Operation *lastGlobalOp = nullptr;
 
+  /// Tell the consumer that this variable has been instantiated.
+  void handleCXXStaticMemberVarInstantiation(VarDecl *vd);
+
   llvm::DenseMap<const Decl *, cir::GlobalOp> staticLocalDeclMap;
 
   mlir::Operation *getGlobalValue(llvm::StringRef ref);
@@ -140,6 +150,7 @@ class CIRGenModule : public CIRGenTypeCache {
 
   static cir::GlobalOp createGlobalOp(CIRGenModule &cgm, mlir::Location loc,
                                       llvm::StringRef name, mlir::Type t,
+                                      bool isConstant = false,
                                       mlir::Operation *insertPoint = nullptr);
 
   llvm::StringMap<unsigned> cgGlobalNames;
@@ -155,6 +166,9 @@ class CIRGenModule : public CIRGenTypeCache {
   getAddrOfGlobalVar(const VarDecl *d, mlir::Type ty = {},
                      ForDefinition_t isForDefinition = NotForDefinition);
 
+  /// Return the mlir::GlobalViewAttr for the address of the given global.
+  cir::GlobalViewAttr getAddrOfGlobalVarAttr(const VarDecl *d);
+
   CharUnits computeNonVirtualBaseClassOffset(
       const CXXRecordDecl *derivedClass,
       llvm::iterator_range<CastExpr::path_const_iterator> path);
@@ -168,6 +182,24 @@ class CIRGenModule : public CIRGenTypeCache {
   void constructAttributeList(CIRGenCalleeInfo calleeInfo,
                               mlir::NamedAttrList &attrs);
 
+  /// Will return a global variable of the given type. If a variable with a
+  /// different type already exists then a new variable with the right type
+  /// will be created and all uses of the old variable will be replaced with a
+  /// bitcast to the new variable.
+  cir::GlobalOp createOrReplaceCXXRuntimeVariable(
+      mlir::Location loc, llvm::StringRef name, mlir::Type ty,
+      cir::GlobalLinkageKind linkage, clang::CharUnits alignment);
+
+  void emitVTable(const CXXRecordDecl *rd);
+
+  /// Return the appropriate linkage for the vtable, VTT, and type information
+  /// of the given class.
+  cir::GlobalLinkageKind getVTableLinkage(const CXXRecordDecl *rd);
+
+  /// Get the address of the RTTI descriptor for the given type.
+  mlir::Attribute getAddrOfRTTIDescriptor(mlir::Location loc, QualType ty,
+                                          bool forEH = false);
+
   /// Return a constant array for the given string.
   mlir::Attribute getConstantArrayFromStringLiteral(const StringLiteral *e);
 
@@ -176,6 +208,12 @@ class CIRGenModule : public CIRGenTypeCache {
   cir::GlobalOp getGlobalForStringLiteral(const StringLiteral *s,
                                           llvm::StringRef name = ".str");
 
+  /// Return a global symbol reference to a constant array for the given string
+  /// literal.
+  cir::GlobalViewAttr
+  getAddrOfConstantStringFromLiteral(const StringLiteral *s,
+                                     llvm::StringRef name = ".str");
+
   /// Set attributes which are common to any form of a global definition (alias,
   /// Objective-C method, function, global variable).
   ///
@@ -213,6 +251,16 @@ class CIRGenModule : public CIRGenTypeCache {
       cir::FuncType fnType = nullptr, bool dontDefer = false,
       ForDefinition_t isForDefinition = NotForDefinition);
 
+  mlir::Type getVTableComponentType();
+  CIRGenVTables &getVTables() { return vtables; }
+
+  ItaniumVTableContext &getItaniumVTableContext() {
+    return vtables.getItaniumVTableContext();
+  }
+  const ItaniumVTableContext &getItaniumVTableContext() const {
+    return vtables.getItaniumVTableContext();
+  }
+
   /// This contains all the decls which have definitions but which are deferred
   /// for emission and therefore should only be output if they are actually
   /// used. If a decl is in this, then it is known to have not been referenced
@@ -252,6 +300,13 @@ class CIRGenModule : public CIRGenTypeCache {
   getAddrOfGlobal(clang::GlobalDecl gd,
                   ForDefinition_t isForDefinition = NotForDefinition);
 
+  // Return whether RTTI information should be emitted for this target.
+  bool shouldEmitRTTI(bool forEH = false) {
+    return (forEH || getLangOpts().RTTI) && !getLangOpts().CUDAIsDevice &&
+           !(getLangOpts().OpenMP && getLangOpts().OpenMPIsTargetDevice &&
+             getTriple().isNVPTX());
+  }
+
   /// Emit type info if type of an expression is a variably modified
   /// type. Also emit proper debug info for cast types.
   void emitExplicitCastExprType(const ExplicitCastExpr *e,
@@ -355,8 +410,8 @@ class CIRGenModule : public CIRGenTypeCache {
   static cir::VisibilityKind getGlobalVisibilityKindFromClangVisibility(
       clang::VisibilityAttr::VisibilityType visibility);
   cir::VisibilityAttr getGlobalVisibilityAttrFromDecl(const Decl *decl);
-  static mlir::SymbolTable::Visibility getMLIRVisibility(cir::GlobalOp op);
   cir::GlobalLinkageKind getFunctionLinkage(GlobalDecl gd);
+  static mlir::SymbolTable::Visibility getMLIRVisibility(cir::GlobalOp op);
   cir::GlobalLinkageKind getCIRLinkageForDeclarator(const DeclaratorDecl *dd,
                                                     GVALinkage linkage,
                                                     bool isConstantVariable);
@@ -423,6 +478,9 @@ class CIRGenModule : public CIRGenTypeCache {
   void replacePointerTypeArgs(cir::FuncOp oldF, cir::FuncOp newF);
 
   void setNonAliasAttributes(GlobalDecl gd, mlir::Operation *op);
+
+  /// Map source language used to a CIR attribute.
+  std::optional<cir::SourceLanguage> getCIRSourceLanguage() const;
 };
 } // namespace CIRGen
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp b/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
index bb9054a68b5c7..4ad15e6922ddd 100644
--- a/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp
@@ -357,15 +357,11 @@ class OpenACCClauseCIREmitter final
   }
 
   template <typename RecipeTy>
-  RecipeTy getOrCreateRecipe(ASTContext &astCtx, const Expr *varRef,
-                             const VarDecl *varRecipe, DeclContext *dc,
-                             QualType baseType, mlir::Value mainOp) {
-    mlir::ModuleOp mod =
-        builder.getBlock()->getParent()->getParentOfType<mlir::ModuleOp>();
-
+  std::string getRecipeName(SourceRange loc, QualType baseType) {
     std::string recipeName;
     {
       llvm::raw_string_ostream stream(recipeName);
+
       if constexpr (std::is_same_v<RecipeTy, mlir::acc::PrivateRecipeOp>) {
         stream << "privatization_";
       } else if constexpr (std::is_same_v<RecipeTy,
@@ -375,11 +371,12 @@ class OpenACCClauseCIREmitter final
       } else if constexpr (std::is_same_v<RecipeTy,
                                           mlir::acc::ReductionRecipeOp>) {
         stream << "reduction_";
-        // We don't have the reduction operation here well enough to know how to
-        // spell this correctly (+ == 'add', etc), so when we implement
-        // 'reduction' we have to do that here.
-        cgf.cgm.errorNYI(varRef->getSourceRange(),
-                         "OpeNACC reduction recipe creation");
+        // TODO: OpenACC: once we have this part implemented, we can remove the
+        // SourceRange `loc` variable from this function. We don't have the
+        // reduction operation here well enough to know how to spell this
+        // correctly (+ == 'add', etc), so when we implement 'reduction' we have
+        // to do that here.
+        cgf.cgm.errorNYI(loc, "OpenACC reduction recipe name creation");
       } else {
         static_assert(!sizeof(RecipeTy), "Unknown Recipe op kind");
       }
@@ -387,72 +384,150 @@ class OpenACCClauseCIREmitter final
       MangleContext &mc = cgf.cgm.getCXXABI().getMangleContext();
       mc.mangleCanonicalTypeName(baseType, stream);
     }
+    return recipeName;
+  }
 
-    if (auto recipe = mod.lookupSymbol<RecipeTy>(recipeName))
-      return recipe;
-
-    mlir::Location loc = cgf.cgm.getLoc(varRef->getBeginLoc());
-    mlir::Location locEnd = cgf.cgm.getLoc(varRef->getEndLoc());
+  void createFirstprivateRecipeCopy(
+      mlir::Location loc, mlir::Location locEnd, mlir::Value mainOp,
+      CIRGenFunction::AutoVarEmission tempDeclEmission,
+      mlir::acc::FirstprivateRecipeOp recipe, const VarDecl *varRecipe,
+      const VarDecl *temporary) {
+    mlir::Block *block = builder.createBlock(
+        &recipe.getCopyRegion(), recipe.getCopyRegion().end(),
+        {mainOp.getType(), mainOp.getType()}, {loc, loc});
+    builder.setInsertionPointToEnd(&recipe.getCopyRegion().back());
+
+    mlir::BlockArgument fromArg = block->getArgument(0);
+    mlir::BlockArgument toArg = block->getArgument(1);
+
+    mlir::Type elementTy =
+        mlir::cast<cir::PointerType>(mainOp.getType()).getPointee();
+
+    // Set the address of the emission to be the argument, so that we initialize
+    // that instead of the variable in the other block.
+    tempDeclEmission.setAllocatedAddress(
+        Address{toArg, elementTy, cgf.getContext().getDeclAlign(varRecipe)});
+    tempDeclEmission.EmittedAsOffload = true;
+
+    CIRGenFunction::DeclMapRevertingRAII declMapRAII{cgf, temporary};
+    cgf.setAddrOfLocalVar(
+        temporary,
+        Address{fromArg, elementTy, cgf.getContext().getDeclAlign(varRecipe)});
+
+    cgf.emitAutoVarInit(tempDeclEmission);
+    mlir::acc::YieldOp::create(builder, locEnd);
+  }
 
-    mlir::OpBuilder modBuilder(mod.getBodyRegion());
-    auto recipe =
-        RecipeTy::create(modBuilder, loc, recipeName, mainOp.getType());
+  // Create the 'init' section of the recipe, including the 'copy' section for
+  // 'firstprivate'.
+  template <typename RecipeTy>
+  void createRecipeInitCopy(mlir::Location loc, mlir::Location locEnd,
+                            SourceRange exprRange, mlir::Value mainOp,
+                            RecipeTy recipe, const VarDecl *varRecipe,
+                            const VarDecl *temporary) {
+    assert(varRecipe && "Required recipe variable not set?");
+    if constexpr (std::is_same_v<RecipeTy, mlir::acc::ReductionRecipeOp>) {
+      // We haven't implemented the 'init' recipe for Reduction yet, so NYI
+      // it.
+      cgf.cgm.errorNYI(exprRange, "OpenACC Reduction recipe init");
+    }
 
     CIRGenFunction::AutoVarEmission tempDeclEmission{
         CIRGenFunction::AutoVarEmission::invalid()};
-
-    // Init section.
-    {
-      llvm::SmallVector<mlir::Type> argsTys{mainOp.getType()};
-      llvm::SmallVector<mlir::Location> argsLocs{loc};
-      builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(),
-                          argsTys, argsLocs);
-      builder.setInsertionPointToEnd(&recipe.getInitRegion().back());
-
-      if constexpr (!std::is_same_v<RecipeTy, mlir::acc::PrivateRecipeOp>) {
-        // We have only implemented 'init' for private, so make this NYI until
-        // we have explicitly implemented everything.
-        cgf.cgm.errorNYI(varRef->getSourceRange(),
-                         "OpenACC non-private recipe init");
+    CIRGenFunction::DeclMapRevertingRAII declMapRAII{cgf, varRecipe};
+
+    // Do the 'init' section of the recipe IR, which does an alloca, then the
+    // initialization (except for firstprivate).
+    builder.createBlock(&recipe.getInitRegion(), recipe.getInitRegion().end(),
+                        {mainOp.getType()}, {loc});
+    builder.setInsertionPointToEnd(&recipe.getInitRegion().back());
+    tempDeclEmission =
+        cgf.emitAutoVarAlloca(*varRecipe, builder.saveInsertionPoint());
+
+    // 'firstprivate' doesn't do its initialization in the 'init' section,
+    // instead does it in the 'copy' section.  SO only do init here.
+    // 'reduction' appears to use it too (rather than a 'copy' section), so
+    // we probably have to do it here too, but we can do that when we get to
+    // reduction implementation.
+    if constexpr (std::is_same_v<RecipeTy, mlir::acc::PrivateRecipeOp>) {
+      // We are OK with no init for builtins, arrays of builtins, or pointers,
+      // else we should NYI so we know to go look for these.
+      if (!varRecipe->getType()
+               ->getPointeeOrArrayElementType()
+               ->isBuiltinType() &&
+          !varRecipe->getType()->isPointerType() && !varRecipe->getInit()) {
+        // If we don't have any initialization recipe, we failed during Sema to
+        // initialize this correctly. If we disable the
+        // Sema::TentativeAnalysisScopes in SemaOpenACC::CreateInitRecipe, it'll
+        // emit an error to tell us.  However, emitting those errors during
+        // production is a violation of the standard, so we cannot do them.
+        cgf.cgm.errorNYI(exprRange, "private default-init recipe");
       }
+      cgf.emitAutoVarInit(tempDeclEmission);
+    }
 
-      if (varRecipe) {
-        tempDeclEmission =
-            cgf.emitAutoVarAlloca(*varRecipe, builder.saveInsertionPoint());
-        cgf.emitAutoVarInit(tempDeclEmission);
+    mlir::acc::YieldOp::create(builder, locEnd);
+
+    if constexpr (std::is_same_v<RecipeTy, mlir::acc::FirstprivateRecipeOp>) {
+      if (!varRecipe->getInit()) {
+        // If we don't have any initialization recipe, we failed during Sema to
+        // initialize this correctly. If we disable the
+        // Sema::TentativeAnalysisScopes in SemaOpenACC::CreateInitRecipe, it'll
+        // emit an error to tell us.  However, emitting those errors during
+        // production is a violation of the standard, so we cannot do them.
+        cgf.cgm.errorNYI(
+            exprRange, "firstprivate copy-init recipe not properly generated");
       }
 
-      mlir::acc::YieldOp::create(builder, locEnd);
+      createFirstprivateRecipeCopy(loc, locEnd, mainOp, tempDeclEmission,
+                                   recipe, varRecipe, temporary);
     }
+  }
 
-    // Copy section.
-    if constexpr (std::is_same_v<RecipeTy, mlir::acc::FirstprivateRecipeOp> ||
-                  std::is_same_v<RecipeTy, mlir::acc::ReductionRecipeOp>) {
-      // TODO: OpenACC: 'private' doesn't emit this, but for the other two we
-      // have to figure out what 'copy' means here.
-      cgf.cgm.errorNYI(varRef->getSourceRange(),
-                       "OpenACC record type privatization copy section");
-    }
+  void createRecipeDestroySection(mlir::Location loc, mlir::Location locEnd,
+                                  mlir::Value mainOp, CharUnits alignment,
+                                  QualType baseType,
+                                  mlir::Region &destroyRegion) {
+    mlir::Block *block = builder.createBlock(
+        &destroyRegion, destroyRegion.end(), {mainOp.getType()}, {loc});
+    builder.setInsertionPointToEnd(&destroyRegion.back());
+
+    mlir::Type elementTy =
+        mlir::cast<cir::PointerType>(mainOp.getType()).getPointee();
+    Address addr{block->getArgument(0), elementTy, alignment};
+    cgf.emitDestroy(addr, baseType,
+                    cgf.getDestroyer(QualType::DK_cxx_destructor));
+
+    mlir::acc::YieldOp::create(builder, locEnd);
+  }
 
-    // Destroy section (doesn't currently exist).
-    if (varRecipe && varRecipe->needsDestruction(cgf.getContext())) {
-      llvm::SmallVector<mlir::Type> argsTys{mainOp.getType()};
-      llvm::SmallVector<mlir::Location> argsLocs{loc};
-      mlir::Block *block = builder.createBlock(&recipe.getDestroyRegion(),
-                                               recipe.getDestroyRegion().end(),
-                                               argsTys, argsLocs);
-      builder.setInsertionPointToEnd(&recipe.getDestroyRegion().back());
-
-      mlir::Type elementTy =
-          mlir::cast<cir::PointerType>(mainOp.getType()).getPointee();
-      Address addr{block->getArgument(0), elementTy,
-                   cgf.getContext().getDeclAlign(varRecipe)};
-      cgf.emitDestroy(addr, baseType,
-                      cgf.getDestroyer(QualType::DK_cxx_destructor));
-
-      mlir::acc::YieldOp::create(builder, locEnd);
-    }
+  template <typename RecipeTy>
+  RecipeTy getOrCreateRecipe(ASTContext &astCtx, const Expr *varRef,
+                             const VarDecl *varRecipe, const VarDecl *temporary,
+                             DeclContext *dc, QualType baseType,
+                             mlir::Value mainOp) {
+    mlir::ModuleOp mod =
+        builder.getBlock()->getParent()->getParentOfType<mlir::ModuleOp>();
 
+    std::string recipeName =
+        getRecipeName<RecipeTy>(varRef->getSourceRange(), baseType);
+    if (auto recipe = mod.lookupSymbol<RecipeTy>(recipeName))
+      return recipe;
+
+    mlir::Location loc = cgf.cgm.getLoc(varRef->getBeginLoc());
+    mlir::Location locEnd = cgf.cgm.getLoc(varRef->getEndLoc());
+
+    mlir::OpBuilder modBuilder(mod.getBodyRegion());
+    auto recipe =
+        RecipeTy::create(modBuilder, loc, recipeName, mainOp.getType());
+
+    createRecipeInitCopy(loc, locEnd, varRef->getSourceRange(), mainOp, recipe,
+                         varRecipe, temporary);
+
+    if (varRecipe && varRecipe->needsDestruction(cgf.getContext()))
+      createRecipeDestroySection(loc, locEnd, mainOp,
+                                 cgf.getContext().getDeclAlign(varRecipe),
+                                 baseType, recipe.getDestroyRegion());
     return recipe;
   }
 
@@ -1088,7 +1163,7 @@ class OpenACCClauseCIREmitter final
         {
           mlir::OpBuilder::InsertionGuard guardCase(builder);
           auto recipe = getOrCreateRecipe<mlir::acc::PrivateRecipeOp>(
-              cgf.getContext(), varExpr, varRecipe,
+              cgf.getContext(), varExpr, varRecipe, /*temporary=*/nullptr,
               Decl::castToDeclContext(cgf.curFuncDecl), opInfo.baseType,
               privateOp.getResult());
           // TODO: OpenACC: The dialect is going to change in the near future to
@@ -1105,6 +1180,43 @@ class OpenACCClauseCIREmitter final
       llvm_unreachable("Unknown construct kind in VisitPrivateClause");
     }
   }
+
+  void VisitFirstPrivateClause(const OpenACCFirstPrivateClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp,
+                               mlir::acc::SerialOp>) {
+      for (const auto [varExpr, varRecipe] :
+           llvm::zip_equal(clause.getVarList(), clause.getInitRecipes())) {
+        CIRGenFunction::OpenACCDataOperandInfo opInfo =
+            cgf.getOpenACCDataOperandInfo(varExpr);
+        auto firstPrivateOp = mlir::acc::FirstprivateOp::create(
+            builder, opInfo.beginLoc, opInfo.varValue, /*structured=*/true,
+            /*implicit=*/false, opInfo.name, opInfo.bounds);
+
+        firstPrivateOp.setDataClause(mlir::acc::DataClause::acc_firstprivate);
+
+        {
+          mlir::OpBuilder::InsertionGuard guardCase(builder);
+          auto recipe = getOrCreateRecipe<mlir::acc::FirstprivateRecipeOp>(
+              cgf.getContext(), varExpr, varRecipe.RecipeDecl,
+              varRecipe.InitFromTemporary,
+              Decl::castToDeclContext(cgf.curFuncDecl), opInfo.baseType,
+              firstPrivateOp.getResult());
+
+          // TODO: OpenACC: The dialect is going to change in the near future to
+          // have these be on a different operation, so when that changes, we
+          // probably need to change these here.
+          operation.addFirstPrivatization(builder.getContext(), firstPrivateOp,
+                                          recipe);
+        }
+      }
+    } else if constexpr (isCombinedType<OpTy>) {
+      // Unlike 'private', 'firstprivate' applies to the compute op, not the
+      // loop op.
+      applyToComputeOp(clause);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitFirstPrivateClause");
+    }
+  }
 };
 
 template <typename OpTy>
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index 50642e7ca48d7..3b0eabe92284b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -14,6 +14,8 @@
 #include "CIRGenFunction.h"
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/Location.h"
+#include "mlir/Support/LLVM.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/Stmt.h"
 #include "clang/AST/StmtOpenACC.h"
@@ -23,28 +25,80 @@ using namespace clang;
 using namespace clang::CIRGen;
 using namespace cir;
 
-void CIRGenFunction::emitCompoundStmtWithoutScope(const CompoundStmt &s) {
-  for (auto *curStmt : s.body()) {
-    if (emitStmt(curStmt, /*useCurrentScope=*/false).failed())
-      getCIRGenModule().errorNYI(curStmt->getSourceRange(),
-                                 std::string("emitCompoundStmtWithoutScope: ") +
-                                     curStmt->getStmtClassName());
+static mlir::LogicalResult emitStmtWithResult(CIRGenFunction &cgf,
+                                              const Stmt *exprResult,
+                                              AggValueSlot slot,
+                                              Address *lastValue) {
+  // We have to special case labels here. They are statements, but when put
+  // at the end of a statement expression, they yield the value of their
+  // subexpression. Handle this by walking through all labels we encounter,
+  // emitting them before we evaluate the subexpr.
+  // Similar issues arise for attributed statements.
+  while (!isa<Expr>(exprResult)) {
+    if (const auto *ls = dyn_cast<LabelStmt>(exprResult)) {
+      if (cgf.emitLabel(*ls->getDecl()).failed())
+        return mlir::failure();
+      exprResult = ls->getSubStmt();
+    } else if (const auto *as = dyn_cast<AttributedStmt>(exprResult)) {
+      // FIXME: Update this if we ever have attributes that affect the
+      // semantics of an expression.
+      exprResult = as->getSubStmt();
+    } else {
+      llvm_unreachable("Unknown value statement");
+    }
   }
+
+  const Expr *e = cast<Expr>(exprResult);
+  QualType exprTy = e->getType();
+  if (cgf.hasAggregateEvaluationKind(exprTy)) {
+    cgf.emitAggExpr(e, slot);
+  } else {
+    // We can't return an RValue here because there might be cleanups at
+    // the end of the StmtExpr.  Because of that, we have to emit the result
+    // here into a temporary alloca.
+    cgf.emitAnyExprToMem(e, *lastValue, Qualifiers(),
+                         /*IsInit*/ false);
+  }
+
+  return mlir::success();
 }
 
-void CIRGenFunction::emitCompoundStmt(const CompoundStmt &s) {
+mlir::LogicalResult CIRGenFunction::emitCompoundStmtWithoutScope(
+    const CompoundStmt &s, Address *lastValue, AggValueSlot slot) {
+  mlir::LogicalResult result = mlir::success();
+  const Stmt *exprResult = s.getStmtExprResult();
+  assert((!lastValue || (lastValue && exprResult)) &&
+         "If lastValue is not null then the CompoundStmt must have a "
+         "StmtExprResult");
+
+  for (const Stmt *curStmt : s.body()) {
+    const bool saveResult = lastValue && exprResult == curStmt;
+    if (saveResult) {
+      if (emitStmtWithResult(*this, exprResult, slot, lastValue).failed())
+        result = mlir::failure();
+    } else {
+      if (emitStmt(curStmt, /*useCurrentScope=*/false).failed())
+        result = mlir::failure();
+    }
+  }
+  return result;
+}
+
+mlir::LogicalResult CIRGenFunction::emitCompoundStmt(const CompoundStmt &s,
+                                                     Address *lastValue,
+                                                     AggValueSlot slot) {
+  // Add local scope to track new declared variables.
+  SymTableScopeTy varScope(symbolTable);
   mlir::Location scopeLoc = getLoc(s.getSourceRange());
   mlir::OpBuilder::InsertPoint scopeInsPt;
   builder.create<cir::ScopeOp>(
       scopeLoc, [&](mlir::OpBuilder &b, mlir::Type &type, mlir::Location loc) {
         scopeInsPt = b.saveInsertionPoint();
       });
-  {
-    mlir::OpBuilder::InsertionGuard guard(builder);
-    builder.restoreInsertionPoint(scopeInsPt);
-    LexicalScope lexScope(*this, scopeLoc, builder.getInsertionBlock());
-    emitCompoundStmtWithoutScope(s);
-  }
+  mlir::OpBuilder::InsertionGuard guard(builder);
+  builder.restoreInsertionPoint(scopeInsPt);
+  LexicalScope lexScope(*this, scopeLoc, builder.getInsertionBlock());
+  return emitCompoundStmtWithoutScope(s, lastValue, slot);
 }
 
 void CIRGenFunction::emitStopPoint(const Stmt *s) {
@@ -130,6 +184,9 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     return emitOpenACCCacheConstruct(cast<OpenACCCacheConstruct>(*s));
   case Stmt::OpenACCAtomicConstructClass:
     return emitOpenACCAtomicConstruct(cast<OpenACCAtomicConstruct>(*s));
+  case Stmt::GCCAsmStmtClass:
+  case Stmt::MSAsmStmtClass:
+    return emitAsmStmt(cast<AsmStmt>(*s));
   case Stmt::OMPScopeDirectiveClass:
   case Stmt::OMPErrorDirectiveClass:
   case Stmt::LabelStmtClass:
@@ -143,8 +200,6 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
   case Stmt::CoreturnStmtClass:
   case Stmt::CXXTryStmtClass:
   case Stmt::IndirectGotoStmtClass:
-  case Stmt::GCCAsmStmtClass:
-  case Stmt::MSAsmStmtClass:
   case Stmt::OMPParallelDirectiveClass:
   case Stmt::OMPTaskwaitDirectiveClass:
   case Stmt::OMPTaskyieldDirectiveClass:
@@ -246,16 +301,19 @@ mlir::LogicalResult CIRGenFunction::emitSimpleStmt(const Stmt *s,
     return emitDeclStmt(cast<DeclStmt>(*s));
   case Stmt::CompoundStmtClass:
     if (useCurrentScope)
-      emitCompoundStmtWithoutScope(cast<CompoundStmt>(*s));
-    else
-      emitCompoundStmt(cast<CompoundStmt>(*s));
-    break;
+      return emitCompoundStmtWithoutScope(cast<CompoundStmt>(*s));
+    return emitCompoundStmt(cast<CompoundStmt>(*s));
+  case Stmt::GotoStmtClass:
+    return emitGotoStmt(cast<GotoStmt>(*s));
   case Stmt::ContinueStmtClass:
     return emitContinueStmt(cast<ContinueStmt>(*s));
 
   // NullStmt doesn't need any handling, but we need to say we handled it.
   case Stmt::NullStmtClass:
     break;
+
+  case Stmt::LabelStmtClass:
+    return emitLabelStmt(cast<LabelStmt>(*s));
   case Stmt::CaseStmtClass:
   case Stmt::DefaultStmtClass:
     // If we reached here, we must not handling a switch case in the top level.
@@ -272,6 +330,17 @@ mlir::LogicalResult CIRGenFunction::emitSimpleStmt(const Stmt *s,
   return mlir::success();
 }
 
+mlir::LogicalResult CIRGenFunction::emitLabelStmt(const clang::LabelStmt &s) {
+
+  if (emitLabel(*s.getDecl()).failed())
+    return mlir::failure();
+
+  if (getContext().getLangOpts().EHAsynch && s.isSideEntry())
+    getCIRGenModule().errorNYI(s.getSourceRange(), "IsEHa: not implemented.");
+
+  return emitStmt(s.getSubStmt(), /*useCurrentScope*/ true);
+}
+
 // Add a terminating yield on a body region if no other terminators are used.
 static void terminateBody(CIRGenBuilderTy &builder, mlir::Region &r,
                           mlir::Location loc) {
@@ -412,13 +481,31 @@ mlir::LogicalResult CIRGenFunction::emitReturnStmt(const ReturnStmt &s) {
   auto *retBlock = curLexScope->getOrCreateRetBlock(*this, loc);
   // This should emit a branch through the cleanup block if one exists.
   builder.create<cir::BrOp>(loc, retBlock);
-  if (ehStack.getStackDepth() != currentCleanupStackDepth)
+  if (ehStack.stable_begin() != currentCleanupStackDepth)
     cgm.errorNYI(s.getSourceRange(), "return with cleanup stack");
   builder.createBlock(builder.getBlock()->getParent());
 
   return mlir::success();
 }
 
+mlir::LogicalResult CIRGenFunction::emitGotoStmt(const clang::GotoStmt &s) {
+  // FIXME: LLVM codegen inserts emit a stop point here for debug info
+  // sake when the insertion point is available, but doesn't do
+  // anything special when there isn't. We haven't implemented debug
+  // info support just yet, look at this again once we have it.
+  assert(!cir::MissingFeatures::generateDebugInfo());
+
+  cir::GotoOp::create(builder, getLoc(s.getSourceRange()),
+                      s.getLabel()->getName());
+
+  // A goto marks the end of a block, create a new one for codegen after
+  // emitGotoStmt can resume building in that block.
+  // Insert the new block to continue codegen after goto.
+  builder.createBlock(builder.getBlock()->getParent());
+
+  return mlir::success();
+}
+
 mlir::LogicalResult
 CIRGenFunction::emitContinueStmt(const clang::ContinueStmt &s) {
   builder.createContinue(getLoc(s.getContinueLoc()));
@@ -429,6 +516,32 @@ CIRGenFunction::emitContinueStmt(const clang::ContinueStmt &s) {
   return mlir::success();
 }
 
+mlir::LogicalResult CIRGenFunction::emitLabel(const clang::LabelDecl &d) {
+  // Create a new block to tag with a label and add a branch from
+  // the current one to it. If the block is empty just call attach it
+  // to this label.
+  mlir::Block *currBlock = builder.getBlock();
+  mlir::Block *labelBlock = currBlock;
+
+  if (!currBlock->empty()) {
+    {
+      mlir::OpBuilder::InsertionGuard guard(builder);
+      labelBlock = builder.createBlock(builder.getBlock()->getParent());
+    }
+    builder.create<cir::BrOp>(getLoc(d.getSourceRange()), labelBlock);
+  }
+
+  builder.setInsertionPointToEnd(labelBlock);
+  builder.create<cir::LabelOp>(getLoc(d.getSourceRange()), d.getName());
+  builder.setInsertionPointToEnd(labelBlock);
+
+  //  FIXME: emit debug info for labels, incrementProfileCounter
+  assert(!cir::MissingFeatures::ehstackBranches());
+  assert(!cir::MissingFeatures::incrementProfileCounter());
+  assert(!cir::MissingFeatures::generateDebugInfo());
+  return mlir::success();
+}
+
 mlir::LogicalResult CIRGenFunction::emitBreakStmt(const clang::BreakStmt &s) {
   builder.createBreak(getLoc(s.getBreakLoc()));
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
index 1d081d53ad15c..eb8dcd67a4a81 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypeCache.h
@@ -59,6 +59,7 @@ struct CIRGenTypeCache {
 
   /// void* in address space 0
   cir::PointerType VoidPtrTy;
+  cir::PointerType UInt8PtrTy;
 
   /// The size and alignment of a pointer into the generic address space.
   union {
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index 3e07f6d3e54cc..aada6094d0fd5 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -103,7 +103,8 @@ std::string CIRGenTypes::getRecordTypeName(const clang::RecordDecl *recordDecl,
   policy.SuppressTagKeyword = true;
 
   if (recordDecl->getIdentifier())
-    astContext.getRecordType(recordDecl).print(outStream, policy);
+    QualType(astContext.getCanonicalTagType(recordDecl))
+        .print(outStream, policy);
   else if (auto *typedefNameDecl = recordDecl->getTypedefNameForAnonDecl())
     typedefNameDecl->printQualifiedName(outStream, policy);
   else
@@ -138,7 +139,9 @@ isSafeToConvert(const RecordDecl *rd, CIRGenTypes &cgt,
   if (!alreadyChecked.insert(rd).second)
     return true;
 
-  const Type *key = cgt.getASTContext().getTagDeclType(rd).getTypePtr();
+  assert(rd->isCompleteDefinition() &&
+         "Expect RecordDecl to be CompleteDefinition");
+  const Type *key = cgt.getASTContext().getCanonicalTagType(rd).getTypePtr();
 
   // If this type is already laid out, converting it is a noop.
   if (cgt.isRecordLayoutComplete(key))
@@ -152,13 +155,14 @@ isSafeToConvert(const RecordDecl *rd, CIRGenTypes &cgt,
   // out, don't do it.  This includes virtual base classes which get laid out
   // when a class is translated, even though they aren't embedded by-value into
   // the class.
-  if (auto *crd = dyn_cast<CXXRecordDecl>(rd)) {
-    if (crd->getNumBases() > 0) {
-      assert(!cir::MissingFeatures::cxxSupport());
-      cgt.getCGModule().errorNYI(rd->getSourceRange(),
-                                 "isSafeToConvert: CXXRecordDecl with bases");
-      return false;
-    }
+  if (const CXXRecordDecl *crd = dyn_cast<CXXRecordDecl>(rd)) {
+    for (const clang::CXXBaseSpecifier &i : crd->bases())
+      if (!isSafeToConvert(i.getType()
+                               ->castAs<RecordType>()
+                               ->getOriginalDecl()
+                               ->getDefinitionOrSelf(),
+                           cgt, alreadyChecked))
+        return false;
   }
 
   // If this type would require laying out members that are currently being laid
@@ -182,7 +186,8 @@ isSafeToConvert(QualType qt, CIRGenTypes &cgt,
 
   // If this is a record, check it.
   if (const auto *rt = qt->getAs<RecordType>())
-    return isSafeToConvert(rt->getDecl(), cgt, alreadyChecked);
+    return isSafeToConvert(rt->getOriginalDecl()->getDefinitionOrSelf(), cgt,
+                           alreadyChecked);
 
   // If this is an array, check the elements, which are embedded inline.
   if (const auto *at = cgt.getASTContext().getAsArrayType(qt))
@@ -210,7 +215,7 @@ static bool isSafeToConvert(const RecordDecl *rd, CIRGenTypes &cgt) {
 mlir::Type CIRGenTypes::convertRecordDeclType(const clang::RecordDecl *rd) {
   // TagDecl's are not necessarily unique, instead use the (clang) type
   // connected to the decl.
-  const Type *key = astContext.getTagDeclType(rd).getTypePtr();
+  const Type *key = astContext.getCanonicalTagType(rd).getTypePtr();
   cir::RecordType entry = recordDeclTypes[key];
 
   // If we don't have an entry for this record yet, create one.
@@ -242,7 +247,10 @@ mlir::Type CIRGenTypes::convertRecordDeclType(const clang::RecordDecl *rd) {
     for (const auto &base : cxxRecordDecl->bases()) {
       if (base.isVirtual())
         continue;
-      convertRecordDeclType(base.getType()->castAs<RecordType>()->getDecl());
+      convertRecordDeclType(base.getType()
+                                ->castAs<RecordType>()
+                                ->getOriginalDecl()
+                                ->getDefinitionOrSelf());
     }
   }
 
@@ -275,7 +283,8 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
 
   // Process record types before the type cache lookup.
   if (const auto *recordType = dyn_cast<RecordType>(type))
-    return convertRecordDeclType(recordType->getDecl());
+    return convertRecordDeclType(
+        recordType->getOriginalDecl()->getDefinitionOrSelf());
 
   // Has the type already been processed?
   TypeCacheTy::iterator tci = typeCache.find(ty);
@@ -457,7 +466,8 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
   }
 
   case Type::Enum: {
-    const EnumDecl *ed = cast<EnumType>(ty)->getDecl();
+    const EnumDecl *ed =
+        cast<EnumType>(ty)->getOriginalDecl()->getDefinitionOrSelf();
     if (auto integerType = ed->getIntegerType(); !integerType.isNull())
       return convertType(integerType);
     // Return a placeholder 'i32' type.  This can be changed later when the
@@ -484,6 +494,20 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
     break;
   }
 
+  case Type::Atomic: {
+    QualType valueType = cast<AtomicType>(ty)->getValueType();
+    resultType = convertTypeForMem(valueType);
+
+    // Pad out to the inflated size if necessary.
+    uint64_t valueSize = astContext.getTypeSize(valueType);
+    uint64_t atomicSize = astContext.getTypeSize(ty);
+    if (valueSize != atomicSize) {
+      cgm.errorNYI("convertType: atomic type value size != atomic size");
+    }
+
+    break;
+  }
+
   default:
     cgm.errorNYI(SourceLocation(), "processing of type",
                  type->getTypeClassName());
@@ -516,7 +540,7 @@ mlir::Type CIRGenTypes::convertTypeForMem(clang::QualType qualType,
 /// Return record layout info for the given record decl.
 const CIRGenRecordLayout &
 CIRGenTypes::getCIRGenRecordLayout(const RecordDecl *rd) {
-  const auto *key = astContext.getTagDeclType(rd).getTypePtr();
+  const auto *key = astContext.getCanonicalTagType(rd).getTypePtr();
 
   // If we have already computed the layout, return it.
   auto it = cirGenRecordLayouts.find(key);
@@ -548,7 +572,7 @@ bool CIRGenTypes::isZeroInitializable(clang::QualType t) {
   }
 
   if (const RecordType *rt = t->getAs<RecordType>()) {
-    const RecordDecl *rd = rt->getDecl();
+    const RecordDecl *rd = rt->getOriginalDecl()->getDefinitionOrSelf();
     return isZeroInitializable(rd);
   }
 
@@ -623,8 +647,10 @@ void CIRGenTypes::updateCompletedType(const TagDecl *td) {
     // declaration of enums, and C doesn't allow an incomplete forward
     // declaration with a non-default type.
     assert(
-        !typeCache.count(ed->getTypeForDecl()) ||
-        (convertType(ed->getIntegerType()) == typeCache[ed->getTypeForDecl()]));
+        !typeCache.count(
+            ed->getASTContext().getCanonicalTagType(ed)->getTypePtr()) ||
+        (convertType(ed->getIntegerType()) ==
+         typeCache[ed->getASTContext().getCanonicalTagType(ed)->getTypePtr()]));
     // If necessary, provide the full definition of a type only used with a
     // declaration so far.
     assert(!cir::MissingFeatures::generateDebugInfo());
@@ -639,7 +665,7 @@ void CIRGenTypes::updateCompletedType(const TagDecl *td) {
 
   // Only complete if we converted it already. If we haven't converted it yet,
   // we'll just do it lazily.
-  if (recordDeclTypes.count(astContext.getTagDeclType(rd).getTypePtr()))
+  if (recordDeclTypes.count(astContext.getCanonicalTagType(rd).getTypePtr()))
     convertRecordDeclType(rd);
 
   // If necessary, provide the full definition of a type only used with a
diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.h b/clang/lib/CIR/CodeGen/CIRGenTypes.h
index c2813d79bf63b..7af0d956e7d56 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.h
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.h
@@ -130,6 +130,13 @@ class CIRGenTypes {
   /// Get the CIR function type for \arg Info.
   cir::FuncType getFunctionType(const CIRGenFunctionInfo &info);
 
+  cir::FuncType getFunctionType(clang::GlobalDecl gd);
+
+  /// Get the CIR function type for use in a vtable, given a CXXMethodDecl. If
+  /// the method has an incomplete return type, and/or incomplete argument
+  /// types, this will return the opaque type.
+  cir::FuncType getFunctionTypeForVTable(clang::GlobalDecl gd);
+
   // The arrangement methods are split into three families:
   //   - those meant to drive the signature and prologue/epilogue
   //     of a function declaration or definition,
diff --git a/clang/lib/CIR/CodeGen/CIRGenVTables.cpp b/clang/lib/CIR/CodeGen/CIRGenVTables.cpp
new file mode 100644
index 0000000000000..438b483ccb9c2
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenVTables.cpp
@@ -0,0 +1,249 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code dealing with C++ code generation of virtual tables.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CIRGenVTables.h"
+
+#include "CIRGenCXXABI.h"
+#include "CIRGenModule.h"
+#include "mlir/IR/Types.h"
+#include "clang/AST/VTableBuilder.h"
+#include "llvm/ADT/SmallVector.h"
+
+using namespace llvm;
+using namespace clang;
+using namespace clang::CIRGen;
+
+CIRGenVTables::CIRGenVTables(CIRGenModule &cgm)
+    : cgm(cgm), vtContext(cgm.getASTContext().getVTableContext()) {}
+
+mlir::Type CIRGenModule::getVTableComponentType() {
+  mlir::Type ptrTy = builder.getUInt8PtrTy();
+  assert(!cir::MissingFeatures::vtableRelativeLayout());
+  return ptrTy;
+}
+
+mlir::Type CIRGenVTables::getVTableComponentType() {
+  return cgm.getVTableComponentType();
+}
+
+cir::RecordType CIRGenVTables::getVTableType(const VTableLayout &layout) {
+  SmallVector<mlir::Type, 4> tys;
+  mlir::Type componentType = getVTableComponentType();
+  for (unsigned i = 0, e = layout.getNumVTables(); i != e; ++i)
+    tys.push_back(cir::ArrayType::get(componentType, layout.getVTableSize(i)));
+
+  // FIXME(cir): should VTableLayout be encoded like we do for some
+  // AST nodes?
+  return cgm.getBuilder().getAnonRecordTy(tys, /*incomplete=*/false);
+}
+
+/// This is a callback from Sema to tell us that a particular vtable is
+/// required to be emitted in this translation unit.
+///
+/// This is only called for vtables that _must_ be emitted (mainly due to key
+/// functions).  For weak vtables, CodeGen tracks when they are needed and
+/// emits them as-needed.
+void CIRGenModule::emitVTable(const CXXRecordDecl *rd) {
+  vtables.generateClassData(rd);
+}
+
+void CIRGenVTables::generateClassData(const CXXRecordDecl *rd) {
+  assert(!cir::MissingFeatures::generateDebugInfo());
+
+  if (rd->getNumVBases())
+    cgm.errorNYI(rd->getSourceRange(), "emitVirtualInheritanceTables");
+
+  cgm.getCXXABI().emitVTableDefinitions(*this, rd);
+}
+
+mlir::Attribute CIRGenVTables::getVTableComponent(
+    const VTableLayout &layout, unsigned componentIndex, mlir::Attribute rtti,
+    unsigned &nextVTableThunkIndex, unsigned vtableAddressPoint,
+    bool vtableHasLocalLinkage) {
+  const VTableComponent &component = layout.vtable_components()[componentIndex];
+
+  CIRGenBuilderTy builder = cgm.getBuilder();
+
+  assert(!cir::MissingFeatures::vtableRelativeLayout());
+
+  switch (component.getKind()) {
+  case VTableComponent::CK_VCallOffset:
+    cgm.errorNYI("getVTableComponent: VCallOffset");
+    return mlir::Attribute();
+  case VTableComponent::CK_VBaseOffset:
+    cgm.errorNYI("getVTableComponent: VBaseOffset");
+    return mlir::Attribute();
+  case VTableComponent::CK_CompleteDtorPointer:
+    cgm.errorNYI("getVTableComponent: CompleteDtorPointer");
+    return mlir::Attribute();
+  case VTableComponent::CK_DeletingDtorPointer:
+    cgm.errorNYI("getVTableComponent: DeletingDtorPointer");
+    return mlir::Attribute();
+  case VTableComponent::CK_UnusedFunctionPointer:
+    cgm.errorNYI("getVTableComponent: UnusedFunctionPointer");
+    return mlir::Attribute();
+
+  case VTableComponent::CK_OffsetToTop:
+    return builder.getConstPtrAttr(builder.getUInt8PtrTy(),
+                                   component.getOffsetToTop().getQuantity());
+
+  case VTableComponent::CK_RTTI:
+    assert((mlir::isa<cir::GlobalViewAttr>(rtti) ||
+            mlir::isa<cir::ConstPtrAttr>(rtti)) &&
+           "expected GlobalViewAttr or ConstPtrAttr");
+    return rtti;
+
+  case VTableComponent::CK_FunctionPointer: {
+    GlobalDecl gd = component.getGlobalDecl();
+
+    assert(!cir::MissingFeatures::cudaSupport());
+
+    cir::FuncOp fnPtr;
+    if (cast<CXXMethodDecl>(gd.getDecl())->isPureVirtual()) {
+      cgm.errorNYI("getVTableComponent: CK_FunctionPointer: pure virtual");
+      return mlir::Attribute();
+    } else if (cast<CXXMethodDecl>(gd.getDecl())->isDeleted()) {
+      cgm.errorNYI("getVTableComponent: CK_FunctionPointer: deleted virtual");
+      return mlir::Attribute();
+    } else if (nextVTableThunkIndex < layout.vtable_thunks().size() &&
+               layout.vtable_thunks()[nextVTableThunkIndex].first ==
+                   componentIndex) {
+      cgm.errorNYI("getVTableComponent: CK_FunctionPointer: thunk");
+      return mlir::Attribute();
+    } else {
+      // Otherwise we can use the method definition directly.
+      cir::FuncType fnTy = cgm.getTypes().getFunctionTypeForVTable(gd);
+      fnPtr = cgm.getAddrOfFunction(gd, fnTy, /*ForVTable=*/true);
+    }
+
+    return cir::GlobalViewAttr::get(
+        builder.getUInt8PtrTy(),
+        mlir::FlatSymbolRefAttr::get(fnPtr.getSymNameAttr()));
+  }
+  }
+
+  llvm_unreachable("Unexpected vtable component kind");
+}
+
+void CIRGenVTables::createVTableInitializer(cir::GlobalOp &vtableOp,
+                                            const clang::VTableLayout &layout,
+                                            mlir::Attribute rtti,
+                                            bool vtableHasLocalLinkage) {
+  mlir::Type componentType = getVTableComponentType();
+
+  const llvm::SmallVectorImpl<unsigned> &addressPoints =
+      layout.getAddressPointIndices();
+  unsigned nextVTableThunkIndex = 0;
+
+  if (layout.getNumVTables() > 1)
+    cgm.errorNYI("emitVTableDefinitions: multiple vtables");
+
+  // We'll need a loop here to handle multiple vtables, but for now we only
+  // support one.
+  unsigned vtableIndex = 0;
+  size_t vtableStart = layout.getVTableOffset(vtableIndex);
+  size_t vtableEnd = vtableStart + layout.getVTableSize(vtableIndex);
+
+  // Build a ConstArrayAttr of the vtable components.
+  llvm::SmallVector<mlir::Attribute> components;
+  for (size_t componentIndex = vtableStart; componentIndex < vtableEnd;
+       ++componentIndex) {
+    components.push_back(
+        getVTableComponent(layout, componentIndex, rtti, nextVTableThunkIndex,
+                           addressPoints[vtableIndex], vtableHasLocalLinkage));
+  }
+
+  mlir::MLIRContext *mlirContext = &cgm.getMLIRContext();
+
+  // Create a ConstArrayAttr to hold the components.
+  auto arr = cir::ConstArrayAttr::get(
+      cir::ArrayType::get(componentType, components.size()),
+      mlir::ArrayAttr::get(mlirContext, components));
+
+  // Create a ConstRecordAttr to hold the component array.
+  const auto members = mlir::ArrayAttr::get(mlirContext, {arr});
+  cir::ConstRecordAttr record = cgm.getBuilder().getAnonConstRecord(members);
+
+  // Create a VTableAttr
+  auto vtableAttr = cir::VTableAttr::get(record.getType(), record.getMembers());
+
+  // Add the vtable initializer to the vtable global op.
+  cgm.setInitializer(vtableOp, vtableAttr);
+}
+
+/// Compute the required linkage of the vtable for the given class.
+///
+/// Note that we only call this at the end of the translation unit.
+cir::GlobalLinkageKind CIRGenModule::getVTableLinkage(const CXXRecordDecl *rd) {
+  if (!rd->isExternallyVisible())
+    return cir::GlobalLinkageKind::InternalLinkage;
+
+  // We're at the end of the translation unit, so the current key
+  // function is fully correct.
+  const CXXMethodDecl *keyFunction = astContext.getCurrentKeyFunction(rd);
+  if (keyFunction && !rd->hasAttr<DLLImportAttr>()) {
+    // If this class has a key function, use that to determine the
+    // linkage of the vtable.
+    const FunctionDecl *def = nullptr;
+    if (keyFunction->hasBody(def))
+      keyFunction = cast<CXXMethodDecl>(def);
+
+    // All of the cases below do something different with AppleKext enabled.
+    assert(!cir::MissingFeatures::appleKext());
+    switch (keyFunction->getTemplateSpecializationKind()) {
+    case TSK_Undeclared:
+    case TSK_ExplicitSpecialization:
+      assert(
+          (def || codeGenOpts.OptimizationLevel > 0 ||
+           codeGenOpts.getDebugInfo() != llvm::codegenoptions::NoDebugInfo) &&
+          "Shouldn't query vtable linkage without key function, "
+          "optimizations, or debug info");
+      if (!def && codeGenOpts.OptimizationLevel > 0)
+        return cir::GlobalLinkageKind::AvailableExternallyLinkage;
+
+      if (keyFunction->isInlined())
+        return !astContext.getLangOpts().AppleKext
+                   ? cir::GlobalLinkageKind::LinkOnceODRLinkage
+                   : cir::GlobalLinkageKind::InternalLinkage;
+      return cir::GlobalLinkageKind::ExternalLinkage;
+
+    case TSK_ImplicitInstantiation:
+      return cir::GlobalLinkageKind::LinkOnceODRLinkage;
+
+    case TSK_ExplicitInstantiationDefinition:
+      return cir::GlobalLinkageKind::WeakODRLinkage;
+
+    case TSK_ExplicitInstantiationDeclaration:
+      llvm_unreachable("Should not have been asked to emit this");
+    }
+  }
+
+  errorNYI(rd->getSourceRange(), "getVTableLinkage: no key function");
+  return cir::GlobalLinkageKind::ExternalLinkage;
+}
+
+void CIRGenVTables::emitThunks(GlobalDecl gd) {
+  const CXXMethodDecl *md =
+      cast<CXXMethodDecl>(gd.getDecl())->getCanonicalDecl();
+
+  // We don't need to generate thunks for the base destructor.
+  if (isa<CXXDestructorDecl>(md) && gd.getDtorType() == Dtor_Base)
+    return;
+
+  const VTableContextBase::ThunkInfoVectorTy *thunkInfoVector =
+      vtContext->getThunkInfo(gd);
+
+  if (!thunkInfoVector)
+    return;
+
+  cgm.errorNYI(md->getSourceRange(), "emitThunks");
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenVTables.h b/clang/lib/CIR/CodeGen/CIRGenVTables.h
new file mode 100644
index 0000000000000..518d7d78f1737
--- /dev/null
+++ b/clang/lib/CIR/CodeGen/CIRGenVTables.h
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This contains code dealing with C++ code generation of virtual tables.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef CLANG_LIB_CIR_CODEGEN_CIRGENVTABLES_H
+#define CLANG_LIB_CIR_CODEGEN_CIRGENVTABLES_H
+
+#include "mlir/IR/Types.h"
+#include "clang/AST/GlobalDecl.h"
+#include "clang/AST/VTableBuilder.h"
+#include "clang/CIR/Dialect/IR/CIRDialect.h"
+
+namespace clang {
+class CXXRecordDecl;
+}
+
+namespace clang::CIRGen {
+class CIRGenModule;
+
+class CIRGenVTables {
+  CIRGenModule &cgm;
+
+  clang::VTableContextBase *vtContext;
+
+  mlir::Attribute
+  getVTableComponent(const VTableLayout &layout, unsigned componentIndex,
+                     mlir::Attribute rtti, unsigned &nextVTableThunkIndex,
+                     unsigned vtableAddressPoint, bool vtableHasLocalLinkage);
+
+  mlir::Type getVTableComponentType();
+
+public:
+  CIRGenVTables(CIRGenModule &cgm);
+
+  /// Add vtable components for the given vtable layout to the given
+  /// global initializer.
+  void createVTableInitializer(cir::GlobalOp &vtable,
+                               const clang::VTableLayout &layout,
+                               mlir::Attribute rtti,
+                               bool vtableHasLocalLinkage);
+
+  clang::ItaniumVTableContext &getItaniumVTableContext() {
+    return *llvm::cast<clang::ItaniumVTableContext>(vtContext);
+  }
+
+  const clang::ItaniumVTableContext &getItaniumVTableContext() const {
+    return *llvm::cast<clang::ItaniumVTableContext>(vtContext);
+  }
+
+  /// Emit the associated thunks for the given global decl.
+  void emitThunks(GlobalDecl gd);
+
+  /// Generate all the class data required to be generated upon definition of a
+  /// KeyFunction. This includes the vtable, the RTTI data structure (if RTTI
+  /// is enabled) and the VTT (if the class has virtual bases).
+  void generateClassData(const CXXRecordDecl *rd);
+
+  /// Returns the type of a vtable with the given layout. Normally a struct of
+  /// arrays of pointers, with one struct element for each vtable in the vtable
+  /// group.
+  cir::RecordType getVTableType(const clang::VTableLayout &layout);
+};
+
+} // namespace clang::CIRGen
+
+#endif // CLANG_LIB_CIR_CODEGEN_CIRGENVTABLES_H
diff --git a/clang/lib/CIR/CodeGen/CIRGenValue.h b/clang/lib/CIR/CodeGen/CIRGenValue.h
index 0832c4141a10f..ac7e1cc1a1db6 100644
--- a/clang/lib/CIR/CodeGen/CIRGenValue.h
+++ b/clang/lib/CIR/CodeGen/CIRGenValue.h
@@ -190,6 +190,7 @@ class LValue {
   bool isSimple() const { return lvType == Simple; }
   bool isVectorElt() const { return lvType == VectorElt; }
   bool isBitField() const { return lvType == BitField; }
+  bool isGlobalReg() const { return lvType == GlobalReg; }
   bool isVolatile() const { return quals.hasVolatile(); }
 
   bool isVolatileQualified() const { return quals.hasVolatile(); }
@@ -211,6 +212,14 @@ class LValue {
     return Address(getPointer(), elementType, getAlignment());
   }
 
+  void setAddress(Address address) {
+    assert(isSimple());
+    v = address.getPointer();
+    elementType = address.getElementType();
+    alignment = address.getAlignment().getQuantity();
+    assert(!cir::MissingFeatures::addressIsKnownNonNull());
+  }
+
   const clang::Qualifiers &getQuals() const { return quals; }
   clang::Qualifiers &getQuals() { return quals; }
 
diff --git a/clang/lib/CIR/CodeGen/CIRGenerator.cpp b/clang/lib/CIR/CodeGen/CIRGenerator.cpp
index b0357d9d3b7fa..aa4d9eba35c04 100644
--- a/clang/lib/CIR/CodeGen/CIRGenerator.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenerator.cpp
@@ -163,7 +163,7 @@ void CIRGenerator::HandleCXXStaticMemberVarInstantiation(VarDecl *D) {
   if (diags.hasErrorOccurred())
     return;
 
-  cgm->errorNYI(D->getSourceRange(), "HandleCXXStaticMemberVarInstantiation");
+  cgm->handleCXXStaticMemberVarInstantiation(D);
 }
 
 void CIRGenerator::CompleteTentativeDefinition(VarDecl *d) {
@@ -177,5 +177,5 @@ void CIRGenerator::HandleVTable(CXXRecordDecl *rd) {
   if (diags.hasErrorOccurred())
     return;
 
-  cgm->errorNYI(rd->getSourceRange(), "HandleVTable");
+  cgm->emitVTable(rd);
 }
diff --git a/clang/lib/CIR/CodeGen/CMakeLists.txt b/clang/lib/CIR/CodeGen/CMakeLists.txt
index ca3a329d0c56d..7366446a33c6e 100644
--- a/clang/lib/CIR/CodeGen/CMakeLists.txt
+++ b/clang/lib/CIR/CodeGen/CMakeLists.txt
@@ -8,6 +8,8 @@ get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 
 add_clang_library(clangCIR
   CIRGenerator.cpp
+  CIRGenAsm.cpp
+  CIRGenAtomic.cpp
   CIRGenBuilder.cpp
   CIRGenCall.cpp
   CIRGenClass.cpp
@@ -22,6 +24,7 @@ add_clang_library(clangCIR
   CIRGenExprAggregate.cpp
   CIRGenExprComplex.cpp
   CIRGenExprConstant.cpp
+  CIRGenExprCXX.cpp
   CIRGenExprScalar.cpp
   CIRGenFunction.cpp
   CIRGenItaniumCXXABI.cpp
@@ -33,6 +36,7 @@ add_clang_library(clangCIR
   CIRGenStmtOpenACC.cpp
   CIRGenStmtOpenACCLoop.cpp
   CIRGenTypes.cpp
+  CIRGenVTables.cpp
   TargetInfo.cpp
 
   DEPENDS
diff --git a/clang/lib/CIR/CodeGen/EHScopeStack.h b/clang/lib/CIR/CodeGen/EHScopeStack.h
index 22750aca3c4fc..47478f6cf4bb3 100644
--- a/clang/lib/CIR/CodeGen/EHScopeStack.h
+++ b/clang/lib/CIR/CodeGen/EHScopeStack.h
@@ -42,7 +42,47 @@ enum CleanupKind : unsigned {
 /// A stack of scopes which respond to exceptions, including cleanups
 /// and catch blocks.
 class EHScopeStack {
+  friend class CIRGenFunction;
+
 public:
+  // TODO(ogcg): Switch to alignof(uint64_t) instead of 8
+  enum { ScopeStackAlignment = 8 };
+
+  /// A saved depth on the scope stack.  This is necessary because
+  /// pushing scopes onto the stack invalidates iterators.
+  class stable_iterator {
+    friend class EHScopeStack;
+
+    /// Offset from startOfData to endOfBuffer.
+    ptrdiff_t size = -1;
+
+    explicit stable_iterator(ptrdiff_t size) : size(size) {}
+
+  public:
+    static stable_iterator invalid() { return stable_iterator(-1); }
+    stable_iterator() = default;
+
+    bool isValid() const { return size >= 0; }
+
+    /// Returns true if this scope encloses I.
+    /// Returns false if I is invalid.
+    /// This scope must be valid.
+    bool encloses(stable_iterator other) const { return size <= other.size; }
+
+    /// Returns true if this scope strictly encloses I: that is,
+    /// if it encloses I and is not I.
+    /// Returns false is I is invalid.
+    /// This scope must be valid.
+    bool strictlyEncloses(stable_iterator I) const { return size < I.size; }
+
+    friend bool operator==(stable_iterator A, stable_iterator B) {
+      return A.size == B.size;
+    }
+    friend bool operator!=(stable_iterator A, stable_iterator B) {
+      return A.size != B.size;
+    }
+  };
+
   /// Information for lazily generating a cleanup.  Subclasses must be
   /// POD-like: cleanups will not be destructed, and they will be
   /// allocated on the cleanup stack and freely copied and moved
@@ -68,30 +108,75 @@ class EHScopeStack {
     ///
     // \param flags cleanup kind.
     virtual void emit(CIRGenFunction &cgf) = 0;
-  };
 
-  // Classic codegen has a finely tuned custom allocator and a complex stack
-  // management scheme. We'll probably eventually want to find a way to share
-  // that implementation. For now, we will use a very simplified implementation
-  // to get cleanups working.
-  llvm::SmallVector<std::unique_ptr<Cleanup>, 8> cleanupStack;
+    // This is a placeholder until EHScope is implemented.
+    virtual size_t getSize() const = 0;
+  };
 
 private:
+  // The implementation for this class is in CIRGenCleanup.h and
+  // CIRGenCleanup.cpp; the definition is here because it's used as a
+  // member of CIRGenFunction.
+
+  /// The start of the scope-stack buffer, i.e. the allocated pointer
+  /// for the buffer.  All of these pointers are either simultaneously
+  /// null or simultaneously valid.
+  std::unique_ptr<char[]> startOfBuffer;
+
+  /// The end of the buffer.
+  char *endOfBuffer = nullptr;
+
+  /// The first valid entry in the buffer.
+  char *startOfData = nullptr;
+
   /// The CGF this Stack belong to
   CIRGenFunction *cgf = nullptr;
 
+  // This class uses a custom allocator for maximum efficiency because cleanups
+  // are allocated and freed very frequently. It's basically a bump pointer
+  // allocator, but we can't use LLVM's BumpPtrAllocator because we use offsets
+  // into the buffer as stable iterators.
+  char *allocate(size_t size);
+  void deallocate(size_t size);
+
+  void *pushCleanup(CleanupKind kind, size_t dataSize);
+
 public:
   EHScopeStack() = default;
   ~EHScopeStack() = default;
 
   /// Push a lazily-created cleanup on the stack.
   template <class T, class... As> void pushCleanup(CleanupKind kind, As... a) {
-    cleanupStack.push_back(std::make_unique<T>(a...));
+    static_assert(alignof(T) <= ScopeStackAlignment,
+                  "Cleanup's alignment is too large.");
+    void *buffer = pushCleanup(kind, sizeof(T));
+    [[maybe_unused]] Cleanup *obj = new (buffer) T(a...);
   }
 
   void setCGF(CIRGenFunction *inCGF) { cgf = inCGF; }
 
-  size_t getStackDepth() const { return cleanupStack.size(); }
+  /// Pops a cleanup scope off the stack.  This is private to CIRGenCleanup.cpp.
+  void popCleanup();
+
+  /// Determines whether the exception-scopes stack is empty.
+  bool empty() const { return startOfData == endOfBuffer; }
+
+  /// An unstable reference to a scope-stack depth.  Invalidated by
+  /// pushes but not pops.
+  class iterator;
+
+  /// Returns an iterator pointing to the innermost EH scope.
+  iterator begin() const;
+
+  /// Create a stable reference to the top of the EH stack.  The
+  /// returned reference is valid until that scope is popped off the
+  /// stack.
+  stable_iterator stable_begin() const {
+    return stable_iterator(endOfBuffer - startOfData);
+  }
+
+  /// Create a stable reference to the bottom of the EH stack.
+  static stable_iterator stable_end() { return stable_iterator(0); }
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/TargetInfo.cpp b/clang/lib/CIR/CodeGen/TargetInfo.cpp
index d2d32bbd9403c..7b6259b04122d 100644
--- a/clang/lib/CIR/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CIR/CodeGen/TargetInfo.cpp
@@ -10,20 +10,20 @@ bool clang::CIRGen::isEmptyRecordForLayout(const ASTContext &context,
   if (!rt)
     return false;
 
-  const RecordDecl *rd = rt->getDecl();
+  const RecordDecl *rd = rt->getOriginalDecl()->getDefinitionOrSelf();
 
   // If this is a C++ record, check the bases first.
   if (const CXXRecordDecl *cxxrd = dyn_cast<CXXRecordDecl>(rd)) {
     if (cxxrd->isDynamicClass())
       return false;
 
-    for (const auto &I : cxxrd->bases())
-      if (!isEmptyRecordForLayout(context, I.getType()))
+    for (const auto &i : cxxrd->bases())
+      if (!isEmptyRecordForLayout(context, i.getType()))
         return false;
   }
 
-  for (const auto *I : rd->fields())
-    if (!isEmptyFieldForLayout(context, I))
+  for (const auto *i : rd->fields())
+    if (!isEmptyFieldForLayout(context, i))
       return false;
 
   return true;
diff --git a/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp b/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
index c039bdcd7f6a1..95faad6746955 100644
--- a/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRAttrs.cpp
@@ -15,6 +15,14 @@
 #include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
 
+//===-----------------------------------------------------------------===//
+// RecordMembers
+//===-----------------------------------------------------------------===//
+
+static void printRecordMembers(mlir::AsmPrinter &p, mlir::ArrayAttr members);
+static mlir::ParseResult parseRecordMembers(mlir::AsmParser &parser,
+                                            mlir::ArrayAttr &members);
+
 //===-----------------------------------------------------------------===//
 // IntLiteral
 //===-----------------------------------------------------------------===//
@@ -68,6 +76,61 @@ void CIRDialect::printAttribute(Attribute attr, DialectAsmPrinter &os) const {
     llvm_unreachable("unexpected CIR type kind");
 }
 
+static void printRecordMembers(mlir::AsmPrinter &printer,
+                               mlir::ArrayAttr members) {
+  printer << '{';
+  llvm::interleaveComma(members, printer);
+  printer << '}';
+}
+
+static ParseResult parseRecordMembers(mlir::AsmParser &parser,
+                                      mlir::ArrayAttr &members) {
+  llvm::SmallVector<mlir::Attribute, 4> elts;
+
+  auto delimiter = AsmParser::Delimiter::Braces;
+  auto result = parser.parseCommaSeparatedList(delimiter, [&]() {
+    mlir::TypedAttr attr;
+    if (parser.parseAttribute(attr).failed())
+      return mlir::failure();
+    elts.push_back(attr);
+    return mlir::success();
+  });
+
+  if (result.failed())
+    return mlir::failure();
+
+  members = mlir::ArrayAttr::get(parser.getContext(), elts);
+  return mlir::success();
+}
+
+//===----------------------------------------------------------------------===//
+// ConstRecordAttr definitions
+//===----------------------------------------------------------------------===//
+
+LogicalResult
+ConstRecordAttr::verify(function_ref<InFlightDiagnostic()> emitError,
+                        mlir::Type type, ArrayAttr members) {
+  auto sTy = mlir::dyn_cast_if_present<cir::RecordType>(type);
+  if (!sTy)
+    return emitError() << "expected !cir.record type";
+
+  if (sTy.getMembers().size() != members.size())
+    return emitError() << "number of elements must match";
+
+  unsigned attrIdx = 0;
+  for (auto &member : sTy.getMembers()) {
+    auto m = mlir::cast<mlir::TypedAttr>(members[attrIdx]);
+    if (member != m.getType())
+      return emitError() << "element at index " << attrIdx << " has type "
+                         << m.getType()
+                         << " but the expected type for this element is "
+                         << member;
+    attrIdx++;
+  }
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // OptInfoAttr definitions
 //===----------------------------------------------------------------------===//
@@ -361,6 +424,44 @@ cir::ConstVectorAttr::verify(function_ref<InFlightDiagnostic()> emitError,
   return elementTypeCheck;
 }
 
+//===----------------------------------------------------------------------===//
+// CIR VTableAttr
+//===----------------------------------------------------------------------===//
+
+LogicalResult cir::VTableAttr::verify(
+    llvm::function_ref<mlir::InFlightDiagnostic()> emitError, mlir::Type type,
+    mlir::ArrayAttr data) {
+  auto sTy = mlir::dyn_cast_if_present<cir::RecordType>(type);
+  if (!sTy)
+    return emitError() << "expected !cir.record type result";
+  if (sTy.getMembers().empty() || data.empty())
+    return emitError() << "expected record type with one or more subtype";
+
+  if (cir::ConstRecordAttr::verify(emitError, type, data).failed())
+    return failure();
+
+  for (const auto &element : data.getAsRange<mlir::Attribute>()) {
+    const auto &constArrayAttr = mlir::dyn_cast<cir::ConstArrayAttr>(element);
+    if (!constArrayAttr)
+      return emitError() << "expected constant array subtype";
+
+    LogicalResult eltTypeCheck = success();
+    auto arrayElts = mlir::cast<ArrayAttr>(constArrayAttr.getElts());
+    arrayElts.walkImmediateSubElements(
+        [&](mlir::Attribute attr) {
+          if (mlir::isa<ConstPtrAttr, GlobalViewAttr>(attr))
+            return;
+
+          eltTypeCheck = emitError()
+                         << "expected GlobalViewAttr or ConstPtrAttr";
+        },
+        [&](mlir::Type type) {});
+    if (eltTypeCheck.failed())
+      return eltTypeCheck;
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // CIR Dialect
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/Dialect/IR/CIRDataLayout.cpp b/clang/lib/CIR/Dialect/IR/CIRDataLayout.cpp
index d835c4076224a..42d45819de0f3 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDataLayout.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDataLayout.cpp
@@ -1,4 +1,6 @@
 #include "clang/CIR/Dialect/IR/CIRDataLayout.h"
+#include "clang/CIR/Dialect/IR/CIRTypes.h"
+#include "clang/CIR/MissingFeatures.h"
 
 using namespace cir;
 
@@ -20,3 +22,42 @@ void CIRDataLayout::reset(mlir::DataLayoutSpecInterface spec) {
         bigEndian = str == mlir::DLTIDialect::kDataLayoutEndiannessBig;
   }
 }
+
+llvm::Align CIRDataLayout::getAlignment(mlir::Type ty, bool useABIAlign) const {
+  if (auto recTy = llvm::dyn_cast<cir::RecordType>(ty)) {
+    // Packed record types always have an ABI alignment of one.
+    if (recTy && recTy.getPacked() && useABIAlign)
+      return llvm::Align(1);
+
+    // Get the layout annotation... which is lazily created on demand.
+    llvm_unreachable("getAlignment()) for record type is not implemented");
+  }
+
+  // FIXME(cir): This does not account for differnt address spaces, and relies
+  // on CIR's data layout to give the proper alignment.
+  assert(!cir::MissingFeatures::addressSpace());
+
+  // Fetch type alignment from MLIR's data layout.
+  unsigned align = useABIAlign ? layout.getTypeABIAlignment(ty)
+                               : layout.getTypePreferredAlignment(ty);
+  return llvm::Align(align);
+}
+
+// The implementation of this method is provided inline as it is particularly
+// well suited to constant folding when called on a specific Type subclass.
+llvm::TypeSize CIRDataLayout::getTypeSizeInBits(mlir::Type ty) const {
+  assert(cir::isSized(ty) && "Cannot getTypeInfo() on a type that is unsized!");
+
+  if (auto recordTy = llvm::dyn_cast<cir::RecordType>(ty)) {
+    // FIXME(cir): CIR record's data layout implementation doesn't do a good job
+    // of handling unions particularities. We should have a separate union type.
+    return recordTy.getTypeSizeInBits(layout, {});
+  }
+
+  // FIXME(cir): This does not account for different address spaces, and relies
+  // on CIR's data layout to give the proper ABI-specific type width.
+  assert(!cir::MissingFeatures::addressSpace());
+
+  // This is calling mlir::DataLayout::getTypeSizeInBits().
+  return llvm::TypeSize::getFixed(layout.getTypeSizeInBits(ty));
+}
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index d3fcac1edeb5a..8c53939e89d01 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -22,6 +22,8 @@
 #include "clang/CIR/Dialect/IR/CIROpsDialect.cpp.inc"
 #include "clang/CIR/Dialect/IR/CIROpsEnums.cpp.inc"
 #include "clang/CIR/MissingFeatures.h"
+#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/Support/LogicalResult.h"
 
 #include <numeric>
@@ -339,7 +341,9 @@ static LogicalResult checkConstantTypes(mlir::Operation *op, mlir::Type opType,
   }
 
   if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr,
-                cir::ConstComplexAttr, cir::PoisonAttr>(attrType))
+                cir::ConstComplexAttr, cir::ConstRecordAttr,
+                cir::GlobalViewAttr, cir::PoisonAttr, cir::VTableAttr>(
+          attrType))
     return success();
 
   assert(isa<TypedAttr>(attrType) && "What else could we be looking at here?");
@@ -1355,11 +1359,14 @@ mlir::LogicalResult cir::GlobalOp::verify() {
 
 void cir::GlobalOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                           llvm::StringRef sym_name, mlir::Type sym_type,
-                          cir::GlobalLinkageKind linkage) {
+                          bool isConstant, cir::GlobalLinkageKind linkage) {
   odsState.addAttribute(getSymNameAttrName(odsState.name),
                         odsBuilder.getStringAttr(sym_name));
   odsState.addAttribute(getSymTypeAttrName(odsState.name),
                         mlir::TypeAttr::get(sym_type));
+  if (isConstant)
+    odsState.addAttribute(getConstantAttrName(odsState.name),
+                          odsBuilder.getUnitAttr());
 
   cir::GlobalLinkageKindAttr linkageAttr =
       cir::GlobalLinkageKindAttr::get(odsBuilder.getContext(), linkage);
@@ -1443,6 +1450,30 @@ cir::GetGlobalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// VTableAddrPointOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult
+cir::VTableAddrPointOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
+  StringRef name = getName();
+
+  // Verify that the result type underlying pointer type matches the type of
+  // the referenced cir.global.
+  auto op =
+      symbolTable.lookupNearestSymbolFrom<cir::GlobalOp>(*this, getNameAttr());
+  if (!op)
+    return emitOpError("'")
+           << name << "' does not reference a valid cir.global";
+  std::optional<mlir::Attribute> init = op.getInitialValue();
+  if (!init)
+    return success();
+  if (!isa<cir::VTableAttr>(*init))
+    return emitOpError("Expected #cir.vtable in initializer for global '")
+           << name << "'";
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // FuncOp
 //===----------------------------------------------------------------------===//
@@ -1625,9 +1656,28 @@ void cir::FuncOp::print(OpAsmPrinter &p) {
   }
 }
 
-// TODO(CIR): The properties of functions that require verification haven't
-// been implemented yet.
-mlir::LogicalResult cir::FuncOp::verify() { return success(); }
+mlir::LogicalResult cir::FuncOp::verify() {
+
+  llvm::SmallSet<llvm::StringRef, 16> labels;
+  llvm::SmallSet<llvm::StringRef, 16> gotos;
+
+  getOperation()->walk([&](mlir::Operation *op) {
+    if (auto lab = dyn_cast<cir::LabelOp>(op)) {
+      labels.insert(lab.getLabel());
+    } else if (auto goTo = dyn_cast<cir::GotoOp>(op)) {
+      gotos.insert(goTo.getLabel());
+    }
+  });
+
+  if (!labels.empty() || !gotos.empty()) {
+    llvm::SmallSet<llvm::StringRef, 16> mismatched =
+        llvm::set_difference(gotos, labels);
+
+    if (!mismatched.empty())
+      return emitOpError() << "goto/label mismatch";
+  }
+  return success();
+}
 
 //===----------------------------------------------------------------------===//
 // BinOp
@@ -1762,6 +1812,19 @@ LogicalResult cir::ShiftOp::verify() {
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// LabelOp Definitions
+//===----------------------------------------------------------------------===//
+
+LogicalResult cir::LabelOp::verify() {
+  mlir::Operation *op = getOperation();
+  mlir::Block *blk = op->getBlock();
+  if (&blk->front() != op)
+    return emitError() << "must be the first operation in a block";
+
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // UnaryOp
 //===----------------------------------------------------------------------===//
@@ -2384,6 +2447,209 @@ OpFoldResult RotateOp::fold(FoldAdaptor adaptor) {
   return IntAttr::get(input.getContext(), input.getType(), resultValue);
 }
 
+//===----------------------------------------------------------------------===//
+// InlineAsmOp
+//===----------------------------------------------------------------------===//
+
+void cir::InlineAsmOp::print(OpAsmPrinter &p) {
+  p << '(' << getAsmFlavor() << ", ";
+  p.increaseIndent();
+  p.printNewline();
+
+  llvm::SmallVector<std::string, 3> names{"out", "in", "in_out"};
+  auto *nameIt = names.begin();
+  auto *attrIt = getOperandAttrs().begin();
+
+  for (mlir::OperandRange ops : getAsmOperands()) {
+    p << *nameIt << " = ";
+
+    p << '[';
+    llvm::interleaveComma(llvm::make_range(ops.begin(), ops.end()), p,
+                          [&](Value value) {
+                            p.printOperand(value);
+                            p << " : " << value.getType();
+                            if (*attrIt)
+                              p << " (maybe_memory)";
+                            attrIt++;
+                          });
+    p << "],";
+    p.printNewline();
+    ++nameIt;
+  }
+
+  p << "{";
+  p.printString(getAsmString());
+  p << " ";
+  p.printString(getConstraints());
+  p << "}";
+  p.decreaseIndent();
+  p << ')';
+  if (getSideEffects())
+    p << " side_effects";
+
+  std::array elidedAttrs{
+      llvm::StringRef("asm_flavor"),        llvm::StringRef("asm_string"),
+      llvm::StringRef("constraints"),       llvm::StringRef("operand_attrs"),
+      llvm::StringRef("operands_segments"), llvm::StringRef("side_effects")};
+  p.printOptionalAttrDict(getOperation()->getAttrs(), elidedAttrs);
+
+  if (auto v = getRes())
+    p << " -> " << v.getType();
+}
+
+void cir::InlineAsmOp::build(OpBuilder &odsBuilder, OperationState &odsState,
+                             ArrayRef<ValueRange> asmOperands,
+                             StringRef asmString, StringRef constraints,
+                             bool sideEffects, cir::AsmFlavor asmFlavor,
+                             ArrayRef<Attribute> operandAttrs) {
+  // Set up the operands_segments for VariadicOfVariadic
+  SmallVector<int32_t> segments;
+  for (auto operandRange : asmOperands) {
+    segments.push_back(operandRange.size());
+    odsState.addOperands(operandRange);
+  }
+
+  odsState.addAttribute(
+      "operands_segments",
+      DenseI32ArrayAttr::get(odsBuilder.getContext(), segments));
+  odsState.addAttribute("asm_string", odsBuilder.getStringAttr(asmString));
+  odsState.addAttribute("constraints", odsBuilder.getStringAttr(constraints));
+  odsState.addAttribute("asm_flavor",
+                        AsmFlavorAttr::get(odsBuilder.getContext(), asmFlavor));
+
+  if (sideEffects)
+    odsState.addAttribute("side_effects", odsBuilder.getUnitAttr());
+
+  odsState.addAttribute("operand_attrs", odsBuilder.getArrayAttr(operandAttrs));
+}
+
+ParseResult cir::InlineAsmOp::parse(OpAsmParser &parser,
+                                    OperationState &result) {
+  llvm::SmallVector<mlir::Attribute> operandAttrs;
+  llvm::SmallVector<int32_t> operandsGroupSizes;
+  std::string asmString, constraints;
+  Type resType;
+  MLIRContext *ctxt = parser.getBuilder().getContext();
+
+  auto error = [&](const Twine &msg) -> LogicalResult {
+    return parser.emitError(parser.getCurrentLocation(), msg);
+  };
+
+  auto expected = [&](const std::string &c) {
+    return error("expected '" + c + "'");
+  };
+
+  if (parser.parseLParen().failed())
+    return expected("(");
+
+  auto flavor = FieldParser<AsmFlavor, AsmFlavor>::parse(parser);
+  if (failed(flavor))
+    return error("Unknown AsmFlavor");
+
+  if (parser.parseComma().failed())
+    return expected(",");
+
+  auto parseValue = [&](Value &v) {
+    OpAsmParser::UnresolvedOperand op;
+
+    if (parser.parseOperand(op) || parser.parseColon())
+      return error("can't parse operand");
+
+    Type typ;
+    if (parser.parseType(typ).failed())
+      return error("can't parse operand type");
+    llvm::SmallVector<mlir::Value> tmp;
+    if (parser.resolveOperand(op, typ, tmp))
+      return error("can't resolve operand");
+    v = tmp[0];
+    return mlir::success();
+  };
+
+  auto parseOperands = [&](llvm::StringRef name) {
+    if (parser.parseKeyword(name).failed())
+      return error("expected " + name + " operands here");
+    if (parser.parseEqual().failed())
+      return expected("=");
+    if (parser.parseLSquare().failed())
+      return expected("[");
+
+    int size = 0;
+    if (parser.parseOptionalRSquare().succeeded()) {
+      operandsGroupSizes.push_back(size);
+      if (parser.parseComma())
+        return expected(",");
+      return mlir::success();
+    }
+
+    auto parseOperand = [&]() {
+      Value val;
+      if (parseValue(val).succeeded()) {
+        result.operands.push_back(val);
+        size++;
+
+        if (parser.parseOptionalLParen().failed()) {
+          operandAttrs.push_back(mlir::Attribute());
+          return mlir::success();
+        }
+
+        if (parser.parseKeyword("maybe_memory").succeeded()) {
+          operandAttrs.push_back(mlir::UnitAttr::get(ctxt));
+          if (parser.parseRParen())
+            return expected(")");
+          return mlir::success();
+        } else {
+          return expected("maybe_memory");
+        }
+      }
+      return mlir::failure();
+    };
+
+    if (parser.parseCommaSeparatedList(parseOperand).failed())
+      return mlir::failure();
+
+    if (parser.parseRSquare().failed() || parser.parseComma().failed())
+      return expected("]");
+    operandsGroupSizes.push_back(size);
+    return mlir::success();
+  };
+
+  if (parseOperands("out").failed() || parseOperands("in").failed() ||
+      parseOperands("in_out").failed())
+    return error("failed to parse operands");
+
+  if (parser.parseLBrace())
+    return expected("{");
+  if (parser.parseString(&asmString))
+    return error("asm string parsing failed");
+  if (parser.parseString(&constraints))
+    return error("constraints string parsing failed");
+  if (parser.parseRBrace())
+    return expected("}");
+  if (parser.parseRParen())
+    return expected(")");
+
+  if (parser.parseOptionalKeyword("side_effects").succeeded())
+    result.attributes.set("side_effects", UnitAttr::get(ctxt));
+
+  if (parser.parseOptionalArrow().succeeded() &&
+      parser.parseType(resType).failed())
+    return mlir::failure();
+
+  if (parser.parseOptionalAttrDict(result.attributes).failed())
+    return mlir::failure();
+
+  result.attributes.set("asm_flavor", AsmFlavorAttr::get(ctxt, *flavor));
+  result.attributes.set("asm_string", StringAttr::get(ctxt, asmString));
+  result.attributes.set("constraints", StringAttr::get(ctxt, constraints));
+  result.attributes.set("operand_attrs", ArrayAttr::get(ctxt, operandAttrs));
+  result.getOrAddProperties<InlineAsmOp::Properties>().operands_segments =
+      parser.getBuilder().getDenseI32ArrayAttr(operandsGroupSizes);
+  if (resType)
+    result.addTypes(TypeRange{resType});
+
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp b/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
index 2eaa60c631a12..d41ea0af58938 100644
--- a/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/CIRCanonicalize.cpp
@@ -47,8 +47,8 @@ struct RemoveRedundantBranches : public OpRewritePattern<BrOp> {
     Block *block = op.getOperation()->getBlock();
     Block *dest = op.getDest();
 
-    assert(!cir::MissingFeatures::labelOp());
-
+    if (isa<cir::LabelOp>(dest->front()))
+      return failure();
     // Single edge between blocks: merge it.
     if (block->getNumSuccessors() == 1 &&
         dest->getSinglePredecessor() == block) {
diff --git a/clang/lib/CIR/Dialect/Transforms/CMakeLists.txt b/clang/lib/CIR/Dialect/Transforms/CMakeLists.txt
index 18beca7b9a680..df7a1a3e0acb5 100644
--- a/clang/lib/CIR/Dialect/Transforms/CMakeLists.txt
+++ b/clang/lib/CIR/Dialect/Transforms/CMakeLists.txt
@@ -4,6 +4,7 @@ add_clang_library(MLIRCIRTransforms
   FlattenCFG.cpp
   HoistAllocas.cpp
   LoweringPrepare.cpp
+  GotoSolver.cpp
 
   DEPENDS
   MLIRCIRPassIncGen
diff --git a/clang/lib/CIR/Dialect/Transforms/GotoSolver.cpp b/clang/lib/CIR/Dialect/Transforms/GotoSolver.cpp
new file mode 100644
index 0000000000000..c0db98440a902
--- /dev/null
+++ b/clang/lib/CIR/Dialect/Transforms/GotoSolver.cpp
@@ -0,0 +1,57 @@
+//====- GotoSolver.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "PassDetail.h"
+#include "clang/CIR/Dialect/IR/CIRDialect.h"
+#include "clang/CIR/Dialect/Passes.h"
+#include "llvm/Support/TimeProfiler.h"
+#include <memory>
+
+using namespace mlir;
+using namespace cir;
+
+namespace {
+
+struct GotoSolverPass : public GotoSolverBase<GotoSolverPass> {
+  GotoSolverPass() = default;
+  void runOnOperation() override;
+};
+
+static void process(cir::FuncOp func) {
+  mlir::OpBuilder rewriter(func.getContext());
+  llvm::StringMap<Block *> labels;
+  llvm::SmallVector<cir::GotoOp, 4> gotos;
+
+  func.getBody().walk([&](mlir::Operation *op) {
+    if (auto lab = dyn_cast<cir::LabelOp>(op)) {
+      // Will construct a string copy inplace. Safely erase the label
+      labels.try_emplace(lab.getLabel(), lab->getBlock());
+      lab.erase();
+    } else if (auto goTo = dyn_cast<cir::GotoOp>(op)) {
+      gotos.push_back(goTo);
+    }
+  });
+
+  for (auto goTo : gotos) {
+    mlir::OpBuilder::InsertionGuard guard(rewriter);
+    rewriter.setInsertionPoint(goTo);
+    Block *dest = labels[goTo.getLabel()];
+    cir::BrOp::create(rewriter, goTo.getLoc(), dest);
+    goTo.erase();
+  }
+}
+
+void GotoSolverPass::runOnOperation() {
+  llvm::TimeTraceScope scope("Goto Solver");
+  getOperation()->walk(&process);
+}
+
+} // namespace
+
+std::unique_ptr<Pass> mlir::createGotoSolverPass() {
+  return std::make_unique<GotoSolverPass>();
+}
diff --git a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
index 66260eb36e002..c15637d297cd1 100644
--- a/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/LoweringPrepare.cpp
@@ -8,7 +8,7 @@
 
 #include "PassDetail.h"
 #include "clang/AST/ASTContext.h"
-#include "clang/AST/CharUnits.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/CIR/Dialect/Builder/CIRBaseBuilder.h"
 #include "clang/CIR/Dialect/IR/CIRDialect.h"
 #include "clang/CIR/Dialect/IR/CIROpsEnums.h"
@@ -27,6 +27,7 @@ struct LoweringPreparePass : public LoweringPrepareBase<LoweringPreparePass> {
 
   void runOnOp(mlir::Operation *op);
   void lowerCastOp(cir::CastOp op);
+  void lowerComplexDivOp(cir::ComplexDivOp op);
   void lowerComplexMulOp(cir::ComplexMulOp op);
   void lowerUnaryOp(cir::UnaryOp op);
   void lowerArrayDtor(cir::ArrayDtor op);
@@ -181,6 +182,280 @@ static mlir::Value buildComplexBinOpLibCall(
   return call.getResult();
 }
 
+static llvm::StringRef
+getComplexDivLibCallName(llvm::APFloat::Semantics semantics) {
+  switch (semantics) {
+  case llvm::APFloat::S_IEEEhalf:
+    return "__divhc3";
+  case llvm::APFloat::S_IEEEsingle:
+    return "__divsc3";
+  case llvm::APFloat::S_IEEEdouble:
+    return "__divdc3";
+  case llvm::APFloat::S_PPCDoubleDouble:
+    return "__divtc3";
+  case llvm::APFloat::S_x87DoubleExtended:
+    return "__divxc3";
+  case llvm::APFloat::S_IEEEquad:
+    return "__divtc3";
+  default:
+    llvm_unreachable("unsupported floating point type");
+  }
+}
+
+static mlir::Value
+buildAlgebraicComplexDiv(CIRBaseBuilderTy &builder, mlir::Location loc,
+                         mlir::Value lhsReal, mlir::Value lhsImag,
+                         mlir::Value rhsReal, mlir::Value rhsImag) {
+  // (a+bi) / (c+di) = ((ac+bd)/(cc+dd)) + ((bc-ad)/(cc+dd))i
+  mlir::Value &a = lhsReal;
+  mlir::Value &b = lhsImag;
+  mlir::Value &c = rhsReal;
+  mlir::Value &d = rhsImag;
+
+  mlir::Value ac = builder.createBinop(loc, a, cir::BinOpKind::Mul, c); // a*c
+  mlir::Value bd = builder.createBinop(loc, b, cir::BinOpKind::Mul, d); // b*d
+  mlir::Value cc = builder.createBinop(loc, c, cir::BinOpKind::Mul, c); // c*c
+  mlir::Value dd = builder.createBinop(loc, d, cir::BinOpKind::Mul, d); // d*d
+  mlir::Value acbd =
+      builder.createBinop(loc, ac, cir::BinOpKind::Add, bd); // ac+bd
+  mlir::Value ccdd =
+      builder.createBinop(loc, cc, cir::BinOpKind::Add, dd); // cc+dd
+  mlir::Value resultReal =
+      builder.createBinop(loc, acbd, cir::BinOpKind::Div, ccdd);
+
+  mlir::Value bc = builder.createBinop(loc, b, cir::BinOpKind::Mul, c); // b*c
+  mlir::Value ad = builder.createBinop(loc, a, cir::BinOpKind::Mul, d); // a*d
+  mlir::Value bcad =
+      builder.createBinop(loc, bc, cir::BinOpKind::Sub, ad); // bc-ad
+  mlir::Value resultImag =
+      builder.createBinop(loc, bcad, cir::BinOpKind::Div, ccdd);
+  return builder.createComplexCreate(loc, resultReal, resultImag);
+}
+
+static mlir::Value
+buildRangeReductionComplexDiv(CIRBaseBuilderTy &builder, mlir::Location loc,
+                              mlir::Value lhsReal, mlir::Value lhsImag,
+                              mlir::Value rhsReal, mlir::Value rhsImag) {
+  // Implements Smith's algorithm for complex division.
+  // SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962).
+
+  // Let:
+  //   - lhs := a+bi
+  //   - rhs := c+di
+  //   - result := lhs / rhs = e+fi
+  //
+  // The algorithm pseudocode looks like follows:
+  //   if fabs(c) >= fabs(d):
+  //     r := d / c
+  //     tmp := c + r*d
+  //     e = (a + b*r) / tmp
+  //     f = (b - a*r) / tmp
+  //   else:
+  //     r := c / d
+  //     tmp := d + r*c
+  //     e = (a*r + b) / tmp
+  //     f = (b*r - a) / tmp
+
+  mlir::Value &a = lhsReal;
+  mlir::Value &b = lhsImag;
+  mlir::Value &c = rhsReal;
+  mlir::Value &d = rhsImag;
+
+  auto trueBranchBuilder = [&](mlir::OpBuilder &, mlir::Location) {
+    mlir::Value r = builder.createBinop(loc, d, cir::BinOpKind::Div,
+                                        c); // r := d / c
+    mlir::Value rd = builder.createBinop(loc, r, cir::BinOpKind::Mul, d); // r*d
+    mlir::Value tmp = builder.createBinop(loc, c, cir::BinOpKind::Add,
+                                          rd); // tmp := c + r*d
+
+    mlir::Value br = builder.createBinop(loc, b, cir::BinOpKind::Mul, r); // b*r
+    mlir::Value abr =
+        builder.createBinop(loc, a, cir::BinOpKind::Add, br); // a + b*r
+    mlir::Value e = builder.createBinop(loc, abr, cir::BinOpKind::Div, tmp);
+
+    mlir::Value ar = builder.createBinop(loc, a, cir::BinOpKind::Mul, r); // a*r
+    mlir::Value bar =
+        builder.createBinop(loc, b, cir::BinOpKind::Sub, ar); // b - a*r
+    mlir::Value f = builder.createBinop(loc, bar, cir::BinOpKind::Div, tmp);
+
+    mlir::Value result = builder.createComplexCreate(loc, e, f);
+    builder.createYield(loc, result);
+  };
+
+  auto falseBranchBuilder = [&](mlir::OpBuilder &, mlir::Location) {
+    mlir::Value r = builder.createBinop(loc, c, cir::BinOpKind::Div,
+                                        d); // r := c / d
+    mlir::Value rc = builder.createBinop(loc, r, cir::BinOpKind::Mul, c); // r*c
+    mlir::Value tmp = builder.createBinop(loc, d, cir::BinOpKind::Add,
+                                          rc); // tmp := d + r*c
+
+    mlir::Value ar = builder.createBinop(loc, a, cir::BinOpKind::Mul, r); // a*r
+    mlir::Value arb =
+        builder.createBinop(loc, ar, cir::BinOpKind::Add, b); // a*r + b
+    mlir::Value e = builder.createBinop(loc, arb, cir::BinOpKind::Div, tmp);
+
+    mlir::Value br = builder.createBinop(loc, b, cir::BinOpKind::Mul, r); // b*r
+    mlir::Value bra =
+        builder.createBinop(loc, br, cir::BinOpKind::Sub, a); // b*r - a
+    mlir::Value f = builder.createBinop(loc, bra, cir::BinOpKind::Div, tmp);
+
+    mlir::Value result = builder.createComplexCreate(loc, e, f);
+    builder.createYield(loc, result);
+  };
+
+  auto cFabs = builder.create<cir::FAbsOp>(loc, c);
+  auto dFabs = builder.create<cir::FAbsOp>(loc, d);
+  cir::CmpOp cmpResult =
+      builder.createCompare(loc, cir::CmpOpKind::ge, cFabs, dFabs);
+  auto ternary = builder.create<cir::TernaryOp>(
+      loc, cmpResult, trueBranchBuilder, falseBranchBuilder);
+
+  return ternary.getResult();
+}
+
+static mlir::Type higherPrecisionElementTypeForComplexArithmetic(
+    mlir::MLIRContext &context, clang::ASTContext &cc,
+    CIRBaseBuilderTy &builder, mlir::Type elementType) {
+
+  auto getHigherPrecisionFPType = [&context](mlir::Type type) -> mlir::Type {
+    if (mlir::isa<cir::FP16Type>(type))
+      return cir::SingleType::get(&context);
+
+    if (mlir::isa<cir::SingleType>(type) || mlir::isa<cir::BF16Type>(type))
+      return cir::DoubleType::get(&context);
+
+    if (mlir::isa<cir::DoubleType>(type))
+      return cir::LongDoubleType::get(&context, type);
+
+    return type;
+  };
+
+  auto getFloatTypeSemantics =
+      [&cc](mlir::Type type) -> const llvm::fltSemantics & {
+    const clang::TargetInfo &info = cc.getTargetInfo();
+    if (mlir::isa<cir::FP16Type>(type))
+      return info.getHalfFormat();
+
+    if (mlir::isa<cir::BF16Type>(type))
+      return info.getBFloat16Format();
+
+    if (mlir::isa<cir::SingleType>(type))
+      return info.getFloatFormat();
+
+    if (mlir::isa<cir::DoubleType>(type))
+      return info.getDoubleFormat();
+
+    if (mlir::isa<cir::LongDoubleType>(type)) {
+      if (cc.getLangOpts().OpenMP && cc.getLangOpts().OpenMPIsTargetDevice)
+        llvm_unreachable("NYI Float type semantics with OpenMP");
+      return info.getLongDoubleFormat();
+    }
+
+    if (mlir::isa<cir::FP128Type>(type)) {
+      if (cc.getLangOpts().OpenMP && cc.getLangOpts().OpenMPIsTargetDevice)
+        llvm_unreachable("NYI Float type semantics with OpenMP");
+      return info.getFloat128Format();
+    }
+
+    assert(false && "Unsupported float type semantics");
+  };
+
+  const mlir::Type higherElementType = getHigherPrecisionFPType(elementType);
+  const llvm::fltSemantics &elementTypeSemantics =
+      getFloatTypeSemantics(elementType);
+  const llvm::fltSemantics &higherElementTypeSemantics =
+      getFloatTypeSemantics(higherElementType);
+
+  // Check that the promoted type can handle the intermediate values without
+  // overflowing. This can be interpreted as:
+  // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal) * 2 <=
+  //      LargerType.LargestFiniteVal.
+  // In terms of exponent it gives this formula:
+  // (SmallerType.LargestFiniteVal * SmallerType.LargestFiniteVal
+  // doubles the exponent of SmallerType.LargestFiniteVal)
+  if (llvm::APFloat::semanticsMaxExponent(elementTypeSemantics) * 2 + 1 <=
+      llvm::APFloat::semanticsMaxExponent(higherElementTypeSemantics)) {
+    return higherElementType;
+  }
+
+  // The intermediate values can't be represented in the promoted type
+  // without overflowing.
+  return {};
+}
+
+static mlir::Value
+lowerComplexDiv(LoweringPreparePass &pass, CIRBaseBuilderTy &builder,
+                mlir::Location loc, cir::ComplexDivOp op, mlir::Value lhsReal,
+                mlir::Value lhsImag, mlir::Value rhsReal, mlir::Value rhsImag,
+                mlir::MLIRContext &mlirCx, clang::ASTContext &cc) {
+  cir::ComplexType complexTy = op.getType();
+  if (mlir::isa<cir::FPTypeInterface>(complexTy.getElementType())) {
+    cir::ComplexRangeKind range = op.getRange();
+    if (range == cir::ComplexRangeKind::Improved)
+      return buildRangeReductionComplexDiv(builder, loc, lhsReal, lhsImag,
+                                           rhsReal, rhsImag);
+
+    if (range == cir::ComplexRangeKind::Full)
+      return buildComplexBinOpLibCall(pass, builder, &getComplexDivLibCallName,
+                                      loc, complexTy, lhsReal, lhsImag, rhsReal,
+                                      rhsImag);
+
+    if (range == cir::ComplexRangeKind::Promoted) {
+      mlir::Type originalElementType = complexTy.getElementType();
+      mlir::Type higherPrecisionElementType =
+          higherPrecisionElementTypeForComplexArithmetic(mlirCx, cc, builder,
+                                                         originalElementType);
+
+      if (!higherPrecisionElementType)
+        return buildRangeReductionComplexDiv(builder, loc, lhsReal, lhsImag,
+                                             rhsReal, rhsImag);
+
+      cir::CastKind floatingCastKind = cir::CastKind::floating;
+      lhsReal = builder.createCast(floatingCastKind, lhsReal,
+                                   higherPrecisionElementType);
+      lhsImag = builder.createCast(floatingCastKind, lhsImag,
+                                   higherPrecisionElementType);
+      rhsReal = builder.createCast(floatingCastKind, rhsReal,
+                                   higherPrecisionElementType);
+      rhsImag = builder.createCast(floatingCastKind, rhsImag,
+                                   higherPrecisionElementType);
+
+      mlir::Value algebraicResult = buildAlgebraicComplexDiv(
+          builder, loc, lhsReal, lhsImag, rhsReal, rhsImag);
+
+      mlir::Value resultReal = builder.createComplexReal(loc, algebraicResult);
+      mlir::Value resultImag = builder.createComplexImag(loc, algebraicResult);
+
+      mlir::Value finalReal =
+          builder.createCast(floatingCastKind, resultReal, originalElementType);
+      mlir::Value finalImag =
+          builder.createCast(floatingCastKind, resultImag, originalElementType);
+      return builder.createComplexCreate(loc, finalReal, finalImag);
+    }
+  }
+
+  return buildAlgebraicComplexDiv(builder, loc, lhsReal, lhsImag, rhsReal,
+                                  rhsImag);
+}
+
+void LoweringPreparePass::lowerComplexDivOp(cir::ComplexDivOp op) {
+  cir::CIRBaseBuilderTy builder(getContext());
+  builder.setInsertionPointAfter(op);
+  mlir::Location loc = op.getLoc();
+  mlir::TypedValue<cir::ComplexType> lhs = op.getLhs();
+  mlir::TypedValue<cir::ComplexType> rhs = op.getRhs();
+  mlir::Value lhsReal = builder.createComplexReal(loc, lhs);
+  mlir::Value lhsImag = builder.createComplexImag(loc, lhs);
+  mlir::Value rhsReal = builder.createComplexReal(loc, rhs);
+  mlir::Value rhsImag = builder.createComplexImag(loc, rhs);
+
+  mlir::Value loweredResult =
+      lowerComplexDiv(*this, builder, loc, op, lhsReal, lhsImag, rhsReal,
+                      rhsImag, getContext(), *astCtx);
+  op.replaceAllUsesWith(loweredResult);
+  op.erase();
+}
+
 static llvm::StringRef
 getComplexMulLibCallName(llvm::APFloat::Semantics semantics) {
   switch (semantics) {
@@ -412,6 +687,8 @@ void LoweringPreparePass::runOnOp(mlir::Operation *op) {
     lowerArrayDtor(arrayDtor);
   else if (auto cast = mlir::dyn_cast<cir::CastOp>(op))
     lowerCastOp(cast);
+  else if (auto complexDiv = mlir::dyn_cast<cir::ComplexDivOp>(op))
+    lowerComplexDivOp(complexDiv);
   else if (auto complexMul = mlir::dyn_cast<cir::ComplexMulOp>(op))
     lowerComplexMulOp(complexMul);
   else if (auto unary = mlir::dyn_cast<cir::UnaryOp>(op))
@@ -427,7 +704,7 @@ void LoweringPreparePass::runOnOperation() {
 
   op->walk([&](mlir::Operation *op) {
     if (mlir::isa<cir::ArrayCtor, cir::ArrayDtor, cir::CastOp,
-                  cir::ComplexMulOp, cir::UnaryOp>(op))
+                  cir::ComplexMulOp, cir::ComplexDivOp, cir::UnaryOp>(op))
       opsToTransform.push_back(op);
   });
 
diff --git a/clang/lib/CIR/Lowering/CIRPasses.cpp b/clang/lib/CIR/Lowering/CIRPasses.cpp
index bb9781be897eb..ccc838717e421 100644
--- a/clang/lib/CIR/Lowering/CIRPasses.cpp
+++ b/clang/lib/CIR/Lowering/CIRPasses.cpp
@@ -45,6 +45,7 @@ namespace mlir {
 void populateCIRPreLoweringPasses(OpPassManager &pm) {
   pm.addPass(createHoistAllocasPass());
   pm.addPass(createCIRFlattenCFGPass());
+  pm.addPass(createGotoSolverPass());
 }
 
 } // namespace mlir
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 7e1c9fb9164cd..9f0e4e6ecb8ce 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -201,7 +201,8 @@ class CIRAttrToValue {
   mlir::Value visit(mlir::Attribute attr) {
     return llvm::TypeSwitch<mlir::Attribute, mlir::Value>(attr)
         .Case<cir::IntAttr, cir::FPAttr, cir::ConstComplexAttr,
-              cir::ConstArrayAttr, cir::ConstVectorAttr, cir::ConstPtrAttr,
+              cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
+              cir::ConstPtrAttr, cir::GlobalViewAttr, cir::VTableAttr,
               cir::ZeroAttr>([&](auto attrT) { return visitCirAttr(attrT); })
         .Default([&](auto attrT) { return mlir::Value(); });
   }
@@ -211,7 +212,10 @@ class CIRAttrToValue {
   mlir::Value visitCirAttr(cir::ConstComplexAttr complexAttr);
   mlir::Value visitCirAttr(cir::ConstPtrAttr ptrAttr);
   mlir::Value visitCirAttr(cir::ConstArrayAttr attr);
+  mlir::Value visitCirAttr(cir::ConstRecordAttr attr);
   mlir::Value visitCirAttr(cir::ConstVectorAttr attr);
+  mlir::Value visitCirAttr(cir::GlobalViewAttr attr);
+  mlir::Value visitCirAttr(cir::VTableAttr attr);
   mlir::Value visitCirAttr(cir::ZeroAttr attr);
 
 private:
@@ -265,6 +269,26 @@ void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow,
   }
 }
 
+static mlir::LLVM::CallIntrinsicOp
+createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter,
+                          mlir::Location loc, const llvm::Twine &intrinsicName,
+                          mlir::Type resultTy, mlir::ValueRange operands) {
+  auto intrinsicNameAttr =
+      mlir::StringAttr::get(rewriter.getContext(), intrinsicName);
+  return mlir::LLVM::CallIntrinsicOp::create(rewriter, loc, resultTy,
+                                             intrinsicNameAttr, operands);
+}
+
+static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(
+    mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op,
+    const llvm::Twine &intrinsicName, mlir::Type resultTy,
+    mlir::ValueRange operands) {
+  mlir::LLVM::CallIntrinsicOp callIntrinOp = createCallLLVMIntrinsicOp(
+      rewriter, op->getLoc(), intrinsicName, resultTy, operands);
+  rewriter.replaceOp(op, callIntrinOp.getOperation());
+  return callIntrinOp;
+}
+
 /// IntAttr visitor.
 mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) {
   mlir::Location loc = parentOp->getLoc();
@@ -364,6 +388,21 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) {
   return result;
 }
 
+/// ConstRecord visitor.
+mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstRecordAttr constRecord) {
+  const mlir::Type llvmTy = converter->convertType(constRecord.getType());
+  const mlir::Location loc = parentOp->getLoc();
+  mlir::Value result = rewriter.create<mlir::LLVM::UndefOp>(loc, llvmTy);
+
+  // Iteratively lower each constant element of the record.
+  for (auto [idx, elt] : llvm::enumerate(constRecord.getMembers())) {
+    mlir::Value init = visit(elt);
+    result = rewriter.create<mlir::LLVM::InsertValueOp>(loc, result, init, idx);
+  }
+
+  return result;
+}
+
 /// ConstVectorAttr visitor.
 mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstVectorAttr attr) {
   const mlir::Type llvmTy = converter->convertType(attr.getType());
@@ -391,6 +430,92 @@ mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstVectorAttr attr) {
                                    mlirValues));
 }
 
+// GlobalViewAttr visitor.
+mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) {
+  auto moduleOp = parentOp->getParentOfType<mlir::ModuleOp>();
+  mlir::DataLayout dataLayout(moduleOp);
+  mlir::Type sourceType;
+  assert(!cir::MissingFeatures::addressSpace());
+  llvm::StringRef symName;
+  mlir::Operation *sourceSymbol =
+      mlir::SymbolTable::lookupSymbolIn(moduleOp, globalAttr.getSymbol());
+  if (auto llvmSymbol = dyn_cast<mlir::LLVM::GlobalOp>(sourceSymbol)) {
+    sourceType = llvmSymbol.getType();
+    symName = llvmSymbol.getSymName();
+  } else if (auto cirSymbol = dyn_cast<cir::GlobalOp>(sourceSymbol)) {
+    sourceType =
+        convertTypeForMemory(*converter, dataLayout, cirSymbol.getSymType());
+    symName = cirSymbol.getSymName();
+  } else if (auto llvmFun = dyn_cast<mlir::LLVM::LLVMFuncOp>(sourceSymbol)) {
+    sourceType = llvmFun.getFunctionType();
+    symName = llvmFun.getSymName();
+  } else if (auto fun = dyn_cast<cir::FuncOp>(sourceSymbol)) {
+    sourceType = converter->convertType(fun.getFunctionType());
+    symName = fun.getSymName();
+  } else if (auto alias = dyn_cast<mlir::LLVM::AliasOp>(sourceSymbol)) {
+    sourceType = alias.getType();
+    symName = alias.getSymName();
+  } else {
+    llvm_unreachable("Unexpected GlobalOp type");
+  }
+
+  mlir::Location loc = parentOp->getLoc();
+  mlir::Value addrOp = rewriter.create<mlir::LLVM::AddressOfOp>(
+      loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symName);
+
+  if (globalAttr.getIndices()) {
+    llvm::SmallVector<mlir::LLVM::GEPArg> indices;
+
+    if (mlir::isa<mlir::LLVM::LLVMArrayType, mlir::LLVM::LLVMStructType>(
+            sourceType))
+      indices.push_back(0);
+
+    for (mlir::Attribute idx : globalAttr.getIndices()) {
+      auto intAttr = mlir::cast<mlir::IntegerAttr>(idx);
+      indices.push_back(intAttr.getValue().getSExtValue());
+    }
+    mlir::Type resTy = addrOp.getType();
+    mlir::Type eltTy = converter->convertType(sourceType);
+    addrOp = rewriter.create<mlir::LLVM::GEPOp>(
+        loc, resTy, eltTy, addrOp, indices, mlir::LLVM::GEPNoWrapFlags::none);
+  }
+
+  // The incubator has handling here for the attribute having integer type, but
+  // the only test case I could find that reaches it is a direct CIR-to-LLVM IR
+  // lowering with no clear indication of how the CIR might have been generated.
+  // We'll hit the unreachable below if this happens.
+  assert(!cir::MissingFeatures::globalViewIntLowering());
+
+  if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(globalAttr.getType())) {
+    mlir::Type llvmEltTy =
+        convertTypeForMemory(*converter, dataLayout, ptrTy.getPointee());
+
+    if (llvmEltTy == sourceType)
+      return addrOp;
+
+    mlir::Type llvmDstTy = converter->convertType(globalAttr.getType());
+    return rewriter.create<mlir::LLVM::BitcastOp>(parentOp->getLoc(), llvmDstTy,
+                                                  addrOp);
+  }
+
+  llvm_unreachable("Expecting pointer or integer type for GlobalViewAttr");
+}
+
+// VTableAttr visitor.
+mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) {
+  mlir::Type llvmTy = converter->convertType(vtableArr.getType());
+  mlir::Location loc = parentOp->getLoc();
+  mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
+
+  for (auto [idx, elt] : llvm::enumerate(vtableArr.getData())) {
+    mlir::Value init = visit(elt);
+    result =
+        mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
+  }
+
+  return result;
+}
+
 /// ZeroAttr visitor.
 mlir::Value CIRAttrToValue::visitCirAttr(cir::ZeroAttr attr) {
   mlir::Location loc = parentOp->getLoc();
@@ -460,6 +585,29 @@ mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMAssumeAlignedOpLowering::matchAndRewrite(
+    cir::AssumeAlignedOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  SmallVector<mlir::Value, 3> opBundleArgs{adaptor.getPointer()};
+
+  auto alignment = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
+                                                  adaptor.getAlignmentAttr());
+  opBundleArgs.push_back(alignment);
+
+  if (mlir::Value offset = adaptor.getOffset())
+    opBundleArgs.push_back(offset);
+
+  auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
+                                             rewriter.getI1Type(), 1);
+  mlir::LLVM::AssumeOp::create(rewriter, op.getLoc(), cond, "align",
+                               opBundleArgs);
+
+  // The llvm.assume operation does not have a result, so we need to replace
+  // all uses of this cir.assume_aligned operation with the input ptr itself.
+  rewriter.replaceOp(op, adaptor.getPointer());
+  return mlir::success();
+}
+
 mlir::LogicalResult CIRToLLVMAssumeSepStorageOpLowering::matchAndRewrite(
     cir::AssumeSepStorageOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -989,8 +1137,7 @@ rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands,
     auto calleeTy = op->getOperands().front().getType();
     auto calleePtrTy = cast<cir::PointerType>(calleeTy);
     auto calleeFuncTy = cast<cir::FuncType>(calleePtrTy.getPointee());
-    calleeFuncTy.dump();
-    converter->convertType(calleeFuncTy).dump();
+    llvm::append_range(adjustedCallOperands, callOperands);
     llvmFnTy = cast<mlir::LLVM::LLVMFunctionType>(
         converter->convertType(calleeFuncTy));
   }
@@ -1016,12 +1163,50 @@ mlir::LogicalResult CIRToLLVMCallOpLowering::matchAndRewrite(
                              getTypeConverter(), op.getCalleeAttr());
 }
 
+mlir::LogicalResult CIRToLLVMReturnAddrOpLowering::matchAndRewrite(
+    cir::ReturnAddrOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
+  replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.returnaddress",
+                                   llvmPtrTy, adaptor.getOperands());
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMFrameAddrOpLowering::matchAndRewrite(
+    cir::FrameAddrOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
+  replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.frameaddress", llvmPtrTy,
+                                   adaptor.getOperands());
+  return mlir::success();
+}
+
+static mlir::LLVM::AtomicOrdering
+getLLVMMemOrder(std::optional<cir::MemOrder> memorder) {
+  if (!memorder)
+    return mlir::LLVM::AtomicOrdering::not_atomic;
+  switch (*memorder) {
+  case cir::MemOrder::Relaxed:
+    return mlir::LLVM::AtomicOrdering::monotonic;
+  case cir::MemOrder::Consume:
+  case cir::MemOrder::Acquire:
+    return mlir::LLVM::AtomicOrdering::acquire;
+  case cir::MemOrder::Release:
+    return mlir::LLVM::AtomicOrdering::release;
+  case cir::MemOrder::AcquireRelease:
+    return mlir::LLVM::AtomicOrdering::acq_rel;
+  case cir::MemOrder::SequentiallyConsistent:
+    return mlir::LLVM::AtomicOrdering::seq_cst;
+  }
+  llvm_unreachable("unknown memory order");
+}
+
 mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
     cir::LoadOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
   const mlir::Type llvmTy =
       convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
-  assert(!cir::MissingFeatures::opLoadStoreMemOrder());
+  mlir::LLVM::AtomicOrdering ordering = getLLVMMemOrder(op.getMemOrder());
   std::optional<size_t> opAlign = op.getAlignment();
   unsigned alignment =
       (unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
@@ -1030,11 +1215,10 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
 
   // TODO: nontemporal, syncscope.
   assert(!cir::MissingFeatures::opLoadStoreVolatile());
-  mlir::LLVM::LoadOp newLoad = rewriter.create<mlir::LLVM::LoadOp>(
-      op->getLoc(), llvmTy, adaptor.getAddr(), alignment,
-      /*volatile=*/false, /*nontemporal=*/false,
-      /*invariant=*/false, /*invariantGroup=*/false,
-      mlir::LLVM::AtomicOrdering::not_atomic);
+  mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create(
+      rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment,
+      /*isVolatile=*/false, /*isNonTemporal=*/false,
+      /*isInvariant=*/false, /*isInvariantGroup=*/false, ordering);
 
   // Convert adapted result to its original type if needed.
   mlir::Value result =
@@ -1047,7 +1231,7 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
 mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
     cir::StoreOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
-  assert(!cir::MissingFeatures::opLoadStoreMemOrder());
+  mlir::LLVM::AtomicOrdering memorder = getLLVMMemOrder(op.getMemOrder());
   const mlir::Type llvmTy =
       getTypeConverter()->convertType(op.getValue().getType());
   std::optional<size_t> opAlign = op.getAlignment();
@@ -1061,10 +1245,10 @@ mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
                                    op.getValue().getType(), adaptor.getValue());
   // TODO: nontemporal, syncscope.
   assert(!cir::MissingFeatures::opLoadStoreVolatile());
-  mlir::LLVM::StoreOp storeOp = rewriter.create<mlir::LLVM::StoreOp>(
-      op->getLoc(), value, adaptor.getAddr(), alignment, /*volatile=*/false,
-      /*nontemporal=*/false, /*invariantGroup=*/false,
-      mlir::LLVM::AtomicOrdering::not_atomic);
+  mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create(
+      rewriter, op->getLoc(), value, adaptor.getAddr(), alignment,
+      /*isVolatile=*/false,
+      /*isNonTemporal=*/false, /*isInvariantGroup=*/false, memorder);
   rewriter.replaceOp(op, storeOp);
   assert(!cir::MissingFeatures::opLoadStoreTbaa());
   return mlir::LogicalResult::success();
@@ -1101,7 +1285,13 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
     attr = rewriter.getIntegerAttr(typeConverter->convertType(op.getType()),
                                    value);
   } else if (mlir::isa<cir::IntType>(op.getType())) {
-    assert(!cir::MissingFeatures::opGlobalViewAttr());
+    // Lower GlobalViewAttr to llvm.mlir.addressof + llvm.mlir.ptrtoint
+    if (auto ga = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
+      // See the comment in visitCirAttr for why this isn't implemented.
+      assert(!cir::MissingFeatures::globalViewIntLowering());
+      op.emitError() << "global view with integer type";
+      return mlir::failure();
+    }
 
     attr = rewriter.getIntegerAttr(
         typeConverter->convertType(op.getType()),
@@ -1119,7 +1309,12 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
         return mlir::success();
       }
     }
-    assert(!cir::MissingFeatures::opGlobalViewAttr());
+    // Lower GlobalViewAttr to llvm.mlir.addressof
+    if (auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
+      auto newOp = lowerCirAttrAsValue(op, gv, rewriter, getTypeConverter());
+      rewriter.replaceOp(op, newOp);
+      return mlir::success();
+    }
     attr = op.getValue();
   } else if (const auto arrTy = mlir::dyn_cast<cir::ArrayType>(op.getType())) {
     const auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(op.getValue());
@@ -1142,6 +1337,11 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
       rewriter.eraseOp(op);
       return mlir::success();
     }
+  } else if (const auto recordAttr =
+                 mlir::dyn_cast<cir::ConstRecordAttr>(op.getValue())) {
+    auto initVal = lowerCirAttrAsValue(op, recordAttr, rewriter, typeConverter);
+    rewriter.replaceOp(op, initVal);
+    return mlir::success();
   } else if (const auto vecTy = mlir::dyn_cast<cir::VectorType>(op.getType())) {
     rewriter.replaceOp(op, lowerCirAttrAsValue(op, op.getValue(), rewriter,
                                                getTypeConverter()));
@@ -1204,6 +1404,15 @@ mlir::LogicalResult CIRToLLVMExpectOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMFAbsOpLowering::matchAndRewrite(
+    cir::FAbsOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Type resTy = typeConverter->convertType(op.getType());
+  rewriter.replaceOpWithNewOp<mlir::LLVM::FAbsOp>(op, resTy,
+                                                  adaptor.getOperands()[0]);
+  return mlir::success();
+}
+
 /// Convert the `cir.func` attributes to `llvm.func` attributes.
 /// Only retain those attributes that are not constructed by
 /// `LLVMFuncOp::build`. If `filterArgAttrs` is set, also filter out
@@ -1349,7 +1558,7 @@ void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(
   //        in CIRToLLVMGlobalOpLowering::matchAndRewrite() but that will go
   //        away when the placeholders are no longer needed.
   assert(!cir::MissingFeatures::opGlobalConstant());
-  const bool isConst = false;
+  const bool isConst = op.getConstant();
   assert(!cir::MissingFeatures::addressSpace());
   const unsigned addrSpace = 0;
   const bool isDsoLocal = op.getDsoLocal();
@@ -1374,8 +1583,9 @@ CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal(
     cir::GlobalOp op, mlir::Attribute init,
     mlir::ConversionPatternRewriter &rewriter) const {
   // TODO: Generalize this handling when more types are needed here.
-  assert((isa<cir::ConstArrayAttr, cir::ConstVectorAttr, cir::ConstPtrAttr,
-              cir::ConstComplexAttr, cir::ZeroAttr>(init)));
+  assert((isa<cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
+              cir::ConstPtrAttr, cir::ConstComplexAttr, cir::GlobalViewAttr,
+              cir::VTableAttr, cir::ZeroAttr>(init)));
 
   // TODO(cir): once LLVM's dialect has proper equivalent attributes this
   // should be updated. For now, we use a custom op to initialize globals
@@ -1428,8 +1638,9 @@ mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
         return mlir::failure();
       }
     } else if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr,
-                         cir::ConstPtrAttr, cir::ConstComplexAttr,
-                         cir::ZeroAttr>(init.value())) {
+                         cir::ConstRecordAttr, cir::ConstPtrAttr,
+                         cir::ConstComplexAttr, cir::GlobalViewAttr,
+                         cir::VTableAttr, cir::ZeroAttr>(init.value())) {
       // TODO(cir): once LLVM's dialect has proper equivalent attributes this
       // should be updated. For now, we use a custom op to initialize globals
       // to the appropriate value.
@@ -1981,6 +2192,10 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
 
     return mlir::LLVM::LLVMPointerType::get(type.getContext(), targetAS);
   });
+  converter.addConversion([&](cir::VPtrType type) -> mlir::Type {
+    assert(!cir::MissingFeatures::addressSpace());
+    return mlir::LLVM::LLVMPointerType::get(type.getContext());
+  });
   converter.addConversion([&](cir::ArrayType type) -> mlir::Type {
     mlir::Type ty =
         convertTypeForMemory(converter, dataLayout, type.getElementType());
@@ -2057,6 +2272,9 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
       }
       break;
     }
+    converter.addConversion([&](cir::VoidType type) -> mlir::Type {
+      return mlir::LLVM::LLVMVoidType::get(type.getContext());
+    });
 
     // Record has a name: lower as an identified record.
     mlir::LLVM::LLVMStructType llvmStruct;
@@ -2072,6 +2290,9 @@ static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
 
     return llvmStruct;
   });
+  converter.addConversion([&](cir::VoidType type) -> mlir::Type {
+    return mlir::LLVM::LLVMVoidType::get(type.getContext());
+  });
 }
 
 // The applyPartialConversion function traverses blocks in the dominance order,
@@ -2143,6 +2364,11 @@ void ConvertCIRToLLVMPass::processCIRAttrs(mlir::ModuleOp module) {
           module->getAttr(cir::CIRDialect::getTripleAttrName()))
     module->setAttr(mlir::LLVM::LLVMDialect::getTargetTripleAttrName(),
                     tripleAttr);
+
+  if (mlir::Attribute asmAttr =
+          module->getAttr(cir::CIRDialect::getModuleLevelAsmAttrName()))
+    module->setAttr(mlir::LLVM::LLVMDialect::getModuleLevelAsmAttrName(),
+                    asmAttr);
 }
 
 void ConvertCIRToLLVMPass::runOnOperation() {
@@ -2165,9 +2391,12 @@ void ConvertCIRToLLVMPass::runOnOperation() {
   patterns.add<CIRToLLVMCastOpLowering>(converter, patterns.getContext(), dl);
   patterns.add<CIRToLLVMPtrStrideOpLowering>(converter, patterns.getContext(),
                                              dl);
+  patterns.add<CIRToLLVMInlineAsmOpLowering>(converter, patterns.getContext(),
+                                             dl);
   patterns.add<
       // clang-format off
                CIRToLLVMAssumeOpLowering,
+               CIRToLLVMAssumeAlignedOpLowering,
                CIRToLLVMAssumeSepStorageOpLowering,
                CIRToLLVMBaseClassAddrOpLowering,
                CIRToLLVMBinOpLowering,
@@ -2192,10 +2421,13 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                CIRToLLVMComplexSubOpLowering,
                CIRToLLVMConstantOpLowering,
                CIRToLLVMExpectOpLowering,
+               CIRToLLVMFAbsOpLowering,
+               CIRToLLVMFrameAddrOpLowering,
                CIRToLLVMFuncOpLowering,
                CIRToLLVMGetBitfieldOpLowering,
                CIRToLLVMGetGlobalOpLowering,
                CIRToLLVMGetMemberOpLowering,
+               CIRToLLVMReturnAddrOpLowering,
                CIRToLLVMRotateOpLowering,
                CIRToLLVMSelectOpLowering,
                CIRToLLVMSetBitfieldOpLowering,
@@ -2205,6 +2437,10 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                CIRToLLVMSwitchFlatOpLowering,
                CIRToLLVMTrapOpLowering,
                CIRToLLVMUnaryOpLowering,
+               CIRToLLVMUnreachableOpLowering,
+               CIRToLLVMVAArgOpLowering,
+               CIRToLLVMVAEndOpLowering,
+               CIRToLLVMVAStartOpLowering,
                CIRToLLVMVecCmpOpLowering,
                CIRToLLVMVecCreateOpLowering,
                CIRToLLVMVecExtractOpLowering,
@@ -2213,7 +2449,9 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                CIRToLLVMVecShuffleOpLowering,
                CIRToLLVMVecSplatOpLowering,
                CIRToLLVMVecTernaryOpLowering,
-               CIRToLLVMUnreachableOpLowering
+               CIRToLLVMVTableAddrPointOpLowering,
+               CIRToLLVMVTableGetVPtrOpLowering,
+               CIRToLLVMVTableGetVirtualFnAddrOpLowering
       // clang-format on
       >(converter, patterns.getContext());
 
@@ -2292,6 +2530,76 @@ mlir::LogicalResult CIRToLLVMTrapOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+static mlir::Value
+getValueForVTableSymbol(mlir::Operation *op,
+                        mlir::ConversionPatternRewriter &rewriter,
+                        const mlir::TypeConverter *converter,
+                        mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType) {
+  auto module = op->getParentOfType<mlir::ModuleOp>();
+  mlir::Operation *symbol = mlir::SymbolTable::lookupSymbolIn(module, nameAttr);
+  if (auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(symbol)) {
+    eltType = llvmSymbol.getType();
+  } else if (auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(symbol)) {
+    eltType = converter->convertType(cirSymbol.getSymType());
+  } else {
+    op->emitError() << "unexpected symbol type for " << symbol;
+    return {};
+  }
+
+  return mlir::LLVM::AddressOfOp::create(
+      rewriter, op->getLoc(),
+      mlir::LLVM::LLVMPointerType::get(op->getContext()), nameAttr.getValue());
+}
+
+mlir::LogicalResult CIRToLLVMVTableAddrPointOpLowering::matchAndRewrite(
+    cir::VTableAddrPointOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  const mlir::TypeConverter *converter = getTypeConverter();
+  mlir::Type targetType = converter->convertType(op.getType());
+  llvm::SmallVector<mlir::LLVM::GEPArg> offsets;
+  mlir::Type eltType;
+  mlir::Value symAddr = getValueForVTableSymbol(op, rewriter, converter,
+                                                op.getNameAttr(), eltType);
+  if (!symAddr)
+    return op.emitError() << "Unable to get value for vtable symbol";
+
+  offsets = llvm::SmallVector<mlir::LLVM::GEPArg>{
+      0, op.getAddressPointAttr().getIndex(),
+      op.getAddressPointAttr().getOffset()};
+
+  assert(eltType && "Shouldn't ever be missing an eltType here");
+  mlir::LLVM::GEPNoWrapFlags inboundsNuw =
+      mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
+  rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, targetType, eltType,
+                                                 symAddr, offsets, inboundsNuw);
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMVTableGetVPtrOpLowering::matchAndRewrite(
+    cir::VTableGetVPtrOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  // cir.vtable.get_vptr is equivalent to a bitcast from the source object
+  // pointer to the vptr type. Since the LLVM dialect uses opaque pointers
+  // we can just replace uses of this operation with the original pointer.
+  mlir::Value srcVal = adaptor.getSrc();
+  rewriter.replaceAllUsesWith(op, srcVal);
+  rewriter.eraseOp(op);
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMVTableGetVirtualFnAddrOpLowering::matchAndRewrite(
+    cir::VTableGetVirtualFnAddrOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Type targetType = getTypeConverter()->convertType(op.getType());
+  auto eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
+  llvm::SmallVector<mlir::LLVM::GEPArg> offsets =
+      llvm::SmallVector<mlir::LLVM::GEPArg>{op.getIndex()};
+  rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
+      op, targetType, eltType, adaptor.getVptr(), offsets,
+      mlir::LLVM::GEPNoWrapFlags::inbounds);
+  return mlir::success();
+}
+
 mlir::LogicalResult CIRToLLVMStackSaveOpLowering::matchAndRewrite(
     cir::StackSaveOp op, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -2797,6 +3105,105 @@ mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite(
   return mlir::success();
 }
 
+mlir::LogicalResult CIRToLLVMInlineAsmOpLowering::matchAndRewrite(
+    cir::InlineAsmOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  mlir::Type llResTy;
+  if (op.getNumResults())
+    llResTy = getTypeConverter()->convertType(op.getType(0));
+
+  cir::AsmFlavor dialect = op.getAsmFlavor();
+  mlir::LLVM::AsmDialect llDialect = dialect == cir::AsmFlavor::x86_att
+                                         ? mlir::LLVM::AsmDialect::AD_ATT
+                                         : mlir::LLVM::AsmDialect::AD_Intel;
+
+  SmallVector<mlir::Attribute> opAttrs;
+  StringRef llvmAttrName = mlir::LLVM::InlineAsmOp::getElementTypeAttrName();
+
+  // this is for the lowering to LLVM from LLVM dialect. Otherwise, if we
+  // don't have the result (i.e. void type as a result of operation), the
+  // element type attribute will be attached to the whole instruction, but not
+  // to the operand
+  if (!op.getNumResults())
+    opAttrs.push_back(mlir::Attribute());
+
+  SmallVector<mlir::Value> llvmOperands;
+  SmallVector<mlir::Value> cirOperands;
+  for (auto const&[llvmOp, cirOp] :
+       zip(adaptor.getAsmOperands(), op.getAsmOperands())) {
+    append_range(llvmOperands, llvmOp);
+    append_range(cirOperands, cirOp);
+  }
+
+  // so far we infer the llvm dialect element type attr from
+  // CIR operand type.
+  for (auto const&[cirOpAttr, cirOp] : zip(op.getOperandAttrs(), cirOperands)) {    
+    if (!cirOpAttr) {
+      opAttrs.push_back(mlir::Attribute());
+      continue;
+    }
+
+    llvm::SmallVector<mlir::NamedAttribute, 1> attrs;
+    cir::PointerType typ =
+        mlir::cast<cir::PointerType>(cirOp.getType());
+    mlir::TypeAttr typAttr = mlir::TypeAttr::get(convertTypeForMemory(
+        *getTypeConverter(), dataLayout, typ.getPointee()));
+
+    attrs.push_back(rewriter.getNamedAttr(llvmAttrName, typAttr));
+    mlir::DictionaryAttr newDict = rewriter.getDictionaryAttr(attrs);
+    opAttrs.push_back(newDict);
+  }
+
+  rewriter.replaceOpWithNewOp<mlir::LLVM::InlineAsmOp>(
+      op, llResTy, llvmOperands, op.getAsmStringAttr(), op.getConstraintsAttr(),
+      op.getSideEffectsAttr(),
+      /*is_align_stack*/ mlir::UnitAttr(),
+      /*tail_call_kind*/
+      mlir::LLVM::TailCallKindAttr::get(
+          getContext(), mlir::LLVM::tailcallkind::TailCallKind::None),
+      mlir::LLVM::AsmDialectAttr::get(getContext(), llDialect),
+      rewriter.getArrayAttr(opAttrs));
+
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMVAStartOpLowering::matchAndRewrite(
+    cir::VAStartOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
+  auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
+                                              adaptor.getArgList());
+  rewriter.replaceOpWithNewOp<mlir::LLVM::VaStartOp>(op, vaList);
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMVAEndOpLowering::matchAndRewrite(
+    cir::VAEndOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
+  auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
+                                              adaptor.getArgList());
+  rewriter.replaceOpWithNewOp<mlir::LLVM::VaEndOp>(op, vaList);
+  return mlir::success();
+}
+
+mlir::LogicalResult CIRToLLVMVAArgOpLowering::matchAndRewrite(
+    cir::VAArgOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  assert(!cir::MissingFeatures::vaArgABILowering());
+  auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
+  auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
+                                              adaptor.getArgList());
+
+  mlir::Type llvmType =
+      getTypeConverter()->convertType(op->getResultTypes().front());
+  if (!llvmType)
+    return mlir::failure();
+
+  rewriter.replaceOpWithNewOp<mlir::LLVM::VaArgOp>(op, llvmType, vaList);
+  return mlir::success();
+}
+
 std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
   return std::make_unique<ConvertCIRToLLVMPass>();
 }
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index c5106cb33f452..7b109c5cef9d3 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -44,6 +44,16 @@ class CIRToLLVMAssumeOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMAssumeAlignedOpLowering
+    : public mlir::OpConversionPattern<cir::AssumeAlignedOp> {
+public:
+  using mlir::OpConversionPattern<cir::AssumeAlignedOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::AssumeAlignedOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMAssumeSepStorageOpLowering
     : public mlir::OpConversionPattern<cir::AssumeSepStorageOp> {
 public:
@@ -199,6 +209,26 @@ class CIRToLLVMCallOpLowering : public mlir::OpConversionPattern<cir::CallOp> {
                   mlir::ConversionPatternRewriter &rewriter) const override;
 };
 
+class CIRToLLVMReturnAddrOpLowering
+    : public mlir::OpConversionPattern<cir::ReturnAddrOp> {
+public:
+  using mlir::OpConversionPattern<cir::ReturnAddrOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::ReturnAddrOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMFrameAddrOpLowering
+    : public mlir::OpConversionPattern<cir::FrameAddrOp> {
+public:
+  using mlir::OpConversionPattern<cir::FrameAddrOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::FrameAddrOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMAllocaOpLowering
     : public mlir::OpConversionPattern<cir::AllocaOp> {
   mlir::DataLayout const &dataLayout;
@@ -447,6 +477,37 @@ class CIRToLLVMBaseClassAddrOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMVTableAddrPointOpLowering
+    : public mlir::OpConversionPattern<cir::VTableAddrPointOp> {
+public:
+  using mlir::OpConversionPattern<cir::VTableAddrPointOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::VTableAddrPointOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMVTableGetVPtrOpLowering
+    : public mlir::OpConversionPattern<cir::VTableGetVPtrOp> {
+public:
+  using mlir::OpConversionPattern<cir::VTableGetVPtrOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::VTableGetVPtrOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMVTableGetVirtualFnAddrOpLowering
+    : public mlir::OpConversionPattern<cir::VTableGetVirtualFnAddrOp> {
+public:
+  using mlir::OpConversionPattern<
+      cir::VTableGetVirtualFnAddrOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::VTableGetVirtualFnAddrOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMStackSaveOpLowering
     : public mlir::OpConversionPattern<cir::StackSaveOp> {
 public:
@@ -638,6 +699,62 @@ class CIRToLLVMGetBitfieldOpLowering
                   mlir::ConversionPatternRewriter &) const override;
 };
 
+class CIRToLLVMFAbsOpLowering : public mlir::OpConversionPattern<cir::FAbsOp> {
+public:
+  using mlir::OpConversionPattern<cir::FAbsOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::FAbsOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMInlineAsmOpLowering
+    : public mlir::OpConversionPattern<cir::InlineAsmOp> {
+  mlir::DataLayout const &dataLayout;
+
+public:
+  CIRToLLVMInlineAsmOpLowering(const mlir::TypeConverter &typeConverter,
+                               mlir::MLIRContext *context,
+                               mlir::DataLayout const &dataLayout)
+      : OpConversionPattern(typeConverter, context), dataLayout(dataLayout) {}
+
+  using mlir::OpConversionPattern<cir::InlineAsmOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::InlineAsmOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMVAStartOpLowering
+    : public mlir::OpConversionPattern<cir::VAStartOp> {
+public:
+  using mlir::OpConversionPattern<cir::VAStartOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::VAStartOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMVAEndOpLowering
+    : public mlir::OpConversionPattern<cir::VAEndOp> {
+public:
+  using mlir::OpConversionPattern<cir::VAEndOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::VAEndOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
+class CIRToLLVMVAArgOpLowering
+    : public mlir::OpConversionPattern<cir::VAArgOp> {
+public:
+  using mlir::OpConversionPattern<cir::VAArgOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::VAArgOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 } // namespace direct
 } // namespace cir
 
diff --git a/clang/lib/CodeGen/ABIInfo.cpp b/clang/lib/CodeGen/ABIInfo.cpp
index 3ef430e19ebd3..83cc6377d502c 100644
--- a/clang/lib/CodeGen/ABIInfo.cpp
+++ b/clang/lib/CodeGen/ABIInfo.cpp
@@ -68,7 +68,7 @@ bool ABIInfo::isHomogeneousAggregate(QualType Ty, const Type *&Base,
       return false;
     Members *= NElements;
   } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->hasFlexibleArrayMember())
       return false;
 
@@ -244,6 +244,14 @@ ABIInfo::getOptimalVectorMemoryType(llvm::FixedVectorType *T,
   return T;
 }
 
+llvm::Value *ABIInfo::createCoercedLoad(Address SrcAddr, const ABIArgInfo &AI,
+                                        CodeGenFunction &CGF) const {
+  return nullptr;
+}
+
+void ABIInfo::createCoercedStore(llvm::Value *Val, Address DstAddr,
+                                 const ABIArgInfo &AI, bool DestIsVolatile,
+                                 CodeGenFunction &CGF) const {}
 // Pin the vtable to this file.
 SwiftABIInfo::~SwiftABIInfo() = default;
 
diff --git a/clang/lib/CodeGen/ABIInfo.h b/clang/lib/CodeGen/ABIInfo.h
index 9c7029c99bd44..130fcd375e90a 100644
--- a/clang/lib/CodeGen/ABIInfo.h
+++ b/clang/lib/CodeGen/ABIInfo.h
@@ -132,6 +132,12 @@ class ABIInfo {
   virtual llvm::FixedVectorType *
   getOptimalVectorMemoryType(llvm::FixedVectorType *T,
                              const LangOptions &Opt) const;
+
+  virtual llvm::Value *createCoercedLoad(Address SrcAddr, const ABIArgInfo &AI,
+                                         CodeGenFunction &CGF) const;
+  virtual void createCoercedStore(llvm::Value *Val, Address DstAddr,
+                                  const ABIArgInfo &AI, bool DestIsVolatile,
+                                  CodeGenFunction &CGF) const;
 };
 
 /// Target specific hooks for defining how a type should be passed or returned
diff --git a/clang/lib/CodeGen/ABIInfoImpl.cpp b/clang/lib/CodeGen/ABIInfoImpl.cpp
index 0a612d3461dc2..79dbe70a0c8eb 100644
--- a/clang/lib/CodeGen/ABIInfoImpl.cpp
+++ b/clang/lib/CodeGen/ABIInfoImpl.cpp
@@ -29,7 +29,7 @@ ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty) const {
 
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   ASTContext &Context = getContext();
   if (const auto *EIT = Ty->getAs<BitIntType>())
@@ -53,7 +53,7 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
 
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-    RetTy = EnumTy->getDecl()->getIntegerType();
+    RetTy = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   if (const auto *EIT = RetTy->getAs<BitIntType>())
     if (EIT->getNumBits() >
@@ -105,13 +105,12 @@ llvm::Type *CodeGen::getVAListElementType(CodeGenFunction &CGF) {
 
 CGCXXABI::RecordArgABI CodeGen::getRecordArgABI(const RecordType *RT,
                                                 CGCXXABI &CXXABI) {
-  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
-  if (!RD) {
-    if (!RT->getDecl()->canPassInRegisters())
-      return CGCXXABI::RAA_Indirect;
-    return CGCXXABI::RAA_Default;
-  }
-  return CXXABI.getRecordArgABI(RD);
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
+  if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD))
+    return CXXABI.getRecordArgABI(CXXRD);
+  if (!RD->canPassInRegisters())
+    return CGCXXABI::RAA_Indirect;
+  return CGCXXABI::RAA_Default;
 }
 
 CGCXXABI::RecordArgABI CodeGen::getRecordArgABI(QualType T, CGCXXABI &CXXABI) {
@@ -125,20 +124,21 @@ bool CodeGen::classifyReturnType(const CGCXXABI &CXXABI, CGFunctionInfo &FI,
                                  const ABIInfo &Info) {
   QualType Ty = FI.getReturnType();
 
-  if (const auto *RT = Ty->getAs<RecordType>())
-    if (!isa<CXXRecordDecl>(RT->getDecl()) &&
-        !RT->getDecl()->canPassInRegisters()) {
+  if (const auto *RT = Ty->getAs<RecordType>()) {
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
+    if (!isa<CXXRecordDecl>(RD) && !RD->canPassInRegisters()) {
       FI.getReturnInfo() = Info.getNaturalAlignIndirect(
           Ty, Info.getDataLayout().getAllocaAddrSpace());
       return true;
     }
+  }
 
   return CXXABI.classifyReturnType(FI);
 }
 
 QualType CodeGen::useFirstFieldIfTransparentUnion(QualType Ty) {
   if (const RecordType *UT = Ty->getAsUnionType()) {
-    const RecordDecl *UD = UT->getDecl();
+    const RecordDecl *UD = UT->getOriginalDecl()->getDefinitionOrSelf();
     if (UD->hasAttr<TransparentUnionAttr>()) {
       assert(!UD->field_empty() && "sema created an empty transparent union");
       return UD->field_begin()->getType();
@@ -276,7 +276,7 @@ bool CodeGen::isEmptyField(ASTContext &Context, const FieldDecl *FD,
   // according to the Itanium ABI.  The exception applies only to records,
   // not arrays of records, so we must also check whether we stripped off an
   // array type above.
-  if (isa<CXXRecordDecl>(RT->getDecl()) &&
+  if (isa<CXXRecordDecl>(RT->getOriginalDecl()) &&
       (WasArray || (!AsIfNoUniqueAddr && !FD->hasAttr<NoUniqueAddressAttr>())))
     return false;
 
@@ -288,7 +288,7 @@ bool CodeGen::isEmptyRecord(ASTContext &Context, QualType T, bool AllowArrays,
   const RecordType *RT = T->getAs<RecordType>();
   if (!RT)
     return false;
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
   if (RD->hasFlexibleArrayMember())
     return false;
 
@@ -320,7 +320,7 @@ bool CodeGen::isEmptyRecordForLayout(const ASTContext &Context, QualType T) {
   if (!RT)
     return false;
 
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
 
   // If this is a C++ record, check the bases first.
   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
@@ -344,7 +344,7 @@ const Type *CodeGen::isSingleElementStruct(QualType T, ASTContext &Context) {
   if (!RT)
     return nullptr;
 
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
   if (RD->hasFlexibleArrayMember())
     return nullptr;
 
@@ -463,7 +463,7 @@ bool CodeGen::isRecordWithSIMDVectorType(ASTContext &Context, QualType Ty) {
   const RecordType *RT = Ty->getAs<RecordType>();
   if (!RT)
     return false;
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
 
   // If this is a C++ record, check the bases first.
   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
diff --git a/clang/lib/CodeGen/Address.h b/clang/lib/CodeGen/Address.h
index a748ddaa110a5..4e7f3561ac049 100644
--- a/clang/lib/CodeGen/Address.h
+++ b/clang/lib/CodeGen/Address.h
@@ -176,6 +176,11 @@ class Address {
   static Address invalid() { return Address(nullptr); }
   bool isValid() const { return Pointer.getPointer() != nullptr; }
 
+  llvm::Value *getPointerIfNotSigned() const {
+    assert(isValid() && "pointer isn't valid");
+    return !isSigned() ? Pointer.getPointer() : nullptr;
+  }
+
   /// This function is used in situations where the caller is doing some sort of
   /// opaque "laundering" of the pointer.
   void replaceBasePointer(llvm::Value *P) {
diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index 0e80522536e15..74d92ef038eb9 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -188,13 +188,14 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
   // Optional copy/dispose helpers.
   bool hasInternalHelper = false;
   if (blockInfo.NeedsCopyDispose) {
+    auto &Schema = CGM.getCodeGenOpts().PointerAuth.BlockHelperFunctionPointers;
     // copy_func_helper_decl
     llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo);
-    elements.add(copyHelper);
+    elements.addSignedPointer(copyHelper, Schema, GlobalDecl(), QualType());
 
     // destroy_func_decl
     llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo);
-    elements.add(disposeHelper);
+    elements.addSignedPointer(disposeHelper, Schema, GlobalDecl(), QualType());
 
     if (cast<llvm::Function>(copyHelper->stripPointerCasts())
             ->hasInternalLinkage() ||
@@ -425,7 +426,8 @@ static bool isSafeForCXXConstantCapture(QualType type) {
   // Only records can be unsafe.
   if (!recordType) return true;
 
-  const auto *record = cast<CXXRecordDecl>(recordType->getDecl());
+  const auto *record =
+      cast<CXXRecordDecl>(recordType->getOriginalDecl())->getDefinitionOrSelf();
 
   // Maintain semantics for classes with non-trivial dtors or copy ctors.
   if (!record->hasTrivialDestructor()) return false;
@@ -567,9 +569,8 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
       llvm::StructType::get(CGM.getLLVMContext(), elementTypes, true);
     info.CanBeGlobal = true;
     return;
-  }
-  else if (C.getLangOpts().ObjC &&
-           CGM.getLangOpts().getGC() == LangOptions::NonGC)
+  } else if (C.getLangOpts().ObjC &&
+             CGM.getLangOpts().getGC() == LangOptions::NonGC)
     info.HasCapturedVariableLayout = true;
 
   if (block->doesNotEscape())
@@ -783,7 +784,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
 
 llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
   bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL;
-  auto GenVoidPtrTy =
+  llvm::PointerType *GenVoidPtrTy =
       IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy;
   LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default;
   auto GenVoidPtrSize = CharUnits::fromQuantity(
@@ -817,9 +818,6 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
                                    : CGM.getNSConcreteStackBlock();
     isa = blockISA;
 
-    // Build the block descriptor.
-    descriptor = buildBlockDescriptor(CGM, blockInfo);
-
     // Compute the initial on-stack block flags.
     if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
       flags = BLOCK_HAS_SIGNATURE;
@@ -833,6 +831,9 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
       flags |= BLOCK_USE_STRET;
     if (blockInfo.NoEscape)
       flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL;
+
+    // Build the block descriptor.
+    descriptor = buildBlockDescriptor(CGM, blockInfo);
   }
 
   auto projectField = [&](unsigned index, const Twine &name) -> Address {
@@ -883,11 +884,25 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
           llvm::ConstantInt::get(IntTy, blockInfo.BlockAlign.getQuantity()),
           getIntSize(), "block.align");
     }
-    addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
-    if (!IsOpenCL)
-      addHeaderField(descriptor, getPointerSize(), "block.descriptor");
-    else if (auto *Helper =
-                 CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
+
+    if (!IsOpenCL) {
+      llvm::Value *blockFnPtr =
+          llvm::ConstantExpr::getBitCast(InvokeFn, VoidPtrTy);
+      QualType type = blockInfo.getBlockExpr()
+                          ->getType()
+                          ->castAs<BlockPointerType>()
+                          ->getPointeeType();
+      addSignedHeaderField(
+          blockFnPtr,
+          CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers,
+          GlobalDecl(), type, getPointerSize(), "block.invoke");
+
+      addSignedHeaderField(
+          descriptor, CGM.getCodeGenOpts().PointerAuth.BlockDescriptorPointers,
+          GlobalDecl(), type, getPointerSize(), "block.descriptor");
+    } else if (auto *Helper =
+                   CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
+      addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
       for (auto I : Helper->getCustomFieldValues(*this, blockInfo)) {
         addHeaderField(
             I.first,
@@ -895,7 +910,8 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
                 CGM.getDataLayout().getTypeAllocSize(I.first->getType())),
             I.second);
       }
-    }
+    } else
+      addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
   }
 
   // Finally, capture all the values into the block.
@@ -1166,6 +1182,8 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
   ASTContext &Ctx = getContext();
   CallArgList Args;
 
+  llvm::Value *FuncPtr = nullptr;
+
   if (getLangOpts().OpenCL) {
     // For OpenCL, BlockPtr is already casted to generic block literal.
 
@@ -1185,7 +1203,7 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
     if (!isa<ParmVarDecl>(E->getCalleeDecl()))
       Func = CGM.getOpenCLRuntime().getInvokeFunction(E->getCallee());
     else {
-      llvm::Value *FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 2);
+      FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 2);
       Func = Builder.CreateAlignedLoad(GenericVoidPtrTy, FuncPtr,
                                        getPointerAlign());
     }
@@ -1194,7 +1212,7 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
     BlockPtr =
         Builder.CreatePointerCast(BlockPtr, UnqualPtrTy, "block.literal");
     // Get pointer to the block invoke function
-    llvm::Value *FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 3);
+    FuncPtr = Builder.CreateStructGEP(GenBlockTy, BlockPtr, 3);
 
     // First argument is a block literal casted to a void pointer
     BlockPtr = Builder.CreatePointerCast(BlockPtr, VoidPtrTy);
@@ -1211,7 +1229,15 @@ RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
     CGM.getTypes().arrangeBlockFunctionCall(Args, FuncTy);
 
   // Prepare the callee.
-  CGCallee Callee(CGCalleeInfo(), Func);
+  CGPointerAuthInfo PointerAuth;
+  if (auto &AuthSchema =
+          CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers) {
+    assert(FuncPtr != nullptr && "Missing function pointer for AuthInfo");
+    PointerAuth =
+        EmitPointerAuthInfo(AuthSchema, FuncPtr, GlobalDecl(), FnType);
+  }
+
+  CGCallee Callee(CGCalleeInfo(), Func, PointerAuth);
 
   // And call the block.
   return EmitCall(FnInfo, Callee, ReturnValue, Args, CallOrInvoke);
@@ -1295,14 +1321,15 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
 
   bool IsOpenCL = CGM.getLangOpts().OpenCL;
   bool IsWindows = CGM.getTarget().getTriple().isOSWindows();
+  auto &CGOPointerAuth = CGM.getCodeGenOpts().PointerAuth;
   if (!IsOpenCL) {
     // isa
     if (IsWindows)
       fields.addNullPointer(CGM.Int8PtrPtrTy);
     else
       fields.addSignedPointer(CGM.getNSConcreteGlobalBlock(),
-                              CGM.getCodeGenOpts().PointerAuth.ObjCIsaPointers,
-                              GlobalDecl(), QualType());
+                              CGOPointerAuth.ObjCIsaPointers, GlobalDecl(),
+                              QualType());
 
     // __flags
     BlockFlags flags = BLOCK_IS_GLOBAL;
@@ -1321,11 +1348,20 @@ static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
   }
 
   // Function
-  fields.add(blockFn);
+  if (auto &Schema = CGOPointerAuth.BlockInvocationFunctionPointers) {
+    QualType FnType = blockInfo.getBlockExpr()
+                          ->getType()
+                          ->castAs<BlockPointerType>()
+                          ->getPointeeType();
+    fields.addSignedPointer(blockFn, Schema, GlobalDecl(), FnType);
+  } else
+    fields.add(blockFn);
 
   if (!IsOpenCL) {
     // Descriptor
-    fields.add(buildBlockDescriptor(CGM, blockInfo));
+    llvm::Constant *Descriptor = buildBlockDescriptor(CGM, blockInfo);
+    fields.addSignedPointer(Descriptor, CGOPointerAuth.BlockDescriptorPointers,
+                            GlobalDecl(), QualType());
   } else if (auto *Helper =
                  CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
     for (auto *I : Helper->getCustomFieldValues(CGM, blockInfo)) {
@@ -1995,8 +2031,8 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
         // it. It's not quite worth the annoyance to avoid creating it in the
         // first place.
         if (!needsEHCleanup(captureType.isDestructedType()))
-          if (auto *I =
-                  cast_or_null<llvm::Instruction>(dstField.getBasePointer()))
+          if (auto *I = cast_or_null<llvm::Instruction>(
+                  dstField.getPointerIfNotSigned()))
             I->eraseFromParent();
       }
       break;
@@ -2730,8 +2766,16 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
   unsigned nextHeaderIndex = 0;
   CharUnits nextHeaderOffset;
   auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize,
-                              const Twine &name) {
+                              const Twine &name, bool isFunction = false) {
     auto fieldAddr = Builder.CreateStructGEP(addr, nextHeaderIndex, name);
+    if (isFunction) {
+      if (auto &Schema = CGM.getCodeGenOpts()
+                             .PointerAuth.BlockByrefHelperFunctionPointers) {
+        auto PointerAuth = EmitPointerAuthInfo(
+            Schema, fieldAddr.emitRawPointer(*this), GlobalDecl(), QualType());
+        value = EmitPointerAuthSign(PointerAuth, value);
+      }
+    }
     Builder.CreateStore(value, fieldAddr);
 
     nextHeaderIndex++;
@@ -2814,10 +2858,10 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
   storeHeaderField(V, getIntSize(), "byref.size");
 
   if (helpers) {
-    storeHeaderField(helpers->CopyHelper, getPointerSize(),
-                     "byref.copyHelper");
+    storeHeaderField(helpers->CopyHelper, getPointerSize(), "byref.copyHelper",
+                     /*isFunction=*/true);
     storeHeaderField(helpers->DisposeHelper, getPointerSize(),
-                     "byref.disposeHelper");
+                     "byref.disposeHelper", /*isFunction=*/true);
   }
 
   if (ByRefHasLifetime && HasByrefExtendedLayout) {
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index a648bde5f7544..d9cc37d123fb4 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -3326,9 +3326,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_ctz:
   case Builtin::BI__builtin_ctzl:
   case Builtin::BI__builtin_ctzll:
-  case Builtin::BI__builtin_ctzg: {
-    bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
-                       E->getNumArgs() > 1;
+  case Builtin::BI__builtin_ctzg:
+  case Builtin::BI__builtin_elementwise_cttz: {
+    bool HasFallback =
+        (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg ||
+         BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_cttz) &&
+        E->getNumArgs() > 1;
 
     Value *ArgValue =
         HasFallback ? EmitScalarExpr(E->getArg(0))
@@ -3338,8 +3341,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Intrinsic::cttz, ArgType);
 
     llvm::Type *ResultType = ConvertType(E->getType());
-    Value *ZeroUndef =
-        Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
+    // The elementwise builtins always exhibit zero-is-undef behaviour
+    Value *ZeroUndef = Builder.getInt1(
+        HasFallback || getTarget().isCLZForZeroUndef() ||
+        BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_cttz);
     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
     if (Result->getType() != ResultType)
       Result =
@@ -3358,9 +3363,12 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_clz:
   case Builtin::BI__builtin_clzl:
   case Builtin::BI__builtin_clzll:
-  case Builtin::BI__builtin_clzg: {
-    bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
-                       E->getNumArgs() > 1;
+  case Builtin::BI__builtin_clzg:
+  case Builtin::BI__builtin_elementwise_ctlz: {
+    bool HasFallback =
+        (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg ||
+         BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctlz) &&
+        E->getNumArgs() > 1;
 
     Value *ArgValue =
         HasFallback ? EmitScalarExpr(E->getArg(0))
@@ -3370,8 +3378,10 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Intrinsic::ctlz, ArgType);
 
     llvm::Type *ResultType = ConvertType(E->getType());
-    Value *ZeroUndef =
-        Builder.getInt1(HasFallback || getTarget().isCLZForZeroUndef());
+    // The elementwise builtins always exhibit zero-is-undef behaviour
+    Value *ZeroUndef = Builder.getInt1(
+        HasFallback || getTarget().isCLZForZeroUndef() ||
+        BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_ctlz);
     Value *Result = Builder.CreateCall(F, {ArgValue, ZeroUndef});
     if (Result->getType() != ResultType)
       Result =
@@ -4030,6 +4040,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_elementwise_fma:
     return RValue::get(
         emitBuiltinWithOneOverloadedType<3>(*this, E, Intrinsic::fma));
+  case Builtin::BI__builtin_elementwise_fshl:
+    return RValue::get(
+        emitBuiltinWithOneOverloadedType<3>(*this, E, Intrinsic::fshl));
+  case Builtin::BI__builtin_elementwise_fshr:
+    return RValue::get(
+        emitBuiltinWithOneOverloadedType<3>(*this, E, Intrinsic::fshr));
+
   case Builtin::BI__builtin_elementwise_add_sat:
   case Builtin::BI__builtin_elementwise_sub_sat: {
     Value *Op0 = EmitScalarExpr(E->getArg(0));
@@ -4238,6 +4255,44 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     return RValue::get(Result);
   }
 
+  case Builtin::BI__builtin_masked_load: {
+    llvm::Value *Mask = EmitScalarExpr(E->getArg(0));
+    llvm::Value *Ptr = EmitScalarExpr(E->getArg(1));
+
+    llvm::Type *RetTy = CGM.getTypes().ConvertType(E->getType());
+    CharUnits Align = CGM.getNaturalTypeAlignment(E->getType(), nullptr);
+    llvm::Value *AlignVal =
+        llvm::ConstantInt::get(Int32Ty, Align.getQuantity());
+
+    llvm::Value *PassThru = llvm::PoisonValue::get(RetTy);
+
+    Function *F =
+        CGM.getIntrinsic(Intrinsic::masked_load, {RetTy, UnqualPtrTy});
+
+    llvm::Value *Result =
+        Builder.CreateCall(F, {Ptr, AlignVal, Mask, PassThru}, "masked_load");
+    return RValue::get(Result);
+  };
+  case Builtin::BI__builtin_masked_store: {
+    llvm::Value *Mask = EmitScalarExpr(E->getArg(0));
+    llvm::Value *Val = EmitScalarExpr(E->getArg(1));
+    llvm::Value *Ptr = EmitScalarExpr(E->getArg(2));
+
+    QualType ValTy = E->getArg(1)->getType();
+    llvm::Type *ValLLTy = CGM.getTypes().ConvertType(ValTy);
+    llvm::Type *PtrTy = Ptr->getType();
+
+    CharUnits Align = CGM.getNaturalTypeAlignment(ValTy, nullptr);
+    llvm::Value *AlignVal =
+        llvm::ConstantInt::get(Int32Ty, Align.getQuantity());
+
+    llvm::Function *F =
+        CGM.getIntrinsic(llvm::Intrinsic::masked_store, {ValLLTy, PtrTy});
+
+    Builder.CreateCall(F, {Val, Ptr, AlignVal, Mask});
+    return RValue::get(nullptr);
+  }
+
   case Builtin::BI__builtin_isinf_sign: {
     // isinf_sign(x) -> fabs(x) == infinity ? (signbit(x) ? -1 : 1) : 0
     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(*this, E);
@@ -5985,8 +6040,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
 
     // Create a temporary array to hold the sizes of local pointer arguments
     // for the block. \p First is the position of the first size argument.
-    auto CreateArrayForSizeVar = [=](unsigned First)
-        -> std::tuple<llvm::Value *, llvm::Value *, llvm::Value *> {
+    auto CreateArrayForSizeVar =
+        [=](unsigned First) -> std::pair<llvm::Value *, llvm::Value *> {
       llvm::APInt ArraySize(32, NumArgs - First);
       QualType SizeArrayTy = getContext().getConstantArrayType(
           getContext().getSizeType(), ArraySize, nullptr,
@@ -5999,9 +6054,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       // actually the Alloca ascasted to the default AS, hence the
       // stripPointerCasts()
       llvm::Value *Alloca = TmpPtr->stripPointerCasts();
-      llvm::Value *TmpSize = EmitLifetimeStart(
-          CGM.getDataLayout().getTypeAllocSize(Tmp.getElementType()), Alloca);
       llvm::Value *ElemPtr;
+      EmitLifetimeStart(Alloca);
       // Each of the following arguments specifies the size of the corresponding
       // argument passed to the enqueued block.
       auto *Zero = llvm::ConstantInt::get(IntTy, 0);
@@ -6018,7 +6072,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       }
       // Return the Alloca itself rather than a potential ascast as this is only
       // used by the paired EmitLifetimeEnd.
-      return {ElemPtr, TmpSize, Alloca};
+      return {ElemPtr, Alloca};
     };
 
     // Could have events and/or varargs.
@@ -6030,7 +6084,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       llvm::Value *Kernel =
           Builder.CreatePointerCast(Info.KernelHandle, GenericVoidPtrTy);
       auto *Block = Builder.CreatePointerCast(Info.BlockArg, GenericVoidPtrTy);
-      auto [ElemPtr, TmpSize, TmpPtr] = CreateArrayForSizeVar(4);
+      auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(4);
 
       // Create a vector of the arguments, as well as a constant value to
       // express to the runtime the number of variadic arguments.
@@ -6045,8 +6099,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       llvm::FunctionType *FTy = llvm::FunctionType::get(Int32Ty, ArgTys, false);
       auto Call = RValue::get(
           EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Args));
-      if (TmpSize)
-        EmitLifetimeEnd(TmpSize, TmpPtr);
+      EmitLifetimeEnd(TmpPtr);
       return Call;
     }
     // Any calls now have event arguments passed.
@@ -6111,15 +6164,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       ArgTys.push_back(Int32Ty);
       Name = "__enqueue_kernel_events_varargs";
 
-      auto [ElemPtr, TmpSize, TmpPtr] = CreateArrayForSizeVar(7);
+      auto [ElemPtr, TmpPtr] = CreateArrayForSizeVar(7);
       Args.push_back(ElemPtr);
       ArgTys.push_back(ElemPtr->getType());
 
       llvm::FunctionType *FTy = llvm::FunctionType::get(Int32Ty, ArgTys, false);
       auto Call = RValue::get(
           EmitRuntimeCall(CGM.CreateRuntimeFunction(FTy, Name), Args));
-      if (TmpSize)
-        EmitLifetimeEnd(TmpSize, TmpPtr);
+      EmitLifetimeEnd(TmpPtr);
       return Call;
     }
     llvm_unreachable("Unexpected enqueue_kernel signature");
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index dd26be74e561b..c7f4bf8a21354 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -1131,7 +1131,8 @@ void CGNVCUDARuntime::handleVarRegistration(const VarDecl *D,
     // Builtin surfaces and textures and their template arguments are
     // also registered with CUDA runtime.
     const auto *TD = cast<ClassTemplateSpecializationDecl>(
-        D->getType()->castAs<RecordType>()->getDecl());
+                         D->getType()->castAs<RecordType>()->getOriginalDecl())
+                         ->getDefinitionOrSelf();
     const TemplateArgumentList &Args = TD->getTemplateArgs();
     if (TD->hasAttr<CUDADeviceBuiltinSurfaceTypeAttr>()) {
       assert(Args.size() == 2 &&
diff --git a/clang/lib/CodeGen/CGCXX.cpp b/clang/lib/CodeGen/CGCXX.cpp
index 78a7b021855b7..f9aff893eb0f0 100644
--- a/clang/lib/CodeGen/CGCXX.cpp
+++ b/clang/lib/CodeGen/CGCXX.cpp
@@ -83,8 +83,9 @@ bool CodeGenModule::TryEmitBaseDestructorAsAlias(const CXXDestructorDecl *D) {
     if (I.isVirtual()) continue;
 
     // Skip base classes with trivial destructors.
-    const auto *Base =
-        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+    const auto *Base = cast<CXXRecordDecl>(
+                           I.getType()->castAs<RecordType>()->getOriginalDecl())
+                           ->getDefinitionOrSelf();
     if (Base->hasTrivialDestructor()) continue;
 
     // If we've already found a base class with a non-trivial
@@ -277,18 +278,18 @@ static CGCallee BuildAppleKextVirtualCall(CodeGenFunction &CGF,
 /// BuildAppleKextVirtualCall - This routine is to support gcc's kext ABI making
 /// indirect call to virtual functions. It makes the call through indexing
 /// into the vtable.
-CGCallee
-CodeGenFunction::BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
-                                           NestedNameSpecifier *Qual,
-                                           llvm::Type *Ty) {
-  assert((Qual->getKind() == NestedNameSpecifier::TypeSpec) &&
+CGCallee CodeGenFunction::BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
+                                                    NestedNameSpecifier Qual,
+                                                    llvm::Type *Ty) {
+  assert(Qual.getKind() == NestedNameSpecifier::Kind::Type &&
          "BuildAppleKextVirtualCall - bad Qual kind");
 
-  const Type *QTy = Qual->getAsType();
+  const Type *QTy = Qual.getAsType();
   QualType T = QualType(QTy, 0);
   const RecordType *RT = T->getAs<RecordType>();
   assert(RT && "BuildAppleKextVirtualCall - Qual type must be record");
-  const auto *RD = cast<CXXRecordDecl>(RT->getDecl());
+  const auto *RD =
+      cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
 
   if (const auto *DD = dyn_cast<CXXDestructorDecl>(MD))
     return BuildAppleKextVirtualDestructorCall(DD, Dtor_Complete, RD);
diff --git a/clang/lib/CodeGen/CGCXXABI.cpp b/clang/lib/CodeGen/CGCXXABI.cpp
index d42e0bb814a88..cca675838644e 100644
--- a/clang/lib/CodeGen/CGCXXABI.cpp
+++ b/clang/lib/CodeGen/CGCXXABI.cpp
@@ -52,7 +52,7 @@ CGCallee CGCXXABI::EmitLoadOfMemberFunctionPointer(
 
   const auto *RD = MPT->getMostRecentCXXRecordDecl();
   ThisPtrForCall =
-      CGF.getAsNaturalPointerTo(This, CGF.getContext().getRecordType(RD));
+      CGF.getAsNaturalPointerTo(This, CGF.getContext().getCanonicalTagType(RD));
   const FunctionProtoType *FPT =
       MPT->getPointeeType()->getAs<FunctionProtoType>();
   llvm::Constant *FnPtr = llvm::Constant::getNullValue(
@@ -106,7 +106,7 @@ CGCXXABI::EmitNullMemberPointer(const MemberPointerType *MPT) {
 
 llvm::Constant *CGCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
   return GetBogusMemberPointer(CGM.getContext().getMemberPointerType(
-      MD->getType(), /*Qualifier=*/nullptr, MD->getParent()));
+      MD->getType(), /*Qualifier=*/std::nullopt, MD->getParent()));
 }
 
 llvm::Constant *CGCXXABI::EmitMemberDataPointer(const MemberPointerType *MPT,
diff --git a/clang/lib/CodeGen/CGCXXABI.h b/clang/lib/CodeGen/CGCXXABI.h
index 96fe04661944d..2dd320dbda976 100644
--- a/clang/lib/CodeGen/CGCXXABI.h
+++ b/clang/lib/CodeGen/CGCXXABI.h
@@ -294,14 +294,22 @@ class CGCXXABI {
                                              Address Value,
                                              QualType SrcRecordTy) = 0;
 
+  struct ExactDynamicCastInfo {
+    bool RequiresCastToPrimaryBase;
+    CharUnits Offset;
+  };
+
+  virtual std::optional<ExactDynamicCastInfo>
+  getExactDynamicCastInfo(QualType SrcRecordTy, QualType DestTy,
+                          QualType DestRecordTy) = 0;
+
   /// Emit a dynamic_cast from SrcRecordTy to DestRecordTy. The cast fails if
   /// the dynamic type of Value is not exactly DestRecordTy.
-  virtual llvm::Value *emitExactDynamicCast(CodeGenFunction &CGF, Address Value,
-                                            QualType SrcRecordTy,
-                                            QualType DestTy,
-                                            QualType DestRecordTy,
-                                            llvm::BasicBlock *CastSuccess,
-                                            llvm::BasicBlock *CastFail) = 0;
+  virtual llvm::Value *emitExactDynamicCast(
+      CodeGenFunction &CGF, Address Value, QualType SrcRecordTy,
+      QualType DestTy, QualType DestRecordTy,
+      const ExactDynamicCastInfo &CastInfo, llvm::BasicBlock *CastSuccess,
+      llvm::BasicBlock *CastFail) = 0;
 
   virtual bool EmitBadCastCall(CodeGenFunction &CGF) = 0;
 
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index d9bd443455e0f..9c4fa12f18b20 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -125,16 +125,16 @@ unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
 /// calling a method pointer.
 CanQualType CodeGenTypes::DeriveThisType(const CXXRecordDecl *RD,
                                          const CXXMethodDecl *MD) {
-  QualType RecTy;
+  CanQualType RecTy;
   if (RD)
-    RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
+    RecTy = Context.getCanonicalTagType(RD);
   else
     RecTy = Context.VoidTy;
 
   if (MD)
-    RecTy = Context.getAddrSpaceQualType(
-        RecTy, MD->getMethodQualifiers().getAddressSpace());
-  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
+    RecTy = CanQualType::CreateUnsafe(Context.getAddrSpaceQualType(
+        RecTy, MD->getMethodQualifiers().getAddressSpace()));
+  return Context.getPointerType(RecTy);
 }
 
 /// Returns the canonical formal type of the given C++ method.
@@ -1008,7 +1008,7 @@ getTypeExpansion(QualType Ty, const ASTContext &Context) {
   if (const RecordType *RT = Ty->getAs<RecordType>()) {
     SmallVector<const CXXBaseSpecifier *, 1> Bases;
     SmallVector<const FieldDecl *, 1> Fields;
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     assert(!RD->hasFlexibleArrayMember() &&
            "Cannot expand structure with flexible array.");
     if (RD->isUnion()) {
@@ -1602,6 +1602,7 @@ void ClangToLLVMArgMapping::construct(const ASTContext &Context,
       IRArgs.PaddingArgIndex = IRArgNo++;
 
     switch (AI.getKind()) {
+    case ABIArgInfo::TargetSpecific:
     case ABIArgInfo::Extend:
     case ABIArgInfo::Direct: {
       // FIXME: handle sseregparm someday...
@@ -1712,6 +1713,7 @@ llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
   case ABIArgInfo::IndirectAliased:
     llvm_unreachable("Invalid ABI kind for return argument");
 
+  case ABIArgInfo::TargetSpecific:
   case ABIArgInfo::Extend:
   case ABIArgInfo::Direct:
     resultType = retAI.getCoerceToType();
@@ -1784,6 +1786,7 @@ llvm::FunctionType *CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
       ArgTypes[FirstIRArg] = llvm::PointerType::get(
           getLLVMContext(), ArgInfo.getIndirectAddrSpace());
       break;
+    case ABIArgInfo::TargetSpecific:
     case ABIArgInfo::Extend:
     case ABIArgInfo::Direct: {
       // Fast-isel and the optimizer generally like scalar values better than
@@ -1895,7 +1898,7 @@ bool CodeGenModule::MayDropFunctionReturn(const ASTContext &Context,
   // complex destructor or a non-trivially copyable type.
   if (const RecordType *RT =
           ReturnType.getCanonicalType()->getAs<RecordType>()) {
-    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
+    if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl()))
       return ClassDecl->hasTrivialDestructor();
   }
   return ReturnType.isTriviallyCopyableType(Context);
@@ -2697,6 +2700,7 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
     else
       RetAttrs.addAttribute(llvm::Attribute::NoExt);
     [[fallthrough]];
+  case ABIArgInfo::TargetSpecific:
   case ABIArgInfo::Direct:
     if (RetAI.getInReg())
       RetAttrs.addAttribute(llvm::Attribute::InReg);
@@ -2838,6 +2842,7 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
       else
         Attrs.addAttribute(llvm::Attribute::NoExt);
       [[fallthrough]];
+    case ABIArgInfo::TargetSpecific:
     case ABIArgInfo::Direct:
       if (ArgNo == 0 && FI.isChainCall())
         Attrs.addAttribute(llvm::Attribute::Nest);
@@ -2870,7 +2875,8 @@ void CodeGenModule::ConstructAttributeList(StringRef Name,
           // (e.g., Obj-C ARC-managed structs, MSVC callee-destroyed objects).
           if (!ParamType.isDestructedType() || !ParamType->isRecordType() ||
               ParamType->castAs<RecordType>()
-                  ->getDecl()
+                  ->getOriginalDecl()
+                  ->getDefinitionOrSelf()
                   ->isParamDestroyedInCallee())
             Attrs.addAttribute(llvm::Attribute::DeadOnReturn);
         }
@@ -3356,17 +3362,6 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
         }
       }
 
-      // Struct of fixed-length vectors and struct of array of fixed-length
-      // vector in VLS calling convention are coerced to vector tuple
-      // type(represented as TargetExtType) and scalable vector type
-      // respectively, they're no longer handled as struct.
-      if (ArgI.isDirect() && isa<llvm::StructType>(ConvertType(Ty)) &&
-          (isa<llvm::TargetExtType>(ArgI.getCoerceToType()) ||
-           isa<llvm::ScalableVectorType>(ArgI.getCoerceToType()))) {
-        ArgVals.push_back(ParamValue::forDirect(AI));
-        break;
-      }
-
       llvm::StructType *STy =
           dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
       Address Alloca =
@@ -3507,6 +3502,25 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       break;
     }
 
+    case ABIArgInfo::TargetSpecific: {
+      auto *AI = Fn->getArg(FirstIRArg);
+      AI->setName(Arg->getName() + ".target_coerce");
+      Address Alloca =
+          CreateMemTemp(Ty, getContext().getDeclAlign(Arg), Arg->getName());
+      Address Ptr = emitAddressAtOffset(*this, Alloca, ArgI);
+      CGM.getABIInfo().createCoercedStore(AI, Ptr, ArgI, false, *this);
+      if (CodeGenFunction::hasScalarEvaluationKind(Ty)) {
+        llvm::Value *V =
+            EmitLoadOfScalar(Alloca, false, Ty, Arg->getBeginLoc());
+        if (isPromoted) {
+          V = emitArgumentDemotion(*this, Arg, V);
+        }
+        ArgVals.push_back(ParamValue::forDirect(V));
+      } else {
+        ArgVals.push_back(ParamValue::forIndirect(Alloca));
+      }
+      break;
+    }
     case ABIArgInfo::Ignore:
       assert(NumIRArgs == 0);
       // Initialize the local variable appropriately.
@@ -3828,7 +3842,7 @@ static void setUsedBits(CodeGenModule &CGM, const RecordType *RTy, int Offset,
                         SmallVectorImpl<uint64_t> &Bits) {
   ASTContext &Context = CGM.getContext();
   int CharWidth = Context.getCharWidth();
-  const RecordDecl *RD = RTy->getDecl()->getDefinition();
+  const RecordDecl *RD = RTy->getOriginalDecl()->getDefinition();
   const ASTRecordLayout &ASTLayout = Context.getASTRecordLayout(RD);
   const CGRecordLayout &Layout = CGM.getTypes().getCGRecordLayout(RD);
 
@@ -4135,6 +4149,11 @@ void CodeGenFunction::EmitFunctionEpilog(
     }
     break;
   }
+  case ABIArgInfo::TargetSpecific: {
+    Address V = emitAddressAtOffset(*this, ReturnValue, RetAI);
+    RV = CGM.getABIInfo().createCoercedLoad(V, RetAI, *this);
+    break;
+  }
   case ABIArgInfo::Expand:
   case ABIArgInfo::IndirectAliased:
     llvm_unreachable("Invalid ABI kind for return argument");
@@ -4289,7 +4308,10 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
 
   // Deactivate the cleanup for the callee-destructed param that was pushed.
   if (type->isRecordType() && !CurFuncIsThunk &&
-      type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
+      type->castAs<RecordType>()
+          ->getOriginalDecl()
+          ->getDefinitionOrSelf()
+          ->isParamDestroyedInCallee() &&
       param->needsDestruction(getContext())) {
     EHScopeStack::stable_iterator cleanup =
         CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
@@ -4319,10 +4341,7 @@ static void emitWriteback(CodeGenFunction &CGF,
 
   if (writeback.WritebackExpr) {
     CGF.EmitIgnoredExpr(writeback.WritebackExpr);
-
-    if (writeback.LifetimeSz)
-      CGF.EmitLifetimeEnd(writeback.LifetimeSz,
-                          writeback.Temporary.getBasePointer());
+    CGF.EmitLifetimeEnd(writeback.Temporary.getBasePointer());
     return;
   }
 
@@ -4885,8 +4904,10 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
   // In the Microsoft C++ ABI, aggregate arguments are destructed by the callee.
   // However, we still have to push an EH-only cleanup in case we unwind before
   // we make it to the call.
-  if (type->isRecordType() &&
-      type->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
+  if (type->isRecordType() && type->castAs<RecordType>()
+                                  ->getOriginalDecl()
+                                  ->getDefinitionOrSelf()
+                                  ->isParamDestroyedInCallee()) {
     // If we're using inalloca, use the argument memory.  Otherwise, use a
     // temporary.
     AggValueSlot Slot = args.isUsingInAlloca()
@@ -5240,9 +5261,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     // since otherwise we could be making a conditional call after a check for
     // the proper cpu features (and it won't cause code generation issues due to
     // function based code generation).
-    if (TargetDecl->hasAttr<AlwaysInlineAttr>() &&
-        (TargetDecl->hasAttr<TargetAttr>() ||
-         (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>())))
+    if ((TargetDecl->hasAttr<AlwaysInlineAttr>() &&
+         (TargetDecl->hasAttr<TargetAttr>() ||
+          (CurFuncDecl && CurFuncDecl->hasAttr<TargetAttr>()))) ||
+        (CurFuncDecl && CurFuncDecl->hasAttr<FlattenAttr>() &&
+         (CurFuncDecl->hasAttr<TargetAttr>() ||
+          TargetDecl->hasAttr<TargetAttr>())))
       checkTargetFeatures(Loc, FD);
   }
 
@@ -5282,7 +5306,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // If the call returns a temporary with struct return, create a temporary
   // alloca to hold the result, unless one is given to us.
   Address SRetPtr = Address::invalid();
-  llvm::Value *UnusedReturnSizePtr = nullptr;
+  bool NeedSRetLifetimeEnd = false;
   if (RetAI.isIndirect() || RetAI.isInAlloca() || RetAI.isCoerceAndExpand()) {
     // For virtual function pointer thunks and musttail calls, we must always
     // forward an incoming SRet pointer to the callee, because a local alloca
@@ -5296,11 +5320,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       SRetPtr = ReturnValue.getAddress();
     } else {
       SRetPtr = CreateMemTempWithoutCast(RetTy, "tmp");
-      if (HaveInsertPoint() && ReturnValue.isUnused()) {
-        llvm::TypeSize size =
-            CGM.getDataLayout().getTypeAllocSize(ConvertTypeForMem(RetTy));
-        UnusedReturnSizePtr = EmitLifetimeStart(size, SRetPtr.getBasePointer());
-      }
+      if (HaveInsertPoint() && ReturnValue.isUnused())
+        NeedSRetLifetimeEnd = EmitLifetimeStart(SRetPtr.getBasePointer());
     }
     if (IRFunctionArgs.hasSRetArg()) {
       // A mismatch between the allocated return value's AS and the target's
@@ -5484,15 +5505,10 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         Val = Builder.CreateFreeze(Val);
       IRCallArgs[FirstIRArg] = Val;
 
-      // Emit lifetime markers for the temporary alloca.
-      llvm::TypeSize ByvalTempElementSize =
-          CGM.getDataLayout().getTypeAllocSize(AI.getElementType());
-      llvm::Value *LifetimeSize =
-          EmitLifetimeStart(ByvalTempElementSize, AI.getPointer());
-
-      // Add cleanup code to emit the end lifetime marker after the call.
-      if (LifetimeSize) // In case we disabled lifetime markers.
-        CallLifetimeEndAfterCall.emplace_back(AI, LifetimeSize);
+      // Emit lifetime markers for the temporary alloca and add cleanup code to
+      // emit the end lifetime marker after the call.
+      if (EmitLifetimeStart(AI.getPointer()))
+        CallLifetimeEndAfterCall.emplace_back(AI);
 
       // Generate the copy.
       I->copyInto(*this, AI);
@@ -5653,9 +5669,9 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       auto unpaddedCoercionType = ArgInfo.getUnpaddedCoerceAndExpandType();
       auto *unpaddedStruct = dyn_cast<llvm::StructType>(unpaddedCoercionType);
 
-      llvm::Value *tempSize = nullptr;
       Address addr = Address::invalid();
       RawAddress AllocaAddr = RawAddress::invalid();
+      bool NeedLifetimeEnd = false;
       if (I->isAggregate()) {
         addr = I->hasLValue() ? I->getKnownLValue().getAddress()
                               : I->getKnownRValue().getAggregateAddress();
@@ -5665,7 +5681,6 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         assert(RV.isScalar()); // complex should always just be direct
 
         llvm::Type *scalarType = RV.getScalarVal()->getType();
-        auto scalarSize = CGM.getDataLayout().getTypeAllocSize(scalarType);
         auto scalarAlign = CGM.getDataLayout().getPrefTypeAlign(scalarType);
 
         // Materialize to a temporary.
@@ -5674,7 +5689,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
                                     layout->getAlignment(), scalarAlign)),
                                 "tmp",
                                 /*ArraySize=*/nullptr, &AllocaAddr);
-        tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
+        NeedLifetimeEnd = EmitLifetimeStart(AllocaAddr.getPointer());
 
         Builder.CreateStore(RV.getScalarVal(), addr);
       }
@@ -5699,10 +5714,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       }
       assert(IRArgPos == FirstIRArg + NumIRArgs);
 
-      if (tempSize) {
-        EmitLifetimeEnd(tempSize, AllocaAddr.getPointer());
-      }
-
+      if (NeedLifetimeEnd)
+        EmitLifetimeEnd(AllocaAddr.getPointer());
       break;
     }
 
@@ -5712,6 +5725,24 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       assert(IRArgPos == FirstIRArg + NumIRArgs);
       break;
     }
+
+    case ABIArgInfo::TargetSpecific: {
+      Address Src = Address::invalid();
+      if (!I->isAggregate()) {
+        Src = CreateMemTemp(I->Ty, "target_coerce");
+        I->copyInto(*this, Src);
+      } else {
+        Src = I->hasLValue() ? I->getKnownLValue().getAddress()
+                             : I->getKnownRValue().getAggregateAddress();
+      }
+
+      // If the value is offset in memory, apply the offset now.
+      Src = emitAddressAtOffset(*this, Src, ArgInfo);
+      llvm::Value *Load =
+          CGM.getABIInfo().createCoercedLoad(Src, ArgInfo, *this);
+      IRCallArgs[FirstIRArg] = Load;
+      break;
+    }
     }
   }
 
@@ -5871,9 +5902,8 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // can't depend on being inside of an ExprWithCleanups, so we need to manually
   // pop this cleanup later on. Being eager about this is OK, since this
   // temporary is 'invisible' outside of the callee.
-  if (UnusedReturnSizePtr)
-    pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetPtr,
-                                         UnusedReturnSizePtr);
+  if (NeedSRetLifetimeEnd)
+    pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, SRetPtr);
 
   llvm::BasicBlock *InvokeDest = CannotThrow ? nullptr : getInvokeDest();
 
@@ -6007,7 +6037,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   // insertion point; this allows the rest of IRGen to discard
   // unreachable code.
   if (CI->doesNotReturn()) {
-    if (UnusedReturnSizePtr)
+    if (NeedSRetLifetimeEnd)
       PopCleanupBlock();
 
     // Strip away the noreturn attribute to better diagnose unreachable UB.
@@ -6122,7 +6152,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       case ABIArgInfo::InAlloca:
       case ABIArgInfo::Indirect: {
         RValue ret = convertTempToRValue(SRetPtr, RetTy, SourceLocation());
-        if (UnusedReturnSizePtr)
+        if (NeedSRetLifetimeEnd)
           PopCleanupBlock();
         return ret;
       }
@@ -6198,6 +6228,19 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         return convertTempToRValue(DestPtr, RetTy, SourceLocation());
       }
 
+      case ABIArgInfo::TargetSpecific: {
+        Address DestPtr = ReturnValue.getValue();
+        Address StorePtr = emitAddressAtOffset(*this, DestPtr, RetAI);
+        bool DestIsVolatile = ReturnValue.isVolatile();
+        if (!DestPtr.isValid()) {
+          DestPtr = CreateMemTemp(RetTy, "target_coerce");
+          DestIsVolatile = false;
+        }
+        CGM.getABIInfo().createCoercedStore(CI, StorePtr, RetAI, DestIsVolatile,
+                                            *this);
+        return convertTempToRValue(DestPtr, RetTy, SourceLocation());
+      }
+
       case ABIArgInfo::Expand:
       case ABIArgInfo::IndirectAliased:
         llvm_unreachable("Invalid ABI kind for return argument");
diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h
index 0b4e3f9cb0365..3157b7f16f294 100644
--- a/clang/lib/CodeGen/CGCall.h
+++ b/clang/lib/CodeGen/CGCall.h
@@ -289,9 +289,6 @@ class CallArgList : public SmallVector<CallArg, 8> {
     /// An Expression (optional) that performs the writeback with any required
     /// casting.
     const Expr *WritebackExpr;
-
-    // Size for optional lifetime end on the temporary.
-    llvm::Value *LifetimeSz;
   };
 
   struct CallArgCleanup {
@@ -321,9 +318,8 @@ class CallArgList : public SmallVector<CallArg, 8> {
   }
 
   void addWriteback(LValue srcLV, Address temporary, llvm::Value *toUse,
-                    const Expr *writebackExpr = nullptr,
-                    llvm::Value *lifetimeSz = nullptr) {
-    Writeback writeback = {srcLV, temporary, toUse, writebackExpr, lifetimeSz};
+                    const Expr *writebackExpr = nullptr) {
+    Writeback writeback = {srcLV, temporary, toUse, writebackExpr};
     Writebacks.push_back(writeback);
   }
 
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index 4a465e6526da0..e9a92ae0f01cb 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -181,7 +181,9 @@ CharUnits CodeGenModule::computeNonVirtualBaseClassOffset(
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
 
     const auto *BaseDecl =
-        cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(
+            Base->getType()->castAs<RecordType>()->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     // Add the offset.
     Offset += Layout.getBaseClassOffset(BaseDecl);
@@ -301,7 +303,8 @@ Address CodeGenFunction::GetAddressOfBaseClass(
   // and hence will not require any further steps.
   if ((*Start)->isVirtual()) {
     VBase = cast<CXXRecordDecl>(
-        (*Start)->getType()->castAs<RecordType>()->getDecl());
+                (*Start)->getType()->castAs<RecordType>()->getOriginalDecl())
+                ->getDefinitionOrSelf();
     ++Start;
   }
 
@@ -326,7 +329,7 @@ Address CodeGenFunction::GetAddressOfBaseClass(
   llvm::Type *PtrTy = llvm::PointerType::get(
       CGM.getLLVMContext(), Value.getType()->getPointerAddressSpace());
 
-  QualType DerivedTy = getContext().getRecordType(Derived);
+  CanQualType DerivedTy = getContext().getCanonicalTagType(Derived);
   CharUnits DerivedAlign = CGM.getClassPointerAlignment(Derived);
 
   // If the static offset is zero and we don't have a virtual step,
@@ -401,8 +404,7 @@ CodeGenFunction::GetAddressOfDerivedClass(Address BaseAddr,
                                           bool NullCheckValue) {
   assert(PathBegin != PathEnd && "Base path should not be empty!");
 
-  QualType DerivedTy =
-      getContext().getCanonicalType(getContext().getTagDeclType(Derived));
+  CanQualType DerivedTy = getContext().getCanonicalTagType(Derived);
   llvm::Type *DerivedValueTy = ConvertType(DerivedTy);
 
   llvm::Value *NonVirtualOffset =
@@ -559,7 +561,8 @@ static void EmitBaseInitializer(CodeGenFunction &CGF,
 
   const Type *BaseType = BaseInit->getBaseClass();
   const auto *BaseClassDecl =
-      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getOriginalDecl())
+          ->getDefinitionOrSelf();
 
   bool isBaseVirtual = BaseInit->isBaseVirtual();
 
@@ -638,7 +641,7 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
   QualType FieldType = Field->getType();
 
   llvm::Value *ThisPtr = CGF.LoadCXXThis();
-  QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
+  CanQualType RecordTy = CGF.getContext().getCanonicalTagType(ClassDecl);
   LValue LHS;
 
   // If a base constructor is being emitted, create an LValue that has the
@@ -974,7 +977,7 @@ namespace {
       }
 
       CharUnits MemcpySize = getMemcpySize(FirstByteOffset);
-      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
+      CanQualType RecordTy = CGF.getContext().getCanonicalTagType(ClassDecl);
       Address ThisPtr = CGF.LoadCXXThisAddress();
       LValue DestLV = CGF.MakeAddrLValue(ThisPtr, RecordTy);
       LValue Dest = CGF.EmitLValueForFieldInitialization(DestLV, FirstField);
@@ -1122,7 +1125,7 @@ namespace {
 
     void pushEHDestructors() {
       Address ThisPtr = CGF.LoadCXXThisAddress();
-      QualType RecordTy = CGF.getContext().getTypeDeclType(ClassDecl);
+      CanQualType RecordTy = CGF.getContext().getCanonicalTagType(ClassDecl);
       LValue LHS = CGF.MakeAddrLValue(ThisPtr, RecordTy);
 
       for (unsigned i = 0; i < AggregatedInits.size(); ++i) {
@@ -1265,7 +1268,8 @@ namespace {
 static bool isInitializerOfDynamicClass(const CXXCtorInitializer *BaseInit) {
   const Type *BaseType = BaseInit->getBaseClass();
   const auto *BaseClassDecl =
-      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(BaseType->castAs<RecordType>()->getOriginalDecl())
+          ->getDefinitionOrSelf();
   return BaseClassDecl->isDynamicClass();
 }
 
@@ -1374,7 +1378,9 @@ HasTrivialDestructorBody(ASTContext &Context,
       continue;
 
     const CXXRecordDecl *NonVirtualBase =
-      cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(
+            I.getType()->castAs<RecordType>()->getOriginalDecl())
+            ->getDefinitionOrSelf();
     if (!HasTrivialDestructorBody(Context, NonVirtualBase,
                                   MostDerivedClassDecl))
       return false;
@@ -1383,8 +1389,10 @@ HasTrivialDestructorBody(ASTContext &Context,
   if (BaseClassDecl == MostDerivedClassDecl) {
     // Check virtual bases.
     for (const auto &I : BaseClassDecl->vbases()) {
-      const CXXRecordDecl *VirtualBase =
-        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+      const auto *VirtualBase =
+          cast<CXXRecordDecl>(
+              I.getType()->castAs<RecordType>()->getOriginalDecl())
+              ->getDefinitionOrSelf();
       if (!HasTrivialDestructorBody(Context, VirtualBase,
                                     MostDerivedClassDecl))
         return false;
@@ -1404,7 +1412,8 @@ FieldHasTrivialDestructorBody(ASTContext &Context,
   if (!RT)
     return true;
 
-  CXXRecordDecl *FieldClassDecl = cast<CXXRecordDecl>(RT->getDecl());
+  auto *FieldClassDecl =
+      cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
 
   // The destructor for an implicit anonymous union member is never invoked.
   if (FieldClassDecl->isUnion() && FieldClassDecl->isAnonymousStructOrUnion())
@@ -1588,7 +1597,7 @@ namespace {
       const CXXRecordDecl *ClassDecl = Dtor->getParent();
       CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
                          LoadThisForDtorDelete(CGF, Dtor),
-                         CGF.getContext().getTagDeclType(ClassDecl));
+                         CGF.getContext().getCanonicalTagType(ClassDecl));
     }
   };
 
@@ -1606,7 +1615,7 @@ namespace {
     const CXXRecordDecl *ClassDecl = Dtor->getParent();
     CGF.EmitDeleteCall(Dtor->getOperatorDelete(),
                        LoadThisForDtorDelete(CGF, Dtor),
-                       CGF.getContext().getTagDeclType(ClassDecl));
+                       CGF.getContext().getCanonicalTagType(ClassDecl));
     assert(Dtor->getOperatorDelete()->isDestroyingOperatorDelete() ==
                ReturnAfterDelete &&
            "unexpected value for ReturnAfterDelete");
@@ -1647,7 +1656,8 @@ namespace {
     void Emit(CodeGenFunction &CGF, Flags flags) override {
       // Find the address of the field.
       Address thisValue = CGF.LoadCXXThisAddress();
-      QualType RecordTy = CGF.getContext().getTagDeclType(field->getParent());
+      CanQualType RecordTy =
+          CGF.getContext().getCanonicalTagType(field->getParent());
       LValue ThisLV = CGF.MakeAddrLValue(thisValue, RecordTy);
       LValue LV = CGF.EmitLValueForField(ThisLV, field);
       assert(LV.isSimple());
@@ -1870,7 +1880,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
         const CXXRecordDecl *ClassDecl = DD->getParent();
         EmitDeleteCall(DD->getOperatorDelete(),
                        LoadThisForDtorDelete(*this, DD),
-                       getContext().getTagDeclType(ClassDecl));
+                       getContext().getCanonicalTagType(ClassDecl));
         EmitBranchThroughCleanup(ReturnBlock);
       } else {
         EHStack.pushCleanup<CallDtorDelete>(NormalAndEHCleanup);
@@ -1898,7 +1908,9 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
     // the reverse order.
     for (const auto &Base : ClassDecl->vbases()) {
       auto *BaseClassDecl =
-          cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
+          cast<CXXRecordDecl>(
+              Base.getType()->castAs<RecordType>()->getOriginalDecl())
+              ->getDefinitionOrSelf();
 
       if (BaseClassDecl->hasTrivialDestructor()) {
         // Under SanitizeMemoryUseAfterDtor, poison the trivial base class
@@ -1964,7 +1976,7 @@ void CodeGenFunction::EnterDtorCleanups(const CXXDestructorDecl *DD,
 
     // Anonymous union members do not have their destructors called.
     const RecordType *RT = type->getAsUnionType();
-    if (RT && RT->getDecl()->isAnonymousStructOrUnion())
+    if (RT && RT->getOriginalDecl()->isAnonymousStructOrUnion())
       continue;
 
     CleanupKind cleanupKind = getCleanupKind(dtorKind);
@@ -2057,7 +2069,7 @@ void CodeGenFunction::EmitCXXAggrConstructorCall(const CXXConstructorDecl *ctor,
   //
   // Note that these are complete objects and so we don't need to
   // use the non-virtual size or alignment.
-  QualType type = getContext().getTypeDeclType(ctor->getParent());
+  CanQualType type = getContext().getCanonicalTagType(ctor->getParent());
   CharUnits eltAlignment =
     arrayBase.getAlignment()
              .alignmentOfArrayElement(getContext().getTypeSizeInChars(type));
@@ -2119,7 +2131,8 @@ void CodeGenFunction::destroyCXXObject(CodeGenFunction &CGF,
                                        Address addr,
                                        QualType type) {
   const RecordType *rtype = type->castAs<RecordType>();
-  const CXXRecordDecl *record = cast<CXXRecordDecl>(rtype->getDecl());
+  const auto *record =
+      cast<CXXRecordDecl>(rtype->getOriginalDecl())->getDefinitionOrSelf();
   const CXXDestructorDecl *dtor = record->getDestructor();
   assert(!dtor->isTrivial());
   CGF.EmitCXXDestructorCall(dtor, Dtor_Complete, /*for vbase*/ false,
@@ -2158,7 +2171,7 @@ void CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
 
     const Expr *Arg = E->getArg(0);
     LValue Src = EmitLValue(Arg);
-    QualType DestTy = getContext().getTypeDeclType(D->getParent());
+    CanQualType DestTy = getContext().getCanonicalTagType(D->getParent());
     LValue Dest = MakeAddrLValue(This, DestTy);
     EmitAggregateCopyCtor(Dest, Src, ThisAVS.mayOverlap());
     return;
@@ -2210,7 +2223,8 @@ void CodeGenFunction::EmitCXXConstructorCall(
 
   if (!NewPointerIsChecked)
     EmitTypeCheck(CodeGenFunction::TCK_ConstructorCall, Loc, This,
-                  getContext().getRecordType(ClassDecl), CharUnits::Zero());
+                  getContext().getCanonicalTagType(ClassDecl),
+                  CharUnits::Zero());
 
   if (D->isTrivial() && D->isDefaultConstructor()) {
     assert(Args.size() == 1 && "trivial default ctor with args");
@@ -2226,7 +2240,7 @@ void CodeGenFunction::EmitCXXConstructorCall(
     Address Src = makeNaturalAddressForPointer(
         Args[1].getRValue(*this).getScalarVal(), SrcTy);
     LValue SrcLVal = MakeAddrLValue(Src, SrcTy);
-    QualType DestTy = getContext().getTypeDeclType(ClassDecl);
+    CanQualType DestTy = getContext().getCanonicalTagType(ClassDecl);
     LValue DestLVal = MakeAddrLValue(This, DestTy);
     EmitAggregateCopyCtor(DestLVal, SrcLVal, Overlap);
     return;
@@ -2638,8 +2652,9 @@ void CodeGenFunction::getVTablePointers(BaseSubobject Base,
 
   // Traverse bases.
   for (const auto &I : RD->bases()) {
-    auto *BaseDecl =
-        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+    auto *BaseDecl = cast<CXXRecordDecl>(
+                         I.getType()->castAs<RecordType>()->getOriginalDecl())
+                         ->getDefinitionOrSelf();
 
     // Ignore classes without a vtable.
     if (!BaseDecl->isDynamicClass())
@@ -2772,7 +2787,7 @@ void CodeGenFunction::EmitTypeMetadataCodeForVCall(const CXXRecordDecl *RD,
            // Don't insert type test assumes if we are forcing public
            // visibility.
            !CGM.AlwaysHasLTOVisibilityPublic(RD)) {
-    QualType Ty = QualType(RD->getTypeForDecl(), 0);
+    CanQualType Ty = CGM.getContext().getCanonicalTagType(RD);
     llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(Ty);
     llvm::Value *TypeId =
         llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
@@ -2839,7 +2854,8 @@ void CodeGenFunction::EmitVTablePtrCheckForCast(QualType T, Address Derived,
   if (!ClassTy)
     return;
 
-  const CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(ClassTy->getDecl());
+  const auto *ClassDecl =
+      cast<CXXRecordDecl>(ClassTy->getOriginalDecl())->getDefinitionOrSelf();
 
   if (!ClassDecl->isCompleteDefinition() || !ClassDecl->isDynamicClass())
     return;
@@ -2896,8 +2912,8 @@ void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
 
   EmitSanitizerStatReport(SSK);
 
-  llvm::Metadata *MD =
-      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
+  CanQualType T = CGM.getContext().getCanonicalTagType(RD);
+  llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T);
   llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
 
   llvm::Value *TypeTest = Builder.CreateCall(
@@ -2906,7 +2922,7 @@ void CodeGenFunction::EmitVTablePtrCheck(const CXXRecordDecl *RD,
   llvm::Constant *StaticData[] = {
       llvm::ConstantInt::get(Int8Ty, TCK),
       EmitCheckSourceLocation(Loc),
-      EmitCheckTypeDescriptor(QualType(RD->getTypeForDecl(), 0)),
+      EmitCheckTypeDescriptor(T),
   };
 
   auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
@@ -2956,8 +2972,8 @@ llvm::Value *CodeGenFunction::EmitVTableTypeCheckedLoad(
 
   EmitSanitizerStatReport(llvm::SanStat_CFI_VCall);
 
-  llvm::Metadata *MD =
-      CGM.CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
+  CanQualType T = CGM.getContext().getCanonicalTagType(RD);
+  llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(T);
   llvm::Value *TypeId = llvm::MetadataAsValue::get(CGM.getLLVMContext(), MD);
 
   auto CheckedLoadIntrinsic = CGM.getVTables().useRelativeLayout()
@@ -3039,7 +3055,8 @@ void CodeGenFunction::EmitLambdaBlockInvokeBody() {
   // Start building arguments for forwarding call
   CallArgList CallArgs;
 
-  QualType ThisType = getContext().getPointerType(getContext().getRecordType(Lambda));
+  CanQualType ThisType =
+      getContext().getPointerType(getContext().getCanonicalTagType(Lambda));
   Address ThisPtr = GetAddrOfBlockDecl(variable);
   CallArgs.add(RValue::get(getAsNaturalPointerTo(ThisPtr, ThisType)), ThisType);
 
@@ -3066,8 +3083,8 @@ void CodeGenFunction::EmitLambdaStaticInvokeBody(const CXXMethodDecl *MD) {
   // Start building arguments for forwarding call
   CallArgList CallArgs;
 
-  QualType LambdaType = getContext().getRecordType(Lambda);
-  QualType ThisType = getContext().getPointerType(LambdaType);
+  CanQualType LambdaType = getContext().getCanonicalTagType(Lambda);
+  CanQualType ThisType = getContext().getPointerType(LambdaType);
   Address ThisPtr = CreateMemTemp(LambdaType, "unused.capture");
   CallArgs.add(RValue::get(ThisPtr.emitRawPointer(*this)), ThisType);
 
@@ -3118,8 +3135,8 @@ void CodeGenFunction::EmitLambdaInAllocaCallOpBody(const CXXMethodDecl *MD) {
 
   // Forward %this argument.
   CallArgList CallArgs;
-  QualType LambdaType = getContext().getRecordType(MD->getParent());
-  QualType ThisType = getContext().getPointerType(LambdaType);
+  CanQualType LambdaType = getContext().getCanonicalTagType(MD->getParent());
+  CanQualType ThisType = getContext().getPointerType(LambdaType);
   llvm::Value *ThisArg = CurFn->getArg(0);
   CallArgs.add(RValue::get(ThisArg), ThisType);
 
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 2b469f2e265b4..c44fea3e6383d 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -366,7 +366,7 @@ llvm::DIScope *CGDebugInfo::getContextDescriptor(const Decl *Context,
 
   if (const auto *RDecl = dyn_cast<RecordDecl>(Context))
     if (!RDecl->isDependentType())
-      return getOrCreateType(CGM.getContext().getTypeDeclType(RDecl),
+      return getOrCreateType(CGM.getContext().getCanonicalTagType(RDecl),
                              TheCU->getFile());
   return Default;
 }
@@ -1285,7 +1285,7 @@ static bool needsTypeIdentifier(const TagDecl *TD, CodeGenModule &CGM,
 static SmallString<256> getTypeIdentifier(const TagType *Ty, CodeGenModule &CGM,
                                           llvm::DICompileUnit *TheCU) {
   SmallString<256> Identifier;
-  const TagDecl *TD = Ty->getDecl();
+  const TagDecl *TD = Ty->getOriginalDecl()->getDefinitionOrSelf();
 
   if (!needsTypeIdentifier(TD, CGM, TheCU))
     return Identifier;
@@ -1321,8 +1321,8 @@ static llvm::dwarf::Tag getTagForRecord(const RecordDecl *RD) {
 llvm::DICompositeType *
 CGDebugInfo::getOrCreateRecordFwdDecl(const RecordType *Ty,
                                       llvm::DIScope *Ctx) {
-  const RecordDecl *RD = Ty->getDecl();
-  if (llvm::DIType *T = getTypeOrNull(CGM.getContext().getRecordType(RD)))
+  const RecordDecl *RD = Ty->getOriginalDecl()->getDefinitionOrSelf();
+  if (llvm::DIType *T = getTypeOrNull(QualType(Ty, 0)))
     return cast<llvm::DICompositeType>(T);
   llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation());
   const unsigned Line =
@@ -2015,6 +2015,8 @@ void CGDebugInfo::CollectRecordNestedType(
     const TypeDecl *TD, SmallVectorImpl<llvm::Metadata *> &elements) {
   QualType Ty = CGM.getContext().getTypeDeclType(TD);
   // Injected class names are not considered nested records.
+  // FIXME: Is this supposed to be testing for injected class name declarations
+  // instead?
   if (isa<InjectedClassNameType>(Ty))
     return;
   SourceLocation Loc = TD->getLocation();
@@ -2356,7 +2358,9 @@ void CGDebugInfo::CollectCXXBasesAux(
   const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
   for (const auto &BI : Bases) {
     const auto *Base =
-        cast<CXXRecordDecl>(BI.getType()->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(
+            BI.getType()->castAs<RecordType>()->getOriginalDecl())
+            ->getDefinition();
     if (!SeenTypes.insert(Base).second)
       continue;
     auto *BaseTy = getOrCreateType(BI.getType(), Unit);
@@ -2825,12 +2829,12 @@ void CGDebugInfo::addHeapAllocSiteMetadata(llvm::CallBase *CI,
 void CGDebugInfo::completeType(const EnumDecl *ED) {
   if (DebugKind <= llvm::codegenoptions::DebugLineTablesOnly)
     return;
-  QualType Ty = CGM.getContext().getEnumType(ED);
+  CanQualType Ty = CGM.getContext().getCanonicalTagType(ED);
   void *TyPtr = Ty.getAsOpaquePtr();
   auto I = TypeCache.find(TyPtr);
   if (I == TypeCache.end() || !cast<llvm::DIType>(I->second)->isForwardDecl())
     return;
-  llvm::DIType *Res = CreateTypeDefinition(Ty->castAs<EnumType>());
+  llvm::DIType *Res = CreateTypeDefinition(dyn_cast<EnumType>(Ty));
   assert(!Res->isForwardDecl());
   TypeCache[TyPtr].reset(Res);
 }
@@ -2900,7 +2904,7 @@ void CGDebugInfo::completeClassData(const RecordDecl *RD) {
 void CGDebugInfo::completeClass(const RecordDecl *RD) {
   if (DebugKind <= llvm::codegenoptions::DebugLineTablesOnly)
     return;
-  QualType Ty = CGM.getContext().getRecordType(RD);
+  CanQualType Ty = CGM.getContext().getCanonicalTagType(RD);
   void *TyPtr = Ty.getAsOpaquePtr();
   auto I = TypeCache.find(TyPtr);
   if (I != TypeCache.end() && !cast<llvm::DIType>(I->second)->isForwardDecl())
@@ -2909,7 +2913,7 @@ void CGDebugInfo::completeClass(const RecordDecl *RD) {
   // We want the canonical definition of the structure to not
   // be the typedef. Since that would lead to circular typedef
   // metadata.
-  auto [Res, PrefRes] = CreateTypeDefinition(Ty->castAs<RecordType>());
+  auto [Res, PrefRes] = CreateTypeDefinition(dyn_cast<RecordType>(Ty));
   assert(!Res->isForwardDecl());
   TypeCache[TyPtr].reset(Res);
 }
@@ -3013,14 +3017,14 @@ void CGDebugInfo::completeRequiredType(const RecordDecl *RD) {
   if (shouldOmitDefinition(DebugKind, DebugTypeExtRefs, RD, CGM.getLangOpts()))
     return;
 
-  QualType Ty = CGM.getContext().getRecordType(RD);
+  CanQualType Ty = CGM.getContext().getCanonicalTagType(RD);
   llvm::DIType *T = getTypeOrNull(Ty);
   if (T && T->isForwardDecl())
     completeClassData(RD);
 }
 
 llvm::DIType *CGDebugInfo::CreateType(const RecordType *Ty) {
-  RecordDecl *RD = Ty->getDecl();
+  RecordDecl *RD = Ty->getOriginalDecl()->getDefinitionOrSelf();
   llvm::DIType *T = cast_or_null<llvm::DIType>(getTypeOrNull(QualType(Ty, 0)));
   if (T || shouldOmitDefinition(DebugKind, DebugTypeExtRefs, RD,
                                 CGM.getLangOpts())) {
@@ -3048,7 +3052,7 @@ llvm::DIType *CGDebugInfo::GetPreferredNameType(const CXXRecordDecl *RD,
 
 std::pair<llvm::DIType *, llvm::DIType *>
 CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) {
-  RecordDecl *RD = Ty->getDecl();
+  RecordDecl *RD = Ty->getOriginalDecl()->getDefinitionOrSelf();
 
   // Get overall information about the record type for the debug info.
   llvm::DIFile *DefUnit = getOrCreateFile(RD->getLocation());
@@ -3070,7 +3074,7 @@ CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) {
 
   // Push the struct on region stack.
   LexicalBlockStack.emplace_back(&*FwdDecl);
-  RegionMap[Ty->getDecl()].reset(FwdDecl);
+  RegionMap[RD].reset(FwdDecl);
 
   // Convert all the elements.
   SmallVector<llvm::Metadata *, 16> EltTys;
@@ -3092,7 +3096,7 @@ CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) {
     CollectCXXMemberFunctions(CXXDecl, DefUnit, EltTys, FwdDecl);
 
   LexicalBlockStack.pop_back();
-  RegionMap.erase(Ty->getDecl());
+  RegionMap.erase(RD);
 
   llvm::DINodeArray Elements = DBuilder.getOrCreateArray(EltTys);
   DBuilder.replaceArrays(FwdDecl, Elements);
@@ -3101,7 +3105,7 @@ CGDebugInfo::CreateTypeDefinition(const RecordType *Ty) {
     FwdDecl =
         llvm::MDNode::replaceWithPermanent(llvm::TempDICompositeType(FwdDecl));
 
-  RegionMap[Ty->getDecl()].reset(FwdDecl);
+  RegionMap[RD].reset(FwdDecl);
 
   if (CGM.getCodeGenOpts().getDebuggerTuning() == llvm::DebuggerKind::LLDB)
     if (auto *PrefDI = GetPreferredNameType(CXXDecl, DefUnit))
@@ -3651,8 +3655,9 @@ llvm::DIType *CGDebugInfo::CreateType(const MemberPointerType *Ty,
     }
   }
 
-  llvm::DIType *ClassType = getOrCreateType(
-      QualType(Ty->getMostRecentCXXRecordDecl()->getTypeForDecl(), 0), U);
+  CanQualType T =
+      CGM.getContext().getCanonicalTagType(Ty->getMostRecentCXXRecordDecl());
+  llvm::DIType *ClassType = getOrCreateType(T, U);
   if (Ty->isMemberDataPointerType())
     return DBuilder.createMemberPointerType(
         getOrCreateType(Ty->getPointeeType(), U), ClassType, Size, /*Align=*/0,
@@ -3687,17 +3692,21 @@ llvm::DIType *CGDebugInfo::CreateType(const HLSLInlineSpirvType *Ty,
   return nullptr;
 }
 
-llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) {
-  const EnumDecl *ED = Ty->getDecl();
+static auto getEnumInfo(CodeGenModule &CGM, llvm::DICompileUnit *TheCU,
+                        const EnumType *Ty) {
+  const EnumDecl *ED = Ty->getOriginalDecl()->getDefinitionOrSelf();
 
   uint64_t Size = 0;
   uint32_t Align = 0;
-  if (!ED->getTypeForDecl()->isIncompleteType()) {
-    Size = CGM.getContext().getTypeSize(ED->getTypeForDecl());
+  if (ED->isComplete()) {
+    Size = CGM.getContext().getTypeSize(QualType(Ty, 0));
     Align = getDeclAlignIfRequired(ED, CGM.getContext());
   }
+  return std::make_tuple(ED, Size, Align, getTypeIdentifier(Ty, CGM, TheCU));
+}
 
-  SmallString<256> Identifier = getTypeIdentifier(Ty, CGM, TheCU);
+llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) {
+  auto [ED, Size, Align, Identifier] = getEnumInfo(CGM, TheCU, Ty);
 
   bool isImportedFromModule =
       DebugTypeExtRefs && ED->isFromASTFile() && ED->getDefinition();
@@ -3732,15 +3741,7 @@ llvm::DIType *CGDebugInfo::CreateEnumType(const EnumType *Ty) {
 }
 
 llvm::DIType *CGDebugInfo::CreateTypeDefinition(const EnumType *Ty) {
-  const EnumDecl *ED = Ty->getDecl();
-  uint64_t Size = 0;
-  uint32_t Align = 0;
-  if (!ED->getTypeForDecl()->isIncompleteType()) {
-    Size = CGM.getContext().getTypeSize(ED->getTypeForDecl());
-    Align = getDeclAlignIfRequired(ED, CGM.getContext());
-  }
-
-  SmallString<256> Identifier = getTypeIdentifier(Ty, CGM, TheCU);
+  auto [ED, Size, Align, Identifier] = getEnumInfo(CGM, TheCU, Ty);
 
   SmallVector<llvm::Metadata *, 16> Enumerators;
   ED = ED->getDefinition();
@@ -3815,6 +3816,11 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) {
     switch (T->getTypeClass()) {
     default:
       return C.getQualifiedType(T.getTypePtr(), Quals);
+    case Type::Enum:
+    case Type::Record:
+    case Type::InjectedClassName:
+      return C.getQualifiedType(T->getCanonicalTypeUnqualified().getTypePtr(),
+                                Quals);
     case Type::TemplateSpecialization: {
       const auto *Spec = cast<TemplateSpecializationType>(T);
       if (Spec->isTypeAlias())
@@ -3843,11 +3849,8 @@ static QualType UnwrapTypeForDebugInfo(QualType T, const ASTContext &C) {
     case Type::CountAttributed:
       T = cast<CountAttributedType>(T)->desugar();
       break;
-    case Type::Elaborated:
-      T = cast<ElaboratedType>(T)->getNamedType();
-      break;
     case Type::Using:
-      T = cast<UsingType>(T)->getUnderlyingType();
+      T = cast<UsingType>(T)->desugar();
       break;
     case Type::Paren:
       T = cast<ParenType>(T)->getInnerType();
@@ -3906,7 +3909,8 @@ void CGDebugInfo::completeUnusedClass(const CXXRecordDecl &D) {
   completeClassData(&D);
   // In case this type has no member function definitions being emitted, ensure
   // it is retained
-  RetainedTypes.push_back(CGM.getContext().getRecordType(&D).getAsOpaquePtr());
+  RetainedTypes.push_back(
+      CGM.getContext().getCanonicalTagType(&D).getAsOpaquePtr());
 }
 
 llvm::DIType *CGDebugInfo::getOrCreateType(QualType Ty, llvm::DIFile *Unit) {
@@ -4051,7 +4055,6 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
   case Type::Adjusted:
   case Type::Decayed:
   case Type::DeducedTemplateSpecialization:
-  case Type::Elaborated:
   case Type::Using:
   case Type::Paren:
   case Type::MacroQualified:
@@ -4094,7 +4097,7 @@ CGDebugInfo::getOrCreateLimitedType(const RecordType *Ty) {
 
 // TODO: Currently used for context chains when limiting debug info.
 llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
-  RecordDecl *RD = Ty->getDecl();
+  RecordDecl *RD = Ty->getOriginalDecl()->getDefinitionOrSelf();
 
   // Get overall information about the record type for the debug info.
   StringRef RDName = getClassName(RD);
@@ -4111,7 +4114,7 @@ llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
   // If we ended up creating the type during the context chain construction,
   // just return that.
   auto *T = cast_or_null<llvm::DICompositeType>(
-      getTypeOrNull(CGM.getContext().getRecordType(RD)));
+      getTypeOrNull(CGM.getContext().getCanonicalTagType(RD)));
   if (T && (!T->isForwardDecl() || !RD->getDefinition()))
     return T;
 
@@ -4181,7 +4184,14 @@ llvm::DICompositeType *CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
     break;
   }
 
-  RegionMap[Ty->getDecl()].reset(RealDecl);
+  if (auto *CTSD =
+          dyn_cast<ClassTemplateSpecializationDecl>(Ty->getOriginalDecl())) {
+    CXXRecordDecl *TemplateDecl =
+        CTSD->getSpecializedTemplate()->getTemplatedDecl();
+    RegionMap[TemplateDecl].reset(RealDecl);
+  } else {
+    RegionMap[RD].reset(RealDecl);
+  }
   TypeCache[QualType(Ty, 0).getAsOpaquePtr()].reset(RealDecl);
 
   if (const auto *TSpecial = dyn_cast<ClassTemplateSpecializationDecl>(RD))
@@ -4205,8 +4215,8 @@ void CGDebugInfo::CollectContainingType(const CXXRecordDecl *RD,
       else
         break;
     }
-    ContainingType = getOrCreateType(QualType(PBase->getTypeForDecl(), 0),
-                                     getOrCreateFile(RD->getLocation()));
+    CanQualType T = CGM.getContext().getCanonicalTagType(PBase);
+    ContainingType = getOrCreateType(T, getOrCreateFile(RD->getLocation()));
   } else if (RD->isDynamicClass())
     ContainingType = RealDecl;
 
@@ -4412,9 +4422,10 @@ llvm::DINode *CGDebugInfo::getDeclarationOrDefinition(const Decl *D) {
   // we would otherwise do to get a type for a pointee. (forward declarations in
   // limited debug info, full definitions (if the type definition is available)
   // in unlimited debug info)
-  if (const auto *TD = dyn_cast<TypeDecl>(D))
-    return getOrCreateType(CGM.getContext().getTypeDeclType(TD),
-                           getOrCreateFile(TD->getLocation()));
+  if (const auto *TD = dyn_cast<TypeDecl>(D)) {
+    QualType Ty = CGM.getContext().getTypeDeclType(TD);
+    return getOrCreateType(Ty, getOrCreateFile(TD->getLocation()));
+  }
   auto I = DeclCache.find(D->getCanonicalDecl());
 
   if (I != DeclCache.end()) {
@@ -5076,7 +5087,7 @@ llvm::DILocalVariable *CGDebugInfo::EmitDeclare(const VarDecl *VD,
   } else if (const auto *RT = dyn_cast<RecordType>(VD->getType())) {
     // If VD is an anonymous union then Storage represents value for
     // all union fields.
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->isUnion() && RD->isAnonymousStructOrUnion()) {
       // GDB has trouble finding local variables in anonymous unions, so we emit
       // artificial local variables for each of the members.
@@ -5536,7 +5547,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
               cast_or_null<CXXMethodDecl>(blockDecl->getNonClosureContext()))
         type = Method->getThisType();
       else if (auto *RDecl = dyn_cast<CXXRecordDecl>(blockDecl->getParent()))
-        type = QualType(RDecl->getTypeForDecl(), 0);
+        type = CGM.getContext().getCanonicalTagType(RDecl);
       else
         llvm_unreachable("unexpected block declcontext");
 
@@ -5626,8 +5637,9 @@ llvm::DIGlobalVariableExpression *CGDebugInfo::CollectAnonRecordDecls(
     // Ignore unnamed fields, but recurse into anonymous records.
     if (FieldName.empty()) {
       if (const auto *RT = dyn_cast<RecordType>(Field->getType()))
-        GVE = CollectAnonRecordDecls(RT->getDecl(), Unit, LineNo, LinkageName,
-                                     Var, DContext);
+        GVE =
+            CollectAnonRecordDecls(RT->getOriginalDecl()->getDefinitionOrSelf(),
+                                   Unit, LineNo, LinkageName, Var, DContext);
       continue;
     }
     // Use VarDecl's Tag, Scope and Line number.
@@ -5646,7 +5658,7 @@ static bool ReferencesAnonymousEntity(RecordType *RT) {
   // But so long as it's not one of those, it doesn't matter if some sub-type
   // of the record (a template parameter) can't be reconstituted - because the
   // un-reconstitutable type itself will carry its own name.
-  const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
+  const auto *RD = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl());
   if (!RD)
     return false;
   if (!RD->getIdentifier())
@@ -5705,15 +5717,15 @@ struct ReconstitutableType : public RecursiveASTVisitor<ReconstitutableType> {
     }
     return true;
   }
-  bool TraverseEnumType(EnumType *ET) {
+  bool TraverseEnumType(EnumType *ET, bool = false) {
     // Unnamed enums can't be reconstituted due to a lack of column info we
     // produce in the DWARF, so we can't get Clang's full name back.
-    if (const auto *ED = dyn_cast<EnumDecl>(ET->getDecl())) {
+    if (const auto *ED = dyn_cast<EnumDecl>(ET->getOriginalDecl())) {
       if (!ED->getIdentifier()) {
         Reconstitutable = false;
         return false;
       }
-      if (!ED->isExternallyVisible()) {
+      if (!ED->getDefinitionOrSelf()->isExternallyVisible()) {
         Reconstitutable = false;
         return false;
       }
@@ -5726,7 +5738,7 @@ struct ReconstitutableType : public RecursiveASTVisitor<ReconstitutableType> {
     Reconstitutable &= !FT->getNoReturnAttr();
     return Reconstitutable;
   }
-  bool VisitRecordType(RecordType *RT) {
+  bool VisitRecordType(RecordType *RT, bool = false) {
     if (ReferencesAnonymousEntity(RT)) {
       Reconstitutable = false;
       return false;
@@ -5909,7 +5921,8 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
   // variable for each member of the anonymous union so that it's possible
   // to find the name of any field in the union.
   if (T->isUnionType() && DeclName.empty()) {
-    const RecordDecl *RD = T->castAs<RecordType>()->getDecl();
+    const RecordDecl *RD =
+        T->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
     assert(RD->isAnonymousStructOrUnion() &&
            "unnamed non-anonymous struct or union?");
     GVE = CollectAnonRecordDecls(RD, Unit, LineNo, LinkageName, Var, DContext);
@@ -5956,8 +5969,6 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, const APValue &Init) {
 
   if (const auto *ECD = dyn_cast<EnumConstantDecl>(VD)) {
     const auto *ED = cast<EnumDecl>(ECD->getDeclContext());
-    assert(isa<EnumType>(ED->getTypeForDecl()) && "Enum without EnumType?");
-
     if (CGM.getCodeGenOpts().EmitCodeView) {
       // If CodeView, emit enums as global variables, unless they are defined
       // inside a class. We do this because MSVC doesn't emit S_CONSTANTs for
@@ -5969,10 +5980,9 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, const APValue &Init) {
       // If not CodeView, emit DW_TAG_enumeration_type if necessary. For
       // example: for "enum { ZERO };", a DW_TAG_enumeration_type is created the
       // first time `ZERO` is referenced in a function.
-      llvm::DIType *EDTy =
-          getOrCreateType(QualType(ED->getTypeForDecl(), 0), Unit);
-      assert (EDTy->getTag() == llvm::dwarf::DW_TAG_enumeration_type);
-      (void)EDTy;
+      CanQualType T = CGM.getContext().getCanonicalTagType(ED);
+      [[maybe_unused]] llvm::DIType *EDTy = getOrCreateType(T, Unit);
+      assert(EDTy->getTag() == llvm::dwarf::DW_TAG_enumeration_type);
       return;
     }
   }
@@ -5991,7 +6001,7 @@ void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, const APValue &Init) {
     // FIXME: This is probably unnecessary, since Ty should reference RD
     // through its scope.
     RetainedTypes.push_back(
-        CGM.getContext().getRecordType(RD).getAsOpaquePtr());
+        CGM.getContext().getCanonicalTagType(RD).getAsOpaquePtr());
 
     return;
   }
diff --git a/clang/lib/CodeGen/CGDebugInfo.h b/clang/lib/CodeGen/CGDebugInfo.h
index 497d3a6ab17b1..ff9c3cd2d1136 100644
--- a/clang/lib/CodeGen/CGDebugInfo.h
+++ b/clang/lib/CodeGen/CGDebugInfo.h
@@ -977,6 +977,8 @@ class ApplyInlineDebugLocation {
   ApplyInlineDebugLocation(CodeGenFunction &CGF, GlobalDecl InlinedFn);
   /// Restore everything back to the original state.
   ~ApplyInlineDebugLocation();
+  ApplyInlineDebugLocation(const ApplyInlineDebugLocation &) = delete;
+  ApplyInlineDebugLocation &operator=(ApplyInlineDebugLocation &) = delete;
 };
 
 class SanitizerDebugLocation {
@@ -988,6 +990,8 @@ class SanitizerDebugLocation {
                          ArrayRef<SanitizerKind::SanitizerOrdinal> Ordinals,
                          SanitizerHandler Handler);
   ~SanitizerDebugLocation();
+  SanitizerDebugLocation(const SanitizerDebugLocation &) = delete;
+  SanitizerDebugLocation &operator=(SanitizerDebugLocation &) = delete;
 };
 
 } // namespace CodeGen
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index ff2dadad0d4ef..8a1675848e13c 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -113,12 +113,14 @@ void CodeGenFunction::EmitDecl(const Decl &D, bool EvaluateConditionDecl) {
   case Decl::CXXRecord: // struct/union/class X; [C++]
     if (CGDebugInfo *DI = getDebugInfo())
       if (cast<RecordDecl>(D).getDefinition())
-        DI->EmitAndRetainType(getContext().getRecordType(cast<RecordDecl>(&D)));
+        DI->EmitAndRetainType(
+            getContext().getCanonicalTagType(cast<RecordDecl>(&D)));
     return;
   case Decl::Enum:      // enum X;
     if (CGDebugInfo *DI = getDebugInfo())
       if (cast<EnumDecl>(D).getDefinition())
-        DI->EmitAndRetainType(getContext().getEnumType(cast<EnumDecl>(&D)));
+        DI->EmitAndRetainType(
+            getContext().getCanonicalTagType(cast<EnumDecl>(&D)));
     return;
   case Decl::Function:     // void X();
   case Decl::EnumConstant: // enum ? { X = ? }
@@ -1351,30 +1353,27 @@ void CodeGenFunction::EmitAutoVarDecl(const VarDecl &D) {
 }
 
 /// Emit a lifetime.begin marker if some criteria are satisfied.
-/// \return a pointer to the temporary size Value if a marker was emitted, null
-/// otherwise
-llvm::Value *CodeGenFunction::EmitLifetimeStart(llvm::TypeSize Size,
-                                                llvm::Value *Addr) {
+/// \return whether the marker was emitted.
+bool CodeGenFunction::EmitLifetimeStart(llvm::Value *Addr) {
   if (!ShouldEmitLifetimeMarkers)
-    return nullptr;
+    return false;
 
   assert(Addr->getType()->getPointerAddressSpace() ==
              CGM.getDataLayout().getAllocaAddrSpace() &&
          "Pointer should be in alloca address space");
-  llvm::Value *SizeV = llvm::ConstantInt::get(
-      Int64Ty, Size.isScalable() ? -1 : Size.getFixedValue());
-  llvm::CallInst *C =
-      Builder.CreateCall(CGM.getLLVMLifetimeStartFn(), {SizeV, Addr});
+  llvm::CallInst *C = Builder.CreateCall(CGM.getLLVMLifetimeStartFn(), {Addr});
   C->setDoesNotThrow();
-  return SizeV;
+  return true;
 }
 
-void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr) {
+void CodeGenFunction::EmitLifetimeEnd(llvm::Value *Addr) {
+  if (!ShouldEmitLifetimeMarkers)
+    return;
+
   assert(Addr->getType()->getPointerAddressSpace() ==
              CGM.getDataLayout().getAllocaAddrSpace() &&
          "Pointer should be in alloca address space");
-  llvm::CallInst *C =
-      Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Size, Addr});
+  llvm::CallInst *C = Builder.CreateCall(CGM.getLLVMLifetimeEndFn(), {Addr});
   C->setDoesNotThrow();
 }
 
@@ -1564,14 +1563,13 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
       // The named return value optimization: allocate this variable in the
       // return slot, so that we can elide the copy when returning this
       // variable (C++0x [class.copy]p34).
-      address = ReturnValue;
       AllocaAddr =
           RawAddress(ReturnValue.emitRawPointer(*this),
                      ReturnValue.getElementType(), ReturnValue.getAlignment());
-      ;
+      address = MaybeCastStackAddressSpace(AllocaAddr, Ty.getAddressSpace());
 
       if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
-        const auto *RD = RecordTy->getDecl();
+        const auto *RD = RecordTy->getOriginalDecl()->getDefinitionOrSelf();
         const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD);
         if ((CXXRD && !CXXRD->hasTrivialDestructor()) ||
             RD->isNonTrivialToPrimitiveDestroy()) {
@@ -1632,9 +1630,8 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
         // is rare.
         if (!Bypasses.IsBypassed(&D) &&
             !(!getLangOpts().CPlusPlus && hasLabelBeenSeenInCurrentScope())) {
-          llvm::TypeSize Size = CGM.getDataLayout().getTypeAllocSize(allocaTy);
-          emission.SizeForLifetimeMarkers =
-              EmitLifetimeStart(Size, AllocaAddr.getPointer());
+          emission.UseLifetimeMarkers =
+              EmitLifetimeStart(AllocaAddr.getPointer());
         }
       } else {
         assert(!emission.useLifetimeMarkers());
@@ -1727,9 +1724,8 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
 
   // Make sure we call @llvm.lifetime.end.
   if (emission.useLifetimeMarkers())
-    EHStack.pushCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker,
-                                         emission.getOriginalAllocatedAddress(),
-                                         emission.getSizeForLifetimeMarkers());
+    EHStack.pushCleanup<CallLifetimeEnd>(
+        NormalEHLifetimeMarker, emission.getOriginalAllocatedAddress());
 
   // Analogous to lifetime markers, we use a 'cleanup' to emit fake.use
   // calls for local variables. We are exempting volatile variables and
@@ -2732,7 +2728,10 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
     // Don't push a cleanup in a thunk for a method that will also emit a
     // cleanup.
     if (Ty->isRecordType() && !CurFuncIsThunk &&
-        Ty->castAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
+        Ty->castAs<RecordType>()
+            ->getOriginalDecl()
+            ->getDefinitionOrSelf()
+            ->isParamDestroyedInCallee()) {
       if (QualType::DestructionKind DtorKind =
               D.needsDestruction(getContext())) {
         assert((DtorKind == QualType::DK_cxx_destructor ||
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 5a3d4e447b229..0f9de6fe3e0cb 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -16,6 +16,7 @@
 #include "CGCall.h"
 #include "CGCleanup.h"
 #include "CGDebugInfo.h"
+#include "CGHLSLRuntime.h"
 #include "CGObjCRuntime.h"
 #include "CGOpenMPRuntime.h"
 #include "CGRecordLayout.h"
@@ -107,13 +108,10 @@ CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty, CharUnits Align,
   return RawAddress(Alloca, Ty, Align, KnownNonNull);
 }
 
-RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, LangAS DestLangAS,
-                                             CharUnits Align, const Twine &Name,
-                                             llvm::Value *ArraySize,
-                                             RawAddress *AllocaAddr) {
-  RawAddress Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
-  if (AllocaAddr)
-    *AllocaAddr = Alloca;
+RawAddress CodeGenFunction::MaybeCastStackAddressSpace(RawAddress Alloca,
+                                                       LangAS DestLangAS,
+                                                       llvm::Value *ArraySize) {
+
   llvm::Value *V = Alloca.getPointer();
   // Alloca always returns a pointer in alloca address space, which may
   // be different from the type defined by the language. For example,
@@ -133,7 +131,18 @@ RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, LangAS DestLangAS,
         /*IsNonNull=*/true);
   }
 
-  return RawAddress(V, Ty, Align, KnownNonNull);
+  return RawAddress(V, Alloca.getElementType(), Alloca.getAlignment(),
+                    KnownNonNull);
+}
+
+RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, LangAS DestLangAS,
+                                             CharUnits Align, const Twine &Name,
+                                             llvm::Value *ArraySize,
+                                             RawAddress *AllocaAddr) {
+  RawAddress Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
+  if (AllocaAddr)
+    *AllocaAddr = Alloca;
+  return MaybeCastStackAddressSpace(Alloca, DestLangAS, ArraySize);
 }
 
 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
@@ -408,9 +417,10 @@ pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
       if (const RecordType *RT =
               E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
         // Get the destructor for the reference temporary.
-        if (auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl());
+        if (auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl());
             ClassDecl && !ClassDecl->hasTrivialDestructor())
-          ReferenceTemporaryDtor = ClassDecl->getDestructor();
+          ReferenceTemporaryDtor =
+              ClassDecl->getDefinitionOrSelf()->getDestructor();
       }
 
       if (!ReferenceTemporaryDtor)
@@ -588,11 +598,9 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
   } else {
     switch (M->getStorageDuration()) {
     case SD_Automatic:
-      if (auto *Size = EmitLifetimeStart(
-              CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
-              Alloca.getPointer())) {
+      if (EmitLifetimeStart(Alloca.getPointer())) {
         pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker,
-                                                  Alloca, Size);
+                                                  Alloca);
       }
       break;
 
@@ -623,11 +631,8 @@ EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
             Block, llvm::BasicBlock::iterator(Block->back())));
       }
 
-      if (auto *Size = EmitLifetimeStart(
-              CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
-              Alloca.getPointer())) {
-        pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca,
-                                             Size);
+      if (EmitLifetimeStart(Alloca.getPointer())) {
+        pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca);
       }
 
       if (OldConditional) {
@@ -1209,9 +1214,10 @@ llvm::Value *CodeGenFunction::GetCountedByFieldExprGEP(
     return nullptr;
 
   Indices.push_back(Builder.getInt32(0));
-  return Builder.CreateInBoundsGEP(
-      ConvertType(QualType(RD->getTypeForDecl(), 0)), Res,
-      RecIndicesTy(llvm::reverse(Indices)), "counted_by.gep");
+  CanQualType T = CGM.getContext().getCanonicalTagType(RD);
+  return Builder.CreateInBoundsGEP(ConvertType(T), Res,
+                                   RecIndicesTy(llvm::reverse(Indices)),
+                                   "counted_by.gep");
 }
 
 /// This method is typically called in contexts where we can't generate
@@ -1319,6 +1325,57 @@ void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
 //                         LValue Expression Emission
 //===----------------------------------------------------------------------===//
 
+static CharUnits getArrayElementAlign(CharUnits arrayAlign, llvm::Value *idx,
+                                      CharUnits eltSize) {
+  // If we have a constant index, we can use the exact offset of the
+  // element we're accessing.
+  if (auto *constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
+    CharUnits offset = constantIdx->getZExtValue() * eltSize;
+    return arrayAlign.alignmentAtOffset(offset);
+  }
+
+  // Otherwise, use the worst-case alignment for any element.
+  return arrayAlign.alignmentOfArrayElement(eltSize);
+}
+
+/// Emit pointer + index arithmetic.
+static Address emitPointerArithmetic(CodeGenFunction &CGF,
+                                     const BinaryOperator *BO,
+                                     LValueBaseInfo *BaseInfo,
+                                     TBAAAccessInfo *TBAAInfo,
+                                     KnownNonNull_t IsKnownNonNull) {
+  assert(BO->isAdditiveOp() && "Expect an addition or subtraction.");
+  Expr *pointerOperand = BO->getLHS();
+  Expr *indexOperand = BO->getRHS();
+  bool isSubtraction = BO->getOpcode() == BO_Sub;
+
+  Address BaseAddr = Address::invalid();
+  llvm::Value *index = nullptr;
+  // In a subtraction, the LHS is always the pointer.
+  // Note: do not change the evaluation order.
+  if (!isSubtraction && !pointerOperand->getType()->isAnyPointerType()) {
+    std::swap(pointerOperand, indexOperand);
+    index = CGF.EmitScalarExpr(indexOperand);
+    BaseAddr = CGF.EmitPointerWithAlignment(pointerOperand, BaseInfo, TBAAInfo,
+                                            NotKnownNonNull);
+  } else {
+    BaseAddr = CGF.EmitPointerWithAlignment(pointerOperand, BaseInfo, TBAAInfo,
+                                            NotKnownNonNull);
+    index = CGF.EmitScalarExpr(indexOperand);
+  }
+
+  llvm::Value *pointer = BaseAddr.getBasePointer();
+  llvm::Value *Res = CGF.EmitPointerArithmetic(
+      BO, pointerOperand, pointer, indexOperand, index, isSubtraction);
+  QualType PointeeTy = BO->getType()->getPointeeType();
+  CharUnits Align =
+      getArrayElementAlign(BaseAddr.getAlignment(), index,
+                           CGF.getContext().getTypeSizeInChars(PointeeTy));
+  return Address(Res, CGF.ConvertTypeForMem(PointeeTy), Align,
+                 CGF.CGM.getPointerAuthInfoForPointeeType(PointeeTy),
+                 /*Offset=*/nullptr, IsKnownNonNull);
+}
+
 static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
                                         TBAAAccessInfo *TBAAInfo,
                                         KnownNonNull_t IsKnownNonNull,
@@ -1381,6 +1438,7 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
         if (CE->getCastKind() == CK_AddressSpaceConversion)
           Addr = CGF.Builder.CreateAddrSpaceCast(
               Addr, CGF.ConvertType(E->getType()), ElemTy);
+
         return CGF.authPointerToPointerCast(Addr, CE->getSubExpr()->getType(),
                                             CE->getType());
       }
@@ -1441,6 +1499,12 @@ static Address EmitPointerWithAlignment(const Expr *E, LValueBaseInfo *BaseInfo,
     }
   }
 
+  // Pointer arithmetic: pointer +/- index.
+  if (auto *BO = dyn_cast<BinaryOperator>(E)) {
+    if (BO->isAdditiveOp())
+      return emitPointerArithmetic(CGF, BO, BaseInfo, TBAAInfo, IsKnownNonNull);
+  }
+
   // TODO: conditional operators, comma.
 
   // Otherwise, use the alignment of the type.
@@ -1759,9 +1823,11 @@ static bool isConstantEmittableObjectType(QualType type) {
   // Otherwise, all object types satisfy this except C++ classes with
   // mutable subobjects or non-trivial copy/destroy behavior.
   if (const auto *RT = dyn_cast<RecordType>(type))
-    if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
+    if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl())) {
+      RD = RD->getDefinitionOrSelf();
       if (RD->hasMutableFields() || !RD->isTrivial())
         return false;
+    }
 
   return true;
 }
@@ -1922,8 +1988,10 @@ static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,
                             llvm::APInt &Min, llvm::APInt &End,
                             bool StrictEnums, bool IsBool) {
   const EnumType *ET = Ty->getAs<EnumType>();
-  bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums &&
-                                ET && !ET->getDecl()->isFixed();
+  const EnumDecl *ED =
+      ET ? ET->getOriginalDecl()->getDefinitionOrSelf() : nullptr;
+  bool IsRegularCPlusPlusEnum =
+      CGF.getLangOpts().CPlusPlus && StrictEnums && ET && !ED->isFixed();
   if (!IsBool && !IsRegularCPlusPlusEnum)
     return false;
 
@@ -1931,7 +1999,6 @@ static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,
     Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0);
     End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2);
   } else {
-    const EnumDecl *ED = ET->getDecl();
     ED->getValueRange(End, Min);
   }
   return true;
@@ -3789,33 +3856,50 @@ void CodeGenFunction::EmitCheck(
   Branch->setMetadata(llvm::LLVMContext::MD_prof, Node);
   EmitBlock(Handlers);
 
+  // Clear arguments for the MinimalRuntime handler.
+  if (CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
+    switch (CheckHandler) {
+    case SanitizerHandler::TypeMismatch:
+      // Pass value pointer only. It adds minimal overhead.
+      StaticArgs = {};
+      assert(DynamicArgs.size() == 1);
+      break;
+    default:
+      // No arguments for other checks.
+      StaticArgs = {};
+      DynamicArgs = {};
+      break;
+    }
+  }
+
   // Handler functions take an i8* pointing to the (handler-specific) static
   // information block, followed by a sequence of intptr_t arguments
   // representing operand values.
   SmallVector<llvm::Value *, 4> Args;
   SmallVector<llvm::Type *, 4> ArgTypes;
-  if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
-    Args.reserve(DynamicArgs.size() + 1);
-    ArgTypes.reserve(DynamicArgs.size() + 1);
-
-    // Emit handler arguments and create handler function type.
-    if (!StaticArgs.empty()) {
-      llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
-      auto *InfoPtr = new llvm::GlobalVariable(
-          CGM.getModule(), Info->getType(), false,
-          llvm::GlobalVariable::PrivateLinkage, Info, "", nullptr,
-          llvm::GlobalVariable::NotThreadLocal,
-          CGM.getDataLayout().getDefaultGlobalsAddressSpace());
-      InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
-      CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
-      Args.push_back(InfoPtr);
-      ArgTypes.push_back(Args.back()->getType());
-    }
 
-    for (llvm::Value *DynamicArg : DynamicArgs) {
-      Args.push_back(EmitCheckValue(DynamicArg));
-      ArgTypes.push_back(IntPtrTy);
-    }
+  Args.reserve(DynamicArgs.size() + 1);
+  ArgTypes.reserve(DynamicArgs.size() + 1);
+
+  // Emit handler arguments and create handler function type.
+  if (!StaticArgs.empty()) {
+    llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
+    auto *InfoPtr = new llvm::GlobalVariable(
+        CGM.getModule(), Info->getType(),
+        // Non-constant global is used in a handler to deduplicate reports.
+        // TODO: change deduplication logic and make it constant.
+        /*isConstant=*/false, llvm::GlobalVariable::PrivateLinkage, Info, "",
+        nullptr, llvm::GlobalVariable::NotThreadLocal,
+        CGM.getDataLayout().getDefaultGlobalsAddressSpace());
+    InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
+    CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
+    Args.push_back(InfoPtr);
+    ArgTypes.push_back(Args.back()->getType());
+  }
+
+  for (llvm::Value *DynamicArg : DynamicArgs) {
+    Args.push_back(EmitCheckValue(DynamicArg));
+    ArgTypes.push_back(IntPtrTy);
   }
 
   llvm::FunctionType *FnType =
@@ -4210,21 +4294,6 @@ static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
   }
 }
 
-static CharUnits getArrayElementAlign(CharUnits arrayAlign,
-                                      llvm::Value *idx,
-                                      CharUnits eltSize) {
-  // If we have a constant index, we can use the exact offset of the
-  // element we're accessing.
-  if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
-    CharUnits offset = constantIdx->getZExtValue() * eltSize;
-    return arrayAlign.alignmentAtOffset(offset);
-
-  // Otherwise, use the worst-case alignment for any element.
-  } else {
-    return arrayAlign.alignmentOfArrayElement(eltSize);
-  }
-}
-
 static QualType getFixedSizeElementType(const ASTContext &ctx,
                                         const VariableArrayType *vla) {
   QualType eltType;
@@ -4290,7 +4359,9 @@ static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
     const auto *PointeeT = PtrT->getPointeeType()
                              ->getUnqualifiedDesugaredType();
     if (const auto *RecT = dyn_cast<RecordType>(PointeeT))
-      return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
+      return RecT->getOriginalDecl()
+          ->getMostRecentDecl()
+          ->hasAttr<BPFPreserveAccessIndexAttr>();
     return false;
   }
 
@@ -4515,6 +4586,15 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
                                  LHS.getBaseInfo(), TBAAAccessInfo());
   }
 
+  // The HLSL runtime handle the subscript expression on global resource arrays.
+  if (getLangOpts().HLSL && (E->getType()->isHLSLResourceRecord() ||
+                             E->getType()->isHLSLResourceRecordArray())) {
+    std::optional<LValue> LV =
+        CGM.getHLSLRuntime().emitResourceArraySubscriptExpr(E, *this);
+    if (LV.has_value())
+      return *LV;
+  }
+
   // All the other cases basically behave like simple offsetting.
 
   // Handle the extvector case we ignored above.
@@ -5056,10 +5136,12 @@ LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field,
       Address Base = GetAddressOfBaseClass(
           LambdaLV.getAddress(), ThisTy, BasePathArray.begin(),
           BasePathArray.end(), /*NullCheckValue=*/false, SourceLocation());
-      LambdaLV = MakeAddrLValue(Base, QualType{LambdaTy->getTypeForDecl(), 0});
+      CanQualType T = getContext().getCanonicalTagType(LambdaTy);
+      LambdaLV = MakeAddrLValue(Base, T);
     }
   } else {
-    QualType LambdaTagType = getContext().getTagDeclType(Field->getParent());
+    CanQualType LambdaTagType =
+        getContext().getCanonicalTagType(Field->getParent());
     LambdaLV = MakeNaturalAlignAddrLValue(ThisValue, LambdaTagType);
   }
   return EmitLValueForField(LambdaLV, Field);
@@ -5184,7 +5266,7 @@ LValue CodeGenFunction::EmitLValueForField(LValue base, const FieldDecl *field,
         }
       } else {
         llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
-            getContext().getRecordType(rec), rec->getLocation());
+            getContext().getCanonicalTagType(rec), rec->getLocation());
         Addr = Builder.CreatePreserveStructAccessIndex(
             Addr, Idx, getDebugInfoFIndex(rec, field->getFieldIndex()),
             DbgInfo);
@@ -5646,7 +5728,9 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
   case CK_DerivedToBase: {
     const auto *DerivedClassTy =
         E->getSubExpr()->getType()->castAs<RecordType>();
-    auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
+    auto *DerivedClassDecl =
+        cast<CXXRecordDecl>(DerivedClassTy->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     LValue LV = EmitLValue(E->getSubExpr());
     Address This = LV.getAddress();
@@ -5666,7 +5750,9 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
     return EmitAggExprToLValue(E);
   case CK_BaseToDerived: {
     const auto *DerivedClassTy = E->getType()->castAs<RecordType>();
-    auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
+    auto *DerivedClassDecl =
+        cast<CXXRecordDecl>(DerivedClassTy->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     LValue LV = EmitLValue(E->getSubExpr());
 
@@ -5767,13 +5853,10 @@ LValue CodeGenFunction::EmitHLSLOutArgExpr(const HLSLOutArgExpr *E,
   llvm::Value *Addr = TempLV.getAddress().getBasePointer();
   llvm::Type *ElTy = ConvertTypeForMem(TempLV.getType());
 
-  llvm::TypeSize Sz = CGM.getDataLayout().getTypeAllocSize(ElTy);
-
-  llvm::Value *LifetimeSize = EmitLifetimeStart(Sz, Addr);
+  EmitLifetimeStart(Addr);
 
   Address TmpAddr(Addr, ElTy, TempLV.getAlignment());
-  Args.addWriteback(BaseLV, TmpAddr, nullptr, E->getWritebackCast(),
-                    LifetimeSize);
+  Args.addWriteback(BaseLV, TmpAddr, nullptr, E->getWritebackCast());
   Args.add(RValue::get(TmpAddr, *this), Ty);
   return TempLV;
 }
@@ -6724,7 +6807,7 @@ void CodeGenFunction::FlattenAccessAndType(
         WorkList.emplace_back(CAT->getElementType(), IdxListCopy);
       }
     } else if (const auto *RT = dyn_cast<RecordType>(T)) {
-      const RecordDecl *Record = RT->getDecl();
+      const RecordDecl *Record = RT->getOriginalDecl()->getDefinitionOrSelf();
       assert(!Record->isUnion() && "Union types not supported in flat cast.");
 
       const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(Record);
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index cad6731173700..04e125c54b1ca 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -272,7 +272,7 @@ bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
   if (!RecordTy) return false;
 
   // Don't mess with non-trivial C++ types.
-  RecordDecl *Record = RecordTy->getDecl();
+  RecordDecl *Record = RecordTy->getOriginalDecl()->getDefinitionOrSelf();
   if (isa<CXXRecordDecl>(Record) &&
       (cast<CXXRecordDecl>(Record)->hasNonTrivialCopyConstructor() ||
        !cast<CXXRecordDecl>(Record)->hasTrivialDestructor()))
@@ -300,16 +300,12 @@ void AggExprEmitter::withReturnValueSlot(
   Address RetAddr = Address::invalid();
 
   EHScopeStack::stable_iterator LifetimeEndBlock;
-  llvm::Value *LifetimeSizePtr = nullptr;
   llvm::IntrinsicInst *LifetimeStartInst = nullptr;
   if (!UseTemp) {
     RetAddr = Dest.getAddress();
   } else {
     RetAddr = CGF.CreateMemTempWithoutCast(RetTy, "tmp");
-    llvm::TypeSize Size =
-        CGF.CGM.getDataLayout().getTypeAllocSize(CGF.ConvertTypeForMem(RetTy));
-    LifetimeSizePtr = CGF.EmitLifetimeStart(Size, RetAddr.getBasePointer());
-    if (LifetimeSizePtr) {
+    if (CGF.EmitLifetimeStart(RetAddr.getBasePointer())) {
       LifetimeStartInst =
           cast<llvm::IntrinsicInst>(std::prev(Builder.GetInsertPoint()));
       assert(LifetimeStartInst->getIntrinsicID() ==
@@ -317,7 +313,7 @@ void AggExprEmitter::withReturnValueSlot(
              "Last insertion wasn't a lifetime.start?");
 
       CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
-          NormalEHLifetimeMarker, RetAddr, LifetimeSizePtr);
+          NormalEHLifetimeMarker, RetAddr);
       LifetimeEndBlock = CGF.EHStack.stable_begin();
     }
   }
@@ -338,7 +334,7 @@ void AggExprEmitter::withReturnValueSlot(
     // Since we're not guaranteed to be in an ExprWithCleanups, clean up
     // eagerly.
     CGF.DeactivateCleanupBlock(LifetimeEndBlock, LifetimeStartInst);
-    CGF.EmitLifetimeEnd(LifetimeSizePtr, RetAddr.getBasePointer());
+    CGF.EmitLifetimeEnd(RetAddr.getBasePointer());
   }
 }
 
@@ -428,7 +424,10 @@ AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
       Ctx.getAsConstantArrayType(E->getSubExpr()->getType());
   assert(ArrayType && "std::initializer_list constructed from non-array");
 
-  RecordDecl *Record = E->getType()->castAs<RecordType>()->getDecl();
+  RecordDecl *Record = E->getType()
+                           ->castAs<RecordType>()
+                           ->getOriginalDecl()
+                           ->getDefinitionOrSelf();
   RecordDecl::field_iterator Field = Record->field_begin();
   assert(Field != Record->field_end() &&
          Ctx.hasSameType(Field->getType()->getPointeeType(),
@@ -1810,7 +1809,10 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
   // the disadvantage is that the generated code is more difficult for
   // the optimizer, especially with bitfields.
   unsigned NumInitElements = InitExprs.size();
-  RecordDecl *record = ExprToVisit->getType()->castAs<RecordType>()->getDecl();
+  RecordDecl *record = ExprToVisit->getType()
+                           ->castAs<RecordType>()
+                           ->getOriginalDecl()
+                           ->getDefinitionOrSelf();
 
   // We'll need to enter cleanup scopes in case any of the element
   // initializers throws an exception.
@@ -2120,7 +2122,7 @@ static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
   // referencee.  InitListExprs for unions and arrays can't have references.
   if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
     if (!RT->isUnionType()) {
-      RecordDecl *SD = RT->getDecl();
+      RecordDecl *SD = RT->getOriginalDecl()->getDefinitionOrSelf();
       CharUnits NumNonZeroBytes = CharUnits::Zero();
 
       unsigned ILEElement = 0;
@@ -2172,7 +2174,7 @@ static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
   if (CGF.getLangOpts().CPlusPlus)
     if (const RecordType *RT = CGF.getContext()
                        .getBaseElementType(E->getType())->getAs<RecordType>()) {
-      const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+      const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getOriginalDecl());
       if (RD->hasUserDeclaredConstructor())
         return;
     }
@@ -2293,7 +2295,8 @@ void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
 
   if (getLangOpts().CPlusPlus) {
     if (const RecordType *RT = Ty->getAs<RecordType>()) {
-      CXXRecordDecl *Record = cast<CXXRecordDecl>(RT->getDecl());
+      auto *Record =
+          cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
       assert((Record->hasTrivialCopyConstructor() ||
               Record->hasTrivialCopyAssignment() ||
               Record->hasTrivialMoveConstructor() ||
@@ -2377,7 +2380,7 @@ void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
   if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
     // fall through
   } else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
-    RecordDecl *Record = RecordTy->getDecl();
+    RecordDecl *Record = RecordTy->getOriginalDecl()->getDefinitionOrSelf();
     if (Record->hasObjectMember()) {
       CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
                                                     SizeVal);
@@ -2386,7 +2389,9 @@ void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
   } else if (Ty->isArrayType()) {
     QualType BaseType = getContext().getBaseElementType(Ty);
     if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) {
-      if (RecordTy->getDecl()->hasObjectMember()) {
+      if (RecordTy->getOriginalDecl()
+              ->getDefinitionOrSelf()
+              ->hasObjectMember()) {
         CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr,
                                                       SizeVal);
         return;
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index fef1baf25d1d2..57d7eecc62ee1 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -181,7 +181,7 @@ static CXXRecordDecl *getCXXRecord(const Expr *E) {
   if (const PointerType *PTy = T->getAs<PointerType>())
     T = PTy->getPointeeType();
   const RecordType *Ty = T->castAs<RecordType>();
-  return cast<CXXRecordDecl>(Ty->getDecl());
+  return cast<CXXRecordDecl>(Ty->getOriginalDecl())->getDefinitionOrSelf();
 }
 
 // Note: This function also emit constructor calls to support a MSVC
@@ -206,7 +206,7 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
   }
 
   bool HasQualifier = ME->hasQualifier();
-  NestedNameSpecifier *Qualifier = HasQualifier ? ME->getQualifier() : nullptr;
+  NestedNameSpecifier Qualifier = ME->getQualifier();
   bool IsArrow = ME->isArrow();
   const Expr *Base = ME->getBase();
 
@@ -217,7 +217,7 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
 
 RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
     const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
-    bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow,
+    bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,
     const Expr *Base, llvm::CallBase **CallOrInvoke) {
   assert(isa<CXXMemberCallExpr>(CE) || isa<CXXOperatorCallExpr>(CE));
 
@@ -361,7 +361,7 @@ RValue CodeGenFunction::EmitCXXMemberOrOperatorMemberCallExpr(
   if (sanitizePerformTypeCheck())
     EmitTypeCheck(CodeGenFunction::TCK_MemberCall, CallLoc,
                   This.emitRawPointer(*this),
-                  C.getRecordType(CalleeDecl->getParent()),
+                  C.getCanonicalTagType(CalleeDecl->getParent()),
                   /*Alignment=*/CharUnits::Zero(), SkippedChecks);
 
   // C++ [class.virtual]p12:
@@ -461,9 +461,9 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
   else
     This = EmitLValue(BaseExpr, KnownNonNull).getAddress();
 
-  EmitTypeCheck(
-      TCK_MemberCall, E->getExprLoc(), This.emitRawPointer(*this),
-      QualType(MPT->getMostRecentCXXRecordDecl()->getTypeForDecl(), 0));
+  CanQualType ClassType = CGM.getContext().getCanonicalTagType(RD);
+  EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This.emitRawPointer(*this),
+                ClassType);
 
   // Get the member function pointer.
   llvm::Value *MemFnPtr = EmitScalarExpr(MemFnExpr);
@@ -476,8 +476,7 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
 
   CallArgList Args;
 
-  QualType ThisType =
-    getContext().getPointerType(getContext().getTagDeclType(RD));
+  QualType ThisType = getContext().getPointerType(ClassType);
 
   // Push the this ptr.
   Args.add(RValue::get(ThisPtrForCall), ThisType);
@@ -498,7 +497,7 @@ RValue CodeGenFunction::EmitCXXOperatorMemberCallExpr(
   assert(MD->isImplicitObjectMemberFunction() &&
          "Trying to emit a member call expr on a static method!");
   return EmitCXXMemberOrOperatorMemberCallExpr(
-      E, MD, ReturnValue, /*HasQualifier=*/false, /*Qualifier=*/nullptr,
+      E, MD, ReturnValue, /*HasQualifier=*/false, /*Qualifier=*/std::nullopt,
       /*IsArrow=*/false, E->getArg(0), CallOrInvoke);
 }
 
@@ -1237,11 +1236,12 @@ void CodeGenFunction::EmitNewArrayInitializer(
   // usually use memset.
   if (auto *ILE = dyn_cast<InitListExpr>(Init)) {
     if (const RecordType *RType = ILE->getType()->getAs<RecordType>()) {
-      if (RType->getDecl()->isStruct()) {
+      const RecordDecl *RD = RType->getOriginalDecl()->getDefinitionOrSelf();
+      if (RD->isStruct()) {
         unsigned NumElements = 0;
-        if (auto *CXXRD = dyn_cast<CXXRecordDecl>(RType->getDecl()))
+        if (auto *CXXRD = dyn_cast<CXXRecordDecl>(RD))
           NumElements = CXXRD->getNumBases();
-        for (auto *Field : RType->getDecl()->fields())
+        for (auto *Field : RD->fields())
           if (!Field->isUnnamedBitField())
             ++NumElements;
         // FIXME: Recurse into nested InitListExprs.
@@ -1687,9 +1687,11 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
       QualType AlignValT = sizeType;
       if (allocatorType->getNumParams() > IndexOfAlignArg) {
         AlignValT = allocatorType->getParamType(IndexOfAlignArg);
-        assert(getContext().hasSameUnqualifiedType(
-                   AlignValT->castAs<EnumType>()->getDecl()->getIntegerType(),
-                   sizeType) &&
+        assert(getContext().hasSameUnqualifiedType(AlignValT->castAs<EnumType>()
+                                                       ->getOriginalDecl()
+                                                       ->getDefinitionOrSelf()
+                                                       ->getIntegerType(),
+                                                   sizeType) &&
                "wrong type for alignment parameter");
         ++ParamsToSkip;
       } else {
@@ -1972,7 +1974,8 @@ static bool EmitObjectDelete(CodeGenFunction &CGF,
   // destructor is virtual, we'll just emit the vcall and return.
   const CXXDestructorDecl *Dtor = nullptr;
   if (const RecordType *RT = ElementType->getAs<RecordType>()) {
-    CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    auto *RD =
+        cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
     if (RD->hasDefinition() && !RD->hasTrivialDestructor()) {
       Dtor = RD->getDestructor();
 
@@ -2295,6 +2298,18 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
                  CGM.getCXXABI().shouldEmitExactDynamicCast(DestRecordTy) &&
                  !getLangOpts().PointerAuthCalls;
 
+  std::optional<CGCXXABI::ExactDynamicCastInfo> ExactCastInfo;
+  if (IsExact) {
+    ExactCastInfo = CGM.getCXXABI().getExactDynamicCastInfo(SrcRecordTy, DestTy,
+                                                            DestRecordTy);
+    if (!ExactCastInfo) {
+      llvm::Value *NullValue = EmitDynamicCastToNull(*this, DestTy);
+      if (!Builder.GetInsertBlock())
+        EmitBlock(createBasicBlock("dynamic_cast.unreachable"));
+      return NullValue;
+    }
+  }
+
   // C++ [expr.dynamic.cast]p4:
   //   If the value of v is a null pointer value in the pointer case, the result
   //   is the null pointer value of type T.
@@ -2322,7 +2337,8 @@ llvm::Value *CodeGenFunction::EmitDynamicCast(Address ThisAddr,
     // If the destination type is effectively final, this pointer points to the
     // right type if and only if its vptr has the right value.
     Value = CGM.getCXXABI().emitExactDynamicCast(
-        *this, ThisAddr, SrcRecordTy, DestTy, DestRecordTy, CastEnd, CastNull);
+        *this, ThisAddr, SrcRecordTy, DestTy, DestRecordTy, *ExactCastInfo,
+        CastEnd, CastNull);
   } else {
     assert(DestRecordTy->isRecordType() &&
            "destination type must be a record type!");
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 715bd392f59f7..a96c1518d2a1d 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -714,7 +714,10 @@ static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter,
 }
 
 bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
-  RecordDecl *RD = ILE->getType()->castAs<RecordType>()->getDecl();
+  RecordDecl *RD = ILE->getType()
+                       ->castAs<RecordType>()
+                       ->getOriginalDecl()
+                       ->getDefinitionOrSelf();
   const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(RD);
 
   unsigned FieldNo = -1;
@@ -977,7 +980,8 @@ bool ConstStructBuilder::DoZeroInitPadding(const ASTRecordLayout &Layout,
 
 llvm::Constant *ConstStructBuilder::Finalize(QualType Type) {
   Type = Type.getNonReferenceType();
-  RecordDecl *RD = Type->castAs<RecordType>()->getDecl();
+  RecordDecl *RD =
+      Type->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
   llvm::Type *ValTy = CGM.getTypes().ConvertType(Type);
   return Builder.build(ValTy, RD->hasFlexibleArrayMember());
 }
@@ -1000,7 +1004,8 @@ llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter,
   ConstantAggregateBuilder Const(Emitter.CGM);
   ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero());
 
-  const RecordDecl *RD = ValTy->castAs<RecordType>()->getDecl();
+  const RecordDecl *RD =
+      ValTy->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
   const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(RD);
   if (!Builder.Build(Val, RD, false, CD, CharUnits::Zero()))
     return nullptr;
@@ -1506,7 +1511,9 @@ class ConstExprEmitter
     llvm::Type *ValTy = CGM.getTypes().ConvertType(destType);
     bool HasFlexibleArray = false;
     if (const auto *RT = destType->getAs<RecordType>())
-      HasFlexibleArray = RT->getDecl()->hasFlexibleArrayMember();
+      HasFlexibleArray = RT->getOriginalDecl()
+                             ->getDefinitionOrSelf()
+                             ->hasFlexibleArrayMember();
     return Const.build(ValTy, HasFlexibleArray);
   }
 
@@ -2640,7 +2647,9 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
       }
 
       const CXXRecordDecl *base =
-        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+          cast<CXXRecordDecl>(
+              I.getType()->castAs<RecordType>()->getOriginalDecl())
+              ->getDefinitionOrSelf();
 
       // Ignore empty bases.
       if (isEmptyRecordForLayout(CGM.getContext(), I.getType()) ||
@@ -2679,8 +2688,10 @@ static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
   // Fill in the virtual bases, if we're working with the complete object.
   if (CXXR && asCompleteObject) {
     for (const auto &I : CXXR->vbases()) {
-      const CXXRecordDecl *base =
-        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+      const auto *base =
+          cast<CXXRecordDecl>(
+              I.getType()->castAs<RecordType>()->getOriginalDecl())
+              ->getDefinitionOrSelf();
 
       // Ignore empty bases.
       if (isEmptyRecordForLayout(CGM.getContext(), I.getType()))
@@ -2746,7 +2757,9 @@ llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
   }
 
   if (const RecordType *RT = T->getAs<RecordType>())
-    return ::EmitNullConstant(*this, RT->getDecl(), /*complete object*/ true);
+    return ::EmitNullConstant(*this,
+                              RT->getOriginalDecl()->getDefinitionOrSelf(),
+                              /*complete object*/ true);
 
   assert(T->isMemberDataPointerType() &&
          "Should only see pointers to data members here!");
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index 44931d0481e26..1e23f45546748 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -3515,7 +3515,9 @@ Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) {
 
     case OffsetOfNode::Field: {
       FieldDecl *MemberDecl = ON.getField();
-      RecordDecl *RD = CurrentType->castAs<RecordType>()->getDecl();
+      RecordDecl *RD = CurrentType->castAs<RecordType>()
+                           ->getOriginalDecl()
+                           ->getDefinitionOrSelf();
       const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(RD);
 
       // Compute the index of the field in its parent.
@@ -3548,15 +3550,16 @@ Value *ScalarExprEmitter::VisitOffsetOfExpr(OffsetOfExpr *E) {
         continue;
       }
 
-      RecordDecl *RD = CurrentType->castAs<RecordType>()->getDecl();
-      const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(RD);
+      const ASTRecordLayout &RL = CGF.getContext().getASTRecordLayout(
+          CurrentType->castAs<RecordType>()->getOriginalDecl());
 
       // Save the element type.
       CurrentType = ON.getBase()->getType();
 
       // Compute the offset to the base.
       auto *BaseRT = CurrentType->castAs<RecordType>();
-      auto *BaseRD = cast<CXXRecordDecl>(BaseRT->getDecl());
+      auto *BaseRD =
+          cast<CXXRecordDecl>(BaseRT->getOriginalDecl())->getDefinitionOrSelf();
       CharUnits OffsetInt = RL.getBaseClassOffset(BaseRD);
       Offset = llvm::ConstantInt::get(ResultType, OffsetInt.getQuantity());
       break;
@@ -4183,9 +4186,10 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
   return phi;
 }
 
-/// Emit pointer + index arithmetic.
-static Value *emitPointerArithmetic(CodeGenFunction &CGF,
-                                    const BinOpInfo &op,
+/// BO_Add/BO_Sub are handled by EmitPointerWithAlignment to preserve alignment
+/// information.
+/// This function is used for BO_AddAssign/BO_SubAssign.
+static Value *emitPointerArithmetic(CodeGenFunction &CGF, const BinOpInfo &op,
                                     bool isSubtraction) {
   // Must have binary (not unary) expr here.  Unary pointer
   // increment/decrement doesn't use this path.
@@ -4202,11 +4206,19 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
     std::swap(pointerOperand, indexOperand);
   }
 
+  return CGF.EmitPointerArithmetic(expr, pointerOperand, pointer, indexOperand,
+                                   index, isSubtraction);
+}
+
+/// Emit pointer + index arithmetic.
+llvm::Value *CodeGenFunction::EmitPointerArithmetic(
+    const BinaryOperator *BO, Expr *pointerOperand, llvm::Value *pointer,
+    Expr *indexOperand, llvm::Value *index, bool isSubtraction) {
   bool isSigned = indexOperand->getType()->isSignedIntegerOrEnumerationType();
 
   unsigned width = cast<llvm::IntegerType>(index->getType())->getBitWidth();
-  auto &DL = CGF.CGM.getDataLayout();
-  auto PtrTy = cast<llvm::PointerType>(pointer->getType());
+  auto &DL = CGM.getDataLayout();
+  auto *PtrTy = cast<llvm::PointerType>(pointer->getType());
 
   // Some versions of glibc and gcc use idioms (particularly in their malloc
   // routines) that add a pointer-sized integer (known to be a pointer value)
@@ -4227,79 +4239,77 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
   //
   // Note that we do not suppress the pointer overflow check in this case.
   if (BinaryOperator::isNullPointerArithmeticExtension(
-          CGF.getContext(), op.Opcode, expr->getLHS(), expr->getRHS())) {
-    Value *Ptr = CGF.Builder.CreateIntToPtr(index, pointer->getType());
-    if (CGF.getLangOpts().PointerOverflowDefined ||
-        !CGF.SanOpts.has(SanitizerKind::PointerOverflow) ||
-        NullPointerIsDefined(CGF.Builder.GetInsertBlock()->getParent(),
+          getContext(), BO->getOpcode(), pointerOperand, indexOperand)) {
+    llvm::Value *Ptr = Builder.CreateIntToPtr(index, pointer->getType());
+    if (getLangOpts().PointerOverflowDefined ||
+        !SanOpts.has(SanitizerKind::PointerOverflow) ||
+        NullPointerIsDefined(Builder.GetInsertBlock()->getParent(),
                              PtrTy->getPointerAddressSpace()))
       return Ptr;
     // The inbounds GEP of null is valid iff the index is zero.
     auto CheckOrdinal = SanitizerKind::SO_PointerOverflow;
     auto CheckHandler = SanitizerHandler::PointerOverflow;
-    SanitizerDebugLocation SanScope(&CGF, {CheckOrdinal}, CheckHandler);
-    Value *IsZeroIndex = CGF.Builder.CreateIsNull(index);
-    llvm::Constant *StaticArgs[] = {
-        CGF.EmitCheckSourceLocation(op.E->getExprLoc())};
+    SanitizerDebugLocation SanScope(this, {CheckOrdinal}, CheckHandler);
+    llvm::Value *IsZeroIndex = Builder.CreateIsNull(index);
+    llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(BO->getExprLoc())};
     llvm::Type *IntPtrTy = DL.getIntPtrType(PtrTy);
-    Value *IntPtr = llvm::Constant::getNullValue(IntPtrTy);
-    Value *ComputedGEP = CGF.Builder.CreateZExtOrTrunc(index, IntPtrTy);
-    Value *DynamicArgs[] = {IntPtr, ComputedGEP};
-    CGF.EmitCheck({{IsZeroIndex, CheckOrdinal}}, CheckHandler, StaticArgs,
-                  DynamicArgs);
+    llvm::Value *IntPtr = llvm::Constant::getNullValue(IntPtrTy);
+    llvm::Value *ComputedGEP = Builder.CreateZExtOrTrunc(index, IntPtrTy);
+    llvm::Value *DynamicArgs[] = {IntPtr, ComputedGEP};
+    EmitCheck({{IsZeroIndex, CheckOrdinal}}, CheckHandler, StaticArgs,
+              DynamicArgs);
     return Ptr;
   }
 
   if (width != DL.getIndexTypeSizeInBits(PtrTy)) {
     // Zero-extend or sign-extend the pointer value according to
     // whether the index is signed or not.
-    index = CGF.Builder.CreateIntCast(index, DL.getIndexType(PtrTy), isSigned,
-                                      "idx.ext");
+    index = Builder.CreateIntCast(index, DL.getIndexType(PtrTy), isSigned,
+                                  "idx.ext");
   }
 
   // If this is subtraction, negate the index.
   if (isSubtraction)
-    index = CGF.Builder.CreateNeg(index, "idx.neg");
+    index = Builder.CreateNeg(index, "idx.neg");
 
-  if (CGF.SanOpts.has(SanitizerKind::ArrayBounds))
-    CGF.EmitBoundsCheck(op.E, pointerOperand, index, indexOperand->getType(),
-                        /*Accessed*/ false);
+  if (SanOpts.has(SanitizerKind::ArrayBounds))
+    EmitBoundsCheck(BO, pointerOperand, index, indexOperand->getType(),
+                    /*Accessed*/ false);
 
-  const PointerType *pointerType
-    = pointerOperand->getType()->getAs<PointerType>();
+  const PointerType *pointerType =
+      pointerOperand->getType()->getAs<PointerType>();
   if (!pointerType) {
     QualType objectType = pointerOperand->getType()
-                                        ->castAs<ObjCObjectPointerType>()
-                                        ->getPointeeType();
-    llvm::Value *objectSize
-      = CGF.CGM.getSize(CGF.getContext().getTypeSizeInChars(objectType));
+                              ->castAs<ObjCObjectPointerType>()
+                              ->getPointeeType();
+    llvm::Value *objectSize =
+        CGM.getSize(getContext().getTypeSizeInChars(objectType));
 
-    index = CGF.Builder.CreateMul(index, objectSize);
+    index = Builder.CreateMul(index, objectSize);
 
-    Value *result =
-        CGF.Builder.CreateGEP(CGF.Int8Ty, pointer, index, "add.ptr");
-    return CGF.Builder.CreateBitCast(result, pointer->getType());
+    llvm::Value *result = Builder.CreateGEP(Int8Ty, pointer, index, "add.ptr");
+    return Builder.CreateBitCast(result, pointer->getType());
   }
 
   QualType elementType = pointerType->getPointeeType();
-  if (const VariableArrayType *vla
-        = CGF.getContext().getAsVariableArrayType(elementType)) {
+  if (const VariableArrayType *vla =
+          getContext().getAsVariableArrayType(elementType)) {
     // The element count here is the total number of non-VLA elements.
-    llvm::Value *numElements = CGF.getVLASize(vla).NumElts;
+    llvm::Value *numElements = getVLASize(vla).NumElts;
 
     // Effectively, the multiply by the VLA size is part of the GEP.
     // GEP indexes are signed, and scaling an index isn't permitted to
     // signed-overflow, so we use the same semantics for our explicit
     // multiply.  We suppress this if overflow is not undefined behavior.
-    llvm::Type *elemTy = CGF.ConvertTypeForMem(vla->getElementType());
-    if (CGF.getLangOpts().PointerOverflowDefined) {
-      index = CGF.Builder.CreateMul(index, numElements, "vla.index");
-      pointer = CGF.Builder.CreateGEP(elemTy, pointer, index, "add.ptr");
+    llvm::Type *elemTy = ConvertTypeForMem(vla->getElementType());
+    if (getLangOpts().PointerOverflowDefined) {
+      index = Builder.CreateMul(index, numElements, "vla.index");
+      pointer = Builder.CreateGEP(elemTy, pointer, index, "add.ptr");
     } else {
-      index = CGF.Builder.CreateNSWMul(index, numElements, "vla.index");
-      pointer = CGF.EmitCheckedInBoundsGEP(
-          elemTy, pointer, index, isSigned, isSubtraction, op.E->getExprLoc(),
-          "add.ptr");
+      index = Builder.CreateNSWMul(index, numElements, "vla.index");
+      pointer =
+          EmitCheckedInBoundsGEP(elemTy, pointer, index, isSigned,
+                                 isSubtraction, BO->getExprLoc(), "add.ptr");
     }
     return pointer;
   }
@@ -4309,16 +4319,15 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
   // future proof.
   llvm::Type *elemTy;
   if (elementType->isVoidType() || elementType->isFunctionType())
-    elemTy = CGF.Int8Ty;
+    elemTy = Int8Ty;
   else
-    elemTy = CGF.ConvertTypeForMem(elementType);
+    elemTy = ConvertTypeForMem(elementType);
 
-  if (CGF.getLangOpts().PointerOverflowDefined)
-    return CGF.Builder.CreateGEP(elemTy, pointer, index, "add.ptr");
+  if (getLangOpts().PointerOverflowDefined)
+    return Builder.CreateGEP(elemTy, pointer, index, "add.ptr");
 
-  return CGF.EmitCheckedInBoundsGEP(
-      elemTy, pointer, index, isSigned, isSubtraction, op.E->getExprLoc(),
-      "add.ptr");
+  return EmitCheckedInBoundsGEP(elemTy, pointer, index, isSigned, isSubtraction,
+                                BO->getExprLoc(), "add.ptr");
 }
 
 // Construct an fmuladd intrinsic to represent a fused mul-add of MulOp and
@@ -5551,8 +5560,8 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
 
   // OpenCL: If the condition is a vector, we can treat this condition like
   // the select function.
-  if ((CGF.getLangOpts().OpenCL && condExpr->getType()->isVectorType()) ||
-      condExpr->getType()->isExtVectorType()) {
+  if (CGF.getLangOpts().OpenCL && (condExpr->getType()->isVectorType() ||
+                                   condExpr->getType()->isExtVectorType())) {
     CGF.incrementProfileCounter(E);
 
     llvm::Value *CondV = CGF.EmitScalarExpr(condExpr);
@@ -5601,9 +5610,16 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
 
     llvm::Type *CondType = ConvertType(condExpr->getType());
     auto *VecTy = cast<llvm::VectorType>(CondType);
-    llvm::Value *ZeroVec = llvm::Constant::getNullValue(VecTy);
 
-    CondV = Builder.CreateICmpNE(CondV, ZeroVec, "vector_cond");
+    if (VecTy->getElementType()->isIntegerTy(1))
+      return Builder.CreateSelect(CondV, LHS, RHS, "vector_select");
+
+    // OpenCL uses the MSB of the mask vector.
+    llvm::Value *ZeroVec = llvm::Constant::getNullValue(VecTy);
+    if (condExpr->getType()->isExtVectorType())
+      CondV = Builder.CreateICmpSLT(CondV, ZeroVec, "vector_cond");
+    else
+      CondV = Builder.CreateICmpNE(CondV, ZeroVec, "vector_cond");
     return Builder.CreateSelect(CondV, LHS, RHS, "vector_select");
   }
 
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index f64ac20545fa3..12ece217ba563 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -22,6 +22,7 @@
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/TargetOptions.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Frontend/HLSL/RootSignatureMetadata.h"
 #include "llvm/IR/Constants.h"
@@ -84,6 +85,111 @@ void addRootSignature(llvm::dxbc::RootSignatureVersion RootSigVer,
   RootSignatureValMD->addOperand(MDVals);
 }
 
+// Find array variable declaration from nested array subscript AST nodes
+static const ValueDecl *getArrayDecl(const ArraySubscriptExpr *ASE) {
+  const Expr *E = nullptr;
+  while (ASE != nullptr) {
+    E = ASE->getBase()->IgnoreImpCasts();
+    if (!E)
+      return nullptr;
+    ASE = dyn_cast<ArraySubscriptExpr>(E);
+  }
+  if (const DeclRefExpr *DRE = dyn_cast_or_null<DeclRefExpr>(E))
+    return DRE->getDecl();
+  return nullptr;
+}
+
+// Get the total size of the array, or -1 if the array is unbounded.
+static int getTotalArraySize(ASTContext &AST, const clang::Type *Ty) {
+  Ty = Ty->getUnqualifiedDesugaredType();
+  assert(Ty->isArrayType() && "expected array type");
+  if (Ty->isIncompleteArrayType())
+    return -1;
+  return AST.getConstantArrayElementCount(cast<ConstantArrayType>(Ty));
+}
+
+// Find constructor decl for a specific resource record type and binding
+// (implicit vs. explicit). The constructor has 5 parameters.
+// For explicit binding the signature is:
+//   void(unsigned, unsigned, int, unsigned, const char *).
+// For implicit binding the signature is:
+//   void(unsigned, int, unsigned, unsigned, const char *).
+static CXXConstructorDecl *findResourceConstructorDecl(ASTContext &AST,
+                                                       QualType ResTy,
+                                                       bool ExplicitBinding) {
+  std::array<QualType, 5> ExpParmTypes = {
+      AST.UnsignedIntTy, AST.UnsignedIntTy, AST.UnsignedIntTy,
+      AST.UnsignedIntTy, AST.getPointerType(AST.CharTy.withConst())};
+  ExpParmTypes[ExplicitBinding ? 2 : 1] = AST.IntTy;
+
+  CXXRecordDecl *ResDecl = ResTy->getAsCXXRecordDecl();
+  for (auto *Ctor : ResDecl->ctors()) {
+    if (Ctor->getNumParams() != ExpParmTypes.size())
+      continue;
+    auto *ParmIt = Ctor->param_begin();
+    auto ExpTyIt = ExpParmTypes.begin();
+    for (; ParmIt != Ctor->param_end() && ExpTyIt != ExpParmTypes.end();
+         ++ParmIt, ++ExpTyIt) {
+      if ((*ParmIt)->getType() != *ExpTyIt)
+        break;
+    }
+    if (ParmIt == Ctor->param_end())
+      return Ctor;
+  }
+  llvm_unreachable("did not find constructor for resource class");
+}
+
+static Value *buildNameForResource(llvm::StringRef BaseName,
+                                   CodeGenModule &CGM) {
+  llvm::SmallString<64> GlobalName = {BaseName, ".str"};
+  return CGM.GetAddrOfConstantCString(BaseName.str(), GlobalName.c_str())
+      .getPointer();
+}
+
+static void createResourceCtorArgs(CodeGenModule &CGM, CXXConstructorDecl *CD,
+                                   llvm::Value *ThisPtr, llvm::Value *Range,
+                                   llvm::Value *Index, StringRef Name,
+                                   HLSLResourceBindingAttr *RBA,
+                                   HLSLVkBindingAttr *VkBinding,
+                                   CallArgList &Args) {
+  assert((VkBinding || RBA) && "at least one a binding attribute expected");
+
+  std::optional<uint32_t> RegisterSlot;
+  uint32_t SpaceNo = 0;
+  if (VkBinding) {
+    RegisterSlot = VkBinding->getBinding();
+    SpaceNo = VkBinding->getSet();
+  } else {
+    if (RBA->hasRegisterSlot())
+      RegisterSlot = RBA->getSlotNumber();
+    SpaceNo = RBA->getSpaceNumber();
+  }
+
+  ASTContext &AST = CD->getASTContext();
+  Value *NameStr = buildNameForResource(Name, CGM);
+  Value *Space = llvm::ConstantInt::get(CGM.IntTy, SpaceNo);
+
+  Args.add(RValue::get(ThisPtr), CD->getThisType());
+  if (RegisterSlot.has_value()) {
+    // explicit binding
+    auto *RegSlot = llvm::ConstantInt::get(CGM.IntTy, RegisterSlot.value());
+    Args.add(RValue::get(RegSlot), AST.UnsignedIntTy);
+    Args.add(RValue::get(Space), AST.UnsignedIntTy);
+    Args.add(RValue::get(Range), AST.IntTy);
+    Args.add(RValue::get(Index), AST.UnsignedIntTy);
+
+  } else {
+    // implicit binding
+    auto *OrderID =
+        llvm::ConstantInt::get(CGM.IntTy, RBA->getImplicitBindingOrderID());
+    Args.add(RValue::get(Space), AST.UnsignedIntTy);
+    Args.add(RValue::get(Range), AST.IntTy);
+    Args.add(RValue::get(Index), AST.UnsignedIntTy);
+    Args.add(RValue::get(OrderID), AST.UnsignedIntTy);
+  }
+  Args.add(RValue::get(NameStr), AST.getPointerType(AST.CharTy.withConst()));
+}
+
 } // namespace
 
 llvm::Type *
@@ -103,13 +209,6 @@ llvm::Triple::ArchType CGHLSLRuntime::getArch() {
   return CGM.getTarget().getTriple().getArch();
 }
 
-// Returns true if the type is an HLSL resource class or an array of them
-static bool isResourceRecordTypeOrArrayOf(const clang::Type *Ty) {
-  while (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(Ty))
-    Ty = CAT->getArrayElementTypeNoTypeQual();
-  return Ty->isHLSLResourceRecord();
-}
-
 // Emits constant global variables for buffer constants declarations
 // and creates metadata linking the constant globals with the buffer global.
 void CGHLSLRuntime::emitBufferGlobalsAndMetadata(const HLSLBufferDecl *BufDecl,
@@ -146,7 +245,7 @@ void CGHLSLRuntime::emitBufferGlobalsAndMetadata(const HLSLBufferDecl *BufDecl,
     if (VDTy.getAddressSpace() != LangAS::hlsl_constant) {
       if (VD->getStorageClass() == SC_Static ||
           VDTy.getAddressSpace() == LangAS::hlsl_groupshared ||
-          isResourceRecordTypeOrArrayOf(VDTy.getTypePtr())) {
+          VDTy->isHLSLResourceRecord() || VDTy->isHLSLResourceRecordArray()) {
         // Emit static and groupshared variables and resource classes inside
         // cbuffer as regular globals
         CGM.EmitGlobal(VD);
@@ -186,8 +285,7 @@ static const clang::HLSLAttributedResourceType *
 createBufferHandleType(const HLSLBufferDecl *BufDecl) {
   ASTContext &AST = BufDecl->getASTContext();
   QualType QT = AST.getHLSLAttributedResourceType(
-      AST.HLSLResourceTy,
-      QualType(BufDecl->getLayoutStruct()->getTypeForDecl(), 0),
+      AST.HLSLResourceTy, AST.getCanonicalTagType(BufDecl->getLayoutStruct()),
       HLSLAttributedResourceType::Attributes(ResourceClass::CBuffer));
   return cast<HLSLAttributedResourceType>(QT.getTypePtr());
 }
@@ -597,13 +695,6 @@ static void initializeBuffer(CodeGenModule &CGM, llvm::GlobalVariable *GV,
   CGM.AddCXXGlobalInit(InitResFunc);
 }
 
-static Value *buildNameForResource(llvm::StringRef BaseName,
-                                   CodeGenModule &CGM) {
-  std::string Str(BaseName);
-  std::string GlobalName(Str + ".str");
-  return CGM.GetAddrOfConstantCString(Str, GlobalName.c_str()).getPointer();
-}
-
 void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
                                                 llvm::GlobalVariable *GV,
                                                 HLSLVkBindingAttr *VkBinding) {
@@ -631,17 +722,13 @@ void CGHLSLRuntime::initializeBufferFromBinding(const HLSLBufferDecl *BufDecl,
   auto *Index = llvm::ConstantInt::get(CGM.IntTy, 0);
   auto *RangeSize = llvm::ConstantInt::get(CGM.IntTy, 1);
   auto *Space = llvm::ConstantInt::get(CGM.IntTy, RBA->getSpaceNumber());
-  Value *Name = nullptr;
+  Value *Name = buildNameForResource(BufDecl->getName(), CGM);
 
   llvm::Intrinsic::ID IntrinsicID =
       RBA->hasRegisterSlot()
           ? CGM.getHLSLRuntime().getCreateHandleFromBindingIntrinsic()
           : CGM.getHLSLRuntime().getCreateHandleFromImplicitBindingIntrinsic();
 
-  std::string Str(BufDecl->getName());
-  std::string GlobalName(Str + ".str");
-  Name = CGM.GetAddrOfConstantCString(Str, GlobalName.c_str()).getPointer();
-
   // buffer with explicit binding
   if (RBA->hasRegisterSlot()) {
     auto *RegSlot = llvm::ConstantInt::get(CGM.IntTy, RBA->getSlotNumber());
@@ -708,3 +795,91 @@ void CGHLSLRuntime::emitInitListOpaqueValues(CodeGenFunction &CGF,
     }
   }
 }
+
+std::optional<LValue> CGHLSLRuntime::emitResourceArraySubscriptExpr(
+    const ArraySubscriptExpr *ArraySubsExpr, CodeGenFunction &CGF) {
+  assert(ArraySubsExpr->getType()->isHLSLResourceRecord() ||
+         ArraySubsExpr->getType()->isHLSLResourceRecordArray() &&
+             "expected resource array subscript expression");
+
+  // let clang codegen handle local resource array subscripts
+  const VarDecl *ArrayDecl = dyn_cast<VarDecl>(getArrayDecl(ArraySubsExpr));
+  if (!ArrayDecl || !ArrayDecl->hasGlobalStorage())
+    return std::nullopt;
+
+  if (ArraySubsExpr->getType()->isArrayType())
+    // FIXME: this is not yet implemented (llvm/llvm-project#145426)
+    llvm_unreachable(
+        "indexing of sub-arrays of multidimensional arrays not supported yet");
+
+  // get the resource array type
+  ASTContext &AST = ArrayDecl->getASTContext();
+  const Type *ResArrayTy = ArrayDecl->getType().getTypePtr();
+  assert(ResArrayTy->isHLSLResourceRecordArray() &&
+         "expected array of resource classes");
+
+  // Iterate through all nested array subscript expressions to calculate
+  // the index in the flattened resource array (if this is a multi-
+  // dimensional array). The index is calculated as a sum of all indices
+  // multiplied by the total size of the array at that level.
+  Value *Index = nullptr;
+  const ArraySubscriptExpr *ASE = ArraySubsExpr;
+  while (ASE != nullptr) {
+    Value *SubIndex = CGF.EmitScalarExpr(ASE->getIdx());
+    if (const auto *ArrayTy =
+            dyn_cast<ConstantArrayType>(ASE->getType().getTypePtr())) {
+      Value *Multiplier = llvm::ConstantInt::get(
+          CGM.IntTy, AST.getConstantArrayElementCount(ArrayTy));
+      SubIndex = CGF.Builder.CreateMul(SubIndex, Multiplier);
+    }
+
+    Index = Index ? CGF.Builder.CreateAdd(Index, SubIndex) : SubIndex;
+    ASE = dyn_cast<ArraySubscriptExpr>(ASE->getBase()->IgnoreParenImpCasts());
+  }
+
+  // find binding info for the resource array (for implicit binding
+  // an HLSLResourceBindingAttr should have been added by SemaHLSL)
+  QualType ResourceTy = ArraySubsExpr->getType();
+  HLSLVkBindingAttr *VkBinding = ArrayDecl->getAttr<HLSLVkBindingAttr>();
+  HLSLResourceBindingAttr *RBA = ArrayDecl->getAttr<HLSLResourceBindingAttr>();
+  assert((VkBinding || RBA) && "resource array must have a binding attribute");
+
+  // lookup the resource class constructor based on the resource type and
+  // binding
+  CXXConstructorDecl *CD = findResourceConstructorDecl(
+      AST, ResourceTy, VkBinding || RBA->hasRegisterSlot());
+
+  // create a temporary variable for the resource class instance (we need to
+  // return an LValue)
+  RawAddress TmpVar = CGF.CreateMemTemp(ResourceTy);
+  if (CGF.EmitLifetimeStart(TmpVar.getPointer()))
+    CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
+        NormalEHLifetimeMarker, TmpVar);
+
+  AggValueSlot ValueSlot = AggValueSlot::forAddr(
+      TmpVar, Qualifiers(), AggValueSlot::IsDestructed_t(true),
+      AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsAliased_t(false),
+      AggValueSlot::DoesNotOverlap);
+
+  Address ThisAddress = ValueSlot.getAddress();
+  llvm::Value *ThisPtr = CGF.getAsNaturalPointerTo(
+      ThisAddress, CD->getThisType()->getPointeeType());
+
+  // get total array size (= range size)
+  llvm::Value *Range =
+      llvm::ConstantInt::get(CGM.IntTy, getTotalArraySize(AST, ResArrayTy));
+
+  // assemble the constructor parameters
+  CallArgList Args;
+  createResourceCtorArgs(CGM, CD, ThisPtr, Range, Index, ArrayDecl->getName(),
+                         RBA, VkBinding, Args);
+
+  // call the constructor
+  CGF.EmitCXXConstructorCall(CD, Ctor_Complete, false, false, ThisAddress, Args,
+                             ValueSlot.mayOverlap(),
+                             ArraySubsExpr->getExprLoc(),
+                             ValueSlot.isSanitizerChecked());
+
+  return CGF.MakeAddrLValue(TmpVar, ArraySubsExpr->getType(),
+                            AlignmentSource::Decl);
+}
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index 31d1728da9c56..b872f9ef0e9b6 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -68,6 +68,7 @@ class Type;
 class RecordType;
 class DeclContext;
 class HLSLPackOffsetAttr;
+class ArraySubscriptExpr;
 
 class FunctionDecl;
 
@@ -75,6 +76,7 @@ namespace CodeGen {
 
 class CodeGenModule;
 class CodeGenFunction;
+class LValue;
 
 class CGHLSLRuntime {
 public:
@@ -164,6 +166,10 @@ class CGHLSLRuntime {
                                llvm::TargetExtType *LayoutTy);
   void emitInitListOpaqueValues(CodeGenFunction &CGF, InitListExpr *E);
 
+  std::optional<LValue>
+  emitResourceArraySubscriptExpr(const ArraySubscriptExpr *E,
+                                 CodeGenFunction &CGF);
+
 private:
   void emitBufferGlobalsAndMetadata(const HLSLBufferDecl *BufDecl,
                                     llvm::GlobalVariable *BufGV);
diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
index e0983ef256e71..1b941fff8b644 100644
--- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp
+++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
@@ -39,7 +39,8 @@ template <class Derived> struct StructVisitor {
 
   template <class... Ts>
   void visitStructFields(QualType QT, CharUnits CurStructOffset, Ts... Args) {
-    const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
+    const RecordDecl *RD =
+        QT->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
 
     // Iterate over the fields of the struct.
     for (const FieldDecl *FD : RD->fields()) {
@@ -464,7 +465,8 @@ template <class Derived> struct GenFuncBase {
 
       if (WrongType) {
         std::string FuncName = std::string(F->getName());
-        SourceLocation Loc = QT->castAs<RecordType>()->getDecl()->getLocation();
+        SourceLocation Loc =
+            QT->castAs<RecordType>()->getOriginalDecl()->getLocation();
         CGM.Error(Loc, "special function " + FuncName +
                            " for non-trivial C struct has incorrect type");
         return nullptr;
@@ -560,7 +562,8 @@ struct GenBinaryFunc : CopyStructVisitor<Derived, IsMove>,
       if (FD->isZeroLengthBitField())
         return;
 
-      QualType RT = QualType(FD->getParent()->getTypeForDecl(), 0);
+      CanQualType RT =
+          this->CGF->getContext().getCanonicalTagType(FD->getParent());
       llvm::Type *Ty = this->CGF->ConvertType(RT);
       Address DstAddr = this->getAddrWithOffset(Addrs[DstIdx], Offset);
       LValue DstBase =
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index 24b6ce7c1c70d..b5f17b812222a 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -1000,7 +1000,9 @@ PropertyImplStrategy::PropertyImplStrategy(CodeGenModule &CGM,
   // Compute whether the ivar has strong members.
   if (CGM.getLangOpts().getGC())
     if (const RecordType *recordType = ivarType->getAs<RecordType>())
-      HasStrong = recordType->getDecl()->hasObjectMember();
+      HasStrong = recordType->getOriginalDecl()
+                      ->getDefinitionOrSelf()
+                      ->hasObjectMember();
 
   // We can never access structs with object members with a native
   // access, because we need to use write barriers.  This is what
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index 8c66176942cb5..eb4904050ae0f 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -2495,7 +2495,7 @@ void CGObjCCommonMac::BuildRCBlockVarRecordLayout(const RecordType *RT,
                                                   CharUnits BytePos,
                                                   bool &HasUnion,
                                                   bool ByrefLayout) {
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
   SmallVector<const FieldDecl *, 16> Fields(RD->fields());
   llvm::Type *Ty = CGM.getTypes().ConvertType(QualType(RT, 0));
   const llvm::StructLayout *RecLayout =
@@ -3354,7 +3354,8 @@ static bool hasWeakMember(QualType type) {
   }
 
   if (auto recType = type->getAs<RecordType>()) {
-    for (auto *field : recType->getDecl()->fields()) {
+    for (auto *field :
+         recType->getOriginalDecl()->getDefinitionOrSelf()->fields()) {
       if (hasWeakMember(field->getType()))
         return true;
     }
@@ -5184,7 +5185,7 @@ CGObjCCommonMac::GetIvarLayoutName(IdentifierInfo *Ident,
 }
 
 void IvarLayoutBuilder::visitRecord(const RecordType *RT, CharUnits offset) {
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
 
   // If this is a union, remember that we had one, because it might mess
   // up the ordering of layout entries.
@@ -5670,7 +5671,7 @@ ObjCCommonTypesHelper::ObjCCommonTypesHelper(CodeGen::CodeGenModule &cgm)
                                 nullptr, false, ICIS_NoInit));
   RD->completeDefinition();
 
-  SuperCTy = Ctx.getTagDeclType(RD);
+  SuperCTy = Ctx.getCanonicalTagType(RD);
   SuperPtrCTy = Ctx.getPointerType(SuperCTy);
 
   SuperTy = cast<llvm::StructType>(Types.ConvertType(SuperCTy));
@@ -6016,7 +6017,7 @@ ObjCNonFragileABITypesHelper::ObjCNonFragileABITypesHelper(
                                 false, ICIS_NoInit));
   RD->completeDefinition();
 
-  MessageRefCTy = Ctx.getTagDeclType(RD);
+  MessageRefCTy = Ctx.getCanonicalTagType(RD);
   MessageRefCPtrTy = Ctx.getPointerType(MessageRefCTy);
   MessageRefTy = cast<llvm::StructType>(Types.ConvertType(MessageRefCTy));
 
diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp
index 6e2f32022a01e..cbf99534d2ce6 100644
--- a/clang/lib/CodeGen/CGObjCRuntime.cpp
+++ b/clang/lib/CodeGen/CGObjCRuntime.cpp
@@ -440,7 +440,9 @@ void CGObjCRuntime::destroyCalleeDestroyedArguments(CodeGenFunction &CGF,
     } else {
       QualType QT = param->getType();
       auto *RT = QT->getAs<RecordType>();
-      if (RT && RT->getDecl()->isParamDestroyedInCallee()) {
+      if (RT && RT->getOriginalDecl()
+                    ->getDefinitionOrSelf()
+                    ->isParamDestroyedInCallee()) {
         RValue RV = I->getRValue(CGF);
         QualType::DestructionKind DtorKind = QT.isDestructedType();
         switch (DtorKind) {
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 91237cfe3a372..f98339d472fa9 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -2915,7 +2915,7 @@ createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind,
   addFieldToRecordDecl(C, UD, KmpInt32Ty);
   addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy);
   UD->completeDefinition();
-  QualType KmpCmplrdataTy = C.getRecordType(UD);
+  CanQualType KmpCmplrdataTy = C.getCanonicalTagType(UD);
   RecordDecl *RD = C.buildImplicitRecord("kmp_task_t");
   RD->startDefinition();
   addFieldToRecordDecl(C, RD, C.VoidPtrTy);
@@ -2950,7 +2950,7 @@ createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
   RD->startDefinition();
   addFieldToRecordDecl(C, RD, KmpTaskTQTy);
   if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates))
-    addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
+    addFieldToRecordDecl(C, RD, C.getCanonicalTagType(PrivateRD));
   RD->completeDefinition();
   return RD;
 }
@@ -3582,7 +3582,7 @@ static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
     addFieldToRecordDecl(C, KmpAffinityInfoRD, C.getSizeType());
     addFieldToRecordDecl(C, KmpAffinityInfoRD, FlagsTy);
     KmpAffinityInfoRD->completeDefinition();
-    KmpTaskAffinityInfoTy = C.getRecordType(KmpAffinityInfoRD);
+    KmpTaskAffinityInfoTy = C.getCanonicalTagType(KmpAffinityInfoRD);
   }
 }
 
@@ -3640,7 +3640,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
   // Build type kmp_task_t (if not built yet).
   if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) {
     if (SavedKmpTaskloopTQTy.isNull()) {
-      SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl(
+      SavedKmpTaskloopTQTy = C.getCanonicalTagType(createKmpTaskTRecordDecl(
           CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
     }
     KmpTaskTQTy = SavedKmpTaskloopTQTy;
@@ -3650,7 +3650,7 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
             isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) &&
            "Expected taskloop, task or target directive");
     if (SavedKmpTaskTQTy.isNull()) {
-      SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl(
+      SavedKmpTaskTQTy = C.getCanonicalTagType(createKmpTaskTRecordDecl(
           CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy));
     }
     KmpTaskTQTy = SavedKmpTaskTQTy;
@@ -3659,7 +3659,8 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
   // Build particular struct kmp_task_t for the given task.
   const RecordDecl *KmpTaskTWithPrivatesQTyRD =
       createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates);
-  QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
+  CanQualType KmpTaskTWithPrivatesQTy =
+      C.getCanonicalTagType(KmpTaskTWithPrivatesQTyRD);
   QualType KmpTaskTWithPrivatesPtrQTy =
       C.getPointerType(KmpTaskTWithPrivatesQTy);
   llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.Builder.getPtrTy(0);
@@ -3914,7 +3915,10 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
   // Fill the data in the resulting kmp_task_t record.
   // Copy shareds if there are any.
   Address KmpTaskSharedsPtr = Address::invalid();
-  if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) {
+  if (!SharedsTy->getAsStructureType()
+           ->getOriginalDecl()
+           ->getDefinitionOrSelf()
+           ->field_empty()) {
     KmpTaskSharedsPtr = Address(
         CGF.EmitLoadOfScalar(
             CGF.EmitLValueForField(
@@ -3944,8 +3948,11 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
   enum { Priority = 0, Destructors = 1 };
   // Provide pointer to function with destructors for privates.
   auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
-  const RecordDecl *KmpCmplrdataUD =
-      (*FI)->getType()->getAsUnionType()->getDecl();
+  const RecordDecl *KmpCmplrdataUD = (*FI)
+                                         ->getType()
+                                         ->getAsUnionType()
+                                         ->getOriginalDecl()
+                                         ->getDefinitionOrSelf();
   if (NeedsCleanup) {
     llvm::Value *DestructorFn = emitDestructorsFunction(
         CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
@@ -4015,7 +4022,7 @@ static void getDependTypes(ASTContext &C, QualType &KmpDependInfoTy,
     addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType());
     addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy);
     KmpDependInfoRD->completeDefinition();
-    KmpDependInfoTy = C.getRecordType(KmpDependInfoRD);
+    KmpDependInfoTy = C.getCanonicalTagType(KmpDependInfoRD);
   }
 }
 
@@ -5714,7 +5721,7 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
   const FieldDecl *FlagsFD = addFieldToRecordDecl(
       C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false));
   RD->completeDefinition();
-  QualType RDType = C.getRecordType(RD);
+  CanQualType RDType = C.getCanonicalTagType(RD);
   unsigned Size = Data.ReductionVars.size();
   llvm::APInt ArraySize(/*numBits=*/64, Size);
   QualType ArrayRDType =
@@ -7487,7 +7494,32 @@ class MappableExprsHandler {
     // dimension.
     uint64_t DimSize = 1;
 
-    bool IsNonContiguous = CombinedInfo.NonContigInfo.IsNonContiguous;
+    // Detects non-contiguous updates due to strided accesses.
+    // Sets the 'IsNonContiguous' flag so that the 'MapType' bits are set
+    // correctly when generating information to be passed to the runtime. The
+    // flag is set to true if any array section has a stride not equal to 1, or
+    // if the stride is not a constant expression (conservatively assumed
+    // non-contiguous).
+    bool IsNonContiguous =
+        CombinedInfo.NonContigInfo.IsNonContiguous ||
+        any_of(Components, [&](const auto &Component) {
+          const auto *OASE =
+              dyn_cast<ArraySectionExpr>(Component.getAssociatedExpression());
+          if (!OASE)
+            return false;
+
+          const Expr *StrideExpr = OASE->getStride();
+          if (!StrideExpr)
+            return false;
+
+          const auto Constant =
+              StrideExpr->getIntegerConstantExpr(CGF.getContext());
+          if (!Constant)
+            return false;
+
+          return !Constant->isOne();
+        });
+
     bool IsPrevMemberReference = false;
 
     bool IsPartialMapped =
@@ -10703,7 +10735,7 @@ static unsigned evaluateCDTSize(const FunctionDecl *FD,
     unsigned Offset = 0;
     if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
       if (ParamAttrs[Offset].Kind == Vector)
-        CDT = C.getPointerType(C.getRecordType(MD->getParent()));
+        CDT = C.getPointerType(C.getCanonicalTagType(MD->getParent()));
       ++Offset;
     }
     if (CDT.isNull()) {
@@ -11285,7 +11317,7 @@ void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF,
     addFieldToRecordDecl(C, RD, Int64Ty);
     addFieldToRecordDecl(C, RD, Int64Ty);
     RD->completeDefinition();
-    KmpDimTy = C.getRecordType(RD);
+    KmpDimTy = C.getCanonicalTagType(RD);
   } else {
     RD = cast<RecordDecl>(KmpDimTy->getAsTagDecl());
   }
@@ -11781,7 +11813,7 @@ Address CGOpenMPRuntime::emitLastprivateConditionalInit(CodeGenFunction &CGF,
     VDField = addFieldToRecordDecl(C, RD, VD->getType().getNonReferenceType());
     FiredField = addFieldToRecordDecl(C, RD, C.CharTy);
     RD->completeDefinition();
-    NewType = C.getRecordType(RD);
+    NewType = C.getCanonicalTagType(RD);
     Address Addr = CGF.CreateMemTemp(NewType, C.getDeclAlign(VD), VD->getName());
     BaseLVal = CGF.MakeAddrLValue(Addr, NewType, AlignmentSource::Decl);
     I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 04c91920c9b7e..cff1071dd1c2e 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -769,7 +769,7 @@ void CGOpenMPRuntimeGPU::emitKernelDeinit(CodeGenFunction &CGF,
       "_openmp_teams_reduction_type_$_", RecordDecl::TagKind::Union);
   StaticRD->startDefinition();
   for (const RecordDecl *TeamReductionRec : TeamsReductions) {
-    QualType RecTy = C.getRecordType(TeamReductionRec);
+    CanQualType RecTy = C.getCanonicalTagType(TeamReductionRec);
     auto *Field = FieldDecl::Create(
         C, StaticRD, SourceLocation(), SourceLocation(), nullptr, RecTy,
         C.getTrivialTypeSourceInfo(RecTy, SourceLocation()),
@@ -779,7 +779,7 @@ void CGOpenMPRuntimeGPU::emitKernelDeinit(CodeGenFunction &CGF,
     StaticRD->addDecl(Field);
   }
   StaticRD->completeDefinition();
-  QualType StaticTy = C.getRecordType(StaticRD);
+  CanQualType StaticTy = C.getCanonicalTagType(StaticRD);
   llvm::Type *LLVMReductionsBufferTy =
       CGM.getTypes().ConvertTypeForMem(StaticTy);
   const auto &DL = CGM.getModule().getDataLayout();
diff --git a/clang/lib/CodeGen/CGPointerAuth.cpp b/clang/lib/CodeGen/CGPointerAuth.cpp
index dcef01a5eb6d3..375f87a68ee6d 100644
--- a/clang/lib/CodeGen/CGPointerAuth.cpp
+++ b/clang/lib/CodeGen/CGPointerAuth.cpp
@@ -466,6 +466,14 @@ llvm::Constant *CodeGenModule::getConstantSignedPointer(
                                   OtherDiscriminator);
 }
 
+llvm::Constant *
+CodeGen::getConstantSignedPointer(CodeGenModule &CGM, llvm::Constant *Pointer,
+                                  unsigned Key, llvm::Constant *StorageAddress,
+                                  llvm::ConstantInt *OtherDiscriminator) {
+  return CGM.getConstantSignedPointer(Pointer, Key, StorageAddress,
+                                      OtherDiscriminator);
+}
+
 /// If applicable, sign a given constant function pointer with the ABI rules for
 /// functionType.
 llvm::Constant *CodeGenModule::getFunctionPointer(llvm::Constant *Pointer,
@@ -531,7 +539,7 @@ llvm::Constant *CodeGenModule::getMemberFunctionPointer(llvm::Constant *Pointer,
 llvm::Constant *CodeGenModule::getMemberFunctionPointer(const FunctionDecl *FD,
                                                         llvm::Type *Ty) {
   QualType FT = FD->getType();
-  FT = getContext().getMemberPointerType(FT, /*Qualifier=*/nullptr,
+  FT = getContext().getMemberPointerType(FT, /*Qualifier=*/std::nullopt,
                                          cast<CXXMethodDecl>(FD)->getParent());
   return getMemberFunctionPointer(getRawFunctionPointer(FD, Ty), FT);
 }
diff --git a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
index e1310aed818ad..5f6136c917ac2 100644
--- a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -1027,8 +1027,11 @@ void CGRecordLowering::fillOutputFields() {
       if (Member.FD)
         Fields[Member.FD->getCanonicalDecl()] = FieldTypes.size() - 1;
       // A field without storage must be a bitfield.
-      if (!Member.Data)
+      if (!Member.Data) {
+        assert(Member.FD &&
+               "Member.Data is a nullptr so Member.FD should not be");
         setBitFieldInfo(Member.FD, Member.Offset, FieldTypes.back());
+      }
     } else if (Member.Kind == MemberInfo::Base)
       NonVirtualBases[Member.RD] = FieldTypes.size() - 1;
     else if (Member.Kind == MemberInfo::VBase)
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 1a8c6f015bda1..031ef73214e76 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -3279,7 +3279,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
 
 LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
   const RecordDecl *RD = S.getCapturedRecordDecl();
-  QualType RecordTy = getContext().getRecordType(RD);
+  CanQualType RecordTy = getContext().getCanonicalTagType(RD);
 
   // Initialize the captured struct.
   LValue SlotLV =
@@ -3359,7 +3359,7 @@ CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
 
   // Initialize variable-length arrays.
   LValue Base = MakeNaturalAlignRawAddrLValue(
-      CapturedStmtInfo->getContextValue(), Ctx.getTagDeclType(RD));
+      CapturedStmtInfo->getContextValue(), Ctx.getCanonicalTagType(RD));
   for (auto *FD : RD->fields()) {
     if (FD->hasCapturedVLAType()) {
       auto *ExprArg =
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 5822e0f6db89a..f6a0ca574a191 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -5273,7 +5273,8 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
   // Emit outlined function for task construct.
   const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
   Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
-  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
+  CanQualType SharedsTy =
+      getContext().getCanonicalTagType(CS->getCapturedRecordDecl());
   auto I = CS->getCapturedDecl()->param_begin();
   auto PartId = std::next(I);
   auto TaskT = std::next(I, 4);
@@ -5507,7 +5508,8 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
   // Emit outlined function for task construct.
   const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
   Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
-  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
+  CanQualType SharedsTy =
+      getContext().getCanonicalTagType(CS->getCapturedRecordDecl());
   const Expr *IfCond = nullptr;
   for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
     if (C->getNameModifier() == OMPD_unknown ||
@@ -7890,7 +7892,8 @@ void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
     OMPLexicalScope Scope(*this, S, OMPD_taskloop, /*EmitPreInitStmt=*/false);
     CapturedStruct = GenerateCapturedStmtArgument(*CS);
   }
-  QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
+  CanQualType SharedsTy =
+      getContext().getCanonicalTagType(CS->getCapturedRecordDecl());
   const Expr *IfCond = nullptr;
   for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
     if (C->getNameModifier() == OMPD_unknown ||
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 0b6e830e0d557..e14e883a55ac5 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -971,7 +971,7 @@ llvm::GlobalVariable *CodeGenVTables::GenerateConstructionVTable(
   VTable->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
 
   llvm::Constant *RTTI = CGM.GetAddrOfRTTIDescriptor(
-      CGM.getContext().getTagDeclType(Base.getBase()));
+      CGM.getContext().getCanonicalTagType(Base.getBase()));
 
   // Create and set the initializer.
   ConstantInitBuilder builder(CGM);
@@ -1382,8 +1382,8 @@ void CodeGenModule::EmitVTableTypeMetadata(const CXXRecordDecl *RD,
                        AP.second.AddressPointIndex,
                    {}};
     llvm::raw_string_ostream Stream(N.TypeName);
-    getCXXABI().getMangleContext().mangleCanonicalTypeName(
-        QualType(N.Base->getTypeForDecl(), 0), Stream);
+    CanQualType T = getContext().getCanonicalTagType(N.Base);
+    getCXXABI().getMangleContext().mangleCanonicalTypeName(T, Stream);
     AddressPoints.push_back(std::move(N));
   }
 
@@ -1404,7 +1404,7 @@ void CodeGenModule::EmitVTableTypeMetadata(const CXXRecordDecl *RD,
         continue;
       llvm::Metadata *MD = CreateMetadataIdentifierForVirtualMemPtrType(
           Context.getMemberPointerType(Comps[I].getFunctionDecl()->getType(),
-                                       /*Qualifier=*/nullptr, AP.Base));
+                                       /*Qualifier=*/std::nullopt, AP.Base));
       VTable->addTypeMetadata((ComponentWidth * I).getQuantity(), MD);
     }
   }
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index ab345a598c4e8..9854dae33d6f3 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -2223,7 +2223,9 @@ CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
   // Ignore empty classes in C++.
   if (getLangOpts().CPlusPlus) {
     if (const RecordType *RT = Ty->getAs<RecordType>()) {
-      if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
+      if (cast<CXXRecordDecl>(RT->getOriginalDecl())
+              ->getDefinitionOrSelf()
+              ->isEmpty())
         return;
     }
   }
@@ -2494,10 +2496,6 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
     case Type::PredefinedSugar:
       llvm_unreachable("type class is never variably-modified!");
 
-    case Type::Elaborated:
-      type = cast<ElaboratedType>(ty)->getNamedType();
-      break;
-
     case Type::Adjusted:
       type = cast<AdjustedType>(ty)->getAdjustedType();
       break;
@@ -2829,6 +2827,9 @@ void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
   if (!FD)
     return;
 
+  bool IsAlwaysInline = TargetDecl->hasAttr<AlwaysInlineAttr>();
+  bool IsFlatten = FD && FD->hasAttr<FlattenAttr>();
+
   // Grab the required features for the call. For a builtin this is listed in
   // the td file with the default cpu, for an always_inline function this is any
   // listed cpu and any listed features.
@@ -2871,25 +2872,39 @@ void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
       if (F.getValue())
         ReqFeatures.push_back(F.getKey());
     }
-    if (!llvm::all_of(ReqFeatures, [&](StringRef Feature) {
-      if (!CallerFeatureMap.lookup(Feature)) {
-        MissingFeature = Feature.str();
-        return false;
-      }
-      return true;
-    }) && !IsHipStdPar)
-      CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
-          << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
+    if (!llvm::all_of(ReqFeatures,
+                      [&](StringRef Feature) {
+                        if (!CallerFeatureMap.lookup(Feature)) {
+                          MissingFeature = Feature.str();
+                          return false;
+                        }
+                        return true;
+                      }) &&
+        !IsHipStdPar) {
+      if (IsAlwaysInline)
+        CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
+            << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
+      else if (IsFlatten)
+        CGM.getDiags().Report(Loc, diag::err_flatten_function_needs_feature)
+            << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
+    }
+
   } else if (!FD->isMultiVersion() && FD->hasAttr<TargetAttr>()) {
     llvm::StringMap<bool> CalleeFeatureMap;
     CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
 
     for (const auto &F : CalleeFeatureMap) {
-      if (F.getValue() && (!CallerFeatureMap.lookup(F.getKey()) ||
-                           !CallerFeatureMap.find(F.getKey())->getValue()) &&
-          !IsHipStdPar)
-        CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
-            << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
+      if (F.getValue() &&
+          (!CallerFeatureMap.lookup(F.getKey()) ||
+           !CallerFeatureMap.find(F.getKey())->getValue()) &&
+          !IsHipStdPar) {
+        if (IsAlwaysInline)
+          CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
+              << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
+        else if (IsFlatten)
+          CGM.getDiags().Report(Loc, diag::err_flatten_function_needs_feature)
+              << FD->getDeclName() << TargetDecl->getDeclName() << F.getKey();
+      }
     }
   }
 }
@@ -2905,10 +2920,16 @@ void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
 
 void CodeGenFunction::EmitKCFIOperandBundle(
     const CGCallee &Callee, SmallVectorImpl<llvm::OperandBundleDef> &Bundles) {
-  const FunctionProtoType *FP =
-      Callee.getAbstractInfo().getCalleeFunctionProtoType();
-  if (FP)
-    Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar()));
+  const CGCalleeInfo &CI = Callee.getAbstractInfo();
+  const FunctionProtoType *FP = CI.getCalleeFunctionProtoType();
+  if (!FP)
+    return;
+
+  StringRef Salt;
+  if (const auto &Info = FP->getExtraAttributeInfo())
+    Salt = Info.CFISalt;
+
+  Bundles.emplace_back("kcfi", CGM.CreateKCFITypeId(FP->desugar(), Salt));
 }
 
 llvm::Value *
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6c32c98cec011..fc65199a0f154 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -701,14 +701,12 @@ class CodeGenFunction : public CodeGenTypeCache {
     bool isRedundantBeforeReturn() override { return true; }
 
     llvm::Value *Addr;
-    llvm::Value *Size;
 
   public:
-    CallLifetimeEnd(RawAddress addr, llvm::Value *size)
-        : Addr(addr.getPointer()), Size(size) {}
+    CallLifetimeEnd(RawAddress addr) : Addr(addr.getPointer()) {}
 
     void Emit(CodeGenFunction &CGF, Flags flags) override {
-      CGF.EmitLifetimeEnd(Size, Addr);
+      CGF.EmitLifetimeEnd(Addr);
     }
   };
 
@@ -727,7 +725,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   };
 
   /// Header for data within LifetimeExtendedCleanupStack.
-  struct LifetimeExtendedCleanupHeader {
+  struct alignas(uint64_t) LifetimeExtendedCleanupHeader {
     /// The size of the following cleanup object.
     unsigned Size;
     /// The kind of cleanup to push.
@@ -949,7 +947,8 @@ class CodeGenFunction : public CodeGenTypeCache {
         LifetimeExtendedCleanupStack.size() + sizeof(Header) + Header.Size +
         (Header.IsConditional ? sizeof(ActiveFlag) : 0));
 
-    static_assert(sizeof(Header) % alignof(T) == 0,
+    static_assert((alignof(LifetimeExtendedCleanupHeader) == alignof(T)) &&
+                      (alignof(T) == alignof(RawAddress)),
                   "Cleanup will be allocated on misaligned address");
     char *Buffer = &LifetimeExtendedCleanupStack[OldSize];
     new (Buffer) LifetimeExtendedCleanupHeader(Header);
@@ -2805,6 +2804,13 @@ class CodeGenFunction : public CodeGenTypeCache {
     AllocaTracker Tracker;
   };
 
+private:
+  /// If \p Alloca is not in the same address space as \p DestLangAS, insert an
+  /// address space cast and return a new RawAddress based on this value.
+  RawAddress MaybeCastStackAddressSpace(RawAddress Alloca, LangAS DestLangAS,
+                                        llvm::Value *ArraySize = nullptr);
+
+public:
   /// CreateTempAlloca - This creates an alloca and inserts it into the entry
   /// block if \p ArraySize is nullptr, otherwise inserts it at the current
   /// insertion point of the builder. The caller is responsible for setting an
@@ -2974,7 +2980,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// member.
   bool hasVolatileMember(QualType T) {
     if (const RecordType *RT = T->getAs<RecordType>()) {
-      const RecordDecl *RD = cast<RecordDecl>(RT->getDecl());
+      const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       return RD->hasVolatileMember();
     }
     return false;
@@ -3233,8 +3239,8 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitSehTryScopeBegin();
   void EmitSehTryScopeEnd();
 
-  llvm::Value *EmitLifetimeStart(llvm::TypeSize Size, llvm::Value *Addr);
-  void EmitLifetimeEnd(llvm::Value *Size, llvm::Value *Addr);
+  bool EmitLifetimeStart(llvm::Value *Addr);
+  void EmitLifetimeEnd(llvm::Value *Addr);
 
   llvm::Value *EmitCXXNewExpr(const CXXNewExpr *E);
   void EmitCXXDeleteExpr(const CXXDeleteExpr *E);
@@ -3417,8 +3423,8 @@ class CodeGenFunction : public CodeGenTypeCache {
     /// initializer.
     bool IsConstantAggregate;
 
-    /// Non-null if we should use lifetime annotations.
-    llvm::Value *SizeForLifetimeMarkers;
+    /// True if lifetime markers should be used.
+    bool UseLifetimeMarkers;
 
     /// Address with original alloca instruction. Invalid if the variable was
     /// emitted as a global constant.
@@ -3432,20 +3438,14 @@ class CodeGenFunction : public CodeGenTypeCache {
     AutoVarEmission(const VarDecl &variable)
         : Variable(&variable), Addr(Address::invalid()), NRVOFlag(nullptr),
           IsEscapingByRef(false), IsConstantAggregate(false),
-          SizeForLifetimeMarkers(nullptr), AllocaAddr(RawAddress::invalid()) {}
+          UseLifetimeMarkers(false), AllocaAddr(RawAddress::invalid()) {}
 
     bool wasEmittedAsGlobal() const { return !Addr.isValid(); }
 
   public:
     static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }
 
-    bool useLifetimeMarkers() const {
-      return SizeForLifetimeMarkers != nullptr;
-    }
-    llvm::Value *getSizeForLifetimeMarkers() const {
-      assert(useLifetimeMarkers());
-      return SizeForLifetimeMarkers;
-    }
+    bool useLifetimeMarkers() const { return UseLifetimeMarkers; }
 
     /// Returns the raw, allocated address, which is not necessarily
     /// the address of the object itself. It is casted to default
@@ -4560,7 +4560,7 @@ class CodeGenFunction : public CodeGenTypeCache {
                                        ArrayRef<llvm::Value *> args);
 
   CGCallee BuildAppleKextVirtualCall(const CXXMethodDecl *MD,
-                                     NestedNameSpecifier *Qual, llvm::Type *Ty);
+                                     NestedNameSpecifier Qual, llvm::Type *Ty);
 
   CGCallee BuildAppleKextVirtualDestructorCall(const CXXDestructorDecl *DD,
                                                CXXDtorType Type,
@@ -4665,7 +4665,7 @@ class CodeGenFunction : public CodeGenTypeCache {
                                llvm::CallBase **CallOrInvoke = nullptr);
   RValue EmitCXXMemberOrOperatorMemberCallExpr(
       const CallExpr *CE, const CXXMethodDecl *MD, ReturnValueSlot ReturnValue,
-      bool HasQualifier, NestedNameSpecifier *Qualifier, bool IsArrow,
+      bool HasQualifier, NestedNameSpecifier Qualifier, bool IsArrow,
       const Expr *Base, llvm::CallBase **CallOrInvoke);
   // Compute the object pointer.
   Address EmitCXXMemberDataPointerAddress(
@@ -5220,6 +5220,12 @@ class CodeGenFunction : public CodeGenTypeCache {
   /// operation is a subtraction.
   enum { NotSubtraction = false, IsSubtraction = true };
 
+  /// Emit pointer + index arithmetic.
+  llvm::Value *EmitPointerArithmetic(const BinaryOperator *BO,
+                                     Expr *pointerOperand, llvm::Value *pointer,
+                                     Expr *indexOperand, llvm::Value *index,
+                                     bool isSubtraction);
+
   /// Same as IRBuilder::CreateInBoundsGEP, but additionally emits a check to
   /// detect undefined behavior when the pointer overflow sanitizer is enabled.
   /// \p SignedIndices indicates whether any of the GEP indices are signed.
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 834b1c067d84c..677d8bc82cb0a 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -2335,7 +2335,40 @@ llvm::ConstantInt *CodeGenModule::CreateCrossDsoCfiTypeId(llvm::Metadata *MD) {
   return llvm::ConstantInt::get(Int64Ty, llvm::MD5Hash(MDS->getString()));
 }
 
-llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T) {
+// Generalize pointer types to a void pointer with the qualifiers of the
+// originally pointed-to type, e.g. 'const char *' and 'char * const *'
+// generalize to 'const void *' while 'char *' and 'const char **' generalize to
+// 'void *'.
+static QualType GeneralizeType(ASTContext &Ctx, QualType Ty) {
+  if (!Ty->isPointerType())
+    return Ty;
+
+  return Ctx.getPointerType(
+      QualType(Ctx.VoidTy)
+          .withCVRQualifiers(Ty->getPointeeType().getCVRQualifiers()));
+}
+
+// Apply type generalization to a FunctionType's return and argument types
+static QualType GeneralizeFunctionType(ASTContext &Ctx, QualType Ty) {
+  if (auto *FnType = Ty->getAs<FunctionProtoType>()) {
+    SmallVector<QualType, 8> GeneralizedParams;
+    for (auto &Param : FnType->param_types())
+      GeneralizedParams.push_back(GeneralizeType(Ctx, Param));
+
+    return Ctx.getFunctionType(GeneralizeType(Ctx, FnType->getReturnType()),
+                               GeneralizedParams, FnType->getExtProtoInfo());
+  }
+
+  if (auto *FnType = Ty->getAs<FunctionNoProtoType>())
+    return Ctx.getFunctionNoProtoType(
+        GeneralizeType(Ctx, FnType->getReturnType()));
+
+  llvm_unreachable("Encountered unknown FunctionType");
+}
+
+llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T, StringRef Salt) {
+  if (getCodeGenOpts().SanitizeCfiICallGeneralizePointers)
+    T = GeneralizeFunctionType(getContext(), T);
   if (auto *FnType = T->getAs<FunctionProtoType>())
     T = getContext().getFunctionType(
         FnType->getReturnType(), FnType->getParamTypes(),
@@ -2346,8 +2379,13 @@ llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T) {
   getCXXABI().getMangleContext().mangleCanonicalTypeName(
       T, Out, getCodeGenOpts().SanitizeCfiICallNormalizeIntegers);
 
+  if (!Salt.empty())
+    Out << "." << Salt;
+
   if (getCodeGenOpts().SanitizeCfiICallNormalizeIntegers)
     Out << ".normalized";
+  if (getCodeGenOpts().SanitizeCfiICallGeneralizePointers)
+    Out << ".generalized";
 
   return llvm::ConstantInt::get(Int32Ty,
                                 static_cast<uint32_t>(llvm::xxHash64(OutName)));
@@ -2793,7 +2831,7 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
     for (const CXXRecordDecl *Base : getMostBaseClasses(MD->getParent())) {
       llvm::Metadata *Id =
           CreateMetadataIdentifierForType(Context.getMemberPointerType(
-              MD->getType(), /*Qualifier=*/nullptr, Base));
+              MD->getType(), /*Qualifier=*/std::nullopt, Base));
       F->addTypeMetadata(0, Id);
     }
   }
@@ -3012,9 +3050,15 @@ void CodeGenModule::createFunctionTypeMetadataForIcall(const FunctionDecl *FD,
 void CodeGenModule::setKCFIType(const FunctionDecl *FD, llvm::Function *F) {
   llvm::LLVMContext &Ctx = F->getContext();
   llvm::MDBuilder MDB(Ctx);
+  llvm::StringRef Salt;
+
+  if (const auto *FP = FD->getType()->getAs<FunctionProtoType>())
+    if (const auto &Info = FP->getExtraAttributeInfo())
+      Salt = Info.CFISalt;
+
   F->setMetadata(llvm::LLVMContext::MD_kcfi_type,
-                 llvm::MDNode::get(
-                     Ctx, MDB.createConstant(CreateKCFITypeId(FD->getType()))));
+                 llvm::MDNode::get(Ctx, MDB.createConstant(CreateKCFITypeId(
+                                            FD->getType(), Salt))));
 }
 
 static bool allowKCFIIdentifier(StringRef Name) {
@@ -4152,9 +4196,11 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
 // Check if T is a class type with a destructor that's not dllimport.
 static bool HasNonDllImportDtor(QualType T) {
   if (const auto *RT = T->getBaseElementTypeUnsafe()->getAs<RecordType>())
-    if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
+    if (auto *RD = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl())) {
+      RD = RD->getDefinitionOrSelf();
       if (RD->getDestructor() && !RD->getDestructor()->hasAttr<DLLImportAttr>())
         return true;
+    }
 
   return false;
 }
@@ -5750,11 +5796,16 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
       (D->getType()->isCUDADeviceBuiltinSurfaceType() ||
        D->getType()->isCUDADeviceBuiltinTextureType());
   if (getLangOpts().CUDA &&
-      (IsCUDASharedVar || IsCUDAShadowVar || IsCUDADeviceShadowVar))
+      (IsCUDASharedVar || IsCUDAShadowVar || IsCUDADeviceShadowVar)) {
     Init = llvm::UndefValue::get(getTypes().ConvertTypeForMem(ASTTy));
-  else if (D->hasAttr<LoaderUninitializedAttr>())
+  } else if (getLangOpts().HLSL &&
+             (D->getType()->isHLSLResourceRecord() ||
+              D->getType()->isHLSLResourceRecordArray())) {
+    Init = llvm::PoisonValue::get(getTypes().ConvertType(ASTTy));
+    NeedsGlobalCtor = D->getType()->isHLSLResourceRecord();
+  } else if (D->hasAttr<LoaderUninitializedAttr>()) {
     Init = llvm::UndefValue::get(getTypes().ConvertTypeForMem(ASTTy));
-  else if (!InitExpr) {
+  } else if (!InitExpr) {
     // This is a tentative definition; tentative definitions are
     // implicitly initialized with { 0 }.
     //
@@ -5775,11 +5826,7 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
       if (D->getType()->isReferenceType())
         T = D->getType();
 
-      if (getLangOpts().HLSL &&
-          D->getType().getTypePtr()->isHLSLResourceRecord()) {
-        Init = llvm::PoisonValue::get(getTypes().ConvertType(ASTTy));
-        NeedsGlobalCtor = true;
-      } else if (getLangOpts().CPlusPlus) {
+      if (getLangOpts().CPlusPlus) {
         Init = EmitNullConstant(T);
         if (!IsDefinitionAvailableExternally)
           NeedsGlobalCtor = true;
@@ -6029,7 +6076,7 @@ static bool isVarDeclStrongDefinition(const ASTContext &Context,
       return true;
 
     if (const auto *RT = VarType->getAs<RecordType>()) {
-      const RecordDecl *RD = RT->getDecl();
+      const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       for (const FieldDecl *FD : RD->fields()) {
         if (FD->isBitField())
           continue;
@@ -6738,7 +6785,7 @@ QualType CodeGenModule::getObjCFastEnumerationStateType() {
     }
 
     D->completeDefinition();
-    ObjCFastEnumerationStateType = Context.getTagDeclType(D);
+    ObjCFastEnumerationStateType = Context.getCanonicalTagType(D);
   }
 
   return ObjCFastEnumerationStateType;
@@ -7248,7 +7295,8 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
     CXXRecordDecl *CRD = cast<CXXRecordDecl>(D);
     if (CGDebugInfo *DI = getModuleDebugInfo()) {
       if (CRD->hasDefinition())
-        DI->EmitAndRetainType(getContext().getRecordType(cast<RecordDecl>(D)));
+        DI->EmitAndRetainType(
+            getContext().getCanonicalTagType(cast<RecordDecl>(D)));
       if (auto *ES = D->getASTContext().getExternalSource())
         if (ES->hasExternalDefinitions(D) == ExternalASTSource::EK_Never)
           DI->completeUnusedClass(*CRD);
@@ -7467,20 +7515,23 @@ void CodeGenModule::EmitTopLevelDecl(Decl *D) {
   case Decl::Typedef:
   case Decl::TypeAlias: // using foo = bar; [C++11]
     if (CGDebugInfo *DI = getModuleDebugInfo())
-      DI->EmitAndRetainType(
-          getContext().getTypedefType(cast<TypedefNameDecl>(D)));
+      DI->EmitAndRetainType(getContext().getTypedefType(
+          ElaboratedTypeKeyword::None, /*Qualifier=*/std::nullopt,
+          cast<TypedefNameDecl>(D)));
     break;
 
   case Decl::Record:
     if (CGDebugInfo *DI = getModuleDebugInfo())
       if (cast<RecordDecl>(D)->getDefinition())
-        DI->EmitAndRetainType(getContext().getRecordType(cast<RecordDecl>(D)));
+        DI->EmitAndRetainType(
+            getContext().getCanonicalTagType(cast<RecordDecl>(D)));
     break;
 
   case Decl::Enum:
     if (CGDebugInfo *DI = getModuleDebugInfo())
       if (cast<EnumDecl>(D)->getDefinition())
-        DI->EmitAndRetainType(getContext().getEnumType(cast<EnumDecl>(D)));
+        DI->EmitAndRetainType(
+            getContext().getCanonicalTagType(cast<EnumDecl>(D)));
     break;
 
   case Decl::HLSLBuffer:
@@ -7880,38 +7931,6 @@ CodeGenModule::CreateMetadataIdentifierForVirtualMemPtrType(QualType T) {
   return CreateMetadataIdentifierImpl(T, VirtualMetadataIdMap, ".virtual");
 }
 
-// Generalize pointer types to a void pointer with the qualifiers of the
-// originally pointed-to type, e.g. 'const char *' and 'char * const *'
-// generalize to 'const void *' while 'char *' and 'const char **' generalize to
-// 'void *'.
-static QualType GeneralizeType(ASTContext &Ctx, QualType Ty) {
-  if (!Ty->isPointerType())
-    return Ty;
-
-  return Ctx.getPointerType(
-      QualType(Ctx.VoidTy).withCVRQualifiers(
-          Ty->getPointeeType().getCVRQualifiers()));
-}
-
-// Apply type generalization to a FunctionType's return and argument types
-static QualType GeneralizeFunctionType(ASTContext &Ctx, QualType Ty) {
-  if (auto *FnType = Ty->getAs<FunctionProtoType>()) {
-    SmallVector<QualType, 8> GeneralizedParams;
-    for (auto &Param : FnType->param_types())
-      GeneralizedParams.push_back(GeneralizeType(Ctx, Param));
-
-    return Ctx.getFunctionType(
-        GeneralizeType(Ctx, FnType->getReturnType()),
-        GeneralizedParams, FnType->getExtProtoInfo());
-  }
-
-  if (auto *FnType = Ty->getAs<FunctionNoProtoType>())
-    return Ctx.getFunctionNoProtoType(
-        GeneralizeType(Ctx, FnType->getReturnType()));
-
-  llvm_unreachable("Encountered unknown FunctionType");
-}
-
 llvm::Metadata *CodeGenModule::CreateMetadataIdentifierGeneralized(QualType T) {
   return CreateMetadataIdentifierImpl(GeneralizeFunctionType(getContext(), T),
                                       GeneralizedMetadataIdMap, ".generalized");
@@ -7934,8 +7953,8 @@ bool CodeGenModule::NeedAllVtablesTypeId() const {
 void CodeGenModule::AddVTableTypeMetadata(llvm::GlobalVariable *VTable,
                                           CharUnits Offset,
                                           const CXXRecordDecl *RD) {
-  llvm::Metadata *MD =
-      CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
+  CanQualType T = getContext().getCanonicalTagType(RD);
+  llvm::Metadata *MD = CreateMetadataIdentifierForType(T);
   VTable->addTypeMetadata(Offset.getQuantity(), MD);
 
   if (CodeGenOpts.SanitizeCfiCrossDso)
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index cb013feb769fc..705d9a3cb9de3 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -1621,7 +1621,7 @@ class CodeGenModule : public CodeGenTypeCache {
   llvm::ConstantInt *CreateCrossDsoCfiTypeId(llvm::Metadata *MD);
 
   /// Generate a KCFI type identifier for T.
-  llvm::ConstantInt *CreateKCFITypeId(QualType T);
+  llvm::ConstantInt *CreateKCFITypeId(QualType T, StringRef Salt);
 
   /// Create a metadata identifier for the given type. This may either be an
   /// MDString (for external identifiers) or a distinct unnamed MDNode (for
diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp
index 90eafe26b548a..bd2442f01cc50 100644
--- a/clang/lib/CodeGen/CodeGenTBAA.cpp
+++ b/clang/lib/CodeGen/CodeGenTBAA.cpp
@@ -143,7 +143,7 @@ static bool TypeHasMayAlias(QualType QTy) {
 /// Check if the given type is a valid base type to be used in access tags.
 static bool isValidBaseType(QualType QTy) {
   if (const RecordType *TTy = QTy->getAs<RecordType>()) {
-    const RecordDecl *RD = TTy->getDecl()->getDefinition();
+    const RecordDecl *RD = TTy->getOriginalDecl()->getDefinition();
     // Incomplete types are not valid base access types.
     if (!RD)
       return false;
@@ -311,7 +311,7 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
       // This also covers anonymous structs and unions, which have a different
       // compatibility rule, but it doesn't matter because you can never have a
       // pointer to an anonymous struct or union.
-      if (!RT->getDecl()->getDeclName())
+      if (!RT->getOriginalDecl()->getDeclName())
         return getAnyPtr(PtrDepth);
 
       // For non-builtin types use the mangled name of the canonical type.
@@ -333,14 +333,15 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) {
   // Enum types are distinct types. In C++ they have "underlying types",
   // however they aren't related for TBAA.
   if (const EnumType *ETy = dyn_cast<EnumType>(Ty)) {
+    const EnumDecl *ED = ETy->getOriginalDecl()->getDefinitionOrSelf();
     if (!Features.CPlusPlus)
-      return getTypeInfo(ETy->getDecl()->getIntegerType());
+      return getTypeInfo(ED->getIntegerType());
 
     // In C++ mode, types have linkage, so we can rely on the ODR and
     // on their mangled names, if they're external.
     // TODO: Is there a way to get a program-wide unique name for a
     // decl with local linkage or no linkage?
-    if (!ETy->getDecl()->isExternallyVisible())
+    if (!ED->isExternallyVisible())
       return getChar();
 
     SmallString<256> OutName;
@@ -433,7 +434,7 @@ CodeGenTBAA::CollectFields(uint64_t BaseOffset,
           llvm::MDBuilder::TBAAStructField(BaseOffset, Size, TBAATag));
       return true;
     }
-    const RecordDecl *RD = TTy->getDecl()->getDefinition();
+    const RecordDecl *RD = TTy->getOriginalDecl()->getDefinition();
     if (RD->hasFlexibleArrayMember())
       return false;
 
@@ -514,7 +515,7 @@ CodeGenTBAA::getTBAAStructInfo(QualType QTy) {
 
 llvm::MDNode *CodeGenTBAA::getBaseTypeInfoHelper(const Type *Ty) {
   if (auto *TTy = dyn_cast<RecordType>(Ty)) {
-    const RecordDecl *RD = TTy->getDecl()->getDefinition();
+    const RecordDecl *RD = TTy->getOriginalDecl()->getDefinition();
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
     using TBAAStructField = llvm::MDBuilder::TBAAStructField;
     SmallVector<TBAAStructField, 4> Fields;
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index c98503e4bbd26..f2a0a649a88fd 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -229,12 +229,13 @@ bool CodeGenTypes::isFuncTypeConvertible(const FunctionType *FT) {
 /// UpdateCompletedType - When we find the full definition for a TagDecl,
 /// replace the 'opaque' type we previously made for it if applicable.
 void CodeGenTypes::UpdateCompletedType(const TagDecl *TD) {
+  CanQualType T = CGM.getContext().getCanonicalTagType(TD);
   // If this is an enum being completed, then we flush all non-struct types from
   // the cache.  This allows function types and other things that may be derived
   // from the enum to be recomputed.
   if (const EnumDecl *ED = dyn_cast<EnumDecl>(TD)) {
     // Only flush the cache if we've actually already converted this type.
-    if (TypeCache.count(ED->getTypeForDecl())) {
+    if (TypeCache.count(T->getTypePtr())) {
       // Okay, we formed some types based on this.  We speculated that the enum
       // would be lowered to i32, so we only need to flush the cache if this
       // didn't happen.
@@ -255,7 +256,7 @@ void CodeGenTypes::UpdateCompletedType(const TagDecl *TD) {
 
   // Only complete it if we converted it already.  If we haven't converted it
   // yet, we'll just do it lazily.
-  if (RecordDeclTypes.count(Context.getTagDeclType(RD).getTypePtr()))
+  if (RecordDeclTypes.count(T.getTypePtr()))
     ConvertRecordDeclType(RD);
 
   // If necessary, provide the full definition of a type only used with a
@@ -265,7 +266,7 @@ void CodeGenTypes::UpdateCompletedType(const TagDecl *TD) {
 }
 
 void CodeGenTypes::RefreshTypeCacheForClass(const CXXRecordDecl *RD) {
-  QualType T = Context.getRecordType(RD);
+  CanQualType T = Context.getCanonicalTagType(RD);
   T = Context.getCanonicalType(T);
 
   const Type *Ty = T.getTypePtr();
@@ -311,11 +312,11 @@ llvm::Type *CodeGenTypes::ConvertFunctionTypeInternal(QualType QFT) {
     // Force conversion of all the relevant record types, to make sure
     // we re-convert the FunctionType when appropriate.
     if (const RecordType *RT = FT->getReturnType()->getAs<RecordType>())
-      ConvertRecordDeclType(RT->getDecl());
+      ConvertRecordDeclType(RT->getOriginalDecl()->getDefinitionOrSelf());
     if (const FunctionProtoType *FPT = dyn_cast<FunctionProtoType>(FT))
       for (unsigned i = 0, e = FPT->getNumParams(); i != e; i++)
         if (const RecordType *RT = FPT->getParamType(i)->getAs<RecordType>())
-          ConvertRecordDeclType(RT->getDecl());
+          ConvertRecordDeclType(RT->getOriginalDecl()->getDefinitionOrSelf());
 
     SkippedLayout = true;
 
@@ -373,7 +374,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
 
   // RecordTypes are cached and processed specially.
   if (const RecordType *RT = dyn_cast<RecordType>(Ty))
-    return ConvertRecordDeclType(RT->getDecl());
+    return ConvertRecordDeclType(RT->getOriginalDecl()->getDefinitionOrSelf());
 
   llvm::Type *CachedType = nullptr;
   auto TCI = TypeCache.find(Ty);
@@ -699,7 +700,8 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
     break;
 
   case Type::Enum: {
-    const EnumDecl *ED = cast<EnumType>(Ty)->getDecl();
+    const EnumDecl *ED =
+        cast<EnumType>(Ty)->getOriginalDecl()->getDefinitionOrSelf();
     if (ED->isCompleteDefinition() || ED->isFixed())
       return ConvertType(ED->getIntegerType());
     // Return a placeholder 'i32' type.  This can be changed later when the
@@ -725,8 +727,10 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
   case Type::MemberPointer: {
     auto *MPTy = cast<MemberPointerType>(Ty);
     if (!getCXXABI().isMemberPointerConvertible(MPTy)) {
-      auto *C = MPTy->getMostRecentCXXRecordDecl()->getTypeForDecl();
-      auto Insertion = RecordsWithOpaqueMemberPointers.try_emplace(C);
+      CanQualType T = CGM.getContext().getCanonicalTagType(
+          MPTy->getMostRecentCXXRecordDecl());
+      auto Insertion =
+          RecordsWithOpaqueMemberPointers.try_emplace(T.getTypePtr());
       if (Insertion.second)
         Insertion.first->second = llvm::StructType::create(getLLVMContext());
       ResultType = Insertion.first->second;
@@ -789,7 +793,7 @@ bool CodeGenModule::isPaddedAtomicType(const AtomicType *type) {
 llvm::StructType *CodeGenTypes::ConvertRecordDeclType(const RecordDecl *RD) {
   // TagDecl's are not necessarily unique, instead use the (clang)
   // type connected to the decl.
-  const Type *Key = Context.getTagDeclType(RD).getTypePtr();
+  const Type *Key = Context.getCanonicalTagType(RD).getTypePtr();
 
   llvm::StructType *&Entry = RecordDeclTypes[Key];
 
@@ -810,7 +814,10 @@ llvm::StructType *CodeGenTypes::ConvertRecordDeclType(const RecordDecl *RD) {
   if (const CXXRecordDecl *CRD = dyn_cast<CXXRecordDecl>(RD)) {
     for (const auto &I : CRD->bases()) {
       if (I.isVirtual()) continue;
-      ConvertRecordDeclType(I.getType()->castAs<RecordType>()->getDecl());
+      ConvertRecordDeclType(I.getType()
+                                ->castAs<RecordType>()
+                                ->getOriginalDecl()
+                                ->getDefinitionOrSelf());
     }
   }
 
@@ -830,7 +837,7 @@ llvm::StructType *CodeGenTypes::ConvertRecordDeclType(const RecordDecl *RD) {
 /// getCGRecordLayout - Return record layout info for the given record decl.
 const CGRecordLayout &
 CodeGenTypes::getCGRecordLayout(const RecordDecl *RD) {
-  const Type *Key = Context.getTagDeclType(RD).getTypePtr();
+  const Type *Key = Context.getCanonicalTagType(RD).getTypePtr();
 
   auto I = CGRecordLayouts.find(Key);
   if (I != CGRecordLayouts.end())
@@ -869,7 +876,7 @@ bool CodeGenTypes::isZeroInitializable(QualType T) {
   // Records are non-zero-initializable if they contain any
   // non-zero-initializable subobjects.
   if (const RecordType *RT = T->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     return isZeroInitializable(RD);
   }
 
diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index 38aaceb6c462d..05fb137ca0575 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -2269,6 +2269,11 @@ struct CounterCoverageMappingBuilder
     // Track LHS True/False Decision.
     const auto DecisionLHS = MCDCBuilder.pop();
 
+    if (auto Gap =
+            findGapAreaBetween(getEnd(E->getLHS()), getStart(E->getRHS()))) {
+      fillGapAreaWithCount(Gap->getBegin(), Gap->getEnd(), getRegionCounter(E));
+    }
+
     // Counter tracks the right hand side of a logical and operator.
     extendRegion(E->getRHS());
     propagateCounts(getRegionCounter(E), E->getRHS());
@@ -2330,6 +2335,11 @@ struct CounterCoverageMappingBuilder
     // Track LHS True/False Decision.
     const auto DecisionLHS = MCDCBuilder.pop();
 
+    if (auto Gap =
+            findGapAreaBetween(getEnd(E->getLHS()), getStart(E->getRHS()))) {
+      fillGapAreaWithCount(Gap->getBegin(), Gap->getEnd(), getRegionCounter(E));
+    }
+
     // Counter tracks the right hand side of a logical or operator.
     extendRegion(E->getRHS());
     propagateCounts(getRegionCounter(E), E->getRHS());
diff --git a/clang/lib/CodeGen/EHScopeStack.h b/clang/lib/CodeGen/EHScopeStack.h
index ed11dc2bb05d7..54f6ceaa52b95 100644
--- a/clang/lib/CodeGen/EHScopeStack.h
+++ b/clang/lib/CodeGen/EHScopeStack.h
@@ -143,7 +143,7 @@ class EHScopeStack {
   ///
   /// Cleanup implementations should generally be declared in an
   /// anonymous namespace.
-  class Cleanup {
+  class alignas(uint64_t) Cleanup {
     // Anchor the construction vtable.
     virtual void anchor();
 
diff --git a/clang/lib/CodeGen/HLSLBufferLayoutBuilder.cpp b/clang/lib/CodeGen/HLSLBufferLayoutBuilder.cpp
index 1ed33894b15aa..ac56dda74abb7 100644
--- a/clang/lib/CodeGen/HLSLBufferLayoutBuilder.cpp
+++ b/clang/lib/CodeGen/HLSLBufferLayoutBuilder.cpp
@@ -101,7 +101,8 @@ llvm::TargetExtType *HLSLBufferLayoutBuilder::createLayoutType(
     const RecordType *RT = RecordTypes.back();
     RecordTypes.pop_back();
 
-    for (const auto *FD : RT->getDecl()->fields()) {
+    for (const auto *FD :
+         RT->getOriginalDecl()->getDefinitionOrSelf()->fields()) {
       assert((!PackOffsets || Index < PackOffsets->size()) &&
              "number of elements in layout struct does not match number of "
              "packoffset annotations");
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index aae1481444067..4ed3775f156c9 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -226,6 +226,10 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
     return hasUniqueVTablePointer(DestRecordTy);
   }
 
+  std::optional<ExactDynamicCastInfo>
+  getExactDynamicCastInfo(QualType SrcRecordTy, QualType DestTy,
+                          QualType DestRecordTy) override;
+
   llvm::Value *emitDynamicCastCall(CodeGenFunction &CGF, Address Value,
                                    QualType SrcRecordTy, QualType DestTy,
                                    QualType DestRecordTy,
@@ -234,6 +238,7 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
   llvm::Value *emitExactDynamicCast(CodeGenFunction &CGF, Address ThisAddr,
                                     QualType SrcRecordTy, QualType DestTy,
                                     QualType DestRecordTy,
+                                    const ExactDynamicCastInfo &CastInfo,
                                     llvm::BasicBlock *CastSuccess,
                                     llvm::BasicBlock *CastFail) override;
 
@@ -826,7 +831,7 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
       for (const CXXRecordDecl *Base : CGM.getMostBaseClasses(RD)) {
         llvm::Metadata *MD = CGM.CreateMetadataIdentifierForType(
             getContext().getMemberPointerType(MPT->getPointeeType(),
-                                              /*Qualifier=*/nullptr,
+                                              /*Qualifier=*/std::nullopt,
                                               Base->getCanonicalDecl()));
         llvm::Value *TypeId =
             llvm::MetadataAsValue::get(CGF.getLLVMContext(), MD);
@@ -1236,7 +1241,7 @@ llvm::Constant *ItaniumCXXABI::EmitMemberPointer(const APValue &MP,
   if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(MPD)) {
     llvm::Constant *Src = BuildMemberPointer(MD, ThisAdjustment);
     QualType SrcType = getContext().getMemberPointerType(
-        MD->getType(), /*Qualifier=*/nullptr, MD->getParent());
+        MD->getType(), /*Qualifier=*/std::nullopt, MD->getParent());
     return pointerAuthResignMemberFunctionPointer(Src, MPType, SrcType, CGM);
   }
 
@@ -1392,8 +1397,9 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
     // to pass to the deallocation function.
 
     // Grab the vtable pointer as an intptr_t*.
-    auto *ClassDecl =
-        cast<CXXRecordDecl>(ElementType->castAs<RecordType>()->getDecl());
+    auto *ClassDecl = cast<CXXRecordDecl>(
+                          ElementType->castAs<RecordType>()->getOriginalDecl())
+                          ->getDefinitionOrSelf();
     llvm::Value *VTable = CGF.GetVTablePtr(Ptr, CGF.UnqualPtrTy, ClassDecl);
 
     // Track back to entry -2 and pull out the offset there.
@@ -1479,7 +1485,8 @@ void ItaniumCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
   // trivial destructor (or isn't a record), we just pass null.
   llvm::Constant *Dtor = nullptr;
   if (const RecordType *RecordTy = ThrowType->getAs<RecordType>()) {
-    CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl());
+    CXXRecordDecl *Record =
+        cast<CXXRecordDecl>(RecordTy->getOriginalDecl())->getDefinitionOrSelf();
     if (!Record->hasTrivialDestructor()) {
       // __cxa_throw is declared to take its destructor as void (*)(void *). We
       // must match that if function pointers can be authenticated with a
@@ -1606,7 +1613,8 @@ llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
                                        Address ThisPtr,
                                        llvm::Type *StdTypeInfoPtrTy) {
   auto *ClassDecl =
-      cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getOriginalDecl())
+          ->getDefinitionOrSelf();
   llvm::Value *Value = CGF.GetVTablePtr(ThisPtr, CGM.GlobalsInt8PtrTy,
                                         ClassDecl);
 
@@ -1681,10 +1689,11 @@ llvm::Value *ItaniumCXXABI::emitDynamicCastCall(
   return Value;
 }
 
-llvm::Value *ItaniumCXXABI::emitExactDynamicCast(
-    CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
-    QualType DestTy, QualType DestRecordTy, llvm::BasicBlock *CastSuccess,
-    llvm::BasicBlock *CastFail) {
+std::optional<CGCXXABI::ExactDynamicCastInfo>
+ItaniumCXXABI::getExactDynamicCastInfo(QualType SrcRecordTy, QualType DestTy,
+                                       QualType DestRecordTy) {
+  assert(shouldEmitExactDynamicCast(DestRecordTy));
+
   ASTContext &Context = getContext();
 
   // Find all the inheritance paths.
@@ -1722,48 +1731,64 @@ llvm::Value *ItaniumCXXABI::emitExactDynamicCast(
     if (!Offset)
       Offset = PathOffset;
     else if (Offset != PathOffset) {
-      // Base appears in at least two different places. Find the most-derived
-      // object and see if it's a DestDecl. Note that the most-derived object
-      // must be at least as aligned as this base class subobject, and must
-      // have a vptr at offset 0.
-      ThisAddr = Address(emitDynamicCastToVoid(CGF, ThisAddr, SrcRecordTy),
-                         CGF.VoidPtrTy, ThisAddr.getAlignment());
-      SrcDecl = DestDecl;
-      Offset = CharUnits::Zero();
-      break;
+      // Base appears in at least two different places.
+      return ExactDynamicCastInfo{/*RequiresCastToPrimaryBase=*/true,
+                                  CharUnits::Zero()};
     }
   }
+  if (!Offset)
+    return std::nullopt;
+  return ExactDynamicCastInfo{/*RequiresCastToPrimaryBase=*/false, *Offset};
+}
 
-  if (!Offset) {
-    // If there are no public inheritance paths, the cast always fails.
-    CGF.EmitBranch(CastFail);
-    return llvm::PoisonValue::get(CGF.VoidPtrTy);
-  }
+llvm::Value *ItaniumCXXABI::emitExactDynamicCast(
+    CodeGenFunction &CGF, Address ThisAddr, QualType SrcRecordTy,
+    QualType DestTy, QualType DestRecordTy,
+    const ExactDynamicCastInfo &ExactCastInfo, llvm::BasicBlock *CastSuccess,
+    llvm::BasicBlock *CastFail) {
+  const CXXRecordDecl *SrcDecl = SrcRecordTy->getAsCXXRecordDecl();
+  const CXXRecordDecl *DestDecl = DestRecordTy->getAsCXXRecordDecl();
+
+  llvm::Value *VTable = nullptr;
+  if (ExactCastInfo.RequiresCastToPrimaryBase) {
+    // Base appears in at least two different places. Find the most-derived
+    // object and see if it's a DestDecl. Note that the most-derived object
+    // must be at least as aligned as this base class subobject, and must
+    // have a vptr at offset 0.
+    llvm::Value *PrimaryBase =
+        emitDynamicCastToVoid(CGF, ThisAddr, SrcRecordTy);
+    ThisAddr = Address(PrimaryBase, CGF.VoidPtrTy, ThisAddr.getAlignment());
+    SrcDecl = DestDecl;
+    Address VTablePtrPtr = ThisAddr.withElementType(CGF.VoidPtrPtrTy);
+    VTable = CGF.Builder.CreateLoad(VTablePtrPtr, "vtable");
+  } else
+    VTable = CGF.GetVTablePtr(ThisAddr, CGF.UnqualPtrTy, SrcDecl);
 
   // Compare the vptr against the expected vptr for the destination type at
-  // this offset. Note that we do not know what type ThisAddr points to in
-  // the case where the derived class multiply inherits from the base class
-  // so we can't use GetVTablePtr, so we load the vptr directly instead.
-  llvm::Instruction *VPtr = CGF.Builder.CreateLoad(
-      ThisAddr.withElementType(CGF.VoidPtrPtrTy), "vtable");
-  CGM.DecorateInstructionWithTBAA(
-      VPtr, CGM.getTBAAVTablePtrAccessInfo(CGF.VoidPtrPtrTy));
-  llvm::Value *Success = CGF.Builder.CreateICmpEQ(
-      VPtr, getVTableAddressPoint(BaseSubobject(SrcDecl, *Offset), DestDecl));
-  llvm::Value *Result = ThisAddr.emitRawPointer(CGF);
-  if (!Offset->isZero())
-    Result = CGF.Builder.CreateInBoundsGEP(
-        CGF.CharTy, Result,
-        {llvm::ConstantInt::get(CGF.PtrDiffTy, -Offset->getQuantity())});
+  // this offset.
+  llvm::Constant *ExpectedVTable = getVTableAddressPoint(
+      BaseSubobject(SrcDecl, ExactCastInfo.Offset), DestDecl);
+  llvm::Value *Success = CGF.Builder.CreateICmpEQ(VTable, ExpectedVTable);
+  llvm::Value *AdjustedThisPtr = ThisAddr.emitRawPointer(CGF);
+
+  if (!ExactCastInfo.Offset.isZero()) {
+    CharUnits::QuantityType Offset = ExactCastInfo.Offset.getQuantity();
+    llvm::Constant *OffsetConstant =
+        llvm::ConstantInt::get(CGF.PtrDiffTy, -Offset);
+    AdjustedThisPtr = CGF.Builder.CreateInBoundsGEP(CGF.CharTy, AdjustedThisPtr,
+                                                    OffsetConstant);
+  }
+
   CGF.Builder.CreateCondBr(Success, CastSuccess, CastFail);
-  return Result;
+  return AdjustedThisPtr;
 }
 
 llvm::Value *ItaniumCXXABI::emitDynamicCastToVoid(CodeGenFunction &CGF,
                                                   Address ThisAddr,
                                                   QualType SrcRecordTy) {
   auto *ClassDecl =
-      cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getOriginalDecl())
+          ->getDefinitionOrSelf();
   llvm::Value *OffsetToTop;
   if (CGM.getItaniumVTableContext().isRelativeLayout()) {
     // Get the vtable pointer.
@@ -2016,7 +2041,7 @@ void ItaniumCXXABI::emitVTableDefinitions(CodeGenVTables &CGVT,
   const VTableLayout &VTLayout = VTContext.getVTableLayout(RD);
   llvm::GlobalVariable::LinkageTypes Linkage = CGM.getVTableLinkage(RD);
   llvm::Constant *RTTI =
-      CGM.GetAddrOfRTTIDescriptor(CGM.getContext().getTagDeclType(RD));
+      CGM.GetAddrOfRTTIDescriptor(CGM.getContext().getCanonicalTagType(RD));
 
   // Create and set the initializer.
   ConstantInitBuilder builder(CGM);
@@ -3755,7 +3780,8 @@ static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM,
   if (!Context.getLangOpts().RTTI) return false;
 
   if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
-    const CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl());
+    const CXXRecordDecl *RD =
+        cast<CXXRecordDecl>(RecordTy->getOriginalDecl())->getDefinitionOrSelf();
     if (!RD->hasDefinition())
       return false;
 
@@ -3789,7 +3815,9 @@ static bool ShouldUseExternalRTTIDescriptor(CodeGenModule &CGM,
 
 /// IsIncompleteClassType - Returns whether the given record type is incomplete.
 static bool IsIncompleteClassType(const RecordType *RecordTy) {
-  return !RecordTy->getDecl()->isCompleteDefinition();
+  return !RecordTy->getOriginalDecl()
+              ->getDefinitionOrSelf()
+              ->isCompleteDefinition();
 }
 
 /// ContainsIncompleteClassType - Returns whether the given type contains an
@@ -3815,9 +3843,7 @@ static bool ContainsIncompleteClassType(QualType Ty) {
   if (const MemberPointerType *MemberPointerTy =
       dyn_cast<MemberPointerType>(Ty)) {
     // Check if the class type is incomplete.
-    const auto *ClassType = cast<RecordType>(
-        MemberPointerTy->getMostRecentCXXRecordDecl()->getTypeForDecl());
-    if (IsIncompleteClassType(ClassType))
+    if (!MemberPointerTy->getMostRecentCXXRecordDecl()->hasDefinition())
       return true;
 
     return ContainsIncompleteClassType(MemberPointerTy->getPointeeType());
@@ -3846,8 +3872,9 @@ static bool CanUseSingleInheritance(const CXXRecordDecl *RD) {
     return false;
 
   // Check that the class is dynamic iff the base is.
-  auto *BaseDecl =
-      cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
+  auto *BaseDecl = cast<CXXRecordDecl>(
+                       Base->getType()->castAs<RecordType>()->getOriginalDecl())
+                       ->getDefinitionOrSelf();
   if (!BaseDecl->isEmpty() &&
       BaseDecl->isDynamicClass() != RD->isDynamicClass())
     return false;
@@ -3926,7 +3953,8 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty,
 
   case Type::Record: {
     const CXXRecordDecl *RD =
-      cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
+        cast<CXXRecordDecl>(cast<RecordType>(Ty)->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     if (!RD->hasDefinition() || !RD->getNumBases()) {
       VTableName = ClassTypeInfo;
@@ -4048,7 +4076,8 @@ static llvm::GlobalVariable::LinkageTypes getTypeInfoLinkage(CodeGenModule &CGM,
       return llvm::GlobalValue::LinkOnceODRLinkage;
 
     if (const RecordType *Record = dyn_cast<RecordType>(Ty)) {
-      const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
+      const CXXRecordDecl *RD =
+          cast<CXXRecordDecl>(Record->getOriginalDecl())->getDefinitionOrSelf();
       if (RD->hasAttr<WeakAttr>())
         return llvm::GlobalValue::WeakODRLinkage;
       if (CGM.getTriple().isWindowsItaniumEnvironment())
@@ -4212,7 +4241,8 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
 
   case Type::Record: {
     const CXXRecordDecl *RD =
-      cast<CXXRecordDecl>(cast<RecordType>(Ty)->getDecl());
+        cast<CXXRecordDecl>(cast<RecordType>(Ty)->getOriginalDecl())
+            ->getDefinitionOrSelf();
     if (!RD->hasDefinition() || !RD->getNumBases()) {
       // We don't need to emit any fields.
       break;
@@ -4259,7 +4289,8 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
   if (CGM.getTarget().hasPS4DLLImportExport() &&
       GVDLLStorageClass != llvm::GlobalVariable::DLLExportStorageClass) {
     if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
-      const CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl());
+      const CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getOriginalDecl())
+                                    ->getDefinitionOrSelf();
       if (RD->hasAttr<DLLExportAttr>() ||
           CXXRecordNonInlineHasAttr<DLLExportAttr>(RD))
         GVDLLStorageClass = llvm::GlobalVariable::DLLExportStorageClass;
@@ -4363,8 +4394,9 @@ static unsigned ComputeVMIClassTypeInfoFlags(const CXXBaseSpecifier *Base,
 
   unsigned Flags = 0;
 
-  auto *BaseDecl =
-      cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
+  auto *BaseDecl = cast<CXXRecordDecl>(
+                       Base->getType()->castAs<RecordType>()->getOriginalDecl())
+                       ->getDefinitionOrSelf();
 
   if (Base->isVirtual()) {
     // Mark the virtual base as seen.
@@ -4464,7 +4496,9 @@ void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
     Fields.push_back(ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Base.getType()));
 
     auto *BaseDecl =
-        cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(
+            Base.getType()->castAs<RecordType>()->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     int64_t OffsetFlags = 0;
 
@@ -4554,9 +4588,8 @@ ItaniumRTTIBuilder::BuildPointerToMemberTypeInfo(const MemberPointerType *Ty) {
   //   attributes of the type pointed to.
   unsigned Flags = extractPBaseFlags(CGM.getContext(), PointeeTy);
 
-  const auto *ClassType =
-      cast<RecordType>(Ty->getMostRecentCXXRecordDecl()->getTypeForDecl());
-  if (IsIncompleteClassType(ClassType))
+  const auto *RD = Ty->getMostRecentCXXRecordDecl();
+  if (!RD->hasDefinition())
     Flags |= PTI_ContainingClassIncomplete;
 
   llvm::Type *UnsignedIntLTy =
@@ -4574,8 +4607,8 @@ ItaniumRTTIBuilder::BuildPointerToMemberTypeInfo(const MemberPointerType *Ty) {
   //   __context is a pointer to an abi::__class_type_info corresponding to the
   //   class type containing the member pointed to
   //   (e.g., the "A" in "int A::*").
-  Fields.push_back(
-      ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(QualType(ClassType, 0)));
+  CanQualType T = CGM.getContext().getCanonicalTagType(RD);
+  Fields.push_back(ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(T));
 }
 
 llvm::Constant *ItaniumCXXABI::getAddrOfRTTIDescriptor(QualType Ty) {
@@ -5155,7 +5188,7 @@ ItaniumCXXABI::getSignedVirtualMemberFunctionPointer(const CXXMethodDecl *MD) {
                               .getDecl());
   llvm::Constant *thunk = getOrCreateVirtualFunctionPointerThunk(origMD);
   QualType funcType = CGM.getContext().getMemberPointerType(
-      MD->getType(), /*Qualifier=*/nullptr, MD->getParent());
+      MD->getType(), /*Qualifier=*/std::nullopt, MD->getParent());
   return CGM.getMemberFunctionPointer(thunk, funcType);
 }
 
diff --git a/clang/lib/CodeGen/MicrosoftCXXABI.cpp b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
index 700ffa4beffc1..88f0648660965 100644
--- a/clang/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/clang/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -158,9 +158,15 @@ class MicrosoftCXXABI : public CGCXXABI {
     // TODO: Add support for exact dynamic_casts.
     return false;
   }
+  std::optional<ExactDynamicCastInfo>
+  getExactDynamicCastInfo(QualType SrcRecordTy, QualType DestTy,
+                          QualType DestRecordTy) override {
+    llvm_unreachable("unsupported");
+  }
   llvm::Value *emitExactDynamicCast(CodeGenFunction &CGF, Address Value,
                                     QualType SrcRecordTy, QualType DestTy,
                                     QualType DestRecordTy,
+                                    const ExactDynamicCastInfo &CastInfo,
                                     llvm::BasicBlock *CastSuccess,
                                     llvm::BasicBlock *CastFail) override {
     llvm_unreachable("unsupported");
@@ -870,7 +876,8 @@ MicrosoftCXXABI::getRecordArgABI(const CXXRecordDecl *RD) const {
     // it indirectly. Prior to MSVC version 19.14, passing overaligned
     // arguments was not supported and resulted in a compiler error. In 19.14
     // and later versions, such arguments are now passed indirectly.
-    TypeInfo Info = getContext().getTypeInfo(RD->getTypeForDecl());
+    TypeInfo Info =
+        getContext().getTypeInfo(getContext().getCanonicalTagType(RD));
     if (Info.isAlignRequired() && Info.Align > 4)
       return RAA_Indirect;
 
@@ -2918,15 +2925,15 @@ llvm::Constant *MicrosoftCXXABI::EmitMemberPointer(const APValue &MP,
     if (!FD)
       FD = cast<FieldDecl>(*cast<IndirectFieldDecl>(MPD)->chain_begin());
     const CXXRecordDecl *RD = cast<CXXRecordDecl>(FD->getParent());
-    RD = RD->getMostRecentNonInjectedDecl();
+    RD = RD->getMostRecentDecl();
     C = EmitMemberDataPointer(RD, FieldOffset);
   }
 
   if (!MemberPointerPath.empty()) {
     const CXXRecordDecl *SrcRD = cast<CXXRecordDecl>(MPD->getDeclContext());
     const MemberPointerType *SrcTy =
-        Ctx.getMemberPointerType(DstTy->getPointeeType(), /*Qualifier=*/nullptr,
-                                 SrcRD)
+        Ctx.getMemberPointerType(DstTy->getPointeeType(),
+                                 /*Qualifier=*/std::nullopt, SrcRD)
             ->castAs<MemberPointerType>();
 
     bool DerivedMember = MP.isMemberPointerToDerivedMember();
@@ -2963,7 +2970,7 @@ MicrosoftCXXABI::EmitMemberFunctionPointer(const CXXMethodDecl *MD) {
   assert(MD->isInstance() && "Member function must not be static!");
 
   CharUnits NonVirtualBaseAdjustment = CharUnits::Zero();
-  const CXXRecordDecl *RD = MD->getParent()->getMostRecentNonInjectedDecl();
+  const CXXRecordDecl *RD = MD->getParent()->getMostRecentDecl();
   CodeGenTypes &Types = CGM.getTypes();
 
   unsigned VBTableIndex = 0;
@@ -3684,7 +3691,7 @@ struct MSRTTIBuilder {
   MSRTTIBuilder(MicrosoftCXXABI &ABI, const CXXRecordDecl *RD)
       : CGM(ABI.CGM), Context(CGM.getContext()),
         VMContext(CGM.getLLVMContext()), Module(CGM.getModule()), RD(RD),
-        Linkage(getLinkageForRTTI(CGM.getContext().getTagDeclType(RD))),
+        Linkage(getLinkageForRTTI(CGM.getContext().getCanonicalTagType(RD))),
         ABI(ABI) {}
 
   llvm::GlobalVariable *getBaseClassDescriptor(const MSRTTIClass &Classes);
@@ -3858,7 +3865,7 @@ MSRTTIBuilder::getBaseClassDescriptor(const MSRTTIClass &Class) {
   // Initialize the BaseClassDescriptor.
   llvm::Constant *Fields[] = {
       ABI.getImageRelativeConstant(
-          ABI.getAddrOfRTTIDescriptor(Context.getTypeDeclType(Class.RD))),
+          ABI.getAddrOfRTTIDescriptor(Context.getCanonicalTagType(Class.RD))),
       llvm::ConstantInt::get(CGM.IntTy, Class.NumBases),
       llvm::ConstantInt::get(CGM.IntTy, Class.OffsetInVBase),
       llvm::ConstantInt::get(CGM.IntTy, VBPtrOffset),
@@ -3905,7 +3912,7 @@ MSRTTIBuilder::getCompleteObjectLocator(const VPtrInfo &Info) {
       llvm::ConstantInt::get(CGM.IntTy, OffsetToTop),
       llvm::ConstantInt::get(CGM.IntTy, VFPtrOffset),
       ABI.getImageRelativeConstant(
-          CGM.GetAddrOfRTTIDescriptor(Context.getTypeDeclType(RD))),
+          CGM.GetAddrOfRTTIDescriptor(Context.getCanonicalTagType(RD))),
       ABI.getImageRelativeConstant(getClassHierarchyDescriptor()),
       ABI.getImageRelativeConstant(COL),
   };
@@ -4076,7 +4083,7 @@ MicrosoftCXXABI::getAddrOfCXXCtorClosure(const CXXConstructorDecl *CD,
   const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeMSCtorClosure(CD, CT);
   llvm::FunctionType *ThunkTy = CGM.getTypes().GetFunctionType(FnInfo);
   const CXXRecordDecl *RD = CD->getParent();
-  QualType RecordTy = getContext().getRecordType(RD);
+  CanQualType RecordTy = getContext().getCanonicalTagType(RD);
   llvm::Function *ThunkFn = llvm::Function::Create(
       ThunkTy, getLinkageForRTTI(RecordTy), ThunkName.str(), &CGM.getModule());
   ThunkFn->setCallingConv(static_cast<llvm::CallingConv::ID>(
@@ -4312,7 +4319,7 @@ llvm::GlobalVariable *MicrosoftCXXABI::getCatchableTypeArray(QualType T) {
 
       // Turn our record back into a pointer if the exception object is a
       // pointer.
-      QualType RTTITy = QualType(Class.RD->getTypeForDecl(), 0);
+      CanQualType RTTITy = Context.getCanonicalTagType(Class.RD);
       if (IsPointer)
         RTTITy = Context.getPointerType(RTTITy);
       CatchableTypes.insert(getCatchableType(RTTITy, Class.OffsetInVBase,
@@ -4463,8 +4470,8 @@ void MicrosoftCXXABI::emitThrow(CodeGenFunction &CGF, const CXXThrowExpr *E) {
 std::pair<llvm::Value *, const CXXRecordDecl *>
 MicrosoftCXXABI::LoadVTablePtr(CodeGenFunction &CGF, Address This,
                                const CXXRecordDecl *RD) {
-  std::tie(This, std::ignore, RD) =
-      performBaseAdjustment(CGF, This, QualType(RD->getTypeForDecl(), 0));
+  CanQualType T = CGF.getContext().getCanonicalTagType(RD);
+  std::tie(This, std::ignore, RD) = performBaseAdjustment(CGF, This, T);
   return {CGF.GetVTablePtr(This, CGM.Int8PtrTy, RD), RD};
 }
 
diff --git a/clang/lib/CodeGen/SwiftCallingConv.cpp b/clang/lib/CodeGen/SwiftCallingConv.cpp
index 10f9f20bca313..de58e0d95a866 100644
--- a/clang/lib/CodeGen/SwiftCallingConv.cpp
+++ b/clang/lib/CodeGen/SwiftCallingConv.cpp
@@ -66,9 +66,9 @@ void SwiftAggLowering::addTypedData(QualType type, CharUnits begin) {
 
   // Record types.
   if (auto recType = type->getAs<RecordType>()) {
-    addTypedData(recType->getDecl(), begin);
+    addTypedData(recType->getOriginalDecl(), begin);
 
-  // Array types.
+    // Array types.
   } else if (type->isArrayType()) {
     // Incomplete array types (flexible array members?) don't provide
     // data to lay out, and the other cases shouldn't be possible.
@@ -814,7 +814,7 @@ static ABIArgInfo classifyType(CodeGenModule &CGM, CanQualType type,
                                bool forReturn) {
   unsigned IndirectAS = CGM.getDataLayout().getAllocaAddrSpace();
   if (auto recordType = dyn_cast<RecordType>(type)) {
-    auto record = recordType->getDecl();
+    auto record = recordType->getOriginalDecl();
     auto &layout = CGM.getContext().getASTRecordLayout(record);
 
     if (mustPassRecordIndirectly(CGM, record))
@@ -822,7 +822,8 @@ static ABIArgInfo classifyType(CodeGenModule &CGM, CanQualType type,
                                      /*AddrSpace=*/IndirectAS, /*byval=*/false);
 
     SwiftAggLowering lowering(CGM);
-    lowering.addTypedData(recordType->getDecl(), CharUnits::Zero(), layout);
+    lowering.addTypedData(recordType->getOriginalDecl(), CharUnits::Zero(),
+                          layout);
     lowering.finish();
 
     return classifyExpandedType(lowering, forReturn, layout.getAlignment(),
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 980f7eb714bbf..60413e7b18e85 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -358,7 +358,7 @@ static Value *emitCallMaybeConstrainedFPBuiltin(CodeGenFunction &CGF,
 
 static llvm::FixedVectorType *GetNeonType(CodeGenFunction *CGF,
                                           NeonTypeFlags TypeFlags,
-                                          bool HasLegalHalfType = true,
+                                          bool HasFastHalfType = true,
                                           bool V1Ty = false,
                                           bool AllowBFloatArgsAndRet = true) {
   int IsQuad = TypeFlags.isQuad();
@@ -376,7 +376,7 @@ static llvm::FixedVectorType *GetNeonType(CodeGenFunction *CGF,
     else
       return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
   case NeonTypeFlags::Float16:
-    if (HasLegalHalfType)
+    if (HasFastHalfType)
       return llvm::FixedVectorType::get(CGF->HalfTy, V1Ty ? 1 : (4 << IsQuad));
     else
       return llvm::FixedVectorType::get(CGF->Int16Ty, V1Ty ? 1 : (4 << IsQuad));
@@ -1754,12 +1754,12 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
   const bool Usgn = Type.isUnsigned();
   const bool Quad = Type.isQuad();
   const bool Floating = Type.isFloatingPoint();
-  const bool HasLegalHalfType = getTarget().hasLegalHalfType();
+  const bool HasFastHalfType = getTarget().hasFastHalfType();
   const bool AllowBFloatArgsAndRet =
       getTargetHooks().getABIInfo().allowBFloatArgsAndRet();
 
   llvm::FixedVectorType *VTy =
-      GetNeonType(this, Type, HasLegalHalfType, false, AllowBFloatArgsAndRet);
+      GetNeonType(this, Type, HasFastHalfType, false, AllowBFloatArgsAndRet);
   llvm::Type *Ty = VTy;
   if (!Ty)
     return nullptr;
@@ -1886,7 +1886,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
   case NEON::BI__builtin_neon_vcvtq_f32_v:
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, Quad),
-                     HasLegalHalfType);
+                     HasFastHalfType);
     return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
   case NEON::BI__builtin_neon_vcvt_f16_s16:
@@ -1895,7 +1895,7 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
   case NEON::BI__builtin_neon_vcvtq_f16_u16:
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float16, false, Quad),
-                     HasLegalHalfType);
+                     HasFastHalfType);
     return Usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
   case NEON::BI__builtin_neon_vcvt_n_f16_s16:
@@ -3211,7 +3211,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   bool rightShift = false;
 
   llvm::FixedVectorType *VTy =
-      GetNeonType(this, Type, getTarget().hasLegalHalfType(), false,
+      GetNeonType(this, Type, getTarget().hasFastHalfType(), false,
                   getTarget().hasBFloat16Type());
   llvm::Type *Ty = VTy;
   if (!Ty)
diff --git a/clang/lib/CodeGen/TargetBuiltins/WebAssembly.cpp b/clang/lib/CodeGen/TargetBuiltins/WebAssembly.cpp
index 33a8d8f8d1754..1a1889a4139d3 100644
--- a/clang/lib/CodeGen/TargetBuiltins/WebAssembly.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/WebAssembly.cpp
@@ -246,35 +246,26 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
     llvm::FunctionType *LLVMFuncTy =
         cast<llvm::FunctionType>(ConvertType(QualType(FuncTy, 0)));
 
+    bool VarArg = LLVMFuncTy->isVarArg();
     unsigned NParams = LLVMFuncTy->getNumParams();
     std::vector<Value *> Args;
-    Args.reserve(NParams + 3);
+    Args.reserve(NParams + 3 + VarArg);
     // The only real argument is the FuncRef
     Args.push_back(FuncRef);
 
     // Add the type information
-    auto addType = [this, &Args](llvm::Type *T) {
-      if (T->isVoidTy()) {
-        // Do nothing
-      } else if (T->isFloatingPointTy()) {
-        Args.push_back(ConstantFP::get(T, 0));
-      } else if (T->isIntegerTy()) {
-        Args.push_back(ConstantInt::get(T, 0));
-      } else if (T->isPointerTy()) {
-        Args.push_back(ConstantPointerNull::get(llvm::PointerType::get(
-            getLLVMContext(), T->getPointerAddressSpace())));
-      } else {
-        // TODO: Handle reference types. For now, we reject them in Sema.
-        llvm_unreachable("Unhandled type");
-      }
-    };
-
-    addType(LLVMFuncTy->getReturnType());
+    llvm::Type *RetType = LLVMFuncTy->getReturnType();
+    if (!RetType->isVoidTy()) {
+      Args.push_back(PoisonValue::get(RetType));
+    }
     // The token type indicates the boundary between return types and param
     // types.
     Args.push_back(PoisonValue::get(llvm::Type::getTokenTy(getLLVMContext())));
     for (unsigned i = 0; i < NParams; i++) {
-      addType(LLVMFuncTy->getParamType(i));
+      Args.push_back(PoisonValue::get(LLVMFuncTy->getParamType(i)));
+    }
+    if (VarArg) {
+      Args.push_back(PoisonValue::get(Builder.getPtrTy()));
     }
     Function *Callee = CGM.getIntrinsic(Intrinsic::wasm_ref_test_func);
     return Builder.CreateCall(Callee, Args);
diff --git a/clang/lib/CodeGen/TargetBuiltins/X86.cpp b/clang/lib/CodeGen/TargetBuiltins/X86.cpp
index e23d19d2f6b6b..b9248a7d43f85 100644
--- a/clang/lib/CodeGen/TargetBuiltins/X86.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/X86.cpp
@@ -1051,18 +1051,9 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
   case X86::BI__builtin_ia32_vfmsubsd3_mask3:
     return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2,
                              /*NegAcc*/ true);
-  case X86::BI__builtin_ia32_vfmaddph:
-  case X86::BI__builtin_ia32_vfmaddps:
-  case X86::BI__builtin_ia32_vfmaddpd:
-  case X86::BI__builtin_ia32_vfmaddph256:
-  case X86::BI__builtin_ia32_vfmaddps256:
-  case X86::BI__builtin_ia32_vfmaddpd256:
   case X86::BI__builtin_ia32_vfmaddph512_mask:
   case X86::BI__builtin_ia32_vfmaddph512_maskz:
   case X86::BI__builtin_ia32_vfmaddph512_mask3:
-  case X86::BI__builtin_ia32_vfmaddbf16128:
-  case X86::BI__builtin_ia32_vfmaddbf16256:
-  case X86::BI__builtin_ia32_vfmaddbf16512:
   case X86::BI__builtin_ia32_vfmaddps512_mask:
   case X86::BI__builtin_ia32_vfmaddps512_maskz:
   case X86::BI__builtin_ia32_vfmaddps512_mask3:
@@ -1941,10 +1932,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     return SI;
   }
   // Rotate is a special case of funnel shift - 1st 2 args are the same.
-  case X86::BI__builtin_ia32_vprotb:
-  case X86::BI__builtin_ia32_vprotw:
-  case X86::BI__builtin_ia32_vprotd:
-  case X86::BI__builtin_ia32_vprotq:
   case X86::BI__builtin_ia32_vprotbi:
   case X86::BI__builtin_ia32_vprotwi:
   case X86::BI__builtin_ia32_vprotdi:
@@ -1955,12 +1942,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
   case X86::BI__builtin_ia32_prolq128:
   case X86::BI__builtin_ia32_prolq256:
   case X86::BI__builtin_ia32_prolq512:
-  case X86::BI__builtin_ia32_prolvd128:
-  case X86::BI__builtin_ia32_prolvd256:
-  case X86::BI__builtin_ia32_prolvd512:
-  case X86::BI__builtin_ia32_prolvq128:
-  case X86::BI__builtin_ia32_prolvq256:
-  case X86::BI__builtin_ia32_prolvq512:
     return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], false);
   case X86::BI__builtin_ia32_prord128:
   case X86::BI__builtin_ia32_prord256:
@@ -1968,12 +1949,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
   case X86::BI__builtin_ia32_prorq128:
   case X86::BI__builtin_ia32_prorq256:
   case X86::BI__builtin_ia32_prorq512:
-  case X86::BI__builtin_ia32_prorvd128:
-  case X86::BI__builtin_ia32_prorvd256:
-  case X86::BI__builtin_ia32_prorvd512:
-  case X86::BI__builtin_ia32_prorvq128:
-  case X86::BI__builtin_ia32_prorvq256:
-  case X86::BI__builtin_ia32_prorvq512:
     return EmitX86FunnelShift(*this, Ops[0], Ops[0], Ops[1], true);
   case X86::BI__builtin_ia32_selectb_128:
   case X86::BI__builtin_ia32_selectb_256:
@@ -2366,29 +2341,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     // Ops 0 and 1 are swapped.
     return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true);
 
-  case X86::BI__builtin_ia32_vpshldvd128:
-  case X86::BI__builtin_ia32_vpshldvd256:
-  case X86::BI__builtin_ia32_vpshldvd512:
-  case X86::BI__builtin_ia32_vpshldvq128:
-  case X86::BI__builtin_ia32_vpshldvq256:
-  case X86::BI__builtin_ia32_vpshldvq512:
-  case X86::BI__builtin_ia32_vpshldvw128:
-  case X86::BI__builtin_ia32_vpshldvw256:
-  case X86::BI__builtin_ia32_vpshldvw512:
-    return EmitX86FunnelShift(*this, Ops[0], Ops[1], Ops[2], false);
-
-  case X86::BI__builtin_ia32_vpshrdvd128:
-  case X86::BI__builtin_ia32_vpshrdvd256:
-  case X86::BI__builtin_ia32_vpshrdvd512:
-  case X86::BI__builtin_ia32_vpshrdvq128:
-  case X86::BI__builtin_ia32_vpshrdvq256:
-  case X86::BI__builtin_ia32_vpshrdvq512:
-  case X86::BI__builtin_ia32_vpshrdvw128:
-  case X86::BI__builtin_ia32_vpshrdvw256:
-  case X86::BI__builtin_ia32_vpshrdvw512:
-    // Ops 0 and 1 are swapped.
-    return EmitX86FunnelShift(*this, Ops[1], Ops[0], Ops[2], true);
-
   // Reductions
   case X86::BI__builtin_ia32_reduce_fadd_pd512:
   case X86::BI__builtin_ia32_reduce_fadd_ps512:
@@ -2850,8 +2802,6 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
     return getCmpIntrinsicCall(Intrinsic::x86_sse2_cmp_sd, 7);
 
   // f16c half2float intrinsics
-  case X86::BI__builtin_ia32_vcvtph2ps:
-  case X86::BI__builtin_ia32_vcvtph2ps256:
   case X86::BI__builtin_ia32_vcvtph2ps_mask:
   case X86::BI__builtin_ia32_vcvtph2ps256_mask:
   case X86::BI__builtin_ia32_vcvtph2ps512_mask: {
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index 277d69daf493c..1e58c3f217812 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -63,6 +63,13 @@ LLVM_DUMP_METHOD void ABIArgInfo::dump() const {
     OS << "CoerceAndExpand Type=";
     getCoerceAndExpandType()->print(OS);
     break;
+  case TargetSpecific:
+    OS << "TargetSpecific Type=";
+    if (llvm::Type *Ty = getCoerceToType())
+      Ty->print(OS);
+    else
+      OS << "null";
+    break;
   }
   OS << ")\n";
 }
diff --git a/clang/lib/CodeGen/Targets/AArch64.cpp b/clang/lib/CodeGen/Targets/AArch64.cpp
index b82c46966cf0b..289f8a9dcf211 100644
--- a/clang/lib/CodeGen/Targets/AArch64.cpp
+++ b/clang/lib/CodeGen/Targets/AArch64.cpp
@@ -375,7 +375,7 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
   if (!passAsAggregateType(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
+      Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     if (const auto *EIT = Ty->getAs<BitIntType>())
       if (EIT->getNumBits() > 128)
@@ -496,7 +496,7 @@ ABIArgInfo AArch64ABIInfo::classifyArgumentType(QualType Ty, bool IsVariadicFn,
       const RecordType *RT = Ty->getAs<RecordType>();
       if (!RT)
         return false;
-      const RecordDecl *RD = RT->getDecl();
+      const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
         for (const auto &I : CXXRD->bases())
           if (!Self(Self, I.getType()))
@@ -548,7 +548,8 @@ ABIArgInfo AArch64ABIInfo::classifyReturnType(QualType RetTy,
   if (!passAsAggregateType(RetTy)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-      RetTy = EnumTy->getDecl()->getIntegerType();
+      RetTy =
+          EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     if (const auto *EIT = RetTy->getAs<BitIntType>())
       if (EIT->getNumBits() > 128)
@@ -744,7 +745,7 @@ bool AArch64ABIInfo::passAsPureScalableType(
       return false;
 
     // Pure scalable types are never unions and never contain unions.
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->isUnion())
       return false;
 
diff --git a/clang/lib/CodeGen/Targets/AMDGPU.cpp b/clang/lib/CodeGen/Targets/AMDGPU.cpp
index 47a552a7bf495..41bccbb5721b2 100644
--- a/clang/lib/CodeGen/Targets/AMDGPU.cpp
+++ b/clang/lib/CodeGen/Targets/AMDGPU.cpp
@@ -96,7 +96,7 @@ unsigned AMDGPUABIInfo::numRegsForType(QualType Ty) const {
   }
 
   if (const RecordType *RT = Ty->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     assert(!RD->hasFlexibleArrayMember());
 
     for (const FieldDecl *Field : RD->fields()) {
@@ -153,7 +153,7 @@ ABIArgInfo AMDGPUABIInfo::classifyReturnType(QualType RetTy) const {
         return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
 
       if (const RecordType *RT = RetTy->getAs<RecordType>()) {
-        const RecordDecl *RD = RT->getDecl();
+        const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
         if (RD->hasFlexibleArrayMember())
           return DefaultABIInfo::classifyReturnType(RetTy);
       }
@@ -246,7 +246,7 @@ ABIArgInfo AMDGPUABIInfo::classifyArgumentType(QualType Ty, bool Variadic,
       return ABIArgInfo::getDirect(CGT.ConvertType(QualType(SeltTy, 0)));
 
     if (const RecordType *RT = Ty->getAs<RecordType>()) {
-      const RecordDecl *RD = RT->getDecl();
+      const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       if (RD->hasFlexibleArrayMember())
         return DefaultABIInfo::classifyArgumentType(Ty);
     }
diff --git a/clang/lib/CodeGen/Targets/ARC.cpp b/clang/lib/CodeGen/Targets/ARC.cpp
index c8db7e8f9706e..ace524e1976d9 100644
--- a/clang/lib/CodeGen/Targets/ARC.cpp
+++ b/clang/lib/CodeGen/Targets/ARC.cpp
@@ -106,13 +106,14 @@ ABIArgInfo ARCABIInfo::classifyArgumentType(QualType Ty,
 
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   auto SizeInRegs = llvm::alignTo(getContext().getTypeSize(Ty), 32) / 32;
 
   if (isAggregateTypeForABI(Ty)) {
     // Structures with flexible arrays are always indirect.
-    if (RT && RT->getDecl()->hasFlexibleArrayMember())
+    if (RT &&
+        RT->getOriginalDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
       return getIndirectByValue(Ty);
 
     // Ignore empty structs/unions.
diff --git a/clang/lib/CodeGen/Targets/ARM.cpp b/clang/lib/CodeGen/Targets/ARM.cpp
index 68f9e01856486..3739e16788c3e 100644
--- a/clang/lib/CodeGen/Targets/ARM.cpp
+++ b/clang/lib/CodeGen/Targets/ARM.cpp
@@ -316,7 +316,7 @@ ABIArgInfo ARMABIInfo::classifyHomogeneousAggregate(QualType Ty,
   // Base can be a floating-point or a vector.
   if (const VectorType *VT = Base->getAs<VectorType>()) {
     // FP16 vectors should be converted to integer vectors
-    if (!getTarget().hasLegalHalfType() && containsAnyFP16Vectors(Ty)) {
+    if (!getTarget().hasFastHalfType() && containsAnyFP16Vectors(Ty)) {
       uint64_t Size = getContext().getTypeSize(VT);
       auto *NewVecTy = llvm::FixedVectorType::get(
           llvm::Type::getInt32Ty(getVMContext()), Size / 32);
@@ -383,7 +383,7 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, bool isVariadic,
   if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>()) {
-      Ty = EnumTy->getDecl()->getIntegerType();
+      Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
     }
 
     if (const auto *EIT = Ty->getAs<BitIntType>())
@@ -516,7 +516,7 @@ static bool isIntegerLikeType(QualType Ty, ASTContext &Context,
   if (!RT) return false;
 
   // Ignore records with flexible arrays.
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
   if (RD->hasFlexibleArrayMember())
     return false;
 
@@ -582,7 +582,7 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic,
                                      getDataLayout().getAllocaAddrSpace());
     // TODO: FP16/BF16 vectors should be converted to integer vectors
     // This check is similar  to isIllegalVectorType - refactor?
-    if ((!getTarget().hasLegalHalfType() &&
+    if ((!getTarget().hasFastHalfType() &&
         (VT->getElementType()->isFloat16Type() ||
          VT->getElementType()->isHalfType())) ||
         (IsFloatABISoftFP &&
@@ -593,7 +593,8 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy, bool isVariadic,
   if (!isAggregateTypeForABI(RetTy)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-      RetTy = EnumTy->getDecl()->getIntegerType();
+      RetTy =
+          EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     if (const auto *EIT = RetTy->getAs<BitIntType>())
       if (EIT->getNumBits() > 64)
@@ -678,9 +679,9 @@ bool ARMABIInfo::isIllegalVectorType(QualType Ty) const {
     // into float, and we don't want the ABI to depend on whether or not they
     // are supported in hardware. Thus return false to coerce vectors of these
     // types into integer vectors.
-    // We do not depend on hasLegalHalfType for bfloat as it is a
+    // We do not depend on hasFastHalfType for bfloat as it is a
     // separate IR type.
-    if ((!getTarget().hasLegalHalfType() &&
+    if ((!getTarget().hasFastHalfType() &&
         (VT->getElementType()->isFloat16Type() ||
          VT->getElementType()->isHalfType())) ||
         (IsFloatABISoftFP &&
@@ -718,7 +719,7 @@ bool ARMABIInfo::containsAnyFP16Vectors(QualType Ty) const {
       return false;
     return containsAnyFP16Vectors(AT->getElementType());
   } else if (const RecordType *RT = Ty->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
 
     // If this is a C++ record, check the bases first.
     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
diff --git a/clang/lib/CodeGen/Targets/BPF.cpp b/clang/lib/CodeGen/Targets/BPF.cpp
index 880a891083c3a..87d50e671d251 100644
--- a/clang/lib/CodeGen/Targets/BPF.cpp
+++ b/clang/lib/CodeGen/Targets/BPF.cpp
@@ -48,7 +48,7 @@ class BPFABIInfo : public DefaultABIInfo {
     }
 
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
+      Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     ASTContext &Context = getContext();
     if (const auto *EIT = Ty->getAs<BitIntType>())
@@ -70,7 +70,8 @@ class BPFABIInfo : public DefaultABIInfo {
 
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-      RetTy = EnumTy->getDecl()->getIntegerType();
+      RetTy =
+          EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     ASTContext &Context = getContext();
     if (const auto *EIT = RetTy->getAs<BitIntType>())
diff --git a/clang/lib/CodeGen/Targets/CSKY.cpp b/clang/lib/CodeGen/Targets/CSKY.cpp
index ef26d483a180a..7e5a16f30727f 100644
--- a/clang/lib/CodeGen/Targets/CSKY.cpp
+++ b/clang/lib/CodeGen/Targets/CSKY.cpp
@@ -116,7 +116,7 @@ ABIArgInfo CSKYABIInfo::classifyArgumentType(QualType Ty, int &ArgGPRsLeft,
   if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
+      Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     // All integral types are promoted to XLen width, unless passed on the
     // stack.
diff --git a/clang/lib/CodeGen/Targets/Hexagon.cpp b/clang/lib/CodeGen/Targets/Hexagon.cpp
index 2976657360650..0c423429eda4f 100644
--- a/clang/lib/CodeGen/Targets/Hexagon.cpp
+++ b/clang/lib/CodeGen/Targets/Hexagon.cpp
@@ -98,7 +98,7 @@ ABIArgInfo HexagonABIInfo::classifyArgumentType(QualType Ty,
   if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
+      Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     uint64_t Size = getContext().getTypeSize(Ty);
     if (Size <= 64)
@@ -161,7 +161,8 @@ ABIArgInfo HexagonABIInfo::classifyReturnType(QualType RetTy) const {
   if (!isAggregateTypeForABI(RetTy)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-      RetTy = EnumTy->getDecl()->getIntegerType();
+      RetTy =
+          EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     if (Size > 64 && RetTy->isBitIntType())
       return getNaturalAlignIndirect(
diff --git a/clang/lib/CodeGen/Targets/Lanai.cpp b/clang/lib/CodeGen/Targets/Lanai.cpp
index 6f75bd54a8ef2..08cb36034f6fd 100644
--- a/clang/lib/CodeGen/Targets/Lanai.cpp
+++ b/clang/lib/CodeGen/Targets/Lanai.cpp
@@ -102,7 +102,8 @@ ABIArgInfo LanaiABIInfo::classifyArgumentType(QualType Ty,
 
   if (isAggregateTypeForABI(Ty)) {
     // Structures with flexible arrays are always indirect.
-    if (RT && RT->getDecl()->hasFlexibleArrayMember())
+    if (RT &&
+        RT->getOriginalDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
       return getIndirectResult(Ty, /*ByVal=*/true, State);
 
     // Ignore empty structs/unions.
@@ -125,7 +126,7 @@ ABIArgInfo LanaiABIInfo::classifyArgumentType(QualType Ty,
 
   // Treat an enum type as its underlying type.
   if (const auto *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   bool InReg = shouldUseInReg(Ty, State);
 
diff --git a/clang/lib/CodeGen/Targets/LoongArch.cpp b/clang/lib/CodeGen/Targets/LoongArch.cpp
index 7640f3779816a..af863e6101e2c 100644
--- a/clang/lib/CodeGen/Targets/LoongArch.cpp
+++ b/clang/lib/CodeGen/Targets/LoongArch.cpp
@@ -150,7 +150,7 @@ bool LoongArchABIInfo::detectFARsEligibleStructHelper(
     // Non-zero-length arrays of empty records make the struct ineligible to be
     // passed via FARs in C++.
     if (const auto *RTy = EltTy->getAs<RecordType>()) {
-      if (ArraySize != 0 && isa<CXXRecordDecl>(RTy->getDecl()) &&
+      if (ArraySize != 0 && isa<CXXRecordDecl>(RTy->getOriginalDecl()) &&
           isEmptyRecord(getContext(), EltTy, true, true))
         return false;
     }
@@ -169,7 +169,7 @@ bool LoongArchABIInfo::detectFARsEligibleStructHelper(
     // copy constructor are not eligible for the FP calling convention.
     if (getRecordArgABI(Ty, CGT.getCXXABI()))
       return false;
-    const RecordDecl *RD = RTy->getDecl();
+    const RecordDecl *RD = RTy->getOriginalDecl()->getDefinitionOrSelf();
     if (isEmptyRecord(getContext(), Ty, true, true) &&
         (!RD->isUnion() || !isa<CXXRecordDecl>(RD)))
       return true;
@@ -181,7 +181,9 @@ bool LoongArchABIInfo::detectFARsEligibleStructHelper(
     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
       for (const CXXBaseSpecifier &B : CXXRD->bases()) {
         const auto *BDecl =
-            cast<CXXRecordDecl>(B.getType()->castAs<RecordType>()->getDecl());
+            cast<CXXRecordDecl>(
+                B.getType()->castAs<RecordType>()->getOriginalDecl())
+                ->getDefinitionOrSelf();
         if (!detectFARsEligibleStructHelper(
                 B.getType(), CurOff + Layout.getBaseClassOffset(BDecl),
                 Field1Ty, Field1Off, Field2Ty, Field2Off))
@@ -369,7 +371,7 @@ ABIArgInfo LoongArchABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
   if (!isAggregateTypeForABI(Ty) && !Ty->isVectorType()) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
+      Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     // All integral types are promoted to GRLen width.
     if (Size < GRLen && Ty->isIntegralOrEnumerationType())
diff --git a/clang/lib/CodeGen/Targets/Mips.cpp b/clang/lib/CodeGen/Targets/Mips.cpp
index c025f7312959c..e12a34ce07bbe 100644
--- a/clang/lib/CodeGen/Targets/Mips.cpp
+++ b/clang/lib/CodeGen/Targets/Mips.cpp
@@ -161,7 +161,7 @@ llvm::Type* MipsABIInfo::HandleAggregates(QualType Ty, uint64_t TySize) const {
     return llvm::StructType::get(getVMContext(), ArgList);
   }
 
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
   assert(!(TySize % 8) && "Size of structure must be multiple of 8.");
 
@@ -242,7 +242,7 @@ MipsABIInfo::classifyArgumentType(QualType Ty, uint64_t &Offset) const {
 
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   // Make sure we pass indirectly things that are too large.
   if (const auto *EIT = Ty->getAs<BitIntType>())
@@ -265,7 +265,7 @@ MipsABIInfo::returnAggregateInRegs(QualType RetTy, uint64_t Size) const {
   SmallVector<llvm::Type*, 8> RTList;
 
   if (RT && RT->isStructureOrClassType()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
     unsigned FieldCnt = Layout.getFieldCount();
 
@@ -333,7 +333,7 @@ ABIArgInfo MipsABIInfo::classifyReturnType(QualType RetTy) const {
 
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-    RetTy = EnumTy->getDecl()->getIntegerType();
+    RetTy = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   // Make sure we pass indirectly things that are too large.
   if (const auto *EIT = RetTy->getAs<BitIntType>())
diff --git a/clang/lib/CodeGen/Targets/NVPTX.cpp b/clang/lib/CodeGen/Targets/NVPTX.cpp
index 82bdfe2666b52..e874617796f86 100644
--- a/clang/lib/CodeGen/Targets/NVPTX.cpp
+++ b/clang/lib/CodeGen/Targets/NVPTX.cpp
@@ -133,7 +133,7 @@ bool NVPTXABIInfo::isUnsupportedType(QualType T) const {
   const auto *RT = T->getAs<RecordType>();
   if (!RT)
     return false;
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
 
   // If this is a C++ record, check the bases first.
   if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
@@ -174,7 +174,7 @@ ABIArgInfo NVPTXABIInfo::classifyReturnType(QualType RetTy) const {
 
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-    RetTy = EnumTy->getDecl()->getIntegerType();
+    RetTy = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   return (isPromotableIntegerTypeForABI(RetTy) ? ABIArgInfo::getExtend(RetTy)
                                                : ABIArgInfo::getDirect());
@@ -183,7 +183,7 @@ ABIArgInfo NVPTXABIInfo::classifyReturnType(QualType RetTy) const {
 ABIArgInfo NVPTXABIInfo::classifyArgumentType(QualType Ty) const {
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   // Return aggregates type as indirect by value
   if (isAggregateTypeForABI(Ty)) {
diff --git a/clang/lib/CodeGen/Targets/PPC.cpp b/clang/lib/CodeGen/Targets/PPC.cpp
index 4df4c9f3c3475..38e76399299ec 100644
--- a/clang/lib/CodeGen/Targets/PPC.cpp
+++ b/clang/lib/CodeGen/Targets/PPC.cpp
@@ -154,7 +154,7 @@ class AIXTargetCodeGenInfo : public TargetCodeGenInfo {
 bool AIXABIInfo::isPromotableTypeForABI(QualType Ty) const {
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   // Promotable integer types are required to be promoted by the ABI.
   if (getContext().isPromotableIntegerType(Ty))
@@ -295,7 +295,9 @@ void AIXTargetCodeGenInfo::setTargetAttributes(
     unsigned Alignment = Context.toBits(Context.getDeclAlign(D)) / 8;
     const auto *Ty = VarD->getType().getTypePtr();
     const RecordDecl *RDecl =
-        Ty->isRecordType() ? Ty->getAs<RecordType>()->getDecl() : nullptr;
+        Ty->isRecordType()
+            ? Ty->getAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf()
+            : nullptr;
 
     bool EmitDiagnostic = UserSpecifiedTOC && GV->hasExternalLinkage();
     auto reportUnsupportedWarning = [&](bool ShouldEmitWarning, StringRef Msg) {
@@ -707,7 +709,7 @@ bool
 PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   // Promotable integer types are required to be promoted by the ABI.
   if (isPromotableIntegerTypeForABI(Ty))
diff --git a/clang/lib/CodeGen/Targets/RISCV.cpp b/clang/lib/CodeGen/Targets/RISCV.cpp
index a7f92981cfbd4..264595a6e4b76 100644
--- a/clang/lib/CodeGen/Targets/RISCV.cpp
+++ b/clang/lib/CodeGen/Targets/RISCV.cpp
@@ -8,6 +8,7 @@
 
 #include "ABIInfoImpl.h"
 #include "TargetInfo.h"
+#include "llvm/IR/IntrinsicsRISCV.h"
 #include "llvm/TargetParser/RISCVTargetParser.h"
 
 using namespace clang;
@@ -73,6 +74,11 @@ class RISCVABIInfo : public DefaultABIInfo {
                                raw_ostream &Out) const override;
   void appendAttributeMangling(StringRef AttrStr,
                                raw_ostream &Out) const override;
+  llvm::Value *createCoercedLoad(Address SrcAddr, const ABIArgInfo &AI,
+                                 CodeGenFunction &CGF) const override;
+  void createCoercedStore(llvm::Value *Val, Address DstAddr,
+                          const ABIArgInfo &AI, bool DestIsVolatile,
+                          CodeGenFunction &CGF) const override;
 };
 } // end anonymous namespace
 
@@ -228,7 +234,7 @@ bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
     // Non-zero-length arrays of empty records make the struct ineligible for
     // the FP calling convention in C++.
     if (const auto *RTy = EltTy->getAs<RecordType>()) {
-      if (ArraySize != 0 && isa<CXXRecordDecl>(RTy->getDecl()) &&
+      if (ArraySize != 0 && isa<CXXRecordDecl>(RTy->getOriginalDecl()) &&
           isEmptyRecord(getContext(), EltTy, true, true))
         return false;
     }
@@ -250,7 +256,7 @@ bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
       return false;
     if (isEmptyRecord(getContext(), Ty, true, true))
       return true;
-    const RecordDecl *RD = RTy->getDecl();
+    const RecordDecl *RD = RTy->getOriginalDecl()->getDefinitionOrSelf();
     // Unions aren't eligible unless they're empty (which is caught above).
     if (RD->isUnion())
       return false;
@@ -259,7 +265,9 @@ bool RISCVABIInfo::detectFPCCEligibleStructHelper(QualType Ty, CharUnits CurOff,
     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
       for (const CXXBaseSpecifier &B : CXXRD->bases()) {
         const auto *BDecl =
-            cast<CXXRecordDecl>(B.getType()->castAs<RecordType>()->getDecl());
+            cast<CXXRecordDecl>(
+                B.getType()->castAs<RecordType>()->getOriginalDecl())
+                ->getDefinitionOrSelf();
         CharUnits BaseOff = Layout.getBaseClassOffset(BDecl);
         bool Ret = detectFPCCEligibleStructHelper(B.getType(), CurOff + BaseOff,
                                                   Field1Ty, Field1Off, Field2Ty,
@@ -648,7 +656,7 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
   if (IsFixed && Ty->isStructureOrClassType()) {
     llvm::Type *VLSType = nullptr;
     if (detectVLSCCEligibleStruct(Ty, ABIVLen, VLSType))
-      return ABIArgInfo::getDirect(VLSType);
+      return ABIArgInfo::getTargetSpecific(VLSType);
   }
 
   uint64_t NeededAlign = getContext().getTypeAlign(Ty);
@@ -673,7 +681,7 @@ ABIArgInfo RISCVABIInfo::classifyArgumentType(QualType Ty, bool IsFixed,
   if (!isAggregateTypeForABI(Ty) && !Ty->isVectorType()) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
+      Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     // All integral types are promoted to XLen width
     if (Size < XLen && Ty->isIntegralOrEnumerationType()) {
@@ -780,6 +788,175 @@ ABIArgInfo RISCVABIInfo::extendType(QualType Ty, llvm::Type *CoerceTy) const {
   return ABIArgInfo::getExtend(Ty, CoerceTy);
 }
 
+llvm::Value *RISCVABIInfo::createCoercedLoad(Address Src, const ABIArgInfo &AI,
+                                             CodeGenFunction &CGF) const {
+  llvm::Type *Ty = AI.getCoerceToType();
+  llvm::Type *SrcTy = Src.getElementType();
+  llvm::StructType *SrcSTy = cast<llvm::StructType>(SrcTy);
+  assert((Ty->isScalableTy() || Ty->isTargetExtTy()) &&
+         "Only scalable vector type and vector tuple type are allowed for load "
+         "type.");
+  if (llvm::TargetExtType *TupTy = dyn_cast<llvm::TargetExtType>(Ty)) {
+    // In RISC-V VLS calling convention, struct of fixed vectors or struct of
+    // array of fixed vector of length >1 might be lowered using vector tuple
+    // type, we consider it as a valid load, e.g.
+    // struct i32x4x2 {
+    //     __attribute__((vector_size(16))) int i;
+    //     __attribute__((vector_size(16))) int i;
+    // };
+    // or
+    // struct i32x4 {
+    //     __attribute__((vector_size(16))) int i[2];
+    // };
+    // is lowered to target("riscv.vector.tuple", <vscale x 8 x i8>, 2)
+    // when ABI_VLEN = 128 bits, please checkout
+    // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
+    // for more information.
+    assert(TupTy->getName() == "riscv.vector.tuple");
+    llvm::Type *EltTy = TupTy->getTypeParameter(0);
+    unsigned NumElts = TupTy->getIntParameter(0);
+
+    if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(SrcSTy->getElementType(0)))
+      Src = Src.withElementType(ArrayTy);
+
+    // Perform extract element and load
+    llvm::Value *TupleVal = llvm::PoisonValue::get(Ty);
+    auto *Load = CGF.Builder.CreateLoad(Src);
+    for (unsigned i = 0; i < NumElts; ++i) {
+      // Extract from struct
+      llvm::Value *ExtractFromLoad = CGF.Builder.CreateExtractValue(Load, i);
+      // Element in vector tuple type is always i8, so we need to cast back to
+      // it's original element type.
+      EltTy =
+          cast<llvm::ScalableVectorType>(llvm::VectorType::getWithSizeAndScalar(
+              cast<llvm::VectorType>(EltTy), ExtractFromLoad->getType()));
+      llvm::Value *VectorVal = llvm::PoisonValue::get(EltTy);
+      // Insert to scalable vector
+      VectorVal = CGF.Builder.CreateInsertVector(
+          EltTy, VectorVal, ExtractFromLoad, uint64_t(0), "cast.scalable");
+      // Insert scalable vector to vector tuple
+      llvm::Value *Idx = CGF.Builder.getInt32(i);
+      TupleVal =
+          CGF.Builder.CreateIntrinsic(llvm::Intrinsic::riscv_tuple_insert,
+                                      {Ty, EltTy}, {TupleVal, VectorVal, Idx});
+    }
+    return TupleVal;
+  }
+
+  // In RISC-V VLS calling convention, struct of fixed vector or struct of
+  // fixed vector array of length 1 might be lowered using scalable vector,
+  // we consider it as a valid load, e.g.
+  // struct i32x4 {
+  //     __attribute__((vector_size(16))) int i;
+  // };
+  // or
+  // struct i32x4 {
+  //     __attribute__((vector_size(16))) int i[1];
+  // };
+  // is lowered to <vscale x 2 x i32>
+  // when ABI_VLEN = 128 bits, please checkout
+  // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
+  // for more information.
+  auto *ScalableDstTy = cast<llvm::ScalableVectorType>(Ty);
+  SrcTy = SrcSTy->getElementType(0);
+  if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(SrcTy))
+    SrcTy = ArrayTy->getElementType();
+  Src = Src.withElementType(SrcTy);
+  [[maybe_unused]] auto *FixedSrcTy = cast<llvm::FixedVectorType>(SrcTy);
+  assert(ScalableDstTy->getElementType() == FixedSrcTy->getElementType());
+  auto *Load = CGF.Builder.CreateLoad(Src);
+  auto *VectorVal = llvm::PoisonValue::get(ScalableDstTy);
+  llvm::Value *Result = CGF.Builder.CreateInsertVector(
+      ScalableDstTy, VectorVal, Load, uint64_t(0), "cast.scalable");
+  return Result;
+}
+
+void RISCVABIInfo::createCoercedStore(llvm::Value *Val, Address Dst,
+                                      const ABIArgInfo &AI, bool DestIsVolatile,
+                                      CodeGenFunction &CGF) const {
+  llvm::Type *SrcTy = Val->getType();
+  llvm::StructType *DstSTy = cast<llvm::StructType>(Dst.getElementType());
+  assert((SrcTy->isScalableTy() || SrcTy->isTargetExtTy()) &&
+         "Only scalable vector type and vector tuple type are allowed for "
+         "store value.");
+  if (llvm::TargetExtType *TupTy = dyn_cast<llvm::TargetExtType>(SrcTy)) {
+    // In RISC-V VLS calling convention, struct of fixed vectors or struct
+    // of array of fixed vector of length >1 might be lowered using vector
+    // tuple type, we consider it as a valid load, e.g.
+    // struct i32x4x2 {
+    //     __attribute__((vector_size(16))) int i;
+    //     __attribute__((vector_size(16))) int i;
+    // };
+    // or
+    // struct i32x4 {
+    //     __attribute__((vector_size(16))) int i[2];
+    // };
+    // is lowered to target("riscv.vector.tuple", <vscale x 8 x i8>, 2)
+    // when ABI_VLEN = 128 bits, please checkout
+    // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
+    // for more information.
+    assert(TupTy->getName() == "riscv.vector.tuple");
+    llvm::Type *EltTy = TupTy->getTypeParameter(0);
+    unsigned NumElts = TupTy->getIntParameter(0);
+
+    llvm::Type *FixedVecTy = DstSTy->getElementType(0);
+    if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(DstSTy->getElementType(0))) {
+      Dst = Dst.withElementType(ArrayTy);
+      FixedVecTy = ArrayTy->getArrayElementType();
+    }
+
+    // Perform extract element and store
+    for (unsigned i = 0; i < NumElts; ++i) {
+      // Element in vector tuple type is always i8, so we need to cast back
+      // to it's original element type.
+      EltTy =
+          cast<llvm::ScalableVectorType>(llvm::VectorType::getWithSizeAndScalar(
+              cast<llvm::VectorType>(EltTy), FixedVecTy));
+      // Extract scalable vector from tuple
+      llvm::Value *Idx = CGF.Builder.getInt32(i);
+      auto *TupleElement = CGF.Builder.CreateIntrinsic(
+          llvm::Intrinsic::riscv_tuple_extract, {EltTy, TupTy}, {Val, Idx});
+
+      // Extract fixed vector from scalable vector
+      auto *ExtractVec = CGF.Builder.CreateExtractVector(
+          FixedVecTy, TupleElement, uint64_t(0));
+      // Store fixed vector to corresponding address
+      Address EltPtr = Address::invalid();
+      if (Dst.getElementType()->isStructTy())
+        EltPtr = CGF.Builder.CreateStructGEP(Dst, i);
+      else
+        EltPtr = CGF.Builder.CreateConstArrayGEP(Dst, i);
+      auto *I = CGF.Builder.CreateStore(ExtractVec, EltPtr, DestIsVolatile);
+      CGF.addInstToCurrentSourceAtom(I, ExtractVec);
+    }
+    return;
+  }
+
+  // In RISC-V VLS calling convention, struct of fixed vector or struct of
+  // fixed vector array of length 1 might be lowered using scalable
+  // vector, we consider it as a valid load, e.g.
+  // struct i32x4 {
+  //     __attribute__((vector_size(16))) int i;
+  // };
+  // or
+  // struct i32x4 {
+  //     __attribute__((vector_size(16))) int i[1];
+  // };
+  // is lowered to <vscale x 2 x i32>
+  // when ABI_VLEN = 128 bits, please checkout
+  // clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
+  // for more information.
+  llvm::Type *EltTy = DstSTy->getElementType(0);
+  if (auto *ArrayTy = dyn_cast<llvm::ArrayType>(EltTy)) {
+    assert(ArrayTy->getNumElements() == 1);
+    EltTy = ArrayTy->getElementType();
+  }
+  auto *Coerced = CGF.Builder.CreateExtractVector(
+      cast<llvm::FixedVectorType>(EltTy), Val, uint64_t(0));
+  auto *I = CGF.Builder.CreateStore(Coerced, Dst, DestIsVolatile);
+  CGF.addInstToCurrentSourceAtom(I, Val);
+}
+
 namespace {
 class RISCVTargetCodeGenInfo : public TargetCodeGenInfo {
 public:
diff --git a/clang/lib/CodeGen/Targets/SPIR.cpp b/clang/lib/CodeGen/Targets/SPIR.cpp
index d952c6e117607..237aea755fa29 100644
--- a/clang/lib/CodeGen/Targets/SPIR.cpp
+++ b/clang/lib/CodeGen/Targets/SPIR.cpp
@@ -119,7 +119,7 @@ ABIArgInfo SPIRVABIInfo::classifyReturnType(QualType RetTy) const {
     return DefaultABIInfo::classifyReturnType(RetTy);
 
   if (const RecordType *RT = RetTy->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->hasFlexibleArrayMember())
       return DefaultABIInfo::classifyReturnType(RetTy);
   }
@@ -187,7 +187,7 @@ ABIArgInfo SPIRVABIInfo::classifyArgumentType(QualType Ty) const {
                                    RAA == CGCXXABI::RAA_DirectInMemory);
 
   if (const RecordType *RT = Ty->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->hasFlexibleArrayMember())
       return DefaultABIInfo::classifyArgumentType(Ty);
   }
@@ -432,7 +432,7 @@ static llvm::Type *getInlineSpirvType(CodeGenModule &CGM,
     case SpirvOperandKind::TypeId: {
       QualType TypeOperand = Operand.getResultType();
       if (auto *RT = TypeOperand->getAs<RecordType>()) {
-        auto *RD = RT->getDecl();
+        auto *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
         assert(RD->isCompleteDefinition() &&
                "Type completion should have been required in Sema");
 
diff --git a/clang/lib/CodeGen/Targets/Sparc.cpp b/clang/lib/CodeGen/Targets/Sparc.cpp
index 9642196b78c63..13a42f66f5843 100644
--- a/clang/lib/CodeGen/Targets/Sparc.cpp
+++ b/clang/lib/CodeGen/Targets/Sparc.cpp
@@ -238,7 +238,7 @@ SparcV9ABIInfo::classifyType(QualType Ty, unsigned SizeLimit) const {
 
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   // Integer types smaller than a register are extended.
   if (Size < 64 && Ty->isIntegerType())
@@ -303,6 +303,7 @@ RValue SparcV9ABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   case ABIArgInfo::Expand:
   case ABIArgInfo::CoerceAndExpand:
   case ABIArgInfo::InAlloca:
+  case ABIArgInfo::TargetSpecific:
     llvm_unreachable("Unsupported ABI kind for va_arg");
 
   case ABIArgInfo::Extend: {
diff --git a/clang/lib/CodeGen/Targets/SystemZ.cpp b/clang/lib/CodeGen/Targets/SystemZ.cpp
index 6ea6c7a546436..38cc4d39126be 100644
--- a/clang/lib/CodeGen/Targets/SystemZ.cpp
+++ b/clang/lib/CodeGen/Targets/SystemZ.cpp
@@ -146,7 +146,7 @@ class SystemZTargetCodeGenInfo : public TargetCodeGenInfo {
 bool SystemZABIInfo::isPromotableIntegerTypeForABI(QualType Ty) const {
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   // Promotable integer types are required to be promoted by the ABI.
   if (ABIInfo::isPromotableIntegerTypeForABI(Ty))
@@ -211,7 +211,7 @@ QualType SystemZABIInfo::GetSingleElementType(QualType Ty) const {
   const RecordType *RT = Ty->getAs<RecordType>();
 
   if (RT && RT->isStructureOrClassType()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     QualType Found;
 
     // If this is a C++ record, check the bases first.
@@ -455,7 +455,7 @@ ABIArgInfo SystemZABIInfo::classifyArgumentType(QualType Ty) const {
   if (const RecordType *RT = Ty->getAs<RecordType>()) {
     // Structures with flexible arrays have variable length, so really
     // fail the size test above.
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->hasFlexibleArrayMember())
       return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),
                                      /*ByVal=*/false);
@@ -526,7 +526,7 @@ bool SystemZTargetCodeGenInfo::isVectorTypeBased(const Type *Ty,
       return true;
 
   if (const auto *RecordTy = Ty->getAs<RecordType>()) {
-    const RecordDecl *RD = RecordTy->getDecl();
+    const RecordDecl *RD = RecordTy->getOriginalDecl()->getDefinitionOrSelf();
     if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD))
       if (CXXRD->hasDefinition())
         for (const auto &I : CXXRD->bases())
diff --git a/clang/lib/CodeGen/Targets/WebAssembly.cpp b/clang/lib/CodeGen/Targets/WebAssembly.cpp
index 9217c78a540a3..ac8dcd2a0540a 100644
--- a/clang/lib/CodeGen/Targets/WebAssembly.cpp
+++ b/clang/lib/CodeGen/Targets/WebAssembly.cpp
@@ -118,7 +118,8 @@ ABIArgInfo WebAssemblyABIInfo::classifyArgumentType(QualType Ty) const {
       const RecordType *RT = Ty->getAs<RecordType>();
       assert(RT);
       bool HasBitField = false;
-      for (auto *Field : RT->getDecl()->fields()) {
+      for (auto *Field :
+           RT->getOriginalDecl()->getDefinitionOrSelf()->fields()) {
         if (Field->isBitField()) {
           HasBitField = true;
           break;
diff --git a/clang/lib/CodeGen/Targets/X86.cpp b/clang/lib/CodeGen/Targets/X86.cpp
index abb91486e7ee6..61b833a9c1b43 100644
--- a/clang/lib/CodeGen/Targets/X86.cpp
+++ b/clang/lib/CodeGen/Targets/X86.cpp
@@ -359,7 +359,8 @@ bool X86_32ABIInfo::shouldReturnTypeInRegister(QualType Ty,
 
   // Structure types are passed in register if all fields would be
   // passed in a register.
-  for (const auto *FD : RT->getDecl()->fields()) {
+  for (const auto *FD :
+       RT->getOriginalDecl()->getDefinitionOrSelf()->fields()) {
     // Empty fields are ignored.
     if (isEmptyField(Context, FD, true))
       continue;
@@ -429,9 +430,9 @@ bool X86_32ABIInfo::canExpandIndirectArgument(QualType Ty) const {
   const RecordType *RT = Ty->getAs<RecordType>();
   if (!RT)
     return false;
-  const RecordDecl *RD = RT->getDecl();
   uint64_t Size = 0;
-  if (const CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
+  if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
     if (!IsWin32StructABI) {
       // On non-Windows, we have to conservatively match our old bitcode
       // prototypes in order to be ABI-compatible at the bitcode level.
@@ -509,7 +510,9 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
   if (isAggregateTypeForABI(RetTy)) {
     if (const RecordType *RT = RetTy->getAs<RecordType>()) {
       // Structures with flexible arrays are always indirect.
-      if (RT->getDecl()->hasFlexibleArrayMember())
+      if (RT->getOriginalDecl()
+              ->getDefinitionOrSelf()
+              ->hasFlexibleArrayMember())
         return getIndirectReturnResult(RetTy, State);
     }
 
@@ -554,7 +557,7 @@ ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
 
   // Treat an enum type as its underlying type.
   if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-    RetTy = EnumTy->getDecl()->getIntegerType();
+    RetTy = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   if (const auto *EIT = RetTy->getAs<BitIntType>())
     if (EIT->getNumBits() > 64)
@@ -796,7 +799,8 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,
   if (isAggregateTypeForABI(Ty)) {
     // Structures with flexible arrays are always indirect.
     // FIXME: This should not be byval!
-    if (RT && RT->getDecl()->hasFlexibleArrayMember())
+    if (RT &&
+        RT->getOriginalDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
       return getIndirectResult(Ty, true, State);
 
     // Ignore empty structs/unions on non-Windows.
@@ -831,7 +835,7 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,
       unsigned AlignInBits = 0;
       if (RT) {
         const ASTRecordLayout &Layout =
-          getContext().getASTRecordLayout(RT->getDecl());
+            getContext().getASTRecordLayout(RT->getOriginalDecl());
         AlignInBits = getContext().toBits(Layout.getRequiredAlignment());
       } else if (TI.isAlignRequired()) {
         AlignInBits = TI.Align;
@@ -883,7 +887,7 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty, CCState &State,
 
 
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   bool InReg = shouldPrimitiveUseInReg(Ty, State);
 
@@ -1008,6 +1012,7 @@ static bool isArgInAlloca(const ABIArgInfo &Info) {
     return true;
   case ABIArgInfo::Ignore:
   case ABIArgInfo::IndirectAliased:
+  case ABIArgInfo::TargetSpecific:
     return false;
   case ABIArgInfo::Indirect:
   case ABIArgInfo::Direct:
@@ -1847,7 +1852,8 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
 
   if (const EnumType *ET = Ty->getAs<EnumType>()) {
     // Classify the underlying integer type.
-    classify(ET->getDecl()->getIntegerType(), OffsetBase, Lo, Hi, isNamedArg);
+    classify(ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType(),
+             OffsetBase, Lo, Hi, isNamedArg);
     return;
   }
 
@@ -2053,7 +2059,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
     if (getRecordArgABI(RT, getCXXABI()))
       return;
 
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
 
     // Assume variable sized types are passed in memory.
     if (RD->hasFlexibleArrayMember())
@@ -2070,7 +2076,9 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase, Class &Lo,
         assert(!I.isVirtual() && !I.getType()->isDependentType() &&
                "Unexpected base class!");
         const auto *Base =
-            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+            cast<CXXRecordDecl>(
+                I.getType()->castAs<RecordType>()->getOriginalDecl())
+                ->getDefinitionOrSelf();
 
         // Classify this field.
         //
@@ -2184,7 +2192,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectReturnResult(QualType Ty) const {
   if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
+      Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     if (Ty->isBitIntType())
       return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace());
@@ -2226,7 +2234,7 @@ ABIArgInfo X86_64ABIInfo::getIndirectResult(QualType Ty,
       !Ty->isBitIntType()) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-      Ty = EnumTy->getDecl()->getIntegerType();
+      Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
     return (isPromotableIntegerTypeForABI(Ty) ? ABIArgInfo::getExtend(Ty)
                                               : ABIArgInfo::getDirect());
@@ -2347,7 +2355,7 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
   }
 
   if (const RecordType *RT = Ty->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
 
     // If this is a C++ record, check the bases first.
@@ -2356,7 +2364,9 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
         assert(!I.isVirtual() && !I.getType()->isDependentType() &&
                "Unexpected base class!");
         const auto *Base =
-            cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+            cast<CXXRecordDecl>(
+                I.getType()->castAs<RecordType>()->getOriginalDecl())
+                ->getDefinitionOrSelf();
 
         // If the base is after the span we care about, ignore it.
         unsigned BaseOffset = Context.toBits(Layout.getBaseClassOffset(Base));
@@ -2637,7 +2647,8 @@ ABIArgInfo X86_64ABIInfo::classifyReturnType(QualType RetTy) const {
     if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
       // Treat an enum type as its underlying type.
       if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
-        RetTy = EnumTy->getDecl()->getIntegerType();
+        RetTy =
+            EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
       if (RetTy->isIntegralOrEnumerationType() &&
           isPromotableIntegerTypeForABI(RetTy))
@@ -2787,7 +2798,7 @@ X86_64ABIInfo::classifyArgumentType(QualType Ty, unsigned freeIntRegs,
     if (Hi == NoClass && isa<llvm::IntegerType>(ResType)) {
       // Treat an enum type as its underlying type.
       if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-        Ty = EnumTy->getDecl()->getIntegerType();
+        Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
       if (Ty->isIntegralOrEnumerationType() &&
           isPromotableIntegerTypeForABI(Ty))
@@ -2866,14 +2877,15 @@ ABIArgInfo
 X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
                                              unsigned &NeededSSE,
                                              unsigned &MaxVectorWidth) const {
-  auto RT = Ty->getAs<RecordType>();
-  assert(RT && "classifyRegCallStructType only valid with struct types");
+  auto *RD = cast<RecordType>(Ty.getCanonicalType())
+                 ->getOriginalDecl()
+                 ->getDefinitionOrSelf();
 
-  if (RT->getDecl()->hasFlexibleArrayMember())
+  if (RD->hasFlexibleArrayMember())
     return getIndirectReturnResult(Ty);
 
   // Sum up bases
-  if (auto CXXRD = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
+  if (auto CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
     if (CXXRD->isDynamicClass()) {
       NeededInt = NeededSSE = 0;
       return getIndirectReturnResult(Ty);
@@ -2889,7 +2901,7 @@ X86_64ABIInfo::classifyRegCallStructTypeImpl(QualType Ty, unsigned &NeededInt,
   }
 
   // Sum up members
-  for (const auto *FD : RT->getDecl()->fields()) {
+  for (const auto *FD : RD->fields()) {
     QualType MTy = FD->getType();
     if (MTy->isRecordType() && !MTy->isUnionType()) {
       if (classifyRegCallStructTypeImpl(MTy, NeededInt, NeededSSE,
@@ -3313,7 +3325,7 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
     return ABIArgInfo::getIgnore();
 
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
-    Ty = EnumTy->getDecl()->getIntegerType();
+    Ty = EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   TypeInfo Info = getContext().getTypeInfo(Ty);
   uint64_t Width = Info.Width;
@@ -3327,7 +3339,7 @@ ABIArgInfo WinX86_64ABIInfo::classify(QualType Ty, unsigned &FreeSSERegs,
                                        RAA == CGCXXABI::RAA_DirectInMemory);
     }
 
-    if (RT->getDecl()->hasFlexibleArrayMember())
+    if (RT->getOriginalDecl()->getDefinitionOrSelf()->hasFlexibleArrayMember())
       return getNaturalAlignIndirect(Ty, getDataLayout().getAllocaAddrSpace(),
                                      /*ByVal=*/false);
   }
diff --git a/clang/lib/CodeGen/Targets/XCore.cpp b/clang/lib/CodeGen/Targets/XCore.cpp
index b7824bde5f55a..d65af5f82b7f8 100644
--- a/clang/lib/CodeGen/Targets/XCore.cpp
+++ b/clang/lib/CodeGen/Targets/XCore.cpp
@@ -157,6 +157,7 @@ RValue XCoreABIInfo::EmitVAArg(CodeGenFunction &CGF, Address VAListAddr,
   case ABIArgInfo::Expand:
   case ABIArgInfo::CoerceAndExpand:
   case ABIArgInfo::InAlloca:
+  case ABIArgInfo::TargetSpecific:
     llvm_unreachable("Unsupported ABI kind for va_arg");
   case ABIArgInfo::Ignore:
     Val = Address(llvm::UndefValue::get(ArgPtrTy), ArgTy, TypeAlign);
@@ -379,7 +380,7 @@ static bool appendRecordType(SmallStringEnc &Enc, const RecordType *RT,
 
   // We collect all encoded fields and order as necessary.
   bool IsRecursive = false;
-  const RecordDecl *RD = RT->getDecl()->getDefinition();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinition();
   if (RD && !RD->field_empty()) {
     // An incomplete TypeString stub is placed in the cache for this RecordType
     // so that recursive calls to this RecordType will use it whilst building a
@@ -428,7 +429,7 @@ static bool appendEnumType(SmallStringEnc &Enc, const EnumType *ET,
   Enc += "){";
 
   // We collect all encoded enumerations and order them alphanumerically.
-  if (const EnumDecl *ED = ET->getDecl()->getDefinition()) {
+  if (const EnumDecl *ED = ET->getOriginalDecl()->getDefinition()) {
     SmallVector<FieldEncoding, 16> FE;
     for (auto I = ED->enumerator_begin(), E = ED->enumerator_end(); I != E;
          ++I) {
diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt
index 45782cbd9d16d..7c4f70b966c48 100644
--- a/clang/lib/Driver/CMakeLists.txt
+++ b/clang/lib/Driver/CMakeLists.txt
@@ -98,5 +98,6 @@ add_clang_library(clangDriver
 
   LINK_LIBS
   clangBasic
+  clangLex
   ${system_libs}
   )
diff --git a/clang/lib/Driver/Compilation.cpp b/clang/lib/Driver/Compilation.cpp
index a39952e3c280a..4e300316ae9ba 100644
--- a/clang/lib/Driver/Compilation.cpp
+++ b/clang/lib/Driver/Compilation.cpp
@@ -232,11 +232,6 @@ static bool ActionFailed(const Action *A,
   return false;
 }
 
-static bool InputsOk(const Command &C,
-                     const FailingCommandList &FailingCommands) {
-  return !ActionFailed(&C.getSource(), FailingCommands);
-}
-
 void Compilation::ExecuteJobs(const JobList &Jobs,
                               FailingCommandList &FailingCommands,
                               bool LogOnly) const {
@@ -245,7 +240,7 @@ void Compilation::ExecuteJobs(const JobList &Jobs,
   // In all but CLMode, execute all the jobs unless the necessary inputs for the
   // job is missing due to previous failures.
   for (const auto &Job : Jobs) {
-    if (!InputsOk(Job, FailingCommands))
+    if (ActionFailed(&Job.getSource(), FailingCommands))
       continue;
     const Command *FailingCommand = nullptr;
     if (int Res = ExecuteCommand(Job, FailingCommand, LogOnly)) {
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 8c0bba938a09b..34eaad43d6f79 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -66,6 +66,7 @@
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
 #include "clang/Driver/Types.h"
+#include "clang/Lex/DependencyDirectivesScanner.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallSet.h"
@@ -4188,6 +4189,11 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
     YcArg = nullptr;
   }
 
+  if (Args.hasArgNoClaim(options::OPT_fmodules_driver))
+    // TODO: Check against all incompatible -fmodules-driver arguments
+    if (!ModulesModeCXX20 && !Args.hasArgNoClaim(options::OPT_fmodules))
+      Args.eraseArg(options::OPT_fmodules_driver);
+
   Arg *FinalPhaseArg;
   phases::ID FinalPhase = getFinalPhase(Args, &FinalPhaseArg);
 
@@ -4314,6 +4320,33 @@ void Driver::handleArguments(Compilation &C, DerivedArgList &Args,
   }
 }
 
+static bool hasCXXModuleInputType(const Driver::InputList &Inputs) {
+  const auto IsTypeCXXModule = [](const auto &Input) -> bool {
+    const auto TypeID = Input.first;
+    return (TypeID == types::TY_CXXModule);
+  };
+  return llvm::any_of(Inputs, IsTypeCXXModule);
+}
+
+llvm::ErrorOr<bool>
+Driver::ScanInputsForCXX20ModulesUsage(const InputList &Inputs) const {
+  const auto CXXInputs = llvm::make_filter_range(
+      Inputs, [](const auto &Input) { return types::isCXX(Input.first); });
+  for (const auto &Input : CXXInputs) {
+    StringRef Filename = Input.second->getSpelling();
+    auto ErrOrBuffer = VFS->getBufferForFile(Filename);
+    if (!ErrOrBuffer)
+      return ErrOrBuffer.getError();
+    const auto Buffer = std::move(*ErrOrBuffer);
+
+    if (scanInputForCXX20ModulesUsage(Buffer->getBuffer())) {
+      Diags.Report(diag::remark_found_cxx20_module_usage) << Filename;
+      return true;
+    }
+  }
+  return false;
+}
+
 void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
                           const InputList &Inputs, ActionList &Actions) const {
   llvm::PrettyStackTraceString CrashInfo("Building compilation actions");
@@ -4325,6 +4358,33 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
 
   handleArguments(C, Args, Inputs, Actions);
 
+  if (Args.hasFlag(options::OPT_fmodules_driver,
+                   options::OPT_fno_modules_driver, false)) {
+    // TODO: Move the logic for implicitly enabling explicit-module-builds out
+    // of -fmodules-driver once it is no longer experimental.
+    // Currently, this serves diagnostic purposes only.
+    bool UsesCXXModules = hasCXXModuleInputType(Inputs);
+    if (!UsesCXXModules) {
+      const auto ErrOrScanResult = ScanInputsForCXX20ModulesUsage(Inputs);
+      if (!ErrOrScanResult) {
+        Diags.Report(diag::err_cannot_open_file)
+            << ErrOrScanResult.getError().message();
+        return;
+      }
+      UsesCXXModules = *ErrOrScanResult;
+    }
+    if (UsesCXXModules || Args.hasArg(options::OPT_fmodules))
+      BuildDriverManagedModuleBuildActions(C, Args, Inputs, Actions);
+    return;
+  }
+
+  BuildDefaultActions(C, Args, Inputs, Actions);
+}
+
+void Driver::BuildDefaultActions(Compilation &C, DerivedArgList &Args,
+                                 const InputList &Inputs,
+                                 ActionList &Actions) const {
+
   bool UseNewOffloadingDriver =
       C.isOffloadingHostKind(Action::OFK_OpenMP) ||
       C.isOffloadingHostKind(Action::OFK_SYCL) ||
@@ -4608,6 +4668,12 @@ void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
   Args.ClaimAllArgs(options::OPT_cl_ignored_Group);
 }
 
+void Driver::BuildDriverManagedModuleBuildActions(
+    Compilation &C, llvm::opt::DerivedArgList &Args, const InputList &Inputs,
+    ActionList &Actions) const {
+  Diags.Report(diag::remark_performing_driver_managed_module_build);
+}
+
 /// Returns the canonical name for the offloading architecture when using a HIP
 /// or CUDA architecture.
 static StringRef getCanonicalArchString(Compilation &C,
diff --git a/clang/lib/Driver/SanitizerArgs.cpp b/clang/lib/Driver/SanitizerArgs.cpp
index 98793a5bb9979..54f0e63b98070 100644
--- a/clang/lib/Driver/SanitizerArgs.cpp
+++ b/clang/lib/Driver/SanitizerArgs.cpp
@@ -851,6 +851,8 @@ SanitizerArgs::SanitizerArgs(const ToolChain &TC,
   }
 
   if (AllAddedKinds & SanitizerKind::KCFI) {
+    CfiICallGeneralizePointers =
+        Args.hasArg(options::OPT_fsanitize_cfi_icall_generalize_pointers);
     CfiICallNormalizeIntegers =
         Args.hasArg(options::OPT_fsanitize_cfi_icall_normalize_integers);
 
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 25c6b5a486fd5..65b36217a940f 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -855,17 +855,30 @@ void ToolChain::addFortranRuntimeLibs(const ArgList &Args,
 
 void ToolChain::addFortranRuntimeLibraryPath(const llvm::opt::ArgList &Args,
                                              ArgStringList &CmdArgs) const {
-  // Default to the <driver-path>/../lib directory. This works fine on the
-  // platforms that we have tested so far. We will probably have to re-fine
-  // this in the future. In particular, on some platforms, we may need to use
-  // lib64 instead of lib.
+  auto AddLibSearchPathIfExists = [&](const Twine &Path) {
+    // Linker may emit warnings about non-existing directories
+    if (!llvm::sys::fs::is_directory(Path))
+      return;
+
+    if (getTriple().isKnownWindowsMSVCEnvironment())
+      CmdArgs.push_back(Args.MakeArgString("-libpath:" + Path));
+    else
+      CmdArgs.push_back(Args.MakeArgString("-L" + Path));
+  };
+
+  // Search for flang_rt.* at the same location as clang_rt.* with
+  // LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=0. On most platforms, flang_rt is
+  // located at the path returned by getRuntimePath() which is already added to
+  // the library search path. This exception is for Apple-Darwin.
+  AddLibSearchPathIfExists(getCompilerRTPath());
+
+  // Fall back to the non-resource directory <driver-path>/../lib. We will
+  // probably have to refine this in the future. In particular, on some
+  // platforms, we may need to use lib64 instead of lib.
   SmallString<256> DefaultLibPath =
       llvm::sys::path::parent_path(getDriver().Dir);
   llvm::sys::path::append(DefaultLibPath, "lib");
-  if (getTriple().isKnownWindowsMSVCEnvironment())
-    CmdArgs.push_back(Args.MakeArgString("-libpath:" + DefaultLibPath));
-  else
-    CmdArgs.push_back(Args.MakeArgString("-L" + DefaultLibPath));
+  AddLibSearchPathIfExists(DefaultLibPath);
 }
 
 void ToolChain::addFlangRTLibPath(const ArgList &Args,
@@ -1396,13 +1409,6 @@ void ToolChain::addSystemFrameworkInclude(const llvm::opt::ArgList &DriverArgs,
   CC1Args.push_back(DriverArgs.MakeArgString(Path));
 }
 
-/// Utility function to add a system include directory to CC1 arguments.
-void ToolChain::addSystemInclude(const ArgList &DriverArgs,
-                                 ArgStringList &CC1Args, const Twine &Path) {
-  CC1Args.push_back("-internal-isystem");
-  CC1Args.push_back(DriverArgs.MakeArgString(Path));
-}
-
 /// Utility function to add a system include directory with extern "C"
 /// semantics to CC1 arguments.
 ///
@@ -1425,6 +1431,14 @@ void ToolChain::addExternCSystemIncludeIfExists(const ArgList &DriverArgs,
     addExternCSystemInclude(DriverArgs, CC1Args, Path);
 }
 
+/// Utility function to add a system include directory to CC1 arguments.
+/*static*/ void ToolChain::addSystemInclude(const ArgList &DriverArgs,
+                                            ArgStringList &CC1Args,
+                                            const Twine &Path) {
+  CC1Args.push_back("-internal-isystem");
+  CC1Args.push_back(DriverArgs.MakeArgString(Path));
+}
+
 /// Utility function to add a list of system framework directories to CC1.
 void ToolChain::addSystemFrameworkIncludes(const ArgList &DriverArgs,
                                            ArgStringList &CC1Args,
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index 418f9fd9ca4c4..98f5efbe5652f 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -52,6 +52,22 @@ std::string aarch64::getAArch64TargetCPU(const ArgList &Args,
     return "apple-m1";
   }
 
+  if (Triple.getOS() == llvm::Triple::IOS) {
+    assert(!Triple.isSimulatorEnvironment() && "iossim should be mac-like");
+    // iOS 26 only runs on apple-a12 and later CPUs.
+    if (!Triple.isOSVersionLT(26))
+      return "apple-a12";
+  }
+
+  if (Triple.isWatchOS()) {
+    assert(!Triple.isSimulatorEnvironment() && "watchossim should be mac-like");
+    // arm64_32/arm64e watchOS requires S4 before watchOS 26, S6 after.
+    if (Triple.getArch() == llvm::Triple::aarch64_32 || Triple.isArm64e())
+      return Triple.isOSVersionLT(26) ? "apple-s4" : "apple-s6";
+    // arm64 (non-e, non-32) watchOS comes later, and requires S6 anyway.
+    return "apple-s6";
+  }
+
   if (Triple.isXROS()) {
     // The xrOS simulator runs on M1 as well, it should have been covered above.
     assert(!Triple.isSimulatorEnvironment() && "xrossim should be mac-like");
diff --git a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
index baa2c8c0bcfb2..76dde0da8e849 100644
--- a/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/RISCV.cpp
@@ -273,9 +273,12 @@ std::string riscv::getRISCVArch(const llvm::opt::ArgList &Args,
   // Clang does not yet support MULTILIB_REUSE, so we use `rv{XLEN}imafdc`
   // instead of `rv{XLEN}gc` though they are (currently) equivalent.
 
-  // 1. If `-march=` is specified, use it.
-  if (const Arg *A = Args.getLastArg(options::OPT_march_EQ))
-    return A->getValue();
+  // 1. If `-march=` is specified, use it unless the value is "unset".
+  if (const Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
+    StringRef MArch = A->getValue();
+    if (MArch != "unset")
+      return MArch.str();
+  }
 
   // 2. Get march (isa string) based on `-mcpu=`
   if (const Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) {
@@ -300,7 +303,7 @@ std::string riscv::getRISCVArch(const llvm::opt::ArgList &Args,
 
     StringRef MArch = llvm::RISCV::getMArchFromMcpu(CPU);
     // Bypass if target cpu's default march is empty.
-    if (MArch != "")
+    if (!MArch.empty())
       return MArch.str();
   }
 
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 25a16fe16eead..9b7f58c392885 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -586,11 +586,18 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
   const Driver &D = getToolChain().getDriver();
   const llvm::Triple::ArchType Arch = TC.getArch();
   const llvm::Triple &Triple = getToolChain().getEffectiveTriple();
+  const bool IsStaticPIE = getStaticPIE(Args, TC);
 
   if (!D.SysRoot.empty())
     CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
 
   CmdArgs.push_back("-Bstatic");
+  if (IsStaticPIE) {
+    CmdArgs.push_back("-pie");
+    CmdArgs.push_back("--no-dynamic-linker");
+    CmdArgs.push_back("-z");
+    CmdArgs.push_back("text");
+  }
 
   if (const char *LDMOption = getLDMOption(TC.getTriple(), Args)) {
     CmdArgs.push_back("-m");
@@ -620,14 +627,18 @@ void baremetal::Linker::ConstructJob(Compilation &C, const JobAction &JA,
 
   const char *CRTBegin, *CRTEnd;
   if (NeedCRTs) {
-    if (!Args.hasArg(options::OPT_r))
-      CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath("crt0.o")));
+    if (!Args.hasArg(options::OPT_r)) {
+      const char *crt = "crt0.o";
+      if (IsStaticPIE)
+        crt = "rcrt1.o";
+      CmdArgs.push_back(Args.MakeArgString(TC.GetFilePath(crt)));
+    }
     if (TC.hasValidGCCInstallation() || detectGCCToolchainAdjacent(D)) {
       auto RuntimeLib = TC.GetRuntimeLibType(Args);
       switch (RuntimeLib) {
       case (ToolChain::RLT_Libgcc): {
-        CRTBegin = "crtbegin.o";
-        CRTEnd = "crtend.o";
+        CRTBegin = IsStaticPIE ? "crtbeginS.o" : "crtbegin.o";
+        CRTEnd = IsStaticPIE ? "crtendS.o" : "crtend.o";
         break;
       }
       case (ToolChain::RLT_CompilerRT): {
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 6eb77610079b7..e8181dca59c17 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -1736,7 +1736,6 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
                     options::OPT_fno_ptrauth_objc_interface_sel);
   Args.addOptInFlag(CmdArgs, options::OPT_fptrauth_objc_class_ro,
                     options::OPT_fno_ptrauth_objc_class_ro);
-
   if (Triple.getEnvironment() == llvm::Triple::PAuthTest)
     handlePAuthABI(Args, CmdArgs);
 
@@ -5982,12 +5981,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddLastArg(CmdArgs, options::OPT_fno_knr_functions);
 
-  // This is a coarse approximation of what llvm-gcc actually does, both
-  // -fasynchronous-unwind-tables and -fnon-call-exceptions interact in more
-  // complicated ways.
   auto SanitizeArgs = TC.getSanitizerArgs(Args);
   Args.AddLastArg(CmdArgs,
                   options::OPT_fallow_runtime_check_skip_hot_cutoff_EQ);
+
+  // This is a coarse approximation of what llvm-gcc actually does, both
+  // -fasynchronous-unwind-tables and -fnon-call-exceptions interact in more
+  // complicated ways.
   bool IsAsyncUnwindTablesDefault =
       TC.getDefaultUnwindTableLevel(Args) == ToolChain::UnwindTableLevel::Asynchronous;
   bool IsSyncUnwindTablesDefault =
@@ -9117,10 +9117,16 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
       OPT_flto_partitions_EQ,
       OPT_flto_EQ};
   const llvm::DenseSet<unsigned> LinkerOptions{OPT_mllvm, OPT_Zlinker_input};
-  auto ShouldForward = [&](const llvm::DenseSet<unsigned> &Set, Arg *A) {
-    return Set.contains(A->getOption().getID()) ||
-           (A->getOption().getGroup().isValid() &&
-            Set.contains(A->getOption().getGroup().getID()));
+  auto ShouldForwardForToolChain = [&](Arg *A, const ToolChain &TC) {
+    // Don't forward -mllvm to toolchains that don't support LLVM.
+    return TC.HasNativeLLVMSupport() || A->getOption().getID() != OPT_mllvm;
+  };
+  auto ShouldForward = [&](const llvm::DenseSet<unsigned> &Set, Arg *A,
+                           const ToolChain &TC) {
+    return (Set.contains(A->getOption().getID()) ||
+            (A->getOption().getGroup().isValid() &&
+             Set.contains(A->getOption().getGroup().getID()))) &&
+           ShouldForwardForToolChain(A, TC);
   };
 
   ArgStringList CmdArgs;
@@ -9139,9 +9145,9 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
       for (Arg *A : ToolChainArgs) {
         if (A->getOption().matches(OPT_Zlinker_input))
           LinkerArgs.emplace_back(A->getValue());
-        else if (ShouldForward(CompilerOptions, A))
+        else if (ShouldForward(CompilerOptions, A, *TC))
           A->render(Args, CompilerArgs);
-        else if (ShouldForward(LinkerOptions, A))
+        else if (ShouldForward(LinkerOptions, A, *TC))
           A->render(Args, LinkerArgs);
       }
 
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 53fd525e941e3..055906a8d721d 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -2132,6 +2132,18 @@ tools::ParsePICArgs(const ToolChain &ToolChain, const ArgList &Args) {
   return std::make_tuple(RelocM, 0U, false);
 }
 
+bool tools::getStaticPIE(const ArgList &Args, const ToolChain &TC) {
+  bool HasStaticPIE = Args.hasArg(options::OPT_static_pie);
+  if (HasStaticPIE && Args.hasArg(options::OPT_no_pie)) {
+    const Driver &D = TC.getDriver();
+    const llvm::opt::OptTable &Opts = D.getOpts();
+    StringRef StaticPIEName = Opts.getOptionName(options::OPT_static_pie);
+    StringRef NoPIEName = Opts.getOptionName(options::OPT_nopie);
+    D.Diag(diag::err_drv_cannot_mix_options) << StaticPIEName << NoPIEName;
+  }
+  return HasStaticPIE;
+}
+
 // `-falign-functions` indicates that the functions should be aligned to the
 // backend's preferred alignment.
 //
@@ -2974,7 +2986,8 @@ void tools::addHIPRuntimeLibArgs(const ToolChain &TC, Compilation &C,
                                  const llvm::opt::ArgList &Args,
                                  llvm::opt::ArgStringList &CmdArgs) {
   if ((C.getActiveOffloadKinds() & Action::OFK_HIP) &&
-      !Args.hasArg(options::OPT_nostdlib) &&
+      (!Args.hasArg(options::OPT_nostdlib) ||
+       TC.getTriple().isKnownWindowsMSVCEnvironment()) &&
       !Args.hasArg(options::OPT_no_hip_rt) && !Args.hasArg(options::OPT_r)) {
     TC.AddHIPRuntimeLibArgs(Args, CmdArgs);
   } else {
@@ -3113,6 +3126,8 @@ void tools::addMCModel(const Driver &D, const llvm::opt::ArgList &Args,
       else if (CM == "medany")
         CM = "large";
       Ok = CM == "small" || CM == "medium" || CM == "large";
+    } else if (Triple.getArch() == llvm::Triple::lanai) {
+      Ok = llvm::is_contained({"small", "medium", "large"}, CM);
     }
     if (Ok) {
       CmdArgs.push_back(Args.MakeArgString("-mcmodel=" + CM));
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index fdfcea852b4f2..327cb5183f837 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -611,9 +611,12 @@ void NVPTX::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(
         "--pxtas-path=" + Args.getLastArgValue(options::OPT_ptxas_path_EQ)));
 
-  if (Args.hasArg(options::OPT_cuda_path_EQ))
-    CmdArgs.push_back(Args.MakeArgString(
-        "--cuda-path=" + Args.getLastArgValue(options::OPT_cuda_path_EQ)));
+  if (Args.hasArg(options::OPT_cuda_path_EQ) || TC.CudaInstallation.isValid()) {
+    StringRef CudaPath = Args.getLastArgValue(
+        options::OPT_cuda_path_EQ,
+        llvm::sys::path::parent_path(TC.CudaInstallation.getBinPath()));
+    CmdArgs.push_back(Args.MakeArgString("--cuda-path=" + CudaPath));
+  }
 
   // Add paths specified in LIBRARY_PATH environment variable as -L options.
   addDirectoryList(Args, CmdArgs, "-L", "LIBRARY_PATH");
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index e5075cbcaf660..234683f2f4882 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -3187,28 +3187,46 @@ void MachO::addClangTargetOptions(const llvm::opt::ArgList &DriverArgs,
 
   ToolChain::addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadKind);
 
-  // On arm64e, enable pointer authentication (for the return address and
-  // indirect calls), as well as usage of the intrinsics.
-  if (getArchName() == "arm64e") {
+  // On arm64e, we enable all the features required for the Darwin userspace
+  // ABI
+  if (getTriple().isArm64e()) {
+    // Core platform ABI
+    if (!DriverArgs.hasArg(options::OPT_fptrauth_calls,
+                           options::OPT_fno_ptrauth_calls))
+      CC1Args.push_back("-fptrauth-calls");
     if (!DriverArgs.hasArg(options::OPT_fptrauth_returns,
                            options::OPT_fno_ptrauth_returns))
       CC1Args.push_back("-fptrauth-returns");
-
     if (!DriverArgs.hasArg(options::OPT_fptrauth_intrinsics,
                            options::OPT_fno_ptrauth_intrinsics))
       CC1Args.push_back("-fptrauth-intrinsics");
-
-    if (!DriverArgs.hasArg(options::OPT_fptrauth_calls,
-                           options::OPT_fno_ptrauth_calls))
-      CC1Args.push_back("-fptrauth-calls");
-
     if (!DriverArgs.hasArg(options::OPT_fptrauth_indirect_gotos,
                            options::OPT_fno_ptrauth_indirect_gotos))
       CC1Args.push_back("-fptrauth-indirect-gotos");
-
     if (!DriverArgs.hasArg(options::OPT_fptrauth_auth_traps,
                            options::OPT_fno_ptrauth_auth_traps))
       CC1Args.push_back("-fptrauth-auth-traps");
+
+    // C++ v-table ABI
+    if (!DriverArgs.hasArg(
+            options::OPT_fptrauth_vtable_pointer_address_discrimination,
+            options::OPT_fno_ptrauth_vtable_pointer_address_discrimination))
+      CC1Args.push_back("-fptrauth-vtable-pointer-address-discrimination");
+    if (!DriverArgs.hasArg(
+            options::OPT_fptrauth_vtable_pointer_type_discrimination,
+            options::OPT_fno_ptrauth_vtable_pointer_type_discrimination))
+      CC1Args.push_back("-fptrauth-vtable-pointer-type-discrimination");
+
+    // Objective-C ABI
+    if (!DriverArgs.hasArg(options::OPT_fptrauth_objc_isa,
+                           options::OPT_fno_ptrauth_objc_isa))
+      CC1Args.push_back("-fptrauth-objc-isa");
+    if (!DriverArgs.hasArg(options::OPT_fptrauth_objc_class_ro,
+                           options::OPT_fno_ptrauth_objc_class_ro))
+      CC1Args.push_back("-fptrauth-objc-class-ro");
+    if (!DriverArgs.hasArg(options::OPT_fptrauth_objc_interface_sel,
+                           options::OPT_fno_ptrauth_objc_interface_sel))
+      CC1Args.push_back("-fptrauth-objc-interface-sel");
   }
 }
 
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 7ab41e9b85a04..1535f4cebf436 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -178,6 +178,8 @@ void Flang::addCodegenOptions(const ArgList &Args,
        options::OPT_fstack_repack_arrays, options::OPT_fno_stack_repack_arrays,
        options::OPT_ftime_report, options::OPT_ftime_report_EQ,
        options::OPT_funroll_loops, options::OPT_fno_unroll_loops});
+  if (Args.hasArg(clang::driver::options::OPT_fcoarray))
+    CmdArgs.push_back("-fcoarray");
 }
 
 void Flang::addPicOptions(const ArgList &Args, ArgStringList &CmdArgs) const {
@@ -532,7 +534,14 @@ void Flang::addTargetOptions(const ArgList &Args,
   }
 
   Args.addAllArgs(CmdArgs,
-                  {options::OPT_fverbose_asm, options::OPT_fno_verbose_asm});
+                  {options::OPT_fverbose_asm, options::OPT_fno_verbose_asm,
+                   options::OPT_fatomic_ignore_denormal_mode,
+                   options::OPT_fno_atomic_ignore_denormal_mode,
+                   options::OPT_fatomic_fine_grained_memory,
+                   options::OPT_fno_atomic_fine_grained_memory,
+                   options::OPT_fatomic_remote_memory,
+                   options::OPT_fno_atomic_remote_memory,
+                   options::OPT_munsafe_fp_atomics});
 }
 
 void Flang::addOffloadOptions(Compilation &C, const InputInfoList &Inputs,
@@ -937,6 +946,8 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
 
       if (Args.hasArg(options::OPT_fopenmp_force_usm))
         CmdArgs.push_back("-fopenmp-force-usm");
+      Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
+                      options::OPT_fno_openmp_simd);
 
       // FIXME: Clang supports a whole bunch more flags here.
       break;
@@ -952,6 +963,9 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
           << A->getSpelling() << A->getValue();
       break;
     }
+  } else {
+    Args.AddLastArg(CmdArgs, options::OPT_fopenmp_simd,
+                    options::OPT_fno_openmp_simd);
   }
 
   // Pass the path to compiler resource files.
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index f5e2655857432..3dade2bdf2277 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -219,18 +219,6 @@ void tools::gcc::Linker::RenderExtraToolArgs(const JobAction &JA,
   // The types are (hopefully) good enough.
 }
 
-static bool getStaticPIE(const ArgList &Args, const ToolChain &TC) {
-  bool HasStaticPIE = Args.hasArg(options::OPT_static_pie);
-  if (HasStaticPIE && Args.hasArg(options::OPT_no_pie)) {
-    const Driver &D = TC.getDriver();
-    const llvm::opt::OptTable &Opts = D.getOpts();
-    StringRef StaticPIEName = Opts.getOptionName(options::OPT_static_pie);
-    StringRef NoPIEName = Opts.getOptionName(options::OPT_nopie);
-    D.Diag(diag::err_drv_cannot_mix_options) << StaticPIEName << NoPIEName;
-  }
-  return HasStaticPIE;
-}
-
 static bool getStatic(const ArgList &Args) {
   return Args.hasArg(options::OPT_static) &&
       !Args.hasArg(options::OPT_static_pie);
@@ -2135,10 +2123,11 @@ void Generic_GCC::GCCInstallationDetector::init(
       StringRef TripleText =
           llvm::sys::path::filename(llvm::sys::path::parent_path(InstallDir));
 
-      Version = GCCVersion::Parse(VersionText);
-      GCCTriple.setTriple(TripleText);
-      GCCInstallPath = std::string(InstallDir);
-      GCCParentLibPath = GCCInstallPath + "/../../..";
+      SelectedInstallation.Version = GCCVersion::Parse(VersionText);
+      SelectedInstallation.GCCTriple.setTriple(TripleText);
+      SelectedInstallation.GCCInstallPath = std::string(InstallDir);
+      SelectedInstallation.GCCParentLibPath =
+          SelectedInstallation.GCCInstallPath + "/../../..";
       IsValid = true;
     }
     return;
@@ -2198,7 +2187,7 @@ void Generic_GCC::GCCInstallationDetector::init(
   // Loop over the various components which exist and select the best GCC
   // installation available. GCC installs are ranked by version number.
   const GCCVersion VersionZero = GCCVersion::Parse("0.0.0");
-  Version = VersionZero;
+  SelectedInstallation.Version = VersionZero;
   for (const std::string &Prefix : Prefixes) {
     auto &VFS = D.getVFS();
     if (!VFS.exists(Prefix))
@@ -2226,7 +2215,7 @@ void Generic_GCC::GCCInstallationDetector::init(
     }
 
     // Skip other prefixes once a GCC installation is found.
-    if (Version > VersionZero)
+    if (SelectedInstallation.Version > VersionZero)
       break;
   }
 }
@@ -2235,14 +2224,17 @@ void Generic_GCC::GCCInstallationDetector::print(raw_ostream &OS) const {
   for (const auto &InstallPath : CandidateGCCInstallPaths)
     OS << "Found candidate GCC installation: " << InstallPath << "\n";
 
-  if (!GCCInstallPath.empty())
-    OS << "Selected GCC installation: " << GCCInstallPath << "\n";
+  if (!SelectedInstallation.GCCInstallPath.empty())
+    OS << "Selected GCC installation: " << SelectedInstallation.GCCInstallPath
+       << "\n";
 
   for (const auto &Multilib : Multilibs)
     OS << "Candidate multilib: " << Multilib << "\n";
 
-  if (Multilibs.size() != 0 || !SelectedMultilib.isDefault())
-    OS << "Selected multilib: " << SelectedMultilib << "\n";
+  if (Multilibs.size() != 0 ||
+      !SelectedInstallation.SelectedMultilib.isDefault())
+    OS << "Selected multilib: " << SelectedInstallation.SelectedMultilib
+       << "\n";
 }
 
 bool Generic_GCC::GCCInstallationDetector::getBiarchSibling(Multilib &M) const {
@@ -2780,14 +2772,50 @@ bool Generic_GCC::GCCInstallationDetector::ScanGCCForMultilibs(
   }
 
   Multilibs = Detected.Multilibs;
-  SelectedMultilib = Detected.SelectedMultilibs.empty()
-                         ? Multilib()
-                         : Detected.SelectedMultilibs.back();
+  SelectedInstallation.SelectedMultilib =
+      Detected.SelectedMultilibs.empty() ? Multilib()
+                                         : Detected.SelectedMultilibs.back();
   BiarchSibling = Detected.BiarchSibling;
 
   return true;
 }
 
+bool Generic_GCC::GCCInstallationDetector::SelectGCCInstallationDirectory(
+    const SmallVector<Generic_GCC::GCCInstallCandidate, 3> &Installations,
+    const ArgList &Args,
+    Generic_GCC::GCCInstallCandidate &SelectedInstallation) const {
+  if (Installations.empty())
+    return false;
+
+  SelectedInstallation =
+      *max_element(Installations, [](const auto &Max, const auto &I) {
+        return I.Version > Max.Version;
+      });
+
+  // FIXME Start selecting installation with libstdc++ in clang 22,
+  // using the current way of selecting the installation as a fallback
+  // only.  For now, warn if the installation with libstdc++ differs
+  // from SelectedInstallation.
+  const GCCInstallCandidate *InstallWithIncludes = nullptr;
+  for (const auto &I : Installations) {
+    if ((!InstallWithIncludes || I.Version > InstallWithIncludes->Version) &&
+        GCCInstallationHasLibStdcxxIncludePaths(I, Args))
+      InstallWithIncludes = &I;
+  }
+
+  if (InstallWithIncludes && SelectedInstallation.GCCInstallPath !=
+                                 InstallWithIncludes->GCCInstallPath)
+    D.Diag(diag::warn_drv_gcc_install_dir_libstdcxx)
+        << InstallWithIncludes->GCCInstallPath
+        << SelectedInstallation.GCCInstallPath;
+
+  // TODO Warn if SelectedInstallation does not contain libstdc++ includes
+  // although compiler flags indicate that it is required (C++ compilation,
+  // libstdc++ not explicitly disabled).
+
+  return true;
+}
+
 void Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple(
     const llvm::Triple &TargetTriple, const ArgList &Args,
     const std::string &LibDir, StringRef CandidateTriple,
@@ -2817,6 +2845,7 @@ void Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple(
        TargetTriple.getVendor() == llvm::Triple::Freescale ||
            TargetTriple.getVendor() == llvm::Triple::OpenEmbedded}};
 
+  SmallVector<GCCInstallCandidate, 3> Installations;
   for (auto &Suffix : Suffixes) {
     if (!Suffix.Active)
       continue;
@@ -2834,23 +2863,31 @@ void Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple(
           continue; // Saw this path before; no need to look at it again.
       if (CandidateVersion.isOlderThan(4, 1, 1))
         continue;
-      if (CandidateVersion <= Version)
+      if (CandidateVersion <= SelectedInstallation.Version && IsValid)
         continue;
 
       if (!ScanGCCForMultilibs(TargetTriple, Args, LI->path(),
                                NeedsBiarchSuffix))
         continue;
 
-      Version = CandidateVersion;
-      GCCTriple.setTriple(CandidateTriple);
+      GCCInstallCandidate Installation;
+      Installation.Version = CandidateVersion;
+      Installation.GCCTriple.setTriple(CandidateTriple);
       // FIXME: We hack together the directory name here instead of
       // using LI to ensure stable path separators across Windows and
       // Linux.
-      GCCInstallPath = (LibDir + "/" + LibSuffix + "/" + VersionText).str();
-      GCCParentLibPath = (GCCInstallPath + "/../" + Suffix.ReversePath).str();
-      IsValid = true;
+      Installation.GCCInstallPath =
+          (LibDir + "/" + LibSuffix + "/" + VersionText).str();
+      Installation.GCCParentLibPath =
+          (Installation.GCCInstallPath + "/../" + Suffix.ReversePath).str();
+      Installation.SelectedMultilib = getMultilib();
+
+      Installations.push_back(Installation);
     }
   }
+
+  IsValid |=
+      SelectGCCInstallationDirectory(Installations, Args, SelectedInstallation);
 }
 
 bool Generic_GCC::GCCInstallationDetector::ScanGentooConfigs(
@@ -2928,10 +2965,12 @@ bool Generic_GCC::GCCInstallationDetector::ScanGentooGccConfig(
                                    NeedsBiarchSuffix))
             continue;
 
-          Version = GCCVersion::Parse(ActiveVersion.second);
-          GCCInstallPath = GentooPath;
-          GCCParentLibPath = GentooPath + std::string("/../../..");
-          GCCTriple.setTriple(ActiveVersion.first);
+          SelectedInstallation.Version =
+              GCCVersion::Parse(ActiveVersion.second);
+          SelectedInstallation.GCCInstallPath = GentooPath;
+          SelectedInstallation.GCCParentLibPath =
+              GentooPath + std::string("/../../..");
+          SelectedInstallation.GCCTriple.setTriple(ActiveVersion.first);
           IsValid = true;
           return true;
         }
@@ -3134,8 +3173,9 @@ void Generic_GCC::AddMultilibIncludeArgs(const ArgList &DriverArgs,
   // gcc TOOL_INCLUDE_DIR.
   const llvm::Triple &GCCTriple = GCCInstallation.getTriple();
   std::string LibPath(GCCInstallation.getParentLibPath());
-  addSystemInclude(DriverArgs, CC1Args,
-                   Twine(LibPath) + "/../" + GCCTriple.str() + "/include");
+  ToolChain::addSystemInclude(DriverArgs, CC1Args,
+                              Twine(LibPath) + "/../" + GCCTriple.str() +
+                                  "/include");
 
   const auto &Callback = Multilibs.includeDirsCallback();
   if (Callback) {
@@ -3222,12 +3262,14 @@ Generic_GCC::addLibCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
     return;
 }
 
-bool Generic_GCC::addLibStdCXXIncludePaths(Twine IncludeDir, StringRef Triple,
-                                           Twine IncludeSuffix,
-                                           const llvm::opt::ArgList &DriverArgs,
-                                           llvm::opt::ArgStringList &CC1Args,
-                                           bool DetectDebian) const {
-  if (!getVFS().exists(IncludeDir))
+static bool addLibStdCXXIncludePaths(llvm::vfs::FileSystem &vfs,
+                                     Twine IncludeDir, StringRef Triple,
+                                     Twine IncludeSuffix,
+                                     const llvm::opt::ArgList &DriverArgs,
+                                     llvm::opt::ArgStringList &CC1Args,
+                                     bool DetectDebian = false) {
+
+  if (!vfs.exists(IncludeDir))
     return false;
 
   // Debian native gcc uses g++-multiarch-incdir.diff which uses
@@ -3239,39 +3281,48 @@ bool Generic_GCC::addLibStdCXXIncludePaths(Twine IncludeDir, StringRef Triple,
   std::string Path =
       (Include + "/" + Triple + Dir.substr(Include.size()) + IncludeSuffix)
           .str();
-  if (DetectDebian && !getVFS().exists(Path))
+  if (DetectDebian && !vfs.exists(Path))
     return false;
 
   // GPLUSPLUS_INCLUDE_DIR
-  addSystemInclude(DriverArgs, CC1Args, IncludeDir);
+  ToolChain::addSystemInclude(DriverArgs, CC1Args, IncludeDir);
   // GPLUSPLUS_TOOL_INCLUDE_DIR. If Triple is not empty, add a target-dependent
   // include directory.
   if (DetectDebian)
-    addSystemInclude(DriverArgs, CC1Args, Path);
+    ToolChain::addSystemInclude(DriverArgs, CC1Args, Path);
   else if (!Triple.empty())
-    addSystemInclude(DriverArgs, CC1Args,
-                     IncludeDir + "/" + Triple + IncludeSuffix);
+    ToolChain::addSystemInclude(DriverArgs, CC1Args,
+                                IncludeDir + "/" + Triple + IncludeSuffix);
   // GPLUSPLUS_BACKWARD_INCLUDE_DIR
-  addSystemInclude(DriverArgs, CC1Args, IncludeDir + "/backward");
+  ToolChain::addSystemInclude(DriverArgs, CC1Args, IncludeDir + "/backward");
   return true;
 }
 
-bool Generic_GCC::addGCCLibStdCxxIncludePaths(
-    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
-    StringRef DebianMultiarch) const {
-  assert(GCCInstallation.isValid());
+bool Generic_GCC::addLibStdCXXIncludePaths(Twine IncludeDir, StringRef Triple,
+                                           Twine IncludeSuffix,
+                                           const llvm::opt::ArgList &DriverArgs,
+                                           llvm::opt::ArgStringList &CC1Args,
+                                           bool DetectDebian) const {
+  return ::addLibStdCXXIncludePaths(getVFS(), IncludeDir, Triple, IncludeSuffix,
+                                    DriverArgs, CC1Args, DetectDebian);
+}
+
+bool Generic_GCC::GCCInstallCandidate::addGCCLibStdCxxIncludePaths(
+    llvm::vfs::FileSystem &vfs, const llvm::opt::ArgList &DriverArgs,
+    llvm::opt::ArgStringList &CC1Args, StringRef DebianMultiarch) const {
 
   // By default, look for the C++ headers in an include directory adjacent to
   // the lib directory of the GCC installation. Note that this is expect to be
   // equivalent to '/usr/include/c++/X.Y' in almost all cases.
-  StringRef LibDir = GCCInstallation.getParentLibPath();
-  StringRef InstallDir = GCCInstallation.getInstallPath();
-  StringRef TripleStr = GCCInstallation.getTriple().str();
-  const Multilib &Multilib = GCCInstallation.getMultilib();
-  const GCCVersion &Version = GCCInstallation.getVersion();
+  StringRef LibDir = getParentLibPath();
+  StringRef InstallDir = getInstallPath();
+  StringRef TripleStr = getTriple().str();
+  const Multilib &Multilib = getMultilib();
+  const GCCVersion &Version = getVersion();
 
   // Try /../$triple/include/c++/$version (gcc --print-multiarch is not empty).
-  if (addLibStdCXXIncludePaths(
+  if (::addLibStdCXXIncludePaths(
+          vfs,
           LibDir.str() + "/../" + TripleStr + "/include/c++/" + Version.Text,
           TripleStr, Multilib.includeSuffix(), DriverArgs, CC1Args))
     return true;
@@ -3279,22 +3330,24 @@ bool Generic_GCC::addGCCLibStdCxxIncludePaths(
   // Try /gcc/$triple/$version/include/c++/ (gcc --print-multiarch is not
   // empty). Like above but for GCC built with
   // --enable-version-specific-runtime-libs.
-  if (addLibStdCXXIncludePaths(LibDir.str() + "/gcc/" + TripleStr + "/" +
-                                   Version.Text + "/include/c++/",
-                               TripleStr, Multilib.includeSuffix(), DriverArgs,
-                               CC1Args))
+  if (::addLibStdCXXIncludePaths(vfs,
+                                 LibDir.str() + "/gcc/" + TripleStr + "/" +
+                                     Version.Text + "/include/c++/",
+                                 TripleStr, Multilib.includeSuffix(),
+                                 DriverArgs, CC1Args))
     return true;
 
   // Detect Debian g++-multiarch-incdir.diff.
-  if (addLibStdCXXIncludePaths(LibDir.str() + "/../include/c++/" + Version.Text,
-                               DebianMultiarch, Multilib.includeSuffix(),
-                               DriverArgs, CC1Args, /*Debian=*/true))
+  if (::addLibStdCXXIncludePaths(
+          vfs, LibDir.str() + "/../include/c++/" + Version.Text,
+          DebianMultiarch, Multilib.includeSuffix(), DriverArgs, CC1Args,
+          /*Debian=*/true))
     return true;
 
   // Try /../include/c++/$version (gcc --print-multiarch is empty).
-  if (addLibStdCXXIncludePaths(LibDir.str() + "/../include/c++/" + Version.Text,
-                               TripleStr, Multilib.includeSuffix(), DriverArgs,
-                               CC1Args))
+  if (::addLibStdCXXIncludePaths(
+          vfs, LibDir.str() + "/../include/c++/" + Version.Text, TripleStr,
+          Multilib.includeSuffix(), DriverArgs, CC1Args))
     return true;
 
   // Otherwise, fall back on a bunch of options which don't use multiarch
@@ -3309,20 +3362,50 @@ bool Generic_GCC::addGCCLibStdCxxIncludePaths(
   };
 
   for (const auto &IncludePath : LibStdCXXIncludePathCandidates) {
-    if (addLibStdCXXIncludePaths(IncludePath, TripleStr,
-                                 Multilib.includeSuffix(), DriverArgs, CC1Args))
+    if (::addLibStdCXXIncludePaths(vfs, IncludePath, TripleStr,
+                                   Multilib.includeSuffix(), DriverArgs,
+                                   CC1Args))
       return true;
   }
   return false;
 }
 
+bool Generic_GCC::GCCInstallationDetector::
+    GCCInstallationHasLibStdcxxIncludePaths(
+        const GCCInstallCandidate &GCCInstallation,
+        const llvm::opt::ArgList &DriverArgs) const {
+  StringRef DebianMultiarch =
+      TripleToDebianMultiarch(GCCInstallation.getTriple());
+
+  // The following function checks for libstdc++ include paths and
+  // adds them to the provided argument list.  Here we just need the
+  // check.
+  llvm::opt::ArgStringList dummyCC1Args;
+  return GCCInstallation.addGCCLibStdCxxIncludePaths(
+      D.getVFS(), DriverArgs, dummyCC1Args, DebianMultiarch);
+}
+
+bool Generic_GCC::addGCCLibStdCxxIncludePaths(
+    const llvm::opt::ArgList &DriverArgs,
+    llvm::opt::ArgStringList &CC1Args) const {
+  assert(GCCInstallation.isValid());
+
+  // Detect Debian g++-multiarch-incdir.diff.
+  StringRef DebianMultiarch =
+      GCCInstallation.TripleToDebianMultiarch(GCCInstallation.getTriple());
+
+  return GCCInstallation.getSelectedInstallation().addGCCLibStdCxxIncludePaths(
+      getVFS(), DriverArgs, CC1Args, DebianMultiarch);
+}
+
 void
 Generic_GCC::addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
                                       llvm::opt::ArgStringList &CC1Args) const {
-  if (GCCInstallation.isValid()) {
-    addGCCLibStdCxxIncludePaths(DriverArgs, CC1Args,
-                                GCCInstallation.getTriple().str());
-  }
+  if (!GCCInstallation.isValid())
+    return;
+
+  GCCInstallation.getSelectedInstallation().addGCCLibStdCxxIncludePaths(
+      getVFS(), DriverArgs, CC1Args, GCCInstallation.getTriple().str());
 }
 
 llvm::opt::DerivedArgList *
diff --git a/clang/lib/Driver/ToolChains/Gnu.h b/clang/lib/Driver/ToolChains/Gnu.h
index 4c42a5e535d56..5fe143b4aa035 100644
--- a/clang/lib/Driver/ToolChains/Gnu.h
+++ b/clang/lib/Driver/ToolChains/Gnu.h
@@ -184,6 +184,39 @@ class LLVM_LIBRARY_VISIBILITY Generic_GCC : public ToolChain {
     bool operator>=(const GCCVersion &RHS) const { return !(*this < RHS); }
   };
 
+  struct GCCInstallCandidate {
+    // FIXME: These might be better as path objects.
+    std::string GCCInstallPath;
+    std::string GCCParentLibPath;
+
+    llvm::Triple GCCTriple;
+
+    /// The primary multilib appropriate for the given flags.
+    Multilib SelectedMultilib;
+
+    GCCVersion Version;
+
+    /// Get the GCC triple for the detected install.
+    const llvm::Triple &getTriple() const { return GCCTriple; }
+
+    /// Get the detected GCC installation path.
+    StringRef getInstallPath() const { return GCCInstallPath; }
+
+    /// Get the detected GCC parent lib path.
+    StringRef getParentLibPath() const { return GCCParentLibPath; }
+
+    /// Get the detected Multilib
+    const Multilib &getMultilib() const { return SelectedMultilib; }
+
+    /// Get the detected GCC version string.
+    const GCCVersion &getVersion() const { return Version; }
+
+    bool addGCCLibStdCxxIncludePaths(llvm::vfs::FileSystem &vfs,
+                                     const llvm::opt::ArgList &DriverArgs,
+                                     llvm::opt::ArgStringList &CC1Args,
+                                     StringRef DebianMultiarch) const;
+  };
+
   /// This is a class to find a viable GCC installation for Clang to
   /// use.
   ///
@@ -192,21 +225,15 @@ class LLVM_LIBRARY_VISIBILITY Generic_GCC : public ToolChain {
   /// Driver, and has logic for fuzzing that where appropriate.
   class GCCInstallationDetector {
     bool IsValid;
-    llvm::Triple GCCTriple;
+
     const Driver &D;
 
-    // FIXME: These might be better as path objects.
-    std::string GCCInstallPath;
-    std::string GCCParentLibPath;
+    GCCInstallCandidate SelectedInstallation;
 
-    /// The primary multilib appropriate for the given flags.
-    Multilib SelectedMultilib;
     /// On Biarch systems, this corresponds to the default multilib when
     /// targeting the non-default multilib. Otherwise, it is empty.
     std::optional<Multilib> BiarchSibling;
 
-    GCCVersion Version;
-
     // We retain the list of install paths that were considered and rejected in
     // order to print out detailed information in verbose mode.
     std::set<std::string> CandidateGCCInstallPaths;
@@ -218,23 +245,50 @@ class LLVM_LIBRARY_VISIBILITY Generic_GCC : public ToolChain {
     const std::string GentooConfigDir = "/etc/env.d/gcc";
 
   public:
+    /// Function for converting a triple to a Debian multiarch.  The
+    /// toolchains use this to adjust the target specific component of
+    /// include paths for Debian.
+    std::function<StringRef(const llvm::Triple &)> TripleToDebianMultiarch =
+        [](const llvm::Triple &T) {
+          StringRef S = T.str();
+          return S;
+        };
+
     explicit GCCInstallationDetector(const Driver &D) : IsValid(false), D(D) {}
+
     void init(const llvm::Triple &TargetTriple, const llvm::opt::ArgList &Args);
 
+    // TODO Replace isValid by changing SelectedInstallation into
+    // std::optional<SelectedInstallation>
+    // and move all accessors for fields of GCCInstallCandidate into
+    // that struct.
+
     /// Check whether we detected a valid GCC install.
     bool isValid() const { return IsValid; }
 
+    const GCCInstallCandidate &getSelectedInstallation() const {
+      return SelectedInstallation;
+    }
+
     /// Get the GCC triple for the detected install.
-    const llvm::Triple &getTriple() const { return GCCTriple; }
+    const llvm::Triple &getTriple() const {
+      return SelectedInstallation.GCCTriple;
+    }
 
     /// Get the detected GCC installation path.
-    StringRef getInstallPath() const { return GCCInstallPath; }
+    StringRef getInstallPath() const {
+      return SelectedInstallation.GCCInstallPath;
+    }
 
     /// Get the detected GCC parent lib path.
-    StringRef getParentLibPath() const { return GCCParentLibPath; }
+    StringRef getParentLibPath() const {
+      return SelectedInstallation.GCCParentLibPath;
+    }
 
     /// Get the detected Multilib
-    const Multilib &getMultilib() const { return SelectedMultilib; }
+    const Multilib &getMultilib() const {
+      return SelectedInstallation.SelectedMultilib;
+    }
 
     /// Get the whole MultilibSet
     const MultilibSet &getMultilibs() const { return Multilibs; }
@@ -244,7 +298,9 @@ class LLVM_LIBRARY_VISIBILITY Generic_GCC : public ToolChain {
     bool getBiarchSibling(Multilib &M) const;
 
     /// Get the detected GCC version string.
-    const GCCVersion &getVersion() const { return Version; }
+    const GCCVersion &getVersion() const {
+      return SelectedInstallation.Version;
+    }
 
     /// Print information about the detected GCC installation.
     void print(raw_ostream &OS) const;
@@ -262,9 +318,21 @@ class LLVM_LIBRARY_VISIBILITY Generic_GCC : public ToolChain {
                                SmallVectorImpl<std::string> &Prefixes,
                                StringRef SysRoot);
 
+    /// Checks if the \p GCCInstallation has libstdc++ include
+    /// directories.
+    bool GCCInstallationHasLibStdcxxIncludePaths(
+        const GCCInstallCandidate &GCCInstallation,
+        const llvm::opt::ArgList &DriverArgs) const;
+
+    /// Select a GCC installation directory from \p Installations and
+    /// set \p SelectedInstallation accordingly.
+    bool SelectGCCInstallationDirectory(
+        const SmallVector<GCCInstallCandidate, 3> &Installations,
+        const llvm::opt::ArgList &Args,
+        GCCInstallCandidate &SelectedInstallation) const;
+
     bool ScanGCCForMultilibs(const llvm::Triple &TargetTriple,
-                             const llvm::opt::ArgList &Args,
-                             StringRef Path,
+                             const llvm::opt::ArgList &Args, StringRef Path,
                              bool NeedsBiarchSuffix = false);
 
     void ScanLibDirForGCCTriple(const llvm::Triple &TargetArch,
@@ -349,8 +417,7 @@ class LLVM_LIBRARY_VISIBILITY Generic_GCC : public ToolChain {
                            llvm::opt::ArgStringList &CC1Args) const;
 
   bool addGCCLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
-                                   llvm::opt::ArgStringList &CC1Args,
-                                   StringRef DebianMultiarch) const;
+                                   llvm::opt::ArgStringList &CC) const;
 
   bool addLibStdCXXIncludePaths(Twine IncludeDir, StringRef Triple,
                                 Twine IncludeSuffix,
diff --git a/clang/lib/Driver/ToolChains/HLSL.cpp b/clang/lib/Driver/ToolChains/HLSL.cpp
index 38f4643abad98..eaa7d736719b5 100644
--- a/clang/lib/Driver/ToolChains/HLSL.cpp
+++ b/clang/lib/Driver/ToolChains/HLSL.cpp
@@ -173,24 +173,71 @@ bool isLegalValidatorVersion(StringRef ValVersionStr, const Driver &D) {
   return true;
 }
 
-std::string getSpirvExtArg(ArrayRef<std::string> SpvExtensionArgs) {
+void getSpirvExtOperand(StringRef SpvExtensionArg, raw_ostream &out) {
+  // The extensions that are commented out are supported in DXC, but the SPIR-V
+  // backend does not know about them yet.
+  static const std::vector<StringRef> DxcSupportedExtensions = {
+      "SPV_KHR_16bit_storage", "SPV_KHR_device_group",
+      "SPV_KHR_fragment_shading_rate", "SPV_KHR_multiview",
+      "SPV_KHR_post_depth_coverage", "SPV_KHR_non_semantic_info",
+      "SPV_KHR_shader_draw_parameters", "SPV_KHR_ray_tracing",
+      "SPV_KHR_shader_clock", "SPV_EXT_demote_to_helper_invocation",
+      "SPV_EXT_descriptor_indexing", "SPV_EXT_fragment_fully_covered",
+      "SPV_EXT_fragment_invocation_density",
+      "SPV_EXT_fragment_shader_interlock", "SPV_EXT_mesh_shader",
+      "SPV_EXT_shader_stencil_export", "SPV_EXT_shader_viewport_index_layer",
+      // "SPV_AMD_shader_early_and_late_fragment_tests",
+      "SPV_GOOGLE_hlsl_functionality1", "SPV_GOOGLE_user_type",
+      "SPV_KHR_ray_query", "SPV_EXT_shader_image_int64",
+      "SPV_KHR_fragment_shader_barycentric", "SPV_KHR_physical_storage_buffer",
+      "SPV_KHR_vulkan_memory_model",
+      // "SPV_KHR_compute_shader_derivatives",
+      // "SPV_KHR_maximal_reconvergence",
+      "SPV_KHR_float_controls", "SPV_NV_shader_subgroup_partitioned",
+      // "SPV_KHR_quad_control"
+  };
+
+  if (SpvExtensionArg.starts_with("SPV_")) {
+    out << "+" << SpvExtensionArg;
+    return;
+  }
+
+  if (SpvExtensionArg.compare_insensitive("DXC") == 0) {
+    bool first = true;
+    std::string Operand;
+    for (StringRef E : DxcSupportedExtensions) {
+      if (!first)
+        out << ",";
+      else
+        first = false;
+      out << "+" << E;
+    }
+    return;
+  }
+  out << SpvExtensionArg;
+}
+
+SmallString<1024> getSpirvExtArg(ArrayRef<std::string> SpvExtensionArgs) {
   if (SpvExtensionArgs.empty()) {
-    return "-spirv-ext=all";
+    return StringRef("-spirv-ext=all");
   }
 
-  std::string LlvmOption =
-      (Twine("-spirv-ext=+") + SpvExtensionArgs.front()).str();
+  llvm::SmallString<1024> LlvmOption;
+  raw_svector_ostream out(LlvmOption);
+
+  out << "-spirv-ext=";
+  getSpirvExtOperand(SpvExtensionArgs[0], out);
+
   SpvExtensionArgs = SpvExtensionArgs.slice(1);
-  for (auto Extension : SpvExtensionArgs) {
-    if (Extension != "KHR")
-      Extension = (Twine("+") + Extension).str();
-    LlvmOption = (Twine(LlvmOption) + "," + Extension).str();
+  for (StringRef Extension : SpvExtensionArgs) {
+    out << ",";
+    getSpirvExtOperand(Extension, out);
   }
   return LlvmOption;
 }
 
 bool isValidSPIRVExtensionName(const std::string &str) {
-  std::regex pattern("KHR|SPV_[a-zA-Z0-9_]+");
+  std::regex pattern("dxc|DXC|khr|KHR|SPV_[a-zA-Z0-9_]+");
   return std::regex_match(str, pattern);
 }
 
@@ -371,7 +418,7 @@ HLSLToolChain::TranslateArgs(const DerivedArgList &Args, StringRef BoundArch,
     std::vector<std::string> SpvExtensionArgs =
         Args.getAllArgValues(options::OPT_fspv_extension_EQ);
     if (checkExtensionArgsAreValid(SpvExtensionArgs, getDriver())) {
-      std::string LlvmOption = getSpirvExtArg(SpvExtensionArgs);
+      SmallString<1024> LlvmOption = getSpirvExtArg(SpvExtensionArgs);
       DAL->AddSeparateArg(nullptr, Opts.getOption(options::OPT_mllvm),
                           LlvmOption);
     }
diff --git a/clang/lib/Driver/ToolChains/Hurd.cpp b/clang/lib/Driver/ToolChains/Hurd.cpp
index a22a8face1797..8bcc7e6d9759e 100644
--- a/clang/lib/Driver/ToolChains/Hurd.cpp
+++ b/clang/lib/Driver/ToolChains/Hurd.cpp
@@ -71,6 +71,13 @@ static StringRef getOSLibDir(const llvm::Triple &Triple, const ArgList &Args) {
 
 Hurd::Hurd(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
     : Generic_ELF(D, Triple, Args) {
+  GCCInstallation.TripleToDebianMultiarch = [](const llvm::Triple &T) {
+    StringRef TripleStr = T.str();
+    StringRef DebianMultiarch =
+        T.getArch() == llvm::Triple::x86 ? "i386-gnu" : TripleStr;
+    return DebianMultiarch;
+  };
+
   GCCInstallation.init(Triple, Args);
   Multilibs = GCCInstallation.getMultilibs();
   SelectedMultilibs.assign({GCCInstallation.getMultilib()});
@@ -207,12 +214,7 @@ void Hurd::addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
   if (!GCCInstallation.isValid())
     return;
 
-  StringRef TripleStr = GCCInstallation.getTriple().str();
-  StringRef DebianMultiarch =
-      GCCInstallation.getTriple().getArch() == llvm::Triple::x86 ? "i386-gnu"
-                                                                 : TripleStr;
-
-  addGCCLibStdCxxIncludePaths(DriverArgs, CC1Args, DebianMultiarch);
+  addGCCLibStdCxxIncludePaths(DriverArgs, CC1Args);
 }
 
 void Hurd::addExtraOpts(llvm::opt::ArgStringList &CmdArgs) const {
diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 8ac8d4eb91812..16e35b08cfbd6 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -211,6 +211,13 @@ static StringRef getOSLibDir(const llvm::Triple &Triple, const ArgList &Args) {
 
 Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
     : Generic_ELF(D, Triple, Args) {
+  GCCInstallation.TripleToDebianMultiarch = [](const llvm::Triple &T) {
+    StringRef TripleStr = T.str();
+    StringRef DebianMultiarch =
+        T.getArch() == llvm::Triple::x86 ? "i386-linux-gnu" : TripleStr;
+    return DebianMultiarch;
+  };
+
   GCCInstallation.init(Triple, Args);
   Multilibs = GCCInstallation.getMultilibs();
   SelectedMultilibs.assign({GCCInstallation.getMultilib()});
@@ -693,22 +700,15 @@ void Linux::addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
   if (!GCCInstallation.isValid())
     return;
 
-  // Detect Debian g++-multiarch-incdir.diff.
-  StringRef TripleStr = GCCInstallation.getTriple().str();
-  StringRef DebianMultiarch =
-      GCCInstallation.getTriple().getArch() == llvm::Triple::x86
-          ? "i386-linux-gnu"
-          : TripleStr;
-
   // Try generic GCC detection first.
-  if (Generic_GCC::addGCCLibStdCxxIncludePaths(DriverArgs, CC1Args,
-                                               DebianMultiarch))
+  if (Generic_GCC::addGCCLibStdCxxIncludePaths(DriverArgs, CC1Args))
     return;
 
   StringRef LibDir = GCCInstallation.getParentLibPath();
   const Multilib &Multilib = GCCInstallation.getMultilib();
   const GCCVersion &Version = GCCInstallation.getVersion();
 
+  StringRef TripleStr = GCCInstallation.getTriple().str();
   const std::string LibStdCXXIncludePathCandidates[] = {
       // Android standalone toolchain has C++ headers in yet another place.
       LibDir.str() + "/../" + TripleStr.str() + "/include/c++/" + Version.Text,
diff --git a/clang/lib/Driver/ToolChains/Managarm.cpp b/clang/lib/Driver/ToolChains/Managarm.cpp
index 0f56f0f6aad77..da4a9072317f4 100644
--- a/clang/lib/Driver/ToolChains/Managarm.cpp
+++ b/clang/lib/Driver/ToolChains/Managarm.cpp
@@ -193,10 +193,8 @@ void Managarm::addLibStdCxxIncludePaths(
   if (!GCCInstallation.isValid())
     return;
 
-  StringRef TripleStr = GCCInstallation.getTriple().str();
-
   // Try generic GCC detection.
-  Generic_GCC::addGCCLibStdCxxIncludePaths(DriverArgs, CC1Args, TripleStr);
+  addGCCLibStdCxxIncludePaths(DriverArgs, CC1Args);
 }
 
 SanitizerMask Managarm::getSupportedSanitizers() const {
diff --git a/clang/lib/ExtractAPI/DeclarationFragments.cpp b/clang/lib/ExtractAPI/DeclarationFragments.cpp
index 51a6f6b779e77..541af6d587174 100644
--- a/clang/lib/ExtractAPI/DeclarationFragments.cpp
+++ b/clang/lib/ExtractAPI/DeclarationFragments.cpp
@@ -205,45 +205,39 @@ DeclarationFragments::getStructureTypeFragment(const RecordDecl *Record) {
 // Build declaration fragments for NNS recursively so that we have the USR for
 // every part in a qualified name, and also leaves the actual underlying type
 // cleaner for its own fragment.
-DeclarationFragments
-DeclarationFragmentsBuilder::getFragmentsForNNS(const NestedNameSpecifier *NNS,
-                                                ASTContext &Context,
-                                                DeclarationFragments &After) {
+DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForNNS(
+    NestedNameSpecifier NNS, ASTContext &Context, DeclarationFragments &After) {
   DeclarationFragments Fragments;
-  if (NNS->getPrefix())
-    Fragments.append(getFragmentsForNNS(NNS->getPrefix(), Context, After));
-
-  switch (NNS->getKind()) {
-  case NestedNameSpecifier::Identifier:
-    Fragments.append(NNS->getAsIdentifier()->getName(),
-                     DeclarationFragments::FragmentKind::Identifier);
-    break;
+  switch (NNS.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+    return Fragments;
 
-  case NestedNameSpecifier::Namespace: {
-    const NamespaceBaseDecl *NS = NNS->getAsNamespace();
-    if (const auto *Namespace = dyn_cast<NamespaceDecl>(NS);
-        Namespace && Namespace->isAnonymousNamespace())
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [Namespace, Prefix] = NNS.getAsNamespaceAndPrefix();
+    Fragments.append(getFragmentsForNNS(Prefix, Context, After));
+    if (const auto *NS = dyn_cast<NamespaceDecl>(Namespace);
+        NS && NS->isAnonymousNamespace())
       return Fragments;
     SmallString<128> USR;
-    index::generateUSRForDecl(NS, USR);
-    Fragments.append(NS->getName(),
-                     DeclarationFragments::FragmentKind::Identifier, USR, NS);
+    index::generateUSRForDecl(Namespace, USR);
+    Fragments.append(Namespace->getName(),
+                     DeclarationFragments::FragmentKind::Identifier, USR,
+                     Namespace);
     break;
   }
 
-  case NestedNameSpecifier::Global:
+  case NestedNameSpecifier::Kind::Global:
     // The global specifier `::` at the beginning. No stored value.
     break;
 
-  case NestedNameSpecifier::Super:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     // Microsoft's `__super` specifier.
     Fragments.append("__super", DeclarationFragments::FragmentKind::Keyword);
     break;
 
-  case NestedNameSpecifier::TypeSpec: {
-    const Type *T = NNS->getAsType();
+  case NestedNameSpecifier::Kind::Type: {
     // FIXME: Handle C++ template specialization type
-    Fragments.append(getFragmentsForType(T, Context, After));
+    Fragments.append(getFragmentsForType(NNS.getAsType(), Context, After));
     break;
   }
   }
@@ -273,26 +267,6 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
     return Fragments;
   }
 
-  // An ElaboratedType is a sugar for types that are referred to using an
-  // elaborated keyword, e.g., `struct S`, `enum E`, or (in C++) via a
-  // qualified name, e.g., `N::M::type`, or both.
-  if (const ElaboratedType *ET = dyn_cast<ElaboratedType>(T)) {
-    ElaboratedTypeKeyword Keyword = ET->getKeyword();
-    if (Keyword != ElaboratedTypeKeyword::None) {
-      Fragments
-          .append(ElaboratedType::getKeywordName(Keyword),
-                  DeclarationFragments::FragmentKind::Keyword)
-          .appendSpace();
-    }
-
-    if (const NestedNameSpecifier *NNS = ET->getQualifier())
-      Fragments.append(getFragmentsForNNS(NNS, Context, After));
-
-    // After handling the elaborated keyword or qualified name, build
-    // declaration fragments for the desugared underlying type.
-    return Fragments.append(getFragmentsForType(ET->desugar(), Context, After));
-  }
-
   // If the type is a typedefed type, get the underlying TypedefNameDecl for a
   // direct reference to the typedef instead of the wrapped type.
 
@@ -303,7 +277,18 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
     TypedefUnderlyingTypeResolver TypedefResolver(Context);
     std::string USR = TypedefResolver.getUSRForType(QualType(T, 0));
 
-    if (T->isObjCIdType()) {
+    if (ElaboratedTypeKeyword Keyword = TypedefTy->getKeyword();
+        Keyword != ElaboratedTypeKeyword::None) {
+      Fragments
+          .append(KeywordHelpers::getKeywordName(Keyword),
+                  DeclarationFragments::FragmentKind::Keyword)
+          .appendSpace();
+    }
+
+    Fragments.append(
+        getFragmentsForNNS(TypedefTy->getQualifier(), Context, After));
+
+    if (TypedefTy->isObjCIdType()) {
       return Fragments.append(Decl->getName(),
                               DeclarationFragments::FragmentKind::Keyword);
     }
@@ -396,14 +381,26 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
 
   if (const TemplateSpecializationType *TemplSpecTy =
           dyn_cast<TemplateSpecializationType>(T)) {
-    const auto TemplName = TemplSpecTy->getTemplateName();
+    if (ElaboratedTypeKeyword Keyword = TemplSpecTy->getKeyword();
+        Keyword != ElaboratedTypeKeyword::None)
+      Fragments
+          .append(KeywordHelpers::getKeywordName(Keyword),
+                  DeclarationFragments::FragmentKind::Keyword)
+          .appendSpace();
+
+    auto TemplName = TemplSpecTy->getTemplateName();
     std::string Str;
     raw_string_ostream Stream(Str);
     TemplName.print(Stream, Context.getPrintingPolicy(),
                     TemplateName::Qualified::AsWritten);
     SmallString<64> USR("");
+    if (const auto *QTN = TemplName.getAsQualifiedTemplateName()) {
+      Fragments.append(getFragmentsForNNS(QTN->getQualifier(), Context, After));
+      TemplName = QTN->getUnderlyingTemplate();
+    }
     if (const auto *TemplDecl = TemplName.getAsTemplateDecl())
       index::generateUSRForDecl(TemplDecl, USR);
+    // FIXME: Handle other kinds of TemplateNames.
 
     return Fragments
         .append(Str, DeclarationFragments::FragmentKind::TypeIdentifier, USR)
@@ -413,14 +410,19 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
         .append(">", DeclarationFragments::FragmentKind::Text);
   }
 
-  // Everything we care about has been handled now, reduce to the canonical
-  // unqualified base type.
-  QualType Base = T->getCanonicalTypeUnqualified();
-
   // If the base type is a TagType (struct/interface/union/class/enum), let's
   // get the underlying Decl for better names and USRs.
-  if (const TagType *TagTy = dyn_cast<TagType>(Base)) {
-    const TagDecl *Decl = TagTy->getDecl();
+  if (const TagType *TagTy = dyn_cast<TagType>(T)) {
+    if (ElaboratedTypeKeyword Keyword = TagTy->getKeyword();
+        Keyword != ElaboratedTypeKeyword::None)
+      Fragments
+          .append(KeywordHelpers::getKeywordName(Keyword),
+                  DeclarationFragments::FragmentKind::Keyword)
+          .appendSpace();
+
+    Fragments.append(getFragmentsForNNS(TagTy->getQualifier(), Context, After));
+
+    const TagDecl *Decl = TagTy->getOriginalDecl();
     // Anonymous decl, skip this fragment.
     if (Decl->getName().empty())
       return Fragments.append("{ ... }",
@@ -432,6 +434,10 @@ DeclarationFragments DeclarationFragmentsBuilder::getFragmentsForType(
                             TagUSR, Decl);
   }
 
+  // Everything we care about has been handled now, reduce to the canonical
+  // unqualified base type.
+  QualType Base = T->getCanonicalTypeUnqualified();
+
   // If the base type is an ObjCInterfaceType, use the underlying
   // ObjCInterfaceDecl for the true USR.
   if (const auto *ObjCIT = dyn_cast<ObjCInterfaceType>(Base)) {
diff --git a/clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.cpp b/clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.cpp
index 41e4e0cf1795f..5adbbc6d1c34c 100644
--- a/clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.cpp
+++ b/clang/lib/ExtractAPI/TypedefUnderlyingTypeResolver.cpp
@@ -26,7 +26,7 @@ TypedefUnderlyingTypeResolver::getUnderlyingTypeDecl(QualType Type) const {
   if (TypedefTy)
     TypeDecl = TypedefTy->getDecl();
   if (const TagType *TagTy = Type->getAs<TagType>()) {
-    TypeDecl = TagTy->getDecl();
+    TypeDecl = TagTy->getOriginalDecl();
   } else if (const ObjCInterfaceType *ObjCITy =
                  Type->getAs<ObjCInterfaceType>()) {
     TypeDecl = ObjCITy->getDecl();
diff --git a/clang/lib/Format/CMakeLists.txt b/clang/lib/Format/CMakeLists.txt
index 9f4939824fdb8..24f435d2caee1 100644
--- a/clang/lib/Format/CMakeLists.txt
+++ b/clang/lib/Format/CMakeLists.txt
@@ -13,6 +13,7 @@ add_clang_library(clangFormat
   MacroExpander.cpp
   MatchFilePath.cpp
   NamespaceEndCommentsFixer.cpp
+  NumericLiteralInfo.cpp
   ObjCPropertyAttributeOrderFixer.cpp
   QualifierAlignmentFixer.cpp
   SortJavaScriptImports.cpp
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 9a10403b858f9..888d0faf80931 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -629,9 +629,16 @@ bool ContinuationIndenter::mustBreak(const LineState &State) {
       // name.
       !Style.isJavaScript() && Previous.isNot(tok::kw_template) &&
       CurrentState.BreakBeforeParameter) {
-    for (const auto *Tok = &Previous; Tok; Tok = Tok->Previous)
-      if (Tok->FirstAfterPPLine || Tok->is(TT_LineComment))
+    for (const auto *Tok = &Previous; Tok; Tok = Tok->Previous) {
+      if (Tok->is(TT_LineComment))
         return false;
+      if (Tok->is(TT_TemplateCloser)) {
+        Tok = Tok->MatchingParen;
+        assert(Tok);
+      }
+      if (Tok->FirstAfterPPLine)
+        return false;
+    }
 
     return true;
   }
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 063780721423f..e3b22cdabaccd 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -763,6 +763,15 @@ struct ScalarEnumerationTraits<FormatStyle::SpaceBeforeParensStyle> {
   }
 };
 
+template <>
+struct ScalarEnumerationTraits<FormatStyle::SpaceInEmptyBracesStyle> {
+  static void enumeration(IO &IO, FormatStyle::SpaceInEmptyBracesStyle &Value) {
+    IO.enumCase(Value, "Always", FormatStyle::SIEB_Always);
+    IO.enumCase(Value, "Block", FormatStyle::SIEB_Block);
+    IO.enumCase(Value, "Never", FormatStyle::SIEB_Never);
+  }
+};
+
 template <> struct ScalarEnumerationTraits<FormatStyle::SpacesInAnglesStyle> {
   static void enumeration(IO &IO, FormatStyle::SpacesInAnglesStyle &Value) {
     IO.enumCase(Value, "Never", FormatStyle::SIAS_Never);
@@ -931,6 +940,7 @@ template <> struct MappingTraits<FormatStyle> {
     bool DeriveLineEnding = true;
     bool UseCRLF = false;
 
+    bool SpaceInEmptyBlock = false;
     bool SpaceInEmptyParentheses = false;
     bool SpacesInConditionalStatement = false;
     bool SpacesInCStyleCastParentheses = false;
@@ -960,6 +970,7 @@ template <> struct MappingTraits<FormatStyle> {
       IO.mapOptional("PointerBindsToType", Style.PointerAlignment);
       IO.mapOptional("SpaceAfterControlStatementKeyword",
                      Style.SpaceBeforeParens);
+      IO.mapOptional("SpaceInEmptyBlock", SpaceInEmptyBlock);
       IO.mapOptional("SpaceInEmptyParentheses", SpaceInEmptyParentheses);
       IO.mapOptional("SpacesInConditionalStatement",
                      SpacesInConditionalStatement);
@@ -1193,7 +1204,7 @@ template <> struct MappingTraits<FormatStyle> {
                    Style.SpaceBeforeRangeBasedForLoopColon);
     IO.mapOptional("SpaceBeforeSquareBrackets",
                    Style.SpaceBeforeSquareBrackets);
-    IO.mapOptional("SpaceInEmptyBlock", Style.SpaceInEmptyBlock);
+    IO.mapOptional("SpaceInEmptyBraces", Style.SpaceInEmptyBraces);
     IO.mapOptional("SpacesBeforeTrailingComments",
                    Style.SpacesBeforeTrailingComments);
     IO.mapOptional("SpacesInAngles", Style.SpacesInAngles);
@@ -1276,6 +1287,13 @@ template <> struct MappingTraits<FormatStyle> {
         Style.LineEnding = FormatStyle::LE_DeriveCRLF;
     }
 
+    // If SpaceInEmptyBlock was specified but SpaceInEmptyBraces was not,
+    // initialize the latter from the former for backward compatibility.
+    if (SpaceInEmptyBlock &&
+        Style.SpaceInEmptyBraces == FormatStyle::SIEB_Never) {
+      Style.SpaceInEmptyBraces = FormatStyle::SIEB_Block;
+    }
+
     if (Style.SpacesInParens != FormatStyle::SIPO_Custom &&
         (SpacesInParentheses || SpaceInEmptyParentheses ||
          SpacesInConditionalStatement || SpacesInCStyleCastParentheses)) {
@@ -1677,7 +1695,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.SpaceBeforeParensOptions.AfterIfMacros = true;
   LLVMStyle.SpaceBeforeRangeBasedForLoopColon = true;
   LLVMStyle.SpaceBeforeSquareBrackets = false;
-  LLVMStyle.SpaceInEmptyBlock = false;
+  LLVMStyle.SpaceInEmptyBraces = FormatStyle::SIEB_Never;
   LLVMStyle.SpacesBeforeTrailingComments = 1;
   LLVMStyle.SpacesInAngles = FormatStyle::SIAS_Never;
   LLVMStyle.SpacesInContainerLiterals = true;
@@ -1984,7 +2002,7 @@ FormatStyle getWebKitStyle() {
   Style.ObjCSpaceAfterProperty = true;
   Style.PointerAlignment = FormatStyle::PAS_Left;
   Style.SpaceBeforeCpp11BracedList = true;
-  Style.SpaceInEmptyBlock = true;
+  Style.SpaceInEmptyBraces = FormatStyle::SIEB_Always;
   return Style;
 }
 
diff --git a/clang/lib/Format/NumericLiteralInfo.cpp b/clang/lib/Format/NumericLiteralInfo.cpp
new file mode 100644
index 0000000000000..81e6dd5e58bbc
--- /dev/null
+++ b/clang/lib/Format/NumericLiteralInfo.cpp
@@ -0,0 +1,65 @@
+//===--- NumericLiteralInfo.cpp ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the functionality of getting information about a
+/// numeric literal string, including 0-based positions of the base letter, the
+/// decimal/hexadecimal point, the exponent letter, and the suffix, or npos if
+/// absent.
+///
+//===----------------------------------------------------------------------===//
+
+#include "NumericLiteralInfo.h"
+#include "llvm/ADT/StringExtras.h"
+
+namespace clang {
+namespace format {
+
+using namespace llvm;
+
+NumericLiteralInfo::NumericLiteralInfo(StringRef Text, char Separator) {
+  if (Text.size() < 2)
+    return;
+
+  bool IsHex = false;
+  if (Text[0] == '0') {
+    switch (Text[1]) {
+    case 'x':
+    case 'X':
+      IsHex = true;
+      [[fallthrough]];
+    case 'b':
+    case 'B':
+    case 'o': // JavaScript octal.
+    case 'O':
+      BaseLetterPos = 1; // e.g. 0xF
+      break;
+    }
+  }
+
+  DotPos = Text.find('.', BaseLetterPos + 1); // e.g. 0x.1 or .1
+
+  // e.g. 1.e2 or 0xFp2
+  const auto Pos = DotPos != StringRef::npos ? DotPos + 1 : BaseLetterPos + 2;
+
+  ExponentLetterPos =
+      // Trim C++ user-defined suffix as in `1_Pa`.
+      (Separator == '\'' ? Text.take_front(Text.find('_')) : Text)
+          .find_insensitive(IsHex ? 'p' : 'e', Pos);
+
+  const bool HasExponent = ExponentLetterPos != StringRef::npos;
+  SuffixPos = Text.find_if_not(
+      [&](char C) {
+        return (HasExponent || !IsHex ? isDigit : isHexDigit)(C) ||
+               C == Separator;
+      },
+      HasExponent ? ExponentLetterPos + 2 : Pos); // e.g. 1e-2f
+}
+
+} // namespace format
+} // namespace clang
diff --git a/clang/lib/Format/NumericLiteralInfo.h b/clang/lib/Format/NumericLiteralInfo.h
new file mode 100644
index 0000000000000..0210f6c94d3e7
--- /dev/null
+++ b/clang/lib/Format/NumericLiteralInfo.h
@@ -0,0 +1,29 @@
+//===--- NumericLiteralInfo.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_FORMAT_NUMERICLITERALINFO_H
+#define LLVM_CLANG_LIB_FORMAT_NUMERICLITERALINFO_H
+
+#include "llvm/ADT/StringRef.h"
+
+namespace clang {
+namespace format {
+
+struct NumericLiteralInfo {
+  size_t BaseLetterPos = llvm::StringRef::npos;     // as in 0b1, 0xF, etc.
+  size_t DotPos = llvm::StringRef::npos;            // pos of decimal/hex point
+  size_t ExponentLetterPos = llvm::StringRef::npos; // as in 9e9 and 0xFp9
+  size_t SuffixPos = llvm::StringRef::npos;         // starting pos of suffix
+
+  NumericLiteralInfo(llvm::StringRef Text, char Separator = '\'');
+};
+
+} // end namespace format
+} // end namespace clang
+
+#endif
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 4801d27b1395a..a220de54f46bf 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -2590,6 +2590,9 @@ class AnnotatingParser {
     if (!Tok.Previous || Tok.isNot(tok::identifier) || Tok.is(TT_ClassHeadName))
       return false;
 
+    if (Tok.endsSequence(Keywords.kw_final, TT_ClassHeadName))
+      return false;
+
     if ((Style.isJavaScript() || Style.isJava()) && Tok.is(Keywords.kw_extends))
       return false;
 
@@ -4513,16 +4516,9 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
     return Left.is(tok::hash);
   if (Left.isOneOf(tok::hashhash, tok::hash))
     return Right.is(tok::hash);
-  if (Left.is(BK_Block) && Right.is(tok::r_brace) &&
-      Right.MatchingParen == &Left && Line.Children.empty()) {
-    return Style.SpaceInEmptyBlock;
-  }
   if (Style.SpacesInParens == FormatStyle::SIPO_Custom) {
-    if ((Left.is(tok::l_paren) && Right.is(tok::r_paren)) ||
-        (Left.is(tok::l_brace) && Left.isNot(BK_Block) &&
-         Right.is(tok::r_brace) && Right.isNot(BK_Block))) {
+    if (Left.is(tok::l_paren) && Right.is(tok::r_paren))
       return Style.SpacesInParensOptions.InEmptyParentheses;
-    }
     if (Style.SpacesInParensOptions.ExceptDoubleParentheses &&
         Left.is(tok::r_paren) && Right.is(tok::r_paren)) {
       auto *InnerLParen = Left.MatchingParen;
@@ -4800,8 +4796,6 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
         Right.is(TT_ArraySubscriptLSquare))) {
     return false;
   }
-  if (Left.is(tok::l_brace) && Right.is(tok::r_brace))
-    return !Left.Children.empty(); // No spaces in "{}".
   if ((Left.is(tok::l_brace) && Left.isNot(BK_Block)) ||
       (Right.is(tok::r_brace) && Right.MatchingParen &&
        Right.MatchingParen->isNot(BK_Block))) {
@@ -4983,6 +4977,17 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
   if (Left.is(tok::star) && Right.is(tok::comment))
     return true;
 
+  if (Left.is(tok::l_brace) && Right.is(tok::r_brace) &&
+      Left.Children.empty()) {
+    if (Left.is(BK_Block))
+      return Style.SpaceInEmptyBraces != FormatStyle::SIEB_Never;
+    if (Style.Cpp11BracedListStyle) {
+      return Style.SpacesInParens == FormatStyle::SIPO_Custom &&
+             Style.SpacesInParensOptions.InEmptyParentheses;
+    }
+    return Style.SpaceInEmptyBraces == FormatStyle::SIEB_Always;
+  }
+
   const auto *BeforeLeft = Left.Previous;
 
   if (IsCpp) {
@@ -6269,7 +6274,8 @@ bool TokenAnnotator::canBreakBefore(const AnnotatedLine &Line,
   }
 
   if (Right.is(tok::colon) &&
-      !Right.isOneOf(TT_CtorInitializerColon, TT_InlineASMColon)) {
+      !Right.isOneOf(TT_CtorInitializerColon, TT_InlineASMColon,
+                     TT_BitFieldColon)) {
     return false;
   }
   if (Left.is(tok::colon) && Left.isOneOf(TT_DictLiteral, TT_ObjCMethodExpr)) {
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index 0adf7ee9ed543..c938ff3965f9e 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -864,7 +864,8 @@ class LineJoiner {
       if (ShouldMerge()) {
         // We merge empty blocks even if the line exceeds the column limit.
         Tok->SpacesRequiredBefore =
-            (Style.SpaceInEmptyBlock || Line.Last->is(tok::comment)) ? 1 : 0;
+            Style.SpaceInEmptyBraces != FormatStyle::SIEB_Never ||
+            Line.Last->is(tok::comment);
         Tok->CanBreakBefore = true;
         return 1;
       } else if (Limit != 0 && !Line.startsWithNamespace() &&
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 91b8fdc8a3c38..ca7e2da3799c4 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -1182,10 +1182,8 @@ void UnwrappedLineParser::parsePPDefine() {
   if (MaybeIncludeGuard && !eof())
     IncludeGuard = IG_Rejected;
 
-  if (FormatTok->Tok.getKind() == tok::l_paren &&
-      !FormatTok->hasWhitespaceBefore()) {
+  if (FormatTok->is(tok::l_paren) && !FormatTok->hasWhitespaceBefore())
     parseParens();
-  }
   if (Style.IndentPPDirectives != FormatStyle::PPDIS_None)
     Line->Level += PPBranchLevel + 1;
   addUnwrappedLine();
@@ -1193,23 +1191,34 @@ void UnwrappedLineParser::parsePPDefine() {
 
   Line->PPLevel = PPBranchLevel + (IncludeGuard == IG_Defined ? 0 : 1);
   assert((int)Line->PPLevel >= 0);
+
+  if (eof())
+    return;
+
   Line->InMacroBody = true;
 
-  if (Style.SkipMacroDefinitionBody) {
-    while (!eof()) {
-      FormatTok->Finalized = true;
-      FormatTok = Tokens->getNextToken();
-    }
-    addUnwrappedLine();
+  if (!Style.SkipMacroDefinitionBody) {
+    // Errors during a preprocessor directive can only affect the layout of the
+    // preprocessor directive, and thus we ignore them. An alternative approach
+    // would be to use the same approach we use on the file level (no
+    // re-indentation if there was a structural error) within the macro
+    // definition.
+    parseFile();
     return;
   }
 
-  // Errors during a preprocessor directive can only affect the layout of the
-  // preprocessor directive, and thus we ignore them. An alternative approach
-  // would be to use the same approach we use on the file level (no
-  // re-indentation if there was a structural error) within the macro
-  // definition.
-  parseFile();
+  if (auto *Prev = Tokens->getPreviousToken(); Prev->is(tok::comment) &&
+                                               Prev->NewlinesBefore > 0 &&
+                                               !Prev->HasUnescapedNewline) {
+    Prev->Finalized = true;
+  }
+
+  do {
+    FormatTok->Finalized = true;
+    FormatTok = Tokens->getNextToken();
+  } while (!eof());
+
+  addUnwrappedLine();
 }
 
 void UnwrappedLineParser::parsePPPragma() {
diff --git a/clang/lib/Frontend/ASTConsumers.cpp b/clang/lib/Frontend/ASTConsumers.cpp
index ab8a35a189250..67c8761511e0c 100644
--- a/clang/lib/Frontend/ASTConsumers.cpp
+++ b/clang/lib/Frontend/ASTConsumers.cpp
@@ -97,6 +97,7 @@ namespace {
           Out << "Not a DeclContext\n";
       } else if (OutputKind == Print) {
         PrintingPolicy Policy(D->getASTContext().getLangOpts());
+        Policy.IncludeTagDefinition = true;
         D->print(Out, Policy, /*Indentation=*/0, /*PrintInstantiation=*/true);
       } else if (OutputKind != None) {
         D->dump(Out, OutputKind == DumpFull, OutputFormat);
@@ -112,8 +113,10 @@ namespace {
         // FIXME: Support combining -ast-dump-decl-types with -ast-dump-lookups.
         if (auto *VD = dyn_cast<ValueDecl>(InnerD))
           VD->getType().dump(Out, VD->getASTContext());
-        if (auto *TD = dyn_cast<TypeDecl>(InnerD))
-          TD->getTypeForDecl()->dump(Out, TD->getASTContext());
+        if (auto *TD = dyn_cast<TypeDecl>(InnerD)) {
+          const ASTContext &Ctx = TD->getASTContext();
+          Ctx.getTypeDeclType(TD)->dump(Out, Ctx);
+        }
       }
     }
 
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index a4078257a889f..03b08cdabe39e 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -396,7 +396,7 @@ void ASTUnit::CacheCodeCompletionResults() {
 
       // Keep track of the type of this completion in an ASTContext-agnostic
       // way.
-      QualType UsageType = getDeclUsageType(*Ctx, R.Declaration);
+      QualType UsageType = getDeclUsageType(*Ctx, R.Qualifier, R.Declaration);
       if (UsageType.isNull()) {
         CachedResult.TypeClass = STC_Void;
         CachedResult.Type = 0;
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index d64290f5c4673..9f99edadbb70f 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -1485,6 +1485,14 @@ static bool compileModuleAndReadASTImpl(CompilerInstance &ImportingInstance,
     return false;
   }
 
+  // The module is built successfully, we can update its timestamp now.
+  if (ImportingInstance.getPreprocessor()
+          .getHeaderSearchInfo()
+          .getHeaderSearchOpts()
+          .ModulesValidateOncePerBuildSession) {
+    ImportingInstance.getModuleCache().updateModuleTimestamp(ModuleFileName);
+  }
+
   return readASTAfterCompileModule(ImportingInstance, ImportLoc, ModuleNameLoc,
                                    Module, ModuleFileName,
                                    /*OutOfDate=*/nullptr, /*Missing=*/nullptr);
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index 9f77e621a5e68..da96352e1d82c 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1542,6 +1542,17 @@ void CompilerInvocation::setDefaultPointerAuthOptions(
           Discrimination::Constant, InitFiniPointerConstantDiscriminator);
     }
 
+    Opts.BlockInvocationFunctionPointers =
+        PointerAuthSchema(Key::ASIA, true, Discrimination::None);
+    Opts.BlockHelperFunctionPointers =
+        PointerAuthSchema(Key::ASIA, true, Discrimination::None);
+    Opts.BlockByrefHelperFunctionPointers =
+        PointerAuthSchema(Key::ASIA, true, Discrimination::None);
+    if (LangOpts.PointerAuthBlockDescriptorPointers)
+      Opts.BlockDescriptorPointers =
+          PointerAuthSchema(Key::ASDA, true, Discrimination::Constant,
+                            BlockDescriptorConstantDiscriminator);
+
     Opts.ObjCMethodListFunctionPointers =
         PointerAuthSchema(Key::ASIA, true, Discrimination::None);
     Opts.ObjCMethodListPointer =
@@ -3598,6 +3609,8 @@ static void GeneratePointerAuthArgs(const LangOptions &Opts,
     GenerateArg(Consumer, OPT_fptrauth_objc_interface_sel);
   if (Opts.PointerAuthObjcClassROPointers)
     GenerateArg(Consumer, OPT_fptrauth_objc_class_ro);
+  if (Opts.PointerAuthBlockDescriptorPointers)
+    GenerateArg(Consumer, OPT_fptrauth_block_descriptor_pointers);
 }
 
 static void ParsePointerAuthArgs(LangOptions &Opts, ArgList &Args,
@@ -3621,7 +3634,8 @@ static void ParsePointerAuthArgs(LangOptions &Opts, ArgList &Args,
   Opts.PointerAuthELFGOT = Args.hasArg(OPT_fptrauth_elf_got);
   Opts.AArch64JumpTableHardening =
       Args.hasArg(OPT_faarch64_jump_table_hardening);
-
+  Opts.PointerAuthBlockDescriptorPointers =
+      Args.hasArg(OPT_fptrauth_block_descriptor_pointers);
   Opts.PointerAuthObjcIsa = Args.hasArg(OPT_fptrauth_objc_isa);
   Opts.PointerAuthObjcClassROPointers = Args.hasArg(OPT_fptrauth_objc_class_ro);
   Opts.PointerAuthObjcInterfaceSel =
@@ -3936,47 +3950,18 @@ void CompilerInvocationBase::GenerateLangArgs(const LangOptions &Opts,
     GenerateArg(Consumer, OPT_fsanitize_ignorelist_EQ, F);
 
   switch (Opts.getClangABICompat()) {
-  case LangOptions::ClangABI::Ver3_8:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "3.8");
-    break;
-  case LangOptions::ClangABI::Ver4:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "4.0");
-    break;
-  case LangOptions::ClangABI::Ver6:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "6.0");
-    break;
-  case LangOptions::ClangABI::Ver7:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "7.0");
-    break;
-  case LangOptions::ClangABI::Ver9:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "9.0");
+#define ABI_VER_MAJOR_MINOR(Major, Minor)                                      \
+  case LangOptions::ClangABI::Ver##Major##_##Minor:                            \
+    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, #Major "." #Minor);        \
     break;
-  case LangOptions::ClangABI::Ver11:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "11.0");
+#define ABI_VER_MAJOR(Major)                                                   \
+  case LangOptions::ClangABI::Ver##Major:                                      \
+    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, #Major ".0");              \
     break;
-  case LangOptions::ClangABI::Ver12:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "12.0");
-    break;
-  case LangOptions::ClangABI::Ver14:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "14.0");
-    break;
-  case LangOptions::ClangABI::Ver15:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "15.0");
-    break;
-  case LangOptions::ClangABI::Ver17:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "17.0");
-    break;
-  case LangOptions::ClangABI::Ver18:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "18.0");
-    break;
-  case LangOptions::ClangABI::Ver19:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "19.0");
-    break;
-  case LangOptions::ClangABI::Ver20:
-    GenerateArg(Consumer, OPT_fclang_abi_compat_EQ, "20.0");
-    break;
-  case LangOptions::ClangABI::Latest:
+#define ABI_VER_LATEST(Latest)                                                 \
+  case LangOptions::ClangABI::Latest:                                          \
     break;
+#include "clang/Basic/ABIVersions.def"
   }
 
   if (Opts.getSignReturnAddressScope() ==
@@ -4470,7 +4455,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
 
     StringRef Ver = A->getValue();
     std::pair<StringRef, StringRef> VerParts = Ver.split('.');
-    unsigned Major, Minor = 0;
+    int Major, Minor = 0;
 
     // Check the version number is valid: either 3.x (0 <= x <= 9) or
     // y or y.0 (4 <= y <= current version).
@@ -4482,32 +4467,18 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
                    !VerParts.second.getAsInteger(10, Minor)
              : VerParts.first.size() == Ver.size() || VerParts.second == "0")) {
       // Got a valid version number.
-      if (Major == 3 && Minor <= 8)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver3_8);
-      else if (Major <= 4)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver4);
-      else if (Major <= 6)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver6);
-      else if (Major <= 7)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver7);
-      else if (Major <= 9)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver9);
-      else if (Major <= 11)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver11);
-      else if (Major <= 12)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver12);
-      else if (Major <= 14)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver14);
-      else if (Major <= 15)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver15);
-      else if (Major <= 17)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver17);
-      else if (Major <= 18)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver18);
-      else if (Major <= 19)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver19);
-      else if (Major <= 20)
-        Opts.setClangABICompat(LangOptions::ClangABI::Ver20);
+#define ABI_VER_MAJOR_MINOR(Major_, Minor_)                                    \
+  if (std::tuple(Major, Minor) <= std::tuple(Major_, Minor_))                  \
+    Opts.setClangABICompat(LangOptions::ClangABI::Ver##Major_##_##Minor_);     \
+  else
+#define ABI_VER_MAJOR(Major_)                                                  \
+  if (Major <= Major_)                                                         \
+    Opts.setClangABICompat(LangOptions::ClangABI::Ver##Major_);                \
+  else
+#define ABI_VER_LATEST(Latest)                                                 \
+  { /* Equivalent to latest version - do nothing */                            \
+  }
+#include "clang/Basic/ABIVersions.def"
     } else if (Ver != "latest") {
       Diags.Report(diag::err_drv_invalid_value)
           << A->getAsString(Args) << A->getValue();
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 008a35d5265e1..4865c0b889044 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -1519,6 +1519,13 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
   if (TI.getTriple().isOSBinFormatELF())
     Builder.defineMacro("__ELF__");
 
+  if (LangOpts.Sanitize.has(SanitizerKind::Address))
+    Builder.defineMacro("__SANITIZE_ADDRESS__");
+  if (LangOpts.Sanitize.has(SanitizerKind::HWAddress))
+    Builder.defineMacro("__SANITIZE_HWADDRESS__");
+  if (LangOpts.Sanitize.has(SanitizerKind::Thread))
+    Builder.defineMacro("__SANITIZE_THREAD__");
+
   // Target OS macro definitions.
   if (PPOpts.DefineTargetOSMacros) {
     const llvm::Triple &Triple = TI.getTriple();
@@ -1528,6 +1535,9 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
 #undef TARGET_OS
   }
 
+  if (LangOpts.PointerAuthIntrinsics)
+    Builder.defineMacro("__PTRAUTH__");
+
   // Get other target #defines.
   TI.getTargetDefines(LangOpts, Builder);
 }
diff --git a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
index 8f275536b98a6..2b55820b38804 100644
--- a/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteModernObjC.cpp
@@ -852,7 +852,7 @@ RewriteModernObjC::getIvarAccessString(ObjCIvarDecl *D) {
     IvarT = GetGroupRecordTypeForObjCIvarBitfield(D);
 
   if (!IvarT->getAs<TypedefType>() && IvarT->isRecordType()) {
-    RecordDecl *RD = IvarT->castAs<RecordType>()->getDecl();
+    RecordDecl *RD = IvarT->castAs<RecordType>()->getOriginalDecl();
     RD = RD->getDefinition();
     if (RD && !RD->getDeclName().getAsIdentifierInfo()) {
       // decltype(((Foo_IMPL*)0)->bar) *
@@ -865,7 +865,8 @@ RewriteModernObjC::getIvarAccessString(ObjCIvarDecl *D) {
       RecordDecl *RD = RecordDecl::Create(*Context, TagTypeKind::Struct, TUDecl,
                                           SourceLocation(), SourceLocation(),
                                           &Context->Idents.get(RecName));
-      QualType PtrStructIMPL = Context->getPointerType(Context->getTagDeclType(RD));
+      QualType PtrStructIMPL =
+          Context->getPointerType(Context->getCanonicalTagType(RD));
       unsigned UnsignedIntSize =
       static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
       Expr *Zero = IntegerLiteral::Create(*Context,
@@ -2999,7 +3000,7 @@ QualType RewriteModernObjC::getSuperStructType() {
 
     SuperStructDecl->completeDefinition();
   }
-  return Context->getTagDeclType(SuperStructDecl);
+  return Context->getCanonicalTagType(SuperStructDecl);
 }
 
 QualType RewriteModernObjC::getConstantStringStructType() {
@@ -3032,7 +3033,7 @@ QualType RewriteModernObjC::getConstantStringStructType() {
 
     ConstantStringDecl->completeDefinition();
   }
-  return Context->getTagDeclType(ConstantStringDecl);
+  return Context->getCanonicalTagType(ConstantStringDecl);
 }
 
 /// getFunctionSourceLocation - returns start location of a function
@@ -3637,7 +3638,8 @@ bool RewriteModernObjC::RewriteObjCFieldDeclType(QualType &Type,
     return RewriteObjCFieldDeclType(ElemTy, Result);
   }
   else if (Type->isRecordType()) {
-    RecordDecl *RD = Type->castAs<RecordType>()->getDecl();
+    RecordDecl *RD =
+        Type->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->isCompleteDefinition()) {
       if (RD->isStruct())
         Result += "\n\tstruct ";
@@ -3660,7 +3662,8 @@ bool RewriteModernObjC::RewriteObjCFieldDeclType(QualType &Type,
     }
   }
   else if (Type->isEnumeralType()) {
-    EnumDecl *ED = Type->castAs<EnumType>()->getDecl();
+    EnumDecl *ED =
+        Type->castAs<EnumType>()->getOriginalDecl()->getDefinitionOrSelf();
     if (ED->isCompleteDefinition()) {
       Result += "\n\tenum ";
       Result += ED->getName();
@@ -3732,10 +3735,10 @@ void RewriteModernObjC::RewriteLocallyDefinedNamedAggregates(FieldDecl *fieldDec
 
   TagDecl *TD = nullptr;
   if (Type->isRecordType()) {
-    TD = Type->castAs<RecordType>()->getDecl();
+    TD = Type->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
   }
   else if (Type->isEnumeralType()) {
-    TD = Type->castAs<EnumType>()->getDecl();
+    TD = Type->castAs<EnumType>()->getOriginalDecl()->getDefinitionOrSelf();
   }
 
   if (TD) {
@@ -3793,7 +3796,7 @@ QualType RewriteModernObjC::SynthesizeBitfieldGroupStructType(
                                   false, ICIS_NoInit));
   }
   RD->completeDefinition();
-  return Context->getTagDeclType(RD);
+  return Context->getCanonicalTagType(RD);
 }
 
 QualType RewriteModernObjC::GetGroupRecordTypeForObjCIvarBitfield(ObjCIvarDecl *IV) {
@@ -4572,7 +4575,7 @@ Stmt *RewriteModernObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp
   RecordDecl *RD = RecordDecl::Create(*Context, TagTypeKind::Struct, TUDecl,
                                       SourceLocation(), SourceLocation(),
                                       &Context->Idents.get("__block_impl"));
-  QualType PtrBlock = Context->getPointerType(Context->getTagDeclType(RD));
+  QualType PtrBlock = Context->getPointerType(Context->getCanonicalTagType(RD));
 
   // Generate a funky cast.
   SmallVector<QualType, 8> ArgTypes;
@@ -5316,7 +5319,8 @@ Stmt *RewriteModernObjC::SynthBlockInitExpr(BlockExpr *Exp,
           RecordDecl::Create(*Context, TagTypeKind::Struct, TUDecl,
                              SourceLocation(), SourceLocation(), II);
       assert(RD && "SynthBlockInitExpr(): Can't find RecordDecl");
-      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
+      QualType castT =
+          Context->getPointerType(Context->getCanonicalTagType(RD));
 
       FD = SynthBlockInitFunctionDecl(ND->getName());
       Exp = new (Context) DeclRefExpr(*Context, FD, false, FD->getType(),
@@ -5719,7 +5723,10 @@ void RewriteModernObjC::HandleDeclInMainFile(Decl *D) {
           }
         }
       } else if (VD->getType()->isRecordType()) {
-        RecordDecl *RD = VD->getType()->castAs<RecordType>()->getDecl();
+        RecordDecl *RD = VD->getType()
+                             ->castAs<RecordType>()
+                             ->getOriginalDecl()
+                             ->getDefinitionOrSelf();
         if (RD->isCompleteDefinition())
           RewriteRecordBody(RD);
       }
@@ -7460,7 +7467,7 @@ Stmt *RewriteModernObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
         IvarT = GetGroupRecordTypeForObjCIvarBitfield(D);
 
       if (!IvarT->getAs<TypedefType>() && IvarT->isRecordType()) {
-        RecordDecl *RD = IvarT->castAs<RecordType>()->getDecl();
+        RecordDecl *RD = IvarT->castAs<RecordType>()->getOriginalDecl();
         RD = RD->getDefinition();
         if (RD && !RD->getDeclName().getAsIdentifierInfo()) {
           // decltype(((Foo_IMPL*)0)->bar) *
@@ -7473,7 +7480,8 @@ Stmt *RewriteModernObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
           RecordDecl *RD = RecordDecl::Create(
               *Context, TagTypeKind::Struct, TUDecl, SourceLocation(),
               SourceLocation(), &Context->Idents.get(RecName));
-          QualType PtrStructIMPL = Context->getPointerType(Context->getTagDeclType(RD));
+          QualType PtrStructIMPL =
+              Context->getPointerType(Context->getCanonicalTagType(RD));
           unsigned UnsignedIntSize =
             static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
           Expr *Zero = IntegerLiteral::Create(*Context,
diff --git a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
index f49ccf7be68e2..0242fc1f6e827 100644
--- a/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
+++ b/clang/lib/Frontend/Rewrite/RewriteObjC.cpp
@@ -2358,7 +2358,7 @@ void RewriteObjC::SynthMsgSendSuperFunctionDecl() {
   RecordDecl *RD = RecordDecl::Create(*Context, TagTypeKind::Struct, TUDecl,
                                       SourceLocation(), SourceLocation(),
                                       &Context->Idents.get("objc_super"));
-  QualType argT = Context->getPointerType(Context->getTagDeclType(RD));
+  QualType argT = Context->getPointerType(Context->getCanonicalTagType(RD));
   assert(!argT.isNull() && "Can't build 'struct objc_super *' type");
   ArgTys.push_back(argT);
   argT = Context->getObjCSelType();
@@ -2401,7 +2401,7 @@ void RewriteObjC::SynthMsgSendSuperStretFunctionDecl() {
   RecordDecl *RD = RecordDecl::Create(*Context, TagTypeKind::Struct, TUDecl,
                                       SourceLocation(), SourceLocation(),
                                       &Context->Idents.get("objc_super"));
-  QualType argT = Context->getPointerType(Context->getTagDeclType(RD));
+  QualType argT = Context->getPointerType(Context->getCanonicalTagType(RD));
   assert(!argT.isNull() && "Can't build 'struct objc_super *' type");
   ArgTys.push_back(argT);
   argT = Context->getObjCSelType();
@@ -2552,7 +2552,7 @@ QualType RewriteObjC::getSuperStructType() {
 
     SuperStructDecl->completeDefinition();
   }
-  return Context->getTagDeclType(SuperStructDecl);
+  return Context->getCanonicalTagType(SuperStructDecl);
 }
 
 QualType RewriteObjC::getConstantStringStructType() {
@@ -2585,7 +2585,7 @@ QualType RewriteObjC::getConstantStringStructType() {
 
     ConstantStringDecl->completeDefinition();
   }
-  return Context->getTagDeclType(ConstantStringDecl);
+  return Context->getCanonicalTagType(ConstantStringDecl);
 }
 
 CallExpr *RewriteObjC::SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavor,
@@ -3750,7 +3750,7 @@ Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
   RecordDecl *RD = RecordDecl::Create(*Context, TagTypeKind::Struct, TUDecl,
                                       SourceLocation(), SourceLocation(),
                                       &Context->Idents.get("__block_impl"));
-  QualType PtrBlock = Context->getPointerType(Context->getTagDeclType(RD));
+  QualType PtrBlock = Context->getPointerType(Context->getCanonicalTagType(RD));
 
   // Generate a funky cast.
   SmallVector<QualType, 8> ArgTypes;
@@ -4468,7 +4468,8 @@ Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
           RecordDecl::Create(*Context, TagTypeKind::Struct, TUDecl,
                              SourceLocation(), SourceLocation(), II);
       assert(RD && "SynthBlockInitExpr(): Can't find RecordDecl");
-      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
+      QualType castT =
+          Context->getPointerType(Context->getCanonicalTagType(RD));
 
       FD = SynthBlockInitFunctionDecl((*I)->getName());
       Exp = new (Context) DeclRefExpr(*Context, FD, false, FD->getType(),
@@ -4834,7 +4835,10 @@ void RewriteObjC::HandleDeclInMainFile(Decl *D) {
           }
         }
       } else if (VD->getType()->isRecordType()) {
-        RecordDecl *RD = VD->getType()->castAs<RecordType>()->getDecl();
+        RecordDecl *RD = VD->getType()
+                             ->castAs<RecordType>()
+                             ->getOriginalDecl()
+                             ->getDefinitionOrSelf();
         if (RD->isCompleteDefinition())
           RewriteRecordBody(RD);
       }
@@ -5804,7 +5808,8 @@ Stmt *RewriteObjCFragileABI::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
           RecordDecl::Create(*Context, TagTypeKind::Struct, TUDecl,
                              SourceLocation(), SourceLocation(), II);
       assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
-      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
+      QualType castT =
+          Context->getPointerType(Context->getCanonicalTagType(RD));
       CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
                                                     CK_BitCast,
                                                     IV->getBase());
@@ -5845,7 +5850,8 @@ Stmt *RewriteObjCFragileABI::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
           RecordDecl::Create(*Context, TagTypeKind::Struct, TUDecl,
                              SourceLocation(), SourceLocation(), II);
       assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
-      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
+      QualType castT =
+          Context->getPointerType(Context->getCanonicalTagType(RD));
       CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
                                                     CK_BitCast,
                                                     IV->getBase());
diff --git a/clang/lib/Frontend/TextDiagnostic.cpp b/clang/lib/Frontend/TextDiagnostic.cpp
index ccdd59da89bd1..58885712fbdcc 100644
--- a/clang/lib/Frontend/TextDiagnostic.cpp
+++ b/clang/lib/Frontend/TextDiagnostic.cpp
@@ -1095,6 +1095,8 @@ prepareAndFilterRanges(const SmallVectorImpl<CharSourceRange> &Ranges,
 
     unsigned StartColumn = SM.getExpansionColumnNumber(Begin);
     unsigned EndColumn = SM.getExpansionColumnNumber(End);
+    assert(StartColumn && "StartColumn must be valid, 0 is invalid");
+    assert(EndColumn && "EndColumn must be valid, 0 is invalid");
     if (R.isTokenRange())
       EndColumn += Lexer::MeasureTokenLength(End, SM, LangOpts);
 
diff --git a/clang/lib/Headers/avx10_2_512bf16intrin.h b/clang/lib/Headers/avx10_2_512bf16intrin.h
index 75290d22ef259..95e9bd7a36f9b 100644
--- a/clang/lib/Headers/avx10_2_512bf16intrin.h
+++ b/clang/lib/Headers/avx10_2_512bf16intrin.h
@@ -441,8 +441,8 @@ _mm512_maskz_sqrt_pbh(__mmask32 __U, __m512bh __A) {
 
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
 _mm512_fmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
-  return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, (__v32bf)__B,
-                                                (__v32bf)__C);
+  return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, (__v32bf)__B,
+                                             (__v32bf)__C);
 }
 
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
@@ -469,8 +469,8 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmadd_pbh(
 
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
 _mm512_fmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
-  return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, (__v32bf)__B,
-                                                -(__v32bf)__C);
+  return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, (__v32bf)__B,
+                                             -(__v32bf)__C);
 }
 
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
@@ -497,8 +497,8 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmsub_pbh(
 
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
 _mm512_fnmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
-  return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, -(__v32bf)__B,
-                                                (__v32bf)__C);
+  return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, -(__v32bf)__B,
+                                             (__v32bf)__C);
 }
 
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmadd_pbh(
@@ -527,8 +527,8 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fnmadd_pbh(
 
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512
 _mm512_fnmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
-  return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, -(__v32bf)__B,
-                                                -(__v32bf)__C);
+  return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, -(__v32bf)__B,
+                                             -(__v32bf)__C);
 }
 
 static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmsub_pbh(
diff --git a/clang/lib/Headers/avx10_2bf16intrin.h b/clang/lib/Headers/avx10_2bf16intrin.h
index 66797ae00fe4f..0c7f381f04fa5 100644
--- a/clang/lib/Headers/avx10_2bf16intrin.h
+++ b/clang/lib/Headers/avx10_2bf16intrin.h
@@ -852,8 +852,8 @@ _mm_maskz_sqrt_pbh(__mmask8 __U, __m128bh __A) {
 
 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
 _mm256_fmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
-  return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, (__v16bf)__B,
-                                                (__v16bf)__C);
+  return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, (__v16bf)__B,
+                                             (__v16bf)__C);
 }
 
 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
@@ -880,8 +880,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmadd_pbh(
 
 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
 _mm256_fmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
-  return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, (__v16bf)__B,
-                                                -(__v16bf)__C);
+  return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, (__v16bf)__B,
+                                             -(__v16bf)__C);
 }
 
 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
@@ -908,8 +908,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmsub_pbh(
 
 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
 _mm256_fnmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
-  return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, -(__v16bf)__B,
-                                                (__v16bf)__C);
+  return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, -(__v16bf)__B,
+                                             (__v16bf)__C);
 }
 
 static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmadd_pbh(
@@ -938,8 +938,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmadd_pbh(
 
 static __inline__ __m256bh __DEFAULT_FN_ATTRS256
 _mm256_fnmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
-  return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, -(__v16bf)__B,
-                                                -(__v16bf)__C);
+  return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, -(__v16bf)__B,
+                                             -(__v16bf)__C);
 }
 
 static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmsub_pbh(
@@ -969,8 +969,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmsub_pbh(
 static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmadd_pbh(__m128bh __A,
                                                                __m128bh __B,
                                                                __m128bh __C) {
-  return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, (__v8bf)__B,
-                                                (__v8bf)__C);
+  return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, (__v8bf)__B,
+                                             (__v8bf)__C);
 }
 
 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
@@ -997,8 +997,8 @@ _mm_maskz_fmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
 static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmsub_pbh(__m128bh __A,
                                                                __m128bh __B,
                                                                __m128bh __C) {
-  return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, (__v8bf)__B,
-                                                -(__v8bf)__C);
+  return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, (__v8bf)__B,
+                                             -(__v8bf)__C);
 }
 
 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
@@ -1025,8 +1025,8 @@ _mm_maskz_fmsub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
 static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmadd_pbh(__m128bh __A,
                                                                 __m128bh __B,
                                                                 __m128bh __C) {
-  return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, -(__v8bf)__B,
-                                                (__v8bf)__C);
+  return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, -(__v8bf)__B,
+                                             (__v8bf)__C);
 }
 
 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
@@ -1053,8 +1053,8 @@ _mm_maskz_fnmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
 static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmsub_pbh(__m128bh __A,
                                                                 __m128bh __B,
                                                                 __m128bh __C) {
-  return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, -(__v8bf)__B,
-                                                -(__v8bf)__C);
+  return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, -(__v8bf)__B,
+                                             -(__v8bf)__C);
 }
 
 static __inline__ __m128bh __DEFAULT_FN_ATTRS128
diff --git a/clang/lib/Headers/avx2intrin.h b/clang/lib/Headers/avx2intrin.h
index dc9fc07314323..ce5b2b7544d8c 100644
--- a/clang/lib/Headers/avx2intrin.h
+++ b/clang/lib/Headers/avx2intrin.h
@@ -31,6 +31,14 @@
                  __min_vector_width__(128)))
 #endif
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#else
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#endif
+
 /* SSE4 Multiple Packed Sums of Absolute Difference.  */
 /// Computes sixteen sum of absolute difference (SAD) operations on sets of
 ///    four unsigned 8-bit integers from the 256-bit integer vectors \a X and
@@ -359,9 +367,8 @@ _mm256_add_epi64(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit integer vector containing one of the source operands.
 /// \returns A 256-bit integer vector containing the sums.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_adds_epi8(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_adds_epi8(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_elementwise_add_sat((__v32qs)__a, (__v32qs)__b);
 }
 
@@ -377,9 +384,8 @@ _mm256_adds_epi8(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit vector of [16 x i16] containing one of the source operands.
 /// \returns A 256-bit vector of [16 x i16] containing the sums.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_adds_epi16(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_adds_epi16(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_elementwise_add_sat((__v16hi)__a, (__v16hi)__b);
 }
 
@@ -396,9 +402,8 @@ _mm256_adds_epi16(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit integer vector containing one of the source operands.
 /// \returns A 256-bit integer vector containing the sums.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_adds_epu8(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_adds_epu8(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_elementwise_add_sat((__v32qu)__a, (__v32qu)__b);
 }
 
@@ -414,9 +419,8 @@ _mm256_adds_epu8(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit vector of [16 x i16] containing one of the source operands.
 /// \returns A 256-bit vector of [16 x i16] containing the sums.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_adds_epu16(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_adds_epu16(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_elementwise_add_sat((__v16hu)__a, (__v16hu)__b);
 }
 
@@ -460,7 +464,7 @@ _mm256_adds_epu16(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit integer vector.
 /// \returns A 256-bit integer vector containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_and_si256(__m256i __a, __m256i __b)
 {
   return (__m256i)((__v4du)__a & (__v4du)__b);
@@ -478,7 +482,7 @@ _mm256_and_si256(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit integer vector.
 /// \returns A 256-bit integer vector containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_andnot_si256(__m256i __a, __m256i __b)
 {
   return (__m256i)(~(__v4du)__a & (__v4du)__b);
@@ -1363,9 +1367,8 @@ _mm256_movemask_epi8(__m256i __a)
 ///    A 128-bit integer vector containing the source bytes.
 /// \returns A 256-bit vector of [16 x i16] containing the sign-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepi8_epi16(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepi8_epi16(__m128i __V) {
   /* This function always performs a signed extension, but __v16qi is a char
      which may be signed or unsigned, so use __v16qs. */
   return (__m256i)__builtin_convertvector((__v16qs)__V, __v16hi);
@@ -1391,9 +1394,8 @@ _mm256_cvtepi8_epi16(__m128i __V)
 ///    A 128-bit integer vector containing the source bytes.
 /// \returns A 256-bit vector of [8 x i32] containing the sign-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepi8_epi32(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepi8_epi32(__m128i __V) {
   /* This function always performs a signed extension, but __v16qi is a char
      which may be signed or unsigned, so use __v16qs. */
   return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si);
@@ -1418,9 +1420,8 @@ _mm256_cvtepi8_epi32(__m128i __V)
 ///    A 128-bit integer vector containing the source bytes.
 /// \returns A 256-bit vector of [4 x i64] containing the sign-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepi8_epi64(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepi8_epi64(__m128i __V) {
   /* This function always performs a signed extension, but __v16qi is a char
      which may be signed or unsigned, so use __v16qs. */
   return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3), __v4di);
@@ -1446,9 +1447,8 @@ _mm256_cvtepi8_epi64(__m128i __V)
 ///    A 128-bit vector of [8 x i16] containing the source values.
 /// \returns A 256-bit vector of [8 x i32] containing the sign-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepi16_epi32(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepi16_epi32(__m128i __V) {
   return (__m256i)__builtin_convertvector((__v8hi)__V, __v8si);
 }
 
@@ -1471,9 +1471,8 @@ _mm256_cvtepi16_epi32(__m128i __V)
 ///    A 128-bit vector of [8 x i16] containing the source values.
 /// \returns A 256-bit vector of [4 x i64] containing the sign-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepi16_epi64(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepi16_epi64(__m128i __V) {
   return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4di);
 }
 
@@ -1496,9 +1495,8 @@ _mm256_cvtepi16_epi64(__m128i __V)
 ///    A 128-bit vector of [4 x i32] containing the source values.
 /// \returns A 256-bit vector of [4 x i64] containing the sign-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepi32_epi64(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepi32_epi64(__m128i __V) {
   return (__m256i)__builtin_convertvector((__v4si)__V, __v4di);
 }
 
@@ -1522,9 +1520,8 @@ _mm256_cvtepi32_epi64(__m128i __V)
 ///    A 128-bit integer vector containing the source bytes.
 /// \returns A 256-bit vector of [16 x i16] containing the zero-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepu8_epi16(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepu8_epi16(__m128i __V) {
   return (__m256i)__builtin_convertvector((__v16qu)__V, __v16hi);
 }
 
@@ -1548,9 +1545,8 @@ _mm256_cvtepu8_epi16(__m128i __V)
 ///    A 128-bit integer vector containing the source bytes.
 /// \returns A 256-bit vector of [8 x i32] containing the zero-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepu8_epi32(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepu8_epi32(__m128i __V) {
   return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8si);
 }
 
@@ -1573,9 +1569,8 @@ _mm256_cvtepu8_epi32(__m128i __V)
 ///    A 128-bit integer vector containing the source bytes.
 /// \returns A 256-bit vector of [4 x i64] containing the zero-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepu8_epi64(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepu8_epi64(__m128i __V) {
   return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3), __v4di);
 }
 
@@ -1599,9 +1594,8 @@ _mm256_cvtepu8_epi64(__m128i __V)
 ///    A 128-bit vector of [8 x i16] containing the source values.
 /// \returns A 256-bit vector of [8 x i32] containing the zero-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepu16_epi32(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepu16_epi32(__m128i __V) {
   return (__m256i)__builtin_convertvector((__v8hu)__V, __v8si);
 }
 
@@ -1624,9 +1618,8 @@ _mm256_cvtepu16_epi32(__m128i __V)
 ///    A 128-bit vector of [8 x i16] containing the source values.
 /// \returns A 256-bit vector of [4 x i64] containing the zero-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepu16_epi64(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepu16_epi64(__m128i __V) {
   return (__m256i)__builtin_convertvector(__builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1, 2, 3), __v4di);
 }
 
@@ -1649,9 +1642,8 @@ _mm256_cvtepu16_epi64(__m128i __V)
 ///    A 128-bit vector of [4 x i32] containing the source values.
 /// \returns A 256-bit vector of [4 x i64] containing the zero-extended
 ///    values.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_cvtepu32_epi64(__m128i __V)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_cvtepu32_epi64(__m128i __V) {
   return (__m256i)__builtin_convertvector((__v4su)__V, __v4di);
 }
 
@@ -1675,9 +1667,8 @@ _mm256_cvtepu32_epi64(__m128i __V)
 /// \param __b
 ///    A 256-bit vector of [8 x i32] containing one of the source operands.
 /// \returns A 256-bit vector of [4 x i64] containing the products.
-static __inline__  __m256i __DEFAULT_FN_ATTRS256
-_mm256_mul_epi32(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_mul_epi32(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_ia32_pmuldq256((__v8si)__a, (__v8si)__b);
 }
 
@@ -1721,10 +1712,10 @@ _mm256_mulhrs_epi16(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit vector of [16 x i16] containing one of the source operands.
 /// \returns A 256-bit vector of [16 x i16] containing the products.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_mulhi_epu16(__m256i __a, __m256i __b)
 {
-  return (__m256i)__builtin_ia32_pmulhuw256((__v16hi)__a, (__v16hi)__b);
+  return (__m256i)__builtin_ia32_pmulhuw256((__v16hu)__a, (__v16hu)__b);
 }
 
 /// Multiplies signed 16-bit integer elements of two 256-bit vectors of
@@ -1740,7 +1731,7 @@ _mm256_mulhi_epu16(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit vector of [16 x i16] containing one of the source operands.
 /// \returns A 256-bit vector of [16 x i16] containing the products.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_mulhi_epi16(__m256i __a, __m256i __b)
 {
   return (__m256i)__builtin_ia32_pmulhw256((__v16hi)__a, (__v16hi)__b);
@@ -1759,7 +1750,7 @@ _mm256_mulhi_epi16(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit vector of [16 x i16] containing one of the source operands.
 /// \returns A 256-bit vector of [16 x i16] containing the products.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_mullo_epi16(__m256i __a, __m256i __b)
 {
   return (__m256i)((__v16hu)__a * (__v16hu)__b);
@@ -1804,9 +1795,8 @@ _mm256_mullo_epi32 (__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit vector of [8 x i32] containing one of the source operands.
 /// \returns A 256-bit vector of [4 x i64] containing the products.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_mul_epu32(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_mul_epu32(__m256i __a, __m256i __b) {
   return __builtin_ia32_pmuludq256((__v8si)__a, (__v8si)__b);
 }
 
@@ -1822,7 +1812,7 @@ _mm256_mul_epu32(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit integer vector.
 /// \returns A 256-bit integer vector containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_or_si256(__m256i __a, __m256i __b)
 {
   return (__m256i)((__v4du)__a | (__v4du)__b);
@@ -2611,9 +2601,8 @@ _mm256_sub_epi64(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit integer vector containing the subtrahends.
 /// \returns A 256-bit integer vector containing the differences.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_subs_epi8(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_subs_epi8(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_elementwise_sub_sat((__v32qs)__a, (__v32qs)__b);
 }
 
@@ -2637,9 +2626,8 @@ _mm256_subs_epi8(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit vector of [16 x i16] containing the subtrahends.
 /// \returns A 256-bit vector of [16 x i16] containing the differences.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_subs_epi16(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_subs_epi16(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_elementwise_sub_sat((__v16hi)__a, (__v16hi)__b);
 }
 
@@ -2664,9 +2652,8 @@ _mm256_subs_epi16(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit integer vector containing the subtrahends.
 /// \returns A 256-bit integer vector containing the differences.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_subs_epu8(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_subs_epu8(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_elementwise_sub_sat((__v32qu)__a, (__v32qu)__b);
 }
 
@@ -2690,9 +2677,8 @@ _mm256_subs_epu8(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit vector of [16 x i16] containing the subtrahends.
 /// \returns A 256-bit vector of [16 x i16] containing the differences.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_subs_epu16(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_subs_epu16(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_elementwise_sub_sat((__v16hu)__a, (__v16hu)__b);
 }
 
@@ -2724,9 +2710,8 @@ _mm256_subs_epu16(__m256i __a, __m256i __b)
 ///    A 256-bit integer vector used as the source for the odd-numbered bytes
 ///    of the result.
 /// \returns A 256-bit integer vector containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_unpackhi_epi8(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_unpackhi_epi8(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 8, 32+8, 9, 32+9, 10, 32+10, 11, 32+11, 12, 32+12, 13, 32+13, 14, 32+14, 15, 32+15, 24, 32+24, 25, 32+25, 26, 32+26, 27, 32+27, 28, 32+28, 29, 32+29, 30, 32+30, 31, 32+31);
 }
 
@@ -2759,9 +2744,8 @@ _mm256_unpackhi_epi8(__m256i __a, __m256i __b)
 ///    A 256-bit vector of [16 x i16] used as the source for the odd-numbered
 ///    elements of the result.
 /// \returns A 256-bit vector of [16 x i16] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_unpackhi_epi16(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_unpackhi_epi16(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 4, 16+4, 5, 16+5, 6, 16+6, 7, 16+7, 12, 16+12, 13, 16+13, 14, 16+14, 15, 16+15);
 }
 
@@ -2793,9 +2777,8 @@ _mm256_unpackhi_epi16(__m256i __a, __m256i __b)
 ///    A 256-bit vector of [8 x i32] used as the source for the odd-numbered
 ///    elements of the result.
 /// \returns A 256-bit vector of [8 x i32] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_unpackhi_epi32(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_unpackhi_epi32(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 2, 8+2, 3, 8+3, 6, 8+6, 7, 8+7);
 }
 
@@ -2823,9 +2806,8 @@ _mm256_unpackhi_epi32(__m256i __a, __m256i __b)
 ///    A 256-bit vector of [4 x i64] used as the source for the odd-numbered
 ///    elements of the result.
 /// \returns A 256-bit vector of [4 x i64] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_unpackhi_epi64(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_unpackhi_epi64(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 1, 4+1, 3, 4+3);
 }
 
@@ -2857,9 +2839,8 @@ _mm256_unpackhi_epi64(__m256i __a, __m256i __b)
 ///    A 256-bit integer vector used as the source for the odd-numbered bytes
 ///    of the result.
 /// \returns A 256-bit integer vector containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_unpacklo_epi8(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_unpacklo_epi8(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_shufflevector((__v32qi)__a, (__v32qi)__b, 0, 32+0, 1, 32+1, 2, 32+2, 3, 32+3, 4, 32+4, 5, 32+5, 6, 32+6, 7, 32+7, 16, 32+16, 17, 32+17, 18, 32+18, 19, 32+19, 20, 32+20, 21, 32+21, 22, 32+22, 23, 32+23);
 }
 
@@ -2892,9 +2873,8 @@ _mm256_unpacklo_epi8(__m256i __a, __m256i __b)
 ///    A 256-bit vector of [16 x i16] used as the source for the odd-numbered
 ///    elements of the result.
 /// \returns A 256-bit vector of [16 x i16] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_unpacklo_epi16(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_unpacklo_epi16(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_shufflevector((__v16hi)__a, (__v16hi)__b, 0, 16+0, 1, 16+1, 2, 16+2, 3, 16+3, 8, 16+8, 9, 16+9, 10, 16+10, 11, 16+11);
 }
 
@@ -2926,9 +2906,8 @@ _mm256_unpacklo_epi16(__m256i __a, __m256i __b)
 ///    A 256-bit vector of [8 x i32] used as the source for the odd-numbered
 ///    elements of the result.
 /// \returns A 256-bit vector of [8 x i32] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_unpacklo_epi32(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_unpacklo_epi32(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_shufflevector((__v8si)__a, (__v8si)__b, 0, 8+0, 1, 8+1, 4, 8+4, 5, 8+5);
 }
 
@@ -2956,9 +2935,8 @@ _mm256_unpacklo_epi32(__m256i __a, __m256i __b)
 ///    A 256-bit vector of [4 x i64] used as the source for the odd-numbered
 ///    elements of the result.
 /// \returns A 256-bit vector of [4 x i64] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_unpacklo_epi64(__m256i __a, __m256i __b)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_unpacklo_epi64(__m256i __a, __m256i __b) {
   return (__m256i)__builtin_shufflevector((__v4di)__a, (__v4di)__b, 0, 4+0, 2, 4+2);
 }
 
@@ -2974,7 +2952,7 @@ _mm256_unpacklo_epi64(__m256i __a, __m256i __b)
 /// \param __b
 ///    A 256-bit integer vector.
 /// \returns A 256-bit integer vector containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_xor_si256(__m256i __a, __m256i __b)
 {
   return (__m256i)((__v4du)__a ^ (__v4du)__b);
@@ -3009,9 +2987,8 @@ _mm256_stream_load_si256(const void *__V)
 /// \param __X
 ///    A 128-bit vector of [4 x float] whose low element will be broadcast.
 /// \returns A 128-bit vector of [4 x float] containing the result.
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
-_mm_broadcastss_ps(__m128 __X)
-{
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_broadcastss_ps(__m128 __X) {
   return (__m128)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0);
 }
 
@@ -3026,9 +3003,8 @@ _mm_broadcastss_ps(__m128 __X)
 /// \param __a
 ///    A 128-bit vector of [2 x double] whose low element will be broadcast.
 /// \returns A 128-bit vector of [2 x double] containing the result.
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
-_mm_broadcastsd_pd(__m128d __a)
-{
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_broadcastsd_pd(__m128d __a) {
   return __builtin_shufflevector((__v2df)__a, (__v2df)__a, 0, 0);
 }
 
@@ -3043,9 +3019,8 @@ _mm_broadcastsd_pd(__m128d __a)
 /// \param __X
 ///    A 128-bit vector of [4 x float] whose low element will be broadcast.
 /// \returns A 256-bit vector of [8 x float] containing the result.
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
-_mm256_broadcastss_ps(__m128 __X)
-{
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcastss_ps(__m128 __X) {
   return (__m256)__builtin_shufflevector((__v4sf)__X, (__v4sf)__X, 0, 0, 0, 0, 0, 0, 0, 0);
 }
 
@@ -3060,9 +3035,8 @@ _mm256_broadcastss_ps(__m128 __X)
 /// \param __X
 ///    A 128-bit vector of [2 x double] whose low element will be broadcast.
 /// \returns A 256-bit vector of [4 x double] containing the result.
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
-_mm256_broadcastsd_pd(__m128d __X)
-{
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcastsd_pd(__m128d __X) {
   return (__m256d)__builtin_shufflevector((__v2df)__X, (__v2df)__X, 0, 0, 0, 0);
 }
 
@@ -3076,9 +3050,8 @@ _mm256_broadcastsd_pd(__m128d __X)
 /// \param __X
 ///    A 128-bit integer vector to be broadcast.
 /// \returns A 256-bit integer vector containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_broadcastsi128_si256(__m128i __X)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcastsi128_si256(__m128i __X) {
   return (__m256i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 1, 0, 1);
 }
 
@@ -3168,9 +3141,8 @@ _mm256_broadcastsi128_si256(__m128i __X)
 /// \param __X
 ///    A 128-bit integer vector whose low byte will be broadcast.
 /// \returns A 256-bit integer vector containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_broadcastb_epi8(__m128i __X)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcastb_epi8(__m128i __X) {
   return (__m256i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 }
 
@@ -3184,9 +3156,8 @@ _mm256_broadcastb_epi8(__m128i __X)
 /// \param __X
 ///    A 128-bit vector of [8 x i16] whose low element will be broadcast.
 /// \returns A 256-bit vector of [16 x i16] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_broadcastw_epi16(__m128i __X)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcastw_epi16(__m128i __X) {
   return (__m256i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 }
 
@@ -3200,9 +3171,8 @@ _mm256_broadcastw_epi16(__m128i __X)
 /// \param __X
 ///    A 128-bit vector of [4 x i32] whose low element will be broadcast.
 /// \returns A 256-bit vector of [8 x i32] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_broadcastd_epi32(__m128i __X)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcastd_epi32(__m128i __X) {
   return (__m256i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0, 0, 0, 0, 0);
 }
 
@@ -3216,9 +3186,8 @@ _mm256_broadcastd_epi32(__m128i __X)
 /// \param __X
 ///    A 128-bit vector of [2 x i64] whose low element will be broadcast.
 /// \returns A 256-bit vector of [4 x i64] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_broadcastq_epi64(__m128i __X)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcastq_epi64(__m128i __X) {
   return (__m256i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0, 0, 0);
 }
 
@@ -3232,9 +3201,8 @@ _mm256_broadcastq_epi64(__m128i __X)
 /// \param __X
 ///    A 128-bit integer vector whose low byte will be broadcast.
 /// \returns A 128-bit integer vector containing the result.
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_broadcastb_epi8(__m128i __X)
-{
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_broadcastb_epi8(__m128i __X) {
   return (__m128i)__builtin_shufflevector((__v16qi)__X, (__v16qi)__X, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 }
 
@@ -3248,9 +3216,8 @@ _mm_broadcastb_epi8(__m128i __X)
 /// \param __X
 ///    A 128-bit vector of [8 x i16] whose low element will be broadcast.
 /// \returns A 128-bit vector of [8 x i16] containing the result.
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_broadcastw_epi16(__m128i __X)
-{
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_broadcastw_epi16(__m128i __X) {
   return (__m128i)__builtin_shufflevector((__v8hi)__X, (__v8hi)__X, 0, 0, 0, 0, 0, 0, 0, 0);
 }
 
@@ -3264,9 +3231,8 @@ _mm_broadcastw_epi16(__m128i __X)
 /// \param __X
 ///    A 128-bit vector of [4 x i32] whose low element will be broadcast.
 /// \returns A 128-bit vector of [4 x i32] containing the result.
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_broadcastd_epi32(__m128i __X)
-{
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_broadcastd_epi32(__m128i __X) {
   return (__m128i)__builtin_shufflevector((__v4si)__X, (__v4si)__X, 0, 0, 0, 0);
 }
 
@@ -3280,9 +3246,8 @@ _mm_broadcastd_epi32(__m128i __X)
 /// \param __X
 ///    A 128-bit vector of [2 x i64] whose low element will be broadcast.
 /// \returns A 128-bit vector of [2 x i64] containing the result.
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_broadcastq_epi64(__m128i __X)
-{
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_broadcastq_epi64(__m128i __X) {
   return (__m128i)__builtin_shufflevector((__v2di)__X, (__v2di)__X, 0, 0);
 }
 
@@ -3756,7 +3721,7 @@ _mm_maskstore_epi64(long long *__X, __m128i __M, __m128i __Y)
 ///    A 256-bit vector of [8 x i32] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 256-bit vector of [8 x i32] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_sllv_epi32(__m256i __X, __m256i __Y)
 {
   return (__m256i)__builtin_ia32_psllv8si((__v8si)__X, (__v8si)__Y);
@@ -3778,7 +3743,7 @@ _mm256_sllv_epi32(__m256i __X, __m256i __Y)
 ///    A 128-bit vector of [4 x i32] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 128-bit vector of [4 x i32] containing the result.
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_sllv_epi32(__m128i __X, __m128i __Y)
 {
   return (__m128i)__builtin_ia32_psllv4si((__v4si)__X, (__v4si)__Y);
@@ -3800,7 +3765,7 @@ _mm_sllv_epi32(__m128i __X, __m128i __Y)
 ///    A 256-bit vector of [4 x i64] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 256-bit vector of [4 x i64] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_sllv_epi64(__m256i __X, __m256i __Y)
 {
   return (__m256i)__builtin_ia32_psllv4di((__v4di)__X, (__v4di)__Y);
@@ -3822,7 +3787,7 @@ _mm256_sllv_epi64(__m256i __X, __m256i __Y)
 ///    A 128-bit vector of [2 x i64] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 128-bit vector of [2 x i64] containing the result.
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_sllv_epi64(__m128i __X, __m128i __Y)
 {
   return (__m128i)__builtin_ia32_psllv2di((__v2di)__X, (__v2di)__Y);
@@ -3845,7 +3810,7 @@ _mm_sllv_epi64(__m128i __X, __m128i __Y)
 ///    A 256-bit vector of [8 x i32] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 256-bit vector of [8 x i32] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_srav_epi32(__m256i __X, __m256i __Y)
 {
   return (__m256i)__builtin_ia32_psrav8si((__v8si)__X, (__v8si)__Y);
@@ -3868,7 +3833,7 @@ _mm256_srav_epi32(__m256i __X, __m256i __Y)
 ///    A 128-bit vector of [4 x i32] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 128-bit vector of [4 x i32] containing the result.
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_srav_epi32(__m128i __X, __m128i __Y)
 {
   return (__m128i)__builtin_ia32_psrav4si((__v4si)__X, (__v4si)__Y);
@@ -3890,7 +3855,7 @@ _mm_srav_epi32(__m128i __X, __m128i __Y)
 ///    A 256-bit vector of [8 x i32] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 256-bit vector of [8 x i32] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_srlv_epi32(__m256i __X, __m256i __Y)
 {
   return (__m256i)__builtin_ia32_psrlv8si((__v8si)__X, (__v8si)__Y);
@@ -3912,7 +3877,7 @@ _mm256_srlv_epi32(__m256i __X, __m256i __Y)
 ///    A 128-bit vector of [4 x i32] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 128-bit vector of [4 x i32] containing the result.
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_srlv_epi32(__m128i __X, __m128i __Y)
 {
   return (__m128i)__builtin_ia32_psrlv4si((__v4si)__X, (__v4si)__Y);
@@ -3934,7 +3899,7 @@ _mm_srlv_epi32(__m128i __X, __m128i __Y)
 ///    A 256-bit vector of [4 x i64] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 256-bit vector of [4 x i64] containing the result.
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_srlv_epi64(__m256i __X, __m256i __Y)
 {
   return (__m256i)__builtin_ia32_psrlv4di((__v4di)__X, (__v4di)__Y);
@@ -3956,7 +3921,7 @@ _mm256_srlv_epi64(__m256i __X, __m256i __Y)
 ///    A 128-bit vector of [2 x i64] containing the unsigned shift counts (in
 ///    bits).
 /// \returns A 128-bit vector of [2 x i64] containing the result.
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_srlv_epi64(__m128i __X, __m128i __Y)
 {
   return (__m128i)__builtin_ia32_psrlv2di((__v2di)__X, (__v2di)__Y);
@@ -5289,5 +5254,7 @@ _mm_srlv_epi64(__m128i __X, __m128i __Y)
 
 #undef __DEFAULT_FN_ATTRS256
 #undef __DEFAULT_FN_ATTRS128
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
 
 #endif /* __AVX2INTRIN_H */
diff --git a/clang/lib/Headers/avx512bitalgintrin.h b/clang/lib/Headers/avx512bitalgintrin.h
index 3c446b34e7885..5cc32077c2c04 100644
--- a/clang/lib/Headers/avx512bitalgintrin.h
+++ b/clang/lib/Headers/avx512bitalgintrin.h
@@ -20,48 +20,42 @@
                  __target__("avx512bitalg,evex512"),                           \
                  __min_vector_width__(512)))
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS
-_mm512_popcnt_epi16(__m512i __A)
-{
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
+#else
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#endif
+
+static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm512_popcnt_epi16(__m512i __A) {
   return (__m512i)__builtin_elementwise_popcount((__v32hu)__A);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS
-_mm512_mask_popcnt_epi16(__m512i __A, __mmask32 __U, __m512i __B)
-{
-  return (__m512i) __builtin_ia32_selectw_512((__mmask32) __U,
-              (__v32hi) _mm512_popcnt_epi16(__B),
-              (__v32hi) __A);
+static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm512_mask_popcnt_epi16(__m512i __A, __mmask32 __U, __m512i __B) {
+  return (__m512i)__builtin_ia32_selectw_512(
+      (__mmask32)__U, (__v32hi)_mm512_popcnt_epi16(__B), (__v32hi)__A);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS
-_mm512_maskz_popcnt_epi16(__mmask32 __U, __m512i __B)
-{
-  return _mm512_mask_popcnt_epi16((__m512i) _mm512_setzero_si512(),
-              __U,
-              __B);
+static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm512_maskz_popcnt_epi16(__mmask32 __U, __m512i __B) {
+  return _mm512_mask_popcnt_epi16((__m512i)_mm512_setzero_si512(), __U, __B);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS
-_mm512_popcnt_epi8(__m512i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm512_popcnt_epi8(__m512i __A) {
   return (__m512i)__builtin_elementwise_popcount((__v64qu)__A);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS
-_mm512_mask_popcnt_epi8(__m512i __A, __mmask64 __U, __m512i __B)
-{
-  return (__m512i) __builtin_ia32_selectb_512((__mmask64) __U,
-              (__v64qi) _mm512_popcnt_epi8(__B),
-              (__v64qi) __A);
+static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm512_mask_popcnt_epi8(__m512i __A, __mmask64 __U, __m512i __B) {
+  return (__m512i)__builtin_ia32_selectb_512(
+      (__mmask64)__U, (__v64qi)_mm512_popcnt_epi8(__B), (__v64qi)__A);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS
-_mm512_maskz_popcnt_epi8(__mmask64 __U, __m512i __B)
-{
-  return _mm512_mask_popcnt_epi8((__m512i) _mm512_setzero_si512(),
-              __U,
-              __B);
+static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm512_maskz_popcnt_epi8(__mmask64 __U, __m512i __B) {
+  return _mm512_mask_popcnt_epi8((__m512i)_mm512_setzero_si512(), __U, __B);
 }
 
 static __inline__ __mmask64 __DEFAULT_FN_ATTRS
@@ -80,7 +74,7 @@ _mm512_bitshuffle_epi64_mask(__m512i __A, __m512i __B)
               __B);
 }
 
-
 #undef __DEFAULT_FN_ATTRS
+#undef __DEFAULT_FN_ATTRS_CONSTEXPR
 
 #endif
diff --git a/clang/lib/Headers/avx512bwintrin.h b/clang/lib/Headers/avx512bwintrin.h
index c854720de6a65..9263f7af3ee2f 100644
--- a/clang/lib/Headers/avx512bwintrin.h
+++ b/clang/lib/Headers/avx512bwintrin.h
@@ -25,6 +25,14 @@ typedef unsigned long long __mmask64;
   __attribute__((__always_inline__, __nodebug__,                               \
                  __target__("avx512bw,no-evex512")))
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
+#else
+#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#endif
+
 static __inline __mmask32 __DEFAULT_FN_ATTRS
 _knot_mask32(__mmask32 __M)
 {
@@ -438,7 +446,7 @@ _mm512_maskz_sub_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
                                              (__v32hi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_mullo_epi16 (__m512i __A, __m512i __B) {
   return (__m512i) ((__v32hu) __A * (__v32hu) __B);
 }
@@ -605,9 +613,8 @@ _mm512_maskz_packus_epi16(__mmask64 __M, __m512i __A, __m512i __B)
                                         (__v64qi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_adds_epi8 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_adds_epi8(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_elementwise_add_sat((__v64qs)__A, (__v64qs)__B);
 }
 
@@ -627,9 +634,8 @@ _mm512_maskz_adds_epi8 (__mmask64 __U, __m512i __A, __m512i __B)
                                         (__v64qi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_adds_epi16 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_adds_epi16(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_elementwise_add_sat((__v32hi)__A, (__v32hi)__B);
 }
 
@@ -649,9 +655,8 @@ _mm512_maskz_adds_epi16 (__mmask32 __U, __m512i __A, __m512i __B)
                                         (__v32hi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_adds_epu8 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_adds_epu8(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_elementwise_add_sat((__v64qu) __A, (__v64qu) __B);
 }
 
@@ -671,9 +676,8 @@ _mm512_maskz_adds_epu8 (__mmask64 __U, __m512i __A, __m512i __B)
                                         (__v64qi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_adds_epu16 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_adds_epu16(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_elementwise_add_sat((__v32hu) __A, (__v32hu) __B);
 }
 
@@ -938,9 +942,8 @@ _mm512_maskz_shuffle_epi8(__mmask64 __U, __m512i __A, __m512i __B)
                                          (__v64qi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_subs_epi8 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_subs_epi8(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_elementwise_sub_sat((__v64qs)__A, (__v64qs)__B);
 }
 
@@ -960,9 +963,8 @@ _mm512_maskz_subs_epi8 (__mmask64 __U, __m512i __A, __m512i __B)
                                         (__v64qi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_subs_epi16 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_subs_epi16(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_elementwise_sub_sat((__v32hi)__A, (__v32hi)__B);
 }
 
@@ -982,9 +984,8 @@ _mm512_maskz_subs_epi16 (__mmask32 __U, __m512i __A, __m512i __B)
                                         (__v32hi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_subs_epu8 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_subs_epu8(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_elementwise_sub_sat((__v64qu) __A, (__v64qu) __B);
 }
 
@@ -1004,9 +1005,8 @@ _mm512_maskz_subs_epu8 (__mmask64 __U, __m512i __A, __m512i __B)
                                         (__v64qi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_subs_epu16 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_subs_epu16(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_elementwise_sub_sat((__v32hu) __A, (__v32hu) __B);
 }
 
@@ -1082,49 +1082,40 @@ _mm512_maskz_mulhrs_epi16(__mmask32 __U, __m512i __A, __m512i __B)
                                          (__v32hi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_mulhi_epi16(__m512i __A, __m512i __B)
-{
-  return (__m512i)__builtin_ia32_pmulhw512((__v32hi) __A, (__v32hi) __B);
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mulhi_epi16(__m512i __A, __m512i __B) {
+  return (__m512i)__builtin_ia32_pmulhw512((__v32hi)__A, (__v32hi)__B);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A,
-       __m512i __B)
-{
-  return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
-                                          (__v32hi)_mm512_mulhi_epi16(__A, __B),
-                                          (__v32hi)__W);
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
+  return (__m512i)__builtin_ia32_selectw_512(
+      (__mmask32)__U, (__v32hi)_mm512_mulhi_epi16(__A, __B), (__v32hi)__W);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_maskz_mulhi_epi16(__mmask32 __U, __m512i __A, __m512i __B)
-{
-  return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
-                                          (__v32hi)_mm512_mulhi_epi16(__A, __B),
-                                          (__v32hi)_mm512_setzero_si512());
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_maskz_mulhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
+  return (__m512i)__builtin_ia32_selectw_512(
+      (__mmask32)__U, (__v32hi)_mm512_mulhi_epi16(__A, __B),
+      (__v32hi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_mulhi_epu16(__m512i __A, __m512i __B)
-{
-  return (__m512i)__builtin_ia32_pmulhuw512((__v32hi) __A, (__v32hi) __B);
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mulhi_epu16(__m512i __A, __m512i __B) {
+  return (__m512i)__builtin_ia32_pmulhuw512((__v32hu)__A, (__v32hu)__B);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B)
-{
-  return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
-                                          (__v32hi)_mm512_mulhi_epu16(__A, __B),
-                                          (__v32hi)__W);
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
+  return (__m512i)__builtin_ia32_selectw_512(
+      (__mmask32)__U, (__v32hi)_mm512_mulhi_epu16(__A, __B), (__v32hi)__W);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_maskz_mulhi_epu16 (__mmask32 __U, __m512i __A, __m512i __B)
-{
-  return (__m512i)__builtin_ia32_selectw_512((__mmask32)__U,
-                                          (__v32hi)_mm512_mulhi_epu16(__A, __B),
-                                          (__v32hi)_mm512_setzero_si512());
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_maskz_mulhi_epu16(__mmask32 __U, __m512i __A, __m512i __B) {
+  return (__m512i)__builtin_ia32_selectw_512(
+      (__mmask32)__U, (__v32hi)_mm512_mulhi_epu16(__A, __B),
+      (__v32hi)_mm512_setzero_si512());
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
@@ -1247,7 +1238,7 @@ _mm512_mask_cvtusepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A)
   __builtin_ia32_pmovuswb512mem_mask ((__v32qi *) __P, (__v32hi) __A, __M);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_unpackhi_epi8(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_shufflevector((__v64qi)__A, (__v64qi)__B,
                                           8,  64+8,   9, 64+9,
@@ -1282,7 +1273,7 @@ _mm512_maskz_unpackhi_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
                                         (__v64qi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_unpackhi_epi16(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_shufflevector((__v32hi)__A, (__v32hi)__B,
                                           4,  32+4,   5, 32+5,
@@ -1309,7 +1300,7 @@ _mm512_maskz_unpackhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
                                        (__v32hi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_unpacklo_epi8(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_shufflevector((__v64qi)__A, (__v64qi)__B,
                                           0,  64+0,   1, 64+1,
@@ -1344,7 +1335,7 @@ _mm512_maskz_unpacklo_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
                                         (__v64qi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_unpacklo_epi16(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_shufflevector((__v32hi)__A, (__v32hi)__B,
                                           0,  32+0,   1, 32+1,
@@ -1371,9 +1362,8 @@ _mm512_maskz_unpacklo_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
                                        (__v32hi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepi8_epi16(__m256i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepi8_epi16(__m256i __A) {
   /* This function always performs a signed extension, but __v32qi is a char
      which may be signed or unsigned, so use __v32qs. */
   return (__m512i)__builtin_convertvector((__v32qs)__A, __v32hi);
@@ -1395,9 +1385,8 @@ _mm512_maskz_cvtepi8_epi16(__mmask32 __U, __m256i __A)
                                              (__v32hi)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepu8_epi16(__m256i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepu8_epi16(__m256i __A) {
   return (__m512i)__builtin_convertvector((__v32qu)__A, __v32hi);
 }
 
@@ -1883,9 +1872,8 @@ _mm512_movm_epi16 (__mmask32 __A)
   return (__m512i) __builtin_ia32_cvtmask2w512 (__A);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_broadcastb_epi8 (__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcastb_epi8(__m128i __A) {
   return (__m512i)__builtin_shufflevector((__v16qi) __A, (__v16qi) __A,
                                           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                                           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
@@ -1925,9 +1913,8 @@ _mm512_maskz_set1_epi16 (__mmask32 __M, short __A)
                                               (__v32hi) _mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_broadcastw_epi16 (__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcastw_epi16(__m128i __A) {
   return (__m512i)__builtin_shufflevector((__v8hi) __A, (__v8hi) __A,
                                           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                                           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
@@ -2010,5 +1997,7 @@ _mm512_sad_epu8 (__m512i __A, __m512i __B)
 
 #undef __DEFAULT_FN_ATTRS512
 #undef __DEFAULT_FN_ATTRS
+#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS_CONSTEXPR
 
 #endif
diff --git a/clang/lib/Headers/avx512dqintrin.h b/clang/lib/Headers/avx512dqintrin.h
index 88b48e3a32070..87d16b474d466 100644
--- a/clang/lib/Headers/avx512dqintrin.h
+++ b/clang/lib/Headers/avx512dqintrin.h
@@ -20,6 +20,14 @@
   __attribute__((__always_inline__, __nodebug__,                               \
                  __target__("avx512dq,no-evex512")))
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
+#else
+#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#endif
+
 static __inline __mmask8 __DEFAULT_FN_ATTRS
 _knot_mask8(__mmask8 __M)
 {
@@ -167,7 +175,7 @@ _mm512_maskz_mullo_epi64(__mmask8 __U, __m512i __A, __m512i __B) {
                                              (__v8di)_mm512_setzero_si512());
 }
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_xor_pd(__m512d __A, __m512d __B) {
   return (__m512d)((__v8du)__A ^ (__v8du)__B);
 }
@@ -186,7 +194,7 @@ _mm512_maskz_xor_pd(__mmask8 __U, __m512d __A, __m512d __B) {
                                               (__v8df)_mm512_setzero_pd());
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_xor_ps (__m512 __A, __m512 __B) {
   return (__m512)((__v16su)__A ^ (__v16su)__B);
 }
@@ -205,7 +213,7 @@ _mm512_maskz_xor_ps(__mmask16 __U, __m512 __A, __m512 __B) {
                                              (__v16sf)_mm512_setzero_ps());
 }
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_or_pd(__m512d __A, __m512d __B) {
   return (__m512d)((__v8du)__A | (__v8du)__B);
 }
@@ -224,7 +232,7 @@ _mm512_maskz_or_pd(__mmask8 __U, __m512d __A, __m512d __B) {
                                               (__v8df)_mm512_setzero_pd());
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_or_ps(__m512 __A, __m512 __B) {
   return (__m512)((__v16su)__A | (__v16su)__B);
 }
@@ -243,7 +251,7 @@ _mm512_maskz_or_ps(__mmask16 __U, __m512 __A, __m512 __B) {
                                              (__v16sf)_mm512_setzero_ps());
 }
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_and_pd(__m512d __A, __m512d __B) {
   return (__m512d)((__v8du)__A & (__v8du)__B);
 }
@@ -262,7 +270,7 @@ _mm512_maskz_and_pd(__mmask8 __U, __m512d __A, __m512d __B) {
                                               (__v8df)_mm512_setzero_pd());
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_and_ps(__m512 __A, __m512 __B) {
   return (__m512)((__v16su)__A & (__v16su)__B);
 }
@@ -281,7 +289,7 @@ _mm512_maskz_and_ps(__mmask16 __U, __m512 __A, __m512 __B) {
                                              (__v16sf)_mm512_setzero_ps());
 }
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_andnot_pd(__m512d __A, __m512d __B) {
   return (__m512d)(~(__v8du)__A & (__v8du)__B);
 }
@@ -300,7 +308,7 @@ _mm512_maskz_andnot_pd(__mmask8 __U, __m512d __A, __m512d __B) {
                                               (__v8df)_mm512_setzero_pd());
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_andnot_ps(__m512 __A, __m512 __B) {
   return (__m512)(~(__v16su)__A & (__v16su)__B);
 }
@@ -1076,10 +1084,8 @@ _mm512_movepi64_mask (__m512i __A)
   return (__mmask8) __builtin_ia32_cvtq2mask512 ((__v8di) __A);
 }
 
-
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
-_mm512_broadcast_f32x2 (__m128 __A)
-{
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_f32x2(__m128 __A) {
   return (__m512)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
                                          0, 1, 0, 1, 0, 1, 0, 1,
                                          0, 1, 0, 1, 0, 1, 0, 1);
@@ -1101,9 +1107,8 @@ _mm512_maskz_broadcast_f32x2 (__mmask16 __M, __m128 __A)
                                              (__v16sf)_mm512_setzero_ps());
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
-_mm512_broadcast_f32x8(__m256 __A)
-{
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_f32x8(__m256 __A) {
   return (__m512)__builtin_shufflevector((__v8sf)__A, (__v8sf)__A,
                                          0, 1, 2, 3, 4, 5, 6, 7,
                                          0, 1, 2, 3, 4, 5, 6, 7);
@@ -1125,9 +1130,8 @@ _mm512_maskz_broadcast_f32x8(__mmask16 __M, __m256 __A)
                                            (__v16sf)_mm512_setzero_ps());
 }
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
-_mm512_broadcast_f64x2(__m128d __A)
-{
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_f64x2(__m128d __A) {
   return (__m512d)__builtin_shufflevector((__v2df)__A, (__v2df)__A,
                                           0, 1, 0, 1, 0, 1, 0, 1);
 }
@@ -1148,9 +1152,8 @@ _mm512_maskz_broadcast_f64x2(__mmask8 __M, __m128d __A)
                                             (__v8df)_mm512_setzero_pd());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_broadcast_i32x2 (__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_i32x2(__m128i __A) {
   return (__m512i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
                                           0, 1, 0, 1, 0, 1, 0, 1,
                                           0, 1, 0, 1, 0, 1, 0, 1);
@@ -1172,9 +1175,8 @@ _mm512_maskz_broadcast_i32x2 (__mmask16 __M, __m128i __A)
                                              (__v16si)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_broadcast_i32x8(__m256i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_i32x8(__m256i __A) {
   return (__m512i)__builtin_shufflevector((__v8si)__A, (__v8si)__A,
                                           0, 1, 2, 3, 4, 5, 6, 7,
                                           0, 1, 2, 3, 4, 5, 6, 7);
@@ -1196,9 +1198,8 @@ _mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i __A)
                                            (__v16si)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_broadcast_i64x2(__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_i64x2(__m128i __A) {
   return (__m512i)__builtin_shufflevector((__v2di)__A, (__v2di)__A,
                                           0, 1, 0, 1, 0, 1, 0, 1);
 }
@@ -1375,5 +1376,7 @@ _mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i __A)
 
 #undef __DEFAULT_FN_ATTRS512
 #undef __DEFAULT_FN_ATTRS
+#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS_CONSTEXPR
 
 #endif
diff --git a/clang/lib/Headers/avx512fintrin.h b/clang/lib/Headers/avx512fintrin.h
index 45e7eeb5327d0..81c8e8e934493 100644
--- a/clang/lib/Headers/avx512fintrin.h
+++ b/clang/lib/Headers/avx512fintrin.h
@@ -180,9 +180,9 @@ typedef enum
 #define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
 #define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
 #else
-#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS128
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
 #define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
-#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
 #endif
 
 /* Create vectors with repeated elements */
@@ -218,9 +218,8 @@ _mm512_undefined_epi32(void)
   return (__m512i)__builtin_ia32_undef512();
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_broadcastd_epi32 (__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcastd_epi32(__m128i __A) {
   return (__m512i)__builtin_shufflevector((__v4si) __A, (__v4si) __A,
                                           0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 }
@@ -241,9 +240,8 @@ _mm512_maskz_broadcastd_epi32 (__mmask16 __M, __m128i __A)
                                              (__v16si) _mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_broadcastq_epi64 (__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcastq_epi64(__m128i __A) {
   return (__m512i)__builtin_shufflevector((__v2di) __A, (__v2di) __A,
                                           0, 0, 0, 0, 0, 0, 0, 0);
 }
@@ -277,20 +275,20 @@ _mm512_setzero_pd(void) {
   return __extension__(__m512d){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
 }
 
-static __inline __m512 __DEFAULT_FN_ATTRS512
+static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_set1_ps(float __w)
 {
   return __extension__ (__m512){ __w, __w, __w, __w, __w, __w, __w, __w,
                                  __w, __w, __w, __w, __w, __w, __w, __w  };
 }
 
-static __inline __m512d __DEFAULT_FN_ATTRS512
+static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_set1_pd(double __w)
 {
   return __extension__ (__m512d){ __w, __w, __w, __w, __w, __w, __w, __w };
 }
 
-static __inline __m512i __DEFAULT_FN_ATTRS512
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_set1_epi8(char __w)
 {
   return __extension__ (__m512i)(__v64qi){
@@ -304,7 +302,7 @@ _mm512_set1_epi8(char __w)
     __w, __w, __w, __w, __w, __w, __w, __w  };
 }
 
-static __inline __m512i __DEFAULT_FN_ATTRS512
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_set1_epi16(short __w)
 {
   return __extension__ (__m512i)(__v32hi){
@@ -314,7 +312,7 @@ _mm512_set1_epi16(short __w)
     __w, __w, __w, __w, __w, __w, __w, __w };
 }
 
-static __inline __m512i __DEFAULT_FN_ATTRS512
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_set1_epi32(int __s)
 {
   return __extension__ (__m512i)(__v16si){
@@ -330,7 +328,7 @@ _mm512_maskz_set1_epi32(__mmask16 __M, int __A)
                                              (__v16si)_mm512_setzero_si512());
 }
 
-static __inline __m512i __DEFAULT_FN_ATTRS512
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_set1_epi64(long long __d)
 {
   return __extension__(__m512i)(__v8di){ __d, __d, __d, __d, __d, __d, __d, __d };
@@ -344,39 +342,33 @@ _mm512_maskz_set1_epi64(__mmask8 __M, long long __A)
                                              (__v8di)_mm512_setzero_si512());
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
-_mm512_broadcastss_ps(__m128 __A)
-{
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcastss_ps(__m128 __A) {
   return (__m512)__builtin_shufflevector((__v4sf) __A, (__v4sf) __A,
                                          0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 }
 
-static __inline __m512i __DEFAULT_FN_ATTRS512
-_mm512_set4_epi32 (int __A, int __B, int __C, int __D)
-{
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_set4_epi32(int __A, int __B, int __C, int __D) {
   return __extension__ (__m512i)(__v16si)
    { __D, __C, __B, __A, __D, __C, __B, __A,
      __D, __C, __B, __A, __D, __C, __B, __A };
 }
 
-static __inline __m512i __DEFAULT_FN_ATTRS512
-_mm512_set4_epi64 (long long __A, long long __B, long long __C,
-       long long __D)
-{
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_set4_epi64(long long __A, long long __B, long long __C, long long __D) {
   return __extension__ (__m512i) (__v8di)
    { __D, __C, __B, __A, __D, __C, __B, __A };
 }
 
-static __inline __m512d __DEFAULT_FN_ATTRS512
-_mm512_set4_pd (double __A, double __B, double __C, double __D)
-{
+static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_set4_pd(double __A, double __B, double __C, double __D) {
   return __extension__ (__m512d)
    { __D, __C, __B, __A, __D, __C, __B, __A };
 }
 
-static __inline __m512 __DEFAULT_FN_ATTRS512
-_mm512_set4_ps (float __A, float __B, float __C, float __D)
-{
+static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_set4_ps(float __A, float __B, float __C, float __D) {
   return __extension__ (__m512)
    { __D, __C, __B, __A, __D, __C, __B, __A,
      __D, __C, __B, __A, __D, __C, __B, __A };
@@ -394,9 +386,8 @@ _mm512_set4_ps (float __A, float __B, float __C, float __D)
 #define _mm512_setr4_ps(e0,e1,e2,e3)                \
   _mm512_set4_ps((e3),(e2),(e1),(e0))
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
-_mm512_broadcastsd_pd(__m128d __A)
-{
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcastsd_pd(__m128d __A) {
   return (__m512d)__builtin_shufflevector((__v2df) __A, (__v2df) __A,
                                           0, 0, 0, 0, 0, 0, 0, 0);
 }
@@ -645,7 +636,7 @@ _mm512_zextsi256_si512(__m256i __a)
 }
 
 /* Bitwise operators */
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_and_epi32(__m512i __a, __m512i __b)
 {
   return (__m512i)((__v16su)__a & (__v16su)__b);
@@ -666,7 +657,7 @@ _mm512_maskz_and_epi32(__mmask16 __k, __m512i __a, __m512i __b)
                                          __k, __a, __b);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_and_epi64(__m512i __a, __m512i __b)
 {
   return (__m512i)((__v8du)__a & (__v8du)__b);
@@ -687,13 +678,13 @@ _mm512_maskz_and_epi64(__mmask8 __k, __m512i __a, __m512i __b)
                                          __k, __a, __b);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_andnot_si512 (__m512i __A, __m512i __B)
 {
   return (__m512i)(~(__v8du)__A & (__v8du)__B);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_andnot_epi32 (__m512i __A, __m512i __B)
 {
   return (__m512i)(~(__v16su)__A & (__v16su)__B);
@@ -714,7 +705,7 @@ _mm512_maskz_andnot_epi32(__mmask16 __U, __m512i __A, __m512i __B)
                                            __U, __A, __B);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_andnot_epi64(__m512i __A, __m512i __B)
 {
   return (__m512i)(~(__v8du)__A & (__v8du)__B);
@@ -735,7 +726,7 @@ _mm512_maskz_andnot_epi64(__mmask8 __U, __m512i __A, __m512i __B)
                                            __U, __A, __B);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_or_epi32(__m512i __a, __m512i __b)
 {
   return (__m512i)((__v16su)__a | (__v16su)__b);
@@ -755,7 +746,7 @@ _mm512_maskz_or_epi32(__mmask16 __k, __m512i __a, __m512i __b)
   return (__m512i)_mm512_mask_or_epi32(_mm512_setzero_si512(), __k, __a, __b);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_or_epi64(__m512i __a, __m512i __b)
 {
   return (__m512i)((__v8du)__a | (__v8du)__b);
@@ -775,7 +766,7 @@ _mm512_maskz_or_epi64(__mmask8 __k, __m512i __a, __m512i __b)
   return (__m512i)_mm512_mask_or_epi64(_mm512_setzero_si512(), __k, __a, __b);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_xor_epi32(__m512i __a, __m512i __b)
 {
   return (__m512i)((__v16su)__a ^ (__v16su)__b);
@@ -795,7 +786,7 @@ _mm512_maskz_xor_epi32(__mmask16 __k, __m512i __a, __m512i __b)
   return (__m512i)_mm512_mask_xor_epi32(_mm512_setzero_si512(), __k, __a, __b);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_xor_epi64(__m512i __a, __m512i __b)
 {
   return (__m512i)((__v8du)__a ^ (__v8du)__b);
@@ -815,19 +806,19 @@ _mm512_maskz_xor_epi64(__mmask8 __k, __m512i __a, __m512i __b)
   return (__m512i)_mm512_mask_xor_epi64(_mm512_setzero_si512(), __k, __a, __b);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_and_si512(__m512i __a, __m512i __b)
 {
   return (__m512i)((__v8du)__a & (__v8du)__b);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_or_si512(__m512i __a, __m512i __b)
 {
   return (__m512i)((__v8du)__a | (__v8du)__b);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_xor_si512(__m512i __a, __m512i __b)
 {
   return (__m512i)((__v8du)__a ^ (__v8du)__b);
@@ -835,45 +826,38 @@ _mm512_xor_si512(__m512i __a, __m512i __b)
 
 /* Arithmetic */
 
-static __inline __m512d __DEFAULT_FN_ATTRS512
-_mm512_add_pd(__m512d __a, __m512d __b)
-{
+static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_add_pd(__m512d __a, __m512d __b) {
   return (__m512d)((__v8df)__a + (__v8df)__b);
 }
 
-static __inline __m512 __DEFAULT_FN_ATTRS512
-_mm512_add_ps(__m512 __a, __m512 __b)
-{
+static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_add_ps(__m512 __a, __m512 __b) {
   return (__m512)((__v16sf)__a + (__v16sf)__b);
 }
 
-static __inline __m512d __DEFAULT_FN_ATTRS512
-_mm512_mul_pd(__m512d __a, __m512d __b)
-{
+static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mul_pd(__m512d __a, __m512d __b) {
   return (__m512d)((__v8df)__a * (__v8df)__b);
 }
 
-static __inline __m512 __DEFAULT_FN_ATTRS512
-_mm512_mul_ps(__m512 __a, __m512 __b)
-{
+static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mul_ps(__m512 __a, __m512 __b) {
   return (__m512)((__v16sf)__a * (__v16sf)__b);
 }
 
-static __inline __m512d __DEFAULT_FN_ATTRS512
-_mm512_sub_pd(__m512d __a, __m512d __b)
-{
+static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_sub_pd(__m512d __a, __m512d __b) {
   return (__m512d)((__v8df)__a - (__v8df)__b);
 }
 
-static __inline __m512 __DEFAULT_FN_ATTRS512
-_mm512_sub_ps(__m512 __a, __m512 __b)
-{
+static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_sub_ps(__m512 __a, __m512 __b) {
   return (__m512)((__v16sf)__a - (__v16sf)__b);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_add_epi64 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_add_epi64(__m512i __A, __m512i __B) {
   return (__m512i) ((__v8du) __A + (__v8du) __B);
 }
 
@@ -1429,9 +1413,8 @@ _mm512_maskz_min_epu64 (__mmask8 __M, __m512i __A, __m512i __B)
                                              (__v8di)_mm512_setzero_si512());
 }
 
-static __inline __m512i __DEFAULT_FN_ATTRS512
-_mm512_mul_epi32(__m512i __X, __m512i __Y)
-{
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mul_epi32(__m512i __X, __m512i __Y) {
   return (__m512i)__builtin_ia32_pmuldq512((__v16si)__X, (__v16si) __Y);
 }
 
@@ -1451,9 +1434,8 @@ _mm512_maskz_mul_epi32(__mmask8 __M, __m512i __X, __m512i __Y)
                                              (__v8di)_mm512_setzero_si512 ());
 }
 
-static __inline __m512i __DEFAULT_FN_ATTRS512
-_mm512_mul_epu32(__m512i __X, __m512i __Y)
-{
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mul_epu32(__m512i __X, __m512i __Y) {
   return (__m512i)__builtin_ia32_pmuludq512((__v16si)__X, (__v16si)__Y);
 }
 
@@ -2315,9 +2297,8 @@ _mm_maskz_div_sd(__mmask8 __U,__m128d __A, __m128d __B) {
                                             (__v2df)_mm_setzero_pd(), \
                                             (__mmask8)(U), (int)(R)))
 
-static __inline __m512d __DEFAULT_FN_ATTRS512
-_mm512_div_pd(__m512d __a, __m512d __b)
-{
+static __inline __m512d
+    __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_div_pd(__m512d __a, __m512d __b) {
   return (__m512d)((__v8df)__a/(__v8df)__b);
 }
 
@@ -2335,9 +2316,8 @@ _mm512_maskz_div_pd(__mmask8 __U, __m512d __A, __m512d __B) {
                                               (__v8df)_mm512_setzero_pd());
 }
 
-static __inline __m512 __DEFAULT_FN_ATTRS512
-_mm512_div_ps(__m512 __a, __m512 __b)
-{
+static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_div_ps(__m512 __a, __m512 __b) {
   return (__m512)((__v16sf)__a/(__v16sf)__b);
 }
 
@@ -4123,9 +4103,8 @@ _mm512_cvtss_f32(__m512 __a)
 
 /* Unpack and Interleave */
 
-static __inline __m512d __DEFAULT_FN_ATTRS512
-_mm512_unpackhi_pd(__m512d __a, __m512d __b)
-{
+static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_unpackhi_pd(__m512d __a, __m512d __b) {
   return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b,
                                           1, 9, 1+2, 9+2, 1+4, 9+4, 1+6, 9+6);
 }
@@ -4146,9 +4125,8 @@ _mm512_maskz_unpackhi_pd(__mmask8 __U, __m512d __A, __m512d __B)
                                            (__v8df)_mm512_setzero_pd());
 }
 
-static __inline __m512d __DEFAULT_FN_ATTRS512
-_mm512_unpacklo_pd(__m512d __a, __m512d __b)
-{
+static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_unpacklo_pd(__m512d __a, __m512d __b) {
   return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b,
                                           0, 8, 0+2, 8+2, 0+4, 8+4, 0+6, 8+6);
 }
@@ -4169,9 +4147,8 @@ _mm512_maskz_unpacklo_pd (__mmask8 __U, __m512d __A, __m512d __B)
                                            (__v8df)_mm512_setzero_pd());
 }
 
-static __inline __m512 __DEFAULT_FN_ATTRS512
-_mm512_unpackhi_ps(__m512 __a, __m512 __b)
-{
+static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_unpackhi_ps(__m512 __a, __m512 __b) {
   return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b,
                                          2,    18,    3,    19,
                                          2+4,  18+4,  3+4,  19+4,
@@ -4195,9 +4172,8 @@ _mm512_maskz_unpackhi_ps (__mmask16 __U, __m512 __A, __m512 __B)
                                           (__v16sf)_mm512_setzero_ps());
 }
 
-static __inline __m512 __DEFAULT_FN_ATTRS512
-_mm512_unpacklo_ps(__m512 __a, __m512 __b)
-{
+static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_unpacklo_ps(__m512 __a, __m512 __b) {
   return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b,
                                          0,    16,    1,    17,
                                          0+4,  16+4,  1+4,  17+4,
@@ -4221,9 +4197,8 @@ _mm512_maskz_unpacklo_ps (__mmask16 __U, __m512 __A, __m512 __B)
                                           (__v16sf)_mm512_setzero_ps());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_unpackhi_epi32(__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_unpackhi_epi32(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_shufflevector((__v16si)__A, (__v16si)__B,
                                           2,    18,    3,    19,
                                           2+4,  18+4,  3+4,  19+4,
@@ -4247,9 +4222,8 @@ _mm512_maskz_unpackhi_epi32(__mmask16 __U, __m512i __A, __m512i __B)
                                        (__v16si)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_unpacklo_epi32(__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_unpacklo_epi32(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_shufflevector((__v16si)__A, (__v16si)__B,
                                           0,    16,    1,    17,
                                           0+4,  16+4,  1+4,  17+4,
@@ -4273,9 +4247,8 @@ _mm512_maskz_unpacklo_epi32(__mmask16 __U, __m512i __A, __m512i __B)
                                        (__v16si)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_unpackhi_epi64(__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_unpackhi_epi64(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_shufflevector((__v8di)__A, (__v8di)__B,
                                           1, 9, 1+2, 9+2, 1+4, 9+4, 1+6, 9+6);
 }
@@ -4296,9 +4269,8 @@ _mm512_maskz_unpackhi_epi64(__mmask8 __U, __m512i __A, __m512i __B)
                                         (__v8di)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_unpacklo_epi64 (__m512i __A, __m512i __B)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_unpacklo_epi64(__m512i __A, __m512i __B) {
   return (__m512i)__builtin_shufflevector((__v8di)__A, (__v8di)__B,
                                           0, 8, 0+2, 8+2, 0+4, 8+4, 0+6, 8+6);
 }
@@ -4727,9 +4699,8 @@ _mm512_knot(__mmask16 __M)
 #define _mm512_mask_cmpneq_epu64_mask(k, A, B) \
     _mm512_mask_cmp_epu64_mask((k), (A), (B), _MM_CMPINT_NE)
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepi8_epi32(__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepi8_epi32(__m128i __A) {
   /* This function always performs a signed extension, but __v16qi is a char
      which may be signed or unsigned, so use __v16qs. */
   return (__m512i)__builtin_convertvector((__v16qs)__A, __v16si);
@@ -4751,9 +4722,8 @@ _mm512_maskz_cvtepi8_epi32(__mmask16 __U, __m128i __A)
                                              (__v16si)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepi8_epi64(__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepi8_epi64(__m128i __A) {
   /* This function always performs a signed extension, but __v16qi is a char
      which may be signed or unsigned, so use __v16qs. */
   return (__m512i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__A, (__v16qs)__A, 0, 1, 2, 3, 4, 5, 6, 7), __v8di);
@@ -4775,9 +4745,8 @@ _mm512_maskz_cvtepi8_epi64(__mmask8 __U, __m128i __A)
                                              (__v8di)_mm512_setzero_si512 ());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepi32_epi64(__m256i __X)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepi32_epi64(__m256i __X) {
   return (__m512i)__builtin_convertvector((__v8si)__X, __v8di);
 }
 
@@ -4797,9 +4766,8 @@ _mm512_maskz_cvtepi32_epi64(__mmask8 __U, __m256i __X)
                                              (__v8di)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepi16_epi32(__m256i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepi16_epi32(__m256i __A) {
   return (__m512i)__builtin_convertvector((__v16hi)__A, __v16si);
 }
 
@@ -4819,9 +4787,8 @@ _mm512_maskz_cvtepi16_epi32(__mmask16 __U, __m256i __A)
                                             (__v16si)_mm512_setzero_si512 ());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepi16_epi64(__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepi16_epi64(__m128i __A) {
   return (__m512i)__builtin_convertvector((__v8hi)__A, __v8di);
 }
 
@@ -4841,9 +4808,8 @@ _mm512_maskz_cvtepi16_epi64(__mmask8 __U, __m128i __A)
                                              (__v8di)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepu8_epi32(__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepu8_epi32(__m128i __A) {
   return (__m512i)__builtin_convertvector((__v16qu)__A, __v16si);
 }
 
@@ -4863,9 +4829,8 @@ _mm512_maskz_cvtepu8_epi32(__mmask16 __U, __m128i __A)
                                              (__v16si)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepu8_epi64(__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepu8_epi64(__m128i __A) {
   return (__m512i)__builtin_convertvector(__builtin_shufflevector((__v16qu)__A, (__v16qu)__A, 0, 1, 2, 3, 4, 5, 6, 7), __v8di);
 }
 
@@ -4885,9 +4850,8 @@ _mm512_maskz_cvtepu8_epi64(__mmask8 __U, __m128i __A)
                                              (__v8di)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepu32_epi64(__m256i __X)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepu32_epi64(__m256i __X) {
   return (__m512i)__builtin_convertvector((__v8su)__X, __v8di);
 }
 
@@ -4907,9 +4871,8 @@ _mm512_maskz_cvtepu32_epi64(__mmask8 __U, __m256i __X)
                                              (__v8di)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepu16_epi32(__m256i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepu16_epi32(__m256i __A) {
   return (__m512i)__builtin_convertvector((__v16hu)__A, __v16si);
 }
 
@@ -4929,9 +4892,8 @@ _mm512_maskz_cvtepu16_epi32(__mmask16 __U, __m256i __A)
                                             (__v16si)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_cvtepu16_epi64(__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_cvtepu16_epi64(__m128i __A) {
   return (__m512i)__builtin_convertvector((__v8hu)__A, __v8di);
 }
 
@@ -4954,7 +4916,7 @@ _mm512_maskz_cvtepu16_epi64(__mmask8 __U, __m128i __A)
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
 _mm512_rorv_epi32 (__m512i __A, __m512i __B)
 {
-  return (__m512i)__builtin_ia32_prorvd512((__v16si)__A, (__v16si)__B);
+  return (__m512i)__builtin_elementwise_fshr((__v16su)__A,(__v16su)__A, (__v16su)__B);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
@@ -4976,7 +4938,7 @@ _mm512_maskz_rorv_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
 _mm512_rorv_epi64 (__m512i __A, __m512i __B)
 {
-  return (__m512i)__builtin_ia32_prorvq512((__v8di)__A, (__v8di)__B);
+  return (__m512i)__builtin_elementwise_fshr((__v8du)__A, (__v8du)__A, (__v8du)__B);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
@@ -5066,7 +5028,7 @@ _mm512_maskz_rorv_epi64 (__mmask8 __U, __m512i __A, __m512i __B)
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
 _mm512_rolv_epi32 (__m512i __A, __m512i __B)
 {
-  return (__m512i)__builtin_ia32_prolvd512((__v16si)__A, (__v16si)__B);
+  return (__m512i)__builtin_elementwise_fshl((__v16su)__A, (__v16su)__A, (__v16su)__B);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
@@ -5088,7 +5050,7 @@ _mm512_maskz_rolv_epi32 (__mmask16 __U, __m512i __A, __m512i __B)
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
 _mm512_rolv_epi64 (__m512i __A, __m512i __B)
 {
-  return (__m512i)__builtin_ia32_prolvq512((__v8di)__A, (__v8di)__B);
+  return (__m512i)__builtin_elementwise_fshl((__v8du)__A, (__v8du)__A, (__v8du)__B);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS512
@@ -5303,7 +5265,7 @@ _mm512_mask_store_epi64 (void *__P, __mmask8 __U, __m512i __A)
           (__mmask8) __U);
 }
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_movedup_pd (__m512d __A)
 {
   return (__m512d)__builtin_shufflevector((__v8df)__A, (__v8df)__A,
@@ -6827,9 +6789,8 @@ _mm_maskz_sqrt_ss (__mmask8 __U, __m128 __A, __m128 __B)
                                             (__v4sf)_mm_setzero_ps(), \
                                             (__mmask8)(U), (int)(R)))
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
-_mm512_broadcast_f32x4(__m128 __A)
-{
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_f32x4(__m128 __A) {
   return (__m512)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
                                          0, 1, 2, 3, 0, 1, 2, 3,
                                          0, 1, 2, 3, 0, 1, 2, 3);
@@ -6851,9 +6812,8 @@ _mm512_maskz_broadcast_f32x4(__mmask16 __M, __m128 __A)
                                            (__v16sf)_mm512_setzero_ps());
 }
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
-_mm512_broadcast_f64x4(__m256d __A)
-{
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_f64x4(__m256d __A) {
   return (__m512d)__builtin_shufflevector((__v4df)__A, (__v4df)__A,
                                           0, 1, 2, 3, 0, 1, 2, 3);
 }
@@ -6874,9 +6834,8 @@ _mm512_maskz_broadcast_f64x4(__mmask8 __M, __m256d __A)
                                             (__v8df)_mm512_setzero_pd());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_broadcast_i32x4(__m128i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_i32x4(__m128i __A) {
   return (__m512i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
                                           0, 1, 2, 3, 0, 1, 2, 3,
                                           0, 1, 2, 3, 0, 1, 2, 3);
@@ -6898,9 +6857,8 @@ _mm512_maskz_broadcast_i32x4(__mmask16 __M, __m128i __A)
                                            (__v16si)_mm512_setzero_si512());
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_broadcast_i64x4(__m256i __A)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_broadcast_i64x4(__m256i __A) {
   return (__m512i)__builtin_shufflevector((__v4di)__A, (__v4di)__A,
                                           0, 1, 2, 3, 0, 1, 2, 3);
 }
@@ -8665,7 +8623,7 @@ _mm512_mask_testn_epi64_mask (__mmask8 __U, __m512i __A, __m512i __B)
                                        _mm512_setzero_si512());
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_movehdup_ps (__m512 __A)
 {
   return (__m512)__builtin_shufflevector((__v16sf)__A, (__v16sf)__A,
@@ -8688,7 +8646,7 @@ _mm512_maskz_movehdup_ps (__mmask16 __U, __m512 __A)
                                              (__v16sf)_mm512_setzero_ps());
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_moveldup_ps (__m512 __A)
 {
   return (__m512)__builtin_shufflevector((__v16sf)__A, (__v16sf)__A,
@@ -8975,36 +8933,28 @@ _mm512_mask_cvtpslo_pd (__m512d __W, __mmask8 __U, __m512 __A)
   return (__m512d) _mm512_mask_cvtps_pd(__W, __U, _mm512_castps512_ps256(__A));
 }
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
-_mm512_mask_mov_pd (__m512d __W, __mmask8 __U, __m512d __A)
-{
-  return (__m512d) __builtin_ia32_selectpd_512 ((__mmask8) __U,
-              (__v8df) __A,
-              (__v8df) __W);
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mask_mov_pd(__m512d __W, __mmask8 __U, __m512d __A) {
+  return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, (__v8df)__A,
+                                              (__v8df)__W);
 }
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
-_mm512_maskz_mov_pd (__mmask8 __U, __m512d __A)
-{
-  return (__m512d) __builtin_ia32_selectpd_512 ((__mmask8) __U,
-              (__v8df) __A,
-              (__v8df) _mm512_setzero_pd ());
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_maskz_mov_pd(__mmask8 __U, __m512d __A) {
+  return (__m512d)__builtin_ia32_selectpd_512((__mmask8)__U, (__v8df)__A,
+                                              (__v8df)_mm512_setzero_pd());
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
-_mm512_mask_mov_ps (__m512 __W, __mmask16 __U, __m512 __A)
-{
-  return (__m512) __builtin_ia32_selectps_512 ((__mmask16) __U,
-             (__v16sf) __A,
-             (__v16sf) __W);
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_mask_mov_ps(__m512 __W, __mmask16 __U, __m512 __A) {
+  return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, (__v16sf)__A,
+                                             (__v16sf)__W);
 }
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
-_mm512_maskz_mov_ps (__mmask16 __U, __m512 __A)
-{
-  return (__m512) __builtin_ia32_selectps_512 ((__mmask16) __U,
-             (__v16sf) __A,
-             (__v16sf) _mm512_setzero_ps ());
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_maskz_mov_ps(__mmask16 __U, __m512 __A) {
+  return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, (__v16sf)__A,
+                                             (__v16sf)_mm512_setzero_ps());
 }
 
 static __inline__ void __DEFAULT_FN_ATTRS512
@@ -9204,18 +9154,18 @@ _mm512_mask_set1_epi64 (__m512i __O, __mmask8 __M, long long __A)
                                               (__v8di) __O);
 }
 
-static  __inline __m512i __DEFAULT_FN_ATTRS512
-_mm512_set_epi8 (char __e63, char __e62, char __e61, char __e60, char __e59,
-    char __e58, char __e57, char __e56, char __e55, char __e54, char __e53,
-    char __e52, char __e51, char __e50, char __e49, char __e48, char __e47,
-    char __e46, char __e45, char __e44, char __e43, char __e42, char __e41,
-    char __e40, char __e39, char __e38, char __e37, char __e36, char __e35,
-    char __e34, char __e33, char __e32, char __e31, char __e30, char __e29,
-    char __e28, char __e27, char __e26, char __e25, char __e24, char __e23,
-    char __e22, char __e21, char __e20, char __e19, char __e18, char __e17,
-    char __e16, char __e15, char __e14, char __e13, char __e12, char __e11,
-    char __e10, char __e9, char __e8, char __e7, char __e6, char __e5,
-    char __e4, char __e3, char __e2, char __e1, char __e0) {
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_set_epi8(
+    char __e63, char __e62, char __e61, char __e60, char __e59, char __e58,
+    char __e57, char __e56, char __e55, char __e54, char __e53, char __e52,
+    char __e51, char __e50, char __e49, char __e48, char __e47, char __e46,
+    char __e45, char __e44, char __e43, char __e42, char __e41, char __e40,
+    char __e39, char __e38, char __e37, char __e36, char __e35, char __e34,
+    char __e33, char __e32, char __e31, char __e30, char __e29, char __e28,
+    char __e27, char __e26, char __e25, char __e24, char __e23, char __e22,
+    char __e21, char __e20, char __e19, char __e18, char __e17, char __e16,
+    char __e15, char __e14, char __e13, char __e12, char __e11, char __e10,
+    char __e9, char __e8, char __e7, char __e6, char __e5, char __e4, char __e3,
+    char __e2, char __e1, char __e0) {
 
   return __extension__ (__m512i)(__v64qi)
     {__e0, __e1, __e2, __e3, __e4, __e5, __e6, __e7,
@@ -9228,14 +9178,13 @@ _mm512_set_epi8 (char __e63, char __e62, char __e61, char __e60, char __e59,
      __e56, __e57, __e58, __e59, __e60, __e61, __e62, __e63};
 }
 
-static  __inline __m512i __DEFAULT_FN_ATTRS512
-_mm512_set_epi16(short __e31, short __e30, short __e29, short __e28,
-    short __e27, short __e26, short __e25, short __e24, short __e23,
-    short __e22, short __e21, short __e20, short __e19, short __e18,
-    short __e17, short __e16, short __e15, short __e14, short __e13,
-    short __e12, short __e11, short __e10, short __e9, short __e8,
-    short __e7, short __e6, short __e5, short __e4, short __e3,
-    short __e2, short __e1, short __e0) {
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_set_epi16(
+    short __e31, short __e30, short __e29, short __e28, short __e27,
+    short __e26, short __e25, short __e24, short __e23, short __e22,
+    short __e21, short __e20, short __e19, short __e18, short __e17,
+    short __e16, short __e15, short __e14, short __e13, short __e12,
+    short __e11, short __e10, short __e9, short __e8, short __e7, short __e6,
+    short __e5, short __e4, short __e3, short __e2, short __e1, short __e0) {
   return __extension__ (__m512i)(__v32hi)
     {__e0, __e1, __e2, __e3, __e4, __e5, __e6, __e7,
      __e8, __e9, __e10, __e11, __e12, __e13, __e14, __e15,
@@ -9243,12 +9192,9 @@ _mm512_set_epi16(short __e31, short __e30, short __e29, short __e28,
      __e24, __e25, __e26, __e27, __e28, __e29, __e30, __e31 };
 }
 
-static __inline __m512i __DEFAULT_FN_ATTRS512
-_mm512_set_epi32 (int __A, int __B, int __C, int __D,
-     int __E, int __F, int __G, int __H,
-     int __I, int __J, int __K, int __L,
-     int __M, int __N, int __O, int __P)
-{
+static __inline __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_set_epi32(
+    int __A, int __B, int __C, int __D, int __E, int __F, int __G, int __H,
+    int __I, int __J, int __K, int __L, int __M, int __N, int __O, int __P) {
   return __extension__ (__m512i)(__v16si)
   { __P, __O, __N, __M, __L, __K, __J, __I,
     __H, __G, __F, __E, __D, __C, __B, __A };
@@ -9259,11 +9205,9 @@ _mm512_set_epi32 (int __A, int __B, int __C, int __D,
   _mm512_set_epi32((e15),(e14),(e13),(e12),(e11),(e10),(e9),(e8),(e7),(e6), \
                    (e5),(e4),(e3),(e2),(e1),(e0))
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS512
-_mm512_set_epi64 (long long __A, long long __B, long long __C,
-     long long __D, long long __E, long long __F,
-     long long __G, long long __H)
-{
+static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_set_epi64(long long __A, long long __B, long long __C, long long __D,
+                 long long __E, long long __F, long long __G, long long __H) {
   return __extension__ (__m512i) (__v8di)
   { __H, __G, __F, __E, __D, __C, __B, __A };
 }
@@ -9271,10 +9215,9 @@ _mm512_set_epi64 (long long __A, long long __B, long long __C,
 #define _mm512_setr_epi64(e0,e1,e2,e3,e4,e5,e6,e7)           \
   _mm512_set_epi64((e7),(e6),(e5),(e4),(e3),(e2),(e1),(e0))
 
-static __inline__ __m512d __DEFAULT_FN_ATTRS512
-_mm512_set_pd (double __A, double __B, double __C, double __D,
-        double __E, double __F, double __G, double __H)
-{
+static __inline__ __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_set_pd(double __A, double __B, double __C, double __D, double __E,
+              double __F, double __G, double __H) {
   return __extension__ (__m512d)
   { __H, __G, __F, __E, __D, __C, __B, __A };
 }
@@ -9282,12 +9225,10 @@ _mm512_set_pd (double __A, double __B, double __C, double __D,
 #define _mm512_setr_pd(e0,e1,e2,e3,e4,e5,e6,e7)              \
   _mm512_set_pd((e7),(e6),(e5),(e4),(e3),(e2),(e1),(e0))
 
-static __inline__ __m512 __DEFAULT_FN_ATTRS512
-_mm512_set_ps (float __A, float __B, float __C, float __D,
-        float __E, float __F, float __G, float __H,
-        float __I, float __J, float __K, float __L,
-        float __M, float __N, float __O, float __P)
-{
+static __inline__ __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_set_ps(float __A, float __B, float __C, float __D, float __E, float __F,
+              float __G, float __H, float __I, float __J, float __K, float __L,
+              float __M, float __N, float __O, float __P) {
   return __extension__ (__m512)
   { __P, __O, __N, __M, __L, __K, __J, __I,
     __H, __G, __F, __E, __D, __C, __B, __A };
@@ -9337,19 +9278,23 @@ _mm512_mask_abs_pd(__m512d __W, __mmask8 __K, __m512d __A)
  * This takes log2(n) steps where n is the number of elements in the vector.
  */
 
-static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_add_epi64(__m512i __W) {
+static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_reduce_add_epi64(__m512i __W) {
   return __builtin_reduce_add((__v8di)__W);
 }
 
-static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_mul_epi64(__m512i __W) {
+static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_reduce_mul_epi64(__m512i __W) {
   return __builtin_reduce_mul((__v8di)__W);
 }
 
-static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_and_epi64(__m512i __W) {
+static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_reduce_and_epi64(__m512i __W) {
   return __builtin_reduce_and((__v8di)__W);
 }
 
-static __inline__ long long __DEFAULT_FN_ATTRS512 _mm512_reduce_or_epi64(__m512i __W) {
+static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_reduce_or_epi64(__m512i __W) {
   return __builtin_reduce_or((__v8di)__W);
 }
 
@@ -9400,22 +9345,22 @@ _mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W) {
   return __builtin_ia32_reduce_fmul_pd512(1.0, __W);
 }
 
-static __inline__ int __DEFAULT_FN_ATTRS512
+static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_add_epi32(__m512i __W) {
   return __builtin_reduce_add((__v16si)__W);
 }
 
-static __inline__ int __DEFAULT_FN_ATTRS512
+static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_mul_epi32(__m512i __W) {
   return __builtin_reduce_mul((__v16si)__W);
 }
 
-static __inline__ int __DEFAULT_FN_ATTRS512
+static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_and_epi32(__m512i __W) {
   return __builtin_reduce_and((__v16si)__W);
 }
 
-static __inline__ int __DEFAULT_FN_ATTRS512
+static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_or_epi32(__m512i __W) {
   return __builtin_reduce_or((__v16si)__W);
 }
@@ -9466,22 +9411,22 @@ _mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W) {
   return __builtin_ia32_reduce_fmul_ps512(1.0f, __W);
 }
 
-static __inline__ long long __DEFAULT_FN_ATTRS512
+static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_max_epi64(__m512i __V) {
   return __builtin_reduce_max((__v8di)__V);
 }
 
-static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
+static __inline__ unsigned long long __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_max_epu64(__m512i __V) {
   return __builtin_reduce_max((__v8du)__V);
 }
 
-static __inline__ long long __DEFAULT_FN_ATTRS512
+static __inline__ long long __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_min_epi64(__m512i __V) {
   return __builtin_reduce_min((__v8di)__V);
 }
 
-static __inline__ unsigned long long __DEFAULT_FN_ATTRS512
+static __inline__ unsigned long long __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_min_epu64(__m512i __V) {
   return __builtin_reduce_min((__v8du)__V);
 }
@@ -9509,22 +9454,22 @@ _mm512_mask_reduce_min_epu64(__mmask8 __M, __m512i __V) {
   __V = _mm512_mask_mov_epi64(_mm512_set1_epi64(-1LL), __M, __V);
   return __builtin_reduce_min((__v8du)__V);
 }
-static __inline__ int __DEFAULT_FN_ATTRS512
+static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_max_epi32(__m512i __V) {
   return __builtin_reduce_max((__v16si)__V);
 }
 
-static __inline__ unsigned int __DEFAULT_FN_ATTRS512
+static __inline__ unsigned int __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_max_epu32(__m512i __V) {
   return __builtin_reduce_max((__v16su)__V);
 }
 
-static __inline__ int __DEFAULT_FN_ATTRS512
+static __inline__ int __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_min_epi32(__m512i __V) {
   return __builtin_reduce_min((__v16si)__V);
 }
 
-static __inline__ unsigned int __DEFAULT_FN_ATTRS512
+static __inline__ unsigned int __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_reduce_min_epu32(__m512i __V) {
   return __builtin_reduce_min((__v16su)__V);
 }
diff --git a/clang/lib/Headers/avx512fp16intrin.h b/clang/lib/Headers/avx512fp16intrin.h
index 92df320b45006..e1b198b7fc5c9 100644
--- a/clang/lib/Headers/avx512fp16intrin.h
+++ b/clang/lib/Headers/avx512fp16intrin.h
@@ -33,15 +33,26 @@ typedef _Float16 __m512h_u __attribute__((__vector_size__(64), __aligned__(1)));
                  __target__("avx512fp16,no-evex512"),                          \
                  __min_vector_width__(128)))
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512 constexpr
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#else
+#define __DEFAULT_FN_ATTRS512_CONSTEXPR __DEFAULT_FN_ATTRS512
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#endif
+
 static __inline__ _Float16 __DEFAULT_FN_ATTRS512 _mm512_cvtsh_h(__m512h __a) {
   return __a[0];
 }
 
-static __inline __m128h __DEFAULT_FN_ATTRS128 _mm_setzero_ph(void) {
+static __inline __m128h __DEFAULT_FN_ATTRS128_CONSTEXPR _mm_setzero_ph(void) {
   return (__m128h){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
 }
 
-static __inline __m256h __DEFAULT_FN_ATTRS256 _mm256_setzero_ph(void) {
+static __inline __m256h __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_setzero_ph(void) {
   return (__m256h){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
 }
@@ -50,7 +61,8 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_undefined_ph(void) {
   return (__m256h)__builtin_ia32_undef256();
 }
 
-static __inline __m512h __DEFAULT_FN_ATTRS512 _mm512_setzero_ph(void) {
+static __inline __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_setzero_ph(void) {
   return (__m512h){0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
                    0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
@@ -64,14 +76,15 @@ static __inline__ __m512h __DEFAULT_FN_ATTRS512 _mm512_undefined_ph(void) {
   return (__m512h)__builtin_ia32_undef512();
 }
 
-static __inline __m512h __DEFAULT_FN_ATTRS512 _mm512_set1_ph(_Float16 __h) {
+static __inline __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
+_mm512_set1_ph(_Float16 __h) {
   return (__m512h)(__v32hf){__h, __h, __h, __h, __h, __h, __h, __h,
                             __h, __h, __h, __h, __h, __h, __h, __h,
                             __h, __h, __h, __h, __h, __h, __h, __h,
                             __h, __h, __h, __h, __h, __h, __h, __h};
 }
 
-static __inline __m512h __DEFAULT_FN_ATTRS512
+static __inline __m512h __DEFAULT_FN_ATTRS512_CONSTEXPR
 _mm512_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16 __h4,
               _Float16 __h5, _Float16 __h6, _Float16 __h7, _Float16 __h8,
               _Float16 __h9, _Float16 __h10, _Float16 __h11, _Float16 __h12,
@@ -3348,6 +3361,9 @@ _mm512_permutexvar_ph(__m512i __A, __m512h __B) {
 #undef __DEFAULT_FN_ATTRS128
 #undef __DEFAULT_FN_ATTRS256
 #undef __DEFAULT_FN_ATTRS512
+#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS512_CONSTEXPR
 
 #endif
 #endif
diff --git a/clang/lib/Headers/avx512vbmi2intrin.h b/clang/lib/Headers/avx512vbmi2intrin.h
index 11598c888787c..f9a5f82b61f82 100644
--- a/clang/lib/Headers/avx512vbmi2intrin.h
+++ b/clang/lib/Headers/avx512vbmi2intrin.h
@@ -215,8 +215,8 @@ _mm512_maskz_expandloadu_epi8(__mmask64 __U, void const *__P)
 static __inline__ __m512i __DEFAULT_FN_ATTRS
 _mm512_shldv_epi64(__m512i __A, __m512i __B, __m512i __C)
 {
-  return (__m512i)__builtin_ia32_vpshldvq512((__v8di)__A, (__v8di)__B,
-                                             (__v8di)__C);
+  return (__m512i)__builtin_elementwise_fshl((__v8du)__A, (__v8du)__B,
+                                             (__v8du)__C);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS
@@ -238,8 +238,8 @@ _mm512_maskz_shldv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C)
 static __inline__ __m512i __DEFAULT_FN_ATTRS
 _mm512_shldv_epi32(__m512i __A, __m512i __B, __m512i __C)
 {
-  return (__m512i)__builtin_ia32_vpshldvd512((__v16si)__A, (__v16si)__B,
-                                             (__v16si)__C);
+  return (__m512i)__builtin_elementwise_fshl((__v16su)__A, (__v16su)__B,
+                                             (__v16su)__C);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS
@@ -261,8 +261,8 @@ _mm512_maskz_shldv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C)
 static __inline__ __m512i __DEFAULT_FN_ATTRS
 _mm512_shldv_epi16(__m512i __A, __m512i __B, __m512i __C)
 {
-  return (__m512i)__builtin_ia32_vpshldvw512((__v32hi)__A, (__v32hi)__B,
-                                             (__v32hi)__C);
+  return (__m512i)__builtin_elementwise_fshl((__v32hu)__A, (__v32hu)__B,
+                                             (__v32hu)__C);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS
@@ -284,8 +284,9 @@ _mm512_maskz_shldv_epi16(__mmask32 __U, __m512i __A, __m512i __B, __m512i __C)
 static __inline__ __m512i __DEFAULT_FN_ATTRS
 _mm512_shrdv_epi64(__m512i __A, __m512i __B, __m512i __C)
 {
-  return (__m512i)__builtin_ia32_vpshrdvq512((__v8di)__A, (__v8di)__B,
-                                             (__v8di)__C);
+  // Ops __A and __B are swapped.
+  return (__m512i)__builtin_elementwise_fshr((__v8du)__B, (__v8du)__A,
+                                             (__v8du)__C);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS
@@ -307,8 +308,9 @@ _mm512_maskz_shrdv_epi64(__mmask8 __U, __m512i __A, __m512i __B, __m512i __C)
 static __inline__ __m512i __DEFAULT_FN_ATTRS
 _mm512_shrdv_epi32(__m512i __A, __m512i __B, __m512i __C)
 {
-  return (__m512i)__builtin_ia32_vpshrdvd512((__v16si)__A, (__v16si)__B,
-                                             (__v16si)__C);
+  // Ops __A and __B are swapped.
+  return (__m512i)__builtin_elementwise_fshr((__v16su)__B, (__v16su)__A,
+                                             (__v16su)__C);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS
@@ -330,8 +332,9 @@ _mm512_maskz_shrdv_epi32(__mmask16 __U, __m512i __A, __m512i __B, __m512i __C)
 static __inline__ __m512i __DEFAULT_FN_ATTRS
 _mm512_shrdv_epi16(__m512i __A, __m512i __B, __m512i __C)
 {
-  return (__m512i)__builtin_ia32_vpshrdvw512((__v32hi)__A, (__v32hi)__B,
-                                             (__v32hi)__C);
+  // Ops __A and __B are swapped.
+  return (__m512i)__builtin_elementwise_fshr((__v32hu)__B, (__v32hu)__A,
+                                             (__v32hu)__C);
 }
 
 static __inline__ __m512i __DEFAULT_FN_ATTRS
diff --git a/clang/lib/Headers/avx512vlbitalgintrin.h b/clang/lib/Headers/avx512vlbitalgintrin.h
index 1b01fe0b9d815..21bf858a20c59 100644
--- a/clang/lib/Headers/avx512vlbitalgintrin.h
+++ b/clang/lib/Headers/avx512vlbitalgintrin.h
@@ -24,92 +24,76 @@
                  __target__("avx512vl,avx512bitalg,no-evex512"),               \
                  __min_vector_width__(256)))
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_popcnt_epi16(__m256i __A)
-{
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#else
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#endif
+
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_popcnt_epi16(__m256i __A) {
   return (__m256i)__builtin_elementwise_popcount((__v16hu)__A);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_mask_popcnt_epi16(__m256i __A, __mmask16 __U, __m256i __B)
-{
-  return (__m256i) __builtin_ia32_selectw_256((__mmask16) __U,
-              (__v16hi) _mm256_popcnt_epi16(__B),
-              (__v16hi) __A);
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_mask_popcnt_epi16(__m256i __A, __mmask16 __U, __m256i __B) {
+  return (__m256i)__builtin_ia32_selectw_256(
+      (__mmask16)__U, (__v16hi)_mm256_popcnt_epi16(__B), (__v16hi)__A);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_maskz_popcnt_epi16(__mmask16 __U, __m256i __B)
-{
-  return _mm256_mask_popcnt_epi16((__m256i) _mm256_setzero_si256(),
-              __U,
-              __B);
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_maskz_popcnt_epi16(__mmask16 __U, __m256i __B) {
+  return _mm256_mask_popcnt_epi16((__m256i)_mm256_setzero_si256(), __U, __B);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_popcnt_epi16(__m128i __A)
-{
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_popcnt_epi16(__m128i __A) {
   return (__m128i)__builtin_elementwise_popcount((__v8hu)__A);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_mask_popcnt_epi16(__m128i __A, __mmask8 __U, __m128i __B)
-{
-  return (__m128i) __builtin_ia32_selectw_128((__mmask8) __U,
-              (__v8hi) _mm_popcnt_epi16(__B),
-              (__v8hi) __A);
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_mask_popcnt_epi16(__m128i __A, __mmask8 __U, __m128i __B) {
+  return (__m128i)__builtin_ia32_selectw_128(
+      (__mmask8)__U, (__v8hi)_mm_popcnt_epi16(__B), (__v8hi)__A);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_maskz_popcnt_epi16(__mmask8 __U, __m128i __B)
-{
-  return _mm_mask_popcnt_epi16((__m128i) _mm_setzero_si128(),
-              __U,
-              __B);
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_maskz_popcnt_epi16(__mmask8 __U, __m128i __B) {
+  return _mm_mask_popcnt_epi16((__m128i)_mm_setzero_si128(), __U, __B);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_popcnt_epi8(__m256i __A)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_popcnt_epi8(__m256i __A) {
   return (__m256i)__builtin_elementwise_popcount((__v32qu)__A);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_mask_popcnt_epi8(__m256i __A, __mmask32 __U, __m256i __B)
-{
-  return (__m256i) __builtin_ia32_selectb_256((__mmask32) __U,
-              (__v32qi) _mm256_popcnt_epi8(__B),
-              (__v32qi) __A);
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_mask_popcnt_epi8(__m256i __A, __mmask32 __U, __m256i __B) {
+  return (__m256i)__builtin_ia32_selectb_256(
+      (__mmask32)__U, (__v32qi)_mm256_popcnt_epi8(__B), (__v32qi)__A);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_maskz_popcnt_epi8(__mmask32 __U, __m256i __B)
-{
-  return _mm256_mask_popcnt_epi8((__m256i) _mm256_setzero_si256(),
-              __U,
-              __B);
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_maskz_popcnt_epi8(__mmask32 __U, __m256i __B) {
+  return _mm256_mask_popcnt_epi8((__m256i)_mm256_setzero_si256(), __U, __B);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_popcnt_epi8(__m128i __A)
-{
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_popcnt_epi8(__m128i __A) {
   return (__m128i)__builtin_elementwise_popcount((__v16qu)__A);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_mask_popcnt_epi8(__m128i __A, __mmask16 __U, __m128i __B)
-{
-  return (__m128i) __builtin_ia32_selectb_128((__mmask16) __U,
-              (__v16qi) _mm_popcnt_epi8(__B),
-              (__v16qi) __A);
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_mask_popcnt_epi8(__m128i __A, __mmask16 __U, __m128i __B) {
+  return (__m128i)__builtin_ia32_selectb_128(
+      (__mmask16)__U, (__v16qi)_mm_popcnt_epi8(__B), (__v16qi)__A);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_maskz_popcnt_epi8(__mmask16 __U, __m128i __B)
-{
-  return _mm_mask_popcnt_epi8((__m128i) _mm_setzero_si128(),
-              __U,
-              __B);
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_maskz_popcnt_epi8(__mmask16 __U, __m128i __B) {
+  return _mm_mask_popcnt_epi8((__m128i)_mm_setzero_si128(), __U, __B);
 }
 
 static __inline__ __mmask32 __DEFAULT_FN_ATTRS256
@@ -147,5 +131,7 @@ _mm_bitshuffle_epi64_mask(__m128i __A, __m128i __B)
 
 #undef __DEFAULT_FN_ATTRS128
 #undef __DEFAULT_FN_ATTRS256
+#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
 
 #endif
diff --git a/clang/lib/Headers/avx512vlbwintrin.h b/clang/lib/Headers/avx512vlbwintrin.h
index 9aedba0669991..2e2052ad1b682 100644
--- a/clang/lib/Headers/avx512vlbwintrin.h
+++ b/clang/lib/Headers/avx512vlbwintrin.h
@@ -24,6 +24,14 @@
                  __target__("avx512vl,avx512bw,no-evex512"),                   \
                  __min_vector_width__(256)))
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#else
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#endif
+
 /* Integer compare */
 
 #define _mm_cmp_epi8_mask(a, b, p) \
@@ -1592,67 +1600,62 @@ _mm256_maskz_mulhrs_epi16(__mmask16 __U, __m256i __X, __m256i __Y) {
                                          (__v16hi)_mm256_setzero_si256());
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_mask_mulhi_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
-                                             (__v8hi)_mm_mulhi_epu16(__A, __B),
-                                             (__v8hi)__W);
+  return (__m128i)__builtin_ia32_selectw_128(
+      (__mmask8)__U, (__v8hi)_mm_mulhi_epu16(__A, __B), (__v8hi)__W);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_maskz_mulhi_epu16(__mmask8 __U, __m128i __A, __m128i __B) {
   return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
                                              (__v8hi)_mm_mulhi_epu16(__A, __B),
                                              (__v8hi)_mm_setzero_si128());
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_mask_mulhi_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
-                                          (__v16hi)_mm256_mulhi_epu16(__A, __B),
-                                          (__v16hi)__W);
+  return (__m256i)__builtin_ia32_selectw_256(
+      (__mmask16)__U, (__v16hi)_mm256_mulhi_epu16(__A, __B), (__v16hi)__W);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_maskz_mulhi_epu16(__mmask16 __U, __m256i __A, __m256i __B) {
-  return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
-                                          (__v16hi)_mm256_mulhi_epu16(__A, __B),
-                                          (__v16hi)_mm256_setzero_si256());
+  return (__m256i)__builtin_ia32_selectw_256(
+      (__mmask16)__U, (__v16hi)_mm256_mulhi_epu16(__A, __B),
+      (__v16hi)_mm256_setzero_si256());
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_mask_mulhi_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
-                                             (__v8hi)_mm_mulhi_epi16(__A, __B),
-                                             (__v8hi)__W);
+  return (__m128i)__builtin_ia32_selectw_128(
+      (__mmask8)__U, (__v8hi)_mm_mulhi_epi16(__A, __B), (__v8hi)__W);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_maskz_mulhi_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
   return (__m128i)__builtin_ia32_selectw_128((__mmask8)__U,
                                              (__v8hi)_mm_mulhi_epi16(__A, __B),
                                              (__v8hi)_mm_setzero_si128());
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_mask_mulhi_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
-                                          (__v16hi)_mm256_mulhi_epi16(__A, __B),
-                                          (__v16hi)__W);
+  return (__m256i)__builtin_ia32_selectw_256(
+      (__mmask16)__U, (__v16hi)_mm256_mulhi_epi16(__A, __B), (__v16hi)__W);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_maskz_mulhi_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  return (__m256i)__builtin_ia32_selectw_256((__mmask16)__U,
-                                          (__v16hi)_mm256_mulhi_epi16(__A, __B),
-                                          (__v16hi)_mm256_setzero_si256());
+  return (__m256i)__builtin_ia32_selectw_256(
+      (__mmask16)__U, (__v16hi)_mm256_mulhi_epi16(__A, __B),
+      (__v16hi)_mm256_setzero_si256());
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_mask_unpackhi_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  return (__m128i)__builtin_ia32_selectb_128((__mmask16)__U,
-                                           (__v16qi)_mm_unpackhi_epi8(__A, __B),
-                                           (__v16qi)__W);
+  return (__m128i)__builtin_ia32_selectb_128(
+      (__mmask16)__U, (__v16qi)_mm_unpackhi_epi8(__A, __B), (__v16qi)__W);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -3163,5 +3166,7 @@ _mm256_mask_reduce_min_epu8(__mmask32 __M, __m256i __V) {
 
 #undef __DEFAULT_FN_ATTRS128
 #undef __DEFAULT_FN_ATTRS256
+#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
 
 #endif /* __AVX512VLBWINTRIN_H */
diff --git a/clang/lib/Headers/avx512vldqintrin.h b/clang/lib/Headers/avx512vldqintrin.h
index 272cdd89e2d24..ceebd09e1d3af 100644
--- a/clang/lib/Headers/avx512vldqintrin.h
+++ b/clang/lib/Headers/avx512vldqintrin.h
@@ -24,6 +24,14 @@
                  __target__("avx512vl,avx512dq,no-evex512"),                   \
                  __min_vector_width__(256)))
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#else
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#endif
+
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_mullo_epi64 (__m256i __A, __m256i __B) {
   return (__m256i) ((__v4du) __A * (__v4du) __B);
@@ -956,9 +964,8 @@ _mm256_movepi64_mask (__m256i __A)
   return (__mmask8) __builtin_ia32_cvtq2mask256 ((__v4di) __A);
 }
 
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
-_mm256_broadcast_f32x2 (__m128 __A)
-{
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcast_f32x2(__m128 __A) {
   return (__m256)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
                                          0, 1, 0, 1, 0, 1, 0, 1);
 }
@@ -979,9 +986,8 @@ _mm256_maskz_broadcast_f32x2 (__mmask8 __M, __m128 __A)
                                              (__v8sf)_mm256_setzero_ps());
 }
 
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
-_mm256_broadcast_f64x2(__m128d __A)
-{
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcast_f64x2(__m128d __A) {
   return (__m256d)__builtin_shufflevector((__v2df)__A, (__v2df)__A,
                                           0, 1, 0, 1);
 }
@@ -1002,9 +1008,8 @@ _mm256_maskz_broadcast_f64x2 (__mmask8 __M, __m128d __A)
                                             (__v4df)_mm256_setzero_pd());
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128
-_mm_broadcast_i32x2 (__m128i __A)
-{
+static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_broadcast_i32x2(__m128i __A) {
   return (__m128i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
                                           0, 1, 0, 1);
 }
@@ -1025,9 +1030,8 @@ _mm_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A)
                                              (__v4si)_mm_setzero_si128());
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_broadcast_i32x2 (__m128i __A)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcast_i32x2(__m128i __A) {
   return (__m256i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
                                           0, 1, 0, 1, 0, 1, 0, 1);
 }
@@ -1048,9 +1052,8 @@ _mm256_maskz_broadcast_i32x2 (__mmask8 __M, __m128i __A)
                                              (__v8si)_mm256_setzero_si256());
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_broadcast_i64x2(__m128i __A)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcast_i64x2(__m128i __A) {
   return (__m256i)__builtin_shufflevector((__v2di)__A, (__v2di)__A,
                                           0, 1, 0, 1);
 }
@@ -1169,5 +1172,7 @@ _mm256_maskz_broadcast_i64x2 (__mmask8 __M, __m128i __A)
 
 #undef __DEFAULT_FN_ATTRS128
 #undef __DEFAULT_FN_ATTRS256
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
 
 #endif
diff --git a/clang/lib/Headers/avx512vlfp16intrin.h b/clang/lib/Headers/avx512vlfp16intrin.h
index a12acb7d9a24a..1f8cca711f9f3 100644
--- a/clang/lib/Headers/avx512vlfp16intrin.h
+++ b/clang/lib/Headers/avx512vlfp16intrin.h
@@ -1419,8 +1419,8 @@ _mm256_maskz_cvtxps_ph(__mmask8 __U, __m256 __A) {
 static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_fmadd_ph(__m128h __A,
                                                              __m128h __B,
                                                              __m128h __C) {
-  return (__m128h)__builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B,
-                                          (__v8hf)__C);
+  return (__m128h)__builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B,
+                                            (__v8hf)__C);
 }
 
 static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_fmadd_ph(__m128h __A,
@@ -1429,7 +1429,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_fmadd_ph(__m128h __A,
                                                                   __m128h __C) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
+      __builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
       (__v8hf)__A);
 }
 
@@ -1437,7 +1437,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
 _mm_mask3_fmadd_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
+      __builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
       (__v8hf)__C);
 }
 
@@ -1445,15 +1445,15 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
 _mm_maskz_fmadd_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
+      __builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
       (__v8hf)_mm_setzero_ph());
 }
 
 static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_fmsub_ph(__m128h __A,
                                                              __m128h __B,
                                                              __m128h __C) {
-  return (__m128h)__builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B,
-                                          -(__v8hf)__C);
+  return (__m128h)__builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B,
+                                            -(__v8hf)__C);
 }
 
 static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_mask_fmsub_ph(__m128h __A,
@@ -1476,7 +1476,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
 _mm_mask3_fnmadd_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph(-(__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
+      __builtin_elementwise_fma(-(__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
       (__v8hf)__C);
 }
 
@@ -1484,7 +1484,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
 _mm_maskz_fnmadd_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph(-(__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
+      __builtin_elementwise_fma(-(__v8hf)__A, (__v8hf)__B, (__v8hf)__C),
       (__v8hf)_mm_setzero_ph());
 }
 
@@ -1492,22 +1492,22 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
 _mm_maskz_fnmsub_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph(-(__v8hf)__A, (__v8hf)__B, -(__v8hf)__C),
+      __builtin_elementwise_fma(-(__v8hf)__A, (__v8hf)__B, -(__v8hf)__C),
       (__v8hf)_mm_setzero_ph());
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_fmadd_ph(__m256h __A,
                                                                 __m256h __B,
                                                                 __m256h __C) {
-  return (__m256h)__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B,
-                                             (__v16hf)__C);
+  return (__m256h)__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B,
+                                            (__v16hf)__C);
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_mask_fmadd_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
+      __builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
       (__v16hf)__A);
 }
 
@@ -1515,7 +1515,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_mask3_fmadd_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
+      __builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
       (__v16hf)__C);
 }
 
@@ -1523,22 +1523,22 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_maskz_fmadd_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
+      __builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
       (__v16hf)_mm256_setzero_ph());
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_fmsub_ph(__m256h __A,
                                                                 __m256h __B,
                                                                 __m256h __C) {
-  return (__m256h)__builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B,
-                                             -(__v16hf)__C);
+  return (__m256h)__builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B,
+                                            -(__v16hf)__C);
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_mask_fmsub_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
+      __builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
       (__v16hf)__A);
 }
 
@@ -1546,7 +1546,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_maskz_fmsub_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
+      __builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
       (__v16hf)_mm256_setzero_ph());
 }
 
@@ -1554,7 +1554,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_mask3_fnmadd_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256(-(__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
+      __builtin_elementwise_fma(-(__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
       (__v16hf)__C);
 }
 
@@ -1562,7 +1562,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_maskz_fnmadd_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256(-(__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
+      __builtin_elementwise_fma(-(__v16hf)__A, (__v16hf)__B, (__v16hf)__C),
       (__v16hf)_mm256_setzero_ph());
 }
 
@@ -1570,7 +1570,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_maskz_fnmsub_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256(-(__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
+      __builtin_elementwise_fma(-(__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
       (__v16hf)_mm256_setzero_ph());
 }
 
@@ -1684,7 +1684,7 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
 _mm_mask3_fmsub_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph((__v8hf)__A, (__v8hf)__B, -(__v8hf)__C),
+      __builtin_elementwise_fma((__v8hf)__A, (__v8hf)__B, -(__v8hf)__C),
       (__v8hf)__C);
 }
 
@@ -1692,7 +1692,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_mask3_fmsub_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
+      __builtin_elementwise_fma((__v16hf)__A, (__v16hf)__B, -(__v16hf)__C),
       (__v16hf)__C);
 }
 
@@ -1715,45 +1715,45 @@ _mm256_mask3_fmsubadd_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
 static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_fnmadd_ph(__m128h __A,
                                                               __m128h __B,
                                                               __m128h __C) {
-  return (__m128h)__builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B,
-                                          (__v8hf)__C);
+  return (__m128h)__builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B,
+                                            (__v8hf)__C);
 }
 
 static __inline__ __m128h __DEFAULT_FN_ATTRS128
 _mm_mask_fnmadd_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B, (__v8hf)__C),
+      __builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B, (__v8hf)__C),
       (__v8hf)__A);
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_fnmadd_ph(__m256h __A,
                                                                  __m256h __B,
                                                                  __m256h __C) {
-  return (__m256h)__builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B,
-                                             (__v16hf)__C);
+  return (__m256h)__builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B,
+                                            (__v16hf)__C);
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_mask_fnmadd_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B, (__v16hf)__C),
+      __builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B, (__v16hf)__C),
       (__v16hf)__A);
 }
 
 static __inline__ __m128h __DEFAULT_FN_ATTRS128 _mm_fnmsub_ph(__m128h __A,
                                                               __m128h __B,
                                                               __m128h __C) {
-  return (__m128h)__builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B,
-                                          -(__v8hf)__C);
+  return (__m128h)__builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B,
+                                            -(__v8hf)__C);
 }
 
 static __inline__ __m128h __DEFAULT_FN_ATTRS128
 _mm_mask_fnmsub_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B, -(__v8hf)__C),
+      __builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B, -(__v8hf)__C),
       (__v8hf)__A);
 }
 
@@ -1761,22 +1761,22 @@ static __inline__ __m128h __DEFAULT_FN_ATTRS128
 _mm_mask3_fnmsub_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
   return (__m128h)__builtin_ia32_selectph_128(
       (__mmask8)__U,
-      __builtin_ia32_vfmaddph((__v8hf)__A, -(__v8hf)__B, -(__v8hf)__C),
+      __builtin_elementwise_fma((__v8hf)__A, -(__v8hf)__B, -(__v8hf)__C),
       (__v8hf)__C);
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256 _mm256_fnmsub_ph(__m256h __A,
                                                                  __m256h __B,
                                                                  __m256h __C) {
-  return (__m256h)__builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B,
-                                             -(__v16hf)__C);
+  return (__m256h)__builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B,
+                                            -(__v16hf)__C);
 }
 
 static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_mask_fnmsub_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B, -(__v16hf)__C),
+      __builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B, -(__v16hf)__C),
       (__v16hf)__A);
 }
 
@@ -1784,7 +1784,7 @@ static __inline__ __m256h __DEFAULT_FN_ATTRS256
 _mm256_mask3_fnmsub_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
   return (__m256h)__builtin_ia32_selectph_256(
       (__mmask16)__U,
-      __builtin_ia32_vfmaddph256((__v16hf)__A, -(__v16hf)__B, -(__v16hf)__C),
+      __builtin_elementwise_fma((__v16hf)__A, -(__v16hf)__B, -(__v16hf)__C),
       (__v16hf)__C);
 }
 
diff --git a/clang/lib/Headers/avx512vlintrin.h b/clang/lib/Headers/avx512vlintrin.h
index 2a5f7b43f63fc..a1f2a1c92a863 100644
--- a/clang/lib/Headers/avx512vlintrin.h
+++ b/clang/lib/Headers/avx512vlintrin.h
@@ -23,6 +23,14 @@
                  __target__("avx512vl,no-evex512"),                            \
                  __min_vector_width__(256)))
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#else
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#endif
+
 typedef short __v2hi __attribute__((__vector_size__(4)));
 typedef char __v4qi __attribute__((__vector_size__(4)));
 typedef char __v2qi __attribute__((__vector_size__(2)));
@@ -899,321 +907,289 @@ _mm_maskz_xor_epi64(__mmask8 __U, __m128i __A, __m128i __B)
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_mask_fmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd ((__v2df) __A,
-                                             (__v2df) __B,
-                                             (__v2df) __C),
-                    (__v2df) __A);
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v2df)__A, (__v2df)__B, (__v2df)__C),
+      (__v2df)__A);
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_mask3_fmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd ((__v2df) __A,
-                                             (__v2df) __B,
-                                             (__v2df) __C),
-                    (__v2df) __C);
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v2df)__A, (__v2df)__B, (__v2df)__C),
+      (__v2df)__C);
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_maskz_fmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd ((__v2df) __A,
-                                             (__v2df) __B,
-                                             (__v2df) __C),
-                    (__v2df)_mm_setzero_pd());
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v2df)__A, (__v2df)__B, (__v2df)__C),
+      (__v2df)_mm_setzero_pd());
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_mask_fmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd ((__v2df) __A,
-                                             (__v2df) __B,
-                                             -(__v2df) __C),
-                    (__v2df) __A);
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v2df)__A, (__v2df)__B, -(__v2df)__C),
+      (__v2df)__A);
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_maskz_fmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd ((__v2df) __A,
-                                             (__v2df) __B,
-                                             -(__v2df) __C),
-                    (__v2df)_mm_setzero_pd());
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v2df)__A, (__v2df)__B, -(__v2df)__C),
+      (__v2df)_mm_setzero_pd());
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_mask3_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd (-(__v2df) __A,
-                                             (__v2df) __B,
-                                             (__v2df) __C),
-                    (__v2df) __C);
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B, (__v2df)__C),
+      (__v2df)__C);
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_maskz_fnmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd (-(__v2df) __A,
-                                             (__v2df) __B,
-                                             (__v2df) __C),
-                    (__v2df)_mm_setzero_pd());
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B, (__v2df)__C),
+      (__v2df)_mm_setzero_pd());
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_maskz_fnmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd (-(__v2df) __A,
-                                             (__v2df) __B,
-                                             -(__v2df) __C),
-                    (__v2df)_mm_setzero_pd());
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B, -(__v2df)__C),
+      (__v2df)_mm_setzero_pd());
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_mask_fmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 ((__v4df) __A,
-                                                (__v4df) __B,
-                                                (__v4df) __C),
-                    (__v4df) __A);
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4df)__A, (__v4df)__B, (__v4df)__C),
+      (__v4df)__A);
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_mask3_fmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 ((__v4df) __A,
-                                                (__v4df) __B,
-                                                (__v4df) __C),
-                    (__v4df) __C);
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4df)__A, (__v4df)__B, (__v4df)__C),
+      (__v4df)__C);
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_maskz_fmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 ((__v4df) __A,
-                                                (__v4df) __B,
-                                                (__v4df) __C),
-                    (__v4df)_mm256_setzero_pd());
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4df)__A, (__v4df)__B, (__v4df)__C),
+      (__v4df)_mm256_setzero_pd());
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_mask_fmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 ((__v4df) __A,
-                                                (__v4df) __B,
-                                                -(__v4df) __C),
-                    (__v4df) __A);
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4df)__A, (__v4df)__B, -(__v4df)__C),
+      (__v4df)__A);
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_maskz_fmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 ((__v4df) __A,
-                                                (__v4df) __B,
-                                                -(__v4df) __C),
-                    (__v4df)_mm256_setzero_pd());
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4df)__A, (__v4df)__B, -(__v4df)__C),
+      (__v4df)_mm256_setzero_pd());
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_mask3_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 (-(__v4df) __A,
-                                                (__v4df) __B,
-                                                (__v4df) __C),
-                    (__v4df) __C);
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B, (__v4df)__C),
+      (__v4df)__C);
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_maskz_fnmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 (-(__v4df) __A,
-                                                (__v4df) __B,
-                                                (__v4df) __C),
-                    (__v4df)_mm256_setzero_pd());
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B, (__v4df)__C),
+      (__v4df)_mm256_setzero_pd());
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_maskz_fnmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 (-(__v4df) __A,
-                                                (__v4df) __B,
-                                                -(__v4df) __C),
-                    (__v4df)_mm256_setzero_pd());
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B, -(__v4df)__C),
+      (__v4df)_mm256_setzero_pd());
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_mask_fmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps ((__v4sf) __A,
-                                             (__v4sf) __B,
-                                             (__v4sf) __C),
-                    (__v4sf) __A);
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B, (__v4sf)__C),
+      (__v4sf)__A);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_mask3_fmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps ((__v4sf) __A,
-                                             (__v4sf) __B,
-                                             (__v4sf) __C),
-                    (__v4sf) __C);
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B, (__v4sf)__C),
+      (__v4sf)__C);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_maskz_fmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps ((__v4sf) __A,
-                                             (__v4sf) __B,
-                                             (__v4sf) __C),
-                    (__v4sf)_mm_setzero_ps());
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B, (__v4sf)__C),
+      (__v4sf)_mm_setzero_ps());
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_mask_fmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps ((__v4sf) __A,
-                                             (__v4sf) __B,
-                                             -(__v4sf) __C),
-                    (__v4sf) __A);
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C),
+      (__v4sf)__A);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_maskz_fmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps ((__v4sf) __A,
-                                             (__v4sf) __B,
-                                             -(__v4sf) __C),
-                    (__v4sf)_mm_setzero_ps());
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C),
+      (__v4sf)_mm_setzero_ps());
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_mask3_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps (-(__v4sf) __A,
-                                             (__v4sf) __B,
-                                             (__v4sf) __C),
-                    (__v4sf) __C);
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C),
+      (__v4sf)__C);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_maskz_fnmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps (-(__v4sf) __A,
-                                             (__v4sf) __B,
-                                             (__v4sf) __C),
-                    (__v4sf)_mm_setzero_ps());
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C),
+      (__v4sf)_mm_setzero_ps());
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_maskz_fnmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps (-(__v4sf) __A,
-                                             (__v4sf) __B,
-                                             -(__v4sf) __C),
-                    (__v4sf)_mm_setzero_ps());
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C),
+      (__v4sf)_mm_setzero_ps());
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_mask_fmadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 ((__v8sf) __A,
-                                                (__v8sf) __B,
-                                                (__v8sf) __C),
-                    (__v8sf) __A);
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B, (__v8sf)__C),
+      (__v8sf)__A);
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_mask3_fmadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 ((__v8sf) __A,
-                                                (__v8sf) __B,
-                                                (__v8sf) __C),
-                    (__v8sf) __C);
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B, (__v8sf)__C),
+      (__v8sf)__C);
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_maskz_fmadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 ((__v8sf) __A,
-                                                (__v8sf) __B,
-                                                (__v8sf) __C),
-                    (__v8sf)_mm256_setzero_ps());
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B, (__v8sf)__C),
+      (__v8sf)_mm256_setzero_ps());
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_mask_fmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 ((__v8sf) __A,
-                                                (__v8sf) __B,
-                                                -(__v8sf) __C),
-                    (__v8sf) __A);
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C),
+      (__v8sf)__A);
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_maskz_fmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 ((__v8sf) __A,
-                                                (__v8sf) __B,
-                                                -(__v8sf) __C),
-                    (__v8sf)_mm256_setzero_ps());
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C),
+      (__v8sf)_mm256_setzero_ps());
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_mask3_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 (-(__v8sf) __A,
-                                                (__v8sf) __B,
-                                                (__v8sf) __C),
-                    (__v8sf) __C);
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C),
+      (__v8sf)__C);
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_maskz_fnmadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 (-(__v8sf) __A,
-                                                (__v8sf) __B,
-                                                (__v8sf) __C),
-                    (__v8sf)_mm256_setzero_ps());
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C),
+      (__v8sf)_mm256_setzero_ps());
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_maskz_fnmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 (-(__v8sf) __A,
-                                                (__v8sf) __B,
-                                                -(__v8sf) __C),
-                    (__v8sf)_mm256_setzero_ps());
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C),
+      (__v8sf)_mm256_setzero_ps());
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
@@ -1420,41 +1396,37 @@ _mm256_maskz_fmsubadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C)
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_mask3_fmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd ((__v2df) __A,
-                                             (__v2df) __B,
-                                             -(__v2df) __C),
-                    (__v2df) __C);
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v2df)__A, (__v2df)__B, -(__v2df)__C),
+      (__v2df)__C);
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_mask3_fmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 ((__v4df) __A,
-                                                (__v4df) __B,
-                                                -(__v4df) __C),
-                    (__v4df) __C);
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4df)__A, (__v4df)__B, -(__v4df)__C),
+      (__v4df)__C);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_mask3_fmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps ((__v4sf) __A,
-                                             (__v4sf) __B,
-                                             -(__v4sf) __C),
-                    (__v4sf) __C);
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C),
+      (__v4sf)__C);
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_mask3_fmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 ((__v8sf) __A,
-                                                (__v8sf) __B,
-                                                -(__v8sf) __C),
-                    (__v8sf) __C);
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C),
+      (__v8sf)__C);
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
@@ -1500,121 +1472,109 @@ _mm256_mask3_fmsubadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_mask_fnmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd ((__v2df) __A,
-                                             -(__v2df) __B,
-                                             (__v2df) __C),
-                    (__v2df) __A);
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v2df)__A, -(__v2df)__B, (__v2df)__C),
+      (__v2df)__A);
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_mask_fnmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 ((__v4df) __A,
-                                                -(__v4df) __B,
-                                                (__v4df) __C),
-                    (__v4df) __A);
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4df)__A, -(__v4df)__B, (__v4df)__C),
+      (__v4df)__A);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_mask_fnmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps ((__v4sf) __A,
-                                             -(__v4sf) __B,
-                                             (__v4sf) __C),
-                    (__v4sf) __A);
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4sf)__A, -(__v4sf)__B, (__v4sf)__C),
+      (__v4sf)__A);
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_mask_fnmadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 ((__v8sf) __A,
-                                                -(__v8sf) __B,
-                                                (__v8sf) __C),
-                    (__v8sf) __A);
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v8sf)__A, -(__v8sf)__B, (__v8sf)__C),
+      (__v8sf)__A);
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_mask_fnmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd ((__v2df) __A,
-                                             -(__v2df) __B,
-                                             -(__v2df) __C),
-                    (__v2df) __A);
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v2df)__A, -(__v2df)__B, -(__v2df)__C),
+      (__v2df)__A);
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
 _mm_mask3_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U)
 {
-  return (__m128d) __builtin_ia32_selectpd_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd ((__v2df) __A,
-                                             -(__v2df) __B,
-                                             -(__v2df) __C),
-                    (__v2df) __C);
+  return (__m128d)__builtin_ia32_selectpd_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v2df)__A, -(__v2df)__B, -(__v2df)__C),
+      (__v2df)__C);
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_mask_fnmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 ((__v4df) __A,
-                                                -(__v4df) __B,
-                                                -(__v4df) __C),
-                    (__v4df) __A);
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4df)__A, -(__v4df)__B, -(__v4df)__C),
+      (__v4df)__A);
 }
 
 static __inline__ __m256d __DEFAULT_FN_ATTRS256
 _mm256_mask3_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U)
 {
-  return (__m256d) __builtin_ia32_selectpd_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddpd256 ((__v4df) __A,
-                                                -(__v4df) __B,
-                                                -(__v4df) __C),
-                    (__v4df) __C);
+  return (__m256d)__builtin_ia32_selectpd_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4df)__A, -(__v4df)__B, -(__v4df)__C),
+      (__v4df)__C);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_mask_fnmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps ((__v4sf) __A,
-                                             -(__v4sf) __B,
-                                             -(__v4sf) __C),
-                    (__v4sf) __A);
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4sf)__A, -(__v4sf)__B, -(__v4sf)__C),
+      (__v4sf)__A);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
 _mm_mask3_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
 {
-  return (__m128) __builtin_ia32_selectps_128((__mmask8) __U,
-                    __builtin_ia32_vfmaddps ((__v4sf) __A,
-                                             -(__v4sf) __B,
-                                             -(__v4sf) __C),
-                    (__v4sf) __C);
+  return (__m128)__builtin_ia32_selectps_128(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v4sf)__A, -(__v4sf)__B, -(__v4sf)__C),
+      (__v4sf)__C);
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_mask_fnmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 ((__v8sf) __A,
-                                                -(__v8sf) __B,
-                                                -(__v8sf) __C),
-                    (__v8sf) __A);
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v8sf)__A, -(__v8sf)__B, -(__v8sf)__C),
+      (__v8sf)__A);
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
 _mm256_mask3_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U)
 {
-  return (__m256) __builtin_ia32_selectps_256((__mmask8) __U,
-                    __builtin_ia32_vfmaddps256 ((__v8sf) __A,
-                                                -(__v8sf) __B,
-                                                -(__v8sf) __C),
-                    (__v8sf) __C);
+  return (__m256)__builtin_ia32_selectps_256(
+      (__mmask8)__U,
+      __builtin_elementwise_fma((__v8sf)__A, -(__v8sf)__B, -(__v8sf)__C),
+      (__v8sf)__C);
 }
 
 static __inline__ __m128d __DEFAULT_FN_ATTRS128
@@ -4358,7 +4318,7 @@ _mm256_maskz_scalef_ps (__mmask8 __U, __m256 __A, __m256 __B) {
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_rolv_epi32 (__m128i __A, __m128i __B)
 {
-  return (__m128i)__builtin_ia32_prolvd128((__v4si)__A, (__v4si)__B);
+  return (__m128i)__builtin_elementwise_fshl((__v4su)__A, (__v4su)__A, (__v4su)__B);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -4380,7 +4340,7 @@ _mm_maskz_rolv_epi32 (__mmask8 __U, __m128i __A, __m128i __B)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_rolv_epi32 (__m256i __A, __m256i __B)
 {
-  return (__m256i)__builtin_ia32_prolvd256((__v8si)__A, (__v8si)__B);
+  return (__m256i)__builtin_elementwise_fshl((__v8su)__A, (__v8su)__A, (__v8su)__B);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -4402,7 +4362,7 @@ _mm256_maskz_rolv_epi32 (__mmask8 __U, __m256i __A, __m256i __B)
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_rolv_epi64 (__m128i __A, __m128i __B)
 {
-  return (__m128i)__builtin_ia32_prolvq128((__v2di)__A, (__v2di)__B);
+  return (__m128i)__builtin_elementwise_fshl((__v2du)__A, (__v2du)__A, (__v2du)__B);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -4424,7 +4384,7 @@ _mm_maskz_rolv_epi64 (__mmask8 __U, __m128i __A, __m128i __B)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_rolv_epi64 (__m256i __A, __m256i __B)
 {
-  return (__m256i)__builtin_ia32_prolvq256((__v4di)__A, (__v4di)__B);
+  return (__m256i)__builtin_elementwise_fshl((__v4du)__A, (__v4du)__A, (__v4du)__B);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -4626,7 +4586,7 @@ _mm256_maskz_slli_epi64(__mmask8 __U, __m256i __A, unsigned int __B)
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_rorv_epi32 (__m128i __A, __m128i __B)
 {
-  return (__m128i)__builtin_ia32_prorvd128((__v4si)__A, (__v4si)__B);
+  return (__m128i)__builtin_elementwise_fshr((__v4su)__A, (__v4su)__A, (__v4su)__B);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -4648,7 +4608,7 @@ _mm_maskz_rorv_epi32 (__mmask8 __U, __m128i __A, __m128i __B)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_rorv_epi32 (__m256i __A, __m256i __B)
 {
-  return (__m256i)__builtin_ia32_prorvd256((__v8si)__A, (__v8si)__B);
+  return (__m256i)__builtin_elementwise_fshr((__v8su)__A, (__v8su)__A, (__v8su)__B);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -4670,7 +4630,7 @@ _mm256_maskz_rorv_epi32 (__mmask8 __U, __m256i __A, __m256i __B)
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_rorv_epi64 (__m128i __A, __m128i __B)
 {
-  return (__m128i)__builtin_ia32_prorvq128((__v2di)__A, (__v2di)__B);
+  return (__m128i)__builtin_elementwise_fshr((__v2du)__A, (__v2du)__A, (__v2du)__B);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -4692,7 +4652,7 @@ _mm_maskz_rorv_epi64 (__mmask8 __U, __m128i __A, __m128i __B)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_rorv_epi64 (__m256i __A, __m256i __B)
 {
-  return (__m256i)__builtin_ia32_prorvq256((__v4di)__A, (__v4di)__B);
+  return (__m256i)__builtin_elementwise_fshr((__v4du)__A, (__v4du)__A, (__v4du)__B);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -6792,9 +6752,8 @@ _mm256_maskz_rsqrt14_ps (__mmask8 __U, __m256 __A)
                 (__mmask8) __U);
 }
 
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
-_mm256_broadcast_f32x4(__m128 __A)
-{
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcast_f32x4(__m128 __A) {
   return (__m256)__builtin_shufflevector((__v4sf)__A, (__v4sf)__A,
                                          0, 1, 2, 3, 0, 1, 2, 3);
 }
@@ -6815,9 +6774,8 @@ _mm256_maskz_broadcast_f32x4 (__mmask8 __M, __m128 __A)
                                             (__v8sf)_mm256_setzero_ps());
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
-_mm256_broadcast_i32x4(__m128i __A)
-{
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_broadcast_i32x4(__m128i __A) {
   return (__m256i)__builtin_shufflevector((__v4si)__A, (__v4si)__A,
                                           0, 1, 2, 3, 0, 1, 2, 3);
 }
@@ -8306,68 +8264,52 @@ _mm256_maskz_moveldup_ps (__mmask8 __U, __m256 __A)
                                        (__v4si)_mm_shuffle_epi32((A), (I)), \
                                        (__v4si)_mm_setzero_si128()))
 
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
-_mm_mask_mov_pd (__m128d __W, __mmask8 __U, __m128d __A)
-{
-  return (__m128d) __builtin_ia32_selectpd_128 ((__mmask8) __U,
-              (__v2df) __A,
-              (__v2df) __W);
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_mask_mov_pd(__m128d __W, __mmask8 __U, __m128d __A) {
+  return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U, (__v2df)__A,
+                                              (__v2df)__W);
 }
 
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
-_mm_maskz_mov_pd (__mmask8 __U, __m128d __A)
-{
-  return (__m128d) __builtin_ia32_selectpd_128 ((__mmask8) __U,
-              (__v2df) __A,
-              (__v2df) _mm_setzero_pd ());
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_maskz_mov_pd(__mmask8 __U, __m128d __A) {
+  return (__m128d)__builtin_ia32_selectpd_128((__mmask8)__U, (__v2df)__A,
+                                              (__v2df)_mm_setzero_pd());
 }
 
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
-_mm256_mask_mov_pd (__m256d __W, __mmask8 __U, __m256d __A)
-{
-  return (__m256d) __builtin_ia32_selectpd_256 ((__mmask8) __U,
-              (__v4df) __A,
-              (__v4df) __W);
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_mask_mov_pd(__m256d __W, __mmask8 __U, __m256d __A) {
+  return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U, (__v4df)__A,
+                                              (__v4df)__W);
 }
 
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
-_mm256_maskz_mov_pd (__mmask8 __U, __m256d __A)
-{
-  return (__m256d) __builtin_ia32_selectpd_256 ((__mmask8) __U,
-              (__v4df) __A,
-              (__v4df) _mm256_setzero_pd ());
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_maskz_mov_pd(__mmask8 __U, __m256d __A) {
+  return (__m256d)__builtin_ia32_selectpd_256((__mmask8)__U, (__v4df)__A,
+                                              (__v4df)_mm256_setzero_pd());
 }
 
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
-_mm_mask_mov_ps (__m128 __W, __mmask8 __U, __m128 __A)
-{
-  return (__m128) __builtin_ia32_selectps_128 ((__mmask8) __U,
-             (__v4sf) __A,
-             (__v4sf) __W);
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_mask_mov_ps(__m128 __W, __mmask8 __U, __m128 __A) {
+  return (__m128)__builtin_ia32_selectps_128((__mmask8)__U, (__v4sf)__A,
+                                             (__v4sf)__W);
 }
 
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
-_mm_maskz_mov_ps (__mmask8 __U, __m128 __A)
-{
-  return (__m128) __builtin_ia32_selectps_128 ((__mmask8) __U,
-             (__v4sf) __A,
-             (__v4sf) _mm_setzero_ps ());
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_maskz_mov_ps(__mmask8 __U, __m128 __A) {
+  return (__m128)__builtin_ia32_selectps_128((__mmask8)__U, (__v4sf)__A,
+                                             (__v4sf)_mm_setzero_ps());
 }
 
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
-_mm256_mask_mov_ps (__m256 __W, __mmask8 __U, __m256 __A)
-{
-  return (__m256) __builtin_ia32_selectps_256 ((__mmask8) __U,
-             (__v8sf) __A,
-             (__v8sf) __W);
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_mask_mov_ps(__m256 __W, __mmask8 __U, __m256 __A) {
+  return (__m256)__builtin_ia32_selectps_256((__mmask8)__U, (__v8sf)__A,
+                                             (__v8sf)__W);
 }
 
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
-_mm256_maskz_mov_ps (__mmask8 __U, __m256 __A)
-{
-  return (__m256) __builtin_ia32_selectps_256 ((__mmask8) __U,
-             (__v8sf) __A,
-             (__v8sf) _mm256_setzero_ps ());
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_maskz_mov_ps(__mmask8 __U, __m256 __A) {
+  return (__m256)__builtin_ia32_selectps_256((__mmask8)__U, (__v8sf)__A,
+                                             (__v8sf)_mm256_setzero_ps());
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
@@ -8430,8 +8372,9 @@ _mm256_maskz_cvtph_ps (__mmask8 __U, __m128i __A)
 #define _mm256_mask_cvtps_ph  _mm256_mask_cvt_roundps_ph
 #define _mm256_maskz_cvtps_ph _mm256_maskz_cvt_roundps_ph
 
-
 #undef __DEFAULT_FN_ATTRS128
 #undef __DEFAULT_FN_ATTRS256
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
 
 #endif /* __AVX512VLINTRIN_H */
diff --git a/clang/lib/Headers/avx512vlvbmi2intrin.h b/clang/lib/Headers/avx512vlvbmi2intrin.h
index 77af2d5cbd2a0..04db52c822640 100644
--- a/clang/lib/Headers/avx512vlvbmi2intrin.h
+++ b/clang/lib/Headers/avx512vlvbmi2intrin.h
@@ -415,8 +415,8 @@ _mm256_maskz_expandloadu_epi8(__mmask32 __U, void const *__P)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_shldv_epi64(__m256i __A, __m256i __B, __m256i __C)
 {
-  return (__m256i)__builtin_ia32_vpshldvq256((__v4di)__A, (__v4di)__B,
-                                             (__v4di)__C);
+  return (__m256i)__builtin_elementwise_fshl((__v4du)__A, (__v4du)__B,
+                                             (__v4du)__C);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -438,8 +438,8 @@ _mm256_maskz_shldv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_shldv_epi64(__m128i __A, __m128i __B, __m128i __C)
 {
-  return (__m128i)__builtin_ia32_vpshldvq128((__v2di)__A, (__v2di)__B,
-                                             (__v2di)__C);
+  return (__m128i)__builtin_elementwise_fshl((__v2du)__A, (__v2du)__B,
+                                             (__v2du)__C);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -461,8 +461,8 @@ _mm_maskz_shldv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_shldv_epi32(__m256i __A, __m256i __B, __m256i __C)
 {
-  return (__m256i)__builtin_ia32_vpshldvd256((__v8si)__A, (__v8si)__B,
-                                             (__v8si)__C);
+  return (__m256i)__builtin_elementwise_fshl((__v8su)__A, (__v8su)__B,
+                                             (__v8su)__C);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -484,8 +484,8 @@ _mm256_maskz_shldv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_shldv_epi32(__m128i __A, __m128i __B, __m128i __C)
 {
-  return (__m128i)__builtin_ia32_vpshldvd128((__v4si)__A, (__v4si)__B,
-                                             (__v4si)__C);
+  return (__m128i)__builtin_elementwise_fshl((__v4su)__A, (__v4su)__B,
+                                             (__v4su)__C);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -507,8 +507,8 @@ _mm_maskz_shldv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_shldv_epi16(__m256i __A, __m256i __B, __m256i __C)
 {
-  return (__m256i)__builtin_ia32_vpshldvw256((__v16hi)__A, (__v16hi)__B,
-                                             (__v16hi)__C);
+  return (__m256i)__builtin_elementwise_fshl((__v16hu)__A, (__v16hu)__B,
+                                             (__v16hu)__C);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -530,8 +530,8 @@ _mm256_maskz_shldv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_shldv_epi16(__m128i __A, __m128i __B, __m128i __C)
 {
-  return (__m128i)__builtin_ia32_vpshldvw128((__v8hi)__A, (__v8hi)__B,
-                                             (__v8hi)__C);
+  return (__m128i)__builtin_elementwise_fshl((__v8hu)__A, (__v8hu)__B,
+                                             (__v8hu)__C);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -553,8 +553,9 @@ _mm_maskz_shldv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_shrdv_epi64(__m256i __A, __m256i __B, __m256i __C)
 {
-  return (__m256i)__builtin_ia32_vpshrdvq256((__v4di)__A, (__v4di)__B,
-                                             (__v4di)__C);
+  // Ops __A and __B are swapped.
+  return (__m256i)__builtin_elementwise_fshr((__v4du)__B, (__v4du)__A,
+                                             (__v4du)__C);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -576,8 +577,9 @@ _mm256_maskz_shrdv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_shrdv_epi64(__m128i __A, __m128i __B, __m128i __C)
 {
-  return (__m128i)__builtin_ia32_vpshrdvq128((__v2di)__A, (__v2di)__B,
-                                             (__v2di)__C);
+  // Ops __A and __B are swapped.
+  return (__m128i)__builtin_elementwise_fshr((__v2du)__B, (__v2du)__A,
+                                             (__v2du)__C);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -599,8 +601,9 @@ _mm_maskz_shrdv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_shrdv_epi32(__m256i __A, __m256i __B, __m256i __C)
 {
-  return (__m256i)__builtin_ia32_vpshrdvd256((__v8si)__A, (__v8si)__B,
-                                             (__v8si)__C);
+  // Ops __A and __B are swapped.
+  return (__m256i)__builtin_elementwise_fshr((__v8su)__B, (__v8su)__A,
+                                             (__v8su)__C);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -622,8 +625,9 @@ _mm256_maskz_shrdv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_shrdv_epi32(__m128i __A, __m128i __B, __m128i __C)
 {
-  return (__m128i)__builtin_ia32_vpshrdvd128((__v4si)__A, (__v4si)__B,
-                                             (__v4si)__C);
+  // Ops __A and __B are swapped.
+  return (__m128i)__builtin_elementwise_fshr((__v4su)__B, (__v4su)__A,
+                                             (__v4su)__C);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
@@ -645,8 +649,9 @@ _mm_maskz_shrdv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_shrdv_epi16(__m256i __A, __m256i __B, __m256i __C)
 {
-  return (__m256i)__builtin_ia32_vpshrdvw256((__v16hi)__A, (__v16hi)__B,
-                                             (__v16hi)__C);
+  // Ops __A and __B are swapped.
+  return (__m256i)__builtin_elementwise_fshr((__v16hu)__B, (__v16hu)__A,
+                                             (__v16hu)__C);
 }
 
 static __inline__ __m256i __DEFAULT_FN_ATTRS256
@@ -668,8 +673,9 @@ _mm256_maskz_shrdv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
 _mm_shrdv_epi16(__m128i __A, __m128i __B, __m128i __C)
 {
-  return (__m128i)__builtin_ia32_vpshrdvw128((__v8hi)__A, (__v8hi)__B,
-                                             (__v8hi)__C);
+  // Ops __A and __B are swapped.
+  return (__m128i)__builtin_elementwise_fshr((__v8hu)__B, (__v8hu)__A,
+                                             (__v8hu)__C);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS128
diff --git a/clang/lib/Headers/avx512vpopcntdqintrin.h b/clang/lib/Headers/avx512vpopcntdqintrin.h
index e24c2c5e1bcd9..ac71808f69803 100644
--- a/clang/lib/Headers/avx512vpopcntdqintrin.h
+++ b/clang/lib/Headers/avx512vpopcntdqintrin.h
@@ -16,19 +16,19 @@
 #define __AVX512VPOPCNTDQINTRIN_H
 
 /* Define the default attributes for the functions in this file. */
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
 #define __DEFAULT_FN_ATTRS                                                     \
   __attribute__((__always_inline__, __nodebug__,                               \
                  __target__("avx512vpopcntdq,evex512"),                        \
-                 __min_vector_width__(512)))
-
-#if defined(__cplusplus) && (__cplusplus >= 201103L)
-#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
+                 __min_vector_width__(512))) constexpr
 #else
-#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#define __DEFAULT_FN_ATTRS                                                     \
+  __attribute__((__always_inline__, __nodebug__,                               \
+                 __target__("avx512vpopcntdq,evex512"),                        \
+                 __min_vector_width__(512)))
 #endif
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
-_mm512_popcnt_epi64(__m512i __A) {
+static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi64(__m512i __A) {
   return (__m512i)__builtin_elementwise_popcount((__v8du)__A);
 }
 
@@ -43,8 +43,7 @@ _mm512_maskz_popcnt_epi64(__mmask8 __U, __m512i __A) {
   return _mm512_mask_popcnt_epi64((__m512i)_mm512_setzero_si512(), __U, __A);
 }
 
-static __inline__ __m512i __DEFAULT_FN_ATTRS_CONSTEXPR
-_mm512_popcnt_epi32(__m512i __A) {
+static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_popcnt_epi32(__m512i __A) {
   return (__m512i)__builtin_elementwise_popcount((__v16su)__A);
 }
 
diff --git a/clang/lib/Headers/avx512vpopcntdqvlintrin.h b/clang/lib/Headers/avx512vpopcntdqvlintrin.h
index b6c819b0cb85e..bed951b764cf7 100644
--- a/clang/lib/Headers/avx512vpopcntdqvlintrin.h
+++ b/clang/lib/Headers/avx512vpopcntdqvlintrin.h
@@ -16,6 +16,17 @@
 #define __AVX512VPOPCNTDQVLINTRIN_H
 
 /* Define the default attributes for the functions in this file. */
+
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS128                                                  \
+  __attribute__((__always_inline__, __nodebug__,                               \
+                 __target__("avx512vpopcntdq,avx512vl,no-evex512"),            \
+                 __min_vector_width__(128))) constexpr
+#define __DEFAULT_FN_ATTRS256                                                  \
+  __attribute__((__always_inline__, __nodebug__,                               \
+                 __target__("avx512vpopcntdq,avx512vl,no-evex512"),            \
+                 __min_vector_width__(256))) constexpr
+#else
 #define __DEFAULT_FN_ATTRS128                                                  \
   __attribute__((__always_inline__, __nodebug__,                               \
                  __target__("avx512vpopcntdq,avx512vl,no-evex512"),            \
@@ -24,17 +35,9 @@
   __attribute__((__always_inline__, __nodebug__,                               \
                  __target__("avx512vpopcntdq,avx512vl,no-evex512"),            \
                  __min_vector_width__(256)))
-
-#if defined(__cplusplus) && (__cplusplus >= 201103L)
-#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
-#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
-#else
-#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
-#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
 #endif
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
-_mm_popcnt_epi64(__m128i __A) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_popcnt_epi64(__m128i __A) {
   return (__m128i)__builtin_elementwise_popcount((__v2du)__A);
 }
 
@@ -49,8 +52,7 @@ _mm_maskz_popcnt_epi64(__mmask8 __U, __m128i __A) {
   return _mm_mask_popcnt_epi64((__m128i)_mm_setzero_si128(), __U, __A);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS128_CONSTEXPR
-_mm_popcnt_epi32(__m128i __A) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS128 _mm_popcnt_epi32(__m128i __A) {
   return (__m128i)__builtin_elementwise_popcount((__v4su)__A);
 }
 
@@ -65,7 +67,7 @@ _mm_maskz_popcnt_epi32(__mmask8 __U, __m128i __A) {
   return _mm_mask_popcnt_epi32((__m128i)_mm_setzero_si128(), __U, __A);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_popcnt_epi64(__m256i __A) {
   return (__m256i)__builtin_elementwise_popcount((__v4du)__A);
 }
@@ -81,7 +83,7 @@ _mm256_maskz_popcnt_epi64(__mmask8 __U, __m256i __A) {
   return _mm256_mask_popcnt_epi64((__m256i)_mm256_setzero_si256(), __U, __A);
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
+static __inline__ __m256i __DEFAULT_FN_ATTRS256
 _mm256_popcnt_epi32(__m256i __A) {
   return (__m256i)__builtin_elementwise_popcount((__v8su)__A);
 }
diff --git a/clang/lib/Headers/avxintrin.h b/clang/lib/Headers/avxintrin.h
index b9ca013c25c7a..b8cfaee7cfb46 100644
--- a/clang/lib/Headers/avxintrin.h
+++ b/clang/lib/Headers/avxintrin.h
@@ -70,8 +70,8 @@ typedef __bf16 __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
 #define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
 #define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
 #else
-#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS128
-#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
 #endif
 
 /* Arithmetic */
@@ -87,9 +87,8 @@ typedef __bf16 __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
 ///    A 256-bit vector of [4 x double] containing one of the source operands.
 /// \returns A 256-bit vector of [4 x double] containing the sums of both
 ///    operands.
-static __inline __m256d __DEFAULT_FN_ATTRS
-_mm256_add_pd(__m256d __a, __m256d __b)
-{
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm256_add_pd(__m256d __a, __m256d __b) {
   return (__m256d)((__v4df)__a+(__v4df)__b);
 }
 
@@ -105,9 +104,8 @@ _mm256_add_pd(__m256d __a, __m256d __b)
 ///    A 256-bit vector of [8 x float] containing one of the source operands.
 /// \returns A 256-bit vector of [8 x float] containing the sums of both
 ///    operands.
-static __inline __m256 __DEFAULT_FN_ATTRS
-_mm256_add_ps(__m256 __a, __m256 __b)
-{
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_add_ps(__m256 __a,
+                                                                  __m256 __b) {
   return (__m256)((__v8sf)__a+(__v8sf)__b);
 }
 
@@ -123,9 +121,8 @@ _mm256_add_ps(__m256 __a, __m256 __b)
 ///    A 256-bit vector of [4 x double] containing the subtrahend.
 /// \returns A 256-bit vector of [4 x double] containing the differences between
 ///    both operands.
-static __inline __m256d __DEFAULT_FN_ATTRS
-_mm256_sub_pd(__m256d __a, __m256d __b)
-{
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm256_sub_pd(__m256d __a, __m256d __b) {
   return (__m256d)((__v4df)__a-(__v4df)__b);
 }
 
@@ -141,9 +138,8 @@ _mm256_sub_pd(__m256d __a, __m256d __b)
 ///    A 256-bit vector of [8 x float] containing the subtrahend.
 /// \returns A 256-bit vector of [8 x float] containing the differences between
 ///    both operands.
-static __inline __m256 __DEFAULT_FN_ATTRS
-_mm256_sub_ps(__m256 __a, __m256 __b)
-{
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_sub_ps(__m256 __a,
+                                                                  __m256 __b) {
   return (__m256)((__v8sf)__a-(__v8sf)__b);
 }
 
@@ -197,9 +193,8 @@ _mm256_addsub_ps(__m256 __a, __m256 __b)
 ///    A 256-bit vector of [4 x double] containing the divisor.
 /// \returns A 256-bit vector of [4 x double] containing the quotients of both
 ///    operands.
-static __inline __m256d __DEFAULT_FN_ATTRS
-_mm256_div_pd(__m256d __a, __m256d __b)
-{
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm256_div_pd(__m256d __a, __m256d __b) {
   return (__m256d)((__v4df)__a/(__v4df)__b);
 }
 
@@ -215,9 +210,8 @@ _mm256_div_pd(__m256d __a, __m256d __b)
 ///    A 256-bit vector of [8 x float] containing the divisor.
 /// \returns A 256-bit vector of [8 x float] containing the quotients of both
 ///    operands.
-static __inline __m256 __DEFAULT_FN_ATTRS
-_mm256_div_ps(__m256 __a, __m256 __b)
-{
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_div_ps(__m256 __a,
+                                                                  __m256 __b) {
   return (__m256)((__v8sf)__a/(__v8sf)__b);
 }
 
@@ -317,9 +311,8 @@ _mm256_min_ps(__m256 __a, __m256 __b)
 ///    A 256-bit vector of [4 x double] containing one of the operands.
 /// \returns A 256-bit vector of [4 x double] containing the products of both
 ///    operands.
-static __inline __m256d __DEFAULT_FN_ATTRS
-_mm256_mul_pd(__m256d __a, __m256d __b)
-{
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm256_mul_pd(__m256d __a, __m256d __b) {
   return (__m256d)((__v4df)__a * (__v4df)__b);
 }
 
@@ -335,9 +328,8 @@ _mm256_mul_pd(__m256d __a, __m256d __b)
 ///    A 256-bit vector of [8 x float] containing one of the operands.
 /// \returns A 256-bit vector of [8 x float] containing the products of both
 ///    operands.
-static __inline __m256 __DEFAULT_FN_ATTRS
-_mm256_mul_ps(__m256 __a, __m256 __b)
-{
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_mul_ps(__m256 __a,
+                                                                  __m256 __b) {
   return (__m256)((__v8sf)__a * (__v8sf)__b);
 }
 
@@ -555,7 +547,7 @@ _mm256_rcp_ps(__m256 __a)
 ///    A 256-bit vector of [4 x double] containing one of the source operands.
 /// \returns A 256-bit vector of [4 x double] containing the bitwise AND of the
 ///    values between both operands.
-static __inline __m256d __DEFAULT_FN_ATTRS
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_and_pd(__m256d __a, __m256d __b)
 {
   return (__m256d)((__v4du)__a & (__v4du)__b);
@@ -573,7 +565,7 @@ _mm256_and_pd(__m256d __a, __m256d __b)
 ///    A 256-bit vector of [8 x float] containing one of the source operands.
 /// \returns A 256-bit vector of [8 x float] containing the bitwise AND of the
 ///    values between both operands.
-static __inline __m256 __DEFAULT_FN_ATTRS
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_and_ps(__m256 __a, __m256 __b)
 {
   return (__m256)((__v8su)__a & (__v8su)__b);
@@ -594,7 +586,7 @@ _mm256_and_ps(__m256 __a, __m256 __b)
 /// \returns A 256-bit vector of [4 x double] containing the bitwise AND of the
 ///    values of the second operand and the one's complement of the first
 ///    operand.
-static __inline __m256d __DEFAULT_FN_ATTRS
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_andnot_pd(__m256d __a, __m256d __b)
 {
   return (__m256d)(~(__v4du)__a & (__v4du)__b);
@@ -615,7 +607,7 @@ _mm256_andnot_pd(__m256d __a, __m256d __b)
 /// \returns A 256-bit vector of [8 x float] containing the bitwise AND of the
 ///    values of the second operand and the one's complement of the first
 ///    operand.
-static __inline __m256 __DEFAULT_FN_ATTRS
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_andnot_ps(__m256 __a, __m256 __b)
 {
   return (__m256)(~(__v8su)__a & (__v8su)__b);
@@ -633,7 +625,7 @@ _mm256_andnot_ps(__m256 __a, __m256 __b)
 ///    A 256-bit vector of [4 x double] containing one of the source operands.
 /// \returns A 256-bit vector of [4 x double] containing the bitwise OR of the
 ///    values between both operands.
-static __inline __m256d __DEFAULT_FN_ATTRS
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_or_pd(__m256d __a, __m256d __b)
 {
   return (__m256d)((__v4du)__a | (__v4du)__b);
@@ -651,7 +643,7 @@ _mm256_or_pd(__m256d __a, __m256d __b)
 ///    A 256-bit vector of [8 x float] containing one of the source operands.
 /// \returns A 256-bit vector of [8 x float] containing the bitwise OR of the
 ///    values between both operands.
-static __inline __m256 __DEFAULT_FN_ATTRS
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_or_ps(__m256 __a, __m256 __b)
 {
   return (__m256)((__v8su)__a | (__v8su)__b);
@@ -669,7 +661,7 @@ _mm256_or_ps(__m256 __a, __m256 __b)
 ///    A 256-bit vector of [4 x double] containing one of the source operands.
 /// \returns A 256-bit vector of [4 x double] containing the bitwise XOR of the
 ///    values between both operands.
-static __inline __m256d __DEFAULT_FN_ATTRS
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_xor_pd(__m256d __a, __m256d __b)
 {
   return (__m256d)((__v4du)__a ^ (__v4du)__b);
@@ -687,7 +679,7 @@ _mm256_xor_pd(__m256d __a, __m256d __b)
 ///    A 256-bit vector of [8 x float] containing one of the source operands.
 /// \returns A 256-bit vector of [8 x float] containing the bitwise XOR of the
 ///    values between both operands.
-static __inline __m256 __DEFAULT_FN_ATTRS
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_xor_ps(__m256 __a, __m256 __b)
 {
   return (__m256)((__v8su)__a ^ (__v8su)__b);
@@ -2392,7 +2384,7 @@ _mm256_cvtss_f32(__m256 __a)
 ///    return value.
 /// \returns A 256-bit vector of [8 x float] containing the moved and duplicated
 ///    values.
-static __inline __m256 __DEFAULT_FN_ATTRS
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_movehdup_ps(__m256 __a)
 {
   return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 1, 1, 3, 3, 5, 5, 7, 7);
@@ -2417,7 +2409,7 @@ _mm256_movehdup_ps(__m256 __a)
 ///    return value.
 /// \returns A 256-bit vector of [8 x float] containing the moved and duplicated
 ///    values.
-static __inline __m256 __DEFAULT_FN_ATTRS
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_moveldup_ps(__m256 __a)
 {
   return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 0, 0, 2, 2, 4, 4, 6, 6);
@@ -2439,7 +2431,7 @@ _mm256_moveldup_ps(__m256 __a)
 ///    the return value.
 /// \returns A 256-bit vector of [4 x double] containing the moved and
 ///    duplicated values.
-static __inline __m256d __DEFAULT_FN_ATTRS
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_movedup_pd(__m256d __a)
 {
   return __builtin_shufflevector((__v4df)__a, (__v4df)__a, 0, 0, 2, 2);
@@ -2462,9 +2454,8 @@ _mm256_movedup_pd(__m256d __a)
 ///    Bits [127:64] are written to bits [127:64] of the return value. \n
 ///    Bits [255:192] are written to bits [255:192] of the return value. \n
 /// \returns A 256-bit vector of [4 x double] containing the interleaved values.
-static __inline __m256d __DEFAULT_FN_ATTRS
-_mm256_unpackhi_pd(__m256d __a, __m256d __b)
-{
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm256_unpackhi_pd(__m256d __a, __m256d __b) {
   return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 1, 5, 1+2, 5+2);
 }
 
@@ -2484,9 +2475,8 @@ _mm256_unpackhi_pd(__m256d __a, __m256d __b)
 ///    Bits [63:0] are written to bits [127:64] of the return value. \n
 ///    Bits [191:128] are written to bits [255:192] of the return value. \n
 /// \returns A 256-bit vector of [4 x double] containing the interleaved values.
-static __inline __m256d __DEFAULT_FN_ATTRS
-_mm256_unpacklo_pd(__m256d __a, __m256d __b)
-{
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm256_unpacklo_pd(__m256d __a, __m256d __b) {
   return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 0, 4, 0+2, 4+2);
 }
 
@@ -2511,9 +2501,8 @@ _mm256_unpacklo_pd(__m256d __a, __m256d __b)
 ///    Bits [223:192] are written to bits [191:160] of the return value. \n
 ///    Bits [255:224] are written to bits [255:224] of the return value.
 /// \returns A 256-bit vector of [8 x float] containing the interleaved values.
-static __inline __m256 __DEFAULT_FN_ATTRS
-_mm256_unpackhi_ps(__m256 __a, __m256 __b)
-{
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm256_unpackhi_ps(__m256 __a, __m256 __b) {
   return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 2, 10, 2+1, 10+1, 6, 14, 6+1, 14+1);
 }
 
@@ -2538,9 +2527,8 @@ _mm256_unpackhi_ps(__m256 __a, __m256 __b)
 ///    Bits [159:128] are written to bits [191:160] of the return value. \n
 ///    Bits [191:160] are written to bits [255:224] of the return value.
 /// \returns A 256-bit vector of [8 x float] containing the interleaved values.
-static __inline __m256 __DEFAULT_FN_ATTRS
-_mm256_unpacklo_ps(__m256 __a, __m256 __b)
-{
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm256_unpacklo_ps(__m256 __a, __m256 __b) {
   return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 0, 8, 0+1, 8+1, 4, 12, 4+1, 12+1);
 }
 
@@ -4379,7 +4367,7 @@ _mm256_setzero_si256(void) {
 ///    A 256-bit floating-point vector of [4 x double].
 /// \returns A 256-bit floating-point vector of [8 x float] containing the same
 ///    bitwise pattern as the parameter.
-static __inline __m256 __DEFAULT_FN_ATTRS
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_castpd_ps(__m256d __a)
 {
   return (__m256)__a;
@@ -4396,7 +4384,7 @@ _mm256_castpd_ps(__m256d __a)
 ///    A 256-bit floating-point vector of [4 x double].
 /// \returns A 256-bit integer vector containing the same bitwise pattern as the
 ///    parameter.
-static __inline __m256i __DEFAULT_FN_ATTRS
+static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_castpd_si256(__m256d __a)
 {
   return (__m256i)__a;
@@ -4413,7 +4401,7 @@ _mm256_castpd_si256(__m256d __a)
 ///    A 256-bit floating-point vector of [8 x float].
 /// \returns A 256-bit floating-point vector of [4 x double] containing the same
 ///    bitwise pattern as the parameter.
-static __inline __m256d __DEFAULT_FN_ATTRS
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_castps_pd(__m256 __a)
 {
   return (__m256d)__a;
@@ -4430,7 +4418,7 @@ _mm256_castps_pd(__m256 __a)
 ///    A 256-bit floating-point vector of [8 x float].
 /// \returns A 256-bit integer vector containing the same bitwise pattern as the
 ///    parameter.
-static __inline __m256i __DEFAULT_FN_ATTRS
+static __inline __m256i __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_castps_si256(__m256 __a)
 {
   return (__m256i)__a;
@@ -4447,7 +4435,7 @@ _mm256_castps_si256(__m256 __a)
 ///    A 256-bit integer vector.
 /// \returns A 256-bit floating-point vector of [8 x float] containing the same
 ///    bitwise pattern as the parameter.
-static __inline __m256 __DEFAULT_FN_ATTRS
+static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_castsi256_ps(__m256i __a)
 {
   return (__m256)__a;
@@ -4464,7 +4452,7 @@ _mm256_castsi256_ps(__m256i __a)
 ///    A 256-bit integer vector.
 /// \returns A 256-bit floating-point vector of [4 x double] containing the same
 ///    bitwise pattern as the parameter.
-static __inline __m256d __DEFAULT_FN_ATTRS
+static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_castsi256_pd(__m256i __a)
 {
   return (__m256d)__a;
@@ -4481,7 +4469,7 @@ _mm256_castsi256_pd(__m256i __a)
 ///    A 256-bit floating-point vector of [4 x double].
 /// \returns A 128-bit floating-point vector of [2 x double] containing the
 ///    lower 128 bits of the parameter.
-static __inline __m128d __DEFAULT_FN_ATTRS
+static __inline __m128d __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_castpd256_pd128(__m256d __a)
 {
   return __builtin_shufflevector((__v4df)__a, (__v4df)__a, 0, 1);
@@ -4498,7 +4486,7 @@ _mm256_castpd256_pd128(__m256d __a)
 ///    A 256-bit floating-point vector of [8 x float].
 /// \returns A 128-bit floating-point vector of [4 x float] containing the
 ///    lower 128 bits of the parameter.
-static __inline __m128 __DEFAULT_FN_ATTRS
+static __inline __m128 __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_castps256_ps128(__m256 __a)
 {
   return __builtin_shufflevector((__v8sf)__a, (__v8sf)__a, 0, 1, 2, 3);
@@ -4514,7 +4502,7 @@ _mm256_castps256_ps128(__m256 __a)
 ///    A 256-bit integer vector.
 /// \returns A 128-bit integer vector containing the lower 128 bits of the
 ///    parameter.
-static __inline __m128i __DEFAULT_FN_ATTRS
+static __inline __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm256_castsi256_si128(__m256i __a)
 {
   return __builtin_shufflevector((__v4di)__a, (__v4di)__a, 0, 1);
diff --git a/clang/lib/Headers/cpuid.h b/clang/lib/Headers/cpuid.h
index 52addb7bfa856..ce8c79e77dc18 100644
--- a/clang/lib/Headers/cpuid.h
+++ b/clang/lib/Headers/cpuid.h
@@ -345,10 +345,15 @@ static __inline int __get_cpuid_count (unsigned int __leaf,
 // In some configurations, __cpuidex is defined as a builtin (primarily
 // -fms-extensions) which will conflict with the __cpuidex definition below.
 #if !(__has_builtin(__cpuidex))
+// In some cases, offloading will set the host as the aux triple and define the
+// builtin. Given __has_builtin does not detect builtins on aux triples, we need
+// to explicitly check for some offloading cases.
+#ifndef __NVPTX__
 static __inline void __cpuidex(int __cpu_info[4], int __leaf, int __subleaf) {
   __cpuid_count(__leaf, __subleaf, __cpu_info[0], __cpu_info[1], __cpu_info[2],
                 __cpu_info[3]);
 }
 #endif
+#endif
 
 #endif /* __CPUID_H */
diff --git a/clang/lib/Headers/emmintrin.h b/clang/lib/Headers/emmintrin.h
index 78e8a422db4c1..8b6b62458dac1 100644
--- a/clang/lib/Headers/emmintrin.h
+++ b/clang/lib/Headers/emmintrin.h
@@ -17,7 +17,6 @@
 #include <xmmintrin.h>
 
 typedef double __m128d __attribute__((__vector_size__(16), __aligned__(16)));
-typedef long long __m128i __attribute__((__vector_size__(16), __aligned__(16)));
 
 typedef double __m128d_u __attribute__((__vector_size__(16), __aligned__(1)));
 typedef long long __m128i_u
@@ -25,13 +24,9 @@ typedef long long __m128i_u
 
 /* Type defines.  */
 typedef double __v2df __attribute__((__vector_size__(16)));
-typedef long long __v2di __attribute__((__vector_size__(16)));
-typedef short __v8hi __attribute__((__vector_size__(16)));
-typedef char __v16qi __attribute__((__vector_size__(16)));
 
 /* Unsigned types */
 typedef unsigned long long __v2du __attribute__((__vector_size__(16)));
-typedef unsigned short __v8hu __attribute__((__vector_size__(16)));
 typedef unsigned char __v16qu __attribute__((__vector_size__(16)));
 
 /* We need an explicitly signed variant for char. Note that this shouldn't
@@ -67,6 +62,9 @@ typedef __bf16 __m128bh __attribute__((__vector_size__(16), __aligned__(16)));
 
 #define __trunc64(x)                                                           \
   (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
+#define __zext128(x)                                                           \
+  (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0,   \
+                                    1, 2, 3)
 #define __anyext128(x)                                                         \
   (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0,   \
                                     1, -1, -1)
@@ -2127,8 +2125,9 @@ _mm_add_epi32(__m128i __a, __m128i __b) {
 /// \param __b
 ///    A 64-bit integer.
 /// \returns A 64-bit integer containing the sum of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_add_si64(__m64 __a, __m64 __b) {
-  return (__m64)(((unsigned long long)__a) + ((unsigned long long)__b));
+static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_add_si64(__m64 __a,
+                                                                  __m64 __b) {
+  return (__m64)(((__v1du)__a)[0] + ((__v1du)__b)[0]);
 }
 
 /// Adds the corresponding elements of two 128-bit vectors of [2 x i64],
@@ -2169,8 +2168,8 @@ _mm_add_epi64(__m128i __a, __m128i __b) {
 ///    A 128-bit signed [16 x i8] vector.
 /// \returns A 128-bit signed [16 x i8] vector containing the saturated sums of
 ///    both parameters.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epi8(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_adds_epi8(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_elementwise_add_sat((__v16qs)__a, (__v16qs)__b);
 }
 
@@ -2191,8 +2190,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epi8(__m128i __a,
 ///    A 128-bit signed [8 x i16] vector.
 /// \returns A 128-bit signed [8 x i16] vector containing the saturated sums of
 ///    both parameters.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epi16(__m128i __a,
-                                                            __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_adds_epi16(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_elementwise_add_sat((__v8hi)__a, (__v8hi)__b);
 }
 
@@ -2213,8 +2212,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epi16(__m128i __a,
 ///    A 128-bit unsigned [16 x i8] vector.
 /// \returns A 128-bit unsigned [16 x i8] vector containing the saturated sums
 ///    of both parameters.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epu8(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_adds_epu8(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_elementwise_add_sat((__v16qu)__a, (__v16qu)__b);
 }
 
@@ -2235,8 +2234,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epu8(__m128i __a,
 ///    A 128-bit unsigned [8 x i16] vector.
 /// \returns A 128-bit unsigned [8 x i16] vector containing the saturated sums
 ///    of both parameters.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_adds_epu16(__m128i __a,
-                                                            __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_adds_epu16(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_elementwise_add_sat((__v8hu)__a, (__v8hu)__b);
 }
 
@@ -2393,8 +2392,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epu8(__m128i __a,
 ///    A 128-bit signed [8 x i16] vector.
 /// \returns A 128-bit signed [8 x i16] vector containing the upper 16 bits of
 ///    each of the eight 32-bit products.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mulhi_epi16(__m128i __a,
-                                                             __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_mulhi_epi16(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_ia32_pmulhw128((__v8hi)__a, (__v8hi)__b);
 }
 
@@ -2412,9 +2411,9 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mulhi_epi16(__m128i __a,
 ///    A 128-bit unsigned [8 x i16] vector.
 /// \returns A 128-bit unsigned [8 x i16] vector containing the upper 16 bits
 ///    of each of the eight 32-bit products.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mulhi_epu16(__m128i __a,
-                                                             __m128i __b) {
-  return (__m128i)__builtin_ia32_pmulhuw128((__v8hi)__a, (__v8hi)__b);
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_mulhi_epu16(__m128i __a, __m128i __b) {
+  return (__m128i)__builtin_ia32_pmulhuw128((__v8hu)__a, (__v8hu)__b);
 }
 
 /// Multiplies the corresponding elements of two signed [8 x i16]
@@ -2431,8 +2430,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mulhi_epu16(__m128i __a,
 ///    A 128-bit signed [8 x i16] vector.
 /// \returns A 128-bit signed [8 x i16] vector containing the lower 16 bits of
 ///    each of the eight 32-bit products.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi16(__m128i __a,
-                                                             __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_mullo_epi16(__m128i __a, __m128i __b) {
   return (__m128i)((__v8hu)__a * (__v8hu)__b);
 }
 
@@ -2449,9 +2448,10 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi16(__m128i __a,
 /// \param __b
 ///    A 64-bit integer containing one of the source operands.
 /// \returns A 64-bit integer vector containing the product of both operands.
-static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_mul_su32(__m64 __a, __m64 __b) {
-  return __trunc64(__builtin_ia32_pmuludq128((__v4si)__anyext128(__a),
-                                             (__v4si)__anyext128(__b)));
+static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_mul_su32(__m64 __a,
+                                                                  __m64 __b) {
+  return __trunc64(__builtin_ia32_pmuludq128((__v4si)__zext128(__a),
+                                             (__v4si)__zext128(__b)));
 }
 
 /// Multiplies 32-bit unsigned integer values contained in the lower
@@ -2467,8 +2467,8 @@ static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_mul_su32(__m64 __a, __m64 __b) {
 /// \param __b
 ///    A [2 x i64] vector containing one of the source operands.
 /// \returns A [2 x i64] vector containing the product of both operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mul_epu32(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_mul_epu32(__m128i __a, __m128i __b) {
   return __builtin_ia32_pmuludq128((__v4si)__a, (__v4si)__b);
 }
 
@@ -2557,8 +2557,9 @@ _mm_sub_epi32(__m128i __a, __m128i __b) {
 ///    A 64-bit integer vector containing the subtrahend.
 /// \returns A 64-bit integer vector containing the difference of the values in
 ///    the operands.
-static __inline__ __m64 __DEFAULT_FN_ATTRS _mm_sub_si64(__m64 __a, __m64 __b) {
-  return (__m64)((unsigned long long)__a - (unsigned long long)__b);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_CONSTEXPR _mm_sub_si64(__m64 __a,
+                                                                  __m64 __b) {
+  return (__m64)(((__v1du)__a)[0] - ((__v1du)__b)[0]);
 }
 
 /// Subtracts the corresponding elements of two [2 x i64] vectors.
@@ -2595,8 +2596,8 @@ _mm_sub_epi64(__m128i __a, __m128i __b) {
 ///    A 128-bit integer vector containing the subtrahends.
 /// \returns A 128-bit integer vector containing the differences of the values
 ///    in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epi8(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_subs_epi8(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_elementwise_sub_sat((__v16qs)__a, (__v16qs)__b);
 }
 
@@ -2617,8 +2618,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epi8(__m128i __a,
 ///    A 128-bit integer vector containing the subtrahends.
 /// \returns A 128-bit integer vector containing the differences of the values
 ///    in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epi16(__m128i __a,
-                                                            __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_subs_epi16(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_elementwise_sub_sat((__v8hi)__a, (__v8hi)__b);
 }
 
@@ -2638,8 +2639,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epi16(__m128i __a,
 ///    A 128-bit integer vector containing the subtrahends.
 /// \returns A 128-bit integer vector containing the unsigned integer
 ///    differences of the values in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epu8(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_subs_epu8(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_elementwise_sub_sat((__v16qu)__a, (__v16qu)__b);
 }
 
@@ -2659,8 +2660,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epu8(__m128i __a,
 ///    A 128-bit integer vector containing the subtrahends.
 /// \returns A 128-bit integer vector containing the unsigned integer
 ///    differences of the values in the operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epu16(__m128i __a,
-                                                            __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_subs_epu16(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_elementwise_sub_sat((__v8hu)__a, (__v8hu)__b);
 }
 
@@ -2676,8 +2677,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_subs_epu16(__m128i __a,
 ///    A 128-bit integer vector containing one of the source operands.
 /// \returns A 128-bit integer vector containing the bitwise AND of the values
 ///    in both operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_and_si128(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_and_si128(__m128i __a, __m128i __b) {
   return (__m128i)((__v2du)__a & (__v2du)__b);
 }
 
@@ -2695,8 +2696,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_and_si128(__m128i __a,
 ///    A 128-bit vector containing the right source operand.
 /// \returns A 128-bit integer vector containing the bitwise AND of the one's
 ///    complement of the first operand and the values in the second operand.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_andnot_si128(__m128i __a,
-                                                              __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_andnot_si128(__m128i __a, __m128i __b) {
   return (__m128i)(~(__v2du)__a & (__v2du)__b);
 }
 /// Performs a bitwise OR of two 128-bit integer vectors.
@@ -2711,8 +2712,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_andnot_si128(__m128i __a,
 ///    A 128-bit integer vector containing one of the source operands.
 /// \returns A 128-bit integer vector containing the bitwise OR of the values
 ///    in both operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_or_si128(__m128i __a,
-                                                          __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_or_si128(__m128i __a, __m128i __b) {
   return (__m128i)((__v2du)__a | (__v2du)__b);
 }
 
@@ -2728,8 +2729,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_or_si128(__m128i __a,
 ///    A 128-bit integer vector containing one of the source operands.
 /// \returns A 128-bit integer vector containing the bitwise exclusive OR of the
 ///    values in both operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_xor_si128(__m128i __a,
-                                                           __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_xor_si128(__m128i __a, __m128i __b) {
   return (__m128i)((__v2du)__a ^ (__v2du)__b);
 }
 
@@ -3379,7 +3380,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvttps_epi32(__m128 __a) {
 /// \param __a
 ///    A 32-bit signed integer operand.
 /// \returns A 128-bit vector of [4 x i32].
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtsi32_si128(int __a) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtsi32_si128(int __a) {
   return __extension__(__m128i)(__v4si){__a, 0, 0, 0};
 }
 
@@ -3394,7 +3396,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtsi32_si128(int __a) {
 /// \param __a
 ///    A 64-bit signed integer operand containing the value to be converted.
 /// \returns A 128-bit vector of [2 x i64] containing the converted value.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtsi64_si128(long long __a) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtsi64_si128(long long __a) {
   return __extension__(__m128i)(__v2di){__a, 0};
 }
 
@@ -3409,7 +3412,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtsi64_si128(long long __a) {
 ///    A vector of [4 x i32]. The least significant 32 bits are moved to the
 ///    destination.
 /// \returns A 32-bit signed integer containing the moved value.
-static __inline__ int __DEFAULT_FN_ATTRS _mm_cvtsi128_si32(__m128i __a) {
+static __inline__ int __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtsi128_si32(__m128i __a) {
   __v4si __b = (__v4si)__a;
   return __b[0];
 }
@@ -3425,7 +3429,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_cvtsi128_si32(__m128i __a) {
 ///    A vector of [2 x i64]. The least significant 64 bits are moved to the
 ///    destination.
 /// \returns A 64-bit signed integer containing the moved value.
-static __inline__ long long __DEFAULT_FN_ATTRS _mm_cvtsi128_si64(__m128i __a) {
+static __inline__ long long __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtsi128_si64(__m128i __a) {
   return __a[0];
 }
 
@@ -4415,8 +4420,8 @@ static __inline__ int __DEFAULT_FN_ATTRS _mm_movemask_epi8(__m128i __a) {
 ///    Bits [119:112] are written to bits [111:104] of the result. \n
 ///    Bits [127:120] are written to bits [127:120] of the result.
 /// \returns A 128-bit vector of [16 x i8] containing the interleaved values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi8(__m128i __a,
-                                                               __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_unpackhi_epi8(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_shufflevector(
       (__v16qi)__a, (__v16qi)__b, 8, 16 + 8, 9, 16 + 9, 10, 16 + 10, 11,
       16 + 11, 12, 16 + 12, 13, 16 + 13, 14, 16 + 14, 15, 16 + 15);
@@ -4443,8 +4448,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi8(__m128i __a,
 ///    Bits [111:96] are written to bits [95:80] of the result. \n
 ///    Bits [127:112] are written to bits [127:112] of the result.
 /// \returns A 128-bit vector of [8 x i16] containing the interleaved values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi16(__m128i __a,
-                                                                __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_unpackhi_epi16(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi)__b, 4, 8 + 4, 5,
                                           8 + 5, 6, 8 + 6, 7, 8 + 7);
 }
@@ -4466,8 +4471,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi16(__m128i __a,
 ///    Bits [95:64] are written to bits [64:32] of the destination. \n
 ///    Bits [127:96] are written to bits [127:96] of the destination.
 /// \returns A 128-bit vector of [4 x i32] containing the interleaved values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi32(__m128i __a,
-                                                                __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_unpackhi_epi32(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si)__b, 2, 4 + 2, 3,
                                           4 + 3);
 }
@@ -4487,8 +4492,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi32(__m128i __a,
 ///    A 128-bit vector of [2 x i64]. \n
 ///    Bits [127:64] are written to bits [127:64] of the destination.
 /// \returns A 128-bit vector of [2 x i64] containing the interleaved values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi64(__m128i __a,
-                                                                __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_unpackhi_epi64(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_shufflevector((__v2di)__a, (__v2di)__b, 1, 2 + 1);
 }
 
@@ -4521,8 +4526,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpackhi_epi64(__m128i __a,
 ///    Bits [55:48] are written to bits [111:104] of the result. \n
 ///    Bits [63:56] are written to bits [127:120] of the result.
 /// \returns A 128-bit vector of [16 x i8] containing the interleaved values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi8(__m128i __a,
-                                                               __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_unpacklo_epi8(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_shufflevector(
       (__v16qi)__a, (__v16qi)__b, 0, 16 + 0, 1, 16 + 1, 2, 16 + 2, 3, 16 + 3, 4,
       16 + 4, 5, 16 + 5, 6, 16 + 6, 7, 16 + 7);
@@ -4550,8 +4555,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi8(__m128i __a,
 ///    Bits [47:32] are written to bits [95:80] of the result. \n
 ///    Bits [63:48] are written to bits [127:112] of the result.
 /// \returns A 128-bit vector of [8 x i16] containing the interleaved values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi16(__m128i __a,
-                                                                __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_unpacklo_epi16(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_shufflevector((__v8hi)__a, (__v8hi)__b, 0, 8 + 0, 1,
                                           8 + 1, 2, 8 + 2, 3, 8 + 3);
 }
@@ -4573,8 +4578,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi16(__m128i __a,
 ///    Bits [31:0] are written to bits [64:32] of the destination. \n
 ///    Bits [63:32] are written to bits [127:96] of the destination.
 /// \returns A 128-bit vector of [4 x i32] containing the interleaved values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi32(__m128i __a,
-                                                                __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_unpacklo_epi32(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_shufflevector((__v4si)__a, (__v4si)__b, 0, 4 + 0, 1,
                                           4 + 1);
 }
@@ -4594,8 +4599,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi32(__m128i __a,
 ///    A 128-bit vector of [2 x i64]. \n
 ///    Bits [63:0] are written to bits [127:64] of the destination. \n
 /// \returns A 128-bit vector of [2 x i64] containing the interleaved values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_unpacklo_epi64(__m128i __a,
-                                                                __m128i __b) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_unpacklo_epi64(__m128i __a, __m128i __b) {
   return (__m128i)__builtin_shufflevector((__v2di)__a, (__v2di)__b, 0, 2 + 0);
 }
 
diff --git a/clang/lib/Headers/f16cintrin.h b/clang/lib/Headers/f16cintrin.h
index 94a662c1d93a8..ede67afada766 100644
--- a/clang/lib/Headers/f16cintrin.h
+++ b/clang/lib/Headers/f16cintrin.h
@@ -38,9 +38,7 @@
 static __inline float __DEFAULT_FN_ATTRS128
 _cvtsh_ss(unsigned short __a)
 {
-  __v8hi __v = {(short)__a, 0, 0, 0, 0, 0, 0, 0};
-  __v4sf __r = __builtin_ia32_vcvtph2ps(__v);
-  return __r[0];
+  return (float)__builtin_bit_cast(__fp16, __a);
 }
 
 /// Converts a 32-bit single-precision float value to a 16-bit
@@ -109,7 +107,10 @@ _cvtsh_ss(unsigned short __a)
 static __inline __m128 __DEFAULT_FN_ATTRS128
 _mm_cvtph_ps(__m128i __a)
 {
-  return (__m128)__builtin_ia32_vcvtph2ps((__v8hi)__a);
+  typedef __fp16 __v4fp16 __attribute__((__vector_size__(8)));
+
+  __v4hi __v = __builtin_shufflevector((__v8hi)__a, (__v8hi)__a, 0, 1, 2, 3);
+  return (__m128) __builtin_convertvector((__v4fp16)__v, __v4sf);
 }
 
 /// Converts a 256-bit vector of [8 x float] into a 128-bit vector
@@ -153,7 +154,9 @@ _mm_cvtph_ps(__m128i __a)
 static __inline __m256 __DEFAULT_FN_ATTRS256
 _mm256_cvtph_ps(__m128i __a)
 {
-  return (__m256)__builtin_ia32_vcvtph2ps256((__v8hi)__a);
+  typedef __fp16 __v8fp16 __attribute__((__vector_size__(16), __aligned__(16)));
+
+  return (__m256) __builtin_convertvector((__v8fp16)__a, __v8sf);
 }
 
 #undef __DEFAULT_FN_ATTRS128
diff --git a/clang/lib/Headers/fma4intrin.h b/clang/lib/Headers/fma4intrin.h
index 694801b3e8dd4..e0a0e4c968950 100644
--- a/clang/lib/Headers/fma4intrin.h
+++ b/clang/lib/Headers/fma4intrin.h
@@ -20,16 +20,24 @@
 #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("fma4"), __min_vector_width__(128)))
 #define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("fma4"), __min_vector_width__(256)))
 
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
-_mm_macc_ps(__m128 __A, __m128 __B, __m128 __C)
-{
-  return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#else
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#endif
+
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_macc_ps(__m128 __A, __m128 __B, __m128 __C) {
+  return (__m128)__builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B,
+                                           (__v4sf)__C);
 }
 
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
-_mm_macc_pd(__m128d __A, __m128d __B, __m128d __C)
-{
-  return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, (__v2df)__C);
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_macc_pd(__m128d __A, __m128d __B, __m128d __C) {
+  return (__m128d)__builtin_elementwise_fma((__v2df)__A, (__v2df)__B,
+                                            (__v2df)__C);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
@@ -44,16 +52,16 @@ _mm_macc_sd(__m128d __A, __m128d __B, __m128d __C)
   return (__m128d)__builtin_ia32_vfmaddsd((__v2df)__A, (__v2df)__B, (__v2df)__C);
 }
 
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
-_mm_msub_ps(__m128 __A, __m128 __B, __m128 __C)
-{
-  return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_msub_ps(__m128 __A, __m128 __B, __m128 __C) {
+  return (__m128)__builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B,
+                                           -(__v4sf)__C);
 }
 
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
-_mm_msub_pd(__m128d __A, __m128d __B, __m128d __C)
-{
-  return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_msub_pd(__m128d __A, __m128d __B, __m128d __C) {
+  return (__m128d)__builtin_elementwise_fma((__v2df)__A, (__v2df)__B,
+                                            -(__v2df)__C);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
@@ -68,16 +76,16 @@ _mm_msub_sd(__m128d __A, __m128d __B, __m128d __C)
   return (__m128d)__builtin_ia32_vfmaddsd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
 }
 
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
-_mm_nmacc_ps(__m128 __A, __m128 __B, __m128 __C)
-{
-  return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_nmacc_ps(__m128 __A, __m128 __B, __m128 __C) {
+  return (__m128)__builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B,
+                                           (__v4sf)__C);
 }
 
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
-_mm_nmacc_pd(__m128d __A, __m128d __B, __m128d __C)
-{
-  return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, (__v2df)__C);
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_nmacc_pd(__m128d __A, __m128d __B, __m128d __C) {
+  return (__m128d)__builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B,
+                                            (__v2df)__C);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
@@ -92,16 +100,16 @@ _mm_nmacc_sd(__m128d __A, __m128d __B, __m128d __C)
   return (__m128d)__builtin_ia32_vfmaddsd(-(__v2df)__A, (__v2df)__B, (__v2df)__C);
 }
 
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
-_mm_nmsub_ps(__m128 __A, __m128 __B, __m128 __C)
-{
-  return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_nmsub_ps(__m128 __A, __m128 __B, __m128 __C) {
+  return (__m128)__builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B,
+                                           -(__v4sf)__C);
 }
 
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
-_mm_nmsub_pd(__m128d __A, __m128d __B, __m128d __C)
-{
-  return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, -(__v2df)__C);
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
+_mm_nmsub_pd(__m128d __A, __m128d __B, __m128d __C) {
+  return (__m128d)__builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B,
+                                            -(__v2df)__C);
 }
 
 static __inline__ __m128 __DEFAULT_FN_ATTRS128
@@ -140,52 +148,52 @@ _mm_msubadd_pd(__m128d __A, __m128d __B, __m128d __C)
   return (__m128d)__builtin_ia32_vfmaddsubpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
 }
 
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
-_mm256_macc_ps(__m256 __A, __m256 __B, __m256 __C)
-{
-  return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_macc_ps(__m256 __A, __m256 __B, __m256 __C) {
+  return (__m256)__builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B,
+                                           (__v8sf)__C);
 }
 
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
-_mm256_macc_pd(__m256d __A, __m256d __B, __m256d __C)
-{
-  return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, (__v4df)__C);
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_macc_pd(__m256d __A, __m256d __B, __m256d __C) {
+  return (__m256d)__builtin_elementwise_fma((__v4df)__A, (__v4df)__B,
+                                            (__v4df)__C);
 }
 
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
-_mm256_msub_ps(__m256 __A, __m256 __B, __m256 __C)
-{
-  return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_msub_ps(__m256 __A, __m256 __B, __m256 __C) {
+  return (__m256)__builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B,
+                                           -(__v8sf)__C);
 }
 
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
-_mm256_msub_pd(__m256d __A, __m256d __B, __m256d __C)
-{
-  return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, -(__v4df)__C);
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_msub_pd(__m256d __A, __m256d __B, __m256d __C) {
+  return (__m256d)__builtin_elementwise_fma((__v4df)__A, (__v4df)__B,
+                                            -(__v4df)__C);
 }
 
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
-_mm256_nmacc_ps(__m256 __A, __m256 __B, __m256 __C)
-{
-  return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_nmacc_ps(__m256 __A, __m256 __B, __m256 __C) {
+  return (__m256)__builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B,
+                                           (__v8sf)__C);
 }
 
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
-_mm256_nmacc_pd(__m256d __A, __m256d __B, __m256d __C)
-{
-  return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, (__v4df)__C);
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_nmacc_pd(__m256d __A, __m256d __B, __m256d __C) {
+  return (__m256d)__builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B,
+                                            (__v4df)__C);
 }
 
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
-_mm256_nmsub_ps(__m256 __A, __m256 __B, __m256 __C)
-{
-  return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_nmsub_ps(__m256 __A, __m256 __B, __m256 __C) {
+  return (__m256)__builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B,
+                                           -(__v8sf)__C);
 }
 
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
-_mm256_nmsub_pd(__m256d __A, __m256d __B, __m256d __C)
-{
-  return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, -(__v4df)__C);
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
+_mm256_nmsub_pd(__m256d __A, __m256d __B, __m256d __C) {
+  return (__m256d)__builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B,
+                                            -(__v4df)__C);
 }
 
 static __inline__ __m256 __DEFAULT_FN_ATTRS256
@@ -214,5 +222,7 @@ _mm256_msubadd_pd(__m256d __A, __m256d __B, __m256d __C)
 
 #undef __DEFAULT_FN_ATTRS128
 #undef __DEFAULT_FN_ATTRS256
+#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
 
 #endif /* __FMA4INTRIN_H */
diff --git a/clang/lib/Headers/fmaintrin.h b/clang/lib/Headers/fmaintrin.h
index 22d1a780bbfd4..d8ea489022b8f 100644
--- a/clang/lib/Headers/fmaintrin.h
+++ b/clang/lib/Headers/fmaintrin.h
@@ -18,6 +18,14 @@
 #define __DEFAULT_FN_ATTRS128 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(128)))
 #define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("fma"), __min_vector_width__(256)))
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128 constexpr
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#else
+#define __DEFAULT_FN_ATTRS128_CONSTEXPR __DEFAULT_FN_ATTRS128
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#endif
+
 /// Computes a multiply-add of 128-bit vectors of [4 x float].
 ///    For each element, computes <c> (__A * __B) + __C </c>.
 ///
@@ -32,10 +40,11 @@
 /// \param __C
 ///    A 128-bit vector of [4 x float] containing the addend.
 /// \returns A 128-bit vector of [4 x float] containing the result.
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_fmadd_ps(__m128 __A, __m128 __B, __m128 __C)
 {
-  return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
+  return (__m128)__builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B,
+                                           (__v4sf)__C);
 }
 
 /// Computes a multiply-add of 128-bit vectors of [2 x double].
@@ -52,10 +61,11 @@ _mm_fmadd_ps(__m128 __A, __m128 __B, __m128 __C)
 /// \param __C
 ///    A 128-bit vector of [2 x double] containing the addend.
 /// \returns A 128-bit [2 x double] vector containing the result.
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_fmadd_pd(__m128d __A, __m128d __B, __m128d __C)
 {
-  return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, (__v2df)__C);
+  return (__m128d)__builtin_elementwise_fma((__v2df)__A, (__v2df)__B,
+                                            (__v2df)__C);
 }
 
 /// Computes a scalar multiply-add of the single-precision values in the
@@ -130,10 +140,11 @@ _mm_fmadd_sd(__m128d __A, __m128d __B, __m128d __C)
 /// \param __C
 ///    A 128-bit vector of [4 x float] containing the subtrahend.
 /// \returns A 128-bit vector of [4 x float] containing the result.
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_fmsub_ps(__m128 __A, __m128 __B, __m128 __C)
 {
-  return (__m128)__builtin_ia32_vfmaddps((__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
+  return (__m128)__builtin_elementwise_fma((__v4sf)__A, (__v4sf)__B,
+                                           -(__v4sf)__C);
 }
 
 /// Computes a multiply-subtract of 128-bit vectors of [2 x double].
@@ -150,10 +161,11 @@ _mm_fmsub_ps(__m128 __A, __m128 __B, __m128 __C)
 /// \param __C
 ///    A 128-bit vector of [2 x double] containing the addend.
 /// \returns A 128-bit vector of [2 x double] containing the result.
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_fmsub_pd(__m128d __A, __m128d __B, __m128d __C)
 {
-  return (__m128d)__builtin_ia32_vfmaddpd((__v2df)__A, (__v2df)__B, -(__v2df)__C);
+  return (__m128d)__builtin_elementwise_fma((__v2df)__A, (__v2df)__B,
+                                            -(__v2df)__C);
 }
 
 /// Computes a scalar multiply-subtract of the single-precision values in
@@ -228,10 +240,11 @@ _mm_fmsub_sd(__m128d __A, __m128d __B, __m128d __C)
 /// \param __C
 ///    A 128-bit vector of [4 x float] containing the addend.
 /// \returns A 128-bit [4 x float] vector containing the result.
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C)
 {
-  return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, (__v4sf)__C);
+  return (__m128)__builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B,
+                                           (__v4sf)__C);
 }
 
 /// Computes a negated multiply-add of 128-bit vectors of [2 x double].
@@ -248,10 +261,11 @@ _mm_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C)
 /// \param __C
 ///    A 128-bit vector of [2 x double] containing the addend.
 /// \returns A 128-bit vector of [2 x double] containing the result.
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C)
 {
-  return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, (__v2df)__C);
+  return (__m128d)__builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B,
+                                            (__v2df)__C);
 }
 
 /// Computes a scalar negated multiply-add of the single-precision values in
@@ -326,10 +340,11 @@ _mm_fnmadd_sd(__m128d __A, __m128d __B, __m128d __C)
 /// \param __C
 ///    A 128-bit vector of [4 x float] containing the subtrahend.
 /// \returns A 128-bit vector of [4 x float] containing the result.
-static __inline__ __m128 __DEFAULT_FN_ATTRS128
+static __inline__ __m128 __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C)
 {
-  return (__m128)__builtin_ia32_vfmaddps(-(__v4sf)__A, (__v4sf)__B, -(__v4sf)__C);
+  return (__m128)__builtin_elementwise_fma(-(__v4sf)__A, (__v4sf)__B,
+                                           -(__v4sf)__C);
 }
 
 /// Computes a negated multiply-subtract of 128-bit vectors of [2 x double].
@@ -346,10 +361,11 @@ _mm_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C)
 /// \param __C
 ///    A 128-bit vector of [2 x double] containing the subtrahend.
 /// \returns A 128-bit vector of [2 x double] containing the result.
-static __inline__ __m128d __DEFAULT_FN_ATTRS128
+static __inline__ __m128d __DEFAULT_FN_ATTRS128_CONSTEXPR
 _mm_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C)
 {
-  return (__m128d)__builtin_ia32_vfmaddpd(-(__v2df)__A, (__v2df)__B, -(__v2df)__C);
+  return (__m128d)__builtin_elementwise_fma(-(__v2df)__A, (__v2df)__B,
+                                            -(__v2df)__C);
 }
 
 /// Computes a scalar negated multiply-subtract of the single-precision
@@ -528,10 +544,11 @@ _mm_fmsubadd_pd(__m128d __A, __m128d __B, __m128d __C)
 /// \param __C
 ///    A 256-bit vector of [8 x float] containing the addend.
 /// \returns A 256-bit vector of [8 x float] containing the result.
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_fmadd_ps(__m256 __A, __m256 __B, __m256 __C)
 {
-  return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
+  return (__m256)__builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B,
+                                           (__v8sf)__C);
 }
 
 /// Computes a multiply-add of 256-bit vectors of [4 x double].
@@ -548,10 +565,11 @@ _mm256_fmadd_ps(__m256 __A, __m256 __B, __m256 __C)
 /// \param __C
 ///    A 256-bit vector of [4 x double] containing the addend.
 /// \returns A 256-bit vector of [4 x double] containing the result.
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_fmadd_pd(__m256d __A, __m256d __B, __m256d __C)
 {
-  return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, (__v4df)__C);
+  return (__m256d)__builtin_elementwise_fma((__v4df)__A, (__v4df)__B,
+                                            (__v4df)__C);
 }
 
 /// Computes a multiply-subtract of 256-bit vectors of [8 x float].
@@ -568,10 +586,11 @@ _mm256_fmadd_pd(__m256d __A, __m256d __B, __m256d __C)
 /// \param __C
 ///    A 256-bit vector of [8 x float] containing the subtrahend.
 /// \returns A 256-bit vector of [8 x float] containing the result.
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_fmsub_ps(__m256 __A, __m256 __B, __m256 __C)
 {
-  return (__m256)__builtin_ia32_vfmaddps256((__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
+  return (__m256)__builtin_elementwise_fma((__v8sf)__A, (__v8sf)__B,
+                                           -(__v8sf)__C);
 }
 
 /// Computes a multiply-subtract of 256-bit vectors of [4 x double].
@@ -588,10 +607,11 @@ _mm256_fmsub_ps(__m256 __A, __m256 __B, __m256 __C)
 /// \param __C
 ///    A 256-bit vector of [4 x double] containing the subtrahend.
 /// \returns A 256-bit vector of [4 x double] containing the result.
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_fmsub_pd(__m256d __A, __m256d __B, __m256d __C)
 {
-  return (__m256d)__builtin_ia32_vfmaddpd256((__v4df)__A, (__v4df)__B, -(__v4df)__C);
+  return (__m256d)__builtin_elementwise_fma((__v4df)__A, (__v4df)__B,
+                                            -(__v4df)__C);
 }
 
 /// Computes a negated multiply-add of 256-bit vectors of [8 x float].
@@ -608,10 +628,11 @@ _mm256_fmsub_pd(__m256d __A, __m256d __B, __m256d __C)
 /// \param __C
 ///    A 256-bit vector of [8 x float] containing the addend.
 /// \returns A 256-bit vector of [8 x float] containing the result.
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C)
 {
-  return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, (__v8sf)__C);
+  return (__m256)__builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B,
+                                           (__v8sf)__C);
 }
 
 /// Computes a negated multiply-add of 256-bit vectors of [4 x double].
@@ -628,10 +649,11 @@ _mm256_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C)
 /// \param __C
 ///    A 256-bit vector of [4 x double] containing the addend.
 /// \returns A 256-bit vector of [4 x double] containing the result.
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C)
 {
-  return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, (__v4df)__C);
+  return (__m256d)__builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B,
+                                            (__v4df)__C);
 }
 
 /// Computes a negated multiply-subtract of 256-bit vectors of [8 x float].
@@ -648,10 +670,11 @@ _mm256_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C)
 /// \param __C
 ///    A 256-bit vector of [8 x float] containing the subtrahend.
 /// \returns A 256-bit vector of [8 x float] containing the result.
-static __inline__ __m256 __DEFAULT_FN_ATTRS256
+static __inline__ __m256 __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C)
 {
-  return (__m256)__builtin_ia32_vfmaddps256(-(__v8sf)__A, (__v8sf)__B, -(__v8sf)__C);
+  return (__m256)__builtin_elementwise_fma(-(__v8sf)__A, (__v8sf)__B,
+                                           -(__v8sf)__C);
 }
 
 /// Computes a negated multiply-subtract of 256-bit vectors of [4 x double].
@@ -668,10 +691,11 @@ _mm256_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C)
 /// \param __C
 ///    A 256-bit vector of [4 x double] containing the subtrahend.
 /// \returns A 256-bit vector of [4 x double] containing the result.
-static __inline__ __m256d __DEFAULT_FN_ATTRS256
+static __inline__ __m256d __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C)
 {
-  return (__m256d)__builtin_ia32_vfmaddpd256(-(__v4df)__A, (__v4df)__B, -(__v4df)__C);
+  return (__m256d)__builtin_elementwise_fma(-(__v4df)__A, (__v4df)__B,
+                                            -(__v4df)__C);
 }
 
 /// Computes a multiply with alternating add/subtract of 256-bit vectors of
@@ -792,5 +816,7 @@ _mm256_fmsubadd_pd(__m256d __A, __m256d __B, __m256d __C)
 
 #undef __DEFAULT_FN_ATTRS128
 #undef __DEFAULT_FN_ATTRS256
+#undef __DEFAULT_FN_ATTRS128_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
 
 #endif /* __FMAINTRIN_H */
diff --git a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
index cbc518d050583..21a9c30d9f445 100644
--- a/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
+++ b/clang/lib/Headers/hlsl/hlsl_alias_intrinsics.h
@@ -2190,10 +2190,10 @@ float4 sqrt(float4);
 // step builtins
 //===----------------------------------------------------------------------===//
 
-/// \fn T step(T x, T y)
+/// \fn T step(T y, T x)
 /// \brief Returns 1 if the x parameter is greater than or equal to the y
-/// parameter; otherwise, 0. vector. \param x [in] The first floating-point
-/// value to compare. \param y [in] The first floating-point value to compare.
+/// parameter; otherwise, 0. vector. \param y [in] The first floating-point
+/// value to compare. \param x [in] The second floating-point value to compare.
 ///
 /// Step is based on the following formula: (x >= y) ? 1 : 0
 
diff --git a/clang/lib/Headers/mmintrin.h b/clang/lib/Headers/mmintrin.h
index dc0fa5c523eeb..6fe9d67b8976d 100644
--- a/clang/lib/Headers/mmintrin.h
+++ b/clang/lib/Headers/mmintrin.h
@@ -57,6 +57,9 @@ typedef char __v16qi __attribute__((__vector_size__(16)));
 
 #define __trunc64(x)                                                           \
   (__m64) __builtin_shufflevector((__v2di)(x), __extension__(__v2di){}, 0)
+#define __zext128(x)                                                           \
+  (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0,   \
+                                    1, 2, 3)
 #define __anyext128(x)                                                         \
   (__m128i) __builtin_shufflevector((__v2si)(x), __extension__(__v2si){}, 0,   \
                                     1, -1, -1)
@@ -85,7 +88,7 @@ _mm_empty(void) {
 ///    A 32-bit integer value.
 /// \returns A 64-bit integer vector. The lower 32 bits contain the value of the
 ///    parameter. The upper 32 bits are set to 0.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtsi32_si64(int __i)
 {
     return __extension__ (__m64)(__v2si){__i, 0};
@@ -102,7 +105,7 @@ _mm_cvtsi32_si64(int __i)
 ///    A 64-bit integer vector.
 /// \returns A 32-bit signed integer value containing the lower 32 bits of the
 ///    parameter.
-static __inline__ int __DEFAULT_FN_ATTRS_SSE2
+static __inline__ int __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtsi64_si32(__m64 __m)
 {
     return ((__v2si)__m)[0];
@@ -118,10 +121,10 @@ _mm_cvtsi64_si32(__m64 __m)
 ///    A 64-bit signed integer.
 /// \returns A 64-bit integer vector containing the same bitwise pattern as the
 ///    parameter.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtsi64_m64(long long __i)
 {
-    return (__m64)__i;
+    return __extension__ (__m64)(__v1di){__i};
 }
 
 /// Casts a 64-bit integer vector into a 64-bit signed integer value.
@@ -134,10 +137,10 @@ _mm_cvtsi64_m64(long long __i)
 ///    A 64-bit integer vector.
 /// \returns A 64-bit signed integer containing the same bitwise pattern as the
 ///    parameter.
-static __inline__ long long __DEFAULT_FN_ATTRS_SSE2
+static __inline__ long long __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtm64_si64(__m64 __m)
 {
-    return (long long)__m;
+    return ((__v1di)__m)[0];
 }
 
 /// Converts, with saturation, 16-bit signed integers from both 64-bit integer
@@ -239,11 +242,10 @@ _mm_packs_pu16(__m64 __m1, __m64 __m2)
 ///    Bits [63:56] are written to bits [63:56] of the result.
 /// \returns A 64-bit integer vector of [8 x i8] containing the interleaved
 ///    values.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_unpackhi_pi8(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2,
-                                          4, 12, 5, 13, 6, 14, 7, 15);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_unpackhi_pi8(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2, 4, 12, 5,
+                                        13, 6, 14, 7, 15);
 }
 
 /// Unpacks the upper 32 bits from two 64-bit integer vectors of
@@ -263,11 +265,9 @@ _mm_unpackhi_pi8(__m64 __m1, __m64 __m2)
 ///    Bits [63:48] are written to bits [63:48] of the result.
 /// \returns A 64-bit integer vector of [4 x i16] containing the interleaved
 ///    values.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_unpackhi_pi16(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2,
-                                          2, 6, 3, 7);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_unpackhi_pi16(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2, 2, 6, 3, 7);
 }
 
 /// Unpacks the upper 32 bits from two 64-bit integer vectors of
@@ -285,10 +285,9 @@ _mm_unpackhi_pi16(__m64 __m1, __m64 __m2)
 ///    the upper 32 bits of the result.
 /// \returns A 64-bit integer vector of [2 x i32] containing the interleaved
 ///    values.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_unpackhi_pi32(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 1, 3);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_unpackhi_pi32(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 1, 3);
 }
 
 /// Unpacks the lower 32 bits from two 64-bit integer vectors of [8 x i8]
@@ -312,11 +311,10 @@ _mm_unpackhi_pi32(__m64 __m1, __m64 __m2)
 ///    Bits [31:24] are written to bits [63:56] of the result.
 /// \returns A 64-bit integer vector of [8 x i8] containing the interleaved
 ///    values.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_unpacklo_pi8(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2,
-                                          0, 8, 1, 9, 2, 10, 3, 11);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_unpacklo_pi8(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_shufflevector((__v8qi)__m1, (__v8qi)__m2, 0, 8, 1, 9,
+                                        2, 10, 3, 11);
 }
 
 /// Unpacks the lower 32 bits from two 64-bit integer vectors of
@@ -336,11 +334,9 @@ _mm_unpacklo_pi8(__m64 __m1, __m64 __m2)
 ///    Bits [31:16] are written to bits [63:48] of the result.
 /// \returns A 64-bit integer vector of [4 x i16] containing the interleaved
 ///    values.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_unpacklo_pi16(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2,
-                                          0, 4, 1, 5);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_unpacklo_pi16(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_shufflevector((__v4hi)__m1, (__v4hi)__m2, 0, 4, 1, 5);
 }
 
 /// Unpacks the lower 32 bits from two 64-bit integer vectors of
@@ -358,10 +354,9 @@ _mm_unpacklo_pi16(__m64 __m1, __m64 __m2)
 ///    the upper 32 bits of the result.
 /// \returns A 64-bit integer vector of [2 x i32] containing the interleaved
 ///    values.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_unpacklo_pi32(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 0, 2);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_unpacklo_pi32(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_shufflevector((__v2si)__m1, (__v2si)__m2, 0, 2);
 }
 
 /// Adds each 8-bit integer element of the first 64-bit integer vector
@@ -379,7 +374,7 @@ _mm_unpacklo_pi32(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [8 x i8].
 /// \returns A 64-bit integer vector of [8 x i8] containing the sums of both
 ///    parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_add_pi8(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v8qu)__m1) + ((__v8qu)__m2));
@@ -400,7 +395,7 @@ _mm_add_pi8(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16].
 /// \returns A 64-bit integer vector of [4 x i16] containing the sums of both
 ///    parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_add_pi16(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v4hu)__m1) + ((__v4hu)__m2));
@@ -421,7 +416,7 @@ _mm_add_pi16(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [2 x i32].
 /// \returns A 64-bit integer vector of [2 x i32] containing the sums of both
 ///    parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_add_pi32(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v2su)__m1) + ((__v2su)__m2));
@@ -445,10 +440,9 @@ _mm_add_pi32(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [8 x i8].
 /// \returns A 64-bit integer vector of [8 x i8] containing the saturated sums
 ///    of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_adds_pi8(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_elementwise_add_sat((__v8qs)__m1, (__v8qs)__m2);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_adds_pi8(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_elementwise_add_sat((__v8qs)__m1, (__v8qs)__m2);
 }
 
 /// Adds, with saturation, each 16-bit signed integer element of the first
@@ -469,10 +463,9 @@ _mm_adds_pi8(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16].
 /// \returns A 64-bit integer vector of [4 x i16] containing the saturated sums
 ///    of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_adds_pi16(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_elementwise_add_sat((__v4hi)__m1, (__v4hi)__m2);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_adds_pi16(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_elementwise_add_sat((__v4hi)__m1, (__v4hi)__m2);
 }
 
 /// Adds, with saturation, each 8-bit unsigned integer element of the first
@@ -492,10 +485,9 @@ _mm_adds_pi16(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [8 x i8].
 /// \returns A 64-bit integer vector of [8 x i8] containing the saturated
 ///    unsigned sums of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_adds_pu8(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_elementwise_add_sat((__v8qu)__m1, (__v8qu)__m2);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_adds_pu8(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_elementwise_add_sat((__v8qu)__m1, (__v8qu)__m2);
 }
 
 /// Adds, with saturation, each 16-bit unsigned integer element of the first
@@ -515,10 +507,9 @@ _mm_adds_pu8(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16].
 /// \returns A 64-bit integer vector of [4 x i16] containing the saturated
 ///    unsigned sums of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_adds_pu16(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_elementwise_add_sat((__v4hu)__m1, (__v4hu)__m2);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_adds_pu16(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_elementwise_add_sat((__v4hu)__m1, (__v4hu)__m2);
 }
 
 /// Subtracts each 8-bit integer element of the second 64-bit integer
@@ -536,7 +527,7 @@ _mm_adds_pu16(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [8 x i8] containing the subtrahends.
 /// \returns A 64-bit integer vector of [8 x i8] containing the differences of
 ///    both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_sub_pi8(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v8qu)__m1) - ((__v8qu)__m2));
@@ -557,7 +548,7 @@ _mm_sub_pi8(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16] containing the subtrahends.
 /// \returns A 64-bit integer vector of [4 x i16] containing the differences of
 ///    both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_sub_pi16(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v4hu)__m1) - ((__v4hu)__m2));
@@ -578,7 +569,7 @@ _mm_sub_pi16(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [2 x i32] containing the subtrahends.
 /// \returns A 64-bit integer vector of [2 x i32] containing the differences of
 ///    both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_sub_pi32(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v2su)__m1) - ((__v2su)__m2));
@@ -602,10 +593,9 @@ _mm_sub_pi32(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [8 x i8] containing the subtrahends.
 /// \returns A 64-bit integer vector of [8 x i8] containing the saturated
 ///    differences of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_subs_pi8(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_elementwise_sub_sat((__v8qs)__m1, (__v8qs)__m2);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_subs_pi8(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_elementwise_sub_sat((__v8qs)__m1, (__v8qs)__m2);
 }
 
 /// Subtracts, with saturation, each 16-bit signed integer element of the
@@ -626,10 +616,9 @@ _mm_subs_pi8(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16] containing the subtrahends.
 /// \returns A 64-bit integer vector of [4 x i16] containing the saturated
 ///    differences of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_subs_pi16(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_elementwise_sub_sat((__v4hi)__m1, (__v4hi)__m2);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_subs_pi16(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_elementwise_sub_sat((__v4hi)__m1, (__v4hi)__m2);
 }
 
 /// Subtracts each 8-bit unsigned integer element of the second 64-bit
@@ -650,10 +639,9 @@ _mm_subs_pi16(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [8 x i8] containing the subtrahends.
 /// \returns A 64-bit integer vector of [8 x i8] containing the saturated
 ///    differences of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_subs_pu8(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_elementwise_sub_sat((__v8qu)__m1, (__v8qu)__m2);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_subs_pu8(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_elementwise_sub_sat((__v8qu)__m1, (__v8qu)__m2);
 }
 
 /// Subtracts each 16-bit unsigned integer element of the second 64-bit
@@ -674,10 +662,9 @@ _mm_subs_pu8(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16] containing the subtrahends.
 /// \returns A 64-bit integer vector of [4 x i16] containing the saturated
 ///    differences of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
-_mm_subs_pu16(__m64 __m1, __m64 __m2)
-{
-    return (__m64)__builtin_elementwise_sub_sat((__v4hu)__m1, (__v4hu)__m2);
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
+_mm_subs_pu16(__m64 __m1, __m64 __m2) {
+  return (__m64)__builtin_elementwise_sub_sat((__v4hu)__m1, (__v4hu)__m2);
 }
 
 /// Multiplies each 16-bit signed integer element of the first 64-bit
@@ -723,11 +710,11 @@ _mm_madd_pi16(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16].
 /// \returns A 64-bit integer vector of [4 x i16] containing the upper 16 bits
 ///    of the products of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_mulhi_pi16(__m64 __m1, __m64 __m2)
 {
-    return __trunc64(__builtin_ia32_pmulhw128((__v8hi)__anyext128(__m1),
-                                              (__v8hi)__anyext128(__m2)));
+    return __trunc64(__builtin_ia32_pmulhw128((__v8hi)__zext128(__m1),
+                                              (__v8hi)__zext128(__m2)));
 }
 
 /// Multiplies each 16-bit signed integer element of the first 64-bit
@@ -745,7 +732,7 @@ _mm_mulhi_pi16(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16].
 /// \returns A 64-bit integer vector of [4 x i16] containing the lower 16 bits
 ///    of the products of both parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_mullo_pi16(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v4hu)__m1) * ((__v4hu)__m2));
@@ -1134,7 +1121,7 @@ _mm_srli_si64(__m64 __m, int __count)
 ///    A 64-bit integer vector.
 /// \returns A 64-bit integer vector containing the bitwise AND of both
 ///    parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_and_si64(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v1du)__m1) & ((__v1du)__m2));
@@ -1155,7 +1142,7 @@ _mm_and_si64(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector.
 /// \returns A 64-bit integer vector containing the bitwise AND of the second
 ///    parameter and the one's complement of the first parameter.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_andnot_si64(__m64 __m1, __m64 __m2)
 {
     return (__m64)(~((__v1du)__m1) & ((__v1du)__m2));
@@ -1173,7 +1160,7 @@ _mm_andnot_si64(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector.
 /// \returns A 64-bit integer vector containing the bitwise OR of both
 ///    parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_or_si64(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v1du)__m1) | ((__v1du)__m2));
@@ -1191,7 +1178,7 @@ _mm_or_si64(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector.
 /// \returns A 64-bit integer vector containing the bitwise exclusive OR of both
 ///    parameters.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_xor_si64(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v1du)__m1) ^ ((__v1du)__m2));
@@ -1213,7 +1200,7 @@ _mm_xor_si64(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [8 x i8].
 /// \returns A 64-bit integer vector of [8 x i8] containing the comparison
 ///    results.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cmpeq_pi8(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v8qi)__m1) == ((__v8qi)__m2));
@@ -1235,7 +1222,7 @@ _mm_cmpeq_pi8(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16].
 /// \returns A 64-bit integer vector of [4 x i16] containing the comparison
 ///    results.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cmpeq_pi16(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v4hi)__m1) == ((__v4hi)__m2));
@@ -1257,7 +1244,7 @@ _mm_cmpeq_pi16(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [2 x i32].
 /// \returns A 64-bit integer vector of [2 x i32] containing the comparison
 ///    results.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cmpeq_pi32(__m64 __m1, __m64 __m2)
 {
     return (__m64)(((__v2si)__m1) == ((__v2si)__m2));
@@ -1279,7 +1266,7 @@ _mm_cmpeq_pi32(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [8 x i8].
 /// \returns A 64-bit integer vector of [8 x i8] containing the comparison
 ///    results.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cmpgt_pi8(__m64 __m1, __m64 __m2)
 {
   /* This function always performs a signed comparison, but __v8qi is a char
@@ -1303,7 +1290,7 @@ _mm_cmpgt_pi8(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [4 x i16].
 /// \returns A 64-bit integer vector of [4 x i16] containing the comparison
 ///    results.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cmpgt_pi16(__m64 __m1, __m64 __m2)
 {
     return (__m64)((__v4hi)__m1 > (__v4hi)__m2);
@@ -1325,7 +1312,7 @@ _mm_cmpgt_pi16(__m64 __m1, __m64 __m2)
 ///    A 64-bit integer vector of [2 x i32].
 /// \returns A 64-bit integer vector of [2 x i32] containing the comparison
 ///    results.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cmpgt_pi32(__m64 __m1, __m64 __m2)
 {
     return (__m64)((__v2si)__m1 > (__v2si)__m2);
diff --git a/clang/lib/Headers/ptrauth.h b/clang/lib/Headers/ptrauth.h
index 7f7d387cbdfda..f902ca1e3bbd3 100644
--- a/clang/lib/Headers/ptrauth.h
+++ b/clang/lib/Headers/ptrauth.h
@@ -95,7 +95,7 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
    __ptrauth qualifier; the compiler will perform this check
    automatically. */
 
-#if __has_feature(ptrauth_intrinsics)
+#if __has_feature(ptrauth_intrinsics) || defined(__PTRAUTH__)
 
 /* Strip the signature from a value without authenticating it.
 
@@ -388,6 +388,6 @@ typedef __UINTPTR_TYPE__ ptrauth_generic_signature_t;
 #define __ptrauth_objc_isa_uintptr
 #define __ptrauth_objc_super_pointer
 
-#endif /* __has_feature(ptrauth_intrinsics) */
+#endif /* __has_feature(ptrauth_intrinsics) || defined(__PTRAUTH__) */
 
 #endif /* __PTRAUTH_H */
diff --git a/clang/lib/Headers/smmintrin.h b/clang/lib/Headers/smmintrin.h
index bc6fe4c801d7e..57d0d329312af 100644
--- a/clang/lib/Headers/smmintrin.h
+++ b/clang/lib/Headers/smmintrin.h
@@ -27,6 +27,12 @@
                  __min_vector_width__(128)))
 #endif
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
+#else
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#endif
+
 /* SSE4 Rounding macros. */
 #define _MM_FROUND_TO_NEAREST_INT 0x00
 #define _MM_FROUND_TO_NEG_INF 0x01
@@ -561,8 +567,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi32(__m128i __V1,
 ///    A 128-bit vector of [4 x i32].
 /// \returns A 128-bit vector of [2 x i64] containing the products of both
 ///    operands.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mul_epi32(__m128i __V1,
-                                                           __m128i __V2) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_mul_epi32(__m128i __V1, __m128i __V2) {
   return (__m128i)__builtin_ia32_pmuldq128((__v4si)__V1, (__v4si)__V2);
 }
 
@@ -1224,7 +1230,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi64(__m128i __V1,
 ///    A 128-bit vector of [16 x i8]. The lower eight 8-bit elements are
 ///    sign-extended to 16-bit values.
 /// \returns A 128-bit vector of [8 x i16] containing the sign-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi16(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepi8_epi16(__m128i __V) {
   /* This function always performs a signed extension, but __v16qi is a char
      which may be signed or unsigned, so use __v16qs. */
   return (__m128i) __builtin_convertvector(
@@ -1246,7 +1253,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi16(__m128i __V) {
 ///    A 128-bit vector of [16 x i8]. The lower four 8-bit elements are
 ///    sign-extended to 32-bit values.
 /// \returns A 128-bit vector of [4 x i32] containing the sign-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi32(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepi8_epi32(__m128i __V) {
   /* This function always performs a signed extension, but __v16qi is a char
      which may be signed or unsigned, so use __v16qs. */
   return (__m128i) __builtin_convertvector(
@@ -1266,7 +1274,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi32(__m128i __V) {
 ///    A 128-bit vector of [16 x i8]. The lower two 8-bit elements are
 ///    sign-extended to 64-bit values.
 /// \returns A 128-bit vector of [2 x i64] containing the sign-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi64(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepi8_epi64(__m128i __V) {
   /* This function always performs a signed extension, but __v16qi is a char
      which may be signed or unsigned, so use __v16qs. */
   return (__m128i) __builtin_convertvector(
@@ -1286,7 +1295,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi64(__m128i __V) {
 ///    A 128-bit vector of [8 x i16]. The lower four 16-bit elements are
 ///    sign-extended to 32-bit values.
 /// \returns A 128-bit vector of [4 x i32] containing the sign-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi32(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepi16_epi32(__m128i __V) {
   return (__m128i) __builtin_convertvector(
       __builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4si);
 }
@@ -1304,7 +1314,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi32(__m128i __V) {
 ///    A 128-bit vector of [8 x i16]. The lower two 16-bit elements are
 ///     sign-extended to 64-bit values.
 /// \returns A 128-bit vector of [2 x i64] containing the sign-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi64(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepi16_epi64(__m128i __V) {
   return (__m128i) __builtin_convertvector(
       __builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1), __v2di);
 }
@@ -1322,7 +1333,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi64(__m128i __V) {
 ///    A 128-bit vector of [4 x i32]. The lower two 32-bit elements are
 ///    sign-extended to 64-bit values.
 /// \returns A 128-bit vector of [2 x i64] containing the sign-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi32_epi64(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepi32_epi64(__m128i __V) {
   return (__m128i) __builtin_convertvector(
       __builtin_shufflevector((__v4si)__V, (__v4si)__V, 0, 1), __v2di);
 }
@@ -1341,7 +1353,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi32_epi64(__m128i __V) {
 ///    A 128-bit vector of [16 x i8]. The lower eight 8-bit elements are
 ///    zero-extended to 16-bit values.
 /// \returns A 128-bit vector of [8 x i16] containing the zero-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi16(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepu8_epi16(__m128i __V) {
   return (__m128i) __builtin_convertvector(
       __builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3, 4, 5, 6,
                               7),
@@ -1361,7 +1374,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi16(__m128i __V) {
 ///    A 128-bit vector of [16 x i8]. The lower four 8-bit elements are
 ///    zero-extended to 32-bit values.
 /// \returns A 128-bit vector of [4 x i32] containing the zero-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi32(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepu8_epi32(__m128i __V) {
   return (__m128i) __builtin_convertvector(
       __builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1, 2, 3), __v4si);
 }
@@ -1379,7 +1393,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi32(__m128i __V) {
 ///    A 128-bit vector of [16 x i8]. The lower two 8-bit elements are
 ///    zero-extended to 64-bit values.
 /// \returns A 128-bit vector of [2 x i64] containing the zero-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi64(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepu8_epi64(__m128i __V) {
   return (__m128i) __builtin_convertvector(
       __builtin_shufflevector((__v16qu)__V, (__v16qu)__V, 0, 1), __v2di);
 }
@@ -1397,7 +1412,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi64(__m128i __V) {
 ///    A 128-bit vector of [8 x i16]. The lower four 16-bit elements are
 ///    zero-extended to 32-bit values.
 /// \returns A 128-bit vector of [4 x i32] containing the zero-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi32(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepu16_epi32(__m128i __V) {
   return (__m128i) __builtin_convertvector(
       __builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1, 2, 3), __v4si);
 }
@@ -1415,7 +1431,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi32(__m128i __V) {
 ///    A 128-bit vector of [8 x i16]. The lower two 16-bit elements are
 ///    zero-extended to 64-bit values.
 /// \returns A 128-bit vector of [2 x i64] containing the zero-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi64(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepu16_epi64(__m128i __V) {
   return (__m128i) __builtin_convertvector(
       __builtin_shufflevector((__v8hu)__V, (__v8hu)__V, 0, 1), __v2di);
 }
@@ -1433,7 +1450,8 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi64(__m128i __V) {
 ///    A 128-bit vector of [4 x i32]. The lower two 32-bit elements are
 ///    zero-extended to 64-bit values.
 /// \returns A 128-bit vector of [2 x i64] containing the zero-extended values.
-static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu32_epi64(__m128i __V) {
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
+_mm_cvtepu32_epi64(__m128i __V) {
   return (__m128i) __builtin_convertvector(
       __builtin_shufflevector((__v4su)__V, (__v4su)__V, 0, 1), __v2di);
 }
@@ -2326,6 +2344,7 @@ static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi64(__m128i __V1,
 }
 
 #undef __DEFAULT_FN_ATTRS
+#undef __DEFAULT_FN_ATTRS_CONSTEXPR
 
 #include <popcntintrin.h>
 
diff --git a/clang/lib/Headers/xmmintrin.h b/clang/lib/Headers/xmmintrin.h
index 6a6436977303f..6d44cff46661f 100644
--- a/clang/lib/Headers/xmmintrin.h
+++ b/clang/lib/Headers/xmmintrin.h
@@ -16,7 +16,6 @@
 
 #include <mmintrin.h>
 
-typedef int __v4si __attribute__((__vector_size__(16)));
 typedef float __v4sf __attribute__((__vector_size__(16)));
 typedef float __m128 __attribute__((__vector_size__(16), __aligned__(16)));
 
@@ -24,6 +23,7 @@ typedef float __m128_u __attribute__((__vector_size__(16), __aligned__(1)));
 
 /* Unsigned types */
 typedef unsigned int __v4su __attribute__((__vector_size__(16)));
+typedef unsigned short __v8hu __attribute__((__vector_size__(16)));
 
 /* This header should only be included in a hosted environment as it depends on
  * a standard library to provide allocation routines. */
@@ -1688,7 +1688,7 @@ _mm_cvtsi64_ss(__m128 __a, long long __b) {
 /// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
 ///    converted value of the second operand. The upper 64 bits are copied from
 ///    the upper 64 bits of the first operand.
-static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtpi32_ps(__m128 __a, __m64 __b)
 {
   return (__m128)__builtin_shufflevector(
@@ -1714,7 +1714,7 @@ _mm_cvtpi32_ps(__m128 __a, __m64 __b)
 /// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
 ///    converted value from the second operand. The upper 64 bits are copied
 ///    from the upper 64 bits of the first operand.
-static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvt_pi2ps(__m128 __a, __m64 __b)
 {
   return _mm_cvtpi32_ps(__a, __b);
@@ -2447,11 +2447,11 @@ _mm_movemask_pi8(__m64 __a)
 /// \param __b
 ///    A 64-bit integer vector containing one of the source operands.
 /// \returns A 64-bit integer vector containing the products of both operands.
-static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m64 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_mulhi_pu16(__m64 __a, __m64 __b)
 {
-  return __trunc64(__builtin_ia32_pmulhuw128((__v8hi)__anyext128(__a),
-                                             (__v8hi)__anyext128(__b)));
+  return __trunc64(__builtin_ia32_pmulhuw128((__v8hu)__zext128(__a),
+                                             (__v8hu)__zext128(__b)));
 }
 
 /// Shuffles the 4 16-bit integers from a 64-bit integer vector to the
@@ -2873,7 +2873,7 @@ _mm_movelh_ps(__m128 __a, __m128 __b) {
 ///    from the corresponding elements in this operand.
 /// \returns A 128-bit vector of [4 x float] containing the copied and converted
 ///    values from the operand.
-static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtpi16_ps(__m64 __a)
 {
   return __builtin_convertvector((__v4hi)__a, __v4sf);
@@ -2891,7 +2891,7 @@ _mm_cvtpi16_ps(__m64 __a)
 ///    destination are copied from the corresponding elements in this operand.
 /// \returns A 128-bit vector of [4 x float] containing the copied and converted
 ///    values from the operand.
-static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtpu16_ps(__m64 __a)
 {
   return __builtin_convertvector((__v4hu)__a, __v4sf);
@@ -2909,7 +2909,7 @@ _mm_cvtpu16_ps(__m64 __a)
 ///    from the corresponding lower 4 elements in this operand.
 /// \returns A 128-bit vector of [4 x float] containing the copied and converted
 ///    values from the operand.
-static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtpi8_ps(__m64 __a)
 {
   return __builtin_convertvector(
@@ -2930,7 +2930,7 @@ _mm_cvtpi8_ps(__m64 __a)
 ///    operand.
 /// \returns A 128-bit vector of [4 x float] containing the copied and converted
 ///    values from the source operand.
-static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtpu8_ps(__m64 __a)
 {
   return __builtin_convertvector(
@@ -2954,7 +2954,7 @@ _mm_cvtpu8_ps(__m64 __a)
 /// \returns A 128-bit vector of [4 x float] whose lower 64 bits contain the
 ///    copied and converted values from the first operand. The upper 64 bits
 ///    contain the copied and converted values from the second operand.
-static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2
+static __inline__ __m128 __DEFAULT_FN_ATTRS_SSE2_CONSTEXPR
 _mm_cvtpi32x2_ps(__m64 __a, __m64 __b)
 {
   return __builtin_convertvector(
diff --git a/clang/lib/Headers/xopintrin.h b/clang/lib/Headers/xopintrin.h
index 976cdf4902a40..7015719659139 100644
--- a/clang/lib/Headers/xopintrin.h
+++ b/clang/lib/Headers/xopintrin.h
@@ -20,6 +20,14 @@
 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("xop"), __min_vector_width__(128)))
 #define __DEFAULT_FN_ATTRS256 __attribute__((__always_inline__, __nodebug__, __target__("xop"), __min_vector_width__(256)))
 
+#if defined(__cplusplus) && (__cplusplus >= 201103L)
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS constexpr
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256 constexpr
+#else
+#define __DEFAULT_FN_ATTRS_CONSTEXPR __DEFAULT_FN_ATTRS
+#define __DEFAULT_FN_ATTRS256_CONSTEXPR __DEFAULT_FN_ATTRS256
+#endif
+
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_maccs_epi16(__m128i __A, __m128i __B, __m128i __C)
 {
@@ -182,13 +190,13 @@ _mm_hsubq_epi32(__m128i __A)
   return (__m128i)__builtin_ia32_vphsubdq((__v4si)__A);
 }
 
-static __inline__ __m128i __DEFAULT_FN_ATTRS
+static __inline__ __m128i __DEFAULT_FN_ATTRS_CONSTEXPR
 _mm_cmov_si128(__m128i __A, __m128i __B, __m128i __C)
 {
   return (__m128i)(((__v2du)__A & (__v2du)__C) | ((__v2du)__B & ~(__v2du)__C));
 }
 
-static __inline__ __m256i __DEFAULT_FN_ATTRS256
+static __inline__ __m256i __DEFAULT_FN_ATTRS256_CONSTEXPR
 _mm256_cmov_si256(__m256i __A, __m256i __B, __m256i __C)
 {
   return (__m256i)(((__v4du)__A & (__v4du)__C) | ((__v4du)__B & ~(__v4du)__C));
@@ -203,25 +211,25 @@ _mm_perm_epi8(__m128i __A, __m128i __B, __m128i __C)
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_rot_epi8(__m128i __A, __m128i __B)
 {
-  return (__m128i)__builtin_ia32_vprotb((__v16qi)__A, (__v16qi)__B);
+  return (__m128i)__builtin_elementwise_fshl((__v16qu)__A, (__v16qu)__A, (__v16qu)__B);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_rot_epi16(__m128i __A, __m128i __B)
 {
-  return (__m128i)__builtin_ia32_vprotw((__v8hi)__A, (__v8hi)__B);
+  return (__m128i)__builtin_elementwise_fshl((__v8hu)__A, (__v8hu)__A, (__v8hu)__B);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_rot_epi32(__m128i __A, __m128i __B)
 {
-  return (__m128i)__builtin_ia32_vprotd((__v4si)__A, (__v4si)__B);
+  return (__m128i)__builtin_elementwise_fshl((__v4su)__A, (__v4su)__A, (__v4su)__B);
 }
 
 static __inline__ __m128i __DEFAULT_FN_ATTRS
 _mm_rot_epi64(__m128i __A, __m128i __B)
 {
-  return (__m128i)__builtin_ia32_vprotq((__v2di)__A, (__v2di)__B);
+  return (__m128i)__builtin_elementwise_fshl((__v2du)__A, (__v2du)__A, (__v2du)__B);
 }
 
 #define _mm_roti_epi8(A, N) \
@@ -766,5 +774,7 @@ _mm256_frcz_pd(__m256d __A)
 
 #undef __DEFAULT_FN_ATTRS
 #undef __DEFAULT_FN_ATTRS256
+#undef __DEFAULT_FN_ATTRS_CONSTEXPR
+#undef __DEFAULT_FN_ATTRS256_CONSTEXPR
 
 #endif /* __XOPINTRIN_H */
diff --git a/clang/lib/Index/IndexTypeSourceInfo.cpp b/clang/lib/Index/IndexTypeSourceInfo.cpp
index adc33b3abd822..9a699c3c896de 100644
--- a/clang/lib/Index/IndexTypeSourceInfo.cpp
+++ b/clang/lib/Index/IndexTypeSourceInfo.cpp
@@ -59,7 +59,7 @@ class TypeIndexer : public RecursiveASTVisitor<TypeIndexer> {
 
   bool VisitTypedefTypeLoc(TypedefTypeLoc TL) {
     SourceLocation Loc = TL.getNameLoc();
-    TypedefNameDecl *ND = TL.getTypedefNameDecl();
+    TypedefNameDecl *ND = TL.getDecl();
     if (ND->isTransparentTag()) {
       TagDecl *Underlying = ND->getUnderlyingType()->getAsTagDecl();
       return IndexCtx.handleReference(Underlying, Loc, Parent,
@@ -117,7 +117,7 @@ class TypeIndexer : public RecursiveASTVisitor<TypeIndexer> {
   }
 
   bool VisitTagTypeLoc(TagTypeLoc TL) {
-    TagDecl *D = TL.getDecl();
+    TagDecl *D = TL.getOriginalDecl();
     if (!IndexCtx.shouldIndexFunctionLocalSymbols() &&
         D->getParentFunctionOrMethod())
       return true;
@@ -172,7 +172,8 @@ class TypeIndexer : public RecursiveASTVisitor<TypeIndexer> {
     return true;
   }
 
-  bool TraverseTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc TL) {
+  bool TraverseTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc TL,
+                                             bool TraverseQualifier) {
     if (!WalkUpFromTemplateSpecializationTypeLoc(TL))
       return false;
     if (!TraverseTemplateName(TL.getTypePtr()->getTemplateName()))
@@ -202,11 +203,6 @@ class TypeIndexer : public RecursiveASTVisitor<TypeIndexer> {
     return true;
   }
 
-  bool VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc TL) {
-    return IndexCtx.handleReference(TL.getDecl(), TL.getNameLoc(), Parent,
-                                    ParentDC, SymbolRoleSet(), Relations);
-  }
-
   bool VisitDependentNameTypeLoc(DependentNameTypeLoc TL) {
     std::vector<const NamedDecl *> Symbols =
         IndexCtx.getResolver()->resolveDependentNameType(TL.getTypePtr());
@@ -248,32 +244,28 @@ void IndexingContext::indexTypeLoc(TypeLoc TL,
   TypeIndexer(*this, Parent, DC, isBase, isIBType).TraverseTypeLoc(TL);
 }
 
-void IndexingContext::indexNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS,
-                                                  const NamedDecl *Parent,
-                                                  const DeclContext *DC) {
-  if (!NNS)
-    return;
-
-  if (NestedNameSpecifierLoc Prefix = NNS.getPrefix())
-    indexNestedNameSpecifierLoc(Prefix, Parent, DC);
-
+void IndexingContext::indexNestedNameSpecifierLoc(
+    NestedNameSpecifierLoc QualifierLoc, const NamedDecl *Parent,
+    const DeclContext *DC) {
   if (!DC)
     DC = Parent->getLexicalDeclContext();
-  SourceLocation Loc = NNS.getLocalBeginLoc();
-
-  switch (NNS.getNestedNameSpecifier()->getKind()) {
-  case NestedNameSpecifier::Identifier:
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Super:
+  switch (NestedNameSpecifier Qualifier = QualifierLoc.getNestedNameSpecifier();
+          Qualifier.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     break;
 
-  case NestedNameSpecifier::Namespace:
-    handleReference(NNS.getNestedNameSpecifier()->getAsNamespace(),
-                    Loc, Parent, DC, SymbolRoleSet());
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [Namespace, Prefix] = QualifierLoc.castAsNamespaceAndPrefix();
+    indexNestedNameSpecifierLoc(Prefix, Parent, DC);
+    handleReference(Namespace, QualifierLoc.getLocalBeginLoc(), Parent, DC,
+                    SymbolRoleSet());
     break;
+  }
 
-  case NestedNameSpecifier::TypeSpec:
-    indexTypeLoc(NNS.getTypeLoc(), Parent, DC);
+  case NestedNameSpecifier::Kind::Type:
+    indexTypeLoc(QualifierLoc.castAsTypeLoc(), Parent, DC);
     break;
   }
 }
diff --git a/clang/lib/Index/USRGeneration.cpp b/clang/lib/Index/USRGeneration.cpp
index 6a884f7be00e1..6bdddcc6967c9 100644
--- a/clang/lib/Index/USRGeneration.cpp
+++ b/clang/lib/Index/USRGeneration.cpp
@@ -653,14 +653,14 @@ bool USRGenerator::GenLoc(const Decl *D, bool IncludeOffset) {
 }
 
 static void printQualifier(llvm::raw_ostream &Out, const LangOptions &LangOpts,
-                           NestedNameSpecifier *NNS) {
+                           NestedNameSpecifier NNS) {
   // FIXME: Encode the qualifier, don't just print it.
   PrintingPolicy PO(LangOpts);
   PO.SuppressTagKeyword = true;
   PO.SuppressUnwrittenScope = true;
   PO.ConstantArraySizeAsWritten = false;
   PO.AnonymousTagLocations = false;
-  NNS->print(Out, PO);
+  NNS.print(Out, PO);
 }
 
 void USRGenerator::VisitType(QualType T) {
@@ -910,9 +910,14 @@ void USRGenerator::VisitType(QualType T) {
       continue;
     }
     if (const TagType *TT = T->getAs<TagType>()) {
-      Out << '$';
-      VisitTagDecl(TT->getDecl());
-      return;
+      if (const auto *ICNT = dyn_cast<InjectedClassNameType>(TT)) {
+        T = ICNT->getOriginalDecl()->getCanonicalTemplateSpecializationType(
+            Ctx);
+      } else {
+        Out << '$';
+        VisitTagDecl(TT->getOriginalDecl());
+        return;
+      }
     }
     if (const ObjCInterfaceType *OIT = T->getAs<ObjCInterfaceType>()) {
       Out << '$';
@@ -945,10 +950,6 @@ void USRGenerator::VisitType(QualType T) {
       Out << ':' << DNT->getIdentifier()->getName();
       return;
     }
-    if (const InjectedClassNameType *InjT = T->getAs<InjectedClassNameType>()) {
-      T = InjT->getInjectedSpecializationType();
-      continue;
-    }
     if (const auto *VT = T->getAs<VectorType>()) {
       Out << (T->isExtVectorType() ? ']' : '[');
       Out << VT->getNumElements();
diff --git a/clang/lib/InstallAPI/Visitor.cpp b/clang/lib/InstallAPI/Visitor.cpp
index 487be2c300887..f12e04069817b 100644
--- a/clang/lib/InstallAPI/Visitor.cpp
+++ b/clang/lib/InstallAPI/Visitor.cpp
@@ -424,7 +424,7 @@ std::string
 InstallAPIVisitor::getMangledCXXRTTIName(const CXXRecordDecl *D) const {
   SmallString<256> Name;
   raw_svector_ostream NameStream(Name);
-  MC->mangleCXXRTTIName(QualType(D->getTypeForDecl(), 0), NameStream);
+  MC->mangleCXXRTTIName(MC->getASTContext().getCanonicalTagType(D), NameStream);
 
   return getBackendMangledName(Name);
 }
@@ -432,7 +432,7 @@ InstallAPIVisitor::getMangledCXXRTTIName(const CXXRecordDecl *D) const {
 std::string InstallAPIVisitor::getMangledCXXRTTI(const CXXRecordDecl *D) const {
   SmallString<256> Name;
   raw_svector_ostream NameStream(Name);
-  MC->mangleCXXRTTI(QualType(D->getTypeForDecl(), 0), NameStream);
+  MC->mangleCXXRTTI(MC->getASTContext().getCanonicalTagType(D), NameStream);
 
   return getBackendMangledName(Name);
 }
@@ -543,8 +543,8 @@ void InstallAPIVisitor::emitVTableSymbols(const CXXRecordDecl *D,
   }
 
   for (const auto &It : D->bases()) {
-    const CXXRecordDecl *Base =
-        cast<CXXRecordDecl>(It.getType()->castAs<RecordType>()->getDecl());
+    const CXXRecordDecl *Base = cast<CXXRecordDecl>(
+        It.getType()->castAs<RecordType>()->getOriginalDecl());
     const auto BaseAccess = getAccessForDecl(Base);
     if (!BaseAccess)
       continue;
diff --git a/clang/lib/Interpreter/InterpreterValuePrinter.cpp b/clang/lib/Interpreter/InterpreterValuePrinter.cpp
index 0ea6274b79cba..4f5f43e1b3ade 100644
--- a/clang/lib/Interpreter/InterpreterValuePrinter.cpp
+++ b/clang/lib/Interpreter/InterpreterValuePrinter.cpp
@@ -66,10 +66,10 @@ static std::string QualTypeToString(ASTContext &Ctx, QualType QT) {
   const QualType NonRefTy = QT.getNonReferenceType();
 
   if (const auto *TTy = llvm::dyn_cast<TagType>(NonRefTy))
-    return DeclTypeToString(NonRefTy, TTy->getDecl());
+    return DeclTypeToString(NonRefTy, TTy->getOriginalDecl());
 
   if (const auto *TRy = dyn_cast<RecordType>(NonRefTy))
-    return DeclTypeToString(NonRefTy, TRy->getDecl());
+    return DeclTypeToString(NonRefTy, TRy->getOriginalDecl());
 
   const QualType Canon = NonRefTy.getCanonicalType();
 
@@ -105,7 +105,7 @@ static std::string EnumToString(const Value &V) {
   const EnumType *EnumTy = DesugaredTy.getNonReferenceType()->getAs<EnumType>();
   assert(EnumTy && "Fail to cast to enum type");
 
-  EnumDecl *ED = EnumTy->getDecl();
+  EnumDecl *ED = EnumTy->getOriginalDecl()->getDefinitionOrSelf();
   uint64_t Data = V.convertTo<uint64_t>();
   bool IsFirst = true;
   llvm::APSInt AP = Ctx.MakeIntValue(Data, DesugaredTy);
@@ -666,7 +666,7 @@ __clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType,
     VRef.setPtr(va_arg(args, void *));
   } else {
     if (const auto *ET = QT->getAs<EnumType>())
-      QT = ET->getDecl()->getIntegerType();
+      QT = ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
     switch (QT->castAs<BuiltinType>()->getKind()) {
     default:
       llvm_unreachable("unknown type kind!");
diff --git a/clang/lib/Interpreter/Value.cpp b/clang/lib/Interpreter/Value.cpp
index be2ab5587a980..84ba508e9cbc8 100644
--- a/clang/lib/Interpreter/Value.cpp
+++ b/clang/lib/Interpreter/Value.cpp
@@ -102,7 +102,7 @@ static Value::Kind ConvertQualTypeToKind(const ASTContext &Ctx, QualType QT) {
     return Value::K_Void;
 
   if (const auto *ET = QT->getAs<EnumType>())
-    QT = ET->getDecl()->getIntegerType();
+    QT = ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   const auto *BT = QT->getAs<BuiltinType>();
   if (!BT || BT->isNullPtrType())
@@ -149,9 +149,9 @@ Value::Value(const Interpreter *In, void *Ty) : Interp(In), OpaqueType(Ty) {
       }
       if (const auto *RT = DtorTy->getAs<RecordType>()) {
         if (CXXRecordDecl *CXXRD =
-                llvm::dyn_cast<CXXRecordDecl>(RT->getDecl())) {
+                llvm::dyn_cast<CXXRecordDecl>(RT->getOriginalDecl())) {
           if (llvm::Expected<llvm::orc::ExecutorAddr> Addr =
-                  Interp.CompileDtorCall(CXXRD))
+                  Interp.CompileDtorCall(CXXRD->getDefinitionOrSelf()))
             DtorF = reinterpret_cast<void *>(Addr->getValue());
           else
             llvm::logAllUnhandledErrors(Addr.takeError(), llvm::errs());
diff --git a/clang/lib/Lex/DependencyDirectivesScanner.cpp b/clang/lib/Lex/DependencyDirectivesScanner.cpp
index 9ccff5e3342d5..eee57c786442a 100644
--- a/clang/lib/Lex/DependencyDirectivesScanner.cpp
+++ b/clang/lib/Lex/DependencyDirectivesScanner.cpp
@@ -83,6 +83,8 @@ struct Scanner {
   /// \returns True on error.
   bool scan(SmallVectorImpl<Directive> &Directives);
 
+  friend bool clang::scanInputForCXX20ModulesUsage(StringRef Source);
+
 private:
   /// Lexes next token and advances \p First and the \p Lexer.
   [[nodiscard]] dependency_directives_scan::Token &
@@ -1075,3 +1077,51 @@ void clang::printDependencyDirectivesAsSource(
     }
   }
 }
+
+static void skipUntilMaybeCXX20ModuleDirective(const char *&First,
+                                               const char *const End) {
+  assert(First <= End);
+  while (First != End) {
+    if (*First == '#') {
+      ++First;
+      skipToNewlineRaw(First, End);
+    }
+    skipWhitespace(First, End);
+    if (const auto Len = isEOL(First, End)) {
+      First += Len;
+      continue;
+    }
+    break;
+  }
+}
+
+bool clang::scanInputForCXX20ModulesUsage(StringRef Source) {
+  const char *First = Source.begin();
+  const char *const End = Source.end();
+  skipUntilMaybeCXX20ModuleDirective(First, End);
+  if (First == End)
+    return false;
+
+  // Check if the next token can even be a module directive before creating a
+  // full lexer.
+  if (!(*First == 'i' || *First == 'e' || *First == 'm'))
+    return false;
+
+  llvm::SmallVector<dependency_directives_scan::Token> Tokens;
+  Scanner S(StringRef(First, End - First), Tokens, nullptr, SourceLocation());
+  S.TheLexer.setParsingPreprocessorDirective(true);
+  if (S.lexModule(First, End))
+    return false;
+  auto IsCXXNamedModuleDirective = [](const DirectiveWithTokens &D) {
+    switch (D.Kind) {
+    case dependency_directives_scan::cxx_module_decl:
+    case dependency_directives_scan::cxx_import_decl:
+    case dependency_directives_scan::cxx_export_module_decl:
+    case dependency_directives_scan::cxx_export_import_decl:
+      return true;
+    default:
+      return false;
+    }
+  };
+  return llvm::any_of(S.DirsWithToks, IsCXXNamedModuleDirective);
+}
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index 1f695b4a8676c..b282a600c0e56 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -174,8 +174,6 @@ void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
   ExtendedTokenMode = 0;
 
   NewLinePtr = nullptr;
-
-  IsFirstPPToken = true;
 }
 
 /// Lexer constructor - Create a new lexer object for the specified buffer
@@ -3225,7 +3223,6 @@ std::optional<Token> Lexer::peekNextPPToken() {
   bool atStartOfLine = IsAtStartOfLine;
   bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
   bool leadingSpace = HasLeadingSpace;
-  bool isFirstPPToken = IsFirstPPToken;
 
   Token Tok;
   Lex(Tok);
@@ -3236,7 +3233,6 @@ std::optional<Token> Lexer::peekNextPPToken() {
   HasLeadingSpace = leadingSpace;
   IsAtStartOfLine = atStartOfLine;
   IsAtPhysicalStartOfLine = atPhysicalStartOfLine;
-  IsFirstPPToken = isFirstPPToken;
   // Restore the lexer back to non-skipping mode.
   LexingRawMode = false;
 
@@ -3726,11 +3722,6 @@ bool Lexer::Lex(Token &Result) {
     HasLeadingEmptyMacro = false;
   }
 
-  if (IsFirstPPToken) {
-    Result.setFlag(Token::FirstPPToken);
-    IsFirstPPToken = false;
-  }
-
   bool atPhysicalStartOfLine = IsAtPhysicalStartOfLine;
   IsAtPhysicalStartOfLine = false;
   bool isRawLex = isLexingRawMode();
diff --git a/clang/lib/Lex/ModuleMapFile.cpp b/clang/lib/Lex/ModuleMapFile.cpp
index 183e919d14c22..f0cd9d2bee82a 100644
--- a/clang/lib/Lex/ModuleMapFile.cpp
+++ b/clang/lib/Lex/ModuleMapFile.cpp
@@ -118,7 +118,8 @@ struct ModuleMapFileParser {
   std::optional<ExcludeDecl> parseExcludeDecl(clang::SourceLocation LeadingLoc);
   std::optional<UmbrellaDirDecl>
   parseUmbrellaDirDecl(SourceLocation UmbrellaLoc);
-  std::optional<LinkDecl> parseLinkDecl();
+  std::optional<LinkDecl>
+  parseLinkDecl(llvm::StringMap<SourceLocation> &SeenLinkDecl, bool Allowed);
 
   SourceLocation consumeToken();
   void skipUntil(MMToken::TokenKind K);
@@ -325,6 +326,7 @@ std::optional<ModuleDecl> ModuleMapFileParser::parseModuleDecl(bool TopLevel) {
   SourceLocation LBraceLoc = consumeToken();
 
   bool Done = false;
+  llvm::StringMap<SourceLocation> SeenLinkDecl;
   do {
     std::optional<Decl> SubDecl;
     switch (Tok.Kind) {
@@ -405,7 +407,9 @@ std::optional<ModuleDecl> ModuleMapFileParser::parseModuleDecl(bool TopLevel) {
       break;
 
     case MMToken::LinkKeyword:
-      SubDecl = parseLinkDecl();
+      // Link decls are only allowed in top level modules or explicit
+      // submodules.
+      SubDecl = parseLinkDecl(SeenLinkDecl, TopLevel || MDecl.Explicit);
       break;
 
     default:
@@ -822,7 +826,8 @@ ModuleMapFileParser::parseUmbrellaDirDecl(clang::SourceLocation UmbrellaLoc) {
 ///
 ///   module-declaration:
 ///     'link' 'framework'[opt] string-literal
-std::optional<LinkDecl> ModuleMapFileParser::parseLinkDecl() {
+std::optional<LinkDecl> ModuleMapFileParser::parseLinkDecl(
+    llvm::StringMap<SourceLocation> &SeenLinkDecl, bool Allowed) {
   assert(Tok.is(MMToken::LinkKeyword));
   LinkDecl LD;
   LD.Location = consumeToken();
@@ -838,12 +843,33 @@ std::optional<LinkDecl> ModuleMapFileParser::parseLinkDecl() {
   if (!Tok.is(MMToken::StringLiteral)) {
     Diags.Report(Tok.getLocation(), diag::err_mmap_expected_library_name)
         << LD.Framework << SourceRange(LD.Location);
+    consumeToken();
     HadError = true;
     return std::nullopt;
   }
 
-  LD.Library = Tok.getString();
+  StringRef Library = Tok.getString();
+
+  LD.Library = Library;
   consumeToken();
+
+  // Make sure we eat all the tokens when we report the errors so parsing
+  // can continue.
+  if (!Allowed) {
+    Diags.Report(LD.Location, diag::err_mmap_submodule_link_decl);
+    HadError = true;
+    return std::nullopt;
+  }
+
+  auto [It, Inserted] =
+      SeenLinkDecl.insert(std::make_pair(Library, LD.Location));
+  if (!Inserted) {
+    Diags.Report(LD.Location, diag::warn_mmap_link_redeclaration) << Library;
+    Diags.Report(It->second, diag::note_mmap_prev_link_declaration);
+    HadError = true;
+    return std::nullopt;
+  }
+
   return std::move(LD);
 }
 
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 3fa060f7ec1bd..9d01b8d99e227 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -3793,9 +3793,13 @@ Preprocessor::LexEmbedParameters(Token &CurTok, bool ForHasEmbed) {
             [[fallthrough]];
           case tok::r_brace:
           case tok::r_square: {
+            if (BracketStack.empty()) {
+              ExpectOrDiagAndSkipToEOD(tok::r_paren);
+              return false;
+            }
             tok::TokenKind Matching =
                 GetMatchingCloseBracket(BracketStack.back().first);
-            if (BracketStack.empty() || CurTok.getKind() != Matching) {
+            if (CurTok.getKind() != Matching) {
               DiagMismatchedBracesAndSkipToEOD(Matching, BracketStack.back());
               return false;
             }
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index e278846f6f36d..e003ad3a95570 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -43,6 +43,7 @@
 #include "clang/Lex/MacroArgs.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/ModuleLoader.h"
+#include "clang/Lex/NoTrivialPPDirectiveTracer.h"
 #include "clang/Lex/Pragma.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/PreprocessorLexer.h"
@@ -247,8 +248,6 @@ void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
     llvm::errs() << " [LeadingSpace]";
   if (Tok.isExpandDisabled())
     llvm::errs() << " [ExpandDisabled]";
-  if (Tok.isFirstPPToken())
-    llvm::errs() << " [First pp-token]";
   if (Tok.needsCleaning()) {
     const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
     llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
@@ -577,8 +576,11 @@ void Preprocessor::EnterMainSourceFile() {
     // export module M; // error: module declaration must occur
     //                  //        at the start of the translation unit.
     if (getLangOpts().CPlusPlusModules) {
+      auto Tracer = std::make_unique<NoTrivialPPDirectiveTracer>(*this);
+      DirTracer = Tracer.get();
+      addPPCallbacks(std::move(Tracer));
       std::optional<Token> FirstPPTok = CurLexer->peekNextPPToken();
-      if (FirstPPTok && FirstPPTok->isFirstPPToken())
+      if (FirstPPTok)
         FirstPPTokenLoc = FirstPPTok->getLocation();
     }
   }
@@ -940,6 +942,8 @@ void Preprocessor::Lex(Token &Result) {
       StdCXXImportSeqState.handleHeaderName();
       break;
     case tok::kw_export:
+      if (hasSeenNoTrivialPPDirective())
+        Result.setFlag(Token::HasSeenNoTrivialPPDirective);
       TrackGMFState.handleExport();
       StdCXXImportSeqState.handleExport();
       ModuleDeclState.handleExport();
@@ -968,6 +972,8 @@ void Preprocessor::Lex(Token &Result) {
           }
           break;
         } else if (Result.getIdentifierInfo() == getIdentifierInfo("module")) {
+          if (hasSeenNoTrivialPPDirective())
+            Result.setFlag(Token::HasSeenNoTrivialPPDirective);
           TrackGMFState.handleModule(StdCXXImportSeqState.afterTopLevelSeq());
           ModuleDeclState.handleModule();
           break;
@@ -1682,3 +1688,31 @@ const char *Preprocessor::getCheckPoint(FileID FID, const char *Start) const {
 
   return nullptr;
 }
+
+bool Preprocessor::hasSeenNoTrivialPPDirective() const {
+  return DirTracer && DirTracer->hasSeenNoTrivialPPDirective();
+}
+
+bool NoTrivialPPDirectiveTracer::hasSeenNoTrivialPPDirective() const {
+  return SeenNoTrivialPPDirective;
+}
+
+void NoTrivialPPDirectiveTracer::setSeenNoTrivialPPDirective() {
+  if (InMainFile && !SeenNoTrivialPPDirective)
+    SeenNoTrivialPPDirective = true;
+}
+
+void NoTrivialPPDirectiveTracer::LexedFileChanged(
+    FileID FID, LexedFileChangeReason Reason,
+    SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) {
+  InMainFile = (FID == PP.getSourceManager().getMainFileID());
+}
+
+void NoTrivialPPDirectiveTracer::MacroExpands(const Token &MacroNameTok,
+                                              const MacroDefinition &MD,
+                                              SourceRange Range,
+                                              const MacroArgs *Args) {
+  // FIXME: Does only enable builtin macro expansion make sense?
+  if (!MD.getMacroInfo()->isBuiltinMacro())
+    setSeenNoTrivialPPDirective();
+}
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index 9a010fb5f3427..74e25002e468b 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -1161,6 +1161,12 @@ bool Parser::ConsumeAndStoreInitializer(CachedTokens &Toks,
 
   while (true) {
     switch (Tok.getKind()) {
+    case tok::ellipsis:
+      // We found an elipsis at the end of the parameter list;
+      // it is not part of a parameter declaration.
+      if (ParenCount == 1 && NextToken().is(tok::r_paren))
+        return true;
+      goto consume_token;
     case tok::comma:
       // If we might be in a template, perform a tentative parse to check.
       if (!AngleCount)
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index fd53cca5a13ff..10355bb874762 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -6224,7 +6224,6 @@ void Parser::ParseTypeQualifierListOpt(
     case tok::kw___funcref:
       ParseWebAssemblyFuncrefTypeAttribute(DS.getAttributes());
       continue;
-      goto DoneWithTypeQuals;
 
     case tok::kw___pascal:
       if (AttrReqs & AR_VendorAttributesParsed) {
@@ -7395,7 +7394,7 @@ void Parser::ParseFunctionDeclaratorIdentifierList(
     Diag(Tok, diag::ext_ident_list_in_param);
 
   // Maintain an efficient lookup of params we have seen so far.
-  llvm::SmallSet<const IdentifierInfo*, 16> ParamsSoFar;
+  llvm::SmallPtrSet<const IdentifierInfo *, 16> ParamsSoFar;
 
   do {
     // If this isn't an identifier, report the error and skip until ')'.
@@ -7879,9 +7878,9 @@ void Parser::ParseMisplacedBracketDeclarator(Declarator &D) {
     D.AddTypeInfo(Chunk, TempDeclarator.getAttributePool(), SourceLocation());
   }
 
-  // The missing identifier would have been diagnosed in ParseDirectDeclarator.
+  // The missing name would have been diagnosed in ParseDirectDeclarator.
   // If parentheses are required, always suggest them.
-  if (!D.getIdentifier() && !NeedParens)
+  if (!D.hasName() && !NeedParens)
     return;
 
   SourceLocation EndBracketLoc = TempDeclarator.getEndLoc();
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index bc8841c68b42c..005ad524605ff 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -591,7 +591,7 @@ bool Parser::ParseUsingDeclarator(DeclaratorContext Context,
        NextToken().isRegularKeywordAttribute() ||
        NextToken().is(tok::kw___attribute)) &&
       D.SS.isNotEmpty() && LastII == Tok.getIdentifierInfo() &&
-      D.SS.getScopeRep()->getKind() != NestedNameSpecifier::Namespace) {
+      D.SS.getScopeRep().getKind() != NestedNameSpecifier::Kind::Namespace) {
     SourceLocation IdLoc = ConsumeToken();
     ParsedType Type =
         Actions.getInheritingConstructorName(D.SS, IdLoc, *LastII);
@@ -4512,6 +4512,27 @@ bool Parser::ParseCXX11AttributeArgs(
       Form = ParsedAttr::Form::Microsoft();
   }
 
+  if (LO.CPlusPlus) {
+    TentativeParsingAction TPA(*this);
+    bool HasInvalidArgument = false;
+    while (Tok.isNot(tok::r_paren) && Tok.isNot(tok::eof)) {
+      if (Tok.isOneOf(tok::hash, tok::hashhash)) {
+        Diag(Tok.getLocation(), diag::ext_invalid_attribute_argument)
+            << PP.getSpelling(Tok);
+        HasInvalidArgument = true;
+      }
+      ConsumeAnyToken();
+    }
+
+    if (HasInvalidArgument) {
+      SkipUntil(tok::r_paren);
+      TPA.Commit();
+      return true;
+    }
+
+    TPA.Revert();
+  }
+
   // If the attribute isn't known, we will not attempt to parse any
   // arguments.
   if (Form.getSyntax() != ParsedAttr::AS_Microsoft &&
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index bc238a9517a37..3515343202de1 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -3342,7 +3342,8 @@ ExprResult Parser::ParseBlockLiteralExpression() {
     Actions.ActOnBlockError(CaretLoc, getCurScope());
     return ExprError();
   }
-
+  EnterExpressionEvaluationContextForFunction PotentiallyEvaluated(
+       Actions, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
   StmtResult Stmt(ParseCompoundStatementBody());
   BlockScope.Exit();
   if (!Stmt.isInvalid())
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 8dce2268082d9..8605ba2cdb49b 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -195,9 +195,10 @@ bool Parser::ParseOptionalCXXScopeSpecifier(
     if (DS.getTypeSpecType() == DeclSpec::TST_error)
       return false;
 
-    QualType Type = Actions.ActOnPackIndexingType(
-        DS.getRepAsType().get(), DS.getPackIndexingExpr(), DS.getBeginLoc(),
-        DS.getEllipsisLoc());
+    QualType Pattern = Sema::GetTypeFromParser(DS.getRepAsType());
+    QualType Type =
+        Actions.ActOnPackIndexingType(Pattern, DS.getPackIndexingExpr(),
+                                      DS.getBeginLoc(), DS.getEllipsisLoc());
 
     if (Type.isNull())
       return false;
@@ -2355,8 +2356,10 @@ bool Parser::ParseUnqualifiedIdTemplateId(
 
   // Constructor and destructor names.
   TypeResult Type = Actions.ActOnTemplateIdType(
-      getCurScope(), SS, TemplateKWLoc, Template, Name, NameLoc, LAngleLoc,
-      TemplateArgsPtr, RAngleLoc, /*IsCtorOrDtorName=*/true);
+      getCurScope(), ElaboratedTypeKeyword::None,
+      /*ElaboratedKeywordLoc=*/SourceLocation(), SS, TemplateKWLoc, Template,
+      Name, NameLoc, LAngleLoc, TemplateArgsPtr, RAngleLoc,
+      /*IsCtorOrDtorName=*/true);
   if (Type.isInvalid())
     return true;
 
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index 98dc458f7adc5..5490c61f52356 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -234,15 +234,15 @@ std::optional<RootDescriptor> RootSignatureParser::parseRootDescriptor() {
   default:
     llvm_unreachable("Switch for consumed token was not provided");
   case TokenKind::kw_CBV:
-    Descriptor.Type = DescriptorType::CBuffer;
+    Descriptor.Type = ResourceClass::CBuffer;
     ExpectedReg = TokenKind::bReg;
     break;
   case TokenKind::kw_SRV:
-    Descriptor.Type = DescriptorType::SRV;
+    Descriptor.Type = ResourceClass::SRV;
     ExpectedReg = TokenKind::tReg;
     break;
   case TokenKind::kw_UAV:
-    Descriptor.Type = DescriptorType::UAV;
+    Descriptor.Type = ResourceClass::UAV;
     ExpectedReg = TokenKind::uReg;
     break;
   }
@@ -360,19 +360,19 @@ RootSignatureParser::parseDescriptorTableClause() {
   default:
     llvm_unreachable("Switch for consumed token was not provided");
   case TokenKind::kw_CBV:
-    Clause.Type = ClauseType::CBuffer;
+    Clause.Type = ResourceClass::CBuffer;
     ExpectedReg = TokenKind::bReg;
     break;
   case TokenKind::kw_SRV:
-    Clause.Type = ClauseType::SRV;
+    Clause.Type = ResourceClass::SRV;
     ExpectedReg = TokenKind::tReg;
     break;
   case TokenKind::kw_UAV:
-    Clause.Type = ClauseType::UAV;
+    Clause.Type = ResourceClass::UAV;
     ExpectedReg = TokenKind::uReg;
     break;
   case TokenKind::kw_Sampler:
-    Clause.Type = ClauseType::Sampler;
+    Clause.Type = ResourceClass::Sampler;
     ExpectedReg = TokenKind::sReg;
     break;
   }
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index 0277dfb5aac13..74aff0b804057 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -1128,12 +1128,14 @@ bool Parser::AnnotateTemplateIdToken(TemplateTy Template, TemplateNameKind TNK,
 
   // Build the annotation token.
   if (TNK == TNK_Type_template && AllowTypeAnnotation) {
-    TypeResult Type = ArgsInvalid
-                          ? TypeError()
-                          : Actions.ActOnTemplateIdType(
-                                getCurScope(), SS, TemplateKWLoc, Template,
-                                TemplateName.Identifier, TemplateNameLoc,
-                                LAngleLoc, TemplateArgsPtr, RAngleLoc);
+    TypeResult Type =
+        ArgsInvalid
+            ? TypeError()
+            : Actions.ActOnTemplateIdType(
+                  getCurScope(), ElaboratedTypeKeyword::None,
+                  /*ElaboratedKeywordLoc=*/SourceLocation(), SS, TemplateKWLoc,
+                  Template, TemplateName.Identifier, TemplateNameLoc, LAngleLoc,
+                  TemplateArgsPtr, RAngleLoc);
 
     Tok.setKind(tok::annot_typename);
     setTypeAnnotation(Tok, Type);
@@ -1194,10 +1196,11 @@ void Parser::AnnotateTemplateIdTokenAsType(
       TemplateId->isInvalid()
           ? TypeError()
           : Actions.ActOnTemplateIdType(
-                getCurScope(), SS, TemplateId->TemplateKWLoc,
-                TemplateId->Template, TemplateId->Name,
-                TemplateId->TemplateNameLoc, TemplateId->LAngleLoc,
-                TemplateArgsPtr, TemplateId->RAngleLoc,
+                getCurScope(), ElaboratedTypeKeyword::None,
+                /*ElaboratedKeywordLoc=*/SourceLocation(), SS,
+                TemplateId->TemplateKWLoc, TemplateId->Template,
+                TemplateId->Name, TemplateId->TemplateNameLoc,
+                TemplateId->LAngleLoc, TemplateArgsPtr, TemplateId->RAngleLoc,
                 /*IsCtorOrDtorName=*/false, IsClassName, AllowImplicitTypename);
   // Create the new "type" annotation token.
   Tok.setKind(tok::annot_typename);
@@ -1263,7 +1266,8 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
           Actions.ActOnTemplateName(getCurScope(), SS, TemplateKWLoc, Name,
                                     /*ObjectType=*/nullptr,
                                     /*EnteringContext=*/false, Template))
-        Result = ParsedTemplateArgument(SS, Template, Name.StartLocation);
+        Result = ParsedTemplateArgument(TemplateKWLoc, SS, Template,
+                                        Name.StartLocation);
     }
   } else if (Tok.is(tok::identifier) || Tok.is(tok::annot_template_id) ||
              Tok.is(tok::annot_non_type)) {
@@ -1300,11 +1304,14 @@ ParsedTemplateArgument Parser::ParseTemplateTemplateArgument() {
           TNK == TNK_Var_template || TNK == TNK_Concept_template) {
         // We have an id-expression that refers to a class template or
         // (C++0x) alias template.
-        Result = ParsedTemplateArgument(SS, Template, Name.StartLocation);
+        Result = ParsedTemplateArgument(/*TemplateKwLoc=*/SourceLocation(), SS,
+                                        Template, Name.StartLocation);
       }
     }
   }
 
+  Result = Actions.ActOnTemplateTemplateArgument(Result);
+
   // If this is a pack expansion, build it as such.
   if (EllipsisLoc.isValid() && !Result.isInvalid())
     Result = Actions.ActOnPackExpansion(Result, EllipsisLoc);
diff --git a/clang/lib/Parse/ParseTentative.cpp b/clang/lib/Parse/ParseTentative.cpp
index 2a731a18ea82a..82f2294ff5bb7 100644
--- a/clang/lib/Parse/ParseTentative.cpp
+++ b/clang/lib/Parse/ParseTentative.cpp
@@ -1328,7 +1328,7 @@ Parser::isCXXDeclarationSpecifier(ImplicitTypenameContext AllowImplicitTypename,
         Actions.RestoreNestedNameSpecifierAnnotation(Tok.getAnnotationValue(),
                                                      Tok.getAnnotationRange(),
                                                      SS);
-        if (SS.getScopeRep() && SS.getScopeRep()->isDependent()) {
+        if (SS.getScopeRep().isDependent()) {
           RevertingTentativeParsingAction PA(*this);
           ConsumeAnnotationToken();
           ConsumeToken();
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index a8cfe202c5214..a17398b84c6a6 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1418,6 +1418,10 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
     // parameter list was specified.
     CurTemplateDepthTracker.addDepth(1);
 
+  // Late attributes are parsed in the same scope as the function body.
+  if (LateParsedAttrs)
+    ParseLexedAttributeList(*LateParsedAttrs, Res, false, true);
+
   if (SkipFunctionBodies && (!Res || Actions.canSkipFunctionBody(Res)) &&
       trySkippingFunctionBody()) {
     BodyScope.Exit();
@@ -1442,10 +1446,6 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
   } else
     Actions.ActOnDefaultCtorInitializers(Res);
 
-  // Late attributes are parsed in the same scope as the function body.
-  if (LateParsedAttrs)
-    ParseLexedAttributeList(*LateParsedAttrs, Res, false, true);
-
   return ParseFunctionStatementBody(Res, BodyScope);
 }
 
@@ -1774,9 +1774,9 @@ Parser::TryAnnotateName(CorrectionCandidateCallback *CCC,
     /// An Objective-C object type followed by '<' is a specialization of
     /// a parameterized class type or a protocol-qualified type.
     ParsedType Ty = Classification.getType();
+    QualType T = Actions.GetTypeFromParser(Ty);
     if (getLangOpts().ObjC && NextToken().is(tok::less) &&
-        (Ty.get()->isObjCObjectType() ||
-         Ty.get()->isObjCObjectPointerType())) {
+        (T->isObjCObjectType() || T->isObjCObjectPointerType())) {
       // Consume the name.
       SourceLocation IdentifierLoc = ConsumeToken();
       SourceLocation NewEndLoc;
@@ -2032,11 +2032,12 @@ bool Parser::TryAnnotateTypeOrScopeTokenAfterScopeSpec(
       if (SS.isNotEmpty()) // it was a C++ qualified type name.
         BeginLoc = SS.getBeginLoc();
 
+      QualType T = Actions.GetTypeFromParser(Ty);
+
       /// An Objective-C object type followed by '<' is a specialization of
       /// a parameterized class type or a protocol-qualified type.
       if (getLangOpts().ObjC && NextToken().is(tok::less) &&
-          (Ty.get()->isObjCObjectType() ||
-           Ty.get()->isObjCObjectPointerType())) {
+          (T->isObjCObjectType() || T->isObjCObjectPointerType())) {
         // Consume the name.
         SourceLocation IdentifierLoc = ConsumeToken();
         SourceLocation NewEndLoc;
@@ -2362,9 +2363,10 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
   // Parse a global-module-fragment, if present.
   if (getLangOpts().CPlusPlusModules && Tok.is(tok::semi)) {
     SourceLocation SemiLoc = ConsumeToken();
-    if (!Introducer.isFirstPPToken()) {
+    if (ImportState != Sema::ModuleImportState::FirstDecl ||
+        Introducer.hasSeenNoTrivialPPDirective()) {
       Diag(StartLoc, diag::err_global_module_introducer_not_at_start)
-        << SourceRange(StartLoc, SemiLoc);
+          << SourceRange(StartLoc, SemiLoc);
       return nullptr;
     }
     if (MDK == Sema::ModuleDeclKind::Interface) {
@@ -2419,7 +2421,8 @@ Parser::ParseModuleDecl(Sema::ModuleImportState &ImportState) {
   ExpectAndConsumeSemi(diag::err_module_expected_semi);
 
   return Actions.ActOnModuleDecl(StartLoc, ModuleLoc, MDK, Path, Partition,
-                                 ImportState, Introducer.isFirstPPToken());
+                                 ImportState,
+                                 Introducer.hasSeenNoTrivialPPDirective());
 }
 
 Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
diff --git a/clang/lib/Sema/AnalysisBasedWarnings.cpp b/clang/lib/Sema/AnalysisBasedWarnings.cpp
index fdb630221889b..0b94b1044f072 100644
--- a/clang/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/clang/lib/Sema/AnalysisBasedWarnings.cpp
@@ -227,14 +227,11 @@ static bool hasRecursiveCallInPath(const FunctionDecl *FD, CFGBlock &Block) {
 
     // Skip function calls which are qualified with a templated class.
     if (const DeclRefExpr *DRE =
-            dyn_cast<DeclRefExpr>(CE->getCallee()->IgnoreParenImpCasts())) {
-      if (NestedNameSpecifier *NNS = DRE->getQualifier()) {
-        if (NNS->getKind() == NestedNameSpecifier::TypeSpec &&
-            isa<TemplateSpecializationType>(NNS->getAsType())) {
+            dyn_cast<DeclRefExpr>(CE->getCallee()->IgnoreParenImpCasts()))
+      if (NestedNameSpecifier NNS = DRE->getQualifier();
+          NNS.getKind() == NestedNameSpecifier::Kind::Type)
+        if (isa_and_nonnull<TemplateSpecializationType>(NNS.getAsType()))
           continue;
-        }
-      }
-    }
 
     const CXXMemberCallExpr *MCE = dyn_cast<CXXMemberCallExpr>(CE);
     if (!MCE || isa<CXXThisExpr>(MCE->getImplicitObjectArgument()) ||
@@ -2783,6 +2780,31 @@ class CallableVisitor : public DynamicRecursiveASTVisitor {
   }
 };
 
+namespace clang::lifetimes {
+namespace {
+class LifetimeSafetyReporterImpl : public LifetimeSafetyReporter {
+
+public:
+  LifetimeSafetyReporterImpl(Sema &S) : S(S) {}
+
+  void reportUseAfterFree(const Expr *IssueExpr, const Expr *UseExpr,
+                          SourceLocation FreeLoc, Confidence C) override {
+    S.Diag(IssueExpr->getExprLoc(),
+           C == Confidence::Definite
+               ? diag::warn_lifetime_safety_loan_expires_permissive
+               : diag::warn_lifetime_safety_loan_expires_strict)
+        << IssueExpr->getEndLoc();
+    S.Diag(FreeLoc, diag::note_lifetime_safety_destroyed_here);
+    S.Diag(UseExpr->getExprLoc(), diag::note_lifetime_safety_used_here)
+        << UseExpr->getEndLoc();
+  }
+
+private:
+  Sema &S;
+};
+} // namespace
+} // namespace clang::lifetimes
+
 void clang::sema::AnalysisBasedWarnings::IssueWarnings(
      TranslationUnitDecl *TU) {
   if (!TU)
@@ -3032,8 +3054,10 @@ void clang::sema::AnalysisBasedWarnings::IssueWarnings(
   // TODO: Enable lifetime safety analysis for other languages once it is
   // stable.
   if (EnableLifetimeSafetyAnalysis && S.getLangOpts().CPlusPlus) {
-    if (AC.getCFG())
-      lifetimes::runLifetimeSafetyAnalysis(AC);
+    if (AC.getCFG()) {
+      lifetimes::LifetimeSafetyReporterImpl LifetimeSafetyReporter(S);
+      lifetimes::runLifetimeSafetyAnalysis(AC, &LifetimeSafetyReporter);
+    }
   }
   // Check for violations of "called once" parameter properties.
   if (S.getLangOpts().ObjC && !S.getLangOpts().CPlusPlus &&
diff --git a/clang/lib/Sema/DeclSpec.cpp b/clang/lib/Sema/DeclSpec.cpp
index f0f1d66f66e93..8756ce5f0d850 100644
--- a/clang/lib/Sema/DeclSpec.cpp
+++ b/clang/lib/Sema/DeclSpec.cpp
@@ -48,9 +48,9 @@ void UnqualifiedId::setConstructorTemplateId(TemplateIdAnnotation *TemplateId) {
   EndLocation = TemplateId->RAngleLoc;
 }
 
-void CXXScopeSpec::Extend(ASTContext &Context, TypeLoc TL,
-                          SourceLocation ColonColonLoc) {
-  Builder.Extend(Context, TL, ColonColonLoc);
+void CXXScopeSpec::Make(ASTContext &Context, TypeLoc TL,
+                        SourceLocation ColonColonLoc) {
+  Builder.Make(Context, TL, ColonColonLoc);
   if (Range.getBegin().isInvalid())
     Range.setBegin(TL.getBeginLoc());
   Range.setEnd(ColonColonLoc);
@@ -59,19 +59,6 @@ void CXXScopeSpec::Extend(ASTContext &Context, TypeLoc TL,
          "NestedNameSpecifierLoc range computation incorrect");
 }
 
-void CXXScopeSpec::Extend(ASTContext &Context, IdentifierInfo *Identifier,
-                          SourceLocation IdentifierLoc,
-                          SourceLocation ColonColonLoc) {
-  Builder.Extend(Context, Identifier, IdentifierLoc, ColonColonLoc);
-
-  if (Range.getBegin().isInvalid())
-    Range.setBegin(IdentifierLoc);
-  Range.setEnd(ColonColonLoc);
-
-  assert(Range == Builder.getSourceRange() &&
-         "NestedNameSpecifierLoc range computation incorrect");
-}
-
 void CXXScopeSpec::Extend(ASTContext &Context, NamespaceBaseDecl *Namespace,
                           SourceLocation NamespaceLoc,
                           SourceLocation ColonColonLoc) {
@@ -95,10 +82,10 @@ void CXXScopeSpec::MakeGlobal(ASTContext &Context,
          "NestedNameSpecifierLoc range computation incorrect");
 }
 
-void CXXScopeSpec::MakeSuper(ASTContext &Context, CXXRecordDecl *RD,
-                             SourceLocation SuperLoc,
-                             SourceLocation ColonColonLoc) {
-  Builder.MakeSuper(Context, RD, SuperLoc, ColonColonLoc);
+void CXXScopeSpec::MakeMicrosoftSuper(ASTContext &Context, CXXRecordDecl *RD,
+                                      SourceLocation SuperLoc,
+                                      SourceLocation ColonColonLoc) {
+  Builder.MakeMicrosoftSuper(Context, RD, SuperLoc, ColonColonLoc);
 
   Range.setBegin(SuperLoc);
   Range.setEnd(ColonColonLoc);
@@ -108,7 +95,7 @@ void CXXScopeSpec::MakeSuper(ASTContext &Context, CXXRecordDecl *RD,
 }
 
 void CXXScopeSpec::MakeTrivial(ASTContext &Context,
-                               NestedNameSpecifier *Qualifier, SourceRange R) {
+                               NestedNameSpecifier Qualifier, SourceRange R) {
   Builder.MakeTrivial(Context, Qualifier, R);
   Range = R;
 }
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
index 87f9ae07550c2..806800cb7b213 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.cpp
@@ -313,9 +313,6 @@ TemplateParameterListBuilder::finalizeTemplateArgs(ConceptDecl *CD) {
   Builder.Record->getDeclContext()->addDecl(Builder.Template);
   Params.clear();
 
-  QualType T = Builder.Template->getInjectedClassNameSpecialization();
-  T = AST.getInjectedClassNameType(Builder.Record, T);
-
   return Builder;
 }
 
@@ -351,7 +348,7 @@ BuiltinTypeMethodBuilder::BuiltinTypeMethodBuilder(BuiltinTypeDeclBuilder &DB,
   ASTContext &AST = DB.SemaRef.getASTContext();
   if (IsCtor) {
     Name = AST.DeclarationNames.getCXXConstructorName(
-        DB.Record->getTypeForDecl()->getCanonicalTypeUnqualified());
+        AST.getCanonicalTagType(DB.Record));
   } else {
     const IdentifierInfo &II =
         AST.Idents.get(NameStr, tok::TokenKind::identifier);
@@ -553,9 +550,9 @@ BuiltinTypeDeclBuilder::BuiltinTypeDeclBuilder(Sema &SemaRef,
     return;
   }
 
-  Record = CXXRecordDecl::Create(AST, TagDecl::TagKind::Class, HLSLNamespace,
-                                 SourceLocation(), SourceLocation(), &II,
-                                 PrevDecl, true);
+  Record =
+      CXXRecordDecl::Create(AST, TagDecl::TagKind::Class, HLSLNamespace,
+                            SourceLocation(), SourceLocation(), &II, PrevDecl);
   Record->setImplicit(true);
   Record->setLexicalDeclContext(HLSLNamespace);
   Record->setHasExternalLexicalStorage();
@@ -570,18 +567,6 @@ BuiltinTypeDeclBuilder::~BuiltinTypeDeclBuilder() {
     HLSLNamespace->addDecl(Record);
 }
 
-CXXRecordDecl *BuiltinTypeDeclBuilder::finalizeForwardDeclaration() {
-  // Force the QualType to be generated for the record declaration. In most
-  // cases this will happen naturally when something uses the type the
-  // QualType gets lazily created. Unfortunately, with our injected types if a
-  // type isn't used in a translation unit the QualType may not get
-  // automatically generated before a PCH is generated. To resolve this we
-  // just force that the QualType is generated after we create a forward
-  // declaration.
-  (void)Record->getASTContext().getRecordType(Record);
-  return Record;
-}
-
 BuiltinTypeDeclBuilder &
 BuiltinTypeDeclBuilder::addMemberVariable(StringRef Name, QualType Type,
                                           llvm::ArrayRef<Attr *> Attrs,
diff --git a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
index 36c4add20b225..098b72692bd3a 100644
--- a/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
+++ b/clang/lib/Sema/HLSLBuiltinTypeDeclBuilder.h
@@ -64,7 +64,7 @@ class BuiltinTypeDeclBuilder {
 
   BuiltinTypeDeclBuilder &addSimpleTemplateParams(ArrayRef<StringRef> Names,
                                                   ConceptDecl *CD);
-  CXXRecordDecl *finalizeForwardDeclaration();
+  CXXRecordDecl *finalizeForwardDeclaration() { return Record; }
   BuiltinTypeDeclBuilder &completeDefinition();
 
   BuiltinTypeDeclBuilder &
diff --git a/clang/lib/Sema/HeuristicResolver.cpp b/clang/lib/Sema/HeuristicResolver.cpp
index 6874d30516f8f..933841beeac3d 100644
--- a/clang/lib/Sema/HeuristicResolver.cpp
+++ b/clang/lib/Sema/HeuristicResolver.cpp
@@ -44,7 +44,7 @@ class HeuristicResolverImpl {
   resolveDependentNameType(const DependentNameType *DNT);
   std::vector<const NamedDecl *> resolveTemplateSpecializationType(
       const DependentTemplateSpecializationType *DTST);
-  QualType resolveNestedNameSpecifierToType(const NestedNameSpecifier *NNS);
+  QualType resolveNestedNameSpecifierToType(NestedNameSpecifier NNS);
   QualType getPointeeType(QualType T);
   std::vector<const NamedDecl *>
   lookupDependentName(CXXRecordDecl *RD, DeclarationName Name,
@@ -57,7 +57,7 @@ class HeuristicResolverImpl {
   ASTContext &Ctx;
 
   // Recursion protection sets
-  llvm::SmallSet<const DependentNameType *, 4> SeenDependentNameTypes;
+  llvm::SmallPtrSet<const DependentNameType *, 4> SeenDependentNameTypes;
 
   // Given a tag-decl type and a member name, heuristically resolve the
   // name to one or more declarations.
@@ -101,9 +101,8 @@ QualType resolveDeclsToType(const std::vector<const NamedDecl *> &Decls,
                             ASTContext &Ctx) {
   if (Decls.size() != 1) // Names an overload set -- just bail.
     return QualType();
-  if (const auto *TD = dyn_cast<TypeDecl>(Decls[0])) {
-    return Ctx.getTypeDeclType(TD);
-  }
+  if (const auto *TD = dyn_cast<TypeDecl>(Decls[0]))
+    return Ctx.getCanonicalTypeDeclType(TD);
   if (const auto *VD = dyn_cast<ValueDecl>(Decls[0])) {
     return VD->getType();
   }
@@ -139,8 +138,7 @@ TagDecl *HeuristicResolverImpl::resolveTypeToTagDecl(QualType QT) {
     T = T->getCanonicalTypeInternal().getTypePtr();
   }
 
-  if (auto *TT = T->getAs<TagType>()) {
-    TagDecl *TD = TT->getDecl();
+  if (auto *TD = T->getAsTagDecl()) {
     // Template might not be instantiated yet, fall back to primary template
     // in such cases.
     if (const auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(TD)) {
@@ -151,11 +149,6 @@ TagDecl *HeuristicResolverImpl::resolveTypeToTagDecl(QualType QT) {
     return TD;
   }
 
-  if (const auto *ICNT = T->getAs<InjectedClassNameType>())
-    T = ICNT->getInjectedSpecializationType().getTypePtrOrNull();
-  if (!T)
-    return nullptr;
-
   TemplateName TN = getReferencedTemplateName(T);
   if (TN.isNull())
     return nullptr;
@@ -292,7 +285,7 @@ std::vector<const NamedDecl *> HeuristicResolverImpl::resolveMemberExpr(
   //      an instance method, it's represented as a CXXDependentScopeMemberExpr
   //      with `this` as the base expression as `X` as the qualifier
   //      (which could be valid if `X` names a base class after instantiation).
-  if (NestedNameSpecifier *NNS = ME->getQualifier()) {
+  if (NestedNameSpecifier NNS = ME->getQualifier()) {
     if (QualType QualifierType = resolveNestedNameSpecifierToType(NNS);
         !QualifierType.isNull()) {
       auto Decls =
@@ -348,7 +341,10 @@ HeuristicResolverImpl::resolveCalleeOfCallExpr(const CallExpr *CE) {
 
 std::vector<const NamedDecl *> HeuristicResolverImpl::resolveUsingValueDecl(
     const UnresolvedUsingValueDecl *UUVD) {
-  return resolveDependentMember(QualType(UUVD->getQualifier()->getAsType(), 0),
+  NestedNameSpecifier Qualifier = UUVD->getQualifier();
+  if (Qualifier.getKind() != NestedNameSpecifier::Kind::Type)
+    return {};
+  return resolveDependentMember(QualType(Qualifier.getAsType(), 0),
                                 UUVD->getNameInfo().getName(), ValueFilter);
 }
 
@@ -399,23 +395,23 @@ QualType HeuristicResolverImpl::resolveExprToType(const Expr *E) {
 }
 
 QualType HeuristicResolverImpl::resolveNestedNameSpecifierToType(
-    const NestedNameSpecifier *NNS) {
-  if (!NNS)
-    return QualType();
-
+    NestedNameSpecifier NNS) {
   // The purpose of this function is to handle the dependent (Kind ==
   // Identifier) case, but we need to recurse on the prefix because
   // that may be dependent as well, so for convenience handle
   // the TypeSpec cases too.
-  switch (NNS->getKind()) {
-  case NestedNameSpecifier::TypeSpec:
-    return QualType(NNS->getAsType(), 0);
-  case NestedNameSpecifier::Identifier: {
-    return resolveDeclsToType(
-        resolveDependentMember(
-            resolveNestedNameSpecifierToType(NNS->getPrefix()),
-            NNS->getAsIdentifier(), TypeFilter),
-        Ctx);
+  switch (NNS.getKind()) {
+  case NestedNameSpecifier::Kind::Type: {
+    const auto *T = NNS.getAsType();
+    // FIXME: Should this handle the DependentTemplateSpecializationType as
+    // well?
+    if (const auto *DTN = dyn_cast<DependentNameType>(T))
+      return resolveDeclsToType(
+          resolveDependentMember(
+              resolveNestedNameSpecifierToType(DTN->getQualifier()),
+              DTN->getIdentifier(), TypeFilter),
+          Ctx);
+    return QualType(T, 0);
   }
   default:
     break;
@@ -590,7 +586,7 @@ HeuristicResolver::resolveTemplateSpecializationType(
   return HeuristicResolverImpl(Ctx).resolveTemplateSpecializationType(DTST);
 }
 QualType HeuristicResolver::resolveNestedNameSpecifierToType(
-    const NestedNameSpecifier *NNS) const {
+    NestedNameSpecifier NNS) const {
   return HeuristicResolverImpl(Ctx).resolveNestedNameSpecifierToType(NNS);
 }
 std::vector<const NamedDecl *> HeuristicResolver::lookupDependentName(
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index cfb2f6013fb1f..b870c5af1e588 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -1434,7 +1434,7 @@ void Sema::ActOnEndOfTranslationUnit() {
   //   translation unit contains a file scope declaration of that
   //   identifier, with the composite type as of the end of the
   //   translation unit, with an initializer equal to 0.
-  llvm::SmallSet<VarDecl *, 32> Seen;
+  llvm::SmallPtrSet<VarDecl *, 32> Seen;
   for (TentativeDefinitionsType::iterator
            T = TentativeDefinitions.begin(ExternalSource.get()),
            TEnd = TentativeDefinitions.end();
@@ -1883,9 +1883,10 @@ class DeferredDiagnosticsEmitter
     for (const FieldDecl *FD : RD->fields()) {
       QualType FT = FD->getType();
       if (const auto *RT = FT->getAs<RecordType>())
-        if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
-          if (ClassDecl->hasDefinition())
-            if (CXXDestructorDecl *MemberDtor = ClassDecl->getDestructor())
+        if (const auto *ClassDecl =
+                dyn_cast<CXXRecordDecl>(RT->getOriginalDecl()))
+          if (const auto *Def = ClassDecl->getDefinition())
+            if (CXXDestructorDecl *MemberDtor = Def->getDestructor())
               asImpl().visitUsedDecl(MemberDtor->getLocation(), MemberDtor);
     }
 
@@ -1893,9 +1894,10 @@ class DeferredDiagnosticsEmitter
     for (const auto &Base : RD->bases()) {
       QualType BaseType = Base.getType();
       if (const auto *RT = BaseType->getAs<RecordType>())
-        if (const auto *BaseDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
-          if (BaseDecl->hasDefinition())
-            if (CXXDestructorDecl *BaseDtor = BaseDecl->getDestructor())
+        if (const auto *BaseDecl =
+                dyn_cast<CXXRecordDecl>(RT->getOriginalDecl()))
+          if (const auto *Def = BaseDecl->getDefinition())
+            if (CXXDestructorDecl *BaseDtor = Def->getDestructor())
               asImpl().visitUsedDecl(BaseDtor->getLocation(), BaseDtor);
     }
   }
@@ -1908,9 +1910,10 @@ class DeferredDiagnosticsEmitter
             VD->needsDestruction(S.Context)) {
           QualType VT = VD->getType();
           if (const auto *RT = VT->getAs<RecordType>())
-            if (const auto *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl()))
-              if (ClassDecl->hasDefinition())
-                if (CXXDestructorDecl *Dtor = ClassDecl->getDestructor())
+            if (const auto *ClassDecl =
+                    dyn_cast<CXXRecordDecl>(RT->getOriginalDecl()))
+              if (const auto *Def = ClassDecl->getDefinition())
+                if (CXXDestructorDecl *Dtor = Def->getDestructor())
                   asImpl().visitUsedDecl(Dtor->getLocation(), Dtor);
         }
 
diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index 044abb0ee08a8..4cc1b76264340 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -336,6 +336,10 @@ static void ProcessAPINotes(Sema &S, Decl *D,
         });
   }
 
+  if (auto ConformsTo = Info.getSwiftConformance())
+    D->addAttr(
+        SwiftAttrAttr::Create(S.Context, "conforms_to:" + ConformsTo.value()));
+
   ProcessAPINotes(S, D, static_cast<const api_notes::CommonEntityInfo &>(Info),
                   Metadata);
 }
@@ -694,14 +698,13 @@ static void ProcessAPINotes(Sema &S, TagDecl *D, const api_notes::TagInfo &Info,
   if (auto ReleaseOp = Info.SwiftReleaseOp)
     D->addAttr(
         SwiftAttrAttr::Create(S.Context, "release:" + ReleaseOp.value()));
+  if (auto DestroyOp = Info.SwiftDestroyOp)
+    D->addAttr(
+        SwiftAttrAttr::Create(S.Context, "destroy:" + DestroyOp.value()));
   if (auto DefaultOwnership = Info.SwiftDefaultOwnership)
     D->addAttr(SwiftAttrAttr::Create(
         S.Context, "returned_as_" + DefaultOwnership.value() + "_by_default"));
 
-  if (auto ConformsTo = Info.SwiftConformance)
-    D->addAttr(
-        SwiftAttrAttr::Create(S.Context, "conforms_to:" + ConformsTo.value()));
-
   if (auto Copyable = Info.isSwiftCopyable()) {
     if (!*Copyable)
       D->addAttr(SwiftAttrAttr::Create(S.Context, "~Copyable"));
diff --git a/clang/lib/Sema/SemaAccess.cpp b/clang/lib/Sema/SemaAccess.cpp
index 83a07a23f3414..ba560d3c52340 100644
--- a/clang/lib/Sema/SemaAccess.cpp
+++ b/clang/lib/Sema/SemaAccess.cpp
@@ -318,11 +318,8 @@ static AccessResult IsDerivedFromInclusive(const CXXRecordDecl *Derived,
       const CXXRecordDecl *RD;
 
       QualType T = I.getType();
-      if (const RecordType *RT = T->getAs<RecordType>()) {
-        RD = cast<CXXRecordDecl>(RT->getDecl());
-      } else if (const InjectedClassNameType *IT
-                   = T->getAs<InjectedClassNameType>()) {
-        RD = IT->getDecl();
+      if (CXXRecordDecl *Rec = T->getAsCXXRecordDecl()) {
+        RD = Rec;
       } else {
         assert(T->isDependentType() && "non-dependent base wasn't a record?");
         OnFailure = AR_dependent;
@@ -443,7 +440,9 @@ static AccessResult MatchesFriend(Sema &S,
                                   const EffectiveContext &EC,
                                   CanQualType Friend) {
   if (const RecordType *RT = Friend->getAs<RecordType>())
-    return MatchesFriend(S, EC, cast<CXXRecordDecl>(RT->getDecl()));
+    return MatchesFriend(
+        S, EC,
+        cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf());
 
   // TODO: we can do better than this
   if (Friend->isDependentType())
@@ -671,11 +670,8 @@ struct ProtectedFriendContext {
       const CXXRecordDecl *RD;
 
       QualType T = I.getType();
-      if (const RecordType *RT = T->getAs<RecordType>()) {
-        RD = cast<CXXRecordDecl>(RT->getDecl());
-      } else if (const InjectedClassNameType *IT
-                   = T->getAs<InjectedClassNameType>()) {
-        RD = IT->getDecl();
+      if (CXXRecordDecl *Rec = T->getAsCXXRecordDecl()) {
+        RD = Rec;
       } else {
         assert(T->isDependentType() && "non-dependent base wasn't a record?");
         EverDependent = true;
@@ -1072,7 +1068,7 @@ static bool TryDiagnoseProtectedAccess(Sema &S, const EffectiveContext &EC,
       // TODO: it would be great to have a fixit here, since this is
       // such an obvious error.
       S.Diag(D->getLocation(), diag::note_access_protected_restricted_noobject)
-        << S.Context.getTypeDeclType(ECRecord);
+          << S.Context.getCanonicalTagType(ECRecord);
       return true;
     }
 
@@ -1101,7 +1097,7 @@ static bool TryDiagnoseProtectedAccess(Sema &S, const EffectiveContext &EC,
     // Otherwise, use the generic diagnostic.
     return S.Diag(D->getLocation(),
                   diag::note_access_protected_restricted_object)
-           << S.Context.getTypeDeclType(ECRecord);
+           << S.Context.getCanonicalTagType(ECRecord);
   }
 
   return false;
@@ -1129,7 +1125,7 @@ static void diagnoseBadDirectAccess(Sema &S,
     else if (TypedefNameDecl *TND = dyn_cast<TypedefNameDecl>(D))
       PrevDecl = TND->getPreviousDecl();
     else if (TagDecl *TD = dyn_cast<TagDecl>(D)) {
-      if (auto *RD = dyn_cast<CXXRecordDecl>(D);
+      if (const auto *RD = dyn_cast<CXXRecordDecl>(TD);
           RD && RD->isInjectedClassName())
         break;
       PrevDecl = TD->getPreviousDecl();
@@ -1284,10 +1280,10 @@ static void DiagnoseBadAccess(Sema &S, SourceLocation Loc,
   NamedDecl *D = (Entity.isMemberAccess() ? Entity.getTargetDecl() : nullptr);
 
   S.Diag(Loc, Entity.getDiag())
-    << (Entity.getAccess() == AS_protected)
-    << (D ? D->getDeclName() : DeclarationName())
-    << S.Context.getTypeDeclType(NamingClass)
-    << S.Context.getTypeDeclType(DeclaringClass);
+      << (Entity.getAccess() == AS_protected)
+      << (D ? D->getDeclName() : DeclarationName())
+      << S.Context.getCanonicalTagType(NamingClass)
+      << S.Context.getCanonicalTagType(DeclaringClass);
   DiagnoseAccessPath(S, EC, Entity);
 }
 
@@ -1640,7 +1636,8 @@ Sema::AccessResult Sema::CheckDestructorAccess(SourceLocation Loc,
     return AR_accessible;
 
   CXXRecordDecl *NamingClass = Dtor->getParent();
-  if (ObjectTy.isNull()) ObjectTy = Context.getTypeDeclType(NamingClass);
+  if (ObjectTy.isNull())
+    ObjectTy = Context.getCanonicalTagType(NamingClass);
 
   AccessTarget Entity(Context, AccessTarget::Member, NamingClass,
                       DeclAccessPair::make(Dtor, Access),
@@ -1728,7 +1725,7 @@ Sema::AccessResult Sema::CheckConstructorAccess(SourceLocation UseLoc,
   AccessTarget AccessEntity(
       Context, AccessTarget::Member, NamingClass,
       DeclAccessPair::make(Constructor, Found.getAccess()),
-      Context.getTypeDeclType(ObjectClass));
+      Context.getCanonicalTagType(ObjectClass));
   AccessEntity.setDiag(PD);
 
   return CheckAccess(*this, UseLoc, AccessEntity);
@@ -1776,7 +1773,7 @@ Sema::CheckStructuredBindingMemberAccess(SourceLocation UseLoc,
     return AR_accessible;
 
   AccessTarget Entity(Context, AccessTarget::Member, DecomposedClass, Field,
-                      Context.getRecordType(DecomposedClass));
+                      Context.getCanonicalTagType(DecomposedClass));
   Entity.setDiag(diag::err_decomp_decl_inaccessible_field);
 
   return CheckAccess(*this, UseLoc, Entity);
@@ -1790,7 +1787,8 @@ Sema::AccessResult Sema::CheckMemberOperatorAccess(SourceLocation OpLoc,
     return AR_accessible;
 
   const RecordType *RT = ObjectExpr->getType()->castAs<RecordType>();
-  CXXRecordDecl *NamingClass = cast<CXXRecordDecl>(RT->getDecl());
+  CXXRecordDecl *NamingClass =
+      cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
 
   AccessTarget Entity(Context, AccessTarget::Member, NamingClass, Found,
                       ObjectExpr->getType());
diff --git a/clang/lib/Sema/SemaAvailability.cpp b/clang/lib/Sema/SemaAvailability.cpp
index 68a698fea1815..1c48b3ca86fbe 100644
--- a/clang/lib/Sema/SemaAvailability.cpp
+++ b/clang/lib/Sema/SemaAvailability.cpp
@@ -102,7 +102,7 @@ Sema::ShouldDiagnoseAvailabilityOfDecl(const NamedDecl *D, std::string *Message,
       break;
     for (const Type *T = TD->getUnderlyingType().getTypePtr(); /**/; /**/) {
       if (auto *TT = dyn_cast<TagType>(T)) {
-        D = TT->getDecl();
+        D = TT->getOriginalDecl()->getDefinitionOrSelf();
       } else if (isa<SubstTemplateTypeParmType>(T)) {
         // A Subst* node represents a use through a template.
         // Any uses of the underlying declaration happened through it's template
@@ -1017,7 +1017,7 @@ bool DiagnoseUnguardedAvailability::VisitTypeLoc(TypeLoc Ty) {
     return true;
 
   if (const auto *TT = dyn_cast<TagType>(TyPtr)) {
-    TagDecl *TD = TT->getDecl();
+    TagDecl *TD = TT->getOriginalDecl()->getDefinitionOrSelf();
     DiagnoseDeclAvailability(TD, Range);
 
   } else if (const auto *TD = dyn_cast<TypedefType>(TyPtr)) {
diff --git a/clang/lib/Sema/SemaBPF.cpp b/clang/lib/Sema/SemaBPF.cpp
index 7c00084d62dd9..6428435ed9d2a 100644
--- a/clang/lib/Sema/SemaBPF.cpp
+++ b/clang/lib/Sema/SemaBPF.cpp
@@ -58,10 +58,10 @@ static bool isValidPreserveTypeInfoArg(Expr *Arg) {
   // Record type or Enum type.
   const Type *Ty = ArgType->getUnqualifiedDesugaredType();
   if (const auto *RT = Ty->getAs<RecordType>()) {
-    if (!RT->getDecl()->getDeclName().isEmpty())
+    if (!RT->getOriginalDecl()->getDeclName().isEmpty())
       return true;
   } else if (const auto *ET = Ty->getAs<EnumType>()) {
-    if (!ET->getDecl()->getDeclName().isEmpty())
+    if (!ET->getOriginalDecl()->getDeclName().isEmpty())
       return true;
   }
 
@@ -105,7 +105,7 @@ static bool isValidPreserveEnumValueArg(Expr *Arg) {
     return false;
 
   // The enum value must be supported.
-  return llvm::is_contained(ET->getDecl()->enumerators(), Enumerator);
+  return llvm::is_contained(ET->getOriginalDecl()->enumerators(), Enumerator);
 }
 
 bool SemaBPF::CheckBPFBuiltinFunctionCall(unsigned BuiltinID,
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 24cb8c369790b..fbf64d3d57050 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -426,7 +426,8 @@ bool SemaCUDA::inferTargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
       continue;
     }
 
-    CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(BaseType->getDecl());
+    CXXRecordDecl *BaseClassDecl =
+        cast<CXXRecordDecl>(BaseType->getOriginalDecl())->getDefinitionOrSelf();
     Sema::SpecialMemberOverloadResult SMOR =
         SemaRef.LookupSpecialMember(BaseClassDecl, CSM,
                                     /* ConstArg */ ConstRHS,
@@ -471,7 +472,9 @@ bool SemaCUDA::inferTargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
       continue;
     }
 
-    CXXRecordDecl *FieldRecDecl = cast<CXXRecordDecl>(FieldType->getDecl());
+    CXXRecordDecl *FieldRecDecl =
+        cast<CXXRecordDecl>(FieldType->getOriginalDecl())
+            ->getDefinitionOrSelf();
     Sema::SpecialMemberOverloadResult SMOR =
         SemaRef.LookupSpecialMember(FieldRecDecl, CSM,
                                     /* ConstArg */ ConstRHS && !F->isMutable(),
diff --git a/clang/lib/Sema/SemaCXXScopeSpec.cpp b/clang/lib/Sema/SemaCXXScopeSpec.cpp
index 6ac04837708f6..45de8ff3ba264 100644
--- a/clang/lib/Sema/SemaCXXScopeSpec.cpp
+++ b/clang/lib/Sema/SemaCXXScopeSpec.cpp
@@ -28,24 +28,21 @@ static CXXRecordDecl *getCurrentInstantiationOf(QualType T,
   if (T.isNull())
     return nullptr;
 
-  const Type *Ty = T->getCanonicalTypeInternal().getTypePtr();
-  if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
-    CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl());
-    if (Record->isCurrentInstantiation(CurContext))
-      return Record;
-
-    return nullptr;
-  } else if (isa<InjectedClassNameType>(Ty))
-    return cast<InjectedClassNameType>(Ty)->getDecl();
-  else
+  const TagType *TagTy = dyn_cast<TagType>(T->getCanonicalTypeInternal());
+  if (!isa_and_present<RecordType, InjectedClassNameType>(TagTy))
     return nullptr;
+  auto *RD =
+      cast<CXXRecordDecl>(TagTy->getOriginalDecl())->getDefinitionOrSelf();
+  if (isa<InjectedClassNameType>(TagTy) ||
+      RD->isCurrentInstantiation(CurContext))
+    return RD;
+  return nullptr;
 }
 
 DeclContext *Sema::computeDeclContext(QualType T) {
   if (!T->isDependentType())
-    if (const TagType *Tag = T->getAs<TagType>())
-      return Tag->getDecl();
-
+    if (auto *D = T->getAsTagDecl())
+      return D;
   return ::getCurrentInstantiationOf(T, CurContext);
 }
 
@@ -54,18 +51,17 @@ DeclContext *Sema::computeDeclContext(const CXXScopeSpec &SS,
   if (!SS.isSet() || SS.isInvalid())
     return nullptr;
 
-  NestedNameSpecifier *NNS = SS.getScopeRep();
-  if (NNS->isDependent()) {
+  NestedNameSpecifier NNS = SS.getScopeRep();
+  if (NNS.isDependent()) {
     // If this nested-name-specifier refers to the current
     // instantiation, return its DeclContext.
     if (CXXRecordDecl *Record = getCurrentInstantiationOf(NNS))
       return Record;
 
     if (EnteringContext) {
-      const Type *NNSType = NNS->getAsType();
-      if (!NNSType) {
+      if (NNS.getKind() != NestedNameSpecifier::Kind::Type)
         return nullptr;
-      }
+      const Type *NNSType = NNS.getAsType();
 
       // Look through type alias templates, per C++0x [temp.dep.type]p1.
       NNSType = Context.getCanonicalType(NNSType);
@@ -118,38 +114,39 @@ DeclContext *Sema::computeDeclContext(const CXXScopeSpec &SS,
           // If the type of the nested name specifier is the same as the
           // injected class name of the named class template, we're entering
           // into that class template definition.
-          QualType Injected =
-              ClassTemplate->getInjectedClassNameSpecialization();
+          CanQualType Injected =
+              ClassTemplate->getCanonicalInjectedSpecializationType(Context);
           if (Context.hasSameType(Injected, QualType(SpecType, 0)))
             return ClassTemplate->getTemplatedDecl();
         }
-      } else if (const RecordType *RecordT = NNSType->getAs<RecordType>()) {
+      } else if (const auto *RecordT = dyn_cast<RecordType>(NNSType)) {
         // The nested name specifier refers to a member of a class template.
-        return RecordT->getDecl();
+        return RecordT->getOriginalDecl()->getDefinitionOrSelf();
       }
     }
 
     return nullptr;
   }
 
-  switch (NNS->getKind()) {
-  case NestedNameSpecifier::Identifier:
-    llvm_unreachable("Dependent nested-name-specifier has no DeclContext");
-
-  case NestedNameSpecifier::Namespace:
-    return NNS->getAsNamespace()->getNamespace();
+  switch (NNS.getKind()) {
+  case NestedNameSpecifier::Kind::Namespace:
+    return const_cast<NamespaceDecl *>(
+        NNS.getAsNamespaceAndPrefix().Namespace->getNamespace());
 
-  case NestedNameSpecifier::TypeSpec: {
-    const TagType *Tag = NNS->getAsType()->getAs<TagType>();
-    assert(Tag && "Non-tag type in nested-name-specifier");
-    return Tag->getDecl();
+  case NestedNameSpecifier::Kind::Type: {
+    auto *TD = NNS.getAsType()->getAsTagDecl();
+    assert(TD && "Non-tag type in nested-name-specifier");
+    return TD;
   }
 
-  case NestedNameSpecifier::Global:
+  case NestedNameSpecifier::Kind::Global:
     return Context.getTranslationUnitDecl();
 
-  case NestedNameSpecifier::Super:
-    return NNS->getAsRecordDecl();
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    return NNS.getAsMicrosoftSuper();
+
+  case NestedNameSpecifier::Kind::Null:
+    llvm_unreachable("unexpected null nested name specifier");
   }
 
   llvm_unreachable("Invalid NestedNameSpecifier::Kind!");
@@ -159,17 +156,17 @@ bool Sema::isDependentScopeSpecifier(const CXXScopeSpec &SS) {
   if (!SS.isSet() || SS.isInvalid())
     return false;
 
-  return SS.getScopeRep()->isDependent();
+  return SS.getScopeRep().isDependent();
 }
 
-CXXRecordDecl *Sema::getCurrentInstantiationOf(NestedNameSpecifier *NNS) {
+CXXRecordDecl *Sema::getCurrentInstantiationOf(NestedNameSpecifier NNS) {
   assert(getLangOpts().CPlusPlus && "Only callable in C++");
-  assert(NNS->isDependent() && "Only dependent nested-name-specifier allowed");
+  assert(NNS.isDependent() && "Only dependent nested-name-specifier allowed");
 
-  if (!NNS->getAsType())
+  if (NNS.getKind() != NestedNameSpecifier::Kind::Type)
     return nullptr;
 
-  QualType T = QualType(NNS->getAsType(), 0);
+  QualType T = QualType(NNS.getAsType(), 0);
   return ::getCurrentInstantiationOf(T, CurContext);
 }
 
@@ -195,8 +192,7 @@ bool Sema::RequireCompleteDeclContext(CXXScopeSpec &SS,
     return false;
 
   // Grab the tag definition, if there is one.
-  QualType type = Context.getTypeDeclType(tag);
-  tag = type->getAsTagDecl();
+  tag = tag->getDefinitionOrSelf();
 
   // If we're currently defining this type, then lookup into the
   // type is okay: don't complain that it isn't complete yet.
@@ -207,7 +203,8 @@ bool Sema::RequireCompleteDeclContext(CXXScopeSpec &SS,
   if (loc.isInvalid()) loc = SS.getRange().getBegin();
 
   // The type must be complete.
-  if (RequireCompleteType(loc, type, diag::err_incomplete_nested_name_spec,
+  if (RequireCompleteType(loc, Context.getCanonicalTagType(tag),
+                          diag::err_incomplete_nested_name_spec,
                           SS.getRange())) {
     SS.SetInvalid(SS.getRange());
     return true;
@@ -259,10 +256,10 @@ bool Sema::RequireCompleteEnumDecl(EnumDecl *EnumD, SourceLocation L,
 
   if (SS) {
     Diag(L, diag::err_incomplete_nested_name_spec)
-        << QualType(EnumD->getTypeForDecl(), 0) << SS->getRange();
+        << Context.getCanonicalTagType(EnumD) << SS->getRange();
     SS->SetInvalid(SS->getRange());
   } else {
-    Diag(L, diag::err_incomplete_enum) << QualType(EnumD->getTypeForDecl(), 0);
+    Diag(L, diag::err_incomplete_enum) << Context.getCanonicalTagType(EnumD);
     Diag(EnumD->getLocation(), diag::note_declared_at);
   }
 
@@ -304,7 +301,7 @@ bool Sema::ActOnSuperScopeSpecifier(SourceLocation SuperLoc,
     return true;
   }
 
-  SS.MakeSuper(Context, RD, SuperLoc, ColonColonLoc);
+  SS.MakeMicrosoftSuper(Context, RD, SuperLoc, ColonColonLoc);
   return false;
 }
 
@@ -324,9 +321,6 @@ bool Sema::isAcceptableNestedNameSpecifier(const NamedDecl *SD,
 
   // Determine whether we have a class (or, in C++11, an enum) or
   // a typedef thereof. If so, build the nested-name-specifier.
-  QualType T = Context.getTypeDeclType(cast<TypeDecl>(SD));
-  if (T->isDependentType())
-    return true;
   if (const TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(SD)) {
     if (TD->getUnderlyingType()->isRecordType())
       return true;
@@ -344,32 +338,42 @@ bool Sema::isAcceptableNestedNameSpecifier(const NamedDecl *SD,
     if (IsExtension)
       *IsExtension = true;
   }
+  if (auto *TD = dyn_cast<TagDecl>(SD)) {
+    if (TD->isDependentType())
+      return true;
+  } else if (Context.getCanonicalTypeDeclType(cast<TypeDecl>(SD))
+                 ->isDependentType()) {
+    return true;
+  }
 
   return false;
 }
 
-NamedDecl *Sema::FindFirstQualifierInScope(Scope *S, NestedNameSpecifier *NNS) {
-  if (!S || !NNS)
+NamedDecl *Sema::FindFirstQualifierInScope(Scope *S, NestedNameSpecifier NNS) {
+  if (!S)
     return nullptr;
 
-  while (NNS->getPrefix())
-    NNS = NNS->getPrefix();
-
-  if (NNS->getKind() != NestedNameSpecifier::Identifier)
-    return nullptr;
+  while (NNS.getKind() == NestedNameSpecifier::Kind::Type) {
+    const Type *T = NNS.getAsType();
+    if ((NNS = T->getPrefix()))
+      continue;
 
-  LookupResult Found(*this, NNS->getAsIdentifier(), SourceLocation(),
-                     LookupNestedNameSpecifierName);
-  LookupName(Found, S);
-  assert(!Found.isAmbiguous() && "Cannot handle ambiguities here yet");
+    const auto *DNT = dyn_cast<DependentNameType>(T);
+    if (!DNT)
+      break;
 
-  if (!Found.isSingleResult())
-    return nullptr;
+    LookupResult Found(*this, DNT->getIdentifier(), SourceLocation(),
+                       LookupNestedNameSpecifierName);
+    LookupName(Found, S);
+    assert(!Found.isAmbiguous() && "Cannot handle ambiguities here yet");
 
-  NamedDecl *Result = Found.getFoundDecl();
-  if (isAcceptableNestedNameSpecifier(Result))
-    return Result;
+    if (!Found.isSingleResult())
+      return nullptr;
 
+    NamedDecl *Result = Found.getFoundDecl();
+    if (isAcceptableNestedNameSpecifier(Result))
+      return Result;
+  }
   return nullptr;
 }
 
@@ -493,7 +497,18 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo,
     // base object type or prior nested-name-specifier, so this
     // nested-name-specifier refers to an unknown specialization. Just build
     // a dependent nested-name-specifier.
-    SS.Extend(Context, IdInfo.Identifier, IdInfo.IdentifierLoc, IdInfo.CCLoc);
+
+    TypeLocBuilder TLB;
+
+    QualType DTN = Context.getDependentNameType(
+        ElaboratedTypeKeyword::None, SS.getScopeRep(), IdInfo.Identifier);
+    auto DTNL = TLB.push<DependentNameTypeLoc>(DTN);
+    DTNL.setElaboratedKeywordLoc(SourceLocation());
+    DTNL.setNameLoc(IdInfo.IdentifierLoc);
+    DTNL.setQualifierLoc(SS.getWithLocInContext(Context));
+
+    SS.clear();
+    SS.Make(Context, TLB.getTypeLocInContext(Context, DTN), IdInfo.CCLoc);
     return false;
   }
 
@@ -599,8 +614,8 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo,
           OuterDecl->getCanonicalDecl() != SD->getCanonicalDecl() &&
           (!isa<TypeDecl>(OuterDecl) || !isa<TypeDecl>(SD) ||
            !Context.hasSameType(
-                            Context.getTypeDeclType(cast<TypeDecl>(OuterDecl)),
-                               Context.getTypeDeclType(cast<TypeDecl>(SD))))) {
+               Context.getCanonicalTypeDeclType(cast<TypeDecl>(OuterDecl)),
+               Context.getCanonicalTypeDeclType(cast<TypeDecl>(SD))))) {
         if (ErrorRecoveryLookup)
           return true;
 
@@ -613,7 +628,7 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo,
 
          // Fall through so that we'll pick the name we found in the object
          // type, since that's probably what the user wanted anyway.
-       }
+      }
     }
 
     if (auto *TD = dyn_cast_or_null<TypedefNameDecl>(SD))
@@ -637,50 +652,44 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo,
       return false;
     }
 
-    QualType T =
-        Context.getTypeDeclType(cast<TypeDecl>(SD->getUnderlyingDecl()));
-
-    if (T->isEnumeralType())
+    const auto *TD = cast<TypeDecl>(SD->getUnderlyingDecl());
+    if (isa<EnumDecl>(TD))
       Diag(IdInfo.IdentifierLoc, diag::warn_cxx98_compat_enum_nested_name_spec);
 
+    QualType T;
     TypeLocBuilder TLB;
     if (const auto *USD = dyn_cast<UsingShadowDecl>(SD)) {
-      T = Context.getUsingType(USD, T);
-      TLB.pushTypeSpec(T).setNameLoc(IdInfo.IdentifierLoc);
-    } else if (isa<InjectedClassNameType>(T)) {
-      InjectedClassNameTypeLoc InjectedTL
-        = TLB.push<InjectedClassNameTypeLoc>(T);
-      InjectedTL.setNameLoc(IdInfo.IdentifierLoc);
-    } else if (isa<RecordType>(T)) {
-      RecordTypeLoc RecordTL = TLB.push<RecordTypeLoc>(T);
-      RecordTL.setNameLoc(IdInfo.IdentifierLoc);
-    } else if (isa<TypedefType>(T)) {
-      TypedefTypeLoc TypedefTL = TLB.push<TypedefTypeLoc>(T);
-      TypedefTL.setNameLoc(IdInfo.IdentifierLoc);
-    } else if (isa<EnumType>(T)) {
-      EnumTypeLoc EnumTL = TLB.push<EnumTypeLoc>(T);
-      EnumTL.setNameLoc(IdInfo.IdentifierLoc);
-    } else if (isa<TemplateTypeParmType>(T)) {
-      TemplateTypeParmTypeLoc TemplateTypeTL
-        = TLB.push<TemplateTypeParmTypeLoc>(T);
-      TemplateTypeTL.setNameLoc(IdInfo.IdentifierLoc);
-    } else if (isa<UnresolvedUsingType>(T)) {
-      UnresolvedUsingTypeLoc UnresolvedTL
-        = TLB.push<UnresolvedUsingTypeLoc>(T);
-      UnresolvedTL.setNameLoc(IdInfo.IdentifierLoc);
-    } else if (isa<SubstTemplateTypeParmType>(T)) {
-      SubstTemplateTypeParmTypeLoc TL
-        = TLB.push<SubstTemplateTypeParmTypeLoc>(T);
-      TL.setNameLoc(IdInfo.IdentifierLoc);
-    } else if (isa<SubstTemplateTypeParmPackType>(T)) {
-      SubstTemplateTypeParmPackTypeLoc TL
-        = TLB.push<SubstTemplateTypeParmPackTypeLoc>(T);
-      TL.setNameLoc(IdInfo.IdentifierLoc);
+      T = Context.getUsingType(ElaboratedTypeKeyword::None, SS.getScopeRep(),
+                               USD);
+      TLB.push<UsingTypeLoc>(T).set(/*ElaboratedKeywordLoc=*/SourceLocation(),
+                                    SS.getWithLocInContext(Context),
+                                    IdInfo.IdentifierLoc);
+    } else if (const auto *Tag = dyn_cast<TagDecl>(TD)) {
+      T = Context.getTagType(ElaboratedTypeKeyword::None, SS.getScopeRep(), Tag,
+                             /*OwnsTag=*/false);
+      auto TTL = TLB.push<TagTypeLoc>(T);
+      TTL.setElaboratedKeywordLoc(SourceLocation());
+      TTL.setQualifierLoc(SS.getWithLocInContext(SemaRef.Context));
+      TTL.setNameLoc(IdInfo.IdentifierLoc);
+    } else if (auto *TN = dyn_cast<TypedefNameDecl>(TD)) {
+      T = Context.getTypedefType(ElaboratedTypeKeyword::None, SS.getScopeRep(),
+                                 TN);
+      TLB.push<TypedefTypeLoc>(T).set(/*ElaboratedKeywordLoc=*/SourceLocation(),
+                                      SS.getWithLocInContext(SemaRef.Context),
+                                      IdInfo.IdentifierLoc);
+    } else if (auto *UD = dyn_cast<UnresolvedUsingTypenameDecl>(TD)) {
+      T = Context.getUnresolvedUsingType(ElaboratedTypeKeyword::None,
+                                         SS.getScopeRep(), UD);
+      TLB.push<UnresolvedUsingTypeLoc>(T).set(
+          /*ElaboratedKeywordLoc=*/SourceLocation(),
+          SS.getWithLocInContext(SemaRef.Context), IdInfo.IdentifierLoc);
     } else {
-      llvm_unreachable("Unhandled TypeDecl node in nested-name-specifier");
+      assert(SS.isEmpty());
+      T = Context.getTypeDeclType(TD);
+      TLB.pushTypeSpec(T).setNameLoc(IdInfo.IdentifierLoc);
     }
-
-    SS.Extend(Context, TLB.getTypeLocInContext(Context, T), IdInfo.CCLoc);
+    SS.clear();
+    SS.Make(Context, TLB.getTypeLocInContext(Context, T), IdInfo.CCLoc);
     return false;
   }
 
@@ -722,16 +731,34 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo,
         Diag(IdInfo.IdentifierLoc,
              diag::ext_undeclared_unqual_id_with_dependent_base)
             << IdInfo.Identifier << ContainingClass;
+
+        TypeLocBuilder TLB;
+
         // Fake up a nested-name-specifier that starts with the
         // injected-class-name of the enclosing class.
-        QualType T = Context.getTypeDeclType(ContainingClass);
-        TypeLocBuilder TLB;
-        TLB.pushTrivial(Context, T, IdInfo.IdentifierLoc);
-        SS.Extend(Context, TLB.getTypeLocInContext(Context, T),
-                  IdInfo.IdentifierLoc);
-        // Add the identifier to form a dependent name.
-        SS.Extend(Context, IdInfo.Identifier, IdInfo.IdentifierLoc,
-                  IdInfo.CCLoc);
+        // FIXME: This should be done as part of an adjustment, so that this
+        // doesn't get confused with something written in source.
+        QualType Result =
+            Context.getTagType(ElaboratedTypeKeyword::None, SS.getScopeRep(),
+                               ContainingClass, /*OwnsTag=*/false);
+        auto TTL = TLB.push<TagTypeLoc>(Result);
+        TTL.setElaboratedKeywordLoc(SourceLocation());
+        TTL.setQualifierLoc(SS.getWithLocInContext(Context));
+        TTL.setNameLoc(IdInfo.IdentifierLoc);
+        SS.Make(Context, TLB.getTypeLocInContext(Context, Result),
+                SourceLocation());
+
+        TLB.clear();
+
+        // Form a DependentNameType.
+        QualType DTN = Context.getDependentNameType(
+            ElaboratedTypeKeyword::None, SS.getScopeRep(), IdInfo.Identifier);
+        auto DTNL = TLB.push<DependentNameTypeLoc>(DTN);
+        DTNL.setElaboratedKeywordLoc(SourceLocation());
+        DTNL.setNameLoc(IdInfo.IdentifierLoc);
+        DTNL.setQualifierLoc(SS.getWithLocInContext(Context));
+        SS.clear();
+        SS.Make(Context, TLB.getTypeLocInContext(Context, DTN), IdInfo.CCLoc);
         return false;
       }
     }
@@ -739,8 +766,19 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S, NestedNameSpecInfo &IdInfo,
 
   if (!Found.empty()) {
     if (TypeDecl *TD = Found.getAsSingle<TypeDecl>()) {
+      QualType T;
+      if (auto *TN = dyn_cast<TypedefNameDecl>(TD)) {
+        T = Context.getTypedefType(ElaboratedTypeKeyword::None,
+                                   SS.getScopeRep(), TN);
+      } else {
+        // FIXME: Enumerate the possibilities here.
+        assert(!isa<TagDecl>(TD));
+        assert(SS.isEmpty());
+        T = Context.getTypeDeclType(TD);
+      }
+
       Diag(IdInfo.IdentifierLoc, diag::err_expected_class_or_namespace)
-          << Context.getTypeDeclType(TD) << getLangOpts().CPlusPlus;
+          << T << getLangOpts().CPlusPlus;
     } else if (Found.getAsSingle<TemplateDecl>()) {
       ParsedType SuggestedType;
       DiagnoseUnknownTypeName(IdInfo.Identifier, IdInfo.IdentifierLoc, S, &SS,
@@ -786,17 +824,19 @@ bool Sema::ActOnCXXNestedNameSpecifierDecltype(CXXScopeSpec &SS,
   if (T.isNull())
     return true;
 
-  if (!T->isDependentType() && !T->getAs<TagType>()) {
+  if (!T->isDependentType() && !isa<TagType>(T.getCanonicalType())) {
     Diag(DS.getTypeSpecTypeLoc(), diag::err_expected_class_or_namespace)
       << T << getLangOpts().CPlusPlus;
     return true;
   }
 
+  assert(SS.isEmpty());
+
   TypeLocBuilder TLB;
   DecltypeTypeLoc DecltypeTL = TLB.push<DecltypeTypeLoc>(T);
   DecltypeTL.setDecltypeLoc(DS.getTypeSpecTypeLoc());
   DecltypeTL.setRParenLoc(DS.getTypeofParensRange().getEnd());
-  SS.Extend(Context, TLB.getTypeLocInContext(Context, T), ColonColonLoc);
+  SS.Make(Context, TLB.getTypeLocInContext(Context, T), ColonColonLoc);
   return false;
 }
 
@@ -812,13 +852,15 @@ bool Sema::ActOnCXXNestedNameSpecifierIndexedPack(CXXScopeSpec &SS,
   if (Type.isNull())
     return true;
 
+  assert(SS.isEmpty());
+
   TypeLocBuilder TLB;
   TLB.pushTrivial(getASTContext(),
                   cast<PackIndexingType>(Type.getTypePtr())->getPattern(),
                   DS.getBeginLoc());
   PackIndexingTypeLoc PIT = TLB.push<PackIndexingTypeLoc>(Type);
   PIT.setEllipsisLoc(DS.getEllipsisLoc());
-  SS.Extend(Context, TLB.getTypeLocInContext(Context, Type), ColonColonLoc);
+  SS.Make(Context, TLB.getTypeLocInContext(Context, Type), ColonColonLoc);
   return false;
 }
 
@@ -858,7 +900,7 @@ bool Sema::ActOnCXXNestedNameSpecifier(Scope *S,
     assert(DTN->getQualifier() == SS.getScopeRep());
     QualType T = Context.getDependentTemplateSpecializationType(
         ElaboratedTypeKeyword::None,
-        {/*Qualifier=*/nullptr, DTN->getName().getIdentifier(),
+        {SS.getScopeRep(), DTN->getName().getIdentifier(),
          TemplateKWLoc.isValid()},
         TemplateArgs.arguments());
 
@@ -867,7 +909,7 @@ bool Sema::ActOnCXXNestedNameSpecifier(Scope *S,
     DependentTemplateSpecializationTypeLoc SpecTL
       = Builder.push<DependentTemplateSpecializationTypeLoc>(T);
     SpecTL.setElaboratedKeywordLoc(SourceLocation());
-    SpecTL.setQualifierLoc(NestedNameSpecifierLoc());
+    SpecTL.setQualifierLoc(SS.getWithLocInContext(Context));
     SpecTL.setTemplateKeywordLoc(TemplateKWLoc);
     SpecTL.setTemplateNameLoc(TemplateNameLoc);
     SpecTL.setLAngleLoc(LAngleLoc);
@@ -875,7 +917,8 @@ bool Sema::ActOnCXXNestedNameSpecifier(Scope *S,
     for (unsigned I = 0, N = TemplateArgs.size(); I != N; ++I)
       SpecTL.setArgLocInfo(I, TemplateArgs[I].getLocInfo());
 
-    SS.Extend(Context, Builder.getTypeLocInContext(Context, T), CCLoc);
+    SS.clear();
+    SS.Make(Context, Builder.getTypeLocInContext(Context, T), CCLoc);
     return false;
   }
 
@@ -900,30 +943,28 @@ bool Sema::ActOnCXXNestedNameSpecifier(Scope *S,
 
   // We were able to resolve the template name to an actual template.
   // Build an appropriate nested-name-specifier.
-  QualType T = CheckTemplateIdType(Template, TemplateNameLoc, TemplateArgs);
+  QualType T = CheckTemplateIdType(ElaboratedTypeKeyword::None, Template,
+                                   TemplateNameLoc, TemplateArgs);
   if (T.isNull())
     return true;
 
   // Alias template specializations can produce types which are not valid
   // nested name specifiers.
-  if (!T->isDependentType() && !T->getAs<TagType>()) {
+  if (!T->isDependentType() && !isa<TagType>(T.getCanonicalType())) {
     Diag(TemplateNameLoc, diag::err_nested_name_spec_non_tag) << T;
     NoteAllFoundTemplates(Template);
     return true;
   }
 
   // Provide source-location information for the template specialization type.
-  TypeLocBuilder Builder;
-  TemplateSpecializationTypeLoc SpecTL
-    = Builder.push<TemplateSpecializationTypeLoc>(T);
-  SpecTL.setTemplateKeywordLoc(TemplateKWLoc);
-  SpecTL.setTemplateNameLoc(TemplateNameLoc);
-  SpecTL.setLAngleLoc(LAngleLoc);
-  SpecTL.setRAngleLoc(RAngleLoc);
-  for (unsigned I = 0, N = TemplateArgs.size(); I != N; ++I)
-    SpecTL.setArgLocInfo(I, TemplateArgs[I].getLocInfo());
-
-  SS.Extend(Context, Builder.getTypeLocInContext(Context, T), CCLoc);
+  TypeLocBuilder TLB;
+  TLB.push<TemplateSpecializationTypeLoc>(T).set(
+      /*ElaboratedKeywordLoc=*/SourceLocation(),
+      SS.getWithLocInContext(Context), TemplateKWLoc, TemplateNameLoc,
+      TemplateArgs);
+
+  SS.clear();
+  SS.Make(Context, TLB.getTypeLocInContext(Context, T), CCLoc);
   return false;
 }
 
@@ -931,7 +972,7 @@ namespace {
   /// A structure that stores a nested-name-specifier annotation,
   /// including both the nested-name-specifier
   struct NestedNameSpecifierAnnotation {
-    NestedNameSpecifier *NNS;
+    NestedNameSpecifier NNS = std::nullopt;
   };
 }
 
@@ -970,8 +1011,6 @@ bool Sema::ShouldEnterDeclaratorScope(Scope *S, const CXXScopeSpec &SS) {
   if (isa<ObjCContainerDecl>(CurContext) || isa<ObjCMethodDecl>(CurContext))
     return false;
 
-  NestedNameSpecifier *Qualifier = SS.getScopeRep();
-
   // There are only two places a well-formed program may qualify a
   // declarator: first, when defining a namespace or class member
   // out-of-line, and second, when naming an explicitly-qualified
@@ -986,18 +1025,20 @@ bool Sema::ShouldEnterDeclaratorScope(Scope *S, const CXXScopeSpec &SS) {
   //   granting friendship.
   // i.e. we don't push a scope unless it's a class member.
 
-  switch (Qualifier->getKind()) {
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Namespace:
+  switch (SS.getScopeRep().getKind()) {
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::Namespace:
     // These are always namespace scopes.  We never want to enter a
     // namespace scope from anything but a file context.
     return CurContext->getRedeclContext()->isFileContext();
 
-  case NestedNameSpecifier::Identifier:
-  case NestedNameSpecifier::TypeSpec:
-  case NestedNameSpecifier::Super:
+  case NestedNameSpecifier::Kind::Type:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     // These are never namespace scopes.
     return true;
+
+  case NestedNameSpecifier::Kind::Null:
+    llvm_unreachable("unexpected null nested name specifier");
   }
 
   llvm_unreachable("Invalid NestedNameSpecifier::Kind!");
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 01252a4bc69c6..da43848a1a7d0 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -964,7 +964,7 @@ void CastOperation::CheckDynamicCast() {
   }
 
   // C++ 5.2.7p6: Otherwise, v shall be [polymorphic].
-  const RecordDecl *SrcDecl = SrcRecord->getDecl()->getDefinition();
+  const RecordDecl *SrcDecl = SrcRecord->getOriginalDecl()->getDefinition();
   assert(SrcDecl && "Definition missing");
   if (!cast<CXXRecordDecl>(SrcDecl)->isPolymorphic()) {
     Self.Diag(OpRange.getBegin(), diag::err_bad_dynamic_cast_not_polymorphic)
@@ -1455,7 +1455,7 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
   // converted to an integral type. [...] A value of a scoped enumeration type
   // can also be explicitly converted to a floating-point type [...].
   if (const EnumType *Enum = SrcType->getAs<EnumType>()) {
-    if (Enum->getDecl()->isScoped()) {
+    if (Enum->getOriginalDecl()->isScoped()) {
       if (DestType->isBooleanType()) {
         Kind = CK_IntegralToBoolean;
         return TC_Success;
@@ -1487,8 +1487,8 @@ static TryCastResult TryStaticCast(Sema &Self, ExprResult &SrcExpr,
       // [expr.static.cast]p10 If the enumeration type has a fixed underlying
       // type, the value is first converted to that type by integral conversion
       const EnumType *Enum = DestType->castAs<EnumType>();
-      Kind = Enum->getDecl()->isFixed() &&
-                     Enum->getDecl()->getIntegerType()->isBooleanType()
+      const EnumDecl *ED = Enum->getOriginalDecl()->getDefinitionOrSelf();
+      Kind = ED->isFixed() && ED->getIntegerType()->isBooleanType()
                  ? CK_IntegralToBoolean
                  : CK_IntegralCast;
       return TC_Success;
@@ -1856,7 +1856,7 @@ TryCastResult TryStaticMemberPointerUpcast(Sema &Self, ExprResult &SrcExpr,
                                                     FoundOverload)) {
       CXXMethodDecl *M = cast<CXXMethodDecl>(Fn);
       SrcType = Self.Context.getMemberPointerType(
-          Fn->getType(), /*Qualifier=*/nullptr, M->getParent());
+          Fn->getType(), /*Qualifier=*/std::nullopt, M->getParent());
       WasOverloadedFunction = true;
     }
   }
@@ -2102,9 +2102,9 @@ void Sema::CheckCompatibleReinterpretCast(QualType SrcType, QualType DestType,
     return;
   }
   // or one of the types is a tag type.
-  if (SrcTy->getAs<TagType>() || DestTy->getAs<TagType>()) {
+  if (isa<TagType>(SrcTy.getCanonicalType()) ||
+      isa<TagType>(DestTy.getCanonicalType()))
     return;
-  }
 
   // FIXME: Scoped enums?
   if ((SrcTy->isUnsignedIntegerType() && DestTy->isSignedIntegerType()) ||
@@ -3097,27 +3097,26 @@ void CastOperation::CheckCStyleCast() {
 
   if (!DestType->isScalarType() && !DestType->isVectorType() &&
       !DestType->isMatrixType()) {
-    const RecordType *DestRecordTy = DestType->getAs<RecordType>();
-
-    if (DestRecordTy && Self.Context.hasSameUnqualifiedType(DestType, SrcType)){
-      // GCC struct/union extension: allow cast to self.
-      Self.Diag(OpRange.getBegin(), diag::ext_typecheck_cast_nonscalar)
-        << DestType << SrcExpr.get()->getSourceRange();
-      Kind = CK_NoOp;
-      return;
-    }
-
-    // GCC's cast to union extension.
-    if (DestRecordTy && DestRecordTy->getDecl()->isUnion()) {
-      RecordDecl *RD = DestRecordTy->getDecl();
-      if (CastExpr::getTargetFieldForToUnionCast(RD, SrcType)) {
-        Self.Diag(OpRange.getBegin(), diag::ext_typecheck_cast_to_union)
-          << SrcExpr.get()->getSourceRange();
-        Kind = CK_ToUnion;
+    if (const RecordType *DestRecordTy = DestType->getAs<RecordType>()) {
+      if (Self.Context.hasSameUnqualifiedType(DestType, SrcType)) {
+        // GCC struct/union extension: allow cast to self.
+        Self.Diag(OpRange.getBegin(), diag::ext_typecheck_cast_nonscalar)
+            << DestType << SrcExpr.get()->getSourceRange();
+        Kind = CK_NoOp;
         return;
-      } else {
+      }
+
+      // GCC's cast to union extension.
+      if (RecordDecl *RD = DestRecordTy->getOriginalDecl(); RD->isUnion()) {
+        if (CastExpr::getTargetFieldForToUnionCast(RD->getDefinitionOrSelf(),
+                                                   SrcType)) {
+          Self.Diag(OpRange.getBegin(), diag::ext_typecheck_cast_to_union)
+              << SrcExpr.get()->getSourceRange();
+          Kind = CK_ToUnion;
+          return;
+        }
         Self.Diag(OpRange.getBegin(), diag::err_typecheck_cast_to_union_no_type)
-          << SrcType << SrcExpr.get()->getSourceRange();
+            << SrcType << SrcExpr.get()->getSourceRange();
         SrcExpr = ExprError();
         return;
       }
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 3c4511ba26330..6e777fb9aec8e 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -41,6 +41,7 @@
 #include "clang/AST/UnresolvedSet.h"
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Basic/LangOptions.h"
@@ -604,7 +605,7 @@ struct BuiltinDumpStructGenerator {
 
   bool dumpUnnamedRecord(const RecordDecl *RD, Expr *E, unsigned Depth) {
     Expr *IndentLit = getIndentString(Depth);
-    Expr *TypeLit = getTypeString(S.Context.getRecordType(RD));
+    Expr *TypeLit = getTypeString(S.Context.getCanonicalTagType(RD));
     if (IndentLit ? callPrintFunction("%s%s", {IndentLit, TypeLit})
                   : callPrintFunction("%s", {TypeLit}))
       return true;
@@ -2265,6 +2266,85 @@ static bool BuiltinCountZeroBitsGeneric(Sema &S, CallExpr *TheCall) {
   return false;
 }
 
+static bool CheckMaskedBuiltinArgs(Sema &S, Expr *MaskArg, Expr *PtrArg,
+                                   unsigned Pos) {
+  QualType MaskTy = MaskArg->getType();
+  if (!MaskTy->isExtVectorBoolType())
+    return S.Diag(MaskArg->getBeginLoc(), diag::err_builtin_invalid_arg_type)
+           << 1 << /* vector of */ 4 << /* booleans */ 6 << /* no fp */ 0
+           << MaskTy;
+
+  QualType PtrTy = PtrArg->getType();
+  if (!PtrTy->isPointerType() || !PtrTy->getPointeeType()->isVectorType())
+    return S.Diag(PtrArg->getExprLoc(), diag::err_vec_masked_load_store_ptr)
+           << Pos << "pointer to vector";
+  return false;
+}
+
+static ExprResult BuiltinMaskedLoad(Sema &S, CallExpr *TheCall) {
+  if (S.checkArgCount(TheCall, 2))
+    return ExprError();
+
+  Expr *MaskArg = TheCall->getArg(0);
+  Expr *PtrArg = TheCall->getArg(1);
+  if (CheckMaskedBuiltinArgs(S, MaskArg, PtrArg, 2))
+    return ExprError();
+
+  QualType MaskTy = MaskArg->getType();
+  QualType PtrTy = PtrArg->getType();
+  QualType PointeeTy = PtrTy->getPointeeType();
+  const VectorType *MaskVecTy = MaskTy->getAs<VectorType>();
+  const VectorType *DataVecTy = PointeeTy->getAs<VectorType>();
+  if (MaskVecTy->getNumElements() != DataVecTy->getNumElements())
+    return ExprError(
+        S.Diag(TheCall->getBeginLoc(), diag::err_vec_masked_load_store_size)
+        << "__builtin_masked_load" << MaskTy << PointeeTy);
+
+  TheCall->setType(PointeeTy);
+  return TheCall;
+}
+
+static ExprResult BuiltinMaskedStore(Sema &S, CallExpr *TheCall) {
+  if (S.checkArgCount(TheCall, 3))
+    return ExprError();
+
+  Expr *MaskArg = TheCall->getArg(0);
+  Expr *ValArg = TheCall->getArg(1);
+  Expr *PtrArg = TheCall->getArg(2);
+
+  if (CheckMaskedBuiltinArgs(S, MaskArg, PtrArg, 3))
+    return ExprError();
+
+  QualType MaskTy = MaskArg->getType();
+  QualType PtrTy = PtrArg->getType();
+  QualType ValTy = ValArg->getType();
+  if (!ValTy->isVectorType())
+    return ExprError(
+        S.Diag(ValArg->getExprLoc(), diag::err_vec_masked_load_store_ptr)
+        << 2 << "vector");
+
+  QualType PointeeTy = PtrTy->getPointeeType();
+  const VectorType *MaskVecTy = MaskTy->getAs<VectorType>();
+  const VectorType *ValVecTy = ValTy->getAs<VectorType>();
+  const VectorType *PtrVecTy = PointeeTy->getAs<VectorType>();
+
+  if (MaskVecTy->getNumElements() != ValVecTy->getNumElements() ||
+      MaskVecTy->getNumElements() != PtrVecTy->getNumElements())
+    return ExprError(
+        S.Diag(TheCall->getBeginLoc(), diag::err_vec_masked_load_store_size)
+        << "__builtin_masked_store" << MaskTy << PointeeTy);
+
+  if (!S.Context.hasSameType(ValTy, PointeeTy))
+    return ExprError(S.Diag(TheCall->getBeginLoc(),
+                            diag::err_vec_builtin_incompatible_vector)
+                     << TheCall->getDirectCallee() << /*isMorethantwoArgs*/ 2
+                     << SourceRange(TheCall->getArg(1)->getBeginLoc(),
+                                    TheCall->getArg(1)->getEndLoc()));
+
+  TheCall->setType(S.Context.VoidTy);
+  return TheCall;
+}
+
 static ExprResult BuiltinInvoke(Sema &S, CallExpr *TheCall) {
   SourceLocation Loc = TheCall->getBeginLoc();
   MutableArrayRef Args(TheCall->getArgs(), TheCall->getNumArgs());
@@ -2289,7 +2369,7 @@ static ExprResult BuiltinInvoke(Sema &S, CallExpr *TheCall) {
       return ExprError();
     }
 
-    const Type *MemPtrClass = MPT->getQualifier()->getAsType();
+    const Type *MemPtrClass = MPT->getQualifier().getAsType();
     QualType ObjectT = Args[1]->getType();
 
     if (MPT->isMemberDataPointer() && S.checkArgCount(TheCall, 2))
@@ -2517,6 +2597,10 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     return BuiltinShuffleVector(TheCall);
     // TheCall will be freed by the smart pointer here, but that's fine, since
     // BuiltinShuffleVector guts it, but then doesn't release it.
+  case Builtin::BI__builtin_masked_load:
+    return BuiltinMaskedLoad(*this, TheCall);
+  case Builtin::BI__builtin_masked_store:
+    return BuiltinMaskedStore(*this, TheCall);
   case Builtin::BI__builtin_invoke:
     return BuiltinInvoke(*this, TheCall);
   case Builtin::BI__builtin_prefetch:
@@ -3031,6 +3115,12 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
                                EltwiseBuiltinArgTyRestriction::IntegerTy))
       return ExprError();
     break;
+  case Builtin::BI__builtin_elementwise_fshl:
+  case Builtin::BI__builtin_elementwise_fshr:
+    if (BuiltinElementwiseTernaryMath(
+            TheCall, EltwiseBuiltinArgTyRestriction::IntegerTy))
+      return ExprError();
+    break;
   case Builtin::BI__builtin_elementwise_min:
   case Builtin::BI__builtin_elementwise_max:
     if (BuiltinElementwiseMath(TheCall))
@@ -3073,6 +3163,19 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
     TheCall->setType(Magnitude.get()->getType());
     break;
   }
+  case Builtin::BI__builtin_elementwise_ctlz:
+  case Builtin::BI__builtin_elementwise_cttz:
+    // These builtins can be unary or binary. Note for empty calls we call the
+    // unary checker in order to not emit an error that says the function
+    // expects 2 arguments, which would be misleading.
+    if (TheCall->getNumArgs() <= 1) {
+      if (PrepareBuiltinElementwiseMathOneArgCall(
+              TheCall, EltwiseBuiltinArgTyRestriction::IntegerTy))
+        return ExprError();
+    } else if (BuiltinElementwiseMath(
+                   TheCall, EltwiseBuiltinArgTyRestriction::IntegerTy))
+      return ExprError();
+    break;
   case Builtin::BI__builtin_reduce_max:
   case Builtin::BI__builtin_reduce_min: {
     if (PrepareBuiltinReduceMathOneArgCall(TheCall))
@@ -3318,7 +3421,9 @@ static bool CheckNonNullExpr(Sema &S, const Expr *Expr) {
   // As a special case, transparent unions initialized with zero are
   // considered null for the purposes of the nonnull attribute.
   if (const RecordType *UT = Expr->getType()->getAsUnionType();
-      UT && UT->getDecl()->hasAttr<TransparentUnionAttr>()) {
+      UT && UT->getOriginalDecl()
+                ->getMostRecentDecl()
+                ->hasAttr<TransparentUnionAttr>()) {
     if (const auto *CLE = dyn_cast<CompoundLiteralExpr>(Expr))
       if (const auto *ILE = dyn_cast<InitListExpr>(CLE->getInitializer()))
         Expr = ILE->getInit(0);
@@ -5175,7 +5280,9 @@ bool Sema::BuiltinVAStart(unsigned BuiltinID, CallExpr *TheCall) {
                return false;
              if (!Type->isEnumeralType())
                return true;
-             const EnumDecl *ED = Type->castAs<EnumType>()->getDecl();
+             const EnumDecl *ED = Type->castAs<EnumType>()
+                                      ->getOriginalDecl()
+                                      ->getDefinitionOrSelf();
              return !(ED &&
                       Context.typesAreCompatible(ED->getPromotionType(), Type));
            }()) {
@@ -5494,8 +5601,10 @@ ExprResult Sema::BuiltinShuffleVector(CallExpr *TheCall) {
                                       TheCall->getArg(1)->getEndLoc()));
     } else if (numElements != numResElements) {
       QualType eltType = LHSType->castAs<VectorType>()->getElementType();
-      resType =
-          Context.getVectorType(eltType, numResElements, VectorKind::Generic);
+      resType = resType->isExtVectorType()
+                    ? Context.getExtVectorType(eltType, numResElements)
+                    : Context.getVectorType(eltType, numResElements,
+                                            VectorKind::Generic);
     }
   }
 
@@ -7646,17 +7755,13 @@ bool EquatableFormatArgument::VerifyCompatible(
     break;
 
   case MK::NoMatchSignedness:
-    if (!S.getDiagnostics().isIgnored(
-            diag::warn_format_conversion_argument_type_mismatch_signedness,
-            ElementLoc)) {
-      EmitDiagnostic(S,
-                     S.PDiag(diag::warn_format_cmp_specifier_sign_mismatch)
-                         << buildFormatSpecifier()
-                         << Other.buildFormatSpecifier(),
-                     FmtExpr, InFunctionCall);
-      HadError = S.Diag(Other.ElementLoc, diag::note_format_cmp_with)
-                 << 0 << Other.Range;
-    }
+    EmitDiagnostic(S,
+                   S.PDiag(diag::warn_format_cmp_specifier_sign_mismatch)
+                       << buildFormatSpecifier()
+                       << Other.buildFormatSpecifier(),
+                   FmtExpr, InFunctionCall);
+    HadError = S.Diag(Other.ElementLoc, diag::note_format_cmp_with)
+               << 0 << Other.Range;
     break;
   }
   return !HadError;
@@ -7791,8 +7896,11 @@ CXXRecordMembersNamed(StringRef Name, Sema &S, QualType Ty) {
 
   if (!RT)
     return Results;
-  const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
-  if (!RD || !RD->getDefinition())
+  CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl());
+  if (!RD)
+    return Results;
+  RD = RD->getDefinition();
+  if (!RD)
     return Results;
 
   LookupResult R(S, &S.Context.Idents.get(Name), SourceLocation(),
@@ -7801,7 +7909,7 @@ CXXRecordMembersNamed(StringRef Name, Sema &S, QualType Ty) {
 
   // We just need to include all members of the right kind turned up by the
   // filter, at this point.
-  if (S.LookupQualifiedName(R, RT->getDecl()))
+  if (S.LookupQualifiedName(R, RD))
     for (LookupResult::iterator I = R.begin(), E = R.end(); I != E; ++I) {
       NamedDecl *decl = (*I)->getUnderlyingDecl();
       if (MemberKind *FK = dyn_cast<MemberKind>(decl))
@@ -8190,11 +8298,14 @@ static analyze_format_string::ArgType::MatchKind
 handleFormatSignedness(analyze_format_string::ArgType::MatchKind Match,
                        DiagnosticsEngine &Diags, SourceLocation Loc) {
   if (Match == analyze_format_string::ArgType::NoMatchSignedness) {
-    Match =
+    if (Diags.isIgnored(
+            diag::warn_format_conversion_argument_type_mismatch_signedness,
+            Loc) ||
         Diags.isIgnored(
-            diag::warn_format_conversion_argument_type_mismatch_signedness, Loc)
-            ? analyze_format_string::ArgType::Match
-            : analyze_format_string::ArgType::NoMatch;
+            // Arbitrary -Wformat diagnostic to detect -Wno-format:
+            diag::warn_format_conversion_argument_type_mismatch, Loc)) {
+      return analyze_format_string::ArgType::Match;
+    }
   }
   return Match;
 }
@@ -8328,7 +8439,8 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS,
   bool IsScopedEnum = false;
   QualType IntendedTy = ExprTy;
   if (auto EnumTy = ExprTy->getAs<EnumType>()) {
-    IntendedTy = EnumTy->getDecl()->getIntegerType();
+    IntendedTy =
+        EnumTy->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
     if (EnumTy->isUnscopedEnumerationType()) {
       ExprTy = IntendedTy;
       // This controls whether we're talking about the underlying type or not,
@@ -8364,7 +8476,9 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS,
         NamedDecl *ND = Result.getFoundDecl();
         if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(ND))
           if (TD->getUnderlyingType() == IntendedTy)
-            IntendedTy = S.Context.getTypedefType(TD);
+            IntendedTy =
+                S.Context.getTypedefType(ElaboratedTypeKeyword::None,
+                                         /*Qualifier=*/std::nullopt, TD);
       }
     }
   }
@@ -8408,8 +8522,10 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS,
       case ArgType::Match:
       case ArgType::MatchPromotion:
       case ArgType::NoMatchPromotionTypeConfusion:
-      case ArgType::NoMatchSignedness:
         llvm_unreachable("expected non-matching");
+      case ArgType::NoMatchSignedness:
+        Diag = diag::warn_format_conversion_argument_type_mismatch_signedness;
+        break;
       case ArgType::NoMatchPedantic:
         Diag = diag::warn_format_conversion_argument_type_mismatch_pedantic;
         break;
@@ -8734,9 +8850,10 @@ bool CheckScanfHandler::HandleScanfSpecifier(
   analyze_format_string::ArgType::MatchKind Match =
       AT.matchesType(S.Context, Ex->getType());
   Match = handleFormatSignedness(Match, S.getDiagnostics(), Ex->getExprLoc());
-  bool Pedantic = Match == analyze_format_string::ArgType::NoMatchPedantic;
   if (Match == analyze_format_string::ArgType::Match)
     return true;
+  bool Pedantic = Match == analyze_format_string::ArgType::NoMatchPedantic;
+  bool Signedness = Match == analyze_format_string::ArgType::NoMatchSignedness;
 
   ScanfSpecifier fixedFS = FS;
   bool Success = fixedFS.fixType(Ex->getType(), Ex->IgnoreImpCasts()->getType(),
@@ -8744,7 +8861,9 @@ bool CheckScanfHandler::HandleScanfSpecifier(
 
   unsigned Diag =
       Pedantic ? diag::warn_format_conversion_argument_type_mismatch_pedantic
-               : diag::warn_format_conversion_argument_type_mismatch;
+      : Signedness
+          ? diag::warn_format_conversion_argument_type_mismatch_signedness
+          : diag::warn_format_conversion_argument_type_mismatch;
 
   if (Success) {
     // Get the fix string from the fixed format specifier.
@@ -9622,7 +9741,10 @@ struct SearchNonTrivialToInitializeField
     S.DiagRuntimeBehavior(SL, E, S.PDiag(diag::note_nontrivial_field) << 1);
   }
   void visitStruct(QualType FT, SourceLocation SL) {
-    for (const FieldDecl *FD : FT->castAs<RecordType>()->getDecl()->fields())
+    for (const FieldDecl *FD : FT->castAs<RecordType>()
+                                   ->getOriginalDecl()
+                                   ->getDefinitionOrSelf()
+                                   ->fields())
       visit(FD->getType(), FD->getLocation());
   }
   void visitArray(QualType::PrimitiveDefaultInitializeKind PDIK,
@@ -9667,7 +9789,10 @@ struct SearchNonTrivialToCopyField
     S.DiagRuntimeBehavior(SL, E, S.PDiag(diag::note_nontrivial_field) << 0);
   }
   void visitStruct(QualType FT, SourceLocation SL) {
-    for (const FieldDecl *FD : FT->castAs<RecordType>()->getDecl()->fields())
+    for (const FieldDecl *FD : FT->castAs<RecordType>()
+                                   ->getOriginalDecl()
+                                   ->getDefinitionOrSelf()
+                                   ->fields())
       visit(FD->getType(), FD->getLocation());
   }
   void visitArray(QualType::PrimitiveCopyKind PCK, const ArrayType *AT,
@@ -9948,8 +10073,9 @@ void Sema::CheckMemaccessArguments(const CallExpr *Call,
           getLangOpts().CPlusPlus && !RT->isIncompleteType() &&
           !RT->desugar().isTriviallyCopyableType(Context);
 
+      const auto *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       if ((BId == Builtin::BImemset || BId == Builtin::BIbzero) &&
-          RT->getDecl()->isNonTrivialToPrimitiveDefaultInitialize()) {
+          RD->isNonTrivialToPrimitiveDefaultInitialize()) {
         DiagRuntimeBehavior(Dest->getExprLoc(), Dest,
                             PDiag(diag::warn_cstruct_memaccess)
                                 << ArgIdx << FnName << PointeeTy << 0);
@@ -9962,7 +10088,7 @@ void Sema::CheckMemaccessArguments(const CallExpr *Call,
                             PDiag(diag::warn_cxxstruct_memaccess)
                                 << FnName << PointeeTy);
       } else if ((BId == Builtin::BImemcpy || BId == Builtin::BImemmove) &&
-                 RT->getDecl()->isNonTrivialToPrimitiveCopy()) {
+                 RD->isNonTrivialToPrimitiveCopy()) {
         DiagRuntimeBehavior(Dest->getExprLoc(), Dest,
                             PDiag(diag::warn_cstruct_memaccess)
                                 << ArgIdx << FnName << PointeeTy << 1);
@@ -10468,10 +10594,14 @@ struct IntRange {
     if (!C.getLangOpts().CPlusPlus) {
       // For enum types in C code, use the underlying datatype.
       if (const auto *ET = dyn_cast<EnumType>(T))
-        T = ET->getDecl()->getIntegerType().getDesugaredType(C).getTypePtr();
+        T = ET->getOriginalDecl()
+                ->getDefinitionOrSelf()
+                ->getIntegerType()
+                .getDesugaredType(C)
+                .getTypePtr();
     } else if (const auto *ET = dyn_cast<EnumType>(T)) {
       // For enum types in C++, use the known bit width of the enumerators.
-      EnumDecl *Enum = ET->getDecl();
+      EnumDecl *Enum = ET->getOriginalDecl()->getDefinitionOrSelf();
       // In C++11, enums can have a fixed underlying type. Use this type to
       // compute the range.
       if (Enum->isFixed()) {
@@ -10513,7 +10643,9 @@ struct IntRange {
     if (const AtomicType *AT = dyn_cast<AtomicType>(T))
       T = AT->getValueType().getTypePtr();
     if (const EnumType *ET = dyn_cast<EnumType>(T))
-      T = C.getCanonicalType(ET->getDecl()->getIntegerType()).getTypePtr();
+      T = C.getCanonicalType(
+               ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType())
+              .getTypePtr();
 
     if (const auto *EIT = dyn_cast<BitIntType>(T))
       return IntRange(EIT->getNumBits(), EIT->isUnsigned());
@@ -11485,7 +11617,9 @@ static bool AnalyzeBitFieldAssignment(Sema &S, FieldDecl *Bitfield, Expr *Init,
      return false;
 
   if (BitfieldType->isEnumeralType()) {
-    EnumDecl *BitfieldEnumDecl = BitfieldType->castAs<EnumType>()->getDecl();
+    EnumDecl *BitfieldEnumDecl = BitfieldType->castAs<EnumType>()
+                                     ->getOriginalDecl()
+                                     ->getDefinitionOrSelf();
     // If the underlying enum type was not explicitly specified as an unsigned
     // type and the enum contain only positive values, MSVC++ will cause an
     // inconsistency by storing this as a signed type.
@@ -11522,7 +11656,7 @@ static bool AnalyzeBitFieldAssignment(Sema &S, FieldDecl *Bitfield, Expr *Init,
         EnumTy = PTAttr->getType()->getAs<EnumType>();
     }
     if (EnumTy) {
-      EnumDecl *ED = EnumTy->getDecl();
+      EnumDecl *ED = EnumTy->getOriginalDecl()->getDefinitionOrSelf();
       bool SignedBitfield = BitfieldType->isSignedIntegerOrEnumerationType();
 
       // Enum types are implicitly signed on Windows, so check if there are any
@@ -12588,8 +12722,8 @@ void Sema::CheckImplicitConversion(Expr *E, QualType T, SourceLocation CC,
 
   if (const EnumType *SourceEnum = Source->getAs<EnumType>())
     if (const EnumType *TargetEnum = Target->getAs<EnumType>())
-      if (SourceEnum->getDecl()->hasNameForLinkage() &&
-          TargetEnum->getDecl()->hasNameForLinkage() &&
+      if (SourceEnum->getOriginalDecl()->hasNameForLinkage() &&
+          TargetEnum->getOriginalDecl()->hasNameForLinkage() &&
           SourceEnum != TargetEnum) {
         if (SourceMgr.isInSystemMacro(CC))
           return;
@@ -14089,7 +14223,6 @@ void Sema::CheckCompletedExpr(Expr *E, SourceLocation CheckLoc,
     CheckUnsequencedOperations(E);
   if (!IsConstexpr && !E->isValueDependent())
     CheckForIntOverflow(E);
-  DiagnoseMisalignedMembers();
 }
 
 void Sema::CheckBitFieldInitialization(SourceLocation InitLoc,
@@ -15113,10 +15246,9 @@ static bool isLayoutCompatible(const ASTContext &C, const EnumDecl *ED1,
 static bool isLayoutCompatible(const ASTContext &C, const FieldDecl *Field1,
                                const FieldDecl *Field2,
                                bool AreUnionMembers = false) {
-  [[maybe_unused]] const Type *Field1Parent =
-      Field1->getParent()->getTypeForDecl();
-  [[maybe_unused]] const Type *Field2Parent =
-      Field2->getParent()->getTypeForDecl();
+#ifndef NDEBUG
+  CanQualType Field1Parent = C.getCanonicalTagType(Field1->getParent());
+  CanQualType Field2Parent = C.getCanonicalTagType(Field2->getParent());
   assert(((Field1Parent->isStructureOrClassType() &&
            Field2Parent->isStructureOrClassType()) ||
           (Field1Parent->isUnionType() && Field2Parent->isUnionType())) &&
@@ -15125,6 +15257,7 @@ static bool isLayoutCompatible(const ASTContext &C, const FieldDecl *Field1,
   assert(((!AreUnionMembers && Field1Parent->isStructureOrClassType()) ||
           (AreUnionMembers && Field1Parent->isUnionType())) &&
          "AreUnionMembers should be 'true' for union fields (only).");
+#endif
 
   if (!isLayoutCompatible(C, Field1->getType(), Field2->getType()))
     return false;
@@ -15229,16 +15362,16 @@ static bool isLayoutCompatible(const ASTContext &C, QualType T1, QualType T2) {
     return false;
 
   if (TC1 == Type::Enum) {
-    return isLayoutCompatible(C,
-                              cast<EnumType>(T1)->getDecl(),
-                              cast<EnumType>(T2)->getDecl());
+    return isLayoutCompatible(
+        C, cast<EnumType>(T1)->getOriginalDecl()->getDefinitionOrSelf(),
+        cast<EnumType>(T2)->getOriginalDecl()->getDefinitionOrSelf());
   } else if (TC1 == Type::Record) {
     if (!T1->isStandardLayoutType() || !T2->isStandardLayoutType())
       return false;
 
-    return isLayoutCompatible(C,
-                              cast<RecordType>(T1)->getDecl(),
-                              cast<RecordType>(T2)->getDecl());
+    return isLayoutCompatible(
+        C, cast<RecordType>(T1)->getOriginalDecl()->getDefinitionOrSelf(),
+        cast<RecordType>(T2)->getOriginalDecl()->getDefinitionOrSelf());
   }
 
   return false;
@@ -15534,11 +15667,12 @@ void Sema::CheckArgumentWithTypeTag(const ArgumentWithTypeTagAttr *Attr,
 
 void Sema::AddPotentialMisalignedMembers(Expr *E, RecordDecl *RD, ValueDecl *MD,
                                          CharUnits Alignment) {
-  MisalignedMembers.emplace_back(E, RD, MD, Alignment);
+  currentEvaluationContext().MisalignedMembers.emplace_back(E, RD, MD,
+                                                            Alignment);
 }
 
 void Sema::DiagnoseMisalignedMembers() {
-  for (MisalignedMember &m : MisalignedMembers) {
+  for (MisalignedMember &m : currentEvaluationContext().MisalignedMembers) {
     const NamedDecl *ND = m.RD;
     if (ND->getName().empty()) {
       if (const TypedefNameDecl *TD = m.RD->getTypedefNameForAnonDecl())
@@ -15547,7 +15681,7 @@ void Sema::DiagnoseMisalignedMembers() {
     Diag(m.E->getBeginLoc(), diag::warn_taking_address_of_packed_member)
         << m.MD << ND << m.E->getSourceRange();
   }
-  MisalignedMembers.clear();
+  currentEvaluationContext().MisalignedMembers.clear();
 }
 
 void Sema::DiscardMisalignedMemberAddress(const Type *T, Expr *E) {
@@ -15558,13 +15692,15 @@ void Sema::DiscardMisalignedMemberAddress(const Type *T, Expr *E) {
       cast<UnaryOperator>(E)->getOpcode() == UO_AddrOf) {
     auto *Op = cast<UnaryOperator>(E)->getSubExpr()->IgnoreParens();
     if (isa<MemberExpr>(Op)) {
-      auto *MA = llvm::find(MisalignedMembers, MisalignedMember(Op));
-      if (MA != MisalignedMembers.end() &&
+      auto &MisalignedMembersForExpr =
+          currentEvaluationContext().MisalignedMembers;
+      auto *MA = llvm::find(MisalignedMembersForExpr, MisalignedMember(Op));
+      if (MA != MisalignedMembersForExpr.end() &&
           (T->isDependentType() || T->isIntegerType() ||
            (T->isPointerType() && (T->getPointeeType()->isIncompleteType() ||
                                    Context.getTypeAlignInChars(
                                        T->getPointeeType()) <= MA->Alignment))))
-        MisalignedMembers.erase(MA);
+        MisalignedMembersForExpr.erase(MA);
     }
   }
 }
@@ -15592,7 +15728,9 @@ void Sema::RefersToMemberWithReducedAlignment(
       return;
     if (ME->isArrow())
       BaseType = BaseType->getPointeeType();
-    RecordDecl *RD = BaseType->castAs<RecordType>()->getDecl();
+    RecordDecl *RD = BaseType->castAs<RecordType>()
+                         ->getOriginalDecl()
+                         ->getDefinitionOrSelf();
     if (RD->isInvalidDecl())
       return;
 
@@ -15637,7 +15775,7 @@ void Sema::RefersToMemberWithReducedAlignment(
 
   // Compute the CompleteObjectAlignment as the alignment of the whole chain.
   CharUnits CompleteObjectAlignment = Context.getTypeAlignInChars(
-      ReverseMemberChain.back()->getParent()->getTypeForDecl());
+      Context.getCanonicalTagType(ReverseMemberChain.back()->getParent()));
 
   // The base expression of the innermost MemberExpr may give
   // stronger guarantees than the class containing the member.
@@ -15667,9 +15805,9 @@ void Sema::RefersToMemberWithReducedAlignment(
       if (FDI->hasAttr<PackedAttr>() ||
           FDI->getParent()->hasAttr<PackedAttr>()) {
         FD = FDI;
-        Alignment = std::min(
-            Context.getTypeAlignInChars(FD->getType()),
-            Context.getTypeAlignInChars(FD->getParent()->getTypeForDecl()));
+        Alignment = std::min(Context.getTypeAlignInChars(FD->getType()),
+                             Context.getTypeAlignInChars(
+                                 Context.getCanonicalTagType(FD->getParent())));
         break;
       }
     }
@@ -15817,6 +15955,7 @@ bool Sema::BuiltinElementwiseTernaryMath(
       return true;
   }
 
+  TheCall->setArg(0, Args[0]);
   for (int I = 1; I < 3; ++I) {
     if (Args[0]->getType().getCanonicalType() !=
         Args[I]->getType().getCanonicalType()) {
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 0de55800ccdd1..c6adead5a65f5 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -197,6 +197,9 @@ class ResultBuilder {
   /// Whether the \p ObjectTypeQualifiers field is active.
   bool HasObjectTypeQualifiers;
 
+  // Whether the member function is using an explicit object parameter
+  bool IsExplicitObjectMemberFunction;
+
   /// The selector that we prefer.
   Selector PreferredSelector;
 
@@ -218,8 +221,8 @@ class ResultBuilder {
                          LookupFilter Filter = nullptr)
       : SemaRef(SemaRef), Allocator(Allocator), CCTUInfo(CCTUInfo),
         Filter(Filter), AllowNestedNameSpecifiers(false),
-        HasObjectTypeQualifiers(false), CompletionContext(CompletionContext),
-        ObjCImplementation(nullptr) {
+        HasObjectTypeQualifiers(false), IsExplicitObjectMemberFunction(false),
+        CompletionContext(CompletionContext), ObjCImplementation(nullptr) {
     // If this is an Objective-C instance method definition, dig out the
     // corresponding implementation.
     switch (CompletionContext.getKind()) {
@@ -275,6 +278,10 @@ class ResultBuilder {
     HasObjectTypeQualifiers = true;
   }
 
+  void setExplicitObjectMemberFn(bool IsExplicitObjectFn) {
+    IsExplicitObjectMemberFunction = IsExplicitObjectFn;
+  }
+
   /// Set the preferred selector.
   ///
   /// When an Objective-C method declaration result is added, and that
@@ -732,7 +739,7 @@ ResultBuilder::ShadowMapEntry::end() const {
 ///
 /// \returns a nested name specifier that refers into the target context, or
 /// NULL if no qualification is needed.
-static NestedNameSpecifier *
+static NestedNameSpecifier
 getRequiredQualification(ASTContext &Context, const DeclContext *CurContext,
                          const DeclContext *TargetContext) {
   SmallVector<const DeclContext *, 4> TargetParents;
@@ -747,7 +754,7 @@ getRequiredQualification(ASTContext &Context, const DeclContext *CurContext,
     TargetParents.push_back(CommonAncestor);
   }
 
-  NestedNameSpecifier *Result = nullptr;
+  NestedNameSpecifier Result = std::nullopt;
   while (!TargetParents.empty()) {
     const DeclContext *Parent = TargetParents.pop_back_val();
 
@@ -755,10 +762,12 @@ getRequiredQualification(ASTContext &Context, const DeclContext *CurContext,
       if (!Namespace->getIdentifier())
         continue;
 
-      Result = NestedNameSpecifier::Create(Context, Result, Namespace);
-    } else if (const auto *TD = dyn_cast<TagDecl>(Parent))
-      Result = NestedNameSpecifier::Create(
-          Context, Result, Context.getTypeDeclType(TD).getTypePtr());
+      Result = NestedNameSpecifier(Context, Namespace, Result);
+    } else if (const auto *TD = dyn_cast<TagDecl>(Parent)) {
+      QualType TT = Context.getTagType(ElaboratedTypeKeyword::None, Result, TD,
+                                       /*OwnsTag=*/false);
+      Result = NestedNameSpecifier(TT.getTypePtr());
+    }
   }
   return Result;
 }
@@ -937,11 +946,12 @@ SimplifiedTypeClass clang::getSimplifiedTypeClass(CanQualType T) {
 
 /// Get the type that a given expression will have if this declaration
 /// is used as an expression in its "typical" code-completion form.
-QualType clang::getDeclUsageType(ASTContext &C, const NamedDecl *ND) {
+QualType clang::getDeclUsageType(ASTContext &C, NestedNameSpecifier Qualifier,
+                                 const NamedDecl *ND) {
   ND = ND->getUnderlyingDecl();
 
   if (const auto *Type = dyn_cast<TypeDecl>(ND))
-    return C.getTypeDeclType(Type);
+    return C.getTypeDeclType(ElaboratedTypeKeyword::None, Qualifier, Type);
   if (const auto *Iface = dyn_cast<ObjCInterfaceDecl>(ND))
     return C.getObjCInterfaceType(Iface);
 
@@ -951,7 +961,9 @@ QualType clang::getDeclUsageType(ASTContext &C, const NamedDecl *ND) {
   else if (const auto *Method = dyn_cast<ObjCMethodDecl>(ND))
     T = Method->getSendResultType();
   else if (const auto *Enumerator = dyn_cast<EnumConstantDecl>(ND))
-    T = C.getTypeDeclType(cast<EnumDecl>(Enumerator->getDeclContext()));
+    T = C.getTagType(ElaboratedTypeKeyword::None, Qualifier,
+                     cast<EnumDecl>(Enumerator->getDeclContext()),
+                     /*OwnsTag=*/false);
   else if (const auto *Property = dyn_cast<ObjCPropertyDecl>(ND))
     T = Property->getType();
   else if (const auto *Value = dyn_cast<ValueDecl>(ND))
@@ -1053,7 +1065,7 @@ void ResultBuilder::AdjustResultPriorityForDecl(Result &R) {
   // If we have a preferred type, adjust the priority for results with exactly-
   // matching or nearly-matching types.
   if (!PreferredType.isNull()) {
-    QualType T = getDeclUsageType(SemaRef.Context, R.Declaration);
+    QualType T = getDeclUsageType(SemaRef.Context, R.Qualifier, R.Declaration);
     if (!T.isNull()) {
       CanQualType TC = SemaRef.Context.getCanonicalType(T);
       // Check for exactly-matching types (modulo qualifiers).
@@ -1070,10 +1082,9 @@ void ResultBuilder::AdjustResultPriorityForDecl(Result &R) {
 
 static DeclContext::lookup_result getConstructors(ASTContext &Context,
                                                   const CXXRecordDecl *Record) {
-  QualType RecordTy = Context.getTypeDeclType(Record);
+  CanQualType RecordTy = Context.getCanonicalTagType(Record);
   DeclarationName ConstructorName =
-      Context.DeclarationNames.getCXXConstructorName(
-          Context.getCanonicalType(RecordTy));
+      Context.DeclarationNames.getCXXConstructorName(RecordTy);
   return Record->lookup(ConstructorName);
 }
 
@@ -1216,11 +1227,13 @@ void ResultBuilder::MaybeAddResult(Result R, DeclContext *CurContext) {
     const DeclContext *Ctx = R.Declaration->getDeclContext();
     if (const NamespaceDecl *Namespace = dyn_cast<NamespaceDecl>(Ctx))
       R.Qualifier =
-          NestedNameSpecifier::Create(SemaRef.Context, nullptr, Namespace);
+          NestedNameSpecifier(SemaRef.Context, Namespace, std::nullopt);
     else if (const TagDecl *Tag = dyn_cast<TagDecl>(Ctx))
-      R.Qualifier = NestedNameSpecifier::Create(
-          SemaRef.Context, nullptr,
-          SemaRef.Context.getTypeDeclType(Tag).getTypePtr());
+      R.Qualifier = NestedNameSpecifier(
+          SemaRef.Context
+              .getTagType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt, Tag, /*OwnsTag=*/false)
+              .getTypePtr());
     else
       R.QualifierIsInformative = false;
   }
@@ -1405,11 +1418,13 @@ void ResultBuilder::AddResult(Result R, DeclContext *CurContext,
     const DeclContext *Ctx = R.Declaration->getDeclContext();
     if (const auto *Namespace = dyn_cast<NamespaceDecl>(Ctx))
       R.Qualifier =
-          NestedNameSpecifier::Create(SemaRef.Context, nullptr, Namespace);
+          NestedNameSpecifier(SemaRef.Context, Namespace, std::nullopt);
     else if (const auto *Tag = dyn_cast<TagDecl>(Ctx))
-      R.Qualifier = NestedNameSpecifier::Create(
-          SemaRef.Context, nullptr,
-          SemaRef.Context.getTypeDeclType(Tag).getTypePtr());
+      R.Qualifier = NestedNameSpecifier(
+          SemaRef.Context
+              .getTagType(ElaboratedTypeKeyword::None,
+                          /*Qualifier=*/std::nullopt, Tag, /*OwnsTag=*/false)
+              .getTypePtr());
     else
       R.QualifierIsInformative = false;
   }
@@ -1420,10 +1435,27 @@ void ResultBuilder::AddResult(Result R, DeclContext *CurContext,
 
   AdjustResultPriorityForDecl(R);
 
+  // Account for explicit object parameter
+  const auto GetQualifiers = [&](const CXXMethodDecl *MethodDecl) {
+    if (MethodDecl->isExplicitObjectMemberFunction())
+      return MethodDecl->getFunctionObjectParameterType().getQualifiers();
+    else
+      return MethodDecl->getMethodQualifiers();
+  };
+
+  if (IsExplicitObjectMemberFunction &&
+      R.Kind == CodeCompletionResult::RK_Declaration &&
+      (isa<CXXMethodDecl>(R.Declaration) || isa<FieldDecl>(R.Declaration))) {
+    // If result is a member in the context of an explicit-object member
+    // function, drop it because it must be accessed through the object
+    // parameter
+    return;
+  }
+
   if (HasObjectTypeQualifiers)
     if (const auto *Method = dyn_cast<CXXMethodDecl>(R.Declaration))
       if (Method->isInstance()) {
-        Qualifiers MethodQuals = Method->getMethodQualifiers();
+        Qualifiers MethodQuals = GetQualifiers(Method);
         if (ObjectTypeQualifiers == MethodQuals)
           R.Priority += CCD_ObjectQualifierMatch;
         else if (ObjectTypeQualifiers - MethodQuals) {
@@ -1664,7 +1696,8 @@ static bool isObjCReceiverType(ASTContext &C, QualType T) {
 }
 
 bool ResultBuilder::IsObjCMessageReceiver(const NamedDecl *ND) const {
-  QualType T = getDeclUsageType(SemaRef.Context, ND);
+  QualType T =
+      getDeclUsageType(SemaRef.Context, /*Qualifier=*/std::nullopt, ND);
   if (T.isNull())
     return false;
 
@@ -1689,7 +1722,8 @@ bool ResultBuilder::IsObjCCollection(const NamedDecl *ND) const {
       (!SemaRef.getLangOpts().CPlusPlus && !IsOrdinaryNonTypeName(ND)))
     return false;
 
-  QualType T = getDeclUsageType(SemaRef.Context, ND);
+  QualType T =
+      getDeclUsageType(SemaRef.Context, /*Qualifier=*/std::nullopt, ND);
   if (T.isNull())
     return false;
 
@@ -1745,8 +1779,10 @@ class CodeCompletionDeclConsumer : public VisibleDeclConsumer {
 
   void FoundDecl(NamedDecl *ND, NamedDecl *Hiding, DeclContext *Ctx,
                  bool InBaseClass) override {
-    ResultBuilder::Result Result(ND, Results.getBasePriority(ND), nullptr,
-                                 false, IsAccessible(ND, Ctx), FixIts);
+    ResultBuilder::Result Result(ND, Results.getBasePriority(ND),
+                                 /*Qualifier=*/std::nullopt,
+                                 /*QualifierIsInformative=*/false,
+                                 IsAccessible(ND, Ctx), FixIts);
     Results.AddResult(Result, InitialLookupCtx, Hiding, InBaseClass, BaseType);
   }
 
@@ -2010,7 +2046,6 @@ static PrintingPolicy getCompletionPrintingPolicy(const ASTContext &Context,
   Policy.AnonymousTagLocations = false;
   Policy.SuppressStrongLifetime = true;
   Policy.SuppressUnwrittenScope = true;
-  Policy.SuppressScope = true;
   Policy.CleanUglifiedParameters = true;
   return Policy;
 }
@@ -2035,7 +2070,7 @@ static const char *GetCompletionTypeString(QualType T, ASTContext &Context,
 
     // Anonymous tag types are constant strings.
     if (const TagType *TagT = dyn_cast<TagType>(T))
-      if (TagDecl *Tag = TagT->getDecl())
+      if (TagDecl *Tag = TagT->getOriginalDecl())
         if (!Tag->hasNameForLinkage()) {
           switch (Tag->getTagKind()) {
           case TagTypeKind::Struct:
@@ -2925,8 +2960,8 @@ static void AddResultTypeChunk(ASTContext &Context,
     else
       T = Method->getReturnType();
   } else if (const auto *Enumerator = dyn_cast<EnumConstantDecl>(ND)) {
-    T = Context.getTypeDeclType(cast<TypeDecl>(Enumerator->getDeclContext()));
-    T = clang::TypeName::getFullyQualifiedType(T, Context);
+    T = Context.getCanonicalTagType(
+        cast<EnumDecl>(Enumerator->getDeclContext()));
   } else if (isa<UnresolvedUsingValueDecl>(ND)) {
     /* Do nothing: ignore unresolved using declarations*/
   } else if (const auto *Ivar = dyn_cast<ObjCIvarDecl>(ND)) {
@@ -3021,7 +3056,7 @@ static void findTypeLocationForBlockDecl(const TypeSourceInfo *TSInfo,
     if (!SuppressBlock) {
       if (TypedefTypeLoc TypedefTL = TL.getAsAdjusted<TypedefTypeLoc>()) {
         if (TypeSourceInfo *InnerTSInfo =
-                TypedefTL.getTypedefNameDecl()->getTypeSourceInfo()) {
+                TypedefTL.getDecl()->getTypeSourceInfo()) {
           TL = InnerTSInfo->getTypeLoc().getUnqualifiedLoc();
           continue;
         }
@@ -3381,7 +3416,7 @@ static void AddTemplateParameterChunks(
 /// Add a qualifier to the given code-completion string, if the
 /// provided nested-name-specifier is non-NULL.
 static void AddQualifierToCompletionString(CodeCompletionBuilder &Result,
-                                           NestedNameSpecifier *Qualifier,
+                                           NestedNameSpecifier Qualifier,
                                            bool QualifierIsInformative,
                                            ASTContext &Context,
                                            const PrintingPolicy &Policy) {
@@ -3391,7 +3426,7 @@ static void AddQualifierToCompletionString(CodeCompletionBuilder &Result,
   std::string PrintedNNS;
   {
     llvm::raw_string_ostream OS(PrintedNNS);
-    Qualifier->print(OS, Policy);
+    Qualifier.print(OS, Policy);
   }
   if (QualifierIsInformative)
     Result.AddInformativeChunk(Result.getAllocator().CopyString(PrintedNNS));
@@ -3399,42 +3434,57 @@ static void AddQualifierToCompletionString(CodeCompletionBuilder &Result,
     Result.AddTextChunk(Result.getAllocator().CopyString(PrintedNNS));
 }
 
-static void
-AddFunctionTypeQualsToCompletionString(CodeCompletionBuilder &Result,
-                                       const FunctionDecl *Function) {
-  const auto *Proto = Function->getType()->getAs<FunctionProtoType>();
-  if (!Proto || !Proto->getMethodQuals())
-    return;
-
+static void AddFunctionTypeQuals(CodeCompletionBuilder &Result,
+                                 const Qualifiers Quals) {
   // FIXME: Add ref-qualifier!
 
   // Handle single qualifiers without copying
-  if (Proto->getMethodQuals().hasOnlyConst()) {
+  if (Quals.hasOnlyConst()) {
     Result.AddInformativeChunk(" const");
     return;
   }
 
-  if (Proto->getMethodQuals().hasOnlyVolatile()) {
+  if (Quals.hasOnlyVolatile()) {
     Result.AddInformativeChunk(" volatile");
     return;
   }
 
-  if (Proto->getMethodQuals().hasOnlyRestrict()) {
+  if (Quals.hasOnlyRestrict()) {
     Result.AddInformativeChunk(" restrict");
     return;
   }
 
   // Handle multiple qualifiers.
   std::string QualsStr;
-  if (Proto->isConst())
+  if (Quals.hasConst())
     QualsStr += " const";
-  if (Proto->isVolatile())
+  if (Quals.hasVolatile())
     QualsStr += " volatile";
-  if (Proto->isRestrict())
+  if (Quals.hasRestrict())
     QualsStr += " restrict";
   Result.AddInformativeChunk(Result.getAllocator().CopyString(QualsStr));
 }
 
+static void
+AddFunctionTypeQualsToCompletionString(CodeCompletionBuilder &Result,
+                                       const FunctionDecl *Function) {
+  if (auto *CxxMethodDecl = llvm::dyn_cast_if_present<CXXMethodDecl>(Function);
+      CxxMethodDecl && CxxMethodDecl->hasCXXExplicitFunctionObjectParameter()) {
+    // if explicit object method, infer quals from the object parameter
+    const auto Quals = CxxMethodDecl->getFunctionObjectParameterType();
+    if (!Quals.hasQualifiers())
+      return;
+
+    AddFunctionTypeQuals(Result, Quals.getQualifiers());
+  } else {
+    const auto *Proto = Function->getType()->getAs<FunctionProtoType>();
+    if (!Proto || !Proto->getMethodQuals())
+      return;
+
+    AddFunctionTypeQuals(Result, Proto->getMethodQuals());
+  }
+}
+
 static void
 AddFunctionExceptSpecToCompletionString(std::string &NameAndSignature,
                                         const FunctionDecl *Function) {
@@ -3520,11 +3570,9 @@ static void AddTypedNameChunk(ASTContext &Context, const PrintingPolicy &Policy,
   case DeclarationName::CXXConstructorName: {
     CXXRecordDecl *Record = nullptr;
     QualType Ty = Name.getCXXNameType();
-    if (const auto *RecordTy = Ty->getAs<RecordType>())
-      Record = cast<CXXRecordDecl>(RecordTy->getDecl());
-    else if (const auto *InjectedTy = Ty->getAs<InjectedClassNameType>())
-      Record = InjectedTy->getDecl();
-    else {
+    if (auto *RD = Ty->getAsCXXRecordDecl()) {
+      Record = RD;
+    } else {
       Result.AddTypedTextChunk(
           Result.getAllocator().CopyString(ND->getNameAsString()));
       break;
@@ -4506,12 +4554,12 @@ static void MaybeAddOverrideCalls(Sema &S, DeclContext *InContext,
 
     // If we need a nested-name-specifier, add one now.
     if (!InContext) {
-      NestedNameSpecifier *NNS = getRequiredQualification(
+      NestedNameSpecifier NNS = getRequiredQualification(
           S.Context, CurContext, Overridden->getDeclContext());
       if (NNS) {
         std::string Str;
         llvm::raw_string_ostream OS(Str);
-        NNS->print(OS, Policy);
+        NNS.print(OS, Policy);
         Builder.AddTextChunk(Results.getAllocator().CopyString(Str));
       }
     } else if (!InContext->Equals(Overridden->getDeclContext()))
@@ -4627,12 +4675,19 @@ void SemaCodeCompletion::CodeCompleteOrdinaryName(
     break;
   }
 
-  // If we are in a C++ non-static member function, check the qualifiers on
-  // the member function to filter/prioritize the results list.
   auto ThisType = SemaRef.getCurrentThisType();
-  if (!ThisType.isNull())
+  if (ThisType.isNull()) {
+    // check if function scope is an explicit object function
+    if (auto *MethodDecl = llvm::dyn_cast_if_present<CXXMethodDecl>(
+            SemaRef.getCurFunctionDecl()))
+      Results.setExplicitObjectMemberFn(
+          MethodDecl->isExplicitObjectMemberFunction());
+  } else {
+    // If we are in a C++ non-static member function, check the qualifiers on
+    // the member function to filter/prioritize the results list.
     Results.setObjectTypeQualifiers(ThisType->getPointeeType().getQualifiers(),
                                     VK_LValue);
+  }
 
   CodeCompletionDeclConsumer Consumer(Results, SemaRef.CurContext);
   SemaRef.LookupVisibleDecls(S, SemaRef.LookupOrdinaryName, Consumer,
@@ -4920,14 +4975,14 @@ namespace {
 /// Information that allows to avoid completing redundant enumerators.
 struct CoveredEnumerators {
   llvm::SmallPtrSet<EnumConstantDecl *, 8> Seen;
-  NestedNameSpecifier *SuggestedQualifier = nullptr;
+  NestedNameSpecifier SuggestedQualifier = std::nullopt;
 };
 } // namespace
 
 static void AddEnumerators(ResultBuilder &Results, ASTContext &Context,
                            EnumDecl *Enum, DeclContext *CurContext,
                            const CoveredEnumerators &Enumerators) {
-  NestedNameSpecifier *Qualifier = Enumerators.SuggestedQualifier;
+  NestedNameSpecifier Qualifier = Enumerators.SuggestedQualifier;
   if (Context.getLangOpts().CPlusPlus && !Qualifier && Enumerators.Seen.empty()) {
     // If there are no prior enumerators in C++, check whether we have to
     // qualify the names of the enumerators that we suggest, because they
@@ -5059,9 +5114,9 @@ void SemaCodeCompletion::CodeCompleteExpression(
                              Data.PreferredType->isMemberPointerType() ||
                              Data.PreferredType->isBlockPointerType();
     if (Data.PreferredType->isEnumeralType()) {
-      EnumDecl *Enum = Data.PreferredType->castAs<EnumType>()->getDecl();
-      if (auto *Def = Enum->getDefinition())
-        Enum = Def;
+      EnumDecl *Enum = Data.PreferredType->castAs<EnumType>()
+                           ->getOriginalDecl()
+                           ->getDefinitionOrSelf();
       // FIXME: collect covered enumerators in cases like:
       //        if (x == my_enum::one) { ... } else if (x == ^) {}
       AddEnumerators(Results, getASTContext(), Enum, SemaRef.CurContext,
@@ -5185,7 +5240,8 @@ AddObjCProperties(const CodeCompletionContext &CCContext,
     // expressions.
     if (!P->getType().getTypePtr()->isBlockPointerType() ||
         !IsBaseExprStatement) {
-      Result R = Result(P, Results.getBasePriority(P), nullptr);
+      Result R =
+          Result(P, Results.getBasePriority(P), /*Qualifier=*/std::nullopt);
       if (!InOriginalClass)
         setInBaseClass(R);
       Results.MaybeAddResult(R, CurContext);
@@ -5199,7 +5255,8 @@ AddObjCProperties(const CodeCompletionContext &CCContext,
     findTypeLocationForBlockDecl(P->getTypeSourceInfo(), BlockLoc,
                                  BlockProtoLoc);
     if (!BlockLoc) {
-      Result R = Result(P, Results.getBasePriority(P), nullptr);
+      Result R =
+          Result(P, Results.getBasePriority(P), /*Qualifier=*/std::nullopt);
       if (!InOriginalClass)
         setInBaseClass(R);
       Results.MaybeAddResult(R, CurContext);
@@ -5611,15 +5668,18 @@ class ConceptInfo {
 
     // In T::foo, `foo` is a static member function/variable.
     bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) override {
-      if (E->getQualifier() && isApprox(E->getQualifier()->getAsType(), T))
+      NestedNameSpecifier Qualifier = E->getQualifier();
+      if (Qualifier.getKind() == NestedNameSpecifier::Kind::Type &&
+          isApprox(Qualifier.getAsType(), T))
         addValue(E, E->getDeclName(), Member::Colons);
       return true;
     }
 
     // In T::typename foo, `foo` is a type.
     bool VisitDependentNameType(DependentNameType *DNT) override {
-      const auto *Q = DNT->getQualifier();
-      if (Q && isApprox(Q->getAsType(), T))
+      NestedNameSpecifier Q = DNT->getQualifier();
+      if (Q.getKind() == NestedNameSpecifier::Kind::Type &&
+          isApprox(Q.getAsType(), T))
         addType(DNT->getIdentifier());
       return true;
     }
@@ -5628,10 +5688,15 @@ class ConceptInfo {
     // VisitNNS() doesn't exist, and TraverseNNS isn't always called :-(
     bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNSL) override {
       if (NNSL) {
-        NestedNameSpecifier *NNS = NNSL.getNestedNameSpecifier();
-        const auto *Q = NNS->getPrefix();
-        if (Q && isApprox(Q->getAsType(), T))
-          addType(NNS->getAsIdentifier());
+        NestedNameSpecifier NNS = NNSL.getNestedNameSpecifier();
+        if (NNS.getKind() == NestedNameSpecifier::Kind::Type) {
+          const Type *NNST = NNS.getAsType();
+          if (NestedNameSpecifier Q = NNST->getPrefix();
+              Q.getKind() == NestedNameSpecifier::Kind::Type &&
+              isApprox(Q.getAsType(), T))
+            if (const auto *DNT = dyn_cast_or_null<DependentNameType>(NNST))
+              addType(DNT->getIdentifier());
+        }
       }
       // FIXME: also handle T::foo<X>::bar
       return DynamicRecursiveASTVisitor::TraverseNestedNameSpecifierLoc(NNSL);
@@ -5777,7 +5842,7 @@ QualType getApproximateType(const Expr *E, HeuristicResolver &Resolver) {
     if (auto Decls = Resolver.resolveDependentNameType(DNT);
         Decls.size() == 1) {
       if (const auto *TD = dyn_cast<TypeDecl>(Decls[0]))
-        return QualType(TD->getTypeForDecl(), 0);
+        return TD->getASTContext().getTypeDeclType(TD);
     }
   }
   // We only resolve DependentTy, or undeduced autos (including auto* etc).
@@ -6184,9 +6249,8 @@ void SemaCodeCompletion::CodeCompleteCase(Scope *S) {
 
   // Code-complete the cases of a switch statement over an enumeration type
   // by providing the list of
-  EnumDecl *Enum = type->castAs<EnumType>()->getDecl();
-  if (EnumDecl *Def = Enum->getDefinition())
-    Enum = Def;
+  EnumDecl *Enum =
+      type->castAs<EnumType>()->getOriginalDecl()->getDefinitionOrSelf();
 
   // Determine which enumerators we have already seen in the switch statement.
   // FIXME: Ideally, we would also be able to look *past* the code-completion
@@ -6887,8 +6951,8 @@ void SemaCodeCompletion::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
 
   // Try to instantiate any non-dependent declaration contexts before
   // we look in them. Bail out if we fail.
-  NestedNameSpecifier *NNS = SS.getScopeRep();
-  if (NNS != nullptr && SS.isValid() && !NNS->isDependent()) {
+  NestedNameSpecifier NNS = SS.getScopeRep();
+  if (NNS && !NNS.isDependent()) {
     if (Ctx == nullptr || SemaRef.RequireCompleteDeclContext(SS, Ctx))
       return;
   }
@@ -6902,14 +6966,13 @@ void SemaCodeCompletion::CodeCompleteQualifiedId(Scope *S, CXXScopeSpec &SS,
   // The "template" keyword can follow "::" in the grammar, but only
   // put it into the grammar if the nested-name-specifier is dependent.
   // FIXME: results is always empty, this appears to be dead.
-  if (!Results.empty() && NNS && NNS->isDependent())
+  if (!Results.empty() && NNS.isDependent())
     Results.AddResult("template");
 
   // If the scope is a concept-constrained type parameter, infer nested
   // members based on the constraints.
-  if (NNS) {
-    if (const auto *TTPT =
-            dyn_cast_or_null<TemplateTypeParmType>(NNS->getAsType())) {
+  if (NNS.getKind() == NestedNameSpecifier::Kind::Type) {
+    if (const auto *TTPT = dyn_cast<TemplateTypeParmType>(NNS.getAsType())) {
       for (const auto &R : ConceptInfo(*TTPT, S).members()) {
         if (R.Operator != ConceptInfo::Member::Colons)
           continue;
@@ -7034,7 +7097,7 @@ void SemaCodeCompletion::CodeCompleteNamespaceDecl(Scope *S) {
          NS != NSEnd; ++NS)
       Results.AddResult(
           CodeCompletionResult(NS->second, Results.getBasePriority(NS->second),
-                               nullptr),
+                               /*Qualifier=*/std::nullopt),
           SemaRef.CurContext, nullptr, false);
     Results.ExitScope();
   }
@@ -7821,7 +7884,8 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
       if (!Selectors.insert(M->getSelector()).second)
         continue;
 
-      Result R = Result(M, Results.getBasePriority(M), nullptr);
+      Result R =
+          Result(M, Results.getBasePriority(M), /*Qualifier=*/std::nullopt);
       R.StartParameter = SelIdents.size();
       R.AllParametersAreInformative = (WantKind != MK_Any);
       if (!InOriginalClass)
@@ -8412,7 +8476,8 @@ AddClassMessageCompletions(Sema &SemaRef, Scope *S, ParsedType Receiver,
           continue;
 
         Result R(MethList->getMethod(),
-                 Results.getBasePriority(MethList->getMethod()), nullptr);
+                 Results.getBasePriority(MethList->getMethod()),
+                 /*Qualifier=*/std::nullopt);
         R.StartParameter = SelIdents.size();
         R.AllParametersAreInformative = false;
         Results.MaybeAddResult(R, SemaRef.CurContext);
@@ -8588,7 +8653,8 @@ void SemaCodeCompletion::CodeCompleteObjCInstanceMessage(
           continue;
 
         Result R(MethList->getMethod(),
-                 Results.getBasePriority(MethList->getMethod()), nullptr);
+                 Results.getBasePriority(MethList->getMethod()),
+                 /*Qualifier=*/std::nullopt);
         R.StartParameter = SelIdents.size();
         R.AllParametersAreInformative = false;
         Results.MaybeAddResult(R, SemaRef.CurContext);
@@ -8704,9 +8770,9 @@ static void AddProtocolResults(DeclContext *Ctx, DeclContext *CurContext,
     // Record any protocols we find.
     if (const auto *Proto = dyn_cast<ObjCProtocolDecl>(D))
       if (!OnlyForwardDeclarations || !Proto->hasDefinition())
-        Results.AddResult(
-            Result(Proto, Results.getBasePriority(Proto), nullptr), CurContext,
-            nullptr, false);
+        Results.AddResult(Result(Proto, Results.getBasePriority(Proto),
+                                 /*Qualifier=*/std::nullopt),
+                          CurContext, nullptr, false);
   }
 }
 
@@ -8772,9 +8838,9 @@ static void AddInterfaceResults(DeclContext *Ctx, DeclContext *CurContext,
     if (const auto *Class = dyn_cast<ObjCInterfaceDecl>(D))
       if ((!OnlyForwardDeclarations || !Class->hasDefinition()) &&
           (!OnlyUnimplemented || !Class->getImplementation()))
-        Results.AddResult(
-            Result(Class, Results.getBasePriority(Class), nullptr), CurContext,
-            nullptr, false);
+        Results.AddResult(Result(Class, Results.getBasePriority(Class),
+                                 /*Qualifier=*/std::nullopt),
+                          CurContext, nullptr, false);
   }
 }
 
@@ -8886,9 +8952,9 @@ void SemaCodeCompletion::CodeCompleteObjCInterfaceCategory(
   for (const auto *D : TU->decls())
     if (const auto *Category = dyn_cast<ObjCCategoryDecl>(D))
       if (CategoryNames.insert(Category->getIdentifier()).second)
-        Results.AddResult(
-            Result(Category, Results.getBasePriority(Category), nullptr),
-            SemaRef.CurContext, nullptr, false);
+        Results.AddResult(Result(Category, Results.getBasePriority(Category),
+                                 /*Qualifier=*/std::nullopt),
+                          SemaRef.CurContext, nullptr, false);
   Results.ExitScope();
 
   HandleCodeCompleteResults(&SemaRef, CodeCompleter,
@@ -8923,7 +8989,8 @@ void SemaCodeCompletion::CodeCompleteObjCImplementationCategory(
     for (const auto *Cat : Class->visible_categories()) {
       if ((!IgnoreImplemented || !Cat->getImplementation()) &&
           CategoryNames.insert(Cat->getIdentifier()).second)
-        Results.AddResult(Result(Cat, Results.getBasePriority(Cat), nullptr),
+        Results.AddResult(Result(Cat, Results.getBasePriority(Cat),
+                                 /*Qualifier=*/std::nullopt),
                           SemaRef.CurContext, nullptr, false);
     }
 
@@ -9023,7 +9090,8 @@ void SemaCodeCompletion::CodeCompleteObjCPropertySynthesizeIvar(
   for (; Class; Class = Class->getSuperClass()) {
     for (ObjCIvarDecl *Ivar = Class->all_declared_ivar_begin(); Ivar;
          Ivar = Ivar->getNextIvar()) {
-      Results.AddResult(Result(Ivar, Results.getBasePriority(Ivar), nullptr),
+      Results.AddResult(Result(Ivar, Results.getBasePriority(Ivar),
+                               /*Qualifier=*/std::nullopt),
                         SemaRef.CurContext, nullptr, false);
 
       // Determine whether we've seen an ivar with a name similar to the
@@ -10039,7 +10107,8 @@ void SemaCodeCompletion::CodeCompleteObjCMethodDeclSelector(
       }
 
       Result R(MethList->getMethod(),
-               Results.getBasePriority(MethList->getMethod()), nullptr);
+               Results.getBasePriority(MethList->getMethod()),
+               /*Qualifier=*/std::nullopt);
       R.StartParameter = SelIdents.size();
       R.AllParametersAreInformative = false;
       R.DeclaringEntity = true;
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index da85959c005fe..f114173a42c21 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -307,7 +307,8 @@ static UnsignedOrNone EvaluateFoldExpandedConstraintSize(
   UnsignedOrNone NumExpansions = FE->getNumExpansions();
   if (S.CheckParameterPacksForExpansion(
           FE->getEllipsisLoc(), Pattern->getSourceRange(), Unexpanded, MLTAL,
-          Expand, RetainExpansion, NumExpansions) ||
+          /*FailOnPackProducingTemplates=*/true, Expand, RetainExpansion,
+          NumExpansions) ||
       !Expand || RetainExpansion)
     return std::nullopt;
 
@@ -1696,11 +1697,13 @@ bool FoldExpandedConstraint::AreCompatibleForSubsumption(
   Sema::collectUnexpandedParameterPacks(const_cast<Expr *>(B.Pattern), BPacks);
 
   for (const UnexpandedParameterPack &APack : APacks) {
-    std::pair<unsigned, unsigned> DepthAndIndex = getDepthAndIndex(APack);
-    auto it = llvm::find_if(BPacks, [&](const UnexpandedParameterPack &BPack) {
-      return getDepthAndIndex(BPack) == DepthAndIndex;
+    auto ADI = getDepthAndIndex(APack);
+    if (!ADI)
+      continue;
+    auto It = llvm::find_if(BPacks, [&](const UnexpandedParameterPack &BPack) {
+      return getDepthAndIndex(BPack) == ADI;
     });
-    if (it != BPacks.end())
+    if (It != BPacks.end())
       return true;
   }
   return false;
diff --git a/clang/lib/Sema/SemaCoroutine.cpp b/clang/lib/Sema/SemaCoroutine.cpp
index d193a33f22393..cc03616e0dfe1 100644
--- a/clang/lib/Sema/SemaCoroutine.cpp
+++ b/clang/lib/Sema/SemaCoroutine.cpp
@@ -89,8 +89,8 @@ static QualType lookupPromiseType(Sema &S, const FunctionDecl *FD,
     AddArg(T);
 
   // Build the template-id.
-  QualType CoroTrait =
-      S.CheckTemplateIdType(TemplateName(CoroTraits), KwLoc, Args);
+  QualType CoroTrait = S.CheckTemplateIdType(
+      ElaboratedTypeKeyword::None, TemplateName(CoroTraits), KwLoc, Args);
   if (CoroTrait.isNull())
     return QualType();
   if (S.RequireCompleteType(KwLoc, CoroTrait,
@@ -111,23 +111,18 @@ static QualType lookupPromiseType(Sema &S, const FunctionDecl *FD,
         << RD;
     return QualType();
   }
-  // The promise type is required to be a class type.
-  QualType PromiseType = S.Context.getTypeDeclType(Promise);
-
-  auto buildElaboratedType = [&]() {
-    auto *NNS = NestedNameSpecifier::Create(S.Context, nullptr, S.getStdNamespace());
-    NNS = NestedNameSpecifier::Create(S.Context, NNS, CoroTrait.getTypePtr());
-    return S.Context.getElaboratedType(ElaboratedTypeKeyword::None, NNS,
-                                       PromiseType);
-  };
 
+  NestedNameSpecifier Qualifier(CoroTrait.getTypePtr());
+  QualType PromiseType = S.Context.getTypeDeclType(ElaboratedTypeKeyword::None,
+                                                   Qualifier, Promise);
+  // The promise type is required to be a class type.
   if (!PromiseType->getAsCXXRecordDecl()) {
     S.Diag(FuncLoc,
            diag::err_implied_std_coroutine_traits_promise_type_not_class)
-        << buildElaboratedType();
+        << PromiseType;
     return QualType();
   }
-  if (S.RequireCompleteType(FuncLoc, buildElaboratedType(),
+  if (S.RequireCompleteType(FuncLoc, PromiseType,
                             diag::err_coroutine_promise_type_incomplete))
     return QualType();
 
@@ -167,8 +162,8 @@ static QualType lookupCoroutineHandleType(Sema &S, QualType PromiseType,
       S.Context.getTrivialTypeSourceInfo(PromiseType, Loc)));
 
   // Build the template-id.
-  QualType CoroHandleType =
-      S.CheckTemplateIdType(TemplateName(CoroHandle), Loc, Args);
+  QualType CoroHandleType = S.CheckTemplateIdType(
+      ElaboratedTypeKeyword::None, TemplateName(CoroHandle), Loc, Args);
   if (CoroHandleType.isNull())
     return QualType();
   if (S.RequireCompleteType(Loc, CoroHandleType,
@@ -643,7 +638,9 @@ static void checkNoThrow(Sema &S, const Stmt *E,
         QualType::DestructionKind::DK_cxx_destructor) {
       const auto *T =
           cast<RecordType>(ReturnType.getCanonicalType().getTypePtr());
-      checkDeclNoexcept(cast<CXXRecordDecl>(T->getDecl())->getDestructor(),
+      checkDeclNoexcept(cast<CXXRecordDecl>(T->getOriginalDecl())
+                            ->getDefinition()
+                            ->getDestructor(),
                         /*IsDtor=*/true);
     }
   } else
@@ -1083,9 +1080,9 @@ static Expr *buildStdNoThrowDeclRef(Sema &S, SourceLocation Loc) {
 
 static TypeSourceInfo *getTypeSourceInfoForStdAlignValT(Sema &S,
                                                         SourceLocation Loc) {
-  EnumDecl *StdAlignValT = S.getStdAlignValT();
-  QualType StdAlignValDecl = S.Context.getTypeDeclType(StdAlignValT);
-  return S.Context.getTrivialTypeSourceInfo(StdAlignValDecl);
+  EnumDecl *StdAlignValDecl = S.getStdAlignValT();
+  CanQualType StdAlignValT = S.Context.getCanonicalTagType(StdAlignValDecl);
+  return S.Context.getTrivialTypeSourceInfo(StdAlignValT);
 }
 
 // When searching for custom allocators on the PromiseType we want to
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index b5eb825eb52cc..160d7353cacd9 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -140,8 +140,8 @@ class TypeNameValidatorCCC final : public CorrectionCandidateCallback {
 
 } // end anonymous namespace
 
-QualType Sema::getTypeDeclType(DeclContext *LookupCtx, DiagCtorKind DCK,
-                               TypeDecl *TD, SourceLocation NameLoc) {
+void Sema::checkTypeDeclType(DeclContext *LookupCtx, DiagCtorKind DCK,
+                             TypeDecl *TD, SourceLocation NameLoc) {
   auto *LookupRD = dyn_cast_or_null<CXXRecordDecl>(LookupCtx);
   auto *FoundRD = dyn_cast<CXXRecordDecl>(TD);
   if (DCK != DiagCtorKind::None && LookupRD && FoundRD &&
@@ -157,7 +157,6 @@ QualType Sema::getTypeDeclType(DeclContext *LookupCtx, DiagCtorKind DCK,
 
   DiagnoseUseOfDecl(TD, NameLoc);
   MarkAnyDeclReferenced(TD->getLocation(), TD, /*OdrUse=*/false);
-  return Context.getTypeDeclType(TD);
 }
 
 namespace {
@@ -182,13 +181,13 @@ lookupUnqualifiedTypeNameInBase(Sema &S, const IdentifierInfo &II,
   UnqualifiedTypeNameLookupResult FoundTypeDecl =
       UnqualifiedTypeNameLookupResult::NotFound;
   for (const auto &Base : RD->bases()) {
-    const CXXRecordDecl *BaseRD = nullptr;
-    if (auto *BaseTT = Base.getType()->getAs<TagType>())
-      BaseRD = BaseTT->getAsCXXRecordDecl();
-    else if (auto *TST = Base.getType()->getAs<TemplateSpecializationType>()) {
+    const CXXRecordDecl *BaseRD = Base.getType()->getAsCXXRecordDecl();
+    if (BaseRD) {
+    } else if (auto *TST = dyn_cast<TemplateSpecializationType>(
+                   Base.getType().getCanonicalType())) {
       // Look for type decls in dependent base classes that have known primary
       // templates.
-      if (!TST || !TST->isDependentType())
+      if (!TST->isDependentType())
         continue;
       auto *TD = TST->getTemplateName().getAsTemplateDecl();
       if (!TD)
@@ -253,8 +252,7 @@ static ParsedType recoverFromTypeInKnownDependentBase(Sema &S,
   S.Diag(NameLoc, diag::ext_found_in_dependent_base) << &II;
 
   ASTContext &Context = S.Context;
-  auto *NNS = NestedNameSpecifier::Create(
-      Context, nullptr, cast<Type>(Context.getRecordType(RD)));
+  NestedNameSpecifier NNS(Context.getCanonicalTagType(RD).getTypePtr());
   QualType T =
       Context.getDependentNameType(ElaboratedTypeKeyword::None, NNS, &II);
 
@@ -269,45 +267,6 @@ static ParsedType recoverFromTypeInKnownDependentBase(Sema &S,
   return S.CreateParsedType(T, Builder.getTypeSourceInfo(Context, T));
 }
 
-/// Build a ParsedType for a simple-type-specifier with a nested-name-specifier.
-static ParsedType buildNamedType(Sema &S, const CXXScopeSpec *SS, QualType T,
-                                 SourceLocation NameLoc,
-                                 bool WantNontrivialTypeSourceInfo = true) {
-  switch (T->getTypeClass()) {
-  case Type::DeducedTemplateSpecialization:
-  case Type::Enum:
-  case Type::InjectedClassName:
-  case Type::Record:
-  case Type::Typedef:
-  case Type::UnresolvedUsing:
-  case Type::Using:
-    break;
-  // These can never be qualified so an ElaboratedType node
-  // would carry no additional meaning.
-  case Type::ObjCInterface:
-  case Type::ObjCTypeParam:
-  case Type::TemplateTypeParm:
-    return ParsedType::make(T);
-  default:
-    llvm_unreachable("Unexpected Type Class");
-  }
-
-  if (!SS || SS->isEmpty())
-    return ParsedType::make(S.Context.getElaboratedType(
-        ElaboratedTypeKeyword::None, nullptr, T, nullptr));
-
-  QualType ElTy = S.getElaboratedType(ElaboratedTypeKeyword::None, *SS, T);
-  if (!WantNontrivialTypeSourceInfo)
-    return ParsedType::make(ElTy);
-
-  TypeLocBuilder Builder;
-  Builder.pushTypeSpec(T).setNameLoc(NameLoc);
-  ElaboratedTypeLoc ElabTL = Builder.push<ElaboratedTypeLoc>(ElTy);
-  ElabTL.setElaboratedKeywordLoc(SourceLocation());
-  ElabTL.setQualifierLoc(SS->getWithLocInContext(S.Context));
-  return S.CreateParsedType(ElTy, Builder.getTypeSourceInfo(S.Context, ElTy));
-}
-
 ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
                              Scope *S, CXXScopeSpec *SS, bool isClassName,
                              bool HasTrailingDot, ParsedType ObjectTypePtr,
@@ -348,9 +307,10 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
           if (AllowImplicitTypename == ImplicitTypenameContext::No)
             return nullptr;
           SourceLocation QualifiedLoc = SS->getRange().getBegin();
-          auto DB =
-              DiagCompat(QualifiedLoc, diag_compat::implicit_typename)
-              << NestedNameSpecifier::Create(Context, SS->getScopeRep(), &II);
+          // FIXME: Defer the diagnostic after we build the type and use it.
+          auto DB = DiagCompat(QualifiedLoc, diag_compat::implicit_typename)
+                    << Context.getDependentNameType(ElaboratedTypeKeyword::None,
+                                                    SS->getScopeRep(), &II);
           if (!getLangOpts().CPlusPlus20)
             DB << FixItHint::CreateInsertion(QualifiedLoc, "typename ");
         }
@@ -430,7 +390,7 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
       bool MemberOfUnknownSpecialization;
       UnqualifiedId TemplateName;
       TemplateName.setIdentifier(NewII, NameLoc);
-      NestedNameSpecifier *NNS = Correction.getCorrectionSpecifier();
+      NestedNameSpecifier NNS = Correction.getCorrectionSpecifier();
       CXXScopeSpec NewSS, *NewSSPtr = SS;
       if (SS && NNS) {
         NewSS.MakeTrivial(Context, NNS, SourceRange(NameLoc));
@@ -530,20 +490,78 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
 
   assert(IIDecl && "Didn't find decl");
 
-  QualType T;
+  TypeLocBuilder TLB;
   if (TypeDecl *TD = dyn_cast<TypeDecl>(IIDecl)) {
-    // C++ [class.qual]p2: A lookup that would find the injected-class-name
-    // instead names the constructors of the class, except when naming a class.
-    // This is ill-formed when we're not actually forming a ctor or dtor name.
-    T = getTypeDeclType(LookupCtx,
-                        IsImplicitTypename ? DiagCtorKind::Implicit
-                                           : DiagCtorKind::None,
-                        TD, NameLoc);
-  } else if (ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(IIDecl)) {
+    checkTypeDeclType(LookupCtx,
+                      IsImplicitTypename ? DiagCtorKind::Implicit
+                                         : DiagCtorKind::None,
+                      TD, NameLoc);
+    QualType T;
+    if (FoundUsingShadow) {
+      T = Context.getUsingType(ElaboratedTypeKeyword::None,
+                               SS ? SS->getScopeRep() : std::nullopt,
+                               FoundUsingShadow);
+      if (!WantNontrivialTypeSourceInfo)
+        return ParsedType::make(T);
+      TLB.push<UsingTypeLoc>(T).set(/*ElaboratedKeywordLoc=*/SourceLocation(),
+                                    SS ? SS->getWithLocInContext(Context)
+                                       : NestedNameSpecifierLoc(),
+                                    NameLoc);
+    } else if (auto *Tag = dyn_cast<TagDecl>(TD)) {
+      T = Context.getTagType(ElaboratedTypeKeyword::None,
+                             SS ? SS->getScopeRep() : std::nullopt, Tag,
+                             /*OwnsTag=*/false);
+      if (!WantNontrivialTypeSourceInfo)
+        return ParsedType::make(T);
+      auto TL = TLB.push<TagTypeLoc>(T);
+      TL.setElaboratedKeywordLoc(SourceLocation());
+      TL.setQualifierLoc(SS ? SS->getWithLocInContext(Context)
+                            : NestedNameSpecifierLoc());
+      TL.setNameLoc(NameLoc);
+    } else if (auto *TN = dyn_cast<TypedefNameDecl>(TD);
+               TN && !isa<ObjCTypeParamDecl>(TN)) {
+      T = Context.getTypedefType(ElaboratedTypeKeyword::None,
+                                 SS ? SS->getScopeRep() : std::nullopt, TN);
+      if (!WantNontrivialTypeSourceInfo)
+        return ParsedType::make(T);
+      TLB.push<TypedefTypeLoc>(T).set(
+          /*ElaboratedKeywordLoc=*/SourceLocation(),
+          SS ? SS->getWithLocInContext(Context) : NestedNameSpecifierLoc(),
+          NameLoc);
+    } else if (auto *UD = dyn_cast<UnresolvedUsingTypenameDecl>(TD)) {
+      T = Context.getUnresolvedUsingType(ElaboratedTypeKeyword::None,
+                                         SS ? SS->getScopeRep() : std::nullopt,
+                                         UD);
+      if (!WantNontrivialTypeSourceInfo)
+        return ParsedType::make(T);
+      TLB.push<UnresolvedUsingTypeLoc>(T).set(
+          /*ElaboratedKeywordLoc=*/SourceLocation(),
+          SS ? SS->getWithLocInContext(Context) : NestedNameSpecifierLoc(),
+          NameLoc);
+    } else {
+      T = Context.getTypeDeclType(TD);
+      if (!WantNontrivialTypeSourceInfo)
+        return ParsedType::make(T);
+      if (isa<ObjCTypeParamType>(T))
+        TLB.push<ObjCTypeParamTypeLoc>(T).setNameLoc(NameLoc);
+      else
+        TLB.pushTypeSpec(T).setNameLoc(NameLoc);
+    }
+    return CreateParsedType(T, TLB.getTypeSourceInfo(Context, T));
+  }
+  if (ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(IIDecl)) {
     (void)DiagnoseUseOfDecl(IDecl, NameLoc);
-    if (!HasTrailingDot)
-      T = Context.getObjCInterfaceType(IDecl);
-    FoundUsingShadow = nullptr; // FIXME: Target must be a TypeDecl.
+    if (!HasTrailingDot) {
+      // FIXME: Support UsingType for this case.
+      QualType T = Context.getObjCInterfaceType(IDecl);
+      if (!WantNontrivialTypeSourceInfo)
+        return ParsedType::make(T);
+      auto TL = TLB.push<ObjCInterfaceTypeLoc>(T);
+      TL.setNameLoc(NameLoc);
+      // FIXME: Pass in this source location.
+      TL.setNameEndLoc(NameLoc);
+      return CreateParsedType(T, TLB.getTypeSourceInfo(Context, T));
+    }
   } else if (auto *UD = dyn_cast<UnresolvedUsingIfExistsDecl>(IIDecl)) {
     (void)DiagnoseUseOfDecl(UD, NameLoc);
     // Recover with 'int'
@@ -551,41 +569,38 @@ ParsedType Sema::getTypeName(const IdentifierInfo &II, SourceLocation NameLoc,
   } else if (AllowDeducedTemplate) {
     if (auto *TD = getAsTypeTemplateDecl(IIDecl)) {
       assert(!FoundUsingShadow || FoundUsingShadow->getTargetDecl() == TD);
+      // FIXME: Support UsingType here.
       TemplateName Template = Context.getQualifiedTemplateName(
-          SS ? SS->getScopeRep() : nullptr, /*TemplateKeyword=*/false,
+          SS ? SS->getScopeRep() : std::nullopt, /*TemplateKeyword=*/false,
           FoundUsingShadow ? TemplateName(FoundUsingShadow) : TemplateName(TD));
-      T = Context.getDeducedTemplateSpecializationType(Template, QualType(),
-                                                       false);
-      // Don't wrap in a further UsingType.
-      FoundUsingShadow = nullptr;
+      QualType T = Context.getDeducedTemplateSpecializationType(
+          ElaboratedTypeKeyword::None, Template, QualType(), false);
+      auto TL = TLB.push<DeducedTemplateSpecializationTypeLoc>(T);
+      TL.setElaboratedKeywordLoc(SourceLocation());
+      TL.setNameLoc(NameLoc);
+      TL.setQualifierLoc(SS ? SS->getWithLocInContext(Context)
+                            : NestedNameSpecifierLoc());
+      return CreateParsedType(T, TLB.getTypeSourceInfo(Context, T));
     }
   }
 
-  if (T.isNull()) {
-    // If it's not plausibly a type, suppress diagnostics.
-    Result.suppressDiagnostics();
-    return nullptr;
-  }
-
-  if (FoundUsingShadow)
-    T = Context.getUsingType(FoundUsingShadow, T);
-
-  return buildNamedType(*this, SS, T, NameLoc, WantNontrivialTypeSourceInfo);
+  // As it's not plausibly a type, suppress diagnostics.
+  Result.suppressDiagnostics();
+  return nullptr;
 }
 
 // Builds a fake NNS for the given decl context.
-static NestedNameSpecifier *
+static NestedNameSpecifier
 synthesizeCurrentNestedNameSpecifier(ASTContext &Context, DeclContext *DC) {
   for (;; DC = DC->getLookupParent()) {
     DC = DC->getPrimaryContext();
     auto *ND = dyn_cast<NamespaceDecl>(DC);
     if (ND && !ND->isInline() && !ND->isAnonymousNamespace())
-      return NestedNameSpecifier::Create(Context, nullptr, ND);
+      return NestedNameSpecifier(Context, ND, std::nullopt);
     if (auto *RD = dyn_cast<CXXRecordDecl>(DC))
-      return NestedNameSpecifier::Create(Context, nullptr,
-                                         RD->getTypeForDecl());
+      return NestedNameSpecifier(Context.getCanonicalTagType(RD)->getTypePtr());
     if (isa<TranslationUnitDecl>(DC))
-      return NestedNameSpecifier::GlobalSpecifier(Context);
+      return NestedNameSpecifier::getGlobal();
   }
   llvm_unreachable("something isn't in TU scope?");
 }
@@ -610,7 +625,7 @@ ParsedType Sema::ActOnMSVCUnknownTypeName(const IdentifierInfo &II,
                                           bool IsTemplateTypeArg) {
   assert(getLangOpts().MSVCCompat && "shouldn't be called in non-MSVC mode");
 
-  NestedNameSpecifier *NNS = nullptr;
+  NestedNameSpecifier NNS = std::nullopt;
   if (IsTemplateTypeArg && getCurScope()->isTemplateParamScope()) {
     // If we weren't able to parse a default template argument, delay lookup
     // until instantiation time by making a non-dependent DependentTypeName. We
@@ -625,7 +640,7 @@ ParsedType Sema::ActOnMSVCUnknownTypeName(const IdentifierInfo &II,
                  findRecordWithDependentBasesOfEnclosingMethod(CurContext)) {
     // Build a DependentNameType that will perform lookup into RD at
     // instantiation time.
-    NNS = NestedNameSpecifier::Create(Context, nullptr, RD->getTypeForDecl());
+    NNS = NestedNameSpecifier(Context.getCanonicalTagType(RD)->getTypePtr());
 
     // Diagnose that this identifier was undeclared, and retry the lookup during
     // template instantiation.
@@ -678,19 +693,22 @@ DeclSpec::TST Sema::isTagName(IdentifierInfo &II, Scope *S) {
 }
 
 bool Sema::isMicrosoftMissingTypename(const CXXScopeSpec *SS, Scope *S) {
-  if (CurContext->isRecord()) {
-    if (SS->getScopeRep()->getKind() == NestedNameSpecifier::Super)
-      return true;
-
-    const Type *Ty = SS->getScopeRep()->getAsType();
+  if (!CurContext->isRecord())
+    return CurContext->isFunctionOrMethod() || S->isFunctionPrototypeScope();
 
-    CXXRecordDecl *RD = cast<CXXRecordDecl>(CurContext);
-    for (const auto &Base : RD->bases())
-      if (Ty && Context.hasSameUnqualifiedType(QualType(Ty, 1), Base.getType()))
+  switch (SS->getScopeRep().getKind()) {
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    return true;
+  case NestedNameSpecifier::Kind::Type: {
+    QualType T(SS->getScopeRep().getAsType(), 0);
+    for (const auto &Base : cast<CXXRecordDecl>(CurContext)->bases())
+      if (Context.hasSameUnqualifiedType(T, Base.getType()))
         return true;
+    [[fallthrough]];
+  }
+  default:
     return S->isFunctionPrototypeScope();
   }
-  return CurContext->isFunctionOrMethod() || S->isFunctionPrototypeScope();
 }
 
 void Sema::DiagnoseUnknownTypeName(IdentifierInfo *&II,
@@ -786,7 +804,7 @@ void Sema::DiagnoseUnknownTypeName(IdentifierInfo *&II,
     Diag(IILoc, IsTemplateName ? diag::err_no_member_template
                                : diag::err_typename_nested_not_found)
         << II << DC << SS->getRange();
-  else if (SS->isValid() && SS->getScopeRep()->containsErrors()) {
+  else if (SS->isValid() && SS->getScopeRep().containsErrors()) {
     SuggestedType =
         ActOnTypenameType(S, SourceLocation(), *SS, *II, IILoc).get();
   } else if (isDependentScopeSpecifier(*SS)) {
@@ -794,12 +812,13 @@ void Sema::DiagnoseUnknownTypeName(IdentifierInfo *&II,
     if (getLangOpts().MSVCCompat && isMicrosoftMissingTypename(SS, S))
       DiagID = diag::ext_typename_missing;
 
+    SuggestedType =
+        ActOnTypenameType(S, SourceLocation(), *SS, *II, IILoc).get();
+
     Diag(SS->getRange().getBegin(), DiagID)
-        << NestedNameSpecifier::Create(Context, SS->getScopeRep(), II)
+        << GetTypeFromParser(SuggestedType)
         << SourceRange(SS->getRange().getBegin(), IILoc)
         << FixItHint::CreateInsertion(SS->getRange().getBegin(), "typename ");
-    SuggestedType = ActOnTypenameType(S, SourceLocation(),
-                                      *SS, *II, IILoc).get();
   } else {
     assert(SS && SS->isInvalid() &&
            "Invalid scope specifier has already been diagnosed");
@@ -1156,10 +1175,34 @@ Sema::NameClassification Sema::ClassifyName(Scope *S, CXXScopeSpec &SS,
   }
 
   auto BuildTypeFor = [&](TypeDecl *Type, NamedDecl *Found) {
-    QualType T = Context.getTypeDeclType(Type);
-    if (const auto *USD = dyn_cast<UsingShadowDecl>(Found))
-      T = Context.getUsingType(USD, T);
-    return buildNamedType(*this, &SS, T, NameLoc);
+    QualType T;
+    TypeLocBuilder TLB;
+    if (const auto *USD = dyn_cast<UsingShadowDecl>(Found)) {
+      T = Context.getUsingType(ElaboratedTypeKeyword::None, SS.getScopeRep(),
+                               USD);
+      TLB.push<UsingTypeLoc>(T).set(/*ElaboratedKeywordLoc=*/SourceLocation(),
+                                    SS.getWithLocInContext(Context), NameLoc);
+    } else {
+      T = Context.getTypeDeclType(ElaboratedTypeKeyword::None, SS.getScopeRep(),
+                                  Type);
+      if (isa<TagType>(T)) {
+        auto TTL = TLB.push<TagTypeLoc>(T);
+        TTL.setElaboratedKeywordLoc(SourceLocation());
+        TTL.setQualifierLoc(SS.getWithLocInContext(Context));
+        TTL.setNameLoc(NameLoc);
+      } else if (isa<TypedefType>(T)) {
+        TLB.push<TypedefTypeLoc>(T).set(
+            /*ElaboratedKeywordLoc=*/SourceLocation(),
+            SS.getWithLocInContext(Context), NameLoc);
+      } else if (isa<UnresolvedUsingType>(T)) {
+        TLB.push<UnresolvedUsingTypeLoc>(T).set(
+            /*ElaboratedKeywordLoc=*/SourceLocation(),
+            SS.getWithLocInContext(Context), NameLoc);
+      } else {
+        TLB.pushTypeSpec(T).setNameLoc(NameLoc);
+      }
+    }
+    return CreateParsedType(T, TLB.getTypeSourceInfo(Context, T));
   };
 
   NamedDecl *FirstDecl = (*Result.begin())->getUnderlyingDecl();
@@ -2009,8 +2052,7 @@ static bool ShouldDiagnoseUnusedDecl(const LangOptions &LangOpts,
     // consistent for both scalars and arrays.
     Ty = Ty->getBaseElementTypeUnsafe();
 
-    if (const TagType *TT = Ty->getAs<TagType>()) {
-      const TagDecl *Tag = TT->getDecl();
+    if (const TagDecl *Tag = Ty->getAsTagDecl()) {
       if (Tag->hasAttr<UnusedAttr>())
         return false;
 
@@ -2070,7 +2112,7 @@ void Sema::DiagnoseUnusedNestedTypedefs(const RecordDecl *D) {
 
 void Sema::DiagnoseUnusedNestedTypedefs(const RecordDecl *D,
                                         DiagReceiverTy DiagReceiver) {
-  if (D->getTypeForDecl()->isDependentType())
+  if (D->isDependentType())
     return;
 
   for (auto *TmpD : D->decls()) {
@@ -2128,8 +2170,7 @@ void Sema::DiagnoseUnusedButSetDecl(const VarDecl *VD,
   if (Ty->isReferenceType() || Ty->isDependentType())
     return;
 
-  if (const TagType *TT = Ty->getAs<TagType>()) {
-    const TagDecl *Tag = TT->getDecl();
+  if (const TagDecl *Tag = Ty->getAsTagDecl()) {
     if (Tag->hasAttr<UnusedAttr>())
       return;
     // In C++, don't warn for record types that don't have WarnUnusedAttr, to
@@ -2508,7 +2549,8 @@ void Sema::MergeTypedefNameDecl(Scope *S, TypedefNameDecl *New,
         }
         Context.setObjCIdRedefinitionType(T);
         // Install the built-in type for 'id', ignoring the current definition.
-        New->setTypeForDecl(Context.getObjCIdType().getTypePtr());
+        New->setModedTypeSourceInfo(New->getTypeSourceInfo(),
+                                    Context.getObjCIdType());
         return;
       }
     case 5:
@@ -2516,14 +2558,16 @@ void Sema::MergeTypedefNameDecl(Scope *S, TypedefNameDecl *New,
         break;
       Context.setObjCClassRedefinitionType(New->getUnderlyingType());
       // Install the built-in type for 'Class', ignoring the current definition.
-      New->setTypeForDecl(Context.getObjCClassType().getTypePtr());
+      New->setModedTypeSourceInfo(New->getTypeSourceInfo(),
+                                  Context.getObjCClassType());
       return;
     case 3:
       if (!TypeID->isStr("SEL"))
         break;
       Context.setObjCSelRedefinitionType(New->getUnderlyingType());
       // Install the built-in type for 'SEL', ignoring the current definition.
-      New->setTypeForDecl(Context.getObjCSelType().getTypePtr());
+      New->setModedTypeSourceInfo(New->getTypeSourceInfo(),
+                                  Context.getObjCSelType());
       return;
     }
     // Fall through - the typedef name was not a builtin type.
@@ -2555,7 +2599,6 @@ void Sema::MergeTypedefNameDecl(Scope *S, TypedefNameDecl *New,
         !hasVisibleDefinition(OldTag, &Hidden)) {
       // There is a definition of this tag, but it is not visible. Use it
       // instead of our tag.
-      New->setTypeForDecl(OldTD->getTypeForDecl());
       if (OldTD->isModed())
         New->setModedTypeSourceInfo(OldTD->getTypeSourceInfo(),
                                     OldTD->getUnderlyingType());
@@ -2742,7 +2785,7 @@ static bool mergeAlignedAttrs(Sema &S, NamedDecl *New, Decl *Old) {
       if (ValueDecl *VD = dyn_cast<ValueDecl>(New))
         Ty = VD->getType();
       else
-        Ty = S.Context.getTagDeclType(cast<TagDecl>(New));
+        Ty = S.Context.getCanonicalTagType(cast<TagDecl>(New));
 
       if (OldAlign == 0)
         OldAlign = S.Context.getTypeAlign(Ty);
@@ -2913,8 +2956,11 @@ static bool mergeDeclAttribute(Sema &S, NamedDecl *D,
 }
 
 static const NamedDecl *getDefinition(const Decl *D) {
-  if (const TagDecl *TD = dyn_cast<TagDecl>(D))
-    return TD->getDefinition();
+  if (const TagDecl *TD = dyn_cast<TagDecl>(D)) {
+    if (const auto *Def = TD->getDefinition(); Def && !Def->isBeingDefined())
+      return Def;
+    return nullptr;
+  }
   if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
     const VarDecl *Def = VD->getDefinition();
     if (Def)
@@ -3069,6 +3115,10 @@ static void checkNewAttributesAfterDef(Sema &S, Decl *New, const Decl *Old) {
       cast<SYCLKernelEntryPointAttr>(NewAttribute)->setInvalidAttr();
       ++I;
       continue;
+    } else if (isa<SYCLExternalAttr>(NewAttribute)) {
+      // SYCLExternalAttr may be added after a definition.
+      ++I;
+      continue;
     }
 
     S.Diag(NewAttribute->getLocation(),
@@ -3607,7 +3657,9 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, NamedDecl *&OldD, Scope *S,
   FunctionDecl *Old = OldD->getAsFunction();
   if (!Old) {
     if (UsingShadowDecl *Shadow = dyn_cast<UsingShadowDecl>(OldD)) {
-      if (New->getFriendObjectKind()) {
+      // We don't need to check the using friend pattern from other module unit
+      // since we should have diagnosed such cases in its unit already.
+      if (New->getFriendObjectKind() && !OldD->isInAnotherModuleUnit()) {
         Diag(New->getLocation(), diag::err_using_decl_friend);
         Diag(Shadow->getTargetDecl()->getLocation(),
              diag::note_using_decl_target);
@@ -4092,6 +4144,18 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, NamedDecl *&OldD, Scope *S,
            diag::note_carries_dependency_missing_first_decl) << 0/*Function*/;
     }
 
+    // SYCL 2020 section 5.10.1, "SYCL functions and member functions linkage":
+    //   When a function is declared with SYCL_EXTERNAL, that macro must be
+    //   used on the first declaration of that function in the translation unit.
+    //   Redeclarations of the function in the same translation unit may
+    //   optionally use SYCL_EXTERNAL, but this is not required.
+    const SYCLExternalAttr *SEA = New->getAttr<SYCLExternalAttr>();
+    if (SEA && !Old->hasAttr<SYCLExternalAttr>()) {
+      Diag(SEA->getLocation(), diag::warn_sycl_external_missing_on_first_decl)
+          << SEA;
+      Diag(Old->getLocation(), diag::note_previous_declaration);
+    }
+
     // (C++98 8.3.5p3):
     //   All declarations for a function shall agree exactly in both the
     //   return type and the parameter-type-list.
@@ -5024,7 +5088,7 @@ void Sema::setTagNameForLinkagePurposes(TagDecl *TagFromDeclSpec,
 
   // The type must match the tag exactly;  no qualifiers allowed.
   if (!Context.hasSameType(NewTD->getUnderlyingType(),
-                           Context.getTagDeclType(TagFromDeclSpec))) {
+                           Context.getCanonicalTagType(TagFromDeclSpec))) {
     if (getLangOpts().CPlusPlus)
       Context.addTypedefNameForUnnamedTagDecl(TagFromDeclSpec, NewTD);
     return;
@@ -5232,9 +5296,9 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
         Record = dyn_cast<RecordDecl>(Tag);
       else if (const RecordType *RT =
                    DS.getRepAsType().get()->getAsStructureType())
-        Record = RT->getDecl();
+        Record = RT->getOriginalDecl()->getDefinitionOrSelf();
       else if (const RecordType *UT = DS.getRepAsType().get()->getAsUnionType())
-        Record = UT->getDecl();
+        Record = UT->getOriginalDecl()->getDefinitionOrSelf();
 
       if (Record && getLangOpts().MicrosoftExt) {
         Diag(DS.getBeginLoc(), diag::ext_ms_anonymous_record)
@@ -5455,48 +5519,54 @@ InjectAnonymousStructOrUnionMembers(Sema &SemaRef, Scope *S, DeclContext *Owner,
     if ((isa<FieldDecl>(D) || isa<IndirectFieldDecl>(D)) &&
         cast<NamedDecl>(D)->getDeclName()) {
       ValueDecl *VD = cast<ValueDecl>(D);
-      if (CheckAnonMemberRedeclaration(SemaRef, S, Owner, VD->getDeclName(),
-                                       VD->getLocation(), AnonRecord->isUnion(),
-                                       SC)) {
-        // C++ [class.union]p2:
-        //   The names of the members of an anonymous union shall be
-        //   distinct from the names of any other entity in the
-        //   scope in which the anonymous union is declared.
+      // C++ [class.union]p2:
+      //   The names of the members of an anonymous union shall be
+      //   distinct from the names of any other entity in the
+      //   scope in which the anonymous union is declared.
+
+      bool FieldInvalid = CheckAnonMemberRedeclaration(
+          SemaRef, S, Owner, VD->getDeclName(), VD->getLocation(),
+          AnonRecord->isUnion(), SC);
+      if (FieldInvalid)
         Invalid = true;
-      } else {
-        // C++ [class.union]p2:
-        //   For the purpose of name lookup, after the anonymous union
-        //   definition, the members of the anonymous union are
-        //   considered to have been defined in the scope in which the
-        //   anonymous union is declared.
-        unsigned OldChainingSize = Chaining.size();
-        if (IndirectFieldDecl *IF = dyn_cast<IndirectFieldDecl>(VD))
-          Chaining.append(IF->chain_begin(), IF->chain_end());
-        else
-          Chaining.push_back(VD);
 
-        assert(Chaining.size() >= 2);
-        NamedDecl **NamedChain =
-          new (SemaRef.Context)NamedDecl*[Chaining.size()];
-        for (unsigned i = 0; i < Chaining.size(); i++)
-          NamedChain[i] = Chaining[i];
+      // Inject the IndirectFieldDecl even if invalid, because later
+      // diagnostics may depend on it being present, see findDefaultInitializer.
+
+      // C++ [class.union]p2:
+      //   For the purpose of name lookup, after the anonymous union
+      //   definition, the members of the anonymous union are
+      //   considered to have been defined in the scope in which the
+      //   anonymous union is declared.
+      unsigned OldChainingSize = Chaining.size();
+      if (IndirectFieldDecl *IF = dyn_cast<IndirectFieldDecl>(VD))
+        Chaining.append(IF->chain_begin(), IF->chain_end());
+      else
+        Chaining.push_back(VD);
 
-        IndirectFieldDecl *IndirectField = IndirectFieldDecl::Create(
-            SemaRef.Context, Owner, VD->getLocation(), VD->getIdentifier(),
-            VD->getType(), {NamedChain, Chaining.size()});
+      assert(Chaining.size() >= 2);
+      NamedDecl **NamedChain =
+          new (SemaRef.Context) NamedDecl *[Chaining.size()];
+      for (unsigned i = 0; i < Chaining.size(); i++)
+        NamedChain[i] = Chaining[i];
 
-        for (const auto *Attr : VD->attrs())
-          IndirectField->addAttr(Attr->clone(SemaRef.Context));
+      IndirectFieldDecl *IndirectField = IndirectFieldDecl::Create(
+          SemaRef.Context, Owner, VD->getLocation(), VD->getIdentifier(),
+          VD->getType(), {NamedChain, Chaining.size()});
 
-        IndirectField->setAccess(AS);
-        IndirectField->setImplicit();
-        SemaRef.PushOnScopeChains(IndirectField, S);
+      for (const auto *Attr : VD->attrs())
+        IndirectField->addAttr(Attr->clone(SemaRef.Context));
 
-        // That includes picking up the appropriate access specifier.
-        if (AS != AS_none) IndirectField->setAccess(AS);
+      IndirectField->setAccess(AS);
+      IndirectField->setImplicit();
+      IndirectField->setInvalidDecl(FieldInvalid);
+      SemaRef.PushOnScopeChains(IndirectField, S);
 
-        Chaining.resize(OldChainingSize);
-      }
+      // That includes picking up the appropriate access specifier.
+      if (AS != AS_none)
+        IndirectField->setAccess(AS);
+
+      Chaining.resize(OldChainingSize);
     }
   }
 
@@ -5762,7 +5832,7 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
   if (RecordDecl *OwningClass = dyn_cast<RecordDecl>(Owner)) {
     Anon = FieldDecl::Create(
         Context, OwningClass, DS.getBeginLoc(), Record->getLocation(),
-        /*IdentifierInfo=*/nullptr, Context.getTypeDeclType(Record), TInfo,
+        /*IdentifierInfo=*/nullptr, Context.getCanonicalTagType(Record), TInfo,
         /*BitWidth=*/nullptr, /*Mutable=*/false,
         /*InitStyle=*/ICIS_NoInit);
     Anon->setAccess(AS);
@@ -5782,7 +5852,7 @@ Decl *Sema::BuildAnonymousStructOrUnion(Scope *S, DeclSpec &DS,
 
     Anon = VarDecl::Create(Context, Owner, DS.getBeginLoc(),
                            Record->getLocation(), /*IdentifierInfo=*/nullptr,
-                           Context.getTypeDeclType(Record), TInfo, SC);
+                           Context.getCanonicalTagType(Record), TInfo, SC);
     if (Invalid)
       Anon->setInvalidDecl();
 
@@ -5845,7 +5915,7 @@ Decl *Sema::BuildMicrosoftCAnonymousStruct(Scope *S, DeclSpec &DS,
   assert(TInfo && "couldn't build declarator info for anonymous struct");
 
   auto *ParentDecl = cast<RecordDecl>(CurContext);
-  QualType RecTy = Context.getTypeDeclType(Record);
+  CanQualType RecTy = Context.getCanonicalTagType(Record);
 
   // Create a declaration for this anonymous struct.
   NamedDecl *Anon =
@@ -5964,14 +6034,14 @@ Sema::GetNameFromUnqualifiedId(const UnqualifiedId &Name) {
       return DeclarationNameInfo();
 
     // Determine the type of the class being constructed.
-    QualType CurClassType = Context.getTypeDeclType(CurClass);
+    CanQualType CurClassType = Context.getCanonicalTagType(CurClass);
 
     // FIXME: Check two things: that the template-id names the same type as
     // CurClassType, and that the template-id does not occur when the name
     // was qualified.
 
-    NameInfo.setName(Context.DeclarationNames.getCXXConstructorName(
-                                    Context.getCanonicalType(CurClassType)));
+    NameInfo.setName(
+        Context.DeclarationNames.getCXXConstructorName(CurClassType));
     // FIXME: should we retrieve TypeSourceInfo?
     NameInfo.setNamedTypeInfo(nullptr);
     return NameInfo;
@@ -6259,8 +6329,9 @@ bool Sema::diagnoseQualifiedDeclaration(CXXScopeSpec &SS, DeclContext *DC,
     // that's the case, then drop this declaration entirely.
     if ((Name.getNameKind() == DeclarationName::CXXConstructorName ||
          Name.getNameKind() == DeclarationName::CXXDestructorName) &&
-        !Context.hasSameType(Name.getCXXNameType(),
-                             Context.getTypeDeclType(cast<CXXRecordDecl>(Cur))))
+        !Context.hasSameType(
+            Name.getCXXNameType(),
+            Context.getCanonicalTagType(cast<CXXRecordDecl>(Cur))))
       return true;
 
     return false;
@@ -6280,36 +6351,48 @@ bool Sema::diagnoseQualifiedDeclaration(CXXScopeSpec &SS, DeclContext *DC,
         << FixItHint::CreateRemoval(TemplateId->TemplateKWLoc);
 
   NestedNameSpecifierLoc SpecLoc(SS.getScopeRep(), SS.location_data());
-  do {
-    if (TypeLoc TL = SpecLoc.getTypeLoc()) {
-      if (SourceLocation TemplateKeywordLoc = TL.getTemplateKeywordLoc();
-          TemplateKeywordLoc.isValid())
-        Diag(Loc, diag::ext_template_after_declarative_nns)
-            << FixItHint::CreateRemoval(TemplateKeywordLoc);
-    }
-
-    if (const Type *T = SpecLoc.getNestedNameSpecifier()->getAsType()) {
-      if (const auto *TST = T->getAsAdjusted<TemplateSpecializationType>()) {
-        // C++23 [expr.prim.id.qual]p3:
-        //   [...] If a nested-name-specifier N is declarative and has a
-        //   simple-template-id with a template argument list A that involves a
-        //   template parameter, let T be the template nominated by N without A.
-        //   T shall be a class template.
-        if (TST->isDependentType() && TST->isTypeAlias())
-          Diag(Loc, diag::ext_alias_template_in_declarative_nns)
-              << SpecLoc.getLocalSourceRange();
-      } else if (T->isDecltypeType() || T->getAsAdjusted<PackIndexingType>()) {
-        // C++23 [expr.prim.id.qual]p2:
-        //   [...] A declarative nested-name-specifier shall not have a
-        //   computed-type-specifier.
-        //
-        // CWG2858 changed this from 'decltype-specifier' to
-        // 'computed-type-specifier'.
-        Diag(Loc, diag::err_computed_type_in_declarative_nns)
-            << T->isDecltypeType() << SpecLoc.getTypeLoc().getSourceRange();
-      }
+  for (TypeLoc TL = SpecLoc.getAsTypeLoc(), NextTL; TL;
+       TL = std::exchange(NextTL, TypeLoc())) {
+    SourceLocation TemplateKeywordLoc;
+    switch (TL.getTypeLocClass()) {
+    case TypeLoc::TemplateSpecialization: {
+      auto TST = TL.castAs<TemplateSpecializationTypeLoc>();
+      TemplateKeywordLoc = TST.getTemplateKeywordLoc();
+      if (auto *T = TST.getTypePtr(); T->isDependentType() && T->isTypeAlias())
+        Diag(Loc, diag::ext_alias_template_in_declarative_nns)
+            << TST.getLocalSourceRange();
+      break;
+    }
+    case TypeLoc::Decltype:
+    case TypeLoc::PackIndexing: {
+      const Type *T = TL.getTypePtr();
+      // C++23 [expr.prim.id.qual]p2:
+      //   [...] A declarative nested-name-specifier shall not have a
+      //   computed-type-specifier.
+      //
+      // CWG2858 changed this from 'decltype-specifier' to
+      // 'computed-type-specifier'.
+      Diag(Loc, diag::err_computed_type_in_declarative_nns)
+          << T->isDecltypeType() << TL.getSourceRange();
+      break;
+    }
+    case TypeLoc::DependentName:
+      NextTL =
+          TL.castAs<DependentNameTypeLoc>().getQualifierLoc().getAsTypeLoc();
+      break;
+    case TypeLoc::DependentTemplateSpecialization: {
+      auto TST = TL.castAs<DependentTemplateSpecializationTypeLoc>();
+      TemplateKeywordLoc = TST.getTemplateKeywordLoc();
+      NextTL = TST.getQualifierLoc().getAsTypeLoc();
+      break;
+    }
+    default:
+      break;
     }
-  } while ((SpecLoc = SpecLoc.getPrefix()));
+    if (TemplateKeywordLoc.isValid())
+      Diag(Loc, diag::ext_template_after_declarative_nns)
+          << FixItHint::CreateRemoval(TemplateKeywordLoc);
+  }
 
   return false;
 }
@@ -9046,9 +9129,8 @@ bool Sema::AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
 
     if (Name.getNameKind() == DeclarationName::CXXDestructorName) {
       // We really want to find the base class destructor here.
-      QualType T = Context.getTypeDeclType(BaseRecord);
-      CanQualType CT = Context.getCanonicalType(T);
-      Name = Context.DeclarationNames.getCXXDestructorName(CT);
+      Name = Context.DeclarationNames.getCXXDestructorName(
+          Context.getCanonicalTagType(BaseRecord));
     }
 
     for (NamedDecl *BaseND : BaseRecord->lookup(Name)) {
@@ -9734,7 +9816,8 @@ static void checkIsValidOpenCLKernelParameter(
 
   // At this point we already handled everything except of a RecordType.
   assert(PT->isRecordType() && "Unexpected type.");
-  const RecordDecl *PD = PT->castAs<RecordType>()->getDecl();
+  const RecordDecl *PD =
+      PT->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
   VisitStack.push_back(PD);
   assert(VisitStack.back() && "First decl null?");
 
@@ -9762,7 +9845,9 @@ static void checkIsValidOpenCLKernelParameter(
              "Unexpected type.");
       const Type *FieldRecTy = FieldTy->getPointeeOrArrayElementType();
 
-      RD = FieldRecTy->castAs<RecordType>()->getDecl();
+      RD = FieldRecTy->castAs<RecordType>()
+               ->getOriginalDecl()
+               ->getDefinitionOrSelf();
     } else {
       RD = cast<RecordDecl>(Next);
     }
@@ -10768,7 +10853,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
         // redeclaration lookup found nothing at all. Diagnose that now;
         // nothing will diagnose that error later.
         if (isFriend &&
-            (D.getCXXScopeSpec().getScopeRep()->isDependent() ||
+            (D.getCXXScopeSpec().getScopeRep().isDependent() ||
              (!Previous.empty() && CurContext->isDependentContext()))) {
           // ignore these
         } else if (NewFD->isCPUDispatchMultiVersion() ||
@@ -12256,6 +12341,9 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
   if (NewFD->hasAttr<SYCLKernelEntryPointAttr>())
     SYCL().CheckSYCLEntryPointFunctionDecl(NewFD);
 
+  if (NewFD->hasAttr<SYCLExternalAttr>())
+    SYCL().CheckSYCLExternalFunctionDecl(NewFD);
+
   // Semantic checking for this function declaration (in isolation).
 
   if (getLangOpts().CPlusPlus) {
@@ -12272,11 +12360,11 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
       // template struct A<B>;
       if (NewFD->getFriendObjectKind() == Decl::FriendObjectKind::FOK_None ||
           !Destructor->getFunctionObjectParameterType()->isDependentType()) {
-        CXXRecordDecl *Record = Destructor->getParent();
-        QualType ClassType = Context.getTypeDeclType(Record);
+        CanQualType ClassType =
+            Context.getCanonicalTagType(Destructor->getParent());
 
-        DeclarationName Name = Context.DeclarationNames.getCXXDestructorName(
-            Context.getCanonicalType(ClassType));
+        DeclarationName Name =
+            Context.DeclarationNames.getCXXDestructorName(ClassType);
         if (NewFD->getDeclName() != Name) {
           Diag(NewFD->getLocation(), diag::err_destructor_name);
           NewFD->setInvalidDecl();
@@ -12444,6 +12532,13 @@ void Sema::CheckMain(FunctionDecl *FD, const DeclSpec &DS) {
     return;
   }
 
+  if (FD->hasAttr<SYCLExternalAttr>()) {
+    Diag(FD->getLocation(), diag::err_sycl_external_invalid_main)
+        << FD->getAttr<SYCLExternalAttr>();
+    FD->setInvalidDecl();
+    return;
+  }
+
   // Functions named main in hlsl are default entries, but don't have specific
   // signatures they are required to conform to.
   if (getLangOpts().HLSL)
@@ -13280,7 +13375,8 @@ struct DiagNonTrivalCUnionDefaultInitializeVisitor
   }
 
   void visitStruct(QualType QT, const FieldDecl *FD, bool InNonTrivialUnion) {
-    const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
+    const RecordDecl *RD =
+        QT->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->isUnion()) {
       if (OrigLoc.isValid()) {
         bool IsUnion = false;
@@ -13346,7 +13442,8 @@ struct DiagNonTrivalCUnionDestructedTypeVisitor
   }
 
   void visitStruct(QualType QT, const FieldDecl *FD, bool InNonTrivialUnion) {
-    const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
+    const RecordDecl *RD =
+        QT->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->isUnion()) {
       if (OrigLoc.isValid()) {
         bool IsUnion = false;
@@ -13411,7 +13508,8 @@ struct DiagNonTrivalCUnionCopyVisitor
   }
 
   void visitStruct(QualType QT, const FieldDecl *FD, bool InNonTrivialUnion) {
-    const RecordDecl *RD = QT->castAs<RecordType>()->getDecl();
+    const RecordDecl *RD =
+        QT->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
     if (RD->isUnion()) {
       if (OrigLoc.isValid()) {
         bool IsUnion = false;
@@ -14384,7 +14482,9 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl) {
     if (getLangOpts().CPlusPlus && Var->hasLocalStorage()) {
       if (const RecordType *Record
             = Context.getBaseElementType(Type)->getAs<RecordType>()) {
-        CXXRecordDecl *CXXRecord = cast<CXXRecordDecl>(Record->getDecl());
+        CXXRecordDecl *CXXRecord =
+            cast<CXXRecordDecl>(Record->getOriginalDecl())
+                ->getDefinitionOrSelf();
         // Mark the function (if we're in one) for further checking even if the
         // looser rules of C++11 do not require such checks, so that we can
         // diagnose incompatibilities with C++98.
@@ -15763,17 +15863,18 @@ Sema::CheckForFunctionRedefinition(FunctionDecl *FD,
   if (TypoCorrectedFunctionDefinitions.count(Definition))
     return;
 
-  // If we don't have a visible definition of the function, and it's inline or
-  // a template, skip the new definition.
-  if (SkipBody && !hasVisibleDefinition(Definition) &&
+  bool DefinitionVisible = false;
+  if (SkipBody && isRedefinitionAllowedFor(Definition, DefinitionVisible) &&
       (Definition->getFormalLinkage() == Linkage::Internal ||
        Definition->isInlined() || Definition->getDescribedFunctionTemplate() ||
        Definition->getNumTemplateParameterLists())) {
     SkipBody->ShouldSkip = true;
     SkipBody->Previous = const_cast<FunctionDecl*>(Definition);
-    if (auto *TD = Definition->getDescribedFunctionTemplate())
-      makeMergedDefinitionVisible(TD);
-    makeMergedDefinitionVisible(const_cast<FunctionDecl*>(Definition));
+    if (!DefinitionVisible) {
+      if (auto *TD = Definition->getDescribedFunctionTemplate())
+        makeMergedDefinitionVisible(TD);
+      makeMergedDefinitionVisible(const_cast<FunctionDecl *>(Definition));
+    }
     return;
   }
 
@@ -16149,10 +16250,6 @@ Decl *Sema::ActOnSkippedFunctionBody(Decl *Decl) {
   return Decl;
 }
 
-Decl *Sema::ActOnFinishFunctionBody(Decl *D, Stmt *BodyArg) {
-  return ActOnFinishFunctionBody(D, BodyArg, /*IsInstantiation=*/false);
-}
-
 /// RAII object that pops an ExpressionEvaluationContext when exiting a function
 /// body.
 class ExitFunctionBodyRAII {
@@ -16223,8 +16320,8 @@ void Sema::CheckCoroutineWrapper(FunctionDecl *FD) {
     Diag(FD->getLocation(), diag::err_coroutine_return_type) << RD;
 }
 
-Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
-                                    bool IsInstantiation) {
+Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body, bool IsInstantiation,
+                                    bool RetainFunctionScopeInfo) {
   FunctionScopeInfo *FSI = getCurFunction();
   FunctionDecl *FD = dcl ? dcl->getAsFunction() : nullptr;
 
@@ -16280,6 +16377,14 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
     }
   }
 
+  if (FD && !FD->isInvalidDecl() && FD->hasAttr<SYCLExternalAttr>()) {
+    SYCLExternalAttr *SEAttr = FD->getAttr<SYCLExternalAttr>();
+    if (FD->isDeletedAsWritten())
+      Diag(SEAttr->getLocation(),
+           diag::err_sycl_external_invalid_deleted_function)
+          << SEAttr;
+  }
+
   {
     // Do not call PopExpressionEvaluationContext() if it is a lambda because
     // one is already popped when finishing the lambda in BuildLambdaExpr().
@@ -16681,7 +16786,8 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
   if (!IsInstantiation)
     PopDeclContext();
 
-  PopFunctionScopeInfo(ActivePolicy, dcl);
+  if (!RetainFunctionScopeInfo)
+    PopFunctionScopeInfo(ActivePolicy, dcl);
   // If any errors have occurred, clear out any temporaries that may have
   // been leftover. This ensures that these temporaries won't be picked up for
   // deletion in some later function.
@@ -17483,6 +17589,7 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
 
   // FIXME: Check member specializations more carefully.
   bool isMemberSpecialization = false;
+  bool IsInjectedClassName = false;
   bool Invalid = false;
 
   // We only need to do this matching if we have template parameters
@@ -17948,8 +18055,7 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
     // see http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_active.html#407
     if (getLangOpts().CPlusPlus) {
       if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(PrevDecl)) {
-        if (const TagType *TT = TD->getUnderlyingType()->getAs<TagType>()) {
-          TagDecl *Tag = TT->getDecl();
+        if (TagDecl *Tag = TD->getUnderlyingType()->getAsTagDecl()) {
           if (Tag->getDeclName() == Name &&
               Tag->getDeclContext()->getRedeclContext()
                           ->Equals(TD->getDeclContext()->getRedeclContext())) {
@@ -17959,6 +18065,15 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
             Previous.resolveKind();
           }
         }
+      } else if (auto *RD = dyn_cast<CXXRecordDecl>(PrevDecl);
+                 RD && RD->isInjectedClassName()) {
+        // If lookup found the injected class name, the previous declaration is
+        // the class being injected into.
+        PrevDecl = cast<TagDecl>(RD->getDeclContext());
+        Previous.clear();
+        Previous.addDecl(PrevDecl);
+        Previous.resolveKind();
+        IsInjectedClassName = true;
       }
     }
 
@@ -18082,78 +18197,83 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
 
           // Diagnose attempts to redefine a tag.
           if (TUK == TagUseKind::Definition) {
-            if (NamedDecl *Def = PrevTagDecl->getDefinition()) {
-              // If we're defining a specialization and the previous definition
-              // is from an implicit instantiation, don't emit an error
-              // here; we'll catch this in the general case below.
-              bool IsExplicitSpecializationAfterInstantiation = false;
-              if (isMemberSpecialization) {
-                if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(Def))
-                  IsExplicitSpecializationAfterInstantiation =
-                    RD->getTemplateSpecializationKind() !=
-                    TSK_ExplicitSpecialization;
-                else if (EnumDecl *ED = dyn_cast<EnumDecl>(Def))
-                  IsExplicitSpecializationAfterInstantiation =
-                    ED->getTemplateSpecializationKind() !=
-                    TSK_ExplicitSpecialization;
-              }
-
-              // Note that clang allows ODR-like semantics for ObjC/C, i.e., do
-              // not keep more that one definition around (merge them). However,
-              // ensure the decl passes the structural compatibility check in
-              // C11 6.2.7/1 (or 6.1.2.6/1 in C89).
-              NamedDecl *Hidden = nullptr;
-              if (SkipBody &&
-                  (!hasVisibleDefinition(Def, &Hidden) || getLangOpts().C23)) {
-                // There is a definition of this tag, but it is not visible. We
-                // explicitly make use of C++'s one definition rule here, and
-                // assume that this definition is identical to the hidden one
-                // we already have. Make the existing definition visible and
-                // use it in place of this one.
-                if (!getLangOpts().CPlusPlus) {
-                  // Postpone making the old definition visible until after we
-                  // complete parsing the new one and do the structural
-                  // comparison.
-                  SkipBody->CheckSameAsPrevious = true;
-                  SkipBody->New = createTagFromNewDecl();
-                  SkipBody->Previous = Def;
-
-                  ProcessDeclAttributeList(S, SkipBody->New, Attrs);
-                  return Def;
-                } else {
-                  SkipBody->ShouldSkip = true;
-                  SkipBody->Previous = Def;
-                  makeMergedDefinitionVisible(Hidden);
-                  // Carry on and handle it like a normal definition. We'll
-                  // skip starting the definition later.
-                }
-              } else if (!IsExplicitSpecializationAfterInstantiation) {
-                // A redeclaration in function prototype scope in C isn't
-                // visible elsewhere, so merely issue a warning.
-                if (!getLangOpts().CPlusPlus && S->containedInPrototypeScope())
-                  Diag(NameLoc, diag::warn_redefinition_in_param_list) << Name;
-                else
-                  Diag(NameLoc, diag::err_redefinition) << Name;
-                notePreviousDefinition(Def,
-                                       NameLoc.isValid() ? NameLoc : KWLoc);
-                // If this is a redefinition, recover by making this
-                // struct be anonymous, which will make any later
-                // references get the previous definition.
-                Name = nullptr;
-                Previous.clear();
-                Invalid = true;
-              }
-            } else {
+            if (TagDecl *Def = PrevTagDecl->getDefinition()) {
               // If the type is currently being defined, complain
               // about a nested redefinition.
-              auto *TD = Context.getTagDeclType(PrevTagDecl)->getAsTagDecl();
-              if (TD->isBeingDefined()) {
+              if (Def->isBeingDefined()) {
                 Diag(NameLoc, diag::err_nested_redefinition) << Name;
                 Diag(PrevTagDecl->getLocation(),
                      diag::note_previous_definition);
                 Name = nullptr;
                 Previous.clear();
                 Invalid = true;
+              } else {
+                // If we're defining a specialization and the previous
+                // definition is from an implicit instantiation, don't emit an
+                // error here; we'll catch this in the general case below.
+                bool IsExplicitSpecializationAfterInstantiation = false;
+                if (isMemberSpecialization) {
+                  if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(Def))
+                    IsExplicitSpecializationAfterInstantiation =
+                        RD->getTemplateSpecializationKind() !=
+                        TSK_ExplicitSpecialization;
+                  else if (EnumDecl *ED = dyn_cast<EnumDecl>(Def))
+                    IsExplicitSpecializationAfterInstantiation =
+                        ED->getTemplateSpecializationKind() !=
+                        TSK_ExplicitSpecialization;
+                }
+
+                // Note that clang allows ODR-like semantics for ObjC/C, i.e.,
+                // do not keep more that one definition around (merge them).
+                // However, ensure the decl passes the structural compatibility
+                // check in C11 6.2.7/1 (or 6.1.2.6/1 in C89).
+                NamedDecl *Hidden = nullptr;
+                bool HiddenDefVisible = false;
+                if (SkipBody &&
+                    (isRedefinitionAllowedFor(Def, &Hidden, HiddenDefVisible) ||
+                     getLangOpts().C23)) {
+                  // There is a definition of this tag, but it is not visible.
+                  // We explicitly make use of C++'s one definition rule here,
+                  // and assume that this definition is identical to the hidden
+                  // one we already have. Make the existing definition visible
+                  // and use it in place of this one.
+                  if (!getLangOpts().CPlusPlus) {
+                    // Postpone making the old definition visible until after we
+                    // complete parsing the new one and do the structural
+                    // comparison.
+                    SkipBody->CheckSameAsPrevious = true;
+                    SkipBody->New = createTagFromNewDecl();
+                    SkipBody->Previous = Def;
+
+                    ProcessDeclAttributeList(S, SkipBody->New, Attrs);
+                    return Def;
+                  }
+
+                  SkipBody->ShouldSkip = true;
+                  SkipBody->Previous = Def;
+                  if (!HiddenDefVisible && Hidden)
+                    makeMergedDefinitionVisible(Hidden);
+                  // Carry on and handle it like a normal definition. We'll
+                  // skip starting the definition later.
+
+                } else if (!IsExplicitSpecializationAfterInstantiation) {
+                  // A redeclaration in function prototype scope in C isn't
+                  // visible elsewhere, so merely issue a warning.
+                  if (!getLangOpts().CPlusPlus &&
+                      S->containedInPrototypeScope())
+                    Diag(NameLoc, diag::warn_redefinition_in_param_list)
+                        << Name;
+                  else
+                    Diag(NameLoc, diag::err_redefinition) << Name;
+                  notePreviousDefinition(Def,
+                                         NameLoc.isValid() ? NameLoc : KWLoc);
+                  // If this is a redefinition, recover by making this
+                  // struct be anonymous, which will make any later
+                  // references get the previous definition.
+                  Name = nullptr;
+                  Previous.clear();
+                  Invalid = true;
+                }
               }
             }
 
@@ -18324,14 +18444,14 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
       (IsTypeSpecifier || IsTemplateParamOrArg) &&
       TUK == TagUseKind::Definition) {
     Diag(New->getLocation(), diag::err_type_defined_in_type_specifier)
-      << Context.getTagDeclType(New);
+        << Context.getCanonicalTagType(New);
     Invalid = true;
   }
 
   if (!Invalid && getLangOpts().CPlusPlus && TUK == TagUseKind::Definition &&
       DC->getDeclKind() == Decl::Enum) {
     Diag(New->getLocation(), diag::err_type_defined_in_enum)
-      << Context.getTagDeclType(New);
+        << Context.getCanonicalTagType(New);
     Invalid = true;
   }
 
@@ -18412,7 +18532,8 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
       // In C23 mode, if the declaration is complete, we do not want to
       // diagnose.
       if (!getLangOpts().C23 || TUK != TagUseKind::Definition)
-        Diag(Loc, diag::warn_decl_in_param_list) << Context.getTagDeclType(New);
+        Diag(Loc, diag::warn_decl_in_param_list)
+            << Context.getCanonicalTagType(New);
     }
   }
 
@@ -18444,7 +18565,7 @@ Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK, SourceLocation KWLoc,
   AddPragmaAttributes(S, New);
 
   // If this has an identifier, add it to the scope stack.
-  if (TUK == TagUseKind::Friend) {
+  if (TUK == TagUseKind::Friend || IsInjectedClassName) {
     // We might be replacing an existing declaration in the lookup tables;
     // if so, borrow its access specifier.
     if (PrevDecl)
@@ -18564,14 +18685,12 @@ void Sema::ActOnStartCXXMemberDeclarations(
   //   as if it were a public member name.
   CXXRecordDecl *InjectedClassName = CXXRecordDecl::Create(
       Context, Record->getTagKind(), CurContext, Record->getBeginLoc(),
-      Record->getLocation(), Record->getIdentifier(),
-      /*PrevDecl=*/nullptr,
-      /*DelayTypeCreation=*/true);
-  Context.getTypeDeclType(InjectedClassName, Record);
+      Record->getLocation(), Record->getIdentifier());
   InjectedClassName->setImplicit();
   InjectedClassName->setAccess(AS_public);
   if (ClassTemplateDecl *Template = Record->getDescribedClassTemplate())
       InjectedClassName->setDescribedClassTemplate(Template);
+
   PushOnScopeChains(InjectedClassName, S);
   assert(InjectedClassName->isInjectedClassName() &&
          "Broken injected-class-name");
@@ -19001,7 +19120,7 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
   if (!InvalidDecl && getLangOpts().CPlusPlus) {
     if (Record->isUnion()) {
       if (const RecordType *RT = EltTy->getAs<RecordType>()) {
-        CXXRecordDecl* RDecl = cast<CXXRecordDecl>(RT->getDecl());
+        CXXRecordDecl *RDecl = cast<CXXRecordDecl>(RT->getOriginalDecl());
         if (RDecl->getDefinition()) {
           // C++ [class.union]p1: An object of a class with a non-trivial
           // constructor, a non-trivial copy constructor, a non-trivial
@@ -19067,7 +19186,8 @@ bool Sema::CheckNontrivialField(FieldDecl *FD) {
 
   QualType EltTy = Context.getBaseElementType(FD->getType());
   if (const RecordType *RT = EltTy->getAs<RecordType>()) {
-    CXXRecordDecl *RDecl = cast<CXXRecordDecl>(RT->getDecl());
+    CXXRecordDecl *RDecl =
+        cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
     if (RDecl->getDefinition()) {
       // We check for copy constructors before constructors
       // because otherwise we'll never get complaints about
@@ -19390,7 +19510,7 @@ bool Sema::EntirelyFunctionPointers(const RecordDecl *Record) {
     }
     // If a member is a struct entirely of function pointers, that counts too.
     if (const RecordType *RT = FieldType->getAs<RecordType>()) {
-      const RecordDecl *Record = RT->getDecl();
+      const RecordDecl *Record = RT->getOriginalDecl()->getDefinitionOrSelf();
       if (Record->isStruct() && EntirelyFunctionPointers(Record))
         return true;
     }
@@ -19548,7 +19668,9 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
       EnclosingDecl->setInvalidDecl();
       continue;
     } else if (const RecordType *FDTTy = FDTy->getAs<RecordType>()) {
-      if (Record && FDTTy->getDecl()->hasFlexibleArrayMember()) {
+      if (Record && FDTTy->getOriginalDecl()
+                        ->getDefinitionOrSelf()
+                        ->hasFlexibleArrayMember()) {
         // A type which contains a flexible array member is considered to be a
         // flexible array member.
         Record->setHasFlexibleArrayMember(true);
@@ -19574,9 +19696,10 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
         // Ivars can not have abstract class types
         FD->setInvalidDecl();
       }
-      if (Record && FDTTy->getDecl()->hasObjectMember())
+      const RecordDecl *RD = FDTTy->getOriginalDecl()->getDefinitionOrSelf();
+      if (Record && RD->hasObjectMember())
         Record->setHasObjectMember(true);
-      if (Record && FDTTy->getDecl()->hasVolatileMember())
+      if (Record && RD->hasVolatileMember())
         Record->setHasVolatileMember(true);
     } else if (FDTy->isObjCObjectType()) {
       /// A field cannot be an Objective-c object
@@ -19607,8 +19730,10 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
         Record->setHasObjectMember(true);
       else if (Context.getAsArrayType(FD->getType())) {
         QualType BaseType = Context.getBaseElementType(FD->getType());
-        if (BaseType->isRecordType() &&
-            BaseType->castAs<RecordType>()->getDecl()->hasObjectMember())
+        if (BaseType->isRecordType() && BaseType->castAs<RecordType>()
+                                            ->getOriginalDecl()
+                                            ->getDefinitionOrSelf()
+                                            ->hasObjectMember())
           Record->setHasObjectMember(true);
         else if (BaseType->isObjCObjectPointerType() ||
                  BaseType.isObjCGCStrong())
@@ -19641,7 +19766,9 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
       }
 
       if (const auto *RT = FT->getAs<RecordType>()) {
-        if (RT->getDecl()->getArgPassingRestrictions() ==
+        if (RT->getOriginalDecl()
+                ->getDefinitionOrSelf()
+                ->getArgPassingRestrictions() ==
             RecordArgPassingKind::CanNeverPassInRegs)
           Record->setArgPassingRestrictions(
               RecordArgPassingKind::CanNeverPassInRegs);
@@ -20179,9 +20306,10 @@ Decl *Sema::ActOnEnumConstant(Scope *S, Decl *theEnumDecl, Decl *lastEnumConst,
   // different from T:
   // - every enumerator of every member of class T that is an unscoped
   // enumerated type
-  if (getLangOpts().CPlusPlus && !TheEnumDecl->isScoped())
-    DiagnoseClassNameShadow(TheEnumDecl->getDeclContext(),
-                            DeclarationNameInfo(Id, IdLoc));
+  if (getLangOpts().CPlusPlus && !TheEnumDecl->isScoped() &&
+      DiagnoseClassNameShadow(TheEnumDecl->getDeclContext(),
+                              DeclarationNameInfo(Id, IdLoc)))
+    return nullptr;
 
   EnumConstantDecl *New =
     CheckEnumConstant(TheEnumDecl, LastEnumConst, IdLoc, Id, Val);
@@ -20407,7 +20535,7 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceRange BraceRange,
                          Decl *EnumDeclX, ArrayRef<Decl *> Elements, Scope *S,
                          const ParsedAttributesView &Attrs) {
   EnumDecl *Enum = cast<EnumDecl>(EnumDeclX);
-  QualType EnumType = Context.getTypeDeclType(Enum);
+  CanQualType EnumType = Context.getCanonicalTagType(Enum);
 
   ProcessDeclAttributeList(S, Enum, Attrs);
   ProcessAPINotes(Enum);
@@ -20750,3 +20878,13 @@ bool Sema::shouldIgnoreInHostDeviceCheck(FunctionDecl *Callee) {
   return LangOpts.CUDA && !LangOpts.CUDAIsDevice &&
          CUDA().IdentifyTarget(Callee) == CUDAFunctionTarget::Global;
 }
+
+bool Sema::isRedefinitionAllowedFor(NamedDecl *D, NamedDecl **Suggested,
+                                    bool &Visible) {
+  Visible = hasVisibleDefinition(D, Suggested);
+  // The redefinition of D in the **current** TU is allowed if D is invisible or
+  // D is defined in the global module of other module units. We didn't check if
+  // it is in global module as, we'll check the redefinition in named module
+  // later with better diagnostic message.
+  return D->isInAnotherModuleUnit() || !Visible;
+}
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index f5f18b06c2fff..3685bcefbf764 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -177,7 +177,7 @@ static bool threadSafetyCheckIsSmartPointer(Sema &S, const RecordType* RT) {
     return !Result.empty();
   };
 
-  const RecordDecl *Record = RT->getDecl();
+  const RecordDecl *Record = RT->getOriginalDecl()->getDefinitionOrSelf();
   bool foundStarOperator = IsOverloadedOperatorPresent(Record, OO_Star);
   bool foundArrowOperator = IsOverloadedOperatorPresent(Record, OO_Arrow);
   if (foundStarOperator && foundArrowOperator)
@@ -271,7 +271,8 @@ static bool checkRecordTypeForCapability(Sema &S, QualType Ty) {
   if (threadSafetyCheckIsSmartPointer(S, RT))
     return true;
 
-  return checkRecordDeclForAttr<CapabilityAttr>(RT->getDecl());
+  return checkRecordDeclForAttr<CapabilityAttr>(
+      RT->getOriginalDecl()->getDefinitionOrSelf());
 }
 
 static bool checkRecordTypeForScopedCapability(Sema &S, QualType Ty) {
@@ -284,7 +285,8 @@ static bool checkRecordTypeForScopedCapability(Sema &S, QualType Ty) {
   if (RT->isIncompleteType())
     return true;
 
-  return checkRecordDeclForAttr<ScopedLockableAttr>(RT->getDecl());
+  return checkRecordDeclForAttr<ScopedLockableAttr>(
+      RT->getOriginalDecl()->getDefinitionOrSelf());
 }
 
 static bool checkTypedefTypeForCapability(QualType Ty) {
@@ -1254,8 +1256,8 @@ bool Sema::isValidPointerAttrType(QualType T, bool RefOkay) {
   // The nonnull attribute, and other similar attributes, can be applied to a
   // transparent union that contains a pointer type.
   if (const RecordType *UT = T->getAsUnionType()) {
-    if (UT && UT->getDecl()->hasAttr<TransparentUnionAttr>()) {
-      RecordDecl *UD = UT->getDecl();
+    RecordDecl *UD = UT->getOriginalDecl()->getDefinitionOrSelf();
+    if (UD->hasAttr<TransparentUnionAttr>()) {
       for (const auto *I : UD->fields()) {
         QualType QT = I->getType();
         if (QT->isAnyPointerType() || QT->isBlockPointerType())
@@ -1987,6 +1989,11 @@ void clang::inferNoReturnAttr(Sema &S, const Decl *D) {
       isKnownToAlwaysThrow(FD)) {
     NonConstFD->addAttr(InferredNoReturnAttr::CreateImplicit(S.Context));
 
+    // [[noreturn]] can only be added to lambdas since C++23
+    if (const auto *MD = dyn_cast<CXXMethodDecl>(FD);
+        MD && !S.getLangOpts().CPlusPlus23 && isLambdaCallOperator(MD))
+      return;
+
     // Emit a diagnostic suggesting the function being marked [[noreturn]].
     S.Diag(FD->getLocation(), diag::warn_suggest_noreturn_function)
         << /*isFunction=*/0 << FD;
@@ -3560,7 +3567,9 @@ static void handleFormatArgAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
   Ty = getFunctionOrMethodResultType(D);
   // replace instancetype with the class type
-  auto Instancetype = S.Context.getObjCInstanceTypeDecl()->getTypeForDecl();
+  auto *Instancetype = cast<TypedefType>(S.Context.getTypedefType(
+      ElaboratedTypeKeyword::None, /*Qualifier=*/std::nullopt,
+      S.Context.getObjCInstanceTypeDecl()));
   if (Ty->getAs<TypedefType>() == Instancetype)
     if (auto *OMD = dyn_cast<ObjCMethodDecl>(D))
       if (auto *Interface = OMD->getClassInterface())
@@ -4154,7 +4163,10 @@ static void handleTransparentUnionAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   RecordDecl *RD = nullptr;
   const auto *TD = dyn_cast<TypedefNameDecl>(D);
   if (TD && TD->getUnderlyingType()->isUnionType())
-    RD = TD->getUnderlyingType()->getAsUnionType()->getDecl();
+    RD = TD->getUnderlyingType()
+             ->getAsUnionType()
+             ->getOriginalDecl()
+             ->getDefinitionOrSelf();
   else
     RD = dyn_cast<RecordDecl>(D);
 
@@ -4507,7 +4519,7 @@ void Sema::CheckAlignasUnderalignment(Decl *D) {
   if (const auto *VD = dyn_cast<ValueDecl>(D)) {
     UnderlyingTy = DiagTy = VD->getType();
   } else {
-    UnderlyingTy = DiagTy = Context.getTagDeclType(cast<TagDecl>(D));
+    UnderlyingTy = DiagTy = Context.getCanonicalTagType(cast<TagDecl>(D));
     if (const auto *ED = dyn_cast<EnumDecl>(D))
       UnderlyingTy = ED->getIntegerType();
   }
@@ -7054,6 +7066,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_EnumExtensibility:
     handleEnumExtensibilityAttr(S, D, AL);
     break;
+  case ParsedAttr::AT_SYCLExternal:
+    handleSimpleAttribute<SYCLExternalAttr>(S, D, AL);
+    break;
   case ParsedAttr::AT_SYCLKernelEntryPoint:
     S.SYCL().handleKernelEntryPointAttr(D, AL);
     break;
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index f5b4614576086..bb57968830a65 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TypeLocBuilder.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTMutationListener.h"
@@ -1101,15 +1102,13 @@ static std::string printTemplateArgs(const PrintingPolicy &PrintingPolicy,
   return std::string(OS.str());
 }
 
-static bool lookupStdTypeTraitMember(Sema &S, LookupResult &TraitMemberLookup,
-                                     SourceLocation Loc, StringRef Trait,
-                                     TemplateArgumentListInfo &Args,
-                                     unsigned DiagID) {
+static QualType getStdTrait(Sema &S, SourceLocation Loc, StringRef Trait,
+                            TemplateArgumentListInfo &Args, unsigned DiagID) {
   auto DiagnoseMissing = [&] {
     if (DiagID)
       S.Diag(Loc, DiagID) << printTemplateArgs(S.Context.getPrintingPolicy(),
                                                Args, /*Params*/ nullptr);
-    return true;
+    return QualType();
   };
 
   // FIXME: Factor out duplication with lookupPromiseType in SemaCoroutine.
@@ -1122,12 +1121,12 @@ static bool lookupStdTypeTraitMember(Sema &S, LookupResult &TraitMemberLookup,
   // missing specialization, because this can only fail if the user has been
   // declaring their own names in namespace std or we don't support the
   // standard library implementation in use.
-  LookupResult Result(S, &S.PP.getIdentifierTable().get(Trait),
-                      Loc, Sema::LookupOrdinaryName);
+  LookupResult Result(S, &S.PP.getIdentifierTable().get(Trait), Loc,
+                      Sema::LookupOrdinaryName);
   if (!S.LookupQualifiedName(Result, Std))
     return DiagnoseMissing();
   if (Result.isAmbiguous())
-    return true;
+    return QualType();
 
   ClassTemplateDecl *TraitTD = Result.getAsSingle<ClassTemplateDecl>();
   if (!TraitTD) {
@@ -1135,28 +1134,31 @@ static bool lookupStdTypeTraitMember(Sema &S, LookupResult &TraitMemberLookup,
     NamedDecl *Found = *Result.begin();
     S.Diag(Loc, diag::err_std_type_trait_not_class_template) << Trait;
     S.Diag(Found->getLocation(), diag::note_declared_at);
-    return true;
+    return QualType();
   }
 
   // Build the template-id.
-  QualType TraitTy = S.CheckTemplateIdType(TemplateName(TraitTD), Loc, Args);
+  QualType TraitTy = S.CheckTemplateIdType(ElaboratedTypeKeyword::None,
+                                           TemplateName(TraitTD), Loc, Args);
   if (TraitTy.isNull())
-    return true;
+    return QualType();
+
   if (!S.isCompleteType(Loc, TraitTy)) {
     if (DiagID)
       S.RequireCompleteType(
           Loc, TraitTy, DiagID,
           printTemplateArgs(S.Context.getPrintingPolicy(), Args,
                             TraitTD->getTemplateParameters()));
-    return true;
+    return QualType();
   }
+  return TraitTy;
+}
 
-  CXXRecordDecl *RD = TraitTy->getAsCXXRecordDecl();
+static bool lookupMember(Sema &S, CXXRecordDecl *RD,
+                         LookupResult &MemberLookup) {
   assert(RD && "specialization of class template is not a class?");
-
-  // Look up the member of the trait type.
-  S.LookupQualifiedName(TraitMemberLookup, RD);
-  return TraitMemberLookup.isAmbiguous();
+  S.LookupQualifiedName(MemberLookup, RD);
+  return MemberLookup.isAmbiguous();
 }
 
 static TemplateArgumentLoc
@@ -1178,17 +1180,20 @@ static IsTupleLike isTupleLike(Sema &S, SourceLocation Loc, QualType T,
   EnterExpressionEvaluationContext ContextRAII(
       S, Sema::ExpressionEvaluationContext::ConstantEvaluated);
 
-  DeclarationName Value = S.PP.getIdentifierInfo("value");
-  LookupResult R(S, Value, Loc, Sema::LookupOrdinaryName);
-
   // Form template argument list for tuple_size<T>.
   TemplateArgumentListInfo Args(Loc, Loc);
   Args.addArgument(getTrivialTypeTemplateArgument(S, Loc, T));
 
+  QualType TraitTy = getStdTrait(S, Loc, "tuple_size", Args, /*DiagID=*/0);
+  if (TraitTy.isNull())
+    return IsTupleLike::NotTupleLike;
+
+  DeclarationName Value = S.PP.getIdentifierInfo("value");
+  LookupResult R(S, Value, Loc, Sema::LookupOrdinaryName);
+
   // If there's no tuple_size specialization or the lookup of 'value' is empty,
   // it's not tuple-like.
-  if (lookupStdTypeTraitMember(S, R, Loc, "tuple_size", Args, /*DiagID*/ 0) ||
-      R.empty())
+  if (lookupMember(S, TraitTy->getAsCXXRecordDecl(), R) || R.empty())
     return IsTupleLike::NotTupleLike;
 
   // If we get this far, we've committed to the tuple interpretation, but
@@ -1228,11 +1233,15 @@ static QualType getTupleLikeElementType(Sema &S, SourceLocation Loc,
       getTrivialIntegralTemplateArgument(S, Loc, S.Context.getSizeType(), I));
   Args.addArgument(getTrivialTypeTemplateArgument(S, Loc, T));
 
+  QualType TraitTy =
+      getStdTrait(S, Loc, "tuple_element", Args,
+                  diag::err_decomp_decl_std_tuple_element_not_specialized);
+  if (TraitTy.isNull())
+    return QualType();
+
   DeclarationName TypeDN = S.PP.getIdentifierInfo("type");
   LookupResult R(S, TypeDN, Loc, Sema::LookupOrdinaryName);
-  if (lookupStdTypeTraitMember(
-          S, R, Loc, "tuple_element", Args,
-          diag::err_decomp_decl_std_tuple_element_not_specialized))
+  if (lookupMember(S, TraitTy->getAsCXXRecordDecl(), R))
     return QualType();
 
   auto *TD = R.getAsSingle<TypeDecl>();
@@ -1246,7 +1255,8 @@ static QualType getTupleLikeElementType(Sema &S, SourceLocation Loc,
     return QualType();
   }
 
-  return S.Context.getTypeDeclType(TD);
+  NestedNameSpecifier Qualifier(TraitTy.getTypePtr());
+  return S.Context.getTypeDeclType(ElaboratedTypeKeyword::None, Qualifier, TD);
 }
 
 namespace {
@@ -1363,10 +1373,13 @@ static bool checkTupleLikeDecomposition(Sema &S,
         S.BuildReferenceType(T, E.get()->isLValue(), Loc, B->getDeclName());
     if (RefType.isNull())
       return true;
-    auto *RefVD = VarDecl::Create(
-        S.Context, Src->getDeclContext(), Loc, Loc,
-        B->getDeclName().getAsIdentifierInfo(), RefType,
-        S.Context.getTrivialTypeSourceInfo(T, Loc), Src->getStorageClass());
+
+    // Don't give this VarDecl a TypeSourceInfo, since this is a synthesized
+    // entity and this type was never written in source code.
+    auto *RefVD =
+        VarDecl::Create(S.Context, Src->getDeclContext(), Loc, Loc,
+                        B->getDeclName().getAsIdentifierInfo(), RefType,
+                        /*TInfo=*/nullptr, Src->getStorageClass());
     RefVD->setLexicalDeclContext(Src->getLexicalDeclContext());
     RefVD->setTSCSpec(Src->getTSCSpec());
     RefVD->setImplicit();
@@ -1452,7 +1465,7 @@ static DeclAccessPair findDecomposableBaseClass(Sema &S, SourceLocation Loc,
     }
 
     //   ... [accessible, implied by other rules] base class of E.
-    S.CheckBaseClassAccess(Loc, BaseType, S.Context.getRecordType(RD),
+    S.CheckBaseClassAccess(Loc, BaseType, S.Context.getCanonicalTagType(RD),
                            *BestPath, diag::err_decomp_decl_inaccessible_base);
     AS = BestPath->Access;
 
@@ -1528,8 +1541,8 @@ static bool checkMemberDecomposition(Sema &S, ArrayRef<BindingDecl*> Bindings,
   const auto *RD = cast_or_null<CXXRecordDecl>(BasePair.getDecl());
   if (!RD)
     return true;
-  QualType BaseType = S.Context.getQualifiedType(S.Context.getRecordType(RD),
-                                                 DecompType.getQualifiers());
+  QualType BaseType = S.Context.getQualifiedType(
+      S.Context.getCanonicalTagType(RD), DecompType.getQualifiers());
 
   auto *DD = cast<DecompositionDecl>(Src);
   unsigned NumFields = llvm::count_if(
@@ -2138,7 +2151,7 @@ static bool CheckConstexprDeclStmt(Sema &SemaRef, const FunctionDecl *Dcl,
 static bool CheckConstexprCtorInitializer(Sema &SemaRef,
                                           const FunctionDecl *Dcl,
                                           FieldDecl *Field,
-                                          llvm::SmallSet<Decl*, 16> &Inits,
+                                          llvm::SmallPtrSet<Decl *, 16> &Inits,
                                           bool &Diagnosed,
                                           Sema::CheckConstexprKind Kind) {
   // In C++20 onwards, there's nothing to check for validity.
@@ -2174,7 +2187,10 @@ static bool CheckConstexprCtorInitializer(Sema &SemaRef,
       return false;
     }
   } else if (Field->isAnonymousStructOrUnion()) {
-    const RecordDecl *RD = Field->getType()->castAs<RecordType>()->getDecl();
+    const RecordDecl *RD = Field->getType()
+                               ->castAs<RecordType>()
+                               ->getOriginalDecl()
+                               ->getDefinitionOrSelf();
     for (auto *I : RD->fields())
       // If an anonymous union contains an anonymous struct of which any member
       // is initialized, all members must be initialized.
@@ -2457,7 +2473,7 @@ static bool CheckConstexprFunctionBody(Sema &SemaRef, const FunctionDecl *Dcl,
         // Check initialization of non-static data members. Base classes are
         // always initialized so do not need to be checked. Dependent bases
         // might not have initializers in the member initializer list.
-        llvm::SmallSet<Decl*, 16> Inits;
+        llvm::SmallPtrSet<Decl *, 16> Inits;
         for (const auto *I: Constructor->inits()) {
           if (FieldDecl *FD = I->getMember())
             Inits.insert(FD);
@@ -2666,7 +2682,9 @@ void Sema::DiagnoseImmediateEscalatingReason(FunctionDecl *FD) {
       return DynamicRecursiveASTVisitor::TraverseCXXConstructorDecl(Ctr);
     }
 
-    bool TraverseType(QualType T) override { return true; }
+    bool TraverseType(QualType T, bool TraverseQualifier) override {
+      return true;
+    }
     bool VisitBlockExpr(BlockExpr *T) override { return true; }
 
   } Visitor(*this, FD);
@@ -2969,7 +2987,8 @@ bool Sema::AttachBaseSpecifiers(CXXRecordDecl *Class,
         NoteIndirectBases(Context, IndirectBaseTypes, NewBaseType);
 
       if (const RecordType *Record = NewBaseType->getAs<RecordType>()) {
-        const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
+        const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getOriginalDecl())
+                                      ->getDefinitionOrSelf();
         if (Class->isInterface() &&
               (!RD->isInterfaceLike() ||
                KnownBase->getAccessSpecifier() != AS_public)) {
@@ -3050,7 +3069,7 @@ bool Sema::IsDerivedFrom(SourceLocation Loc, CXXRecordDecl *Derived,
 
   // FIXME: In a modules build, do we need the entire path to be visible for us
   // to be able to use the inheritance relationship?
-  if (!isCompleteType(Loc, Context.getTypeDeclType(Derived)) &&
+  if (!isCompleteType(Loc, Context.getCanonicalTagType(Derived)) &&
       !Derived->isBeingDefined())
     return false;
 
@@ -3205,7 +3224,8 @@ std::string Sema::getAmbiguousPathsDisplayString(CXXBasePaths &Paths) {
       // We haven't displayed a path to this particular base
       // class subobject yet.
       PathDisplayStr += "\n    ";
-      PathDisplayStr += Context.getTypeDeclType(Paths.getOrigin()).getAsString();
+      PathDisplayStr += QualType(Context.getCanonicalTagType(Paths.getOrigin()))
+                            .getAsString();
       for (CXXBasePath::const_iterator Element = Path->begin();
            Element != Path->end(); ++Element)
         PathDisplayStr += " -> " + Element->Base->getType().getAsString();
@@ -4245,7 +4265,7 @@ static bool FindBaseInitializer(Sema &SemaRef,
     CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
                        /*DetectVirtual=*/false);
     if (SemaRef.IsDerivedFrom(ClassDecl->getLocation(),
-                              SemaRef.Context.getTypeDeclType(ClassDecl),
+                              SemaRef.Context.getCanonicalTagType(ClassDecl),
                               BaseType, Paths)) {
       for (CXXBasePaths::paths_iterator Path = Paths.begin();
            Path != Paths.end(); ++Path) {
@@ -4481,7 +4501,7 @@ Sema::BuildMemInitializer(Decl *ConstructorD,
       if (getLangOpts().MSVCCompat && !getLangOpts().CPlusPlus20) {
         if (auto UnqualifiedBase = R.getAsSingle<ClassTemplateDecl>()) {
           auto *TempSpec = cast<TemplateSpecializationType>(
-              UnqualifiedBase->getInjectedClassNameSpecialization());
+              UnqualifiedBase->getCanonicalInjectedSpecializationType(Context));
           TemplateName TN = TempSpec->getTemplateName();
           for (auto const &Base : ClassDecl->bases()) {
             auto BaseTemplate =
@@ -4545,14 +4565,36 @@ Sema::BuildMemInitializer(Decl *ConstructorD,
     }
 
     if (BaseType.isNull()) {
-      BaseType = getElaboratedType(ElaboratedTypeKeyword::None, SS,
-                                   Context.getTypeDeclType(TyD));
       MarkAnyDeclReferenced(TyD->getLocation(), TyD, /*OdrUse=*/false);
-      TInfo = Context.CreateTypeSourceInfo(BaseType);
-      ElaboratedTypeLoc TL = TInfo->getTypeLoc().castAs<ElaboratedTypeLoc>();
-      TL.getNamedTypeLoc().castAs<TypeSpecTypeLoc>().setNameLoc(IdLoc);
-      TL.setElaboratedKeywordLoc(SourceLocation());
-      TL.setQualifierLoc(SS.getWithLocInContext(Context));
+
+      TypeLocBuilder TLB;
+      // FIXME: This is missing building the UsingType for TyD, if any.
+      if (const auto *TD = dyn_cast<TagDecl>(TyD)) {
+        BaseType = Context.getTagType(ElaboratedTypeKeyword::None,
+                                      SS.getScopeRep(), TD, /*OwnsTag=*/false);
+        auto TL = TLB.push<TagTypeLoc>(BaseType);
+        TL.setElaboratedKeywordLoc(SourceLocation());
+        TL.setQualifierLoc(SS.getWithLocInContext(Context));
+        TL.setNameLoc(IdLoc);
+      } else if (auto *TN = dyn_cast<TypedefNameDecl>(TyD)) {
+        BaseType = Context.getTypedefType(ElaboratedTypeKeyword::None,
+                                          SS.getScopeRep(), TN);
+        TLB.push<TypedefTypeLoc>(BaseType).set(
+            /*ElaboratedKeywordLoc=*/SourceLocation(),
+            SS.getWithLocInContext(Context), IdLoc);
+      } else if (auto *UD = dyn_cast<UnresolvedUsingTypenameDecl>(TyD)) {
+        BaseType = Context.getUnresolvedUsingType(ElaboratedTypeKeyword::None,
+                                                  SS.getScopeRep(), UD);
+        TLB.push<UnresolvedUsingTypeLoc>(BaseType).set(
+            /*ElaboratedKeywordLoc=*/SourceLocation(),
+            SS.getWithLocInContext(Context), IdLoc);
+      } else {
+        // FIXME: What else can appear here?
+        assert(SS.isEmpty());
+        BaseType = Context.getTypeDeclType(TyD);
+        TLB.pushTypeSpec(BaseType).setNameLoc(IdLoc);
+      }
+      TInfo = TLB.getTypeSourceInfo(Context, BaseType);
     }
   }
 
@@ -4661,10 +4703,12 @@ Sema::BuildDelegatingInitializer(TypeSourceInfo *TInfo, Expr *Init,
     Args = MultiExprArg(ParenList->getExprs(), ParenList->getNumExprs());
   }
 
+  CanQualType ClassType = Context.getCanonicalTagType(ClassDecl);
+
   SourceRange InitRange = Init->getSourceRange();
   // Initialize the object.
-  InitializedEntity DelegationEntity = InitializedEntity::InitializeDelegation(
-                                     QualType(ClassDecl->getTypeForDecl(), 0));
+  InitializedEntity DelegationEntity =
+      InitializedEntity::InitializeDelegation(ClassType);
   InitializationKind Kind =
       InitList ? InitializationKind::CreateDirectList(
                      NameLoc, Init->getBeginLoc(), Init->getEndLoc())
@@ -4686,9 +4730,8 @@ Sema::BuildDelegatingInitializer(TypeSourceInfo *TInfo, Expr *Init,
   }
 
   if (DelegationInit.isInvalid()) {
-    DelegationInit =
-        CreateRecoveryExpr(InitRange.getBegin(), InitRange.getEnd(), Args,
-                           QualType(ClassDecl->getTypeForDecl(), 0));
+    DelegationInit = CreateRecoveryExpr(InitRange.getBegin(),
+                                        InitRange.getEnd(), Args, ClassType);
     if (DelegationInit.isInvalid())
       return true;
   } else {
@@ -4753,8 +4796,7 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
   const CXXBaseSpecifier *DirectBaseSpec = nullptr;
   const CXXBaseSpecifier *VirtualBaseSpec = nullptr;
   if (!Dependent) {
-    if (Context.hasSameUnqualifiedType(QualType(ClassDecl->getTypeForDecl(),0),
-                                       BaseType))
+    if (declaresSameEntity(ClassDecl, BaseType->getAsCXXRecordDecl()))
       return BuildDelegatingInitializer(BaseTInfo, Init, ClassDecl);
 
     FindBaseInitializer(*this, ClassDecl, BaseType, DirectBaseSpec,
@@ -4774,7 +4816,7 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
         Dependent = true;
       else
         return Diag(BaseLoc, diag::err_not_direct_base_or_virtual)
-               << BaseType << Context.getTypeDeclType(ClassDecl)
+               << BaseType << Context.getCanonicalTagType(ClassDecl)
                << BaseTInfo->getTypeLoc().getSourceRange();
     }
   }
@@ -5078,9 +5120,9 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     if (FieldBaseElementType->isReferenceType()) {
       SemaRef.Diag(Constructor->getLocation(),
                    diag::err_uninitialized_member_in_ctor)
-      << (int)Constructor->isImplicit()
-      << SemaRef.Context.getTagDeclType(Constructor->getParent())
-      << 0 << Field->getDeclName();
+          << (int)Constructor->isImplicit()
+          << SemaRef.Context.getCanonicalTagType(Constructor->getParent()) << 0
+          << Field->getDeclName();
       SemaRef.Diag(Field->getLocation(), diag::note_declared_at);
       return true;
     }
@@ -5088,9 +5130,9 @@ BuildImplicitMemberInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     if (FieldBaseElementType.isConstQualified()) {
       SemaRef.Diag(Constructor->getLocation(),
                    diag::err_uninitialized_member_in_ctor)
-      << (int)Constructor->isImplicit()
-      << SemaRef.Context.getTagDeclType(Constructor->getParent())
-      << 1 << Field->getDeclName();
+          << (int)Constructor->isImplicit()
+          << SemaRef.Context.getCanonicalTagType(Constructor->getParent()) << 1
+          << Field->getDeclName();
       SemaRef.Diag(Field->getLocation(), diag::note_declared_at);
       return true;
     }
@@ -5391,7 +5433,7 @@ static void MarkBaseDestructorsReferenced(Sema &S, SourceLocation Location,
     S.CheckDestructorAccess(Base.getBeginLoc(), Dtor,
                             S.PDiag(diag::err_access_dtor_base)
                                 << Base.getType() << Base.getSourceRange(),
-                            S.Context.getTypeDeclType(ClassDecl));
+                            S.Context.getCanonicalTagType(ClassDecl));
 
     S.MarkFunctionReferenced(Location, Dtor);
     S.DiagnoseUseOfDecl(Dtor, Location);
@@ -5594,9 +5636,9 @@ bool Sema::SetCtorInitializers(CXXConstructorDecl *Constructor, bool AnyErrors,
 
 static void PopulateKeysForFields(FieldDecl *Field, SmallVectorImpl<const void*> &IdealInits) {
   if (const RecordType *RT = Field->getType()->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl();
     if (RD->isAnonymousStructOrUnion()) {
-      for (auto *Field : RD->fields())
+      for (auto *Field : RD->getDefinitionOrSelf()->fields())
         PopulateKeysForFields(Field, IdealInits);
       return;
     }
@@ -5928,16 +5970,14 @@ void Sema::MarkVirtualBaseDestructorsReferenced(
     if (!Dtor)
       continue;
 
-    if (CheckDestructorAccess(
-            ClassDecl->getLocation(), Dtor,
-            PDiag(diag::err_access_dtor_vbase)
-                << Context.getTypeDeclType(ClassDecl) << VBase.getType(),
-            Context.getTypeDeclType(ClassDecl)) ==
-        AR_accessible) {
+    CanQualType CT = Context.getCanonicalTagType(ClassDecl);
+    if (CheckDestructorAccess(ClassDecl->getLocation(), Dtor,
+                              PDiag(diag::err_access_dtor_vbase)
+                                  << CT << VBase.getType(),
+                              CT) == AR_accessible) {
       CheckDerivedToBaseConversion(
-          Context.getTypeDeclType(ClassDecl), VBase.getType(),
-          diag::err_access_dtor_vbase, 0, ClassDecl->getLocation(),
-          SourceRange(), DeclarationName(), nullptr);
+          CT, VBase.getType(), diag::err_access_dtor_vbase, 0,
+          ClassDecl->getLocation(), SourceRange(), DeclarationName(), nullptr);
     }
 
     MarkFunctionReferenced(Location, Dtor);
@@ -6052,10 +6092,8 @@ struct AbstractUsageInfo {
   bool Invalid;
 
   AbstractUsageInfo(Sema &S, CXXRecordDecl *Record)
-    : S(S), Record(Record),
-      AbstractType(S.Context.getCanonicalType(
-                   S.Context.getTypeDeclType(Record))),
-      Invalid(false) {}
+      : S(S), Record(Record),
+        AbstractType(S.Context.getCanonicalTagType(Record)), Invalid(false) {}
 
   void DiagnoseAbstractType() {
     if (Invalid) return;
@@ -6868,8 +6906,8 @@ static bool canPassInRegisters(Sema &S, CXXRecordDecl *D,
     bool isAArch64 = S.Context.getTargetInfo().getTriple().isAArch64();
     uint64_t TypeSize = isAArch64 ? 128 : 64;
 
-    if (CopyCtorIsTrivial &&
-        S.getASTContext().getTypeSize(D->getTypeForDecl()) <= TypeSize)
+    if (CopyCtorIsTrivial && S.getASTContext().getTypeSize(
+                                 S.Context.getCanonicalTagType(D)) <= TypeSize)
       return true;
     return false;
   }
@@ -6984,9 +7022,12 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
     for (DeclContext::lookup_iterator I = R.begin(), E = R.end(); I != E;
          ++I) {
       NamedDecl *D = (*I)->getUnderlyingDecl();
+      // Invalid IndirectFieldDecls have already been diagnosed with
+      // err_anonymous_record_member_redecl in
+      // SemaDecl.cpp:CheckAnonMemberRedeclaration.
       if (((isa<FieldDecl>(D) || isa<UnresolvedUsingValueDecl>(D)) &&
            Record->hasUserDeclaredConstructor()) ||
-          isa<IndirectFieldDecl>(D)) {
+          (isa<IndirectFieldDecl>(D) && !D->isInvalidDecl())) {
         Diag((*I)->getLocation(), diag::err_member_name_of_class)
           << D->getDeclName();
         break;
@@ -7000,7 +7041,8 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
     if ((!dtor || (!dtor->isVirtual() && dtor->getAccess() == AS_public)) &&
         !Record->hasAttr<FinalAttr>())
       Diag(dtor ? dtor->getLocation() : Record->getLocation(),
-           diag::warn_non_virtual_dtor) << Context.getRecordType(Record);
+           diag::warn_non_virtual_dtor)
+          << Context.getCanonicalTagType(Record);
   }
 
   if (Record->isAbstract()) {
@@ -7022,7 +7064,7 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
                    (FA->isSpelledAsSealed() ? " sealed" : " final"));
         Diag(Record->getLocation(),
              diag::note_final_dtor_non_final_class_silence)
-            << Context.getRecordType(Record) << FA->isSpelledAsSealed();
+            << Context.getCanonicalTagType(Record) << FA->isSpelledAsSealed();
       }
     }
   }
@@ -7177,7 +7219,7 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
         return true;
       };
       EffectivelyConstexprDestructor =
-          Check(QualType(Record->getTypeForDecl(), 0), Check);
+          Check(Context.getCanonicalTagType(Record), Check);
     }
 
     // Define defaulted constexpr virtual functions that override a base class
@@ -7333,7 +7375,7 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
                 NewDecls.empty() ? NewKind : DeleteKind);
         Diag(Record->getLocation(),
              diag::err_type_aware_allocator_missing_matching_operator)
-            << FoundOperator << Context.getRecordType(Record)
+            << FoundOperator << Context.getCanonicalTagType(Record)
             << MissingOperator;
         for (auto MD : NewDecls)
           Diag(MD->getLocation(),
@@ -7572,7 +7614,9 @@ static bool defaultedSpecialMemberIsConstexpr(
       const RecordType *BaseType = B.getType()->getAs<RecordType>();
       if (!BaseType)
         continue;
-      CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(BaseType->getDecl());
+      CXXRecordDecl *BaseClassDecl =
+          cast<CXXRecordDecl>(BaseType->getOriginalDecl())
+              ->getDefinitionOrSelf();
       if (!specialMemberIsConstexpr(S, BaseClassDecl, CSM, 0, ConstArg,
                                     InheritedCtor, Inherited))
         return false;
@@ -7595,7 +7639,9 @@ static bool defaultedSpecialMemberIsConstexpr(
         continue;
       QualType BaseType = S.Context.getBaseElementType(F->getType());
       if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) {
-        CXXRecordDecl *FieldRecDecl = cast<CXXRecordDecl>(RecordTy->getDecl());
+        CXXRecordDecl *FieldRecDecl =
+            cast<CXXRecordDecl>(RecordTy->getOriginalDecl())
+                ->getDefinitionOrSelf();
         if (!specialMemberIsConstexpr(S, FieldRecDecl, CSM,
                                       BaseType.getCVRQualifiers(),
                                       ConstArg && !F->isMutable()))
@@ -7782,9 +7828,9 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
     ReturnType = Type->getReturnType();
     QualType ThisType = MD->getFunctionObjectParameterType();
 
-    QualType DeclType = Context.getTypeDeclType(RD);
-    DeclType = Context.getElaboratedType(ElaboratedTypeKeyword::None, nullptr,
-                                         DeclType, nullptr);
+    QualType DeclType =
+        Context.getTagType(ElaboratedTypeKeyword::None,
+                           /*Qualifier=*/std::nullopt, RD, /*OwnsTag=*/false);
     DeclType = Context.getAddrSpaceQualType(
         DeclType, ThisType.getQualifiers().getAddressSpace());
     QualType ExpectedReturnType = Context.getLValueReferenceType(DeclType);
@@ -7819,7 +7865,7 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
     if (!ExplicitObjectParameter.isNull() &&
         (!ExplicitObjectParameter->isReferenceType() ||
          !Context.hasSameType(ExplicitObjectParameter.getNonReferenceType(),
-                              Context.getRecordType(RD)))) {
+                              Context.getCanonicalTagType(RD)))) {
       if (DeleteOnTypeMismatch)
         ShouldDeleteForTypeMismatch = true;
       else {
@@ -8278,7 +8324,7 @@ class DefaultedComparisonAnalyzer
           Best->FoundDecl.getDecl()->isCXXClassMember()) {
         QualType ObjectType = Subobj.Kind == Subobject::Member
                                   ? Args[0]->getType()
-                                  : S.Context.getRecordType(RD);
+                                  : S.Context.getCanonicalTagType(RD);
         if (!S.isMemberAccessibleForDeletion(
                 ArgClass, Best->FoundDecl, ObjectType, Subobj.Loc,
                 Diagnose == ExplainDeleted
@@ -8979,7 +9025,7 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
       bool Ok = !IsMethod || FD->hasCXXExplicitFunctionObjectParameter();
       QualType ExpectedTy;
       if (RD)
-        ExpectedTy = Context.getRecordType(RD);
+        ExpectedTy = Context.getCanonicalTagType(RD);
       if (auto *Ref = CTy->getAs<LValueReferenceType>()) {
         CTy = Ref->getPointeeType();
         if (RD)
@@ -9002,7 +9048,7 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
         // corresponding defaulted 'operator<=>' already.
         if (!FD->isImplicit()) {
           if (RD) {
-            QualType PlainTy = Context.getRecordType(RD);
+            CanQualType PlainTy = Context.getCanonicalTagType(RD);
             QualType RefTy =
                 Context.getLValueReferenceType(PlainTy.withConst());
             Diag(FD->getLocation(), diag::err_defaulted_comparison_param)
@@ -9033,7 +9079,7 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
   } else {
     // Out of class, require the defaulted comparison to be a friend (of a
     // complete type, per CWG2547).
-    if (RequireCompleteType(FD->getLocation(), Context.getRecordType(RD),
+    if (RequireCompleteType(FD->getLocation(), Context.getCanonicalTagType(RD),
                             diag::err_defaulted_comparison_not_friend, int(DCK),
                             int(1)))
       return true;
@@ -9486,15 +9532,15 @@ bool SpecialMemberDeletionInfo::isAccessible(Subobject Subobj,
                                              CXXMethodDecl *target) {
   /// If we're operating on a base class, the object type is the
   /// type of this special member.
-  QualType objectTy;
+  CanQualType objectTy;
   AccessSpecifier access = target->getAccess();
   if (CXXBaseSpecifier *base = Subobj.dyn_cast<CXXBaseSpecifier*>()) {
-    objectTy = S.Context.getTypeDeclType(MD->getParent());
+    objectTy = S.Context.getCanonicalTagType(MD->getParent());
     access = CXXRecordDecl::MergeAccess(base->getAccessSpecifier(), access);
 
   // If we're operating on a field, the object type is the type of the field.
   } else {
-    objectTy = S.Context.getTypeDeclType(target->getParent());
+    objectTy = S.Context.getCanonicalTagType(target->getParent());
   }
 
   return S.isMemberAccessibleForDeletion(
@@ -9912,7 +9958,7 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD,
   //    results in an ambiguity or in a function that is deleted or inaccessible
   if (CSM == CXXSpecialMemberKind::Destructor && MD->isVirtual()) {
     FunctionDecl *OperatorDelete = nullptr;
-    QualType DeallocType = Context.getRecordType(RD);
+    CanQualType DeallocType = Context.getCanonicalTagType(RD);
     DeclarationName Name =
       Context.DeclarationNames.getCXXOperatorName(OO_Delete);
     ImplicitDeallocationParameters IDP = {
@@ -10245,7 +10291,7 @@ static bool checkTrivialClassMembers(Sema &S, CXXRecordDecl *RD,
 
 void Sema::DiagnoseNontrivial(const CXXRecordDecl *RD,
                               CXXSpecialMemberKind CSM) {
-  QualType Ty = Context.getRecordType(RD);
+  CanQualType Ty = Context.getCanonicalTagType(RD);
 
   bool ConstArg = (CSM == CXXSpecialMemberKind::CopyConstructor ||
                    CSM == CXXSpecialMemberKind::CopyAssignment);
@@ -10293,9 +10339,9 @@ bool Sema::SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMemberKind CSM,
          ClangABICompat14)) {
       if (Diagnose)
         Diag(Param0->getLocation(), diag::note_nontrivial_param_type)
-          << Param0->getSourceRange() << Param0->getType()
-          << Context.getLValueReferenceType(
-               Context.getRecordType(RD).withConst());
+            << Param0->getSourceRange() << Param0->getType()
+            << Context.getLValueReferenceType(
+                   Context.getCanonicalTagType(RD).withConst());
       return false;
     }
 
@@ -10312,8 +10358,8 @@ bool Sema::SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMemberKind CSM,
     if (!RT || RT->getPointeeType().getCVRQualifiers()) {
       if (Diagnose)
         Diag(Param0->getLocation(), diag::note_nontrivial_param_type)
-          << Param0->getSourceRange() << Param0->getType()
-          << Context.getRValueReferenceType(Context.getRecordType(RD));
+            << Param0->getSourceRange() << Param0->getType()
+            << Context.getRValueReferenceType(Context.getCanonicalTagType(RD));
       return false;
     }
     break;
@@ -10432,8 +10478,10 @@ struct FindHiddenVirtualMethod {
   /// method overloads virtual methods in a base class without overriding any,
   /// to be used with CXXRecordDecl::lookupInBases().
   bool operator()(const CXXBaseSpecifier *Specifier, CXXBasePath &Path) {
-    RecordDecl *BaseRecord =
-        Specifier->getType()->castAs<RecordType>()->getDecl();
+    RecordDecl *BaseRecord = Specifier->getType()
+                                 ->castAs<RecordType>()
+                                 ->getOriginalDecl()
+                                 ->getDefinitionOrSelf();
 
     DeclarationName Name = Method->getDeclName();
     assert(Name.getNameKind() == DeclarationName::Identifier);
@@ -10591,7 +10639,8 @@ void Sema::checkIllFormedTrivialABIStruct(CXXRecordDecl &RD) {
 
     if (const auto *RT = FT->getBaseElementTypeUnsafe()->getAs<RecordType>())
       if (!RT->isDependentType() &&
-          !cast<CXXRecordDecl>(RT->getDecl())->canPassInRegisters()) {
+          !cast<CXXRecordDecl>(RT->getOriginalDecl()->getDefinitionOrSelf())
+               ->canPassInRegisters()) {
         PrintDiagAndRemoveAttr(5);
         return;
       }
@@ -10626,7 +10675,7 @@ void Sema::checkIllFormedTrivialABIStruct(CXXRecordDecl &RD) {
 
 void Sema::checkIncorrectVTablePointerAuthenticationAttribute(
     CXXRecordDecl &RD) {
-  if (RequireCompleteType(RD.getLocation(), Context.getRecordType(&RD),
+  if (RequireCompleteType(RD.getLocation(), Context.getCanonicalTagType(&RD),
                           diag::err_incomplete_type_vtable_pointer_auth))
     return;
 
@@ -11056,9 +11105,10 @@ void Sema::CheckConstructor(CXXConstructorDecl *Constructor) {
   if (!Constructor->isInvalidDecl() &&
       Constructor->hasOneParamOrDefaultArgs() &&
       !Constructor->isFunctionTemplateSpecialization()) {
-    QualType ParamType = Constructor->getParamDecl(0)->getType();
-    QualType ClassTy = Context.getTagDeclType(ClassDecl);
-    if (Context.getCanonicalType(ParamType).getUnqualifiedType() == ClassTy) {
+    CanQualType ParamType =
+        Constructor->getParamDecl(0)->getType()->getCanonicalTypeUnqualified();
+    CanQualType ClassTy = Context.getCanonicalTagType(ClassDecl);
+    if (ParamType == ClassTy) {
       SourceLocation ParamLoc = Constructor->getParamDecl(0)->getLocation();
       const char *ConstRef
         = Constructor->getParamDecl(0)->getIdentifier() ? "const &"
@@ -11429,8 +11479,7 @@ Decl *Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
   //   same object type (or a reference to it), to a (possibly
   //   cv-qualified) base class of that type (or a reference to it),
   //   or to (possibly cv-qualified) void.
-  QualType ClassType
-    = Context.getCanonicalType(Context.getTypeDeclType(ClassDecl));
+  CanQualType ClassType = Context.getCanonicalTagType(ClassDecl);
   if (const ReferenceType *ConvTypeRef = ConvType->getAs<ReferenceType>())
     ConvType = ConvTypeRef->getPointeeType();
   if (Conversion->getTemplateSpecializationKind() != TSK_Undeclared &&
@@ -11698,8 +11747,8 @@ bool Sema::CheckDeductionGuideDeclarator(Declarator &D, QualType &R,
       const QualifiedTemplateName *Qualifiers =
           SpecifiedName.getAsQualifiedTemplateName();
       assert(Qualifiers && "expected QualifiedTemplate");
-      bool SimplyWritten = !Qualifiers->hasTemplateKeyword() &&
-                           Qualifiers->getQualifier() == nullptr;
+      bool SimplyWritten =
+          !Qualifiers->hasTemplateKeyword() && !Qualifiers->getQualifier();
       if (SimplyWritten && TemplateMatches)
         AcceptableReturnType = true;
       else {
@@ -12004,10 +12053,11 @@ QualType Sema::CheckComparisonCategoryType(ComparisonCategoryType Kind,
   // Use an elaborated type for diagnostics which has a name containing the
   // prepended 'std' namespace but not any inline namespace names.
   auto TyForDiags = [&](ComparisonCategoryInfo *Info) {
-    auto *NNS =
-        NestedNameSpecifier::Create(Context, nullptr, getStdNamespace());
-    return Context.getElaboratedType(ElaboratedTypeKeyword::None, NNS,
-                                     Info->getType());
+    NestedNameSpecifier Qualifier(Context, getStdNamespace(),
+                                  /*Prefix=*/std::nullopt);
+    return Context.getTagType(ElaboratedTypeKeyword::None, Qualifier,
+                              Info->Record,
+                              /*OwnsTag=*/false);
   };
 
   // Check if we've already successfully checked the comparison category type
@@ -12143,26 +12193,14 @@ static bool isStdClassTemplate(Sema &S, QualType SugaredType, QualType *TypeArg,
 
   ClassTemplateDecl *Template = nullptr;
   ArrayRef<TemplateArgument> Arguments;
-  {
-    const TemplateSpecializationType *TST =
-        SugaredType->getAsNonAliasTemplateSpecializationType();
-    if (!TST)
-      if (const auto *ICN = SugaredType->getAs<InjectedClassNameType>())
-        TST = ICN->getInjectedTST();
-    if (TST) {
-      Template = dyn_cast_or_null<ClassTemplateDecl>(
-          TST->getTemplateName().getAsTemplateDecl());
-      Arguments = TST->template_arguments();
-    } else if (const RecordType *RT = SugaredType->getAs<RecordType>()) {
-      ClassTemplateSpecializationDecl *Specialization =
-          dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl());
-      if (!Specialization) {
-        ReportMatchingNameAsMalformed(RT->getDecl());
-        return false;
-      }
-      Template = Specialization->getSpecializedTemplate();
-      Arguments = Specialization->getTemplateArgs().asArray();
-    }
+  if (const TemplateSpecializationType *TST =
+          SugaredType->getAsNonAliasTemplateSpecializationType()) {
+    Template = dyn_cast_or_null<ClassTemplateDecl>(
+        TST->getTemplateName().getAsTemplateDecl());
+    Arguments = TST->template_arguments();
+  } else if (const auto *TT = SugaredType->getAs<TagType>()) {
+    Template = TT->getTemplateDecl();
+    Arguments = TT->getTemplateArgs(S.Context);
   }
 
   if (!Template) {
@@ -12288,13 +12326,8 @@ static QualType BuildStdClassTemplate(Sema &S, ClassTemplateDecl *CTD,
   auto TSI = S.Context.getTrivialTypeSourceInfo(TypeParam, Loc);
   Args.addArgument(TemplateArgumentLoc(TemplateArgument(TypeParam), TSI));
 
-  QualType T = S.CheckTemplateIdType(TemplateName(CTD), Loc, Args);
-  if (T.isNull())
-    return QualType();
-
-  return S.Context.getElaboratedType(
-      ElaboratedTypeKeyword::None,
-      NestedNameSpecifier::Create(S.Context, nullptr, S.getStdNamespace()), T);
+  return S.CheckTemplateIdType(ElaboratedTypeKeyword::None, TemplateName(CTD),
+                               Loc, Args);
 }
 
 QualType Sema::BuildStdInitializerList(QualType Element, SourceLocation Loc) {
@@ -12436,9 +12469,7 @@ Decl *Sema::ActOnUsingDirective(Scope *S, SourceLocation UsingLoc,
   S = S->getDeclParent();
 
   UsingDirectiveDecl *UDir = nullptr;
-  NestedNameSpecifier *Qualifier = nullptr;
-  if (SS.isSet())
-    Qualifier = SS.getScopeRep();
+  NestedNameSpecifier Qualifier = SS.getScopeRep();
 
   // Lookup namespace name.
   LookupResult R(*this, NamespcName, IdentLoc, LookupNamespaceName);
@@ -12450,14 +12481,16 @@ Decl *Sema::ActOnUsingDirective(Scope *S, SourceLocation UsingLoc,
     R.clear();
     // Allow "using namespace std;" or "using namespace ::std;" even if
     // "std" hasn't been defined yet, for GCC compatibility.
-    if ((!Qualifier || Qualifier->getKind() == NestedNameSpecifier::Global) &&
+    if ((!Qualifier ||
+         Qualifier.getKind() == NestedNameSpecifier::Kind::Global) &&
         NamespcName->isStr("std")) {
       Diag(IdentLoc, diag::ext_using_undefined_std);
       R.addDecl(getOrCreateStdNamespace());
       R.resolveKind();
     }
     // Otherwise, attempt typo correction.
-    else TryNamespaceTypoCorrection(*this, R, S, SS, IdentLoc, NamespcName);
+    else
+      TryNamespaceTypoCorrection(*this, R, S, SS, IdentLoc, NamespcName);
   }
 
   if (!R.empty()) {
@@ -12584,7 +12617,7 @@ Decl *Sema::ActOnUsingDeclaration(Scope *S, AccessSpecifier AS,
         DiagnoseUnexpandedParameterPack(TargetNameInfo, UPPC_UsingDeclaration))
       return nullptr;
   } else {
-    if (!SS.getScopeRep()->containsUnexpandedParameterPack() &&
+    if (!SS.getScopeRep().containsUnexpandedParameterPack() &&
         !TargetNameInfo.containsUnexpandedParameterPack()) {
       Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
         << SourceRange(SS.getBeginLoc(), TargetNameInfo.getEndLoc());
@@ -12869,7 +12902,7 @@ UsingShadowDecl *Sema::BuildUsingShadowDecl(Scope *S, BaseUsingDecl *BUD,
     UsingDecl *Using = cast<UsingDecl>(BUD);
     bool IsVirtualBase =
         isVirtualDirectBase(cast<CXXRecordDecl>(CurContext),
-                            Using->getQualifier()->getAsRecordDecl());
+                            Using->getQualifier().getAsRecordDecl());
     Shadow = ConstructorUsingShadowDecl::Create(
         Context, CurContext, Using->getLocation(), Using, Orig, IsVirtualBase);
   } else {
@@ -12935,7 +12968,7 @@ namespace {
 class UsingValidatorCCC final : public CorrectionCandidateCallback {
 public:
   UsingValidatorCCC(bool HasTypenameKeyword, bool IsInstantiation,
-                    NestedNameSpecifier *NNS, CXXRecordDecl *RequireMemberOf)
+                    NestedNameSpecifier NNS, CXXRecordDecl *RequireMemberOf)
       : HasTypenameKeyword(HasTypenameKeyword),
         IsInstantiation(IsInstantiation), OldNNS(NNS),
         RequireMemberOf(RequireMemberOf) {}
@@ -12962,24 +12995,23 @@ class UsingValidatorCCC final : public CorrectionCandidateCallback {
         ASTContext &Ctx = ND->getASTContext();
         if (!Ctx.getLangOpts().CPlusPlus11)
           return false;
-        QualType FoundType = Ctx.getRecordType(FoundRecord);
+        CanQualType FoundType = Ctx.getCanonicalTagType(FoundRecord);
 
         // Check that the injected-class-name is named as a member of its own
         // type; we don't want to suggest 'using Derived::Base;', since that
         // means something else.
-        NestedNameSpecifier *Specifier =
-            Candidate.WillReplaceSpecifier()
-                ? Candidate.getCorrectionSpecifier()
-                : OldNNS;
-        if (!Specifier->getAsType() ||
-            !Ctx.hasSameType(QualType(Specifier->getAsType(), 0), FoundType))
+        NestedNameSpecifier Specifier = Candidate.WillReplaceSpecifier()
+                                            ? Candidate.getCorrectionSpecifier()
+                                            : OldNNS;
+        if (Specifier.getKind() != NestedNameSpecifier::Kind::Type ||
+            !Ctx.hasSameType(QualType(Specifier.getAsType(), 0), FoundType))
           return false;
 
         // Check that this inheriting constructor declaration actually names a
         // direct base class of the current class.
         bool AnyDependentBases = false;
         if (!findDirectBaseWithType(RequireMemberOf,
-                                    Ctx.getRecordType(FoundRecord),
+                                    Ctx.getCanonicalTagType(FoundRecord),
                                     AnyDependentBases) &&
             !AnyDependentBases)
           return false;
@@ -13009,7 +13041,7 @@ class UsingValidatorCCC final : public CorrectionCandidateCallback {
 private:
   bool HasTypenameKeyword;
   bool IsInstantiation;
-  NestedNameSpecifier *OldNNS;
+  NestedNameSpecifier OldNNS;
   CXXRecordDecl *RequireMemberOf;
 };
 } // end anonymous namespace
@@ -13051,7 +13083,7 @@ NamedDecl *Sema::BuildUsingDeclaration(
   if (UsingName.getName().getNameKind() == DeclarationName::CXXConstructorName)
     if (auto *RD = dyn_cast<CXXRecordDecl>(CurContext))
       UsingName.setName(Context.DeclarationNames.getCXXConstructorName(
-          Context.getCanonicalType(Context.getRecordType(RD))));
+          Context.getCanonicalTagType(RD)));
 
   // Do the redeclaration lookup in the current scope.
   LookupResult Previous(*this, UsingName, LookupUsingDeclName,
@@ -13154,7 +13186,7 @@ NamedDecl *Sema::BuildUsingDeclaration(
   // equal to that of the current context.
   if (CurContext->isRecord()) {
     R.setBaseObjectType(
-                   Context.getTypeDeclType(cast<CXXRecordDecl>(CurContext)));
+        Context.getCanonicalTagType(cast<CXXRecordDecl>(CurContext)));
   }
 
   LookupQualifiedName(R, LookupContext);
@@ -13220,7 +13252,7 @@ NamedDecl *Sema::BuildUsingDeclaration(
         // constructor.
         auto *CurClass = cast<CXXRecordDecl>(CurContext);
         UsingName.setName(Context.DeclarationNames.getCXXConstructorName(
-            Context.getCanonicalType(Context.getRecordType(CurClass))));
+            Context.getCanonicalTagType(CurClass)));
         UsingName.setNamedTypeInfo(nullptr);
         for (auto *Ctor : LookupConstructors(RD))
           R.addDecl(Ctor);
@@ -13366,20 +13398,16 @@ NamedDecl *Sema::BuildUsingPackDecl(NamedDecl *InstantiatedFrom,
 bool Sema::CheckInheritingConstructorUsingDecl(UsingDecl *UD) {
   assert(!UD->hasTypename() && "expecting a constructor name");
 
-  const Type *SourceType = UD->getQualifier()->getAsType();
-  assert(SourceType &&
-         "Using decl naming constructor doesn't have type in scope spec.");
+  QualType SourceType(UD->getQualifier().getAsType(), 0);
   CXXRecordDecl *TargetClass = cast<CXXRecordDecl>(CurContext);
 
   // Check whether the named type is a direct base class.
   bool AnyDependentBases = false;
-  auto *Base = findDirectBaseWithType(TargetClass, QualType(SourceType, 0),
-                                      AnyDependentBases);
+  auto *Base =
+      findDirectBaseWithType(TargetClass, SourceType, AnyDependentBases);
   if (!Base && !AnyDependentBases) {
-    Diag(UD->getUsingLoc(),
-         diag::err_using_decl_constructor_not_in_direct_base)
-      << UD->getNameInfo().getSourceRange()
-      << QualType(SourceType, 0) << TargetClass;
+    Diag(UD->getUsingLoc(), diag::err_using_decl_constructor_not_in_direct_base)
+        << UD->getNameInfo().getSourceRange() << SourceType << TargetClass;
     UD->setInvalidDecl();
     return true;
   }
@@ -13395,7 +13423,7 @@ bool Sema::CheckUsingDeclRedeclaration(SourceLocation UsingLoc,
                                        const CXXScopeSpec &SS,
                                        SourceLocation NameLoc,
                                        const LookupResult &Prev) {
-  NestedNameSpecifier *Qual = SS.getScopeRep();
+  NestedNameSpecifier Qual = SS.getScopeRep();
 
   // C++03 [namespace.udecl]p8:
   // C++0x [namespace.udecl]p10:
@@ -13410,7 +13438,7 @@ bool Sema::CheckUsingDeclRedeclaration(SourceLocation UsingLoc,
     // declaration in the same scope.
     // FIXME: How should we check for dependent type-type conflicts at block
     // scope?
-    if (Qual->isDependent() && !HasTypenameKeyword) {
+    if (Qual.isDependent() && !HasTypenameKeyword) {
       for (auto *D : Prev) {
         if (!isa<TypeDecl>(D) && !isa<UsingDecl>(D) && !isa<UsingPackDecl>(D)) {
           bool OldCouldBeEnumerator =
@@ -13427,13 +13455,12 @@ bool Sema::CheckUsingDeclRedeclaration(SourceLocation UsingLoc,
     return false;
   }
 
-  const NestedNameSpecifier *CNNS =
-      Context.getCanonicalNestedNameSpecifier(Qual);
+  NestedNameSpecifier CNNS = Qual.getCanonical();
   for (LookupResult::iterator I = Prev.begin(), E = Prev.end(); I != E; ++I) {
     NamedDecl *D = *I;
 
     bool DTypename;
-    NestedNameSpecifier *DQual;
+    NestedNameSpecifier DQual = std::nullopt;
     if (UsingDecl *UD = dyn_cast<UsingDecl>(D)) {
       DTypename = UD->hasTypename();
       DQual = UD->getQualifier();
@@ -13454,7 +13481,7 @@ bool Sema::CheckUsingDeclRedeclaration(SourceLocation UsingLoc,
     // using decls differ if they name different scopes (but note that
     // template instantiation can cause this check to trigger when it
     // didn't before instantiation).
-    if (CNNS != Context.getCanonicalNestedNameSpecifier(DQual))
+    if (CNNS != DQual.getCanonical())
       continue;
 
     Diag(NameLoc, diag::err_using_decl_redeclaration) << SS.getRange();
@@ -13914,7 +13941,8 @@ bool SpecialMemberExceptionSpecInfo::visitBase(CXXBaseSpecifier *Base) {
   if (!RT)
     return false;
 
-  auto *BaseClass = cast<CXXRecordDecl>(RT->getDecl());
+  auto *BaseClass =
+      cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
   Sema::SpecialMemberOverloadResult SMOR = lookupInheritedCtor(BaseClass);
   if (auto *BaseCtor = SMOR.getMethod()) {
     visitSubobjectCall(Base, BaseCtor);
@@ -13940,8 +13968,9 @@ bool SpecialMemberExceptionSpecInfo::visitField(FieldDecl *FD) {
       ExceptSpec.CalledExpr(E);
   } else if (auto *RT = S.Context.getBaseElementType(FD->getType())
                             ->getAs<RecordType>()) {
-    visitClassSubobject(cast<CXXRecordDecl>(RT->getDecl()), FD,
-                        FD->getType().getCVRQualifiers());
+    visitClassSubobject(
+        cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf(), FD,
+        FD->getType().getCVRQualifiers());
   }
   return false;
 }
@@ -14003,7 +14032,7 @@ ComputeDefaultedSpecialMemberExceptionSpec(
   // attempting to resolve an exception specification before it's known
   // at a higher level.
   if (S.RequireCompleteType(MD->getLocation(),
-                            S.Context.getRecordType(ClassDecl),
+                            S.Context.getCanonicalTagType(ClassDecl),
                             diag::err_exception_spec_incomplete_type))
     return Info.ExceptSpec;
 
@@ -14134,8 +14163,7 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
       *this, ClassDecl, CXXSpecialMemberKind::DefaultConstructor, false);
 
   // Create the actual constructor declaration.
-  CanQualType ClassType
-    = Context.getCanonicalType(Context.getTypeDeclType(ClassDecl));
+  CanQualType ClassType = Context.getCanonicalTagType(ClassDecl);
   SourceLocation ClassLoc = ClassDecl->getLocation();
   DeclarationName Name
     = Context.DeclarationNames.getCXXConstructorName(ClassType);
@@ -14420,8 +14448,7 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
       *this, ClassDecl, CXXSpecialMemberKind::Destructor, false);
 
   // Create the actual destructor declaration.
-  CanQualType ClassType
-    = Context.getCanonicalType(Context.getTypeDeclType(ClassDecl));
+  CanQualType ClassType = Context.getCanonicalTagType(ClassDecl);
   SourceLocation ClassLoc = ClassDecl->getLocation();
   DeclarationName Name
     = Context.DeclarationNames.getCXXDestructorName(ClassType);
@@ -14759,9 +14786,10 @@ buildMemcpyForAssignmentOp(Sema &S, SourceLocation Loc, QualType T,
       VK_PRValue, OK_Ordinary, Loc, false, S.CurFPFeatureOverrides());
 
   const Type *E = T->getBaseElementTypeUnsafe();
-  bool NeedsCollectableMemCpy =
-      E->isRecordType() &&
-      E->castAs<RecordType>()->getDecl()->hasObjectMember();
+  bool NeedsCollectableMemCpy = E->isRecordType() && E->castAs<RecordType>()
+                                                         ->getOriginalDecl()
+                                                         ->getDefinitionOrSelf()
+                                                         ->hasObjectMember();
 
   // Create a reference to the __builtin_objc_memmove_collectable function
   StringRef MemCpyName = NeedsCollectableMemCpy ?
@@ -14838,7 +14866,8 @@ buildSingleCopyAssignRecursively(Sema &S, SourceLocation Loc, QualType T,
   //       ignoring any possible virtual overriding functions in more derived
   //       classes);
   if (const RecordType *RecordTy = T->getAs<RecordType>()) {
-    CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(RecordTy->getDecl());
+    CXXRecordDecl *ClassDecl =
+        cast<CXXRecordDecl>(RecordTy->getOriginalDecl())->getDefinitionOrSelf();
 
     // Look for operator=.
     DeclarationName Name
@@ -14885,10 +14914,9 @@ buildSingleCopyAssignRecursively(Sema &S, SourceLocation Loc, QualType T,
     // reference to operator=; this is required to suppress the virtual
     // call mechanism.
     CXXScopeSpec SS;
+    // FIXME: Don't canonicalize this.
     const Type *CanonicalT = S.Context.getCanonicalType(T.getTypePtr());
-    SS.MakeTrivial(S.Context,
-                   NestedNameSpecifier::Create(S.Context, nullptr, CanonicalT),
-                   Loc);
+    SS.MakeTrivial(S.Context, NestedNameSpecifier(CanonicalT), Loc);
 
     // Create the reference to operator=.
     ExprResult OpEqualRef
@@ -15042,9 +15070,9 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
-  QualType ArgType = Context.getTypeDeclType(ClassDecl);
-  ArgType = Context.getElaboratedType(ElaboratedTypeKeyword::None, nullptr,
-                                      ArgType, nullptr);
+  QualType ArgType = Context.getTagType(ElaboratedTypeKeyword::None,
+                                        /*Qualifier=*/std::nullopt, ClassDecl,
+                                        /*OwnsTag=*/false);
   LangAS AS = getDefaultCXXMethodAddrSpace();
   if (AS != LangAS::Default)
     ArgType = Context.getAddrSpaceQualType(ArgType, AS);
@@ -15303,7 +15331,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     // Check for members of reference type; we can't copy those.
     if (Field->getType()->isReferenceType()) {
       Diag(ClassDecl->getLocation(), diag::err_uninitialized_member_for_assign)
-        << Context.getTagDeclType(ClassDecl) << 0 << Field->getDeclName();
+          << Context.getCanonicalTagType(ClassDecl) << 0
+          << Field->getDeclName();
       Diag(Field->getLocation(), diag::note_declared_at);
       Invalid = true;
       continue;
@@ -15313,7 +15342,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
     QualType BaseType = Context.getBaseElementType(Field->getType());
     if (!BaseType->getAs<RecordType>() && BaseType.isConstQualified()) {
       Diag(ClassDecl->getLocation(), diag::err_uninitialized_member_for_assign)
-        << Context.getTagDeclType(ClassDecl) << 1 << Field->getDeclName();
+          << Context.getCanonicalTagType(ClassDecl) << 1
+          << Field->getDeclName();
       Diag(Field->getLocation(), diag::note_declared_at);
       Invalid = true;
       continue;
@@ -15398,9 +15428,9 @@ CXXMethodDecl *Sema::DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl) {
   // Note: The following rules are largely analoguous to the move
   // constructor rules.
 
-  QualType ArgType = Context.getTypeDeclType(ClassDecl);
-  ArgType = Context.getElaboratedType(ElaboratedTypeKeyword::None, nullptr,
-                                      ArgType, nullptr);
+  QualType ArgType = Context.getTagType(ElaboratedTypeKeyword::None,
+                                        /*Qualifier=*/std::nullopt, ClassDecl,
+                                        /*OwnsTag=*/false);
   LangAS AS = getDefaultCXXMethodAddrSpace();
   if (AS != LangAS::Default)
     ArgType = Context.getAddrSpaceQualType(ArgType, AS);
@@ -15690,7 +15720,8 @@ void Sema::DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
     // Check for members of reference type; we can't move those.
     if (Field->getType()->isReferenceType()) {
       Diag(ClassDecl->getLocation(), diag::err_uninitialized_member_for_assign)
-        << Context.getTagDeclType(ClassDecl) << 0 << Field->getDeclName();
+          << Context.getCanonicalTagType(ClassDecl) << 0
+          << Field->getDeclName();
       Diag(Field->getLocation(), diag::note_declared_at);
       Invalid = true;
       continue;
@@ -15700,7 +15731,8 @@ void Sema::DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
     QualType BaseType = Context.getBaseElementType(Field->getType());
     if (!BaseType->getAs<RecordType>() && BaseType.isConstQualified()) {
       Diag(ClassDecl->getLocation(), diag::err_uninitialized_member_for_assign)
-        << Context.getTagDeclType(ClassDecl) << 1 << Field->getDeclName();
+          << Context.getCanonicalTagType(ClassDecl) << 1
+          << Field->getDeclName();
       Diag(Field->getLocation(), diag::note_declared_at);
       Invalid = true;
       continue;
@@ -15791,10 +15823,10 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
-  QualType ClassType = Context.getTypeDeclType(ClassDecl);
+  QualType ClassType = Context.getTagType(ElaboratedTypeKeyword::None,
+                                          /*Qualifier=*/std::nullopt, ClassDecl,
+                                          /*OwnsTag=*/false);
   QualType ArgType = ClassType;
-  ArgType = Context.getElaboratedType(ElaboratedTypeKeyword::None, nullptr,
-                                      ArgType, nullptr);
   bool Const = ClassDecl->implicitCopyConstructorHasConstParam();
   if (Const)
     ArgType = ArgType.withConst();
@@ -15938,11 +15970,11 @@ CXXConstructorDecl *Sema::DeclareImplicitMoveConstructor(
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
-  QualType ClassType = Context.getTypeDeclType(ClassDecl);
+  QualType ClassType = Context.getTagType(ElaboratedTypeKeyword::None,
+                                          /*Qualifier=*/std::nullopt, ClassDecl,
+                                          /*OwnsTag=*/false);
 
   QualType ArgType = ClassType;
-  ArgType = Context.getElaboratedType(ElaboratedTypeKeyword::None, nullptr,
-                                      ArgType, nullptr);
   LangAS AS = getDefaultCXXMethodAddrSpace();
   if (AS != LangAS::Default)
     ArgType = Context.getAddrSpaceQualType(ClassType, AS);
@@ -16297,7 +16329,8 @@ void Sema::FinalizeVarWithDestructor(VarDecl *VD, const RecordType *Record) {
   if (VD->getInit() && VD->getInit()->containsErrors())
     return;
 
-  CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(Record->getDecl());
+  CXXRecordDecl *ClassDecl =
+      cast<CXXRecordDecl>(Record->getOriginalDecl())->getDefinitionOrSelf();
   if (ClassDecl->isInvalidDecl()) return;
   if (ClassDecl->hasIrrelevantDestructor()) return;
   if (ClassDecl->isDependentContext()) return;
@@ -16639,10 +16672,10 @@ static inline bool CheckOperatorNewDeleteTypes(
 
   if (CheckType(SizeParameterIndex, SemaRef.Context.getSizeType(), "size_t"))
     return true;
-  TypeDecl *StdAlignValTDecl = SemaRef.getStdAlignValT();
-  QualType StdAlignValT =
-      StdAlignValTDecl ? SemaRef.Context.getTypeDeclType(StdAlignValTDecl)
-                       : QualType();
+  TagDecl *StdAlignValTDecl = SemaRef.getStdAlignValT();
+  CanQualType StdAlignValT =
+      StdAlignValTDecl ? SemaRef.Context.getCanonicalTagType(StdAlignValTDecl)
+                       : CanQualType();
   if (CheckType(SizeParameterIndex + 1, StdAlignValT, "std::align_val_t"))
     return true;
 
@@ -16682,8 +16715,8 @@ CheckOperatorDeleteDeclaration(Sema &SemaRef, FunctionDecl *FnDecl) {
   auto *MD = dyn_cast<CXXMethodDecl>(FnDecl);
   auto ConstructDestroyingDeleteAddressType = [&]() {
     assert(MD);
-    return SemaRef.Context.getCanonicalType(SemaRef.Context.getPointerType(
-        SemaRef.Context.getRecordType(MD->getParent())));
+    return SemaRef.Context.getPointerType(
+        SemaRef.Context.getCanonicalTagType(MD->getParent()));
   };
 
   // C++ P2719: A destroying operator delete cannot be type aware
@@ -16722,8 +16755,8 @@ CheckOperatorDeleteDeclaration(Sema &SemaRef, FunctionDecl *FnDecl) {
   //   function shall be of type void *.
   CanQualType ExpectedAddressParamType =
       MD && IsPotentiallyDestroyingOperatorDelete(SemaRef, MD)
-          ? SemaRef.Context.getCanonicalType(SemaRef.Context.getPointerType(
-                SemaRef.Context.getRecordType(MD->getParent())))
+          ? SemaRef.Context.getPointerType(
+                SemaRef.Context.getCanonicalTagType(MD->getParent()))
           : SemaRef.Context.VoidPtrTy;
 
   // C++ [basic.stc.dynamic.deallocation]p2:
@@ -17954,27 +17987,14 @@ DeclResult Sema::ActOnTemplatedFriendTag(
                       /*OOK=*/OffsetOfKind::Outside);
     }
 
+    TypeSourceInfo *TSI = nullptr;
     ElaboratedTypeKeyword Keyword
       = TypeWithKeyword::getKeywordForTagTypeKind(Kind);
-    QualType T = CheckTypenameType(Keyword, TagLoc, QualifierLoc,
-                                   *Name, NameLoc);
+    QualType T = CheckTypenameType(Keyword, TagLoc, QualifierLoc, *Name,
+                                   NameLoc, &TSI, /*DeducedTSTContext=*/true);
     if (T.isNull())
       return true;
 
-    TypeSourceInfo *TSI = Context.CreateTypeSourceInfo(T);
-    if (isa<DependentNameType>(T)) {
-      DependentNameTypeLoc TL =
-          TSI->getTypeLoc().castAs<DependentNameTypeLoc>();
-      TL.setElaboratedKeywordLoc(TagLoc);
-      TL.setQualifierLoc(QualifierLoc);
-      TL.setNameLoc(NameLoc);
-    } else {
-      ElaboratedTypeLoc TL = TSI->getTypeLoc().castAs<ElaboratedTypeLoc>();
-      TL.setElaboratedKeywordLoc(TagLoc);
-      TL.setQualifierLoc(QualifierLoc);
-      TL.getNamedTypeLoc().castAs<TypeSpecTypeLoc>().setNameLoc(NameLoc);
-    }
-
     FriendDecl *Friend =
         FriendDecl::Create(Context, CurContext, NameLoc, TSI, FriendLoc,
                            EllipsisLoc, TempParamLists);
@@ -17992,7 +18012,8 @@ DeclResult Sema::ActOnTemplatedFriendTag(
   collectUnexpandedParameterPacks(QualifierLoc, Unexpanded);
   unsigned FriendDeclDepth = TempParamLists.front()->getDepth();
   for (UnexpandedParameterPack &U : Unexpanded) {
-    if (getDepthAndIndex(U).first >= FriendDeclDepth) {
+    if (std::optional<std::pair<unsigned, unsigned>> DI = getDepthAndIndex(U);
+        DI && DI->first >= FriendDeclDepth) {
       auto *ND = dyn_cast<NamedDecl *>(U.first);
       if (!ND)
         ND = cast<const TemplateTypeParmType *>(U.first)->getDecl();
@@ -18286,7 +18307,7 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D,
   //   - There's a non-dependent scope specifier, in which case we
   //     compute it and do a previous lookup there for a function
   //     or function template.
-  } else if (!SS.getScopeRep()->isDependent()) {
+  } else if (!SS.getScopeRep().isDependent()) {
     DC = computeDeclContext(SS);
     if (!DC) return nullptr;
 
@@ -18793,7 +18814,7 @@ bool Sema::CheckOverridingFunctionReturnType(const CXXMethodDecl *New,
     //   complete at the point of declaration of D::f or shall be the class
     //   type D.
     if (const RecordType *RT = NewClassTy->getAs<RecordType>()) {
-      if (!RT->isBeingDefined() &&
+      if (!RT->getOriginalDecl()->isEntityBeingDefined() &&
           RequireCompleteType(New->getLocation(), NewClassTy,
                               diag::err_covariant_return_incomplete,
                               New->getDeclName()))
@@ -19148,8 +19169,9 @@ void Sema::MarkVirtualMembersReferenced(SourceLocation Loc,
     return;
 
   for (const auto &I : RD->bases()) {
-    const auto *Base =
-        cast<CXXRecordDecl>(I.getType()->castAs<RecordType>()->getDecl());
+    const auto *Base = cast<CXXRecordDecl>(
+                           I.getType()->castAs<RecordType>()->getOriginalDecl())
+                           ->getDefinitionOrSelf();
     if (Base->getNumVBases() == 0)
       continue;
     MarkVirtualMembersReferenced(Loc, Base);
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index bbd104909956f..88ed83eca243e 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -591,12 +591,13 @@ void SemaObjC::ActOnSuperClassOfClassInterface(
       // The previous declaration was not a class decl. Check if we have a
       // typedef. If we do, get the underlying class type.
       if (const TypedefNameDecl *TDecl =
-          dyn_cast_or_null<TypedefNameDecl>(PrevDecl)) {
+              dyn_cast_or_null<TypedefNameDecl>(PrevDecl)) {
         QualType T = TDecl->getUnderlyingType();
         if (T->isObjCObjectType()) {
           if (NamedDecl *IDecl = T->castAs<ObjCObjectType>()->getInterface()) {
             SuperClassDecl = dyn_cast<ObjCInterfaceDecl>(IDecl);
-            SuperClassType = Context.getTypeDeclType(TDecl);
+            SuperClassType = Context.getTypeDeclType(
+                ElaboratedTypeKeyword::None, /*Qualifier=*/std::nullopt, TDecl);
 
             // This handles the following case:
             // @interface NewI @end
@@ -1389,11 +1390,9 @@ class ObjCTypeArgOrProtocolValidatorCCC final
 
         // Make sure the type is something we would accept as a type
         // argument.
-        auto type = Context.getTypeDeclType(typeDecl);
-        if (type->isObjCObjectPointerType() ||
-            type->isBlockPointerType() ||
+        if (CanQualType type = Context.getCanonicalTypeDeclType(typeDecl);
             type->isDependentType() ||
-            type->isObjCObjectType())
+            isa<ObjCObjectPointerType, BlockPointerType, ObjCObjectType>(type))
           return true;
 
         return false;
@@ -1589,7 +1588,9 @@ void SemaObjC::actOnObjCTypeArgsOrProtocolQualifiers(
     unsigned diagID; // unused
     QualType type;
     if (auto *actualTypeDecl = dyn_cast<TypeDecl *>(typeDecl))
-      type = Context.getTypeDeclType(actualTypeDecl);
+      type =
+          Context.getTypeDeclType(ElaboratedTypeKeyword::None,
+                                  /*Qualifier=*/std::nullopt, actualTypeDecl);
     else
       type = Context.getObjCInterfaceType(cast<ObjCInterfaceDecl *>(typeDecl));
     TypeSourceInfo *parsedTSInfo = Context.getTrivialTypeSourceInfo(type, loc);
@@ -3231,8 +3232,10 @@ static bool tryMatchRecordTypes(ASTContext &Context,
   assert(lt && rt && lt != rt);
 
   if (!isa<RecordType>(lt) || !isa<RecordType>(rt)) return false;
-  RecordDecl *left = cast<RecordType>(lt)->getDecl();
-  RecordDecl *right = cast<RecordType>(rt)->getDecl();
+  RecordDecl *left =
+      cast<RecordType>(lt)->getOriginalDecl()->getDefinitionOrSelf();
+  RecordDecl *right =
+      cast<RecordType>(rt)->getOriginalDecl()->getDefinitionOrSelf();
 
   // Require union-hood to match.
   if (left->isUnion() != right->isUnion()) return false;
@@ -3846,7 +3849,9 @@ static bool IsVariableSizedType(QualType T) {
   if (T->isIncompleteArrayType())
     return true;
   const auto *RecordTy = T->getAs<RecordType>();
-  return (RecordTy && RecordTy->getDecl()->hasFlexibleArrayMember());
+  return (RecordTy && RecordTy->getOriginalDecl()
+                          ->getDefinitionOrSelf()
+                          ->hasFlexibleArrayMember());
 }
 
 static void DiagnoseVariableSizedIvars(Sema &S, ObjCContainerDecl *OCD) {
@@ -3892,7 +3897,9 @@ static void DiagnoseVariableSizedIvars(Sema &S, ObjCContainerDecl *OCD) {
           << TagTypeKind::Class; // Use "class" for Obj-C.
       IsInvalidIvar = true;
     } else if (const RecordType *RecordTy = IvarTy->getAs<RecordType>()) {
-      if (RecordTy->getDecl()->hasFlexibleArrayMember()) {
+      if (RecordTy->getOriginalDecl()
+              ->getDefinitionOrSelf()
+              ->hasFlexibleArrayMember()) {
         S.Diag(ivar->getLocation(),
                diag::err_objc_variable_sized_type_not_at_end)
             << ivar->getDeclName() << IvarTy;
@@ -5537,7 +5544,8 @@ void SemaObjC::SetIvarInitializers(ObjCImplementationDecl *ObjCImplementation) {
       if (const RecordType *RecordTy =
               Context.getBaseElementType(Field->getType())
                   ->getAs<RecordType>()) {
-        CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getDecl());
+        CXXRecordDecl *RD = cast<CXXRecordDecl>(RecordTy->getOriginalDecl())
+                                ->getDefinitionOrSelf();
         if (CXXDestructorDecl *Destructor = SemaRef.LookupDestructor(RD)) {
           SemaRef.MarkFunctionReferenced(Field->getLocation(), Destructor);
           SemaRef.CheckDestructorAccess(
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 0a6cea8869c14..94413b5b92d22 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -163,8 +163,9 @@ bool Sema::CheckSpecifiedExceptionType(QualType &T, SourceRange Range) {
     DiagID = diag::ext_incomplete_in_exception_spec;
     ReturnValueOnError = false;
   }
-  if (!(PointeeT->isRecordType() &&
-        PointeeT->castAs<RecordType>()->isBeingDefined()) &&
+  if (!(PointeeT->isRecordType() && PointeeT->castAs<RecordType>()
+                                        ->getOriginalDecl()
+                                        ->isEntityBeingDefined()) &&
       RequireCompleteType(Range.getBegin(), PointeeT, DiagID, Kind, Range))
     return ReturnValueOnError;
 
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 6793d6da85cb1..a73093545aa7f 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -1528,8 +1528,8 @@ void Sema::checkEnumArithmeticConversions(Expr *LHS, Expr *RHS,
     // are ill-formed.
     if (getLangOpts().CPlusPlus26)
       DiagID = diag::warn_conv_mixed_enum_types_cxx26;
-    else if (!L->castAs<EnumType>()->getDecl()->hasNameForLinkage() ||
-             !R->castAs<EnumType>()->getDecl()->hasNameForLinkage()) {
+    else if (!L->castAs<EnumType>()->getOriginalDecl()->hasNameForLinkage() ||
+             !R->castAs<EnumType>()->getOriginalDecl()->hasNameForLinkage()) {
       // If either enumeration type is unnamed, it's less likely that the
       // user cares about this, but this situation is still deprecated in
       // C++2a. Use a different warning group.
@@ -2531,7 +2531,7 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
     if (isa<CXXRecordDecl>(DC)) {
       if (ExplicitTemplateArgs) {
         if (LookupTemplateName(
-                R, S, SS, Context.getRecordType(cast<CXXRecordDecl>(DC)),
+                R, S, SS, Context.getCanonicalTagType(cast<CXXRecordDecl>(DC)),
                 /*EnteringContext*/ false, TemplateNameIsRequired,
                 /*RequiredTemplateKind*/ nullptr, /*AllowTypoCorrection*/ true))
           return true;
@@ -2607,11 +2607,8 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
       }
       R.addDecl(ND);
       if (getLangOpts().CPlusPlus && ND->isCXXClassMember()) {
-        CXXRecordDecl *Record = nullptr;
-        if (Corrected.getCorrectionSpecifier()) {
-          const Type *Ty = Corrected.getCorrectionSpecifier()->getAsType();
-          Record = Ty->getAsCXXRecordDecl();
-        }
+        CXXRecordDecl *Record =
+            Corrected.getCorrectionSpecifier().getAsRecordDecl();
         if (!Record)
           Record = cast<CXXRecordDecl>(
               ND->getDeclContext()->getRedeclContext());
@@ -2707,8 +2704,7 @@ recoverFromMSUnqualifiedLookup(Sema &S, ASTContext &Context,
   // Synthesize a fake NNS that points to the derived class.  This will
   // perform name lookup during template instantiation.
   CXXScopeSpec SS;
-  auto *NNS =
-      NestedNameSpecifier::Create(Context, nullptr, RD->getTypeForDecl());
+  NestedNameSpecifier NNS(Context.getCanonicalTagType(RD)->getTypePtr());
   SS.MakeTrivial(Context, NNS, SourceRange(Loc, Loc));
   return DependentScopeDeclRefExpr::Create(
       Context, SS.getWithLocInContext(Context), TemplateKWLoc, NameInfo,
@@ -2831,8 +2827,7 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
 
     // If this name wasn't predeclared and if this is not a function
     // call, diagnose the problem.
-    DefaultFilterCCC DefaultValidator(II, SS.isValid() ? SS.getScopeRep()
-                                                       : nullptr);
+    DefaultFilterCCC DefaultValidator(II, SS.getScopeRep());
     DefaultValidator.IsAddressOfOperand = IsAddressOfOperand;
     assert((!CCC || CCC->IsAddressOfOperand == IsAddressOfOperand) &&
            "Typo correction callback misconfigured");
@@ -2943,8 +2938,28 @@ ExprResult Sema::BuildQualifiedDeclarationNameExpr(
   }
 
   if (const TypeDecl *TD = R.getAsSingle<TypeDecl>()) {
-    QualType Ty = Context.getTypeDeclType(TD);
-    QualType ET = getElaboratedType(ElaboratedTypeKeyword::None, SS, Ty);
+    QualType ET;
+    TypeLocBuilder TLB;
+    if (auto *TagD = dyn_cast<TagDecl>(TD)) {
+      ET = SemaRef.Context.getTagType(ElaboratedTypeKeyword::None,
+                                      SS.getScopeRep(), TagD,
+                                      /*OwnsTag=*/false);
+      auto TL = TLB.push<TagTypeLoc>(ET);
+      TL.setElaboratedKeywordLoc(SourceLocation());
+      TL.setQualifierLoc(SS.getWithLocInContext(Context));
+      TL.setNameLoc(NameInfo.getLoc());
+    } else if (auto *TypedefD = dyn_cast<TypedefNameDecl>(TD)) {
+      ET = SemaRef.Context.getTypedefType(ElaboratedTypeKeyword::None,
+                                          SS.getScopeRep(), TypedefD);
+      TLB.push<TypedefTypeLoc>(ET).set(
+          /*ElaboratedKeywordLoc=*/SourceLocation(),
+          SS.getWithLocInContext(Context), NameInfo.getLoc());
+    } else {
+      // FIXME: What else can appear here?
+      ET = SemaRef.Context.getTypeDeclType(TD);
+      TLB.pushTypeSpec(ET).setNameLoc(NameInfo.getLoc());
+      assert(SS.isEmpty());
+    }
 
     // Diagnose a missing typename if this resolved unambiguously to a type in
     // a dependent context.  If we can recover with a type, downgrade this to
@@ -2965,13 +2980,6 @@ ExprResult Sema::BuildQualifiedDeclarationNameExpr(
     D << FixItHint::CreateInsertion(Loc, "typename ");
 
     // Recover by pretending this was an elaborated type.
-    TypeLocBuilder TLB;
-    TLB.pushTypeSpec(Ty).setNameLoc(NameInfo.getLoc());
-
-    ElaboratedTypeLoc QTL = TLB.push<ElaboratedTypeLoc>(ET);
-    QTL.setElaboratedKeywordLoc(SourceLocation());
-    QTL.setQualifierLoc(SS.getWithLocInContext(Context));
-
     *RecoveryTSI = TLB.getTypeSourceInfo(Context, ET);
 
     return ExprEmpty();
@@ -2987,11 +2995,10 @@ ExprResult Sema::BuildQualifiedDeclarationNameExpr(
   return BuildDeclarationNameExpr(SS, R, /*ADL=*/false);
 }
 
-ExprResult
-Sema::PerformObjectMemberConversion(Expr *From,
-                                    NestedNameSpecifier *Qualifier,
-                                    NamedDecl *FoundDecl,
-                                    NamedDecl *Member) {
+ExprResult Sema::PerformObjectMemberConversion(Expr *From,
+                                               NestedNameSpecifier Qualifier,
+                                               NamedDecl *FoundDecl,
+                                               NamedDecl *Member) {
   const auto *RD = dyn_cast<CXXRecordDecl>(Member->getDeclContext());
   if (!RD)
     return From;
@@ -3002,7 +3009,7 @@ Sema::PerformObjectMemberConversion(Expr *From,
   QualType FromType = From->getType();
   bool PointerConversions = false;
   if (isa<FieldDecl>(Member)) {
-    DestRecordType = Context.getCanonicalType(Context.getTypeDeclType(RD));
+    DestRecordType = Context.getCanonicalTagType(RD);
     auto FromPtrType = FromType->getAs<PointerType>();
     DestRecordType = Context.getAddrSpaceQualType(
         DestRecordType, FromPtrType
@@ -3080,8 +3087,8 @@ Sema::PerformObjectMemberConversion(Expr *From,
   //     x = 17; // error: ambiguous base subobjects
   //     Derived1::x = 17; // okay, pick the Base subobject of Derived1
   //   }
-  if (Qualifier && Qualifier->getAsType()) {
-    QualType QType = QualType(Qualifier->getAsType(), 0);
+  if (Qualifier.getKind() == NestedNameSpecifier::Kind::Type) {
+    QualType QType = QualType(Qualifier.getAsType(), 0);
     assert(QType->isRecordType() && "lookup done with non-record type");
 
     QualType QRecordType = QualType(QType->castAs<RecordType>(), 0);
@@ -3248,10 +3255,6 @@ ExprResult Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
   return ULE;
 }
 
-static void diagnoseUncapturableValueReferenceOrBinding(Sema &S,
-                                                        SourceLocation loc,
-                                                        ValueDecl *var);
-
 ExprResult Sema::BuildDeclarationNameExpr(
     const CXXScopeSpec &SS, const DeclarationNameInfo &NameInfo, NamedDecl *D,
     NamedDecl *FoundD, const TemplateArgumentListInfo *TemplateArgs,
@@ -4483,9 +4486,6 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
     case Type::BitInt:
     case Type::HLSLInlineSpirv:
       llvm_unreachable("type class is never variably-modified!");
-    case Type::Elaborated:
-      T = cast<ElaboratedType>(Ty)->getNamedType();
-      break;
     case Type::Adjusted:
       T = cast<AdjustedType>(Ty)->getOriginalType();
       break;
@@ -5877,7 +5877,7 @@ static bool isParenthetizedAndQualifiedAddressOfExpr(Expr *Fn) {
     return DRE->hasQualifier();
   }
   if (auto *OVL = dyn_cast<OverloadExpr>(UO->getSubExpr()->IgnoreParens()))
-    return OVL->getQualifier();
+    return bool(OVL->getQualifier());
   return false;
 }
 
@@ -7076,7 +7076,7 @@ ExprResult Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
     for (unsigned i = 0, e = Args.size(); i != e; i++) {
       if (const auto *RT =
               dyn_cast<RecordType>(Args[i]->getType().getCanonicalType())) {
-        if (RT->getDecl()->isOrContainsUnion())
+        if (RT->getOriginalDecl()->isOrContainsUnion())
           Diag(Args[i]->getBeginLoc(), diag::warn_cmse_nonsecure_union)
               << 0 << i;
       }
@@ -8430,9 +8430,11 @@ static bool checkVectorResult(Sema &S, QualType CondTy, QualType VecResTy,
   QualType CVE = CV->getElementType();
   QualType RVE = RV->getElementType();
 
-  if (S.Context.getTypeSize(CVE) != S.Context.getTypeSize(RVE)) {
+  // Boolean vectors are permitted outside of OpenCL mode.
+  if (S.Context.getTypeSize(CVE) != S.Context.getTypeSize(RVE) &&
+      (!CVE->isBooleanType() || S.LangOpts.OpenCL)) {
     S.Diag(QuestionLoc, diag::err_conditional_vector_element_size)
-      << CondTy << VecResTy;
+        << CondTy << VecResTy;
     return true;
   }
 
@@ -8605,7 +8607,8 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
   // type.
   if (const RecordType *LHSRT = LHSTy->getAs<RecordType>()) {    // C99 6.5.15p3
     if (const RecordType *RHSRT = RHSTy->getAs<RecordType>())
-      if (LHSRT->getDecl() == RHSRT->getDecl())
+      if (declaresSameEntity(LHSRT->getOriginalDecl(),
+                             RHSRT->getOriginalDecl()))
         // "If both the operands have structure or union type, the result has
         // that type."  This implies that CV qualifiers are dropped.
         return Context.getCommonSugaredType(LHSTy.getUnqualifiedType(),
@@ -9312,14 +9315,14 @@ AssignConvertType Sema::CheckAssignmentConstraints(QualType LHSType,
   // If we have an atomic type, try a non-atomic assignment, then just add an
   // atomic qualification step.
   if (const AtomicType *AtomicTy = dyn_cast<AtomicType>(LHSType)) {
-    AssignConvertType result =
+    AssignConvertType Result =
         CheckAssignmentConstraints(AtomicTy->getValueType(), RHS, Kind);
-    if (result != AssignConvertType::Compatible)
-      return result;
+    if (!IsAssignConvertCompatible(Result))
+      return Result;
     if (Kind != CK_NoOp && ConvertRHS)
       RHS = ImpCastExprToType(RHS.get(), AtomicTy->getValueType(), Kind);
     Kind = CK_NonAtomicToAtomic;
-    return AssignConvertType::Compatible;
+    return Result;
   }
 
   // If the left-hand side is a reference type, then we are in a
@@ -9674,11 +9677,14 @@ Sema::CheckTransparentUnionArgumentConstraints(QualType ArgType,
   // If the ArgType is a Union type, we want to handle a potential
   // transparent_union GCC extension.
   const RecordType *UT = ArgType->getAsUnionType();
-  if (!UT || !UT->getDecl()->hasAttr<TransparentUnionAttr>())
+  if (!UT)
+    return AssignConvertType::Incompatible;
+
+  RecordDecl *UD = UT->getOriginalDecl()->getDefinitionOrSelf();
+  if (!UD->hasAttr<TransparentUnionAttr>())
     return AssignConvertType::Incompatible;
 
   // The field to initialize within the transparent union.
-  RecordDecl *UD = UT->getDecl();
   FieldDecl *InitField = nullptr;
   // It's compatible if the expression matches any of the fields.
   for (auto *it : UD->fields()) {
@@ -10742,9 +10748,50 @@ static void DiagnoseBadDivideOrRemainderValues(Sema& S, ExprResult &LHS,
                             << IsDiv << RHS.get()->getSourceRange());
 }
 
+static void diagnoseScopedEnums(Sema &S, const SourceLocation Loc,
+                                const ExprResult &LHS, const ExprResult &RHS,
+                                BinaryOperatorKind Opc) {
+  if (!LHS.isUsable() || !RHS.isUsable())
+    return;
+  const Expr *LHSExpr = LHS.get();
+  const Expr *RHSExpr = RHS.get();
+  const QualType LHSType = LHSExpr->getType();
+  const QualType RHSType = RHSExpr->getType();
+  const bool LHSIsScoped = LHSType->isScopedEnumeralType();
+  const bool RHSIsScoped = RHSType->isScopedEnumeralType();
+  if (!LHSIsScoped && !RHSIsScoped)
+    return;
+  if (BinaryOperator::isAssignmentOp(Opc) && LHSIsScoped)
+    return;
+  if (!LHSIsScoped && !LHSType->isIntegralOrUnscopedEnumerationType())
+    return;
+  if (!RHSIsScoped && !RHSType->isIntegralOrUnscopedEnumerationType())
+    return;
+  auto DiagnosticHelper = [&S](const Expr *expr, const QualType type) {
+    SourceLocation BeginLoc = expr->getBeginLoc();
+    QualType IntType = type->castAs<EnumType>()
+                           ->getOriginalDecl()
+                           ->getDefinitionOrSelf()
+                           ->getIntegerType();
+    std::string InsertionString = "static_cast<" + IntType.getAsString() + ">(";
+    S.Diag(BeginLoc, diag::note_no_implicit_conversion_for_scoped_enum)
+        << FixItHint::CreateInsertion(BeginLoc, InsertionString)
+        << FixItHint::CreateInsertion(expr->getEndLoc(), ")");
+  };
+  if (LHSIsScoped) {
+    DiagnosticHelper(LHSExpr, LHSType);
+  }
+  if (RHSIsScoped) {
+    DiagnosticHelper(RHSExpr, RHSType);
+  }
+}
+
 QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
                                            SourceLocation Loc,
-                                           bool IsCompAssign, bool IsDiv) {
+                                           BinaryOperatorKind Opc) {
+  bool IsCompAssign = Opc == BO_MulAssign || Opc == BO_DivAssign;
+  bool IsDiv = Opc == BO_Div || Opc == BO_DivAssign;
+
   checkArithmeticNull(*this, LHS, RHS, Loc, /*IsCompare=*/false);
 
   QualType LHSTy = LHS.get()->getType();
@@ -10772,9 +10819,11 @@ QualType Sema::CheckMultiplyDivideOperands(ExprResult &LHS, ExprResult &RHS,
   if (LHS.isInvalid() || RHS.isInvalid())
     return QualType();
 
-
-  if (compType.isNull() || !compType->isArithmeticType())
-    return InvalidOperands(Loc, LHS, RHS);
+  if (compType.isNull() || !compType->isArithmeticType()) {
+    QualType ResultTy = InvalidOperands(Loc, LHS, RHS);
+    diagnoseScopedEnums(*this, Loc, LHS, RHS, Opc);
+    return ResultTy;
+  }
   if (IsDiv) {
     DetectPrecisionLossInComplexDivision(*this, RHS.get()->getType(), Loc);
     DiagnoseBadDivideOrRemainderValues(*this, LHS, RHS, Loc, IsDiv);
@@ -10837,8 +10886,12 @@ QualType Sema::CheckRemainderOperands(
 
   if (compType.isNull() ||
       (!compType->isIntegerType() &&
-       !(getLangOpts().HLSL && compType->isFloatingType())))
-    return InvalidOperands(Loc, LHS, RHS);
+       !(getLangOpts().HLSL && compType->isFloatingType()))) {
+    QualType ResultTy = InvalidOperands(Loc, LHS, RHS);
+    diagnoseScopedEnums(*this, Loc, LHS, RHS,
+                        IsCompAssign ? BO_RemAssign : BO_Rem);
+    return ResultTy;
+  }
   DiagnoseBadDivideOrRemainderValues(*this, LHS, RHS, Loc, false /* IsDiv */);
   return compType;
 }
@@ -11194,7 +11247,9 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
     } else if (PExp->getType()->isObjCObjectPointerType()) {
       isObjCPointer = true;
     } else {
-      return InvalidOperands(Loc, LHS, RHS);
+      QualType ResultTy = InvalidOperands(Loc, LHS, RHS);
+      diagnoseScopedEnums(*this, Loc, LHS, RHS, Opc);
+      return ResultTy;
     }
   }
   assert(PExp->getType()->isAnyPointerType());
@@ -11251,7 +11306,8 @@ QualType Sema::CheckAdditionOperands(ExprResult &LHS, ExprResult &RHS,
 // C99 6.5.6
 QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
                                         SourceLocation Loc,
-                                        QualType* CompLHSTy) {
+                                        BinaryOperatorKind Opc,
+                                        QualType *CompLHSTy) {
   checkArithmeticNull(*this, LHS, RHS, Loc, /*IsCompare=*/false);
 
   if (LHS.get()->getType()->isVectorType() ||
@@ -11396,12 +11452,14 @@ QualType Sema::CheckSubtractionOperands(ExprResult &LHS, ExprResult &RHS,
     }
   }
 
-  return InvalidOperands(Loc, LHS, RHS);
+  QualType ResultTy = InvalidOperands(Loc, LHS, RHS);
+  diagnoseScopedEnums(*this, Loc, LHS, RHS, Opc);
+  return ResultTy;
 }
 
 static bool isScopedEnumerationType(QualType T) {
   if (const EnumType *ET = T->getAs<EnumType>())
-    return ET->getDecl()->isScoped();
+    return ET->getOriginalDecl()->isScoped();
   return false;
 }
 
@@ -11744,15 +11802,12 @@ QualType Sema::CheckShiftOperands(ExprResult &LHS, ExprResult &RHS,
   // Embedded-C 4.1.6.2.2: The LHS may also be fixed-point.
   if ((!LHSType->isFixedPointOrIntegerType() &&
        !LHSType->hasIntegerRepresentation()) ||
-      !RHSType->hasIntegerRepresentation())
-    return InvalidOperands(Loc, LHS, RHS);
-
-  // C++0x: Don't allow scoped enums. FIXME: Use something better than
-  // hasIntegerRepresentation() above instead of this.
-  if (isScopedEnumerationType(LHSType) ||
-      isScopedEnumerationType(RHSType)) {
-    return InvalidOperands(Loc, LHS, RHS);
+      !RHSType->hasIntegerRepresentation()) {
+    QualType ResultTy = InvalidOperands(Loc, LHS, RHS);
+    diagnoseScopedEnums(*this, Loc, LHS, RHS, Opc);
+    return ResultTy;
   }
+
   DiagnoseBadShiftValues(*this, LHS, RHS, Loc, Opc, LHSType);
 
   // "The type of the result is that of the promoted left operand."
@@ -12290,8 +12345,10 @@ static QualType checkArithmeticOrEnumeralThreeWayCompare(Sema &S,
       S.InvalidOperands(Loc, LHS, RHS);
       return QualType();
     }
-    QualType IntType =
-        LHSStrippedType->castAs<EnumType>()->getDecl()->getIntegerType();
+    QualType IntType = LHSStrippedType->castAs<EnumType>()
+                           ->getOriginalDecl()
+                           ->getDefinitionOrSelf()
+                           ->getIntegerType();
     assert(IntType->isArithmeticType());
 
     // We can't use `CK_IntegralCast` when the underlying type is 'bool', so we
@@ -12311,8 +12368,11 @@ static QualType checkArithmeticOrEnumeralThreeWayCompare(Sema &S,
       S.UsualArithmeticConversions(LHS, RHS, Loc, ArithConvKind::Comparison);
   if (LHS.isInvalid() || RHS.isInvalid())
     return QualType();
-  if (Type.isNull())
-    return S.InvalidOperands(Loc, LHS, RHS);
+  if (Type.isNull()) {
+    QualType ResultTy = S.InvalidOperands(Loc, LHS, RHS);
+    diagnoseScopedEnums(S, Loc, LHS, RHS, BO_Cmp);
+    return ResultTy;
+  }
 
   std::optional<ComparisonCategoryType> CCT =
       getComparisonCategoryForBuiltinCmp(Type);
@@ -12344,8 +12404,11 @@ static QualType checkArithmeticOrEnumeralCompare(Sema &S, ExprResult &LHS,
       S.UsualArithmeticConversions(LHS, RHS, Loc, ArithConvKind::Comparison);
   if (LHS.isInvalid() || RHS.isInvalid())
     return QualType();
-  if (Type.isNull())
-    return S.InvalidOperands(Loc, LHS, RHS);
+  if (Type.isNull()) {
+    QualType ResultTy = S.InvalidOperands(Loc, LHS, RHS);
+    diagnoseScopedEnums(S, Loc, LHS, RHS, Opc);
+    return ResultTy;
+  }
   assert(Type->isArithmeticType() || Type->isEnumeralType());
 
   if (Type->isAnyComplexType() && BinaryOperator::isRelationalOp(Opc))
@@ -12457,7 +12520,7 @@ QualType Sema::CheckCompareOperands(ExprResult &LHS, ExprResult &RHS,
 
   auto computeResultTy = [&]() {
     if (Opc != BO_Cmp)
-      return Context.getLogicalOperationType();
+      return QualType(Context.getLogicalOperationType());
     assert(getLangOpts().CPlusPlus);
     assert(Context.hasSameType(LHS.get()->getType(), RHS.get()->getType()));
 
@@ -13355,7 +13418,9 @@ inline QualType Sema::CheckBitwiseOperands(ExprResult &LHS, ExprResult &RHS,
 
   if (!compType.isNull() && compType->isIntegralOrUnscopedEnumerationType())
     return compType;
-  return InvalidOperands(Loc, LHS, RHS);
+  QualType ResultTy = InvalidOperands(Loc, LHS, RHS);
+  diagnoseScopedEnums(*this, Loc, LHS, RHS, Opc);
+  return ResultTy;
 }
 
 // C99 6.5.[13,14]
@@ -13457,13 +13522,19 @@ inline QualType Sema::CheckLogicalOperands(ExprResult &LHS, ExprResult &RHS,
   // C++ [expr.log.or]p1
   // The operands are both contextually converted to type bool.
   ExprResult LHSRes = PerformContextuallyConvertToBool(LHS.get());
-  if (LHSRes.isInvalid())
-    return InvalidOperands(Loc, LHS, RHS);
+  if (LHSRes.isInvalid()) {
+    QualType ResultTy = InvalidOperands(Loc, LHS, RHS);
+    diagnoseScopedEnums(*this, Loc, LHS, RHS, Opc);
+    return ResultTy;
+  }
   LHS = LHSRes;
 
   ExprResult RHSRes = PerformContextuallyConvertToBool(RHS.get());
-  if (RHSRes.isInvalid())
-    return InvalidOperands(Loc, LHS, RHS);
+  if (RHSRes.isInvalid()) {
+    QualType ResultTy = InvalidOperands(Loc, LHS, RHS);
+    diagnoseScopedEnums(*this, Loc, LHS, RHS, Opc);
+    return ResultTy;
+  }
   RHS = RHSRes;
 
   // C++ [expr.log.and]p2
@@ -13694,8 +13765,10 @@ static void DiagnoseRecursiveConstFields(Sema &S, const ValueDecl *VD,
   // diagnostics in field nesting order.
   while (RecordTypeList.size() > NextToCheckIndex) {
     bool IsNested = NextToCheckIndex > 0;
-    for (const FieldDecl *Field :
-         RecordTypeList[NextToCheckIndex]->getDecl()->fields()) {
+    for (const FieldDecl *Field : RecordTypeList[NextToCheckIndex]
+                                      ->getOriginalDecl()
+                                      ->getDefinitionOrSelf()
+                                      ->fields()) {
       // First, check every field for constness.
       QualType FieldTy = Field->getType();
       if (FieldTy.isConstQualified()) {
@@ -15057,8 +15130,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
   case BO_Mul:
   case BO_Div:
     ConvertHalfVec = true;
-    ResultTy = CheckMultiplyDivideOperands(LHS, RHS, OpLoc, false,
-                                           Opc == BO_Div);
+    ResultTy = CheckMultiplyDivideOperands(LHS, RHS, OpLoc, Opc);
     break;
   case BO_Rem:
     ResultTy = CheckRemainderOperands(LHS, RHS, OpLoc);
@@ -15069,7 +15141,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
     break;
   case BO_Sub:
     ConvertHalfVec = true;
-    ResultTy = CheckSubtractionOperands(LHS, RHS, OpLoc);
+    ResultTy = CheckSubtractionOperands(LHS, RHS, OpLoc, Opc);
     break;
   case BO_Shl:
   case BO_Shr:
@@ -15113,8 +15185,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
   case BO_MulAssign:
   case BO_DivAssign:
     ConvertHalfVec = true;
-    CompResultTy = CheckMultiplyDivideOperands(LHS, RHS, OpLoc, true,
-                                               Opc == BO_DivAssign);
+    CompResultTy = CheckMultiplyDivideOperands(LHS, RHS, OpLoc, Opc);
     CompLHSTy = CompResultTy;
     if (!CompResultTy.isNull() && !LHS.isInvalid() && !RHS.isInvalid())
       ResultTy =
@@ -15136,7 +15207,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
     break;
   case BO_SubAssign:
     ConvertHalfVec = true;
-    CompResultTy = CheckSubtractionOperands(LHS, RHS, OpLoc, &CompLHSTy);
+    CompResultTy = CheckSubtractionOperands(LHS, RHS, OpLoc, Opc, &CompLHSTy);
     if (!CompResultTy.isNull() && !LHS.isInvalid() && !RHS.isInvalid())
       ResultTy =
           CheckAssignmentOperands(LHS.get(), RHS, OpLoc, CompResultTy, Opc);
@@ -16150,7 +16221,7 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
     if (!RC)
       return ExprError(Diag(OC.LocEnd, diag::err_offsetof_record_type)
                        << CurrentType);
-    RecordDecl *RD = RC->getDecl();
+    RecordDecl *RD = RC->getOriginalDecl()->getDefinitionOrSelf();
 
     // C++ [lib.support.types]p5:
     //   The macro offsetof accepts a restricted set of type arguments in this
@@ -16211,8 +16282,8 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
     // If the member was found in a base class, introduce OffsetOfNodes for
     // the base class indirections.
     CXXBasePaths Paths;
-    if (IsDerivedFrom(OC.LocStart, CurrentType, Context.getTypeDeclType(Parent),
-                      Paths)) {
+    if (IsDerivedFrom(OC.LocStart, CurrentType,
+                      Context.getCanonicalTagType(Parent), Paths)) {
       if (Paths.getDetectedVirtual()) {
         Diag(OC.LocEnd, diag::err_offsetof_field_of_virtual_base)
           << MemberDecl->getDeclName()
@@ -16777,7 +16848,8 @@ ExprResult Sema::BuildVAArgExpr(SourceLocation BuiltinLoc,
       // that.
       QualType UnderlyingType = TInfo->getType();
       if (const auto *ET = UnderlyingType->getAs<EnumType>())
-        UnderlyingType = ET->getDecl()->getIntegerType();
+        UnderlyingType =
+            ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
       if (Context.typesAreCompatible(PromoteType, UnderlyingType,
                                      /*CompareUnqualified*/ true))
         PromoteType = QualType();
@@ -16923,7 +16995,7 @@ ExprResult Sema::ActOnSourceLocExpr(SourceLocIdentKind Kind,
         return ExprError();
     }
     ResultTy = Context.getPointerType(
-        Context.getRecordType(StdSourceLocationImplDecl).withConst());
+        Context.getCanonicalTagType(StdSourceLocationImplDecl).withConst());
     break;
   }
 
@@ -18108,6 +18180,8 @@ void Sema::PopExpressionEvaluationContext() {
     MaybeODRUseExprs.insert_range(Rec.SavedMaybeODRUseExprs);
   }
 
+  DiagnoseMisalignedMembers();
+
   // Pop the current expression evaluation context off the stack.
   ExprEvalContexts.pop_back();
 }
@@ -18642,8 +18716,9 @@ void Sema::MarkCaptureUsedInEnclosingContext(ValueDecl *Capture,
   MarkVarDeclODRUsed(Capture, Loc, *this, &CapturingScopeIndex);
 }
 
-void diagnoseUncapturableValueReferenceOrBinding(Sema &S, SourceLocation loc,
-                                                 ValueDecl *var) {
+static void diagnoseUncapturableValueReferenceOrBinding(Sema &S,
+                                                        SourceLocation loc,
+                                                        ValueDecl *var) {
   DeclContext *VarDC = var->getDeclContext();
 
   //  If the parameter still belongs to the translation unit, then
@@ -18772,7 +18847,9 @@ static bool isVariableCapturable(CapturingScopeInfo *CSI, ValueDecl *Var,
   // Prohibit structs with flexible array members too.
   // We cannot capture what is in the tail end of the struct.
   if (const RecordType *VTTy = Var->getType()->getAs<RecordType>()) {
-    if (VTTy->getDecl()->hasFlexibleArrayMember()) {
+    if (VTTy->getOriginalDecl()
+            ->getDefinitionOrSelf()
+            ->hasFlexibleArrayMember()) {
       if (Diagnose) {
         if (IsBlock)
           S.Diag(Loc, diag::err_ref_flexarray_type);
@@ -21213,7 +21290,7 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
     NamedDecl *Temp = *ULE->decls_begin();
     const bool IsTypeAliasTemplateDecl = isa<TypeAliasTemplateDecl>(Temp);
 
-    NestedNameSpecifier *NNS = ULE->getQualifierLoc().getNestedNameSpecifier();
+    NestedNameSpecifier NNS = ULE->getQualifierLoc().getNestedNameSpecifier();
     // FIXME: AssumedTemplate is not very appropriate for error recovery here,
     // as it models only the unqualified-id case, where this case can clearly be
     // qualified. Thus we can't just qualify an assumed template.
@@ -21239,16 +21316,16 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
     QualType TST;
     {
       SFINAETrap Trap(*this);
-      TST = CheckTemplateIdType(TN, NameInfo.getBeginLoc(), TAL);
+      TST = CheckTemplateIdType(ElaboratedTypeKeyword::None, TN,
+                                NameInfo.getBeginLoc(), TAL);
     }
     if (TST.isNull())
       TST = Context.getTemplateSpecializationType(
-          TN, ULE->template_arguments(), /*CanonicalArgs=*/{},
+          ElaboratedTypeKeyword::None, TN, ULE->template_arguments(),
+          /*CanonicalArgs=*/{},
           HasAnyDependentTA ? Context.DependentTy : Context.IntTy);
-    QualType ET =
-        Context.getElaboratedType(ElaboratedTypeKeyword::None, NNS, TST);
     return CreateRecoveryExpr(NameInfo.getBeginLoc(), NameInfo.getEndLoc(), {},
-                              ET);
+                              TST);
   }
 
   // Overloaded expressions.
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 0edfd6015cbd9..2a82842095446 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -57,11 +57,11 @@ using namespace sema;
 ParsedType Sema::getInheritingConstructorName(CXXScopeSpec &SS,
                                               SourceLocation NameLoc,
                                               const IdentifierInfo &Name) {
-  NestedNameSpecifier *NNS = SS.getScopeRep();
-  if ([[maybe_unused]] const IdentifierInfo *II = NNS->getAsIdentifier())
-    assert(II == &Name && "not a constructor name");
+  NestedNameSpecifier NNS = SS.getScopeRep();
+  QualType Type(NNS.getAsType(), 0);
+  if ([[maybe_unused]] const auto *DNT = dyn_cast<DependentNameType>(Type))
+    assert(DNT->getIdentifier() == &Name && "not a constructor name");
 
-  QualType Type(NNS->translateToType(Context), 0);
   // This reference to the type is located entirely at the location of the
   // final identifier in the qualified-id.
   return CreateParsedType(Type,
@@ -111,10 +111,8 @@ ParsedType Sema::getConstructorName(const IdentifierInfo &II,
     return ParsedType();
   }
 
-  QualType T = Context.getTypeDeclType(InjectedClassName);
-  DiagnoseUseOfDecl(InjectedClassName, NameLoc);
-  MarkAnyDeclReferenced(NameLoc, InjectedClassName, /*OdrUse=*/false);
-
+  QualType T = Context.getTagType(ElaboratedTypeKeyword::None, SS.getScopeRep(),
+                                  InjectedClassName, /*OwnsTag=*/false);
   return ParsedType::make(T);
 }
 
@@ -175,7 +173,7 @@ ParsedType Sema::getDestructorName(const IdentifierInfo &II,
       if (SearchType.isNull() || SearchType->isDependentType())
         return true;
 
-      QualType T = Context.getTypeDeclType(Type);
+      CanQualType T = Context.getCanonicalTypeDeclType(Type);
       return Context.hasSameUnqualifiedType(T, SearchType);
     };
 
@@ -207,7 +205,8 @@ ParsedType Sema::getDestructorName(const IdentifierInfo &II,
         NamedDecl *D = F.next();
         if (auto *TD = dyn_cast<TypeDecl>(D->getUnderlyingDecl()))
           Diag(D->getLocation(), diag::note_destructor_type_here)
-              << Context.getTypeDeclType(TD);
+              << Context.getTypeDeclType(ElaboratedTypeKeyword::None,
+                                         /*Qualifier=*/std::nullopt, TD);
         else
           Diag(D->getLocation(), diag::note_destructor_nontype_here);
 
@@ -222,11 +221,11 @@ ParsedType Sema::getDestructorName(const IdentifierInfo &II,
 
     if (TypeDecl *Type = Found.getAsSingle<TypeDecl>()) {
       if (IsAcceptableResult(Type)) {
-        QualType T = Context.getTypeDeclType(Type);
+        QualType T = Context.getTypeDeclType(ElaboratedTypeKeyword::None,
+                                             /*Qualifier=*/std::nullopt, Type);
         MarkAnyDeclReferenced(Type->getLocation(), Type, /*OdrUse=*/false);
-        return CreateParsedType(
-            Context.getElaboratedType(ElaboratedTypeKeyword::None, nullptr, T),
-            Context.getTrivialTypeSourceInfo(T, NameLoc));
+        return CreateParsedType(T,
+                                Context.getTrivialTypeSourceInfo(T, NameLoc));
       }
     }
 
@@ -311,15 +310,23 @@ ParsedType Sema::getDestructorName(const IdentifierInfo &II,
   // If both lookups succeed and find a dependent result, which result should
   // we retain? (Same question for p->~type-name().)
 
-  if (NestedNameSpecifier *Prefix =
-      SS.isSet() ? SS.getScopeRep()->getPrefix() : nullptr) {
+  auto Prefix = [&]() -> NestedNameSpecifierLoc {
+    NestedNameSpecifierLoc NNS = SS.getWithLocInContext(Context);
+    if (!NNS)
+      return NestedNameSpecifierLoc();
+    if (auto TL = NNS.getAsTypeLoc())
+      return TL.getPrefix();
+    return NNS.getAsNamespaceAndPrefix().Prefix;
+  }();
+
+  if (Prefix) {
     // This is
     //
     //   nested-name-specifier type-name :: ~ type-name
     //
     // Look for the second type-name in the nested-name-specifier.
     CXXScopeSpec PrefixSS;
-    PrefixSS.Adopt(NestedNameSpecifierLoc(Prefix, SS.location_data()));
+    PrefixSS.Adopt(Prefix);
     if (ParsedType T = LookupInNestedNameSpec(PrefixSS))
       return T;
   } else {
@@ -374,7 +381,7 @@ ParsedType Sema::getDestructorName(const IdentifierInfo &II,
     //
     // also looks for type-name in the scope. Unfortunately, we can't
     // reasonably apply this fallback for dependent nested-name-specifiers.
-    if (SS.isValid() && SS.getScopeRep()->getPrefix()) {
+    if (Prefix) {
       if (ParsedType T = LookupInScope()) {
         Diag(SS.getEndLoc(), diag::ext_qualified_dtor_named_in_lexical_scope)
             << FixItHint::CreateRemoval(SS.getRange());
@@ -419,9 +426,10 @@ ParsedType Sema::getDestructorName(const IdentifierInfo &II,
     if (auto *TD = dyn_cast<TypeDecl>(FoundDecls[0]->getUnderlyingDecl())) {
       assert(!SearchType.isNull() &&
              "should only reject a type result if we have a search type");
-      QualType T = Context.getTypeDeclType(TD);
       Diag(NameLoc, diag::err_destructor_expr_type_mismatch)
-          << T << SearchType << MakeFixItHint();
+          << Context.getTypeDeclType(ElaboratedTypeKeyword::None,
+                                     /*Qualifier=*/std::nullopt, TD)
+          << SearchType << MakeFixItHint();
     } else {
       Diag(NameLoc, diag::err_destructor_expr_nontype)
           << &II << MakeFixItHint();
@@ -435,7 +443,8 @@ ParsedType Sema::getDestructorName(const IdentifierInfo &II,
   for (NamedDecl *FoundD : FoundDecls) {
     if (auto *TD = dyn_cast<TypeDecl>(FoundD->getUnderlyingDecl()))
       Diag(FoundD->getLocation(), diag::note_destructor_type_here)
-          << Context.getTypeDeclType(TD);
+          << Context.getTypeDeclType(ElaboratedTypeKeyword::None,
+                                     /*Qualifier=*/std::nullopt, TD);
     else
       Diag(FoundD->getLocation(), diag::note_destructor_nontype_here)
           << FoundD;
@@ -501,12 +510,8 @@ bool Sema::checkLiteralOperatorId(const CXXScopeSpec &SS,
           << II << static_cast<int>(Status) << Hint;
   }
 
-  if (!SS.isValid())
-    return false;
-
-  switch (SS.getScopeRep()->getKind()) {
-  case NestedNameSpecifier::Identifier:
-  case NestedNameSpecifier::TypeSpec:
+  switch (SS.getScopeRep().getKind()) {
+  case NestedNameSpecifier::Kind::Type:
     // Per C++11 [over.literal]p2, literal operators can only be declared at
     // namespace scope. Therefore, this unqualified-id cannot name anything.
     // Reject it early, because we have no AST representation for this in the
@@ -515,9 +520,10 @@ bool Sema::checkLiteralOperatorId(const CXXScopeSpec &SS,
         << SS.getScopeRep();
     return true;
 
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Super:
-  case NestedNameSpecifier::Namespace:
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+  case NestedNameSpecifier::Kind::Namespace:
     return false;
   }
 
@@ -565,7 +571,8 @@ ExprResult Sema::BuildCXXTypeId(QualType TypeInfoType,
 
     QualType T = E->getType();
     if (const RecordType *RecordT = T->getAs<RecordType>()) {
-      CXXRecordDecl *RecordD = cast<CXXRecordDecl>(RecordT->getDecl());
+      CXXRecordDecl *RecordD = cast<CXXRecordDecl>(RecordT->getOriginalDecl())
+                                   ->getDefinitionOrSelf();
       // C++ [expr.typeid]p3:
       //   [...] If the type of the expression is a class type, the class
       //   shall be completely-defined.
@@ -659,7 +666,7 @@ Sema::ActOnCXXTypeid(SourceLocation OpLoc, SourceLocation LParenLoc,
     return ExprError(Diag(OpLoc, diag::err_no_typeid_with_fno_rtti));
   }
 
-  QualType TypeInfoType = Context.getTypeDeclType(CXXTypeInfoDecl);
+  CanQualType TypeInfoType = Context.getCanonicalTagType(CXXTypeInfoDecl);
 
   if (isType) {
     // The operand is a type; handle it as such.
@@ -939,7 +946,7 @@ collectPublicBases(CXXRecordDecl *RD,
 static void getUnambiguousPublicSubobjects(
     CXXRecordDecl *RD, llvm::SmallVectorImpl<CXXRecordDecl *> &Objects) {
   llvm::DenseMap<CXXRecordDecl *, unsigned> SubobjectsSeen;
-  llvm::SmallSet<CXXRecordDecl *, 2> VBases;
+  llvm::SmallPtrSet<CXXRecordDecl *, 2> VBases;
   llvm::SetVector<CXXRecordDecl *> PublicSubobjectsSeen;
   SubobjectsSeen[RD] = 1;
   PublicSubobjectsSeen.insert(RD);
@@ -1214,7 +1221,7 @@ QualType Sema::getCurrentThisType() {
     // This is a lambda call operator that is being instantiated as a default
     // initializer. DC must point to the enclosing class type, so we can recover
     // the 'this' type from it.
-    QualType ClassTy = Context.getTypeDeclType(cast<CXXRecordDecl>(DC));
+    CanQualType ClassTy = Context.getCanonicalTagType(cast<CXXRecordDecl>(DC));
     // There are no cv-qualifiers for 'this' within default initializers,
     // per [expr.prim.general]p4.
     ThisTy = Context.getPointerType(ClassTy);
@@ -1244,7 +1251,7 @@ Sema::CXXThisScopeRAII::CXXThisScopeRAII(Sema &S,
   else
     Record = cast<CXXRecordDecl>(ContextDecl);
 
-  QualType T = S.Context.getRecordType(Record);
+  QualType T = S.Context.getCanonicalTagType(Record);
   T = S.getASTContext().getQualifiedType(T, CXXThisTypeQuals);
 
   S.CXXThisTypeOverride =
@@ -1732,7 +1739,7 @@ static bool isNonPlacementDeallocationFunction(Sema &S, FunctionDecl *FD) {
   if (S.getLangOpts().AlignedAllocation && UsualParams < FD->getNumParams() &&
       S.Context.hasSameUnqualifiedType(
           FD->getParamDecl(UsualParams)->getType(),
-          S.Context.getTypeDeclType(S.getStdAlignValT())))
+          S.Context.getCanonicalTagType(S.getStdAlignValT())))
     ++UsualParams;
 
   return UsualParams == FD->getNumParams();
@@ -1860,8 +1867,8 @@ namespace {
           if (FunctionTemplateDecl *Best = S.getMoreSpecializedTemplate(
                   PrimaryTemplate, OtherPrimaryTemplate, SourceLocation(),
                   TPOC_Call, ImplicitArgCount,
-                  DC ? QualType(DC->getTypeForDecl(), 0) : QualType{},
-                  OtherDC ? QualType(OtherDC->getTypeForDecl(), 0) : QualType{},
+                  DC ? S.Context.getCanonicalTagType(DC) : QualType{},
+                  OtherDC ? S.Context.getCanonicalTagType(OtherDC) : QualType{},
                   false)) {
             return Best == PrimaryTemplate ? 1 : -1;
           }
@@ -1977,7 +1984,7 @@ static bool doesUsualArrayDeleteWantSize(Sema &S, SourceLocation loc,
   DeclarationName deleteName =
     S.Context.DeclarationNames.getCXXOperatorName(OO_Array_Delete);
   LookupResult ops(S, deleteName, loc, Sema::LookupOrdinaryName);
-  S.LookupQualifiedName(ops, record->getDecl());
+  S.LookupQualifiedName(ops, record->getOriginalDecl()->getDefinitionOrSelf());
 
   // We're just doing this for information.
   ops.suppressDiagnostics();
@@ -2505,7 +2512,7 @@ ExprResult Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
     // because there might not be a `std::align_val_t` type.
     EnumDecl *StdAlignValT = getStdAlignValT();
     QualType AlignValT =
-        StdAlignValT ? Context.getTypeDeclType(StdAlignValT) : SizeTy;
+        StdAlignValT ? Context.getCanonicalTagType(StdAlignValT) : SizeTy;
     IntegerLiteral AlignmentLiteral(
         Context,
         llvm::APInt(Context.getTypeSize(SizeTy),
@@ -2966,7 +2973,7 @@ bool Sema::FindAllocationFunctions(
                            isTypeAwareAllocation(IAP.PassTypeIdentity);
   if (IncludeAlignParam) {
     DeclareGlobalNewDelete();
-    AlignValT = Context.getTypeDeclType(getStdAlignValT());
+    AlignValT = Context.getCanonicalTagType(getStdAlignValT());
   }
   CXXScalarValueInitExpr Align(AlignValT, nullptr, SourceLocation());
   if (IncludeAlignParam)
@@ -3048,8 +3055,9 @@ bool Sema::FindAllocationFunctions(
   LookupResult FoundDelete(*this, DeleteName, StartLoc, LookupOrdinaryName);
   if (AllocElemType->isRecordType() &&
       DeleteScope != AllocationFunctionScope::Global) {
-    auto *RD =
-        cast<CXXRecordDecl>(AllocElemType->castAs<RecordType>()->getDecl());
+    auto *RD = cast<CXXRecordDecl>(
+                   AllocElemType->castAs<RecordType>()->getOriginalDecl())
+                   ->getDefinitionOrSelf();
     LookupQualifiedName(FoundDelete, RD);
   }
   if (FoundDelete.isAmbiguous())
@@ -3426,7 +3434,7 @@ void Sema::DeclareGlobalNewDelete() {
 
       for (int Aligned = 0; Aligned < NumAlignVariants; ++Aligned) {
         if (Aligned)
-          Params.push_back(Context.getTypeDeclType(getStdAlignValT()));
+          Params.push_back(Context.getCanonicalTagType(getStdAlignValT()));
 
         DeclareGlobalAllocationFunction(
             Context.DeclarationNames.getCXXOperatorName(Kind), Return, Params);
@@ -3483,7 +3491,7 @@ void Sema::DeclareGlobalAllocationFunction(DeclarationName Name,
   bool HasBadAllocExceptionSpec = Name.isAnyOperatorNew();
   if (HasBadAllocExceptionSpec) {
     if (!getLangOpts().CPlusPlus11) {
-      BadAllocType = Context.getTypeDeclType(getStdBadAlloc());
+      BadAllocType = Context.getCanonicalTagType(getStdBadAlloc());
       assert(StdBadAlloc && "Must have std::bad_alloc declared");
       EPI.ExceptionSpec.Type = EST_Dynamic;
       EPI.ExceptionSpec.Exceptions = llvm::ArrayRef(BadAllocType);
@@ -3497,6 +3505,19 @@ void Sema::DeclareGlobalAllocationFunction(DeclarationName Name,
   }
 
   auto CreateAllocationFunctionDecl = [&](Attr *ExtraAttr) {
+    // The MSVC STL has explicit cdecl on its (host-side) allocation function
+    // specializations for the allocation, so in order to prevent a CC clash
+    // we use the host's CC, if available, or CC_C as a fallback, for the
+    // host-side implicit decls, knowing these do not get emitted when compiling
+    // for device.
+    if (getLangOpts().CUDAIsDevice && ExtraAttr &&
+        isa<CUDAHostAttr>(ExtraAttr) &&
+        Context.getTargetInfo().getTriple().isSPIRV()) {
+      if (auto *ATI = Context.getAuxTargetInfo())
+        EPI.ExtInfo = EPI.ExtInfo.withCallingConv(ATI->getDefaultCallingConv());
+      else
+        EPI.ExtInfo = EPI.ExtInfo.withCallingConv(CallingConv::CC_C);
+    }
     QualType FnType = Context.getFunctionType(Return, Params, EPI);
     FunctionDecl *Alloc = FunctionDecl::Create(
         Context, GlobalCtx, SourceLocation(), SourceLocation(), Name, FnType,
@@ -3593,7 +3614,7 @@ FunctionDecl *Sema::FindDeallocationFunctionForDestructor(SourceLocation Loc,
   DeclarationName Name = Context.DeclarationNames.getCXXOperatorName(OO_Delete);
 
   FunctionDecl *OperatorDelete = nullptr;
-  QualType DeallocType = Context.getRecordType(RD);
+  CanQualType DeallocType = Context.getCanonicalTagType(RD);
   ImplicitDeallocationParameters IDP = {
       DeallocType, ShouldUseTypeAwareOperatorNewOrDelete(),
       AlignedAllocationMode::No, SizedDeallocationMode::No};
@@ -3627,7 +3648,7 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
   Found.suppressDiagnostics();
 
   if (!isAlignedAllocation(IDP.PassAlignment) &&
-      hasNewExtendedAlignment(*this, Context.getRecordType(RD)))
+      hasNewExtendedAlignment(*this, Context.getCanonicalTagType(RD)))
     IDP.PassAlignment = AlignedAllocationMode::Yes;
 
   // C++17 [expr.delete]p10:
@@ -4050,7 +4071,8 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
                                    : diag::warn_delete_incomplete,
                                Ex.get())) {
         if (const RecordType *RT = PointeeElem->getAs<RecordType>())
-          PointeeRD = cast<CXXRecordDecl>(RT->getDecl());
+          PointeeRD =
+              cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
       }
     }
 
@@ -4574,7 +4596,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
         // If the user-defined conversion is specified by a conversion function,
         // the initial standard conversion sequence converts the source type to
         // the implicit object parameter of the conversion function.
-        BeforeToType = Context.getTagDeclType(Conv->getParent());
+        BeforeToType = Context.getCanonicalTagType(Conv->getParent());
       } else {
         const CXXConstructorDecl *Ctor = cast<CXXConstructorDecl>(FD);
         CastKind = CK_ConstructorConversion;
@@ -4818,7 +4840,10 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     if (FromType->isVectorType() || ToType->isVectorType())
       StepTy = adjustVectorType(Context, FromType, ToType, &ElTy);
     if (ElTy->isBooleanType()) {
-      assert(FromType->castAs<EnumType>()->getDecl()->isFixed() &&
+      assert(FromType->castAs<EnumType>()
+                 ->getOriginalDecl()
+                 ->getDefinitionOrSelf()
+                 ->isFixed() &&
              SCS.Second == ICK_Integral_Promotion &&
              "only enums with fixed underlying type can promote to bool");
       From = ImpCastExprToType(From, StepTy, CK_IntegralToBoolean, VK_PRValue,
@@ -5370,16 +5395,18 @@ QualType Sema::CheckPointerToMemberOperands(ExprResult &LHS, ExprResult &RHS,
       return QualType();
     }
 
+    // FIXME: use sugared type from member pointer.
+    CanQualType RHSClassType = Context.getCanonicalTagType(RHSClass);
     CXXCastPath BasePath;
     if (CheckDerivedToBaseConversion(
-            LHSType, QualType(RHSClass->getTypeForDecl(), 0), Loc,
+            LHSType, RHSClassType, Loc,
             SourceRange(LHS.get()->getBeginLoc(), RHS.get()->getEndLoc()),
             &BasePath))
       return QualType();
 
     // Cast LHS to type of use.
-    QualType UseType = Context.getQualifiedType(RHSClass->getTypeForDecl(),
-                                                LHSType.getQualifiers());
+    QualType UseType =
+        Context.getQualifiedType(RHSClassType, LHSType.getQualifiers());
     if (isIndirect)
       UseType = Context.getPointerType(UseType);
     ExprValueKind VK = isIndirect ? VK_PRValue : LHS.get()->getValueKind();
@@ -5726,10 +5753,12 @@ QualType Sema::CheckVectorConditionalTypes(ExprResult &Cond, ExprResult &LHS,
     return {};
   }
 
+  // Boolean vectors are permitted outside of OpenCL mode.
   if (Context.getTypeSize(ResultElementTy) !=
-      Context.getTypeSize(CondElementTy)) {
-    Diag(QuestionLoc, diag::err_conditional_vector_element_size) << CondType
-                                                                 << ResultType;
+          Context.getTypeSize(CondElementTy) &&
+      (!CondElementTy->isBooleanType() || LangOpts.OpenCL)) {
+    Diag(QuestionLoc, diag::err_conditional_vector_element_size)
+        << CondType << ResultType;
     return {};
   }
 
@@ -6184,7 +6213,7 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
       case Pointer:
         return Ctx.getPointerType(T);
       case MemberPointer:
-        return Ctx.getMemberPointerType(T, /*Qualifier=*/nullptr,
+        return Ctx.getMemberPointerType(T, /*Qualifier=*/std::nullopt,
                                         ClassOrBound->getAsCXXRecordDecl());
       case ObjCPointer:
         return Ctx.getObjCObjectPointerType(T);
@@ -6357,7 +6386,7 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
         return QualType();
 
       Steps.emplace_back(Step::MemberPointer,
-                         Context.getTypeDeclType(Cls).getTypePtr());
+                         Context.getCanonicalTagType(Cls).getTypePtr());
       continue;
     }
 
@@ -6636,7 +6665,8 @@ ExprResult Sema::MaybeBindToTemporary(Expr *E) {
 
   // That should be enough to guarantee that this type is complete, if we're
   // not processing a decltype expression.
-  CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+  CXXRecordDecl *RD =
+      cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
   if (RD->isInvalidDecl() || RD->isDependentContext())
     return E;
 
@@ -7243,16 +7273,13 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
     TemplateIdAnnotation *TemplateId = SecondTypeName.TemplateId;
     ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                        TemplateId->NumArgs);
-    TypeResult T = ActOnTemplateIdType(S,
-                                       SS,
-                                       TemplateId->TemplateKWLoc,
-                                       TemplateId->Template,
-                                       TemplateId->Name,
-                                       TemplateId->TemplateNameLoc,
-                                       TemplateId->LAngleLoc,
-                                       TemplateArgsPtr,
-                                       TemplateId->RAngleLoc,
-                                       /*IsCtorOrDtorName*/true);
+    TypeResult T = ActOnTemplateIdType(
+        S, ElaboratedTypeKeyword::None,
+        /*ElaboratedKeywordLoc=*/SourceLocation(), SS,
+        TemplateId->TemplateKWLoc, TemplateId->Template, TemplateId->Name,
+        TemplateId->TemplateNameLoc, TemplateId->LAngleLoc, TemplateArgsPtr,
+        TemplateId->RAngleLoc,
+        /*IsCtorOrDtorName*/ true);
     if (T.isInvalid() || !T.get()) {
       // Recover by assuming we had the right type all along.
       DestructedType = ObjectType;
@@ -7296,16 +7323,13 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
       TemplateIdAnnotation *TemplateId = FirstTypeName.TemplateId;
       ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                          TemplateId->NumArgs);
-      TypeResult T = ActOnTemplateIdType(S,
-                                         SS,
-                                         TemplateId->TemplateKWLoc,
-                                         TemplateId->Template,
-                                         TemplateId->Name,
-                                         TemplateId->TemplateNameLoc,
-                                         TemplateId->LAngleLoc,
-                                         TemplateArgsPtr,
-                                         TemplateId->RAngleLoc,
-                                         /*IsCtorOrDtorName*/true);
+      TypeResult T = ActOnTemplateIdType(
+          S, ElaboratedTypeKeyword::None,
+          /*ElaboratedKeywordLoc=*/SourceLocation(), SS,
+          TemplateId->TemplateKWLoc, TemplateId->Template, TemplateId->Name,
+          TemplateId->TemplateNameLoc, TemplateId->LAngleLoc, TemplateArgsPtr,
+          TemplateId->RAngleLoc,
+          /*IsCtorOrDtorName*/ true);
       if (T.isInvalid() || !T.get()) {
         // Recover by dropping this type.
         ScopeType = QualType();
@@ -7505,7 +7529,7 @@ ExprResult Sema::IgnoredValueConversions(Expr *E) {
 
   // GCC seems to also exclude expressions of incomplete enum type.
   if (const EnumType *T = E->getType()->getAs<EnumType>()) {
-    if (!T->getDecl()->isComplete()) {
+    if (!T->getOriginalDecl()->getDefinitionOrSelf()->isComplete()) {
       // FIXME: stupid workaround for a codegen bug!
       E = ImpCastExprToType(E, Context.VoidTy, CK_ToVoid).get();
       return E;
diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index 5dca509d46fdb..4a31a139eaf4f 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -492,13 +492,14 @@ CheckExtVectorComponent(Sema &S, QualType baseType, ExprValueKind &VK,
 
   QualType VT = S.Context.getExtVectorType(vecType->getElementType(), CompSize);
   // Now look up the TypeDefDecl from the vector type. Without this,
-  // diagostics look bad. We want extended vector types to appear built-in.
+  // diagnostics look bad. We want extended vector types to appear built-in.
   for (Sema::ExtVectorDeclsType::iterator
          I = S.ExtVectorDecls.begin(S.getExternalSource()),
          E = S.ExtVectorDecls.end();
        I != E; ++I) {
     if ((*I)->getUnderlyingType() == VT)
-      return S.Context.getTypedefType(*I);
+      return S.Context.getTypedefType(ElaboratedTypeKeyword::None,
+                                      /*Qualifier=*/std::nullopt, *I);
   }
 
   return VT; // should never get here (a typedef type should always be found).
@@ -881,7 +882,7 @@ Sema::BuildMemberReferenceExpr(Expr *BaseExpr, QualType BaseExprType,
   // build a CXXDependentScopeMemberExpr.
   if (R.wasNotFoundInCurrentInstantiation() ||
       (R.getLookupName().getCXXOverloadedOperator() == OO_Equal &&
-       (SS.isSet() ? SS.getScopeRep()->isDependent()
+       (SS.isSet() ? SS.getScopeRep().isDependent()
                    : BaseExprType->isDependentType())))
     return ActOnDependentMemberExpr(BaseExpr, BaseExprType, IsArrow, OpLoc, SS,
                                     TemplateKWLoc, FirstQualifierInScope,
diff --git a/clang/lib/Sema/SemaExprObjC.cpp b/clang/lib/Sema/SemaExprObjC.cpp
index e0662d82914f4..03b5c79cf70e3 100644
--- a/clang/lib/Sema/SemaExprObjC.cpp
+++ b/clang/lib/Sema/SemaExprObjC.cpp
@@ -639,14 +639,14 @@ ExprResult SemaObjC::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
     BoxingMethod = getNSNumberFactoryMethod(*this, Loc, ValueType);
     BoxedType = NSNumberPointer;
   } else if (const EnumType *ET = ValueType->getAs<EnumType>()) {
-    if (!ET->getDecl()->isComplete()) {
+    const EnumDecl *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+    if (!ED->isComplete()) {
       Diag(Loc, diag::err_objc_incomplete_boxed_expression_type)
         << ValueType << ValueExpr->getSourceRange();
       return ExprError();
     }
 
-    BoxingMethod = getNSNumberFactoryMethod(*this, Loc,
-                                            ET->getDecl()->getIntegerType());
+    BoxingMethod = getNSNumberFactoryMethod(*this, Loc, ED->getIntegerType());
     BoxedType = NSNumberPointer;
   } else if (ValueType->isObjCBoxableRecordType()) {
     // Support for structure types, that marked as objc_boxable
@@ -2337,10 +2337,10 @@ SemaObjC::getObjCMessageKind(Scope *S, IdentifierInfo *Name,
     if (ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(ND))
       T = Context.getObjCInterfaceType(Class);
     else if (TypeDecl *Type = dyn_cast<TypeDecl>(ND)) {
-      T = Context.getTypeDeclType(Type);
       SemaRef.DiagnoseUseOfDecl(Type, NameLoc);
-    }
-    else
+      T = Context.getTypeDeclType(ElaboratedTypeKeyword::None,
+                                  /*Qualifier=*/std::nullopt, Type);
+    } else
       return ObjCInstanceMessage;
 
     //  We have a class message, and T is the type we're
@@ -3847,7 +3847,8 @@ static inline T *getObjCBridgeAttr(const TypedefType *TD) {
   if (QT->isPointerType()) {
     QT = QT->getPointeeType();
     if (const RecordType *RT = QT->getAs<RecordType>()) {
-      for (auto *Redecl : RT->getDecl()->getMostRecentDecl()->redecls()) {
+      for (auto *Redecl :
+           RT->getOriginalDecl()->getMostRecentDecl()->redecls()) {
         if (auto *attr = Redecl->getAttr<T>())
           return attr;
       }
diff --git a/clang/lib/Sema/SemaFunctionEffects.cpp b/clang/lib/Sema/SemaFunctionEffects.cpp
index 1592862416bf9..8590ee831084f 100644
--- a/clang/lib/Sema/SemaFunctionEffects.cpp
+++ b/clang/lib/Sema/SemaFunctionEffects.cpp
@@ -1352,11 +1352,15 @@ class Analyzer {
       return true;
     }
 
-    bool TraverseTypeOfExprTypeLoc(TypeOfExprTypeLoc Node) override {
+    bool TraverseTypeOfExprTypeLoc(TypeOfExprTypeLoc Node,
+                                   bool TraverseQualifier) override {
       return true;
     }
 
-    bool TraverseDecltypeTypeLoc(DecltypeTypeLoc Node) override { return true; }
+    bool TraverseDecltypeTypeLoc(DecltypeTypeLoc Node,
+                                 bool TraverseQualifier) override {
+      return true;
+    }
 
     bool TraverseCXXNoexceptExpr(CXXNoexceptExpr *Node) override {
       return true;
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 8536e04a129ee..f87715950c74c 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -71,6 +71,10 @@ static RegisterType getRegisterType(ResourceClass RC) {
   llvm_unreachable("unexpected ResourceClass value");
 }
 
+static RegisterType getRegisterType(const HLSLAttributedResourceType *ResTy) {
+  return getRegisterType(ResTy->getAttrs().ResourceClass);
+}
+
 // Converts the first letter of string Slot to RegisterType.
 // Returns false if the letter does not correspond to a valid register type.
 static bool convertToRegisterType(StringRef Slot, RegisterType *RT) {
@@ -234,7 +238,7 @@ static unsigned calculateLegacyCbufferSize(const ASTContext &Context,
   constexpr unsigned CBufferAlign = 16;
   if (const RecordType *RT = T->getAs<RecordType>()) {
     unsigned Size = 0;
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     for (const FieldDecl *Field : RD->fields()) {
       QualType Ty = Field->getType();
       unsigned FieldSize = calculateLegacyCbufferSize(Context, Ty);
@@ -337,16 +341,19 @@ static bool isZeroSizedArray(const ConstantArrayType *CAT) {
   return CAT != nullptr;
 }
 
-// Returns true if the record type is an HLSL resource class or an array of
-// resource classes
-static bool isResourceRecordTypeOrArrayOf(const Type *Ty) {
-  while (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(Ty))
-    Ty = CAT->getArrayElementTypeNoTypeQual();
-  return HLSLAttributedResourceType::findHandleTypeOnResource(Ty) != nullptr;
+static bool isResourceRecordTypeOrArrayOf(VarDecl *VD) {
+  const Type *Ty = VD->getType().getTypePtr();
+  return Ty->isHLSLResourceRecord() || Ty->isHLSLResourceRecordArray();
 }
 
-static bool isResourceRecordTypeOrArrayOf(VarDecl *VD) {
-  return isResourceRecordTypeOrArrayOf(VD->getType().getTypePtr());
+static const HLSLAttributedResourceType *
+getResourceArrayHandleType(VarDecl *VD) {
+  assert(VD->getType()->isHLSLResourceRecordArray() &&
+         "expected array of resource records");
+  const Type *Ty = VD->getType()->getUnqualifiedDesugaredType();
+  while (const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(Ty))
+    Ty = CAT->getArrayElementTypeNoTypeQual()->getUnqualifiedDesugaredType();
+  return HLSLAttributedResourceType::findHandleTypeOnResource(Ty);
 }
 
 // Returns true if the type is a leaf element type that is not valid to be
@@ -355,7 +362,7 @@ static bool isResourceRecordTypeOrArrayOf(VarDecl *VD) {
 // type or if it is a record type that needs to be inspected further.
 static bool isInvalidConstantBufferLeafElementType(const Type *Ty) {
   Ty = Ty->getUnqualifiedDesugaredType();
-  if (isResourceRecordTypeOrArrayOf(Ty))
+  if (Ty->isHLSLResourceRecord() || Ty->isHLSLResourceRecordArray())
     return true;
   if (Ty->isRecordType())
     return Ty->getAsCXXRecordDecl()->isEmpty();
@@ -373,7 +380,7 @@ static bool isInvalidConstantBufferLeafElementType(const Type *Ty) {
 // needs to be created for HLSL Buffer use that will exclude these unwanted
 // declarations (see createHostLayoutStruct function).
 static bool requiresImplicitBufferLayoutStructure(const CXXRecordDecl *RD) {
-  if (RD->getTypeForDecl()->isHLSLIntangibleType() || RD->isEmpty())
+  if (RD->isHLSLIntangible() || RD->isEmpty())
     return true;
   // check fields
   for (const FieldDecl *Field : RD->fields()) {
@@ -458,7 +465,7 @@ static FieldDecl *createFieldForHostLayoutStruct(Sema &S, const Type *Ty,
       RD = createHostLayoutStruct(S, RD);
       if (!RD)
         return nullptr;
-      Ty = RD->getTypeForDecl();
+      Ty = S.Context.getCanonicalTagType(RD)->getTypePtr();
     }
   }
 
@@ -508,8 +515,8 @@ static CXXRecordDecl *createHostLayoutStruct(Sema &S,
     if (requiresImplicitBufferLayoutStructure(BaseDecl)) {
       BaseDecl = createHostLayoutStruct(S, BaseDecl);
       if (BaseDecl) {
-        TypeSourceInfo *TSI = AST.getTrivialTypeSourceInfo(
-            QualType(BaseDecl->getTypeForDecl(), 0));
+        TypeSourceInfo *TSI =
+            AST.getTrivialTypeSourceInfo(AST.getCanonicalTagType(BaseDecl));
         Base = CXXBaseSpecifier(SourceRange(), false, StructDecl->isClass(),
                                 AS_none, TSI, SourceLocation());
       }
@@ -575,16 +582,13 @@ void createHostLayoutStructForBuffer(Sema &S, HLSLBufferDecl *BufDecl) {
   BufDecl->addLayoutStruct(LS);
 }
 
-static void addImplicitBindingAttrToBuffer(Sema &S, HLSLBufferDecl *BufDecl,
-                                           uint32_t ImplicitBindingOrderID) {
-  RegisterType RT =
-      BufDecl->isCBuffer() ? RegisterType::CBuffer : RegisterType::SRV;
+static void addImplicitBindingAttrToDecl(Sema &S, Decl *D, RegisterType RT,
+                                         uint32_t ImplicitBindingOrderID) {
   auto *Attr =
       HLSLResourceBindingAttr::CreateImplicit(S.getASTContext(), "", "0", {});
-  std::optional<unsigned> RegSlot;
-  Attr->setBinding(RT, RegSlot, 0);
+  Attr->setBinding(RT, std::nullopt, 0);
   Attr->setImplicitBindingOrderID(ImplicitBindingOrderID);
-  BufDecl->addAttr(Attr);
+  D->addAttr(Attr);
 }
 
 // Handle end of cbuffer/tbuffer declaration
@@ -607,7 +611,10 @@ void SemaHLSL::ActOnFinishBuffer(Decl *Dcl, SourceLocation RBrace) {
     if (RBA)
       RBA->setImplicitBindingOrderID(OrderID);
     else
-      addImplicitBindingAttrToBuffer(SemaRef, BufDecl, OrderID);
+      addImplicitBindingAttrToDecl(SemaRef, BufDecl,
+                                   BufDecl->isCBuffer() ? RegisterType::CBuffer
+                                                        : RegisterType::SRV,
+                                   OrderID);
   }
 
   SemaRef.PopDeclContext();
@@ -1843,7 +1850,7 @@ SemaHLSL::TakeLocForHLSLAttribute(const HLSLAttributedResourceType *RT) {
 // requirements and adds them to Bindings
 void SemaHLSL::collectResourceBindingsOnUserRecordDecl(const VarDecl *VD,
                                                        const RecordType *RT) {
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
   for (FieldDecl *FD : RD->fields()) {
     const Type *Ty = FD->getType()->getUnqualifiedDesugaredType();
 
@@ -1913,7 +1920,7 @@ static bool DiagnoseLocalRegisterBinding(Sema &S, SourceLocation &ArgLoc,
   if (const HLSLAttributedResourceType *AttrResType =
           HLSLAttributedResourceType::findHandleTypeOnResource(
               VD->getType().getTypePtr())) {
-    if (RegType == getRegisterType(AttrResType->getAttrs().ResourceClass))
+    if (RegType == getRegisterType(AttrResType))
       return true;
 
     S.Diag(D->getLocation(), diag::err_hlsl_binding_type_mismatch)
@@ -2446,8 +2453,8 @@ void SemaHLSL::ActOnEndOfTranslationUnit(TranslationUnitDecl *TU) {
     HLSLBufferDecl *DefaultCBuffer = HLSLBufferDecl::CreateDefaultCBuffer(
         SemaRef.getASTContext(), SemaRef.getCurLexicalContext(),
         DefaultCBufferDecls);
-    addImplicitBindingAttrToBuffer(SemaRef, DefaultCBuffer,
-                                   getNextImplicitBindingOrderID());
+    addImplicitBindingAttrToDecl(SemaRef, DefaultCBuffer, RegisterType::CBuffer,
+                                 getNextImplicitBindingOrderID());
     SemaRef.getCurLexicalContext()->addDecl(DefaultCBuffer);
     createHostLayoutStructForBuffer(SemaRef, DefaultCBuffer);
 
@@ -3396,7 +3403,7 @@ bool SemaHLSL::ContainsBitField(QualType BaseTy) {
       continue;
     }
     if (const auto *RT = dyn_cast<RecordType>(T)) {
-      const RecordDecl *RD = RT->getDecl();
+      const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
       if (RD->isUnion())
         continue;
 
@@ -3597,7 +3604,7 @@ void SemaHLSL::deduceAddressSpace(VarDecl *Decl) {
     return;
 
   // Resource handles.
-  if (isResourceRecordTypeOrArrayOf(Type->getUnqualifiedDesugaredType()))
+  if (Type->isHLSLResourceRecord() || Type->isHLSLResourceRecordArray())
     return;
 
   // Only static globals belong to the Private address space.
@@ -3637,10 +3644,7 @@ void SemaHLSL::ActOnVariableDeclarator(VarDecl *VD) {
     if (VD->getType()->isHLSLIntangibleType())
       collectResourceBindingsOnVarDecl(VD);
 
-    const Type *VarType = VD->getType().getTypePtr();
-    while (VarType->isArrayType())
-      VarType = VarType->getArrayElementTypeNoTypeQual();
-    if (VarType->isHLSLResourceRecord() ||
+    if (isResourceRecordTypeOrArrayOf(VD) ||
         VD->hasAttr<HLSLVkConstantIdAttr>()) {
       // Make the variable for resources static. The global externally visible
       // storage is accessed through the handle, which is a member. The variable
@@ -3650,6 +3654,24 @@ void SemaHLSL::ActOnVariableDeclarator(VarDecl *VD) {
 
     // process explicit bindings
     processExplicitBindingsOnDecl(VD);
+
+    if (VD->getType()->isHLSLResourceRecordArray()) {
+      // If the resource array does not have an explicit binding attribute,
+      // create an implicit one. It will be used to transfer implicit binding
+      // order_ID to codegen.
+      if (!VD->hasAttr<HLSLVkBindingAttr>()) {
+        HLSLResourceBindingAttr *RBA = VD->getAttr<HLSLResourceBindingAttr>();
+        if (!RBA || !RBA->hasRegisterSlot()) {
+          uint32_t OrderID = getNextImplicitBindingOrderID();
+          if (RBA)
+            RBA->setImplicitBindingOrderID(OrderID);
+          else
+            addImplicitBindingAttrToDecl(
+                SemaRef, VD, getRegisterType(getResourceArrayHandleType(VD)),
+                OrderID);
+        }
+      }
+    }
   }
 
   deduceAddressSpace(VD);
@@ -3675,11 +3697,12 @@ static bool initVarDeclWithCtor(Sema &S, VarDecl *VD,
   return true;
 }
 
-bool SemaHLSL::initGlobalResourceDecl(VarDecl *VD) {
+void SemaHLSL::createResourceRecordCtorArgs(
+    const Type *ResourceTy, StringRef VarName, HLSLResourceBindingAttr *RBA,
+    HLSLVkBindingAttr *VkBinding, uint32_t ArrayIndex,
+    llvm::SmallVectorImpl<Expr *> &Args) {
   std::optional<uint32_t> RegisterSlot;
   uint32_t SpaceNo = 0;
-  HLSLVkBindingAttr *VkBinding = VD->getAttr<HLSLVkBindingAttr>();
-  HLSLResourceBindingAttr *RBA = VD->getAttr<HLSLResourceBindingAttr>();
   if (VkBinding) {
     RegisterSlot = VkBinding->getBinding();
     SpaceNo = VkBinding->getSet();
@@ -3694,12 +3717,12 @@ bool SemaHLSL::initGlobalResourceDecl(VarDecl *VD) {
   uint64_t IntTySize = AST.getTypeSize(AST.IntTy);
   IntegerLiteral *RangeSize = IntegerLiteral::Create(
       AST, llvm::APInt(IntTySize, 1), AST.IntTy, SourceLocation());
-  IntegerLiteral *Index = IntegerLiteral::Create(
-      AST, llvm::APInt(UIntTySize, 0), AST.UnsignedIntTy, SourceLocation());
+  IntegerLiteral *Index =
+      IntegerLiteral::Create(AST, llvm::APInt(UIntTySize, ArrayIndex),
+                             AST.UnsignedIntTy, SourceLocation());
   IntegerLiteral *Space =
       IntegerLiteral::Create(AST, llvm::APInt(UIntTySize, SpaceNo),
                              AST.UnsignedIntTy, SourceLocation());
-  StringRef VarName = VD->getName();
   StringLiteral *Name = StringLiteral::Create(
       AST, VarName, StringLiteralKind::Ordinary, false,
       AST.getStringLiteralArrayType(AST.CharTy.withConst(), VarName.size()),
@@ -3710,18 +3733,57 @@ bool SemaHLSL::initGlobalResourceDecl(VarDecl *VD) {
     IntegerLiteral *RegSlot = IntegerLiteral::Create(
         AST, llvm::APInt(UIntTySize, RegisterSlot.value()), AST.UnsignedIntTy,
         SourceLocation());
-    Expr *Args[] = {RegSlot, Space, RangeSize, Index, Name};
-    return initVarDeclWithCtor(SemaRef, VD, Args);
+    Args.append({RegSlot, Space, RangeSize, Index, Name});
+  } else {
+    // resource with implicit binding
+    uint32_t OrderID = (RBA && RBA->hasImplicitBindingOrderID())
+                           ? RBA->getImplicitBindingOrderID()
+                           : getNextImplicitBindingOrderID();
+    IntegerLiteral *OrderId =
+        IntegerLiteral::Create(AST, llvm::APInt(UIntTySize, OrderID),
+                               AST.UnsignedIntTy, SourceLocation());
+    Args.append({Space, RangeSize, Index, OrderId, Name});
   }
+}
 
-  // resource with implicit binding
-  IntegerLiteral *OrderId = IntegerLiteral::Create(
-      AST, llvm::APInt(UIntTySize, getNextImplicitBindingOrderID()),
-      AST.UnsignedIntTy, SourceLocation());
-  Expr *Args[] = {Space, RangeSize, Index, OrderId, Name};
+bool SemaHLSL::initGlobalResourceDecl(VarDecl *VD) {
+  SmallVector<Expr *> Args;
+  createResourceRecordCtorArgs(VD->getType().getTypePtr(), VD->getName(),
+                               VD->getAttr<HLSLResourceBindingAttr>(),
+                               VD->getAttr<HLSLVkBindingAttr>(), 0, Args);
   return initVarDeclWithCtor(SemaRef, VD, Args);
 }
 
+bool SemaHLSL::initGlobalResourceArrayDecl(VarDecl *VD) {
+  assert(VD->getType()->isHLSLResourceRecordArray() &&
+         "expected array of resource records");
+
+  // Individual resources in a resource array are not initialized here. They
+  // are initialized later on during codegen when the individual resources are
+  // accessed. Codegen will emit a call to the resource constructor with the
+  // specified array index. We need to make sure though that the constructor
+  // for the specific resource type is instantiated, so codegen can emit a call
+  // to it when the array element is accessed.
+  SmallVector<Expr *> Args;
+  QualType ResElementTy = VD->getASTContext().getBaseElementType(VD->getType());
+  createResourceRecordCtorArgs(ResElementTy.getTypePtr(), VD->getName(),
+                               VD->getAttr<HLSLResourceBindingAttr>(),
+                               VD->getAttr<HLSLVkBindingAttr>(), 0, Args);
+
+  SourceLocation Loc = VD->getLocation();
+  InitializedEntity Entity =
+      InitializedEntity::InitializeTemporary(ResElementTy);
+  InitializationKind Kind = InitializationKind::CreateDirect(Loc, Loc, Loc);
+  InitializationSequence InitSeq(SemaRef, Entity, Kind, Args);
+  if (InitSeq.Failed())
+    return false;
+
+  // This takes care of instantiating and emitting of the constructor that will
+  // be called from codegen when the array is accessed.
+  ExprResult OneResInit = InitSeq.Perform(SemaRef, Entity, Kind, Args);
+  return !OneResInit.isInvalid();
+}
+
 // Returns true if the initialization has been handled.
 // Returns false to use default initialization.
 bool SemaHLSL::ActOnUninitializedVarDecl(VarDecl *VD) {
@@ -3730,17 +3792,14 @@ bool SemaHLSL::ActOnUninitializedVarDecl(VarDecl *VD) {
   if (VD->getType().getAddressSpace() == LangAS::hlsl_constant)
     return true;
 
-  // Initialize resources
-  if (!isResourceRecordTypeOrArrayOf(VD))
-    return false;
-
-  // FIXME: We currectly support only simple resources - no arrays of resources
-  // or resources in user defined structs.
-  // (llvm/llvm-project#133835, llvm/llvm-project#133837)
   // Initialize resources at the global scope
-  if (VD->hasGlobalStorage() && VD->getType()->isHLSLResourceRecord())
-    return initGlobalResourceDecl(VD);
-
+  if (VD->hasGlobalStorage()) {
+    const Type *Ty = VD->getType().getTypePtr();
+    if (Ty->isHLSLResourceRecord())
+      return initGlobalResourceDecl(VD);
+    if (Ty->isHLSLResourceRecordArray())
+      return initGlobalResourceArrayDecl(VD);
+  }
   return false;
 }
 
@@ -3919,7 +3978,8 @@ class InitListTransformer {
       }
       while (!RecordTypes.empty()) {
         const RecordType *RT = RecordTypes.pop_back_val();
-        for (auto *FD : RT->getDecl()->fields()) {
+        for (auto *FD :
+             RT->getOriginalDecl()->getDefinitionOrSelf()->fields()) {
           DeclAccessPair Found = DeclAccessPair::make(FD, FD->getAccess());
           DeclarationNameInfo NameInfo(FD->getDeclName(), E->getBeginLoc());
           ExprResult Res = S.BuildFieldReferenceExpr(
@@ -3967,7 +4027,8 @@ class InitListTransformer {
       }
       while (!RecordTypes.empty()) {
         const RecordType *RT = RecordTypes.pop_back_val();
-        for (auto *FD : RT->getDecl()->fields()) {
+        for (auto *FD :
+             RT->getOriginalDecl()->getDefinitionOrSelf()->fields()) {
           Inits.push_back(generateInitListsImpl(FD->getType()));
         }
       }
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 1dd38c0ef6773..60f9d449fc037 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -446,7 +446,6 @@ class InitListChecker {
                                    unsigned ExpectedNumInits);
   int numArrayElements(QualType DeclType);
   int numStructUnionElements(QualType DeclType);
-  static RecordDecl *getRecordDecl(QualType DeclType);
 
   ExprResult PerformEmptyInit(SourceLocation Loc,
                               const InitializedEntity &Entity);
@@ -776,7 +775,7 @@ void InitListChecker::FillInEmptyInitForField(unsigned Init, FieldDecl *Field,
 
   if (Init >= NumInits || !ILE->getInit(Init)) {
     if (const RecordType *RType = ILE->getType()->getAs<RecordType>())
-      if (!RType->getDecl()->isUnion())
+      if (!RType->getOriginalDecl()->isUnion())
         assert((Init < NumInits || VerifyOnly) &&
                "This ILE should have been expanded");
 
@@ -923,7 +922,7 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity,
     return;
 
   if (const RecordType *RType = ILE->getType()->getAs<RecordType>()) {
-    const RecordDecl *RDecl = RType->getDecl();
+    const RecordDecl *RDecl = RType->getOriginalDecl()->getDefinitionOrSelf();
     if (RDecl->isUnion() && ILE->getInitializedFieldInUnion()) {
       FillInEmptyInitForField(0, ILE->getInitializedFieldInUnion(), Entity, ILE,
                               RequiresSecondPass, FillWithNoInit);
@@ -1127,7 +1126,8 @@ int InitListChecker::numArrayElements(QualType DeclType) {
 }
 
 int InitListChecker::numStructUnionElements(QualType DeclType) {
-  RecordDecl *structDecl = DeclType->castAs<RecordType>()->getDecl();
+  RecordDecl *structDecl =
+      DeclType->castAs<RecordType>()->getOriginalDecl()->getDefinitionOrSelf();
   int InitializableMembers = 0;
   if (auto *CXXRD = dyn_cast<CXXRecordDecl>(structDecl))
     InitializableMembers += CXXRD->getNumBases();
@@ -1140,14 +1140,6 @@ int InitListChecker::numStructUnionElements(QualType DeclType) {
   return InitializableMembers - structDecl->hasFlexibleArrayMember();
 }
 
-RecordDecl *InitListChecker::getRecordDecl(QualType DeclType) {
-  if (const auto *RT = DeclType->getAs<RecordType>())
-    return RT->getDecl();
-  if (const auto *Inject = DeclType->getAs<InjectedClassNameType>())
-    return Inject->getDecl();
-  return nullptr;
-}
-
 /// Determine whether Entity is an entity for which it is idiomatic to elide
 /// the braces in aggregate initialization.
 static bool isIdiomaticBraceElisionEntity(const InitializedEntity &Entity) {
@@ -1164,16 +1156,22 @@ static bool isIdiomaticBraceElisionEntity(const InitializedEntity &Entity) {
 
   // Allows elide brace initialization for aggregates with empty base.
   if (Entity.getKind() == InitializedEntity::EK_Base) {
-    auto *ParentRD =
-        Entity.getParent()->getType()->castAs<RecordType>()->getDecl();
+    auto *ParentRD = Entity.getParent()
+                         ->getType()
+                         ->castAs<RecordType>()
+                         ->getOriginalDecl()
+                         ->getDefinitionOrSelf();
     CXXRecordDecl *CXXRD = cast<CXXRecordDecl>(ParentRD);
     return CXXRD->getNumBases() == 1 && CXXRD->field_empty();
   }
 
   // Allow brace elision if the only subobject is a field.
   if (Entity.getKind() == InitializedEntity::EK_Member) {
-    auto *ParentRD =
-        Entity.getParent()->getType()->castAs<RecordType>()->getDecl();
+    auto *ParentRD = Entity.getParent()
+                         ->getType()
+                         ->castAs<RecordType>()
+                         ->getOriginalDecl()
+                         ->getDefinitionOrSelf();
     if (CXXRecordDecl *CXXRD = dyn_cast<CXXRecordDecl>(ParentRD)) {
       if (CXXRD->getNumBases()) {
         return false;
@@ -1442,7 +1440,7 @@ void InitListChecker::CheckListElementTypes(const InitializedEntity &Entity,
   } else if (DeclType->isVectorType()) {
     CheckVectorType(Entity, IList, DeclType, Index,
                     StructuredList, StructuredIndex);
-  } else if (const RecordDecl *RD = getRecordDecl(DeclType)) {
+  } else if (const RecordDecl *RD = DeclType->getAsRecordDecl()) {
     auto Bases =
         CXXRecordDecl::base_class_const_range(CXXRecordDecl::base_class_const_iterator(),
                                         CXXRecordDecl::base_class_const_iterator());
@@ -2320,7 +2318,7 @@ void InitListChecker::CheckStructUnionTypes(
     bool SubobjectIsDesignatorContext, unsigned &Index,
     InitListExpr *StructuredList, unsigned &StructuredIndex,
     bool TopLevelObject) {
-  const RecordDecl *RD = getRecordDecl(DeclType);
+  const RecordDecl *RD = DeclType->getAsRecordDecl();
 
   // If the record is invalid, some of it's members are invalid. To avoid
   // confusion, we forgo checking the initializer for the entire record.
@@ -2889,7 +2887,7 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
     //   then the current object (defined below) shall have
     //   structure or union type and the identifier shall be the
     //   name of a member of that type.
-    RecordDecl *RD = getRecordDecl(CurrentObjectType);
+    RecordDecl *RD = CurrentObjectType->getAsRecordDecl();
     if (!RD) {
       SourceLocation Loc = D->getDotLoc();
       if (Loc.isInvalid())
@@ -3296,8 +3294,9 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
     if (StringLiteral *SL = dyn_cast<StringLiteral>(SubExpr)) {
       // Get the length of the string.
       uint64_t StrLen = SL->getLength();
-      if (cast<ConstantArrayType>(AT)->getSize().ult(StrLen))
-        StrLen = cast<ConstantArrayType>(AT)->getZExtSize();
+      if (const auto *CAT = dyn_cast<ConstantArrayType>(AT);
+          CAT && CAT->getSize().ult(StrLen))
+        StrLen = CAT->getZExtSize();
       StructuredList->resizeInits(Context, StrLen);
 
       // Build a literal for each character in the string, and put them into
@@ -3319,8 +3318,9 @@ InitListChecker::CheckDesignatedInitializer(const InitializedEntity &Entity,
 
       // Get the length of the string.
       uint64_t StrLen = Str.size();
-      if (cast<ConstantArrayType>(AT)->getSize().ult(StrLen))
-        StrLen = cast<ConstantArrayType>(AT)->getZExtSize();
+      if (const auto *CAT = dyn_cast<ConstantArrayType>(AT);
+          CAT && CAT->getSize().ult(StrLen))
+        StrLen = CAT->getZExtSize();
       StructuredList->resizeInits(Context, StrLen);
 
       // Build a literal for each character in the string, and put them into
@@ -4337,8 +4337,8 @@ static bool hasCopyOrMoveCtorParam(ASTContext &Ctx,
 
   QualType ParmT =
       Info.Constructor->getParamDecl(0)->getType().getNonReferenceType();
-  QualType ClassT =
-      Ctx.getRecordType(cast<CXXRecordDecl>(Info.FoundDecl->getDeclContext()));
+  CanQualType ClassT = Ctx.getCanonicalTagType(
+      cast<CXXRecordDecl>(Info.FoundDecl->getDeclContext()));
 
   return Ctx.hasSameUnqualifiedType(ParmT, ClassT);
 }
@@ -4537,8 +4537,8 @@ static void TryConstructorInitialization(Sema &S,
 
   const RecordType *DestRecordType = DestType->getAs<RecordType>();
   assert(DestRecordType && "Constructor initialization requires record type");
-  CXXRecordDecl *DestRecordDecl
-    = cast<CXXRecordDecl>(DestRecordType->getDecl());
+  auto *DestRecordDecl = cast<CXXRecordDecl>(DestRecordType->getOriginalDecl())
+                             ->getDefinitionOrSelf();
 
   // Build the candidate set directly in the initialization sequence
   // structure, so that it will persist if we fail.
@@ -5059,8 +5059,8 @@ static void TryListInitialization(Sema &S,
     //     the underlying type of T, the program is ill-formed.
     auto *ET = DestType->getAs<EnumType>();
     if (S.getLangOpts().CPlusPlus17 &&
-        Kind.getKind() == InitializationKind::IK_DirectList &&
-        ET && ET->getDecl()->isFixed() &&
+        Kind.getKind() == InitializationKind::IK_DirectList && ET &&
+        ET->getOriginalDecl()->getDefinitionOrSelf()->isFixed() &&
         !S.Context.hasSameUnqualifiedType(E->getType(), DestType) &&
         (E->getType()->isIntegralOrUnscopedEnumerationType() ||
          E->getType()->isFloatingType())) {
@@ -5170,7 +5170,8 @@ static OverloadingResult TryRefInitWithConversionFunction(
       S.isCompleteType(Kind.getLocation(), T1)) {
     // The type we're converting to is a class type. Enumerate its constructors
     // to see if there is a suitable conversion.
-    CXXRecordDecl *T1RecordDecl = cast<CXXRecordDecl>(T1RecordType->getDecl());
+    auto *T1RecordDecl = cast<CXXRecordDecl>(T1RecordType->getOriginalDecl())
+                             ->getDefinitionOrSelf();
 
     for (NamedDecl *D : S.LookupConstructors(T1RecordDecl)) {
       auto Info = getConstructorInfo(D);
@@ -5193,7 +5194,8 @@ static OverloadingResult TryRefInitWithConversionFunction(
       }
     }
   }
-  if (T1RecordType && T1RecordType->getDecl()->isInvalidDecl())
+  if (T1RecordType &&
+      T1RecordType->getOriginalDecl()->getDefinitionOrSelf()->isInvalidDecl())
     return OR_No_Viable_Function;
 
   const RecordType *T2RecordType = nullptr;
@@ -5201,7 +5203,8 @@ static OverloadingResult TryRefInitWithConversionFunction(
       S.isCompleteType(Kind.getLocation(), T2)) {
     // The type we're converting from is a class type, enumerate its conversion
     // functions.
-    CXXRecordDecl *T2RecordDecl = cast<CXXRecordDecl>(T2RecordType->getDecl());
+    auto *T2RecordDecl = cast<CXXRecordDecl>(T2RecordType->getOriginalDecl())
+                             ->getDefinitionOrSelf();
 
     const auto &Conversions = T2RecordDecl->getVisibleConversionFunctions();
     for (auto I = Conversions.begin(), E = Conversions.end(); I != E; ++I) {
@@ -5237,7 +5240,8 @@ static OverloadingResult TryRefInitWithConversionFunction(
       }
     }
   }
-  if (T2RecordType && T2RecordType->getDecl()->isInvalidDecl())
+  if (T2RecordType &&
+      T2RecordType->getOriginalDecl()->getDefinitionOrSelf()->isInvalidDecl())
     return OR_No_Viable_Function;
 
   SourceLocation DeclLoc = Initializer->getBeginLoc();
@@ -5715,7 +5719,9 @@ static void TryValueInitialization(Sema &S,
   T = S.Context.getBaseElementType(T);
 
   if (const RecordType *RT = T->getAs<RecordType>()) {
-    if (CXXRecordDecl *ClassDecl = dyn_cast<CXXRecordDecl>(RT->getDecl())) {
+    if (CXXRecordDecl *ClassDecl =
+            dyn_cast<CXXRecordDecl>(RT->getOriginalDecl())) {
+      ClassDecl = ClassDecl->getDefinitionOrSelf();
       bool NeedZeroInitialization = true;
       // C++98:
       // -- if T is a class type (clause 9) with a user-declared constructor
@@ -5913,7 +5919,8 @@ static void TryOrBuildParenListInitialization(
     }
   } else if (auto *RT = Entity.getType()->getAs<RecordType>()) {
     bool IsUnion = RT->isUnionType();
-    const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+    const auto *RD =
+        cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
     if (RD->isInvalidDecl()) {
       // Exit early to avoid confusion when processing members.
       // We do the same for braced list initialization in
@@ -6102,8 +6109,9 @@ static void TryUserDefinedConversion(Sema &S,
   if (const RecordType *DestRecordType = DestType->getAs<RecordType>()) {
     // The type we're converting to is a class type. Enumerate its constructors
     // to see if there is a suitable conversion.
-    CXXRecordDecl *DestRecordDecl
-      = cast<CXXRecordDecl>(DestRecordType->getDecl());
+    auto *DestRecordDecl =
+        cast<CXXRecordDecl>(DestRecordType->getOriginalDecl())
+            ->getDefinitionOrSelf();
 
     // Try to complete the type we're converting to.
     if (S.isCompleteType(Kind.getLocation(), DestType)) {
@@ -6139,8 +6147,9 @@ static void TryUserDefinedConversion(Sema &S,
     // We can only enumerate the conversion functions for a complete type; if
     // the type isn't complete, simply skip this step.
     if (S.isCompleteType(DeclLoc, SourceType)) {
-      CXXRecordDecl *SourceRecordDecl
-        = cast<CXXRecordDecl>(SourceRecordType->getDecl());
+      auto *SourceRecordDecl =
+          cast<CXXRecordDecl>(SourceRecordType->getOriginalDecl())
+              ->getDefinitionOrSelf();
 
       const auto &Conversions =
           SourceRecordDecl->getVisibleConversionFunctions();
@@ -7173,7 +7182,8 @@ static ExprResult CopyObject(Sema &S,
   Expr *CurInitExpr = (Expr *)CurInit.get();
   CXXRecordDecl *Class = nullptr;
   if (const RecordType *Record = T->getAs<RecordType>())
-    Class = cast<CXXRecordDecl>(Record->getDecl());
+    Class =
+        cast<CXXRecordDecl>(Record->getOriginalDecl())->getDefinitionOrSelf();
   if (!Class)
     return CurInit;
 
@@ -7328,8 +7338,8 @@ static void CheckCXX98CompatAccessibleCopy(Sema &S,
 
   // Find constructors which would have been considered.
   OverloadCandidateSet CandidateSet(Loc, OverloadCandidateSet::CSK_Normal);
-  DeclContext::lookup_result Ctors =
-      S.LookupConstructors(cast<CXXRecordDecl>(Record->getDecl()));
+  DeclContext::lookup_result Ctors = S.LookupConstructors(
+      cast<CXXRecordDecl>(Record->getOriginalDecl())->getDefinitionOrSelf());
 
   // Perform overload resolution.
   OverloadCandidateSet::iterator Best;
@@ -8160,8 +8170,9 @@ ExprResult InitializationSequence::Perform(Sema &S,
         // regardless of how we initialized the entity.
         QualType T = CurInit.get()->getType();
         if (const RecordType *Record = T->getAs<RecordType>()) {
-          CXXDestructorDecl *Destructor
-            = S.LookupDestructor(cast<CXXRecordDecl>(Record->getDecl()));
+          CXXDestructorDecl *Destructor =
+              S.LookupDestructor(cast<CXXRecordDecl>(Record->getOriginalDecl())
+                                     ->getDefinitionOrSelf());
           S.CheckDestructorAccess(CurInit.get()->getBeginLoc(), Destructor,
                                   S.PDiag(diag::err_access_dtor_temp) << T);
           S.MarkFunctionReferenced(CurInit.get()->getBeginLoc(), Destructor);
@@ -8808,8 +8819,8 @@ static void emitBadConversionNotes(Sema &S, const InitializedEntity &entity,
       destPointeeType.getQualifiers().compatiblyIncludes(
           fromPointeeType.getQualifiers(), S.getASTContext()))
     S.Diag(fromDecl->getLocation(), diag::note_forward_class_conversion)
-        << S.getASTContext().getTagDeclType(fromDecl)
-        << S.getASTContext().getTagDeclType(destDecl);
+        << S.getASTContext().getCanonicalTagType(fromDecl)
+        << S.getASTContext().getCanonicalTagType(destDecl);
 }
 
 static void diagnoseListInit(Sema &S, const InitializedEntity &Entity,
@@ -9208,32 +9219,34 @@ bool InitializationSequence::Diagnose(Sema &S,
             InheritedFrom = Inherited.getShadowDecl()->getNominatedBaseClass();
           if (Entity.getKind() == InitializedEntity::EK_Base) {
             S.Diag(Kind.getLocation(), diag::err_missing_default_ctor)
-              << (InheritedFrom ? 2 : Constructor->isImplicit() ? 1 : 0)
-              << S.Context.getTypeDeclType(Constructor->getParent())
-              << /*base=*/0
-              << Entity.getType()
-              << InheritedFrom;
-
-            RecordDecl *BaseDecl
-              = Entity.getBaseSpecifier()->getType()->castAs<RecordType>()
-                                                                  ->getDecl();
+                << (InheritedFrom               ? 2
+                    : Constructor->isImplicit() ? 1
+                                                : 0)
+                << S.Context.getCanonicalTagType(Constructor->getParent())
+                << /*base=*/0 << Entity.getType() << InheritedFrom;
+
+            RecordDecl *BaseDecl = Entity.getBaseSpecifier()
+                                       ->getType()
+                                       ->castAs<RecordType>()
+                                       ->getOriginalDecl()
+                                       ->getDefinitionOrSelf();
             S.Diag(BaseDecl->getLocation(), diag::note_previous_decl)
-              << S.Context.getTagDeclType(BaseDecl);
+                << S.Context.getCanonicalTagType(BaseDecl);
           } else {
             S.Diag(Kind.getLocation(), diag::err_missing_default_ctor)
-              << (InheritedFrom ? 2 : Constructor->isImplicit() ? 1 : 0)
-              << S.Context.getTypeDeclType(Constructor->getParent())
-              << /*member=*/1
-              << Entity.getName()
-              << InheritedFrom;
+                << (InheritedFrom               ? 2
+                    : Constructor->isImplicit() ? 1
+                                                : 0)
+                << S.Context.getCanonicalTagType(Constructor->getParent())
+                << /*member=*/1 << Entity.getName() << InheritedFrom;
             S.Diag(Entity.getDecl()->getLocation(),
                    diag::note_member_declared_at);
 
             if (const RecordType *Record
                                  = Entity.getType()->getAs<RecordType>())
-              S.Diag(Record->getDecl()->getLocation(),
+              S.Diag(Record->getOriginalDecl()->getLocation(),
                      diag::note_previous_decl)
-                << S.Context.getTagDeclType(Record->getDecl());
+                  << S.Context.getCanonicalTagType(Record->getOriginalDecl());
           }
           break;
         }
@@ -9300,11 +9313,11 @@ bool InitializationSequence::Diagnose(Sema &S,
       // initialized.
       CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(S.CurContext);
       S.Diag(Kind.getLocation(), diag::err_uninitialized_member_in_ctor)
-        << (Constructor->getInheritedConstructor() ? 2 :
-            Constructor->isImplicit() ? 1 : 0)
-        << S.Context.getTypeDeclType(Constructor->getParent())
-        << /*const=*/1
-        << Entity.getName();
+          << (Constructor->getInheritedConstructor() ? 2
+              : Constructor->isImplicit()            ? 1
+                                                     : 0)
+          << S.Context.getCanonicalTagType(Constructor->getParent())
+          << /*const=*/1 << Entity.getName();
       S.Diag(Entity.getDecl()->getLocation(), diag::note_previous_decl)
         << Entity.getName();
     } else if (const auto *VD = dyn_cast_if_present<VarDecl>(Entity.getDecl());
@@ -10157,7 +10170,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
       auto *RD = cast<CXXRecordDecl>(Pattern->getTemplatedDecl());
       if (!(RD->getDefinition() && RD->isAggregate()))
         return;
-      QualType Ty = Context.getRecordType(RD);
+      QualType Ty = Context.getCanonicalTagType(RD);
       SmallVector<QualType, 8> ElementTypes;
 
       InitListChecker CheckInitList(*this, Entity, ListInit, Ty, ElementTypes);
@@ -10297,8 +10310,8 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
   case OR_No_Viable_Function: {
     CXXRecordDecl *Primary =
         cast<ClassTemplateDecl>(Template)->getTemplatedDecl();
-    bool Complete =
-        isCompleteType(Kind.getLocation(), Context.getTypeDeclType(Primary));
+    bool Complete = isCompleteType(Kind.getLocation(),
+                                   Context.getCanonicalTagType(Primary));
     Candidates.NoteCandidates(
         PartialDiagnosticAt(
             Kind.getLocation(),
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index bc3c4b0addeba..8a81cbe2623d8 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -425,7 +425,7 @@ bool Sema::DiagnoseInvalidExplicitObjectParameterInLambda(
                                              .getNonReferenceType()
                                              .getUnqualifiedType()
                                              .getDesugaredType(getASTContext());
-  QualType LambdaType = getASTContext().getRecordType(RD);
+  CanQualType LambdaType = getASTContext().getCanonicalTagType(RD);
   if (LambdaType == ExplicitObjectParameterType)
     return false;
 
@@ -457,7 +457,7 @@ bool Sema::DiagnoseInvalidExplicitObjectParameterInLambda(
     return true;
   }
 
-  if (Paths.isAmbiguous(LambdaType->getCanonicalTypeUnqualified())) {
+  if (Paths.isAmbiguous(LambdaType)) {
     std::string PathsDisplay = getAmbiguousPathsDisplayString(Paths);
     Diag(CallLoc, diag::err_explicit_object_lambda_ambiguous_base)
         << LambdaType << PathsDisplay;
@@ -642,7 +642,7 @@ static EnumDecl *findEnumForBlockReturn(Expr *E) {
 
   //   - it is an expression of that formal enum type.
   if (const EnumType *ET = E->getType()->getAs<EnumType>()) {
-    return ET->getDecl();
+    return ET->getOriginalDecl()->getDefinitionOrSelf();
   }
 
   // Otherwise, nope.
@@ -759,7 +759,7 @@ void Sema::deduceClosureReturnType(CapturingScopeInfo &CSI) {
     assert(isa<BlockScopeInfo>(CSI));
     const EnumDecl *ED = findCommonEnumForBlockReturns(CSI.Returns);
     if (ED) {
-      CSI.ReturnType = Context.getTypeDeclType(ED);
+      CSI.ReturnType = Context.getCanonicalTagType(ED);
       adjustBlockReturnsToEnum(*this, CSI.Returns, CSI.ReturnType);
       return;
     }
@@ -1968,14 +1968,15 @@ ExprResult Sema::BuildCaptureInit(const Capture &Cap,
 }
 
 ExprResult Sema::ActOnLambdaExpr(SourceLocation StartLoc, Stmt *Body) {
-  LambdaScopeInfo LSI = *cast<LambdaScopeInfo>(FunctionScopes.back());
+  LambdaScopeInfo &LSI = *cast<LambdaScopeInfo>(FunctionScopes.back());
 
   if (LSI.CallOperator->hasAttr<SYCLKernelEntryPointAttr>())
     SYCL().CheckSYCLEntryPointFunctionDecl(LSI.CallOperator);
 
-  ActOnFinishFunctionBody(LSI.CallOperator, Body);
+  ActOnFinishFunctionBody(LSI.CallOperator, Body, /*IsInstantiation=*/false,
+                          /*RetainFunctionScopeInfo=*/true);
 
-  return BuildLambdaExpr(StartLoc, Body->getEndLoc(), &LSI);
+  return BuildLambdaExpr(StartLoc, Body->getEndLoc());
 }
 
 static LambdaCaptureDefault
@@ -2132,156 +2133,149 @@ ConstructFixItRangeForUnusedCapture(Sema &S, SourceRange CaptureRange,
   return SourceRange(FixItStart, FixItEnd);
 }
 
-ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
-                                 LambdaScopeInfo *LSI) {
+ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc,
+                                 SourceLocation EndLoc) {
+  LambdaScopeInfo *LSI = cast<LambdaScopeInfo>(FunctionScopes.back());
   // Collect information from the lambda scope.
   SmallVector<LambdaCapture, 4> Captures;
   SmallVector<Expr *, 4> CaptureInits;
   SourceLocation CaptureDefaultLoc = LSI->CaptureDefaultLoc;
   LambdaCaptureDefault CaptureDefault =
       mapImplicitCaptureStyle(LSI->ImpCaptureStyle);
-  CXXRecordDecl *Class;
-  CXXMethodDecl *CallOperator;
-  SourceRange IntroducerRange;
-  bool ExplicitParams;
-  bool ExplicitResultType;
-  CleanupInfo LambdaCleanup;
-  bool ContainsUnexpandedParameterPack;
-  bool IsGenericLambda;
-  {
-    CallOperator = LSI->CallOperator;
-    Class = LSI->Lambda;
-    IntroducerRange = LSI->IntroducerRange;
-    ExplicitParams = LSI->ExplicitParams;
-    ExplicitResultType = !LSI->HasImplicitReturnType;
-    LambdaCleanup = LSI->Cleanup;
-    ContainsUnexpandedParameterPack = LSI->ContainsUnexpandedParameterPack;
-    IsGenericLambda = Class->isGenericLambda();
-
-    CallOperator->setLexicalDeclContext(Class);
-    Decl *TemplateOrNonTemplateCallOperatorDecl =
-        CallOperator->getDescribedFunctionTemplate()
-        ? CallOperator->getDescribedFunctionTemplate()
-        : cast<Decl>(CallOperator);
-
-    // FIXME: Is this really the best choice? Keeping the lexical decl context
-    // set as CurContext seems more faithful to the source.
-    TemplateOrNonTemplateCallOperatorDecl->setLexicalDeclContext(Class);
-
-    PopExpressionEvaluationContext();
-
-    // True if the current capture has a used capture or default before it.
-    bool CurHasPreviousCapture = CaptureDefault != LCD_None;
-    SourceLocation PrevCaptureLoc = CurHasPreviousCapture ?
-        CaptureDefaultLoc : IntroducerRange.getBegin();
-
-    for (unsigned I = 0, N = LSI->Captures.size(); I != N; ++I) {
-      const Capture &From = LSI->Captures[I];
-
-      if (From.isInvalid())
-        return ExprError();
-
-      assert(!From.isBlockCapture() && "Cannot capture __block variables");
-      bool IsImplicit = I >= LSI->NumExplicitCaptures;
-      SourceLocation ImplicitCaptureLoc =
-          IsImplicit ? CaptureDefaultLoc : SourceLocation();
-
-      // Use source ranges of explicit captures for fixits where available.
-      SourceRange CaptureRange = LSI->ExplicitCaptureRanges[I];
-
-      // Warn about unused explicit captures.
-      bool IsCaptureUsed = true;
-      if (!CurContext->isDependentContext() && !IsImplicit &&
-          !From.isODRUsed()) {
-        // Initialized captures that are non-ODR used may not be eliminated.
-        // FIXME: Where did the IsGenericLambda here come from?
-        bool NonODRUsedInitCapture =
-            IsGenericLambda && From.isNonODRUsed() && From.isInitCapture();
-        if (!NonODRUsedInitCapture) {
-          bool IsLast = (I + 1) == LSI->NumExplicitCaptures;
-          SourceRange FixItRange = ConstructFixItRangeForUnusedCapture(
-              *this, CaptureRange, PrevCaptureLoc, CurHasPreviousCapture,
-              IsLast);
-          IsCaptureUsed =
-              !DiagnoseUnusedLambdaCapture(CaptureRange, FixItRange, From);
-        }
-      }
+  CXXRecordDecl *Class = LSI->Lambda;
+  CXXMethodDecl *CallOperator = LSI->CallOperator;
+  SourceRange IntroducerRange = LSI->IntroducerRange;
+  bool ExplicitParams = LSI->ExplicitParams;
+  bool ExplicitResultType = !LSI->HasImplicitReturnType;
+  CleanupInfo LambdaCleanup = LSI->Cleanup;
+  bool ContainsUnexpandedParameterPack = LSI->ContainsUnexpandedParameterPack;
+  bool IsGenericLambda = Class->isGenericLambda();
+
+  CallOperator->setLexicalDeclContext(Class);
+  Decl *TemplateOrNonTemplateCallOperatorDecl =
+      CallOperator->getDescribedFunctionTemplate()
+          ? CallOperator->getDescribedFunctionTemplate()
+          : cast<Decl>(CallOperator);
+
+  // FIXME: Is this really the best choice? Keeping the lexical decl context
+  // set as CurContext seems more faithful to the source.
+  TemplateOrNonTemplateCallOperatorDecl->setLexicalDeclContext(Class);
 
-      if (CaptureRange.isValid()) {
-        CurHasPreviousCapture |= IsCaptureUsed;
-        PrevCaptureLoc = CaptureRange.getEnd();
+  PopExpressionEvaluationContext();
+
+  sema::AnalysisBasedWarnings::Policy WP =
+      AnalysisWarnings.getPolicyInEffectAt(EndLoc);
+  // We cannot release LSI until we finish computing captures, which
+  // requires the scope to be popped.
+  Sema::PoppedFunctionScopePtr _ = PopFunctionScopeInfo(&WP, LSI->CallOperator);
+
+  // True if the current capture has a used capture or default before it.
+  bool CurHasPreviousCapture = CaptureDefault != LCD_None;
+  SourceLocation PrevCaptureLoc =
+      CurHasPreviousCapture ? CaptureDefaultLoc : IntroducerRange.getBegin();
+
+  for (unsigned I = 0, N = LSI->Captures.size(); I != N; ++I) {
+    const Capture &From = LSI->Captures[I];
+
+    if (From.isInvalid())
+      return ExprError();
+
+    assert(!From.isBlockCapture() && "Cannot capture __block variables");
+    bool IsImplicit = I >= LSI->NumExplicitCaptures;
+    SourceLocation ImplicitCaptureLoc =
+        IsImplicit ? CaptureDefaultLoc : SourceLocation();
+
+    // Use source ranges of explicit captures for fixits where available.
+    SourceRange CaptureRange = LSI->ExplicitCaptureRanges[I];
+
+    // Warn about unused explicit captures.
+    bool IsCaptureUsed = true;
+    if (!CurContext->isDependentContext() && !IsImplicit && !From.isODRUsed()) {
+      // Initialized captures that are non-ODR used may not be eliminated.
+      // FIXME: Where did the IsGenericLambda here come from?
+      bool NonODRUsedInitCapture =
+          IsGenericLambda && From.isNonODRUsed() && From.isInitCapture();
+      if (!NonODRUsedInitCapture) {
+        bool IsLast = (I + 1) == LSI->NumExplicitCaptures;
+        SourceRange FixItRange = ConstructFixItRangeForUnusedCapture(
+            *this, CaptureRange, PrevCaptureLoc, CurHasPreviousCapture, IsLast);
+        IsCaptureUsed =
+            !DiagnoseUnusedLambdaCapture(CaptureRange, FixItRange, From);
       }
+    }
 
-      // Map the capture to our AST representation.
-      LambdaCapture Capture = [&] {
-        if (From.isThisCapture()) {
-          // Capturing 'this' implicitly with a default of '[=]' is deprecated,
-          // because it results in a reference capture. Don't warn prior to
-          // C++2a; there's nothing that can be done about it before then.
-          if (getLangOpts().CPlusPlus20 && IsImplicit &&
-              CaptureDefault == LCD_ByCopy) {
-            Diag(From.getLocation(), diag::warn_deprecated_this_capture);
-            Diag(CaptureDefaultLoc, diag::note_deprecated_this_capture)
-                << FixItHint::CreateInsertion(
-                       getLocForEndOfToken(CaptureDefaultLoc), ", this");
-          }
-          return LambdaCapture(From.getLocation(), IsImplicit,
-                               From.isCopyCapture() ? LCK_StarThis : LCK_This);
-        } else if (From.isVLATypeCapture()) {
-          return LambdaCapture(From.getLocation(), IsImplicit, LCK_VLAType);
-        } else {
-          assert(From.isVariableCapture() && "unknown kind of capture");
-          ValueDecl *Var = From.getVariable();
-          LambdaCaptureKind Kind =
-              From.isCopyCapture() ? LCK_ByCopy : LCK_ByRef;
-          return LambdaCapture(From.getLocation(), IsImplicit, Kind, Var,
-                               From.getEllipsisLoc());
-        }
-      }();
+    if (CaptureRange.isValid()) {
+      CurHasPreviousCapture |= IsCaptureUsed;
+      PrevCaptureLoc = CaptureRange.getEnd();
+    }
 
-      // Form the initializer for the capture field.
-      ExprResult Init = BuildCaptureInit(From, ImplicitCaptureLoc);
+    // Map the capture to our AST representation.
+    LambdaCapture Capture = [&] {
+      if (From.isThisCapture()) {
+        // Capturing 'this' implicitly with a default of '[=]' is deprecated,
+        // because it results in a reference capture. Don't warn prior to
+        // C++2a; there's nothing that can be done about it before then.
+        if (getLangOpts().CPlusPlus20 && IsImplicit &&
+            CaptureDefault == LCD_ByCopy) {
+          Diag(From.getLocation(), diag::warn_deprecated_this_capture);
+          Diag(CaptureDefaultLoc, diag::note_deprecated_this_capture)
+              << FixItHint::CreateInsertion(
+                     getLocForEndOfToken(CaptureDefaultLoc), ", this");
+        }
+        return LambdaCapture(From.getLocation(), IsImplicit,
+                             From.isCopyCapture() ? LCK_StarThis : LCK_This);
+      } else if (From.isVLATypeCapture()) {
+        return LambdaCapture(From.getLocation(), IsImplicit, LCK_VLAType);
+      } else {
+        assert(From.isVariableCapture() && "unknown kind of capture");
+        ValueDecl *Var = From.getVariable();
+        LambdaCaptureKind Kind = From.isCopyCapture() ? LCK_ByCopy : LCK_ByRef;
+        return LambdaCapture(From.getLocation(), IsImplicit, Kind, Var,
+                             From.getEllipsisLoc());
+      }
+    }();
 
-      // FIXME: Skip this capture if the capture is not used, the initializer
-      // has no side-effects, the type of the capture is trivial, and the
-      // lambda is not externally visible.
+    // Form the initializer for the capture field.
+    ExprResult Init = BuildCaptureInit(From, ImplicitCaptureLoc);
 
-      // Add a FieldDecl for the capture and form its initializer.
-      BuildCaptureField(Class, From);
-      Captures.push_back(Capture);
-      CaptureInits.push_back(Init.get());
+    // FIXME: Skip this capture if the capture is not used, the initializer
+    // has no side-effects, the type of the capture is trivial, and the
+    // lambda is not externally visible.
 
-      if (LangOpts.CUDA)
-        CUDA().CheckLambdaCapture(CallOperator, From);
-    }
+    // Add a FieldDecl for the capture and form its initializer.
+    BuildCaptureField(Class, From);
+    Captures.push_back(Capture);
+    CaptureInits.push_back(Init.get());
 
-    Class->setCaptures(Context, Captures);
-
-    // C++11 [expr.prim.lambda]p6:
-    //   The closure type for a lambda-expression with no lambda-capture
-    //   has a public non-virtual non-explicit const conversion function
-    //   to pointer to function having the same parameter and return
-    //   types as the closure type's function call operator.
-    if (Captures.empty() && CaptureDefault == LCD_None)
-      addFunctionPointerConversions(*this, IntroducerRange, Class,
-                                    CallOperator);
-
-    // Objective-C++:
-    //   The closure type for a lambda-expression has a public non-virtual
-    //   non-explicit const conversion function to a block pointer having the
-    //   same parameter and return types as the closure type's function call
-    //   operator.
-    // FIXME: Fix generic lambda to block conversions.
-    if (getLangOpts().Blocks && getLangOpts().ObjC && !IsGenericLambda)
-      addBlockPointerConversion(*this, IntroducerRange, Class, CallOperator);
-
-    // Finalize the lambda class.
-    SmallVector<Decl*, 4> Fields(Class->fields());
-    ActOnFields(nullptr, Class->getLocation(), Class, Fields, SourceLocation(),
-                SourceLocation(), ParsedAttributesView());
-    CheckCompletedCXXClass(nullptr, Class);
+    if (LangOpts.CUDA)
+      CUDA().CheckLambdaCapture(CallOperator, From);
   }
 
+  Class->setCaptures(Context, Captures);
+
+  // C++11 [expr.prim.lambda]p6:
+  //   The closure type for a lambda-expression with no lambda-capture
+  //   has a public non-virtual non-explicit const conversion function
+  //   to pointer to function having the same parameter and return
+  //   types as the closure type's function call operator.
+  if (Captures.empty() && CaptureDefault == LCD_None)
+    addFunctionPointerConversions(*this, IntroducerRange, Class, CallOperator);
+
+  // Objective-C++:
+  //   The closure type for a lambda-expression has a public non-virtual
+  //   non-explicit const conversion function to a block pointer having the
+  //   same parameter and return types as the closure type's function call
+  //   operator.
+  // FIXME: Fix generic lambda to block conversions.
+  if (getLangOpts().Blocks && getLangOpts().ObjC && !IsGenericLambda)
+    addBlockPointerConversion(*this, IntroducerRange, Class, CallOperator);
+
+  // Finalize the lambda class.
+  SmallVector<Decl *, 4> Fields(Class->fields());
+  ActOnFields(nullptr, Class->getLocation(), Class, Fields, SourceLocation(),
+              SourceLocation(), ParsedAttributesView());
+  CheckCompletedCXXClass(nullptr, Class);
+
   Cleanup.mergeFrom(LambdaCleanup);
 
   LambdaExpr *Lambda =
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index dc73dedfb5598..e28492b579564 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -563,9 +563,8 @@ void LookupResult::resolveKind() {
     // no ambiguity if they all refer to the same type, so unique based on the
     // canonical type.
     if (const auto *TD = dyn_cast<TypeDecl>(D)) {
-      QualType T = getSema().Context.getTypeDeclType(TD);
       auto UniqueResult = UniqueTypes.insert(
-          std::make_pair(getSema().Context.getCanonicalType(T), I));
+          std::make_pair(getSema().Context.getCanonicalTypeDeclType(TD), I));
       if (!UniqueResult.second) {
         // The type is not unique.
         ExistingI = UniqueResult.first->second;
@@ -717,7 +716,7 @@ static QualType getOpenCLEnumType(Sema &S, llvm::StringRef Name) {
   EnumDecl *Decl = Result.getAsSingle<EnumDecl>();
   if (!Decl)
     return diagOpenCLBuiltinTypeError(S, "enum", Name);
-  return S.Context.getEnumType(Decl);
+  return S.Context.getCanonicalTagType(Decl);
 }
 
 /// Lookup an OpenCL typedef type.
@@ -730,7 +729,8 @@ static QualType getOpenCLTypedefType(Sema &S, llvm::StringRef Name) {
   TypedefNameDecl *Decl = Result.getAsSingle<TypedefNameDecl>();
   if (!Decl)
     return diagOpenCLBuiltinTypeError(S, "typedef", Name);
-  return S.Context.getTypedefType(Decl);
+  return S.Context.getTypedefType(ElaboratedTypeKeyword::None,
+                                  /*Qualifier=*/std::nullopt, Decl);
 }
 
 /// Get the QualType instances of the return type and arguments for an OpenCL
@@ -1001,7 +1001,7 @@ static void LookupPredefedObjCSuperType(Sema &Sema, Scope *S) {
   Sema.LookupName(Result, S);
   if (Result.getResultKind() == LookupResultKind::Found)
     if (const TagDecl *TD = Result.getAsSingle<TagDecl>())
-      Context.setObjCSuperType(Context.getTagDeclType(TD));
+      Context.setObjCSuperType(Context.getCanonicalTagType(TD));
 }
 
 void Sema::LookupNecessaryTypesForBuiltin(Scope *S, unsigned ID) {
@@ -2435,12 +2435,12 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
   if (!R.getLookupName())
     return false;
 
+#ifndef NDEBUG
   // Make sure that the declaration context is complete.
-  assert((!isa<TagDecl>(LookupCtx) ||
-          LookupCtx->isDependentContext() ||
-          cast<TagDecl>(LookupCtx)->isCompleteDefinition() ||
-          cast<TagDecl>(LookupCtx)->isBeingDefined()) &&
-         "Declaration context must already be complete!");
+  if (const auto *TD = dyn_cast<TagDecl>(LookupCtx);
+      TD && !TD->isDependentType() && TD->getDefinition() == nullptr)
+    llvm_unreachable("Declaration context must already be complete!");
+#endif
 
   struct QualifiedLookupInScope {
     bool oldVal;
@@ -2596,10 +2596,8 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
         // C++ [class.member.lookup]p3:
         //   type declarations (including injected-class-names) are replaced by
         //   the types they designate
-        if (const TypeDecl *TD = dyn_cast<TypeDecl>(ND->getUnderlyingDecl())) {
-          QualType T = Context.getTypeDeclType(TD);
-          return T.getCanonicalType().getAsOpaquePtr();
-        }
+        if (const TypeDecl *TD = dyn_cast<TypeDecl>(ND->getUnderlyingDecl()))
+          return Context.getCanonicalTypeDeclType(TD).getAsOpaquePtr();
 
         return ND->getUnderlyingDecl()->getCanonicalDecl();
       }
@@ -2704,12 +2702,10 @@ bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
 
 bool Sema::LookupQualifiedName(LookupResult &R, DeclContext *LookupCtx,
                                CXXScopeSpec &SS) {
-  auto *NNS = SS.getScopeRep();
-  if (NNS && NNS->getKind() == NestedNameSpecifier::Super)
-    return LookupInSuper(R, NNS->getAsRecordDecl());
-  else
-
-    return LookupQualifiedName(R, LookupCtx);
+  NestedNameSpecifier Qualifier = SS.getScopeRep();
+  if (Qualifier.getKind() == NestedNameSpecifier::Kind::MicrosoftSuper)
+    return LookupInSuper(R, Qualifier.getAsMicrosoftSuper());
+  return LookupQualifiedName(R, LookupCtx);
 }
 
 bool Sema::LookupParsedName(LookupResult &R, Scope *S, CXXScopeSpec *SS,
@@ -2731,7 +2727,9 @@ bool Sema::LookupParsedName(LookupResult &R, Scope *S, CXXScopeSpec *SS,
     IsDependent = !DC && ObjectType->isDependentType();
     assert(((!DC && ObjectType->isDependentType()) ||
             !ObjectType->isIncompleteType() || !ObjectType->getAs<TagType>() ||
-            ObjectType->castAs<TagType>()->isBeingDefined()) &&
+            ObjectType->castAs<TagType>()
+                ->getOriginalDecl()
+                ->isEntityBeingDefined()) &&
            "Caller should have completed object type");
   } else if (SS && SS->isNotEmpty()) {
     // This nested-name-specifier occurs after another nested-name-specifier,
@@ -2744,9 +2742,9 @@ bool Sema::LookupParsedName(LookupResult &R, Scope *S, CXXScopeSpec *SS,
       // FIXME: '__super' lookup semantics could be implemented by a
       // LookupResult::isSuperLookup flag which skips the initial search of
       // the lookup context in LookupQualified.
-      if (NestedNameSpecifier *NNS = SS->getScopeRep();
-          NNS->getKind() == NestedNameSpecifier::Super)
-        return LookupInSuper(R, NNS->getAsRecordDecl());
+      if (NestedNameSpecifier Qualifier = SS->getScopeRep();
+          Qualifier.getKind() == NestedNameSpecifier::Kind::MicrosoftSuper)
+        return LookupInSuper(R, Qualifier.getAsMicrosoftSuper());
     }
     IsDependent = !DC && isDependentScopeSpecifier(*SS);
   } else {
@@ -2772,10 +2770,12 @@ bool Sema::LookupInSuper(LookupResult &R, CXXRecordDecl *Class) {
   // members of Class itself.  That is, the naming class is Class, and the
   // access includes the access of the base.
   for (const auto &BaseSpec : Class->bases()) {
-    CXXRecordDecl *RD = cast<CXXRecordDecl>(
-        BaseSpec.getType()->castAs<RecordType>()->getDecl());
+    CXXRecordDecl *RD =
+        cast<CXXRecordDecl>(
+            BaseSpec.getType()->castAs<RecordType>()->getOriginalDecl())
+            ->getDefinitionOrSelf();
     LookupResult Result(*this, R.getLookupNameInfo(), R.getLookupKind());
-    Result.setBaseObjectType(Context.getRecordType(Class));
+    Result.setBaseObjectType(Context.getCanonicalTagType(Class));
     LookupQualifiedName(Result, RD);
 
     // Copy the lookup results into the target, merging the base's access into
@@ -3101,7 +3101,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result,
 
   // Only recurse into base classes for complete types.
   if (!Result.S.isCompleteType(Result.InstantiationLoc,
-                               Result.S.Context.getRecordType(Class)))
+                               Result.S.Context.getCanonicalTagType(Class)))
     return;
 
   // Add direct and indirect base classes along with their associated
@@ -3123,7 +3123,8 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result,
       // the classes and namespaces of known non-dependent arguments.
       if (!BaseType)
         continue;
-      CXXRecordDecl *BaseDecl = cast<CXXRecordDecl>(BaseType->getDecl());
+      CXXRecordDecl *BaseDecl = cast<CXXRecordDecl>(BaseType->getOriginalDecl())
+                                    ->getDefinitionOrSelf();
       if (Result.addClassTransitive(BaseDecl)) {
         // Find the associated namespace for this base class.
         DeclContext *BaseCtx = BaseDecl->getDeclContext();
@@ -3194,8 +3195,10 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
     //        Its associated namespaces are the innermost enclosing
     //        namespaces of its associated classes.
     case Type::Record: {
+      // FIXME: This should use the original decl.
       CXXRecordDecl *Class =
-          cast<CXXRecordDecl>(cast<RecordType>(T)->getDecl());
+          cast<CXXRecordDecl>(cast<RecordType>(T)->getOriginalDecl())
+              ->getDefinitionOrSelf();
       addAssociatedClassesAndNamespaces(Result, Class);
       break;
     }
@@ -3205,7 +3208,9 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
     //        If it is a class member, its associated class is the
     //        member’s class; else it has no associated class.
     case Type::Enum: {
-      EnumDecl *Enum = cast<EnumType>(T)->getDecl();
+      // FIXME: This should use the original decl.
+      EnumDecl *Enum =
+          cast<EnumType>(T)->getOriginalDecl()->getDefinitionOrSelf();
 
       DeclContext *Ctx = Enum->getDeclContext();
       if (CXXRecordDecl *EnclosingClass = dyn_cast<CXXRecordDecl>(Ctx))
@@ -3438,7 +3443,7 @@ Sema::LookupSpecialMember(CXXRecordDecl *RD, CXXSpecialMemberKind SM,
 
   // Prepare for overload resolution. Here we construct a synthetic argument
   // if necessary and make sure that implicit functions are declared.
-  CanQualType CanTy = Context.getCanonicalType(Context.getTagDeclType(RD));
+  CanQualType CanTy = Context.getCanonicalTagType(RD);
   DeclarationName Name;
   Expr *Arg = nullptr;
   unsigned NumArgs;
@@ -3645,7 +3650,7 @@ DeclContext::lookup_result Sema::LookupConstructors(CXXRecordDecl *Class) {
     });
   }
 
-  CanQualType T = Context.getCanonicalType(Context.getTypeDeclType(Class));
+  CanQualType T = Context.getCanonicalTagType(Class);
   DeclarationName Name = Context.DeclarationNames.getCXXConstructorName(T);
   return Class->lookup(Name);
 }
@@ -4260,7 +4265,7 @@ class LookupVisibleHelper {
           const auto *Record = BaseType->getAs<RecordType>();
           if (!Record)
             continue;
-          RD = Record->getDecl();
+          RD = Record->getOriginalDecl()->getDefinitionOrSelf();
         }
 
         // FIXME: It would be nice to be able to determine whether referencing
@@ -4546,40 +4551,101 @@ static void checkCorrectionVisibility(Sema &SemaRef, TypoCorrection &TC) {
 // the given NestedNameSpecifier (i.e. given a NestedNameSpecifier "foo::bar::",
 // fill the vector with the IdentifierInfo pointers for "foo" and "bar").
 static void getNestedNameSpecifierIdentifiers(
-    NestedNameSpecifier *NNS,
-    SmallVectorImpl<const IdentifierInfo*> &Identifiers) {
-  if (NestedNameSpecifier *Prefix = NNS->getPrefix())
-    getNestedNameSpecifierIdentifiers(Prefix, Identifiers);
-  else
+    NestedNameSpecifier NNS,
+    SmallVectorImpl<const IdentifierInfo *> &Identifiers) {
+  switch (NNS.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
     Identifiers.clear();
+    return;
 
-  const IdentifierInfo *II = nullptr;
-
-  switch (NNS->getKind()) {
-  case NestedNameSpecifier::Identifier:
-    II = NNS->getAsIdentifier();
-    break;
-
-  case NestedNameSpecifier::Namespace: {
-    const NamespaceBaseDecl *Namespace = NNS->getAsNamespace();
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [Namespace, Prefix] = NNS.getAsNamespaceAndPrefix();
+    getNestedNameSpecifierIdentifiers(Prefix, Identifiers);
     if (const auto *NS = dyn_cast<NamespaceDecl>(Namespace);
         NS && NS->isAnonymousNamespace())
       return;
-    II = Namespace->getIdentifier();
-    break;
+    Identifiers.push_back(Namespace->getIdentifier());
+    return;
   }
 
-  case NestedNameSpecifier::TypeSpec:
-    II = QualType(NNS->getAsType(), 0).getBaseTypeIdentifier();
+  case NestedNameSpecifier::Kind::Type: {
+    for (const Type *T = NNS.getAsType(); /**/; /**/) {
+      switch (T->getTypeClass()) {
+      case Type::DependentName: {
+        auto *DT = cast<DependentNameType>(T);
+        getNestedNameSpecifierIdentifiers(DT->getQualifier(), Identifiers);
+        Identifiers.push_back(DT->getIdentifier());
+        return;
+      }
+      case Type::TemplateSpecialization: {
+        TemplateName Name =
+            cast<TemplateSpecializationType>(T)->getTemplateName();
+        if (const QualifiedTemplateName *QTN =
+                Name.getAsAdjustedQualifiedTemplateName()) {
+          getNestedNameSpecifierIdentifiers(QTN->getQualifier(), Identifiers);
+          Name = QTN->getUnderlyingTemplate();
+        }
+        if (const auto *TD = Name.getAsTemplateDecl(/*IgnoreDeduced=*/true))
+          Identifiers.push_back(TD->getIdentifier());
+        return;
+      }
+      case Type::DependentTemplateSpecialization: {
+        const DependentTemplateStorage &S =
+            cast<DependentTemplateSpecializationType>(T)
+                ->getDependentTemplateName();
+        getNestedNameSpecifierIdentifiers(S.getQualifier(), Identifiers);
+        // FIXME: Should this dig into the Name as well?
+        // Identifiers.push_back(S.getName().getIdentifier());
+        return;
+      }
+      case Type::SubstTemplateTypeParm:
+        T = cast<SubstTemplateTypeParmType>(T)
+                ->getReplacementType()
+                .getTypePtr();
+        continue;
+      case Type::TemplateTypeParm:
+        Identifiers.push_back(cast<TemplateTypeParmType>(T)->getIdentifier());
+        return;
+      case Type::Decltype:
+        return;
+      case Type::Enum:
+      case Type::Record:
+      case Type::InjectedClassName: {
+        auto *TT = cast<TagType>(T);
+        getNestedNameSpecifierIdentifiers(TT->getQualifier(), Identifiers);
+        Identifiers.push_back(TT->getOriginalDecl()->getIdentifier());
+        return;
+      }
+      case Type::Typedef: {
+        auto *TT = cast<TypedefType>(T);
+        getNestedNameSpecifierIdentifiers(TT->getQualifier(), Identifiers);
+        Identifiers.push_back(TT->getDecl()->getIdentifier());
+        return;
+      }
+      case Type::Using: {
+        auto *TT = cast<UsingType>(T);
+        getNestedNameSpecifierIdentifiers(TT->getQualifier(), Identifiers);
+        Identifiers.push_back(TT->getDecl()->getIdentifier());
+        return;
+      }
+      case Type::UnresolvedUsing: {
+        auto *TT = cast<UnresolvedUsingType>(T);
+        getNestedNameSpecifierIdentifiers(TT->getQualifier(), Identifiers);
+        Identifiers.push_back(TT->getDecl()->getIdentifier());
+        return;
+      }
+      default:
+        Identifiers.push_back(QualType(T, 0).getBaseTypeIdentifier());
+        return;
+      }
+    }
     break;
+  }
 
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Super:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     return;
   }
-
-  if (II)
-    Identifiers.push_back(II);
 }
 
 void TypoCorrectionConsumer::FoundDecl(NamedDecl *ND, NamedDecl *Hiding,
@@ -4612,11 +4678,11 @@ void TypoCorrectionConsumer::FoundName(StringRef Name) {
 void TypoCorrectionConsumer::addKeywordResult(StringRef Keyword) {
   // Compute the edit distance between the typo and this keyword,
   // and add the keyword to the list of results.
-  addName(Keyword, nullptr, nullptr, true);
+  addName(Keyword, /*ND=*/nullptr, /*NNS=*/std::nullopt, /*isKeyword=*/true);
 }
 
 void TypoCorrectionConsumer::addName(StringRef Name, NamedDecl *ND,
-                                     NestedNameSpecifier *NNS, bool isKeyword) {
+                                     NestedNameSpecifier NNS, bool isKeyword) {
   // Use a simple length-based heuristic to determine the minimum possible
   // edit distance. If the minimum isn't good enough, bail out early.
   StringRef TypoStr = Typo->getName();
@@ -4708,10 +4774,10 @@ void TypoCorrectionConsumer::addNamespaces(
     Namespaces.addNameSpecifier(KNPair.first);
 
   bool SSIsTemplate = false;
-  if (NestedNameSpecifier *NNS =
-          (SS && SS->isValid()) ? SS->getScopeRep() : nullptr) {
-    if (const Type *T = NNS->getAsType())
-      SSIsTemplate = T->getTypeClass() == Type::TemplateSpecialization;
+  if (NestedNameSpecifier NNS = (SS ? SS->getScopeRep() : std::nullopt)) {
+    if (NNS.getKind() == NestedNameSpecifier::Kind::Type)
+      SSIsTemplate =
+          NNS.getAsType()->getTypeClass() == Type::TemplateSpecialization;
   }
   // Do not transform this into an iterator-based loop. The loop body can
   // trigger the creation of further types (through lazy deserialization) and
@@ -4813,17 +4879,15 @@ void TypoCorrectionConsumer::performQualifiedLookups() {
   for (const TypoCorrection &QR : QualifiedResults) {
     for (const auto &NSI : Namespaces) {
       DeclContext *Ctx = NSI.DeclCtx;
-      const Type *NSType = NSI.NameSpecifier->getAsType();
+      CXXRecordDecl *NamingClass = NSI.NameSpecifier.getAsRecordDecl();
 
       // If the current NestedNameSpecifier refers to a class and the
       // current correction candidate is the name of that class, then skip
       // it as it is unlikely a qualified version of the class' constructor
       // is an appropriate correction.
-      if (CXXRecordDecl *NSDecl = NSType ? NSType->getAsCXXRecordDecl() :
-                                           nullptr) {
-        if (NSDecl->getIdentifier() == QR.getCorrectionAsIdentifierInfo())
-          continue;
-      }
+      if (NamingClass &&
+          NamingClass->getIdentifier() == QR.getCorrectionAsIdentifierInfo())
+        continue;
 
       TypoCorrection TC(QR);
       TC.ClearCorrectionDecls();
@@ -4853,7 +4917,7 @@ void TypoCorrectionConsumer::performQualifiedLookups() {
           std::string NewQualified = TC.getAsString(SemaRef.getLangOpts());
           std::string OldQualified;
           llvm::raw_string_ostream OldOStream(OldQualified);
-          SS->getScopeRep()->print(OldOStream, SemaRef.getPrintingPolicy());
+          SS->getScopeRep().print(OldOStream, SemaRef.getPrintingPolicy());
           OldOStream << Typo->getName();
           // If correction candidate would be an identical written qualified
           // identifier, then the existing CXXScopeSpec probably included a
@@ -4864,8 +4928,7 @@ void TypoCorrectionConsumer::performQualifiedLookups() {
         for (LookupResult::iterator TRD = Result.begin(), TRDEnd = Result.end();
              TRD != TRDEnd; ++TRD) {
           if (SemaRef.CheckMemberAccess(TC.getCorrectionRange().getBegin(),
-                                        NSType ? NSType->getAsCXXRecordDecl()
-                                               : nullptr,
+                                        NamingClass,
                                         TRD.getPair()) == Sema::AR_accessible)
             TC.addCorrectionDecl(*TRD);
         }
@@ -4889,10 +4952,10 @@ void TypoCorrectionConsumer::performQualifiedLookups() {
 TypoCorrectionConsumer::NamespaceSpecifierSet::NamespaceSpecifierSet(
     ASTContext &Context, DeclContext *CurContext, CXXScopeSpec *CurScopeSpec)
     : Context(Context), CurContextChain(buildContextChain(CurContext)) {
-  if (NestedNameSpecifier *NNS =
-          CurScopeSpec ? CurScopeSpec->getScopeRep() : nullptr) {
+  if (NestedNameSpecifier NNS =
+          CurScopeSpec ? CurScopeSpec->getScopeRep() : std::nullopt) {
     llvm::raw_string_ostream SpecifierOStream(CurNameSpecifier);
-    NNS->print(SpecifierOStream, Context.getPrintingPolicy());
+    NNS.print(SpecifierOStream, Context.getPrintingPolicy());
 
     getNestedNameSpecifierIdentifiers(NNS, CurNameSpecifierIdentifiers);
   }
@@ -4906,7 +4969,7 @@ TypoCorrectionConsumer::NamespaceSpecifierSet::NamespaceSpecifierSet(
 
   // Add the global context as a NestedNameSpecifier
   SpecifierInfo SI = {cast<DeclContext>(Context.getTranslationUnitDecl()),
-                      NestedNameSpecifier::GlobalSpecifier(Context), 1};
+                      NestedNameSpecifier::getGlobal(), 1};
   DistanceMap[1].push_back(SI);
 }
 
@@ -4926,14 +4989,16 @@ auto TypoCorrectionConsumer::NamespaceSpecifierSet::buildContextChain(
 
 unsigned
 TypoCorrectionConsumer::NamespaceSpecifierSet::buildNestedNameSpecifier(
-    DeclContextList &DeclChain, NestedNameSpecifier *&NNS) {
+    DeclContextList &DeclChain, NestedNameSpecifier &NNS) {
   unsigned NumSpecifiers = 0;
   for (DeclContext *C : llvm::reverse(DeclChain)) {
     if (auto *ND = dyn_cast_or_null<NamespaceDecl>(C)) {
-      NNS = NestedNameSpecifier::Create(Context, NNS, ND);
+      NNS = NestedNameSpecifier(Context, ND, NNS);
       ++NumSpecifiers;
     } else if (auto *RD = dyn_cast_or_null<RecordDecl>(C)) {
-      NNS = NestedNameSpecifier::Create(Context, NNS, RD->getTypeForDecl());
+      QualType T = Context.getTagType(ElaboratedTypeKeyword::None, NNS, RD,
+                                      /*OwnsTag=*/false);
+      NNS = NestedNameSpecifier(T.getTypePtr());
       ++NumSpecifiers;
     }
   }
@@ -4942,7 +5007,7 @@ TypoCorrectionConsumer::NamespaceSpecifierSet::buildNestedNameSpecifier(
 
 void TypoCorrectionConsumer::NamespaceSpecifierSet::addNameSpecifier(
     DeclContext *Ctx) {
-  NestedNameSpecifier *NNS = nullptr;
+  NestedNameSpecifier NNS = std::nullopt;
   unsigned NumSpecifiers = 0;
   DeclContextList NamespaceDeclChain(buildContextChain(Ctx));
   DeclContextList FullNamespaceDeclChain(NamespaceDeclChain);
@@ -4960,7 +5025,7 @@ void TypoCorrectionConsumer::NamespaceSpecifierSet::addNameSpecifier(
   // Add an explicit leading '::' specifier if needed.
   if (NamespaceDeclChain.empty()) {
     // Rebuild the NestedNameSpecifier as a globally-qualified specifier.
-    NNS = NestedNameSpecifier::GlobalSpecifier(Context);
+    NNS = NestedNameSpecifier::getGlobal();
     NumSpecifiers =
         buildNestedNameSpecifier(FullNamespaceDeclChain, NNS);
   } else if (NamedDecl *ND =
@@ -4972,12 +5037,12 @@ void TypoCorrectionConsumer::NamespaceSpecifierSet::addNameSpecifier(
       llvm::raw_string_ostream SpecifierOStream(NewNameSpecifier);
       SmallVector<const IdentifierInfo *, 4> NewNameSpecifierIdentifiers;
       getNestedNameSpecifierIdentifiers(NNS, NewNameSpecifierIdentifiers);
-      NNS->print(SpecifierOStream, Context.getPrintingPolicy());
+      NNS.print(SpecifierOStream, Context.getPrintingPolicy());
       SameNameSpecifier = NewNameSpecifier == CurNameSpecifier;
     }
     if (SameNameSpecifier || llvm::is_contained(CurContextIdentifiers, Name)) {
       // Rebuild the NestedNameSpecifier as a globally-qualified specifier.
-      NNS = NestedNameSpecifier::GlobalSpecifier(Context);
+      NNS = NestedNameSpecifier::getGlobal();
       NumSpecifiers =
           buildNestedNameSpecifier(FullNamespaceDeclChain, NNS);
     }
@@ -5463,7 +5528,7 @@ std::string TypoCorrection::getAsString(const LangOptions &LO) const {
   if (CorrectionNameSpec) {
     std::string tmpBuffer;
     llvm::raw_string_ostream PrefixOStream(tmpBuffer);
-    CorrectionNameSpec->print(PrefixOStream, PrintingPolicy(LO));
+    CorrectionNameSpec.print(PrefixOStream, PrintingPolicy(LO));
     PrefixOStream << CorrectionName;
     return PrefixOStream.str();
   }
diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index ff9f85f960d93..773bcb225c188 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -137,7 +137,7 @@ makeTransitiveImportsVisible(ASTContext &Ctx, VisibleModuleSet &VisibleModules,
          "modules only.");
 
   llvm::SmallVector<Module *, 4> Worklist;
-  llvm::SmallSet<Module *, 16> Visited;
+  llvm::SmallPtrSet<Module *, 16> Visited;
   Worklist.push_back(Imported);
 
   Module *FoundPrimaryModuleInterface =
@@ -265,10 +265,11 @@ Sema::DeclGroupPtrTy
 Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
                       ModuleDeclKind MDK, ModuleIdPath Path,
                       ModuleIdPath Partition, ModuleImportState &ImportState,
-                      bool IntroducerIsFirstPPToken) {
+                      bool SeenNoTrivialPPDirective) {
   assert(getLangOpts().CPlusPlusModules &&
          "should only have module decl in standard C++ modules");
 
+  bool IsFirstDecl = ImportState == ModuleImportState::FirstDecl;
   bool SeenGMF = ImportState == ModuleImportState::GlobalFragment;
   // If any of the steps here fail, we count that as invalidating C++20
   // module state;
@@ -336,7 +337,8 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
 
   // In C++20, A module directive may only appear as the first preprocessing
   // tokens in a file (excluding the global module fragment.).
-  if (getLangOpts().CPlusPlusModules && !IntroducerIsFirstPPToken && !SeenGMF) {
+  if (getLangOpts().CPlusPlusModules &&
+      (!IsFirstDecl || SeenNoTrivialPPDirective) && !SeenGMF) {
     Diag(ModuleLoc, diag::err_module_decl_not_at_start);
     SourceLocation BeginLoc = PP.getMainFileFirstPPTokenLoc();
     Diag(BeginLoc, diag::note_global_module_introducer_missing)
diff --git a/clang/lib/Sema/SemaObjC.cpp b/clang/lib/Sema/SemaObjC.cpp
index 0f39a9817ce7f..8d8d5e87afe73 100644
--- a/clang/lib/Sema/SemaObjC.cpp
+++ b/clang/lib/Sema/SemaObjC.cpp
@@ -691,7 +691,7 @@ static QualType applyObjCTypeArgs(Sema &S, SourceLocation loc, QualType type,
   if (!anyPackExpansions && finalTypeArgs.size() != numTypeParams) {
     S.Diag(loc, diag::err_objc_type_args_wrong_arity)
         << (typeArgs.size() < typeParams->size()) << objcClass->getDeclName()
-        << (unsigned)finalTypeArgs.size() << (unsigned)numTypeParams;
+        << (unsigned)finalTypeArgs.size() << numTypeParams;
     S.Diag(objcClass->getLocation(), diag::note_previous_decl) << objcClass;
 
     if (failOnError)
@@ -1407,7 +1407,8 @@ SemaObjC::ObjCSubscriptKind SemaObjC::CheckSubscriptingKind(Expr *FromE) {
   int NoIntegrals = 0, NoObjCIdPointers = 0;
   SmallVector<CXXConversionDecl *, 4> ConversionDecls;
 
-  for (NamedDecl *D : cast<CXXRecordDecl>(RecordTy->getDecl())
+  for (NamedDecl *D : cast<CXXRecordDecl>(RecordTy->getOriginalDecl())
+                          ->getDefinitionOrSelf()
                           ->getVisibleConversionFunctions()) {
     if (CXXConversionDecl *Conversion =
             dyn_cast<CXXConversionDecl>(D->getUnderlyingDecl())) {
@@ -1510,7 +1511,7 @@ bool SemaObjC::isCFStringType(QualType T) {
   if (!RT)
     return false;
 
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl();
   if (RD->getTagKind() != TagTypeKind::Struct)
     return false;
 
diff --git a/clang/lib/Sema/SemaObjCProperty.cpp b/clang/lib/Sema/SemaObjCProperty.cpp
index 9dbb1d28aa722..bf6c364e40cc4 100644
--- a/clang/lib/Sema/SemaObjCProperty.cpp
+++ b/clang/lib/Sema/SemaObjCProperty.cpp
@@ -1321,7 +1321,9 @@ Decl *SemaObjC::ActOnPropertyImplDecl(
       }
       if (!CompleteTypeErr) {
         const RecordType *RecordTy = PropertyIvarType->getAs<RecordType>();
-        if (RecordTy && RecordTy->getDecl()->hasFlexibleArrayMember()) {
+        if (RecordTy && RecordTy->getOriginalDecl()
+                            ->getDefinitionOrSelf()
+                            ->hasFlexibleArrayMember()) {
           Diag(PropertyIvarLoc, diag::err_synthesize_variable_sized_ivar)
             << PropertyIvarType;
           CompleteTypeErr = true; // suppress later diagnostics about the ivar
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index 62fe3d1836ab5..1457522406531 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -646,8 +646,17 @@ ExprResult CheckVarType(SemaOpenACC &S, OpenACCClauseKind CK, Expr *VarExpr,
   if (auto *RefTy = InnerTy->getAs<ReferenceType>())
     InnerTy = RefTy->getPointeeType();
 
-  if (auto *ArrTy = InnerTy->getAsArrayTypeUnsafe())
+  if (auto *ArrTy = InnerTy->getAsArrayTypeUnsafe()) {
+    // Non constant arrays decay to 'pointer', so warn and return that we're
+    // successful.
+    if (!ArrTy->isConstantArrayType()) {
+      S.Diag(InnerLoc, clang::diag::warn_acc_var_referenced_non_const_array)
+          << InnerTy << CK;
+      return VarExpr;
+    }
+
     return CheckVarType(S, CK, VarExpr, InnerLoc, ArrTy->getElementType());
+  }
 
   auto *RD = InnerTy->getAsCXXRecordDecl();
 
@@ -681,9 +690,9 @@ ExprResult CheckVarType(SemaOpenACC &S, OpenACCClauseKind CK, Expr *VarExpr,
       }
     }
   } else if (CK == OpenACCClauseKind::Reduction) {
-    // TODO: OpenACC:
-    // Reduction must have copyctor + dtor + operation in InnerTy I think?
-    // Need to confirm when implementing this part.
+    // TODO: Reduction needs to be an aggregate, which gets checked later, so
+    // construction here isn't a problem.  However, we need to make sure that we
+    // can compare it correctly still.
   }
 
   // All 3 things need to make sure they have a dtor.
@@ -1912,8 +1921,13 @@ void SemaOpenACC::ActOnVariableDeclarator(VarDecl *VD) {
     return;
 
   // This cast should be safe, since a static-local can only happen in a
-  // function declaration.
-  auto *ContextDecl = cast<FunctionDecl>(getCurContext());
+  // function declaration. However, in error cases (or perhaps ObjC/C++?), this
+  // could possibly be something like a 'block' decl, so if this is NOT a
+  // function decl, just give up.
+  auto *ContextDecl = dyn_cast<FunctionDecl>(getCurContext());
+
+  if (!ContextDecl)
+      return;
 
   // OpenACC 3.3 2.15:
   // In C and C++, function static variables are not supported in functions to
@@ -2575,8 +2589,8 @@ SemaOpenACC::ActOnOpenACCAsteriskSizeExpr(SourceLocation AsteriskLoc) {
   return BuildOpenACCAsteriskSizeExpr(AsteriskLoc);
 }
 
-VarDecl *SemaOpenACC::CreateInitRecipe(OpenACCClauseKind CK,
-                                       const Expr *VarExpr) {
+std::pair<VarDecl *, VarDecl *>
+SemaOpenACC::CreateInitRecipe(OpenACCClauseKind CK, const Expr *VarExpr) {
   // Strip off any array subscripts/array section exprs to get to the type of
   // the variable.
   while (isa_and_present<ArraySectionExpr, ArraySubscriptExpr>(VarExpr)) {
@@ -2590,35 +2604,128 @@ VarDecl *SemaOpenACC::CreateInitRecipe(OpenACCClauseKind CK,
   // fill in with nullptr.  We'll count on TreeTransform to make this if
   // necessary.
   if (!VarExpr || VarExpr->getType()->isDependentType())
-    return nullptr;
+    return {nullptr, nullptr};
 
   QualType VarTy =
       VarExpr->getType().getNonReferenceType().getUnqualifiedType();
 
+  IdentifierInfo *VarName = [&]() {
+    switch (CK) {
+    case OpenACCClauseKind::Private:
+      return &getASTContext().Idents.get("openacc.private.init");
+    case OpenACCClauseKind::FirstPrivate:
+      return &getASTContext().Idents.get("openacc.firstprivate.init");
+    case OpenACCClauseKind::Reduction:
+      return &getASTContext().Idents.get("openacc.reduction.init");
+    default:
+      llvm_unreachable("Unknown clause kind?");
+    }
+  }();
+
   VarDecl *Recipe = VarDecl::Create(
       getASTContext(), SemaRef.getCurContext(), VarExpr->getBeginLoc(),
-      VarExpr->getBeginLoc(),
-      &getASTContext().Idents.get("openacc.private.init"), VarTy,
+      VarExpr->getBeginLoc(), VarName, VarTy,
       getASTContext().getTrivialTypeSourceInfo(VarTy), SC_Auto);
 
   ExprResult Init;
-
-  if (CK == OpenACCClauseKind::Private) {
+  VarDecl *Temporary = nullptr;
+  {
     // Trap errors so we don't get weird ones here. If we can't init, we'll just
     // swallow the errors.
     Sema::TentativeAnalysisScope Trap{SemaRef};
     InitializedEntity Entity = InitializedEntity::InitializeVariable(Recipe);
-    InitializationKind Kind =
-        InitializationKind::CreateDefault(Recipe->getLocation());
 
-    InitializationSequence InitSeq(SemaRef.SemaRef, Entity, Kind, {});
-    Init = InitSeq.Perform(SemaRef.SemaRef, Entity, Kind, {});
-  } else if (CK == OpenACCClauseKind::FirstPrivate) {
-    // TODO: OpenACC: Implement this to do a 'copy' operation.
-  } else if (CK == OpenACCClauseKind::Reduction) {
-    // TODO: OpenACC: Implement this for whatever reduction needs.
-  } else {
-    llvm_unreachable("Unknown clause kind in CreateInitRecipe");
+    if (CK == OpenACCClauseKind::Private) {
+      InitializationKind Kind =
+          InitializationKind::CreateDefault(Recipe->getLocation());
+
+      InitializationSequence InitSeq(SemaRef.SemaRef, Entity, Kind, {});
+      Init = InitSeq.Perform(SemaRef.SemaRef, Entity, Kind, {});
+
+    } else if (CK == OpenACCClauseKind::FirstPrivate) {
+      // Create a VarDecl to be the 'copied-from' for the copy section of the
+      // recipe. This allows us to make the association so that we can use the
+      // standard 'generation' ability of the init.
+      Temporary = VarDecl::Create(
+          getASTContext(), SemaRef.getCurContext(), VarExpr->getBeginLoc(),
+          VarExpr->getBeginLoc(), &getASTContext().Idents.get("openacc.temp"),
+          VarTy, getASTContext().getTrivialTypeSourceInfo(VarTy), SC_Auto);
+      auto *TemporaryDRE = DeclRefExpr::Create(
+          getASTContext(), NestedNameSpecifierLoc{}, SourceLocation{},
+          Temporary,
+          /*ReferstoEnclosingVariableOrCapture=*/false,
+          DeclarationNameInfo{DeclarationName{Temporary->getDeclName()},
+                              VarExpr->getBeginLoc()},
+          VarTy, clang::VK_LValue, Temporary, nullptr, NOUR_None);
+
+      Expr *InitExpr = nullptr;
+
+      if (const auto *ArrTy = getASTContext().getAsConstantArrayType(VarTy)) {
+        // Arrays need to have each individual element initialized as there
+        // isn't a normal 'equals' feature in C/C++. This section sets these up
+        // as an init list after 'initializing' each individual element.
+        llvm::SmallVector<Expr *> Args;
+
+        // Decay to pointer for the array subscript expression.
+        auto *CastToPtr = ImplicitCastExpr::Create(
+            getASTContext(),
+            getASTContext().getPointerType(ArrTy->getElementType()),
+            CK_ArrayToPointerDecay, TemporaryDRE, /*BasePath=*/nullptr,
+            clang::VK_LValue, FPOptionsOverride{});
+
+        for (std::size_t I = 0; I < ArrTy->getLimitedSize(); ++I) {
+          // Each element needs to be some sort of copy initialization from an
+          // array-index of the original temporary (referenced via a
+          // DeclRefExpr).
+
+          auto *Idx = IntegerLiteral::Create(
+              getASTContext(),
+              llvm::APInt(
+                  getASTContext().getTypeSize(getASTContext().getSizeType()),
+                  I),
+              getASTContext().getSizeType(), VarExpr->getBeginLoc());
+
+          Expr *Subscript = new (getASTContext()) ArraySubscriptExpr(
+              CastToPtr, Idx, ArrTy->getElementType(), clang::VK_LValue,
+              OK_Ordinary, VarExpr->getBeginLoc());
+
+          // Generate a simple copy from the result of the subscript. This will
+          // do a bitwise copy or a copy-constructor, as necessary.
+          InitializedEntity CopyEntity =
+              InitializedEntity::InitializeElement(getASTContext(), I, Entity);
+          InitializationKind CopyKind =
+              InitializationKind::CreateCopy(VarExpr->getBeginLoc(), {});
+          InitializationSequence CopySeq(SemaRef.SemaRef, CopyEntity, CopyKind,
+                                         Subscript,
+                                         /*TopLevelOfInitList=*/true);
+
+          ExprResult ElemRes =
+              CopySeq.Perform(SemaRef.SemaRef, CopyEntity, CopyKind, Subscript);
+          Args.push_back(ElemRes.get());
+        }
+
+        InitExpr = new (getASTContext())
+            InitListExpr(getASTContext(), VarExpr->getBeginLoc(), Args,
+                         VarExpr->getEndLoc());
+        InitExpr->setType(VarTy);
+
+      } else {
+        // If this isn't an array, we can just do normal copy init from a simple
+        // variable reference, so set that up.
+        InitExpr = TemporaryDRE;
+      }
+
+      InitializationKind Kind = InitializationKind::CreateForInit(
+          Recipe->getLocation(), /*DirectInit=*/true, InitExpr);
+      InitializationSequence InitSeq(SemaRef.SemaRef, Entity, Kind, InitExpr,
+                                     /*TopLevelOfInitList=*/false,
+                                     /*TreatUnavailableAsInvalid=*/false);
+      Init = InitSeq.Perform(SemaRef.SemaRef, Entity, Kind, InitExpr, &VarTy);
+    } else if (CK == OpenACCClauseKind::Reduction) {
+      // TODO: OpenACC: Implement this for whatever reduction needs.
+    } else {
+      llvm_unreachable("Unknown clause kind in CreateInitRecipe");
+    }
   }
 
   if (Init.get()) {
@@ -2626,5 +2733,5 @@ VarDecl *SemaOpenACC::CreateInitRecipe(OpenACCClauseKind CK,
     Recipe->setInitStyle(VarDecl::CallInit);
   }
 
-  return Recipe;
+  return {Recipe, Temporary};
 }
diff --git a/clang/lib/Sema/SemaOpenACCClause.cpp b/clang/lib/Sema/SemaOpenACCClause.cpp
index 88d217f2c8d25..43ae4f4d3011c 100644
--- a/clang/lib/Sema/SemaOpenACCClause.cpp
+++ b/clang/lib/Sema/SemaOpenACCClause.cpp
@@ -800,7 +800,7 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitPrivateClause(
   // Assemble the recipes list.
   for (const Expr *VarExpr : Clause.getVarList())
     InitRecipes.push_back(
-        SemaRef.CreateInitRecipe(OpenACCClauseKind::Private, VarExpr));
+        SemaRef.CreateInitRecipe(OpenACCClauseKind::Private, VarExpr).first);
 
   return OpenACCPrivateClause::Create(
       Ctx, Clause.getBeginLoc(), Clause.getLParenLoc(), Clause.getVarList(),
@@ -813,7 +813,7 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitFirstPrivateClause(
   // really isn't anything to do here. GCC does some duplicate-finding, though
   // it isn't apparent in the standard where this is justified.
 
-  llvm::SmallVector<VarDecl *> InitRecipes;
+  llvm::SmallVector<OpenACCFirstPrivateRecipe> InitRecipes;
 
   // Assemble the recipes list.
   for (const Expr *VarExpr : Clause.getVarList())
@@ -1054,13 +1054,17 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitWaitClause(
 OpenACCClause *SemaOpenACCClauseVisitor::VisitDeviceTypeClause(
     SemaOpenACC::OpenACCParsedClause &Clause) {
 
-  // Based on discussions, having more than 1 'architecture' on a 'set' is
-  // nonsensical, so we're going to fix the standard to reflect this.  Implement
-  // the limitation, since the Dialect requires this.
-  if (Clause.getDirectiveKind() == OpenACCDirectiveKind::Set &&
+  // OpenACC Pull #550 (https://github.com/OpenACC/openacc-spec/pull/550)
+  // clarified that Init, Shutdown, and Set only support a single architecture.
+  // Though the dialect only requires it for 'set' as far as we know, we'll just
+  // implement all 3 here.
+  if ((Clause.getDirectiveKind() == OpenACCDirectiveKind::Init ||
+       Clause.getDirectiveKind() == OpenACCDirectiveKind::Shutdown ||
+       Clause.getDirectiveKind() == OpenACCDirectiveKind::Set) &&
       Clause.getDeviceTypeArchitectures().size() > 1) {
     SemaRef.Diag(Clause.getDeviceTypeArchitectures()[1].getLoc(),
-                 diag::err_acc_device_type_multiple_archs);
+                 diag::err_acc_device_type_multiple_archs)
+        << Clause.getDirectiveKind();
     return nullptr;
   }
 
@@ -1770,18 +1774,27 @@ OpenACCClause *SemaOpenACCClauseVisitor::VisitReductionClause(
   }
 
   SmallVector<Expr *> ValidVars;
+  SmallVector<OpenACCReductionRecipe> Recipes;
 
   for (Expr *Var : Clause.getVarList()) {
     ExprResult Res = SemaRef.CheckReductionVar(Clause.getDirectiveKind(),
                                                Clause.getReductionOp(), Var);
 
-    if (Res.isUsable())
+    if (Res.isUsable()) {
       ValidVars.push_back(Res.get());
+
+      VarDecl *InitRecipe =
+          SemaRef.CreateInitRecipe(OpenACCClauseKind::Reduction, Res.get())
+              .first;
+      // TODO: OpenACC: Create the reduction operation recipe here too.
+      Recipes.push_back({InitRecipe});
+    }
   }
 
   return SemaRef.CheckReductionClause(
       ExistingClauses, Clause.getDirectiveKind(), Clause.getBeginLoc(),
       Clause.getLParenLoc(), Clause.getReductionOp(), ValidVars,
+      Recipes,
       Clause.getEndLoc());
 }
 
@@ -1917,61 +1930,95 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
 ExprResult SemaOpenACC::CheckReductionVar(OpenACCDirectiveKind DirectiveKind,
                                           OpenACCReductionOperator ReductionOp,
                                           Expr *VarExpr) {
+  // For now, we only support 'scalar' types, or composites/arrays of scalar
+  // types.
   VarExpr = VarExpr->IgnoreParenCasts();
+  SourceLocation VarLoc = VarExpr->getBeginLoc();
+
+  SmallVector<PartialDiagnosticAt> Notes;
+  QualType CurType = VarExpr->getType();
+
+  // For array like things, the expression can either be an array element
+  // (subscript expr), array section, or array type. Peel those off, and add
+  // notes in case we find an illegal kind.  We'll allow scalar or composite of
+  // scalars inside of this.
+  if (auto *ASE = dyn_cast<ArraySectionExpr>(VarExpr)) {
+    QualType BaseType = ArraySectionExpr::getBaseOriginalType(ASE);
+
+    PartialDiagnostic PD = PDiag(diag::note_acc_reduction_array)
+                           << diag::OACCReductionArray::Section << BaseType;
+    Notes.push_back({ASE->getBeginLoc(), PD});
+
+    CurType = getASTContext().getBaseElementType(BaseType);
+  } else if (auto *SubExpr = dyn_cast<ArraySubscriptExpr>(VarExpr)) {
+    // Array subscript already results in the type of the thing as its type, so
+    // there is no type to change here.
+    PartialDiagnostic PD =
+        PDiag(diag::note_acc_reduction_array)
+        << diag::OACCReductionArray::Subscript
+        << SubExpr->getBase()->IgnoreParenImpCasts()->getType();
+    Notes.push_back({SubExpr->getBeginLoc(), PD});
+  } else if (auto *AT = getASTContext().getAsArrayType(CurType)) {
+    // If we're already the array type, peel off the array and leave the element
+    // type.
+    CurType = getASTContext().getBaseElementType(AT);
+    PartialDiagnostic PD = PDiag(diag::note_acc_reduction_array)
+                           << diag::OACCReductionArray::ArrayTy << CurType;
+    Notes.push_back({VarLoc, PD});
+  }
 
-  auto TypeIsValid = [](QualType Ty) {
+  auto IsValidMemberOfComposite = [](QualType Ty) {
     return Ty->isDependentType() || Ty->isScalarType();
   };
 
-  if (isa<ArraySectionExpr>(VarExpr)) {
-    Expr *ASExpr = VarExpr;
-    QualType BaseTy = ArraySectionExpr::getBaseOriginalType(ASExpr);
-    QualType EltTy = getASTContext().getBaseElementType(BaseTy);
+  auto EmitDiags = [&](SourceLocation Loc, PartialDiagnostic PD) {
+    Diag(Loc, PD);
 
-    if (!TypeIsValid(EltTy)) {
-      Diag(VarExpr->getExprLoc(), diag::err_acc_reduction_type)
-          << EltTy << /*Sub array base type*/ 1;
-      return ExprError();
-    }
-  } else if (VarExpr->getType()->isArrayType()) {
-    // Arrays are considered an 'aggregate variable' explicitly, so are OK, no
-    // additional checking required.
-    //
-    // Glossary: Aggregate variables – a variable of any non-scalar datatype,
-    // including array or composite variables.
-    //
-    // The next branch (record decl) checks for composite variables.
-  } else if (auto *RD = VarExpr->getType()->getAsRecordDecl()) {
+    for (auto [Loc, PD] : Notes)
+      Diag(Loc, PD);
+
+    Diag(VarLoc, diag::note_acc_reduction_type_summary);
+  };
+
+  // If the type is already scalar, or is dependent, just give up.
+  if (IsValidMemberOfComposite(CurType)) {
+    // Nothing to do here, is valid.
+  } else if (auto *RD = CurType->getAsRecordDecl()) {
     if (!RD->isStruct() && !RD->isClass()) {
-      Diag(VarExpr->getExprLoc(), diag::err_acc_reduction_composite_type)
-          << /*not class or struct*/ 0 << VarExpr->getType();
+      EmitDiags(VarLoc, PDiag(diag::err_acc_reduction_type)
+                            << RD << diag::OACCReductionTy::NotClassStruct);
       return ExprError();
     }
 
     if (!RD->isCompleteDefinition()) {
-      Diag(VarExpr->getExprLoc(), diag::err_acc_reduction_composite_type)
-          << /*incomplete*/ 1 << VarExpr->getType();
+      EmitDiags(VarLoc, PDiag(diag::err_acc_reduction_type)
+                            << RD << diag::OACCReductionTy::NotComplete);
       return ExprError();
     }
+
     if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(RD);
         CXXRD && !CXXRD->isAggregate()) {
-      Diag(VarExpr->getExprLoc(), diag::err_acc_reduction_composite_type)
-          << /*aggregate*/ 2 << VarExpr->getType();
+      EmitDiags(VarLoc, PDiag(diag::err_acc_reduction_type)
+                            << CXXRD << diag::OACCReductionTy::NotAgg);
       return ExprError();
     }
 
     for (FieldDecl *FD : RD->fields()) {
-      if (!TypeIsValid(FD->getType())) {
-        Diag(VarExpr->getExprLoc(),
-             diag::err_acc_reduction_composite_member_type);
-        Diag(FD->getLocation(), diag::note_acc_reduction_composite_member_loc);
+      if (!IsValidMemberOfComposite(FD->getType())) {
+        PartialDiagnostic PD =
+            PDiag(diag::note_acc_reduction_member_of_composite)
+            << FD->getName() << RD->getName();
+        Notes.push_back({FD->getBeginLoc(), PD});
+        // TODO: member here.note_acc_reduction_member_of_composite
+        EmitDiags(VarLoc, PDiag(diag::err_acc_reduction_type)
+                              << FD->getType()
+                              << diag::OACCReductionTy::MemberNotScalar);
         return ExprError();
       }
     }
-  } else if (!TypeIsValid(VarExpr->getType())) {
-    Diag(VarExpr->getExprLoc(), diag::err_acc_reduction_type)
-        << VarExpr->getType() << /*Sub array base type*/ 0;
-    return ExprError();
+  } else {
+    EmitDiags(VarLoc, PDiag(diag::err_acc_reduction_type)
+                          << CurType << diag::OACCReductionTy::NotScalar);
   }
 
   // OpenACC3.3: 2.9.11: Reduction clauses on nested constructs for the same
@@ -2154,7 +2201,8 @@ OpenACCClause *SemaOpenACC::CheckReductionClause(
     ArrayRef<const OpenACCClause *> ExistingClauses,
     OpenACCDirectiveKind DirectiveKind, SourceLocation BeginLoc,
     SourceLocation LParenLoc, OpenACCReductionOperator ReductionOp,
-    ArrayRef<Expr *> Vars, SourceLocation EndLoc) {
+    ArrayRef<Expr *> Vars, ArrayRef<OpenACCReductionRecipe> Recipes,
+    SourceLocation EndLoc) {
   if (DirectiveKind == OpenACCDirectiveKind::Loop ||
       isOpenACCCombinedDirectiveKind(DirectiveKind)) {
     // OpenACC 3.3 2.9.11: A reduction clause may not appear on a loop directive
@@ -2183,7 +2231,7 @@ OpenACCClause *SemaOpenACC::CheckReductionClause(
   }
 
   auto *Ret = OpenACCReductionClause::Create(
-      getASTContext(), BeginLoc, LParenLoc, ReductionOp, Vars, EndLoc);
+      getASTContext(), BeginLoc, LParenLoc, ReductionOp, Vars, Recipes, EndLoc);
   return Ret;
 }
 
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 2c5d97c3064ac..7d800c446b595 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -7358,7 +7358,9 @@ SemaOpenMP::checkOpenMPDeclareVariantFunction(SemaOpenMP::DeclGroupPtrTy DG,
       Diag(SR.getBegin(), diag::err_omp_interop_type_not_found) << SR;
       return std::nullopt;
     }
-    QualType InteropType = Context.getTypeDeclType(TD);
+    QualType InteropType =
+        Context.getTypeDeclType(ElaboratedTypeKeyword::None,
+                                /*Qualifier=*/std::nullopt, TD);
     if (PTy->isVariadic()) {
       Diag(FD->getLocation(), diag::err_omp_append_args_with_varargs) << SR;
       return std::nullopt;
@@ -7378,7 +7380,7 @@ SemaOpenMP::checkOpenMPDeclareVariantFunction(SemaOpenMP::DeclGroupPtrTy DG,
     auto *Method = dyn_cast<CXXMethodDecl>(FD);
     if (Method && !Method->isStatic()) {
       FnPtrType = Context.getMemberPointerType(
-          AdjustedFnType, /*Qualifier=*/nullptr, Method->getParent());
+          AdjustedFnType, /*Qualifier=*/std::nullopt, Method->getParent());
       ExprResult ER;
       {
         // Build addr_of unary op to correctly handle type checks for member
@@ -18629,10 +18631,11 @@ buildDeclareReductionRef(Sema &SemaRef, SourceLocation Loc, SourceRange Range,
   if (const auto *TyRec = Ty->getAs<RecordType>()) {
     // Complete the type if it can be completed.
     // If the type is neither complete nor being defined, bail out now.
-    if (SemaRef.isCompleteType(Loc, Ty) || TyRec->isBeingDefined() ||
-        TyRec->getDecl()->getDefinition()) {
+    bool IsComplete = SemaRef.isCompleteType(Loc, Ty);
+    RecordDecl *RD = TyRec->getOriginalDecl()->getDefinition();
+    if (IsComplete || RD) {
       Lookup.clear();
-      SemaRef.LookupQualifiedName(Lookup, TyRec->getDecl());
+      SemaRef.LookupQualifiedName(Lookup, RD);
       if (Lookup.empty()) {
         Lookups.emplace_back();
         Lookups.back().append(Lookup.begin(), Lookup.end());
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 76e189d536849..7c4405b414c47 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -370,7 +370,7 @@ NarrowingKind StandardConversionSequence::getNarrowingKind(
   // the underlying type is narrowing. This only arises for expressions of
   // the form 'Enum{init}'.
   if (auto *ET = ToType->getAs<EnumType>())
-    ToType = ET->getDecl()->getIntegerType();
+    ToType = ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   switch (Second) {
   // 'bool' is an integral type; dispatch to the right place to handle it.
@@ -1063,7 +1063,8 @@ static bool shouldAddReversedEqEq(Sema &S, SourceLocation OpLoc,
       return true;
     LookupResult Members(S, NotEqOp, OpLoc,
                          Sema::LookupNameKind::LookupMemberName);
-    S.LookupQualifiedName(Members, RHSRec->getDecl());
+    S.LookupQualifiedName(Members,
+                          RHSRec->getOriginalDecl()->getDefinitionOrSelf());
     Members.suppressAccessDiagnostics();
     for (NamedDecl *Op : Members)
       if (FunctionsCorrespond(S.Context, EqFD, Op->getAsFunction()))
@@ -1268,7 +1269,7 @@ OverloadKind Sema::CheckOverload(Scope *S, FunctionDecl *New,
       //
       // Exception: if the scope is dependent and this is not a class
       // member, the using declaration can only introduce an enumerator.
-      if (UUD->getQualifier()->isDependent() && !UUD->isCXXClassMember()) {
+      if (UUD->getQualifier().isDependent() && !UUD->isCXXClassMember()) {
         Match = *I;
         return OverloadKind::NonFunction;
       }
@@ -1471,9 +1472,8 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
 
     if (OldMethod->isImplicitObjectMemberFunction() &&
         OldMethod->getParent() != NewMethod->getParent()) {
-      QualType ParentType =
-          SemaRef.Context.getTypeDeclType(OldMethod->getParent())
-              .getCanonicalType();
+      CanQualType ParentType =
+          SemaRef.Context.getCanonicalTagType(OldMethod->getParent());
       if (ParentType.getTypePtr() != BS.Ty)
         return false;
       BS.Ty = DS.Ty;
@@ -1979,7 +1979,7 @@ bool Sema::IsFunctionConversion(QualType FromType, QualType ToType,
   }
 
   // Drop 'noexcept' if not present in target type.
-  if (FromFPT) {
+  if (FromFPT && ToFPT) {
     if (FromFPT->isNothrow() && !ToFPT->isNothrow()) {
       FromFn = cast<FunctionType>(
           Context.getFunctionTypeWithExceptionSpec(QualType(FromFPT, 0),
@@ -2293,7 +2293,7 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
                == UO_AddrOf &&
                "Non-address-of operator on non-static member address");
         FromType = S.Context.getMemberPointerType(
-            FromType, /*Qualifier=*/nullptr, Method->getParent());
+            FromType, /*Qualifier=*/std::nullopt, Method->getParent());
       } else if (isa<UnaryOperator>(From->IgnoreParens())) {
         assert(cast<UnaryOperator>(From->IgnoreParens())->getOpcode() ==
                UO_AddrOf &&
@@ -2602,10 +2602,12 @@ IsTransparentUnionStandardConversion(Sema &S, Expr* From,
                                      bool CStyle) {
 
   const RecordType *UT = ToType->getAsUnionType();
-  if (!UT || !UT->getDecl()->hasAttr<TransparentUnionAttr>())
+  if (!UT)
     return false;
   // The field to initialize within the transparent union.
-  RecordDecl *UD = UT->getDecl();
+  const RecordDecl *UD = UT->getOriginalDecl()->getDefinitionOrSelf();
+  if (!UD->hasAttr<TransparentUnionAttr>())
+    return false;
   // It's compatible if the expression matches any of the fields.
   for (const auto *it : UD->fields()) {
     if (IsStandardConversion(S, From, it->getType(), InOverloadResolution, SCS,
@@ -2663,15 +2665,17 @@ bool Sema::IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType) {
   if (const EnumType *FromEnumType = FromType->getAs<EnumType>()) {
     // C++0x 7.2p9: Note that this implicit enum to int conversion is not
     // provided for a scoped enumeration.
-    if (FromEnumType->getDecl()->isScoped())
+    const EnumDecl *FromED =
+        FromEnumType->getOriginalDecl()->getDefinitionOrSelf();
+    if (FromED->isScoped())
       return false;
 
     // We can perform an integral promotion to the underlying type of the enum,
     // even if that's not the promoted type. Note that the check for promoting
     // the underlying type is based on the type alone, and does not consider
     // the bitfield-ness of the actual source expression.
-    if (FromEnumType->getDecl()->isFixed()) {
-      QualType Underlying = FromEnumType->getDecl()->getIntegerType();
+    if (FromED->isFixed()) {
+      QualType Underlying = FromED->getIntegerType();
       return Context.hasSameUnqualifiedType(Underlying, ToType) ||
              IsIntegralPromotion(nullptr, Underlying, ToType);
     }
@@ -2679,8 +2683,7 @@ bool Sema::IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType) {
     // We have already pre-calculated the promotion type, so this is trivial.
     if (ToType->isIntegerType() &&
         isCompleteType(From->getBeginLoc(), FromType))
-      return Context.hasSameUnqualifiedType(
-          ToType, FromEnumType->getDecl()->getPromotionType());
+      return Context.hasSameUnqualifiedType(ToType, FromED->getPromotionType());
 
     // C++ [conv.prom]p5:
     //   If the bit-field has an enumerated type, it is treated as any other
@@ -3347,12 +3350,12 @@ void Sema::HandleFunctionTypeMismatch(PartialDiagnostic &PDiag,
                             ToMember->getMostRecentCXXRecordDecl())) {
       PDiag << ft_different_class;
       if (ToMember->isSugared())
-        PDiag << Context.getTypeDeclType(
+        PDiag << Context.getCanonicalTagType(
             ToMember->getMostRecentCXXRecordDecl());
       else
         PDiag << ToMember->getQualifier();
       if (FromMember->isSugared())
-        PDiag << Context.getTypeDeclType(
+        PDiag << Context.getCanonicalTagType(
             FromMember->getMostRecentCXXRecordDecl());
       else
         PDiag << FromMember->getQualifier();
@@ -3636,12 +3639,12 @@ Sema::MemberPointerConversionResult Sema::CheckMemberPointerConversion(
   CXXRecordDecl *FromClass = FromPtrType->getMostRecentCXXRecordDecl(),
                 *ToClass = ToPtrType->getMostRecentCXXRecordDecl();
 
-  auto DiagCls = [](PartialDiagnostic &PD, NestedNameSpecifier *Qual,
-                    const CXXRecordDecl *Cls) {
-    if (declaresSameEntity(Qual->getAsRecordDecl(), Cls))
+  auto DiagCls = [&](PartialDiagnostic &PD, NestedNameSpecifier Qual,
+                     const CXXRecordDecl *Cls) {
+    if (declaresSameEntity(Qual.getAsRecordDecl(), Cls))
       PD << Qual;
     else
-      PD << QualType(Cls->getTypeForDecl(), 0);
+      PD << Context.getCanonicalTagType(Cls);
   };
   auto DiagFromTo = [&](PartialDiagnostic &PD) -> PartialDiagnostic & {
     DiagCls(PD, FromPtrType->getQualifier(), FromClass);
@@ -3658,8 +3661,7 @@ Sema::MemberPointerConversionResult Sema::CheckMemberPointerConversion(
   if (!IsDerivedFrom(OpRange.getBegin(), Derived, Base, Paths))
     return MemberPointerConversionResult::NotDerived;
 
-  if (Paths.isAmbiguous(
-          Base->getTypeForDecl()->getCanonicalTypeUnqualified())) {
+  if (Paths.isAmbiguous(Context.getCanonicalTagType(Base))) {
     PartialDiagnostic PD = PDiag(diag::err_ambiguous_memptr_conv);
     PD << int(Direction);
     DiagFromTo(PD) << getAmbiguousPathsDisplayString(Paths) << OpRange;
@@ -3687,8 +3689,8 @@ Sema::MemberPointerConversionResult Sema::CheckMemberPointerConversion(
             ? diag::err_upcast_to_inaccessible_base
             : diag::err_downcast_from_inaccessible_base,
         [&](PartialDiagnostic &PD) {
-          NestedNameSpecifier *BaseQual = FromPtrType->getQualifier(),
-                              *DerivedQual = ToPtrType->getQualifier();
+          NestedNameSpecifier BaseQual = FromPtrType->getQualifier(),
+                              DerivedQual = ToPtrType->getQualifier();
           if (Direction == MemberPointerConversionDirection::Upcast)
             std::swap(BaseQual, DerivedQual);
           DiagCls(PD, DerivedQual, Derived);
@@ -3983,8 +3985,9 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
 
     if (!S.isCompleteType(From->getExprLoc(), ToType)) {
       // We're not going to find any constructors.
-    } else if (CXXRecordDecl *ToRecordDecl
-                 = dyn_cast<CXXRecordDecl>(ToRecordType->getDecl())) {
+    } else if (auto *ToRecordDecl =
+                   dyn_cast<CXXRecordDecl>(ToRecordType->getOriginalDecl())) {
+      ToRecordDecl = ToRecordDecl->getDefinitionOrSelf();
 
       Expr **Args = &From;
       unsigned NumArgs = 1;
@@ -4057,8 +4060,9 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
     // No conversion functions from incomplete types.
   } else if (const RecordType *FromRecordType =
                  From->getType()->getAs<RecordType>()) {
-    if (CXXRecordDecl *FromRecordDecl
-         = dyn_cast<CXXRecordDecl>(FromRecordType->getDecl())) {
+    if (auto *FromRecordDecl =
+            dyn_cast<CXXRecordDecl>(FromRecordType->getOriginalDecl())) {
+      FromRecordDecl = FromRecordDecl->getDefinitionOrSelf();
       // Add all of the conversion functions as candidates.
       const auto &Conversions = FromRecordDecl->getVisibleConversionFunctions();
       for (auto I = Conversions.begin(), E = Conversions.end(); I != E; ++I) {
@@ -4509,7 +4513,8 @@ getFixedEnumPromtion(Sema &S, const StandardConversionSequence &SCS) {
   if (!FromType->isEnumeralType())
     return FixedEnumPromotion::None;
 
-  EnumDecl *Enum = FromType->castAs<EnumType>()->getDecl();
+  EnumDecl *Enum =
+      FromType->castAs<EnumType>()->getOriginalDecl()->getDefinitionOrSelf();
   if (!Enum->isFixed())
     return FixedEnumPromotion::None;
 
@@ -5145,7 +5150,9 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
                          Expr *Init, QualType T2, bool AllowRvalues,
                          bool AllowExplicit) {
   assert(T2->isRecordType() && "Can only find conversions of record types.");
-  auto *T2RecordDecl = cast<CXXRecordDecl>(T2->castAs<RecordType>()->getDecl());
+  auto *T2RecordDecl =
+      cast<CXXRecordDecl>(T2->castAs<RecordType>()->getOriginalDecl())
+          ->getDefinitionOrSelf();
 
   OverloadCandidateSet CandidateSet(
       DeclLoc, OverloadCandidateSet::CSK_InitByUserDefinedConversion);
@@ -5934,7 +5941,7 @@ static ImplicitConversionSequence TryObjectArgumentInitialization(
 
   assert(FromType->isRecordType());
 
-  QualType ClassType = S.Context.getTypeDeclType(ActingContext);
+  CanQualType ClassType = S.Context.getCanonicalTagType(ActingContext);
   // C++98 [class.dtor]p2:
   //   A destructor can be invoked for a const, volatile or const volatile
   //   object.
@@ -6055,7 +6062,7 @@ static ImplicitConversionSequence TryObjectArgumentInitialization(
 /// the implicit object parameter for the given Method with the given
 /// expression.
 ExprResult Sema::PerformImplicitObjectArgumentInitialization(
-    Expr *From, NestedNameSpecifier *Qualifier, NamedDecl *FoundDecl,
+    Expr *From, NestedNameSpecifier Qualifier, NamedDecl *FoundDecl,
     CXXMethodDecl *Method) {
   QualType FromRecordType, DestType;
   QualType ImplicitParamRecordType = Method->getFunctionObjectParameterType();
@@ -6268,7 +6275,9 @@ static ExprResult BuildConvertedConstantExpression(Sema &S, Expr *From,
                                                    QualType T, CCEKind CCE,
                                                    NamedDecl *Dest,
                                                    APValue &PreNarrowingValue) {
-  assert((S.getLangOpts().CPlusPlus11 || CCE == CCEKind::TempArgStrict) &&
+  [[maybe_unused]] bool isCCEAllowedPreCXX11 =
+      (CCE == CCEKind::TempArgStrict || CCE == CCEKind::ExplicitBool);
+  assert((S.getLangOpts().CPlusPlus11 || isCCEAllowedPreCXX11) &&
          "converted constant expression outside C++11 or TTP matching");
 
   if (checkPlaceholderForOverload(S, From))
@@ -6840,8 +6849,9 @@ ExprResult Sema::PerformContextualImplicitConversion(
   UnresolvedSet<4>
       ViableConversions; // These are *potentially* viable in C++1y.
   UnresolvedSet<4> ExplicitConversions;
-  const auto &Conversions =
-      cast<CXXRecordDecl>(RecordTy->getDecl())->getVisibleConversionFunctions();
+  const auto &Conversions = cast<CXXRecordDecl>(RecordTy->getOriginalDecl())
+                                ->getDefinitionOrSelf()
+                                ->getVisibleConversionFunctions();
 
   bool HadMultipleCandidates =
       (std::distance(Conversions.begin(), Conversions.end()) > 1);
@@ -7160,7 +7170,8 @@ void Sema::AddOverloadCandidate(
     // C++ [class.copy]p3:
     //   A member function template is never instantiated to perform the copy
     //   of a class object to an object of its class type.
-    QualType ClassType = Context.getTypeDeclType(Constructor->getParent());
+    CanQualType ClassType =
+        Context.getCanonicalTagType(Constructor->getParent());
     if (Args.size() == 1 && Constructor->isSpecializationCopyingObject() &&
         (Context.hasSameUnqualifiedType(ClassType, Args[0]->getType()) ||
          IsDerivedFrom(Args[0]->getBeginLoc(), Args[0]->getType(),
@@ -7181,8 +7192,8 @@ void Sema::AddOverloadCandidate(
     if (Shadow && Args.size() == 1 && Constructor->getNumParams() >= 1 &&
         Constructor->getParamDecl(0)->getType()->isReferenceType()) {
       QualType P = Constructor->getParamDecl(0)->getType()->getPointeeType();
-      QualType C = Context.getRecordType(Constructor->getParent());
-      QualType D = Context.getRecordType(Shadow->getParent());
+      CanQualType C = Context.getCanonicalTagType(Constructor->getParent());
+      CanQualType D = Context.getCanonicalTagType(Shadow->getParent());
       SourceLocation Loc = Args.front()->getExprLoc();
       if ((Context.hasSameUnqualifiedType(P, C) || IsDerivedFrom(Loc, P, C)) &&
           (Context.hasSameUnqualifiedType(D, P) || IsDerivedFrom(Loc, D, P))) {
@@ -7403,7 +7414,7 @@ static bool convertArgsForAvailabilityChecks(
            "Shouldn't have `this` for ctors!");
     assert(!Method->isStatic() && "Shouldn't have `this` for static methods!");
     ExprResult R = S.PerformImplicitObjectArgumentInitialization(
-        ThisArg, /*Qualifier=*/nullptr, Method, Method);
+        ThisArg, /*Qualifier=*/std::nullopt, Method, Method);
     if (R.isInvalid())
       return false;
     ConvertedThis = R.get();
@@ -8159,29 +8170,24 @@ bool Sema::CheckNonDependentConversions(
       ArgType = ArgType->getPointeeType();
     }
 
-    if (auto *RT = ParamType->getAs<RecordType>())
-      if (auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
-          RD && RD->hasDefinition()) {
-        if (llvm::any_of(LookupConstructors(RD), [](NamedDecl *ND) {
-              auto Info = getConstructorInfo(ND);
-              if (!Info)
-                return false;
-              CXXConstructorDecl *Ctor = Info.Constructor;
-              /// isConvertingConstructor takes copy/move constructors into
-              /// account!
-              return !Ctor->isCopyOrMoveConstructor() &&
-                     Ctor->isConvertingConstructor(
-                         /*AllowExplicit=*/true);
-            }))
-          return true;
-      }
-
-    if (auto *RT = ArgType->getAs<RecordType>())
-      if (auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl());
-          RD && RD->hasDefinition() &&
-          !RD->getVisibleConversionFunctions().empty()) {
-        return true;
-      }
+    if (auto *RD = ParamType->getAsCXXRecordDecl();
+        RD && RD->hasDefinition() &&
+        llvm::any_of(LookupConstructors(RD), [](NamedDecl *ND) {
+          auto Info = getConstructorInfo(ND);
+          if (!Info)
+            return false;
+          CXXConstructorDecl *Ctor = Info.Constructor;
+          /// isConvertingConstructor takes copy/move constructors into
+          /// account!
+          return !Ctor->isCopyOrMoveConstructor() &&
+                 Ctor->isConvertingConstructor(
+                     /*AllowExplicit=*/true);
+        }))
+      return true;
+    if (auto *RD = ArgType->getAsCXXRecordDecl();
+        RD && RD->hasDefinition() &&
+        !RD->getVisibleConversionFunctions().empty())
+      return true;
 
     return false;
   };
@@ -8336,7 +8342,8 @@ void Sema::AddConversionCandidate(
   if (const auto *FromPtrType = ObjectType->getAs<PointerType>())
     ObjectType = FromPtrType->getPointeeType();
   const auto *ConversionContext =
-      cast<CXXRecordDecl>(ObjectType->castAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(ObjectType->castAs<RecordType>()->getOriginalDecl())
+          ->getDefinitionOrSelf();
 
   // C++23 [over.best.ics.general]
   // However, if the target is [...]
@@ -8736,15 +8743,16 @@ void Sema::AddMemberOperatorCandidates(OverloadedOperatorKind Op,
   //        qualified lookup of T1::operator@ (13.3.1.1.1); otherwise,
   //        the set of member candidates is empty.
   if (const RecordType *T1Rec = T1->getAs<RecordType>()) {
+    bool IsComplete = isCompleteType(OpLoc, T1);
+    CXXRecordDecl *T1RD =
+        cast<CXXRecordDecl>(T1Rec->getOriginalDecl())->getDefinition();
     // Complete the type if it can be completed.
-    if (!isCompleteType(OpLoc, T1) && !T1Rec->isBeingDefined())
-      return;
     // If the type is neither complete nor being defined, bail out now.
-    if (!T1Rec->getDecl()->getDefinition())
+    if (!T1RD || (!IsComplete && !T1RD->isBeingDefined()))
       return;
 
     LookupResult Operators(*this, OpName, OpLoc, LookupOrdinaryName);
-    LookupQualifiedName(Operators, T1Rec->getDecl());
+    LookupQualifiedName(Operators, T1RD);
     Operators.suppressAccessDiagnostics();
 
     for (LookupResult::iterator Oper = Operators.begin(),
@@ -9009,8 +9017,8 @@ BuiltinCandidateTypeSet::AddMemberPointerWithMoreQualifiedTypeVariants(
     if ((CVR | BaseCVR) != CVR) continue;
 
     QualType QPointeeTy = Context.getCVRQualifiedType(PointeeTy, CVR);
-    MemberPointerTypes.insert(
-        Context.getMemberPointerType(QPointeeTy, /*Qualifier=*/nullptr, Cls));
+    MemberPointerTypes.insert(Context.getMemberPointerType(
+        QPointeeTy, /*Qualifier=*/std::nullopt, Cls));
   }
 
   return true;
@@ -9087,7 +9095,8 @@ BuiltinCandidateTypeSet::AddTypesConvertedFrom(QualType Ty,
     if (!SemaRef.isCompleteType(Loc, Ty))
       return;
 
-    CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(TyRec->getDecl());
+    auto *ClassDecl =
+        cast<CXXRecordDecl>(TyRec->getOriginalDecl())->getDefinitionOrSelf();
     for (NamedDecl *D : ClassDecl->getVisibleConversionFunctions()) {
       if (isa<UsingShadowDecl>(D))
         D = cast<UsingShadowDecl>(D)->getTargetDecl();
@@ -10199,7 +10208,7 @@ class BuiltinOperatorOverloadBuilder {
 
       if (S.getLangOpts().CPlusPlus11) {
         for (QualType EnumTy : CandidateTypes[ArgIdx].enumeration_types()) {
-          if (!EnumTy->castAs<EnumType>()->getDecl()->isScoped())
+          if (!EnumTy->castAs<EnumType>()->getOriginalDecl()->isScoped())
             continue;
 
           if (!AddedTypes.insert(S.Context.getCanonicalType(EnumTy)).second)
@@ -10955,9 +10964,9 @@ bool clang::isBetterOverloadCandidate(
             isa<CXXConversionDecl>(Cand1.Function) ? TPOC_Conversion
                                                    : TPOC_Call,
             Cand1.ExplicitCallArguments,
-            Obj1Context ? QualType(Obj1Context->getTypeForDecl(), 0)
+            Obj1Context ? S.Context.getCanonicalTagType(Obj1Context)
                         : QualType{},
-            Obj2Context ? QualType(Obj2Context->getTypeForDecl(), 0)
+            Obj2Context ? S.Context.getCanonicalTagType(Obj2Context)
                         : QualType{},
             Cand1.isReversed() ^ Cand2.isReversed(), PartialOverloading)) {
       return BetterTemplate == Cand1.Function->getPrimaryTemplate();
@@ -14863,8 +14872,8 @@ ExprResult Sema::BuildCXXMemberCallExpr(Expr *E, NamedDecl *FoundDecl,
   if (Method->isExplicitObjectMemberFunction())
     Exp = InitializeExplicitObjectArgument(*this, E, Method);
   else
-    Exp = PerformImplicitObjectArgumentInitialization(E, /*Qualifier=*/nullptr,
-                                                      FoundDecl, Method);
+    Exp = PerformImplicitObjectArgumentInitialization(
+        E, /*Qualifier=*/std::nullopt, FoundDecl, Method);
   if (Exp.isInvalid())
     return true;
 
@@ -15025,7 +15034,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
           InputInit = InitializeExplicitObjectArgument(*this, Input, Method);
         else
           InputInit = PerformImplicitObjectArgumentInitialization(
-              Input, /*Qualifier=*/nullptr, Best->FoundDecl, Method);
+              Input, /*Qualifier=*/std::nullopt, Best->FoundDecl, Method);
         if (InputInit.isInvalid())
           return ExprError();
         Base = Input = InputInit.get();
@@ -15408,7 +15417,7 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
             ParamIdx = 1;
           } else {
             Arg0 = PerformImplicitObjectArgumentInitialization(
-                Args[0], /*Qualifier=*/nullptr, Best->FoundDecl, Method);
+                Args[0], /*Qualifier=*/std::nullopt, Best->FoundDecl, Method);
           }
           Arg1 = PerformCopyInitialization(
               InitializedEntity::InitializeParameter(
@@ -15878,7 +15887,7 @@ ExprResult Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
           ArgExpr = Args;
         } else {
           ExprResult Arg0 = PerformImplicitObjectArgumentInitialization(
-              Args[0], /*Qualifier=*/nullptr, Best->FoundDecl, Method);
+              Args[0], /*Qualifier=*/std::nullopt, Best->FoundDecl, Method);
           if (Arg0.isInvalid())
             return ExprError();
 
@@ -16074,7 +16083,7 @@ ExprResult Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
   CXXMethodDecl *Method = nullptr;
   bool HadMultipleCandidates = false;
   DeclAccessPair FoundDecl = DeclAccessPair::make(nullptr, AS_public);
-  NestedNameSpecifier *Qualifier = nullptr;
+  NestedNameSpecifier Qualifier = std::nullopt;
   if (isa<MemberExpr>(NakedMemExpr)) {
     MemExpr = cast<MemberExpr>(NakedMemExpr);
     Method = cast<CXXMethodDecl>(MemExpr->getMemberDecl());
@@ -16346,7 +16355,7 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
 
   const auto *Record = Object.get()->getType()->castAs<RecordType>();
   LookupResult R(*this, OpName, LParenLoc, LookupOrdinaryName);
-  LookupQualifiedName(R, Record->getDecl());
+  LookupQualifiedName(R, Record->getOriginalDecl()->getDefinitionOrSelf());
   R.suppressAccessDiagnostics();
 
   for (LookupResult::iterator Oper = R.begin(), OperEnd = R.end();
@@ -16390,8 +16399,9 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
   //   functions for each conversion function declared in an
   //   accessible base class provided the function is not hidden
   //   within T by another intervening declaration.
-  const auto &Conversions =
-      cast<CXXRecordDecl>(Record->getDecl())->getVisibleConversionFunctions();
+  const auto &Conversions = cast<CXXRecordDecl>(Record->getOriginalDecl())
+                                ->getDefinitionOrSelf()
+                                ->getVisibleConversionFunctions();
   for (auto I = Conversions.begin(), E = Conversions.end();
        !IgnoreSurrogateFunctions && I != E; ++I) {
     NamedDecl *D = *I;
@@ -16541,7 +16551,7 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
     IsError |= PrepareExplicitObjectArgument(*this, Method, Obj, Args, NewArgs);
   } else {
     ExprResult ObjRes = PerformImplicitObjectArgumentInitialization(
-        Object.get(), /*Qualifier=*/nullptr, Best->FoundDecl, Method);
+        Object.get(), /*Qualifier=*/std::nullopt, Best->FoundDecl, Method);
     if (ObjRes.isInvalid())
       IsError = true;
     else
@@ -16612,7 +16622,10 @@ ExprResult Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base,
     return ExprError();
 
   LookupResult R(*this, OpName, OpLoc, LookupOrdinaryName);
-  LookupQualifiedName(R, Base->getType()->castAs<RecordType>()->getDecl());
+  LookupQualifiedName(R, Base->getType()
+                             ->castAs<RecordType>()
+                             ->getOriginalDecl()
+                             ->getDefinitionOrSelf());
   R.suppressAccessDiagnostics();
 
   for (LookupResult::iterator Oper = R.begin(), OperEnd = R.end();
@@ -16686,7 +16699,7 @@ ExprResult Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base,
     Base = R.get();
   } else {
     ExprResult BaseResult = PerformImplicitObjectArgumentInitialization(
-        Base, /*Qualifier=*/nullptr, Best->FoundDecl, Method);
+        Base, /*Qualifier=*/std::nullopt, Best->FoundDecl, Method);
     if (BaseResult.isInvalid())
       return ExprError();
     Base = BaseResult.get();
@@ -16940,7 +16953,7 @@ ExprResult Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
 
         assert(isa<DeclRefExpr>(SubExpr.get()) &&
                "fixed to something other than a decl ref");
-        NestedNameSpecifier *Qualifier =
+        NestedNameSpecifier Qualifier =
             cast<DeclRefExpr>(SubExpr.get())->getQualifier();
         assert(Qualifier &&
                "fixed to a member ref with no nested name qualifier");
diff --git a/clang/lib/Sema/SemaPPC.cpp b/clang/lib/Sema/SemaPPC.cpp
index d5c83aedb3008..46d7372dd056b 100644
--- a/clang/lib/Sema/SemaPPC.cpp
+++ b/clang/lib/Sema/SemaPPC.cpp
@@ -41,8 +41,10 @@ void SemaPPC::checkAIXMemberAlignment(SourceLocation Loc, const Expr *Arg) {
     return;
 
   QualType ArgType = Arg->getType();
-  for (const FieldDecl *FD :
-       ArgType->castAs<RecordType>()->getDecl()->fields()) {
+  for (const FieldDecl *FD : ArgType->castAs<RecordType>()
+                                 ->getOriginalDecl()
+                                 ->getDefinitionOrSelf()
+                                 ->fields()) {
     if (const auto *AA = FD->getAttr<AlignedAttr>()) {
       CharUnits Alignment = getASTContext().toCharUnitsFromBits(
           AA->getAlignment(getASTContext()));
@@ -106,6 +108,7 @@ bool SemaPPC::CheckPPCBuiltinFunctionCall(const TargetInfo &TI,
   switch (BuiltinID) {
   default:
     return false;
+  case PPC::BI__builtin_ppc_bcdsetsign:
   case PPC::BI__builtin_ppc_national2packed:
   case PPC::BI__builtin_ppc_packed2zoned:
   case PPC::BI__builtin_ppc_zoned2packed:
diff --git a/clang/lib/Sema/SemaSYCL.cpp b/clang/lib/Sema/SemaSYCL.cpp
index 4683c81bd1c60..2f97f6290f0e8 100644
--- a/clang/lib/Sema/SemaSYCL.cpp
+++ b/clang/lib/Sema/SemaSYCL.cpp
@@ -221,7 +221,7 @@ static SourceLocation SourceLocationForUserDeclaredType(QualType QT) {
   SourceLocation Loc;
   const Type *T = QT->getUnqualifiedDesugaredType();
   if (const TagType *TT = dyn_cast<TagType>(T))
-    Loc = TT->getDecl()->getLocation();
+    Loc = TT->getOriginalDecl()->getLocation();
   else if (const ObjCInterfaceType *ObjCIT = dyn_cast<ObjCInterfaceType>(T))
     Loc = ObjCIT->getDecl()->getLocation();
   return Loc;
@@ -250,6 +250,23 @@ static bool CheckSYCLKernelName(Sema &S, SourceLocation Loc,
   return false;
 }
 
+void SemaSYCL::CheckSYCLExternalFunctionDecl(FunctionDecl *FD) {
+  const auto *SEAttr = FD->getAttr<SYCLExternalAttr>();
+  assert(SEAttr && "Missing sycl_external attribute");
+  if (!FD->isInvalidDecl() && !FD->isTemplated()) {
+    if (!FD->isExternallyVisible())
+      if (!FD->isFunctionTemplateSpecialization() ||
+          FD->getTemplateSpecializationInfo()->isExplicitSpecialization())
+        Diag(SEAttr->getLocation(), diag::err_sycl_external_invalid_linkage)
+            << SEAttr;
+  }
+  if (FD->isDeletedAsWritten()) {
+    Diag(SEAttr->getLocation(),
+         diag::err_sycl_external_invalid_deleted_function)
+        << SEAttr;
+  }
+}
+
 void SemaSYCL::CheckSYCLEntryPointFunctionDecl(FunctionDecl *FD) {
   // Ensure that all attributes present on the declaration are consistent
   // and warn about any redundant ones.
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index a5f92020f49f8..bc1ddb80961a2 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -316,17 +316,10 @@ void DiagnoseUnused(Sema &S, const Expr *E, std::optional<unsigned> DiagID) {
       }
     }
   } else if (const auto *CE = dyn_cast<CXXConstructExpr>(E)) {
-    if (const CXXConstructorDecl *Ctor = CE->getConstructor()) {
-      const NamedDecl *OffendingDecl = nullptr;
-      const auto *A = Ctor->getAttr<WarnUnusedResultAttr>();
-      if (!A) {
-        OffendingDecl = Ctor->getParent();
-        A = OffendingDecl->getAttr<WarnUnusedResultAttr>();
-      }
-      if (DiagnoseNoDiscard(S, OffendingDecl, A, Loc, R1, R2,
-                            /*isCtor=*/true))
-        return;
-    }
+    auto [OffendingDecl, A] = CE->getUnusedResultAttr(S.Context);
+    if (DiagnoseNoDiscard(S, OffendingDecl, A, Loc, R1, R2,
+                          /*isCtor=*/true))
+      return;
   } else if (const auto *ILE = dyn_cast<InitListExpr>(E)) {
     if (const TagDecl *TD = ILE->getType()->getAsTagDecl()) {
 
@@ -798,7 +791,7 @@ bool Sema::checkMustTailAttr(const Stmt *St, const Attr &MTA) {
     const auto *MPT =
         CalleeBinOp->getRHS()->getType()->castAs<MemberPointerType>();
     CalleeType.This =
-        Context.getTypeDeclType(MPT->getMostRecentCXXRecordDecl());
+        Context.getCanonicalTagType(MPT->getMostRecentCXXRecordDecl());
     CalleeType.Func = MPT->getPointeeType()->castAs<FunctionProtoType>();
     CalleeType.MemberType = FuncType::ft_pointer_to_member;
   } else if (isa<CXXPseudoDestructorExpr>(CalleeExpr)) {
@@ -1254,7 +1247,7 @@ static bool ShouldDiagnoseSwitchCaseNotInEnum(const Sema &S,
           dyn_cast<DeclRefExpr>(CaseExpr->IgnoreParenImpCasts())) {
     if (const VarDecl *VD = dyn_cast<VarDecl>(DRE->getDecl())) {
       QualType VarType = VD->getType();
-      QualType EnumType = S.Context.getTypeDeclType(ED);
+      CanQualType EnumType = S.Context.getCanonicalTagType(ED);
       if (VD->hasGlobalStorage() && VarType.isConstQualified() &&
           S.Context.hasSameUnqualifiedType(EnumType, VarType))
         return false;
@@ -1284,11 +1277,11 @@ static void checkEnumTypesInSwitchStmt(Sema &S, const Expr *Cond,
     return;
 
   // Ignore anonymous enums.
-  if (!CondEnumType->getDecl()->getIdentifier() &&
-      !CondEnumType->getDecl()->getTypedefNameForAnonDecl())
+  if (!CondEnumType->getOriginalDecl()->getIdentifier() &&
+      !CondEnumType->getOriginalDecl()->getTypedefNameForAnonDecl())
     return;
-  if (!CaseEnumType->getDecl()->getIdentifier() &&
-      !CaseEnumType->getDecl()->getTypedefNameForAnonDecl())
+  if (!CaseEnumType->getOriginalDecl()->getIdentifier() &&
+      !CaseEnumType->getOriginalDecl()->getTypedefNameForAnonDecl())
     return;
 
   if (S.Context.hasSameUnqualifiedType(CondType, CaseType))
@@ -1601,9 +1594,11 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
 
     // If switch has default case, then ignore it.
     if (!CaseListIsErroneous && !CaseListIsIncomplete && !HasConstantCond &&
-        ET && ET->getDecl()->isCompleteDefinition() &&
-        !ET->getDecl()->enumerators().empty()) {
-      const EnumDecl *ED = ET->getDecl();
+        ET) {
+      const EnumDecl *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
+      if (!ED->isCompleteDefinition() || ED->enumerators().empty())
+        goto enum_out;
+
       EnumValsTy EnumVals;
 
       // Gather all enum values, set their type and sort them,
@@ -1716,6 +1711,7 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch,
       if (!hasCasesNotInSwitch)
         SS->setAllEnumCasesCovered();
     }
+  enum_out:;
   }
 
   if (BodyStmt)
@@ -1745,7 +1741,7 @@ Sema::DiagnoseAssignmentEnum(QualType DstType, QualType SrcType,
   if (SrcExpr->isTypeDependent() || SrcExpr->isValueDependent())
     return;
 
-  const EnumDecl *ED = ET->getDecl();
+  const EnumDecl *ED = ET->getOriginalDecl()->getDefinitionOrSelf();
   if (!ED->isClosed())
     return;
 
@@ -3710,7 +3706,7 @@ class LocalTypedefNameReferencer : public DynamicRecursiveASTVisitor {
   Sema &S;
 };
 bool LocalTypedefNameReferencer::VisitRecordType(RecordType *RT) {
-  auto *R = dyn_cast<CXXRecordDecl>(RT->getDecl());
+  auto *R = dyn_cast<CXXRecordDecl>(RT->getOriginalDecl());
   if (!R || !R->isLocalClass() || !R->isLocalClass()->isExternallyVisible() ||
       R->isDependentType())
     return true;
@@ -3929,7 +3925,7 @@ StmtResult Sema::BuildReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp,
             << RetValExp->getSourceRange();
     if (FD->hasAttr<CmseNSEntryAttr>() && RetValExp) {
       if (const auto *RT = dyn_cast<RecordType>(FnRetType.getCanonicalType())) {
-        if (RT->getDecl()->isOrContainsUnion())
+        if (RT->getOriginalDecl()->isOrContainsUnion())
           Diag(RetValExp->getBeginLoc(), diag::warn_cmse_nonsecure_union) << 1;
       }
     }
@@ -4620,7 +4616,8 @@ void Sema::ActOnCapturedRegionStart(SourceLocation Loc, Scope *CurScope,
   // Build the context parameter
   DeclContext *DC = CapturedDecl::castToDeclContext(CD);
   IdentifierInfo *ParamName = &Context.Idents.get("__context");
-  QualType ParamType = Context.getPointerType(Context.getTagDeclType(RD));
+  CanQualType ParamType =
+      Context.getPointerType(Context.getCanonicalTagType(RD));
   auto *Param =
       ImplicitParamDecl::Create(Context, DC, Loc, ParamName, ParamType,
                                 ImplicitParamKind::CapturedContext);
@@ -4662,9 +4659,10 @@ void Sema::ActOnCapturedRegionStart(SourceLocation Loc, Scope *CurScope,
       assert(!ContextIsFound &&
              "null type has been found already for '__context' parameter");
       IdentifierInfo *ParamName = &Context.Idents.get("__context");
-      QualType ParamType = Context.getPointerType(Context.getTagDeclType(RD))
-                               .withConst()
-                               .withRestrict();
+      QualType ParamType =
+          Context.getPointerType(Context.getCanonicalTagType(RD))
+              .withConst()
+              .withRestrict();
       auto *Param =
           ImplicitParamDecl::Create(Context, DC, Loc, ParamName, ParamType,
                                     ImplicitParamKind::CapturedContext);
@@ -4684,7 +4682,8 @@ void Sema::ActOnCapturedRegionStart(SourceLocation Loc, Scope *CurScope,
   if (!ContextIsFound) {
     // Add __context implicitly if it is not specified.
     IdentifierInfo *ParamName = &Context.Idents.get("__context");
-    QualType ParamType = Context.getPointerType(Context.getTagDeclType(RD));
+    CanQualType ParamType =
+        Context.getPointerType(Context.getCanonicalTagType(RD));
     auto *Param =
         ImplicitParamDecl::Create(Context, DC, Loc, ParamName, ParamType,
                                   ImplicitParamKind::CapturedContext);
diff --git a/clang/lib/Sema/SemaStmtAsm.cpp b/clang/lib/Sema/SemaStmtAsm.cpp
index 4507a21a4c111..cd8b98c7444eb 100644
--- a/clang/lib/Sema/SemaStmtAsm.cpp
+++ b/clang/lib/Sema/SemaStmtAsm.cpp
@@ -894,7 +894,7 @@ bool Sema::LookupInlineAsmField(StringRef Base, StringRef Member,
         QT = PT->getPointeeType();
       RT = QT->getAs<RecordType>();
     } else if (TypeDecl *TD = dyn_cast<TypeDecl>(FoundDecl))
-      RT = TD->getTypeForDecl()->getAs<RecordType>();
+      RT = Context.getTypeDeclType(TD)->getAs<RecordType>();
     else if (FieldDecl *TD = dyn_cast<FieldDecl>(FoundDecl))
       RT = TD->getType()->getAs<RecordType>();
     if (!RT)
@@ -907,7 +907,8 @@ bool Sema::LookupInlineAsmField(StringRef Base, StringRef Member,
     LookupResult FieldResult(*this, &Context.Idents.get(NextMember),
                              SourceLocation(), LookupMemberName);
 
-    if (!LookupQualifiedName(FieldResult, RT->getDecl()))
+    RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
+    if (!LookupQualifiedName(FieldResult, RD))
       return true;
 
     if (!FieldResult.isSingleResult())
@@ -919,7 +920,7 @@ bool Sema::LookupInlineAsmField(StringRef Base, StringRef Member,
     if (!FD)
       return true;
 
-    const ASTRecordLayout &RL = Context.getASTRecordLayout(RT->getDecl());
+    const ASTRecordLayout &RL = Context.getASTRecordLayout(RD);
     unsigned i = FD->getFieldIndex();
     CharUnits Result = Context.toCharUnitsFromBits(RL.getFieldOffset(i));
     Offset += (unsigned)Result.getQuantity();
@@ -951,7 +952,8 @@ Sema::LookupInlineAsmVarDeclField(Expr *E, StringRef Member,
   LookupResult FieldResult(*this, &Context.Idents.get(Member), AsmLoc,
                            LookupMemberName);
 
-  if (!LookupQualifiedName(FieldResult, RT->getDecl()))
+  if (!LookupQualifiedName(FieldResult,
+                           RT->getOriginalDecl()->getDefinitionOrSelf()))
     return ExprResult();
 
   // Only normal and indirect field results will work.
diff --git a/clang/lib/Sema/SemaSwift.cpp b/clang/lib/Sema/SemaSwift.cpp
index 4000beff7dc49..a99222c5ed55f 100644
--- a/clang/lib/Sema/SemaSwift.cpp
+++ b/clang/lib/Sema/SemaSwift.cpp
@@ -130,7 +130,7 @@ static bool isErrorParameter(Sema &S, QualType QT) {
   // Check for CFError**.
   if (const auto *PT = Pointee->getAs<PointerType>())
     if (const auto *RT = PT->getPointeeType()->getAs<RecordType>())
-      if (S.ObjC().isCFError(RT->getDecl()))
+      if (S.ObjC().isCFError(RT->getOriginalDecl()->getDefinitionOrSelf()))
         return true;
 
   return false;
@@ -272,7 +272,8 @@ static void checkSwiftAsyncErrorBlock(Sema &S, Decl *D,
       // Check for CFError *.
       if (const auto *PtrTy = Param->getAs<PointerType>()) {
         if (const auto *RT = PtrTy->getPointeeType()->getAs<RecordType>()) {
-          if (S.ObjC().isCFError(RT->getDecl())) {
+          if (S.ObjC().isCFError(
+                  RT->getOriginalDecl()->getDefinitionOrSelf())) {
             AnyErrorParams = true;
             break;
           }
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index b6b8932588909..36bffc5e5e3c9 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -19,6 +19,7 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/TemplateName.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/TypeOrdering.h"
 #include "clang/AST/TypeVisitor.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/DiagnosticSema.h"
@@ -293,7 +294,7 @@ TemplateNameKind Sema::isTemplateName(Scope *S,
         FoundUsingShadow ? TemplateName(FoundUsingShadow) : TemplateName(TD);
     assert(!FoundUsingShadow || FoundUsingShadow->getTargetDecl() == TD);
     if (!SS.isInvalid()) {
-      NestedNameSpecifier *Qualifier = SS.getScopeRep();
+      NestedNameSpecifier Qualifier = SS.getScopeRep();
       Template = Context.getQualifiedTemplateName(Qualifier, hasTemplateKeyword,
                                                   Template);
     }
@@ -316,6 +317,12 @@ TemplateNameKind Sema::isTemplateName(Scope *S,
     }
   }
 
+  if (isPackProducingBuiltinTemplateName(Template) &&
+      S->getTemplateParamParent() == nullptr)
+    Diag(Name.getBeginLoc(), diag::err_builtin_pack_outside_template) << TName;
+  // Recover by returning the template, even though we would never be able to
+  // substitute it.
+
   TemplateResult = TemplateTy::make(Template);
   return TemplateKind;
 }
@@ -367,9 +374,8 @@ bool Sema::DiagnoseUnknownTemplateName(const IdentifierInfo &II,
 
   // The code is missing a 'template' keyword prior to the dependent template
   // name.
-  NestedNameSpecifier *Qualifier = (NestedNameSpecifier *)SS->getScopeRep();
   SuggestedTemplate = TemplateTy::make(Context.getDependentTemplateName(
-      {Qualifier, &II, /*HasTemplateKeyword=*/false}));
+      {SS->getScopeRep(), &II, /*HasTemplateKeyword=*/false}));
   Diag(IILoc, diag::err_template_kw_missing)
       << SuggestedTemplate.get()
       << FixItHint::CreateInsertion(IILoc, "template ");
@@ -401,7 +407,9 @@ bool Sema::LookupTemplateName(LookupResult &Found, Scope *S, CXXScopeSpec &SS,
     IsDependent = !LookupCtx && ObjectType->isDependentType();
     assert((IsDependent || !ObjectType->isIncompleteType() ||
             !ObjectType->getAs<TagType>() ||
-            ObjectType->castAs<TagType>()->isBeingDefined()) &&
+            ObjectType->castAs<TagType>()
+                ->getOriginalDecl()
+                ->isEntityBeingDefined()) &&
            "Caller should have completed object type");
 
     // Template names cannot appear inside an Objective-C class or object type
@@ -801,9 +809,9 @@ bool Sema::DiagnoseUninstantiableTemplate(SourceLocation PointOfInstantiation,
   if (!Complain || (PatternDef && PatternDef->isInvalidDecl()))
     return true;
 
-  QualType InstantiationTy;
+  CanQualType InstantiationTy;
   if (TagDecl *TD = dyn_cast<TagDecl>(Instantiation))
-    InstantiationTy = Context.getTypeDeclType(TD);
+    InstantiationTy = Context.getCanonicalTagType(TD);
   if (PatternDef) {
     Diag(PointOfInstantiation,
          diag::err_template_instantiate_within_definition)
@@ -911,12 +919,12 @@ static TemplateArgumentLoc translateTemplateArgument(Sema &SemaRef,
     TypeSourceInfo *DI;
     QualType T = SemaRef.GetTypeFromParser(Arg.getAsType(), &DI);
     if (!DI)
-      DI = SemaRef.Context.getTrivialTypeSourceInfo(T, Arg.getLocation());
+      DI = SemaRef.Context.getTrivialTypeSourceInfo(T, Arg.getNameLoc());
     return TemplateArgumentLoc(TemplateArgument(T), DI);
   }
 
   case ParsedTemplateArgument::NonType: {
-    Expr *E = static_cast<Expr *>(Arg.getAsExpr());
+    Expr *E = Arg.getAsExpr();
     return TemplateArgumentLoc(TemplateArgument(E, /*IsCanonical=*/false), E);
   }
 
@@ -928,9 +936,9 @@ static TemplateArgumentLoc translateTemplateArgument(Sema &SemaRef,
     else
       TArg = Template;
     return TemplateArgumentLoc(
-        SemaRef.Context, TArg,
+        SemaRef.Context, TArg, Arg.getTemplateKwLoc(),
         Arg.getScopeSpec().getWithLocInContext(SemaRef.Context),
-        Arg.getLocation(), Arg.getEllipsisLoc());
+        Arg.getNameLoc(), Arg.getEllipsisLoc());
   }
   }
 
@@ -971,15 +979,12 @@ ParsedTemplateArgument Sema::ActOnTemplateTypeArgument(TypeResult ParsedType) {
       TL = PET.getPatternLoc();
     }
 
-    CXXScopeSpec SS;
-    if (auto ET = TL.getAs<ElaboratedTypeLoc>()) {
-      SS.Adopt(ET.getQualifierLoc());
-      TL = ET.getNamedTypeLoc();
-    }
-
     if (auto DTST = TL.getAs<DeducedTemplateSpecializationTypeLoc>()) {
       TemplateName Name = DTST.getTypePtr()->getTemplateName();
-      ParsedTemplateArgument Result(SS, TemplateTy::make(Name),
+      CXXScopeSpec SS;
+      SS.Adopt(DTST.getQualifierLoc());
+      ParsedTemplateArgument Result(/*TemplateKwLoc=*/SourceLocation(), SS,
+                                    TemplateTy::make(Name),
                                     DTST.getTemplateNameLoc());
       if (EllipsisLoc.isValid())
         Result = Result.getTemplatePackExpansion(EllipsisLoc);
@@ -2080,14 +2085,19 @@ DeclResult Sema::CheckClassTemplate(
         // If we have a prior definition that is not visible, treat this as
         // simply making that previous definition visible.
         NamedDecl *Hidden = nullptr;
-        if (SkipBody && !hasVisibleDefinition(Def, &Hidden)) {
+        bool HiddenDefVisible = false;
+        if (SkipBody &&
+            isRedefinitionAllowedFor(Def, &Hidden, HiddenDefVisible)) {
           SkipBody->ShouldSkip = true;
           SkipBody->Previous = Def;
-          auto *Tmpl = cast<CXXRecordDecl>(Hidden)->getDescribedClassTemplate();
-          assert(Tmpl && "original definition of a class template is not a "
-                         "class template?");
-          makeMergedDefinitionVisible(Hidden);
-          makeMergedDefinitionVisible(Tmpl);
+          if (!HiddenDefVisible && Hidden) {
+            auto *Tmpl =
+                cast<CXXRecordDecl>(Hidden)->getDescribedClassTemplate();
+            assert(Tmpl && "original definition of a class template is not a "
+                           "class template?");
+            makeMergedDefinitionVisible(Hidden);
+            makeMergedDefinitionVisible(Tmpl);
+          }
         } else {
           Diag(NameLoc, diag::err_redefinition) << Name;
           Diag(Def->getLocation(), diag::note_previous_definition);
@@ -2145,11 +2155,11 @@ DeclResult Sema::CheckClassTemplate(
   bool ShouldAddRedecl =
       !(TUK == TagUseKind::Friend && CurContext->isDependentContext());
 
-  CXXRecordDecl *NewClass =
-    CXXRecordDecl::Create(Context, Kind, SemanticContext, KWLoc, NameLoc, Name,
-                          PrevClassTemplate && ShouldAddRedecl ?
-                            PrevClassTemplate->getTemplatedDecl() : nullptr,
-                          /*DelayTypeCreation=*/true);
+  CXXRecordDecl *NewClass = CXXRecordDecl::Create(
+      Context, Kind, SemanticContext, KWLoc, NameLoc, Name,
+      PrevClassTemplate && ShouldAddRedecl
+          ? PrevClassTemplate->getTemplatedDecl()
+          : nullptr);
   SetNestedNameSpecifier(*this, NewClass, SS);
   if (NumOuterTemplateParamLists > 0)
     NewClass->setTemplateParameterListsInfo(
@@ -2178,12 +2188,6 @@ DeclResult Sema::CheckClassTemplate(
   if (ModulePrivateLoc.isValid())
     NewTemplate->setModulePrivate();
 
-  // Build the type for the class template declaration now.
-  QualType T = NewTemplate->getInjectedClassNameSpecialization();
-  T = Context.getInjectedClassNameType(NewClass, T);
-  assert(T->isDependentType() && "Class template type is not dependent?");
-  (void)T;
-
   // If we are providing an explicit specialization of a member that is a
   // class template, make a note of that.
   if (PrevClassTemplate &&
@@ -2674,11 +2678,11 @@ struct DependencyChecker : DynamicRecursiveASTVisitor {
     return DynamicRecursiveASTVisitor::TraverseStmt(S);
   }
 
-  bool TraverseTypeLoc(TypeLoc TL) override {
+  bool TraverseTypeLoc(TypeLoc TL, bool TraverseQualifier = true) override {
     if (IgnoreNonTypeDependent && !TL.isNull() &&
         !TL.getType()->isDependentType())
       return true;
-    return DynamicRecursiveASTVisitor::TraverseTypeLoc(TL);
+    return DynamicRecursiveASTVisitor::TraverseTypeLoc(TL, TraverseQualifier);
   }
 
   bool VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc TL) override {
@@ -2727,8 +2731,12 @@ struct DependencyChecker : DynamicRecursiveASTVisitor {
     return TraverseTemplateArgument(T->getArgumentPack());
   }
 
-  bool TraverseInjectedClassNameType(InjectedClassNameType *T) override {
-    return TraverseType(T->getInjectedSpecializationType());
+  bool TraverseInjectedClassNameType(InjectedClassNameType *T,
+                                     bool TraverseQualifier) override {
+    // An InjectedClassNameType will never have a dependent template name,
+    // so no need to traverse it.
+    return TraverseTemplateArguments(
+        T->getTemplateArgs(T->getOriginalDecl()->getASTContext()));
   }
 };
 } // end anonymous namespace
@@ -2751,14 +2759,14 @@ static SourceRange getRangeOfTypeInNestedNameSpecifier(ASTContext &Context,
                                                        QualType T,
                                                        const CXXScopeSpec &SS) {
   NestedNameSpecifierLoc NNSLoc(SS.getScopeRep(), SS.location_data());
-  while (NestedNameSpecifier *NNS = NNSLoc.getNestedNameSpecifier()) {
-    if (const Type *CurType = NNS->getAsType()) {
-      if (Context.hasSameUnqualifiedType(T, QualType(CurType, 0)))
-        return NNSLoc.getTypeLoc().getSourceRange();
-    } else
+  for (;;) {
+    NestedNameSpecifier NNS = NNSLoc.getNestedNameSpecifier();
+    if (NNS.getKind() != NestedNameSpecifier::Kind::Type)
       break;
-
-    NNSLoc = NNSLoc.getPrefix();
+    if (Context.hasSameUnqualifiedType(T, QualType(NNS.getAsType(), 0)))
+      return NNSLoc.castAsTypeLoc().getSourceRange();
+    // FIXME: This will always be empty.
+    NNSLoc = NNSLoc.getAsNamespaceAndPrefix().Prefix;
   }
 
   return SourceRange();
@@ -2777,12 +2785,13 @@ TemplateParameterList *Sema::MatchTemplateParametersToScopeSpecifier(
   // by the nested-name-specifier and walking out until we run out of types.
   SmallVector<QualType, 4> NestedTypes;
   QualType T;
-  if (SS.getScopeRep()) {
-    if (CXXRecordDecl *Record
-              = dyn_cast_or_null<CXXRecordDecl>(computeDeclContext(SS, true)))
-      T = Context.getTypeDeclType(Record);
+  if (NestedNameSpecifier Qualifier = SS.getScopeRep();
+      Qualifier.getKind() == NestedNameSpecifier::Kind::Type) {
+    if (CXXRecordDecl *Record =
+            dyn_cast_or_null<CXXRecordDecl>(computeDeclContext(SS, true)))
+      T = Context.getCanonicalTagType(Record);
     else
-      T = QualType(SS.getScopeRep()->getAsType(), 0);
+      T = QualType(Qualifier.getAsType(), 0);
   }
 
   // If we found an explicit specialization that prevents us from needing
@@ -2830,9 +2839,10 @@ TemplateParameterList *Sema::MatchTemplateParametersToScopeSpecifier(
     // Look one step prior in a dependent template specialization type.
     if (const DependentTemplateSpecializationType *DependentTST
                           = T->getAs<DependentTemplateSpecializationType>()) {
-      if (NestedNameSpecifier *NNS =
-              DependentTST->getDependentTemplateName().getQualifier())
-        T = QualType(NNS->getAsType(), 0);
+      if (NestedNameSpecifier NNS =
+              DependentTST->getDependentTemplateName().getQualifier();
+          NNS.getKind() == NestedNameSpecifier::Kind::Type)
+        T = QualType(NNS.getAsType(), 0);
       else
         T = QualType();
       continue;
@@ -2840,8 +2850,9 @@ TemplateParameterList *Sema::MatchTemplateParametersToScopeSpecifier(
 
     // Look one step prior in a dependent name type.
     if (const DependentNameType *DependentName = T->getAs<DependentNameType>()){
-      if (NestedNameSpecifier *NNS = DependentName->getQualifier())
-        T = QualType(NNS->getAsType(), 0);
+      if (NestedNameSpecifier NNS = DependentName->getQualifier();
+          NNS.getKind() == NestedNameSpecifier::Kind::Type)
+        T = QualType(NNS.getAsType(), 0);
       else
         T = QualType();
       continue;
@@ -2851,7 +2862,7 @@ TemplateParameterList *Sema::MatchTemplateParametersToScopeSpecifier(
     if (const EnumType *EnumT = T->getAs<EnumType>()) {
       // FIXME: Forward-declared enums require a TSK_ExplicitSpecialization
       // check here.
-      EnumDecl *Enum = EnumT->getDecl();
+      EnumDecl *Enum = EnumT->getOriginalDecl();
 
       // Get to the parent type.
       if (TypeDecl *Parent = dyn_cast<TypeDecl>(Enum->getParent()))
@@ -3151,7 +3162,8 @@ void Sema::NoteAllFoundTemplates(TemplateName Name) {
   }
 }
 
-static QualType builtinCommonTypeImpl(Sema &S, TemplateName BaseTemplate,
+static QualType builtinCommonTypeImpl(Sema &S, ElaboratedTypeKeyword Keyword,
+                                      TemplateName BaseTemplate,
                                       SourceLocation TemplateLoc,
                                       ArrayRef<TemplateArgument> Ts) {
   auto lookUpCommonType = [&](TemplateArgument T1,
@@ -3159,7 +3171,8 @@ static QualType builtinCommonTypeImpl(Sema &S, TemplateName BaseTemplate,
     // Don't bother looking for other specializations if both types are
     // builtins - users aren't allowed to specialize for them
     if (T1.getAsType()->isBuiltinType() && T2.getAsType()->isBuiltinType())
-      return builtinCommonTypeImpl(S, BaseTemplate, TemplateLoc, {T1, T2});
+      return builtinCommonTypeImpl(S, Keyword, BaseTemplate, TemplateLoc,
+                                   {T1, T2});
 
     TemplateArgumentListInfo Args;
     Args.addArgument(TemplateArgumentLoc(
@@ -3173,7 +3186,7 @@ static QualType builtinCommonTypeImpl(Sema &S, TemplateName BaseTemplate,
     Sema::ContextRAII TUContext(S, S.Context.getTranslationUnitDecl());
 
     QualType BaseTemplateInst =
-        S.CheckTemplateIdType(BaseTemplate, TemplateLoc, Args);
+        S.CheckTemplateIdType(Keyword, BaseTemplate, TemplateLoc, Args);
 
     if (SFINAE.hasErrorOccurred())
       return QualType();
@@ -3286,7 +3299,7 @@ static QualType builtinCommonTypeImpl(Sema &S, TemplateName BaseTemplate,
 }
 
 static bool isInVkNamespace(const RecordType *RT) {
-  DeclContext *DC = RT->getDecl()->getDeclContext();
+  DeclContext *DC = RT->getOriginalDecl()->getDeclContext();
   if (!DC)
     return false;
 
@@ -3303,8 +3316,9 @@ static SpirvOperand checkHLSLSpirvTypeOperand(Sema &SemaRef,
   if (auto *RT = OperandArg->getAs<RecordType>()) {
     bool Literal = false;
     SourceLocation LiteralLoc;
-    if (isInVkNamespace(RT) && RT->getDecl()->getName() == "Literal") {
-      auto SpecDecl = dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl());
+    if (isInVkNamespace(RT) && RT->getOriginalDecl()->getName() == "Literal") {
+      auto SpecDecl =
+          dyn_cast<ClassTemplateSpecializationDecl>(RT->getOriginalDecl());
       assert(SpecDecl);
 
       const TemplateArgumentList &LiteralArgs = SpecDecl->getTemplateArgs();
@@ -3315,8 +3329,9 @@ static SpirvOperand checkHLSLSpirvTypeOperand(Sema &SemaRef,
     }
 
     if (RT && isInVkNamespace(RT) &&
-        RT->getDecl()->getName() == "integral_constant") {
-      auto SpecDecl = dyn_cast<ClassTemplateSpecializationDecl>(RT->getDecl());
+        RT->getOriginalDecl()->getName() == "integral_constant") {
+      auto SpecDecl =
+          dyn_cast<ClassTemplateSpecializationDecl>(RT->getOriginalDecl());
       assert(SpecDecl);
 
       const TemplateArgumentList &ConstantArgs = SpecDecl->getTemplateArgs();
@@ -3338,11 +3353,10 @@ static SpirvOperand checkHLSLSpirvTypeOperand(Sema &SemaRef,
   return SpirvOperand::createType(OperandArg);
 }
 
-static QualType
-checkBuiltinTemplateIdType(Sema &SemaRef, BuiltinTemplateDecl *BTD,
-                           ArrayRef<TemplateArgument> Converted,
-                           SourceLocation TemplateLoc,
-                           TemplateArgumentListInfo &TemplateArgs) {
+static QualType checkBuiltinTemplateIdType(
+    Sema &SemaRef, ElaboratedTypeKeyword Keyword, BuiltinTemplateDecl *BTD,
+    ArrayRef<TemplateArgument> Converted, SourceLocation TemplateLoc,
+    TemplateArgumentListInfo &TemplateArgs) {
   ASTContext &Context = SemaRef.getASTContext();
 
   switch (BTD->getBuiltinTemplateKind()) {
@@ -3389,7 +3403,7 @@ checkBuiltinTemplateIdType(Sema &SemaRef, BuiltinTemplateDecl *BTD,
 
     // The first template argument will be reused as the template decl that
     // our synthetic template arguments will be applied to.
-    return SemaRef.CheckTemplateIdType(Converted[0].getAsTemplate(),
+    return SemaRef.CheckTemplateIdType(Keyword, Converted[0].getAsTemplate(),
                                        TemplateLoc, SyntheticTemplateArgs);
   }
 
@@ -3426,14 +3440,16 @@ checkBuiltinTemplateIdType(Sema &SemaRef, BuiltinTemplateDecl *BTD,
 
     TemplateName BaseTemplate = Converted[0].getAsTemplate();
     ArrayRef<TemplateArgument> Ts = Converted[3].getPackAsArray();
-    if (auto CT = builtinCommonTypeImpl(SemaRef, BaseTemplate, TemplateLoc, Ts);
+    if (auto CT = builtinCommonTypeImpl(SemaRef, Keyword, BaseTemplate,
+                                        TemplateLoc, Ts);
         !CT.isNull()) {
       TemplateArgumentListInfo TAs;
       TAs.addArgument(TemplateArgumentLoc(
           TemplateArgument(CT), SemaRef.Context.getTrivialTypeSourceInfo(
                                     CT, TemplateArgs[1].getLocation())));
       TemplateName HasTypeMember = Converted[1].getAsTemplate();
-      return SemaRef.CheckTemplateIdType(HasTypeMember, TemplateLoc, TAs);
+      return SemaRef.CheckTemplateIdType(Keyword, HasTypeMember, TemplateLoc,
+                                         TAs);
     }
     QualType HasNoTypeMember = Converted[2].getAsType();
     return HasNoTypeMember;
@@ -3468,6 +3484,28 @@ checkBuiltinTemplateIdType(Sema &SemaRef, BuiltinTemplateDecl *BTD,
 
     return Context.getHLSLInlineSpirvType(Opcode, Size, Alignment, Operands);
   }
+  case BTK__builtin_dedup_pack: {
+    assert(Converted.size() == 1 && "__builtin_dedup_pack should be given "
+                                    "a parameter pack");
+    TemplateArgument Ts = Converted[0];
+    // Delay the computation until we can compute the final result. We choose
+    // not to remove the duplicates upfront before substitution to keep the code
+    // simple.
+    if (Ts.isDependent())
+      return QualType();
+    assert(Ts.getKind() == clang::TemplateArgument::Pack);
+    llvm::SmallVector<TemplateArgument> OutArgs;
+    llvm::SmallDenseSet<QualType> Seen;
+    // Synthesize a new template argument list, removing duplicates.
+    for (auto T : Ts.getPackAsArray()) {
+      assert(T.getKind() == clang::TemplateArgument::Type);
+      if (!Seen.insert(T.getAsType().getCanonicalType()).second)
+        continue;
+      OutArgs.push_back(T);
+    }
+    return Context.getSubstBuiltinTemplatePack(
+        TemplateArgument::CreatePackCopy(Context, OutArgs));
+  }
   }
   llvm_unreachable("unexpected BuiltinTemplateDecl!");
 }
@@ -3545,7 +3583,7 @@ class FailedBooleanConditionPrinterHelper : public PrinterHelper {
     if (DR && DR->getQualifier()) {
       // If this is a qualified name, expand the template arguments in nested
       // qualifiers.
-      DR->getQualifier()->print(OS, Policy, true);
+      DR->getQualifier().print(OS, Policy, true);
       // Then print the decl itself.
       const ValueDecl *VD = DR->getDecl();
       OS << VD->getName();
@@ -3610,7 +3648,8 @@ Sema::findFailedBooleanCondition(Expr *Cond) {
   return { FailedCond, Description };
 }
 
-QualType Sema::CheckTemplateIdType(TemplateName Name,
+QualType Sema::CheckTemplateIdType(ElaboratedTypeKeyword Keyword,
+                                   TemplateName Name,
                                    SourceLocation TemplateLoc,
                                    TemplateArgumentListInfo &TemplateArgs) {
   // FIXME: 'getUnderlying' loses SubstTemplateTemplateParm nodes from alias
@@ -3661,6 +3700,18 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
   } else if (TypeAliasTemplateDecl *AliasTemplate =
                  dyn_cast<TypeAliasTemplateDecl>(Template)) {
 
+    // C++0x [dcl.type.elab]p2:
+    //   If the identifier resolves to a typedef-name or the simple-template-id
+    //   resolves to an alias template specialization, the
+    //   elaborated-type-specifier is ill-formed.
+    if (Keyword != ElaboratedTypeKeyword::None &&
+        Keyword != ElaboratedTypeKeyword::Typename) {
+      SemaRef.Diag(TemplateLoc, diag::err_tag_reference_non_tag)
+          << AliasTemplate << NonTagKind::TypeAliasTemplate
+          << KeywordHelpers::getTagTypeKindForKeyword(Keyword);
+      SemaRef.Diag(AliasTemplate->getLocation(), diag::note_declared_at);
+    }
+
     // Find the canonical type for this type alias template specialization.
     TypeAliasDecl *Pattern = AliasTemplate->getTemplatedDecl();
     if (Pattern->isInvalidDecl())
@@ -3728,8 +3779,8 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
       return QualType();
     }
   } else if (auto *BTD = dyn_cast<BuiltinTemplateDecl>(Template)) {
-    CanonType = checkBuiltinTemplateIdType(*this, BTD, CTAI.SugaredConverted,
-                                           TemplateLoc, TemplateArgs);
+    CanonType = checkBuiltinTemplateIdType(
+        *this, Keyword, BTD, CTAI.SugaredConverted, TemplateLoc, TemplateArgs);
   } else if (Name.isDependent() ||
              TemplateSpecializationType::anyDependentTemplateArguments(
                  TemplateArgs, CTAI.CanonicalConverted)) {
@@ -3768,16 +3819,15 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
 
         // Fetch the injected class name type and check whether its
         // injected type is equal to the type we just built.
-        QualType ICNT = Context.getTypeDeclType(Record);
-        QualType Injected = cast<InjectedClassNameType>(ICNT)
-          ->getInjectedSpecializationType();
+        CanQualType ICNT = Context.getCanonicalTagType(Record);
+        CanQualType Injected =
+            Record->getCanonicalTemplateSpecializationType(Context);
 
-        if (CanonType != Injected->getCanonicalTypeInternal())
+        if (CanonType != Injected)
           continue;
 
         // If so, the canonical type of this TST is the injected
         // class name type of the record we just found.
-        assert(ICNT.isCanonical());
         CanonType = ICNT;
         break;
       }
@@ -3819,7 +3869,7 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
     // Diagnose uses of this specialization.
     (void)DiagnoseUseOfDecl(Decl, TemplateLoc);
 
-    CanonType = Context.getTypeDeclType(Decl);
+    CanonType = Context.getCanonicalTagType(Decl);
     assert(isa<RecordType>(CanonType) &&
            "type of non-dependent specialization is not a RecordType");
   } else {
@@ -3830,7 +3880,8 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
   // specialization, which refers back to the class template
   // specialization we created or found.
   return Context.getTemplateSpecializationType(
-      Name, TemplateArgs.arguments(), CTAI.CanonicalConverted, CanonType);
+      Keyword, Name, TemplateArgs.arguments(), CTAI.CanonicalConverted,
+      CanonType);
 }
 
 void Sema::ActOnUndeclaredTypeTemplateName(Scope *S, TemplateTy &ParsedName,
@@ -3878,7 +3929,7 @@ bool Sema::resolveAssumedTemplateNameAsType(Scope *S, TemplateName &Name,
     diagnoseTypo(Corrected, PDiag(diag::err_no_template_suggest)
                                 << ATN->getDeclName());
     Name = Context.getQualifiedTemplateName(
-        /*NNS=*/nullptr, /*TemplateKeyword=*/false,
+        /*Qualifier=*/std::nullopt, /*TemplateKeyword=*/false,
         TemplateName(Corrected.getCorrectionDeclAs<TemplateDecl>()));
     return false;
   }
@@ -3889,11 +3940,12 @@ bool Sema::resolveAssumedTemplateNameAsType(Scope *S, TemplateName &Name,
 }
 
 TypeResult Sema::ActOnTemplateIdType(
-    Scope *S, CXXScopeSpec &SS, SourceLocation TemplateKWLoc,
-    TemplateTy TemplateD, const IdentifierInfo *TemplateII,
-    SourceLocation TemplateIILoc, SourceLocation LAngleLoc,
-    ASTTemplateArgsPtr TemplateArgsIn, SourceLocation RAngleLoc,
-    bool IsCtorOrDtorName, bool IsClassName,
+    Scope *S, ElaboratedTypeKeyword ElaboratedKeyword,
+    SourceLocation ElaboratedKeywordLoc, CXXScopeSpec &SS,
+    SourceLocation TemplateKWLoc, TemplateTy TemplateD,
+    const IdentifierInfo *TemplateII, SourceLocation TemplateIILoc,
+    SourceLocation LAngleLoc, ASTTemplateArgsPtr TemplateArgsIn,
+    SourceLocation RAngleLoc, bool IsCtorOrDtorName, bool IsClassName,
     ImplicitTypenameContext AllowImplicitTypename) {
   if (SS.isInvalid())
     return true;
@@ -3909,8 +3961,9 @@ TypeResult Sema::ActOnTemplateIdType(
     //   elaborated-type-specifier (7.1.5.3).
     if (!LookupCtx && isDependentScopeSpecifier(SS)) {
       // C++2a relaxes some of those restrictions in [temp.res]p5.
-      NestedNameSpecifier *NNS =
-          NestedNameSpecifier::Create(Context, SS.getScopeRep(), TemplateII);
+      QualType DNT = Context.getDependentNameType(ElaboratedTypeKeyword::None,
+                                                  SS.getScopeRep(), TemplateII);
+      NestedNameSpecifier NNS(DNT.getTypePtr());
       if (AllowImplicitTypename == ImplicitTypenameContext::Yes) {
         auto DB = DiagCompat(SS.getBeginLoc(), diag_compat::implicit_typename)
                   << NNS;
@@ -3954,12 +4007,12 @@ TypeResult Sema::ActOnTemplateIdType(
   if (DependentTemplateName *DTN = Template.getAsDependentTemplateName()) {
     assert(SS.getScopeRep() == DTN->getQualifier());
     QualType T = Context.getDependentTemplateSpecializationType(
-        ElaboratedTypeKeyword::None, *DTN, TemplateArgs.arguments());
+        ElaboratedKeyword, *DTN, TemplateArgs.arguments());
     // Build type-source information.
     TypeLocBuilder TLB;
     DependentTemplateSpecializationTypeLoc SpecTL
       = TLB.push<DependentTemplateSpecializationTypeLoc>(T);
-    SpecTL.setElaboratedKeywordLoc(SourceLocation());
+    SpecTL.setElaboratedKeywordLoc(ElaboratedKeywordLoc);
     SpecTL.setQualifierLoc(SS.getWithLocInContext(Context));
     SpecTL.setTemplateKeywordLoc(TemplateKWLoc);
     SpecTL.setTemplateNameLoc(TemplateIILoc);
@@ -3970,30 +4023,17 @@ TypeResult Sema::ActOnTemplateIdType(
     return CreateParsedType(T, TLB.getTypeSourceInfo(Context, T));
   }
 
-  QualType SpecTy = CheckTemplateIdType(Template, TemplateIILoc, TemplateArgs);
+  QualType SpecTy = CheckTemplateIdType(ElaboratedKeyword, Template,
+                                        TemplateIILoc, TemplateArgs);
   if (SpecTy.isNull())
     return true;
 
   // Build type-source information.
   TypeLocBuilder TLB;
-  TemplateSpecializationTypeLoc SpecTL =
-      TLB.push<TemplateSpecializationTypeLoc>(SpecTy);
-  SpecTL.setTemplateKeywordLoc(TemplateKWLoc);
-  SpecTL.setTemplateNameLoc(TemplateIILoc);
-  SpecTL.setLAngleLoc(LAngleLoc);
-  SpecTL.setRAngleLoc(RAngleLoc);
-  for (unsigned i = 0, e = SpecTL.getNumArgs(); i != e; ++i)
-    SpecTL.setArgLocInfo(i, TemplateArgs[i].getLocInfo());
-
-  // Create an elaborated-type-specifier containing the nested-name-specifier.
-  QualType ElTy =
-      getElaboratedType(ElaboratedTypeKeyword::None,
-                        !IsCtorOrDtorName ? SS : CXXScopeSpec(), SpecTy);
-  ElaboratedTypeLoc ElabTL = TLB.push<ElaboratedTypeLoc>(ElTy);
-  ElabTL.setElaboratedKeywordLoc(SourceLocation());
-  if (!ElabTL.isEmpty())
-    ElabTL.setQualifierLoc(SS.getWithLocInContext(Context));
-  return CreateParsedType(ElTy, TLB.getTypeSourceInfo(Context, ElTy));
+  TLB.push<TemplateSpecializationTypeLoc>(SpecTy).set(
+      ElaboratedKeywordLoc, SS.getWithLocInContext(Context), TemplateKWLoc,
+      TemplateIILoc, TemplateArgs);
+  return CreateParsedType(SpecTy, TLB.getTypeSourceInfo(Context, SpecTy));
 }
 
 TypeResult Sema::ActOnTagTemplateIdType(TagUseKind TUK,
@@ -4040,24 +4080,14 @@ TypeResult Sema::ActOnTagTemplateIdType(TagUseKind TUK,
     return CreateParsedType(T, TLB.getTypeSourceInfo(Context, T));
   }
 
-  if (TypeAliasTemplateDecl *TAT =
-        dyn_cast_or_null<TypeAliasTemplateDecl>(Template.getAsTemplateDecl())) {
-    // C++0x [dcl.type.elab]p2:
-    //   If the identifier resolves to a typedef-name or the simple-template-id
-    //   resolves to an alias template specialization, the
-    //   elaborated-type-specifier is ill-formed.
-    Diag(TemplateLoc, diag::err_tag_reference_non_tag)
-        << TAT << NonTagKind::TypeAliasTemplate << TagKind;
-    Diag(TAT->getLocation(), diag::note_declared_at);
-  }
-
-  QualType Result = CheckTemplateIdType(Template, TemplateLoc, TemplateArgs);
+  QualType Result =
+      CheckTemplateIdType(Keyword, Template, TemplateLoc, TemplateArgs);
   if (Result.isNull())
     return TypeResult(true);
 
   // Check the tag kind
   if (const RecordType *RT = Result->getAs<RecordType>()) {
-    RecordDecl *D = RT->getDecl();
+    RecordDecl *D = RT->getOriginalDecl()->getDefinitionOrSelf();
 
     IdentifierInfo *Id = D->getIdentifier();
     assert(Id && "templated class must have an identifier");
@@ -4073,21 +4103,9 @@ TypeResult Sema::ActOnTagTemplateIdType(TagUseKind TUK,
 
   // Provide source-location information for the template specialization.
   TypeLocBuilder TLB;
-  TemplateSpecializationTypeLoc SpecTL
-    = TLB.push<TemplateSpecializationTypeLoc>(Result);
-  SpecTL.setTemplateKeywordLoc(TemplateKWLoc);
-  SpecTL.setTemplateNameLoc(TemplateLoc);
-  SpecTL.setLAngleLoc(LAngleLoc);
-  SpecTL.setRAngleLoc(RAngleLoc);
-  for (unsigned i = 0, e = SpecTL.getNumArgs(); i != e; ++i)
-    SpecTL.setArgLocInfo(i, TemplateArgs[i].getLocInfo());
-
-  // Construct an elaborated type containing the nested-name-specifier (if any)
-  // and tag keyword.
-  Result = Context.getElaboratedType(Keyword, SS.getScopeRep(), Result);
-  ElaboratedTypeLoc ElabTL = TLB.push<ElaboratedTypeLoc>(Result);
-  ElabTL.setElaboratedKeywordLoc(TagLoc);
-  ElabTL.setQualifierLoc(SS.getWithLocInContext(Context));
+  TLB.push<TemplateSpecializationTypeLoc>(Result).set(
+      TagLoc, SS.getWithLocInContext(Context), TemplateKWLoc, TemplateLoc,
+      TemplateArgs);
   return CreateParsedType(Result, TLB.getTypeSourceInfo(Context, Result));
 }
 
@@ -4099,7 +4117,6 @@ static bool CheckTemplateSpecializationScope(Sema &S, NamedDecl *Specialized,
 static TemplateSpecializationKind getTemplateSpecializationKind(Decl *D);
 
 static bool isTemplateArgumentTemplateParameter(const TemplateArgument &Arg,
-                                                NamedDecl *Param,
                                                 unsigned Depth,
                                                 unsigned Index) {
   switch (Arg.getKind()) {
@@ -4139,8 +4156,9 @@ static bool isTemplateArgumentTemplateParameter(const TemplateArgument &Arg,
 }
 
 static bool isSameAsPrimaryTemplate(TemplateParameterList *Params,
+                                    TemplateParameterList *SpecParams,
                                     ArrayRef<TemplateArgument> Args) {
-  if (Params->size() != Args.size())
+  if (Params->size() != Args.size() || Params->size() != SpecParams->size())
     return false;
 
   unsigned Depth = Params->getDepth();
@@ -4157,9 +4175,19 @@ static bool isSameAsPrimaryTemplate(TemplateParameterList *Params,
       Arg = Arg.pack_begin()->getPackExpansionPattern();
     }
 
-    if (!isTemplateArgumentTemplateParameter(Arg, Params->getParam(I), Depth,
-                                             I))
+    if (!isTemplateArgumentTemplateParameter(Arg, Depth, I))
       return false;
+
+    // For NTTPs further specialization is allowed via deduced types, so
+    // we need to make sure to only reject here if primary template and
+    // specialization use the same type for the NTTP.
+    if (auto *SpecNTTP =
+            dyn_cast<NonTypeTemplateParmDecl>(SpecParams->getParam(I))) {
+      auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(Params->getParam(I));
+      if (!NTTP || NTTP->getType().getCanonicalType() !=
+                       SpecNTTP->getType().getCanonicalType())
+        return false;
+    }
   }
 
   return true;
@@ -4357,7 +4385,7 @@ DeclResult Sema::ActOnVarTemplateSpecialization(
     }
 
     if (isSameAsPrimaryTemplate(VarTemplate->getTemplateParameters(),
-                                CTAI.CanonicalConverted) &&
+                                TemplateParams, CTAI.CanonicalConverted) &&
         (!Context.getLangOpts().CPlusPlus20 ||
          !TemplateParams->hasAssociatedConstraints())) {
       // C++ [temp.class.spec]p9b3:
@@ -5055,7 +5083,7 @@ TemplateNameKind Sema::ActOnTemplateName(Scope *S,
     return TNK_Non_template;
   }
 
-  NestedNameSpecifier *Qualifier = SS.getScopeRep();
+  NestedNameSpecifier Qualifier = SS.getScopeRep();
 
   switch (Name.getKind()) {
   case UnqualifiedIdKind::IK_Identifier:
@@ -5343,8 +5371,9 @@ static bool SubstDefaultTemplateArgument(
 ///
 /// \returns the substituted template argument, or NULL if an error occurred.
 static TemplateName SubstDefaultTemplateArgument(
-    Sema &SemaRef, TemplateDecl *Template, SourceLocation TemplateLoc,
-    SourceLocation RAngleLoc, TemplateTemplateParmDecl *Param,
+    Sema &SemaRef, TemplateDecl *Template, SourceLocation TemplateKWLoc,
+    SourceLocation TemplateLoc, SourceLocation RAngleLoc,
+    TemplateTemplateParmDecl *Param,
     ArrayRef<TemplateArgument> SugaredConverted,
     ArrayRef<TemplateArgument> CanonicalConverted,
     NestedNameSpecifierLoc &QualifierLoc) {
@@ -5361,25 +5390,17 @@ static TemplateName SubstDefaultTemplateArgument(
     TemplateArgLists.addOuterTemplateArguments(std::nullopt);
 
   Sema::ContextRAII SavedContext(SemaRef, Template->getDeclContext());
-  // Substitute into the nested-name-specifier first,
-  QualifierLoc = Param->getDefaultArgument().getTemplateQualifierLoc();
-  if (QualifierLoc) {
-    QualifierLoc =
-        SemaRef.SubstNestedNameSpecifierLoc(QualifierLoc, TemplateArgLists);
-    if (!QualifierLoc)
-      return TemplateName();
-  }
 
-  return SemaRef.SubstTemplateName(
-             QualifierLoc,
-             Param->getDefaultArgument().getArgument().getAsTemplate(),
-             Param->getDefaultArgument().getTemplateNameLoc(),
-             TemplateArgLists);
+  const TemplateArgumentLoc &A = Param->getDefaultArgument();
+  QualifierLoc = A.getTemplateQualifierLoc();
+  return SemaRef.SubstTemplateName(TemplateKWLoc, QualifierLoc,
+                                   A.getArgument().getAsTemplate(),
+                                   A.getTemplateNameLoc(), TemplateArgLists);
 }
 
 TemplateArgumentLoc Sema::SubstDefaultTemplateArgumentIfAvailable(
-    TemplateDecl *Template, SourceLocation TemplateLoc,
-    SourceLocation RAngleLoc, Decl *Param,
+    TemplateDecl *Template, SourceLocation TemplateKWLoc,
+    SourceLocation TemplateNameLoc, SourceLocation RAngleLoc, Decl *Param,
     ArrayRef<TemplateArgument> SugaredConverted,
     ArrayRef<TemplateArgument> CanonicalConverted, bool &HasDefaultArg) {
   HasDefaultArg = false;
@@ -5390,8 +5411,8 @@ TemplateArgumentLoc Sema::SubstDefaultTemplateArgumentIfAvailable(
 
     HasDefaultArg = true;
     TemplateArgumentLoc Output;
-    if (SubstDefaultTemplateArgument(*this, Template, TemplateLoc, RAngleLoc,
-                                     TypeParm, SugaredConverted,
+    if (SubstDefaultTemplateArgument(*this, Template, TemplateNameLoc,
+                                     RAngleLoc, TypeParm, SugaredConverted,
                                      CanonicalConverted, Output))
       return TemplateArgumentLoc();
     return Output;
@@ -5404,8 +5425,8 @@ TemplateArgumentLoc Sema::SubstDefaultTemplateArgumentIfAvailable(
 
     HasDefaultArg = true;
     TemplateArgumentLoc Output;
-    if (SubstDefaultTemplateArgument(*this, Template, TemplateLoc, RAngleLoc,
-                                     NonTypeParm, SugaredConverted,
+    if (SubstDefaultTemplateArgument(*this, Template, TemplateNameLoc,
+                                     RAngleLoc, NonTypeParm, SugaredConverted,
                                      CanonicalConverted, Output))
       return TemplateArgumentLoc();
     return Output;
@@ -5417,17 +5438,16 @@ TemplateArgumentLoc Sema::SubstDefaultTemplateArgumentIfAvailable(
     return TemplateArgumentLoc();
 
   HasDefaultArg = true;
+  const TemplateArgumentLoc &A = TempTempParm->getDefaultArgument();
   NestedNameSpecifierLoc QualifierLoc;
   TemplateName TName = SubstDefaultTemplateArgument(
-      *this, Template, TemplateLoc, RAngleLoc, TempTempParm, SugaredConverted,
-      CanonicalConverted, QualifierLoc);
+      *this, Template, TemplateKWLoc, TemplateNameLoc, RAngleLoc, TempTempParm,
+      SugaredConverted, CanonicalConverted, QualifierLoc);
   if (TName.isNull())
     return TemplateArgumentLoc();
 
-  return TemplateArgumentLoc(
-      Context, TemplateArgument(TName),
-      TempTempParm->getDefaultArgument().getTemplateQualifierLoc(),
-      TempTempParm->getDefaultArgument().getTemplateNameLoc());
+  return TemplateArgumentLoc(Context, TemplateArgument(TName), TemplateKWLoc,
+                             QualifierLoc, A.getTemplateNameLoc());
 }
 
 /// Convert a template-argument that we parsed as a type into a template, if
@@ -5435,33 +5455,24 @@ TemplateArgumentLoc Sema::SubstDefaultTemplateArgumentIfAvailable(
 /// template template arguments and as template type arguments.
 static TemplateArgumentLoc
 convertTypeTemplateArgumentToTemplate(ASTContext &Context, TypeLoc TLoc) {
-  // Extract and step over any surrounding nested-name-specifier.
-  NestedNameSpecifierLoc QualLoc;
-  if (auto ETLoc = TLoc.getAs<ElaboratedTypeLoc>()) {
-    if (ETLoc.getTypePtr()->getKeyword() != ElaboratedTypeKeyword::None)
-      return TemplateArgumentLoc();
+  auto TagLoc = TLoc.getAs<TagTypeLoc>();
+  if (!TagLoc)
+    return TemplateArgumentLoc();
 
-    QualLoc = ETLoc.getQualifierLoc();
-    TLoc = ETLoc.getNamedTypeLoc();
-  }
   // If this type was written as an injected-class-name, it can be used as a
   // template template argument.
-  if (auto InjLoc = TLoc.getAs<InjectedClassNameTypeLoc>())
-    return TemplateArgumentLoc(Context, InjLoc.getTypePtr()->getTemplateName(),
-                               QualLoc, InjLoc.getNameLoc());
-
   // If this type was written as an injected-class-name, it may have been
   // converted to a RecordType during instantiation. If the RecordType is
   // *not* wrapped in a TemplateSpecializationType and denotes a class
   // template specialization, it must have come from an injected-class-name.
-  if (auto RecLoc = TLoc.getAs<RecordTypeLoc>())
-    if (auto *CTSD =
-            dyn_cast<ClassTemplateSpecializationDecl>(RecLoc.getDecl()))
-      return TemplateArgumentLoc(Context,
-                                 TemplateName(CTSD->getSpecializedTemplate()),
-                                 QualLoc, RecLoc.getNameLoc());
 
-  return TemplateArgumentLoc();
+  TemplateName Name = TagLoc.getTypePtr()->getTemplateName(Context);
+  if (Name.isNull())
+    return TemplateArgumentLoc();
+
+  return TemplateArgumentLoc(Context, Name,
+                             /*TemplateKWLoc=*/SourceLocation(),
+                             TagLoc.getQualifierLoc(), TagLoc.getNameLoc());
 }
 
 bool Sema::CheckTemplateArgument(NamedDecl *Param, TemplateArgumentLoc &ArgLoc,
@@ -5857,6 +5868,29 @@ bool Sema::CheckTemplateArgumentList(
       }
     }
 
+    // Check for builtins producing template packs in this context, we do not
+    // support them yet.
+    if (const NonTypeTemplateParmDecl *NTTP =
+            dyn_cast<NonTypeTemplateParmDecl>(*Param);
+        NTTP && NTTP->isPackExpansion()) {
+      auto TL = NTTP->getTypeSourceInfo()
+                    ->getTypeLoc()
+                    .castAs<PackExpansionTypeLoc>();
+      llvm::SmallVector<UnexpandedParameterPack> Unexpanded;
+      collectUnexpandedParameterPacks(TL.getPatternLoc(), Unexpanded);
+      for (const auto &UPP : Unexpanded) {
+        auto *TST = UPP.first.dyn_cast<const TemplateSpecializationType *>();
+        if (!TST)
+          continue;
+        assert(isPackProducingBuiltinTemplateName(TST->getTemplateName()));
+        // Expanding a built-in pack in this context is not yet supported.
+        Diag(TL.getEllipsisLoc(),
+             diag::err_unsupported_builtin_template_pack_expansion)
+            << TST->getTemplateName();
+        return true;
+      }
+    }
+
     if (ArgIdx < NumArgs) {
       TemplateArgumentLoc &ArgLoc = NewArgs[ArgIdx];
       bool NonPackParameter =
@@ -6007,8 +6041,8 @@ bool Sema::CheckTemplateArgumentList(
     // (when the template parameter was part of a nested template) into
     // the default argument.
     TemplateArgumentLoc Arg = SubstDefaultTemplateArgumentIfAvailable(
-        Template, TemplateLoc, RAngleLoc, *Param, CTAI.SugaredConverted,
-        CTAI.CanonicalConverted, HasDefaultArg);
+        Template, /*TemplateKWLoc=*/SourceLocation(), TemplateLoc, RAngleLoc,
+        *Param, CTAI.SugaredConverted, CTAI.CanonicalConverted, HasDefaultArg);
 
     if (Arg.getArgument().isNull()) {
       if (!HasDefaultArg) {
@@ -6152,7 +6186,7 @@ namespace {
 #include "clang/AST/TypeNodes.inc"
 
     bool VisitTagDecl(const TagDecl *Tag);
-    bool VisitNestedNameSpecifier(NestedNameSpecifier *NNS);
+    bool VisitNestedNameSpecifier(NestedNameSpecifier NNS);
   };
 } // end anonymous namespace
 
@@ -6297,11 +6331,11 @@ bool UnnamedLocalNoLinkageFinder::VisitDeducedTemplateSpecializationType(
 }
 
 bool UnnamedLocalNoLinkageFinder::VisitRecordType(const RecordType* T) {
-  return VisitTagDecl(T->getDecl());
+  return VisitTagDecl(T->getOriginalDecl()->getDefinitionOrSelf());
 }
 
 bool UnnamedLocalNoLinkageFinder::VisitEnumType(const EnumType* T) {
-  return VisitTagDecl(T->getDecl());
+  return VisitTagDecl(T->getOriginalDecl()->getDefinitionOrSelf());
 }
 
 bool UnnamedLocalNoLinkageFinder::VisitTemplateTypeParmType(
@@ -6314,6 +6348,11 @@ bool UnnamedLocalNoLinkageFinder::VisitSubstTemplateTypeParmPackType(
   return false;
 }
 
+bool UnnamedLocalNoLinkageFinder::VisitSubstBuiltinTemplatePackType(
+    const SubstBuiltinTemplatePackType *) {
+  return false;
+}
+
 bool UnnamedLocalNoLinkageFinder::VisitTemplateSpecializationType(
                                             const TemplateSpecializationType*) {
   return false;
@@ -6321,7 +6360,7 @@ bool UnnamedLocalNoLinkageFinder::VisitTemplateSpecializationType(
 
 bool UnnamedLocalNoLinkageFinder::VisitInjectedClassNameType(
                                               const InjectedClassNameType* T) {
-  return VisitTagDecl(T->getDecl());
+  return VisitTagDecl(T->getOriginalDecl()->getDefinitionOrSelf());
 }
 
 bool UnnamedLocalNoLinkageFinder::VisitDependentNameType(
@@ -6331,10 +6370,7 @@ bool UnnamedLocalNoLinkageFinder::VisitDependentNameType(
 
 bool UnnamedLocalNoLinkageFinder::VisitDependentTemplateSpecializationType(
                                  const DependentTemplateSpecializationType* T) {
-  if (auto *Q = T->getDependentTemplateName().getQualifier())
-    return VisitNestedNameSpecifier(Q);
-
-  return false;
+  return VisitNestedNameSpecifier(T->getDependentTemplateName().getQualifier());
 }
 
 bool UnnamedLocalNoLinkageFinder::VisitPackExpansionType(
@@ -6380,11 +6416,10 @@ bool UnnamedLocalNoLinkageFinder::VisitDependentBitIntType(
 
 bool UnnamedLocalNoLinkageFinder::VisitTagDecl(const TagDecl *Tag) {
   if (Tag->getDeclContext()->isFunctionOrMethod()) {
-    S.Diag(SR.getBegin(),
-           S.getLangOpts().CPlusPlus11 ?
-             diag::warn_cxx98_compat_template_arg_local_type :
-             diag::ext_template_arg_local_type)
-      << S.Context.getTypeDeclType(Tag) << SR;
+    S.Diag(SR.getBegin(), S.getLangOpts().CPlusPlus11
+                              ? diag::warn_cxx98_compat_template_arg_local_type
+                              : diag::ext_template_arg_local_type)
+        << S.Context.getCanonicalTagType(Tag) << SR;
     return true;
   }
 
@@ -6401,20 +6436,15 @@ bool UnnamedLocalNoLinkageFinder::VisitTagDecl(const TagDecl *Tag) {
 }
 
 bool UnnamedLocalNoLinkageFinder::VisitNestedNameSpecifier(
-                                                    NestedNameSpecifier *NNS) {
-  assert(NNS);
-  if (NNS->getPrefix() && VisitNestedNameSpecifier(NNS->getPrefix()))
-    return true;
-
-  switch (NNS->getKind()) {
-  case NestedNameSpecifier::Identifier:
-  case NestedNameSpecifier::Namespace:
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Super:
+    NestedNameSpecifier NNS) {
+  switch (NNS.getKind()) {
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Namespace:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
     return false;
-
-  case NestedNameSpecifier::TypeSpec:
-    return Visit(QualType(NNS->getAsType(), 0));
+  case NestedNameSpecifier::Kind::Type:
+    return Visit(QualType(NNS.getAsType(), 0));
   }
   llvm_unreachable("Invalid NestedNameSpecifier::Kind!");
 }
@@ -7382,7 +7412,8 @@ ExprResult Sema::CheckTemplateArgument(NamedDecl *Param, QualType ParamType,
       // that case, this may extend the argument from 1 bit to 8 bits.
       QualType IntegerType = ParamType;
       if (const EnumType *Enum = IntegerType->getAs<EnumType>())
-        IntegerType = Enum->getDecl()->getIntegerType();
+        IntegerType =
+            Enum->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
       Value = Value.extOrTrunc(IntegerType->isBitIntType()
                                    ? Context.getIntWidth(IntegerType)
                                    : Context.getTypeSize(IntegerType));
@@ -7480,7 +7511,8 @@ ExprResult Sema::CheckTemplateArgument(NamedDecl *Param, QualType ParamType,
 
     QualType IntegerType = ParamType;
     if (const EnumType *Enum = IntegerType->getAs<EnumType>()) {
-      IntegerType = Enum->getDecl()->getIntegerType();
+      IntegerType =
+          Enum->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
     }
 
     if (ParamType->isBooleanType()) {
@@ -7903,10 +7935,9 @@ ExprResult Sema::BuildExpressionFromDeclTemplateArgument(
     assert(VD->getDeclContext()->isRecord() &&
            (isa<CXXMethodDecl>(VD) || isa<FieldDecl>(VD) ||
             isa<IndirectFieldDecl>(VD)));
-    QualType ClassType
-      = Context.getTypeDeclType(cast<RecordDecl>(VD->getDeclContext()));
-    NestedNameSpecifier *Qualifier =
-        NestedNameSpecifier::Create(Context, nullptr, ClassType.getTypePtr());
+    CanQualType ClassType =
+        Context.getCanonicalTagType(cast<RecordDecl>(VD->getDeclContext()));
+    NestedNameSpecifier Qualifier(ClassType.getTypePtr());
     SS.MakeTrivial(Context, Qualifier, Loc);
   }
 
@@ -7998,7 +8029,7 @@ static Expr *BuildExpressionFromIntegralTemplateArgumentValue(
   // type of literal for it.
   QualType T = OrigT;
   if (const EnumType *ET = OrigT->getAs<EnumType>())
-    T = ET->getDecl()->getIntegerType();
+    T = ET->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType();
 
   Expr *E;
   if (T->isAnyCharacterType()) {
@@ -8881,7 +8912,6 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
                                        isPartialSpecialization))
     return true;
 
-  QualType CanonType;
   if (!isPartialSpecialization) {
     // Create a new class template specialization declaration node for
     // this explicit specialization or friend declaration.
@@ -8897,18 +8927,14 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
 
     if (!PrevDecl)
       ClassTemplate->AddSpecialization(Specialization, InsertPos);
-
-    if (!CurContext->isDependentContext())
-      CanonType = Context.getTypeDeclType(Specialization);
-  }
-
-  TypeSourceInfo *WrittenTy = Context.getTemplateSpecializationTypeInfo(
-      Name, TemplateNameLoc, TemplateArgs, CTAI.CanonicalConverted, CanonType);
-
-  if (isPartialSpecialization) {
+  } else {
+    CanQualType CanonType = CanQualType::CreateUnsafe(
+        Context.getCanonicalTemplateSpecializationType(
+            TemplateName(ClassTemplate->getCanonicalDecl()),
+            CTAI.CanonicalConverted));
     if (Context.hasSameType(
-            WrittenTy->getType(),
-            ClassTemplate->getInjectedClassNameSpecialization()) &&
+            CanonType,
+            ClassTemplate->getCanonicalInjectedSpecializationType(Context)) &&
         (!Context.getLangOpts().CPlusPlus20 ||
          !TemplateParams->hasAssociatedConstraints())) {
       // C++ [temp.class.spec]p9b3:
@@ -8930,13 +8956,12 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     }
 
     // Create a new class template partial specialization declaration node.
-    ClassTemplatePartialSpecializationDecl *PrevPartial
-      = cast_or_null<ClassTemplatePartialSpecializationDecl>(PrevDecl);
+    ClassTemplatePartialSpecializationDecl *PrevPartial =
+        cast_or_null<ClassTemplatePartialSpecializationDecl>(PrevDecl);
     ClassTemplatePartialSpecializationDecl *Partial =
         ClassTemplatePartialSpecializationDecl::Create(
             Context, Kind, DC, KWLoc, TemplateNameLoc, TemplateParams,
-            ClassTemplate, CTAI.CanonicalConverted, WrittenTy->getType(),
-            PrevPartial);
+            ClassTemplate, CTAI.CanonicalConverted, CanonType, PrevPartial);
     Partial->setTemplateArgsAsWritten(TemplateArgs);
     SetNestedNameSpecifier(*this, Partial, SS);
     if (TemplateParameterLists.size() > 1 && SS.isSet()) {
@@ -8975,7 +9000,7 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     if (!Okay) {
       SourceRange Range(TemplateNameLoc, RAngleLoc);
       Diag(TemplateNameLoc, diag::err_specialization_after_instantiation)
-        << Context.getTypeDeclType(Specialization) << Range;
+          << Context.getCanonicalTagType(Specialization) << Range;
 
       Diag(PrevDecl->getPointOfInstantiation(),
            diag::note_instantiation_required_here)
@@ -8993,10 +9018,13 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
   if (TUK == TagUseKind::Definition) {
     RecordDecl *Def = Specialization->getDefinition();
     NamedDecl *Hidden = nullptr;
-    if (Def && SkipBody && !hasVisibleDefinition(Def, &Hidden)) {
+    bool HiddenDefVisible = false;
+    if (Def && SkipBody &&
+        isRedefinitionAllowedFor(Def, &Hidden, HiddenDefVisible)) {
       SkipBody->ShouldSkip = true;
       SkipBody->Previous = Def;
-      makeMergedDefinitionVisible(Hidden);
+      if (!HiddenDefVisible && Hidden)
+        makeMergedDefinitionVisible(Hidden);
     } else if (Def) {
       SourceRange Range(TemplateNameLoc, RAngleLoc);
       Diag(TemplateNameLoc, diag::err_redefinition) << Specialization << Range;
@@ -9038,6 +9066,13 @@ DeclResult Sema::ActOnClassTemplateSpecialization(
     Specialization->startDefinition();
 
   if (TUK == TagUseKind::Friend) {
+    CanQualType CanonType = Context.getCanonicalTagType(Specialization);
+    TypeSourceInfo *WrittenTy = Context.getTemplateSpecializationTypeInfo(
+        ElaboratedTypeKeyword::None, /*ElaboratedKeywordLoc=*/SourceLocation(),
+        SS.getWithLocInContext(Context),
+        /*TemplateKeywordLoc=*/SourceLocation(), Name, TemplateNameLoc,
+        TemplateArgs, CTAI.CanonicalConverted, CanonType);
+
     // Build the fully-sugared type for this class template
     // specialization as the user wrote in the specialization
     // itself. This means that we'll pretty-print the type retrieved
@@ -10078,9 +10113,6 @@ static bool CheckExplicitInstantiation(Sema &S, NamedDecl *D,
 
 /// Determine whether the given scope specifier has a template-id in it.
 static bool ScopeSpecifierHasTemplateId(const CXXScopeSpec &SS) {
-  if (!SS.isSet())
-    return false;
-
   // C++11 [temp.explicit]p3:
   //   If the explicit instantiation is for a member function, a member class
   //   or a static data member of a class template specialization, the name of
@@ -10088,12 +10120,14 @@ static bool ScopeSpecifierHasTemplateId(const CXXScopeSpec &SS) {
   //   name shall be a simple-template-id.
   //
   // C++98 has the same restriction, just worded differently.
-  for (NestedNameSpecifier *NNS = SS.getScopeRep(); NNS;
-       NNS = NNS->getPrefix())
-    if (const Type *T = NNS->getAsType())
-      if (isa<TemplateSpecializationType>(T))
-        return true;
-
+  for (NestedNameSpecifier NNS = SS.getScopeRep();
+       NNS.getKind() == NestedNameSpecifier::Kind::Type;
+       /**/) {
+    const Type *T = NNS.getAsType();
+    if (isa<TemplateSpecializationType>(T))
+      return true;
+    NNS = T->getPrefix();
+  }
   return false;
 }
 
@@ -10448,7 +10482,7 @@ Sema::ActOnExplicitInstantiation(Scope *S, SourceLocation ExternLoc,
   CXXRecordDecl *Pattern = Record->getInstantiatedFromMemberClass();
   if (!Pattern) {
     Diag(TemplateLoc, diag::err_explicit_instantiation_nontemplate_type)
-      << Context.getTypeDeclType(Record);
+        << Context.getCanonicalTagType(Record);
     Diag(Record->getLocation(), diag::note_nontemplate_decl_here);
     return true;
   }
@@ -10995,7 +11029,7 @@ TypeResult Sema::ActOnDependentTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
   // This has to hold, because SS is expected to be defined.
   assert(Name && "Expected a name in a dependent tag");
 
-  NestedNameSpecifier *NNS = SS.getScopeRep();
+  NestedNameSpecifier NNS = SS.getScopeRep();
   if (!NNS)
     return true;
 
@@ -11107,7 +11141,10 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
     return CreateParsedType(T, Builder.getTypeSourceInfo(Context, T));
   }
 
-  QualType T = CheckTemplateIdType(Template, TemplateIILoc, TemplateArgs);
+  QualType T = CheckTemplateIdType(TypenameLoc.isValid()
+                                       ? ElaboratedTypeKeyword::Typename
+                                       : ElaboratedTypeKeyword::None,
+                                   Template, TemplateIILoc, TemplateArgs);
   if (T.isNull())
     return true;
 
@@ -11115,18 +11152,8 @@ Sema::ActOnTypenameType(Scope *S, SourceLocation TypenameLoc,
   TypeLocBuilder Builder;
   TemplateSpecializationTypeLoc SpecTL
     = Builder.push<TemplateSpecializationTypeLoc>(T);
-  SpecTL.setTemplateKeywordLoc(TemplateKWLoc);
-  SpecTL.setTemplateNameLoc(TemplateIILoc);
-  SpecTL.setLAngleLoc(LAngleLoc);
-  SpecTL.setRAngleLoc(RAngleLoc);
-  for (unsigned I = 0, N = TemplateArgs.size(); I != N; ++I)
-    SpecTL.setArgLocInfo(I, TemplateArgs[I].getLocInfo());
-
-  T = Context.getElaboratedType(Keyword, SS.getScopeRep(), T);
-  ElaboratedTypeLoc TL = Builder.push<ElaboratedTypeLoc>(T);
-  TL.setElaboratedKeywordLoc(TypenameLoc);
-  TL.setQualifierLoc(SS.getWithLocInContext(Context));
-
+  SpecTL.set(TypenameLoc, SS.getWithLocInContext(Context), TemplateKWLoc,
+             TemplateIILoc, TemplateArgs);
   TypeSourceInfo *TSI = Builder.getTypeSourceInfo(Context, T);
   return CreateParsedType(T, TSI);
 }
@@ -11140,11 +11167,12 @@ static bool isEnableIf(NestedNameSpecifierLoc NNS, const IdentifierInfo &II,
     return false;
 
   // ... within an explicitly-written template specialization...
-  if (!NNS || !NNS.getNestedNameSpecifier()->getAsType())
+  if (NNS.getNestedNameSpecifier().getKind() != NestedNameSpecifier::Kind::Type)
     return false;
-  TypeLoc EnableIfTy = NNS.getTypeLoc();
-  TemplateSpecializationTypeLoc EnableIfTSTLoc =
-      EnableIfTy.getAs<TemplateSpecializationTypeLoc>();
+
+  // FIXME: Look through sugar.
+  auto EnableIfTSTLoc =
+      NNS.castAsTypeLoc().getAs<TemplateSpecializationTypeLoc>();
   if (!EnableIfTSTLoc || EnableIfTSTLoc.getNumArgs() == 0)
     return false;
   const TemplateSpecializationType *EnableIfTST = EnableIfTSTLoc.getTypePtr();
@@ -11192,19 +11220,33 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword,
   if (T.isNull())
     return QualType();
 
-  *TSI = Context.CreateTypeSourceInfo(T);
+  TypeLocBuilder TLB;
   if (isa<DependentNameType>(T)) {
-    DependentNameTypeLoc TL =
-        (*TSI)->getTypeLoc().castAs<DependentNameTypeLoc>();
+    auto TL = TLB.push<DependentNameTypeLoc>(T);
     TL.setElaboratedKeywordLoc(KeywordLoc);
     TL.setQualifierLoc(QualifierLoc);
     TL.setNameLoc(IILoc);
-  } else {
-    ElaboratedTypeLoc TL = (*TSI)->getTypeLoc().castAs<ElaboratedTypeLoc>();
+  } else if (isa<DeducedTemplateSpecializationType>(T)) {
+    auto TL = TLB.push<DeducedTemplateSpecializationTypeLoc>(T);
+    TL.setElaboratedKeywordLoc(KeywordLoc);
+    TL.setQualifierLoc(QualifierLoc);
+    TL.setNameLoc(IILoc);
+  } else if (isa<TemplateTypeParmType>(T)) {
+    // FIXME: There might be a 'typename' keyword here, but we just drop it
+    // as it can't be represented.
+    assert(!QualifierLoc);
+    TLB.pushTypeSpec(T).setNameLoc(IILoc);
+  } else if (isa<TagType>(T)) {
+    auto TL = TLB.push<TagTypeLoc>(T);
     TL.setElaboratedKeywordLoc(KeywordLoc);
     TL.setQualifierLoc(QualifierLoc);
-    TL.getNamedTypeLoc().castAs<TypeSpecTypeLoc>().setNameLoc(IILoc);
+    TL.setNameLoc(IILoc);
+  } else if (isa<TypedefType>(T)) {
+    TLB.push<TypedefTypeLoc>(T).set(KeywordLoc, QualifierLoc, IILoc);
+  } else {
+    TLB.push<UnresolvedUsingTypeLoc>(T).set(KeywordLoc, QualifierLoc, IILoc);
   }
+  *TSI = TLB.getTypeSourceInfo(Context, T);
   return T;
 }
 
@@ -11227,7 +11269,7 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword,
     if (!Ctx) {
       // If the nested-name-specifier is dependent and couldn't be
       // resolved to a type, build a typename type.
-      assert(QualifierLoc.getNestedNameSpecifier()->isDependent());
+      assert(QualifierLoc.getNestedNameSpecifier().isDependent());
       return Context.getDependentNameType(Keyword,
                                           QualifierLoc.getNestedNameSpecifier(),
                                           &II);
@@ -11309,6 +11351,7 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword,
                                         &II);
 
   case LookupResultKind::Found:
+    // FXIME: Missing support for UsingShadowDecl on this path?
     if (TypeDecl *Type = dyn_cast<TypeDecl>(Result.getFoundDecl())) {
       // C++ [class.qual]p2:
       //   In a lookup in which function names are not ignored and the
@@ -11324,15 +11367,20 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword,
       //
       // FIXME: That's not strictly true: mem-initializer-id lookup does not
       // ignore functions, but that appears to be an oversight.
-      QualType T = getTypeDeclType(Ctx,
-                                   Keyword == ElaboratedTypeKeyword::Typename
-                                       ? DiagCtorKind::Typename
-                                       : DiagCtorKind::None,
-                                   Type, IILoc);
-      // We found a type. Build an ElaboratedType, since the
-      // typename-specifier was just sugar.
-      return Context.getElaboratedType(
-          Keyword, QualifierLoc.getNestedNameSpecifier(), T);
+      checkTypeDeclType(Ctx,
+                        Keyword == ElaboratedTypeKeyword::Typename
+                            ? DiagCtorKind::Typename
+                            : DiagCtorKind::None,
+                        Type, IILoc);
+      // FIXME: This appears to be the only case where a template type parameter
+      // can have an elaborated keyword. We should preserve it somehow.
+      if (isa<TemplateTypeParmDecl>(Type)) {
+        assert(Keyword == ElaboratedTypeKeyword::Typename);
+        assert(!QualifierLoc);
+        Keyword = ElaboratedTypeKeyword::None;
+      }
+      return Context.getTypeDeclType(
+          Keyword, QualifierLoc.getNestedNameSpecifier(), Type);
     }
 
     // C++ [dcl.type.simple]p2:
@@ -11342,22 +11390,22 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword,
     if (getLangOpts().CPlusPlus17) {
       if (auto *TD = getAsTypeTemplateDecl(Result.getFoundDecl())) {
         if (!DeducedTSTContext) {
-          QualType T(QualifierLoc
-                         ? QualifierLoc.getNestedNameSpecifier()->getAsType()
-                         : nullptr, 0);
-          if (!T.isNull())
+          NestedNameSpecifier Qualifier = QualifierLoc.getNestedNameSpecifier();
+          if (Qualifier.getKind() == NestedNameSpecifier::Kind::Type)
             Diag(IILoc, diag::err_dependent_deduced_tst)
-              << (int)getTemplateNameKindForDiagnostics(TemplateName(TD)) << T;
+                << (int)getTemplateNameKindForDiagnostics(TemplateName(TD))
+                << QualType(Qualifier.getAsType(), 0);
           else
             Diag(IILoc, diag::err_deduced_tst)
               << (int)getTemplateNameKindForDiagnostics(TemplateName(TD));
           NoteTemplateLocation(*TD);
           return QualType();
         }
-        return Context.getElaboratedType(
-            Keyword, QualifierLoc.getNestedNameSpecifier(),
-            Context.getDeducedTemplateSpecializationType(TemplateName(TD),
-                                                         QualType(), false));
+        TemplateName Name = Context.getQualifiedTemplateName(
+            QualifierLoc.getNestedNameSpecifier(), /*TemplateKeyword=*/false,
+            TemplateName(TD));
+        return Context.getDeducedTemplateSpecializationType(
+            Keyword, Name, /*DeducedType=*/QualType(), /*IsDependent=*/false);
       }
     }
 
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index 0d70321335467..93983bf6d1607 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -691,38 +691,32 @@ DeduceTemplateSpecArguments(Sema &S, TemplateParameterList *TemplateParams,
                             TemplateDeductionInfo &Info, bool PartialOrdering,
                             SmallVectorImpl<DeducedTemplateArgument> &Deduced,
                             bool *HasDeducedAnyParam) {
-  QualType UP = P;
-  if (const auto *IP = P->getAs<InjectedClassNameType>())
-    UP = IP->getInjectedSpecializationType();
-
-  assert(isa<TemplateSpecializationType>(UP.getCanonicalType()));
-  const TemplateSpecializationType *TP = ::getLastTemplateSpecType(UP);
-  TemplateName TNP = TP->getTemplateName();
+  TemplateName TNP;
+  ArrayRef<TemplateArgument> PResolved;
+  if (isa<TemplateSpecializationType>(P.getCanonicalType())) {
+    const TemplateSpecializationType *TP = ::getLastTemplateSpecType(P);
+    TNP = TP->getTemplateName();
+    // FIXME: To preserve sugar, the TST needs to carry sugared resolved
+    // arguments.
+    PResolved = TP->getCanonicalTypeInternal()
+                    ->castAs<TemplateSpecializationType>()
+                    ->template_arguments();
+  } else {
+    const auto *TT = P->castAs<InjectedClassNameType>();
+    TNP = TT->getTemplateName(S.Context);
+    PResolved = TT->getTemplateArgs(S.Context);
+  }
 
   // If the parameter is an alias template, there is nothing to deduce.
   if (const auto *TD = TNP.getAsTemplateDecl(); TD && TD->isTypeAlias())
     return TemplateDeductionResult::Success;
-
-  // FIXME: To preserve sugar, the TST needs to carry sugared resolved
-  // arguments.
-  ArrayRef<TemplateArgument> PResolved =
-      TP->getCanonicalTypeInternal()
-          ->castAs<TemplateSpecializationType>()
-          ->template_arguments();
-
-  QualType UA = A;
-  std::optional<NestedNameSpecifier *> NNS;
-  // Treat an injected-class-name as its underlying template-id.
-  if (const auto *Elaborated = A->getAs<ElaboratedType>()) {
-    NNS = Elaborated->getQualifier();
-  } else if (const auto *Injected = A->getAs<InjectedClassNameType>()) {
-    UA = Injected->getInjectedSpecializationType();
-    NNS = nullptr;
-  }
+  // Pack-producing templates can only be matched after substitution.
+  if (isPackProducingBuiltinTemplateName(TNP))
+    return TemplateDeductionResult::Success;
 
   // Check whether the template argument is a dependent template-id.
-  if (isa<TemplateSpecializationType>(UA.getCanonicalType())) {
-    const TemplateSpecializationType *SA = ::getLastTemplateSpecType(UA);
+  if (isa<TemplateSpecializationType>(A.getCanonicalType())) {
+    const TemplateSpecializationType *SA = ::getLastTemplateSpecType(A);
     TemplateName TNA = SA->getTemplateName();
 
     // If the argument is an alias template, there is nothing to deduce.
@@ -756,34 +750,36 @@ DeduceTemplateSpecArguments(Sema &S, TemplateParameterList *TemplateParams,
   // If the argument type is a class template specialization, we
   // perform template argument deduction using its template
   // arguments.
-  const auto *RA = UA->getAs<RecordType>();
-  const auto *SA =
-      RA ? dyn_cast<ClassTemplateSpecializationDecl>(RA->getDecl()) : nullptr;
-  if (!SA) {
+  const auto *TA = A->getAs<TagType>();
+  TemplateName TNA;
+  if (TA) {
+    // FIXME: Can't use the template arguments from this TST, as they are not
+    // resolved.
+    if (const auto *TST = A->getAsNonAliasTemplateSpecializationType())
+      TNA = TST->getTemplateName();
+    else
+      TNA = TA->getTemplateName(S.Context);
+  }
+  if (TNA.isNull()) {
     Info.FirstArg = TemplateArgument(P);
     Info.SecondArg = TemplateArgument(A);
     return TemplateDeductionResult::NonDeducedMismatch;
   }
 
-  TemplateName TNA = TemplateName(SA->getSpecializedTemplate());
-  if (NNS)
-    TNA = S.Context.getQualifiedTemplateName(
-        *NNS, false, TemplateName(SA->getSpecializedTemplate()));
-
+  ArrayRef<TemplateArgument> AResolved = TA->getTemplateArgs(S.Context);
   // Perform template argument deduction for the template name.
-  if (auto Result = DeduceTemplateArguments(
-          S, TemplateParams, TNP, TNA, Info,
-          /*DefaultArguments=*/SA->getTemplateArgs().asArray(), PartialOrdering,
-          Deduced, HasDeducedAnyParam);
+  if (auto Result =
+          DeduceTemplateArguments(S, TemplateParams, TNP, TNA, Info,
+                                  /*DefaultArguments=*/AResolved,
+                                  PartialOrdering, Deduced, HasDeducedAnyParam);
       Result != TemplateDeductionResult::Success)
     return Result;
 
   // Perform template argument deduction for the template arguments.
-  return DeduceTemplateArguments(S, TemplateParams, PResolved,
-                                 SA->getTemplateArgs().asArray(), Info, Deduced,
-                                 /*NumberOfArgumentsMustMatch=*/true,
-                                 PartialOrdering, PackFold::ParameterToArgument,
-                                 HasDeducedAnyParam);
+  return DeduceTemplateArguments(
+      S, TemplateParams, PResolved, AResolved, Info, Deduced,
+      /*NumberOfArgumentsMustMatch=*/true, PartialOrdering,
+      PackFold::ParameterToArgument, HasDeducedAnyParam);
 }
 
 static bool IsPossiblyOpaquelyQualifiedTypeInternal(const Type *T) {
@@ -935,7 +931,11 @@ class PackDeductionScope {
       S.collectUnexpandedParameterPacks(Pattern, Unexpanded);
       for (unsigned I = 0, N = Unexpanded.size(); I != N; ++I) {
         unsigned Depth, Index;
-        std::tie(Depth, Index) = getDepthAndIndex(Unexpanded[I]);
+        if (auto DI = getDepthAndIndex(Unexpanded[I]))
+          std::tie(Depth, Index) = *DI;
+        else
+          continue;
+
         if (Depth == Info.getDeducedDepth())
           AddPack(Index);
       }
@@ -943,7 +943,6 @@ class PackDeductionScope {
 
     // Look for unexpanded packs in the pattern.
     Collect(Pattern);
-    assert(!Packs.empty() && "Pack expansion without unexpanded packs?");
 
     unsigned NumNamedPacks = Packs.size();
 
@@ -1865,6 +1864,7 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
 
     case Type::TemplateTypeParm:
     case Type::SubstTemplateTypeParmPack:
+    case Type::SubstBuiltinTemplatePack:
       llvm_unreachable("Type nodes handled above");
 
     case Type::Auto:
@@ -2188,26 +2188,19 @@ static TemplateDeductionResult DeduceTemplateArgumentsByTypeMatch(
           Result != TemplateDeductionResult::Success)
         return Result;
 
-      QualType TP;
-      if (MPP->isSugared()) {
-        TP = S.Context.getTypeDeclType(MPP->getMostRecentCXXRecordDecl());
-      } else {
-        NestedNameSpecifier *QP = MPP->getQualifier();
-        if (QP->getKind() == NestedNameSpecifier::Identifier)
-          // Skip translation if it's a non-deduced context anyway.
-          return TemplateDeductionResult::Success;
-        TP = QualType(QP->translateToType(S.Context), 0);
-      }
+      QualType TP =
+          MPP->isSugared()
+              ? S.Context.getCanonicalTagType(MPP->getMostRecentCXXRecordDecl())
+              : QualType(MPP->getQualifier().getAsType(), 0);
       assert(!TP.isNull() && "member pointer with non-type class");
 
-      QualType TA;
-      if (MPA->isSugared()) {
-        TA = S.Context.getTypeDeclType(MPA->getMostRecentCXXRecordDecl());
-      } else {
-        NestedNameSpecifier *QA = MPA->getQualifier();
-        TA = QualType(QA->translateToType(S.Context), 0).getUnqualifiedType();
-      }
+      QualType TA =
+          MPA->isSugared()
+              ? S.Context.getCanonicalTagType(MPA->getMostRecentCXXRecordDecl())
+              : QualType(MPA->getQualifier().getAsType(), 0)
+                    .getUnqualifiedType();
       assert(!TA.isNull() && "member pointer with non-type class");
+
       return DeduceTemplateArgumentsByTypeMatch(
           S, TemplateParams, TP, TA, Info, Deduced, SubTDF,
           degradeCallPartialOrderingKind(POK),
@@ -2925,18 +2918,12 @@ Sema::getTrivialTemplateArgumentLoc(const TemplateArgument &Arg,
     case TemplateArgument::TemplateExpansion: {
       NestedNameSpecifierLocBuilder Builder;
       TemplateName Template = Arg.getAsTemplateOrTemplatePattern();
-      if (DependentTemplateName *DTN = Template.getAsDependentTemplateName())
-        Builder.MakeTrivial(Context, DTN->getQualifier(), Loc);
-      else if (QualifiedTemplateName *QTN =
-                   Template.getAsQualifiedTemplateName())
-        Builder.MakeTrivial(Context, QTN->getQualifier(), Loc);
-
-      if (Arg.getKind() == TemplateArgument::Template)
-        return TemplateArgumentLoc(Context, Arg,
-                                   Builder.getWithLocInContext(Context), Loc);
-
+      Builder.MakeTrivial(Context, Template.getQualifier(), Loc);
       return TemplateArgumentLoc(
-          Context, Arg, Builder.getWithLocInContext(Context), Loc, Loc);
+          Context, Arg, Loc, Builder.getWithLocInContext(Context), Loc,
+          /*EllipsisLoc=*/Arg.getKind() == TemplateArgument::TemplateExpansion
+              ? Loc
+              : SourceLocation());
     }
 
   case TemplateArgument::Expression:
@@ -3154,8 +3141,9 @@ static TemplateDeductionResult ConvertDeducedTemplateArguments(
                                        S.getLangOpts().CPlusPlus17);
 
       DefArg = S.SubstDefaultTemplateArgumentIfAvailable(
-          TD, TD->getLocation(), TD->getSourceRange().getEnd(), Param,
-          CTAI.SugaredConverted, CTAI.CanonicalConverted, HasDefaultArg);
+          TD, /*TemplateKWLoc=*/SourceLocation(), TD->getLocation(),
+          TD->getSourceRange().getEnd(), Param, CTAI.SugaredConverted,
+          CTAI.CanonicalConverted, HasDefaultArg);
     }
 
     // If there was no default argument, deduction is incomplete.
@@ -3512,7 +3500,7 @@ Sema::DeduceTemplateArgumentsFromType(TemplateDecl *TD, QualType FromType,
   QualType PType;
   if (const auto *CTD = dyn_cast<ClassTemplateDecl>(TD)) {
     // Use the InjectedClassNameType.
-    PType = Context.getTypeDeclType(CTD->getTemplatedDecl());
+    PType = Context.getCanonicalTagType(CTD->getTemplatedDecl());
   } else if (const auto *AliasTemplate = dyn_cast<TypeAliasTemplateDecl>(TD)) {
     PType = AliasTemplate->getTemplatedDecl()->getUnderlyingType();
   } else {
@@ -4182,7 +4170,7 @@ static QualType GetTypeOfFunction(Sema &S, const OverloadExpr::FindResult &R,
         return {};
 
       return S.Context.getMemberPointerType(
-          Fn->getType(), /*Qualifier=*/nullptr, Method->getParent());
+          Fn->getType(), /*Qualifier=*/std::nullopt, Method->getParent());
     }
 
   if (!R.IsAddressOfOperand) return Fn->getType();
@@ -5138,10 +5126,12 @@ namespace {
         return TransformDesugared(TLB, TL);
 
       QualType Result = SemaRef.Context.getDeducedTemplateSpecializationType(
-          TL.getTypePtr()->getTemplateName(),
+          TL.getTypePtr()->getKeyword(), TL.getTypePtr()->getTemplateName(),
           Replacement, Replacement.isNull());
       auto NewTL = TLB.push<DeducedTemplateSpecializationTypeLoc>(Result);
+      NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
       NewTL.setNameLoc(TL.getNameLoc());
+      NewTL.setQualifierLoc(TL.getQualifierLoc());
       return Result;
     }
 
@@ -5607,7 +5597,8 @@ static TemplateDeductionResult CheckDeductionConsistency(
   bool IsDeductionGuide = isa<CXXDeductionGuideDecl>(FTD->getTemplatedDecl());
   if (IsDeductionGuide) {
     if (auto *Injected = P->getAs<InjectedClassNameType>())
-      P = Injected->getInjectedSpecializationType();
+      P = Injected->getOriginalDecl()->getCanonicalTemplateSpecializationType(
+          S.Context);
   }
   QualType InstP = S.SubstType(P.getCanonicalType(), MLTAL, FTD->getLocation(),
                                FTD->getDeclName(), &IsIncompleteSubstitution);
@@ -5627,9 +5618,11 @@ static TemplateDeductionResult CheckDeductionConsistency(
   auto T2 = S.Context.getUnqualifiedArrayType(A.getNonReferenceType());
   if (IsDeductionGuide) {
     if (auto *Injected = T1->getAs<InjectedClassNameType>())
-      T1 = Injected->getInjectedSpecializationType();
+      T1 = Injected->getOriginalDecl()->getCanonicalTemplateSpecializationType(
+          S.Context);
     if (auto *Injected = T2->getAs<InjectedClassNameType>())
-      T2 = Injected->getInjectedSpecializationType();
+      T2 = Injected->getOriginalDecl()->getCanonicalTemplateSpecializationType(
+          S.Context);
   }
   if (!S.Context.hasSameType(T1, T2))
     return TemplateDeductionResult::NonDeducedMismatch;
@@ -6471,8 +6464,8 @@ Sema::getMoreSpecializedPartialSpecialization(
                                   ClassTemplatePartialSpecializationDecl *PS1,
                                   ClassTemplatePartialSpecializationDecl *PS2,
                                               SourceLocation Loc) {
-  QualType PT1 = PS1->getInjectedSpecializationType().getCanonicalType();
-  QualType PT2 = PS2->getInjectedSpecializationType().getCanonicalType();
+  QualType PT1 = PS1->getCanonicalInjectedSpecializationType(Context);
+  QualType PT2 = PS2->getCanonicalInjectedSpecializationType(Context);
 
   TemplateDeductionInfo Info(Loc);
   return getMoreSpecialized(*this, PT1, PT2, PS1, PS2, Info);
@@ -6481,9 +6474,8 @@ Sema::getMoreSpecializedPartialSpecialization(
 bool Sema::isMoreSpecializedThanPrimary(
     ClassTemplatePartialSpecializationDecl *Spec, TemplateDeductionInfo &Info) {
   ClassTemplateDecl *Primary = Spec->getSpecializedTemplate();
-  QualType PrimaryT =
-      Primary->getInjectedClassNameSpecialization().getCanonicalType();
-  QualType PartialT = Spec->getInjectedSpecializationType().getCanonicalType();
+  QualType PrimaryT = Primary->getCanonicalInjectedSpecializationType(Context);
+  QualType PartialT = Spec->getCanonicalInjectedSpecializationType(Context);
 
   ClassTemplatePartialSpecializationDecl *MaybeSpec =
       getMoreSpecialized(*this, PartialT, PrimaryT, Spec, Primary, Info);
@@ -6790,19 +6782,13 @@ MarkUsedTemplateParameters(ASTContext &Ctx,
 
 /// Mark the template parameters that are used by the given
 /// nested name specifier.
-static void
-MarkUsedTemplateParameters(ASTContext &Ctx,
-                           NestedNameSpecifier *NNS,
-                           bool OnlyDeduced,
-                           unsigned Depth,
-                           llvm::SmallBitVector &Used) {
-  if (!NNS)
+static void MarkUsedTemplateParameters(ASTContext &Ctx, NestedNameSpecifier NNS,
+                                       bool OnlyDeduced, unsigned Depth,
+                                       llvm::SmallBitVector &Used) {
+  if (NNS.getKind() != NestedNameSpecifier::Kind::Type)
     return;
-
-  MarkUsedTemplateParameters(Ctx, NNS->getPrefix(), OnlyDeduced, Depth,
-                             Used);
-  MarkUsedTemplateParameters(Ctx, QualType(NNS->getAsType(), 0),
-                             OnlyDeduced, Depth, Used);
+  MarkUsedTemplateParameters(Ctx, QualType(NNS.getAsType(), 0), OnlyDeduced,
+                             Depth, Used);
 }
 
 /// Mark the template parameters that are used by the given
@@ -6876,7 +6862,7 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
     MarkUsedTemplateParameters(Ctx, MemPtr->getPointeeType(), OnlyDeduced,
                                Depth, Used);
     MarkUsedTemplateParameters(Ctx,
-                               QualType(MemPtr->getQualifier()->getAsType(), 0),
+                               QualType(MemPtr->getQualifier().getAsType(), 0),
                                OnlyDeduced, Depth, Used);
     break;
   }
@@ -6988,13 +6974,20 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
       = cast<SubstTemplateTypeParmPackType>(T);
     if (Subst->getReplacedParameter()->getDepth() == Depth)
       Used[Subst->getIndex()] = true;
-    MarkUsedTemplateParameters(Ctx, Subst->getArgumentPack(),
+    MarkUsedTemplateParameters(Ctx, Subst->getArgumentPack(), OnlyDeduced,
+                               Depth, Used);
+    break;
+  }
+  case Type::SubstBuiltinTemplatePack: {
+    MarkUsedTemplateParameters(Ctx, cast<SubstPackType>(T)->getArgumentPack(),
                                OnlyDeduced, Depth, Used);
     break;
   }
 
   case Type::InjectedClassName:
-    T = cast<InjectedClassNameType>(T)->getInjectedSpecializationType();
+    T = cast<InjectedClassNameType>(T)
+            ->getOriginalDecl()
+            ->getCanonicalTemplateSpecializationType(Ctx);
     [[fallthrough]];
 
   case Type::TemplateSpecialization: {
diff --git a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
index 9be1c9c356cb2..604591408728c 100644
--- a/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
+++ b/clang/lib/Sema/SemaTemplateDeductionGuide.cpp
@@ -105,22 +105,21 @@ class ExtractTypeForDeductionGuide
     return false;
   }
 
-  QualType
-  RebuildTemplateSpecializationType(TemplateName Template,
-                                    SourceLocation TemplateNameLoc,
-                                    TemplateArgumentListInfo &TemplateArgs) {
+  QualType RebuildTemplateSpecializationType(
+      ElaboratedTypeKeyword Keyword, TemplateName Template,
+      SourceLocation TemplateNameLoc, TemplateArgumentListInfo &TemplateArgs) {
     if (!OuterInstantiationArgs ||
         !isa_and_present<TypeAliasTemplateDecl>(Template.getAsTemplateDecl()))
-      return Base::RebuildTemplateSpecializationType(Template, TemplateNameLoc,
-                                                     TemplateArgs);
+      return Base::RebuildTemplateSpecializationType(
+          Keyword, Template, TemplateNameLoc, TemplateArgs);
 
     auto *TATD = cast<TypeAliasTemplateDecl>(Template.getAsTemplateDecl());
     auto *Pattern = TATD;
     while (Pattern->getInstantiatedFromMemberTemplate())
       Pattern = Pattern->getInstantiatedFromMemberTemplate();
     if (!mightReferToOuterTemplateParameters(Pattern->getTemplatedDecl()))
-      return Base::RebuildTemplateSpecializationType(Template, TemplateNameLoc,
-                                                     TemplateArgs);
+      return Base::RebuildTemplateSpecializationType(
+          Keyword, Template, TemplateNameLoc, TemplateArgs);
 
     Decl *NewD =
         TypedefNameInstantiator->InstantiateTypeAliasTemplateDecl(TATD);
@@ -131,13 +130,14 @@ class ExtractTypeForDeductionGuide
     MaterializedTypedefs.push_back(NewTATD->getTemplatedDecl());
 
     return Base::RebuildTemplateSpecializationType(
-        TemplateName(NewTATD), TemplateNameLoc, TemplateArgs);
+        Keyword, TemplateName(NewTATD), TemplateNameLoc, TemplateArgs);
   }
 
   QualType TransformTypedefType(TypeLocBuilder &TLB, TypedefTypeLoc TL) {
     ASTContext &Context = SemaRef.getASTContext();
-    TypedefNameDecl *OrigDecl = TL.getTypedefNameDecl();
+    TypedefNameDecl *OrigDecl = TL.getDecl();
     TypedefNameDecl *Decl = OrigDecl;
+    const TypedefType *T = TL.getTypePtr();
     // Transform the underlying type of the typedef and clone the Decl only if
     // the typedef has a dependent context.
     bool InDependentContext = OrigDecl->getDeclContext()->isDependentContext();
@@ -155,7 +155,7 @@ class ExtractTypeForDeductionGuide
     //     };
     //   };
     if (OuterInstantiationArgs && InDependentContext &&
-        TL.getTypePtr()->isInstantiationDependentType()) {
+        T->isInstantiationDependentType()) {
       Decl = cast_if_present<TypedefNameDecl>(
           TypedefNameInstantiator->InstantiateTypedefNameDecl(
               OrigDecl, /*IsTypeAlias=*/isa<TypeAliasDecl>(OrigDecl)));
@@ -180,10 +180,17 @@ class ExtractTypeForDeductionGuide
       MaterializedTypedefs.push_back(Decl);
     }
 
-    QualType TDTy = Context.getTypedefType(Decl);
-    TypedefTypeLoc TypedefTL = TLB.push<TypedefTypeLoc>(TDTy);
-    TypedefTL.setNameLoc(TL.getNameLoc());
+    NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+    if (QualifierLoc) {
+      QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(QualifierLoc);
+      if (!QualifierLoc)
+        return QualType();
+    }
 
+    QualType TDTy = Context.getTypedefType(
+        T->getKeyword(), QualifierLoc.getNestedNameSpecifier(), Decl);
+    TLB.push<TypedefTypeLoc>(TDTy).set(TL.getElaboratedKeywordLoc(),
+                                       QualifierLoc, TL.getNameLoc());
     return TDTy;
   }
 };
@@ -327,7 +334,7 @@ struct ConvertConstructorToDeductionGuideTransform {
   DeclarationName DeductionGuideName =
       SemaRef.Context.DeclarationNames.getCXXDeductionGuideName(Template);
 
-  QualType DeducedType = SemaRef.Context.getTypeDeclType(Primary);
+  QualType DeducedType = SemaRef.Context.getCanonicalTagType(Primary);
 
   // Index adjustment to apply to convert depth-1 template parameters into
   // depth-0 template parameters.
@@ -593,7 +600,10 @@ struct ConvertConstructorToDeductionGuideTransform {
     // context of the template), so implicit deduction guides can never collide
     // with explicit ones.
     QualType ReturnType = DeducedType;
-    TLB.pushTypeSpec(ReturnType).setNameLoc(Primary->getLocation());
+    auto TTL = TLB.push<TagTypeLoc>(ReturnType);
+    TTL.setElaboratedKeywordLoc(SourceLocation());
+    TTL.setQualifierLoc(NestedNameSpecifierLoc());
+    TTL.setNameLoc(Primary->getLocation());
 
     // Resolving a wording defect, we also inherit the variadicness of the
     // constructor.
@@ -954,7 +964,8 @@ Expr *buildIsDeducibleConstraint(Sema &SemaRef,
   SmallVector<TypeSourceInfo *> IsDeducibleTypeTraitArgs = {
       Context.getTrivialTypeSourceInfo(
           Context.getDeducedTemplateSpecializationType(
-              TemplateName(AliasTemplate), /*DeducedType=*/QualType(),
+              ElaboratedTypeKeyword::None, TemplateName(AliasTemplate),
+              /*DeducedType=*/QualType(),
               /*IsDependent=*/true),
           AliasTemplate->getLocation()), // template specialization type whose
                                          // arguments will be deduced.
@@ -970,10 +981,7 @@ Expr *buildIsDeducibleConstraint(Sema &SemaRef,
 
 std::pair<TemplateDecl *, llvm::ArrayRef<TemplateArgument>>
 getRHSTemplateDeclAndArgs(Sema &SemaRef, TypeAliasTemplateDecl *AliasTemplate) {
-  // Unwrap the sugared ElaboratedType.
-  auto RhsType = AliasTemplate->getTemplatedDecl()
-                     ->getUnderlyingType()
-                     .getSingleStepDesugaredType(SemaRef.Context);
+  auto RhsType = AliasTemplate->getTemplatedDecl()->getUnderlyingType();
   TemplateDecl *Template = nullptr;
   llvm::ArrayRef<TemplateArgument> AliasRhsTemplateArgs;
   if (const auto *TST = RhsType->getAs<TemplateSpecializationType>()) {
@@ -1048,12 +1056,11 @@ BuildDeductionGuideForTypeAlias(Sema &SemaRef,
   // The (trailing) return type of the deduction guide.
   const TemplateSpecializationType *FReturnType =
       RType->getAs<TemplateSpecializationType>();
-  if (const auto *InjectedCNT = RType->getAs<InjectedClassNameType>())
+  if (const auto *ICNT = RType->getAs<InjectedClassNameType>())
     // implicitly-generated deduction guide.
-    FReturnType = InjectedCNT->getInjectedTST();
-  else if (const auto *ET = RType->getAs<ElaboratedType>())
-    // explicit deduction guide.
-    FReturnType = ET->getNamedType()->getAsNonAliasTemplateSpecializationType();
+    FReturnType = cast<TemplateSpecializationType>(
+        ICNT->getOriginalDecl()->getCanonicalTemplateSpecializationType(
+            SemaRef.Context));
   assert(FReturnType && "expected to see a return type");
   // Deduce template arguments of the deduction guide f from the RHS of
   // the alias.
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index 0d96d18de2d85..fe1c5faba9e40 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -353,45 +353,50 @@ Response HandleFunctionTemplateDecl(Sema &SemaRef,
             SemaRef.Context),
         /*Final=*/false);
 
-    NestedNameSpecifier *NNS = FTD->getTemplatedDecl()->getQualifier();
-
-    while (const Type *Ty = NNS ? NNS->getAsType() : nullptr) {
-      if (NNS->isInstantiationDependent()) {
-        if (const auto *TSTy = Ty->getAs<TemplateSpecializationType>()) {
-          ArrayRef<TemplateArgument> Arguments = TSTy->template_arguments();
-          // Prefer template arguments from the injected-class-type if possible.
-          // For example,
-          // ```cpp
-          // template <class... Pack> struct S {
-          //   template <class T> void foo();
-          // };
-          // template <class... Pack> template <class T>
-          //           ^^^^^^^^^^^^^ InjectedTemplateArgs
-          //           They're of kind TemplateArgument::Pack, not of
-          //           TemplateArgument::Type.
-          // void S<Pack...>::foo() {}
-          //        ^^^^^^^
-          //        TSTy->template_arguments() (which are of PackExpansionType)
-          // ```
-          // This meets the contract in
-          // TreeTransform::TryExpandParameterPacks that the template arguments
-          // for unexpanded parameters should be of a Pack kind.
-          if (TSTy->isCurrentInstantiation()) {
-            auto *RD = TSTy->getCanonicalTypeInternal()->getAsCXXRecordDecl();
-            if (ClassTemplateDecl *CTD = RD->getDescribedClassTemplate())
-              Arguments = CTD->getInjectedTemplateArgs(SemaRef.Context);
-            else if (auto *Specialization =
-                         dyn_cast<ClassTemplateSpecializationDecl>(RD))
-              Arguments =
-                  Specialization->getTemplateInstantiationArgs().asArray();
-          }
-          Result.addOuterTemplateArguments(
-              TSTy->getTemplateName().getAsTemplateDecl(), Arguments,
-              /*Final=*/false);
-        }
-      }
+    NestedNameSpecifier NNS = FTD->getTemplatedDecl()->getQualifier();
+
+    for (const Type *Ty = NNS.getKind() == NestedNameSpecifier::Kind::Type
+                              ? NNS.getAsType()
+                              : nullptr,
+                    *NextTy = nullptr;
+         Ty && Ty->isInstantiationDependentType();
+         Ty = std::exchange(NextTy, nullptr)) {
+      if (NestedNameSpecifier P = Ty->getPrefix();
+          P.getKind() == NestedNameSpecifier::Kind::Type)
+        NextTy = P.getAsType();
+      const auto *TSTy = dyn_cast<TemplateSpecializationType>(Ty);
+      if (!TSTy)
+        continue;
 
-      NNS = NNS->getPrefix();
+      ArrayRef<TemplateArgument> Arguments = TSTy->template_arguments();
+      // Prefer template arguments from the injected-class-type if possible.
+      // For example,
+      // ```cpp
+      // template <class... Pack> struct S {
+      //   template <class T> void foo();
+      // };
+      // template <class... Pack> template <class T>
+      //           ^^^^^^^^^^^^^ InjectedTemplateArgs
+      //           They're of kind TemplateArgument::Pack, not of
+      //           TemplateArgument::Type.
+      // void S<Pack...>::foo() {}
+      //        ^^^^^^^
+      //        TSTy->template_arguments() (which are of PackExpansionType)
+      // ```
+      // This meets the contract in
+      // TreeTransform::TryExpandParameterPacks that the template arguments
+      // for unexpanded parameters should be of a Pack kind.
+      if (TSTy->isCurrentInstantiation()) {
+        auto *RD = TSTy->getCanonicalTypeInternal()->getAsCXXRecordDecl();
+        if (ClassTemplateDecl *CTD = RD->getDescribedClassTemplate())
+          Arguments = CTD->getInjectedTemplateArgs(SemaRef.Context);
+        else if (auto *Specialization =
+                     dyn_cast<ClassTemplateSpecializationDecl>(RD))
+          Arguments = Specialization->getTemplateInstantiationArgs().asArray();
+      }
+      Result.addOuterTemplateArguments(
+          TSTy->getTemplateName().getAsTemplateDecl(), Arguments,
+          /*Final=*/false);
     }
   }
 
@@ -1165,7 +1170,7 @@ void Sema::PrintInstantiationStack(InstantiationContextDiagFuncRef DiagFunc) {
         DiagFunc(Active->PointOfInstantiation,
                  PDiag(diag::note_member_synthesized_at)
                      << MD->isExplicitlyDefaulted() << DFK.asSpecialMember()
-                     << Context.getTagDeclType(MD->getParent()));
+                     << Context.getCanonicalTagType(MD->getParent()));
       } else if (DFK.isComparison()) {
         QualType RecordType = FD->getParamDecl(0)
                                   ->getType()
@@ -1454,6 +1459,7 @@ namespace {
     bool TryExpandParameterPacks(SourceLocation EllipsisLoc,
                                  SourceRange PatternRange,
                                  ArrayRef<UnexpandedParameterPack> Unexpanded,
+                                 bool FailOnPackProducingTemplates,
                                  bool &ShouldExpand, bool &RetainExpansion,
                                  UnsignedOrNone &NumExpansions) {
       if (SemaRef.CurrentInstantiationScope &&
@@ -1467,8 +1473,9 @@ namespace {
       }
 
       return getSema().CheckParameterPacksForExpansion(
-          EllipsisLoc, PatternRange, Unexpanded, TemplateArgs, ShouldExpand,
-          RetainExpansion, NumExpansions);
+          EllipsisLoc, PatternRange, Unexpanded, TemplateArgs,
+          FailOnPackProducingTemplates, ShouldExpand, RetainExpansion,
+          NumExpansions);
     }
 
     void ExpandingFunctionParameterPack(ParmVarDecl *Pack) {
@@ -1510,6 +1517,21 @@ namespace {
       }
     }
 
+    MultiLevelTemplateArgumentList ForgetSubstitution() {
+      MultiLevelTemplateArgumentList New;
+      New.addOuterRetainedLevels(this->TemplateArgs.getNumLevels());
+
+      MultiLevelTemplateArgumentList Old =
+          const_cast<MultiLevelTemplateArgumentList &>(this->TemplateArgs);
+      const_cast<MultiLevelTemplateArgumentList &>(this->TemplateArgs) =
+          std::move(New);
+      return Old;
+    }
+    void RememberSubstitution(MultiLevelTemplateArgumentList Old) {
+      const_cast<MultiLevelTemplateArgumentList &>(this->TemplateArgs) =
+          std::move(Old);
+    }
+
     TemplateArgument
     getTemplateArgumentPackPatternForRewrite(const TemplateArgument &TA) {
       if (TA.getKind() != TemplateArgument::Pack)
@@ -1595,15 +1617,9 @@ namespace {
     VarDecl *RebuildObjCExceptionDecl(VarDecl *ExceptionDecl,
                                       TypeSourceInfo *TSInfo, QualType T);
 
-    /// Check for tag mismatches when instantiating an
-    /// elaborated type.
-    QualType RebuildElaboratedType(SourceLocation KeywordLoc,
-                                   ElaboratedTypeKeyword Keyword,
-                                   NestedNameSpecifierLoc QualifierLoc,
-                                   QualType T);
-
     TemplateName
-    TransformTemplateName(CXXScopeSpec &SS, TemplateName Name,
+    TransformTemplateName(NestedNameSpecifierLoc &QualifierLoc,
+                          SourceLocation TemplateKWLoc, TemplateName Name,
                           SourceLocation NameLoc,
                           QualType ObjectType = QualType(),
                           NamedDecl *FirstQualifierInScope = nullptr,
@@ -1649,22 +1665,20 @@ namespace {
       return inherited::TransformFunctionProtoType(TLB, TL);
     }
 
-    QualType TransformInjectedClassNameType(TypeLocBuilder &TLB,
-                                            InjectedClassNameTypeLoc TL) {
-      auto Type = inherited::TransformInjectedClassNameType(TLB, TL);
+    QualType TransformTagType(TypeLocBuilder &TLB, TagTypeLoc TL) {
+      auto Type = inherited::TransformTagType(TLB, TL);
+      if (!Type.isNull())
+        return Type;
       // Special case for transforming a deduction guide, we return a
       // transformed TemplateSpecializationType.
-      if (Type.isNull() &&
-          SemaRef.CodeSynthesisContexts.back().Kind ==
-              Sema::CodeSynthesisContext::BuildingDeductionGuides) {
-        // Return a TemplateSpecializationType for transforming a deduction
-        // guide.
-        if (auto *ICT = TL.getType()->getAs<InjectedClassNameType>()) {
-          auto Type =
-              inherited::TransformType(ICT->getInjectedSpecializationType());
-          TLB.pushTrivial(SemaRef.Context, Type, TL.getNameLoc());
-          return Type;
-        }
+      // FIXME: Why is this hack necessary?
+      if (const auto *ICNT = dyn_cast<InjectedClassNameType>(TL.getTypePtr());
+          ICNT && SemaRef.CodeSynthesisContexts.back().Kind ==
+                      Sema::CodeSynthesisContext::BuildingDeductionGuides) {
+        Type = inherited::TransformType(
+            ICNT->getOriginalDecl()->getCanonicalTemplateSpecializationType(
+                SemaRef.Context));
+        TLB.pushTrivial(SemaRef.Context, Type, TL.getNameLoc());
       }
       return Type;
     }
@@ -1700,6 +1714,26 @@ namespace {
       return inherited::TransformTemplateArgument(Input, Output, Uneval);
     }
 
+    using TreeTransform::TransformTemplateSpecializationType;
+    QualType
+    TransformTemplateSpecializationType(TypeLocBuilder &TLB,
+                                        TemplateSpecializationTypeLoc TL) {
+      auto *T = TL.getTypePtr();
+      if (!getSema().ArgPackSubstIndex || !T->isSugared() ||
+          !isPackProducingBuiltinTemplateName(T->getTemplateName()))
+        return TreeTransform::TransformTemplateSpecializationType(TLB, TL);
+      // Look through sugar to get to the SubstBuiltinTemplatePackType that we
+      // need to substitute into.
+
+      // `TransformType` code below will handle picking the element from a pack
+      // with the index `ArgPackSubstIndex`.
+      // FIXME: add ability to represent sugarred type for N-th element of a
+      // builtin pack and produce the sugar here.
+      QualType R = TransformType(T->desugar());
+      TLB.pushTrivial(getSema().getASTContext(), R, TL.getBeginLoc());
+      return R;
+    }
+
     UnsignedOrNone ComputeSizeOfPackExprWithoutSubstitution(
         ArrayRef<TemplateArgument> PackArgs) {
       // Don't do this when rewriting template parameters for CTAD:
@@ -1749,6 +1783,9 @@ namespace {
     TransformSubstTemplateTypeParmPackType(TypeLocBuilder &TLB,
                                            SubstTemplateTypeParmPackTypeLoc TL,
                                            bool SuppressObjCLifetime);
+    QualType
+    TransformSubstBuiltinTemplatePackType(TypeLocBuilder &TLB,
+                                          SubstBuiltinTemplatePackTypeLoc TL);
 
     CXXRecordDecl::LambdaDependencyKind
     ComputeLambdaDependency(LambdaScopeInfo *LSI) {
@@ -1986,6 +2023,7 @@ bool TemplateInstantiator::maybeInstantiateFunctionParameterToScope(
   UnsignedOrNone NumExpansions = OrigNumExpansions;
   if (TryExpandParameterPacks(ExpansionTL.getEllipsisLoc(),
                               Pattern.getSourceRange(), Unexpanded,
+                              /*FailOnPackProducingTemplates=*/true,
                               ShouldExpand, RetainExpansion, NumExpansions))
     return true;
 
@@ -2049,7 +2087,7 @@ TemplateInstantiator::TransformFirstQualifierInScope(NamedDecl *D,
         return cast_or_null<NamedDecl>(TransformDecl(Loc, D));
 
       if (const TagType *Tag = T->getAs<TagType>())
-        return Tag->getDecl();
+        return Tag->getOriginalDecl();
 
       // The resulting type is not a tag; complain.
       getSema().Diag(Loc, diag::err_nested_name_spec_non_tag) << T;
@@ -2082,41 +2120,10 @@ VarDecl *TemplateInstantiator::RebuildObjCExceptionDecl(VarDecl *ExceptionDecl,
   return Var;
 }
 
-QualType
-TemplateInstantiator::RebuildElaboratedType(SourceLocation KeywordLoc,
-                                            ElaboratedTypeKeyword Keyword,
-                                            NestedNameSpecifierLoc QualifierLoc,
-                                            QualType T) {
-  if (const TagType *TT = T->getAs<TagType>()) {
-    TagDecl* TD = TT->getDecl();
-
-    SourceLocation TagLocation = KeywordLoc;
-
-    IdentifierInfo *Id = TD->getIdentifier();
-
-    // TODO: should we even warn on struct/class mismatches for this?  Seems
-    // like it's likely to produce a lot of spurious errors.
-    if (Id && Keyword != ElaboratedTypeKeyword::None &&
-        Keyword != ElaboratedTypeKeyword::Typename) {
-      TagTypeKind Kind = TypeWithKeyword::getTagTypeKindForKeyword(Keyword);
-      if (!SemaRef.isAcceptableTagRedeclaration(TD, Kind, /*isDefinition*/false,
-                                                TagLocation, Id)) {
-        SemaRef.Diag(TagLocation, diag::err_use_with_wrong_tag)
-          << Id
-          << FixItHint::CreateReplacement(SourceRange(TagLocation),
-                                          TD->getKindName());
-        SemaRef.Diag(TD->getLocation(), diag::note_previous_use);
-      }
-    }
-  }
-
-  return inherited::RebuildElaboratedType(KeywordLoc, Keyword, QualifierLoc, T);
-}
-
 TemplateName TemplateInstantiator::TransformTemplateName(
-    CXXScopeSpec &SS, TemplateName Name, SourceLocation NameLoc,
-    QualType ObjectType, NamedDecl *FirstQualifierInScope,
-    bool AllowInjectedClassName) {
+    NestedNameSpecifierLoc &QualifierLoc, SourceLocation TemplateKWLoc,
+    TemplateName Name, SourceLocation NameLoc, QualType ObjectType,
+    NamedDecl *FirstQualifierInScope, bool AllowInjectedClassName) {
   if (TemplateTemplateParmDecl *TTP
        = dyn_cast_or_null<TemplateTemplateParmDecl>(Name.getAsTemplateDecl())) {
     if (TTP->getDepth() < TemplateArgs.getNumLevels()) {
@@ -2163,6 +2170,12 @@ TemplateName TemplateInstantiator::TransformTemplateName(
       TemplateName Template = Arg.getAsTemplate();
       assert(!Template.isNull() && "Null template template argument");
 
+      if (NestedNameSpecifier Qualifier = Template.getQualifier()) {
+        NestedNameSpecifierLocBuilder Builder;
+        Builder.MakeTrivial(SemaRef.Context, Qualifier, NameLoc);
+        QualifierLoc = Builder.getWithLocInContext(SemaRef.Context);
+      }
+
       return getSema().Context.getSubstTemplateTemplateParm(
           Template, AssociatedDecl, TTP->getIndex(), PackIndex, Final);
     }
@@ -2181,9 +2194,9 @@ TemplateName TemplateInstantiator::TransformTemplateName(
         getPackIndex(Pack), SubstPack->getFinal());
   }
 
-  return inherited::TransformTemplateName(SS, Name, NameLoc, ObjectType,
-                                          FirstQualifierInScope,
-                                          AllowInjectedClassName);
+  return inherited::TransformTemplateName(
+      QualifierLoc, TemplateKWLoc, Name, NameLoc, ObjectType,
+      FirstQualifierInScope, AllowInjectedClassName);
 }
 
 ExprResult
@@ -2747,6 +2760,17 @@ QualType TemplateInstantiator::TransformSubstTemplateTypeParmPackType(
       getPackIndex(Pack), Arg, TL.getNameLoc());
 }
 
+QualType TemplateInstantiator::TransformSubstBuiltinTemplatePackType(
+    TypeLocBuilder &TLB, SubstBuiltinTemplatePackTypeLoc TL) {
+  if (!getSema().ArgPackSubstIndex)
+    return TreeTransform::TransformSubstBuiltinTemplatePackType(TLB, TL);
+  auto &Sema = getSema();
+  TemplateArgument Result = getPackSubstitutedTemplateArgument(
+      Sema, TL.getTypePtr()->getArgumentPack());
+  TLB.pushTrivial(Sema.getASTContext(), Result.getAsType(), TL.getBeginLoc());
+  return Result.getAsType();
+}
+
 static concepts::Requirement::SubstitutionDiagnostic *
 createSubstDiag(Sema &S, TemplateDeductionInfo &Info,
                 Sema::EntityPrinter Printer) {
@@ -3159,10 +3183,6 @@ namespace {
     // Only these types can contain 'auto' types, and subsequently be replaced
     // by references to invented parameters.
 
-    TemplateTypeParmDecl *VisitElaboratedType(const ElaboratedType *T) {
-      return Visit(T->getNamedType());
-    }
-
     TemplateTypeParmDecl *VisitPointerType(const PointerType *T) {
       return Visit(T->getPointeeType());
     }
@@ -3478,6 +3498,72 @@ bool Sema::SubstDefaultArgument(
   return false;
 }
 
+// See TreeTransform::PreparePackForExpansion for the relevant comment.
+// This function implements the same concept for base specifiers.
+static bool
+PreparePackForExpansion(Sema &S, const CXXBaseSpecifier &Base,
+                        const MultiLevelTemplateArgumentList &TemplateArgs,
+                        TypeSourceInfo *&Out, UnexpandedInfo &Info) {
+  SourceRange BaseSourceRange = Base.getSourceRange();
+  SourceLocation BaseEllipsisLoc = Base.getEllipsisLoc();
+  Info.Ellipsis = Base.getEllipsisLoc();
+  auto ComputeInfo = [&S, &TemplateArgs, BaseSourceRange, BaseEllipsisLoc](
+                         TypeSourceInfo *BaseTypeInfo,
+                         bool IsLateExpansionAttempt, UnexpandedInfo &Info) {
+    // This is a pack expansion. See whether we should expand it now, or
+    // wait until later.
+    SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+    S.collectUnexpandedParameterPacks(BaseTypeInfo->getTypeLoc(), Unexpanded);
+    if (IsLateExpansionAttempt) {
+      // Request expansion only when there is an opportunity to expand a pack
+      // that required a substituion first.
+      bool SawPackTypes =
+          llvm::any_of(Unexpanded, [](UnexpandedParameterPack P) {
+            return P.first.dyn_cast<const SubstBuiltinTemplatePackType *>();
+          });
+      if (!SawPackTypes) {
+        Info.Expand = false;
+        return false;
+      }
+    }
+
+    // Determine whether the set of unexpanded parameter packs can and should be
+    // expanded.
+    Info.Expand = false;
+    Info.RetainExpansion = false;
+    Info.NumExpansions = std::nullopt;
+    return S.CheckParameterPacksForExpansion(
+        BaseEllipsisLoc, BaseSourceRange, Unexpanded, TemplateArgs,
+        /*FailOnPackProducingTemplates=*/false, Info.Expand,
+        Info.RetainExpansion, Info.NumExpansions);
+  };
+
+  if (ComputeInfo(Base.getTypeSourceInfo(), false, Info))
+    return true;
+
+  if (Info.Expand) {
+    Out = Base.getTypeSourceInfo();
+    return false;
+  }
+
+  // The resulting base specifier will (still) be a pack expansion.
+  {
+    Sema::ArgPackSubstIndexRAII SubstIndex(S, std::nullopt);
+    Out = S.SubstType(Base.getTypeSourceInfo(), TemplateArgs,
+                      BaseSourceRange.getBegin(), DeclarationName());
+  }
+  if (!Out->getType()->containsUnexpandedParameterPack())
+    return false;
+
+  // Some packs will learn their length after substitution.
+  // We may need to request their expansion.
+  if (ComputeInfo(Out, /*IsLateExpansionAttempt=*/true, Info))
+    return true;
+  if (Info.Expand)
+    Info.ExpandUnderForgetSubstitions = true;
+  return false;
+}
+
 bool
 Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
                           CXXRecordDecl *Pattern,
@@ -3495,47 +3581,37 @@ Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
     }
 
     SourceLocation EllipsisLoc;
-    TypeSourceInfo *BaseTypeLoc;
+    TypeSourceInfo *BaseTypeLoc = nullptr;
     if (Base.isPackExpansion()) {
-      // This is a pack expansion. See whether we should expand it now, or
-      // wait until later.
-      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
-      collectUnexpandedParameterPacks(Base.getTypeSourceInfo()->getTypeLoc(),
-                                      Unexpanded);
-      bool ShouldExpand = false;
-      bool RetainExpansion = false;
-      UnsignedOrNone NumExpansions = std::nullopt;
-      if (CheckParameterPacksForExpansion(Base.getEllipsisLoc(),
-                                          Base.getSourceRange(),
-                                          Unexpanded,
-                                          TemplateArgs, ShouldExpand,
-                                          RetainExpansion,
-                                          NumExpansions)) {
+      UnexpandedInfo Info;
+      if (PreparePackForExpansion(*this, Base, TemplateArgs, BaseTypeLoc,
+                                  Info)) {
         Invalid = true;
         continue;
       }
 
       // If we should expand this pack expansion now, do so.
-      if (ShouldExpand) {
-        for (unsigned I = 0; I != *NumExpansions; ++I) {
+      MultiLevelTemplateArgumentList EmptyList;
+      const MultiLevelTemplateArgumentList *ArgsForSubst = &TemplateArgs;
+      if (Info.ExpandUnderForgetSubstitions)
+        ArgsForSubst = &EmptyList;
+
+      if (Info.Expand) {
+        for (unsigned I = 0; I != *Info.NumExpansions; ++I) {
           Sema::ArgPackSubstIndexRAII SubstIndex(*this, I);
 
-          TypeSourceInfo *BaseTypeLoc = SubstType(Base.getTypeSourceInfo(),
-                                                  TemplateArgs,
-                                              Base.getSourceRange().getBegin(),
-                                                  DeclarationName());
-          if (!BaseTypeLoc) {
+          TypeSourceInfo *Expanded =
+              SubstType(BaseTypeLoc, *ArgsForSubst,
+                        Base.getSourceRange().getBegin(), DeclarationName());
+          if (!Expanded) {
             Invalid = true;
             continue;
           }
 
-          if (CXXBaseSpecifier *InstantiatedBase
-                = CheckBaseSpecifier(Instantiation,
-                                     Base.getSourceRange(),
-                                     Base.isVirtual(),
-                                     Base.getAccessSpecifierAsWritten(),
-                                     BaseTypeLoc,
-                                     SourceLocation()))
+          if (CXXBaseSpecifier *InstantiatedBase = CheckBaseSpecifier(
+                  Instantiation, Base.getSourceRange(), Base.isVirtual(),
+                  Base.getAccessSpecifierAsWritten(), Expanded,
+                  SourceLocation()))
             InstantiatedBases.push_back(InstantiatedBase);
           else
             Invalid = true;
@@ -3547,10 +3623,9 @@ Sema::SubstBaseSpecifiers(CXXRecordDecl *Instantiation,
       // The resulting base specifier will (still) be a pack expansion.
       EllipsisLoc = Base.getEllipsisLoc();
       Sema::ArgPackSubstIndexRAII SubstIndex(*this, std::nullopt);
-      BaseTypeLoc = SubstType(Base.getTypeSourceInfo(),
-                              TemplateArgs,
-                              Base.getSourceRange().getBegin(),
-                              DeclarationName());
+      BaseTypeLoc =
+          SubstType(BaseTypeLoc, *ArgsForSubst,
+                    Base.getSourceRange().getBegin(), DeclarationName());
     } else {
       BaseTypeLoc = SubstType(Base.getTypeSourceInfo(),
                               TemplateArgs,
@@ -4557,14 +4632,14 @@ Sema::SubstDeclarationNameInfo(const DeclarationNameInfo &NameInfo,
 }
 
 TemplateName
-Sema::SubstTemplateName(NestedNameSpecifierLoc QualifierLoc,
-                        TemplateName Name, SourceLocation Loc,
+Sema::SubstTemplateName(SourceLocation TemplateKWLoc,
+                        NestedNameSpecifierLoc &QualifierLoc, TemplateName Name,
+                        SourceLocation NameLoc,
                         const MultiLevelTemplateArgumentList &TemplateArgs) {
-  TemplateInstantiator Instantiator(*this, TemplateArgs, Loc,
+  TemplateInstantiator Instantiator(*this, TemplateArgs, NameLoc,
                                     DeclarationName());
-  CXXScopeSpec SS;
-  SS.Adopt(QualifierLoc);
-  return Instantiator.TransformTemplateName(SS, Name, Loc);
+  return Instantiator.TransformTemplateName(QualifierLoc, TemplateKWLoc, Name,
+                                            NameLoc);
 }
 
 static const Decl *getCanonicalParmVarDecl(const Decl *D) {
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 87ec4f75863fe..681ee796440f3 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -133,8 +133,9 @@ static void instantiateDependentAlignedAttr(
   // FIXME: Use the actual location of the ellipsis.
   SourceLocation EllipsisLoc = Aligned->getLocation();
   if (S.CheckParameterPacksForExpansion(EllipsisLoc, Aligned->getRange(),
-                                        Unexpanded, TemplateArgs, Expand,
-                                        RetainExpansion, NumExpansions))
+                                        Unexpanded, TemplateArgs,
+                                        /*FailOnPackProducingTemplates=*/true,
+                                        Expand, RetainExpansion, NumExpansions))
     return;
 
   if (!Expand) {
@@ -1483,9 +1484,9 @@ Decl *TemplateDeclInstantiator::InstantiateTypedefNameDecl(TypedefNameDecl *D,
   // If the old typedef was the name for linkage purposes of an anonymous
   // tag decl, re-establish that relationship for the new typedef.
   if (const TagType *oldTagType = D->getUnderlyingType()->getAs<TagType>()) {
-    TagDecl *oldTag = oldTagType->getDecl();
+    TagDecl *oldTag = oldTagType->getOriginalDecl();
     if (oldTag->getTypedefNameForAnonDecl() == D && !Invalid) {
-      TagDecl *newTag = DI->getType()->castAs<TagType>()->getDecl();
+      TagDecl *newTag = DI->getType()->castAs<TagType>()->getOriginalDecl();
       assert(!newTag->hasNameForLinkage());
       newTag->setTypedefNameForAnonDecl(Typedef);
     }
@@ -1914,7 +1915,8 @@ Decl *TemplateDeclInstantiator::VisitFriendDecl(FriendDecl *D) {
         UnsignedOrNone NumExpansions = std::nullopt;
         if (SemaRef.CheckParameterPacksForExpansion(
                 D->getEllipsisLoc(), D->getSourceRange(), Unexpanded,
-                TemplateArgs, ShouldExpand, RetainExpansion, NumExpansions))
+                TemplateArgs, /*FailOnPackProducingTemplates=*/true,
+                ShouldExpand, RetainExpansion, NumExpansions))
           return nullptr;
 
         assert(!RetainExpansion &&
@@ -2250,8 +2252,7 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
 
   CXXRecordDecl *RecordInst = CXXRecordDecl::Create(
       SemaRef.Context, Pattern->getTagKind(), DC, Pattern->getBeginLoc(),
-      Pattern->getLocation(), Pattern->getIdentifier(), PrevDecl,
-      /*DelayTypeCreation=*/true);
+      Pattern->getLocation(), Pattern->getIdentifier(), PrevDecl);
   if (QualifierLoc)
     RecordInst->setQualifierInfo(QualifierLoc);
 
@@ -2271,8 +2272,6 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
 
     if (PrevClassTemplate) {
       Inst->setCommonPtr(PrevClassTemplate->getCommonPtr());
-      RecordInst->setTypeForDecl(
-          PrevClassTemplate->getTemplatedDecl()->getTypeForDecl());
       const ClassTemplateDecl *MostRecentPrevCT =
           PrevClassTemplate->getMostRecentDecl();
       TemplateParameterList *PrevParams =
@@ -2306,10 +2305,6 @@ Decl *TemplateDeclInstantiator::VisitClassTemplateDecl(ClassTemplateDecl *D) {
 
   Inst->setPreviousDecl(PrevClassTemplate);
 
-  // Trigger creation of the type for the instantiation.
-  SemaRef.Context.getInjectedClassNameType(
-      RecordInst, Inst->getInjectedClassNameSpecialization());
-
   // Finish handling of friends.
   if (isFriend) {
     DC->makeDeclVisibleInContext(Inst);
@@ -2515,11 +2510,9 @@ Decl *TemplateDeclInstantiator::VisitCXXRecordDecl(CXXRecordDecl *D) {
   else
     Record = CXXRecordDecl::Create(SemaRef.Context, D->getTagKind(), Owner,
                                    D->getBeginLoc(), D->getLocation(),
-                                   D->getIdentifier(), PrevDecl,
-                                   /*DelayTypeCreation=*/IsInjectedClassName);
-  // Link the type of the injected-class-name to that of the outer class.
-  if (IsInjectedClassName)
-    (void)SemaRef.Context.getTypeDeclType(Record, cast<CXXRecordDecl>(Owner));
+                                   D->getIdentifier(), PrevDecl);
+
+  Record->setImplicit(D->isImplicit());
 
   // Substitute the nested name specifier, if any.
   if (SubstQualifier(D, Record))
@@ -2528,7 +2521,6 @@ Decl *TemplateDeclInstantiator::VisitCXXRecordDecl(CXXRecordDecl *D) {
   SemaRef.InstantiateAttrsForDecl(TemplateArgs, D, Record, LateAttrs,
                                                               StartingScope);
 
-  Record->setImplicit(D->isImplicit());
   // FIXME: Check against AS_none is an ugly hack to work around the issue that
   // the tag decls introduced by friend class declarations don't have an access
   // specifier. Remove once this area of the code gets sorted out.
@@ -3164,8 +3156,8 @@ Decl *TemplateDeclInstantiator::VisitCXXMethodDecl(
     Method->setIneligibleOrNotSelected(true);
     Method->setRangeEnd(Destructor->getEndLoc());
     Method->setDeclName(SemaRef.Context.DeclarationNames.getCXXDestructorName(
-        SemaRef.Context.getCanonicalType(
-            SemaRef.Context.getTypeDeclType(Record))));
+
+        SemaRef.Context.getCanonicalTagType(Record)));
   } else if (CXXConversionDecl *Conversion = dyn_cast<CXXConversionDecl>(D)) {
     Method = CXXConversionDecl::Create(
         SemaRef.Context, Record, StartLoc, NameInfo, T, TInfo,
@@ -3474,10 +3466,11 @@ Decl *TemplateDeclInstantiator::VisitTemplateTypeParmDecl(
               cast<CXXFoldExpr>(TC->getImmediatelyDeclaredConstraint())
                   ->getEllipsisLoc(),
               SourceRange(TC->getConceptNameLoc(),
-                          TC->hasExplicitTemplateArgs() ?
-                          TC->getTemplateArgsAsWritten()->getRAngleLoc() :
-                          TC->getConceptNameInfo().getEndLoc()),
-              Unexpanded, TemplateArgs, Expand, RetainExpansion, NumExpanded))
+                          TC->hasExplicitTemplateArgs()
+                              ? TC->getTemplateArgsAsWritten()->getRAngleLoc()
+                              : TC->getConceptNameInfo().getEndLoc()),
+              Unexpanded, TemplateArgs, /*FailOnPackProducingTemplates=*/true,
+              Expand, RetainExpansion, NumExpanded))
         return nullptr;
     }
   }
@@ -3565,12 +3558,10 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
     UnsignedOrNone OrigNumExpansions =
         Expansion.getTypePtr()->getNumExpansions();
     UnsignedOrNone NumExpansions = OrigNumExpansions;
-    if (SemaRef.CheckParameterPacksForExpansion(Expansion.getEllipsisLoc(),
-                                                Pattern.getSourceRange(),
-                                                Unexpanded,
-                                                TemplateArgs,
-                                                Expand, RetainExpansion,
-                                                NumExpansions))
+    if (SemaRef.CheckParameterPacksForExpansion(
+            Expansion.getEllipsisLoc(), Pattern.getSourceRange(), Unexpanded,
+            TemplateArgs, /*FailOnPackProducingTemplates=*/true, Expand,
+            RetainExpansion, NumExpansions))
       return nullptr;
 
     if (Expand) {
@@ -3736,12 +3727,10 @@ TemplateDeclInstantiator::VisitTemplateTemplateParmDecl(
     bool Expand = true;
     bool RetainExpansion = false;
     UnsignedOrNone NumExpansions = std::nullopt;
-    if (SemaRef.CheckParameterPacksForExpansion(D->getLocation(),
-                                                TempParams->getSourceRange(),
-                                                Unexpanded,
-                                                TemplateArgs,
-                                                Expand, RetainExpansion,
-                                                NumExpansions))
+    if (SemaRef.CheckParameterPacksForExpansion(
+            D->getLocation(), TempParams->getSourceRange(), Unexpanded,
+            TemplateArgs, /*FailOnPackProducingTemplates=*/true, Expand,
+            RetainExpansion, NumExpansions))
       return nullptr;
 
     if (Expand) {
@@ -3793,19 +3782,18 @@ TemplateDeclInstantiator::VisitTemplateTemplateParmDecl(
         D->getPosition(), D->isParameterPack(), D->getIdentifier(),
         D->templateParameterKind(), D->wasDeclaredWithTypename(), InstParams);
   if (D->hasDefaultArgument() && !D->defaultArgumentWasInherited()) {
-    NestedNameSpecifierLoc QualifierLoc =
-        D->getDefaultArgument().getTemplateQualifierLoc();
-    QualifierLoc =
-        SemaRef.SubstNestedNameSpecifierLoc(QualifierLoc, TemplateArgs);
+    const TemplateArgumentLoc &A = D->getDefaultArgument();
+    NestedNameSpecifierLoc QualifierLoc = A.getTemplateQualifierLoc();
+    // FIXME: Pass in the template keyword location.
     TemplateName TName = SemaRef.SubstTemplateName(
-        QualifierLoc, D->getDefaultArgument().getArgument().getAsTemplate(),
-        D->getDefaultArgument().getTemplateNameLoc(), TemplateArgs);
+        A.getTemplateKWLoc(), QualifierLoc, A.getArgument().getAsTemplate(),
+        A.getTemplateNameLoc(), TemplateArgs);
     if (!TName.isNull())
       Param->setDefaultArgument(
           SemaRef.Context,
           TemplateArgumentLoc(SemaRef.Context, TemplateArgument(TName),
-                              D->getDefaultArgument().getTemplateQualifierLoc(),
-                              D->getDefaultArgument().getTemplateNameLoc()));
+                              A.getTemplateKWLoc(), QualifierLoc,
+                              A.getTemplateNameLoc()));
   }
   Param->setAccess(AS_public);
   Param->setImplicit(D->isImplicit());
@@ -3907,7 +3895,7 @@ Decl *TemplateDeclInstantiator::VisitUsingDecl(UsingDecl *D) {
   if (NameInfo.getName().getNameKind() == DeclarationName::CXXConstructorName)
     if (auto *RD = dyn_cast<CXXRecordDecl>(SemaRef.CurContext))
       NameInfo.setName(SemaRef.Context.DeclarationNames.getCXXConstructorName(
-          SemaRef.Context.getCanonicalType(SemaRef.Context.getRecordType(RD))));
+          SemaRef.Context.getCanonicalTagType(RD)));
 
   // We only need to do redeclaration lookups if we're in a class scope (in
   // fact, it's not really even possible in non-class scopes).
@@ -4014,8 +4002,9 @@ Decl *TemplateDeclInstantiator::instantiateUnresolvedUsingDecl(
     bool RetainExpansion = false;
     UnsignedOrNone NumExpansions = std::nullopt;
     if (SemaRef.CheckParameterPacksForExpansion(
-          D->getEllipsisLoc(), D->getSourceRange(), Unexpanded, TemplateArgs,
-            Expand, RetainExpansion, NumExpansions))
+            D->getEllipsisLoc(), D->getSourceRange(), Unexpanded, TemplateArgs,
+            /*FailOnPackProducingTemplates=*/true, Expand, RetainExpansion,
+            NumExpansions))
       return nullptr;
 
     // This declaration cannot appear within a function template signature,
@@ -4819,18 +4808,13 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
       ClassTemplate->findPartialSpecialization(CTAI.CanonicalConverted,
                                                InstParams, InsertPos);
 
-  // Build the type that describes the converted template arguments of the class
-  // template partial specialization.
-  TypeSourceInfo *WrittenTy = SemaRef.Context.getTemplateSpecializationTypeInfo(
-      TemplateName(ClassTemplate), TemplArgInfo->getLAngleLoc(),
-      InstTemplateArgs, CTAI.CanonicalConverted);
-
   // Create the class template partial specialization declaration.
   ClassTemplatePartialSpecializationDecl *InstPartialSpec =
       ClassTemplatePartialSpecializationDecl::Create(
           SemaRef.Context, PartialSpec->getTagKind(), Owner,
           PartialSpec->getBeginLoc(), PartialSpec->getLocation(), InstParams,
-          ClassTemplate, CTAI.CanonicalConverted, WrittenTy->getType(),
+          ClassTemplate, CTAI.CanonicalConverted,
+          /*CanonInjectedTST=*/CanQualType(),
           /*PrevDecl=*/nullptr);
 
   InstPartialSpec->setTemplateArgsAsWritten(InstTemplateArgs);
@@ -4861,7 +4845,7 @@ TemplateDeclInstantiator::InstantiateClassTemplatePartialSpecialization(
                  diag::err_partial_spec_redeclared)
         << InstPartialSpec;
     SemaRef.Diag(PrevDecl->getLocation(), diag::note_prev_partial_spec_here)
-      << SemaRef.Context.getTypeDeclType(PrevDecl);
+        << SemaRef.Context.getCanonicalTagType(PrevDecl);
     return nullptr;
   }
 
@@ -5685,7 +5669,7 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   };
   Function->setDeclarationNameLoc(NameLocPointsToPattern());
 
-  EnterExpressionEvaluationContext EvalContext(
+  EnterExpressionEvaluationContextForFunction EvalContext(
       *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
 
   Qualifiers ThisTypeQuals;
@@ -5750,15 +5734,19 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
       QualType TransformRecordType(TypeLocBuilder &TLB, RecordTypeLoc TL) {
         const RecordType *T = TL.getTypePtr();
         RecordDecl *Record = cast_or_null<RecordDecl>(
-            getDerived().TransformDecl(TL.getNameLoc(), T->getDecl()));
+            getDerived().TransformDecl(TL.getNameLoc(), T->getOriginalDecl()));
         if (Record != OldDecl)
           return Base::TransformRecordType(TLB, TL);
 
-        QualType Result = getDerived().RebuildRecordType(NewDecl);
+        // FIXME: transform the rest of the record type.
+        QualType Result = getDerived().RebuildTagType(
+            ElaboratedTypeKeyword::None, /*Qualifier=*/std::nullopt, NewDecl);
         if (Result.isNull())
           return QualType();
 
-        RecordTypeLoc NewTL = TLB.push<RecordTypeLoc>(Result);
+        TagTypeLoc NewTL = TLB.push<RecordTypeLoc>(Result);
+        NewTL.setElaboratedKeywordLoc(SourceLocation());
+        NewTL.setQualifierLoc(NestedNameSpecifierLoc());
         NewTL.setNameLoc(TL.getNameLoc());
         return Result;
       }
@@ -6413,12 +6401,10 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
       bool ShouldExpand = false;
       bool RetainExpansion = false;
       UnsignedOrNone NumExpansions = std::nullopt;
-      if (CheckParameterPacksForExpansion(Init->getEllipsisLoc(),
-                                          BaseTL.getSourceRange(),
-                                          Unexpanded,
-                                          TemplateArgs, ShouldExpand,
-                                          RetainExpansion,
-                                          NumExpansions)) {
+      if (CheckParameterPacksForExpansion(
+              Init->getEllipsisLoc(), BaseTL.getSourceRange(), Unexpanded,
+              TemplateArgs, /*FailOnPackProducingTemplates=*/true, ShouldExpand,
+              RetainExpansion, NumExpansions)) {
         AnyErrors = true;
         New->setInvalidDecl();
         continue;
@@ -6922,7 +6908,8 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
               Args.addArgument(
                   getTrivialTemplateArgumentLoc(UnpackedArg, QualType(), Loc));
           }
-          QualType T = CheckTemplateIdType(TemplateName(TD), Loc, Args);
+          QualType T = CheckTemplateIdType(ElaboratedTypeKeyword::None,
+                                           TemplateName(TD), Loc, Args);
           // We may get a non-null type with errors, in which case
           // `getAsCXXRecordDecl` will return `nullptr`. For instance, this
           // happens when one of the template arguments is an invalid
@@ -6988,16 +6975,17 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
     bool IsBeingInstantiated = false;
     if (CXXRecordDecl *Spec = dyn_cast<CXXRecordDecl>(ParentDC)) {
       if (!Spec->isDependentContext()) {
-        QualType T = Context.getTypeDeclType(Spec);
+        CanQualType T = Context.getCanonicalTagType(Spec);
         const RecordType *Tag = T->getAs<RecordType>();
         assert(Tag && "type of non-dependent record is not a RecordType");
-        if (Tag->isBeingDefined())
+        auto *TagDecl =
+            cast<CXXRecordDecl>(Tag->getOriginalDecl())->getDefinitionOrSelf();
+        if (TagDecl->isBeingDefined())
           IsBeingInstantiated = true;
-        if (!Tag->isBeingDefined() &&
-            RequireCompleteType(Loc, T, diag::err_incomplete_type))
+        else if (RequireCompleteType(Loc, T, diag::err_incomplete_type))
           return nullptr;
 
-        ParentDC = Tag->getDecl();
+        ParentDC = TagDecl;
       }
     }
 
@@ -7043,8 +7031,8 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
         // of member classes, and introduces ordering dependencies via
         // template instantiation.
         Diag(Loc, diag::err_member_not_yet_instantiated)
-          << D->getDeclName()
-          << Context.getTypeDeclType(cast<CXXRecordDecl>(ParentDC));
+            << D->getDeclName()
+            << Context.getCanonicalTagType(cast<CXXRecordDecl>(ParentDC));
         Diag(D->getLocation(), diag::note_non_instantiated_member_here);
       } else if (EnumConstantDecl *ED = dyn_cast<EnumConstantDecl>(D)) {
         // This enumeration constant was found when the template was defined,
@@ -7059,7 +7047,7 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
           << D->getDeclName()
           << Context.getTypeDeclType(cast<TypeDecl>(Spec->getDeclContext()));
         Diag(Spec->getLocation(), diag::note_enum_specialized_here)
-          << Context.getTypeDeclType(Spec);
+            << Context.getCanonicalTagType(Spec);
       } else {
         // We should have found something, but didn't.
         llvm_unreachable("Unable to find instantiation of declaration!");
diff --git a/clang/lib/Sema/SemaTemplateVariadic.cpp b/clang/lib/Sema/SemaTemplateVariadic.cpp
index d2baa2efb6121..0f72d6a13ae06 100644
--- a/clang/lib/Sema/SemaTemplateVariadic.cpp
+++ b/clang/lib/Sema/SemaTemplateVariadic.cpp
@@ -65,6 +65,41 @@ class CollectUnexpandedParameterPacksVisitor
         Unexpanded.push_back({T, Loc});
     }
 
+    bool addUnexpanded(const SubstBuiltinTemplatePackType *T,
+                       SourceLocation Loc = SourceLocation()) {
+      Unexpanded.push_back({T, Loc});
+      return true;
+    }
+
+    bool addUnexpanded(const TemplateSpecializationType *T,
+                       SourceLocation Loc = SourceLocation()) {
+      assert(T->isCanonicalUnqualified() &&
+             isPackProducingBuiltinTemplateName(T->getTemplateName()));
+      Unexpanded.push_back({T, Loc});
+      return true;
+    }
+
+    /// Returns true iff it handled the traversal. On false, the callers must
+    /// traverse themselves.
+    bool
+    TryTraverseSpecializationProducingPacks(const TemplateSpecializationType *T,
+                                            SourceLocation Loc) {
+      if (!isPackProducingBuiltinTemplateName(T->getTemplateName()))
+        return false;
+      // Canonical types are inputs to the initial substitution. Report them and
+      // do not recurse any further.
+      if (T->isCanonicalUnqualified()) {
+        addUnexpanded(T, Loc);
+        return true;
+      }
+      // For sugared types, do not use the default traversal as it would be
+      // looking at (now irrelevant) template arguments. Instead, look at the
+      // result of substitution, it usually contains SubstPackType that needs to
+      // be expanded further.
+      DynamicRecursiveASTVisitor::TraverseType(T->desugar());
+      return true;
+    }
+
   public:
     explicit CollectUnexpandedParameterPacksVisitor(
         SmallVectorImpl<UnexpandedParameterPack> &Unexpanded)
@@ -123,6 +158,23 @@ class CollectUnexpandedParameterPacksVisitor
       return DynamicRecursiveASTVisitor::TraverseTemplateName(Template);
     }
 
+    bool
+    TraverseTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc T,
+                                          bool TraverseQualifier) override {
+      if (TryTraverseSpecializationProducingPacks(T.getTypePtr(),
+                                                  T.getBeginLoc()))
+        return true;
+      return DynamicRecursiveASTVisitor::TraverseTemplateSpecializationTypeLoc(
+          T, TraverseQualifier);
+    }
+
+    bool TraverseTemplateSpecializationType(TemplateSpecializationType *T,
+                                            bool TraverseQualfier) override {
+      if (TryTraverseSpecializationProducingPacks(T, SourceLocation()))
+        return true;
+      return DynamicRecursiveASTVisitor::TraverseTemplateSpecializationType(T);
+    }
+
     /// Suppress traversal into Objective-C container literal
     /// elements that are pack expansions.
     bool TraverseObjCDictionaryLiteral(ObjCDictionaryLiteral *E) override {
@@ -155,21 +207,22 @@ class CollectUnexpandedParameterPacksVisitor
 
     /// Suppress traversal into types that do not contain
     /// unexpanded parameter packs.
-    bool TraverseType(QualType T) override {
+    bool TraverseType(QualType T, bool TraverseQualifier = true) override {
       if ((!T.isNull() && T->containsUnexpandedParameterPack()) ||
           InLambdaOrBlock)
-        return DynamicRecursiveASTVisitor::TraverseType(T);
+        return DynamicRecursiveASTVisitor::TraverseType(T, TraverseQualifier);
 
       return true;
     }
 
     /// Suppress traversal into types with location information
     /// that do not contain unexpanded parameter packs.
-    bool TraverseTypeLoc(TypeLoc TL) override {
+    bool TraverseTypeLoc(TypeLoc TL, bool TraverseQualifier = true) override {
       if ((!TL.getType().isNull() &&
            TL.getType()->containsUnexpandedParameterPack()) ||
           InLambdaOrBlock)
-        return DynamicRecursiveASTVisitor::TraverseTypeLoc(TL);
+        return DynamicRecursiveASTVisitor::TraverseTypeLoc(TL,
+                                                           TraverseQualifier);
 
       return true;
     }
@@ -195,10 +248,12 @@ class CollectUnexpandedParameterPacksVisitor
 
     /// Suppress traversal of pack expansion expressions and types.
     ///@{
-    bool TraversePackExpansionType(PackExpansionType *T) override {
+    bool TraversePackExpansionType(PackExpansionType *T,
+                                   bool TraverseQualifier) override {
       return true;
     }
-    bool TraversePackExpansionTypeLoc(PackExpansionTypeLoc TL) override {
+    bool TraversePackExpansionTypeLoc(PackExpansionTypeLoc TL,
+                                      bool TraverseQualifier) override {
       return true;
     }
     bool TraversePackExpansionExpr(PackExpansionExpr *E) override {
@@ -208,10 +263,12 @@ class CollectUnexpandedParameterPacksVisitor
     bool TraversePackIndexingExpr(PackIndexingExpr *E) override {
       return DynamicRecursiveASTVisitor::TraverseStmt(E->getIndexExpr());
     }
-    bool TraversePackIndexingType(PackIndexingType *E) override {
+    bool TraversePackIndexingType(PackIndexingType *E,
+                                  bool TraverseQualifier) override {
       return DynamicRecursiveASTVisitor::TraverseStmt(E->getIndexExpr());
     }
-    bool TraversePackIndexingTypeLoc(PackIndexingTypeLoc TL) override {
+    bool TraversePackIndexingTypeLoc(PackIndexingTypeLoc TL,
+                                     bool TraverseQualifier) override {
       return DynamicRecursiveASTVisitor::TraverseStmt(TL.getIndexExpr());
     }
 
@@ -320,6 +377,14 @@ class CollectUnexpandedParameterPacksVisitor
       return DynamicRecursiveASTVisitor::TraverseUnresolvedLookupExpr(E);
     }
 
+    bool TraverseSubstBuiltinTemplatePackType(SubstBuiltinTemplatePackType *T,
+                                              bool TraverseQualifier) override {
+      addUnexpanded(T);
+      // Do not call into base implementation to supress traversal of the
+      // substituted types.
+      return true;
+    }
+
 #ifndef NDEBUG
     bool TraverseFunctionParmPackExpr(FunctionParmPackExpr *) override {
       ContainsIntermediatePacks = true;
@@ -525,8 +590,7 @@ bool Sema::DiagnoseUnexpandedParameterPack(const CXXScopeSpec &SS,
   // C++0x [temp.variadic]p5:
   //   An appearance of a name of a parameter pack that is not expanded is
   //   ill-formed.
-  if (!SS.getScopeRep() ||
-      !SS.getScopeRep()->containsUnexpandedParameterPack())
+  if (!SS.getScopeRep().containsUnexpandedParameterPack())
     return false;
 
   SmallVector<UnexpandedParameterPack, 2> Unexpanded;
@@ -641,6 +705,23 @@ void Sema::collectUnexpandedParameterPacks(
   CollectUnexpandedParameterPacksVisitor(Unexpanded).TraverseStmt(E);
 }
 
+ParsedTemplateArgument
+Sema::ActOnTemplateTemplateArgument(const ParsedTemplateArgument &Arg) {
+  if (Arg.isInvalid())
+    return Arg;
+
+  // We do not allow to reference builtin templates that produce multiple
+  // values, they would not have a well-defined semantics outside template
+  // arguments.
+  auto *T = dyn_cast_or_null<BuiltinTemplateDecl>(
+      Arg.getAsTemplate().get().getAsTemplateDecl());
+  if (T && T->isPackProducingBuiltinTemplate())
+    diagnoseMissingTemplateArguments(Arg.getAsTemplate().get(),
+                                     Arg.getNameLoc());
+
+  return Arg;
+}
+
 ParsedTemplateArgument
 Sema::ActOnPackExpansion(const ParsedTemplateArgument &Arg,
                          SourceLocation EllipsisLoc) {
@@ -654,7 +735,7 @@ Sema::ActOnPackExpansion(const ParsedTemplateArgument &Arg,
       return ParsedTemplateArgument();
 
     return ParsedTemplateArgument(Arg.getKind(), Result.get().getAsOpaquePtr(),
-                                  Arg.getLocation());
+                                  Arg.getNameLoc());
   }
 
   case ParsedTemplateArgument::NonType: {
@@ -663,12 +744,12 @@ Sema::ActOnPackExpansion(const ParsedTemplateArgument &Arg,
       return ParsedTemplateArgument();
 
     return ParsedTemplateArgument(Arg.getKind(), Result.get(),
-                                  Arg.getLocation());
+                                  Arg.getNameLoc());
   }
 
   case ParsedTemplateArgument::Template:
     if (!Arg.getAsTemplate().get().containsUnexpandedParameterPack()) {
-      SourceRange R(Arg.getLocation());
+      SourceRange R(Arg.getNameLoc());
       if (Arg.getScopeSpec().isValid())
         R.setBegin(Arg.getScopeSpec().getBeginLoc());
       Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
@@ -727,7 +808,7 @@ QualType Sema::CheckPackExpansion(QualType Pattern, SourceRange PatternRange,
   if (!Pattern->containsUnexpandedParameterPack() &&
       !Pattern->getContainedDeducedType()) {
     Diag(EllipsisLoc, diag::err_pack_expansion_without_parameter_packs)
-      << PatternRange;
+        << PatternRange;
     return QualType();
   }
 
@@ -761,7 +842,8 @@ ExprResult Sema::CheckPackExpansion(Expr *Pattern, SourceLocation EllipsisLoc,
 bool Sema::CheckParameterPacksForExpansion(
     SourceLocation EllipsisLoc, SourceRange PatternRange,
     ArrayRef<UnexpandedParameterPack> Unexpanded,
-    const MultiLevelTemplateArgumentList &TemplateArgs, bool &ShouldExpand,
+    const MultiLevelTemplateArgumentList &TemplateArgs,
+    bool FailOnPackProducingTemplates, bool &ShouldExpand,
     bool &RetainExpansion, UnsignedOrNone &NumExpansions) {
   ShouldExpand = true;
   RetainExpansion = false;
@@ -777,12 +859,31 @@ bool Sema::CheckParameterPacksForExpansion(
     IdentifierInfo *Name;
     bool IsVarDeclPack = false;
     FunctionParmPackExpr *BindingPack = nullptr;
+    std::optional<unsigned> NumPrecomputedArguments;
 
-    if (const TemplateTypeParmType *TTP =
-            ParmPack.first.dyn_cast<const TemplateTypeParmType *>()) {
+    if (auto *TTP = ParmPack.first.dyn_cast<const TemplateTypeParmType *>()) {
       Depth = TTP->getDepth();
       Index = TTP->getIndex();
       Name = TTP->getIdentifier();
+    } else if (auto *TST =
+                   ParmPack.first
+                       .dyn_cast<const TemplateSpecializationType *>()) {
+      assert(isPackProducingBuiltinTemplateName(TST->getTemplateName()));
+      // Delay expansion, substitution is required to know the size.
+      ShouldExpand = false;
+      if (!FailOnPackProducingTemplates)
+        continue;
+
+      // It is not yet supported in certain contexts.
+      return Diag(PatternRange.getBegin().isValid() ? PatternRange.getBegin()
+                                                    : EllipsisLoc,
+                  diag::err_unsupported_builtin_template_pack_expansion)
+             << TST->getTemplateName();
+    } else if (auto *S =
+                   ParmPack.first
+                       .dyn_cast<const SubstBuiltinTemplatePackType *>()) {
+      Name = nullptr;
+      NumPrecomputedArguments = S->getNumArgs();
     } else {
       NamedDecl *ND = cast<NamedDecl *>(ParmPack.first);
       if (isa<VarDecl>(ND))
@@ -822,6 +923,8 @@ bool Sema::CheckParameterPacksForExpansion(
       }
     } else if (BindingPack) {
       NewPackSize = BindingPack->getNumExpansions();
+    } else if (NumPrecomputedArguments) {
+      NewPackSize = *NumPrecomputedArguments;
     } else {
       // If we don't have a template argument at this depth/index, then we
       // cannot expand the pack expansion. Make a note of this, but we still
@@ -963,6 +1066,21 @@ UnsignedOrNone Sema::getNumArgumentsInExpansionFromUnexpanded(
             Unexpanded[I].first.dyn_cast<const TemplateTypeParmType *>()) {
       Depth = TTP->getDepth();
       Index = TTP->getIndex();
+    } else if (auto *TST =
+                   Unexpanded[I]
+                       .first.dyn_cast<const TemplateSpecializationType *>()) {
+      // This is a dependent pack, we are not ready to expand it yet.
+      assert(isPackProducingBuiltinTemplateName(TST->getTemplateName()));
+      (void)TST;
+      return std::nullopt;
+    } else if (auto *PST =
+                   Unexpanded[I]
+                       .first
+                       .dyn_cast<const SubstBuiltinTemplatePackType *>()) {
+      assert((!Result || *Result == PST->getNumArgs()) &&
+             "inconsistent pack sizes");
+      Result = PST->getNumArgs();
+      continue;
     } else {
       NamedDecl *ND = cast<NamedDecl *>(Unexpanded[I].first);
       if (isa<VarDecl>(ND)) {
@@ -1115,8 +1233,7 @@ bool Sema::containsUnexpandedParameterPacks(Declarator &D) {
       break;
 
     case DeclaratorChunk::MemberPointer:
-      if (Chunk.Mem.Scope().getScopeRep() &&
-          Chunk.Mem.Scope().getScopeRep()->containsUnexpandedParameterPack())
+      if (Chunk.Mem.Scope().getScopeRep().containsUnexpandedParameterPack())
         return true;
       break;
     }
@@ -1300,9 +1417,9 @@ TemplateArgumentLoc Sema::getTemplateArgumentPackExpansionPattern(
   case TemplateArgument::TemplateExpansion:
     Ellipsis = OrigLoc.getTemplateEllipsisLoc();
     NumExpansions = Argument.getNumTemplateExpansions();
-    return TemplateArgumentLoc(Context, Argument.getPackExpansionPattern(),
-                               OrigLoc.getTemplateQualifierLoc(),
-                               OrigLoc.getTemplateNameLoc());
+    return TemplateArgumentLoc(
+        Context, Argument.getPackExpansionPattern(), OrigLoc.getTemplateKWLoc(),
+        OrigLoc.getTemplateQualifierLoc(), OrigLoc.getTemplateNameLoc());
 
   case TemplateArgument::Declaration:
   case TemplateArgument::NullPtr:
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 1289bede3f192..d745cdbf0526f 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -156,6 +156,7 @@ static void diagnoseBadTypeAttribute(Sema &S, const ParsedAttr &attr,
   case ParsedAttr::AT_Allocating:                                              \
   case ParsedAttr::AT_Regparm:                                                 \
   case ParsedAttr::AT_CFIUncheckedCallee:                                      \
+  case ParsedAttr::AT_CFISalt:                                                 \
   case ParsedAttr::AT_CmseNSCall:                                              \
   case ParsedAttr::AT_ArmStreaming:                                            \
   case ParsedAttr::AT_ArmStreamingCompatible:                                  \
@@ -1211,14 +1212,11 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
            DS.getTypeSpecSign() == TypeSpecifierSign::Unspecified &&
            "No qualifiers on tag names!");
 
+    ElaboratedTypeKeyword Keyword =
+        KeywordHelpers::getKeywordForTypeSpec(DS.getTypeSpecType());
     // TypeQuals handled by caller.
-    Result = Context.getTypeDeclType(D);
-
-    // In both C and C++, make an ElaboratedType.
-    ElaboratedTypeKeyword Keyword
-      = ElaboratedType::getKeywordForTypeSpec(DS.getTypeSpecType());
-    Result = S.getElaboratedType(Keyword, DS.getTypeSpecScope(), Result,
-                                 DS.isTypeSpecOwned() ? D : nullptr);
+    Result = Context.getTagType(Keyword, DS.getTypeSpecScope().getScopeRep(), D,
+                                DS.isTypeSpecOwned());
     break;
   }
   case DeclSpec::TST_typename: {
@@ -1241,7 +1239,7 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
     assert(!Result.isNull() && "Didn't get a type for typeof?");
     if (!Result->isDependentType())
       if (const TagType *TT = Result->getAs<TagType>())
-        S.DiagnoseUseOfDecl(TT->getDecl(), DS.getTypeSpecTypeLoc());
+        S.DiagnoseUseOfDecl(TT->getOriginalDecl(), DS.getTypeSpecTypeLoc());
     // TypeQuals handled by caller.
     Result = Context.getTypeOfType(
         Result, DS.getTypeSpecType() == DeclSpec::TST_typeof_unqualType
@@ -2085,7 +2083,7 @@ QualType Sema::BuildArrayType(QualType T, ArraySizeModifier ASM,
     // an inheritance model, even if it's inside an unused typedef.
     if (Context.getTargetInfo().getCXXABI().isMicrosoft())
       if (const MemberPointerType *MPTy = T->getAs<MemberPointerType>())
-        if (!MPTy->getQualifier()->isDependent())
+        if (!MPTy->getQualifier().isDependent())
           (void)isCompleteType(Loc, T);
 
   } else {
@@ -2120,7 +2118,9 @@ QualType Sema::BuildArrayType(QualType T, ArraySizeModifier ASM,
   if (const RecordType *EltTy = T->getAs<RecordType>()) {
     // If the element type is a struct or union that contains a variadic
     // array, accept it as a GNU extension: C99 6.7.2.1p2.
-    if (EltTy->getDecl()->hasFlexibleArrayMember())
+    if (EltTy->getOriginalDecl()
+            ->getDefinitionOrSelf()
+            ->hasFlexibleArrayMember())
       Diag(Loc, diag::ext_flexible_array_in_array) << T;
   } else if (T->isObjCObjectType()) {
     Diag(Loc, diag::err_objc_array_of_interfaces) << T;
@@ -3460,7 +3460,7 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
 
     if (DiagID != 0) {
       SemaRef.Diag(OwnedTagDecl->getLocation(), DiagID)
-          << SemaRef.Context.getTypeDeclType(OwnedTagDecl);
+          << SemaRef.Context.getCanonicalTagType(OwnedTagDecl);
       D.setInvalidType(true);
     }
   }
@@ -3653,11 +3653,22 @@ static void warnAboutRedundantParens(Sema &S, Declarator &D, QualType T) {
     // here: even (e.g.) "int ::x" is visually ambiguous even though it's
     // formally unambiguous.
     if (StartsWithDeclaratorId && D.getCXXScopeSpec().isValid()) {
-      for (NestedNameSpecifier *NNS = D.getCXXScopeSpec().getScopeRep(); NNS;
-           NNS = NNS->getPrefix()) {
-        if (NNS->getKind() == NestedNameSpecifier::Global)
+      NestedNameSpecifier NNS = D.getCXXScopeSpec().getScopeRep();
+      for (;;) {
+        switch (NNS.getKind()) {
+        case NestedNameSpecifier::Kind::Global:
           return;
+        case NestedNameSpecifier::Kind::Type:
+          NNS = NNS.getAsType()->getPrefix();
+          continue;
+        case NestedNameSpecifier::Kind::Namespace:
+          NNS = NNS.getAsNamespaceAndPrefix().Prefix;
+          continue;
+        default:
+          goto out;
+        }
       }
+    out:;
     }
 
     S.Diag(Paren.Loc, diag::warn_redundant_parens_around_declarator)
@@ -3965,7 +3976,8 @@ classifyPointerDeclarator(Sema &S, QualType type, Declarator &declarator,
       return PointerDeclaratorKind::NonPointer;
 
     if (auto recordType = type->getAs<RecordType>()) {
-      RecordDecl *recordDecl = recordType->getDecl();
+      RecordDecl *recordDecl =
+          recordType->getOriginalDecl()->getDefinitionOrSelf();
 
       // If this is CFErrorRef*, report it as such.
       if (numNormalPointers == 2 && numTypeSpecifierPointers < 2 &&
@@ -5101,7 +5113,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
         //   Types shall not be defined in return or parameter types.
         TagDecl *Tag = cast<TagDecl>(D.getDeclSpec().getRepAsDecl());
         S.Diag(Tag->getLocation(), diag::err_type_defined_in_result_type)
-          << Context.getTypeDeclType(Tag);
+            << Context.getCanonicalTagType(Tag);
       }
 
       // Exception specs are not allowed in typedefs. Complain, but add it
@@ -5319,7 +5331,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
                   state.getDeclarator()
                           .getCXXScopeSpec()
                           .getScopeRep()
-                          ->getKind() == NestedNameSpecifier::TypeSpec) ||
+                          .getKind() == NestedNameSpecifier::Kind::Type) ||
                  state.getDeclarator().getContext() ==
                      DeclaratorContext::Member ||
                  state.getDeclarator().getContext() ==
@@ -5898,7 +5910,49 @@ namespace {
     //   int __attr * __attr * __attr *p;
     void VisitPointerTypeLoc(PointerTypeLoc TL) { Visit(TL.getNextTypeLoc()); }
     void VisitTypedefTypeLoc(TypedefTypeLoc TL) {
-      TL.setNameLoc(DS.getTypeSpecTypeLoc());
+      if (DS.getTypeSpecType() == TST_typename) {
+        TypeSourceInfo *TInfo = nullptr;
+        Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
+        if (TInfo) {
+          TL.copy(TInfo->getTypeLoc().castAs<TypedefTypeLoc>());
+          return;
+        }
+      }
+      TL.set(TL.getTypePtr()->getKeyword() != ElaboratedTypeKeyword::None
+                 ? DS.getTypeSpecTypeLoc()
+                 : SourceLocation(),
+             DS.getTypeSpecScope().getWithLocInContext(Context),
+             DS.getTypeSpecTypeNameLoc());
+    }
+    void VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc TL) {
+      if (DS.getTypeSpecType() == TST_typename) {
+        TypeSourceInfo *TInfo = nullptr;
+        Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
+        if (TInfo) {
+          TL.copy(TInfo->getTypeLoc().castAs<UnresolvedUsingTypeLoc>());
+          return;
+        }
+      }
+      TL.set(TL.getTypePtr()->getKeyword() != ElaboratedTypeKeyword::None
+                 ? DS.getTypeSpecTypeLoc()
+                 : SourceLocation(),
+             DS.getTypeSpecScope().getWithLocInContext(Context),
+             DS.getTypeSpecTypeNameLoc());
+    }
+    void VisitUsingTypeLoc(UsingTypeLoc TL) {
+      if (DS.getTypeSpecType() == TST_typename) {
+        TypeSourceInfo *TInfo = nullptr;
+        Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
+        if (TInfo) {
+          TL.copy(TInfo->getTypeLoc().castAs<UsingTypeLoc>());
+          return;
+        }
+      }
+      TL.set(TL.getTypePtr()->getKeyword() != ElaboratedTypeKeyword::None
+                 ? DS.getTypeSpecTypeLoc()
+                 : SourceLocation(),
+             DS.getTypeSpecScope().getWithLocInContext(Context),
+             DS.getTypeSpecTypeNameLoc());
     }
     void VisitObjCInterfaceTypeLoc(ObjCInterfaceTypeLoc TL) {
       TL.setNameLoc(DS.getTypeSpecTypeLoc());
@@ -5929,16 +5983,9 @@ namespace {
       }
 
       TypeLoc OldTL = TInfo->getTypeLoc();
-      if (TInfo->getType()->getAs<ElaboratedType>()) {
-        ElaboratedTypeLoc ElabTL = OldTL.castAs<ElaboratedTypeLoc>();
-        TemplateSpecializationTypeLoc NamedTL = ElabTL.getNamedTypeLoc()
-            .castAs<TemplateSpecializationTypeLoc>();
-        TL.copy(NamedTL);
-      } else {
-        TL.copy(OldTL.castAs<TemplateSpecializationTypeLoc>());
-        assert(TL.getRAngleLoc() == OldTL.castAs<TemplateSpecializationTypeLoc>().getRAngleLoc());
-      }
-
+      TL.copy(OldTL.castAs<TemplateSpecializationTypeLoc>());
+      assert(TL.getRAngleLoc() ==
+             OldTL.castAs<TemplateSpecializationTypeLoc>().getRAngleLoc());
     }
     void VisitTypeOfExprTypeLoc(TypeOfExprTypeLoc TL) {
       assert(DS.getTypeSpecType() == DeclSpec::TST_typeofExpr ||
@@ -5987,24 +6034,6 @@ namespace {
           TL.expandBuiltinRange(DS.getTypeSpecWidthRange());
       }
     }
-    void VisitElaboratedTypeLoc(ElaboratedTypeLoc TL) {
-      if (DS.getTypeSpecType() == TST_typename) {
-        TypeSourceInfo *TInfo = nullptr;
-        Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
-        if (TInfo)
-          if (auto ETL = TInfo->getTypeLoc().getAs<ElaboratedTypeLoc>()) {
-            TL.copy(ETL);
-            return;
-          }
-      }
-      const ElaboratedType *T = TL.getTypePtr();
-      TL.setElaboratedKeywordLoc(T->getKeyword() != ElaboratedTypeKeyword::None
-                                     ? DS.getTypeSpecTypeLoc()
-                                     : SourceLocation());
-      const CXXScopeSpec& SS = DS.getTypeSpecScope();
-      TL.setQualifierLoc(SS.getWithLocInContext(Context));
-      Visit(TL.getNextTypeLoc().getUnqualifiedLoc());
-    }
     void VisitDependentNameTypeLoc(DependentNameTypeLoc TL) {
       assert(DS.getTypeSpecType() == TST_typename);
       TypeSourceInfo *TInfo = nullptr;
@@ -6063,7 +6092,29 @@ namespace {
           ASTTemplateArgumentListInfo::Create(Context, TemplateArgsInfo));
       TL.setConceptReference(CR);
     }
+    void VisitDeducedTemplateSpecializationTypeLoc(
+        DeducedTemplateSpecializationTypeLoc TL) {
+      assert(DS.getTypeSpecType() == TST_typename);
+      TypeSourceInfo *TInfo = nullptr;
+      Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
+      assert(TInfo);
+      TL.copy(
+          TInfo->getTypeLoc().castAs<DeducedTemplateSpecializationTypeLoc>());
+    }
     void VisitTagTypeLoc(TagTypeLoc TL) {
+      if (DS.getTypeSpecType() == TST_typename) {
+        TypeSourceInfo *TInfo = nullptr;
+        Sema::GetTypeFromParser(DS.getRepAsType(), &TInfo);
+        if (TInfo) {
+          TL.copy(TInfo->getTypeLoc().castAs<TagTypeLoc>());
+          return;
+        }
+      }
+      TL.setElaboratedKeywordLoc(TL.getTypePtr()->getKeyword() !=
+                                         ElaboratedTypeKeyword::None
+                                     ? DS.getTypeSpecTypeLoc()
+                                     : SourceLocation());
+      TL.setQualifierLoc(DS.getTypeSpecScope().getWithLocInContext(Context));
       TL.setNameLoc(DS.getTypeSpecTypeNameLoc());
     }
     void VisitAtomicTypeLoc(AtomicTypeLoc TL) {
@@ -7029,9 +7080,6 @@ static bool handleMSPointerTypeQualifierAttr(TypeProcessingState &State,
     if (const TypedefType *TT = dyn_cast<TypedefType>(Desugared)) {
       Desugared = TT->desugar();
       continue;
-    } else if (const ElaboratedType *ET = dyn_cast<ElaboratedType>(Desugared)) {
-      Desugared = ET->desugar();
-      continue;
     }
     const AttributedType *AT = dyn_cast<AttributedType>(Desugared);
     if (!AT)
@@ -7939,6 +7987,36 @@ static bool handleFunctionTypeAttr(TypeProcessingState &state, ParsedAttr &attr,
     return true;
   }
 
+  if (attr.getKind() == ParsedAttr::AT_CFISalt) {
+    if (attr.getNumArgs() != 1)
+      return true;
+
+    StringRef Argument;
+    if (!S.checkStringLiteralArgumentAttr(attr, 0, Argument))
+      return true;
+
+    // Delay if this is not a function type.
+    if (!unwrapped.isFunctionType())
+      return false;
+
+    const auto *FnTy = unwrapped.get()->getAs<FunctionProtoType>();
+    if (!FnTy) {
+      S.Diag(attr.getLoc(), diag::err_attribute_wrong_decl_type)
+          << attr << attr.isRegularKeywordAttribute()
+          << ExpectedFunctionWithProtoType;
+      attr.setInvalid();
+      return true;
+    }
+
+    FunctionProtoType::ExtProtoInfo EPI = FnTy->getExtProtoInfo();
+    EPI.ExtraAttributeInfo.CFISalt = Argument;
+
+    QualType newtype = S.Context.getFunctionType(FnTy->getReturnType(),
+                                                 FnTy->getParamTypes(), EPI);
+    type = unwrapped.wrap(S, newtype->getAs<FunctionType>());
+    return true;
+  }
+
   if (attr.getKind() == ParsedAttr::AT_ArmStreaming ||
       attr.getKind() == ParsedAttr::AT_ArmStreamingCompatible ||
       attr.getKind() == ParsedAttr::AT_ArmPreserves ||
@@ -9170,11 +9248,9 @@ bool Sema::RequireCompleteType(SourceLocation Loc, QualType T,
                                TypeDiagnoser &Diagnoser) {
   if (RequireCompleteTypeImpl(Loc, T, Kind, &Diagnoser))
     return true;
-  if (const TagType *Tag = T->getAs<TagType>()) {
-    if (!Tag->getDecl()->isCompleteDefinitionRequired()) {
-      Tag->getDecl()->setCompleteDefinitionRequired();
-      Consumer.HandleTagDeclRequiredDefinition(Tag->getDecl());
-    }
+  if (auto *TD = T->getAsTagDecl(); TD && !TD->isCompleteDefinitionRequired()) {
+    TD->setCompleteDefinitionRequired();
+    Consumer.HandleTagDeclRequiredDefinition(TD);
   }
   return false;
 }
@@ -9312,7 +9388,7 @@ bool Sema::hasReachableDefinition(NamedDecl *D, NamedDecl **Suggested,
 
 /// Locks in the inheritance model for the given class and all of its bases.
 static void assignInheritanceModel(Sema &S, CXXRecordDecl *RD) {
-  RD = RD->getMostRecentNonInjectedDecl();
+  RD = RD->getMostRecentDecl();
   if (!RD->hasAttr<MSInheritanceAttr>()) {
     MSInheritanceModel IM;
     bool BestCase = false;
@@ -9352,10 +9428,10 @@ bool Sema::RequireCompleteTypeImpl(SourceLocation Loc, QualType T,
   //  assert(!T->isDependentType() &&
   //         "Can't ask whether a dependent type is complete");
 
-  if (const MemberPointerType *MPTy = T->getAs<MemberPointerType>()) {
+  if (const auto *MPTy = dyn_cast<MemberPointerType>(T.getCanonicalType())) {
     if (CXXRecordDecl *RD = MPTy->getMostRecentCXXRecordDecl();
         RD && !RD->isDependentType()) {
-      QualType T = Context.getTypeDeclType(RD);
+      CanQualType T = Context.getCanonicalTagType(RD);
       if (getLangOpts().CompleteMemberPointers && !RD->isBeingDefined() &&
           RequireCompleteType(Loc, T, Kind, diag::err_memptr_incomplete))
         return true;
@@ -9492,10 +9568,10 @@ bool Sema::RequireCompleteTypeImpl(SourceLocation Loc, QualType T,
   // If the type was a forward declaration of a class/struct/union
   // type, produce a note.
   if (Tag && !Tag->isInvalidDecl() && !Tag->getLocation().isInvalid())
-    Diag(Tag->getLocation(),
-         Tag->isBeingDefined() ? diag::note_type_being_defined
-                               : diag::note_forward_declaration)
-      << Context.getTagDeclType(Tag);
+    Diag(Tag->getLocation(), Tag->isBeingDefined()
+                                 ? diag::note_type_being_defined
+                                 : diag::note_forward_declaration)
+        << Context.getCanonicalTagType(Tag);
 
   // If the Objective-C class was a forward declaration, produce a note.
   if (IFace && !IFace->isInvalidDecl() && !IFace->getLocation().isInvalid())
@@ -9550,7 +9626,8 @@ bool Sema::RequireLiteralType(SourceLocation Loc, QualType T,
   if (!RT)
     return true;
 
-  const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+  const CXXRecordDecl *RD =
+      cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
 
   // A partially-defined class type can't be a literal type, because a literal
   // class type must have a trivial destructor (which can't be checked until
@@ -9628,15 +9705,6 @@ bool Sema::RequireLiteralType(SourceLocation Loc, QualType T, unsigned DiagID) {
   return RequireLiteralType(Loc, T, Diagnoser);
 }
 
-QualType Sema::getElaboratedType(ElaboratedTypeKeyword Keyword,
-                                 const CXXScopeSpec &SS, QualType T,
-                                 TagDecl *OwnedTagDecl) {
-  if (T.isNull())
-    return T;
-  return Context.getElaboratedType(
-      Keyword, SS.isValid() ? SS.getScopeRep() : nullptr, T, OwnedTagDecl);
-}
-
 QualType Sema::BuildTypeofExprType(Expr *E, TypeOfKind Kind) {
   assert(!E->hasPlaceholderType() && "unexpected placeholder");
 
@@ -9647,7 +9715,7 @@ QualType Sema::BuildTypeofExprType(Expr *E, TypeOfKind Kind) {
   if (!E->isTypeDependent()) {
     QualType T = E->getType();
     if (const TagType *TT = T->getAs<TagType>())
-      DiagnoseUseOfDecl(TT->getDecl(), E->getExprLoc());
+      DiagnoseUseOfDecl(TT->getOriginalDecl(), E->getExprLoc());
   }
   return Context.getTypeOfExprType(E, Kind);
 }
@@ -9813,8 +9881,7 @@ QualType Sema::BuildPackIndexingType(QualType Pattern, Expr *IndexExpr,
 static QualType GetEnumUnderlyingType(Sema &S, QualType BaseType,
                                       SourceLocation Loc) {
   assert(BaseType->isEnumeralType());
-  EnumDecl *ED = BaseType->castAs<EnumType>()->getDecl();
-  assert(ED && "EnumType has no EnumDecl");
+  EnumDecl *ED = BaseType->castAs<EnumType>()->getOriginalDecl();
 
   S.DiagnoseUseOfDecl(ED, Loc);
 
diff --git a/clang/lib/Sema/SemaTypeTraits.cpp b/clang/lib/Sema/SemaTypeTraits.cpp
index 1d8687e4bf1c1..0b4d5916f8dc3 100644
--- a/clang/lib/Sema/SemaTypeTraits.cpp
+++ b/clang/lib/Sema/SemaTypeTraits.cpp
@@ -32,8 +32,7 @@ static CXXMethodDecl *LookupSpecialMemberFromXValue(Sema &SemaRef,
   RD = RD->getDefinition();
   SourceLocation LookupLoc = RD->getLocation();
 
-  CanQualType CanTy = SemaRef.getASTContext().getCanonicalType(
-      SemaRef.getASTContext().getTagDeclType(RD));
+  CanQualType CanTy = SemaRef.getASTContext().getCanonicalTagType(RD);
   DeclarationName Name;
   Expr *Arg = nullptr;
   unsigned NumArgs;
@@ -562,7 +561,8 @@ static bool HasNoThrowOperator(const RecordType *RT, OverloadedOperatorKind Op,
                                bool (CXXRecordDecl::*HasTrivial)() const,
                                bool (CXXRecordDecl::*HasNonTrivial)() const,
                                bool (CXXMethodDecl::*IsDesiredOp)() const) {
-  CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
+  CXXRecordDecl *RD =
+      cast<CXXRecordDecl>(RT->getOriginalDecl())->getDefinitionOrSelf();
   if ((RD->*HasTrivial)() && !(RD->*HasNonTrivial)())
     return true;
 
@@ -599,6 +599,7 @@ static bool HasNonDeletedDefaultedEqualityComparison(Sema &S,
   if (Decl->isLambda())
     return Decl->isCapturelessLambda();
 
+  CanQualType T = S.Context.getCanonicalTagType(Decl);
   {
     EnterExpressionEvaluationContext UnevaluatedContext(
         S, Sema::ExpressionEvaluationContext::Unevaluated);
@@ -606,10 +607,7 @@ static bool HasNonDeletedDefaultedEqualityComparison(Sema &S,
     Sema::ContextRAII TUContext(S, S.Context.getTranslationUnitDecl());
 
     // const ClassT& obj;
-    OpaqueValueExpr Operand(
-        KeyLoc,
-        Decl->getTypeForDecl()->getCanonicalTypeUnqualified().withConst(),
-        ExprValueKind::VK_LValue);
+    OpaqueValueExpr Operand(KeyLoc, T.withConst(), ExprValueKind::VK_LValue);
     UnresolvedSet<16> Functions;
     // obj == obj;
     S.LookupBinOp(S.TUScope, {}, BinaryOperatorKind::BO_EQ, Functions);
@@ -628,8 +626,7 @@ static bool HasNonDeletedDefaultedEqualityComparison(Sema &S,
       return false;
     if (!ParamT->isReferenceType() && !Decl->isTriviallyCopyable())
       return false;
-    if (ParamT.getNonReferenceType()->getUnqualifiedDesugaredType() !=
-        Decl->getTypeForDecl())
+    if (!S.Context.hasSameUnqualifiedType(ParamT.getNonReferenceType(), T))
       return false;
   }
 
@@ -1613,9 +1610,9 @@ bool Sema::BuiltinIsBaseOf(SourceLocation RhsTLoc, QualType LhsT,
 
   // Unions are never base classes, and never have base classes.
   // It doesn't matter if they are complete or not. See PR#41843
-  if (lhsRecord && lhsRecord->getDecl()->isUnion())
+  if (lhsRecord && lhsRecord->getOriginalDecl()->isUnion())
     return false;
-  if (rhsRecord && rhsRecord->getDecl()->isUnion())
+  if (rhsRecord && rhsRecord->getOriginalDecl()->isUnion())
     return false;
 
   if (lhsRecord == rhsRecord)
@@ -1629,8 +1626,8 @@ bool Sema::BuiltinIsBaseOf(SourceLocation RhsTLoc, QualType LhsT,
                           diag::err_incomplete_type_used_in_type_trait_expr))
     return false;
 
-  return cast<CXXRecordDecl>(rhsRecord->getDecl())
-      ->isDerivedFrom(cast<CXXRecordDecl>(lhsRecord->getDecl()));
+  return cast<CXXRecordDecl>(rhsRecord->getOriginalDecl())
+      ->isDerivedFrom(cast<CXXRecordDecl>(lhsRecord->getOriginalDecl()));
 }
 
 static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT,
@@ -1670,8 +1667,9 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT,
                                  diag::err_incomplete_type))
       return false;
 
-    return cast<CXXRecordDecl>(DerivedRecord->getDecl())
-        ->isVirtuallyDerivedFrom(cast<CXXRecordDecl>(BaseRecord->getDecl()));
+    return cast<CXXRecordDecl>(DerivedRecord->getOriginalDecl())
+        ->isVirtuallyDerivedFrom(
+            cast<CXXRecordDecl>(BaseRecord->getOriginalDecl()));
   }
   case BTT_IsSame:
     return Self.Context.hasSameType(LhsT, RhsT);
@@ -1966,6 +1964,7 @@ static std::optional<TypeTrait> StdNameToTypeTrait(StringRef Name) {
       .Case("is_empty", TypeTrait::UTT_IsEmpty)
       .Case("is_standard_layout", TypeTrait::UTT_IsStandardLayout)
       .Case("is_constructible", TypeTrait::TT_IsConstructible)
+      .Case("is_final", TypeTrait::UTT_IsFinal)
       .Default(std::nullopt);
 }
 
@@ -2018,11 +2017,10 @@ static ExtractedTypeTraitInfo ExtractTypeTraitFromExpression(const Expr *E) {
   // std::is_xxx<>::value
   if (const auto *VD = dyn_cast<VarDecl>(Ref->getDecl());
       Ref->hasQualifier() && VD && VD->getIdentifier()->isStr("value")) {
-    const Type *T = Ref->getQualifier()->getAsType();
-    if (!T)
+    NestedNameSpecifier Qualifier = Ref->getQualifier();
+    if (Qualifier.getKind() != NestedNameSpecifier::Kind::Type)
       return std::nullopt;
-    const TemplateSpecializationType *Ts =
-        T->getAs<TemplateSpecializationType>();
+    const auto *Ts = Qualifier.getAsType()->getAs<TemplateSpecializationType>();
     if (!Ts)
       return std::nullopt;
     const TemplateDecl *D = Ts->getTemplateName().getAsTemplateDecl();
@@ -2451,6 +2449,52 @@ static void DiagnoseIsEmptyReason(Sema &S, SourceLocation Loc, QualType T) {
   }
 }
 
+static void DiagnoseIsFinalReason(Sema &S, SourceLocation Loc,
+                                  const CXXRecordDecl *D) {
+  if (!D || D->isInvalidDecl())
+    return;
+
+  // Complete record but not 'final'.
+  if (!D->isEffectivelyFinal()) {
+    S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::NotMarkedFinal;
+    S.Diag(D->getLocation(), diag::note_defined_here) << D;
+    return;
+  }
+}
+
+static void DiagnoseIsFinalReason(Sema &S, SourceLocation Loc, QualType T) {
+  // Primary: “%0 is not final”
+  S.Diag(Loc, diag::note_unsatisfied_trait) << T << diag::TraitName::Final;
+  if (T->isReferenceType()) {
+    S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::Ref;
+    S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::NotClassOrUnion;
+    return;
+  }
+  // Arrays / functions / non-records → not a class/union.
+  if (S.Context.getAsArrayType(T)) {
+    S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::NotClassOrUnion;
+    return;
+  }
+  if (T->isFunctionType()) {
+    S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::FunctionType;
+    S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::NotClassOrUnion;
+    return;
+  }
+  if (!T->isRecordType()) {
+    S.Diag(Loc, diag::note_unsatisfied_trait_reason)
+        << diag::TraitNotSatisfiedReason::NotClassOrUnion;
+    return;
+  }
+  if (const auto *D = T->getAsCXXRecordDecl())
+    DiagnoseIsFinalReason(S, Loc, D);
+}
+
 static bool hasMultipleDataBaseClassesWithFields(const CXXRecordDecl *D) {
   int NumBasesWithFields = 0;
   for (const CXXBaseSpecifier &Base : D->bases()) {
@@ -2627,6 +2671,15 @@ void Sema::DiagnoseTypeTraitDetails(const Expr *E) {
   case TT_IsConstructible:
     DiagnoseNonConstructibleReason(*this, E->getBeginLoc(), Args);
     break;
+  case UTT_IsFinal: {
+    QualType QT = Args[0];
+    if (QT->isDependentType())
+      break;
+    const auto *RD = QT->getAsCXXRecordDecl();
+    if (!RD || !RD->isEffectivelyFinal())
+      DiagnoseIsFinalReason(*this, E->getBeginLoc(), QT); // unsatisfied
+    break;
+  }
   default:
     break;
   }
diff --git a/clang/lib/Sema/SemaWasm.cpp b/clang/lib/Sema/SemaWasm.cpp
index 8998492a71619..e7731136720e8 100644
--- a/clang/lib/Sema/SemaWasm.cpp
+++ b/clang/lib/Sema/SemaWasm.cpp
@@ -17,6 +17,7 @@
 #include "clang/Basic/AddressSpaces.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetBuiltins.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Attr.h"
 #include "clang/Sema/Sema.h"
 
@@ -227,7 +228,8 @@ bool SemaWasm::BuiltinWasmTableCopy(CallExpr *TheCall) {
   return false;
 }
 
-bool SemaWasm::BuiltinWasmTestFunctionPointerSignature(CallExpr *TheCall) {
+bool SemaWasm::BuiltinWasmTestFunctionPointerSignature(const TargetInfo &TI,
+                                                       CallExpr *TheCall) {
   if (SemaRef.checkArgCount(TheCall, 1))
     return true;
 
@@ -250,27 +252,31 @@ bool SemaWasm::BuiltinWasmTestFunctionPointerSignature(CallExpr *TheCall) {
            << ArgType << FuncPtrArg->getSourceRange();
   }
 
-  // Check that the function pointer doesn't use reference types
-  if (FuncTy->getReturnType().isWebAssemblyReferenceType()) {
-    return Diag(
-               FuncPtrArg->getBeginLoc(),
-               diag::err_wasm_builtin_test_fp_sig_cannot_include_reference_type)
-           << 0 << FuncTy->getReturnType() << FuncPtrArg->getSourceRange();
-  }
-  auto NParams = FuncTy->getNumParams();
-  for (unsigned I = 0; I < NParams; I++) {
-    if (FuncTy->getParamType(I).isWebAssemblyReferenceType()) {
+  if (TI.getABI() == "experimental-mv") {
+    auto isStructOrUnion = [](QualType T) {
+      return T->isUnionType() || T->isStructureType();
+    };
+    if (isStructOrUnion(FuncTy->getReturnType())) {
       return Diag(
                  FuncPtrArg->getBeginLoc(),
                  diag::
-                     err_wasm_builtin_test_fp_sig_cannot_include_reference_type)
-             << 1 << FuncPtrArg->getSourceRange();
+                     err_wasm_builtin_test_fp_sig_cannot_include_struct_or_union)
+             << 0 << FuncTy->getReturnType() << FuncPtrArg->getSourceRange();
+    }
+    auto NParams = FuncTy->getNumParams();
+    for (unsigned I = 0; I < NParams; I++) {
+      if (isStructOrUnion(FuncTy->getParamType(I))) {
+        return Diag(
+                   FuncPtrArg->getBeginLoc(),
+                   diag::
+                       err_wasm_builtin_test_fp_sig_cannot_include_struct_or_union)
+               << 1 << FuncPtrArg->getSourceRange();
+      }
     }
   }
 
   // Set return type to int (the result of the test)
   TheCall->setType(getASTContext().IntTy);
-
   return false;
 }
 
@@ -297,7 +303,7 @@ bool SemaWasm::CheckWebAssemblyBuiltinFunctionCall(const TargetInfo &TI,
   case WebAssembly::BI__builtin_wasm_table_copy:
     return BuiltinWasmTableCopy(TheCall);
   case WebAssembly::BI__builtin_wasm_test_function_pointer_signature:
-    return BuiltinWasmTestFunctionPointerSignature(TheCall);
+    return BuiltinWasmTestFunctionPointerSignature(TI, TheCall);
   }
 
   return false;
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 6ce5535c7f3f7..1d14ead778446 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -45,6 +45,7 @@
 #include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaPseudoObject.h"
 #include "clang/Sema/SemaSYCL.h"
+#include "clang/Sema/Template.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <algorithm>
@@ -55,6 +56,17 @@ using namespace llvm::omp;
 namespace clang {
 using namespace sema;
 
+// This helper class is used to facilitate pack expansion during tree transform.
+struct UnexpandedInfo {
+  SourceLocation Ellipsis;
+  UnsignedOrNone OrigNumExpansions = std::nullopt;
+
+  bool Expand = false;
+  bool RetainExpansion = false;
+  UnsignedOrNone NumExpansions = std::nullopt;
+  bool ExpandUnderForgetSubstitions = false;
+};
+
 /// A semantic tree transformation that allows one to transform one
 /// abstract syntax tree into another.
 ///
@@ -292,6 +304,7 @@ class TreeTransform {
   bool TryExpandParameterPacks(SourceLocation EllipsisLoc,
                                SourceRange PatternRange,
                                ArrayRef<UnexpandedParameterPack> Unexpanded,
+                               bool FailOnPackProducingTemplates,
                                bool &ShouldExpand, bool &RetainExpansion,
                                UnsignedOrNone &NumExpansions) {
     ShouldExpand = false;
@@ -314,6 +327,27 @@ class TreeTransform {
   /// This routine is meant to be overridden by the template instantiator.
   void RememberPartiallySubstitutedPack(TemplateArgument Arg) { }
 
+  /// "Forget" the template substitution to allow transforming the AST without
+  /// any template instantiations. This is used to expand template packs when
+  /// their size is not known in advance (e.g. for builtins that produce type
+  /// packs).
+  MultiLevelTemplateArgumentList ForgetSubstitution() { return {}; }
+  void RememberSubstitution(MultiLevelTemplateArgumentList) {}
+
+private:
+  struct ForgetSubstitutionRAII {
+    Derived &Self;
+    MultiLevelTemplateArgumentList Old;
+
+  public:
+    ForgetSubstitutionRAII(Derived &Self) : Self(Self) {
+      Old = Self.ForgetSubstitution();
+    }
+
+    ~ForgetSubstitutionRAII() { Self.RememberSubstitution(std::move(Old)); }
+  };
+
+public:
   /// Note to the derived class when a function parameter pack is
   /// being expanded.
   void ExpandingFunctionParameterPack(ParmVarDecl *Pack) { }
@@ -591,12 +625,12 @@ class TreeTransform {
   /// By default, transforms the template name by transforming the declarations
   /// and nested-name-specifiers that occur within the template name.
   /// Subclasses may override this function to provide alternate behavior.
-  TemplateName
-  TransformTemplateName(CXXScopeSpec &SS, TemplateName Name,
-                        SourceLocation NameLoc,
-                        QualType ObjectType = QualType(),
-                        NamedDecl *FirstQualifierInScope = nullptr,
-                        bool AllowInjectedClassName = false);
+  TemplateName TransformTemplateName(NestedNameSpecifierLoc &QualifierLoc,
+                                     SourceLocation TemplateKWLoc,
+                                     TemplateName Name, SourceLocation NameLoc,
+                                     QualType ObjectType = QualType(),
+                                     NamedDecl *FirstQualifierInScope = nullptr,
+                                     bool AllowInjectedClassName = false);
 
   /// Transform the given template argument.
   ///
@@ -610,9 +644,9 @@ class TreeTransform {
                                  TemplateArgumentLoc &Output,
                                  bool Uneval = false);
 
-  TemplateArgument
-  TransformNamedTemplateTemplateArgument(CXXScopeSpec &SS, TemplateName Name,
-                                         SourceLocation NameLoc);
+  TemplateArgument TransformNamedTemplateTemplateArgument(
+      NestedNameSpecifierLoc &QualifierLoc, SourceLocation TemplateKeywordLoc,
+      TemplateName Name, SourceLocation NameLoc);
 
   /// Transform the given set of template arguments.
   ///
@@ -660,6 +694,19 @@ class TreeTransform {
                                   TemplateArgumentListInfo &Outputs,
                                   bool Uneval = false);
 
+  /// Checks if the argument pack from \p In will need to be expanded and does
+  /// the necessary prework.
+  /// Whether the expansion is needed is captured in Info.Expand.
+  ///
+  /// - When the expansion is required, \p Out will be a template pattern that
+  ///   would need to be expanded.
+  /// - When the expansion must not happen, \p Out will be a pack that must be
+  ///   returned to the outputs directly.
+  ///
+  /// \return true iff the error occurred
+  bool PreparePackForExpansion(TemplateArgumentLoc In, bool Uneval,
+                               TemplateArgumentLoc &Out, UnexpandedInfo &Info);
+
   /// Fakes up a TemplateArgumentLoc for a given TemplateArgument.
   void InventTemplateArgumentLoc(const TemplateArgument &Arg,
                                  TemplateArgumentLoc &ArgLoc);
@@ -697,20 +744,17 @@ class TreeTransform {
 
   StmtResult TransformSEHHandler(Stmt *Handler);
 
-  QualType
-  TransformTemplateSpecializationType(TypeLocBuilder &TLB,
-                                      TemplateSpecializationTypeLoc TL,
-                                      TemplateName Template);
-
-  QualType
-  TransformDependentTemplateSpecializationType(TypeLocBuilder &TLB,
-                                      DependentTemplateSpecializationTypeLoc TL,
+  QualType TransformTemplateSpecializationType(TypeLocBuilder &TLB,
+                                               TemplateSpecializationTypeLoc TL,
                                                TemplateName Template,
                                                CXXScopeSpec &SS);
 
   QualType TransformDependentTemplateSpecializationType(
       TypeLocBuilder &TLB, DependentTemplateSpecializationTypeLoc TL,
-      CXXScopeSpec &SS);
+      QualType ObjectType, NamedDecl *UnqualLookup,
+      bool AllowInjectedClassName);
+
+  QualType TransformTagType(TypeLocBuilder &TLB, TagTypeLoc TL);
 
   /// Transforms the parameters of a function type into the
   /// given vectors.
@@ -1020,16 +1064,22 @@ class TreeTransform {
 
   /// Rebuild an unresolved typename type, given the decl that
   /// the UnresolvedUsingTypenameDecl was transformed to.
-  QualType RebuildUnresolvedUsingType(SourceLocation NameLoc, Decl *D);
+  QualType RebuildUnresolvedUsingType(ElaboratedTypeKeyword Keyword,
+                                      NestedNameSpecifier Qualifier,
+                                      SourceLocation NameLoc, Decl *D);
 
   /// Build a new type found via an alias.
-  QualType RebuildUsingType(UsingShadowDecl *Found, QualType Underlying) {
-    return SemaRef.Context.getUsingType(Found, Underlying);
+  QualType RebuildUsingType(ElaboratedTypeKeyword Keyword,
+                            NestedNameSpecifier Qualifier, UsingShadowDecl *D,
+                            QualType UnderlyingType) {
+    return SemaRef.Context.getUsingType(Keyword, Qualifier, D, UnderlyingType);
   }
 
   /// Build a new typedef type.
-  QualType RebuildTypedefType(TypedefNameDecl *Typedef) {
-    return SemaRef.Context.getTypeDeclType(Typedef);
+  QualType RebuildTypedefType(ElaboratedTypeKeyword Keyword,
+                              NestedNameSpecifier Qualifier,
+                              TypedefNameDecl *Typedef) {
+    return SemaRef.Context.getTypedefType(Keyword, Qualifier, Typedef);
   }
 
   /// Build a new MacroDefined type.
@@ -1038,14 +1088,14 @@ class TreeTransform {
     return SemaRef.Context.getMacroQualifiedType(T, MacroII);
   }
 
-  /// Build a new class/struct/union type.
-  QualType RebuildRecordType(RecordDecl *Record) {
-    return SemaRef.Context.getTypeDeclType(Record);
+  /// Build a new class/struct/union/enum type.
+  QualType RebuildTagType(ElaboratedTypeKeyword Keyword,
+                          NestedNameSpecifier Qualifier, TagDecl *Tag) {
+    return SemaRef.Context.getTagType(Keyword, Qualifier, Tag,
+                                      /*OwnsTag=*/false);
   }
-
-  /// Build a new Enum type.
-  QualType RebuildEnumType(EnumDecl *Enum) {
-    return SemaRef.Context.getTypeDeclType(Enum);
+  QualType RebuildCanonicalTagType(TagDecl *Tag) {
+    return SemaRef.Context.getCanonicalTagType(Tag);
   }
 
   /// Build a new typeof(expr) type.
@@ -1094,10 +1144,10 @@ class TreeTransform {
 
   /// By default, builds a new DeducedTemplateSpecializationType with the given
   /// deduced type.
-  QualType RebuildDeducedTemplateSpecializationType(TemplateName Template,
-      QualType Deduced) {
+  QualType RebuildDeducedTemplateSpecializationType(
+      ElaboratedTypeKeyword Keyword, TemplateName Template, QualType Deduced) {
     return SemaRef.Context.getDeducedTemplateSpecializationType(
-        Template, Deduced, /*IsDependent*/ false);
+        Keyword, Template, Deduced, /*IsDependent*/ false);
   }
 
   /// Build a new template specialization type.
@@ -1105,7 +1155,8 @@ class TreeTransform {
   /// By default, performs semantic analysis when building the template
   /// specialization type. Subclasses may override this routine to provide
   /// different behavior.
-  QualType RebuildTemplateSpecializationType(TemplateName Template,
+  QualType RebuildTemplateSpecializationType(ElaboratedTypeKeyword Keyword,
+                                             TemplateName Template,
                                              SourceLocation TemplateLoc,
                                              TemplateArgumentListInfo &Args);
 
@@ -1117,41 +1168,22 @@ class TreeTransform {
     return SemaRef.BuildParenType(InnerType);
   }
 
-  /// Build a new qualified name type.
-  ///
-  /// By default, builds a new ElaboratedType type from the keyword,
-  /// the nested-name-specifier and the named type.
-  /// Subclasses may override this routine to provide different behavior.
-  QualType RebuildElaboratedType(SourceLocation KeywordLoc,
-                                 ElaboratedTypeKeyword Keyword,
-                                 NestedNameSpecifierLoc QualifierLoc,
-                                 QualType Named) {
-    return SemaRef.Context.getElaboratedType(Keyword,
-                                         QualifierLoc.getNestedNameSpecifier(),
-                                             Named);
-  }
-
   /// Build a new typename type that refers to a template-id.
   ///
   /// By default, builds a new DependentNameType type from the
   /// nested-name-specifier and the given type. Subclasses may override
   /// this routine to provide different behavior.
   QualType RebuildDependentTemplateSpecializationType(
-      ElaboratedTypeKeyword Keyword, NestedNameSpecifier *NNS,
-      SourceLocation TemplateKWLoc, TemplateName Name, SourceLocation NameLoc,
-      TemplateArgumentListInfo &Args, bool AllowInjectedClassName) {
+      ElaboratedTypeKeyword Keyword, SourceLocation TemplateKWLoc,
+      TemplateName Name, SourceLocation NameLoc, TemplateArgumentListInfo &Args,
+      bool AllowInjectedClassName) {
     // If it's still dependent, make a dependent specialization.
     if (const DependentTemplateStorage *S = Name.getAsDependentTemplateName())
       return SemaRef.Context.getDependentTemplateSpecializationType(
           Keyword, *S, Args.arguments());
 
-    // Otherwise, make an elaborated type wrapping a non-dependent
-    // specialization.
-    QualType T =
-        getDerived().RebuildTemplateSpecializationType(Name, NameLoc, Args);
-    if (T.isNull())
-      return QualType();
-    return SemaRef.Context.getElaboratedType(Keyword, NNS, T);
+    return getDerived().RebuildTemplateSpecializationType(Keyword, Name,
+                                                          NameLoc, Args);
   }
 
   /// Build a new typename type that refers to an identifier.
@@ -1168,7 +1200,7 @@ class TreeTransform {
     CXXScopeSpec SS;
     SS.Adopt(QualifierLoc);
 
-    if (QualifierLoc.getNestedNameSpecifier()->isDependent()) {
+    if (QualifierLoc.getNestedNameSpecifier().isDependent()) {
       // If the name is still dependent, just build a new dependent name type.
       if (!SemaRef.computeDeclContext(SS))
         return SemaRef.Context.getDependentNameType(Keyword,
@@ -1238,19 +1270,14 @@ class TreeTransform {
       }
       return QualType();
     }
-
     if (!SemaRef.isAcceptableTagRedeclaration(Tag, Kind, /*isDefinition*/false,
                                               IdLoc, Id)) {
       SemaRef.Diag(KeywordLoc, diag::err_use_with_wrong_tag) << Id;
       SemaRef.Diag(Tag->getLocation(), diag::note_previous_use);
       return QualType();
     }
-
-    // Build the elaborated-type-specifier type.
-    QualType T = SemaRef.Context.getTypeDeclType(Tag);
-    return SemaRef.Context.getElaboratedType(Keyword,
-                                         QualifierLoc.getNestedNameSpecifier(),
-                                             T);
+    return getDerived().RebuildTagType(
+        Keyword, QualifierLoc.getNestedNameSpecifier(), Tag);
   }
 
   /// Build a new pack expansion type.
@@ -1303,7 +1330,6 @@ class TreeTransform {
                                    SourceLocation TemplateKWLoc,
                                    const IdentifierInfo &Name,
                                    SourceLocation NameLoc, QualType ObjectType,
-                                   NamedDecl *FirstQualifierInScope,
                                    bool AllowInjectedClassName);
 
   /// Build a new template name given a nested name specifier and the
@@ -1323,7 +1349,6 @@ class TreeTransform {
                                    SourceLocation TemplateKWLoc,
                                    IdentifierOrOverloadedOperator IO,
                                    SourceLocation NameLoc, QualType ObjectType,
-                                   NamedDecl *FirstQualifierInScope,
                                    bool AllowInjectedClassName);
 
   /// Build a new template name given a template template parameter pack
@@ -4012,8 +4037,8 @@ class TreeTransform {
           SemaRef.Context,
           TemplateArgument(Pattern.getArgument().getAsTemplate(),
                            NumExpansions),
-          Pattern.getTemplateQualifierLoc(), Pattern.getTemplateNameLoc(),
-          EllipsisLoc);
+          Pattern.getTemplateKWLoc(), Pattern.getTemplateQualifierLoc(),
+          Pattern.getTemplateNameLoc(), EllipsisLoc);
 
     case TemplateArgument::Null:
     case TemplateArgument::Integral:
@@ -4073,7 +4098,7 @@ class TreeTransform {
             PVD->getUninstantiatedDefaultArg()
                 ->containsUnexpandedParameterPack();
     }
-    return getSema().BuildLambdaExpr(StartLoc, EndLoc, LSI);
+    return getSema().BuildLambdaExpr(StartLoc, EndLoc);
   }
 
   /// Build an empty C++1z fold-expression with the given operator.
@@ -4258,23 +4283,29 @@ class TreeTransform {
   }
 
 private:
-  TypeLoc TransformTypeInObjectScope(TypeLoc TL,
-                                     QualType ObjectType,
-                                     NamedDecl *FirstQualifierInScope,
-                                     CXXScopeSpec &SS);
+  QualType TransformTypeInObjectScope(TypeLocBuilder &TLB, TypeLoc TL,
+                                      QualType ObjectType,
+                                      NamedDecl *FirstQualifierInScope);
 
   TypeSourceInfo *TransformTypeInObjectScope(TypeSourceInfo *TSInfo,
                                              QualType ObjectType,
-                                             NamedDecl *FirstQualifierInScope,
-                                             CXXScopeSpec &SS);
+                                             NamedDecl *FirstQualifierInScope) {
+    if (getDerived().AlreadyTransformed(TSInfo->getType()))
+      return TSInfo;
 
-  TypeSourceInfo *TransformTSIInObjectScope(TypeLoc TL, QualType ObjectType,
-                                            NamedDecl *FirstQualifierInScope,
-                                            CXXScopeSpec &SS);
+    TypeLocBuilder TLB;
+    QualType T = TransformTypeInObjectScope(TLB, TSInfo->getTypeLoc(),
+                                            ObjectType, FirstQualifierInScope);
+    if (T.isNull())
+      return nullptr;
+    return TLB.getTypeSourceInfo(SemaRef.Context, T);
+  }
 
   QualType TransformDependentNameType(TypeLocBuilder &TLB,
                                       DependentNameTypeLoc TL,
-                                      bool DeducibleTSTContext);
+                                      bool DeducibleTSTContext,
+                                      QualType ObjectType = QualType(),
+                                      NamedDecl *UnqualLookup = nullptr);
 
   llvm::SmallVector<OpenACCClause *>
   TransformOpenACCClauseList(OpenACCDirectiveKind DirKind,
@@ -4477,11 +4508,10 @@ bool TreeTransform<Derived>::TransformExprs(Expr *const *Inputs,
       bool RetainExpansion = false;
       UnsignedOrNone OrigNumExpansions = Expansion->getNumExpansions();
       UnsignedOrNone NumExpansions = OrigNumExpansions;
-      if (getDerived().TryExpandParameterPacks(Expansion->getEllipsisLoc(),
-                                               Pattern->getSourceRange(),
-                                               Unexpanded,
-                                               Expand, RetainExpansion,
-                                               NumExpansions))
+      if (getDerived().TryExpandParameterPacks(
+              Expansion->getEllipsisLoc(), Pattern->getSourceRange(),
+              Unexpanded, /*FailOnPackProducingTemplates=*/true, Expand,
+              RetainExpansion, NumExpansions))
         return true;
 
       if (!Expand) {
@@ -4603,7 +4633,7 @@ NestedNameSpecifierLoc TreeTransform<Derived>::TransformNestedNameSpecifierLoc(
 
   auto insertNNS = [&Qualifiers](NestedNameSpecifierLoc NNS) {
     for (NestedNameSpecifierLoc Qualifier = NNS; Qualifier;
-         Qualifier = Qualifier.getPrefix())
+         Qualifier = Qualifier.getAsNamespaceAndPrefix().Prefix)
       Qualifiers.push_back(Qualifier);
   };
   insertNNS(NNS);
@@ -4611,76 +4641,87 @@ NestedNameSpecifierLoc TreeTransform<Derived>::TransformNestedNameSpecifierLoc(
   CXXScopeSpec SS;
   while (!Qualifiers.empty()) {
     NestedNameSpecifierLoc Q = Qualifiers.pop_back_val();
-    NestedNameSpecifier *QNNS = Q.getNestedNameSpecifier();
-
-    switch (QNNS->getKind()) {
-    case NestedNameSpecifier::Identifier: {
-      Sema::NestedNameSpecInfo IdInfo(QNNS->getAsIdentifier(),
-                                      Q.getLocalBeginLoc(), Q.getLocalEndLoc(),
-                                      ObjectType);
-      if (SemaRef.BuildCXXNestedNameSpecifier(/*Scope=*/nullptr, IdInfo, false,
-                                              SS, FirstQualifierInScope, false))
-        return NestedNameSpecifierLoc();
-      break;
-    }
+    NestedNameSpecifier QNNS = Q.getNestedNameSpecifier();
+
+    switch (QNNS.getKind()) {
+    case NestedNameSpecifier::Kind::Null:
+      llvm_unreachable("unexpected null nested name specifier");
 
-    case NestedNameSpecifier::Namespace: {
+    case NestedNameSpecifier::Kind::Namespace: {
       auto *NS = cast<NamespaceBaseDecl>(getDerived().TransformDecl(
-          Q.getLocalBeginLoc(), QNNS->getAsNamespace()));
+          Q.getLocalBeginLoc(), const_cast<NamespaceBaseDecl *>(
+                                    QNNS.getAsNamespaceAndPrefix().Namespace)));
       SS.Extend(SemaRef.Context, NS, Q.getLocalBeginLoc(), Q.getLocalEndLoc());
       break;
     }
 
-    case NestedNameSpecifier::Global:
+    case NestedNameSpecifier::Kind::Global:
       // There is no meaningful transformation that one could perform on the
       // global scope.
       SS.MakeGlobal(SemaRef.Context, Q.getBeginLoc());
       break;
 
-    case NestedNameSpecifier::Super: {
-      CXXRecordDecl *RD =
-          cast_or_null<CXXRecordDecl>(getDerived().TransformDecl(
-              SourceLocation(), QNNS->getAsRecordDecl()));
-      SS.MakeSuper(SemaRef.Context, RD, Q.getBeginLoc(), Q.getEndLoc());
+    case NestedNameSpecifier::Kind::MicrosoftSuper: {
+      CXXRecordDecl *RD = cast_or_null<CXXRecordDecl>(
+          getDerived().TransformDecl(SourceLocation(), QNNS.getAsRecordDecl()));
+      SS.MakeMicrosoftSuper(SemaRef.Context, RD, Q.getBeginLoc(),
+                            Q.getEndLoc());
       break;
     }
 
-    case NestedNameSpecifier::TypeSpec: {
-      TypeLoc TL = TransformTypeInObjectScope(Q.getTypeLoc(), ObjectType,
-                                              FirstQualifierInScope, SS);
-
-      if (!TL)
-        return NestedNameSpecifierLoc();
+    case NestedNameSpecifier::Kind::Type: {
+      assert(SS.isEmpty());
+      TypeLoc TL = Q.castAsTypeLoc();
+
+      if (auto DNT = TL.getAs<DependentNameTypeLoc>()) {
+        NestedNameSpecifierLoc QualifierLoc = DNT.getQualifierLoc();
+        if (QualifierLoc) {
+          QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(
+              QualifierLoc, ObjectType, FirstQualifierInScope);
+          if (!QualifierLoc)
+            return NestedNameSpecifierLoc();
+          ObjectType = QualType();
+          FirstQualifierInScope = nullptr;
+        }
+        SS.Adopt(QualifierLoc);
+        Sema::NestedNameSpecInfo IdInfo(
+            const_cast<IdentifierInfo *>(DNT.getTypePtr()->getIdentifier()),
+            DNT.getNameLoc(), Q.getLocalEndLoc(), ObjectType);
+        if (SemaRef.BuildCXXNestedNameSpecifier(/*Scope=*/nullptr, IdInfo,
+                                                false, SS,
+                                                FirstQualifierInScope, false))
+          return NestedNameSpecifierLoc();
+        return SS.getWithLocInContext(SemaRef.Context);
+      }
 
       QualType T = TL.getType();
+      TypeLocBuilder TLB;
+      if (!getDerived().AlreadyTransformed(T)) {
+        T = TransformTypeInObjectScope(TLB, TL, ObjectType,
+                                       FirstQualifierInScope);
+        if (T.isNull())
+          return NestedNameSpecifierLoc();
+        TL = TLB.getTypeLocInContext(SemaRef.Context, T);
+      }
+
       if (T->isDependentType() || T->isRecordType() ||
           (SemaRef.getLangOpts().CPlusPlus11 && T->isEnumeralType())) {
         if (T->isEnumeralType())
           SemaRef.Diag(TL.getBeginLoc(),
                        diag::warn_cxx98_compat_enum_nested_name_spec);
-
-        if (const auto ETL = TL.getAs<ElaboratedTypeLoc>()) {
-          SS.Adopt(ETL.getQualifierLoc());
-          TL = ETL.getNamedTypeLoc();
-        }
-
-        SS.Extend(SemaRef.Context, TL, Q.getLocalEndLoc());
+        SS.Make(SemaRef.Context, TL, Q.getLocalEndLoc());
         break;
       }
       // If the nested-name-specifier is an invalid type def, don't emit an
       // error because a previous error should have already been emitted.
       TypedefTypeLoc TTL = TL.getAsAdjusted<TypedefTypeLoc>();
-      if (!TTL || !TTL.getTypedefNameDecl()->isInvalidDecl()) {
+      if (!TTL || !TTL.getDecl()->isInvalidDecl()) {
         SemaRef.Diag(TL.getBeginLoc(), diag::err_nested_name_spec_non_tag)
             << T << SS.getRange();
       }
       return NestedNameSpecifierLoc();
     }
     }
-
-    // The qualifier-in-scope and object type only apply to the leftmost entity.
-    FirstQualifierInScope = nullptr;
-    ObjectType = QualType();
   }
 
   // Don't rebuild the nested-name-specifier if we don't have to.
@@ -4766,30 +4807,32 @@ template <typename Derived>
 TemplateName TreeTransform<Derived>::RebuildTemplateName(
     CXXScopeSpec &SS, SourceLocation TemplateKWLoc,
     IdentifierOrOverloadedOperator IO, SourceLocation NameLoc,
-    QualType ObjectType, NamedDecl *FirstQualifierInScope,
-    bool AllowInjectedClassName) {
-  if (const IdentifierInfo *II = IO.getIdentifier()) {
+    QualType ObjectType, bool AllowInjectedClassName) {
+  if (const IdentifierInfo *II = IO.getIdentifier())
     return getDerived().RebuildTemplateName(SS, TemplateKWLoc, *II, NameLoc,
-                                            ObjectType, FirstQualifierInScope,
-                                            AllowInjectedClassName);
-  }
+                                            ObjectType, AllowInjectedClassName);
   return getDerived().RebuildTemplateName(SS, TemplateKWLoc, IO.getOperator(),
                                           NameLoc, ObjectType,
                                           AllowInjectedClassName);
 }
 
-template<typename Derived>
-TemplateName
-TreeTransform<Derived>::TransformTemplateName(CXXScopeSpec &SS,
-                                              TemplateName Name,
-                                              SourceLocation NameLoc,
-                                              QualType ObjectType,
-                                              NamedDecl *FirstQualifierInScope,
-                                              bool AllowInjectedClassName) {
+template <typename Derived>
+TemplateName TreeTransform<Derived>::TransformTemplateName(
+    NestedNameSpecifierLoc &QualifierLoc, SourceLocation TemplateKWLoc,
+    TemplateName Name, SourceLocation NameLoc, QualType ObjectType,
+    NamedDecl *FirstQualifierInScope, bool AllowInjectedClassName) {
   if (QualifiedTemplateName *QTN = Name.getAsQualifiedTemplateName()) {
+    // FIXME: Preserve UsingTemplateName.
     TemplateDecl *Template = QTN->getUnderlyingTemplate().getAsTemplateDecl();
     assert(Template && "qualified template name must refer to a template");
 
+    if (QualifierLoc) {
+      QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(
+          QualifierLoc, ObjectType, FirstQualifierInScope);
+      if (!QualifierLoc)
+        return TemplateName();
+    }
+
     TemplateDecl *TransTemplate
       = cast_or_null<TemplateDecl>(getDerived().TransformDecl(NameLoc,
                                                               Template));
@@ -4797,41 +4840,67 @@ TreeTransform<Derived>::TransformTemplateName(CXXScopeSpec &SS,
       return TemplateName();
 
     if (!getDerived().AlwaysRebuild() &&
-        SS.getScopeRep() == QTN->getQualifier() &&
+        QualifierLoc.getNestedNameSpecifier() == QTN->getQualifier() &&
         TransTemplate == Template)
       return Name;
-
+    CXXScopeSpec SS;
+    SS.Adopt(QualifierLoc);
     return getDerived().RebuildTemplateName(SS, QTN->hasTemplateKeyword(),
                                             TransTemplate);
   }
 
   if (DependentTemplateName *DTN = Name.getAsDependentTemplateName()) {
-    if (SS.getScopeRep()) {
-      // These apply to the scope specifier, not the template.
+    if (QualifierLoc) {
+      QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(
+          QualifierLoc, ObjectType, FirstQualifierInScope);
+      if (!QualifierLoc)
+        return TemplateName();
+      // The qualifier-in-scope and object type only apply to the leftmost
+      // entity.
       ObjectType = QualType();
-      FirstQualifierInScope = nullptr;
     }
 
     if (!getDerived().AlwaysRebuild() &&
-        SS.getScopeRep() == DTN->getQualifier() &&
+        QualifierLoc.getNestedNameSpecifier() == DTN->getQualifier() &&
         ObjectType.isNull())
       return Name;
 
-    // FIXME: Preserve the location of the "template" keyword.
-    SourceLocation TemplateKWLoc = NameLoc;
-    return getDerived().RebuildTemplateName(
-        SS, TemplateKWLoc, DTN->getName(), NameLoc, ObjectType,
+    CXXScopeSpec SS;
+    SS.Adopt(QualifierLoc);
+    return getDerived().RebuildTemplateName(SS, TemplateKWLoc, DTN->getName(),
+                                            NameLoc, ObjectType,
+                                            AllowInjectedClassName);
+  }
+
+  if (SubstTemplateTemplateParmStorage *S =
+          Name.getAsSubstTemplateTemplateParm()) {
+    TemplateName NewName = getDerived().TransformTemplateName(
+        QualifierLoc, TemplateKWLoc, S->getReplacement(), NameLoc, ObjectType,
         FirstQualifierInScope, AllowInjectedClassName);
+    if (NewName.isNull())
+      return TemplateName();
+    Decl *AssociatedDecl =
+        getDerived().TransformDecl(NameLoc, S->getAssociatedDecl());
+    if (!getDerived().AlwaysRebuild() && NewName == S->getReplacement() &&
+        AssociatedDecl == S->getAssociatedDecl())
+      return Name;
+    return SemaRef.Context.getSubstTemplateTemplateParm(
+        NewName, AssociatedDecl, S->getIndex(), S->getPackIndex(),
+        S->getFinal());
   }
 
-  // FIXME: Try to preserve more of the TemplateName.
+  assert(!Name.getAsDeducedTemplateName() &&
+         "DeducedTemplateName should not escape partial ordering");
+
   if (TemplateDecl *Template = Name.getAsTemplateDecl()) {
+    assert(!QualifierLoc && "missed a Qualified Template");
     TemplateDecl *TransTemplate
       = cast_or_null<TemplateDecl>(getDerived().TransformDecl(NameLoc,
                                                               Template));
     if (!TransTemplate)
       return TemplateName();
 
+    CXXScopeSpec SS;
     return getDerived().RebuildTemplateName(SS, /*TemplateKeyword=*/false,
                                             TransTemplate);
   }
@@ -4849,8 +4918,10 @@ TreeTransform<Derived>::TransformTemplateName(CXXScopeSpec &SS,
 
 template <typename Derived>
 TemplateArgument TreeTransform<Derived>::TransformNamedTemplateTemplateArgument(
-    CXXScopeSpec &SS, TemplateName Name, SourceLocation NameLoc) {
-  TemplateName TN = getDerived().TransformTemplateName(SS, Name, NameLoc);
+    NestedNameSpecifierLoc &QualifierLoc, SourceLocation TemplateKeywordLoc,
+    TemplateName Name, SourceLocation NameLoc) {
+  TemplateName TN = getDerived().TransformTemplateName(
+      QualifierLoc, TemplateKeywordLoc, Name, NameLoc);
   if (TN.isNull())
     return TemplateArgument();
   return TemplateArgument(TN);
@@ -4932,21 +5003,14 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
 
   case TemplateArgument::Template: {
     NestedNameSpecifierLoc QualifierLoc = Input.getTemplateQualifierLoc();
-    if (QualifierLoc) {
-      QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(QualifierLoc);
-      if (!QualifierLoc)
-        return true;
-    }
-
-    CXXScopeSpec SS;
-    SS.Adopt(QualifierLoc);
 
     TemplateArgument Out = getDerived().TransformNamedTemplateTemplateArgument(
-        SS, Arg.getAsTemplate(), Input.getTemplateNameLoc());
+        QualifierLoc, Input.getTemplateKWLoc(), Arg.getAsTemplate(),
+        Input.getTemplateNameLoc());
     if (Out.isNull())
       return true;
-    Output = TemplateArgumentLoc(SemaRef.Context, Out, QualifierLoc,
-                                 Input.getTemplateNameLoc());
+    Output = TemplateArgumentLoc(SemaRef.Context, Out, Input.getTemplateKWLoc(),
+                                 QualifierLoc, Input.getTemplateNameLoc());
     return false;
   }
 
@@ -5063,60 +5127,30 @@ bool TreeTransform<Derived>::TransformTemplateArguments(
     }
 
     if (In.getArgument().isPackExpansion()) {
-      // We have a pack expansion, for which we will be substituting into
-      // the pattern.
-      SourceLocation Ellipsis;
-      UnsignedOrNone OrigNumExpansions = std::nullopt;
-      TemplateArgumentLoc Pattern
-        = getSema().getTemplateArgumentPackExpansionPattern(
-              In, Ellipsis, OrigNumExpansions);
-
-      SmallVector<UnexpandedParameterPack, 2> Unexpanded;
-      getSema().collectUnexpandedParameterPacks(Pattern, Unexpanded);
-      assert(!Unexpanded.empty() && "Pack expansion without parameter packs?");
-
-      // Determine whether the set of unexpanded parameter packs can and should
-      // be expanded.
-      bool Expand = true;
-      bool RetainExpansion = false;
-      UnsignedOrNone NumExpansions = OrigNumExpansions;
-      if (getDerived().TryExpandParameterPacks(Ellipsis,
-                                               Pattern.getSourceRange(),
-                                               Unexpanded,
-                                               Expand,
-                                               RetainExpansion,
-                                               NumExpansions))
+      UnexpandedInfo Info;
+      TemplateArgumentLoc Prepared;
+      if (PreparePackForExpansion(In, Uneval, Prepared, Info))
         return true;
-
-      if (!Expand) {
-        // The transform has determined that we should perform a simple
-        // transformation on the pack expansion, producing another pack
-        // expansion.
-        TemplateArgumentLoc OutPattern;
-        Sema::ArgPackSubstIndexRAII SubstIndex(getSema(), std::nullopt);
-        if (getDerived().TransformTemplateArgument(Pattern, OutPattern, Uneval))
-          return true;
-
-        Out = getDerived().RebuildPackExpansion(OutPattern, Ellipsis,
-                                                NumExpansions);
-        if (Out.getArgument().isNull())
-          return true;
-
-        Outputs.addArgument(Out);
+      if (!Info.Expand) {
+        Outputs.addArgument(Prepared);
         continue;
       }
 
       // The transform has determined that we should perform an elementwise
       // expansion of the pattern. Do so.
-      for (unsigned I = 0; I != *NumExpansions; ++I) {
+      std::optional<ForgetSubstitutionRAII> ForgetSubst;
+      if (Info.ExpandUnderForgetSubstitions)
+        ForgetSubst.emplace(getDerived());
+      for (unsigned I = 0; I != *Info.NumExpansions; ++I) {
         Sema::ArgPackSubstIndexRAII SubstIndex(getSema(), I);
 
-        if (getDerived().TransformTemplateArgument(Pattern, Out, Uneval))
+        TemplateArgumentLoc Out;
+        if (getDerived().TransformTemplateArgument(Prepared, Out, Uneval))
           return true;
 
         if (Out.getArgument().containsUnexpandedParameterPack()) {
-          Out = getDerived().RebuildPackExpansion(Out, Ellipsis,
-                                                  OrigNumExpansions);
+          Out = getDerived().RebuildPackExpansion(Out, Info.Ellipsis,
+                                                  Info.OrigNumExpansions);
           if (Out.getArgument().isNull())
             return true;
         }
@@ -5126,14 +5160,15 @@ bool TreeTransform<Derived>::TransformTemplateArguments(
 
       // If we're supposed to retain a pack expansion, do so by temporarily
       // forgetting the partially-substituted parameter pack.
-      if (RetainExpansion) {
+      if (Info.RetainExpansion) {
         ForgetPartiallySubstitutedPackRAII Forget(getDerived());
 
-        if (getDerived().TransformTemplateArgument(Pattern, Out, Uneval))
+        TemplateArgumentLoc Out;
+        if (getDerived().TransformTemplateArgument(Prepared, Out, Uneval))
           return true;
 
-        Out = getDerived().RebuildPackExpansion(Out, Ellipsis,
-                                                OrigNumExpansions);
+        Out = getDerived().RebuildPackExpansion(Out, Info.Ellipsis,
+                                                Info.OrigNumExpansions);
         if (Out.getArgument().isNull())
           return true;
 
@@ -5154,6 +5189,114 @@ bool TreeTransform<Derived>::TransformTemplateArguments(
 
 }
 
+// FIXME: Find ways to reduce code duplication for pack expansions.
+template <typename Derived>
+bool TreeTransform<Derived>::PreparePackForExpansion(TemplateArgumentLoc In,
+                                                     bool Uneval,
+                                                     TemplateArgumentLoc &Out,
+                                                     UnexpandedInfo &Info) {
+  auto ComputeInfo = [this](TemplateArgumentLoc Arg,
+                            bool IsLateExpansionAttempt, UnexpandedInfo &Info,
+                            TemplateArgumentLoc &Pattern) {
+    assert(Arg.getArgument().isPackExpansion());
+    // We have a pack expansion, for which we will be substituting into the
+    // pattern.
+    Pattern = getSema().getTemplateArgumentPackExpansionPattern(
+        Arg, Info.Ellipsis, Info.OrigNumExpansions);
+    SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+    getSema().collectUnexpandedParameterPacks(Pattern, Unexpanded);
+    if (IsLateExpansionAttempt) {
+      // Request expansion only when there is an opportunity to expand a pack
+      // that required a substituion first.
+      bool SawPackTypes =
+          llvm::any_of(Unexpanded, [](UnexpandedParameterPack P) {
+            return P.first.dyn_cast<const SubstBuiltinTemplatePackType *>();
+          });
+      if (!SawPackTypes) {
+        Info.Expand = false;
+        return false;
+      }
+    }
+    assert(!Unexpanded.empty() && "Pack expansion without parameter packs?");
+
+    // Determine whether the set of unexpanded parameter packs can and
+    // should be expanded.
+    Info.Expand = true;
+    Info.RetainExpansion = false;
+    Info.NumExpansions = Info.OrigNumExpansions;
+    return getDerived().TryExpandParameterPacks(
+        Info.Ellipsis, Pattern.getSourceRange(), Unexpanded,
+        /*FailOnPackProducingTemplates=*/false, Info.Expand,
+        Info.RetainExpansion, Info.NumExpansions);
+  };
+
+  TemplateArgumentLoc Pattern;
+  if (ComputeInfo(In, false, Info, Pattern))
+    return true;
+
+  if (Info.Expand) {
+    Out = Pattern;
+    return false;
+  }
+
+  // The transform has determined that we should perform a simple
+  // transformation on the pack expansion, producing another pack
+  // expansion.
+  TemplateArgumentLoc OutPattern;
+  std::optional<Sema::ArgPackSubstIndexRAII> SubstIndex(
+      std::in_place, getSema(), std::nullopt);
+  if (getDerived().TransformTemplateArgument(Pattern, OutPattern, Uneval))
+    return true;
+
+  Out = getDerived().RebuildPackExpansion(OutPattern, Info.Ellipsis,
+                                          Info.NumExpansions);
+  if (Out.getArgument().isNull())
+    return true;
+  SubstIndex.reset();
+
+  if (!OutPattern.getArgument().containsUnexpandedParameterPack())
+    return false;
+
+  // Some packs will learn their length after substitution, e.g.
+  // __builtin_dedup_pack<T,int> has size 1 or 2, depending on the substitution
+  // value of `T`.
+  //
+  // We only expand after we know sizes of all packs, check if this is the case
+  // or not. However, we avoid a full template substitution and only do
+  // expanstions after this point.
+
+  // E.g. when substituting template arguments of tuple with {T -> int} in the
+  // following example:
+  //   template <class T>
+  //   struct TupleWithInt {
+  //     using type = std::tuple<__builtin_dedup_pack<T, int>...>;
+  //   };
+  //   TupleWithInt<int>::type y;
+  // At this point we will see the `__builtin_dedup_pack<int, int>` with a known
+  // lenght and run `ComputeInfo()` to provide the necessary information to our
+  // caller.
+  //
+  // Note that we may still have situations where builtin is not going to be
+  // expanded. For example:
+  //   template <class T>
+  //   struct Foo {
+  //     template <class U> using tuple_with_t =
+  //     std::tuple<__builtin_dedup_pack<T, U, int>...>; using type =
+  //     tuple_with_t<short>;
+  //   }
+  // Because the substitution into `type` happens in dependent context, `type`
+  // will be `tuple<builtin_dedup_pack<T, short, int>...>` after substitution
+  // and the caller will not be able to expand it.
+  ForgetSubstitutionRAII ForgetSubst(getDerived());
+  if (ComputeInfo(Out, true, Info, OutPattern))
+    return true;
+  if (!Info.Expand)
+    return false;
+  Out = OutPattern;
+  Info.ExpandUnderForgetSubstitions = true;
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // Type transformation
 //===----------------------------------------------------------------------===//
@@ -5380,85 +5523,38 @@ QualType TreeTransform<Derived>::RebuildQualifiedType(QualType T,
   return SemaRef.BuildQualifiedType(T, Loc, Quals);
 }
 
-template<typename Derived>
-TypeLoc
-TreeTransform<Derived>::TransformTypeInObjectScope(TypeLoc TL,
-                                                   QualType ObjectType,
-                                                   NamedDecl *UnqualLookup,
-                                                   CXXScopeSpec &SS) {
-  if (getDerived().AlreadyTransformed(TL.getType()))
-    return TL;
-
-  TypeSourceInfo *TSI =
-      TransformTSIInObjectScope(TL, ObjectType, UnqualLookup, SS);
-  if (TSI)
-    return TSI->getTypeLoc();
-  return TypeLoc();
-}
-
-template<typename Derived>
-TypeSourceInfo *
-TreeTransform<Derived>::TransformTypeInObjectScope(TypeSourceInfo *TSInfo,
-                                                   QualType ObjectType,
-                                                   NamedDecl *UnqualLookup,
-                                                   CXXScopeSpec &SS) {
-  if (getDerived().AlreadyTransformed(TSInfo->getType()))
-    return TSInfo;
-
-  return TransformTSIInObjectScope(TSInfo->getTypeLoc(), ObjectType,
-                                   UnqualLookup, SS);
-}
-
 template <typename Derived>
-TypeSourceInfo *TreeTransform<Derived>::TransformTSIInObjectScope(
-    TypeLoc TL, QualType ObjectType, NamedDecl *UnqualLookup,
-    CXXScopeSpec &SS) {
-  QualType T = TL.getType();
-  assert(!getDerived().AlreadyTransformed(T));
-
-  TypeLocBuilder TLB;
-  QualType Result;
-
-  if (isa<TemplateSpecializationType>(T)) {
-    TemplateSpecializationTypeLoc SpecTL =
-        TL.castAs<TemplateSpecializationTypeLoc>();
-
-    TemplateName Template = getDerived().TransformTemplateName(
-        SS, SpecTL.getTypePtr()->getTemplateName(), SpecTL.getTemplateNameLoc(),
-        ObjectType, UnqualLookup, /*AllowInjectedClassName*/true);
-    if (Template.isNull())
-      return nullptr;
-
-    Result = getDerived().TransformTemplateSpecializationType(TLB, SpecTL,
-                                                              Template);
-  } else if (isa<DependentTemplateSpecializationType>(T)) {
-    DependentTemplateSpecializationTypeLoc SpecTL =
-        TL.castAs<DependentTemplateSpecializationTypeLoc>();
-
-    const IdentifierInfo *II = SpecTL.getTypePtr()
-                                   ->getDependentTemplateName()
-                                   .getName()
-                                   .getIdentifier();
-    TemplateName Template = getDerived().RebuildTemplateName(
-        SS, SpecTL.getTemplateKeywordLoc(), *II, SpecTL.getTemplateNameLoc(),
-        ObjectType, UnqualLookup,
-        /*AllowInjectedClassName*/ true);
-    if (Template.isNull())
-      return nullptr;
-
-    Result = getDerived().TransformDependentTemplateSpecializationType(TLB,
-                                                                       SpecTL,
-                                                                       Template,
-                                                                       SS);
-  } else {
-    // Nothing special needs to be done for these.
-    Result = getDerived().TransformType(TLB, TL);
+QualType TreeTransform<Derived>::TransformTypeInObjectScope(
+    TypeLocBuilder &TLB, TypeLoc TL, QualType ObjectType,
+    NamedDecl *UnqualLookup) {
+  assert(!getDerived().AlreadyTransformed(TL.getType()));
+
+  switch (TL.getTypeLocClass()) {
+  case TypeLoc::DependentTemplateSpecialization:
+    return getDerived().TransformDependentTemplateSpecializationType(
+        TLB, TL.castAs<DependentTemplateSpecializationTypeLoc>(), ObjectType,
+        UnqualLookup, /*AllowInjectedClassName=*/true);
+  case TypeLoc::DependentName: {
+    return getDerived().TransformDependentNameType(
+        TLB, TL.castAs<DependentNameTypeLoc>(), /*DeducedTSTContext=*/false,
+        ObjectType, UnqualLookup);
+  }
+  case TypeLoc::Typedef:
+  case TypeLoc::TemplateSpecialization:
+  case TypeLoc::SubstTemplateTypeParm:
+  case TypeLoc::SubstTemplateTypeParmPack:
+  case TypeLoc::PackIndexing:
+  case TypeLoc::Enum:
+  case TypeLoc::Record:
+  case TypeLoc::InjectedClassName:
+  case TypeLoc::TemplateTypeParm:
+  case TypeLoc::Decltype:
+  case TypeLoc::UnresolvedUsing:
+  case TypeLoc::Using:
+    return getDerived().TransformType(TLB, TL);
+  default:
+    llvm_unreachable("unexpected type class");
   }
-
-  if (Result.isNull())
-    return nullptr;
-
-  return TLB.getTypeSourceInfo(SemaRef.Context, Result);
 }
 
 template <class TyLoc> static inline
@@ -6211,12 +6307,10 @@ bool TreeTransform<Derived>::TransformFunctionTypeParams(
         if (Unexpanded.size() > 0) {
           OrigNumExpansions = ExpansionTL.getTypePtr()->getNumExpansions();
           NumExpansions = OrigNumExpansions;
-          if (getDerived().TryExpandParameterPacks(ExpansionTL.getEllipsisLoc(),
-                                                   Pattern.getSourceRange(),
-                                                   Unexpanded,
-                                                   ShouldExpand,
-                                                   RetainExpansion,
-                                                   NumExpansions)) {
+          if (getDerived().TryExpandParameterPacks(
+                  ExpansionTL.getEllipsisLoc(), Pattern.getSourceRange(),
+                  Unexpanded, /*FailOnPackProducingTemplates=*/true,
+                  ShouldExpand, RetainExpansion, NumExpansions)) {
             return true;
           }
         } else {
@@ -6322,11 +6416,10 @@ bool TreeTransform<Derived>::TransformFunctionTypeParams(
       // Determine whether we should expand the parameter packs.
       bool ShouldExpand = false;
       bool RetainExpansion = false;
-      if (getDerived().TryExpandParameterPacks(Loc, SourceRange(),
-                                               Unexpanded,
-                                               ShouldExpand,
-                                               RetainExpansion,
-                                               NumExpansions)) {
+      if (getDerived().TryExpandParameterPacks(
+              Loc, SourceRange(), Unexpanded,
+              /*FailOnPackProducingTemplates=*/true, ShouldExpand,
+              RetainExpansion, NumExpansions)) {
         return true;
       }
 
@@ -6623,8 +6716,9 @@ bool TreeTransform<Derived>::TransformExceptionSpec(
       // FIXME: Track the location of the ellipsis (and track source location
       // information for the types in the exception specification in general).
       if (getDerived().TryExpandParameterPacks(
-              Loc, SourceRange(), Unexpanded, Expand,
-              RetainExpansion, NumExpansions))
+              Loc, SourceRange(), Unexpanded,
+              /*FailOnPackProducingTemplates=*/true, Expand, RetainExpansion,
+              NumExpansions))
         return true;
 
       if (!Expand) {
@@ -6694,23 +6788,38 @@ QualType TreeTransform<Derived>::TransformFunctionNoProtoType(
 template <typename Derived>
 QualType TreeTransform<Derived>::TransformUnresolvedUsingType(
     TypeLocBuilder &TLB, UnresolvedUsingTypeLoc TL) {
+
   const UnresolvedUsingType *T = TL.getTypePtr();
-  Decl *D = getDerived().TransformDecl(TL.getNameLoc(), T->getDecl());
+  bool Changed = false;
+
+  NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+  if (NestedNameSpecifierLoc OldQualifierLoc = QualifierLoc) {
+    QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(QualifierLoc);
+    if (!QualifierLoc)
+      return QualType();
+    Changed |= QualifierLoc != OldQualifierLoc;
+  }
+
+  auto *D = getDerived().TransformDecl(TL.getNameLoc(), T->getDecl());
   if (!D)
     return QualType();
+  Changed |= D != T->getDecl();
 
   QualType Result = TL.getType();
-  if (getDerived().AlwaysRebuild() || D != T->getDecl()) {
-    Result = getDerived().RebuildUnresolvedUsingType(TL.getNameLoc(), D);
+  if (getDerived().AlwaysRebuild() || Changed) {
+    Result = getDerived().RebuildUnresolvedUsingType(
+        T->getKeyword(), QualifierLoc.getNestedNameSpecifier(), TL.getNameLoc(),
+        D);
     if (Result.isNull())
       return QualType();
   }
 
-  // We might get an arbitrary type spec type back.  We should at
-  // least always get a type spec type, though.
-  TypeSpecTypeLoc NewTL = TLB.pushTypeSpec(Result);
-  NewTL.setNameLoc(TL.getNameLoc());
-
+  if (isa<UsingType>(Result))
+    TLB.push<UsingTypeLoc>(Result).set(TL.getElaboratedKeywordLoc(),
+                                       QualifierLoc, TL.getNameLoc());
+  else
+    TLB.push<UnresolvedUsingTypeLoc>(Result).set(TL.getElaboratedKeywordLoc(),
+                                                 QualifierLoc, TL.getNameLoc());
   return Result;
 }
 
@@ -6718,25 +6827,37 @@ template <typename Derived>
 QualType TreeTransform<Derived>::TransformUsingType(TypeLocBuilder &TLB,
                                                     UsingTypeLoc TL) {
   const UsingType *T = TL.getTypePtr();
+  bool Changed = false;
 
-  auto *Found = cast_or_null<UsingShadowDecl>(getDerived().TransformDecl(
-      TL.getLocalSourceRange().getBegin(), T->getFoundDecl()));
-  if (!Found)
+  NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+  if (NestedNameSpecifierLoc OldQualifierLoc = QualifierLoc) {
+    QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(QualifierLoc);
+    if (!QualifierLoc)
+      return QualType();
+    Changed |= QualifierLoc != OldQualifierLoc;
+  }
+
+  auto *D = cast_or_null<UsingShadowDecl>(
+      getDerived().TransformDecl(TL.getNameLoc(), T->getDecl()));
+  if (!D)
     return QualType();
+  Changed |= D != T->getDecl();
 
-  QualType Underlying = getDerived().TransformType(T->desugar());
-  if (Underlying.isNull())
+  QualType UnderlyingType = getDerived().TransformType(T->desugar());
+  if (UnderlyingType.isNull())
     return QualType();
+  Changed |= UnderlyingType != T->desugar();
 
   QualType Result = TL.getType();
-  if (getDerived().AlwaysRebuild() || Found != T->getFoundDecl() ||
-      Underlying != T->getUnderlyingType()) {
-    Result = getDerived().RebuildUsingType(Found, Underlying);
+  if (getDerived().AlwaysRebuild() || Changed) {
+    Result = getDerived().RebuildUsingType(
+        T->getKeyword(), QualifierLoc.getNestedNameSpecifier(), D,
+        UnderlyingType);
     if (Result.isNull())
       return QualType();
   }
-
-  TLB.pushTypeSpec(Result).setNameLoc(TL.getNameLoc());
+  TLB.push<UsingTypeLoc>(Result).set(TL.getElaboratedKeywordLoc(), QualifierLoc,
+                                     TL.getNameLoc());
   return Result;
 }
 
@@ -6744,23 +6865,34 @@ template<typename Derived>
 QualType TreeTransform<Derived>::TransformTypedefType(TypeLocBuilder &TLB,
                                                       TypedefTypeLoc TL) {
   const TypedefType *T = TL.getTypePtr();
-  TypedefNameDecl *Typedef
-    = cast_or_null<TypedefNameDecl>(getDerived().TransformDecl(TL.getNameLoc(),
-                                                               T->getDecl()));
+  bool Changed = false;
+
+  NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+  if (NestedNameSpecifierLoc OldQualifierLoc = QualifierLoc) {
+    QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(QualifierLoc);
+    if (!QualifierLoc)
+      return QualType();
+    Changed |= QualifierLoc != OldQualifierLoc;
+  }
+
+  auto *Typedef = cast_or_null<TypedefNameDecl>(
+      getDerived().TransformDecl(TL.getNameLoc(), T->getDecl()));
   if (!Typedef)
     return QualType();
+  Changed |= Typedef != T->getDecl();
+
+  // FIXME: Transform the UnderlyingType if different from decl.
 
   QualType Result = TL.getType();
-  if (getDerived().AlwaysRebuild() ||
-      Typedef != T->getDecl()) {
-    Result = getDerived().RebuildTypedefType(Typedef);
+  if (getDerived().AlwaysRebuild() || Changed) {
+    Result = getDerived().RebuildTypedefType(
+        T->getKeyword(), QualifierLoc.getNestedNameSpecifier(), Typedef);
     if (Result.isNull())
       return QualType();
   }
 
-  TypedefTypeLoc NewTL = TLB.push<TypedefTypeLoc>(Result);
-  NewTL.setNameLoc(TL.getNameLoc());
-
+  TLB.push<TypedefTypeLoc>(Result).set(TL.getElaboratedKeywordLoc(),
+                                       QualifierLoc, TL.getNameLoc());
   return Result;
 }
 
@@ -6898,9 +7030,10 @@ TreeTransform<Derived>::TransformPackIndexingType(TypeLocBuilder &TLB,
     bool ShouldExpand = true;
     bool RetainExpansion = false;
     UnsignedOrNone NumExpansions = std::nullopt;
-    if (getDerived().TryExpandParameterPacks(TL.getEllipsisLoc(), SourceRange(),
-                                             Unexpanded, ShouldExpand,
-                                             RetainExpansion, NumExpansions))
+    if (getDerived().TryExpandParameterPacks(
+            TL.getEllipsisLoc(), SourceRange(), Unexpanded,
+            /*FailOnPackProducingTemplates=*/true, ShouldExpand,
+            RetainExpansion, NumExpansions))
       return QualType();
     if (!ShouldExpand) {
       Sema::ArgPackSubstIndexRAII SubstIndex(getSema(), std::nullopt);
@@ -6996,9 +7129,10 @@ QualType TreeTransform<Derived>::TransformDeducedTemplateSpecializationType(
     TypeLocBuilder &TLB, DeducedTemplateSpecializationTypeLoc TL) {
   const DeducedTemplateSpecializationType *T = TL.getTypePtr();
 
-  CXXScopeSpec SS;
+  NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
   TemplateName TemplateName = getDerived().TransformTemplateName(
-      SS, T->getTemplateName(), TL.getTemplateNameLoc());
+      QualifierLoc, /*TemplateKELoc=*/SourceLocation(), T->getTemplateName(),
+      TL.getTemplateNameLoc());
   if (TemplateName.isNull())
     return QualType();
 
@@ -7011,76 +7145,71 @@ QualType TreeTransform<Derived>::TransformDeducedTemplateSpecializationType(
   }
 
   QualType Result = getDerived().RebuildDeducedTemplateSpecializationType(
-      TemplateName, NewDeduced);
+      T->getKeyword(), TemplateName, NewDeduced);
   if (Result.isNull())
     return QualType();
 
-  DeducedTemplateSpecializationTypeLoc NewTL =
-      TLB.push<DeducedTemplateSpecializationTypeLoc>(Result);
+  auto NewTL = TLB.push<DeducedTemplateSpecializationTypeLoc>(Result);
+  NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
   NewTL.setTemplateNameLoc(TL.getTemplateNameLoc());
-
+  NewTL.setQualifierLoc(QualifierLoc);
   return Result;
 }
 
-template<typename Derived>
-QualType TreeTransform<Derived>::TransformRecordType(TypeLocBuilder &TLB,
-                                                     RecordTypeLoc TL) {
-  const RecordType *T = TL.getTypePtr();
-  RecordDecl *Record
-    = cast_or_null<RecordDecl>(getDerived().TransformDecl(TL.getNameLoc(),
-                                                          T->getDecl()));
-  if (!Record)
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformTagType(TypeLocBuilder &TLB,
+                                                  TagTypeLoc TL) {
+  const TagType *T = TL.getTypePtr();
+
+  NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+  if (QualifierLoc) {
+    QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(QualifierLoc);
+    if (!QualifierLoc)
+      return QualType();
+  }
+
+  auto *TD = cast_or_null<TagDecl>(
+      getDerived().TransformDecl(TL.getNameLoc(), T->getOriginalDecl()));
+  if (!TD)
     return QualType();
 
   QualType Result = TL.getType();
-  if (getDerived().AlwaysRebuild() ||
-      Record != T->getDecl()) {
-    Result = getDerived().RebuildRecordType(Record);
+  if (getDerived().AlwaysRebuild() || QualifierLoc != TL.getQualifierLoc() ||
+      TD != T->getOriginalDecl()) {
+    if (T->isCanonicalUnqualified())
+      Result = getDerived().RebuildCanonicalTagType(TD);
+    else
+      Result = getDerived().RebuildTagType(
+          T->getKeyword(), QualifierLoc.getNestedNameSpecifier(), TD);
     if (Result.isNull())
       return QualType();
   }
 
-  RecordTypeLoc NewTL = TLB.push<RecordTypeLoc>(Result);
+  TagTypeLoc NewTL = TLB.push<TagTypeLoc>(Result);
+  NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
+  NewTL.setQualifierLoc(QualifierLoc);
   NewTL.setNameLoc(TL.getNameLoc());
 
   return Result;
 }
 
-template<typename Derived>
+template <typename Derived>
 QualType TreeTransform<Derived>::TransformEnumType(TypeLocBuilder &TLB,
                                                    EnumTypeLoc TL) {
-  const EnumType *T = TL.getTypePtr();
-  EnumDecl *Enum
-    = cast_or_null<EnumDecl>(getDerived().TransformDecl(TL.getNameLoc(),
-                                                        T->getDecl()));
-  if (!Enum)
-    return QualType();
-
-  QualType Result = TL.getType();
-  if (getDerived().AlwaysRebuild() ||
-      Enum != T->getDecl()) {
-    Result = getDerived().RebuildEnumType(Enum);
-    if (Result.isNull())
-      return QualType();
-  }
-
-  EnumTypeLoc NewTL = TLB.push<EnumTypeLoc>(Result);
-  NewTL.setNameLoc(TL.getNameLoc());
+  return getDerived().TransformTagType(TLB, TL);
+}
 
-  return Result;
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformRecordType(TypeLocBuilder &TLB,
+                                                     RecordTypeLoc TL) {
+  return getDerived().TransformTagType(TLB, TL);
 }
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformInjectedClassNameType(
                                          TypeLocBuilder &TLB,
                                          InjectedClassNameTypeLoc TL) {
-  Decl *D = getDerived().TransformDecl(TL.getNameLoc(),
-                                       TL.getTypePtr()->getDecl());
-  if (!D) return QualType();
-
-  QualType T = SemaRef.Context.getTypeDeclType(cast<TypeDecl>(D));
-  TLB.pushTypeSpec(T).setNameLoc(TL.getNameLoc());
-  return T;
+  return getDerived().TransformTagType(TLB, TL);
 }
 
 template<typename Derived>
@@ -7126,6 +7255,11 @@ QualType TreeTransform<Derived>::TransformSubstTemplateTypeParmType(
   return Result;
 
 }
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformSubstBuiltinTemplatePackType(
+    TypeLocBuilder &TLB, SubstBuiltinTemplatePackTypeLoc TL) {
+  return TransformTypeSpecType(TLB, TL);
+}
 
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformSubstTemplateTypeParmPackType(
@@ -7141,24 +7275,6 @@ QualType TreeTransform<Derived>::TransformSubstTemplateTypeParmPackType(
   return TransformTypeSpecType(TLB, TL);
 }
 
-template<typename Derived>
-QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
-                                                        TypeLocBuilder &TLB,
-                                           TemplateSpecializationTypeLoc TL) {
-  const TemplateSpecializationType *T = TL.getTypePtr();
-
-  // The nested-name-specifier never matters in a TemplateSpecializationType,
-  // because we can't have a dependent nested-name-specifier anyway.
-  CXXScopeSpec SS;
-  TemplateName Template
-    = getDerived().TransformTemplateName(SS, T->getTemplateName(),
-                                         TL.getTemplateNameLoc());
-  if (Template.isNull())
-    return QualType();
-
-  return getDerived().TransformTemplateSpecializationType(TLB, TL, Template);
-}
-
 template<typename Derived>
 QualType TreeTransform<Derived>::TransformAtomicType(TypeLocBuilder &TLB,
                                                      AtomicTypeLoc TL) {
@@ -7398,9 +7514,16 @@ QualType TreeTransform<Derived>::TransformAutoType(TypeLocBuilder &TLB,
 
 template <typename Derived>
 QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
-                                                        TypeLocBuilder &TLB,
-                                           TemplateSpecializationTypeLoc TL,
-                                                      TemplateName Template) {
+    TypeLocBuilder &TLB, TemplateSpecializationTypeLoc TL) {
+  const TemplateSpecializationType *T = TL.getTypePtr();
+
+  NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+  TemplateName Template = getDerived().TransformTemplateName(
+      QualifierLoc, TL.getTemplateKeywordLoc(), T->getTemplateName(),
+      TL.getTemplateNameLoc());
+  if (Template.isNull())
+    return QualType();
+
   TemplateArgumentListInfo NewTemplateArgs;
   NewTemplateArgs.setLAngleLoc(TL.getLAngleLoc());
   NewTemplateArgs.setRAngleLoc(TL.getRAngleLoc());
@@ -7415,10 +7538,9 @@ QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
   // original template changed. If the template changed, and even if the
   // arguments didn't change, these arguments might not correspond to their
   // respective parameters, therefore needing conversions.
-  QualType Result =
-    getDerived().RebuildTemplateSpecializationType(Template,
-                                                   TL.getTemplateNameLoc(),
-                                                   NewTemplateArgs);
+  QualType Result = getDerived().RebuildTemplateSpecializationType(
+      TL.getTypePtr()->getKeyword(), Template, TL.getTemplateNameLoc(),
+      NewTemplateArgs);
 
   if (!Result.isNull()) {
     // Specializations of template template parameters are represented as
@@ -7428,8 +7550,8 @@ QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
     if (isa<DependentTemplateSpecializationType>(Result)) {
       DependentTemplateSpecializationTypeLoc NewTL
         = TLB.push<DependentTemplateSpecializationTypeLoc>(Result);
-      NewTL.setElaboratedKeywordLoc(SourceLocation());
-      NewTL.setQualifierLoc(NestedNameSpecifierLoc());
+      NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
+      NewTL.setQualifierLoc(QualifierLoc);
       NewTL.setTemplateKeywordLoc(TL.getTemplateKeywordLoc());
       NewTL.setTemplateNameLoc(TL.getTemplateNameLoc());
       NewTL.setLAngleLoc(TL.getLAngleLoc());
@@ -7438,132 +7560,14 @@ QualType TreeTransform<Derived>::TransformTemplateSpecializationType(
         NewTL.setArgLocInfo(i, NewTemplateArgs[i].getLocInfo());
       return Result;
     }
-
-    TemplateSpecializationTypeLoc NewTL
-      = TLB.push<TemplateSpecializationTypeLoc>(Result);
-    NewTL.setTemplateKeywordLoc(TL.getTemplateKeywordLoc());
-    NewTL.setTemplateNameLoc(TL.getTemplateNameLoc());
-    NewTL.setLAngleLoc(TL.getLAngleLoc());
-    NewTL.setRAngleLoc(TL.getRAngleLoc());
-    for (unsigned i = 0, e = NewTemplateArgs.size(); i != e; ++i)
-      NewTL.setArgLocInfo(i, NewTemplateArgs[i].getLocInfo());
+    TLB.push<TemplateSpecializationTypeLoc>(Result).set(
+        TL.getElaboratedKeywordLoc(), QualifierLoc, TL.getTemplateKeywordLoc(),
+        TL.getTemplateNameLoc(), NewTemplateArgs);
   }
 
   return Result;
 }
 
-template <typename Derived>
-QualType TreeTransform<Derived>::TransformDependentTemplateSpecializationType(
-                                     TypeLocBuilder &TLB,
-                                     DependentTemplateSpecializationTypeLoc TL,
-                                     TemplateName Template,
-                                     CXXScopeSpec &SS) {
-  TemplateArgumentListInfo NewTemplateArgs;
-  NewTemplateArgs.setLAngleLoc(TL.getLAngleLoc());
-  NewTemplateArgs.setRAngleLoc(TL.getRAngleLoc());
-  typedef TemplateArgumentLocContainerIterator<
-            DependentTemplateSpecializationTypeLoc> ArgIterator;
-  if (getDerived().TransformTemplateArguments(ArgIterator(TL, 0),
-                                              ArgIterator(TL, TL.getNumArgs()),
-                                              NewTemplateArgs))
-    return QualType();
-
-  // FIXME: maybe don't rebuild if all the template arguments are the same.
-
-  if (DependentTemplateName *DTN = Template.getAsDependentTemplateName()) {
-    assert(DTN->getQualifier() == SS.getScopeRep());
-    QualType Result = getSema().Context.getDependentTemplateSpecializationType(
-        TL.getTypePtr()->getKeyword(), *DTN, NewTemplateArgs.arguments());
-
-    DependentTemplateSpecializationTypeLoc NewTL
-      = TLB.push<DependentTemplateSpecializationTypeLoc>(Result);
-    NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
-    NewTL.setQualifierLoc(SS.getWithLocInContext(SemaRef.Context));
-    NewTL.setTemplateKeywordLoc(TL.getTemplateKeywordLoc());
-    NewTL.setTemplateNameLoc(TL.getTemplateNameLoc());
-    NewTL.setLAngleLoc(TL.getLAngleLoc());
-    NewTL.setRAngleLoc(TL.getRAngleLoc());
-    for (unsigned i = 0, e = NewTemplateArgs.size(); i != e; ++i)
-      NewTL.setArgLocInfo(i, NewTemplateArgs[i].getLocInfo());
-    return Result;
-  }
-
-  QualType Result
-    = getDerived().RebuildTemplateSpecializationType(Template,
-                                                     TL.getTemplateNameLoc(),
-                                                     NewTemplateArgs);
-
-  if (!Result.isNull()) {
-    /// FIXME: Wrap this in an elaborated-type-specifier?
-    TemplateSpecializationTypeLoc NewTL
-      = TLB.push<TemplateSpecializationTypeLoc>(Result);
-    NewTL.setTemplateKeywordLoc(TL.getTemplateKeywordLoc());
-    NewTL.setTemplateNameLoc(TL.getTemplateNameLoc());
-    NewTL.setLAngleLoc(TL.getLAngleLoc());
-    NewTL.setRAngleLoc(TL.getRAngleLoc());
-    for (unsigned i = 0, e = NewTemplateArgs.size(); i != e; ++i)
-      NewTL.setArgLocInfo(i, NewTemplateArgs[i].getLocInfo());
-  }
-
-  return Result;
-}
-
-template<typename Derived>
-QualType
-TreeTransform<Derived>::TransformElaboratedType(TypeLocBuilder &TLB,
-                                                ElaboratedTypeLoc TL) {
-  const ElaboratedType *T = TL.getTypePtr();
-
-  NestedNameSpecifierLoc QualifierLoc;
-  // NOTE: the qualifier in an ElaboratedType is optional.
-  if (TL.getQualifierLoc()) {
-    QualifierLoc
-      = getDerived().TransformNestedNameSpecifierLoc(TL.getQualifierLoc());
-    if (!QualifierLoc)
-      return QualType();
-  }
-
-  QualType NamedT = getDerived().TransformType(TLB, TL.getNamedTypeLoc());
-  if (NamedT.isNull())
-    return QualType();
-
-  // C++0x [dcl.type.elab]p2:
-  //   If the identifier resolves to a typedef-name or the simple-template-id
-  //   resolves to an alias template specialization, the
-  //   elaborated-type-specifier is ill-formed.
-  if (T->getKeyword() != ElaboratedTypeKeyword::None &&
-      T->getKeyword() != ElaboratedTypeKeyword::Typename) {
-    if (const TemplateSpecializationType *TST =
-          NamedT->getAs<TemplateSpecializationType>()) {
-      TemplateName Template = TST->getTemplateName();
-      if (TypeAliasTemplateDecl *TAT = dyn_cast_or_null<TypeAliasTemplateDecl>(
-              Template.getAsTemplateDecl())) {
-        SemaRef.Diag(TL.getNamedTypeLoc().getBeginLoc(),
-                     diag::err_tag_reference_non_tag)
-            << TAT << NonTagKind::TypeAliasTemplate
-            << ElaboratedType::getTagTypeKindForKeyword(T->getKeyword());
-        SemaRef.Diag(TAT->getLocation(), diag::note_declared_at);
-      }
-    }
-  }
-
-  QualType Result = TL.getType();
-  if (getDerived().AlwaysRebuild() ||
-      QualifierLoc != TL.getQualifierLoc() ||
-      NamedT != T->getNamedType()) {
-    Result = getDerived().RebuildElaboratedType(TL.getElaboratedKeywordLoc(),
-                                                T->getKeyword(),
-                                                QualifierLoc, NamedT);
-    if (Result.isNull())
-      return QualType();
-  }
-
-  ElaboratedTypeLoc NewTL = TLB.push<ElaboratedTypeLoc>(Result);
-  NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
-  NewTL.setQualifierLoc(QualifierLoc);
-  return Result;
-}
-
 template <typename Derived>
 QualType TreeTransform<Derived>::TransformAttributedType(TypeLocBuilder &TLB,
                                                          AttributedTypeLoc TL) {
@@ -7754,15 +7758,22 @@ QualType TreeTransform<Derived>::TransformDependentNameType(
   return TransformDependentNameType(TLB, TL, false);
 }
 
-template<typename Derived>
+template <typename Derived>
 QualType TreeTransform<Derived>::TransformDependentNameType(
-    TypeLocBuilder &TLB, DependentNameTypeLoc TL, bool DeducedTSTContext) {
+    TypeLocBuilder &TLB, DependentNameTypeLoc TL, bool DeducedTSTContext,
+    QualType ObjectType, NamedDecl *UnqualLookup) {
   const DependentNameType *T = TL.getTypePtr();
 
-  NestedNameSpecifierLoc QualifierLoc
-    = getDerived().TransformNestedNameSpecifierLoc(TL.getQualifierLoc());
-  if (!QualifierLoc)
-    return QualType();
+  NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+  if (QualifierLoc) {
+    QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(
+        QualifierLoc, ObjectType, UnqualLookup);
+    if (!QualifierLoc)
+      return QualType();
+  } else {
+    assert((ObjectType.isNull() && !UnqualLookup) &&
+           "must be transformed by TransformNestedNameSpecifierLoc");
+  }
 
   QualType Result
     = getDerived().RebuildDependentNameType(T->getKeyword(),
@@ -7774,15 +7785,24 @@ QualType TreeTransform<Derived>::TransformDependentNameType(
   if (Result.isNull())
     return QualType();
 
-  if (const ElaboratedType* ElabT = Result->getAs<ElaboratedType>()) {
-    QualType NamedT = ElabT->getNamedType();
-    TLB.pushTypeSpec(NamedT).setNameLoc(TL.getNameLoc());
-
-    ElaboratedTypeLoc NewTL = TLB.push<ElaboratedTypeLoc>(Result);
+  if (isa<TagType>(Result)) {
+    auto NewTL = TLB.push<TagTypeLoc>(Result);
+    NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
+    NewTL.setQualifierLoc(QualifierLoc);
+    NewTL.setNameLoc(TL.getNameLoc());
+  } else if (isa<DeducedTemplateSpecializationType>(Result)) {
+    auto NewTL = TLB.push<DeducedTemplateSpecializationTypeLoc>(Result);
     NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
+    NewTL.setTemplateNameLoc(TL.getNameLoc());
     NewTL.setQualifierLoc(QualifierLoc);
+  } else if (isa<TypedefType>(Result)) {
+    TLB.push<TypedefTypeLoc>(Result).set(TL.getElaboratedKeywordLoc(),
+                                         QualifierLoc, TL.getNameLoc());
+  } else if (isa<UnresolvedUsingType>(Result)) {
+    auto NewTL = TLB.push<UnresolvedUsingTypeLoc>(Result);
+    NewTL.set(TL.getElaboratedKeywordLoc(), QualifierLoc, TL.getNameLoc());
   } else {
-    DependentNameTypeLoc NewTL = TLB.push<DependentNameTypeLoc>(Result);
+    auto NewTL = TLB.push<DependentNameTypeLoc>(Result);
     NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
     NewTL.setQualifierLoc(QualifierLoc);
     NewTL.setNameLoc(TL.getNameLoc());
@@ -7790,33 +7810,34 @@ QualType TreeTransform<Derived>::TransformDependentNameType(
   return Result;
 }
 
-template<typename Derived>
-QualType TreeTransform<Derived>::
-          TransformDependentTemplateSpecializationType(TypeLocBuilder &TLB,
-                                 DependentTemplateSpecializationTypeLoc TL) {
-  NestedNameSpecifierLoc QualifierLoc;
-  if (TL.getQualifierLoc()) {
-    QualifierLoc
-      = getDerived().TransformNestedNameSpecifierLoc(TL.getQualifierLoc());
-    if (!QualifierLoc)
-      return QualType();
-  }
-
-  CXXScopeSpec SS;
-  SS.Adopt(QualifierLoc);
-  return getDerived().TransformDependentTemplateSpecializationType(TLB, TL, SS);
+template <typename Derived>
+QualType TreeTransform<Derived>::TransformDependentTemplateSpecializationType(
+    TypeLocBuilder &TLB, DependentTemplateSpecializationTypeLoc TL) {
+  return getDerived().TransformDependentTemplateSpecializationType(
+      TLB, TL, QualType(), nullptr, false);
 }
 
 template <typename Derived>
 QualType TreeTransform<Derived>::TransformDependentTemplateSpecializationType(
     TypeLocBuilder &TLB, DependentTemplateSpecializationTypeLoc TL,
-    CXXScopeSpec &SS) {
+    QualType ObjectType, NamedDecl *UnqualLookup, bool AllowInjectedClassName) {
   const DependentTemplateSpecializationType *T = TL.getTypePtr();
 
-  TemplateArgumentListInfo NewTemplateArgs;
-  NewTemplateArgs.setLAngleLoc(TL.getLAngleLoc());
-  NewTemplateArgs.setRAngleLoc(TL.getRAngleLoc());
+  NestedNameSpecifierLoc QualifierLoc = TL.getQualifierLoc();
+  if (QualifierLoc) {
+    QualifierLoc = getDerived().TransformNestedNameSpecifierLoc(
+        QualifierLoc, ObjectType, UnqualLookup);
+    if (!QualifierLoc)
+      return QualType();
+    // These only apply to the leftmost prefix.
+    ObjectType = QualType();
+    UnqualLookup = nullptr;
+  }
+  CXXScopeSpec SS;
+  SS.Adopt(QualifierLoc);
 
+  TemplateArgumentListInfo NewTemplateArgs(TL.getLAngleLoc(),
+                                           TL.getRAngleLoc());
   auto ArgsRange = llvm::make_range<TemplateArgumentLocContainerIterator<
       DependentTemplateSpecializationTypeLoc>>({TL, 0}, {TL, TL.getNumArgs()});
 
@@ -7833,43 +7854,27 @@ QualType TreeTransform<Derived>::TransformDependentTemplateSpecializationType(
 
   QualType Result = TL.getType();
   if (getDerived().AlwaysRebuild() || SS.getScopeRep() != DTN.getQualifier() ||
-      TemplateArgumentsChanged) {
+      TemplateArgumentsChanged || !ObjectType.isNull()) {
     TemplateName Name = getDerived().RebuildTemplateName(
         SS, TL.getTemplateKeywordLoc(), DTN.getName(), TL.getTemplateNameLoc(),
-        /*ObjectType=*/QualType(), /*FirstQualifierInScope=*/nullptr,
-        /*AllowInjectedClassName=*/false);
+        ObjectType, AllowInjectedClassName);
     if (Name.isNull())
       return QualType();
     Result = getDerived().RebuildDependentTemplateSpecializationType(
-        T->getKeyword(), SS.getScopeRep(), TL.getTemplateKeywordLoc(), Name,
+        T->getKeyword(), TL.getTemplateKeywordLoc(), Name,
         TL.getTemplateNameLoc(), NewTemplateArgs,
         /*AllowInjectedClassName=*/false);
     if (Result.isNull())
       return QualType();
   }
 
-  NestedNameSpecifierLoc QualifierLoc = SS.getWithLocInContext(SemaRef.Context);
-  if (const ElaboratedType *ElabT = dyn_cast<ElaboratedType>(Result)) {
-    QualType NamedT = ElabT->getNamedType();
-
-    // Copy information relevant to the template specialization.
-    TemplateSpecializationTypeLoc NamedTL
-      = TLB.push<TemplateSpecializationTypeLoc>(NamedT);
-    NamedTL.setTemplateKeywordLoc(TL.getTemplateKeywordLoc());
-    NamedTL.setTemplateNameLoc(TL.getTemplateNameLoc());
-    NamedTL.setLAngleLoc(TL.getLAngleLoc());
-    NamedTL.setRAngleLoc(TL.getRAngleLoc());
-    for (unsigned I = 0, E = NewTemplateArgs.size(); I != E; ++I)
-      NamedTL.setArgLocInfo(I, NewTemplateArgs[I].getLocInfo());
-
-    // Copy information relevant to the elaborated type.
-    ElaboratedTypeLoc NewTL = TLB.push<ElaboratedTypeLoc>(Result);
-    NewTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
-    NewTL.setQualifierLoc(QualifierLoc);
+  QualifierLoc = SS.getWithLocInContext(SemaRef.Context);
+  if (isa<TemplateSpecializationType>(Result)) {
+    TLB.push<TemplateSpecializationTypeLoc>(Result).set(
+        TL.getElaboratedKeywordLoc(), QualifierLoc, TL.getTemplateKeywordLoc(),
+        TL.getTemplateNameLoc(), NewTemplateArgs);
   } else {
-    assert(isa<DependentTemplateSpecializationType>(Result));
-    DependentTemplateSpecializationTypeLoc SpecTL
-      = TLB.push<DependentTemplateSpecializationTypeLoc>(Result);
+    auto SpecTL = TLB.push<DependentTemplateSpecializationTypeLoc>(Result);
     SpecTL.setElaboratedKeywordLoc(TL.getElaboratedKeywordLoc());
     SpecTL.setQualifierLoc(QualifierLoc);
     SpecTL.setTemplateKeywordLoc(TL.getTemplateKeywordLoc());
@@ -7981,8 +7986,9 @@ TreeTransform<Derived>::TransformObjCObjectType(TypeLocBuilder &TLB,
       bool RetainExpansion = false;
       UnsignedOrNone NumExpansions = PackExpansion->getNumExpansions();
       if (getDerived().TryExpandParameterPacks(
-            PackExpansionLoc.getEllipsisLoc(), PatternLoc.getSourceRange(),
-            Unexpanded, Expand, RetainExpansion, NumExpansions))
+              PackExpansionLoc.getEllipsisLoc(), PatternLoc.getSourceRange(),
+              Unexpanded, /*FailOnPackProducingTemplates=*/true, Expand,
+              RetainExpansion, NumExpansions))
         return QualType();
 
       if (!Expand) {
@@ -8590,14 +8596,18 @@ TreeTransform<Derived>::TransformDeclStmt(DeclStmt *S) {
       DeclChanged = true;
 
     if (LSI) {
-      if (auto *TD = dyn_cast<TypeDecl>(Transformed))
-        LSI->ContainsUnexpandedParameterPack |=
-            getSema()
-                .getASTContext()
-                .getTypeDeclType(TD)
-                .getSingleStepDesugaredType(getSema().getASTContext())
-                ->containsUnexpandedParameterPack();
-
+      if (auto *TD = dyn_cast<TypeDecl>(Transformed)) {
+        if (auto *TN = dyn_cast<TypedefNameDecl>(TD)) {
+          LSI->ContainsUnexpandedParameterPack |=
+              TN->getUnderlyingType()->containsUnexpandedParameterPack();
+        } else {
+          LSI->ContainsUnexpandedParameterPack |=
+              getSema()
+                  .getASTContext()
+                  .getTypeDeclType(TD)
+                  ->containsUnexpandedParameterPack();
+        }
+      }
       if (auto *VD = dyn_cast<VarDecl>(Transformed))
         LSI->ContainsUnexpandedParameterPack |=
             VD->getType()->containsUnexpandedParameterPack();
@@ -11901,8 +11911,11 @@ void OpenACCClauseTransform<Derived>::VisitPrivateClause(
       if (InitRecipe)
         InitRecipes.push_back(InitRecipe);
       else
-        InitRecipes.push_back(Self.getSema().OpenACC().CreateInitRecipe(
-            OpenACCClauseKind::Private, VarRef.get()));
+        InitRecipes.push_back(
+            Self.getSema()
+                .OpenACC()
+                .CreateInitRecipe(OpenACCClauseKind::Private, VarRef.get())
+                .first);
     }
   }
   ParsedClause.setVarListDetails(InstantiatedVarList,
@@ -11942,7 +11955,7 @@ template <typename Derived>
 void OpenACCClauseTransform<Derived>::VisitFirstPrivateClause(
     const OpenACCFirstPrivateClause &C) {
   llvm::SmallVector<Expr *> InstantiatedVarList;
-  llvm::SmallVector<VarDecl *> InitRecipes;
+  llvm::SmallVector<OpenACCFirstPrivateRecipe> InitRecipes;
 
   for (const auto [RefExpr, InitRecipe] :
        llvm::zip(C.getVarList(), C.getInitRecipes())) {
@@ -11953,7 +11966,7 @@ void OpenACCClauseTransform<Derived>::VisitFirstPrivateClause(
 
       // We only have to create a new one if it is dependent, and Sema won't
       // make one of these unless the type is non-dependent.
-      if (InitRecipe)
+      if (InitRecipe.RecipeDecl)
         InitRecipes.push_back(InitRecipe);
       else
         InitRecipes.push_back(Self.getSema().OpenACC().CreateInitRecipe(
@@ -12413,18 +12426,36 @@ void OpenACCClauseTransform<Derived>::VisitReductionClause(
     const OpenACCReductionClause &C) {
   SmallVector<Expr *> TransformedVars = VisitVarList(C.getVarList());
   SmallVector<Expr *> ValidVars;
+  llvm::SmallVector<OpenACCReductionRecipe> Recipes;
 
-  for (Expr *Var : TransformedVars) {
+  for (const auto [Var, OrigRecipes] :
+       llvm::zip(TransformedVars, C.getRecipes())) {
     ExprResult Res = Self.getSema().OpenACC().CheckReductionVar(
         ParsedClause.getDirectiveKind(), C.getReductionOp(), Var);
-    if (Res.isUsable())
+    if (Res.isUsable()) {
       ValidVars.push_back(Res.get());
+
+      // TODO OpenACC: When the recipe changes, make sure we get these right
+      // too. We probably need something similar for the operation.
+      static_assert(sizeof(OpenACCReductionRecipe) == sizeof(int*));
+      VarDecl *InitRecipe = nullptr;
+      if (OrigRecipes.RecipeDecl)
+        InitRecipe = OrigRecipes.RecipeDecl;
+       else
+         InitRecipe =
+             Self.getSema()
+                 .OpenACC()
+                 .CreateInitRecipe(OpenACCClauseKind::Reduction, Res.get())
+                 .first;
+
+      Recipes.push_back({InitRecipe});
+    }
   }
 
   NewClause = Self.getSema().OpenACC().CheckReductionClause(
       ExistingClauses, ParsedClause.getDirectiveKind(),
       ParsedClause.getBeginLoc(), ParsedClause.getLParenLoc(),
-      C.getReductionOp(), ValidVars, ParsedClause.getEndLoc());
+      C.getReductionOp(), ValidVars, Recipes, ParsedClause.getEndLoc());
 }
 
 template <typename Derived>
@@ -14350,7 +14381,7 @@ TreeTransform<Derived>::TransformCXXTypeidExpr(CXXTypeidExpr *E) {
   auto EvalCtx = Sema::ExpressionEvaluationContext::Unevaluated;
   if (E->isGLValue())
     if (auto *RecordT = Op->getType()->getAs<RecordType>())
-      if (cast<CXXRecordDecl>(RecordT->getDecl())->isPolymorphic())
+      if (cast<CXXRecordDecl>(RecordT->getOriginalDecl())->isPolymorphic())
         EvalCtx = SemaRef.ExprEvalContexts.back().Context;
 
   EnterExpressionEvaluationContext Unevaluated(SemaRef, EvalCtx,
@@ -14425,7 +14456,9 @@ TreeTransform<Derived>::TransformCXXThisExpr(CXXThisExpr *E) {
   // for type deduction, so we need to recompute it.
   //
   // Always recompute the type if we're in the body of a lambda, and
-  // 'this' is dependent on a lambda's explicit object parameter.
+  // 'this' is dependent on a lambda's explicit object parameter; we
+  // also need to always rebuild the expression in this case to clear
+  // the flag.
   QualType T = [&]() {
     auto &S = getSema();
     if (E->isCapturedByCopyInLambdaWithExplicitObjectParameter())
@@ -14435,7 +14468,8 @@ TreeTransform<Derived>::TransformCXXThisExpr(CXXThisExpr *E) {
     return S.getCurrentThisType();
   }();
 
-  if (!getDerived().AlwaysRebuild() && T == E->getType()) {
+  if (!getDerived().AlwaysRebuild() && T == E->getType() &&
+      !E->isCapturedByCopyInLambdaWithExplicitObjectParameter()) {
     // Mark it referenced in the new context regardless.
     // FIXME: this is a bit instantiation-specific.
     getSema().MarkThisReferenced(E);
@@ -14590,7 +14624,8 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
       QualType ElementType
         = SemaRef.Context.getBaseElementType(E->getAllocatedType());
       if (const RecordType *RecordT = ElementType->getAs<RecordType>()) {
-        CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordT->getDecl());
+        CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordT->getOriginalDecl())
+                                    ->getDefinitionOrSelf();
         if (CXXDestructorDecl *Destructor = SemaRef.LookupDestructor(Record)) {
           SemaRef.MarkFunctionReferenced(E->getBeginLoc(), Destructor);
         }
@@ -14660,7 +14695,9 @@ TreeTransform<Derived>::TransformCXXDeleteExpr(CXXDeleteExpr *E) {
       QualType Destroyed = SemaRef.Context.getBaseElementType(
                                                          E->getDestroyedType());
       if (const RecordType *DestroyedRec = Destroyed->getAs<RecordType>()) {
-        CXXRecordDecl *Record = cast<CXXRecordDecl>(DestroyedRec->getDecl());
+        CXXRecordDecl *Record =
+            cast<CXXRecordDecl>(DestroyedRec->getOriginalDecl())
+                ->getDefinitionOrSelf();
         SemaRef.MarkFunctionReferenced(E->getBeginLoc(),
                                        SemaRef.LookupDestructor(Record));
       }
@@ -14704,9 +14741,9 @@ TreeTransform<Derived>::TransformCXXPseudoDestructorExpr(
 
   PseudoDestructorTypeStorage Destroyed;
   if (E->getDestroyedTypeInfo()) {
-    TypeSourceInfo *DestroyedTypeInfo
-      = getDerived().TransformTypeInObjectScope(E->getDestroyedTypeInfo(),
-                                                ObjectType, nullptr, SS);
+    TypeSourceInfo *DestroyedTypeInfo = getDerived().TransformTypeInObjectScope(
+        E->getDestroyedTypeInfo(), ObjectType,
+        /*FirstQualifierInScope=*/nullptr);
     if (!DestroyedTypeInfo)
       return ExprError();
     Destroyed = DestroyedTypeInfo;
@@ -14730,9 +14767,8 @@ TreeTransform<Derived>::TransformCXXPseudoDestructorExpr(
 
   TypeSourceInfo *ScopeTypeInfo = nullptr;
   if (E->getScopeTypeInfo()) {
-    CXXScopeSpec EmptySS;
     ScopeTypeInfo = getDerived().TransformTypeInObjectScope(
-                      E->getScopeTypeInfo(), ObjectType, nullptr, EmptySS);
+        E->getScopeTypeInfo(), ObjectType, nullptr);
     if (!ScopeTypeInfo)
       return ExprError();
   }
@@ -14935,11 +14971,10 @@ TreeTransform<Derived>::TransformTypeTraitExpr(TypeTraitExpr *E) {
     UnsignedOrNone OrigNumExpansions =
         ExpansionTL.getTypePtr()->getNumExpansions();
     UnsignedOrNone NumExpansions = OrigNumExpansions;
-    if (getDerived().TryExpandParameterPacks(ExpansionTL.getEllipsisLoc(),
-                                             PatternTL.getSourceRange(),
-                                             Unexpanded,
-                                             Expand, RetainExpansion,
-                                             NumExpansions))
+    if (getDerived().TryExpandParameterPacks(
+            ExpansionTL.getEllipsisLoc(), PatternTL.getSourceRange(),
+            Unexpanded, /*FailOnPackProducingTemplates=*/true, Expand,
+            RetainExpansion, NumExpansions))
       return ExprError();
 
     if (!Expand) {
@@ -15513,8 +15548,8 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
           ExpansionTL.getTypePtr()->getNumExpansions();
       UnsignedOrNone NumExpansions = OrigNumExpansions;
       if (getDerived().TryExpandParameterPacks(
-              ExpansionTL.getEllipsisLoc(),
-              OldVD->getInit()->getSourceRange(), Unexpanded, Expand,
+              ExpansionTL.getEllipsisLoc(), OldVD->getInit()->getSourceRange(),
+              Unexpanded, /*FailOnPackProducingTemplates=*/true, Expand,
               RetainExpansion, NumExpansions))
         return ExprError();
       assert(!RetainExpansion && "Should not need to retain expansion after a "
@@ -15674,11 +15709,10 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
       bool ShouldExpand = false;
       bool RetainExpansion = false;
       UnsignedOrNone NumExpansions = std::nullopt;
-      if (getDerived().TryExpandParameterPacks(C->getEllipsisLoc(),
-                                               C->getLocation(),
-                                               Unexpanded,
-                                               ShouldExpand, RetainExpansion,
-                                               NumExpansions)) {
+      if (getDerived().TryExpandParameterPacks(
+              C->getEllipsisLoc(), C->getLocation(), Unexpanded,
+              /*FailOnPackProducingTemplates=*/true, ShouldExpand,
+              RetainExpansion, NumExpansions)) {
         Invalid = true;
         continue;
       }
@@ -15810,12 +15844,9 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
     return ExprError();
   }
 
-  // Copy the LSI before ActOnFinishFunctionBody removes it.
-  // FIXME: This is dumb. Store the lambda information somewhere that outlives
-  // the call operator.
-  auto LSICopy = *LSI;
   getSema().ActOnFinishFunctionBody(NewCallOperator, Body.get(),
-                                    /*IsInstantiation*/ true);
+                                    /*IsInstantiation=*/true,
+                                    /*RetainFunctionScopeInfo=*/true);
   SavedContext.pop();
 
   // Recompute the dependency of the lambda so that we can defer the lambda call
@@ -15851,15 +15882,11 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
   // *after* the substitution in case we can't decide the dependency
   // so early, e.g. because we want to see if any of the *substituted*
   // parameters are dependent.
-  DependencyKind = getDerived().ComputeLambdaDependency(&LSICopy);
+  DependencyKind = getDerived().ComputeLambdaDependency(LSI);
   Class->setLambdaDependencyKind(DependencyKind);
-  // Clean up the type cache created previously. Then, we re-create a type for
-  // such Decl with the new DependencyKind.
-  Class->setTypeForDecl(nullptr);
-  getSema().Context.getTypeDeclType(Class);
 
   return getDerived().RebuildLambdaExpr(E->getBeginLoc(),
-                                        Body.get()->getEndLoc(), &LSICopy);
+                                        Body.get()->getEndLoc(), LSI);
 }
 
 template<typename Derived>
@@ -16199,10 +16226,10 @@ TreeTransform<Derived>::TransformSizeOfPackExpr(SizeOfPackExpr *E) {
     bool ShouldExpand = false;
     bool RetainExpansion = false;
     UnsignedOrNone NumExpansions = std::nullopt;
-    if (getDerived().TryExpandParameterPacks(E->getOperatorLoc(), E->getPackLoc(),
-                                             Unexpanded,
-                                             ShouldExpand, RetainExpansion,
-                                             NumExpansions))
+    if (getDerived().TryExpandParameterPacks(
+            E->getOperatorLoc(), E->getPackLoc(), Unexpanded,
+            /*FailOnPackProducingTemplates=*/true, ShouldExpand,
+            RetainExpansion, NumExpansions))
       return ExprError();
 
     // If we need to expand the pack, build a template argument from it and
@@ -16319,7 +16346,8 @@ TreeTransform<Derived>::TransformPackIndexingExpr(PackIndexingExpr *E) {
                    NumExpansions = std::nullopt;
     if (getDerived().TryExpandParameterPacks(
             E->getEllipsisLoc(), Pattern->getSourceRange(), Unexpanded,
-            ShouldExpand, RetainExpansion, NumExpansions))
+            /*FailOnPackProducingTemplates=*/true, ShouldExpand,
+            RetainExpansion, NumExpansions))
       return true;
     if (!ShouldExpand) {
       Sema::ArgPackSubstIndexRAII SubstIndex(getSema(), std::nullopt);
@@ -16425,11 +16453,10 @@ TreeTransform<Derived>::TransformCXXFoldExpr(CXXFoldExpr *E) {
   bool RetainExpansion = false;
   UnsignedOrNone OrigNumExpansions = E->getNumExpansions(),
                  NumExpansions = OrigNumExpansions;
-  if (getDerived().TryExpandParameterPacks(E->getEllipsisLoc(),
-                                           Pattern->getSourceRange(),
-                                           Unexpanded,
-                                           Expand, RetainExpansion,
-                                           NumExpansions))
+  if (getDerived().TryExpandParameterPacks(
+          E->getEllipsisLoc(), Pattern->getSourceRange(), Unexpanded,
+          /*FailOnPackProducingTemplates=*/true, Expand, RetainExpansion,
+          NumExpansions))
     return true;
 
   if (!Expand) {
@@ -16663,9 +16690,10 @@ TreeTransform<Derived>::TransformObjCDictionaryLiteral(
       UnsignedOrNone NumExpansions = OrigNumExpansions;
       SourceRange PatternRange(OrigElement.Key->getBeginLoc(),
                                OrigElement.Value->getEndLoc());
-      if (getDerived().TryExpandParameterPacks(OrigElement.EllipsisLoc,
-                                               PatternRange, Unexpanded, Expand,
-                                               RetainExpansion, NumExpansions))
+      if (getDerived().TryExpandParameterPacks(
+              OrigElement.EllipsisLoc, PatternRange, Unexpanded,
+              /*FailOnPackProducingTemplates=*/true, Expand, RetainExpansion,
+              NumExpansions))
         return ExprError();
 
       if (!Expand) {
@@ -17359,9 +17387,10 @@ QualType TreeTransform<Derived>::RebuildFunctionNoProtoType(QualType T) {
   return SemaRef.Context.getFunctionNoProtoType(T);
 }
 
-template<typename Derived>
-QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(SourceLocation Loc,
-                                                            Decl *D) {
+template <typename Derived>
+QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(
+    ElaboratedTypeKeyword Keyword, NestedNameSpecifier Qualifier,
+    SourceLocation NameLoc, Decl *D) {
   assert(D && "no decl found");
   if (D->isInvalidDecl()) return QualType();
 
@@ -17371,7 +17400,7 @@ QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(SourceLocation Loc,
     // UsingDecls, but they must each have exactly one type, and it must be
     // the same type in every case. But we must have at least one expansion!
     if (UPD->expansions().empty()) {
-      getSema().Diag(Loc, diag::err_using_pack_expansion_empty)
+      getSema().Diag(NameLoc, diag::err_using_pack_expansion_empty)
           << UPD->isCXXClassMember() << UPD;
       return QualType();
     }
@@ -17382,10 +17411,11 @@ QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(SourceLocation Loc,
     QualType FallbackT;
     QualType T;
     for (auto *E : UPD->expansions()) {
-      QualType ThisT = RebuildUnresolvedUsingType(Loc, E);
+      QualType ThisT =
+          RebuildUnresolvedUsingType(Keyword, Qualifier, NameLoc, E);
       if (ThisT.isNull())
         continue;
-      else if (ThisT->getAs<UnresolvedUsingType>())
+      if (ThisT->getAs<UnresolvedUsingType>())
         FallbackT = ThisT;
       else if (T.isNull())
         T = ThisT;
@@ -17394,7 +17424,8 @@ QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(SourceLocation Loc,
                "mismatched resolved types in using pack expansion");
     }
     return T.isNull() ? FallbackT : T;
-  } else if (auto *Using = dyn_cast<UsingDecl>(D)) {
+  }
+  if (auto *Using = dyn_cast<UsingDecl>(D)) {
     assert(Using->hasTypename() &&
            "UnresolvedUsingTypenameDecl transformed to non-typename using");
 
@@ -17402,17 +17433,14 @@ QualType TreeTransform<Derived>::RebuildUnresolvedUsingType(SourceLocation Loc,
     assert(++Using->shadow_begin() == Using->shadow_end());
 
     UsingShadowDecl *Shadow = *Using->shadow_begin();
-    if (SemaRef.DiagnoseUseOfDecl(Shadow->getTargetDecl(), Loc))
+    if (SemaRef.DiagnoseUseOfDecl(Shadow->getTargetDecl(), NameLoc))
       return QualType();
-    return SemaRef.Context.getUsingType(
-        Shadow, SemaRef.Context.getTypeDeclType(
-                    cast<TypeDecl>(Shadow->getTargetDecl())));
-  } else {
-    assert(isa<UnresolvedUsingTypenameDecl>(D) &&
-           "UnresolvedUsingTypenameDecl transformed to non-using decl");
-    return SemaRef.Context.getTypeDeclType(
-        cast<UnresolvedUsingTypenameDecl>(D));
+    return SemaRef.Context.getUsingType(Keyword, Qualifier, Shadow);
   }
+  assert(isa<UnresolvedUsingTypenameDecl>(D) &&
+         "UnresolvedUsingTypenameDecl transformed to non-using decl");
+  return SemaRef.Context.getUnresolvedUsingType(
+      Keyword, Qualifier, cast<UnresolvedUsingTypenameDecl>(D));
 }
 
 template <typename Derived>
@@ -17448,12 +17476,12 @@ QualType TreeTransform<Derived>::RebuildUnaryTransformType(QualType BaseType,
   return SemaRef.BuildUnaryTransformType(BaseType, UKind, Loc);
 }
 
-template<typename Derived>
+template <typename Derived>
 QualType TreeTransform<Derived>::RebuildTemplateSpecializationType(
-                                                      TemplateName Template,
-                                             SourceLocation TemplateNameLoc,
-                                     TemplateArgumentListInfo &TemplateArgs) {
-  return SemaRef.CheckTemplateIdType(Template, TemplateNameLoc, TemplateArgs);
+    ElaboratedTypeKeyword Keyword, TemplateName Template,
+    SourceLocation TemplateNameLoc, TemplateArgumentListInfo &TemplateArgs) {
+  return SemaRef.CheckTemplateIdType(Keyword, Template, TemplateNameLoc,
+                                     TemplateArgs);
 }
 
 template<typename Derived>
@@ -17496,15 +17524,10 @@ TreeTransform<Derived>::RebuildTemplateName(CXXScopeSpec &SS,
                                                   TemplateName(Template));
 }
 
-template<typename Derived>
-TemplateName
-TreeTransform<Derived>::RebuildTemplateName(CXXScopeSpec &SS,
-                                            SourceLocation TemplateKWLoc,
-                                            const IdentifierInfo &Name,
-                                            SourceLocation NameLoc,
-                                            QualType ObjectType,
-                                            NamedDecl *FirstQualifierInScope,
-                                            bool AllowInjectedClassName) {
+template <typename Derived>
+TemplateName TreeTransform<Derived>::RebuildTemplateName(
+    CXXScopeSpec &SS, SourceLocation TemplateKWLoc, const IdentifierInfo &Name,
+    SourceLocation NameLoc, QualType ObjectType, bool AllowInjectedClassName) {
   UnqualifiedId TemplateName;
   TemplateName.setIdentifier(&Name, NameLoc);
   Sema::TemplateTy Template;
@@ -17646,15 +17669,16 @@ TreeTransform<Derived>::RebuildCXXPseudoDestructorExpr(Expr *Base,
   NameInfo.setNamedTypeInfo(DestroyedType);
 
   // The scope type is now known to be a valid nested name specifier
-  // component. Tack it on to the end of the nested name specifier.
+  // component. Tack it on to the nested name specifier.
   if (ScopeType) {
-    if (!ScopeType->getType()->getAs<TagType>()) {
+    if (!isa<TagType>(ScopeType->getType().getCanonicalType())) {
       getSema().Diag(ScopeType->getTypeLoc().getBeginLoc(),
                      diag::err_expected_class_or_namespace)
           << ScopeType->getType() << getSema().getLangOpts().CPlusPlus;
       return ExprError();
     }
-    SS.Extend(SemaRef.Context, ScopeType->getTypeLoc(), CCLoc);
+    SS.clear();
+    SS.Make(SemaRef.Context, ScopeType->getTypeLoc(), CCLoc);
   }
 
   SourceLocation TemplateKWLoc; // FIXME: retrieve it from caller.
diff --git a/clang/lib/Sema/UsedDeclVisitor.h b/clang/lib/Sema/UsedDeclVisitor.h
index 580d702f96fe5..ad475ab0f42ae 100644
--- a/clang/lib/Sema/UsedDeclVisitor.h
+++ b/clang/lib/Sema/UsedDeclVisitor.h
@@ -71,9 +71,10 @@ class UsedDeclVisitor : public EvaluatedExprVisitor<Derived> {
     if (!DestroyedOrNull.isNull()) {
       QualType Destroyed = S.Context.getBaseElementType(DestroyedOrNull);
       if (const RecordType *DestroyedRec = Destroyed->getAs<RecordType>()) {
-        CXXRecordDecl *Record = cast<CXXRecordDecl>(DestroyedRec->getDecl());
-        if (Record->getDefinition())
-          asImpl().visitUsedDecl(E->getBeginLoc(), S.LookupDestructor(Record));
+        CXXRecordDecl *Record =
+            cast<CXXRecordDecl>(DestroyedRec->getOriginalDecl());
+        if (auto *Def = Record->getDefinition())
+          asImpl().visitUsedDecl(E->getBeginLoc(), S.LookupDestructor(Def));
       }
     }
 
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 1402f40894ab8..32f7779278286 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -5500,7 +5500,7 @@ void ASTReader::InitializeContext() {
             Error("Invalid FILE type in AST file");
             return;
           }
-          Context.setFILEDecl(Tag->getDecl());
+          Context.setFILEDecl(Tag->getOriginalDecl());
         }
       }
     }
@@ -5521,7 +5521,7 @@ void ASTReader::InitializeContext() {
             Error("Invalid jmp_buf type in AST file");
             return;
           }
-          Context.setjmp_bufDecl(Tag->getDecl());
+          Context.setjmp_bufDecl(Tag->getOriginalDecl());
         }
       }
     }
@@ -5539,7 +5539,7 @@ void ASTReader::InitializeContext() {
         else {
           const TagType *Tag = Sigjmp_bufType->getAs<TagType>();
           assert(Tag && "Invalid sigjmp_buf type in AST file");
-          Context.setsigjmp_bufDecl(Tag->getDecl());
+          Context.setsigjmp_bufDecl(Tag->getOriginalDecl());
         }
       }
     }
@@ -5574,7 +5574,7 @@ void ASTReader::InitializeContext() {
         else {
           const TagType *Tag = Ucontext_tType->getAs<TagType>();
           assert(Tag && "Invalid ucontext_t type in AST file");
-          Context.setucontext_tDecl(Tag->getDecl());
+          Context.setucontext_tDecl(Tag->getOriginalDecl());
         }
       }
     }
@@ -7226,6 +7226,7 @@ class TypeLocReader : public TypeLocVisitor<TypeLocReader> {
 
   void VisitFunctionTypeLoc(FunctionTypeLoc);
   void VisitArrayTypeLoc(ArrayTypeLoc);
+  void VisitTagTypeLoc(TagTypeLoc TL);
 };
 
 } // namespace clang
@@ -7372,15 +7373,24 @@ void TypeLocReader::VisitFunctionNoProtoTypeLoc(FunctionNoProtoTypeLoc TL) {
 }
 
 void TypeLocReader::VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc TL) {
-  TL.setNameLoc(readSourceLocation());
+  SourceLocation ElaboratedKeywordLoc = readSourceLocation();
+  NestedNameSpecifierLoc QualifierLoc = ReadNestedNameSpecifierLoc();
+  SourceLocation NameLoc = readSourceLocation();
+  TL.set(ElaboratedKeywordLoc, QualifierLoc, NameLoc);
 }
 
 void TypeLocReader::VisitUsingTypeLoc(UsingTypeLoc TL) {
-  TL.setNameLoc(readSourceLocation());
+  SourceLocation ElaboratedKeywordLoc = readSourceLocation();
+  NestedNameSpecifierLoc QualifierLoc = ReadNestedNameSpecifierLoc();
+  SourceLocation NameLoc = readSourceLocation();
+  TL.set(ElaboratedKeywordLoc, QualifierLoc, NameLoc);
 }
 
 void TypeLocReader::VisitTypedefTypeLoc(TypedefTypeLoc TL) {
-  TL.setNameLoc(readSourceLocation());
+  SourceLocation ElaboratedKeywordLoc = readSourceLocation();
+  NestedNameSpecifierLoc QualifierLoc = ReadNestedNameSpecifierLoc();
+  SourceLocation NameLoc = readSourceLocation();
+  TL.set(ElaboratedKeywordLoc, QualifierLoc, NameLoc);
 }
 
 void TypeLocReader::VisitTypeOfExprTypeLoc(TypeOfExprTypeLoc TL) {
@@ -7434,17 +7444,27 @@ void TypeLocReader::VisitAutoTypeLoc(AutoTypeLoc TL) {
 
 void TypeLocReader::VisitDeducedTemplateSpecializationTypeLoc(
     DeducedTemplateSpecializationTypeLoc TL) {
+  TL.setElaboratedKeywordLoc(readSourceLocation());
+  TL.setQualifierLoc(ReadNestedNameSpecifierLoc());
   TL.setTemplateNameLoc(readSourceLocation());
 }
 
-void TypeLocReader::VisitRecordTypeLoc(RecordTypeLoc TL) {
+void TypeLocReader::VisitTagTypeLoc(TagTypeLoc TL) {
+  TL.setElaboratedKeywordLoc(readSourceLocation());
+  TL.setQualifierLoc(ReadNestedNameSpecifierLoc());
   TL.setNameLoc(readSourceLocation());
 }
 
-void TypeLocReader::VisitEnumTypeLoc(EnumTypeLoc TL) {
-  TL.setNameLoc(readSourceLocation());
+void TypeLocReader::VisitRecordTypeLoc(RecordTypeLoc TL) {
+  VisitTagTypeLoc(TL);
+}
+
+void TypeLocReader::VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc TL) {
+  VisitTagTypeLoc(TL);
 }
 
+void TypeLocReader::VisitEnumTypeLoc(EnumTypeLoc TL) { VisitTagTypeLoc(TL); }
+
 void TypeLocReader::VisitAttributedTypeLoc(AttributedTypeLoc TL) {
   TL.setAttr(ReadAttr());
 }
@@ -7480,16 +7500,25 @@ void TypeLocReader::VisitSubstTemplateTypeParmPackTypeLoc(
   TL.setNameLoc(readSourceLocation());
 }
 
+void TypeLocReader::VisitSubstBuiltinTemplatePackTypeLoc(
+    SubstBuiltinTemplatePackTypeLoc TL) {
+  TL.setNameLoc(readSourceLocation());
+}
+
 void TypeLocReader::VisitTemplateSpecializationTypeLoc(
                                            TemplateSpecializationTypeLoc TL) {
-  TL.setTemplateKeywordLoc(readSourceLocation());
-  TL.setTemplateNameLoc(readSourceLocation());
-  TL.setLAngleLoc(readSourceLocation());
-  TL.setRAngleLoc(readSourceLocation());
-  for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
-    TL.setArgLocInfo(i,
-                     Reader.readTemplateArgumentLocInfo(
-                         TL.getTypePtr()->template_arguments()[i].getKind()));
+  SourceLocation ElaboratedKeywordLoc = readSourceLocation();
+  NestedNameSpecifierLoc QualifierLoc = ReadNestedNameSpecifierLoc();
+  SourceLocation TemplateKeywordLoc = readSourceLocation();
+  SourceLocation NameLoc = readSourceLocation();
+  SourceLocation LAngleLoc = readSourceLocation();
+  SourceLocation RAngleLoc = readSourceLocation();
+  TL.set(ElaboratedKeywordLoc, QualifierLoc, TemplateKeywordLoc, NameLoc,
+         LAngleLoc, RAngleLoc);
+  MutableArrayRef<TemplateArgumentLocInfo> Args = TL.getArgLocInfos();
+  for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I)
+    Args[I] = Reader.readTemplateArgumentLocInfo(
+        TL.getTypePtr()->template_arguments()[I].getKind());
 }
 
 void TypeLocReader::VisitParenTypeLoc(ParenTypeLoc TL) {
@@ -7497,15 +7526,6 @@ void TypeLocReader::VisitParenTypeLoc(ParenTypeLoc TL) {
   TL.setRParenLoc(readSourceLocation());
 }
 
-void TypeLocReader::VisitElaboratedTypeLoc(ElaboratedTypeLoc TL) {
-  TL.setElaboratedKeywordLoc(readSourceLocation());
-  TL.setQualifierLoc(ReadNestedNameSpecifierLoc());
-}
-
-void TypeLocReader::VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc TL) {
-  TL.setNameLoc(readSourceLocation());
-}
-
 void TypeLocReader::VisitDependentNameTypeLoc(DependentNameTypeLoc TL) {
   TL.setElaboratedKeywordLoc(readSourceLocation());
   TL.setQualifierLoc(ReadNestedNameSpecifierLoc());
@@ -7962,18 +7982,15 @@ ASTRecordReader::readTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind) {
     return readExpr();
   case TemplateArgument::Type:
     return readTypeSourceInfo();
-  case TemplateArgument::Template: {
-    NestedNameSpecifierLoc QualifierLoc =
-      readNestedNameSpecifierLoc();
-    SourceLocation TemplateNameLoc = readSourceLocation();
-    return TemplateArgumentLocInfo(getASTContext(), QualifierLoc,
-                                   TemplateNameLoc, SourceLocation());
-  }
+  case TemplateArgument::Template:
   case TemplateArgument::TemplateExpansion: {
+    SourceLocation TemplateKWLoc = readSourceLocation();
     NestedNameSpecifierLoc QualifierLoc = readNestedNameSpecifierLoc();
     SourceLocation TemplateNameLoc = readSourceLocation();
-    SourceLocation EllipsisLoc = readSourceLocation();
-    return TemplateArgumentLocInfo(getASTContext(), QualifierLoc,
+    SourceLocation EllipsisLoc = Kind == TemplateArgument::TemplateExpansion
+                                     ? readSourceLocation()
+                                     : SourceLocation();
+    return TemplateArgumentLocInfo(getASTContext(), TemplateKWLoc, QualifierLoc,
                                    TemplateNameLoc, EllipsisLoc);
   }
   case TemplateArgument::Null:
@@ -9586,12 +9603,6 @@ void ASTReader::AssignedLambdaNumbering(CXXRecordDecl *Lambda) {
     CXXRecordDecl *Previous =
         cast<CXXRecordDecl>(Iter->second)->getMostRecentDecl();
     Lambda->setPreviousDecl(Previous);
-    // FIXME: It will be best to use the Previous type when we creating the
-    // lambda directly. But that requires us to get the lambda context decl and
-    // lambda index before creating the lambda, which needs a drastic change in
-    // the parser.
-    const_cast<QualType &>(Lambda->TypeForDecl->CanonicalType) =
-        Previous->TypeForDecl->CanonicalType;
     return;
   }
 
@@ -10107,41 +10118,37 @@ ASTRecordReader::readNestedNameSpecifierLoc() {
   for (unsigned I = 0; I != N; ++I) {
     auto Kind = readNestedNameSpecifierKind();
     switch (Kind) {
-    case NestedNameSpecifier::Identifier: {
-      IdentifierInfo *II = readIdentifier();
-      SourceRange Range = readSourceRange();
-      Builder.Extend(Context, II, Range.getBegin(), Range.getEnd());
-      break;
-    }
-
-    case NestedNameSpecifier::Namespace: {
+    case NestedNameSpecifier::Kind::Namespace: {
       auto *NS = readDeclAs<NamespaceBaseDecl>();
       SourceRange Range = readSourceRange();
       Builder.Extend(Context, NS, Range.getBegin(), Range.getEnd());
       break;
     }
 
-    case NestedNameSpecifier::TypeSpec: {
+    case NestedNameSpecifier::Kind::Type: {
       TypeSourceInfo *T = readTypeSourceInfo();
       if (!T)
         return NestedNameSpecifierLoc();
       SourceLocation ColonColonLoc = readSourceLocation();
-      Builder.Extend(Context, T->getTypeLoc(), ColonColonLoc);
+      Builder.Make(Context, T->getTypeLoc(), ColonColonLoc);
       break;
     }
 
-    case NestedNameSpecifier::Global: {
+    case NestedNameSpecifier::Kind::Global: {
       SourceLocation ColonColonLoc = readSourceLocation();
       Builder.MakeGlobal(Context, ColonColonLoc);
       break;
     }
 
-    case NestedNameSpecifier::Super: {
+    case NestedNameSpecifier::Kind::MicrosoftSuper: {
       CXXRecordDecl *RD = readDeclAs<CXXRecordDecl>();
       SourceRange Range = readSourceRange();
-      Builder.MakeSuper(Context, RD, Range.getBegin(), Range.getEnd());
+      Builder.MakeMicrosoftSuper(Context, RD, Range.getBegin(), Range.getEnd());
       break;
     }
+
+    case NestedNameSpecifier::Kind::Null:
+      llvm_unreachable("unexpected null nested name specifier");
     }
   }
 
@@ -10537,12 +10544,7 @@ void ASTReader::finishPendingActions() {
   // happen now, after the redeclaration chains have been fully wired.
   for (Decl *D : PendingDefinitions) {
     if (TagDecl *TD = dyn_cast<TagDecl>(D)) {
-      if (const TagType *TagT = dyn_cast<TagType>(TD->getTypeForDecl())) {
-        // Make sure that the TagType points at the definition.
-        const_cast<TagType*>(TagT)->decl = TD;
-      }
-
-      if (auto RD = dyn_cast<CXXRecordDecl>(D)) {
+      if (auto *RD = dyn_cast<CXXRecordDecl>(TD)) {
         for (auto *R = getMostRecentExistingDecl(RD); R;
              R = R->getPreviousDecl()) {
           assert((R == D) ==
@@ -11006,8 +11008,9 @@ void ASTReader::diagnoseOdrViolations() {
 }
 
 void ASTReader::StartedDeserializing() {
-  if (++NumCurrentElementsDeserializing == 1 && ReadTimer.get())
-    ReadTimer->startTimer();
+  if (llvm::Timer *T = ReadTimer.get();
+      ++NumCurrentElementsDeserializing == 1 && T)
+    ReadTimeRegion.emplace(T);
 }
 
 void ASTReader::FinishedDeserializing() {
@@ -11065,8 +11068,7 @@ void ASTReader::FinishedDeserializing() {
           (void)UndeducedFD->getMostRecentDecl();
       }
 
-      if (ReadTimer)
-        ReadTimer->stopTimer();
+      ReadTimeRegion.reset();
 
       diagnoseOdrViolations();
     }
@@ -12877,9 +12879,12 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::FirstPrivate: {
     SourceLocation LParenLoc = readSourceLocation();
     llvm::SmallVector<Expr *> VarList = readOpenACCVarList();
-    llvm::SmallVector<VarDecl *> RecipeList;
-    for (unsigned I = 0; I < VarList.size(); ++I)
-      RecipeList.push_back(readDeclAs<VarDecl>());
+    llvm::SmallVector<OpenACCFirstPrivateRecipe> RecipeList;
+    for (unsigned I = 0; I < VarList.size(); ++I) {
+      VarDecl *Recipe = readDeclAs<VarDecl>();
+      VarDecl *RecipeTemp = readDeclAs<VarDecl>();
+      RecipeList.push_back({Recipe, RecipeTemp});
+    }
 
     return OpenACCFirstPrivateClause::Create(getContext(), BeginLoc, LParenLoc,
                                              VarList, RecipeList, EndLoc);
@@ -12996,8 +13001,16 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     SourceLocation LParenLoc = readSourceLocation();
     OpenACCReductionOperator Op = readEnum<OpenACCReductionOperator>();
     llvm::SmallVector<Expr *> VarList = readOpenACCVarList();
+    llvm::SmallVector<OpenACCReductionRecipe> RecipeList;
+
+    for (unsigned I = 0; I < VarList.size(); ++I) {
+      VarDecl *Recipe = readDeclAs<VarDecl>();
+      static_assert(sizeof(OpenACCReductionRecipe) == sizeof(int *));
+      RecipeList.push_back({Recipe});
+    }
+
     return OpenACCReductionClause::Create(getContext(), BeginLoc, LParenLoc, Op,
-                                          VarList, EndLoc);
+                                          VarList, RecipeList, EndLoc);
   }
   case OpenACCClauseKind::Seq:
     return OpenACCSeqClause::Create(getContext(), BeginLoc, EndLoc);
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 2c7beb431170e..6b35b205079e5 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -538,7 +538,11 @@ void ASTDeclReader::Visit(Decl *D) {
 
   if (auto *TD = dyn_cast<TypeDecl>(D)) {
     // We have a fully initialized TypeDecl. Read its type now.
-    TD->setTypeForDecl(Reader.GetType(DeferredTypeID).getTypePtrOrNull());
+    if (isa<TagDecl, TypedefDecl, TypeAliasDecl>(TD))
+      assert(DeferredTypeID == 0 &&
+             "Deferred type not used for TagDecls and Typedefs");
+    else
+      TD->setTypeForDecl(Reader.GetType(DeferredTypeID).getTypePtrOrNull());
 
     // If this is a tag declaration with a typedef name for linkage, it's safe
     // to load that typedef now.
@@ -695,7 +699,8 @@ void ASTDeclReader::VisitTypeDecl(TypeDecl *TD) {
   VisitNamedDecl(TD);
   TD->setLocStart(readSourceLocation());
   // Delay type reading until after we have fully initialized the decl.
-  DeferredTypeID = Record.getGlobalTypeID(Record.readInt());
+  if (!isa<TagDecl, TypedefDecl, TypeAliasDecl>(TD))
+    DeferredTypeID = Record.getGlobalTypeID(Record.readInt());
 }
 
 RedeclarableResult ASTDeclReader::VisitTypedefNameDecl(TypedefNameDecl *TD) {
@@ -2237,15 +2242,6 @@ RedeclarableResult ASTDeclReader::VisitCXXRecordDeclImpl(CXXRecordDecl *D) {
     // Merged when we merge the template.
     auto *Template = readDeclAs<ClassTemplateDecl>();
     D->TemplateOrInstantiation = Template;
-    if (!Template->getTemplatedDecl()) {
-      // We've not actually loaded the ClassTemplateDecl yet, because we're
-      // currently being loaded as its pattern. Rely on it to set up our
-      // TypeForDecl (see VisitClassTemplateDecl).
-      //
-      // Beware: we do not yet know our canonical declaration, and may still
-      // get merged once the surrounding class template has got off the ground.
-      DeferredTypeID = 0;
-    }
     break;
   }
   case CXXRecMemberSpecialization: {
@@ -2479,14 +2475,6 @@ void ASTDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
     ReadSpecializations(*Loc.F, D, Loc.F->DeclsCursor, /*IsPartial=*/false);
     ReadSpecializations(*Loc.F, D, Loc.F->DeclsCursor, /*IsPartial=*/true);
   }
-
-  if (D->getTemplatedDecl()->TemplateOrInstantiation) {
-    // We were loaded before our templated declaration was. We've not set up
-    // its corresponding type yet (see VisitCXXRecordDeclImpl), so reconstruct
-    // it now.
-    Reader.getContext().getInjectedClassNameType(
-        D->getTemplatedDecl(), D->getInjectedClassNameSpecialization());
-  }
 }
 
 void ASTDeclReader::VisitBuiltinTemplateDecl(BuiltinTemplateDecl *D) {
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index c038d4dd5c942..87e462f179ce6 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -345,6 +345,7 @@ class TypeLocWriter : public TypeLocVisitor<TypeLocWriter> {
 
   void VisitArrayTypeLoc(ArrayTypeLoc TyLoc);
   void VisitFunctionTypeLoc(FunctionTypeLoc TyLoc);
+  void VisitTagTypeLoc(TagTypeLoc TL);
 };
 
 } // namespace
@@ -490,14 +491,20 @@ void TypeLocWriter::VisitFunctionNoProtoTypeLoc(FunctionNoProtoTypeLoc TL) {
 }
 
 void TypeLocWriter::VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc TL) {
+  addSourceLocation(TL.getElaboratedKeywordLoc());
+  Record.AddNestedNameSpecifierLoc(TL.getQualifierLoc());
   addSourceLocation(TL.getNameLoc());
 }
 
 void TypeLocWriter::VisitUsingTypeLoc(UsingTypeLoc TL) {
+  addSourceLocation(TL.getElaboratedKeywordLoc());
+  Record.AddNestedNameSpecifierLoc(TL.getQualifierLoc());
   addSourceLocation(TL.getNameLoc());
 }
 
 void TypeLocWriter::VisitTypedefTypeLoc(TypedefTypeLoc TL) {
+  addSourceLocation(TL.getElaboratedKeywordLoc());
+  Record.AddNestedNameSpecifierLoc(TL.getQualifierLoc());
   addSourceLocation(TL.getNameLoc());
 }
 
@@ -564,17 +571,27 @@ void TypeLocWriter::VisitAutoTypeLoc(AutoTypeLoc TL) {
 
 void TypeLocWriter::VisitDeducedTemplateSpecializationTypeLoc(
     DeducedTemplateSpecializationTypeLoc TL) {
+  addSourceLocation(TL.getElaboratedKeywordLoc());
+  Record.AddNestedNameSpecifierLoc(TL.getQualifierLoc());
   addSourceLocation(TL.getTemplateNameLoc());
 }
 
-void TypeLocWriter::VisitRecordTypeLoc(RecordTypeLoc TL) {
+void TypeLocWriter::VisitTagTypeLoc(TagTypeLoc TL) {
+  addSourceLocation(TL.getElaboratedKeywordLoc());
+  Record.AddNestedNameSpecifierLoc(TL.getQualifierLoc());
   addSourceLocation(TL.getNameLoc());
 }
 
-void TypeLocWriter::VisitEnumTypeLoc(EnumTypeLoc TL) {
-  addSourceLocation(TL.getNameLoc());
+void TypeLocWriter::VisitRecordTypeLoc(RecordTypeLoc TL) {
+  VisitTagTypeLoc(TL);
+}
+
+void TypeLocWriter::VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc TL) {
+  VisitTagTypeLoc(TL);
 }
 
+void TypeLocWriter::VisitEnumTypeLoc(EnumTypeLoc TL) { VisitTagTypeLoc(TL); }
+
 void TypeLocWriter::VisitAttributedTypeLoc(AttributedTypeLoc TL) {
   Record.AddAttr(TL.getAttr());
 }
@@ -610,15 +627,21 @@ void TypeLocWriter::VisitSubstTemplateTypeParmPackTypeLoc(
   addSourceLocation(TL.getNameLoc());
 }
 
+void TypeLocWriter::VisitSubstBuiltinTemplatePackTypeLoc(
+    SubstBuiltinTemplatePackTypeLoc TL) {
+  addSourceLocation(TL.getNameLoc());
+}
+
 void TypeLocWriter::VisitTemplateSpecializationTypeLoc(
                                            TemplateSpecializationTypeLoc TL) {
+  addSourceLocation(TL.getElaboratedKeywordLoc());
+  Record.AddNestedNameSpecifierLoc(TL.getQualifierLoc());
   addSourceLocation(TL.getTemplateKeywordLoc());
   addSourceLocation(TL.getTemplateNameLoc());
   addSourceLocation(TL.getLAngleLoc());
   addSourceLocation(TL.getRAngleLoc());
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
-    Record.AddTemplateArgumentLocInfo(TL.getArgLoc(i).getArgument().getKind(),
-                                      TL.getArgLoc(i).getLocInfo());
+    Record.AddTemplateArgumentLocInfo(TL.getArgLoc(i));
 }
 
 void TypeLocWriter::VisitParenTypeLoc(ParenTypeLoc TL) {
@@ -630,15 +653,6 @@ void TypeLocWriter::VisitMacroQualifiedTypeLoc(MacroQualifiedTypeLoc TL) {
   addSourceLocation(TL.getExpansionLoc());
 }
 
-void TypeLocWriter::VisitElaboratedTypeLoc(ElaboratedTypeLoc TL) {
-  addSourceLocation(TL.getElaboratedKeywordLoc());
-  Record.AddNestedNameSpecifierLoc(TL.getQualifierLoc());
-}
-
-void TypeLocWriter::VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc TL) {
-  addSourceLocation(TL.getNameLoc());
-}
-
 void TypeLocWriter::VisitDependentNameTypeLoc(DependentNameTypeLoc TL) {
   addSourceLocation(TL.getElaboratedKeywordLoc());
   Record.AddNestedNameSpecifierLoc(TL.getQualifierLoc());
@@ -654,8 +668,7 @@ void TypeLocWriter::VisitDependentTemplateSpecializationTypeLoc(
   addSourceLocation(TL.getLAngleLoc());
   addSourceLocation(TL.getRAngleLoc());
   for (unsigned I = 0, E = TL.getNumArgs(); I != E; ++I)
-    Record.AddTemplateArgumentLocInfo(TL.getArgLoc(I).getArgument().getKind(),
-                                      TL.getArgLoc(I).getLocInfo());
+    Record.AddTemplateArgumentLocInfo(TL.getArgLoc(I));
 }
 
 void TypeLocWriter::VisitPackExpansionTypeLoc(PackExpansionTypeLoc TL) {
@@ -1038,7 +1051,6 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(TYPE_OBJC_INTERFACE);
   RECORD(TYPE_OBJC_OBJECT_POINTER);
   RECORD(TYPE_DECLTYPE);
-  RECORD(TYPE_ELABORATED);
   RECORD(TYPE_SUBST_TEMPLATE_TYPE_PARM);
   RECORD(TYPE_UNRESOLVED_USING);
   RECORD(TYPE_INJECTED_CLASS_NAME);
@@ -1053,6 +1065,7 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(TYPE_PACK_EXPANSION);
   RECORD(TYPE_ATTRIBUTED);
   RECORD(TYPE_SUBST_TEMPLATE_TYPE_PARM_PACK);
+  RECORD(TYPE_SUBST_BUILTIN_TEMPLATE_PACK);
   RECORD(TYPE_AUTO);
   RECORD(TYPE_UNARY_TRANSFORM);
   RECORD(TYPE_ATOMIC);
@@ -5461,11 +5474,6 @@ ASTWriter::WriteAST(llvm::PointerUnion<Sema *, Preprocessor *> Subject,
 
   WritingAST = false;
 
-  if (WritingModule && PPRef.getHeaderSearchInfo()
-                           .getHeaderSearchOpts()
-                           .ModulesValidateOncePerBuildSession)
-    ModCache.updateModuleTimestamp(OutputFile);
-
   if (ShouldCacheASTInMemory) {
     // Construct MemoryBuffer and update buffer manager.
     ModCache.getInMemoryModuleCache().addBuiltPCM(
@@ -6774,22 +6782,22 @@ void ASTRecordWriter::AddCXXTemporary(const CXXTemporary *Temp) {
 }
 
 void ASTRecordWriter::AddTemplateArgumentLocInfo(
-    TemplateArgument::ArgKind Kind, const TemplateArgumentLocInfo &Arg) {
-  switch (Kind) {
+    const TemplateArgumentLoc &Arg) {
+  const TemplateArgumentLocInfo &Info = Arg.getLocInfo();
+  switch (auto K = Arg.getArgument().getKind()) {
   case TemplateArgument::Expression:
-    AddStmt(Arg.getAsExpr());
+    AddStmt(Info.getAsExpr());
     break;
   case TemplateArgument::Type:
-    AddTypeSourceInfo(Arg.getAsTypeSourceInfo());
+    AddTypeSourceInfo(Info.getAsTypeSourceInfo());
     break;
   case TemplateArgument::Template:
-    AddNestedNameSpecifierLoc(Arg.getTemplateQualifierLoc());
-    AddSourceLocation(Arg.getTemplateNameLoc());
-    break;
   case TemplateArgument::TemplateExpansion:
+    AddSourceLocation(Arg.getTemplateKWLoc());
     AddNestedNameSpecifierLoc(Arg.getTemplateQualifierLoc());
     AddSourceLocation(Arg.getTemplateNameLoc());
-    AddSourceLocation(Arg.getTemplateEllipsisLoc());
+    if (K == TemplateArgument::TemplateExpansion)
+      AddSourceLocation(Arg.getTemplateEllipsisLoc());
     break;
   case TemplateArgument::Null:
   case TemplateArgument::Integral:
@@ -6812,7 +6820,7 @@ void ASTRecordWriter::AddTemplateArgumentLoc(const TemplateArgumentLoc &Arg) {
     if (InfoHasSameExpr)
       return; // Avoid storing the same expr twice.
   }
-  AddTemplateArgumentLocInfo(Arg.getArgument().getKind(), Arg.getLocInfo());
+  AddTemplateArgumentLocInfo(Arg);
 }
 
 void ASTRecordWriter::AddTypeSourceInfo(TypeSourceInfo *TInfo) {
@@ -7070,49 +7078,50 @@ void ASTRecordWriter::AddQualifierInfo(const QualifierInfo &Info) {
     AddTemplateParameterList(Info.TemplParamLists[i]);
 }
 
-void ASTRecordWriter::AddNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS) {
+void ASTRecordWriter::AddNestedNameSpecifierLoc(
+    NestedNameSpecifierLoc QualifierLoc) {
   // Nested name specifiers usually aren't too long. I think that 8 would
   // typically accommodate the vast majority.
   SmallVector<NestedNameSpecifierLoc , 8> NestedNames;
 
   // Push each of the nested-name-specifiers's onto a stack for
   // serialization in reverse order.
-  while (NNS) {
-    NestedNames.push_back(NNS);
-    NNS = NNS.getPrefix();
+  while (QualifierLoc) {
+    NestedNames.push_back(QualifierLoc);
+    QualifierLoc = QualifierLoc.getAsNamespaceAndPrefix().Prefix;
   }
 
   Record->push_back(NestedNames.size());
   while(!NestedNames.empty()) {
-    NNS = NestedNames.pop_back_val();
-    NestedNameSpecifier::SpecifierKind Kind
-      = NNS.getNestedNameSpecifier()->getKind();
-    Record->push_back(Kind);
+    QualifierLoc = NestedNames.pop_back_val();
+    NestedNameSpecifier Qualifier = QualifierLoc.getNestedNameSpecifier();
+    NestedNameSpecifier::Kind Kind = Qualifier.getKind();
+    Record->push_back(llvm::to_underlying(Kind));
     switch (Kind) {
-    case NestedNameSpecifier::Identifier:
-      AddIdentifierRef(NNS.getNestedNameSpecifier()->getAsIdentifier());
-      AddSourceRange(NNS.getLocalSourceRange());
+    case NestedNameSpecifier::Kind::Namespace:
+      AddDeclRef(Qualifier.getAsNamespaceAndPrefix().Namespace);
+      AddSourceRange(QualifierLoc.getLocalSourceRange());
       break;
 
-    case NestedNameSpecifier::Namespace:
-      AddDeclRef(NNS.getNestedNameSpecifier()->getAsNamespace());
-      AddSourceRange(NNS.getLocalSourceRange());
+    case NestedNameSpecifier::Kind::Type: {
+      TypeLoc TL = QualifierLoc.castAsTypeLoc();
+      AddTypeRef(TL.getType());
+      AddTypeLoc(TL);
+      AddSourceLocation(QualifierLoc.getLocalSourceRange().getEnd());
       break;
+    }
 
-    case NestedNameSpecifier::TypeSpec:
-      AddTypeRef(NNS.getTypeLoc().getType());
-      AddTypeLoc(NNS.getTypeLoc());
-      AddSourceLocation(NNS.getLocalSourceRange().getEnd());
+    case NestedNameSpecifier::Kind::Global:
+      AddSourceLocation(QualifierLoc.getLocalSourceRange().getEnd());
       break;
 
-    case NestedNameSpecifier::Global:
-      AddSourceLocation(NNS.getLocalSourceRange().getEnd());
+    case NestedNameSpecifier::Kind::MicrosoftSuper:
+      AddDeclRef(Qualifier.getAsMicrosoftSuper());
+      AddSourceRange(QualifierLoc.getLocalSourceRange());
       break;
 
-    case NestedNameSpecifier::Super:
-      AddDeclRef(NNS.getNestedNameSpecifier()->getAsRecordDecl());
-      AddSourceRange(NNS.getLocalSourceRange());
-      break;
+    case NestedNameSpecifier::Kind::Null:
+      llvm_unreachable("unexpected null nested name specifier");
     }
   }
 }
@@ -8762,8 +8771,10 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeSourceLocation(FPC->getLParenLoc());
     writeOpenACCVarList(FPC);
 
-    for (VarDecl *VD : FPC->getInitRecipes())
-      AddDeclRef(VD);
+    for (const OpenACCFirstPrivateRecipe &R : FPC->getInitRecipes()) {
+      AddDeclRef(R.RecipeDecl);
+      AddDeclRef(R.InitFromTemporary);
+    }
     return;
   }
   case OpenACCClauseKind::Attach: {
@@ -8881,6 +8892,11 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeSourceLocation(RC->getLParenLoc());
     writeEnum(RC->getReductionOp());
     writeOpenACCVarList(RC);
+
+    for (const OpenACCReductionRecipe &R : RC->getRecipes()) {
+      static_assert(sizeof(OpenACCReductionRecipe) == sizeof(int *));
+      AddDeclRef(R.RecipeDecl);
+    }
     return;
   }
   case OpenACCClauseKind::Seq:
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 6fd25b9e0a9ec..ec3dda1fcdf48 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -524,7 +524,7 @@ void ASTDeclWriter::VisitDecl(Decl *D) {
   // bits actually. However, if we changed the order to be 0x0f, then we can
   // store it as 0b001111, which takes 6 bits only now.
   DeclBits.addBits((uint64_t)D->getModuleOwnershipKind(), /*BitWidth=*/3);
-  DeclBits.addBit(D->isReferenced());
+  DeclBits.addBit(D->isThisDeclarationReferenced());
   DeclBits.addBit(D->isUsed(false));
   DeclBits.addBits(D->getAccess(), /*BitWidth=*/2);
   DeclBits.addBit(D->isImplicit());
@@ -601,7 +601,8 @@ void ASTDeclWriter::VisitNamedDecl(NamedDecl *D) {
 void ASTDeclWriter::VisitTypeDecl(TypeDecl *D) {
   VisitNamedDecl(D);
   Record.AddSourceLocation(D->getBeginLoc());
-  Record.AddTypeRef(QualType(D->getTypeForDecl(), 0));
+  if (!isa<TagDecl, TypedefDecl, TypeAliasDecl>(D))
+    Record.AddTypeRef(QualType(D->getTypeForDecl(), 0));
 }
 
 void ASTDeclWriter::VisitTypedefNameDecl(TypedefNameDecl *D) {
@@ -2561,7 +2562,6 @@ void ASTWriter::WriteDeclAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // AnonDeclNumber
   // TypeDecl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location
-  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type Ref
   // TagDecl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // IdentifierNamespace
   Abv->Add(BitCodeAbbrevOp(
@@ -2607,7 +2607,6 @@ void ASTWriter::WriteDeclAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // AnonDeclNumber
   // TypeDecl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Source Location
-  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type Ref
   // TagDecl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));   // IdentifierNamespace
   Abv->Add(BitCodeAbbrevOp(
diff --git a/clang/lib/Serialization/TemplateArgumentHasher.cpp b/clang/lib/Serialization/TemplateArgumentHasher.cpp
index c56138e8893c1..3e8ffea78c2f1 100644
--- a/clang/lib/Serialization/TemplateArgumentHasher.cpp
+++ b/clang/lib/Serialization/TemplateArgumentHasher.cpp
@@ -320,7 +320,7 @@ class TypeVisitorHelper : public TypeVisitor<TypeVisitorHelper> {
 
   void VisitMemberPointerType(const MemberPointerType *T) {
     AddQualType(T->getPointeeType());
-    AddType(T->getQualifier()->getAsType());
+    AddType(T->getQualifier().getAsType());
     if (auto *RD = T->getMostRecentCXXRecordDecl())
       AddDecl(RD->getCanonicalDecl());
   }
@@ -358,7 +358,7 @@ class TypeVisitorHelper : public TypeVisitor<TypeVisitorHelper> {
     AddQualType(T->getReplacementType());
   }
 
-  void VisitTagType(const TagType *T) { AddDecl(T->getDecl()); }
+  void VisitTagType(const TagType *T) { AddDecl(T->getOriginalDecl()); }
 
   void VisitRecordType(const RecordType *T) { VisitTagType(T); }
   void VisitEnumType(const EnumType *T) { VisitTagType(T); }
@@ -379,10 +379,6 @@ class TypeVisitorHelper : public TypeVisitor<TypeVisitorHelper> {
 
   void VisitTypedefType(const TypedefType *T) { AddDecl(T->getDecl()); }
 
-  void VisitElaboratedType(const ElaboratedType *T) {
-    AddQualType(T->getNamedType());
-  }
-
   void VisitUnaryTransformType(const UnaryTransformType *T) {
     AddQualType(T->getUnderlyingType());
     AddQualType(T->getBaseType());
diff --git a/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp
index 3b3def7ce3324..e64153d53bbd6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/AnalysisOrderChecker.cpp
@@ -183,7 +183,8 @@ class AnalysisOrderChecker
       llvm::errs() << "NewAllocator\n";
   }
 
-  void checkBind(SVal Loc, SVal Val, const Stmt *S, CheckerContext &C) const {
+  void checkBind(SVal Loc, SVal Val, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const {
     if (isCallbackEnabled(C, "Bind"))
       llvm::errs() << "Bind\n";
   }
diff --git a/clang/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp
index 837cbbce8f45f..921114a7a215f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp
@@ -29,7 +29,8 @@ class BoolAssignmentChecker : public Checker<check::Bind> {
                   bool IsTainted = false) const;
 
 public:
-  void checkBind(SVal Loc, SVal Val, const Stmt *S, CheckerContext &C) const;
+  void checkBind(SVal Loc, SVal Val, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const;
 };
 } // end anonymous namespace
 
@@ -55,6 +56,7 @@ static bool isBooleanType(QualType Ty) {
 }
 
 void BoolAssignmentChecker::checkBind(SVal Loc, SVal Val, const Stmt *S,
+                                      bool AtDeclInit,
                                       CheckerContext &C) const {
 
   // We are only interested in stores into Booleans.
diff --git a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
index fd0a3982bf461..cfc6d34a75ca2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
@@ -97,10 +97,6 @@ class CStringChecker
   CheckerFrontendWithBugType UninitializedRead{
       "Accessing unitialized/garbage values"};
 
-  // FIXME: This bug type should be removed because it is only emitted in a
-  // situation that is practically impossible.
-  const BugType AdditionOverflow{&OutOfBounds, "API"};
-
   StringRef getDebugTag() const override { return "MallocChecker"; }
 
   static void *getTag() { static int tag; return &tag; }
@@ -330,7 +326,6 @@ class CStringChecker
                           const Stmt *S, StringRef WarningMsg) const;
   void emitNotCStringBug(CheckerContext &C, ProgramStateRef State,
                          const Stmt *S, StringRef WarningMsg) const;
-  void emitAdditionOverflowBug(CheckerContext &C, ProgramStateRef State) const;
   void emitUninitializedReadBug(CheckerContext &C, ProgramStateRef State,
                                 const Expr *E, const MemRegion *R,
                                 StringRef Msg) const;
@@ -843,22 +838,6 @@ void CStringChecker::emitNotCStringBug(CheckerContext &C, ProgramStateRef State,
   }
 }
 
-void CStringChecker::emitAdditionOverflowBug(CheckerContext &C,
-                                             ProgramStateRef State) const {
-  if (ExplodedNode *N = C.generateErrorNode(State)) {
-    // This isn't a great error message, but this should never occur in real
-    // code anyway -- you'd have to create a buffer longer than a size_t can
-    // represent, which is sort of a contradiction.
-    const char *WarningMsg =
-        "This expression will create a string whose length is too big to "
-        "be represented as a size_t";
-
-    auto Report = std::make_unique<PathSensitiveBugReport>(AdditionOverflow,
-                                                           WarningMsg, N);
-    C.emitReport(std::move(Report));
-  }
-}
-
 ProgramStateRef CStringChecker::checkAdditionOverflow(CheckerContext &C,
                                                      ProgramStateRef state,
                                                      NonLoc left,
@@ -896,19 +875,22 @@ ProgramStateRef CStringChecker::checkAdditionOverflow(CheckerContext &C,
     SVal willOverflow = svalBuilder.evalBinOpNN(state, BO_GT, left,
                                                 *maxMinusRightNL, cmpTy);
 
-    ProgramStateRef stateOverflow, stateOkay;
-    std::tie(stateOverflow, stateOkay) =
-      state->assume(willOverflow.castAs<DefinedOrUnknownSVal>());
+    auto [StateOverflow, StateOkay] =
+        state->assume(willOverflow.castAs<DefinedOrUnknownSVal>());
 
-    if (stateOverflow && !stateOkay) {
-      // We have an overflow. Emit a bug report.
-      emitAdditionOverflowBug(C, stateOverflow);
+    if (StateOverflow && !StateOkay) {
+      // On this path the analyzer is convinced that the addition of these two
+      // values would overflow `size_t` which must be caused by the inaccuracy
+      // of our modeling because this method is called in situations where the
+      // summands are size/length values which are much less than SIZE_MAX. To
+      // avoid false positives let's just sink this invalid path.
+      C.addSink(StateOverflow);
       return nullptr;
     }
 
     // From now on, assume an overflow didn't occur.
-    assert(stateOkay);
-    state = stateOkay;
+    assert(StateOkay);
+    state = StateOkay;
   }
 
   return state;
@@ -1147,9 +1129,9 @@ bool CStringChecker::isFirstBufInBound(CheckerContext &C, ProgramStateRef State,
   if (!ER)
     return true; // cf top comment.
 
-  // FIXME: Does this crash when a non-standard definition
-  // of a library function is encountered?
-  assert(ER->getValueType() == C.getASTContext().CharTy &&
+  // Support library functions defined with non-default address spaces
+  assert(ER->getValueType()->getCanonicalTypeUnqualified() ==
+             C.getASTContext().CharTy &&
          "isFirstBufInBound should only be called with char* ElementRegions");
 
   // Get the size of the array.
diff --git a/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
index 23935647a5826..731e506a5d259 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
@@ -250,7 +250,7 @@ class FindUninitializedField {
   bool Find(const TypedValueRegion *R) {
     QualType T = R->getValueType();
     if (const RecordType *RT = T->getAsStructureType()) {
-      const RecordDecl *RD = RT->getDecl()->getDefinition();
+      const RecordDecl *RD = RT->getOriginalDecl()->getDefinition();
       assert(RD && "Referred record has no definition");
       for (const auto *I : RD->fields()) {
         if (I->isUnnamedBitField())
diff --git a/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
index 0b52c9bd8ac2a..90c6537d71d9d 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
@@ -53,7 +53,7 @@ static bool evenFlexibleArraySize(ASTContext &Ctx, CharUnits RegionSize,
   if (!RT)
     return false;
 
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
   RecordDecl::field_iterator Iter(RD->field_begin());
   RecordDecl::field_iterator End(RD->field_end());
   const FieldDecl *Last = nullptr;
diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp
index 350db4b1bc2bc..392c7eeea234a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp
@@ -175,9 +175,12 @@ class CheckerDocumentation
   /// \param Loc The value of the location (pointer).
   /// \param Val The value which will be stored at the location Loc.
   /// \param S   The bind is performed while processing the statement S.
+  /// \param AtDeclInit Whether the bind is performed during declaration
+  ///                   initialization.
   ///
   /// check::Bind
-  void checkBind(SVal Loc, SVal Val, const Stmt *S, CheckerContext &) const {}
+  void checkBind(SVal Loc, SVal Val, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &) const {}
 
   /// Called after a CFG edge is taken within a function.
   ///
diff --git a/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
index 152129ea91fd4..395d724cdfd11 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
@@ -48,7 +48,8 @@ class DereferenceChecker
 public:
   void checkLocation(SVal location, bool isLoad, const Stmt* S,
                      CheckerContext &C) const;
-  void checkBind(SVal L, SVal V, const Stmt *S, CheckerContext &C) const;
+  void checkBind(SVal L, SVal V, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const;
 
   static void AddDerefSource(raw_ostream &os,
                              SmallVectorImpl<SourceRange> &Ranges,
@@ -309,7 +310,7 @@ void DereferenceChecker::checkLocation(SVal l, bool isLoad, const Stmt* S,
 }
 
 void DereferenceChecker::checkBind(SVal L, SVal V, const Stmt *S,
-                                   CheckerContext &C) const {
+                                   bool AtDeclInit, CheckerContext &C) const {
   // If we're binding to a reference, check if the value is known to be null.
   if (V.isUndef())
     return;
diff --git a/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp b/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
index 4982cd59b0a4a..cee744aecb686 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
@@ -245,7 +245,7 @@ static void recordFixedType(const MemRegion *Region, const CXXMethodDecl *MD,
   assert(MD);
 
   ASTContext &Ctx = C.getASTContext();
-  QualType Ty = Ctx.getPointerType(Ctx.getRecordType(MD->getParent()));
+  CanQualType Ty = Ctx.getPointerType(Ctx.getCanonicalTagType(MD->getParent()));
 
   ProgramStateRef State = C.getState();
   State = setDynamicTypeInfo(State, Region, Ty, /*CanBeSubClassed=*/false);
diff --git a/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp
index 355e82e465e82..054b2e96bd13b 100644
--- a/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp
@@ -148,7 +148,8 @@ void EnumCastOutOfRangeChecker::checkPreStmt(const CastExpr *CE,
   // If the isEnumeralType() returned true, then the declaration must exist
   // even if it is a stub declaration. It is up to the getDeclValuesForEnum()
   // function to handle this.
-  const EnumDecl *ED = T->castAs<EnumType>()->getDecl();
+  const EnumDecl *ED =
+      T->castAs<EnumType>()->getOriginalDecl()->getDefinitionOrSelf();
 
   // [[clang::flag_enum]] annotated enums are by definition should be ignored.
   if (ED->hasAttr<FlagEnumAttr>())
diff --git a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
index 7ad54c08d5c71..7eb9a1da90bb4 100644
--- a/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/IteratorModeling.cpp
@@ -150,7 +150,8 @@ class IteratorModeling
   IteratorModeling() = default;
 
   void checkPostCall(const CallEvent &Call, CheckerContext &C) const;
-  void checkBind(SVal Loc, SVal Val, const Stmt *S, CheckerContext &C) const;
+  void checkBind(SVal Loc, SVal Val, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const;
   void checkPostStmt(const UnaryOperator *UO, CheckerContext &C) const;
   void checkPostStmt(const BinaryOperator *BO, CheckerContext &C) const;
   void checkPostStmt(const MaterializeTemporaryExpr *MTE,
@@ -234,7 +235,7 @@ void IteratorModeling::checkPostCall(const CallEvent &Call,
 }
 
 void IteratorModeling::checkBind(SVal Loc, SVal Val, const Stmt *S,
-                                 CheckerContext &C) const {
+                                 bool AtDeclInit, CheckerContext &C) const {
   auto State = C.getState();
   const auto *Pos = getIteratorPosition(State, Val);
   if (Pos) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
index 1cb3848cfed2a..828b6f91d81c2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/LLVMConventionsChecker.cpp
@@ -47,9 +47,6 @@ static bool InNamespace(const Decl *D, StringRef NS) {
 }
 
 static bool IsStdString(QualType T) {
-  if (const ElaboratedType *QT = T->getAs<ElaboratedType>())
-    T = QT->getNamedType();
-
   const TypedefType *TT = T->getAs<TypedefType>();
   if (!TT)
     return false;
@@ -201,7 +198,7 @@ static bool IsPartOfAST(const CXXRecordDecl *R) {
   for (const auto &BS : R->bases()) {
     QualType T = BS.getType();
     if (const RecordType *baseT = T->getAs<RecordType>()) {
-      CXXRecordDecl *baseD = cast<CXXRecordDecl>(baseT->getDecl());
+      CXXRecordDecl *baseD = cast<CXXRecordDecl>(baseT->getOriginalDecl());
       if (IsPartOfAST(baseD))
         return true;
     }
@@ -247,7 +244,7 @@ void ASTFieldVisitor::Visit(FieldDecl *D) {
     ReportError(T);
 
   if (const RecordType *RT = T->getAs<RecordType>()) {
-    const RecordDecl *RD = RT->getDecl()->getDefinition();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinition();
     for (auto *I : RD->fields())
       Visit(I);
   }
diff --git a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
index 538104d13e697..62d568f2f40e2 100644
--- a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
@@ -71,7 +71,7 @@ class NonLocalizedStringChecker
   // Methods that return a localized string
   mutable llvm::SmallSet<std::pair<const IdentifierInfo *, Selector>, 12> LSM;
   // C Functions that return a localized string
-  mutable llvm::SmallSet<const IdentifierInfo *, 5> LSF;
+  mutable llvm::SmallPtrSet<const IdentifierInfo *, 5> LSF;
 
   void initUIMethods(ASTContext &Ctx) const;
   void initLocStringsMethods(ASTContext &Ctx) const;
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 369d6194dbb65..efb980962e811 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -3156,7 +3156,7 @@ void MallocChecker::checkPreCall(const CallEvent &Call,
   for (unsigned I = 0, E = Call.getNumArgs(); I != E; ++I) {
     SVal ArgSVal = Call.getArgSVal(I);
     if (isa<Loc>(ArgSVal)) {
-      SymbolRef Sym = ArgSVal.getAsSymbol();
+      SymbolRef Sym = ArgSVal.getAsSymbol(/*IncludeBaseRegions=*/true);
       if (!Sym)
         continue;
       if (checkUseAfterFree(Sym, C, Call.getArgExpr(I)))
diff --git a/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp
index 0aea981252259..b1a7cd7620424 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp
@@ -148,7 +148,9 @@ void NonNullParamChecker::checkPreCall(const CallEvent &Call,
 
       QualType T = ArgE->getType();
       const RecordType *UT = T->getAsUnionType();
-      if (!UT || !UT->getDecl()->hasAttr<TransparentUnionAttr>())
+      if (!UT || !UT->getOriginalDecl()
+                      ->getMostRecentDecl()
+                      ->hasAttr<TransparentUnionAttr>())
         continue;
 
       auto CSV = DV->getAs<nonloc::CompoundVal>();
diff --git a/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp
index 7927967093dcb..b5e32495235aa 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NonnullGlobalConstantsChecker.cpp
@@ -114,10 +114,7 @@ bool NonnullGlobalConstantsChecker::isGlobalConstString(SVal V) const {
       if (AT->getAttrKind() == attr::TypeNonNull)
         return true;
       Ty = AT->getModifiedType();
-    } else if (const auto *ET = dyn_cast<ElaboratedType>(T)) {
-      const auto *TT = dyn_cast<TypedefType>(ET->getNamedType());
-      if (!TT)
-        return false;
+    } else if (const auto *TT = dyn_cast<TypedefType>(T)) {
       Ty = TT->getDecl()->getUnderlyingType();
       // It is sufficient for any intermediate typedef
       // to be classified const.
diff --git a/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
index 9744d1abf7790..eeb6b720cce2a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
@@ -97,7 +97,8 @@ class NullabilityChecker
   // libraries.
   bool NoDiagnoseCallsToSystemHeaders = false;
 
-  void checkBind(SVal L, SVal V, const Stmt *S, CheckerContext &C) const;
+  void checkBind(SVal L, SVal V, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const;
   void checkPostStmt(const ExplicitCastExpr *CE, CheckerContext &C) const;
   void checkPreStmt(const ReturnStmt *S, CheckerContext &C) const;
   void checkPostObjCMessage(const ObjCMethodCall &M, CheckerContext &C) const;
@@ -1250,7 +1251,7 @@ static bool isARCNilInitializedLocal(CheckerContext &C, const Stmt *S) {
 /// Propagate the nullability information through binds and warn when nullable
 /// pointer or null symbol is assigned to a pointer with a nonnull type.
 void NullabilityChecker::checkBind(SVal L, SVal V, const Stmt *S,
-                                   CheckerContext &C) const {
+                                   bool AtDeclInit, CheckerContext &C) const {
   const TypedValueRegion *TVR =
       dyn_cast_or_null<TypedValueRegion>(L.getAsRegion());
   if (!TVR)
diff --git a/clang/lib/StaticAnalyzer/Checkers/NumberObjectConversionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NumberObjectConversionChecker.cpp
index f217520d8f4a0..68ab22a44e7b6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NumberObjectConversionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NumberObjectConversionChecker.cpp
@@ -197,9 +197,9 @@ void NumberObjectConversionChecker::checkASTCodeBody(const Decl *D,
                                                      BugReporter &BR) const {
   // Currently this matches CoreFoundation opaque pointer typedefs.
   auto CSuspiciousNumberObjectExprM = expr(ignoringParenImpCasts(
-      expr(hasType(elaboratedType(namesType(typedefType(
+      expr(hasType(typedefType(
                hasDeclaration(anyOf(typedefDecl(hasName("CFNumberRef")),
-                                    typedefDecl(hasName("CFBooleanRef")))))))))
+                                    typedefDecl(hasName("CFBooleanRef")))))))
           .bind("c_object")));
 
   // Currently this matches XNU kernel number-object pointers.
@@ -238,8 +238,7 @@ void NumberObjectConversionChecker::checkASTCodeBody(const Decl *D,
 
   // The .bind here is in order to compose the error message more accurately.
   auto ObjCSuspiciousScalarBooleanTypeM =
-      qualType(elaboratedType(namesType(
-                   typedefType(hasDeclaration(typedefDecl(hasName("BOOL")))))))
+      qualType(typedefType(hasDeclaration(typedefDecl(hasName("BOOL")))))
           .bind("objc_bool_type");
 
   // The .bind here is in order to compose the error message more accurately.
@@ -252,8 +251,8 @@ void NumberObjectConversionChecker::checkASTCodeBody(const Decl *D,
   // for storing pointers.
   auto SuspiciousScalarNumberTypeM =
       qualType(hasCanonicalType(isInteger()),
-               unless(elaboratedType(namesType(typedefType(hasDeclaration(
-                   typedefDecl(matchesName("^::u?intptr_t$"))))))))
+               unless(typedefType(
+                   hasDeclaration(typedefDecl(matchesName("^::u?intptr_t$"))))))
           .bind("int_type");
 
   auto SuspiciousScalarTypeM =
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
index ace3426387568..e40b4f84d55ec 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
@@ -73,7 +73,8 @@ class ObjCSelfInitChecker : public Checker<  check::PostObjCMessage,
   void checkPreStmt(const ReturnStmt *S, CheckerContext &C) const;
   void checkLocation(SVal location, bool isLoad, const Stmt *S,
                      CheckerContext &C) const;
-  void checkBind(SVal loc, SVal val, const Stmt *S, CheckerContext &C) const;
+  void checkBind(SVal loc, SVal val, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const;
 
   void checkPreCall(const CallEvent &CE, CheckerContext &C) const;
   void checkPostCall(const CallEvent &CE, CheckerContext &C) const;
@@ -311,9 +312,8 @@ void ObjCSelfInitChecker::checkLocation(SVal location, bool isLoad,
                 C);
 }
 
-
 void ObjCSelfInitChecker::checkBind(SVal loc, SVal val, const Stmt *S,
-                                    CheckerContext &C) const {
+                                    bool AtDeclInit, CheckerContext &C) const {
   // Allow assignment of anything to self. Self is a local variable in the
   // initializer, so it is legal to assign anything to it, like results of
   // static functions/method calls. After self is assigned something we cannot
diff --git a/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp
index d4efbdd54fc22..7ef659518ab1b 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PaddingChecker.cpp
@@ -104,7 +104,7 @@ class PaddingChecker : public Checker<check::ASTDecl<TranslationUnitDecl>> {
       // There is not enough excess padding to trigger a warning.
       return;
     }
-    reportRecord(RD, BaselinePad, OptimalPad, OptimalFieldsOrder);
+    reportRecord(ASTContext, RD, BaselinePad, OptimalPad, OptimalFieldsOrder);
   }
 
   /// Look for arrays of overly padded types. If the padding of the
@@ -123,7 +123,7 @@ class PaddingChecker : public Checker<check::ASTDecl<TranslationUnitDecl>> {
       return;
 
     // TODO: Recurse into the fields to see if they have excess padding.
-    visitRecord(RT->getDecl(), Elts);
+    visitRecord(RT->getOriginalDecl()->getDefinitionOrSelf(), Elts);
   }
 
   bool shouldSkipDecl(const RecordDecl *RD) const {
@@ -159,9 +159,7 @@ class PaddingChecker : public Checker<check::ASTDecl<TranslationUnitDecl>> {
         return true;
       // Can't layout a template, so skip it. We do still layout the
       // instantiations though.
-      if (CXXRD->getTypeForDecl()->isDependentType())
-        return true;
-      if (CXXRD->getTypeForDecl()->isInstantiationDependentType())
+      if (CXXRD->isDependentType())
         return true;
     }
     // How do you reorder fields if you haven't got any?
@@ -306,14 +304,14 @@ class PaddingChecker : public Checker<check::ASTDecl<TranslationUnitDecl>> {
   }
 
   void reportRecord(
-      const RecordDecl *RD, CharUnits BaselinePad, CharUnits OptimalPad,
+      const ASTContext &Ctx, const RecordDecl *RD, CharUnits BaselinePad,
+      CharUnits OptimalPad,
       const SmallVector<const FieldDecl *, 20> &OptimalFieldsOrder) const {
     SmallString<100> Buf;
     llvm::raw_svector_ostream Os(Buf);
     Os << "Excessive padding in '";
-    Os << QualType::getAsString(RD->getTypeForDecl(), Qualifiers(),
-                                LangOptions())
-       << "'";
+    QualType(Ctx.getCanonicalTagType(RD)).print(Os, LangOptions());
+    Os << "'";
 
     if (auto *TSD = dyn_cast<ClassTemplateSpecializationDecl>(RD)) {
       // TODO: make this show up better in the console output and in
diff --git a/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
index 1141f07428b4c..30e01e73eb18f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
@@ -58,7 +58,7 @@ class PointerArithChecker
 
   const BugType BT_pointerArith{this, "Dangerous pointer arithmetic"};
   const BugType BT_polyArray{this, "Dangerous pointer arithmetic"};
-  mutable llvm::SmallSet<IdentifierInfo *, 8> AllocFunctions;
+  mutable llvm::SmallPtrSet<IdentifierInfo *, 8> AllocFunctions;
 
 public:
   void checkPreStmt(const UnaryOperator *UOp, CheckerContext &C) const;
diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
index 62bc3218d9ced..1762505f58f0f 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.cpp
@@ -840,20 +840,27 @@ ProgramStateRef RetainCountChecker::updateSymbol(ProgramStateRef state,
 const RefCountBug &
 RetainCountChecker::errorKindToBugKind(RefVal::Kind ErrorKind,
                                        SymbolRef Sym) const {
+  const RefCountFrontend &FE = getPreferredFrontend();
+
   switch (ErrorKind) {
     case RefVal::ErrorUseAfterRelease:
-      return *UseAfterRelease;
+      return FE.UseAfterRelease;
     case RefVal::ErrorReleaseNotOwned:
-      return *ReleaseNotOwned;
+      return FE.ReleaseNotOwned;
     case RefVal::ErrorDeallocNotOwned:
       if (Sym->getType()->getPointeeCXXRecordDecl())
-        return *FreeNotOwned;
-      return *DeallocNotOwned;
+        return FE.FreeNotOwned;
+      return FE.DeallocNotOwned;
     default:
       llvm_unreachable("Unhandled error.");
   }
 }
 
+bool RetainCountChecker::isReleaseUnownedError(RefVal::Kind ErrorKind) const {
+  return ErrorKind == RefVal::ErrorReleaseNotOwned ||
+         ErrorKind == RefVal::ErrorDeallocNotOwned;
+}
+
 void RetainCountChecker::processNonLeakError(ProgramStateRef St,
                                              SourceRange ErrorRange,
                                              RefVal::Kind ErrorKind,
@@ -874,8 +881,8 @@ void RetainCountChecker::processNonLeakError(ProgramStateRef St,
     return;
 
   auto report = std::make_unique<RefCountReport>(
-      errorKindToBugKind(ErrorKind, Sym),
-      C.getASTContext().getLangOpts(), N, Sym);
+      errorKindToBugKind(ErrorKind, Sym), C.getASTContext().getLangOpts(), N,
+      Sym, /*isLeak=*/false, isReleaseUnownedError(ErrorKind));
   report->addRange(ErrorRange);
   C.emitReport(std::move(report));
 }
@@ -1090,8 +1097,8 @@ ExplodedNode * RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S,
         ExplodedNode *N = C.addTransition(state, Pred);
         if (N) {
           const LangOptions &LOpts = C.getASTContext().getLangOpts();
-          auto R =
-              std::make_unique<RefLeakReport>(*LeakAtReturn, LOpts, N, Sym, C);
+          auto R = std::make_unique<RefLeakReport>(
+              getPreferredFrontend().LeakAtReturn, LOpts, N, Sym, C);
           C.emitReport(std::move(R));
         }
         return N;
@@ -1113,7 +1120,8 @@ ExplodedNode * RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S,
         ExplodedNode *N = C.addTransition(state, Pred);
         if (N) {
           auto R = std::make_unique<RefCountReport>(
-              *ReturnNotOwnedForOwned, C.getASTContext().getLangOpts(), N, Sym);
+              getPreferredFrontend().ReturnNotOwnedForOwned,
+              C.getASTContext().getLangOpts(), N, Sym);
           C.emitReport(std::move(R));
         }
         return N;
@@ -1128,7 +1136,7 @@ ExplodedNode * RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S,
 //===----------------------------------------------------------------------===//
 
 void RetainCountChecker::checkBind(SVal loc, SVal val, const Stmt *S,
-                                   CheckerContext &C) const {
+                                   bool AtDeclInit, CheckerContext &C) const {
   ProgramStateRef state = C.getState();
   const MemRegion *MR = loc.getAsRegion();
 
@@ -1261,8 +1269,8 @@ ProgramStateRef RetainCountChecker::handleAutoreleaseCounts(
     os << "has a +" << V.getCount() << " retain count";
 
     const LangOptions &LOpts = Ctx.getASTContext().getLangOpts();
-    auto R = std::make_unique<RefCountReport>(*OverAutorelease, LOpts, N, Sym,
-                                              os.str());
+    auto R = std::make_unique<RefCountReport>(
+        getPreferredFrontend().OverAutorelease, LOpts, N, Sym, os.str());
     Ctx.emitReport(std::move(R));
   }
 
@@ -1307,8 +1315,10 @@ RetainCountChecker::processLeaks(ProgramStateRef state,
   const LangOptions &LOpts = Ctx.getASTContext().getLangOpts();
 
   if (N) {
+    const RefCountFrontend &FE = getPreferredFrontend();
+    const RefCountBug &BT = Pred ? FE.LeakWithinFunction : FE.LeakAtReturn;
+
     for (SymbolRef L : Leaked) {
-      const RefCountBug &BT = Pred ? *LeakWithinFunction : *LeakAtReturn;
       Ctx.emitReport(std::make_unique<RefLeakReport>(BT, LOpts, N, L, Ctx));
     }
   }
@@ -1463,44 +1473,31 @@ std::unique_ptr<SimpleProgramPointTag> RetainCountChecker::DeallocSentTag;
 std::unique_ptr<SimpleProgramPointTag> RetainCountChecker::CastFailTag;
 
 void ento::registerRetainCountBase(CheckerManager &Mgr) {
-  auto *Chk = Mgr.registerChecker<RetainCountChecker>();
+  auto *Chk = Mgr.getChecker<RetainCountChecker>();
   Chk->DeallocSentTag = std::make_unique<SimpleProgramPointTag>(
       "RetainCountChecker", "DeallocSent");
   Chk->CastFailTag = std::make_unique<SimpleProgramPointTag>(
       "RetainCountChecker", "DynamicCastFail");
 }
 
-bool ento::shouldRegisterRetainCountBase(const CheckerManager &mgr) {
+bool ento::shouldRegisterRetainCountBase(const CheckerManager &) {
   return true;
 }
+
 void ento::registerRetainCountChecker(CheckerManager &Mgr) {
   auto *Chk = Mgr.getChecker<RetainCountChecker>();
-  Chk->TrackObjCAndCFObjects = true;
+  Chk->RetainCount.enable(Mgr);
   Chk->TrackNSCFStartParam = Mgr.getAnalyzerOptions().getCheckerBooleanOption(
       Mgr.getCurrentCheckerName(), "TrackNSCFStartParam");
-
-#define INIT_BUGTYPE(KIND)                                                     \
-  Chk->KIND = std::make_unique<RefCountBug>(Mgr.getCurrentCheckerName(),       \
-                                            RefCountBug::KIND);
-  // TODO: Ideally, we should have a checker for each of these bug types.
-  INIT_BUGTYPE(UseAfterRelease)
-  INIT_BUGTYPE(ReleaseNotOwned)
-  INIT_BUGTYPE(DeallocNotOwned)
-  INIT_BUGTYPE(FreeNotOwned)
-  INIT_BUGTYPE(OverAutorelease)
-  INIT_BUGTYPE(ReturnNotOwnedForOwned)
-  INIT_BUGTYPE(LeakWithinFunction)
-  INIT_BUGTYPE(LeakAtReturn)
-#undef INIT_BUGTYPE
 }
 
-bool ento::shouldRegisterRetainCountChecker(const CheckerManager &mgr) {
+bool ento::shouldRegisterRetainCountChecker(const CheckerManager &) {
   return true;
 }
 
 void ento::registerOSObjectRetainCountChecker(CheckerManager &Mgr) {
   auto *Chk = Mgr.getChecker<RetainCountChecker>();
-  Chk->TrackOSObjects = true;
+  Chk->OSObjectRetainCount.enable(Mgr);
 
   // FIXME: We want bug reports to always have the same checker name associated
   // with them, yet here, if RetainCountChecker is disabled but
@@ -1511,21 +1508,8 @@ void ento::registerOSObjectRetainCountChecker(CheckerManager &Mgr) {
   // diagnostics, and **hidden checker options** with the fine-tuning of
   // modeling. Following this logic, OSObjectRetainCountChecker should be the
   // latter, but we can't just remove it for backward compatibility reasons.
-#define LAZY_INIT_BUGTYPE(KIND)                                                \
-  if (!Chk->KIND)                                                              \
-    Chk->KIND = std::make_unique<RefCountBug>(Mgr.getCurrentCheckerName(),     \
-                                              RefCountBug::KIND);
-  LAZY_INIT_BUGTYPE(UseAfterRelease)
-  LAZY_INIT_BUGTYPE(ReleaseNotOwned)
-  LAZY_INIT_BUGTYPE(DeallocNotOwned)
-  LAZY_INIT_BUGTYPE(FreeNotOwned)
-  LAZY_INIT_BUGTYPE(OverAutorelease)
-  LAZY_INIT_BUGTYPE(ReturnNotOwnedForOwned)
-  LAZY_INIT_BUGTYPE(LeakWithinFunction)
-  LAZY_INIT_BUGTYPE(LeakAtReturn)
-#undef LAZY_INIT_BUGTYPE
 }
 
-bool ento::shouldRegisterOSObjectRetainCountChecker(const CheckerManager &mgr) {
+bool ento::shouldRegisterOSObjectRetainCountChecker(const CheckerManager &) {
   return true;
 }
diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h
index 0e811436605ff..dc8bad60944b4 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h
+++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountChecker.h
@@ -235,51 +235,32 @@ class RefVal {
 };
 
 class RetainCountChecker
-  : public Checker< check::Bind,
-                    check::DeadSymbols,
-                    check::BeginFunction,
-                    check::EndFunction,
-                    check::PostStmt<BlockExpr>,
-                    check::PostStmt<CastExpr>,
-                    check::PostStmt<ObjCArrayLiteral>,
-                    check::PostStmt<ObjCDictionaryLiteral>,
-                    check::PostStmt<ObjCBoxedExpr>,
-                    check::PostStmt<ObjCIvarRefExpr>,
-                    check::PostCall,
-                    check::RegionChanges,
-                    eval::Assume,
-                    eval::Call > {
+    : public CheckerFamily<
+          check::Bind, check::DeadSymbols, check::BeginFunction,
+          check::EndFunction, check::PostStmt<BlockExpr>,
+          check::PostStmt<CastExpr>, check::PostStmt<ObjCArrayLiteral>,
+          check::PostStmt<ObjCDictionaryLiteral>,
+          check::PostStmt<ObjCBoxedExpr>, check::PostStmt<ObjCIvarRefExpr>,
+          check::PostCall, check::RegionChanges, eval::Assume, eval::Call> {
 
 public:
-  std::unique_ptr<RefCountBug> UseAfterRelease;
-  std::unique_ptr<RefCountBug> ReleaseNotOwned;
-  std::unique_ptr<RefCountBug> DeallocNotOwned;
-  std::unique_ptr<RefCountBug> FreeNotOwned;
-  std::unique_ptr<RefCountBug> OverAutorelease;
-  std::unique_ptr<RefCountBug> ReturnNotOwnedForOwned;
-  std::unique_ptr<RefCountBug> LeakWithinFunction;
-  std::unique_ptr<RefCountBug> LeakAtReturn;
+  RefCountFrontend RetainCount;
+  RefCountFrontend OSObjectRetainCount;
 
   mutable std::unique_ptr<RetainSummaryManager> Summaries;
 
   static std::unique_ptr<SimpleProgramPointTag> DeallocSentTag;
   static std::unique_ptr<SimpleProgramPointTag> CastFailTag;
 
-  /// Track Objective-C and CoreFoundation objects.
-  bool TrackObjCAndCFObjects = false;
-
-  /// Track sublcasses of OSObject.
-  bool TrackOSObjects = false;
-
   /// Track initial parameters (for the entry point) for NS/CF objects.
   bool TrackNSCFStartParam = false;
 
-  RetainCountChecker() {};
+  StringRef getDebugTag() const override { return "RetainCountChecker"; }
 
   RetainSummaryManager &getSummaryManager(ASTContext &Ctx) const {
     if (!Summaries)
-      Summaries.reset(
-          new RetainSummaryManager(Ctx, TrackObjCAndCFObjects, TrackOSObjects));
+      Summaries = std::make_unique<RetainSummaryManager>(
+          Ctx, RetainCount.isEnabled(), OSObjectRetainCount.isEnabled());
     return *Summaries;
   }
 
@@ -287,10 +268,20 @@ class RetainCountChecker
     return getSummaryManager(C.getASTContext());
   }
 
+  const RefCountFrontend &getPreferredFrontend() const {
+    // FIXME: The two frontends of this checker family are in an unusual
+    // relationship: if they are both enabled, then all bug reports are
+    // reported by RetainCount (i.e. `osx.cocoa.RetainCount`), even the bugs
+    // that "belong to" OSObjectRetainCount (i.e. `osx.OSObjectRetainCount`).
+    // This is counter-intuitive and should be fixed to avoid confusion.
+    return RetainCount.isEnabled() ? RetainCount : OSObjectRetainCount;
+  }
+
   void printState(raw_ostream &Out, ProgramStateRef State,
                   const char *NL, const char *Sep) const override;
 
-  void checkBind(SVal loc, SVal val, const Stmt *S, CheckerContext &C) const;
+  void checkBind(SVal loc, SVal val, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const;
   void checkPostStmt(const BlockExpr *BE, CheckerContext &C) const;
   void checkPostStmt(const CastExpr *CE, CheckerContext &C) const;
 
@@ -337,6 +328,8 @@ class RetainCountChecker
   const RefCountBug &errorKindToBugKind(RefVal::Kind ErrorKind,
                                         SymbolRef Sym) const;
 
+  bool isReleaseUnownedError(RefVal::Kind ErrorKind) const;
+
   void processNonLeakError(ProgramStateRef St, SourceRange ErrorRange,
                            RefVal::Kind ErrorKind, SymbolRef Sym,
                            CheckerContext &C) const;
diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
index c9f5dc99aaf6b..cad2c72438d4a 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.cpp
@@ -21,57 +21,6 @@ using namespace clang;
 using namespace ento;
 using namespace retaincountchecker;
 
-StringRef RefCountBug::bugTypeToName(RefCountBug::RefCountBugKind BT) {
-  switch (BT) {
-  case UseAfterRelease:
-    return "Use-after-release";
-  case ReleaseNotOwned:
-    return "Bad release";
-  case DeallocNotOwned:
-    return "-dealloc sent to non-exclusively owned object";
-  case FreeNotOwned:
-    return "freeing non-exclusively owned object";
-  case OverAutorelease:
-    return "Object autoreleased too many times";
-  case ReturnNotOwnedForOwned:
-    return "Method should return an owned object";
-  case LeakWithinFunction:
-    return "Leak";
-  case LeakAtReturn:
-    return "Leak of returned object";
-  }
-  llvm_unreachable("Unknown RefCountBugKind");
-}
-
-StringRef RefCountBug::getDescription() const {
-  switch (BT) {
-  case UseAfterRelease:
-    return "Reference-counted object is used after it is released";
-  case ReleaseNotOwned:
-    return "Incorrect decrement of the reference count of an object that is "
-           "not owned at this point by the caller";
-  case DeallocNotOwned:
-    return "-dealloc sent to object that may be referenced elsewhere";
-  case FreeNotOwned:
-    return  "'free' called on an object that may be referenced elsewhere";
-  case OverAutorelease:
-    return "Object autoreleased too many times";
-  case ReturnNotOwnedForOwned:
-    return "Object with a +0 retain count returned to caller where a +1 "
-           "(owning) retain count is expected";
-  case LeakWithinFunction:
-  case LeakAtReturn:
-    return "";
-  }
-  llvm_unreachable("Unknown RefCountBugKind");
-}
-
-RefCountBug::RefCountBug(CheckerNameRef Checker, RefCountBugKind BT)
-    : BugType(Checker, bugTypeToName(BT), categories::MemoryRefCount,
-              /*SuppressOnSink=*/BT == LeakWithinFunction ||
-                  BT == LeakAtReturn),
-      BT(BT) {}
-
 static bool isNumericLiteralExpression(const Expr *E) {
   // FIXME: This set of cases was copied from SemaExprObjC.
   return isa<IntegerLiteral, CharacterLiteral, FloatingLiteral,
@@ -312,9 +261,11 @@ namespace retaincountchecker {
 class RefCountReportVisitor : public BugReporterVisitor {
 protected:
   SymbolRef Sym;
+  bool IsReleaseUnowned;
 
 public:
-  RefCountReportVisitor(SymbolRef sym) : Sym(sym) {}
+  RefCountReportVisitor(SymbolRef S, bool IRU)
+      : Sym(S), IsReleaseUnowned(IRU) {}
 
   void Profile(llvm::FoldingSetNodeID &ID) const override {
     static int x = 0;
@@ -334,7 +285,8 @@ class RefCountReportVisitor : public BugReporterVisitor {
 class RefLeakReportVisitor : public RefCountReportVisitor {
 public:
   RefLeakReportVisitor(SymbolRef Sym, const MemRegion *LastBinding)
-      : RefCountReportVisitor(Sym), LastBinding(LastBinding) {}
+      : RefCountReportVisitor(Sym, /*IsReleaseUnowned=*/false),
+        LastBinding(LastBinding) {}
 
   PathDiagnosticPieceRef getEndPath(BugReporterContext &BRC,
                                     const ExplodedNode *N,
@@ -452,12 +404,6 @@ annotateStartParameter(const ExplodedNode *N, SymbolRef Sym,
 PathDiagnosticPieceRef
 RefCountReportVisitor::VisitNode(const ExplodedNode *N, BugReporterContext &BRC,
                                  PathSensitiveBugReport &BR) {
-
-  const auto &BT = static_cast<const RefCountBug&>(BR.getBugType());
-
-  bool IsFreeUnowned = BT.getBugType() == RefCountBug::FreeNotOwned ||
-                       BT.getBugType() == RefCountBug::DeallocNotOwned;
-
   const SourceManager &SM = BRC.getSourceManager();
   CallEventManager &CEMgr = BRC.getStateManager().getCallEventManager();
   if (auto CE = N->getLocationAs<CallExitBegin>())
@@ -490,7 +436,7 @@ RefCountReportVisitor::VisitNode(const ExplodedNode *N, BugReporterContext &BRC,
   std::string sbuf;
   llvm::raw_string_ostream os(sbuf);
 
-  if (PrevT && IsFreeUnowned && CurrV.isNotOwned() && PrevT->isOwned()) {
+  if (PrevT && IsReleaseUnowned && CurrV.isNotOwned() && PrevT->isOwned()) {
     os << "Object is now not exclusively owned";
     auto Pos = PathDiagnosticLocation::create(N->getLocation(), SM);
     return std::make_shared<PathDiagnosticEventPiece>(Pos, sbuf);
@@ -815,10 +761,8 @@ RefLeakReportVisitor::getEndPath(BugReporterContext &BRC,
         if (K == ObjKind::ObjC || K == ObjKind::CF) {
           os << "whose name ('" << *FD
              << "') does not contain 'Copy' or 'Create'.  This violates the "
-                "naming"
-                " convention rules given in the Memory Management Guide for "
-                "Core"
-                " Foundation";
+                "naming convention rules given in the Memory Management Guide "
+                "for Core Foundation";
         } else if (RV->getObjKind() == ObjKind::OS) {
           std::string FuncName = FD->getNameAsString();
           os << "whose name ('" << FuncName << "') starts with '"
@@ -836,19 +780,20 @@ RefLeakReportVisitor::getEndPath(BugReporterContext &BRC,
 }
 
 RefCountReport::RefCountReport(const RefCountBug &D, const LangOptions &LOpts,
-                               ExplodedNode *n, SymbolRef sym, bool isLeak)
-    : PathSensitiveBugReport(D, D.getDescription(), n), Sym(sym),
+                               ExplodedNode *n, SymbolRef sym, bool isLeak,
+                               bool IsReleaseUnowned)
+    : PathSensitiveBugReport(D, D.getReportMessage(), n), Sym(sym),
       isLeak(isLeak) {
   if (!isLeak)
-    addVisitor<RefCountReportVisitor>(sym);
+    addVisitor<RefCountReportVisitor>(sym, IsReleaseUnowned);
 }
 
 RefCountReport::RefCountReport(const RefCountBug &D, const LangOptions &LOpts,
                                ExplodedNode *n, SymbolRef sym,
                                StringRef endText)
-    : PathSensitiveBugReport(D, D.getDescription(), endText, n) {
+    : PathSensitiveBugReport(D, D.getReportMessage(), endText, n) {
 
-  addVisitor<RefCountReportVisitor>(sym);
+  addVisitor<RefCountReportVisitor>(sym, /*IsReleaseUnowned=*/false);
 }
 
 void RefLeakReport::deriveParamLocation(CheckerContext &Ctx) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.h b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.h
index d05900895c6a6..6ceb86fec6433 100644
--- a/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.h
+++ b/clang/lib/StaticAnalyzer/Checkers/RetainCountChecker/RetainCountDiagnostics.h
@@ -25,25 +25,44 @@ namespace ento {
 namespace retaincountchecker {
 
 class RefCountBug : public BugType {
+  StringRef ReportMessage;
+
 public:
-  enum RefCountBugKind {
-    UseAfterRelease,
-    ReleaseNotOwned,
-    DeallocNotOwned,
-    FreeNotOwned,
-    OverAutorelease,
-    ReturnNotOwnedForOwned,
-    LeakWithinFunction,
-    LeakAtReturn,
-  };
-  RefCountBug(CheckerNameRef Checker, RefCountBugKind BT);
-  StringRef getDescription() const;
-
-  RefCountBugKind getBugType() const { return BT; }
-
-private:
-  RefCountBugKind BT;
-  static StringRef bugTypeToName(RefCountBugKind BT);
+  RefCountBug(const CheckerFrontend *CF, StringRef Desc, StringRef ReportMsg,
+              bool SuppressOnSink = false)
+      : BugType(CF, Desc, categories::MemoryRefCount, SuppressOnSink),
+        ReportMessage(ReportMsg) {}
+  StringRef getReportMessage() const { return ReportMessage; }
+};
+
+class RefCountFrontend : public CheckerFrontend {
+public:
+  const RefCountBug UseAfterRelease{
+      this, "Use-after-release",
+      "Reference-counted object is used after it is released"};
+  const RefCountBug ReleaseNotOwned{
+      this, "Bad release",
+      "Incorrect decrement of the reference count of an object that is not "
+      "owned at this point by the caller"};
+  const RefCountBug DeallocNotOwned{
+      this, "-dealloc sent to non-exclusively owned object",
+      "-dealloc sent to object that may be referenced elsewhere"};
+  const RefCountBug FreeNotOwned{
+      this, "freeing non-exclusively owned object",
+      "'free' called on an object that may be referenced elsewhere"};
+  const RefCountBug OverAutorelease{this, "Object autoreleased too many times",
+                                    "Object autoreleased too many times"};
+  const RefCountBug ReturnNotOwnedForOwned{
+      this, "Method should return an owned object",
+      "Object with a +0 retain count returned to caller where a +1 (owning) "
+      "retain count is expected"};
+  // For these two bug types the report message will be generated dynamically
+  // by `RefLeakReport::createDescription` so the empty string taken from the
+  // BugType will be ignored (overwritten).
+  const RefCountBug LeakWithinFunction{this, "Leak", /*ReportMsg=*/"",
+                                       /*SuppressOnSink=*/true};
+  const RefCountBug LeakAtReturn{this, "Leak of returned object",
+                                 /*ReportMsg=*/"", /*SuppressOnSink=*/true};
 };
 
 class RefCountReport : public PathSensitiveBugReport {
@@ -53,8 +72,8 @@ class RefCountReport : public PathSensitiveBugReport {
 
 public:
   RefCountReport(const RefCountBug &D, const LangOptions &LOpts,
-              ExplodedNode *n, SymbolRef sym,
-              bool isLeak=false);
+                 ExplodedNode *n, SymbolRef sym, bool isLeak = false,
+                 bool IsReleaseUnowned = false);
 
   RefCountReport(const RefCountBug &D, const LangOptions &LOpts,
               ExplodedNode *n, SymbolRef sym,
diff --git a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
index 52b3d1e95942c..844447f59f7f6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StdLibraryFunctionsChecker.cpp
@@ -1589,7 +1589,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
       // and we have a TypedefDecl with the name 'FILE'.
       for (Decl *D : LookupRes)
         if (auto *TD = dyn_cast<TypedefNameDecl>(D))
-          return ACtx.getTypeDeclType(TD).getCanonicalType();
+          return ACtx.getCanonicalTypeDeclType(TD);
 
       // Find the first TypeDecl.
       // There maybe cases when a function has the same name as a struct.
@@ -1597,7 +1597,7 @@ void StdLibraryFunctionsChecker::initFunctionSummaries(
       //   int stat(const char *restrict path, struct stat *restrict buf);
       for (Decl *D : LookupRes)
         if (auto *TD = dyn_cast<TypeDecl>(D))
-          return ACtx.getTypeDeclType(TD).getCanonicalType();
+          return ACtx.getCanonicalTypeDeclType(TD);
       return std::nullopt;
     }
   } lookupTy(ACtx);
diff --git a/clang/lib/StaticAnalyzer/Checkers/StoreToImmutableChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/StoreToImmutableChecker.cpp
index afad41939cdca..2bb391799d696 100644
--- a/clang/lib/StaticAnalyzer/Checkers/StoreToImmutableChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/StoreToImmutableChecker.cpp
@@ -26,53 +26,11 @@ class StoreToImmutableChecker : public Checker<check::Bind> {
   const BugType BT{this, "Write to immutable memory", "CERT Environment (ENV)"};
 
 public:
-  void checkBind(SVal Loc, SVal Val, const Stmt *S, CheckerContext &C) const;
+  void checkBind(SVal Loc, SVal Val, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const;
 };
 } // end anonymous namespace
 
-static bool isInitializationContext(const Stmt *S, CheckerContext &C) {
-  // Check if this is a DeclStmt (variable declaration)
-  if (isa<DeclStmt>(S))
-    return true;
-
-  // This part is specific for initialization of const lambdas pre-C++17.
-  // Lets look at the AST of the statement:
-  // ```
-  // const auto lambda = [](){};
-  // ```
-  //
-  // The relevant part of the AST for this case prior to C++17 is:
-  // ...
-  // `-DeclStmt
-  //   `-VarDecl
-  //     `-ExprWithCleanups
-  //       `-CXXConstructExpr
-  // ...
-  // In C++17 and later, the AST is different:
-  // ...
-  // `-DeclStmt
-  //   `-VarDecl
-  //     `-ImplicitCastExpr
-  //       `-LambdaExpr
-  //         |-CXXRecordDecl
-  //         `-CXXConstructExpr
-  // ...
-  // And even beside this, the statement `S` that is given to the checkBind
-  // callback is the VarDecl in C++17 and later, and the CXXConstructExpr in
-  // C++14 and before. So in order to support the C++14 we need the following
-  // ugly hack to detect whether this construction is used to initialize a
-  // variable.
-  //
-  // FIXME: This should be eliminated by improving the API of checkBind to
-  // ensure that it consistently passes the `VarDecl` (instead of the
-  // `CXXConstructExpr`) when the constructor call denotes the initialization
-  // of a variable with a lambda, or maybe less preferably, try the more
-  // invasive approach of passing the information forward to the checkers
-  // whether the current bind is an initialization or an assignment.
-  const auto *ConstructExp = dyn_cast<CXXConstructExpr>(S);
-  return ConstructExp && ConstructExp->isElidable();
-}
-
 static bool isEffectivelyConstRegion(const MemRegion *MR, CheckerContext &C) {
   if (isa<GlobalImmutableSpaceRegion>(MR))
     return true;
@@ -128,6 +86,7 @@ getInnermostEnclosingConstDeclRegion(const MemRegion *MR, CheckerContext &C) {
 }
 
 void StoreToImmutableChecker::checkBind(SVal Loc, SVal Val, const Stmt *S,
+                                        bool AtDeclInit,
                                         CheckerContext &C) const {
   // We are only interested in stores to memory regions
   const MemRegion *MR = Loc.getAsRegion();
@@ -136,9 +95,7 @@ void StoreToImmutableChecker::checkBind(SVal Loc, SVal Val, const Stmt *S,
 
   // Skip variable declarations and initializations - we only want to catch
   // actual writes
-  // FIXME: If the API of checkBind would allow to distinguish between
-  // initialization and assignment, we could use that instead.
-  if (isInitializationContext(S, C))
+  if (AtDeclInit)
     return;
 
   // Check if the region is in the global immutable space
diff --git a/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
index e98de333f8dd3..7f8923c7c09fc 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
@@ -26,13 +26,13 @@ class UndefinedAssignmentChecker
   const BugType BT{this, "Assigned value is uninitialized"};
 
 public:
-  void checkBind(SVal location, SVal val, const Stmt *S,
+  void checkBind(SVal location, SVal val, const Stmt *S, bool AtDeclInit,
                  CheckerContext &C) const;
 };
 }
 
 void UndefinedAssignmentChecker::checkBind(SVal location, SVal val,
-                                           const Stmt *StoreE,
+                                           const Stmt *StoreE, bool AtDeclInit,
                                            CheckerContext &C) const {
   if (!val.isUndef())
     return;
diff --git a/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedPointee.cpp b/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedPointee.cpp
index fcb7664f36ceb..d090748850c0c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedPointee.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UninitializedObject/UninitializedPointee.cpp
@@ -217,7 +217,7 @@ bool FindUninitializedFields::isDereferencableUninit(
 static std::optional<DereferenceInfo> dereference(ProgramStateRef State,
                                                   const FieldRegion *FR) {
 
-  llvm::SmallSet<const TypedValueRegion *, 5> VisitedRegions;
+  llvm::SmallPtrSet<const TypedValueRegion *, 5> VisitedRegions;
 
   SVal V = State->getSVal(FR);
   assert(V.getAsRegion() && "V must have an underlying region!");
diff --git a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
index e12261603757f..4df751d203973 100644
--- a/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
@@ -137,8 +137,7 @@ ProgramStateRef UnixAPIMisuseChecker::EnsurePtrNotNull(
       auto R = std::make_unique<PathSensitiveBugReport>(
           BT.value_or(std::cref(BT_ArgumentNull)),
           (PtrDescr + " pointer might be NULL.").str(), N);
-      if (PtrExpr)
-        bugreporter::trackExpressionValue(N, PtrExpr, *R);
+      bugreporter::trackExpressionValue(N, PtrExpr, *R);
       C.emitReport(std::move(R));
     }
     return nullptr;
diff --git a/clang/lib/StaticAnalyzer/Checkers/VforkChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/VforkChecker.cpp
index cb73ac68edd1e..d6224873d7474 100644
--- a/clang/lib/StaticAnalyzer/Checkers/VforkChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/VforkChecker.cpp
@@ -45,7 +45,7 @@ namespace {
 class VforkChecker : public Checker<check::PreCall, check::PostCall,
                                     check::Bind, check::PreStmt<ReturnStmt>> {
   const BugType BT{this, "Dangerous construct in a vforked process"};
-  mutable llvm::SmallSet<const IdentifierInfo *, 10> VforkAllowlist;
+  mutable llvm::SmallPtrSet<const IdentifierInfo *, 10> VforkAllowlist;
   mutable const IdentifierInfo *II_vfork = nullptr;
 
   static bool isChildProcess(const ProgramStateRef State);
@@ -62,7 +62,8 @@ class VforkChecker : public Checker<check::PreCall, check::PostCall,
 
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
   void checkPostCall(const CallEvent &Call, CheckerContext &C) const;
-  void checkBind(SVal L, SVal V, const Stmt *S, CheckerContext &C) const;
+  void checkBind(SVal L, SVal V, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const;
   void checkPreStmt(const ReturnStmt *RS, CheckerContext &C) const;
 };
 
@@ -188,7 +189,7 @@ void VforkChecker::checkPreCall(const CallEvent &Call,
 }
 
 // Prohibit writes in child process (except for vfork's lhs).
-void VforkChecker::checkBind(SVal L, SVal V, const Stmt *S,
+void VforkChecker::checkBind(SVal L, SVal V, const Stmt *S, bool AtDeclInit,
                              CheckerContext &C) const {
   ProgramStateRef State = C.getState();
   if (!isChildProcess(State))
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
index 72199af2f80a5..36c12582a5787 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/PtrTypesSemantics.cpp
@@ -181,10 +181,6 @@ template <typename Predicate>
 static bool isPtrOfType(const clang::QualType T, Predicate Pred) {
   QualType type = T;
   while (!type.isNull()) {
-    if (auto *elaboratedT = type->getAs<ElaboratedType>()) {
-      type = elaboratedT->desugar();
-      continue;
-    }
     if (auto *SpecialT = type->getAs<TemplateSpecializationType>()) {
       auto *Decl = SpecialT->getTemplateName().getAsTemplateDecl();
       return Decl && Pred(Decl->getNameAsString());
@@ -248,13 +244,15 @@ void RetainTypeChecker::visitTypedef(const TypedefDecl *TD) {
   const RecordType *RT = PointeeQT->getAs<RecordType>();
   if (!RT) {
     if (TD->hasAttr<ObjCBridgeAttr>() || TD->hasAttr<ObjCBridgeMutableAttr>()) {
-      if (auto *Type = TD->getTypeForDecl())
-        RecordlessTypes.insert(Type);
+      RecordlessTypes.insert(TD->getASTContext()
+                                 .getTypedefType(ElaboratedTypeKeyword::None,
+                                                 /*Qualifier=*/std::nullopt, TD)
+                                 .getTypePtr());
     }
     return;
   }
 
-  for (auto *Redecl : RT->getDecl()->getMostRecentDecl()->redecls()) {
+  for (auto *Redecl : RT->getOriginalDecl()->getMostRecentDecl()->redecls()) {
     if (Redecl->getAttr<ObjCBridgeAttr>() ||
         Redecl->getAttr<ObjCBridgeMutableAttr>()) {
       CFPointees.insert(RT);
@@ -266,21 +264,10 @@ void RetainTypeChecker::visitTypedef(const TypedefDecl *TD) {
 bool RetainTypeChecker::isUnretained(const QualType QT, bool ignoreARC) {
   if (ento::cocoa::isCocoaObjectRef(QT) && (!IsARCEnabled || ignoreARC))
     return true;
-  auto CanonicalType = QT.getCanonicalType();
-  auto PointeeType = CanonicalType->getPointeeType();
-  auto *RT = dyn_cast_or_null<RecordType>(PointeeType.getTypePtrOrNull());
-  if (!RT) {
-    auto *Type = QT.getTypePtrOrNull();
-    while (Type) {
-      if (RecordlessTypes.contains(Type))
-        return true;
-      auto *ET = dyn_cast_or_null<ElaboratedType>(Type);
-      if (!ET)
-        break;
-      Type = ET->desugar().getTypePtrOrNull();
-    }
-  }
-  return RT && CFPointees.contains(RT);
+  if (auto *RT = dyn_cast_or_null<RecordType>(
+          QT.getCanonicalType()->getPointeeType().getTypePtrOrNull()))
+    return CFPointees.contains(RT);
+  return RecordlessTypes.contains(QT.getTypePtr());
 }
 
 std::optional<bool> isUnretained(const QualType T, bool IsARCEnabled) {
@@ -306,7 +293,7 @@ std::optional<bool> isUnretained(const QualType T, bool IsARCEnabled) {
   auto *Record = PointeeType->getAsStructureType();
   if (!Record)
     return false;
-  auto *Decl = Record->getDecl();
+  auto *Decl = Record->getOriginalDecl();
   if (!Decl)
     return false;
   auto TypeName = Decl->getName();
diff --git a/clang/lib/StaticAnalyzer/Checkers/WebKit/RefCntblBaseVirtualDtorChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/WebKit/RefCntblBaseVirtualDtorChecker.cpp
index 98c587d62978b..6f3a280971cb8 100644
--- a/clang/lib/StaticAnalyzer/Checkers/WebKit/RefCntblBaseVirtualDtorChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/WebKit/RefCntblBaseVirtualDtorChecker.cpp
@@ -113,10 +113,6 @@ class DerefFuncDeleteExprVisitor
         auto CastType = Cast->getType();
         if (auto *PtrType = dyn_cast<PointerType>(CastType)) {
           auto PointeeType = PtrType->getPointeeType();
-          while (auto *ET = dyn_cast<ElaboratedType>(PointeeType)) {
-            if (ET->isSugared())
-              PointeeType = ET->desugar();
-          }
           if (auto *ParmType = dyn_cast<TemplateTypeParmType>(PointeeType)) {
             if (ArgList) {
               auto ParmIndex = ParmType->getIndex();
@@ -125,13 +121,13 @@ class DerefFuncDeleteExprVisitor
                 return true;
             }
           } else if (auto *RD = dyn_cast<RecordType>(PointeeType)) {
-            if (RD->getDecl() == ClassDecl)
+            if (declaresSameEntity(RD->getOriginalDecl(), ClassDecl))
               return true;
           } else if (auto *ST =
                          dyn_cast<SubstTemplateTypeParmType>(PointeeType)) {
             auto Type = ST->getReplacementType();
             if (auto *RD = dyn_cast<RecordType>(Type)) {
-              if (RD->getDecl() == ClassDecl)
+              if (declaresSameEntity(RD->getOriginalDecl(), ClassDecl))
                 return true;
             }
           }
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index 4631e0cad5546..63f0d70238992 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -1950,7 +1950,7 @@ class TrackControlDependencyCondBRVisitor final
     : public TrackingBugReporterVisitor {
   const ExplodedNode *Origin;
   ControlDependencyCalculator ControlDeps;
-  llvm::SmallSet<const CFGBlock *, 32> VisitedBlocks;
+  llvm::SmallPtrSet<const CFGBlock *, 32> VisitedBlocks;
 
 public:
   TrackControlDependencyCondBRVisitor(TrackerRef ParentTracker,
diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
index 34fcb9b64d555..180056cf68b64 100644
--- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -89,7 +89,7 @@ static bool isCallback(QualType T) {
     T = T->getPointeeType();
 
   if (const RecordType *RT = T->getAsStructureType()) {
-    const RecordDecl *RD = RT->getDecl();
+    const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
     for (const auto *I : RD->fields()) {
       QualType FieldT = I->getType();
       if (FieldT->isBlockPointerType() || FieldT->isFunctionPointerType())
@@ -391,7 +391,9 @@ bool CallEvent::isVariadic(const Decl *D) {
 
 static bool isTransparentUnion(QualType T) {
   const RecordType *UT = T->getAsUnionType();
-  return UT && UT->getDecl()->hasAttr<TransparentUnionAttr>();
+  return UT && UT->getOriginalDecl()
+                   ->getMostRecentDecl()
+                   ->hasAttr<TransparentUnionAttr>();
 }
 
 // In some cases, symbolic cases should be transformed before we associate
@@ -843,7 +845,7 @@ void CXXInstanceCall::getInitialStackFrameContents(
     if (MD->getCanonicalDecl() != getDecl()->getCanonicalDecl()) {
       ASTContext &Ctx = SVB.getContext();
       const CXXRecordDecl *Class = MD->getParent();
-      QualType Ty = Ctx.getPointerType(Ctx.getRecordType(Class));
+      CanQualType Ty = Ctx.getPointerType(Ctx.getCanonicalTagType(Class));
 
       // FIXME: CallEvent maybe shouldn't be directly accessing StoreManager.
       std::optional<SVal> V =
@@ -854,7 +856,8 @@ void CXXInstanceCall::getInitialStackFrameContents(
         // Fall back to a generic pointer cast for this-value.
         const CXXMethodDecl *StaticMD = cast<CXXMethodDecl>(getDecl());
         const CXXRecordDecl *StaticClass = StaticMD->getParent();
-        QualType StaticTy = Ctx.getPointerType(Ctx.getRecordType(StaticClass));
+        CanQualType StaticTy =
+            Ctx.getPointerType(Ctx.getCanonicalTagType(StaticClass));
         ThisVal = SVB.evalCast(ThisVal, Ty, StaticTy);
       } else
         ThisVal = *V;
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
index 0fe677e4ee435..44c6f9f52cca6 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerManager.cpp
@@ -376,11 +376,13 @@ namespace {
     const Stmt *S;
     ExprEngine &Eng;
     const ProgramPoint &PP;
+    bool AtDeclInit;
 
-    CheckBindContext(const CheckersTy &checkers,
-                     SVal loc, SVal val, const Stmt *s, ExprEngine &eng,
+    CheckBindContext(const CheckersTy &checkers, SVal loc, SVal val,
+                     const Stmt *s, bool AtDeclInit, ExprEngine &eng,
                      const ProgramPoint &pp)
-        : Checkers(checkers), Loc(loc), Val(val), S(s), Eng(eng), PP(pp) {}
+        : Checkers(checkers), Loc(loc), Val(val), S(s), Eng(eng), PP(pp),
+          AtDeclInit(AtDeclInit) {}
 
     CheckersTy::const_iterator checkers_begin() { return Checkers.begin(); }
     CheckersTy::const_iterator checkers_end() { return Checkers.end(); }
@@ -391,7 +393,7 @@ namespace {
       const ProgramPoint &L = PP.withTag(checkFn.Checker);
       CheckerContext C(Bldr, Eng, Pred, L);
 
-      checkFn(Loc, Val, S, C);
+      checkFn(Loc, Val, S, AtDeclInit, C);
     }
   };
 
@@ -408,10 +410,10 @@ namespace {
 /// Run checkers for binding of a value to a location.
 void CheckerManager::runCheckersForBind(ExplodedNodeSet &Dst,
                                         const ExplodedNodeSet &Src,
-                                        SVal location, SVal val,
-                                        const Stmt *S, ExprEngine &Eng,
+                                        SVal location, SVal val, const Stmt *S,
+                                        bool AtDeclInit, ExprEngine &Eng,
                                         const ProgramPoint &PP) {
-  CheckBindContext C(BindCheckers, location, val, S, Eng, PP);
+  CheckBindContext C(BindCheckers, location, val, S, AtDeclInit, Eng, PP);
   llvm::TimeTraceScope TimeScope{
       "CheckerManager::runCheckersForBind",
       [&val]() { return getTimeTraceBindMetadata(val); }};
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index d87484470f8b5..c853c00019c10 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -3714,9 +3714,8 @@ ExprEngine::notifyCheckersOfPointerEscape(ProgramStateRef State,
 /// evalBind - Handle the semantics of binding a value to a specific location.
 ///  This method is used by evalStore and (soon) VisitDeclStmt, and others.
 void ExprEngine::evalBind(ExplodedNodeSet &Dst, const Stmt *StoreE,
-                          ExplodedNode *Pred,
-                          SVal location, SVal Val,
-                          bool atDeclInit, const ProgramPoint *PP) {
+                          ExplodedNode *Pred, SVal location, SVal Val,
+                          bool AtDeclInit, const ProgramPoint *PP) {
   const LocationContext *LC = Pred->getLocationContext();
   PostStmt PS(StoreE, LC);
   if (!PP)
@@ -3725,7 +3724,7 @@ void ExprEngine::evalBind(ExplodedNodeSet &Dst, const Stmt *StoreE,
   // Do a previsit of the bind.
   ExplodedNodeSet CheckedSet;
   getCheckerManager().runCheckersForBind(CheckedSet, Pred, location, Val,
-                                         StoreE, *this, *PP);
+                                         StoreE, AtDeclInit, *this, *PP);
 
   StmtNodeBuilder Bldr(CheckedSet, Dst, *currBldrCtx);
 
@@ -3748,8 +3747,8 @@ void ExprEngine::evalBind(ExplodedNodeSet &Dst, const Stmt *StoreE,
     // When binding the value, pass on the hint that this is a initialization.
     // For initializations, we do not need to inform clients of region
     // changes.
-    state = state->bindLoc(location.castAs<Loc>(),
-                           Val, LC, /* notifyChanges = */ !atDeclInit);
+    state = state->bindLoc(location.castAs<Loc>(), Val, LC,
+                           /* notifyChanges = */ !AtDeclInit);
 
     const MemRegion *LocReg = nullptr;
     if (std::optional<loc::MemRegionVal> LocRegVal =
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
index fe70558dfc45c..c0b28d2ebb212 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
@@ -85,7 +85,7 @@ void ExprEngine::performTrivialCopy(NodeBuilder &Bldr, ExplodedNode *Pred,
   evalLocation(Tmp, CallExpr, VExpr, Pred, Pred->getState(), V,
                /*isLoad=*/true);
   for (ExplodedNode *N : Tmp)
-    evalBind(Dst, CallExpr, N, ThisVal, V, true);
+    evalBind(Dst, CallExpr, N, ThisVal, V, !AlwaysReturnsLValue);
 
   PostStmt PS(CallExpr, LCtx);
   for (ExplodedNode *N : Dst) {
diff --git a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
index 0058a0ddfb0df..5f271963e3d09 100644
--- a/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
+++ b/clang/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -192,11 +192,11 @@ QualType ObjCIvarRegion::getValueType() const {
 }
 
 QualType CXXBaseObjectRegion::getValueType() const {
-  return QualType(getDecl()->getTypeForDecl(), 0);
+  return getContext().getCanonicalTagType(getDecl());
 }
 
 QualType CXXDerivedObjectRegion::getValueType() const {
-  return QualType(getDecl()->getTypeForDecl(), 0);
+  return getContext().getCanonicalTagType(getDecl());
 }
 
 QualType ParamVarRegion::getValueType() const {
diff --git a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
index 388034b087789..02375b0c3469a 100644
--- a/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
+++ b/clang/lib/StaticAnalyzer/Core/RegionStore.cpp
@@ -2453,7 +2453,8 @@ NonLoc RegionStoreManager::createLazyBinding(RegionBindingsConstRef B,
 
 SVal RegionStoreManager::getBindingForStruct(RegionBindingsConstRef B,
                                              const TypedValueRegion *R) {
-  const RecordDecl *RD = R->getValueType()->castAs<RecordType>()->getDecl();
+  const RecordDecl *RD =
+      R->getValueType()->castAs<RecordType>()->getOriginalDecl();
   if (!RD->getDefinition())
     return UnknownVal();
 
@@ -2844,7 +2845,7 @@ RegionStoreManager::bindStruct(LimitedRegionBindingsConstRef B,
   assert(T->isStructureOrClassType());
 
   const RecordType* RT = T->castAs<RecordType>();
-  const RecordDecl *RD = RT->getDecl();
+  const RecordDecl *RD = RT->getOriginalDecl()->getDefinitionOrSelf();
 
   if (!RD->isCompleteDefinition())
     return B;
diff --git a/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp b/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
index 2276c452cce76..a6f4463852a9b 100644
--- a/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
+++ b/clang/lib/StaticAnalyzer/Core/SValBuilder.cpp
@@ -320,8 +320,8 @@ loc::MemRegionVal SValBuilder::getCXXThis(const CXXMethodDecl *D,
 /// Return a memory region for the 'this' object reference.
 loc::MemRegionVal SValBuilder::getCXXThis(const CXXRecordDecl *D,
                                           const StackFrameContext *SFC) {
-  const Type *T = D->getTypeForDecl();
-  QualType PT = getContext().getPointerType(QualType(T, 0));
+  CanQualType PT =
+      getContext().getPointerType(getContext().getCanonicalTagType(D));
   return loc::MemRegionVal(getRegionManager().getCXXThisRegion(PT, SFC));
 }
 
diff --git a/clang/lib/Tooling/ASTDiff/ASTDiff.cpp b/clang/lib/Tooling/ASTDiff/ASTDiff.cpp
index 5f7153cd53ac2..d70a679cc8dd0 100644
--- a/clang/lib/Tooling/ASTDiff/ASTDiff.cpp
+++ b/clang/lib/Tooling/ASTDiff/ASTDiff.cpp
@@ -246,7 +246,7 @@ struct PreorderVisitor : public RecursiveASTVisitor<PreorderVisitor> {
     PostTraverse(SavedState);
     return true;
   }
-  bool TraverseType(QualType T) { return true; }
+  bool TraverseType(QualType T, bool TraverseQualifier = true) { return true; }
   bool TraverseConstructorInitializer(CXXCtorInitializer *Init) {
     if (isNodeExcluded(Tree.AST.getSourceManager(), Init))
       return true;
@@ -428,11 +428,12 @@ std::string SyntaxTree::Impl::getDeclValue(const Decl *D) const {
     Value += getRelativeName(N) + ";";
   if (auto *T = dyn_cast<TypedefNameDecl>(D))
     return Value + T->getUnderlyingType().getAsString(TypePP) + ";";
-  if (auto *T = dyn_cast<TypeDecl>(D))
-    if (T->getTypeForDecl())
-      Value +=
-          T->getTypeForDecl()->getCanonicalTypeInternal().getAsString(TypePP) +
-          ";";
+  if (auto *T = dyn_cast<TypeDecl>(D)) {
+    const ASTContext &Ctx = T->getASTContext();
+    Value +=
+        Ctx.getTypeDeclType(T)->getCanonicalTypeInternal().getAsString(TypePP) +
+        ";";
+  }
   if (auto *U = dyn_cast<UsingDirectiveDecl>(D))
     return std::string(U->getNominatedNamespace()->getName());
   if (auto *A = dyn_cast<AccessSpecDecl>(D)) {
diff --git a/clang/lib/Tooling/Refactoring/Lookup.cpp b/clang/lib/Tooling/Refactoring/Lookup.cpp
index 757fba0404e62..dedde860516ab 100644
--- a/clang/lib/Tooling/Refactoring/Lookup.cpp
+++ b/clang/lib/Tooling/Refactoring/Lookup.cpp
@@ -108,16 +108,6 @@ static StringRef getBestNamespaceSubstr(const DeclContext *DeclA,
   }
 }
 
-/// Check if the name specifier begins with a written "::".
-static bool isFullyQualified(const NestedNameSpecifier *NNS) {
-  while (NNS) {
-    if (NNS->getKind() == NestedNameSpecifier::Global)
-      return true;
-    NNS = NNS->getPrefix();
-  }
-  return false;
-}
-
 // Adds more scope specifier to the spelled name until the spelling is not
 // ambiguous. A spelling is ambiguous if the resolution of the symbol is
 // ambiguous. For example, if QName is "::y::bar", the spelling is "y::bar", and
@@ -182,7 +172,7 @@ static std::string disambiguateSpellingInScope(StringRef Spelling,
   return Disambiguated;
 }
 
-std::string tooling::replaceNestedName(const NestedNameSpecifier *Use,
+std::string tooling::replaceNestedName(NestedNameSpecifier Use,
                                        SourceLocation UseLoc,
                                        const DeclContext *UseContext,
                                        const NamedDecl *FromDecl,
@@ -217,7 +207,7 @@ std::string tooling::replaceNestedName(const NestedNameSpecifier *Use,
   // We work backwards (from most specific possible namespace to least
   // specific).
   StringRef Suggested = getBestNamespaceSubstr(UseContext, ReplacementString,
-                                               isFullyQualified(Use));
+                                               Use.isFullyQualified());
 
   return disambiguateSpellingInScope(Suggested, ReplacementString, *UseContext,
                                      UseLoc);
diff --git a/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp b/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp
index 8eff77861f753..d9444110d421c 100644
--- a/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp
+++ b/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp
@@ -107,45 +107,83 @@ class USRLocFindingASTVisitor
 };
 
 SourceLocation StartLocationForType(TypeLoc TL) {
+  if (auto QTL = TL.getAs<QualifiedTypeLoc>())
+    TL = QTL.getUnqualifiedLoc();
+
   // For elaborated types (e.g. `struct a::A`) we want the portion after the
-  // `struct` but including the namespace qualifier, `a::`.
-  if (auto ElaboratedTypeLoc = TL.getAs<clang::ElaboratedTypeLoc>()) {
-    NestedNameSpecifierLoc NestedNameSpecifier =
-        ElaboratedTypeLoc.getQualifierLoc();
-    if (NestedNameSpecifier.getNestedNameSpecifier())
-      return NestedNameSpecifier.getBeginLoc();
-    TL = TL.getNextTypeLoc();
+  // `struct`, including the namespace qualifier, `a::`.
+  switch (TL.getTypeLocClass()) {
+  case TypeLoc::Record:
+  case TypeLoc::InjectedClassName:
+  case TypeLoc::Enum: {
+    auto TTL = TL.castAs<TagTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = TTL.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return TTL.getNameLoc();
+  }
+  case TypeLoc::Typedef: {
+    auto TTL = TL.castAs<TypedefTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = TTL.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return TTL.getNameLoc();
+  }
+  case TypeLoc::UnresolvedUsing: {
+    auto TTL = TL.castAs<UnresolvedUsingTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = TTL.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return TTL.getNameLoc();
+  }
+  case TypeLoc::Using: {
+    auto TTL = TL.castAs<UsingTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = TTL.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return TTL.getNameLoc();
+  }
+  case TypeLoc::TemplateSpecialization: {
+    auto TTL = TL.castAs<TemplateSpecializationTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = TTL.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return TTL.getTemplateNameLoc();
+  }
+  case TypeLoc::DeducedTemplateSpecialization: {
+    auto DTL = TL.castAs<clang::DeducedTemplateSpecializationTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = DTL.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return DTL.getTemplateNameLoc();
+  }
+  case TypeLoc::DependentName: {
+    auto TTL = TL.castAs<DependentNameTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = TTL.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return TTL.getNameLoc();
+  }
+  case TypeLoc::DependentTemplateSpecialization: {
+    auto TTL = TL.castAs<DependentTemplateSpecializationTypeLoc>();
+    if (NestedNameSpecifierLoc QualifierLoc = TTL.getQualifierLoc())
+      return QualifierLoc.getBeginLoc();
+    return TTL.getTemplateNameLoc();
+  }
+  default:
+    llvm_unreachable("unhandled TypeLoc class");
   }
-  return TL.getBeginLoc();
 }
 
 SourceLocation EndLocationForType(TypeLoc TL) {
-  // Dig past any namespace or keyword qualifications.
-  while (TL.getTypeLocClass() == TypeLoc::Elaborated ||
-         TL.getTypeLocClass() == TypeLoc::Qualified)
-    TL = TL.getNextTypeLoc();
+  if (auto QTL = TL.getAs<QualifiedTypeLoc>())
+    TL = QTL.getUnqualifiedLoc();
 
   // The location for template specializations (e.g. Foo<int>) includes the
   // templated types in its location range.  We want to restrict this to just
   // before the `<` character.
-  if (TL.getTypeLocClass() == TypeLoc::TemplateSpecialization) {
-    return TL.castAs<TemplateSpecializationTypeLoc>()
-        .getLAngleLoc()
-        .getLocWithOffset(-1);
-  }
+  if (auto TTL = TL.getAs<TemplateSpecializationTypeLoc>())
+    return TTL.getLAngleLoc().getLocWithOffset(-1);
   return TL.getEndLoc();
 }
 
-NestedNameSpecifier *GetNestedNameForType(TypeLoc TL) {
-  // Dig past any keyword qualifications.
-  while (TL.getTypeLocClass() == TypeLoc::Qualified)
-    TL = TL.getNextTypeLoc();
-
-  // For elaborated types (e.g. `struct a::A`) we want the portion after the
-  // `struct` but including the namespace qualifier, `a::`.
-  if (auto ElaboratedTypeLoc = TL.getAs<clang::ElaboratedTypeLoc>())
-    return ElaboratedTypeLoc.getQualifierLoc().getNestedNameSpecifier();
-  return nullptr;
+NestedNameSpecifier GetNestedNameForType(TypeLoc TL) {
+  if (auto QTL = TL.getAs<QualifiedTypeLoc>())
+    TL = QTL.getUnqualifiedLoc();
+  return TL.getPrefix().getNestedNameSpecifier();
 }
 
 // Find all locations identified by the given USRs for rename.
@@ -168,14 +206,14 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
     const NamedDecl *FromDecl;
     // The declaration in which the nested name is contained (can be nullptr).
     const Decl *Context;
-    // The nested name being replaced (can be nullptr).
-    const NestedNameSpecifier *Specifier;
+    // The nested name being replaced.
+    NestedNameSpecifier Specifier;
     // Determine whether the prefix qualifiers of the NewName should be ignored.
     // Normally, we set it to true for the symbol declaration and definition to
     // avoid adding prefix qualifiers.
     // For example, if it is true and NewName is "a::b::foo", then the symbol
     // occurrence which the RenameInfo points to will be renamed to "foo".
-    bool IgnorePrefixQualifers;
+    bool IgnorePrefixQualifiers;
   };
 
   bool VisitNamedDecl(const NamedDecl *Decl) {
@@ -203,8 +241,8 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
                            EndLoc,
                            /*FromDecl=*/nullptr,
                            /*Context=*/nullptr,
-                           /*Specifier=*/nullptr,
-                           /*IgnorePrefixQualifers=*/true};
+                           /*Specifier=*/std::nullopt,
+                           /*IgnorePrefixQualifiers=*/true};
         RenameInfos.push_back(Info);
       }
     }
@@ -217,10 +255,10 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
     auto EndLoc = Expr->getMemberLoc();
     if (isInUSRSet(Decl)) {
       RenameInfos.push_back({StartLoc, EndLoc,
-                            /*FromDecl=*/nullptr,
-                            /*Context=*/nullptr,
-                            /*Specifier=*/nullptr,
-                            /*IgnorePrefixQualifiers=*/true});
+                             /*FromDecl=*/nullptr,
+                             /*Context=*/nullptr,
+                             /*Specifier=*/std::nullopt,
+                             /*IgnorePrefixQualifiers=*/true});
     }
     return true;
   }
@@ -235,7 +273,7 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
             RenameInfos.push_back({StartLoc, EndLoc,
                                    /*FromDecl=*/nullptr,
                                    /*Context=*/nullptr,
-                                   /*Specifier=*/nullptr,
+                                   /*Specifier=*/std::nullopt,
                                    /*IgnorePrefixQualifiers=*/true});
           }
         }
@@ -257,7 +295,7 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
           RenameInfos.push_back({Loc, Loc,
                                  /*FromDecl=*/nullptr,
                                  /*Context=*/nullptr,
-                                 /*Specifier=*/nullptr,
+                                 /*Specifier=*/std::nullopt,
                                  /*IgnorePrefixQualifiers=*/true});
         }
       }
@@ -288,7 +326,7 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
         RenameInfos.push_back({EndLoc, EndLoc,
                                /*FromDecl=*/nullptr,
                                /*Context=*/nullptr,
-                               /*Specifier=*/nullptr,
+                               /*Specifier=*/std::nullopt,
                                /*IgnorePrefixQualifiers=*/true});
         return true;
       }
@@ -332,7 +370,7 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
                          Decl,
                          getClosestAncestorDecl(*Expr),
                          Expr->getQualifier(),
-                         /*IgnorePrefixQualifers=*/false};
+                         /*IgnorePrefixQualifiers=*/false};
       RenameInfos.push_back(Info);
     }
 
@@ -350,18 +388,18 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
   }
 
   bool VisitNestedNameSpecifierLocations(NestedNameSpecifierLoc NestedLoc) {
-    if (!NestedLoc.getNestedNameSpecifier()->getAsType())
+    TypeLoc TL = NestedLoc.getAsTypeLoc();
+    if (!TL)
       return true;
 
-    if (const auto *TargetDecl =
-            getSupportedDeclFromTypeLoc(NestedLoc.getTypeLoc())) {
+    if (const auto *TargetDecl = getSupportedDeclFromTypeLoc(TL)) {
       if (isInUSRSet(TargetDecl)) {
         RenameInfo Info = {NestedLoc.getBeginLoc(),
-                           EndLocationForType(NestedLoc.getTypeLoc()),
+                           EndLocationForType(TL),
                            TargetDecl,
                            getClosestAncestorDecl(NestedLoc),
-                           NestedLoc.getNestedNameSpecifier()->getPrefix(),
-                           /*IgnorePrefixQualifers=*/false};
+                           /*Specifier=*/std::nullopt,
+                           /*IgnorePrefixQualifiers=*/false};
         RenameInfos.push_back(Info);
       }
     }
@@ -411,7 +449,7 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
                              TargetDecl,
                              getClosestAncestorDecl(Loc),
                              GetNestedNameForType(Loc),
-                             /*IgnorePrefixQualifers=*/false};
+                             /*IgnorePrefixQualifiers=*/false};
           RenameInfos.push_back(Info);
         }
         return true;
@@ -421,33 +459,17 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
     // Handle specific template class specialiation cases.
     if (const auto *TemplateSpecType =
             dyn_cast<TemplateSpecializationType>(Loc.getType())) {
-      TypeLoc TargetLoc = Loc;
-      if (!ParentTypeLoc.isNull()) {
-        if (llvm::isa<ElaboratedType>(ParentTypeLoc.getType()))
-          TargetLoc = ParentTypeLoc;
-      }
-
       if (isInUSRSet(TemplateSpecType->getTemplateName().getAsTemplateDecl())) {
-        TypeLoc TargetLoc = Loc;
-        // FIXME: Find a better way to handle this case.
-        // For the qualified template class specification type like
-        // "ns::Foo<int>" in "ns::Foo<int>& f();", we want the parent typeLoc
-        // (ElaboratedType) of the TemplateSpecializationType in order to
-        // catch the prefix qualifiers "ns::".
-        if (!ParentTypeLoc.isNull() &&
-            llvm::isa<ElaboratedType>(ParentTypeLoc.getType()))
-          TargetLoc = ParentTypeLoc;
-
-        auto StartLoc = StartLocationForType(TargetLoc);
-        auto EndLoc = EndLocationForType(TargetLoc);
+        auto StartLoc = StartLocationForType(Loc);
+        auto EndLoc = EndLocationForType(Loc);
         if (IsValidEditLoc(Context.getSourceManager(), StartLoc)) {
           RenameInfo Info = {
               StartLoc,
               EndLoc,
               TemplateSpecType->getTemplateName().getAsTemplateDecl(),
-              getClosestAncestorDecl(DynTypedNode::create(TargetLoc)),
-              GetNestedNameForType(TargetLoc),
-              /*IgnorePrefixQualifers=*/false};
+              getClosestAncestorDecl(DynTypedNode::create(Loc)),
+              GetNestedNameForType(Loc),
+              /*IgnorePrefixQualifiers=*/false};
           RenameInfos.push_back(Info);
         }
       }
@@ -469,12 +491,7 @@ class RenameLocFinder : public RecursiveASTVisitor<RenameLocFinder> {
   const NamedDecl *getSupportedDeclFromTypeLoc(TypeLoc Loc) {
     if (const auto* TT = Loc.getType()->getAs<clang::TypedefType>())
       return TT->getDecl();
-    if (const auto *RD = Loc.getType()->getAsCXXRecordDecl())
-      return RD;
-    if (const auto *ED =
-            llvm::dyn_cast_or_null<EnumDecl>(Loc.getType()->getAsTagDecl()))
-      return ED;
-    return nullptr;
+    return Loc.getType()->getAsTagDecl();
   }
 
   // Get the closest ancester which is a declaration of a given AST node.
@@ -549,7 +566,7 @@ createRenameAtomicChanges(llvm::ArrayRef<std::string> USRs,
 
   for (const auto &RenameInfo : Finder.getRenameInfos()) {
     std::string ReplacedName = NewName.str();
-    if (RenameInfo.IgnorePrefixQualifers) {
+    if (RenameInfo.IgnorePrefixQualifiers) {
       // Get the name without prefix qualifiers from NewName.
       size_t LastColonPos = NewName.find_last_of(':');
       if (LastColonPos != std::string::npos)
diff --git a/clang/lib/Tooling/Syntax/BuildTree.cpp b/clang/lib/Tooling/Syntax/BuildTree.cpp
index eb9fa7a7fa1e8..546161cee33f4 100644
--- a/clang/lib/Tooling/Syntax/BuildTree.cpp
+++ b/clang/lib/Tooling/Syntax/BuildTree.cpp
@@ -918,97 +918,91 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
     return true;
   }
 
-  // FIXME: Fix `NestedNameSpecifierLoc::getLocalSourceRange` for the
-  // `DependentTemplateSpecializationType` case.
-  /// Given a nested-name-specifier return the range for the last name
-  /// specifier.
-  ///
-  /// e.g. `std::T::template X<U>::` => `template X<U>::`
-  SourceRange getLocalSourceRange(const NestedNameSpecifierLoc &NNSLoc) {
-    auto SR = NNSLoc.getLocalSourceRange();
-
-    // The method `NestedNameSpecifierLoc::getLocalSourceRange` *should*
-    // return the desired `SourceRange`, but there is a corner case. For a
-    // `DependentTemplateSpecializationType` this method returns its
-    // qualifiers as well, in other words in the example above this method
-    // returns `T::template X<U>::` instead of only `template X<U>::`
-    if (auto TL = NNSLoc.getTypeLoc()) {
-      if (auto DependentTL =
-              TL.getAs<DependentTemplateSpecializationTypeLoc>()) {
-        // The 'template' keyword is always present in dependent template
-        // specializations. Except in the case of incorrect code
-        // TODO: Treat the case of incorrect code.
-        SR.setBegin(DependentTL.getTemplateKeywordLoc());
-      }
-    }
-
-    return SR;
-  }
-
-  syntax::NodeKind getNameSpecifierKind(const NestedNameSpecifier &NNS) {
-    switch (NNS.getKind()) {
-    case NestedNameSpecifier::Global:
-      return syntax::NodeKind::GlobalNameSpecifier;
-    case NestedNameSpecifier::Namespace:
-    case NestedNameSpecifier::Identifier:
-      return syntax::NodeKind::IdentifierNameSpecifier;
-    case NestedNameSpecifier::TypeSpec: {
-      const auto *NNSType = NNS.getAsType();
-      assert(NNSType);
-      if (isa<DecltypeType>(NNSType))
-        return syntax::NodeKind::DecltypeNameSpecifier;
-      if (isa<TemplateSpecializationType, DependentTemplateSpecializationType>(
-              NNSType))
-        return syntax::NodeKind::SimpleTemplateNameSpecifier;
-      return syntax::NodeKind::IdentifierNameSpecifier;
-    }
-    default:
-      // FIXME: Support Microsoft's __super
-      llvm::report_fatal_error("We don't yet support the __super specifier",
-                               true);
-    }
+  syntax::NameSpecifier *buildIdentifier(SourceRange SR,
+                                         bool DropBack = false) {
+    auto NameSpecifierTokens = Builder.getRange(SR).drop_back(DropBack);
+    assert(NameSpecifierTokens.size() == 1);
+    Builder.markChildToken(NameSpecifierTokens.begin(),
+                           syntax::NodeRole::Unknown);
+    auto *NS = new (allocator()) syntax::IdentifierNameSpecifier;
+    Builder.foldNode(NameSpecifierTokens, NS, nullptr);
+    return NS;
+  }
+
+  syntax::NameSpecifier *buildSimpleTemplateName(SourceRange SR) {
+    auto NameSpecifierTokens = Builder.getRange(SR);
+    // TODO: Build `SimpleTemplateNameSpecifier` children and implement
+    // accessors to them.
+    // Be aware, we cannot do that simply by calling `TraverseTypeLoc`,
+    // some `TypeLoc`s have inside them the previous name specifier and
+    // we want to treat them independently.
+    auto *NS = new (allocator()) syntax::SimpleTemplateNameSpecifier;
+    Builder.foldNode(NameSpecifierTokens, NS, nullptr);
+    return NS;
   }
 
   syntax::NameSpecifier *
   buildNameSpecifier(const NestedNameSpecifierLoc &NNSLoc) {
     assert(NNSLoc.hasQualifier());
-    auto NameSpecifierTokens =
-        Builder.getRange(getLocalSourceRange(NNSLoc)).drop_back();
-    switch (getNameSpecifierKind(*NNSLoc.getNestedNameSpecifier())) {
-    case syntax::NodeKind::GlobalNameSpecifier:
+    switch (NNSLoc.getNestedNameSpecifier().getKind()) {
+    case NestedNameSpecifier::Kind::Global:
       return new (allocator()) syntax::GlobalNameSpecifier;
-    case syntax::NodeKind::IdentifierNameSpecifier: {
-      assert(NameSpecifierTokens.size() == 1);
-      Builder.markChildToken(NameSpecifierTokens.begin(),
-                             syntax::NodeRole::Unknown);
-      auto *NS = new (allocator()) syntax::IdentifierNameSpecifier;
-      Builder.foldNode(NameSpecifierTokens, NS, nullptr);
-      return NS;
-    }
-    case syntax::NodeKind::SimpleTemplateNameSpecifier: {
-      // TODO: Build `SimpleTemplateNameSpecifier` children and implement
-      // accessors to them.
-      // Be aware, we cannot do that simply by calling `TraverseTypeLoc`,
-      // some `TypeLoc`s have inside them the previous name specifier and
-      // we want to treat them independently.
-      auto *NS = new (allocator()) syntax::SimpleTemplateNameSpecifier;
-      Builder.foldNode(NameSpecifierTokens, NS, nullptr);
-      return NS;
-    }
-    case syntax::NodeKind::DecltypeNameSpecifier: {
-      const auto TL = NNSLoc.getTypeLoc().castAs<DecltypeTypeLoc>();
-      if (!RecursiveASTVisitor::TraverseDecltypeTypeLoc(TL))
-        return nullptr;
-      auto *NS = new (allocator()) syntax::DecltypeNameSpecifier;
-      // TODO: Implement accessor to `DecltypeNameSpecifier` inner
-      // `DecltypeTypeLoc`.
-      // For that add mapping from `TypeLoc` to `syntax::Node*` then:
-      // Builder.markChild(TypeLoc, syntax::NodeRole);
-      Builder.foldNode(NameSpecifierTokens, NS, nullptr);
-      return NS;
+
+    case NestedNameSpecifier::Kind::Namespace:
+      return buildIdentifier(NNSLoc.getLocalSourceRange(), /*DropBack=*/true);
+
+    case NestedNameSpecifier::Kind::Type: {
+      TypeLoc TL = NNSLoc.castAsTypeLoc();
+      switch (TL.getTypeLocClass()) {
+      case TypeLoc::Record:
+      case TypeLoc::InjectedClassName:
+      case TypeLoc::Enum:
+        return buildIdentifier(TL.castAs<TagTypeLoc>().getNameLoc());
+      case TypeLoc::Typedef:
+        return buildIdentifier(TL.castAs<TypedefTypeLoc>().getNameLoc());
+      case TypeLoc::UnresolvedUsing:
+        return buildIdentifier(
+            TL.castAs<UnresolvedUsingTypeLoc>().getNameLoc());
+      case TypeLoc::Using:
+        return buildIdentifier(TL.castAs<UsingTypeLoc>().getNameLoc());
+      case TypeLoc::DependentName:
+        return buildIdentifier(TL.castAs<DependentNameTypeLoc>().getNameLoc());
+      case TypeLoc::TemplateSpecialization: {
+        auto TST = TL.castAs<TemplateSpecializationTypeLoc>();
+        SourceLocation BeginLoc = TST.getTemplateKeywordLoc();
+        if (BeginLoc.isInvalid())
+          BeginLoc = TST.getTemplateNameLoc();
+        return buildSimpleTemplateName({BeginLoc, TST.getEndLoc()});
+      }
+      case TypeLoc::DependentTemplateSpecialization: {
+        auto DT = TL.castAs<DependentTemplateSpecializationTypeLoc>();
+        SourceLocation BeginLoc = DT.getTemplateKeywordLoc();
+        if (BeginLoc.isInvalid())
+          BeginLoc = DT.getTemplateNameLoc();
+        return buildSimpleTemplateName({BeginLoc, DT.getEndLoc()});
+      }
+      case TypeLoc::Decltype: {
+        const auto DTL = TL.castAs<DecltypeTypeLoc>();
+        if (!RecursiveASTVisitor::TraverseDecltypeTypeLoc(
+                DTL, /*TraverseQualifier=*/true))
+          return nullptr;
+        auto *NS = new (allocator()) syntax::DecltypeNameSpecifier;
+        // TODO: Implement accessor to `DecltypeNameSpecifier` inner
+        // `DecltypeTypeLoc`.
+        // For that add mapping from `TypeLoc` to `syntax::Node*` then:
+        // Builder.markChild(TypeLoc, syntax::NodeRole);
+        Builder.foldNode(Builder.getRange(DTL.getLocalSourceRange()), NS,
+                         nullptr);
+        return NS;
+      }
+      default:
+        return buildIdentifier(TL.getLocalSourceRange());
+      }
     }
     default:
-      llvm_unreachable("getChildKind() does not return this value");
+      // FIXME: Support Microsoft's __super
+      llvm::report_fatal_error("We don't yet support the __super specifier",
+                               true);
     }
   }
 
@@ -1019,12 +1013,16 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
   bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc QualifierLoc) {
     if (!QualifierLoc)
       return true;
-    for (auto It = QualifierLoc; It; It = It.getPrefix()) {
+    for (auto It = QualifierLoc; It; /**/) {
       auto *NS = buildNameSpecifier(It);
       if (!NS)
         return false;
       Builder.markChild(NS, syntax::NodeRole::ListElement);
       Builder.markChildToken(It.getEndLoc(), syntax::NodeRole::ListDelimiter);
+      if (TypeLoc TL = It.getAsTypeLoc())
+        It = TL.getPrefix();
+      else
+        It = It.getAsNamespaceAndPrefix().Prefix;
     }
     Builder.foldNode(Builder.getRange(QualifierLoc.getSourceRange()),
                      new (allocator()) syntax::NestedNameSpecifier,
@@ -1328,7 +1326,7 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
 
   // FIXME: Deleting the `TraverseParenTypeLoc` override doesn't change test
   // results. Find test coverage or remove it.
-  bool TraverseParenTypeLoc(ParenTypeLoc L) {
+  bool TraverseParenTypeLoc(ParenTypeLoc L, bool TraverseQualifier) {
     // We reverse order of traversal to get the proper syntax structure.
     if (!WalkUpFromParenTypeLoc(L))
       return false;
@@ -1391,7 +1389,8 @@ class BuildTreeVisitor : public RecursiveASTVisitor<BuildTreeVisitor> {
     return WalkUpFromFunctionTypeLoc(L);
   }
 
-  bool TraverseMemberPointerTypeLoc(MemberPointerTypeLoc L) {
+  bool TraverseMemberPointerTypeLoc(MemberPointerTypeLoc L,
+                                    bool TraverseQualifier) {
     // In the source code "void (Y::*mp)()" `MemberPointerTypeLoc` corresponds
     // to "Y::*" but it points to a `ParenTypeLoc` that corresponds to
     // "(Y::*mp)" We thus reverse the order of traversal to get the proper
diff --git a/clang/lib/Tooling/Transformer/RangeSelector.cpp b/clang/lib/Tooling/Transformer/RangeSelector.cpp
index 00f4611ddb5d4..171c786bc366f 100644
--- a/clang/lib/Tooling/Transformer/RangeSelector.cpp
+++ b/clang/lib/Tooling/Transformer/RangeSelector.cpp
@@ -222,14 +222,10 @@ RangeSelector transformer::name(std::string ID) {
       return CharSourceRange::getTokenRange(L, L);
     }
     if (const auto *T = Node.get<TypeLoc>()) {
-      TypeLoc Loc = *T;
-      auto ET = Loc.getAs<ElaboratedTypeLoc>();
-      if (!ET.isNull())
-        Loc = ET.getNamedTypeLoc();
-      if (auto SpecLoc = Loc.getAs<TemplateSpecializationTypeLoc>();
+      if (auto SpecLoc = T->getAs<TemplateSpecializationTypeLoc>();
           !SpecLoc.isNull())
         return CharSourceRange::getTokenRange(SpecLoc.getTemplateNameLoc());
-      return CharSourceRange::getTokenRange(Loc.getSourceRange());
+      return CharSourceRange::getTokenRange(T->getSourceRange());
     }
     return typeError(ID, Node.getNodeKind(),
                      "DeclRefExpr, NamedDecl, CXXCtorInitializer, TypeLoc");
diff --git a/clang/test/APINotes/Inputs/Frameworks/Simple.framework/Headers/Simple.apinotes b/clang/test/APINotes/Inputs/Frameworks/Simple.framework/Headers/Simple.apinotes
index 8c915bd8b5913..9ba38edb8a6f1 100644
--- a/clang/test/APINotes/Inputs/Frameworks/Simple.framework/Headers/Simple.apinotes
+++ b/clang/test/APINotes/Inputs/Frameworks/Simple.framework/Headers/Simple.apinotes
@@ -26,3 +26,6 @@ Classes:
       - Name:            scalarNewProperty
         PropertyKind:    Instance
         Nullability:     Scalar
+Typedefs:
+  - Name:            MyTypedef
+    SwiftConformsTo: Swift.Equatable
diff --git a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes
index c096822fb92b5..15c806842d08f 100644
--- a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes
+++ b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.apinotes
@@ -32,6 +32,9 @@ Tags:
   SwiftEscapable: false
 - Name: EscapableType
   SwiftEscapable: true
+- Name: NoncopyableWithDestroyType
+  SwiftCopyable: false
+  SwiftDestroyOp: NCDDestroy
 
 Functions:
   - Name: functionReturningFrt__
@@ -39,3 +42,7 @@ Functions:
     SwiftReturnOwnership: unretained
   - Name: functionReturningFrt_returns_retained
     SwiftReturnOwnership: retained
+Typedefs:
+  - Name: WrappedOptions
+    SwiftWrapper: struct
+    SwiftConformsTo: Swift.OptionSet
diff --git a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h
index 5f817ac89bdd3..978b4fbbb3b00 100644
--- a/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h
+++ b/clang/test/APINotes/Inputs/Headers/SwiftImportAs.h
@@ -29,3 +29,10 @@ struct OpaqueRefCountedType; // redeclaration
 
 inline void ORCRetain(struct OpaqueRefCountedType *x);
 inline void ORCRelease(struct OpaqueRefCountedType *x);
+
+typedef unsigned WrappedOptions;
+
+struct NoncopyableWithDestroyType {
+};
+
+void NCDDestroy(NoncopyableWithDestroyType instance);
diff --git a/clang/test/APINotes/swift-import-as.cpp b/clang/test/APINotes/swift-import-as.cpp
index 179170fbc0994..f5d08df7c6a1b 100644
--- a/clang/test/APINotes/swift-import-as.cpp
+++ b/clang/test/APINotes/swift-import-as.cpp
@@ -14,6 +14,8 @@
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter methodReturningFrt__ | FileCheck -check-prefix=CHECK-METHOD-RETURNING-FRT %s
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter methodReturningFrt_returns_unretained | FileCheck -check-prefix=CHECK-METHOD-RETURNING-FRT-UNRETAINED %s
 // RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter methodReturningFrt_returns_retained | FileCheck -check-prefix=CHECK-METHOD-RETURNING-FRT-RETAINED %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter WrappedOptions | FileCheck -check-prefix=CHECK-WRAPPED-OPTIONS %s
+// RUN: %clang_cc1 -fmodules -fblocks -fimplicit-module-maps -fmodules-cache-path=%t/ModulesCache -fdisable-module-hash -fapinotes-modules -I %S/Inputs/Headers %s -x c++ -ast-dump -ast-dump-filter NoncopyableWithDestroyType | FileCheck -check-prefix=CHECK-NONCOPYABLE-WITH-DESTROY %s
 
 #include <SwiftImportAs.h>
 
@@ -51,8 +53,8 @@
 // CHECK-OPAQUE-REF-COUNTED-NOT: SwiftAttrAttr {{.+}} <<invalid sloc>> "release:
 // CHECK-NON-COPYABLE: Dumping NonCopyableType:
 // CHECK-NON-COPYABLE-NEXT: CXXRecordDecl {{.+}} imported in SwiftImportAs {{.+}} struct NonCopyableType
-// CHECK-NON-COPYABLE: SwiftAttrAttr {{.+}} <<invalid sloc>> "conforms_to:MySwiftModule.MySwiftNonCopyableProtocol"
 // CHECK-NON-COPYABLE: SwiftAttrAttr {{.+}} <<invalid sloc>> "~Copyable"
+// CHECK-NON-COPYABLE: SwiftAttrAttr {{.+}} <<invalid sloc>> "conforms_to:MySwiftModule.MySwiftNonCopyableProtocol"
 
 // CHECK-COPYABLE: Dumping CopyableType:
 // CHECK-COPYABLE-NEXT: CXXRecordDecl {{.+}} imported in SwiftImportAs {{.+}} struct CopyableType
@@ -91,3 +93,13 @@
 // CHECK-METHOD-RETURNING-FRT-RETAINED: Dumping ImmortalRefType::methodReturningFrt_returns_retained:
 // CHECK-METHOD-RETURNING-FRT-RETAINED: CXXMethodDecl {{.+}} imported in SwiftImportAs methodReturningFrt_returns_retained 'ImmortalRefType *()'
 // CHECK-METHOD-RETURNING-FRT-RETAINED: `-SwiftAttrAttr {{.+}} "returns_retained"
+
+// CHECK-WRAPPED-OPTIONS: Dumping WrappedOptions
+// CHECK-WRAPPED-OPTIONS: TypedefDecl{{.*}}WrappedOptions 'unsigned int'
+// CHECK-WRAPPED-OPTIONS: SwiftNewTypeAttr {{.*}} swift_wrapper NK_Struct
+// CHECK-WRAPPED-OPTIONS: SwiftAttrAttr {{.*}} "conforms_to:Swift.OptionSet"
+
+// CHECK-NONCOPYABLE-WITH-DESTROY: Dumping NoncopyableWithDestroyType
+// CHECK-NONCOPYABLE-WITH-DESTROY: RecordDecl {{.*}}struct NoncopyableWithDestroyType
+// CHECK-NONCOPYABLE-WITH-DESTROY: SwiftAttrAttr {{.+}} "destroy:NCDDestroy"
+// CHECK-NONCOPYABLE-WITH-DESTROY: SwiftAttrAttr {{.+}} "~Copyable"
diff --git a/clang/test/APINotes/yaml-roundtrip.test b/clang/test/APINotes/yaml-roundtrip.test
index bcf84afda8df0..f69038ca828b1 100644
--- a/clang/test/APINotes/yaml-roundtrip.test
+++ b/clang/test/APINotes/yaml-roundtrip.test
@@ -24,7 +24,7 @@ CHECK-NEXT: 25c26
 CHECK-NEXT: <         Nullability:     S
 CHECK-NEXT: ---
 CHECK-NEXT: >         Nullability:     Unspecified
-CHECK-NEXT: 28c29,30
+CHECK-NEXT: 28c29
 CHECK-NEXT: <         Nullability:     Scalar
 CHECK-NEXT: ---
 CHECK-NEXT: >         Nullability:     Unspecified
diff --git a/clang/test/AST/ByteCode/arrays.cpp b/clang/test/AST/ByteCode/arrays.cpp
index 2dd51c2fa6711..22a4b41041eb3 100644
--- a/clang/test/AST/ByteCode/arrays.cpp
+++ b/clang/test/AST/ByteCode/arrays.cpp
@@ -779,3 +779,44 @@ namespace DiscardedSubScriptExpr {
     return true;
   }
 }
+
+namespace ZeroSizeArrayRead {
+  constexpr char str[0] = {};
+  constexpr  unsigned checksum(const char *s) {
+    unsigned result = 0;
+    for (const char *p = s; *p != '\0'; ++p) { // both-note {{read of dereferenced one-past-the-end pointer}}
+      result += *p;
+    }
+    return result;
+  }
+  constexpr unsigned C = checksum(str); // both-error {{must be initialized by a constant expression}} \
+                                        // both-note {{in call to}}
+
+  constexpr const char *p1 = &str[0];
+  constexpr const char *p2 = &str[1]; // both-error {{must be initialized by a constant expression}} \
+                                      // both-note {{cannot refer to element 1 of array of 0 elements in a constant expression}}
+
+  constexpr char s[] = {};
+  static_assert(s[0] == '0', ""); // both-error {{not an integral constant expression}} \
+                                  // both-note {{read of dereferenced one-past-the-end pointer}}
+}
+
+namespace FAM {
+  char *strchr(const char *, int);
+
+  struct A {
+    char n, a[2];
+  };
+  struct B {
+    int n;
+    struct A a[]; // both-note {{here}}
+  };
+
+  const struct B b = {0, {{1, {2, 3}}, {4, {5, 6}}}};
+  void foo(void) { int sch = 0 != strchr(b.a[1].a, '\0'); }
+
+  int foo2() {
+    struct B b = {0, {{1, {2, 3}}, {4, {5, 6}}}}; // both-error {{initialization of flexible array member is not allowed}}
+    return 1;
+  }
+}
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index e554ef41a8d0f..3277ef65a880b 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -21,6 +21,27 @@
 #error "huh?"
 #endif
 
+
+inline constexpr void* operator new(__SIZE_TYPE__, void* p) noexcept { return p; }
+namespace std {
+  using size_t = decltype(sizeof(0));
+  template<typename T> struct allocator {
+    constexpr T *allocate(size_t N) {
+      return (T*)__builtin_operator_new(sizeof(T) * N); // #alloc
+    }
+    constexpr void deallocate(void *p, __SIZE_TYPE__) {
+      __builtin_operator_delete(p);
+    }
+  };
+template<typename T, typename... Args>
+constexpr T* construct_at(T* p, Args&&... args) { return ::new((void*)p) T(static_cast<Args&&>(args)...); }
+
+  template<typename T>
+  constexpr void destroy_at(T* p) {
+    p->~T();
+  }
+}
+
 extern "C" {
   typedef decltype(sizeof(int)) size_t;
   extern size_t wcslen(const wchar_t *p);
@@ -1514,7 +1535,7 @@ namespace Memchr {
   extern struct Incomplete incomplete;
   static_assert(__builtin_memchr(&incomplete, 0, 0u) == nullptr);
   static_assert(__builtin_memchr(&incomplete, 0, 1u) == nullptr); // both-error {{not an integral constant}} \
-                                                                  // ref-note {{read of incomplete type 'struct Incomplete'}}
+                                                                  // both-note {{read of incomplete type 'struct Incomplete'}}
 
   const unsigned char &u1 = 0xf0;
   auto &&i1 = (const signed char []){-128};
@@ -1697,6 +1718,15 @@ namespace WMemMove {
                                                                      // both-note {{source of 'wmemmove' is nullptr}}
   static_assert(__builtin_wmemmove(null, &global, sizeof(wchar_t))); // both-error {{}} \
                                                                      // both-note {{destination of 'wmemmove' is nullptr}}
+
+  // Check that a pointer to an incomplete array is rejected.
+  constexpr int test_address_of_incomplete_array_type() { // both-error {{never produces a constant}}
+    extern int arr[];
+    __builtin_memmove(&arr, &arr, 4 * sizeof(arr[0])); // both-note 2{{cannot constant evaluate 'memmove' between objects of incomplete type 'int[]'}}
+    return arr[0] * 1000 + arr[1] * 100 + arr[2] * 10 + arr[3];
+  }
+  static_assert(test_address_of_incomplete_array_type() == 1234); // both-error {{constant}} \
+                                                                  // both-note {{in call}}
 }
 
 namespace Invalid {
@@ -1758,6 +1788,30 @@ namespace WithinLifetime {
     }
   } xstd; // both-error {{is not a constant expression}} \
           // both-note {{in call to}}
+
+  /// FIXME: We do not have per-element lifetime information for primitive arrays.
+  /// See https://github.com/llvm/llvm-project/issues/147528
+  consteval bool test_dynamic(bool read_after_deallocate) {
+    std::allocator<int> a;
+    int* p = a.allocate(1); // expected-note 2{{allocation performed here was not deallocated}}
+    // a.allocate starts the lifetime of an array,
+    // the complete object of *p has started its lifetime
+    if (__builtin_is_within_lifetime(p))
+      return false;
+    std::construct_at(p);
+    if (!__builtin_is_within_lifetime(p))
+      return false;
+    std::destroy_at(p);
+    if (__builtin_is_within_lifetime(p))
+      return false;
+    a.deallocate(p, 1);
+    if (read_after_deallocate)
+      __builtin_is_within_lifetime(p); // ref-note {{read of heap allocated object that has been deleted}}
+    return true;
+  }
+  static_assert(test_dynamic(false)); // expected-error {{not an integral constant expression}}
+  static_assert(test_dynamic(true)); // both-error {{not an integral constant expression}} \
+                                     // ref-note {{in call to}}
 }
 
 #ifdef __SIZEOF_INT128__
diff --git a/clang/test/AST/ByteCode/builtin-object-size-codegen.cpp b/clang/test/AST/ByteCode/builtin-object-size-codegen.cpp
new file mode 100644
index 0000000000000..f6ddbeb12b7a2
--- /dev/null
+++ b/clang/test/AST/ByteCode/builtin-object-size-codegen.cpp
@@ -0,0 +1,87 @@
+// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -triple x86_64-apple-darwin -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1                                         -triple x86_64-apple-darwin -emit-llvm -o - %s | FileCheck %s
+
+void foo() {
+  struct A { char buf[16]; };
+  struct B : A {};
+  struct C { int i; B bs[1]; } *c;
+
+  int gi;
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 false, i1 true, i1 false)
+  gi = __builtin_object_size(&c->bs[0], 0);
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 false, i1 true, i1 false)
+  gi = __builtin_object_size(&c->bs[0], 1);
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 true, i1 true, i1 false)
+  gi = __builtin_object_size(&c->bs[0], 2);
+  // CHECK: store i32 16
+  gi = __builtin_object_size(&c->bs[0], 3);
+}
+
+
+void foo2() {
+  struct A { int a; };
+  struct B { int b; };
+  struct C: public A, public B {};
+
+  C c;
+
+  int gi;
+  // CHECK: store i32 8
+  gi = __builtin_object_size(&c, 0);
+  // CHECK: store i32 8
+  gi = __builtin_object_size((A*)&c, 0);
+  // CHECK: store i32 4
+  gi = __builtin_object_size((B*)&c, 0);
+
+  // CHECK: store i32 8
+  gi = __builtin_object_size((char*)&c, 0);
+  // CHECK: store i32 8
+  gi = __builtin_object_size((char*)(A*)&c, 0);
+  // CHECK: store i32 4
+  gi = __builtin_object_size((char*)(B*)&c, 0);
+}
+
+
+typedef struct {
+  double c[0];
+  float f;
+} foofoo0_t;
+
+unsigned babar0(foofoo0_t *f) {
+  // CHECK: ret i32 0
+  return __builtin_object_size(f->c, 1);
+}
+
+void test2() {
+  struct A { char buf[16]; };
+  struct B : A {};
+  struct C { int i; B bs[1]; } *c;
+
+  int gi;
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 false, i1 true, i1 false)
+  gi = __builtin_object_size(&c->bs[0], 0);
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 false, i1 true, i1 false)
+  gi = __builtin_object_size(&c->bs[0], 1);
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 true, i1 true, i1 false)
+  gi = __builtin_object_size(&c->bs[0], 2);
+  // CHECK: store i32 16
+  gi = __builtin_object_size(&c->bs[0], 3);
+
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 false, i1 true, i1 false)
+  gi = __builtin_object_size((A*)&c->bs[0], 0);
+  // CHECK: store i32 16
+  gi = __builtin_object_size((A*)&c->bs[0], 1);
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 true, i1 true, i1 false)
+  gi = __builtin_object_size(&c->bs[0].buf[0], 2);
+  // CHECK: store i32 16
+  gi = __builtin_object_size(&c->bs[0].buf[0], 3);
+
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 false, i1 true, i1 false)
+  gi = __builtin_object_size(&c->bs[0].buf[0], 0);
+  // CHECK: store i32 16
+  gi = __builtin_object_size(&c->bs[0].buf[0], 1);
+  // CHECK: call i64 @llvm.objectsize.i64.p0(ptr %{{.*}}, i1 true, i1 true, i1 false)
+  gi = __builtin_object_size(&c->bs[0].buf[0], 2);
+  // CHECK: store i32 16
+  gi = __builtin_object_size(&c->bs[0].buf[0], 3);
+}
diff --git a/clang/test/AST/ByteCode/c.c b/clang/test/AST/ByteCode/c.c
index a7b1fe07f6d84..73469d7fd6cc4 100644
--- a/clang/test/AST/ByteCode/c.c
+++ b/clang/test/AST/ByteCode/c.c
@@ -329,3 +329,21 @@ void foo3 (void)
  void* x = 0;
  void* y = &*x;
 }
+
+static void *FooTable[1] = {
+    [0] = (void *[1]) { // 1
+        [0] = (void *[1]) { // 2
+            [0] = (void *[1]) {} // pedantic-warning {{use of an empty initializer}}
+        },
+    }
+};
+
+int strcmp(const char *, const char *); // all-note {{passing argument to parameter here}}
+#define S "\x01\x02\x03\x04\x05\x06\x07\x08"
+const char _str[] = {S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7]};
+const unsigned char _str2[] = {S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7]};
+const int compared = strcmp(_str, (const char *)_str2); // all-error {{initializer element is not a compile-time constant}}
+
+
+const int compared2 = strcmp(strcmp, _str); // all-warning {{incompatible pointer types}} \
+                                            // all-error {{initializer element is not a compile-time constant}}
diff --git a/clang/test/AST/ByteCode/constexpr-vectors.cpp b/clang/test/AST/ByteCode/constexpr-vectors.cpp
index f19adad3323f2..81ec6aac4bee3 100644
--- a/clang/test/AST/ByteCode/constexpr-vectors.cpp
+++ b/clang/test/AST/ByteCode/constexpr-vectors.cpp
@@ -15,7 +15,6 @@ using FourFloatsExtVec __attribute__((ext_vector_type(4))) = float;
 using FourDoublesExtVec __attribute__((ext_vector_type(4))) = double;
 using FourI128ExtVec __attribute__((ext_vector_type(4))) = __int128;
 
-
 // Next a series of tests to make sure these operations are usable in
 // constexpr functions. Template instantiations don't emit Winvalid-constexpr,
 // so we have to do these as macros.
@@ -875,3 +874,9 @@ void BoolVecUsage() {
   constexpr auto k = ~FourBoolsExtVec{true, false, true, false};
   static_assert(k[0] == false && k[1] == true && k[2] == false && k[3] == true, "");
 }
+
+using EightBoolsExtVec __attribute__((ext_vector_type(8))) = bool;
+void BoolVecShuffle() {
+  constexpr EightBoolsExtVec a = __builtin_shufflevector(
+      FourBoolsExtVec{}, FourBoolsExtVec{}, 0, 1, 2, 3, 4, 5, 6, 7);
+}
diff --git a/clang/test/AST/ByteCode/cxx11.cpp b/clang/test/AST/ByteCode/cxx11.cpp
index 7aecf23b0f149..08caca03b8056 100644
--- a/clang/test/AST/ByteCode/cxx11.cpp
+++ b/clang/test/AST/ByteCode/cxx11.cpp
@@ -330,3 +330,33 @@ namespace ReadMutableInCopyCtor {
                        // both-note {{read of mutable member 'u'}} \
                        // both-note {{in call to 'G(g1)'}}
 }
+
+namespace GH150709 {
+  struct C { };
+  struct D : C {
+    constexpr int f() const { return 1; };
+  };
+  struct E : C { };
+  struct F : D { };
+  struct G : E { };
+  
+  constexpr C c1, c2[2];
+  constexpr D d1, d2[2];
+  constexpr E e1, e2[2];
+  constexpr F f;
+  constexpr G g;
+
+  constexpr auto mp = static_cast<int (C::*)() const>(&D::f);
+
+  // sanity checks for fix of GH150709 (unchanged behavior)
+  static_assert((c1.*mp)() == 1, ""); // both-error {{constant expression}}
+  static_assert((d1.*mp)() == 1, "");
+  static_assert((f.*mp)() == 1, "");
+  static_assert((c2[0].*mp)() == 1, ""); // ref-error {{constant expression}}
+  static_assert((d2[0].*mp)() == 1, "");
+
+  // incorrectly undiagnosed before fix of GH150709
+  static_assert((e1.*mp)() == 1, ""); // ref-error {{constant expression}}
+  static_assert((e2[0].*mp)() == 1, ""); // ref-error {{constant expression}}
+  static_assert((g.*mp)() == 1, ""); // ref-error {{constant expression}}
+}
diff --git a/clang/test/AST/ByteCode/cxx20.cpp b/clang/test/AST/ByteCode/cxx20.cpp
index cc315d36456d5..67bf9a732d8b7 100644
--- a/clang/test/AST/ByteCode/cxx20.cpp
+++ b/clang/test/AST/ByteCode/cxx20.cpp
@@ -1084,3 +1084,19 @@ namespace Virtual {
   static_assert(l.b == 10);
   static_assert(l.c == 10);
 }
+
+namespace DiscardedTrivialCXXConstructExpr {
+  struct S {
+    constexpr S(int a) : x(a) {}
+    int x;
+  };
+
+  constexpr int foo(int x) { // ref-error {{never produces a constant expression}}
+    throw S(3); // both-note {{not valid in a constant expression}} \
+                // ref-note {{not valid in a constant expression}}
+    return 1;
+  }
+
+  constexpr int y = foo(12); // both-error {{must be initialized by a constant expression}} \
+                             // both-note {{in call to}}
+}
diff --git a/clang/test/AST/ByteCode/cxx23.cpp b/clang/test/AST/ByteCode/cxx23.cpp
index 45dd4f528aefb..2182d7c4e4325 100644
--- a/clang/test/AST/ByteCode/cxx23.cpp
+++ b/clang/test/AST/ByteCode/cxx23.cpp
@@ -309,15 +309,14 @@ namespace NonLiteralDtorInParam {
     ~NonLiteral() {} // all23-note {{declared here}}
   };
   constexpr int F2(NonLiteral N) { // all20-error {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} \
-                                   // ref23-note {{non-constexpr function '~NonLiteral' cannot be used in a constant expression}}
+                                   // all23-note {{non-constexpr function '~NonLiteral' cannot be used in a constant expression}}
     return 8;
   }
 
 
   void test() {
     NonLiteral L;
-    constexpr auto D = F2(L); // all23-error {{must be initialized by a constant expression}} \
-                              // expected23-note {{non-constexpr function '~NonLiteral' cannot be used in a constant expression}}
+    constexpr auto D = F2(L); // all23-error {{must be initialized by a constant expression}}
   }
 }
 
diff --git a/clang/test/AST/ByteCode/cxx2a.cpp b/clang/test/AST/ByteCode/cxx2a.cpp
index ac2f9883d49b6..744c99eaa1e68 100644
--- a/clang/test/AST/ByteCode/cxx2a.cpp
+++ b/clang/test/AST/ByteCode/cxx2a.cpp
@@ -225,3 +225,17 @@ namespace Dtor {
   static_assert(pseudo(true, false)); // both-error {{constant expression}} both-note {{in call}}
   static_assert(pseudo(false, true));
 }
+
+namespace GH150705 {
+  struct A { };
+  struct B : A { };
+  struct C : A {
+    constexpr virtual int foo() const { return 0; }
+  };
+
+  constexpr auto p = &C::foo;
+  constexpr auto q = static_cast<int (A::*)() const>(p);
+  constexpr B b;
+  constexpr const A& a = b;
+  constexpr auto x = (a.*q)(); // both-error {{constant expression}}
+}
diff --git a/clang/test/AST/ByteCode/cxx98.cpp b/clang/test/AST/ByteCode/cxx98.cpp
index c17049b01c1da..1150a4e41edc1 100644
--- a/clang/test/AST/ByteCode/cxx98.cpp
+++ b/clang/test/AST/ByteCode/cxx98.cpp
@@ -18,13 +18,12 @@ template struct C<cval>;
 
 /// FIXME: This example does not get properly diagnosed in the new interpreter.
 extern const int recurse1;
-const int recurse2 = recurse1; // both-note {{declared here}}
+const int recurse2 = recurse1; // ref-note {{declared here}}
 const int recurse1 = 1;
 int array1[recurse1];
 int array2[recurse2]; // ref-warning 2{{variable length array}} \
-                      // both-note {{initializer of 'recurse2' is not a constant expression}} \
-                      // expected-warning {{variable length array}} \
-                      // expected-error {{variable length array}}
+                      // ref-note {{initializer of 'recurse2' is not a constant expression}} \
+                      // expected-warning 2{{variable length array}}
 
 int NCI; // both-note {{declared here}}
 int NCIA[NCI]; // both-warning {{variable length array}} \
@@ -64,3 +63,20 @@ const int b = 1 / 0; // both-warning {{division by zero is undefined}} \
                      // both-note {{declared here}}
 _Static_assert(b, ""); // both-error {{not an integral constant expression}} \
                        // both-note {{initializer of 'b' is not a constant expression}}
+
+#ifdef __SIZEOF_INT128__
+/// The if statement tries an ltor conversion on an inactive union member.
+union InactiveReadUnion{
+  int a;
+  __uint128_t n;
+};
+
+int inactiveRead(void) {
+  const InactiveReadUnion U = {1};
+
+  if (U.n)
+    return 1;
+
+  return 0;
+}
+#endif
diff --git a/clang/test/AST/ByteCode/functions.cpp b/clang/test/AST/ByteCode/functions.cpp
index 363b6a58ac54d..01bf0a55bd19a 100644
--- a/clang/test/AST/ByteCode/functions.cpp
+++ b/clang/test/AST/ByteCode/functions.cpp
@@ -622,7 +622,7 @@ namespace FromIntegral {
   int a[(int)DoubleFn((void*)-1)()]; // both-error {{not allowed at file scope}} \
                                     // both-warning {{variable length arrays}}
   int b[(int)DoubleFn((void*)(-1 + 1))()]; // both-error {{not allowed at file scope}} \
-                                           // expected-note {{evaluates to a null function pointer}} \
+                                           // both-note {{evaluates to a null function pointer}} \
                                            // both-warning {{variable length arrays}}
 #endif
 }
@@ -708,3 +708,32 @@ namespace NoDiags {
     return true;
   }
 }
+
+namespace EnableIfWithTemporary {
+  struct A { ~A(); };
+  int &h() __attribute__((enable_if((A(), true), ""))); // both-warning {{clang extension}}
+}
+
+namespace LocalVarForParmVarDecl {
+  struct Iter {
+    void *p;
+  };
+  constexpr bool bar2(Iter A) {
+    return true;
+  }
+  constexpr bool bar(Iter A, bool b) {
+    if (b)
+      return true;
+
+    return bar(A, true);
+  }
+  constexpr int foo() {
+    return bar(Iter(), false);
+  }
+  static_assert(foo(), "");
+}
+
+namespace PtrPtrCast {
+  void foo() { ; }
+  void bar(int *a) { a = (int *)(void *)(foo); }
+}
diff --git a/clang/test/AST/ByteCode/lifetimes.cpp b/clang/test/AST/ByteCode/lifetimes.cpp
index 5c8d56291f079..d3b02d215b442 100644
--- a/clang/test/AST/ByteCode/lifetimes.cpp
+++ b/clang/test/AST/ByteCode/lifetimes.cpp
@@ -94,14 +94,25 @@ namespace CallScope {
     int n = 0;
     constexpr int f() const { return 0; }
   };
-  constexpr Q *out_of_lifetime(Q q) { return &q; } // both-warning {{address of stack}}
+  constexpr Q *out_of_lifetime(Q q) { return &q; } // both-warning {{address of stack}} \
+                                                   // expected-note 2{{declared here}}
   constexpr int k3 = out_of_lifetime({})->n; // both-error {{must be initialized by a constant expression}} \
-                                             // expected-note {{read of temporary whose lifetime has ended}} \
-                                             // expected-note {{temporary created here}} \
+                                             // expected-note {{read of variable whose lifetime has ended}} \
                                              // ref-note {{read of object outside its lifetime}}
 
   constexpr int k4 = out_of_lifetime({})->f(); // both-error {{must be initialized by a constant expression}} \
-                                               // expected-note {{member call on temporary whose lifetime has ended}} \
-                                               // expected-note {{temporary created here}} \
+                                               // expected-note {{member call on variable whose lifetime has ended}} \
                                                // ref-note {{member call on object outside its lifetime}}
 }
+
+namespace ExprDoubleDestroy {
+  template <typename T>
+  constexpr bool test() {
+    T{}.~T(); // both-note {{lifetime has already ended}}
+    return true;
+  }
+
+  struct S { int x; };
+  constexpr bool t = test<S>(); // both-error {{must be initialized by a constant expression}} \
+                                // both-note {{in call to}}
+}
diff --git a/clang/test/AST/ByteCode/lifetimes26.cpp b/clang/test/AST/ByteCode/lifetimes26.cpp
index a5203ae77bc13..c3163f8a562bf 100644
--- a/clang/test/AST/ByteCode/lifetimes26.cpp
+++ b/clang/test/AST/ByteCode/lifetimes26.cpp
@@ -17,8 +17,8 @@ namespace std {
 
 constexpr void *operator new(std::size_t, void *p) { return p; }
 namespace std {
-  template<typename T> constexpr T *construct(T *p) { return new (p) T; }
-  template<typename T> constexpr void destroy(T *p) { p->~T(); }
+  template<typename T> constexpr T *construct_at(T *p) { return new (p) T; }
+  template<typename T> constexpr void destroy_at(T *p) { p->~T(); }
 }
 
 constexpr bool foo() {
@@ -43,7 +43,24 @@ constexpr void destroy_pointer() {
   using T = int*;
   T p;
   p.~T();
-  std::construct(&p);
+  std::construct_at(&p);
 }
 static_assert((destroy_pointer(), true));
 
+
+namespace DestroyArrayElem {
+  /// This is proof that std::destroy_at'ing an array element
+  /// ends the lifetime of the entire array.
+  /// See https://github.com/llvm/llvm-project/issues/147528
+  /// Using destroy_at on array elements is currently a no-op due to this.
+  constexpr int test() {
+    int a[4] = {};
+
+    std::destroy_at(&a[3]);
+    int r = a[1];
+    std::construct_at(&a[3]);
+
+    return r;
+  }
+  static_assert(test() == 0);
+}
diff --git a/clang/test/AST/ByteCode/literals.cpp b/clang/test/AST/ByteCode/literals.cpp
index ddf1d2bebdbd0..5bc3f7f4c815c 100644
--- a/clang/test/AST/ByteCode/literals.cpp
+++ b/clang/test/AST/ByteCode/literals.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -Wno-vla -fms-extensions -std=c++11 -verify=expected,both %s
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -Wno-vla -fms-extensions -std=c++20 -verify=expected,both %s
-// RUN: %clang_cc1 -std=c++11 -fms-extensions -Wno-vla -verify=ref,both %s
-// RUN: %clang_cc1 -std=c++20 -fms-extensions -Wno-vla -verify=ref,both %s
+// RUN: %clang_cc1 -Wno-vla -fms-extensions -std=c++11 -fexperimental-new-constant-interpreter -verify=expected,both %s
+// RUN: %clang_cc1 -Wno-vla -fms-extensions -std=c++20 -fexperimental-new-constant-interpreter -verify=expected,both %s
+// RUN: %clang_cc1 -Wno-vla -fms-extensions -std=c++11                                         -verify=ref,both %s
+// RUN: %clang_cc1 -Wno-vla -fms-extensions -std=c++20                                         -verify=ref,both %s
 
 #define INT_MIN (~__INT_MAX__)
 #define INT_MAX __INT_MAX__
@@ -546,16 +546,14 @@ namespace IncDec {
              // expected-note 2{{increment of uninitialized}} \
              // expected-note {{read of uninitialized}}
       else
-        a++; // ref-note 2{{increment of uninitialized}} \
-             // expected-note 2{{increment of uninitialized}}
+        a++; // both-note 2{{increment of uninitialized}}
     } else {
       if (Pre)
         --a; // ref-note 3{{decrement of uninitialized}} \
              // expected-note 2{{decrement of uninitialized}} \
              // expected-note {{read of uninitialized}}
       else
-        a--; // ref-note 2{{decrement of uninitialized}} \
-             // expected-note 2{{decrement of uninitialized}}
+        a--; // both-note 2{{decrement of uninitialized}}
     }
     return 1;
   }
@@ -1430,3 +1428,10 @@ namespace OnePastEndCmp {
   constexpr const int *q = &s.a + 1;
   static_assert(p != q, "");
 }
+
+namespace ExternRedecl {
+  extern const int a;
+  constexpr const int *p = &a;
+  constexpr int a = 10;
+  static_assert(*p == 10, "");
+}
diff --git a/clang/test/AST/ByteCode/new-delete.cpp b/clang/test/AST/ByteCode/new-delete.cpp
index c5f1878c41734..3f0e928c7664e 100644
--- a/clang/test/AST/ByteCode/new-delete.cpp
+++ b/clang/test/AST/ByteCode/new-delete.cpp
@@ -82,8 +82,7 @@ static_assert(noInit() == 0, "");
 /// Try to delete a pointer that hasn't been heap allocated.
 constexpr int notHeapAllocated() { // both-error {{never produces a constant expression}}
   int A = 0; // both-note 2{{declared here}}
-  delete &A; // ref-note 2{{delete of pointer '&A' that does not point to a heap-allocated object}} \
-             // expected-note 2{{delete of pointer '&A' that does not point to a heap-allocated object}}
+  delete &A; // both-note 2{{delete of pointer '&A' that does not point to a heap-allocated object}}
 
   return 1;
 }
@@ -374,8 +373,7 @@ namespace delete_random_things {
   static_assert((delete &(new A)->n, true)); // both-error {{}} \
                                              // both-note {{delete of pointer to subobject }}
   static_assert((delete (new int + 1), true)); // both-error {{}} \
-                                               // ref-note {{delete of pointer '&{*new int#0} + 1' that does not point to complete object}} \
-                                               // expected-note {{delete of pointer '&{*new int#1} + 1' that does not point to complete object}}
+                                               // both-note {{delete of pointer '&{*new int#0} + 1' that does not point to complete object}}
   static_assert((delete[] (new int[3] + 1), true)); // both-error {{}} \
                                                     // both-note {{delete of pointer to subobject}}
   static_assert((delete &(int&)(int&&)0, true)); // both-error {{}} \
diff --git a/clang/test/AST/ByteCode/records.cpp b/clang/test/AST/ByteCode/records.cpp
index 5ca3e2d12e2df..48cf81182c1de 100644
--- a/clang/test/AST/ByteCode/records.cpp
+++ b/clang/test/AST/ByteCode/records.cpp
@@ -1,11 +1,11 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++14 -verify=expected,both %s
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++17 -verify=expected,both %s
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++17 -triple i686 -verify=expected,both %s
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -std=c++20 -verify=expected,both %s
-// RUN: %clang_cc1 -verify=ref,both -std=c++14 %s
-// RUN: %clang_cc1 -verify=ref,both -std=c++17 %s
-// RUN: %clang_cc1 -verify=ref,both -std=c++17 -triple i686 %s
-// RUN: %clang_cc1 -verify=ref,both -std=c++20 %s
+// RUN: %clang_cc1 -std=c++14 -verify=expected,both              %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++17 -verify=expected,both              %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++17 -verify=expected,both -triple i686 %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++20 -verify=expected,both              %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -std=c++14 -verify=ref,both                   %s
+// RUN: %clang_cc1 -std=c++17 -verify=ref,both                   %s
+// RUN: %clang_cc1 -std=c++17 -verify=ref,both -triple i686      %s
+// RUN: %clang_cc1 -std=c++20 -verify=ref,both                   %s
 
 /// Used to crash.
 struct Empty {};
diff --git a/clang/test/AST/ByteCode/typeid.cpp b/clang/test/AST/ByteCode/typeid.cpp
index 5be5604016db5..179a66fd7fd0a 100644
--- a/clang/test/AST/ByteCode/typeid.cpp
+++ b/clang/test/AST/ByteCode/typeid.cpp
@@ -13,7 +13,12 @@ struct __type_info_implementations {
   typedef __unique_impl __impl;
 };
 
-class type_info {
+class __pointer_type_info {
+public:
+  int __flags = 0;
+};
+
+class type_info : public __pointer_type_info {
 protected:
   typedef __type_info_implementations::__impl __impl;
   __impl::__type_name_t __type_name;
@@ -40,3 +45,10 @@ constexpr bool test() {
   return true;
 }
 static_assert(test());
+
+int dontcrash() {
+  auto& pti = static_cast<const std::__pointer_type_info&>(
+      typeid(int)
+  );
+  return pti.__flags == 0 ? 1 : 0;
+}
diff --git a/clang/test/AST/ByteCode/unions.cpp b/clang/test/AST/ByteCode/unions.cpp
index 8ed76258b879a..6bccbda23f468 100644
--- a/clang/test/AST/ByteCode/unions.cpp
+++ b/clang/test/AST/ByteCode/unions.cpp
@@ -966,3 +966,15 @@ namespace AddressComparison {
   static_assert(&U2.a[0] != &U2.b[1]);
   static_assert(&U2.a[0] == &U2.b[1]); // both-error {{failed}}
 }
+
+#if __cplusplus >= 202002L
+namespace UnionMemberOnePastEnd {
+  constexpr bool b() {
+    union  {
+      int p;
+    };
+    return &p == (&p + 1);
+  }
+  static_assert(!b());
+}
+#endif
diff --git a/clang/test/AST/ByteCode/vectors.cpp b/clang/test/AST/ByteCode/vectors.cpp
index a04b678a623a1..091caf8c9a275 100644
--- a/clang/test/AST/ByteCode/vectors.cpp
+++ b/clang/test/AST/ByteCode/vectors.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=expected,both %s
-// RUN: %clang_cc1 -verify=ref,both %s
+// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=expected,both -flax-vector-conversions=none %s
+// RUN: %clang_cc1 -verify=ref,both -flax-vector-conversions=none %s
 
 typedef int __attribute__((vector_size(16))) VI4;
 constexpr VI4 A = {1,2,3,4};
@@ -58,7 +58,7 @@ namespace Vector {
 
 namespace {
   typedef float __attribute__((vector_size(16))) VI42;
-  constexpr VI42 A2 = A;
+  constexpr VI42 A2 = {1.f, 2.f, 3.f, 4.f};
 }
 
 namespace BoolToSignedIntegralCast{
diff --git a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
index 1c8b9c10f5a98..76de39ca4b60e 100644
--- a/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/StructuredBuffers-AST.hlsl
@@ -91,21 +91,21 @@ RESOURCE<float> Buffer;
 
 // Default constructor
 
-// CHECK: CXXConstructorDecl {{.*}} [[RESOURCE]]<element_type> 'void ()' inline
+// CHECK: CXXConstructorDecl {{.*}} hlsl::[[RESOURCE]]<element_type> 'void ()' inline
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: BinaryOperator {{.*}} '='
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: CallExpr {{.*}} '__hlsl_resource_t
 // CHECK-NEXT: ImplicitCastExpr {{.*}} <BuiltinFnToFnPtr>
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_uninitializedhandle'
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: AlwaysInlineAttr
 
 // Constructor from binding
 
-// CHECK: CXXConstructorDecl {{.*}} [[RESOURCE]]<element_type> 'void (unsigned int, unsigned int, int, unsigned int, const char *)' inline
+// CHECK: CXXConstructorDecl {{.*}} hlsl::[[RESOURCE]]<element_type> 'void (unsigned int, unsigned int, int, unsigned int, const char *)' inline
 // CHECK-NEXT: ParmVarDecl {{.*}} registerNo 'unsigned int'
 // CHECK-NEXT: ParmVarDecl {{.*}} spaceNo 'unsigned int'
 // CHECK-NEXT: ParmVarDecl {{.*}} range 'int'
@@ -114,12 +114,12 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: CompoundStmt {{.*}}
 // CHECK-NEXT: BinaryOperator {{.*}} '='
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: CallExpr {{.*}} '__hlsl_resource_t
 // CHECK-NEXT: ImplicitCastExpr {{.*}} <BuiltinFnToFnPtr>
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_handlefrombinding'
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'registerNo' 'unsigned int'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'spaceNo' 'unsigned int'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'int' ParmVar {{.*}} 'range' 'int'
@@ -129,7 +129,7 @@ RESOURCE<float> Buffer;
 
 // Constructor from implicit binding
 
-// CHECK: CXXConstructorDecl {{.*}} [[RESOURCE]]<element_type> 'void (unsigned int, int, unsigned int, unsigned int, const char *)' inline
+// CHECK: CXXConstructorDecl {{.*}} hlsl::[[RESOURCE]]<element_type> 'void (unsigned int, int, unsigned int, unsigned int, const char *)' inline
 // CHECK-NEXT: ParmVarDecl {{.*}} spaceNo 'unsigned int'
 // CHECK-NEXT: ParmVarDecl {{.*}} range 'int'
 // CHECK-NEXT: ParmVarDecl {{.*}} index 'unsigned int'
@@ -138,12 +138,12 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: CompoundStmt {{.*}}
 // CHECK-NEXT: BinaryOperator {{.*}} '='
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: CallExpr {{.*}} '__hlsl_resource_t
 // CHECK-NEXT: ImplicitCastExpr {{.*}} <BuiltinFnToFnPtr>
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_handlefromimplicitbinding'
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'spaceNo' 'unsigned int'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'int' ParmVar {{.*}} 'range' 'int'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'index' 'unsigned int'
@@ -166,7 +166,7 @@ RESOURCE<float> Buffer;
 // CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-SUBSCRIPT-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SUBSCRIPT-SAME: ' lvalue .__handle {{.*}}
-// CHECK-SUBSCRIPT-NEXT: CXXThisExpr {{.*}} 'const [[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-SUBSCRIPT-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-SUBSCRIPT-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'Index' 'unsigned int'
 // CHECK-SUBSCRIPT-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
 
@@ -183,7 +183,7 @@ RESOURCE<float> Buffer;
 // CHECK-SUBSCRIPT-UAV-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-SUBSCRIPT-UAV-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SUBSCRIPT-UAV-SAME: ' lvalue .__handle {{.*}}
-// CHECK-SUBSCRIPT-UAV-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-SUBSCRIPT-UAV-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-SUBSCRIPT-UAV-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'Index' 'unsigned int'
 // CHECK-SUBSCRIPT-UAV-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
 
@@ -204,7 +204,7 @@ RESOURCE<float> Buffer;
 // CHECK-LOAD-SAME{LITERAL}: [[hlsl::resource_class(
 // CHECK-LOAD-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-LOAD-SAME: ' lvalue .__handle {{.*}}
-// CHECK-LOAD-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-LOAD-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-LOAD-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'Index' 'unsigned int'
 // CHECK-LOAD-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
 
@@ -220,7 +220,7 @@ RESOURCE<float> Buffer;
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-COUNTER-NEXT: CXXThisExpr {{.*}} 'RWStructuredBuffer<element_type>' lvalue implicit this
+// CHECK-COUNTER-NEXT: CXXThisExpr {{.*}} 'hlsl::RWStructuredBuffer<element_type>' lvalue implicit this
 // CHECK-COUNTER-NEXT: IntegerLiteral {{.*}} 'int' 1
 // CHECK-COUNTER-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
 
@@ -236,7 +236,7 @@ RESOURCE<float> Buffer;
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-COUNTER-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-COUNTER-NEXT: CXXThisExpr {{.*}} 'RWStructuredBuffer<element_type>' lvalue implicit this
+// CHECK-COUNTER-NEXT: CXXThisExpr {{.*}} 'hlsl::RWStructuredBuffer<element_type>' lvalue implicit this
 // CHECK-COUNTER-NEXT: IntegerLiteral {{.*}} 'int' -1
 // CHECK-COUNTER-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
 
@@ -254,7 +254,7 @@ RESOURCE<float> Buffer;
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-APPEND-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-APPEND-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-APPEND-NEXT: CallExpr {{.*}} 'unsigned int'
 // CHECK-APPEND-NEXT: ImplicitCastExpr {{.*}} <BuiltinFnToFnPtr>
 // CHECK-APPEND-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
@@ -262,7 +262,7 @@ RESOURCE<float> Buffer;
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-APPEND-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-APPEND-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-APPEND-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-APPEND-NEXT: IntegerLiteral {{.*}} 'int' 1
 // CHECK-APPEND-NEXT: DeclRefExpr {{.*}} 'element_type' ParmVar {{.*}} 'value' 'element_type'
 
@@ -279,7 +279,7 @@ RESOURCE<float> Buffer;
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-CONSUME-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-CONSUME-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-CONSUME-NEXT: CallExpr {{.*}} 'unsigned int'
 // CHECK-CONSUME-NEXT: ImplicitCastExpr {{.*}} <BuiltinFnToFnPtr>
 // CHECK-CONSUME-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_buffer_update_counter' 'unsigned int (...) noexcept'
@@ -287,7 +287,7 @@ RESOURCE<float> Buffer;
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::raw_buffer]]
 // CHECK-CONSUME-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]' lvalue .__handle
-// CHECK-CONSUME-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-CONSUME-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-CONSUME-NEXT: IntegerLiteral {{.*}} 'int' -1
 
 // CHECK: ClassTemplateSpecializationDecl {{.*}} class [[RESOURCE]] definition
diff --git a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
index d6b88e276762e..936a060a305cd 100644
--- a/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
+++ b/clang/test/AST/HLSL/TypedBuffers-AST.hlsl
@@ -66,21 +66,21 @@ RESOURCE<float> Buffer;
 
 // Default constructor
 
-// CHECK: CXXConstructorDecl {{.*}} [[RESOURCE]]<element_type> 'void ()' inline
+// CHECK: CXXConstructorDecl {{.*}} hlsl::[[RESOURCE]]<element_type> 'void ()' inline
 // CHECK-NEXT: CompoundStmt
 // CHECK-NEXT: BinaryOperator {{.*}} '='
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: CallExpr {{.*}} '__hlsl_resource_t
 // CHECK-NEXT: ImplicitCastExpr {{.*}} <BuiltinFnToFnPtr>
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_uninitializedhandle'
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: AlwaysInlineAttr
 
 // Constructor from binding
 
-// CHECK: CXXConstructorDecl {{.*}} [[RESOURCE]]<element_type> 'void (unsigned int, unsigned int, int, unsigned int, const char *)' inline
+// CHECK: CXXConstructorDecl {{.*}} hlsl::[[RESOURCE]]<element_type> 'void (unsigned int, unsigned int, int, unsigned int, const char *)' inline
 // CHECK-NEXT: ParmVarDecl {{.*}} registerNo 'unsigned int'
 // CHECK-NEXT: ParmVarDecl {{.*}} spaceNo 'unsigned int'
 // CHECK-NEXT: ParmVarDecl {{.*}} range 'int'
@@ -89,12 +89,12 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: CompoundStmt {{.*}}
 // CHECK-NEXT: BinaryOperator {{.*}} '='
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: CallExpr {{.*}} '__hlsl_resource_t
 // CHECK-NEXT: ImplicitCastExpr {{.*}} <BuiltinFnToFnPtr>
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_handlefrombinding'
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'registerNo' 'unsigned int'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'spaceNo' 'unsigned int'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'int' ParmVar {{.*}} 'range' 'int'
@@ -104,7 +104,7 @@ RESOURCE<float> Buffer;
 
 // Constructor from implicit binding
 
-// CHECK: CXXConstructorDecl {{.*}} [[RESOURCE]]<element_type> 'void (unsigned int, int, unsigned int, unsigned int, const char *)' inline
+// CHECK: CXXConstructorDecl {{.*}} hlsl::[[RESOURCE]]<element_type> 'void (unsigned int, int, unsigned int, unsigned int, const char *)' inline
 // CHECK-NEXT: ParmVarDecl {{.*}} spaceNo 'unsigned int'
 // CHECK-NEXT: ParmVarDecl {{.*}} range 'int'
 // CHECK-NEXT: ParmVarDecl {{.*}} index 'unsigned int'
@@ -113,12 +113,12 @@ RESOURCE<float> Buffer;
 // CHECK-NEXT: CompoundStmt {{.*}}
 // CHECK-NEXT: BinaryOperator {{.*}} '='
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: CallExpr {{.*}} '__hlsl_resource_t
 // CHECK-NEXT: ImplicitCastExpr {{.*}} <BuiltinFnToFnPtr>
 // CHECK-NEXT: DeclRefExpr {{.*}} '<builtin fn type>' Function {{.*}} '__builtin_hlsl_resource_handlefromimplicitbinding'
 // CHECK-NEXT: MemberExpr {{.*}} lvalue .__handle
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'spaceNo' 'unsigned int'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'int' ParmVar {{.*}} 'range' 'int'
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}} 'index' 'unsigned int'
@@ -141,7 +141,7 @@ RESOURCE<float> Buffer;
 // CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' lvalue .__handle {{.*}}
-// CHECK-NEXT: CXXThisExpr {{.*}} 'const [[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'const hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}}  'Index' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
 
@@ -157,7 +157,7 @@ RESOURCE<float> Buffer;
 // CHECK-UAV-SAME{LITERAL}: [[hlsl::resource_class(UAV)]]
 // CHECK-UAV-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-UAV-SAME: ' lvalue .__handle {{.*}}
-// CHECK-UAV-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-UAV-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-UAV-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}}  'Index' 'unsigned int'
 // CHECK-UAV-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
 
@@ -176,7 +176,7 @@ RESOURCE<float> Buffer;
 // CHECK-SRV-SAME{LITERAL}: [[hlsl::resource_class(SRV)]]
 // CHECK-SAME{LITERAL}: [[hlsl::contained_type(element_type)]]
 // CHECK-SAME: ' lvalue .__handle {{.*}}
-// CHECK-NEXT: CXXThisExpr {{.*}} '[[RESOURCE]]<element_type>' lvalue implicit this
+// CHECK-NEXT: CXXThisExpr {{.*}} 'hlsl::[[RESOURCE]]<element_type>' lvalue implicit this
 // CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int' ParmVar {{.*}}  'Index' 'unsigned int'
 // CHECK-NEXT: AlwaysInlineAttr {{.*}} Implicit always_inline
 
diff --git a/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl b/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl
index 2f08531000f32..1c5e06789c3e4 100644
--- a/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl
+++ b/clang/test/AST/HLSL/cbuffer_and_namespaces.hlsl
@@ -63,9 +63,9 @@ namespace NS2 {
   // CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 2]]:11 cbuffer CB2
   // CHECK: HLSLResourceClassAttr {{.*}} Implicit CBuffer
   cbuffer CB2 {
-    // CHECK: VarDecl {{.*}} foo0 'hlsl_constant ::Foo':'hlsl_constant Foo'
+    // CHECK: VarDecl {{.*}} foo0 'hlsl_constant ::Foo'
     ::Foo foo0;
-    // CHECK: VarDecl {{.*}} foo1 'hlsl_constant Foo':'hlsl_constant NS2::Foo'
+    // CHECK: VarDecl {{.*}} foo1 'hlsl_constant Foo'
     Foo foo1;
     // CHECK: VarDecl {{.*}} foo2 'hlsl_constant NS1::Foo'
     NS1::Foo foo2;
diff --git a/clang/test/AST/HLSL/resource_binding_attr.hlsl b/clang/test/AST/HLSL/resource_binding_attr.hlsl
index c073cd4dc1476..c6d93b991fbfc 100644
--- a/clang/test/AST/HLSL/resource_binding_attr.hlsl
+++ b/clang/test/AST/HLSL/resource_binding_attr.hlsl
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -finclude-default-header -ast-dump -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -finclude-default-header -ast-dump -o - %s | FileCheck %s -check-prefixes=CHECK,DXIL
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan-library -finclude-default-header -ast-dump -o - %s | FileCheck %s -check-prefixes=CHECK,SPV
 
 // CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 4]]:9 cbuffer CB
 // CHECK-NEXT: HLSLResourceClassAttr {{.*}} Implicit CBuffer
@@ -34,6 +35,10 @@ RWBuffer<float> UAV1 : register(u2), UAV2 : register(u4);
 // CHECK: HLSLResourceBindingAttr {{.*}} "" "space5"
 RWBuffer<float> UAV3 : register(space5);
 
+// CHECK: VarDecl {{.*}} UAV_Array 'RWBuffer<float>[10]'
+// CHECK: HLSLResourceBindingAttr {{.*}} "u10" "space6"
+RWBuffer<float> UAV_Array[10] : register(u10, space6);
+
 //
 // Default constants ($Globals) layout annotations
 
@@ -56,3 +61,44 @@ struct S {
 // CHECK: VarDecl {{.*}} s 'hlsl_constant S'
 // CHECK: HLSLResourceBindingAttr {{.*}} "c10" "space0
 S s : register(c10);
+
+//
+// Implicit binding
+
+// Constant buffers should have implicit binding attribute added by SemaHLSL,
+// unless the target is SPIR-V and there is [[vk::binding]] attribute.
+// CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 3]]:9 cbuffer CB2
+// CHECK-NEXT: HLSLResourceClassAttr {{.*}} Implicit CBuffer
+// CHECK-NEXT: HLSLResourceBindingAttr {{.*}} Implicit "" "0"
+cbuffer CB2 {
+  float4 c;
+}
+
+// CHECK: HLSLBufferDecl {{.*}} line:[[# @LINE + 7]]:9 cbuffer CB3
+// CHECK-NEXT: HLSLResourceClassAttr {{.*}} Implicit CBuffer
+// DXIL: HLSLResourceBindingAttr {{.*}} Implicit
+// DXIL-NOT: HLSLVkBindingAttr
+// SPV: HLSLVkBindingAttr {{.*}} 1 0
+// SPV-NOT: HLSLResourceBindingAttr {{.*}} Implicit
+[[vk::binding(1)]]
+cbuffer CB3 {
+  float2 d;
+}
+
+// Resource arrays should have implicit binding attribute added by SemaHLSL,
+// unless the target is SPIR-V and there is [[vk::binding]] attribute.
+// CHECK: VarDecl {{.*}} SB 'StructuredBuffer<float>[10]'
+// CHECK: HLSLResourceBindingAttr {{.*}} Implicit "" "0"
+StructuredBuffer<float> SB[10];
+
+// CHECK: VarDecl {{.*}} SB2 'StructuredBuffer<float>[10]'
+// DXIL: HLSLResourceBindingAttr {{.*}} Implicit
+// DXIL-NOT: HLSLVkBindingAttr
+// SPV: HLSLVkBindingAttr {{.*}} 2 0
+// SPV-NOT: HLSLResourceBindingAttr {{.*}} Implicit
+[[vk::binding(2)]]
+StructuredBuffer<float> SB2[10];
+
+// $Globals should have implicit binding attribute added by SemaHLSL
+// CHECK: HLSLBufferDecl {{.*}} implicit cbuffer $Globals
+// CHECK: HLSLResourceBindingAttr {{.*}} Implicit "" "0"
diff --git a/clang/test/AST/HLSL/vector-constructors.hlsl b/clang/test/AST/HLSL/vector-constructors.hlsl
index 31d8dd0fdac8e..fd43a7dcbfcca 100644
--- a/clang/test/AST/HLSL/vector-constructors.hlsl
+++ b/clang/test/AST/HLSL/vector-constructors.hlsl
@@ -88,10 +88,10 @@ void entry() {
 // CHECK-NEXT: InitListExpr
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'float' <LValueToRValue>
 // CHECK-NEXT: MemberExpr {{.*}} 'float' lvalue .f {{.*}}
-// CHECK-NEXT: DeclRefExpr {{.*}} 'struct S':'S' lvalue Var {{.*}} 's' 'struct S':'S'
+// CHECK-NEXT: DeclRefExpr {{.*}} 'struct S' lvalue Var {{.*}} 's' 'struct S'
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'float' <LValueToRValue>
 // CHECK-NEXT: MemberExpr {{.*}} 'float' lvalue .f {{.*}}
-// CHECK-NEXT: DeclRefExpr {{.*}} 'struct S':'S' lvalue Var {{.*}} 's' 'struct S':'S'
+// CHECK-NEXT: DeclRefExpr {{.*}} 'struct S' lvalue Var {{.*}} 's' 'struct S'
 
   struct T {
     operator float() const { return 1.0f; }
@@ -105,12 +105,12 @@ void entry() {
 // CHECK-NEXT: CXXMemberCallExpr {{.*}} 'float'
 // CHECK-NEXT: MemberExpr {{.*}} '<bound member function type>' .operator float {{.*}}
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'const T' lvalue <NoOp>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'struct T':'T' lvalue Var {{.*}} 't' 'struct T':'T'
+// CHECK-NEXT: DeclRefExpr {{.*}} 'struct T' lvalue Var {{.*}} 't' 'struct T'
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'float' <UserDefinedConversion>
 // CHECK-NEXT: CXXMemberCallExpr {{.*}} 'float'
 // CHECK-NEXT: MemberExpr {{.*}} '<bound member function type>' .operator float {{.*}}
 // CHECK-NEXT: ImplicitCastExpr {{.*}} 'const T' lvalue <NoOp>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'struct T':'T' lvalue Var {{.*}} 't' 'struct T':'T'
+// CHECK-NEXT: DeclRefExpr {{.*}} 'struct T' lvalue Var {{.*}} 't' 'struct T'
 
   typedef float2 second_level_of_typedefs;
   second_level_of_typedefs foo6 = float2(1.0f, 2.0f);
diff --git a/clang/test/AST/arm-mfp8.cpp b/clang/test/AST/arm-mfp8.cpp
index 195c734fc753e..b1fa04a3a79c1 100644
--- a/clang/test/AST/arm-mfp8.cpp
+++ b/clang/test/AST/arm-mfp8.cpp
@@ -49,7 +49,7 @@ class C1 {
   }
 };
 
-//CHECK:       | |-CXXRecordDecl {{.*}} referenced class C1
+//CHECK:       | |-CXXRecordDecl {{.*}} class C1
 //CHECK-NEXT:  | |-FieldDecl {{.*}} f1c '__mfp8'
 //CHECK-NEXT:  | |-VarDecl {{.*}} f2c 'const __mfp8' static
 //CHECK-NEXT:  | |-FieldDecl {{.*}} f3c 'volatile __mfp8'
diff --git a/clang/test/AST/ast-dump-color.cpp b/clang/test/AST/ast-dump-color.cpp
index 87797f6bffc5b..2e60e760283de 100644
--- a/clang/test/AST/ast-dump-color.cpp
+++ b/clang/test/AST/ast-dump-color.cpp
@@ -82,15 +82,15 @@ struct Invalid {
 //CHECK: {{^}}[[Blue]]| | `-[[RESET]][[GREEN]]ParmVarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:33[[RESET]]> [[Yellow]]col:33[[RESET]] [[Green]]'const Mutex &'[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| `-[[RESET]][[GREEN]]CXXConstructorDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:33[[RESET]]> [[Yellow]]col:33[[RESET]] implicit constexpr[[CYAN]] Mutex[[RESET]] [[Green]]'void (Mutex &&)'[[RESET]] inline{{ .*$}}
 //CHECK: {{^}}[[Blue]]|   `-[[RESET]][[GREEN]]ParmVarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:33[[RESET]]> [[Yellow]]col:33[[RESET]] [[Green]]'Mutex &&'[[RESET]]{{$}}
-//CHECK: {{^}}[[Blue]]|-[[RESET]][[GREEN]]VarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:1[[RESET]], [[Yellow]]line:25:3[[RESET]]> [[Yellow]]col:3[[RESET]] referenced[[CYAN]] mu1[[RESET]] [[Green]]'class Mutex':'Mutex'[[RESET]]
-//CHECK: {{^}}[[Blue]]| `-[[RESET]][[MAGENTA]]CXXConstructExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:3[[RESET]]> [[Green]]'class Mutex':'Mutex'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]] [[Green]]'void () noexcept'[[RESET]]{{$}}
-//CHECK: {{^}}[[Blue]]|-[[RESET]][[GREEN]]VarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:18:1[[RESET]], [[Yellow]]line:25:8[[RESET]]> [[Yellow]]col:8[[RESET]][[CYAN]] mu2[[RESET]] [[Green]]'class Mutex':'Mutex'[[RESET]]
-//CHECK: {{^}}[[Blue]]| `-[[RESET]][[MAGENTA]]CXXConstructExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:8[[RESET]]> [[Green]]'class Mutex':'Mutex'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]] [[Green]]'void () noexcept'[[RESET]]{{$}}
+//CHECK: {{^}}[[Blue]]|-[[RESET]][[GREEN]]VarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:1[[RESET]], [[Yellow]]line:25:3[[RESET]]> [[Yellow]]col:3[[RESET]] referenced[[CYAN]] mu1[[RESET]] [[Green]]'class Mutex'[[RESET]]
+//CHECK: {{^}}[[Blue]]| `-[[RESET]][[MAGENTA]]CXXConstructExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:3[[RESET]]> [[Green]]'class Mutex'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]] [[Green]]'void () noexcept'[[RESET]]{{$}}
+//CHECK: {{^}}[[Blue]]|-[[RESET]][[GREEN]]VarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:18:1[[RESET]], [[Yellow]]line:25:8[[RESET]]> [[Yellow]]col:8[[RESET]][[CYAN]] mu2[[RESET]] [[Green]]'class Mutex'[[RESET]]
+//CHECK: {{^}}[[Blue]]| `-[[RESET]][[MAGENTA]]CXXConstructExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:8[[RESET]]> [[Green]]'class Mutex'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]] [[Green]]'void () noexcept'[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]|-[[RESET]][[GREEN]]VarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:26:1[[RESET]], [[Yellow]]col:5[[RESET]]> [[Yellow]]col:5[[RESET]][[CYAN]] TestExpr[[RESET]] [[Green]]'int'[[RESET]]
 //CHECK: {{^}}[[Blue]]| `-[[RESET]][[BLUE]]GuardedByAttr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:29[[RESET]], [[Yellow]]col:43[[RESET]]>{{$}}
-//CHECK: {{^}}[[Blue]]|   `-[[RESET]][[MAGENTA]]DeclRefExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:40[[RESET]]> [[Green]]'class Mutex':'Mutex'[[RESET]][[Cyan]] lvalue[[RESET]][[Cyan]][[RESET]] [[GREEN]]Var[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]][[CYAN]] 'mu1'[[RESET]] [[Green]]'class Mutex':'Mutex'[[RESET]] non_odr_use_unevaluated{{$}}
+//CHECK: {{^}}[[Blue]]|   `-[[RESET]][[MAGENTA]]DeclRefExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:40[[RESET]]> [[Green]]'class Mutex'[[RESET]][[Cyan]] lvalue[[RESET]][[Cyan]][[RESET]] [[GREEN]]Var[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]][[CYAN]] 'mu1'[[RESET]] [[Green]]'class Mutex'[[RESET]] non_odr_use_unevaluated{{$}}
 //CHECK: {{^}}[[Blue]]|-[[RESET]][[GREEN]]CXXRecordDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:28:1[[RESET]], [[Yellow]]line:30:1[[RESET]]> [[Yellow]]line:28:8[[RESET]] struct[[CYAN]] Invalid[[RESET]] definition
-//CHECK: {{^}}[[Blue]]| |-[[RESET]][[GREEN]]CXXRecordDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:1[[RESET]], [[Yellow]]col:8[[RESET]]> [[Yellow]]col:8[[RESET]] implicit referenced struct[[CYAN]] Invalid[[RESET]]
+//CHECK: {{^}}[[Blue]]| |-[[RESET]][[GREEN]]CXXRecordDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:1[[RESET]], [[Yellow]]col:8[[RESET]]> [[Yellow]]col:8[[RESET]] implicit struct[[CYAN]] Invalid[[RESET]]
 //CHECK: {{^}}[[Blue]]| |-[[RESET]][[GREEN]]CXXConstructorDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:29:3[[RESET]], [[Yellow]]col:42[[RESET]]> [[Yellow]]col:29[[RESET]] invalid[[CYAN]] Invalid[[RESET]] [[Green]]'void (int)'[[RESET]]
 //CHECK: {{^}}[[Blue]]| | |-[[RESET]][[GREEN]]ParmVarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:37[[RESET]], [[Yellow]]<invalid sloc>[[RESET]]> [[Yellow]]col:42[[RESET]] invalid [[Green]]'int'[[RESET]]
 //CHECK: {{^}}[[Blue]]| | `-[[RESET]][[BLUE]]NoInlineAttr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:18[[RESET]]>
@@ -100,5 +100,5 @@ struct Invalid {
 //CHECK: {{^}}[[Blue]]| | `-[[RESET]][[GREEN]]ParmVarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:8[[RESET]]> [[Yellow]]col:8[[RESET]] [[Green]]'const Invalid &'[[RESET]]
 //CHECK: {{^}}[[Blue]]| `-[[RESET]][[GREEN]]CXXConstructorDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:8[[RESET]]> [[Yellow]]col:8[[RESET]] implicit constexpr[[CYAN]] Invalid[[RESET]] [[Green]]'void (Invalid &&)'[[RESET]] inline default trivial noexcept-unevaluated 0x{{[0-9a-fA-F]*}}
 //CHECK: {{^}}[[Blue]]|   `-[[RESET]][[GREEN]]ParmVarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:8[[RESET]]> [[Yellow]]col:8[[RESET]] [[Green]]'Invalid &&'[[RESET]]
-//CHECK: {{^}}[[Blue]]`-[[RESET]][[GREEN]]VarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:1[[RESET]], [[Yellow]]line:30:3[[RESET]]> [[Yellow]]col:3[[RESET]][[CYAN]] Invalid[[RESET]] [[Green]]'struct Invalid':'Invalid'[[RESET]]
-//CHECK: {{^}}[[Blue]]  `-[[RESET]][[MAGENTA]]CXXConstructExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:3[[RESET]]> [[Green]]'struct Invalid':'Invalid'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]] [[Green]]'void () noexcept'[[RESET]]
+//CHECK: {{^}}[[Blue]]`-[[RESET]][[GREEN]]VarDecl[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:1[[RESET]], [[Yellow]]line:30:3[[RESET]]> [[Yellow]]col:3[[RESET]][[CYAN]] Invalid[[RESET]] [[Green]]'struct Invalid'[[RESET]]
+//CHECK: {{^}}[[Blue]]  `-[[RESET]][[MAGENTA]]CXXConstructExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:3[[RESET]]> [[Green]]'struct Invalid'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]] [[Green]]'void () noexcept'[[RESET]]
diff --git a/clang/test/AST/ast-dump-comment.cpp b/clang/test/AST/ast-dump-comment.cpp
index 40c3edb62821b..b5dbe2e317d8c 100644
--- a/clang/test/AST/ast-dump-comment.cpp
+++ b/clang/test/AST/ast-dump-comment.cpp
@@ -131,3 +131,51 @@ void Test_TemplatedFunctionVariadic(int arg, ...);
 // CHECK:        ParamCommandComment{{.*}} [in] implicitly Param="..."
 // CHECK-NEXT:     ParagraphComment
 // CHECK-NEXT:       TextComment{{.*}} Text=" More arguments"
+
+/// \param[out] Aaa <summary>Short summary</summary>
+int Test_HTMLSummaryTag(int Aaa);
+// CHECK:     FunctionDecl{{.*}}Test_HTMLSummaryTag
+// CHECK:       ParamCommandComment{{.*}} [out] explicitly Param="Aaa"
+// CHECK-NEXT:    ParagraphComment
+// CHECK:           HTMLStartTagComment{{.*}} Name="summary"
+// CHECK-NEXT:        TextComment{{.*}} Text="Short summary"
+// CHECK-NEXT:        HTMLEndTagComment{{.*}} Name="summary"
+
+/// \thread_safe test for underscore in special command
+int Test_UnderscoreInSpecialCommand;
+// CHECK:      VarDecl{{.*}}Test_UnderscoreInSpecialCommand 'int'
+// CHECK:        InlineCommandComment{{.*}} Name="thread_safe" RenderNormal
+// CHECK-NEXT:     TextComment{{.*}} Text=" test for underscore in special command"
+
+/// <details>
+///   <summary>
+///     Summary
+///   </summary>
+///   <p>Details</p>
+/// </details>
+///
+/// Some <mark>highlighting</mark>
+///
+/// <figure>
+///   <img src="pic.jpg">
+///   <figcaption>Figure 1</figcaption>
+/// </figure>
+int Test_AdditionalHTMLTags(int Aaa);
+// CHECK:      FunctionDecl{{.*}}Test_AdditionalHTMLTags 'int (int)'
+// CHECK:        HTMLStartTagComment{{.*}} Name="details"
+// CHECK:        HTMLStartTagComment{{.*}} Name="summary"
+// CHECK-NEXT:   TextComment{{.*}} Text="     Summary"
+// CHECK:        HTMLEndTagComment{{.*}} Name="summary"
+// CHECK:        HTMLStartTagComment{{.*}} Name="p"
+// CHECK-NEXT:   TextComment{{.*}} Text="Details"
+// CHECK-NEXT:   HTMLEndTagComment{{.*}} Name="p"
+// CHECK:        HTMLEndTagComment{{.*}} Name="details"
+// CHECK:        HTMLStartTagComment{{.*}} Name="mark"
+// CHECK-NEXT:   TextComment{{.*}} Text="highlighting"
+// CHECK-NEXT:   HTMLEndTagComment{{.*}} Name="mark"
+// CHECK:        HTMLStartTagComment{{.*}} Name="figure"
+// CHECK:        HTMLStartTagComment{{.*}} Name="img" Attrs:  "src="pic.jpg"
+// CHECK:        HTMLStartTagComment{{.*}} Name="figcaption"
+// CHECK-NEXT:   TextComment{{.*}} Text="Figure 1"
+// CHECK-NEXT:   HTMLEndTagComment{{.*}} Name="figcaption"
+// CHECK:        HTMLEndTagComment{{.*}} Name="figure"
diff --git a/clang/test/AST/ast-dump-ctad-alias.cpp b/clang/test/AST/ast-dump-ctad-alias.cpp
index f39a4cee518ce..781fb9f28cb8d 100644
--- a/clang/test/AST/ast-dump-ctad-alias.cpp
+++ b/clang/test/AST/ast-dump-ctad-alias.cpp
@@ -39,15 +39,14 @@ Out2<double>::AInner t(1.0);
 // CHECK-NEXT: |     |   |-DeducedTemplateSpecializationType {{.*}} 'Out2<double>::AInner' dependent
 // CHECK-NEXT: |     |   | `-name: 'Out2<double>::AInner'
 // CHECK-NEXT: |     |   |   `-TypeAliasTemplateDecl {{.+}} AInner{{$}}
-// CHECK-NEXT: |     |   `-ElaboratedType {{.*}} 'Inner<Y>' sugar dependent
-// CHECK-NEXT: |     |     `-TemplateSpecializationType {{.*}} 'Inner<Y>' dependent
-// CHECK-NEXT: |     |       |-name: 'Inner':'Out<int>::Inner' qualified
-// CHECK-NEXT: |     |       | `-ClassTemplateDecl {{.+}} Inner{{$}}
-// CHECK-NEXT: |     |       `-TemplateArgument type 'Y'
-// CHECK-NEXT: |     |         `-SubstTemplateTypeParmType {{.*}} 'Y'
-// CHECK-NEXT: |     |           |-FunctionTemplate {{.*}} '<deduction guide for Inner>'
-// CHECK-NEXT: |     |           `-TemplateTypeParmType {{.*}} 'Y' dependent depth 1 index 0
-// CHECK-NEXT: |     |             `-TemplateTypeParm {{.*}} 'Y'
+// CHECK-NEXT: |     |   `-TemplateSpecializationType {{.*}} 'Inner<Y>' dependent
+// CHECK-NEXT: |     |     |-name: 'Inner':'Out<int>::Inner' qualified
+// CHECK-NEXT: |     |     | `-ClassTemplateDecl {{.+}} Inner{{$}}
+// CHECK-NEXT: |     |     `-TemplateArgument type 'Y'
+// CHECK-NEXT: |     |       `-SubstTemplateTypeParmType {{.*}} 'Y'
+// CHECK-NEXT: |     |         |-FunctionTemplate {{.*}} '<deduction guide for Inner>'
+// CHECK-NEXT: |     |         `-TemplateTypeParmType {{.*}} 'Y' dependent depth 1 index 0
+// CHECK-NEXT: |     |           `-TemplateTypeParm {{.*}} 'Y'
 // CHECK-NEXT: |     |-CXXDeductionGuideDecl {{.*}} <deduction guide for AInner> 'auto (Y) -> Inner<Y>'
 // CHECK-NEXT: |     | `-ParmVarDecl {{.*}} 'Y'
 // CHECK-NEXT: |     `-CXXDeductionGuideDecl {{.*}} used <deduction guide for AInner> 'auto (double) -> Inner<double>' implicit_instantiation
@@ -188,9 +187,9 @@ void foo() {
 // CHECK-NEXT: | | | |-ImplicitConceptSpecializationDecl {{.*}}
 // CHECK-NEXT: | | | | |-TemplateArgument type 'type-parameter-0-2'
 // CHECK-NEXT: | | | | | `-TemplateTypeParmType {{.*}} 'type-parameter-0-2' dependent depth 0 index 2
-// CHECK-NEXT: | | | | `-TemplateArgument pack '<Packs<type-parameter-0-1...>>'
-// CHECK-NEXT: | | | |   `-TemplateArgument type 'Packs<type-parameter-0-1...>'
-// CHECK-NEXT: | | | |     `-TemplateSpecializationType {{.*}} 'Packs<type-parameter-0-1...>' dependent
+// CHECK-NEXT: | | | | `-TemplateArgument pack '<GH124715::Packs<type-parameter-0-1...>>'
+// CHECK-NEXT: | | | |   `-TemplateArgument type 'GH124715::Packs<type-parameter-0-1...>'
+// CHECK-NEXT: | | | |     `-TemplateSpecializationType {{.*}} 'GH124715::Packs<type-parameter-0-1...>' dependent
 // CHECK-NEXT: | | | |       |-name: 'GH124715::Packs'
 // CHECK-NEXT: | | | |       | `-ClassTemplateDecl {{.*}} Packs
 // CHECK-NEXT: | | | |       `-TemplateArgument pack '<type-parameter-0-1...>'
diff --git a/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp b/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp
index fc86aeb3e5ec3..09a274aa46422 100644
--- a/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp
+++ b/clang/test/AST/ast-dump-cxx2b-deducing-this.cpp
@@ -33,5 +33,5 @@ struct B {
   operator A(this B);
 };
 A a = A(B{});
-// CHECK: CallExpr 0x{{[^ ]*}} <col:9, col:11> 'A':'GH130272::A'
+// CHECK: CallExpr 0x{{[^ ]*}} <col:9, col:11> 'A'
 }
diff --git a/clang/test/AST/ast-dump-decl-json.c b/clang/test/AST/ast-dump-decl-json.c
index ec2d75ba7195d..b84ddf93f44c3 100644
--- a/clang/test/AST/ast-dump-decl-json.c
+++ b/clang/test/AST/ast-dump-decl-json.c
@@ -585,7 +585,6 @@ void testParmVarDecl(int TestParmVarDecl);
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "name": "e",
 // CHECK-NEXT:    "type": {
-// CHECK-NEXT:     "desugaredQualType": "enum TestEnumDeclAnon::(unnamed at {{.*}}:31:3)",
 // CHECK-NEXT:     "qualType": "enum (unnamed enum at {{.*}}:31:3)"
 // CHECK-NEXT:    }
 // CHECK-NEXT:   }
@@ -777,7 +776,6 @@ void testParmVarDecl(int TestParmVarDecl);
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "name": "testRecordDeclAnon1",
 // CHECK-NEXT:    "type": {
-// CHECK-NEXT:     "desugaredQualType": "struct TestRecordDeclAnon1::(unnamed at {{.*}}:46:3)",
 // CHECK-NEXT:     "qualType": "struct (unnamed struct at {{.*}}:46:3)"
 // CHECK-NEXT:    }
 // CHECK-NEXT:   }
@@ -1204,7 +1202,6 @@ void testParmVarDecl(int TestParmVarDecl);
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "name": "y",
 // CHECK-NEXT:    "type": {
-// CHECK-NEXT:     "desugaredQualType": "enum (unnamed at {{.*}}:69:29)",
 // CHECK-NEXT:     "qualType": "enum (unnamed enum at {{.*}}:69:29)"
 // CHECK-NEXT:    }
 // CHECK-NEXT:   },
diff --git a/clang/test/AST/ast-dump-decl.cpp b/clang/test/AST/ast-dump-decl.cpp
index 08d8fba795daa..dca0e9b9ca10a 100644
--- a/clang/test/AST/ast-dump-decl.cpp
+++ b/clang/test/AST/ast-dump-decl.cpp
@@ -35,9 +35,9 @@ namespace testVarDeclNRVO {
 // CHECK:      FunctionDecl{{.*}} TestFuncNRVO 'A ()'
 // CHECK-NEXT: `-CompoundStmt
 // CHECK-NEXT: |-DeclStmt
-// CHECK-NEXT: | `-VarDecl{{.*}} TestVarDeclNRVO 'A':'testVarDeclNRVO::A' nrvo callinit
+// CHECK-NEXT: | `-VarDecl{{.*}} TestVarDeclNRVO 'A' nrvo callinit
 // CHECK-NEXT: |   `-CXXConstructExpr
-// CHECK-NEXT: `-ReturnStmt{{.*}} nrvo_candidate(Var {{.*}} 'TestVarDeclNRVO' 'A':'testVarDeclNRVO::A')
+// CHECK-NEXT: `-ReturnStmt{{.*}} nrvo_candidate(Var {{.*}} 'TestVarDeclNRVO' 'A')
 
 void testParmVarDeclInit(int TestParmVarDeclInit = 0);
 // CHECK:      ParmVarDecl{{.*}} TestParmVarDeclInit 'int'
@@ -131,8 +131,8 @@ namespace testCXXRecordDecl {
 // CHECK-NEXT:     CopyAssignment simple non_trivial has_const_param
 // CHECK-NEXT:     MoveAssignment exists simple non_trivial
 // CHECK-NEXT:     Destructor simple irrelevant trivial
-// CHECK-NEXT:   virtual private 'A':'testCXXRecordDecl::A'
-// CHECK-NEXT:   public 'B':'testCXXRecordDecl::B'
+// CHECK-NEXT:   virtual private 'A'
+// CHECK-NEXT:   public 'B'
 // CHECK-NEXT:   CXXRecordDecl{{.*}} class TestCXXRecordDecl
 // CHECK-NEXT:   FieldDecl
 
@@ -269,7 +269,7 @@ namespace testFunctionTemplateDecl {
   // CHECK-NEXT:  |-TemplateArgument type 'testFunctionTemplateDecl::B'
   // CHECK-NEXT:  | `-RecordType 0{{.+}} 'testFunctionTemplateDecl::B'
   // CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'B'
-  // CHECK-NEXT:  `-ParmVarDecl 0x{{.+}} <col:40> col:41 'B':'testFunctionTemplateDecl::B'
+  // CHECK-NEXT:  `-ParmVarDecl 0x{{.+}} <col:40> col:41 'B'
 
 
 namespace testClassTemplateDecl {
@@ -330,8 +330,8 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | | `-Destructor irrelevant non_trivial user_declared{{$}}
 // CHECK-NEXT:  | |-CXXRecordDecl 0x{{.+}} <col:24, col:30> col:30 implicit referenced class TestClassTemplate{{$}}
 // CHECK-NEXT:  | |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-50]]:3, col:9> col:3 public{{$}}
-// CHECK-NEXT:  | |-CXXConstructorDecl 0x[[#%x,TEMPLATE_CONSTRUCTOR_DECL:]] <line:[[@LINE-50]]:5, col:23> col:5 TestClassTemplate<T> 'void ()'{{$}}
-// CHECK-NEXT:  | |-CXXDestructorDecl 0x[[#%x,TEMPLATE_DESTRUCTOR_DECL:]] <line:[[@LINE-50]]:5, col:24> col:5 ~TestClassTemplate<T> 'void ()' not_selected{{$}}
+// CHECK-NEXT:  | |-CXXConstructorDecl 0x[[#%x,TEMPLATE_CONSTRUCTOR_DECL:]] <line:[[@LINE-50]]:5, col:23> col:5 testClassTemplateDecl::TestClassTemplate<T> 'void ()'{{$}}
+// CHECK-NEXT:  | |-CXXDestructorDecl 0x[[#%x,TEMPLATE_DESTRUCTOR_DECL:]] <line:[[@LINE-50]]:5, col:24> col:5 ~testClassTemplateDecl::TestClassTemplate<T> 'void ()' not_selected{{$}}
 // CHECK-NEXT:  | |-CXXMethodDecl 0x[[#%x,TEMPLATE_METHOD_DECL:]] <line:[[@LINE-50]]:5, col:11> col:9 j 'int ()'{{$}}
 // CHECK-NEXT:  | `-FieldDecl 0x{{.+}} <line:[[@LINE-50]]:5, col:9> col:9 i 'int'{{$}}
 // CHECK-NEXT:  |-ClassTemplateSpecializationDecl 0x{{.+}} <line:[[@LINE-56]]:3, line:[[@LINE-50]]:3> line:[[@LINE-56]]:30 class TestClassTemplate definition implicit_instantiation{{$}}
@@ -343,7 +343,7 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | | |-MoveAssignment{{$}}
 // CHECK-NEXT:  | | `-Destructor non_trivial user_declared{{$}}
 // CHECK-NEXT:  | |-TemplateArgument type 'testClassTemplateDecl::A'{{$}}
-// CHECK-NEXT:  | | `-RecordType 0{{.+}} 'testClassTemplateDecl::A'{{$}}
+// CHECK-NEXT:  | | `-RecordType 0{{.+}} 'testClassTemplateDecl::A' canonical{{$}}
 // CHECK-NEXT:  | |   `-CXXRecord 0x{{.+}} 'A'{{$}}
 // CHECK-NEXT:  | |-CXXRecordDecl 0x{{.+}} <col:24, col:30> col:30 implicit class TestClassTemplate{{$}}
 // CHECK-NEXT:  | |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-67]]:3, col:9> col:3 public{{$}}
@@ -366,7 +366,7 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | |-MoveAssignment exists simple trivial needs_implicit{{$}}
 // CHECK-NEXT:  | `-Destructor simple irrelevant trivial needs_implicit{{$}}
 // CHECK-NEXT:  |-TemplateArgument type 'testClassTemplateDecl::B'{{$}}
-// CHECK-NEXT:  | `-RecordType 0{{.+}} 'testClassTemplateDecl::B'{{$}}
+// CHECK-NEXT:  | `-RecordType 0{{.+}} 'testClassTemplateDecl::B' canonical{{$}}
 // CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'B'{{$}}
 // CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} <col:14, col:20> col:20 implicit class TestClassTemplate{{$}}
 // CHECK-NEXT:  `-FieldDecl 0x{{.+}} <line:[[@LINE-78]]:5, col:9> col:9 j 'int'{{$}}
@@ -380,7 +380,7 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | |-MoveAssignment{{$}}
 // CHECK-NEXT:  | `-Destructor non_trivial user_declared{{$}}
 // CHECK-NEXT:  |-TemplateArgument type 'testClassTemplateDecl::C'{{$}}
-// CHECK-NEXT:  | `-RecordType 0{{.+}} 'testClassTemplateDecl::C'{{$}}
+// CHECK-NEXT:  | `-RecordType 0{{.+}} 'testClassTemplateDecl::C' canonical{{$}}
 // CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'C'{{$}}
 // CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} <line:[[@LINE-104]]:24, col:30> col:30 implicit class TestClassTemplate{{$}}
 // CHECK-NEXT:  |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-104]]:3, col:9> col:3 public{{$}}
@@ -398,7 +398,7 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  | |-MoveAssignment{{$}}
 // CHECK-NEXT:  | `-Destructor non_trivial user_declared{{$}}
 // CHECK-NEXT:  |-TemplateArgument type 'testClassTemplateDecl::D'{{$}}
-// CHECK-NEXT:  | `-RecordType 0{{.+}} 'testClassTemplateDecl::D'{{$}}
+// CHECK-NEXT:  | `-RecordType 0{{.+}} 'testClassTemplateDecl::D' canonical{{$}}
 // CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'D'{{$}}
 // CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} <line:[[@LINE-122]]:24, col:30> col:30 implicit class TestClassTemplate{{$}}
 // CHECK-NEXT:  |-AccessSpecDecl 0x{{.+}} <line:[[@LINE-122]]:3, col:9> col:3 public{{$}}
@@ -432,7 +432,7 @@ namespace testClassTemplateDecl {
 // CHECK-NEXT:  |-TemplateArgument type 'type-parameter-0-0'{{$}}
 // CHECK-NEXT:  | `-TemplateTypeParmType 0x{{.+}} 'type-parameter-0-0' dependent depth 0 index 0{{$}}
 // CHECK-NEXT:  |-TemplateArgument type 'testClassTemplateDecl::A'{{$}}
-// CHECK-NEXT:  | `-RecordType 0x{{.+}} 'testClassTemplateDecl::A'{{$}}
+// CHECK-NEXT:  | `-RecordType 0x{{.+}} 'testClassTemplateDecl::A' canonical{{$}}
 // CHECK-NEXT:  |   `-CXXRecord 0x{{.+}} 'A'{{$}}
 // CHECK-NEXT:  |-TemplateTypeParmDecl 0x{{.+}} <col:12, col:21> col:21 referenced typename depth 0 index 0 T1{{$}}
 // CHECK-NEXT:  |-CXXRecordDecl 0x{{.+}} <col:25, col:31> col:31 implicit class TestClassTemplatePartial{{$}}
@@ -605,7 +605,7 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT:   | `-ParmVarDecl 0x{{.*}} <col:50> col:51 'T'{{$}}
   // CHECK-NEXT:   `-FunctionDecl 0x{{.*}} <line:[[@LINE-6]]:24, col:51> col:29 used TestFunctionTemplate 'void (testCanonicalTemplate::A)' implicit_instantiation{{$}}
   // CHECK-NEXT:     |-TemplateArgument type 'testCanonicalTemplate::A'{{$}}
-  // CHECK-NEXT:     | `-RecordType 0x{{.+}} 'testCanonicalTemplate::A'{{$}}
+  // CHECK-NEXT:     | `-RecordType 0x{{.+}} 'testCanonicalTemplate::A' canonical{{$}}
   // CHECK-NEXT:     |   `-CXXRecord 0x{{.+}} 'A'{{$}}
   // CHECK-NEXT:     `-ParmVarDecl 0x{{.*}} <col:50> col:51 'testCanonicalTemplate::A'{{$}}
 
@@ -644,7 +644,7 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT:   | |-MoveAssignment exists simple trivial needs_implicit{{$}}
   // CHECK-NEXT:   | `-Destructor simple irrelevant trivial needs_implicit{{$}}
   // CHECK-NEXT:   |-TemplateArgument type 'testCanonicalTemplate::A'{{$}}
-  // CHECK-NEXT:   | `-RecordType 0x{{.+}} 'testCanonicalTemplate::A'{{$}}
+  // CHECK-NEXT:   | `-RecordType 0x{{.+}} 'testCanonicalTemplate::A' canonical{{$}}
   // CHECK-NEXT:   |   `-CXXRecord 0x{{.+}} 'A'{{$}}
   // CHECK-NEXT:   |-CXXRecordDecl 0x{{.+}} <col:25, col:31> col:31 implicit class TestClassTemplate{{$}}
   // CHECK-NEXT:   |-FriendDecl 0x{{.+}} <line:[[@LINE-30]]:5, col:40> col:40{{$}}
@@ -677,7 +677,7 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT:   | |-MoveAssignment exists simple trivial needs_implicit{{$}}
   // CHECK-NEXT:   | `-Destructor simple irrelevant trivial needs_implicit{{$}}
   // CHECK-NEXT:   |-TemplateArgument type 'testCanonicalTemplate::A'{{$}}
-  // CHECK-NEXT:   | `-RecordType 0x{{.+}} 'testCanonicalTemplate::A'{{$}}
+  // CHECK-NEXT:   | `-RecordType 0x{{.+}} 'testCanonicalTemplate::A' canonical{{$}}
   // CHECK-NEXT:   |   `-CXXRecord 0x{{.+}} 'A'{{$}}
   // CHECK-NEXT:   |-CXXRecordDecl 0x{{.+}} <col:25, col:31> col:31 implicit class TestClassTemplate2{{$}}
   // CHECK-NEXT:   |-CXXConstructorDecl 0x{{.+}} <col:31> col:31 implicit used constexpr TestClassTemplate2 'void () noexcept' inline default trivial{{$}}
@@ -721,7 +721,7 @@ namespace testCanonicalTemplate {
   // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{.+}} <col:16, col:25> col:25 referenced typename depth 0 index 0 T{{$}}
   // CHECK-NEXT: |-VarDecl 0x{{.+}} <col:28, col:43> col:43 TestVarTemplate 'const T' static{{$}}
   // CHECK-NEXT: |-VarTemplateSpecializationDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-12]]:3, line:[[@LINE-11]]:34> col:14 referenced TestVarTemplate 'const int' implicit_instantiation cinit{{$}}
-  // CHECK-NEXT: | |-NestedNameSpecifier TypeSpec 'testCanonicalTemplate::S'{{$}}
+  // CHECK-NEXT: | |-NestedNameSpecifier TypeSpec 'S'{{$}}
   // CHECK-NEXT: | |-TemplateArgument type 'int'{{$}}
   // CHECK-NEXT: | | `-BuiltinType 0x{{.+}} 'int'{{$}}
   // CHECK-NEXT: | `-InitListExpr 0x{{.+}} <col:32, col:34> 'int'{{$}}
@@ -735,13 +735,13 @@ namespace testCanonicalTemplate {
   // CHECK:      VarTemplateDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-24]]:3, line:[[@LINE-23]]:34> col:14 TestVarTemplate{{$}}
   // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{.+}} <line:[[@LINE-25]]:12, col:21> col:21 referenced typename depth 0 index 0 T{{$}}
   // CHECK-NEXT: |-VarDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <line:[[@LINE-25]]:3, col:34> col:14 TestVarTemplate 'const T' cinit{{$}}
-  // CHECK-NEXT: | |-NestedNameSpecifier TypeSpec 'testCanonicalTemplate::S'{{$}}
+  // CHECK-NEXT: | |-NestedNameSpecifier TypeSpec 'S'{{$}}
   // CHECK-NEXT: | `-InitListExpr 0x{{.+}} <col:32, col:34> 'void'{{$}}
   // CHECK-NEXT: |-VarTemplateSpecialization 0x{{.+}} 'TestVarTemplate' 'const int'{{$}}
   // CHECK-NEXT: `-VarTemplateSpecialization 0x{{.+}} 'TestVarTemplate' 'const int'{{$}}
 
   // CHECK:      VarTemplateSpecializationDecl 0x{{.+}} parent 0x{{.+}} prev 0x{{.+}} <{{.+}}:[[@LINE-32]]:3, line:[[@LINE-31]]:34> col:14 referenced TestVarTemplate 'const int' implicit_instantiation cinit{{$}}
-  // CHECK-NEXT: |-NestedNameSpecifier TypeSpec 'testCanonicalTemplate::S'{{$}}
+  // CHECK-NEXT: |-NestedNameSpecifier TypeSpec 'S'{{$}}
   // CHECK-NEXT: |-TemplateArgument type 'int'{{$}}
   // CHECK-NEXT: | `-BuiltinType 0x{{.+}} 'int'{{$}}
   // CHECK-NEXT: `-InitListExpr 0x{{.+}} <col:32, col:34> 'int'{{$}}
@@ -901,7 +901,7 @@ template<typename T> class TestFriendDecl {
 // CHECK:        CXXRecord{{.*}} TestFriendDecl
 // CHECK-NEXT:   FriendDecl
 // CHECK-NEXT:     FunctionDecl{{.*}} foo
-// CHECK-NEXT:   FriendDecl{{.*}} 'class A':'A'
+// CHECK-NEXT:   FriendDecl{{.*}} 'class A'
 // CHECK-NEXT:     CXXRecordDecl{{.*}} class A
 // CHECK-NEXT:   FriendDecl{{.*}} 'T'
 
diff --git a/clang/test/AST/ast-dump-expr-json.cpp b/clang/test/AST/ast-dump-expr-json.cpp
index 11026c9d302f0..6293f8cb3fe0e 100644
--- a/clang/test/AST/ast-dump-expr-json.cpp
+++ b/clang/test/AST/ast-dump-expr-json.cpp
@@ -7962,7 +7962,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:         }
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "type": {
-// CHECK-NEXT:         "desugaredQualType": "NS::X",
 // CHECK-NEXT:         "qualType": "X"
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "valueCategory": "prvalue",
@@ -7988,7 +7987,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:           }
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "type": {
-// CHECK-NEXT:           "desugaredQualType": "const NS::X",
 // CHECK-NEXT:           "qualType": "const X"
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "valueCategory": "lvalue",
@@ -8142,7 +8140,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:           }
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "type": {
-// CHECK-NEXT:           "desugaredQualType": "const NS::X",
 // CHECK-NEXT:           "qualType": "const X"
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "valueCategory": "lvalue",
@@ -8395,7 +8392,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:         }
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "type": {
-// CHECK-NEXT:         "desugaredQualType": "NS::X",
 // CHECK-NEXT:         "qualType": "X"
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "valueCategory": "prvalue",
@@ -8421,7 +8417,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:           }
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "type": {
-// CHECK-NEXT:           "desugaredQualType": "const NS::X",
 // CHECK-NEXT:           "qualType": "const X"
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "valueCategory": "lvalue",
@@ -8720,7 +8715,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:         }
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "type": {
-// CHECK-NEXT:         "desugaredQualType": "NS::X",
 // CHECK-NEXT:         "qualType": "X"
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "valueCategory": "prvalue",
@@ -8746,7 +8740,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:           }
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "type": {
-// CHECK-NEXT:           "desugaredQualType": "const NS::X",
 // CHECK-NEXT:           "qualType": "const X"
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "valueCategory": "lvalue",
@@ -8900,7 +8893,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:           }
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "type": {
-// CHECK-NEXT:           "desugaredQualType": "const NS::X",
 // CHECK-NEXT:           "qualType": "const X"
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "valueCategory": "lvalue",
@@ -9032,7 +9024,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:        "isUsed": true,
 // CHECK-NEXT:        "name": "x",
 // CHECK-NEXT:        "type": {
-// CHECK-NEXT:         "desugaredQualType": "NS::X",
 // CHECK-NEXT:         "qualType": "X"
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "init": "call",
@@ -9053,7 +9044,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:           }
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "type": {
-// CHECK-NEXT:           "desugaredQualType": "NS::X",
 // CHECK-NEXT:           "qualType": "X"
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "valueCategory": "prvalue",
@@ -9155,7 +9145,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:         }
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "type": {
-// CHECK-NEXT:         "desugaredQualType": "NS::X",
 // CHECK-NEXT:         "qualType": "X"
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "valueCategory": "prvalue",
@@ -9181,7 +9170,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:           }
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "type": {
-// CHECK-NEXT:           "desugaredQualType": "const NS::X",
 // CHECK-NEXT:           "qualType": "const X"
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "valueCategory": "lvalue",
@@ -9203,7 +9191,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:             }
 // CHECK-NEXT:            },
 // CHECK-NEXT:            "type": {
-// CHECK-NEXT:             "desugaredQualType": "NS::X",
 // CHECK-NEXT:             "qualType": "X"
 // CHECK-NEXT:            },
 // CHECK-NEXT:            "valueCategory": "lvalue",
@@ -9212,7 +9199,6 @@ void TestNonADLCall3() {
 // CHECK-NEXT:             "kind": "VarDecl",
 // CHECK-NEXT:             "name": "x",
 // CHECK-NEXT:             "type": {
-// CHECK-NEXT:              "desugaredQualType": "NS::X",
 // CHECK-NEXT:              "qualType": "X"
 // CHECK-NEXT:             }
 // CHECK-NEXT:            }
diff --git a/clang/test/AST/ast-dump-expr.cpp b/clang/test/AST/ast-dump-expr.cpp
index 6fd429d1500a4..7a686b28d80d1 100644
--- a/clang/test/AST/ast-dump-expr.cpp
+++ b/clang/test/AST/ast-dump-expr.cpp
@@ -219,13 +219,10 @@ void PostfixExpressions(S a, S *p, U<int> *r) {
   // CHECK-NEXT: MemberExpr 0x{{[^ ]*}} <col:3, col:5> '<bound member function type>' .~S 0x{{[^ ]*}}
   // CHECK-NEXT: DeclRefExpr 0x{{[^ ]*}} <col:3> 'S' lvalue ParmVar 0x{{[^ ]*}} 'a' 'S'
 
-  // FIXME: similarly, there is no way to distinguish the construct below from
-  // the p->~S() case.
   p->::S::~S();
   // CHECK: CXXMemberCallExpr 0x{{[^ ]*}} <line:[[@LINE-1]]:3, col:14> 'void'
   // CHECK-NEXT: MemberExpr 0x{{[^ ]*}} <col:3, col:12> '<bound member function type>' ->~S 0x{{[^ ]*}}
-  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'S'
-  // CHECK-NEXT: NestedNameSpecifier Global
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec '::S'
   // CHECK-NEXT: ImplicitCastExpr
   // CHECK-NEXT: DeclRefExpr 0x{{[^ ]*}} <col:3> 'S *' lvalue ParmVar 0x{{[^ ]*}} 'p' 'S *'
 
@@ -597,5 +594,5 @@ struct S {
 void f() {
   S(S(0, 1));
 }
-// CHECK: CXXTemporaryObjectExpr {{.*}} <col:5, col:11> 'S':'GH143711::S' 'void (int, int)'
+// CHECK: CXXTemporaryObjectExpr {{.*}} <col:5, col:11> 'S' 'void (int, int)'
 }
diff --git a/clang/test/AST/ast-dump-for-range-lifetime.cpp b/clang/test/AST/ast-dump-for-range-lifetime.cpp
index ee046be19ab63..c330342f3328e 100644
--- a/clang/test/AST/ast-dump-for-range-lifetime.cpp
+++ b/clang/test/AST/ast-dump-for-range-lifetime.cpp
@@ -24,14 +24,14 @@ void test1() {
   // CHECK-NEXT: |     |-<<<NULL>>>
   // CHECK-NEXT: |     |-DeclStmt {{.*}}
   // CHECK-NEXT: |     | `-VarDecl {{.*}} implicit used __range1 'const A &' cinit
-  // CHECK-NEXT: |     |   `-ExprWithCleanups {{.*}} 'const A':'const P2718R0::A' lvalue
-  // CHECK-NEXT: |     |     `-CallExpr {{.*}} 'const A':'const P2718R0::A' lvalue
+  // CHECK-NEXT: |     |   `-ExprWithCleanups {{.*}} 'const A' lvalue
+  // CHECK-NEXT: |     |     `-CallExpr {{.*}} 'const A' lvalue
   // CHECK-NEXT: |     |       |-ImplicitCastExpr {{.*}} 'const A &(*)(const A &)' <FunctionToPointerDecay>
   // CHECK-NEXT: |     |       | `-DeclRefExpr {{.*}} 'const A &(const A &)' lvalue Function {{.*}} 'f1' 'const A &(const A &)'
-  // CHECK-NEXT: |     |       `-MaterializeTemporaryExpr {{.*}} 'const A':'const P2718R0::A' lvalue extended by Var {{.*}} '__range1' 'const A &'
-  // CHECK-NEXT: |     |         `-ImplicitCastExpr {{.*}} 'const A':'const P2718R0::A' <NoOp>
-  // CHECK-NEXT: |     |           `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
-  // CHECK-NEXT: |     |             `-CallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT: |     |       `-MaterializeTemporaryExpr {{.*}} 'const A' lvalue extended by Var {{.*}} '__range1' 'const A &'
+  // CHECK-NEXT: |     |         `-ImplicitCastExpr {{.*}} 'const A' <NoOp>
+  // CHECK-NEXT: |     |           `-CXXBindTemporaryExpr {{.*}} 'A' (CXXTemporary {{.*}})
+  // CHECK-NEXT: |     |             `-CallExpr {{.*}} 'A'
   // CHECK-NEXT: |     |               `-ImplicitCastExpr {{.*}} 'A (*)()' <FunctionToPointerDecay>
   // CHECK-NEXT: |     |                 `-DeclRefExpr {{.*}} 'A ()' lvalue Function {{.*}} 'g' 'A ()'
   for (auto e : f1(g()))
@@ -56,11 +56,11 @@ void test2() {
   // CHECK-NEXT: |     |       `-CallExpr {{.*}} 'const A *'
   // CHECK-NEXT: |     |         |-ImplicitCastExpr {{.*}} 'const A *(*)(const A &)' <FunctionToPointerDecay>
   // CHECK-NEXT: |     |         | `-DeclRefExpr {{.*}} 'const A *(const A &)' lvalue Function {{.*}} 'g' 'const A *(const A &)'
-  // CHECK-NEXT: |     |         `-ImplicitCastExpr {{.*}} 'const A':'const P2718R0::A' lvalue <DerivedToBase (A)>
-  // CHECK-NEXT: |     |           `-MaterializeTemporaryExpr {{.*}} 'const B':'const P2718R0::B' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT: |     |             `-ImplicitCastExpr {{.*}} 'const B':'const P2718R0::B' <NoOp>
-  // CHECK-NEXT: |     |               `-CXXBindTemporaryExpr {{.*}} 'B':'P2718R0::B' (CXXTemporary {{.*}})
-  // CHECK-NEXT: |     |                 `-CXXTemporaryObjectExpr {{.*}} 'B':'P2718R0::B' 'void () noexcept(false)' zeroing
+  // CHECK-NEXT: |     |         `-ImplicitCastExpr {{.*}} 'const A' lvalue <DerivedToBase (A)>
+  // CHECK-NEXT: |     |           `-MaterializeTemporaryExpr {{.*}} 'const B' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT: |     |             `-ImplicitCastExpr {{.*}} 'const B' <NoOp>
+  // CHECK-NEXT: |     |               `-CXXBindTemporaryExpr {{.*}} 'B' (CXXTemporary {{.*}})
+  // CHECK-NEXT: |     |                 `-CXXTemporaryObjectExpr {{.*}} 'B' 'void () noexcept(false)' zeroing
   for (auto e : f(g(B())))
     bar(e);
 }
@@ -82,9 +82,9 @@ void test3() {
   // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'int[3]' lvalue
   // CHECK-NEXT:  |     `-BinaryOperator {{.*}} 'int[3]' lvalue ','
   // CHECK-NEXT:  |       |-CXXStaticCastExpr {{.*}} 'void' static_cast<void> <ToVoid>
-  // CHECK-NEXT:  |       | `-MaterializeTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' xvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |       |   `-CXXBindTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |       |     `-CXXTemporaryObjectExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' 'void ()'
+  // CHECK-NEXT:  |       | `-MaterializeTemporaryExpr {{.*}} 'LockGuard' xvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |       |   `-CXXBindTemporaryExpr {{.*}} 'LockGuard' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |       |     `-CXXTemporaryObjectExpr {{.*}} 'LockGuard' 'void ()'
   // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} 'int[3]' lvalue Var {{.*}} 'v' 'int[3]'
   for ([[maybe_unused]] int x : static_cast<void>(LockGuard()), v)
     LockGuard guard;
@@ -96,9 +96,9 @@ void test3() {
   // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'int[3]' lvalue
   // CHECK-NEXT:  |     `-BinaryOperator {{.*}} 'int[3]' lvalue ','
   // CHECK-NEXT:  |       |-CStyleCastExpr {{.*}} 'void' <ToVoid>
-  // CHECK-NEXT:  |       | `-MaterializeTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' xvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |       |   `-CXXBindTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |       |     `-CXXTemporaryObjectExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' 'void ()'
+  // CHECK-NEXT:  |       | `-MaterializeTemporaryExpr {{.*}} 'LockGuard' xvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |       |   `-CXXBindTemporaryExpr {{.*}} 'LockGuard' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |       |     `-CXXTemporaryObjectExpr {{.*}} 'LockGuard' 'void ()'
   // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} 'int[3]' lvalue Var {{.*}} 'v' 'int[3]'
   for ([[maybe_unused]] int x : (void)LockGuard(), v)
     LockGuard guard;
@@ -109,9 +109,9 @@ void test3() {
   // CHECK-NEXT:  | `-VarDecl {{.*}} implicit used __range1 'int (&)[3]' cinit
   // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'int[3]' lvalue
   // CHECK-NEXT:  |     `-BinaryOperator {{.*}} 'int[3]' lvalue ','
-  // CHECK-NEXT:  |       |-MaterializeTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' xvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |       | `-CXXBindTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |       |   `-CXXTemporaryObjectExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' 'void ()'
+  // CHECK-NEXT:  |       |-MaterializeTemporaryExpr {{.*}} 'LockGuard' xvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |       | `-CXXBindTemporaryExpr {{.*}} 'LockGuard' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |       |   `-CXXTemporaryObjectExpr {{.*}} 'LockGuard' 'void ()'
   // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} 'int[3]' lvalue Var {{.*}} 'v' 'int[3]'
   for ([[maybe_unused]] int x : LockGuard(), v)
     LockGuard guard;
@@ -130,12 +130,12 @@ void test4() {
   // CHECK-NEXT:  |     `-CallExpr {{.*}} 'int[3]' lvalue
   // CHECK-NEXT:  |       |-ImplicitCastExpr {{.*}} 'int (&(*)(const A &))[3]' <FunctionToPointerDecay>
   // CHECK-NEXT:  |       | `-DeclRefExpr {{.*}} 'int (&(const A &))[3]' lvalue Function {{.*}} 'default_arg_fn' 'int (&(const A &))[3]'
-  // CHECK-NEXT:  |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const A':'const P2718R0::A' lvalue has rewritten init
-  // CHECK-NEXT:  |         `-MaterializeTemporaryExpr {{.*}} 'const A':'const P2718R0::A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |           `-ImplicitCastExpr {{.*}} 'const A':'const P2718R0::A' <NoOp>
-  // CHECK-NEXT:  |             `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |               `-CXXTemporaryObjectExpr {{.*}} 'A':'P2718R0::A' 'void ()'
-  for (auto e : default_arg_fn()) 
+  // CHECK-NEXT:  |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const A' lvalue has rewritten init
+  // CHECK-NEXT:  |         `-MaterializeTemporaryExpr {{.*}} 'const A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |           `-ImplicitCastExpr {{.*}} 'const A' <NoOp>
+  // CHECK-NEXT:  |             `-CXXBindTemporaryExpr {{.*}} 'A' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |               `-CXXTemporaryObjectExpr {{.*}} 'A' 'void ()'
+  for (auto e : default_arg_fn())
     bar(e);
 }
 
@@ -158,43 +158,43 @@ void test5() {
   // CHECK-NEXT:  |     `-CallExpr {{.*}} 'int[3]' lvalue
   // CHECK-NEXT:  |       |-ImplicitCastExpr {{.*}} 'int (&(*)(const A &))[3]' <FunctionToPointerDecay>
   // CHECK-NEXT:  |       | `-DeclRefExpr {{.*}} 'int (&(const A &))[3]' lvalue Function {{.*}} 'default_arg_fn' 'int (&(const A &))[3]'
-  // CHECK-NEXT:  |       `-MaterializeTemporaryExpr {{.*}} 'const A':'const P2718R0::A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |         `-ImplicitCastExpr {{.*}} 'const A':'const P2718R0::A' <NoOp>
-  // CHECK-NEXT:  |           `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A'
-  // CHECK-NEXT:  |             `-CallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |       `-MaterializeTemporaryExpr {{.*}} 'const A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |         `-ImplicitCastExpr {{.*}} 'const A' <NoOp>
+  // CHECK-NEXT:  |           `-CXXBindTemporaryExpr {{.*}} 'A'
+  // CHECK-NEXT:  |             `-CallExpr {{.*}} 'A'
   // CHECK-NEXT:  |               |-ImplicitCastExpr {{.*}} 'A (*)(const A &, const DefaultA &)' <FunctionToPointerDecay>
   // CHECK-NEXT:  |               | `-DeclRefExpr {{.*}} 'A (const A &, const DefaultA &)' lvalue Function {{.*}} 'foo' 'A (const A &, const DefaultA &)'
-  // CHECK-NEXT:  |               |-MaterializeTemporaryExpr {{.*}} 'const A':'const P2718R0::A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |               | `-ImplicitCastExpr {{.*}} 'const A':'const P2718R0::A' <NoOp>
-  // CHECK-NEXT:  |               |   `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A'
-  // CHECK-NEXT:  |               |     `-CallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |               |-MaterializeTemporaryExpr {{.*}} 'const A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |               | `-ImplicitCastExpr {{.*}} 'const A' <NoOp>
+  // CHECK-NEXT:  |               |   `-CXXBindTemporaryExpr {{.*}} 'A'
+  // CHECK-NEXT:  |               |     `-CallExpr {{.*}} 'A'
   // CHECK-NEXT:  |               |       |-ImplicitCastExpr {{.*}} 'A (*)(const A &, const DefaultA &)' <FunctionToPointerDecay>
   // CHECK-NEXT:  |               |       | `-DeclRefExpr {{.*}} 'A (const A &, const DefaultA &)' lvalue Function {{.*}} 'foo' 'A (const A &, const DefaultA &)'
-  // CHECK-NEXT:  |               |       |-MaterializeTemporaryExpr {{.*}} 'const A':'const P2718R0::A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |               |       | `-ImplicitCastExpr {{.*}} 'const A':'const P2718R0::A' <NoOp>
-  // CHECK-NEXT:  |               |       |   `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A'
-  // CHECK-NEXT:  |               |       |     `-CallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |               |       |-MaterializeTemporaryExpr {{.*}} 'const A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |               |       | `-ImplicitCastExpr {{.*}} 'const A' <NoOp>
+  // CHECK-NEXT:  |               |       |   `-CXXBindTemporaryExpr {{.*}} 'A'
+  // CHECK-NEXT:  |               |       |     `-CallExpr {{.*}} 'A'
   // CHECK-NEXT:  |               |       |       |-ImplicitCastExpr {{.*}} 'A (*)(const A &, const DefaultA &)' <FunctionToPointerDecay>
   // CHECK-NEXT:  |               |       |       | `-DeclRefExpr {{.*}} 'A (const A &, const DefaultA &)' lvalue Function {{.*}} 'foo' 'A (const A &, const DefaultA &)'
-  // CHECK-NEXT:  |               |       |       |-MaterializeTemporaryExpr {{.*}} 'const A':'const P2718R0::A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |               |       |       | `-ImplicitCastExpr {{.*}} 'const A':'const P2718R0::A' <NoOp>
-  // CHECK-NEXT:  |               |       |       |   `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A'
-  // CHECK-NEXT:  |               |       |       |     `-CXXTemporaryObjectExpr {{.*}} 'A':'P2718R0::A' 'void ()'
-  // CHECK-NEXT:  |               |       |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
-  // CHECK-NEXT:  |               |       |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |               |       |           `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
-  // CHECK-NEXT:  |               |       |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |               |       |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
-  // CHECK-NEXT:  |               |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
-  // CHECK-NEXT:  |               |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |               |           `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
-  // CHECK-NEXT:  |               |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |               |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
-  // CHECK-NEXT:  |               `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
-  // CHECK-NEXT:  |                 `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |                   `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
-  // CHECK-NEXT:  |                     `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |                       `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
+  // CHECK-NEXT:  |               |       |       |-MaterializeTemporaryExpr {{.*}} 'const A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |               |       |       | `-ImplicitCastExpr {{.*}} 'const A' <NoOp>
+  // CHECK-NEXT:  |               |       |       |   `-CXXBindTemporaryExpr {{.*}} 'A'
+  // CHECK-NEXT:  |               |       |       |     `-CXXTemporaryObjectExpr {{.*}} 'A' 'void ()'
+  // CHECK-NEXT:  |               |       |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA' lvalue has rewritten init
+  // CHECK-NEXT:  |               |       |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |               |       |           `-ImplicitCastExpr {{.*}} 'const DefaultA' <NoOp>
+  // CHECK-NEXT:  |               |       |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |               |       |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA' 'void ()'
+  // CHECK-NEXT:  |               |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA' lvalue has rewritten init
+  // CHECK-NEXT:  |               |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |               |           `-ImplicitCastExpr {{.*}} 'const DefaultA' <NoOp>
+  // CHECK-NEXT:  |               |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |               |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA' 'void ()'
+  // CHECK-NEXT:  |               `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA' lvalue has rewritten init
+  // CHECK-NEXT:  |                 `-MaterializeTemporaryExpr {{.*}} 'const DefaultA' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |                   `-ImplicitCastExpr {{.*}} 'const DefaultA' <NoOp>
+  // CHECK-NEXT:  |                     `-CXXBindTemporaryExpr {{.*}} 'DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                       `-CXXTemporaryObjectExpr {{.*}} 'DefaultA' 'void ()'
   for (auto e : default_arg_fn(foo(foo(foo(A())))))
     bar(e);
 }
@@ -210,40 +210,40 @@ void test6() {
   // CHECK-NEXT:  |-<<<NULL>>>
   // CHECK-NEXT:  |-DeclStmt {{.*}}
   // CHECK-NEXT:  | `-VarDecl {{.*}} col:17 implicit used __range1 'C &&' cinit
-  // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'C':'P2718R0::C' xvalue
-  // CHECK-NEXT:  |     `-MaterializeTemporaryExpr {{.*}} 'C':'P2718R0::C' xvalue extended by Var {{.*}} '__range1' 'C &&'
-  // CHECK-NEXT:  |       `-CXXBindTemporaryExpr {{.*}} 'C':'P2718R0::C'
-  // CHECK-NEXT:  |         `-CXXTemporaryObjectExpr {{.*}} 'C':'P2718R0::C' 'void (int, const C &, const DefaultA &)'
+  // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'C' xvalue
+  // CHECK-NEXT:  |     `-MaterializeTemporaryExpr {{.*}} 'C' xvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |       `-CXXBindTemporaryExpr {{.*}} 'C'
+  // CHECK-NEXT:  |         `-CXXTemporaryObjectExpr {{.*}} 'C' 'void (int, const C &, const DefaultA &)'
   // CHECK-NEXT:  |           |-IntegerLiteral {{.*}}'int' 0
-  // CHECK-NEXT:  |           |-MaterializeTemporaryExpr {{.*}} 'const C':'const P2718R0::C' lvalue extended by Var {{.*}} '__range1' 'C &&'
-  // CHECK-NEXT:  |           | `-ImplicitCastExpr {{.*}} 'const C':'const P2718R0::C' <NoOp>
-  // CHECK-NEXT:  |           |   `-CXXBindTemporaryExpr {{.*}} 'C':'P2718R0::C'
-  // CHECK-NEXT:  |           |     `-CXXTemporaryObjectExpr {{.*}} 'C':'P2718R0::C' 'void (int, const C &, const DefaultA &)'
+  // CHECK-NEXT:  |           |-MaterializeTemporaryExpr {{.*}} 'const C' lvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |           | `-ImplicitCastExpr {{.*}} 'const C' <NoOp>
+  // CHECK-NEXT:  |           |   `-CXXBindTemporaryExpr {{.*}} 'C'
+  // CHECK-NEXT:  |           |     `-CXXTemporaryObjectExpr {{.*}} 'C' 'void (int, const C &, const DefaultA &)'
   // CHECK-NEXT:  |           |       |-IntegerLiteral {{.*}} 'int' 0
-  // CHECK-NEXT:  |           |       |-MaterializeTemporaryExpr {{.*}} 'const C':'const P2718R0::C' lvalue extended by Var {{.*}} '__range1' 'C &&'
-  // CHECK-NEXT:  |           |       | `-ImplicitCastExpr {{.*}} 'const C':'const P2718R0::C' <NoOp>
-  // CHECK-NEXT:  |           |       |   `-CXXBindTemporaryExpr {{.*}} 'C':'P2718R0::C'
-  // CHECK-NEXT:  |           |       |     `-CXXTemporaryObjectExpr {{.*}} 'C':'P2718R0::C' 'void (int, const C &, const DefaultA &)'
+  // CHECK-NEXT:  |           |       |-MaterializeTemporaryExpr {{.*}} 'const C' lvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |           |       | `-ImplicitCastExpr {{.*}} 'const C' <NoOp>
+  // CHECK-NEXT:  |           |       |   `-CXXBindTemporaryExpr {{.*}} 'C'
+  // CHECK-NEXT:  |           |       |     `-CXXTemporaryObjectExpr {{.*}} 'C' 'void (int, const C &, const DefaultA &)'
   // CHECK-NEXT:  |           |       |       |-IntegerLiteral {{.*}} 'int' 0
-  // CHECK-NEXT:  |           |       |       |-MaterializeTemporaryExpr {{.*}} 'const C':'const P2718R0::C' lvalue extended by Var {{.*}} '__range1' 'C &&'
-  // CHECK-NEXT:  |           |       |       | `-ImplicitCastExpr {{.*}} 'const C':'const P2718R0::C' <NoOp>
-  // CHECK-NEXT:  |           |       |       |   `-CXXBindTemporaryExpr {{.*}} 'C':'P2718R0::C'
-  // CHECK-NEXT:  |           |       |       |     `-CXXTemporaryObjectExpr {{.*}} 'C':'P2718R0::C' 'void ()'
-  // CHECK-NEXT:  |           |       |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
-  // CHECK-NEXT:  |           |       |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'C &&'
-  // CHECK-NEXT:  |           |       |           `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
-  // CHECK-NEXT:  |           |       |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |           |       |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
-  // CHECK-NEXT:  |           |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
-  // CHECK-NEXT:  |           |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'C &&'
-  // CHECK-NEXT:  |           |           `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
-  // CHECK-NEXT:  |           |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |           |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
-  // CHECK-NEXT:  |           `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
-  // CHECK-NEXT:  |             `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'C &&'
-  // CHECK-NEXT:  |               `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
-  // CHECK-NEXT:  |                 `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |                   `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
+  // CHECK-NEXT:  |           |       |       |-MaterializeTemporaryExpr {{.*}} 'const C' lvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |           |       |       | `-ImplicitCastExpr {{.*}} 'const C' <NoOp>
+  // CHECK-NEXT:  |           |       |       |   `-CXXBindTemporaryExpr {{.*}} 'C'
+  // CHECK-NEXT:  |           |       |       |     `-CXXTemporaryObjectExpr {{.*}} 'C' 'void ()'
+  // CHECK-NEXT:  |           |       |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA' lvalue has rewritten init
+  // CHECK-NEXT:  |           |       |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA' lvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |           |       |           `-ImplicitCastExpr {{.*}} 'const DefaultA' <NoOp>
+  // CHECK-NEXT:  |           |       |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |           |       |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA' 'void ()'
+  // CHECK-NEXT:  |           |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA' lvalue has rewritten init
+  // CHECK-NEXT:  |           |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA' lvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |           |           `-ImplicitCastExpr {{.*}} 'const DefaultA' <NoOp>
+  // CHECK-NEXT:  |           |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |           |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA' 'void ()'
+  // CHECK-NEXT:  |           `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA' lvalue has rewritten init
+  // CHECK-NEXT:  |             `-MaterializeTemporaryExpr {{.*}} 'const DefaultA' lvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |               `-ImplicitCastExpr {{.*}} 'const DefaultA' <NoOp>
+  // CHECK-NEXT:  |                 `-CXXBindTemporaryExpr {{.*}} 'DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                   `-CXXTemporaryObjectExpr {{.*}} 'DefaultA' 'void ()'
   for (auto e : C(0, C(0, C(0, C()))))
     bar(e);
 }
@@ -255,28 +255,28 @@ void test7() {
   // CHECK-NEXT:  |-<<<NULL>>>
   // CHECK-NEXT:  |-DeclStmt {{.*}}
   // CHECK-NEXT:  | `-VarDecl {{.*}} implicit used __range1 'A &&' cinit
-  // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'A':'P2718R0::A' xvalue
-  // CHECK-NEXT:  |     `-MaterializeTemporaryExpr {{.*}} 'A':'P2718R0::A' xvalue extended by Var {{.*}} '__range1' 'A &&'
-  // CHECK-NEXT:  |       `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |         `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'A' xvalue
+  // CHECK-NEXT:  |     `-MaterializeTemporaryExpr {{.*}} 'A' xvalue extended by Var {{.*}} '__range1' 'A &&'
+  // CHECK-NEXT:  |       `-CXXBindTemporaryExpr {{.*}} 'A' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |         `-CXXMemberCallExpr {{.*}} 'A'
   // CHECK-NEXT:  |           `-MemberExpr {{.*}} '<bound member function type>' .g {{.*}}
-  // CHECK-NEXT:  |             `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A' lvalue
+  // CHECK-NEXT:  |             `-CXXMemberCallExpr {{.*}} 'A' lvalue
   // CHECK-NEXT:  |               `-MemberExpr {{.*}} '<bound member function type>' .r {{.*}}
-  // CHECK-NEXT:  |                 `-MaterializeTemporaryExpr {{.*}} 'A':'P2718R0::A' xvalue extended by Var {{.*}} '__range1' 'A &&'
-  // CHECK-NEXT:  |                   `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |                     `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |                 `-MaterializeTemporaryExpr {{.*}} 'A' xvalue extended by Var {{.*}} '__range1' 'A &&'
+  // CHECK-NEXT:  |                   `-CXXBindTemporaryExpr {{.*}} 'A' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                     `-CXXMemberCallExpr {{.*}} 'A'
   // CHECK-NEXT:  |                       `-MemberExpr {{.*}} '<bound member function type>' .g {{.*}}
-  // CHECK-NEXT:  |                         `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A' lvalue
+  // CHECK-NEXT:  |                         `-CXXMemberCallExpr {{.*}} 'A' lvalue
   // CHECK-NEXT:  |                           `-MemberExpr {{.*}} '<bound member function type>' .r {{.*}}
-  // CHECK-NEXT:  |                             `-MaterializeTemporaryExpr {{.*}} 'A':'P2718R0::A' xvalue extended by Var {{.*}} '__range1' 'A &&'
-  // CHECK-NEXT:  |                               `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |                                 `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |                             `-MaterializeTemporaryExpr {{.*}} 'A' xvalue extended by Var {{.*}} '__range1' 'A &&'
+  // CHECK-NEXT:  |                               `-CXXBindTemporaryExpr {{.*}} 'A' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                                 `-CXXMemberCallExpr {{.*}} 'A'
   // CHECK-NEXT:  |                                   `-MemberExpr {{.*}} '<bound member function type>' .g {{.*}}
-  // CHECK-NEXT:  |                                     `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A' lvalue
+  // CHECK-NEXT:  |                                     `-CXXMemberCallExpr {{.*}} 'A' lvalue
   // CHECK-NEXT:  |                                       `-MemberExpr {{.*}} '<bound member function type>' .r {{.*}}
-  // CHECK-NEXT:  |                                         `-MaterializeTemporaryExpr {{.*}} 'A':'P2718R0::A' xvalue extended by Var {{.*}} '__range1' 'A &&'
-  // CHECK-NEXT:  |                                           `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |                                             `-CallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |                                         `-MaterializeTemporaryExpr {{.*}} 'A' xvalue extended by Var {{.*}} '__range1' 'A &&'
+  // CHECK-NEXT:  |                                           `-CXXBindTemporaryExpr {{.*}} 'A' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                                             `-CallExpr {{.*}} 'A'
   // CHECK-NEXT:  |                                               `-ImplicitCastExpr {{.*}} 'A (*)()' <FunctionToPointerDecay>
   // CHECK-NEXT:  |                                                 `-DeclRefExpr {{.*}} 'A ()' lvalue Function {{.*}} 'g' 'A ()'
   for (auto e : g().r().g().r().g().r().g())
@@ -324,11 +324,11 @@ void test9() {
   // CHECK-NEXT:  |       `-CallExpr {{.*}} 'const A *'
   // CHECK-NEXT:  |         |-ImplicitCastExpr {{.*}} 'const A *(*)(const A &)' <FunctionToPointerDecay>
   // CHECK-NEXT:  |         | `-DeclRefExpr {{.*}} 'const A *(const A &)' lvalue Function {{.*}} 'dg2' 'const A *(const A &)'
-  // CHECK-NEXT:  |         `-ImplicitCastExpr {{.*}} 'const A':'const P2718R0::A' lvalue <DerivedToBase (A)>
-  // CHECK-NEXT:  |           `-MaterializeTemporaryExpr {{.*}} 'const B':'const P2718R0::B' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |             `-ImplicitCastExpr {{.*}} 'const B':'const P2718R0::B' <NoOp>
-  // CHECK-NEXT:  |               `-CXXBindTemporaryExpr {{.*}} 'B':'P2718R0::B' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |                 `-CXXTemporaryObjectExpr {{.*}} 'B':'P2718R0::B' 'void () noexcept(false)' zeroing
+  // CHECK-NEXT:  |         `-ImplicitCastExpr {{.*}} 'const A' lvalue <DerivedToBase (A)>
+  // CHECK-NEXT:  |           `-MaterializeTemporaryExpr {{.*}} 'const B' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |             `-ImplicitCastExpr {{.*}} 'const B' <NoOp>
+  // CHECK-NEXT:  |               `-CXXBindTemporaryExpr {{.*}} 'B' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                 `-CXXTemporaryObjectExpr {{.*}} 'B' 'void () noexcept(false)' zeroing
   for (auto e : df2(dg2(B())))
     bar(e);
 }
@@ -348,10 +348,10 @@ void test10() {
   // CHECK-NEXT:  |       | `-CallExpr {{.*}} 'const P2718R0::LockGuard' lvalue
   // CHECK-NEXT:  |       |   |-ImplicitCastExpr {{.*}} 'const P2718R0::LockGuard &(*)(const P2718R0::LockGuard &)' <FunctionToPointerDecay>
   // CHECK-NEXT:  |       |   | `-DeclRefExpr {{.*}} 'const P2718R0::LockGuard &(const P2718R0::LockGuard &)' lvalue Function {{.*}} 'df1' 'const P2718R0::LockGuard &(const P2718R0::LockGuard &)' (FunctionTemplate {{.*}} 'df1')
-  // CHECK-NEXT:  |       |   `-MaterializeTemporaryExpr {{.*}} 'const LockGuard':'const P2718R0::LockGuard' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |       |     `-ImplicitCastExpr {{.*}} 'const LockGuard':'const P2718R0::LockGuard' <NoOp>
-  // CHECK-NEXT:  |       |       `-CXXBindTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |       |         `-CXXTemporaryObjectExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' 'void ()'
+  // CHECK-NEXT:  |       |   `-MaterializeTemporaryExpr {{.*}} 'const LockGuard' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |       |     `-ImplicitCastExpr {{.*}} 'const LockGuard' <NoOp>
+  // CHECK-NEXT:  |       |       `-CXXBindTemporaryExpr {{.*}} 'LockGuard' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |       |         `-CXXTemporaryObjectExpr {{.*}} 'LockGuard' 'void ()'
   // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} 'int[3]' lvalue Var {{.*}} 'v' 'int[3]'
   for ([[maybe_unused]] int x : static_cast<void>(df1(LockGuard())), v)
     LockGuard guard;
@@ -366,10 +366,10 @@ void test10() {
   // CHECK-NEXT:  |       | `-CallExpr {{.*}} 'const P2718R0::LockGuard' lvalue
   // CHECK-NEXT:  |       |   |-ImplicitCastExpr {{.*}} 'const P2718R0::LockGuard &(*)(const P2718R0::LockGuard &)' <FunctionToPointerDecay>
   // CHECK-NEXT:  |       |   | `-DeclRefExpr {{.*}} 'const P2718R0::LockGuard &(const P2718R0::LockGuard &)' lvalue Function {{.*}} 'df1' 'const P2718R0::LockGuard &(const P2718R0::LockGuard &)' (FunctionTemplate {{.*}} 'df1')
-  // CHECK-NEXT:  |       |   `-MaterializeTemporaryExpr {{.*}} 'const LockGuard':'const P2718R0::LockGuard' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |       |     `-ImplicitCastExpr {{.*}} 'const LockGuard':'const P2718R0::LockGuard' <NoOp>
-  // CHECK-NEXT:  |       |       `-CXXBindTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |       |         `-CXXTemporaryObjectExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' 'void ()'
+  // CHECK-NEXT:  |       |   `-MaterializeTemporaryExpr {{.*}} 'const LockGuard' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |       |     `-ImplicitCastExpr {{.*}} 'const LockGuard' <NoOp>
+  // CHECK-NEXT:  |       |       `-CXXBindTemporaryExpr {{.*}} 'LockGuard' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |       |         `-CXXTemporaryObjectExpr {{.*}} 'LockGuard' 'void ()'
   // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} 'int[3]' lvalue Var {{.*}} 'v' 'int[3]'
   for ([[maybe_unused]] int x : (void)df1(LockGuard()), v)
     LockGuard guard;
@@ -384,17 +384,17 @@ void test10() {
   // CHECK-NEXT:  |       | |-CallExpr {{.*}} 'const P2718R0::LockGuard' lvalue
   // CHECK-NEXT:  |       | | |-ImplicitCastExpr {{.*}} 'const P2718R0::LockGuard &(*)(const P2718R0::LockGuard &)' <FunctionToPointerDecay>
   // CHECK-NEXT:  |       | | | `-DeclRefExpr {{.*}} 'const P2718R0::LockGuard &(const P2718R0::LockGuard &)' lvalue Function {{.*}} 'df1' 'const P2718R0::LockGuard &(const P2718R0::LockGuard &)' (FunctionTemplate {{.*}} 'df1')
-  // CHECK-NEXT:  |       | | `-MaterializeTemporaryExpr {{.*}} 'const LockGuard':'const P2718R0::LockGuard' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |       | |   `-ImplicitCastExpr {{.*}} 'const LockGuard':'const P2718R0::LockGuard' <NoOp>
-  // CHECK-NEXT:  |       | |     `-CXXBindTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |       | |       `-CXXTemporaryObjectExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' 'void ()'
+  // CHECK-NEXT:  |       | | `-MaterializeTemporaryExpr {{.*}} 'const LockGuard' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |       | |   `-ImplicitCastExpr {{.*}} 'const LockGuard' <NoOp>
+  // CHECK-NEXT:  |       | |     `-CXXBindTemporaryExpr {{.*}} 'LockGuard' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |       | |       `-CXXTemporaryObjectExpr {{.*}} 'LockGuard' 'void ()'
   // CHECK-NEXT:  |       | `-CallExpr {{.*}} 'const P2718R0::LockGuard' lvalue
   // CHECK-NEXT:  |       |   |-ImplicitCastExpr {{.*}} 'const P2718R0::LockGuard &(*)(const P2718R0::LockGuard &)' <FunctionToPointerDecay>
   // CHECK-NEXT:  |       |   | `-DeclRefExpr {{.*}} 'const P2718R0::LockGuard &(const P2718R0::LockGuard &)' lvalue Function {{.*}} 'df1' 'const P2718R0::LockGuard &(const P2718R0::LockGuard &)' (FunctionTemplate {{.*}} 'df1')
-  // CHECK-NEXT:  |       |   `-MaterializeTemporaryExpr {{.*}} 'const LockGuard':'const P2718R0::LockGuard' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
-  // CHECK-NEXT:  |       |     `-ImplicitCastExpr {{.*}} 'const LockGuard':'const P2718R0::LockGuard' <NoOp>
-  // CHECK-NEXT:  |       |       `-CXXBindTemporaryExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |       |         `-CXXTemporaryObjectExpr {{.*}} 'LockGuard':'P2718R0::LockGuard' 'void ()'
+  // CHECK-NEXT:  |       |   `-MaterializeTemporaryExpr {{.*}} 'const LockGuard' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |       |     `-ImplicitCastExpr {{.*}} 'const LockGuard' <NoOp>
+  // CHECK-NEXT:  |       |       `-CXXBindTemporaryExpr {{.*}} 'LockGuard' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |       |         `-CXXTemporaryObjectExpr {{.*}} 'LockGuard' 'void ()'
   // CHECK-NEXT:  |       `-DeclRefExpr {{.*}} 'int[3]' lvalue Var {{.*}} 'v' 'int[3]'
   for ([[maybe_unused]] int x : df1(LockGuard()), df1(LockGuard()), v)
     LockGuard guard;
@@ -403,7 +403,7 @@ void test10() {
 // Test default argument && dependent context
 template <typename T> int (&default_arg_fn2(const T & = T()))[3];
 void test11() {
-  for (auto e : default_arg_fn2<A>()) 
+  for (auto e : default_arg_fn2<A>())
     bar(e);
 }
 
@@ -422,24 +422,24 @@ void test13() {
   // CHECK-NEXT:  |-<<<NULL>>>
   // CHECK-NEXT:  |-DeclStmt {{.*}}
   // CHECK-NEXT:  | `-VarDecl {{.*}} implicit used __range1 'A &&' cinit
-  // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'A':'P2718R0::A' xvalue
-  // CHECK-NEXT:  |     `-MaterializeTemporaryExpr {{.*}} 'A':'P2718R0::A' xvalue extended by Var {{.*}} '__range1' 'A &&'
-  // CHECK-NEXT:  |       `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |         `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'A' xvalue
+  // CHECK-NEXT:  |     `-MaterializeTemporaryExpr {{.*}} 'A' xvalue extended by Var {{.*}} '__range1' 'A &&'
+  // CHECK-NEXT:  |       `-CXXBindTemporaryExpr {{.*}} 'A' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |         `-CXXMemberCallExpr {{.*}} 'A'
   // CHECK-NEXT:  |           `-MemberExpr {{.*}} '<bound member function type>' .g {{.*}}
-  // CHECK-NEXT:  |             `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A' lvalue
+  // CHECK-NEXT:  |             `-CXXMemberCallExpr {{.*}} 'A' lvalue
   // CHECK-NEXT:  |               `-MemberExpr {{.*}} '<bound member function type>' .r {{.*}}
-  // CHECK-NEXT:  |                 `-MaterializeTemporaryExpr {{.*}} 'A':'P2718R0::A' xvalue extended by Var {{.*}} '__range1' 'A &&'
-  // CHECK-NEXT:  |                   `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |                     `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |                 `-MaterializeTemporaryExpr {{.*}} 'A' xvalue extended by Var {{.*}} '__range1' 'A &&'
+  // CHECK-NEXT:  |                   `-CXXBindTemporaryExpr {{.*}} 'A' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                     `-CXXMemberCallExpr {{.*}} 'A'
   // CHECK-NEXT:  |                       `-MemberExpr {{.*}} '<bound member function type>' .g {{.*}}
-  // CHECK-NEXT:  |                         `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A' lvalue
+  // CHECK-NEXT:  |                         `-CXXMemberCallExpr {{.*}} 'A' lvalue
   // CHECK-NEXT:  |                           `-MemberExpr {{.*}} '<bound member function type>' .r {{.*}}
-  // CHECK-NEXT:  |                             `-MaterializeTemporaryExpr {{.*}} 'A':'P2718R0::A' xvalue extended by Var {{.*}} '__range1' 'A &&'
-  // CHECK-NEXT:  |                               `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |                                 `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A'
+  // CHECK-NEXT:  |                             `-MaterializeTemporaryExpr {{.*}} 'A' xvalue extended by Var {{.*}} '__range1' 'A &&'
+  // CHECK-NEXT:  |                               `-CXXBindTemporaryExpr {{.*}} 'A' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                                 `-CXXMemberCallExpr {{.*}} 'A'
   // CHECK-NEXT:  |                                   `-MemberExpr {{.*}} '<bound member function type>' .g {{.*}}
-  // CHECK-NEXT:  |                                     `-CXXMemberCallExpr {{.*}} 'A':'P2718R0::A' lvalue
+  // CHECK-NEXT:  |                                     `-CXXMemberCallExpr {{.*}} 'A' lvalue
   // CHECK-NEXT:  |                                       `-MemberExpr {{.*}} '<bound member function type>' .r {{.*}}
   // CHECK-NEXT:  |                                         `-MaterializeTemporaryExpr {{.*}} 'P2718R0::A' xvalue extended by Var {{.*}} '__range1' 'A &&'
   // CHECK-NEXT:  |                                           `-CXXBindTemporaryExpr {{.*}} 'P2718R0::A' (CXXTemporary {{.*}})
@@ -474,17 +474,17 @@ void test14() {
   // CHECK-NEXT:  | `-VarDecl {{.*}} implicit used __range1 'const int (&)[1]' cinit
   // CHECK-NEXT:  |   `-ExprWithCleanups {{.*}} 'const int[1]' lvalue
   // CHECK-NEXT:  |     `-MemberExpr {{.*}} 'const int[1]' lvalue .arr {{.*}}
-  // CHECK-NEXT:  |       `-MemberExpr {{.*}} 'const A14':'const P2718R0::A14' lvalue .a {{.*}}
-  // CHECK-NEXT:  |         `-MaterializeTemporaryExpr {{.*}} 'B14':'P2718R0::B14' xvalue extended by Var {{.*}} '__range1' 'const int (&)[1]'
-  // CHECK-NEXT:  |           `-CXXFunctionalCastExpr {{.*}} 'B14':'P2718R0::B14' functional cast to B14 <NoOp>
-  // CHECK-NEXT:  |             `-InitListExpr {{.*}} 'B14':'P2718R0::B14'
+  // CHECK-NEXT:  |       `-MemberExpr {{.*}} 'const A14' lvalue .a {{.*}}
+  // CHECK-NEXT:  |         `-MaterializeTemporaryExpr {{.*}} 'B14' xvalue extended by Var {{.*}} '__range1' 'const int (&)[1]'
+  // CHECK-NEXT:  |           `-CXXFunctionalCastExpr {{.*}} 'B14' functional cast to B14 <NoOp>
+  // CHECK-NEXT:  |             `-InitListExpr {{.*}} 'B14'
   // CHECK-NEXT:  |               |-IntegerLiteral {{.*}} 'int' 0
-  // CHECK-NEXT:  |               `-CXXDefaultInitExpr {{.*}} 'const A14':'const P2718R0::A14' lvalue has rewritten init
-  // CHECK-NEXT:  |                 `-MaterializeTemporaryExpr {{.*}} 'const A14':'const P2718R0::A14' lvalue extended by Var {{.*}} '__range1' 'const int (&)[1]'
-  // CHECK-NEXT:  |                   `-ImplicitCastExpr {{.*}} 'const A14':'const P2718R0::A14' <NoOp>
-  // CHECK-NEXT:  |                     `-CXXFunctionalCastExpr {{.*}} 'A14':'P2718R0::A14' functional cast to A14 <NoOp>
-  // CHECK-NEXT:  |                       `-CXXBindTemporaryExpr {{.*}} 'A14':'P2718R0::A14' (CXXTemporary {{.*}})
-  // CHECK-NEXT:  |                         `-InitListExpr {{.*}} 'A14':'P2718R0::A14'
+  // CHECK-NEXT:  |               `-CXXDefaultInitExpr {{.*}} 'const A14' lvalue has rewritten init
+  // CHECK-NEXT:  |                 `-MaterializeTemporaryExpr {{.*}} 'const A14' lvalue extended by Var {{.*}} '__range1' 'const int (&)[1]'
+  // CHECK-NEXT:  |                   `-ImplicitCastExpr {{.*}} 'const A14' <NoOp>
+  // CHECK-NEXT:  |                     `-CXXFunctionalCastExpr {{.*}} 'A14' functional cast to A14 <NoOp>
+  // CHECK-NEXT:  |                       `-CXXBindTemporaryExpr {{.*}} 'A14' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                         `-InitListExpr {{.*}} 'A14'
   // CHECK-NEXT:  |                           `-InitListExpr {{.*}} 'int[1]'
   // CHECK-NEXT:  |                             `-IntegerLiteral {{.*}} 'int' 0
   for (auto &&x : B14{0}.a.arr) { exit(0); }
@@ -493,17 +493,17 @@ void test14() {
   // CHECK-NEXT: |-<<<NULL>>>
   // CHECK-NEXT: |-DeclStmt {{.*}}
   // CHECK-NEXT: | `-VarDecl {{.*}} col:19 implicit used __range1 'B14 &&' cinit
-  // CHECK-NEXT: |   `-ExprWithCleanups {{.*}} 'B14':'P2718R0::B14' xvalue
-  // CHECK-NEXT: |     `-MaterializeTemporaryExpr {{.*}} 'B14':'P2718R0::B14' xvalue extended by Var {{.*}} '__range1' 'B14 &&'
-  // CHECK-NEXT: |       `-CXXFunctionalCastExpr {{.*}} 'B14':'P2718R0::B14' functional cast to B14 <NoOp>
-  // CHECK-NEXT: |         `-InitListExpr {{.*}} 'B14':'P2718R0::B14'
+  // CHECK-NEXT: |   `-ExprWithCleanups {{.*}} 'B14' xvalue
+  // CHECK-NEXT: |     `-MaterializeTemporaryExpr {{.*}} 'B14' xvalue extended by Var {{.*}} '__range1' 'B14 &&'
+  // CHECK-NEXT: |       `-CXXFunctionalCastExpr {{.*}} 'B14' functional cast to B14 <NoOp>
+  // CHECK-NEXT: |         `-InitListExpr {{.*}} 'B14'
   // CHECK-NEXT: |           |-IntegerLiteral {{.*}} 'int' 0
-  // CHECK-NEXT: |           `-CXXDefaultInitExpr {{.*}} 'const A14':'const P2718R0::A14' lvalue has rewritten init
-  // CHECK-NEXT: |             `-MaterializeTemporaryExpr {{.*}} 'const A14':'const P2718R0::A14' lvalue extended by Var {{.*}} '__range1' 'B14 &&'
-  // CHECK-NEXT: |               `-ImplicitCastExpr {{.*}} 'const A14':'const P2718R0::A14' <NoOp>
-  // CHECK-NEXT: |                 `-CXXFunctionalCastExpr {{.*}} 'A14':'P2718R0::A14' functional cast to A14 <NoOp>
-  // CHECK-NEXT: |                   `-CXXBindTemporaryExpr {{.*}} 'A14':'P2718R0::A14' (CXXTemporary {{.*}})
-  // CHECK-NEXT: |                     `-InitListExpr {{.*}} 'A14':'P2718R0::A14'
+  // CHECK-NEXT: |           `-CXXDefaultInitExpr {{.*}} 'const A14' lvalue has rewritten init
+  // CHECK-NEXT: |             `-MaterializeTemporaryExpr {{.*}} 'const A14' lvalue extended by Var {{.*}} '__range1' 'B14 &&'
+  // CHECK-NEXT: |               `-ImplicitCastExpr {{.*}} 'const A14' <NoOp>
+  // CHECK-NEXT: |                 `-CXXFunctionalCastExpr {{.*}} 'A14' functional cast to A14 <NoOp>
+  // CHECK-NEXT: |                   `-CXXBindTemporaryExpr {{.*}} 'A14' (CXXTemporary {{.*}})
+  // CHECK-NEXT: |                     `-InitListExpr {{.*}} 'A14'
   // CHECK-NEXT: |                       `-InitListExpr {{.*}} 'int[1]'
   // CHECK-NEXT: |                         `-IntegerLiteral {{.*}} 'int' 0
   for (auto &&x : B14{0}) { exit(0); }
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_reference.cpp b/clang/test/AST/ast-dump-openmp-begin-declare-variant_reference.cpp
index 1937a5d1c3eb3..9584e77b9e414 100644
--- a/clang/test/AST/ast-dump-openmp-begin-declare-variant_reference.cpp
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_reference.cpp
@@ -192,12 +192,12 @@ int test(float &&f, short &&s) {
 // CHECK-NEXT: | | `-CompoundStmt [[ADDR_44:0x[a-z0-9]*]] <col:17, line:14:1>
 // CHECK-NEXT: | |   |-DeclStmt [[ADDR_45:0x[a-z0-9]*]] <line:12:3, col:51>
 // CHECK-NEXT: | |   | `-TypedefDecl [[ADDR_46:0x[a-z0-9]*]] <col:3, col:48> col:48 referenced _Up 'typename remove_reference<float &>::type':'float'
-// CHECK-NEXT: | |   |   `-ElaboratedType [[ADDR_47:0x[a-z0-9]*]] 'typename remove_reference<float &>::type' sugar
-// CHECK-NEXT: | |   |     `-TypedefType [[ADDR_48:0x[a-z0-9]*]] 'remove_reference<float &>::type' sugar
-// CHECK-NEXT: | |   |       |-Typedef [[ADDR_10]] 'type'
-// CHECK-NEXT: | |   |       `-SubstTemplateTypeParmType [[ADDR_11]] 'float' sugar class depth 0 index 0 _Tp
-// CHECK-NEXT: | |   |         |-ClassTemplateSpecialization [[ADDR_6]] 'remove_reference'
-// CHECK-NEXT: | |   |         `-BuiltinType [[ADDR_8]] 'float'
+// CHECK-NEXT: | |   |   `-TypedefType [[ADDR_48:0x[a-z0-9]*]] 'typename remove_reference<float &>::type' sugar typename
+// CHECK-NEXT: | |   |     |-NestedNameSpecifier TypeSpec 'remove_reference<float &>'
+// CHECK-NEXT: | |   |     |-Typedef [[ADDR_10]] 'type'
+// CHECK-NEXT: | |   |     `-SubstTemplateTypeParmType [[ADDR_11]] 'float' sugar class depth 0 index 0 _Tp
+// CHECK-NEXT: | |   |       |-ClassTemplateSpecialization [[ADDR_6]] 'remove_reference'
+// CHECK-NEXT: | |   |       `-BuiltinType [[ADDR_8]] 'float'
 // CHECK-NEXT: | |   `-ReturnStmt [[ADDR_49:0x[a-z0-9]*]] <line:13:3, col:33>
 // CHECK-NEXT: | |     `-CXXStaticCastExpr [[ADDR_50:0x[a-z0-9]*]] <col:10, col:33> '_Up':'float' xvalue static_cast<_Up &&> <NoOp>
 // CHECK-NEXT: | |       `-DeclRefExpr [[ADDR_51:0x[a-z0-9]*]] <col:30> 'float' {{.*}}ParmVar [[ADDR_43]] '__t' 'float &'
@@ -209,12 +209,12 @@ int test(float &&f, short &&s) {
 // CHECK-NEXT: |   `-CompoundStmt [[ADDR_54:0x[a-z0-9]*]] <col:17, line:14:1>
 // CHECK-NEXT: |     |-DeclStmt [[ADDR_55:0x[a-z0-9]*]] <line:12:3, col:51>
 // CHECK-NEXT: |     | `-TypedefDecl [[ADDR_56:0x[a-z0-9]*]] <col:3, col:48> col:48 referenced _Up 'typename remove_reference<short &>::type':'short'
-// CHECK-NEXT: |     |   `-ElaboratedType [[ADDR_57:0x[a-z0-9]*]] 'typename remove_reference<short &>::type' sugar
-// CHECK-NEXT: |     |     `-TypedefType [[ADDR_58:0x[a-z0-9]*]] 'remove_reference<short &>::type' sugar
-// CHECK-NEXT: |     |       |-Typedef [[ADDR_18]] 'type'
-// CHECK-NEXT: |     |       `-SubstTemplateTypeParmType [[ADDR_19]] 'short' sugar class depth 0 index 0 _Tp
-// CHECK-NEXT: |     |         |-ClassTemplateSpecialization [[ADDR_14]] 'remove_reference'
-// CHECK-NEXT: |     |         `-BuiltinType [[ADDR_16]] 'short'
+// CHECK-NEXT: |     |   `-TypedefType [[ADDR_58:0x[a-z0-9]*]] 'typename remove_reference<short &>::type' sugar typename
+// CHECK-NEXT: |     |     |-NestedNameSpecifier TypeSpec 'remove_reference<short &>'
+// CHECK-NEXT: |     |     |-Typedef [[ADDR_18]] 'type'
+// CHECK-NEXT: |     |     `-SubstTemplateTypeParmType [[ADDR_19]] 'short' sugar class depth 0 index 0 _Tp
+// CHECK-NEXT: |     |       |-ClassTemplateSpecialization [[ADDR_14]] 'remove_reference'
+// CHECK-NEXT: |     |       `-BuiltinType [[ADDR_16]] 'short'
 // CHECK-NEXT: |     `-ReturnStmt [[ADDR_59:0x[a-z0-9]*]] <line:13:3, col:33>
 // CHECK-NEXT: |       `-CXXStaticCastExpr [[ADDR_60:0x[a-z0-9]*]] <col:10, col:33> '_Up':'short' xvalue static_cast<_Up &&> <NoOp>
 // CHECK-NEXT: |         `-DeclRefExpr [[ADDR_61:0x[a-z0-9]*]] <col:30> 'short' {{.*}}ParmVar [[ADDR_53]] '__t' 'short &'
diff --git a/clang/test/AST/ast-dump-openmp-begin-declare-variant_template_3.cpp b/clang/test/AST/ast-dump-openmp-begin-declare-variant_template_3.cpp
index ad269506042c8..44d1cb462cd58 100644
--- a/clang/test/AST/ast-dump-openmp-begin-declare-variant_template_3.cpp
+++ b/clang/test/AST/ast-dump-openmp-begin-declare-variant_template_3.cpp
@@ -54,7 +54,7 @@ int test() {
 // CHECK-NEXT: | | | |-CopyAssignment simple trivial has_const_param needs_implicit implicit_has_const_param
 // CHECK-NEXT: | | | |-MoveAssignment exists simple trivial needs_implicit
 // CHECK-NEXT: | | | `-Destructor simple irrelevant trivial {{(constexpr )?}}needs_implicit
-// CHECK-NEXT: | | |-CXXRecordDecl [[ADDR_3:0x[a-z0-9]*]] <col:23, col:30> col:30 implicit referenced struct S
+// CHECK-NEXT: | | |-CXXRecordDecl [[ADDR_3:0x[a-z0-9]*]] <col:23, col:30> col:30 implicit struct S
 // CHECK-NEXT: | | `-CXXConstructorDecl [[ADDR_4:0x[a-z0-9]*]] <line:6:3, col:16> col:3 S<T> 'void (int, T *)'
 // CHECK-NEXT: | |   |-ParmVarDecl [[ADDR_5:0x[a-z0-9]*]] <col:5> col:8 'int'
 // CHECK-NEXT: | |   |-ParmVarDecl [[ADDR_6:0x[a-z0-9]*]] <col:10, col:12> col:13 'T *'
diff --git a/clang/test/AST/ast-dump-record-definition-data-json.cpp b/clang/test/AST/ast-dump-record-definition-data-json.cpp
index c119089def716..e35bec78c6847 100644
--- a/clang/test/AST/ast-dump-record-definition-data-json.cpp
+++ b/clang/test/AST/ast-dump-record-definition-data-json.cpp
@@ -2516,7 +2516,6 @@ struct DoesNotAllowConstDefaultInit {
 // CHECK-NEXT:     }
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "isImplicit": true,
-// CHECK-NEXT:    "isReferenced": true,
 // CHECK-NEXT:    "name": "IsTrivial",
 // CHECK-NEXT:    "tagUsed": "struct"
 // CHECK-NEXT:   },
@@ -2646,7 +2645,6 @@ struct DoesNotAllowConstDefaultInit {
 // CHECK-NEXT:     }
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "isImplicit": true,
-// CHECK-NEXT:    "isReferenced": true,
 // CHECK-NEXT:    "name": "IsNotTrivial",
 // CHECK-NEXT:    "tagUsed": "struct"
 // CHECK-NEXT:   },
@@ -3980,7 +3978,6 @@ struct DoesNotAllowConstDefaultInit {
 // CHECK-NEXT:     }
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "isImplicit": true,
-// CHECK-NEXT:    "isReferenced": true,
 // CHECK-NEXT:    "name": "HasUserDeclaredConstructor",
 // CHECK-NEXT:    "tagUsed": "struct"
 // CHECK-NEXT:   },
@@ -4234,7 +4231,6 @@ struct DoesNotAllowConstDefaultInit {
 // CHECK-NEXT:     }
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "isImplicit": true,
-// CHECK-NEXT:    "isReferenced": true,
 // CHECK-NEXT:    "name": "HasConstexprNonCopyMoveConstructor",
 // CHECK-NEXT:    "tagUsed": "struct"
 // CHECK-NEXT:   },
@@ -4381,7 +4377,6 @@ struct DoesNotAllowConstDefaultInit {
 // CHECK-NEXT:     }
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "isImplicit": true,
-// CHECK-NEXT:    "isReferenced": true,
 // CHECK-NEXT:    "name": "HasNoConstexprNonCopyMoveConstructor",
 // CHECK-NEXT:    "tagUsed": "struct"
 // CHECK-NEXT:   },
diff --git a/clang/test/AST/ast-dump-records-json.cpp b/clang/test/AST/ast-dump-records-json.cpp
index 7efdcb0b61318..941c6a675448a 100644
--- a/clang/test/AST/ast-dump-records-json.cpp
+++ b/clang/test/AST/ast-dump-records-json.cpp
@@ -795,7 +795,6 @@ struct Derived6 : virtual public Bases... {
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "name": "b",
 // CHECK-NEXT:    "type": {
-// CHECK-NEXT:     "desugaredQualType": "C::(unnamed struct at {{.*}}:16:3)",
 // CHECK-NEXT:     "qualType": "struct (unnamed struct at {{.*}}:16:3)"
 // CHECK-NEXT:    }
 // CHECK-NEXT:   },
@@ -2072,7 +2071,6 @@ struct Derived6 : virtual public Bases... {
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "name": "b",
 // CHECK-NEXT:    "type": {
-// CHECK-NEXT:     "desugaredQualType": "G::(unnamed struct at {{.*}}:50:3)",
 // CHECK-NEXT:     "qualType": "struct (unnamed struct at {{.*}}:50:3)"
 // CHECK-NEXT:    }
 // CHECK-NEXT:   },
diff --git a/clang/test/AST/ast-dump-records.c b/clang/test/AST/ast-dump-records.c
index f4a540f3fa872..1dc175c4a8560 100644
--- a/clang/test/AST/ast-dump-records.c
+++ b/clang/test/AST/ast-dump-records.c
@@ -47,7 +47,7 @@ struct C {
     int a;
     // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:5, col:9> col:9 a 'int'
   } b;
-  // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-5]]:3, line:[[@LINE-1]]:5> col:5 b 'struct (unnamed struct at {{.*}}:[[@LINE-5]]:3)':'struct C::(unnamed at {{.*}}:[[@LINE-5]]:3)'
+  // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-5]]:3, line:[[@LINE-1]]:5> col:5 b 'struct (unnamed struct at {{.*}}:[[@LINE-5]]:3)'
 
   union {
     // CHECK-NEXT: RecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, line:[[@LINE+5]]:3> line:[[@LINE-1]]:3 union definition
@@ -122,15 +122,13 @@ union E {
 };
 
 union G {
-  // CHECK: RecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+38]]:1> line:[[@LINE-1]]:7 union G definition
+  // CHECK: RecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+36]]:1> line:[[@LINE-1]]:7 union G definition
   struct {
     // CHECK-NEXT: RecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, line:[[@LINE+3]]:3> line:[[@LINE-1]]:3 struct definition
     int a;
     // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:5, col:9> col:9 a 'int'
   } b;
-  // FIXME: note that it talks about 'struct G' below; the same happens in
-  // other cases with union G as well.
-  // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-7]]:3, line:[[@LINE-3]]:5> col:5 b 'struct (unnamed struct at {{.*}}:[[@LINE-7]]:3)':'struct G::(unnamed at {{.*}}:[[@LINE-7]]:3)'
+  // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-5]]:3, line:[[@LINE-1]]:5> col:5 b 'struct (unnamed struct at {{.*}}:[[@LINE-5]]:3)'
 
   union {
     // CHECK-NEXT: RecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, line:[[@LINE+5]]:3> line:[[@LINE-1]]:3 union definition
diff --git a/clang/test/AST/ast-dump-records.cpp b/clang/test/AST/ast-dump-records.cpp
index e9b37b73002dd..edd13ba1c6f12 100644
--- a/clang/test/AST/ast-dump-records.cpp
+++ b/clang/test/AST/ast-dump-records.cpp
@@ -72,7 +72,7 @@ struct C {
     int a;
     // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:5, col:9> col:9 a 'int'
   } b;
-  // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-12]]:3, line:[[@LINE-1]]:5> col:5 b 'struct (unnamed struct at {{.*}}:[[@LINE-12]]:3)':'C::(unnamed struct at {{.*}}:[[@LINE-12]]:3)'
+  // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-12]]:3, line:[[@LINE-1]]:5> col:5 b 'struct (unnamed struct at {{.*}}:[[@LINE-12]]:3)'
 
   union {
     // CHECK-NEXT: CXXRecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, line:[[@LINE+12]]:3> line:[[@LINE-1]]:3 union definition
@@ -179,7 +179,7 @@ union E {
 };
 
 union G {
-  // CHECK: CXXRecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+71]]:1> line:[[@LINE-1]]:7 union G definition
+  // CHECK: CXXRecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+69]]:1> line:[[@LINE-1]]:7 union G definition
   // CHECK-NEXT: DefinitionData pass_in_registers aggregate standard_layout trivially_copyable pod trivial literal
   // CHECK-NEXT: DefaultConstructor exists trivial needs_implicit
   // CHECK-NEXT: CopyConstructor simple trivial has_const_param needs_implicit implicit_has_const_param
@@ -202,9 +202,7 @@ union G {
     int a;
     // CHECK-NEXT: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:5, col:9> col:9 a 'int'
   } b;
-  // FIXME: note that it talks about 'struct G' below; the same happens in
-  // other cases with union G as well.
-  // CHECK: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-15]]:3, line:[[@LINE-3]]:5> col:5 b 'struct (unnamed struct at {{.*}}:[[@LINE-15]]:3)':'G::(unnamed struct at {{.*}}:[[@LINE-15]]:3)'
+  // CHECK: FieldDecl 0x{{[^ ]*}} <line:[[@LINE-13]]:3, line:[[@LINE-1]]:5> col:5 b 'struct (unnamed struct at {{.*}}:[[@LINE-13]]:3)'
 
   union {
     // CHECK-NEXT: CXXRecordDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:3, line:[[@LINE+13]]:3> line:[[@LINE-1]]:3 union definition
diff --git a/clang/test/AST/ast-dump-recovery.cpp b/clang/test/AST/ast-dump-recovery.cpp
index a8e30f1759e9f..060ae3b62b6f8 100644
--- a/clang/test/AST/ast-dump-recovery.cpp
+++ b/clang/test/AST/ast-dump-recovery.cpp
@@ -128,7 +128,7 @@ void test2(Foo2 f) {
   // CHECK-NEXT:   | `-DeclRefExpr {{.*}} 'f'
   // CHECK-NEXT: `-IntegerLiteral {{.*}} 'int' 1
   f.func(1);
-  // CHECK:      RecoveryExpr {{.*}} 'ForwardClass':'Foo2::ForwardClass'
+  // CHECK:      RecoveryExpr {{.*}} 'ForwardClass'
   // CHECK-NEXT: `-MemberExpr {{.*}} '<bound member function type>' .createFwd
   // CHECK-NEXT:   `-DeclRefExpr {{.*}} 'f'
   f.createFwd();
@@ -292,8 +292,8 @@ union U {
 // CHECK: FunctionDecl {{.*}} foo 'void ()'
 // CHECK-NEXT:    `-CompoundStmt {{.*}}
 // CHECK-NEXT:      `-DeclStmt {{.*}}
-// CHECK-NEXT:        `-VarDecl {{.*}} g 'U':'GH112560::U' listinit
-// CHECK-NEXT:          `-InitListExpr {{.*}} 'U':'GH112560::U' contains-errors field Field {{.*}} 'f' 'int'
+// CHECK-NEXT:        `-VarDecl {{.*}} g 'U' listinit
+// CHECK-NEXT:          `-InitListExpr {{.*}} 'U' contains-errors field Field {{.*}} 'f' 'int'
 // CHECK-NEXT:            `-CXXDefaultInitExpr {{.*}} 'int' contains-errors has rewritten init
 // CHECK-NEXT:              `-RecoveryExpr {{.*}} 'int' contains-errors
 // DISABLED-NOT: -RecoveryExpr {{.*}} contains-errors
diff --git a/clang/test/AST/ast-dump-stmt-json.cpp b/clang/test/AST/ast-dump-stmt-json.cpp
index a8f113ce6a3d4..ee99ab832c89f 100644
--- a/clang/test/AST/ast-dump-stmt-json.cpp
+++ b/clang/test/AST/ast-dump-stmt-json.cpp
@@ -224,7 +224,55 @@ void TestDependentGenericSelectionExpr(Ty T) {
 // CHECK-NEXT:   "type": {
 // CHECK-NEXT:    "qualType": "void ()"
 // CHECK-NEXT:   }
-// CHECK-NEXT:  }
+// CHECK-NEXT:  },
+// CHECK-NEXT:  "inner": [
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "FunctionDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 125,
+// CHECK-NEXT:     "line": 4,
+// CHECK-NEXT:     "col": 6,
+// CHECK-NEXT:     "tokLen": 8
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 120,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 4
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 137,
+// CHECK-NEXT:      "col": 18,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isUsed": true,
+// CHECK-NEXT:    "name": "function",
+// CHECK-NEXT:    "mangledName": "_ZN1n8functionEv",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "void ()"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CompoundStmt",
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 136,
+// CHECK-NEXT:        "col": 17,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 137,
+// CHECK-NEXT:        "col": 18,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      }
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   }
+// CHECK-NEXT:  ]
 // CHECK-NEXT: }
 
 
@@ -279,7 +327,37 @@ void TestDependentGenericSelectionExpr(Ty T) {
 // CHECK-NEXT:   "type": {
 // CHECK-NEXT:    "qualType": "int"
 // CHECK-NEXT:   }
-// CHECK-NEXT:  }
+// CHECK-NEXT:  },
+// CHECK-NEXT:  "inner": [
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "VarDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 143,
+// CHECK-NEXT:     "line": 5,
+// CHECK-NEXT:     "col": 5,
+// CHECK-NEXT:     "tokLen": 8
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 139,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 3
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 143,
+// CHECK-NEXT:      "col": 5,
+// CHECK-NEXT:      "tokLen": 8
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isUsed": true,
+// CHECK-NEXT:    "name": "Variable",
+// CHECK-NEXT:    "mangledName": "_ZN1n8VariableE",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "int"
+// CHECK-NEXT:    }
+// CHECK-NEXT:   }
+// CHECK-NEXT:  ]
 // CHECK-NEXT: }
 
 
@@ -1869,6 +1947,7 @@ void TestDependentGenericSelectionExpr(Ty T) {
 // CHECK-NEXT:  ]
 // CHECK-NEXT: }
 
+
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {},
@@ -1936,6 +2015,7 @@ void TestDependentGenericSelectionExpr(Ty T) {
 // CHECK-NEXT:  ]
 // CHECK-NEXT: }
 
+
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {},
@@ -2086,6 +2166,7 @@ void TestDependentGenericSelectionExpr(Ty T) {
 // CHECK-NEXT:  ]
 // CHECK-NEXT: }
 
+
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionDecl",
 // CHECK-NEXT:  "loc": {},
@@ -2153,6 +2234,7 @@ void TestDependentGenericSelectionExpr(Ty T) {
 // CHECK-NEXT:  ]
 // CHECK-NEXT: }
 
+
 // CHECK-NOT: {{^}}Dumping
 // CHECK:  "kind": "FunctionTemplateDecl",
 // CHECK-NEXT:  "loc": {
diff --git a/clang/test/AST/ast-dump-stmt.m b/clang/test/AST/ast-dump-stmt.m
index e0fc16b3fa825..b68f5b6bd1d3f 100644
--- a/clang/test/AST/ast-dump-stmt.m
+++ b/clang/test/AST/ast-dump-stmt.m
@@ -55,4 +55,4 @@ id TestCompoundLiteral(id a) {
 // CHECK:     FunctionDecl{{.*}}TestCompoundLiteral
 // CHECK:       ExprWithCleanups
 // CHECK-NEXT:    cleanup CompoundLiteralExpr
-// CHECK:           CompoundLiteralExpr{{.*}}'S' lvalue
+     // CHECK:           CompoundLiteralExpr{{.*}}'S':'struct S' lvalue
diff --git a/clang/test/AST/ast-dump-template-decls.cpp b/clang/test/AST/ast-dump-template-decls.cpp
index d5228d4667304..ba3e405eb18aa 100644
--- a/clang/test/AST/ast-dump-template-decls.cpp
+++ b/clang/test/AST/ast-dump-template-decls.cpp
@@ -115,8 +115,7 @@ template <class T> struct C {
 };
 using type2 = typename C<int>::type1<void>;
 // CHECK:      TypeAliasDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, col:42> col:7 type2 'typename C<int>::type1<void>':'void (int)'
-// CHECK-NEXT: ElaboratedType 0x{{[^ ]*}} 'typename C<int>::type1<void>' sugar
-// CHECK-NEXT: TemplateSpecializationType 0x{{[^ ]*}} 'type1<void>' sugar alias
+// CHECK-NEXT: TemplateSpecializationType 0x{{[^ ]*}} 'typename C<int>::type1<void>' sugar alias
 // CHECK-NEXT: name: 'C<int>::type1':'PR55886::C<int>::type1' qualified
 // CHECK-NEXT: NestedNameSpecifier TypeSpec 'C<int>':'PR55886::C<int>'
 // CHECK-NEXT: TypeAliasTemplateDecl {{.+}} type1
diff --git a/clang/test/AST/ast-dump-template-json-win32-mangler-crash.cpp b/clang/test/AST/ast-dump-template-json-win32-mangler-crash.cpp
index cc9a82cb7bfff..3e0877f131e94 100644
--- a/clang/test/AST/ast-dump-template-json-win32-mangler-crash.cpp
+++ b/clang/test/AST/ast-dump-template-json-win32-mangler-crash.cpp
@@ -537,28 +537,17 @@ int main()
 // CHECK-NEXT:        "inner": [
 // CHECK-NEXT:         {
 // CHECK-NEXT:          "id": "0x{{.*}}",
-// CHECK-NEXT:          "kind": "ElaboratedType",
+// CHECK-NEXT:          "kind": "InjectedClassNameType",
 // CHECK-NEXT:          "type": {
 // CHECK-NEXT:           "qualType": "integral_constant<_Ty, _Val>"
 // CHECK-NEXT:          },
 // CHECK-NEXT:          "isDependent": true,
 // CHECK-NEXT:          "isInstantiationDependent": true,
-// CHECK-NEXT:          "inner": [
-// CHECK-NEXT:           {
-// CHECK-NEXT:            "id": "0x{{.*}}",
-// CHECK-NEXT:            "kind": "InjectedClassNameType",
-// CHECK-NEXT:            "type": {
-// CHECK-NEXT:             "qualType": "integral_constant<_Ty, _Val>"
-// CHECK-NEXT:            },
-// CHECK-NEXT:            "isDependent": true,
-// CHECK-NEXT:            "isInstantiationDependent": true,
-// CHECK-NEXT:            "decl": {
-// CHECK-NEXT:             "id": "0x{{.*}}",
-// CHECK-NEXT:             "kind": "CXXRecordDecl",
-// CHECK-NEXT:             "name": "integral_constant"
-// CHECK-NEXT:            }
-// CHECK-NEXT:           }
-// CHECK-NEXT:          ]
+// CHECK-NEXT:          "decl": {
+// CHECK-NEXT:           "id": "0x{{.*}}",
+// CHECK-NEXT:           "kind": "CXXRecordDecl",
+// CHECK-NEXT:           "name": "integral_constant"
+// CHECK-NEXT:          }
 // CHECK-NEXT:         }
 // CHECK-NEXT:        ]
 // CHECK-NEXT:       },
@@ -885,71 +874,60 @@ int main()
 // CHECK-NEXT:      "inner": [
 // CHECK-NEXT:       {
 // CHECK-NEXT:        "id": "0x{{.*}}",
-// CHECK-NEXT:        "kind": "ElaboratedType",
+// CHECK-NEXT:        "kind": "TemplateSpecializationType",
 // CHECK-NEXT:        "type": {
 // CHECK-NEXT:         "qualType": "integral_constant<bool, _Val>"
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "isDependent": true,
 // CHECK-NEXT:        "isInstantiationDependent": true,
+// CHECK-NEXT:        "templateName": "integral_constant",
 // CHECK-NEXT:        "inner": [
 // CHECK-NEXT:         {
-// CHECK-NEXT:          "id": "0x{{.*}}",
-// CHECK-NEXT:          "kind": "TemplateSpecializationType",
+// CHECK-NEXT:          "kind": "TemplateArgument",
 // CHECK-NEXT:          "type": {
-// CHECK-NEXT:           "qualType": "integral_constant<bool, _Val>"
+// CHECK-NEXT:           "qualType": "bool"
 // CHECK-NEXT:          },
-// CHECK-NEXT:          "isDependent": true,
-// CHECK-NEXT:          "isInstantiationDependent": true,
-// CHECK-NEXT:          "templateName": "integral_constant",
 // CHECK-NEXT:          "inner": [
 // CHECK-NEXT:           {
-// CHECK-NEXT:            "kind": "TemplateArgument",
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "BuiltinType",
 // CHECK-NEXT:            "type": {
 // CHECK-NEXT:             "qualType": "bool"
-// CHECK-NEXT:            },
-// CHECK-NEXT:            "inner": [
-// CHECK-NEXT:             {
-// CHECK-NEXT:              "id": "0x{{.*}}",
-// CHECK-NEXT:              "kind": "BuiltinType",
-// CHECK-NEXT:              "type": {
-// CHECK-NEXT:               "qualType": "bool"
-// CHECK-NEXT:              }
-// CHECK-NEXT:             }
-// CHECK-NEXT:            ]
-// CHECK-NEXT:           },
+// CHECK-NEXT:            }
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         },
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "kind": "TemplateArgument",
+// CHECK-NEXT:          "isExpr": true,
+// CHECK-NEXT:          "inner": [
 // CHECK-NEXT:           {
-// CHECK-NEXT:            "kind": "TemplateArgument",
-// CHECK-NEXT:            "isExpr": true,
-// CHECK-NEXT:            "inner": [
-// CHECK-NEXT:             {
-// CHECK-NEXT:              "id": "0x{{.*}}",
-// CHECK-NEXT:              "kind": "DeclRefExpr",
-// CHECK-NEXT:              "range": {
-// CHECK-NEXT:               "begin": {
-// CHECK-NEXT:                "offset": 554,
-// CHECK-NEXT:                "col": 47,
-// CHECK-NEXT:                "tokLen": 4
-// CHECK-NEXT:               },
-// CHECK-NEXT:               "end": {
-// CHECK-NEXT:                "offset": 554,
-// CHECK-NEXT:                "col": 47,
-// CHECK-NEXT:                "tokLen": 4
-// CHECK-NEXT:               }
-// CHECK-NEXT:              },
-// CHECK-NEXT:              "type": {
-// CHECK-NEXT:               "qualType": "bool"
-// CHECK-NEXT:              },
-// CHECK-NEXT:              "valueCategory": "prvalue",
-// CHECK-NEXT:              "referencedDecl": {
-// CHECK-NEXT:               "id": "0x{{.*}}",
-// CHECK-NEXT:               "kind": "NonTypeTemplateParmDecl",
-// CHECK-NEXT:               "name": "_Val",
-// CHECK-NEXT:               "type": {
-// CHECK-NEXT:                "qualType": "bool"
-// CHECK-NEXT:               }
-// CHECK-NEXT:              }
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "DeclRefExpr",
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 554,
+// CHECK-NEXT:              "col": 47,
+// CHECK-NEXT:              "tokLen": 4
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 554,
+// CHECK-NEXT:              "col": 47,
+// CHECK-NEXT:              "tokLen": 4
 // CHECK-NEXT:             }
-// CHECK-NEXT:            ]
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "bool"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "valueCategory": "prvalue",
+// CHECK-NEXT:            "referencedDecl": {
+// CHECK-NEXT:             "id": "0x{{.*}}",
+// CHECK-NEXT:             "kind": "NonTypeTemplateParmDecl",
+// CHECK-NEXT:             "name": "_Val",
+// CHECK-NEXT:             "type": {
+// CHECK-NEXT:              "qualType": "bool"
+// CHECK-NEXT:             }
+// CHECK-NEXT:            }
 // CHECK-NEXT:           }
 // CHECK-NEXT:          ]
 // CHECK-NEXT:         }
diff --git a/clang/test/AST/ast-dump-template-name.cpp b/clang/test/AST/ast-dump-template-name.cpp
index acacdac857954..7f085086ab6a3 100644
--- a/clang/test/AST/ast-dump-template-name.cpp
+++ b/clang/test/AST/ast-dump-template-name.cpp
@@ -11,13 +11,12 @@ namespace qualified {
 
 // CHECK:      Dumping qualified::TestQualified:
 // CHECK-NEXT: TypeAliasDecl
-// CHECK-NEXT: `-ElaboratedType
-// CHECK-NEXT:   `-TemplateSpecializationType
-// CHECK-NEXT:     |-name: 'N' qualified
-// CHECK-NEXT:     | `-TypeAliasTemplateDecl {{.+}} N{{$}}
-// CHECK-NEXT:     |-TemplateArgument template 'foo::A':'qualified::foo::A' qualified{{$}}
-// CHECK-NEXT:     | |-NestedNameSpecifier Namespace 0x{{.+}} 'foo'{{$}}
-// CHECK-NEXT:     | `-ClassTemplateDecl {{.+}} A{{$}}
+// CHECK-NEXT: `-TemplateSpecializationType
+// CHECK-NEXT:   |-name: 'N' qualified
+// CHECK-NEXT:   | `-TypeAliasTemplateDecl {{.+}} N{{$}}
+// CHECK-NEXT:   |-TemplateArgument template 'foo::A':'qualified::foo::A' qualified{{$}}
+// CHECK-NEXT:   | |-NestedNameSpecifier Namespace 0x{{.+}} 'foo'{{$}}
+// CHECK-NEXT:   | `-ClassTemplateDecl {{.+}} A{{$}}
 
 namespace dependent {
   template <class T> struct B {
@@ -27,12 +26,11 @@ namespace dependent {
 
 // CHECK:      Dumping dependent::B::TestDependent:
 // CHECK-NEXT: TypeAliasDecl
-// CHECK-NEXT: `-ElaboratedType
-// CHECK-NEXT:   `-TemplateSpecializationType
-// CHECK-NEXT:     |-name: 'N' qualified
-// CHECK-NEXT:     | `-TypeAliasTemplateDecl
-// CHECK-NEXT:     |-TemplateArgument template 'T::template X':'type-parameter-0-0::template X' dependent{{$}}
-// CHECK-NEXT:     | `-NestedNameSpecifier TypeSpec 'T'{{$}}
+// CHECK-NEXT: `-TemplateSpecializationType
+// CHECK-NEXT:   |-name: 'N' qualified
+// CHECK-NEXT:   | `-TypeAliasTemplateDecl
+// CHECK-NEXT:   |-TemplateArgument template 'T::template X':'type-parameter-0-0::template X' dependent{{$}}
+// CHECK-NEXT:   | `-NestedNameSpecifier TypeSpec 'T'{{$}}
 
 namespace subst {
   template <class> struct A;
@@ -46,15 +44,14 @@ namespace subst {
 
 // CHECK:      Dumping subst::TestSubst:
 // CHECK-NEXT: TypeAliasDecl
-// CHECK-NEXT: `-ElaboratedType
-// CHECK-NEXT:   `-TypedefType
-// CHECK-NEXT:     |-TypeAlias
-// CHECK-NEXT:     `-ElaboratedType
-// CHECK-NEXT:       `-TemplateSpecializationType
-// CHECK-NEXT:         |-name: 'C':'subst::B<subst::A>::C' qualified
-// CHECK-NEXT:         | `-ClassTemplateDecl {{.+}} C
-// CHECK-NEXT:         |-TemplateArgument template 'subst::A' subst index 0
-// CHECK-NEXT:         | |-parameter: TemplateTemplateParmDecl {{.+}} depth 0 index 0 TT{{$}}
-// CHECK-NEXT:         | |-associated ClassTemplateSpecialization {{.+}} 'B'{{$}}
-// CHECK-NEXT:         | `-replacement:
-// CHECK-NEXT:         |   `-ClassTemplateDecl {{.+}} A{{$}}
+// CHECK-NEXT: `-TypedefType
+// CHECK-NEXT:   |-NestedNameSpecifier TypeSpec 'B<A>':'subst::B<subst::A>'
+// CHECK-NEXT:   |-TypeAlias
+// CHECK-NEXT:   `-TemplateSpecializationType
+// CHECK-NEXT:     |-name: 'C':'subst::B<subst::A>::C' qualified
+// CHECK-NEXT:     | `-ClassTemplateDecl {{.+}} C
+// CHECK-NEXT:     |-TemplateArgument template 'subst::A' subst index 0
+// CHECK-NEXT:     | |-parameter: TemplateTemplateParmDecl {{.+}} depth 0 index 0 TT{{$}}
+// CHECK-NEXT:     | |-associated ClassTemplateSpecialization {{.+}} 'B'{{$}}
+// CHECK-NEXT:     | `-replacement:
+// CHECK-NEXT:     |   `-ClassTemplateDecl {{.+}} A{{$}}
diff --git a/clang/test/AST/ast-dump-templates.cpp b/clang/test/AST/ast-dump-templates.cpp
index b504accb927dc..a842eabf6962f 100644
--- a/clang/test/AST/ast-dump-templates.cpp
+++ b/clang/test/AST/ast-dump-templates.cpp
@@ -171,10 +171,11 @@ namespace TestDependentMemberPointer {
 // DUMP-NEXT:  |   `-BuiltinType {{.+}} 'int'
 // DUMP-NEXT:  |-TypeAliasDecl {{.+}} Y 'int U::test::*'{{$}}
 // DUMP-NEXT:  | `-MemberPointerType {{.+}} 'int U::test::*' dependent
+// DUMP-NEXT:  |   |-DependentNameType {{.+}} 'U::test' dependent
 // DUMP-NEXT:  |   `-BuiltinType {{.+}} 'int'
 // DUMP-NEXT:  `-TypeAliasDecl {{.+}} Z 'int U::template V<int>::*'{{$}}
 // DUMP-NEXT:    `-MemberPointerType {{.+}} 'int U::template V<int>::*' dependent
-// DUMP-NEXT:      |-DependentTemplateSpecializationType {{.+}} 'template V<int>' dependent
+// DUMP-NEXT:      |-DependentTemplateSpecializationType {{.+}} 'U::template V<int>' dependent
 // DUMP-NEXT:      `-BuiltinType {{.+}} 'int'
 } // namespace TestDependentMemberPointer
 
@@ -186,14 +187,14 @@ namespace TestPartialSpecNTTP {
   template <class U1, bool U2, bool U3>
   struct Template2<Template1<U1, U2>, U3> {};
 // DUMP:      ClassTemplatePartialSpecializationDecl {{.+}} struct Template2
-// DUMP:      |-TemplateArgument type 'Template1<type-parameter-0-0, value-parameter-0-1>'
-// DUMP-NEXT: | `-TemplateSpecializationType {{.+}} 'Template1<type-parameter-0-0, value-parameter-0-1>' dependent
+// DUMP:      |-TemplateArgument type 'TestPartialSpecNTTP::Template1<type-parameter-0-0, value-parameter-0-1>'
+// DUMP-NEXT: | `-TemplateSpecializationType {{.+}} 'TestPartialSpecNTTP::Template1<type-parameter-0-0, value-parameter-0-1>' dependent
 // DUMP-NEXT: |   |-name: 'TestPartialSpecNTTP::Template1'
 // DUMP-NEXT: |   | `-ClassTemplateDecl {{.+}} Template1
 // DUMP-NEXT: |   |-TemplateArgument type 'type-parameter-0-0'
 // DUMP-NEXT: |   | `-TemplateTypeParmType {{.+}} 'type-parameter-0-0' dependent depth 0 index 0
 // DUMP-NEXT: |   `-TemplateArgument expr canonical 'value-parameter-0-1'
-// DUMP-NEXT: |     `-DeclRefExpr {{.+}} 'bool' NonTypeTemplateParm {{.+}} 'TA2' 'bool'
+// DUMP-NEXT: |     `-DeclRefExpr {{.+}} 'bool' NonTypeTemplateParm {{.+}} 'U2' 'bool'
 // DUMP-NEXT: |-TemplateArgument expr canonical 'value-parameter-0-2'
 // DUMP-NEXT: | `-DeclRefExpr {{.+}} 'bool' NonTypeTemplateParm {{.+}} 'U3' 'bool'
 // DUMP-NEXT: |-TemplateTypeParmDecl {{.+}} referenced class depth 0 index 0 U1
@@ -204,8 +205,8 @@ namespace TestPartialSpecNTTP {
   template <typename U1, bool U3, bool U2>
   struct Template2<Template1<U1, U2>, U3> {};
 // DUMP:      ClassTemplatePartialSpecializationDecl {{.+}} struct Template2 definition explicit_specialization
-// DUMP:      |-TemplateArgument type 'Template1<type-parameter-0-0, value-parameter-0-2>'
-// DUMP-NEXT: | `-TemplateSpecializationType {{.+}} 'Template1<type-parameter-0-0, value-parameter-0-2>' dependent
+// DUMP:      |-TemplateArgument type 'TestPartialSpecNTTP::Template1<type-parameter-0-0, value-parameter-0-2>'
+// DUMP-NEXT: | `-TemplateSpecializationType {{.+}} 'TestPartialSpecNTTP::Template1<type-parameter-0-0, value-parameter-0-2>' dependent
 // DUMP-NEXT: |   |-name: 'TestPartialSpecNTTP::Template1'
 // DUMP-NEXT: |   | `-ClassTemplateDecl {{.+}} Template1
 // DUMP-NEXT: |   |-TemplateArgument type 'type-parameter-0-0'
@@ -220,6 +221,22 @@ namespace TestPartialSpecNTTP {
 // DUMP-NEXT: `-CXXRecordDecl {{.+}} implicit struct Template2
 } // namespace TestPartialSpecNTTP
 
+namespace GH153540 {
+// DUMP-LABEL: NamespaceDecl {{.*}} GH153540{{$}}
+
+  namespace N {
+    template<typename T> struct S { S(T); };
+  }
+  void f() {
+    N::S(0);
+  }
+
+// DUMP:      FunctionDecl {{.*}} f 'void ()'
+// DUMP-NEXT: CompoundStmt
+// DUMP-NEXT: CXXFunctionalCastExpr {{.*}} 'N::S<int>':'GH153540::N::S<int>'
+// DUMP-NEXT: CXXConstructExpr {{.*}} <col:5, col:11> 'N::S<int>':'GH153540::N::S<int>' 'void (int)'
+} // namespace GH153540
+
 // NOTE: CHECK lines have been autogenerated by gen_ast_dump_json_test.py
 
 
@@ -621,7 +638,6 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:         }
 // JSON-NEXT:        },
 // JSON-NEXT:        "isImplicit": true,
-// JSON-NEXT:        "isReferenced": true,
 // JSON-NEXT:        "name": "foo",
 // JSON-NEXT:        "tagUsed": "struct"
 // JSON-NEXT:       },
@@ -4109,7 +4125,7 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "isImplicit": true,
 // JSON-NEXT:        "name": "<deduction guide for A>",
 // JSON-NEXT:        "type": {
-// JSON-NEXT:         "qualType": "auto () -> A<T>"
+// JSON-NEXT:         "qualType": "auto () -> test3::A<T>"
 // JSON-NEXT:        }
 // JSON-NEXT:       }
 // JSON-NEXT:      ]
@@ -4185,7 +4201,7 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "isImplicit": true,
 // JSON-NEXT:        "name": "<deduction guide for A>",
 // JSON-NEXT:        "type": {
-// JSON-NEXT:         "qualType": "auto (A<T>) -> A<T>"
+// JSON-NEXT:         "qualType": "auto (test3::A<T>) -> test3::A<T>"
 // JSON-NEXT:        },
 // JSON-NEXT:        "inner": [
 // JSON-NEXT:         {
@@ -4209,7 +4225,7 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:           }
 // JSON-NEXT:          },
 // JSON-NEXT:          "type": {
-// JSON-NEXT:           "qualType": "A<T>"
+// JSON-NEXT:           "qualType": "test3::A<T>"
 // JSON-NEXT:          }
 // JSON-NEXT:         }
 // JSON-NEXT:        ]
@@ -6630,8 +6646,8 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:      "tokLen": 9
 // JSON-NEXT:     },
 // JSON-NEXT:     "end": {
-// JSON-NEXT:      "offset": 6356,
-// JSON-NEXT:      "line": 179,
+// JSON-NEXT:      "offset": 6425,
+// JSON-NEXT:      "line": 180,
 // JSON-NEXT:      "col": 1,
 // JSON-NEXT:      "tokLen": 1
 // JSON-NEXT:     }
@@ -6889,6 +6905,15 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:            "inner": [
 // JSON-NEXT:             {
 // JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "DependentNameType",
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "U::test"
+// JSON-NEXT:              },
+// JSON-NEXT:              "isDependent": true,
+// JSON-NEXT:              "isInstantiationDependent": true
+// JSON-NEXT:             },
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
 // JSON-NEXT:              "kind": "BuiltinType",
 // JSON-NEXT:              "type": {
 // JSON-NEXT:               "qualType": "int"
@@ -6938,7 +6963,7 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:              "id": "0x{{.*}}",
 // JSON-NEXT:              "kind": "DependentTemplateSpecializationType",
 // JSON-NEXT:              "type": {
-// JSON-NEXT:               "qualType": "template V<int>"
+// JSON-NEXT:               "qualType": "U::template V<int>"
 // JSON-NEXT:              },
 // JSON-NEXT:              "isDependent": true,
 // JSON-NEXT:              "isInstantiationDependent": true
@@ -6964,20 +6989,20 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:    "id": "0x{{.*}}",
 // JSON-NEXT:    "kind": "NamespaceDecl",
 // JSON-NEXT:    "loc": {
-// JSON-NEXT:     "offset": 6409,
-// JSON-NEXT:     "line": 181,
+// JSON-NEXT:     "offset": 6478,
+// JSON-NEXT:     "line": 182,
 // JSON-NEXT:     "col": 11,
 // JSON-NEXT:     "tokLen": 19
 // JSON-NEXT:    },
 // JSON-NEXT:    "range": {
 // JSON-NEXT:     "begin": {
-// JSON-NEXT:      "offset": 6399,
+// JSON-NEXT:      "offset": 6468,
 // JSON-NEXT:      "col": 1,
 // JSON-NEXT:      "tokLen": 9
 // JSON-NEXT:     },
 // JSON-NEXT:     "end": {
-// JSON-NEXT:      "offset": 9184,
-// JSON-NEXT:      "line": 221,
+// JSON-NEXT:      "offset": 9336,
+// JSON-NEXT:      "line": 222,
 // JSON-NEXT:      "col": 1,
 // JSON-NEXT:      "tokLen": 1
 // JSON-NEXT:     }
@@ -6988,19 +7013,19 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:      "id": "0x{{.*}}",
 // JSON-NEXT:      "kind": "ClassTemplateDecl",
 // JSON-NEXT:      "loc": {
-// JSON-NEXT:       "offset": 6532,
-// JSON-NEXT:       "line": 183,
+// JSON-NEXT:       "offset": 6601,
+// JSON-NEXT:       "line": 184,
 // JSON-NEXT:       "col": 41,
 // JSON-NEXT:       "tokLen": 9
 // JSON-NEXT:      },
 // JSON-NEXT:      "range": {
 // JSON-NEXT:       "begin": {
-// JSON-NEXT:        "offset": 6494,
+// JSON-NEXT:        "offset": 6563,
 // JSON-NEXT:        "col": 3,
 // JSON-NEXT:        "tokLen": 8
 // JSON-NEXT:       },
 // JSON-NEXT:       "end": {
-// JSON-NEXT:        "offset": 6543,
+// JSON-NEXT:        "offset": 6612,
 // JSON-NEXT:        "col": 52,
 // JSON-NEXT:        "tokLen": 1
 // JSON-NEXT:       }
@@ -7011,18 +7036,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "TemplateTypeParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6510,
+// JSON-NEXT:         "offset": 6579,
 // JSON-NEXT:         "col": 19,
 // JSON-NEXT:         "tokLen": 3
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6504,
+// JSON-NEXT:          "offset": 6573,
 // JSON-NEXT:          "col": 13,
 // JSON-NEXT:          "tokLen": 5
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6510,
+// JSON-NEXT:          "offset": 6579,
 // JSON-NEXT:          "col": 19,
 // JSON-NEXT:          "tokLen": 3
 // JSON-NEXT:         }
@@ -7036,18 +7061,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6520,
+// JSON-NEXT:         "offset": 6589,
 // JSON-NEXT:         "col": 29,
 // JSON-NEXT:         "tokLen": 3
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6515,
+// JSON-NEXT:          "offset": 6584,
 // JSON-NEXT:          "col": 24,
 // JSON-NEXT:          "tokLen": 4
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6520,
+// JSON-NEXT:          "offset": 6589,
 // JSON-NEXT:          "col": 29,
 // JSON-NEXT:          "tokLen": 3
 // JSON-NEXT:         }
@@ -7063,18 +7088,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "CXXRecordDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6532,
+// JSON-NEXT:         "offset": 6601,
 // JSON-NEXT:         "col": 41,
 // JSON-NEXT:         "tokLen": 9
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6525,
+// JSON-NEXT:          "offset": 6594,
 // JSON-NEXT:          "col": 34,
 // JSON-NEXT:          "tokLen": 6
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6543,
+// JSON-NEXT:          "offset": 6612,
 // JSON-NEXT:          "col": 52,
 // JSON-NEXT:          "tokLen": 1
 // JSON-NEXT:         }
@@ -7137,18 +7162,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:          "id": "0x{{.*}}",
 // JSON-NEXT:          "kind": "CXXRecordDecl",
 // JSON-NEXT:          "loc": {
-// JSON-NEXT:           "offset": 6532,
+// JSON-NEXT:           "offset": 6601,
 // JSON-NEXT:           "col": 41,
 // JSON-NEXT:           "tokLen": 9
 // JSON-NEXT:          },
 // JSON-NEXT:          "range": {
 // JSON-NEXT:           "begin": {
-// JSON-NEXT:            "offset": 6525,
+// JSON-NEXT:            "offset": 6594,
 // JSON-NEXT:            "col": 34,
 // JSON-NEXT:            "tokLen": 6
 // JSON-NEXT:           },
 // JSON-NEXT:           "end": {
-// JSON-NEXT:            "offset": 6532,
+// JSON-NEXT:            "offset": 6601,
 // JSON-NEXT:            "col": 41,
 // JSON-NEXT:            "tokLen": 9
 // JSON-NEXT:           }
@@ -7165,19 +7190,19 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:      "id": "0x{{.*}}",
 // JSON-NEXT:      "kind": "ClassTemplateDecl",
 // JSON-NEXT:      "loc": {
-// JSON-NEXT:       "offset": 6586,
-// JSON-NEXT:       "line": 184,
+// JSON-NEXT:       "offset": 6655,
+// JSON-NEXT:       "line": 185,
 // JSON-NEXT:       "col": 41,
 // JSON-NEXT:       "tokLen": 9
 // JSON-NEXT:      },
 // JSON-NEXT:      "range": {
 // JSON-NEXT:       "begin": {
-// JSON-NEXT:        "offset": 6548,
+// JSON-NEXT:        "offset": 6617,
 // JSON-NEXT:        "col": 3,
 // JSON-NEXT:        "tokLen": 8
 // JSON-NEXT:       },
 // JSON-NEXT:       "end": {
-// JSON-NEXT:        "offset": 6597,
+// JSON-NEXT:        "offset": 6666,
 // JSON-NEXT:        "col": 52,
 // JSON-NEXT:        "tokLen": 1
 // JSON-NEXT:       }
@@ -7188,18 +7213,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "TemplateTypeParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6564,
+// JSON-NEXT:         "offset": 6633,
 // JSON-NEXT:         "col": 19,
 // JSON-NEXT:         "tokLen": 3
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6558,
+// JSON-NEXT:          "offset": 6627,
 // JSON-NEXT:          "col": 13,
 // JSON-NEXT:          "tokLen": 5
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6564,
+// JSON-NEXT:          "offset": 6633,
 // JSON-NEXT:          "col": 19,
 // JSON-NEXT:          "tokLen": 3
 // JSON-NEXT:         }
@@ -7213,18 +7238,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6574,
+// JSON-NEXT:         "offset": 6643,
 // JSON-NEXT:         "col": 29,
 // JSON-NEXT:         "tokLen": 3
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6569,
+// JSON-NEXT:          "offset": 6638,
 // JSON-NEXT:          "col": 24,
 // JSON-NEXT:          "tokLen": 4
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6574,
+// JSON-NEXT:          "offset": 6643,
 // JSON-NEXT:          "col": 29,
 // JSON-NEXT:          "tokLen": 3
 // JSON-NEXT:         }
@@ -7240,18 +7265,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "CXXRecordDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6586,
+// JSON-NEXT:         "offset": 6655,
 // JSON-NEXT:         "col": 41,
 // JSON-NEXT:         "tokLen": 9
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6579,
+// JSON-NEXT:          "offset": 6648,
 // JSON-NEXT:          "col": 34,
 // JSON-NEXT:          "tokLen": 6
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6597,
+// JSON-NEXT:          "offset": 6666,
 // JSON-NEXT:          "col": 52,
 // JSON-NEXT:          "tokLen": 1
 // JSON-NEXT:         }
@@ -7314,18 +7339,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:          "id": "0x{{.*}}",
 // JSON-NEXT:          "kind": "CXXRecordDecl",
 // JSON-NEXT:          "loc": {
-// JSON-NEXT:           "offset": 6586,
+// JSON-NEXT:           "offset": 6655,
 // JSON-NEXT:           "col": 41,
 // JSON-NEXT:           "tokLen": 9
 // JSON-NEXT:          },
 // JSON-NEXT:          "range": {
 // JSON-NEXT:           "begin": {
-// JSON-NEXT:            "offset": 6579,
+// JSON-NEXT:            "offset": 6648,
 // JSON-NEXT:            "col": 34,
 // JSON-NEXT:            "tokLen": 6
 // JSON-NEXT:           },
 // JSON-NEXT:           "end": {
-// JSON-NEXT:            "offset": 6586,
+// JSON-NEXT:            "offset": 6655,
 // JSON-NEXT:            "col": 41,
 // JSON-NEXT:            "tokLen": 9
 // JSON-NEXT:           }
@@ -7342,21 +7367,21 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:      "id": "0x{{.*}}",
 // JSON-NEXT:      "kind": "ClassTemplatePartialSpecializationDecl",
 // JSON-NEXT:      "loc": {
-// JSON-NEXT:       "offset": 6650,
-// JSON-NEXT:       "line": 187,
+// JSON-NEXT:       "offset": 6719,
+// JSON-NEXT:       "line": 188,
 // JSON-NEXT:       "col": 10,
 // JSON-NEXT:       "tokLen": 9
 // JSON-NEXT:      },
 // JSON-NEXT:      "range": {
 // JSON-NEXT:       "begin": {
-// JSON-NEXT:        "offset": 6603,
-// JSON-NEXT:        "line": 186,
+// JSON-NEXT:        "offset": 6672,
+// JSON-NEXT:        "line": 187,
 // JSON-NEXT:        "col": 3,
 // JSON-NEXT:        "tokLen": 8
 // JSON-NEXT:       },
 // JSON-NEXT:       "end": {
-// JSON-NEXT:        "offset": 6684,
-// JSON-NEXT:        "line": 187,
+// JSON-NEXT:        "offset": 6753,
+// JSON-NEXT:        "line": 188,
 // JSON-NEXT:        "col": 44,
 // JSON-NEXT:        "tokLen": 1
 // JSON-NEXT:       }
@@ -7418,14 +7443,14 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:       {
 // JSON-NEXT:        "kind": "TemplateArgument",
 // JSON-NEXT:        "type": {
-// JSON-NEXT:         "qualType": "Template1<type-parameter-0-0, value-parameter-0-1>"
+// JSON-NEXT:         "qualType": "TestPartialSpecNTTP::Template1<type-parameter-0-0, value-parameter-0-1>"
 // JSON-NEXT:        },
 // JSON-NEXT:        "inner": [
 // JSON-NEXT:         {
 // JSON-NEXT:          "id": "0x{{.*}}",
 // JSON-NEXT:          "kind": "TemplateSpecializationType",
 // JSON-NEXT:          "type": {
-// JSON-NEXT:           "qualType": "Template1<type-parameter-0-0, value-parameter-0-1>"
+// JSON-NEXT:           "qualType": "TestPartialSpecNTTP::Template1<type-parameter-0-0, value-parameter-0-1>"
 // JSON-NEXT:          },
 // JSON-NEXT:          "isDependent": true,
 // JSON-NEXT:          "isInstantiationDependent": true,
@@ -7463,15 +7488,14 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:              "kind": "DeclRefExpr",
 // JSON-NEXT:              "range": {
 // JSON-NEXT:               "begin": {
-// JSON-NEXT:                "offset": 6520,
-// JSON-NEXT:                "line": 183,
-// JSON-NEXT:                "col": 29,
-// JSON-NEXT:                "tokLen": 3
+// JSON-NEXT:                "offset": 6743,
+// JSON-NEXT:                "col": 34,
+// JSON-NEXT:                "tokLen": 2
 // JSON-NEXT:               },
 // JSON-NEXT:               "end": {
-// JSON-NEXT:                "offset": 6520,
-// JSON-NEXT:                "col": 29,
-// JSON-NEXT:                "tokLen": 3
+// JSON-NEXT:                "offset": 6743,
+// JSON-NEXT:                "col": 34,
+// JSON-NEXT:                "tokLen": 2
 // JSON-NEXT:               }
 // JSON-NEXT:              },
 // JSON-NEXT:              "type": {
@@ -7481,7 +7505,7 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:              "referencedDecl": {
 // JSON-NEXT:               "id": "0x{{.*}}",
 // JSON-NEXT:               "kind": "NonTypeTemplateParmDecl",
-// JSON-NEXT:               "name": "TA2",
+// JSON-NEXT:               "name": "U2",
 // JSON-NEXT:               "type": {
 // JSON-NEXT:                "qualType": "bool"
 // JSON-NEXT:               }
@@ -7503,13 +7527,12 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:          "kind": "DeclRefExpr",
 // JSON-NEXT:          "range": {
 // JSON-NEXT:           "begin": {
-// JSON-NEXT:            "offset": 6679,
-// JSON-NEXT:            "line": 187,
+// JSON-NEXT:            "offset": 6748,
 // JSON-NEXT:            "col": 39,
 // JSON-NEXT:            "tokLen": 2
 // JSON-NEXT:           },
 // JSON-NEXT:           "end": {
-// JSON-NEXT:            "offset": 6679,
+// JSON-NEXT:            "offset": 6748,
 // JSON-NEXT:            "col": 39,
 // JSON-NEXT:            "tokLen": 2
 // JSON-NEXT:           }
@@ -7533,19 +7556,19 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "TemplateTypeParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6619,
-// JSON-NEXT:         "line": 186,
+// JSON-NEXT:         "offset": 6688,
+// JSON-NEXT:         "line": 187,
 // JSON-NEXT:         "col": 19,
 // JSON-NEXT:         "tokLen": 2
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6613,
+// JSON-NEXT:          "offset": 6682,
 // JSON-NEXT:          "col": 13,
 // JSON-NEXT:          "tokLen": 5
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6619,
+// JSON-NEXT:          "offset": 6688,
 // JSON-NEXT:          "col": 19,
 // JSON-NEXT:          "tokLen": 2
 // JSON-NEXT:         }
@@ -7560,18 +7583,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6628,
+// JSON-NEXT:         "offset": 6697,
 // JSON-NEXT:         "col": 28,
 // JSON-NEXT:         "tokLen": 2
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6623,
+// JSON-NEXT:          "offset": 6692,
 // JSON-NEXT:          "col": 23,
 // JSON-NEXT:          "tokLen": 4
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6628,
+// JSON-NEXT:          "offset": 6697,
 // JSON-NEXT:          "col": 28,
 // JSON-NEXT:          "tokLen": 2
 // JSON-NEXT:         }
@@ -7588,18 +7611,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6637,
+// JSON-NEXT:         "offset": 6706,
 // JSON-NEXT:         "col": 37,
 // JSON-NEXT:         "tokLen": 2
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6632,
+// JSON-NEXT:          "offset": 6701,
 // JSON-NEXT:          "col": 32,
 // JSON-NEXT:          "tokLen": 4
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6637,
+// JSON-NEXT:          "offset": 6706,
 // JSON-NEXT:          "col": 37,
 // JSON-NEXT:          "tokLen": 2
 // JSON-NEXT:         }
@@ -7616,19 +7639,19 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "CXXRecordDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 6650,
-// JSON-NEXT:         "line": 187,
+// JSON-NEXT:         "offset": 6719,
+// JSON-NEXT:         "line": 188,
 // JSON-NEXT:         "col": 10,
 // JSON-NEXT:         "tokLen": 9
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 6643,
+// JSON-NEXT:          "offset": 6712,
 // JSON-NEXT:          "col": 3,
 // JSON-NEXT:          "tokLen": 6
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 6650,
+// JSON-NEXT:          "offset": 6719,
 // JSON-NEXT:          "col": 10,
 // JSON-NEXT:          "tokLen": 9
 // JSON-NEXT:         }
@@ -7643,21 +7666,21 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:      "id": "0x{{.*}}",
 // JSON-NEXT:      "kind": "ClassTemplatePartialSpecializationDecl",
 // JSON-NEXT:      "loc": {
-// JSON-NEXT:       "offset": 7925,
-// JSON-NEXT:       "line": 205,
+// JSON-NEXT:       "offset": 8035,
+// JSON-NEXT:       "line": 206,
 // JSON-NEXT:       "col": 10,
 // JSON-NEXT:       "tokLen": 9
 // JSON-NEXT:      },
 // JSON-NEXT:      "range": {
 // JSON-NEXT:       "begin": {
-// JSON-NEXT:        "offset": 7875,
-// JSON-NEXT:        "line": 204,
+// JSON-NEXT:        "offset": 7985,
+// JSON-NEXT:        "line": 205,
 // JSON-NEXT:        "col": 3,
 // JSON-NEXT:        "tokLen": 8
 // JSON-NEXT:       },
 // JSON-NEXT:       "end": {
-// JSON-NEXT:        "offset": 7959,
-// JSON-NEXT:        "line": 205,
+// JSON-NEXT:        "offset": 8069,
+// JSON-NEXT:        "line": 206,
 // JSON-NEXT:        "col": 44,
 // JSON-NEXT:        "tokLen": 1
 // JSON-NEXT:       }
@@ -7719,14 +7742,14 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:       {
 // JSON-NEXT:        "kind": "TemplateArgument",
 // JSON-NEXT:        "type": {
-// JSON-NEXT:         "qualType": "Template1<type-parameter-0-0, value-parameter-0-2>"
+// JSON-NEXT:         "qualType": "TestPartialSpecNTTP::Template1<type-parameter-0-0, value-parameter-0-2>"
 // JSON-NEXT:        },
 // JSON-NEXT:        "inner": [
 // JSON-NEXT:         {
 // JSON-NEXT:          "id": "0x{{.*}}",
 // JSON-NEXT:          "kind": "TemplateSpecializationType",
 // JSON-NEXT:          "type": {
-// JSON-NEXT:           "qualType": "Template1<type-parameter-0-0, value-parameter-0-2>"
+// JSON-NEXT:           "qualType": "TestPartialSpecNTTP::Template1<type-parameter-0-0, value-parameter-0-2>"
 // JSON-NEXT:          },
 // JSON-NEXT:          "isDependent": true,
 // JSON-NEXT:          "isInstantiationDependent": true,
@@ -7764,12 +7787,12 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:              "kind": "DeclRefExpr",
 // JSON-NEXT:              "range": {
 // JSON-NEXT:               "begin": {
-// JSON-NEXT:                "offset": 7949,
+// JSON-NEXT:                "offset": 8059,
 // JSON-NEXT:                "col": 34,
 // JSON-NEXT:                "tokLen": 2
 // JSON-NEXT:               },
 // JSON-NEXT:               "end": {
-// JSON-NEXT:                "offset": 7949,
+// JSON-NEXT:                "offset": 8059,
 // JSON-NEXT:                "col": 34,
 // JSON-NEXT:                "tokLen": 2
 // JSON-NEXT:               }
@@ -7803,12 +7826,12 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:          "kind": "DeclRefExpr",
 // JSON-NEXT:          "range": {
 // JSON-NEXT:           "begin": {
-// JSON-NEXT:            "offset": 7954,
+// JSON-NEXT:            "offset": 8064,
 // JSON-NEXT:            "col": 39,
 // JSON-NEXT:            "tokLen": 2
 // JSON-NEXT:           },
 // JSON-NEXT:           "end": {
-// JSON-NEXT:            "offset": 7954,
+// JSON-NEXT:            "offset": 8064,
 // JSON-NEXT:            "col": 39,
 // JSON-NEXT:            "tokLen": 2
 // JSON-NEXT:           }
@@ -7832,19 +7855,19 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "TemplateTypeParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 7894,
-// JSON-NEXT:         "line": 204,
+// JSON-NEXT:         "offset": 8004,
+// JSON-NEXT:         "line": 205,
 // JSON-NEXT:         "col": 22,
 // JSON-NEXT:         "tokLen": 2
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 7885,
+// JSON-NEXT:          "offset": 7995,
 // JSON-NEXT:          "col": 13,
 // JSON-NEXT:          "tokLen": 8
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 7894,
+// JSON-NEXT:          "offset": 8004,
 // JSON-NEXT:          "col": 22,
 // JSON-NEXT:          "tokLen": 2
 // JSON-NEXT:         }
@@ -7859,18 +7882,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 7903,
+// JSON-NEXT:         "offset": 8013,
 // JSON-NEXT:         "col": 31,
 // JSON-NEXT:         "tokLen": 2
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 7898,
+// JSON-NEXT:          "offset": 8008,
 // JSON-NEXT:          "col": 26,
 // JSON-NEXT:          "tokLen": 4
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 7903,
+// JSON-NEXT:          "offset": 8013,
 // JSON-NEXT:          "col": 31,
 // JSON-NEXT:          "tokLen": 2
 // JSON-NEXT:         }
@@ -7887,18 +7910,18 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "NonTypeTemplateParmDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 7912,
+// JSON-NEXT:         "offset": 8022,
 // JSON-NEXT:         "col": 40,
 // JSON-NEXT:         "tokLen": 2
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 7907,
+// JSON-NEXT:          "offset": 8017,
 // JSON-NEXT:          "col": 35,
 // JSON-NEXT:          "tokLen": 4
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 7912,
+// JSON-NEXT:          "offset": 8022,
 // JSON-NEXT:          "col": 40,
 // JSON-NEXT:          "tokLen": 2
 // JSON-NEXT:         }
@@ -7915,19 +7938,19 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:        "id": "0x{{.*}}",
 // JSON-NEXT:        "kind": "CXXRecordDecl",
 // JSON-NEXT:        "loc": {
-// JSON-NEXT:         "offset": 7925,
-// JSON-NEXT:         "line": 205,
+// JSON-NEXT:         "offset": 8035,
+// JSON-NEXT:         "line": 206,
 // JSON-NEXT:         "col": 10,
 // JSON-NEXT:         "tokLen": 9
 // JSON-NEXT:        },
 // JSON-NEXT:        "range": {
 // JSON-NEXT:         "begin": {
-// JSON-NEXT:          "offset": 7918,
+// JSON-NEXT:          "offset": 8028,
 // JSON-NEXT:          "col": 3,
 // JSON-NEXT:          "tokLen": 6
 // JSON-NEXT:         },
 // JSON-NEXT:         "end": {
-// JSON-NEXT:          "offset": 7925,
+// JSON-NEXT:          "offset": 8035,
 // JSON-NEXT:          "col": 10,
 // JSON-NEXT:          "tokLen": 9
 // JSON-NEXT:         }
@@ -7939,6 +7962,959 @@ namespace TestPartialSpecNTTP {
 // JSON-NEXT:      ]
 // JSON-NEXT:     }
 // JSON-NEXT:    ]
+// JSON-NEXT:   },
+// JSON-NEXT:   {
+// JSON-NEXT:    "id": "0x{{.*}}",
+// JSON-NEXT:    "kind": "NamespaceDecl",
+// JSON-NEXT:    "loc": {
+// JSON-NEXT:     "offset": 9382,
+// JSON-NEXT:     "line": 224,
+// JSON-NEXT:     "col": 11,
+// JSON-NEXT:     "tokLen": 8
+// JSON-NEXT:    },
+// JSON-NEXT:    "range": {
+// JSON-NEXT:     "begin": {
+// JSON-NEXT:      "offset": 9372,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 9
+// JSON-NEXT:     },
+// JSON-NEXT:     "end": {
+// JSON-NEXT:      "offset": 9791,
+// JSON-NEXT:      "line": 238,
+// JSON-NEXT:      "col": 1,
+// JSON-NEXT:      "tokLen": 1
+// JSON-NEXT:     }
+// JSON-NEXT:    },
+// JSON-NEXT:    "name": "GH153540",
+// JSON-NEXT:    "inner": [
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "NamespaceDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 9456,
+// JSON-NEXT:       "line": 227,
+// JSON-NEXT:       "col": 13,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 9446,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 9
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 9507,
+// JSON-NEXT:        "line": 229,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "N",
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "ClassTemplateDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 9492,
+// JSON-NEXT:         "line": 228,
+// JSON-NEXT:         "col": 33,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 9464,
+// JSON-NEXT:          "col": 5,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 9502,
+// JSON-NEXT:          "col": 43,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "name": "S",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 9482,
+// JSON-NEXT:           "col": 23,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 9473,
+// JSON-NEXT:            "col": 14,
+// JSON-NEXT:            "tokLen": 8
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 9482,
+// JSON-NEXT:            "col": 23,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isReferenced": true,
+// JSON-NEXT:          "name": "T",
+// JSON-NEXT:          "tagUsed": "typename",
+// JSON-NEXT:          "depth": 0,
+// JSON-NEXT:          "index": 0
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXRecordDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 9492,
+// JSON-NEXT:           "col": 33,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 9485,
+// JSON-NEXT:            "col": 26,
+// JSON-NEXT:            "tokLen": 6
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 9502,
+// JSON-NEXT:            "col": 43,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "name": "S",
+// JSON-NEXT:          "tagUsed": "struct",
+// JSON-NEXT:          "completeDefinition": true,
+// JSON-NEXT:          "definitionData": {
+// JSON-NEXT:           "canConstDefaultInit": true,
+// JSON-NEXT:           "copyAssign": {
+// JSON-NEXT:            "hasConstParam": true,
+// JSON-NEXT:            "implicitHasConstParam": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "copyCtor": {
+// JSON-NEXT:            "hasConstParam": true,
+// JSON-NEXT:            "implicitHasConstParam": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "defaultCtor": {
+// JSON-NEXT:            "defaultedIsConstexpr": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "dtor": {
+// JSON-NEXT:            "irrelevant": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "hasUserDeclaredConstructor": true,
+// JSON-NEXT:           "isEmpty": true,
+// JSON-NEXT:           "isStandardLayout": true,
+// JSON-NEXT:           "isTriviallyCopyable": true,
+// JSON-NEXT:           "moveAssign": {
+// JSON-NEXT:            "exists": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "moveCtor": {
+// JSON-NEXT:            "exists": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXRecordDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9492,
+// JSON-NEXT:             "col": 33,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9485,
+// JSON-NEXT:              "col": 26,
+// JSON-NEXT:              "tokLen": 6
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "isImplicit": true,
+// JSON-NEXT:            "name": "S",
+// JSON-NEXT:            "tagUsed": "struct"
+// JSON-NEXT:           },
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXConstructorDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9496,
+// JSON-NEXT:             "col": 37,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9496,
+// JSON-NEXT:              "col": 37,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9499,
+// JSON-NEXT:              "col": 40,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "name": "GH153540::N::S<T>",
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "void (T)"
+// JSON-NEXT:            },
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "ParmVarDecl",
+// JSON-NEXT:              "loc": {
+// JSON-NEXT:               "offset": 9499,
+// JSON-NEXT:               "col": 40,
+// JSON-NEXT:               "tokLen": 1
+// JSON-NEXT:              },
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 9498,
+// JSON-NEXT:                "col": 39,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 9498,
+// JSON-NEXT:                "col": 39,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "T"
+// JSON-NEXT:              }
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "ClassTemplateSpecializationDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 9492,
+// JSON-NEXT:           "col": 33,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 9464,
+// JSON-NEXT:            "col": 5,
+// JSON-NEXT:            "tokLen": 8
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 9502,
+// JSON-NEXT:            "col": 43,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "name": "S",
+// JSON-NEXT:          "tagUsed": "struct",
+// JSON-NEXT:          "completeDefinition": true,
+// JSON-NEXT:          "definitionData": {
+// JSON-NEXT:           "canConstDefaultInit": true,
+// JSON-NEXT:           "canPassInRegisters": true,
+// JSON-NEXT:           "copyAssign": {
+// JSON-NEXT:            "hasConstParam": true,
+// JSON-NEXT:            "implicitHasConstParam": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "copyCtor": {
+// JSON-NEXT:            "hasConstParam": true,
+// JSON-NEXT:            "implicitHasConstParam": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "defaultCtor": {
+// JSON-NEXT:            "defaultedIsConstexpr": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "dtor": {
+// JSON-NEXT:            "irrelevant": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "hasUserDeclaredConstructor": true,
+// JSON-NEXT:           "isEmpty": true,
+// JSON-NEXT:           "isStandardLayout": true,
+// JSON-NEXT:           "isTriviallyCopyable": true,
+// JSON-NEXT:           "moveAssign": {
+// JSON-NEXT:            "exists": true,
+// JSON-NEXT:            "needsImplicit": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           },
+// JSON-NEXT:           "moveCtor": {
+// JSON-NEXT:            "exists": true,
+// JSON-NEXT:            "simple": true,
+// JSON-NEXT:            "trivial": true
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "kind": "TemplateArgument",
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "int"
+// JSON-NEXT:            },
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "BuiltinType",
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "int"
+// JSON-NEXT:              }
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           },
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXRecordDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9492,
+// JSON-NEXT:             "col": 33,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9485,
+// JSON-NEXT:              "col": 26,
+// JSON-NEXT:              "tokLen": 6
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "isImplicit": true,
+// JSON-NEXT:            "name": "S",
+// JSON-NEXT:            "tagUsed": "struct"
+// JSON-NEXT:           },
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXConstructorDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9496,
+// JSON-NEXT:             "col": 37,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9496,
+// JSON-NEXT:              "col": 37,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9499,
+// JSON-NEXT:              "col": 40,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "isUsed": true,
+// JSON-NEXT:            "name": "S",
+// JSON-NEXT:            "mangledName": "_ZN8GH1535401N1SIiEC1Ei",
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "void (int)"
+// JSON-NEXT:            },
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "ParmVarDecl",
+// JSON-NEXT:              "loc": {
+// JSON-NEXT:               "offset": 9499,
+// JSON-NEXT:               "col": 40,
+// JSON-NEXT:               "tokLen": 1
+// JSON-NEXT:              },
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 9498,
+// JSON-NEXT:                "col": 39,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 9498,
+// JSON-NEXT:                "col": 39,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "int"
+// JSON-NEXT:              }
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           },
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXConstructorDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9492,
+// JSON-NEXT:             "col": 33,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "isImplicit": true,
+// JSON-NEXT:            "name": "S",
+// JSON-NEXT:            "mangledName": "_ZN8GH1535401N1SIiEC1ERKS2_",
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "void (const S<int> &)"
+// JSON-NEXT:            },
+// JSON-NEXT:            "inline": true,
+// JSON-NEXT:            "constexpr": true,
+// JSON-NEXT:            "explicitlyDefaulted": "default",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "ParmVarDecl",
+// JSON-NEXT:              "loc": {
+// JSON-NEXT:               "offset": 9492,
+// JSON-NEXT:               "col": 33,
+// JSON-NEXT:               "tokLen": 1
+// JSON-NEXT:              },
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 9492,
+// JSON-NEXT:                "col": 33,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 9492,
+// JSON-NEXT:                "col": 33,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "const S<int> &"
+// JSON-NEXT:              }
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           },
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXConstructorDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9492,
+// JSON-NEXT:             "col": 33,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "isImplicit": true,
+// JSON-NEXT:            "name": "S",
+// JSON-NEXT:            "mangledName": "_ZN8GH1535401N1SIiEC1EOS2_",
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "void (S<int> &&)"
+// JSON-NEXT:            },
+// JSON-NEXT:            "inline": true,
+// JSON-NEXT:            "constexpr": true,
+// JSON-NEXT:            "explicitlyDefaulted": "default",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "ParmVarDecl",
+// JSON-NEXT:              "loc": {
+// JSON-NEXT:               "offset": 9492,
+// JSON-NEXT:               "col": 33,
+// JSON-NEXT:               "tokLen": 1
+// JSON-NEXT:              },
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 9492,
+// JSON-NEXT:                "col": 33,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 9492,
+// JSON-NEXT:                "col": 33,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "S<int> &&"
+// JSON-NEXT:              }
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           },
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXDestructorDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9492,
+// JSON-NEXT:             "col": 33,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "isImplicit": true,
+// JSON-NEXT:            "isReferenced": true,
+// JSON-NEXT:            "name": "~S",
+// JSON-NEXT:            "mangledName": "_ZN8GH1535401N1SIiED1Ev",
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "void () noexcept"
+// JSON-NEXT:            },
+// JSON-NEXT:            "inline": true,
+// JSON-NEXT:            "constexpr": true,
+// JSON-NEXT:            "explicitlyDefaulted": "default"
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FunctionTemplateDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 9496,
+// JSON-NEXT:         "col": 37,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 9464,
+// JSON-NEXT:          "col": 5,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 9499,
+// JSON-NEXT:          "col": 40,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "<deduction guide for S>",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 9482,
+// JSON-NEXT:           "col": 23,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 9473,
+// JSON-NEXT:            "col": 14,
+// JSON-NEXT:            "tokLen": 8
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 9482,
+// JSON-NEXT:            "col": 23,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isReferenced": true,
+// JSON-NEXT:          "name": "T",
+// JSON-NEXT:          "tagUsed": "typename",
+// JSON-NEXT:          "depth": 0,
+// JSON-NEXT:          "index": 0
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXDeductionGuideDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 9496,
+// JSON-NEXT:           "col": 37,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 9496,
+// JSON-NEXT:            "col": 37,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 9499,
+// JSON-NEXT:            "col": 40,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isImplicit": true,
+// JSON-NEXT:          "name": "<deduction guide for S>",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "auto (T) -> GH153540::N::S<T>"
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "ParmVarDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9499,
+// JSON-NEXT:             "col": 40,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9498,
+// JSON-NEXT:              "col": 39,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9498,
+// JSON-NEXT:              "col": 39,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "T"
+// JSON-NEXT:            }
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXDeductionGuideDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 9496,
+// JSON-NEXT:           "col": 37,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 9496,
+// JSON-NEXT:            "col": 37,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 9499,
+// JSON-NEXT:            "col": 40,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isImplicit": true,
+// JSON-NEXT:          "isUsed": true,
+// JSON-NEXT:          "name": "<deduction guide for S>",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "auto (int) -> GH153540::N::S<int>"
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "kind": "TemplateArgument",
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "int"
+// JSON-NEXT:            },
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "BuiltinType",
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "int"
+// JSON-NEXT:              }
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           },
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "ParmVarDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9499,
+// JSON-NEXT:             "col": 40,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9498,
+// JSON-NEXT:              "col": 39,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9498,
+// JSON-NEXT:              "col": 39,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "int"
+// JSON-NEXT:            }
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       },
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "FunctionTemplateDecl",
+// JSON-NEXT:        "loc": {
+// JSON-NEXT:         "offset": 9492,
+// JSON-NEXT:         "col": 33,
+// JSON-NEXT:         "tokLen": 1
+// JSON-NEXT:        },
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 9464,
+// JSON-NEXT:          "col": 5,
+// JSON-NEXT:          "tokLen": 8
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 9492,
+// JSON-NEXT:          "col": 33,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "isImplicit": true,
+// JSON-NEXT:        "name": "<deduction guide for S>",
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "TemplateTypeParmDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 9482,
+// JSON-NEXT:           "col": 23,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 9473,
+// JSON-NEXT:            "col": 14,
+// JSON-NEXT:            "tokLen": 8
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 9482,
+// JSON-NEXT:            "col": 23,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isReferenced": true,
+// JSON-NEXT:          "name": "T",
+// JSON-NEXT:          "tagUsed": "typename",
+// JSON-NEXT:          "depth": 0,
+// JSON-NEXT:          "index": 0
+// JSON-NEXT:         },
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXDeductionGuideDecl",
+// JSON-NEXT:          "loc": {
+// JSON-NEXT:           "offset": 9492,
+// JSON-NEXT:           "col": 33,
+// JSON-NEXT:           "tokLen": 1
+// JSON-NEXT:          },
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 9492,
+// JSON-NEXT:            "col": 33,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 9492,
+// JSON-NEXT:            "col": 33,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "isImplicit": true,
+// JSON-NEXT:          "name": "<deduction guide for S>",
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "qualType": "auto (GH153540::N::S<T>) -> GH153540::N::S<T>"
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "ParmVarDecl",
+// JSON-NEXT:            "loc": {
+// JSON-NEXT:             "offset": 9492,
+// JSON-NEXT:             "col": 33,
+// JSON-NEXT:             "tokLen": 1
+// JSON-NEXT:            },
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9492,
+// JSON-NEXT:              "col": 33,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "qualType": "GH153540::N::S<T>"
+// JSON-NEXT:            }
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     },
+// JSON-NEXT:     {
+// JSON-NEXT:      "id": "0x{{.*}}",
+// JSON-NEXT:      "kind": "FunctionDecl",
+// JSON-NEXT:      "loc": {
+// JSON-NEXT:       "offset": 9516,
+// JSON-NEXT:       "line": 230,
+// JSON-NEXT:       "col": 8,
+// JSON-NEXT:       "tokLen": 1
+// JSON-NEXT:      },
+// JSON-NEXT:      "range": {
+// JSON-NEXT:       "begin": {
+// JSON-NEXT:        "offset": 9511,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 4
+// JSON-NEXT:       },
+// JSON-NEXT:       "end": {
+// JSON-NEXT:        "offset": 9537,
+// JSON-NEXT:        "line": 232,
+// JSON-NEXT:        "col": 3,
+// JSON-NEXT:        "tokLen": 1
+// JSON-NEXT:       }
+// JSON-NEXT:      },
+// JSON-NEXT:      "name": "f",
+// JSON-NEXT:      "mangledName": "_ZN8GH1535401fEv",
+// JSON-NEXT:      "type": {
+// JSON-NEXT:       "qualType": "void ()"
+// JSON-NEXT:      },
+// JSON-NEXT:      "inner": [
+// JSON-NEXT:       {
+// JSON-NEXT:        "id": "0x{{.*}}",
+// JSON-NEXT:        "kind": "CompoundStmt",
+// JSON-NEXT:        "range": {
+// JSON-NEXT:         "begin": {
+// JSON-NEXT:          "offset": 9520,
+// JSON-NEXT:          "line": 230,
+// JSON-NEXT:          "col": 12,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         },
+// JSON-NEXT:         "end": {
+// JSON-NEXT:          "offset": 9537,
+// JSON-NEXT:          "line": 232,
+// JSON-NEXT:          "col": 3,
+// JSON-NEXT:          "tokLen": 1
+// JSON-NEXT:         }
+// JSON-NEXT:        },
+// JSON-NEXT:        "inner": [
+// JSON-NEXT:         {
+// JSON-NEXT:          "id": "0x{{.*}}",
+// JSON-NEXT:          "kind": "CXXFunctionalCastExpr",
+// JSON-NEXT:          "range": {
+// JSON-NEXT:           "begin": {
+// JSON-NEXT:            "offset": 9526,
+// JSON-NEXT:            "line": 231,
+// JSON-NEXT:            "col": 5,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           },
+// JSON-NEXT:           "end": {
+// JSON-NEXT:            "offset": 9532,
+// JSON-NEXT:            "col": 11,
+// JSON-NEXT:            "tokLen": 1
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "type": {
+// JSON-NEXT:           "desugaredQualType": "GH153540::N::S<int>",
+// JSON-NEXT:           "qualType": "N::S<int>"
+// JSON-NEXT:          },
+// JSON-NEXT:          "valueCategory": "prvalue",
+// JSON-NEXT:          "castKind": "ConstructorConversion",
+// JSON-NEXT:          "conversionFunc": {
+// JSON-NEXT:           "id": "0x{{.*}}",
+// JSON-NEXT:           "kind": "CXXConstructorDecl",
+// JSON-NEXT:           "name": "S",
+// JSON-NEXT:           "type": {
+// JSON-NEXT:            "qualType": "void (int)"
+// JSON-NEXT:           }
+// JSON-NEXT:          },
+// JSON-NEXT:          "inner": [
+// JSON-NEXT:           {
+// JSON-NEXT:            "id": "0x{{.*}}",
+// JSON-NEXT:            "kind": "CXXConstructExpr",
+// JSON-NEXT:            "range": {
+// JSON-NEXT:             "begin": {
+// JSON-NEXT:              "offset": 9526,
+// JSON-NEXT:              "col": 5,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             },
+// JSON-NEXT:             "end": {
+// JSON-NEXT:              "offset": 9532,
+// JSON-NEXT:              "col": 11,
+// JSON-NEXT:              "tokLen": 1
+// JSON-NEXT:             }
+// JSON-NEXT:            },
+// JSON-NEXT:            "type": {
+// JSON-NEXT:             "desugaredQualType": "GH153540::N::S<int>",
+// JSON-NEXT:             "qualType": "N::S<int>"
+// JSON-NEXT:            },
+// JSON-NEXT:            "valueCategory": "prvalue",
+// JSON-NEXT:            "ctorType": {
+// JSON-NEXT:             "qualType": "void (int)"
+// JSON-NEXT:            },
+// JSON-NEXT:            "hadMultipleCandidates": true,
+// JSON-NEXT:            "constructionKind": "complete",
+// JSON-NEXT:            "inner": [
+// JSON-NEXT:             {
+// JSON-NEXT:              "id": "0x{{.*}}",
+// JSON-NEXT:              "kind": "IntegerLiteral",
+// JSON-NEXT:              "range": {
+// JSON-NEXT:               "begin": {
+// JSON-NEXT:                "offset": 9531,
+// JSON-NEXT:                "col": 10,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               },
+// JSON-NEXT:               "end": {
+// JSON-NEXT:                "offset": 9531,
+// JSON-NEXT:                "col": 10,
+// JSON-NEXT:                "tokLen": 1
+// JSON-NEXT:               }
+// JSON-NEXT:              },
+// JSON-NEXT:              "type": {
+// JSON-NEXT:               "qualType": "int"
+// JSON-NEXT:              },
+// JSON-NEXT:              "valueCategory": "prvalue",
+// JSON-NEXT:              "value": "0"
+// JSON-NEXT:             }
+// JSON-NEXT:            ]
+// JSON-NEXT:           }
+// JSON-NEXT:          ]
+// JSON-NEXT:         }
+// JSON-NEXT:        ]
+// JSON-NEXT:       }
+// JSON-NEXT:      ]
+// JSON-NEXT:     }
+// JSON-NEXT:    ]
 // JSON-NEXT:   }
 // JSON-NEXT:  ]
 // JSON-NEXT: }
diff --git a/clang/test/AST/ast-dump-traits.cpp b/clang/test/AST/ast-dump-traits.cpp
index 72d2a2ae8603e..b844fd6bcc49c 100644
--- a/clang/test/AST/ast-dump-traits.cpp
+++ b/clang/test/AST/ast-dump-traits.cpp
@@ -40,9 +40,8 @@ void test_unary_expr_or_type_trait() {
 // CHECK-NEXT: |   | `-EnumDecl {{.*}} <col:3, col:11> col:8{{( imported)?}} referenced E
 // CHECK-NEXT: |   |-CStyleCastExpr {{.*}} <line:13:3, col:21> 'void' <ToVoid>
 // CHECK-NEXT: |   | `-TypeTraitExpr {{.*}} <col:10, col:21> 'bool' __is_enum
-// CHECK-NEXT: |   |   `-ElaboratedType {{.*}} 'E' sugar
-// CHECK-NEXT: |   |     `-EnumType {{.*}} 'E'
-// CHECK-NEXT: |   |       `-Enum {{.*}} 'E'
+// CHECK-NEXT: |   |   `-EnumType {{.*}} 'E'
+// CHECK-NEXT: |   |     `-Enum {{.*}} 'E'
 // CHECK-NEXT: |   |-CStyleCastExpr {{.*}} <line:15:3, col:30> 'void' <ToVoid>
 // CHECK-NEXT: |   | `-TypeTraitExpr {{.*}} <col:10, col:30> 'bool' __is_same
 // CHECK-NEXT: |   |   |-BuiltinType {{.*}} 'int'
diff --git a/clang/test/AST/ast-dump-types-json.cpp b/clang/test/AST/ast-dump-types-json.cpp
index cc4d4d9a64872..aac602780e8c0 100644
--- a/clang/test/AST/ast-dump-types-json.cpp
+++ b/clang/test/AST/ast-dump-types-json.cpp
@@ -51,30 +51,20 @@ using ::TestUsingShadowDeclType;
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "name": "TestElaboratedType1",
 // CHECK-NEXT:  "type": {
-// CHECK-NEXT:   "desugaredQualType": "T",
 // CHECK-NEXT:   "qualType": "struct T"
 // CHECK-NEXT:  },
 // CHECK-NEXT:  "inner": [
 // CHECK-NEXT:   {
 // CHECK-NEXT:    "id": "0x{{.*}}",
-// CHECK-NEXT:    "kind": "ElaboratedType",
+// CHECK-NEXT:    "kind": "RecordType",
 // CHECK-NEXT:    "type": {
 // CHECK-NEXT:     "qualType": "struct T"
 // CHECK-NEXT:    },
-// CHECK-NEXT:    "inner": [
-// CHECK-NEXT:     {
-// CHECK-NEXT:      "id": "0x{{.*}}",
-// CHECK-NEXT:      "kind": "RecordType",
-// CHECK-NEXT:      "type": {
-// CHECK-NEXT:       "qualType": "T"
-// CHECK-NEXT:      },
-// CHECK-NEXT:      "decl": {
-// CHECK-NEXT:       "id": "0x{{.*}}",
-// CHECK-NEXT:       "kind": "CXXRecordDecl",
-// CHECK-NEXT:       "name": "T"
-// CHECK-NEXT:      }
-// CHECK-NEXT:     }
-// CHECK-NEXT:    ]
+// CHECK-NEXT:    "decl": {
+// CHECK-NEXT:     "id": "0x{{.*}}",
+// CHECK-NEXT:     "kind": "CXXRecordDecl",
+// CHECK-NEXT:     "name": "T"
+// CHECK-NEXT:    }
 // CHECK-NEXT:   }
 // CHECK-NEXT:  ]
 // CHECK-NEXT: }
@@ -108,25 +98,16 @@ using ::TestUsingShadowDeclType;
 // CHECK-NEXT:  "inner": [
 // CHECK-NEXT:   {
 // CHECK-NEXT:    "id": "0x{{.*}}",
-// CHECK-NEXT:    "kind": "ElaboratedType",
+// CHECK-NEXT:    "kind": "RecordType",
 // CHECK-NEXT:    "type": {
 // CHECK-NEXT:     "qualType": "NS::S"
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "qualifier": "NS::",
-// CHECK-NEXT:    "inner": [
-// CHECK-NEXT:     {
-// CHECK-NEXT:      "id": "0x{{.*}}",
-// CHECK-NEXT:      "kind": "RecordType",
-// CHECK-NEXT:      "type": {
-// CHECK-NEXT:       "qualType": "NS::S"
-// CHECK-NEXT:      },
-// CHECK-NEXT:      "decl": {
-// CHECK-NEXT:       "id": "0x{{.*}}",
-// CHECK-NEXT:       "kind": "CXXRecordDecl",
-// CHECK-NEXT:       "name": "S"
-// CHECK-NEXT:      }
-// CHECK-NEXT:     }
-// CHECK-NEXT:    ]
+// CHECK-NEXT:    "decl": {
+// CHECK-NEXT:     "id": "0x{{.*}}",
+// CHECK-NEXT:     "kind": "CXXRecordDecl",
+// CHECK-NEXT:     "name": "S"
+// CHECK-NEXT:    }
 // CHECK-NEXT:   }
 // CHECK-NEXT:  ]
 // CHECK-NEXT: }
@@ -538,7 +519,39 @@ using ::TestUsingShadowDeclType;
 // CHECK-NEXT:    },
 // CHECK-NEXT:    "inner": [
 // CHECK-NEXT:     {
-// CHECK-NEXT:      "id": "0x0"
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "TypedefDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 506,
+// CHECK-NEXT:       "line": 23,
+// CHECK-NEXT:       "col": 13,
+// CHECK-NEXT:       "tokLen": 23
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 494,
+// CHECK-NEXT:        "col": 1,
+// CHECK-NEXT:        "tokLen": 7
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 506,
+// CHECK-NEXT:        "col": 13,
+// CHECK-NEXT:        "tokLen": 23
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "name": "TestUsingShadowDeclType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "int"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "BuiltinType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "int"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
 // CHECK-NEXT:     }
 // CHECK-NEXT:    ]
 // CHECK-NEXT:   }
diff --git a/clang/test/AST/ast-dump-using-template.cpp b/clang/test/AST/ast-dump-using-template.cpp
index 2c9584966e843..a5a0e4d07d683 100644
--- a/clang/test/AST/ast-dump-using-template.cpp
+++ b/clang/test/AST/ast-dump-using-template.cpp
@@ -19,40 +19,36 @@ using ns::S2;
 template<typename T>
 using A = S<T>;
 // CHECK:      TypeAliasDecl
-// CHECK-NEXT: `-ElaboratedType {{.*}} 'S<T>' sugar dependent
-// CHECK-NEXT:   `-TemplateSpecializationType {{.*}} 'S<T>' dependent
-// CHECK-NEXT:     |-name: 'S':'ns::S' qualified
-// CHECK-NEXT:     | |-UsingShadowDecl {{.+}} ClassTemplate {{.+}} 'S'
+// CHECK-NEXT: `-TemplateSpecializationType {{.*}} 'S<T>' dependent
+// CHECK-NEXT:   |-name: 'S':'ns::S' qualified
+// CHECk-NEXT:   | |-UsingShadowDecl {{.+}} ClassTemplate {{.+}} 'S'
 
 // TemplateName in TemplateArgument.
 template <template <typename> class T> class X {};
 using B = X<S>;
 // CHECK:      TypeAliasDecl
-// CHECK-NEXT: `-ElaboratedType {{.*}} 'X<S>' sugar
-// CHECK-NEXT:   `-TemplateSpecializationType {{.*}} 'X<S>' sugar
-// CHECK-NEXT:     |-name: 'X' qualified
-// CHECK-NEXT:     | `-ClassTemplateDecl {{.+}} X
-// CHECK-NEXT:     |-TemplateArgument template 'S':'ns::S' qualified
-// CHECK-NEXT:     | |-UsingShadowDecl {{.*}} implicit ClassTemplate {{.*}} 'S'
-// CHECK-NEXT:     | `-target: ClassTemplateDecl {{.*}} S
-// CHECK-NEXT:     `-RecordType {{.*}} 'X<ns::S>'
-// CHECK-NEXT:       `-ClassTemplateSpecialization {{.*}} 'X'
+// CHECK-NEXT: `-TemplateSpecializationType {{.*}} 'X<S>' sugar
+// CHECK-NEXT:   |-name: 'X' qualified
+// CHECK-NEXT:   | `-ClassTemplateDecl {{.+}} X
+// CHECK-NEXT:   |-TemplateArgument template 'S':'ns::S' qualified
+// CHECK-NEXT:   | |-UsingShadowDecl {{.*}} implicit ClassTemplate {{.*}} 'S'
+// CHECK-NEXT:   | `-target: ClassTemplateDecl {{.*}} S
+// CHECK-NEXT:   `-RecordType {{.*}} 'X<ns::S>'
+// CHECK-NEXT:     `-ClassTemplateSpecialization {{.*}} 'X'
 
 // TemplateName in DeducedTemplateSpecializationType.
 S DeducedTemplateSpecializationT(123);
 using C = decltype(DeducedTemplateSpecializationT);
 // CHECK:      DecltypeType {{.*}}
 // CHECK-NEXT:  |-DeclRefExpr {{.*}}
-// CHECK-NEXT:  `-ElaboratedType {{.*}} 'S<int>' sugar
-// CHECK-NEXT:    `-DeducedTemplateSpecializationType {{.*}} 'ns::S<int>' sugar
-// CHECK-NEXT:      |-name: 'S':'ns::S' qualified
-// CHECK-NEXT:      | |-UsingShadowDecl {{.+}} 'S'
+// CHECK-NEXT:  `-DeducedTemplateSpecializationType {{.*}} 'S<int>' sugar
+// CHECK-NEXT:    |-name: 'S':'ns::S' qualified
+// CHECK-NEXT:    | |-UsingShadowDecl {{.+}} 'S'
 
 S2 DeducedTemplateSpecializationT2(123);
 using D = decltype(DeducedTemplateSpecializationT2);
 // CHECK:      DecltypeType {{.*}}
 // CHECK-NEXT:  |-DeclRefExpr {{.*}}
-// CHECK-NEXT:  `-ElaboratedType {{.*}} 'S2<int>' sugar
-// CHECK-NEXT:    `-DeducedTemplateSpecializationType {{.*}} 'S2<int>' sugar
-// CHECK-NEXT:      |-name: 'S2':'ns::S2' qualified
-// CHECK-NEXT:       | |-UsingShadowDecl {{.+}} ClassTemplate {{.+}} 'S2'
+// CHECK-NEXT:  `-DeducedTemplateSpecializationType {{.*}} 'S2<int>' sugar
+// CHECK-NEXT:    |-name: 'S2':'ns::S2' qualified
+// CHECk-NEXT:    | |-UsingShadowDecl {{.+}} ClassTemplate {{.+}} 'S2'
diff --git a/clang/test/AST/ast-dump-using.cpp b/clang/test/AST/ast-dump-using.cpp
index 8e5c60d3aabf4..0c8405bc5a52b 100644
--- a/clang/test/AST/ast-dump-using.cpp
+++ b/clang/test/AST/ast-dump-using.cpp
@@ -9,19 +9,17 @@ using a::S;
 // CHECK:      UsingDecl {{.*}} a::S
 // CHECK-NEXT: | `-NestedNameSpecifier Namespace {{.*}} 'a'
 // CHECK-NEXT: UsingShadowDecl {{.*}} implicit CXXRecord {{.*}} 'S'
-// CHECK-NEXT: `-RecordType {{.*}} 'a::S'
+// CHECK-NEXT: `-CXXRecordDecl {{.*}} referenced struct S
 typedef S f; // to dump the introduced type
 // CHECK:      TypedefDecl
-// CHECK-NEXT: `-ElaboratedType {{.*}} 'S' sugar
-// CHECK-NEXT:   `-UsingType [[TYPE_ADDR:.*]] 'a::S' sugar
-// CHECK-NEXT:     |-UsingShadow [[SHADOW_ADDR:.*]] 'S'
-// CHECK-NEXT:     `-RecordType {{.*}} 'a::S'
+// CHECK-NEXT: `-UsingType [[TYPE_ADDR:.*]] 'S' sugar 'a::S'
+// CHECK-NEXT:   |-UsingShadow [[SHADOW_ADDR:.*]] 'S'
+// CHECK-NEXT:   `-RecordType {{.*}} 'a::S'
 typedef S e; // check the same UsingType is reused.
 // CHECK:      TypedefDecl
-// CHECK-NEXT: `-ElaboratedType {{.*}} 'S' sugar
-// CHECK-NEXT:   `-UsingType [[TYPE_ADDR]] 'a::S' sugar
-// CHECK-NEXT:     |-UsingShadow [[SHADOW_ADDR]] 'S'
-// CHECK-NEXT:     `-RecordType {{.*}} 'a::S'
+// CHECK-NEXT: `-UsingType [[TYPE_ADDR]] 'S' sugar 'a::S'
+// CHECK-NEXT:   |-UsingShadow [[SHADOW_ADDR]] 'S'
+// CHECK-NEXT:   `-RecordType {{.*}} 'a::S'
 using a::x;
 
 void foo() {
diff --git a/clang/test/AST/attr-swift_attr.m b/clang/test/AST/attr-swift_attr.m
index 6888745fe95d4..766da93db811e 100644
--- a/clang/test/AST/attr-swift_attr.m
+++ b/clang/test/AST/attr-swift_attr.m
@@ -43,7 +43,7 @@ @interface Generic<T: SWIFT_SENDABLE id>
   void *ptr;
 } SendableStruct;
 
-// CHECK-LABEL: TypedefDecl {{.*}} SendableStruct 'struct SendableStruct':'SendableStruct'
+// CHECK-LABEL: TypedefDecl {{.*}} SendableStruct 'struct SendableStruct'
 // CHECK: SwiftAttrAttr {{.*}} "@Sendable"
 
 @interface TestAttrPlacementInBlock1
diff --git a/clang/test/AST/coroutine-locals-cleanup.cpp b/clang/test/AST/coroutine-locals-cleanup.cpp
index a7f524b0b1ac9..b0577f9b8e235 100644
--- a/clang/test/AST/coroutine-locals-cleanup.cpp
+++ b/clang/test/AST/coroutine-locals-cleanup.cpp
@@ -86,7 +86,7 @@ Task bar() {
 // CHECK:             ExprWithCleanups {{.*}} 'void'
 // CHECK-NEXT:          CoawaitExpr
 // CHECK-NEXT:            CXXBindTemporaryExpr {{.*}} 'Task' (CXXTemporary {{.*}})
-// CHECK:                 MaterializeTemporaryExpr {{.*}} 'Awaiter':'Task::Awaiter'
+// CHECK:                 MaterializeTemporaryExpr {{.*}} 'Awaiter'
 // CHECK:                 ExprWithCleanups {{.*}} 'bool'
 // CHECK-NEXT:              CXXMemberCallExpr {{.*}} 'bool'
 // CHECK-NEXT:                MemberExpr {{.*}} .await_ready
@@ -96,7 +96,7 @@ Task bar() {
 // CHECK:             ExprWithCleanups {{.*}} 'void'
 // CHECK-NEXT:          CoawaitExpr
 // CHECK-NEXT:            CXXBindTemporaryExpr {{.*}} 'Task' (CXXTemporary {{.*}})
-// CHECK:                 MaterializeTemporaryExpr {{.*}} 'Awaiter':'Task::Awaiter'
+// CHECK:                 MaterializeTemporaryExpr {{.*}} 'Awaiter'
 // CHECK:                 ExprWithCleanups {{.*}} 'bool'
 // CHECK-NEXT:              CXXMemberCallExpr {{.*}} 'bool'
 // CHECK-NEXT:                MemberExpr {{.*}} .await_ready
diff --git a/clang/test/AST/cxx2c-variadic-friends.cpp b/clang/test/AST/cxx2c-variadic-friends.cpp
index fc84e7346fe03..6147d7f75895f 100644
--- a/clang/test/AST/cxx2c-variadic-friends.cpp
+++ b/clang/test/AST/cxx2c-variadic-friends.cpp
@@ -70,11 +70,11 @@ template <typename ...Pack> struct Variadic {
 // CHECK-LABEL: ClassTemplateDecl {{.*}} S2
 // PRINT-LABEL: template <class ...Ts> struct S2 {
 template<class ...Ts> struct S2 {
-  // CHECK: FriendDecl {{.*}} 'class C<Ts>':'C<Ts>'...
+  // CHECK: FriendDecl {{.*}} 'class C<Ts>'...
   // PRINT-NEXT: friend class C<Ts>...;
   friend class C<Ts>...;
 
-  // CHECK-NEXT: FriendDecl {{.*}} 'class N::C<Ts>':'C<Ts>'...
+  // CHECK-NEXT: FriendDecl {{.*}} 'class N::C<Ts>'...
   // PRINT-NEXT: friend class N::C<Ts>...
   friend class N::C<Ts>...;
 };
diff --git a/clang/test/AST/deduction-guides.cpp b/clang/test/AST/deduction-guides.cpp
index d96c7e6bd5e35..5ecb276370bcc 100644
--- a/clang/test/AST/deduction-guides.cpp
+++ b/clang/test/AST/deduction-guides.cpp
@@ -30,11 +30,11 @@ struct HasDeductionGuideTypeAlias {
 HasDeductionGuideTypeAlias()->HasDeductionGuideTypeAlias<int>;
 
 // The parameter to this one shouldn't be an elaborated type.
-// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for HasDeductionGuide> 'auto (typename STy::Child) -> HasDeductionGuide<T>'
-// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for HasDeductionGuide> 'auto (HasDeductionGuide<T>) -> HasDeductionGuide<T>'
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for HasDeductionGuide> 'auto (typename STy::Child) -> PR46111::HasDeductionGuide<T>'
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for HasDeductionGuide> 'auto (PR46111::HasDeductionGuide<T>) -> PR46111::HasDeductionGuide<T>'
 // CHECK: CXXDeductionGuideDecl {{.*}} <deduction guide for HasDeductionGuide> 'auto () -> HasDeductionGuide<int>'
-// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for HasDeductionGuideTypeAlias> 'auto (typename STy::Child) -> HasDeductionGuideTypeAlias<T>'
-// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for HasDeductionGuideTypeAlias> 'auto (HasDeductionGuideTypeAlias<T>) -> HasDeductionGuideTypeAlias<T>'
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for HasDeductionGuideTypeAlias> 'auto (typename STy::Child) -> PR46111::HasDeductionGuideTypeAlias<T>'
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for HasDeductionGuideTypeAlias> 'auto (PR46111::HasDeductionGuideTypeAlias<T>) -> PR46111::HasDeductionGuideTypeAlias<T>'
 // CHECK: CXXDeductionGuideDecl {{.*}} <deduction guide for HasDeductionGuideTypeAlias> 'auto () -> HasDeductionGuideTypeAlias<int>'
 } // namespace PR46111
 
@@ -64,15 +64,15 @@ namespace PR48177 {
 
 // CHECK: CXXRecordDecl {{.*}} struct Derived
 // CHECK: TypeAliasDecl {{.*}} type_alias 'typename Derived<int, 1, int>::type_alias':'int'
-// CHECK-NEXT: ElaboratedType {{.*}} 'typename Derived<int, 1, int>::type_alias' sugar
-// CHECK-NEXT: TypedefType {{.*}} 'PR48177::Base<int>::type_alias' sugar
+// CHECK-NEXT: TypedefType {{.*}} 'typename Derived<int, 1, int>::type_alias' sugar
+// CHECK-NEXT: NestedNameSpecifier TypeSpec 'Derived<int, 1, int>'
 // CHECK-NEXT: TypeAlias {{.*}} 'type_alias'
 // CHECK-NEXT: SubstTemplateTypeParmType {{.*}} 'int' sugar class depth 0 index 0 A
 // CHECK-NEXT: ClassTemplateSpecialization {{.*}} 'Base'
 // CHECK-NEXT: BuiltinType {{.*}} 'int'
 
-// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for Derived> 'auto (Derived<T, S, A> &&, const typename Derived<T, S, A>::type_alias &) -> Derived<T, S, A>'
-// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for Derived> 'auto (T) -> Derived<T, S, A>'
-// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for Derived> 'auto (Derived<T, S, A>) -> Derived<T, S, A>'
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for Derived> 'auto (Derived<T, S, A> &&, const typename Derived<T, S, A>::type_alias &) -> PR48177::Derived<T, S, A>'
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for Derived> 'auto (T) -> PR48177::Derived<T, S, A>'
+// CHECK: CXXDeductionGuideDecl {{.*}} implicit <deduction guide for Derived> 'auto (PR48177::Derived<T, S, A>) -> PR48177::Derived<T, S, A>'
 // CHECK: CXXDeductionGuideDecl {{.*}} <deduction guide for Derived> 'auto (T, A) -> Derived<T, 1, A>'
 // CHECK: CXXDeductionGuideDecl {{.*}} <deduction guide for Derived> 'auto (int, int) -> Derived<int, 1, int>'
diff --git a/clang/test/AST/float16.cpp b/clang/test/AST/float16.cpp
index a9e1144cf0958..643d37cfc9436 100644
--- a/clang/test/AST/float16.cpp
+++ b/clang/test/AST/float16.cpp
@@ -126,7 +126,7 @@ class C1 {
 };
 
 //CHECK:      |-CXXRecordDecl {{.*}} referenced class C1 definition
-//CHECK:      | |-CXXRecordDecl {{.*}} implicit referenced class C1
+//CHECK:      | |-CXXRecordDecl {{.*}} implicit class C1
 //CHECK-NEXT: | |-FieldDecl {{.*}} referenced f1c '_Float16'
 //CHECK-NEXT: | |-VarDecl {{.*}} used f2c 'const _Float16' static
 //CHECK-NEXT: | |-FieldDecl {{.*}} f3c 'volatile _Float16'
diff --git a/clang/test/AST/sourceranges.cpp b/clang/test/AST/sourceranges.cpp
index f78d34c1ee7e3..598a28bddad60 100644
--- a/clang/test/AST/sourceranges.cpp
+++ b/clang/test/AST/sourceranges.cpp
@@ -47,7 +47,7 @@ struct D {
 void construct() {
   using namespace foo;
   A a = A(12);
-  // CHECK: CXXConstructExpr {{0x[0-9a-fA-F]+}} <col:9, col:13> 'A':'foo::A' 'void (int){{( __attribute__\(\(thiscall\)\))?}}'
+  // CHECK: CXXConstructExpr {{0x[0-9a-fA-F]+}} <col:9, col:13> 'A' 'void (int){{( __attribute__\(\(thiscall\)\))?}}'
   D d = D(12);
   // CHECK: CXXConstructExpr {{0x[0-9a-fA-F]+}} <col:9, col:13> 'D' 'void (int){{( __attribute__\(\(thiscall\)\))?}}'
 }
@@ -174,7 +174,7 @@ namespace in_class_init {
 
   // CHECK-1Z: CXXRecordDecl {{.*}} struct B definition
   struct B {
-    // CHECK-1Z: FieldDecl {{.*}} a 'A':'in_class_init::A'
+    // CHECK-1Z: FieldDecl {{.*}} a 'A'
     // CHECK-1Z-NEXT: InitListExpr {{.*}} <col:11, col:12
     A a = {};
   };
@@ -192,7 +192,7 @@ namespace delegating_constructor_init {
   // CHECK-1Z: CXXRecordDecl {{.*}} struct C definition
   struct C : B {
     // CHECK-1Z: CXXConstructorDecl {{.*}} C
-    // CHECK-1Z-NEXT: CXXCtorInitializer 'B':'delegating_constructor_init::B'
+    // CHECK-1Z-NEXT: CXXCtorInitializer 'B'
     // CHECK-1Z-NEXT: CXXConstructExpr {{.*}} <col:11, col:15
     // CHECK-1Z-NEXT: InitListExpr {{.*}} <col:13, col:14
     C() : B({}) {};
diff --git a/clang/test/ASTSYCL/ast-dump-sycl-kernel-call-stmt.cpp b/clang/test/ASTSYCL/ast-dump-sycl-kernel-call-stmt.cpp
index 27604e237adbb..8e8e03c2451a0 100644
--- a/clang/test/ASTSYCL/ast-dump-sycl-kernel-call-stmt.cpp
+++ b/clang/test/ASTSYCL/ast-dump-sycl-kernel-call-stmt.cpp
@@ -64,10 +64,10 @@ void skep2<KN<2>>(K<2>);
 
 // CHECK-NEXT: | `-FunctionDecl {{.*}} skep2 'void (K<2>)' explicit_instantiation_definition
 // CHECK-NEXT: |   |-TemplateArgument type 'KN<2>'
-// CHECK-NEXT: |   | `-RecordType {{.*}} 'KN<2>'
+// CHECK-NEXT: |   | `-RecordType {{.*}} 'KN<2>' canonical
 // CHECK-NEXT: |   |   `-ClassTemplateSpecialization {{.*}} 'KN'
 // CHECK-NEXT: |   |-TemplateArgument type 'K<2>'
-// CHECK-NEXT: |   | `-RecordType {{.*}} 'K<2>'
+// CHECK-NEXT: |   | `-RecordType {{.*}} 'K<2>' canonical
 // CHECK-NEXT: |   |   `-ClassTemplateSpecialization {{.*}} 'K'
 // CHECK-NEXT: |   |-ParmVarDecl {{.*}} k 'K<2>'
 // CHECK-NEXT: |   |-SYCLKernelCallStmt {{.*}}
@@ -110,10 +110,10 @@ void skep3<KN<3>>(K<3> k) {
 // CHECK-NEXT: | `-Function {{.*}} 'skep3' 'void (K<3>)'
 // CHECK-NEXT: |-FunctionDecl {{.*}} skep3 'void (K<3>)' explicit_specialization
 // CHECK-NEXT: | |-TemplateArgument type 'KN<3>'
-// CHECK-NEXT: | | `-RecordType {{.*}} 'KN<3>'
+// CHECK-NEXT: | | `-RecordType {{.*}} 'KN<3>' canonical
 // CHECK-NEXT: | |   `-ClassTemplateSpecialization {{.*}} 'KN'
 // CHECK-NEXT: | |-TemplateArgument type 'K<3>'
-// CHECK-NEXT: | | `-RecordType {{.*}} 'K<3>'
+// CHECK-NEXT: | | `-RecordType {{.*}} 'K<3>' canonical
 // CHECK-NEXT: | |   `-ClassTemplateSpecialization {{.*}} 'K'
 // CHECK-NEXT: | |-ParmVarDecl {{.*}} k 'K<3>'
 // CHECK-NEXT: | |-SYCLKernelCallStmt {{.*}}
diff --git a/clang/test/Analysis/LifetimeSafety/benchmark.py b/clang/test/Analysis/LifetimeSafety/benchmark.py
index 9d5f36c51b9ee..4421fe9a81e21 100644
--- a/clang/test/Analysis/LifetimeSafety/benchmark.py
+++ b/clang/test/Analysis/LifetimeSafety/benchmark.py
@@ -99,28 +99,84 @@ def generate_cpp_merge_test(n: int) -> str:
     return cpp_code
 
 
-def analyze_trace_file(trace_path: str) -> tuple[float, float]:
+def generate_cpp_nested_loop_test(n: int) -> str:
     """
-    Parses the -ftime-trace JSON output to find durations.
+    Generates C++ code with N levels of nested loops.
+    This pattern tests how analysis performance scales with loop nesting depth,
+    which is a key factor in the complexity of dataflow analyses on structured
+    control flow.
 
-    Returns:
-        A tuple of (lifetime_analysis_duration_us, total_clang_duration_us).
+    Example (n=3):
+        struct MyObj { int id; ~MyObj() {} };
+        void nested_loops_3() {
+            MyObj* p = nullptr;
+            for(int i0=0; i0<2; ++i0) {
+                MyObj s0;
+                p = &s0;
+                for(int i1=0; i1<2; ++i1) {
+                    MyObj s1;
+                    p = &s1;
+                    for(int i2=0; i2<2; ++i2) {
+                        MyObj s2;
+                        p = &s2;
+                    }
+                }
+            }
+        }
+    """
+    if n <= 0:
+        return "// Nesting depth must be positive."
+
+    cpp_code = "struct MyObj { int id; ~MyObj() {} };\n\n"
+    cpp_code += f"void nested_loops_{n}() {{\n"
+    cpp_code += "    MyObj* p = nullptr;\n"
+
+    for i in range(n):
+        indent = "    " * (i + 1)
+        cpp_code += f"{indent}for(int i{i}=0; i{i}<2; ++i{i}) {{\n"
+        cpp_code += f"{indent}    MyObj s{i}; p = &s{i};\n"
+
+    for i in range(n - 1, -1, -1):
+        indent = "    " * (i + 1)
+        cpp_code += f"{indent}}}\n"
+
+    cpp_code += "}\n"
+    cpp_code += f"\nint main() {{ nested_loops_{n}(); return 0; }}\n"
+    return cpp_code
+
+
+def analyze_trace_file(trace_path: str) -> dict:
     """
-    lifetime_duration = 0.0
-    total_duration = 0.0
+    Parses the -ftime-trace JSON output to find durations for the lifetime
+    analysis and its sub-phases.
+    Returns a dictionary of durations in microseconds.
+    """
+    durations = {
+        "lifetime_us": 0.0,
+        "total_us": 0.0,
+        "fact_gen_us": 0.0,
+        "loan_prop_us": 0.0,
+        "expired_loans_us": 0.0,
+    }
+    event_name_map = {
+        "LifetimeSafetyAnalysis": "lifetime_us",
+        "ExecuteCompiler": "total_us",
+        "FactGenerator": "fact_gen_us",
+        "LoanPropagation": "loan_prop_us",
+        "ExpiredLoans": "expired_loans_us",
+    }
     try:
         with open(trace_path, "r") as f:
             trace_data = json.load(f)
             for event in trace_data.get("traceEvents", []):
-                if event.get("name") == "LifetimeSafetyAnalysis":
-                    lifetime_duration += float(event.get("dur", 0))
-                if event.get("name") == "ExecuteCompiler":
-                    total_duration += float(event.get("dur", 0))
-
+                event_name = event.get("name")
+                if event_name in event_name_map:
+                    key = event_name_map[event_name]
+                    durations[key] += float(event.get("dur", 0))
     except (IOError, json.JSONDecodeError) as e:
         print(f"Error reading or parsing trace file {trace_path}: {e}", file=sys.stderr)
-        return 0.0, 0.0
-    return lifetime_duration, total_duration
+        return {key: 0.0 for key in durations}
+    return durations
 
 
 def power_law(n, c, k):
@@ -135,8 +191,29 @@ def human_readable_time(ms: float) -> str:
     return f"{ms:.2f} ms"
 
 
+def calculate_complexity(n_data, y_data) -> tuple[float | None, float | None]:
+    """
+    Calculates the exponent 'k' for the power law fit y = c * n^k.
+    Returns a tuple of (k, k_standard_error).
+    """
+    try:
+        if len(n_data) < 3 or np.all(y_data < 1e-6) or np.var(y_data) < 1e-6:
+            return None, None
+
+        non_zero_indices = y_data > 0
+        if np.sum(non_zero_indices) < 3:
+            return None, None
+
+        n_fit, y_fit = n_data[non_zero_indices], y_data[non_zero_indices]
+        popt, pcov = curve_fit(power_law, n_fit, y_fit, p0=[0, 1], maxfev=5000)
+        k_stderr = np.sqrt(np.diag(pcov))[1]
+        return popt[1], k_stderr
+    except (RuntimeError, ValueError):
+        return None, None
+
+
 def generate_markdown_report(results: dict) -> str:
-    """Generates a Markdown-formatted report from the benchmark results."""
+    """Generates a concise, Markdown-formatted report from the benchmark results."""
     report = []
     timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S %Z")
     report.append(f"# Lifetime Analysis Performance Report")
@@ -146,54 +223,52 @@ def generate_markdown_report(results: dict) -> str:
     for test_name, data in results.items():
         title = data["title"]
         report.append(f"## Test Case: {title}")
-        report.append("")
+        report.append("\n**Timing Results:**\n")
 
         # Table header
-        report.append("| N   | Analysis Time | Total Clang Time |")
-        report.append("|:----|--------------:|-----------------:|")
+        report.append(
+            "| N (Input Size) | Total Time | Analysis Time (%) | Fact Generator (%) | Loan Propagation (%) | Expired Loans (%) |"
+        )
+        report.append(
+            "|:---------------|-----------:|------------------:|-------------------:|---------------------:|------------------:|"
+        )
 
         # Table rows
         n_data = np.array(data["n"])
-        analysis_data = np.array(data["lifetime_ms"])
-        total_data = np.array(data["total_ms"])
+        total_ms_data = np.array(data["total_ms"])
         for i in range(len(n_data)):
-            analysis_str = human_readable_time(analysis_data[i])
-            total_str = human_readable_time(total_data[i])
-            report.append(f"| {n_data[i]:<3} | {analysis_str:>13} | {total_str:>16} |")
-
-        report.append("")
-
-        # Complexity analysis
-        report.append(f"**Complexity Analysis:**")
-        try:
-            # Curve fitting requires at least 3 points
-            if len(n_data) < 3:
-                raise ValueError("Not enough data points to perform curve fitting.")
-
-            popt, pcov = curve_fit(
-                power_law, n_data, analysis_data, p0=[0, 2], maxfev=5000
-            )
-            _, k = popt
-
-            # Confidence Interval for k
-            alpha = 0.05  # 95% confidence
-            dof = max(0, len(n_data) - len(popt))  # degrees of freedom
-            t_val = t.ppf(1.0 - alpha / 2.0, dof)
-            # Standard error of the parameters
-            perr = np.sqrt(np.diag(pcov))
-            k_stderr = perr[1]
-            k_ci_lower = k - t_val * k_stderr
-            k_ci_upper = k + t_val * k_stderr
-
-            report.append(
-                f"- The performance for this case scales approx. as **O(n<sup>{k:.2f}</sup>)**."
-            )
-            report.append(
-                f"- **95% Confidence interval for exponent:** `[{k_ci_lower:.2f}, {k_ci_upper:.2f}]`."
-            )
+            total_t = total_ms_data[i]
+            if total_t < 1e-6:
+                total_t = 1.0  # Avoid division by zero
+
+            row = [
+                f"| {n_data[i]:<14} |",
+                f"{human_readable_time(total_t):>10} |",
+                f"{data['lifetime_ms'][i] / total_t * 100:>17.2f}% |",
+                f"{data['fact_gen_ms'][i] / total_t * 100:>18.2f}% |",
+                f"{data['loan_prop_ms'][i] / total_t * 100:>20.2f}% |",
+                f"{data['expired_loans_ms'][i] / total_t * 100:>17.2f}% |",
+            ]
+            report.append(" ".join(row))
+
+        report.append("\n**Complexity Analysis:**\n")
+        report.append("| Analysis Phase    | Complexity O(n<sup>k</sup>) |")
+        report.append("|:------------------|:--------------------------|")
+
+        analysis_phases = {
+            "Total Analysis": data["lifetime_ms"],
+            "FactGenerator": data["fact_gen_ms"],
+            "LoanPropagation": data["loan_prop_ms"],
+            "ExpiredLoans": data["expired_loans_ms"],
+        }
 
-        except (RuntimeError, ValueError) as e:
-            report.append(f"- Could not determine a best-fit curve for the data: {e}")
+        for phase_name, y_data in analysis_phases.items():
+            k, delta = calculate_complexity(n_data, np.array(y_data))
+            if k is not None and delta is not None:
+                complexity_str = f"O(n<sup>{k:.2f}</sup> &pm; {delta:.2f})"
+            else:
+                complexity_str = "(Negligible)"
+            report.append(f"| {phase_name:<17} | {complexity_str:<25} |")
 
         report.append("\n---\n")
 
@@ -202,7 +277,7 @@ def generate_markdown_report(results: dict) -> str:
 
 def run_single_test(
     clang_binary: str, output_dir: str, test_name: str, generator_func, n: int
-) -> tuple[float, float]:
+) -> dict:
     """Generates, compiles, and benchmarks a single test case."""
     print(f"--- Running Test: {test_name.capitalize()} with N={n} ---")
 
@@ -221,7 +296,8 @@ def run_single_test(
         "-o",
         "/dev/null",
         "-ftime-trace=" + trace_file,
-        "-Wexperimental-lifetime-safety",
+        "-Xclang",
+        "-fexperimental-lifetime-safety",
         "-std=c++17",
         source_file,
     ]
@@ -231,11 +307,12 @@ def run_single_test(
     if result.returncode != 0:
         print(f"Compilation failed for N={n}!", file=sys.stderr)
         print(result.stderr, file=sys.stderr)
-        return 0.0, 0.0
+        return {}
 
-    lifetime_us, total_us = analyze_trace_file(trace_file)
-
-    return lifetime_us / 1000.0, total_us / 1000.0
+    durations_us = analyze_trace_file(trace_file)
+    return {
+        key.replace("_us", "_ms"): value / 1000.0 for key, value in durations_us.items()
+    }
 
 
 if __name__ == "__main__":
@@ -270,6 +347,12 @@ def run_single_test(
             "generator_func": generate_cpp_merge_test,
             "n_values": [10, 50, 100, 200, 400, 800],
         },
+        {
+            "name": "nested_loops",
+            "title": "Deeply Nested Loops",
+            "generator_func": generate_cpp_nested_loop_test,
+            "n_values": [10, 50, 100, 200, 400, 800],
+        },
     ]
 
     results = {}
@@ -282,21 +365,28 @@ def run_single_test(
             "n": [],
             "lifetime_ms": [],
             "total_ms": [],
+            "fact_gen_ms": [],
+            "loan_prop_ms": [],
+            "expired_loans_ms": [],
         }
         for n in config["n_values"]:
-            lifetime_ms, total_ms = run_single_test(
+            durations_ms = run_single_test(
                 args.clang_binary,
                 args.output_dir,
                 test_name,
                 config["generator_func"],
                 n,
             )
-            if total_ms > 0:
+            if durations_ms:
                 results[test_name]["n"].append(n)
-                results[test_name]["lifetime_ms"].append(lifetime_ms)
-                results[test_name]["total_ms"].append(total_ms)
+                for key, value in durations_ms.items():
+                    results[test_name][key].append(value)
+
                 print(
-                    f"    Total: {human_readable_time(total_ms)} | Analysis: {human_readable_time(lifetime_ms)}"
+                    f"    Total Analysis: {human_readable_time(durations_ms['lifetime_ms'])} | "
+                    f"FactGen: {human_readable_time(durations_ms['fact_gen_ms'])} | "
+                    f"LoanProp: {human_readable_time(durations_ms['loan_prop_ms'])} | "
+                    f"ExpiredLoans: {human_readable_time(durations_ms['expired_loans_ms'])}"
                 )
 
     print("\n\n" + "=" * 80)
@@ -305,3 +395,8 @@ def run_single_test(
 
     markdown_report = generate_markdown_report(results)
     print(markdown_report)
+
+    report_filename = os.path.join(args.output_dir, "performance_report.md")
+    with open(report_filename, "w") as f:
+        f.write(markdown_report)
+    print(f"Report saved to: {report_filename}")
diff --git a/clang/test/Analysis/anonymous-decls.cpp b/clang/test/Analysis/anonymous-decls.cpp
index 85449caa46972..76e5155b61b67 100644
--- a/clang/test/Analysis/anonymous-decls.cpp
+++ b/clang/test/Analysis/anonymous-decls.cpp
@@ -78,12 +78,12 @@ int main() {
 // CHECK-NEXT:   8: decomposition-a-b
 // CHECK-NEXT:   9: [B3.7]([B3.8])
 // CHECK-NEXT:  10: [B3.9]
-// CHECK-NEXT:  11: std::tuple_element<0, std::pair<int, int>>::type a = get<0UL>(decomposition-a-b);
+// CHECK-NEXT:  11: std::tuple_element<0UL, std::pair<int, int>>::type &&a = get<0UL>(decomposition-a-b);
 // CHECK-NEXT:  12: get<1UL>
 // CHECK-NEXT:  13: [B3.12] (ImplicitCastExpr, FunctionToPointerDecay, tuple_element<1L, pair<int, int> >::type (*)(pair<int, int> &))
 // CHECK-NEXT:  14: decomposition-a-b
 // CHECK-NEXT:  15: [B3.13]([B3.14])
 // CHECK-NEXT:  16: [B3.15]
-// CHECK-NEXT:  17: std::tuple_element<1, std::pair<int, int>>::type b = get<1UL>(decomposition-a-b);
+// CHECK-NEXT:  17: std::tuple_element<1UL, std::pair<int, int>>::type &&b = get<1UL>(decomposition-a-b);
 // CHECK-NEXT:   Preds (1): B1
 // CHECK-NEXT:   Succs (1): B2
diff --git a/clang/test/Analysis/element-region-address-space.c b/clang/test/Analysis/element-region-address-space.c
index dd7066240fef6..da590556848b9 100644
--- a/clang/test/Analysis/element-region-address-space.c
+++ b/clang/test/Analysis/element-region-address-space.c
@@ -1,11 +1,27 @@
 // RUN: %clang_analyze_cc1 -triple amdgcn-unknown-unknown \
-// RUN:   -analyzer-checker=core -verify %s
+// RUN:   -Wno-incompatible-library-redeclaration \
+// RUN:   -analyzer-checker=core,unix -verify %s
 
 // expected-no-diagnostics
 //
 // By default, pointers are 64-bits.
+#define ADDRESS_SPACE_64BITS __attribute__((address_space(0)))
 #define ADDRESS_SPACE_32BITS __attribute__((address_space(3)))
 
 int test(ADDRESS_SPACE_32BITS int *p, ADDRESS_SPACE_32BITS void *q) {
   return p == q; // no-crash
 }
+
+// Make sure that the cstring checker handles non-default address spaces
+ADDRESS_SPACE_64BITS void *
+memcpy(ADDRESS_SPACE_64BITS void *,
+       ADDRESS_SPACE_32BITS const void *,
+       long unsigned int);
+
+ADDRESS_SPACE_64BITS struct {
+  char m[16];
+} n;
+
+void avoid_cstring_checker_crash(ADDRESS_SPACE_32BITS char *p) {
+  memcpy(&n.m[0], p, 4); // no-crash
+}
diff --git a/clang/test/Analysis/malloc-checker-arg-uaf.c b/clang/test/Analysis/malloc-checker-arg-uaf.c
new file mode 100644
index 0000000000000..d6aa85657fd46
--- /dev/null
+++ b/clang/test/Analysis/malloc-checker-arg-uaf.c
@@ -0,0 +1,44 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc -verify %s
+
+#include "Inputs/system-header-simulator-for-malloc.h"
+
+struct Obj {
+  int field;
+};
+
+void use(void *ptr);
+
+void test_direct_param_uaf() {
+  int *p = (int *)malloc(sizeof(int));
+  free(p);
+  use(p); // expected-warning{{Use of memory after it is released}}
+}
+
+void test_struct_field_uaf() {
+  struct Obj *o = (struct Obj *)malloc(sizeof(struct Obj));
+  free(o);
+  use(&o->field); // expected-warning{{Use of memory after it is released}}
+}
+
+void test_no_warning_const_int() {
+  use((void *)0x1234); // no-warning
+}
+
+void test_no_warning_stack() {
+  int x = 42;
+  use(&x); // no-warning
+}
+
+void test_nested_alloc() {
+  struct Obj *o = (struct Obj *)malloc(sizeof(struct Obj));
+  use(o);   // no-warning
+  free(o);
+  use(o);   // expected-warning{{Use of memory after it is released}}
+}
+
+void test_nested_field() {
+    struct Obj *o = (struct Obj *)malloc(sizeof(struct Obj));
+    int *f = &o->field;
+    free(o);
+    use(f); // expected-warning{{Use of memory after it is released}}
+}
diff --git a/clang/test/C/C11/n1285_1.c b/clang/test/C/C11/n1285_1.c
index 50100042e82fd..25b68e3145b04 100644
--- a/clang/test/C/C11/n1285_1.c
+++ b/clang/test/C/C11/n1285_1.c
@@ -26,16 +26,16 @@ struct X f(void);
 // C11-O2-NEXT:  [[ENTRY:.*:]]
 // C11-O2-NEXT:    [[P:%.*]] = alloca ptr, align 8
 // C11-O2-NEXT:    [[REF_TMP:%.*]] = alloca [[STRUCT_X:%.*]], align 4
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[P]]) #[[ATTR5:[0-9]+]]
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]]) #[[ATTR5:[0-9]+]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR5]]
 // C11-O2-NEXT:    call void @f(ptr dead_on_unwind writable sret([[STRUCT_X]]) align 4 [[REF_TMP]])
 // C11-O2-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_X]], ptr [[REF_TMP]], i32 0, i32 0
 // C11-O2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [5 x i32], ptr [[A]], i64 0, i64 0
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR5]]
 // C11-O2-NEXT:    store ptr [[ARRAYDECAY]], ptr [[P]], align 8, !tbaa [[TBAA2:![0-9]+]]
 // C11-O2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[P]], align 8, !tbaa [[TBAA2]]
 // C11-O2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA7:![0-9]+]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[P]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[P]]) #[[ATTR5]]
 // C11-O2-NEXT:    ret i32 [[TMP1]]
 //
 int func_return(void) {
@@ -79,7 +79,7 @@ int func_return(void) {
 // C11-O2-NEXT:    [[REF_TMP:%.*]] = alloca [[STRUCT_X:%.*]], align 4
 // C11-O2-NEXT:    [[Q:%.*]] = alloca ptr, align 8
 // C11-O2-NEXT:    [[DOTCOMPOUNDLITERAL:%.*]] = alloca [[STRUCT_X]], align 4
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR5]]
 // C11-O2-NEXT:    br i1 true, label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // C11-O2:       [[COND_TRUE]]:
 // C11-O2-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[REF_TMP]], i8 0, i64 20, i1 false)
@@ -92,8 +92,8 @@ int func_return(void) {
 // C11-O2-NEXT:    [[A1:%.*]] = getelementptr inbounds nuw [[STRUCT_X]], ptr [[REF_TMP]], i32 0, i32 0
 // C11-O2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [5 x i32], ptr [[A1]], i64 0, i64 0
 // C11-O2-NEXT:    store ptr [[ARRAYDECAY]], ptr @p, align 8, !tbaa [[TBAA2]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[Q]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[Q]]) #[[ATTR5]]
 // C11-O2-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[DOTCOMPOUNDLITERAL]], i8 0, i64 20, i1 false)
 // C11-O2-NEXT:    [[A2:%.*]] = getelementptr inbounds nuw [[STRUCT_X]], ptr [[DOTCOMPOUNDLITERAL]], i32 0, i32 0
 // C11-O2-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw [[STRUCT_X]], ptr [[DOTCOMPOUNDLITERAL]], i32 0, i32 0
@@ -104,7 +104,7 @@ int func_return(void) {
 // C11-O2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[Q]], align 8, !tbaa [[TBAA2]]
 // C11-O2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4, !tbaa [[TBAA7]]
 // C11-O2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP3]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[Q]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[Q]]) #[[ATTR5]]
 // C11-O2-NEXT:    ret i32 [[ADD]]
 //
 int ternary(void) {
@@ -133,16 +133,16 @@ int ternary(void) {
 // C11-O2-NEXT:  [[ENTRY:.*:]]
 // C11-O2-NEXT:    [[X:%.*]] = alloca [[STRUCT_X:%.*]], align 4
 // C11-O2-NEXT:    [[REF_TMP:%.*]] = alloca [[STRUCT_X]], align 4
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[X]]) #[[ATTR5]]
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[X]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR5]]
 // C11-O2-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[REF_TMP]], ptr align 4 [[X]], i64 20, i1 false), !tbaa.struct [[TBAA_STRUCT9:![0-9]+]]
 // C11-O2-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_X]], ptr [[REF_TMP]], i32 0, i32 0
 // C11-O2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [5 x i32], ptr [[A]], i64 0, i64 0
 // C11-O2-NEXT:    store ptr [[ARRAYDECAY]], ptr @p, align 8, !tbaa [[TBAA2]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR5]]
 // C11-O2-NEXT:    [[TMP0:%.*]] = load ptr, ptr @p, align 8, !tbaa [[TBAA2]]
 // C11-O2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA7]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[X]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]]) #[[ATTR5]]
 // C11-O2-NEXT:    ret i32 [[TMP1]]
 //
 int comma(void) {
@@ -170,16 +170,16 @@ int comma(void) {
 // C11-O2-NEXT:  [[ENTRY:.*:]]
 // C11-O2-NEXT:    [[X:%.*]] = alloca [[STRUCT_X:%.*]], align 4
 // C11-O2-NEXT:    [[REF_TMP:%.*]] = alloca [[STRUCT_X]], align 4
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[X]]) #[[ATTR5]]
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[X]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR5]]
 // C11-O2-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[REF_TMP]], ptr align 4 [[X]], i64 20, i1 false), !tbaa.struct [[TBAA_STRUCT9]]
 // C11-O2-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_X]], ptr [[REF_TMP]], i32 0, i32 0
 // C11-O2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [5 x i32], ptr [[A]], i64 0, i64 0
 // C11-O2-NEXT:    store ptr [[ARRAYDECAY]], ptr @p, align 8, !tbaa [[TBAA2]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR5]]
 // C11-O2-NEXT:    [[TMP0:%.*]] = load ptr, ptr @p, align 8, !tbaa [[TBAA2]]
 // C11-O2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA7]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[X]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]]) #[[ATTR5]]
 // C11-O2-NEXT:    ret i32 [[TMP1]]
 //
 int cast(void) {
@@ -210,19 +210,19 @@ int cast(void) {
 // C11-O2-NEXT:    [[X:%.*]] = alloca [[STRUCT_X:%.*]], align 4
 // C11-O2-NEXT:    [[S:%.*]] = alloca [[STRUCT_X]], align 4
 // C11-O2-NEXT:    [[REF_TMP:%.*]] = alloca [[STRUCT_X]], align 4
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[X]]) #[[ATTR5]]
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[S]]) #[[ATTR5]]
-// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[X]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[S]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR5]]
 // C11-O2-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[X]], ptr align 4 [[S]], i64 20, i1 false), !tbaa.struct [[TBAA_STRUCT9]]
 // C11-O2-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[REF_TMP]], ptr align 4 [[X]], i64 20, i1 false), !tbaa.struct [[TBAA_STRUCT9]]
 // C11-O2-NEXT:    [[A:%.*]] = getelementptr inbounds nuw [[STRUCT_X]], ptr [[REF_TMP]], i32 0, i32 0
 // C11-O2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [5 x i32], ptr [[A]], i64 0, i64 0
 // C11-O2-NEXT:    store ptr [[ARRAYDECAY]], ptr @p, align 8, !tbaa [[TBAA2]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[REF_TMP]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR5]]
 // C11-O2-NEXT:    [[TMP0:%.*]] = load ptr, ptr @p, align 8, !tbaa [[TBAA2]]
 // C11-O2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA7]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[S]]) #[[ATTR5]]
-// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[X]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[S]]) #[[ATTR5]]
+// C11-O2-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]]) #[[ATTR5]]
 // C11-O2-NEXT:    ret i32 [[TMP1]]
 //
 int assign(void) {
diff --git a/clang/test/CIR/CodeGen/array.cpp b/clang/test/CIR/CodeGen/array.cpp
index 141b67e0e63c7..a643de2d26189 100644
--- a/clang/test/CIR/CodeGen/array.cpp
+++ b/clang/test/CIR/CodeGen/array.cpp
@@ -45,9 +45,9 @@ int dd[3][2] = {{1, 2}, {3, 4}, {5, 6}};
 // OGCG: [i32 3, i32 4], [2 x i32] [i32 5, i32 6]]
 
 int e[10] = {1, 2};
-// CIR: cir.global external @e = #cir.const_array<[#cir.int<1> : !s32i, #cir.int<2> : !s32i], trailing_zeros> : !cir.array<!s32i x 10>
+// CIR: cir.global external @e = #cir.const_record<{#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.zero : !cir.array<!s32i x 8>}> : !rec_anon_struct
 
-// LLVM: @e = global [10 x i32] [i32 1, i32 2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0]
+// LLVM: @e = global <{ i32, i32, [8 x i32] }> <{ i32 1, i32 2, [8 x i32] zeroinitializer }>
 
 // OGCG: @e = global <{ i32, i32, [8 x i32] }> <{ i32 1, i32 2, [8 x i32] zeroinitializer }>
 
@@ -58,6 +58,28 @@ int f[5] = {1, 2};
 
 // OGCG: @f = global [5 x i32] [i32 1, i32 2, i32 0, i32 0, i32 0]
 
+int g[16] = {1, 2, 3, 4, 5, 6, 7, 8};
+// CIR:      cir.global external @g = #cir.const_record<{
+// CIR-SAME:   #cir.const_array<[#cir.int<1> : !s32i, #cir.int<2> : !s32i,
+// CIR-SAME:                     #cir.int<3> : !s32i, #cir.int<4> : !s32i,
+// CIR-SAME:                     #cir.int<5> : !s32i, #cir.int<6> : !s32i,
+// CIR-SAME:                     #cir.int<7> : !s32i, #cir.int<8> : !s32i]>
+// CIR-SAME:                     : !cir.array<!s32i x 8>,
+// CIR-SAME:   #cir.zero : !cir.array<!s32i x 8>}> : !rec_anon_struct1
+
+// LLVM:       @g = global <{ [8 x i32], [8 x i32] }> 
+// LLVM-SAME:          <{ [8 x i32]
+// LLVM-SAME:              [i32 1, i32 2, i32 3, i32 4,
+// LLVM-SAME:               i32 5, i32 6, i32 7, i32 8],
+// LLVM-SAME:             [8 x i32] zeroinitializer }>
+
+// OGCG:       @g = global <{ [8 x i32], [8 x i32] }> 
+// OGCG-SAME:          <{ [8 x i32]
+// OGCG-SAME:              [i32 1, i32 2, i32 3, i32 4,
+// OGCG-SAME:               i32 5, i32 6, i32 7, i32 8],
+// OGCG-SAME:             [8 x i32] zeroinitializer }>
+
+
 extern int b[10];
 // CIR: cir.global "private" external @b : !cir.array<!s32i x 10>
 // LLVM: @b = external global [10 x i32]
@@ -129,31 +151,50 @@ void func2() {
 }
 
 // CIR: %[[ARR2:.*]] = cir.alloca !cir.array<!s32i x 2>, !cir.ptr<!cir.array<!s32i x 2>>, ["arr", init]
-// CIR: %[[ELE_ALLOCA:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arrayinit.temp", init]
-// CIR: %[[ARR_2_PTR:.*]] = cir.cast(array_to_ptrdecay, %[[ARR2]] : !cir.ptr<!cir.array<!s32i x 2>>), !cir.ptr<!s32i>
-// CIR: %[[V1:.*]] = cir.const #cir.int<5> : !s32i
-// CIR: cir.store{{.*}} %[[V1]], %[[ARR_2_PTR]] : !s32i, !cir.ptr<!s32i>
+// CIR: %[[ARR_PTR:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["arrayinit.temp", init]
+// CIR: %[[ARR_0:.*]] = cir.cast(array_to_ptrdecay, %[[ARR2]] : !cir.ptr<!cir.array<!s32i x 2>>), !cir.ptr<!s32i>
+// CIR: %[[FIVE:.*]] = cir.const #cir.int<5> : !s32i
+// CIR: cir.store{{.*}} %[[FIVE]], %[[ARR_0]] : !s32i, !cir.ptr<!s32i>
 // CIR: %[[OFFSET_0:.*]] = cir.const #cir.int<1> : !s64i
-// CIR: %[[ELE_PTR:.*]] = cir.ptr_stride(%[[ARR_2_PTR]] : !cir.ptr<!s32i>, %[[OFFSET_0]] : !s64i), !cir.ptr<!s32i>
-// CIR: cir.store{{.*}} %[[ELE_PTR]], %[[ELE_ALLOCA]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
-// CIR: %[[LOAD_1:.*]] = cir.load{{.*}} %[[ELE_ALLOCA]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
-// CIR: %[[V2:.*]] = cir.const #cir.int<0> : !s32i
-// CIR: cir.store{{.*}} %[[V2]], %[[LOAD_1]] : !s32i, !cir.ptr<!s32i>
-// CIR: %[[OFFSET_1:.*]] = cir.const #cir.int<1> : !s64i
-// CIR: %[[ELE_1_PTR:.*]] = cir.ptr_stride(%[[LOAD_1]] : !cir.ptr<!s32i>, %[[OFFSET_1]] : !s64i), !cir.ptr<!s32i>
-// CIR: cir.store{{.*}} %[[ELE_1_PTR]], %[[ELE_ALLOCA]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+// CIR: %[[ELE_PTR:.*]] = cir.ptr_stride(%[[ARR_0]] : !cir.ptr<!s32i>, %[[OFFSET_0]] : !s64i), !cir.ptr<!s32i>
+// CIR: cir.store{{.*}} %[[ELE_PTR]], %[[ARR_PTR]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+// CIR: %[[TWO:.*]] = cir.const #cir.int<2> : !s64i
+// CIR: %[[ARR_END:.*]] = cir.ptr_stride(%[[ARR_0]] : !cir.ptr<!s32i>, %[[TWO]] : !s64i), !cir.ptr<!s32i>
+// CIR: cir.do {
+// CIR:   %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR:   %[[ZERO:.*]] = cir.const #cir.int<0> : !s32i
+// CIR:   cir.store{{.*}} %[[ZERO]], %[[ARR_CUR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[ONE:.*]] = cir.const #cir.int<1> : !s64i
+// CIR:   %[[ARR_NEXT:.*]] = cir.ptr_stride(%[[ARR_CUR]] : !cir.ptr<!s32i>, %[[ONE]] : !s64i), !cir.ptr<!s32i>
+// CIR:   cir.store{{.*}} %[[ARR_NEXT]], %[[ARR_PTR]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+// CIR:   cir.yield
+// CIR: } while {
+// CIR:   %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+// CIR:   %[[CMP:.*]] = cir.cmp(ne, %[[ARR_CUR]], %[[ARR_END]]) : !cir.ptr<!s32i>, !cir.bool
+// CIR:   cir.condition(%[[CMP]])
+// CIR: }
 
 // LLVM: define{{.*}} void @_Z5func2v()
-// LLVM:  %[[ARR:.*]] = alloca [2 x i32], i64 1, align 4
-// LLVM:  %[[TMP:.*]] = alloca ptr, i64 1, align 8
-// LLVM:  %[[ARR_PTR:.*]] = getelementptr i32, ptr %[[ARR]], i32 0
-// LLVM:  store i32 5, ptr %[[ARR_PTR]], align 4
-// LLVM:  %[[ELE_1_PTR:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i64 1
-// LLVM:  store ptr %[[ELE_1_PTR]], ptr %[[TMP]], align 8
-// LLVM:  %[[TMP2:.*]] = load ptr, ptr %[[TMP]], align 8
-// LLVM:  store i32 0, ptr %[[TMP2]], align 4
-// LLVM:  %[[ELE_1:.*]] = getelementptr i32, ptr %[[TMP2]], i64 1
-// LLVM:  store ptr %[[ELE_1]], ptr %[[TMP]], align 8
+// LLVM:   %[[ARR:.*]] = alloca [2 x i32], i64 1, align 4
+// LLVM:   %[[TMP:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[ARR_PTR:.*]] = getelementptr i32, ptr %[[ARR]], i32 0
+// LLVM:   store i32 5, ptr %[[ARR_PTR]], align 4
+// LLVM:   %[[ELE_1_PTR:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i64 1
+// LLVM:   store ptr %[[ELE_1_PTR]], ptr %[[TMP]], align 8
+// LLVM:   %[[END_PTR:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i64 2
+// LLVM:   br label %[[LOOP_BODY:.*]]
+// LLVM: [[LOOP_NEXT:.*]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// LLVM:   %[[CMP:.*]] = icmp ne ptr %[[CUR]], %[[END_PTR]]
+// LLVM:   br i1 %[[CMP]], label %[[LOOP_BODY]], label %[[LOOP_END:.*]]
+// LLVM: [[LOOP_BODY]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// LLVM:   store i32 0, ptr %[[CUR]], align 4
+// LLVM:   %[[NEXT:.*]] = getelementptr i32, ptr %[[CUR]], i64 1
+// LLVM:   store ptr %[[NEXT]], ptr %[[TMP]], align 8
+// LLVM:   br label %[[LOOP_NEXT:.*]]
+// LLVM: [[LOOP_END]]:
+// LLVM:   ret void
 
 // OGCG: %[[ARR:.*]] = alloca [2 x i32], align 4
 // OGCG: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[ARR]], ptr align 4 @[[FUN2_ARR]], i64 8, i1 false)
@@ -270,27 +311,46 @@ void func5() {
 // CIR: %[[V_0_0:.*]] = cir.const #cir.int<5> : !s32i
 // CIR: cir.store{{.*}} %[[V_0_0]], %[[ARR_0_PTR]] : !s32i, !cir.ptr<!s32i>
 // CIR: %[[OFFSET:.*]] = cir.const #cir.int<1> : !s64i
-// CIR: %6 = cir.ptr_stride(%[[ARR_0]] : !cir.ptr<!cir.array<!s32i x 1>>, %[[OFFSET]] : !s64i), !cir.ptr<!cir.array<!s32i x 1>>
-// CIR: cir.store{{.*}} %6, %[[ARR_PTR]] : !cir.ptr<!cir.array<!s32i x 1>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 1>>>
-// CIR: %7 = cir.load{{.*}} %[[ARR_PTR]] : !cir.ptr<!cir.ptr<!cir.array<!s32i x 1>>>, !cir.ptr<!cir.array<!s32i x 1>>
-// CIR: %8 = cir.const #cir.zero : !cir.array<!s32i x 1>
-// CIR: cir.store{{.*}} %8, %7 : !cir.array<!s32i x 1>, !cir.ptr<!cir.array<!s32i x 1>>
-// CIR: %[[OFFSET_1:.*]] = cir.const #cir.int<1> : !s64i
-// CIR: %10 = cir.ptr_stride(%7 : !cir.ptr<!cir.array<!s32i x 1>>, %[[OFFSET_1]] : !s64i), !cir.ptr<!cir.array<!s32i x 1>>
-// CIR: cir.store{{.*}} %10, %[[ARR_PTR]] : !cir.ptr<!cir.array<!s32i x 1>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 1>>>
+// CIR: %[[ARR_1:.*]] = cir.ptr_stride(%[[ARR_0]] : !cir.ptr<!cir.array<!s32i x 1>>, %[[OFFSET]] : !s64i), !cir.ptr<!cir.array<!s32i x 1>>
+// CIR: cir.store{{.*}} %[[ARR_1]], %[[ARR_PTR]] : !cir.ptr<!cir.array<!s32i x 1>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 1>>>
+// CIR: %[[TWO:.*]] = cir.const #cir.int<2> : !s64i
+// CIR: %[[ARR_END:.*]] = cir.ptr_stride(%[[ARR_0]] : !cir.ptr<!cir.array<!s32i x 1>>, %[[TWO]] : !s64i), !cir.ptr<!cir.array<!s32i x 1>>
+// CIR: cir.do {
+// CIR:   %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]] : !cir.ptr<!cir.ptr<!cir.array<!s32i x 1>>>, !cir.ptr<!cir.array<!s32i x 1>>
+// CIR:   %[[ZERO:.*]] = cir.const #cir.zero : !cir.array<!s32i x 1>
+// CIR:   cir.store{{.*}} %[[ZERO]], %[[ARR_CUR]] : !cir.array<!s32i x 1>, !cir.ptr<!cir.array<!s32i x 1>>
+// CIR:   %[[ONE:.*]] = cir.const #cir.int<1> : !s64i
+// CIR:   %[[ARR_NEXT:.*]] = cir.ptr_stride(%[[ARR_CUR]] : !cir.ptr<!cir.array<!s32i x 1>>, %[[ONE]] : !s64i), !cir.ptr<!cir.array<!s32i x 1>>
+// CIR:   cir.store{{.*}} %[[ARR_NEXT]], %[[ARR_PTR]] : !cir.ptr<!cir.array<!s32i x 1>>, !cir.ptr<!cir.ptr<!cir.array<!s32i x 1>>>
+// CIR:   cir.yield
+// CIR: } while {
+// CIR:   %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]] : !cir.ptr<!cir.ptr<!cir.array<!s32i x 1>>>, !cir.ptr<!cir.array<!s32i x 1>>
+// CIR:   %[[CMP:.*]] = cir.cmp(ne, %[[ARR_CUR]], %[[ARR_END]]) : !cir.ptr<!cir.array<!s32i x 1>>, !cir.bool
+// CIR:   cir.condition(%[[CMP]])
+// CIR: }
 
 // LLVM: define{{.*}} void @_Z5func5v()
-// LLVM:  %[[ARR:.*]] = alloca [2 x [1 x i32]], i64 1, align 4
-// LLVM:  %[[TMP:.*]] = alloca ptr, i64 1, align 8
-// LLVM:  %[[ARR_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR]], i32 0
-// LLVM:  %[[ARR_0:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i32 0
-// LLVM:  store i32 5, ptr %[[ARR_0]], align 4
-// LLVM:  %[[ARR_1:.*]] = getelementptr [1 x i32], ptr %[[ARR_PTR]], i64 1
-// LLVM:  store ptr %[[ARR_1]], ptr %[[TMP]], align 8
-// LLVM:  %[[ARR_1_VAL:.*]] = load ptr, ptr %[[TMP]], align 8
-// LLVM:  store [1 x i32] zeroinitializer, ptr %[[ARR_1_VAL]], align 4
-// LLVM:  %[[ARR_1_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR_1_VAL]], i64 1
-// LLVM:  store ptr %[[ARR_1_PTR]], ptr %[[TMP]], align 8
+// LLVM:   %[[ARR:.*]] = alloca [2 x [1 x i32]], i64 1, align 4
+// LLVM:   %[[TMP:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[ARR_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR]], i32 0
+// LLVM:   %[[ARR_0:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i32 0
+// LLVM:   store i32 5, ptr %[[ARR_0]], align 4
+// LLVM:   %[[ARR_1:.*]] = getelementptr [1 x i32], ptr %[[ARR_PTR]], i64 1
+// LLVM:   store ptr %[[ARR_1]], ptr %[[TMP]], align 8
+// LLVM:   %[[END_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR_PTR]], i64 2
+// LLVM:   br label %[[LOOP_BODY:.*]]
+// LLVM: [[LOOP_NEXT:.*]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// LLVM:   %[[CMP:.*]] = icmp ne ptr %[[CUR]], %[[END_PTR]]
+// LLVM:   br i1 %[[CMP]], label %[[LOOP_BODY]], label %[[LOOP_END:.*]]
+// LLVM: [[LOOP_BODY]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// LLVM:   store [1 x i32] zeroinitializer, ptr %[[CUR]], align 4
+// LLVM:   %[[NEXT:.*]] = getelementptr [1 x i32], ptr %[[CUR]], i64 1
+// LLVM:   store ptr %[[NEXT]], ptr %[[TMP]], align 8
+// LLVM:   br label %[[LOOP_NEXT:.*]]
+// LLVM: [[LOOP_END]]:
+// LLVM:   ret void
 
 // ORGC: %[[ARR:.*]] = alloca [2 x [1 x i32]], align 4
 // ORGC: call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[ARR]], ptr align 4 @[[FUN5_ARR]], i64 8, i1 false)
@@ -335,25 +395,44 @@ void func7() {
 }
 
 // CIR: %[[ARR:.*]] = cir.alloca !cir.array<!cir.ptr<!s32i> x 1>, !cir.ptr<!cir.array<!cir.ptr<!s32i> x 1>>, ["arr", init]
-// CIR: %[[ARR_TMP:.*]] = cir.alloca !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>, ["arrayinit.temp", init]
-// CIR: %[[ARR_PTR:.*]] = cir.cast(array_to_ptrdecay, %[[ARR]] : !cir.ptr<!cir.array<!cir.ptr<!s32i> x 1>>), !cir.ptr<!cir.ptr<!s32i>>
-// CIR: cir.store{{.*}} %[[ARR_PTR]], %[[ARR_TMP]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>
-// CIR: %[[TMP:.*]] = cir.load{{.*}} %[[ARR_TMP]] : !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>, !cir.ptr<!cir.ptr<!s32i>>
-// CIR: %[[NULL_PTR:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!s32i>
-// CIR: cir.store{{.*}} %[[NULL_PTR]], %[[TMP]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
-// CIR: %[[OFFSET:.*]] = cir.const #cir.int<1> : !s64i
-// CIR: %[[ELE_PTR:.*]] = cir.ptr_stride(%[[TMP]] : !cir.ptr<!cir.ptr<!s32i>>, %[[OFFSET]] : !s64i), !cir.ptr<!cir.ptr<!s32i>>
-// CIR: cir.store{{.*}} %[[ELE_PTR]], %[[ARR_TMP]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>
+// CIR: %[[ARR_PTR:.*]] = cir.alloca !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>, ["arrayinit.temp", init]
+// CIR: %[[ARR_0:.*]] = cir.cast(array_to_ptrdecay, %[[ARR]] : !cir.ptr<!cir.array<!cir.ptr<!s32i> x 1>>), !cir.ptr<!cir.ptr<!s32i>>
+// CIR: cir.store{{.*}} %[[ARR_0]], %[[ARR_PTR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>
+// CIR: %[[ONE:.*]] = cir.const #cir.int<1> : !s64i
+// CIR: %[[ARR_END:.*]] = cir.ptr_stride(%[[ARR_0]] : !cir.ptr<!cir.ptr<!s32i>>, %[[ONE]] : !s64i), !cir.ptr<!cir.ptr<!s32i>>
+// CIR: cir.do {
+// CIR:   %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]] : !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>, !cir.ptr<!cir.ptr<!s32i>>
+// CIR:   %[[NULL_PTR:.*]] = cir.const #cir.ptr<null> : !cir.ptr<!s32i>
+// CIR:   cir.store{{.*}} %[[NULL_PTR]], %[[ARR_CUR]] : !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>
+// CIR:   %[[ONE:.*]] = cir.const #cir.int<1> : !s64i
+// CIR:   %[[ARR_NEXT:.*]] = cir.ptr_stride(%[[ARR_CUR]] : !cir.ptr<!cir.ptr<!s32i>>, %[[ONE]] : !s64i), !cir.ptr<!cir.ptr<!s32i>>
+// CIR:   cir.store{{.*}} %[[ARR_NEXT]], %[[ARR_PTR]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>
+// CIR:   cir.yield
+// CIR: } while {
+// CIR:   %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]] : !cir.ptr<!cir.ptr<!cir.ptr<!s32i>>>, !cir.ptr<!cir.ptr<!s32i>>
+// CIR:   %[[CMP:.*]] = cir.cmp(ne, %[[ARR_CUR]], %[[ARR_END]]) : !cir.ptr<!cir.ptr<!s32i>>, !cir.bool
+// CIR:   cir.condition(%[[CMP]])
+// CIR: }
 
 // LLVM: define{{.*}} void @_Z5func7v()
-// LLVM:  %[[ARR:.*]] = alloca [1 x ptr], i64 1, align 8
-// LLVM:  %[[ALLOCA:.*]] = alloca ptr, i64 1, align 8
-// LLVM:  %[[ELE_PTR:.*]] = getelementptr ptr, ptr %[[ARR]], i32 0
-// LLVM:  store ptr %[[ELE_PTR]], ptr %[[ALLOCA]], align 8
-// LLVM:  %[[TMP:.*]] = load ptr, ptr %[[ALLOCA]], align 8
-// LLVM:  store ptr null, ptr %[[TMP]], align 8
-// LLVM:  %[[ELE:.*]] = getelementptr ptr, ptr %[[TMP]], i64 1
-// LLVM:  store ptr %[[ELE]], ptr %[[ALLOCA]], align 8
+// LLVM:   %[[ARR:.*]] = alloca [1 x ptr], i64 1, align 8
+// LLVM:   %[[TMP:.*]] = alloca ptr, i64 1, align 8
+// LLVM:   %[[ARR_PTR:.*]] = getelementptr ptr, ptr %[[ARR]], i32 0
+// LLVM:   store ptr %[[ARR_PTR]], ptr %[[TMP]], align 8
+// LLVM:   %[[END_PTR:.*]] = getelementptr ptr, ptr %[[ARR_PTR]], i64 1
+// LLVM:   br label %[[LOOP_BODY:.*]]
+// LLVM: [[LOOP_NEXT:.*]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// LLVM:   %[[CMP:.*]] = icmp ne ptr %[[CUR]], %[[END_PTR]]
+// LLVM:   br i1 %[[CMP]], label %[[LOOP_BODY]], label %[[LOOP_END:.*]]
+// LLVM: [[LOOP_BODY]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// LLVM:   store ptr null, ptr %[[CUR]], align 8
+// LLVM:   %[[NEXT:.*]] = getelementptr ptr, ptr %[[CUR]], i64 1
+// LLVM:   store ptr %[[NEXT]], ptr %[[TMP]], align 8
+// LLVM:   br label %[[LOOP_NEXT:.*]]
+// LLVM: [[LOOP_END]]:
+// LLVM:   ret void
 
 // OGCG: %[[ARR:.*]] = alloca [1 x ptr], align 8
 // OGCG: call void @llvm.memset.p0.i64(ptr align 8 %[[ARR]], i8 0, i64 8, i1 false)
diff --git a/clang/test/CIR/CodeGen/atomic.c b/clang/test/CIR/CodeGen/atomic.c
new file mode 100644
index 0000000000000..6c37fb7cd432d
--- /dev/null
+++ b/clang/test/CIR/CodeGen/atomic.c
@@ -0,0 +1,153 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+void f1(void) {
+  _Atomic(int) x = 42;
+}
+
+// CIR-LABEL: @f1
+// CIR:         %[[SLOT:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init] {alignment = 4 : i64}
+// CIR-NEXT:    %[[INIT:.+]] = cir.const #cir.int<42> : !s32i
+// CIR-NEXT:    cir.store align(4) %[[INIT]], %[[SLOT]] : !s32i, !cir.ptr<!s32i>
+// CIR:       }
+
+// LLVM-LABEL: @f1
+// LLVM:         %[[SLOT:.+]] = alloca i32, i64 1, align 4
+// LLVM-NEXT:    store i32 42, ptr %[[SLOT]], align 4
+// LLVM:       }
+
+// OGCG-LABEL: @f1
+// OGCG:         %[[SLOT:.+]] = alloca i32, align 4
+// OGCG-NEXT:    store i32 42, ptr %[[SLOT]], align 4
+// OGCG:       }
+
+void f2(void) {
+  _Atomic(int) x;
+  __c11_atomic_init(&x, 42);
+}
+
+// CIR-LABEL: @f2
+// CIR:         %[[SLOT:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["x"] {alignment = 4 : i64}
+// CIR-NEXT:    %[[INIT:.+]] = cir.const #cir.int<42> : !s32i
+// CIR-NEXT:    cir.store align(4) %[[INIT]], %[[SLOT]] : !s32i, !cir.ptr<!s32i>
+// CIR:       }
+
+// LLVM-LABEL: @f2
+// LLVM:         %[[SLOT:.+]] = alloca i32, i64 1, align 4
+// LLVM-NEXT:    store i32 42, ptr %[[SLOT]], align 4
+// LLVM:       }
+
+// OGCG-LABEL: @f2
+// OGCG:         %[[SLOT:.+]] = alloca i32, align 4
+// OGCG-NEXT:    store i32 42, ptr %[[SLOT]], align 4
+// OGCG:       }
+
+void load(int *ptr) {
+  int x;
+  __atomic_load(ptr, &x, __ATOMIC_RELAXED);
+  __atomic_load(ptr, &x, __ATOMIC_CONSUME);
+  __atomic_load(ptr, &x, __ATOMIC_ACQUIRE);
+  __atomic_load(ptr, &x, __ATOMIC_SEQ_CST);
+}
+
+// CIR-LABEL: @load
+// CIR:   %{{.+}} = cir.load align(4) atomic(relaxed) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR:   %{{.+}} = cir.load align(4) atomic(consume) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR:   %{{.+}} = cir.load align(4) atomic(acquire) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR:   %{{.+}} = cir.load align(4) atomic(seq_cst) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR: }
+
+// LLVM-LABEL: @load
+// LLVM:   %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
+// LLVM:   %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
+// LLVM:   %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
+// LLVM:   %{{.+}} = load atomic i32, ptr %{{.+}} seq_cst, align 4
+// LLVM: }
+
+// OGCG-LABEL: @load
+// OGCG:   %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
+// OGCG:   %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
+// OGCG:   %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
+// OGCG:   %{{.+}} = load atomic i32, ptr %{{.+}} seq_cst, align 4
+// OGCG: }
+
+void c11_load(_Atomic(int) *ptr) {
+  __c11_atomic_load(ptr, __ATOMIC_RELAXED);
+  __c11_atomic_load(ptr, __ATOMIC_CONSUME);
+  __c11_atomic_load(ptr, __ATOMIC_ACQUIRE);
+  __c11_atomic_load(ptr, __ATOMIC_SEQ_CST);
+}
+
+// CIR-LABEL: @c11_load
+// CIR:   %{{.+}} = cir.load align(4) atomic(relaxed) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR:   %{{.+}} = cir.load align(4) atomic(consume) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR:   %{{.+}} = cir.load align(4) atomic(acquire) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR:   %{{.+}} = cir.load align(4) atomic(seq_cst) %{{.+}} : !cir.ptr<!s32i>, !s32i
+// CIR: }
+
+// LLVM-LABEL: @c11_load
+// LLVM:   %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
+// LLVM:   %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
+// LLVM:   %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
+// LLVM:   %{{.+}} = load atomic i32, ptr %{{.+}} seq_cst, align 4
+// LLVM: }
+
+// OGCG-LABEL: @c11_load
+// OGCG:   %{{.+}} = load atomic i32, ptr %{{.+}} monotonic, align 4
+// OGCG:   %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
+// OGCG:   %{{.+}} = load atomic i32, ptr %{{.+}} acquire, align 4
+// OGCG:   %{{.+}} = load atomic i32, ptr %{{.+}} seq_cst, align 4
+// OGCG: }
+
+void store(int *ptr, int x) {
+  __atomic_store(ptr, &x, __ATOMIC_RELAXED);
+  __atomic_store(ptr, &x, __ATOMIC_RELEASE);
+  __atomic_store(ptr, &x, __ATOMIC_SEQ_CST);
+}
+
+// CIR-LABEL: @store
+// CIR:   cir.store align(4) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.store align(4) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.store align(4) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
+// CIR: }
+
+// LLVM-LABEL: @store
+// LLVM:   store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
+// LLVM:   store atomic i32 %{{.+}}, ptr %{{.+}} release, align 4
+// LLVM:   store atomic i32 %{{.+}}, ptr %{{.+}} seq_cst, align 4
+// LLVM: }
+
+// OGCG-LABEL: @store
+// OGCG:   store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
+// OGCG:   store atomic i32 %{{.+}}, ptr %{{.+}} release, align 4
+// OGCG:   store atomic i32 %{{.+}}, ptr %{{.+}} seq_cst, align 4
+// OGCG: }
+
+void c11_store(_Atomic(int) *ptr, int x) {
+  __c11_atomic_store(ptr, x, __ATOMIC_RELAXED);
+  __c11_atomic_store(ptr, x, __ATOMIC_RELEASE);
+  __c11_atomic_store(ptr, x, __ATOMIC_SEQ_CST);
+}
+
+// CIR-LABEL: @c11_store
+// CIR:   cir.store align(4) atomic(relaxed) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.store align(4) atomic(release) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
+// CIR:   cir.store align(4) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
+// CIR: }
+
+// LLVM-LABEL: @c11_store
+// LLVM:   store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
+// LLVM:   store atomic i32 %{{.+}}, ptr %{{.+}} release, align 4
+// LLVM:   store atomic i32 %{{.+}}, ptr %{{.+}} seq_cst, align 4
+// LLVM: }
+
+// OGCG-LABEL: @c11_store
+// OGCG:   store atomic i32 %{{.+}}, ptr %{{.+}} monotonic, align 4
+// OGCG:   store atomic i32 %{{.+}}, ptr %{{.+}} release, align 4
+// OGCG:   store atomic i32 %{{.+}}, ptr %{{.+}} seq_cst, align 4
+// OGCG: }
+
diff --git a/clang/test/CIR/CodeGen/bitfield-union.c b/clang/test/CIR/CodeGen/bitfield-union.c
index b5d14540f488f..14a2aaf68d318 100644
--- a/clang/test/CIR/CodeGen/bitfield-union.c
+++ b/clang/test/CIR/CodeGen/bitfield-union.c
@@ -28,3 +28,44 @@ typedef union {
 
 demo d;
 zero_bit z;
+
+void f() {
+    demo d;
+    d.x = 1;
+    d.y = 2;
+    d.z = 0;
+}
+
+// CIR: #bfi_y = #cir.bitfield_info<name = "y", storage_type = !u8i, size = 4, offset = 0, is_signed = true>
+// CIR: #bfi_z = #cir.bitfield_info<name = "z", storage_type = !u8i, size = 8, offset = 0, is_signed = true>
+
+// CIR:   cir.func no_proto dso_local @f
+// CIR:    [[ALLOC:%.*]] = cir.alloca !rec_demo, !cir.ptr<!rec_demo>, ["d"] {alignment = 4 : i64}
+// CIR:    [[ONE:%.*]] = cir.const #cir.int<1> : !s32i
+// CIR:    [[X:%.*]] = cir.get_member [[ALLOC]][0] {name = "x"} : !cir.ptr<!rec_demo> -> !cir.ptr<!s32i>
+// CIR:    cir.store align(4) [[ONE]], [[X]] : !s32i, !cir.ptr<!s32i>
+// CIR:    [[TWO:%.*]] = cir.const #cir.int<2> : !s32i
+// CIR:    [[Y:%.*]] = cir.get_member [[ALLOC]][1] {name = "y"} : !cir.ptr<!rec_demo> -> !cir.ptr<!u8i>
+// CIR:    [[SET:%.*]] = cir.set_bitfield align(4) (#bfi_y, [[Y]] : !cir.ptr<!u8i>, [[TWO]] : !s32i) -> !s32i
+// CIR:    [[ZERO:%.*]] = cir.const #cir.int<0> : !s32i
+// CIR:    [[Z:%.*]] = cir.get_member [[ALLOC]][2] {name = "z"} : !cir.ptr<!rec_demo> -> !cir.ptr<!u8i>
+// CIR:    [[SET2:%.*]] = cir.set_bitfield align(4) (#bfi_z, [[Z]] : !cir.ptr<!u8i>, [[ZERO]] : !s32i) -> !s32i
+// CIR:    cir.return
+
+// LLVM: define dso_local void @f
+// LLVM:   [[ALLOC:%.*]] = alloca %union.demo, i64 1, align 4
+// LLVM:   store i32 1, ptr [[ALLOC]], align 4
+// LLVM:   [[BFLOAD:%.*]] = load i8, ptr [[ALLOC]], align 4
+// LLVM:   [[CLEAR:%.*]] = and i8 [[BFLOAD]], -16
+// LLVM:   [[SET:%.*]] = or i8 [[CLEAR]], 2
+// LLVM:   store i8 [[SET]], ptr [[ALLOC]], align 4
+// LLVM:   store i8 0, ptr [[ALLOC]], align 4
+
+// OGCG: define dso_local void @f
+// OGCG:   [[ALLOC:%.*]] = alloca %union.demo, align 4
+// OGCG:   store i32 1, ptr [[ALLOC]], align 4
+// OGCG:   [[BFLOAD:%.*]] = load i8, ptr [[ALLOC]], align 4
+// OGCG:   [[CLEAR:%.*]] = and i8 [[BFLOAD]], -16
+// OGCG:   [[SET:%.*]] = or i8 [[CLEAR]], 2
+// OGCG:   store i8 [[SET]], ptr [[ALLOC]], align 4
+// OGCG:   store i8 0, ptr [[ALLOC]], align 4
diff --git a/clang/test/CIR/CodeGen/builtin_call.cpp b/clang/test/CIR/CodeGen/builtin_call.cpp
index c266f1a6d1637..09be7937a4330 100644
--- a/clang/test/CIR/CodeGen/builtin_call.cpp
+++ b/clang/test/CIR/CodeGen/builtin_call.cpp
@@ -111,6 +111,38 @@ void assume(bool arg) {
 // OGCG:   call void @llvm.assume(i1 %{{.+}})
 // OGCG: }
 
+void *assume_aligned(void *ptr) {
+  return __builtin_assume_aligned(ptr, 16);
+}
+
+// CIR: @_Z14assume_alignedPv
+// CIR:   %{{.+}} = cir.assume_aligned %{{.+}} alignment 16 : !cir.ptr<!void>
+// CIR: }
+
+// LLVM: @_Z14assume_alignedPv
+// LLVM:   call void @llvm.assume(i1 true) [ "align"(ptr %{{.+}}, i64 16) ]
+// LLVM: }
+
+// OGCG: @_Z14assume_alignedPv
+// OGCG:   call void @llvm.assume(i1 true) [ "align"(ptr %{{.+}}, i64 16) ]
+// OGCG: }
+
+void *assume_aligned_misalignment(void *ptr, unsigned misalignment) {
+  return __builtin_assume_aligned(ptr, 16, misalignment);
+}
+
+// CIR: @_Z27assume_aligned_misalignmentPvj
+// CIR:   %{{.+}} = cir.assume_aligned %{{.+}} alignment 16[offset %{{.+}} : !u64i] : !cir.ptr<!void>
+// CIR: }
+
+// LLVM: @_Z27assume_aligned_misalignmentPvj
+// LLVM:   call void @llvm.assume(i1 true) [ "align"(ptr %{{.+}}, i64 16, i64 %{{.+}}) ]
+// LLVM: }
+
+// OGCG: @_Z27assume_aligned_misalignmentPvj
+// OGCG:   call void @llvm.assume(i1 true) [ "align"(ptr %{{.+}}, i64 16, i64 %{{.+}}) ]
+// OGCG: }
+
 void assume_separate_storage(void *p1, void *p2) {
   __builtin_assume_separate_storage(p1, p2);
 }
diff --git a/clang/test/CIR/CodeGen/builtin_printf.cpp b/clang/test/CIR/CodeGen/builtin_printf.cpp
index 43128e44b19e4..80875c349bfcf 100644
--- a/clang/test/CIR/CodeGen/builtin_printf.cpp
+++ b/clang/test/CIR/CodeGen/builtin_printf.cpp
@@ -5,10 +5,10 @@
 // RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
 // RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
 
-// CIR: cir.global "private" cir_private dso_local @".str" = #cir.const_array<"%s\00" : !cir.array<!s8i x 3>> : !cir.array<!s8i x 3> 
-// CIR: cir.global "private" cir_private dso_local @".str.1" = #cir.const_array<"%s %d\0A\00" : !cir.array<!s8i x 7>> : !cir.array<!s8i x 7>
-// LLVM: @.str = private global [3 x i8] c"%s\00"
-// LLVM: @.str.1 = private global [7 x i8] c"%s %d\0A\00"
+// CIR: cir.global "private" constant cir_private dso_local @".str" = #cir.const_array<"%s\00" : !cir.array<!s8i x 3>> : !cir.array<!s8i x 3> 
+// CIR: cir.global "private" constant cir_private dso_local @".str.1" = #cir.const_array<"%s %d\0A\00" : !cir.array<!s8i x 7>> : !cir.array<!s8i x 7>
+// LLVM: @.str = private constant [3 x i8] c"%s\00"
+// LLVM: @.str.1 = private constant [7 x i8] c"%s %d\0A\00"
 // OGCG: @.str = private unnamed_addr constant [3 x i8] c"%s\00"
 // OGCG: @.str.1 = private unnamed_addr constant [7 x i8] c"%s %d\0A\00"
 
diff --git a/clang/test/CIR/CodeGen/builtins.cpp b/clang/test/CIR/CodeGen/builtins.cpp
new file mode 100644
index 0000000000000..0e434809fe6be
--- /dev/null
+++ b/clang/test/CIR/CodeGen/builtins.cpp
@@ -0,0 +1,42 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+double fabs(double x) {
+  return __builtin_fabs(x);
+}
+
+// CIR: {{.*}} = cir.fabs {{.*}} : !cir.double
+// LLVM: {{.*}} = call double @llvm.fabs.f64(double {{.*}})
+// OGCG: {{.*}} = call double @llvm.fabs.f64(double {{.*}})
+
+extern "C" void *test_return_address(void) {
+  return __builtin_return_address(1);
+
+  // CIR-LABEL: test_return_address
+  // CIR: [[ARG:%.*]] = cir.const #cir.int<1> : !u32i
+  // CIR: {{%.*}} = cir.return_address([[ARG]])
+
+  // LLVM-LABEL: @test_return_address
+  // LLVM: {{%.*}} = call ptr @llvm.returnaddress(i32 1)
+
+  // OGCG-LABEL: @test_return_address
+  // OGCG: {{%.*}} = call ptr @llvm.returnaddress(i32 1)
+}
+
+extern "C" void *test_frame_address(void) {
+  return __builtin_frame_address(1);
+
+  // CIR-LABEL: test_frame_address
+  // CIR: [[ARG:%.*]] = cir.const #cir.int<1> : !u32i
+  // CIR: {{%.*}} = cir.frame_address([[ARG]])
+
+  // LLVM-LABEL: @test_frame_address
+  // LLVM: {{%.*}} = call ptr @llvm.frameaddress.p0(i32 1)
+
+  // OGCG-LABEL: @test_frame_address
+  // OGCG: {{%.*}} = call ptr @llvm.frameaddress.p0(i32 1)
+}
diff --git a/clang/test/CIR/CodeGen/call.cpp b/clang/test/CIR/CodeGen/call.cpp
index 43e5d150a949f..3e8cfc1cceb51 100644
--- a/clang/test/CIR/CodeGen/call.cpp
+++ b/clang/test/CIR/CodeGen/call.cpp
@@ -116,4 +116,18 @@ void f14() {
 // LLVM:         call void @_Z3f13v() #[[LLVM_ATTR_0:.+]]
 // LLVM:       }
 
-// LLLVM: attributes #[[LLVM_ATTR_0]] = { nounwind }
+int f15();
+void f16() {
+  using T = int;
+  f15().~T();
+}
+
+// CIR-LABEL: @_Z3f16v
+// CIR-NEXT:    %{{.+}} = cir.call @_Z3f15v() : () -> !s32i
+// CIR:       }
+
+// LLVM-LABEL: define{{.+}} void @_Z3f16v() {
+// LLVM-NEXT:    %{{.+}} = call i32 @_Z3f15v()
+// LLVM:       }
+
+// LLVM: attributes #[[LLVM_ATTR_0]] = { nounwind }
diff --git a/clang/test/CIR/CodeGen/class.cpp b/clang/test/CIR/CodeGen/class.cpp
index 43dde12df40f0..eb9d5d73c3616 100644
--- a/clang/test/CIR/CodeGen/class.cpp
+++ b/clang/test/CIR/CodeGen/class.cpp
@@ -100,3 +100,22 @@ int use_base_via_pointer(Derived *d) {
 
 // OGCG: define{{.*}} i32 @_Z20use_base_via_pointerP7Derived
 // OGCG:   %[[D_A_ADDR:.*]] = getelementptr inbounds nuw %class.Base, ptr %{{.*}}, i32 0, i32 0
+
+struct EmptyDerived : Base {};
+struct EmptyDerived2 : EmptyDerived {};
+
+void use_empty_derived2() {
+  EmptyDerived2 d2;
+}
+
+// CIR: cir.func{{.*}} @_Z18use_empty_derived2v()
+// CIR:   %0 = cir.alloca !rec_EmptyDerived2, !cir.ptr<!rec_EmptyDerived2>, ["d2"]
+// CIR:   cir.return
+
+// LLVM: define{{.*}} void @_Z18use_empty_derived2v
+// LLVM:   alloca %struct.EmptyDerived2
+// LLVM:   ret void
+
+// OGCG: define{{.*}} void @_Z18use_empty_derived2v
+// OGCG:   alloca %struct.EmptyDerived2
+// OGCG:   ret void
diff --git a/clang/test/CIR/CodeGen/complex-arithmetic.cpp b/clang/test/CIR/CodeGen/complex-arithmetic.cpp
index 5e384cd34ebfd..fbb0335106f88 100644
--- a/clang/test/CIR/CodeGen/complex-arithmetic.cpp
+++ b/clang/test/CIR/CodeGen/complex-arithmetic.cpp
@@ -11,16 +11,16 @@ void foo() {
   int _Complex c = a + b;
 }
 
-// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a"]
-// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b"]
-// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
-// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
 // CIR: %[[ADD:.*]] = cir.complex.add %[[TMP_A]], %[[TMP_B]] : !cir.complex<!s32i>
 
-// LLVM: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, i64 1, align 4
-// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
-// LLVM: %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[COMPLEX_A]], align 4
-// LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[COMPLEX_B]], align 4
+// LLVM: %[[A_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[B_ADDR]], align 4
 // LLVM: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 0
 // LLVM: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 1
 // LLVM: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 0
@@ -30,16 +30,16 @@ void foo() {
 // LLVM: %[[RESULT:.*]] = insertvalue { i32, i32 } poison, i32 %[[ADD_REAL]], 0
 // LLVM: %[[RESULT_2:.*]] = insertvalue { i32, i32 } %[[RESULT]], i32 %[[ADD_IMAG]], 1
 
-// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
-// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_ADDR:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { i32, i32 }, align 4
 // OGCG: %[[RESULT:.*]] = alloca { i32, i32 }, align 4
-// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 0
 // OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
-// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 1
 // OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
-// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
 // OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
-// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
 // OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
 // OGCG: %[[ADD_REAL:.*]] = add i32 %[[A_REAL]], %[[B_REAL]]
 // OGCG: %[[ADD_IMAG:.*]] = add i32 %[[A_IMAG]], %[[B_IMAG]]
@@ -54,16 +54,16 @@ void foo2() {
   float _Complex c = a + b;
 }
 
-// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
-// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
-// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
-// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
 // CIR: %[[ADD:.*]] = cir.complex.add %[[TMP_A]], %[[TMP_B]] : !cir.complex<!cir.float>
 
-// LLVM: %[[COMPLEX_A:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[COMPLEX_B:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[COMPLEX_A]], align 4
-// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[COMPLEX_B]], align 4
+// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
 // LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
 // LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
 // LLVM: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
@@ -73,16 +73,16 @@ void foo2() {
 // LLVM: %[[RESULT:.*]] = insertvalue { float, float } poison, float %[[ADD_REAL]], 0
 // LLVM: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT]], float %[[ADD_IMAG]], 1
 
-// OGCG: %[[COMPLEX_A:.*]] = alloca { float, float }, align 4
-// OGCG: %[[COMPLEX_B:.*]] = alloca { float, float }, align 4
+// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { float, float }, align 4
 // OGCG: %[[RESULT:.*]] = alloca { float, float }, align 4
-// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
 // OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
-// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
 // OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
-// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
 // OGCG: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
-// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
 // OGCG: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
 // OGCG: %[[ADD_REAL:.*]] = fadd float %[[A_REAL]], %[[B_REAL]]
 // OGCG: %[[ADD_IMAG:.*]] = fadd float %[[A_IMAG]], %[[B_IMAG]]
@@ -98,23 +98,23 @@ void foo3() {
   float _Complex d = (a + b) + c;
 }
 
-// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
-// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
-// CIR: %[[COMPLEX_C:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c"]
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR: %[[C_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c"]
 // CIR: %[[RESULT:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["d", init]
-// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
-// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
 // CIR: %[[ADD_A_B:.*]] = cir.complex.add %[[TMP_A]], %[[TMP_B]] : !cir.complex<!cir.float>
-// CIR: %[[TMP_C:.*]] = cir.load{{.*}} %[[COMPLEX_C]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_C:.*]] = cir.load{{.*}} %[[C_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
 // CIR: %[[ADD_A_B_C:.*]] = cir.complex.add %[[ADD_A_B]], %[[TMP_C]] : !cir.complex<!cir.float>
 // CIR: cir.store{{.*}} %[[ADD_A_B_C]], %[[RESULT]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
 
-// LLVM: %[[COMPLEX_A:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[COMPLEX_B:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[COMPLEX_C:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[C_ADDR:.*]] = alloca { float, float }, i64 1, align 4
 // LLVM: %[[RESULT:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[COMPLEX_A]], align 4
-// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[COMPLEX_B]], align 4
+// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
 // LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
 // LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
 // LLVM: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
@@ -123,7 +123,7 @@ void foo3() {
 // LLVM: %[[ADD_IMAG_A_B:.*]] = fadd float %[[A_IMAG]], %[[B_IMAG]]
 // LLVM: %[[A_B:.*]] = insertvalue { float, float } poison, float %[[ADD_REAL_A_B]], 0
 // LLVM: %[[TMP_A_B:.*]] = insertvalue { float, float } %[[A_B]], float %[[ADD_IMAG_A_B]], 1
-// LLVM: %[[TMP_C:.*]] = load { float, float }, ptr %[[COMPLEX_C]], align 4
+// LLVM: %[[TMP_C:.*]] = load { float, float }, ptr %[[C_ADDR]], align 4
 // LLVM: %[[A_B_REAL:.*]] = extractvalue { float, float } %[[TMP_A_B]], 0
 // LLVM: %[[A_B_IMAG:.*]] = extractvalue { float, float } %[[TMP_A_B]], 1
 // LLVM: %[[C_REAL:.*]] = extractvalue { float, float } %[[TMP_C]], 0
@@ -134,23 +134,23 @@ void foo3() {
 // LLVM: %[[TMP_A_B_C:.*]] = insertvalue { float, float } %[[A_B_C]], float %[[ADD_IMAG_A_B_C]], 1
 // LLVM: store { float, float } %[[TMP_A_B_C]], ptr %[[RESULT]], align 4
 
-// OGCG: %[[COMPLEX_A:.*]] = alloca { float, float }, align 4
-// OGCG: %[[COMPLEX_B:.*]] = alloca { float, float }, align 4
-// OGCG: %[[COMPLEX_C:.*]] = alloca { float, float }, align 4
+// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[C_ADDR:.*]] = alloca { float, float }, align 4
 // OGCG: %[[RESULT:.*]] = alloca { float, float }, align 4
-// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
 // OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
-// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
 // OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
-// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
 // OGCG: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
-// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
 // OGCG: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
 // OGCG: %[[ADD_REAL_A_B:.*]] = fadd float %[[A_REAL]], %[[B_REAL]]
 // OGCG: %[[ADD_IMAG_A_B:.*]] = fadd float %[[A_IMAG]], %[[B_IMAG]]
-// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_C]], i32 0, i32 0
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 0
 // OGCG: %[[C_REAL:.*]] = load float, ptr %[[C_REAL_PTR]], align 4
-// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_C]], i32 0, i32 1
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 1
 // OGCG: %[[C_IMAG:.*]] = load float, ptr %[[C_IMAG_PTR]], align 4
 // OGCG: %[[ADD_REAL_A_B_C:.*]] = fadd float %[[ADD_REAL_A_B]], %[[C_REAL]]
 // OGCG: %[[ADD_IMAG_A_B_C:.*]] = fadd float %[[ADD_IMAG_A_B]], %[[C_IMAG]]
@@ -165,17 +165,17 @@ void foo4() {
   int _Complex c = a - b;
 }
 
-// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a"]
-// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b"]
-// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
-// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
 // CIR: %[[SUB:.*]] = cir.complex.sub %[[TMP_A]], %[[TMP_B]] : !cir.complex<!s32i>
 
-// LLVM: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, i64 1, align 4
-// LLVM: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, i64 1, align 4
-// LLVM: %[[COMPLEX_C:.*]] = alloca { i32, i32 }, i64 1, align 4
-// LLVM: %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[COMPLEX_A]], align 4
-// LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[COMPLEX_B]], align 4
+// LLVM: %[[A_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[C_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[B_ADDR]], align 4
 // LLVM: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 0
 // LLVM: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 1
 // LLVM: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 0
@@ -184,18 +184,18 @@ void foo4() {
 // LLVM: %[[SUB_IMAG:.*]] = sub i32 %[[A_IMAG]], %[[B_IMAG]]
 // LLVM: %[[RESULT:.*]] = insertvalue { i32, i32 } poison, i32 %[[SUB_REAL]], 0
 // LLVM: %[[RESULT_2:.*]] = insertvalue { i32, i32 } %[[RESULT]], i32 %[[SUB_IMAG]], 1
-// LLVM: store { i32, i32 } %[[RESULT_2]], ptr %[[COMPLEX_C]], align 4
+// LLVM: store { i32, i32 } %[[RESULT_2]], ptr %[[C_ADDR]], align 4
 
-// OGCG: %[[COMPLEX_A:.*]] = alloca { i32, i32 }, align 4
-// OGCG: %[[COMPLEX_B:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_ADDR:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { i32, i32 }, align 4
 // OGCG: %[[RESULT:.*]] = alloca { i32, i32 }, align 4
-// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 0
 // OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
-// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 1
 // OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
-// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
 // OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
-// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
 // OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
 // OGCG: %[[SUB_REAL:.*]] = sub i32 %[[A_REAL]], %[[B_REAL]]
 // OGCG: %[[SUB_IMAG:.*]] = sub i32 %[[A_IMAG]], %[[B_IMAG]]
@@ -210,16 +210,16 @@ void foo5() {
   float _Complex c = a - b;
 }
 
-// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
-// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
-// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
-// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
 // CIR: %[[SUB:.*]] = cir.complex.sub %[[TMP_A]], %[[TMP_B]] : !cir.complex<!cir.float>
 
-// LLVM: %[[COMPLEX_A:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[COMPLEX_B:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[COMPLEX_A]], align 4
-// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[COMPLEX_B]], align 4
+// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
 // LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
 // LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
 // LLVM: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
@@ -229,16 +229,16 @@ void foo5() {
 // LLVM: %[[RESULT:.*]] = insertvalue { float, float } poison, float %[[SUB_REAL]], 0
 // LLVM: %[[RESULT_2:.*]] = insertvalue { float, float } %[[RESULT]], float %[[SUB_IMAG]], 1
 
-// OGCG: %[[COMPLEX_A:.*]] = alloca { float, float }, align 4
-// OGCG: %[[COMPLEX_B:.*]] = alloca { float, float }, align 4
+// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { float, float }, align 4
 // OGCG: %[[RESULT:.*]] = alloca { float, float }, align 4
-// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
 // OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
-// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
 // OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
-// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
 // OGCG: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
-// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
 // OGCG: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
 // OGCG: %[[SUB_REAL:.*]] = fsub float %[[A_REAL]], %[[B_REAL]]
 // OGCG: %[[SUB_IMAG:.*]] = fsub float %[[A_IMAG]], %[[B_IMAG]]
@@ -254,23 +254,23 @@ void foo6() {
   float _Complex d = (a - b) - c;
 }
 
-// CIR: %[[COMPLEX_A:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
-// CIR: %[[COMPLEX_B:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
-// CIR: %[[COMPLEX_C:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c"]
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR: %[[C_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c"]
 // CIR: %[[RESULT:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["d", init]
-// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[COMPLEX_A]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
-// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[COMPLEX_B]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
 // CIR: %[[SUB_A_B:.*]] = cir.complex.sub %[[TMP_A]], %[[TMP_B]] : !cir.complex<!cir.float>
-// CIR: %[[TMP_C:.*]] = cir.load{{.*}} %[[COMPLEX_C]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_C:.*]] = cir.load{{.*}} %[[C_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
 // CIR: %[[SUB_A_B_C:.*]] = cir.complex.sub %[[SUB_A_B]], %[[TMP_C]] : !cir.complex<!cir.float>
 // CIR: cir.store{{.*}} %[[SUB_A_B_C]], %[[RESULT]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
 
-// LLVM: %[[COMPLEX_A:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[COMPLEX_B:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[COMPLEX_C:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[C_ADDR:.*]] = alloca { float, float }, i64 1, align 4
 // LLVM: %[[RESULT:.*]] = alloca { float, float }, i64 1, align 4
-// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[COMPLEX_A]], align 4
-// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[COMPLEX_B]], align 4
+// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
 // LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
 // LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
 // LLVM: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
@@ -279,7 +279,7 @@ void foo6() {
 // LLVM: %[[SUB_IMAG_A_B:.*]] = fsub float %[[A_IMAG]], %[[B_IMAG]]
 // LLVM: %[[A_B:.*]] = insertvalue { float, float } poison, float %[[SUB_REAL_A_B]], 0
 // LLVM: %[[TMP_A_B:.*]] = insertvalue { float, float } %[[A_B]], float %[[SUB_IMAG_A_B]], 1
-// LLVM: %[[TMP_C:.*]] = load { float, float }, ptr %[[COMPLEX_C]], align 4
+// LLVM: %[[TMP_C:.*]] = load { float, float }, ptr %[[C_ADDR]], align 4
 // LLVM: %[[A_B_REAL:.*]] = extractvalue { float, float } %[[TMP_A_B]], 0
 // LLVM: %[[A_B_IMAG:.*]] = extractvalue { float, float } %[[TMP_A_B]], 1
 // LLVM: %[[C_REAL:.*]] = extractvalue { float, float } %[[TMP_C]], 0
@@ -290,23 +290,23 @@ void foo6() {
 // LLVM: %[[TMP_A_B_C:.*]] = insertvalue { float, float } %[[A_B_C]], float %[[SUB_IMAG_A_B_C]], 1
 // LLVM: store { float, float } %[[TMP_A_B_C]], ptr %[[RESULT]], align 4
 
-// OGCG: %[[COMPLEX_A:.*]] = alloca { float, float }, align 4
-// OGCG: %[[COMPLEX_B:.*]] = alloca { float, float }, align 4
-// OGCG: %[[COMPLEX_C:.*]] = alloca { float, float }, align 4
+// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[C_ADDR:.*]] = alloca { float, float }, align 4
 // OGCG: %[[RESULT:.*]] = alloca { float, float }, align 4
-// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_A]], i32 0, i32 0
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
 // OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
-// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_A]], i32 0, i32 1
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
 // OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
-// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_B]], i32 0, i32 0
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
 // OGCG: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
-// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_B]], i32 0, i32 1
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
 // OGCG: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
 // OGCG: %[[SUB_REAL_A_B:.*]] = fsub float %[[A_REAL]], %[[B_REAL]]
 // OGCG: %[[SUB_IMAG_A_B:.*]] = fsub float %[[A_IMAG]], %[[B_IMAG]]
-// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_C]], i32 0, i32 0
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 0
 // OGCG: %[[C_REAL:.*]] = load float, ptr %[[C_REAL_PTR]], align 4
-// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_C]], i32 0, i32 1
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 1
 // OGCG: %[[C_IMAG:.*]] = load float, ptr %[[C_IMAG_PTR]], align 4
 // OGCG: %[[SUB_REAL_A_B_C:.*]] = fsub float %[[SUB_REAL_A_B]], %[[C_REAL]]
 // OGCG: %[[SUB_IMAG_A_B_C:.*]] = fsub float %[[SUB_IMAG_A_B]], %[[C_IMAG]]
@@ -314,3 +314,4 @@ void foo6() {
 // OGCG: %[[RESULT_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[RESULT]], i32 0, i32 1
 // OGCG: store float %[[SUB_REAL_A_B_C]], ptr %[[RESULT_REAL_PTR]], align 4
 // OGCG: store float %[[SUB_IMAG_A_B_C]], ptr %[[RESULT_IMAG_PTR]], align 4
+
diff --git a/clang/test/CIR/CodeGen/complex-compound-assignment.cpp b/clang/test/CIR/CodeGen/complex-compound-assignment.cpp
new file mode 100644
index 0000000000000..9a6659bd5d93e
--- /dev/null
+++ b/clang/test/CIR/CodeGen/complex-compound-assignment.cpp
@@ -0,0 +1,574 @@
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR,CXX_CIR
+// RUN: %clang_cc1 -x c -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM,CXX_LLVM
+// RUN: %clang_cc1 -x c -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG,CXX_OGCG
+// RUN: %clang_cc1 -x c -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+void foo() {
+  float _Complex a;
+  float _Complex b;
+  b += a;
+}
+
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[RESULT:.*]] = cir.complex.add %[[TMP_B]], %[[TMP_A]] : !cir.complex<!cir.float>
+// CIR: cir.store{{.*}} %[[RESULT]], %[[B_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
+// LLVM: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { float, float } %[[TMP_B]], 1
+// LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM: %[[ADD_REAL_A_B:.*]] = fadd float %[[B_REAL]], %[[A_REAL]]
+// LLVM: %[[ADD_IMAG_A_B:.*]] = fadd float %[[B_IMAG]], %[[A_IMAG]]
+// LLVM: %[[ADD_A_B:.*]] = insertvalue { float, float } poison, float %[[ADD_REAL_A_B]], 0
+// LLVM: %[[RESULT:.*]] = insertvalue { float, float } %[[ADD_A_B]], float %[[ADD_IMAG_A_B]], 1
+// LLVM: store { float, float } %[[RESULT]], ptr %[[B_ADDR]], align 4
+
+// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[ADD_REAL:.*]] = fadd float %[[B_REAL]], %[[A_REAL]]
+// OGCG: %[[ADD_IMAG:.*]] = fadd float %[[B_IMAG]], %[[A_IMAG]]
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: store float %[[ADD_REAL]], ptr %[[B_REAL_PTR]], align 4
+// OGCG: store float %[[ADD_IMAG]], ptr %[[B_IMAG_PTR]], align 4
+
+void foo1() {
+  float _Complex a;
+  float _Complex b;
+  b -= a;
+}
+
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[RESULT:.*]] = cir.complex.sub %[[TMP_B]], %[[TMP_A]] : !cir.complex<!cir.float>
+// CIR: cir.store{{.*}} %[[RESULT]], %[[B_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
+// LLVM: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { float, float } %[[TMP_B]], 1
+// LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM: %[[SUB_REAL_A_B:.*]] = fsub float %[[B_REAL]], %[[A_REAL]]
+// LLVM: %[[SUB_IMAG_A_B:.*]] = fsub float %[[B_IMAG]], %[[A_IMAG]]
+// LLVM: %[[SUB_A_B:.*]] = insertvalue { float, float } poison, float %[[SUB_REAL_A_B]], 0
+// LLVM: %[[RESULT:.*]] = insertvalue { float, float } %[[SUB_A_B]], float %[[SUB_IMAG_A_B]], 1
+// LLVM: store { float, float } %[[RESULT]], ptr %[[B_ADDR]], align 4
+
+// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[SUB_REAL:.*]] = fsub float %[[B_REAL]], %[[A_REAL]]
+// OGCG: %[[SUB_IMAG:.*]] = fsub float %[[B_IMAG]], %[[A_IMAG]]
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: store float %[[SUB_REAL]], ptr %[[B_REAL_PTR]], align 4
+// OGCG: store float %[[SUB_IMAG]], ptr %[[B_IMAG_PTR]], align 4
+
+void foo2() {
+  int _Complex a;
+  int _Complex b;
+  b += a;
+}
+
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[RESULT:.*]] = cir.complex.add %[[TMP_B]], %[[TMP_A]] : !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[RESULT]], %[[B_ADDR]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[A_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[B_ADDR]], align 4
+// LLVM: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 1
+// LLVM: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 1
+// LLVM: %[[ADD_REAL_A_B:.*]] = add i32 %[[B_REAL]], %[[A_REAL]]
+// LLVM: %[[ADD_IMAG_A_B:.*]] = add i32 %[[B_IMAG]], %[[A_IMAG]]
+// LLVM: %[[ADD_A_B:.*]] = insertvalue { i32, i32 } poison, i32 %[[ADD_REAL_A_B]], 0
+// LLVM: %[[RESULT:.*]] = insertvalue { i32, i32 } %[[ADD_A_B]], i32 %[[ADD_IMAG_A_B]], 1
+// LLVM: store { i32, i32 } %[[RESULT]], ptr %[[B_ADDR]], align 4
+
+// OGCG: %[[A_ADDR:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[ADD_REAL:.*]] = add i32 %[[B_REAL]], %[[A_REAL]]
+// OGCG: %[[ADD_IMAG:.*]] = add i32 %[[B_IMAG]], %[[A_IMAG]]
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: store i32 %[[ADD_REAL]], ptr %[[B_REAL_PTR]], align 4
+// OGCG: store i32 %[[ADD_IMAG]], ptr %[[B_IMAG_PTR]], align 4
+
+void foo3() {
+  _Float16 _Complex a;
+  _Float16 _Complex b;
+  b += a;
+}
+
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.f16>, !cir.ptr<!cir.complex<!cir.f16>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.f16>, !cir.ptr<!cir.complex<!cir.f16>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.f16>>, !cir.complex<!cir.f16>
+// CIR: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.f16> -> !cir.f16
+// CIR: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.f16> -> !cir.f16
+// CIR: %[[A_REAL_F32:.*]] = cir.cast(floating, %[[A_REAL]] : !cir.f16), !cir.float
+// CIR: %[[A_IMAG_F32:.*]] = cir.cast(floating, %[[A_IMAG]] : !cir.f16), !cir.float
+// CIR: %[[A_COMPLEX_F32:.*]] = cir.complex.create %[[A_REAL_F32]], %[[A_IMAG_F32]] : !cir.float -> !cir.complex<!cir.float>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.f16>>, !cir.complex<!cir.f16>
+// CIR: %[[B_REAL:.*]] = cir.complex.real %[[TMP_B]] : !cir.complex<!cir.f16> -> !cir.f16
+// CIR: %[[B_IMAG:.*]] = cir.complex.imag %[[TMP_B]] : !cir.complex<!cir.f16> -> !cir.f16
+// CIR: %[[B_REAL_F32:.*]] = cir.cast(floating, %[[B_REAL]] : !cir.f16), !cir.float
+// CIR: %[[B_IMAG_F32:.*]] = cir.cast(floating, %[[B_IMAG]] : !cir.f16), !cir.float
+// CIR: %[[B_COMPLEX_F32:.*]] = cir.complex.create %[[B_REAL_F32]], %[[B_IMAG_F32]] : !cir.float -> !cir.complex<!cir.float>
+// CIR: %[[ADD_A_B:.*]] = cir.complex.add %[[B_COMPLEX_F32]], %[[A_COMPLEX_F32]] : !cir.complex<!cir.float>
+// CIR: %[[ADD_REAL:.*]] = cir.complex.real %[[ADD_A_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[ADD_IMAG:.*]] = cir.complex.imag %[[ADD_A_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[ADD_REAL_F16:.*]] = cir.cast(floating, %[[ADD_REAL]] : !cir.float), !cir.f16
+// CIR: %[[ADD_IMAG_F16:.*]] = cir.cast(floating, %[[ADD_IMAG]] : !cir.float), !cir.f16
+// CIR: %[[RESULT:.*]] = cir.complex.create %[[ADD_REAL_F16]], %[[ADD_IMAG_F16]] : !cir.f16 -> !cir.complex<!cir.f16>
+// CIR: cir.store{{.*}} %[[RESULT]], %[[B_ADDR]] : !cir.complex<!cir.f16>, !cir.ptr<!cir.complex<!cir.f16>>
+
+// LLVM: %[[A_ADDR:.*]] = alloca { half, half }, i64 1, align 2
+// LLVM: %[[B_ADDR:.*]] = alloca { half, half }, i64 1, align 2
+// LLVM: %[[TMP_A:.*]] = load { half, half }, ptr %[[A_ADDR]], align 2
+// LLVM: %[[A_REAL:.*]] = extractvalue { half, half } %[[TMP_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { half, half } %[[TMP_A]], 1
+// LLVM: %[[A_REAL_F32:.*]] = fpext half %[[A_REAL]] to float
+// LLVM: %[[A_IMAG_F32:.*]] = fpext half %[[A_IMAG]] to float
+// LLVM: %[[TMP_A_COMPLEX_F32:.*]] = insertvalue { float, float } {{.*}}, float %[[A_REAL_F32]], 0
+// LLVM: %[[A_COMPLEX_F32:.*]] = insertvalue { float, float } %8, float %[[A_IMAG_F32]], 1
+// LLVM: %[[TMP_B:.*]] = load { half, half }, ptr %[[B_ADDR]], align 2
+// LLVM: %[[B_REAL:.*]] = extractvalue { half, half } %[[TMP_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { half, half } %[[TMP_B]], 1
+// LLVM: %[[B_REAL_F32:.*]] = fpext half %[[B_REAL]] to float
+// LLVM: %[[B_IMAG_F32:.*]] = fpext half %[[B_IMAG]] to float
+// LLVM: %[[TMP_B_COMPLEX_F32:.*]] = insertvalue { float, float } {{.*}}, float %[[B_REAL_F32]], 0
+// LLVM: %[[B_COMPLEX_F32:.*]] = insertvalue { float, float } %[[TMP_B_COMPLEX_F32]], float %[[B_IMAG_F32]], 1
+// LLVM: %[[B_REAL:.*]] = extractvalue { float, float } %[[B_COMPLEX_F32]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { float, float } %[[B_COMPLEX_F32]], 1
+// LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[A_COMPLEX_F32]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[A_COMPLEX_F32]], 1
+// LLVM: %[[ADD_REAL:.*]] = fadd float %[[B_REAL]], %[[A_REAL]]
+// LLVM: %[[ADD_IMAG:.*]] = fadd float %[[B_IMAG]], %[[A_IMAG]]
+// LLVM: %[[TMP_RESULT:.*]] = insertvalue { float, float } poison, float %[[ADD_REAL]], 0
+// LLVM: %[[RESULT:.*]] = insertvalue { float, float } %[[TMP_RESULT]], float %[[ADD_IMAG]], 1
+// LLVM: %[[RESULT_REAL:.*]] = extractvalue { float, float } %[[RESULT]], 0
+// LLVM: %[[RESULT_IMAG:.*]] = extractvalue { float, float } %[[RESULT]], 1
+// LLVM: %[[RESULT_REAL_F16:.*]] = fptrunc float %[[RESULT_REAL]] to half
+// LLVM: %[[RESULT_IMAG_F26:.*]] = fptrunc float %[[RESULT_IMAG]] to half
+// LLVM: %[[TMP_RESULT_F16:.*]] = insertvalue { half, half } undef, half %[[RESULT_REAL_F16]], 0
+// LLVM: %[[RESULT_F16:.*]] = insertvalue { half, half } %29, half %[[RESULT_IMAG_F26]], 1
+// LLVM: store { half, half } %[[RESULT_F16]], ptr %[[B_ADDR]], align 2
+
+// OGCG: %[[A_ADDR:.*]] = alloca { half, half }, align 2
+// OGCG: %[[B_ADDR:.*]] = alloca { half, half }, align 2
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { half, half }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load half, ptr %[[A_REAL_PTR]], align 2
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { half, half }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load half, ptr %[[A_IMAG_PTR]], align 2
+// OGCG: %[[A_REAL_F32:.*]] = fpext half %[[A_REAL]] to float
+// OGCG: %[[A_IMAG_F32:.*]] = fpext half %[[A_IMAG]] to float
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { half, half }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load half, ptr %[[B_REAL_PTR]], align 2
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { half, half }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load half, ptr %[[B_IMAG_PTR]], align 2
+// OGCG: %[[B_REAL_F32:.*]] = fpext half %[[B_REAL]] to float
+// OGCG: %[[B_IMAG_F32:.*]] = fpext half %[[B_IMAG]] to float
+// OGCG: %[[ADD_REAL:.*]] = fadd float %[[B_REAL_F32]], %[[A_REAL_F32]]
+// OGCG: %[[ADD_IMAG:.*]] = fadd float %[[B_IMAG_F32]], %[[A_IMAG_F32]]
+// OGCG: %[[ADD_REAL_F16:.*]] = fptrunc float %[[ADD_REAL]] to half
+// OGCG: %[[ADD_IMAG_F16:.*]] = fptrunc float %[[ADD_IMAG]] to half
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { half, half }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { half, half }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: store half %[[ADD_REAL_F16]], ptr %[[B_REAL_PTR]], align 2
+// OGCG: store half %[[ADD_IMAG_F16]], ptr %[[B_IMAG_PTR]], align 2
+
+#ifdef __cplusplus
+void foo4() {
+  volatile _Complex int a;
+  volatile _Complex int b;
+  int _Complex c = b += a;
+}
+#endif
+
+// CXX_CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a"]
+// CXX_CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b"]
+// CXX_CIR: %[[C_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CXX_CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CXX_CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CXX_CIR: %[[RESULT:.*]] = cir.complex.add %[[TMP_B]], %[[TMP_A]] : !cir.complex<!s32i>
+// CXX_CIR: cir.store{{.*}} %[[RESULT]], %[[B_ADDR]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+// CXX_CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CXX_CIR: cir.store{{.*}} %[[TMP_B]], %[[C_ADDR]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>
+
+// CXX_LLVM: %[[A_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// CXX_LLVM: %[[B_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// CXX_LLVM: %[[C_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// CXX_LLVM: %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[A_ADDR]], align 4
+// CXX_LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[B_ADDR]], align 4
+// CXX_LLVM: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 0
+// CXX_LLVM: %[[B_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 1
+// CXX_LLVM: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 0
+// CXX_LLVM: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 1
+// CXX_LLVM: %[[ADD_REAL:.*]] = add i32 %[[B_REAL]], %[[A_REAL]]
+// CXX_LLVM: %[[ADD_IMAG:.*]] = add i32 %[[B_IMAG]], %[[A_IMAG]]
+// CXX_LLVM: %[[TMP_RESULT:.*]] = insertvalue { i32, i32 } poison, i32 %[[ADD_REAL]], 0
+// CXX_LLVM: %[[RESULT:.*]] = insertvalue { i32, i32 } %[[TMP_RESULT]], i32 %[[ADD_IMAG]], 1
+// CXX_LLVM: store { i32, i32 } %[[RESULT]], ptr %[[B_ADDR]], align 4
+// CXX_LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[B_ADDR]], align 4
+// CXX_LLVM: store { i32, i32 } %[[TMP_B]], ptr %[[C_ADDR]], align 4
+
+// CXX_OGCG: %[[A_ADDR:.*]] = alloca { i32, i32 }, align 4
+// CXX_OGCG: %[[B_ADDR:.*]] = alloca { i32, i32 }, align 4
+// CXX_OGCG: %[[C_ADDR:.*]] = alloca { i32, i32 }, align 4
+// CXX_OGCG: %a.realp = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 0
+// CXX_OGCG: %a.real = load volatile i32, ptr %a.realp, align 4
+// CXX_OGCG: %a.imagp = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 1
+// CXX_OGCG: %a.imag = load volatile i32, ptr %a.imagp, align 4
+// CXX_OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
+// CXX_OGCG: %[[B_REAL:.*]] = load volatile i32, ptr %[[B_REAL_PTR]], align 4
+// CXX_OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
+// CXX_OGCG: %[[B_IMAG:.*]] = load volatile i32, ptr %[[B_IMAG_PTR]], align 4
+// CXX_OGCG: %[[ADD_REAL:.*]] = add i32 %[[B_REAL]], %[[A_REAL]]
+// CXX_OGCG: %[[ADD_IMAG:.*]] = add i32 %[[B_IMAG]], %[[A_IMAG]]
+// CXX_OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
+// CXX_OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
+// CXX_OGCG: store volatile i32 %[[ADD_REAL]], ptr %[[B_REAL_PTR]], align 4
+// CXX_OGCG: store volatile i32 %[[ADD_IMAG]], ptr %[[B_IMAG_PTR]], align 4
+// CXX_OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
+// CXX_OGCG: %[[B_REAL:.*]] = load volatile i32, ptr %[[B_REAL_PTR]], align 4
+// CXX_OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
+// CXX_OGCG: %[[B_IMAG:.*]] = load volatile i32, ptr %[[B_IMAG_PTR]], align 4
+// CXX_OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[C_ADDR]], i32 0, i32 0
+// CXX_OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[C_ADDR]], i32 0, i32 1
+// CXX_OGCG: store i32 %[[B_REAL]], ptr %[[C_REAL_PTR]], align 4
+// CXX_OGCG: store i32 %[[B_IMAG]], ptr %[[C_IMAG_PTR]], align 4
+
+void foo5() {
+  float _Complex a;
+  float b;
+  a += b;
+}
+
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["b"]
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.float>, !cir.float
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[RESULT_REAL:.*]] = cir.binop(add, %[[A_REAL]], %[[TMP_B]]) : !cir.float
+// CIR: %[[RESULT:.*]] = cir.complex.create %[[RESULT_REAL]], %[[A_IMAG]] : !cir.float -> !cir.complex<!cir.float>
+// CIR: cir.store{{.*}} %[[RESULT]], %[[A_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca float, i64 1, align 4
+// LLVM: %[[TMP_B:.*]] = load float, ptr %[[B_ADDR]], align 4
+// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM: %[[RESULT_REAL:.*]] = fadd float %[[A_REAL]], %[[TMP_B]]
+// LLVM: %[[TMP_RESULT:.*]] = insertvalue { float, float } {{.*}}, float %[[RESULT_REAL]], 0
+// LLVM: %[[RESULT:.*]] = insertvalue { float, float } %[[TMP_RESULT]], float %[[A_IMAG]], 1
+// LLVM: store { float, float } %[[RESULT]], ptr %[[A_ADDR]], align 4
+
+// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca float, align 4
+// OGCG: %[[TMP_B:.*]] = load float, ptr %[[B_ADDR]], align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[ADD_REAL:.*]] = fadd float %[[A_REAL]], %[[TMP_B]]
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: store float %[[ADD_REAL]], ptr %[[A_REAL_PTR]], align 4
+// OGCG: store float %[[A_IMAG]], ptr %[[A_IMAG_PTR]], align 4
+
+void foo6() {
+  int _Complex a;
+  int _Complex b;
+  b *= a;
+}
+
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[B_REAL:.*]] = cir.complex.real %[[TMP_B]] : !cir.complex<!s32i> -> !s32i
+// CIR: %[[B_IMAG:.*]] = cir.complex.imag %[[TMP_B]] : !cir.complex<!s32i> -> !s32i
+// CIR: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!s32i> -> !s32i
+// CIR: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!s32i> -> !s32i
+// CIR: %[[MUL_BR_AR:.*]] = cir.binop(mul, %[[B_REAL]], %[[A_REAL]]) : !s32i
+// CIR: %[[MUL_BI_AI:.*]] = cir.binop(mul, %[[B_IMAG]], %[[A_IMAG]]) : !s32i
+// CIR: %[[MUL_BR_AI:.*]] = cir.binop(mul, %[[B_REAL]], %[[A_IMAG]]) : !s32i
+// CIR: %[[MUL_BI_AR:.*]] = cir.binop(mul, %[[B_IMAG]], %[[A_REAL]]) : !s32i
+// CIR: %[[RESULT_REAL:.*]] = cir.binop(sub, %[[MUL_BR_AR]], %[[MUL_BI_AI]]) : !s32i
+// CIR: %[[RESULT_IMAG:.*]] = cir.binop(add, %[[MUL_BR_AI]], %[[MUL_BI_AR]]) : !s32i
+// CIR: %[[RESULT:.*]] = cir.complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : !s32i -> !cir.complex<!s32i>
+// CIR: cir.store{{.*}} %[[RESULT]], %[[B_ADDR]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM: %[[A_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[B_ADDR]], align 4
+// LLVM: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 1
+// LLVM: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 1
+// LLVM: %[[MUL_BR_AR:.*]] = mul i32 %[[B_REAL]], %[[A_REAL]]
+// LLVM: %[[MUL_BI_AI:.*]] = mul i32 %[[B_IMAG]], %[[A_IMAG]]
+// LLVM: %[[MUL_BR_AI:.*]] = mul i32 %[[B_REAL]], %[[A_IMAG]]
+// LLVM: %[[MUL_BI_AR:.*]] = mul i32 %[[B_IMAG]], %[[A_REAL]]
+// LLVM: %[[RESULT_REAL:.*]] = sub i32 %[[MUL_BR_AR]], %[[MUL_BI_AI]]
+// LLVM: %[[RESULT_IMAG:.*]] = add i32 %[[MUL_BR_AI]], %[[MUL_BI_AR]]
+// LLVM: %[[MUL_A_B:.*]] = insertvalue { i32, i32 } {{.*}}, i32 %[[RESULT_REAL]], 0
+// LLVM: %[[RESULT:.*]] = insertvalue { i32, i32 } %[[MUL_A_B]], i32 %[[RESULT_IMAG]], 1
+// LLVM: store { i32, i32 } %[[RESULT]], ptr %[[B_ADDR]], align 4
+
+// OGCG: %[[A_ADDR:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { i32, i32 }, align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[MUL_BR_AR:.*]] = mul i32 %[[B_REAL]], %[[A_REAL]]
+// OGCG: %[[MUL_BI_AI:.*]] = mul i32 %[[B_IMAG]], %[[A_IMAG]]
+// OGCG: %[[RESULT_REAL:.*]] = sub i32 %[[MUL_BR_AR]], %[[MUL_BI_AI]]
+// OGCG: %[[MUL_BI_AR:.*]] = mul i32 %[[B_IMAG]], %[[A_REAL]]
+// OGCG: %[[MUL_BR_AI:.*]] = mul i32 %[[B_REAL]], %[[A_IMAG]]
+// OGCG: %[[RESULT_IMAG:.*]] = add i32 %[[MUL_BI_AR]], %[[MUL_BR_AI]]
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: store i32 %[[RESULT_REAL]], ptr %[[B_REAL_PTR]], align 4
+// OGCG: store i32 %[[RESULT_IMAG]], ptr %[[B_IMAG_PTR]], align 4
+
+void foo7() {
+  float _Complex a;
+  float _Complex b;
+  b *= a;
+}
+
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[B_REAL:.*]] = cir.complex.real %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[B_IMAG:.*]] = cir.complex.imag %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[MUL_BR_AR:.*]] = cir.binop(mul, %[[B_REAL]], %[[A_REAL]]) : !cir.float
+// CIR: %[[MUL_BI_AI:.*]] = cir.binop(mul, %[[B_IMAG]], %[[A_IMAG]]) : !cir.float
+// CIR: %[[MUL_BR_AI:.*]] = cir.binop(mul, %[[B_REAL]], %[[A_IMAG]]) : !cir.float
+// CIR: %[[MUL_BI_AR:.*]] = cir.binop(mul, %[[B_IMAG]], %[[A_REAL]]) : !cir.float
+// CIR: %[[C_REAL:.*]] = cir.binop(sub, %[[MUL_BR_AR]], %[[MUL_BI_AI]]) : !cir.float
+// CIR: %[[C_IMAG:.*]] = cir.binop(add, %[[MUL_BR_AI]], %[[MUL_BI_AR]]) : !cir.float
+// CIR: %[[COMPLEX:.*]] = cir.complex.create %[[C_REAL]], %[[C_IMAG]] : !cir.float -> !cir.complex<!cir.float>
+// CIR: %[[IS_C_REAL_NAN:.*]] = cir.cmp(ne, %[[C_REAL]], %[[C_REAL]]) : !cir.float, !cir.bool
+// CIR: %[[IS_C_IMAG_NAN:.*]] = cir.cmp(ne, %[[C_IMAG]], %[[C_IMAG]]) : !cir.float, !cir.bool
+// CIR: %[[CONST_FALSE:.*]] = cir.const #false
+// CIR: %[[SELECT_CONDITION:.*]] = cir.select if %[[IS_C_REAL_NAN]] then %[[IS_C_IMAG_NAN]] else %[[CONST_FALSE]] : (!cir.bool, !cir.bool, !cir.bool) -> !cir.bool
+// CIR: %[[RESULT:.*]] = cir.ternary(%[[SELECT_CONDITION]], true {
+// CIR:   %[[LIBC_COMPLEX:.*]] = cir.call @__mulsc3(%[[B_REAL]], %[[B_IMAG]], %[[A_REAL]], %[[A_IMAG]]) : (!cir.float, !cir.float, !cir.float, !cir.float) -> !cir.complex<!cir.float>
+// CIR:   cir.yield %[[LIBC_COMPLEX]] : !cir.complex<!cir.float>
+// CIR: }, false {
+// CIR:   cir.yield %[[COMPLEX]] : !cir.complex<!cir.float>
+// CIR: }) : (!cir.bool) -> !cir.complex<!cir.float>
+// CIR: cir.store{{.*}} %[[RESULT]], %[[B_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
+// LLVM: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
+// LLVM: %[[B_IMAG:.*]] = extractvalue { float, float } %[[TMP_B]], 1
+// LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM: %[[MUL_BR_AR:.*]] = fmul float %[[B_REAL]], %[[A_REAL]]
+// LLVM: %[[MUL_BI_AI:.*]] = fmul float %[[B_IMAG]], %[[A_IMAG]]
+// LLVM: %[[MUL_BR_AI:.*]] = fmul float %[[B_REAL]], %[[A_IMAG]]
+// LLVM: %[[MUL_BI_AR:.*]] = fmul float %[[B_IMAG]], %[[A_REAL]]
+// LLVM: %[[C_REAL:.*]] = fsub float %[[MUL_BR_AR]], %[[MUL_BI_AI]]
+// LLVM: %[[C_IMAG:.*]] = fadd float %[[MUL_BR_AI]], %[[MUL_BI_AR]]
+// LLVM: %[[MUL_A_B:.*]] = insertvalue { float, float } {{.*}}, float %[[C_REAL]], 0
+// LLVM: %[[COMPLEX:.*]] = insertvalue { float, float } %[[MUL_A_B]], float %[[C_IMAG]], 1
+// LLVM: %[[IS_C_REAL_NAN:.*]] = fcmp une float %[[C_REAL]], %[[C_REAL]]
+// LLVM: %[[IS_C_IMAG_NAN:.*]] = fcmp une float %[[C_IMAG]], %[[C_IMAG]]
+// LLVM: %[[SELECT_CONDITION:.*]] = and i1 %[[IS_C_REAL_NAN]], %[[IS_C_IMAG_NAN]]
+// LLVM: br i1 %[[SELECT_CONDITION]], label %[[THEN_LABEL:.*]], label %[[ELSE_LABEL:.*]]
+// LLVM: [[THEN_LABEL]]:
+// LLVM:  %[[LIBC_COMPLEX:.*]] = call { float, float } @__mulsc3(float %[[B_REAL]], float %[[B_IMAG]], float %[[A_REAL]], float %[[A_IMAG]])
+// LLVM:  br label %[[PHI_BRANCH:.*]]
+// LLVM: [[ELSE_LABEL]]:
+// LLVM:  br label %[[PHI_BRANCH:]]
+// LLVM: [[PHI_BRANCH:]]:
+// LLVM:  %[[RESULT:.*]] = phi { float, float } [ %[[COMPLEX]], %[[ELSE_LABEL]] ], [ %[[LIBC_COMPLEX]], %[[THEN_LABEL]] ]
+// LLVM:  br label %[[END_LABEL:.*]]
+// LLVM: [[END_LABEL]]:
+// LLVM:  store { float, float } %[[RESULT]], ptr %[[B_ADDR]], align 4
+
+// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[COMPLEX_CALL_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
+// OGCG: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
+// OGCG: %[[MUL_BR_AR:.*]] = fmul float %[[B_REAL]], %[[A_REAL]]
+// OGCG: %[[MUL_BI_AI:.*]] = fmul float %[[B_IMAG]], %[[A_IMAG]]
+// OGCG: %[[MUL_BR_AI:.*]] = fmul float %[[B_REAL]], %[[A_IMAG]]
+// OGCG: %[[MUL_BI_AR:.*]] = fmul float %[[B_IMAG]], %[[A_REAL]]
+// OGCG: %[[C_REAL:.*]] = fsub float %[[MUL_BR_AR]], %[[MUL_BI_AI]]
+// OGCG: %[[C_IMAG:.*]] = fadd float %[[MUL_BR_AI]], %[[MUL_BI_AR]]
+// OGCG: %[[IS_C_REAL_NAN:.*]] = fcmp uno float %[[C_REAL]], %[[C_REAL]]
+// OGCG: br i1 %[[IS_C_REAL_NAN]], label %[[COMPLEX_IS_IMAG_NAN:.*]], label %[[END_LABEL:.*]], !prof !2
+// OGCG: [[COMPLEX_IS_IMAG_NAN]]:
+// OGCG:  %[[IS_C_IMAG_NAN:.*]] = fcmp uno float %[[C_IMAG]], %[[C_IMAG]]
+// OGCG:  br i1 %[[IS_C_IMAG_NAN]], label %[[COMPLEX_LIB_CALL:.*]], label %[[END_LABEL]], !prof !2
+// OGCG: [[COMPLEX_LIB_CALL]]:
+// OGCG:  %[[CALL_RESULT:.*]] = call{{.*}} <2 x float> @__mulsc3(float noundef %[[B_REAL]], float noundef %[[B_IMAG]], float noundef %[[A_REAL]], float noundef %[[A_IMAG]])
+// OGCG:  store <2 x float> %[[CALL_RESULT]], ptr %[[COMPLEX_CALL_ADDR]], align 4
+// OGCG:  %[[COMPLEX_CALL_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_CALL_ADDR]], i32 0, i32 0
+// OGCG:  %[[COMPLEX_CALL_REAL:.*]] = load float, ptr %[[COMPLEX_CALL_REAL_PTR]], align 4
+// OGCG:  %[[COMPLEX_CALL_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[COMPLEX_CALL_ADDR]], i32 0, i32 1
+// OGCG:  %[[COMPLEX_CALL_IMAG:.*]] = load float, ptr %[[COMPLEX_CALL_IMAG_PTR]], align 4
+// OGCG:  br label %[[END_LABEL]]
+// OGCG: [[END_LABEL]]:
+// OGCG:  %[[FINAL_REAL:.*]] = phi float [ %[[C_REAL]], %[[ENTRY:.*]] ], [ %[[C_REAL]], %[[COMPLEX_IS_IMAG_NAN]] ], [ %[[COMPLEX_CALL_REAL]], %[[COMPLEX_LIB_CALL]] ]
+// OGCG:  %[[FINAL_IMAG:.*]] = phi float [ %[[C_IMAG]], %[[ENTRY]] ], [ %[[C_IMAG]], %[[COMPLEX_IS_IMAG_NAN]] ], [ %[[COMPLEX_CALL_IMAG]], %[[COMPLEX_LIB_CALL]] ]
+// OGCG:  %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG:  %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG:  store float %[[FINAL_REAL]], ptr %[[C_REAL_PTR]], align 4
+// OGCG:  store float %[[FINAL_IMAG]], ptr %[[C_IMAG_PTR]], align 4
+
+void foo8() {
+  float _Complex a;
+  float b;
+  a *= b;
+}
+
+// CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR: %[[B_ADDR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["b"]
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.float>, !cir.float
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR: %[[RESULT_REAL:.*]] = cir.binop(mul, %[[A_REAL]], %[[TMP_B]]) : !cir.float
+// CIR: %[[RESULT_IMAG:.*]] = cir.binop(mul, %[[A_IMAG]], %[[TMP_B]]) : !cir.float
+// CIR: %[[RESULT:.*]] = cir.complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : !cir.float -> !cir.complex<!cir.float>
+// CIR: cir.store{{.*}} %[[RESULT]], %[[A_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM: %[[B_ADDR:.*]] = alloca float, i64 1, align 4
+// LLVM: %[[TMP_B:.*]] = load float, ptr %[[B_ADDR]], align 4
+// LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM: %[[RESULT_REAL:.*]] = fmul float %[[A_REAL]], %[[TMP_B]]
+// LLVM: %[[RESULT_IMAG:.*]] = fmul float %[[A_IMAG]], %[[TMP_B]]
+// LLVM: %[[TMP_RESULT:.*]] = insertvalue { float, float } {{.*}}, float %[[RESULT_REAL]], 0
+// LLVM: %[[RESULT:.*]] = insertvalue { float, float } %[[TMP_RESULT]], float %[[RESULT_IMAG]], 1
+// LLVM: store { float, float } %[[RESULT]], ptr %[[A_ADDR]], align 4
+
+// OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG: %[[B_ADDR:.*]] = alloca float, align 4
+// OGCG: %[[TMP_B:.*]] = load float, ptr %[[B_ADDR]], align 4
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// OGCG: %[[RESULT_REAL:.*]] = fmul float %[[A_REAL]], %[[TMP_B]]
+// OGCG: %[[RESULT_IMAG:.*]] = fmul float %[[A_IMAG]], %[[TMP_B]]
+// OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG: store float %[[RESULT_REAL]], ptr %[[A_REAL_PTR]], align 4
+// OGCG: store float %[[RESULT_IMAG]], ptr %[[A_IMAG_PTR]], align 4
+
+#ifndef __cplusplus
+void foo9() {
+  float _Complex a;
+  float b;
+  b += a;
+}
+#endif
+
+// C_CIR: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// C_CIR: %[[B_ADDR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["b"]
+// C_CIR: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// C_CIR: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.float>, !cir.float
+// C_CIR: %[[A_REAL:.*]] = cir.complex.real %[[A_ADDR]] : !cir.complex<!cir.float> -> !cir.float
+// C_CIR: %[[A_IMAG:.*]] = cir.complex.imag %[[A_ADDR]] : !cir.complex<!cir.float> -> !cir.float
+// C_CIR: %[[NEW_REAL:.*]] = cir.binop(add, %[[TMP_B]], %[[A_REAL]]) : !cir.float
+// C_CIR: %[[RESULT:.*]] = cir.complex.create %[[NEW_REAL]], %[[A_IMAG]] : !cir.float -> !cir.complex<!cir.float>
+// C_CIR: %[[RESULT_REAL:.*]] = cir.complex.real %[[RESULT]] : !cir.complex<!cir.float> -> !cir.float
+// C_CIR: cir.store{{.*}} %[[RESULT_REAL]], %[[B_ADDR]] : !cir.float, !cir.ptr<!cir.float>
+
+// C_LLVM: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// C_LLVM: %[[B_ADDR:.*]] = alloca float, i64 1, align 4
+// C_LLVM: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// C_LLVM: %[[TMP_B:.*]] = load float, ptr %[[B_ADDR]], align 4
+// C_LLVM: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// C_LLVM: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// C_LLVM: %[[NEW_REAL:.*]] = fadd float %[[TMP_B]], %[[A_REAL]]
+// C_LLVM: %[[TMP_RESULT:.*]] = insertvalue { float, float } {{.*}}, float %[[NEW_REAL]], 0
+// C_LLVM: store float %[[NEW_REAL]], ptr %[[B_ADDR]], align 4
+
+// C_OGCG: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// C_OGCG: %[[B_ADDR:.*]] = alloca float, align 4
+// C_OGCG: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// C_OGCG: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// C_OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// C_OGCG: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// C_OGCG: %[[TMP_B:.*]] = load float, ptr %[[B_ADDR]], align 4
+// C_OGCG: %[[ADD_REAL:.*]] = fadd float %[[TMP_B]], %[[A_REAL]]
+// C_OGCG: store float %[[ADD_REAL]], ptr %[[B_ADDR]], align 4
diff --git a/clang/test/CIR/CodeGen/complex-mul-div.cpp b/clang/test/CIR/CodeGen/complex-mul-div.cpp
index 9d71ef7453001..0fb82b24f6640 100644
--- a/clang/test/CIR/CodeGen/complex-mul-div.cpp
+++ b/clang/test/CIR/CodeGen/complex-mul-div.cpp
@@ -1,38 +1,38 @@
 // complex-range basic
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -complex-range=basic -Wno-unused-value -fclangir -emit-cir -mmlir --mlir-print-ir-before=cir-canonicalize -o %t.cir %s 2>&1 | FileCheck --check-prefix=CIR-BEFORE-BASIC %s
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=basic -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR-AFTER-INT,CIR-AFTER-MUL-COMBINED
+// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR-AFTER-INT,CIR-AFTER-MUL-COMBINED,CIR-COMBINED,CIR-AFTER-BASIC
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=basic -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM-INT,LLVM-MUL-COMBINED
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM-INT,LLVM-MUL-COMBINED,LLVM-COMBINED,LLVM-BASIC
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=basic -Wno-unused-value -emit-llvm %s -o %t.ll
-// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG-INT,OGCG-MUL-COMBINED
+// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG-INT,OGCG-MUL-COMBINED,OGCG-COMBINED,OGCG-BASIC
 
 // complex-range improved
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -complex-range=improved -Wno-unused-value -fclangir -emit-cir -mmlir --mlir-print-ir-before=cir-canonicalize -o %t.cir %s 2>&1 | FileCheck --check-prefix=CIR-BEFORE-IMPROVED %s
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=improved -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR-AFTER-INT,CIR-AFTER-MUL-COMBINED
+// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR-AFTER-INT,CIR-AFTER-MUL-COMBINED,CIR-COMBINED,CIR-AFTER-IMPROVED
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=improved -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM-INT,LLVM-MUL-COMBINED
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM-INT,LLVM-MUL-COMBINED,LLVM-COMBINED,LLVM-IMPROVED
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=improved -Wno-unused-value -emit-llvm %s -o %t.ll
-// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG-INT,OGCG-MUL-COMBINED
+// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG-INT,OGCG-MUL-COMBINED,OGCG-COMBINED,OGCG-IMPROVED
 
 // complex-range promoted
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -complex-range=promoted -Wno-unused-value -fclangir -emit-cir -mmlir --mlir-print-ir-before=cir-canonicalize -o %t.cir %s 2>&1 | FileCheck --check-prefix=CIR-BEFORE-PROMOTED %s
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=promoted -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR-AFTER-INT,CIR-AFTER-MUL-COMBINED
+// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR-AFTER-INT,CIR-AFTER-MUL-COMBINED,CIR-COMBINED,CIR-AFTER-PROMOTED
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=promoted -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM-INT,LLVM-MUL-COMBINED
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM-INT,LLVM-MUL-COMBINED,LLVM-COMBINED,LLVM-PROMOTED
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=promoted -Wno-unused-value -emit-llvm %s -o %t.ll
-// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG-INT,OGCG-MUL-COMBINED
+// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG-INT,OGCG-MUL-COMBINED,OGCG-COMBINED,OGCG-PROMOTED
 
 // complex-range full
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -complex-range=full -Wno-unused-value -fclangir -emit-cir -mmlir --mlir-print-ir-before=cir-canonicalize -o %t.cir %s 2>&1 | FileCheck --check-prefix=CIR-BEFORE-FULL %s
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=full -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
-// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR-AFTER-FULL,CIR-AFTER-INT
+// RUN: FileCheck --input-file=%t.cir %s --check-prefixes=CIR-AFTER-FULL,CIR-AFTER-INT,CIR-COMBINED
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=full -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
-// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM-FULL,LLVM-INT
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefixes=LLVM-FULL,LLVM-INT,LLVM-COMBINED
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-linux-gnu -complex-range=full -Wno-unused-value -emit-llvm %s -o %t.ll
-// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG-FULL,OGCG-INT
+// RUN: FileCheck --input-file=%t.ll %s --check-prefixes=OGCG-FULL,OGCG-INT,OGCG-COMBINED
 
 void foo() {
   float _Complex a;
@@ -278,3 +278,498 @@ void foo1() {
 // OGCG-INT: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[C_ADDR]], i32 0, i32 1
 // OGCG-INT: store i32 %[[C_REAL]], ptr %[[C_REAL_PTR]], align 4
 // OGCG-INT: store i32 %[[C_IMAG]], ptr %[[C_IMAG_PTR]], align 4
+
+void foo2() {
+  float _Complex a;
+  float b;
+  float _Complex c = a * b;
+}
+
+// CIR-COMBINED: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR-COMBINED: %[[B_ADDR:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["b"]
+// CIR-COMBINED: %[[C_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c", init]
+// CIR-COMBINED: %[[TMP_A:.*]] = cir.load{{.*}} %0 : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR-COMBINED: %[[TMP_B:.*]] = cir.load{{.*}} %1 : !cir.ptr<!cir.float>, !cir.float
+// CIR-COMBINED: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-COMBINED: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-COMBINED: %[[RESULT_REAL:.*]] = cir.binop(mul, %[[A_REAL]], %[[TMP_B]]) : !cir.float
+// CIR-COMBINED: %[[RESULT_IMAG:.*]] = cir.binop(mul, %[[A_IMAG]], %[[TMP_B]]) : !cir.float
+// CIR-COMBINED: %[[RESULT:.*]] = cir.complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : !cir.float -> !cir.complex<!cir.float>
+// CIR-COMBINED: cir.store{{.*}} %[[RESULT]], %[[C_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM-COMBINED: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-COMBINED: %[[B_ADDR:.*]] = alloca float, i64 1, align 4
+// LLVM-COMBINED: %[[C_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-COMBINED: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM-COMBINED: %[[TMP_B:.*]] = load float, ptr %[[B_ADDR]], align 4
+// LLVM-COMBINED: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM-COMBINED: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM-COMBINED: %[[RESULT_REAL:.*]] = fmul float %[[A_REAL]], %[[TMP_B]]
+// LLVM-COMBINED: %[[RESULT_IMAG:.*]] = fmul float %[[A_IMAG]], %[[TMP_B]]
+// LLVM-COMBINED: %[[TMP_RESULT:.*]] = insertvalue { float, float } {{.*}}, float %[[RESULT_REAL]], 0
+// LLVM-COMBINED: %[[RESULT:.*]] = insertvalue { float, float } %[[TMP_RESULT]], float %[[RESULT_IMAG]], 1
+// LLVM-COMBINED: store { float, float } %[[RESULT]], ptr %[[C_ADDR]], align 4
+
+// OGCG-COMBINED: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-COMBINED: %[[B_ADDR:.*]] = alloca float, align 4
+// OGCG-COMBINED: %[[C_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-COMBINED: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG-COMBINED: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// OGCG-COMBINED: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG-COMBINED: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// OGCG-COMBINED: %[[TMP_B:.*]] = load float, ptr %[[B_ADDR]], align 4
+// OGCG-COMBINED: %[[RESULT_REAL:.*]] = fmul float %[[A_REAL]], %[[TMP_B]]
+// OGCG-COMBINED: %[[RESULT_IMAG:.*]] = fmul float %[[A_IMAG]], %[[TMP_B]]
+// OGCG-COMBINED: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 0
+// OGCG-COMBINED: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 1
+// OGCG-COMBINED: store float %[[RESULT_REAL]], ptr %[[C_REAL_PTR]], align 4
+// OGCG-COMBINED: store float %[[RESULT_IMAG]], ptr %[[C_IMAG_PTR]], align 4
+
+void foo3() {
+  float _Complex a;
+  float _Complex b;
+  float _Complex c = a / b;
+}
+
+// CIR-BEFORE-BASIC: %{{.*}} = cir.complex.div {{.*}}, {{.*}} range(basic) : !cir.complex<!cir.float>
+
+// CIR-AFTER-BASIC: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR-AFTER-BASIC: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR-AFTER-BASIC: %[[C_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c", init]
+// CIR-AFTER-BASIC: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR-AFTER-BASIC: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR-AFTER-BASIC: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-BASIC: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-BASIC: %[[B_REAL:.*]] = cir.complex.real %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-BASIC: %[[B_IMAG:.*]] = cir.complex.imag %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-BASIC: %[[MUL_AR_BR:.*]] = cir.binop(mul, %[[A_REAL]], %[[B_REAL]]) : !cir.float
+// CIR-AFTER-BASIC: %[[MUL_AI_BI:.*]] = cir.binop(mul, %[[A_IMAG]], %[[B_IMAG]]) : !cir.float
+// CIR-AFTER-BASIC: %[[MUL_BR_BR:.*]] = cir.binop(mul, %[[B_REAL]], %[[B_REAL]]) : !cir.float
+// CIR-AFTER-BASIC: %[[MUL_BI_BI:.*]] = cir.binop(mul, %[[B_IMAG]], %[[B_IMAG]]) : !cir.float
+// CIR-AFTER-BASIC: %[[ADD_ARBR_AIBI:.*]] = cir.binop(add, %[[MUL_AR_BR]], %[[MUL_AI_BI]]) : !cir.float
+// CIR-AFTER-BASIC: %[[ADD_BRBR_BIBI:.*]] = cir.binop(add, %[[MUL_BR_BR]], %[[MUL_BI_BI]]) : !cir.float
+// CIR-AFTER-BASIC: %[[RESULT_REAL:.*]] = cir.binop(div, %[[ADD_ARBR_AIBI]], %[[ADD_BRBR_BIBI]]) : !cir.float
+// CIR-AFTER-BASIC: %[[MUL_AI_BR:.*]] = cir.binop(mul, %[[A_IMAG]], %[[B_REAL]]) : !cir.float
+// CIR-AFTER-BASIC: %[[MUL_AR_BI:.*]] = cir.binop(mul, %[[A_REAL]], %[[B_IMAG]]) : !cir.float
+// CIR-AFTER-BASIC: %[[SUB_AIBR_ARBI:.*]] = cir.binop(sub, %[[MUL_AI_BR]], %[[MUL_AR_BI]]) : !cir.float
+// CIR-AFTER-BASIC: %[[RESULT_IMAG:.*]] = cir.binop(div, %[[SUB_AIBR_ARBI]], %14) : !cir.float
+// CIR-AFTER-BASIC: %[[RESULT:.*]] = cir.complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : !cir.float -> !cir.complex<!cir.float>
+// CIR-AFTER-BASIC: cir.store{{.*}} %[[RESULT]], %[[C_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM-BASIC: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-BASIC: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-BASIC: %[[C_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-BASIC: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM-BASIC: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
+// LLVM-BASIC: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM-BASIC: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM-BASIC: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
+// LLVM-BASIC: %[[B_IMAG:.*]] = extractvalue { float, float } %[[TMP_B]], 1
+// LLVM-BASIC: %[[MUL_AR_BR:.*]] = fmul float %[[A_REAL]], %[[B_REAL]]
+// LLVM-BASIC: %[[MUL_AI_BI:.*]] = fmul float %[[A_IMAG]], %[[B_IMAG]]
+// LLVM-BASIC: %[[MUL_BR_BR:.*]] = fmul float %[[B_REAL]], %[[B_REAL]]
+// LLVM-BASIC: %[[MUL_BI_BI:.*]] = fmul float %[[B_IMAG]], %[[B_IMAG]]
+// LLVM-BASIC: %[[ADD_ARBR_AIBI:.*]] = fadd float %[[MUL_AR_BR]], %[[MUL_AI_BI]]
+// LLVM-BASIC: %[[ADD_BRBR_BIBI:.*]] = fadd float %[[MUL_BR_BR]], %[[MUL_BI_BI]]
+// LLVM-BASIC: %[[RESULT_REAL:.*]] = fdiv float %[[ADD_ARBR_AIBI]], %[[ADD_BRBR_BIBI]]
+// LLVM-BASIC: %[[MUL_AI_BR:.*]] = fmul float %[[A_IMAG]], %[[B_REAL]]
+// LLVM-BASIC: %[[MUL_BR_BI:.*]] = fmul float %[[A_REAL]], %[[B_IMAG]]
+// LLVM-BASIC: %[[SUB_AIBR_BRBI:.*]] = fsub float %[[MUL_AI_BR]], %[[MUL_BR_BI]]
+// LLVM-BASIC: %[[RESULT_IMAG:.*]] = fdiv float %[[SUB_AIBR_BRBI]], %[[ADD_BRBR_BIBI]]
+// LLVM-BASIC: %[[TMP_RESULT:.*]] = insertvalue { float, float } {{.*}}, float %[[RESULT_REAL]], 0
+// LLVM-BASIC: %[[RESULT:.*]] = insertvalue { float, float } %[[TMP_RESULT]], float %[[RESULT_IMAG]], 1
+// LLVM-BASIC: store { float, float } %[[RESULT]], ptr %[[C_ADDR]], align 4
+
+// OGCG-BASIC: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-BASIC: %[[B_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-BASIC: %[[C_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-BASIC: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG-BASIC: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// OGCG-BASIC: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG-BASIC: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// OGCG-BASIC: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG-BASIC: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
+// OGCG-BASIC: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG-BASIC: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
+// OGCG-BASIC: %[[MUL_AR_BR:.*]] = fmul float %[[A_REAL]], %[[B_REAL]]
+// OGCG-BASIC: %[[MUL_AI_BI:.*]] = fmul float %[[A_IMAG]], %[[B_IMAG]]
+// OGCG-BASIC: %[[ADD_ARBR_AIBI:.*]] = fadd float %[[MUL_AR_BR]], %[[MUL_AI_BI]]
+// OGCG-BASIC: %[[MUL_BR_BR:.*]] = fmul float %[[B_REAL]], %[[B_REAL]]
+// OGCG-BASIC: %[[MUL_BI_BI:.*]] = fmul float %[[B_IMAG]], %[[B_IMAG]]
+// OGCG-BASIC: %[[ADD_BRBR_BIBI:.*]] = fadd float %[[MUL_BR_BR]], %[[MUL_BI_BI]]
+// OGCG-BASIC: %[[MUL_AI_BR:.*]] = fmul float %[[A_IMAG]], %[[B_REAL]]
+// OGCG-BASIC: %[[MUL_AR_BI:.*]] = fmul float %[[A_REAL]], %[[B_IMAG]]
+// OGCG-BASIC: %[[SUB_AIBR_BRBI:.*]] = fsub float %[[MUL_AI_BR]], %[[MUL_AR_BI]]
+// OGCG-BASIC: %[[RESULT_REAL:.*]] = fdiv float %[[ADD_ARBR_AIBI]], %[[ADD_BRBR_BIBI]]
+// OGCG-BASIC: %[[RESULT_IMAG:.*]] = fdiv float %[[SUB_AIBR_BRBI]], %[[ADD_BRBR_BIBI]]
+// OGCG-BASIC: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 0
+// OGCG-BASIC: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 1
+// OGCG-BASIC: store float %[[RESULT_REAL]], ptr %[[C_REAL_PTR]], align 4
+// OGCG-BASIC: store float %[[RESULT_IMAG]], ptr %[[C_IMAG_PTR]], align 4
+
+// CIR-BEFORE-IMPROVED: %{{.*}} = cir.complex.div {{.*}}, {{.*}} range(improved) : !cir.complex<!cir.float>
+
+// CIR-AFTER-IMPROVED: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR-AFTER-IMPROVED: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR-AFTER-IMPROVED: %[[C_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c", init]
+// CIR-AFTER-IMPROVED: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR-AFTER-IMPROVED: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR-AFTER-IMPROVED: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-IMPROVED: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-IMPROVED: %[[B_REAL:.*]] = cir.complex.real %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-IMPROVED: %[[B_IMAG:.*]] = cir.complex.imag %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-IMPROVED: %[[ABS_B_REAL:.*]] = cir.fabs %[[B_REAL]] : !cir.float
+// CIR-AFTER-IMPROVED: %[[ABS_B_IMAG:.*]] = cir.fabs %[[B_IMAG]] : !cir.float
+// CIR-AFTER-IMPROVED: %[[ABS_B_CMP:.*]] = cir.cmp(ge, %[[ABS_B_REAL]], %[[ABS_B_IMAG]]) : !cir.float, !cir.bool
+// CIR-AFTER-IMPROVED: %[[RESULT:.*]] = cir.ternary(%[[ABS_B_CMP]], true {
+// CIR-AFTER-IMPROVED:   %[[DIV_BI_BR:.*]] = cir.binop(div, %[[B_IMAG]], %[[B_REAL]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[MUL_DIV_BIBR_BI:.*]] = cir.binop(mul, %[[DIV_BI_BR]], %[[B_IMAG]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[ADD_BR_MUL_DIV_BIBR_BI:.*]] = cir.binop(add, %[[B_REAL]], %[[MUL_DIV_BIBR_BI]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[MUL_AI_DIV_BIBR:.*]] = cir.binop(mul, %[[A_IMAG]], %[[DIV_BI_BR]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[ADD_AR_MUL_AI_DIV_BIBR:.*]] = cir.binop(add, %[[A_REAL]], %[[MUL_AI_DIV_BIBR]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[RESULT_REAL:.*]] = cir.binop(div, %[[ADD_AR_MUL_AI_DIV_BIBR]], %[[ADD_BR_MUL_DIV_BIBR_BI]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[MUL_AR_DIV_BIBR:.*]] = cir.binop(mul, %[[A_REAL]], %[[DIV_BI_BR]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[SUB_AI_MUL_AR_DIV_BIBR:.*]] = cir.binop(sub, %[[A_IMAG]], %[[MUL_AR_DIV_BIBR]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[RESULT_IMAG:.*]] = cir.binop(div, %[[SUB_AI_MUL_AR_DIV_BIBR]], %[[ADD_BR_MUL_DIV_BIBR_BI]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[RESULT_COMPLEX:.*]] = cir.complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : !cir.float -> !cir.complex<!cir.float>
+// CIR-AFTER-IMPROVED:   cir.yield %[[RESULT_COMPLEX]] : !cir.complex<!cir.float>
+// CIR-AFTER-IMPROVED: }, false {
+// CIR-AFTER-IMPROVED:   %[[DIV_BR_BI:.*]] = cir.binop(div, %[[B_REAL]], %[[B_IMAG]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[MUL_DIV_BRBI_BR:.*]] = cir.binop(mul, %[[DIV_BR_BI]], %[[B_REAL]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[ADD_BI_MUL_DIV_BRBI_BR:.*]] = cir.binop(add, %[[B_IMAG]], %[[MUL_DIV_BRBI_BR]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[MUL_AR_DIV_BIBR:.*]] = cir.binop(mul, %[[A_REAL]], %[[DIV_BR_BI]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[ADD_MUL_AR_DIV_BRBI_AI:.*]] = cir.binop(add, %[[MUL_AR_DIV_BIBR]], %[[A_IMAG]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[RESULT_REAL:.*]] = cir.binop(div, %[[ADD_MUL_AR_DIV_BRBI_AI]], %[[ADD_BI_MUL_DIV_BRBI_BR]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[MUL_AI_DIV_BRBI:.*]] = cir.binop(mul, %[[A_IMAG]], %[[DIV_BR_BI]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[SUB_MUL_AI_DIV_BRBI_AR:.*]] = cir.binop(sub, %[[MUL_AI_DIV_BRBI]], %[[A_REAL]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[RESULT_IMAG:.*]] = cir.binop(div, %[[SUB_MUL_AI_DIV_BRBI_AR]], %[[ADD_BI_MUL_DIV_BRBI_BR]]) : !cir.float
+// CIR-AFTER-IMPROVED:   %[[RESULT_COMPLEX:.*]] = cir.complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : !cir.float -> !cir.complex<!cir.float>
+// CIR-AFTER-IMPROVED:   cir.yield %[[RESULT_COMPLEX]] : !cir.complex<!cir.float>
+// CIR-AFTER-IMPROVED: }) : (!cir.bool) -> !cir.complex<!cir.float>
+// CIR-AFTER-IMPROVED: cir.store{{.*}} %[[RESULT]], %[[C_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM-IMPROVED: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-IMPROVED: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-IMPROVED: %[[C_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-IMPROVED: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM-IMPROVED: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
+// LLVM-IMPROVED: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM-IMPROVED: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM-IMPROVED: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
+// LLVM-IMPROVED: %[[B_IMAG:.*]] = extractvalue { float, float } %[[TMP_B]], 1
+// LLVM-IMPROVED: %[[ABS_B_REAL:.*]] = call float @llvm.fabs.f32(float %[[B_REAL]])
+// LLVM-IMPROVED: %[[ABS_B_IMAG:.*]] = call float @llvm.fabs.f32(float %[[B_IMAG]])
+// LLVM-IMPROVED: %[[ABS_B_CMP:.*]] = fcmp oge float %[[ABS_B_REAL]], %[[ABS_B_IMAG]]
+// LLVM-IMPROVED: br i1 %[[ABS_B_CMP]], label %[[ABS_BR_GT_ABS_BI:.*]], label %[[ABS_BR_LT_ABS_BI:.*]]
+// LLVM-IMPROVED: [[ABS_BR_GT_ABS_BI]]:
+// LLVM-IMPROVED:  %[[DIV_BI_BR:.*]] = fdiv float %[[B_IMAG]], %[[B_REAL]]
+// LLVM-IMPROVED:  %[[MUL_DIV_BIBR_BI:.*]] = fmul float %[[DIV_BI_BR]], %[[B_IMAG]]
+// LLVM-IMPROVED:  %[[ADD_BR_MUL_DIV_BIBR_BI:.*]] = fadd float %[[B_REAL]], %[[MUL_DIV_BIBR_BI]]
+// LLVM-IMPROVED:  %[[MUL_AI_DIV_BIBR:.*]] = fmul float %[[A_IMAG]], %[[DIV_BI_BR]]
+// LLVM-IMPROVED:  %[[ADD_AR_MUL_AI_DIV_BIBR:.*]] = fadd float %[[A_REAL]], %[[MUL_AI_DIV_BIBR]]
+// LLVM-IMPROVED:  %[[RESULT_REAL:.*]] = fdiv float %[[ADD_AR_MUL_AI_DIV_BIBR]], %16
+// LLVM-IMPROVED:  %[[MUL_AR_DIV_BIBR:.*]] = fmul float %[[A_REAL]], %[[DIV_BI_BR]]
+// LLVM-IMPROVED:  %[[SUB_AI_MUL_AR_DIV_BIBR:.*]] = fsub float %[[A_IMAG]], %[[MUL_AR_DIV_BIBR]]
+// LLVM-IMPROVED:  %[[RESULT_IMAG:.*]] = fdiv float %[[SUB_AI_MUL_AR_DIV_BIBR]], %[[ADD_BR_MUL_DIV_BIBR_BI]]
+// LLVM-IMPROVED:  %[[TMP_THEN_RESULT:.*]] = insertvalue { float, float } {{.*}}, float %[[RESULT_REAL]], 0
+// LLVM-IMPROVED:  %[[THEN_RESULT:.*]] = insertvalue { float, float } %[[TMP_THEN_RESULT]], float %[[RESULT_IMAG]], 1
+// LLVM-IMPROVED:  br label %[[PHI_RESULT:.*]]
+// LLVM-IMPROVED: [[ABS_BR_LT_ABS_BI]]:
+// LLVM-IMPROVED:  %[[DIV_BR_BI:.*]] = fdiv float %[[B_REAL]], %[[B_IMAG]]
+// LLVM-IMPROVED:  %[[MUL_DIV_BRBI_BR:.*]] = fmul float %[[DIV_BR_BI]], %[[B_REAL]]
+// LLVM-IMPROVED:  %[[ADD_BI_MUL_DIV_BRBI_BR:.*]] = fadd float %[[B_IMAG]], %[[MUL_DIV_BRBI_BR]]
+// LLVM-IMPROVED:  %[[MUL_AR_DIV_BRBI:.*]] = fmul float %[[A_REAL]], %[[DIV_BR_BI]]
+// LLVM-IMPROVED:  %[[ADD_MUL_AR_DIV_BRBI_AI:.*]] = fadd float %[[MUL_AR_DIV_BRBI]], %[[A_IMAG]]
+// LLVM-IMPROVED:  %[[RESULT_REAL:.*]] = fdiv float %[[ADD_MUL_AR_DIV_BRBI_AI]], %[[ADD_BI_MUL_DIV_BRBI_BR]]
+// LLVM-IMPROVED:  %[[MUL_AI_DIV_BRBI:.*]] = fmul float %[[A_IMAG]], %[[DIV_BR_BI]]
+// LLVM-IMPROVED:  %[[SUB_MUL_AI_DIV_BRBI_AR:.*]] = fsub float %[[MUL_AI_DIV_BRBI]], %[[A_REAL]]
+// LLVM-IMPROVED:  %[[RESULT_IMAG:.*]] = fdiv float %[[SUB_MUL_AI_DIV_BRBI_AR]], %[[ADD_BI_MUL_DIV_BRBI_BR]]
+// LLVM-IMPROVED:  %[[TMP_ELSE_RESULT:.*]] = insertvalue { float, float } {{.*}}, float %[[RESULT_REAL]], 0
+// LLVM-IMPROVED:  %[[ELSE_RESULT:.*]] = insertvalue { float, float } %[[TMP_ELSE_RESULT]], float %[[RESULT_IMAG]], 1
+// LLVM-IMPROVED:  br label %[[PHI_RESULT]]
+// LLVM-IMPROVED: [[PHI_RESULT]]:
+// LLVM-IMPROVED:  %[[RESULT:.*]] = phi { float, float } [ %[[ELSE_RESULT]], %[[ABS_BR_LT_ABS_BI]] ], [ %[[THEN_RESULT]], %[[ABS_BR_GT_ABS_BI]] ]
+// LLVM-IMPROVED:  br label %[[STORE_RESULT:.*]]
+// LLVM-IMPROVED: [[STORE_RESULT]]:
+// LLVM-IMPROVED:  store { float, float } %[[RESULT]], ptr %[[C_ADDR]], align 4
+
+// OGCG-IMPROVED: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-IMPROVED: %[[B_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-IMPROVED: %[[C_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-IMPROVED: %a.realp = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG-IMPROVED: %a.real = load float, ptr %a.realp, align 4
+// OGCG-IMPROVED: %a.imagp = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG-IMPROVED: %a.imag = load float, ptr %a.imagp, align 4
+// OGCG-IMPROVED: %b.realp = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG-IMPROVED: %b.real = load float, ptr %b.realp, align 4
+// OGCG-IMPROVED: %b.imagp = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG-IMPROVED: %b.imag = load float, ptr %b.imagp, align 4
+// OGCG-IMPROVED: %[[ABS_B_REAL:.*]] = call float @llvm.fabs.f32(float %[[B_REAL]])
+// OGCG-IMPROVED: %[[ABS_B_IMAG:.*]] = call float @llvm.fabs.f32(float %[[B_IMAG]])
+// OGCG-IMPROVED: %[[ABS_B_CMP:.*]] = fcmp ugt float %[[ABS_B_REAL]], %[[ABS_B_IMAG]]
+// OGCG-IMPROVED: br i1 %[[ABS_B_CMP]], label %[[ABS_BR_GT_ABS_BI:.*]], label %[[ABS_BR_LT_ABS_BI:.*]]
+// OGCG-IMPROVED: [[ABS_BR_GT_ABS_BI]]:
+// OGCG-IMPROVED:  %[[DIV_BI_BR:.*]] = fdiv float %[[B_IMAG]], %[[B_REAL]]
+// OGCG-IMPROVED:  %[[MUL_DIV_BIBR_BI:.*]] = fmul float %[[DIV_BI_BR]], %[[B_IMAG]]
+// OGCG-IMPROVED:  %[[ADD_BR_MUL_DIV_BIBR_BI:.*]] = fadd float %[[B_REAL]], %[[MUL_DIV_BIBR_BI]]
+// OGCG-IMPROVED:  %[[MUL_AI_DIV_BIBR:.*]] = fmul float %[[A_IMAG]], %[[DIV_BI_BR]]
+// OGCG-IMPROVED:  %[[ADD_AR_MUL_AI_DIV_BIBR:.*]] = fadd float %[[A_REAL]], %[[MUL_AI_DIV_BIBR]]
+// OGCG-IMPROVED:  %[[THEN_RESULT_REAL:.*]] = fdiv float %[[ADD_AR_MUL_AI_DIV_BIBR]], %[[ADD_BR_MUL_DIV_BIBR_BI]]
+// OGCG-IMPROVED:  %[[MUL_AR_DIV_BI_BR:.*]] = fmul float %[[A_REAL]], %[[DIV_BI_BR]]
+// OGCG-IMPROVED:  %[[SUB_AI_MUL_AR_DIV_BIBR:.*]] = fsub float %[[A_IMAG]], %[[MUL_AR_DIV_BI_BR]]
+// OGCG-IMPROVED:  %[[THEN_RESULT_IMAG:.*]] = fdiv float %[[SUB_AI_MUL_AR_DIV_BIBR]], %[[ADD_BR_MUL_DIV_BIBR_BI]]
+// OGCG-IMPROVED:  br label %[[STORE_RESULT:.*]]
+// OGCG-IMPROVED: [[ABS_BR_LT_ABS_BI]]:
+// OGCG-IMPROVED:  %[[DIV_BR_BI:.*]] = fdiv float %[[B_REAL]], %[[B_IMAG]]
+// OGCG-IMPROVED:  %[[MUL_DIV_BRBI_BR:.*]] = fmul float %[[DIV_BR_BI]], %[[B_REAL]]
+// OGCG-IMPROVED:  %[[ADD_BI_MUL_DIV_BRBI_BR:.*]] = fadd float %[[B_IMAG]], %[[MUL_DIV_BRBI_BR]]
+// OGCG-IMPROVED:  %[[MUL_AR_DIV_BRBI:.*]] = fmul float %[[A_REAL]], %[[DIV_BR_BI]]
+// OGCG-IMPROVED:  %[[ADD_MUL_AR_DIV_BRBI_AI:.*]] = fadd float %[[MUL_AR_DIV_BRBI]], %[[A_IMAG]]
+// OGCG-IMPROVED:  %[[ELSE_RESULT_REAL:.*]] = fdiv float %[[ADD_MUL_AR_DIV_BRBI_AI]], %[[ADD_BI_MUL_DIV_BRBI_BR]]
+// OGCG-IMPROVED:  %[[MUL_AI_DIV_BRBI:.*]] = fmul float %[[A_IMAG]], %[[DIV_BR_BI]]
+// OGCG-IMPROVED:  %[[SUB_MUL_AI_DIV_BRBI_AR:.*]] = fsub float %[[MUL_AI_DIV_BRBI]], %[[A_REAL]]
+// OGCG-IMPROVED:  %[[ELSE_RESULT_IMAG:.*]] = fdiv float %[[SUB_MUL_AI_DIV_BRBI_AR]], %[[ADD_BI_MUL_DIV_BRBI_BR]]
+// OGCG-IMPROVED:  br label %[[STORE_RESULT]]
+// OGCG-IMPROVED: [[STORE_RESULT]]:
+// OGCG-IMPROVED:  %[[RESULT_REAL:.*]] = phi float [ %[[THEN_RESULT_REAL]], %[[ABS_BR_GT_ABS_BI]] ], [ %[[ELSE_RESULT_REAL]], %[[ABS_BR_LT_ABS_BI]] ]
+// OGCG-IMPROVED:  %[[RESULT_IMAG:.*]] = phi float [ %[[THEN_RESULT_IMAG]], %[[ABS_BR_GT_ABS_BI]] ], [ %[[ELSE_RESULT_IMAG]], %[[ABS_BR_LT_ABS_BI]] ]
+// OGCG-IMPROVED:  %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 0
+// OGCG-IMPROVED:  %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 1
+// OGCG-IMPROVED:  store float %[[RESULT_REAL]], ptr %[[C_REAL_PTR]], align 4
+// OGCG-IMPROVED:  store float %[[RESULT_IMAG]], ptr %[[C_IMAG_PTR]], align 4
+
+// CIR-BEFORE-PROMOTED: %{{.*}} = cir.complex.div {{.*}}, {{.*}} range(promoted) : !cir.complex<!cir.float>
+
+// CIR-AFTER-PROMOTED: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR-AFTER-PROMOTED: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR-AFTER-PROMOTED: %[[C_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c", init]
+// CIR-AFTER-PROMOTED: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR-AFTER-PROMOTED: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR-AFTER-PROMOTED: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-PROMOTED: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-PROMOTED: %[[B_REAL:.*]] = cir.complex.real %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-PROMOTED: %[[B_IMAG:.*]] = cir.complex.imag %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-PROMOTED: %[[A_REAL_F64:.*]] = cir.cast(floating, %[[A_REAL]] : !cir.float), !cir.double
+// CIR-AFTER-PROMOTED: %[[A_IMAG_F64:.*]] = cir.cast(floating, %[[A_IMAG]] : !cir.float), !cir.double
+// CIR-AFTER-PROMOTED: %[[B_REAL_F64:.*]] = cir.cast(floating, %[[B_REAL]] : !cir.float), !cir.double
+// CIR-AFTER-PROMOTED: %[[B_IMAG_F64:.*]] = cir.cast(floating, %[[B_IMAG]] : !cir.float), !cir.double
+// CIR-AFTER-PROMOTED: %[[MUL_AR_BR:.*]] = cir.binop(mul, %[[A_REAL_F64]], %[[B_REAL_F64]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[MUL_AI_BI:.*]] = cir.binop(mul, %[[A_IMAG_F64]], %[[B_IMAG_F64]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[MUL_BR_BR:.*]] = cir.binop(mul, %[[B_REAL_F64]], %[[B_REAL_F64]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[MUL_BI_BI:.*]] = cir.binop(mul, %[[B_IMAG_F64]], %[[B_IMAG_F64]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[ADD_ARBR_AIBI:.*]] = cir.binop(add, %[[MUL_AR_BR]], %[[MUL_AI_BI]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[ADD_BRBR_BIBI:.*]] = cir.binop(add, %[[MUL_BR_BR]], %[[MUL_BI_BI]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[RESULT_REAL:.*]] = cir.binop(div, %[[ADD_ARBR_AIBI]], %18) : !cir.double
+// CIR-AFTER-PROMOTED: %[[MUL_AI_BR:.*]] = cir.binop(mul, %[[A_IMAG_F64]], %[[B_REAL_F64]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[MUL_AR_BI:.*]] = cir.binop(mul, %[[A_REAL_F64]], %[[B_IMAG_F64]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[SUB_AIBR_ARBI:.*]] = cir.binop(sub, %[[MUL_AI_BR]], %[[MUL_AR_BI]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[RESULT_IMAG:.*]] = cir.binop(div, %[[SUB_AIBR_ARBI]], %[[ADD_BRBR_BIBI]]) : !cir.double
+// CIR-AFTER-PROMOTED: %[[RESULT_F64:.*]] = cir.complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : !cir.double -> !cir.complex<!cir.double>
+// CIR-AFTER-PROMOTED: %[[RESULT_REAL_F64:.*]] = cir.complex.real %[[RESULT_F64]] : !cir.complex<!cir.double> -> !cir.double
+// CIR-AFTER-PROMOTED: %[[RESULT_IMAG_F64:.*]] = cir.complex.imag %[[RESULT_F64]] : !cir.complex<!cir.double> -> !cir.double
+// CIR-AFTER-PROMOTED: %[[RESULT_REAL_F32:.*]] = cir.cast(floating, %[[RESULT_REAL_F64]] : !cir.double), !cir.float
+// CIR-AFTER-PROMOTED: %[[RESULT_IMAG_F32:.*]] = cir.cast(floating, %[[RESULT_IMAG_F64]] : !cir.double), !cir.float
+// CIR-AFTER-PROMOTED: %[[RESULT_F32:.*]] = cir.complex.create %[[RESULT_REAL_F32]], %[[RESULT_IMAG_F32]] : !cir.float -> !cir.complex<!cir.float>
+// CIR-AFTER-PROMOTED: cir.store{{.*}} %[[RESULT_F32]], %[[C_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM-PROMOTED: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-PROMOTED: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-PROMOTED: %[[C_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-PROMOTED: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM-PROMOTED: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
+// LLVM-PROMOTED: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM-PROMOTED: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM-PROMOTED: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
+// LLVM-PROMOTED: %[[B_IMAG:.*]] = extractvalue { float, float } %[[TMP_B]], 1
+// LLVM-PROMOTED: %[[A_REAL_F64:.*]] = fpext float %[[A_REAL]] to double
+// LLVM-PROMOTED: %[[A_IMAG_F64:.*]] = fpext float %[[A_IMAG]] to double
+// LLVM-PROMOTED: %[[B_REAL_F64:.*]] = fpext float %[[B_REAL]] to double
+// LLVM-PROMOTED: %[[B_IMAG_F64:.*]] = fpext float %[[B_IMAG]] to double
+// LLVM-PROMOTED: %[[MUL_AR_BR:.*]] = fmul double %[[A_REAL_F64]], %[[B_REAL_F64]]
+// LLVM-PROMOTED: %[[MUL_AI_BI:.*]] = fmul double %[[A_IMAG_F64]], %[[B_IMAG_F64]]
+// LLVM-PROMOTED: %[[MUL_BR_BR:.*]] = fmul double %[[B_REAL_F64]], %[[B_REAL_F64]]
+// LLVM-PROMOTED: %[[MUL_BI_BI:.*]] = fmul double %[[B_IMAG_F64]], %[[B_IMAG_F64]]
+// LLVM-PROMOTED: %[[ADD_ARBR_AIBI:.*]] = fadd double %[[MUL_AR_BR]], %[[MUL_AI_BI]]
+// LLVM-PROMOTED: %[[ADD_BRBR_BIBI:.*]] = fadd double %[[MUL_BR_BR]], %[[MUL_BI_BI]]
+// LLVM-PROMOTED: %[[RESULT_REAL:.*]] = fdiv double %[[ADD_ARBR_AIBI]], %[[ADD_BRBR_BIBI]]
+// LLVM-PROMOTED: %[[MUL_AI_BR:.*]] = fmul double %[[A_IMAG_F64]], %[[B_REAL_F64]]
+// LLVM-PROMOTED: %[[MUL_AR_BR:.*]] = fmul double %[[A_REAL_F64]], %[[B_IMAG_F64]]
+// LLVM-PROMOTED: %[[SUB_AIBR_ARBI:.*]] = fsub double %[[MUL_AI_BR]], %[[MUL_AR_BR]]
+// LLVM-PROMOTED: %[[RESULT_IMAG:.*]] = fdiv double %[[SUB_AIBR_ARBI]], %[[ADD_BRBR_BIBI]]
+// LLVM-PROMOTED: %[[TMP_RESULT_F64:.*]] = insertvalue { double, double } {{.*}}, double %[[RESULT_REAL]], 0
+// LLVM-PROMOTED: %[[RESULT_F64:.*]] = insertvalue { double, double } %[[TMP_RESULT_F64]], double %[[RESULT_IMAG]], 1
+// LLVM-PROMOTED: %[[RESULT_REAL_F32:.*]] = fptrunc double %[[RESULT_REAL]] to float
+// LLVM-PROMOTED: %[[RESULT_IMAG_F32:.*]] = fptrunc double %[[RESULT_IMAG]] to float
+// LLVM-PROMOTED: %[[TMP_RESULT_F32:.*]] = insertvalue { float, float } {{.*}}, float %[[RESULT_REAL_F32]], 0
+// LLVM-PROMOTED: %[[RESULT_F32:.*]] = insertvalue { float, float } %[[TMP_RESULT_F32]], float %[[RESULT_IMAG_F32]], 1
+// LLVM-PROMOTED: store { float, float } %[[RESULT_F32]], ptr %[[C_ADDR]], align 4
+
+// OGCG-PROMOTED: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-PROMOTED: %[[B_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-PROMOTED: %[[C_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-PROMOTED: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG-PROMOTED: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// OGCG-PROMOTED: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG-PROMOTED: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// OGCG-PROMOTED: %[[A_REAL_F64:.*]] = fpext float %[[A_REAL]] to double
+// OGCG-PROMOTED: %[[A_IMAG_F64:.*]] = fpext float %[[A_IMAG]] to double
+// OGCG-PROMOTED: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG-PROMOTED: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
+// OGCG-PROMOTED: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG-PROMOTED: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
+// OGCG-PROMOTED: %[[B_REAL_F64:.*]] = fpext float %[[B_REAL]] to double
+// OGCG-PROMOTED: %[[B_IMAG_F64:.*]] = fpext float %[[B_IMAG]] to double
+// OGCG-PROMOTED: %[[MUL_AR_BR:.*]] = fmul double %[[A_REAL_F64]], %[[B_REAL_F64]]
+// OGCG-PROMOTED: %[[MUL_AI_BI:.*]] = fmul double %[[A_IMAG_F64]], %[[B_IMAG_F64]]
+// OGCG-PROMOTED: %[[ADD_ARBR_AIBI:.*]] = fadd double %[[MUL_AR_BR]], %[[MUL_AI_BI]]
+// OGCG-PROMOTED: %[[MUL_BR_BR:.*]] = fmul double %[[B_REAL_F64]], %[[B_REAL_F64]]
+// OGCG-PROMOTED: %[[MUL_BI_BI:.*]] = fmul double %[[B_IMAG_F64]], %[[B_IMAG_F64]]
+// OGCG-PROMOTED: %[[ADD_BRBR_BIBI:.*]] = fadd double %[[MUL_BR_BR]], %[[MUL_BI_BI]]
+// OGCG-PROMOTED: %[[MUL_AI_BR:.*]] = fmul double %[[A_IMAG_F64]], %[[B_REAL_F64]]
+// OGCG-PROMOTED: %[[MUL_AR_BI:.*]] = fmul double %[[A_REAL_F64]], %[[B_IMAG_F64]]
+// OGCG-PROMOTED: %[[SUB_AIBR_ARBI:.*]] = fsub double %[[MUL_AI_BR]], %[[MUL_AR_BI]]
+// OGCG-PROMOTED: %[[RESULT_REAL:.*]] = fdiv double %[[ADD_ARBR_AIBI]], %[[ADD_BRBR_BIBI]]
+// OGCG-PROMOTED: %[[RESULT_IMAG:.*]] = fdiv double %[[SUB_AIBR_ARBI]], %[[ADD_BRBR_BIBI]]
+// OGCG-PROMOTED: %[[UNPROMOTION_RESULT_REAL:.*]] = fptrunc double %[[RESULT_REAL]] to float
+// OGCG-PROMOTED: %[[UNPROMOTION_RESULT_IMAG:.*]] = fptrunc double %[[RESULT_IMAG]] to float
+// OGCG-PROMOTED: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 0
+// OGCG-PROMOTED: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 1
+// OGCG-PROMOTED: store float %[[UNPROMOTION_RESULT_REAL]], ptr %[[C_REAL_PTR]], align 4
+// OGCG-PROMOTED: store float %[[UNPROMOTION_RESULT_IMAG]], ptr %[[C_IMAG_PTR]], align 4
+
+// CIR-BEFORE-FULL: %{{.*}} = cir.complex.div {{.*}}, {{.*}} range(full) : !cir.complex<!cir.float>
+
+// CIR-AFTER-FULL: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["a"]
+// CIR-AFTER-FULL: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["b"]
+// CIR-AFTER-FULL: %[[C_ADDR:.*]] = cir.alloca !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>, ["c", init]
+// CIR-AFTER-FULL: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR-AFTER-FULL: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!cir.float>>, !cir.complex<!cir.float>
+// CIR-AFTER-FULL: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-FULL: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-FULL: %[[B_REAL:.*]] = cir.complex.real %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-FULL: %[[B_IMAG:.*]] = cir.complex.imag %[[TMP_B]] : !cir.complex<!cir.float> -> !cir.float
+// CIR-AFTER-FULL: %[[RESULT:.*]] = cir.call @__divsc3(%[[A_REAL]], %[[A_IMAG]], %[[B_REAL]], %[[B_IMAG]]) : (!cir.float, !cir.float, !cir.float, !cir.float) -> !cir.complex<!cir.float>
+// CIR-AFTER-FULL: cir.store{{.*}} %[[RESULT]], %[[C_ADDR]] : !cir.complex<!cir.float>, !cir.ptr<!cir.complex<!cir.float>>
+
+// LLVM-FULL: %[[A_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-FULL: %[[B_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-FULL: %[[C_ADDR:.*]] = alloca { float, float }, i64 1, align 4
+// LLVM-FULL: %[[TMP_A:.*]] = load { float, float }, ptr %[[A_ADDR]], align 4
+// LLVM-FULL: %[[TMP_B:.*]] = load { float, float }, ptr %[[B_ADDR]], align 4
+// LLVM-FULL: %[[A_REAL:.*]] = extractvalue { float, float } %[[TMP_A]], 0
+// LLVM-FULL: %[[A_IMAG:.*]] = extractvalue { float, float } %[[TMP_A]], 1
+// LLVM-FULL: %[[B_REAL:.*]] = extractvalue { float, float } %[[TMP_B]], 0
+// LLVM-FULL: %[[B_IMAG:.*]] = extractvalue { float, float } %[[TMP_B]], 1
+// LLVM-FULL: %[[RESULT:.*]] = call { float, float } @__divsc3(float %[[A_REAL]], float %[[A_IMAG]], float %[[B_REAL]], float %[[B_IMAG]])
+// LLVM-FULL: store { float, float } %[[RESULT]], ptr %[[C_ADDR]], align 4
+
+// OGCG-FULL: %[[A_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-FULL: %[[B_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-FULL: %[[C_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-FULL: %[[RESULT_ADDR:.*]] = alloca { float, float }, align 4
+// OGCG-FULL: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG-FULL: %[[A_REAL:.*]] = load float, ptr %[[A_REAL_PTR]], align 4
+// OGCG-FULL: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG-FULL: %[[A_IMAG:.*]] = load float, ptr %[[A_IMAG_PTR]], align 4
+// OGCG-FULL: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG-FULL: %[[B_REAL:.*]] = load float, ptr %[[B_REAL_PTR]], align 4
+// OGCG-FULL: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG-FULL: %[[B_IMAG:.*]] = load float, ptr %[[B_IMAG_PTR]], align 4
+// OGCG-FULL: %[[RESULT:.*]] = call noundef <2 x float> @__divsc3(float noundef %[[A_REAL]], float noundef %[[A_IMAG]], float noundef %[[B_REAL]], float noundef %[[B_IMAG]]) #2
+// OGCG-FULL: store <2 x float> %[[RESULT]], ptr %[[RESULT_ADDR]], align 4
+// OGCG-FULL: %[[RESULT_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[RESULT_ADDR]], i32 0, i32 0
+// OGCG-FULL: %[[RESULT_REAL:.*]] = load float, ptr %[[RESULT_REAL_PTR]], align 4
+// OGCG-FULL: %[[RESULT_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[RESULT_ADDR]], i32 0, i32 1
+// OGCG-FULL: %[[RESULT_IMAG:.*]] = load float, ptr %[[RESULT_IMAG_PTR]], align 4
+// OGCG-FULL: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 0
+// OGCG-FULL: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[C_ADDR]], i32 0, i32 1
+// OGCG-FULL: store float %[[RESULT_REAL]], ptr %[[C_REAL_PTR]], align 4
+// OGCG-FULL: store float %[[RESULT_IMAG]], ptr %[[C_IMAG_PTR]], align 4
+
+void foo4() {
+  int _Complex a;
+  int _Complex b;
+  int _Complex c = a / b;
+}
+
+// CIR-BEFORE-BASIC: %{{.*}} = cir.complex.div {{.*}}, {{.*}} range(basic) : !cir.complex<!s32i>
+
+// CIR-BEFORE-IMPROVED: %{{.*}} = cir.complex.div {{.*}}, {{.*}} range(improved) : !cir.complex<!s32i>
+
+// CIR-BEFORE-PROMOTED: %{{.*}} = cir.complex.div {{.*}}, {{.*}} range(promoted) : !cir.complex<!s32i>
+
+// CIR-BEFORE-FULL: %{{.*}} = cir.complex.div {{.*}}, {{.*}} range(full) : !cir.complex<!s32i>
+
+// CIR-COMBINED: %[[A_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["a"]
+// CIR-COMBINED: %[[B_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["b"]
+// CIR-COMBINED: %[[C_ADDR:.*]] = cir.alloca !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>, ["c", init]
+// CIR-COMBINED: %[[TMP_A:.*]] = cir.load{{.*}} %[[A_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR-COMBINED: %[[TMP_B:.*]] = cir.load{{.*}} %[[B_ADDR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR-COMBINED: %[[A_REAL:.*]] = cir.complex.real %[[TMP_A]] : !cir.complex<!s32i> -> !s32i
+// CIR-COMBINED: %[[A_IMAG:.*]] = cir.complex.imag %[[TMP_A]] : !cir.complex<!s32i> -> !s32i
+// CIR-COMBINED: %[[B_REAL:.*]] = cir.complex.real %[[TMP_B]] : !cir.complex<!s32i> -> !s32i
+// CIR-COMBINED: %[[B_IMAG:.*]] = cir.complex.imag %[[TMP_B]] : !cir.complex<!s32i> -> !s32i
+// CIR-COMBINED: %[[MUL_AR_BR:.*]] = cir.binop(mul, %[[A_REAL]], %[[B_REAL]]) : !s32i
+// CIR-COMBINED: %[[MUL_AI_BI:.*]] = cir.binop(mul, %[[A_IMAG]], %[[B_IMAG]]) : !s32i
+// CIR-COMBINED: %[[MUL_BR_BR:.*]] = cir.binop(mul, %[[B_REAL]], %[[B_REAL]]) : !s32i
+// CIR-COMBINED: %[[MUL_BI_BI:.*]] = cir.binop(mul, %[[B_IMAG]], %[[B_IMAG]]) : !s32i
+// CIR-COMBINED: %[[ADD_ARBR_AIBI:.*]] = cir.binop(add, %[[MUL_AR_BR]], %[[MUL_AI_BI]]) : !s32i
+// CIR-COMBINED: %[[ADD_BRBR_BIBI:.*]] = cir.binop(add, %[[MUL_BR_BR]], %[[MUL_BI_BI]]) : !s32i
+// CIR-COMBINED: %[[RESULT_REAL:.*]] = cir.binop(div, %[[ADD_ARBR_AIBI]], %[[ADD_BRBR_BIBI]]) : !s32i
+// CIR-COMBINED: %[[MUL_AI_BR:.*]] = cir.binop(mul, %[[A_IMAG]], %[[B_REAL]]) : !s32i
+// CIR-COMBINED: %[[MUL_AR_BI:.*]] = cir.binop(mul, %[[A_REAL]], %[[B_IMAG]]) : !s32i
+// CIR-COMBINED: %[[SUB_AIBR_ARBI:.*]] = cir.binop(sub, %[[MUL_AI_BR]], %[[MUL_AR_BI]]) : !s32i
+// CIR-COMBINED: %[[RESULT_IMAG:.*]] = cir.binop(div, %[[SUB_AIBR_ARBI]], %14) : !s32i
+// CIR-COMBINED: %[[RESULT:.*]] = cir.complex.create %[[RESULT_REAL]], %[[RESULT_IMAG]] : !s32i -> !cir.complex<!s32i>
+// CIR-COMBINED: cir.store{{.*}} %[[RESULT]], %[[C_ADDR]] : !cir.complex<!s32i>, !cir.ptr<!cir.complex<!s32i>>
+
+// LLVM-COMBINED: %[[A_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM-COMBINED: %[[B_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM-COMBINED: %[[C_ADDR:.*]] = alloca { i32, i32 }, i64 1, align 4
+// LLVM-COMBINED: %[[TMP_A:.*]] = load { i32, i32 }, ptr %[[A_ADDR]], align 4
+// LLVM-COMBINED: %[[TMP_B:.*]] = load { i32, i32 }, ptr %[[B_ADDR]], align 4
+// LLVM-COMBINED: %[[A_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 0
+// LLVM-COMBINED: %[[A_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_A]], 1
+// LLVM-COMBINED: %[[B_REAL:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 0
+// LLVM-COMBINED: %[[B_IMAG:.*]] = extractvalue { i32, i32 } %[[TMP_B]], 1
+// LLVM-COMBINED: %[[MUL_AR_BR:.*]] = mul i32 %[[A_REAL]], %[[B_REAL]]
+// LLVM-COMBINED: %[[MUL_AI_BI:.*]] = mul i32 %[[A_IMAG]], %[[B_IMAG]]
+// LLVM-COMBINED: %[[MUL_BR_BR:.*]] = mul i32 %[[B_REAL]], %[[B_REAL]]
+// LLVM-COMBINED: %[[MUL_BI_BI:.*]] = mul i32 %[[B_IMAG]], %[[B_IMAG]]
+// LLVM-COMBINED: %[[ADD_ARBR_AIBI:.*]] = add i32 %[[MUL_AR_BR]], %[[MUL_AI_BI]]
+// LLVM-COMBINED: %[[ADD_BRBR_BIBI:.*]] = add i32 %[[MUL_BR_BR]], %[[MUL_BI_BI]]
+// LLVM-COMBINED: %[[RESULT_REAL:.*]] = sdiv i32 %[[ADD_ARBR_AIBI]], %[[ADD_BRBR_BIBI]]
+// LLVM-COMBINED: %[[MUL_AI_BR:.*]] = mul i32 %[[A_IMAG]], %[[B_REAL]]
+// LLVM-COMBINED: %[[MUL_BR_BI:.*]] = mul i32 %[[A_REAL]], %[[B_IMAG]]
+// LLVM-COMBINED: %[[SUB_AIBR_BRBI:.*]] = sub i32 %[[MUL_AI_BR]], %[[MUL_BR_BI]]
+// LLVM-COMBINED: %[[RESULT_IMAG:.*]] = sdiv i32 %[[SUB_AIBR_BRBI]], %[[ADD_BRBR_BIBI]]
+// LLVM-COMBINED: %[[TMP_RESULT:.*]] = insertvalue { i32, i32 } {{.*}}, i32 %[[RESULT_REAL]], 0
+// LLVM-COMBINED: %[[RESULT:.*]] = insertvalue { i32, i32 } %[[TMP_RESULT]], i32 %[[RESULT_IMAG]], 1
+// LLVM-COMBINED: store { i32, i32 } %[[RESULT]], ptr %[[C_ADDR]], align 4
+
+// OGCG-COMBINED: %[[A_ADDR:.*]] = alloca { i32, i32 }, align 4
+// OGCG-COMBINED: %[[B_ADDR:.*]] = alloca { i32, i32 }, align 4
+// OGCG-COMBINED: %[[C_ADDR:.*]] = alloca { i32, i32 }, align 4
+// OGCG-COMBINED: %[[A_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 0
+// OGCG-COMBINED: %[[A_REAL:.*]] = load i32, ptr %[[A_REAL_PTR]], align 4
+// OGCG-COMBINED: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[A_ADDR]], i32 0, i32 1
+// OGCG-COMBINED: %[[A_IMAG:.*]] = load i32, ptr %[[A_IMAG_PTR]], align 4
+// OGCG-COMBINED: %[[B_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 0
+// OGCG-COMBINED: %[[B_REAL:.*]] = load i32, ptr %[[B_REAL_PTR]], align 4
+// OGCG-COMBINED: %[[B_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[B_ADDR]], i32 0, i32 1
+// OGCG-COMBINED: %[[B_IMAG:.*]] = load i32, ptr %[[B_IMAG_PTR]], align 4
+// OGCG-COMBINED: %[[MUL_AR_BR:.*]] = mul i32 %[[A_REAL]], %[[B_REAL]]
+// OGCG-COMBINED: %[[MUL_AI_BI:.*]] = mul i32 %[[A_IMAG]], %[[B_IMAG]]
+// OGCG-COMBINED: %[[ADD_ARBR_AIBI:.*]] = add i32 %[[MUL_AR_BR]], %[[MUL_AI_BI]]
+// OGCG-COMBINED: %[[MUL_BR_BR:.*]] = mul i32 %[[B_REAL]], %[[B_REAL]]
+// OGCG-COMBINED: %[[MUL_BI_BI:.*]] = mul i32 %[[B_IMAG]], %[[B_IMAG]]
+// OGCG-COMBINED: %[[ADD_BRBR_BIBI:.*]] = add i32 %[[MUL_BR_BR]], %[[MUL_BI_BI]]
+// OGCG-COMBINED: %[[MUL_AI_BR:.*]] = mul i32 %[[A_IMAG]], %[[B_REAL]]
+// OGCG-COMBINED: %[[MUL_AR_BI:.*]] = mul i32 %[[A_REAL]], %[[B_IMAG]]
+// OGCG-COMBINED: %[[SUB_AIBR_BRBI:.*]] = sub i32 %[[MUL_AI_BR]], %[[MUL_AR_BI]]
+// OGCG-COMBINED: %[[RESULT_REAL:.*]] = sdiv i32 %[[ADD_ARBR_AIBI]], %[[ADD_BRBR_BIBI]]
+// OGCG-COMBINED: %[[RESULT_IMAG:.*]] = sdiv i32 %[[SUB_AIBR_BRBI]], %[[ADD_BRBR_BIBI]]
+// OGCG-COMBINED: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[C_ADDR]], i32 0, i32 0
+// OGCG-COMBINED: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[C_ADDR]], i32 0, i32 1
+// OGCG-COMBINED: store i32 %[[RESULT_REAL]], ptr %[[C_REAL_PTR]], align 4
+// OGCG-COMBINED: store i32 %[[RESULT_IMAG]], ptr %[[C_IMAG_PTR]], align 4
diff --git a/clang/test/CIR/CodeGen/complex.cpp b/clang/test/CIR/CodeGen/complex.cpp
index bd7de9a3a5039..e435a5e6ed010 100644
--- a/clang/test/CIR/CodeGen/complex.cpp
+++ b/clang/test/CIR/CodeGen/complex.cpp
@@ -799,3 +799,57 @@ void foo30() {
 // OGCG: %[[A_IMAG_PTR:.*]] = getelementptr inbounds nuw { float, float }, ptr %[[A_ADDR]], i32 0, i32 1
 // OGCG: store float 1.000000e+00, ptr %[[A_REAL_PTR]], align 4
 // OGCG: store float 0.000000e+00, ptr %[[A_IMAG_PTR]], align 4
+
+void foo31() {
+  struct Wrapper {
+    int _Complex c;
+  };
+
+  Wrapper w;
+  int r = __real__ w.c;
+}
+
+// CIR: %[[W_ADDR:.*]] = cir.alloca !rec_Wrapper, !cir.ptr<!rec_Wrapper>, ["w"]
+// CIR: %[[REAL_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["r", init]
+// CIR: %[[ELEM_PTR:.*]] = cir.get_member %[[W_ADDR]][0] {name = "c"} : !cir.ptr<!rec_Wrapper> -> !cir.ptr<!cir.complex<!s32i>>
+// CIR: %[[TMP_ELEM_PTR:.*]] = cir.load{{.*}} %[[ELEM_PTR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[REAL:.*]] = cir.complex.real %[[TMP_ELEM_PTR]] : !cir.complex<!s32i> -> !s32i
+// CIR: cir.store{{.*}} %[[REAL]], %[[REAL_ADDR]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: %[[W_ADDR:.*]] = alloca %struct.Wrapper, i64 1, align 4
+// LLVM: %[[REAL_ADDR:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[ELEM_PTR:.*]] = getelementptr %struct.Wrapper, ptr %[[W_ADDR]], i32 0, i32 0
+// LLVM: %[[TMP_ELEM_PTR:.*]] = load { i32, i32 }, ptr %[[ELEM_PTR]], align 4
+// LLVM: %[[REAL:.*]] = extractvalue { i32, i32 } %[[TMP_ELEM_PTR]], 0
+// LLVM: store i32 %[[REAL]], ptr %[[REAL_ADDR]], align 4
+
+// OGCG: %[[W_ADDR:.*]] = alloca %struct.Wrapper, align 4
+// OGCG: %[[REAL_ADDR:.*]] = alloca i32, align 4
+// OGCG: %[[ELEM_PTR:.*]] = getelementptr inbounds nuw %struct.Wrapper, ptr %[[W_ADDR]], i32 0, i32 0
+// OGCG: %[[REAL_PTR:.*]] = getelementptr inbounds nuw { i32, i32 }, ptr %[[ELEM_PTR]], i32 0, i32 0
+// OGCG: %[[REAL:.*]] = load i32, ptr %[[REAL_PTR]], align 4
+// OGCG: store i32 %[[REAL]], ptr %[[REAL_ADDR]], align 4
+
+struct Container {
+  static int _Complex c;
+};
+
+void foo32() {
+  Container con;
+  int r = __real__ con.c;
+}
+
+// CIR: %[[REAL_ADDR:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["r", init]
+// CIR: %[[ELEM_PTR:.*]] = cir.get_global @_ZN9Container1cE : !cir.ptr<!cir.complex<!s32i>>
+// CIR: %[[ELEM:.*]] = cir.load{{.*}} %[[ELEM_PTR]] : !cir.ptr<!cir.complex<!s32i>>, !cir.complex<!s32i>
+// CIR: %[[REAL:.*]] = cir.complex.real %[[ELEM]] : !cir.complex<!s32i> -> !s32i
+// CIR: cir.store{{.*}} %[[REAL]], %[[REAL_ADDR]] : !s32i, !cir.ptr<!s32i>
+
+// LLVM: %[[REAL_ADDR:.*]] = alloca i32, i64 1, align 4
+// LLVM: %[[ELEM:.*]] = load { i32, i32 }, ptr @_ZN9Container1cE, align 4
+// LLVM: %[[REAL:.*]] = extractvalue { i32, i32 } %[[ELEM]], 0
+// LLVM: store i32 %[[REAL]], ptr %[[REAL_ADDR]], align 4
+
+// OGCG: %[[REAL_ADDR:.*]] = alloca i32, align 4
+// OGCG: %[[REAL:.*]] = load i32, ptr @_ZN9Container1cE, align 4
+// OGCG: store i32 %[[REAL]], ptr %[[REAL_ADDR]], align 4
diff --git a/clang/test/CIR/CodeGen/destructors.cpp b/clang/test/CIR/CodeGen/destructors.cpp
index de7718f0998fc..fde0732a4352f 100644
--- a/clang/test/CIR/CodeGen/destructors.cpp
+++ b/clang/test/CIR/CodeGen/destructors.cpp
@@ -55,3 +55,102 @@ struct inline_destructor {
 // CIR-NOT: cir.func {{.*}}inline_destructor{{.*}}
 // LLVM-NOT: define {{.*}}inline_destructor{{.*}}
 // OGCG-NOT: define {{.*}}inline_destructor{{.*}}
+
+struct array_element {~array_element();};
+void test_array_destructor() {
+  array_element arr[5]{};
+}
+
+// CIR: cir.func dso_local @_Z21test_array_destructorv()
+// CIR:   %[[ARR:.*]] = cir.alloca !cir.array<!rec_array_element x 5>, !cir.ptr<!cir.array<!rec_array_element x 5>>, ["arr", init]
+// CIR:   %[[ARR_PTR:.*]] = cir.alloca !cir.ptr<!rec_array_element>, !cir.ptr<!cir.ptr<!rec_array_element>>, ["arrayinit.temp", init]
+// CIR:   %[[BEGIN:.*]] = cir.cast(array_to_ptrdecay, %[[ARR]] : !cir.ptr<!cir.array<!rec_array_element x 5>>)
+// CIR:   cir.store{{.*}} %[[BEGIN]], %[[ARR_PTR]]
+// CIR:   %[[FIVE:.*]] = cir.const #cir.int<5> : !s64i
+// CIR:   %[[ARR_END:.*]] = cir.ptr_stride(%[[BEGIN]] : !cir.ptr<!rec_array_element>, %[[FIVE]] : !s64i)
+// CIR:   cir.do {
+// CIR:     %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]]
+// CIR:     %[[ONE:.*]] = cir.const #cir.int<1> : !s64i
+// CIR:     %[[ARR_NEXT:.*]] = cir.ptr_stride(%[[ARR_CUR]] : !cir.ptr<!rec_array_element>, %[[ONE]] : !s64i)
+// CIR:     cir.store{{.*}} %[[ARR_NEXT]], %[[ARR_PTR]] : !cir.ptr<!rec_array_element>, !cir.ptr<!cir.ptr<!rec_array_element>>
+// CIR:     cir.yield
+// CIR:   } while {
+// CIR:     %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]]
+// CIR:     %[[CMP:.*]] = cir.cmp(ne, %[[ARR_CUR]], %[[ARR_END]])
+// CIR:     cir.condition(%[[CMP]])
+// CIR:   }
+// CIR:   %[[FOUR:.*]] = cir.const #cir.int<4> : !u64i
+// CIR:   %[[BEGIN:.*]] = cir.cast(array_to_ptrdecay, %[[ARR]] : !cir.ptr<!cir.array<!rec_array_element x 5>>)
+// CIR:   %[[END:.*]] = cir.ptr_stride(%[[BEGIN]] : !cir.ptr<!rec_array_element>, %[[FOUR]] : !u64i)
+// CIR:   %[[ARR_PTR:.*]] = cir.alloca !cir.ptr<!rec_array_element>, !cir.ptr<!cir.ptr<!rec_array_element>>, ["__array_idx"]
+// CIR:   cir.store %[[END]], %[[ARR_PTR]]
+// CIR:   cir.do {
+// CIR:     %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]]
+// CIR:     cir.call @_ZN13array_elementD1Ev(%[[ARR_CUR]]) nothrow : (!cir.ptr<!rec_array_element>) -> ()
+// CIR:     %[[NEG_ONE:.*]] = cir.const #cir.int<-1> : !s64i
+// CIR:     %[[ARR_NEXT:.*]] = cir.ptr_stride(%[[ARR_CUR]] : !cir.ptr<!rec_array_element>, %[[NEG_ONE]] : !s64i)
+// CIR:     cir.store %[[ARR_NEXT]], %[[ARR_PTR]]
+// CIR:     cir.yield
+// CIR:   } while {
+// CIR:     %[[ARR_CUR:.*]] = cir.load{{.*}} %[[ARR_PTR]]
+// CIR:     %[[CMP:.*]] = cir.cmp(ne, %[[ARR_CUR]], %[[BEGIN]])
+// CIR:     cir.condition(%[[CMP]])
+// CIR:   }
+
+// LLVM: define{{.*}} void @_Z21test_array_destructorv()
+// LLVM:   %[[ARR:.*]] = alloca [5 x %struct.array_element]
+// LLVM:   %[[TMP:.*]] = alloca ptr
+// LLVM:   %[[ARR_PTR:.*]] = getelementptr %struct.array_element, ptr %[[ARR]], i32 0
+// LLVM:   store ptr %[[ARR_PTR]], ptr %[[TMP]]
+// LLVM:   %[[END_PTR:.*]] = getelementptr %struct.array_element, ptr %[[ARR_PTR]], i64 5
+// LLVM:   br label %[[INIT_LOOP_BODY:.*]]
+// LLVM: [[INIT_LOOP_NEXT:.*]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[TMP]]
+// LLVM:   %[[CMP:.*]] = icmp ne ptr %[[CUR]], %[[END_PTR]]
+// LLVM:   br i1 %[[CMP]], label %[[INIT_LOOP_BODY]], label %[[INIT_LOOP_END:.*]]
+// LLVM: [[INIT_LOOP_BODY]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[TMP]]
+// LLVM:   %[[NEXT:.*]] = getelementptr %struct.array_element, ptr %[[CUR]], i64 1
+// LLVM:   store ptr %[[NEXT]], ptr %[[TMP]]
+// LLVM:   br label %[[INIT_LOOP_NEXT:.*]]
+// LLVM: [[INIT_LOOP_END]]:
+// LLVM:   %[[ARR_BEGIN:.*]] = getelementptr %struct.array_element, ptr %[[ARR]], i32 0
+// LLVM:   %[[ARR_END:.*]] = getelementptr %struct.array_element, ptr %[[ARR_BEGIN]], i64 4
+// LLVM:   %[[ARR_CUR:.*]] = alloca ptr
+// LLVM:   store ptr %[[ARR_END]], ptr %[[ARR_CUR]]
+// LLVM:   br label %[[DESTROY_LOOP_BODY:.*]]
+// LLVM: [[DESTROY_LOOP_NEXT:.*]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[ARR_CUR]]
+// LLVM:   %[[CMP:.*]] = icmp ne ptr %[[CUR]], %[[ARR_BEGIN]]
+// LLVM:   br i1 %[[CMP]], label %[[DESTROY_LOOP_BODY]], label %[[DESTROY_LOOP_END:.*]]
+// LLVM: [[DESTROY_LOOP_BODY]]:
+// LLVM:   %[[CUR:.*]] = load ptr, ptr %[[ARR_CUR]]
+// LLVM:   call void @_ZN13array_elementD1Ev(ptr %[[CUR]])
+// LLVM:   %[[PREV:.*]] = getelementptr %struct.array_element, ptr %[[CUR]], i64 -1
+// LLVM:   store ptr %[[PREV]], ptr %[[ARR_CUR]]
+// LLVM:   br label %[[DESTROY_LOOP_NEXT]]
+// LLVM: [[DESTROY_LOOP_END]]:
+// LLVM:   ret void
+
+// OGCG: define{{.*}} void @_Z21test_array_destructorv()
+// OGCG: entry:
+// OGCG:   %[[ARR:.*]] = alloca [5 x %struct.array_element]
+// OGCG:   %[[ARRAYINIT_END:.*]] = getelementptr inbounds %struct.array_element, ptr %[[ARR]], i64 5
+// OGCG:   br label %[[INIT_LOOP_BODY:.*]]
+// OGCG: [[INIT_LOOP_BODY]]:
+// OGCG:   %[[CUR:.*]] = phi ptr [ %[[ARR]], %entry ], [ %[[NEXT:.*]], %[[INIT_LOOP_BODY]] ]
+// OGCG:   %[[NEXT]] = getelementptr inbounds %struct.array_element, ptr %[[CUR]], i64 1
+// OGCG:   %[[CMP:.*]] = icmp eq ptr %[[NEXT]], %[[ARRAYINIT_END]]
+// OGCG:   br i1 %[[CMP]], label %[[INIT_LOOP_END:.*]], label %[[INIT_LOOP_BODY]]
+// OGCG: [[INIT_LOOP_END:.*]]:
+// OGCG:   %[[BEGIN:.*]] = getelementptr inbounds [5 x %struct.array_element], ptr %[[ARR]], i32 0, i32 0
+// OGCG:   %[[END:.*]] = getelementptr inbounds %struct.array_element, ptr %[[BEGIN]], i64 5
+// OGCG:   br label %[[DESTROY_LOOP_BODY:.*]]
+// OGCG: [[DESTROY_LOOP_BODY:.*]]:
+// OGCG:   %[[CUR:.*]] = phi ptr [ %[[END]], %[[INIT_LOOP_END]] ], [ %[[PREV:.*]], %[[DESTROY_LOOP_BODY]] ]
+// OGCG:   %[[PREV]] = getelementptr inbounds %struct.array_element, ptr %[[CUR]], i64 -1
+// OGCG:   call void @_ZN13array_elementD1Ev(ptr {{.*}} %[[PREV]])
+// OGCG:   %[[CMP:.*]] = icmp eq ptr %[[PREV]], %[[BEGIN]]
+// OGCG:   br i1 %[[CMP]], label %[[DESTROY_LOOP_END:.*]], label %[[DESTROY_LOOP_BODY]]
+// OGCG: [[DESTROY_LOOP_END:.*]]:
+// OGCG:   ret void
diff --git a/clang/test/CIR/CodeGen/function-to-pointer-decay.c b/clang/test/CIR/CodeGen/function-to-pointer-decay.c
new file mode 100644
index 0000000000000..507957a5a1a91
--- /dev/null
+++ b/clang/test/CIR/CodeGen/function-to-pointer-decay.c
@@ -0,0 +1,47 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+void f(void);
+
+void f1() {
+  (void (*)())f;
+}
+
+void f2() {
+  (*(void (*)(void))f)();
+}
+
+void test_lvalue_cast() {
+  (*(void (*)(int))f)(42);
+}
+
+// CIR-LABEL:   cir.func{{.*}} @f()
+// CIR:         cir.func{{.*}} @f1()
+// CIR:         cir.return{{.*}}
+
+// CIR-LABEL:   cir.func{{.*}} @f2()
+// CIR:         cir.call @f() : () -> ()
+
+// CIR-LABEL:   cir.func{{.*}} @test_lvalue_cast()
+// CIR:         %[[S0:.+]] = {{.*}}@f : !cir.ptr<!cir.func<()>>{{.*}}
+// CIR:         %[[S1:.+]] = cir.cast{{.*}}%[[S0]] : !cir.ptr<!cir.func<()>>{{.*}}
+// CIR:         %[[S2:.+]] = cir.const #cir.int<42> : !s32i
+// CIR:         cir.call %[[S1]](%[[S2]]) : (!cir.ptr<!cir.func<(!s32i)>>, !s32i) -> ()
+
+// LLVM-LABEL:  define{{.*}} void @f1()
+// LLVM:        ret void
+// LLVM:        define{{.*}} void @f2()
+// LLVM:        call void @f()
+// LLVM:        define{{.*}} void @test_lvalue_cast()
+// LLVM:        call void @f(i32 42)
+
+// OGCG-LABEL:  define{{.*}} void @f1()
+// OGCG:        ret void
+// OGCG:        define{{.*}} void @f2()
+// OGCG:        call void @f()
+// OGCG:        define{{.*}} void @test_lvalue_cast()
+// OGCG:        call void @f(i32 noundef 42)
diff --git a/clang/test/CIR/CodeGen/globals.cpp b/clang/test/CIR/CodeGen/globals.cpp
new file mode 100644
index 0000000000000..a3e16139a41a9
--- /dev/null
+++ b/clang/test/CIR/CodeGen/globals.cpp
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+
+// Should constant initialize global with constant address.
+int var = 1;
+int *constAddr = &var;
+
+// CIR: cir.global external @constAddr = #cir.global_view<@var> : !cir.ptr<!s32i>
+
+// LLVM: @constAddr = global ptr @var, align 8
+
+// OGCG: @constAddr = global ptr @var, align 8
+
+// Should constant initialize global with constant address.
+int f();
+int (*constFnAddr)() = f;
+
+// CIR: cir.global external @constFnAddr = #cir.global_view<@_Z1fv> : !cir.ptr<!cir.func<() -> !s32i>>
+
+// LLVM: @constFnAddr = global ptr @_Z1fv, align 8
+
+// OGCG: @constFnAddr = global ptr @_Z1fv, align 8
+
+int arr[4][16];
+int *constArrAddr = &arr[2][1];
+
+// CIR: cir.global external @constArrAddr = #cir.global_view<@arr, [2 : i32, 1 : i32]> : !cir.ptr<!s32i>
+
+// The 'inbounds' and 'nuw' flags are inferred by LLVM's constant folder. The
+// same flags show up at -O1 in OGCG.
+// LLVM: @constArrAddr = global ptr getelementptr inbounds nuw (i8, ptr @arr, i64 132), align 8
+
+// OGCG: @constArrAddr = global ptr getelementptr (i8, ptr @arr, i64 132), align 8
diff --git a/clang/test/CIR/CodeGen/goto.cpp b/clang/test/CIR/CodeGen/goto.cpp
new file mode 100644
index 0000000000000..48cb44ed0f478
--- /dev/null
+++ b/clang/test/CIR/CodeGen/goto.cpp
@@ -0,0 +1,305 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+
+int shouldNotGenBranchRet(int x) {
+  if (x > 5)
+    goto err;
+  return 0;
+err:
+  return -1;
+}
+// CIR:  cir.func dso_local @_Z21shouldNotGenBranchReti
+// CIR:    cir.if {{.*}} {
+// CIR:      cir.goto "err"
+// CIR:    }
+// CIR:    [[ZERO:%.*]] = cir.const #cir.int<0> : !s32i
+// CIR:    cir.store [[ZERO]], [[RETVAL:%.*]] : !s32i, !cir.ptr<!s32i>
+// CIR:    cir.br ^bb1
+// CIR:  ^bb1:
+// CIR:    [[RET:%.*]] = cir.load [[RETVAL]] : !cir.ptr<!s32i>, !s32i
+// CIR:    cir.return [[RET]] : !s32i
+// CIR:  ^bb2:
+// CIR:    cir.label "err"
+// CIR:    [[ONE:%.*]] = cir.const #cir.int<1> : !s32i
+// CIR:    [[MINUS:%.*]] = cir.unary(minus, [[ONE]]) nsw : !s32i, !s32i
+// CIR:    cir.store [[MINUS]], [[RETVAL]] : !s32i, !cir.ptr<!s32i>
+// CIR:    cir.br ^bb1
+
+// LLVM: define dso_local i32 @_Z21shouldNotGenBranchReti
+// LLVM:   [[COND:%.*]] = load i32, ptr {{.*}}, align 4
+// LLVM:   [[CMP:%.*]] = icmp sgt i32 [[COND]], 5
+// LLVM:   br i1 [[CMP]], label %[[IFTHEN:.*]], label %[[IFEND:.*]]
+// LLVM: [[IFTHEN]]:
+// LLVM:   br label %[[ERR:.*]]
+// LLVM: [[IFEND]]:
+// LLVM:   br label %[[BB9:.*]]
+// LLVM: [[BB9]]:
+// LLVM:   store i32 0, ptr %[[RETVAL:.*]], align 4
+// LLVM:   br label %[[BBRET:.*]]
+// LLVM: [[BBRET]]:
+// LLVM:   [[RET:%.*]] = load i32, ptr %[[RETVAL]], align 4
+// LLVM:   ret i32 [[RET]]
+// LLVM: [[ERR]]:
+// LLVM:   store i32 -1, ptr %[[RETVAL]], align 4
+// LLVM:   br label %10
+
+// OGCG: define dso_local noundef i32 @_Z21shouldNotGenBranchReti
+// OGCG: if.then:
+// OGCG:   br label %err
+// OGCG: if.end:
+// OGCG:   br label %return
+// OGCG: err:
+// OGCG:   br label %return
+// OGCG: return:
+
+int shouldGenBranch(int x) {
+  if (x > 5)
+    goto err;
+  x++;
+err:
+  return -1;
+}
+// CIR:  cir.func dso_local @_Z15shouldGenBranchi
+// CIR:    cir.if {{.*}} {
+// CIR:      cir.goto "err"
+// CIR:    }
+// CIR:    cir.br ^bb1
+// CIR:  ^bb1:
+// CIR:    cir.label "err"
+
+// LLVM: define dso_local i32 @_Z15shouldGenBranchi
+// LLVM:   br i1 [[CMP:%.*]], label %[[IFTHEN:.*]], label %[[IFEND:.*]]
+// LLVM: [[IFTHEN]]:
+// LLVM:   br label %[[ERR:.*]]
+// LLVM: [[IFEND]]:
+// LLVM:   br label %[[BB9:.*]]
+// LLVM: [[BB9]]:
+// LLVM:   br label %[[ERR]]
+// LLVM: [[ERR]]:
+// LLVM:   ret i32 [[RET:%.*]]
+
+// OGCG: define dso_local noundef i32 @_Z15shouldGenBranchi
+// OGCG: if.then:
+// OGCG:   br label %err
+// OGCG: if.end:
+// OGCG:   br label %err
+// OGCG: err:
+// OGCG:   ret
+
+void severalLabelsInARow(int a) {
+  int b = a;
+  goto end1;
+  b = b + 1;
+  goto end2;
+end1:
+end2:
+  b = b + 2;
+}
+// CIR:  cir.func dso_local @_Z19severalLabelsInARowi
+// CIR:    cir.goto "end1"
+// CIR:  ^bb[[#BLK1:]]
+// CIR:    cir.goto "end2"
+// CIR:  ^bb[[#BLK2:]]:
+// CIR:    cir.label "end1"
+// CIR:    cir.br ^bb[[#BLK3:]]
+// CIR:  ^bb[[#BLK3]]:
+// CIR:    cir.label "end2"
+
+// LLVM: define dso_local void @_Z19severalLabelsInARowi
+// LLVM:   br label %[[END1:.*]]
+// LLVM: [[UNRE:.*]]:                                                ; No predecessors!
+// LLVM:   br label %[[END2:.*]]
+// LLVM: [[END1]]:
+// LLVM:   br label %[[END2]]
+// LLVM: [[END2]]:
+// LLVM:   ret
+
+// OGCG: define dso_local void @_Z19severalLabelsInARowi
+// OGCG:   br label %end1
+// OGCG: end1:
+// OGCG:   br label %end2
+// OGCG: end2:
+// OGCG:   ret
+
+void severalGotosInARow(int a) {
+  int b = a;
+  goto end;
+  goto end;
+end:
+  b = b + 2;
+}
+// CIR:  cir.func dso_local @_Z18severalGotosInARowi
+// CIR:    cir.goto "end"
+// CIR:  ^bb[[#BLK1:]]:
+// CIR:    cir.goto "end"
+// CIR:  ^bb[[#BLK2:]]:
+// CIR:    cir.label "end"
+
+// LLVM: define dso_local void @_Z18severalGotosInARowi
+// LLVM:   br label %[[END:.*]]
+// LLVM: [[UNRE:.*]]:                                                ; No predecessors!
+// LLVM:   br label %[[END]]
+// LLVM: [[END]]:
+// LLVM:   ret void
+
+// OGCG: define dso_local void @_Z18severalGotosInARowi(i32 noundef %a) #0 {
+// OGCG:   br label %end
+// OGCG: end:
+// OGCG:   ret void
+
+extern "C" void action1();
+extern "C" void action2();
+extern "C" void multiple_non_case(int v) {
+  switch (v) {
+    default:
+        action1();
+      l2:
+        action2();
+        break;
+  }
+}
+
+// CIR: cir.func dso_local @multiple_non_case
+// CIR: cir.switch
+// CIR: cir.case(default, []) {
+// CIR: cir.call @action1()
+// CIR: cir.br ^[[BB1:[a-zA-Z0-9]+]]
+// CIR: ^[[BB1]]:
+// CIR: cir.label
+// CIR: cir.call @action2()
+// CIR: cir.break
+
+// LLVM: define dso_local void @multiple_non_case
+// LLVM: [[SWDEFAULT:.*]]:
+// LLVM:   call void @action1()
+// LLVM:   br label %[[L2:.*]]
+// LLVM: [[L2]]:
+// LLVM:   call void @action2()
+// LLVM:   br label %[[BREAK:.*]]
+
+// OGCG: define dso_local void @multiple_non_case
+// OGCG: sw.default:
+// OGCG:   call void @action1()
+// OGCG:   br label %l2
+// OGCG: l2:
+// OGCG:   call void @action2()
+// OGCG:   br label [[BREAK:%.*]]
+
+extern "C" void case_follow_label(int v) {
+  switch (v) {
+    case 1:
+    label:
+    case 2:
+      action1();
+      break;
+    default:
+      action2();
+      goto label;
+  }
+}
+
+// CIR: cir.func dso_local @case_follow_label
+// CIR: cir.switch
+// CIR: cir.case(equal, [#cir.int<1> : !s32i]) {
+// CIR:   cir.label "label"
+// CIR: cir.case(equal, [#cir.int<2> : !s32i]) {
+// CIR:   cir.call @action1()
+// CIR:   cir.break
+// CIR: cir.case(default, []) {
+// CIR:   cir.call @action2()
+// CIR:   cir.goto "label"
+
+// LLVM: define dso_local void @case_follow_label
+// LLVM:  switch i32 {{.*}}, label %[[SWDEFAULT:.*]] [
+// LLVM:    i32 1, label %[[LABEL:.*]]
+// LLVM:    i32 2, label %[[CASE2:.*]]
+// LLVM:  ]
+// LLVM: [[LABEL]]:
+// LLVM:   br label %[[CASE2]]
+// LLVM: [[CASE2]]:
+// LLVM:   call void @action1()
+// LLVM:   br label %[[BREAK:.*]]
+// LLVM: [[BREAK]]:
+// LLVM:   br label %[[END:.*]]
+// LLVM: [[SWDEFAULT]]:
+// LLVM:   call void @action2()
+// LLVM:   br label %[[LABEL]]
+// LLVM: [[END]]:
+// LLVM:   br label %[[RET:.*]]
+// LLVM: [[RET]]:
+// LLVM:   ret void
+
+// OGCG: define dso_local void @case_follow_label
+// OGCG: sw.bb:
+// OGCG:   br label %label
+// OGCG: label:
+// OGCG:   br label %sw.bb1
+// OGCG: sw.bb1:
+// OGCG:   call void @action1()
+// OGCG:   br label %sw.epilog
+// OGCG: sw.default:
+// OGCG:   call void @action2()
+// OGCG:   br label %label
+// OGCG: sw.epilog:
+// OGCG:   ret void
+
+extern "C" void default_follow_label(int v) {
+  switch (v) {
+    case 1:
+    case 2:
+      action1();
+      break;
+    label:
+    default:
+      action2();
+      goto label;
+  }
+}
+
+// CIR: cir.func dso_local @default_follow_label
+// CIR: cir.switch
+// CIR: cir.case(equal, [#cir.int<1> : !s32i]) {
+// CIR:   cir.yield
+// CIR: cir.case(equal, [#cir.int<2> : !s32i]) {
+// CIR:   cir.call @action1()
+// CIR:   cir.break
+// CIR:   cir.label "label"
+// CIR: cir.case(default, []) {
+// CIR:   cir.call @action2()
+// CIR:   cir.goto "label"
+
+// LLVM: define dso_local void @default_follow_label
+// LLVM: [[CASE1:.*]]:
+// LLVM:   br label %[[BB8:.*]]
+// LLVM: [[BB8]]:
+// LLVM:   br label %[[CASE2:.*]]
+// LLVM: [[CASE2]]:
+// LLVM:   call void @action1()
+// LLVM:   br label %[[BREAK:.*]]
+// LLVM: [[LABEL:.*]]:
+// LLVM:   br label %[[SWDEFAULT:.*]]
+// LLVM: [[SWDEFAULT]]:
+// LLVM:   call void @action2()
+// LLVM:   br label %[[BB9:.*]]
+// LLVM: [[BB9]]:
+// LLVM:   br label %[[LABEL]]
+// LLVM: [[BREAK]]:
+// LLVM:   br label %[[RET:.*]]
+// LLVM: [[RET]]:
+// LLVM:   ret void
+
+// OGCG: define dso_local void @default_follow_label
+// OGCG: sw.bb:
+// OGCG:   call void @action1()
+// OGCG:   br label %sw.epilog
+// OGCG: label:
+// OGCG:   br label %sw.default
+// OGCG: sw.default:
+// OGCG:   call void @action2()
+// OGCG:   br label %label
+// OGCG: sw.epilog:
+// OGCG:   ret void
diff --git a/clang/test/CIR/CodeGen/inline-asm.c b/clang/test/CIR/CodeGen/inline-asm.c
new file mode 100644
index 0000000000000..fc959f9326876
--- /dev/null
+++ b/clang/test/CIR/CodeGen/inline-asm.c
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
+
+void f1() {
+  // CIR: cir.asm(x86_att, 
+  // CIR:   out = [],
+  // CIR:   in = [],
+  // CIR:   in_out = [],
+  // CIR:   {"" "~{dirflag},~{fpsr},~{flags}"}) side_effects
+  // LLVM: call void asm sideeffect "", "~{dirflag},~{fpsr},~{flags}"()
+  __asm__ volatile("" : : : );
+}
+
+void f2() {
+  // CIR: cir.asm(x86_att,
+  // CIR:   out = [],
+  // CIR:   in = [],
+  // CIR:   in_out = [],
+  // CIR:   {"nop" "~{dirflag},~{fpsr},~{flags}"}) side_effects
+  // LLVM: call void asm sideeffect "nop", "~{dirflag},~{fpsr},~{flags}"()
+  __asm__ volatile("nop" : : : );
+}
diff --git a/clang/test/CIR/CodeGen/label.c b/clang/test/CIR/CodeGen/label.c
new file mode 100644
index 0000000000000..a050094de678b
--- /dev/null
+++ b/clang/test/CIR/CodeGen/label.c
@@ -0,0 +1,186 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+
+void label() {
+labelA:
+  return;
+}
+
+// CIR:  cir.func no_proto dso_local @label
+// CIR:    cir.label "labelA"
+// CIR:    cir.return
+
+// LLVM:define dso_local void @label
+// LLVM:  ret void
+
+// OGCG: define dso_local void @label
+// OGCG:   br label %labelA
+// OGCG: labelA:
+// OGCG:   ret void
+
+void multiple_labels() {
+labelB:
+labelC:
+  return;
+}
+
+// CIR:  cir.func no_proto dso_local @multiple_labels
+// CIR:    cir.label "labelB"
+// CIR:    cir.br ^bb1
+// CIR:  ^bb1:  // pred: ^bb0
+// CIR:    cir.label "labelC"
+// CIR:    cir.return
+
+// LLVM: define dso_local void @multiple_labels()
+// LLVM:   br label %1
+// LLVM: 1:
+// LLVM:   ret void
+
+// OGCG: define dso_local void @multiple_labels
+// OGCG:   br label %labelB
+// OGCG: labelB:
+// OGCG:   br label %labelC
+// OGCG: labelC:
+// OGCG:   ret void
+
+void label_in_if(int cond) {
+  if (cond) {
+labelD:
+    cond++;
+  }
+}
+
+// CIR:  cir.func dso_local @label_in_if
+// CIR:      cir.if {{.*}} {
+// CIR:        cir.label "labelD"
+// CIR:        [[LOAD:%.*]] = cir.load align(4) [[COND:%.*]] : !cir.ptr<!s32i>, !s32i
+// CIR:        [[INC:%.*]] = cir.unary(inc, %3) nsw : !s32i, !s32i
+// CIR:        cir.store align(4) [[INC]], [[COND]] : !s32i, !cir.ptr<!s32i>
+// CIR:      }
+// CIR:    cir.return
+
+// LLVM: define dso_local void @label_in_if
+// LLVM:   br label %3
+// LLVM: 3:
+// LLVM:   [[LOAD:%.*]] = load i32, ptr [[COND:%.*]], align 4
+// LLVM:   [[CMP:%.*]] = icmp ne i32 [[LOAD]], 0
+// LLVM:   br i1 [[CMP]], label %6, label %9
+// LLVM: 6:
+// LLVM:   [[LOAD2:%.*]] = load i32, ptr [[COND]], align 4
+// LLVM:   [[ADD1:%.*]] = add nsw i32 [[LOAD2]], 1
+// LLVM:   store i32 [[ADD1]], ptr [[COND]], align 4
+// LLVM:   br label %9
+// LLVM: 9:
+// LLVM:   br label %10
+// LLVM: 10:
+// LLVM:  ret void
+
+// OGCG: define dso_local void @label_in_if
+// OGCG: if.then:
+// OGCG:   br label %labelD
+// OGCG: labelD:
+// OGCG:   [[LOAD:%.*]] = load i32, ptr [[COND:%.*]], align 4
+// OGCG:   [[INC:%.*]] = add nsw i32 %1, 1
+// OGCG:   store i32 [[INC]], ptr [[COND]], align 4
+// OGCG:   br label %if.end
+// OGCG: if.end:
+// OGCG:   ret void
+
+void after_return() {
+  return;
+  label:
+}
+
+// CIR:  cir.func no_proto dso_local @after_return
+// CIR:    cir.br ^bb1
+// CIR:  ^bb1:  // 2 preds: ^bb0, ^bb2
+// CIR:    cir.return
+// CIR:  ^bb2:  // no predecessors
+// CIR:    cir.label "label"
+// CIR:    cir.br ^bb1
+
+// LLVM: define dso_local void @after_return
+// LLVM:   br label %1
+// LLVM: 1:
+// LLVM:   ret void
+// LLVM: 2:
+// LLVM:   br label %1
+
+// OGCG: define dso_local void @after_return
+// OGCG:   br label %label
+// OGCG: label:
+// OGCG:   ret void
+
+
+void after_unreachable() {
+  __builtin_unreachable();
+  label:
+}
+
+// CIR:  cir.func no_proto dso_local @after_unreachable
+// CIR:    cir.unreachable
+// CIR:  ^bb1:
+// CIR:    cir.label "label"
+// CIR:    cir.return
+
+// LLVM: define dso_local void @after_unreachable
+// LLVM:   unreachable
+// LLVM: 1:
+// LLVM:   ret void
+
+// OGCG: define dso_local void @after_unreachable
+// OGCG:   unreachable
+// OGCG: label:
+// OGCG:   ret void
+
+void labelWithoutMatch() {
+end:
+  return;
+}
+// CIR:  cir.func no_proto dso_local @labelWithoutMatch
+// CIR:    cir.label "end"
+// CIR:    cir.return
+// CIR:  }
+
+// LLVM: define dso_local void @labelWithoutMatch
+// LLVM:   ret void
+
+// OGCG: define dso_local void @labelWithoutMatch
+// OGCG:   br label %end
+// OGCG: end:
+// OGCG:   ret void
+
+struct S {};
+struct S get();
+void bar(struct S);
+
+void foo() {
+  {
+    label:
+      bar(get());
+  }
+}
+
+// CIR: cir.func no_proto dso_local @foo
+// CIR:   cir.scope {
+// CIR:     cir.label "label"
+// CIR:     %0 = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["agg.tmp0"]
+
+// LLVM:define dso_local void @foo() {
+// LLVM:  [[ALLOC:%.*]] = alloca %struct.S, i64 1, align 1
+// LLVM:  br label %2
+// LLVM:2:
+// LLVM:  [[CALL:%.*]] = call %struct.S @get()
+// LLVM:  store %struct.S [[CALL]], ptr [[ALLOC]], align 1
+// LLVM:  [[LOAD:%.*]] = load %struct.S, ptr [[ALLOC]], align 1
+// LLVM:  call void @bar(%struct.S [[LOAD]])
+
+// OGCG: define dso_local void @foo()
+// OGCG:   %agg.tmp = alloca %struct.S, align 1
+// OGCG:   %undef.agg.tmp = alloca %struct.S, align 1
+// OGCG:   br label %label
+// OGCG: label:
diff --git a/clang/test/CIR/CodeGen/lang-c-cpp.cpp b/clang/test/CIR/CodeGen/lang-c-cpp.cpp
new file mode 100644
index 0000000000000..e126932104de2
--- /dev/null
+++ b/clang/test/CIR/CodeGen/lang-c-cpp.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cpp.cir
+// RUN: FileCheck --check-prefix=CIR-CPP --input-file=%t.cpp.cir %s
+// RUN: %clang_cc1 -x c -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.c.cir
+// RUN: FileCheck --check-prefix=CIR-C --input-file=%t.c.cir %s
+
+// CIR-CPP: module attributes {{{.*}}cir.lang = #cir.lang<cxx>{{.*}}}
+// CIR-C: module attributes {{{.*}}cir.lang = #cir.lang<c>{{.*}}}
+
+int main() {
+  return 0;
+}
diff --git a/clang/test/CIR/CodeGen/module-asm.c b/clang/test/CIR/CodeGen/module-asm.c
new file mode 100644
index 0000000000000..e6cec5e0ee948
--- /dev/null
+++ b/clang/test/CIR/CodeGen/module-asm.c
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-cir %s -o %t.cir 
+// RUN: FileCheck --input-file=%t.cir %s
+
+// CHECK:  cir.module_asm = [".globl bar", ".globl foo"]
+__asm (".globl bar");
+__asm (".globl foo");
diff --git a/clang/test/CIR/CodeGen/statement-exprs.c b/clang/test/CIR/CodeGen/statement-exprs.c
new file mode 100644
index 0000000000000..927a868336b59
--- /dev/null
+++ b/clang/test/CIR/CodeGen/statement-exprs.c
@@ -0,0 +1,274 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+
+int f19(void) {
+  return ({ 3;;4;; });
+}
+
+// CIR: cir.func dso_local @f19() -> !s32i
+// CIR:   %[[RETVAL:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[TMP:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["tmp"]
+// CIR:   cir.scope {
+// CIR:     %[[C3:.+]] = cir.const #cir.int<3> : !s32i
+// CIR:     %[[C4:.+]] = cir.const #cir.int<4> : !s32i
+// CIR:     cir.store {{.*}} %[[C4]], %[[TMP]] : !s32i, !cir.ptr<!s32i>
+// CIR:   }
+// CIR:   %[[TMP_VAL:.+]] = cir.load {{.*}} %[[TMP]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.store %[[TMP_VAL]], %[[RETVAL]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[RES:.+]] = cir.load %[[RETVAL]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[RES]] : !s32i
+
+// LLVM: define dso_local i32 @f19()
+// LLVM:   %[[VAR1:.+]] = alloca i32, i64 1
+// LLVM:   %[[VAR2:.+]] = alloca i32, i64 1
+// LLVM:   br label %[[LBL3:.+]]
+// LLVM: [[LBL3]]:
+// LLVM:     store i32 4, ptr %[[VAR2]]
+// LLVM:     br label %[[LBL4:.+]]
+// LLVM: [[LBL4]]:
+// LLVM:     %[[V1:.+]] = load i32, ptr %[[VAR2]]
+// LLVM:     store i32 %[[V1]], ptr %[[VAR1]]
+// LLVM:     %[[RES:.+]] = load i32, ptr %[[VAR1]]
+// LLVM:     ret i32 %[[RES]]
+
+// OGCG: define dso_local i32 @f19()
+// OGCG: entry:
+// OGCG:   %[[TMP:.+]] = alloca i32
+// OGCG:   store i32 4, ptr %[[TMP]]
+// OGCG:   %[[TMP_VAL:.+]] = load i32, ptr %[[TMP]]
+// OGCG:   ret i32 %[[TMP_VAL]]
+
+
+int nested(void) {
+  ({123;});
+  {
+    int bar = 987;
+    return ({ ({ int asdf = 123; asdf; }); ({9999;}); });
+  }
+}
+
+// CIR: cir.func dso_local @nested() -> !s32i
+// CIR:   %[[RETVAL:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[TMP_OUTER:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["tmp"]
+// CIR:   cir.scope {
+// CIR:     %[[C123_OUTER:.+]] = cir.const #cir.int<123> : !s32i
+// CIR:     cir.store {{.*}} %[[C123_OUTER]], %[[TMP_OUTER]] : !s32i, !cir.ptr<!s32i>
+// CIR:   }
+// CIR:   %[[LOAD_TMP_OUTER:.+]] = cir.load {{.*}} %[[TMP_OUTER]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.scope {
+// CIR:     %[[BAR:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["bar", init]
+// CIR:     %[[TMP_BARRET:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["tmp"]
+// CIR:     %[[C987:.+]] = cir.const #cir.int<987> : !s32i
+// CIR:     cir.store {{.*}} %[[C987]], %[[BAR]] : !s32i, !cir.ptr<!s32i>
+// CIR:     cir.scope {
+// CIR:       %[[TMP1:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["tmp"]
+// CIR:       %[[TMP2:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["tmp"]
+// CIR:       cir.scope {
+// CIR:         %[[ASDF:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["asdf", init]
+// CIR:         %[[C123_INNER:.+]] = cir.const #cir.int<123> : !s32i
+// CIR:         cir.store {{.*}} %[[C123_INNER]], %[[ASDF]] : !s32i, !cir.ptr<!s32i>
+// CIR:         %[[LOAD_ASDF:.+]] = cir.load {{.*}} %[[ASDF]] : !cir.ptr<!s32i>, !s32i
+// CIR:         cir.store {{.*}} %[[LOAD_ASDF]], %[[TMP1]] : !s32i, !cir.ptr<!s32i>
+// CIR:       }
+// CIR:       %[[V1:.+]] = cir.load {{.*}} %[[TMP1]] : !cir.ptr<!s32i>, !s32i
+// CIR:       cir.scope {
+// CIR:         %[[C9999:.+]] = cir.const #cir.int<9999> : !s32i
+// CIR:         cir.store {{.*}} %[[C9999]], %[[TMP2]] : !s32i, !cir.ptr<!s32i>
+// CIR:       }
+// CIR:       %[[V2:.+]] = cir.load {{.*}} %[[TMP2]] : !cir.ptr<!s32i>, !s32i
+// CIR:       cir.store {{.*}} %[[V2]], %[[TMP_BARRET]] : !s32i, !cir.ptr<!s32i>
+// CIR:     }
+// CIR:     %[[BARRET_VAL:.+]] = cir.load {{.*}} %[[TMP_BARRET]] : !cir.ptr<!s32i>, !s32i
+// CIR:     cir.store %[[BARRET_VAL]], %[[RETVAL]] : !s32i, !cir.ptr<!s32i>
+// CIR:     %[[RES:.+]] = cir.load %[[RETVAL]] : !cir.ptr<!s32i>, !s32i
+// CIR:     cir.return %[[RES]] : !s32i
+// CIR:   }
+// CIR:   %[[FINAL_RES:.+]] = cir.load %[[RETVAL]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[FINAL_RES]] : !s32i
+
+// LLVM: define dso_local i32 @nested()
+// LLVM:   %[[VAR1:.+]] = alloca i32, i64 1
+// LLVM:   %[[VAR2:.+]] = alloca i32, i64 1
+// LLVM:   %[[VAR3:.+]] = alloca i32, i64 1
+// LLVM:   %[[VAR4:.+]] = alloca i32, i64 1
+// LLVM:   %[[VAR5:.+]] = alloca i32, i64 1
+// LLVM:   %[[VAR6:.+]] = alloca i32, i64 1
+// LLVM:   %[[VAR7:.+]] = alloca i32, i64 1
+// LLVM:   br label %[[LBL8:.+]]
+// LLVM: [[LBL8]]:
+// LLVM:     store i32 123, ptr %[[VAR7]]
+// LLVM:     br label %[[LBL9:.+]]
+// LLVM: [[LBL9]]:
+// LLVM:     br label %[[LBL10:.+]]
+// LLVM: [[LBL10]]:
+// LLVM:     store i32 987, ptr %[[VAR1]]
+// LLVM:     br label %[[LBL11:.+]]
+// LLVM: [[LBL11]]:
+// LLVM:     br label %[[LBL12:.+]]
+// LLVM: [[LBL12]]:
+// LLVM:     store i32 123, ptr %[[VAR5]]
+// LLVM:     %[[V1:.+]] = load i32, ptr %[[VAR5]]
+// LLVM:     store i32 %[[V1]], ptr %[[VAR3]]
+// LLVM:     br label %[[LBL14:.+]]
+// LLVM: [[LBL14]]:
+// LLVM:     br label %[[LBL15:.+]]
+// LLVM: [[LBL15]]:
+// LLVM:     store i32 9999, ptr %[[VAR4]]
+// LLVM:     br label %[[LBL16:.+]]
+// LLVM: [[LBL16]]:
+// LLVM:     %[[V2:.+]] = load i32, ptr %[[VAR4]]
+// LLVM:     store i32 %[[V2]], ptr %[[VAR2]]
+// LLVM:     br label %[[LBL18:.+]]
+// LLVM: [[LBL18]]:
+// LLVM:     %[[V3:.+]] = load i32, ptr %[[VAR2]]
+// LLVM:     store i32 %[[V3]], ptr %[[VAR6]]
+// LLVM:     %[[RES:.+]] = load i32, ptr %[[VAR6]]
+// LLVM:     ret i32 %[[RES]]
+
+// OGCG: define dso_local i32 @nested()
+// OGCG: entry:
+// OGCG:   %[[TMP_OUTER:.+]] = alloca i32
+// OGCG:   %[[BAR:.+]] = alloca i32
+// OGCG:   %[[ASDF:.+]] = alloca i32
+// OGCG:   %[[TMP1:.+]] = alloca i32
+// OGCG:   %[[TMP2:.+]] = alloca i32
+// OGCG:   %[[TMP3:.+]] = alloca i32
+// OGCG:   store i32 123, ptr %[[TMP_OUTER]]
+// OGCG:   %[[OUTER_VAL:.+]] = load i32, ptr %[[TMP_OUTER]]
+// OGCG:   store i32 987, ptr %[[BAR]]
+// OGCG:   store i32 123, ptr %[[ASDF]]
+// OGCG:   %[[ASDF_VAL:.+]] = load i32, ptr %[[ASDF]]
+// OGCG:   store i32 %[[ASDF_VAL]], ptr %[[TMP1]]
+// OGCG:   %[[TMP1_VAL:.+]] = load i32, ptr %[[TMP1]]
+// OGCG:   store i32 9999, ptr %[[TMP3]]
+// OGCG:   %[[TMP3_VAL:.+]] = load i32, ptr %[[TMP3]]
+// OGCG:   store i32 %[[TMP3_VAL]], ptr %[[TMP2]]
+// OGCG:   %[[RES:.+]] = load i32, ptr %[[TMP2]]
+// OGCG:   ret i32 %[[RES]]
+
+void empty() {
+  return ({;;;;});
+}
+
+// CIR: cir.func no_proto dso_local @empty()
+// CIR-NEXT:   cir.return
+
+// LLVM: define dso_local void @empty()
+// LLVM:   ret void
+// LLVM: }
+
+// OGCG: define dso_local void @empty()
+// OGCG:   ret void
+// OGCG: }
+
+void empty2() { ({ }); }
+
+// CIR: @empty2
+// CIR-NEXT: cir.return
+
+// LLVM: @empty2()
+// LLVM:   ret void
+// LLVM: }
+
+// OGCG: @empty2()
+// OGCG:   ret void
+// OGCG: }
+
+
+// Yields an out-of-scope scalar.
+void test2() { ({int x = 3; x; }); }
+// CIR: @test2
+// CIR: %[[RETVAL:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>
+// CIR: cir.scope {
+// CIR:   %[[VAR:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init]
+//          [...]
+// CIR:   %[[TMP:.+]] = cir.load{{.*}} %[[VAR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.store{{.*}} %[[TMP]], %[[RETVAL]] : !s32i, !cir.ptr<!s32i>
+// CIR: }
+// CIR: %{{.+}} = cir.load{{.*}} %[[RETVAL]] : !cir.ptr<!s32i>, !s32i
+
+// LLVM: define dso_local void @test2()
+// LLVM:   %[[X:.+]] = alloca i32, i64 1
+// LLVM:   %[[TMP:.+]] = alloca i32, i64 1
+// LLVM:   br label %[[LBL3:.+]]
+// LLVM: [[LBL3]]:
+// LLVM:     store i32 3, ptr %[[X]]
+// LLVM:     %[[X_VAL:.+]] = load i32, ptr %[[X]]
+// LLVM:     store i32 %[[X_VAL]], ptr %[[TMP]]
+// LLVM:     br label %[[LBL5:.+]]
+// LLVM: [[LBL5]]:
+// LLVM:     ret void
+
+// OGCG: define dso_local void @test2()
+// OGCG: entry:
+// OGCG:   %[[X:.+]] = alloca i32
+// OGCG:   %[[TMP:.+]] = alloca i32
+// OGCG:   store i32 3, ptr %[[X]]
+// OGCG:   %[[X_VAL:.+]] = load i32, ptr %[[X]]
+// OGCG:   store i32 %[[X_VAL]], ptr %[[TMP]]
+// OGCG:   %[[TMP_VAL:.+]] = load i32, ptr %[[TMP]]
+// OGCG:   ret void
+
+// Yields an aggregate.
+struct S { int x; };
+int test3() { return ({ struct S s = {1}; s; }).x; }
+// CIR: cir.func no_proto dso_local @test3() -> !s32i
+// CIR:   %[[RETVAL:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[YIELDVAL:.+]] = cir.scope {
+// CIR:     %[[REF_TMP0:.+]] = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["ref.tmp0"]
+// CIR:     %[[TMP:.+]] = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["tmp"]
+// CIR:     cir.scope {
+// CIR:       %[[S:.+]] = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["s", init]
+// CIR:       %[[GEP_X_S:.+]] = cir.get_member %[[S]][0] {name = "x"} : !cir.ptr<!rec_S> -> !cir.ptr<!s32i>
+// CIR:       %[[C1:.+]] = cir.const #cir.int<1> : !s32i
+// CIR:       cir.store {{.*}} %[[C1]], %[[GEP_X_S]] : !s32i, !cir.ptr<!s32i>
+// CIR:     }
+// CIR:     %[[GEP_X_TMP:.+]] = cir.get_member %[[REF_TMP0]][0] {name = "x"} : !cir.ptr<!rec_S> -> !cir.ptr<!s32i>
+// CIR:     %[[XVAL:.+]] = cir.load {{.*}} %[[GEP_X_TMP]] : !cir.ptr<!s32i>, !s32i
+// CIR:     cir.yield %[[XVAL]] : !s32i
+// CIR:   } : !s32i
+// CIR:   cir.store %[[YIELDVAL]], %[[RETVAL]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[RES:.+]] = cir.load %[[RETVAL]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[RES]] : !s32i
+
+// LLVM: define dso_local i32 @test3()
+// LLVM:   %[[VAR1:.+]] = alloca %struct.S, i64 1
+// LLVM:   %[[VAR2:.+]] = alloca %struct.S, i64 1
+// LLVM:   %[[VAR3:.+]] = alloca %struct.S, i64 1
+// LLVM:   %[[VAR4:.+]] = alloca i32, i64 1
+// LLVM:   br label %[[LBL5:.+]]
+// LLVM: [[LBL5]]:
+// LLVM:     br label %[[LBL6:.+]]
+// LLVM: [[LBL6]]:
+// LLVM:     %[[GEP_S:.+]] = getelementptr %struct.S, ptr %[[VAR3]], i32 0, i32 0
+// LLVM:     store i32 1, ptr %[[GEP_S]]
+// LLVM:     br label %[[LBL8:.+]]
+// LLVM: [[LBL8]]:
+// LLVM:     %[[GEP_VAR1:.+]] = getelementptr %struct.S, ptr %[[VAR1]], i32 0, i32 0
+// LLVM:     %[[LOAD_X:.+]] = load i32, ptr %[[GEP_VAR1]]
+// LLVM:     br label %[[LBL11:.+]]
+// LLVM: [[LBL11]]:
+// LLVM:     %[[PHI:.+]] = phi i32 [ %[[LOAD_X]], %[[LBL8]] ]
+// LLVM:     store i32 %[[PHI]], ptr %[[VAR4]]
+// LLVM:     %[[RES:.+]] = load i32, ptr %[[VAR4]]
+// LLVM:     ret i32 %[[RES]]
+
+// OGCG: define dso_local i32 @test3()
+// OGCG: entry:
+// OGCG:   %[[REF_TMP:.+]] = alloca %struct.S
+// OGCG:   %[[S:.+]] = alloca %struct.S
+// OGCG:   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[S]], ptr align 4 @__const.test3.s, i64 4, i1 false)
+// OGCG:   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %[[REF_TMP]], ptr align 4 %[[S]], i64 4, i1 false)
+// OGCG:   %[[GEP:.+]] = getelementptr inbounds nuw %struct.S, ptr %[[REF_TMP]], i32 0, i32 0
+// OGCG:   %[[XVAL:.+]] = load i32, ptr %[[GEP]]
+// OGCG:   ret i32 %[[XVAL]]
+
+// Expression is wrapped in an expression attribute (just ensure it does not crash).
+void test4(int x) { ({[[gsl::suppress("foo")]] x;}); }
+// CIR: @test4
+// LLVM: @test4
+// OGCG: @test4
diff --git a/clang/test/CIR/CodeGen/static-vars.cpp b/clang/test/CIR/CodeGen/static-vars.cpp
index d949936f6bff9..4f22fc7abc541 100644
--- a/clang/test/CIR/CodeGen/static-vars.cpp
+++ b/clang/test/CIR/CodeGen/static-vars.cpp
@@ -2,6 +2,37 @@
 // RUN: FileCheck --input-file=%t.cir %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t1.ll
 // RUN: FileCheck --check-prefix=LLVM --input-file=%t1.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t1.ll
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t1.ll %s
+
+template<typename T>
+struct implicitly_instantiated {
+  static T member;
+};
+
+template<typename T>
+T implicitly_instantiated<T>::member = 12345;
+
+int use_implicitly_instantiated() {
+  return implicitly_instantiated<int>::member;
+}
+
+// CHECK-DAG: cir.global linkonce_odr comdat @_ZN23implicitly_instantiatedIiE6memberE = #cir.int<12345> : !s32i
+// LLVM-DAG: @_ZN23implicitly_instantiatedIiE6memberE = linkonce_odr global i32 12345, comdat
+// OGCG-DAG: @_ZN23implicitly_instantiatedIiE6memberE = linkonce_odr global i32 12345, comdat
+
+template<typename T>
+struct explicitly_instantiated {
+    static T member;
+};
+
+template<typename T>
+T explicitly_instantiated<T>::member = 54321;
+
+template int explicitly_instantiated<int>::member;
+// CHECK-DAG: cir.global weak_odr comdat @_ZN23explicitly_instantiatedIiE6memberE = #cir.int<54321> : !s32i
+// LLVM-DAG: @_ZN23explicitly_instantiatedIiE6memberE = weak_odr global i32 54321, comdat
+// OGCG-DAG: @_ZN23explicitly_instantiatedIiE6memberE = weak_odr global i32 54321, comdat
 
 void func1(void) {
   // Should lower default-initialized static vars.
@@ -42,6 +73,8 @@ void func2(void) {
 
 // LLVM-DAG: $_ZZ4testvE1c = comdat any
 // LLVM-DAG: @_ZZ4testvE1c = linkonce_odr global i32 0, comdat, align 4
+// OGCG-DAG: $_ZZ4testvE1c = comdat any
+// OGCG-DAG: @_ZZ4testvE1c = linkonce_odr global i32 0, comdat, align 4
 
 inline void test() { static int c; }
 // CHECK-LABEL: @_Z4testv
diff --git a/clang/test/CIR/CodeGen/stmt-expr.cpp b/clang/test/CIR/CodeGen/stmt-expr.cpp
new file mode 100644
index 0000000000000..9e3911f638ba7
--- /dev/null
+++ b/clang/test/CIR/CodeGen/stmt-expr.cpp
@@ -0,0 +1,90 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -mconstructor-aliases -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -mconstructor-aliases -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -mconstructor-aliases -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+
+class A {
+public:
+  A(): x(0) {}
+  A(A &a) : x(a.x) {}
+  int x;
+  void Foo() {}
+};
+
+void test1() {
+  ({
+    A a;
+    a;
+  }).Foo();
+}
+
+// CIR: cir.func dso_local @_Z5test1v()
+// CIR:   cir.scope {
+// CIR:     %[[REF_TMP0:.+]] = cir.alloca !rec_A, !cir.ptr<!rec_A>, ["ref.tmp0"]
+// CIR:     %[[TMP:.+]]      = cir.alloca !rec_A, !cir.ptr<!rec_A>, ["tmp"]
+// CIR:     cir.scope {
+// CIR:       %[[A:.+]] = cir.alloca !rec_A, !cir.ptr<!rec_A>, ["a", init]
+// CIR:       cir.call @_ZN1AC2Ev(%[[A]]) : (!cir.ptr<!rec_A>) -> ()
+// CIR:       cir.call @_ZN1AC2ERS_(%[[REF_TMP0]], %[[A]]) : (!cir.ptr<!rec_A>, !cir.ptr<!rec_A>) -> ()
+// CIR:     }
+// CIR:     cir.call @_ZN1A3FooEv(%[[REF_TMP0]]) : (!cir.ptr<!rec_A>) -> ()
+// CIR:   }
+// CIR:   cir.return
+
+// LLVM: define dso_local void @_Z5test1v()
+// LLVM:   %[[VAR1:.+]] = alloca %class.A, i64 1
+// LLVM:   %[[VAR2:.+]] = alloca %class.A, i64 1
+// LLVM:   %[[VAR3:.+]] = alloca %class.A, i64 1
+// LLVM:   br label %[[LBL4:.+]]
+// LLVM: [[LBL4]]:
+// LLVM:     br label %[[LBL5:.+]]
+// LLVM: [[LBL5]]:
+// LLVM:     call void @_ZN1AC2Ev(ptr %[[VAR3]])
+// LLVM:     call void @_ZN1AC2ERS_(ptr %[[VAR1]], ptr %[[VAR3]])
+// LLVM:     br label %[[LBL6:.+]]
+// LLVM: [[LBL6]]:
+// LLVM:     call void @_ZN1A3FooEv(ptr %[[VAR1]])
+// LLVM:     br label %[[LBL7:.+]]
+// LLVM: [[LBL7]]:
+// LLVM:     ret void
+
+// OGCG: define dso_local void @_Z5test1v()
+// OGCG: entry:
+// OGCG:   %[[REF_TMP:.+]] = alloca %class.A
+// OGCG:   %[[A:.+]]       = alloca %class.A
+// OGCG:   call void @_ZN1AC2Ev(ptr {{.*}} %[[A]])
+// OGCG:   call void @_ZN1AC2ERS_(ptr {{.*}} %[[REF_TMP]], ptr {{.*}} %[[A]])
+// OGCG:   call void @_ZN1A3FooEv(ptr {{.*}} %[[REF_TMP]])
+// OGCG:   ret void
+
+struct with_dtor {
+  ~with_dtor();
+};
+
+void cleanup() {
+  ({ with_dtor wd; });
+}
+
+// CIR: cir.func dso_local @_Z7cleanupv()
+// CIR:   cir.scope {
+// CIR:     %[[WD:.+]] = cir.alloca !rec_with_dtor, !cir.ptr<!rec_with_dtor>, ["wd"]
+// CIR:     cir.call @_ZN9with_dtorD1Ev(%[[WD]]) nothrow : (!cir.ptr<!rec_with_dtor>) -> ()
+// CIR:   }
+// CIR:   cir.return
+
+// LLVM: define dso_local void @_Z7cleanupv()
+// LLVM:   %[[WD:.+]] = alloca %struct.with_dtor, i64 1
+// LLVM:   br label %[[LBL2:.+]]
+// LLVM: [[LBL2]]:
+// LLVM:     call void @_ZN9with_dtorD1Ev(ptr %[[WD]])
+// LLVM:     br label %[[LBL3:.+]]
+// LLVM: [[LBL3]]:
+// LLVM:     ret void
+
+// OGCG: define dso_local void @_Z7cleanupv()
+// OGCG: entry:
+// OGCG:   %[[WD:.+]] = alloca %struct.with_dtor
+// OGCG:   call void @_ZN9with_dtorD1Ev(ptr {{.*}} %[[WD]])
+// OGCG:   ret void
diff --git a/clang/test/CIR/CodeGen/string-literals.c b/clang/test/CIR/CodeGen/string-literals.c
index 44fd191173c33..38657b2cd1175 100644
--- a/clang/test/CIR/CodeGen/string-literals.c
+++ b/clang/test/CIR/CodeGen/string-literals.c
@@ -17,13 +17,13 @@ char g_exact[4] = "123";
 
 // CIR: cir.global external @g_exact = #cir.const_array<"123\00" : !cir.array<!s8i x 4>> : !cir.array<!s8i x 4>
 
-// CIR: cir.global "private" cir_private dso_local @[[STR1_GLOBAL:.*]] = #cir.const_array<"1\00" : !cir.array<!s8i x 2>> : !cir.array<!s8i x 2>
-// CIR: cir.global "private" cir_private dso_local @[[STR2_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 1>
-// CIR: cir.global "private" cir_private dso_local @[[STR3_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 2>
+// CIR: cir.global "private" constant cir_private dso_local @[[STR1_GLOBAL:.*]] = #cir.const_array<"1\00" : !cir.array<!s8i x 2>> : !cir.array<!s8i x 2>
+// CIR: cir.global "private" constant cir_private dso_local @[[STR2_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 1>
+// CIR: cir.global "private" constant cir_private dso_local @[[STR3_GLOBAL:.*]] = #cir.zero : !cir.array<!s8i x 2>
 
-// LLVM: @[[STR1_GLOBAL:.*]] = private global [2 x i8] c"1\00"
-// LLVM: @[[STR2_GLOBAL:.*]] = private global [1 x i8] zeroinitializer
-// LLVM: @[[STR3_GLOBAL:.*]] = private global [2 x i8] zeroinitializer
+// LLVM: @[[STR1_GLOBAL:.*]] = private constant [2 x i8] c"1\00"
+// LLVM: @[[STR2_GLOBAL:.*]] = private constant [1 x i8] zeroinitializer
+// LLVM: @[[STR3_GLOBAL:.*]] = private constant [2 x i8] zeroinitializer
 
 // OGCG: @[[STR1_GLOBAL:.*]] = private unnamed_addr constant [2 x i8] c"1\00"
 // OGCG: @[[STR2_GLOBAL:.*]] = private unnamed_addr constant [1 x i8] zeroinitializer
diff --git a/clang/test/CIR/CodeGen/string-literals.cpp b/clang/test/CIR/CodeGen/string-literals.cpp
index 081c4c2c5a4a2..14115672e28e2 100644
--- a/clang/test/CIR/CodeGen/string-literals.cpp
+++ b/clang/test/CIR/CodeGen/string-literals.cpp
@@ -5,11 +5,37 @@
 // RUN: %clang_cc1 -triple aarch64-none-linux-android21 -emit-llvm %s -o %t.ll
 // RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
 
-// CIR: cir.global "private" cir_private dso_local @[[STR1_GLOBAL:.*]] = #cir.const_array<"abcd\00" : !cir.array<!s8i x 5>> : !cir.array<!s8i x 5>
 
-// LLVM: @[[STR1_GLOBAL:.*]] = private global [5 x i8] c"abcd\00"
+char const *array[] {
+    "my", "hands", "are", "typing", "words"
+};
 
-// OGCG: @[[STR1_GLOBAL:.*]] = private unnamed_addr constant [5 x i8] c"abcd\00"
+// CIR: cir.global "private" constant cir_private dso_local @"[[STR:.+]]" = #cir.const_array<"my\00" : !cir.array<!s8i x 3>> : !cir.array<!s8i x 3>
+// CIR: cir.global "private" constant cir_private dso_local @"[[STR1:.+]]" = #cir.const_array<"hands\00" : !cir.array<!s8i x 6>> : !cir.array<!s8i x 6>
+// CIR: cir.global "private" constant cir_private dso_local @"[[STR2:.+]]" = #cir.const_array<"are\00" : !cir.array<!s8i x 4>> : !cir.array<!s8i x 4>
+// CIR: cir.global "private" constant cir_private dso_local @"[[STR3:.+]]" = #cir.const_array<"typing\00" : !cir.array<!s8i x 7>> : !cir.array<!s8i x 7>
+// CIR: cir.global "private" constant cir_private dso_local @"[[STR4:.+]]" = #cir.const_array<"words\00" : !cir.array<!s8i x 6>> : !cir.array<!s8i x 6>
+// CIR: cir.global external @array = #cir.const_array<[#cir.global_view<@"[[STR]]"> : !cir.ptr<!s8i>, #cir.global_view<@"[[STR1]]"> : !cir.ptr<!s8i>, #cir.global_view<@"[[STR2]]"> : !cir.ptr<!s8i>, #cir.global_view<@"[[STR3]]"> : !cir.ptr<!s8i>, #cir.global_view<@"[[STR4]]"> : !cir.ptr<!s8i>]> : !cir.array<!cir.ptr<!s8i> x 5>
+
+// LLVM: @[[STR:.+]] = private constant [3 x i8] c"my\00"
+// LLVM: @[[STR1:.+]] = private constant [6 x i8] c"hands\00"
+// LLVM: @[[STR2:.+]] = private constant [4 x i8] c"are\00"
+// LLVM: @[[STR3:.+]] = private constant [7 x i8] c"typing\00"
+// LLVM: @[[STR4:.+]] = private constant [6 x i8] c"words\00"
+// LLVM: @array = global [5 x ptr] [ptr @[[STR]], ptr @[[STR1]], ptr @[[STR2]], ptr @[[STR3]], ptr @[[STR4]]]
+
+// OGCG: @[[STR:.+]] = private unnamed_addr constant [3 x i8] c"my\00"
+// OGCG: @[[STR1:.+]] = private unnamed_addr constant [6 x i8] c"hands\00"
+// OGCG: @[[STR2:.+]] = private unnamed_addr constant [4 x i8] c"are\00"
+// OGCG: @[[STR3:.+]] = private unnamed_addr constant [7 x i8] c"typing\00"
+// OGCG: @[[STR4:.+]] = private unnamed_addr constant [6 x i8] c"words\00"
+// OGCG: @array = global [5 x ptr] [ptr @[[STR]], ptr @[[STR1]], ptr @[[STR2]], ptr @[[STR3]], ptr @[[STR4]]]
+
+// CIR: cir.global "private" constant cir_private dso_local @[[STR5_GLOBAL:.*]] = #cir.const_array<"abcd\00" : !cir.array<!s8i x 5>> : !cir.array<!s8i x 5>
+
+// LLVM: @[[STR5_GLOBAL:.*]] = private constant [5 x i8] c"abcd\00"
+
+// OGCG: @[[STR5_GLOBAL:.*]] = private unnamed_addr constant [5 x i8] c"abcd\00"
 
 decltype(auto) returns_literal() {
     return "abcd";
@@ -17,7 +43,7 @@ decltype(auto) returns_literal() {
 
 // CIR: cir.func{{.*}} @_Z15returns_literalv() -> !cir.ptr<!cir.array<!s8i x 5>>
 // CIR:   %[[RET_ADDR:.*]] = cir.alloca !cir.ptr<!cir.array<!s8i x 5>>, !cir.ptr<!cir.ptr<!cir.array<!s8i x 5>>>, ["__retval"]
-// CIR:   %[[STR_ADDR:.*]] = cir.get_global @[[STR1_GLOBAL]] : !cir.ptr<!cir.array<!s8i x 5>>
+// CIR:   %[[STR_ADDR:.*]] = cir.get_global @[[STR5_GLOBAL]] : !cir.ptr<!cir.array<!s8i x 5>>
 // CIR:   cir.store{{.*}} %[[STR_ADDR]], %[[RET_ADDR]]
 // CIR:   %[[RET:.*]] = cir.load %[[RET_ADDR]]
 // CIR:   cir.return %[[RET]]
diff --git a/clang/test/CIR/CodeGen/var_arg.c b/clang/test/CIR/CodeGen/var_arg.c
new file mode 100644
index 0000000000000..e9c4acb15d009
--- /dev/null
+++ b/clang/test/CIR/CodeGen/var_arg.c
@@ -0,0 +1,166 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s -check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -Wno-unused-value -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
+
+// CIR: !rec___va_list_tag = !cir.record<struct "__va_list_tag" {!u32i, !u32i, !cir.ptr<!void>, !cir.ptr<!void>}
+// LLVM: %struct.__va_list_tag = type { i32, i32, ptr, ptr }
+// OGCG: %struct.__va_list_tag = type { i32, i32, ptr, ptr }
+
+int varargs(int count, ...) {
+    __builtin_va_list args;
+    __builtin_va_start(args, count);
+    int res = __builtin_va_arg(args, int);
+    __builtin_va_end(args);
+    return res;
+}
+
+// CIR-LABEL: cir.func dso_local @varargs(
+// CIR:   %[[COUNT_ADDR:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["count", init]
+// CIR:   %[[RET_ADDR:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[VAAREA:.+]] = cir.alloca !cir.array<!rec___va_list_tag x 1>, !cir.ptr<!cir.array<!rec___va_list_tag x 1>>, ["args"]
+// CIR:   %[[RES_ADDR:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["res", init]
+// CIR:   cir.store %arg0, %[[COUNT_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[VA_PTR0:.+]] = cir.cast(array_to_ptrdecay, %[[VAAREA]] : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>), !cir.ptr<!rec___va_list_tag>
+// CIR:   %[[COUNT_VAL:.+]] = cir.load{{.*}} %[[COUNT_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.va_start %[[VA_PTR0]] %[[COUNT_VAL]] : !cir.ptr<!rec___va_list_tag>, !s32i
+// CIR:   %[[VA_PTR1:.+]] = cir.cast(array_to_ptrdecay, %[[VAAREA]] : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>), !cir.ptr<!rec___va_list_tag>
+// CIR:   %[[VA_ARG:.+]] = cir.va_arg %[[VA_PTR1]] : (!cir.ptr<!rec___va_list_tag>) -> !s32i
+// CIR:   cir.store{{.*}} %[[VA_ARG]], %[[RES_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[VA_PTR2:.+]] = cir.cast(array_to_ptrdecay, %[[VAAREA]] : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>), !cir.ptr<!rec___va_list_tag>
+// CIR:   cir.va_end %[[VA_PTR2]] : !cir.ptr<!rec___va_list_tag>
+// CIR:   %[[RESULT:.+]] = cir.load{{.*}} %[[RES_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.store %[[RESULT]], %[[RET_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[RETVAL:.+]] = cir.load{{.*}} %[[RET_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[RETVAL]] : !s32i
+
+// LLVM-LABEL: define dso_local i32 @varargs(
+// LLVM:   %[[COUNT_ADDR:.+]] = alloca i32{{.*}}
+// LLVM:   %[[RET_ADDR:.+]] = alloca i32{{.*}}
+// LLVM:   %[[VAAREA:.+]] = alloca [1 x %struct.__va_list_tag]{{.*}}
+// LLVM:   %[[RES_ADDR:.+]] = alloca i32{{.*}}
+// LLVM:   %[[VA_PTR0:.+]] = getelementptr %struct.__va_list_tag, ptr %[[VAAREA]], i32 0
+// LLVM:   call void @llvm.va_start.p0(ptr %[[VA_PTR0]])
+// LLVM:   %[[VA_PTR1:.+]] = getelementptr %struct.__va_list_tag, ptr %[[VAAREA]], i32 0
+// LLVM:   %[[VA_ARG:.+]] = va_arg ptr %[[VA_PTR1]], i32
+// LLVM:   store i32 %[[VA_ARG]], ptr %[[RES_ADDR]], {{.*}}
+// LLVM:   %[[VA_PTR2:.+]] = getelementptr %struct.__va_list_tag, ptr %[[VAAREA]], i32 0
+// LLVM:   call void @llvm.va_end.p0(ptr %[[VA_PTR2]])
+// LLVM:   %[[TMP_LOAD:.+]] = load i32, ptr %[[RES_ADDR]], {{.*}}
+// LLVM:   store i32 %[[TMP_LOAD]], ptr %[[RET_ADDR]], {{.*}}
+// LLVM:   %[[RETVAL:.+]] = load i32, ptr %[[RET_ADDR]], {{.*}}
+// LLVM:   ret i32 %[[RETVAL]]
+
+// OGCG-LABEL: define dso_local i32 @varargs
+// OGCG:   %[[COUNT_ADDR:.+]] = alloca i32
+// OGCG:   %[[VAAREA:.+]] = alloca [1 x %struct.__va_list_tag]
+// OGCG:   %[[RES_ADDR:.+]] = alloca i32
+// OGCG:   %[[DECAY:.+]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %[[VAAREA]]
+// OGCG:   call void @llvm.va_start.p0(ptr %[[DECAY]])
+// OGCG:   %[[DECAY1:.+]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %[[VAAREA]]
+// OGCG:   %[[GPOFFSET_PTR:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %[[DECAY1]], i32 0, i32 0
+// OGCG:   %[[GPOFFSET:.+]] = load i32, ptr %[[GPOFFSET_PTR]]
+// OGCG:   %[[COND:.+]] = icmp ule i32 %[[GPOFFSET]], 40
+// OGCG:   br i1 %[[COND]], label %vaarg.in_reg, label %vaarg.in_mem
+//
+// OGCG: vaarg.in_reg:
+// OGCG:   %[[REGSAVE_PTR:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %[[DECAY1]], i32 0, i32 3
+// OGCG:   %[[REGSAVE:.+]] = load ptr, ptr %[[REGSAVE_PTR]]
+// OGCG:   %[[VAADDR1:.+]] = getelementptr i8, ptr %[[REGSAVE]], i32 %[[GPOFFSET]]
+// OGCG:   br label %vaarg.end
+//
+// OGCG: vaarg.in_mem:
+// OGCG:   %[[OVERFLOW_PTR:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %[[DECAY1]], i32 0, i32 2
+// OGCG:   %[[OVERFLOW:.+]] = load ptr, ptr %[[OVERFLOW_PTR]]
+// OGCG:   br label %vaarg.end
+//
+// OGCG: vaarg.end:
+// OGCG:   %[[PHI:.+]] = phi ptr [ %[[VAADDR1]], %vaarg.in_reg ], [ %[[OVERFLOW]], %vaarg.in_mem ]
+// OGCG:   %[[LOADED:.+]] = load i32, ptr %[[PHI]]
+// OGCG:   store i32 %[[LOADED]], ptr %[[RES_ADDR]]
+// OGCG:   %[[DECAY2:.+]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %[[VAAREA]]
+// OGCG:   call void @llvm.va_end.p0(ptr %[[DECAY2]])
+// OGCG:   %[[VAL:.+]] = load i32, ptr %[[RES_ADDR]]
+// OGCG:   ret i32 %[[VAL]]
+
+int stdarg_start(int count, ...) {
+    __builtin_va_list args;
+    __builtin_stdarg_start(args, 12345);
+    int res = __builtin_va_arg(args, int);
+    __builtin_va_end(args);
+    return res;
+}
+
+// CIR-LABEL: cir.func dso_local @stdarg_start(
+// CIR:   %[[COUNT_ADDR:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["count", init]
+// CIR:   %[[RET_ADDR:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"]
+// CIR:   %[[VAAREA:.+]] = cir.alloca !cir.array<!rec___va_list_tag x 1>, !cir.ptr<!cir.array<!rec___va_list_tag x 1>>, ["args"]
+// CIR:   %[[RES_ADDR:.+]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["res", init]
+// CIR:   cir.store %arg0, %[[COUNT_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[VA_PTR0:.+]] = cir.cast(array_to_ptrdecay, %[[VAAREA]] : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>), !cir.ptr<!rec___va_list_tag>
+// CIR:   %[[C12345:.+]] = cir.const #cir.int<12345> : !s32i
+// CIR:   cir.va_start %[[VA_PTR0]] %[[C12345]] : !cir.ptr<!rec___va_list_tag>, !s32i
+// CIR:   %[[VA_PTR1:.+]] = cir.cast(array_to_ptrdecay, %[[VAAREA]] : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>), !cir.ptr<!rec___va_list_tag>
+// CIR:   %[[VA_ARG:.+]] = cir.va_arg %[[VA_PTR1]] : (!cir.ptr<!rec___va_list_tag>) -> !s32i
+// CIR:   cir.store{{.*}} %[[VA_ARG]], %[[RES_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[VA_PTR2:.+]] = cir.cast(array_to_ptrdecay, %[[VAAREA]] : !cir.ptr<!cir.array<!rec___va_list_tag x 1>>), !cir.ptr<!rec___va_list_tag>
+// CIR:   cir.va_end %[[VA_PTR2]] : !cir.ptr<!rec___va_list_tag>
+// CIR:   %[[RESULT:.+]] = cir.load{{.*}} %[[RES_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.store %[[RESULT]], %[[RET_ADDR]] : !s32i, !cir.ptr<!s32i>
+// CIR:   %[[RETVAL:.+]] = cir.load{{.*}} %[[RET_ADDR]] : !cir.ptr<!s32i>, !s32i
+// CIR:   cir.return %[[RETVAL]] : !s32i
+
+// LLVM-LABEL: define dso_local i32 @stdarg_start(
+// LLVM:   %[[COUNT_ADDR:.+]] = alloca i32{{.*}}
+// LLVM:   %[[RET_ADDR:.+]] = alloca i32{{.*}}
+// LLVM:   %[[VAAREA:.+]] = alloca [1 x %struct.__va_list_tag]{{.*}}
+// LLVM:   %[[RES_ADDR:.+]] = alloca i32{{.*}}
+// LLVM:   %[[VA_PTR0:.+]] = getelementptr %struct.__va_list_tag, ptr %[[VAAREA]], i32 0
+// LLVM:   call void @llvm.va_start.p0(ptr %[[VA_PTR0]])
+// LLVM:   %[[VA_PTR1:.+]] = getelementptr %struct.__va_list_tag, ptr %[[VAAREA]], i32 0
+// LLVM:   %[[VA_ARG:.+]] = va_arg ptr %[[VA_PTR1]], i32
+// LLVM:   store i32 %[[VA_ARG]], ptr %[[RES_ADDR]], {{.*}}
+// LLVM:   %[[VA_PTR2:.+]] = getelementptr %struct.__va_list_tag, ptr %[[VAAREA]], i32 0
+// LLVM:   call void @llvm.va_end.p0(ptr %[[VA_PTR2]])
+// LLVM:   %[[TMP_LOAD:.+]] = load i32, ptr %[[RES_ADDR]], {{.*}}
+// LLVM:   store i32 %[[TMP_LOAD]], ptr %[[RET_ADDR]], {{.*}}
+// LLVM:   %[[RETVAL:.+]] = load i32, ptr %[[RET_ADDR]], {{.*}}
+// LLVM:   ret i32 %[[RETVAL]]
+
+// OGCG-LABEL: define dso_local i32 @stdarg_start
+// OGCG:   %[[COUNT_ADDR:.+]] = alloca i32
+// OGCG:   %[[VAAREA:.+]] = alloca [1 x %struct.__va_list_tag]
+// OGCG:   %[[RES_ADDR:.+]] = alloca i32
+// OGCG:   %[[DECAY:.+]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %[[VAAREA]], i64 0, i64 0
+// OGCG:   call void @llvm.va_start.p0(ptr %[[DECAY]])
+// OGCG:   %[[DECAY1:.+]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %[[VAAREA]], i64 0, i64 0
+// OGCG:   %[[GPOFFSET_PTR:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %[[DECAY1]], i32 0, i32 0
+// OGCG:   %[[GPOFFSET:.+]] = load i32, ptr %[[GPOFFSET_PTR]]
+// OGCG:   %[[COND:.+]] = icmp ule i32 %[[GPOFFSET]], 40
+// OGCG:   br i1 %[[COND]], label %vaarg.in_reg, label %vaarg.in_mem
+//
+// OGCG: vaarg.in_reg:                                   
+// OGCG:   %[[REGSAVE_PTR:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %[[DECAY1]], i32 0, i32 3
+// OGCG:   %[[REGSAVE:.+]] = load ptr, ptr %[[REGSAVE_PTR]]
+// OGCG:   %[[VAADDR1:.+]] = getelementptr i8, ptr %[[REGSAVE]], i32 %[[GPOFFSET]]
+// OGCG:   %[[NEXT_GPOFFSET:.+]] = add i32 %[[GPOFFSET]], 8
+// OGCG:   store i32 %[[NEXT_GPOFFSET]], ptr %[[GPOFFSET_PTR]]
+// OGCG:   br label %vaarg.end
+//
+// OGCG: vaarg.in_mem:
+// OGCG:   %[[OVERFLOW_PTR:.+]] = getelementptr inbounds nuw %struct.__va_list_tag, ptr %[[DECAY1]], i32 0, i32 2
+// OGCG:   %[[OVERFLOW:.+]] = load ptr, ptr %[[OVERFLOW_PTR]]
+// OGCG:   %[[OVERFLOW_NEXT:.+]] = getelementptr i8, ptr %[[OVERFLOW]], i32 8
+// OGCG:   store ptr %[[OVERFLOW_NEXT]], ptr %[[OVERFLOW_PTR]]
+// OGCG:   br label %vaarg.end
+//
+// OGCG: vaarg.end:
+// OGCG:   %[[PHI:.+]] = phi ptr [ %[[VAADDR1]], %vaarg.in_reg ], [ %[[OVERFLOW]], %vaarg.in_mem ]
+// OGCG:   %[[LOADED:.+]] = load i32, ptr %[[PHI]]
+// OGCG:   store i32 %[[LOADED]], ptr %[[RES_ADDR]]
+// OGCG:   %[[DECAY2:.+]] = getelementptr inbounds [1 x %struct.__va_list_tag], ptr %[[VAAREA]], i64 0, i64 0
+// OGCG:   call void @llvm.va_end.p0(ptr %[[DECAY2]])
+// OGCG:   %[[VAL:.+]] = load i32, ptr %[[RES_ADDR]]
+// OGCG:   ret i32 %[[VAL]]
diff --git a/clang/test/CIR/CodeGen/variable-template-specialization.cpp b/clang/test/CIR/CodeGen/variable-template-specialization.cpp
new file mode 100644
index 0000000000000..c13ab51ec64c9
--- /dev/null
+++ b/clang/test/CIR/CodeGen/variable-template-specialization.cpp
@@ -0,0 +1,40 @@
+// RUN: %clang_cc1 -std=c++14 -triple nvptx64-unknown-unknown -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -std=c++14 -triple nvptx64-unknown-unknown -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -std=c++14 -triple nvptx64-unknown-unknown -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
+
+struct some_struct {
+    int x;
+};
+
+template<int I>
+int var_template;
+
+template<> int var_template<0>;
+template<> int var_template<1> = 1;
+template<> some_struct var_template<2>;
+
+// CIR: !rec_some_struct = !cir.record<struct "some_struct" {!s32i}>
+// CIR: cir.global external @_Z12var_templateILi0EE = #cir.int<0> : !s32i
+// CIR: cir.global external @_Z12var_templateILi1EE = #cir.int<1> : !s32i
+// CIR: cir.global external @_Z12var_templateILi2EE = #cir.zero : !rec_some_struct
+
+// LLVM: %[[STRUCT_TYPE:.+]] = type { i32 }
+// LLVM: @_Z12var_templateILi0EE = global i32 0
+// LLVM: @_Z12var_templateILi1EE = global i32 1
+// LLVM: @_Z12var_templateILi2EE = global %[[STRUCT_TYPE]] zeroinitializer
+
+// OGCG: %[[STRUCT_TYPE:.+]] = type { i32 }
+// OGCG: @_Z12var_templateILi0EE = global i32 0
+// OGCG: @_Z12var_templateILi1EE = global i32 1
+// OGCG: @_Z12var_templateILi2EE = global %[[STRUCT_TYPE]] zeroinitializer
+
+template<typename T, int I> int partial_var_template_specialization_shouldnt_hit_codegen;
+template<typename T> int partial_var_template_specialization_shouldnt_hit_codegen<T, 123>;
+template<int I> float partial_var_template_specialization_shouldnt_hit_codegen<float, I>;
+
+// CIR-NOT: partial_var_template_specialization_shouldnt_hit_codegen
+// LLVM-NOT: partial_var_template_specialization_shouldnt_hit_codegen
+// OGCG-NOT: partial_var_template_specialization_shouldnt_hit_codegen
diff --git a/clang/test/CIR/CodeGen/virtual-function-calls.cpp b/clang/test/CIR/CodeGen/virtual-function-calls.cpp
index 3e03b32ce1fd2..e68b38fad3f84 100644
--- a/clang/test/CIR/CodeGen/virtual-function-calls.cpp
+++ b/clang/test/CIR/CodeGen/virtual-function-calls.cpp
@@ -1,13 +1,81 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -mconstructor-aliases -fclangir -emit-cir %s -o %t.cir
 // RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -mconstructor-aliases -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s -check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -mconstructor-aliases -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s -check-prefix=OGCG
 
 struct A {
+  A();
   virtual void f(char);
 };
 
-// This is just here to force the class definition to be emitted without
-// requiring any other support. It will be removed when more complete
-// vtable support is implemented.
-A *a;
+// This should initialize the vtable pointer.
+A::A() {}
 
 // CIR: !rec_A = !cir.record<struct "A" {!cir.vptr}>
+// CIR: !rec_anon_struct = !cir.record<struct  {!cir.array<!cir.ptr<!u8i> x 3>}>
+
+// CIR: cir.global "private" external @_ZTV1A : !rec_anon_struct
+
+// LLVM: @_ZTV1A = external global { [3 x ptr] }
+
+// OGCG: @_ZTV1A = external unnamed_addr constant { [3 x ptr] }
+
+// CIR: cir.func{{.*}} @_ZN1AC2Ev(%arg0: !cir.ptr<!rec_A> {{.*}})
+// CIR:    %[[THIS_ADDR:.*]] = cir.alloca !cir.ptr<!rec_A>, !cir.ptr<!cir.ptr<!rec_A>>, ["this", init]
+// CIR:    cir.store %arg0, %[[THIS_ADDR]] : !cir.ptr<!rec_A>, !cir.ptr<!cir.ptr<!rec_A>>
+// CIR:    %[[THIS:.*]] = cir.load %[[THIS_ADDR]] : !cir.ptr<!cir.ptr<!rec_A>>, !cir.ptr<!rec_A>
+// CIR:    %[[VPTR:.*]] = cir.vtable.address_point(@_ZTV1A, address_point = <index = 0, offset = 2>) : !cir.vptr
+// CIR:    %[[THIS_VPTR_PTR:.*]] = cir.vtable.get_vptr %[[THIS]] : !cir.ptr<!rec_A> -> !cir.ptr<!cir.vptr>
+// CIR:    cir.store{{.*}} align(8) %[[VPTR]], %[[THIS_VPTR_PTR]] : !cir.vptr, !cir.ptr<!cir.vptr>
+// CIR:    cir.return
+
+// LLVM: define{{.*}} void @_ZN1AC2Ev(ptr %[[ARG0:.*]])
+// LLVM:   %[[THIS_ADDR:.*]] = alloca ptr
+// LLVM:   store ptr %[[ARG0]], ptr %[[THIS_ADDR]]
+// LLVM:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]]
+// LLVM:   store ptr getelementptr inbounds nuw (i8, ptr @_ZTV1A, i64 16), ptr %[[THIS]]
+
+// OGCG: define{{.*}} void @_ZN1AC2Ev(ptr {{.*}} %[[ARG0:.*]])
+// OGCG:   %[[THIS_ADDR:.*]] = alloca ptr
+// OGCG:   store ptr %[[ARG0]], ptr %[[THIS_ADDR]]
+// OGCG:   %[[THIS:.*]] = load ptr, ptr %[[THIS_ADDR]]
+// OGCG:   store ptr getelementptr inbounds inrange(-16, 8) ({ [3 x ptr] }, ptr @_ZTV1A, i32 0, i32 0, i32 2), ptr %[[THIS]]
+
+// NOTE: The GEP in OGCG looks very different from the one generated with CIR,
+//       but it is equivalent. The OGCG GEP indexes by base pointer, then
+//       structure, then array, whereas the CIR GEP indexes by byte offset.
+
+void f1(A *a) {
+  a->f('c');
+}
+
+// CIR: cir.func{{.*}} @_Z2f1P1A(%arg0: !cir.ptr<!rec_A> {{.*}})
+// CIR:   %[[A_ADDR:.*]] = cir.alloca !cir.ptr<!rec_A>
+// CIR:   cir.store %arg0, %[[A_ADDR]]
+// CIR:   %[[A:.*]] = cir.load{{.*}} %[[A_ADDR]]
+// CIR:   %[[C_LITERAL:.*]] = cir.const #cir.int<99> : !s8i
+// CIR:   %[[VPTR_ADDR:.*]] = cir.vtable.get_vptr %[[A]] : !cir.ptr<!rec_A> -> !cir.ptr<!cir.vptr>
+// CIR:   %[[VPTR:.*]] = cir.load{{.*}} %[[VPTR_ADDR]] : !cir.ptr<!cir.vptr>, !cir.vptr
+// CIR:   %[[FN_PTR_PTR:.*]] = cir.vtable.get_virtual_fn_addr %[[VPTR]][0] : !cir.vptr -> !cir.ptr<!cir.ptr<!cir.func<(!cir.ptr<!rec_A>, !s8i)>>>
+// CIR:   %[[FN_PTR:.*]] = cir.load{{.*}} %[[FN_PTR_PTR:.*]] : !cir.ptr<!cir.ptr<!cir.func<(!cir.ptr<!rec_A>, !s8i)>>>, !cir.ptr<!cir.func<(!cir.ptr<!rec_A>, !s8i)>>
+// CIR:   cir.call %[[FN_PTR]](%[[A]], %[[C_LITERAL]]) : (!cir.ptr<!cir.func<(!cir.ptr<!rec_A>, !s8i)>>, !cir.ptr<!rec_A>, !s8i) -> ()
+
+// LLVM: define{{.*}} void @_Z2f1P1A(ptr %[[ARG0:.*]])
+// LLVM:   %[[A_ADDR:.*]] = alloca ptr
+// LLVM:   store ptr %[[ARG0]], ptr %[[A_ADDR]]
+// LLVM:   %[[A:.*]] = load ptr, ptr %[[A_ADDR]]
+// LLVM:   %[[VPTR:.*]] = load ptr, ptr %[[A]]
+// LLVM:   %[[FN_PTR_PTR:.*]] = getelementptr inbounds ptr, ptr %[[VPTR]], i32 0
+// LLVM:   %[[FN_PTR:.*]] = load ptr, ptr %[[FN_PTR_PTR]]
+// LLVM:   call void %[[FN_PTR]](ptr %[[A]], i8 99)
+
+// OGCG: define{{.*}} void @_Z2f1P1A(ptr {{.*}} %[[ARG0:.*]])
+// OGCG:   %[[A_ADDR:.*]] = alloca ptr
+// OGCG:   store ptr %[[ARG0]], ptr %[[A_ADDR]]
+// OGCG:   %[[A:.*]] = load ptr, ptr %[[A_ADDR]]
+// OGCG:   %[[VPTR:.*]] = load ptr, ptr %[[A]]
+// OGCG:   %[[FN_PTR_PTR:.*]] = getelementptr inbounds ptr, ptr %[[VPTR]], i64 0
+// OGCG:   %[[FN_PTR:.*]] = load ptr, ptr %[[FN_PTR_PTR]]
+// OGCG:   call void %[[FN_PTR]](ptr {{.*}} %[[A]], i8 {{.*}} 99)
diff --git a/clang/test/CIR/CodeGen/vtable-emission.cpp b/clang/test/CIR/CodeGen/vtable-emission.cpp
new file mode 100644
index 0000000000000..9a34573b475c3
--- /dev/null
+++ b/clang/test/CIR/CodeGen/vtable-emission.cpp
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fno-rtti -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fno-rtti -fclangir  -emit-llvm -o %t-cir.ll %s
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t-cir.ll %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fno-rtti -emit-llvm -o %t.ll %s
+// RUN: FileCheck --check-prefix=OGCG --input-file=%t.ll %s
+
+// Note: This test is using -fno-rtti so that we can delay implemntation of that handling.
+//       When rtti handling for vtables is implemented, that option should be removed.
+
+struct S {
+  virtual void key();
+  virtual void nonKey() {}
+};
+
+void S::key() {}
+
+// CHECK-DAG: !rec_anon_struct = !cir.record<struct  {!cir.array<!cir.ptr<!u8i> x 4>}>
+
+// The definition of the key function should result in the vtable being emitted.
+// CHECK:      cir.global "private" external @_ZTV1S = #cir.vtable<{
+// CHECK-SAME:     #cir.const_array<[
+// CHECK-SAME:         #cir.ptr<null> : !cir.ptr<!u8i>,
+// CHECK-SAME:         #cir.ptr<null> : !cir.ptr<!u8i>,
+// CHECK-SAME:         #cir.global_view<@_ZN1S3keyEv> : !cir.ptr<!u8i>,
+// CHECK-SAME:         #cir.global_view<@_ZN1S6nonKeyEv> : !cir.ptr<!u8i>]>
+// CHECK-SAME:     : !cir.array<!cir.ptr<!u8i> x 4>}> : !rec_anon_struct
+
+// LLVM:      @_ZTV1S = global { [4 x ptr] } { [4 x ptr]
+// LLVM-SAME:      [ptr null, ptr null, ptr @_ZN1S3keyEv, ptr @_ZN1S6nonKeyEv] }
+
+// OGCG:      @_ZTV1S = unnamed_addr constant { [4 x ptr] } { [4 x ptr]
+// OGCG-SAME:      [ptr null, ptr null, ptr @_ZN1S3keyEv, ptr @_ZN1S6nonKeyEv] }
+
+// CHECK: cir.func dso_local @_ZN1S3keyEv
+
+// The reference from the vtable should result in nonKey being emitted.
+// CHECK: cir.func comdat linkonce_odr @_ZN1S6nonKeyEv
diff --git a/clang/test/CIR/CodeGenOpenACC/combined-firstprivate-clause.cpp b/clang/test/CIR/CodeGenOpenACC/combined-firstprivate-clause.cpp
new file mode 100644
index 0000000000000..64e5a73667fff
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/combined-firstprivate-clause.cpp
@@ -0,0 +1,779 @@
+// RUN: %clang_cc1 -fopenacc -triple x86_64-linux-gnu -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir -triple x86_64-linux-pc %s -o - | FileCheck %s
+
+struct NoCopyConstruct {};
+
+struct CopyConstruct {
+  CopyConstruct() = default;
+  CopyConstruct(const CopyConstruct&);
+};
+
+struct NonDefaultCtor {
+  NonDefaultCtor();
+};
+
+struct HasDtor {
+  ~HasDtor();
+};
+
+// CHECK: acc.firstprivate.recipe @firstprivatization__ZTSA5_7HasDtor : !cir.ptr<!cir.array<!rec_HasDtor x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_HasDtor x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!rec_HasDtor x 5>, !cir.ptr<!cir.array<!rec_HasDtor x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!cir.array<!rec_HasDtor x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!rec_HasDtor x 5>> {{.*}}):
+// CHECK-NEXT: %[[DECAY_TO:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor> 
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[ZERO]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[DECAY_TO]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_HasDtor>, %[[ONE]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[ONE_2]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_HasDtor>, %[[TWO]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[TWO_2]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_HasDtor>, %[[THREE]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[THREE_2]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_HasDtor>, %[[FOUR]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[FOUR_2]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: acc.yield
+//
+// CHECK-NEXT: } destroy {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_HasDtor x 5>> {{.*}}):
+// CHECK-NEXT: %[[LAST_IDX:.*]] = cir.const #cir.int<4> : !u64i
+// CHECK-NEXT: %[[ARRPTR:.*]] = cir.cast(array_to_ptrdecay, %[[ARG]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[ELEM:.*]] = cir.ptr_stride(%[[ARRPTR]] : !cir.ptr<!rec_HasDtor>, %[[LAST_IDX]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[ITR:.*]] = cir.alloca !cir.ptr<!rec_HasDtor>, !cir.ptr<!cir.ptr<!rec_HasDtor>>, ["__array_idx"]
+// CHECK-NEXT: cir.store %[[ELEM]], %[[ITR]] : !cir.ptr<!rec_HasDtor>, !cir.ptr<!cir.ptr<!rec_HasDtor>>
+// CHECK-NEXT: cir.do {
+// CHECK-NEXT: %[[ELEM_LOAD:.*]] = cir.load %[[ITR]] : !cir.ptr<!cir.ptr<!rec_HasDtor>>, !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[ELEM_LOAD]]) nothrow : (!cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: %[[NEG_ONE:.*]] =  cir.const #cir.int<-1> : !s64i
+// CHECK-NEXT: %[[PREVELEM:.*]] = cir.ptr_stride(%[[ELEM_LOAD]] : !cir.ptr<!rec_HasDtor>, %[[NEG_ONE]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.store %[[PREVELEM]], %[[ITR]] : !cir.ptr<!rec_HasDtor>, !cir.ptr<!cir.ptr<!rec_HasDtor>>
+// CHECK-NEXT: cir.yield
+// CHECK-NEXT: } while {
+// CHECK-NEXT: %[[ELEM_LOAD:.*]] = cir.load %[[ITR]] : !cir.ptr<!cir.ptr<!rec_HasDtor>>, !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[CMP:.*]] = cir.cmp(ne, %[[ELEM_LOAD]], %[[ARRPTR]]) : !cir.ptr<!rec_HasDtor>, !cir.bool
+// CHECK-NEXT: cir.condition(%[[CMP]])
+// CHECK-NEXT: }
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_14NonDefaultCtor : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {{.*}}):
+// CHECK-NEXT: %[[ALLOCA:.*]] = cir.alloca !cir.array<!rec_NonDefaultCtor x 5>, !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!rec_NonDefaultCtor>, %[[ZERO]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_DECAY]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NonDefaultCtor>, %[[ONE]] : !s64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NonDefaultCtor>, %[[ONE_2]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NonDefaultCtor>, %[[TWO]] : !s64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NonDefaultCtor>, %[[TWO_2]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NonDefaultCtor>, %[[THREE]] : !s64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NonDefaultCtor>, %[[THREE_2]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NonDefaultCtor>, %[[FOUR]] : !s64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NonDefaultCtor>, %[[FOUR_2]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_13CopyConstruct : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!rec_CopyConstruct x 5>, !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!rec_CopyConstruct>, %[[ZERO]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_DECAY]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_CopyConstruct>, %[[ONE]] : !s64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_CopyConstruct>, %[[ONE_2]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_CopyConstruct>, %[[TWO]] : !s64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_CopyConstruct>, %[[TWO_2]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_CopyConstruct>, %[[THREE]] : !s64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_CopyConstruct>, %[[THREE_2]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_CopyConstruct>, %[[FOUR]] : !s64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_CopyConstruct>, %[[FOUR_2]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_15NoCopyConstruct : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!rec_NoCopyConstruct x 5>, !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!rec_NoCopyConstruct>, %[[ZERO]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_DECAY]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NoCopyConstruct>, %[[ONE]] : !s64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NoCopyConstruct>, %[[ONE_2]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NoCopyConstruct>, %[[TWO]] : !s64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NoCopyConstruct>, %[[TWO_2]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NoCopyConstruct>, %[[THREE]] : !s64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NoCopyConstruct>, %[[THREE_2]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NoCopyConstruct>, %[[FOUR]] : !s64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NoCopyConstruct>, %[[FOUR_2]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_f : !cir.ptr<!cir.array<!cir.float x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!cir.float x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!cir.float x 5>, !cir.ptr<!cir.array<!cir.float x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!cir.float x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!cir.float x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!cir.float>, %[[ZERO]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_DECAY]] : !cir.float, !cir.ptr<!cir.float>
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!cir.float>, %[[ONE]] : !s64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!cir.float>, %[[ONE_2]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !cir.float, !cir.ptr<!cir.float>
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!cir.float>, %[[TWO]] : !s64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!cir.float>, %[[TWO_2]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !cir.float, !cir.ptr<!cir.float>
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!cir.float>, %[[THREE]] : !s64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!cir.float>, %[[THREE_2]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !cir.float, !cir.ptr<!cir.float>
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!cir.float>, %[[FOUR]] : !s64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!cir.float>, %[[FOUR_2]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !cir.float, !cir.ptr<!cir.float>
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_i : !cir.ptr<!cir.array<!s32i x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!s32i x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!s32i x 5>, !cir.ptr<!cir.array<!s32i x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!s32i x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!s32i x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!s32i>, %[[ZERO]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_DECAY]] : !s32i, !cir.ptr<!s32i>
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!s32i>, %[[ONE]] : !s64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!s32i>, %[[ONE_2]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !s32i, !cir.ptr<!s32i>
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!s32i>, %[[TWO]] : !s64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!s32i>, %[[TWO_2]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !s32i, !cir.ptr<!s32i>
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!s32i>, %[[THREE]] : !s64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!s32i>, %[[THREE_2]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !s32i, !cir.ptr<!s32i>
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!s32i>, %[[FOUR]] : !s64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!s32i>, %[[FOUR_2]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !s32i, !cir.ptr<!s32i>
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS7HasDtor : !cir.ptr<!rec_HasDtor> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
+// CHECK-NEXT: cir.alloca !rec_HasDtor, !cir.ptr<!rec_HasDtor>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } destroy {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[ARG]]) nothrow : (!cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS14NonDefaultCtor : !cir.ptr<!rec_NonDefaultCtor> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}):
+// CHECK-NEXT: %[[ALLOCA:.*]] = cir.alloca !rec_NonDefaultCtor, !cir.ptr<!rec_NonDefaultCtor>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS13CopyConstruct : !cir.ptr<!rec_CopyConstruct> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}):
+// CHECK-NEXT: cir.alloca !rec_CopyConstruct, !cir.ptr<!rec_CopyConstruct>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS15NoCopyConstruct : !cir.ptr<!rec_NoCopyConstruct> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}):
+// CHECK-NEXT: cir.alloca !rec_NoCopyConstruct, !cir.ptr<!rec_NoCopyConstruct>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSf : !cir.ptr<!cir.float> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.float> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.float, !cir.ptr<!cir.float>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!cir.float> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.float> {{.*}}):
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[ARG_FROM]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[ARG_TO]] : !cir.float, !cir.ptr<!cir.float>
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSi : !cir.ptr<!s32i> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!s32i> {{.*}}):
+// CHECK-NEXT: cir.alloca !s32i, !cir.ptr<!s32i>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!s32i> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!s32i> {{.*}}):
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[ARG_FROM]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[ARG_TO]] : !s32i, !cir.ptr<!s32i>
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+
+extern "C" void acc_combined() {
+  // CHECK: cir.func{{.*}} @acc_combined() {
+
+  int someInt;
+  // CHECK-NEXT: %[[SOMEINT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["someInt"]
+  float someFloat;
+  // CHECK-NEXT: %[[SOMEFLOAT:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["someFloat"]
+  NoCopyConstruct noCopy;
+  // CHECK-NEXT: %[[NOCOPY:.*]] = cir.alloca !rec_NoCopyConstruct, !cir.ptr<!rec_NoCopyConstruct>, ["noCopy"]
+  CopyConstruct hasCopy;
+  // CHECK-NEXT: %[[HASCOPY:.*]] = cir.alloca !rec_CopyConstruct, !cir.ptr<!rec_CopyConstruct>, ["hasCopy"]
+  NonDefaultCtor notDefCtor;
+  // CHECK-NEXT: %[[NOTDEFCTOR:.*]] = cir.alloca !rec_NonDefaultCtor, !cir.ptr<!rec_NonDefaultCtor>, ["notDefCtor", init]
+  HasDtor dtor;
+  // CHECK-NEXT: %[[DTOR:.*]] = cir.alloca !rec_HasDtor, !cir.ptr<!rec_HasDtor>, ["dtor"]
+  int someIntArr[5];
+  // CHECK-NEXT: %[[INTARR:.*]] = cir.alloca !cir.array<!s32i x 5>, !cir.ptr<!cir.array<!s32i x 5>>, ["someIntArr"]
+  float someFloatArr[5];
+  // CHECK-NEXT: %[[FLOATARR:.*]] = cir.alloca !cir.array<!cir.float x 5>, !cir.ptr<!cir.array<!cir.float x 5>>, ["someFloatArr"]
+  NoCopyConstruct noCopyArr[5];
+  // CHECK-NEXT: %[[NOCOPYARR:.*]] = cir.alloca !cir.array<!rec_NoCopyConstruct x 5>, !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>, ["noCopyArr"]
+  CopyConstruct hasCopyArr[5];
+  // CHECK-NEXT: %[[HASCOPYARR:.*]] = cir.alloca !cir.array<!rec_CopyConstruct x 5>, !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>, ["hasCopyArr"]
+  NonDefaultCtor notDefCtorArr[5];
+  // CHECK-NEXT: %[[NOTDEFCTORARR:.*]] = cir.alloca !cir.array<!rec_NonDefaultCtor x 5>, !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>, ["notDefCtorArr", init]
+  HasDtor dtorArr[5];
+  // CHECK-NEXT: %[[DTORARR:.*]] = cir.alloca !cir.array<!rec_HasDtor x 5>, !cir.ptr<!cir.array<!rec_HasDtor x 5>>, ["dtorArr"]
+  // CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1Ev(%[[NOTDEFCTOR]]) : (!cir.ptr<!rec_NonDefaultCtor>) -> ()
+
+#pragma acc parallel loop firstprivate(someInt)
+  for(int i = 0; i < 5; ++i);
+  // CHECK: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[SOMEINT]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "someInt"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSi -> %[[PRIVATE]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial loop firstprivate(someFloat)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[SOMEFLOAT]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {name = "someFloat"}
+  // CHECK-NEXT: acc.serial combined(loop) firstprivate(@firstprivatization__ZTSf -> %[[PRIVATE]] : !cir.ptr<!cir.float>) {
+  // CHECK-NEXT: acc.loop combined(serial)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel loop firstprivate(noCopy)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOCOPY]] : !cir.ptr<!rec_NoCopyConstruct>) -> !cir.ptr<!rec_NoCopyConstruct> {name = "noCopy"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTS15NoCopyConstruct -> %[[PRIVATE]] : !cir.ptr<!rec_NoCopyConstruct>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial loop firstprivate(hasCopy)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[HASCOPY]] : !cir.ptr<!rec_CopyConstruct>) -> !cir.ptr<!rec_CopyConstruct> {name = "hasCopy"}
+  // CHECK-NEXT: acc.serial combined(loop) firstprivate(@firstprivatization__ZTS13CopyConstruct -> %[[PRIVATE]] : !cir.ptr<!rec_CopyConstruct>) {
+  // CHECK-NEXT: acc.loop combined(serial)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial loop firstprivate(notDefCtor)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTOR]] : !cir.ptr<!rec_NonDefaultCtor>) -> !cir.ptr<!rec_NonDefaultCtor> {name = "notDefCtor"}
+  // CHECK-NEXT: acc.serial combined(loop) firstprivate(@firstprivatization__ZTS14NonDefaultCtor -> %[[PRIVATE]] : !cir.ptr<!rec_NonDefaultCtor>) {
+  // CHECK-NEXT: acc.loop combined(serial)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial loop firstprivate(dtor)
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[DTOR]] : !cir.ptr<!rec_HasDtor>) -> !cir.ptr<!rec_HasDtor> {name = "dtor"}
+  // CHECK-NEXT: acc.serial combined(loop) firstprivate(@firstprivatization__ZTS7HasDtor -> %[[PRIVATE]] : !cir.ptr<!rec_HasDtor>) {
+  // CHECK-NEXT: acc.loop combined(serial)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel loop firstprivate(someInt, someFloat, noCopy, hasCopy, notDefCtor, dtor)
+  for(int i = 0; i < 5; ++i);
+  // CHECK: %[[PRIVATE1:.*]] = acc.firstprivate varPtr(%[[SOMEINT]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "someInt"}
+  // CHECK-NEXT: %[[PRIVATE2:.*]] = acc.firstprivate varPtr(%[[SOMEFLOAT]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {name = "someFloat"}
+  // CHECK-NEXT: %[[PRIVATE3:.*]] = acc.firstprivate varPtr(%[[NOCOPY]] : !cir.ptr<!rec_NoCopyConstruct>) -> !cir.ptr<!rec_NoCopyConstruct> {name = "noCopy"}
+  // CHECK-NEXT: %[[PRIVATE4:.*]] = acc.firstprivate varPtr(%[[HASCOPY]] : !cir.ptr<!rec_CopyConstruct>) -> !cir.ptr<!rec_CopyConstruct> {name = "hasCopy"}
+  // CHECK-NEXT: %[[PRIVATE5:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTOR]] : !cir.ptr<!rec_NonDefaultCtor>) -> !cir.ptr<!rec_NonDefaultCtor> {name = "notDefCtor"}
+  // CHECK-NEXT: %[[PRIVATE6:.*]] = acc.firstprivate varPtr(%[[DTOR]] : !cir.ptr<!rec_HasDtor>) -> !cir.ptr<!rec_HasDtor> {name = "dtor"}
+  // CHECK: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSi -> %[[PRIVATE1]] : !cir.ptr<!s32i>,
+  // CHECK-SAME: @firstprivatization__ZTSf -> %[[PRIVATE2]] : !cir.ptr<!cir.float>,
+  // CHECK-SAME: @firstprivatization__ZTS15NoCopyConstruct -> %[[PRIVATE3]] : !cir.ptr<!rec_NoCopyConstruct>,
+  // CHECK-SAME: @firstprivatization__ZTS13CopyConstruct -> %[[PRIVATE4]] : !cir.ptr<!rec_CopyConstruct>,
+  // CHECK-SAME: @firstprivatization__ZTS14NonDefaultCtor -> %[[PRIVATE5]] : !cir.ptr<!rec_NonDefaultCtor>,
+  // CHECK-SAME: @firstprivatization__ZTS7HasDtor -> %[[PRIVATE6]] : !cir.ptr<!rec_HasDtor>)
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial loop firstprivate(someIntArr[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[INTARR]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {name = "someIntArr[1]"}
+  // CHECK-NEXT: acc.serial combined(loop) firstprivate(@firstprivatization__ZTSA5_i -> %[[PRIVATE]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel loop firstprivate(someFloatArr[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[FLOATARR]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {name = "someFloatArr[1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSA5_f -> %[[PRIVATE]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial loop firstprivate(noCopyArr[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOCOPYARR]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {name = "noCopyArr[1]"}
+  // CHECK-NEXT: acc.serial combined(loop) firstprivate(@firstprivatization__ZTSA5_15NoCopyConstruct -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel loop firstprivate(hasCopyArr[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[HASCOPYARR]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {name = "hasCopyArr[1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSA5_13CopyConstruct -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel loop firstprivate(notDefCtorArr[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTORARR]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {name = "notDefCtorArr[1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSA5_14NonDefaultCtor -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel loop firstprivate(dtorArr[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[DTORARR]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_HasDtor x 5>> {name = "dtorArr[1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSA5_7HasDtor -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial loop firstprivate(someIntArr[1], someFloatArr[1], noCopyArr[1], hasCopyArr[1], notDefCtorArr[1], dtorArr[1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE1:.*]] = acc.firstprivate varPtr(%[[INTARR]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {name = "someIntArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE2:.*]] = acc.firstprivate varPtr(%[[FLOATARR]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {name = "someFloatArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE3:.*]] = acc.firstprivate varPtr(%[[NOCOPYARR]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {name = "noCopyArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE4:.*]] = acc.firstprivate varPtr(%[[HASCOPYARR]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {name = "hasCopyArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE5:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTORARR]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {name = "notDefCtorArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE6:.*]] = acc.firstprivate varPtr(%[[DTORARR]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_HasDtor x 5>> {name = "dtorArr[1]"}
+  // CHECK-NEXT: acc.serial combined(loop) firstprivate(@firstprivatization__ZTSA5_i -> %[[PRIVATE1]] : !cir.ptr<!cir.array<!s32i x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_f -> %[[PRIVATE2]] : !cir.ptr<!cir.array<!cir.float x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_15NoCopyConstruct -> %[[PRIVATE3]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_13CopyConstruct -> %[[PRIVATE4]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_14NonDefaultCtor -> %[[PRIVATE5]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_7HasDtor -> %[[PRIVATE6]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>)
+  // CHECK-NEXT: acc.loop combined(serial)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel loop firstprivate(someIntArr[1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[INTARR]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {name = "someIntArr[1:1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSA5_i -> %[[PRIVATE]] : !cir.ptr<!cir.array<!s32i x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial loop firstprivate(someFloatArr[1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[FLOATARR]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {name = "someFloatArr[1:1]"}
+  // CHECK-NEXT: acc.serial combined(loop) firstprivate(@firstprivatization__ZTSA5_f -> %[[PRIVATE]] : !cir.ptr<!cir.array<!cir.float x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel loop firstprivate(noCopyArr[1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOCOPYARR]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {name = "noCopyArr[1:1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSA5_15NoCopyConstruct -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial loop firstprivate(hasCopyArr[1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[HASCOPYARR]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {name = "hasCopyArr[1:1]"}
+  // CHECK-NEXT: acc.serial combined(loop) firstprivate(@firstprivatization__ZTSA5_13CopyConstruct -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) {
+  // CHECK-NEXT: acc.loop combined(serial)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel loop firstprivate(notDefCtorArr[1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTORARR]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {name = "notDefCtorArr[1:1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSA5_14NonDefaultCtor -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel loop firstprivate(dtorArr[1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[DTORARR]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_HasDtor x 5>> {name = "dtorArr[1:1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSA5_7HasDtor -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) {
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel loop firstprivate(someIntArr[1:1], someFloatArr[1:1], noCopyArr[1:1], hasCopyArr[1:1], notDefCtorArr[1:1], dtorArr[1:1])
+  for(int i = 0; i < 5; ++i);
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE1:.*]] = acc.firstprivate varPtr(%[[INTARR]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {name = "someIntArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE2:.*]] = acc.firstprivate varPtr(%[[FLOATARR]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {name = "someFloatArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE3:.*]] = acc.firstprivate varPtr(%[[NOCOPYARR]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {name = "noCopyArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE4:.*]] = acc.firstprivate varPtr(%[[HASCOPYARR]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {name = "hasCopyArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE5:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTORARR]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {name = "notDefCtorArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE6:.*]] = acc.firstprivate varPtr(%[[DTORARR]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_HasDtor x 5>> {name = "dtorArr[1:1]"}
+  // CHECK-NEXT: acc.parallel combined(loop) firstprivate(@firstprivatization__ZTSA5_i -> %[[PRIVATE1]] : !cir.ptr<!cir.array<!s32i x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_f -> %[[PRIVATE2]] : !cir.ptr<!cir.array<!cir.float x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_15NoCopyConstruct -> %[[PRIVATE3]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_13CopyConstruct -> %[[PRIVATE4]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_14NonDefaultCtor -> %[[PRIVATE5]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_7HasDtor -> %[[PRIVATE6]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>)
+  // CHECK-NEXT: acc.loop combined(parallel)
+  // CHECK: acc.yield
+  // CHECK-NEXT: } loc
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause-templates.cpp b/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause-templates.cpp
new file mode 100644
index 0000000000000..50b3219128b1f
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause-templates.cpp
@@ -0,0 +1,95 @@
+// RUN: %clang_cc1 -fopenacc -triple x86_64-linux-gnu -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir -triple x86_64-linux-pc %s -o - | FileCheck %s
+
+struct CopyConstruct {
+  CopyConstruct() = default;
+  CopyConstruct(const CopyConstruct&);
+};
+
+struct NonDefaultCtor {
+  NonDefaultCtor();
+};
+
+struct HasDtor {
+  ~HasDtor();
+};
+
+// CHECK: acc.firstprivate.recipe @firstprivatization__ZTSi : !cir.ptr<!s32i> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!s32i> {{.*}}):
+// CHECK-NEXT: cir.alloca !s32i, !cir.ptr<!s32i>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!s32i> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!s32i> {{.*}}):
+// CHECK-NEXT: %[[LOAD:.*]] = cir.load {{.*}} %[[ARG_FROM]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store{{.*}} %[[LOAD]], %[[ARG_TO]] : !s32i, !cir.ptr<!s32i>
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS7HasDtor : !cir.ptr<!rec_HasDtor> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
+// CHECK-NEXT: cir.alloca !rec_HasDtor, !cir.ptr<!rec_HasDtor>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } destroy {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[ARG]]) nothrow : (!cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS14NonDefaultCtor : !cir.ptr<!rec_NonDefaultCtor> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}):
+// CHECK-NEXT: %[[ALLOCA:.*]] = cir.alloca !rec_NonDefaultCtor, !cir.ptr<!rec_NonDefaultCtor>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS13CopyConstruct : !cir.ptr<!rec_CopyConstruct> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}):
+// CHECK-NEXT: cir.alloca !rec_CopyConstruct, !cir.ptr<!rec_CopyConstruct>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}):
+// CHECK-NEXT:  cir.call @_ZN13CopyConstructC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+
+template<typename T, typename U, typename V, typename W>
+void dependent_version(const T &cc, const U &ndc, const V &dtor, const W &someInt) {
+  // CHECK: cir.func {{.*}}@_Z17dependent_versionI13CopyConstruct14NonDefaultCtor7HasDtoriEvRKT_RKT0_RKT1_RKT2_(%[[ARG0:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}, %[[ARG1:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}, %[[ARG2:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG3:.*]]: !cir.ptr<!s32i> {{.*}}) {
+  // CHECK-NEXT: %[[CC:.*]] = cir.alloca !cir.ptr<!rec_CopyConstruct>, !cir.ptr<!cir.ptr<!rec_CopyConstruct>>, ["cc", init, const]
+  // CHECK-NEXT: %[[NDC:.*]] = cir.alloca !cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!cir.ptr<!rec_NonDefaultCtor>>, ["ndc", init, const]
+  // CHECK-NEXT: %[[DTOR:.*]] = cir.alloca !cir.ptr<!rec_HasDtor>, !cir.ptr<!cir.ptr<!rec_HasDtor>>, ["dtor", init, const]
+  // CHECK-NEXT: %[[SOMEINT:.*]] = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["someInt", init, const]
+  //             %             3 = cir.alloca !cir.ptr<!s32i>, !cir.ptr<!cir.ptr<!s32i>>, ["someInt", init, const]
+
+#pragma acc parallel firstprivate(cc, ndc, dtor, someInt)
+  ;
+  // CHECK: %[[PRIV_LOAD:.*]] = cir.load %[[CC]] : !cir.ptr<!cir.ptr<!rec_CopyConstruct>>, !cir.ptr<!rec_CopyConstruct>
+  // CHECK-NEXT: %[[PRIVATE1:.*]] = acc.firstprivate varPtr(%[[PRIV_LOAD]] : !cir.ptr<!rec_CopyConstruct>) -> !cir.ptr<!rec_CopyConstruct> {name = "cc"}
+  // CHECK-NEXT: %[[PRIV_LOAD:.*]] = cir.load %[[NDC]] : !cir.ptr<!cir.ptr<!rec_NonDefaultCtor>>, !cir.ptr<!rec_NonDefaultCtor>
+  // CHECK-NEXT: %[[PRIVATE2:.*]] = acc.firstprivate varPtr(%[[PRIV_LOAD]] : !cir.ptr<!rec_NonDefaultCtor>) -> !cir.ptr<!rec_NonDefaultCtor> {name = "ndc"}
+  // CHECK-NEXT: %[[PRIV_LOAD:.*]] = cir.load %[[DTOR]] : !cir.ptr<!cir.ptr<!rec_HasDtor>>, !cir.ptr<!rec_HasDtor>
+  // CHECK-NEXT: %[[PRIVATE3:.*]] = acc.firstprivate varPtr(%[[PRIV_LOAD]] : !cir.ptr<!rec_HasDtor>) -> !cir.ptr<!rec_HasDtor> {name = "dtor"}
+  // CHECK-NEXT: %[[PRIV_LOAD:.*]] = cir.load %[[SOMEINT]] : !cir.ptr<!cir.ptr<!s32i>>, !cir.ptr<!s32i>
+  // CHECK-NEXT: %[[PRIVATE4:.*]] = acc.firstprivate varPtr(%[[PRIV_LOAD]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "someInt"}
+
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTS13CopyConstruct -> %[[PRIVATE1]] : !cir.ptr<!rec_CopyConstruct>, 
+  // CHECK-SAME: @firstprivatization__ZTS14NonDefaultCtor -> %[[PRIVATE2]] : !cir.ptr<!rec_NonDefaultCtor>,
+  // CHECK-SAME: @firstprivatization__ZTS7HasDtor -> %[[PRIVATE3]] : !cir.ptr<!rec_HasDtor>,
+  // CHECK-SAME: @firstprivatization__ZTSi -> %[[PRIVATE4]] : !cir.ptr<!s32i>) {
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+}
+
+void use() {
+  CopyConstruct cc;
+  NonDefaultCtor ndc;
+  HasDtor dtor;
+  int i;
+  dependent_version(cc, ndc, dtor, i);
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause.cpp b/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause.cpp
new file mode 100644
index 0000000000000..2bc94a88fa52b
--- /dev/null
+++ b/clang/test/CIR/CodeGenOpenACC/compute-firstprivate-clause.cpp
@@ -0,0 +1,716 @@
+// RUN: %clang_cc1 -fopenacc -triple x86_64-linux-gnu -Wno-openacc-self-if-potential-conflict -emit-cir -fclangir -triple x86_64-linux-pc %s -o - | FileCheck %s
+
+struct NoCopyConstruct {};
+
+struct CopyConstruct {
+  CopyConstruct() = default;
+  CopyConstruct(const CopyConstruct&);
+};
+
+struct NonDefaultCtor {
+  NonDefaultCtor();
+};
+
+struct HasDtor {
+  ~HasDtor();
+};
+
+// CHECK: acc.firstprivate.recipe @firstprivatization__ZTSA5_7HasDtor : !cir.ptr<!cir.array<!rec_HasDtor x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_HasDtor x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!rec_HasDtor x 5>, !cir.ptr<!cir.array<!rec_HasDtor x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!cir.array<!rec_HasDtor x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!rec_HasDtor x 5>> {{.*}}):
+// CHECK-NEXT: %[[DECAY_TO:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor> 
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[ZERO]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[DECAY_TO]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_HasDtor>, %[[ONE]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[ONE_2]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_HasDtor>, %[[TWO]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[TWO_2]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_HasDtor>, %[[THREE]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[THREE_2]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_HasDtor>, %[[FOUR]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_HasDtor>, %[[FOUR_2]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+//
+// CHECK-NEXT: acc.yield
+//
+// CHECK-NEXT: } destroy {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_HasDtor x 5>> {{.*}}):
+// CHECK-NEXT: %[[LAST_IDX:.*]] = cir.const #cir.int<4> : !u64i
+// CHECK-NEXT: %[[ARRPTR:.*]] = cir.cast(array_to_ptrdecay, %[[ARG]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[ELEM:.*]] = cir.ptr_stride(%[[ARRPTR]] : !cir.ptr<!rec_HasDtor>, %[[LAST_IDX]] : !u64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[ITR:.*]] = cir.alloca !cir.ptr<!rec_HasDtor>, !cir.ptr<!cir.ptr<!rec_HasDtor>>, ["__array_idx"]
+// CHECK-NEXT: cir.store %[[ELEM]], %[[ITR]] : !cir.ptr<!rec_HasDtor>, !cir.ptr<!cir.ptr<!rec_HasDtor>>
+// CHECK-NEXT: cir.do {
+// CHECK-NEXT: %[[ELEM_LOAD:.*]] = cir.load %[[ITR]] : !cir.ptr<!cir.ptr<!rec_HasDtor>>, !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[ELEM_LOAD]]) nothrow : (!cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: %[[NEG_ONE:.*]] =  cir.const #cir.int<-1> : !s64i
+// CHECK-NEXT: %[[PREVELEM:.*]] = cir.ptr_stride(%[[ELEM_LOAD]] : !cir.ptr<!rec_HasDtor>, %[[NEG_ONE]] : !s64i), !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: cir.store %[[PREVELEM]], %[[ITR]] : !cir.ptr<!rec_HasDtor>, !cir.ptr<!cir.ptr<!rec_HasDtor>>
+// CHECK-NEXT: cir.yield
+// CHECK-NEXT: } while {
+// CHECK-NEXT: %[[ELEM_LOAD:.*]] = cir.load %[[ITR]] : !cir.ptr<!cir.ptr<!rec_HasDtor>>, !cir.ptr<!rec_HasDtor>
+// CHECK-NEXT: %[[CMP:.*]] = cir.cmp(ne, %[[ELEM_LOAD]], %[[ARRPTR]]) : !cir.ptr<!rec_HasDtor>, !cir.bool
+// CHECK-NEXT: cir.condition(%[[CMP]])
+// CHECK-NEXT: }
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_14NonDefaultCtor : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {{.*}}):
+// CHECK-NEXT: %[[ALLOCA:.*]] = cir.alloca !cir.array<!rec_NonDefaultCtor x 5>, !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!rec_NonDefaultCtor>, %[[ZERO]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_DECAY]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NonDefaultCtor>, %[[ONE]] : !s64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NonDefaultCtor>, %[[ONE_2]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NonDefaultCtor>, %[[TWO]] : !s64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NonDefaultCtor>, %[[TWO_2]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NonDefaultCtor>, %[[THREE]] : !s64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NonDefaultCtor>, %[[THREE_2]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NonDefaultCtor>, %[[FOUR]] : !s64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NonDefaultCtor>, %[[FOUR_2]] : !u64i), !cir.ptr<!rec_NonDefaultCtor>
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+//
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_13CopyConstruct : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!rec_CopyConstruct x 5>, !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!rec_CopyConstruct>, %[[ZERO]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_DECAY]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_CopyConstruct>, %[[ONE]] : !s64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_CopyConstruct>, %[[ONE_2]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_CopyConstruct>, %[[TWO]] : !s64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_CopyConstruct>, %[[TWO_2]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_CopyConstruct>, %[[THREE]] : !s64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_CopyConstruct>, %[[THREE_2]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_CopyConstruct>, %[[FOUR]] : !s64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_CopyConstruct>, %[[FOUR_2]] : !u64i), !cir.ptr<!rec_CopyConstruct>
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+//
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_15NoCopyConstruct : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!rec_NoCopyConstruct x 5>, !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!rec_NoCopyConstruct>, %[[ZERO]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_DECAY]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NoCopyConstruct>, %[[ONE]] : !s64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NoCopyConstruct>, %[[ONE_2]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NoCopyConstruct>, %[[TWO]] : !s64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NoCopyConstruct>, %[[TWO_2]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NoCopyConstruct>, %[[THREE]] : !s64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NoCopyConstruct>, %[[THREE_2]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!rec_NoCopyConstruct>, %[[FOUR]] : !s64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!rec_NoCopyConstruct>, %[[FOUR_2]] : !u64i), !cir.ptr<!rec_NoCopyConstruct>
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[TO_OFFSET]], %[[FROM_OFFSET]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+//
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_f : !cir.ptr<!cir.array<!cir.float x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!cir.float x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!cir.float x 5>, !cir.ptr<!cir.array<!cir.float x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!cir.float x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!cir.float x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!cir.float>, %[[ZERO]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_DECAY]] : !cir.float, !cir.ptr<!cir.float>
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!cir.float>, %[[ONE]] : !s64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!cir.float>, %[[ONE_2]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !cir.float, !cir.ptr<!cir.float>
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!cir.float>, %[[TWO]] : !s64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!cir.float>, %[[TWO_2]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !cir.float, !cir.ptr<!cir.float>
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!cir.float>, %[[THREE]] : !s64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!cir.float>, %[[THREE_2]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !cir.float, !cir.ptr<!cir.float>
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!cir.float>, %[[FOUR]] : !s64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!cir.float x 5>>), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!cir.float>, %[[FOUR_2]] : !u64i), !cir.ptr<!cir.float>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !cir.float, !cir.ptr<!cir.float>
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSA5_i : !cir.ptr<!cir.array<!s32i x 5>> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.array<!s32i x 5>> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.array<!s32i x 5>, !cir.ptr<!cir.array<!s32i x 5>>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FORM:.*]]: !cir.ptr<!cir.array<!s32i x 5>> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.array<!s32i x 5>> {{.*}}):
+// CHECK-NEXT: %[[TO_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_TO]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[ZERO:.*]] = cir.const #cir.int<0>
+// CHECK-NEXT: %[[FROM_DECAY:.*]] = cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[FROM_DECAY]] : !cir.ptr<!s32i>, %[[ZERO]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_DECAY]] : !s32i, !cir.ptr<!s32i>
+//
+// CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!s32i>, %[[ONE]] : !s64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[ONE_2:.*]] = cir.const #cir.int<1>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!s32i>, %[[ONE_2]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !s32i, !cir.ptr<!s32i>
+//
+// CHECK-NEXT: %[[TWO:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!s32i>, %[[TWO]] : !s64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[TWO_2:.*]] = cir.const #cir.int<2>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!s32i>, %[[TWO_2]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !s32i, !cir.ptr<!s32i>
+//
+// CHECK-NEXT: %[[THREE:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!s32i>, %[[THREE]] : !s64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[THREE_2:.*]] = cir.const #cir.int<3>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!s32i>, %[[THREE_2]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !s32i, !cir.ptr<!s32i>
+//
+// CHECK-NEXT: %[[FOUR:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[TO_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_TO]] : !cir.ptr<!s32i>, %[[FOUR]] : !s64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FOUR_2:.*]] = cir.const #cir.int<4>
+// CHECK-NEXT: %[[DECAY_FROM:.*]] =  cir.cast(array_to_ptrdecay, %[[ARG_FROM]] : !cir.ptr<!cir.array<!s32i x 5>>), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_OFFSET:.*]] = cir.ptr_stride(%[[DECAY_FROM]] : !cir.ptr<!s32i>, %[[FOUR_2]] : !u64i), !cir.ptr<!s32i>
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[FROM_OFFSET]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[TO_OFFSET]] : !s32i, !cir.ptr<!s32i>
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS7HasDtor : !cir.ptr<!rec_HasDtor> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
+// CHECK-NEXT: cir.alloca !rec_HasDtor, !cir.ptr<!rec_HasDtor>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN7HasDtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_HasDtor>, !cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } destroy {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_HasDtor> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN7HasDtorD1Ev(%[[ARG]]) nothrow : (!cir.ptr<!rec_HasDtor>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS14NonDefaultCtor : !cir.ptr<!rec_NonDefaultCtor> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}):
+// CHECK-NEXT: %[[ALLOCA:.*]] = cir.alloca !rec_NonDefaultCtor, !cir.ptr<!rec_NonDefaultCtor>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NonDefaultCtor> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NonDefaultCtor>, !cir.ptr<!rec_NonDefaultCtor>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS13CopyConstruct : !cir.ptr<!rec_CopyConstruct> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}):
+// CHECK-NEXT: cir.alloca !rec_CopyConstruct, !cir.ptr<!rec_CopyConstruct>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_CopyConstruct> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN13CopyConstructC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) : (!cir.ptr<!rec_CopyConstruct>, !cir.ptr<!rec_CopyConstruct>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTS15NoCopyConstruct : !cir.ptr<!rec_NoCopyConstruct> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}):
+// CHECK-NEXT: cir.alloca !rec_NoCopyConstruct, !cir.ptr<!rec_NoCopyConstruct>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!rec_NoCopyConstruct> {{.*}}):
+// CHECK-NEXT: cir.call @_ZN15NoCopyConstructC1ERKS_(%[[ARG_TO]], %[[ARG_FROM]]) nothrow : (!cir.ptr<!rec_NoCopyConstruct>, !cir.ptr<!rec_NoCopyConstruct>) -> ()
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSf : !cir.ptr<!cir.float> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!cir.float> {{.*}}):
+// CHECK-NEXT: cir.alloca !cir.float, !cir.ptr<!cir.float>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!cir.float> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!cir.float> {{.*}}):
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[ARG_FROM]] : !cir.ptr<!cir.float>, !cir.float
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[ARG_TO]] : !cir.float, !cir.ptr<!cir.float>
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+//
+// CHECK-NEXT: acc.firstprivate.recipe @firstprivatization__ZTSi : !cir.ptr<!s32i> init {
+// CHECK-NEXT: ^bb0(%[[ARG:.*]]: !cir.ptr<!s32i> {{.*}}):
+// CHECK-NEXT: cir.alloca !s32i, !cir.ptr<!s32i>, ["openacc.firstprivate.init"]
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: } copy {
+// CHECK-NEXT: ^bb0(%[[ARG_FROM:.*]]: !cir.ptr<!s32i> {{.*}}, %[[ARG_TO:.*]]: !cir.ptr<!s32i> {{.*}}):
+// CHECK-NEXT: %[[FROM_LOAD:.*]] = cir.load {{.*}}%[[ARG_FROM]] : !cir.ptr<!s32i>, !s32i
+// CHECK-NEXT: cir.store {{.*}} %[[FROM_LOAD]], %[[ARG_TO]] : !s32i, !cir.ptr<!s32i>
+// CHECK-NEXT: acc.yield
+// CHECK-NEXT: }
+
+extern "C" void acc_compute() {
+  // CHECK: cir.func{{.*}} @acc_compute() {
+
+  int someInt;
+  // CHECK-NEXT: %[[SOMEINT:.*]] = cir.alloca !s32i, !cir.ptr<!s32i>, ["someInt"]
+  float someFloat;
+  // CHECK-NEXT: %[[SOMEFLOAT:.*]] = cir.alloca !cir.float, !cir.ptr<!cir.float>, ["someFloat"]
+  NoCopyConstruct noCopy;
+  // CHECK-NEXT: %[[NOCOPY:.*]] = cir.alloca !rec_NoCopyConstruct, !cir.ptr<!rec_NoCopyConstruct>, ["noCopy"]
+  CopyConstruct hasCopy;
+  // CHECK-NEXT: %[[HASCOPY:.*]] = cir.alloca !rec_CopyConstruct, !cir.ptr<!rec_CopyConstruct>, ["hasCopy"]
+  NonDefaultCtor notDefCtor;
+  // CHECK-NEXT: %[[NOTDEFCTOR:.*]] = cir.alloca !rec_NonDefaultCtor, !cir.ptr<!rec_NonDefaultCtor>, ["notDefCtor", init]
+  HasDtor dtor;
+  // CHECK-NEXT: %[[DTOR:.*]] = cir.alloca !rec_HasDtor, !cir.ptr<!rec_HasDtor>, ["dtor"]
+  int someIntArr[5];
+  // CHECK-NEXT: %[[INTARR:.*]] = cir.alloca !cir.array<!s32i x 5>, !cir.ptr<!cir.array<!s32i x 5>>, ["someIntArr"]
+  float someFloatArr[5];
+  // CHECK-NEXT: %[[FLOATARR:.*]] = cir.alloca !cir.array<!cir.float x 5>, !cir.ptr<!cir.array<!cir.float x 5>>, ["someFloatArr"]
+  NoCopyConstruct noCopyArr[5];
+  // CHECK-NEXT: %[[NOCOPYARR:.*]] = cir.alloca !cir.array<!rec_NoCopyConstruct x 5>, !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>, ["noCopyArr"]
+  CopyConstruct hasCopyArr[5];
+  // CHECK-NEXT: %[[HASCOPYARR:.*]] = cir.alloca !cir.array<!rec_CopyConstruct x 5>, !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>, ["hasCopyArr"]
+  NonDefaultCtor notDefCtorArr[5];
+  // CHECK-NEXT: %[[NOTDEFCTORARR:.*]] = cir.alloca !cir.array<!rec_NonDefaultCtor x 5>, !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>, ["notDefCtorArr", init]
+  HasDtor dtorArr[5];
+  // CHECK-NEXT: %[[DTORARR:.*]] = cir.alloca !cir.array<!rec_HasDtor x 5>, !cir.ptr<!cir.array<!rec_HasDtor x 5>>, ["dtorArr"]
+  // CHECK-NEXT: cir.call @_ZN14NonDefaultCtorC1Ev(%[[NOTDEFCTOR]]) : (!cir.ptr<!rec_NonDefaultCtor>) -> ()
+
+#pragma acc parallel firstprivate(someInt)
+  ;
+  // CHECK: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[SOMEINT]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "someInt"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSi -> %[[PRIVATE]] : !cir.ptr<!s32i>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial firstprivate(someFloat)
+  ;
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[SOMEFLOAT]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {name = "someFloat"}
+  // CHECK-NEXT: acc.serial firstprivate(@firstprivatization__ZTSf -> %[[PRIVATE]] : !cir.ptr<!cir.float>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel firstprivate(noCopy)
+  ;
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOCOPY]] : !cir.ptr<!rec_NoCopyConstruct>) -> !cir.ptr<!rec_NoCopyConstruct> {name = "noCopy"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTS15NoCopyConstruct -> %[[PRIVATE]] : !cir.ptr<!rec_NoCopyConstruct>
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial firstprivate(hasCopy)
+  ;
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[HASCOPY]] : !cir.ptr<!rec_CopyConstruct>) -> !cir.ptr<!rec_CopyConstruct> {name = "hasCopy"}
+  // CHECK-NEXT: acc.serial firstprivate(@firstprivatization__ZTS13CopyConstruct -> %[[PRIVATE]] : !cir.ptr<!rec_CopyConstruct>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial firstprivate(notDefCtor)
+  ;
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTOR]] : !cir.ptr<!rec_NonDefaultCtor>) -> !cir.ptr<!rec_NonDefaultCtor> {name = "notDefCtor"}
+  // CHECK-NEXT: acc.serial firstprivate(@firstprivatization__ZTS14NonDefaultCtor -> %[[PRIVATE]] : !cir.ptr<!rec_NonDefaultCtor>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial firstprivate(dtor)
+  ;
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[DTOR]] : !cir.ptr<!rec_HasDtor>) -> !cir.ptr<!rec_HasDtor> {name = "dtor"}
+  // CHECK-NEXT: acc.serial firstprivate(@firstprivatization__ZTS7HasDtor -> %[[PRIVATE]] : !cir.ptr<!rec_HasDtor>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel firstprivate(someInt, someFloat, noCopy, hasCopy, notDefCtor, dtor)
+  ;
+  // CHECK: %[[PRIVATE1:.*]] = acc.firstprivate varPtr(%[[SOMEINT]] : !cir.ptr<!s32i>) -> !cir.ptr<!s32i> {name = "someInt"}
+  // CHECK-NEXT: %[[PRIVATE2:.*]] = acc.firstprivate varPtr(%[[SOMEFLOAT]] : !cir.ptr<!cir.float>) -> !cir.ptr<!cir.float> {name = "someFloat"}
+  // CHECK-NEXT: %[[PRIVATE3:.*]] = acc.firstprivate varPtr(%[[NOCOPY]] : !cir.ptr<!rec_NoCopyConstruct>) -> !cir.ptr<!rec_NoCopyConstruct> {name = "noCopy"}
+  // CHECK-NEXT: %[[PRIVATE4:.*]] = acc.firstprivate varPtr(%[[HASCOPY]] : !cir.ptr<!rec_CopyConstruct>) -> !cir.ptr<!rec_CopyConstruct> {name = "hasCopy"}
+  // CHECK-NEXT: %[[PRIVATE5:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTOR]] : !cir.ptr<!rec_NonDefaultCtor>) -> !cir.ptr<!rec_NonDefaultCtor> {name = "notDefCtor"}
+  // CHECK-NEXT: %[[PRIVATE6:.*]] = acc.firstprivate varPtr(%[[DTOR]] : !cir.ptr<!rec_HasDtor>) -> !cir.ptr<!rec_HasDtor> {name = "dtor"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSi -> %[[PRIVATE1]] : !cir.ptr<!s32i>,
+  // CHECK-SAME: @firstprivatization__ZTSf -> %[[PRIVATE2]] : !cir.ptr<!cir.float>,
+  // CHECK-SAME: @firstprivatization__ZTS15NoCopyConstruct -> %[[PRIVATE3]] : !cir.ptr<!rec_NoCopyConstruct>,
+  // CHECK-SAME: @firstprivatization__ZTS13CopyConstruct -> %[[PRIVATE4]] : !cir.ptr<!rec_CopyConstruct>,
+  // CHECK-SAME: @firstprivatization__ZTS14NonDefaultCtor -> %[[PRIVATE5]] : !cir.ptr<!rec_NonDefaultCtor>,
+  // CHECK-SAME: @firstprivatization__ZTS7HasDtor -> %[[PRIVATE6]] : !cir.ptr<!rec_HasDtor>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc serial firstprivate(someIntArr[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[INTARR]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {name = "someIntArr[1]"}
+  // CHECK-NEXT: acc.serial firstprivate(@firstprivatization__ZTSA5_i -> %[[PRIVATE]] : !cir.ptr<!cir.array<!s32i x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel firstprivate(someFloatArr[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[FLOATARR]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {name = "someFloatArr[1]"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSA5_f -> %[[PRIVATE]] : !cir.ptr<!cir.array<!cir.float x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial firstprivate(noCopyArr[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOCOPYARR]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {name = "noCopyArr[1]"}
+  // CHECK-NEXT: acc.serial firstprivate(@firstprivatization__ZTSA5_15NoCopyConstruct -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel firstprivate(hasCopyArr[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[HASCOPYARR]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {name = "hasCopyArr[1]"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSA5_13CopyConstruct -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel firstprivate(notDefCtorArr[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTORARR]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {name = "notDefCtorArr[1]"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSA5_14NonDefaultCtor -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel firstprivate(dtorArr[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[DTORARR]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_HasDtor x 5>> {name = "dtorArr[1]"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSA5_7HasDtor -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial firstprivate(someIntArr[1], someFloatArr[1], noCopyArr[1], hasCopyArr[1], notDefCtorArr[1], dtorArr[1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE1:.*]] = acc.firstprivate varPtr(%[[INTARR]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {name = "someIntArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE2:.*]] = acc.firstprivate varPtr(%[[FLOATARR]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {name = "someFloatArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE3:.*]] = acc.firstprivate varPtr(%[[NOCOPYARR]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {name = "noCopyArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE4:.*]] = acc.firstprivate varPtr(%[[HASCOPYARR]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {name = "hasCopyArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE5:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTORARR]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {name = "notDefCtorArr[1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE_CONST:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CONST]] : i64) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE6:.*]] = acc.firstprivate varPtr(%[[DTORARR]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_HasDtor x 5>> {name = "dtorArr[1]"}
+  // CHECK-NEXT: acc.serial firstprivate(@firstprivatization__ZTSA5_i -> %[[PRIVATE1]] : !cir.ptr<!cir.array<!s32i x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_f -> %[[PRIVATE2]] : !cir.ptr<!cir.array<!cir.float x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_15NoCopyConstruct -> %[[PRIVATE3]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_13CopyConstruct -> %[[PRIVATE4]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_14NonDefaultCtor -> %[[PRIVATE5]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_7HasDtor -> %[[PRIVATE6]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+
+#pragma acc parallel firstprivate(someIntArr[1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[INTARR]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {name = "someIntArr[1:1]"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSA5_i -> %[[PRIVATE]] : !cir.ptr<!cir.array<!s32i x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial firstprivate(someFloatArr[1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[FLOATARR]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {name = "someFloatArr[1:1]"}
+  // CHECK-NEXT: acc.serial firstprivate(@firstprivatization__ZTSA5_f -> %[[PRIVATE]] : !cir.ptr<!cir.array<!cir.float x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel firstprivate(noCopyArr[1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOCOPYARR]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {name = "noCopyArr[1:1]"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSA5_15NoCopyConstruct -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc serial firstprivate(hasCopyArr[1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[HASCOPYARR]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {name = "hasCopyArr[1:1]"}
+  // CHECK-NEXT: acc.serial firstprivate(@firstprivatization__ZTSA5_13CopyConstruct -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel firstprivate(notDefCtorArr[1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTORARR]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {name = "notDefCtorArr[1:1]"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSA5_14NonDefaultCtor -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel firstprivate(dtorArr[1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE:.*]] = acc.firstprivate varPtr(%[[DTORARR]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_HasDtor x 5>> {name = "dtorArr[1:1]"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSA5_7HasDtor -> %[[PRIVATE]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+#pragma acc parallel firstprivate(someIntArr[1:1], someFloatArr[1:1], noCopyArr[1:1], hasCopyArr[1:1], notDefCtorArr[1:1], dtorArr[1:1])
+  ;
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE1:.*]] = acc.firstprivate varPtr(%[[INTARR]] : !cir.ptr<!cir.array<!s32i x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!s32i x 5>> {name = "someIntArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE2:.*]] = acc.firstprivate varPtr(%[[FLOATARR]] : !cir.ptr<!cir.array<!cir.float x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!cir.float x 5>> {name = "someFloatArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE3:.*]] = acc.firstprivate varPtr(%[[NOCOPYARR]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>> {name = "noCopyArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE4:.*]] = acc.firstprivate varPtr(%[[HASCOPYARR]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_CopyConstruct x 5>> {name = "hasCopyArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE5:.*]] = acc.firstprivate varPtr(%[[NOTDEFCTORARR]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>> {name = "notDefCtorArr[1:1]"}
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ONE:.*]] = cir.const #cir.int<1> : !s32i
+  // CHECK-NEXT: %[[ONE_CAST2:.*]] = builtin.unrealized_conversion_cast %[[ONE]] : !s32i to si32
+  // CHECK-NEXT: %[[ZERO_CONST:.*]] = arith.constant 0
+  // CHECK-NEXT: %[[ONE_CONST2:.*]] = arith.constant 1
+  // CHECK-NEXT: %[[BOUNDS:.*]] = acc.bounds lowerbound(%[[ONE_CAST]] : si32) extent(%[[ONE_CAST2]] : si32) stride(%[[ONE_CONST2]] : i64) startIdx(%[[ZERO_CONST]] : i64)
+  // CHECK-NEXT: %[[PRIVATE6:.*]] = acc.firstprivate varPtr(%[[DTORARR]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>) bounds(%[[BOUNDS]]) -> !cir.ptr<!cir.array<!rec_HasDtor x 5>> {name = "dtorArr[1:1]"}
+  // CHECK-NEXT: acc.parallel firstprivate(@firstprivatization__ZTSA5_i -> %[[PRIVATE1]] : !cir.ptr<!cir.array<!s32i x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_f -> %[[PRIVATE2]] : !cir.ptr<!cir.array<!cir.float x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_15NoCopyConstruct -> %[[PRIVATE3]] : !cir.ptr<!cir.array<!rec_NoCopyConstruct x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_13CopyConstruct -> %[[PRIVATE4]] : !cir.ptr<!cir.array<!rec_CopyConstruct x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_14NonDefaultCtor -> %[[PRIVATE5]] : !cir.ptr<!cir.array<!rec_NonDefaultCtor x 5>>,
+  // CHECK-SAME: @firstprivatization__ZTSA5_7HasDtor -> %[[PRIVATE6]] : !cir.ptr<!cir.array<!rec_HasDtor x 5>>)
+  // CHECK-NEXT: acc.yield
+  // CHECK-NEXT: } loc
+}
diff --git a/clang/test/CIR/CodeGenOpenACC/init.c b/clang/test/CIR/CodeGenOpenACC/init.c
index 177e5a6ea2117..805fb08dbf487 100644
--- a/clang/test/CIR/CodeGenOpenACC/init.c
+++ b/clang/test/CIR/CodeGenOpenACC/init.c
@@ -11,12 +11,8 @@ void acc_init(int cond) {
   // CHECK-NEXT: acc.init attributes {device_types = [#acc.device_type<star>]}
 #pragma acc init device_type(nvidia)
   // CHECK-NEXT: acc.init attributes {device_types = [#acc.device_type<nvidia>]}
-#pragma acc init device_type(host, multicore)
-  // CHECK-NEXT: acc.init attributes {device_types = [#acc.device_type<host>, #acc.device_type<multicore>]}
 #pragma acc init device_type(NVIDIA)
   // CHECK-NEXT: acc.init attributes {device_types = [#acc.device_type<nvidia>]}
-#pragma acc init device_type(HoSt, MuLtIcORe)
-  // CHECK-NEXT: acc.init attributes {device_types = [#acc.device_type<host>, #acc.device_type<multicore>]}
 #pragma acc init device_type(HoSt) device_type(MuLtIcORe)
   // CHECK-NEXT: acc.init attributes {device_types = [#acc.device_type<host>, #acc.device_type<multicore>]}
 
diff --git a/clang/test/CIR/CodeGenOpenACC/openacc-not-implemented.cpp b/clang/test/CIR/CodeGenOpenACC/openacc-not-implemented.cpp
index 0bf932ea62ceb..e6d33aeb3c7e5 100644
--- a/clang/test/CIR/CodeGenOpenACC/openacc-not-implemented.cpp
+++ b/clang/test/CIR/CodeGenOpenACC/openacc-not-implemented.cpp
@@ -2,17 +2,13 @@
 
 void HelloWorld(int *A, int *B, int *C, int N) {
 
-// expected-error@+2{{ClangIR code gen Not Yet Implemented: OpenACC Atomic Construct}}
-// expected-error@+1{{ClangIR code gen Not Yet Implemented: emitCompoundStmtWithoutScope: OpenACCAtomicConstruct}}
+// expected-error@+1{{ClangIR code gen Not Yet Implemented: OpenACC Atomic Construct}}
 #pragma acc atomic
   N = N + 1;
 
 // expected-error@+1{{ClangIR code gen Not Yet Implemented: OpenACC Declare Construct}}
 #pragma acc declare create(A)
 
-  // expected-error@+1{{ClangIR code gen Not Yet Implemented: OpenACC Clause: firstprivate}}
-#pragma acc parallel loop firstprivate(A)
-  for(int i = 0; i <5; ++i);
   // expected-error@+1{{ClangIR code gen Not Yet Implemented: OpenACC Clause: reduction}}
 #pragma acc parallel loop reduction(+:A)
   for(int i = 0; i <5; ++i);
diff --git a/clang/test/CIR/CodeGenOpenACC/shutdown.c b/clang/test/CIR/CodeGenOpenACC/shutdown.c
index 52db382df217e..b68ef90e07252 100644
--- a/clang/test/CIR/CodeGenOpenACC/shutdown.c
+++ b/clang/test/CIR/CodeGenOpenACC/shutdown.c
@@ -11,12 +11,8 @@ void acc_shutdown(int cond) {
   // CHECK-NEXT: acc.shutdown attributes {device_types = [#acc.device_type<star>]}
 #pragma acc shutdown device_type(nvidia)
   // CHECK-NEXT: acc.shutdown attributes {device_types = [#acc.device_type<nvidia>]}
-#pragma acc shutdown device_type(host, multicore)
-  // CHECK-NEXT: acc.shutdown attributes {device_types = [#acc.device_type<host>, #acc.device_type<multicore>]}
 #pragma acc shutdown device_type(NVIDIA)
   // CHECK-NEXT: acc.shutdown attributes {device_types = [#acc.device_type<nvidia>]}
-#pragma acc shutdown device_type(HoSt, MuLtIcORe)
-  // CHECK-NEXT: acc.shutdown attributes {device_types = [#acc.device_type<host>, #acc.device_type<multicore>]}
 #pragma acc shutdown device_type(HoSt) device_type(MuLtIcORe)
   // CHECK-NEXT: acc.shutdown attributes {device_types = [#acc.device_type<host>, #acc.device_type<multicore>]}
 
diff --git a/clang/test/CIR/IR/inline-asm.cir b/clang/test/CIR/IR/inline-asm.cir
new file mode 100644
index 0000000000000..fb1f6315a7d72
--- /dev/null
+++ b/clang/test/CIR/IR/inline-asm.cir
@@ -0,0 +1,112 @@
+// RUN: cir-opt %s | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+!u32i = !cir.int<u, 32>
+
+module {
+cir.func @f1() {
+  // CHECK: cir.asm(x86_att, 
+  // CHECK:   out = [],
+  // CHECK:   in = [],
+  // CHECK:   in_out = [],
+  // CHECK:   {"" "~{dirflag},~{fpsr},~{flags}"})
+  cir.asm(x86_att, 
+    out = [],
+    in = [],
+    in_out = [],
+    {"" "~{dirflag},~{fpsr},~{flags}"})
+  cir.return
+}
+
+cir.func @f2() {
+  // CHECK: cir.asm(x86_att, 
+  // CHECK:   out = [],
+  // CHECK:   in = [],
+  // CHECK:   in_out = [],
+  // CHECK:   {"" "~{dirflag},~{fpsr},~{flags}"}) side_effects
+  cir.asm(x86_att, 
+    out = [],
+    in = [],
+    in_out = [],
+    {"" "~{dirflag},~{fpsr},~{flags}"}) side_effects
+  cir.return
+}
+
+cir.func @f3() {
+  // CHECK: cir.asm(x86_att, 
+  // CHECK:   out = [],
+  // CHECK:   in = [],
+  // CHECK:   in_out = [],
+  // CHECK:   {"abc" "~{dirflag},~{fpsr},~{flags}"}) side_effects
+  cir.asm(x86_att,
+    out = [],
+    in = [],
+    in_out = [],
+    {"abc" "~{dirflag},~{fpsr},~{flags}"}) side_effects
+  cir.return
+}
+
+cir.func @f4(%arg0: !s32i) {
+  %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init] {alignment = 4 : i64}
+  cir.store %arg0, %0 : !s32i, !cir.ptr<!s32i>
+  // CHECK: cir.asm(x86_att, 
+  // CHECK:   out = [],
+  // CHECK:   in = [%0 : !cir.ptr<!s32i> (maybe_memory)],
+  // CHECK:   in_out = [],
+  // CHECK:   {"" "*m,~{dirflag},~{fpsr},~{flags}"}) side_effects
+  cir.asm(x86_att, 
+    out = [],
+    in = [%0 : !cir.ptr<!s32i> (maybe_memory)],
+    in_out = [],
+    {"" "*m,~{dirflag},~{fpsr},~{flags}"}) side_effects
+  cir.return
+}
+
+cir.func @f5() {
+  // CHECK: cir.asm(x86_intel, 
+  // CHECK:   out = [],
+  // CHECK:   in = [],
+  // CHECK:   in_out = [],
+  // CHECK:   {"" "~{dirflag},~{fpsr},~{flags}"})
+  cir.asm(x86_intel, 
+    out = [],
+    in = [],
+    in_out = [],
+    {"" "~{dirflag},~{fpsr},~{flags}"})
+  cir.return
+}
+cir.func @f6() -> !s32i {
+  %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init] {alignment = 4 : i64}
+  // CHECK: %1 = cir.asm(x86_att, 
+  // CHECK:   out = [],
+  // CHECK:   in = [],
+  // CHECK:   in_out = [],
+  // CHECK:   {"movl $$42, $0" "=r,~{dirflag},~{fpsr},~{flags}"}) side_effects -> !s32i
+  %1 = cir.asm(x86_att, 
+    out = [],
+    in = [],
+    in_out = [],
+    {"movl $$42, $0" "=r,~{dirflag},~{fpsr},~{flags}"}) side_effects -> !s32i
+  cir.store align(4) %1, %0 : !s32i, !cir.ptr<!s32i>
+  %3 = cir.load align(4) %0 : !cir.ptr<!s32i>, !s32i
+  cir.return %3 : !s32i
+}
+cir.func @f7(%arg0: !u32i) -> !u32i {
+  %0 = cir.alloca !u32i, !cir.ptr<!u32i>, ["x", init] {alignment = 4 : i64}
+  cir.store %arg0, %0 : !u32i, !cir.ptr<!u32i>
+  %1 = cir.load align(4) %0 : !cir.ptr<!u32i>, !u32i
+  // CHECK: %2 = cir.asm(x86_att, 
+  // CHECK:   out = [],
+  // CHECK:   in = [],
+  // CHECK:   in_out = [%1 : !u32i],
+  // CHECK:   {"addl $$42, $0" "=r,0,~{dirflag},~{fpsr},~{flags}"}) side_effects -> !u32i
+  %2 = cir.asm(x86_att, 
+    out = [],
+    in = [],
+    in_out = [%1 : !u32i],
+    {"addl $$42, $0" "=r,0,~{dirflag},~{fpsr},~{flags}"}) side_effects -> !u32i
+   cir.store align(4) %2, %0 : !u32i, !cir.ptr<!u32i>
+   %3 = cir.load align(4) %0 : !cir.ptr<!u32i>, !u32i
+   cir.return %3 : !u32i
+}
+}
diff --git a/clang/test/CIR/IR/invalid-const-record.cir b/clang/test/CIR/IR/invalid-const-record.cir
new file mode 100644
index 0000000000000..37d7789d4bd45
--- /dev/null
+++ b/clang/test/CIR/IR/invalid-const-record.cir
@@ -0,0 +1,23 @@
+// RUN: cir-opt %s -verify-diagnostics -split-input-file
+
+!s32i = !cir.int<s, 32>
+!rec_anon_struct = !cir.record<struct  packed {!s32i, !s32i, !cir.array<!s32i x 8>}>
+
+// expected-error @below {{expected !cir.record type}}
+cir.global external @e = #cir.const_record<{#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.zero : !cir.array<!s32i x 8>}> : !cir.ptr<!rec_anon_struct>
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!rec_anon_struct = !cir.record<struct  packed {!s32i, !s32i, !cir.array<!s32i x 8>}>
+
+// expected-error @below {{number of elements must match}}
+cir.global external @e = #cir.const_record<{#cir.int<1> : !s32i, #cir.zero : !cir.array<!s32i x 8>}> : !rec_anon_struct
+
+// -----
+
+!s32i = !cir.int<s, 32>
+!rec_anon_struct = !cir.record<struct  packed {!s32i, !s32i, !cir.array<!s32i x 8>}>
+
+// expected-error @below {{element at index 1 has type '!cir.float' but the expected type for this element is '!cir.int<s, 32>'}}
+cir.global external @e = #cir.const_record<{#cir.int<1> : !s32i, #cir.fp<2.000000e+00> : !cir.float, #cir.zero : !cir.array<!s32i x 8>}> : !rec_anon_struct
diff --git a/clang/test/CIR/IR/invalid-goto.cir b/clang/test/CIR/IR/invalid-goto.cir
new file mode 100644
index 0000000000000..9f58bac92fa3f
--- /dev/null
+++ b/clang/test/CIR/IR/invalid-goto.cir
@@ -0,0 +1,9 @@
+// RUN: cir-opt %s -verify-diagnostics -split-input-file
+
+// expected-error@+1 {{goto/label mismatch}}
+cir.func @bad_goto() -> () {
+  cir.goto "somewhere"
+^bb1:
+  cir.label "label"
+  cir.return
+}
diff --git a/clang/test/CIR/IR/invalid-label.cir b/clang/test/CIR/IR/invalid-label.cir
new file mode 100644
index 0000000000000..4cb8d01864441
--- /dev/null
+++ b/clang/test/CIR/IR/invalid-label.cir
@@ -0,0 +1,12 @@
+// RUN: cir-opt %s -verify-diagnostics -split-input-file
+
+!s32i = !cir.int<s, 32>
+
+module  {
+  // expected-error@+3 {{must be the first operation in a block}}
+  cir.func @error(){
+    %0 = cir.const #cir.int<0> : !s32i
+    cir.label "label"
+    cir.return
+  }
+}
diff --git a/clang/test/CIR/IR/invalid-lang-attr.cir b/clang/test/CIR/IR/invalid-lang-attr.cir
new file mode 100644
index 0000000000000..ffe523b1ad401
--- /dev/null
+++ b/clang/test/CIR/IR/invalid-lang-attr.cir
@@ -0,0 +1,5 @@
+// RUN: cir-opt %s -verify-diagnostics
+
+// expected-error@below {{expected ::cir::SourceLanguage to be one of}}
+// expected-error@below {{failed to parse CIR_SourceLanguageAttr parameter 'value'}}
+module attributes {cir.lang = #cir.lang<dummy>} { }
diff --git a/clang/test/CIR/IR/invalid-vtable.cir b/clang/test/CIR/IR/invalid-vtable.cir
new file mode 100644
index 0000000000000..41ddd4c2419be
--- /dev/null
+++ b/clang/test/CIR/IR/invalid-vtable.cir
@@ -0,0 +1,84 @@
+// RUN: cir-opt %s -verify-diagnostics -split-input-file
+
+!s8i = !cir.int<s, 8>
+!u32i = !cir.int<u, 32>
+cir.func @reference_unknown_vtable() {
+  // expected-error @below {{'cir.vtable.address_point' op 'some_vtable' does not reference a valid cir.global}}
+  %0 = cir.vtable.address_point(@some_vtable, address_point = <index = 0, offset = 2>) : !cir.vptr
+  cir.return
+}
+
+// -----
+
+!u8i = !cir.int<u, 8>
+!u32i = !cir.int<u, 32>
+cir.global linkonce_odr @_ZTT1D = #cir.const_array<[#cir.global_view<@_ZTV1D, [0 : i32, 3 : i32]> : !cir.ptr<!u8i>, #cir.global_view<@_ZTC1D0_1B, [0 : i32, 3 : i32]> : !cir.ptr<!u8i>]> : !cir.array<!cir.ptr<!u8i> x 2>
+cir.func @reference_unknown_vtable() {
+  // expected-error @below {{Expected #cir.vtable in initializer for global '_ZTT1D'}}
+  %0 = cir.vtable.address_point(@_ZTT1D, address_point = <index = 0, offset = 2>) : !cir.vptr
+  cir.return
+}
+
+// -----
+
+!rec_S = !cir.record<struct "S" {!cir.vptr}>
+!u8i = !cir.int<u, 8>
+!rec_anon_struct = !cir.record<struct {!cir.array<!cir.ptr<!u8i> x 4>}>
+module {
+  // expected-error @below {{expected !cir.record type result}}
+  cir.global external @_ZTV1S = #cir.vtable<{#cir.const_array<[#cir.ptr<null> : !cir.ptr<!u8i>, #cir.ptr<null> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S3keyEv> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S6nonKeyEv> : !cir.ptr<!u8i>]> : !cir.array<!cir.ptr<!u8i> x 4>}> : !cir.ptr<!rec_anon_struct>
+  cir.func private dso_local @_ZN1S3keyEv(%arg0: !cir.ptr<!rec_S>)
+  cir.func private dso_local @_ZN1S6nonKeyEv(%arg0: !cir.ptr<!rec_S>)
+}
+
+// -----
+
+!rec_S = !cir.record<struct "S" {!cir.vptr}>
+!u8i = !cir.int<u, 8>
+!rec_anon_struct = !cir.record<struct  {}>
+module {
+  // expected-error @below {{expected record type with one or more subtype}}
+  cir.global external @_ZTV1S = #cir.vtable<{}> : !rec_anon_struct {alignment = 8 : i64}
+  cir.func private dso_local @_ZN1S3keyEv(%arg0: !cir.ptr<!rec_S>)
+  cir.func private dso_local @_ZN1S6nonKeyEv(%arg0: !cir.ptr<!rec_S>)
+}
+
+// -----
+
+!rec_S = !cir.record<struct "S" {!cir.vptr}>
+!u8i = !cir.int<u, 8>
+!rec_anon_struct = !cir.record<struct  {!cir.ptr<!u8i>}>
+module {
+  // expected-error @below {{expected constant array subtype}}
+  cir.global external @_ZTV1S = #cir.vtable<{#cir.ptr<null> : !cir.ptr<!u8i>}> : !rec_anon_struct {alignment = 8 : i64}
+  cir.func private dso_local @_ZN1S3keyEv(%arg0: !cir.ptr<!rec_S>)
+  cir.func private dso_local @_ZN1S6nonKeyEv(%arg0: !cir.ptr<!rec_S>)
+}
+
+// -----
+
+!rec_S = !cir.record<struct "S" {!cir.vptr}>
+!u64i = !cir.int<u, 64>
+!rec_anon_struct = !cir.record<struct  {!cir.array<!u64i x 4>}>
+module {
+  // expected-error @below {{expected GlobalViewAttr or ConstPtrAttr}}
+  cir.global external @_ZTV1S = #cir.vtable<{#cir.const_array<[#cir.int<1> : !u64i, #cir.int<1> : !u64i, #cir.int<3> : !u64i, #cir.int<4> : !u64i]> : !cir.array<!u64i x 4>}> : !rec_anon_struct {alignment = 8 : i64}
+  cir.func private dso_local @_ZN1S3keyEv(%arg0: !cir.ptr<!rec_S>)
+  cir.func private dso_local @_ZN1S6nonKeyEv(%arg0: !cir.ptr<!rec_S>)
+}
+
+// -----
+
+!rec_Q = !cir.record<struct "Q" {!cir.vptr}>
+!rec_S = !cir.record<struct "S" {!cir.vptr}>
+!rec_S2 = !cir.record<struct "S2" {!rec_Q, !rec_S}>
+!u8i = !cir.int<u, 8>
+!rec_anon_struct = !cir.record<struct  {!cir.array<!cir.ptr<!u8i> x 4>, !cir.ptr<!u8i>}>
+module {
+  // expected-error @below {{expected constant array subtype}}
+  cir.global external @_ZTV2S2 = #cir.vtable<{#cir.const_array<[#cir.ptr<null> : !cir.ptr<!u8i>, #cir.ptr<null> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S3keyEv> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S6nonKeyEv> : !cir.ptr<!u8i>]> : !cir.array<!cir.ptr<!u8i> x 4>, #cir.ptr<null> : !cir.ptr<!u8i>}> : !rec_anon_struct {alignment = 8 : i64}
+
+  cir.func private dso_local @_ZN1S3keyEv(%arg0: !cir.ptr<!rec_S>)
+  cir.func private dso_local @_ZN1S6nonKeyEv(%arg0: !cir.ptr<!rec_S>)
+  cir.func private dso_local @_ZN2S23keyEv(%arg0: !cir.ptr<!rec_S2>)
+}
diff --git a/clang/test/CIR/IR/label.cir b/clang/test/CIR/IR/label.cir
new file mode 100644
index 0000000000000..2211a4e8da331
--- /dev/null
+++ b/clang/test/CIR/IR/label.cir
@@ -0,0 +1,26 @@
+// RUN: cir-opt %s | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module  {
+  cir.func @label() {
+    cir.label "label"
+    cir.return
+  }
+
+  cir.func @label2() {
+    %0 = cir.const #cir.int<0> : !s32i
+    cir.br ^bb1
+  ^bb1:  // pred: ^bb0
+    cir.label "label2"
+    cir.return
+  }
+}
+
+// CHECK:       cir.func @label
+// CHECK-NEXT:    cir.label "label"
+
+// CHECK:       cir.func @label2
+// CHECK:        cir.br ^bb1
+// CHECK-NEXT:  ^bb1:  // pred: ^bb0
+// CHECK-NEXT:    cir.label "label2"
diff --git a/clang/test/CIR/IR/module.cir b/clang/test/CIR/IR/module.cir
new file mode 100644
index 0000000000000..7ce2c0ba21cb0
--- /dev/null
+++ b/clang/test/CIR/IR/module.cir
@@ -0,0 +1,12 @@
+// RUN: cir-opt %s -split-input-file -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s
+
+// Should parse and print C source language attribute.
+module attributes {cir.lang = #cir.lang<c>} { }
+// CHECK: module attributes {cir.lang = #cir.lang<c>}
+
+// -----
+
+// Should parse and print C++ source language attribute.
+module attributes {cir.lang = #cir.lang<cxx>} { }
+// CHECK: module attributes {cir.lang = #cir.lang<cxx>}
diff --git a/clang/test/CIR/IR/struct.cir b/clang/test/CIR/IR/struct.cir
index 85f475f643ee5..33f2e9860c5cb 100644
--- a/clang/test/CIR/IR/struct.cir
+++ b/clang/test/CIR/IR/struct.cir
@@ -13,8 +13,9 @@
 // CHECK-DAG: !rec_S = !cir.record<struct "S" incomplete>
 // CHECK-DAG: !rec_U = !cir.record<union "U" incomplete>
 
-!rec_anon_struct = !cir.record<struct {!cir.array<!cir.ptr<!u8i> x 5>}>
-!rec_anon_struct1 = !cir.record<struct {!cir.ptr<!u8i>, !cir.ptr<!u8i>, !cir.ptr<!u8i>}>
+!rec_anon_struct = !cir.record<struct  packed {!s32i, !s32i, !cir.array<!s32i x 8>}>
+!rec_anon_struct1 = !cir.record<struct {!cir.array<!cir.ptr<!u8i> x 5>}>
+!rec_anon_struct2 = !cir.record<struct {!cir.ptr<!u8i>, !cir.ptr<!u8i>, !cir.ptr<!u8i>}>
 !rec_S1 = !cir.record<struct "S1" {!s32i, !s32i}>
 !rec_Sc = !cir.record<struct "Sc" {!u8i, !u16i, !u32i}>
 
@@ -42,18 +43,22 @@
 !rec_Node = !cir.record<struct "Node" {!cir.ptr<!cir.record<struct "Node">>}>
 // CHECK-DAG: !cir.record<struct "Node" {!cir.ptr<!cir.record<struct "Node">>}>
 
+
+
 module  {
   cir.global external @p1 = #cir.ptr<null> : !cir.ptr<!rec_S>
   cir.global external @p2 = #cir.ptr<null> : !cir.ptr<!rec_U>
   cir.global external @p3 = #cir.ptr<null> : !cir.ptr<!rec_C>
+  cir.global external @arr = #cir.const_record<{#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.zero : !cir.array<!s32i x 8>}> : !rec_anon_struct
 // CHECK: cir.global external @p1 = #cir.ptr<null> : !cir.ptr<!rec_S>
 // CHECK: cir.global external @p2 = #cir.ptr<null> : !cir.ptr<!rec_U>
 // CHECK: cir.global external @p3 = #cir.ptr<null> : !cir.ptr<!rec_C>
+// CHECK: cir.global external @arr = #cir.const_record<{#cir.int<1> : !s32i, #cir.int<2> : !s32i, #cir.zero : !cir.array<!s32i x 8>}> : !rec_anon_struct
 
   // Dummy function to use types and force them to be printed.
   cir.func @useTypes(%arg0: !rec_Node,
-                     %arg1: !rec_anon_struct1,
-                     %arg2: !rec_anon_struct,
+                     %arg1: !rec_anon_struct2,
+                     %arg2: !rec_anon_struct1,
                      %arg3: !rec_S1,
                      %arg4: !rec_Ac,
                      %arg5: !rec_P1,
diff --git a/clang/test/CIR/IR/vtable-addrpt.cir b/clang/test/CIR/IR/vtable-addrpt.cir
new file mode 100644
index 0000000000000..0b809cc2506e6
--- /dev/null
+++ b/clang/test/CIR/IR/vtable-addrpt.cir
@@ -0,0 +1,23 @@
+// RUN: cir-opt %s | FileCheck %s
+
+// Test the parsing and printing of a constructor that uses a vtable addess_point op.
+
+!u32i = !cir.int<u, 32>
+!u8i = !cir.int<u, 8>
+!rec_anon_struct = !cir.record<struct  {!cir.array<!cir.ptr<!u8i> x 4>}>
+!rec_S = !cir.record<struct "S" {!cir.vptr}>
+
+module {
+  cir.global "private" external @_ZTV1S : !rec_anon_struct {alignment = 8 : i64}
+  cir.func @_ZN1SC2Ev(%arg0: !cir.ptr<!rec_S>) {
+    %0 = cir.alloca !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>, ["this", init] {alignment = 8 : i64}
+    cir.store %arg0, %0 : !cir.ptr<!rec_S>, !cir.ptr<!cir.ptr<!rec_S>>
+    %1 = cir.load %0 : !cir.ptr<!cir.ptr<!rec_S>>, !cir.ptr<!rec_S>
+    %2 = cir.vtable.address_point(@_ZTV1S, address_point = <index = 0, offset = 2>) : !cir.vptr
+    %3 = cir.cast(bitcast, %1 : !cir.ptr<!rec_S>), !cir.ptr<!cir.vptr>
+    cir.store align(8) %2, %3 : !cir.vptr, !cir.ptr<!cir.vptr>
+    cir.return
+  }
+}
+
+// CHECK: cir.vtable.address_point(@_ZTV1S, address_point = <index = 0, offset = 2>) : !cir.vptr
diff --git a/clang/test/CIR/IR/vtable-attr.cir b/clang/test/CIR/IR/vtable-attr.cir
new file mode 100644
index 0000000000000..3854208ff78cc
--- /dev/null
+++ b/clang/test/CIR/IR/vtable-attr.cir
@@ -0,0 +1,19 @@
+// RUN: cir-opt %s | FileCheck %s
+
+!rec_Q = !cir.record<struct "Q" {!cir.vptr}>
+!rec_S = !cir.record<struct "S" {!cir.vptr}>
+!rec_S2 = !cir.record<struct "S2" {!rec_Q, !rec_S}>
+!u8i = !cir.int<u, 8>
+!rec_anon_struct = !cir.record<struct  {!cir.array<!cir.ptr<!u8i> x 4>}>
+!rec_anon_struct1 = !cir.record<struct  {!cir.array<!cir.ptr<!u8i> x 4>, !cir.array<!cir.ptr<!u8i> x 3>}>
+module {
+  cir.global external @_ZTV1S = #cir.vtable<{#cir.const_array<[#cir.ptr<null> : !cir.ptr<!u8i>, #cir.ptr<null> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S3keyEv> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S6nonKeyEv> : !cir.ptr<!u8i>]> : !cir.array<!cir.ptr<!u8i> x 4>}> : !rec_anon_struct {alignment = 8 : i64}
+  // CHECK: cir.global external @_ZTV1S = #cir.vtable<{#cir.const_array<[#cir.ptr<null> : !cir.ptr<!u8i>, #cir.ptr<null> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S3keyEv> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S6nonKeyEv> : !cir.ptr<!u8i>]> : !cir.array<!cir.ptr<!u8i> x 4>}> : !rec_anon_struct {alignment = 8 : i64}
+
+  cir.global external @_ZTV2S2 = #cir.vtable<{#cir.const_array<[#cir.ptr<null> : !cir.ptr<!u8i>, #cir.ptr<null> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S3keyEv> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S6nonKeyEv> : !cir.ptr<!u8i>]> : !cir.array<!cir.ptr<!u8i> x 4>, #cir.const_array<[#cir.ptr<-8 : i64> : !cir.ptr<!u8i>, #cir.ptr<null> : !cir.ptr<!u8i>, #cir.global_view<@_ZN2S23keyEv> : !cir.ptr<!u8i>]> : !cir.array<!cir.ptr<!u8i> x 3>}> : !rec_anon_struct1 {alignment = 8 : i64}
+  // CHECK: cir.global external @_ZTV2S2 = #cir.vtable<{#cir.const_array<[#cir.ptr<null> : !cir.ptr<!u8i>, #cir.ptr<null> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S3keyEv> : !cir.ptr<!u8i>, #cir.global_view<@_ZN1S6nonKeyEv> : !cir.ptr<!u8i>]> : !cir.array<!cir.ptr<!u8i> x 4>, #cir.const_array<[#cir.ptr<-8 : i64> : !cir.ptr<!u8i>, #cir.ptr<null> : !cir.ptr<!u8i>, #cir.global_view<@_ZN2S23keyEv> : !cir.ptr<!u8i>]> : !cir.array<!cir.ptr<!u8i> x 3>}> : !rec_anon_struct1 {alignment = 8 : i64}
+
+  cir.func private dso_local @_ZN1S3keyEv(%arg0: !cir.ptr<!rec_S>)
+  cir.func private dso_local @_ZN1S6nonKeyEv(%arg0: !cir.ptr<!rec_S>)
+  cir.func private dso_local @_ZN2S23keyEv(%arg0: !cir.ptr<!rec_S2>)
+}
diff --git a/clang/test/CIR/Lowering/array.cpp b/clang/test/CIR/Lowering/array.cpp
index 438d41e2c2c2f..40ad986b7fdfa 100644
--- a/clang/test/CIR/Lowering/array.cpp
+++ b/clang/test/CIR/Lowering/array.cpp
@@ -19,7 +19,7 @@ int dd[3][2] = {{1, 2}, {3, 4}, {5, 6}};
 // CHECK: [i32 3, i32 4], [2 x i32] [i32 5, i32 6]]
 
 int e[10] = {1, 2};
-// CHECK: @e = global [10 x i32] [i32 1, i32 2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0]
+// CHECK: @e = global <{ i32, i32, [8 x i32] }> <{ i32 1, i32 2, [8 x i32] zeroinitializer }>
 
 int f[5] = {1, 2};
 // CHECK: @f = global [5 x i32] [i32 1, i32 2, i32 0, i32 0, i32 0]
@@ -57,17 +57,28 @@ void func() {
 void func2() {
   int arr[2] = {5};
 }
+
 // CHECK: define{{.*}} void @_Z5func2v()
-// CHECK:  %[[ARR_ALLOCA:.*]] = alloca [2 x i32], i64 1, align 4
-// CHECK:  %[[TMP:.*]] = alloca ptr, i64 1, align 8
-// CHECK:  %[[ARR_PTR:.*]] = getelementptr i32, ptr %[[ARR_ALLOCA]], i32 0
-// CHECK:  store i32 5, ptr %[[ARR_PTR]], align 4
-// CHECK:  %[[ELE_1_PTR:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i64 1
-// CHECK:  store ptr %[[ELE_1_PTR]], ptr %[[TMP]], align 8
-// CHECK:  %[[TMP2:.*]] = load ptr, ptr %[[TMP]], align 8
-// CHECK:  store i32 0, ptr %[[TMP2]], align 4
-// CHECK:  %[[ELE_1:.*]] = getelementptr i32, ptr %[[TMP2]], i64 1
-// CHECK:  store ptr %[[ELE_1]], ptr %[[TMP]], align 8
+// CHECK:   %[[ARR:.*]] = alloca [2 x i32], i64 1, align 4
+// CHECK:   %[[TMP:.*]] = alloca ptr, i64 1, align 8
+// CHECK:   %[[ARR_PTR:.*]] = getelementptr i32, ptr %[[ARR]], i32 0
+// CHECK:   store i32 5, ptr %[[ARR_PTR]], align 4
+// CHECK:   %[[ELE_1_PTR:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i64 1
+// CHECK:   store ptr %[[ELE_1_PTR]], ptr %[[TMP]], align 8
+// CHECK:   %[[END_PTR:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i64 2
+// CHECK:   br label %[[LOOP_BODY:.*]]
+// CHECK: [[LOOP_NEXT:.*]]:
+// CHECK:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// CHECK:   %[[CMP:.*]] = icmp ne ptr %[[CUR]], %[[END_PTR]]
+// CHECK:   br i1 %[[CMP]], label %[[LOOP_BODY]], label %[[LOOP_END:.*]]
+// CHECK: [[LOOP_BODY]]:
+// CHECK:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// CHECK:   store i32 0, ptr %[[CUR]], align 4
+// CHECK:   %[[NEXT:.*]] = getelementptr i32, ptr %[[CUR]], i64 1
+// CHECK:   store ptr %[[NEXT]], ptr %[[TMP]], align 8
+// CHECK:   br label %[[LOOP_NEXT:.*]]
+// CHECK: [[LOOP_END]]:
+// CHECK:   ret void
 
 void func3() {
   int arr3[2] = {5, 6};
@@ -103,17 +114,27 @@ void func5() {
   int arr[2][1] = {{5}};
 }
 // CHECK: define{{.*}} void @_Z5func5v()
-// CHECK:  %[[ARR_ALLOCA:.*]] = alloca [2 x [1 x i32]], i64 1, align 4
-// CHECK:  %[[TMP:.*]] = alloca ptr, i64 1, align 8
-// CHECK:  %[[ARR_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR_ALLOCA]], i32 0
-// CHECK:  %[[ARR_0:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i32 0
-// CHECK:  store i32 5, ptr %[[ARR_0]], align 4
-// CHECK:  %[[ARR_1:.*]] = getelementptr [1 x i32], ptr %[[ARR_PTR]], i64 1
-// CHECK:  store ptr %[[ARR_1]], ptr %[[TMP]], align 8
-// CHECK:  %[[ARR_1_VAL:.*]] = load ptr, ptr %[[TMP]], align 8
-// CHECK:  store [1 x i32] zeroinitializer, ptr %[[ARR_1_VAL]], align 4
-// CHECK:  %[[ARR_1_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR_1_VAL]], i64 1
-// CHECK:  store ptr %[[ARR_1_PTR]], ptr %[[TMP]], align 8
+// CHECK:   %[[ARR:.*]] = alloca [2 x [1 x i32]], i64 1, align 4
+// CHECK:   %[[TMP:.*]] = alloca ptr, i64 1, align 8
+// CHECK:   %[[ARR_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR]], i32 0
+// CHECK:   %[[ARR_0:.*]] = getelementptr i32, ptr %[[ARR_PTR]], i32 0
+// CHECK:   store i32 5, ptr %[[ARR_0]], align 4
+// CHECK:   %[[ARR_1:.*]] = getelementptr [1 x i32], ptr %[[ARR_PTR]], i64 1
+// CHECK:   store ptr %[[ARR_1]], ptr %[[TMP]], align 8
+// CHECK:   %[[END_PTR:.*]] = getelementptr [1 x i32], ptr %[[ARR_PTR]], i64 2
+// CHECK:   br label %[[LOOP_BODY:.*]]
+// CHECK: [[LOOP_NEXT:.*]]:
+// CHECK:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// CHECK:   %[[CMP:.*]] = icmp ne ptr %[[CUR]], %[[END_PTR]]
+// CHECK:   br i1 %[[CMP]], label %[[LOOP_BODY]], label %[[LOOP_END:.*]]
+// CHECK: [[LOOP_BODY]]:
+// CHECK:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// CHECK:   store [1 x i32] zeroinitializer, ptr %[[CUR]], align 4
+// CHECK:   %[[NEXT:.*]] = getelementptr [1 x i32], ptr %[[CUR]], i64 1
+// CHECK:   store ptr %[[NEXT]], ptr %[[TMP]], align 8
+// CHECK:   br label %[[LOOP_NEXT:.*]]
+// CHECK: [[LOOP_END]]:
+// CHECK:   ret void
 
 void func6() {
   int x = 4;
@@ -133,14 +154,24 @@ void func7() {
   int* arr[1] = {};
 }
 // CHECK: define{{.*}} void @_Z5func7v()
-// CHECK:  %[[ARR:.*]] = alloca [1 x ptr], i64 1, align 8
-// CHECK:  %[[ALLOCA:.*]] = alloca ptr, i64 1, align 8
-// CHECK:  %[[ELE_PTR:.*]] = getelementptr ptr, ptr %[[ARR]], i32 0
-// CHECK:  store ptr %[[ELE_PTR]], ptr %[[ALLOCA]], align 8
-// CHECK:  %[[TMP:.*]] = load ptr, ptr %[[ALLOCA]], align 8
-// CHECK:  store ptr null, ptr %[[TMP]], align 8
-// CHECK:  %[[ELE:.*]] = getelementptr ptr, ptr %[[TMP]], i64 1
-// CHECK:  store ptr %[[ELE]], ptr %[[ALLOCA]], align 8
+// CHECK:   %[[ARR:.*]] = alloca [1 x ptr], i64 1, align 8
+// CHECK:   %[[TMP:.*]] = alloca ptr, i64 1, align 8
+// CHECK:   %[[ARR_PTR:.*]] = getelementptr ptr, ptr %[[ARR]], i32 0
+// CHECK:   store ptr %[[ARR_PTR]], ptr %[[TMP]], align 8
+// CHECK:   %[[END_PTR:.*]] = getelementptr ptr, ptr %[[ARR_PTR]], i64 1
+// CHECK:   br label %[[LOOP_BODY:.*]]
+// CHECK: [[LOOP_NEXT:.*]]:
+// CHECK:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// CHECK:   %[[CMP:.*]] = icmp ne ptr %[[CUR]], %[[END_PTR]]
+// CHECK:   br i1 %[[CMP]], label %[[LOOP_BODY]], label %[[LOOP_END:.*]]
+// CHECK: [[LOOP_BODY]]:
+// CHECK:   %[[CUR:.*]] = load ptr, ptr %[[TMP]], align 8
+// CHECK:   store ptr null, ptr %[[CUR]], align 8
+// CHECK:   %[[NEXT:.*]] = getelementptr ptr, ptr %[[CUR]], i64 1
+// CHECK:   store ptr %[[NEXT]], ptr %[[TMP]], align 8
+// CHECK:   br label %[[LOOP_NEXT:.*]]
+// CHECK: [[LOOP_END]]:
+// CHECK:   ret void
 
 void func8(int p[10]) {}
 // CHECK: define{{.*}} void @_Z5func8Pi(ptr {{%.*}})
diff --git a/clang/test/CIR/Lowering/goto.cir b/clang/test/CIR/Lowering/goto.cir
new file mode 100644
index 0000000000000..cd3a57d2e7138
--- /dev/null
+++ b/clang/test/CIR/Lowering/goto.cir
@@ -0,0 +1,52 @@
+// RUN: cir-opt %s --pass-pipeline='builtin.module(cir-to-llvm,canonicalize{region-simplify=disabled})' -o - | FileCheck %s -check-prefix=MLIR
+
+!s32i = !cir.int<s, 32>
+
+module {
+
+  cir.func @gotoFromIf(%arg0: !s32i) -> !s32i {
+    %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init] {alignment = 4 : i64}
+    %1 = cir.alloca !s32i, !cir.ptr<!s32i>, ["__retval"] {alignment = 4 : i64}
+    cir.store %arg0, %0 : !s32i, !cir.ptr<!s32i>
+    cir.scope {
+      %6 = cir.load %0 : !cir.ptr<!s32i>, !s32i
+      %7 = cir.const #cir.int<5> : !s32i
+      %8 = cir.cmp(gt, %6, %7) : !s32i, !cir.bool
+      cir.if %8 {
+        cir.goto "err"
+      }
+    }
+    %2 = cir.const #cir.int<0> : !s32i
+    cir.store %2, %1 : !s32i, !cir.ptr<!s32i>
+    cir.br ^bb1
+  ^bb1:
+    %3 = cir.load %1 : !cir.ptr<!s32i>, !s32i
+    cir.return %3 : !s32i
+  ^bb2:
+    cir.label "err"
+    %4 = cir.const #cir.int<1> : !s32i
+    %5 = cir.unary(minus, %4) : !s32i, !s32i
+    cir.store %5, %1 : !s32i, !cir.ptr<!s32i>
+    cir.br ^bb1
+  }
+
+// MLIR:  llvm.func @gotoFromIf
+// MLIR:    %[[#One:]] = llvm.mlir.constant(1 : i32) : i32
+// MLIR:    %[[#Zero:]] = llvm.mlir.constant(0 : i32) : i32
+// MLIR:    llvm.cond_br {{.*}}, ^bb[[#COND_YES:]], ^bb[[#COND_NO:]]
+// MLIR:  ^bb[[#COND_YES]]:
+// MLIR:    llvm.br ^bb[[#GOTO_BLK:]]
+// MLIR:   ^bb[[#COND_NO]]:
+// MLIR:    llvm.br ^bb[[#BLK:]]
+// MLIR:  ^bb[[#BLK]]:
+// MLIR:    llvm.store %[[#Zero]], %[[#Ret_val_addr:]] {{.*}}: i32, !llvm.ptr
+// MLIR:    llvm.br ^bb[[#RETURN:]]
+// MLIR:  ^bb[[#RETURN]]:
+// MLIR:    %[[#Ret_val:]] = llvm.load %[[#Ret_val_addr]] {alignment = 4 : i64} : !llvm.ptr -> i32
+// MLIR:    llvm.return %[[#Ret_val]] : i32
+// MLIR:  ^bb[[#GOTO_BLK]]:
+// MLIR:    %[[#Neg_one:]] = llvm.sub %[[#Zero]], %[[#One]]  : i32
+// MLIR:    llvm.store %[[#Neg_one]], %[[#Ret_val_addr]] {{.*}}: i32, !llvm.ptr
+// MLIR:    llvm.br ^bb[[#RETURN]]
+// MLIR: }
+}
diff --git a/clang/test/CIR/Lowering/inline-asm.cir b/clang/test/CIR/Lowering/inline-asm.cir
new file mode 100644
index 0000000000000..a8545d4c0f059
--- /dev/null
+++ b/clang/test/CIR/Lowering/inline-asm.cir
@@ -0,0 +1,86 @@
+// RUN: cir-translate %s -cir-to-llvmir --target x86_64-unknown-linux-gnu --disable-cc-lowering  | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+!u32i = !cir.int<u, 32>
+
+module {
+cir.func @f1() {
+  // CHECK: call void asm "", "~{dirflag},~{fpsr},~{flags}"()
+  cir.asm(x86_att, 
+    out = [],
+    in = [],
+    in_out = [],
+    {"" "~{dirflag},~{fpsr},~{flags}"})
+  cir.return
+}
+
+cir.func @f2() {
+  // CHECK: call void asm sideeffect "", "~{dirflag},~{fpsr},~{flags}"()
+  cir.asm(x86_att, 
+    out = [],
+    in = [],
+    in_out = [],
+    {"" "~{dirflag},~{fpsr},~{flags}"}) side_effects
+  cir.return
+}
+
+cir.func @f3() {
+  // CHECK: call void asm sideeffect "abc", "~{dirflag},~{fpsr},~{flags}"()
+  cir.asm(x86_att,
+    out = [],
+    in = [],
+    in_out = [],
+    {"abc" "~{dirflag},~{fpsr},~{flags}"}) side_effects
+  cir.return
+}
+
+cir.func @f4(%arg0: !s32i) {
+  %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init] {alignment = 4 : i64}
+  cir.store %arg0, %0 : !s32i, !cir.ptr<!s32i>
+  // CHECK: call void asm sideeffect "", "*m,~{dirflag},~{fpsr},~{flags}"(ptr elementtype(i32) %2)
+  cir.asm(x86_att, 
+    out = [],
+    in = [%0 : !cir.ptr<!s32i> (maybe_memory)],
+    in_out = [],
+    {"" "*m,~{dirflag},~{fpsr},~{flags}"}) side_effects
+  cir.return
+}
+
+cir.func @f5() {
+  // CHECK: call void asm inteldialect "", "~{dirflag},~{fpsr},~{flags}"()
+  cir.asm(x86_intel, 
+    out = [],
+    in = [],
+    in_out = [],
+    {"" "~{dirflag},~{fpsr},~{flags}"})
+  cir.return
+}
+
+cir.func @f6() -> !s32i {
+  %0 = cir.alloca !s32i, !cir.ptr<!s32i>, ["x", init] {alignment = 4 : i64}
+  // CHECK: %2 = call i32 asm sideeffect "movl $$42, $0", "=r,~{dirflag},~{fpsr},~{flags}"()
+  %1 = cir.asm(x86_att, 
+    out = [],
+    in = [],
+    in_out = [],
+    {"movl $$42, $0" "=r,~{dirflag},~{fpsr},~{flags}"}) side_effects -> !s32i
+  cir.store align(4) %1, %0 : !s32i, !cir.ptr<!s32i>
+  %3 = cir.load align(4) %0 : !cir.ptr<!s32i>, !s32i
+  cir.return %3 : !s32i
+}
+
+cir.func @f7(%arg0: !u32i) -> !u32i {
+  %0 = cir.alloca !u32i, !cir.ptr<!u32i>, ["x", init] {alignment = 4 : i64}
+  cir.store %arg0, %0 : !u32i, !cir.ptr<!u32i>
+  %1 = cir.load align(4) %0 : !cir.ptr<!u32i>, !u32i
+  // CHECK: %4 = call i32 asm sideeffect "addl $$42, $0", "=r,0,~{dirflag},~{fpsr},~{flags}"(i32 %3)
+  %2 = cir.asm(x86_att, 
+    out = [],
+    in = [],
+    in_out = [%1 : !u32i],
+    {"addl $$42, $0" "=r,0,~{dirflag},~{fpsr},~{flags}"}) side_effects -> !u32i
+   cir.store align(4) %2, %0 : !u32i, !cir.ptr<!u32i>
+   %3 = cir.load align(4) %0 : !cir.ptr<!u32i>, !u32i
+   cir.return %3 : !u32i
+}
+}
diff --git a/clang/test/CIR/Lowering/module-asm.cir b/clang/test/CIR/Lowering/module-asm.cir
new file mode 100644
index 0000000000000..b802cda8f6e65
--- /dev/null
+++ b/clang/test/CIR/Lowering/module-asm.cir
@@ -0,0 +1,11 @@
+// RUN: cir-opt %s -cir-to-llvm -o %t.cir
+// RUN: FileCheck %s --input-file=%t.cir
+
+// RUN: cir-translate -cir-to-llvmir --disable-cc-lowering -o %t.ll %s
+// RUN: FileCheck -check-prefix=LLVM --input-file=%t.ll %s
+
+// CHECK: llvm.module_asm =  [".globl bar", ".globl foo"]
+// LLVM: module asm ".globl bar"
+// LLVM: module asm ".globl foo"
+module attributes {cir.module_asm = [".globl bar", ".globl foo"]} {
+}
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index 286c9d40d2dab..e9f4f83f98923 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -26,7 +26,6 @@ llvm_canonicalize_cmake_booleans(
   PPC_LINUX_DEFAULT_IEEELONGDOUBLE
   LLVM_TOOL_LLVM_DRIVER_BUILD
   LLVM_INCLUDE_SPIRV_TOOLS_TESTS
-  LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS
   )
 
 # Run tests requiring Z3 headers only if LLVM was built with Z3
diff --git a/clang/test/CXX/basic/basic.lookup/basic.lookup.qual/class.qual/p2.cpp b/clang/test/CXX/basic/basic.lookup/basic.lookup.qual/class.qual/p2.cpp
index 5c281ac806836..76e2afb3a8515 100644
--- a/clang/test/CXX/basic/basic.lookup/basic.lookup.qual/class.qual/p2.cpp
+++ b/clang/test/CXX/basic/basic.lookup/basic.lookup.qual/class.qual/p2.cpp
@@ -197,7 +197,7 @@ namespace InhCtor {
   // FIXME: Consider reusing the same diagnostic between dependent and non-dependent contexts
   typedef int I;
   struct UsingInt {
-    using I::I; // expected-error {{'InhCtor::I' (aka 'int') is not a class, namespace, or enumeration}}
+    using I::I; // expected-error {{'I' (aka 'int') is not a class, namespace, or enumeration}}
   };
   template<typename T> struct UsingIntTemplate {
     using T::T; // expected-error {{type 'int' cannot be used prior to '::' because it has no members}}
diff --git a/clang/test/CXX/class.access/p6.cpp b/clang/test/CXX/class.access/p6.cpp
index 15f2644f6ac1d..9a8aebe76706a 100644
--- a/clang/test/CXX/class.access/p6.cpp
+++ b/clang/test/CXX/class.access/p6.cpp
@@ -176,7 +176,7 @@ namespace test8 {
   };
 
   void test(A &a) {
-    if (a) return; // expected-error-re {{'operator void *(A::*)(){{( __attribute__\(\(thiscall\)\))?}} const' is a private member of 'test8::A'}}
+    if (a) return; // expected-error-re {{'operator void *(test8::A::*)(){{( __attribute__\(\(thiscall\)\))?}} const' is a private member of 'test8::A'}}
   }
 }
 
diff --git a/clang/test/CXX/class.derived/class.derived.general/p2.cpp b/clang/test/CXX/class.derived/class.derived.general/p2.cpp
index 1423eeabebf59..c700e8a3ce14e 100644
--- a/clang/test/CXX/class.derived/class.derived.general/p2.cpp
+++ b/clang/test/CXX/class.derived/class.derived.general/p2.cpp
@@ -2,7 +2,7 @@
 
 namespace CurrentInstantiation {
   template<typename T>
-  struct A0 { // expected-note 6{{definition of 'A0<T>' is not complete until the closing '}'}}
+  struct A0 { // expected-note 6{{definition of 'CurrentInstantiation::A0<T>' is not complete until the closing '}'}}
     struct B0 : A0 { }; // expected-error {{base class has incomplete type}}
 
     template<typename U>
@@ -26,7 +26,7 @@ namespace CurrentInstantiation {
     };
 
     template<typename U>
-    struct B5 { // expected-note 2{{definition of 'B5<U>' is not complete until the closing '}'}}
+    struct B5 { // expected-note 2{{definition of 'CurrentInstantiation::A0::B5<U>' is not complete until the closing '}'}}
       struct C0 : A0, B5 { }; // expected-error 2{{base class has incomplete type}}
 
       template<typename V>
@@ -63,7 +63,7 @@ namespace CurrentInstantiation {
   struct A0<T>::B5<U>::C3 : A0, B5 { };
 
   template<typename T>
-  struct A0<T*> { // expected-note 2{{definition of 'A0<T *>' is not complete until the closing '}'}}
+  struct A0<T*> { // expected-note 2{{definition of 'CurrentInstantiation::A0<T *>' is not complete until the closing '}'}}
     struct B0 : A0 { }; // expected-error {{base class has incomplete type}}
 
     template<typename U>
@@ -91,7 +91,7 @@ namespace MemberOfCurrentInstantiation {
 
     template<typename U>
     struct C : C<U> { }; // expected-error {{base class has incomplete type}}
-                         // expected-note@-1 {{definition of 'C<U>' is not complete until the closing '}'}}
+                         // expected-note@-1 {{definition of 'MemberOfCurrentInstantiation::A0::C<U>' is not complete until the closing '}'}}
   };
 
   template<typename T>
diff --git a/clang/test/CXX/class/class.mem/class.mem.general/p8.cpp b/clang/test/CXX/class/class.mem/class.mem.general/p8.cpp
index 8cc9b41eaca91..c4aca327fbb37 100644
--- a/clang/test/CXX/class/class.mem/class.mem.general/p8.cpp
+++ b/clang/test/CXX/class/class.mem/class.mem.general/p8.cpp
@@ -45,7 +45,7 @@ namespace N1 {
     void g0() noexcept(y); // expected-error {{use of undeclared identifier 'y'}}
 
     void f1() noexcept(A::x);
-    void g1() noexcept(A::y); // expected-error {{no member named 'y' in 'A<T>'}}
+    void g1() noexcept(A::y); // expected-error {{no member named 'y' in 'N1::A<T>'}}
 
     template<typename U>
     void f2() noexcept(x);
@@ -55,13 +55,13 @@ namespace N1 {
     template<typename U>
     void f3() noexcept(A::x);
     template<typename U>
-    void g3() noexcept(A::y); // expected-error {{no member named 'y' in 'A<T>'}}
+    void g3() noexcept(A::y); // expected-error {{no member named 'y' in 'N1::A<T>'}}
 
     friend void f4() noexcept(x);
     friend void g4() noexcept(y); // expected-error {{use of undeclared identifier 'y'}}
 
     friend void f5() noexcept(A::x);
-    friend void g5() noexcept(A::y); // expected-error {{no member named 'y' in 'A<T>'}}
+    friend void g5() noexcept(A::y); // expected-error {{no member named 'y' in 'N1::A<T>'}}
 
     template<typename U>
     friend void f6() noexcept(x);
@@ -71,7 +71,7 @@ namespace N1 {
     template<typename U>
     friend void f7() noexcept(A::x);
     template<typename U>
-    friend void g7() noexcept(A::y); // expected-error {{no member named 'y' in 'A<T>'}}
+    friend void g7() noexcept(A::y); // expected-error {{no member named 'y' in 'N1::A<T>'}}
 
     static constexpr bool x = true;
   };
diff --git a/clang/test/CXX/class/class.mem/p13.cpp b/clang/test/CXX/class/class.mem/p13.cpp
index d947586c41940..a30aa5d0b2ee3 100644
--- a/clang/test/CXX/class/class.mem/p13.cpp
+++ b/clang/test/CXX/class/class.mem/p13.cpp
@@ -114,3 +114,12 @@ template<typename B> struct CtorDtorName : B {
   CtorDtorName();
   ~CtorDtorName(); // expected-error {{identifier 'CtorDtorName' after '~' in destructor name does not name a type}}
 };
+
+struct S { // expected-note {{'S' declared here}}
+  enum E {
+    R = 11,
+    S = 12  // expected-error {{member 'S' has the same name as its class}}
+  };
+  static_assert(E::R == 11, "E::R is not 11");
+  static_assert(E::S == 12, "E::S is not 12");  // expected-error {{no member named 'S' in 'S::E'}}
+};
diff --git a/clang/test/CXX/class/class.union/class.union.anon/p4.cpp b/clang/test/CXX/class/class.union/class.union.anon/p4.cpp
index a12ec38503fa8..f124ac1052b7e 100644
--- a/clang/test/CXX/class/class.union/class.union.anon/p4.cpp
+++ b/clang/test/CXX/class/class.union/class.union.anon/p4.cpp
@@ -8,3 +8,28 @@ union U {
     int y = 1; // expected-error {{initializing multiple members of union}}
   };
 };
+
+namespace GH149985 {
+  union X {
+    enum {
+      csize = 42,
+      cs = sizeof(int) // expected-note {{previous declaration is here}}
+    };
+    struct {
+      int data; // expected-note {{previous declaration is here}}
+      union X *cs[csize] = {}; // expected-error {{member of anonymous struct redeclares}} expected-note {{previous initialization is here}}
+    };
+    struct {
+      int data; // expected-error {{member of anonymous struct redeclares}}
+      union X *ds[2] = {}; // expected-error {{initializing multiple members of union}}
+    };
+  };
+
+  union U {
+    int x; // expected-note {{previous declaration is here}}
+    union {
+      int x = {}; // expected-error {{member of anonymous union redeclares}} expected-note {{previous initialization is here}}
+    };
+    int y = {}; // expected-error {{initializing multiple members of union}}
+  };
+}
diff --git a/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.nodiscard/p2.cpp b/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.nodiscard/p2.cpp
index 0012ab976baa5..7f933a4dcc6b2 100644
--- a/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.nodiscard/p2.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.attr/dcl.attr.nodiscard/p2.cpp
@@ -115,7 +115,7 @@ void usage() {
   S();    // expected-warning {{ignoring temporary created by a constructor declared with 'nodiscard' attribute}}
   S('A'); // expected-warning {{ignoring temporary created by a constructor declared with 'nodiscard' attribute: Don't let that S-Char go!}}
   S(1);
-  S(2.2);
+  S(2.2); // expected-warning {{ignoring temporary created by a constructor declared with 'gnu::warn_unused_result' attribute}}
   Y(); // expected-warning {{ignoring temporary of type 'Y' declared with 'nodiscard' attribute: Don't throw me away either!}}
   S s;
   ConvertTo{}; // expected-warning {{ignoring return value of type 'ConvertTo' declared with 'nodiscard' attribute: Don't throw me away!}}
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p3.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p3.cpp
index 8bdd490531119..c7b331e84cc0f 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p3.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.type/dcl.type.elab/p3.cpp
@@ -38,7 +38,7 @@ template <> struct B<A> {
 
 void b1(struct B<float>);
 void b2(class B<float>);
-void b3(union B<float>); // expected-error {{use of 'B<float>' with tag type that does not match previous declaration}}
+void b3(union B<float>); // expected-error {{use of 'union B<float>' with tag type that does not match previous declaration}}
 //void b4(enum B<float>); // this just doesn't parse; you can't template an enum directly
 
 void c1(struct B<float>::Member);
diff --git a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.ref/p5-examples.cpp b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.ref/p5-examples.cpp
index 77aef99eaa7c8..ad3b0145d6cbb 100644
--- a/clang/test/CXX/dcl.decl/dcl.init/dcl.init.ref/p5-examples.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.init/dcl.init.ref/p5-examples.cpp
@@ -22,7 +22,7 @@ void example1() {
   // CHECK: VarDecl{{.*}}rca 'const A &'
   // CHECK: ImplicitCastExpr{{.*}}'const A' lvalue <DerivedToBase (A)>
   // CHECK-NOT: MaterializeTemporaryExpr
-  // CHECK: ImplicitCastExpr{{.*}}'const B' lvalue <NoOp>
+  // CHECK: ImplicitCastExpr{{.*}}'const struct B' lvalue <NoOp>
   const A& rca = b;
 }
 
diff --git a/clang/test/CXX/dcl.decl/dcl.meaning/dcl.mptr/p2.cpp b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.mptr/p2.cpp
index a06b107755596..bb1eb732b063b 100644
--- a/clang/test/CXX/dcl.decl/dcl.meaning/dcl.mptr/p2.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.meaning/dcl.mptr/p2.cpp
@@ -59,6 +59,6 @@ namespace TypoCorrection {
   int A<T>::template typo<int>::* f();
 
   template<typename T>
-  int A<T>::typo<int>::* g(); // expected-error {{no template named 'typo' in 'A<T>'; did you mean 'Typo'?}}
+  int A<T>::typo<int>::* g(); // expected-error {{no template named 'typo' in 'TypoCorrection::A<T>'; did you mean 'Typo'?}}
                               // expected-error@-1 {{expected unqualified-id}}
 }
diff --git a/clang/test/CXX/dcl/dcl.fct/p17.cpp b/clang/test/CXX/dcl/dcl.fct/p17.cpp
index 4a81875cb4f89..431a8fbdf38a0 100644
--- a/clang/test/CXX/dcl/dcl.fct/p17.cpp
+++ b/clang/test/CXX/dcl/dcl.fct/p17.cpp
@@ -109,7 +109,7 @@ namespace unconstrained {
   template<typename T>
   template<typename U>
   constexpr auto S<T>::f2(auto x, U u, T t) -> decltype(x + u + t) { return x + u + t; }
-  // expected-error@-1 {{out-of-line definition of 'f2' does not match any declaration in 'S<T>'}}
+  // expected-error@-1 {{out-of-line definition of 'f2' does not match any declaration in 'unconstrained::S<T>'}}
   // expected-note@#defined-here {{S defined here}}
 
   template<typename T>
diff --git a/clang/test/CXX/drs/cwg0xx.cpp b/clang/test/CXX/drs/cwg0xx.cpp
index 4d4e2f64ff643..9aeee7a76903d 100644
--- a/clang/test/CXX/drs/cwg0xx.cpp
+++ b/clang/test/CXX/drs/cwg0xx.cpp
@@ -859,7 +859,7 @@ namespace cwg54 { // cwg54: 2.8
   // expected-error@-1 {{cannot cast 'struct B' to its private base class 'A'}}
   //   expected-note@#cwg54-B {{declared private here}}
   int A::*smab = static_cast<int A::*>(&B::b);
-  // expected-error@-1 {{cannot cast 'B' to its private base class 'A'}}
+  // expected-error@-1 {{cannot cast 'cwg54::B' to its private base class 'A'}}
   //   expected-note@#cwg54-B {{declared private here}}
   B &sba = static_cast<B&>(a);
   // expected-error@-1 {{cannot cast private base class 'cwg54::A' to 'cwg54::B'}}
@@ -874,7 +874,7 @@ namespace cwg54 { // cwg54: 2.8
   V &svb = static_cast<V&>(b);
   V *spvb = static_cast<V*>(&b);
   int V::*smvb = static_cast<int V::*>(&B::b);
-  // expected-error@-1 {{conversion from pointer to member of class 'B' to pointer to member of class 'V' via virtual base 'cwg54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::B' to pointer to member of class 'V' via virtual base 'cwg54::V' is not allowed}}
   B &sbv = static_cast<B&>(v);
   // expected-error@-1 {{cannot cast 'struct V' to 'B &' via virtual base 'cwg54::V'}}
   B *spbv = static_cast<B*>(&v);
@@ -892,7 +892,7 @@ namespace cwg54 { // cwg54: 2.8
   V &cvb = (V&)(b);
   V *cpvb = (V*)(&b);
   int V::*cmvb = (int V::*)(&B::b);
-  // expected-error@-1 {{conversion from pointer to member of class 'B' to pointer to member of class 'V' via virtual base 'cwg54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::B' to pointer to member of class 'V' via virtual base 'cwg54::V' is not allowed}}
   B &cbv = (B&)(v);
   // expected-error@-1 {{cannot cast 'struct V' to 'B &' via virtual base 'cwg54::V'}}
   B *cpbv = (B*)(&v);
diff --git a/clang/test/CXX/drs/cwg12xx.cpp b/clang/test/CXX/drs/cwg12xx.cpp
index e02a7e11b80b2..03a92287008c0 100644
--- a/clang/test/CXX/drs/cwg12xx.cpp
+++ b/clang/test/CXX/drs/cwg12xx.cpp
@@ -81,7 +81,7 @@ void g() {
     A b(auto ()->C);
     static_assert(sizeof(B ()->C[1] == sizeof(int)), "");
     sizeof(auto () -> C[1]);
-    // since-cxx11-error@-1 {{function cannot return array type 'C[1]' (aka 'cwg1223::BB[1]')}}
+    // since-cxx11-error@-1 {{function cannot return array type 'C[1]' (aka 'struct BB[1]')}}
 }
 #endif
 } // namespace cwg1223
diff --git a/clang/test/CXX/drs/cwg13xx.cpp b/clang/test/CXX/drs/cwg13xx.cpp
index c4acafd5fd79a..ad6ee01483b4a 100644
--- a/clang/test/CXX/drs/cwg13xx.cpp
+++ b/clang/test/CXX/drs/cwg13xx.cpp
@@ -252,7 +252,7 @@ namespace cwg1330 { // cwg1330: 4 c++11
   void (B<P>::*bpf3)() = &B<P>::f;
   void (B<P>::*bpf4)() throw() = &B<P>::f;
   // cxx98-14-error@-1 {{target exception specification is not superset of source}}
-  // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (B<P>::*)() throw()' with an rvalue of type 'void (B<P>::*)() throw(T, typename P::type)': different exception specifications}}
+  // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (B<P>::*)() throw()' with an rvalue of type 'void (B<P>::*)() throw(T, typename cwg1330::P::type)': different exception specifications}}
 
 #if __cplusplus >= 201103L
   static_assert(noexcept(B<P>().g()), "");
diff --git a/clang/test/CXX/drs/cwg2149.cpp b/clang/test/CXX/drs/cwg2149.cpp
index 416c895e9ff30..6b54223eb6785 100644
--- a/clang/test/CXX/drs/cwg2149.cpp
+++ b/clang/test/CXX/drs/cwg2149.cpp
@@ -56,22 +56,22 @@ void f() {
 
 // CXX98:       VarDecl {{.+}} a 'X[2]'
 // CXX98-NEXT:  `-InitListExpr {{.+}} 'X[2]'
-// CXX98-NEXT:    |-InitListExpr {{.+}} 'X':'cwg2149::X'
+// CXX98-NEXT:    |-InitListExpr {{.+}} 'X'{{$}}
 // CXX98-NEXT:    | |-IntegerLiteral {{.+}} 'int' 1
 // CXX98-NEXT:    | |-IntegerLiteral {{.+}} 'int' 2
 // CXX98-NEXT:    | `-IntegerLiteral {{.+}} 'int' 3
-// CXX98-NEXT:    `-InitListExpr {{.+}} 'X':'cwg2149::X'
+// CXX98-NEXT:    `-InitListExpr {{.+}} 'X'{{$}}
 // CXX98-NEXT:      |-IntegerLiteral {{.+}} 'int' 4
 // CXX98-NEXT:      |-IntegerLiteral {{.+}} 'int' 5
 // CXX98-NEXT:      `-IntegerLiteral {{.+}} 'int' 6
 
 // CXX98:       VarDecl {{.+}} b 'X[2]'
 // CXX98-NEXT:  `-InitListExpr {{.+}} 'X[2]'
-// CXX98-NEXT:    |-InitListExpr {{.+}} 'X':'cwg2149::X'
+// CXX98-NEXT:    |-InitListExpr {{.+}} 'X'{{$}}
 // CXX98-NEXT:    | |-IntegerLiteral {{.+}} 'int' 1
 // CXX98-NEXT:    | |-IntegerLiteral {{.+}} 'int' 2
 // CXX98-NEXT:    | `-IntegerLiteral {{.+}} 'int' 3
-// CXX98-NEXT:    `-InitListExpr {{.+}} 'X':'cwg2149::X'
+// CXX98-NEXT:    `-InitListExpr {{.+}} 'X'{{$}}
 // CXX98-NEXT:      |-IntegerLiteral {{.+}} 'int' 4
 // CXX98-NEXT:      |-IntegerLiteral {{.+}} 'int' 5
 // CXX98-NEXT:      `-IntegerLiteral {{.+}} 'int' 6
diff --git a/clang/test/CXX/drs/cwg26xx.cpp b/clang/test/CXX/drs/cwg26xx.cpp
index 426fc6cab66a8..bceef6419b00a 100644
--- a/clang/test/CXX/drs/cwg26xx.cpp
+++ b/clang/test/CXX/drs/cwg26xx.cpp
@@ -291,12 +291,12 @@ static_assert(__is_same(decltype(i), I<char, 4>));
 
 J j = { "ghi" };
 // since-cxx20-error@-1 {{no viable constructor or deduction guide}}
-//   since-cxx20-note@#cwg2681-J {{candidate template ignored: could not match 'J<N>' against 'const char *'}}
-//   since-cxx20-note@#cwg2681-J {{implicit deduction guide declared as 'template <size_t N> J(J<N>) -> J<N>'}}
+//   since-cxx20-note@#cwg2681-J {{candidate template ignored: could not match 'cwg2681::J<N>' against 'const char *'}}
+//   since-cxx20-note@#cwg2681-J {{implicit deduction guide declared as 'template <size_t N> J(cwg2681::J<N>) -> cwg2681::J<N>'}}
 //   since-cxx20-note@#cwg2681-J {{candidate template ignored: could not match 'const unsigned char' against 'const char'}}
-//   since-cxx20-note@#cwg2681-J {{implicit deduction guide declared as 'template <size_t N> J(const unsigned char (&)[N]) -> J<N>'}}
+//   since-cxx20-note@#cwg2681-J {{implicit deduction guide declared as 'template <size_t N> J(const unsigned char (&)[N]) -> cwg2681::J<N>'}}
 //   since-cxx20-note@#cwg2681-J {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
-//   since-cxx20-note@#cwg2681-J {{implicit deduction guide declared as 'template <size_t N> J() -> J<N>'}}
+//   since-cxx20-note@#cwg2681-J {{implicit deduction guide declared as 'template <size_t N> J() -> cwg2681::J<N>'}}
 #endif
 } // namespace cwg2681
 
diff --git a/clang/test/CXX/drs/cwg279.cpp b/clang/test/CXX/drs/cwg279.cpp
index 3c63486cc0dd5..d10ceb2f05d30 100644
--- a/clang/test/CXX/drs/cwg279.cpp
+++ b/clang/test/CXX/drs/cwg279.cpp
@@ -46,8 +46,8 @@ extern S2 *q2;
 
 // FIXME: This is well-formed, because [basic.def.odr]/15 is satisfied.
 struct S3 {};
-// since-cxx20-error@-1 {{redefinition of 'S3'}}
-//   since-cxx20-note@cwg279_A.cppm:23 {{previous definition is here}}
+// since-cxx20-error@-1 {{declaration of 'S3' in the global module follows declaration in module cwg279_A}}
+//   since-cxx20-note@cwg279_A.cppm:23 {{previous declaration is here}}
 extern S3 *q3;
 // since-cxx20-error@-1 {{declaration of 'q3' in the global module follows declaration in module cwg279_A}}
 //   since-cxx20-note@cwg279_A.cppm:24 {{previous declaration is here}}
diff --git a/clang/test/CXX/drs/cwg28xx.cpp b/clang/test/CXX/drs/cwg28xx.cpp
index b32e649374893..a6b2b99e0c3f1 100644
--- a/clang/test/CXX/drs/cwg28xx.cpp
+++ b/clang/test/CXX/drs/cwg28xx.cpp
@@ -227,12 +227,12 @@ void f() {
   auto L2 = [&](this auto&& self) { (void) &x; };
   O1<decltype(L1)>{L1, L1}();
   /* since-cxx23-error-re@-1 {{inaccessible due to ambiguity:
-    struct cwg2881::O1<class (lambda at {{.+}})> -> A<(lambda at {{.+}})> -> class (lambda at {{.+}})
-    struct cwg2881::O1<class (lambda at {{.+}})> -> B<(lambda at {{.+}})> -> class (lambda at {{.+}})}}*/
+    struct cwg2881::O1<class (lambda at {{.+}})> -> A<class (lambda at {{.+}})> -> class (lambda at {{.+}})
+    struct cwg2881::O1<class (lambda at {{.+}})> -> B<class (lambda at {{.+}})> -> class (lambda at {{.+}})}}*/
   O1<decltype(L2)>{L2, L2}();
   /* since-cxx23-error-re@-1 {{inaccessible due to ambiguity:
-    struct cwg2881::O1<class (lambda at {{.+}})> -> A<(lambda at {{.+}})> -> class (lambda at {{.+}})
-    struct cwg2881::O1<class (lambda at {{.+}})> -> B<(lambda at {{.+}})> -> class (lambda at {{.+}})}}*/
+    struct cwg2881::O1<class (lambda at {{.+}})> -> A<class (lambda at {{.+}})> -> class (lambda at {{.+}})
+    struct cwg2881::O1<class (lambda at {{.+}})> -> B<class (lambda at {{.+}})> -> class (lambda at {{.+}})}}*/
   O2{L1}();
   // since-cxx23-error-re@-1 {{invalid explicit object parameter type 'cwg2881::O2<(lambda at {{.+}})>' in lambda with capture; the type must derive publicly from the lambda}}
   //   since-cxx23-note@#cwg2881-O2 {{declared protected here}}
@@ -277,7 +277,7 @@ struct Indirect : T {
 template<typename T>
 struct Ambiguous : Indirect<T>, T {
 /* since-cxx23-warning-re@-1 {{direct base '(lambda at {{.+}})' is inaccessible due to ambiguity:
-    struct cwg2881::Ambiguous<class (lambda at {{.+}})> -> Indirect<(lambda at {{.+}})> -> class (lambda at {{.+}})
+    struct cwg2881::Ambiguous<class (lambda at {{.+}})> -> Indirect<class (lambda at {{.+}})> -> class (lambda at {{.+}})
     struct cwg2881::Ambiguous<class (lambda at {{.+}})> -> class (lambda at {{.+}})}}*/
 //   since-cxx23-note-re@#cwg2881-f4 {{in instantiation of template class 'cwg2881::Ambiguous<(lambda at {{.+}})>' requested here}}
 //   since-cxx34-note-re@#cwg2881-f4-call {{while substituting deduced template arguments into function template 'f4' [with L = (lambda at {{.+}})]}}
@@ -303,7 +303,7 @@ void g() {
   // expected-error@-1 {{no matching function for call to 'f4'}}
   //   expected-note-re@-2 {{while substituting deduced template arguments into function template 'f4' [with L = (lambda at {{.+}})]}}
   /*   expected-note-re@#cwg2881-f4 {{candidate template ignored: substitution failure [with L = (lambda at {{.+}})]: lambda '(lambda at {{.+}})' is inaccessible due to ambiguity:
-    struct cwg2881::Ambiguous<class (lambda at {{.+}})> -> Indirect<(lambda at {{.+}})> -> class (lambda at {{.+}})
+    struct cwg2881::Ambiguous<class (lambda at {{.+}})> -> Indirect<class (lambda at {{.+}})> -> class (lambda at {{.+}})
     struct cwg2881::Ambiguous<class (lambda at {{.+}})> -> class (lambda at {{.+}})}}*/
   static_assert(!is_callable<Private<decltype(lambda)>>);
   static_assert(!is_callable<Ambiguous<decltype(lambda)>>);
diff --git a/clang/test/CXX/drs/cwg2xx.cpp b/clang/test/CXX/drs/cwg2xx.cpp
index 556407afa2641..37186e3c3f205 100644
--- a/clang/test/CXX/drs/cwg2xx.cpp
+++ b/clang/test/CXX/drs/cwg2xx.cpp
@@ -98,8 +98,8 @@ template <class T> class Templ { // #cwg203-ex3-Templ
 
 void foo() { Templ<Derived> x(&Derived::func); }
 // expected-error@-1 {{no matching constructor for initialization of 'Templ<Derived>'}}
-//   expected-note@#cwg203-ex3-Templ {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int (Derived::*)() const' (aka 'int (Base::*)() const') to 'const Templ<cwg203::ex3::Derived>' for 1st argument}}
-//   since-cxx11-note@#cwg203-ex3-Templ {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int (Derived::*)() const' (aka 'int (Base::*)() const') to 'Templ<cwg203::ex3::Derived>' for 1st argument}}
+//   expected-note@#cwg203-ex3-Templ {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int (Derived::*)() const' (aka 'int (cwg203::ex3::Base::*)() const') to 'const Templ<cwg203::ex3::Derived>' for 1st argument}}
+//   since-cxx11-note@#cwg203-ex3-Templ {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int (Derived::*)() const' (aka 'int (cwg203::ex3::Base::*)() const') to 'Templ<cwg203::ex3::Derived>' for 1st argument}}
 //   expected-note@#cwg203-ex3-Templ-ctor {{candidate template ignored: could not match 'cwg203::ex3::Derived' against 'cwg203::ex3::Base'}}
 } // namespace ex3
 
@@ -690,8 +690,8 @@ namespace cwg244 { // cwg244: 11
 
   void f() {
     D_object.~B();
-    // expected-error@-1 {{destructor type 'cwg244::B' in object destruction expression does not match the type 'D' of the object being destroyed}}
-    //   expected-note@#cwg244-B {{type 'cwg244::B' found by destructor name lookup}}
+    // expected-error@-1 {{destructor type 'B' in object destruction expression does not match the type 'D' of the object being destroyed}}
+    //   expected-note@#cwg244-B {{type 'B' found by destructor name lookup}}
     D_object.B::~B();
     D_object.D::~B(); // FIXME: Missing diagnostic for this.
     B_ptr->~B();
@@ -1400,7 +1400,7 @@ namespace cwg298 { // cwg298: 3.1
   // expected-error@-1 {{a type specifier is required for all declarations}}
   B::A() {} // ok
   C::~C() {}
-  // expected-error@-1 {{destructor cannot be declared using a typedef 'C' (aka 'const cwg298::A') of the class name}}
+  // expected-error@-1 {{destructor cannot be declared using a typedef 'C' (aka 'const A') of the class name}}
 
   typedef struct D E; // #cwg298-E
   struct E {};
diff --git a/clang/test/CXX/drs/cwg3xx.cpp b/clang/test/CXX/drs/cwg3xx.cpp
index 8b035cf6f2370..5d09697a3cf6b 100644
--- a/clang/test/CXX/drs/cwg3xx.cpp
+++ b/clang/test/CXX/drs/cwg3xx.cpp
@@ -143,7 +143,7 @@ namespace cwg305 { // cwg305: no
   void h(B *b) {
     struct B {}; // #cwg305-h-B
     b->~B();
-    // expected-error@-1 {{destructor type 'B' in object destruction expression does not match the type 'B' (aka 'cwg305::A') of the object being destroyed}}
+    // expected-error@-1 {{destructor type 'B' in object destruction expression does not match the type 'B' (aka 'A') of the object being destroyed}}
     //   expected-note@#cwg305-h-B {{type 'B' found by destructor name lookup}}
   }
 
@@ -1027,7 +1027,7 @@ namespace cwg357 { // cwg357: 2.7
     void f() const; // #cwg357-f
   };
   template<typename T> void A<T>::f() {}
-  // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'A<T>'}}
+  // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'cwg357::A<T>'}}
   //   expected-note@#cwg357-A {{defined here}}
   //   expected-note@#cwg357-f {{member declaration does not match because it is const qualified}}
 
@@ -1800,8 +1800,8 @@ namespace cwg399 { // cwg399: 11
 
   void f() {
     D_object.~B();
-    // expected-error@-1 {{destructor type 'cwg399::B' in object destruction expression does not match the type 'D' of the object being destroyed}}
-    //   expected-note@#cwg399-B {{type 'cwg399::B' found by destructor name lookup}}
+    // expected-error@-1 {{destructor type 'B' in object destruction expression does not match the type 'D' of the object being destroyed}}
+    //   expected-note@#cwg399-B {{type 'B' found by destructor name lookup}}
     D_object.B::~B();
     D_object.D::~B(); // FIXME: Missing diagnostic for this.
     B_ptr->~B();
diff --git a/clang/test/CXX/drs/cwg4xx.cpp b/clang/test/CXX/drs/cwg4xx.cpp
index dc53ae712a0b5..8497f974b2ece 100644
--- a/clang/test/CXX/drs/cwg4xx.cpp
+++ b/clang/test/CXX/drs/cwg4xx.cpp
@@ -1185,7 +1185,7 @@ namespace cwg480 { // cwg480: 2.7
 
   extern int D::*c;
   int A::*d = static_cast<int A::*>(c);
-  // expected-error@-1 {{conversion from pointer to member of class 'D' to pointer to member of class 'A' via virtual base 'cwg480::B' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg480::D' to pointer to member of class 'A' via virtual base 'cwg480::B' is not allowed}}
 
   D *e;
   A *f = e;
diff --git a/clang/test/CXX/drs/cwg6xx.cpp b/clang/test/CXX/drs/cwg6xx.cpp
index e2eb009508b52..659fa499a2f49 100644
--- a/clang/test/CXX/drs/cwg6xx.cpp
+++ b/clang/test/CXX/drs/cwg6xx.cpp
@@ -383,7 +383,7 @@ namespace cwg635 { // cwg635: 2.7
   template<typename T> template<typename U> D<T>::D() {}
   template<typename T> D<T>::D<T>() {} // #cwg635-D-T
   // expected-error@#cwg635-D-T {{out-of-line constructor for 'D' cannot have template arguments}}
-  // expected-error@#cwg635-D-T {{redefinition of 'D<T>'}}
+  // expected-error@#cwg635-D-T {{redefinition of 'cwg635::D<T>'}}
   //   expected-note@#cwg635-D {{previous definition is here}}
 } // namespace cwg635
 
@@ -895,12 +895,12 @@ namespace cwg666 { // cwg666: 2.8
   template<int> int f();
   template<typename T> int f() {
     T::type *p = 0;
-    // expected-error@-1 {{missing 'typename' prior to dependent type name 'Y::type'}}
+    // expected-error@-1 {{missing 'typename' prior to dependent type name 'cwg666::Y::type'}}
     //   expected-note@#cwg666-f-Y {{in instantiation of function template specialization 'cwg666::f<cwg666::Y>' requested here}}
     int a(T::type);
-    // expected-error@-1 {{missing 'typename' prior to dependent type name 'Y::type'}}
+    // expected-error@-1 {{missing 'typename' prior to dependent type name 'cwg666::Y::type'}}
     return f<T::type>();
-    // expected-error@-1 {{missing 'typename' prior to dependent type name 'Y::type'}}
+    // expected-error@-1 {{missing 'typename' prior to dependent type name 'cwg666::Y::type'}}
   }
   struct X { static const int type = 0; };
   struct Y { typedef int type; };
diff --git a/clang/test/CXX/expr/expr.prim/expr.prim.req/compound-requirement.cpp b/clang/test/CXX/expr/expr.prim/expr.prim.req/compound-requirement.cpp
index dc0e84280e056..31587a956b8ab 100644
--- a/clang/test/CXX/expr/expr.prim/expr.prim.req/compound-requirement.cpp
+++ b/clang/test/CXX/expr/expr.prim/expr.prim.req/compound-requirement.cpp
@@ -160,7 +160,7 @@ namespace std_example {
   template<typename T> concept C2 =
     requires(T x) {
       {*x} -> same_as<typename T::inner>;
-      // expected-note@-1{{because type constraint 'same_as<int, typename T2::inner>' was not satisfied:}}
+      // expected-note@-1{{because type constraint 'same_as<int, typename std_example::T2::inner>' was not satisfied:}}
       // expected-note@-2{{because '*x' would be invalid: indirection requires pointer operand ('int' invalid)}}
     };
 
diff --git a/clang/test/CXX/module/cpp.pre/module_decl.cpp b/clang/test/CXX/module/cpp.pre/module_decl.cpp
index 6238347c167ac..5c29aeff1b632 100644
--- a/clang/test/CXX/module/cpp.pre/module_decl.cpp
+++ b/clang/test/CXX/module/cpp.pre/module_decl.cpp
@@ -1,8 +1,147 @@
 // RUN: rm -rf %t
 // RUN: mkdir -p %t
-// RUN: %clang_cc1 -std=c++20 -emit-module-interface %s -verify -o %t/M.pcm
+// RUN: split-file %s %t
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/line.cpp -verify -o %t/line.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/gnu_line_marker.cpp -verify -o %t/gnu_line_marker.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/include.cpp -verify -o %t/include.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/ident.cpp -verify -o %t/ident.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_comment.cpp -verify -o %t/pragma_comment.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_mark.cpp -verify -o %t/pragma_mark.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_detect_mismatch.cpp -verify -o %t/pragma_detect_mismatch.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_clang_debug.cpp -verify -o %t/pragma_clang_debug.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_message.cpp -verify -o %t/pragma_message.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_gcc_warn.cpp -verify -o %t/pragma_gcc_warn.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_gcc_error.cpp -verify -o %t/pragma_gcc_error.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_diag_push_pop.cpp -verify -o %t/pragma_diag_push_pop.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_diag_ignore.cpp -verify -o %t/pragma_diag_ignore.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_opencl_ext.cpp -verify -o %t/pragma_opencl_ext.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_push_pop.cpp -verify -o %t/pragma_push_pop.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_exec_charset.cpp -verify -o %t/pragma_exec_charset.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/pragma_clang_assume_nonnull.cpp -verify -o %t/pragma_clang_assume_nonnull.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/marco_expand.cpp -DMACRO="" -verify -o %t/marco_expand.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/define.cpp -verify -o %t/define.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/undef.cpp -verify -o %t/undef.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/defined.cpp -verify -o %t/defined.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/has_embed.cpp -verify -o %t/has_embed.pcm
+// RUN: %clang_cc1 -std=c++20 -emit-module-interface %t/has_include.cpp -verify -o %t/has_include.pcm
 
+//--- header.h
+#ifndef HEADER_H
+#define HEADER_H
+
+#endif // HEADER_H
+
+//--- line.cpp
+// expected-no-diagnostics
+#line 3
+export module M;
+
+//--- gnu_line_marker.cpp
+// expected-no-diagnostics
+# 1 __FILE__ 1 3
+export module M;
+
+//--- include.cpp
+#include "header.h" // expected-note {{add 'module;' to the start of the file to introduce a global module fragment}}
+export module M; // expected-error {{module declaration must occur at the start of the translation unit}}
+
+//--- ident.cpp
+// expected-no-diagnostics
+#ident "$Header:$"
+export module M;
+
+//--- pragma_comment.cpp
+// expected-no-diagnostics
+#pragma comment(lib, "msvcrt.lib")
+export module M;
+
+//--- pragma_mark.cpp
+// expected-no-diagnostics
+#pragma mark LLVM's world
+export module M;
+
+//--- pragma_detect_mismatch.cpp
+// expected-no-diagnostics
+#pragma detect_mismatch("test", "1")
+export module M;
+
+//--- pragma_clang_debug.cpp
+// expected-no-diagnostics
+#pragma clang __debug dump Test
+export module M;
+
+//--- pragma_message.cpp
+#pragma message "test" // expected-warning {{test}}
+export module M;
+
+//--- pragma_gcc_warn.cpp
+#pragma GCC warning "Foo" // expected-warning {{Foo}}
+export module M;
+
+//--- pragma_gcc_error.cpp
+#pragma GCC error "Foo" // expected-error {{Foo}}
+export module M;
+
+//--- pragma_diag_push_pop.cpp
+// expected-no-diagnostics
+#pragma gcc diagnostic push
+#pragma gcc diagnostic pop
+export module M;
+
+//--- pragma_diag_ignore.cpp
+// expected-no-diagnostics
+#pragma GCC diagnostic ignored "-Wframe-larger-than"
+export module M;
+
+//--- pragma_opencl_ext.cpp
+// expected-no-diagnostics
+#pragma OPENCL EXTENSION __cl_clang_variadic_functions : enable
+export module M;
+
+//--- pragma_push_pop.cpp
+// expected-no-diagnostics
+#pragma warning(push)
+#pragma warning(pop)
+export module M;
+
+//--- pragma_exec_charset.cpp
+// expected-no-diagnostics
+#pragma execution_character_set(push, "UTF-8")
+#pragma execution_character_set(pop)
+export module M;
+
+//--- pragma_clang_assume_nonnull.cpp
+// expected-no-diagnostics
+#pragma clang assume_nonnull begin
+#pragma clang assume_nonnull end
+export module M;
+
+//--- marco_expand.cpp
+MACRO // expected-note {{add 'module;' to the start of the file to introduce a global module fragment}}
+export module M; // expected-error {{module declaration must occur at the start of the translation unit}}
+
+//--- define.cpp
 // This is a comment
 #define I32 int // expected-note {{add 'module;' to the start of the file to introduce a global module fragment}}
 export module M; // expected-error {{module declaration must occur at the start of the translation unit}}
 export I32 i32;
+
+//--- undef.cpp
+#undef FOO // expected-note {{add 'module;' to the start of the file to introduce a global module fragment}}
+export module M; // expected-error {{module declaration must occur at the start of the translation unit}}
+
+//--- defined.cpp
+#if defined(FOO) // expected-note {{add 'module;' to the start of the file to introduce a global module fragment}}
+#endif
+export module M; // expected-error {{module declaration must occur at the start of the translation unit}}
+
+//--- has_embed.cpp
+#if __has_embed(__FILE__ ext::token(0xB055)) // expected-note {{add 'module;' to the start of the file to introduce a global module fragment}}
+#endif
+export module M; // expected-error {{module declaration must occur at the start of the translation unit}}
+
+//--- has_include.cpp
+#if __has_include(<stdio.h>) || __has_include_next(<stdlib.h>) // expected-note {{add 'module;' to the start of the file to introduce a global module fragment}} \
+                                                               // expected-warning {{#include_next in primary source file; will search from start of include path}}
+#endif
+export module M; // expected-error {{module declaration must occur at the start of the translation unit}}
diff --git a/clang/test/CXX/over/over.match/over.match.funcs/over.match.class.deduct/p2.cpp b/clang/test/CXX/over/over.match/over.match.funcs/over.match.class.deduct/p2.cpp
index d192070132d78..a32295ee224fb 100644
--- a/clang/test/CXX/over/over.match/over.match.funcs/over.match.class.deduct/p2.cpp
+++ b/clang/test/CXX/over/over.match/over.match.funcs/over.match.class.deduct/p2.cpp
@@ -37,7 +37,7 @@ namespace std {
 namespace p0702r1 {
   template<typename T> struct X { // expected-note {{candidate}} expected-note {{implicit deduction guide}}
     X(std::initializer_list<T>); // expected-note {{candidate template ignored: could not match 'std::initializer_list<T>' against 'Z'}} \
-                                 // expected-note {{implicit deduction guide declared as 'template <typename T> X(std::initializer_list<T>) -> X<T>'}}
+                                 // expected-note {{implicit deduction guide declared as 'template <typename T> X(std::initializer_list<T>) -> p0702r1::X<T>'}}
   };
 
   X xi = {0};
diff --git a/clang/test/CXX/over/over.match/over.match.funcs/over.match.oper/p3.cpp b/clang/test/CXX/over/over.match/over.match.funcs/over.match.oper/p3.cpp
index d88d5beb4e99e..7a0cacc2e65be 100644
--- a/clang/test/CXX/over/over.match/over.match.funcs/over.match.oper/p3.cpp
+++ b/clang/test/CXX/over/over.match/over.match.funcs/over.match.oper/p3.cpp
@@ -13,6 +13,7 @@ enum class E { e };
 
 template<typename T> int f(T t) { return ~t; } // expected-error {{invalid argument type}}
 template<typename T, typename U> int f(T t, U u) { return t % u; } // expected-error {{invalid operands to}}
+                                                                   // expected-note@-1 {{no implicit conversion for scoped enum}}
 
 int b1 = ~E::e; // expected-error {{invalid argument type}}
 int b2 = f(E::e); // expected-note {{in instantiation of}}
diff --git a/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp b/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp
index abb42447d3e0b..05830de9891fe 100644
--- a/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp
+++ b/clang/test/CXX/stmt.stmt/stmt.select/stmt.if/p2.cpp
@@ -239,5 +239,21 @@ void f2() {
 
 }
 
+namespace GH153884 {
+  bool f1() {
+    auto f = [](auto) { return true; };
+    if constexpr (0)
+      return f(1);
+    return false;
+  }
+  bool f2() {
+    auto f = [](auto x) { if (x) return 1.5; else return "wat"; };
+    // expected-error@-1 {{'auto' in return type deduced as 'const char *' here but deduced as 'double' in earlier return statement}}
+    if constexpr (0)
+      return f(1);
+    // expected-note@-1 {{in instantiation of function template specialization 'GH153884::f2()}}
+    return false;
+  }
+}
 
 #endif
diff --git a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp
index 01ce33b6ef51d..4b93d865c8b19 100644
--- a/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp
+++ b/clang/test/CXX/temp/temp.arg/temp.arg.nontype/p5.cpp
@@ -238,9 +238,9 @@ namespace pointer_to_member_data {
   X0<&Y::y> x0a;
   X0<&Y::x> x0b;
 #if __cplusplus <= 201402L
-  // expected-error@-2 {{non-type template argument of type 'int Y::*' (aka 'int X::*') cannot be converted to a value of type 'int Y::*'}}
+  // expected-error@-2 {{non-type template argument of type 'int Y::*' (aka 'int pointer_to_member_data::X::*') cannot be converted to a value of type 'int Y::*'}}
 #else
-  // expected-error@-4 {{conversion from 'int Y::*' (aka 'int X::*') to 'int Y::*' is not allowed in a converted constant expression}}
+  // expected-error@-4 {{conversion from 'int Y::*' (aka 'int pointer_to_member_data::X::*') to 'int Y::*' (aka 'int pointer_to_member_data::Y::*') is not allowed in a converted constant expression}}
 #endif
 
   // Test qualification conversions
diff --git a/clang/test/CXX/temp/temp.constr/temp.constr.decl/func-template-decl.cpp b/clang/test/CXX/temp/temp.constr/temp.constr.decl/func-template-decl.cpp
index 9a8148bdf625f..499e6abc1df1a 100644
--- a/clang/test/CXX/temp/temp.constr/temp.constr.decl/func-template-decl.cpp
+++ b/clang/test/CXX/temp/temp.constr/temp.constr.decl/func-template-decl.cpp
@@ -54,7 +54,7 @@ struct TA { // #defined-here
 
 template <unsigned N>
 template <template <unsigned> class TT> int TA<N>::A() { return sizeof(TT<N>); }
-// expected-error@-1{{out-of-line definition of 'A' does not match any declaration in 'TA<N>'}}
+// expected-error@-1{{out-of-line definition of 'A' does not match any declaration in 'diag::TA<N>'}}
 // expected-note@#defined-here{{defined here}}
 
 } // end namespace diag
diff --git a/clang/test/CXX/temp/temp.decls/temp.class.spec/p6.cpp b/clang/test/CXX/temp/temp.decls/temp.class.spec/p6.cpp
index 2da0382accff3..5345b080ea3da 100644
--- a/clang/test/CXX/temp/temp.decls/temp.class.spec/p6.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.class.spec/p6.cpp
@@ -38,7 +38,7 @@ A<short>::C::B<int*> absip;
 template<typename T, typename U>
 struct Outer {
   template<typename X, typename Y> struct Inner;
-  template<typename Y> struct Inner<T, Y> {}; // expected-note{{previous declaration of class template partial specialization 'Inner<int, Y>' is here}}
+  template<typename Y> struct Inner<T, Y> {}; // expected-note{{previous declaration of class template partial specialization 'Outer<int, int>::Inner<int, Y>' is here}}
   template<typename Y> struct Inner<U, Y> {}; // expected-error{{cannot be redeclared}}
 };
 
@@ -80,7 +80,7 @@ namespace print_dependent_TemplateSpecializationType {
 template <class T, class U> struct Foo {
   template <unsigned long, class X, class Y> struct Bar;
   template <class Y> struct Bar<0, T, Y> {};
-  // expected-note-re@-1 {{previous declaration {{.*}} 'Bar<0, int, Y>' is here}}
+  // expected-note-re@-1 {{previous declaration {{.*}} 'print_dependent_TemplateSpecializationType::Foo<int, int>::Bar<0, int, Y>' is here}}
   template <class Y> struct Bar<0, U, Y> {};
   // expected-error@-1 {{partial specialization 'Bar<0, int, Y>' cannot be redeclared}}
 };
diff --git a/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.call/p3-0x.cpp b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.call/p3-0x.cpp
index 7d5be01a8d017..1eb35ab221ea4 100644
--- a/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.call/p3-0x.cpp
+++ b/clang/test/CXX/temp/temp.fct.spec/temp.deduct/temp.deduct.call/p3-0x.cpp
@@ -78,9 +78,9 @@ namespace std_example {
   template<class T> struct A { // expected-note {{candidate}} expected-note {{implicit deduction guide}}
     template<class U>
     A(T &&, U &&, int *); // expected-note {{[with T = int, U = int] not viable: expects an rvalue}} \
-                          // expected-note {{implicit deduction guide declared as 'template <class T, class U> A(T &&, U &&, int *) -> A<T>'}}
+                          // expected-note {{implicit deduction guide declared as 'template <class T, class U> A(T &&, U &&, int *) -> std_example::A<T>'}}
     A(T &&, int *);       // expected-note {{requires 2}} \
-                          // expected-note {{implicit deduction guide declared as 'template <class T> A(T &&, int *) -> A<T>'}}
+                          // expected-note {{implicit deduction guide declared as 'template <class T> A(T &&, int *) -> std_example::A<T>'}}
   };
   template<class T> A(T &&, int *) -> A<T>; // expected-note {{requires 2}}
 
diff --git a/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp b/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp
index 83144a494937b..238490c3e0d82 100644
--- a/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp
+++ b/clang/test/CXX/temp/temp.param/p15-cxx0x.cpp
@@ -97,14 +97,14 @@ template<unsigned N, typename...Ts> struct drop {
 using T1 = take<3, int, char, double, long>::type; // expected-note {{previous}}
 // FIXME: Desguar the types on the RHS in this diagnostic.
 // desired-error {{'types<void, void, void, void>' vs 'types<int, char, double, (no argument)>'}}
-using T1 = types<void, void, void, void>; // expected-error {{'types<void, void, void, void>' vs 'types<typename inner<_>::type, typename inner<_>::type, typename inner<_>::type, (no argument)>'}}
+using T1 = types<void, void, void, void>; // expected-error {{'types<void, void, void, void>' vs 'types<typename ParameterPackExpansions::wrap<int>::inner<_>::type, typename ParameterPackExpansions::wrap<char>::inner<_>::type, typename ParameterPackExpansions::wrap<double>::inner<_>::type, (no argument)>'}}
 using D1 = drop<3, int, char, double, long>::type;
 using D1 = types<long>;
 
 using T2 = take<4, int, char, double, long>::type; // expected-note {{previous}}
 // FIXME: Desguar the types on the RHS in this diagnostic.
 // desired-error {{'types<void, void, void, void>' vs 'types<int, char, double, long>'}}
-using T2 = types<void, void, void, void>; // expected-error {{'types<void, void, void, void>' vs 'types<typename inner<_>::type, typename inner<_>::type, typename inner<_>::type, typename inner<_>::type>'}}
+using T2 = types<void, void, void, void>; // expected-error {{'types<void, void, void, void>' vs 'types<typename ParameterPackExpansions::wrap<int>::inner<_>::type, typename ParameterPackExpansions::wrap<char>::inner<_>::type, typename ParameterPackExpansions::wrap<double>::inner<_>::type, typename ParameterPackExpansions::wrap<long>::inner<_>::type>'}}
 using T2 = types<int, char, double, long>;
 using D2 = drop<4, int, char, double, long>::type;
 using D2 = types<>;
diff --git a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
index f32f49ef4539a..75805a764c666 100644
--- a/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
+++ b/clang/test/CXX/temp/temp.res/temp.dep/temp.dep.type/p4.cpp
@@ -326,52 +326,52 @@ namespace N0 {
       // None of the following should be found in the current instantiation.
 
       new M4; // expected-error{{unknown type name 'M4'}}
-      new B::M4; // expected-error{{no type named 'M4' in 'B<T>'}}
+      new B::M4; // expected-error{{no type named 'M4' in 'N0::B<T>'}}
       new A::M4; // expected-error{{no type named 'M4' in 'N0::A'}}
       new B::A::M4; // expected-error{{no type named 'M4' in 'N0::A'}}
 
       x4; // expected-error{{use of undeclared identifier 'x4'}}
-      B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
+      B::x4; // expected-error{{no member named 'x4' in 'N0::B<T>'}}
       A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
       B::A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
       f4(); // expected-error{{use of undeclared identifier 'f4'}}
-      B::f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
+      B::f4(); // expected-error{{no member named 'f4' in 'N0::B<T>'}}
       A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
       B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
 
-      this->x4; // expected-error{{no member named 'x4' in 'B<T>'}}
-      this->B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
+      this->x4; // expected-error{{no member named 'x4' in 'N0::B<T>'}}
+      this->B::x4; // expected-error{{no member named 'x4' in 'N0::B<T>'}}
       this->A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
       this->B::A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
-      this->f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
-      this->B::f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
+      this->f4(); // expected-error{{no member named 'f4' in 'N0::B<T>'}}
+      this->B::f4(); // expected-error{{no member named 'f4' in 'N0::B<T>'}}
       this->A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
       this->B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
 
-      a->x4; // expected-error{{no member named 'x4' in 'B<T>'}}
-      a->B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
+      a->x4; // expected-error{{no member named 'x4' in 'N0::B<T>'}}
+      a->B::x4; // expected-error{{no member named 'x4' in 'N0::B<T>'}}
       a->A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
       a->B::A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
-      a->f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
-      a->B::f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
+      a->f4(); // expected-error{{no member named 'f4' in 'N0::B<T>'}}
+      a->B::f4(); // expected-error{{no member named 'f4' in 'N0::B<T>'}}
       a->A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
       a->B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
 
-      (*this).x4; // expected-error{{no member named 'x4' in 'B<T>'}}
-      (*this).B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
+      (*this).x4; // expected-error{{no member named 'x4' in 'N0::B<T>'}}
+      (*this).B::x4; // expected-error{{no member named 'x4' in 'N0::B<T>'}}
       (*this).A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
       (*this).B::A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
-      (*this).f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
-      (*this).B::f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
+      (*this).f4(); // expected-error{{no member named 'f4' in 'N0::B<T>'}}
+      (*this).B::f4(); // expected-error{{no member named 'f4' in 'N0::B<T>'}}
       (*this).A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
       (*this).B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
 
-      b.x4; // expected-error{{no member named 'x4' in 'B<T>'}}
-      b.B::x4; // expected-error{{no member named 'x4' in 'B<T>'}}
+      b.x4; // expected-error{{no member named 'x4' in 'N0::B<T>'}}
+      b.B::x4; // expected-error{{no member named 'x4' in 'N0::B<T>'}}
       b.A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
       b.B::A::x4; // expected-error{{no member named 'x4' in 'N0::A'}}
-      b.f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
-      b.B::f4(); // expected-error{{no member named 'f4' in 'B<T>'}}
+      b.f4(); // expected-error{{no member named 'f4' in 'N0::B<T>'}}
+      b.B::f4(); // expected-error{{no member named 'f4' in 'N0::B<T>'}}
       b.A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
       b.B::A::f4(); // expected-error{{no member named 'f4' in 'N0::A'}}
     }
@@ -424,7 +424,7 @@ namespace N2 {
       void not_instantiated(A *a, B *b) {
         b->x; // expected-error{{no member named 'x' in 'N2::A::B'}}
         b->B::x; // expected-error{{no member named 'x' in 'N2::A::B'}}
-        a->B::C::x; // expected-error{{no member named 'x' in 'A<T>'}}
+        a->B::C::x; // expected-error{{no member named 'x' in 'N2::A<T>'}}
       }
     };
 
diff --git a/clang/test/ClangScanDeps/link-libraries-diag-dup.c b/clang/test/ClangScanDeps/link-libraries-diag-dup.c
new file mode 100644
index 0000000000000..e6612ca7bd216
--- /dev/null
+++ b/clang/test/ClangScanDeps/link-libraries-diag-dup.c
@@ -0,0 +1,57 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: split-file %s %t
+
+//--- module.modulemap
+module A {
+  umbrella header "A.h"
+
+  module B {
+    header "B.h"
+    link "libraryB"
+  }
+
+  explicit module D {
+    header "D.h"
+    link "libraryD"
+  }
+
+  link "libraryA"
+  link "libraryA"
+}
+
+module C {
+  header "C.h"
+  link "libraryA"
+}
+
+//--- A.h
+#include "B.h"
+//--- B.h
+// empty
+//--- C.h
+// empty
+//--- D.h
+// empty
+//--- TU.c
+#include "A.h"
+#include "C.h"
+#include "D.h"
+
+//--- cdb.json.template
+[{
+  "file": "DIR/TU.c",
+  "directory": "DIR",
+  "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -c DIR/TU.c"
+}]
+
+// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json
+// RUN: not clang-scan-deps -compilation-database %t/cdb.json -format \
+// RUN:   experimental-full 2>&1 | FileCheck %s
+
+// Note that module D does not report an error because it is explicit.
+// Therefore we can use CHECK-NEXT for the redeclaration error on line 15.
+// CHECK:      module.modulemap:6:5:  error: link declaration is not allowed in submodules
+// CHECK-NEXT: module.modulemap:15:3: error: redeclaration of link library 'libraryA' [-Wmodule-link-redeclaration]
+// CHECK-NEXT: module.modulemap:14:3: note: previously declared here
+// CHECK-NOT:  module.modulemap:20:3: error: redeclaration of link library 'libraryA'
diff --git a/clang/test/CodeCompletion/cpp23-explicit-object.cpp b/clang/test/CodeCompletion/cpp23-explicit-object.cpp
new file mode 100644
index 0000000000000..ea97237ecd24b
--- /dev/null
+++ b/clang/test/CodeCompletion/cpp23-explicit-object.cpp
@@ -0,0 +1,153 @@
+struct A {
+  void foo(this auto&& self, int arg);
+  void bar(this A self, int arg);
+};
+
+int func1() {
+  A a {};
+  a.
+}
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-2):5 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC1 %s
+// CHECK-CC1: COMPLETION: A : A::
+// CHECK-NEXT-CC1: COMPLETION: bar : [#void#]bar(<#int arg#>)
+// CHECK-NEXT-CC1: COMPLETION: foo : [#void#]foo(<#int arg#>)
+// CHECK-NEXT-CC1: COMPLETION: operator= : [#A &#]operator=(<#const A &#>)
+// CHECK-NEXT-CC1: COMPLETION: operator= : [#A &#]operator=(<#A &&#>)
+// CHECK-NEXT-CC1: COMPLETION: ~A : [#void#]~A()
+
+struct B {
+  template <typename T>
+  void foo(this T&& self, int arg);
+};
+
+int func2() {
+  B b {};
+  b.foo();
+}
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-2):9 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC2 %s
+// CHECK-CC2: OVERLOAD: [#void#]foo(int arg)
+
+// TODO: llvm/llvm-project/146649
+// This is incorrect behavior. Correct Result should be a variant of, 
+// CC3: should be something like [#void#]foo(<#A self#>, <#int arg#>)
+// CC4: should be something like [#void#]bar(<#A self#>, <#int arg#>)
+int func3() {
+  (&A::foo)
+  (&A::bar)
+}
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-3):10 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC3 %s
+// CHECK-CC3: COMPLETION: foo : [#void#]foo<<#class self:auto#>>(<#int arg#>)
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-4):10 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC4 %s
+// CHECK-CC4: COMPLETION: bar : [#void#]bar(<#int arg#>)
+
+int func4() {
+  // TODO (&A::foo)(
+  (&A::bar)()
+}
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-2):13 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC5 %s
+// CHECK-CC5: OVERLOAD: [#void#](<#A#>, int)
+
+struct C {
+  int member {};
+  int memberFnA(int a);
+  int memberFnA(this C&, float a);
+
+  void foo(this C& self) {
+    // Should not offer any members here, since 
+    // it needs to be referenced through `self`.
+    mem
+    // RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):8 -std=c++23 %s | FileCheck --allow-empty %s
+    // CHECK-NOT: COMPLETION: member : [#int#]member
+    // CHECK-NOT: COMPLETION: memberFnA : [#int#]memberFnA(<#int a#>)
+    // CHECK-NOT: COMPLETION: memberFnA : [#int#]memberFnA(<#float a#>)
+  }
+  void bar(this C& self) {
+    // should offer all results
+    self.mem
+    // RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):13 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC6 %s
+    // CHECK-CC6: COMPLETION: member : [#int#]member
+    // CHECK-CC6: COMPLETION: memberFnA : [#int#]memberFnA(<#int a#>)
+    // CHECK-CC6: COMPLETION: memberFnA : [#int#]memberFnA(<#float a#>)
+  }
+  void baz(this C& self) {
+    [&]() {
+      // Should not offer any results
+      mem
+      // RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):10 -std=c++23 %s | FileCheck --allow-empty %s
+      // CHECK-NOT: COMPLETION: member : [#int#]member
+      // CHECK-NOT: COMPLETION: memberFnA : [#int#]memberFnA(<#int a#>)
+      // CHECK-NOT: COMPLETION: memberFnA : [#int#]memberFnA(<#float a#>)
+    }();
+  }
+};
+
+
+struct S {
+  void foo1(int a);
+  void foo2(int a) const;
+  void foo2(this const S& self, float a);  
+  void foo3(this const S& self, int a);
+  void foo4(this S& self, int a);
+};
+
+void S::foo1(int a) {
+  this->;
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):9 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC7 %s
+// CHECK-CC7: COMPLETION: foo1 : [#void#]foo1(<#int a#>)
+// CHECK-CC7: COMPLETION: foo2 : [#void#]foo2(<#int a#>)[# const#]
+// CHECK-CC7: COMPLETION: foo2 : [#void#]foo2(<#float a#>)[# const#]
+// CHECK-CC7: COMPLETION: foo3 : [#void#]foo3(<#int a#>)[# const#]
+// CHECK-CC7: COMPLETION: foo4 : [#void#]foo4(<#int a#>)
+}
+
+void S::foo2(int a) const {
+  this->;
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):9 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC8 %s
+// CHECK-CC8: COMPLETION: foo2 : [#void#]foo2(<#int a#>)[# const#]
+// CHECK-CC8: COMPLETION: foo2 : [#void#]foo2(<#float a#>)[# const#]
+// CHECK-CC8: COMPLETION: foo3 : [#void#]foo3(<#int a#>)[# const#]
+}
+
+void S::foo3(this const S& self, int a) {
+  self.;
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):8 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC9 %s
+// CHECK-CC9: COMPLETION: foo2 : [#void#]foo2(<#int a#>)[# const#]
+// CHECK-CC9: COMPLETION: foo2 : [#void#]foo2(<#float a#>)[# const#]
+// CHECK-CC9: COMPLETION: foo3 : [#void#]foo3(<#int a#>)[# const#]
+}
+
+void S::foo4(this S& self, int a) {
+  self.;
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):8 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC10 %s
+// CHECK-CC10: COMPLETION: foo1 : [#void#]foo1(<#int a#>)
+// CHECK-CC10: COMPLETION: foo2 : [#void#]foo2(<#int a#>)[# const#]
+// CHECK-CC10: COMPLETION: foo2 : [#void#]foo2(<#float a#>)[# const#]
+// CHECK-CC10: COMPLETION: foo3 : [#void#]foo3(<#int a#>)[# const#]
+// CHECK-CC10: COMPLETION: foo4 : [#void#]foo4(<#int a#>)
+}
+
+void test1(S s) {
+  s.;
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):5 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC11 %s
+// CHECK-CC11: COMPLETION: foo1 : [#void#]foo1(<#int a#>)
+// CHECK-CC11: COMPLETION: foo2 : [#void#]foo2(<#int a#>)[# const#]
+// CHECK-CC11: COMPLETION: foo2 : [#void#]foo2(<#float a#>)[# const#]
+// CHECK-CC11: COMPLETION: foo3 : [#void#]foo3(<#int a#>)[# const#]
+// CHECK-CC11: COMPLETION: foo4 : [#void#]foo4(<#int a#>)
+}
+
+void test2(const S s) {
+  s.;
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):5 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC12 %s
+// CHECK-CC12: COMPLETION: foo2 : [#void#]foo2(<#int a#>)[# const#]
+// CHECK-CC12: COMPLETION: foo2 : [#void#]foo2(<#float a#>)[# const#]
+// CHECK-CC12: COMPLETION: foo3 : [#void#]foo3(<#int a#>)[# const#]
+}
+
+void test3(S s) {
+  s.foo2();
+// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-1):10 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC13 %s
+// CHECK-CC13: OVERLOAD: [#void#]foo2(<#int a#>)
+// CHECK-CC13: OVERLOAD: [#void#]foo2(float a)
+// TODO: foo2 should be OVERLOAD: [#void#]foo2(<#float a#>)
+}
diff --git a/clang/test/CodeCompletion/skip-explicit-object-parameter.cpp b/clang/test/CodeCompletion/skip-explicit-object-parameter.cpp
deleted file mode 100644
index 587d6cb044d19..0000000000000
--- a/clang/test/CodeCompletion/skip-explicit-object-parameter.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-struct A {
-  void foo(this auto&& self, int arg);
-  void bar(this A self, int arg);
-};
-
-int func1() {
-  A a {};
-  a.
-}
-// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-2):5 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC1 %s
-// CHECK-CC1: COMPLETION: A : A::
-// CHECK-NEXT-CC1: COMPLETION: bar : [#void#]bar(<#int arg#>)
-// CHECK-NEXT-CC1: COMPLETION: foo : [#void#]foo(<#int arg#>)
-// CHECK-NEXT-CC1: COMPLETION: operator= : [#A &#]operator=(<#const A &#>)
-// CHECK-NEXT-CC1: COMPLETION: operator= : [#A &#]operator=(<#A &&#>)
-// CHECK-NEXT-CC1: COMPLETION: ~A : [#void#]~A()
-
-struct B {
-  template <typename T>
-  void foo(this T&& self, int arg);
-};
-
-int func2() {
-  B b {};
-  b.foo();
-}
-// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-2):9 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC2 %s
-// CHECK-CC2: OVERLOAD: [#void#]foo(int arg)
-
-// TODO: llvm/llvm-project/146649
-// This is incorrect behavior. Correct Result should be a variant of, 
-// CC3: should be something like [#void#]foo(<#A self#>, <#int arg#>)
-// CC4: should be something like [#void#]bar(<#A self#>, <#int arg#>)
-int func3() {
-  (&A::foo)
-  (&A::bar)
-}
-// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-3):10 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC3 %s
-// CHECK-CC3: COMPLETION: foo : [#void#]foo<<#class self:auto#>>(<#int arg#>)
-// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-4):10 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC4 %s
-// CHECK-CC4: COMPLETION: bar : [#void#]bar(<#int arg#>)
-
-int func4() {
-  // TODO (&A::foo)(
-  (&A::bar)(
-}
-// RUN: %clang_cc1 -fsyntax-only -code-completion-at=%s:%(line-2):13 -std=c++23 %s | FileCheck -check-prefix=CHECK-CC5 %s
-// CHECK-CC5: OVERLOAD: [#void#](<#A#>, int)
diff --git a/clang/test/CodeGen/2007-01-20-VectorICE.c b/clang/test/CodeGen/2007-01-20-VectorICE.c
index 286b8a1b3de08..61013ac467692 100644
--- a/clang/test/CodeGen/2007-01-20-VectorICE.c
+++ b/clang/test/CodeGen/2007-01-20-VectorICE.c
@@ -1,11 +1,11 @@
-// RUN: %clang_cc1 %s -emit-llvm -o - 
+// RUN: %clang_cc1 %s -emit-llvm -flax-vector-conversions=none -o -
 
 typedef float __m128 __attribute__((__vector_size__(16)));
 typedef long long __v2di __attribute__((__vector_size__(16)));
 typedef int __v4si __attribute__((__vector_size__(16)));
 
-__v2di  bar(void);
+__v2di bar(void);
 void foo(int X, __v4si *P) {
-	*P = X == 2 ? bar() : bar();
+	*P = X == 2 ? (__v4si)bar() : (__v4si)bar();
 }
 
diff --git a/clang/test/CodeGen/2007-06-18-SextAttrAggregate.c b/clang/test/CodeGen/2007-06-18-SextAttrAggregate.c
index daed3baf4643e..ccfdc1a1c6cfa 100644
--- a/clang/test/CodeGen/2007-06-18-SextAttrAggregate.c
+++ b/clang/test/CodeGen/2007-06-18-SextAttrAggregate.c
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -no-enable-noundef-analysis %s -o - -emit-llvm | FileCheck %s
-// XFAIL: target={{(aarch64|arm64).*}}, target=x86_64-pc-windows-msvc, target=x86_64-{{(pc|w64)}}-windows-gnu
+// XFAIL: target={{(aarch64|arm64).*}}, target=x86_64-pc-windows-msvc, target=x86_64-{{(pc|w64)}}-windows-gnu, target=x86_64-pc-windows-cygnus
 
 // PR1513
 
diff --git a/clang/test/CodeGen/AArch64/ABI-align-packed.c b/clang/test/CodeGen/AArch64/ABI-align-packed.c
index 0349ebc8cc639..09f9180cdb0a7 100644
--- a/clang/test/CodeGen/AArch64/ABI-align-packed.c
+++ b/clang/test/CodeGen/AArch64/ABI-align-packed.c
@@ -72,9 +72,9 @@ __attribute__((noinline)) void named_arg_non_packed_struct(double d0, double d1,
 // CHECK-SAME: (double [[D0:%.*]], double [[D1:%.*]], double [[D2:%.*]], double [[D3:%.*]], double [[D4:%.*]], double [[D5:%.*]], double [[D6:%.*]], double [[D7:%.*]], double [[D8:%.*]], ...) local_unnamed_addr #[[ATTR1:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6:[0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VL]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_non_packed_struct(double d0, double d1, double d2, double d3,
                                  double d4, double d5, double d6, double d7,
@@ -89,7 +89,7 @@ void variadic_non_packed_struct(double d0, double d1, double d2, double d3,
 // CHECK-SAME: () local_unnamed_addr #[[ATTR4:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[S_NON_PACKED_STRUCT:%.*]] = alloca [[STRUCT_NON_PACKED_STRUCT:%.*]], align 16
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[S_NON_PACKED_STRUCT]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S_NON_PACKED_STRUCT]]) #[[ATTR6]]
 // CHECK-NEXT:    call void (i32, ...) @init(i32 noundef 1, ptr noundef nonnull [[S_NON_PACKED_STRUCT]]) #[[ATTR6]]
 // CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load <8 x i16>, ptr [[S_NON_PACKED_STRUCT]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD]], 0
@@ -97,7 +97,7 @@ void variadic_non_packed_struct(double d0, double d1, double d2, double d3,
 // CHECK-NEXT:    [[DOTFCA_0_LOAD3:%.*]] = load <8 x i16>, ptr [[S_NON_PACKED_STRUCT]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT4:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD3]], 0
 // CHECK-NEXT:    call void (double, double, double, double, double, double, double, double, double, ...) @variadic_non_packed_struct(double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, [1 x <8 x i16>] alignstack(16) [[DOTFCA_0_INSERT4]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[S_NON_PACKED_STRUCT]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S_NON_PACKED_STRUCT]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void test_non_packed_struct() {
     struct non_packed_struct s_non_packed_struct;
@@ -127,9 +127,9 @@ __attribute__((noinline)) void named_arg_packed_struct(double d0, double d1, dou
 // CHECK-SAME: (double [[D0:%.*]], double [[D1:%.*]], double [[D2:%.*]], double [[D3:%.*]], double [[D4:%.*]], double [[D5:%.*]], double [[D6:%.*]], double [[D7:%.*]], double [[D8:%.*]], ...) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_packed_struct(double d0, double d1, double d2, double d3,
                                  double d4, double d5, double d6, double d7,
@@ -144,7 +144,7 @@ void variadic_packed_struct(double d0, double d1, double d2, double d3,
 // CHECK-SAME: () local_unnamed_addr #[[ATTR4]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[S_PACKED_STRUCT:%.*]] = alloca [[STRUCT_PACKED_STRUCT:%.*]], align 16
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[S_PACKED_STRUCT]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S_PACKED_STRUCT]]) #[[ATTR6]]
 // CHECK-NEXT:    call void (i32, ...) @init(i32 noundef 1, ptr noundef nonnull [[S_PACKED_STRUCT]]) #[[ATTR6]]
 // CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load <8 x i16>, ptr [[S_PACKED_STRUCT]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD]], 0
@@ -152,7 +152,7 @@ void variadic_packed_struct(double d0, double d1, double d2, double d3,
 // CHECK-NEXT:    [[DOTFCA_0_LOAD3:%.*]] = load <8 x i16>, ptr [[S_PACKED_STRUCT]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT4:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD3]], 0
 // CHECK-NEXT:    call void (double, double, double, double, double, double, double, double, double, ...) @variadic_packed_struct(double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, [1 x <8 x i16>] alignstack(8) [[DOTFCA_0_INSERT4]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[S_PACKED_STRUCT]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S_PACKED_STRUCT]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void test_packed_struct() {
     struct packed_struct s_packed_struct;
@@ -182,9 +182,9 @@ __attribute__((noinline)) void named_arg_packed_member(double d0, double d1, dou
 // CHECK-SAME: (double [[D0:%.*]], double [[D1:%.*]], double [[D2:%.*]], double [[D3:%.*]], double [[D4:%.*]], double [[D5:%.*]], double [[D6:%.*]], double [[D7:%.*]], double [[D8:%.*]], ...) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_packed_member(double d0, double d1, double d2, double d3,
                                  double d4, double d5, double d6, double d7,
@@ -199,7 +199,7 @@ void variadic_packed_member(double d0, double d1, double d2, double d3,
 // CHECK-SAME: () local_unnamed_addr #[[ATTR4]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[S_PACKED_MEMBER:%.*]] = alloca [[STRUCT_PACKED_MEMBER:%.*]], align 16
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[S_PACKED_MEMBER]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S_PACKED_MEMBER]]) #[[ATTR6]]
 // CHECK-NEXT:    call void (i32, ...) @init(i32 noundef 1, ptr noundef nonnull [[S_PACKED_MEMBER]]) #[[ATTR6]]
 // CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load <8 x i16>, ptr [[S_PACKED_MEMBER]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD]], 0
@@ -207,7 +207,7 @@ void variadic_packed_member(double d0, double d1, double d2, double d3,
 // CHECK-NEXT:    [[DOTFCA_0_LOAD3:%.*]] = load <8 x i16>, ptr [[S_PACKED_MEMBER]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT4:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD3]], 0
 // CHECK-NEXT:    call void (double, double, double, double, double, double, double, double, double, ...) @variadic_packed_member(double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, [1 x <8 x i16>] alignstack(8) [[DOTFCA_0_INSERT4]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[S_PACKED_MEMBER]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S_PACKED_MEMBER]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void test_packed_member() {
     struct packed_member s_packed_member;
@@ -237,9 +237,9 @@ __attribute__((noinline)) void named_arg_aligned_struct_8(double d0, double d1,
 // CHECK-SAME: (double [[D0:%.*]], double [[D1:%.*]], double [[D2:%.*]], double [[D3:%.*]], double [[D4:%.*]], double [[D5:%.*]], double [[D6:%.*]], double [[D7:%.*]], double [[D8:%.*]], ...) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_aligned_struct_8(double d0, double d1, double d2, double d3,
                                  double d4, double d5, double d6, double d7,
@@ -254,7 +254,7 @@ void variadic_aligned_struct_8(double d0, double d1, double d2, double d3,
 // CHECK-SAME: () local_unnamed_addr #[[ATTR4]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[S_ALIGNED_STRUCT_8:%.*]] = alloca [[STRUCT_ALIGNED_STRUCT_8:%.*]], align 16
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[S_ALIGNED_STRUCT_8]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S_ALIGNED_STRUCT_8]]) #[[ATTR6]]
 // CHECK-NEXT:    call void (i32, ...) @init(i32 noundef 1, ptr noundef nonnull [[S_ALIGNED_STRUCT_8]]) #[[ATTR6]]
 // CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load <8 x i16>, ptr [[S_ALIGNED_STRUCT_8]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD]], 0
@@ -262,7 +262,7 @@ void variadic_aligned_struct_8(double d0, double d1, double d2, double d3,
 // CHECK-NEXT:    [[DOTFCA_0_LOAD3:%.*]] = load <8 x i16>, ptr [[S_ALIGNED_STRUCT_8]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT4:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD3]], 0
 // CHECK-NEXT:    call void (double, double, double, double, double, double, double, double, double, ...) @variadic_aligned_struct_8(double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, [1 x <8 x i16>] alignstack(16) [[DOTFCA_0_INSERT4]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[S_ALIGNED_STRUCT_8]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S_ALIGNED_STRUCT_8]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void test_aligned_struct_8() {
     struct aligned_struct_8 s_aligned_struct_8;
@@ -292,9 +292,9 @@ __attribute__((noinline)) void named_arg_aligned_member_8(double d0, double d1,
 // CHECK-SAME: (double [[D0:%.*]], double [[D1:%.*]], double [[D2:%.*]], double [[D3:%.*]], double [[D4:%.*]], double [[D5:%.*]], double [[D6:%.*]], double [[D7:%.*]], double [[D8:%.*]], ...) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_aligned_member_8(double d0, double d1, double d2, double d3,
                                  double d4, double d5, double d6, double d7,
@@ -309,7 +309,7 @@ void variadic_aligned_member_8(double d0, double d1, double d2, double d3,
 // CHECK-SAME: () local_unnamed_addr #[[ATTR4]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[S_ALIGNED_MEMBER_8:%.*]] = alloca [[STRUCT_ALIGNED_MEMBER_8:%.*]], align 16
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[S_ALIGNED_MEMBER_8]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S_ALIGNED_MEMBER_8]]) #[[ATTR6]]
 // CHECK-NEXT:    call void (i32, ...) @init(i32 noundef 1, ptr noundef nonnull [[S_ALIGNED_MEMBER_8]]) #[[ATTR6]]
 // CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load <8 x i16>, ptr [[S_ALIGNED_MEMBER_8]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD]], 0
@@ -317,7 +317,7 @@ void variadic_aligned_member_8(double d0, double d1, double d2, double d3,
 // CHECK-NEXT:    [[DOTFCA_0_LOAD3:%.*]] = load <8 x i16>, ptr [[S_ALIGNED_MEMBER_8]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT4:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD3]], 0
 // CHECK-NEXT:    call void (double, double, double, double, double, double, double, double, double, ...) @variadic_aligned_member_8(double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, [1 x <8 x i16>] alignstack(16) [[DOTFCA_0_INSERT4]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[S_ALIGNED_MEMBER_8]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S_ALIGNED_MEMBER_8]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void test_aligned_member_8() {
     struct aligned_member_8 s_aligned_member_8;
@@ -347,9 +347,9 @@ __attribute__((noinline)) void named_arg_pragma_packed_struct_8(double d0, doubl
 // CHECK-SAME: (double [[D0:%.*]], double [[D1:%.*]], double [[D2:%.*]], double [[D3:%.*]], double [[D4:%.*]], double [[D5:%.*]], double [[D6:%.*]], double [[D7:%.*]], double [[D8:%.*]], ...) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_pragma_packed_struct_8(double d0, double d1, double d2, double d3,
                                  double d4, double d5, double d6, double d7,
@@ -364,7 +364,7 @@ void variadic_pragma_packed_struct_8(double d0, double d1, double d2, double d3,
 // CHECK-SAME: () local_unnamed_addr #[[ATTR4]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[S_PRAGMA_PACKED_STRUCT_8:%.*]] = alloca [[STRUCT_PRAGMA_PACKED_STRUCT_8:%.*]], align 16
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[S_PRAGMA_PACKED_STRUCT_8]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S_PRAGMA_PACKED_STRUCT_8]]) #[[ATTR6]]
 // CHECK-NEXT:    call void (i32, ...) @init(i32 noundef 1, ptr noundef nonnull [[S_PRAGMA_PACKED_STRUCT_8]]) #[[ATTR6]]
 // CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load <8 x i16>, ptr [[S_PRAGMA_PACKED_STRUCT_8]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD]], 0
@@ -372,7 +372,7 @@ void variadic_pragma_packed_struct_8(double d0, double d1, double d2, double d3,
 // CHECK-NEXT:    [[DOTFCA_0_LOAD3:%.*]] = load <8 x i16>, ptr [[S_PRAGMA_PACKED_STRUCT_8]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT4:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD3]], 0
 // CHECK-NEXT:    call void (double, double, double, double, double, double, double, double, double, ...) @variadic_pragma_packed_struct_8(double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, [1 x <8 x i16>] alignstack(8) [[DOTFCA_0_INSERT4]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[S_PRAGMA_PACKED_STRUCT_8]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S_PRAGMA_PACKED_STRUCT_8]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void test_pragma_packed_struct_8() {
     struct pragma_packed_struct_8 s_pragma_packed_struct_8;
@@ -402,9 +402,9 @@ __attribute__((noinline)) void named_arg_pragma_packed_struct_4(double d0, doubl
 // CHECK-SAME: (double [[D0:%.*]], double [[D1:%.*]], double [[D2:%.*]], double [[D3:%.*]], double [[D4:%.*]], double [[D5:%.*]], double [[D6:%.*]], double [[D7:%.*]], double [[D8:%.*]], ...) local_unnamed_addr #[[ATTR1]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[VL:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VL]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VL]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VL]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void variadic_pragma_packed_struct_4(double d0, double d1, double d2, double d3,
                                  double d4, double d5, double d6, double d7,
@@ -419,7 +419,7 @@ void variadic_pragma_packed_struct_4(double d0, double d1, double d2, double d3,
 // CHECK-SAME: () local_unnamed_addr #[[ATTR4]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[S_PRAGMA_PACKED_STRUCT_4:%.*]] = alloca [[STRUCT_PRAGMA_PACKED_STRUCT_4:%.*]], align 16
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[S_PRAGMA_PACKED_STRUCT_4]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S_PRAGMA_PACKED_STRUCT_4]]) #[[ATTR6]]
 // CHECK-NEXT:    call void (i32, ...) @init(i32 noundef 1, ptr noundef nonnull [[S_PRAGMA_PACKED_STRUCT_4]]) #[[ATTR6]]
 // CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load <8 x i16>, ptr [[S_PRAGMA_PACKED_STRUCT_4]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD]], 0
@@ -427,7 +427,7 @@ void variadic_pragma_packed_struct_4(double d0, double d1, double d2, double d3,
 // CHECK-NEXT:    [[DOTFCA_0_LOAD3:%.*]] = load <8 x i16>, ptr [[S_PRAGMA_PACKED_STRUCT_4]], align 16
 // CHECK-NEXT:    [[DOTFCA_0_INSERT4:%.*]] = insertvalue [1 x <8 x i16>] poison, <8 x i16> [[DOTFCA_0_LOAD3]], 0
 // CHECK-NEXT:    call void (double, double, double, double, double, double, double, double, double, ...) @variadic_pragma_packed_struct_4(double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, double poison, [1 x <8 x i16>] alignstack(8) [[DOTFCA_0_INSERT4]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[S_PRAGMA_PACKED_STRUCT_4]]) #[[ATTR6]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S_PRAGMA_PACKED_STRUCT_4]]) #[[ATTR6]]
 // CHECK-NEXT:    ret void
 void test_pragma_packed_struct_4() {
     struct pragma_packed_struct_4 s_pragma_packed_struct_4;
diff --git a/clang/test/CodeGen/AArch64/byval-temp.c b/clang/test/CodeGen/AArch64/byval-temp.c
index 5033b6cf5ac03..353bfa753885b 100644
--- a/clang/test/CodeGen/AArch64/byval-temp.c
+++ b/clang/test/CodeGen/AArch64/byval-temp.c
@@ -47,13 +47,13 @@ void example(void) {
 // CHECK-O3-NEXT: %[[byvaltemp1:[0-9A-Za-z-]+]] = alloca %struct.large, align 8
 //
 // Mark the start of the lifetime for `l`
-// CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr %[[l]])
+// CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(ptr %[[l]])
 //
 // First, memset `l` to 0.
 // CHECK-O3-NEXT: call void @llvm.memset.p0.i64(ptr align 8 %[[l]], i8 0, i64 64, i1 false)
 //
 // Lifetime of the first temporary starts here and ends right after the call.
-// CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr %[[byvaltemp]])
+// CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(ptr %[[byvaltemp]])
 //
 // Then, memcpy `l` to the temporary stack space.
 // CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp]], ptr align 8 %[[l]], i64 64, i1 false)
@@ -61,16 +61,16 @@ void example(void) {
 // CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp]])
 //
 // The lifetime of the temporary used to pass a pointer to the struct ends here.
-// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp]])
+// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(ptr %[[byvaltemp]])
 //
 // Now, do the same for the second call, using the second temporary alloca.
-// CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(i64 64, ptr %[[byvaltemp1]])
+// CHECK-O3-NEXT: call void @llvm.lifetime.start.p0(ptr %[[byvaltemp1]])
 // CHECK-O3-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 %[[byvaltemp1]], ptr align 8 %[[l]], i64 64, i1 false)
 // CHECK-O3-NEXT: call void @pass_large(ptr dead_on_return noundef %[[byvaltemp1]])
-// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %[[byvaltemp1]])
+// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(ptr %[[byvaltemp1]])
 //
 // Mark the end of the lifetime of `l`.
-// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(i64 64, ptr %l)
+// CHECK-O3-NEXT: call void @llvm.lifetime.end.p0(ptr %l)
 // CHECK-O3-NEXT: ret void
 
 void example_BitInt(void) {
@@ -101,20 +101,20 @@ void example_BitInt(void) {
 // CHECK-O3-NEXT:    [[L:%.*]] = alloca i256, align 16
 // CHECK-O3-NEXT:    [[INDIRECT_ARG_TEMP:%.*]] = alloca i256, align 16
 // CHECK-O3-NEXT:    [[INDIRECT_ARG_TEMP1:%.*]] = alloca i256, align 16
-// CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[L]]) 
+// CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(ptr [[L]]) 
 // CHECK-O3-NEXT:    store i256 0, ptr [[L]], align 16, !tbaa [[TBAA6:![0-9]+]]
 // CHECK-O3-NEXT:    [[TMP0:%.*]] = load i256, ptr [[L]], align 16, !tbaa [[TBAA6]]
 // CHECK-O3-NEXT:    [[LOADEDV:%.*]] = trunc i256 [[TMP0]] to i129
-// CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]]) 
+// CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(ptr [[INDIRECT_ARG_TEMP]]) 
 // CHECK-O3-NEXT:    [[STOREDV:%.*]] = sext i129 [[LOADEDV]] to i256
 // CHECK-O3-NEXT:    store i256 [[STOREDV]], ptr [[INDIRECT_ARG_TEMP]], align 16, !tbaa [[TBAA6]]
 // CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP]])
-// CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP]]) 
+// CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(ptr [[INDIRECT_ARG_TEMP]]) 
 // CHECK-O3-NEXT:    [[TMP1:%.*]] = load i256, ptr [[L]], align 16, !tbaa [[TBAA6]]
 // CHECK-O3-NEXT:    [[LOADEDV1:%.*]] = trunc i256 [[TMP1]] to i129
-// CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]]) 
+// CHECK-O3-NEXT:    call void @llvm.lifetime.start.p0(ptr [[INDIRECT_ARG_TEMP1]]) 
 // CHECK-O3-NEXT:    [[STOREDV1:%.*]] = sext i129 [[LOADEDV1]] to i256
 // CHECK-O3-NEXT:    store i256 [[STOREDV1]], ptr [[INDIRECT_ARG_TEMP1]], align 16, !tbaa [[TBAA6]]
 // CHECK-O3-NEXT:    call void @pass_large_BitInt(ptr dead_on_return noundef [[INDIRECT_ARG_TEMP1]])
-// CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[INDIRECT_ARG_TEMP1]]) 
-// CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[L]]) 
+// CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(ptr [[INDIRECT_ARG_TEMP1]]) 
+// CHECK-O3-NEXT:    call void @llvm.lifetime.end.p0(ptr [[L]]) 
diff --git a/clang/test/CodeGen/AArch64/pure-scalable-args.c b/clang/test/CodeGen/AArch64/pure-scalable-args.c
index 48988f7a1722b..d34c7f95a70a3 100644
--- a/clang/test/CodeGen/AArch64/pure-scalable-args.c
+++ b/clang/test/CodeGen/AArch64/pure-scalable-args.c
@@ -329,7 +329,7 @@ void test_pass_variadic(PST *p, PST *q) {
 // CHECK-AAPCS: call void (<vscale x 16 x i1>, <vscale x 2 x double>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 16 x i8>, <vscale x 16 x i1>, ...) @pass_variadic_callee(<vscale x 16 x i1> %1, <vscale x 2 x double> %cast.scalable1, <vscale x 4 x float> %cast.scalable2, <vscale x 4 x float> %cast.scalable3, <vscale x 16 x i8> %cast.scalable4, <vscale x 16 x i1> %12, ptr dead_on_return noundef nonnull %byval-temp)
 
 // CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp, ptr noundef nonnull align 16 dereferenceable(96) %p, i64 96, i1 false)
-// CHECK-DARWIN: call void @llvm.lifetime.start.p0(i64 96, ptr nonnull %byval-temp1)
+// CHECK-DARWIN: call void @llvm.lifetime.start.p0(ptr nonnull %byval-temp1)
 // CHECK-DARWIN: call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(96) %byval-temp1, ptr noundef nonnull align 16 dereferenceable(96) %q, i64 96, i1 false)
 // CHECK-DARWIN: call void (ptr, ...) @pass_variadic_callee(ptr dead_on_return noundef nonnull %byval-temp, ptr dead_on_return noundef nonnull %byval-temp1)
 
@@ -392,7 +392,7 @@ void test_va_arg(int n, ...) {
 // CHECK-AAPCS: define dso_local void @test_va_arg(i32 noundef %n, ...)
 // CHECK-AAPCS-NEXT: entry:
 // CHECK-AAPCS-NEXT:   %ap = alloca %struct.__va_list, align 8
-// CHECK-AAPCS-NEXT:   call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %ap)
+// CHECK-AAPCS-NEXT:   call void @llvm.lifetime.start.p0(ptr nonnull %ap)
 // CHECK-AAPCS-NEXT:   call void @llvm.va_start.p0(ptr nonnull %ap)
 // CHECK-AAPCS-NEXT:   %gr_offs_p = getelementptr inbounds nuw i8, ptr %ap, i64 24
 // CHECK-AAPCS-NEXT:   %gr_offs = load i32, ptr %gr_offs_p, align 8
@@ -435,14 +435,14 @@ void test_va_arg(int n, ...) {
 // CHECK-AAPCS-NEXT:   %3 = bitcast <vscale x 2 x i8> %cast.scalable to <vscale x 16 x i1>
 // CHECK-AAPCS-NEXT:   %cast.scalable2 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %v.sroa.43.0.copyload, i64 0)
 // CHECK-AAPCS-NEXT:   call void @use1(<vscale x 16 x i1> noundef %3, <vscale x 4 x float> noundef %cast.scalable2)
-// CHECK-AAPCS-NEXT:   call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %ap)
+// CHECK-AAPCS-NEXT:   call void @llvm.lifetime.end.p0(ptr nonnull %ap)
 // CHECK-AAPCS-NEXT:   ret void
 // CHECK-AAPCS-NEXT: }
 
 // CHECK-DARWIN: define void @test_va_arg(i32 noundef %n, ...)
 // CHECK-DARWIN-NEXT: entry:
 // CHECK-DARWIN-NEXT:   %ap = alloca ptr, align 8
-// CHECK-DARWIN-NEXT:   call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %ap)
+// CHECK-DARWIN-NEXT:   call void @llvm.lifetime.start.p0(ptr nonnull %ap)
 // CHECK-DARWIN-NEXT:   call void @llvm.va_start.p0(ptr nonnull %ap)
 // CHECK-DARWIN-NEXT:   %argp.cur = load ptr, ptr %ap, align 8
 // CHECK-DARWIN-NEXT:   %argp.next = getelementptr inbounds nuw i8, ptr %argp.cur, i64 8
@@ -456,7 +456,7 @@ void test_va_arg(int n, ...) {
 // CHECK-DARWIN-NEXT:   %1 = bitcast <vscale x 2 x i8> %cast.scalable to <vscale x 16 x i1>
 // CHECK-DARWIN-NEXT:   %cast.scalable2 = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %v.sroa.43.0.copyload, i64 0)
 // CHECK-DARWIN-NEXT:   call void @use1(<vscale x 16 x i1> noundef %1, <vscale x 4 x float> noundef %cast.scalable2)
-// CHECK-DARWIN-NEXT:   call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %ap)
+// CHECK-DARWIN-NEXT:   call void @llvm.lifetime.end.p0(ptr nonnull %ap)
 // CHECK-DARWIN-NEXT:   ret void
 // CHECK-DARWIN-NEXT: }
 
diff --git a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_ld1_vnum.c b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_ld1_vnum.c
index fb86690f07f1d..e4c93ade35d53 100644
--- a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_ld1_vnum.c
+++ b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_ld1_vnum.c
@@ -9,29 +9,31 @@
 // CHECK-C-LABEL: define dso_local void @test_svld1_hor_vnum_za8(
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-C-NEXT:  entry:
-// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP1:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[TMP0]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z23test_svld1_hor_vnum_za8ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[TMP0]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -43,30 +45,32 @@ void test_svld1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 1, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_hor_vnum_za16ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 1, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -78,30 +82,32 @@ void test_svld1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 3, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_hor_vnum_za32ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 3, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -113,30 +119,32 @@ void test_svld1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 7, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_hor_vnum_za64ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 7, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -148,26 +156,28 @@ void test_svld1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 15, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z25test_svld1_hor_vnum_za128ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 15, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -178,29 +188,31 @@ void test_svld1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr
 // CHECK-C-LABEL: define dso_local void @test_svld1_ver_hor_za8(
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
-// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP1:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[TMP0]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z22test_svld1_ver_hor_za8ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[TMP0]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -212,30 +224,32 @@ void test_svld1_ver_hor_za8(uint32_t slice_base, svbool_t pg, const void *ptr, i
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 1, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_ver_vnum_za16ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 1, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -247,30 +261,32 @@ void test_svld1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 3, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_ver_vnum_za32ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 3, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -282,30 +298,32 @@ void test_svld1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 7, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svld1_ver_vnum_za64ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 7, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
@@ -317,26 +335,28 @@ void test_svld1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, const void *ptr,
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 15, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z25test_svld1_ver_vnum_za128ju10__SVBool_tPKvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.ld1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 15, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svld1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, const void *ptr, int64_t vnum) __arm_streaming __arm_out("za") {
diff --git a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_st1_vnum.c b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_st1_vnum.c
index dafc3d61a05f1..22a0b9eabaea3 100644
--- a/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_st1_vnum.c
+++ b/clang/test/CodeGen/AArch64/sme-intrinsics/acle_sme_st1_vnum.c
@@ -9,29 +9,31 @@
 // CHECK-C-LABEL: define dso_local void @test_svst1_hor_vnum_za8(
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-C-NEXT:  entry:
-// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP1:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[TMP0]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z23test_svst1_hor_vnum_za8ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[TMP0]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.horiz(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -43,30 +45,32 @@ void test_svst1_hor_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 1, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_hor_vnum_za16ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.horiz(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 1, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -78,30 +82,32 @@ void test_svst1_hor_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 3, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_hor_vnum_za32ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.horiz(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 3, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -113,30 +119,32 @@ void test_svst1_hor_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 7, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_hor_vnum_za64ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.horiz(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 7, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -148,26 +156,28 @@ void test_svst1_hor_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 15, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z25test_svst1_hor_vnum_za128ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.horiz(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 15, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -178,29 +188,31 @@ void test_svst1_hor_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int6
 // CHECK-C-LABEL: define dso_local void @test_svst1_ver_vnum_za8(
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
-// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP1:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[TMP0]]
+// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 15
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z23test_svst1_ver_vnum_za8ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
-// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP0]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP3]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP2]]
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[ADD]], 15
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP1]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[TMP0]]
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 15
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1b.vert(<vscale x 16 x i1> [[PG]], ptr [[TMP2]], i32 0, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -212,30 +224,32 @@ void test_svst1_ver_vnum_za8(uint32_t slice_base, svbool_t pg, void *ptr, int64_
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 7
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 1, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_ver_vnum_za16ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 7
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP2]], i32 1, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 7
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1h.vert(<vscale x 8 x i1> [[TMP0]], ptr [[TMP3]], i32 1, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -247,30 +261,32 @@ void test_svst1_ver_vnum_za16(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 3
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 3, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_ver_vnum_za32ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 3
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP2]], i32 3, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 3
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1w.vert(<vscale x 4 x i1> [[TMP0]], ptr [[TMP3]], i32 3, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -282,30 +298,32 @@ void test_svst1_ver_vnum_za32(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 1
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 7, i32 [[TMP6]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z24test_svst1_ver_vnum_za64ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[ADD]], 1
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP2]], i32 7, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    [[ADD:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    [[TMP6:%.*]] = add i32 [[ADD]], 1
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1d.vert(<vscale x 2 x i1> [[TMP0]], ptr [[TMP3]], i32 7, i32 [[TMP6]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
@@ -317,26 +335,28 @@ void test_svst1_ver_vnum_za64(uint32_t slice_base, svbool_t pg, void *ptr, int64
 // CHECK-C-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-C-NEXT:  entry:
 // CHECK-C-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
-// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-C-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-C-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-C-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
+// CHECK-C-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-C-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-C-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-C-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-C-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-C-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-C-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 15, i32 [[TMP5]])
 // CHECK-C-NEXT:    ret void
 //
 // CHECK-CXX-LABEL: define dso_local void @_Z25test_svst1_ver_vnum_za128ju10__SVBool_tPvl(
 // CHECK-CXX-SAME: i32 noundef [[SLICE_BASE:%.*]], <vscale x 16 x i1> [[PG:%.*]], ptr noundef [[PTR:%.*]], i64 noundef [[VNUM:%.*]]) local_unnamed_addr #[[ATTR0]] {
 // CHECK-CXX-NEXT:  entry:
 // CHECK-CXX-NEXT:    [[TMP0:%.*]] = tail call <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1> [[PG]])
-// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.aarch64.sme.cntsb()
-// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP1]], [[VNUM]]
-// CHECK-CXX-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
-// CHECK-CXX-NEXT:    [[TMP3:%.*]] = trunc i64 [[VNUM]] to i32
-// CHECK-CXX-NEXT:    [[TMP4:%.*]] = add i32 [[SLICE_BASE]], [[TMP3]]
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 0, i32 [[TMP4]])
-// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP2]], i32 15, i32 [[TMP4]])
+// CHECK-CXX-NEXT:    [[TMP1:%.*]] = tail call i64 @llvm.vscale.i64()
+// CHECK-CXX-NEXT:    [[TMP2:%.*]] = shl i64 [[VNUM]], 4
+// CHECK-CXX-NEXT:    [[MULVL:%.*]] = mul i64 [[TMP2]], [[TMP1]]
+// CHECK-CXX-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[MULVL]]
+// CHECK-CXX-NEXT:    [[TMP4:%.*]] = trunc i64 [[VNUM]] to i32
+// CHECK-CXX-NEXT:    [[TMP5:%.*]] = add i32 [[SLICE_BASE]], [[TMP4]]
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 0, i32 [[TMP5]])
+// CHECK-CXX-NEXT:    tail call void @llvm.aarch64.sme.st1q.vert(<vscale x 1 x i1> [[TMP0]], ptr [[TMP3]], i32 15, i32 [[TMP5]])
 // CHECK-CXX-NEXT:    ret void
 //
 void test_svst1_ver_vnum_za128(uint32_t slice_base, svbool_t pg, void *ptr, int64_t vnum) __arm_streaming __arm_in("za") {
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
index d244a8ba88572..4a1185d02981b 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.c
@@ -86,10 +86,10 @@ typedef svint8_t vec2 __attribute__((arm_sve_vector_bits(N)));
 // CHECK-NEXT: entry:
 // CHECK-NEXT:   [[INDIRECT_ARG_TEMP:%.*]] = alloca <[[#div(VBITS,8)]] x i8>, align 16
 // CHECK-NEXT:   [[X:%.*]] = tail call <[[#div(VBITS,8)]] x i8> @llvm.vector.extract.v[[#div(VBITS,8)]]i8.nxv16i8(<vscale x 16 x i8> [[X_COERCE:%.*]], i64 0)
-// CHECK-NEXT:   call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
+// CHECK-NEXT:   call void @llvm.lifetime.start.p0(ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:   store <[[#div(VBITS,8)]] x i8> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6]]
 // CHECK-NEXT:   call void @f3(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
-// CHECK-NEXT:   call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
+// CHECK-NEXT:   call void @llvm.lifetime.end.p0(ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:   ret void
 
 // CHECK128-LABEL: declare void @f3(<16 x i8> noundef)
diff --git a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
index d42ecb663050f..6211b609d6d28 100644
--- a/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
+++ b/clang/test/CodeGen/AArch64/sve-acle-__ARM_FEATURE_SVE_VECTOR_OPERATORS.cpp
@@ -73,10 +73,10 @@ typedef svint16_t vec2 __attribute__((arm_sve_vector_bits(N)));
 // CHECK128-NEXT:   ret void
 // CHECKWIDE-NEXT:   [[INDIRECT_ARG_TEMP:%.*]] = alloca <[[#div(VBITS, 16)]] x i16>, align 16
 // CHECKWIDE-NEXT:   [[X:%.*]] = tail call <[[#div(VBITS, 16)]] x i16> @llvm.vector.extract.v[[#div(VBITS, 16)]]i16.nxv8i16(<vscale x 8 x i16> [[X_COERCE:%.*]], i64 0)
-// CHECKWIDE-NEXT:   call void @llvm.lifetime.start.p0(i64 [[SIZE:[0-9]+]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
+// CHECKWIDE-NEXT:   call void @llvm.lifetime.start.p0(ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECKWIDE-NEXT:   store <[[#div(VBITS, 16)]] x i16> [[X]], ptr [[INDIRECT_ARG_TEMP]], align 16, [[TBAA6:!tbaa !.*]]
 // CHECKWIDE-NEXT:   call void @_Z1fDv[[#div(VBITS, 16)]]_s(ptr dead_on_return noundef nonnull [[INDIRECT_ARG_TEMP]]) [[ATTR5:#.*]]
-// CHECKWIDE-NEXT:   call void @llvm.lifetime.end.p0(i64 [[SIZE]], ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
+// CHECKWIDE-NEXT:   call void @llvm.lifetime.end.p0(ptr nonnull [[INDIRECT_ARG_TEMP]]) #[[ATTR6:[0-9]+]]
 // CHECKWIDE-NEXT:   ret void
 void g(vec2 x) { f(x); } // OK
 #endif
diff --git a/clang/test/CodeGen/LoongArch/targetattr-lasx.c b/clang/test/CodeGen/LoongArch/targetattr-lasx.c
new file mode 100644
index 0000000000000..56fd6573ed34c
--- /dev/null
+++ b/clang/test/CodeGen/LoongArch/targetattr-lasx.c
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -triple loongarch64 -target-feature -lsx -emit-llvm %s -o - | FileCheck %s
+
+__attribute__((target("lasx")))
+// CHECK: #[[ATTR0:[0-9]+]] {
+void testlasx() {}
+
+// CHECK: attributes #[[ATTR0]] = { {{.*}}"target-features"="+64bit,+lasx,+lsx"{{.*}} }
diff --git a/clang/test/CodeGen/PowerPC/builtins-bcd-format-conversion.c b/clang/test/CodeGen/PowerPC/builtins-bcd-format-conversion.c
new file mode 100644
index 0000000000000..0aeb720e545ed
--- /dev/null
+++ b/clang/test/CodeGen/PowerPC/builtins-bcd-format-conversion.c
@@ -0,0 +1,29 @@
+// NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+// REQUIRES: powerpc-registered-target
+// RUN: %clang_cc1 -triple powerpc64le-unknown-unknown -O2 -target-cpu pwr9 \
+// RUN:   -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple powerpc64-unknown-unknown -O2 -target-cpu pwr9 \
+// RUN:   -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple powerpc-unknown-unknown -O2 -target-cpu pwr9 \
+// RUN:   -emit-llvm %s -o - | FileCheck %s
+
+// CHECK-LABEL: test_bcdcopysign
+// CHECK:         [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.bcdcopysign(<16 x i8> %a, <16 x i8> %b)
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+vector unsigned char test_bcdcopysign(vector unsigned char a, vector unsigned char b) {
+    return __builtin_ppc_bcdcopysign(a, b);
+}
+
+// CHECK-LABEL: test_bcdsetsign_imm0
+// CHECK:         [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.bcdsetsign(<16 x i8> %a, i32 0)
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+vector unsigned char test_bcdsetsign_imm0(vector unsigned char a) {
+    return __builtin_ppc_bcdsetsign(a, '\0');
+}
+
+// CHECK-LABEL: test_bcdsetsign_imm1
+// CHECK:         [[TMP0:%.*]] = tail call <16 x i8> @llvm.ppc.bcdsetsign(<16 x i8> %a, i32 1)
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+vector unsigned char test_bcdsetsign_imm1(vector unsigned char a) {
+    return __builtin_ppc_bcdsetsign(a, '\1');
+}
diff --git a/clang/test/CodeGen/RISCV/riscv-inline-asm-fixed-length-vector.c b/clang/test/CodeGen/RISCV/riscv-inline-asm-fixed-length-vector.c
new file mode 100644
index 0000000000000..699c588950c62
--- /dev/null
+++ b/clang/test/CodeGen/RISCV/riscv-inline-asm-fixed-length-vector.c
@@ -0,0 +1,66 @@
+// REQUIRES: riscv-registered-target
+
+// RUN: %clang_cc1 -triple riscv32 -target-feature +v \
+// RUN:     -mvscale-min=2 -mvscale-max=2 -O2 -emit-llvm %s -o - \
+// RUN:     | FileCheck %s
+// RUN: %clang_cc1 -triple riscv64 -target-feature +v \
+// RUN:     -mvscale-min=2 -mvscale-max=2 -O2 -emit-llvm %s -o - \
+// RUN:     | FileCheck %s
+
+// Test RISC-V V-extension fixed-length vector inline assembly constraints.
+#include <riscv_vector.h>
+#include <stdbool.h>
+
+typedef vbool1_t fixed_bool1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint32m1_t fixed_i32m1_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen)));
+typedef vint8mf2_t fixed_i8mf2_t __attribute__((riscv_rvv_vector_bits(__riscv_v_fixed_vlen / 2)));
+
+typedef bool bx2 __attribute__((ext_vector_type(16)));
+typedef int i32x2 __attribute__((ext_vector_type(2)));
+typedef char  i8x4 __attribute__((ext_vector_type(4)));
+
+fixed_i32m1_t test_vr(fixed_i32m1_t a) {
+// CHECK-LABEL: define{{.*}} @test_vr
+// CHECK: %0 = tail call <4 x i32> asm sideeffect "vadd.vv $0, $1, $2", "=^vr,^vr,^vr"(<4 x i32> %a, <4 x i32> %a)
+  fixed_i32m1_t ret;
+  asm volatile ("vadd.vv %0, %1, %2" : "=vr"(ret) : "vr"(a), "vr"(a));
+  return ret;
+}
+
+i32x2 test_vr2(i32x2 a) {
+// CHECK-LABEL: define{{.*}} @test_vr2
+// CHECK: %1 = tail call <2 x i32> asm sideeffect "vadd.vv $0, $1, $2", "=^vr,^vr,^vr"(<2 x i32> %0, <2 x i32> %0)
+  i32x2 ret;
+  asm volatile ("vadd.vv %0, %1, %2" : "=vr"(ret) : "vr"(a), "vr"(a));
+  return ret;
+}
+
+fixed_i8mf2_t test_vd(fixed_i8mf2_t a) {
+// CHECK-LABEL: define{{.*}} @test_vd
+// CHECK: %0 = tail call <8 x i8> asm sideeffect "vadd.vv $0, $1, $2", "=^vd,^vr,^vr"(<8 x i8> %a, <8 x i8> %a)
+  fixed_i8mf2_t ret;
+  asm volatile ("vadd.vv %0, %1, %2" : "=vd"(ret) : "vr"(a), "vr"(a));
+  return ret;
+}
+
+i8x4 test_vd2(i8x4 a) {
+// CHECK-LABEL: define{{.*}} @test_vd2
+// CHECK: %1 = tail call <4 x i8> asm sideeffect "vadd.vv $0, $1, $2", "=^vd,^vr,^vr"(<4 x i8> %0, <4 x i8> %0)
+  i8x4 ret;
+  asm volatile ("vadd.vv %0, %1, %2" : "=vd"(ret) : "vr"(a), "vr"(a));
+  return ret;
+}
+
+fixed_bool1_t test_vm(fixed_bool1_t a) {
+// CHECK-LABEL: define{{.*}} @test_vm
+// CHECK: %1 = tail call <16 x i8> asm sideeffect "vmand.mm $0, $1, $2", "=^vm,^vm,^vm"(<16 x i8> %a, <16 x i8> %a)
+  fixed_bool1_t ret;
+  asm volatile ("vmand.mm %0, %1, %2" : "=vm"(ret) : "vm"(a), "vm"(a));
+  return ret;
+}
+
+void test_vm2(bx2 a) {
+// CHECK-LABEL: define{{.*}} @test_vm2
+// CHECK: tail call void asm sideeffect "dummy $0", "^vm"(<16 x i1> %a1)
+  asm volatile ("dummy %0" :: "vm"(a));
+}
diff --git a/clang/test/CodeGen/RISCV/riscv-v-lifetime.cpp b/clang/test/CodeGen/RISCV/riscv-v-lifetime.cpp
index 4c91fefad749d..c5b863bf2b60d 100644
--- a/clang/test/CodeGen/RISCV/riscv-v-lifetime.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-v-lifetime.cpp
@@ -10,10 +10,10 @@ vint32m1_t Baz();
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[REF_TMP:%.*]] = alloca <vscale x 2 x i32>, align 4
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[A]]) #[[ATTR3:[0-9]+]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[REF_TMP]]) #[[ATTR3]]
-// CHECK:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[REF_TMP]]) #[[ATTR3]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[A]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]]) #[[ATTR3:[0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR3]]
+// CHECK:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]]) #[[ATTR3]]
 //
 vint32m1_t Test() {
   const vint32m1_t &a = Baz();
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
index bc89cb532bdcc..6866fe053b4b0 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.c
@@ -143,34 +143,34 @@ void __attribute__((riscv_vls_cc)) test_too_large(int32x64_t arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_too_large_256(<vscale x 16 x i32> noundef %arg.coerce)
 void __attribute__((riscv_vls_cc(256))) test_too_large_256(int32x64_t arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4(<vscale x 2 x i32> %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4(<vscale x 2 x i32> %arg.target_coerce)
 void __attribute__((riscv_vls_cc)) test_st_i32x4(struct st_i32x4 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4_256(<vscale x 1 x i32> %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4_256(<vscale x 1 x i32> %arg.target_coerce)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4_256(struct st_i32x4 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4_arr1(<vscale x 2 x i32> %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4_arr1(<vscale x 2 x i32> %arg.target_coerce)
 void __attribute__((riscv_vls_cc)) test_st_i32x4_arr1(struct st_i32x4_arr1 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4_arr1_256(<vscale x 1 x i32> %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4_arr1_256(<vscale x 1 x i32> %arg.target_coerce)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4_arr1_256(struct st_i32x4_arr1 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4_arr4(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4_arr4(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %arg.target_coerce)
 void __attribute__((riscv_vls_cc)) test_st_i32x4_arr4(struct st_i32x4_arr4 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4_arr4_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4_arr4_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %arg.target_coerce)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4_arr4_256(struct st_i32x4_arr4 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4_arr8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4_arr8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %arg.target_coerce)
 void __attribute__((riscv_vls_cc)) test_st_i32x4_arr8(struct st_i32x4_arr8 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4_arr8_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4_arr8_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg.target_coerce)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4_arr8_256(struct st_i32x4_arr8 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg.target_coerce)
 void __attribute__((riscv_vls_cc)) test_st_i32x4x2(struct st_i32x4x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x2_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x2_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %arg.target_coerce)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x2_256(struct st_i32x4x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x8x2(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x8x2(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %arg.target_coerce)
 void __attribute__((riscv_vls_cc)) test_st_i32x8x2(struct st_i32x8x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x8x2_256(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x8x2_256(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg.target_coerce)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x64x2(ptr dead_on_return noundef %arg)
@@ -178,17 +178,78 @@ void __attribute__((riscv_vls_cc)) test_st_i32x64x2(struct st_i32x64x2 arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x64x2_256(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg.target_coerce)
 void __attribute__((riscv_vls_cc)) test_st_i32x4x3(struct st_i32x4x3 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x3_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x3_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %arg.target_coerce)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x3_256(struct st_i32x4x3 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %arg.target_coerce)
 void __attribute__((riscv_vls_cc)) test_st_i32x4x8(struct st_i32x4x8 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x8_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x8_256(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg.target_coerce)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @test_st_i32x4x9(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc)) test_st_i32x4x9(struct st_i32x4x9 arg) {}
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @test_st_i32x4x9_256(ptr dead_on_return noundef %arg)
 void __attribute__((riscv_vls_cc(256))) test_st_i32x4x9_256(struct st_i32x4x9 arg) {}
+
+// CHECK-LLVM-LABEL: define dso_local riscv_vls_cc(128) target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @test_function_prolog_epilog(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %arg.target_coerce) #0 {
+// CHECK-LLVM-NEXT: entry:
+// CHECK-LLVM-NEXT:   %retval = alloca %struct.st_i32x4_arr4, align 16
+// CHECK-LLVM-NEXT:   %arg = alloca %struct.st_i32x4_arr4, align 16
+// CHECK-LLVM-NEXT:   %0 = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %arg.target_coerce, i32 0)
+// CHECK-LLVM-NEXT:   %1 = call <4 x i32> @llvm.vector.extract.v4i32.nxv2i32(<vscale x 2 x i32> %0, i64 0)
+// CHECK-LLVM-NEXT:   %2 = getelementptr inbounds [4 x <4 x i32>], ptr %arg, i64 0, i64 0
+// CHECK-LLVM-NEXT:   store <4 x i32> %1, ptr %2, align 16
+// CHECK-LLVM-NEXT:   %3 = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %arg.target_coerce, i32 1)
+// CHECK-LLVM-NEXT:   %4 = call <4 x i32> @llvm.vector.extract.v4i32.nxv2i32(<vscale x 2 x i32> %3, i64 0)
+// CHECK-LLVM-NEXT:   %5 = getelementptr inbounds [4 x <4 x i32>], ptr %arg, i64 0, i64 1
+// CHECK-LLVM-NEXT:   store <4 x i32> %4, ptr %5, align 16
+// CHECK-LLVM-NEXT:   %6 = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %arg.target_coerce, i32 2)
+// CHECK-LLVM-NEXT:   %7 = call <4 x i32> @llvm.vector.extract.v4i32.nxv2i32(<vscale x 2 x i32> %6, i64 0)
+// CHECK-LLVM-NEXT:   %8 = getelementptr inbounds [4 x <4 x i32>], ptr %arg, i64 0, i64 2
+// CHECK-LLVM-NEXT:   store <4 x i32> %7, ptr %8, align 16
+// CHECK-LLVM-NEXT:   %9 = call <vscale x 2 x i32> @llvm.riscv.tuple.extract.nxv2i32.triscv.vector.tuple_nxv8i8_4t(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %arg.target_coerce, i32 3)
+// CHECK-LLVM-NEXT:   %10 = call <4 x i32> @llvm.vector.extract.v4i32.nxv2i32(<vscale x 2 x i32> %9, i64 0)
+// CHECK-LLVM-NEXT:   %11 = getelementptr inbounds [4 x <4 x i32>], ptr %arg, i64 0, i64 3
+// CHECK-LLVM-NEXT:   store <4 x i32> %10, ptr %11, align 16
+// CHECK-LLVM-NEXT:   call void @llvm.memcpy.p0.p0.i64(ptr align 16 %retval, ptr align 16 %arg, i64 64, i1 false)
+// CHECK-LLVM-NEXT:   %12 = load [4 x <4 x i32>], ptr %retval, align 16
+// CHECK-LLVM-NEXT:   %13 = extractvalue [4 x <4 x i32>] %12, 0
+// CHECK-LLVM-NEXT:   %cast.scalable = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v4i32(<vscale x 2 x i32> poison, <4 x i32> %13, i64 0)
+// CHECK-LLVM-NEXT:   %14 = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv2i32(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) poison, <vscale x 2 x i32> %cast.scalable, i32 0)
+// CHECK-LLVM-NEXT:   %15 = extractvalue [4 x <4 x i32>] %12, 1
+// CHECK-LLVM-NEXT:   %cast.scalable1 = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v4i32(<vscale x 2 x i32> poison, <4 x i32> %15, i64 0)
+// CHECK-LLVM-NEXT:   %16 = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv2i32(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %14, <vscale x 2 x i32> %cast.scalable1, i32 1)
+// CHECK-LLVM-NEXT:   %17 = extractvalue [4 x <4 x i32>] %12, 2
+// CHECK-LLVM-NEXT:   %cast.scalable2 = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v4i32(<vscale x 2 x i32> poison, <4 x i32> %17, i64 0)
+// CHECK-LLVM-NEXT:   %18 = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv2i32(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %16, <vscale x 2 x i32> %cast.scalable2, i32 2)
+// CHECK-LLVM-NEXT:   %19 = extractvalue [4 x <4 x i32>] %12, 3
+// CHECK-LLVM-NEXT:   %cast.scalable3 = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v4i32(<vscale x 2 x i32> poison, <4 x i32> %19, i64 0)
+// CHECK-LLVM-NEXT:   %20 = call target("riscv.vector.tuple", <vscale x 8 x i8>, 4) @llvm.riscv.tuple.insert.triscv.vector.tuple_nxv8i8_4t.nxv2i32(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %18, <vscale x 2 x i32> %cast.scalable3, i32 3)
+// CHECK-LLVM-NEXT:   ret target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %20
+// CHECK-LLVM-NEXT: }
+struct st_i32x4_arr4 __attribute__((riscv_vls_cc)) test_function_prolog_epilog(struct st_i32x4_arr4 arg) {
+  return arg;
+}
+
+struct st_i32x4 __attribute__((riscv_vls_cc)) dummy(struct st_i32x4);
+// CHECK-LLVM-LABEL: define dso_local riscv_vls_cc(128) <vscale x 2 x i32> @test_call(<vscale x 2 x i32> %arg.target_coerce) #0 {
+// CHECK-LLVM-NEXT: entry:
+// CHECK-LLVM-NEXT:   %retval = alloca %struct.st_i32x4, align 16
+// CHECK-LLVM-NEXT:   %arg = alloca %struct.st_i32x4, align 16
+// CHECK-LLVM-NEXT:   %0 = call <4 x i32> @llvm.vector.extract.v4i32.nxv2i32(<vscale x 2 x i32> %arg.target_coerce, i64 0)
+// CHECK-LLVM-NEXT:   store <4 x i32> %0, ptr %arg, align 16
+// CHECK-LLVM-NEXT:   %1 = load <4 x i32>, ptr %arg, align 16
+// CHECK-LLVM-NEXT:   %cast.scalable = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v4i32(<vscale x 2 x i32> poison, <4 x i32> %1, i64 0)
+// CHECK-LLVM-NEXT:   %call = call riscv_vls_cc(128) <vscale x 2 x i32> @dummy(<vscale x 2 x i32> %cast.scalable)
+// CHECK-LLVM-NEXT:   %2 = call <4 x i32> @llvm.vector.extract.v4i32.nxv2i32(<vscale x 2 x i32> %call, i64 0)
+// CHECK-LLVM-NEXT:   store <4 x i32> %2, ptr %retval, align 16
+// CHECK-LLVM-NEXT:   %3 = load <4 x i32>, ptr %retval, align 16
+// CHECK-LLVM-NEXT:   %cast.scalable1 = call <vscale x 2 x i32> @llvm.vector.insert.nxv2i32.v4i32(<vscale x 2 x i32> poison, <4 x i32> %3, i64 0)
+// CHECK-LLVM-NEXT:   ret <vscale x 2 x i32> %cast.scalable1
+// CHECK-LLVM-NEXT: }
+struct st_i32x4 __attribute__((riscv_vls_cc)) test_call(struct st_i32x4 arg) {
+  struct st_i32x4 abc = dummy(arg);
+  return abc;
+}
diff --git a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
index 128610e578c26..3ed4f594d55d8 100644
--- a/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
+++ b/clang/test/CodeGen/RISCV/riscv-vector-callingconv-llvm-ir.cpp
@@ -123,34 +123,34 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z18test_too_large_256Dv64_i(<vscale x 16 x i32> noundef %arg.coerce)
 [[riscv::vls_cc(256)]] void test_too_large_256(int32x64_t arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z13test_st_i32x48st_i32x4(<vscale x 2 x i32> %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z13test_st_i32x48st_i32x4(<vscale x 2 x i32> %arg.target_coerce)
 [[riscv::vls_cc]] void test_st_i32x4(struct st_i32x4 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z17test_st_i32x4_2568st_i32x4(<vscale x 1 x i32> %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z17test_st_i32x4_2568st_i32x4(<vscale x 1 x i32> %arg.target_coerce)
 [[riscv::vls_cc(256)]] void test_st_i32x4_256(struct st_i32x4 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z18test_st_i32x4_arr113st_i32x4_arr1(<vscale x 2 x i32> %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z18test_st_i32x4_arr113st_i32x4_arr1(<vscale x 2 x i32> %arg.target_coerce)
 [[riscv::vls_cc]] void test_st_i32x4_arr1(struct st_i32x4_arr1 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z22test_st_i32x4_arr1_25613st_i32x4_arr1(<vscale x 1 x i32> %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z22test_st_i32x4_arr1_25613st_i32x4_arr1(<vscale x 1 x i32> %arg.target_coerce)
 [[riscv::vls_cc(256)]] void test_st_i32x4_arr1_256(struct st_i32x4_arr1 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z18test_st_i32x4_arr413st_i32x4_arr4(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z18test_st_i32x4_arr413st_i32x4_arr4(target("riscv.vector.tuple", <vscale x 8 x i8>, 4) %arg.target_coerce)
 [[riscv::vls_cc]] void test_st_i32x4_arr4(struct st_i32x4_arr4 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z22test_st_i32x4_arr4_25613st_i32x4_arr4(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z22test_st_i32x4_arr4_25613st_i32x4_arr4(target("riscv.vector.tuple", <vscale x 4 x i8>, 4) %arg.target_coerce)
 [[riscv::vls_cc(256)]] void test_st_i32x4_arr4_256(struct st_i32x4_arr4 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z18test_st_i32x4_arr813st_i32x4_arr8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z18test_st_i32x4_arr813st_i32x4_arr8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %arg.target_coerce)
 [[riscv::vls_cc]] void test_st_i32x4_arr8(struct st_i32x4_arr8 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z22test_st_i32x4_arr8_25613st_i32x4_arr8(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z22test_st_i32x4_arr8_25613st_i32x4_arr8(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg.target_coerce)
 [[riscv::vls_cc(256)]] void test_st_i32x4_arr8_256(struct st_i32x4_arr8 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x210st_i32x4x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x210st_i32x4x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg.target_coerce)
 [[riscv::vls_cc]] void test_st_i32x4x2(struct st_i32x4x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x2_25610st_i32x4x2(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x2_25610st_i32x4x2(target("riscv.vector.tuple", <vscale x 4 x i8>, 2) %arg.target_coerce)
 [[riscv::vls_cc(256)]] void test_st_i32x4x2_256(struct st_i32x4x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x8x210st_i32x8x2(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x8x210st_i32x8x2(target("riscv.vector.tuple", <vscale x 16 x i8>, 2) %arg.target_coerce)
 [[riscv::vls_cc]] void test_st_i32x8x2(struct st_i32x8x2 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x8x2_25610st_i32x8x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x8x2_25610st_i32x8x2(target("riscv.vector.tuple", <vscale x 8 x i8>, 2) %arg.target_coerce)
 [[riscv::vls_cc(256)]] void test_st_i32x8x2_256(struct st_i32x8x2 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z16test_st_i32x64x211st_i32x64x2(ptr dead_on_return noundef %arg)
@@ -158,14 +158,14 @@ typedef int __attribute__((vector_size(256))) int32x64_t;
 // CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z20test_st_i32x64x2_25611st_i32x64x2(ptr dead_on_return noundef %arg)
 [[riscv::vls_cc(256)]] void test_st_i32x64x2_256(struct st_i32x64x2 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x310st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x310st_i32x4x3(target("riscv.vector.tuple", <vscale x 8 x i8>, 3) %arg.target_coerce)
 [[riscv::vls_cc]] void test_st_i32x4x3(struct st_i32x4x3 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x3_25610st_i32x4x3(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x3_25610st_i32x4x3(target("riscv.vector.tuple", <vscale x 4 x i8>, 3) %arg.target_coerce)
 [[riscv::vls_cc(256)]] void test_st_i32x4x3_256(struct st_i32x4x3 arg) {}
 
-// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x810st_i32x4x8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x810st_i32x4x8(target("riscv.vector.tuple", <vscale x 8 x i8>, 8) %arg.target_coerce)
 [[riscv::vls_cc]] void test_st_i32x4x8(struct st_i32x4x8 arg) {}
-// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x8_25610st_i32x4x8(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg)
+// CHECK-LLVM: define dso_local riscv_vls_cc(256) void @_Z19test_st_i32x4x8_25610st_i32x4x8(target("riscv.vector.tuple", <vscale x 4 x i8>, 8) %arg.target_coerce)
 [[riscv::vls_cc(256)]] void test_st_i32x4x8_256(struct st_i32x4x8 arg) {}
 
 // CHECK-LLVM: define dso_local riscv_vls_cc(128) void @_Z15test_st_i32x4x910st_i32x4x9(ptr dead_on_return noundef %arg)
diff --git a/clang/test/CodeGen/WebAssembly/builtins-test-fp-sig.c b/clang/test/CodeGen/WebAssembly/builtins-test-fp-sig.c
new file mode 100644
index 0000000000000..88447f7fa232d
--- /dev/null
+++ b/clang/test/CodeGen/WebAssembly/builtins-test-fp-sig.c
@@ -0,0 +1,70 @@
+// RUN: %clang_cc1 -triple wasm32-unknown-unknown -target-feature +gc -O3 -emit-llvm -DSINGLE_VALUE -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY-SV
+// RUN: %clang_cc1 -triple wasm64-unknown-unknown -target-feature +gc -O3 -emit-llvm -DSINGLE_VALUE -o - %s | FileCheck %s -check-prefixes WEBASSEMBLY,WEBASSEMBLY-SV
+// RUN: %clang_cc1 -triple wasm64-unknown-unknown -target-feature +gc -target-abi experimental-mv -O3 -emit-llvm  -o - %s 2>&1 | FileCheck %s -check-prefixes WEBASSEMBLY
+// RUN: not %clang_cc1 -triple wasm64-unknown-unknown -O3 -emit-llvm -o - %s 2>&1 | FileCheck %s -check-prefixes MISSING-GC
+
+void use(int);
+
+typedef void (*Fvoid)(void);
+void test_function_pointer_signature_void(Fvoid func) {
+  // MISSING-GC: error: '__builtin_wasm_test_function_pointer_signature' needs target feature gc
+  // WEBASSEMBLY:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, token poison)
+  use(__builtin_wasm_test_function_pointer_signature(func));
+}
+
+typedef float (*Ffloats)(float, double, int);
+void test_function_pointer_signature_floats(Ffloats func) {
+  // WEBASSEMBLY:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, float poison, token poison, float poison, double poison, i32 poison)
+  use(__builtin_wasm_test_function_pointer_signature(func));
+}
+
+typedef void (*Fpointers)(Fvoid, Ffloats, void*, int*, int***, char[5]);
+void test_function_pointer_signature_pointers(Fpointers func) {
+  // WEBASSEMBLY:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, token poison, ptr poison, ptr poison, ptr poison, ptr poison, ptr poison, ptr poison)
+  use(__builtin_wasm_test_function_pointer_signature(func));
+}
+
+typedef void (*FVarArgs)(int, ...);
+void test_function_pointer_signature_varargs(FVarArgs func) {
+  // WEBASSEMBLY:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, token poison, i32 poison, ptr poison)
+  use(__builtin_wasm_test_function_pointer_signature(func));
+}
+
+typedef __externref_t (*FExternRef)(__externref_t, __externref_t);
+void test_function_pointer_externref(FExternRef func) {
+  // WEBASSEMBLY:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, ptr addrspace(10) poison, token poison, ptr addrspace(10) poison, ptr addrspace(10) poison)
+  use(__builtin_wasm_test_function_pointer_signature(func));
+}
+
+typedef __funcref Fpointers (*FFuncRef)(__funcref Fvoid, __funcref Ffloats);
+void test_function_pointer_funcref(FFuncRef func) {
+  // WEBASSEMBLY:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, ptr addrspace(20) poison, token poison, ptr addrspace(20) poison, ptr addrspace(20) poison)
+  use(__builtin_wasm_test_function_pointer_signature(func));
+}
+
+#ifdef SINGLE_VALUE
+// Some tests that we get struct ABIs correct. There is no special code in
+// __builtin_wasm_test_function_pointer_signature for this, it gets handled by
+// the normal type lowering code.
+// Single element structs are unboxed, multi element structs are passed on
+// stack.
+typedef struct {double x;} (*Fstructs1)(struct {double x;}, struct {float x;}, struct {double x; float y;});
+void test_function_pointer_structs1(Fstructs1 func) {
+  // WEBASSEMBLY-SV:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, double poison, token poison, double poison, float poison, ptr poison)
+  use(__builtin_wasm_test_function_pointer_signature(func));
+}
+
+// Two element return struct ==> return ptr on stack
+typedef struct {double x; double y;} (*Fstructs2)(void);
+void test_function_pointer_structs2(Fstructs2 func) {
+  // WEBASSEMBLY-SV:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, token poison, ptr poison)
+  use(__builtin_wasm_test_function_pointer_signature(func));
+}
+
+// Return union ==> return ptr on stack, one element union => unboxed
+typedef union {double x; float y;} (*FUnions)(union {double x; float y;}, union {double x;});
+void test_function_pointer_unions(FUnions func) {
+  // WEBASSEMBLY-SV:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, token poison, ptr poison, ptr poison, double poison)
+  use(__builtin_wasm_test_function_pointer_signature(func));
+}
+#endif
diff --git a/clang/test/CodeGen/X86/avx-builtins.c b/clang/test/CodeGen/X86/avx-builtins.c
index a6e70aae420ea..e0a21c2ac0d23 100644
--- a/clang/test/CodeGen/X86/avx-builtins.c
+++ b/clang/test/CodeGen/X86/avx-builtins.c
@@ -20,12 +20,14 @@ __m256d test_mm256_add_pd(__m256d A, __m256d B) {
   // CHECK: fadd <4 x double>
   return _mm256_add_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d( _mm256_add_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-4.0, -5.0, +6.0, +7.0}), -8.0, -10.0, +12.0, +14.0));
 
 __m256 test_mm256_add_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_add_ps
   // CHECK: fadd <8 x float>
   return _mm256_add_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256(_mm256_add_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}), -8.0f, -10.0f, +12.0f, +14.0f, +14.0f, +12.0f, -10.0f, -8.0f));
 
 __m256d test_mm256_addsub_pd(__m256d A, __m256d B) {
   // CHECK-LABEL: test_mm256_addsub_pd
@@ -44,12 +46,14 @@ __m256d test_mm256_and_pd(__m256d A, __m256d B) {
   // CHECK: and <4 x i64>
   return _mm256_and_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_and_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){+0.0, -0.0, -0.0, +7.0}), +0.0, -0.0, +0.0, +7.0));
 
 __m256 test_mm256_and_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_and_ps
   // CHECK: and <8 x i32>
   return _mm256_and_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256(_mm256_and_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), +0.0f, -0.0f, +0.0f, +7.0f, +7.0f, +0.0f, -0.0f, +0.0f));
 
 __m256d test_mm256_andnot_pd(__m256d A, __m256d B) {
   // CHECK-LABEL: test_mm256_andnot_pd
@@ -57,6 +61,7 @@ __m256d test_mm256_andnot_pd(__m256d A, __m256d B) {
   // CHECK: and <4 x i64>
   return _mm256_andnot_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_andnot_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){+0.0, -0.0, -0.0, +7.0}), +0.0, +0.0, -0.0, +0.0));
 
 __m256 test_mm256_andnot_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_andnot_ps
@@ -64,6 +69,7 @@ __m256 test_mm256_andnot_ps(__m256 A, __m256 B) {
   // CHECK: and <8 x i32>
   return _mm256_andnot_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256(_mm256_andnot_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), +0.0f, +0.0f, -0.0f, +0.0f, +0.0f, -0.0f, +0.0f, +0.0f));
 
 __m256d test_mm256_blend_pd(__m256d A, __m256d B) {
   // CHECK-LABEL: test_mm256_blend_pd
@@ -141,11 +147,13 @@ __m256 test_mm256_castpd_ps(__m256d A) {
   // CHECK-LABEL: test_mm256_castpd_ps
   return _mm256_castpd_ps(A);
 }
+TEST_CONSTEXPR(match_m256(_mm256_castpd_ps((__m256d){-1.0, +2.0, +4.0, -6.0}), +0.0f, -1.875f, +0.0f, +2.0f, +0.0f, +2.25f, 0.0f, -2.375f));
 
 __m256i test_mm256_castpd_si256(__m256d A) {
   // CHECK-LABEL: test_mm256_castpd_si256
   return _mm256_castpd_si256(A);
 }
+TEST_CONSTEXPR(match_m256i(_mm256_castpd_si256((__m256d){-1.0, +2.0, -3.0, +4.0}), 0xBFF0000000000000ULL, 0x4000000000000000ULL, 0xC008000000000000ULL, 0x4010000000000000ULL));
 
 __m256d test_mm256_castpd128_pd256(__m128d A) {
   // CHECK-LABEL: test_mm256_castpd128_pd256
@@ -159,16 +167,19 @@ __m128d test_mm256_castpd256_pd128(__m256d A) {
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <2 x i32> <i32 0, i32 1>
   return _mm256_castpd256_pd128(A);
 }
+TEST_CONSTEXPR(match_m128d(_mm256_castpd256_pd128((__m256d){-1.0, +2.0, -3.0, +4.0}), -1.0, +2.0));
 
 __m256d test_mm256_castps_pd(__m256 A) {
   // CHECK-LABEL: test_mm256_castps_pd
   return _mm256_castps_pd(A);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_castps_pd((__m256){0.0f, -1.0f, 0.0f, 4.0f, 0.0f, -2.0f, 0.0f, 6.0f}), -0.0078125, 512.0, -2.0, +8192.0));
 
 __m256i test_mm256_castps_si256(__m256 A) {
   // CHECK-LABEL: test_mm256_castps_si256
   return _mm256_castps_si256(A);
 }
+TEST_CONSTEXPR(match_m256i(_mm256_castps_si256((__m256){1.0f, -2.0f, -4.0f, 8.0f, -16.0f, +16.0f, +32.0f, -32.0f}), 0xC00000003F800000ULL, 0x41000000c0800000ULL, 0x41800000C1800000ULL, 0xC200000042000000ULL));
 
 __m256 test_mm256_castps128_ps256(__m128 A) {
   // CHECK-LABEL: test_mm256_castps128_ps256
@@ -182,6 +193,7 @@ __m128 test_mm256_castps256_ps128(__m256 A) {
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   return _mm256_castps256_ps128(A);
 }
+TEST_CONSTEXPR(match_m128(_mm256_castps256_ps128((__m256){1.0f, -2.0f, -4.0f, 8.0f, -16.0f, +16.0f, +32.0f, -32.0f}), 1.0f, -2.0f, -4.0f, 8.0f));
 
 __m256i test_mm256_castsi128_si256(__m128i A) {
   // CHECK-LABEL: test_mm256_castsi128_si256
@@ -194,17 +206,20 @@ __m256d test_mm256_castsi256_pd(__m256i A) {
   // CHECK-LABEL: test_mm256_castsi256_pd
   return _mm256_castsi256_pd(A);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_castsi256_pd((__m256i)(__v4du){0x4070000000000000ULL, 0xC000000000000000ULL, 0xBFF0000000000000ULL, 0xC008000000000000ULL}), 256.0, -2.0, -1.0, -3.0));
 
 __m256 test_mm256_castsi256_ps(__m256i A) {
   // CHECK-LABEL: test_mm256_castsi256_ps
   return _mm256_castsi256_ps(A);
 }
+TEST_CONSTEXPR(match_m256(_mm256_castsi256_ps((__m256i)(__v4du){0x42000000c1800000ULL, 0x43000000c2800000ULL, 0x41000000c0800000ULL, 0xC00000003F800000ULL}), -16.0f, 32.0f, -64.0f, 128.0f, -4.0f, 8.0f, 1.0f, -2.0f));
 
 __m128i test_mm256_castsi256_si128(__m256i A) {
   // CHECK-LABEL: test_mm256_castsi256_si128
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <2 x i32> <i32 0, i32 1>
   return _mm256_castsi256_si128(A);
 }
+TEST_CONSTEXPR(match_m128i(_mm256_castsi256_si128((__m256i)(__v4du){0xBFF0000000000000ULL, 0x4070000000000000ULL, 0xC000000000000000ULL, 0xC008000000000000ULL}), 0xBFF0000000000000ULL, 0x4070000000000000ULL));
 
 __m256d test_mm256_ceil_pd(__m256d x) {
   // CHECK-LABEL: test_mm256_ceil_pd
@@ -973,12 +988,14 @@ __m256d test_mm256_div_pd(__m256d A, __m256d B) {
   // CHECK: fdiv <4 x double>
   return _mm256_div_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d( _mm256_div_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-1.0, +1.0, -1.0, +1.0}), +4.0, -5.0, -6.0, +7.0));
 
 __m256 test_mm256_div_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_div_ps
   // CHECK: fdiv <8 x float>
   return _mm256_div_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256( _mm256_div_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-1.0f, +1.0f, -1.0f, +1.0f, +1.0f, -1.0f, +1.0f, -1.0f}), +4.0f, -5.0f, -6.0f, +7.0f, +7.0f, -6.0f, -5.0f, +4.0f));
 
 __m256 test_mm256_dp_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_dp_ps
@@ -1258,18 +1275,21 @@ __m256d test_mm256_movedup_pd(__m256d A) {
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   return _mm256_movedup_pd(A);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_movedup_pd((__m256d){+7.0, -7.0, -42.0, +42.0}), +7.0, +7.0, -42.0, -42.0));
 
 __m256 test_mm256_movehdup_ps(__m256 A) {
   // CHECK-LABEL: test_mm256_movehdup_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
   return _mm256_movehdup_ps(A);
 }
+TEST_CONSTEXPR(match_m256(_mm256_movehdup_ps((__m256){+1.0f,-1.0f,+2.0f,+4.0f,+8.0f,-8.0f,-3.0f,+3.0f}), -1.0f, -1.0f, +4.0f, +4.0f, -8.0f, -8.0f, +3.0f, +3.0f));
 
 __m256 test_mm256_moveldup_ps(__m256 A) {
   // CHECK-LABEL: test_mm256_moveldup_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   return _mm256_moveldup_ps(A);
 }
+TEST_CONSTEXPR(match_m256(_mm256_moveldup_ps((__m256){+1.0f,-1.0f,+2.0f,+4.0f,+8.0f,-8.0f,-3.0f,+3.0f}), +1.0f, +1.0f, +2.0f, +2.0f, +8.0f, +8.0f, -3.0f, -3.0f));
 
 int test_mm256_movemask_pd(__m256d A) {
   // CHECK-LABEL: test_mm256_movemask_pd
@@ -1288,24 +1308,28 @@ __m256d test_mm256_mul_pd(__m256d A, __m256d B) {
   // CHECK: fmul <4 x double>
   return _mm256_mul_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d( _mm256_mul_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-4.0, -5.0, +6.0, +7.0}), +16.0, +25.0, +36.0, +49.0));
 
 __m256 test_mm256_mul_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_mul_ps
   // CHECK: fmul <8 x float>
   return _mm256_mul_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256( _mm256_mul_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}), +16.0f, +25.0f, +36.0f, +49.0f, +49.0f, +36.0f, +25.0f, +16.0f));
 
 __m256d test_mm256_or_pd(__m256d A, __m256d B) {
   // CHECK-LABEL: test_mm256_or_pd
   // CHECK: or <4 x i64>
   return _mm256_or_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_or_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){+0.0, -0.0, -0.0, +7.0}), -4.0, -5.0, -6.0, +7.0));
 
 __m256 test_mm256_or_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_or_ps
   // CHECK: or <8 x i32>
   return _mm256_or_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256(_mm256_or_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -4.0f, -5.0f, -6.0f, +7.0f, +7.0f, -6.0f, -5.0f, -4.0f));
 
 __m128d test_mm_permute_pd(__m128d A) {
   // CHECK-LABEL: test_mm_permute_pd
@@ -1924,12 +1948,14 @@ __m256d test_mm256_sub_pd(__m256d A, __m256d B) {
   // CHECK: fsub <4 x double>
   return _mm256_sub_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d( _mm256_sub_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-0.0, +0.0, +2.0, -1.0}), -4.0, -5.0, 4.0, 8.0));
 
 __m256 test_mm256_sub_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_sub_ps
   // CHECK: fsub <8 x float>
   return _mm256_sub_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256( _mm256_sub_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-0.0f, +0.0f, +2.0f, -1.0f, -1.0f, +2.0f, +0.0f, -0.0f}), -4.0f, -5.0f, 4.0f, 8.0f, 8.0f, 4.0f, -5.0f, -4.0f));
 
 int test_mm_testc_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_testc_pd
@@ -2053,36 +2079,42 @@ __m256d test_mm256_unpackhi_pd(__m256d A, __m256d B) {
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
   return _mm256_unpackhi_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_unpackhi_pd((__m256d){+1.0, +2.0, +3.0, +4.0}, (__m256d){+5.0, +6.0, +7.0, +8.0}), +2.0, +6.0, +4.0, +8.0));
 
 __m256 test_mm256_unpackhi_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_unpackhi_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   return _mm256_unpackhi_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256(_mm256_unpackhi_ps((__m256){+0.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f}, (__m256){+10.0f, +11.0f, +12.0f, +13.0f, +14.0f, +15.0f, +16.0f, +17.0f}), +2.0f, +12.0f, +3.0f, +13.0f, +6.0f, +16.0f, +7.0f, +17.0f));
 
 __m256d test_mm256_unpacklo_pd(__m256d A, __m256d B) {
   // CHECK-LABEL: test_mm256_unpacklo_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   return _mm256_unpacklo_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_unpacklo_pd((__m256d){+1.0, +2.0, +3.0, +4.0}, (__m256d){+5.0, +6.0, +7.0, +8.0}), +1.0, +5.0, +3.0, +7.0));
 
 __m256 test_mm256_unpacklo_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_unpacklo_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   return _mm256_unpacklo_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256(_mm256_unpacklo_ps((__m256){+0.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f}, (__m256){+10.0f, +11.0f, +12.0f, +13.0f, +14.0f, +15.0f, +16.0f, +17.0f}), +0.0f, +10.0f, +1.0f, +11.0f, +4.0f, +14.0f, +5.0f, +15.0f));
 
 __m256d test_mm256_xor_pd(__m256d A, __m256d B) {
   // CHECK-LABEL: test_mm256_xor_pd
   // CHECK: xor <4 x i64>
   return _mm256_xor_pd(A, B);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_xor_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){+0.0, -0.0, -0.0, +7.0}), -4.0, +5.0, -6.0, +0.0));
 
 __m256 test_mm256_xor_ps(__m256 A, __m256 B) {
   // CHECK-LABEL: test_mm256_xor_ps
   // CHECK: xor <8 x i32>
   return _mm256_xor_ps(A, B);
 }
+TEST_CONSTEXPR(match_m256(_mm256_xor_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -4.0f, +5.0f, -6.0f, +0.0f, +0.0f, -6.0f, +5.0f, -4.0f));
 
 void test_mm256_zeroall(void) {
   // CHECK-LABEL: test_mm256_zeroall
diff --git a/clang/test/CodeGen/X86/avx2-builtins.c b/clang/test/CodeGen/X86/avx2-builtins.c
index 27da56fb75723..29cb3e8860be9 100644
--- a/clang/test/CodeGen/X86/avx2-builtins.c
+++ b/clang/test/CodeGen/X86/avx2-builtins.c
@@ -9,6 +9,7 @@
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 // NOTE: This should match the tests in llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
 
@@ -59,12 +60,14 @@ __m256i test_mm256_adds_epi8(__m256i a, __m256i b) {
   // CHECK: call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_adds_epi8(a, b);
 }
+TEST_CONSTEXPR(match_v32qi(_mm256_adds_epi8((__m256i)(__v32qs){0, +1, +2, +3, +4, +5, +6, +7, +8, +9, +10, +11, +12, +13, +14, +15, +100, +50, -100, +20, +80, -50, +120, -20, -100, -50, +100, -20, -80, +50, -120, +20}, (__m256i)(__v32qs){0, +1, +2, +3, +4, +5, +6, +7, +8, +9, +10, +11, +12, +13, +14, +15, +50, +80, -50, +110, +60, -30, +20, -10, +50, +80, -50, +110, +60, -30, +20, -10}), 0, +2, +4, +6, +8, +10, +12, +14, +16, +18, +20, +22, +24, +26, +28, +30, +127, +127, -128, +127, +127, -80, +127, -30, -50, +30, +50, +90, -20, +20, -100, +10));
 
 __m256i test_mm256_adds_epi16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_adds_epi16
   // CHECK: call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_adds_epi16(a, b);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_adds_epi16((__m256i)(__v16hi){0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, +32000, -32000, +32000, -32000}, (__m256i)(__v16hi){0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, +800, -800, -800, +800}), 0, -2, -4, -6, -8, -10, -12, -14, -16, -18, -20, -22, +32767, -32768, +31200, -31200));
 
 __m256i test_mm256_adds_epu8(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_adds_epu8
@@ -72,6 +75,7 @@ __m256i test_mm256_adds_epu8(__m256i a, __m256i b) {
   // CHECK: call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_adds_epu8(a, b);
 }
+TEST_CONSTEXPR(match_v32qu(_mm256_adds_epu8((__m256i)(__v32qu){0, 0, 0, 0, +64, +64, +64, +64, +64, +64, +127, +127, +127, +127, +127, +127, +128, +128, +128, +128, +128, +128, +192, +192, +192, +192, +192, +192, +255, +255, +255, +255}, (__m256i)(__v32qu){0, +127, +128, +255, 0, +64, +127, +128, +192, +255, 0, +64, +127, +128, +192, +255, 0, +64, +127, +128, +192, +255, 0, +64, +127, +128, +192, +255, 0, +127, +128, +255}), 0, +127, +128, +255, +64, +128, +191, +192, +255, +255, +127, +191, +254, +255, +255, +255, +128, +192, +255, +255, +255, +255, +192, +255, +255, +255, +255, +255, +255, +255, +255, +255));
 
 __m256i test_mm256_adds_epu16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_adds_epu16
@@ -79,6 +83,7 @@ __m256i test_mm256_adds_epu16(__m256i a, __m256i b) {
   // CHECK: call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_adds_epu16(a, b);
 }
+TEST_CONSTEXPR(match_v16hu(_mm256_adds_epu16((__m256i)(__v16hu){0, 0, 0, 0, +32767, +32767, +32767, +32767, +32768, +32768, +32768, +32768, +65535, +65535, +65535, +65535}, (__m256i)(__v16hu){0, +32767, +32768, +65535, 0, +32767, +32768, +65535, 0, +32767, +32768, +65535, 0, +32767, +32768, +65535}), 0, +32767, +32768, +65535, +32767, +65534, +65535, +65535, +32768, +65535, +65535, +65535, +65535, +65535, +65535, +65535));
 
 __m256i test_mm256_alignr_epi8(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_alignr_epi8
@@ -97,6 +102,7 @@ __m256i test_mm256_and_si256(__m256i a, __m256i b) {
   // CHECK: and <4 x i64>
   return _mm256_and_si256(a, b);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_and_si256((__m256i)(__v4di){0, -1, 0, -1}, (__m256i)(__v4di){0, 0, -1, -1}), 0, 0, 0, -1));
 
 __m256i test_mm256_andnot_si256(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_andnot_si256
@@ -104,6 +110,7 @@ __m256i test_mm256_andnot_si256(__m256i a, __m256i b) {
   // CHECK: and <4 x i64>
   return _mm256_andnot_si256(a, b);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_andnot_si256((__m256i)(__v4di){0, -1, 0, -1}, (__m256i)(__v4di){0, 0, -1, -1}), 0, 0, -1, 0));
 
 __m256i test_mm256_avg_epu8(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_avg_epu8
@@ -153,6 +160,7 @@ __m128i test_mm_broadcastb_epi8(__m128i a) {
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> zeroinitializer
   return _mm_broadcastb_epi8(a);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_broadcastb_epi8((__m128i)(__v16qi){42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}), 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42));
 
 __m256i test_mm256_broadcastb_epi8(__m128i a) {
   // CHECK-LABEL: test_mm256_broadcastb_epi8
@@ -160,6 +168,7 @@ __m256i test_mm256_broadcastb_epi8(__m128i a) {
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <32 x i32> zeroinitializer
   return _mm256_broadcastb_epi8(a);
 }
+TEST_CONSTEXPR(match_v32qi(_mm256_broadcastb_epi8((__m128i)(__v16qi){42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}), 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42));
 
 __m128i test_mm_broadcastd_epi32(__m128i a) {
   // CHECK-LABEL: test_mm_broadcastd_epi32
@@ -167,6 +176,7 @@ __m128i test_mm_broadcastd_epi32(__m128i a) {
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> zeroinitializer
   return _mm_broadcastd_epi32(a);
 }
+TEST_CONSTEXPR(match_v4si(_mm_broadcastd_epi32((__m128i)(__v4si){-42, 0, 0, 0}), -42, -42, -42, -42));
 
 __m256i test_mm256_broadcastd_epi32(__m128i a) {
   // CHECK-LABEL: test_mm256_broadcastd_epi32
@@ -174,6 +184,7 @@ __m256i test_mm256_broadcastd_epi32(__m128i a) {
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> zeroinitializer
   return _mm256_broadcastd_epi32(a);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_broadcastd_epi32((__m128i)(__v4si){-42, 0, 0, 0}), -42, -42, -42, -42, -42, -42, -42, -42));
 
 __m128i test_mm_broadcastq_epi64(__m128i a) {
   // CHECK-LABEL: test_mm_broadcastq_epi64
@@ -181,6 +192,7 @@ __m128i test_mm_broadcastq_epi64(__m128i a) {
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> zeroinitializer
   return _mm_broadcastq_epi64(a);
 }
+TEST_CONSTEXPR(match_v2di(_mm_broadcastq_epi64((__m128i)(__v2di){-42, 0}), -42, -42));
 
 __m256i test_mm256_broadcastq_epi64(__m128i a) {
   // CHECK-LABEL: test_mm256_broadcastq_epi64
@@ -188,12 +200,14 @@ __m256i test_mm256_broadcastq_epi64(__m128i a) {
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <4 x i32> zeroinitializer
   return _mm256_broadcastq_epi64(a);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_broadcastq_epi64((__m128i)(__v2di){-42, 0}), -42, -42, -42, -42));
 
 __m128d test_mm_broadcastsd_pd(__m128d a) {
   // CHECK-LABEL: test_mm_broadcastsd_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> zeroinitializer
   return _mm_broadcastsd_pd(a);
 }
+TEST_CONSTEXPR(match_m128d(_mm_broadcastsd_pd((__m128d){+7.0, -7.0}), +7.0, +7.0));
 
 __m256d test_mm256_broadcastsd_pd(__m128d a) {
   // CHECK-LABEL: test_mm256_broadcastsd_pd
@@ -201,12 +215,14 @@ __m256d test_mm256_broadcastsd_pd(__m128d a) {
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <4 x i32> zeroinitializer
   return _mm256_broadcastsd_pd(a);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_broadcastsd_pd((__m128d){+7.0, -7.0}), +7.0, +7.0, +7.0, +7.0));
 
 __m256i test_mm256_broadcastsi128_si256(__m128i a) {
   // CHECK-LABEL: test_mm256_broadcastsi128_si256
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   return _mm256_broadcastsi128_si256(a);
 }
+TEST_CONSTEXPR(match_m256i(_mm256_broadcastsi128_si256((__m128i)(__v2di){3, 45}), 3, 45, 3, 45));
 
 __m256i test_mm_broadcastsi128_si256(__m128i a) {
   // CHECK-LABEL: test_mm_broadcastsi128_si256
@@ -220,6 +236,7 @@ __m128 test_mm_broadcastss_ps(__m128 a) {
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> zeroinitializer
   return _mm_broadcastss_ps(a);
 }
+TEST_CONSTEXPR(match_m128(_mm_broadcastss_ps((__m128){-4.0f, +5.0f, +6.0f, +7.0f}), -4.0f, -4.0f, -4.0f, -4.0f));
 
 __m256 test_mm256_broadcastss_ps(__m128 a) {
   // CHECK-LABEL: test_mm256_broadcastss_ps
@@ -227,6 +244,7 @@ __m256 test_mm256_broadcastss_ps(__m128 a) {
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <8 x i32> zeroinitializer
   return _mm256_broadcastss_ps(a);
 }
+TEST_CONSTEXPR(match_m256(_mm256_broadcastss_ps((__m128){-4.0f, +5.0f, +6.0f, +7.0f}), -4.0f, -4.0f, -4.0f, -4.0f, -4.0f, -4.0f, -4.0f, -4.0f));
 
 __m128i test_mm_broadcastw_epi16(__m128i a) {
   // CHECK-LABEL: test_mm_broadcastw_epi16
@@ -234,6 +252,7 @@ __m128i test_mm_broadcastw_epi16(__m128i a) {
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i32> zeroinitializer
   return _mm_broadcastw_epi16(a);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_broadcastw_epi16((__m128i)(__v8hi){42, 0, 0, 0, 0, 0, 0, 0}), 42, 42, 42, 42, 42, 42, 42, 42));
 
 __m256i test_mm256_broadcastw_epi16(__m128i a) {
   // CHECK-LABEL: test_mm256_broadcastw_epi16
@@ -241,6 +260,7 @@ __m256i test_mm256_broadcastw_epi16(__m128i a) {
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <16 x i32> zeroinitializer
   return _mm256_broadcastw_epi16(a);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_broadcastw_epi16((__m128i)(__v8hi){42, 0, 0, 0, 0, 0, 0, 0}), 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42));
 
 __m256i test_mm256_bslli_epi128(__m256i a) {
   // CHECK-LABEL: test_mm256_bslli_epi128
@@ -307,6 +327,7 @@ __m256i test_mm256_cvtepi8_epi16(__m128i a) {
   // CHECK: sext <16 x i8> %{{.*}} to <16 x i16>
   return _mm256_cvtepi8_epi16(a);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_cvtepi8_epi16(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), -3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12));
 
 __m256i test_mm256_cvtepi8_epi32(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepi8_epi32
@@ -314,6 +335,7 @@ __m256i test_mm256_cvtepi8_epi32(__m128i a) {
   // CHECK: sext <8 x i8> %{{.*}} to <8 x i32>
   return _mm256_cvtepi8_epi32(a);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_cvtepi8_epi32(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), -3, 2, -1, 0, 1, -2, 3, -4));
 
 __m256i test_mm256_cvtepi8_epi64(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepi8_epi64
@@ -321,12 +343,14 @@ __m256i test_mm256_cvtepi8_epi64(__m128i a) {
   // CHECK: sext <4 x i8> %{{.*}} to <4 x i64>
   return _mm256_cvtepi8_epi64(a);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_cvtepi8_epi64(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), -3, 2, -1, 0));
 
 __m256i test_mm256_cvtepi16_epi32(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepi16_epi32
   // CHECK: sext <8 x i16> %{{.*}} to <8 x i32>
   return _mm256_cvtepi16_epi32(a);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_cvtepi16_epi32(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), -300, 2, -1, 0, 1, -2, 3, -4));
 
 __m256i test_mm256_cvtepi16_epi64(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepi16_epi64
@@ -334,18 +358,21 @@ __m256i test_mm256_cvtepi16_epi64(__m128i a) {
   // CHECK: sext <4 x i16> %{{.*}} to <4 x i64>
   return _mm256_cvtepi16_epi64(a);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_cvtepi16_epi64(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), -300, 2, -1, 0));
 
 __m256i test_mm256_cvtepi32_epi64(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepi32_epi64
   // CHECK: sext <4 x i32> %{{.*}} to <4 x i64>
   return _mm256_cvtepi32_epi64(a);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_cvtepi32_epi64(_mm_setr_epi32(-70000, 2, -1, 0)), -70000, 2, -1, 0));
 
 __m256i test_mm256_cvtepu8_epi16(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepu8_epi16
   // CHECK: zext <16 x i8> %{{.*}} to <16 x i16>
   return _mm256_cvtepu8_epi16(a);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_cvtepu8_epi16(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), 253, 2, 255, 0, 1, 254, 3, 252, 5, 250, 7, 248, 9, 246, 11, 244));
 
 __m256i test_mm256_cvtepu8_epi32(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepu8_epi32
@@ -353,6 +380,7 @@ __m256i test_mm256_cvtepu8_epi32(__m128i a) {
   // CHECK: zext <8 x i8> %{{.*}} to <8 x i32>
   return _mm256_cvtepu8_epi32(a);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_cvtepu8_epi32(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), 253, 2, 255, 0, 1, 254, 3, 252));
 
 __m256i test_mm256_cvtepu8_epi64(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepu8_epi64
@@ -360,12 +388,14 @@ __m256i test_mm256_cvtepu8_epi64(__m128i a) {
   // CHECK: zext <4 x i8> %{{.*}} to <4 x i64>
   return _mm256_cvtepu8_epi64(a);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_cvtepu8_epi64(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), 253, 2, 255, 0));
 
 __m256i test_mm256_cvtepu16_epi32(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepu16_epi32
   // CHECK: zext <8 x i16> {{.*}} to <8 x i32>
   return _mm256_cvtepu16_epi32(a);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_cvtepu16_epi32(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), 65236, 2, 65535, 0, 1, 65534, 3, 65532));
 
 __m256i test_mm256_cvtepu16_epi64(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepu16_epi64
@@ -373,12 +403,14 @@ __m256i test_mm256_cvtepu16_epi64(__m128i a) {
   // CHECK: zext <4 x i16> %{{.*}} to <4 x i64>
   return _mm256_cvtepu16_epi64(a);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_cvtepu16_epi64(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), 65236, 2, 65535, 0));
 
 __m256i test_mm256_cvtepu32_epi64(__m128i a) {
   // CHECK-LABEL: test_mm256_cvtepu32_epi64
   // CHECK: zext <4 x i32> %{{.*}} to <4 x i64>
   return _mm256_cvtepu32_epi64(a);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_cvtepu32_epi64(_mm_setr_epi32(-70000, 2, -1, 0)), 4294897296, 2, 4294967295, 0));
 
 __m128i test0_mm256_extracti128_si256_0(__m256i a) {
   // CHECK-LABEL: test0_mm256_extracti128_si256
@@ -876,6 +908,7 @@ __m256i test_mm256_mul_epi32(__m256i a, __m256i b) {
   // CHECK: mul <4 x i64> %{{.*}}, %{{.*}}
   return _mm256_mul_epi32(a, b);
 }
+TEST_CONSTEXPR(match_m256i(_mm256_mul_epi32((__m256i)(__v8si){+1, -2, +3, -4, +5, -6, +7, -8}, (__m256i)(__v8si){-16, -14, +12, +10, -8, +6, -4, +2}), -16, 36, -40, -28));
 
 __m256i test_mm256_mul_epu32(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_mul_epu32
@@ -884,18 +917,21 @@ __m256i test_mm256_mul_epu32(__m256i a, __m256i b) {
   // CHECK: mul <4 x i64> %{{.*}}, %{{.*}}
   return _mm256_mul_epu32(a, b);
 }
+TEST_CONSTEXPR(match_m256i(_mm256_mul_epu32((__m256i)(__v8si){+1, -2, +3, -4, +5, -6, +7, -8}, (__m256i)(__v8si){-16, -14, +12, +10, -8, +6, -4, +2}), 4294967280, 36, 21474836440, 30064771044));
 
 __m256i test_mm256_mulhi_epu16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_mulhi_epu16
   // CHECK: call <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_mulhi_epu16(a, b);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_mulhi_epu16((__m256i)(__v16hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16}, (__m256i)(__v16hi){-32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), 0, -32, 0, 25, 4, -28, 0, 17, 8, -24, 0, 9, 12, 5, 14, 1));
 
 __m256i test_mm256_mulhi_epi16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_mulhi_epi16
   // CHECK: call <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_mulhi_epi16(a, b);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_mulhi_epi16((__m256i)(__v16hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16}, (__m256i)(__v16hi){-32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, -1, -1, -1));
 
 __m256i test_mm256_mulhrs_epi16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_mulhrs_epi16
@@ -908,6 +944,7 @@ __m256i test_mm256_mullo_epi16(__m256i a, __m256i b) {
   // CHECK: mul <16 x i16>
   return _mm256_mullo_epi16(a, b);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_mullo_epi16((__m256i)(__v16hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16}, (__m256i)(__v16hi){-32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), -32, 60, 84, -104, -120, 132, 140, -144, -144, 140, 132, -120, -104, -84, -60, -32));
 
 __m256i test_mm256_mullo_epi32(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_mullo_epi32
@@ -920,6 +957,7 @@ __m256i test_mm256_or_si256(__m256i a, __m256i b) {
   // CHECK: or <4 x i64>
   return _mm256_or_si256(a, b);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_or_si256((__m256i)(__v4di){0, -1, 0, -1}, (__m256i)(__v4di){0, 0, -1, -1}), 0, -1, -1, -1));
 
 __m256i test_mm256_packs_epi16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_packs_epi16
@@ -1070,24 +1108,28 @@ __m128i test_mm_sllv_epi32(__m128i a, __m128i b) {
   // CHECK: call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_sllv_epi32(a, b);
 }
+TEST_CONSTEXPR(match_v4si(_mm_sllv_epi32((__m128i)(__v4si){1, -2, 3, -4}, (__m128i)(__v4si){1, 2, 3, -4}), 2, -8, 24, 0));
 
 __m256i test_mm256_sllv_epi32(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_sllv_epi32
   // CHECK: call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_sllv_epi32(a, b);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_sllv_epi32((__m256i)(__v8si){1, -2, 3, -4, 5, -6, 7, -8}, (__m256i)(__v8si){1, 2, 3, 4, -17, 31, 33, 29}), 2, -8, 24, -64, 0, 0, 0, 0));
 
 __m128i test_mm_sllv_epi64(__m128i a, __m128i b) {
   // CHECK-LABEL: test_mm_sllv_epi64
   // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_sllv_epi64(a, b);
 }
+TEST_CONSTEXPR(match_m128i(_mm_sllv_epi64((__m128i)(__v2di){1, -3}, (__m128i)(__v2di){8, 63}), 256, 0x8000000000000000ULL));
 
 __m256i test_mm256_sllv_epi64(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_sllv_epi64
   // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_sllv_epi64(a, b);
 }
+TEST_CONSTEXPR(match_m256i(_mm256_sllv_epi64((__m256i)(__v4di){1, -2, 3, -4}, (__m256i)(__v4di){1, 2, 3, -4}), 2, -8, 24, 0));
 
 __m256i test_mm256_sra_epi16(__m256i a, __m128i b) {
   // CHECK-LABEL: test_mm256_sra_epi16
@@ -1130,12 +1172,14 @@ __m128i test_mm_srav_epi32(__m128i a, __m128i b) {
   // CHECK: call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_srav_epi32(a, b);
 }
+TEST_CONSTEXPR(match_v4si(_mm_srav_epi32((__m128i)(__v4si){1, -2, 3, -4}, (__m128i)(__v4si){1, 2, 3, -4}), 0, -1, 0, -1));
 
 __m256i test_mm256_srav_epi32(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_srav_epi32
   // CHECK: call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_srav_epi32(a, b);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_srav_epi32((__m256i)(__v8si){1, -2, 3, -4, 5, -6, 7, -8}, (__m256i)(__v8si){1, 2, 3, 4, -17, 31, 33, 29}), 0, -1, 0, -1, 0, -1, 0, -1));
 
 __m256i test_mm256_srl_epi16(__m256i a, __m128i b) {
   // CHECK-LABEL: test_mm256_srl_epi16
@@ -1202,24 +1246,28 @@ __m128i test_mm_srlv_epi32(__m128i a, __m128i b) {
   // CHECK: call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_srlv_epi32(a, b);
 }
+TEST_CONSTEXPR(match_v4si(_mm_srlv_epi32((__m128i)(__v4si){1, -2, 3, -4}, (__m128i)(__v4si){1, 2, 3, -4}), 0, 1073741823, 0, 0));
 
 __m256i test_mm256_srlv_epi32(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_srlv_epi32
   // CHECK: call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_srlv_epi32(a, b);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_srlv_epi32((__m256i)(__v8si){1, -2, 3, -4, 5, -6, 7, -8}, (__m256i)(__v8si){1, 2, 3, 4, -17, 31, 33, 29}), 0, 1073741823, 0, 268435455, 0, 1, 0, 7));
 
 __m128i test_mm_srlv_epi64(__m128i a, __m128i b) {
   // CHECK-LABEL: test_mm_srlv_epi64
   // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_srlv_epi64(a, b);
 }
+TEST_CONSTEXPR(match_m128i(_mm_srlv_epi64((__m128i)(__v2di){1, -3}, (__m128i)(__v2di){8, 63}), 0, 1));
 
 __m256i test_mm256_srlv_epi64(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_srlv_epi64
   // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_srlv_epi64(a, b);
 }
+TEST_CONSTEXPR(match_m256i(_mm256_srlv_epi64((__m256i)(__v4di){1, -2, 3, -4}, (__m256i)(__v4di){1, 2, 3, -4}), 0, 0x3FFFFFFFFFFFFFFFULL, 0, 0));
 
 __m256i test_mm256_stream_load_si256(__m256i const *a) {
   // CHECK-LABEL: test_mm256_stream_load_si256
@@ -1262,12 +1310,14 @@ __m256i test_mm256_subs_epi8(__m256i a, __m256i b) {
   // CHECK: call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_subs_epi8(a, b);
 }
+TEST_CONSTEXPR(match_v32qi(_mm256_subs_epi8((__m256i)(__v32qs){0, +1, +2, +3, +4, +5, +6, +7, +8, +9, +10, +11, +12, +13, +14, +15, +100, +50, -100, +20, +80, -50, +120, -20, -100, -50, +100, -20, -80, +50, -120, +20}, (__m256i)(__v32qs){0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15, -50, -80, +50, -110, -60, +30, -20, +10, -50, -80, +50, -110, -60, +30, -20, +10}), 0, +2, +4, +6, +8, +10, +12, +14, +16, +18, +20, +22, +24, +26, +28, +30, +127, +127, -128, +127, +127, -80, +127, -30, -50, +30, +50, +90, -20, +20, -100, +10));
 
 __m256i test_mm256_subs_epi16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_subs_epi16
   // CHECK: call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_subs_epi16(a, b);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_subs_epi16((__m256i)(__v16hi){0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, +32000, -32000, +32000, -32000}, (__m256i)(__v16hi){0, +1, +2, +3, +4, +5, +6, +7, +8, +9, +10, +11, -800, +800, +800, -800}),0, -2, -4, -6, -8, -10, -12, -14, -16, -18, -20, -22, +32767, -32768, +31200, -31200));
 
 __m256i test_mm256_subs_epu8(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_subs_epu8
@@ -1275,6 +1325,7 @@ __m256i test_mm256_subs_epu8(__m256i a, __m256i b) {
   // CHECK: call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_subs_epu8(a, b);
 }
+TEST_CONSTEXPR(match_v32qu(_mm256_subs_epu8((__m256i)(__v32qu){0, 0, 0, 0, +64, +64, +64, +64, +64, +64, +127, +127, +127, +127, +127, +127, +128, +128, +128, +128, +128, +128, +192, +192, +192, +192, +192, +192, +255, +255, +255, +255}, (__m256i)(__v32qu){0, +127, +128, +255, 0, +64, +127, +128, +192, +255, 0, +64, +127, +128, +192, +255, 0, +64, +127, +128, +192, +255, 0, +64, +127, +128, +192, +255, 0, +127, +128, +255}), 0, 0, 0, 0, +64, 0, 0, 0, 0, 0, +127, +63, 0, 0, 0, 0, +128, +64, +1, 0, 0, 0, +192, +128, +65, +64, 0, 0, +255, +128, +127, 0));
 
 __m256i test_mm256_subs_epu16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_subs_epu16
@@ -1282,57 +1333,67 @@ __m256i test_mm256_subs_epu16(__m256i a, __m256i b) {
   // CHECK: call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_subs_epu16(a, b);
 }
+TEST_CONSTEXPR(match_v16hu(_mm256_subs_epu16((__m256i)(__v16hu){0, 0, 0, 0, +32767, +32767, +32767, +32767, +32768, +32768, +32768, +32768, +65535, +65535, +65535, +65535}, (__m256i)(__v16hu){0, +32767, +32768, +65535, 0, +32767, +32768, +65535, 0, +32767, +32768, +65535, 0, +32767, +32768, +65535}), 0, 0, 0, 0, +32767, 0, 0, 0, +32768, +1, 0, 0, +65535, +32768, +32767, 0));
 
 __m256i test_mm256_unpackhi_epi8(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_unpackhi_epi8
   // CHECK: shufflevector <32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   return _mm256_unpackhi_epi8(a, b);
 }
+TEST_CONSTEXPR(match_v32qi(_mm256_unpackhi_epi8((__m256i)(__v32qi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}, (__m256i)(__v32qi){32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}), 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47, 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63));
 
 __m256i test_mm256_unpackhi_epi16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_unpackhi_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   return _mm256_unpackhi_epi16(a, b);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_unpackhi_epi16((__m256i)(__v16hi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, (__m256i)(__v16hi){16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}), 4, 20, 5, 21, 6, 22, 7, 23, 12, 28, 13, 29, 14, 30, 15, 31));
 
 __m256i test_mm256_unpackhi_epi32(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_unpackhi_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   return _mm256_unpackhi_epi32(a, b);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_unpackhi_epi32((__m256i)(__v8si){0, 1, 2, 3, 4, 5, 6, 7}, (__m256i)(__v8si){8, 9, 10, 11, 12, 13, 14, 15}), 2, 10, 3, 11, 6, 14, 7, 15));
 
 __m256i test_mm256_unpackhi_epi64(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_unpackhi_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
   return _mm256_unpackhi_epi64(a, b);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_unpackhi_epi64((__m256i)(__v4di){0, 1, 2, 3}, (__m256i)(__v4di){ 4, 5, 6, 7}), 1, 5, 3, 7));
 
 __m256i test_mm256_unpacklo_epi8(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_unpacklo_epi8
   // CHECK: shufflevector <32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
   return _mm256_unpacklo_epi8(a, b);
 }
+TEST_CONSTEXPR(match_v32qi(_mm256_unpacklo_epi8((__m256i)(__v32qi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}, (__m256i)(__v32qi){32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}), 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55));
 
 __m256i test_mm256_unpacklo_epi16(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_unpacklo_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27>
   return _mm256_unpacklo_epi16(a, b);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_unpacklo_epi16((__m256i)(__v16hi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, (__m256i)(__v16hi){16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}), 0, 16, 1, 17, 2, 18, 3, 19, 8, 24, 9, 25, 10, 26, 11, 27));
 
 __m256i test_mm256_unpacklo_epi32(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_unpacklo_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   return _mm256_unpacklo_epi32(a, b);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_unpacklo_epi32((__m256i)(__v8si){0, 1, 2, 3, 4, 5, 6, 7}, (__m256i)(__v8si){ 8, 9, 10, 11, 12, 13, 14, 15}), 0, 8, 1, 9, 4, 12, 5, 13));
 
 __m256i test_mm256_unpacklo_epi64(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_unpacklo_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   return _mm256_unpacklo_epi64(a, b);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_unpacklo_epi64((__m256i)(__v4di){0, 1, 2, 3}, (__m256i)(__v4di){ 4, 5, 6, 7}), 0, 4, 2, 6));
 
 __m256i test_mm256_xor_si256(__m256i a, __m256i b) {
   // CHECK-LABEL: test_mm256_xor_si256
   // CHECK: xor <4 x i64>
   return _mm256_xor_si256(a, b);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_xor_si256((__m256i)(__v4di){0, -1, 0, -1}, (__m256i)(__v4di){0, 0, -1, -1}), 0, -1, -1, 0));
diff --git a/clang/test/CodeGen/X86/avx512-reduceIntrin.c b/clang/test/CodeGen/X86/avx512-reduceIntrin.c
index 2ceac3a8b2670..4069b463e3f11 100644
--- a/clang/test/CodeGen/X86/avx512-reduceIntrin.c
+++ b/clang/test/CodeGen/X86/avx512-reduceIntrin.c
@@ -1,162 +1,174 @@
-// RUN: %clang_cc1 -ffreestanding %s -O0 -triple=x86_64-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -O0 -triple=x86_64-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -O0 -triple=i386-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -O0 -triple=x86_64-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -O0 -triple=i386-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 long long test_mm512_reduce_add_epi64(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_add_epi64(
-// CHECK:    call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_add_epi64
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %{{.*}})
   return _mm512_reduce_add_epi64(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_add_epi64((__m512i)(__v8di){-4, -3, -2, -1, 0, 1, 2, 3}) == -4);
 
 long long test_mm512_reduce_mul_epi64(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_mul_epi64(
-// CHECK:    call i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_mul_epi64
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %{{.*}})
   return _mm512_reduce_mul_epi64(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_mul_epi64((__m512i)(__v8di){1, 2, 3, 4, 5, 6, 7, 8}) == 40320);
 
 long long test_mm512_reduce_or_epi64(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_or_epi64(
-// CHECK:    call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_or_epi64
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %{{.*}})
   return _mm512_reduce_or_epi64(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_or_epi64((__m512i)(__v8di){0x100, 0x200, 0x400, 0x800, 0, 0, 0, 0}) == 0xF00);
 
 long long test_mm512_reduce_and_epi64(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_and_epi64(
-// CHECK:    call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_and_epi64
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %{{.*}})
   return _mm512_reduce_and_epi64(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_and_epi64((__m512i)(__v8di){0xFFFF, 0xFF00, 0x00FF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFF00, 0x00FF}) == 0x0000);
 
 long long test_mm512_mask_reduce_add_epi64(__mmask8 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_add_epi64(
+// CHECK-LABEL: test_mm512_mask_reduce_add_epi64
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
-// CHECK:    call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %{{.*}})
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %{{.*}})
   return _mm512_mask_reduce_add_epi64(__M, __W);
 }
 
 long long test_mm512_mask_reduce_mul_epi64(__mmask8 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_mul_epi64(
+// CHECK-LABEL: test_mm512_mask_reduce_mul_epi64
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
-// CHECK:    call i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %{{.*}})
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %{{.*}})
   return _mm512_mask_reduce_mul_epi64(__M, __W);
 }
 
 long long test_mm512_mask_reduce_and_epi64(__mmask8 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_and_epi64(
+// CHECK-LABEL: test_mm512_mask_reduce_and_epi64
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
-// CHECK:    call i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %{{.*}})
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.and.v8i64(<8 x i64> %{{.*}})
   return _mm512_mask_reduce_and_epi64(__M, __W);
 }
 
 long long test_mm512_mask_reduce_or_epi64(__mmask8 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_or_epi64(
+// CHECK-LABEL: test_mm512_mask_reduce_or_epi64
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
-// CHECK:    call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %{{.*}})
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.or.v8i64(<8 x i64> %{{.*}})
   return _mm512_mask_reduce_or_epi64(__M, __W);
 }
 
 int test_mm512_reduce_add_epi32(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_add_epi32(
-// CHECK:    call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_add_epi32
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %{{.*}})
   return _mm512_reduce_add_epi32(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_add_epi32((__m512i)(__v16si){-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7}) == -8);
 
 int test_mm512_reduce_mul_epi32(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_mul_epi32(
-// CHECK:    call i32 @llvm.vector.reduce.mul.v16i32(<16 x i32> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_mul_epi32
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.mul.v16i32(<16 x i32> %{{.*}})
   return _mm512_reduce_mul_epi32(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_mul_epi32((__m512i)(__v16si){1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 1, 1, -3, 1, 1}) == -36);
 
 int test_mm512_reduce_or_epi32(__m512i __W){
-// CHECK:    call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> %{{.*}})
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.or.v16i32(<16 x i32> %{{.*}})
   return _mm512_reduce_or_epi32(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_or_epi32((__m512i)(__v16si){0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0, 0, 0, 0, 0, 0, 0, 0}) == 0xFF);
 
 int test_mm512_reduce_and_epi32(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_and_epi32(
-// CHECK:    call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_and_epi32
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %{{.*}})
   return _mm512_reduce_and_epi32(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_and_epi32((__m512i)(__v16si){0xFF, 0xF0, 0x0F, 0xFF, 0xFF, 0xFF, 0xF0, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xF0, 0xF0, 0x0F, 0x0F}) == 0x00);
 
 int test_mm512_mask_reduce_add_epi32(__mmask16 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_add_epi32(
+// CHECK-LABEL: test_mm512_mask_reduce_add_epi32
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-// CHECK:    call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %{{.*}})
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %{{.*}})
   return _mm512_mask_reduce_add_epi32(__M, __W);
 }
 
 int test_mm512_mask_reduce_mul_epi32(__mmask16 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_mul_epi32(
+// CHECK-LABEL: test_mm512_mask_reduce_mul_epi32
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-// CHECK:    call i32 @llvm.vector.reduce.mul.v16i32(<16 x i32> %{{.*}})
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.mul.v16i32(<16 x i32> %{{.*}})
   return _mm512_mask_reduce_mul_epi32(__M, __W);
 }
 
 int test_mm512_mask_reduce_and_epi32(__mmask16 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_and_epi32(
+// CHECK-LABEL: test_mm512_mask_reduce_and_epi32
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-// CHECK:    call i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %{{.*}})
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.and.v16i32(<16 x i32> %{{.*}})
   return _mm512_mask_reduce_and_epi32(__M, __W);
 }
 
 int test_mm512_mask_reduce_or_epi32(__mmask16 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_or_epi32(
+// CHECK-LABEL: test_mm512_mask_reduce_or_epi32
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-// CHECK:    call i32 @llvm.vector.reduce.or.v16i32(<16 x i32> %{{.*}})
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.or.v16i32(<16 x i32> %{{.*}})
   return _mm512_mask_reduce_or_epi32(__M, __W);
 }
 
 double test_mm512_reduce_add_pd(__m512d __W, double ExtraAddOp){
-// CHECK-LABEL: @test_mm512_reduce_add_pd(
+// CHECK-LABEL: test_mm512_reduce_add_pd
 // CHECK-NOT: reassoc
-// CHECK:    call reassoc double @llvm.vector.reduce.fadd.v8f64(double -0.000000e+00, <8 x double> %{{.*}})
+// CHECK:    call reassoc {{.*}}double @llvm.vector.reduce.fadd.v8f64(double -0.000000e+00, <8 x double> %{{.*}})
 // CHECK-NOT: reassoc
   return _mm512_reduce_add_pd(__W) + ExtraAddOp;
 }
 
 double test_mm512_reduce_mul_pd(__m512d __W, double ExtraMulOp){
-// CHECK-LABEL: @test_mm512_reduce_mul_pd(
+// CHECK-LABEL: test_mm512_reduce_mul_pd
 // CHECK-NOT: reassoc
-// CHECK:    call reassoc double @llvm.vector.reduce.fmul.v8f64(double 1.000000e+00, <8 x double> %{{.*}})
+// CHECK:    call reassoc {{.*}}double @llvm.vector.reduce.fmul.v8f64(double 1.000000e+00, <8 x double> %{{.*}})
 // CHECK-NOT: reassoc
   return _mm512_reduce_mul_pd(__W) * ExtraMulOp;
 }
 
 float test_mm512_reduce_add_ps(__m512 __W){
-// CHECK-LABEL: @test_mm512_reduce_add_ps(
-// CHECK:    call reassoc float @llvm.vector.reduce.fadd.v16f32(float -0.000000e+00, <16 x float> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_add_ps
+// CHECK:    call reassoc {{.*}}float @llvm.vector.reduce.fadd.v16f32(float -0.000000e+00, <16 x float> %{{.*}})
   return _mm512_reduce_add_ps(__W);
 }
 
 float test_mm512_reduce_mul_ps(__m512 __W){
-// CHECK-LABEL: @test_mm512_reduce_mul_ps(
-// CHECK:    call reassoc float @llvm.vector.reduce.fmul.v16f32(float 1.000000e+00, <16 x float> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_mul_ps
+// CHECK:    call reassoc {{.*}}float @llvm.vector.reduce.fmul.v16f32(float 1.000000e+00, <16 x float> %{{.*}})
   return _mm512_reduce_mul_ps(__W);
 }
 
 double test_mm512_mask_reduce_add_pd(__mmask8 __M, __m512d __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_add_pd(
+// CHECK-LABEL: test_mm512_mask_reduce_add_pd
 // CHECK:    select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
-// CHECK:    call reassoc double @llvm.vector.reduce.fadd.v8f64(double -0.000000e+00, <8 x double> %{{.*}})
+// CHECK:    call reassoc {{.*}}double @llvm.vector.reduce.fadd.v8f64(double -0.000000e+00, <8 x double> %{{.*}})
   return _mm512_mask_reduce_add_pd(__M, __W);
 }
 
 double test_mm512_mask_reduce_mul_pd(__mmask8 __M, __m512d __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_mul_pd(
+// CHECK-LABEL: test_mm512_mask_reduce_mul_pd
 // CHECK:    select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
-// CHECK:    call reassoc double @llvm.vector.reduce.fmul.v8f64(double 1.000000e+00, <8 x double> %{{.*}})
+// CHECK:    call reassoc {{.*}}double @llvm.vector.reduce.fmul.v8f64(double 1.000000e+00, <8 x double> %{{.*}})
   return _mm512_mask_reduce_mul_pd(__M, __W);
 }
 
 float test_mm512_mask_reduce_add_ps(__mmask16 __M, __m512 __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_add_ps(
+// CHECK-LABEL: test_mm512_mask_reduce_add_ps
 // CHECK:    select <16 x i1> %{{.*}}, <16 x float> {{.*}}, <16 x float> {{.*}}
-// CHECK:    call reassoc float @llvm.vector.reduce.fadd.v16f32(float -0.000000e+00, <16 x float> %{{.*}})
+// CHECK:    call reassoc {{.*}}float @llvm.vector.reduce.fadd.v16f32(float -0.000000e+00, <16 x float> %{{.*}})
   return _mm512_mask_reduce_add_ps(__M, __W);
 }
 
 float test_mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_mul_ps(
+// CHECK-LABEL: test_mm512_mask_reduce_mul_ps
 // CHECK:    select <16 x i1> %{{.*}}, <16 x float> {{.*}}, <16 x float> %{{.*}}
-// CHECK:    call reassoc float @llvm.vector.reduce.fmul.v16f32(float 1.000000e+00, <16 x float> %{{.*}})
+// CHECK:    call reassoc {{.*}}float @llvm.vector.reduce.fmul.v16f32(float 1.000000e+00, <16 x float> %{{.*}})
   return _mm512_mask_reduce_mul_ps(__M, __W);
 }
diff --git a/clang/test/CodeGen/X86/avx512-reduceMinMaxIntrin.c b/clang/test/CodeGen/X86/avx512-reduceMinMaxIntrin.c
index 3e33ec566d281..0110079a007d5 100644
--- a/clang/test/CodeGen/X86/avx512-reduceMinMaxIntrin.c
+++ b/clang/test/CodeGen/X86/avx512-reduceMinMaxIntrin.c
@@ -1,164 +1,175 @@
-// RUN: %clang_cc1 -ffreestanding %s -O0 -triple=x86_64-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -O0 -triple=x86_64-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -O0 -triple=i386-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -O0 -triple=x86_64-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -O0 -triple=i386-apple-darwin -target-cpu skylake-avx512 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 long long test_mm512_reduce_max_epi64(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_max_epi64(
-// CHECK:    call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_max_epi64
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> %{{.*}})
   return _mm512_reduce_max_epi64(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_max_epi64((__m512i)(__v8di){-4, -3, -2, -1, 0, 1, 2, 3}) == 3);
 
 unsigned long long test_mm512_reduce_max_epu64(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_max_epu64(
-// CHECK:    call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_max_epu64
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> %{{.*}})
   return _mm512_reduce_max_epu64(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_max_epu64((__m512i)(__v8du){0, 1, 2, 3, 4, 5, 6, 7}) == 7);
 
 double test_mm512_reduce_max_pd(__m512d __W, double ExtraAddOp){
-// CHECK-LABEL: @test_mm512_reduce_max_pd(
+// CHECK-LABEL: test_mm512_reduce_max_pd
 // CHECK-NOT: nnan
-// CHECK:    call nnan double @llvm.vector.reduce.fmax.v8f64(<8 x double> %{{.*}})
+// CHECK:    call nnan {{.*}}double @llvm.vector.reduce.fmax.v8f64(<8 x double> %{{.*}})
 // CHECK-NOT: nnan
   return _mm512_reduce_max_pd(__W) + ExtraAddOp;
 }
 
 long long test_mm512_reduce_min_epi64(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_min_epi64(
-// CHECK:    call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_min_epi64
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> %{{.*}})
   return _mm512_reduce_min_epi64(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_min_epi64((__m512i)(__v8di){-4, -3, -2, -1, 0, 1, 2, 3}) == -4);
 
 unsigned long long test_mm512_reduce_min_epu64(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_min_epu64(
-// CHECK:    call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_min_epu64
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> %{{.*}})
   return _mm512_reduce_min_epu64(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_min_epu64((__m512i)(__v8du){0, 1, 2, 3, 4, 5, 6, 7}) == 0);
 
 double test_mm512_reduce_min_pd(__m512d __W, double ExtraMulOp){
-// CHECK-LABEL: @test_mm512_reduce_min_pd(
+// CHECK-LABEL: test_mm512_reduce_min_pd
 // CHECK-NOT: nnan
-// CHECK:    call nnan double @llvm.vector.reduce.fmin.v8f64(<8 x double> %{{.*}})
+// CHECK:    call nnan {{.*}}double @llvm.vector.reduce.fmin.v8f64(<8 x double> %{{.*}})
 // CHECK-NOT: nnan
   return _mm512_reduce_min_pd(__W) * ExtraMulOp;
 }
 
 long long test_mm512_mask_reduce_max_epi64(__mmask8 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_max_epi64(
+// CHECK-LABEL: test_mm512_mask_reduce_max_epi64
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
-// CHECK:    call i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> %{{.*}})
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.smax.v8i64(<8 x i64> %{{.*}})
   return _mm512_mask_reduce_max_epi64(__M, __W);
 }
 
 unsigned long test_mm512_mask_reduce_max_epu64(__mmask8 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_max_epu64(
+// CHECK-LABEL: test_mm512_mask_reduce_max_epu64
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
-// CHECK:    call i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> %{{.*}})
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.umax.v8i64(<8 x i64> %{{.*}})
   return _mm512_mask_reduce_max_epu64(__M, __W);
 }
 
 double test_mm512_mask_reduce_max_pd(__mmask8 __M, __m512d __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_max_pd(
+// CHECK-LABEL: test_mm512_mask_reduce_max_pd
 // CHECK:    select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
-// CHECK:    call nnan double @llvm.vector.reduce.fmax.v8f64(<8 x double> %{{.*}})
+// CHECK:    call nnan {{.*}}double @llvm.vector.reduce.fmax.v8f64(<8 x double> %{{.*}})
   return _mm512_mask_reduce_max_pd(__M, __W);
 }
 
 long long test_mm512_mask_reduce_min_epi64(__mmask8 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_min_epi64(
+// CHECK-LABEL: test_mm512_mask_reduce_min_epi64
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
-// CHECK:    call i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> %{{.*}})
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.smin.v8i64(<8 x i64> %{{.*}})
   return _mm512_mask_reduce_min_epi64(__M, __W);
 }
 
 unsigned long long test_mm512_mask_reduce_min_epu64(__mmask8 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_min_epu64(
+// CHECK-LABEL: test_mm512_mask_reduce_min_epu64
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
-// CHECK:    call i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> %{{.*}})
+// CHECK:    call {{.*}}i64 @llvm.vector.reduce.umin.v8i64(<8 x i64> %{{.*}})
   return _mm512_mask_reduce_min_epu64(__M, __W);
 }
 
 double test_mm512_mask_reduce_min_pd(__mmask8 __M, __m512d __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_min_pd(
+// CHECK-LABEL: test_mm512_mask_reduce_min_pd
 // CHECK:    select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
-// CHECK:    call nnan double @llvm.vector.reduce.fmin.v8f64(<8 x double> %{{.*}})
+// CHECK:    call nnan {{.*}}double @llvm.vector.reduce.fmin.v8f64(<8 x double> %{{.*}})
   return _mm512_mask_reduce_min_pd(__M, __W);
 }
 
 int test_mm512_reduce_max_epi32(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_max_epi32(
-// CHECK:    call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_max_epi32
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> %{{.*}})
   return _mm512_reduce_max_epi32(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_max_epi32((__m512i)(__v16si){-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7}) == 7);
 
 unsigned int test_mm512_reduce_max_epu32(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_max_epu32(
-// CHECK:    call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_max_epu32
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> %{{.*}})
   return _mm512_reduce_max_epu32(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_max_epu32((__m512i)(__v16su){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}) == 15);
 
 float test_mm512_reduce_max_ps(__m512 __W){
-// CHECK-LABEL: @test_mm512_reduce_max_ps(
-// CHECK:    call nnan float @llvm.vector.reduce.fmax.v16f32(<16 x float> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_max_ps
+// CHECK:    call nnan {{.*}}float @llvm.vector.reduce.fmax.v16f32(<16 x float> %{{.*}})
   return _mm512_reduce_max_ps(__W);
 }
 
 int test_mm512_reduce_min_epi32(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_min_epi32(
-// CHECK:    call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_min_epi32
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> %{{.*}})
   return _mm512_reduce_min_epi32(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_min_epi32((__m512i)(__v16si){-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7}) == -8);
 
 unsigned int test_mm512_reduce_min_epu32(__m512i __W){
-// CHECK-LABEL: @test_mm512_reduce_min_epu32(
-// CHECK:    call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_min_epu32
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> %{{.*}})
   return _mm512_reduce_min_epu32(__W);
 }
+TEST_CONSTEXPR(_mm512_reduce_min_epu32((__m512i)(__v16su){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}) == 0);
 
 float test_mm512_reduce_min_ps(__m512 __W){
-// CHECK-LABEL: @test_mm512_reduce_min_ps(
-// CHECK:    call nnan float @llvm.vector.reduce.fmin.v16f32(<16 x float> %{{.*}})
+// CHECK-LABEL: test_mm512_reduce_min_ps
+// CHECK:    call nnan {{.*}}float @llvm.vector.reduce.fmin.v16f32(<16 x float> %{{.*}})
   return _mm512_reduce_min_ps(__W);
 }
 
 int test_mm512_mask_reduce_max_epi32(__mmask16 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_max_epi32(
+// CHECK-LABEL: test_mm512_mask_reduce_max_epi32
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-// CHECK:    call i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> %{{.*}})
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.smax.v16i32(<16 x i32> %{{.*}})
   return _mm512_mask_reduce_max_epi32(__M, __W);
 }
 
 unsigned int test_mm512_mask_reduce_max_epu32(__mmask16 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_max_epu32(
+// CHECK-LABEL: test_mm512_mask_reduce_max_epu32
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-// CHECK:    call i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> %{{.*}})
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.umax.v16i32(<16 x i32> %{{.*}})
   return _mm512_mask_reduce_max_epu32(__M, __W);
 }
 
 float test_mm512_mask_reduce_max_ps(__mmask16 __M, __m512 __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_max_ps(
+// CHECK-LABEL: test_mm512_mask_reduce_max_ps
 // CHECK:    select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
-// CHECK:    call nnan float @llvm.vector.reduce.fmax.v16f32(<16 x float> %{{.*}})
+// CHECK:    call nnan {{.*}}float @llvm.vector.reduce.fmax.v16f32(<16 x float> %{{.*}})
   return _mm512_mask_reduce_max_ps(__M, __W);
 }
 
 int test_mm512_mask_reduce_min_epi32(__mmask16 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_min_epi32(
+// CHECK-LABEL: test_mm512_mask_reduce_min_epi32
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-// CHECK:    call i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> %{{.*}})
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.smin.v16i32(<16 x i32> %{{.*}})
   return _mm512_mask_reduce_min_epi32(__M, __W);
 }
 
 unsigned int test_mm512_mask_reduce_min_epu32(__mmask16 __M, __m512i __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_min_epu32(
+// CHECK-LABEL: test_mm512_mask_reduce_min_epu32
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-// CHECK:    call i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> %{{.*}})
+// CHECK:    call {{.*}}i32 @llvm.vector.reduce.umin.v16i32(<16 x i32> %{{.*}})
   return _mm512_mask_reduce_min_epu32(__M, __W);
 }
 
 float test_mm512_mask_reduce_min_ps(__mmask16 __M, __m512 __W){
-// CHECK-LABEL: @test_mm512_mask_reduce_min_ps(
+// CHECK-LABEL: test_mm512_mask_reduce_min_ps
 // CHECK:    select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
-// CHECK:    call nnan float @llvm.vector.reduce.fmin.v16f32(<16 x float> %{{.*}})
+// CHECK:    call nnan {{.*}}float @llvm.vector.reduce.fmin.v16f32(<16 x float> %{{.*}})
   return _mm512_mask_reduce_min_ps(__M, __W);
 }
-
diff --git a/clang/test/CodeGen/X86/avx512bf16-builtins.c b/clang/test/CodeGen/X86/avx512bf16-builtins.c
index 8eb93e6889bea..52c20aa15a568 100644
--- a/clang/test/CodeGen/X86/avx512bf16-builtins.c
+++ b/clang/test/CodeGen/X86/avx512bf16-builtins.c
@@ -1,107 +1,96 @@
-//  RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin \
-//  RUN:            -target-feature +avx512bf16 -emit-llvm -o - -Wall -Werror \
-//  RUN:            | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bf16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512bf16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bf16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512bf16 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 float test_mm_cvtsbh_ss(__bf16 A) {
-  // CHECK-LABEL: @test_mm_cvtsbh_ss
+  // CHECK-LABEL: test_mm_cvtsbh_ss
   // CHECK: fpext bfloat %{{.*}} to float
   // CHECK: ret float %{{.*}}
   return _mm_cvtsbh_ss(A);
 }
 
 __m512bh test_mm512_cvtne2ps_pbh(__m512 A, __m512 B) {
-  // CHECK-LABEL: @test_mm512_cvtne2ps_pbh
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512
-  // CHECK: ret <32 x bfloat> %{{.*}}
+  // CHECK-LABEL: test_mm512_cvtne2ps_pbh
+  // CHECK: call {{.*}}<32 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float> %{{.*}}, <16 x float> %{{.*}})
   return _mm512_cvtne2ps_pbh(A, B);
 }
 
 __m512bh test_mm512_maskz_cvtne2ps_pbh(__m512 A, __m512 B, __mmask32 U) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtne2ps_pbh
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512
+  // CHECK-LABEL: test_mm512_maskz_cvtne2ps_pbh
+  // CHECK: call {{.*}}<32 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float> %{{.*}}, <16 x float> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}}
-  // CHECK: ret <32 x bfloat> %{{.*}}
   return _mm512_maskz_cvtne2ps_pbh(U, A, B);
 }
 
 __m512bh test_mm512_mask_cvtne2ps_pbh(__m512bh C, __mmask32 U, __m512 A, __m512 B) {
-  // CHECK-LABEL: @test_mm512_mask_cvtne2ps_pbh
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512
+  // CHECK-LABEL: test_mm512_mask_cvtne2ps_pbh
+  // CHECK: call {{.*}}<32 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float> %{{.*}}, <16 x float> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}}
-  // CHECK: ret <32 x bfloat> %{{.*}}
   return _mm512_mask_cvtne2ps_pbh(C, U, A, B);
 }
 
 __m256bh test_mm512_cvtneps_pbh(__m512 A) {
-  // CHECK-LABEL: @test_mm512_cvtneps_pbh
-  // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.512
-  // CHECK: ret <16 x bfloat> %{{.*}}
+  // CHECK-LABEL: test_mm512_cvtneps_pbh
+  // CHECK: call {{.*}}<16 x bfloat> @llvm.x86.avx512bf16.cvtneps2bf16.512(<16 x float> %{{.*}})
   return _mm512_cvtneps_pbh(A);
 }
 
 __m256bh test_mm512_mask_cvtneps_pbh(__m256bh C, __mmask16 U, __m512 A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtneps_pbh
-  // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.512
+  // CHECK-LABEL: test_mm512_mask_cvtneps_pbh
+  // CHECK: call {{.*}}<16 x bfloat> @llvm.x86.avx512bf16.cvtneps2bf16.512(<16 x float> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}}
-  // CHECK: ret <16 x bfloat> %{{.*}}
   return _mm512_mask_cvtneps_pbh(C, U, A);
 }
 
 __m256bh test_mm512_maskz_cvtneps_pbh(__m512 A, __mmask16 U) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtneps_pbh
-  // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.512
+  // CHECK-LABEL: test_mm512_maskz_cvtneps_pbh
+  // CHECK: call {{.*}}<16 x bfloat> @llvm.x86.avx512bf16.cvtneps2bf16.512(<16 x float> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}}
-  // CHECK: ret <16 x bfloat> %{{.*}}
   return _mm512_maskz_cvtneps_pbh(U, A);
 }
 
 __m512 test_mm512_dpbf16_ps(__m512 D, __m512bh A, __m512bh B) {
-  // CHECK-LABEL: @test_mm512_dpbf16_ps
-  // CHECK: @llvm.x86.avx512bf16.dpbf16ps.512
-  // CHECK: ret <16 x float> %{{.*}}
+  // CHECK-LABEL: test_mm512_dpbf16_ps
+  // CHECK: call {{.*}}<16 x float> @llvm.x86.avx512bf16.dpbf16ps.512(<16 x float> %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
   return _mm512_dpbf16_ps(D, A, B);
 }
 
 __m512 test_mm512_maskz_dpbf16_ps(__m512 D, __m512bh A, __m512bh B, __mmask16 U) {
-  // CHECK-LABEL: @test_mm512_maskz_dpbf16_ps
-  // CHECK: @llvm.x86.avx512bf16.dpbf16ps.512
+  // CHECK-LABEL: test_mm512_maskz_dpbf16_ps
+  // CHECK: call {{.*}}<16 x float> @llvm.x86.avx512bf16.dpbf16ps.512(<16 x float> %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
-  // CHECK: ret <16 x float> %{{.*}}
   return _mm512_maskz_dpbf16_ps(U, D, A, B);
 }
 
 __m512 test_mm512_mask_dpbf16_ps(__m512 D, __m512bh A, __m512bh B, __mmask16 U) {
-  // CHECK-LABEL: @test_mm512_mask_dpbf16_ps
-  // CHECK: @llvm.x86.avx512bf16.dpbf16ps.512
+  // CHECK-LABEL: test_mm512_mask_dpbf16_ps
+  // CHECK: call {{.*}}<16 x float> @llvm.x86.avx512bf16.dpbf16ps.512(<16 x float> %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
-  // CHECK: ret <16 x float> %{{.*}}
   return _mm512_mask_dpbf16_ps(D, U, A, B);
 }
 
 __m512 test_mm512_cvtpbh_ps(__m256bh A) {
-  // CHECK-LABEL: @test_mm512_cvtpbh_ps
+  // CHECK-LABEL: test_mm512_cvtpbh_ps
   // CHECK: sext <16 x i16> %{{.*}} to <16 x i32>
-  // CHECK: @llvm.x86.avx512.pslli.d.512
-  // CHECK: ret <16 x float> %{{.*}}
+  // CHECK: call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %{{.*}}, i32 %{{.*}})
   return _mm512_cvtpbh_ps(A);
 }
 
 __m512 test_mm512_maskz_cvtpbh_ps(__mmask16 M, __m256bh A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtpbh_ps
+  // CHECK-LABEL: test_mm512_maskz_cvtpbh_ps
   // CHECK: sext <16 x i16> %{{.*}} to <16 x i32>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-  // CHECK: @llvm.x86.avx512.pslli.d.512
-  // CHECK: ret <16 x float> %{{.*}}
+  // CHECK: call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %{{.*}}, i32 %{{.*}})
   return _mm512_maskz_cvtpbh_ps(M, A);
 }
 
 __m512 test_mm512_mask_cvtpbh_ps(__m512 S, __mmask16 M, __m256bh A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtpbh_ps
+  // CHECK-LABEL: test_mm512_mask_cvtpbh_ps
   // CHECK: sext <16 x i16> %{{.*}} to <16 x i32>
-  // CHECK: @llvm.x86.avx512.pslli.d.512
+  // CHECK: call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %{{.*}}, i32 %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
-  // CHECK: ret <16 x float> %{{.*}}
   return _mm512_mask_cvtpbh_ps(S, M, A);
 }
diff --git a/clang/test/CodeGen/X86/avx512bitalg-builtins.c b/clang/test/CodeGen/X86/avx512bitalg-builtins.c
index c80fb5e708c55..8092f2d30214a 100644
--- a/clang/test/CodeGen/X86/avx512bitalg-builtins.c
+++ b/clang/test/CodeGen/X86/avx512bitalg-builtins.c
@@ -1,54 +1,66 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bitalg -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bitalg -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512bitalg -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bitalg -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512bitalg -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __m512i test_mm512_popcnt_epi16(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_popcnt_epi16
+  // CHECK-LABEL: test_mm512_popcnt_epi16
   // CHECK: @llvm.ctpop.v32i16
   return _mm512_popcnt_epi16(__A);
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_popcnt_epi16((__m512i)(__v32hi){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025, +5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), 2, 15, 14, 1, 0, 8, 1, 9, 2, 2, 4, 2, 6, 2, 9, 2, 2, 15, 14, 1, 0, 8, 1, 9, 2, 2, 4, 2, 6, 2, 9, 2));
 
 __m512i test_mm512_mask_popcnt_epi16(__m512i __A, __mmask32 __U, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_popcnt_epi16
+  // CHECK-LABEL: test_mm512_mask_popcnt_epi16
   // CHECK: @llvm.ctpop.v32i16
   // CHECK: select <32 x i1> %{{[0-9]+}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_popcnt_epi16(__A, __U, __B);
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_mask_popcnt_epi16(_mm512_set1_epi16(-1), 0xF0F0F0F0, (__m512i)(__v32hi){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025, +5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), -1, -1, -1, -1, 0, 8, 1, 9, -1, -1, -1, -1, 6, 2, 9, 2, -1, -1, -1, -1, 0, 8, 1, 9, -1, -1, -1, -1, 6, 2, 9, 2));
+
 __m512i test_mm512_maskz_popcnt_epi16(__mmask32 __U, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_popcnt_epi16
+  // CHECK-LABEL: test_mm512_maskz_popcnt_epi16
   // CHECK: @llvm.ctpop.v32i16
   // CHECK: select <32 x i1> %{{[0-9]+}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_popcnt_epi16(__U, __B);
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_maskz_popcnt_epi16(0x0F0F0F0F, (__m512i)(__v32hi){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025, +5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), 2, 15, 14, 1, 0, 0, 0, 0, 2, 2, 4, 2, 0, 0, 0, 0, 2, 15, 14, 1, 0, 0, 0, 0, 2, 2, 4, 2, 0, 0, 0, 0));
 
 __m512i test_mm512_popcnt_epi8(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_popcnt_epi8
+  // CHECK-LABEL: test_mm512_popcnt_epi8
   // CHECK: @llvm.ctpop.v64i8
   return _mm512_popcnt_epi8(__A);
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_popcnt_epi8((__m512i)(__v64qi){+5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73}), 2, 7, 6, 1, 0, 4, 1, 4, 2, 2, 4, 2, 6, 2, 4, 3, 2, 7, 6, 1, 0, 4, 1, 4, 2, 2, 4, 2, 6, 2, 4, 3, 2, 7, 6, 1, 0, 4, 1, 4, 2, 2, 4, 2, 6, 2, 4, 3, 2, 7, 6, 1, 0, 4, 1, 4, 2, 2, 4, 2, 6, 2, 4, 3));
 
 __m512i test_mm512_mask_popcnt_epi8(__m512i __A, __mmask64 __U, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_popcnt_epi8
+  // CHECK-LABEL: test_mm512_mask_popcnt_epi8
   // CHECK: @llvm.ctpop.v64i8
   // CHECK: select <64 x i1> %{{[0-9]+}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_popcnt_epi8(__A, __U, __B);
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_mask_popcnt_epi8(_mm512_set1_epi8(-1), 0xF0F0F0F00F0F0F0FULL, (__m512i)(__v64qi){+5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73}), 2, 7, 6, 1, -1, -1, -1, -1, 2, 2, 4, 2, -1, -1, -1, -1, 2, 7, 6, 1, -1, -1, -1, -1, 2, 2, 4, 2, -1, -1, -1, -1, -1, -1, -1, -1, 0, 4, 1, 4, -1, -1, -1, -1, 6, 2, 4, 3, -1, -1, -1, -1, 0, 4, 1, 4, -1, -1, -1, -1, 6, 2, 4, 3));
+
 __m512i test_mm512_maskz_popcnt_epi8(__mmask64 __U, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_popcnt_epi8
+  // CHECK-LABEL: test_mm512_maskz_popcnt_epi8
   // CHECK: @llvm.ctpop.v64i8
   // CHECK: select <64 x i1> %{{[0-9]+}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_popcnt_epi8(__U, __B);
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_maskz_popcnt_epi8(0x0F0F0F0FF0F0F0F0ULL, (__m512i)(__v64qi){+5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73}), 0, 0, 0, 0, 0, 4, 1, 4, 0, 0, 0, 0, 6, 2, 4, 3, 0, 0, 0, 0, 0, 4, 1, 4, 0, 0, 0, 0, 6, 2, 4, 3, 2, 7, 6, 1, 0, 0, 0, 0, 2, 2, 4, 2, 0, 0, 0, 0, 2, 7, 6, 1, 0, 0, 0, 0, 2, 2, 4, 2, 0, 0, 0, 0));
 
 __mmask64 test_mm512_mask_bitshuffle_epi64_mask(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_bitshuffle_epi64_mask
+  // CHECK-LABEL: test_mm512_mask_bitshuffle_epi64_mask
   // CHECK: @llvm.x86.avx512.vpshufbitqmb.512
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return _mm512_mask_bitshuffle_epi64_mask(__U, __A, __B);
 }
 
 __mmask64 test_mm512_bitshuffle_epi64_mask(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_bitshuffle_epi64_mask
+  // CHECK-LABEL: test_mm512_bitshuffle_epi64_mask
   // CHECK: @llvm.x86.avx512.vpshufbitqmb.512
   return _mm512_bitshuffle_epi64_mask(__A, __B);
 }
diff --git a/clang/test/CodeGen/X86/avx512bw-builtins.c b/clang/test/CodeGen/X86/avx512bw-builtins.c
index 1d18ca8548a3a..02cedc3c73fb7 100644
--- a/clang/test/CodeGen/X86/avx512bw-builtins.c
+++ b/clang/test/CodeGen/X86/avx512bw-builtins.c
@@ -1,25 +1,28 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __mmask32 test_knot_mask32(__mmask32 a) {
-  // CHECK-LABEL: @test_knot_mask32
+  // CHECK-LABEL: test_knot_mask32
   // CHECK: [[IN:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[NOT:%.*]] = xor <32 x i1> [[IN]], splat (i1 true)
   return _knot_mask32(a);
 }
 
 __mmask64 test_knot_mask64(__mmask64 a) {
-  // CHECK-LABEL: @test_knot_mask64
+  // CHECK-LABEL: test_knot_mask64
   // CHECK: [[IN:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[NOT:%.*]] = xor <64 x i1> [[IN]], splat (i1 true)
   return _knot_mask64(a);
 }
 
 __mmask32 test_kand_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kand_mask32
+  // CHECK-LABEL: test_kand_mask32
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RES:%.*]] = and <32 x i1> [[LHS]], [[RHS]]
@@ -29,7 +32,7 @@ __mmask32 test_kand_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, _
 }
 
 __mmask64 test_kand_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kand_mask64
+  // CHECK-LABEL: test_kand_mask64
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RES:%.*]] = and <64 x i1> [[LHS]], [[RHS]]
@@ -39,7 +42,7 @@ __mmask64 test_kand_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, _
 }
 
 __mmask32 test_kandn_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kandn_mask32
+  // CHECK-LABEL: test_kandn_mask32
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[NOT:%.*]] = xor <32 x i1> [[LHS]], splat (i1 true)
@@ -50,7 +53,7 @@ __mmask32 test_kandn_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D,
 }
 
 __mmask64 test_kandn_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kandn_mask64
+  // CHECK-LABEL: test_kandn_mask64
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[NOT:%.*]] = xor <64 x i1> [[LHS]], splat (i1 true)
@@ -61,7 +64,7 @@ __mmask64 test_kandn_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D,
 }
 
 __mmask32 test_kor_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kor_mask32
+  // CHECK-LABEL: test_kor_mask32
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RES:%.*]] = or <32 x i1> [[LHS]], [[RHS]]
@@ -71,7 +74,7 @@ __mmask32 test_kor_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __
 }
 
 __mmask64 test_kor_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kor_mask64
+  // CHECK-LABEL: test_kor_mask64
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RES:%.*]] = or <64 x i1> [[LHS]], [[RHS]]
@@ -81,7 +84,7 @@ __mmask64 test_kor_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __
 }
 
 __mmask32 test_kxnor_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kxnor_mask32
+  // CHECK-LABEL: test_kxnor_mask32
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[NOT:%.*]] = xor <32 x i1> [[LHS]], splat (i1 true)
@@ -92,7 +95,7 @@ __mmask32 test_kxnor_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D,
 }
 
 __mmask64 test_kxnor_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kxnor_mask64
+  // CHECK-LABEL: test_kxnor_mask64
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[NOT:%.*]] = xor <64 x i1> [[LHS]], splat (i1 true)
@@ -103,7 +106,7 @@ __mmask64 test_kxnor_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D,
 }
 
 __mmask32 test_kxor_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kxor_mask32
+  // CHECK-LABEL: test_kxor_mask32
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RES:%.*]] = xor <32 x i1> [[LHS]], [[RHS]]
@@ -113,7 +116,7 @@ __mmask32 test_kxor_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, _
 }
 
 __mmask64 test_kxor_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kxor_mask64
+  // CHECK-LABEL: test_kxor_mask64
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RES:%.*]] = xor <64 x i1> [[LHS]], [[RHS]]
@@ -123,7 +126,7 @@ __mmask64 test_kxor_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, _
 }
 
 unsigned char test_kortestz_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_kortestz_mask32_u8
+  // CHECK-LABEL: test_kortestz_mask32_u8
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[OR:%.*]] = or <32 x i1> [[LHS]], [[RHS]]
@@ -136,7 +139,7 @@ unsigned char test_kortestz_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m
 }
 
 unsigned char test_kortestc_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_kortestc_mask32_u8
+  // CHECK-LABEL: test_kortestc_mask32_u8
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[OR:%.*]] = or <32 x i1> [[LHS]], [[RHS]]
@@ -149,7 +152,7 @@ unsigned char test_kortestc_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m
 }
 
 unsigned char test_kortest_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) {
-  // CHECK-LABEL: @test_kortest_mask32_u8
+  // CHECK-LABEL: test_kortest_mask32_u8
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[OR:%.*]] = or <32 x i1> [[LHS]], [[RHS]]
@@ -169,7 +172,7 @@ unsigned char test_kortest_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m5
 }
 
 unsigned char test_kortestz_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_kortestz_mask64_u8
+  // CHECK-LABEL: test_kortestz_mask64_u8
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[OR:%.*]] = or <64 x i1> [[LHS]], [[RHS]]
@@ -182,7 +185,7 @@ unsigned char test_kortestz_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m
 }
 
 unsigned char test_kortestc_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_kortestc_mask64_u8
+  // CHECK-LABEL: test_kortestc_mask64_u8
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[OR:%.*]] = or <64 x i1> [[LHS]], [[RHS]]
@@ -195,7 +198,7 @@ unsigned char test_kortestc_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m
 }
 
 unsigned char test_kortest_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) {
-  // CHECK-LABEL: @test_kortest_mask64_u8
+  // CHECK-LABEL: test_kortest_mask64_u8
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[OR:%.*]] = or <64 x i1> [[LHS]], [[RHS]]
@@ -215,7 +218,7 @@ unsigned char test_kortest_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m5
 }
 
 unsigned char test_ktestz_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_ktestz_mask32_u8
+  // CHECK-LABEL: test_ktestz_mask32_u8
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestz.d(<32 x i1> [[LHS]], <32 x i1> [[RHS]])
@@ -225,7 +228,7 @@ unsigned char test_ktestz_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m51
 }
 
 unsigned char test_ktestc_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_ktestc_mask32_u8
+  // CHECK-LABEL: test_ktestc_mask32_u8
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.d(<32 x i1> [[LHS]], <32 x i1> [[RHS]])
@@ -235,7 +238,7 @@ unsigned char test_ktestc_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m51
 }
 
 unsigned char test_ktest_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) {
-  // CHECK-LABEL: @test_ktest_mask32_u8
+  // CHECK-LABEL: test_ktest_mask32_u8
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.d(<32 x i1> [[LHS]], <32 x i1> [[RHS]])
@@ -249,7 +252,7 @@ unsigned char test_ktest_mask32_u8(__m512i __A, __m512i __B, __m512i __C, __m512
 }
 
 unsigned char test_ktestz_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_ktestz_mask64_u8
+  // CHECK-LABEL: test_ktestz_mask64_u8
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestz.q(<64 x i1> [[LHS]], <64 x i1> [[RHS]])
@@ -259,7 +262,7 @@ unsigned char test_ktestz_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m51
 }
 
 unsigned char test_ktestc_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_ktestc_mask64_u8
+  // CHECK-LABEL: test_ktestc_mask64_u8
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.q(<64 x i1> [[LHS]], <64 x i1> [[RHS]])
@@ -269,7 +272,7 @@ unsigned char test_ktestc_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m51
 }
 
 unsigned char test_ktest_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) {
-  // CHECK-LABEL: @test_ktest_mask64_u8
+  // CHECK-LABEL: test_ktest_mask64_u8
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.q(<64 x i1> [[LHS]], <64 x i1> [[RHS]])
@@ -283,7 +286,7 @@ unsigned char test_ktest_mask64_u8(__m512i __A, __m512i __B, __m512i __C, __m512
 }
 
 __mmask32 test_kadd_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kadd_mask32
+  // CHECK-LABEL: test_kadd_mask32
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RES:%.*]] = call <32 x i1> @llvm.x86.avx512.kadd.d(<32 x i1> [[LHS]], <32 x i1> [[RHS]])
@@ -293,7 +296,7 @@ __mmask32 test_kadd_mask32(__m512i __A, __m512i __B, __m512i __C, __m512i __D, _
 }
 
 __mmask64 test_kadd_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kadd_mask64
+  // CHECK-LABEL: test_kadd_mask64
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RES:%.*]] = call <64 x i1> @llvm.x86.avx512.kadd.q(<64 x i1> [[LHS]], <64 x i1> [[RHS]])
@@ -303,1592 +306,1625 @@ __mmask64 test_kadd_mask64(__m512i __A, __m512i __B, __m512i __C, __m512i __D, _
 }
 
 __mmask32 test_kshiftli_mask32(__m512i A, __m512i B, __m512i C, __m512i D) {
-  // CHECK-LABEL: @test_kshiftli_mask32
+  // CHECK-LABEL: test_kshiftli_mask32
   // CHECK: [[VAL:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RES:%.*]] = shufflevector <32 x i1> zeroinitializer, <32 x i1> [[VAL]], <32 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32>
   return _mm512_mask_cmpneq_epu16_mask(_kshiftli_mask32(_mm512_cmpneq_epu16_mask(A, B), 31), C, D);
 }
 
 __mmask32 test_kshiftri_mask32(__m512i A, __m512i B, __m512i C, __m512i D) {
-  // CHECK-LABEL: @test_kshiftri_mask32
+  // CHECK-LABEL: test_kshiftri_mask32
   // CHECK: [[VAL:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RES:%.*]] = shufflevector <32 x i1> [[VAL]], <32 x i1> zeroinitializer, <32 x i32> <i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62>
   return _mm512_mask_cmpneq_epu16_mask(_kshiftri_mask32(_mm512_cmpneq_epu16_mask(A, B), 31), C, D);
 }
 
 __mmask64 test_kshiftli_mask64(__m512i A, __m512i B, __m512i C, __m512i D) {
-  // CHECK-LABEL: @test_kshiftli_mask64
+  // CHECK-LABEL: test_kshiftli_mask64
   // CHECK: [[VAL:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RES:%.*]] = shufflevector <64 x i1> zeroinitializer, <64 x i1> [[VAL]], <64 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
   return _mm512_mask_cmpneq_epu8_mask(_kshiftli_mask64(_mm512_cmpneq_epu8_mask(A, B), 32), C, D);
 }
 
 __mmask64 test_kshiftri_mask64(__m512i A, __m512i B, __m512i C, __m512i D) {
-  // CHECK-LABEL: @test_kshiftri_mask64
+  // CHECK-LABEL: test_kshiftri_mask64
   // CHECK: [[VAL:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RES:%.*]] = shufflevector <64 x i1> [[VAL]], <64 x i1> zeroinitializer, <64 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 91, i32 92, i32 93, i32 94, i32 95>
   return _mm512_mask_cmpneq_epu8_mask(_kshiftri_mask64(_mm512_cmpneq_epu8_mask(A, B), 32), C, D);
 }
 
 unsigned int test_cvtmask32_u32(__m512i A, __m512i B) {
-  // CHECK-LABEL: @test_cvtmask32_u32
+  // CHECK-LABEL: test_cvtmask32_u32
   return _cvtmask32_u32(_mm512_cmpneq_epu16_mask(A, B));
 }
 
 unsigned long long test_cvtmask64_u64(__m512i A, __m512i B) {
-  // CHECK-LABEL: @test_cvtmask64_u64
+  // CHECK-LABEL: test_cvtmask64_u64
   return _cvtmask64_u64(_mm512_cmpneq_epu8_mask(A, B));
 }
 
 __mmask32 test_cvtu32_mask32(__m512i A, __m512i B, unsigned int C) {
-  // CHECK-LABEL: @test_cvtu32_mask32
+  // CHECK-LABEL: test_cvtu32_mask32
   return _mm512_mask_cmpneq_epu16_mask(_cvtu32_mask32(C), A, B);
 }
 
 __mmask64 test_cvtu64_mask64(__m512i A, __m512i B, unsigned long long C) {
-  // CHECK-LABEL: @test_cvtu64_mask64
+  // CHECK-LABEL: test_cvtu64_mask64
   return _mm512_mask_cmpneq_epu8_mask(_cvtu64_mask64(C), A, B);
 }
 
 __mmask32 test_load_mask32(__mmask32 *A, __m512i B, __m512i C) {
-  // CHECK-LABEL: @test_load_mask32
+  // CHECK-LABEL: test_load_mask32
   // CHECK: [[LOAD:%.*]] = load i32, ptr %{{.*}}
   return _mm512_mask_cmpneq_epu16_mask(_load_mask32(A), B, C);
 }
 
 __mmask64 test_load_mask64(__mmask64 *A, __m512i B, __m512i C) {
-  // CHECK-LABEL: @test_load_mask64
+  // CHECK-LABEL: test_load_mask64
   // CHECK: [[LOAD:%.*]] = load i64, ptr %{{.*}}
   return _mm512_mask_cmpneq_epu8_mask(_load_mask64(A), B, C);
 }
 
 void test_store_mask32(__mmask32 *A, __m512i B, __m512i C) {
-  // CHECK-LABEL: @test_store_mask32
+  // CHECK-LABEL: test_store_mask32
   // CHECK: store i32 %{{.*}}, ptr %{{.*}}
   _store_mask32(A, _mm512_cmpneq_epu16_mask(B, C));
 }
 
 void test_store_mask64(__mmask64 *A, __m512i B, __m512i C) {
-  // CHECK-LABEL: @test_store_mask64
+  // CHECK-LABEL: test_store_mask64
   // CHECK: store i64 %{{.*}}, ptr %{{.*}}
   _store_mask64(A, _mm512_cmpneq_epu8_mask(B, C));
 }
 
 __mmask64 test_mm512_cmpeq_epi8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpeq_epi8_mask
+  // CHECK-LABEL: test_mm512_cmpeq_epi8_mask
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmpeq_epi8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmpeq_epi8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpeq_epi8_mask
+  // CHECK-LABEL: test_mm512_mask_cmpeq_epi8_mask
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmpeq_epi8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmpeq_epi16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpeq_epi16_mask
+  // CHECK-LABEL: test_mm512_cmpeq_epi16_mask
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmpeq_epi16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmpeq_epi16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpeq_epi16_mask
+  // CHECK-LABEL: test_mm512_mask_cmpeq_epi16_mask
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmpeq_epi16_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmpgt_epi8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpgt_epi8_mask
+  // CHECK-LABEL: test_mm512_cmpgt_epi8_mask
   // CHECK: icmp sgt <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmpgt_epi8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmpgt_epi8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpgt_epi8_mask
+  // CHECK-LABEL: test_mm512_mask_cmpgt_epi8_mask
   // CHECK: icmp sgt <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmpgt_epi8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmpgt_epi16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpgt_epi16_mask
+  // CHECK-LABEL: test_mm512_cmpgt_epi16_mask
   // CHECK: icmp sgt <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmpgt_epi16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmpgt_epi16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpgt_epi16_mask
+  // CHECK-LABEL: test_mm512_mask_cmpgt_epi16_mask
   // CHECK: icmp sgt <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmpgt_epi16_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmpeq_epu8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpeq_epu8_mask
+  // CHECK-LABEL: test_mm512_cmpeq_epu8_mask
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmpeq_epu8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmpeq_epu8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpeq_epu8_mask
+  // CHECK-LABEL: test_mm512_mask_cmpeq_epu8_mask
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmpeq_epu8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmpeq_epu16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpeq_epu16_mask
+  // CHECK-LABEL: test_mm512_cmpeq_epu16_mask
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmpeq_epu16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmpeq_epu16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpeq_epu16_mask
+  // CHECK-LABEL: test_mm512_mask_cmpeq_epu16_mask
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmpeq_epu16_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmpgt_epu8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpgt_epu8_mask
+  // CHECK-LABEL: test_mm512_cmpgt_epu8_mask
   // CHECK: icmp ugt <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmpgt_epu8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmpgt_epu8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpgt_epu8_mask
+  // CHECK-LABEL: test_mm512_mask_cmpgt_epu8_mask
   // CHECK: icmp ugt <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmpgt_epu8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmpgt_epu16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpgt_epu16_mask
+  // CHECK-LABEL: test_mm512_cmpgt_epu16_mask
   // CHECK: icmp ugt <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmpgt_epu16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmpgt_epu16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpgt_epu16_mask
+  // CHECK-LABEL: test_mm512_mask_cmpgt_epu16_mask
   // CHECK: icmp ugt <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmpgt_epu16_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmpge_epi8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpge_epi8_mask
+  // CHECK-LABEL: test_mm512_cmpge_epi8_mask
   // CHECK: icmp sge <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmpge_epi8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmpge_epi8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpge_epi8_mask
+  // CHECK-LABEL: test_mm512_mask_cmpge_epi8_mask
   // CHECK: icmp sge <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmpge_epi8_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmpge_epu8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpge_epu8_mask
+  // CHECK-LABEL: test_mm512_cmpge_epu8_mask
   // CHECK: icmp uge <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmpge_epu8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmpge_epu8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpge_epu8_mask
+  // CHECK-LABEL: test_mm512_mask_cmpge_epu8_mask
   // CHECK: icmp uge <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmpge_epu8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmpge_epi16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpge_epi16_mask
+  // CHECK-LABEL: test_mm512_cmpge_epi16_mask
   // CHECK: icmp sge <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmpge_epi16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmpge_epi16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpge_epi16_mask
+  // CHECK-LABEL: test_mm512_mask_cmpge_epi16_mask
   // CHECK: icmp sge <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmpge_epi16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmpge_epu16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpge_epu16_mask
+  // CHECK-LABEL: test_mm512_cmpge_epu16_mask
   // CHECK: icmp uge <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmpge_epu16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmpge_epu16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpge_epu16_mask
+  // CHECK-LABEL: test_mm512_mask_cmpge_epu16_mask
   // CHECK: icmp uge <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmpge_epu16_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmple_epi8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmple_epi8_mask
+  // CHECK-LABEL: test_mm512_cmple_epi8_mask
   // CHECK: icmp sle <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmple_epi8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmple_epi8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmple_epi8_mask
+  // CHECK-LABEL: test_mm512_mask_cmple_epi8_mask
   // CHECK: icmp sle <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmple_epi8_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmple_epu8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmple_epu8_mask
+  // CHECK-LABEL: test_mm512_cmple_epu8_mask
   // CHECK: icmp ule <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmple_epu8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmple_epu8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmple_epu8_mask
+  // CHECK-LABEL: test_mm512_mask_cmple_epu8_mask
   // CHECK: icmp ule <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmple_epu8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmple_epi16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmple_epi16_mask
+  // CHECK-LABEL: test_mm512_cmple_epi16_mask
   // CHECK: icmp sle <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmple_epi16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmple_epi16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmple_epi16_mask
+  // CHECK-LABEL: test_mm512_mask_cmple_epi16_mask
   // CHECK: icmp sle <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmple_epi16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmple_epu16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmple_epu16_mask
+  // CHECK-LABEL: test_mm512_cmple_epu16_mask
   // CHECK: icmp ule <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmple_epu16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmple_epu16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmple_epu16_mask
+  // CHECK-LABEL: test_mm512_mask_cmple_epu16_mask
   // CHECK: icmp ule <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmple_epu16_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmplt_epi8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmplt_epi8_mask
+  // CHECK-LABEL: test_mm512_cmplt_epi8_mask
   // CHECK: icmp slt <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmplt_epi8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmplt_epi8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmplt_epi8_mask
+  // CHECK-LABEL: test_mm512_mask_cmplt_epi8_mask
   // CHECK: icmp slt <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmplt_epi8_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmplt_epu8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmplt_epu8_mask
+  // CHECK-LABEL: test_mm512_cmplt_epu8_mask
   // CHECK: icmp ult <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmplt_epu8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmplt_epu8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmplt_epu8_mask
+  // CHECK-LABEL: test_mm512_mask_cmplt_epu8_mask
   // CHECK: icmp ult <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmplt_epu8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmplt_epi16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmplt_epi16_mask
+  // CHECK-LABEL: test_mm512_cmplt_epi16_mask
   // CHECK: icmp slt <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmplt_epi16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmplt_epi16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmplt_epi16_mask
+  // CHECK-LABEL: test_mm512_mask_cmplt_epi16_mask
   // CHECK: icmp slt <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmplt_epi16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmplt_epu16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmplt_epu16_mask
+  // CHECK-LABEL: test_mm512_cmplt_epu16_mask
   // CHECK: icmp ult <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmplt_epu16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmplt_epu16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmplt_epu16_mask
+  // CHECK-LABEL: test_mm512_mask_cmplt_epu16_mask
   // CHECK: icmp ult <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmplt_epu16_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmpneq_epi8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpneq_epi8_mask
+  // CHECK-LABEL: test_mm512_cmpneq_epi8_mask
   // CHECK: icmp ne <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmpneq_epi8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmpneq_epi8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpneq_epi8_mask
+  // CHECK-LABEL: test_mm512_mask_cmpneq_epi8_mask
   // CHECK: icmp ne <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmpneq_epi8_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmpneq_epu8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpneq_epu8_mask
+  // CHECK-LABEL: test_mm512_cmpneq_epu8_mask
   // CHECK: icmp ne <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmpneq_epu8_mask(__a, __b);
 }
 
 __mmask64 test_mm512_mask_cmpneq_epu8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpneq_epu8_mask
+  // CHECK-LABEL: test_mm512_mask_cmpneq_epu8_mask
   // CHECK: icmp ne <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmpneq_epu8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmpneq_epi16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpneq_epi16_mask
+  // CHECK-LABEL: test_mm512_cmpneq_epi16_mask
   // CHECK: icmp ne <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmpneq_epi16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmpneq_epi16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpneq_epi16_mask
+  // CHECK-LABEL: test_mm512_mask_cmpneq_epi16_mask
   // CHECK: icmp ne <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmpneq_epi16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm512_cmpneq_epu16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmpneq_epu16_mask
+  // CHECK-LABEL: test_mm512_cmpneq_epu16_mask
   // CHECK: icmp ne <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmpneq_epu16_mask(__a, __b);
 }
 
 __mmask32 test_mm512_mask_cmpneq_epu16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmpneq_epu16_mask
+  // CHECK-LABEL: test_mm512_mask_cmpneq_epu16_mask
   // CHECK: icmp ne <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmpneq_epu16_mask(__u, __a, __b);
 }
 
 __mmask64 test_mm512_cmp_epi8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmp_epi8_mask
+  // CHECK-LABEL: test_mm512_cmp_epi8_mask
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmp_epi8_mask(__a, __b, 0);
 }
 
 __mmask64 test_mm512_mask_cmp_epi8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmp_epi8_mask
+  // CHECK-LABEL: test_mm512_mask_cmp_epi8_mask
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmp_epi8_mask(__u, __a, __b, 0);
 }
 
 __mmask64 test_mm512_cmp_epu8_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmp_epu8_mask
+  // CHECK-LABEL: test_mm512_cmp_epu8_mask
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_cmp_epu8_mask(__a, __b, 0);
 }
 
 __mmask64 test_mm512_mask_cmp_epu8_mask(__mmask64 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmp_epu8_mask
+  // CHECK-LABEL: test_mm512_mask_cmp_epu8_mask
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
   return (__mmask64)_mm512_mask_cmp_epu8_mask(__u, __a, __b, 0);
 }
 
 __mmask32 test_mm512_cmp_epi16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmp_epi16_mask
+  // CHECK-LABEL: test_mm512_cmp_epi16_mask
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmp_epi16_mask(__a, __b, 0);
 }
 
 __mmask32 test_mm512_mask_cmp_epi16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmp_epi16_mask
+  // CHECK-LABEL: test_mm512_mask_cmp_epi16_mask
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmp_epi16_mask(__u, __a, __b, 0);
 }
 
 __mmask32 test_mm512_cmp_epu16_mask(__m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_cmp_epu16_mask
+  // CHECK-LABEL: test_mm512_cmp_epu16_mask
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_cmp_epu16_mask(__a, __b, 0);
 }
 
 __mmask32 test_mm512_mask_cmp_epu16_mask(__mmask32 __u, __m512i __a, __m512i __b) {
-  // CHECK-LABEL: @test_mm512_mask_cmp_epu16_mask
+  // CHECK-LABEL: test_mm512_mask_cmp_epu16_mask
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm512_mask_cmp_epu16_mask(__u, __a, __b, 0);
 }
 
 __m512i test_mm512_add_epi8 (__m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_add_epi8
+  //CHECK-LABEL: test_mm512_add_epi8
   //CHECK: add <64 x i8>
   return _mm512_add_epi8(__A,__B);
 }
 
 __m512i test_mm512_mask_add_epi8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_mask_add_epi8
+  //CHECK-LABEL: test_mm512_mask_add_epi8
   //CHECK: add <64 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_add_epi8(__W, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_add_epi8 (__mmask64 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_maskz_add_epi8
+  //CHECK-LABEL: test_mm512_maskz_add_epi8
   //CHECK: add <64 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_add_epi8(__U, __A, __B);
 }
 
 __m512i test_mm512_sub_epi8 (__m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_sub_epi8
+  //CHECK-LABEL: test_mm512_sub_epi8
   //CHECK: sub <64 x i8>
   return _mm512_sub_epi8(__A, __B);
 }
 
 __m512i test_mm512_mask_sub_epi8 (__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_mask_sub_epi8
+  //CHECK-LABEL: test_mm512_mask_sub_epi8
   //CHECK: sub <64 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_sub_epi8(__W, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_sub_epi8 (__mmask64 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_maskz_sub_epi8
+  //CHECK-LABEL: test_mm512_maskz_sub_epi8
   //CHECK: sub <64 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_sub_epi8(__U, __A, __B);
 }
 
 __m512i test_mm512_add_epi16 (__m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_add_epi16
+  //CHECK-LABEL: test_mm512_add_epi16
   //CHECK: add <32 x i16>
   return _mm512_add_epi16(__A, __B);
 }
 
 __m512i test_mm512_mask_add_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_mask_add_epi16
+  //CHECK-LABEL: test_mm512_mask_add_epi16
   //CHECK: add <32 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_add_epi16(__W, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_add_epi16 (__mmask32 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_maskz_add_epi16
+  //CHECK-LABEL: test_mm512_maskz_add_epi16
   //CHECK: add <32 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_add_epi16(__U, __A, __B);
 }
 
 __m512i test_mm512_sub_epi16 (__m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_sub_epi16
+  //CHECK-LABEL: test_mm512_sub_epi16
   //CHECK: sub <32 x i16>
   return _mm512_sub_epi16(__A, __B);
 }
 
 __m512i test_mm512_mask_sub_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_mask_sub_epi16
+  //CHECK-LABEL: test_mm512_mask_sub_epi16
   //CHECK: sub <32 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_sub_epi16(__W, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_sub_epi16 (__mmask32 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_maskz_sub_epi16
+  //CHECK-LABEL: test_mm512_maskz_sub_epi16
   //CHECK: sub <32 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_sub_epi16(__U, __A, __B);
 }
 
 __m512i test_mm512_mullo_epi16 (__m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_mullo_epi16
+  //CHECK-LABEL: test_mm512_mullo_epi16
   //CHECK: mul <32 x i16>
   return _mm512_mullo_epi16(__A, __B);
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_mullo_epi16((__m512i)(__v32hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32}, (__m512i)(__v32hi){-64, -62, +60, +58, -56, -54, +52, +50, -48, -46, +44, +42, -40, -38, +36, +34, -32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), -64, 124, 180, -232, -280, 324, 364, -400, -432, 460, 484, -504, -520, 532, 540, -544, -544, 540, 532, -520, -504, 484, 460, -432, -400, 364, 324, -280, -232, -180, -124, -64));
 
 __m512i test_mm512_mask_mullo_epi16 (__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_mask_mullo_epi16
+  //CHECK-LABEL: test_mm512_mask_mullo_epi16
   //CHECK: mul <32 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_mullo_epi16(__W, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_mullo_epi16 (__mmask32 __U, __m512i __A, __m512i __B) {
-  //CHECK-LABEL: @test_mm512_maskz_mullo_epi16
+  //CHECK-LABEL: test_mm512_maskz_mullo_epi16
   //CHECK: mul <32 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_mullo_epi16(__U, __A, __B);
 }
 
 __m512i test_mm512_mask_blend_epi8(__mmask64 __U, __m512i __A, __m512i __W) {
-  // CHECK-LABEL: @test_mm512_mask_blend_epi8
+  // CHECK-LABEL: test_mm512_mask_blend_epi8
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_blend_epi8(__U,__A,__W); 
 }
 __m512i test_mm512_mask_blend_epi16(__mmask32 __U, __m512i __A, __m512i __W) {
-  // CHECK-LABEL: @test_mm512_mask_blend_epi16
+  // CHECK-LABEL: test_mm512_mask_blend_epi16
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_blend_epi16(__U,__A,__W); 
 }
 __m512i test_mm512_abs_epi8(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_abs_epi8
+  // CHECK-LABEL: test_mm512_abs_epi8
   // CHECK: [[ABS:%.*]] = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %{{.*}}, i1 false)
   return _mm512_abs_epi8(__A); 
 }
 __m512i test_mm512_mask_abs_epi8(__m512i __W, __mmask64 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_abs_epi8
+  // CHECK-LABEL: test_mm512_mask_abs_epi8
   // CHECK: [[ABS:%.*]] = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %{{.*}}, i1 false)
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> [[ABS]], <64 x i8> %{{.*}}
   return _mm512_mask_abs_epi8(__W,__U,__A); 
 }
 __m512i test_mm512_maskz_abs_epi8(__mmask64 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_abs_epi8
+  // CHECK-LABEL: test_mm512_maskz_abs_epi8
   // CHECK: [[ABS:%.*]] = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %{{.*}}, i1 false)
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> [[ABS]], <64 x i8> %{{.*}}
   return _mm512_maskz_abs_epi8(__U,__A); 
 }
 __m512i test_mm512_abs_epi16(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_abs_epi16
+  // CHECK-LABEL: test_mm512_abs_epi16
   // CHECK: [[ABS:%.*]] = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %{{.*}}, i1 false)
   return _mm512_abs_epi16(__A); 
 }
 __m512i test_mm512_mask_abs_epi16(__m512i __W, __mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_abs_epi16
+  // CHECK-LABEL: test_mm512_mask_abs_epi16
   // CHECK: [[ABS:%.*]] = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %{{.*}}, i1 false)
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> [[ABS]], <32 x i16> %{{.*}}
   return _mm512_mask_abs_epi16(__W,__U,__A); 
 }
 __m512i test_mm512_maskz_abs_epi16(__mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_abs_epi16
+  // CHECK-LABEL: test_mm512_maskz_abs_epi16
   // CHECK: [[ABS:%.*]] = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %{{.*}}, i1 false)
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> [[ABS]], <32 x i16> %{{.*}}
   return _mm512_maskz_abs_epi16(__U,__A); 
 }
 __m512i test_mm512_packs_epi32(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_packs_epi32
+  // CHECK-LABEL: test_mm512_packs_epi32
   // CHECK: @llvm.x86.avx512.packssdw.512
   return _mm512_packs_epi32(__A,__B); 
 }
 __m512i test_mm512_maskz_packs_epi32(__mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_packs_epi32
+  // CHECK-LABEL: test_mm512_maskz_packs_epi32
   // CHECK: @llvm.x86.avx512.packssdw.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_packs_epi32(__M,__A,__B); 
 }
 __m512i test_mm512_mask_packs_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_packs_epi32
+  // CHECK-LABEL: test_mm512_mask_packs_epi32
   // CHECK: @llvm.x86.avx512.packssdw.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_packs_epi32(__W,__M,__A,__B); 
 }
 __m512i test_mm512_packs_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_packs_epi16
+  // CHECK-LABEL: test_mm512_packs_epi16
   // CHECK: @llvm.x86.avx512.packsswb.512
   return _mm512_packs_epi16(__A,__B); 
 }
 __m512i test_mm512_mask_packs_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_packs_epi16
+  // CHECK-LABEL: test_mm512_mask_packs_epi16
   // CHECK: @llvm.x86.avx512.packsswb.512
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_packs_epi16(__W,__M,__A,__B); 
 }
 __m512i test_mm512_maskz_packs_epi16(__mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_packs_epi16
+  // CHECK-LABEL: test_mm512_maskz_packs_epi16
   // CHECK: @llvm.x86.avx512.packsswb.512
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_packs_epi16(__M,__A,__B); 
 }
 __m512i test_mm512_packus_epi32(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_packus_epi32
+  // CHECK-LABEL: test_mm512_packus_epi32
   // CHECK: @llvm.x86.avx512.packusdw.512
   return _mm512_packus_epi32(__A,__B); 
 }
 __m512i test_mm512_maskz_packus_epi32(__mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_packus_epi32
+  // CHECK-LABEL: test_mm512_maskz_packus_epi32
   // CHECK: @llvm.x86.avx512.packusdw.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_packus_epi32(__M,__A,__B); 
 }
 __m512i test_mm512_mask_packus_epi32(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_packus_epi32
+  // CHECK-LABEL: test_mm512_mask_packus_epi32
   // CHECK: @llvm.x86.avx512.packusdw.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_packus_epi32(__W,__M,__A,__B); 
 }
 __m512i test_mm512_packus_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_packus_epi16
+  // CHECK-LABEL: test_mm512_packus_epi16
   // CHECK: @llvm.x86.avx512.packuswb.512
   return _mm512_packus_epi16(__A,__B); 
 }
 __m512i test_mm512_mask_packus_epi16(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_packus_epi16
+  // CHECK-LABEL: test_mm512_mask_packus_epi16
   // CHECK: @llvm.x86.avx512.packuswb.512
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_packus_epi16(__W,__M,__A,__B); 
 }
 __m512i test_mm512_maskz_packus_epi16(__mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_packus_epi16
+  // CHECK-LABEL: test_mm512_maskz_packus_epi16
   // CHECK: @llvm.x86.avx512.packuswb.512
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_packus_epi16(__M,__A,__B); 
 }
 __m512i test_mm512_adds_epi8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_adds_epi8
+  // CHECK-LABEL: test_mm512_adds_epi8
   // CHECK: @llvm.sadd.sat.v64i8
   return _mm512_adds_epi8(__A,__B); 
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_adds_epi8((__m512i)(__v64qs){0, +1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32, +33, -34, +35, -36, +37, -38, +39, -40, +41, -42, +43, -44, +45, -46, +47, +100, +50, -100, +20, +80, -50, +120, -20, -100, -50, +100, -20, -80, +50, -120, +20}, (__m512i)(__v64qs){0, +1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32, +33, -34, +35, -36, +37, -38, +39, -40, +41, -42, +43, -44, +45, -46, +47, +50, +80, -50, +110, +60, -30, +20, -10, +50, +80, -50, +110, +60, -30, +20, -10}), 0, +2, -4, +6, -8, +10, -12, +14, -16, +18, -20, +22, -24, +26, -28, +30, -32, +34, -36, +38, -40, +42, -44, +46, -48, +50, -52, +54, -56, +58, -60, +62, -64, +66, -68, +70, -72, +74, -76, +78, -80, +82, -84, +86, -88, +90, -92, +94, +127, +127, -128, +127, +127, -80, +127, -30, -50, +30, +50, +90, -20, +20, -100, +10));
+
 __m512i test_mm512_mask_adds_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_adds_epi8
+  // CHECK-LABEL: test_mm512_mask_adds_epi8
   // CHECK: @llvm.sadd.sat.v64i8
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
  return _mm512_mask_adds_epi8(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_adds_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_adds_epi8
+  // CHECK-LABEL: test_mm512_maskz_adds_epi8
   // CHECK: @llvm.sadd.sat.v64i8
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_adds_epi8(__U,__A,__B); 
 }
 __m512i test_mm512_adds_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_adds_epi16
+  // CHECK-LABEL: test_mm512_adds_epi16
   // CHECK: @llvm.sadd.sat.v32i16
  return _mm512_adds_epi16(__A,__B); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_adds_epi16((__m512i)(__v32hi){0, +1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, +32000, -32000, +32000, -32000}, (__m512i)(__v32hi){0, +1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, +800, -800, -800, +800}), 0, +2, -4, +6, -8, +10, -12, +14, -16, +18, -20, +22, -24, +26, -28, +30, -32, +34, -36, +38, -40, +42, -44, +46, -48, +50, -52, +54, +32767, -32768, +31200, -31200));
+
 __m512i test_mm512_mask_adds_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_adds_epi16
+  // CHECK-LABEL: test_mm512_mask_adds_epi16
   // CHECK: @llvm.sadd.sat.v32i16
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_adds_epi16(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_adds_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_adds_epi16
+  // CHECK-LABEL: test_mm512_maskz_adds_epi16
   // CHECK: @llvm.sadd.sat.v32i16
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
 return _mm512_maskz_adds_epi16(__U,__A,__B); 
 }
 __m512i test_mm512_adds_epu8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_adds_epu8
+  // CHECK-LABEL: test_mm512_adds_epu8
   // CHECK-NOT: @llvm.x86.avx512.mask.paddus.b.512
   // CHECK: call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_adds_epu8(__A,__B); 
 }
+TEST_CONSTEXPR(match_v64qu(_mm512_adds_epu8((__m512i)(__v64qu){0, 0, 0, 0, 0, 0, 0, 0, +63, +63, +63, +63, +63, +63, +63, +63, +64, +64, +64, +64, +64, +64, +64, +64, +127, +127, +127, +127, +127, +127, +127, +127, +128, +128, +128, +128, +128, +128, +128, +128, +191, +191, +191, +191, +191, +191, +191, +191, +192, +192, +192, +192, +192, +192, +192, +192, +255, +255, +255, +255, +255, +255, +255, +255}, (__m512i)(__v64qu){0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255}), 0, +63, +64, +127, +128, +191, +192, +255, +63, +126, +127, +190, +191, +254, +255, +255, +64, +127, +128, +191, +192, +255, +255, +255, +127, +190, +191, +254, +255, +255, +255, +255, +128, +191, +192, +255, +255, +255, +255, +255, +191, +254, +255, +255, +255, +255, +255, +255, +192, +255, +255, +255, +255, +255, +255, +255, +255, +255, +255, +255, +255, +255, +255, +255));
+
 __m512i test_mm512_mask_adds_epu8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_adds_epu8
+  // CHECK-LABEL: test_mm512_mask_adds_epu8
   // CHECK-NOT: @llvm.x86.avx512.mask.paddus.b.512
   // CHECK: call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_adds_epu8(__W,__U,__A,__B); 
 }
+TEST_CONSTEXPR(match_v32hu(_mm512_adds_epu16((__m512i)(__v32hu){0, 0, 0, 0, +16384, +16384, +16384, +16384, +16384, +16384, +32767, +32767, +32767, +32767, +32767, +32767, +32768, +32768, +32768, +32768, +32768, +32768, +49152, +49152, +49152, +49152, +49152, +49152, +65535, +65535, +65535, +65535}, (__m512i)(__v32hu){0, +32767, +32768, +65535, 0, +16384, +32767, +32768, +49152, +65535, 0, +16384, +32767, +32768, +49152, +65535, 0, +16384, +32767, +32768, +49152, +65535, 0, +16384, +32767, +32768, +49152, +65535, 0, +32767, +32768, +65535}), 0, +32767, +32768, +65535, +16384, +32768, +49151, +49152, +65535, +65535, +32767, +49151, +65534, +65535, +65535, +65535, +32768, +49152, +65535, +65535, +65535, +65535, +49152, +65535, +65535, +65535, +65535, +65535, +65535, +65535, +65535, +65535));
+
 __m512i test_mm512_maskz_adds_epu8(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_adds_epu8
+  // CHECK-LABEL: test_mm512_maskz_adds_epu8
   // CHECK-NOT: @llvm.x86.avx512.mask.paddus.b.512
   // CHECK: call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_adds_epu8(__U,__A,__B); 
 }
 __m512i test_mm512_adds_epu16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_adds_epu16
+  // CHECK-LABEL: test_mm512_adds_epu16
   // CHECK-NOT: @llvm.x86.avx512.mask.paddus.w.512
   // CHECK: call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_adds_epu16(__A,__B); 
 }
 __m512i test_mm512_mask_adds_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_adds_epu16
+  // CHECK-LABEL: test_mm512_mask_adds_epu16
   // CHECK-NOT: @llvm.x86.avx512.mask.paddus.w.512
   // CHECK: call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_adds_epu16(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_adds_epu16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_adds_epu16
+  // CHECK-LABEL: test_mm512_maskz_adds_epu16
   // CHECK-NOT: @llvm.x86.avx512.mask.paddus.w.512
   // CHECK: call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_adds_epu16(__U,__A,__B); 
 }
 __m512i test_mm512_avg_epu8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_avg_epu8
+  // CHECK-LABEL: test_mm512_avg_epu8
   // CHECK: @llvm.x86.avx512.pavg.b.512
   return _mm512_avg_epu8(__A,__B); 
 }
 __m512i test_mm512_mask_avg_epu8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_avg_epu8
+  // CHECK-LABEL: test_mm512_mask_avg_epu8
   // CHECK: @llvm.x86.avx512.pavg.b.512
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_avg_epu8(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_avg_epu8(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_avg_epu8
+  // CHECK-LABEL: test_mm512_maskz_avg_epu8
   // CHECK: @llvm.x86.avx512.pavg.b.512
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_avg_epu8(__U,__A,__B); 
 }
 __m512i test_mm512_avg_epu16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_avg_epu16
+  // CHECK-LABEL: test_mm512_avg_epu16
   // CHECK: @llvm.x86.avx512.pavg.w.512
   return _mm512_avg_epu16(__A,__B); 
 }
 __m512i test_mm512_mask_avg_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_avg_epu16
+  // CHECK-LABEL: test_mm512_mask_avg_epu16
   // CHECK: @llvm.x86.avx512.pavg.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_avg_epu16(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_avg_epu16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_avg_epu16
+  // CHECK-LABEL: test_mm512_maskz_avg_epu16
   // CHECK: @llvm.x86.avx512.pavg.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_avg_epu16(__U,__A,__B); 
 }
 __m512i test_mm512_max_epi8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_max_epi8
+  // CHECK-LABEL: test_mm512_max_epi8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.smax.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_max_epi8(__A,__B); 
 }
 __m512i test_mm512_maskz_max_epi8(__mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_max_epi8
+  // CHECK-LABEL: test_mm512_maskz_max_epi8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.smax.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK:       select <64 x i1> {{.*}}, <64 x i8> [[RES]], <64 x i8> {{.*}}
   return _mm512_maskz_max_epi8(__M,__A,__B); 
 }
 __m512i test_mm512_mask_max_epi8(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_max_epi8
+  // CHECK-LABEL: test_mm512_mask_max_epi8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.smax.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK:       select <64 x i1> {{.*}}, <64 x i8> [[RES]], <64 x i8> {{.*}}
   return _mm512_mask_max_epi8(__W,__M,__A,__B); 
 }
 __m512i test_mm512_max_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_max_epi16
+  // CHECK-LABEL: test_mm512_max_epi16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.smax.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_max_epi16(__A,__B); 
 }
 __m512i test_mm512_maskz_max_epi16(__mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_max_epi16
+  // CHECK-LABEL: test_mm512_maskz_max_epi16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.smax.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK:       select <32 x i1> {{.*}}, <32 x i16> [[RES]], <32 x i16> {{.*}}
   return _mm512_maskz_max_epi16(__M,__A,__B); 
 }
 __m512i test_mm512_mask_max_epi16(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_max_epi16
+  // CHECK-LABEL: test_mm512_mask_max_epi16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.smax.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK:       select <32 x i1> {{.*}}, <32 x i16> [[RES]], <32 x i16> {{.*}}
   return _mm512_mask_max_epi16(__W,__M,__A,__B); 
 }
 __m512i test_mm512_max_epu8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_max_epu8
+  // CHECK-LABEL: test_mm512_max_epu8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.umax.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_max_epu8(__A,__B); 
 }
 __m512i test_mm512_maskz_max_epu8(__mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_max_epu8
+  // CHECK-LABEL: test_mm512_maskz_max_epu8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.umax.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK:       select <64 x i1> {{.*}}, <64 x i8> [[RES]], <64 x i8> {{.*}}
   return _mm512_maskz_max_epu8(__M,__A,__B); 
 }
 __m512i test_mm512_mask_max_epu8(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_max_epu8
+  // CHECK-LABEL: test_mm512_mask_max_epu8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.umax.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK:       select <64 x i1> {{.*}}, <64 x i8> [[RES]], <64 x i8> {{.*}}
   return _mm512_mask_max_epu8(__W,__M,__A,__B); 
 }
 __m512i test_mm512_max_epu16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_max_epu16
+  // CHECK-LABEL: test_mm512_max_epu16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.umax.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_max_epu16(__A,__B); 
 }
 __m512i test_mm512_maskz_max_epu16(__mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_max_epu16
+  // CHECK-LABEL: test_mm512_maskz_max_epu16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.umax.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK:       select <32 x i1> {{.*}}, <32 x i16> [[RES]], <32 x i16> {{.*}}
   return _mm512_maskz_max_epu16(__M,__A,__B); 
 }
 __m512i test_mm512_mask_max_epu16(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_max_epu16
+  // CHECK-LABEL: test_mm512_mask_max_epu16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.umax.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK:       select <32 x i1> {{.*}}, <32 x i16> [[RES]], <32 x i16> {{.*}}
   return _mm512_mask_max_epu16(__W,__M,__A,__B); 
 }
 __m512i test_mm512_min_epi8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_min_epi8
+  // CHECK-LABEL: test_mm512_min_epi8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.smin.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_min_epi8(__A,__B); 
 }
 __m512i test_mm512_maskz_min_epi8(__mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_min_epi8
+  // CHECK-LABEL: test_mm512_maskz_min_epi8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.smin.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK:       select <64 x i1> {{.*}}, <64 x i8> [[RES]], <64 x i8> {{.*}}
   return _mm512_maskz_min_epi8(__M,__A,__B); 
 }
 __m512i test_mm512_mask_min_epi8(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_min_epi8
+  // CHECK-LABEL: test_mm512_mask_min_epi8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.smin.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK:       select <64 x i1> {{.*}}, <64 x i8> [[RES]], <64 x i8> {{.*}}
   return _mm512_mask_min_epi8(__W,__M,__A,__B); 
 }
 __m512i test_mm512_min_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_min_epi16
+  // CHECK-LABEL: test_mm512_min_epi16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.smin.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_min_epi16(__A,__B); 
 }
 __m512i test_mm512_maskz_min_epi16(__mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_min_epi16
+  // CHECK-LABEL: test_mm512_maskz_min_epi16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.smin.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK:       select <32 x i1> {{.*}}, <32 x i16> [[RES]], <32 x i16> {{.*}}
   return _mm512_maskz_min_epi16(__M,__A,__B); 
 }
 __m512i test_mm512_mask_min_epi16(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_min_epi16
+  // CHECK-LABEL: test_mm512_mask_min_epi16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.smin.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK:       select <32 x i1> {{.*}}, <32 x i16> [[RES]], <32 x i16> {{.*}}
   return _mm512_mask_min_epi16(__W,__M,__A,__B); 
 }
 __m512i test_mm512_min_epu8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_min_epu8
+  // CHECK-LABEL: test_mm512_min_epu8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.umin.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_min_epu8(__A,__B); 
 }
 __m512i test_mm512_maskz_min_epu8(__mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_min_epu8
+  // CHECK-LABEL: test_mm512_maskz_min_epu8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.umin.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK:       select <64 x i1> {{.*}}, <64 x i8> [[RES]], <64 x i8> {{.*}}
   return _mm512_maskz_min_epu8(__M,__A,__B); 
 }
 __m512i test_mm512_mask_min_epu8(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_min_epu8
+  // CHECK-LABEL: test_mm512_mask_min_epu8
   // CHECK:       [[RES:%.*]] = call <64 x i8> @llvm.umin.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK:       select <64 x i1> {{.*}}, <64 x i8> [[RES]], <64 x i8> {{.*}}
   return _mm512_mask_min_epu8(__W,__M,__A,__B); 
 }
 __m512i test_mm512_min_epu16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_min_epu16
+  // CHECK-LABEL: test_mm512_min_epu16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.umin.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_min_epu16(__A,__B); 
 }
 __m512i test_mm512_maskz_min_epu16(__mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_min_epu16
+  // CHECK-LABEL: test_mm512_maskz_min_epu16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.umin.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK:       select <32 x i1> {{.*}}, <32 x i16> [[RES]], <32 x i16> {{.*}}
   return _mm512_maskz_min_epu16(__M,__A,__B); 
 }
 __m512i test_mm512_mask_min_epu16(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_min_epu16
+  // CHECK-LABEL: test_mm512_mask_min_epu16
   // CHECK:       [[RES:%.*]] = call <32 x i16> @llvm.umin.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK:       select <32 x i1> {{.*}}, <32 x i16> [[RES]], <32 x i16> {{.*}}
   return _mm512_mask_min_epu16(__W,__M,__A,__B); 
 }
 __m512i test_mm512_shuffle_epi8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shuffle_epi8
+  // CHECK-LABEL: test_mm512_shuffle_epi8
   // CHECK: @llvm.x86.avx512.pshuf.b.512
   return _mm512_shuffle_epi8(__A,__B); 
 }
 __m512i test_mm512_mask_shuffle_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shuffle_epi8
+  // CHECK-LABEL: test_mm512_mask_shuffle_epi8
   // CHECK: @llvm.x86.avx512.pshuf.b.512
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_shuffle_epi8(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_shuffle_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shuffle_epi8
+  // CHECK-LABEL: test_mm512_maskz_shuffle_epi8
   // CHECK: @llvm.x86.avx512.pshuf.b.512
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_shuffle_epi8(__U,__A,__B); 
 }
 __m512i test_mm512_subs_epi8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_subs_epi8
+  // CHECK-LABEL: test_mm512_subs_epi8
   // CHECK: @llvm.ssub.sat.v64i8
 return _mm512_subs_epi8(__A,__B); 
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_subs_epi8((__m512i)(__v64qs){0, +1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32, +33, -34, +35, -36, +37, -38, +39, -40, +41, -42, +43, -44, +45, -46, +47, +100, +50, -100, +20, +80, -50, +120, -20, -100, -50, +100, -20, -80, +50, -120, +20}, (__m512i)(__v64qs){0, -1, +2, -3, +4, -5, +6, -7, +8, -9, +10, -11, +12, -13, +14, -15, +16, -17, +18, -19, +20, -21, +22, -23, +24, -25, +26, -27, +28, -29, +30, -31, +32, -33, +34, -35, +36, -37, +38, -39, +40, -41, +42, -43, +44, -45, +46, -47, -50, -80, +50, -110, -60, +30, -20, +10, -50, -80, +50, -110, -60, +30, -20, +10}), 0, +2, -4, +6, -8, +10, -12, +14, -16, +18, -20, +22, -24, +26, -28, +30, -32, +34, -36, +38, -40, +42, -44, +46, -48, +50, -52, +54, -56, +58, -60, +62, -64, +66, -68, +70, -72, +74, -76, +78, -80, +82, -84, +86, -88, +90, -92, +94, +127, +127, -128, +127, +127, -80, +127, -30, -50, +30, +50, +90, -20, +20, -100, +10));
+
 __m512i test_mm512_mask_subs_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_subs_epi8
+  // CHECK-LABEL: test_mm512_mask_subs_epi8
   // CHECK: @llvm.ssub.sat.v64i8
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
 return _mm512_mask_subs_epi8(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_subs_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_subs_epi8
+  // CHECK-LABEL: test_mm512_maskz_subs_epi8
   // CHECK: @llvm.ssub.sat.v64i8
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
 return _mm512_maskz_subs_epi8(__U,__A,__B); 
 }
 __m512i test_mm512_subs_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_subs_epi16
+  // CHECK-LABEL: test_mm512_subs_epi16
   // CHECK: @llvm.ssub.sat.v32i16
 return _mm512_subs_epi16(__A,__B); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_subs_epi16((__m512i)(__v32hi){0, +1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, +32000, -32000, +32000, -32000}, (__m512i)(__v32hi){0, -1, +2, -3, +4, -5, +6, -7, +8, -9, +10, -11, +12, -13, +14, -15, +16, -17, +18, -19, +20, -21, +22, -23, +24, -25, +26, -27, -800, +800, +800, -800}), 0, +2, -4, +6, -8, +10, -12, +14, -16, +18, -20, +22, -24, +26, -28, +30, -32, +34, -36, +38, -40, +42, -44, +46, -48, +50, -52, +54, +32767, -32768, +31200, -31200));
 __m512i test_mm512_mask_subs_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_subs_epi16
+  // CHECK-LABEL: test_mm512_mask_subs_epi16
   // CHECK: @llvm.ssub.sat.v32i16
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
 return _mm512_mask_subs_epi16(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_subs_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_subs_epi16
+  // CHECK-LABEL: test_mm512_maskz_subs_epi16
   // CHECK: @llvm.ssub.sat.v32i16
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
 return _mm512_maskz_subs_epi16(__U,__A,__B); 
 }
 __m512i test_mm512_subs_epu8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_subs_epu8
+  // CHECK-LABEL: test_mm512_subs_epu8
   // CHECK-NOT: @llvm.x86.avx512.mask.psubus.b.512
   // CHECK: call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
 return _mm512_subs_epu8(__A,__B); 
 }
 __m512i test_mm512_mask_subs_epu8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_subs_epu8
+  // CHECK-LABEL: test_mm512_mask_subs_epu8
   // CHECK-NOT: @llvm.x86.avx512.mask.psubus.b.512
   // CHECK: call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
 return _mm512_mask_subs_epu8(__W,__U,__A,__B); 
 }
+TEST_CONSTEXPR(match_v64qu(_mm512_subs_epu8((__m512i)(__v64qu){0, 0, 0, 0, 0, 0, 0, 0, +63, +63, +63, +63, +63, +63, +63, +63, +64, +64, +64, +64, +64, +64, +64, +64, +127, +127, +127, +127, +127, +127, +127, +127, +128, +128, +128, +128, +128, +128, +128, +128, +191, +191, +191, +191, +191, +191, +191, +191, +192, +192, +192, +192, +192, +192, +192, +192, +255, +255, +255, +255, +255, +255, +255, +255}, (__m512i)(__v64qu){0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255, 0, +63, +64, +127, +128, +191, +192, +255}), 0, 0, 0, 0, 0, 0, 0, 0, +63, 0, 0, 0, 0, 0, 0, 0, +64, +1, 0, 0, 0, 0, 0, 0, +127, +64, +63, 0, 0, 0, 0, 0, +128, +65, +64, +1, 0, 0, 0, 0, +191, +128, +127, +64, +63, 0, 0, 0, +192, +129, +128, +65, +64, +1, 0, 0, +255, +192, +191, +128, +127, +64, +63, +0));
+
 __m512i test_mm512_maskz_subs_epu8(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_subs_epu8
+  // CHECK-LABEL: test_mm512_maskz_subs_epu8
   // CHECK-NOT: @llvm.x86.avx512.mask.psubus.b.512
   // CHECK: call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
 return _mm512_maskz_subs_epu8(__U,__A,__B); 
 }
 __m512i test_mm512_subs_epu16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_subs_epu16
+  // CHECK-LABEL: test_mm512_subs_epu16
   // CHECK-NOT: @llvm.x86.avx512.mask.psubus.w.512
   // CHECK: call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
 return _mm512_subs_epu16(__A,__B); 
 }
 __m512i test_mm512_mask_subs_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_subs_epu16
+  // CHECK-LABEL: test_mm512_mask_subs_epu16
   // CHECK-NOT: @llvm.x86.avx512.mask.psubus.w.512
   // CHECK: call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
 return _mm512_mask_subs_epu16(__W,__U,__A,__B); 
+TEST_CONSTEXPR(match_v32hu(_mm512_subs_epu16((__m512i)(__v32hu){0, 0, 0, 0, +16384, +16384, +16384, +16384, +16384, +16384, +32767, +32767, +32767, +32767, +32767, +32767, +32768, +32768, +32768, +32768, +32768, +32768, +49152, +49152, +49152, +49152, +49152, +49152, +65535, +65535, +65535, +65535}, (__m512i)(__v32hu){0, +32767, +32768, +65535, 0, +16384, +32767, +32768, +49152, +65535, 0, +16384, +32767, +32768, +49152, +65535, 0, +16384, +32767, +32768, +49152, +65535, 0, +16384, +32767, +32768, +49152, +65535, 0, +32767, +32768, +65535}), 0, 0, 0, 0, +16384, 0, 0, 0, 0, 0, +32767, +16383, 0, 0, 0, 0, +32768, +16384, +1, 0, 0, 0, +49152, +32768, +16385, +16384, 0, 0, +65535, +32768, +32767, 0));
 }
 __m512i test_mm512_maskz_subs_epu16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_subs_epu16
+  // CHECK-LABEL: test_mm512_maskz_subs_epu16
   // CHECK-NOT: @llvm.x86.avx512.mask.psubus.w.512
   // CHECK: call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
 return _mm512_maskz_subs_epu16(__U,__A,__B); 
 }
 __m512i test_mm512_mask2_permutex2var_epi16(__m512i __A, __m512i __I, __mmask32 __U, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask2_permutex2var_epi16
+  // CHECK-LABEL: test_mm512_mask2_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask2_permutex2var_epi16(__A,__I,__U,__B); 
 }
 __m512i test_mm512_permutex2var_epi16(__m512i __A, __m512i __I, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_permutex2var_epi16
+  // CHECK-LABEL: test_mm512_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.512
   return _mm512_permutex2var_epi16(__A,__I,__B); 
 }
 __m512i test_mm512_mask_permutex2var_epi16(__m512i __A, __mmask32 __U, __m512i __I, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_permutex2var_epi16
+  // CHECK-LABEL: test_mm512_mask_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_permutex2var_epi16(__A,__U,__I,__B); 
 }
 __m512i test_mm512_maskz_permutex2var_epi16(__mmask32 __U, __m512i __A, __m512i __I, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_permutex2var_epi16
+  // CHECK-LABEL: test_mm512_maskz_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_permutex2var_epi16(__U,__A,__I,__B); 
 }
 
 __m512i test_mm512_mulhrs_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mulhrs_epi16
+  // CHECK-LABEL: test_mm512_mulhrs_epi16
   // CHECK: @llvm.x86.avx512.pmul.hr.sw.512
   return _mm512_mulhrs_epi16(__A,__B); 
 }
-__m512i test_mm512_mask_mulhrs_epi16(__m512i __W, __mmask32 __U, __m512i __A,        __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_mulhrs_epi16
+__m512i test_mm512_mask_mulhrs_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
+  // CHECK-LABEL: test_mm512_mask_mulhrs_epi16
   // CHECK: @llvm.x86.avx512.pmul.hr.sw.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_mulhrs_epi16(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_mulhrs_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_mulhrs_epi16
+  // CHECK-LABEL: test_mm512_maskz_mulhrs_epi16
   // CHECK: @llvm.x86.avx512.pmul.hr.sw.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_mulhrs_epi16(__U,__A,__B); 
 }
 __m512i test_mm512_mulhi_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mulhi_epi16
+  // CHECK-LABEL: test_mm512_mulhi_epi16
   // CHECK: @llvm.x86.avx512.pmulh.w.512
   return _mm512_mulhi_epi16(__A,__B); 
 }
-__m512i test_mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A,       __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_mulhi_epi16
+TEST_CONSTEXPR(match_v32hi(_mm512_mulhi_epi16((__m512i)(__v32hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32}, (__m512i)(__v32hi){-64, -62, +60, +58, -56, -54, +52, +50, -48, -46, +44, +42, -40, -38, +36, +34, -32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, -1, -1, -1));
+
+__m512i test_mm512_mask_mulhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
+  // CHECK-LABEL: test_mm512_mask_mulhi_epi16
   // CHECK: @llvm.x86.avx512.pmulh.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_mulhi_epi16(__W,__U,__A,__B); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_mask_mulhi_epi16(_mm512_set1_epi16(1), 0xF00FF00F, (__m512i)(__v32hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32}, (__m512i)(__v32hi){-64, -62, +60, +58, -56, -54, +52, +50, -48, -46, +44, +42, -40, -38, +36, +34, -32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), -1, 0, 0, -1, 1, 1, 1, 1, 1, 1, 1, 1, -1, 0, 0, -1, -1, 0, 0, -1, 1, 1, 1, 1, 1, 1, 1, 1, -1, -1, -1, -1));
+
 __m512i test_mm512_maskz_mulhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_mulhi_epi16
+  // CHECK-LABEL: test_mm512_maskz_mulhi_epi16
   // CHECK: @llvm.x86.avx512.pmulh.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_mulhi_epi16(__U,__A,__B); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_maskz_mulhi_epi16(0x0FF00FF0, (__m512i)(__v32hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32}, (__m512i)(__v32hi){-64, -62, +60, +58, -56, -54, +52, +50, -48, -46, +44, +42, -40, -38, +36, +34, -32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), 0, 0, 0, 0, -1, 0, 0, -1, -1, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, -1, 0, 0, -1, -1, 0, 0, -1, 0, 0, 0, 0));
+
 __m512i test_mm512_mulhi_epu16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mulhi_epu16
+  // CHECK-LABEL: test_mm512_mulhi_epu16
   // CHECK: @llvm.x86.avx512.pmulhu.w.512
   return _mm512_mulhi_epu16(__A,__B); 
 }
-__m512i test_mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A,       __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_mulhi_epu16
+TEST_CONSTEXPR(match_v32hi(_mm512_mulhi_epu16((__m512i)(__v32hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32}, (__m512i)(__v32hi){-64, -62, +60, +58, -56, -54, +52, +50, -48, -46, +44, +42, -40, -38, +36, +34, -32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), 0, -64, 0, 57, 4, -60, 0, 49, 8, -56, 0, 41, 12, -52, 0, 33, 16, -48, 0, 25, 20, -44, 0, 17, 24, -40, 0, 9, 28, 5, 30, 1));
+
+__m512i test_mm512_mask_mulhi_epu16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
+  // CHECK-LABEL: test_mm512_mask_mulhi_epu16
   // CHECK: @llvm.x86.avx512.pmulhu.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_mulhi_epu16(__W,__U,__A,__B); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_mask_mulhi_epu16(_mm512_set1_epi16(1), 0x0FF00FF0, (__m512i)(__v32hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32}, (__m512i)(__v32hi){-64, -62, +60, +58, -56, -54, +52, +50, -48, -46, +44, +42, -40, -38, +36, +34, -32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), 1, 1, 1, 1, 4, -60, 0, 49, 8, -56, 0, 41, 1, 1, 1, 1, 1, 1, 1, 1, 20, -44, 0, 17, 24, -40, 0, 9, 1, 1, 1, 1));
+
 __m512i test_mm512_maskz_mulhi_epu16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_mulhi_epu16
+  // CHECK-LABEL: test_mm512_maskz_mulhi_epu16
   // CHECK: @llvm.x86.avx512.pmulhu.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_mulhi_epu16(__U,__A,__B); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_maskz_mulhi_epu16(0xF00FF00F, (__m512i)(__v32hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16, +17, -18, +19, -20, +21, -22, +23, -24, +25, -26, +27, -28, +29, -30, +31, -32}, (__m512i)(__v32hi){-64, -62, +60, +58, -56, -54, +52, +50, -48, -46, +44, +42, -40, -38, +36, +34, -32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), 0, -64, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 12, -52, 0, 33, 16, -48, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 28, 5, 30, 1));
 
 __m512i test_mm512_maddubs_epi16(__m512i __X, __m512i __Y) {
-  // CHECK-LABEL: @test_mm512_maddubs_epi16
+  // CHECK-LABEL: test_mm512_maddubs_epi16
   // CHECK: @llvm.x86.avx512.pmaddubs.w.512
   return _mm512_maddubs_epi16(__X,__Y); 
 }
-__m512i test_mm512_mask_maddubs_epi16(__m512i __W, __mmask32 __U, __m512i __X,         __m512i __Y) {
-  // CHECK-LABEL: @test_mm512_mask_maddubs_epi16
+__m512i test_mm512_mask_maddubs_epi16(__m512i __W, __mmask32 __U, __m512i __X, __m512i __Y) {
+  // CHECK-LABEL: test_mm512_mask_maddubs_epi16
   // CHECK: @llvm.x86.avx512.pmaddubs.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_maddubs_epi16(__W,__U,__X,__Y); 
 }
 __m512i test_mm512_maskz_maddubs_epi16(__mmask32 __U, __m512i __X, __m512i __Y) {
-  // CHECK-LABEL: @test_mm512_maskz_maddubs_epi16
+  // CHECK-LABEL: test_mm512_maskz_maddubs_epi16
   // CHECK: @llvm.x86.avx512.pmaddubs.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_maddubs_epi16(__U,__X,__Y); 
 }
 __m512i test_mm512_madd_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_madd_epi16
+  // CHECK-LABEL: test_mm512_madd_epi16
   // CHECK: @llvm.x86.avx512.pmaddw.d.512
   return _mm512_madd_epi16(__A,__B); 
 }
 __m512i test_mm512_mask_madd_epi16(__m512i __W, __mmask16 __U, __m512i __A,      __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_madd_epi16
+  // CHECK-LABEL: test_mm512_mask_madd_epi16
   // CHECK: @llvm.x86.avx512.pmaddw.d.512
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_madd_epi16(__W,__U,__A,__B); 
 }
 __m512i test_mm512_maskz_madd_epi16(__mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_madd_epi16
+  // CHECK-LABEL: test_mm512_maskz_madd_epi16
   // CHECK: @llvm.x86.avx512.pmaddw.d.512
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_madd_epi16(__U,__A,__B); 
 }
 
 __m256i test_mm512_cvtsepi16_epi8(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvtsepi16_epi8
+  // CHECK-LABEL: test_mm512_cvtsepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.wb.512
   return _mm512_cvtsepi16_epi8(__A); 
 }
 
 __m256i test_mm512_mask_cvtsepi16_epi8(__m256i __O, __mmask32 __M, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtsepi16_epi8
+  // CHECK-LABEL: test_mm512_mask_cvtsepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.wb.512
   return _mm512_mask_cvtsepi16_epi8(__O, __M, __A); 
 }
 
 __m256i test_mm512_maskz_cvtsepi16_epi8(__mmask32 __M, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtsepi16_epi8
+  // CHECK-LABEL: test_mm512_maskz_cvtsepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.wb.512
   return _mm512_maskz_cvtsepi16_epi8(__M, __A); 
 }
 
 __m256i test_mm512_cvtusepi16_epi8(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvtusepi16_epi8
+  // CHECK-LABEL: test_mm512_cvtusepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.wb.512
   return _mm512_cvtusepi16_epi8(__A); 
 }
 
 __m256i test_mm512_mask_cvtusepi16_epi8(__m256i __O, __mmask32 __M, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtusepi16_epi8
+  // CHECK-LABEL: test_mm512_mask_cvtusepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.wb.512
   return _mm512_mask_cvtusepi16_epi8(__O, __M, __A); 
 }
 
 __m256i test_mm512_maskz_cvtusepi16_epi8(__mmask32 __M, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtusepi16_epi8
+  // CHECK-LABEL: test_mm512_maskz_cvtusepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.wb.512
   return _mm512_maskz_cvtusepi16_epi8(__M, __A); 
 }
 
 __m256i test_mm512_cvtepi16_epi8(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvtepi16_epi8
+  // CHECK-LABEL: test_mm512_cvtepi16_epi8
   // CHECK: trunc <32 x i16> %{{.*}} to <32 x i8>
   return _mm512_cvtepi16_epi8(__A); 
 }
 
 __m256i test_mm512_mask_cvtepi16_epi8(__m256i __O, __mmask32 __M, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtepi16_epi8
+  // CHECK-LABEL: test_mm512_mask_cvtepi16_epi8
   // CHECK: trunc <32 x i16> %{{.*}} to <32 x i8>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm512_mask_cvtepi16_epi8(__O, __M, __A); 
 }
 
 __m256i test_mm512_maskz_cvtepi16_epi8(__mmask32 __M, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtepi16_epi8
+  // CHECK-LABEL: test_mm512_maskz_cvtepi16_epi8
   // CHECK: trunc <32 x i16> %{{.*}} to <32 x i8>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm512_maskz_cvtepi16_epi8(__M, __A); 
 }
 
 __m512i test_mm512_unpackhi_epi8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_unpackhi_epi8
+  // CHECK-LABEL: test_mm512_unpackhi_epi8
   // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
   return _mm512_unpackhi_epi8(__A, __B); 
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_unpackhi_epi8((__m512i)(__v64qi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}, (__m512i)(__v64qi){64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127}), 8, 72, 9, 73, 10, 74, 11, 75, 12, 76, 13, 77, 14, 78, 15, 79, 24, 88, 25, 89, 26, 90, 27, 91, 28, 92, 29, 93, 30, 94, 31, 95, 40, 104, 41, 105, 42, 106, 43, 107, 44, 108, 45, 109, 46, 110, 47, 111, 56, 120, 57, 121, 58, 122, 59, 123, 60, 124, 61, 125, 62, 126, 63, 127));
 
 __m512i test_mm512_mask_unpackhi_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_unpackhi_epi8
+  // CHECK-LABEL: test_mm512_mask_unpackhi_epi8
   // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_unpackhi_epi8(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_unpackhi_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_unpackhi_epi8
+  // CHECK-LABEL: test_mm512_maskz_unpackhi_epi8
   // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i32> <i32 8, i32 72, i32 9, i32 73, i32 10, i32 74, i32 11, i32 75, i32 12, i32 76, i32 13, i32 77, i32 14, i32 78, i32 15, i32 79, i32 24, i32 88, i32 25, i32 89, i32 26, i32 90, i32 27, i32 91, i32 28, i32 92, i32 29, i32 93, i32 30, i32 94, i32 31, i32 95, i32 40, i32 104, i32 41, i32 105, i32 42, i32 106, i32 43, i32 107, i32 44, i32 108, i32 45, i32 109, i32 46, i32 110, i32 47, i32 111, i32 56, i32 120, i32 57, i32 121, i32 58, i32 122, i32 59, i32 123, i32 60, i32 124, i32 61, i32 125, i32 62, i32 126, i32 63, i32 127>
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_unpackhi_epi8(__U, __A, __B); 
 }
 
 __m512i test_mm512_unpackhi_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_unpackhi_epi16
+  // CHECK-LABEL: test_mm512_unpackhi_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   return _mm512_unpackhi_epi16(__A, __B); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_unpackhi_epi16((__m512i)(__v32hi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}, (__m512i)(__v32hi){32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}), 4, 36, 5, 37, 6, 38, 7, 39, 12, 44, 13, 45, 14, 46, 15, 47, 20, 52, 21, 53, 22, 54, 23, 55, 28, 60, 29, 61, 30, 62, 31, 63));
+
 
 __m512i test_mm512_mask_unpackhi_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_unpackhi_epi16
+  // CHECK-LABEL: test_mm512_mask_unpackhi_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_unpackhi_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_unpackhi_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_unpackhi_epi16
+  // CHECK-LABEL: test_mm512_maskz_unpackhi_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i32> <i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_unpackhi_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_unpacklo_epi8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_unpacklo_epi8
+  // CHECK-LABEL: test_mm512_unpacklo_epi8
   // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
   return _mm512_unpacklo_epi8(__A, __B); 
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_unpacklo_epi8((__m512i)(__v64qi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}, (__m512i)(__v64qi){64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127}), 0, 64, 1, 65, 2, 66, 3, 67, 4, 68, 5, 69, 6, 70, 7, 71, 16, 80, 17, 81, 18, 82, 19, 83, 20, 84, 21, 85, 22, 86, 23, 87, 32, 96, 33, 97, 34, 98, 35, 99, 36, 100, 37, 101, 38, 102, 39, 103, 48, 112, 49, 113, 50, 114, 51, 115, 52, 116, 53, 117, 54, 118, 55, 119));
 
 __m512i test_mm512_mask_unpacklo_epi8(__m512i __W, __mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_unpacklo_epi8
+  // CHECK-LABEL: test_mm512_mask_unpacklo_epi8
   // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_unpacklo_epi8(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_unpacklo_epi8(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_unpacklo_epi8
+  // CHECK-LABEL: test_mm512_maskz_unpacklo_epi8
   // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_unpacklo_epi8(__U, __A, __B); 
 }
 
 __m512i test_mm512_unpacklo_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_unpacklo_epi16
+  // CHECK-LABEL: test_mm512_unpacklo_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59>
   return _mm512_unpacklo_epi16(__A, __B); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_unpacklo_epi16((__m512i)(__v32hi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}, (__m512i)(__v32hi){32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}), 0, 32, 1, 33, 2, 34, 3, 35, 8, 40, 9, 41, 10, 42, 11, 43, 16, 48, 17, 49, 18, 50, 19, 51, 24, 56, 25, 57, 26, 58, 27, 59));
 
 __m512i test_mm512_mask_unpacklo_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_unpacklo_epi16
+  // CHECK-LABEL: test_mm512_mask_unpacklo_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_unpacklo_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_unpacklo_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_unpacklo_epi16
+  // CHECK-LABEL: test_mm512_maskz_unpacklo_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_unpacklo_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_cvtepi8_epi16(__m256i __A) {
-  // CHECK-LABEL: @test_mm512_cvtepi8_epi16
+  // CHECK-LABEL: test_mm512_cvtepi8_epi16
   // CHECK: sext <32 x i8> %{{.*}} to <32 x i16>
   return _mm512_cvtepi8_epi16(__A); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_cvtepi8_epi16((__m256i)(__v32qs){-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12, 13, -14, 15, -16, 17, -18, 19, -20, 21, -22, 23, -24, 25, -26, 27, -28}), -3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12, 13, -14, 15, -16, 17, -18, 19, -20, 21, -22, 23, -24, 25, -26, 27, -28));
 
 __m512i test_mm512_mask_cvtepi8_epi16(__m512i __W, __mmask32 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtepi8_epi16
+  // CHECK-LABEL: test_mm512_mask_cvtepi8_epi16
   // CHECK: sext <32 x i8> %{{.*}} to <32 x i16>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_cvtepi8_epi16(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvtepi8_epi16(__mmask32 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtepi8_epi16
+  // CHECK-LABEL: test_mm512_maskz_cvtepi8_epi16
   // CHECK: sext <32 x i8> %{{.*}} to <32 x i16>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_cvtepi8_epi16(__U, __A); 
 }
 
 __m512i test_mm512_cvtepu8_epi16(__m256i __A) {
-  // CHECK-LABEL: @test_mm512_cvtepu8_epi16
+  // CHECK-LABEL: test_mm512_cvtepu8_epi16
   // CHECK: zext <32 x i8> %{{.*}} to <32 x i16>
   return _mm512_cvtepu8_epi16(__A); 
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_cvtepu8_epi16((__m256i)(__v32qs){-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12, 13, -14, 15, -16, 17, -18, 19, -20, 21, -22, 23, -24, 25, -26, 27, -28}), 253, 2, 255, 0, 1, 254, 3, 252, 5, 250, 7, 248, 9, 246, 11, 244, 13, 242, 15, 240, 17, 238, 19, 236, 21, 234, 23, 232, 25, 230, 27, 228));
 
 __m512i test_mm512_mask_cvtepu8_epi16(__m512i __W, __mmask32 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtepu8_epi16
+  // CHECK-LABEL: test_mm512_mask_cvtepu8_epi16
   // CHECK: zext <32 x i8> %{{.*}} to <32 x i16>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_cvtepu8_epi16(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvtepu8_epi16(__mmask32 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtepu8_epi16
+  // CHECK-LABEL: test_mm512_maskz_cvtepu8_epi16
   // CHECK: zext <32 x i8> %{{.*}} to <32 x i16>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_cvtepu8_epi16(__U, __A); 
 }
 
 __m512i test_mm512_shufflehi_epi16(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_shufflehi_epi16
+  // CHECK-LABEL: test_mm512_shufflehi_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 13, i32 13, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 21, i32 21, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 29, i32 29, i32 28, i32 28>
   return _mm512_shufflehi_epi16(__A, 5); 
 }
 
 __m512i test_mm512_mask_shufflehi_epi16(__m512i __W, __mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_shufflehi_epi16
+  // CHECK-LABEL: test_mm512_mask_shufflehi_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 13, i32 13, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 21, i32 21, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 29, i32 29, i32 28, i32 28>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_shufflehi_epi16(__W, __U, __A, 5); 
 }
 
 __m512i test_mm512_maskz_shufflehi_epi16(__mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_shufflehi_epi16
+  // CHECK-LABEL: test_mm512_maskz_shufflehi_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 13, i32 13, i32 12, i32 12, i32 16, i32 17, i32 18, i32 19, i32 21, i32 21, i32 20, i32 20, i32 24, i32 25, i32 26, i32 27, i32 29, i32 29, i32 28, i32 28>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_shufflehi_epi16(__U, __A, 5); 
 }
 
 __m512i test_mm512_shufflelo_epi16(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_shufflelo_epi16
+  // CHECK-LABEL: test_mm512_shufflelo_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 1, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 9, i32 9, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 17, i32 17, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 25, i32 25, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
   return _mm512_shufflelo_epi16(__A, 5); 
 }
 
 __m512i test_mm512_mask_shufflelo_epi16(__m512i __W, __mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_shufflelo_epi16
+  // CHECK-LABEL: test_mm512_mask_shufflelo_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 1, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 9, i32 9, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 17, i32 17, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 25, i32 25, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_shufflelo_epi16(__W, __U, __A, 5); 
 }
 
 __m512i test_mm512_maskz_shufflelo_epi16(__mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_shufflelo_epi16
+  // CHECK-LABEL: test_mm512_maskz_shufflelo_epi16
   // CHECK: shufflevector <32 x i16> %{{.*}}, <32 x i16> poison, <32 x i32> <i32 1, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 9, i32 9, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 17, i32 17, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 25, i32 25, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_shufflelo_epi16(__U, __A, 5); 
 }
 
 __m512i test_mm512_sllv_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_sllv_epi16
+  // CHECK-LABEL: test_mm512_sllv_epi16
   // CHECK: @llvm.x86.avx512.psllv.w.512(
   return _mm512_sllv_epi16(__A, __B); 
 }
 
 __m512i test_mm512_mask_sllv_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_sllv_epi16
+  // CHECK-LABEL: test_mm512_mask_sllv_epi16
   // CHECK: @llvm.x86.avx512.psllv.w.512(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_sllv_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_sllv_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_sllv_epi16
+  // CHECK-LABEL: test_mm512_maskz_sllv_epi16
   // CHECK: @llvm.x86.avx512.psllv.w.512(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_sllv_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_sll_epi16(__m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_sll_epi16
+  // CHECK-LABEL: test_mm512_sll_epi16
   // CHECK: @llvm.x86.avx512.psll.w.512
   return _mm512_sll_epi16(__A, __B); 
 }
 
 __m512i test_mm512_mask_sll_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_mask_sll_epi16
+  // CHECK-LABEL: test_mm512_mask_sll_epi16
   // CHECK: @llvm.x86.avx512.psll.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_sll_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_sll_epi16(__mmask32 __U, __m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_sll_epi16
+  // CHECK-LABEL: test_mm512_maskz_sll_epi16
   // CHECK: @llvm.x86.avx512.psll.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_sll_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_slli_epi16(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_slli_epi16
+  // CHECK-LABEL: test_mm512_slli_epi16
   // CHECK: @llvm.x86.avx512.pslli.w.512
   return _mm512_slli_epi16(__A, 5); 
 }
 
 __m512i test_mm512_slli_epi16_2(__m512i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm512_slli_epi16_2
+  // CHECK-LABEL: test_mm512_slli_epi16_2
   // CHECK: @llvm.x86.avx512.pslli.w.512
   return _mm512_slli_epi16(__A, __B); 
 }
 
 __m512i test_mm512_mask_slli_epi16(__m512i __W, __mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_slli_epi16
+  // CHECK-LABEL: test_mm512_mask_slli_epi16
   // CHECK: @llvm.x86.avx512.pslli.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_slli_epi16(__W, __U, __A, 5); 
 }
 
 __m512i test_mm512_mask_slli_epi16_2(__m512i __W, __mmask32 __U, __m512i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm512_mask_slli_epi16_2
+  // CHECK-LABEL: test_mm512_mask_slli_epi16_2
   // CHECK: @llvm.x86.avx512.pslli.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_slli_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_slli_epi16(__mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_slli_epi16
+  // CHECK-LABEL: test_mm512_maskz_slli_epi16
   // CHECK: @llvm.x86.avx512.pslli.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_slli_epi16(__U, __A, 5); 
 }
 
 __m512i test_mm512_maskz_slli_epi16_2(__mmask32 __U, __m512i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm512_maskz_slli_epi16_2
+  // CHECK-LABEL: test_mm512_maskz_slli_epi16_2
   // CHECK: @llvm.x86.avx512.pslli.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_slli_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_bslli_epi128(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_bslli_epi128
+  // CHECK-LABEL: test_mm512_bslli_epi128
   // CHECK: shufflevector <64 x i8> zeroinitializer, <64 x i8> %{{.*}}, <64 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 85, i32 86, i32 87, i32 88, i32 89, i32 90, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122>
   return _mm512_bslli_epi128(__A, 5);
 }
 
 __m512i test_mm512_srlv_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_srlv_epi16
+  // CHECK-LABEL: test_mm512_srlv_epi16
   // CHECK: @llvm.x86.avx512.psrlv.w.512(
   return _mm512_srlv_epi16(__A, __B); 
 }
 
 __m512i test_mm512_mask_srlv_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_srlv_epi16
+  // CHECK-LABEL: test_mm512_mask_srlv_epi16
   // CHECK: @llvm.x86.avx512.psrlv.w.512(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_srlv_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_srlv_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_srlv_epi16
+  // CHECK-LABEL: test_mm512_maskz_srlv_epi16
   // CHECK: @llvm.x86.avx512.psrlv.w.512(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_srlv_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_srav_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_srav_epi16
+  // CHECK-LABEL: test_mm512_srav_epi16
   // CHECK: @llvm.x86.avx512.psrav.w.512(
   return _mm512_srav_epi16(__A, __B); 
 }
 
 __m512i test_mm512_mask_srav_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_srav_epi16
+  // CHECK-LABEL: test_mm512_mask_srav_epi16
   // CHECK: @llvm.x86.avx512.psrav.w.512(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_srav_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_srav_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_srav_epi16
+  // CHECK-LABEL: test_mm512_maskz_srav_epi16
   // CHECK: @llvm.x86.avx512.psrav.w.512(
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_srav_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_sra_epi16(__m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_sra_epi16
+  // CHECK-LABEL: test_mm512_sra_epi16
   // CHECK: @llvm.x86.avx512.psra.w.512
   return _mm512_sra_epi16(__A, __B); 
 }
 
 __m512i test_mm512_mask_sra_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_mask_sra_epi16
+  // CHECK-LABEL: test_mm512_mask_sra_epi16
   // CHECK: @llvm.x86.avx512.psra.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_sra_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_sra_epi16(__mmask32 __U, __m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_sra_epi16
+  // CHECK-LABEL: test_mm512_maskz_sra_epi16
   // CHECK: @llvm.x86.avx512.psra.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_sra_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_srai_epi16(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_srai_epi16
+  // CHECK-LABEL: test_mm512_srai_epi16
   // CHECK: @llvm.x86.avx512.psrai.w.512
   return _mm512_srai_epi16(__A, 5); 
 }
 
 __m512i test_mm512_srai_epi16_2(__m512i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm512_srai_epi16_2
+  // CHECK-LABEL: test_mm512_srai_epi16_2
   // CHECK: @llvm.x86.avx512.psrai.w.512
   return _mm512_srai_epi16(__A, __B); 
 }
 
 __m512i test_mm512_mask_srai_epi16(__m512i __W, __mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_srai_epi16
+  // CHECK-LABEL: test_mm512_mask_srai_epi16
   // CHECK: @llvm.x86.avx512.psrai.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_srai_epi16(__W, __U, __A, 5); 
 }
 
 __m512i test_mm512_mask_srai_epi16_2(__m512i __W, __mmask32 __U, __m512i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm512_mask_srai_epi16_2
+  // CHECK-LABEL: test_mm512_mask_srai_epi16_2
   // CHECK: @llvm.x86.avx512.psrai.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_srai_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_srai_epi16(__mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_srai_epi16
+  // CHECK-LABEL: test_mm512_maskz_srai_epi16
   // CHECK: @llvm.x86.avx512.psrai.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_srai_epi16(__U, __A, 5); 
 }
 
 __m512i test_mm512_maskz_srai_epi16_2(__mmask32 __U, __m512i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm512_maskz_srai_epi16_2
+  // CHECK-LABEL: test_mm512_maskz_srai_epi16_2
   // CHECK: @llvm.x86.avx512.psrai.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_srai_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_srl_epi16(__m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_srl_epi16
+  // CHECK-LABEL: test_mm512_srl_epi16
   // CHECK: @llvm.x86.avx512.psrl.w.512
   return _mm512_srl_epi16(__A, __B); 
 }
 
 __m512i test_mm512_mask_srl_epi16(__m512i __W, __mmask32 __U, __m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_mask_srl_epi16
+  // CHECK-LABEL: test_mm512_mask_srl_epi16
   // CHECK: @llvm.x86.avx512.psrl.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_srl_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_srl_epi16(__mmask32 __U, __m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_srl_epi16
+  // CHECK-LABEL: test_mm512_maskz_srl_epi16
   // CHECK: @llvm.x86.avx512.psrl.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_srl_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_srli_epi16(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_srli_epi16
+  // CHECK-LABEL: test_mm512_srli_epi16
   // CHECK: @llvm.x86.avx512.psrli.w.512
   return _mm512_srli_epi16(__A, 5); 
 }
 
 __m512i test_mm512_srli_epi16_2(__m512i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm512_srli_epi16_2
+  // CHECK-LABEL: test_mm512_srli_epi16_2
   // CHECK: @llvm.x86.avx512.psrli.w.512
   return _mm512_srli_epi16(__A, __B); 
 }
 
 __m512i test_mm512_mask_srli_epi16(__m512i __W, __mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_srli_epi16
+  // CHECK-LABEL: test_mm512_mask_srli_epi16
   // CHECK: @llvm.x86.avx512.psrli.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_srli_epi16(__W, __U, __A, 5); 
 }
 
 __m512i test_mm512_mask_srli_epi16_2(__m512i __W, __mmask32 __U, __m512i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm512_mask_srli_epi16_2
+  // CHECK-LABEL: test_mm512_mask_srli_epi16_2
   // CHECK: @llvm.x86.avx512.psrli.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_srli_epi16(__W, __U, __A, __B); 
 }
 
 __m512i test_mm512_maskz_srli_epi16(__mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_srli_epi16
+  // CHECK-LABEL: test_mm512_maskz_srli_epi16
   // CHECK: @llvm.x86.avx512.psrli.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_srli_epi16(__U, __A, 5); 
 }
 
 __m512i test_mm512_maskz_srli_epi16_2(__mmask32 __U, __m512i __A, int __B) {
-  // CHECK-LABEL: @test_mm512_maskz_srli_epi16_2
+  // CHECK-LABEL: test_mm512_maskz_srli_epi16_2
   // CHECK: @llvm.x86.avx512.psrli.w.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_srli_epi16(__U, __A, __B); 
 }
 
 __m512i test_mm512_bsrli_epi128(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_bsrli_epi128
+  // CHECK-LABEL: test_mm512_bsrli_epi128
   // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> zeroinitializer, <64 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 66, i32 67, i32 68, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 82, i32 83, i32 84, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 98, i32 99, i32 100, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113, i32 114, i32 115, i32 116>
   return _mm512_bsrli_epi128(__A, 5);
 }
 __m512i test_mm512_mask_mov_epi16(__m512i __W, __mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_mov_epi16
+  // CHECK-LABEL: test_mm512_mask_mov_epi16
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_mov_epi16(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_mov_epi16(__mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_mov_epi16
+  // CHECK-LABEL: test_mm512_maskz_mov_epi16
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_mov_epi16(__U, __A); 
 }
 
 __m512i test_mm512_mask_mov_epi8(__m512i __W, __mmask64 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_mov_epi8
+  // CHECK-LABEL: test_mm512_mask_mov_epi8
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_mov_epi8(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_mov_epi8(__mmask64 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_mov_epi8
+  // CHECK-LABEL: test_mm512_maskz_mov_epi8
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_mov_epi8(__U, __A); 
 }
 
 __m512i test_mm512_mask_set1_epi8(__m512i __O, __mmask64 __M, char __A) {
-  // CHECK-LABEL: @test_mm512_mask_set1_epi8
+  // CHECK-LABEL: test_mm512_mask_set1_epi8
   // CHECK: insertelement <64 x i8> poison, i8 %{{.*}}, i32 0
   // CHECK: insertelement <64 x i8> %{{.*}}, i8 %{{.*}}, i32 1
   // CHECK: insertelement <64 x i8> %{{.*}}, i8 %{{.*}}, i32 2
@@ -1956,7 +1992,7 @@ __m512i test_mm512_mask_set1_epi8(__m512i __O, __mmask64 __M, char __A) {
 }
 
 __m512i test_mm512_maskz_set1_epi8(__mmask64 __M, char __A) {
-  // CHECK-LABEL: @test_mm512_maskz_set1_epi8
+  // CHECK-LABEL: test_mm512_maskz_set1_epi8
   // CHECK: insertelement <64 x i8> poison, i8 %{{.*}}, i32 0
   // CHECK: insertelement <64 x i8> %{{.*}}, i8 %{{.*}}, i32 1
   // CHECK: insertelement <64 x i8> %{{.*}}, i8 %{{.*}}, i32 2
@@ -2026,7 +2062,7 @@ __m512i test_mm512_maskz_set1_epi8(__mmask64 __M, char __A) {
 }
 
 __mmask64 test_mm512_kunpackd(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_mm512_kunpackd
+  // CHECK-LABEL: test_mm512_kunpackd
   // CHECK: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK: [[LHS2:%.*]] = shufflevector <64 x i1> [[LHS]], <64 x i1> [[LHS]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -2036,7 +2072,7 @@ __mmask64 test_mm512_kunpackd(__m512i __A, __m512i __B, __m512i __C, __m512i __D
 }
 
 __mmask32 test_mm512_kunpackw(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_mm512_kunpackw
+  // CHECK-LABEL: test_mm512_kunpackw
   // CHECK: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: [[LHS2:%.*]] = shufflevector <32 x i1> [[LHS]], <32 x i1> [[LHS]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -2047,74 +2083,74 @@ __mmask32 test_mm512_kunpackw(__m512i __A, __m512i __B, __m512i __C, __m512i __D
 
 __m512i test_mm512_loadu_epi16 (void *__P)
 {
-  // CHECK-LABEL: @test_mm512_loadu_epi16
+  // CHECK-LABEL: test_mm512_loadu_epi16
   // CHECK: load <8 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm512_loadu_epi16 (__P);
 }
 
 __m512i test_mm512_mask_loadu_epi16(__m512i __W, __mmask32 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm512_mask_loadu_epi16
+  // CHECK-LABEL: test_mm512_mask_loadu_epi16
   // CHECK: @llvm.masked.load.v32i16.p0(ptr %{{.*}}, i32 1, <32 x i1> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_mask_loadu_epi16(__W, __U, __P); 
 }
 
 __m512i test_mm512_maskz_loadu_epi16(__mmask32 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm512_maskz_loadu_epi16
+  // CHECK-LABEL: test_mm512_maskz_loadu_epi16
   // CHECK: @llvm.masked.load.v32i16.p0(ptr %{{.*}}, i32 1, <32 x i1> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_maskz_loadu_epi16(__U, __P); 
 }
 
 __m512i test_mm512_loadu_epi8 (void *__P)
 {
-  // CHECK-LABEL: @test_mm512_loadu_epi8
+  // CHECK-LABEL: test_mm512_loadu_epi8
   // CHECK: load <8 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm512_loadu_epi8 (__P);
 }
 
 __m512i test_mm512_mask_loadu_epi8(__m512i __W, __mmask64 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm512_mask_loadu_epi8
+  // CHECK-LABEL: test_mm512_mask_loadu_epi8
   // CHECK: @llvm.masked.load.v64i8.p0(ptr %{{.*}}, i32 1, <64 x i1> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_mask_loadu_epi8(__W, __U, __P); 
 }
 
 __m512i test_mm512_maskz_loadu_epi8(__mmask64 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm512_maskz_loadu_epi8
+  // CHECK-LABEL: test_mm512_maskz_loadu_epi8
   // CHECK: @llvm.masked.load.v64i8.p0(ptr %{{.*}}, i32 1, <64 x i1> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_maskz_loadu_epi8(__U, __P); 
 }
 
 void test_mm512_storeu_epi16(void *__P, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_storeu_epi16
+  // CHECK-LABEL: test_mm512_storeu_epi16
   // CHECK: store <8 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm512_storeu_epi16(__P, __A); 
 }
 
 void test_mm512_mask_storeu_epi16(void *__P, __mmask32 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_storeu_epi16
+  // CHECK-LABEL: test_mm512_mask_storeu_epi16
   // CHECK: @llvm.masked.store.v32i16.p0(<32 x i16> %{{.*}}, ptr %{{.*}}, i32 1, <32 x i1> %{{.*}})
   return _mm512_mask_storeu_epi16(__P, __U, __A);
 }
 
 __mmask64 test_mm512_test_epi8_mask(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_test_epi8_mask
+  // CHECK-LABEL: test_mm512_test_epi8_mask
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <64 x i8> %{{.*}}, %{{.*}}
   return _mm512_test_epi8_mask(__A, __B); 
 }
 
 void test_mm512_storeu_epi8(void *__P, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_storeu_epi8
+  // CHECK-LABEL: test_mm512_storeu_epi8
   // CHECK: store <8 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm512_storeu_epi8(__P, __A);
 }
 
 void test_mm512_mask_storeu_epi8(void *__P, __mmask64 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_storeu_epi8
+  // CHECK-LABEL: test_mm512_mask_storeu_epi8
   // CHECK: @llvm.masked.store.v64i8.p0(<64 x i8> %{{.*}}, ptr %{{.*}}, i32 1, <64 x i1> %{{.*}})
   return _mm512_mask_storeu_epi8(__P, __U, __A); 
 }
 __mmask64 test_mm512_mask_test_epi8_mask(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_test_epi8_mask
+  // CHECK-LABEL: test_mm512_mask_test_epi8_mask
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
@@ -2122,14 +2158,14 @@ __mmask64 test_mm512_mask_test_epi8_mask(__mmask64 __U, __m512i __A, __m512i __B
 }
 
 __mmask32 test_mm512_test_epi16_mask(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_test_epi16_mask
+  // CHECK-LABEL: test_mm512_test_epi16_mask
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <32 x i16> %{{.*}}, %{{.*}}
   return _mm512_test_epi16_mask(__A, __B); 
 }
 
 __mmask32 test_mm512_mask_test_epi16_mask(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_test_epi16_mask
+  // CHECK-LABEL: test_mm512_mask_test_epi16_mask
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
@@ -2137,14 +2173,14 @@ __mmask32 test_mm512_mask_test_epi16_mask(__mmask32 __U, __m512i __A, __m512i __
 }
 
 __mmask64 test_mm512_testn_epi8_mask(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_testn_epi8_mask
+  // CHECK-LABEL: test_mm512_testn_epi8_mask
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   return _mm512_testn_epi8_mask(__A, __B); 
 }
 
 __mmask64 test_mm512_mask_testn_epi8_mask(__mmask64 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_testn_epi8_mask
+  // CHECK-LABEL: test_mm512_mask_testn_epi8_mask
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <64 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <64 x i1> %{{.*}}, %{{.*}}
@@ -2152,14 +2188,14 @@ __mmask64 test_mm512_mask_testn_epi8_mask(__mmask64 __U, __m512i __A, __m512i __
 }
 
 __mmask32 test_mm512_testn_epi16_mask(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_testn_epi16_mask
+  // CHECK-LABEL: test_mm512_testn_epi16_mask
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   return _mm512_testn_epi16_mask(__A, __B); 
 }
 
 __mmask32 test_mm512_mask_testn_epi16_mask(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_testn_epi16_mask
+  // CHECK-LABEL: test_mm512_mask_testn_epi16_mask
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <32 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
@@ -2167,67 +2203,69 @@ __mmask32 test_mm512_mask_testn_epi16_mask(__mmask32 __U, __m512i __A, __m512i _
 }
 
 __mmask64 test_mm512_movepi8_mask(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_movepi8_mask
+  // CHECK-LABEL: test_mm512_movepi8_mask
   // CHECK: [[CMP:%.*]] = icmp slt <64 x i8> %{{.*}}, zeroinitializer
   return _mm512_movepi8_mask(__A); 
 }
 
 __m512i test_mm512_movm_epi8(__mmask64 __A) {
-  // CHECK-LABEL: @test_mm512_movm_epi8
+  // CHECK-LABEL: test_mm512_movm_epi8
   // CHECK:  %{{.*}} = bitcast i64 %{{.*}} to <64 x i1>
   // CHECK:  %vpmovm2.i = sext <64 x i1> %{{.*}} to <64 x i8>
   return _mm512_movm_epi8(__A); 
 }
 
 __m512i test_mm512_movm_epi16(__mmask32 __A) {
-  // CHECK-LABEL: @test_mm512_movm_epi16
+  // CHECK-LABEL: test_mm512_movm_epi16
   // CHECK:  %{{.*}} = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK:  %vpmovm2.i = sext <32 x i1> %{{.*}} to <32 x i16>
   return _mm512_movm_epi16(__A); 
 }
 
 __m512i test_mm512_broadcastb_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm512_broadcastb_epi8
+  // CHECK-LABEL: test_mm512_broadcastb_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <64 x i32> zeroinitializer
   return _mm512_broadcastb_epi8(__A);
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_broadcastb_epi8((__m128i)(__v16qi){42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}), 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42));
 
 __m512i test_mm512_mask_broadcastb_epi8(__m512i __O, __mmask64 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm512_mask_broadcastb_epi8
+  // CHECK-LABEL: test_mm512_mask_broadcastb_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <64 x i32> zeroinitializer
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_broadcastb_epi8(__O, __M, __A);
 }
 
 __m512i test_mm512_maskz_broadcastb_epi8(__mmask64 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_broadcastb_epi8
+  // CHECK-LABEL: test_mm512_maskz_broadcastb_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <64 x i32> zeroinitializer
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_broadcastb_epi8(__M, __A);
 }
 
 __m512i test_mm512_broadcastw_epi16(__m128i __A) {
-  // CHECK-LABEL: @test_mm512_broadcastw_epi16
+  // CHECK-LABEL: test_mm512_broadcastw_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <32 x i32> zeroinitializer
   return _mm512_broadcastw_epi16(__A);
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_broadcastw_epi16((__m128i)(__v8hi){42, 3, 10, 8, 0, 256, 256, 128}), 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42));
 
 __m512i test_mm512_mask_broadcastw_epi16(__m512i __O, __mmask32 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm512_mask_broadcastw_epi16
+  // CHECK-LABEL: test_mm512_mask_broadcastw_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <32 x i32> zeroinitializer
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_broadcastw_epi16(__O, __M, __A);
 }
 
 __m512i test_mm512_maskz_broadcastw_epi16(__mmask32 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_broadcastw_epi16
+  // CHECK-LABEL: test_mm512_maskz_broadcastw_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <32 x i32> zeroinitializer
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_broadcastw_epi16(__M, __A);
 }
 
 __m512i test_mm512_mask_set1_epi16(__m512i __O, __mmask32 __M, short __A) {
-  // CHECK-LABEL: @test_mm512_mask_set1_epi16
+  // CHECK-LABEL: test_mm512_mask_set1_epi16
   // CHECK: insertelement <32 x i16> poison, i16 %{{.*}}, i32 0
   // CHECK: insertelement <32 x i16> %{{.*}}, i16 %{{.*}}, i32 1
   // CHECK: insertelement <32 x i16> %{{.*}}, i16 %{{.*}}, i32 2
@@ -2265,7 +2303,7 @@ __m512i test_mm512_mask_set1_epi16(__m512i __O, __mmask32 __M, short __A) {
 }
 
 __m512i test_mm512_maskz_set1_epi16(__mmask32 __M, short __A) {
-  // CHECK-LABEL: @test_mm512_maskz_set1_epi16
+  // CHECK-LABEL: test_mm512_maskz_set1_epi16
   // CHECK: insertelement <32 x i16> poison, i16 %{{.*}}, i32 0
   // CHECK: insertelement <32 x i16> %{{.*}}, i16 %{{.*}}, i32 1
   // CHECK: insertelement <32 x i16> %{{.*}}, i16 %{{.*}}, i32 2
@@ -2302,39 +2340,39 @@ __m512i test_mm512_maskz_set1_epi16(__mmask32 __M, short __A) {
   return _mm512_maskz_set1_epi16(__M, __A); 
 }
 __m512i test_mm512_permutexvar_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_permutexvar_epi16
+  // CHECK-LABEL: test_mm512_permutexvar_epi16
   // CHECK: @llvm.x86.avx512.permvar.hi.512
  return _mm512_permutexvar_epi16(__A, __B); 
 }
 
 __m512i test_mm512_maskz_permutexvar_epi16(__mmask32 __M, __m512i __A, __m512i __B) {
- // CHECK-LABEL: @test_mm512_maskz_permutexvar_epi16
+ // CHECK-LABEL: test_mm512_maskz_permutexvar_epi16
   // CHECK: @llvm.x86.avx512.permvar.hi.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_permutexvar_epi16(__M, __A, __B); 
 }
 
 __m512i test_mm512_mask_permutexvar_epi16(__m512i __W, __mmask32 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_permutexvar_epi16
+  // CHECK-LABEL: test_mm512_mask_permutexvar_epi16
   // CHECK: @llvm.x86.avx512.permvar.hi.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_permutexvar_epi16(__W, __M, __A, __B); 
 }
 __m512i test_mm512_alignr_epi8(__m512i __A,__m512i __B){
-    // CHECK-LABEL: @test_mm512_alignr_epi8
+    // CHECK-LABEL: test_mm512_alignr_epi8
     // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113>
     return _mm512_alignr_epi8(__A, __B, 2); 
 }
 
 __m512i test_mm512_mask_alignr_epi8(__m512i __W, __mmask64 __U, __m512i __A,__m512i __B){
-    // CHECK-LABEL: @test_mm512_mask_alignr_epi8
+    // CHECK-LABEL: test_mm512_mask_alignr_epi8
     // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113>
     // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
     return _mm512_mask_alignr_epi8(__W, __U, __A, __B, 2); 
 }
 
 __m512i test_mm512_maskz_alignr_epi8(__mmask64 __U, __m512i __A,__m512i __B){
-    // CHECK-LABEL: @test_mm512_maskz_alignr_epi8
+    // CHECK-LABEL: test_mm512_maskz_alignr_epi8
     // CHECK: shufflevector <64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 64, i32 65, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 80, i32 81, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 96, i32 97, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 112, i32 113>
     // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
    return _mm512_maskz_alignr_epi8(__U, __A, __B, 2); 
@@ -2343,54 +2381,54 @@ __m512i test_mm512_maskz_alignr_epi8(__mmask64 __U, __m512i __A,__m512i __B){
 
 
 __m512i test_mm512_mm_dbsad_epu8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mm_dbsad_epu8
+  // CHECK-LABEL: test_mm512_mm_dbsad_epu8
   // CHECK: @llvm.x86.avx512.dbpsadbw.512
   return _mm512_dbsad_epu8(__A, __B, 170); 
 }
 
 __m512i test_mm512_mm_mask_dbsad_epu8(__m512i __W, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mm_mask_dbsad_epu8
+  // CHECK-LABEL: test_mm512_mm_mask_dbsad_epu8
   // CHECK: @llvm.x86.avx512.dbpsadbw.512
   //CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_dbsad_epu8(__W, __U, __A, __B, 170); 
 }
 
 __m512i test_mm512_mm_maskz_dbsad_epu8(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mm_maskz_dbsad_epu8
+  // CHECK-LABEL: test_mm512_mm_maskz_dbsad_epu8
   // CHECK: @llvm.x86.avx512.dbpsadbw.512
   //CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_dbsad_epu8(__U, __A, __B, 170); 
 }
 
 __m512i test_mm512_sad_epu8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_sad_epu8
+  // CHECK-LABEL: test_mm512_sad_epu8
   // CHECK: @llvm.x86.avx512.psad.bw.512
   return _mm512_sad_epu8(__A, __B); 
 }
 
 __mmask32 test_mm512_movepi16_mask(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_movepi16_mask
+  // CHECK-LABEL: test_mm512_movepi16_mask
   // CHECK: [[CMP:%.*]] = icmp slt <32 x i16> %{{.*}}, zeroinitializer
   return _mm512_movepi16_mask(__A); 
 }
 
 void test_mm512_mask_cvtepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A)
 {
- // CHECK-LABEL: @test_mm512_mask_cvtepi16_storeu_epi8
+ // CHECK-LABEL: test_mm512_mask_cvtepi16_storeu_epi8
  // CHECK: @llvm.x86.avx512.mask.pmov.wb.mem.512
  _mm512_mask_cvtepi16_storeu_epi8 ( __P,  __M, __A);
 }
 
 void test_mm512_mask_cvtsepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A)
 {
- // CHECK-LABEL: @test_mm512_mask_cvtsepi16_storeu_epi8
+ // CHECK-LABEL: test_mm512_mask_cvtsepi16_storeu_epi8
  // CHECK: @llvm.x86.avx512.mask.pmovs.wb.mem.512
  _mm512_mask_cvtsepi16_storeu_epi8 ( __P,  __M, __A);
 }
 
 void test_mm512_mask_cvtusepi16_storeu_epi8 (void * __P, __mmask32 __M, __m512i __A)
 {
- // CHECK-LABEL: @test_mm512_mask_cvtusepi16_storeu_epi8
+ // CHECK-LABEL: test_mm512_mask_cvtusepi16_storeu_epi8
  // CHECK: @llvm.x86.avx512.mask.pmovus.wb.mem.512
  _mm512_mask_cvtusepi16_storeu_epi8 ( __P, __M, __A);
 }
diff --git a/clang/test/CodeGen/X86/avx512cd-builtins.c b/clang/test/CodeGen/X86/avx512cd-builtins.c
index 3c1415cd62565..45df0fccb322f 100644
--- a/clang/test/CodeGen/X86/avx512cd-builtins.c
+++ b/clang/test/CodeGen/X86/avx512cd-builtins.c
@@ -1,79 +1,82 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
 
 __m512i test_mm512_conflict_epi64(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_conflict_epi64
-  // CHECK: @llvm.x86.avx512.conflict.q.512
+  // CHECK-LABEL: test_mm512_conflict_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64> %{{.*}})
   return _mm512_conflict_epi64(__A); 
 }
 __m512i test_mm512_mask_conflict_epi64(__m512i __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_conflict_epi64
-  // CHECK: @llvm.x86.avx512.conflict.q.512
+  // CHECK-LABEL: test_mm512_mask_conflict_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_conflict_epi64(__W,__U,__A); 
 }
 __m512i test_mm512_maskz_conflict_epi64(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_conflict_epi64
-  // CHECK: @llvm.x86.avx512.conflict.q.512
+  // CHECK-LABEL: test_mm512_maskz_conflict_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_conflict_epi64(__U,__A); 
 }
 __m512i test_mm512_conflict_epi32(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_conflict_epi32
-  // CHECK: @llvm.x86.avx512.conflict.d.512
+  // CHECK-LABEL: test_mm512_conflict_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32> %{{.*}})
   return _mm512_conflict_epi32(__A); 
 }
 __m512i test_mm512_mask_conflict_epi32(__m512i __W, __mmask16 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_conflict_epi32
-  // CHECK: @llvm.x86.avx512.conflict.d.512
+  // CHECK-LABEL: test_mm512_mask_conflict_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_conflict_epi32(__W,__U,__A); 
 }
 __m512i test_mm512_maskz_conflict_epi32(__mmask16 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_conflict_epi32
-  // CHECK: @llvm.x86.avx512.conflict.d.512
+  // CHECK-LABEL: test_mm512_maskz_conflict_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_conflict_epi32(__U,__A); 
 }
 __m512i test_mm512_lzcnt_epi32(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_lzcnt_epi32
+  // CHECK-LABEL: test_mm512_lzcnt_epi32
   // CHECK: call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %{{.*}}, i1 false)
   return _mm512_lzcnt_epi32(__A); 
 }
 __m512i test_mm512_mask_lzcnt_epi32(__m512i __W, __mmask16 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_lzcnt_epi32
+  // CHECK-LABEL: test_mm512_mask_lzcnt_epi32
   // CHECK: call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %{{.*}}, i1 false)
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_lzcnt_epi32(__W,__U,__A); 
 }
 __m512i test_mm512_maskz_lzcnt_epi32(__mmask16 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_lzcnt_epi32
+  // CHECK-LABEL: test_mm512_maskz_lzcnt_epi32
   // CHECK: call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %{{.*}}, i1 false)
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_lzcnt_epi32(__U,__A); 
 }
 __m512i test_mm512_lzcnt_epi64(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_lzcnt_epi64
-  // CHECK: call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm512_lzcnt_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.ctlz.v8i64(<8 x i64> %{{.*}}, i1 false)
   return _mm512_lzcnt_epi64(__A); 
 }
 __m512i test_mm512_mask_lzcnt_epi64(__m512i __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_lzcnt_epi64
-  // CHECK: call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm512_mask_lzcnt_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.ctlz.v8i64(<8 x i64> %{{.*}}, i1 false)
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_lzcnt_epi64(__W,__U,__A); 
 }
 __m512i test_mm512_maskz_lzcnt_epi64(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_lzcnt_epi64
-  // CHECK: call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm512_maskz_lzcnt_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.ctlz.v8i64(<8 x i64> %{{.*}}, i1 false)
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_lzcnt_epi64(__U,__A); 
 }
 
 __m512i test_mm512_broadcastmb_epi64(__m512i a, __m512i b) {
-  // CHECK-LABEL: @test_mm512_broadcastmb_epi64
+  // CHECK-LABEL: test_mm512_broadcastmb_epi64
   // CHECK: icmp eq <8 x i64> %{{.*}}, %{{.*}}
   // CHECK: zext i8 %{{.*}} to i64
   // CHECK: insertelement <8 x i64> poison, i64 %{{.*}}, i32 0
@@ -88,7 +91,7 @@ __m512i test_mm512_broadcastmb_epi64(__m512i a, __m512i b) {
 }
 
 __m512i test_mm512_broadcastmw_epi32(__m512i a, __m512i b) {
-  // CHECK-LABEL: @test_mm512_broadcastmw_epi32
+  // CHECK-LABEL: test_mm512_broadcastmw_epi32
   // CHECK: icmp eq <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: zext i16 %{{.*}} to i32
   // CHECK: insertelement <16 x i32> poison, i32 %{{.*}}
diff --git a/clang/test/CodeGen/X86/avx512dq-builtins.c b/clang/test/CodeGen/X86/avx512dq-builtins.c
index 1ebd3696bce54..47e5a3ffeeff4 100644
--- a/clang/test/CodeGen/X86/avx512dq-builtins.c
+++ b/clang/test/CodeGen/X86/avx512dq-builtins.c
@@ -1,17 +1,21 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512dq -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __mmask8 test_knot_mask8(__mmask8 a) {
-  // CHECK-LABEL: @test_knot_mask8
+  // CHECK-LABEL: test_knot_mask8
   // CHECK: [[IN:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[NOT:%.*]] = xor <8 x i1> [[IN]], splat (i1 true)
   return _knot_mask8(a);
 }
 
 __mmask8 test_kand_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kand_mask8
+  // CHECK-LABEL: test_kand_mask8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RES:%.*]] = and <8 x i1> [[LHS]], [[RHS]]
@@ -21,7 +25,7 @@ __mmask8 test_kand_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m
 }
 
 __mmask8 test_kandn_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kandn_mask8
+  // CHECK-LABEL: test_kandn_mask8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[NOT:%.*]] = xor <8 x i1> [[LHS]], splat (i1 true)
@@ -32,7 +36,7 @@ __mmask8 test_kandn_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __
 }
 
 __mmask8 test_kor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kor_mask8
+  // CHECK-LABEL: test_kor_mask8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RES:%.*]] = or <8 x i1> [[LHS]], [[RHS]]
@@ -42,7 +46,7 @@ __mmask8 test_kor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m5
 }
 
 __mmask8 test_kxnor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kxnor_mask8
+  // CHECK-LABEL: test_kxnor_mask8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[NOT:%.*]] = xor <8 x i1> [[LHS]], splat (i1 true)
@@ -53,7 +57,7 @@ __mmask8 test_kxnor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __
 }
 
 __mmask8 test_kxor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kxor_mask8
+  // CHECK-LABEL: test_kxor_mask8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RES:%.*]] = xor <8 x i1> [[LHS]], [[RHS]]
@@ -63,7 +67,7 @@ __mmask8 test_kxor_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m
 }
 
 unsigned char test_kortestz_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_kortestz_mask8_u8
+  // CHECK-LABEL: test_kortestz_mask8_u8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[OR:%.*]] = or <8 x i1> [[LHS]], [[RHS]]
@@ -76,7 +80,7 @@ unsigned char test_kortestz_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m5
 }
 
 unsigned char test_kortestc_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_kortestc_mask8_u8
+  // CHECK-LABEL: test_kortestc_mask8_u8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[OR:%.*]] = or <8 x i1> [[LHS]], [[RHS]]
@@ -89,7 +93,7 @@ unsigned char test_kortestc_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m5
 }
 
 unsigned char test_kortest_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) {
-  // CHECK-LABEL: @test_kortest_mask8_u8
+  // CHECK-LABEL: test_kortest_mask8_u8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[OR:%.*]] = or <8 x i1> [[LHS]], [[RHS]]
@@ -109,7 +113,7 @@ unsigned char test_kortest_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m51
 }
 
 unsigned char test_ktestz_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_ktestz_mask8_u8
+  // CHECK-LABEL: test_ktestz_mask8_u8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestz.b(<8 x i1> [[LHS]], <8 x i1> [[RHS]])
@@ -119,7 +123,7 @@ unsigned char test_ktestz_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512
 }
 
 unsigned char test_ktestc_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_ktestc_mask8_u8
+  // CHECK-LABEL: test_ktestc_mask8_u8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.b(<8 x i1> [[LHS]], <8 x i1> [[RHS]])
@@ -129,7 +133,7 @@ unsigned char test_ktestc_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512
 }
 
 unsigned char test_ktest_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) {
-  // CHECK-LABEL: @test_ktest_mask8_u8
+  // CHECK-LABEL: test_ktest_mask8_u8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.b(<8 x i1> [[LHS]], <8 x i1> [[RHS]])
@@ -143,7 +147,7 @@ unsigned char test_ktest_mask8_u8(__m512i __A, __m512i __B, __m512i __C, __m512i
 }
 
 unsigned char test_ktestz_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_ktestz_mask16_u8
+  // CHECK-LABEL: test_ktestz_mask16_u8
   // CHECK: [[LHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestz.w(<16 x i1> [[LHS]], <16 x i1> [[RHS]])
@@ -153,7 +157,7 @@ unsigned char test_ktestz_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m51
 }
 
 unsigned char test_ktestc_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D) {
-  // CHECK-LABEL: @test_ktestc_mask16_u8
+  // CHECK-LABEL: test_ktestc_mask16_u8
   // CHECK: [[LHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.w(<16 x i1> [[LHS]], <16 x i1> [[RHS]])
@@ -163,7 +167,7 @@ unsigned char test_ktestc_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m51
 }
 
 unsigned char test_ktest_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, unsigned char *CF) {
-  // CHECK-LABEL: @test_ktest_mask16_u8
+  // CHECK-LABEL: test_ktest_mask16_u8
   // CHECK: [[LHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: [[RES:%.*]] = call i32 @llvm.x86.avx512.ktestc.w(<16 x i1> [[LHS]], <16 x i1> [[RHS]])
@@ -177,7 +181,7 @@ unsigned char test_ktest_mask16_u8(__m512i __A, __m512i __B, __m512i __C, __m512
 }
 
 __mmask8 test_kadd_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kadd_mask8
+  // CHECK-LABEL: test_kadd_mask8
   // CHECK: [[LHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RES:%.*]] = call <8 x i1> @llvm.x86.avx512.kadd.b(<8 x i1> [[LHS]], <8 x i1> [[RHS]])
@@ -187,7 +191,7 @@ __mmask8 test_kadd_mask8(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m
 }
 
 __mmask16 test_kadd_mask16(__m512i __A, __m512i __B, __m512i __C, __m512i __D, __m512i __E, __m512i __F) {
-  // CHECK-LABEL: @test_kadd_mask16
+  // CHECK-LABEL: test_kadd_mask16
   // CHECK: [[LHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: [[RHS:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: [[RES:%.*]] = call <16 x i1> @llvm.x86.avx512.kadd.w(<16 x i1> [[LHS]], <16 x i1> [[RHS]])
@@ -197,79 +201,80 @@ __mmask16 test_kadd_mask16(__m512i __A, __m512i __B, __m512i __C, __m512i __D, _
 }
 
 __mmask8 test_kshiftli_mask8(__m512i A, __m512i B, __m512i C, __m512i D) {
-  // CHECK-LABEL: @test_kshiftli_mask8
+  // CHECK-LABEL: test_kshiftli_mask8
   // CHECK: [[VAL:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RES:%.*]] = shufflevector <8 x i1> zeroinitializer, <8 x i1> [[VAL]], <8 x i32> <i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13>
   return _mm512_mask_cmpneq_epu64_mask(_kshiftli_mask8(_mm512_cmpneq_epu64_mask(A, B), 2), C, D);
 }
 
 __mmask8 test_kshiftri_mask8(__m512i A, __m512i B, __m512i C, __m512i D) {
-  // CHECK-LABEL: @test_kshiftri_mask8
+  // CHECK-LABEL: test_kshiftri_mask8
   // CHECK: [[VAL:%.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: [[RES:%.*]] = shufflevector <8 x i1> [[VAL]], <8 x i1> zeroinitializer, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>
   return _mm512_mask_cmpneq_epu64_mask(_kshiftri_mask8(_mm512_cmpneq_epu64_mask(A, B), 2), C, D);
 }
 
 unsigned int test_cvtmask8_u32(__m512i A, __m512i B) {
-  // CHECK-LABEL: @test_cvtmask8_u32
+  // CHECK-LABEL: test_cvtmask8_u32
   // CHECK: zext i8 %{{.*}} to i32
   return _cvtmask8_u32(_mm512_cmpneq_epu64_mask(A, B));
 }
 
 __mmask8 test_cvtu32_mask8(__m512i A, __m512i B, unsigned int C) {
-  // CHECK-LABEL: @test_cvtu32_mask8
+  // CHECK-LABEL: test_cvtu32_mask8
   // CHECK: trunc i32 %{{.*}} to i8
   return _mm512_mask_cmpneq_epu64_mask(_cvtu32_mask8(C), A, B);
 }
 
 __mmask8 test_load_mask8(__mmask8 *A, __m512i B, __m512i C) {
-  // CHECK-LABEL: @test_load_mask8
+  // CHECK-LABEL: test_load_mask8
   // CHECK: [[LOAD:%.*]] = load i8, ptr %{{.*}}
   return _mm512_mask_cmpneq_epu64_mask(_load_mask8(A), B, C);
 }
 
 void test_store_mask8(__mmask8 *A, __m512i B, __m512i C) {
-  // CHECK-LABEL: @test_store_mask8
+  // CHECK-LABEL: test_store_mask8
   // CHECK: store i8 %{{.*}}, ptr %{{.*}}
   _store_mask8(A, _mm512_cmpneq_epu64_mask(B, C));
 }
 
 __m512i test_mm512_mullo_epi64 (__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mullo_epi64
+  // CHECK-LABEL: test_mm512_mullo_epi64
   // CHECK: mul <8 x i64>
   return (__m512i) _mm512_mullo_epi64(__A, __B);
 }
 
 __m512i test_mm512_mask_mullo_epi64 (__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_mullo_epi64
+  // CHECK-LABEL: test_mm512_mask_mullo_epi64
   // CHECK: mul <8 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return (__m512i) _mm512_mask_mullo_epi64(__W, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_mullo_epi64 (__mmask8 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_mullo_epi64
+  // CHECK-LABEL: test_mm512_maskz_mullo_epi64
   // CHECK: mul <8 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return (__m512i) _mm512_maskz_mullo_epi64(__U, __A, __B);
 }
 
 __m512d test_mm512_xor_pd (__m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_xor_pd
+  // CHECK-LABEL: test_mm512_xor_pd
   // CHECK: xor <8 x i64>
   return (__m512d) _mm512_xor_pd(__A, __B);
 }
 
 __m512d test_mm512_mask_xor_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_mask_xor_pd
+  // CHECK-LABEL: test_mm512_mask_xor_pd
   // CHECK: xor <8 x i64>
   // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return (__m512d) _mm512_mask_xor_pd(__W, __U, __A, __B);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_xor_pd((__m512d){-4.0, -5.0, +6.0, +7.0, +7.0, +6.0, -5.0, -4.0}, (__m512d){+0.0, -0.0, -0.0, +7.0, +7.0, -0.0, -0.0, +0.0}), -4.0, +5.0, -6.0, +0.0, +0.0, -6.0, +5.0, -4.0));
 
 __m512d test_mm512_maskz_xor_pd (__mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_maskz_xor_pd
+  // CHECK-LABEL: test_mm512_maskz_xor_pd
   // CHECK: xor <8 x i64>
   // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}}
@@ -277,13 +282,14 @@ __m512d test_mm512_maskz_xor_pd (__mmask8 __U, __m512d __A, __m512d __B) {
 }
 
 __m512 test_mm512_xor_ps (__m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_xor_ps
+  // CHECK-LABEL: test_mm512_xor_ps
   // CHECK: xor <16 x i32>
   return (__m512) _mm512_xor_ps(__A, __B);
 }
+TEST_CONSTEXPR(match_m512(_mm512_xor_ps((__m512){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f, -4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m512){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f, +0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -4.0f, +5.0f, -6.0f, +0.0f, +0.0f, -6.0f, +5.0f, -4.0f, -4.0f, +5.0f, -6.0f, +0.0f, +0.0f, -6.0f, +5.0f, -4.0f));
 
 __m512 test_mm512_mask_xor_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_mask_xor_ps
+  // CHECK-LABEL: test_mm512_mask_xor_ps
   // CHECK: xor <16 x i32>
   // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}}
@@ -291,7 +297,7 @@ __m512 test_mm512_mask_xor_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B
 }
 
 __m512 test_mm512_maskz_xor_ps (__mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_maskz_xor_ps
+  // CHECK-LABEL: test_mm512_maskz_xor_ps
   // CHECK: xor <16 x i32>
   // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}}
@@ -299,13 +305,14 @@ __m512 test_mm512_maskz_xor_ps (__mmask16 __U, __m512 __A, __m512 __B) {
 }
 
 __m512d test_mm512_or_pd (__m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_or_pd
+  // CHECK-LABEL: test_mm512_or_pd
   // CHECK: or <8 x i64>
   return (__m512d) _mm512_or_pd(__A, __B);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_or_pd((__m512d){-4.0, -5.0, +6.0, +7.0, +7.0, +6.0, -5.0, -4.0}, (__m512d){+0.0, -0.0, -0.0, +7.0, +7.0, -0.0, -0.0, +0.0}), -4.0, -5.0, -6.0, +7.0, +7.0, -6.0, -5.0, -4.0));
 
 __m512d test_mm512_mask_or_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_mask_or_pd
+  // CHECK-LABEL: test_mm512_mask_or_pd
   // CHECK: or <8 x i64>
   // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}}
@@ -313,7 +320,7 @@ __m512d test_mm512_mask_or_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d _
 }
 
 __m512d test_mm512_maskz_or_pd (__mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_maskz_or_pd
+  // CHECK-LABEL: test_mm512_maskz_or_pd
   // CHECK: or <8 x i64>
   // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}}
@@ -321,13 +328,14 @@ __m512d test_mm512_maskz_or_pd (__mmask8 __U, __m512d __A, __m512d __B) {
 }
 
 __m512 test_mm512_or_ps (__m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_or_ps
+  // CHECK-LABEL: test_mm512_or_ps
   // CHECK: or <16 x i32>
   return (__m512) _mm512_or_ps(__A, __B);
 }
+TEST_CONSTEXPR(match_m512(_mm512_or_ps((__m512){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f, -4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m512){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f, +0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), -4.0f, -5.0f, -6.0f, +7.0f, +7.0f, -6.0f, -5.0f, -4.0f, -4.0f, -5.0f, -6.0f, +7.0f, +7.0f, -6.0f, -5.0f, -4.0f));
 
 __m512 test_mm512_mask_or_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_mask_or_ps
+  // CHECK-LABEL: test_mm512_mask_or_ps
   // CHECK: or <16 x i32>
   // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}}
@@ -335,7 +343,7 @@ __m512 test_mm512_mask_or_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B)
 }
 
 __m512 test_mm512_maskz_or_ps (__mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_maskz_or_ps
+  // CHECK-LABEL: test_mm512_maskz_or_ps
   // CHECK: or <16 x i32>
   // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}}
@@ -343,13 +351,14 @@ __m512 test_mm512_maskz_or_ps (__mmask16 __U, __m512 __A, __m512 __B) {
 }
 
 __m512d test_mm512_and_pd (__m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_and_pd
+  // CHECK-LABEL: test_mm512_and_pd
   // CHECK: and <8 x i64>
   return (__m512d) _mm512_and_pd(__A, __B);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_and_pd((__m512d){-4.0, -5.0, +6.0, +7.0, +7.0, +6.0, -5.0, -4.0}, (__m512d){+0.0, -0.0, -0.0, +7.0, +7.0, -0.0, -0.0, +0.0}), +0.0, -0.0, +0.0, +7.0, +7.0, +0.0, -0.0, +0.0));
 
 __m512d test_mm512_mask_and_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_mask_and_pd
+  // CHECK-LABEL: test_mm512_mask_and_pd
   // CHECK: and <8 x i64>
   // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}}
@@ -357,7 +366,7 @@ __m512d test_mm512_mask_and_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d
 }
 
 __m512d test_mm512_maskz_and_pd (__mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_maskz_and_pd
+  // CHECK-LABEL: test_mm512_maskz_and_pd
   // CHECK: and <8 x i64>
   // CHECK: %[[MASK:.*]] = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: select <8 x i1> %[[MASK]], <8 x double> %{{.*}}, <8 x double> %{{.*}}
@@ -365,13 +374,14 @@ __m512d test_mm512_maskz_and_pd (__mmask8 __U, __m512d __A, __m512d __B) {
 }
 
 __m512 test_mm512_and_ps (__m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_and_ps
+  // CHECK-LABEL: test_mm512_and_ps
   // CHECK: and <16 x i32>
   return (__m512) _mm512_and_ps(__A, __B);
 }
+TEST_CONSTEXPR(match_m512(_mm512_and_ps((__m512){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f, -4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m512){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f, +0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), +0.0f, -0.0f, +0.0f, +7.0f, +7.0f, +0.0f, -0.0f, +0.0f, +0.0f, -0.0f, +0.0f, +7.0f, +7.0f, +0.0f, -0.0f, +0.0f));
 
 __m512 test_mm512_mask_and_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_mask_and_ps
+  // CHECK-LABEL: test_mm512_mask_and_ps
   // CHECK: and <16 x i32>
   // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}}
@@ -379,7 +389,7 @@ __m512 test_mm512_mask_and_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B
 }
 
 __m512 test_mm512_maskz_and_ps (__mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_maskz_and_ps
+  // CHECK-LABEL: test_mm512_maskz_and_ps
   // CHECK: and <16 x i32>
   // CHECK: %[[MASK:.*]] = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: select <16 x i1> %[[MASK]], <16 x float> %{{.*}}, <16 x float> %{{.*}}
@@ -387,14 +397,15 @@ __m512 test_mm512_maskz_and_ps (__mmask16 __U, __m512 __A, __m512 __B) {
 }
 
 __m512d test_mm512_andnot_pd (__m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_andnot_pd
+  // CHECK-LABEL: test_mm512_andnot_pd
   // CHECK: xor <8 x i64> %{{.*}}, splat (i64 -1)
   // CHECK: and <8 x i64>
   return (__m512d) _mm512_andnot_pd(__A, __B);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_andnot_pd((__m512d){-4.0, -5.0, +6.0, +7.0, +7.0, +6.0, -5.0, -4.0}, (__m512d){+0.0, -0.0, -0.0, +7.0, +7.0, -0.0, -0.0, +0.0}), +0.0, +0.0, -0.0, +0.0, +0.0, -0.0, +0.0, +0.0));
 
 __m512d test_mm512_mask_andnot_pd (__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_mask_andnot_pd
+  // CHECK-LABEL: test_mm512_mask_andnot_pd
   // CHECK: xor <8 x i64> %{{.*}}, splat (i64 -1)
   // CHECK: and <8 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
@@ -402,7 +413,7 @@ __m512d test_mm512_mask_andnot_pd (__m512d __W, __mmask8 __U, __m512d __A, __m51
 }
 
 __m512d test_mm512_maskz_andnot_pd (__mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_maskz_andnot_pd
+  // CHECK-LABEL: test_mm512_maskz_andnot_pd
   // CHECK: xor <8 x i64> %{{.*}}, splat (i64 -1)
   // CHECK: and <8 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
@@ -410,14 +421,15 @@ __m512d test_mm512_maskz_andnot_pd (__mmask8 __U, __m512d __A, __m512d __B) {
 }
 
 __m512 test_mm512_andnot_ps (__m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_andnot_ps
+  // CHECK-LABEL: test_mm512_andnot_ps
   // CHECK: xor <16 x i32> %{{.*}}, splat (i32 -1)
   // CHECK: and <16 x i32>
   return (__m512) _mm512_andnot_ps(__A, __B);
 }
+TEST_CONSTEXPR(match_m512(_mm512_andnot_ps((__m512){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f, -4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m512){+0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f, +0.0f, -0.0f, -0.0f, +7.0f, +7.0f, -0.0f, -0.0f, +0.0f}), +0.0f, +0.0f, -0.0f, +0.0f, +0.0f, -0.0f, +0.0f, +0.0f, +0.0f, +0.0f, -0.0f, +0.0f, +0.0f, -0.0f, +0.0f, +0.0f));
 
 __m512 test_mm512_mask_andnot_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_mask_andnot_ps
+  // CHECK-LABEL: test_mm512_mask_andnot_ps
   // CHECK: xor <16 x i32> %{{.*}}, splat (i32 -1)
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
@@ -425,7 +437,7 @@ __m512 test_mm512_mask_andnot_ps (__m512 __W, __mmask16 __U, __m512 __A, __m512
 }
 
 __m512 test_mm512_maskz_andnot_ps (__mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_maskz_andnot_ps
+  // CHECK-LABEL: test_mm512_maskz_andnot_ps
   // CHECK: xor <16 x i32> %{{.*}}, splat (i32 -1)
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
@@ -433,491 +445,491 @@ __m512 test_mm512_maskz_andnot_ps (__mmask16 __U, __m512 __A, __m512 __B) {
 }
 
 __m512i test_mm512_cvtpd_epi64(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_cvtpd_epi64
+  // CHECK-LABEL: test_mm512_cvtpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512
   return _mm512_cvtpd_epi64(__A); 
 }
 
 __m512i test_mm512_mask_cvtpd_epi64(__m512i __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtpd_epi64
+  // CHECK-LABEL: test_mm512_mask_cvtpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512
   return _mm512_mask_cvtpd_epi64(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvtpd_epi64(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtpd_epi64
+  // CHECK-LABEL: test_mm512_maskz_cvtpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512
   return _mm512_maskz_cvtpd_epi64(__U, __A); 
 }
 
 __m512i test_mm512_cvt_roundpd_epi64(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_cvt_roundpd_epi64
+  // CHECK-LABEL: test_mm512_cvt_roundpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512
   return _mm512_cvt_roundpd_epi64(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_mask_cvt_roundpd_epi64(__m512i __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvt_roundpd_epi64
+  // CHECK-LABEL: test_mm512_mask_cvt_roundpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512
   return _mm512_mask_cvt_roundpd_epi64(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_maskz_cvt_roundpd_epi64(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvt_roundpd_epi64
+  // CHECK-LABEL: test_mm512_maskz_cvt_roundpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.512
   return _mm512_maskz_cvt_roundpd_epi64(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_cvtpd_epu64(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_cvtpd_epu64
+  // CHECK-LABEL: test_mm512_cvtpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512
   return _mm512_cvtpd_epu64(__A); 
 }
 
 __m512i test_mm512_mask_cvtpd_epu64(__m512i __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtpd_epu64
+  // CHECK-LABEL: test_mm512_mask_cvtpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512
   return _mm512_mask_cvtpd_epu64(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvtpd_epu64(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtpd_epu64
+  // CHECK-LABEL: test_mm512_maskz_cvtpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512
   return _mm512_maskz_cvtpd_epu64(__U, __A); 
 }
 
 __m512i test_mm512_cvt_roundpd_epu64(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_cvt_roundpd_epu64
+  // CHECK-LABEL: test_mm512_cvt_roundpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512
   return _mm512_cvt_roundpd_epu64(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_mask_cvt_roundpd_epu64(__m512i __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvt_roundpd_epu64
+  // CHECK-LABEL: test_mm512_mask_cvt_roundpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512
   return _mm512_mask_cvt_roundpd_epu64(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_maskz_cvt_roundpd_epu64(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvt_roundpd_epu64
+  // CHECK-LABEL: test_mm512_maskz_cvt_roundpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.512
   return _mm512_maskz_cvt_roundpd_epu64(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_cvtps_epi64(__m256 __A) {
-  // CHECK-LABEL: @test_mm512_cvtps_epi64
+  // CHECK-LABEL: test_mm512_cvtps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512
   return _mm512_cvtps_epi64(__A); 
 }
 
 __m512i test_mm512_mask_cvtps_epi64(__m512i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtps_epi64
+  // CHECK-LABEL: test_mm512_mask_cvtps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512
   return _mm512_mask_cvtps_epi64(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvtps_epi64(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtps_epi64
+  // CHECK-LABEL: test_mm512_maskz_cvtps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512
   return _mm512_maskz_cvtps_epi64(__U, __A); 
 }
 
 __m512i test_mm512_cvt_roundps_epi64(__m256 __A) {
-  // CHECK-LABEL: @test_mm512_cvt_roundps_epi64
+  // CHECK-LABEL: test_mm512_cvt_roundps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512
   return _mm512_cvt_roundps_epi64(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_mask_cvt_roundps_epi64(__m512i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvt_roundps_epi64
+  // CHECK-LABEL: test_mm512_mask_cvt_roundps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512
   return _mm512_mask_cvt_roundps_epi64(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_maskz_cvt_roundps_epi64(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvt_roundps_epi64
+  // CHECK-LABEL: test_mm512_maskz_cvt_roundps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.512
   return _mm512_maskz_cvt_roundps_epi64(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_cvtps_epu64(__m256 __A) {
-  // CHECK-LABEL: @test_mm512_cvtps_epu64
+  // CHECK-LABEL: test_mm512_cvtps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512
   return _mm512_cvtps_epu64(__A); 
 }
 
 __m512i test_mm512_mask_cvtps_epu64(__m512i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtps_epu64
+  // CHECK-LABEL: test_mm512_mask_cvtps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512
   return _mm512_mask_cvtps_epu64(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvtps_epu64(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtps_epu64
+  // CHECK-LABEL: test_mm512_maskz_cvtps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512
   return _mm512_maskz_cvtps_epu64(__U, __A); 
 }
 
 __m512i test_mm512_cvt_roundps_epu64(__m256 __A) {
-  // CHECK-LABEL: @test_mm512_cvt_roundps_epu64
+  // CHECK-LABEL: test_mm512_cvt_roundps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512
   return _mm512_cvt_roundps_epu64(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_mask_cvt_roundps_epu64(__m512i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvt_roundps_epu64
+  // CHECK-LABEL: test_mm512_mask_cvt_roundps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512
   return _mm512_mask_cvt_roundps_epu64(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_maskz_cvt_roundps_epu64(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvt_roundps_epu64
+  // CHECK-LABEL: test_mm512_maskz_cvt_roundps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.512
   return _mm512_maskz_cvt_roundps_epu64(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512d test_mm512_cvtepi64_pd(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvtepi64_pd
+  // CHECK-LABEL: test_mm512_cvtepi64_pd
   // CHECK: sitofp <8 x i64> %{{.*}} to <8 x double>
   return _mm512_cvtepi64_pd(__A); 
 }
 
 __m512d test_mm512_mask_cvtepi64_pd(__m512d __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtepi64_pd
+  // CHECK-LABEL: test_mm512_mask_cvtepi64_pd
   // CHECK: sitofp <8 x i64> %{{.*}} to <8 x double>
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_mask_cvtepi64_pd(__W, __U, __A); 
 }
 
 __m512d test_mm512_maskz_cvtepi64_pd(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtepi64_pd
+  // CHECK-LABEL: test_mm512_maskz_cvtepi64_pd
   // CHECK: sitofp <8 x i64> %{{.*}} to <8 x double>
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_maskz_cvtepi64_pd(__U, __A); 
 }
 
 __m512d test_mm512_cvt_roundepi64_pd(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvt_roundepi64_pd
+  // CHECK-LABEL: test_mm512_cvt_roundepi64_pd
   // CHECK: @llvm.x86.avx512.sitofp.round.v8f64.v8i64
   return _mm512_cvt_roundepi64_pd(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512d test_mm512_mask_cvt_roundepi64_pd(__m512d __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvt_roundepi64_pd
+  // CHECK-LABEL: test_mm512_mask_cvt_roundepi64_pd
   // CHECK: @llvm.x86.avx512.sitofp.round.v8f64.v8i64
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_mask_cvt_roundepi64_pd(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512d test_mm512_maskz_cvt_roundepi64_pd(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvt_roundepi64_pd
+  // CHECK-LABEL: test_mm512_maskz_cvt_roundepi64_pd
   // CHECK: @llvm.x86.avx512.sitofp.round.v8f64.v8i64
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_maskz_cvt_roundepi64_pd(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m256 test_mm512_cvtepi64_ps(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvtepi64_ps
+  // CHECK-LABEL: test_mm512_cvtepi64_ps
   // CHECK: sitofp <8 x i64> %{{.*}} to <8 x float>
   return _mm512_cvtepi64_ps(__A); 
 }
 
 __m256 test_mm512_mask_cvtepi64_ps(__m256 __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtepi64_ps
+  // CHECK-LABEL: test_mm512_mask_cvtepi64_ps
   // CHECK: sitofp <8 x i64> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_mask_cvtepi64_ps(__W, __U, __A); 
 }
 
 __m256 test_mm512_maskz_cvtepi64_ps(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtepi64_ps
+  // CHECK-LABEL: test_mm512_maskz_cvtepi64_ps
   // CHECK: sitofp <8 x i64> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_maskz_cvtepi64_ps(__U, __A); 
 }
 
 __m256 test_mm512_cvt_roundepi64_ps(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvt_roundepi64_ps
+  // CHECK-LABEL: test_mm512_cvt_roundepi64_ps
   // CHECK: @llvm.x86.avx512.sitofp.round.v8f32.v8i64
   return _mm512_cvt_roundepi64_ps(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m256 test_mm512_mask_cvt_roundepi64_ps(__m256 __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvt_roundepi64_ps
+  // CHECK-LABEL: test_mm512_mask_cvt_roundepi64_ps
   // CHECK: @llvm.x86.avx512.sitofp.round.v8f32.v8i64
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_mask_cvt_roundepi64_ps(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m256 test_mm512_maskz_cvt_roundepi64_ps(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvt_roundepi64_ps
+  // CHECK-LABEL: test_mm512_maskz_cvt_roundepi64_ps
   // CHECK: @llvm.x86.avx512.sitofp.round.v8f32.v8i64
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_maskz_cvt_roundepi64_ps(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512i test_mm512_cvttpd_epi64(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_cvttpd_epi64
+  // CHECK-LABEL: test_mm512_cvttpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512
   return _mm512_cvttpd_epi64(__A); 
 }
 
 __m512i test_mm512_mask_cvttpd_epi64(__m512i __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvttpd_epi64
+  // CHECK-LABEL: test_mm512_mask_cvttpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512
   return _mm512_mask_cvttpd_epi64(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvttpd_epi64(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvttpd_epi64
+  // CHECK-LABEL: test_mm512_maskz_cvttpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512
   return _mm512_maskz_cvttpd_epi64(__U, __A); 
 }
 
 __m512i test_mm512_cvtt_roundpd_epi64(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_cvtt_roundpd_epi64
+  // CHECK-LABEL: test_mm512_cvtt_roundpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512
   return _mm512_cvtt_roundpd_epi64(__A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_mask_cvtt_roundpd_epi64(__m512i __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtt_roundpd_epi64
+  // CHECK-LABEL: test_mm512_mask_cvtt_roundpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512
   return _mm512_mask_cvtt_roundpd_epi64(__W, __U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_maskz_cvtt_roundpd_epi64(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtt_roundpd_epi64
+  // CHECK-LABEL: test_mm512_maskz_cvtt_roundpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.512
   return _mm512_maskz_cvtt_roundpd_epi64(__U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_cvttpd_epu64(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_cvttpd_epu64
+  // CHECK-LABEL: test_mm512_cvttpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512
   return _mm512_cvttpd_epu64(__A); 
 }
 
 __m512i test_mm512_mask_cvttpd_epu64(__m512i __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvttpd_epu64
+  // CHECK-LABEL: test_mm512_mask_cvttpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512
   return _mm512_mask_cvttpd_epu64(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvttpd_epu64(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvttpd_epu64
+  // CHECK-LABEL: test_mm512_maskz_cvttpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512
   return _mm512_maskz_cvttpd_epu64(__U, __A); 
 }
 
 __m512i test_mm512_cvtt_roundpd_epu64(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_cvtt_roundpd_epu64
+  // CHECK-LABEL: test_mm512_cvtt_roundpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512
   return _mm512_cvtt_roundpd_epu64(__A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_mask_cvtt_roundpd_epu64(__m512i __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtt_roundpd_epu64
+  // CHECK-LABEL: test_mm512_mask_cvtt_roundpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512
   return _mm512_mask_cvtt_roundpd_epu64(__W, __U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_maskz_cvtt_roundpd_epu64(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtt_roundpd_epu64
+  // CHECK-LABEL: test_mm512_maskz_cvtt_roundpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.512
   return _mm512_maskz_cvtt_roundpd_epu64(__U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_cvttps_epi64(__m256 __A) {
-  // CHECK-LABEL: @test_mm512_cvttps_epi64
+  // CHECK-LABEL: test_mm512_cvttps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512
   return _mm512_cvttps_epi64(__A); 
 }
 
 __m512i test_mm512_mask_cvttps_epi64(__m512i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvttps_epi64
+  // CHECK-LABEL: test_mm512_mask_cvttps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512
   return _mm512_mask_cvttps_epi64(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvttps_epi64(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvttps_epi64
+  // CHECK-LABEL: test_mm512_maskz_cvttps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512
   return _mm512_maskz_cvttps_epi64(__U, __A); 
 }
 
 __m512i test_mm512_cvtt_roundps_epi64(__m256 __A) {
-  // CHECK-LABEL: @test_mm512_cvtt_roundps_epi64
+  // CHECK-LABEL: test_mm512_cvtt_roundps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512
   return _mm512_cvtt_roundps_epi64(__A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_mask_cvtt_roundps_epi64(__m512i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtt_roundps_epi64
+  // CHECK-LABEL: test_mm512_mask_cvtt_roundps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512
   return _mm512_mask_cvtt_roundps_epi64(__W, __U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_maskz_cvtt_roundps_epi64(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtt_roundps_epi64
+  // CHECK-LABEL: test_mm512_maskz_cvtt_roundps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.512
   return _mm512_maskz_cvtt_roundps_epi64(__U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_cvttps_epu64(__m256 __A) {
-  // CHECK-LABEL: @test_mm512_cvttps_epu64
+  // CHECK-LABEL: test_mm512_cvttps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512
   return _mm512_cvttps_epu64(__A); 
 }
 
 __m512i test_mm512_mask_cvttps_epu64(__m512i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvttps_epu64
+  // CHECK-LABEL: test_mm512_mask_cvttps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512
   return _mm512_mask_cvttps_epu64(__W, __U, __A); 
 }
 
 __m512i test_mm512_maskz_cvttps_epu64(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvttps_epu64
+  // CHECK-LABEL: test_mm512_maskz_cvttps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512
   return _mm512_maskz_cvttps_epu64(__U, __A); 
 }
 
 __m512i test_mm512_cvtt_roundps_epu64(__m256 __A) {
-  // CHECK-LABEL: @test_mm512_cvtt_roundps_epu64
+  // CHECK-LABEL: test_mm512_cvtt_roundps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512
   return _mm512_cvtt_roundps_epu64(__A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_mask_cvtt_roundps_epu64(__m512i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtt_roundps_epu64
+  // CHECK-LABEL: test_mm512_mask_cvtt_roundps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512
   return _mm512_mask_cvtt_roundps_epu64(__W, __U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512i test_mm512_maskz_cvtt_roundps_epu64(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtt_roundps_epu64
+  // CHECK-LABEL: test_mm512_maskz_cvtt_roundps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.512
   return _mm512_maskz_cvtt_roundps_epu64(__U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512d test_mm512_cvtepu64_pd(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvtepu64_pd
+  // CHECK-LABEL: test_mm512_cvtepu64_pd
   // CHECK: uitofp <8 x i64> %{{.*}} to <8 x double>
   return _mm512_cvtepu64_pd(__A); 
 }
 
 __m512d test_mm512_mask_cvtepu64_pd(__m512d __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtepu64_pd
+  // CHECK-LABEL: test_mm512_mask_cvtepu64_pd
   // CHECK: uitofp <8 x i64> %{{.*}} to <8 x double>
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_mask_cvtepu64_pd(__W, __U, __A); 
 }
 
 __m512d test_mm512_maskz_cvtepu64_pd(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtepu64_pd
+  // CHECK-LABEL: test_mm512_maskz_cvtepu64_pd
   // CHECK: uitofp <8 x i64> %{{.*}} to <8 x double>
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_maskz_cvtepu64_pd(__U, __A); 
 }
 
 __m512d test_mm512_cvt_roundepu64_pd(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvt_roundepu64_pd
+  // CHECK-LABEL: test_mm512_cvt_roundepu64_pd
   // CHECK: @llvm.x86.avx512.uitofp.round.v8f64.v8i64
   return _mm512_cvt_roundepu64_pd(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512d test_mm512_mask_cvt_roundepu64_pd(__m512d __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvt_roundepu64_pd
+  // CHECK-LABEL: test_mm512_mask_cvt_roundepu64_pd
   // CHECK: @llvm.x86.avx512.uitofp.round.v8f64.v8i64
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_mask_cvt_roundepu64_pd(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512d test_mm512_maskz_cvt_roundepu64_pd(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvt_roundepu64_pd
+  // CHECK-LABEL: test_mm512_maskz_cvt_roundepu64_pd
   // CHECK: @llvm.x86.avx512.uitofp.round.v8f64.v8i64
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_maskz_cvt_roundepu64_pd(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m256 test_mm512_cvtepu64_ps(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvtepu64_ps
+  // CHECK-LABEL: test_mm512_cvtepu64_ps
   // CHECK: uitofp <8 x i64> %{{.*}} to <8 x float>
   return _mm512_cvtepu64_ps(__A); 
 }
 
 __m256 test_mm512_mask_cvtepu64_ps(__m256 __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvtepu64_ps
+  // CHECK-LABEL: test_mm512_mask_cvtepu64_ps
   // CHECK: uitofp <8 x i64> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_mask_cvtepu64_ps(__W, __U, __A); 
 }
 
 __m256 test_mm512_maskz_cvtepu64_ps(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtepu64_ps
+  // CHECK-LABEL: test_mm512_maskz_cvtepu64_ps
   // CHECK: uitofp <8 x i64> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_maskz_cvtepu64_ps(__U, __A); 
 }
 
 __m256 test_mm512_cvt_roundepu64_ps(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_cvt_roundepu64_ps
+  // CHECK-LABEL: test_mm512_cvt_roundepu64_ps
   // CHECK: @llvm.x86.avx512.uitofp.round.v8f32.v8i64
   return _mm512_cvt_roundepu64_ps(__A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m256 test_mm512_mask_cvt_roundepu64_ps(__m256 __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_cvt_roundepu64_ps
+  // CHECK-LABEL: test_mm512_mask_cvt_roundepu64_ps
   // CHECK: @llvm.x86.avx512.uitofp.round.v8f32.v8i64
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_mask_cvt_roundepu64_ps(__W, __U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m256 test_mm512_maskz_cvt_roundepu64_ps(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_cvt_roundepu64_ps
+  // CHECK-LABEL: test_mm512_maskz_cvt_roundepu64_ps
   // CHECK: @llvm.x86.avx512.uitofp.round.v8f32.v8i64
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_maskz_cvt_roundepu64_ps(__U, __A, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC); 
 }
 
 __m512d test_mm512_range_pd(__m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_range_pd
+  // CHECK-LABEL: test_mm512_range_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.512
   return _mm512_range_pd(__A, __B, 4); 
 }
 
 __m512d test_mm512_mask_range_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_mask_range_pd
+  // CHECK-LABEL: test_mm512_mask_range_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.512
   return _mm512_mask_range_pd(__W, __U, __A, __B, 4); 
 }
 
 __m512d test_mm512_maskz_range_pd(__mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_maskz_range_pd
+  // CHECK-LABEL: test_mm512_maskz_range_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.512
   return _mm512_maskz_range_pd(__U, __A, __B, 4); 
 }
 
 __m512d test_mm512_range_round_pd(__m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_range_round_pd
+  // CHECK-LABEL: test_mm512_range_round_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.512
   return _mm512_range_round_pd(__A, __B, 4, 8); 
 }
 
 __m512d test_mm512_mask_range_round_pd(__m512d __W, __mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_mask_range_round_pd
+  // CHECK-LABEL: test_mm512_mask_range_round_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.512
   return _mm512_mask_range_round_pd(__W, __U, __A, __B, 4, 8); 
 }
 
 __m512d test_mm512_maskz_range_round_pd(__mmask8 __U, __m512d __A, __m512d __B) {
-  // CHECK-LABEL: @test_mm512_maskz_range_round_pd
+  // CHECK-LABEL: test_mm512_maskz_range_round_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.512
   return _mm512_maskz_range_round_pd(__U, __A, __B, 4, 8); 
 }
 
 __m128d test_mm512_range_round_sd(__m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm512_range_round_sd
+  // CHECK-LABEL: test_mm512_range_round_sd
   // CHECK: @llvm.x86.avx512.mask.range.sd
   return _mm_range_round_sd(__A, __B, 4, 8); 
 }
@@ -929,31 +941,31 @@ __m128d test_mm512_mask_range_round_sd(__m128d __W, __mmask8 __U, __m128d __A, _
 }
 
 __m128d test_mm512_maskz_range_round_sd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm512_maskz_range_round_sd
+  // CHECK-LABEL: test_mm512_maskz_range_round_sd
   // CHECK: @llvm.x86.avx512.mask.range.sd
   return _mm_maskz_range_round_sd(__U, __A, __B, 4, 8); 
 }
 
 __m128 test_mm512_range_round_ss(__m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm512_range_round_ss
+  // CHECK-LABEL: test_mm512_range_round_ss
   // CHECK: @llvm.x86.avx512.mask.range.ss
   return _mm_range_round_ss(__A, __B, 4, 8); 
 }
 
 __m128 test_mm512_mask_range_round_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm512_mask_range_round_ss
+  // CHECK-LABEL: test_mm512_mask_range_round_ss
   // CHECK: @llvm.x86.avx512.mask.range.ss
   return _mm_mask_range_round_ss(__W, __U, __A, __B, 4, 8); 
 }
 
 __m128 test_mm512_maskz_range_round_ss(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm512_maskz_range_round_ss
+  // CHECK-LABEL: test_mm512_maskz_range_round_ss
   // CHECK: @llvm.x86.avx512.mask.range.ss
   return _mm_maskz_range_round_ss(__U, __A, __B, 4, 8); 
 }
 
 __m128d test_mm_range_sd(__m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_range_sd
+  // CHECK-LABEL: test_mm_range_sd
   // CHECK: @llvm.x86.avx512.mask.range.sd
   return _mm_range_sd(__A, __B, 4); 
 }
@@ -965,558 +977,564 @@ __m128d test_mm_mask_range_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __
 }
 
 __m128d test_mm_maskz_range_sd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_range_sd
+  // CHECK-LABEL: test_mm_maskz_range_sd
   // CHECK: @llvm.x86.avx512.mask.range.sd
   return _mm_maskz_range_sd(__U, __A, __B, 4); 
 }
 
 __m128 test_mm_range_ss(__m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_range_ss
+  // CHECK-LABEL: test_mm_range_ss
   // CHECK: @llvm.x86.avx512.mask.range.ss
   return _mm_range_ss(__A, __B, 4); 
 }
 
 __m128 test_mm_mask_range_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_range_ss
+  // CHECK-LABEL: test_mm_mask_range_ss
   // CHECK: @llvm.x86.avx512.mask.range.ss
   return _mm_mask_range_ss(__W, __U, __A, __B, 4); 
 }
 
 __m128 test_mm_maskz_range_ss(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_range_ss
+  // CHECK-LABEL: test_mm_maskz_range_ss
   // CHECK: @llvm.x86.avx512.mask.range.ss
   return _mm_maskz_range_ss(__U, __A, __B, 4); 
 }
 
 __m512 test_mm512_range_ps(__m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_range_ps
+  // CHECK-LABEL: test_mm512_range_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.512
   return _mm512_range_ps(__A, __B, 4); 
 }
 
 __m512 test_mm512_mask_range_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_mask_range_ps
+  // CHECK-LABEL: test_mm512_mask_range_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.512
   return _mm512_mask_range_ps(__W, __U, __A, __B, 4); 
 }
 
 __m512 test_mm512_maskz_range_ps(__mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_maskz_range_ps
+  // CHECK-LABEL: test_mm512_maskz_range_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.512
   return _mm512_maskz_range_ps(__U, __A, __B, 4); 
 }
 
 __m512 test_mm512_range_round_ps(__m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_range_round_ps
+  // CHECK-LABEL: test_mm512_range_round_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.512
   return _mm512_range_round_ps(__A, __B, 4, 8); 
 }
 
 __m512 test_mm512_mask_range_round_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_mask_range_round_ps
+  // CHECK-LABEL: test_mm512_mask_range_round_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.512
   return _mm512_mask_range_round_ps(__W, __U, __A, __B, 4, 8); 
 }
 
 __m512 test_mm512_maskz_range_round_ps(__mmask16 __U, __m512 __A, __m512 __B) {
-  // CHECK-LABEL: @test_mm512_maskz_range_round_ps
+  // CHECK-LABEL: test_mm512_maskz_range_round_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.512
   return _mm512_maskz_range_round_ps(__U, __A, __B, 4, 8); 
 }
 
 __m512d test_mm512_reduce_pd(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_reduce_pd
+  // CHECK-LABEL: test_mm512_reduce_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.512
   return _mm512_reduce_pd(__A, 4); 
 }
 
 __m512d test_mm512_mask_reduce_pd(__m512d __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_reduce_pd
+  // CHECK-LABEL: test_mm512_mask_reduce_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.512
   return _mm512_mask_reduce_pd(__W, __U, __A, 4); 
 }
 
 __m512d test_mm512_maskz_reduce_pd(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_reduce_pd
+  // CHECK-LABEL: test_mm512_maskz_reduce_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.512
   return _mm512_maskz_reduce_pd(__U, __A, 4); 
 }
 
 __m512 test_mm512_reduce_ps(__m512 __A) {
-  // CHECK-LABEL: @test_mm512_reduce_ps
+  // CHECK-LABEL: test_mm512_reduce_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.512
   return _mm512_reduce_ps(__A, 4); 
 }
 
 __m512 test_mm512_mask_reduce_ps(__m512 __W, __mmask16 __U, __m512 __A) {
-  // CHECK-LABEL: @test_mm512_mask_reduce_ps
+  // CHECK-LABEL: test_mm512_mask_reduce_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.512
   return _mm512_mask_reduce_ps(__W, __U, __A, 4); 
 }
 
 __m512 test_mm512_maskz_reduce_ps(__mmask16 __U, __m512 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_reduce_ps
+  // CHECK-LABEL: test_mm512_maskz_reduce_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.512
   return _mm512_maskz_reduce_ps(__U, __A, 4); 
 }
 
 __m512d test_mm512_reduce_round_pd(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_reduce_round_pd
+  // CHECK-LABEL: test_mm512_reduce_round_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.512
   return _mm512_reduce_round_pd(__A, 4, 8); 
 }
 
 __m512d test_mm512_mask_reduce_round_pd(__m512d __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_reduce_round_pd
+  // CHECK-LABEL: test_mm512_mask_reduce_round_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.512
   return _mm512_mask_reduce_round_pd(__W, __U, __A, 4, 8); 
 }
 
 __m512d test_mm512_maskz_reduce_round_pd(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_reduce_round_pd
+  // CHECK-LABEL: test_mm512_maskz_reduce_round_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.512
   return _mm512_maskz_reduce_round_pd(__U, __A, 4, 8);
 }
 
 __m512 test_mm512_reduce_round_ps(__m512 __A) {
-  // CHECK-LABEL: @test_mm512_reduce_round_ps
+  // CHECK-LABEL: test_mm512_reduce_round_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.512
   return _mm512_reduce_round_ps(__A, 4, 8); 
 }
 
 __m512 test_mm512_mask_reduce_round_ps(__m512 __W, __mmask16 __U, __m512 __A) {
-  // CHECK-LABEL: @test_mm512_mask_reduce_round_ps
+  // CHECK-LABEL: test_mm512_mask_reduce_round_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.512
   return _mm512_mask_reduce_round_ps(__W, __U, __A, 4, 8); 
 }
 
 __m512 test_mm512_maskz_reduce_round_ps(__mmask16 __U, __m512 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_reduce_round_ps
+  // CHECK-LABEL: test_mm512_maskz_reduce_round_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.512
   return _mm512_maskz_reduce_round_ps(__U, __A, 4, 8); 
 }
 
 __m128 test_mm_reduce_ss(__m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_reduce_ss
+  // CHECK-LABEL: test_mm_reduce_ss
   // CHECK: @llvm.x86.avx512.mask.reduce.ss
   return _mm_reduce_ss(__A, __B, 4);
 }
 
 __m128 test_mm_mask_reduce_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_reduce_ss
+  // CHECK-LABEL: test_mm_mask_reduce_ss
   // CHECK: @llvm.x86.avx512.mask.reduce.ss
   return _mm_mask_reduce_ss(__W, __U, __A, __B, 4);
 }
 
 __m128 test_mm_maskz_reduce_ss(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_reduce_ss
+  // CHECK-LABEL: test_mm_maskz_reduce_ss
   // CHECK: @llvm.x86.avx512.mask.reduce.ss
   return _mm_maskz_reduce_ss(__U, __A, __B, 4);
 }
 
 __m128 test_mm_reduce_round_ss(__m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_reduce_round_ss
+  // CHECK-LABEL: test_mm_reduce_round_ss
   // CHECK: @llvm.x86.avx512.mask.reduce.ss
   return _mm_reduce_round_ss(__A, __B, 4, 8);
 }
 
 __m128 test_mm_mask_reduce_round_ss(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_reduce_round_ss
+  // CHECK-LABEL: test_mm_mask_reduce_round_ss
   // CHECK: @llvm.x86.avx512.mask.reduce.ss
   return _mm_mask_reduce_round_ss(__W, __U, __A, __B, 4, 8);
 }
 
 __m128 test_mm_maskz_reduce_round_ss(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_reduce_round_ss
+  // CHECK-LABEL: test_mm_maskz_reduce_round_ss
   // CHECK: @llvm.x86.avx512.mask.reduce.ss
   return _mm_maskz_reduce_round_ss(__U, __A, __B, 4, 8);
 }
 
 __m128d test_mm_reduce_sd(__m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_reduce_sd
+  // CHECK-LABEL: test_mm_reduce_sd
   // CHECK: @llvm.x86.avx512.mask.reduce.sd
   return _mm_reduce_sd(__A, __B, 4);
 }
 
 __m128d test_mm_mask_reduce_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_reduce_sd
+  // CHECK-LABEL: test_mm_mask_reduce_sd
   // CHECK: @llvm.x86.avx512.mask.reduce.sd
   return _mm_mask_reduce_sd(__W, __U, __A, __B, 4);
 }
 
 __m128d test_mm_maskz_reduce_sd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_reduce_sd
+  // CHECK-LABEL: test_mm_maskz_reduce_sd
   // CHECK: @llvm.x86.avx512.mask.reduce.sd
   return _mm_maskz_reduce_sd(__U, __A, __B, 4);
 }
 
 __m128d test_mm_reduce_round_sd(__m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_reduce_round_sd
+  // CHECK-LABEL: test_mm_reduce_round_sd
   // CHECK: @llvm.x86.avx512.mask.reduce.sd
   return _mm_reduce_round_sd(__A, __B, 4, 8);
 }
 
 __m128d test_mm_mask_reduce_round_sd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_reduce_round_sd
+  // CHECK-LABEL: test_mm_mask_reduce_round_sd
   // CHECK: @llvm.x86.avx512.mask.reduce.sd
   return _mm_mask_reduce_round_sd(__W, __U, __A, __B, 4, 8);
 }
 
 __m128d test_mm_maskz_reduce_round_sd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_reduce_round_sd
+  // CHECK-LABEL: test_mm_maskz_reduce_round_sd
   // CHECK: @llvm.x86.avx512.mask.reduce.sd
   return _mm_maskz_reduce_round_sd(__U, __A, __B, 4, 8);
 }
 
 __mmask16 test_mm512_movepi32_mask(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_movepi32_mask
+  // CHECK-LABEL: test_mm512_movepi32_mask
   // CHECK: [[CMP:%.*]] = icmp slt <16 x i32> %{{.*}}, zeroinitializer
   return _mm512_movepi32_mask(__A); 
 }
 
 __m512i test_mm512_movm_epi32(__mmask16 __A) {
-  // CHECK-LABEL: @test_mm512_movm_epi32
+  // CHECK-LABEL: test_mm512_movm_epi32
   // CHECK: %{{.*}} = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: %vpmovm2.i = sext <16 x i1> %{{.*}} to <16 x i32>
   return _mm512_movm_epi32(__A); 
 }
 
 __m512i test_mm512_movm_epi64(__mmask8 __A) {
-  // CHECK-LABEL: @test_mm512_movm_epi64
+  // CHECK-LABEL: test_mm512_movm_epi64
   // CHECK: %{{.*}} = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: %vpmovm2.i = sext <8 x i1> %{{.*}} to <8 x i64>
   return _mm512_movm_epi64(__A); 
 }
 
 __mmask8 test_mm512_movepi64_mask(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_movepi64_mask
+  // CHECK-LABEL: test_mm512_movepi64_mask
   // CHECK: [[CMP:%.*]] = icmp slt <8 x i64> %{{.*}}, zeroinitializer
   return _mm512_movepi64_mask(__A); 
 }
 
 __m512 test_mm512_broadcast_f32x2(__m128 __A) {
-  // CHECK-LABEL: @test_mm512_broadcast_f32x2
+  // CHECK-LABEL: test_mm512_broadcast_f32x2
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   return _mm512_broadcast_f32x2(__A); 
 }
+TEST_CONSTEXPR(match_m512(_mm512_broadcast_f32x2((__m128)(__v4sf){1.0, -2.0, 3.0, -4.0}), 1.0, -2.0, 1.0, -2.0, 1.0, -2.0, 1.0, -2.0, 1.0, -2.0, 1.0, -2.0, 1.0, -2.0, 1.0, -2.0));
 
 __m512 test_mm512_mask_broadcast_f32x2(__m512 __O, __mmask16 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm512_mask_broadcast_f32x2
+  // CHECK-LABEL: test_mm512_mask_broadcast_f32x2
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask_broadcast_f32x2(__O, __M, __A); 
 }
 
 __m512 test_mm512_maskz_broadcast_f32x2(__mmask16 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_broadcast_f32x2
+  // CHECK-LABEL: test_mm512_maskz_broadcast_f32x2
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_maskz_broadcast_f32x2(__M, __A); 
 }
 
 __m512 test_mm512_broadcast_f32x8(float const* __A) {
-  // CHECK-LABEL: @test_mm512_broadcast_f32x8
+  // CHECK-LABEL: test_mm512_broadcast_f32x8
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   return _mm512_broadcast_f32x8(_mm256_loadu_ps(__A)); 
 }
+TEST_CONSTEXPR(match_m512(_mm512_broadcast_f32x8((__m256)(__v8sf){1.0f, 2.0f, 3.0f, 4.0f, -5.0f, -6.0f, -7.0f, -8.0f}), 1.0f, 2.0f, 3.0f, 4.0f, -5.0f, -6.0f, -7.0f, -8.0f, 1.0f, 2.0f, 3.0f, 4.0f, -5.0f, -6.0f, -7.0f, -8.0f));
 
 __m512 test_mm512_mask_broadcast_f32x8(__m512 __O, __mmask16 __M, float const* __A) {
-  // CHECK-LABEL: @test_mm512_mask_broadcast_f32x8
+  // CHECK-LABEL: test_mm512_mask_broadcast_f32x8
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask_broadcast_f32x8(__O, __M, _mm256_loadu_ps(__A)); 
 }
 
 __m512 test_mm512_maskz_broadcast_f32x8(__mmask16 __M, float const* __A) {
-  // CHECK-LABEL: @test_mm512_maskz_broadcast_f32x8
+  // CHECK-LABEL: test_mm512_maskz_broadcast_f32x8
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_maskz_broadcast_f32x8(__M, _mm256_loadu_ps(__A)); 
 }
 
 __m512d test_mm512_broadcast_f64x2(double const* __A) {
-  // CHECK-LABEL: @test_mm512_broadcast_f64x2
+  // CHECK-LABEL: test_mm512_broadcast_f64x2
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   return _mm512_broadcast_f64x2(_mm_loadu_pd(__A)); 
 }
+TEST_CONSTEXPR(match_m512d(_mm512_broadcast_f64x2((__m128d)(__v2df){1.0, -2.0}), 1.0, -2.0, 1.0, -2.0, 1.0, -2.0, 1.0, -2.0));
 
 __m512d test_mm512_mask_broadcast_f64x2(__m512d __O, __mmask8 __M, double const* __A) {
-  // CHECK-LABEL: @test_mm512_mask_broadcast_f64x2
+  // CHECK-LABEL: test_mm512_mask_broadcast_f64x2
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_mask_broadcast_f64x2(__O, __M, _mm_loadu_pd(__A)); 
 }
 
 __m512d test_mm512_maskz_broadcast_f64x2(__mmask8 __M, double const* __A) {
-  // CHECK-LABEL: @test_mm512_maskz_broadcast_f64x2
+  // CHECK-LABEL: test_mm512_maskz_broadcast_f64x2
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_maskz_broadcast_f64x2(__M, _mm_loadu_pd(__A)); 
 }
 
 __m512i test_mm512_broadcast_i32x2(__m128i __A) {
-  // CHECK-LABEL: @test_mm512_broadcast_i32x2
+  // CHECK-LABEL: test_mm512_broadcast_i32x2
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   return _mm512_broadcast_i32x2(__A); 
 }
+TEST_CONSTEXPR(match_v16si(_mm512_broadcast_i32x2((__m128i)(__v4si){1, -2, 3, -4}), 1, -2, 1, -2, 1, -2, 1, -2, 1, -2, 1, -2, 1, -2, 1, -2));
 
 __m512i test_mm512_mask_broadcast_i32x2(__m512i __O, __mmask16 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm512_mask_broadcast_i32x2
+  // CHECK-LABEL: test_mm512_mask_broadcast_i32x2
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_broadcast_i32x2(__O, __M, __A); 
 }
 
 __m512i test_mm512_maskz_broadcast_i32x2(__mmask16 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_broadcast_i32x2
+  // CHECK-LABEL: test_mm512_maskz_broadcast_i32x2
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_broadcast_i32x2(__M, __A); 
 }
 
 __m512i test_mm512_broadcast_i32x8(__m256i const* __A) {
-  // CHECK-LABEL: @test_mm512_broadcast_i32x8
+  // CHECK-LABEL: test_mm512_broadcast_i32x8
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   return _mm512_broadcast_i32x8(_mm256_loadu_si256(__A)); 
 }
+TEST_CONSTEXPR(match_v16si(_mm512_broadcast_i32x8((__m256i)(__v8si){1, 2, 3, 4, -5, -6, -7, -8}), 1, 2, 3, 4, -5, -6, -7, -8, 1, 2, 3, 4, -5, -6, -7, -8));
 
 __m512i test_mm512_mask_broadcast_i32x8(__m512i __O, __mmask16 __M, __m256i const* __A) {
-  // CHECK-LABEL: @test_mm512_mask_broadcast_i32x8
+  // CHECK-LABEL: test_mm512_mask_broadcast_i32x8
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_broadcast_i32x8(__O, __M, _mm256_loadu_si256(__A)); 
 }
 
 __m512i test_mm512_maskz_broadcast_i32x8(__mmask16 __M, __m256i const* __A) {
-  // CHECK-LABEL: @test_mm512_maskz_broadcast_i32x8
+  // CHECK-LABEL: test_mm512_maskz_broadcast_i32x8
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_broadcast_i32x8(__M, _mm256_loadu_si256(__A)); 
 }
 
 __m512i test_mm512_broadcast_i64x2(__m128i const* __A) {
-  // CHECK-LABEL: @test_mm512_broadcast_i64x2
+  // CHECK-LABEL: test_mm512_broadcast_i64x2
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   return _mm512_broadcast_i64x2(_mm_loadu_si128(__A)); 
 }
+TEST_CONSTEXPR(match_v8di(_mm512_broadcast_i64x2((__m128i)(__v2di){1, -2}), 1, -2, 1, -2, 1, -2, 1, -2));
 
 __m512i test_mm512_mask_broadcast_i64x2(__m512i __O, __mmask8 __M, __m128i const* __A) {
-  // CHECK-LABEL: @test_mm512_mask_broadcast_i64x2
+  // CHECK-LABEL: test_mm512_mask_broadcast_i64x2
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_broadcast_i64x2(__O, __M, _mm_loadu_si128(__A)); 
 }
 
 __m512i test_mm512_maskz_broadcast_i64x2(__mmask8 __M, __m128i const* __A) {
-  // CHECK-LABEL: @test_mm512_maskz_broadcast_i64x2
+  // CHECK-LABEL: test_mm512_maskz_broadcast_i64x2
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_broadcast_i64x2(__M, _mm_loadu_si128(__A)); 
 }
 
 __m256 test_mm512_extractf32x8_ps(__m512 __A) {
-  // CHECK-LABEL: @test_mm512_extractf32x8_ps
+  // CHECK-LABEL: test_mm512_extractf32x8_ps
   // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   return _mm512_extractf32x8_ps(__A, 1); 
 }
 
 __m256 test_mm512_mask_extractf32x8_ps(__m256 __W, __mmask8 __U, __m512 __A) {
-  // CHECK-LABEL: @test_mm512_mask_extractf32x8_ps
+  // CHECK-LABEL: test_mm512_mask_extractf32x8_ps
   // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_mask_extractf32x8_ps(__W, __U, __A, 1); 
 }
 
 __m256 test_mm512_maskz_extractf32x8_ps(__mmask8 __U, __m512 __A) {
-  // CHECK-LABEL: @test_mm512_maskz_extractf32x8_ps
+  // CHECK-LABEL: test_mm512_maskz_extractf32x8_ps
   // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm512_maskz_extractf32x8_ps(__U, __A, 1); 
 }
 
 __m128d test_mm512_extractf64x2_pd(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_extractf64x2_pd
+  // CHECK-LABEL: test_mm512_extractf64x2_pd
   // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   return _mm512_extractf64x2_pd(__A, 3); 
 }
 
 __m128d test_mm512_mask_extractf64x2_pd(__m128d __W, __mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_extractf64x2_pd
+  // CHECK-LABEL: test_mm512_mask_extractf64x2_pd
   // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm512_mask_extractf64x2_pd(__W, __U, __A, 3); 
 }
 
 __m128d test_mm512_maskz_extractf64x2_pd(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_maskz_extractf64x2_pd
+  // CHECK-LABEL: test_mm512_maskz_extractf64x2_pd
   // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> poison, <2 x i32> <i32 6, i32 7>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm512_maskz_extractf64x2_pd(__U, __A, 3); 
 }
 
 __m256i test_mm512_extracti32x8_epi32(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_extracti32x8_epi32
+  // CHECK-LABEL: test_mm512_extracti32x8_epi32
   // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   return _mm512_extracti32x8_epi32(__A, 1); 
 }
 
 __m256i test_mm512_mask_extracti32x8_epi32(__m256i __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_extracti32x8_epi32
+  // CHECK-LABEL: test_mm512_mask_extracti32x8_epi32
   // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm512_mask_extracti32x8_epi32(__W, __U, __A, 1); 
 }
 
 __m256i test_mm512_maskz_extracti32x8_epi32(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_extracti32x8_epi32
+  // CHECK-LABEL: test_mm512_maskz_extracti32x8_epi32
   // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm512_maskz_extracti32x8_epi32(__U, __A, 1); 
 }
 
 __m128i test_mm512_extracti64x2_epi64(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_extracti64x2_epi64
+  // CHECK-LABEL: test_mm512_extracti64x2_epi64
   // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> poison, <2 x i32> <i32 6, i32 7>
   return _mm512_extracti64x2_epi64(__A, 3); 
 }
 
 __m128i test_mm512_mask_extracti64x2_epi64(__m128i __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_extracti64x2_epi64
+  // CHECK-LABEL: test_mm512_mask_extracti64x2_epi64
   // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> poison, <2 x i32> <i32 6, i32 7>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm512_mask_extracti64x2_epi64(__W, __U, __A, 3); 
 }
 
 __m128i test_mm512_maskz_extracti64x2_epi64(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_extracti64x2_epi64
+  // CHECK-LABEL: test_mm512_maskz_extracti64x2_epi64
   // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> poison, <2 x i32> <i32 6, i32 7>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm512_maskz_extracti64x2_epi64(__U, __A, 3); 
 }
 
 __m512 test_mm512_insertf32x8(__m512 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm512_insertf32x8
+  // CHECK-LABEL: test_mm512_insertf32x8
   // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
   return _mm512_insertf32x8(__A, __B, 1); 
 }
 
 __m512 test_mm512_mask_insertf32x8(__m512 __W, __mmask16 __U, __m512 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm512_mask_insertf32x8
+  // CHECK-LABEL: test_mm512_mask_insertf32x8
   // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask_insertf32x8(__W, __U, __A, __B, 1); 
 }
 
 __m512 test_mm512_maskz_insertf32x8(__mmask16 __U, __m512 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm512_maskz_insertf32x8
+  // CHECK-LABEL: test_mm512_maskz_insertf32x8
   // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_maskz_insertf32x8(__U, __A, __B, 1); 
 }
 
 __m512d test_mm512_insertf64x2(__m512d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm512_insertf64x2
+  // CHECK-LABEL: test_mm512_insertf64x2
   // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
   return _mm512_insertf64x2(__A, __B, 3); 
 }
 
 __m512d test_mm512_mask_insertf64x2(__m512d __W, __mmask8 __U, __m512d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm512_mask_insertf64x2
+  // CHECK-LABEL: test_mm512_mask_insertf64x2
   // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_mask_insertf64x2(__W, __U, __A, __B, 3); 
 }
 
 __m512d test_mm512_maskz_insertf64x2(__mmask8 __U, __m512d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm512_maskz_insertf64x2
+  // CHECK-LABEL: test_mm512_maskz_insertf64x2
   // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_maskz_insertf64x2(__U, __A, __B, 3); 
 }
 
 __m512i test_mm512_inserti32x8(__m512i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm512_inserti32x8
+  // CHECK-LABEL: test_mm512_inserti32x8
   // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
   return _mm512_inserti32x8(__A, __B, 1); 
 }
 
 __m512i test_mm512_mask_inserti32x8(__m512i __W, __mmask16 __U, __m512i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm512_mask_inserti32x8
+  // CHECK-LABEL: test_mm512_mask_inserti32x8
   // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_inserti32x8(__W, __U, __A, __B, 1); 
 }
 
 __m512i test_mm512_maskz_inserti32x8(__mmask16 __U, __m512i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_inserti32x8
+  // CHECK-LABEL: test_mm512_maskz_inserti32x8
   // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_inserti32x8(__U, __A, __B, 1); 
 }
 
 __m512i test_mm512_inserti64x2(__m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_inserti64x2
+  // CHECK-LABEL: test_mm512_inserti64x2
   // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
   return _mm512_inserti64x2(__A, __B, 1); 
 }
 
 __m512i test_mm512_mask_inserti64x2(__m512i __W, __mmask8 __U, __m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_mask_inserti64x2
+  // CHECK-LABEL: test_mm512_mask_inserti64x2
   // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_inserti64x2(__W, __U, __A, __B, 1); 
 }
 
 __m512i test_mm512_maskz_inserti64x2(__mmask8 __U, __m512i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_inserti64x2
+  // CHECK-LABEL: test_mm512_maskz_inserti64x2
   // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_inserti64x2(__U, __A, __B, 1); 
 }
 __mmask8 test_mm512_mask_fpclass_pd_mask(__mmask8 __U, __m512d __A) {
-  // CHECK-LABEL: @test_mm512_mask_fpclass_pd_mask
+  // CHECK-LABEL: test_mm512_mask_fpclass_pd_mask
   // CHECK: @llvm.x86.avx512.fpclass.pd.512
   return _mm512_mask_fpclass_pd_mask(__U, __A, 4); 
 }
 
 __mmask8 test_mm512_fpclass_pd_mask(__m512d __A) {
-  // CHECK-LABEL: @test_mm512_fpclass_pd_mask
+  // CHECK-LABEL: test_mm512_fpclass_pd_mask
   // CHECK: @llvm.x86.avx512.fpclass.pd.512
   return _mm512_fpclass_pd_mask(__A, 4); 
 }
 
 __mmask16 test_mm512_mask_fpclass_ps_mask(__mmask16 __U, __m512 __A) {
-  // CHECK-LABEL: @test_mm512_mask_fpclass_ps_mask
+  // CHECK-LABEL: test_mm512_mask_fpclass_ps_mask
   // CHECK: @llvm.x86.avx512.fpclass.ps.512
   return _mm512_mask_fpclass_ps_mask(__U, __A, 4); 
 }
 
 __mmask16 test_mm512_fpclass_ps_mask(__m512 __A) {
-  // CHECK-LABEL: @test_mm512_fpclass_ps_mask
+  // CHECK-LABEL: test_mm512_fpclass_ps_mask
   // CHECK: @llvm.x86.avx512.fpclass.ps.512
   return _mm512_fpclass_ps_mask(__A, 4); 
 }
 
 __mmask8 test_mm_fpclass_sd_mask(__m128d __A)  { 
-  // CHECK-LABEL: @test_mm_fpclass_sd_mask
+  // CHECK-LABEL: test_mm_fpclass_sd_mask
   // CHECK: @llvm.x86.avx512.mask.fpclass.sd
  return _mm_fpclass_sd_mask (__A, 2);
 }
 
 __mmask8 test_mm_mask_fpclass_sd_mask(__mmask8 __U, __m128d __A)  {
- // CHECK-LABEL: @test_mm_mask_fpclass_sd_mask
+ // CHECK-LABEL: test_mm_mask_fpclass_sd_mask
  // CHECK: @llvm.x86.avx512.mask.fpclass.sd
  return _mm_mask_fpclass_sd_mask (__U,  __A, 2);
 }
 
 __mmask8 test_mm_fpclass_ss_mask(__m128 __A)  { 
- // CHECK-LABEL: @test_mm_fpclass_ss_mask
+ // CHECK-LABEL: test_mm_fpclass_ss_mask
  // CHECK: @llvm.x86.avx512.mask.fpclass.ss
  return _mm_fpclass_ss_mask ( __A, 2);
 }
 
 __mmask8 test_mm_mask_fpclass_ss_mask(__mmask8 __U, __m128 __A)  {
- // CHECK-LABEL: @test_mm_mask_fpclass_ss_mask
+ // CHECK-LABEL: test_mm_mask_fpclass_ss_mask
  // CHECK: @llvm.x86.avx512.mask.fpclass.ss
  return _mm_mask_fpclass_ss_mask (__U, __A, 2);
 }
diff --git a/clang/test/CodeGen/X86/avx512f-builtins.c b/clang/test/CodeGen/X86/avx512f-builtins.c
index 84e700cfbd378..29916be23cbae 100644
--- a/clang/test/CodeGen/X86/avx512f-builtins.c
+++ b/clang/test/CodeGen/X86/avx512f-builtins.c
@@ -3,6 +3,11 @@
 // RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512f -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
 // RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +avx512f -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
 
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512f -emit-llvm -o - -Wall -Werror -Wsign-conversion -fexperimental-new-constant-interpreter | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +avx512f -emit-llvm -o - -Wall -Werror -Wsign-conversion -fexperimental-new-constant-interpreter | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512f -emit-llvm -o - -Wall -Werror -Wsign-conversion -fexperimental-new-constant-interpreter | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +avx512f -emit-llvm -o - -Wall -Werror -Wsign-conversion -fexperimental-new-constant-interpreter | FileCheck %s
+
 #include <immintrin.h>
 #include "builtin_test_helpers.h"
 
@@ -154,6 +159,7 @@ __m512 test_mm512_add_ps(__m512 a, __m512 b)
   // CHECK: fadd <16 x float>
   return _mm512_add_ps(a, b);
 }
+TEST_CONSTEXPR(match_m512(_mm512_add_ps((__m512){-1.0f, -2.0f, -3.0f, -4.0f, -5.0f, -6.0f, -7.0f, -8.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}, (__m512){-1.0f, -2.0f, -3.0f, -4.0f, -5.0f, -6.0f, -7.0f, -8.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}), -2.0f, -4.0f, -6.0f, -8.0f, -10.0f, -12.0f, -14.0f, -16.0f, +2.0f, +4.0f, +6.0f, +8.0f, +10.0f, +12.0f, +14.0f, +16.0f));
 
 __m512d test_mm512_add_pd(__m512d a, __m512d b)
 {
@@ -161,6 +167,7 @@ __m512d test_mm512_add_pd(__m512d a, __m512d b)
   // CHECK: fadd <8 x double>
   return _mm512_add_pd(a, b);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_add_pd((__m512d){-1.0, -2.0, -3.0, -4.0, +1.0, +2.0, +3.0, +4.0}, (__m512d){-1.0, -2.0, -3.0, -4.0, +1.0, +2.0, +3.0, +4.0}), -2.0, -4.0, -6.0, -8.0, +2.0, +4.0, +6.0, +8.0));
 
 __m512 test_mm512_mul_ps(__m512 a, __m512 b)
 {
@@ -168,6 +175,7 @@ __m512 test_mm512_mul_ps(__m512 a, __m512 b)
   // CHECK: fmul <16 x float>
   return _mm512_mul_ps(a, b);
 }
+TEST_CONSTEXPR(match_m512(_mm512_mul_ps((__m512){-1.0f, -2.0f, -3.0f, -4.0f, -5.0f, -6.0f, -7.0f, -8.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}, (__m512){-1.0f, -2.0f, -3.0f, -4.0f, -5.0f, -6.0f, -7.0f, -8.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}), +1.0f, +4.0f, +9.0f, +16.0f, +25.0f, +36.0f, +49.0f, +64.0f, +1.0f, +4.0f, +9.0f, +16.0f, +25.0f, +36.0f, +49.0f, +64.0f));
 
 __m512d test_mm512_mul_pd(__m512d a, __m512d b)
 {
@@ -175,6 +183,7 @@ __m512d test_mm512_mul_pd(__m512d a, __m512d b)
   // CHECK: fmul <8 x double>
   return _mm512_mul_pd(a, b);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_mul_pd((__m512d){-1.0, -2.0, -3.0, -4.0, +1.0, +2.0, +3.0, +4.0}, (__m512d){-1.0, -2.0, -3.0, -4.0, +1.0, +2.0, +3.0, +4.0}), +1.0, +4.0, +9.0, +16.0, +1.0, +4.0, +9.0, +16.0));
 
 void test_mm512_storeu_si512 (void *__P, __m512i __A)
 {
@@ -435,6 +444,22 @@ __m512d test_mm512_set1_pd(double d)
   // CHECK: insertelement <8 x double> {{.*}}, i32 7
   return _mm512_set1_pd(d);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_set1_pd(-100.0), -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0, -100.0));
+
+__m512 test_mm512_set1_ps(float d)
+{
+  // CHECK-LABEL: test_mm512_set1_ps
+  // CHECK: insertelement <16 x float> {{.*}}, i32 0
+  // CHECK: insertelement <16 x float> {{.*}}, i32 1
+  // CHECK: insertelement <16 x float> {{.*}}, i32 2
+  // CHECK: insertelement <16 x float> {{.*}}, i32 3
+  // CHECK: insertelement <16 x float> {{.*}}, i32 4
+  // CHECK: insertelement <16 x float> {{.*}}, i32 5
+  // CHECK: insertelement <16 x float> {{.*}}, i32 6
+  // CHECK: insertelement <16 x float> {{.*}}, i32 15
+  return _mm512_set1_ps(d);
+}
+TEST_CONSTEXPR(match_m512(_mm512_set1_ps(-55.0f), -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f, -55.0f));
 
 __mmask16 test_mm512_knot(__mmask16 a)
 {
@@ -758,6 +783,8 @@ __m512 test_mm512_fmadd_ps(__m512 __A, __m512 __B, __m512 __C) {
 __m512 test_mm512_mask_fmadd_ps(__m512 __A, __mmask16 __U, __m512 __B, __m512 __C) {
   // CHECK-LABEL: test_mm512_mask_fmadd_ps
   // CHECK: call {{.*}}<16 x float> @llvm.fma.v16f32(<16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}})
+  // CHECK: bitcast i16 %{{.*}} to <16 x i1>
+  // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask_fmadd_ps(__A, __U, __B, __C);
 }
 __m512 test_mm512_mask3_fmadd_ps(__m512 __A, __m512 __B, __m512 __C, __mmask16 __U) {
@@ -1261,6 +1288,7 @@ __m512d test_mm512_unpackhi_pd(__m512d a, __m512d b)
   // CHECK: shufflevector <8 x double> {{.*}} <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
   return _mm512_unpackhi_pd(a, b);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_unpackhi_pd((__m512d){1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}, (__m512d){9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0}), +2.0, +10.0, +4.0, +12.0, +6.0, +14.0, +8.0, +16.0));
 
 __m512d test_mm512_unpacklo_pd(__m512d a, __m512d b)
 {
@@ -1268,6 +1296,7 @@ __m512d test_mm512_unpacklo_pd(__m512d a, __m512d b)
   // CHECK: shufflevector <8 x double> {{.*}} <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
   return _mm512_unpacklo_pd(a, b);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_unpacklo_pd((__m512d){1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}, (__m512d){9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0}), +1.0, +9.0, +3.0, +11.0, +5.0, +13.0, +7.0, +15.0));
 
 __m512 test_mm512_unpackhi_ps(__m512 a, __m512 b)
 {
@@ -1275,6 +1304,7 @@ __m512 test_mm512_unpackhi_ps(__m512 a, __m512 b)
   // CHECK: shufflevector <16 x float> {{.*}} <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
   return _mm512_unpackhi_ps(a, b);
 }
+TEST_CONSTEXPR(match_m512(_mm512_unpackhi_ps((__m512){0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}, (__m512){16.0f, 17.0f, 18.0f, 19.0f, 20.0f, 21.0f, 22.0f, 23.0f, 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0f, 31.0f}), +2.0f, +18.0f, +3.0f, +19.0f, +6.0f, +22.0f, +7.0f, +23.0f, +10.0f, +26.0f, +11.0f, +27.0f, +14.0f, +30.0f, +15.0f, +31.0f));
 
 __m512 test_mm512_unpacklo_ps(__m512 a, __m512 b)
 {
@@ -1282,6 +1312,7 @@ __m512 test_mm512_unpacklo_ps(__m512 a, __m512 b)
   // CHECK: shufflevector <16 x float> {{.*}} <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
   return _mm512_unpacklo_ps(a, b);
 }
+TEST_CONSTEXPR(match_m512(_mm512_unpacklo_ps((__m512){0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, 13.0f, 14.0f, 15.0f}, (__m512){16.0f, 17.0f, 18.0f, 19.0f, 20.0f, 21.0f, 22.0f, 23.0f, 24.0f, 25.0f, 26.0f, 27.0f, 28.0f, 29.0f, 30.0f, 31.0f}), +0.0f, +16.0f, +1.0f, +17.0f, +4.0f, +20.0f, +5.0f, +21.0f, +8.0f, +24.0f, +9.0f, +25.0f, +12.0f, +28.0f, +13.0f, +29.0f));
 
 __mmask16 test_mm512_cmp_round_ps_mask(__m512 a, __m512 b) {
   // CHECK-LABEL: test_mm512_cmp_round_ps_mask
@@ -2811,36 +2842,42 @@ __m512i test_mm512_and_epi32(__m512i __src,__mmask16 __k, __m512i __a, __m512i _
   // CHECK: and <16 x i32>
   return _mm512_and_epi32(__a, __b);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_and_epi32((__m512i)(__v16si){0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v16si){0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1));
 
 __m512i test_mm512_and_epi64(__m512i __src,__mmask8 __k, __m512i __a, __m512i __b) {
   // CHECK-LABEL: test_mm512_and_epi64
   // CHECK: and <8 x i64>
   return _mm512_and_epi64(__a, __b);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_and_epi64((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, 0, -1, 0, 0, 0, -1));
 
 __m512i test_mm512_or_epi32(__m512i __src,__mmask16 __k, __m512i __a, __m512i __b) {
   // CHECK-LABEL: test_mm512_or_epi32
   // CHECK: or <16 x i32>
   return _mm512_or_epi32(__a, __b);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_or_epi32((__m512i)(__v16si){0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v16si){0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1}), 0, -1, -1, -1, 0, -1, -1, -1, 0, -1, -1, -1, 0, -1, -1, -1));
 
 __m512i test_mm512_or_epi64(__m512i __src,__mmask8 __k, __m512i __a, __m512i __b) {
   // CHECK-LABEL: test_mm512_or_epi64
   // CHECK: or <8 x i64>
   return _mm512_or_epi64(__a, __b);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_or_epi64((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, -1, -1, -1, 0, -1, -1, -1));
 
 __m512i test_mm512_xor_epi32(__m512i __src,__mmask16 __k, __m512i __a, __m512i __b) {
   // CHECK-LABEL: test_mm512_xor_epi32
   // CHECK: xor <16 x i32>
   return _mm512_xor_epi32(__a, __b);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_xor_epi32((__m512i)(__v16si){0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v16si){0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1}), 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0, 0, -1, -1, 0));
 
 __m512i test_mm512_xor_epi64(__m512i __src,__mmask8 __k, __m512i __a, __m512i __b) {
   // CHECK-LABEL: test_mm512_xor_epi64
   // CHECK: xor <8 x i64>
   return _mm512_xor_epi64(__a, __b);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_xor_epi64((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, -1, -1, 0, 0, -1, -1, 0));
 
 __m512i test_mm512_maskz_andnot_epi32 (__mmask16 __k,__m512i __A, __m512i __B){
   // CHECK-LABEL: test_mm512_maskz_andnot_epi32
@@ -2869,6 +2906,7 @@ __m512i test_mm512_andnot_si512(__m512i __A, __m512i __B)
 
   return _mm512_andnot_si512(__A, __B);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_andnot_si512((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, -1, 0, 0, -1, 0, 0));
 
 __m512i test_mm512_andnot_epi32(__m512i __A, __m512i __B) {
   // CHECK-LABEL: test_mm512_andnot_epi32
@@ -2876,6 +2914,7 @@ __m512i test_mm512_andnot_epi32(__m512i __A, __m512i __B) {
   // CHECK: and <16 x i32> %{{.*}}, %{{.*}}
   return _mm512_andnot_epi32(__A,__B);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_andnot_epi32((__m512i)(__v16si){0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v16si){0, 0, -1, -1, 0, -1, 0, -1, 0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, -1, 0, 0, -1, 0, 0, 0, 0, -1, 0, 0, -1, 0, 0));
 
 __m512i test_mm512_maskz_andnot_epi64 (__mmask8 __k,__m512i __A, __m512i __B) {
   // CHECK-LABEL: test_mm512_maskz_andnot_epi64
@@ -2900,6 +2939,7 @@ __m512i test_mm512_andnot_epi64(__m512i __A, __m512i __B) {
   // CHECK: and <8 x i64> %{{.*}}, %{{.*}}
   return _mm512_andnot_epi64(__A,__B);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_andnot_epi64((__m512i)(__v8di){0, -1, 0, -1, 0, 0, -1, -1}, (__m512i)(__v8di){0, 0, -1, -1, 0, -1, 0, -1}), 0, 0, -1, 0, 0, -1, 0, 0));
 
 __m512i test_mm512_maskz_sub_epi32 (__mmask16 __k,__m512i __A, __m512i __B) {
   //CHECK-LABEL: test_mm512_maskz_sub_epi32
@@ -2994,6 +3034,7 @@ __m512i test_mm512_mul_epi32(__m512i __A, __m512i __B) {
   //CHECK: mul <8 x i64> %{{.*}}, %{{.*}}
   return _mm512_mul_epi32(__A,__B);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_mul_epi32((__m512i)(__v16si){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16}, (__m512i)(__v16si){-32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), -32, 84, -120, 140, -144, 132, -104, -60));
 
 __m512i test_mm512_maskz_mul_epi32 (__mmask8 __k,__m512i __A, __m512i __B) {
   //CHECK-LABEL: test_mm512_maskz_mul_epi32
@@ -3024,6 +3065,7 @@ __m512i test_mm512_mul_epu32 (__m512i __A, __m512i __B) {
   //CHECK: mul <8 x i64> %{{.*}}, %{{.*}}
   return _mm512_mul_epu32(__A,__B);
 }
+TEST_CONSTEXPR(match_m512i(_mm512_mul_epu32((__m512i)(__v16si){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16}, (__m512i)(__v16si){-32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), 4294967264, 84, 21474836360, 140, 38654705520, 132, 55834574744, 64424509380));
 
 __m512i test_mm512_maskz_mul_epu32 (__mmask8 __k,__m512i __A, __m512i __B) {
   //CHECK-LABEL: test_mm512_maskz_mul_epu32
@@ -3542,6 +3584,7 @@ __m512d test_mm512_div_pd(__m512d __a, __m512d __b) {
   // CHECK: fdiv <8 x double>
   return _mm512_div_pd(__a,__b); 
 }
+TEST_CONSTEXPR(match_m512d(_mm512_div_pd((__m512d){+8.0, +6.0, +4.0, +2.0, -8.0, -6.0, -4.0, -2.0}, (__m512d){+2.0, +2.0, +2.0, +2.0, -2.0, -2.0, -2.0, -2.0}), +4.0, +3.0, +2.0, +1.0, +4.0, +3.0, +2.0, +1.0));
 __m512d test_mm512_mask_div_pd(__m512d __w, __mmask8 __u, __m512d __a, __m512d __b) {
   // CHECK-LABEL: test_mm512_mask_div_pd
   // CHECK: fdiv <8 x double> %{{.*}}, %{{.*}}
@@ -3576,6 +3619,7 @@ __m512 test_mm512_div_ps(__m512 __A, __m512 __B) {
   // CHECK: fdiv <16 x float>
   return _mm512_div_ps(__A,__B); 
 }
+TEST_CONSTEXPR(match_m512(_mm512_div_ps((__m512){+16.0f, +14.0f, +12.0f, +10.0f, +8.0f, +6.0f, +4.0f, +2.0f, -16.0f, -14.0f, -12.0f, -10.0f, -8.0f, -6.0f, -4.0f, -2.0f}, (__m512){+2.0f, +2.0f, +2.0f, +2.0f, +2.0f, +2.0f, +2.0f, +2.0f, -2.0f, -2.0f, -2.0f, -2.0f, -2.0f, -2.0f, -2.0f, -2.0f}), +8.0f, +7.0f, +6.0f, +5.0f, +4.0f, +3.0f, +2.0f, +1.0f, +8.0f, +7.0f, +6.0f, +5.0f, +4.0f, +3.0f, +2.0f, +1.0f));
 __m512 test_mm512_mask_div_ps(__m512 __W, __mmask16 __U, __m512 __A, __m512 __B) {
   // CHECK-LABEL: test_mm512_mask_div_ps
   // CHECK: fdiv <16 x float> %{{.*}}, %{{.*}}
@@ -3805,6 +3849,8 @@ __m512i test_mm512_cvtepi8_epi32(__m128i __A) {
   return _mm512_cvtepi8_epi32(__A); 
 }
 
+TEST_CONSTEXPR(match_v16si(_mm512_cvtepi8_epi32(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), -3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12));
+
 __m512i test_mm512_mask_cvtepi8_epi32(__m512i __W, __mmask16 __U, __m128i __A) {
   // CHECK-LABEL: test_mm512_mask_cvtepi8_epi32
   // CHECK: sext <16 x i8> %{{.*}} to <16 x i32>
@@ -3825,6 +3871,8 @@ __m512i test_mm512_cvtepi8_epi64(__m128i __A) {
   return _mm512_cvtepi8_epi64(__A); 
 }
 
+TEST_CONSTEXPR(match_v8di(_mm512_cvtepi8_epi64(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 0, 0, 0, 0, 0, 0, 0, 0)), -3, 2, -1, 0, 1, -2, 3, -4));
+
 __m512i test_mm512_mask_cvtepi8_epi64(__m512i __W, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: test_mm512_mask_cvtepi8_epi64
   // CHECK: sext <8 x i8> %{{.*}} to <8 x i64>
@@ -3845,6 +3893,8 @@ __m512i test_mm512_cvtepi32_epi64(__m256i __X) {
   return _mm512_cvtepi32_epi64(__X); 
 }
 
+TEST_CONSTEXPR(match_v8di(_mm512_cvtepi32_epi64(_mm256_setr_epi32(-70000, 2, -1, 0, 1, -2, 3, -4)), -70000, 2, -1, 0, 1, -2, 3, -4));
+
 __m512i test_mm512_mask_cvtepi32_epi64(__m512i __W, __mmask8 __U, __m256i __X) {
   // CHECK-LABEL: test_mm512_mask_cvtepi32_epi64
   // CHECK: sext <8 x i32> %{{.*}} to <8 x i64>
@@ -3865,6 +3915,8 @@ __m512i test_mm512_cvtepi16_epi32(__m256i __A) {
   return _mm512_cvtepi16_epi32(__A); 
 }
 
+TEST_CONSTEXPR(match_v16si(_mm512_cvtepi16_epi32(_mm256_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), -300, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12));
+
 __m512i test_mm512_mask_cvtepi16_epi32(__m512i __W, __mmask16 __U, __m256i __A) {
   // CHECK-LABEL: test_mm512_mask_cvtepi16_epi32
   // CHECK: sext <16 x i16> %{{.*}} to <16 x i32>
@@ -3885,6 +3937,8 @@ __m512i test_mm512_cvtepi16_epi64(__m128i __A) {
   return _mm512_cvtepi16_epi64(__A); 
 }
 
+TEST_CONSTEXPR(match_v8di(_mm512_cvtepi16_epi64(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), -300, 2, -1, 0, 1, -2, 3, -4));
+
 __m512i test_mm512_mask_cvtepi16_epi64(__m512i __W, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: test_mm512_mask_cvtepi16_epi64
   // CHECK: sext <8 x i16> %{{.*}} to <8 x i64>
@@ -3905,6 +3959,8 @@ __m512i test_mm512_cvtepu8_epi32(__m128i __A) {
   return _mm512_cvtepu8_epi32(__A); 
 }
 
+TEST_CONSTEXPR(match_v16si(_mm512_cvtepu8_epi32(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), 253, 2, 255, 0, 1, 254, 3, 252, 5, 250, 7, 248, 9, 246, 11, 244));
+
 __m512i test_mm512_mask_cvtepu8_epi32(__m512i __W, __mmask16 __U, __m128i __A) {
   // CHECK-LABEL: test_mm512_mask_cvtepu8_epi32
   // CHECK: zext <16 x i8> %{{.*}} to <16 x i32>
@@ -3925,6 +3981,8 @@ __m512i test_mm512_cvtepu8_epi64(__m128i __A) {
   return _mm512_cvtepu8_epi64(__A); 
 }
 
+TEST_CONSTEXPR(match_v8di(_mm512_cvtepu8_epi64(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 0, 0, 0, 0, 0, 0, 0, 0)), 253, 2, 255, 0, 1, 254, 3, 252));
+
 __m512i test_mm512_mask_cvtepu8_epi64(__m512i __W, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: test_mm512_mask_cvtepu8_epi64
   // CHECK: zext <8 x i8> %{{.*}} to <8 x i64>
@@ -3945,6 +4003,8 @@ __m512i test_mm512_cvtepu32_epi64(__m256i __X) {
   return _mm512_cvtepu32_epi64(__X); 
 }
 
+TEST_CONSTEXPR(match_v8di(_mm512_cvtepu32_epi64(_mm256_setr_epi32(-70000, 2, -1, 0, 1, -2, 3, -4)), 4294897296, 2, 4294967295, 0, 1, 4294967294, 3, 4294967292));
+
 __m512i test_mm512_mask_cvtepu32_epi64(__m512i __W, __mmask8 __U, __m256i __X) {
   // CHECK-LABEL: test_mm512_mask_cvtepu32_epi64
   // CHECK: zext <8 x i32> %{{.*}} to <8 x i64>
@@ -3965,6 +4025,8 @@ __m512i test_mm512_cvtepu16_epi32(__m256i __A) {
   return _mm512_cvtepu16_epi32(__A); 
 }
 
+TEST_CONSTEXPR(match_v16si(_mm512_cvtepu16_epi32(_mm256_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4, 5, -6, 7, -8, 9, -10, 11, -12)), 65236, 2, 65535, 0, 1, 65534, 3, 65532, 5, 65530, 7, 65528, 9, 65526, 11, 65524));
+
 __m512i test_mm512_mask_cvtepu16_epi32(__m512i __W, __mmask16 __U, __m256i __A) {
   // CHECK-LABEL: test_mm512_mask_cvtepu16_epi32
   // CHECK: zext <16 x i16> %{{.*}} to <16 x i32>
@@ -3985,6 +4047,8 @@ __m512i test_mm512_cvtepu16_epi64(__m128i __A) {
   return _mm512_cvtepu16_epi64(__A); 
 }
 
+TEST_CONSTEXPR(match_v8di(_mm512_cvtepu16_epi64(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), 65236, 2, 65535, 0, 1, 65534, 3, 65532));
+
 __m512i test_mm512_mask_cvtepu16_epi64(__m512i __W, __mmask8 __U, __m128i __A) {
   // CHECK-LABEL: test_mm512_mask_cvtepu16_epi64
   // CHECK: zext <8 x i16> %{{.*}} to <8 x i64>
@@ -4386,6 +4450,7 @@ __m512d test_mm512_movedup_pd(__m512d __A) {
   // CHECK: shufflevector <8 x double> %{{.*}}, <8 x double> %{{.*}}, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   return _mm512_movedup_pd(__A);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_movedup_pd((__m512d){-1.0, +2.0, +3.0, +4.0, -5.0, -6.0, +7.0, +8.0}), -1.0, -1.0, +3.0, +3.0, -5.0, -5.0, +7.0, +7.0));
 
 __m512d test_mm512_mask_movedup_pd(__m512d __W, __mmask8 __U, __m512d __A) {
   // CHECK-LABEL: test_mm512_mask_movedup_pd
@@ -4635,6 +4700,7 @@ __m512i test_mm512_unpackhi_epi32(__m512i __A, __m512i __B) {
   // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
   return _mm512_unpackhi_epi32(__A, __B); 
 }
+TEST_CONSTEXPR(match_v16si(_mm512_unpackhi_epi32((__m512i)(__v16si){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, (__m512i)(__v16si){16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}), 2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31));
 
 __m512d test_mm512_maskz_unpackhi_pd(__mmask8 __U, __m512d __A, __m512d __B) {
   // CHECK-LABEL: test_mm512_maskz_unpackhi_pd
@@ -5421,6 +5487,7 @@ __m512i test_mm512_unpackhi_epi64(__m512i __A, __m512i __B) {
   // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
   return _mm512_unpackhi_epi64(__A, __B); 
 }
+TEST_CONSTEXPR(match_m512i(_mm512_unpackhi_epi64((__m512i){0, 1, 2, 3, 4, 5, 6, 7}, (__m512i){8, 9, 10, 11, 12, 13, 14, 15}), 1, 9, 3, 11, 5, 13, 7, 15));
 
 __m512i test_mm512_mask_unpackhi_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {
   // CHECK-LABEL: test_mm512_mask_unpackhi_epi64
@@ -5441,6 +5508,7 @@ __m512i test_mm512_unpacklo_epi32(__m512i __A, __m512i __B) {
   // CHECK: shufflevector <16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
   return _mm512_unpacklo_epi32(__A, __B); 
 }
+TEST_CONSTEXPR(match_v16si(_mm512_unpacklo_epi32((__m512i)(__v16si){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, (__m512i)(__v16si){16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}), 0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29));
 
 __m512i test_mm512_mask_unpacklo_epi32(__m512i __W, __mmask16 __U, __m512i __A, __m512i __B) {
   // CHECK-LABEL: test_mm512_mask_unpacklo_epi32
@@ -5461,6 +5529,7 @@ __m512i test_mm512_unpacklo_epi64(__m512i __A, __m512i __B) {
   // CHECK: shufflevector <8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
   return _mm512_unpacklo_epi64(__A, __B); 
 }
+TEST_CONSTEXPR(match_m512i(_mm512_unpacklo_epi64((__m512i){0, 1, 2, 3, 4, 5, 6, 7}, (__m512i){8, 9, 10, 11, 12, 13, 14, 15}), 0, 8, 2, 10, 4, 12, 6, 14));
 
 __m512i test_mm512_mask_unpacklo_epi64(__m512i __W, __mmask8 __U, __m512i __A, __m512i __B) {
   // CHECK-LABEL: test_mm512_mask_unpacklo_epi64
@@ -6251,6 +6320,7 @@ __m512 test_mm512_broadcast_f32x4(float const* __A) {
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   return _mm512_broadcast_f32x4(_mm_loadu_ps(__A)); 
 }
+TEST_CONSTEXPR(match_m512(_mm512_broadcast_f32x4((__m128)(__v4sf){1.0f, 2.0f, -3.0f, -4.0f}), 1.0f, 2.0f, -3.0f, -4.0f, 1.0f, 2.0f, -3.0f, -4.0f, 1.0f, 2.0f, -3.0f, -4.0f, 1.0f, 2.0f, -3.0f, -4.0f));
 
 __m512 test_mm512_mask_broadcast_f32x4(__m512 __O, __mmask16 __M, float const* __A) {
   // CHECK-LABEL: test_mm512_mask_broadcast_f32x4
@@ -6271,6 +6341,7 @@ __m512d test_mm512_broadcast_f64x4(double const* __A) {
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   return _mm512_broadcast_f64x4(_mm256_loadu_pd(__A)); 
 }
+TEST_CONSTEXPR(match_m512d(_mm512_broadcast_f64x4((__m256d)(__v4df){1.0, 2.0, -3.0, -4.0}), 1.0, 2.0, -3.0, -4.0, 1.0, 2.0, -3.0, -4.0));
 
 __m512d test_mm512_mask_broadcast_f64x4(__m512d __O, __mmask8 __M, double const* __A) {
   // CHECK-LABEL: test_mm512_mask_broadcast_f64x4
@@ -6291,6 +6362,7 @@ __m512i test_mm512_broadcast_i32x4(__m128i const* __A) {
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   return _mm512_broadcast_i32x4(_mm_loadu_si128(__A)); 
 }
+TEST_CONSTEXPR(match_v16si(_mm512_broadcast_i32x4((__m128i)(__v4si){1, 2, -3, -4}), 1, 2, -3, -4, 1, 2, -3, -4, 1, 2, -3, -4, 1, 2, -3, -4));
 
 __m512i test_mm512_mask_broadcast_i32x4(__m512i __O, __mmask16 __M, __m128i const* __A) {
   // CHECK-LABEL: test_mm512_mask_broadcast_i32x4
@@ -6311,6 +6383,7 @@ __m512i test_mm512_broadcast_i64x4(__m256i const* __A) {
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   return _mm512_broadcast_i64x4(_mm256_loadu_si256(__A)); 
 }
+TEST_CONSTEXPR(match_v8di(_mm512_broadcast_i64x4((__m256i)(__v4di){1, 2, -3, -4}), 1, 2, -3, -4, 1, 2, -3, -4));
 
 __m512i test_mm512_mask_broadcast_i64x4(__m512i __O, __mmask8 __M, __m256i const* __A) {
   // CHECK-LABEL: test_mm512_mask_broadcast_i64x4
@@ -6331,6 +6404,7 @@ __m512d test_mm512_broadcastsd_pd(__m128d __A) {
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <8 x i32> zeroinitializer
   return _mm512_broadcastsd_pd(__A);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_broadcastsd_pd((__m128d)(__v2df){1.0, 2.0}), 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0));
 
 __m512d test_mm512_mask_broadcastsd_pd(__m512d __O, __mmask8 __M, __m128d __A) {
   // CHECK-LABEL: test_mm512_mask_broadcastsd_pd
@@ -6351,6 +6425,7 @@ __m512 test_mm512_broadcastss_ps(__m128 __A) {
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <16 x i32> zeroinitializer
   return _mm512_broadcastss_ps(__A);
 }
+TEST_CONSTEXPR(match_m512(_mm512_broadcastss_ps((__m128)(__v4sf){1.0f, 2.0f, -3.0f, -4.0f}), 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f));
 
 __m512 test_mm512_mask_broadcastss_ps(__m512 __O, __mmask16 __M, __m128 __A) {
   // CHECK-LABEL: test_mm512_mask_broadcastss_ps
@@ -6371,6 +6446,7 @@ __m512i test_mm512_broadcastd_epi32(__m128i __A) {
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <16 x i32> zeroinitializer
   return _mm512_broadcastd_epi32(__A);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_broadcastd_epi32((__m128i)(__v4si){-42, 0, 0, 0}), -42, -42, -42, -42, -42, -42, -42, -42, -42, -42, -42, -42, -42, -42, -42, -42));
 
 __m512i test_mm512_mask_broadcastd_epi32(__m512i __O, __mmask16 __M, __m128i __A) {
   // CHECK-LABEL: test_mm512_mask_broadcastd_epi32
@@ -6391,6 +6467,7 @@ __m512i test_mm512_broadcastq_epi64(__m128i __A) {
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <8 x i32> zeroinitializer
   return _mm512_broadcastq_epi64(__A);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_broadcastq_epi64((__m128i)(__v2di){-42, 0}), -42, -42, -42, -42, -42, -42, -42, -42));
 
 __m512i test_mm512_mask_broadcastq_epi64(__m512i __O, __mmask8 __M, __m128i __A) {
   // CHECK-LABEL: test_mm512_mask_broadcastq_epi64
@@ -8682,6 +8759,7 @@ __m512 test_mm512_movehdup_ps(__m512 __A) {
   // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
   return _mm512_movehdup_ps(__A);
 }
+TEST_CONSTEXPR(match_m512(_mm512_movehdup_ps((__m512){+1.0f,-1.0f,+2.0f,-2.0f,+3.0f,-3.0f,+4.0f,-4.0f,+5.0f,-5.0f,+6.0f,-6.0f,+7.0f,-7.0f,+8.0f,-8.0f}), -1.0f, -1.0f, -2.0f, -2.0f, -3.0f, -3.0f, -4.0f, -4.0f, -5.0f, -5.0f, -6.0f, -6.0f, -7.0f, -7.0f, -8.0f, -8.0f));
 
 __m512 test_mm512_mask_movehdup_ps(__m512 __W, __mmask16 __U, __m512 __A) {
   // CHECK-LABEL: test_mm512_mask_movehdup_ps
@@ -8702,6 +8780,7 @@ __m512 test_mm512_moveldup_ps(__m512 __A) {
   // CHECK: shufflevector <16 x float> %{{.*}}, <16 x float> %{{.*}}, <16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>
   return _mm512_moveldup_ps(__A);
 }
+TEST_CONSTEXPR(match_m512(_mm512_moveldup_ps((__m512){+1.0f,-1.0f,+2.0f,-2.0f,+3.0f,-3.0f,+4.0f,-4.0f,+5.0f,-5.0f,+6.0f,-6.0f,+7.0f,-7.0f,+8.0f,-8.0f}), +1.0f, +1.0f, +2.0f, +2.0f, +3.0f, +3.0f, +4.0f, +4.0f, +5.0f, +5.0f, +6.0f, +6.0f, +7.0f, +7.0f, +8.0f, +8.0f));
 
 __m512 test_mm512_mask_moveldup_ps(__m512 __W, __mmask16 __U, __m512 __A) {
   // CHECK-LABEL: test_mm512_mask_moveldup_ps
@@ -8883,29 +8962,34 @@ __m512d test_mm512_maskz_cvtps_pd(__mmask8 __U, __m256 __A) {
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_maskz_cvtps_pd(__U, __A); 
 }
+
 __m512d test_mm512_mask_mov_pd(__m512d __W, __mmask8 __U, __m512d __A) {
   // CHECK-LABEL: test_mm512_mask_mov_pd
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_mask_mov_pd(__W, __U, __A); 
 }
+TEST_CONSTEXPR(match_m512d(_mm512_mask_mov_pd((__m512d){-8.0, -7.0, -6.0, -5.0, -4.0, -3.0, -2.0, -1.0}, 0xC3, (__m512d){+1.0, +2.0, +3.0, +4.0, +5.0, +6.0, +7.0, +8.0}), +1.0, +2.0, -6.0, -5.0, -4.0, -3.0, +7.0, +8.0));
 
 __m512d test_mm512_maskz_mov_pd(__mmask8 __U, __m512d __A) {
   // CHECK-LABEL: test_mm512_maskz_mov_pd
   // CHECK: select <8 x i1> %{{.*}}, <8 x double> %{{.*}}, <8 x double> %{{.*}}
   return _mm512_maskz_mov_pd(__U, __A); 
 }
+TEST_CONSTEXPR(match_m512d(_mm512_maskz_mov_pd(0xC3, (__m512d){+1.0, +2.0, +3.0, +4.0, +5.0, +6.0, +7.0, +8.0}), +1.0, +2.0, +0.0, +0.0, +0.0, +0.0, +7.0, +8.0));
 
 __m512 test_mm512_mask_mov_ps(__m512 __W, __mmask16 __U, __m512 __A) {
   // CHECK-LABEL: test_mm512_mask_mov_ps
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask_mov_ps(__W, __U, __A); 
 }
+TEST_CONSTEXPR(match_m512(_mm512_mask_mov_ps((__m512){-16.0f, -15.0f, -14.0f, -13.0f, -12.0f, -11.0f, -10.0f, -9.0f, -8.0f, -7.0f, -6.0f, -5.0f, -4.0f, -3.0f, -2.0f, -1.0f}, 0x0FF0, (__m512){+1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f, +9.0f, +10.0f, +11.0f, +12.0f, +13.0f, +14.0f, +15.0f, +16.0f}), -16.0f, -15.0f, -14.0f, -13.0f, 5.0f, 6.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f, -4.0f, -3.0f, -2.0f, -1.0f));
 
 __m512 test_mm512_maskz_mov_ps(__mmask16 __U, __m512 __A) {
   // CHECK-LABEL: test_mm512_maskz_mov_ps
   // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_maskz_mov_ps(__U, __A); 
 }
+TEST_CONSTEXPR(match_m512(_mm512_maskz_mov_ps(0xF3F3, (__m512){+1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f, +9.0f, +10.0f, +11.0f, +12.0f, +13.0f, +14.0f, +15.0f, +16.0f}), +1.0f, +2.0f, 0.0f, 0.0f, +5.0f, +6.0f, +7.0f, +8.0f, +9.0f, +10.0f, 0.0f, 0.0f, +13.0f, +14.0f, +15.0f, +16.0f));
 
 void test_mm512_mask_compressstoreu_pd(void *__P, __mmask8 __U, __m512d __A) {
   // CHECK-LABEL: test_mm512_mask_compressstoreu_pd
@@ -9024,6 +9108,7 @@ __m512i test_mm512_set1_epi8(char d)
   // CHECK: insertelement <64 x i8> {{.*}}, i32 63
   return _mm512_set1_epi8(d);
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_set1_epi8(127), 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127, 127));
 
 __m512i test_mm512_set1_epi16(short d)
 {
@@ -9039,6 +9124,37 @@ __m512i test_mm512_set1_epi16(short d)
   // CHECK: insertelement <32 x i16> {{.*}}, i32 31
   return _mm512_set1_epi16(d);
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_set1_epi16(-511), -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511, -511));
+
+__m512i test_mm512_set1_epi32(int d)
+{
+  // CHECK-LABEL: test_mm512_set1_epi32
+  // CHECK: insertelement <16 x i32> {{.*}}, i32 0
+  // CHECK: insertelement <16 x i32> {{.*}}, i32 1
+  // CHECK: insertelement <16 x i32> {{.*}}, i32 2
+  // CHECK: insertelement <16 x i32> {{.*}}, i32 3
+  // CHECK: insertelement <16 x i32> {{.*}}, i32 4
+  // CHECK: insertelement <16 x i32> {{.*}}, i32 5
+  // CHECK: insertelement <16 x i32> {{.*}}, i32 6
+  // CHECK: insertelement <16 x i32> {{.*}}, i32 15
+  return _mm512_set1_epi32(d);
+}
+TEST_CONSTEXPR(match_v16si(_mm512_set1_epi32(99), 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99));
+
+__m512i test_mm512_set1_epi64(long long d)
+{
+  // CHECK-LABEL: test_mm512_set1_epi64
+  // CHECK: insertelement <8 x i64> {{.*}}, i32 0
+  // CHECK: insertelement <8 x i64> {{.*}}, i32 1
+  // CHECK: insertelement <8 x i64> {{.*}}, i32 2
+  // CHECK: insertelement <8 x i64> {{.*}}, i32 3
+  // CHECK: insertelement <8 x i64> {{.*}}, i32 4
+  // CHECK: insertelement <8 x i64> {{.*}}, i32 5
+  // CHECK: insertelement <8 x i64> {{.*}}, i32 6
+  // CHECK: insertelement <8 x i64> {{.*}}, i32 7
+  return _mm512_set1_epi64(d);
+}
+TEST_CONSTEXPR(match_v8di(_mm512_set1_epi64(-42), -42, -42, -42, -42, -42, -42, -42, -42));
 
 __m512i test_mm512_set4_epi32 (int __A, int __B, int __C, int __D)
 {
@@ -9046,6 +9162,7 @@ __m512i test_mm512_set4_epi32 (int __A, int __B, int __C, int __D)
   // CHECK: insertelement <16 x i32> {{.*}}, i32 15
   return _mm512_set4_epi32 (__A,__B,__C,__D);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_set4_epi32(10, 20, 30, 40), 40, 30, 20, 10, 40, 30, 20, 10, 40, 30, 20, 10, 40, 30, 20, 10));
 
 __m512i test_mm512_set4_epi64 (long long __A, long long __B, long long __C, long long __D)
 {
@@ -9053,6 +9170,7 @@ __m512i test_mm512_set4_epi64 (long long __A, long long __B, long long __C, long
   // CHECK: insertelement <8 x i64> {{.*}}, i32 7
   return _mm512_set4_epi64 (__A,__B,__C,__D);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_set4_epi64(1, -3, 5, -7), -7, 5, -3, 1, -7, 5, -3, 1));
 
 __m512d test_mm512_set4_pd (double __A, double __B, double __C, double __D)
 {
@@ -9060,6 +9178,7 @@ __m512d test_mm512_set4_pd (double __A, double __B, double __C, double __D)
   // CHECK: insertelement <8 x double> {{.*}}, i32 7
   return _mm512_set4_pd (__A,__B,__C,__D);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_set4_pd(10.0, 20.0, 30.0, 40.0), 40.0, 30.0, 20.0, 10.0, 40.0, 30.0, 20.0, 10.0));
 
 __m512 test_mm512_set4_ps (float __A, float __B, float __C, float __D)
 {
@@ -9067,6 +9186,7 @@ __m512 test_mm512_set4_ps (float __A, float __B, float __C, float __D)
   // CHECK: insertelement <16 x float> {{.*}}, i32 15
   return _mm512_set4_ps (__A,__B,__C,__D);
 }
+TEST_CONSTEXPR(match_m512(_mm512_set4_ps(1.1f, 2.2f, 3.3f, 4.4f), 4.4f, 3.3f, 2.2f, 1.1f, 4.4f, 3.3f, 2.2f, 1.1f, 4.4f, 3.3f, 2.2f, 1.1f, 4.4f, 3.3f, 2.2f, 1.1f));
 
 __m512i test_mm512_setr4_epi32(int e0, int e1, int e2, int e3)
 {
@@ -10206,6 +10326,11 @@ __m512i test_mm512_set_epi8(char e63, char e62, char e61, char e60, char e59,
       e25, e24, e23, e22, e21, e20, e19, e18, e17, e16, e15, e14, e13, e12,
       e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0);
 }
+TEST_CONSTEXPR(match_v64qi(_mm512_set_epi8(63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48,
+                                          47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 
+                                          31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16,
+                                          15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
+                                          32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63));
 
 __m512i test_mm512_set_epi16(short e31, short e30, short e29, short e28,
     short e27, short e26, short e25, short e24, short e23, short e22,
@@ -10249,8 +10374,9 @@ __m512i test_mm512_set_epi16(short e31, short e30, short e29, short e28,
   return _mm512_set_epi16(e31, e30, e29, e28, e27, e26, e25, e24, e23, e22,
       e21, e20, e19, e18, e17, e16, e15, e14, e13, e12, e11, e10, e9, e8, e7,
       e6, e5, e4, e3, e2, e1, e0);
-
 }
+TEST_CONSTEXPR(match_v32hi(_mm512_set_epi16(-31, 30, -29, 28, -27, 26, -25, 24, -23, 22, -21, 20, -19, 18, -17, 16, -15, 14, -13, 12, -11, 10, -9, 8, -7, 6, -5, 4, -3, 2, -1, 0), 0, -1, 2, -3, 4, -5, 6, -7, 8, -9, 10, -11, 12, -13, 14, -15, 16, -17, 18, -19, 20, -21, 22, -23, 24, -25, 26, -27, 28, -29, 30, -31));
+
 __m512i test_mm512_set_epi32 (int __A, int __B, int __C, int __D,
                int __E, int __F, int __G, int __H,
                int __I, int __J, int __K, int __L,
@@ -10276,6 +10402,7 @@ __m512i test_mm512_set_epi32 (int __A, int __B, int __C, int __D,
  return _mm512_set_epi32( __A, __B, __C, __D,__E, __F, __G, __H,
               __I, __J, __K, __L,__M, __N, __O, __P);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_set_epi32(-15, -14, -13, -12, -11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0), 0, -1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12, -13, -14, -15));
 
 __m512i test_mm512_setr_epi32 (int __A, int __B, int __C, int __D,
                int __E, int __F, int __G, int __H,
@@ -10318,6 +10445,7 @@ __m512i test_mm512_setr_epi32 (int __A, int __B, int __C, int __D,
  return _mm512_setr_epi32( __A, __B, __C, __D,__E, __F, __G, __H,
               __I, __J, __K, __L,__M, __N, __O, __P);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_setr_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15));
 
 __m512i test_mm512_mask_set1_epi64 (__m512i __O, __mmask8 __M, long long __A)
 {
@@ -10365,6 +10493,7 @@ __m512i test_mm512_set_epi64 (long long __A, long long __B, long long __C,
     //CHECK: insertelement{{.*}}i32 7
   return _mm512_set_epi64(__A, __B, __C, __D, __E, __F, __G, __H );
 }
+TEST_CONSTEXPR(match_v8di(_mm512_set_epi64(-15, 13, -11, 9, -7, 5, -3, 1),  1, -3, 5, -7, 9, -11, 13, -15));
 
 __m512i test_mm512_setr_epi64 (long long __A, long long __B, long long __C,
                               long long __D, long long __E, long long __F,
@@ -10389,6 +10518,7 @@ __m512i test_mm512_setr_epi64 (long long __A, long long __B, long long __C,
     //CHECK: insertelement{{.*}}i32 7
   return _mm512_setr_epi64(__A, __B, __C, __D, __E, __F, __G, __H );
 }
+TEST_CONSTEXPR(match_v8di(_mm512_setr_epi64(-1, 3, -5, 7, -9, 11, -13, 15), -1, 3, -5, 7, -9, 11, -13, 15));
 
 __m512d test_mm512_set_pd (double __A, double __B, double __C, double __D,
                            double __E, double __F, double __G, double __H)
@@ -10404,6 +10534,7 @@ __m512d test_mm512_set_pd (double __A, double __B, double __C, double __D,
     //CHECK: insertelement{{.*}}i32 7
   return _mm512_set_pd( __A, __B, __C, __D, __E, __F, __G, __H);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_set_pd(20.0, 40.0, 60.0, 80.0, 100.0, 120.0, 140.0, 160.0), 160.0, 140.0, 120.0, 100.0, 80.0, 60.0, 40.0, 20.0));
 
 __m512d test_mm512_setr_pd (double __A, double __B, double __C, double __D,
                            double __E, double __F, double __G, double __H)
@@ -10427,6 +10558,7 @@ __m512d test_mm512_setr_pd (double __A, double __B, double __C, double __D,
     //CHECK: insertelement{{.*}}i32 7
   return _mm512_setr_pd( __A, __B, __C, __D, __E, __F, __G, __H);
 }
+TEST_CONSTEXPR(match_m512d(_mm512_setr_pd(-20.0, 40.0, -60.0, 80.0, -100.0, 120.0, -140.0, 160.0), -20.0, 40.0, -60.0, 80.0, -100.0, 120.0, -140.0, 160.0));
 
 __m512 test_mm512_set_ps (float __A, float __B, float __C, float __D,
                           float __E, float __F, float __G, float __H,
@@ -10453,6 +10585,7 @@ __m512 test_mm512_set_ps (float __A, float __B, float __C, float __D,
     return _mm512_set_ps( __A, __B, __C, __D, __E, __F, __G, __H,
                           __I, __J, __K, __L, __M, __N, __O, __P);
 }
+TEST_CONSTEXPR(match_m512(_mm512_set_ps(-16.0f, 15.0f, -14.0f, 13.0f, -12.0f, 11.0f, -10.0f, 9.0f, -8.0f, 7.0f, -6.0f, 5.0f, -4.0f, 3.0f, -2.0f, 1.0f), 1.0f, -2.0f, 3.0f, -4.0f, 5.0f, -6.0f, 7.0f, -8.0f, 9.0f, -10.0f, 11.0f, -12.0f, 13.0f, -14.0f, 15.0f, -16.0f));
 
 __m512i test_mm512_mask_abs_epi64 (__m512i __W, __mmask8 __U, __m512i __A)
 {
@@ -10531,6 +10664,7 @@ __m512 test_mm512_setr_ps (float __A, float __B, float __C, float __D,
     return _mm512_setr_ps( __A, __B, __C, __D, __E, __F, __G, __H,
                           __I, __J, __K, __L, __M, __N, __O, __P);
 }
+TEST_CONSTEXPR(match_m512(_mm512_setr_ps(-1.0f, 2.0f, -3.0f, 4.0f, -5.0f, 6.0f, -7.0f, 8.0f, -9.0f, 10.0f, -11.0f, 12.0f, -13.0f, 14.0f, -15.0f, 16.0f), -1.0f, 2.0f, -3.0f, 4.0f, -5.0f, 6.0f, -7.0f, 8.0f, -9.0f, 10.0f, -11.0f, 12.0f, -13.0f, 14.0f, -15.0f, 16.0f));
 
 int test_mm_cvtss_i32(__m128 A) {
   // CHECK-LABEL: test_mm_cvtss_i32
diff --git a/clang/test/CodeGen/X86/avx512fp16-builtins.c b/clang/test/CodeGen/X86/avx512fp16-builtins.c
index d277d053147fd..3fcddb02d16c7 100644
--- a/clang/test/CodeGen/X86/avx512fp16-builtins.c
+++ b/clang/test/CodeGen/X86/avx512fp16-builtins.c
@@ -1,51 +1,58 @@
-// RUN: %clang_cc1 -ffreestanding -flax-vector-conversions=none %s -triple=x86_64-unknown-unknown -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding -flax-vector-conversions=none %s -triple=x86_64-unknown-unknown -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c -ffreestanding -flax-vector-conversions=none %s -triple=i686-unknown-unknown -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
+// RUN: %clang_cc1 -x c++ -ffreestanding -flax-vector-conversions=none %s -triple=x86_64-unknown-unknown -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK,X64
+// RUN: %clang_cc1 -x c++ -ffreestanding -flax-vector-conversions=none %s -triple=i686-unknown-unknown -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefixes=CHECK
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 _Float16 test_mm512_cvtsh_h(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_cvtsh_h
+  // CHECK-LABEL: test_mm512_cvtsh_h
   // CHECK: extractelement <32 x half> %{{.*}}, i32 0
   return _mm512_cvtsh_h(__A);
 }
 
 __m128h test_mm_setzero_ph(void) {
-  // CHECK-LABEL: @test_mm_setzero_ph
+  // CHECK-LABEL: test_mm_setzero_ph
   // CHECK: zeroinitializer
   return _mm_setzero_ph();
 }
+TEST_CONSTEXPR(match_m128h(_mm_setzero_ph(), +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f));
 
 __m256h test_mm256_setzero_ph(void) {
-  // CHECK-LABEL: @test_mm256_setzero_ph
+  // CHECK-LABEL: test_mm256_setzero_ph
   // CHECK: zeroinitializer
   return _mm256_setzero_ph();
 }
+TEST_CONSTEXPR(match_m256h(_mm256_setzero_ph(), +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f));
 
 __m256h test_mm256_undefined_ph(void) {
-  // CHECK-LABEL: @test_mm256_undefined_ph
+  // CHECK-LABEL: test_mm256_undefined_ph
   // CHECK: ret <16 x half> zeroinitializer
   return _mm256_undefined_ph();
 }
 
 __m512h test_mm512_setzero_ph(void) {
-  // CHECK-LABEL: @test_mm512_setzero_ph
+  // CHECK-LABEL: test_mm512_setzero_ph
   // CHECK: zeroinitializer
   return _mm512_setzero_ph();
 }
+TEST_CONSTEXPR(match_m512h(_mm512_setzero_ph(), +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f, +0.0f));
 
 __m128h test_mm_undefined_ph(void) {
-  // CHECK-LABEL: @test_mm_undefined_ph
+  // CHECK-LABEL: test_mm_undefined_ph
   // CHECK: ret <8 x half> zeroinitializer
   return _mm_undefined_ph();
 }
 
 __m512h test_mm512_undefined_ph(void) {
-  // CHECK-LABEL: @test_mm512_undefined_ph
+  // CHECK-LABEL: test_mm512_undefined_ph
   // CHECK: ret <32 x half> zeroinitializer
   return _mm512_undefined_ph();
 }
 
 __m512h test_mm512_set1_ph(_Float16 h) {
-  // CHECK-LABEL: @test_mm512_set1_ph
+  // CHECK-LABEL: test_mm512_set1_ph
   // CHECK: insertelement <32 x half> {{.*}}, i32 0
   // CHECK: insertelement <32 x half> {{.*}}, i32 1
   // CHECK: insertelement <32 x half> {{.*}}, i32 2
@@ -80,9 +87,10 @@ __m512h test_mm512_set1_ph(_Float16 h) {
   // CHECK: insertelement <32 x half> {{.*}}, i32 31
   return _mm512_set1_ph(h);
 }
+TEST_CONSTEXPR(match_m512h(_mm512_set1_ph(-101.0f), -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f, -101.0f));
 
 __m512h test_mm512_set1_pch(_Float16 _Complex h) {
-  // CHECK-LABEL: @test_mm512_set1_pch
+  // CHECK-LABEL: test_mm512_set1_pch
   // CHECK: insertelement <16 x float> {{.*}}, i32 0
   // CHECK: insertelement <16 x float> {{.*}}, i32 1
   // CHECK: insertelement <16 x float> {{.*}}, i32 2
@@ -111,7 +119,7 @@ __m512h test_mm512_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16
                           _Float16 __h21, _Float16 __h22, _Float16 __h23, _Float16 __h24,
                           _Float16 __h25, _Float16 __h26, _Float16 __h27, _Float16 __h28,
                           _Float16 __h29, _Float16 __h30, _Float16 __h31, _Float16 __h32) {
-  // CHECK-LABEL: @test_mm512_set_ph
+  // CHECK-LABEL: test_mm512_set_ph
   // CHECK: insertelement <32 x half> {{.*}}, i32 0
   // CHECK: insertelement <32 x half> {{.*}}, i32 1
   // CHECK: insertelement <32 x half> {{.*}}, i32 2
@@ -149,6 +157,13 @@ __m512h test_mm512_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16
                        __h17, __h18, __h19, __h20, __h21, __h22, __h23, __h24,
                        __h25, __h26, __h27, __h28, __h29, __h30, __h31, __h32);
 }
+TEST_CONSTEXPR(match_m512h(_mm512_set_ph(32.0f, -31.0f, 30.0f, -29.0f, 28.0f, -27.0f, 26.0f, -25.0f,
+                                        24.0f, -23.0f, 22.0f, -21.0f, 20.0f, -19.0f, 18.0f, -17.0f,
+                                        16.0f, -15.0f, 14.0f, -13.0f, 12.0f, -11.0f, 10.0f, -9.0f,
+                                        8.0f, -7.0f, 6.0f, -5.0f, 4.0f, -3.0f, 2.0f, -1.0f), -1.0f, 2.0f, -3.0f, 4.0f, -5.0f, 6.0f, -7.0f, 8.0f,
+                                        -9.0f, 10.0f, -11.0f, 12.0f, -13.0f, 14.0f, -15.0f, 16.0f,
+                                        -17.0f, 18.0f, -19.0f, 20.0f, -21.0f, 22.0f, -23.0f, 24.0f,
+                                        -25.0f, 26.0f, -27.0f, 28.0f, -29.0f, 30.0f, -31.0f, 32.0f));
 
 __m512h test_mm512_setr_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16 __h4,
                            _Float16 __h5, _Float16 __h6, _Float16 __h7, _Float16 __h8,
@@ -158,7 +173,7 @@ __m512h test_mm512_setr_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16
                            _Float16 __h21, _Float16 __h22, _Float16 __h23, _Float16 __h24,
                            _Float16 __h25, _Float16 __h26, _Float16 __h27, _Float16 __h28,
                            _Float16 __h29, _Float16 __h30, _Float16 __h31, _Float16 __h32) {
-  // CHECK-LABEL: @test_mm512_setr_ph
+  // CHECK-LABEL: test_mm512_setr_ph
   // CHECK: insertelement <32 x half> {{.*}}, i32 0
   // CHECK: insertelement <32 x half> {{.*}}, i32 1
   // CHECK: insertelement <32 x half> {{.*}}, i32 2
@@ -196,6 +211,14 @@ __m512h test_mm512_setr_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16
                         __h17, __h18, __h19, __h20, __h21, __h22, __h23, __h24,
                         __h25, __h26, __h27, __h28, __h29, __h30, __h31, __h32);
 }
+TEST_CONSTEXPR(match_m512h(_mm512_setr_ph(1.0f, -3.0f, 5.0f, -7.0f, 9.0f, -11.0f, 13.0f, -15.0f,
+                                         17.0f, -19.0f, 21.0f, -23.0f, 25.0f, -27.0f, 29.0f, -31.0f,
+                                         33.0f, -35.0f, 37.0f, -39.0f, 41.0f, -43.0f, 45.0f, -47.0f,
+                                         49.0f, -51.0f, 53.0f, -55.0f, 57.0f, -59.0f, 61.0f, -63.0f),
+                          1.0f, -3.0f, 5.0f, -7.0f, 9.0f, -11.0f, 13.0f, -15.0f,
+                          17.0f, -19.0f, 21.0f, -23.0f, 25.0f, -27.0f, 29.0f, -31.0f,
+                          33.0f, -35.0f, 37.0f, -39.0f, 41.0f, -43.0f, 45.0f, -47.0f,
+                          49.0f, -51.0f, 53.0f, -55.0f, 57.0f, -59.0f, 61.0f, -63.0f));
 
 __m128 test_mm_castph_ps(__m128h A) {
   // CHECK-LABEL: test_mm_castph_ps
@@ -368,327 +391,327 @@ __m512h test_mm512_zextph256_ph512(__m256h __a) {
 
 int test_mm_comi_round_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_comi_round_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 0, i32 8)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 0, i32 8)
   return _mm_comi_round_sh(__A, __B, 0, _MM_FROUND_NO_EXC);
 }
 
 int test_mm_comi_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_comi_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 0, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 0, i32 4)
   return _mm_comi_sh(__A, __B, 0);
 }
 
 int test_mm_comieq_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_comieq_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 16, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 16, i32 4)
   return _mm_comieq_sh(__A, __B);
 }
 
 int test_mm_comilt_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_comilt_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 1, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 1, i32 4)
   return _mm_comilt_sh(__A, __B);
 }
 
 int test_mm_comile_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_comile_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 2, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 2, i32 4)
   return _mm_comile_sh(__A, __B);
 }
 
 int test_mm_comigt_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_comigt_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 14, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 14, i32 4)
   return _mm_comigt_sh(__A, __B);
 }
 
 int test_mm_comige_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_comige_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 13, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 13, i32 4)
   return _mm_comige_sh(__A, __B);
 }
 
 int test_mm_comineq_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_comineq_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 20, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 20, i32 4)
   return _mm_comineq_sh(__A, __B);
 }
 
 int test_mm_ucomieq_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_ucomieq_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 0, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 0, i32 4)
   return _mm_ucomieq_sh(__A, __B);
 }
 
 int test_mm_ucomilt_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_ucomilt_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 17, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 17, i32 4)
   return _mm_ucomilt_sh(__A, __B);
 }
 
 int test_mm_ucomile_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_ucomile_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 18, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 18, i32 4)
   return _mm_ucomile_sh(__A, __B);
 }
 
 int test_mm_ucomigt_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_ucomigt_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 30, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 30, i32 4)
   return _mm_ucomigt_sh(__A, __B);
 }
 
 int test_mm_ucomige_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_ucomige_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 29, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 29, i32 4)
   return _mm_ucomige_sh(__A, __B);
 }
 
 int test_mm_ucomineq_sh(__m128h __A, __m128h __B) {
   // CHECK-LABEL: test_mm_ucomineq_sh
-  // CHECK: %{{.}} = call i32 @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 4, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vcomi.sh(<8 x half> %{{.}}, <8 x half> %{{.}}, i32 4, i32 4)
   return _mm_ucomineq_sh(__A, __B);
 }
 
 __m512h test_mm512_add_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_add_ph
+  // CHECK-LABEL: test_mm512_add_ph
   // CHECK: %{{.*}} = fadd <32 x half> %{{.*}}, %{{.*}}
   return _mm512_add_ph(__A, __B);
 }
 
 __m512h test_mm512_mask_add_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_add_ph
+  // CHECK-LABEL: test_mm512_mask_add_ph
   // CHECK: %{{.*}} = fadd <32 x half> %{{.*}}, %{{.*}}
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return (__m512h)_mm512_mask_add_ph(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_add_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_add_ph
+  // CHECK-LABEL: test_mm512_maskz_add_ph
   // CHECK: %{{.*}} = fadd <32 x half> %{{.*}}, %{{.*}}
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_add_ph(__U, __A, __B);
 }
 
 __m512h test_mm512_add_round_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_add_round_ph
+  // CHECK-LABEL: test_mm512_add_round_ph
   // CHECK: @llvm.x86.avx512fp16.add.ph.512
   return _mm512_add_round_ph(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_mask_add_round_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_add_round_ph
+  // CHECK-LABEL: test_mm512_mask_add_round_ph
   // CHECK: @llvm.x86.avx512fp16.add.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_add_round_ph(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_maskz_add_round_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_add_round_ph
+  // CHECK-LABEL: test_mm512_maskz_add_round_ph
   // CHECK: @llvm.x86.avx512fp16.add.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_add_round_ph(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_sub_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_sub_ph
+  // CHECK-LABEL: test_mm512_sub_ph
   // CHECK: %{{.*}} = fsub <32 x half> %{{.*}}, %{{.*}}
   return _mm512_sub_ph(__A, __B);
 }
 
 __m512h test_mm512_mask_sub_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_sub_ph
+  // CHECK-LABEL: test_mm512_mask_sub_ph
   // CHECK: %{{.*}} = fsub <32 x half> %{{.*}}, %{{.*}}
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return (__m512h)_mm512_mask_sub_ph(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_sub_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_sub_ph
+  // CHECK-LABEL: test_mm512_maskz_sub_ph
   // CHECK: %{{.*}} = fsub <32 x half> %{{.*}}, %{{.*}}
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_sub_ph(__U, __A, __B);
 }
 
 __m512h test_mm512_sub_round_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_sub_round_ph
+  // CHECK-LABEL: test_mm512_sub_round_ph
   // CHECK: @llvm.x86.avx512fp16.sub.ph.512
   return _mm512_sub_round_ph(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_mask_sub_round_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_sub_round_ph
+  // CHECK-LABEL: test_mm512_mask_sub_round_ph
   // CHECK: @llvm.x86.avx512fp16.sub.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_sub_round_ph(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_maskz_sub_round_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_sub_round_ph
+  // CHECK-LABEL: test_mm512_maskz_sub_round_ph
   // CHECK: @llvm.x86.avx512fp16.sub.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_sub_round_ph(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mul_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mul_ph
+  // CHECK-LABEL: test_mm512_mul_ph
   // CHECK: %{{.*}} = fmul <32 x half> %{{.*}}, %{{.*}}
   return _mm512_mul_ph(__A, __B);
 }
 
 __m512h test_mm512_mask_mul_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_mul_ph
+  // CHECK-LABEL: test_mm512_mask_mul_ph
   // CHECK: %{{.*}} = fmul <32 x half> %{{.*}}, %{{.*}}
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return (__m512h)_mm512_mask_mul_ph(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_mul_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_mul_ph
+  // CHECK-LABEL: test_mm512_maskz_mul_ph
   // CHECK: %{{.*}} = fmul <32 x half> %{{.*}}, %{{.*}}
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_mul_ph(__U, __A, __B);
 }
 
 __m512h test_mm512_mul_round_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mul_round_ph
+  // CHECK-LABEL: test_mm512_mul_round_ph
   // CHECK: @llvm.x86.avx512fp16.mul.ph.512
   return _mm512_mul_round_ph(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_mask_mul_round_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_mul_round_ph
+  // CHECK-LABEL: test_mm512_mask_mul_round_ph
   // CHECK: @llvm.x86.avx512fp16.mul.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_mul_round_ph(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_maskz_mul_round_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_mul_round_ph
+  // CHECK-LABEL: test_mm512_maskz_mul_round_ph
   // CHECK: @llvm.x86.avx512fp16.mul.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_mul_round_ph(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_div_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_div_ph
+  // CHECK-LABEL: test_mm512_div_ph
   // CHECK: %{{.*}} = fdiv <32 x half> %{{.*}}, %{{.*}}
   return _mm512_div_ph(__A, __B);
 }
 
 __m512h test_mm512_mask_div_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_div_ph
+  // CHECK-LABEL: test_mm512_mask_div_ph
   // CHECK: %{{.*}} = fdiv <32 x half> %{{.*}}, %{{.*}}
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return (__m512h)_mm512_mask_div_ph(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_div_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_div_ph
+  // CHECK-LABEL: test_mm512_maskz_div_ph
   // CHECK: %{{.*}} = fdiv <32 x half> %{{.*}}, %{{.*}}
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_div_ph(__U, __A, __B);
 }
 
 __m512h test_mm512_div_round_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_div_round_ph
+  // CHECK-LABEL: test_mm512_div_round_ph
   // CHECK: @llvm.x86.avx512fp16.div.ph.512
   return _mm512_div_round_ph(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_mask_div_round_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_div_round_ph
+  // CHECK-LABEL: test_mm512_mask_div_round_ph
   // CHECK: @llvm.x86.avx512fp16.div.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_div_round_ph(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_maskz_div_round_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_div_round_ph
+  // CHECK-LABEL: test_mm512_maskz_div_round_ph
   // CHECK: @llvm.x86.avx512fp16.div.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_div_round_ph(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_min_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_min_ph
+  // CHECK-LABEL: test_mm512_min_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.512
   return _mm512_min_ph(__A, __B);
 }
 
 __m512h test_mm512_mask_min_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_min_ph
+  // CHECK-LABEL: test_mm512_mask_min_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return (__m512h)_mm512_mask_min_ph(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_min_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_min_ph
+  // CHECK-LABEL: test_mm512_maskz_min_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_min_ph(__U, __A, __B);
 }
 
 __m512h test_mm512_min_round_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_min_round_ph
+  // CHECK-LABEL: test_mm512_min_round_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.512
   return _mm512_min_round_ph(__A, __B, _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_mask_min_round_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_min_round_ph
+  // CHECK-LABEL: test_mm512_mask_min_round_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_min_round_ph(__W, __U, __A, __B, _MM_FROUND_NO_EXC);
 }
 __m512h test_mm512_maskz_min_round_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_min_round_ph
+  // CHECK-LABEL: test_mm512_maskz_min_round_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_min_round_ph(__U, __A, __B, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_max_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_max_ph
+  // CHECK-LABEL: test_mm512_max_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.512
 
   return _mm512_max_ph(__A, __B);
 }
 
 __m512h test_mm512_mask_max_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_max_ph
+  // CHECK-LABEL: test_mm512_mask_max_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return (__m512h)_mm512_mask_max_ph(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_max_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_max_ph
+  // CHECK-LABEL: test_mm512_maskz_max_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_max_ph(__U, __A, __B);
 }
 
 __m512h test_mm512_max_round_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_max_round_ph
+  // CHECK-LABEL: test_mm512_max_round_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.512
   return _mm512_max_round_ph(__A, __B, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_max_round_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_max_round_ph
+  // CHECK-LABEL: test_mm512_mask_max_round_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_max_round_ph(__W, __U, __A, __B, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_max_round_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_max_round_ph
+  // CHECK-LABEL: test_mm512_maskz_max_round_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.512
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_maskz_max_round_ph(__U, __A, __B, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_abs_ph(__m512h a) {
-  // CHECK-LABEL: @test_mm512_abs_ph
+  // CHECK-LABEL: test_mm512_abs_ph
   // CHECK: and <16 x i32>
   return _mm512_abs_ph(a);
 }
 
 __m512h test_mm512_conj_pch(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_conj_pch
+  // CHECK-LABEL: test_mm512_conj_pch
   // CHECK:  %{{.*}} = bitcast <32 x half> %{{.*}} to <8 x i64>
   // CHECK:  %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
   // CHECK:  %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
@@ -699,7 +722,7 @@ __m512h test_mm512_conj_pch(__m512h __A) {
 }
 
 __m512h test_mm512_mask_conj_pch(__m512h __W, __mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_conj_pch
+  // CHECK-LABEL: test_mm512_mask_conj_pch
   // CHECK:  %{{.*}} = trunc i32 %{{.*}} to i16
   // CHECK:  %{{.*}} = bitcast <32 x half> %{{.*}} to <8 x i64>
   // CHECK:  %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
@@ -715,7 +738,7 @@ __m512h test_mm512_mask_conj_pch(__m512h __W, __mmask32 __U, __m512h __A) {
 }
 
 __m512h test_mm512_maskz_conj_pch(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_conj_pch
+  // CHECK-LABEL: test_mm512_maskz_conj_pch
   // CHECK:  %{{.*}} = trunc i32 %{{.*}} to i16
   // CHECK:  %{{.*}} = bitcast <32 x half> %{{.*}} to <8 x i64>
   // CHECK:  %{{.*}} = bitcast <8 x i64> %{{.*}} to <16 x i32>
@@ -730,22 +753,22 @@ __m512h test_mm512_maskz_conj_pch(__mmask32 __U, __m512h __A) {
 }
 
 __m128h test_mm_add_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_add_round_sh
+  // CHECK-LABEL: test_mm_add_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.add.sh.round
   return _mm_add_round_sh(__A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_mask_add_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_add_round_sh
+  // CHECK-LABEL: test_mm_mask_add_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.add.sh.round
   return _mm_mask_add_round_sh(__W, __U, __A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_maskz_add_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_add_round_sh
+  // CHECK-LABEL: test_mm_maskz_add_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.add.sh.round
   return _mm_maskz_add_round_sh(__U, __A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_mask_add_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_add_sh
+  // CHECK-LABEL: test_mm_mask_add_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fadd half %{{.*}}, %{{.*}}
@@ -759,7 +782,7 @@ __m128h test_mm_mask_add_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B)
   return _mm_mask_add_sh(__W, __U, __A, __B);
 }
 __m128h test_mm_maskz_add_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_add_sh
+  // CHECK-LABEL: test_mm_maskz_add_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fadd half %{{.*}}, %{{.*}}
@@ -774,7 +797,7 @@ __m128h test_mm_maskz_add_sh(__mmask8 __U, __m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_add_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_add_sh
+  // CHECK-LABEL: test_mm_add_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fadd half %{{.*}}, %{{.*}}
@@ -783,22 +806,22 @@ __m128h test_mm_add_sh(__m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_sub_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_sub_round_sh
+  // CHECK-LABEL: test_mm_sub_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.sub.sh.round
   return _mm_sub_round_sh(__A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_mask_sub_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_sub_round_sh
+  // CHECK-LABEL: test_mm_mask_sub_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.sub.sh.round
   return _mm_mask_sub_round_sh(__W, __U, __A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_maskz_sub_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_sub_round_sh
+  // CHECK-LABEL: test_mm_maskz_sub_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.sub.sh.round
   return _mm_maskz_sub_round_sh(__U, __A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_mask_sub_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_sub_sh
+  // CHECK-LABEL: test_mm_mask_sub_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fsub half %{{.*}}, %{{.*}}
@@ -812,7 +835,7 @@ __m128h test_mm_mask_sub_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B)
   return _mm_mask_sub_sh(__W, __U, __A, __B);
 }
 __m128h test_mm_maskz_sub_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_sub_sh
+  // CHECK-LABEL: test_mm_maskz_sub_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fsub half %{{.*}}, %{{.*}}
@@ -827,7 +850,7 @@ __m128h test_mm_maskz_sub_sh(__mmask8 __U, __m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_sub_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_sub_sh
+  // CHECK-LABEL: test_mm_sub_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fsub half %{{.*}}, %{{.*}}
@@ -836,22 +859,22 @@ __m128h test_mm_sub_sh(__m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_mul_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mul_round_sh
+  // CHECK-LABEL: test_mm_mul_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.mul.sh.round
   return _mm_mul_round_sh(__A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_mask_mul_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_mul_round_sh
+  // CHECK-LABEL: test_mm_mask_mul_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.mul.sh.round
   return _mm_mask_mul_round_sh(__W, __U, __A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_maskz_mul_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_mul_round_sh
+  // CHECK-LABEL: test_mm_maskz_mul_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.mul.sh.round
   return _mm_maskz_mul_round_sh(__U, __A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_mask_mul_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_mul_sh
+  // CHECK-LABEL: test_mm_mask_mul_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fmul half %{{.*}}, %{{.*}}
@@ -865,7 +888,7 @@ __m128h test_mm_mask_mul_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B)
   return _mm_mask_mul_sh(__W, __U, __A, __B);
 }
 __m128h test_mm_maskz_mul_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_mul_sh
+  // CHECK-LABEL: test_mm_maskz_mul_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fmul half %{{.*}}, %{{.*}}
@@ -880,7 +903,7 @@ __m128h test_mm_maskz_mul_sh(__mmask8 __U, __m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_mul_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mul_sh
+  // CHECK-LABEL: test_mm_mul_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fmul half %{{.*}}, %{{.*}}
@@ -889,22 +912,22 @@ __m128h test_mm_mul_sh(__m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_div_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_div_round_sh
+  // CHECK-LABEL: test_mm_div_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.div.sh.round
   return _mm_div_round_sh(__A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_mask_div_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_div_round_sh
+  // CHECK-LABEL: test_mm_mask_div_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.div.sh.round
   return _mm_mask_div_round_sh(__W, __U, __A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_maskz_div_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_div_round_sh
+  // CHECK-LABEL: test_mm_maskz_div_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.div.sh.round
   return _mm_maskz_div_round_sh(__U, __A, __B, _MM_FROUND_NO_EXC | _MM_FROUND_TO_ZERO);
 }
 __m128h test_mm_mask_div_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_div_sh
+  // CHECK-LABEL: test_mm_mask_div_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fdiv half %{{.*}}, %{{.*}}
@@ -918,7 +941,7 @@ __m128h test_mm_mask_div_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B)
   return _mm_mask_div_sh(__W, __U, __A, __B);
 }
 __m128h test_mm_maskz_div_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_div_sh
+  // CHECK-LABEL: test_mm_maskz_div_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fdiv half %{{.*}}, %{{.*}}
@@ -933,7 +956,7 @@ __m128h test_mm_maskz_div_sh(__mmask8 __U, __m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_div_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_div_sh
+  // CHECK-LABEL: test_mm_div_sh
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: %{{.*}} = fdiv half %{{.*}}, %{{.*}}
@@ -942,83 +965,83 @@ __m128h test_mm_div_sh(__m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_min_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_min_round_sh
+  // CHECK-LABEL: test_mm_min_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.min.sh.round
   return _mm_min_round_sh(__A, __B, 0x08);
 }
 __m128h test_mm_mask_min_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_min_round_sh
+  // CHECK-LABEL: test_mm_mask_min_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.min.sh.round
   return _mm_mask_min_round_sh(__W, __U, __A, __B, 0x08);
 }
 __m128h test_mm_maskz_min_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_round_sh
+  // CHECK-LABEL: test_mm_maskz_min_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.min.sh.round
   return _mm_maskz_min_round_sh(__U, __A, __B, 0x08);
 }
 __m128h test_mm_mask_min_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_min_sh
+  // CHECK-LABEL: test_mm_mask_min_sh
   // CHECK: @llvm.x86.avx512fp16.mask.min.sh.round
   return _mm_mask_min_sh(__W, __U, __A, __B);
 }
 __m128h test_mm_maskz_min_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_sh
+  // CHECK-LABEL: test_mm_maskz_min_sh
   // CHECK: @llvm.x86.avx512fp16.mask.min.sh.round
   return _mm_maskz_min_sh(__U, __A, __B);
 }
 
 __m128h test_mm_min_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_min_sh
+  // CHECK-LABEL: test_mm_min_sh
   // CHECK: @llvm.x86.avx512fp16.mask.min.sh.round
   return _mm_min_sh(__A, __B);
 }
 
 __m128h test_mm_max_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_max_round_sh
+  // CHECK-LABEL: test_mm_max_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.max.sh.round
   return _mm_max_round_sh(__A, __B, 0x08);
 }
 __m128h test_mm_mask_max_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_max_round_sh
+  // CHECK-LABEL: test_mm_mask_max_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.max.sh.round
   return _mm_mask_max_round_sh(__W, __U, __A, __B, 0x08);
 }
 __m128h test_mm_maskz_max_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_round_sh
+  // CHECK-LABEL: test_mm_maskz_max_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.max.sh.round
   return _mm_maskz_max_round_sh(__U, __A, __B, 0x08);
 }
 __m128h test_mm_mask_max_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_max_sh
+  // CHECK-LABEL: test_mm_mask_max_sh
   // CHECK: @llvm.x86.avx512fp16.mask.max.sh.round
   return _mm_mask_max_sh(__W, __U, __A, __B);
 }
 __m128h test_mm_maskz_max_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_sh
+  // CHECK-LABEL: test_mm_maskz_max_sh
   // CHECK: @llvm.x86.avx512fp16.mask.max.sh.round
   return _mm_maskz_max_sh(__U, __A, __B);
 }
 
 __m128h test_mm_max_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_max_sh
+  // CHECK-LABEL: test_mm_max_sh
   // CHECK: @llvm.x86.avx512fp16.mask.max.sh.round
   return _mm_max_sh(__A, __B);
 }
 __mmask32 test_mm512_cmp_round_ph_mask(__m512h a, __m512h b) {
-  // CHECK-LABEL: @test_mm512_cmp_round_ph_mask
+  // CHECK-LABEL: test_mm512_cmp_round_ph_mask
   // CHECK: fcmp oeq <32 x half> %{{.*}}, %{{.*}}
   return _mm512_cmp_round_ph_mask(a, b, 0, _MM_FROUND_NO_EXC);
 }
 
 __mmask32 test_mm512_mask_cmp_round_ph_mask(__mmask32 m, __m512h a, __m512h b) {
-  // CHECK-LABEL: @test_mm512_mask_cmp_round_ph_mask
+  // CHECK-LABEL: test_mm512_mask_cmp_round_ph_mask
   // CHECK: [[CMP:%.*]] = fcmp oeq <32 x half> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> [[CMP]], {{.*}}
   return _mm512_mask_cmp_round_ph_mask(m, a, b, 0, _MM_FROUND_NO_EXC);
 }
 
 __mmask32 test_mm512_cmp_ph_mask_eq_oq(__m512h a, __m512h b) {
-  // CHECK-LABEL: @test_mm512_cmp_ph_mask_eq_oq
+  // CHECK-LABEL: test_mm512_cmp_ph_mask_eq_oq
   // CHECK: fcmp oeq <32 x half> %{{.*}}, %{{.*}}
   return _mm512_cmp_ph_mask(a, b, _CMP_EQ_OQ);
 }
@@ -1210,7 +1233,7 @@ __mmask32 test_mm512_cmp_ph_mask_true_us(__m512h a, __m512h b) {
 }
 
 __mmask32 test_mm512_mask_cmp_ph_mask_eq_oq(__mmask32 m, __m512h a, __m512h b) {
-  // CHECK-LABEL: @test_mm512_mask_cmp_ph_mask_eq_oq
+  // CHECK-LABEL: test_mm512_mask_cmp_ph_mask_eq_oq
   // CHECK: [[CMP:%.*]] = fcmp oeq <32 x half> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> [[CMP]], {{.*}}
   return _mm512_mask_cmp_ph_mask(m, a, b, _CMP_EQ_OQ);
@@ -1434,25 +1457,25 @@ __mmask32 test_mm512_mask_cmp_ph_mask_true_us(__mmask32 m, __m512h a, __m512h b)
 }
 
 __mmask8 test_mm_cmp_round_sh_mask(__m128h __X, __m128h __Y) {
-  // CHECK-LABEL: @test_mm_cmp_round_sh_mask
+  // CHECK-LABEL: test_mm_cmp_round_sh_mask
   // CHECK: @llvm.x86.avx512fp16.mask.cmp.sh
   return _mm_cmp_round_sh_mask(__X, __Y, _CMP_NLT_US, _MM_FROUND_NO_EXC);
 }
 
 __mmask8 test_mm_mask_cmp_round_sh_mask(__mmask8 __M, __m128h __X, __m128h __Y) {
-  // CHECK-LABEL: @test_mm_mask_cmp_round_sh_mask
+  // CHECK-LABEL: test_mm_mask_cmp_round_sh_mask
   // CHECK: @llvm.x86.avx512fp16.mask.cmp.sh
   return _mm_mask_cmp_round_sh_mask(__M, __X, __Y, _CMP_NLT_US, _MM_FROUND_NO_EXC);
 }
 
 __mmask8 test_mm_cmp_sh_mask(__m128h __X, __m128h __Y) {
-  // CHECK-LABEL: @test_mm_cmp_sh_mask
+  // CHECK-LABEL: test_mm_cmp_sh_mask
   // CHECK: @llvm.x86.avx512fp16.mask.cmp.sh
   return _mm_cmp_sh_mask(__X, __Y, _CMP_NLT_US);
 }
 
 __mmask8 test_mm_mask_cmp_sh_mask(__mmask8 __M, __m128h __X, __m128h __Y) {
-  // CHECK-LABEL: @test_mm_mask_cmp_sh_mask
+  // CHECK-LABEL: test_mm_mask_cmp_sh_mask
   // CHECK: @llvm.x86.avx512fp16.mask.cmp.sh
   return _mm_mask_cmp_sh_mask(__M, __X, __Y, _CMP_NLT_US);
 }
@@ -1466,49 +1489,49 @@ __m128h test_mm_load_sh(void const *A) {
 }
 
 __m128h test_mm_mask_load_sh(__m128h __A, __mmask8 __U, const void *__W) {
-  // CHECK-LABEL: @test_mm_mask_load_sh
-  // CHECK: %{{.*}} = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_load_sh
+  // CHECK: @llvm.masked.load.v8f16.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x half> %{{.*}})
   return _mm_mask_load_sh(__A, __U, __W);
 }
 
 __m128h test_mm_maskz_load_sh(__mmask8 __U, const void *__W) {
-  // CHECK-LABEL: @test_mm_maskz_load_sh
-  // CHECK: %{{.*}} = call <8 x half> @llvm.masked.load.v8f16.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_load_sh
+  // CHECK: @llvm.masked.load.v8f16.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x half> %{{.*}})
   return _mm_maskz_load_sh(__U, __W);
 }
 
 __m512h test_mm512_load_ph(void *p) {
-  // CHECK-LABEL: @test_mm512_load_ph
+  // CHECK-LABEL: test_mm512_load_ph
   // CHECK: load <32 x half>, ptr %{{.*}}, align 64{{$}}
   return _mm512_load_ph(p);
 }
 
 __m256h test_mm256_load_ph(void *p) {
-  // CHECK-LABEL: @test_mm256_load_ph
+  // CHECK-LABEL: test_mm256_load_ph
   // CHECK: load <16 x half>, ptr %{{.*}}, align 32{{$}}
   return _mm256_load_ph(p);
 }
 
 __m128h test_mm_load_ph(void *p) {
-  // CHECK-LABEL: @test_mm_load_ph
+  // CHECK-LABEL: test_mm_load_ph
   // CHECK: load <8 x half>, ptr %{{.*}}, align 16{{$}}
   return _mm_load_ph(p);
 }
 
 __m512h test_mm512_loadu_ph(void *p) {
-  // CHECK-LABEL: @test_mm512_loadu_ph
+  // CHECK-LABEL: test_mm512_loadu_ph
   // CHECK: load <32 x half>, ptr {{.*}}, align 1{{$}}
   return _mm512_loadu_ph(p);
 }
 
 __m256h test_mm256_loadu_ph(void *p) {
-  // CHECK-LABEL: @test_mm256_loadu_ph
+  // CHECK-LABEL: test_mm256_loadu_ph
   // CHECK: load <16 x half>, ptr {{.*}}, align 1{{$}}
   return _mm256_loadu_ph(p);
 }
 
 __m128h test_mm_loadu_ph(void *p) {
-  // CHECK-LABEL: @test_mm_loadu_ph
+  // CHECK-LABEL: test_mm_loadu_ph
   // CHECK: load <8 x half>, ptr {{.*}}, align 1{{$}}
   return _mm_loadu_ph(p);
 }
@@ -1521,45 +1544,45 @@ void test_mm_store_sh(void *A, __m128h B) {
 }
 
 void test_mm_mask_store_sh(void *__P, __mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_store_sh
+  // CHECK-LABEL: test_mm_mask_store_sh
   // CHECK: call void @llvm.masked.store.v8f16.p0(<8 x half> %{{.*}}, ptr %{{.*}}, i32 1, <8 x i1> %{{.*}})
   _mm_mask_store_sh(__P, __U, __A);
 }
 
 void test_mm512_store_ph(void *p, __m512h a) {
-  // CHECK-LABEL: @test_mm512_store_ph
+  // CHECK-LABEL: test_mm512_store_ph
   // CHECK: store <32 x half> %{{.*}}, ptr %{{.*}}, align 64
   _mm512_store_ph(p, a);
 }
 
 void test_mm256_store_ph(void *p, __m256h a) {
-  // CHECK-LABEL: @test_mm256_store_ph
+  // CHECK-LABEL: test_mm256_store_ph
   // CHECK: store <16 x half> %{{.*}}, ptr %{{.*}}, align 32
   _mm256_store_ph(p, a);
 }
 
 void test_mm_store_ph(void *p, __m128h a) {
-  // CHECK-LABEL: @test_mm_store_ph
+  // CHECK-LABEL: test_mm_store_ph
   // CHECK: store <8 x half> %{{.*}}, ptr %{{.*}}, align 16
   _mm_store_ph(p, a);
 }
 
 void test_mm512_storeu_ph(void *p, __m512h a) {
-  // CHECK-LABEL: @test_mm512_storeu_ph
+  // CHECK-LABEL: test_mm512_storeu_ph
   // CHECK: store <32 x half> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   // CHECK-NEXT: ret void
   _mm512_storeu_ph(p, a);
 }
 
 void test_mm256_storeu_ph(void *p, __m256h a) {
-  // CHECK-LABEL: @test_mm256_storeu_ph
+  // CHECK-LABEL: test_mm256_storeu_ph
   // CHECK: store <16 x half> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   // CHECK-NEXT: ret void
   _mm256_storeu_ph(p, a);
 }
 
 void test_mm_storeu_ph(void *p, __m128h a) {
-  // CHECK-LABEL: @test_mm_storeu_ph
+  // CHECK-LABEL: test_mm_storeu_ph
   // CHECK: store <8 x half> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   // CHECK-NEXT: ret void
   _mm_storeu_ph(p, a);
@@ -1573,7 +1596,7 @@ __m128h test_mm_move_sh(__m128h A, __m128h B) {
 }
 
 __m128h test_mm_mask_move_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_move_sh
+  // CHECK-LABEL: test_mm_mask_move_sh
   // CHECK: [[EXT:%.*]] = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: insertelement <8 x half> %{{.*}}, half [[EXT]], i32 0
   // CHECK: [[A:%.*]] = extractelement <8 x half> [[VEC:%.*]], i64 0
@@ -1586,7 +1609,7 @@ __m128h test_mm_mask_move_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B
 }
 
 __m128h test_mm_maskz_move_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_move_sh
+  // CHECK-LABEL: test_mm_maskz_move_sh
   // CHECK: [[EXT:%.*]] = extractelement <8 x half> %{{.*}}, i32 0
   // CHECK: insertelement <8 x half> %{{.*}}, half [[EXT]], i32 0
   // CHECK: [[A:%.*]] = extractelement <8 x half> [[VEC:%.*]], i64 0
@@ -1614,532 +1637,532 @@ __m128i test_mm_cvtsi16_si128(short A) {
 }
 
 __m512h test_mm512_rcp_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_rcp_ph
+  // CHECK-LABEL: test_mm512_rcp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.ph.512
   return _mm512_rcp_ph(__A);
 }
 
 __m512h test_mm512_mask_rcp_ph(__m512h __W, __mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_rcp_ph
+  // CHECK-LABEL: test_mm512_mask_rcp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.ph.512
   return (__m512h)_mm512_mask_rcp_ph(__W, __U, __A);
 }
 
 __m512h test_mm512_maskz_rcp_ph(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_rcp_ph
+  // CHECK-LABEL: test_mm512_maskz_rcp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.ph.512
   return _mm512_maskz_rcp_ph(__U, __A);
 }
 
 __m512h test_mm512_rsqrt_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_rsqrt_ph
+  // CHECK-LABEL: test_mm512_rsqrt_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.ph.512
   return _mm512_rsqrt_ph(__A);
 }
 
 __m512h test_mm512_mask_rsqrt_ph(__m512h __W, __mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_rsqrt_ph
+  // CHECK-LABEL: test_mm512_mask_rsqrt_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.ph.512
   return (__m512h)_mm512_mask_rsqrt_ph(__W, __U, __A);
 }
 
 __m512h test_mm512_maskz_rsqrt_ph(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_rsqrt_ph
+  // CHECK-LABEL: test_mm512_maskz_rsqrt_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.ph.512
   return _mm512_maskz_rsqrt_ph(__U, __A);
 }
 
 __m512h test_mm512_getmant_round_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_getmant_round_ph
+  // CHECK-LABEL: test_mm512_getmant_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.512
   return _mm512_getmant_round_ph(__A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_getmant_round_ph(__m512h __W, __mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_getmant_round_ph
+  // CHECK-LABEL: test_mm512_mask_getmant_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.512
   return _mm512_mask_getmant_round_ph(__W, __U, __A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_getmant_round_ph(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_getmant_round_ph
+  // CHECK-LABEL: test_mm512_maskz_getmant_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.512
   return _mm512_maskz_getmant_round_ph(__U, __A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_getmant_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_getmant_ph
+  // CHECK-LABEL: test_mm512_getmant_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.512
   return _mm512_getmant_ph(__A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan);
 }
 
 __m512h test_mm512_mask_getmant_ph(__m512h __W, __mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_getmant_ph
+  // CHECK-LABEL: test_mm512_mask_getmant_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.512
   return _mm512_mask_getmant_ph(__W, __U, __A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan);
 }
 
 __m512h test_mm512_maskz_getmant_ph(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_getmant_ph
+  // CHECK-LABEL: test_mm512_maskz_getmant_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.512
   return _mm512_maskz_getmant_ph(__U, __A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan);
 }
 
 __m512h test_mm512_scalef_round_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_scalef_round_ph
+  // CHECK-LABEL: test_mm512_scalef_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.512
   return _mm512_scalef_round_ph(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_scalef_round_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_scalef_round_ph
+  // CHECK-LABEL: test_mm512_mask_scalef_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.512
   return _mm512_mask_scalef_round_ph(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_scalef_round_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_scalef_round_ph
+  // CHECK-LABEL: test_mm512_maskz_scalef_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.512
   return _mm512_maskz_scalef_round_ph(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_scalef_ph(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_scalef_ph
+  // CHECK-LABEL: test_mm512_scalef_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.512
   return _mm512_scalef_ph(__A, __B);
 }
 
 __m512h test_mm512_mask_scalef_ph(__m512h __W, __mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_scalef_ph
+  // CHECK-LABEL: test_mm512_mask_scalef_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.512
   return _mm512_mask_scalef_ph(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_scalef_ph(__mmask32 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_scalef_ph
+  // CHECK-LABEL: test_mm512_maskz_scalef_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.512
   return _mm512_maskz_scalef_ph(__U, __A, __B);
 }
 
 __m512h test_mm512_mask_roundscale_ph(__m512h __W, __mmask16 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_roundscale_ph
+  // CHECK-LABEL: test_mm512_mask_roundscale_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.512
   return _mm512_mask_roundscale_ph(__W, __U, __A, 1);
 }
 
 __m512h test_mm512_maskz_roundscale_ph(__mmask16 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_roundscale_ph
+  // CHECK-LABEL: test_mm512_maskz_roundscale_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.512
   return _mm512_maskz_roundscale_ph(__U, __A, 1);
 }
 
 __m512h test_mm512_mask_roundscale_round_ph(__m512h __A, __mmask16 __U, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_roundscale_round_ph
+  // CHECK-LABEL: test_mm512_mask_roundscale_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.512
   return _mm512_mask_roundscale_round_ph(__A, __U, __C, 3, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_roundscale_round_ph(__m512h __A, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm512_maskz_roundscale_round_ph
+  // CHECK-LABEL: test_mm512_maskz_roundscale_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.512
   return _mm512_maskz_roundscale_round_ph(__U, __A, 3, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_roundscale_round_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_roundscale_round_ph
+  // CHECK-LABEL: test_mm512_roundscale_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.512
   return _mm512_roundscale_round_ph(__A, 3, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_roundscale_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_roundscale_ph
+  // CHECK-LABEL: test_mm512_roundscale_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.512
   return _mm512_roundscale_ph(__A, 3);
 }
 
 __m512h test_mm512_getexp_round_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_getexp_round_ph
+  // CHECK-LABEL: test_mm512_getexp_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.512
   return _mm512_getexp_round_ph(__A, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_getexp_round_ph(__m512h __W, __mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_getexp_round_ph
+  // CHECK-LABEL: test_mm512_mask_getexp_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.512
   return _mm512_mask_getexp_round_ph(__W, __U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_getexp_round_ph(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_getexp_round_ph
+  // CHECK-LABEL: test_mm512_maskz_getexp_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.512
   return _mm512_maskz_getexp_round_ph(__U, __A, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_getexp_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_getexp_ph
+  // CHECK-LABEL: test_mm512_getexp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.512
   return _mm512_getexp_ph(__A);
 }
 
 __m512h test_mm512_mask_getexp_ph(__m512h __W, __mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_getexp_ph
+  // CHECK-LABEL: test_mm512_mask_getexp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.512
   return _mm512_mask_getexp_ph(__W, __U, __A);
 }
 
 __m512h test_mm512_maskz_getexp_ph(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_getexp_ph
+  // CHECK-LABEL: test_mm512_maskz_getexp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.512
   return _mm512_maskz_getexp_ph(__U, __A);
 }
 
 __m512h test_mm512_mask_reduce_ph(__m512h __W, __mmask16 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_reduce_ph
+  // CHECK-LABEL: test_mm512_mask_reduce_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.512
   return _mm512_mask_reduce_ph(__W, __U, __A, 1);
 }
 
 __m512h test_mm512_maskz_reduce_ph(__mmask16 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_reduce_ph
+  // CHECK-LABEL: test_mm512_maskz_reduce_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.512
   return _mm512_maskz_reduce_ph(__U, __A, 1);
 }
 
 __m512h test_mm512_mask_reduce_round_ph(__m512h __A, __mmask16 __U, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_reduce_round_ph
+  // CHECK-LABEL: test_mm512_mask_reduce_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.512
   return _mm512_mask_reduce_round_ph(__A, __U, __C, 3, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_reduce_round_ph(__m512h __A, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm512_maskz_reduce_round_ph
+  // CHECK-LABEL: test_mm512_maskz_reduce_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.512
   return _mm512_maskz_reduce_round_ph(__U, __A, 3, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_reduce_round_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_reduce_round_ph
+  // CHECK-LABEL: test_mm512_reduce_round_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.512
   return _mm512_reduce_round_ph(__A, 3, _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_reduce_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_reduce_ph
+  // CHECK-LABEL: test_mm512_reduce_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.512
   return _mm512_reduce_ph(__A, 3);
 }
 __m128h test_mm_rcp_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_rcp_sh
+  // CHECK-LABEL: test_mm_rcp_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.sh
   return _mm_rcp_sh(__A, __B);
 }
 
 __m128h test_mm_mask_rcp_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_rcp_sh
+  // CHECK-LABEL: test_mm_mask_rcp_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.sh
   return _mm_mask_rcp_sh(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_rcp_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_rcp_sh
+  // CHECK-LABEL: test_mm_maskz_rcp_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.sh
   return _mm_maskz_rcp_sh(__U, __A, __B);
 }
 
 __m128h test_mm_rsqrt_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_rsqrt_sh
+  // CHECK-LABEL: test_mm_rsqrt_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.sh
   return _mm_rsqrt_sh(__A, __B);
 }
 
 __m128h test_mm_mask_rsqrt_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_rsqrt_sh
+  // CHECK-LABEL: test_mm_mask_rsqrt_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.sh
   return _mm_mask_rsqrt_sh(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_rsqrt_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_rsqrt_sh
+  // CHECK-LABEL: test_mm_maskz_rsqrt_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.sh
   return _mm_maskz_rsqrt_sh(__U, __A, __B);
 }
 
 __m128h test_mm_getmant_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_getmant_round_sh
+  // CHECK-LABEL: test_mm_getmant_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.sh
   return _mm_getmant_round_sh(__A, __B, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_src, 8);
 }
 
 __m128h test_mm_getmant_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_getmant_sh
+  // CHECK-LABEL: test_mm_getmant_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.sh
   return _mm_getmant_sh(__A, __B, _MM_MANT_NORM_1_2, _MM_MANT_SIGN_src);
 }
 
 __m128h test_mm_mask_getmant_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_getmant_sh
+  // CHECK-LABEL: test_mm_mask_getmant_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.sh
   return _mm_mask_getmant_sh(__W, __U, __A, __B, 1, 2);
 }
 
 __m128h test_mm_mask_getmant_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_getmant_round_sh
+  // CHECK-LABEL: test_mm_mask_getmant_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.sh
   return _mm_mask_getmant_round_sh(__W, __U, __A, __B, 1, 2, _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_getmant_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_getmant_sh
+  // CHECK-LABEL: test_mm_maskz_getmant_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.sh
   return _mm_maskz_getmant_sh(__U, __A, __B, 1, 2);
 }
 
 __m128h test_mm_maskz_getmant_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_getmant_round_sh
+  // CHECK-LABEL: test_mm_maskz_getmant_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.sh
   return _mm_maskz_getmant_round_sh(__U, __A, __B, 1, 2, _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_getexp_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_getexp_round_sh
+  // CHECK-LABEL: test_mm_getexp_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.sh
   return _mm_getexp_round_sh(__A, __B, 8);
 }
 
 __m128h test_mm_getexp_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_getexp_sh
+  // CHECK-LABEL: test_mm_getexp_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.sh
   return _mm_getexp_sh(__A, __B);
 }
 
 __m128h test_mm_mask_getexp_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_getexp_sh
+  // CHECK-LABEL: test_mm_mask_getexp_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.sh
   return _mm_mask_getexp_sh(__W, __U, __A, __B);
 }
 
 __m128h test_mm_mask_getexp_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_getexp_round_sh
+  // CHECK-LABEL: test_mm_mask_getexp_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.sh
   return _mm_mask_getexp_round_sh(__W, __U, __A, __B, _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_getexp_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_getexp_sh
+  // CHECK-LABEL: test_mm_maskz_getexp_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.sh
   return _mm_maskz_getexp_sh(__U, __A, __B);
 }
 
 __m128h test_mm_maskz_getexp_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_getexp_round_sh
+  // CHECK-LABEL: test_mm_maskz_getexp_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.sh
   return _mm_maskz_getexp_round_sh(__U, __A, __B, _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_scalef_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_scalef_round_sh
+  // CHECK-LABEL: test_mm_scalef_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}, i8 -1, i32 11)
   return _mm_scalef_round_sh(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_scalef_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_scalef_sh
+  // CHECK-LABEL: test_mm_scalef_sh
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.sh
   return _mm_scalef_sh(__A, __B);
 }
 
 __m128h test_mm_mask_scalef_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_scalef_sh
+  // CHECK-LABEL: test_mm_mask_scalef_sh
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.sh
   return _mm_mask_scalef_sh(__W, __U, __A, __B);
 }
 
 __m128h test_mm_mask_scalef_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_scalef_round_sh
+  // CHECK-LABEL: test_mm_mask_scalef_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}, i8 %{{.*}}, i32 11)
   return _mm_mask_scalef_round_sh(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_scalef_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_scalef_sh
+  // CHECK-LABEL: test_mm_maskz_scalef_sh
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.sh
   return _mm_maskz_scalef_sh(__U, __A, __B);
 }
 
 __m128h test_mm_maskz_scalef_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_scalef_round_sh
+  // CHECK-LABEL: test_mm_maskz_scalef_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.sh(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}, i8 %{{.*}}, i32 11)
   return _mm_maskz_scalef_round_sh(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_roundscale_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_roundscale_round_sh
+  // CHECK-LABEL: test_mm_roundscale_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.sh
   return _mm_roundscale_round_sh(__A, __B, 3, _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_roundscale_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_roundscale_sh
+  // CHECK-LABEL: test_mm_roundscale_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.sh
   return _mm_roundscale_sh(__A, __B, 3);
 }
 
 __m128h test_mm_mask_roundscale_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_roundscale_sh
+  // CHECK-LABEL: test_mm_mask_roundscale_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.sh
   return _mm_mask_roundscale_sh(__W, __U, __A, __B, 3);
 }
 
 __m128h test_mm_mask_roundscale_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_roundscale_round_sh
+  // CHECK-LABEL: test_mm_mask_roundscale_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.sh
   return _mm_mask_roundscale_round_sh(__W, __U, __A, __B, 3, _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_roundscale_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_roundscale_round_sh
+  // CHECK-LABEL: test_mm_maskz_roundscale_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.sh
   return _mm_maskz_roundscale_round_sh(__U, __A, __B, 3, _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_roundscale_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_roundscale_sh
+  // CHECK-LABEL: test_mm_maskz_roundscale_sh
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.sh
   return _mm_maskz_roundscale_sh(__U, __A, __B, 3);
 }
 
 __m128h test_mm_reduce_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_reduce_sh
+  // CHECK-LABEL: test_mm_reduce_sh
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.sh
   return _mm_reduce_sh(__A, __B, 4);
 }
 
 __m128h test_mm_mask_reduce_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_reduce_sh
+  // CHECK-LABEL: test_mm_mask_reduce_sh
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.sh
   return _mm_mask_reduce_sh(__W, __U, __A, __B, 4);
 }
 
 __m128h test_mm_maskz_reduce_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_reduce_sh
+  // CHECK-LABEL: test_mm_maskz_reduce_sh
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.sh
   return _mm_maskz_reduce_sh(__U, __A, __B, 4);
 }
 
 __m128h test_mm_reduce_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_reduce_round_sh
+  // CHECK-LABEL: test_mm_reduce_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.sh
   return _mm_reduce_round_sh(__A, __B, 4, 8);
 }
 
 __m128h test_mm_mask_reduce_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_reduce_round_sh
+  // CHECK-LABEL: test_mm_mask_reduce_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.sh
   return _mm_mask_reduce_round_sh(__W, __U, __A, __B, 4, 8);
 }
 
 __m128h test_mm_maskz_reduce_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_reduce_round_sh
+  // CHECK-LABEL: test_mm_maskz_reduce_round_sh
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.sh
   return _mm_maskz_reduce_round_sh(__U, __A, __B, 4, 8);
 }
 
 __m512h test_mm512_sqrt_round_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_sqrt_round_ph
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> %{{.*}}, i32 11)
+  // CHECK-LABEL: test_mm512_sqrt_round_ph
+  // CHECK: @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> %{{.*}}, i32 11)
   return _mm512_sqrt_round_ph(__A, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_sqrt_round_ph(__m512h __W, __mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_sqrt_round_ph
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> %{{.*}}, i32 11)
+  // CHECK-LABEL: test_mm512_mask_sqrt_round_ph
+  // CHECK: @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> %{{.*}}, i32 11)
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_sqrt_round_ph(__W, __U, __A, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_sqrt_round_ph(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_sqrt_round_ph
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> %{{.*}}, i32 11)
+  // CHECK-LABEL: test_mm512_maskz_sqrt_round_ph
+  // CHECK: @llvm.x86.avx512fp16.sqrt.ph.512(<32 x half> %{{.*}}, i32 11)
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> {{.*}}
   return _mm512_maskz_sqrt_round_ph(__U, __A, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_sqrt_ph(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_sqrt_ph
-  // CHECK: %{{.*}} = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_sqrt_ph
+  // CHECK: @llvm.sqrt.v32f16(<32 x half> %{{.*}})
   return _mm512_sqrt_ph(__A);
 }
 __m512h test_mm512_mask_sqrt_ph(__m512h __W, __mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_sqrt_ph
-  // CHECK: %{{.*}} = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_sqrt_ph
+  // CHECK: @llvm.sqrt.v32f16(<32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_sqrt_ph(__W, __U, __A);
 }
 __m512h test_mm512_maskz_sqrt_ph(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_maskz_sqrt_ph
-  // CHECK: %{{.*}} = call <32 x half> @llvm.sqrt.v32f16(<32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_sqrt_ph
+  // CHECK: @llvm.sqrt.v32f16(<32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> {{.*}}
   return _mm512_maskz_sqrt_ph(__U, __A);
 }
 
 __m128h test_mm_sqrt_round_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_sqrt_round_sh
+  // CHECK-LABEL: test_mm_sqrt_round_sh
   // CHECK: call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}, i8 -1, i32 11)
   return _mm_sqrt_round_sh(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_mask_sqrt_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_sqrt_round_sh
+  // CHECK-LABEL: test_mm_mask_sqrt_round_sh
   // CHECK: call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}, i8 {{.*}}, i32 11)
   return _mm_mask_sqrt_round_sh(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_sqrt_round_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_sqrt_round_sh
+  // CHECK-LABEL: test_mm_maskz_sqrt_round_sh
   // CHECK: call <8 x half> @llvm.x86.avx512fp16.mask.sqrt.sh(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}, i8 {{.*}}, i32 11)
   return _mm_maskz_sqrt_round_sh(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_sqrt_sh(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_sqrt_sh
+  // CHECK-LABEL: test_mm_sqrt_sh
   // CHECK: %{{.*}} = call half @llvm.sqrt.f16(half %{{.*}})
   return _mm_sqrt_sh(__A, __B);
 }
 __m128h test_mm_mask_sqrt_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_sqrt_sh
+  // CHECK-LABEL: test_mm_mask_sqrt_sh
   // CHECK: %{{.*}} = call half @llvm.sqrt.f16(half %{{.*}})
   return _mm_mask_sqrt_sh(__W, __U, __A, __B);
 }
 __m128h test_mm_maskz_sqrt_sh(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_sqrt_sh
+  // CHECK-LABEL: test_mm_maskz_sqrt_sh
   // CHECK: %{{.*}} = call half @llvm.sqrt.f16(half %{{.*}})
   return _mm_maskz_sqrt_sh(__U, __A, __B);
 }
 
 __mmask32 test_mm512_mask_fpclass_ph_mask(__mmask32 __U, __m512h __A) {
-  // CHECK-LABEL: @test_mm512_mask_fpclass_ph_mask
+  // CHECK-LABEL: test_mm512_mask_fpclass_ph_mask
   // CHECK: @llvm.x86.avx512fp16.fpclass.ph.512
   return _mm512_mask_fpclass_ph_mask(__U, __A, 4);
 }
 
 __mmask32 test_mm512_fpclass_ph_mask(__m512h __A) {
-  // CHECK-LABEL: @test_mm512_fpclass_ph_mask
+  // CHECK-LABEL: test_mm512_fpclass_ph_mask
   // CHECK: @llvm.x86.avx512fp16.fpclass.ph.512
   return _mm512_fpclass_ph_mask(__A, 4);
 }
 
 __mmask8 test_mm_fpclash_sh_mask(__m128 __A) {
-  // CHECK-LABEL: @test_mm_fpclash_sh_mask
+  // CHECK-LABEL: test_mm_fpclash_sh_mask
   // CHECK: @llvm.x86.avx512fp16.mask.fpclass.sh
   return _mm_fpclass_sh_mask(__A, 2);
 }
 
 __mmask8 test_mm_mask_fpclash_sh_mask(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_fpclash_sh_mask
+  // CHECK-LABEL: test_mm_mask_fpclash_sh_mask
   // CHECK: @llvm.x86.avx512fp16.mask.fpclass.sh
   return _mm_mask_fpclass_sh_mask(__U, __A, 2);
 }
@@ -3034,26 +3057,26 @@ unsigned int test_mm_cvtsh_u32(__m128h A) {
 
 #ifdef __x86_64__
 long long test_mm_cvt_roundsh_i64(__m128h A) {
-  // CHECK-LABEL: test_mm_cvt_roundsh_i64
-  // CHECK: @llvm.x86.avx512fp16.vcvtsh2si64
+  // X64-LABEL: test_mm_cvt_roundsh_i64
+  // X64: @llvm.x86.avx512fp16.vcvtsh2si64
   return _mm_cvt_roundsh_i64(A, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 long long test_mm_cvtsh_i64(__m128h A) {
-  // CHECK-LABEL: test_mm_cvtsh_i64
-  // CHECK: @llvm.x86.avx512fp16.vcvtsh2si64
+  // X64-LABEL: test_mm_cvtsh_i64
+  // X64: @llvm.x86.avx512fp16.vcvtsh2si64
   return _mm_cvtsh_i64(A);
 }
 
 unsigned long long test_mm_cvt_roundsh_u64(__m128h A) {
-  // CHECK-LABEL: test_mm_cvt_roundsh_u64
-  // CHECK: @llvm.x86.avx512fp16.vcvtsh2usi64
+  // X64-LABEL: test_mm_cvt_roundsh_u64
+  // X64: @llvm.x86.avx512fp16.vcvtsh2usi64
   return _mm_cvt_roundsh_u64(A, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 unsigned long long test_mm_cvtsh_u64(__m128h A) {
-  // CHECK-LABEL: test_mm_cvtsh_u64
-  // CHECK: @llvm.x86.avx512fp16.vcvtsh2usi64
+  // X64-LABEL: test_mm_cvtsh_u64
+  // X64: @llvm.x86.avx512fp16.vcvtsh2usi64
   return _mm_cvtsh_u64(A);
 }
 #endif
@@ -3072,14 +3095,14 @@ __m128h test_mm_cvtu32_sh(__m128h A, unsigned int B) {
 
 #ifdef __x86_64__
 __m128h test_mm_cvt_roundu64_sh(__m128h A, unsigned long long B) {
-  // CHECK-LABEL: test_mm_cvt_roundu64_sh
-  // CHECK: @llvm.x86.avx512fp16.vcvtusi642sh
+  // X64-LABEL: test_mm_cvt_roundu64_sh
+  // X64: @llvm.x86.avx512fp16.vcvtusi642sh
   return _mm_cvt_roundu64_sh(A, B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_cvtu64_sh(__m128h A, unsigned long long B) {
-  // CHECK-LABEL: test_mm_cvtu64_sh
-  // CHECK: %{{.*}} = uitofp i64 %{{.*}} to half
+  // X64-LABEL: test_mm_cvtu64_sh
+  // X64: %{{.*}} = uitofp i64 %{{.*}} to half
   return _mm_cvtu64_sh(A, B);
 }
 #endif
@@ -3098,14 +3121,14 @@ __m128h test_mm_cvti32_sh(__m128h A, int B) {
 
 #ifdef __x86_64__
 __m128h test_mm_cvt_roundi64_sh(__m128h A, long long B) {
-  // CHECK-LABEL: test_mm_cvt_roundi64_sh
-  // CHECK: @llvm.x86.avx512fp16.vcvtsi642sh
+  // X64-LABEL: test_mm_cvt_roundi64_sh
+  // X64: @llvm.x86.avx512fp16.vcvtsi642sh
   return _mm_cvt_roundi64_sh(A, B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_cvti64_sh(__m128h A, long long B) {
-  // CHECK-LABEL: test_mm_cvti64_sh
-  // CHECK: %{{.*}} = sitofp i64 %{{.*}} to half
+  // X64-LABEL: test_mm_cvti64_sh
+  // X64: %{{.*}} = sitofp i64 %{{.*}} to half
   return _mm_cvti64_sh(A, B);
 }
 #endif
@@ -3124,14 +3147,14 @@ int test_mm_cvttsh_i32(__m128h A) {
 
 #ifdef __x86_64__
 long long test_mm_cvtt_roundsh_i64(__m128h A) {
-  // CHECK-LABEL: test_mm_cvtt_roundsh_i64
-  // CHECK: @llvm.x86.avx512fp16.vcvttsh2si64
+  // X64-LABEL: test_mm_cvtt_roundsh_i64
+  // X64: @llvm.x86.avx512fp16.vcvttsh2si64
   return _mm_cvtt_roundsh_i64(A, _MM_FROUND_NO_EXC);
 }
 
 long long test_mm_cvttsh_i64(__m128h A) {
-  // CHECK-LABEL: test_mm_cvttsh_i64
-  // CHECK: @llvm.x86.avx512fp16.vcvttsh2si64
+  // X64-LABEL: test_mm_cvttsh_i64
+  // X64: @llvm.x86.avx512fp16.vcvttsh2si64
   return _mm_cvttsh_i64(A);
 }
 #endif
@@ -3150,14 +3173,14 @@ unsigned int test_mm_cvttsh_u32(__m128h A) {
 
 #ifdef __x86_64__
 unsigned long long test_mm_cvtt_roundsh_u64(__m128h A) {
-  // CHECK-LABEL: test_mm_cvtt_roundsh_u64
-  // CHECK: @llvm.x86.avx512fp16.vcvttsh2usi64
+  // X64-LABEL: test_mm_cvtt_roundsh_u64
+  // X64: @llvm.x86.avx512fp16.vcvttsh2usi64
   return _mm_cvtt_roundsh_u64(A, _MM_FROUND_NO_EXC);
 }
 
 unsigned long long test_mm_cvttsh_u64(__m128h A) {
-  // CHECK-LABEL: test_mm_cvttsh_u64
-  // CHECK: @llvm.x86.avx512fp16.vcvttsh2usi64
+  // X64-LABEL: test_mm_cvttsh_u64
+  // X64: @llvm.x86.avx512fp16.vcvttsh2usi64
   return _mm_cvttsh_u64(A);
 }
 #endif
@@ -3235,13 +3258,13 @@ __m256h test_mm512_maskz_cvtxps_ph(__mmask16 A, __m512 B) {
 }
 
 __m512h test_mm512_fmadd_round_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmadd_round_ph
+  // CHECK-LABEL: test_mm512_fmadd_round_ph
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   return _mm512_fmadd_round_ph(__A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_fmadd_round_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmadd_round_ph
+  // CHECK-LABEL: test_mm512_mask_fmadd_round_ph
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
@@ -3249,7 +3272,7 @@ __m512h test_mm512_mask_fmadd_round_ph(__m512h __A, __mmask32 __U, __m512h __B,
 }
 
 __m512h test_mm512_mask3_fmadd_round_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmadd_round_ph
+  // CHECK-LABEL: test_mm512_mask3_fmadd_round_ph
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
@@ -3257,7 +3280,7 @@ __m512h test_mm512_mask3_fmadd_round_ph(__m512h __A, __m512h __B, __m512h __C, _
 }
 
 __m512h test_mm512_maskz_fmadd_round_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmadd_round_ph
+  // CHECK-LABEL: test_mm512_maskz_fmadd_round_ph
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> zeroinitializer
@@ -3265,14 +3288,14 @@ __m512h test_mm512_maskz_fmadd_round_ph(__mmask32 __U, __m512h __A, __m512h __B,
 }
 
 __m512h test_mm512_fmsub_round_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmsub_round_ph
+  // CHECK-LABEL: test_mm512_fmsub_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   return _mm512_fmsub_round_ph(__A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_fmsub_round_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmsub_round_ph
+  // CHECK-LABEL: test_mm512_mask_fmsub_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
@@ -3281,7 +3304,7 @@ __m512h test_mm512_mask_fmsub_round_ph(__m512h __A, __mmask32 __U, __m512h __B,
 }
 
 __m512h test_mm512_maskz_fmsub_round_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmsub_round_ph
+  // CHECK-LABEL: test_mm512_maskz_fmsub_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
@@ -3290,14 +3313,14 @@ __m512h test_mm512_maskz_fmsub_round_ph(__mmask32 __U, __m512h __A, __m512h __B,
 }
 
 __m512h test_mm512_fnmadd_round_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fnmadd_round_ph
+  // CHECK-LABEL: test_mm512_fnmadd_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   return _mm512_fnmadd_round_ph(__A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask3_fnmadd_round_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fnmadd_round_ph
+  // CHECK-LABEL: test_mm512_mask3_fnmadd_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
@@ -3306,7 +3329,7 @@ __m512h test_mm512_mask3_fnmadd_round_ph(__m512h __A, __m512h __B, __m512h __C,
 }
 
 __m512h test_mm512_maskz_fnmadd_round_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fnmadd_round_ph
+  // CHECK-LABEL: test_mm512_maskz_fnmadd_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
@@ -3315,7 +3338,7 @@ __m512h test_mm512_maskz_fnmadd_round_ph(__mmask32 __U, __m512h __A, __m512h __B
 }
 
 __m512h test_mm512_fnmsub_round_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fnmsub_round_ph
+  // CHECK-LABEL: test_mm512_fnmsub_round_ph
   // CHECK: fneg
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
@@ -3323,7 +3346,7 @@ __m512h test_mm512_fnmsub_round_ph(__m512h __A, __m512h __B, __m512h __C) {
 }
 
 __m512h test_mm512_maskz_fnmsub_round_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fnmsub_round_ph
+  // CHECK-LABEL: test_mm512_maskz_fnmsub_round_ph
   // CHECK: fneg
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
@@ -3333,109 +3356,109 @@ __m512h test_mm512_maskz_fnmsub_round_ph(__mmask32 __U, __m512h __A, __m512h __B
 }
 
 __m512h test_mm512_fmadd_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmadd_ph
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_fmadd_ph
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   return _mm512_fmadd_ph(__A, __B, __C);
 }
 
 __m512h test_mm512_mask_fmadd_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmadd_ph
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_fmadd_ph
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   return _mm512_mask_fmadd_ph(__A, __U, __B, __C);
 }
 
 __m512h test_mm512_mask3_fmadd_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmadd_ph
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask3_fmadd_ph
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask3_fmadd_ph(__A, __B, __C, __U);
 }
 
 __m512h test_mm512_maskz_fmadd_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmadd_ph
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_fmadd_ph
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> zeroinitializer
   return _mm512_maskz_fmadd_ph(__U, __A, __B, __C);
 }
 
 __m512h test_mm512_fmsub_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmsub_ph
+  // CHECK-LABEL: test_mm512_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   return _mm512_fmsub_ph(__A, __B, __C);
 }
 
 __m512h test_mm512_mask_fmsub_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmsub_ph
+  // CHECK-LABEL: test_mm512_mask_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_fmsub_ph(__A, __U, __B, __C);
 }
 
 __m512h test_mm512_maskz_fmsub_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmsub_ph
+  // CHECK-LABEL: test_mm512_maskz_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> zeroinitializer
   return _mm512_maskz_fmsub_ph(__U, __A, __B, __C);
 }
 
 __m512h test_mm512_fnmadd_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fnmadd_ph
+  // CHECK-LABEL: test_mm512_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   return _mm512_fnmadd_ph(__A, __B, __C);
 }
 
 __m512h test_mm512_mask3_fnmadd_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fnmadd_ph
+  // CHECK-LABEL: test_mm512_mask3_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask3_fnmadd_ph(__A, __B, __C, __U);
 }
 
 __m512h test_mm512_maskz_fnmadd_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fnmadd_ph
+  // CHECK-LABEL: test_mm512_maskz_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> zeroinitializer
   return _mm512_maskz_fnmadd_ph(__U, __A, __B, __C);
 }
 
 __m512h test_mm512_fnmsub_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fnmsub_ph
+  // CHECK-LABEL: test_mm512_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   return _mm512_fnmsub_ph(__A, __B, __C);
 }
 
 __m512h test_mm512_maskz_fnmsub_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fnmsub_ph
+  // CHECK-LABEL: test_mm512_maskz_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> zeroinitializer
   return _mm512_maskz_fnmsub_ph(__U, __A, __B, __C);
 }
 
 __m512h test_mm512_fmaddsub_round_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmaddsub_round_ph
+  // CHECK-LABEL: test_mm512_fmaddsub_round_ph
   // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512
   return _mm512_fmaddsub_round_ph(__A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_fmaddsub_round_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmaddsub_round_ph
+  // CHECK-LABEL: test_mm512_mask_fmaddsub_round_ph
   // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
@@ -3443,7 +3466,7 @@ __m512h test_mm512_mask_fmaddsub_round_ph(__m512h __A, __mmask32 __U, __m512h __
 }
 
 __m512h test_mm512_mask3_fmaddsub_round_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmaddsub_round_ph
+  // CHECK-LABEL: test_mm512_mask3_fmaddsub_round_ph
   // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
@@ -3451,7 +3474,7 @@ __m512h test_mm512_mask3_fmaddsub_round_ph(__m512h __A, __m512h __B, __m512h __C
 }
 
 __m512h test_mm512_maskz_fmaddsub_round_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmaddsub_round_ph
+  // CHECK-LABEL: test_mm512_maskz_fmaddsub_round_ph
   // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> zeroinitializer
@@ -3459,14 +3482,14 @@ __m512h test_mm512_maskz_fmaddsub_round_ph(__mmask32 __U, __m512h __A, __m512h _
 }
 
 __m512h test_mm512_fmsubadd_round_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmsubadd_round_ph
+  // CHECK-LABEL: test_mm512_fmsubadd_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512
   return _mm512_fmsubadd_round_ph(__A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_fmsubadd_round_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmsubadd_round_ph
+  // CHECK-LABEL: test_mm512_mask_fmsubadd_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
@@ -3475,7 +3498,7 @@ __m512h test_mm512_mask_fmsubadd_round_ph(__m512h __A, __mmask32 __U, __m512h __
 }
 
 __m512h test_mm512_maskz_fmsubadd_round_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmsubadd_round_ph
+  // CHECK-LABEL: test_mm512_maskz_fmsubadd_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
@@ -3484,66 +3507,66 @@ __m512h test_mm512_maskz_fmsubadd_round_ph(__mmask32 __U, __m512h __A, __m512h _
 }
 
 __m512h test_mm512_fmaddsub_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmaddsub_ph
+  // CHECK-LABEL: test_mm512_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}, i32 4)
   return _mm512_fmaddsub_ph(__A, __B, __C);
 }
 
 __m512h test_mm512_mask_fmaddsub_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmaddsub_ph
+  // CHECK-LABEL: test_mm512_mask_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}, i32 4)
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_fmaddsub_ph(__A, __U, __B, __C);
 }
 
 __m512h test_mm512_mask3_fmaddsub_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmaddsub_ph
+  // CHECK-LABEL: test_mm512_mask3_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}, i32 4)
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask3_fmaddsub_ph(__A, __B, __C, __U);
 }
 
 __m512h test_mm512_maskz_fmaddsub_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmaddsub_ph
+  // CHECK-LABEL: test_mm512_maskz_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}, i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}, i32 4)
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> zeroinitializer
   return _mm512_maskz_fmaddsub_ph(__U, __A, __B, __C);
 }
 
 __m512h test_mm512_fmsubadd_ph(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmsubadd_ph
+  // CHECK-LABEL: test_mm512_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> [[NEG]], i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> [[NEG]], i32 4)
   return _mm512_fmsubadd_ph(__A, __B, __C);
 }
 
 __m512h test_mm512_mask_fmsubadd_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmsubadd_ph
+  // CHECK-LABEL: test_mm512_mask_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> [[NEG]], i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> [[NEG]], i32 4)
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_fmsubadd_ph(__A, __U, __B, __C);
 }
 
 __m512h test_mm512_maskz_fmsubadd_ph(__mmask32 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmsubadd_ph
+  // CHECK-LABEL: test_mm512_maskz_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> [[NEG]], i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> [[NEG]], i32 4)
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> zeroinitializer
   return _mm512_maskz_fmsubadd_ph(__U, __A, __B, __C);
 }
 
 __m512h test_mm512_mask3_fmsub_round_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmsub_round_ph
+  // CHECK-LABEL: test_mm512_mask3_fmsub_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
@@ -3552,16 +3575,16 @@ __m512h test_mm512_mask3_fmsub_round_ph(__m512h __A, __m512h __B, __m512h __C, _
 }
 
 __m512h test_mm512_mask3_fmsub_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmsub_ph
+  // CHECK-LABEL: test_mm512_mask3_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask3_fmsub_ph(__A, __B, __C, __U);
 }
 
 __m512h test_mm512_mask3_fmsubadd_round_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmsubadd_round_ph
+  // CHECK-LABEL: test_mm512_mask3_fmsubadd_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
@@ -3570,16 +3593,16 @@ __m512h test_mm512_mask3_fmsubadd_round_ph(__m512h __A, __m512h __B, __m512h __C
 }
 
 __m512h test_mm512_mask3_fmsubadd_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmsubadd_ph
+  // CHECK-LABEL: test_mm512_mask3_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <32 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> [[NEG]], i32 4)
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.512(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> [[NEG]], i32 4)
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask3_fmsubadd_ph(__A, __B, __C, __U);
 }
 
 __m512h test_mm512_mask_fnmadd_round_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fnmadd_round_ph
+  // CHECK-LABEL: test_mm512_mask_fnmadd_round_ph
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
@@ -3588,16 +3611,16 @@ __m512h test_mm512_mask_fnmadd_round_ph(__m512h __A, __mmask32 __U, __m512h __B,
 }
 
 __m512h test_mm512_mask_fnmadd_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fnmadd_ph
+  // CHECK-LABEL: test_mm512_mask_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_fnmadd_ph(__A, __U, __B, __C);
 }
 
 __m512h test_mm512_mask_fnmsub_round_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fnmsub_round_ph
+  // CHECK-LABEL: test_mm512_mask_fnmsub_round_ph
   // CHECK: fneg
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
@@ -3607,7 +3630,7 @@ __m512h test_mm512_mask_fnmsub_round_ph(__m512h __A, __mmask32 __U, __m512h __B,
 }
 
 __m512h test_mm512_mask3_fnmsub_round_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fnmsub_round_ph
+  // CHECK-LABEL: test_mm512_mask3_fnmsub_round_ph
   // CHECK: fneg
   // CHECK: fneg
   // CHECK: @llvm.x86.avx512fp16.vfmadd.ph.512
@@ -3617,27 +3640,27 @@ __m512h test_mm512_mask3_fnmsub_round_ph(__m512h __A, __m512h __B, __m512h __C,
 }
 
 __m512h test_mm512_mask_fnmsub_ph(__m512h __A, __mmask32 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fnmsub_ph
+  // CHECK-LABEL: test_mm512_mask_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_fnmsub_ph(__A, __U, __B, __C);
 }
 
 __m512h test_mm512_mask3_fnmsub_ph(__m512h __A, __m512h __B, __m512h __C, __mmask32 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fnmsub_ph
+  // CHECK-LABEL: test_mm512_mask3_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <32 x half> @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
+  // CHECK: @llvm.fma.v32f16(<32 x half> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}})
   // CHECK: bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask3_fnmsub_ph(__A, __B, __C, __U);
 }
 
 __m128h test_mm_fmadd_sh(__m128h __W, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fmadd_sh
+  // CHECK-LABEL: test_mm_fmadd_sh
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[C:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3646,7 +3669,7 @@ __m128h test_mm_fmadd_sh(__m128h __W, __m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_mask_fmadd_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fmadd_sh
+  // CHECK-LABEL: test_mm_mask_fmadd_sh
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[C:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3659,7 +3682,7 @@ __m128h test_mm_mask_fmadd_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __
 }
 
 __m128h test_mm_fmadd_round_sh(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fmadd_round_sh
+  // CHECK-LABEL: test_mm_fmadd_round_sh
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[C:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3669,7 +3692,7 @@ __m128h test_mm_fmadd_round_sh(__m128h __A, __m128h __B, __m128h __C) {
 }
 
 __m128h test_mm_mask_fmadd_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fmadd_round_sh
+  // CHECK-LABEL: test_mm_mask_fmadd_round_sh
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[C:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3682,7 +3705,7 @@ __m128h test_mm_mask_fmadd_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m1
 }
 
 __m128h test_mm_maskz_fmadd_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmadd_sh
+  // CHECK-LABEL: test_mm_maskz_fmadd_sh
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[C:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3695,7 +3718,7 @@ __m128h test_mm_maskz_fmadd_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h _
 }
 
 __m128h test_mm_maskz_fmadd_round_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmadd_round_sh
+  // CHECK-LABEL: test_mm_maskz_fmadd_round_sh
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[C:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3708,7 +3731,7 @@ __m128h test_mm_maskz_fmadd_round_sh(__mmask8 __U, __m128h __A, __m128h __B, __m
 }
 
 __m128h test_mm_mask3_fmadd_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmadd_sh
+  // CHECK-LABEL: test_mm_mask3_fmadd_sh
   // CHECK: [[A:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[C:%.+]] = extractelement <8 x half> [[ORIGC:%.+]], i64 0
@@ -3721,7 +3744,7 @@ __m128h test_mm_mask3_fmadd_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 _
 }
 
 __m128h test_mm_mask3_fmadd_round_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmadd_round_sh
+  // CHECK-LABEL: test_mm_mask3_fmadd_round_sh
   // CHECK: [[A:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[C:%.+]] = extractelement <8 x half> [[ORIGC:%.+]], i64 0
@@ -3734,7 +3757,7 @@ __m128h test_mm_mask3_fmadd_round_sh(__m128h __W, __m128h __X, __m128h __Y, __mm
 }
 
 __m128h test_mm_fmsub_sh(__m128h __W, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fmsub_sh
+  // CHECK-LABEL: test_mm_fmsub_sh
   // CHECK: %{{.*}} = fneg <8 x half> %{{.*}}
   // CHECK-NEXT: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
@@ -3746,7 +3769,7 @@ __m128h test_mm_fmsub_sh(__m128h __W, __m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_mask_fmsub_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fmsub_sh
+  // CHECK-LABEL: test_mm_mask_fmsub_sh
   // CHECK: %{{.*}} = fneg <8 x half> %{{.*}}
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
@@ -3761,7 +3784,7 @@ __m128h test_mm_mask_fmsub_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __
 }
 
 __m128h test_mm_fmsub_round_sh(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fmsub_round_sh
+  // CHECK-LABEL: test_mm_fmsub_round_sh
   // CHECK: %{{.*}} = fneg <8 x half> %{{.*}}
   // CHECK-NEXT: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
@@ -3773,7 +3796,7 @@ __m128h test_mm_fmsub_round_sh(__m128h __A, __m128h __B, __m128h __C) {
 }
 
 __m128h test_mm_mask_fmsub_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fmsub_round_sh
+  // CHECK-LABEL: test_mm_mask_fmsub_round_sh
   // CHECK: %{{.*}} = fneg <8 x half> %{{.*}}
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
@@ -3788,7 +3811,7 @@ __m128h test_mm_mask_fmsub_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m1
 }
 
 __m128h test_mm_maskz_fmsub_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmsub_sh
+  // CHECK-LABEL: test_mm_maskz_fmsub_sh
   // CHECK: %{{.*}} = fneg <8 x half> %{{.*}}
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
@@ -3803,7 +3826,7 @@ __m128h test_mm_maskz_fmsub_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h _
 }
 
 __m128h test_mm_maskz_fmsub_round_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmsub_round_sh
+  // CHECK-LABEL: test_mm_maskz_fmsub_round_sh
   // CHECK: %{{.*}} = fneg <8 x half> %{{.*}}
   // CHECK: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: %{{.*}} = extractelement <8 x half> %{{.*}}, i64 0
@@ -3818,7 +3841,7 @@ __m128h test_mm_maskz_fmsub_round_sh(__mmask8 __U, __m128h __A, __m128h __B, __m
 }
 
 __m128h test_mm_mask3_fmsub_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmsub_sh
+  // CHECK-LABEL: test_mm_mask3_fmsub_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3833,7 +3856,7 @@ __m128h test_mm_mask3_fmsub_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 _
 }
 
 __m128h test_mm_mask3_fmsub_round_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmsub_round_sh
+  // CHECK-LABEL: test_mm_mask3_fmsub_round_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3848,7 +3871,7 @@ __m128h test_mm_mask3_fmsub_round_sh(__m128h __W, __m128h __X, __m128h __Y, __mm
 }
 
 __m128h test_mm_fnmadd_sh(__m128h __W, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fnmadd_sh
+  // CHECK-LABEL: test_mm_fnmadd_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3858,7 +3881,7 @@ __m128h test_mm_fnmadd_sh(__m128h __W, __m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_mask_fnmadd_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fnmadd_sh
+  // CHECK-LABEL: test_mm_mask_fnmadd_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3872,7 +3895,7 @@ __m128h test_mm_mask_fnmadd_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h _
 }
 
 __m128h test_mm_fnmadd_round_sh(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fnmadd_round_sh
+  // CHECK-LABEL: test_mm_fnmadd_round_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3883,7 +3906,7 @@ __m128h test_mm_fnmadd_round_sh(__m128h __A, __m128h __B, __m128h __C) {
 }
 
 __m128h test_mm_mask_fnmadd_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fnmadd_round_sh
+  // CHECK-LABEL: test_mm_mask_fnmadd_round_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3897,7 +3920,7 @@ __m128h test_mm_mask_fnmadd_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m
 }
 
 __m128h test_mm_maskz_fnmadd_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmadd_sh
+  // CHECK-LABEL: test_mm_maskz_fnmadd_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3911,7 +3934,7 @@ __m128h test_mm_maskz_fnmadd_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h
 }
 
 __m128h test_mm_maskz_fnmadd_round_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmadd_round_sh
+  // CHECK-LABEL: test_mm_maskz_fnmadd_round_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3925,7 +3948,7 @@ __m128h test_mm_maskz_fnmadd_round_sh(__mmask8 __U, __m128h __A, __m128h __B, __
 }
 
 __m128h test_mm_mask3_fnmadd_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmadd_sh
+  // CHECK-LABEL: test_mm_mask3_fnmadd_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3939,7 +3962,7 @@ __m128h test_mm_mask3_fnmadd_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8
 }
 
 __m128h test_mm_mask3_fnmadd_round_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmadd_round_sh
+  // CHECK-LABEL: test_mm_mask3_fnmadd_round_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
   // CHECK-NEXT: [[B:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -3953,7 +3976,7 @@ __m128h test_mm_mask3_fnmadd_round_sh(__m128h __W, __m128h __X, __m128h __Y, __m
 }
 
 __m128h test_mm_fnmsub_sh(__m128h __W, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fnmsub_sh
+  // CHECK-LABEL: test_mm_fnmsub_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[NEG2:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
@@ -3964,7 +3987,7 @@ __m128h test_mm_fnmsub_sh(__m128h __W, __m128h __A, __m128h __B) {
 }
 
 __m128h test_mm_mask_fnmsub_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fnmsub_sh
+  // CHECK-LABEL: test_mm_mask_fnmsub_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[NEG2:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
@@ -3979,7 +4002,7 @@ __m128h test_mm_mask_fnmsub_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h _
 }
 
 __m128h test_mm_fnmsub_round_sh(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fnmsub_round_sh
+  // CHECK-LABEL: test_mm_fnmsub_round_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[NEG2:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
@@ -3991,7 +4014,7 @@ __m128h test_mm_fnmsub_round_sh(__m128h __A, __m128h __B, __m128h __C) {
 }
 
 __m128h test_mm_mask_fnmsub_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fnmsub_round_sh
+  // CHECK-LABEL: test_mm_mask_fnmsub_round_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[NEG2:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
@@ -4006,7 +4029,7 @@ __m128h test_mm_mask_fnmsub_round_sh(__m128h __W, __mmask8 __U, __m128h __A, __m
 }
 
 __m128h test_mm_maskz_fnmsub_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmsub_sh
+  // CHECK-LABEL: test_mm_maskz_fnmsub_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[NEG2:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
@@ -4021,7 +4044,7 @@ __m128h test_mm_maskz_fnmsub_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h
 }
 
 __m128h test_mm_maskz_fnmsub_round_sh(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmsub_round_sh
+  // CHECK-LABEL: test_mm_maskz_fnmsub_round_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[NEG2:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> [[ORIGA:%.+]], i64 0
@@ -4036,7 +4059,7 @@ __m128h test_mm_maskz_fnmsub_round_sh(__mmask8 __U, __m128h __A, __m128h __B, __
 }
 
 __m128h test_mm_mask3_fnmsub_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmsub_sh
+  // CHECK-LABEL: test_mm_mask3_fnmsub_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[NEG2:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -4052,7 +4075,7 @@ __m128h test_mm_mask3_fnmsub_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8
 }
 
 __m128h test_mm_mask3_fnmsub_round_sh(__m128h __W, __m128h __X, __m128h __Y, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmsub_round_sh
+  // CHECK-LABEL: test_mm_mask3_fnmsub_round_sh
   // CHECK: [[NEG:%.+]] = fneg
   // CHECK: [[NEG2:%.+]] = fneg
   // CHECK: [[A:%.+]] = extractelement <8 x half> %{{.*}}, i64 0
@@ -4068,13 +4091,13 @@ __m128h test_mm_mask3_fnmsub_round_sh(__m128h __W, __m128h __X, __m128h __Y, __m
 }
 
 __m128h test_mm_fcmadd_sch(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fcmadd_sch
+  // CHECK-LABEL: test_mm_fcmadd_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.csh
   return _mm_fcmadd_sch(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fcmadd_sch(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fcmadd_sch
+  // CHECK-LABEL: test_mm_mask_fcmadd_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.csh
   // CHECK:  %{{.*}} = shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK:  %{{.*}} = select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -4082,13 +4105,13 @@ __m128h test_mm_mask_fcmadd_sch(__m128h __A, __mmask8 __U, __m128h __B, __m128h
 }
 
 __m128h test_mm_maskz_fcmadd_sch(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fcmadd_sch
+  // CHECK-LABEL: test_mm_maskz_fcmadd_sch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfcmadd.csh
   return _mm_maskz_fcmadd_sch(__U, __A, __B, __C);
 }
 
 __m128h test_mm_mask3_fcmadd_sch(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fcmadd_sch
+  // CHECK-LABEL: test_mm_mask3_fcmadd_sch
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
@@ -4099,13 +4122,13 @@ __m128h test_mm_mask3_fcmadd_sch(__m128h __A, __m128h __B, __m128h __C, __mmask8
 }
 
 __m128h test_mm_fcmadd_round_sch(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fcmadd_round_sch
+  // CHECK-LABEL: test_mm_fcmadd_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.csh
   return _mm_fcmadd_round_sch(__A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_mask_fcmadd_round_sch(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fcmadd_round_sch
+  // CHECK-LABEL: test_mm_mask_fcmadd_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.csh
   // CHECK:  %{{.*}} = shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK:  %{{.*}} = select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -4113,13 +4136,13 @@ __m128h test_mm_mask_fcmadd_round_sch(__m128h __A, __mmask8 __U, __m128h __B, __
 }
 
 __m128h test_mm_maskz_fcmadd_round_sch(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fcmadd_round_sch
+  // CHECK-LABEL: test_mm_maskz_fcmadd_round_sch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfcmadd.csh
   return _mm_maskz_fcmadd_round_sch(__U, __A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_mask3_fcmadd_round_sch(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fcmadd_round_sch
+  // CHECK-LABEL: test_mm_mask3_fcmadd_round_sch
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
@@ -4130,13 +4153,13 @@ __m128h test_mm_mask3_fcmadd_round_sch(__m128h __A, __m128h __B, __m128h __C, __
 }
 
 __m128h test_mm_fmadd_sch(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fmadd_sch
+  // CHECK-LABEL: test_mm_fmadd_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.csh
   return _mm_fmadd_sch(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fmadd_sch(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fmadd_sch
+  // CHECK-LABEL: test_mm_mask_fmadd_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.csh
   // CHECK:  %{{.*}} = shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK:  %{{.*}} = select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -4144,13 +4167,13 @@ __m128h test_mm_mask_fmadd_sch(__m128h __A, __mmask8 __U, __m128h __B, __m128h _
 }
 
 __m128h test_mm_maskz_fmadd_sch(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmadd_sch
+  // CHECK-LABEL: test_mm_maskz_fmadd_sch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfmadd.csh
   return _mm_maskz_fmadd_sch(__U, __A, __B, __C);
 }
 
 __m128h test_mm_mask3_fmadd_sch(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmadd_sch
+  // CHECK-LABEL: test_mm_mask3_fmadd_sch
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
@@ -4161,13 +4184,13 @@ __m128h test_mm_mask3_fmadd_sch(__m128h __A, __m128h __B, __m128h __C, __mmask8
 }
 
 __m128h test_mm_fmadd_round_sch(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fmadd_round_sch
+  // CHECK-LABEL: test_mm_fmadd_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.csh
   return _mm_fmadd_round_sch(__A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_mask_fmadd_round_sch(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fmadd_round_sch
+  // CHECK-LABEL: test_mm_mask_fmadd_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.csh
   // CHECK:  %{{.*}} = shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK:  %{{.*}} = select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -4175,13 +4198,13 @@ __m128h test_mm_mask_fmadd_round_sch(__m128h __A, __mmask8 __U, __m128h __B, __m
 }
 
 __m128h test_mm_maskz_fmadd_round_sch(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmadd_round_sch
+  // CHECK-LABEL: test_mm_maskz_fmadd_round_sch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfmadd.csh
   return _mm_maskz_fmadd_round_sch(__U, __A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_mask3_fmadd_round_sch(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmadd_round_sch
+  // CHECK-LABEL: test_mm_mask3_fmadd_round_sch
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
@@ -4192,286 +4215,286 @@ __m128h test_mm_mask3_fmadd_round_sch(__m128h __A, __m128h __B, __m128h __C, __m
 }
 
 __m128h test_mm_fcmul_sch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fcmul_sch
+  // CHECK-LABEL: test_mm_fcmul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_fcmul_sch(__A, __B);
 }
 
 __m128h test_mm_mask_fcmul_sch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fcmul_sch
+  // CHECK-LABEL: test_mm_mask_fcmul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_mask_fcmul_sch(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_fcmul_sch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_fcmul_sch
+  // CHECK-LABEL: test_mm_maskz_fcmul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_maskz_fcmul_sch(__U, __A, __B);
 }
 
 __m128h test_mm_fcmul_round_sch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fcmul_round_sch
+  // CHECK-LABEL: test_mm_fcmul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_fcmul_round_sch(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_mask_fcmul_round_sch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fcmul_round_sch
+  // CHECK-LABEL: test_mm_mask_fcmul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_mask_fcmul_round_sch(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_fcmul_round_sch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_fcmul_round_sch
+  // CHECK-LABEL: test_mm_maskz_fcmul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_maskz_fcmul_round_sch(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_fcmul_pch(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_fcmul_pch
+  // CHECK-LABEL: test_mm512_fcmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_fcmul_pch(__A, __B);
 }
 
 __m512h test_mm512_mask_fcmul_pch(__m512h __W, __mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_fcmul_pch
+  // CHECK-LABEL: test_mm512_mask_fcmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_mask_fcmul_pch(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_fcmul_pch(__mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_fcmul_pch
+  // CHECK-LABEL: test_mm512_maskz_fcmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_maskz_fcmul_pch(__U, __A, __B);
 }
 
 __m512h test_mm512_fcmul_round_pch(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_fcmul_round_pch
+  // CHECK-LABEL: test_mm512_fcmul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_fcmul_round_pch(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_fcmul_round_pch(__m512h __W, __mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_fcmul_round_pch
+  // CHECK-LABEL: test_mm512_mask_fcmul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_mask_fcmul_round_pch(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_fcmul_round_pch(__mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_fcmul_round_pch
+  // CHECK-LABEL: test_mm512_maskz_fcmul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_maskz_fcmul_round_pch(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_fcmadd_pch(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fcmadd_pch
+  // CHECK-LABEL: test_mm512_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.512
   return _mm512_fcmadd_pch(__A, __B, __C);
 }
 
 __m512h test_mm512_mask_fcmadd_pch(__m512h __A, __mmask16 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fcmadd_pch
+  // CHECK-LABEL: test_mm512_mask_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.512
   // CHECK:  %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask_fcmadd_pch(__A, __U, __B, __C);
 }
 
 __m512h test_mm512_mask3_fcmadd_pch(__m512h __A, __m512h __B, __m512h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fcmadd_pch
+  // CHECK-LABEL: test_mm512_mask3_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.512
   // CHECK-NOT:  %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask3_fcmadd_pch(__A, __B, __C, __U);
 }
 
 __m512h test_mm512_maskz_fcmadd_pch(__mmask16 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fcmadd_pch
+  // CHECK-LABEL: test_mm512_maskz_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfcmadd.cph.512
   return _mm512_maskz_fcmadd_pch(__U, __A, __B, __C);
 }
 
 __m512h test_mm512_fcmadd_round_pch(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fcmadd_round_pch
+  // CHECK-LABEL: test_mm512_fcmadd_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.512
   return _mm512_fcmadd_round_pch(__A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_fcmadd_round_pch(__m512h __A, __mmask16 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fcmadd_round_pch
+  // CHECK-LABEL: test_mm512_mask_fcmadd_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.512
   // CHECK:  %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask_fcmadd_round_pch(__A, __U, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask3_fcmadd_round_pch(__m512h __A, __m512h __B, __m512h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fcmadd_round_pch
+  // CHECK-LABEL: test_mm512_mask3_fcmadd_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.512
   // CHECK-NOT:  %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask3_fcmadd_round_pch(__A, __B, __C, __U, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_fcmadd_round_pch(__mmask16 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fcmadd_round_pch
+  // CHECK-LABEL: test_mm512_maskz_fcmadd_round_pch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfcmadd.cph.512
   return _mm512_maskz_fcmadd_round_pch(__U, __A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_fmul_pch(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_fmul_pch
+  // CHECK-LABEL: test_mm512_fmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_fmul_pch(__A, __B);
 }
 
 __m512h test_mm512_mask_fmul_pch(__m512h __W, __mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_fmul_pch
+  // CHECK-LABEL: test_mm512_mask_fmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_mask_fmul_pch(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_fmul_pch(__mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_fmul_pch
+  // CHECK-LABEL: test_mm512_maskz_fmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_maskz_fmul_pch(__U, __A, __B);
 }
 
 __m512h test_mm512_fmul_round_pch(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_fmul_round_pch
+  // CHECK-LABEL: test_mm512_fmul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_fmul_round_pch(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_fmul_round_pch(__m512h __W, __mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_fmul_round_pch
+  // CHECK-LABEL: test_mm512_mask_fmul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_mask_fmul_round_pch(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_fmul_round_pch(__mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_fmul_round_pch
+  // CHECK-LABEL: test_mm512_maskz_fmul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_maskz_fmul_round_pch(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_fmadd_pch(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmadd_pch
+  // CHECK-LABEL: test_mm512_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.512
   return _mm512_fmadd_pch(__A, __B, __C);
 }
 
 __m512h test_mm512_mask_fmadd_pch(__m512h __A, __mmask16 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmadd_pch
+  // CHECK-LABEL: test_mm512_mask_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.512
   // CHECK:  %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask_fmadd_pch(__A, __U, __B, __C);
 }
 
 __m512h test_mm512_mask3_fmadd_pch(__m512h __A, __m512h __B, __m512h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmadd_pch
+  // CHECK-LABEL: test_mm512_mask3_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.512
   // CHECK-NOT:  %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask3_fmadd_pch(__A, __B, __C, __U);
 }
 
 __m512h test_mm512_maskz_fmadd_pch(__mmask16 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmadd_pch
+  // CHECK-LABEL: test_mm512_maskz_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfmadd.cph.512
   return _mm512_maskz_fmadd_pch(__U, __A, __B, __C);
 }
 
 __m512h test_mm512_fmadd_round_pch(__m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_fmadd_round_pch
+  // CHECK-LABEL: test_mm512_fmadd_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.512
   return _mm512_fmadd_round_pch(__A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_fmadd_round_pch(__m512h __A, __mmask16 __U, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_mask_fmadd_round_pch
+  // CHECK-LABEL: test_mm512_mask_fmadd_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.512
   // CHECK:  %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask_fmadd_round_pch(__A, __U, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask3_fmadd_round_pch(__m512h __A, __m512h __B, __m512h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm512_mask3_fmadd_round_pch
+  // CHECK-LABEL: test_mm512_mask3_fmadd_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.512
   // CHECK-NOT:  %{{.*}} = select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}}
   return _mm512_mask3_fmadd_round_pch(__A, __B, __C, __U, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_fmadd_round_pch(__mmask16 __U, __m512h __A, __m512h __B, __m512h __C) {
-  // CHECK-LABEL: @test_mm512_maskz_fmadd_round_pch
+  // CHECK-LABEL: test_mm512_maskz_fmadd_round_pch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfmadd.cph.512
   return _mm512_maskz_fmadd_round_pch(__U, __A, __B, __C, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_fmul_sch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fmul_sch
+  // CHECK-LABEL: test_mm_fmul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_fmul_sch(__A, __B);
 }
 
 __m128h test_mm_mask_fmul_sch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fmul_sch
+  // CHECK-LABEL: test_mm_mask_fmul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_mask_fmul_sch(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_fmul_sch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_fmul_sch
+  // CHECK-LABEL: test_mm_maskz_fmul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_maskz_fmul_sch(__U, __A, __B);
 }
 
 __m128h test_mm_fmul_round_sch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fmul_round_sch
+  // CHECK-LABEL: test_mm_fmul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_fmul_round_sch(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_mask_fmul_round_sch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fmul_round_sch
+  // CHECK-LABEL: test_mm_mask_fmul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_mask_fmul_round_sch(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_fmul_round_sch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_fmul_round_sch
+  // CHECK-LABEL: test_mm_maskz_fmul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_maskz_fmul_round_sch(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 _Float16 test_mm512_reduce_add_ph(__m512h __W) {
-  // CHECK-LABEL: @test_mm512_reduce_add_ph
-  // CHECK: call reassoc half @llvm.vector.reduce.fadd.v32f16(half 0xH8000, <32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_reduce_add_ph
+  // CHECK: @llvm.vector.reduce.fadd.v32f16(half 0xH8000, <32 x half> %{{.*}})
   return _mm512_reduce_add_ph(__W);
 }
 
 _Float16 test_mm512_reduce_mul_ph(__m512h __W) {
-  // CHECK-LABEL: @test_mm512_reduce_mul_ph
-  // CHECK: call reassoc half @llvm.vector.reduce.fmul.v32f16(half 0xH3C00, <32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_reduce_mul_ph
+  // CHECK: @llvm.vector.reduce.fmul.v32f16(half 0xH3C00, <32 x half> %{{.*}})
   return _mm512_reduce_mul_ph(__W);
 }
 
 _Float16 test_mm512_reduce_max_ph(__m512h __W) {
-  // CHECK-LABEL: @test_mm512_reduce_max_ph
-  // CHECK: call nnan half @llvm.vector.reduce.fmax.v32f16(<32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_reduce_max_ph
+  // CHECK: @llvm.vector.reduce.fmax.v32f16(<32 x half> %{{.*}})
   return _mm512_reduce_max_ph(__W);
 }
 
 _Float16 test_mm512_reduce_min_ph(__m512h __W) {
-  // CHECK-LABEL: @test_mm512_reduce_min_ph
-  // CHECK: call nnan half @llvm.vector.reduce.fmin.v32f16(<32 x half> %{{.*}})
+  // CHECK-LABEL: test_mm512_reduce_min_ph
+  // CHECK: @llvm.vector.reduce.fmin.v32f16(<32 x half> %{{.*}})
   return _mm512_reduce_min_ph(__W);
 }
 
 __m512h test_mm512_mask_blend_ph(__mmask32 __U, __m512h __A, __m512h __W) {
-  // CHECK-LABEL: @test_mm512_mask_blend_ph
+  // CHECK-LABEL: test_mm512_mask_blend_ph
   // CHECK:  %{{.*}} = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK:  %{{.*}} = select <32 x i1> %{{.*}}, <32 x half> %{{.*}}, <32 x half> %{{.*}}
   return _mm512_mask_blend_ph(__U, __A, __W);
 }
 
 __m512h test_mm512_permutex2var_ph(__m512h __A, __m512i __I, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_permutex2var_ph
+  // CHECK-LABEL: test_mm512_permutex2var_ph
   // CHECK:  %{{.*}} = bitcast <32 x half> %{{.*}} to <32 x i16>
   // CHECK:  %{{.*}} = bitcast <8 x i64> %{{.*}} to <32 x i16>
   // CHECK:  %{{.*}} = bitcast <32 x half> %{{.*}} to <32 x i16>
@@ -4481,7 +4504,7 @@ __m512h test_mm512_permutex2var_ph(__m512h __A, __m512i __I, __m512h __B) {
 }
 
 __m512h test_mm512_permutexvar_epi16(__m512i __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_permutexvar_epi16
+  // CHECK-LABEL: test_mm512_permutexvar_epi16
   // CHECK:  %{{.*}} = bitcast <32 x half> %{{.*}} to <32 x i16>
   // CHECK:  %{{.*}} = bitcast <8 x i64> %{{.*}} to <32 x i16>
   // CHECK:  %{{.*}} = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %{{.*}}, <32 x i16> %{{.*}})
@@ -4491,144 +4514,144 @@ __m512h test_mm512_permutexvar_epi16(__m512i __A, __m512h __B) {
 
 // tests below are for alias intrinsics.
 __m512h test_mm512_mul_pch(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mul_pch
+  // CHECK-LABEL: test_mm512_mul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_mul_pch(__A, __B);
 }
 
 __m512h test_mm512_mask_mul_pch(__m512h __W, __mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_mul_pch
+  // CHECK-LABEL: test_mm512_mask_mul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_mask_mul_pch(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_mul_pch(__mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_mul_pch
+  // CHECK-LABEL: test_mm512_maskz_mul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_maskz_mul_pch(__U, __A, __B);
 }
 
 __m512h test_mm512_cmul_pch(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_cmul_pch
+  // CHECK-LABEL: test_mm512_cmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_cmul_pch(__A, __B);
 }
 __m512h test_mm512_mask_cmul_pch(__m512h __W, __mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_cmul_pch
+  // CHECK-LABEL: test_mm512_mask_cmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_mask_cmul_pch(__W, __U, __A, __B);
 }
 
 __m512h test_mm512_maskz_cmul_pch(__mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_cmul_pch
+  // CHECK-LABEL: test_mm512_maskz_cmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_maskz_cmul_pch(__U, __A, __B);
 }
 
 __m128h test_mm_mul_sch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mul_sch
+  // CHECK-LABEL: test_mm_mul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_mul_sch(__A, __B);
 }
 
 __m128h test_mm_mask_mul_sch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_mul_sch
+  // CHECK-LABEL: test_mm_mask_mul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_mask_mul_sch(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_mul_sch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_mul_sch
+  // CHECK-LABEL: test_mm_maskz_mul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_maskz_mul_sch(__U, __A, __B);
 }
 
 __m128h test_mm_mul_round_sch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mul_round_sch
+  // CHECK-LABEL: test_mm_mul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_mul_round_sch(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_mask_mul_round_sch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_mul_round_sch
+  // CHECK-LABEL: test_mm_mask_mul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_mask_mul_round_sch(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_mul_round_sch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_mul_round_sch
+  // CHECK-LABEL: test_mm_maskz_mul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.csh
   return _mm_maskz_mul_round_sch(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mul_round_pch(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mul_round_pch
+  // CHECK-LABEL: test_mm512_mul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_mul_round_pch(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_mul_round_pch(__m512h __W, __mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_mul_round_pch
+  // CHECK-LABEL: test_mm512_mask_mul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_mask_mul_round_pch(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_mul_round_pch(__mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_mul_round_pch
+  // CHECK-LABEL: test_mm512_maskz_mul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.512
   return _mm512_maskz_mul_round_pch(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_cmul_round_pch(__m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_cmul_round_pch
+  // CHECK-LABEL: test_mm512_cmul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_cmul_round_pch(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_mask_cmul_round_pch(__m512h __W, __mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_mask_cmul_round_pch
+  // CHECK-LABEL: test_mm512_mask_cmul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_mask_cmul_round_pch(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m512h test_mm512_maskz_cmul_round_pch(__mmask16 __U, __m512h __A, __m512h __B) {
-  // CHECK-LABEL: @test_mm512_maskz_cmul_round_pch
+  // CHECK-LABEL: test_mm512_maskz_cmul_round_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.512
   return _mm512_maskz_cmul_round_pch(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_cmul_sch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_cmul_sch
+  // CHECK-LABEL: test_mm_cmul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_cmul_sch(__A, __B);
 }
 
 __m128h test_mm_mask_cmul_sch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_cmul_sch
+  // CHECK-LABEL: test_mm_mask_cmul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_mask_cmul_sch(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_cmul_sch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_cmul_sch
+  // CHECK-LABEL: test_mm_maskz_cmul_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_maskz_cmul_sch(__U, __A, __B);
 }
 
 __m128h test_mm_cmul_round_sch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_cmul_round_sch
+  // CHECK-LABEL: test_mm_cmul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_cmul_round_sch(__A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_mask_cmul_round_sch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_cmul_round_sch
+  // CHECK-LABEL: test_mm_mask_cmul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_mask_cmul_round_sch(__W, __U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128h test_mm_maskz_cmul_round_sch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_cmul_round_sch
+  // CHECK-LABEL: test_mm_maskz_cmul_round_sch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.csh
   return _mm_maskz_cmul_round_sch(__U, __A, __B, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
diff --git a/clang/test/CodeGen/X86/avx512ifma-builtins.c b/clang/test/CodeGen/X86/avx512ifma-builtins.c
index e2077b5b59124..7c7c492c79c99 100644
--- a/clang/test/CodeGen/X86/avx512ifma-builtins.c
+++ b/clang/test/CodeGen/X86/avx512ifma-builtins.c
@@ -1,44 +1,47 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512ifma -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512ifma -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512ifma -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512ifma -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512ifma -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
 
 __m512i test_mm512_madd52hi_epu64(__m512i __X, __m512i __Y, __m512i __Z) {
-  // CHECK-LABEL: @test_mm512_madd52hi_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52h.uq.512
+  // CHECK-LABEL: test_mm512_madd52hi_epu64
+  // CHECK: call {{.*}}<8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   return _mm512_madd52hi_epu64(__X, __Y, __Z); 
 }
 
 __m512i test_mm512_mask_madd52hi_epu64(__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) {
-  // CHECK-LABEL: @test_mm512_mask_madd52hi_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52h.uq.512
+  // CHECK-LABEL: test_mm512_mask_madd52hi_epu64
+  // CHECK: call {{.*}}<8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_madd52hi_epu64(__W, __M, __X, __Y); 
 }
 
 __m512i test_mm512_maskz_madd52hi_epu64(__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) {
-  // CHECK-LABEL: @test_mm512_maskz_madd52hi_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52h.uq.512
+  // CHECK-LABEL: test_mm512_maskz_madd52hi_epu64
+  // CHECK: call {{.*}}<8 x i64> @llvm.x86.avx512.vpmadd52h.uq.512(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_madd52hi_epu64(__M, __X, __Y, __Z); 
 }
 
 __m512i test_mm512_madd52lo_epu64(__m512i __X, __m512i __Y, __m512i __Z) {
-  // CHECK-LABEL: @test_mm512_madd52lo_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52l.uq.512
+  // CHECK-LABEL: test_mm512_madd52lo_epu64
+  // CHECK: call {{.*}}<8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   return _mm512_madd52lo_epu64(__X, __Y, __Z); 
 }
 
 __m512i test_mm512_mask_madd52lo_epu64(__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) {
-  // CHECK-LABEL: @test_mm512_mask_madd52lo_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52l.uq.512
+  // CHECK-LABEL: test_mm512_mask_madd52lo_epu64
+  // CHECK: call {{.*}}<8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_madd52lo_epu64(__W, __M, __X, __Y); 
 }
 
 __m512i test_mm512_maskz_madd52lo_epu64(__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) {
-  // CHECK-LABEL: @test_mm512_maskz_madd52lo_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52l.uq.512
+  // CHECK-LABEL: test_mm512_maskz_madd52lo_epu64
+  // CHECK: call {{.*}}<8 x i64> @llvm.x86.avx512.vpmadd52l.uq.512(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_madd52lo_epu64(__M, __X, __Y, __Z); 
 }
diff --git a/clang/test/CodeGen/X86/avx512ifmavl-builtins.c b/clang/test/CodeGen/X86/avx512ifmavl-builtins.c
index 3b744787f2abc..c115b60381383 100644
--- a/clang/test/CodeGen/X86/avx512ifmavl-builtins.c
+++ b/clang/test/CodeGen/X86/avx512ifmavl-builtins.c
@@ -1,83 +1,86 @@
-// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-apple-darwin -target-feature +avx512ifma -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-apple-darwin -target-feature +avx512ifma -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -flax-vector-conversions=none -ffreestanding -triple=i386-apple-darwin -target-feature +avx512ifma -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-apple-darwin -target-feature +avx512ifma -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -flax-vector-conversions=none -ffreestanding -triple=i386-apple-darwin -target-feature +avx512ifma -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 __m128i test_mm_madd52hi_epu64(__m128i __X, __m128i __Y, __m128i __Z) {
-  // CHECK-LABEL: @test_mm_madd52hi_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52h.uq.128
+  // CHECK-LABEL: test_mm_madd52hi_epu64
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_madd52hi_epu64(__X, __Y, __Z); 
 }
 
 __m128i test_mm_mask_madd52hi_epu64(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_madd52hi_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52h.uq.128
+  // CHECK-LABEL: test_mm_mask_madd52hi_epu64
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_madd52hi_epu64(__W, __M, __X, __Y); 
 }
 
 __m128i test_mm_maskz_madd52hi_epu64(__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) {
-  // CHECK-LABEL: @test_mm_maskz_madd52hi_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52h.uq.128
+  // CHECK-LABEL: test_mm_maskz_madd52hi_epu64
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_madd52hi_epu64(__M, __X, __Y, __Z); 
 }
 
 __m256i test_mm256_madd52hi_epu64(__m256i __X, __m256i __Y, __m256i __Z) {
-  // CHECK-LABEL: @test_mm256_madd52hi_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52h.uq.256
+  // CHECK-LABEL: test_mm256_madd52hi_epu64
+  // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_madd52hi_epu64(__X, __Y, __Z); 
 }
 
 __m256i test_mm256_mask_madd52hi_epu64(__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_madd52hi_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52h.uq.256
+  // CHECK-LABEL: test_mm256_mask_madd52hi_epu64
+  // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_madd52hi_epu64(__W, __M, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_madd52hi_epu64(__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) {
-  // CHECK-LABEL: @test_mm256_maskz_madd52hi_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52h.uq.256
+  // CHECK-LABEL: test_mm256_maskz_madd52hi_epu64
+  // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_madd52hi_epu64(__M, __X, __Y, __Z); 
 }
 
 __m128i test_mm_madd52lo_epu64(__m128i __X, __m128i __Y, __m128i __Z) {
-  // CHECK-LABEL: @test_mm_madd52lo_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52l.uq.128
+  // CHECK-LABEL: test_mm_madd52lo_epu64
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_madd52lo_epu64(__X, __Y, __Z); 
 }
 
 __m128i test_mm_mask_madd52lo_epu64(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_madd52lo_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52l.uq.128
+  // CHECK-LABEL: test_mm_mask_madd52lo_epu64
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_madd52lo_epu64(__W, __M, __X, __Y); 
 }
 
 __m128i test_mm_maskz_madd52lo_epu64(__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) {
-  // CHECK-LABEL: @test_mm_maskz_madd52lo_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52l.uq.128
+  // CHECK-LABEL: test_mm_maskz_madd52lo_epu64
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_madd52lo_epu64(__M, __X, __Y, __Z); 
 }
 
 __m256i test_mm256_madd52lo_epu64(__m256i __X, __m256i __Y, __m256i __Z) {
-  // CHECK-LABEL: @test_mm256_madd52lo_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52l.uq.256
+  // CHECK-LABEL: test_mm256_madd52lo_epu64
+  // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_madd52lo_epu64(__X, __Y, __Z); 
 }
 
 __m256i test_mm256_mask_madd52lo_epu64(__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_madd52lo_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52l.uq.256
+  // CHECK-LABEL: test_mm256_mask_madd52lo_epu64
+  // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_madd52lo_epu64(__W, __M, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_madd52lo_epu64(__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) {
-  // CHECK-LABEL: @test_mm256_maskz_madd52lo_epu64
-  // CHECK: @llvm.x86.avx512.vpmadd52l.uq.256
+  // CHECK-LABEL: test_mm256_maskz_madd52lo_epu64
+  // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_madd52lo_epu64(__M, __X, __Y, __Z); 
 }
diff --git a/clang/test/CodeGen/X86/avx512vbmi-builtins.c b/clang/test/CodeGen/X86/avx512vbmi-builtins.c
index aefc39fd2bae4..c3b6298a39b59 100644
--- a/clang/test/CodeGen/X86/avx512vbmi-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vbmi-builtins.c
@@ -1,71 +1,74 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vbmi -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vbmi -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
 
 __m512i test_mm512_mask2_permutex2var_epi8(__m512i __A, __m512i __I, __mmask64 __U, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask2_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.512
+  // CHECK-LABEL: test_mm512_mask2_permutex2var_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask2_permutex2var_epi8(__A, __I, __U, __B); 
 }
 
 __m512i test_mm512_permutex2var_epi8(__m512i __A, __m512i __I, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.512
+  // CHECK-LABEL: test_mm512_permutex2var_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_permutex2var_epi8(__A, __I, __B); 
 }
 
 __m512i test_mm512_mask_permutex2var_epi8(__m512i __A, __mmask64 __U, __m512i __I, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.512
+  // CHECK-LABEL: test_mm512_mask_permutex2var_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_permutex2var_epi8(__A, __U, __I, __B); 
 }
 
 __m512i test_mm512_maskz_permutex2var_epi8(__mmask64 __U, __m512i __A, __m512i __I, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.512
+  // CHECK-LABEL: test_mm512_maskz_permutex2var_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_permutex2var_epi8(__U, __A, __I, __B); 
 }
 
 __m512i test_mm512_permutexvar_epi8(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_permutexvar_epi8
-  // CHECK: @llvm.x86.avx512.permvar.qi.512
+  // CHECK-LABEL: test_mm512_permutexvar_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_permutexvar_epi8(__A, __B); 
 }
 
 __m512i test_mm512_maskz_permutexvar_epi8(__mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_permutexvar_epi8
-  // CHECK: @llvm.x86.avx512.permvar.qi.512
+  // CHECK-LABEL: test_mm512_maskz_permutexvar_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_permutexvar_epi8(__M, __A, __B); 
 }
 
 __m512i test_mm512_mask_permutexvar_epi8(__m512i __W, __mmask64 __M, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_permutexvar_epi8
-  // CHECK: @llvm.x86.avx512.permvar.qi.512
+  // CHECK-LABEL: test_mm512_mask_permutexvar_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_permutexvar_epi8(__W, __M, __A, __B); 
 }
 
 __m512i test_mm512_mask_multishift_epi64_epi8(__m512i __W, __mmask64 __M, __m512i __X, __m512i __Y) {
-  // CHECK-LABEL: @test_mm512_mask_multishift_epi64_epi8
-  // CHECK: @llvm.x86.avx512.pmultishift.qb.512
+  // CHECK-LABEL: test_mm512_mask_multishift_epi64_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_mask_multishift_epi64_epi8(__W, __M, __X, __Y); 
 }
 
 __m512i test_mm512_maskz_multishift_epi64_epi8(__mmask64 __M, __m512i __X, __m512i __Y) {
-  // CHECK-LABEL: @test_mm512_maskz_multishift_epi64_epi8
-  // CHECK: @llvm.x86.avx512.pmultishift.qb.512
+  // CHECK-LABEL: test_mm512_maskz_multishift_epi64_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   // CHECK: select <64 x i1> %{{.*}}, <64 x i8> %{{.*}}, <64 x i8> %{{.*}}
   return _mm512_maskz_multishift_epi64_epi8(__M, __X, __Y); 
 }
 
 __m512i test_mm512_multishift_epi64_epi8(__m512i __X, __m512i __Y) {
-  // CHECK-LABEL: @test_mm512_multishift_epi64_epi8
-  // CHECK: @llvm.x86.avx512.pmultishift.qb.512
+  // CHECK-LABEL: test_mm512_multishift_epi64_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_multishift_epi64_epi8(__X, __Y); 
 }
diff --git a/clang/test/CodeGen/X86/avx512vbmi2-builtins.c b/clang/test/CodeGen/X86/avx512vbmi2-builtins.c
index 1e804c3db57d6..4f6139b81960d 100644
--- a/clang/test/CodeGen/X86/avx512vbmi2-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vbmi2-builtins.c
@@ -1,328 +1,331 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 __m512i test_mm512_mask_compress_epi16(__m512i __S, __mmask32 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_mask_compress_epi16
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm512_mask_compress_epi16
+  // CHECK: call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i1> %{{.*}})
   return _mm512_mask_compress_epi16(__S, __U, __D);
 }
 
 __m512i test_mm512_maskz_compress_epi16(__mmask32 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_maskz_compress_epi16
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm512_maskz_compress_epi16
+  // CHECK: call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i1> %{{.*}})
   return _mm512_maskz_compress_epi16(__U, __D);
 }
 
 __m512i test_mm512_mask_compress_epi8(__m512i __S, __mmask64 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_mask_compress_epi8
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm512_mask_compress_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i1> %{{.*}})
   return _mm512_mask_compress_epi8(__S, __U, __D);
 }
 
 __m512i test_mm512_maskz_compress_epi8(__mmask64 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_maskz_compress_epi8
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm512_maskz_compress_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i1> %{{.*}})
   return _mm512_maskz_compress_epi8(__U, __D);
 }
 
 void test_mm512_mask_compressstoreu_epi16(void *__P, __mmask32 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_mask_compressstoreu_epi16
-  // CHECK: @llvm.masked.compressstore.v32i16(<32 x i16> %{{.*}}, ptr %{{.*}}, <32 x i1> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_compressstoreu_epi16
+  // CHECK: call void @llvm.masked.compressstore.v32i16(<32 x i16> %{{.*}}, ptr %{{.*}}, <32 x i1> %{{.*}})
   _mm512_mask_compressstoreu_epi16(__P, __U, __D);
 }
 
 void test_mm512_mask_compressstoreu_epi8(void *__P, __mmask64 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_mask_compressstoreu_epi8
-  // CHECK: @llvm.masked.compressstore.v64i8(<64 x i8> %{{.*}}, ptr %{{.*}}, <64 x i1> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_compressstoreu_epi8
+  // CHECK: call void @llvm.masked.compressstore.v64i8(<64 x i8> %{{.*}}, ptr %{{.*}}, <64 x i1> %{{.*}})
   _mm512_mask_compressstoreu_epi8(__P, __U, __D);
 }
 
 __m512i test_mm512_mask_expand_epi16(__m512i __S, __mmask32 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_mask_expand_epi16
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm512_mask_expand_epi16
+  // CHECK: call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i1> %{{.*}})
   return _mm512_mask_expand_epi16(__S, __U, __D);
 }
 
 __m512i test_mm512_maskz_expand_epi16(__mmask32 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_maskz_expand_epi16
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm512_maskz_expand_epi16
+  // CHECK: call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i1> %{{.*}})
   return _mm512_maskz_expand_epi16(__U, __D);
 }
 
 __m512i test_mm512_mask_expand_epi8(__m512i __S, __mmask64 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_mask_expand_epi8
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm512_mask_expand_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i1> %{{.*}})
   return _mm512_mask_expand_epi8(__S, __U, __D);
 }
 
 __m512i test_mm512_maskz_expand_epi8(__mmask64 __U, __m512i __D) {
-  // CHECK-LABEL: @test_mm512_maskz_expand_epi8
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm512_maskz_expand_epi8
+  // CHECK: call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %{{.*}}, <64 x i8> %{{.*}}, <64 x i1> %{{.*}})
   return _mm512_maskz_expand_epi8(__U, __D);
 }
 
 __m512i test_mm512_mask_expandloadu_epi16(__m512i __S, __mmask32 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm512_mask_expandloadu_epi16
-  // CHECK: @llvm.masked.expandload.v32i16(ptr %{{.*}}, <32 x i1> %{{.*}}, <32 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_expandloadu_epi16
+  // CHECK: call <32 x i16> @llvm.masked.expandload.v32i16(ptr %{{.*}}, <32 x i1> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_mask_expandloadu_epi16(__S, __U, __P);
 }
 
 __m512i test_mm512_maskz_expandloadu_epi16(__mmask32 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm512_maskz_expandloadu_epi16
-  // CHECK: @llvm.masked.expandload.v32i16(ptr %{{.*}}, <32 x i1> %{{.*}}, <32 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_expandloadu_epi16
+  // CHECK: call <32 x i16> @llvm.masked.expandload.v32i16(ptr %{{.*}}, <32 x i1> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_maskz_expandloadu_epi16(__U, __P);
 }
 
 __m512i test_mm512_mask_expandloadu_epi8(__m512i __S, __mmask64 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm512_mask_expandloadu_epi8
-  // CHECK: @llvm.masked.expandload.v64i8(ptr %{{.*}}, <64 x i1> %{{.*}}, <64 x i8> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_expandloadu_epi8
+  // CHECK: call <64 x i8> @llvm.masked.expandload.v64i8(ptr %{{.*}}, <64 x i1> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_mask_expandloadu_epi8(__S, __U, __P);
 }
 
 __m512i test_mm512_maskz_expandloadu_epi8(__mmask64 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm512_maskz_expandloadu_epi8
-  // CHECK: @llvm.masked.expandload.v64i8(ptr %{{.*}}, <64 x i1> %{{.*}}, <64 x i8> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_expandloadu_epi8
+  // CHECK: call <64 x i8> @llvm.masked.expandload.v64i8(ptr %{{.*}}, <64 x i1> %{{.*}}, <64 x i8> %{{.*}})
   return _mm512_maskz_expandloadu_epi8(__U, __P);
 }
 
 __m512i test_mm512_mask_shldi_epi64(__m512i __S, __mmask8 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shldi_epi64
-  // CHECK: @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 47))
+  // CHECK-LABEL: test_mm512_mask_shldi_epi64
+  // CHECK: call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 47))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_shldi_epi64(__S, __U, __A, __B, 47);
 }
 
 __m512i test_mm512_maskz_shldi_epi64(__mmask8 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shldi_epi64
-  // CHECK: @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 63))
+  // CHECK-LABEL: test_mm512_maskz_shldi_epi64
+  // CHECK: call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 63))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_shldi_epi64(__U, __A, __B, 63);
 }
 
 __m512i test_mm512_shldi_epi64(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shldi_epi64
-  // CHECK: @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 31))
+  // CHECK-LABEL: test_mm512_shldi_epi64
+  // CHECK: call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 31))
   return _mm512_shldi_epi64(__A, __B, 31);
 }
 
 __m512i test_mm512_mask_shldi_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shldi_epi32
-  // CHECK: @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 7))
+  // CHECK-LABEL: test_mm512_mask_shldi_epi32
+  // CHECK: call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 7))
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_shldi_epi32(__S, __U, __A, __B, 7);
 }
 
 __m512i test_mm512_maskz_shldi_epi32(__mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shldi_epi32
-  // CHECK: @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 15))
+  // CHECK-LABEL: test_mm512_maskz_shldi_epi32
+  // CHECK: call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 15))
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_shldi_epi32(__U, __A, __B, 15);
 }
 
 __m512i test_mm512_shldi_epi32(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shldi_epi32
-  // CHECK: @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 31))
+  // CHECK-LABEL: test_mm512_shldi_epi32
+  // CHECK: call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 31))
   return _mm512_shldi_epi32(__A, __B, 31);
 }
 
 __m512i test_mm512_mask_shldi_epi16(__m512i __S, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shldi_epi16
-  // CHECK: @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 3))
+  // CHECK-LABEL: test_mm512_mask_shldi_epi16
+  // CHECK: call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 3))
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_shldi_epi16(__S, __U, __A, __B, 3);
 }
 
 __m512i test_mm512_maskz_shldi_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shldi_epi16
-  // CHECK: @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 7))
+  // CHECK-LABEL: test_mm512_maskz_shldi_epi16
+  // CHECK: call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 7))
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_shldi_epi16(__U, __A, __B, 7);
 }
 
 __m512i test_mm512_shldi_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shldi_epi16
-  // CHECK: @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 15))
+  // CHECK-LABEL: test_mm512_shldi_epi16
+  // CHECK: call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 15))
   return _mm512_shldi_epi16(__A, __B, 15);
 }
 
 __m512i test_mm512_mask_shrdi_epi64(__m512i __S, __mmask8 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shrdi_epi64
-  // CHECK: @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 47))
+  // CHECK-LABEL: test_mm512_mask_shrdi_epi64
+  // CHECK: call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 47))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_shrdi_epi64(__S, __U, __A, __B, 47);
 }
 
 __m512i test_mm512_maskz_shrdi_epi64(__mmask8 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shrdi_epi64
-  // CHECK: @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 63))
+  // CHECK-LABEL: test_mm512_maskz_shrdi_epi64
+  // CHECK: call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 63))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_shrdi_epi64(__U, __A, __B, 63);
 }
 
 __m512i test_mm512_shrdi_epi64(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shrdi_epi64
-  // CHECK: @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 31))
+  // CHECK-LABEL: test_mm512_shrdi_epi64
+  // CHECK: call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> splat (i64 31))
   return _mm512_shrdi_epi64(__A, __B, 31);
 }
 
 __m512i test_mm512_mask_shrdi_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shrdi_epi32
-  // CHECK: @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 7))
+  // CHECK-LABEL: test_mm512_mask_shrdi_epi32
+  // CHECK: call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 7))
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_shrdi_epi32(__S, __U, __A, __B, 7);
 }
 
 __m512i test_mm512_maskz_shrdi_epi32(__mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shrdi_epi32
-  // CHECK: @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 15))
+  // CHECK-LABEL: test_mm512_maskz_shrdi_epi32
+  // CHECK: call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 15))
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_shrdi_epi32(__U, __A, __B, 15);
 }
 
 __m512i test_mm512_shrdi_epi32(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shrdi_epi32
-  // CHECK: @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 31))
+  // CHECK-LABEL: test_mm512_shrdi_epi32
+  // CHECK: call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> splat (i32 31))
   return _mm512_shrdi_epi32(__A, __B, 31);
 }
 
 __m512i test_mm512_mask_shrdi_epi16(__m512i __S, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shrdi_epi16
-  // CHECK: @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 3))
+  // CHECK-LABEL: test_mm512_mask_shrdi_epi16
+  // CHECK: call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 3))
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_shrdi_epi16(__S, __U, __A, __B, 3);
 }
 
 __m512i test_mm512_maskz_shrdi_epi16(__mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shrdi_epi16
-  // CHECK: @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 15))
+  // CHECK-LABEL: test_mm512_maskz_shrdi_epi16
+  // CHECK: call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 15))
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_shrdi_epi16(__U, __A, __B, 15);
 }
 
 __m512i test_mm512_shrdi_epi16(__m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shrdi_epi16
-  // CHECK: @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 31))
+  // CHECK-LABEL: test_mm512_shrdi_epi16
+  // CHECK: call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> splat (i16 31))
   return _mm512_shrdi_epi16(__A, __B, 31);
 }
 
 __m512i test_mm512_mask_shldv_epi64(__m512i __S, __mmask8 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shldv_epi64
-  // CHECK: @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_shldv_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_shldv_epi64(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_shldv_epi64(__mmask8 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shldv_epi64
-  // CHECK: @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_shldv_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_shldv_epi64(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_shldv_epi64(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shldv_epi64
-  // CHECK: @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm512_shldv_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.fshl.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   return _mm512_shldv_epi64(__S, __A, __B);
 }
 
 __m512i test_mm512_mask_shldv_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shldv_epi32
-  // CHECK: @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_shldv_epi32
+  // CHECK: call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_shldv_epi32(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_shldv_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shldv_epi32
-  // CHECK: @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_shldv_epi32
+  // CHECK: call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_shldv_epi32(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_shldv_epi32(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shldv_epi32
-  // CHECK: @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm512_shldv_epi32
+  // CHECK: call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   return _mm512_shldv_epi32(__S, __A, __B);
 }
 
 __m512i test_mm512_mask_shldv_epi16(__m512i __S, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shldv_epi16
-  // CHECK: @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_shldv_epi16
+  // CHECK: call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_shldv_epi16(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_shldv_epi16(__mmask32 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shldv_epi16
-  // CHECK: @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_shldv_epi16
+  // CHECK: call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_shldv_epi16(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_shldv_epi16(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shldv_epi16
-  // CHECK: @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm512_shldv_epi16
+  // CHECK: call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_shldv_epi16(__S, __A, __B);
 }
 
 __m512i test_mm512_mask_shrdv_epi64(__m512i __S, __mmask8 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shrdv_epi64
-  // CHECK: @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_shrdv_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_shrdv_epi64(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_shrdv_epi64(__mmask8 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shrdv_epi64
-  // CHECK: @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_shrdv_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_shrdv_epi64(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_shrdv_epi64(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shrdv_epi64
-  // CHECK: @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm512_shrdv_epi64
+  // CHECK: call {{.*}}<8 x i64> @llvm.fshr.v8i64(<8 x i64> %{{.*}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}})
   return _mm512_shrdv_epi64(__S, __A, __B);
 }
 
 __m512i test_mm512_mask_shrdv_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shrdv_epi32
-  // CHECK: @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_shrdv_epi32
+  // CHECK: call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_shrdv_epi32(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_shrdv_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shrdv_epi32
-  // CHECK: @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_shrdv_epi32
+  // CHECK: call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_shrdv_epi32(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_shrdv_epi32(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shrdv_epi32
-  // CHECK: @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm512_shrdv_epi32
+  // CHECK: call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   return _mm512_shrdv_epi32(__S, __A, __B);
 }
 
 __m512i test_mm512_mask_shrdv_epi16(__m512i __S, __mmask32 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_shrdv_epi16
-  // CHECK: @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm512_mask_shrdv_epi16
+  // CHECK: call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_mask_shrdv_epi16(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_shrdv_epi16(__mmask32 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_shrdv_epi16
-  // CHECK: @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm512_maskz_shrdv_epi16
+  // CHECK: call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}}
   return _mm512_maskz_shrdv_epi16(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_shrdv_epi16(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_shrdv_epi16
-  // CHECK: @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm512_shrdv_epi16
+  // CHECK: call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}})
   return _mm512_shrdv_epi16(__S, __A, __B);
 }
 
diff --git a/clang/test/CodeGen/X86/avx512vbmivl-builtin.c b/clang/test/CodeGen/X86/avx512vbmivl-builtin.c
index 2562da26d6ce7..c4d5fc8fb6977 100644
--- a/clang/test/CodeGen/X86/avx512vbmivl-builtin.c
+++ b/clang/test/CodeGen/X86/avx512vbmivl-builtin.c
@@ -1,139 +1,142 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi -target-feature +avx512vl -target-feature +avx512bw -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi -target-feature +avx512vl -target-feature +avx512bw -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vbmi -target-feature +avx512vl -target-feature +avx512bw -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vbmi -target-feature +avx512vl -target-feature +avx512bw -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vbmi -target-feature +avx512vl -target-feature +avx512bw -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
 
 __m128i test_mm_permutexvar_epi8(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_permutexvar_epi8
-  // CHECK: @llvm.x86.avx512.permvar.qi.128
+  // CHECK-LABEL: test_mm_permutexvar_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_permutexvar_epi8(__A, __B); 
 }
 
 __m128i test_mm_maskz_permutexvar_epi8(__mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_permutexvar_epi8
-  // CHECK: @llvm.x86.avx512.permvar.qi.128
+  // CHECK-LABEL: test_mm_maskz_permutexvar_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_permutexvar_epi8(__M, __A, __B); 
 }
 
 __m128i test_mm_mask_permutexvar_epi8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_permutexvar_epi8
-  // CHECK: @llvm.x86.avx512.permvar.qi.128
+  // CHECK-LABEL: test_mm_mask_permutexvar_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_permutexvar_epi8(__W, __M, __A, __B); 
 }
 
 __m256i test_mm256_permutexvar_epi8(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_permutexvar_epi8
-  // CHECK: @llvm.x86.avx512.permvar.qi.256
+  // CHECK-LABEL: test_mm256_permutexvar_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_permutexvar_epi8(__A, __B); 
 }
 
 __m256i test_mm256_maskz_permutexvar_epi8(__mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_permutexvar_epi8
-  // CHECK: @llvm.x86.avx512.permvar.qi.256
+  // CHECK-LABEL: test_mm256_maskz_permutexvar_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_permutexvar_epi8(__M, __A, __B); 
 }
 
 __m256i test_mm256_mask_permutexvar_epi8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_permutexvar_epi8
-  // CHECK: @llvm.x86.avx512.permvar.qi.256
+  // CHECK-LABEL: test_mm256_mask_permutexvar_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_permutexvar_epi8(__W, __M, __A, __B); 
 }
 
 __m128i test_mm_mask2_permutex2var_epi8(__m128i __A, __m128i __I, __mmask16 __U, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask2_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.128
+  // CHECK-LABEL: test_mm_mask2_permutex2var_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask2_permutex2var_epi8(__A, __I, __U, __B); 
 }
 
 __m256i test_mm256_mask2_permutex2var_epi8(__m256i __A, __m256i __I, __mmask32 __U, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask2_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.256
+  // CHECK-LABEL: test_mm256_mask2_permutex2var_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask2_permutex2var_epi8(__A, __I, __U, __B); 
 }
 
 __m128i test_mm_permutex2var_epi8(__m128i __A, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.128
+  // CHECK-LABEL: test_mm_permutex2var_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_permutex2var_epi8(__A, __I, __B); 
 }
 
 __m128i test_mm_mask_permutex2var_epi8(__m128i __A, __mmask16 __U, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.128
+  // CHECK-LABEL: test_mm_mask_permutex2var_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_permutex2var_epi8(__A, __U, __I, __B); 
 }
 
 __m128i test_mm_maskz_permutex2var_epi8(__mmask16 __U, __m128i __A, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.128
+  // CHECK-LABEL: test_mm_maskz_permutex2var_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_permutex2var_epi8(__U, __A, __I, __B); 
 }
 
 __m256i test_mm256_permutex2var_epi8(__m256i __A, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.256
+  // CHECK-LABEL: test_mm256_permutex2var_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_permutex2var_epi8(__A, __I, __B); 
 }
 
 __m256i test_mm256_mask_permutex2var_epi8(__m256i __A, __mmask32 __U, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.256
+  // CHECK-LABEL: test_mm256_mask_permutex2var_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_permutex2var_epi8(__A, __U, __I, __B); 
 }
 
 __m256i test_mm256_maskz_permutex2var_epi8(__mmask32 __U, __m256i __A, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_permutex2var_epi8
-  // CHECK: @llvm.x86.avx512.vpermi2var.qi.256
+  // CHECK-LABEL: test_mm256_maskz_permutex2var_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_permutex2var_epi8(__U, __A, __I, __B); 
 }
 
 __m128i test_mm_mask_multishift_epi64_epi8(__m128i __W, __mmask16 __M, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_multishift_epi64_epi8
-  // CHECK: @llvm.x86.avx512.pmultishift.qb.128
+  // CHECK-LABEL: test_mm_mask_multishift_epi64_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_multishift_epi64_epi8(__W, __M, __X, __Y); 
 }
 
 __m128i test_mm_maskz_multishift_epi64_epi8(__mmask16 __M, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_maskz_multishift_epi64_epi8
-  // CHECK: @llvm.x86.avx512.pmultishift.qb.128
+  // CHECK-LABEL: test_mm_maskz_multishift_epi64_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_multishift_epi64_epi8(__M, __X, __Y); 
 }
 
 __m128i test_mm_multishift_epi64_epi8(__m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_multishift_epi64_epi8
-  // CHECK: @llvm.x86.avx512.pmultishift.qb.128
+  // CHECK-LABEL: test_mm_multishift_epi64_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_multishift_epi64_epi8(__X, __Y); 
 }
 
 __m256i test_mm256_mask_multishift_epi64_epi8(__m256i __W, __mmask32 __M, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_multishift_epi64_epi8
-  // CHECK: @llvm.x86.avx512.pmultishift.qb.256
+  // CHECK-LABEL: test_mm256_mask_multishift_epi64_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_multishift_epi64_epi8(__W, __M, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_multishift_epi64_epi8(__mmask32 __M, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_multishift_epi64_epi8
-  // CHECK: @llvm.x86.avx512.pmultishift.qb.256
+  // CHECK-LABEL: test_mm256_maskz_multishift_epi64_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_multishift_epi64_epi8(__M, __X, __Y); 
 }
 
 __m256i test_mm256_multishift_epi64_epi8(__m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_multishift_epi64_epi8
-  // CHECK: @llvm.x86.avx512.pmultishift.qb.256
+  // CHECK-LABEL: test_mm256_multishift_epi64_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_multishift_epi64_epi8(__X, __Y); 
 }
 
diff --git a/clang/test/CodeGen/X86/avx512vl-builtins.c b/clang/test/CodeGen/X86/avx512vl-builtins.c
index 1c2d467a47428..a1c267dd51bfa 100644
--- a/clang/test/CodeGen/X86/avx512vl-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vl-builtins.c
@@ -1,602 +1,604 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512f -target-feature +avx512vl -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512f -target-feature +avx512vl -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512f -target-feature +avx512vl -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __mmask8 test_mm_cmpeq_epu32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpeq_epu32_mask
+  // CHECK-LABEL: test_mm_cmpeq_epu32_mask
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: shufflevector <4 x i1> %{{.*}}, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   return (__mmask8)_mm_cmpeq_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpeq_epu32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpeq_epu32_mask
+  // CHECK-LABEL: test_mm_mask_cmpeq_epu32_mask
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpeq_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpeq_epu64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpeq_epu64_mask
+  // CHECK-LABEL: test_mm_cmpeq_epu64_mask
   // CHECK: icmp eq <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: shufflevector <2 x i1> %{{.*}}, <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
   return (__mmask8)_mm_cmpeq_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpeq_epu64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpeq_epu64_mask
+  // CHECK-LABEL: test_mm_mask_cmpeq_epu64_mask
   // CHECK: icmp eq <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpeq_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpge_epi32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpge_epi32_mask
+  // CHECK-LABEL: test_mm_cmpge_epi32_mask
   // CHECK: icmp sge <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpge_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpge_epi32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpge_epi32_mask
+  // CHECK-LABEL: test_mm_mask_cmpge_epi32_mask
   // CHECK: icmp sge <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpge_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpge_epi64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpge_epi64_mask
+  // CHECK-LABEL: test_mm_cmpge_epi64_mask
   // CHECK: icmp sge <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpge_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpge_epi64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpge_epi64_mask
+  // CHECK-LABEL: test_mm_mask_cmpge_epi64_mask
   // CHECK: icmp sge <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpge_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpge_epi32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpge_epi32_mask
+  // CHECK-LABEL: test_mm256_cmpge_epi32_mask
   // CHECK: icmp sge <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpge_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpge_epi32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpge_epi32_mask
+  // CHECK-LABEL: test_mm256_mask_cmpge_epi32_mask
   // CHECK: icmp sge <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpge_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpge_epi64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpge_epi64_mask
+  // CHECK-LABEL: test_mm256_cmpge_epi64_mask
   // CHECK: icmp sge <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpge_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpge_epi64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpge_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_cmpge_epi64_mask
   // CHECK: icmp sge <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpge_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpge_epu32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpge_epu32_mask
+  // CHECK-LABEL: test_mm_cmpge_epu32_mask
   // CHECK: icmp uge <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpge_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpge_epu32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpge_epu32_mask
+  // CHECK-LABEL: test_mm_mask_cmpge_epu32_mask
   // CHECK: icmp uge <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpge_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpge_epu64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpge_epu64_mask
+  // CHECK-LABEL: test_mm_cmpge_epu64_mask
   // CHECK: icmp uge <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpge_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpge_epu64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpge_epu64_mask
+  // CHECK-LABEL: test_mm_mask_cmpge_epu64_mask
   // CHECK: icmp uge <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpge_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpge_epu32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpge_epu32_mask
+  // CHECK-LABEL: test_mm256_cmpge_epu32_mask
   // CHECK: icmp uge <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpge_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpge_epu32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpge_epu32_mask
+  // CHECK-LABEL: test_mm256_mask_cmpge_epu32_mask
   // CHECK: icmp uge <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpge_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpge_epu64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpge_epu64_mask
+  // CHECK-LABEL: test_mm256_cmpge_epu64_mask
   // CHECK: icmp uge <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpge_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpge_epu64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpge_epu64_mask
+  // CHECK-LABEL: test_mm256_mask_cmpge_epu64_mask
   // CHECK: icmp uge <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpge_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpgt_epu32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpgt_epu32_mask
+  // CHECK-LABEL: test_mm_cmpgt_epu32_mask
   // CHECK: icmp ugt <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpgt_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpgt_epu32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpgt_epu32_mask
+  // CHECK-LABEL: test_mm_mask_cmpgt_epu32_mask
   // CHECK: icmp ugt <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpgt_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpgt_epu64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpgt_epu64_mask
+  // CHECK-LABEL: test_mm_cmpgt_epu64_mask
   // CHECK: icmp ugt <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpgt_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpgt_epu64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpgt_epu64_mask
+  // CHECK-LABEL: test_mm_mask_cmpgt_epu64_mask
   // CHECK: icmp ugt <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpgt_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpgt_epu32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpgt_epu32_mask
+  // CHECK-LABEL: test_mm256_cmpgt_epu32_mask
   // CHECK: icmp ugt <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpgt_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpgt_epu32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpgt_epu32_mask
+  // CHECK-LABEL: test_mm256_mask_cmpgt_epu32_mask
   // CHECK: icmp ugt <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpgt_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpgt_epu64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpgt_epu64_mask
+  // CHECK-LABEL: test_mm256_cmpgt_epu64_mask
   // CHECK: icmp ugt <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpgt_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpgt_epu64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpgt_epu64_mask
+  // CHECK-LABEL: test_mm256_mask_cmpgt_epu64_mask
   // CHECK: icmp ugt <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpgt_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmple_epi32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmple_epi32_mask
+  // CHECK-LABEL: test_mm_cmple_epi32_mask
   // CHECK: icmp sle <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmple_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmple_epi32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmple_epi32_mask
+  // CHECK-LABEL: test_mm_mask_cmple_epi32_mask
   // CHECK: icmp sle <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmple_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmple_epi64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmple_epi64_mask
+  // CHECK-LABEL: test_mm_cmple_epi64_mask
   // CHECK: icmp sle <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmple_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmple_epi64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmple_epi64_mask
+  // CHECK-LABEL: test_mm_mask_cmple_epi64_mask
   // CHECK: icmp sle <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmple_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmple_epi32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmple_epi32_mask
+  // CHECK-LABEL: test_mm256_cmple_epi32_mask
   // CHECK: icmp sle <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmple_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmple_epi32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmple_epi32_mask
+  // CHECK-LABEL: test_mm256_mask_cmple_epi32_mask
   // CHECK: icmp sle <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmple_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmple_epi64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmple_epi64_mask
+  // CHECK-LABEL: test_mm256_cmple_epi64_mask
   // CHECK: icmp sle <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmple_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmple_epi64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmple_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_cmple_epi64_mask
   // CHECK: icmp sle <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmple_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmple_epu32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmple_epu32_mask
+  // CHECK-LABEL: test_mm_cmple_epu32_mask
   // CHECK: icmp ule <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmple_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmple_epu32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmple_epu32_mask
+  // CHECK-LABEL: test_mm_mask_cmple_epu32_mask
   // CHECK: icmp ule <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmple_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmple_epu64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmple_epu64_mask
+  // CHECK-LABEL: test_mm_cmple_epu64_mask
   // CHECK: icmp ule <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmple_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmple_epu64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmple_epu64_mask
+  // CHECK-LABEL: test_mm_mask_cmple_epu64_mask
   // CHECK: icmp ule <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmple_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmple_epu32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmple_epu32_mask
+  // CHECK-LABEL: test_mm256_cmple_epu32_mask
   // CHECK: icmp ule <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmple_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmple_epu32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmple_epu32_mask
+  // CHECK-LABEL: test_mm256_mask_cmple_epu32_mask
   // CHECK: icmp ule <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmple_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmple_epu64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmple_epu64_mask
+  // CHECK-LABEL: test_mm256_cmple_epu64_mask
   // CHECK: icmp ule <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmple_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmple_epu64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmple_epu64_mask
+  // CHECK-LABEL: test_mm256_mask_cmple_epu64_mask
   // CHECK: icmp ule <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmple_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmplt_epi32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmplt_epi32_mask
+  // CHECK-LABEL: test_mm_cmplt_epi32_mask
   // CHECK: icmp slt <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmplt_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmplt_epi32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmplt_epi32_mask
+  // CHECK-LABEL: test_mm_mask_cmplt_epi32_mask
   // CHECK: icmp slt <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmplt_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmplt_epi64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmplt_epi64_mask
+  // CHECK-LABEL: test_mm_cmplt_epi64_mask
   // CHECK: icmp slt <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmplt_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmplt_epi64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmplt_epi64_mask
+  // CHECK-LABEL: test_mm_mask_cmplt_epi64_mask
   // CHECK: icmp slt <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmplt_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmplt_epi32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmplt_epi32_mask
+  // CHECK-LABEL: test_mm256_cmplt_epi32_mask
   // CHECK: icmp slt <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmplt_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmplt_epi32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmplt_epi32_mask
+  // CHECK-LABEL: test_mm256_mask_cmplt_epi32_mask
   // CHECK: icmp slt <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmplt_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmplt_epi64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmplt_epi64_mask
+  // CHECK-LABEL: test_mm256_cmplt_epi64_mask
   // CHECK: icmp slt <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmplt_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmplt_epi64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmplt_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_cmplt_epi64_mask
   // CHECK: icmp slt <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmplt_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmplt_epu32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmplt_epu32_mask
+  // CHECK-LABEL: test_mm_cmplt_epu32_mask
   // CHECK: icmp ult <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmplt_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmplt_epu32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmplt_epu32_mask
+  // CHECK-LABEL: test_mm_mask_cmplt_epu32_mask
   // CHECK: icmp ult <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmplt_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmplt_epu64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmplt_epu64_mask
+  // CHECK-LABEL: test_mm_cmplt_epu64_mask
   // CHECK: icmp ult <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmplt_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmplt_epu64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmplt_epu64_mask
+  // CHECK-LABEL: test_mm_mask_cmplt_epu64_mask
   // CHECK: icmp ult <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmplt_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmplt_epu32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmplt_epu32_mask
+  // CHECK-LABEL: test_mm256_cmplt_epu32_mask
   // CHECK: icmp ult <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmplt_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmplt_epu32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmplt_epu32_mask
+  // CHECK-LABEL: test_mm256_mask_cmplt_epu32_mask
   // CHECK: icmp ult <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmplt_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmplt_epu64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmplt_epu64_mask
+  // CHECK-LABEL: test_mm256_cmplt_epu64_mask
   // CHECK: icmp ult <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmplt_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmplt_epu64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmplt_epu64_mask
+  // CHECK-LABEL: test_mm256_mask_cmplt_epu64_mask
   // CHECK: icmp ult <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmplt_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpneq_epi32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpneq_epi32_mask
+  // CHECK-LABEL: test_mm_cmpneq_epi32_mask
   // CHECK: icmp ne <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpneq_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpneq_epi32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpneq_epi32_mask
+  // CHECK-LABEL: test_mm_mask_cmpneq_epi32_mask
   // CHECK: icmp ne <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpneq_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpneq_epi64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpneq_epi64_mask
+  // CHECK-LABEL: test_mm_cmpneq_epi64_mask
   // CHECK: icmp ne <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpneq_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpneq_epi64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpneq_epi64_mask
+  // CHECK-LABEL: test_mm_mask_cmpneq_epi64_mask
   // CHECK: icmp ne <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpneq_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpneq_epi32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpneq_epi32_mask
+  // CHECK-LABEL: test_mm256_cmpneq_epi32_mask
   // CHECK: icmp ne <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpneq_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpneq_epi32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpneq_epi32_mask
+  // CHECK-LABEL: test_mm256_mask_cmpneq_epi32_mask
   // CHECK: icmp ne <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpneq_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpneq_epi64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpneq_epi64_mask
+  // CHECK-LABEL: test_mm256_cmpneq_epi64_mask
   // CHECK: icmp ne <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpneq_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpneq_epi64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpneq_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_cmpneq_epi64_mask
   // CHECK: icmp ne <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpneq_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpneq_epu32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpneq_epu32_mask
+  // CHECK-LABEL: test_mm_cmpneq_epu32_mask
   // CHECK: icmp ne <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpneq_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpneq_epu32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpneq_epu32_mask
+  // CHECK-LABEL: test_mm_mask_cmpneq_epu32_mask
   // CHECK: icmp ne <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpneq_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpneq_epu64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpneq_epu64_mask
+  // CHECK-LABEL: test_mm_cmpneq_epu64_mask
   // CHECK: icmp ne <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpneq_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpneq_epu64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpneq_epu64_mask
+  // CHECK-LABEL: test_mm_mask_cmpneq_epu64_mask
   // CHECK: icmp ne <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpneq_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpneq_epu32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpneq_epu32_mask
+  // CHECK-LABEL: test_mm256_cmpneq_epu32_mask
   // CHECK: icmp ne <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpneq_epu32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpneq_epu32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpneq_epu32_mask
+  // CHECK-LABEL: test_mm256_mask_cmpneq_epu32_mask
   // CHECK: icmp ne <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpneq_epu32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpneq_epu64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpneq_epu64_mask
+  // CHECK-LABEL: test_mm256_cmpneq_epu64_mask
   // CHECK: icmp ne <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpneq_epu64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpneq_epu64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpneq_epu64_mask
+  // CHECK-LABEL: test_mm256_mask_cmpneq_epu64_mask
   // CHECK: icmp ne <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpneq_epu64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmp_eq_epi32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmp_eq_epi32_mask
+  // CHECK-LABEL: test_mm_cmp_eq_epi32_mask
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmp_epi32_mask(__a, __b, _MM_CMPINT_EQ);
 }
 
 __mmask8 test_mm_mask_cmp_lt_epi32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_lt_epi32_mask
+  // CHECK-LABEL: test_mm_mask_cmp_lt_epi32_mask
   // CHECK: icmp slt <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmp_epi32_mask(__u, __a, __b, _MM_CMPINT_LT);
 }
 
 __mmask8 test_mm_cmp_lt_epi64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmp_lt_epi64_mask
+  // CHECK-LABEL: test_mm_cmp_lt_epi64_mask
   // CHECK: icmp slt <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmp_epi64_mask(__a, __b, _MM_CMPINT_LT);
 }
 
 __mmask8 test_mm_mask_cmp_eq_epi64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_eq_epi64_mask
+  // CHECK-LABEL: test_mm_mask_cmp_eq_epi64_mask
   // CHECK: icmp eq <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmp_epi64_mask(__u, __a, __b, _MM_CMPINT_EQ);
 }
 
 __mmask8 test_mm256_cmp_eq_epi32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmp_eq_epi32_mask
+  // CHECK-LABEL: test_mm256_cmp_eq_epi32_mask
   // CHECK: icmp eq <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmp_epi32_mask(__a, __b, _MM_CMPINT_EQ);
 }
 
 __mmask8 test_mm256_mask_cmp_le_epi32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_le_epi32_mask
+  // CHECK-LABEL: test_mm256_mask_cmp_le_epi32_mask
   // CHECK: icmp sle <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmp_epi32_mask(__u, __a, __b, _MM_CMPINT_LE);
 }
 
 __mmask8 test_mm256_cmp_eq_epi64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmp_eq_epi64_mask
+  // CHECK-LABEL: test_mm256_cmp_eq_epi64_mask
   // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmp_epi64_mask(__a, __b, _MM_CMPINT_EQ);
 }
 
 __mmask8 test_mm256_mask_cmp_eq_epi64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_eq_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_cmp_eq_epi64_mask
   // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmp_epi64_mask(__u, __a, __b, _MM_CMPINT_EQ);
 }
 
 __mmask8 test_mm_cmp_epu32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmp_epu32_mask
+  // CHECK-LABEL: test_mm_cmp_epu32_mask
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmp_epu32_mask(__a, __b, 0);
 }
 
 __mmask8 test_mm_mask_cmp_epu32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_epu32_mask
+  // CHECK-LABEL: test_mm_mask_cmp_epu32_mask
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmp_epu32_mask(__u, __a, __b, 0);
 }
 
 __mmask8 test_mm_cmp_epu64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmp_epu64_mask
+  // CHECK-LABEL: test_mm_cmp_epu64_mask
   // CHECK: icmp eq <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmp_epu64_mask(__a, __b, 0);
 }
 
 __mmask8 test_mm_mask_cmp_epu64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_epu64_mask
+  // CHECK-LABEL: test_mm_mask_cmp_epu64_mask
   // CHECK: icmp eq <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmp_epu64_mask(__u, __a, __b, 0);
 }
 
 __mmask8 test_mm256_cmp_epu32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmp_epu32_mask
+  // CHECK-LABEL: test_mm256_cmp_epu32_mask
   // CHECK: icmp eq <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmp_epu32_mask(__a, __b, 0);
 }
 
 __mmask8 test_mm256_mask_cmp_epu32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_epu32_mask
+  // CHECK-LABEL: test_mm256_mask_cmp_epu32_mask
   // CHECK: icmp eq <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmp_epu32_mask(__u, __a, __b, 0);
 }
 
 __mmask8 test_mm256_cmp_epu64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmp_epu64_mask
+  // CHECK-LABEL: test_mm256_cmp_epu64_mask
   // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmp_epu64_mask(__a, __b, 0);
 }
 
 __mmask8 test_mm256_mask_cmp_epu64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_epu64_mask
+  // CHECK-LABEL: test_mm256_mask_cmp_epu64_mask
   // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmp_epu64_mask(__u, __a, __b, 0);
@@ -604,14 +606,14 @@ __mmask8 test_mm256_mask_cmp_epu64_mask(__mmask8 __u, __m256i __a, __m256i __b)
 
 __m256i test_mm256_mask_add_epi32 (__m256i __W, __mmask8 __U, __m256i __A,
            __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_add_epi32
+  //CHECK-LABEL: test_mm256_mask_add_epi32
   //CHECK: add <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_add_epi32(__W, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_add_epi32 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_add_epi32
+  //CHECK-LABEL: test_mm256_maskz_add_epi32
   //CHECK: add <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_add_epi32(__U, __A, __B);
@@ -619,14 +621,14 @@ __m256i test_mm256_maskz_add_epi32 (__mmask8 __U, __m256i __A, __m256i __B) {
 
 __m256i test_mm256_mask_add_epi64 (__m256i __W, __mmask8 __U, __m256i __A,
            __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_add_epi64
+  //CHECK-LABEL: test_mm256_mask_add_epi64
   //CHECK: add <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_add_epi64(__W,__U,__A,__B);
 }
 
 __m256i test_mm256_maskz_add_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_add_epi64
+  //CHECK-LABEL: test_mm256_maskz_add_epi64
   //CHECK: add <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_add_epi64 (__U,__A,__B);
@@ -634,14 +636,14 @@ __m256i test_mm256_maskz_add_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
 
 __m256i test_mm256_mask_sub_epi32 (__m256i __W, __mmask8 __U, __m256i __A,
            __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_sub_epi32
+  //CHECK-LABEL: test_mm256_mask_sub_epi32
   //CHECK: sub <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_sub_epi32 (__W,__U,__A,__B);
 }
 
 __m256i test_mm256_maskz_sub_epi32 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_sub_epi32
+  //CHECK-LABEL: test_mm256_maskz_sub_epi32
   //CHECK: sub <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_sub_epi32 (__U,__A,__B);
@@ -649,14 +651,14 @@ __m256i test_mm256_maskz_sub_epi32 (__mmask8 __U, __m256i __A, __m256i __B) {
 
 __m256i test_mm256_mask_sub_epi64 (__m256i __W, __mmask8 __U, __m256i __A,
            __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_sub_epi64
+  //CHECK-LABEL: test_mm256_mask_sub_epi64
   //CHECK: sub <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_sub_epi64 (__W,__U,__A,__B);
 }
 
 __m256i test_mm256_maskz_sub_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_sub_epi64
+  //CHECK-LABEL: test_mm256_maskz_sub_epi64
   //CHECK: sub <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_sub_epi64 (__U,__A,__B);
@@ -664,7 +666,7 @@ __m256i test_mm256_maskz_sub_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
 
 __m128i test_mm_mask_add_epi32 (__m128i __W, __mmask8 __U, __m128i __A,
         __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_add_epi32
+  //CHECK-LABEL: test_mm_mask_add_epi32
   //CHECK: add <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_add_epi32(__W,__U,__A,__B);
@@ -672,7 +674,7 @@ __m128i test_mm_mask_add_epi32 (__m128i __W, __mmask8 __U, __m128i __A,
 
 
 __m128i test_mm_maskz_add_epi32 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_add_epi32
+  //CHECK-LABEL: test_mm_maskz_add_epi32
   //CHECK: add <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_add_epi32 (__U,__A,__B);
@@ -680,14 +682,14 @@ __m128i test_mm_maskz_add_epi32 (__mmask8 __U, __m128i __A, __m128i __B) {
 
 __m128i test_mm_mask_add_epi64 (__m128i __W, __mmask8 __U, __m128i __A,
         __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_add_epi64
+  //CHECK-LABEL: test_mm_mask_add_epi64
   //CHECK: add <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_add_epi64 (__W,__U,__A,__B);
 }
 
 __m128i test_mm_maskz_add_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_add_epi64
+  //CHECK-LABEL: test_mm_maskz_add_epi64
   //CHECK: add <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_add_epi64 (__U,__A,__B);
@@ -695,14 +697,14 @@ __m128i test_mm_maskz_add_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
 
 __m128i test_mm_mask_sub_epi32 (__m128i __W, __mmask8 __U, __m128i __A,
         __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_sub_epi32
+  //CHECK-LABEL: test_mm_mask_sub_epi32
   //CHECK: sub <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_sub_epi32(__W, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_sub_epi32 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_sub_epi32
+  //CHECK-LABEL: test_mm_maskz_sub_epi32
   //CHECK: sub <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_sub_epi32(__U, __A, __B);
@@ -710,14 +712,14 @@ __m128i test_mm_maskz_sub_epi32 (__mmask8 __U, __m128i __A, __m128i __B) {
 
 __m128i test_mm_mask_sub_epi64 (__m128i __W, __mmask8 __U, __m128i __A,
         __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_sub_epi64
+  //CHECK-LABEL: test_mm_mask_sub_epi64
   //CHECK: sub <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_sub_epi64 (__W, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_sub_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_sub_epi64
+  //CHECK-LABEL: test_mm_maskz_sub_epi64
   //CHECK: sub <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_sub_epi64 (__U, __A, __B);
@@ -725,7 +727,7 @@ __m128i test_mm_maskz_sub_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
 
 __m256i test_mm256_mask_mul_epi32 (__m256i __W, __mmask8 __M, __m256i __X,
            __m256i __Y) {
-  //CHECK-LABEL: @test_mm256_mask_mul_epi32
+  //CHECK-LABEL: test_mm256_mask_mul_epi32
   //CHECK: shl <4 x i64> %{{.*}}, splat (i64 32)
   //CHECK: ashr <4 x i64> %{{.*}}, splat (i64 32)
   //CHECK: shl <4 x i64> %{{.*}}, splat (i64 32)
@@ -736,7 +738,7 @@ __m256i test_mm256_mask_mul_epi32 (__m256i __W, __mmask8 __M, __m256i __X,
 }
 
 __m256i test_mm256_maskz_mul_epi32 (__mmask8 __M, __m256i __X, __m256i __Y) {
-  //CHECK-LABEL: @test_mm256_maskz_mul_epi32
+  //CHECK-LABEL: test_mm256_maskz_mul_epi32
   //CHECK: shl <4 x i64> %{{.*}}, splat (i64 32)
   //CHECK: ashr <4 x i64> %{{.*}}, splat (i64 32)
   //CHECK: shl <4 x i64> %{{.*}}, splat (i64 32)
@@ -749,7 +751,7 @@ __m256i test_mm256_maskz_mul_epi32 (__mmask8 __M, __m256i __X, __m256i __Y) {
 
 __m128i test_mm_mask_mul_epi32 (__m128i __W, __mmask8 __M, __m128i __X,
         __m128i __Y) {
-  //CHECK-LABEL: @test_mm_mask_mul_epi32
+  //CHECK-LABEL: test_mm_mask_mul_epi32
   //CHECK: shl <2 x i64> %{{.*}}, splat (i64 32)
   //CHECK: ashr <2 x i64> %{{.*}}, splat (i64 32)
   //CHECK: shl <2 x i64> %{{.*}}, splat (i64 32)
@@ -760,7 +762,7 @@ __m128i test_mm_mask_mul_epi32 (__m128i __W, __mmask8 __M, __m128i __X,
 }
 
 __m128i test_mm_maskz_mul_epi32 (__mmask8 __M, __m128i __X, __m128i __Y) {
-  //CHECK-LABEL: @test_mm_maskz_mul_epi32
+  //CHECK-LABEL: test_mm_maskz_mul_epi32
   //CHECK: shl <2 x i64> %{{.*}}, splat (i64 32)
   //CHECK: ashr <2 x i64> %{{.*}}, splat (i64 32)
   //CHECK: shl <2 x i64> %{{.*}}, splat (i64 32)
@@ -772,7 +774,7 @@ __m128i test_mm_maskz_mul_epi32 (__mmask8 __M, __m128i __X, __m128i __Y) {
 
 __m256i test_mm256_mask_mul_epu32 (__m256i __W, __mmask8 __M, __m256i __X,
            __m256i __Y) {
-  //CHECK-LABEL: @test_mm256_mask_mul_epu32
+  //CHECK-LABEL: test_mm256_mask_mul_epu32
   //CHECK: and <4 x i64> %{{.*}}, splat (i64 4294967295)
   //CHECK: and <4 x i64> %{{.*}}, splat (i64 4294967295)
   //CHECK: mul <4 x i64> %{{.*}}, %{{.*}}
@@ -781,7 +783,7 @@ __m256i test_mm256_mask_mul_epu32 (__m256i __W, __mmask8 __M, __m256i __X,
 }
 
 __m256i test_mm256_maskz_mul_epu32 (__mmask8 __M, __m256i __X, __m256i __Y) {
-  //CHECK-LABEL: @test_mm256_maskz_mul_epu32
+  //CHECK-LABEL: test_mm256_maskz_mul_epu32
   //CHECK: and <4 x i64> %{{.*}}, splat (i64 4294967295)
   //CHECK: and <4 x i64> %{{.*}}, splat (i64 4294967295)
   //CHECK: mul <4 x i64> %{{.*}}, %{{.*}}
@@ -791,7 +793,7 @@ __m256i test_mm256_maskz_mul_epu32 (__mmask8 __M, __m256i __X, __m256i __Y) {
 
 __m128i test_mm_mask_mul_epu32 (__m128i __W, __mmask8 __M, __m128i __X,
         __m128i __Y) {
-  //CHECK-LABEL: @test_mm_mask_mul_epu32
+  //CHECK-LABEL: test_mm_mask_mul_epu32
   //CHECK: and <2 x i64> %{{.*}}, splat (i64 4294967295)
   //CHECK: and <2 x i64> %{{.*}}, splat (i64 4294967295)
   //CHECK: mul <2 x i64> %{{.*}}, %{{.*}}
@@ -800,7 +802,7 @@ __m128i test_mm_mask_mul_epu32 (__m128i __W, __mmask8 __M, __m128i __X,
 }
 
 __m128i test_mm_maskz_mul_epu32 (__mmask8 __M, __m128i __X, __m128i __Y) {
-  //CHECK-LABEL: @test_mm_maskz_mul_epu32
+  //CHECK-LABEL: test_mm_maskz_mul_epu32
   //CHECK: and <2 x i64> %{{.*}}, splat (i64 4294967295)
   //CHECK: and <2 x i64> %{{.*}}, splat (i64 4294967295)
   //CHECK: mul <2 x i64> %{{.*}}, %{{.*}}
@@ -809,7 +811,7 @@ __m128i test_mm_maskz_mul_epu32 (__mmask8 __M, __m128i __X, __m128i __Y) {
 }
 
 __m128i test_mm_maskz_mullo_epi32 (__mmask8 __M, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_mullo_epi32
+  //CHECK-LABEL: test_mm_maskz_mullo_epi32
   //CHECK: mul <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_mullo_epi32(__M, __A, __B);
@@ -817,14 +819,14 @@ __m128i test_mm_maskz_mullo_epi32 (__mmask8 __M, __m128i __A, __m128i __B) {
 
 __m128i test_mm_mask_mullo_epi32 (__m128i __W, __mmask8 __M, __m128i __A,
           __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_mullo_epi32
+  //CHECK-LABEL: test_mm_mask_mullo_epi32
   //CHECK: mul <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_mullo_epi32(__W, __M, __A, __B);
 }
 
 __m256i test_mm256_maskz_mullo_epi32 (__mmask8 __M, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_mullo_epi32
+  //CHECK-LABEL: test_mm256_maskz_mullo_epi32
   //CHECK: mul <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_mullo_epi32(__M, __A, __B);
@@ -832,61 +834,61 @@ __m256i test_mm256_maskz_mullo_epi32 (__mmask8 __M, __m256i __A, __m256i __B) {
 
 __m256i test_mm256_mask_mullo_epi32 (__m256i __W, __mmask8 __M, __m256i __A,
        __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_mullo_epi32
+  //CHECK-LABEL: test_mm256_mask_mullo_epi32
   //CHECK: mul <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_mullo_epi32(__W, __M, __A, __B);
 }
 
 __m256i test_mm256_and_epi32 (__m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_and_epi32
+  //CHECK-LABEL: test_mm256_and_epi32
   //CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   return _mm256_and_epi32(__A, __B);
 }
 
 __m256i test_mm256_mask_and_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_and_epi32
+  //CHECK-LABEL: test_mm256_mask_and_epi32
   //CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_and_epi32(__W, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_and_epi32 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_and_epi32
+  //CHECK-LABEL: test_mm256_maskz_and_epi32
   //CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_and_epi32(__U, __A, __B);
 }
 
 __m128i test_mm_and_epi32 (__m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_and_epi32
+  //CHECK-LABEL: test_mm_and_epi32
   //CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   return _mm_and_epi32(__A, __B);
 }
 
 __m128i test_mm_mask_and_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_and_epi32
+  //CHECK-LABEL: test_mm_mask_and_epi32
   //CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_and_epi32(__W, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_and_epi32 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_and_epi32
+  //CHECK-LABEL: test_mm_maskz_and_epi32
   //CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_and_epi32(__U, __A, __B);
 }
 
 __m256i test_mm256_andnot_epi32 (__m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_andnot_epi32
+  //CHECK-LABEL: test_mm256_andnot_epi32
   //CHECK: xor <8 x i32> %{{.*}}, splat (i32 -1)
   //CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   return _mm256_andnot_epi32(__A, __B);
 }
 
 __m256i test_mm256_mask_andnot_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_andnot_epi32
+  //CHECK-LABEL: test_mm256_mask_andnot_epi32
   //CHECK: xor <8 x i32> %{{.*}}, splat (i32 -1)
   //CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
@@ -894,7 +896,7 @@ __m256i test_mm256_mask_andnot_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __
 }
 
 __m256i test_mm256_maskz_andnot_epi32 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_andnot_epi32
+  //CHECK-LABEL: test_mm256_maskz_andnot_epi32
   //CHECK: xor <8 x i32> %{{.*}}, splat (i32 -1)
   //CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
@@ -902,14 +904,14 @@ __m256i test_mm256_maskz_andnot_epi32 (__mmask8 __U, __m256i __A, __m256i __B) {
 }
 
 __m128i test_mm_andnot_epi32 (__m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_andnot_epi32
+  //CHECK-LABEL: test_mm_andnot_epi32
   //CHECK: xor <4 x i32> %{{.*}}, splat (i32 -1)
   //CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   return _mm_andnot_epi32(__A, __B);
 }
 
 __m128i test_mm_mask_andnot_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_andnot_epi32
+  //CHECK-LABEL: test_mm_mask_andnot_epi32
   //CHECK: xor <4 x i32> %{{.*}}, splat (i32 -1)
   //CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
@@ -917,7 +919,7 @@ __m128i test_mm_mask_andnot_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m12
 }
 
 __m128i test_mm_maskz_andnot_epi32 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_andnot_epi32
+  //CHECK-LABEL: test_mm_maskz_andnot_epi32
   //CHECK: xor <4 x i32> %{{.*}}, splat (i32 -1)
   //CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
@@ -925,134 +927,134 @@ __m128i test_mm_maskz_andnot_epi32 (__mmask8 __U, __m128i __A, __m128i __B) {
 }
 
 __m256i test_mm256_or_epi32 (__m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_or_epi32
+  //CHECK-LABEL: test_mm256_or_epi32
   //CHECK: or <8 x i32> %{{.*}}, %{{.*}}
   return _mm256_or_epi32(__A, __B);
 }
 
 __m256i test_mm256_mask_or_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_or_epi32
+  //CHECK-LABEL: test_mm256_mask_or_epi32
   //CHECK: or <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_or_epi32(__W, __U, __A, __B);
 }
 
  __m256i test_mm256_maskz_or_epi32 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_or_epi32
+  //CHECK-LABEL: test_mm256_maskz_or_epi32
   //CHECK: or <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_or_epi32(__U, __A, __B);
 }
 
 __m128i test_mm_or_epi32 (__m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_or_epi32
+  //CHECK-LABEL: test_mm_or_epi32
   //CHECK: or <4 x i32> %{{.*}}, %{{.*}}
   return _mm_or_epi32(__A, __B);
 }
 
 __m128i test_mm_mask_or_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_or_epi32
+  //CHECK-LABEL: test_mm_mask_or_epi32
   //CHECK: or <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_or_epi32(__W, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_or_epi32 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_or_epi32
+  //CHECK-LABEL: test_mm_maskz_or_epi32
   //CHECK: or <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_or_epi32(__U, __A, __B);
 }
 
 __m256i test_mm256_xor_epi32 (__m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_xor_epi32
+  //CHECK-LABEL: test_mm256_xor_epi32
   //CHECK: or <8 x i32> %{{.*}}, %{{.*}}
   return _mm256_xor_epi32(__A, __B);
 }
 
 __m256i test_mm256_mask_xor_epi32 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_xor_epi32
+  //CHECK-LABEL: test_mm256_mask_xor_epi32
   //CHECK: xor <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_xor_epi32(__W, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_xor_epi32 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_xor_epi32
+  //CHECK-LABEL: test_mm256_maskz_xor_epi32
   //CHECK: xor <8 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_xor_epi32(__U, __A, __B);
 }
 
 __m128i test_mm_xor_epi32 (__m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_xor_epi32
+  //CHECK-LABEL: test_mm_xor_epi32
   //CHECK: xor <4 x i32> %{{.*}}, %{{.*}}
   return _mm_xor_epi32(__A, __B);
 }
 
 __m128i test_mm_mask_xor_epi32 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_xor_epi32
+  //CHECK-LABEL: test_mm_mask_xor_epi32
   //CHECK: xor <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_xor_epi32(__W, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_xor_epi32 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_xor_epi32
+  //CHECK-LABEL: test_mm_maskz_xor_epi32
   //CHECK: xor <4 x i32> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_xor_epi32(__U, __A, __B);
 }
 
 __m256i test_mm256_and_epi64 (__m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_and_epi64
+  //CHECK-LABEL: test_mm256_and_epi64
   //CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   return _mm256_and_epi64(__A, __B);
 }
 
 __m256i test_mm256_mask_and_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_and_epi64
+  //CHECK-LABEL: test_mm256_mask_and_epi64
   //CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_and_epi64(__W, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_and_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_and_epi64
+  //CHECK-LABEL: test_mm256_maskz_and_epi64
   //CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_and_epi64(__U, __A, __B);
 }
 
 __m128i test_mm_and_epi64 (__m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_and_epi64
+  //CHECK-LABEL: test_mm_and_epi64
   //CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   return _mm_and_epi64(__A, __B);
 }
 
 __m128i test_mm_mask_and_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_and_epi64
+  //CHECK-LABEL: test_mm_mask_and_epi64
   //CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_and_epi64(__W,__U, __A, __B);
 }
 
 __m128i test_mm_maskz_and_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_and_epi64
+  //CHECK-LABEL: test_mm_maskz_and_epi64
   //CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_and_epi64(__U, __A, __B);
 }
 
 __m256i test_mm256_andnot_epi64 (__m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_andnot_epi64
+  //CHECK-LABEL: test_mm256_andnot_epi64
   //CHECK: xor <4 x i64> %{{.*}}, splat (i64 -1)
   //CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   return _mm256_andnot_epi64(__A, __B);
 }
 
 __m256i test_mm256_mask_andnot_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_andnot_epi64
+  //CHECK-LABEL: test_mm256_mask_andnot_epi64
   //CHECK: xor <4 x i64> %{{.*}}, splat (i64 -1)
   //CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
@@ -1060,7 +1062,7 @@ __m256i test_mm256_mask_andnot_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __
 }
 
 __m256i test_mm256_maskz_andnot_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_andnot_epi64
+  //CHECK-LABEL: test_mm256_maskz_andnot_epi64
   //CHECK: xor <4 x i64> %{{.*}}, splat (i64 -1)
   //CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
@@ -1068,14 +1070,14 @@ __m256i test_mm256_maskz_andnot_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
 }
 
 __m128i test_mm_andnot_epi64 (__m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_andnot_epi64
+  //CHECK-LABEL: test_mm_andnot_epi64
   //CHECK: xor <2 x i64> %{{.*}}, splat (i64 -1)
   //CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   return _mm_andnot_epi64(__A, __B);
 }
 
 __m128i test_mm_mask_andnot_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_andnot_epi64
+  //CHECK-LABEL: test_mm_mask_andnot_epi64
   //CHECK: xor <2 x i64> %{{.*}}, splat (i64 -1)
   //CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
@@ -1083,7 +1085,7 @@ __m128i test_mm_mask_andnot_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m12
 }
 
 __m128i test_mm_maskz_andnot_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_andnot_epi64
+  //CHECK-LABEL: test_mm_maskz_andnot_epi64
   //CHECK: xor <2 x i64> %{{.*}}, splat (i64 -1)
   //CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
@@ -1091,87 +1093,87 @@ __m128i test_mm_maskz_andnot_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
 }
 
 __m256i test_mm256_or_epi64 (__m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_or_epi64
+  //CHECK-LABEL: test_mm256_or_epi64
   //CHECK: or <4 x i64> %{{.*}}, %{{.*}}
   return _mm256_or_epi64(__A, __B);
 }
 
 __m256i test_mm256_mask_or_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_or_epi64
+  //CHECK-LABEL: test_mm256_mask_or_epi64
   //CHECK: or <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_or_epi64(__W,__U, __A, __B);
 }
 
 __m256i test_mm256_maskz_or_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_or_epi64
+  //CHECK-LABEL: test_mm256_maskz_or_epi64
   //CHECK: or <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_or_epi64(__U, __A, __B);
 }
 
 __m128i test_mm_or_epi64 (__m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_or_epi64
+  //CHECK-LABEL: test_mm_or_epi64
   //CHECK: or <2 x i64> %{{.*}}, %{{.*}}
   return _mm_or_epi64(__A, __B);
 }
 
 __m128i test_mm_mask_or_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_or_epi64
+  //CHECK-LABEL: test_mm_mask_or_epi64
   //CHECK: or <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_or_epi64(__W, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_or_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_or_epi64
+  //CHECK-LABEL: test_mm_maskz_or_epi64
   //CHECK: or <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_or_epi64( __U, __A, __B);
 }
 
 __m256i test_mm256_xor_epi64 (__m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_xor_epi64
+  //CHECK-LABEL: test_mm256_xor_epi64
   //CHECK: xor <4 x i64> %{{.*}}, %{{.*}}
   return _mm256_xor_epi64(__A, __B);
 }
 
 __m256i test_mm256_mask_xor_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_xor_epi64
+  //CHECK-LABEL: test_mm256_mask_xor_epi64
   //CHECK: xor <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_xor_epi64(__W,__U, __A, __B);
 }
 
 __m256i test_mm256_maskz_xor_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_xor_epi64
+  //CHECK-LABEL: test_mm256_maskz_xor_epi64
   //CHECK: xor <4 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_xor_epi64(__U, __A, __B);
 }
 
 __m128i test_mm_xor_epi64 (__m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_xor_epi64
+  //CHECK-LABEL: test_mm_xor_epi64
   //CHECK: xor <2 x i64> %{{.*}}, %{{.*}}
   return _mm_xor_epi64(__A, __B);
 }
 
 __m128i test_mm_mask_xor_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_xor_epi64
+  //CHECK-LABEL: test_mm_mask_xor_epi64
   //CHECK: xor <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_xor_epi64(__W, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_xor_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_xor_epi64
+  //CHECK-LABEL: test_mm_maskz_xor_epi64
   //CHECK: xor <2 x i64> %{{.*}}, %{{.*}}
   //CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_xor_epi64( __U, __A, __B);
 }
 
 __mmask8 test_mm256_cmp_ps_mask_eq_oq(__m256 a, __m256 b) {
-  // CHECK-LABEL: @test_mm256_cmp_ps_mask_eq_oq
+  // CHECK-LABEL: test_mm256_cmp_ps_mask_eq_oq
   // CHECK: fcmp oeq <8 x float> %{{.*}}, %{{.*}}
   return _mm256_cmp_ps_mask(a, b, _CMP_EQ_OQ);
 }
@@ -1363,7 +1365,7 @@ __mmask8 test_mm256_cmp_ps_mask_true_us(__m256 a, __m256 b) {
 }
 
 __mmask8 test_mm256_mask_cmp_ps_mask_eq_oq(__mmask8 m, __m256 a, __m256 b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_ps_mask_eq_oq
+  // CHECK-LABEL: test_mm256_mask_cmp_ps_mask_eq_oq
   // CHECK: [[CMP:%.*]] = fcmp oeq <8 x float> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> [[CMP]], {{.*}}
   return _mm256_mask_cmp_ps_mask(m, a, b, _CMP_EQ_OQ);
@@ -1587,7 +1589,7 @@ __mmask8 test_mm256_mask_cmp_ps_mask_true_us(__mmask8 m, __m256 a, __m256 b) {
 }
 
 __mmask8 test_mm256_cmp_pd_mask_eq_oq(__m256d a, __m256d b) {
-  // CHECK-LABEL: @test_mm256_cmp_pd_mask_eq_oq
+  // CHECK-LABEL: test_mm256_cmp_pd_mask_eq_oq
   // CHECK: fcmp oeq <4 x double> %{{.*}}, %{{.*}}
   return _mm256_cmp_pd_mask(a, b, _CMP_EQ_OQ);
 }
@@ -1779,7 +1781,7 @@ __mmask8 test_mm256_cmp_pd_mask_true_us(__m256d a, __m256d b) {
 }
 
 __mmask8 test_mm256_mask_cmp_pd_mask_eq_oq(__mmask8 m, __m256d a, __m256d b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_pd_mask_eq_oq
+  // CHECK-LABEL: test_mm256_mask_cmp_pd_mask_eq_oq
   // CHECK: [[CMP:%.*]] = fcmp oeq <4 x double> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> [[CMP]], {{.*}}
   return _mm256_mask_cmp_pd_mask(m, a, b, _CMP_EQ_OQ);
@@ -2003,7 +2005,7 @@ __mmask8 test_mm256_mask_cmp_pd_mask_true_us(__mmask8 m, __m256d a, __m256d b) {
 }
 
 __mmask8 test_mm_cmp_ps_mask_eq_oq(__m128 a, __m128 b) {
-  // CHECK-LABEL: @test_mm_cmp_ps_mask_eq_oq
+  // CHECK-LABEL: test_mm_cmp_ps_mask_eq_oq
   // CHECK: fcmp oeq <4 x float> %{{.*}}, %{{.*}}
   return _mm_cmp_ps_mask(a, b, _CMP_EQ_OQ);
 }
@@ -2195,7 +2197,7 @@ __mmask8 test_mm_cmp_ps_mask_true_us(__m128 a, __m128 b) {
 }
 
 __mmask8 test_mm_mask_cmp_ps_mask_eq_oq(__mmask8 m, __m128 a, __m128 b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_ps_mask_eq_oq
+  // CHECK-LABEL: test_mm_mask_cmp_ps_mask_eq_oq
   // CHECK: [[CMP:%.*]] = fcmp oeq <4 x float> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> [[CMP]], {{.*}}
   return _mm_mask_cmp_ps_mask(m, a, b, _CMP_EQ_OQ);
@@ -2419,7 +2421,7 @@ __mmask8 test_mm_mask_cmp_ps_mask_true_us(__mmask8 m, __m128 a, __m128 b) {
 }
 
 __mmask8 test_mm_cmp_pd_mask_eq_oq(__m128d a, __m128d b) {
-  // CHECK-LABEL: @test_mm_cmp_pd_mask_eq_oq
+  // CHECK-LABEL: test_mm_cmp_pd_mask_eq_oq
   // CHECK: fcmp oeq <2 x double> %{{.*}}, %{{.*}}
   return _mm_cmp_pd_mask(a, b, _CMP_EQ_OQ);
 }
@@ -2611,7 +2613,7 @@ __mmask8 test_mm_cmp_pd_mask_true_us(__m128d a, __m128d b) {
 }
 
 __mmask8 test_mm_mask_cmp_pd_mask_eq_oq(__mmask8 m, __m128d a, __m128d b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_pd_mask_eq_oq
+  // CHECK-LABEL: test_mm_mask_cmp_pd_mask_eq_oq
   // CHECK: [[CMP:%.*]] = fcmp oeq <2 x double> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> [[CMP]], {{.*}}
   return _mm_mask_cmp_pd_mask(m, a, b, _CMP_EQ_OQ);
@@ -2835,7 +2837,7 @@ __mmask8 test_mm_mask_cmp_pd_mask_true_us(__mmask8 m, __m128d a, __m128d b) {
 }
 
 __m128d test_mm_mask_fmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_mask_fmadd_pd
+  // CHECK-LABEL: test_mm_mask_fmadd_pd
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
@@ -2843,7 +2845,7 @@ __m128d test_mm_mask_fmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __
 }
 
 __m128d test_mm_mask_fmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_mask_fmsub_pd
+  // CHECK-LABEL: test_mm_mask_fmsub_pd
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -2852,7 +2854,7 @@ __m128d test_mm_mask_fmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __
 }
 
 __m128d test_mm_mask3_fmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmadd_pd
+  // CHECK-LABEL: test_mm_mask3_fmadd_pd
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
@@ -2860,7 +2862,7 @@ __m128d test_mm_mask3_fmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 _
 }
 
 __m128d test_mm_mask3_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmadd_pd
+  // CHECK-LABEL: test_mm_mask3_fnmadd_pd
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -2869,7 +2871,7 @@ __m128d test_mm_mask3_fnmadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8
 }
 
 __m128d test_mm_maskz_fmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmadd_pd
+  // CHECK-LABEL: test_mm_maskz_fmadd_pd
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
@@ -2877,7 +2879,7 @@ __m128d test_mm_maskz_fmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d _
 }
 
 __m128d test_mm_maskz_fmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmsub_pd
+  // CHECK-LABEL: test_mm_maskz_fmsub_pd
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -2886,7 +2888,7 @@ __m128d test_mm_maskz_fmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d _
 }
 
 __m128d test_mm_maskz_fnmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmadd_pd
+  // CHECK-LABEL: test_mm_maskz_fnmadd_pd
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -2895,7 +2897,7 @@ __m128d test_mm_maskz_fnmadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d
 }
 
 __m128d test_mm_maskz_fnmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmsub_pd
+  // CHECK-LABEL: test_mm_maskz_fnmsub_pd
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
@@ -2905,7 +2907,7 @@ __m128d test_mm_maskz_fnmsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d
 }
 
 __m256d test_mm256_mask_fmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmadd_pd
+  // CHECK-LABEL: test_mm256_mask_fmadd_pd
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
@@ -2913,7 +2915,7 @@ __m256d test_mm256_mask_fmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d
 }
 
 __m256d test_mm256_mask_fmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmsub_pd
+  // CHECK-LABEL: test_mm256_mask_fmsub_pd
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -2922,7 +2924,7 @@ __m256d test_mm256_mask_fmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d
 }
 
 __m256d test_mm256_mask3_fmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmadd_pd
+  // CHECK-LABEL: test_mm256_mask3_fmadd_pd
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
@@ -2930,7 +2932,7 @@ __m256d test_mm256_mask3_fmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask
 }
 
 __m256d test_mm256_mask3_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fnmadd_pd
+  // CHECK-LABEL: test_mm256_mask3_fnmadd_pd
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -2939,7 +2941,7 @@ __m256d test_mm256_mask3_fnmadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmas
 }
 
 __m256d test_mm256_maskz_fmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmadd_pd
+  // CHECK-LABEL: test_mm256_maskz_fmadd_pd
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
@@ -2947,7 +2949,7 @@ __m256d test_mm256_maskz_fmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256
 }
 
 __m256d test_mm256_maskz_fmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmsub_pd
+  // CHECK-LABEL: test_mm256_maskz_fmsub_pd
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -2956,7 +2958,7 @@ __m256d test_mm256_maskz_fmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256
 }
 
 __m256d test_mm256_maskz_fnmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fnmadd_pd
+  // CHECK-LABEL: test_mm256_maskz_fnmadd_pd
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -2965,7 +2967,7 @@ __m256d test_mm256_maskz_fnmadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m25
 }
 
 __m256d test_mm256_maskz_fnmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fnmsub_pd
+  // CHECK-LABEL: test_mm256_maskz_fnmsub_pd
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
@@ -2975,7 +2977,7 @@ __m256d test_mm256_maskz_fnmsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m25
 }
 
 __m128 test_mm_mask_fmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_mask_fmadd_ps
+  // CHECK-LABEL: test_mm_mask_fmadd_ps
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -2983,7 +2985,7 @@ __m128 test_mm_mask_fmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
 }
 
 __m128 test_mm_mask_fmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_mask_fmsub_ps
+  // CHECK-LABEL: test_mm_mask_fmsub_ps
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -2992,7 +2994,7 @@ __m128 test_mm_mask_fmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
 }
 
 __m128 test_mm_mask3_fmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmadd_ps
+  // CHECK-LABEL: test_mm_mask3_fmadd_ps
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -3000,7 +3002,7 @@ __m128 test_mm_mask3_fmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
 }
 
 __m128 test_mm_mask3_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmadd_ps
+  // CHECK-LABEL: test_mm_mask3_fnmadd_ps
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3009,7 +3011,7 @@ __m128 test_mm_mask3_fnmadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
 }
 
 __m128 test_mm_maskz_fmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmadd_ps
+  // CHECK-LABEL: test_mm_maskz_fmadd_ps
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -3017,7 +3019,7 @@ __m128 test_mm_maskz_fmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
 }
 
 __m128 test_mm_maskz_fmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmsub_ps
+  // CHECK-LABEL: test_mm_maskz_fmsub_ps
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3026,7 +3028,7 @@ __m128 test_mm_maskz_fmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
 }
 
 __m128 test_mm_maskz_fnmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmadd_ps
+  // CHECK-LABEL: test_mm_maskz_fnmadd_ps
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3035,7 +3037,7 @@ __m128 test_mm_maskz_fnmadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
 }
 
 __m128 test_mm_maskz_fnmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmsub_ps
+  // CHECK-LABEL: test_mm_maskz_fnmsub_ps
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
@@ -3045,14 +3047,14 @@ __m128 test_mm_maskz_fnmsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C)
 }
 
 __m256 test_mm256_mask_fmadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmadd_ps
+  // CHECK-LABEL: test_mm256_mask_fmadd_ps
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_fmadd_ps(__A, __U, __B, __C);
 }
 
 __m256 test_mm256_mask_fmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmsub_ps
+  // CHECK-LABEL: test_mm256_mask_fmsub_ps
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3060,14 +3062,14 @@ __m256 test_mm256_mask_fmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C
 }
 
 __m256 test_mm256_mask3_fmadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmadd_ps
+  // CHECK-LABEL: test_mm256_mask3_fmadd_ps
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask3_fmadd_ps(__A, __B, __C, __U);
 }
 
 __m256 test_mm256_mask3_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fnmadd_ps
+  // CHECK-LABEL: test_mm256_mask3_fnmadd_ps
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3075,14 +3077,14 @@ __m256 test_mm256_mask3_fnmadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 _
 }
 
 __m256 test_mm256_maskz_fmadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmadd_ps
+  // CHECK-LABEL: test_mm256_maskz_fmadd_ps
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_fmadd_ps(__U, __A, __B, __C);
 }
 
 __m256 test_mm256_maskz_fmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmsub_ps
+  // CHECK-LABEL: test_mm256_maskz_fmsub_ps
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3090,7 +3092,7 @@ __m256 test_mm256_maskz_fmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __
 }
 
 __m256 test_mm256_maskz_fnmadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fnmadd_ps
+  // CHECK-LABEL: test_mm256_maskz_fnmadd_ps
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3098,7 +3100,7 @@ __m256 test_mm256_maskz_fnmadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 _
 }
 
 __m256 test_mm256_maskz_fnmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fnmsub_ps
+  // CHECK-LABEL: test_mm256_maskz_fnmsub_ps
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
@@ -3107,7 +3109,7 @@ __m256 test_mm256_maskz_fnmsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 _
 }
 
 __m128d test_mm_mask_fmaddsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_mask_fmaddsub_pd
+  // CHECK-LABEL: test_mm_mask_fmaddsub_pd
   // CHECK-NOT: fneg
   // CHECK: call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -3116,7 +3118,7 @@ __m128d test_mm_mask_fmaddsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d
 }
 
 __m128d test_mm_mask_fmsubadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_mask_fmsubadd_pd
+  // CHECK-LABEL: test_mm_mask_fmsubadd_pd
   // CHECK: [[NEG:%.+]] = fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> [[NEG]])
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -3125,7 +3127,7 @@ __m128d test_mm_mask_fmsubadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d
 }
 
 __m128d test_mm_mask3_fmaddsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmaddsub_pd
+  // CHECK-LABEL: test_mm_mask3_fmaddsub_pd
   // CHECK-NOT: fneg
   // CHECK: call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -3134,7 +3136,7 @@ __m128d test_mm_mask3_fmaddsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask
 }
 
 __m128d test_mm_maskz_fmaddsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmaddsub_pd
+  // CHECK-LABEL: test_mm_maskz_fmaddsub_pd
   // CHECK-NOT: fneg
   // CHECK: call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -3143,7 +3145,7 @@ __m128d test_mm_maskz_fmaddsub_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128
 }
 
 __m128d test_mm_maskz_fmsubadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmsubadd_pd
+  // CHECK-LABEL: test_mm_maskz_fmsubadd_pd
   // CHECK: [[NEG:%.+]] = fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> [[NEG]])
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -3152,7 +3154,7 @@ __m128d test_mm_maskz_fmsubadd_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128
 }
 
 __m256d test_mm256_mask_fmaddsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmaddsub_pd
+  // CHECK-LABEL: test_mm256_mask_fmaddsub_pd
   // CHECK-NOT: fneg
   // CHECK: call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3161,7 +3163,7 @@ __m256d test_mm256_mask_fmaddsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m2
 }
 
 __m256d test_mm256_mask_fmsubadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmsubadd_pd
+  // CHECK-LABEL: test_mm256_mask_fmsubadd_pd
   // CHECK: [[NEG:%.+]] = fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> [[NEG]])
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3170,7 +3172,7 @@ __m256d test_mm256_mask_fmsubadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m2
 }
 
 __m256d test_mm256_mask3_fmaddsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmaddsub_pd
+  // CHECK-LABEL: test_mm256_mask3_fmaddsub_pd
   // CHECK-NOT: fneg
   // CHECK: call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3179,7 +3181,7 @@ __m256d test_mm256_mask3_fmaddsub_pd(__m256d __A, __m256d __B, __m256d __C, __mm
 }
 
 __m256d test_mm256_maskz_fmaddsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmaddsub_pd
+  // CHECK-LABEL: test_mm256_maskz_fmaddsub_pd
   // CHECK-NOT: fneg
   // CHECK: call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3188,7 +3190,7 @@ __m256d test_mm256_maskz_fmaddsub_pd(__mmask8 __U, __m256d __A, __m256d __B, __m
 }
 
 __m256d test_mm256_maskz_fmsubadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmsubadd_pd
+  // CHECK-LABEL: test_mm256_maskz_fmsubadd_pd
   // CHECK: [[NEG:%.+]] = fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> [[NEG]])
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3197,7 +3199,7 @@ __m256d test_mm256_maskz_fmsubadd_pd(__mmask8 __U, __m256d __A, __m256d __B, __m
 }
 
 __m128 test_mm_mask_fmaddsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_mask_fmaddsub_ps
+  // CHECK-LABEL: test_mm_mask_fmaddsub_ps
   // CHECK-NOT: fneg
   // CHECK: call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3206,7 +3208,7 @@ __m128 test_mm_mask_fmaddsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C
 }
 
 __m128 test_mm_mask_fmsubadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_mask_fmsubadd_ps
+  // CHECK-LABEL: test_mm_mask_fmsubadd_ps
   // CHECK: [[NEG:%.+]] = fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> [[NEG]])
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3215,7 +3217,7 @@ __m128 test_mm_mask_fmsubadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C
 }
 
 __m128 test_mm_mask3_fmaddsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmaddsub_ps
+  // CHECK-LABEL: test_mm_mask3_fmaddsub_ps
   // CHECK-NOT: fneg
   // CHECK: call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3224,7 +3226,7 @@ __m128 test_mm_mask3_fmaddsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __
 }
 
 __m128 test_mm_maskz_fmaddsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmaddsub_ps
+  // CHECK-LABEL: test_mm_maskz_fmaddsub_ps
   // CHECK-NOT: fneg
   // CHECK: call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3233,7 +3235,7 @@ __m128 test_mm_maskz_fmaddsub_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __
 }
 
 __m128 test_mm_maskz_fmsubadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmsubadd_ps
+  // CHECK-LABEL: test_mm_maskz_fmsubadd_ps
   // CHECK: [[NEG:%.+]] = fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> [[NEG]])
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3242,7 +3244,7 @@ __m128 test_mm_maskz_fmsubadd_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128 __
 }
 
 __m256 test_mm256_mask_fmaddsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmaddsub_ps
+  // CHECK-LABEL: test_mm256_mask_fmaddsub_ps
   // CHECK-NOT: fneg
   // CHECK: call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3250,7 +3252,7 @@ __m256 test_mm256_mask_fmaddsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256
 }
 
 __m256 test_mm256_mask_fmsubadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmsubadd_ps
+  // CHECK-LABEL: test_mm256_mask_fmsubadd_ps
   // CHECK: [[NEG:%.+]] = fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> [[NEG]])
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3258,7 +3260,7 @@ __m256 test_mm256_mask_fmsubadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256
 }
 
 __m256 test_mm256_mask3_fmaddsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmaddsub_ps
+  // CHECK-LABEL: test_mm256_mask3_fmaddsub_ps
   // CHECK-NOT: fneg
   // CHECK: call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3266,7 +3268,7 @@ __m256 test_mm256_mask3_fmaddsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8
 }
 
 __m256 test_mm256_maskz_fmaddsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmaddsub_ps
+  // CHECK-LABEL: test_mm256_maskz_fmaddsub_ps
   // CHECK-NOT: fneg
   // CHECK: call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3274,7 +3276,7 @@ __m256 test_mm256_maskz_fmaddsub_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256
 }
 
 __m256 test_mm256_maskz_fmsubadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmsubadd_ps
+  // CHECK-LABEL: test_mm256_maskz_fmsubadd_ps
   // CHECK: [[NEG:%.+]] = fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> [[NEG]])
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3282,7 +3284,7 @@ __m256 test_mm256_maskz_fmsubadd_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256
 }
 
 __m128d test_mm_mask3_fmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmsub_pd
+  // CHECK-LABEL: test_mm_mask3_fmsub_pd
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -3291,7 +3293,7 @@ __m128d test_mm_mask3_fmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 _
 }
 
 __m256d test_mm256_mask3_fmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmsub_pd
+  // CHECK-LABEL: test_mm256_mask3_fmsub_pd
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3300,7 +3302,7 @@ __m256d test_mm256_mask3_fmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask
 }
 
 __m128 test_mm_mask3_fmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmsub_ps
+  // CHECK-LABEL: test_mm_mask3_fmsub_ps
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3309,7 +3311,7 @@ __m128 test_mm_mask3_fmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
 }
 
 __m256 test_mm256_mask3_fmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmsub_ps
+  // CHECK-LABEL: test_mm256_mask3_fmsub_ps
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3317,7 +3319,7 @@ __m256 test_mm256_mask3_fmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __
 }
 
 __m128d test_mm_mask3_fmsubadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmsubadd_pd
+  // CHECK-LABEL: test_mm_mask3_fmsubadd_pd
   // CHECK: [[NEG:%.+]] = fneg <2 x double> %{{.+}}
   // CHECK: call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> [[NEG]])
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -3326,7 +3328,7 @@ __m128d test_mm_mask3_fmsubadd_pd(__m128d __A, __m128d __B, __m128d __C, __mmask
 }
 
 __m256d test_mm256_mask3_fmsubadd_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmsubadd_pd
+  // CHECK-LABEL: test_mm256_mask3_fmsubadd_pd
   // CHECK: [[NEG:%.+]] = fneg <4 x double> %{{.+}}
   // CHECK: call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> [[NEG]])
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3335,7 +3337,7 @@ __m256d test_mm256_mask3_fmsubadd_pd(__m256d __A, __m256d __B, __m256d __C, __mm
 }
 
 __m128 test_mm_mask3_fmsubadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmsubadd_ps
+  // CHECK-LABEL: test_mm_mask3_fmsubadd_ps
   // CHECK: [[NEG:%.+]] = fneg <4 x float> %{{.+}}
   // CHECK: call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> [[NEG]])
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3344,7 +3346,7 @@ __m128 test_mm_mask3_fmsubadd_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __
 }
 
 __m256 test_mm256_mask3_fmsubadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmsubadd_ps
+  // CHECK-LABEL: test_mm256_mask3_fmsubadd_ps
   // CHECK: [[NEG:%.+]] = fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> [[NEG]])
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3352,7 +3354,7 @@ __m256 test_mm256_mask3_fmsubadd_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8
 }
 
 __m128d test_mm_mask_fnmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_mask_fnmadd_pd
+  // CHECK-LABEL: test_mm_mask_fnmadd_pd
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -3361,7 +3363,7 @@ __m128d test_mm_mask_fnmadd_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d _
 }
 
 __m256d test_mm256_mask_fnmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_mask_fnmadd_pd
+  // CHECK-LABEL: test_mm256_mask_fnmadd_pd
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3370,7 +3372,7 @@ __m256d test_mm256_mask_fnmadd_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256
 }
 
 __m128 test_mm_mask_fnmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_mask_fnmadd_ps
+  // CHECK-LABEL: test_mm_mask_fnmadd_ps
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -3379,7 +3381,7 @@ __m128 test_mm_mask_fnmadd_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
 }
 
 __m256 test_mm256_mask_fnmadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_mask_fnmadd_ps
+  // CHECK-LABEL: test_mm256_mask_fnmadd_ps
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -3387,7 +3389,7 @@ __m256 test_mm256_mask_fnmadd_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __
 }
 
 __m128d test_mm_mask_fnmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d __C) {
-  // CHECK-LABEL: @test_mm_mask_fnmsub_pd
+  // CHECK-LABEL: test_mm_mask_fnmsub_pd
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
@@ -3397,7 +3399,7 @@ __m128d test_mm_mask_fnmsub_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128d _
 }
 
 __m128d test_mm_mask3_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmsub_pd
+  // CHECK-LABEL: test_mm_mask3_fnmsub_pd
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: fneg <2 x double> %{{.*}}
   // CHECK: call <2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
@@ -3407,7 +3409,7 @@ __m128d test_mm_mask3_fnmsub_pd(__m128d __A, __m128d __B, __m128d __C, __mmask8
 }
 
 __m256d test_mm256_mask_fnmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256d __C) {
-  // CHECK-LABEL: @test_mm256_mask_fnmsub_pd
+  // CHECK-LABEL: test_mm256_mask_fnmsub_pd
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
@@ -3417,7 +3419,7 @@ __m256d test_mm256_mask_fnmsub_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256
 }
 
 __m256d test_mm256_mask3_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fnmsub_pd
+  // CHECK-LABEL: test_mm256_mask3_fnmsub_pd
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: fneg <4 x double> %{{.*}}
   // CHECK: call <4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
@@ -3427,7 +3429,7 @@ __m256d test_mm256_mask3_fnmsub_pd(__m256d __A, __m256d __B, __m256d __C, __mmas
 }
 
 __m128 test_mm_mask_fnmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C) {
-  // CHECK-LABEL: @test_mm_mask_fnmsub_ps
+  // CHECK-LABEL: test_mm_mask_fnmsub_ps
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
@@ -3437,7 +3439,7 @@ __m128 test_mm_mask_fnmsub_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128 __C)
 }
 
 __m128 test_mm_mask3_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmsub_ps
+  // CHECK-LABEL: test_mm_mask3_fnmsub_ps
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: fneg <4 x float> %{{.*}}
   // CHECK: call <4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
@@ -3447,7 +3449,7 @@ __m128 test_mm_mask3_fnmsub_ps(__m128 __A, __m128 __B, __m128 __C, __mmask8 __U)
 }
 
 __m256 test_mm256_mask_fnmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __C) {
-  // CHECK-LABEL: @test_mm256_mask_fnmsub_ps
+  // CHECK-LABEL: test_mm256_mask_fnmsub_ps
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
@@ -3456,7 +3458,7 @@ __m256 test_mm256_mask_fnmsub_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256 __
 }
 
 __m256 test_mm256_mask3_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fnmsub_ps
+  // CHECK-LABEL: test_mm256_mask3_fnmsub_ps
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: fneg <8 x float> %{{.*}}
   // CHECK: call <8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
@@ -3465,1006 +3467,1006 @@ __m256 test_mm256_mask3_fnmsub_ps(__m256 __A, __m256 __B, __m256 __C, __mmask8 _
 }
 
 __m128d test_mm_mask_add_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_add_pd
+  // CHECK-LABEL: test_mm_mask_add_pd
   // CHECK: fadd <2 x double> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_add_pd(__W,__U,__A,__B); 
 }
 __m128d test_mm_maskz_add_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_add_pd
+  // CHECK-LABEL: test_mm_maskz_add_pd
   // CHECK: fadd <2 x double> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_add_pd(__U,__A,__B); 
 }
 __m256d test_mm256_mask_add_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_add_pd
+  // CHECK-LABEL: test_mm256_mask_add_pd
   // CHECK: fadd <4 x double> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_add_pd(__W,__U,__A,__B); 
 }
 __m256d test_mm256_maskz_add_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_add_pd
+  // CHECK-LABEL: test_mm256_maskz_add_pd
   // CHECK: fadd <4 x double> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_add_pd(__U,__A,__B); 
 }
 __m128 test_mm_mask_add_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_add_ps
+  // CHECK-LABEL: test_mm_mask_add_ps
   // CHECK: fadd <4 x float> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_add_ps(__W,__U,__A,__B); 
 }
 __m128 test_mm_maskz_add_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_add_ps
+  // CHECK-LABEL: test_mm_maskz_add_ps
   // CHECK: fadd <4 x float> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_add_ps(__U,__A,__B); 
 }
 __m256 test_mm256_mask_add_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_add_ps
+  // CHECK-LABEL: test_mm256_mask_add_ps
   // CHECK: fadd <8 x float> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_add_ps(__W,__U,__A,__B); 
 }
 __m256 test_mm256_maskz_add_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_add_ps
+  // CHECK-LABEL: test_mm256_maskz_add_ps
   // CHECK: fadd <8 x float> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_add_ps(__U,__A,__B); 
 }
 __m128i test_mm_mask_blend_epi32(__mmask8 __U, __m128i __A, __m128i __W) {
-  // CHECK-LABEL: @test_mm_mask_blend_epi32
+  // CHECK-LABEL: test_mm_mask_blend_epi32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_blend_epi32(__U,__A,__W); 
 }
 __m256i test_mm256_mask_blend_epi32(__mmask8 __U, __m256i __A, __m256i __W) {
-  // CHECK-LABEL: @test_mm256_mask_blend_epi32
+  // CHECK-LABEL: test_mm256_mask_blend_epi32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_blend_epi32(__U,__A,__W); 
 }
 __m128d test_mm_mask_blend_pd(__mmask8 __U, __m128d __A, __m128d __W) {
-  // CHECK-LABEL: @test_mm_mask_blend_pd
+  // CHECK-LABEL: test_mm_mask_blend_pd
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_blend_pd(__U,__A,__W); 
 }
 __m256d test_mm256_mask_blend_pd(__mmask8 __U, __m256d __A, __m256d __W) {
-  // CHECK-LABEL: @test_mm256_mask_blend_pd
+  // CHECK-LABEL: test_mm256_mask_blend_pd
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_blend_pd(__U,__A,__W); 
 }
 __m128 test_mm_mask_blend_ps(__mmask8 __U, __m128 __A, __m128 __W) {
-  // CHECK-LABEL: @test_mm_mask_blend_ps
+  // CHECK-LABEL: test_mm_mask_blend_ps
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_blend_ps(__U,__A,__W); 
 }
 __m256 test_mm256_mask_blend_ps(__mmask8 __U, __m256 __A, __m256 __W) {
-  // CHECK-LABEL: @test_mm256_mask_blend_ps
+  // CHECK-LABEL: test_mm256_mask_blend_ps
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_blend_ps(__U,__A,__W); 
 }
 __m128i test_mm_mask_blend_epi64(__mmask8 __U, __m128i __A, __m128i __W) {
-  // CHECK-LABEL: @test_mm_mask_blend_epi64
+  // CHECK-LABEL: test_mm_mask_blend_epi64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_blend_epi64(__U,__A,__W); 
 }
 __m256i test_mm256_mask_blend_epi64(__mmask8 __U, __m256i __A, __m256i __W) {
-  // CHECK-LABEL: @test_mm256_mask_blend_epi64
+  // CHECK-LABEL: test_mm256_mask_blend_epi64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_blend_epi64(__U,__A,__W); 
 }
 __m128d test_mm_mask_compress_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_compress_pd
+  // CHECK-LABEL: test_mm_mask_compress_pd
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm_mask_compress_pd(__W,__U,__A); 
 }
 __m128d test_mm_maskz_compress_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_compress_pd
+  // CHECK-LABEL: test_mm_maskz_compress_pd
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm_maskz_compress_pd(__U,__A); 
 }
 __m256d test_mm256_mask_compress_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_compress_pd
+  // CHECK-LABEL: test_mm256_mask_compress_pd
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm256_mask_compress_pd(__W,__U,__A); 
 }
 __m256d test_mm256_maskz_compress_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_compress_pd
+  // CHECK-LABEL: test_mm256_maskz_compress_pd
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm256_maskz_compress_pd(__U,__A); 
 }
 __m128i test_mm_mask_compress_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_compress_epi64
+  // CHECK-LABEL: test_mm_mask_compress_epi64
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm_mask_compress_epi64(__W,__U,__A); 
 }
 __m128i test_mm_maskz_compress_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_compress_epi64
+  // CHECK-LABEL: test_mm_maskz_compress_epi64
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm_maskz_compress_epi64(__U,__A); 
 }
 __m256i test_mm256_mask_compress_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_compress_epi64
+  // CHECK-LABEL: test_mm256_mask_compress_epi64
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm256_mask_compress_epi64(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_compress_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_compress_epi64
+  // CHECK-LABEL: test_mm256_maskz_compress_epi64
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm256_maskz_compress_epi64(__U,__A); 
 }
 __m128 test_mm_mask_compress_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_compress_ps
+  // CHECK-LABEL: test_mm_mask_compress_ps
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm_mask_compress_ps(__W,__U,__A); 
 }
 __m128 test_mm_maskz_compress_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_compress_ps
+  // CHECK-LABEL: test_mm_maskz_compress_ps
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm_maskz_compress_ps(__U,__A); 
 }
 __m256 test_mm256_mask_compress_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_compress_ps
+  // CHECK-LABEL: test_mm256_mask_compress_ps
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm256_mask_compress_ps(__W,__U,__A); 
 }
 __m256 test_mm256_maskz_compress_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_compress_ps
+  // CHECK-LABEL: test_mm256_maskz_compress_ps
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm256_maskz_compress_ps(__U,__A); 
 }
 __m128i test_mm_mask_compress_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_compress_epi32
+  // CHECK-LABEL: test_mm_mask_compress_epi32
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm_mask_compress_epi32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_compress_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_compress_epi32
+  // CHECK-LABEL: test_mm_maskz_compress_epi32
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm_maskz_compress_epi32(__U,__A); 
 }
 __m256i test_mm256_mask_compress_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_compress_epi32
+  // CHECK-LABEL: test_mm256_mask_compress_epi32
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm256_mask_compress_epi32(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_compress_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_compress_epi32
+  // CHECK-LABEL: test_mm256_maskz_compress_epi32
   // CHECK: @llvm.x86.avx512.mask.compress
   return _mm256_maskz_compress_epi32(__U,__A); 
 }
 void test_mm_mask_compressstoreu_pd(void *__P, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_compressstoreu_pd
+  // CHECK-LABEL: test_mm_mask_compressstoreu_pd
   // CHECK: @llvm.masked.compressstore.v2f64(<2 x double> %{{.*}}, ptr %{{.*}}, <2 x i1> %{{.*}})
   return _mm_mask_compressstoreu_pd(__P,__U,__A); 
 }
 void test_mm256_mask_compressstoreu_pd(void *__P, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_compressstoreu_pd
+  // CHECK-LABEL: test_mm256_mask_compressstoreu_pd
   // CHECK: @llvm.masked.compressstore.v4f64(<4 x double> %{{.*}}, ptr %{{.*}}, <4 x i1> %{{.*}})
   return _mm256_mask_compressstoreu_pd(__P,__U,__A); 
 }
 void test_mm_mask_compressstoreu_epi64(void *__P, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_compressstoreu_epi64
+  // CHECK-LABEL: test_mm_mask_compressstoreu_epi64
   // CHECK: @llvm.masked.compressstore.v2i64(<2 x i64> %{{.*}}, ptr %{{.*}}, <2 x i1> %{{.*}})
   return _mm_mask_compressstoreu_epi64(__P,__U,__A); 
 }
 void test_mm256_mask_compressstoreu_epi64(void *__P, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_compressstoreu_epi64
+  // CHECK-LABEL: test_mm256_mask_compressstoreu_epi64
   // CHECK: @llvm.masked.compressstore.v4i64(<4 x i64> %{{.*}}, ptr %{{.*}}, <4 x i1> %{{.*}})
   return _mm256_mask_compressstoreu_epi64(__P,__U,__A); 
 }
 void test_mm_mask_compressstoreu_ps(void *__P, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_compressstoreu_ps
+  // CHECK-LABEL: test_mm_mask_compressstoreu_ps
   // CHECK: @llvm.masked.compressstore.v4f32(<4 x float> %{{.*}}, ptr %{{.*}}, <4 x i1> %{{.*}})
   return _mm_mask_compressstoreu_ps(__P,__U,__A); 
 }
 void test_mm256_mask_compressstoreu_ps(void *__P, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_compressstoreu_ps
+  // CHECK-LABEL: test_mm256_mask_compressstoreu_ps
   // CHECK: @llvm.masked.compressstore.v8f32(<8 x float> %{{.*}}, ptr %{{.*}}, <8 x i1> %{{.*}})
   return _mm256_mask_compressstoreu_ps(__P,__U,__A); 
 }
 void test_mm_mask_compressstoreu_epi32(void *__P, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_compressstoreu_epi32
+  // CHECK-LABEL: test_mm_mask_compressstoreu_epi32
   // CHECK: @llvm.masked.compressstore.v4i32(<4 x i32> %{{.*}}, ptr %{{.*}}, <4 x i1> %{{.*}})
   return _mm_mask_compressstoreu_epi32(__P,__U,__A); 
 }
 void test_mm256_mask_compressstoreu_epi32(void *__P, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_compressstoreu_epi32
+  // CHECK-LABEL: test_mm256_mask_compressstoreu_epi32
   // CHECK: @llvm.masked.compressstore.v8i32(<8 x i32> %{{.*}}, ptr %{{.*}}, <8 x i1> %{{.*}})
   return _mm256_mask_compressstoreu_epi32(__P,__U,__A); 
 }
 __m128d test_mm_mask_cvtepi32_pd(__m128d __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi32_pd
+  // CHECK-LABEL: test_mm_mask_cvtepi32_pd
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i32> <i32 0, i32 1>
   // CHECK: sitofp <2 x i32> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> {{.*}}, <2 x double> {{.*}}, <2 x double> {{.*}}
   return _mm_mask_cvtepi32_pd(__W,__U,__A);
 }
 __m128d test_mm_maskz_cvtepi32_pd(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi32_pd
+  // CHECK-LABEL: test_mm_maskz_cvtepi32_pd
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i32> <i32 0, i32 1>
   // CHECK: sitofp <2 x i32> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> {{.*}}, <2 x double> {{.*}}, <2 x double> {{.*}}
   return _mm_maskz_cvtepi32_pd(__U,__A);
 }
 __m256d test_mm256_mask_cvtepi32_pd(__m256d __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi32_pd
+  // CHECK-LABEL: test_mm256_mask_cvtepi32_pd
   // CHECK: sitofp <4 x i32> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> {{.*}}, <4 x double> {{.*}}, <4 x double> {{.*}}
   return _mm256_mask_cvtepi32_pd(__W,__U,__A);
 }
 __m256d test_mm256_maskz_cvtepi32_pd(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi32_pd
+  // CHECK-LABEL: test_mm256_maskz_cvtepi32_pd
   // CHECK: sitofp <4 x i32> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> {{.*}}, <4 x double> {{.*}}, <4 x double> {{.*}}
   return _mm256_maskz_cvtepi32_pd(__U,__A);
 }
 __m128 test_mm_mask_cvtepi32_ps(__m128 __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi32_ps
+  // CHECK-LABEL: test_mm_mask_cvtepi32_ps
   // CHECK: sitofp <4 x i32> %{{.*}} to <4 x float>
   // CHECK: select <4 x i1> {{.*}}, <4 x float> {{.*}}, <4 x float> {{.*}}
   return _mm_mask_cvtepi32_ps(__W,__U,__A); 
 }
 __m128 test_mm_maskz_cvtepi32_ps(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi32_ps
+  // CHECK-LABEL: test_mm_maskz_cvtepi32_ps
   // CHECK: sitofp <4 x i32> %{{.*}} to <4 x float>
   // CHECK: select <4 x i1> {{.*}}, <4 x float> {{.*}}, <4 x float> {{.*}}
   return _mm_maskz_cvtepi32_ps(__U,__A); 
 }
 __m256 test_mm256_mask_cvtepi32_ps(__m256 __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi32_ps
+  // CHECK-LABEL: test_mm256_mask_cvtepi32_ps
   // CHECK: sitofp <8 x i32> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> {{.*}}, <8 x float> {{.*}}, <8 x float> {{.*}}
   return _mm256_mask_cvtepi32_ps(__W,__U,__A); 
 }
 __m256 test_mm256_maskz_cvtepi32_ps(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi32_ps
+  // CHECK-LABEL: test_mm256_maskz_cvtepi32_ps
   // CHECK: sitofp <8 x i32> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> {{.*}}, <8 x float> {{.*}}, <8 x float> {{.*}}
   return _mm256_maskz_cvtepi32_ps(__U,__A); 
 }
 __m128i test_mm_mask_cvtpd_epi32(__m128i __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtpd_epi32
+  // CHECK-LABEL: test_mm_mask_cvtpd_epi32
   // CHECK: @llvm.x86.avx512.mask.cvtpd2dq.128
   return _mm_mask_cvtpd_epi32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_cvtpd_epi32(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtpd_epi32
+  // CHECK-LABEL: test_mm_maskz_cvtpd_epi32
   // CHECK: @llvm.x86.avx512.mask.cvtpd2dq.128
   return _mm_maskz_cvtpd_epi32(__U,__A); 
 }
 __m128i test_mm256_mask_cvtpd_epi32(__m128i __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtpd_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtpd_epi32
   // CHECK: @llvm.x86.avx.cvt.pd2dq.256
   // CHECK: select <4 x i1> {{.*}}, <4 x i32> {{.*}}, <4 x i32> {{.*}}
   return _mm256_mask_cvtpd_epi32(__W,__U,__A); 
 }
 __m128i test_mm256_maskz_cvtpd_epi32(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtpd_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvtpd_epi32
   // CHECK: @llvm.x86.avx.cvt.pd2dq.256
   // CHECK: select <4 x i1> {{.*}}, <4 x i32> {{.*}}, <4 x i32> {{.*}}
   return _mm256_maskz_cvtpd_epi32(__U,__A); 
 }
 __m128 test_mm_mask_cvtpd_ps(__m128 __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtpd_ps
+  // CHECK-LABEL: test_mm_mask_cvtpd_ps
   // CHECK: @llvm.x86.avx512.mask.cvtpd2ps
   return _mm_mask_cvtpd_ps(__W,__U,__A); 
 }
 __m128 test_mm_maskz_cvtpd_ps(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtpd_ps
+  // CHECK-LABEL: test_mm_maskz_cvtpd_ps
   // CHECK: @llvm.x86.avx512.mask.cvtpd2ps
   return _mm_maskz_cvtpd_ps(__U,__A); 
 }
 __m128 test_mm256_mask_cvtpd_ps(__m128 __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtpd_ps
+  // CHECK-LABEL: test_mm256_mask_cvtpd_ps
   // CHECK: @llvm.x86.avx.cvt.pd2.ps.256
   // CHECK: select <4 x i1> {{.*}}, <4 x float> {{.*}}, <4 x float> {{.*}}
   return _mm256_mask_cvtpd_ps(__W,__U,__A); 
 }
 __m128 test_mm256_maskz_cvtpd_ps(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtpd_ps
+  // CHECK-LABEL: test_mm256_maskz_cvtpd_ps
   // CHECK: @llvm.x86.avx.cvt.pd2.ps.256
   // CHECK: select <4 x i1> {{.*}}, <4 x float> {{.*}}, <4 x float> {{.*}}
   return _mm256_maskz_cvtpd_ps(__U,__A); 
 }
 __m128i test_mm_cvtpd_epu32(__m128d __A) {
-  // CHECK-LABEL: @test_mm_cvtpd_epu32
+  // CHECK-LABEL: test_mm_cvtpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtpd2udq.128
   return _mm_cvtpd_epu32(__A); 
 }
 __m128i test_mm_mask_cvtpd_epu32(__m128i __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtpd_epu32
+  // CHECK-LABEL: test_mm_mask_cvtpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtpd2udq.128
   return _mm_mask_cvtpd_epu32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_cvtpd_epu32(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtpd_epu32
+  // CHECK-LABEL: test_mm_maskz_cvtpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtpd2udq.128
   return _mm_maskz_cvtpd_epu32(__U,__A); 
 }
 __m128i test_mm256_cvtpd_epu32(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_cvtpd_epu32
+  // CHECK-LABEL: test_mm256_cvtpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtpd2udq.256
   return _mm256_cvtpd_epu32(__A); 
 }
 __m128i test_mm256_mask_cvtpd_epu32(__m128i __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtpd_epu32
+  // CHECK-LABEL: test_mm256_mask_cvtpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtpd2udq.256
   return _mm256_mask_cvtpd_epu32(__W,__U,__A); 
 }
 __m128i test_mm256_maskz_cvtpd_epu32(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtpd_epu32
+  // CHECK-LABEL: test_mm256_maskz_cvtpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtpd2udq.256
   return _mm256_maskz_cvtpd_epu32(__U,__A); 
 }
 __m128i test_mm_mask_cvtps_epi32(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtps_epi32
+  // CHECK-LABEL: test_mm_mask_cvtps_epi32
   // CHECK: @llvm.x86.sse2.cvtps2dq
   // CHECK: select <4 x i1> {{.*}}, <4 x i32> {{.*}}, <4 x i32> {{.*}}
   return _mm_mask_cvtps_epi32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_cvtps_epi32(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtps_epi32
+  // CHECK-LABEL: test_mm_maskz_cvtps_epi32
   // CHECK: @llvm.x86.sse2.cvtps2dq
   // CHECK: select <4 x i1> {{.*}}, <4 x i32> {{.*}}, <4 x i32> {{.*}}
   return _mm_maskz_cvtps_epi32(__U,__A); 
 }
 __m256i test_mm256_mask_cvtps_epi32(__m256i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtps_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtps_epi32
   // CHECK: @llvm.x86.avx.cvt.ps2dq.256
   // CHECK: select <8 x i1> {{.*}}, <8 x i32> {{.*}}, <8 x i32> {{.*}}
   return _mm256_mask_cvtps_epi32(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_cvtps_epi32(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtps_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvtps_epi32
   // CHECK: @llvm.x86.avx.cvt.ps2dq.256
   // CHECK: select <8 x i1> {{.*}}, <8 x i32> {{.*}}, <8 x i32> {{.*}}
   return _mm256_maskz_cvtps_epi32(__U,__A); 
 }
 __m128d test_mm_mask_cvtps_pd(__m128d __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtps_pd
+  // CHECK-LABEL: test_mm_mask_cvtps_pd
   // CHECK: fpext <2 x float> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> {{.*}}, <2 x double> {{.*}}, <2 x double> {{.*}}
   return _mm_mask_cvtps_pd(__W,__U,__A); 
 }
 __m128d test_mm_maskz_cvtps_pd(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtps_pd
+  // CHECK-LABEL: test_mm_maskz_cvtps_pd
   // CHECK: fpext <2 x float> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> {{.*}}, <2 x double> {{.*}}, <2 x double> {{.*}}
   return _mm_maskz_cvtps_pd(__U,__A); 
 }
 __m256d test_mm256_mask_cvtps_pd(__m256d __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtps_pd
+  // CHECK-LABEL: test_mm256_mask_cvtps_pd
   // CHECK: fpext <4 x float> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> {{.*}}, <4 x double> {{.*}}, <4 x double> {{.*}}
   return _mm256_mask_cvtps_pd(__W,__U,__A); 
 }
 __m256d test_mm256_maskz_cvtps_pd(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtps_pd
+  // CHECK-LABEL: test_mm256_maskz_cvtps_pd
   // CHECK: fpext <4 x float> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> {{.*}}, <4 x double> {{.*}}, <4 x double> {{.*}}
   return _mm256_maskz_cvtps_pd(__U,__A); 
 }
 __m128i test_mm_cvtps_epu32(__m128 __A) {
-  // CHECK-LABEL: @test_mm_cvtps_epu32
+  // CHECK-LABEL: test_mm_cvtps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtps2udq.128
   return _mm_cvtps_epu32(__A); 
 }
 __m128i test_mm_mask_cvtps_epu32(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtps_epu32
+  // CHECK-LABEL: test_mm_mask_cvtps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtps2udq.128
   return _mm_mask_cvtps_epu32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_cvtps_epu32(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtps_epu32
+  // CHECK-LABEL: test_mm_maskz_cvtps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtps2udq.128
   return _mm_maskz_cvtps_epu32(__U,__A); 
 }
 __m256i test_mm256_cvtps_epu32(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_cvtps_epu32
+  // CHECK-LABEL: test_mm256_cvtps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtps2udq.256
   return _mm256_cvtps_epu32(__A); 
 }
 __m256i test_mm256_mask_cvtps_epu32(__m256i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtps_epu32
+  // CHECK-LABEL: test_mm256_mask_cvtps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtps2udq.256
   return _mm256_mask_cvtps_epu32(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_cvtps_epu32(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtps_epu32
+  // CHECK-LABEL: test_mm256_maskz_cvtps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvtps2udq.256
   return _mm256_maskz_cvtps_epu32(__U,__A); 
 }
 __m128i test_mm_mask_cvttpd_epi32(__m128i __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_cvttpd_epi32
+  // CHECK-LABEL: test_mm_mask_cvttpd_epi32
   // CHECK: @llvm.x86.avx512.mask.cvttpd2dq.128
   return _mm_mask_cvttpd_epi32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_cvttpd_epi32(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvttpd_epi32
+  // CHECK-LABEL: test_mm_maskz_cvttpd_epi32
   // CHECK: @llvm.x86.avx512.mask.cvttpd2dq.128
   return _mm_maskz_cvttpd_epi32(__U,__A); 
 }
 __m128i test_mm256_mask_cvttpd_epi32(__m128i __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvttpd_epi32
+  // CHECK-LABEL: test_mm256_mask_cvttpd_epi32
   // CHECK: @llvm.x86.avx.cvtt.pd2dq.256
   // CHECK: select <4 x i1> {{.*}}, <4 x i32> {{.*}}, <4 x i32> {{.*}}
   return _mm256_mask_cvttpd_epi32(__W,__U,__A); 
 }
 __m128i test_mm256_maskz_cvttpd_epi32(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvttpd_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvttpd_epi32
   // CHECK: @llvm.x86.avx.cvtt.pd2dq.256
   // CHECK: select <4 x i1> {{.*}}, <4 x i32> {{.*}}, <4 x i32> {{.*}}
   return _mm256_maskz_cvttpd_epi32(__U,__A); 
 }
 __m128i test_mm_cvttpd_epu32(__m128d __A) {
-  // CHECK-LABEL: @test_mm_cvttpd_epu32
+  // CHECK-LABEL: test_mm_cvttpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttpd2udq.128
   return _mm_cvttpd_epu32(__A); 
 }
 __m128i test_mm_mask_cvttpd_epu32(__m128i __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_cvttpd_epu32
+  // CHECK-LABEL: test_mm_mask_cvttpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttpd2udq.128
   return _mm_mask_cvttpd_epu32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_cvttpd_epu32(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvttpd_epu32
+  // CHECK-LABEL: test_mm_maskz_cvttpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttpd2udq.128
   return _mm_maskz_cvttpd_epu32(__U,__A); 
 }
 __m128i test_mm256_cvttpd_epu32(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_cvttpd_epu32
+  // CHECK-LABEL: test_mm256_cvttpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttpd2udq.256
   return _mm256_cvttpd_epu32(__A); 
 }
 __m128i test_mm256_mask_cvttpd_epu32(__m128i __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvttpd_epu32
+  // CHECK-LABEL: test_mm256_mask_cvttpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttpd2udq.256
   return _mm256_mask_cvttpd_epu32(__W,__U,__A); 
 }
 __m128i test_mm256_maskz_cvttpd_epu32(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvttpd_epu32
+  // CHECK-LABEL: test_mm256_maskz_cvttpd_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttpd2udq.256
   return _mm256_maskz_cvttpd_epu32(__U,__A); 
 }
 __m128i test_mm_mask_cvttps_epi32(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvttps_epi32
+  // CHECK-LABEL: test_mm_mask_cvttps_epi32
   // CHECK: @llvm.x86.sse2.cvttps2dq
   // CHECK: select <4 x i1> {{.*}}, <4 x i32> {{.*}}, <4 x i32> {{.*}}
   return _mm_mask_cvttps_epi32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_cvttps_epi32(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvttps_epi32
+  // CHECK-LABEL: test_mm_maskz_cvttps_epi32
   // CHECK: @llvm.x86.sse2.cvttps2dq
   // CHECK: select <4 x i1> {{.*}}, <4 x i32> {{.*}}, <4 x i32> {{.*}}
   return _mm_maskz_cvttps_epi32(__U,__A); 
 }
 __m256i test_mm256_mask_cvttps_epi32(__m256i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvttps_epi32
+  // CHECK-LABEL: test_mm256_mask_cvttps_epi32
   // CHECK: @llvm.x86.avx.cvtt.ps2dq.256
   // CHECK: select <8 x i1> {{.*}}, <8 x i32> {{.*}}, <8 x i32> {{.*}}
   return _mm256_mask_cvttps_epi32(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_cvttps_epi32(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvttps_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvttps_epi32
   // CHECK: @llvm.x86.avx.cvtt.ps2dq.256
   // CHECK: select <8 x i1> {{.*}}, <8 x i32> {{.*}}, <8 x i32> {{.*}}
   return _mm256_maskz_cvttps_epi32(__U,__A); 
 }
 __m128i test_mm_cvttps_epu32(__m128 __A) {
-  // CHECK-LABEL: @test_mm_cvttps_epu32
+  // CHECK-LABEL: test_mm_cvttps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttps2udq.128
   return _mm_cvttps_epu32(__A); 
 }
 __m128i test_mm_mask_cvttps_epu32(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvttps_epu32
+  // CHECK-LABEL: test_mm_mask_cvttps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttps2udq.128
   return _mm_mask_cvttps_epu32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_cvttps_epu32(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvttps_epu32
+  // CHECK-LABEL: test_mm_maskz_cvttps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttps2udq.128
   return _mm_maskz_cvttps_epu32(__U,__A); 
 }
 __m256i test_mm256_cvttps_epu32(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_cvttps_epu32
+  // CHECK-LABEL: test_mm256_cvttps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttps2udq.256
   return _mm256_cvttps_epu32(__A); 
 }
 __m256i test_mm256_mask_cvttps_epu32(__m256i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvttps_epu32
+  // CHECK-LABEL: test_mm256_mask_cvttps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttps2udq.256
   return _mm256_mask_cvttps_epu32(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_cvttps_epu32(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvttps_epu32
+  // CHECK-LABEL: test_mm256_maskz_cvttps_epu32
   // CHECK: @llvm.x86.avx512.mask.cvttps2udq.256
   return _mm256_maskz_cvttps_epu32(__U,__A); 
 }
 __m128d test_mm_cvtepu32_pd(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepu32_pd
+  // CHECK-LABEL: test_mm_cvtepu32_pd
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i32> <i32 0, i32 1>
   // CHECK: uitofp <2 x i32> %{{.*}} to <2 x double>
   return _mm_cvtepu32_pd(__A);
 }
 __m128d test_mm_mask_cvtepu32_pd(__m128d __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu32_pd
+  // CHECK-LABEL: test_mm_mask_cvtepu32_pd
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i32> <i32 0, i32 1>
   // CHECK: uitofp <2 x i32> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> {{.*}}, <2 x double> {{.*}}, <2 x double> {{.*}}
   return _mm_mask_cvtepu32_pd(__W,__U,__A);
 }
 __m128d test_mm_maskz_cvtepu32_pd(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu32_pd
+  // CHECK-LABEL: test_mm_maskz_cvtepu32_pd
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i32> <i32 0, i32 1>
   // CHECK: uitofp <2 x i32> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> {{.*}}, <2 x double> {{.*}}, <2 x double> {{.*}}
   return _mm_maskz_cvtepu32_pd(__U,__A);
 }
 __m256d test_mm256_cvtepu32_pd(__m128i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepu32_pd
+  // CHECK-LABEL: test_mm256_cvtepu32_pd
   // CHECK: uitofp <4 x i32> %{{.*}} to <4 x double>
   return _mm256_cvtepu32_pd(__A);
 }
 __m256d test_mm256_mask_cvtepu32_pd(__m256d __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu32_pd
+  // CHECK-LABEL: test_mm256_mask_cvtepu32_pd
   // CHECK: uitofp <4 x i32> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> {{.*}}, <4 x double> {{.*}}, <4 x double> {{.*}}
   return _mm256_mask_cvtepu32_pd(__W,__U,__A);
 }
 __m256d test_mm256_maskz_cvtepu32_pd(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu32_pd
+  // CHECK-LABEL: test_mm256_maskz_cvtepu32_pd
   // CHECK: uitofp <4 x i32> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> {{.*}}, <4 x double> {{.*}}, <4 x double> {{.*}}
   return _mm256_maskz_cvtepu32_pd(__U,__A);
 }
 __m128 test_mm_cvtepu32_ps(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepu32_ps
+  // CHECK-LABEL: test_mm_cvtepu32_ps
   // CHECK: uitofp <4 x i32> %{{.*}} to <4 x float>
   return _mm_cvtepu32_ps(__A); 
 }
 __m128 test_mm_mask_cvtepu32_ps(__m128 __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu32_ps
+  // CHECK-LABEL: test_mm_mask_cvtepu32_ps
   // CHECK: uitofp <4 x i32> %{{.*}} to <4 x float>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_cvtepu32_ps(__W,__U,__A); 
 }
 __m128 test_mm_maskz_cvtepu32_ps(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu32_ps
+  // CHECK-LABEL: test_mm_maskz_cvtepu32_ps
   // CHECK: uitofp <4 x i32> %{{.*}} to <4 x float>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_cvtepu32_ps(__U,__A); 
 }
 __m256 test_mm256_cvtepu32_ps(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepu32_ps
+  // CHECK-LABEL: test_mm256_cvtepu32_ps
   // CHECK: uitofp <8 x i32> %{{.*}} to <8 x float>
   return _mm256_cvtepu32_ps(__A); 
 }
 __m256 test_mm256_mask_cvtepu32_ps(__m256 __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu32_ps
+  // CHECK-LABEL: test_mm256_mask_cvtepu32_ps
   // CHECK: uitofp <8 x i32> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_cvtepu32_ps(__W,__U,__A); 
 }
 __m256 test_mm256_maskz_cvtepu32_ps(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu32_ps
+  // CHECK-LABEL: test_mm256_maskz_cvtepu32_ps
   // CHECK: uitofp <8 x i32> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_cvtepu32_ps(__U,__A); 
 }
 __m128d test_mm_mask_div_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_div_pd
+  // CHECK-LABEL: test_mm_mask_div_pd
   // CHECK: fdiv <2 x double> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_div_pd(__W,__U,__A,__B); 
 }
 __m128d test_mm_maskz_div_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_div_pd
+  // CHECK-LABEL: test_mm_maskz_div_pd
   // CHECK: fdiv <2 x double> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_div_pd(__U,__A,__B); 
 }
 __m256d test_mm256_mask_div_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_div_pd
+  // CHECK-LABEL: test_mm256_mask_div_pd
   // CHECK: fdiv <4 x double> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_div_pd(__W,__U,__A,__B); 
 }
 __m256d test_mm256_maskz_div_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_div_pd
+  // CHECK-LABEL: test_mm256_maskz_div_pd
   // CHECK: fdiv <4 x double> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_div_pd(__U,__A,__B); 
 }
 __m128 test_mm_mask_div_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_div_ps
+  // CHECK-LABEL: test_mm_mask_div_ps
   // CHECK: fdiv <4 x float> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_div_ps(__W,__U,__A,__B); 
 }
 __m128 test_mm_maskz_div_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_div_ps
+  // CHECK-LABEL: test_mm_maskz_div_ps
   // CHECK: fdiv <4 x float> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_div_ps(__U,__A,__B); 
 }
 __m256 test_mm256_mask_div_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_div_ps
+  // CHECK-LABEL: test_mm256_mask_div_ps
   // CHECK: fdiv <8 x float> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_div_ps(__W,__U,__A,__B); 
 }
 __m256 test_mm256_maskz_div_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_div_ps
+  // CHECK-LABEL: test_mm256_maskz_div_ps
   // CHECK: fdiv <8 x float> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_div_ps(__U,__A,__B); 
 }
 __m128d test_mm_mask_expand_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_expand_pd
+  // CHECK-LABEL: test_mm_mask_expand_pd
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm_mask_expand_pd(__W,__U,__A); 
 }
 __m128d test_mm_maskz_expand_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_expand_pd
+  // CHECK-LABEL: test_mm_maskz_expand_pd
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm_maskz_expand_pd(__U,__A); 
 }
 __m256d test_mm256_mask_expand_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_expand_pd
+  // CHECK-LABEL: test_mm256_mask_expand_pd
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm256_mask_expand_pd(__W,__U,__A); 
 }
 __m256d test_mm256_maskz_expand_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_expand_pd
+  // CHECK-LABEL: test_mm256_maskz_expand_pd
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm256_maskz_expand_pd(__U,__A); 
 }
 __m128i test_mm_mask_expand_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_expand_epi64
+  // CHECK-LABEL: test_mm_mask_expand_epi64
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm_mask_expand_epi64(__W,__U,__A); 
 }
 __m128i test_mm_maskz_expand_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_expand_epi64
+  // CHECK-LABEL: test_mm_maskz_expand_epi64
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm_maskz_expand_epi64(__U,__A); 
 }
 __m256i test_mm256_mask_expand_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_expand_epi64
+  // CHECK-LABEL: test_mm256_mask_expand_epi64
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm256_mask_expand_epi64(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_expand_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_expand_epi64
+  // CHECK-LABEL: test_mm256_maskz_expand_epi64
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm256_maskz_expand_epi64(__U,__A); 
 }
 __m128d test_mm_mask_expandloadu_pd(__m128d __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_expandloadu_pd
+  // CHECK-LABEL: test_mm_mask_expandloadu_pd
   // CHECK: @llvm.masked.expandload.v2f64(ptr %{{.*}}, <2 x i1> %{{.*}}, <2 x double> %{{.*}})
   return _mm_mask_expandloadu_pd(__W,__U,__P); 
 }
 __m128d test_mm_maskz_expandloadu_pd(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_expandloadu_pd
+  // CHECK-LABEL: test_mm_maskz_expandloadu_pd
   // CHECK: @llvm.masked.expandload.v2f64(ptr %{{.*}}, <2 x i1> %{{.*}}, <2 x double> %{{.*}})
   return _mm_maskz_expandloadu_pd(__U,__P); 
 }
 __m256d test_mm256_mask_expandloadu_pd(__m256d __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_expandloadu_pd
+  // CHECK-LABEL: test_mm256_mask_expandloadu_pd
   // CHECK: @llvm.masked.expandload.v4f64(ptr %{{.*}}, <4 x i1> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_mask_expandloadu_pd(__W,__U,__P); 
 }
 __m256d test_mm256_maskz_expandloadu_pd(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_expandloadu_pd
+  // CHECK-LABEL: test_mm256_maskz_expandloadu_pd
   // CHECK: @llvm.masked.expandload.v4f64(ptr %{{.*}}, <4 x i1> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_maskz_expandloadu_pd(__U,__P); 
 }
 __m128i test_mm_mask_expandloadu_epi64(__m128i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_expandloadu_epi64
+  // CHECK-LABEL: test_mm_mask_expandloadu_epi64
   // CHECK: @llvm.masked.expandload.v2i64(ptr %{{.*}}, <2 x i1> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_mask_expandloadu_epi64(__W,__U,__P); 
 }
 __m128i test_mm_maskz_expandloadu_epi64(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_expandloadu_epi64
+  // CHECK-LABEL: test_mm_maskz_expandloadu_epi64
   // CHECK: @llvm.masked.expandload.v2i64(ptr %{{.*}}, <2 x i1> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_maskz_expandloadu_epi64(__U,__P); 
 }
 __m256i test_mm256_mask_expandloadu_epi64(__m256i __W, __mmask8 __U,   void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_expandloadu_epi64
+  // CHECK-LABEL: test_mm256_mask_expandloadu_epi64
   // CHECK: @llvm.masked.expandload.v4i64(ptr %{{.*}}, <4 x i1> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_mask_expandloadu_epi64(__W,__U,__P); 
 }
 __m256i test_mm256_maskz_expandloadu_epi64(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_expandloadu_epi64
+  // CHECK-LABEL: test_mm256_maskz_expandloadu_epi64
   // CHECK: @llvm.masked.expandload.v4i64(ptr %{{.*}}, <4 x i1> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_maskz_expandloadu_epi64(__U,__P); 
 }
 __m128 test_mm_mask_expandloadu_ps(__m128 __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_expandloadu_ps
+  // CHECK-LABEL: test_mm_mask_expandloadu_ps
   // CHECK: @llvm.masked.expandload.v4f32(ptr %{{.*}}, <4 x i1> %{{.*}}, <4 x float> %{{.*}})
   return _mm_mask_expandloadu_ps(__W,__U,__P); 
 }
 __m128 test_mm_maskz_expandloadu_ps(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_expandloadu_ps
+  // CHECK-LABEL: test_mm_maskz_expandloadu_ps
   // CHECK: @llvm.masked.expandload.v4f32(ptr %{{.*}}, <4 x i1> %{{.*}}, <4 x float> %{{.*}})
   return _mm_maskz_expandloadu_ps(__U,__P); 
 }
 __m256 test_mm256_mask_expandloadu_ps(__m256 __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_expandloadu_ps
+  // CHECK-LABEL: test_mm256_mask_expandloadu_ps
   // CHECK: @llvm.masked.expandload.v8f32(ptr %{{.*}}, <8 x i1> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_mask_expandloadu_ps(__W,__U,__P); 
 }
 __m256 test_mm256_maskz_expandloadu_ps(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_expandloadu_ps
+  // CHECK-LABEL: test_mm256_maskz_expandloadu_ps
   // CHECK: @llvm.masked.expandload.v8f32(ptr %{{.*}}, <8 x i1> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_maskz_expandloadu_ps(__U,__P); 
 }
 __m128i test_mm_mask_expandloadu_epi32(__m128i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_expandloadu_epi32
+  // CHECK-LABEL: test_mm_mask_expandloadu_epi32
   // CHECK: @llvm.masked.expandload.v4i32(ptr %{{.*}}, <4 x i1> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_mask_expandloadu_epi32(__W,__U,__P); 
 }
 __m128i test_mm_maskz_expandloadu_epi32(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_expandloadu_epi32
+  // CHECK-LABEL: test_mm_maskz_expandloadu_epi32
   // CHECK: @llvm.masked.expandload.v4i32(ptr %{{.*}}, <4 x i1> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_maskz_expandloadu_epi32(__U,__P); 
 }
 __m256i test_mm256_mask_expandloadu_epi32(__m256i __W, __mmask8 __U,   void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_expandloadu_epi32
+  // CHECK-LABEL: test_mm256_mask_expandloadu_epi32
   // CHECK: @llvm.masked.expandload.v8i32(ptr %{{.*}}, <8 x i1> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_mask_expandloadu_epi32(__W,__U,__P); 
 }
 __m256i test_mm256_maskz_expandloadu_epi32(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_expandloadu_epi32
+  // CHECK-LABEL: test_mm256_maskz_expandloadu_epi32
   // CHECK: @llvm.masked.expandload.v8i32(ptr %{{.*}}, <8 x i1> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_maskz_expandloadu_epi32(__U,__P); 
 }
 __m128 test_mm_mask_expand_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_expand_ps
+  // CHECK-LABEL: test_mm_mask_expand_ps
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm_mask_expand_ps(__W,__U,__A); 
 }
 __m128 test_mm_maskz_expand_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_expand_ps
+  // CHECK-LABEL: test_mm_maskz_expand_ps
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm_maskz_expand_ps(__U,__A); 
 }
 __m256 test_mm256_mask_expand_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_expand_ps
+  // CHECK-LABEL: test_mm256_mask_expand_ps
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm256_mask_expand_ps(__W,__U,__A); 
 }
 __m256 test_mm256_maskz_expand_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_expand_ps
+  // CHECK-LABEL: test_mm256_maskz_expand_ps
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm256_maskz_expand_ps(__U,__A); 
 }
 __m128i test_mm_mask_expand_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_expand_epi32
+  // CHECK-LABEL: test_mm_mask_expand_epi32
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm_mask_expand_epi32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_expand_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_expand_epi32
+  // CHECK-LABEL: test_mm_maskz_expand_epi32
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm_maskz_expand_epi32(__U,__A); 
 }
 __m256i test_mm256_mask_expand_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_expand_epi32
+  // CHECK-LABEL: test_mm256_mask_expand_epi32
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm256_mask_expand_epi32(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_expand_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_expand_epi32
+  // CHECK-LABEL: test_mm256_maskz_expand_epi32
   // CHECK: @llvm.x86.avx512.mask.expand
   return _mm256_maskz_expand_epi32(__U,__A); 
 }
 __m128d test_mm_getexp_pd(__m128d __A) {
-  // CHECK-LABEL: @test_mm_getexp_pd
+  // CHECK-LABEL: test_mm_getexp_pd
   // CHECK: @llvm.x86.avx512.mask.getexp.pd.128
   return _mm_getexp_pd(__A); 
 }
 __m128d test_mm_mask_getexp_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_getexp_pd
+  // CHECK-LABEL: test_mm_mask_getexp_pd
   // CHECK: @llvm.x86.avx512.mask.getexp.pd.128
   return _mm_mask_getexp_pd(__W,__U,__A); 
 }
 __m128d test_mm_maskz_getexp_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_getexp_pd
+  // CHECK-LABEL: test_mm_maskz_getexp_pd
   // CHECK: @llvm.x86.avx512.mask.getexp.pd.128
   return _mm_maskz_getexp_pd(__U,__A); 
 }
 __m256d test_mm256_getexp_pd(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_getexp_pd
+  // CHECK-LABEL: test_mm256_getexp_pd
   // CHECK: @llvm.x86.avx512.mask.getexp.pd.256
   return _mm256_getexp_pd(__A); 
 }
 __m256d test_mm256_mask_getexp_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_getexp_pd
+  // CHECK-LABEL: test_mm256_mask_getexp_pd
   // CHECK: @llvm.x86.avx512.mask.getexp.pd.256
   return _mm256_mask_getexp_pd(__W,__U,__A); 
 }
 __m256d test_mm256_maskz_getexp_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_getexp_pd
+  // CHECK-LABEL: test_mm256_maskz_getexp_pd
   // CHECK: @llvm.x86.avx512.mask.getexp.pd.256
   return _mm256_maskz_getexp_pd(__U,__A); 
 }
 __m128 test_mm_getexp_ps(__m128 __A) {
-  // CHECK-LABEL: @test_mm_getexp_ps
+  // CHECK-LABEL: test_mm_getexp_ps
   // CHECK: @llvm.x86.avx512.mask.getexp.ps.128
   return _mm_getexp_ps(__A); 
 }
 __m128 test_mm_mask_getexp_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_getexp_ps
+  // CHECK-LABEL: test_mm_mask_getexp_ps
   // CHECK: @llvm.x86.avx512.mask.getexp.ps.128
   return _mm_mask_getexp_ps(__W,__U,__A); 
 }
 __m128 test_mm_maskz_getexp_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_getexp_ps
+  // CHECK-LABEL: test_mm_maskz_getexp_ps
   // CHECK: @llvm.x86.avx512.mask.getexp.ps.128
   return _mm_maskz_getexp_ps(__U,__A); 
 }
 __m256 test_mm256_getexp_ps(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_getexp_ps
+  // CHECK-LABEL: test_mm256_getexp_ps
   // CHECK: @llvm.x86.avx512.mask.getexp.ps.256
   return _mm256_getexp_ps(__A); 
 }
 __m256 test_mm256_mask_getexp_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_getexp_ps
+  // CHECK-LABEL: test_mm256_mask_getexp_ps
   // CHECK: @llvm.x86.avx512.mask.getexp.ps.256
   return _mm256_mask_getexp_ps(__W,__U,__A); 
 }
 __m256 test_mm256_maskz_getexp_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_getexp_ps
+  // CHECK-LABEL: test_mm256_maskz_getexp_ps
   // CHECK: @llvm.x86.avx512.mask.getexp.ps.256
   return _mm256_maskz_getexp_ps(__U,__A); 
 }
 __m128d test_mm_mask_max_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_max_pd
+  // CHECK-LABEL: test_mm_mask_max_pd
   // CHECK: @llvm.x86.sse2.max.pd
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_max_pd(__W,__U,__A,__B); 
 }
 __m128d test_mm_maskz_max_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_pd
+  // CHECK-LABEL: test_mm_maskz_max_pd
   // CHECK: @llvm.x86.sse2.max.pd
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_max_pd(__U,__A,__B); 
 }
 __m256d test_mm256_mask_max_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_pd
+  // CHECK-LABEL: test_mm256_mask_max_pd
   // CHECK: @llvm.x86.avx.max.pd.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_max_pd(__W,__U,__A,__B); 
 }
 __m256d test_mm256_maskz_max_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_pd
+  // CHECK-LABEL: test_mm256_maskz_max_pd
   // CHECK: @llvm.x86.avx.max.pd.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_max_pd(__U,__A,__B); 
 }
 __m128 test_mm_mask_max_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_max_ps
+  // CHECK-LABEL: test_mm_mask_max_ps
   // CHECK: @llvm.x86.sse.max.ps
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_max_ps(__W,__U,__A,__B); 
 }
 __m128 test_mm_maskz_max_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_ps
+  // CHECK-LABEL: test_mm_maskz_max_ps
   // CHECK: @llvm.x86.sse.max.ps
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_max_ps(__U,__A,__B); 
 }
 __m256 test_mm256_mask_max_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_ps
+  // CHECK-LABEL: test_mm256_mask_max_ps
   // CHECK: @llvm.x86.avx.max.ps.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_max_ps(__W,__U,__A,__B); 
 }
 __m256 test_mm256_maskz_max_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_ps
+  // CHECK-LABEL: test_mm256_maskz_max_ps
   // CHECK: @llvm.x86.avx.max.ps.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_max_ps(__U,__A,__B); 
 }
 __m128d test_mm_mask_min_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_min_pd
+  // CHECK-LABEL: test_mm_mask_min_pd
   // CHECK: @llvm.x86.sse2.min.pd
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_min_pd(__W,__U,__A,__B); 
 }
 __m128d test_mm_maskz_min_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_pd
+  // CHECK-LABEL: test_mm_maskz_min_pd
   // CHECK: @llvm.x86.sse2.min.pd
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_min_pd(__U,__A,__B); 
 }
 __m256d test_mm256_mask_min_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_pd
+  // CHECK-LABEL: test_mm256_mask_min_pd
   // CHECK: @llvm.x86.avx.min.pd.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_min_pd(__W,__U,__A,__B); 
 }
 __m256d test_mm256_maskz_min_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_pd
+  // CHECK-LABEL: test_mm256_maskz_min_pd
   // CHECK: @llvm.x86.avx.min.pd.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_min_pd(__U,__A,__B); 
 }
 __m128 test_mm_mask_min_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_min_ps
+  // CHECK-LABEL: test_mm_mask_min_ps
   // CHECK: @llvm.x86.sse.min.ps
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_min_ps(__W,__U,__A,__B); 
 }
 __m128 test_mm_maskz_min_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_ps
+  // CHECK-LABEL: test_mm_maskz_min_ps
   // CHECK: @llvm.x86.sse.min.ps
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_min_ps(__U,__A,__B); 
 }
 __m256 test_mm256_mask_min_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_ps
+  // CHECK-LABEL: test_mm256_mask_min_ps
   // CHECK: @llvm.x86.avx.min.ps.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_min_ps(__W,__U,__A,__B); 
 }
 __m256 test_mm256_maskz_min_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_ps
+  // CHECK-LABEL: test_mm256_maskz_min_ps
   // CHECK: @llvm.x86.avx.min.ps.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_min_ps(__U,__A,__B); 
 }
 __m128d test_mm_mask_mul_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_mul_pd
+  // CHECK-LABEL: test_mm_mask_mul_pd
   // CHECK: fmul <2 x double> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_mul_pd(__W,__U,__A,__B); 
 }
 __m128d test_mm_maskz_mul_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_mul_pd
+  // CHECK-LABEL: test_mm_maskz_mul_pd
   // CHECK: fmul <2 x double> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_mul_pd(__U,__A,__B); 
 }
 __m256d test_mm256_mask_mul_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_mul_pd
+  // CHECK-LABEL: test_mm256_mask_mul_pd
   // CHECK: fmul <4 x double> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_mul_pd(__W,__U,__A,__B); 
 }
 __m256d test_mm256_maskz_mul_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_mul_pd
+  // CHECK-LABEL: test_mm256_maskz_mul_pd
   // CHECK: fmul <4 x double> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_mul_pd(__U,__A,__B); 
 }
 __m128 test_mm_mask_mul_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_mul_ps
+  // CHECK-LABEL: test_mm_mask_mul_ps
   // CHECK: fmul <4 x float> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_mul_ps(__W,__U,__A,__B); 
 }
 __m128 test_mm_maskz_mul_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_mul_ps
+  // CHECK-LABEL: test_mm_maskz_mul_ps
   // CHECK: fmul <4 x float> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_mul_ps(__U,__A,__B); 
 }
 __m256 test_mm256_mask_mul_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_mul_ps
+  // CHECK-LABEL: test_mm256_mask_mul_ps
   // CHECK: fmul <8 x float> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_mul_ps(__W,__U,__A,__B); 
 }
 __m256 test_mm256_maskz_mul_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_mul_ps
+  // CHECK-LABEL: test_mm256_maskz_mul_ps
   // CHECK: fmul <8 x float> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_mul_ps(__U,__A,__B); 
 }
 __m128i test_mm_mask_abs_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_abs_epi32
+  // CHECK-LABEL: test_mm_mask_abs_epi32
   // CHECK: [[ABS:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[ABS]] to <2 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4472,7 +4474,7 @@ __m128i test_mm_mask_abs_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
   return _mm_mask_abs_epi32(__W,__U,__A); 
 }
 __m128i test_mm_maskz_abs_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_abs_epi32
+  // CHECK-LABEL: test_mm_maskz_abs_epi32
   // CHECK: [[ABS:%.*]] = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[ABS]] to <2 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4480,7 +4482,7 @@ __m128i test_mm_maskz_abs_epi32(__mmask8 __U, __m128i __A) {
   return _mm_maskz_abs_epi32(__U,__A); 
 }
 __m256i test_mm256_mask_abs_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_abs_epi32
+  // CHECK-LABEL: test_mm256_mask_abs_epi32
   // CHECK: [[ABS:%.*]] = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[ABS]] to <4 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4488,7 +4490,7 @@ __m256i test_mm256_mask_abs_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
   return _mm256_mask_abs_epi32(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_abs_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_abs_epi32
+  // CHECK-LABEL: test_mm256_maskz_abs_epi32
   // CHECK: [[ABS:%.*]] = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[ABS]] to <4 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4496,41 +4498,41 @@ __m256i test_mm256_maskz_abs_epi32(__mmask8 __U, __m256i __A) {
   return _mm256_maskz_abs_epi32(__U,__A); 
 }
 __m128i test_mm_abs_epi64(__m128i __A) {
-  // CHECK-LABEL: @test_mm_abs_epi64
-  // CHECK: [[ABS:%.*]] = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm_abs_epi64
+  // CHECK: [[ABS:%.*]] = call {{.*}}<2 x i64> @llvm.abs.v2i64(<2 x i64> %{{.*}}, i1 false)
   return _mm_abs_epi64(__A); 
 }
 __m128i test_mm_mask_abs_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_abs_epi64
-  // CHECK: [[ABS:%.*]] = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm_mask_abs_epi64
+  // CHECK: [[ABS:%.*]] = call {{.*}}<2 x i64> @llvm.abs.v2i64(<2 x i64> %{{.*}}, i1 false)
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> [[ABS]], <2 x i64> %{{.*}}
   return _mm_mask_abs_epi64(__W,__U,__A); 
 }
 __m128i test_mm_maskz_abs_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_abs_epi64
-  // CHECK: [[ABS:%.*]] = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm_maskz_abs_epi64
+  // CHECK: [[ABS:%.*]] = call {{.*}}<2 x i64> @llvm.abs.v2i64(<2 x i64> %{{.*}}, i1 false)
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> [[ABS]], <2 x i64> %{{.*}}
   return _mm_maskz_abs_epi64(__U,__A); 
 }
 __m256i test_mm256_abs_epi64(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_abs_epi64
-  // CHECK: [[ABS:%.*]] = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm256_abs_epi64
+  // CHECK: [[ABS:%.*]] = call {{.*}}<4 x i64> @llvm.abs.v4i64(<4 x i64> %{{.*}}, i1 false)
   return _mm256_abs_epi64(__A); 
 }
 __m256i test_mm256_mask_abs_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_abs_epi64
-  // CHECK: [[ABS:%.*]] = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm256_mask_abs_epi64
+  // CHECK: [[ABS:%.*]] = call {{.*}}<4 x i64> @llvm.abs.v4i64(<4 x i64> %{{.*}}, i1 false)
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> [[ABS]], <4 x i64> %{{.*}}
   return _mm256_mask_abs_epi64(__W,__U,__A); 
 }
 __m256i test_mm256_maskz_abs_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_abs_epi64
-  // CHECK: [[ABS:%.*]] = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm256_maskz_abs_epi64
+  // CHECK: [[ABS:%.*]] = call {{.*}}<4 x i64> @llvm.abs.v4i64(<4 x i64> %{{.*}}, i1 false)
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> [[ABS]], <4 x i64> %{{.*}}
   return _mm256_maskz_abs_epi64(__U,__A); 
 }
 __m128i test_mm_maskz_max_epi32(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_epi32
+  // CHECK-LABEL: test_mm_maskz_max_epi32
   // CHECK: [[RES:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
   // CHECK: [[RES:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4538,7 +4540,7 @@ __m128i test_mm_maskz_max_epi32(__mmask8 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_max_epi32(__M,__A,__B); 
 }
 __m128i test_mm_mask_max_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_max_epi32
+  // CHECK-LABEL: test_mm_mask_max_epi32
   // CHECK: [[RES:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
   // CHECK: [[RES:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4546,7 +4548,7 @@ __m128i test_mm_mask_max_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i _
   return _mm_mask_max_epi32(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_max_epi32(__mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_epi32
+  // CHECK-LABEL: test_mm256_maskz_max_epi32
   // CHECK: [[RES:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[RES]] to <4 x i64>
   // CHECK: [[RES:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4554,7 +4556,7 @@ __m256i test_mm256_maskz_max_epi32(__mmask8 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_max_epi32(__M,__A,__B); 
 }
 __m256i test_mm256_mask_max_epi32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_epi32
+  // CHECK-LABEL: test_mm256_mask_max_epi32
   // CHECK: [[RES:%.*]] = call <8 x i32> @llvm.smax.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[RES]] to <4 x i64>
   // CHECK: [[RES:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4562,41 +4564,41 @@ __m256i test_mm256_mask_max_epi32(__m256i __W, __mmask8 __M, __m256i __A, __m256
   return _mm256_mask_max_epi32(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_max_epi64(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_epi64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_max_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.smax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK:       select <2 x i1> {{.*}}, <2 x i64> [[RES]], <2 x i64> {{.*}}
   return _mm_maskz_max_epi64(__M,__A,__B); 
 }
 __m128i test_mm_mask_max_epi64(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_max_epi64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_max_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.smax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK:       select <2 x i1> {{.*}}, <2 x i64> [[RES]], <2 x i64> {{.*}}
   return _mm_mask_max_epi64(__W,__M,__A,__B); 
 }
 __m128i test_mm_max_epi64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_max_epi64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_max_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.smax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_max_epi64(__A,__B); 
 }
 __m256i test_mm256_maskz_max_epi64(__mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_epi64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_max_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.smax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK:       select <4 x i1> {{.*}}, <4 x i64> [[RES]], <4 x i64> {{.*}}
   return _mm256_maskz_max_epi64(__M,__A,__B); 
 }
 __m256i test_mm256_mask_max_epi64(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_epi64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_max_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.smax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK:       select <4 x i1> {{.*}}, <4 x i64> [[RES]], <4 x i64> {{.*}}
   return _mm256_mask_max_epi64(__W,__M,__A,__B); 
 }
 __m256i test_mm256_max_epi64(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_max_epi64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_max_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.smax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_max_epi64(__A,__B); 
 }
 __m128i test_mm_maskz_max_epu32(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_epu32
+  // CHECK-LABEL: test_mm_maskz_max_epu32
   // CHECK: [[RES:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
   // CHECK: [[RES:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4604,7 +4606,7 @@ __m128i test_mm_maskz_max_epu32(__mmask8 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_max_epu32(__M,__A,__B); 
 }
 __m128i test_mm_mask_max_epu32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_max_epu32
+  // CHECK-LABEL: test_mm_mask_max_epu32
   // CHECK: [[RES:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
   // CHECK: [[RES:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4612,7 +4614,7 @@ __m128i test_mm_mask_max_epu32(__m128i __W, __mmask8 __M, __m128i __A, __m128i _
   return _mm_mask_max_epu32(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_max_epu32(__mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_epu32
+  // CHECK-LABEL: test_mm256_maskz_max_epu32
   // CHECK: [[RES:%.*]] = call <8 x i32> @llvm.umax.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[RES]] to <4 x i64>
   // CHECK: [[RES:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4620,7 +4622,7 @@ __m256i test_mm256_maskz_max_epu32(__mmask8 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_max_epu32(__M,__A,__B); 
 }
 __m256i test_mm256_mask_max_epu32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_epu32
+  // CHECK-LABEL: test_mm256_mask_max_epu32
   // CHECK: [[RES:%.*]] = call <8 x i32> @llvm.umax.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[RES]] to <4 x i64>
   // CHECK: [[RES:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4628,41 +4630,41 @@ __m256i test_mm256_mask_max_epu32(__m256i __W, __mmask8 __M, __m256i __A, __m256
   return _mm256_mask_max_epu32(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_max_epu64(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_epu64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_max_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.umax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK:       select <2 x i1> {{.*}}, <2 x i64> [[RES]], <2 x i64> {{.*}}
   return _mm_maskz_max_epu64(__M,__A,__B); 
 }
 __m128i test_mm_max_epu64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_max_epu64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_max_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.umax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_max_epu64(__A,__B); 
 }
 __m128i test_mm_mask_max_epu64(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_max_epu64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.umax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_max_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.umax.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK:       select <2 x i1> {{.*}}, <2 x i64> [[RES]], <2 x i64> {{.*}}
   return _mm_mask_max_epu64(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_max_epu64(__mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_epu64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.umax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_max_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.umax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK:       select <4 x i1> {{.*}}, <4 x i64> [[RES]], <4 x i64> {{.*}}
   return _mm256_maskz_max_epu64(__M,__A,__B); 
 }
 __m256i test_mm256_max_epu64(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_max_epu64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.umax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_max_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.umax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_max_epu64(__A,__B); 
 }
 __m256i test_mm256_mask_max_epu64(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_epu64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.umax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_max_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.umax.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK:       select <4 x i1> {{.*}}, <4 x i64> [[RES]], <4 x i64> {{.*}}
   return _mm256_mask_max_epu64(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_min_epi32(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_epi32
+  // CHECK-LABEL: test_mm_maskz_min_epi32
   // CHECK: [[RES:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
   // CHECK: [[RES:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4670,7 +4672,7 @@ __m128i test_mm_maskz_min_epi32(__mmask8 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_min_epi32(__M,__A,__B); 
 }
 __m128i test_mm_mask_min_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_min_epi32
+  // CHECK-LABEL: test_mm_mask_min_epi32
   // CHECK: [[RES:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
   // CHECK: [[RES:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4678,7 +4680,7 @@ __m128i test_mm_mask_min_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i _
   return _mm_mask_min_epi32(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_min_epi32(__mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_epi32
+  // CHECK-LABEL: test_mm256_maskz_min_epi32
   // CHECK: [[RES:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[RES]] to <4 x i64>
   // CHECK: [[RES:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4686,7 +4688,7 @@ __m256i test_mm256_maskz_min_epi32(__mmask8 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_min_epi32(__M,__A,__B); 
 }
 __m256i test_mm256_mask_min_epi32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_epi32
+  // CHECK-LABEL: test_mm256_mask_min_epi32
   // CHECK: [[RES:%.*]] = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[RES]] to <4 x i64>
   // CHECK: [[RES:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4694,41 +4696,41 @@ __m256i test_mm256_mask_min_epi32(__m256i __W, __mmask8 __M, __m256i __A, __m256
   return _mm256_mask_min_epi32(__W,__M,__A,__B); 
 }
 __m128i test_mm_min_epi64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_min_epi64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_min_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.smin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_min_epi64(__A,__B); 
 }
 __m128i test_mm_mask_min_epi64(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_min_epi64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_min_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.smin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK:       select <2 x i1> {{.*}}, <2 x i64> [[RES]], <2 x i64> {{.*}}
   return _mm_mask_min_epi64(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_min_epi64(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_epi64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_min_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.smin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK:       select <2 x i1> {{.*}}, <2 x i64> [[RES]], <2 x i64> {{.*}}
   return _mm_maskz_min_epi64(__M,__A,__B); 
 }
 __m256i test_mm256_min_epi64(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_min_epi64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_min_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.smin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_min_epi64(__A,__B); 
 }
 __m256i test_mm256_mask_min_epi64(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_epi64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_min_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.smin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK:       select <4 x i1> {{.*}}, <4 x i64> [[RES]], <4 x i64> {{.*}}
   return _mm256_mask_min_epi64(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_min_epi64(__mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_epi64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_min_epi64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.smin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK:       select <4 x i1> {{.*}}, <4 x i64> [[RES]], <4 x i64> {{.*}}
   return _mm256_maskz_min_epi64(__M,__A,__B); 
 }
 __m128i test_mm_maskz_min_epu32(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_epu32
+  // CHECK-LABEL: test_mm_maskz_min_epu32
   // CHECK: [[RES:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
   // CHECK: [[RES:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4736,7 +4738,7 @@ __m128i test_mm_maskz_min_epu32(__mmask8 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_min_epu32(__M,__A,__B); 
 }
 __m128i test_mm_mask_min_epu32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_min_epu32
+  // CHECK-LABEL: test_mm_mask_min_epu32
   // CHECK: [[RES:%.*]] = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <4 x i32> [[RES]] to <2 x i64>
   // CHECK: [[RES:%.*]] = bitcast <2 x i64> [[TMP]] to <4 x i32>
@@ -4744,7 +4746,7 @@ __m128i test_mm_mask_min_epu32(__m128i __W, __mmask8 __M, __m128i __A, __m128i _
   return _mm_mask_min_epu32(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_min_epu32(__mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_epu32
+  // CHECK-LABEL: test_mm256_maskz_min_epu32
   // CHECK: [[RES:%.*]] = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[RES]] to <4 x i64>
   // CHECK: [[RES:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4752,7 +4754,7 @@ __m256i test_mm256_maskz_min_epu32(__mmask8 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_min_epu32(__M,__A,__B); 
 }
 __m256i test_mm256_mask_min_epu32(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_epu32
+  // CHECK-LABEL: test_mm256_mask_min_epu32
   // CHECK: [[RES:%.*]] = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast <8 x i32> [[RES]] to <4 x i64>
   // CHECK: [[RES:%.*]] = bitcast <4 x i64> [[TMP]] to <8 x i32>
@@ -4760,1115 +4762,1115 @@ __m256i test_mm256_mask_min_epu32(__m256i __W, __mmask8 __M, __m256i __A, __m256
   return _mm256_mask_min_epu32(__W,__M,__A,__B); 
 }
 __m128i test_mm_min_epu64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_min_epu64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_min_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.umin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_min_epu64(__A,__B); 
 }
 __m128i test_mm_mask_min_epu64(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_min_epu64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_min_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.umin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK:       select <2 x i1> {{.*}}, <2 x i64> [[RES]], <2 x i64> {{.*}}
   return _mm_mask_min_epu64(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_min_epu64(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_epu64
-  // CHECK: [[RES:%.*]] = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_min_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<2 x i64> @llvm.umin.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK:       select <2 x i1> {{.*}}, <2 x i64> [[RES]], <2 x i64> {{.*}}
   return _mm_maskz_min_epu64(__M,__A,__B); 
 }
 __m256i test_mm256_min_epu64(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_min_epu64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_min_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.umin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_min_epu64(__A,__B); 
 }
 __m256i test_mm256_mask_min_epu64(__m256i __W, __mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_epu64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_min_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.umin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK:       select <4 x i1> {{.*}}, <4 x i64> [[RES]], <4 x i64> {{.*}}
   return _mm256_mask_min_epu64(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_min_epu64(__mmask8 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_epu64
-  // CHECK: [[RES:%.*]] = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_min_epu64
+  // CHECK: [[RES:%.*]] = call {{.*}}<4 x i64> @llvm.umin.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK:       select <4 x i1> {{.*}}, <4 x i64> [[RES]], <4 x i64> {{.*}}
   return _mm256_maskz_min_epu64(__M,__A,__B); 
 }
 __m128d test_mm_roundscale_pd(__m128d __A) {
-  // CHECK-LABEL: @test_mm_roundscale_pd
+  // CHECK-LABEL: test_mm_roundscale_pd
   // CHECK: @llvm.x86.avx512.mask.rndscale.pd.128
   return _mm_roundscale_pd(__A,4); 
 }
 __m128d test_mm_mask_roundscale_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_roundscale_pd
+  // CHECK-LABEL: test_mm_mask_roundscale_pd
   // CHECK: @llvm.x86.avx512.mask.rndscale.pd.128
   return _mm_mask_roundscale_pd(__W,__U,__A,4); 
 }
 __m128d test_mm_maskz_roundscale_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_roundscale_pd
+  // CHECK-LABEL: test_mm_maskz_roundscale_pd
   // CHECK: @llvm.x86.avx512.mask.rndscale.pd.128
   return _mm_maskz_roundscale_pd(__U,__A,4); 
 }
 __m256d test_mm256_roundscale_pd(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_roundscale_pd
+  // CHECK-LABEL: test_mm256_roundscale_pd
   // CHECK: @llvm.x86.avx512.mask.rndscale.pd.256
   return _mm256_roundscale_pd(__A,4); 
 }
 __m256d test_mm256_mask_roundscale_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_roundscale_pd
+  // CHECK-LABEL: test_mm256_mask_roundscale_pd
   // CHECK: @llvm.x86.avx512.mask.rndscale.pd.256
   return _mm256_mask_roundscale_pd(__W,__U,__A,4); 
 }
 __m256d test_mm256_maskz_roundscale_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_roundscale_pd
+  // CHECK-LABEL: test_mm256_maskz_roundscale_pd
   // CHECK: @llvm.x86.avx512.mask.rndscale.pd.256
   return _mm256_maskz_roundscale_pd(__U,__A,4); 
 }
 __m128 test_mm_roundscale_ps(__m128 __A) {
-  // CHECK-LABEL: @test_mm_roundscale_ps
+  // CHECK-LABEL: test_mm_roundscale_ps
   // CHECK: @llvm.x86.avx512.mask.rndscale.ps.128
   return _mm_roundscale_ps(__A,4); 
 }
 __m128 test_mm_mask_roundscale_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_roundscale_ps
+  // CHECK-LABEL: test_mm_mask_roundscale_ps
   // CHECK: @llvm.x86.avx512.mask.rndscale.ps.128
   return _mm_mask_roundscale_ps(__W,__U,__A,4); 
 }
 __m128 test_mm_maskz_roundscale_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_roundscale_ps
+  // CHECK-LABEL: test_mm_maskz_roundscale_ps
   // CHECK: @llvm.x86.avx512.mask.rndscale.ps.128
   return _mm_maskz_roundscale_ps(__U,__A, 4); 
 }
 __m256 test_mm256_roundscale_ps(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_roundscale_ps
+  // CHECK-LABEL: test_mm256_roundscale_ps
   // CHECK: @llvm.x86.avx512.mask.rndscale.ps.256
   return _mm256_roundscale_ps(__A,4); 
 }
 __m256 test_mm256_mask_roundscale_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_roundscale_ps
+  // CHECK-LABEL: test_mm256_mask_roundscale_ps
   // CHECK: @llvm.x86.avx512.mask.rndscale.ps.256
   return _mm256_mask_roundscale_ps(__W,__U,__A,4); 
 }
 __m256 test_mm256_maskz_roundscale_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_roundscale_ps
+  // CHECK-LABEL: test_mm256_maskz_roundscale_ps
   // CHECK: @llvm.x86.avx512.mask.rndscale.ps.256
   return _mm256_maskz_roundscale_ps(__U,__A,4); 
 }
 __m128d test_mm_scalef_pd(__m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_scalef_pd
+  // CHECK-LABEL: test_mm_scalef_pd
   // CHECK: @llvm.x86.avx512.mask.scalef.pd.128
   return _mm_scalef_pd(__A,__B); 
 }
 __m128d test_mm_mask_scalef_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_scalef_pd
+  // CHECK-LABEL: test_mm_mask_scalef_pd
   // CHECK: @llvm.x86.avx512.mask.scalef.pd.128
   return _mm_mask_scalef_pd(__W,__U,__A,__B); 
 }
 __m128d test_mm_maskz_scalef_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_scalef_pd
+  // CHECK-LABEL: test_mm_maskz_scalef_pd
   // CHECK: @llvm.x86.avx512.mask.scalef.pd.128
   return _mm_maskz_scalef_pd(__U,__A,__B); 
 }
 __m256d test_mm256_scalef_pd(__m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_scalef_pd
+  // CHECK-LABEL: test_mm256_scalef_pd
   // CHECK: @llvm.x86.avx512.mask.scalef.pd.256
   return _mm256_scalef_pd(__A,__B); 
 }
 __m256d test_mm256_mask_scalef_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_scalef_pd
+  // CHECK-LABEL: test_mm256_mask_scalef_pd
   // CHECK: @llvm.x86.avx512.mask.scalef.pd.256
   return _mm256_mask_scalef_pd(__W,__U,__A,__B); 
 }
 __m256d test_mm256_maskz_scalef_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_scalef_pd
+  // CHECK-LABEL: test_mm256_maskz_scalef_pd
   // CHECK: @llvm.x86.avx512.mask.scalef.pd.256
   return _mm256_maskz_scalef_pd(__U,__A,__B); 
 }
 __m128 test_mm_scalef_ps(__m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_scalef_ps
+  // CHECK-LABEL: test_mm_scalef_ps
   // CHECK: @llvm.x86.avx512.mask.scalef.ps.128
   return _mm_scalef_ps(__A,__B); 
 }
 __m128 test_mm_mask_scalef_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_scalef_ps
+  // CHECK-LABEL: test_mm_mask_scalef_ps
   // CHECK: @llvm.x86.avx512.mask.scalef.ps.128
   return _mm_mask_scalef_ps(__W,__U,__A,__B); 
 }
 __m128 test_mm_maskz_scalef_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_scalef_ps
+  // CHECK-LABEL: test_mm_maskz_scalef_ps
   // CHECK: @llvm.x86.avx512.mask.scalef.ps.128
   return _mm_maskz_scalef_ps(__U,__A,__B); 
 }
 __m256 test_mm256_scalef_ps(__m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_scalef_ps
+  // CHECK-LABEL: test_mm256_scalef_ps
   // CHECK: @llvm.x86.avx512.mask.scalef.ps.256
   return _mm256_scalef_ps(__A,__B); 
 }
 __m256 test_mm256_mask_scalef_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_scalef_ps
+  // CHECK-LABEL: test_mm256_mask_scalef_ps
   // CHECK: @llvm.x86.avx512.mask.scalef.ps.256
   return _mm256_mask_scalef_ps(__W,__U,__A,__B); 
 }
 __m256 test_mm256_maskz_scalef_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_scalef_ps
+  // CHECK-LABEL: test_mm256_maskz_scalef_ps
   // CHECK: @llvm.x86.avx512.mask.scalef.ps.256
   return _mm256_maskz_scalef_ps(__U,__A,__B); 
 }
 void test_mm_i64scatter_pd(double *__addr, __m128i __index,  __m128d __v1) {
-  // CHECK-LABEL: @test_mm_i64scatter_pd
+  // CHECK-LABEL: test_mm_i64scatter_pd
   // CHECK: @llvm.x86.avx512.mask.scatterdiv2.df
   return _mm_i64scatter_pd(__addr,__index,__v1,2); 
 }
 void test_mm_mask_i64scatter_pd(double *__addr, __mmask8 __mask, __m128i __index, __m128d __v1) {
-  // CHECK-LABEL: @test_mm_mask_i64scatter_pd
+  // CHECK-LABEL: test_mm_mask_i64scatter_pd
   // CHECK: @llvm.x86.avx512.mask.scatterdiv2.df
   return _mm_mask_i64scatter_pd(__addr,__mask,__index,__v1,2); 
 }
 void test_mm_i64scatter_epi64(long long *__addr, __m128i __index,  __m128i __v1) {
-  // CHECK-LABEL: @test_mm_i64scatter_epi64
+  // CHECK-LABEL: test_mm_i64scatter_epi64
   // CHECK: @llvm.x86.avx512.mask.scatterdiv2.di
   return _mm_i64scatter_epi64(__addr,__index,__v1,2); 
 }
 void test_mm_mask_i64scatter_epi64(long long *__addr, __mmask8 __mask, __m128i __index, __m128i __v1) {
-  // CHECK-LABEL: @test_mm_mask_i64scatter_epi64
+  // CHECK-LABEL: test_mm_mask_i64scatter_epi64
   // CHECK: @llvm.x86.avx512.mask.scatterdiv2.di
   return _mm_mask_i64scatter_epi64(__addr,__mask,__index,__v1,2); 
 }
 void test_mm256_i64scatter_pd(double *__addr, __m256i __index,  __m256d __v1) {
-  // CHECK-LABEL: @test_mm256_i64scatter_pd
+  // CHECK-LABEL: test_mm256_i64scatter_pd
   // CHECK: @llvm.x86.avx512.mask.scatterdiv4.df
   return _mm256_i64scatter_pd(__addr,__index,__v1,2); 
 }
 void test_mm256_mask_i64scatter_pd(double *__addr, __mmask8 __mask, __m256i __index, __m256d __v1) {
-  // CHECK-LABEL: @test_mm256_mask_i64scatter_pd
+  // CHECK-LABEL: test_mm256_mask_i64scatter_pd
   // CHECK: @llvm.x86.avx512.mask.scatterdiv4.df
   return _mm256_mask_i64scatter_pd(__addr,__mask,__index,__v1,2); 
 }
 void test_mm256_i64scatter_epi64(long long *__addr, __m256i __index,  __m256i __v1) {
-  // CHECK-LABEL: @test_mm256_i64scatter_epi64
+  // CHECK-LABEL: test_mm256_i64scatter_epi64
   // CHECK: @llvm.x86.avx512.mask.scatterdiv4.di
   return _mm256_i64scatter_epi64(__addr,__index,__v1,2); 
 }
 void test_mm256_mask_i64scatter_epi64(long long *__addr, __mmask8 __mask,  __m256i __index, __m256i __v1) {
-  // CHECK-LABEL: @test_mm256_mask_i64scatter_epi64
+  // CHECK-LABEL: test_mm256_mask_i64scatter_epi64
   // CHECK: @llvm.x86.avx512.mask.scatterdiv4.di
   return _mm256_mask_i64scatter_epi64(__addr,__mask,__index,__v1,2); 
 }
 void test_mm_i64scatter_ps(float *__addr, __m128i __index, __m128 __v1) {
-  // CHECK-LABEL: @test_mm_i64scatter_ps
+  // CHECK-LABEL: test_mm_i64scatter_ps
   // CHECK: @llvm.x86.avx512.mask.scatterdiv4.sf
   return _mm_i64scatter_ps(__addr,__index,__v1,2); 
 }
 void test_mm_mask_i64scatter_ps(float *__addr, __mmask8 __mask, __m128i __index, __m128 __v1) {
-  // CHECK-LABEL: @test_mm_mask_i64scatter_ps
+  // CHECK-LABEL: test_mm_mask_i64scatter_ps
   // CHECK: @llvm.x86.avx512.mask.scatterdiv4.sf
   return _mm_mask_i64scatter_ps(__addr,__mask,__index,__v1,2); 
 }
 void test_mm_i64scatter_epi32(int *__addr, __m128i __index,  __m128i __v1) {
-  // CHECK-LABEL: @test_mm_i64scatter_epi32
+  // CHECK-LABEL: test_mm_i64scatter_epi32
   // CHECK: @llvm.x86.avx512.mask.scatterdiv4.si
   return _mm_i64scatter_epi32(__addr,__index,__v1,2); 
 }
 void test_mm_mask_i64scatter_epi32(int *__addr, __mmask8 __mask, __m128i __index, __m128i __v1) {
-  // CHECK-LABEL: @test_mm_mask_i64scatter_epi32
+  // CHECK-LABEL: test_mm_mask_i64scatter_epi32
   // CHECK: @llvm.x86.avx512.mask.scatterdiv4.si
   return _mm_mask_i64scatter_epi32(__addr,__mask,__index,__v1,2); 
 }
 void test_mm256_i64scatter_ps(float *__addr, __m256i __index,  __m128 __v1) {
-  // CHECK-LABEL: @test_mm256_i64scatter_ps
+  // CHECK-LABEL: test_mm256_i64scatter_ps
   // CHECK: @llvm.x86.avx512.mask.scatterdiv8.sf
   return _mm256_i64scatter_ps(__addr,__index,__v1,2); 
 }
 void test_mm256_mask_i64scatter_ps(float *__addr, __mmask8 __mask, __m256i __index, __m128 __v1) {
-  // CHECK-LABEL: @test_mm256_mask_i64scatter_ps
+  // CHECK-LABEL: test_mm256_mask_i64scatter_ps
   // CHECK: @llvm.x86.avx512.mask.scatterdiv8.sf
   return _mm256_mask_i64scatter_ps(__addr,__mask,__index,__v1,2); 
 }
 void test_mm256_i64scatter_epi32(int *__addr, __m256i __index,  __m128i __v1) {
-  // CHECK-LABEL: @test_mm256_i64scatter_epi32
+  // CHECK-LABEL: test_mm256_i64scatter_epi32
   // CHECK: @llvm.x86.avx512.mask.scatterdiv8.si
   return _mm256_i64scatter_epi32(__addr,__index,__v1,2); 
 }
 void test_mm256_mask_i64scatter_epi32(int *__addr, __mmask8 __mask,  __m256i __index, __m128i __v1) {
-  // CHECK-LABEL: @test_mm256_mask_i64scatter_epi32
+  // CHECK-LABEL: test_mm256_mask_i64scatter_epi32
   // CHECK: @llvm.x86.avx512.mask.scatterdiv8.si
   return _mm256_mask_i64scatter_epi32(__addr,__mask,__index,__v1,2); 
 }
 void test_mm_i32scatter_pd(double *__addr, __m128i __index,  __m128d __v1) {
-  // CHECK-LABEL: @test_mm_i32scatter_pd
+  // CHECK-LABEL: test_mm_i32scatter_pd
   // CHECK: @llvm.x86.avx512.mask.scattersiv2.df
   return _mm_i32scatter_pd(__addr,__index,__v1,2); 
 }
 void test_mm_mask_i32scatter_pd(double *__addr, __mmask8 __mask, __m128i __index, __m128d __v1) {
-  // CHECK-LABEL: @test_mm_mask_i32scatter_pd
+  // CHECK-LABEL: test_mm_mask_i32scatter_pd
   // CHECK: @llvm.x86.avx512.mask.scattersiv2.df
   return _mm_mask_i32scatter_pd(__addr,__mask,__index,__v1,2); 
 }
 void test_mm_i32scatter_epi64(long long *__addr, __m128i __index,  __m128i __v1) {
-  // CHECK-LABEL: @test_mm_i32scatter_epi64
+  // CHECK-LABEL: test_mm_i32scatter_epi64
   // CHECK: @llvm.x86.avx512.mask.scattersiv2.di
   return _mm_i32scatter_epi64(__addr,__index,__v1,2); 
 }
 void test_mm_mask_i32scatter_epi64(long long *__addr, __mmask8 __mask, __m128i __index, __m128i __v1) {
-  // CHECK-LABEL: @test_mm_mask_i32scatter_epi64
+  // CHECK-LABEL: test_mm_mask_i32scatter_epi64
   // CHECK: @llvm.x86.avx512.mask.scattersiv2.di
   return _mm_mask_i32scatter_epi64(__addr,__mask,__index,__v1,2); 
 }
 void test_mm256_i32scatter_pd(double *__addr, __m128i __index,  __m256d __v1) {
-  // CHECK-LABEL: @test_mm256_i32scatter_pd
+  // CHECK-LABEL: test_mm256_i32scatter_pd
   // CHECK: @llvm.x86.avx512.mask.scattersiv4.df
   return _mm256_i32scatter_pd(__addr,__index,__v1,2); 
 }
 void test_mm256_mask_i32scatter_pd(double *__addr, __mmask8 __mask, __m128i __index, __m256d __v1) {
-  // CHECK-LABEL: @test_mm256_mask_i32scatter_pd
+  // CHECK-LABEL: test_mm256_mask_i32scatter_pd
   // CHECK: @llvm.x86.avx512.mask.scattersiv4.df
   return _mm256_mask_i32scatter_pd(__addr,__mask,__index,__v1,2); 
 }
 void test_mm256_i32scatter_epi64(long long *__addr, __m128i __index,  __m256i __v1) {
-  // CHECK-LABEL: @test_mm256_i32scatter_epi64
+  // CHECK-LABEL: test_mm256_i32scatter_epi64
   // CHECK: @llvm.x86.avx512.mask.scattersiv4.di
   return _mm256_i32scatter_epi64(__addr,__index,__v1,2); 
 }
 void test_mm256_mask_i32scatter_epi64(long long *__addr, __mmask8 __mask,  __m128i __index, __m256i __v1) {
-  // CHECK-LABEL: @test_mm256_mask_i32scatter_epi64
+  // CHECK-LABEL: test_mm256_mask_i32scatter_epi64
   // CHECK: @llvm.x86.avx512.mask.scattersiv4.di
   return _mm256_mask_i32scatter_epi64(__addr,__mask,__index,__v1,2); 
 }
 void test_mm_i32scatter_ps(float *__addr, __m128i __index, __m128 __v1) {
-  // CHECK-LABEL: @test_mm_i32scatter_ps
+  // CHECK-LABEL: test_mm_i32scatter_ps
   // CHECK: @llvm.x86.avx512.mask.scattersiv4.sf
   return _mm_i32scatter_ps(__addr,__index,__v1,2); 
 }
 void test_mm_mask_i32scatter_ps(float *__addr, __mmask8 __mask, __m128i __index, __m128 __v1) {
-  // CHECK-LABEL: @test_mm_mask_i32scatter_ps
+  // CHECK-LABEL: test_mm_mask_i32scatter_ps
   // CHECK: @llvm.x86.avx512.mask.scattersiv4.sf
   return _mm_mask_i32scatter_ps(__addr,__mask,__index,__v1,2); 
 }
 void test_mm_i32scatter_epi32(int *__addr, __m128i __index,  __m128i __v1) {
-  // CHECK-LABEL: @test_mm_i32scatter_epi32
+  // CHECK-LABEL: test_mm_i32scatter_epi32
   // CHECK: @llvm.x86.avx512.mask.scattersiv4.si
   return _mm_i32scatter_epi32(__addr,__index,__v1,2); 
 }
 void test_mm_mask_i32scatter_epi32(int *__addr, __mmask8 __mask, __m128i __index, __m128i __v1) {
-  // CHECK-LABEL: @test_mm_mask_i32scatter_epi32
+  // CHECK-LABEL: test_mm_mask_i32scatter_epi32
   // CHECK: @llvm.x86.avx512.mask.scattersiv4.si
   return _mm_mask_i32scatter_epi32(__addr,__mask,__index,__v1,2); 
 }
 void test_mm256_i32scatter_ps(float *__addr, __m256i __index,  __m256 __v1) {
-  // CHECK-LABEL: @test_mm256_i32scatter_ps
+  // CHECK-LABEL: test_mm256_i32scatter_ps
   // CHECK: @llvm.x86.avx512.mask.scattersiv8.sf
   return _mm256_i32scatter_ps(__addr,__index,__v1,2); 
 }
 void test_mm256_mask_i32scatter_ps(float *__addr, __mmask8 __mask, __m256i __index, __m256 __v1) {
-  // CHECK-LABEL: @test_mm256_mask_i32scatter_ps
+  // CHECK-LABEL: test_mm256_mask_i32scatter_ps
   // CHECK: @llvm.x86.avx512.mask.scattersiv8.sf
   return _mm256_mask_i32scatter_ps(__addr,__mask,__index,__v1,2); 
 }
 void test_mm256_i32scatter_epi32(int *__addr, __m256i __index,  __m256i __v1) {
-  // CHECK-LABEL: @test_mm256_i32scatter_epi32
+  // CHECK-LABEL: test_mm256_i32scatter_epi32
   // CHECK: @llvm.x86.avx512.mask.scattersiv8.si
   return _mm256_i32scatter_epi32(__addr,__index,__v1,2); 
 }
 void test_mm256_mask_i32scatter_epi32(int *__addr, __mmask8 __mask,  __m256i __index, __m256i __v1) {
-  // CHECK-LABEL: @test_mm256_mask_i32scatter_epi32
+  // CHECK-LABEL: test_mm256_mask_i32scatter_epi32
   // CHECK: @llvm.x86.avx512.mask.scattersiv8.si
   return _mm256_mask_i32scatter_epi32(__addr,__mask,__index,__v1,2); 
 }
 __m128d test_mm_mask_sqrt_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_sqrt_pd
+  // CHECK-LABEL: test_mm_mask_sqrt_pd
   // CHECK: @llvm.sqrt.v2f64
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_sqrt_pd(__W,__U,__A); 
 }
 __m128d test_mm_maskz_sqrt_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_sqrt_pd
+  // CHECK-LABEL: test_mm_maskz_sqrt_pd
   // CHECK: @llvm.sqrt.v2f64
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_sqrt_pd(__U,__A); 
 }
 __m256d test_mm256_mask_sqrt_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_sqrt_pd
+  // CHECK-LABEL: test_mm256_mask_sqrt_pd
   // CHECK: @llvm.sqrt.v4f64
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_sqrt_pd(__W,__U,__A); 
 }
 __m256d test_mm256_maskz_sqrt_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_sqrt_pd
+  // CHECK-LABEL: test_mm256_maskz_sqrt_pd
   // CHECK: @llvm.sqrt.v4f64
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_sqrt_pd(__U,__A); 
 }
 __m128 test_mm_mask_sqrt_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_sqrt_ps
+  // CHECK-LABEL: test_mm_mask_sqrt_ps
   // CHECK: @llvm.sqrt.v4f32
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_sqrt_ps(__W,__U,__A); 
 }
 __m128 test_mm_maskz_sqrt_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_sqrt_ps
+  // CHECK-LABEL: test_mm_maskz_sqrt_ps
   // CHECK: @llvm.sqrt.v4f32
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_sqrt_ps(__U,__A); 
 }
 __m256 test_mm256_mask_sqrt_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_sqrt_ps
+  // CHECK-LABEL: test_mm256_mask_sqrt_ps
   // CHECK: @llvm.sqrt.v8f32
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_sqrt_ps(__W,__U,__A); 
 }
 __m256 test_mm256_maskz_sqrt_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_sqrt_ps
+  // CHECK-LABEL: test_mm256_maskz_sqrt_ps
   // CHECK: @llvm.sqrt.v8f32
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_sqrt_ps(__U,__A); 
 }
 __m128d test_mm_mask_sub_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_sub_pd
+  // CHECK-LABEL: test_mm_mask_sub_pd
   // CHECK: fsub <2 x double> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_sub_pd(__W,__U,__A,__B); 
 }
 __m128d test_mm_maskz_sub_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_sub_pd
+  // CHECK-LABEL: test_mm_maskz_sub_pd
   // CHECK: fsub <2 x double> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_sub_pd(__U,__A,__B); 
 }
 __m256d test_mm256_mask_sub_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_sub_pd
+  // CHECK-LABEL: test_mm256_mask_sub_pd
   // CHECK: fsub <4 x double> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_sub_pd(__W,__U,__A,__B); 
 }
 __m256d test_mm256_maskz_sub_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sub_pd
+  // CHECK-LABEL: test_mm256_maskz_sub_pd
   // CHECK: fsub <4 x double> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_sub_pd(__U,__A,__B); 
 }
 __m128 test_mm_mask_sub_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_sub_ps
+  // CHECK-LABEL: test_mm_mask_sub_ps
   // CHECK: fsub <4 x float> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_sub_ps(__W,__U,__A,__B); 
 }
 __m128 test_mm_maskz_sub_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_sub_ps
+  // CHECK-LABEL: test_mm_maskz_sub_ps
   // CHECK: fsub <4 x float> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_sub_ps(__U,__A,__B); 
 }
 __m256 test_mm256_mask_sub_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_sub_ps
+  // CHECK-LABEL: test_mm256_mask_sub_ps
   // CHECK: fsub <8 x float> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_sub_ps(__W,__U,__A,__B); 
 }
 __m256 test_mm256_maskz_sub_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sub_ps
+  // CHECK-LABEL: test_mm256_maskz_sub_ps
   // CHECK: fsub <8 x float> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_sub_ps(__U,__A,__B); 
 }
 __m128i test_mm_mask2_permutex2var_epi32(__m128i __A, __m128i __I, __mmask8 __U,  __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask2_permutex2var_epi32
+  // CHECK-LABEL: test_mm_mask2_permutex2var_epi32
   // CHECK: @llvm.x86.avx512.vpermi2var.d.128
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask2_permutex2var_epi32(__A,__I,__U,__B); 
 }
 __m256i test_mm256_mask2_permutex2var_epi32(__m256i __A, __m256i __I, __mmask8 __U, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask2_permutex2var_epi32
+  // CHECK-LABEL: test_mm256_mask2_permutex2var_epi32
   // CHECK: @llvm.x86.avx512.vpermi2var.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask2_permutex2var_epi32(__A,__I,__U,__B); 
 }
 __m128d test_mm_mask2_permutex2var_pd(__m128d __A, __m128i __I, __mmask8 __U, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask2_permutex2var_pd
+  // CHECK-LABEL: test_mm_mask2_permutex2var_pd
   // CHECK: @llvm.x86.avx512.vpermi2var.pd.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask2_permutex2var_pd(__A,__I,__U,__B); 
 }
 __m256d test_mm256_mask2_permutex2var_pd(__m256d __A, __m256i __I, __mmask8 __U,  __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask2_permutex2var_pd
+  // CHECK-LABEL: test_mm256_mask2_permutex2var_pd
   // CHECK: @llvm.x86.avx512.vpermi2var.pd.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask2_permutex2var_pd(__A,__I,__U,__B); 
 }
 __m128 test_mm_mask2_permutex2var_ps(__m128 __A, __m128i __I, __mmask8 __U, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask2_permutex2var_ps
+  // CHECK-LABEL: test_mm_mask2_permutex2var_ps
   // CHECK: @llvm.x86.avx512.vpermi2var.ps.128
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask2_permutex2var_ps(__A,__I,__U,__B); 
 }
 __m256 test_mm256_mask2_permutex2var_ps(__m256 __A, __m256i __I, __mmask8 __U,  __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask2_permutex2var_ps
+  // CHECK-LABEL: test_mm256_mask2_permutex2var_ps
   // CHECK: @llvm.x86.avx512.vpermi2var.ps.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask2_permutex2var_ps(__A,__I,__U,__B); 
 }
 __m128i test_mm_mask2_permutex2var_epi64(__m128i __A, __m128i __I, __mmask8 __U,  __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask2_permutex2var_epi64
+  // CHECK-LABEL: test_mm_mask2_permutex2var_epi64
   // CHECK: @llvm.x86.avx512.vpermi2var.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask2_permutex2var_epi64(__A,__I,__U,__B); 
 }
 __m256i test_mm256_mask2_permutex2var_epi64(__m256i __A, __m256i __I, __mmask8 __U, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask2_permutex2var_epi64
+  // CHECK-LABEL: test_mm256_mask2_permutex2var_epi64
   // CHECK: @llvm.x86.avx512.vpermi2var.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask2_permutex2var_epi64(__A,__I,__U,__B); 
 }
 __m128i test_mm_permutex2var_epi32(__m128i __A, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_permutex2var_epi32
+  // CHECK-LABEL: test_mm_permutex2var_epi32
   // CHECK: @llvm.x86.avx512.vpermi2var.d.128
   return _mm_permutex2var_epi32(__A,__I,__B); 
 }
 __m128i test_mm_mask_permutex2var_epi32(__m128i __A, __mmask8 __U, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_permutex2var_epi32
+  // CHECK-LABEL: test_mm_mask_permutex2var_epi32
   // CHECK: @llvm.x86.avx512.vpermi2var.d.128
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_permutex2var_epi32(__A,__U,__I,__B); 
 }
 __m128i test_mm_maskz_permutex2var_epi32(__mmask8 __U, __m128i __A, __m128i __I,  __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_permutex2var_epi32
+  // CHECK-LABEL: test_mm_maskz_permutex2var_epi32
   // CHECK: @llvm.x86.avx512.vpermi2var.d.128
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_permutex2var_epi32(__U,__A,__I,__B); 
 }
 __m256i test_mm256_permutex2var_epi32(__m256i __A, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_permutex2var_epi32
+  // CHECK-LABEL: test_mm256_permutex2var_epi32
   // CHECK: @llvm.x86.avx512.vpermi2var.d.256
   return _mm256_permutex2var_epi32(__A,__I,__B); 
 }
 __m256i test_mm256_mask_permutex2var_epi32(__m256i __A, __mmask8 __U, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_permutex2var_epi32
+  // CHECK-LABEL: test_mm256_mask_permutex2var_epi32
   // CHECK: @llvm.x86.avx512.vpermi2var.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_permutex2var_epi32(__A,__U,__I,__B); 
 }
 __m256i test_mm256_maskz_permutex2var_epi32(__mmask8 __U, __m256i __A, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_permutex2var_epi32
+  // CHECK-LABEL: test_mm256_maskz_permutex2var_epi32
   // CHECK: @llvm.x86.avx512.vpermi2var.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_permutex2var_epi32(__U,__A,__I,__B); 
 }
 __m128d test_mm_permutex2var_pd(__m128d __A, __m128i __I, __m128d __B) {
-  // CHECK-LABEL: @test_mm_permutex2var_pd
+  // CHECK-LABEL: test_mm_permutex2var_pd
   // CHECK: @llvm.x86.avx512.vpermi2var.pd.128
   return _mm_permutex2var_pd(__A,__I,__B); 
 }
 __m128d test_mm_mask_permutex2var_pd(__m128d __A, __mmask8 __U, __m128i __I, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_permutex2var_pd
+  // CHECK-LABEL: test_mm_mask_permutex2var_pd
   // CHECK: @llvm.x86.avx512.vpermi2var.pd.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_permutex2var_pd(__A,__U,__I,__B); 
 }
 __m128d test_mm_maskz_permutex2var_pd(__mmask8 __U, __m128d __A, __m128i __I, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_permutex2var_pd
+  // CHECK-LABEL: test_mm_maskz_permutex2var_pd
   // CHECK: @llvm.x86.avx512.vpermi2var.pd.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_permutex2var_pd(__U,__A,__I,__B); 
 }
 __m256d test_mm256_permutex2var_pd(__m256d __A, __m256i __I, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_permutex2var_pd
+  // CHECK-LABEL: test_mm256_permutex2var_pd
   // CHECK: @llvm.x86.avx512.vpermi2var.pd.256
   return _mm256_permutex2var_pd(__A,__I,__B); 
 }
 __m256d test_mm256_mask_permutex2var_pd(__m256d __A, __mmask8 __U, __m256i __I, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_permutex2var_pd
+  // CHECK-LABEL: test_mm256_mask_permutex2var_pd
   // CHECK: @llvm.x86.avx512.vpermi2var.pd.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_permutex2var_pd(__A,__U,__I,__B); 
 }
 __m256d test_mm256_maskz_permutex2var_pd(__mmask8 __U, __m256d __A, __m256i __I,  __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_permutex2var_pd
+  // CHECK-LABEL: test_mm256_maskz_permutex2var_pd
   // CHECK: @llvm.x86.avx512.vpermi2var.pd.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_permutex2var_pd(__U,__A,__I,__B); 
 }
 __m128 test_mm_permutex2var_ps(__m128 __A, __m128i __I, __m128 __B) {
-  // CHECK-LABEL: @test_mm_permutex2var_ps
+  // CHECK-LABEL: test_mm_permutex2var_ps
   // CHECK: @llvm.x86.avx512.vpermi2var.ps.128
   return _mm_permutex2var_ps(__A,__I,__B); 
 }
 __m128 test_mm_mask_permutex2var_ps(__m128 __A, __mmask8 __U, __m128i __I, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_permutex2var_ps
+  // CHECK-LABEL: test_mm_mask_permutex2var_ps
   // CHECK: @llvm.x86.avx512.vpermi2var.ps.128
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_permutex2var_ps(__A,__U,__I,__B); 
 }
 __m128 test_mm_maskz_permutex2var_ps(__mmask8 __U, __m128 __A, __m128i __I, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_permutex2var_ps
+  // CHECK-LABEL: test_mm_maskz_permutex2var_ps
   // CHECK: @llvm.x86.avx512.vpermi2var.ps.128
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_permutex2var_ps(__U,__A,__I,__B); 
 }
 __m256 test_mm256_permutex2var_ps(__m256 __A, __m256i __I, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_permutex2var_ps
+  // CHECK-LABEL: test_mm256_permutex2var_ps
   // CHECK: @llvm.x86.avx512.vpermi2var.ps.256
   return _mm256_permutex2var_ps(__A,__I,__B); 
 }
 __m256 test_mm256_mask_permutex2var_ps(__m256 __A, __mmask8 __U, __m256i __I, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_permutex2var_ps
+  // CHECK-LABEL: test_mm256_mask_permutex2var_ps
   // CHECK: @llvm.x86.avx512.vpermi2var.ps.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_permutex2var_ps(__A,__U,__I,__B); 
 }
 __m256 test_mm256_maskz_permutex2var_ps(__mmask8 __U, __m256 __A, __m256i __I, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_permutex2var_ps
+  // CHECK-LABEL: test_mm256_maskz_permutex2var_ps
   // CHECK: @llvm.x86.avx512.vpermi2var.ps.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_permutex2var_ps(__U,__A,__I,__B); 
 }
 __m128i test_mm_permutex2var_epi64(__m128i __A, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_permutex2var_epi64
+  // CHECK-LABEL: test_mm_permutex2var_epi64
   // CHECK: @llvm.x86.avx512.vpermi2var.q.128
   return _mm_permutex2var_epi64(__A,__I,__B); 
 }
 __m128i test_mm_mask_permutex2var_epi64(__m128i __A, __mmask8 __U, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_permutex2var_epi64
+  // CHECK-LABEL: test_mm_mask_permutex2var_epi64
   // CHECK: @llvm.x86.avx512.vpermi2var.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_permutex2var_epi64(__A,__U,__I,__B); 
 }
 __m128i test_mm_maskz_permutex2var_epi64(__mmask8 __U, __m128i __A, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_permutex2var_epi64
+  // CHECK-LABEL: test_mm_maskz_permutex2var_epi64
   // CHECK: @llvm.x86.avx512.vpermi2var.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_permutex2var_epi64(__U,__A,__I,__B); 
 }
 __m256i test_mm256_permutex2var_epi64(__m256i __A, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_permutex2var_epi64
+  // CHECK-LABEL: test_mm256_permutex2var_epi64
   // CHECK: @llvm.x86.avx512.vpermi2var.q.256
   return _mm256_permutex2var_epi64(__A,__I,__B); 
 }
 __m256i test_mm256_mask_permutex2var_epi64(__m256i __A, __mmask8 __U, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_permutex2var_epi64
+  // CHECK-LABEL: test_mm256_mask_permutex2var_epi64
   // CHECK: @llvm.x86.avx512.vpermi2var.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_permutex2var_epi64(__A,__U,__I,__B); 
 }
 __m256i test_mm256_maskz_permutex2var_epi64(__mmask8 __U, __m256i __A, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_permutex2var_epi64
+  // CHECK-LABEL: test_mm256_maskz_permutex2var_epi64
   // CHECK: @llvm.x86.avx512.vpermi2var.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_permutex2var_epi64(__U,__A,__I,__B); 
 }
 
 __m128i test_mm_mask_cvtepi8_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi8_epi32
+  // CHECK-LABEL: test_mm_mask_cvtepi8_epi32
   // CHECK: sext <4 x i8> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_cvtepi8_epi32(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi8_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi8_epi32
+  // CHECK-LABEL: test_mm_maskz_cvtepi8_epi32
   // CHECK: sext <4 x i8> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_cvtepi8_epi32(__U, __A); 
 }
 
 __m256i test_mm256_mask_cvtepi8_epi32(__m256i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi8_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtepi8_epi32
   // CHECK: sext <8 x i8> %{{.*}} to <8 x i32>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_cvtepi8_epi32(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepi8_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi8_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvtepi8_epi32
   // CHECK: sext <8 x i8> %{{.*}} to <8 x i32>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_cvtepi8_epi32(__U, __A); 
 }
 
 __m128i test_mm_mask_cvtepi8_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi8_epi64
+  // CHECK-LABEL: test_mm_mask_cvtepi8_epi64
   // CHECK: sext <2 x i8> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_cvtepi8_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi8_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi8_epi64
+  // CHECK-LABEL: test_mm_maskz_cvtepi8_epi64
   // CHECK: sext <2 x i8> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_cvtepi8_epi64(__U, __A); 
 }
 
 __m256i test_mm256_mask_cvtepi8_epi64(__m256i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi8_epi64
+  // CHECK-LABEL: test_mm256_mask_cvtepi8_epi64
   // CHECK: sext <4 x i8> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_cvtepi8_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepi8_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi8_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvtepi8_epi64
   // CHECK: sext <4 x i8> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_cvtepi8_epi64(__U, __A); 
 }
 
 __m128i test_mm_mask_cvtepi32_epi64(__m128i __W, __mmask8 __U, __m128i __X) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi32_epi64
+  // CHECK-LABEL: test_mm_mask_cvtepi32_epi64
   // CHECK: sext <2 x i32> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_cvtepi32_epi64(__W, __U, __X); 
 }
 
 __m128i test_mm_maskz_cvtepi32_epi64(__mmask8 __U, __m128i __X) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi32_epi64
+  // CHECK-LABEL: test_mm_maskz_cvtepi32_epi64
   // CHECK: sext <2 x i32> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_cvtepi32_epi64(__U, __X); 
 }
 
 __m256i test_mm256_mask_cvtepi32_epi64(__m256i __W, __mmask8 __U, __m128i __X) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi32_epi64
+  // CHECK-LABEL: test_mm256_mask_cvtepi32_epi64
   // CHECK: sext <4 x i32> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_cvtepi32_epi64(__W, __U, __X); 
 }
 
 __m256i test_mm256_maskz_cvtepi32_epi64(__mmask8 __U, __m128i __X) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi32_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvtepi32_epi64
   // CHECK: sext <4 x i32> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_cvtepi32_epi64(__U, __X); 
 }
 
 __m128i test_mm_mask_cvtepi16_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi16_epi32
+  // CHECK-LABEL: test_mm_mask_cvtepi16_epi32
   // CHECK: sext <4 x i16> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_cvtepi16_epi32(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi16_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi16_epi32
+  // CHECK-LABEL: test_mm_maskz_cvtepi16_epi32
   // CHECK: sext <4 x i16> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_cvtepi16_epi32(__U, __A); 
 }
 
 __m256i test_mm256_mask_cvtepi16_epi32(__m256i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi16_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtepi16_epi32
   // CHECK: sext <8 x i16> %{{.*}} to <8 x i32>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_cvtepi16_epi32(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepi16_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi16_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvtepi16_epi32
   // CHECK: sext <8 x i16> %{{.*}} to <8 x i32>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_cvtepi16_epi32(__U, __A); 
 }
 
 __m128i test_mm_mask_cvtepi16_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi16_epi64
+  // CHECK-LABEL: test_mm_mask_cvtepi16_epi64
   // CHECK: sext <2 x i16> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_cvtepi16_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi16_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi16_epi64
+  // CHECK-LABEL: test_mm_maskz_cvtepi16_epi64
   // CHECK: sext <2 x i16> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_cvtepi16_epi64(__U, __A); 
 }
 
 __m256i test_mm256_mask_cvtepi16_epi64(__m256i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi16_epi64
+  // CHECK-LABEL: test_mm256_mask_cvtepi16_epi64
   // CHECK: sext <4 x i16> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_cvtepi16_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepi16_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi16_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvtepi16_epi64
   // CHECK: sext <4 x i16> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_cvtepi16_epi64(__U, __A); 
 }
 
 __m128i test_mm_mask_cvtepu8_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu8_epi32
+  // CHECK-LABEL: test_mm_mask_cvtepu8_epi32
   // CHECK: zext <4 x i8> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_cvtepu8_epi32(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepu8_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu8_epi32
+  // CHECK-LABEL: test_mm_maskz_cvtepu8_epi32
   // CHECK: zext <4 x i8> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_cvtepu8_epi32(__U, __A);
 }
 
 __m256i test_mm256_mask_cvtepu8_epi32(__m256i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu8_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtepu8_epi32
   // CHECK: zext <8 x i8> %{{.*}} to <8 x i32>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_cvtepu8_epi32(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepu8_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu8_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvtepu8_epi32
   // CHECK: zext <8 x i8> %{{.*}} to <8 x i32>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_cvtepu8_epi32(__U, __A); 
 }
 
 __m128i test_mm_mask_cvtepu8_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu8_epi64
+  // CHECK-LABEL: test_mm_mask_cvtepu8_epi64
   // CHECK: zext <2 x i8> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_cvtepu8_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepu8_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu8_epi64
+  // CHECK-LABEL: test_mm_maskz_cvtepu8_epi64
   // CHECK: zext <2 x i8> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_cvtepu8_epi64(__U, __A); 
 }
 
 __m256i test_mm256_mask_cvtepu8_epi64(__m256i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu8_epi64
+  // CHECK-LABEL: test_mm256_mask_cvtepu8_epi64
   // CHECK: zext <4 x i8> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_cvtepu8_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepu8_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu8_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvtepu8_epi64
   // CHECK: zext <4 x i8> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_cvtepu8_epi64(__U, __A); 
 }
 
 __m128i test_mm_mask_cvtepu32_epi64(__m128i __W, __mmask8 __U, __m128i __X) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu32_epi64
+  // CHECK-LABEL: test_mm_mask_cvtepu32_epi64
   // CHECK: zext <2 x i32> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_cvtepu32_epi64(__W, __U, __X); 
 }
 
 __m128i test_mm_maskz_cvtepu32_epi64(__mmask8 __U, __m128i __X) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu32_epi64
+  // CHECK-LABEL: test_mm_maskz_cvtepu32_epi64
   // CHECK: zext <2 x i32> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_cvtepu32_epi64(__U, __X); 
 }
 
 __m256i test_mm256_mask_cvtepu32_epi64(__m256i __W, __mmask8 __U, __m128i __X) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu32_epi64
+  // CHECK-LABEL: test_mm256_mask_cvtepu32_epi64
   // CHECK: zext <4 x i32> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_cvtepu32_epi64(__W, __U, __X); 
 }
 
 __m256i test_mm256_maskz_cvtepu32_epi64(__mmask8 __U, __m128i __X) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu32_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvtepu32_epi64
   // CHECK: zext <4 x i32> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_cvtepu32_epi64(__U, __X); 
 }
 
 __m128i test_mm_mask_cvtepu16_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu16_epi32
+  // CHECK-LABEL: test_mm_mask_cvtepu16_epi32
   // CHECK: zext <4 x i16> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_cvtepu16_epi32(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepu16_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu16_epi32
+  // CHECK-LABEL: test_mm_maskz_cvtepu16_epi32
   // CHECK: zext <4 x i16> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_cvtepu16_epi32(__U, __A); 
 }
 
 __m256i test_mm256_mask_cvtepu16_epi32(__m256i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu16_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtepu16_epi32
   // CHECK: zext <8 x i16> %{{.*}} to <8 x i32>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_cvtepu16_epi32(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepu16_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu16_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvtepu16_epi32
   // CHECK: zext <8 x i16> %{{.*}} to <8 x i32>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_cvtepu16_epi32(__U, __A); 
 }
 
 __m128i test_mm_mask_cvtepu16_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu16_epi64
+  // CHECK-LABEL: test_mm_mask_cvtepu16_epi64
   // CHECK: zext <2 x i16> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_cvtepu16_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepu16_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu16_epi64
+  // CHECK-LABEL: test_mm_maskz_cvtepu16_epi64
   // CHECK: zext <2 x i16> %{{.*}} to <2 x i64>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_cvtepu16_epi64(__U, __A); 
 }
 
 __m256i test_mm256_mask_cvtepu16_epi64(__m256i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu16_epi64
+  // CHECK-LABEL: test_mm256_mask_cvtepu16_epi64
   // CHECK: zext <4 x i16> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_cvtepu16_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepu16_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu16_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvtepu16_epi64
   // CHECK: zext <4 x i16> %{{.*}} to <4 x i64>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_cvtepu16_epi64(__U, __A); 
 }
 
 __m128i test_mm_rol_epi32(__m128i __A) {
-  // CHECK-LABEL: @test_mm_rol_epi32
+  // CHECK-LABEL: test_mm_rol_epi32
   // CHECK: @llvm.fshl.v4i32
   return _mm_rol_epi32(__A, 5); 
 }
 
 __m128i test_mm_mask_rol_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_rol_epi32
+  // CHECK-LABEL: test_mm_mask_rol_epi32
   // CHECK: @llvm.fshl.v4i32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_rol_epi32(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_maskz_rol_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_rol_epi32
+  // CHECK-LABEL: test_mm_maskz_rol_epi32
   // CHECK: @llvm.fshl.v4i32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_rol_epi32(__U, __A, 5); 
 }
 
 __m256i test_mm256_rol_epi32(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_rol_epi32
+  // CHECK-LABEL: test_mm256_rol_epi32
   // CHECK: @llvm.fshl.v8i32
   return _mm256_rol_epi32(__A, 5); 
 }
 
 __m256i test_mm256_mask_rol_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_rol_epi32
+  // CHECK-LABEL: test_mm256_mask_rol_epi32
   // CHECK: @llvm.fshl.v8i32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_rol_epi32(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_rol_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_rol_epi32
+  // CHECK-LABEL: test_mm256_maskz_rol_epi32
   // CHECK: @llvm.fshl.v8i32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_rol_epi32(__U, __A, 5); 
 }
 
 __m128i test_mm_rol_epi64(__m128i __A) {
-  // CHECK-LABEL: @test_mm_rol_epi64
+  // CHECK-LABEL: test_mm_rol_epi64
   // CHECK: @llvm.fshl.v2i64
   return _mm_rol_epi64(__A, 5); 
 }
 
 __m128i test_mm_mask_rol_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_rol_epi64
+  // CHECK-LABEL: test_mm_mask_rol_epi64
   // CHECK: @llvm.fshl.v2i64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_rol_epi64(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_maskz_rol_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_rol_epi64
+  // CHECK-LABEL: test_mm_maskz_rol_epi64
   // CHECK: @llvm.fshl.v2i64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_rol_epi64(__U, __A, 5); 
 }
 
 __m256i test_mm256_rol_epi64(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_rol_epi64
+  // CHECK-LABEL: test_mm256_rol_epi64
   // CHECK: @llvm.fshl.v4i64
   return _mm256_rol_epi64(__A, 5); 
 }
 
 __m256i test_mm256_mask_rol_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_rol_epi64
+  // CHECK-LABEL: test_mm256_mask_rol_epi64
   // CHECK: @llvm.fshl.v4i64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_rol_epi64(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_rol_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_rol_epi64
+  // CHECK-LABEL: test_mm256_maskz_rol_epi64
   // CHECK: @llvm.fshl.v4i64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_rol_epi64(__U, __A, 5); 
 }
 
 __m128i test_mm_rolv_epi32(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_rolv_epi32
+  // CHECK-LABEL: test_mm_rolv_epi32
   // CHECK: llvm.fshl.v4i32
   return _mm_rolv_epi32(__A, __B); 
 }
 
 __m128i test_mm_mask_rolv_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_rolv_epi32
+  // CHECK-LABEL: test_mm_mask_rolv_epi32
   // CHECK: llvm.fshl.v4i32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_rolv_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_rolv_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_rolv_epi32
+  // CHECK-LABEL: test_mm_maskz_rolv_epi32
   // CHECK: llvm.fshl.v4i32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_rolv_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_rolv_epi32(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_rolv_epi32
+  // CHECK-LABEL: test_mm256_rolv_epi32
   // CHECK: @llvm.fshl.v8i32
   return _mm256_rolv_epi32(__A, __B); 
 }
 
 __m256i test_mm256_mask_rolv_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_rolv_epi32
+  // CHECK-LABEL: test_mm256_mask_rolv_epi32
   // CHECK: @llvm.fshl.v8i32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_rolv_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_rolv_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_rolv_epi32
+  // CHECK-LABEL: test_mm256_maskz_rolv_epi32
   // CHECK: @llvm.fshl.v8i32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_rolv_epi32(__U, __A, __B); 
 }
 
 __m128i test_mm_rolv_epi64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_rolv_epi64
+  // CHECK-LABEL: test_mm_rolv_epi64
   // CHECK: @llvm.fshl.v2i64
   return _mm_rolv_epi64(__A, __B); 
 }
 
 __m128i test_mm_mask_rolv_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_rolv_epi64
+  // CHECK-LABEL: test_mm_mask_rolv_epi64
   // CHECK: @llvm.fshl.v2i64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_rolv_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_rolv_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_rolv_epi64
+  // CHECK-LABEL: test_mm_maskz_rolv_epi64
   // CHECK: @llvm.fshl.v2i64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_rolv_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_rolv_epi64(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_rolv_epi64
+  // CHECK-LABEL: test_mm256_rolv_epi64
   // CHECK: @llvm.fshl.v4i64
   return _mm256_rolv_epi64(__A, __B); 
 }
 
 __m256i test_mm256_mask_rolv_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_rolv_epi64
+  // CHECK-LABEL: test_mm256_mask_rolv_epi64
   // CHECK: @llvm.fshl.v4i64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_rolv_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_rolv_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_rolv_epi64
+  // CHECK-LABEL: test_mm256_maskz_rolv_epi64
   // CHECK: @llvm.fshl.v4i64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_rolv_epi64(__U, __A, __B); 
 }
 
 __m128i test_mm_ror_epi32(__m128i __A) {
-  // CHECK-LABEL: @test_mm_ror_epi32
+  // CHECK-LABEL: test_mm_ror_epi32
   // CHECK: @llvm.fshr.v4i32
   return _mm_ror_epi32(__A, 5); 
 }
 
 __m128i test_mm_mask_ror_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_ror_epi32
+  // CHECK-LABEL: test_mm_mask_ror_epi32
   // CHECK: @llvm.fshr.v4i32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_ror_epi32(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_maskz_ror_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_ror_epi32
+  // CHECK-LABEL: test_mm_maskz_ror_epi32
   // CHECK: @llvm.fshr.v4i32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_ror_epi32(__U, __A, 5); 
 }
 
 __m256i test_mm256_ror_epi32(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_ror_epi32
+  // CHECK-LABEL: test_mm256_ror_epi32
   // CHECK: @llvm.fshr.v8i32
   return _mm256_ror_epi32(__A, 5); 
 }
 
 __m256i test_mm256_mask_ror_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_ror_epi32
+  // CHECK-LABEL: test_mm256_mask_ror_epi32
   // CHECK: @llvm.fshr.v8i32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_ror_epi32(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_ror_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_ror_epi32
+  // CHECK-LABEL: test_mm256_maskz_ror_epi32
   // CHECK: @llvm.fshr.v8i32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_ror_epi32(__U, __A, 5); 
 }
 
 __m128i test_mm_ror_epi64(__m128i __A) {
-  // CHECK-LABEL: @test_mm_ror_epi64
+  // CHECK-LABEL: test_mm_ror_epi64
   // CHECK: @llvm.fshr.v2i64
   return _mm_ror_epi64(__A, 5); 
 }
 
 __m128i test_mm_mask_ror_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_ror_epi64
+  // CHECK-LABEL: test_mm_mask_ror_epi64
   // CHECK: @llvm.fshr.v2i64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_ror_epi64(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_maskz_ror_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_ror_epi64
+  // CHECK-LABEL: test_mm_maskz_ror_epi64
   // CHECK: @llvm.fshr.v2i64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_ror_epi64(__U, __A, 5); 
 }
 
 __m256i test_mm256_ror_epi64(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_ror_epi64
+  // CHECK-LABEL: test_mm256_ror_epi64
   // CHECK: @llvm.fshr.v4i64
   return _mm256_ror_epi64(__A, 5); 
 }
 
 __m256i test_mm256_mask_ror_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_ror_epi64
+  // CHECK-LABEL: test_mm256_mask_ror_epi64
   // CHECK: @llvm.fshr.v4i64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_ror_epi64(__W, __U, __A,5); 
 }
 
 __m256i test_mm256_maskz_ror_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_ror_epi64
+  // CHECK-LABEL: test_mm256_maskz_ror_epi64
   // CHECK: @llvm.fshr.v4i64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_ror_epi64(__U, __A, 5); 
@@ -5876,798 +5878,798 @@ __m256i test_mm256_maskz_ror_epi64(__mmask8 __U, __m256i __A) {
 
 
 __m128i test_mm_rorv_epi32(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_rorv_epi32
+  // CHECK-LABEL: test_mm_rorv_epi32
   // CHECK: @llvm.fshr.v4i32
   return _mm_rorv_epi32(__A, __B); 
 }
 
 __m128i test_mm_mask_rorv_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_rorv_epi32
+  // CHECK-LABEL: test_mm_mask_rorv_epi32
   // CHECK: @llvm.fshr.v4i32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_rorv_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_rorv_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_rorv_epi32
+  // CHECK-LABEL: test_mm_maskz_rorv_epi32
   // CHECK: @llvm.fshr.v4i32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_rorv_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_rorv_epi32(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_rorv_epi32
+  // CHECK-LABEL: test_mm256_rorv_epi32
   // CHECK: @llvm.fshr.v8i32
   return _mm256_rorv_epi32(__A, __B); 
 }
 
 __m256i test_mm256_mask_rorv_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_rorv_epi32
+  // CHECK-LABEL: test_mm256_mask_rorv_epi32
   // CHECK: @llvm.fshr.v8i32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_rorv_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_rorv_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_rorv_epi32
+  // CHECK-LABEL: test_mm256_maskz_rorv_epi32
   // CHECK: @llvm.fshr.v8i32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_rorv_epi32(__U, __A, __B); 
 }
 
 __m128i test_mm_rorv_epi64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_rorv_epi64
+  // CHECK-LABEL: test_mm_rorv_epi64
   // CHECK: @llvm.fshr.v2i64
   return _mm_rorv_epi64(__A, __B); 
 }
 
 __m128i test_mm_mask_rorv_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_rorv_epi64
+  // CHECK-LABEL: test_mm_mask_rorv_epi64
   // CHECK: @llvm.fshr.v2i64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_rorv_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_rorv_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_rorv_epi64
+  // CHECK-LABEL: test_mm_maskz_rorv_epi64
   // CHECK: @llvm.fshr.v2i64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_rorv_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_rorv_epi64(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_rorv_epi64
+  // CHECK-LABEL: test_mm256_rorv_epi64
   // CHECK: @llvm.fshr.v4i64
   return _mm256_rorv_epi64(__A, __B); 
 }
 
 __m256i test_mm256_mask_rorv_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_rorv_epi64
+  // CHECK-LABEL: test_mm256_mask_rorv_epi64
   // CHECK: @llvm.fshr.v4i64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_rorv_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_rorv_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_rorv_epi64
+  // CHECK-LABEL: test_mm256_maskz_rorv_epi64
   // CHECK: @llvm.fshr.v4i64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_rorv_epi64(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_sllv_epi64(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_sllv_epi64
+  // CHECK-LABEL: test_mm_mask_sllv_epi64
   // CHECK: @llvm.x86.avx2.psllv.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_sllv_epi64(__W, __U, __X, __Y); 
 }
 
 __m128i test_mm_maskz_sllv_epi64(__mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_maskz_sllv_epi64
+  // CHECK-LABEL: test_mm_maskz_sllv_epi64
   // CHECK: @llvm.x86.avx2.psllv.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_sllv_epi64(__U, __X, __Y); 
 }
 
 __m256i test_mm256_mask_sllv_epi64(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_sllv_epi64
+  // CHECK-LABEL: test_mm256_mask_sllv_epi64
   // CHECK: @llvm.x86.avx2.psllv.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_sllv_epi64(__W, __U, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_sllv_epi64(__mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_sllv_epi64
+  // CHECK-LABEL: test_mm256_maskz_sllv_epi64
   // CHECK: @llvm.x86.avx2.psllv.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_sllv_epi64(__U, __X, __Y); 
 }
 
 __m128i test_mm_mask_sllv_epi32(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_sllv_epi32
+  // CHECK-LABEL: test_mm_mask_sllv_epi32
   // CHECK: @llvm.x86.avx2.psllv.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_sllv_epi32(__W, __U, __X, __Y); 
 }
 
 __m128i test_mm_maskz_sllv_epi32(__mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_maskz_sllv_epi32
+  // CHECK-LABEL: test_mm_maskz_sllv_epi32
   // CHECK: @llvm.x86.avx2.psllv.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_sllv_epi32(__U, __X, __Y); 
 }
 
 __m256i test_mm256_mask_sllv_epi32(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_sllv_epi32
+  // CHECK-LABEL: test_mm256_mask_sllv_epi32
   // CHECK: @llvm.x86.avx2.psllv.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_sllv_epi32(__W, __U, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_sllv_epi32(__mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_sllv_epi32
+  // CHECK-LABEL: test_mm256_maskz_sllv_epi32
   // CHECK: @llvm.x86.avx2.psllv.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_sllv_epi32(__U, __X, __Y); 
 }
 
 __m128i test_mm_mask_srlv_epi64(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_srlv_epi64
+  // CHECK-LABEL: test_mm_mask_srlv_epi64
   // CHECK: @llvm.x86.avx2.psrlv.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_srlv_epi64(__W, __U, __X, __Y); 
 }
 
 __m128i test_mm_maskz_srlv_epi64(__mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_maskz_srlv_epi64
+  // CHECK-LABEL: test_mm_maskz_srlv_epi64
   // CHECK: @llvm.x86.avx2.psrlv.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_srlv_epi64(__U, __X, __Y); 
 }
 
 __m256i test_mm256_mask_srlv_epi64(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_srlv_epi64
+  // CHECK-LABEL: test_mm256_mask_srlv_epi64
   // CHECK: @llvm.x86.avx2.psrlv.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_srlv_epi64(__W, __U, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_srlv_epi64(__mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_srlv_epi64
+  // CHECK-LABEL: test_mm256_maskz_srlv_epi64
   // CHECK: @llvm.x86.avx2.psrlv.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_srlv_epi64(__U, __X, __Y); 
 }
 
 __m128i test_mm_mask_srlv_epi32(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_srlv_epi32
+  // CHECK-LABEL: test_mm_mask_srlv_epi32
   // CHECK: @llvm.x86.avx2.psrlv.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_srlv_epi32(__W, __U, __X, __Y); 
 }
 
 __m128i test_mm_maskz_srlv_epi32(__mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_maskz_srlv_epi32
+  // CHECK-LABEL: test_mm_maskz_srlv_epi32
   // CHECK: @llvm.x86.avx2.psrlv.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_srlv_epi32(__U, __X, __Y); 
 }
 
 __m256i test_mm256_mask_srlv_epi32(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_srlv_epi32
+  // CHECK-LABEL: test_mm256_mask_srlv_epi32
   // CHECK: @llvm.x86.avx2.psrlv.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_srlv_epi32(__W, __U, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_srlv_epi32(__mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_srlv_epi32
+  // CHECK-LABEL: test_mm256_maskz_srlv_epi32
   // CHECK: @llvm.x86.avx2.psrlv.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_srlv_epi32(__U, __X, __Y); 
 }
 
 __m128i test_mm_mask_srl_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_srl_epi32
+  // CHECK-LABEL: test_mm_mask_srl_epi32
   // CHECK: @llvm.x86.sse2.psrl.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_srl_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srl_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_srl_epi32
+  // CHECK-LABEL: test_mm_maskz_srl_epi32
   // CHECK: @llvm.x86.sse2.psrl.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_srl_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_srl_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_srl_epi32
+  // CHECK-LABEL: test_mm256_mask_srl_epi32
   // CHECK: @llvm.x86.avx2.psrl.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_srl_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srl_epi32(__mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srl_epi32
+  // CHECK-LABEL: test_mm256_maskz_srl_epi32
   // CHECK: @llvm.x86.avx2.psrl.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_srl_epi32(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_srli_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_srli_epi32
+  // CHECK-LABEL: test_mm_mask_srli_epi32
   // CHECK: @llvm.x86.sse2.psrli.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_srli_epi32(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_mask_srli_epi32_2(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_mask_srli_epi32_2
+  // CHECK-LABEL: test_mm_mask_srli_epi32_2
   // CHECK: @llvm.x86.sse2.psrli.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_srli_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srli_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_srli_epi32
+  // CHECK-LABEL: test_mm_maskz_srli_epi32
   // CHECK: @llvm.x86.sse2.psrli.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_srli_epi32(__U, __A, 5); 
 }
 
 __m128i test_mm_maskz_srli_epi32_2(__mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_maskz_srli_epi32_2
+  // CHECK-LABEL: test_mm_maskz_srli_epi32_2
   // CHECK: @llvm.x86.sse2.psrli.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_srli_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_srli_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_srli_epi32
+  // CHECK-LABEL: test_mm256_mask_srli_epi32
   // CHECK: @llvm.x86.avx2.psrli.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_srli_epi32(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_mask_srli_epi32_2(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_mask_srli_epi32_2
+  // CHECK-LABEL: test_mm256_mask_srli_epi32_2
   // CHECK: @llvm.x86.avx2.psrli.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_srli_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srli_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_srli_epi32
+  // CHECK-LABEL: test_mm256_maskz_srli_epi32
   // CHECK: @llvm.x86.avx2.psrli.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_srli_epi32(__U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_srli_epi32_2(__mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srli_epi32_2
+  // CHECK-LABEL: test_mm256_maskz_srli_epi32_2
   // CHECK: @llvm.x86.avx2.psrli.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_srli_epi32(__U, __A, __B); 
 }
 __m128i test_mm_mask_srl_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_srl_epi64
+  // CHECK-LABEL: test_mm_mask_srl_epi64
   // CHECK: @llvm.x86.sse2.psrl.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_srl_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srl_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_srl_epi64
+  // CHECK-LABEL: test_mm_maskz_srl_epi64
   // CHECK: @llvm.x86.sse2.psrl.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_srl_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_srl_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_srl_epi64
+  // CHECK-LABEL: test_mm256_mask_srl_epi64
   // CHECK: @llvm.x86.avx2.psrl.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_srl_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srl_epi64(__mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srl_epi64
+  // CHECK-LABEL: test_mm256_maskz_srl_epi64
   // CHECK: @llvm.x86.avx2.psrl.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_srl_epi64(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_srli_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_srli_epi64
+  // CHECK-LABEL: test_mm_mask_srli_epi64
   // CHECK: @llvm.x86.sse2.psrli.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_srli_epi64(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_mask_srli_epi64_2(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_mask_srli_epi64_2
+  // CHECK-LABEL: test_mm_mask_srli_epi64_2
   // CHECK: @llvm.x86.sse2.psrli.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_srli_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srli_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_srli_epi64
+  // CHECK-LABEL: test_mm_maskz_srli_epi64
   // CHECK: @llvm.x86.sse2.psrli.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_srli_epi64(__U, __A, 5); 
 }
 
 __m128i test_mm_maskz_srli_epi64_2(__mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_maskz_srli_epi64_2
+  // CHECK-LABEL: test_mm_maskz_srli_epi64_2
   // CHECK: @llvm.x86.sse2.psrli.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_srli_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_srli_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_srli_epi64
+  // CHECK-LABEL: test_mm256_mask_srli_epi64
   // CHECK: @llvm.x86.avx2.psrli.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_srli_epi64(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_mask_srli_epi64_2(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_mask_srli_epi64_2
+  // CHECK-LABEL: test_mm256_mask_srli_epi64_2
   // CHECK: @llvm.x86.avx2.psrli.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_srli_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srli_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_srli_epi64
+  // CHECK-LABEL: test_mm256_maskz_srli_epi64
   // CHECK: @llvm.x86.avx2.psrli.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_srli_epi64(__U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_srli_epi64_2(__mmask8 __U,__m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srli_epi64_2
+  // CHECK-LABEL: test_mm256_maskz_srli_epi64_2
   // CHECK: @llvm.x86.avx2.psrli.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_srli_epi64(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_sll_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_sll_epi32
+  // CHECK-LABEL: test_mm_mask_sll_epi32
   // CHECK: @llvm.x86.sse2.psll.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_sll_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_sll_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_sll_epi32
+  // CHECK-LABEL: test_mm_maskz_sll_epi32
   // CHECK: @llvm.x86.sse2.psll.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_sll_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_sll_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_sll_epi32
+  // CHECK-LABEL: test_mm256_mask_sll_epi32
   // CHECK: @llvm.x86.avx2.psll.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_sll_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_sll_epi32(__mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sll_epi32
+  // CHECK-LABEL: test_mm256_maskz_sll_epi32
   // CHECK: @llvm.x86.avx2.psll.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_sll_epi32(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_slli_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_slli_epi32
+  // CHECK-LABEL: test_mm_mask_slli_epi32
   // CHECK: @llvm.x86.sse2.pslli.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_slli_epi32(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_mask_slli_epi32_2(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_mask_slli_epi32_2
+  // CHECK-LABEL: test_mm_mask_slli_epi32_2
   // CHECK: @llvm.x86.sse2.pslli.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_slli_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_slli_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_slli_epi32
+  // CHECK-LABEL: test_mm_maskz_slli_epi32
   // CHECK: @llvm.x86.sse2.pslli.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_slli_epi32(__U, __A, 5); 
 }
 
 __m128i test_mm_maskz_slli_epi32_2(__mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_maskz_slli_epi32_2
+  // CHECK-LABEL: test_mm_maskz_slli_epi32_2
   // CHECK: @llvm.x86.sse2.pslli.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_slli_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_slli_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_slli_epi32
+  // CHECK-LABEL: test_mm256_mask_slli_epi32
   // CHECK: @llvm.x86.avx2.pslli.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_slli_epi32(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_mask_slli_epi32_2(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_mask_slli_epi32_2
+  // CHECK-LABEL: test_mm256_mask_slli_epi32_2
   // CHECK: @llvm.x86.avx2.pslli.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_slli_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_slli_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_slli_epi32
+  // CHECK-LABEL: test_mm256_maskz_slli_epi32
   // CHECK: @llvm.x86.avx2.pslli.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_slli_epi32(__U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_slli_epi32_2(__mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_maskz_slli_epi32_2
+  // CHECK-LABEL: test_mm256_maskz_slli_epi32_2
   // CHECK: @llvm.x86.avx2.pslli.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_slli_epi32(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_sll_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_sll_epi64
+  // CHECK-LABEL: test_mm_mask_sll_epi64
   // CHECK: @llvm.x86.sse2.psll.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_sll_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_sll_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_sll_epi64
+  // CHECK-LABEL: test_mm_maskz_sll_epi64
   // CHECK: @llvm.x86.sse2.psll.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_sll_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_sll_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_sll_epi64
+  // CHECK-LABEL: test_mm256_mask_sll_epi64
   // CHECK: @llvm.x86.avx2.psll.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_sll_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_sll_epi64(__mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sll_epi64
+  // CHECK-LABEL: test_mm256_maskz_sll_epi64
   // CHECK: @llvm.x86.avx2.psll.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_sll_epi64(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_slli_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_slli_epi64
+  // CHECK-LABEL: test_mm_mask_slli_epi64
   // CHECK: @llvm.x86.sse2.pslli.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_slli_epi64(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_mask_slli_epi64_2(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_mask_slli_epi64_2
+  // CHECK-LABEL: test_mm_mask_slli_epi64_2
   // CHECK: @llvm.x86.sse2.pslli.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_slli_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_slli_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_slli_epi64
+  // CHECK-LABEL: test_mm_maskz_slli_epi64
   // CHECK: @llvm.x86.sse2.pslli.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_slli_epi64(__U, __A, 5); 
 }
 
 __m128i test_mm_maskz_slli_epi64_2(__mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_maskz_slli_epi64_2
+  // CHECK-LABEL: test_mm_maskz_slli_epi64_2
   // CHECK: @llvm.x86.sse2.pslli.q
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_slli_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_slli_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_slli_epi64
+  // CHECK-LABEL: test_mm256_mask_slli_epi64
   // CHECK: @llvm.x86.avx2.pslli.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_slli_epi64(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_mask_slli_epi64_2(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_mask_slli_epi64_2
+  // CHECK-LABEL: test_mm256_mask_slli_epi64_2
   // CHECK: @llvm.x86.avx2.pslli.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_slli_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_slli_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_slli_epi64
+  // CHECK-LABEL: test_mm256_maskz_slli_epi64
   // CHECK: @llvm.x86.avx2.pslli.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_slli_epi64(__U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_slli_epi64_2(__mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_maskz_slli_epi64_2
+  // CHECK-LABEL: test_mm256_maskz_slli_epi64_2
   // CHECK: @llvm.x86.avx2.pslli.q
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_slli_epi64(__U, __A, __B);
 }
 
 __m128i test_mm_mask_srav_epi32(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_srav_epi32
+  // CHECK-LABEL: test_mm_mask_srav_epi32
   // CHECK: @llvm.x86.avx2.psrav.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_srav_epi32(__W, __U, __X, __Y); 
 }
 
 __m128i test_mm_maskz_srav_epi32(__mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_maskz_srav_epi32
+  // CHECK-LABEL: test_mm_maskz_srav_epi32
   // CHECK: @llvm.x86.avx2.psrav.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_srav_epi32(__U, __X, __Y); 
 }
 
 __m256i test_mm256_mask_srav_epi32(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_srav_epi32
+  // CHECK-LABEL: test_mm256_mask_srav_epi32
   // CHECK: @llvm.x86.avx2.psrav.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_srav_epi32(__W, __U, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_srav_epi32(__mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_srav_epi32
+  // CHECK-LABEL: test_mm256_maskz_srav_epi32
   // CHECK: @llvm.x86.avx2.psrav.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_srav_epi32(__U, __X, __Y); 
 }
 
 __m128i test_mm_srav_epi64(__m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_srav_epi64
+  // CHECK-LABEL: test_mm_srav_epi64
   // CHECK: @llvm.x86.avx512.psrav.q.128
   return _mm_srav_epi64(__X, __Y); 
 }
 
 __m128i test_mm_mask_srav_epi64(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_srav_epi64
+  // CHECK-LABEL: test_mm_mask_srav_epi64
   // CHECK: @llvm.x86.avx512.psrav.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_srav_epi64(__W, __U, __X, __Y); 
 }
 
 __m128i test_mm_maskz_srav_epi64(__mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_maskz_srav_epi64
+  // CHECK-LABEL: test_mm_maskz_srav_epi64
   // CHECK: @llvm.x86.avx512.psrav.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_srav_epi64(__U, __X, __Y); 
 }
 
 __m256i test_mm256_srav_epi64(__m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_srav_epi64
+  // CHECK-LABEL: test_mm256_srav_epi64
   // CHECK: @llvm.x86.avx512.psrav.q.256
   return _mm256_srav_epi64(__X, __Y); 
 }
 
 __m256i test_mm256_mask_srav_epi64(__m256i __W, __mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_srav_epi64
+  // CHECK-LABEL: test_mm256_mask_srav_epi64
   // CHECK: @llvm.x86.avx512.psrav.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_srav_epi64(__W, __U, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_srav_epi64(__mmask8 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_srav_epi64
+  // CHECK-LABEL: test_mm256_maskz_srav_epi64
   // CHECK: @llvm.x86.avx512.psrav.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_srav_epi64(__U, __X, __Y); 
 }
 
 void test_mm_store_epi32(void *__P, __m128i __A) {
-  // CHECK-LABEL: @test_mm_store_epi32
+  // CHECK-LABEL: test_mm_store_epi32
   // CHECK: store <2 x i64> %{{.*}}, ptr %{{.*}}
   return _mm_store_epi32(__P, __A);
 }
 
 void test_mm_mask_store_epi32(void *__P, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_store_epi32
+  // CHECK-LABEL: test_mm_mask_store_epi32
   // CHECK: @llvm.masked.store.v4i32.p0(<4 x i32> %{{.*}}, ptr %{{.}}, i32 16, <4 x i1> %{{.*}})
   return _mm_mask_store_epi32(__P, __U, __A); 
 }
 
 void test_mm256_store_epi32(void *__P, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_store_epi32
+  // CHECK-LABEL: test_mm256_store_epi32
   // CHECK: store <4 x i64> %{{.*}}, ptr %{{.*}}
   return _mm256_store_epi32(__P, __A);
 }
 
 void test_mm256_mask_store_epi32(void *__P, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_store_epi32
+  // CHECK-LABEL: test_mm256_mask_store_epi32
   // CHECK: @llvm.masked.store.v8i32.p0(<8 x i32> %{{.*}}, ptr %{{.}}, i32 32, <8 x i1> %{{.*}})
   return _mm256_mask_store_epi32(__P, __U, __A); 
 }
 
 __m128i test_mm_mask_mov_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_mov_epi32
+  // CHECK-LABEL: test_mm_mask_mov_epi32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_mov_epi32(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_mov_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_mov_epi32
+  // CHECK-LABEL: test_mm_maskz_mov_epi32
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_mov_epi32(__U, __A); 
 }
 
 __m256i test_mm256_mask_mov_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_mov_epi32
+  // CHECK-LABEL: test_mm256_mask_mov_epi32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_mov_epi32(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_mov_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_mov_epi32
+  // CHECK-LABEL: test_mm256_maskz_mov_epi32
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_mov_epi32(__U, __A); 
 }
 
 __m128i test_mm_mask_mov_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_mov_epi64
+  // CHECK-LABEL: test_mm_mask_mov_epi64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_mov_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_mov_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_mov_epi64
+  // CHECK-LABEL: test_mm_maskz_mov_epi64
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_mov_epi64(__U, __A); 
 }
 
 __m256i test_mm256_mask_mov_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_mov_epi64
+  // CHECK-LABEL: test_mm256_mask_mov_epi64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_mov_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_mov_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_mov_epi64
+  // CHECK-LABEL: test_mm256_maskz_mov_epi64
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_mov_epi64(__U, __A); 
 }
 
 __m128i test_mm_load_epi32(void const *__P) {
-  // CHECK-LABEL: @test_mm_load_epi32
+  // CHECK-LABEL: test_mm_load_epi32
   // CHECK: load <2 x i64>, ptr %{{.*}}
   return _mm_load_epi32(__P);
 }
 
 __m128i test_mm_mask_load_epi32(__m128i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_load_epi32
+  // CHECK-LABEL: test_mm_mask_load_epi32
   // CHECK: @llvm.masked.load.v4i32.p0(ptr %{{.*}}, i32 16, <4 x i1> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_mask_load_epi32(__W, __U, __P); 
 }
 
 __m128i test_mm_maskz_load_epi32(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_load_epi32
+  // CHECK-LABEL: test_mm_maskz_load_epi32
   // CHECK: @llvm.masked.load.v4i32.p0(ptr %{{.*}}, i32 16, <4 x i1> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_maskz_load_epi32(__U, __P); 
 }
 
 __m256i test_mm256_load_epi32(void const *__P) {
-  // CHECK-LABEL: @test_mm256_load_epi32
+  // CHECK-LABEL: test_mm256_load_epi32
   // CHECK: load <4 x i64>, ptr %{{.*}}
   return _mm256_load_epi32(__P);
 }
 
 __m256i test_mm256_mask_load_epi32(__m256i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_load_epi32
+  // CHECK-LABEL: test_mm256_mask_load_epi32
   // CHECK: @llvm.masked.load.v8i32.p0(ptr %{{.*}}, i32 32, <8 x i1> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_mask_load_epi32(__W, __U, __P); 
 }
 
 __m256i test_mm256_maskz_load_epi32(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_load_epi32
+  // CHECK-LABEL: test_mm256_maskz_load_epi32
   // CHECK: @llvm.masked.load.v8i32.p0(ptr %{{.*}}, i32 32, <8 x i1> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_maskz_load_epi32(__U, __P); 
 }
 
 __m128i test_mm_load_epi64(void const *__P) {
-  // CHECK-LABEL: @test_mm_load_epi64
+  // CHECK-LABEL: test_mm_load_epi64
   // CHECK: load <2 x i64>, ptr %{{.*}}
   return _mm_load_epi64(__P);
 }
 
 __m128i test_mm_mask_load_epi64(__m128i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_load_epi64
+  // CHECK-LABEL: test_mm_mask_load_epi64
   // CHECK: @llvm.masked.load.v2i64.p0(ptr %{{.*}}, i32 16, <2 x i1> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_mask_load_epi64(__W, __U, __P); 
 }
 
 __m128i test_mm_maskz_load_epi64(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_load_epi64
+  // CHECK-LABEL: test_mm_maskz_load_epi64
   // CHECK: @llvm.masked.load.v2i64.p0(ptr %{{.*}}, i32 16, <2 x i1> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_maskz_load_epi64(__U, __P); 
 }
 
 __m256i test_mm256_load_epi64(void const *__P) {
-  // CHECK-LABEL: @test_mm256_load_epi64
+  // CHECK-LABEL: test_mm256_load_epi64
   // CHECK: load <4 x i64>, ptr %{{.*}}
   return _mm256_load_epi64(__P);
 }
 
 __m256i test_mm256_mask_load_epi64(__m256i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_load_epi64
+  // CHECK-LABEL: test_mm256_mask_load_epi64
   // CHECK: @llvm.masked.load.v4i64.p0(ptr %{{.*}}, i32 32, <4 x i1> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_mask_load_epi64(__W, __U, __P); 
 }
 
 __m256i test_mm256_maskz_load_epi64(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_load_epi64
+  // CHECK-LABEL: test_mm256_maskz_load_epi64
   // CHECK: @llvm.masked.load.v4i64.p0(ptr %{{.*}}, i32 32, <4 x i1> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_maskz_load_epi64(__U, __P); 
 }
 
 void test_mm_store_epi64(void *__P, __m128i __A) {
-  // CHECK-LABEL: @test_mm_store_epi64
+  // CHECK-LABEL: test_mm_store_epi64
   // CHECK: store <2 x i64> %{{.*}}, ptr %{{.*}}
   return _mm_store_epi64(__P, __A);
 }
 
 void test_mm_mask_store_epi64(void *__P, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_store_epi64
+  // CHECK-LABEL: test_mm_mask_store_epi64
   // CHECK: @llvm.masked.store.v2i64.p0(<2 x i64> %{{.*}}, ptr %{{.*}}, i32 16, <2 x i1> %{{.*}})
   return _mm_mask_store_epi64(__P, __U, __A); 
 }
 
 void test_mm256_store_epi64(void *__P, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_store_epi64
+  // CHECK-LABEL: test_mm256_store_epi64
   // CHECK: store <4 x i64> %{{.*}}, ptr %{{.*}}
   return _mm256_store_epi64(__P, __A);
 }
 
 void test_mm256_mask_store_epi64(void *__P, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_store_epi64
+  // CHECK-LABEL: test_mm256_mask_store_epi64
   // CHECK: @llvm.masked.store.v4i64.p0(<4 x i64> %{{.*}}, ptr %{{.*}}, i32 32, <4 x i1> %{{.*}})
   return _mm256_mask_store_epi64(__P, __U, __A); 
 }
 
 __m128d test_mm_mask_movedup_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_movedup_pd
+  // CHECK-LABEL: test_mm_mask_movedup_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> zeroinitializer
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_movedup_pd(__W, __U, __A); 
 }
 
 __m128d test_mm_maskz_movedup_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_movedup_pd
+  // CHECK-LABEL: test_mm_maskz_movedup_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> zeroinitializer
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_movedup_pd(__U, __A); 
 }
 
 __m256d test_mm256_mask_movedup_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_movedup_pd
+  // CHECK-LABEL: test_mm256_mask_movedup_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_movedup_pd(__W, __U, __A); 
 }
 
 __m256d test_mm256_maskz_movedup_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_movedup_pd
+  // CHECK-LABEL: test_mm256_maskz_movedup_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_movedup_pd(__U, __A); 
 }
 
 __m128i test_mm_mask_set1_epi32(__m128i __O, __mmask8 __M) {
-  // CHECK-LABEL: @test_mm_mask_set1_epi32
+  // CHECK-LABEL: test_mm_mask_set1_epi32
   // CHECK: insertelement <4 x i32> poison, i32 %{{.*}}, i32 0
   // CHECK: insertelement <4 x i32> %{{.*}}32 1
   // CHECK: insertelement <4 x i32> %{{.*}}32 2
@@ -6678,7 +6680,7 @@ __m128i test_mm_mask_set1_epi32(__m128i __O, __mmask8 __M) {
 }
 
 __m128i test_mm_maskz_set1_epi32(__mmask8 __M) {
-  // CHECK-LABEL: @test_mm_maskz_set1_epi32
+  // CHECK-LABEL: test_mm_maskz_set1_epi32
   // CHECK: insertelement <4 x i32> poison, i32 %{{.*}}, i32 0
   // CHECK: insertelement <4 x i32> %{{.*}}32 1
   // CHECK: insertelement <4 x i32> %{{.*}}32 2
@@ -6689,7 +6691,7 @@ __m128i test_mm_maskz_set1_epi32(__mmask8 __M) {
 }
 
 __m256i test_mm256_mask_set1_epi32(__m256i __O, __mmask8 __M) {
-  // CHECK-LABEL: @test_mm256_mask_set1_epi32
+  // CHECK-LABEL: test_mm256_mask_set1_epi32
   // CHECK:  insertelement <8 x i32> poison, i32 %{{.*}}, i32 0
   // CHECK:  insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 1
   // CHECK:  insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 2
@@ -6703,7 +6705,7 @@ __m256i test_mm256_mask_set1_epi32(__m256i __O, __mmask8 __M) {
 }
 
 __m256i test_mm256_maskz_set1_epi32(__mmask8 __M) {
-  // CHECK-LABEL: @test_mm256_maskz_set1_epi32
+  // CHECK-LABEL: test_mm256_maskz_set1_epi32
   // CHECK:  insertelement <8 x i32> poison, i32 %{{.*}}, i32 0
   // CHECK:  insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 1
   // CHECK:  insertelement <8 x i32> %{{.*}}, i32 %{{.*}}, i32 2
@@ -6717,7 +6719,7 @@ __m256i test_mm256_maskz_set1_epi32(__mmask8 __M) {
 }
 
 __m128i test_mm_mask_set1_epi64(__m128i __O, __mmask8 __M, long long __A) {
-  // CHECK-LABEL: @test_mm_mask_set1_epi64
+  // CHECK-LABEL: test_mm_mask_set1_epi64
   // CHECK: insertelement <2 x i64> poison, i64 %{{.*}}, i32 0
   // CHECK: insertelement <2 x i64> %{{.*}}, i64 %{{.*}}, i32 1
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -6726,7 +6728,7 @@ __m128i test_mm_mask_set1_epi64(__m128i __O, __mmask8 __M, long long __A) {
 }
 
 __m128i test_mm_maskz_set1_epi64(__mmask8 __M, long long __A) {
-  // CHECK-LABEL: @test_mm_maskz_set1_epi64
+  // CHECK-LABEL: test_mm_maskz_set1_epi64
   // CHECK: insertelement <2 x i64> poison, i64 %{{.*}}, i32 0
   // CHECK: insertelement <2 x i64> %{{.*}}, i64 %{{.*}}, i32 1
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
@@ -6735,7 +6737,7 @@ __m128i test_mm_maskz_set1_epi64(__mmask8 __M, long long __A) {
 }
 
 __m256i test_mm256_mask_set1_epi64(__m256i __O, __mmask8 __M, long long __A) {
-  // CHECK-LABEL: @test_mm256_mask_set1_epi64
+  // CHECK-LABEL: test_mm256_mask_set1_epi64
   // CHECK: insertelement <4 x i64> poison, i64 %{{.*}}, i32 0
   // CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i32 1
   // CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i32 2
@@ -6746,7 +6748,7 @@ __m256i test_mm256_mask_set1_epi64(__m256i __O, __mmask8 __M, long long __A) {
 }
 
 __m256i test_mm256_maskz_set1_epi64(__mmask8 __M, long long __A) {
-  // CHECK-LABEL: @test_mm256_maskz_set1_epi64
+  // CHECK-LABEL: test_mm256_maskz_set1_epi64
   // CHECK: insertelement <4 x i64> poison, i64 %{{.*}}, i32 0
   // CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i32 1
   // CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i32 2
@@ -6757,646 +6759,646 @@ __m256i test_mm256_maskz_set1_epi64(__mmask8 __M, long long __A) {
 }
 
 __m128d test_mm_fixupimm_pd(__m128d __A, __m128d __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_fixupimm_pd
+  // CHECK-LABEL: test_mm_fixupimm_pd
   // CHECK: @llvm.x86.avx512.mask.fixupimm.pd.128
   return _mm_fixupimm_pd(__A, __B, __C, 5); 
 }
 
 __m128d test_mm_mask_fixupimm_pd(__m128d __A, __mmask8 __U, __m128d __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_mask_fixupimm_pd
+  // CHECK-LABEL: test_mm_mask_fixupimm_pd
   // CHECK: @llvm.x86.avx512.mask.fixupimm.pd.128
   return _mm_mask_fixupimm_pd(__A, __U, __B, __C, 5); 
 }
 
 __m128d test_mm_maskz_fixupimm_pd(__mmask8 __U, __m128d __A, __m128d __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_maskz_fixupimm_pd
+  // CHECK-LABEL: test_mm_maskz_fixupimm_pd
   // CHECK: @llvm.x86.avx512.maskz.fixupimm.pd.128
   return _mm_maskz_fixupimm_pd(__U, __A, __B, __C, 5); 
 }
 
 __m256d test_mm256_fixupimm_pd(__m256d __A, __m256d __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_fixupimm_pd
+  // CHECK-LABEL: test_mm256_fixupimm_pd
   // CHECK: @llvm.x86.avx512.mask.fixupimm.pd.256
   return _mm256_fixupimm_pd(__A, __B, __C, 5); 
 }
 
 __m256d test_mm256_mask_fixupimm_pd(__m256d __A, __mmask8 __U, __m256d __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_mask_fixupimm_pd
+  // CHECK-LABEL: test_mm256_mask_fixupimm_pd
   // CHECK: @llvm.x86.avx512.mask.fixupimm.pd.256
   return _mm256_mask_fixupimm_pd(__A, __U, __B, __C, 5); 
 }
 
 __m256d test_mm256_maskz_fixupimm_pd(__mmask8 __U, __m256d __A, __m256d __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fixupimm_pd
+  // CHECK-LABEL: test_mm256_maskz_fixupimm_pd
   // CHECK: @llvm.x86.avx512.maskz.fixupimm.pd.256
   return _mm256_maskz_fixupimm_pd(__U, __A, __B, __C, 5); 
 }
 
 __m128 test_mm_fixupimm_ps(__m128 __A, __m128 __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_fixupimm_ps
+  // CHECK-LABEL: test_mm_fixupimm_ps
   // CHECK: @llvm.x86.avx512.mask.fixupimm.ps.128
   return _mm_fixupimm_ps(__A, __B, __C, 5); 
 }
 
 __m128 test_mm_mask_fixupimm_ps(__m128 __A, __mmask8 __U, __m128 __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_mask_fixupimm_ps
+  // CHECK-LABEL: test_mm_mask_fixupimm_ps
   // CHECK: @llvm.x86.avx512.mask.fixupimm.ps.128
   return _mm_mask_fixupimm_ps(__A, __U, __B, __C, 5); 
 }
 
 __m128 test_mm_maskz_fixupimm_ps(__mmask8 __U, __m128 __A, __m128 __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_maskz_fixupimm_ps
+  // CHECK-LABEL: test_mm_maskz_fixupimm_ps
   // CHECK: @llvm.x86.avx512.maskz.fixupimm.ps.128
   return _mm_maskz_fixupimm_ps(__U, __A, __B, __C, 5); 
 }
 
 __m256 test_mm256_fixupimm_ps(__m256 __A, __m256 __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_fixupimm_ps
+  // CHECK-LABEL: test_mm256_fixupimm_ps
   // CHECK: @llvm.x86.avx512.mask.fixupimm.ps.256
   return _mm256_fixupimm_ps(__A, __B, __C, 5); 
 }
 
 __m256 test_mm256_mask_fixupimm_ps(__m256 __A, __mmask8 __U, __m256 __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_mask_fixupimm_ps
+  // CHECK-LABEL: test_mm256_mask_fixupimm_ps
   // CHECK: @llvm.x86.avx512.mask.fixupimm.ps.256
   return _mm256_mask_fixupimm_ps(__A, __U, __B, __C, 5); 
 }
 
 __m256 test_mm256_maskz_fixupimm_ps(__mmask8 __U, __m256 __A, __m256 __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fixupimm_ps
+  // CHECK-LABEL: test_mm256_maskz_fixupimm_ps
   // CHECK: @llvm.x86.avx512.maskz.fixupimm.ps.256
   return _mm256_maskz_fixupimm_ps(__U, __A, __B, __C, 5); 
 }
 
 __m128d test_mm_mask_load_pd(__m128d __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_load_pd
+  // CHECK-LABEL: test_mm_mask_load_pd
   // CHECK: @llvm.masked.load.v2f64.p0(ptr %{{.*}}, i32 16, <2 x i1> %{{.*}}, <2 x double> %{{.*}})
   return _mm_mask_load_pd(__W, __U, __P); 
 }
 
 __m128d test_mm_maskz_load_pd(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_load_pd
+  // CHECK-LABEL: test_mm_maskz_load_pd
   // CHECK: @llvm.masked.load.v2f64.p0(ptr %{{.*}}, i32 16, <2 x i1> %{{.*}}, <2 x double> %{{.*}})
   return _mm_maskz_load_pd(__U, __P); 
 }
 
 __m256d test_mm256_mask_load_pd(__m256d __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_load_pd
+  // CHECK-LABEL: test_mm256_mask_load_pd
   // CHECK: @llvm.masked.load.v4f64.p0(ptr %{{.*}}, i32 32, <4 x i1> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_mask_load_pd(__W, __U, __P); 
 }
 
 __m256d test_mm256_maskz_load_pd(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_load_pd
+  // CHECK-LABEL: test_mm256_maskz_load_pd
   // CHECK: @llvm.masked.load.v4f64.p0(ptr %{{.*}}, i32 32, <4 x i1> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_maskz_load_pd(__U, __P); 
 }
 
 __m128 test_mm_mask_load_ps(__m128 __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_load_ps
+  // CHECK-LABEL: test_mm_mask_load_ps
   // CHECK: @llvm.masked.load.v4f32.p0(ptr %{{.*}}, i32 16, <4 x i1> %{{.*}}, <4 x float> %{{.*}})
   return _mm_mask_load_ps(__W, __U, __P); 
 }
 
 __m128 test_mm_maskz_load_ps(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_load_ps
+  // CHECK-LABEL: test_mm_maskz_load_ps
   // CHECK: @llvm.masked.load.v4f32.p0(ptr %{{.*}}, i32 16, <4 x i1> %{{.*}}, <4 x float> %{{.*}})
   return _mm_maskz_load_ps(__U, __P); 
 }
 
 __m256 test_mm256_mask_load_ps(__m256 __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_load_ps
+  // CHECK-LABEL: test_mm256_mask_load_ps
   // CHECK: @llvm.masked.load.v8f32.p0(ptr %{{.*}}, i32 32, <8 x i1> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_mask_load_ps(__W, __U, __P); 
 }
 
 __m256 test_mm256_maskz_load_ps(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_load_ps
+  // CHECK-LABEL: test_mm256_maskz_load_ps
   // CHECK: @llvm.masked.load.v8f32.p0(ptr %{{.*}}, i32 32, <8 x i1> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_maskz_load_ps(__U, __P); 
 }
 
 __m128i test_mm_loadu_epi64(void const *__P) {
-  // CHECK-LABEL: @test_mm_loadu_epi64
+  // CHECK-LABEL: test_mm_loadu_epi64
   // CHECK: load <2 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm_loadu_epi64(__P);
 }
 
 __m128i test_mm_mask_loadu_epi64(__m128i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_loadu_epi64
+  // CHECK-LABEL: test_mm_mask_loadu_epi64
   // CHECK: @llvm.masked.load.v2i64.p0(ptr %{{.*}}, i32 1, <2 x i1> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_mask_loadu_epi64(__W, __U, __P); 
 }
 
 __m128i test_mm_maskz_loadu_epi64(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_loadu_epi64
+  // CHECK-LABEL: test_mm_maskz_loadu_epi64
   // CHECK: @llvm.masked.load.v2i64.p0(ptr %{{.*}}, i32 1, <2 x i1> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_maskz_loadu_epi64(__U, __P); 
 }
 
 __m256i test_mm256_loadu_epi64(void const *__P) {
-  // CHECK-LABEL: @test_mm256_loadu_epi64
+  // CHECK-LABEL: test_mm256_loadu_epi64
   // CHECK: load <4 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm256_loadu_epi64(__P);
 }
 
 __m256i test_mm256_mask_loadu_epi64(__m256i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_loadu_epi64
+  // CHECK-LABEL: test_mm256_mask_loadu_epi64
   // CHECK: @llvm.masked.load.v4i64.p0(ptr %{{.*}}, i32 1, <4 x i1> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_mask_loadu_epi64(__W, __U, __P); 
 }
 
 __m256i test_mm256_maskz_loadu_epi64(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_loadu_epi64
+  // CHECK-LABEL: test_mm256_maskz_loadu_epi64
   // CHECK: @llvm.masked.load.v4i64.p0(ptr %{{.*}}, i32 1, <4 x i1> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_maskz_loadu_epi64(__U, __P); 
 }
 
 __m128i test_mm_loadu_epi32(void const *__P) {
-  // CHECK-LABEL: @test_mm_loadu_epi32
+  // CHECK-LABEL: test_mm_loadu_epi32
   // CHECK: load <2 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm_loadu_epi32(__P);
 }
 
 __m128i test_mm_mask_loadu_epi32(__m128i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_loadu_epi32
+  // CHECK-LABEL: test_mm_mask_loadu_epi32
   // CHECK: @llvm.masked.load.v4i32.p0(ptr %{{.*}}, i32 1, <4 x i1> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_mask_loadu_epi32(__W, __U, __P); 
 }
 
 __m128i test_mm_maskz_loadu_epi32(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_loadu_epi32
+  // CHECK-LABEL: test_mm_maskz_loadu_epi32
   // CHECK: @llvm.masked.load.v4i32.p0(ptr %{{.*}}, i32 1, <4 x i1> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_maskz_loadu_epi32(__U, __P); 
 }
 
 __m256i test_mm256_loadu_epi32(void const *__P) {
-  // CHECK-LABEL: @test_mm256_loadu_epi32
+  // CHECK-LABEL: test_mm256_loadu_epi32
   // CHECK: load <4 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm256_loadu_epi32(__P);
 }
 
 __m256i test_mm256_mask_loadu_epi32(__m256i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_loadu_epi32
+  // CHECK-LABEL: test_mm256_mask_loadu_epi32
   // CHECK: @llvm.masked.load.v8i32.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_mask_loadu_epi32(__W, __U, __P); 
 }
 
 __m256i test_mm256_maskz_loadu_epi32(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_loadu_epi32
+  // CHECK-LABEL: test_mm256_maskz_loadu_epi32
   // CHECK: @llvm.masked.load.v8i32.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_maskz_loadu_epi32(__U, __P); 
 }
 
 __m128d test_mm_mask_loadu_pd(__m128d __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_loadu_pd
+  // CHECK-LABEL: test_mm_mask_loadu_pd
   // CHECK: @llvm.masked.load.v2f64.p0(ptr %{{.*}}, i32 1, <2 x i1> %{{.*}}, <2 x double> %{{.*}})
   return _mm_mask_loadu_pd(__W, __U, __P); 
 }
 
 __m128d test_mm_maskz_loadu_pd(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_loadu_pd
+  // CHECK-LABEL: test_mm_maskz_loadu_pd
   // CHECK: @llvm.masked.load.v2f64.p0(ptr %{{.*}}, i32 1, <2 x i1> %{{.*}}, <2 x double> %{{.*}})
   return _mm_maskz_loadu_pd(__U, __P); 
 }
 
 __m256d test_mm256_mask_loadu_pd(__m256d __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_loadu_pd
+  // CHECK-LABEL: test_mm256_mask_loadu_pd
   // CHECK: @llvm.masked.load.v4f64.p0(ptr %{{.*}}, i32 1, <4 x i1> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_mask_loadu_pd(__W, __U, __P); 
 }
 
 __m256d test_mm256_maskz_loadu_pd(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_loadu_pd
+  // CHECK-LABEL: test_mm256_maskz_loadu_pd
   // CHECK: @llvm.masked.load.v4f64.p0(ptr %{{.*}}, i32 1, <4 x i1> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_maskz_loadu_pd(__U, __P); 
 }
 
 __m128 test_mm_mask_loadu_ps(__m128 __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_loadu_ps
+  // CHECK-LABEL: test_mm_mask_loadu_ps
   // CHECK: @llvm.masked.load.v4f32.p0(ptr %{{.*}}, i32 1, <4 x i1> %{{.*}}, <4 x float> %{{.*}})
   return _mm_mask_loadu_ps(__W, __U, __P); 
 }
 
 __m128 test_mm_maskz_loadu_ps(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_loadu_ps
+  // CHECK-LABEL: test_mm_maskz_loadu_ps
   // CHECK: @llvm.masked.load.v4f32.p0(ptr %{{.*}}, i32 1, <4 x i1> %{{.*}}, <4 x float> %{{.*}})
   return _mm_maskz_loadu_ps(__U, __P); 
 }
 
 __m256 test_mm256_mask_loadu_ps(__m256 __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_loadu_ps
+  // CHECK-LABEL: test_mm256_mask_loadu_ps
   // CHECK: @llvm.masked.load.v8f32.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_mask_loadu_ps(__W, __U, __P); 
 }
 
 __m256 test_mm256_maskz_loadu_ps(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_loadu_ps
+  // CHECK-LABEL: test_mm256_maskz_loadu_ps
   // CHECK: @llvm.masked.load.v8f32.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_maskz_loadu_ps(__U, __P); 
 }
 
 void test_mm_mask_store_pd(void *__P, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_store_pd
+  // CHECK-LABEL: test_mm_mask_store_pd
   // CHECK: @llvm.masked.store.v2f64.p0(<2 x double> %{{.*}}, ptr %{{.*}}, i32 16, <2 x i1> %{{.*}})
   return _mm_mask_store_pd(__P, __U, __A); 
 }
 
 void test_mm256_mask_store_pd(void *__P, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_store_pd
+  // CHECK-LABEL: test_mm256_mask_store_pd
   // CHECK: @llvm.masked.store.v4f64.p0(<4 x double> %{{.*}}, ptr %{{.*}}, i32 32, <4 x i1> %{{.*}})
   return _mm256_mask_store_pd(__P, __U, __A); 
 }
 
 void test_mm_mask_store_ps(void *__P, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_store_ps
+  // CHECK-LABEL: test_mm_mask_store_ps
   // CHECK: @llvm.masked.store.v4f32.p0(<4 x float> %{{.*}}, ptr %{{.*}}, i32 16, <4 x i1> %{{.*}})
   return _mm_mask_store_ps(__P, __U, __A); 
 }
 
 void test_mm256_mask_store_ps(void *__P, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_store_ps
+  // CHECK-LABEL: test_mm256_mask_store_ps
   // CHECK: @llvm.masked.store.v8f32.p0(<8 x float> %{{.*}}, ptr %{{.*}}, i32 32, <8 x i1> %{{.*}})
   return _mm256_mask_store_ps(__P, __U, __A); 
 }
 
 void test_mm_storeu_epi64(void *__p, __m128i __a) {
-  // check-label: @test_mm_storeu_epi64
+  // CHECK-LABEL: test_mm_storeu_epi64
   // check: store <2 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm_storeu_epi64(__p, __a);
 }
 
 void test_mm_mask_storeu_epi64(void *__P, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_storeu_epi64
+  // CHECK-LABEL: test_mm_mask_storeu_epi64
   // CHECK: @llvm.masked.store.v2i64.p0(<2 x i64> %{{.*}}, ptr %{{.*}}, i32 1, <2 x i1> %{{.*}})
   return _mm_mask_storeu_epi64(__P, __U, __A); 
 }
 
 void test_mm256_storeu_epi64(void *__P, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_storeu_epi64
+  // CHECK-LABEL: test_mm256_storeu_epi64
   // CHECK: store <4 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm256_storeu_epi64(__P, __A);
 }
 
 void test_mm256_mask_storeu_epi64(void *__P, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_storeu_epi64
+  // CHECK-LABEL: test_mm256_mask_storeu_epi64
   // CHECK: @llvm.masked.store.v4i64.p0(<4 x i64> %{{.*}}, ptr %{{.*}}, i32 1, <4 x i1> %{{.*}})
   return _mm256_mask_storeu_epi64(__P, __U, __A); 
 }
 
 void test_mm_storeu_epi32(void *__P, __m128i __A) {
-  // CHECK-LABEL: @test_mm_storeu_epi32
+  // CHECK-LABEL: test_mm_storeu_epi32
   // CHECK: store <2 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm_storeu_epi32(__P, __A);
 }
 
 void test_mm_mask_storeu_epi32(void *__P, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_storeu_epi32
+  // CHECK-LABEL: test_mm_mask_storeu_epi32
   // CHECK: @llvm.masked.store.v4i32.p0(<4 x i32> %{{.*}}, ptr %{{.*}}, i32 1, <4 x i1> %{{.*}})
   return _mm_mask_storeu_epi32(__P, __U, __A); 
 }
 
 void test_mm256_storeu_epi32(void *__P, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_storeu_epi32
+  // CHECK-LABEL: test_mm256_storeu_epi32
   // CHECK: store <4 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm256_storeu_epi32(__P, __A);
 }
 
 void test_mm256_mask_storeu_epi32(void *__P, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_storeu_epi32
+  // CHECK-LABEL: test_mm256_mask_storeu_epi32
   // CHECK: @llvm.masked.store.v8i32.p0(<8 x i32> %{{.*}}, ptr %{{.*}}, i32 1, <8 x i1> %{{.*}})
   return _mm256_mask_storeu_epi32(__P, __U, __A); 
 }
 
 void test_mm_mask_storeu_pd(void *__P, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_storeu_pd
+  // CHECK-LABEL: test_mm_mask_storeu_pd
   // CHECK: @llvm.masked.store.v2f64.p0(<2 x double> %{{.*}}, ptr %{{.*}}, i32 1, <2 x i1> %{{.*}})
   return _mm_mask_storeu_pd(__P, __U, __A); 
 }
 
 void test_mm256_mask_storeu_pd(void *__P, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_storeu_pd
+  // CHECK-LABEL: test_mm256_mask_storeu_pd
   // CHECK: @llvm.masked.store.v4f64.p0(<4 x double> %{{.*}}, ptr %{{.*}}, i32 1, <4 x i1> %{{.*}})
   return _mm256_mask_storeu_pd(__P, __U, __A); 
 }
 
 void test_mm_mask_storeu_ps(void *__P, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_storeu_ps
+  // CHECK-LABEL: test_mm_mask_storeu_ps
   // CHECK: @llvm.masked.store.v4f32.p0(<4 x float> %{{.*}}, ptr %{{.*}}, i32 1, <4 x i1> %{{.*}})
   return _mm_mask_storeu_ps(__P, __U, __A); 
 }
 
 void test_mm256_mask_storeu_ps(void *__P, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_storeu_ps
+  // CHECK-LABEL: test_mm256_mask_storeu_ps
   // CHECK: @llvm.masked.store.v8f32.p0(<8 x float> %{{.*}}, ptr %{{.*}}, i32 1, <8 x i1> %{{.*}})
   return _mm256_mask_storeu_ps(__P, __U, __A); 
 }
 
 __m128d test_mm_mask_unpackhi_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_unpackhi_pd
+  // CHECK-LABEL: test_mm_mask_unpackhi_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_unpackhi_pd(__W, __U, __A, __B); 
 }
 
 __m128d test_mm_maskz_unpackhi_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpackhi_pd
+  // CHECK-LABEL: test_mm_maskz_unpackhi_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_unpackhi_pd(__U, __A, __B); 
 }
 
 __m256d test_mm256_mask_unpackhi_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpackhi_pd
+  // CHECK-LABEL: test_mm256_mask_unpackhi_pd
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}} <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_unpackhi_pd(__W, __U, __A, __B); 
 }
 
 __m256d test_mm256_maskz_unpackhi_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpackhi_pd
+  // CHECK-LABEL: test_mm256_maskz_unpackhi_pd
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}} <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_unpackhi_pd(__U, __A, __B); 
 }
 
 __m128 test_mm_mask_unpackhi_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_unpackhi_ps
+  // CHECK-LABEL: test_mm_mask_unpackhi_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}} <4 x float> %{{.*}}
   return _mm_mask_unpackhi_ps(__W, __U, __A, __B); 
 }
 
 __m128 test_mm_maskz_unpackhi_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpackhi_ps
+  // CHECK-LABEL: test_mm_maskz_unpackhi_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}} <4 x float> %{{.*}}
   return _mm_maskz_unpackhi_ps(__U, __A, __B); 
 }
 
 __m256 test_mm256_mask_unpackhi_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpackhi_ps
+  // CHECK-LABEL: test_mm256_mask_unpackhi_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_unpackhi_ps(__W, __U, __A, __B); 
 }
 
 __m256 test_mm256_maskz_unpackhi_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpackhi_ps
+  // CHECK-LABEL: test_mm256_maskz_unpackhi_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_unpackhi_ps(__U, __A, __B); 
 }
 
 __m128d test_mm_mask_unpacklo_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_unpacklo_pd
+  // CHECK-LABEL: test_mm_mask_unpacklo_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 0, i32 2>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_unpacklo_pd(__W, __U, __A, __B); 
 }
 
 __m128d test_mm_maskz_unpacklo_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpacklo_pd
+  // CHECK-LABEL: test_mm_maskz_unpacklo_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 0, i32 2>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_unpacklo_pd(__U, __A, __B); 
 }
 
 __m256d test_mm256_mask_unpacklo_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpacklo_pd
+  // CHECK-LABEL: test_mm256_mask_unpacklo_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   // CHECK: select <4 x i1> %{{.*}} <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_unpacklo_pd(__W, __U, __A, __B); 
 }
 
 __m256d test_mm256_maskz_unpacklo_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpacklo_pd
+  // CHECK-LABEL: test_mm256_maskz_unpacklo_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   // CHECK: select <4 x i1> %{{.*}} <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_unpacklo_pd(__U, __A, __B); 
 }
 
 __m128 test_mm_mask_unpacklo_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_unpacklo_ps
+  // CHECK-LABEL: test_mm_mask_unpacklo_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   // CHECK: select <4 x i1> %{{.*}} <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_unpacklo_ps(__W, __U, __A, __B); 
 }
 
 __m128 test_mm_maskz_unpacklo_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpacklo_ps
+  // CHECK-LABEL: test_mm_maskz_unpacklo_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   // CHECK: select <4 x i1> %{{.*}} <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_unpacklo_ps(__U, __A, __B); 
 }
 
 __m256 test_mm256_mask_unpacklo_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpacklo_ps
+  // CHECK-LABEL: test_mm256_mask_unpacklo_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_unpacklo_ps(__W, __U, __A, __B); 
 }
 
 __m256 test_mm256_maskz_unpacklo_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpacklo_ps
+  // CHECK-LABEL: test_mm256_maskz_unpacklo_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_unpacklo_ps(__U, __A, __B); 
 }
 
 __m128d test_mm_rcp14_pd(__m128d __A) {
-  // CHECK-LABEL: @test_mm_rcp14_pd
+  // CHECK-LABEL: test_mm_rcp14_pd
   // CHECK: @llvm.x86.avx512.rcp14.pd.128
   return _mm_rcp14_pd(__A); 
 }
 
 __m128d test_mm_mask_rcp14_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_rcp14_pd
+  // CHECK-LABEL: test_mm_mask_rcp14_pd
   // CHECK: @llvm.x86.avx512.rcp14.pd.128
   return _mm_mask_rcp14_pd(__W, __U, __A); 
 }
 
 __m128d test_mm_maskz_rcp14_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_rcp14_pd
+  // CHECK-LABEL: test_mm_maskz_rcp14_pd
   // CHECK: @llvm.x86.avx512.rcp14.pd.128
   return _mm_maskz_rcp14_pd(__U, __A); 
 }
 
 __m256d test_mm256_rcp14_pd(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_rcp14_pd
+  // CHECK-LABEL: test_mm256_rcp14_pd
   // CHECK: @llvm.x86.avx512.rcp14.pd.256
   return _mm256_rcp14_pd(__A); 
 }
 
 __m256d test_mm256_mask_rcp14_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_rcp14_pd
+  // CHECK-LABEL: test_mm256_mask_rcp14_pd
   // CHECK: @llvm.x86.avx512.rcp14.pd.256
   return _mm256_mask_rcp14_pd(__W, __U, __A); 
 }
 
 __m256d test_mm256_maskz_rcp14_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_rcp14_pd
+  // CHECK-LABEL: test_mm256_maskz_rcp14_pd
   // CHECK: @llvm.x86.avx512.rcp14.pd.256
   return _mm256_maskz_rcp14_pd(__U, __A); 
 }
 
 __m128 test_mm_rcp14_ps(__m128 __A) {
-  // CHECK-LABEL: @test_mm_rcp14_ps
+  // CHECK-LABEL: test_mm_rcp14_ps
   // CHECK: @llvm.x86.avx512.rcp14.ps.128
   return _mm_rcp14_ps(__A); 
 }
 
 __m128 test_mm_mask_rcp14_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_rcp14_ps
+  // CHECK-LABEL: test_mm_mask_rcp14_ps
   // CHECK: @llvm.x86.avx512.rcp14.ps.128
   return _mm_mask_rcp14_ps(__W, __U, __A); 
 }
 
 __m128 test_mm_maskz_rcp14_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_rcp14_ps
+  // CHECK-LABEL: test_mm_maskz_rcp14_ps
   // CHECK: @llvm.x86.avx512.rcp14.ps.128
   return _mm_maskz_rcp14_ps(__U, __A); 
 }
 
 __m256 test_mm256_rcp14_ps(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_rcp14_ps
+  // CHECK-LABEL: test_mm256_rcp14_ps
   // CHECK: @llvm.x86.avx512.rcp14.ps.256
   return _mm256_rcp14_ps(__A); 
 }
 
 __m256 test_mm256_mask_rcp14_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_rcp14_ps
+  // CHECK-LABEL: test_mm256_mask_rcp14_ps
   // CHECK: @llvm.x86.avx512.rcp14.ps.256
   return _mm256_mask_rcp14_ps(__W, __U, __A); 
 }
 
 __m256 test_mm256_maskz_rcp14_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_rcp14_ps
+  // CHECK-LABEL: test_mm256_maskz_rcp14_ps
   // CHECK: @llvm.x86.avx512.rcp14.ps.256
   return _mm256_maskz_rcp14_ps(__U, __A); 
 }
 
 __m128d test_mm_mask_permute_pd(__m128d __W, __mmask8 __U, __m128d __X) {
-  // CHECK-LABEL: @test_mm_mask_permute_pd
+  // CHECK-LABEL: test_mm_mask_permute_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> <i32 1, i32 0>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_permute_pd(__W, __U, __X, 1); 
 }
 
 __m128d test_mm_maskz_permute_pd(__mmask8 __U, __m128d __X) {
-  // CHECK-LABEL: @test_mm_maskz_permute_pd
+  // CHECK-LABEL: test_mm_maskz_permute_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> poison, <2 x i32> <i32 1, i32 0>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_permute_pd(__U, __X, 1); 
 }
 
 __m256d test_mm256_mask_permute_pd(__m256d __W, __mmask8 __U, __m256d __X) {
-  // CHECK-LABEL: @test_mm256_mask_permute_pd
+  // CHECK-LABEL: test_mm256_mask_permute_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_permute_pd(__W, __U, __X, 5); 
 }
 
 __m256d test_mm256_maskz_permute_pd(__mmask8 __U, __m256d __X) {
-  // CHECK-LABEL: @test_mm256_maskz_permute_pd
+  // CHECK-LABEL: test_mm256_maskz_permute_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_permute_pd(__U, __X, 5); 
 }
 
 __m128 test_mm_mask_permute_ps(__m128 __W, __mmask8 __U, __m128 __X) {
-  // CHECK-LABEL: @test_mm_mask_permute_ps
+  // CHECK-LABEL: test_mm_mask_permute_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_permute_ps(__W, __U, __X, 0x1b); 
 }
 
 __m128 test_mm_maskz_permute_ps(__mmask8 __U, __m128 __X) {
-  // CHECK-LABEL: @test_mm_maskz_permute_ps
+  // CHECK-LABEL: test_mm_maskz_permute_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_permute_ps(__U, __X, 0x1b); 
 }
 
 __m256 test_mm256_mask_permute_ps(__m256 __W, __mmask8 __U, __m256 __X) {
-  // CHECK-LABEL: @test_mm256_mask_permute_ps
+  // CHECK-LABEL: test_mm256_mask_permute_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> poison, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_permute_ps(__W, __U, __X, 0x1b); 
 }
 
 __m256 test_mm256_maskz_permute_ps(__mmask8 __U, __m256 __X) {
-  // CHECK-LABEL: @test_mm256_maskz_permute_ps
+  // CHECK-LABEL: test_mm256_maskz_permute_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> poison, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_permute_ps(__U, __X, 0x1b); 
 }
 
 __m128d test_mm_mask_permutevar_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128i __C) {
-  // CHECK-LABEL: @test_mm_mask_permutevar_pd
+  // CHECK-LABEL: test_mm_mask_permutevar_pd
   // CHECK: @llvm.x86.avx.vpermilvar.pd
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_permutevar_pd(__W, __U, __A, __C); 
 }
 
 __m128d test_mm_maskz_permutevar_pd(__mmask8 __U, __m128d __A, __m128i __C) {
-  // CHECK-LABEL: @test_mm_maskz_permutevar_pd
+  // CHECK-LABEL: test_mm_maskz_permutevar_pd
   // CHECK: @llvm.x86.avx.vpermilvar.pd
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_permutevar_pd(__U, __A, __C); 
 }
 
 __m256d test_mm256_mask_permutevar_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_mask_permutevar_pd
+  // CHECK-LABEL: test_mm256_mask_permutevar_pd
   // CHECK: @llvm.x86.avx.vpermilvar.pd.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_permutevar_pd(__W, __U, __A, __C); 
 }
 
 __m256d test_mm256_maskz_permutevar_pd(__mmask8 __U, __m256d __A, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_maskz_permutevar_pd
+  // CHECK-LABEL: test_mm256_maskz_permutevar_pd
   // CHECK: @llvm.x86.avx.vpermilvar.pd.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_permutevar_pd(__U, __A, __C); 
 }
 
 __m128 test_mm_mask_permutevar_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128i __C) {
-  // CHECK-LABEL: @test_mm_mask_permutevar_ps
+  // CHECK-LABEL: test_mm_mask_permutevar_ps
   // CHECK: @llvm.x86.avx.vpermilvar.ps
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_permutevar_ps(__W, __U, __A, __C); 
 }
 
 __m128 test_mm_maskz_permutevar_ps(__mmask8 __U, __m128 __A, __m128i __C) {
-  // CHECK-LABEL: @test_mm_maskz_permutevar_ps
+  // CHECK-LABEL: test_mm_maskz_permutevar_ps
   // CHECK: @llvm.x86.avx.vpermilvar.ps
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_permutevar_ps(__U, __A, __C); 
 }
 
 __m256 test_mm256_mask_permutevar_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_mask_permutevar_ps
+  // CHECK-LABEL: test_mm256_mask_permutevar_ps
   // CHECK: @llvm.x86.avx.vpermilvar.ps.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_permutevar_ps(__W, __U, __A, __C); 
 }
 
 __m256 test_mm256_maskz_permutevar_ps(__mmask8 __U, __m256 __A, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_maskz_permutevar_ps
+  // CHECK-LABEL: test_mm256_maskz_permutevar_ps
   // CHECK: @llvm.x86.avx.vpermilvar.ps.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_permutevar_ps(__U, __A, __C); 
 }
 
 __mmask8 test_mm_test_epi32_mask(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_test_epi32_mask
+  // CHECK-LABEL: test_mm_test_epi32_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <4 x i32> %{{.*}}, %{{.*}}
   return _mm_test_epi32_mask(__A, __B); 
 }
 
 __mmask8 test_mm_mask_test_epi32_mask(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_test_epi32_mask
+  // CHECK-LABEL: test_mm_mask_test_epi32_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
@@ -7404,14 +7406,14 @@ __mmask8 test_mm_mask_test_epi32_mask(__mmask8 __U, __m128i __A, __m128i __B) {
 }
 
 __mmask8 test_mm256_test_epi32_mask(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_test_epi32_mask
+  // CHECK-LABEL: test_mm256_test_epi32_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <8 x i32> %{{.*}}, %{{.*}}
   return _mm256_test_epi32_mask(__A, __B); 
 }
 
 __mmask8 test_mm256_mask_test_epi32_mask(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_test_epi32_mask
+  // CHECK-LABEL: test_mm256_mask_test_epi32_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
@@ -7419,14 +7421,14 @@ __mmask8 test_mm256_mask_test_epi32_mask(__mmask8 __U, __m256i __A, __m256i __B)
 }
 
 __mmask8 test_mm_test_epi64_mask(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_test_epi64_mask
+  // CHECK-LABEL: test_mm_test_epi64_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <2 x i64> %{{.*}}, %{{.*}}
   return _mm_test_epi64_mask(__A, __B); 
 }
 
 __mmask8 test_mm_mask_test_epi64_mask(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_test_epi64_mask
+  // CHECK-LABEL: test_mm_mask_test_epi64_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
@@ -7434,14 +7436,14 @@ __mmask8 test_mm_mask_test_epi64_mask(__mmask8 __U, __m128i __A, __m128i __B) {
 }
 
 __mmask8 test_mm256_test_epi64_mask(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_test_epi64_mask
+  // CHECK-LABEL: test_mm256_test_epi64_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <4 x i64> %{{.*}}, %{{.*}}
   return _mm256_test_epi64_mask(__A, __B); 
 }
 
 __mmask8 test_mm256_mask_test_epi64_mask(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_test_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_test_epi64_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
@@ -7449,14 +7451,14 @@ __mmask8 test_mm256_mask_test_epi64_mask(__mmask8 __U, __m256i __A, __m256i __B)
 }
 
 __mmask8 test_mm_testn_epi32_mask(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_testn_epi32_mask
+  // CHECK-LABEL: test_mm_testn_epi32_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   return _mm_testn_epi32_mask(__A, __B); 
 }
 
 __mmask8 test_mm_mask_testn_epi32_mask(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_testn_epi32_mask
+  // CHECK-LABEL: test_mm_mask_testn_epi32_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
@@ -7464,14 +7466,14 @@ __mmask8 test_mm_mask_testn_epi32_mask(__mmask8 __U, __m128i __A, __m128i __B) {
 }
 
 __mmask8 test_mm256_testn_epi32_mask(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_testn_epi32_mask
+  // CHECK-LABEL: test_mm256_testn_epi32_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <8 x i32> %{{.*}}, %{{.*}}
   return _mm256_testn_epi32_mask(__A, __B); 
 }
 
 __mmask8 test_mm256_mask_testn_epi32_mask(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_testn_epi32_mask
+  // CHECK-LABEL: test_mm256_mask_testn_epi32_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
@@ -7479,14 +7481,14 @@ __mmask8 test_mm256_mask_testn_epi32_mask(__mmask8 __U, __m256i __A, __m256i __B
 }
 
 __mmask8 test_mm_testn_epi64_mask(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_testn_epi64_mask
+  // CHECK-LABEL: test_mm_testn_epi64_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <2 x i64> %{{.*}}, %{{.*}}
   return _mm_testn_epi64_mask(__A, __B); 
 }
 
 __mmask8 test_mm_mask_testn_epi64_mask(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_testn_epi64_mask
+  // CHECK-LABEL: test_mm_mask_testn_epi64_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
@@ -7494,14 +7496,14 @@ __mmask8 test_mm_mask_testn_epi64_mask(__mmask8 __U, __m128i __A, __m128i __B) {
 }
 
 __mmask8 test_mm256_testn_epi64_mask(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_testn_epi64_mask
+  // CHECK-LABEL: test_mm256_testn_epi64_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
   return _mm256_testn_epi64_mask(__A, __B); 
 }
 
 __mmask8 test_mm256_mask_testn_epi64_mask(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_testn_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_testn_epi64_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
@@ -7509,428 +7511,428 @@ __mmask8 test_mm256_mask_testn_epi64_mask(__mmask8 __U, __m256i __A, __m256i __B
 }
 
 __m128i test_mm_mask_unpackhi_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_unpackhi_epi32
+  // CHECK-LABEL: test_mm_mask_unpackhi_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_unpackhi_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_unpackhi_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpackhi_epi32
+  // CHECK-LABEL: test_mm_maskz_unpackhi_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_unpackhi_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_unpackhi_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpackhi_epi32
+  // CHECK-LABEL: test_mm256_mask_unpackhi_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_unpackhi_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_unpackhi_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpackhi_epi32
+  // CHECK-LABEL: test_mm256_maskz_unpackhi_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_unpackhi_epi32(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_unpackhi_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_unpackhi_epi64
+  // CHECK-LABEL: test_mm_mask_unpackhi_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> <i32 1, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_unpackhi_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_unpackhi_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpackhi_epi64
+  // CHECK-LABEL: test_mm_maskz_unpackhi_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> <i32 1, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_unpackhi_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_unpackhi_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpackhi_epi64
+  // CHECK-LABEL: test_mm256_mask_unpackhi_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_unpackhi_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_unpackhi_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpackhi_epi64
+  // CHECK-LABEL: test_mm256_maskz_unpackhi_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_unpackhi_epi64(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_unpacklo_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_unpacklo_epi32
+  // CHECK-LABEL: test_mm_mask_unpacklo_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_unpacklo_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_unpacklo_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpacklo_epi32
+  // CHECK-LABEL: test_mm_maskz_unpacklo_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_unpacklo_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_unpacklo_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpacklo_epi32
+  // CHECK-LABEL: test_mm256_mask_unpacklo_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_unpacklo_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_unpacklo_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpacklo_epi32
+  // CHECK-LABEL: test_mm256_maskz_unpacklo_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_unpacklo_epi32(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_unpacklo_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_unpacklo_epi64
+  // CHECK-LABEL: test_mm_mask_unpacklo_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> <i32 0, i32 2>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_unpacklo_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_unpacklo_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpacklo_epi64
+  // CHECK-LABEL: test_mm_maskz_unpacklo_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> <i32 0, i32 2>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_unpacklo_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_unpacklo_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpacklo_epi64
+  // CHECK-LABEL: test_mm256_mask_unpacklo_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_unpacklo_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_unpacklo_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpacklo_epi64
+  // CHECK-LABEL: test_mm256_maskz_unpacklo_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_unpacklo_epi64(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_sra_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_sra_epi32
+  // CHECK-LABEL: test_mm_mask_sra_epi32
   // CHECK: @llvm.x86.sse2.psra.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_sra_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_sra_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_sra_epi32
+  // CHECK-LABEL: test_mm_maskz_sra_epi32
   // CHECK: @llvm.x86.sse2.psra.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_sra_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_sra_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_sra_epi32
+  // CHECK-LABEL: test_mm256_mask_sra_epi32
   // CHECK: @llvm.x86.avx2.psra.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_sra_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_sra_epi32(__mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sra_epi32
+  // CHECK-LABEL: test_mm256_maskz_sra_epi32
   // CHECK: @llvm.x86.avx2.psra.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_sra_epi32(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_srai_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_srai_epi32
+  // CHECK-LABEL: test_mm_mask_srai_epi32
   // CHECK: @llvm.x86.sse2.psrai.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_srai_epi32(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_mask_srai_epi32_2(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_mask_srai_epi32_2
+  // CHECK-LABEL: test_mm_mask_srai_epi32_2
   // CHECK: @llvm.x86.sse2.psrai.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_srai_epi32(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srai_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_srai_epi32
+  // CHECK-LABEL: test_mm_maskz_srai_epi32
   // CHECK: @llvm.x86.sse2.psrai.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_srai_epi32(__U, __A, 5); 
 }
 
 __m128i test_mm_maskz_srai_epi32_2(__mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_maskz_srai_epi32_2
+  // CHECK-LABEL: test_mm_maskz_srai_epi32_2
   // CHECK: @llvm.x86.sse2.psrai.d
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_srai_epi32(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_srai_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_srai_epi32
+  // CHECK-LABEL: test_mm256_mask_srai_epi32
   // CHECK: @llvm.x86.avx2.psrai.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_srai_epi32(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_mask_srai_epi32_2(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_mask_srai_epi32_2
+  // CHECK-LABEL: test_mm256_mask_srai_epi32_2
   // CHECK: @llvm.x86.avx2.psrai.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_srai_epi32(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srai_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_srai_epi32
+  // CHECK-LABEL: test_mm256_maskz_srai_epi32
   // CHECK: @llvm.x86.avx2.psrai.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_srai_epi32(__U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_srai_epi32_2(__mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srai_epi32_2
+  // CHECK-LABEL: test_mm256_maskz_srai_epi32_2
   // CHECK: @llvm.x86.avx2.psrai.d
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_srai_epi32(__U, __A, __B); 
 }
 
 __m128i test_mm_sra_epi64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_sra_epi64
+  // CHECK-LABEL: test_mm_sra_epi64
   // CHECK: @llvm.x86.avx512.psra.q.128
   return _mm_sra_epi64(__A, __B); 
 }
 
 __m128i test_mm_mask_sra_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_sra_epi64
+  // CHECK-LABEL: test_mm_mask_sra_epi64
   // CHECK: @llvm.x86.avx512.psra.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_sra_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_sra_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_sra_epi64
+  // CHECK-LABEL: test_mm_maskz_sra_epi64
   // CHECK: @llvm.x86.avx512.psra.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_sra_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_sra_epi64(__m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_sra_epi64
+  // CHECK-LABEL: test_mm256_sra_epi64
   // CHECK: @llvm.x86.avx512.psra.q.256
   return _mm256_sra_epi64(__A, __B); 
 }
 
 __m256i test_mm256_mask_sra_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_sra_epi64
+  // CHECK-LABEL: test_mm256_mask_sra_epi64
   // CHECK: @llvm.x86.avx512.psra.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_sra_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_sra_epi64(__mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sra_epi64
+  // CHECK-LABEL: test_mm256_maskz_sra_epi64
   // CHECK: @llvm.x86.avx512.psra.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_sra_epi64(__U, __A, __B); 
 }
 
 __m128i test_mm_srai_epi64(__m128i __A) {
-  // CHECK-LABEL: @test_mm_srai_epi64
+  // CHECK-LABEL: test_mm_srai_epi64
   // CHECK: @llvm.x86.avx512.psrai.q.128
   return _mm_srai_epi64(__A, 5); 
 }
 
 __m128i test_mm_srai_epi64_2(__m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_srai_epi64_2
+  // CHECK-LABEL: test_mm_srai_epi64_2
   // CHECK: @llvm.x86.avx512.psrai.q.128
   return _mm_srai_epi64(__A, __B); 
 }
 
 __m128i test_mm_mask_srai_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_srai_epi64
+  // CHECK-LABEL: test_mm_mask_srai_epi64
   // CHECK: @llvm.x86.avx512.psrai.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_srai_epi64(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_mask_srai_epi64_2(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_mask_srai_epi64_2
+  // CHECK-LABEL: test_mm_mask_srai_epi64_2
   // CHECK: @llvm.x86.avx512.psrai.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_srai_epi64(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srai_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_srai_epi64
+  // CHECK-LABEL: test_mm_maskz_srai_epi64
   // CHECK: @llvm.x86.avx512.psrai.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_srai_epi64(__U, __A, 5); 
 }
 
 __m128i test_mm_maskz_srai_epi64_2(__mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_maskz_srai_epi64_2
+  // CHECK-LABEL: test_mm_maskz_srai_epi64_2
   // CHECK: @llvm.x86.avx512.psrai.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_srai_epi64(__U, __A, __B); 
 }
 
 __m256i test_mm256_srai_epi64(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_srai_epi64
+  // CHECK-LABEL: test_mm256_srai_epi64
   // CHECK: @llvm.x86.avx512.psrai.q.256
   return _mm256_srai_epi64(__A, 5); 
 }
 
 __m256i test_mm256_srai_epi64_2(__m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_srai_epi64_2
+  // CHECK-LABEL: test_mm256_srai_epi64_2
   // CHECK: @llvm.x86.avx512.psrai.q.256
   return _mm256_srai_epi64(__A, __B); 
 }
 
 __m256i test_mm256_mask_srai_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_srai_epi64
+  // CHECK-LABEL: test_mm256_mask_srai_epi64
   // CHECK: @llvm.x86.avx512.psrai.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_srai_epi64(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_mask_srai_epi64_2(__m256i __W, __mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_mask_srai_epi64_2
+  // CHECK-LABEL: test_mm256_mask_srai_epi64_2
   // CHECK: @llvm.x86.avx512.psrai.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_srai_epi64(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srai_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_srai_epi64
+  // CHECK-LABEL: test_mm256_maskz_srai_epi64
   // CHECK: @llvm.x86.avx512.psrai.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_srai_epi64(__U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_srai_epi64_2(__mmask8 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srai_epi64_2
+  // CHECK-LABEL: test_mm256_maskz_srai_epi64_2
   // CHECK: @llvm.x86.avx512.psrai.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_srai_epi64(__U, __A, __B); 
 }
 
 __m128i test_mm_ternarylogic_epi32(__m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_ternarylogic_epi32
+  // CHECK-LABEL: test_mm_ternarylogic_epi32
   // CHECK: @llvm.x86.avx512.pternlog.d.128
   return _mm_ternarylogic_epi32(__A, __B, __C, 4); 
 }
 
 __m128i test_mm_mask_ternarylogic_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_mask_ternarylogic_epi32
+  // CHECK-LABEL: test_mm_mask_ternarylogic_epi32
   // CHECK: @llvm.x86.avx512.pternlog.d.128
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_ternarylogic_epi32(__A, __U, __B, __C, 4); 
 }
 
 __m128i test_mm_maskz_ternarylogic_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_maskz_ternarylogic_epi32
+  // CHECK-LABEL: test_mm_maskz_ternarylogic_epi32
   // CHECK: @llvm.x86.avx512.pternlog.d.128
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> zeroinitializer
   return _mm_maskz_ternarylogic_epi32(__U, __A, __B, __C, 4); 
 }
 
 __m256i test_mm256_ternarylogic_epi32(__m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_ternarylogic_epi32
+  // CHECK-LABEL: test_mm256_ternarylogic_epi32
   // CHECK: @llvm.x86.avx512.pternlog.d.256
   return _mm256_ternarylogic_epi32(__A, __B, __C, 4); 
 }
 
 __m256i test_mm256_mask_ternarylogic_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_mask_ternarylogic_epi32
+  // CHECK-LABEL: test_mm256_mask_ternarylogic_epi32
   // CHECK: @llvm.x86.avx512.pternlog.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_ternarylogic_epi32(__A, __U, __B, __C, 4); 
 }
 
 __m256i test_mm256_maskz_ternarylogic_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_maskz_ternarylogic_epi32
+  // CHECK-LABEL: test_mm256_maskz_ternarylogic_epi32
   // CHECK: @llvm.x86.avx512.pternlog.d.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> zeroinitializer
   return _mm256_maskz_ternarylogic_epi32(__U, __A, __B, __C, 4); 
 }
 
 __m128i test_mm_ternarylogic_epi64(__m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_ternarylogic_epi64
+  // CHECK-LABEL: test_mm_ternarylogic_epi64
   // CHECK: @llvm.x86.avx512.pternlog.q.128
   return _mm_ternarylogic_epi64(__A, __B, __C, 4); 
 }
 
 __m128i test_mm_mask_ternarylogic_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_mask_ternarylogic_epi64
+  // CHECK-LABEL: test_mm_mask_ternarylogic_epi64
   // CHECK: @llvm.x86.avx512.pternlog.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_ternarylogic_epi64(__A, __U, __B, __C, 4); 
 }
 
 __m128i test_mm_maskz_ternarylogic_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_maskz_ternarylogic_epi64
+  // CHECK-LABEL: test_mm_maskz_ternarylogic_epi64
   // CHECK: @llvm.x86.avx512.pternlog.q.128
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> zeroinitializer
   return _mm_maskz_ternarylogic_epi64(__U, __A, __B, __C, 4); 
 }
 
 __m256i test_mm256_ternarylogic_epi64(__m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_ternarylogic_epi64
+  // CHECK-LABEL: test_mm256_ternarylogic_epi64
   // CHECK: @llvm.x86.avx512.pternlog.q.256
   return _mm256_ternarylogic_epi64(__A, __B, __C, 4); 
 }
 
 __m256i test_mm256_mask_ternarylogic_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_mask_ternarylogic_epi64
+  // CHECK-LABEL: test_mm256_mask_ternarylogic_epi64
   // CHECK: @llvm.x86.avx512.pternlog.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_ternarylogic_epi64(__A, __U, __B, __C, 4); 
 }
 
 __m256i test_mm256_maskz_ternarylogic_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_maskz_ternarylogic_epi64
+  // CHECK-LABEL: test_mm256_maskz_ternarylogic_epi64
   // CHECK: @llvm.x86.avx512.pternlog.q.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> zeroinitializer
   return _mm256_maskz_ternarylogic_epi64(__U, __A, __B, __C, 4); 
 }
 __m256 test_mm256_shuffle_f32x4(__m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_shuffle_f32x4
+  // CHECK-LABEL: test_mm256_shuffle_f32x4
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
   return _mm256_shuffle_f32x4(__A, __B, 3); 
 }
 
 __m256 test_mm256_mask_shuffle_f32x4(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_shuffle_f32x4
+  // CHECK-LABEL: test_mm256_mask_shuffle_f32x4
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_shuffle_f32x4(__W, __U, __A, __B, 3); 
 }
 
 __m256 test_mm256_maskz_shuffle_f32x4(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shuffle_f32x4
+  // CHECK-LABEL: test_mm256_maskz_shuffle_f32x4
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_shuffle_f32x4(__U, __A, __B, 3); 
 }
 
 __m256d test_mm256_shuffle_f64x2(__m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_shuffle_f64x2
+  // CHECK-LABEL: test_mm256_shuffle_f64x2
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
   return _mm256_shuffle_f64x2(__A, __B, 3); 
 }
 
 __m256d test_mm256_mask_shuffle_f64x2(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_shuffle_f64x2
+  // CHECK-LABEL: test_mm256_mask_shuffle_f64x2
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
@@ -7938,7 +7940,7 @@ __m256d test_mm256_mask_shuffle_f64x2(__m256d __W, __mmask8 __U, __m256d __A, __
 }
 
 __m256d test_mm256_maskz_shuffle_f64x2(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shuffle_f64x2
+  // CHECK-LABEL: test_mm256_maskz_shuffle_f64x2
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
@@ -7946,33 +7948,33 @@ __m256d test_mm256_maskz_shuffle_f64x2(__mmask8 __U, __m256d __A, __m256d __B) {
 }
 
 __m256i test_mm256_shuffle_i32x4(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shuffle_i32x4
+  // CHECK-LABEL: test_mm256_shuffle_i32x4
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
   return _mm256_shuffle_i32x4(__A, __B, 3); 
 }
 
 __m256i test_mm256_mask_shuffle_i32x4(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shuffle_i32x4
+  // CHECK-LABEL: test_mm256_mask_shuffle_i32x4
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_shuffle_i32x4(__W, __U, __A, __B, 3); 
 }
 
 __m256i test_mm256_maskz_shuffle_i32x4(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shuffle_i32x4
+  // CHECK-LABEL: test_mm256_maskz_shuffle_i32x4
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_shuffle_i32x4(__U, __A, __B, 3); 
 }
 
 __m256i test_mm256_shuffle_i64x2(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shuffle_i64x2
+  // CHECK-LABEL: test_mm256_shuffle_i64x2
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
   return _mm256_shuffle_i64x2(__A, __B, 3); 
 }
 
 __m256i test_mm256_mask_shuffle_i64x2(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shuffle_i64x2
+  // CHECK-LABEL: test_mm256_mask_shuffle_i64x2
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
@@ -7980,7 +7982,7 @@ __m256i test_mm256_mask_shuffle_i64x2(__m256i __W, __mmask8 __U, __m256i __A, __
 }
 
 __m256i test_mm256_maskz_shuffle_i64x2(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shuffle_i64x2
+  // CHECK-LABEL: test_mm256_maskz_shuffle_i64x2
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
   // CHECK: shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
@@ -7988,1576 +7990,1586 @@ __m256i test_mm256_maskz_shuffle_i64x2(__mmask8 __U, __m256i __A, __m256i __B) {
 }
 
 __m128d test_mm_mask_shuffle_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_shuffle_pd
+  // CHECK-LABEL: test_mm_mask_shuffle_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_shuffle_pd(__W, __U, __A, __B, 3); 
 }
 
 __m128d test_mm_maskz_shuffle_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_shuffle_pd
+  // CHECK-LABEL: test_mm_maskz_shuffle_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x i32> <i32 1, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_shuffle_pd(__U, __A, __B, 3); 
 }
 
 __m256d test_mm256_mask_shuffle_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_shuffle_pd
+  // CHECK-LABEL: test_mm256_mask_shuffle_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 1, i32 5, i32 2, i32 6>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_shuffle_pd(__W, __U, __A, __B, 3); 
 }
 
 __m256d test_mm256_maskz_shuffle_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shuffle_pd
+  // CHECK-LABEL: test_mm256_maskz_shuffle_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 1, i32 5, i32 2, i32 6>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_shuffle_pd(__U, __A, __B, 3); 
 }
 
 __m128 test_mm_mask_shuffle_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_shuffle_ps
+  // CHECK-LABEL: test_mm_mask_shuffle_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 4>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_shuffle_ps(__W, __U, __A, __B, 4); 
 }
 
 __m128 test_mm_maskz_shuffle_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_shuffle_ps
+  // CHECK-LABEL: test_mm_maskz_shuffle_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 4>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_shuffle_ps(__U, __A, __B, 4); 
 }
 
 __m256 test_mm256_mask_shuffle_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_shuffle_ps
+  // CHECK-LABEL: test_mm256_mask_shuffle_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 8, i32 8, i32 4, i32 5, i32 12, i32 12>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_shuffle_ps(__W, __U, __A, __B, 4); 
 }
 
 __m256 test_mm256_maskz_shuffle_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shuffle_ps
+  // CHECK-LABEL: test_mm256_maskz_shuffle_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 8, i32 8, i32 4, i32 5, i32 12, i32 12>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_shuffle_ps(__U, __A, __B, 4); 
 }
 
 __m128d test_mm_rsqrt14_pd(__m128d __A) {
-  // CHECK-LABEL: @test_mm_rsqrt14_pd
+  // CHECK-LABEL: test_mm_rsqrt14_pd
   // CHECK: @llvm.x86.avx512.rsqrt14.pd.128
   return _mm_rsqrt14_pd(__A); 
 }
 
 __m128d test_mm_mask_rsqrt14_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_rsqrt14_pd
+  // CHECK-LABEL: test_mm_mask_rsqrt14_pd
   // CHECK: @llvm.x86.avx512.rsqrt14.pd.128
   return _mm_mask_rsqrt14_pd(__W, __U, __A); 
 }
 
 __m128d test_mm_maskz_rsqrt14_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_rsqrt14_pd
+  // CHECK-LABEL: test_mm_maskz_rsqrt14_pd
   // CHECK: @llvm.x86.avx512.rsqrt14.pd.128
   return _mm_maskz_rsqrt14_pd(__U, __A); 
 }
 
 __m256d test_mm256_rsqrt14_pd(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_rsqrt14_pd
+  // CHECK-LABEL: test_mm256_rsqrt14_pd
   // CHECK: @llvm.x86.avx512.rsqrt14.pd.256
   return _mm256_rsqrt14_pd(__A); 
 }
 
 __m256d test_mm256_mask_rsqrt14_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_rsqrt14_pd
+  // CHECK-LABEL: test_mm256_mask_rsqrt14_pd
   // CHECK: @llvm.x86.avx512.rsqrt14.pd.256
   return _mm256_mask_rsqrt14_pd(__W, __U, __A); 
 }
 
 __m256d test_mm256_maskz_rsqrt14_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_rsqrt14_pd
+  // CHECK-LABEL: test_mm256_maskz_rsqrt14_pd
   // CHECK: @llvm.x86.avx512.rsqrt14.pd.256
   return _mm256_maskz_rsqrt14_pd(__U, __A); 
 }
 
 __m128 test_mm_rsqrt14_ps(__m128 __A) {
-  // CHECK-LABEL: @test_mm_rsqrt14_ps
+  // CHECK-LABEL: test_mm_rsqrt14_ps
   // CHECK: @llvm.x86.avx512.rsqrt14.ps.128
   return _mm_rsqrt14_ps(__A); 
 }
 
 __m128 test_mm_mask_rsqrt14_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_rsqrt14_ps
+  // CHECK-LABEL: test_mm_mask_rsqrt14_ps
   // CHECK: @llvm.x86.avx512.rsqrt14.ps.128
   return _mm_mask_rsqrt14_ps(__W, __U, __A); 
 }
 
 __m128 test_mm_maskz_rsqrt14_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_rsqrt14_ps
+  // CHECK-LABEL: test_mm_maskz_rsqrt14_ps
   // CHECK: @llvm.x86.avx512.rsqrt14.ps.128
   return _mm_maskz_rsqrt14_ps(__U, __A); 
 }
 
 __m256 test_mm256_rsqrt14_ps(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_rsqrt14_ps
+  // CHECK-LABEL: test_mm256_rsqrt14_ps
   // CHECK: @llvm.x86.avx512.rsqrt14.ps.256
   return _mm256_rsqrt14_ps(__A); 
 }
 
 __m256 test_mm256_mask_rsqrt14_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_rsqrt14_ps
+  // CHECK-LABEL: test_mm256_mask_rsqrt14_ps
   // CHECK: @llvm.x86.avx512.rsqrt14.ps.256
   return _mm256_mask_rsqrt14_ps(__W, __U, __A); 
 }
 
 __m256 test_mm256_maskz_rsqrt14_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_rsqrt14_ps
+  // CHECK-LABEL: test_mm256_maskz_rsqrt14_ps
   // CHECK: @llvm.x86.avx512.rsqrt14.ps.256
   return _mm256_maskz_rsqrt14_ps(__U, __A); 
 }
 
 __m256 test_mm256_broadcast_f32x4(__m128 __A) {
-  // CHECK-LABEL: @test_mm256_broadcast_f32x4
+  // CHECK-LABEL: test_mm256_broadcast_f32x4
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   return _mm256_broadcast_f32x4(__A); 
 }
+TEST_CONSTEXPR(match_m256(_mm256_broadcast_f32x4((__m128)(__v4sf){1.0f, 3.0f, -5.0f, -8.0f}), 1.0f, 3.0f, -5.0f, -8.0f, 1.0f, 3.0f, -5.0f, -8.0f));
 
 __m256 test_mm256_mask_broadcast_f32x4(__m256 __O, __mmask8 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcast_f32x4
+  // CHECK-LABEL: test_mm256_mask_broadcast_f32x4
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_broadcast_f32x4(__O, __M, __A); 
 }
 
 __m256 test_mm256_maskz_broadcast_f32x4(__mmask8 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcast_f32x4
+  // CHECK-LABEL: test_mm256_maskz_broadcast_f32x4
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_broadcast_f32x4(__M, __A); 
 }
 
 __m256i test_mm256_broadcast_i32x4(__m128i const* __A) {
-  // CHECK-LABEL: @test_mm256_broadcast_i32x4
+  // CHECK-LABEL: test_mm256_broadcast_i32x4
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   return _mm256_broadcast_i32x4(_mm_loadu_si128(__A)); 
 }
+TEST_CONSTEXPR(match_v8si(_mm256_broadcast_i32x4((__m128i)(__v4si){1, 3, -5, -8}), 1, 3, -5, -8, 1, 3, -5, -8));
 
 __m256i test_mm256_mask_broadcast_i32x4(__m256i __O, __mmask8 __M, __m128i const* __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcast_i32x4
+  // CHECK-LABEL: test_mm256_mask_broadcast_i32x4
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_broadcast_i32x4(__O, __M, _mm_loadu_si128(__A)); 
 }
 
 __m256i test_mm256_maskz_broadcast_i32x4(__mmask8 __M, __m128i const* __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcast_i32x4
+  // CHECK-LABEL: test_mm256_maskz_broadcast_i32x4
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_broadcast_i32x4(__M, _mm_loadu_si128(__A)); 
 }
 
 __m256d test_mm256_mask_broadcastsd_pd(__m256d __O, __mmask8 __M, __m128d __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcastsd_pd
+  // CHECK-LABEL: test_mm256_mask_broadcastsd_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <4 x i32> zeroinitializer
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_broadcastsd_pd(__O, __M, __A);
 }
 
 __m256d test_mm256_maskz_broadcastsd_pd(__mmask8 __M, __m128d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcastsd_pd
+  // CHECK-LABEL: test_mm256_maskz_broadcastsd_pd
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <4 x i32> zeroinitializer
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_broadcastsd_pd(__M, __A);
 }
 
 __m128 test_mm_mask_broadcastss_ps(__m128 __O, __mmask8 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_broadcastss_ps
+  // CHECK-LABEL: test_mm_mask_broadcastss_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> zeroinitializer
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_broadcastss_ps(__O, __M, __A);
 }
 
 __m128 test_mm_maskz_broadcastss_ps(__mmask8 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_broadcastss_ps
+  // CHECK-LABEL: test_mm_maskz_broadcastss_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> zeroinitializer
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_broadcastss_ps(__M, __A);
 }
 
 __m256 test_mm256_mask_broadcastss_ps(__m256 __O, __mmask8 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcastss_ps
+  // CHECK-LABEL: test_mm256_mask_broadcastss_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <8 x i32> zeroinitializer
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_broadcastss_ps(__O, __M, __A);
 }
 
 __m256 test_mm256_maskz_broadcastss_ps(__mmask8 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcastss_ps
+  // CHECK-LABEL: test_mm256_maskz_broadcastss_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <8 x i32> zeroinitializer
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_broadcastss_ps(__M, __A);
 }
 
 __m128i test_mm_mask_broadcastd_epi32(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_broadcastd_epi32
+  // CHECK-LABEL: test_mm_mask_broadcastd_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> zeroinitializer
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_broadcastd_epi32(__O, __M, __A);
 }
 
 __m128i test_mm_maskz_broadcastd_epi32(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_broadcastd_epi32
+  // CHECK-LABEL: test_mm_maskz_broadcastd_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> zeroinitializer
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_broadcastd_epi32(__M, __A);
 }
 
 __m256i test_mm256_mask_broadcastd_epi32(__m256i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcastd_epi32
+  // CHECK-LABEL: test_mm256_mask_broadcastd_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> zeroinitializer
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_broadcastd_epi32(__O, __M, __A);
 }
 
 __m256i test_mm256_maskz_broadcastd_epi32(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcastd_epi32
+  // CHECK-LABEL: test_mm256_maskz_broadcastd_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> zeroinitializer
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_broadcastd_epi32(__M, __A);
 }
 
 __m128i test_mm_mask_broadcastq_epi64(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_broadcastq_epi64
+  // CHECK-LABEL: test_mm_mask_broadcastq_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> zeroinitializer
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_broadcastq_epi64(__O, __M, __A);
 }
 
 __m128i test_mm_maskz_broadcastq_epi64(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_broadcastq_epi64
+  // CHECK-LABEL: test_mm_maskz_broadcastq_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> zeroinitializer
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_broadcastq_epi64(__M, __A);
 }
 
 __m256i test_mm256_mask_broadcastq_epi64(__m256i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcastq_epi64
+  // CHECK-LABEL: test_mm256_mask_broadcastq_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <4 x i32> zeroinitializer
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_broadcastq_epi64(__O, __M, __A);
 }
 
 __m256i test_mm256_maskz_broadcastq_epi64(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcastq_epi64
+  // CHECK-LABEL: test_mm256_maskz_broadcastq_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <4 x i32> zeroinitializer
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_broadcastq_epi64(__M, __A);
 }
 
 __m128i test_mm_cvtsepi32_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtsepi32_epi8
+  // CHECK-LABEL: test_mm_cvtsepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.db.128
   return _mm_cvtsepi32_epi8(__A); 
 }
 
 __m128i test_mm_mask_cvtsepi32_epi8(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi32_epi8
+  // CHECK-LABEL: test_mm_mask_cvtsepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.db.128
   return _mm_mask_cvtsepi32_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtsepi32_epi8(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtsepi32_epi8
+  // CHECK-LABEL: test_mm_maskz_cvtsepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.db.128
   return _mm_maskz_cvtsepi32_epi8(__M, __A); 
 }
 
 void test_mm_mask_cvtsepi32_storeu_epi8(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi32_storeu_epi8
+  // CHECK-LABEL: test_mm_mask_cvtsepi32_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.db.mem.128
   return _mm_mask_cvtsepi32_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtsepi32_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtsepi32_epi8
+  // CHECK-LABEL: test_mm256_cvtsepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.db.256
   return _mm256_cvtsepi32_epi8(__A); 
 }
 
 __m128i test_mm256_mask_cvtsepi32_epi8(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi32_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtsepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.db.256
   return _mm256_mask_cvtsepi32_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtsepi32_epi8(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtsepi32_epi8
+  // CHECK-LABEL: test_mm256_maskz_cvtsepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.db.256
   return _mm256_maskz_cvtsepi32_epi8(__M, __A); 
 }
 
 void test_mm256_mask_cvtsepi32_storeu_epi8(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi32_storeu_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtsepi32_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.db.mem.256
   return _mm256_mask_cvtsepi32_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtsepi32_epi16(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtsepi32_epi16
+  // CHECK-LABEL: test_mm_cvtsepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.dw.128
   return _mm_cvtsepi32_epi16(__A); 
 }
 
 __m128i test_mm_mask_cvtsepi32_epi16(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi32_epi16
+  // CHECK-LABEL: test_mm_mask_cvtsepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.dw.128
   return _mm_mask_cvtsepi32_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtsepi32_epi16(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtsepi32_epi16
+  // CHECK-LABEL: test_mm_maskz_cvtsepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.dw.128
   return _mm_maskz_cvtsepi32_epi16(__M, __A); 
 }
 
 void test_mm_mask_cvtsepi32_storeu_epi16(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi32_storeu_epi16
+  // CHECK-LABEL: test_mm_mask_cvtsepi32_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.dw.mem.128
   return _mm_mask_cvtsepi32_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtsepi32_epi16(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtsepi32_epi16
+  // CHECK-LABEL: test_mm256_cvtsepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.dw.256
   return _mm256_cvtsepi32_epi16(__A); 
 }
 
 __m128i test_mm256_mask_cvtsepi32_epi16(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi32_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtsepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.dw.256
   return _mm256_mask_cvtsepi32_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtsepi32_epi16(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtsepi32_epi16
+  // CHECK-LABEL: test_mm256_maskz_cvtsepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.dw.256
   return _mm256_maskz_cvtsepi32_epi16(__M, __A); 
 }
 
 void test_mm256_mask_cvtsepi32_storeu_epi16(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi32_storeu_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtsepi32_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.dw.mem.256
   return _mm256_mask_cvtsepi32_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtsepi64_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtsepi64_epi8
+  // CHECK-LABEL: test_mm_cvtsepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.qb.128
   return _mm_cvtsepi64_epi8(__A); 
 }
 
 __m128i test_mm_mask_cvtsepi64_epi8(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi64_epi8
+  // CHECK-LABEL: test_mm_mask_cvtsepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.qb.128
   return _mm_mask_cvtsepi64_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtsepi64_epi8(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtsepi64_epi8
+  // CHECK-LABEL: test_mm_maskz_cvtsepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.qb.128
   return _mm_maskz_cvtsepi64_epi8(__M, __A); 
 }
 
 void test_mm_mask_cvtsepi64_storeu_epi8(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi64_storeu_epi8
+  // CHECK-LABEL: test_mm_mask_cvtsepi64_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.qb.mem.128
   return _mm_mask_cvtsepi64_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtsepi64_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtsepi64_epi8
+  // CHECK-LABEL: test_mm256_cvtsepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.qb.256
   return _mm256_cvtsepi64_epi8(__A); 
 }
 
 __m128i test_mm256_mask_cvtsepi64_epi8(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi64_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtsepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.qb.256
   return _mm256_mask_cvtsepi64_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtsepi64_epi8(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtsepi64_epi8
+  // CHECK-LABEL: test_mm256_maskz_cvtsepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.qb.256
   return _mm256_maskz_cvtsepi64_epi8(__M, __A); 
 }
 
 void test_mm256_mask_cvtsepi64_storeu_epi8(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi64_storeu_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtsepi64_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.qb.mem.256
   return _mm256_mask_cvtsepi64_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtsepi64_epi32(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtsepi64_epi32
+  // CHECK-LABEL: test_mm_cvtsepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovs.qd.128
   return _mm_cvtsepi64_epi32(__A); 
 }
 
 __m128i test_mm_mask_cvtsepi64_epi32(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi64_epi32
+  // CHECK-LABEL: test_mm_mask_cvtsepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovs.qd.128
   return _mm_mask_cvtsepi64_epi32(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtsepi64_epi32(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtsepi64_epi32
+  // CHECK-LABEL: test_mm_maskz_cvtsepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovs.qd.128
   return _mm_maskz_cvtsepi64_epi32(__M, __A); 
 }
 
 void test_mm_mask_cvtsepi64_storeu_epi32(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi64_storeu_epi32
+  // CHECK-LABEL: test_mm_mask_cvtsepi64_storeu_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovs.qd.mem.128
   return _mm_mask_cvtsepi64_storeu_epi32(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtsepi64_epi32(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtsepi64_epi32
+  // CHECK-LABEL: test_mm256_cvtsepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovs.qd.256
   return _mm256_cvtsepi64_epi32(__A); 
 }
 
 __m128i test_mm256_mask_cvtsepi64_epi32(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi64_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtsepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovs.qd.256
   return _mm256_mask_cvtsepi64_epi32(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtsepi64_epi32(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtsepi64_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvtsepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovs.qd.256
   return _mm256_maskz_cvtsepi64_epi32(__M, __A); 
 }
 
 void test_mm256_mask_cvtsepi64_storeu_epi32(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi64_storeu_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtsepi64_storeu_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovs.qd.mem.256
   return _mm256_mask_cvtsepi64_storeu_epi32(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtsepi64_epi16(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtsepi64_epi16
+  // CHECK-LABEL: test_mm_cvtsepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.qw.128
   return _mm_cvtsepi64_epi16(__A); 
 }
 
 __m128i test_mm_mask_cvtsepi64_epi16(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi64_epi16
+  // CHECK-LABEL: test_mm_mask_cvtsepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.qw.128
   return _mm_mask_cvtsepi64_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtsepi64_epi16(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtsepi64_epi16
+  // CHECK-LABEL: test_mm_maskz_cvtsepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.qw.128
   return _mm_maskz_cvtsepi64_epi16(__M, __A); 
 }
 
 void test_mm_mask_cvtsepi64_storeu_epi16(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi64_storeu_epi16
+  // CHECK-LABEL: test_mm_mask_cvtsepi64_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.qw.mem.128
   return _mm_mask_cvtsepi64_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtsepi64_epi16(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtsepi64_epi16
+  // CHECK-LABEL: test_mm256_cvtsepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.qw.256
   return _mm256_cvtsepi64_epi16(__A); 
 }
 
 __m128i test_mm256_mask_cvtsepi64_epi16(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi64_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtsepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.qw.256
   return _mm256_mask_cvtsepi64_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtsepi64_epi16(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtsepi64_epi16
+  // CHECK-LABEL: test_mm256_maskz_cvtsepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.qw.256
   return _mm256_maskz_cvtsepi64_epi16(__M, __A); 
 }
 
 void test_mm256_mask_cvtsepi64_storeu_epi16(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi64_storeu_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtsepi64_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovs.qw.mem.256
   return _mm256_mask_cvtsepi64_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtusepi32_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtusepi32_epi8
+  // CHECK-LABEL: test_mm_cvtusepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.db.128
   return _mm_cvtusepi32_epi8(__A); 
 }
 
 __m128i test_mm_mask_cvtusepi32_epi8(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi32_epi8
+  // CHECK-LABEL: test_mm_mask_cvtusepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.db.128
   return _mm_mask_cvtusepi32_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtusepi32_epi8(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtusepi32_epi8
+  // CHECK-LABEL: test_mm_maskz_cvtusepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.db.128
   return _mm_maskz_cvtusepi32_epi8(__M, __A); 
 }
 
 void test_mm_mask_cvtusepi32_storeu_epi8(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi32_storeu_epi8
+  // CHECK-LABEL: test_mm_mask_cvtusepi32_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.db.mem.128
   return _mm_mask_cvtusepi32_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtusepi32_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtusepi32_epi8
+  // CHECK-LABEL: test_mm256_cvtusepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.db.256
   return _mm256_cvtusepi32_epi8(__A); 
 }
 
 __m128i test_mm256_mask_cvtusepi32_epi8(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi32_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtusepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.db.256
   return _mm256_mask_cvtusepi32_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtusepi32_epi8(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtusepi32_epi8
+  // CHECK-LABEL: test_mm256_maskz_cvtusepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.db.256
   return _mm256_maskz_cvtusepi32_epi8(__M, __A); 
 }
 
 void test_mm256_mask_cvtusepi32_storeu_epi8(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi32_storeu_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtusepi32_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.db.mem.256
   return _mm256_mask_cvtusepi32_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtusepi32_epi16(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtusepi32_epi16
+  // CHECK-LABEL: test_mm_cvtusepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.dw.128
   return _mm_cvtusepi32_epi16(__A); 
 }
 
 __m128i test_mm_mask_cvtusepi32_epi16(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi32_epi16
+  // CHECK-LABEL: test_mm_mask_cvtusepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.dw.128
   return _mm_mask_cvtusepi32_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtusepi32_epi16(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtusepi32_epi16
+  // CHECK-LABEL: test_mm_maskz_cvtusepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.dw.128
   return _mm_maskz_cvtusepi32_epi16(__M, __A); 
 }
 
 void test_mm_mask_cvtusepi32_storeu_epi16(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi32_storeu_epi16
+  // CHECK-LABEL: test_mm_mask_cvtusepi32_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.dw.mem.128
   return _mm_mask_cvtusepi32_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtusepi32_epi16(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtusepi32_epi16
+  // CHECK-LABEL: test_mm256_cvtusepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.dw.256
   return _mm256_cvtusepi32_epi16(__A); 
 }
 
 __m128i test_mm256_mask_cvtusepi32_epi16(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi32_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtusepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.dw.256
   return _mm256_mask_cvtusepi32_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtusepi32_epi16(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtusepi32_epi16
+  // CHECK-LABEL: test_mm256_maskz_cvtusepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.dw.256
   return _mm256_maskz_cvtusepi32_epi16(__M, __A); 
 }
 
 void test_mm256_mask_cvtusepi32_storeu_epi16(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi32_storeu_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtusepi32_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.dw.mem.256
   return _mm256_mask_cvtusepi32_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtusepi64_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtusepi64_epi8
+  // CHECK-LABEL: test_mm_cvtusepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.qb.128
   return _mm_cvtusepi64_epi8(__A); 
 }
 
 __m128i test_mm_mask_cvtusepi64_epi8(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi64_epi8
+  // CHECK-LABEL: test_mm_mask_cvtusepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.qb.128
   return _mm_mask_cvtusepi64_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtusepi64_epi8(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtusepi64_epi8
+  // CHECK-LABEL: test_mm_maskz_cvtusepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.qb.128
   return _mm_maskz_cvtusepi64_epi8(__M, __A); 
 }
 
 void test_mm_mask_cvtusepi64_storeu_epi8(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi64_storeu_epi8
+  // CHECK-LABEL: test_mm_mask_cvtusepi64_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.qb.mem.128
   return _mm_mask_cvtusepi64_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtusepi64_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtusepi64_epi8
+  // CHECK-LABEL: test_mm256_cvtusepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.qb.256
   return _mm256_cvtusepi64_epi8(__A); 
 }
 
 __m128i test_mm256_mask_cvtusepi64_epi8(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi64_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtusepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.qb.256
   return _mm256_mask_cvtusepi64_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtusepi64_epi8(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtusepi64_epi8
+  // CHECK-LABEL: test_mm256_maskz_cvtusepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.qb.256
   return _mm256_maskz_cvtusepi64_epi8(__M, __A); 
 }
 
 void test_mm256_mask_cvtusepi64_storeu_epi8(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi64_storeu_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtusepi64_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.qb.mem.256
   return _mm256_mask_cvtusepi64_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtusepi64_epi32(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtusepi64_epi32
+  // CHECK-LABEL: test_mm_cvtusepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovus.qd.128
   return _mm_cvtusepi64_epi32(__A); 
 }
 
 __m128i test_mm_mask_cvtusepi64_epi32(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi64_epi32
+  // CHECK-LABEL: test_mm_mask_cvtusepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovus.qd.128
   return _mm_mask_cvtusepi64_epi32(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtusepi64_epi32(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtusepi64_epi32
+  // CHECK-LABEL: test_mm_maskz_cvtusepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovus.qd.128
   return _mm_maskz_cvtusepi64_epi32(__M, __A); 
 }
 
 void test_mm_mask_cvtusepi64_storeu_epi32(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi64_storeu_epi32
+  // CHECK-LABEL: test_mm_mask_cvtusepi64_storeu_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovus.qd.mem.128
   return _mm_mask_cvtusepi64_storeu_epi32(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtusepi64_epi32(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtusepi64_epi32
+  // CHECK-LABEL: test_mm256_cvtusepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovus.qd.256
   return _mm256_cvtusepi64_epi32(__A); 
 }
 
 __m128i test_mm256_mask_cvtusepi64_epi32(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi64_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtusepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovus.qd.256
   return _mm256_mask_cvtusepi64_epi32(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtusepi64_epi32(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtusepi64_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvtusepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovus.qd.256
   return _mm256_maskz_cvtusepi64_epi32(__M, __A); 
 }
 
 void test_mm256_mask_cvtusepi64_storeu_epi32(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi64_storeu_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtusepi64_storeu_epi32
   // CHECK: @llvm.x86.avx512.mask.pmovus.qd.mem.256
   return _mm256_mask_cvtusepi64_storeu_epi32(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtusepi64_epi16(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtusepi64_epi16
+  // CHECK-LABEL: test_mm_cvtusepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.qw.128
   return _mm_cvtusepi64_epi16(__A); 
 }
 
 __m128i test_mm_mask_cvtusepi64_epi16(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi64_epi16
+  // CHECK-LABEL: test_mm_mask_cvtusepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.qw.128
   return _mm_mask_cvtusepi64_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtusepi64_epi16(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtusepi64_epi16
+  // CHECK-LABEL: test_mm_maskz_cvtusepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.qw.128
   return _mm_maskz_cvtusepi64_epi16(__M, __A); 
 }
 
 void test_mm_mask_cvtusepi64_storeu_epi16(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi64_storeu_epi16
+  // CHECK-LABEL: test_mm_mask_cvtusepi64_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.qw.mem.128
   return _mm_mask_cvtusepi64_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtusepi64_epi16(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtusepi64_epi16
+  // CHECK-LABEL: test_mm256_cvtusepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.qw.256
   return _mm256_cvtusepi64_epi16(__A); 
 }
 
 __m128i test_mm256_mask_cvtusepi64_epi16(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi64_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtusepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.qw.256
   return _mm256_mask_cvtusepi64_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtusepi64_epi16(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtusepi64_epi16
+  // CHECK-LABEL: test_mm256_maskz_cvtusepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.qw.256
   return _mm256_maskz_cvtusepi64_epi16(__M, __A); 
 }
 
 void test_mm256_mask_cvtusepi64_storeu_epi16(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi64_storeu_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtusepi64_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmovus.qw.mem.256
   return _mm256_mask_cvtusepi64_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtepi32_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepi32_epi8
+  // CHECK-LABEL: test_mm_cvtepi32_epi8
   // CHECK: trunc <4 x i32> %{{.*}} to <4 x i8>
   // CHECK: shufflevector <4 x i8> %{{.*}}, <4 x i8> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
   return _mm_cvtepi32_epi8(__A); 
 }
 
 __m128i test_mm_mask_cvtepi32_epi8(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi32_epi8
+  // CHECK-LABEL: test_mm_mask_cvtepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.db.128
   return _mm_mask_cvtepi32_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi32_epi8(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi32_epi8
+  // CHECK-LABEL: test_mm_maskz_cvtepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.db.128
   return _mm_maskz_cvtepi32_epi8(__M, __A); 
 }
 
 void test_mm_mask_cvtepi32_storeu_epi8(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi32_storeu_epi8
+  // CHECK-LABEL: test_mm_mask_cvtepi32_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.db.mem.128
   return _mm_mask_cvtepi32_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtepi32_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepi32_epi8
+  // CHECK-LABEL: test_mm256_cvtepi32_epi8
   // CHECK: trunc <8 x i32> %{{.*}} to <8 x i8>
   // CHECK: shufflevector <8 x i8> %{{.*}}, <8 x i8> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   return _mm256_cvtepi32_epi8(__A); 
 }
 
 __m128i test_mm256_mask_cvtepi32_epi8(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi32_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.db.256
   return _mm256_mask_cvtepi32_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtepi32_epi8(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi32_epi8
+  // CHECK-LABEL: test_mm256_maskz_cvtepi32_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.db.256
   return _mm256_maskz_cvtepi32_epi8(__M, __A); 
 }
 
 void test_mm256_mask_cvtepi32_storeu_epi8(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi32_storeu_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtepi32_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.db.mem.256
   return _mm256_mask_cvtepi32_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtepi32_epi16(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepi32_epi16
+  // CHECK-LABEL: test_mm_cvtepi32_epi16
   // CHECK: trunc <4 x i32> %{{.*}} to <4 x i16>
   // CHECK: shufflevector <4 x i16> %{{.*}}, <4 x i16> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   return _mm_cvtepi32_epi16(__A); 
 }
 
 __m128i test_mm_mask_cvtepi32_epi16(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi32_epi16
+  // CHECK-LABEL: test_mm_mask_cvtepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.dw.128
   return _mm_mask_cvtepi32_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi32_epi16(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi32_epi16
+  // CHECK-LABEL: test_mm_maskz_cvtepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.dw.128
   return _mm_maskz_cvtepi32_epi16(__M, __A); 
 }
 
 void test_mm_mask_cvtepi32_storeu_epi16(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi32_storeu_epi16
+  // CHECK-LABEL: test_mm_mask_cvtepi32_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.dw.mem.128
   return _mm_mask_cvtepi32_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtepi32_epi16(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepi32_epi16
+  // CHECK-LABEL: test_mm256_cvtepi32_epi16
   // CHECK: trunc <8 x i32> %{{.*}} to <8 x i16>
   return _mm256_cvtepi32_epi16(__A); 
 }
 
 __m128i test_mm256_mask_cvtepi32_epi16(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi32_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.dw.256
   return _mm256_mask_cvtepi32_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtepi32_epi16(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi32_epi16
+  // CHECK-LABEL: test_mm256_maskz_cvtepi32_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.dw.256
   return _mm256_maskz_cvtepi32_epi16(__M, __A); 
 }
 
 void test_mm256_mask_cvtepi32_storeu_epi16(void *  __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi32_storeu_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtepi32_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.dw.mem.256
   return _mm256_mask_cvtepi32_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtepi64_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepi64_epi8
+  // CHECK-LABEL: test_mm_cvtepi64_epi8
   // CHECK: trunc <2 x i64> %{{.*}} to <2 x i8>
   // CHECK: shufflevector <2 x i8> %{{.*}}, <2 x i8> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   return _mm_cvtepi64_epi8(__A); 
 }
 
 __m128i test_mm_mask_cvtepi64_epi8(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi64_epi8
+  // CHECK-LABEL: test_mm_mask_cvtepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.qb.128
   return _mm_mask_cvtepi64_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi64_epi8(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi64_epi8
+  // CHECK-LABEL: test_mm_maskz_cvtepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.qb.128
   return _mm_maskz_cvtepi64_epi8(__M, __A); 
 }
 
 void test_mm_mask_cvtepi64_storeu_epi8(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi64_storeu_epi8
+  // CHECK-LABEL: test_mm_mask_cvtepi64_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.qb.mem.128
   return _mm_mask_cvtepi64_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtepi64_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepi64_epi8
+  // CHECK-LABEL: test_mm256_cvtepi64_epi8
   // CHECK: trunc <4 x i64> %{{.*}} to <4 x i8>
   // CHECK: shufflevector <4 x i8> %{{.*}}, <4 x i8> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>
   return _mm256_cvtepi64_epi8(__A); 
 }
 
 __m128i test_mm256_mask_cvtepi64_epi8(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi64_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.qb.256
   return _mm256_mask_cvtepi64_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtepi64_epi8(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi64_epi8
+  // CHECK-LABEL: test_mm256_maskz_cvtepi64_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.qb.256
   return _mm256_maskz_cvtepi64_epi8(__M, __A); 
 }
 
 void test_mm256_mask_cvtepi64_storeu_epi8(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi64_storeu_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtepi64_storeu_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.qb.mem.256
   return _mm256_mask_cvtepi64_storeu_epi8(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtepi64_epi32(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepi64_epi32
+  // CHECK-LABEL: test_mm_cvtepi64_epi32
   // CHECK: trunc <2 x i64> %{{.*}} to <2 x i32>
   // CHECK: shufflevector <2 x i32> %{{.*}}, <2 x i32> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   return _mm_cvtepi64_epi32(__A); 
 }
 
 __m128i test_mm_mask_cvtepi64_epi32(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi64_epi32
+  // CHECK-LABEL: test_mm_mask_cvtepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmov.qd.128
   return _mm_mask_cvtepi64_epi32(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi64_epi32(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi64_epi32
+  // CHECK-LABEL: test_mm_maskz_cvtepi64_epi32
   // CHECK: @llvm.x86.avx512.mask.pmov.qd.128
   return _mm_maskz_cvtepi64_epi32(__M, __A); 
 }
 
 void test_mm_mask_cvtepi64_storeu_epi32(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi64_storeu_epi32
+  // CHECK-LABEL: test_mm_mask_cvtepi64_storeu_epi32
   // CHECK: @llvm.x86.avx512.mask.pmov.qd.mem.128
   return _mm_mask_cvtepi64_storeu_epi32(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtepi64_epi32(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepi64_epi32
+  // CHECK-LABEL: test_mm256_cvtepi64_epi32
   // CHECK: trunc <4 x i64> %{{.*}} to <4 x i32>
   return _mm256_cvtepi64_epi32(__A); 
 }
 
 __m128i test_mm256_mask_cvtepi64_epi32(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi64_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtepi64_epi32
   // CHECK: trunc <4 x i64> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm256_mask_cvtepi64_epi32(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtepi64_epi32(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi64_epi32
+  // CHECK-LABEL: test_mm256_maskz_cvtepi64_epi32
   // CHECK: trunc <4 x i64> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm256_maskz_cvtepi64_epi32(__M, __A); 
 }
 
 void test_mm256_mask_cvtepi64_storeu_epi32(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi64_storeu_epi32
+  // CHECK-LABEL: test_mm256_mask_cvtepi64_storeu_epi32
   // CHECK: @llvm.x86.avx512.mask.pmov.qd.mem.256
   return _mm256_mask_cvtepi64_storeu_epi32(__P, __M, __A); 
 }
 
 __m128i test_mm_cvtepi64_epi16(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepi64_epi16
+  // CHECK-LABEL: test_mm_cvtepi64_epi16
   // CHECK: trunc <2 x i64> %{{.*}} to <2 x i16>
   // CHECK: shufflevector <2 x i16> %{{.*}}, <2 x i16> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3>
   return _mm_cvtepi64_epi16(__A); 
 }
 
 __m128i test_mm_mask_cvtepi64_epi16(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi64_epi16
+  // CHECK-LABEL: test_mm_mask_cvtepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.qw.128
   return _mm_mask_cvtepi64_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi64_epi16(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi64_epi16
+  // CHECK-LABEL: test_mm_maskz_cvtepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.qw.128
   return _mm_maskz_cvtepi64_epi16(__M, __A); 
 }
 
 void test_mm_mask_cvtepi64_storeu_epi16(void * __P, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi64_storeu_epi16
+  // CHECK-LABEL: test_mm_mask_cvtepi64_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.qw.mem.128
   return _mm_mask_cvtepi64_storeu_epi16(__P, __M, __A); 
 }
 
 __m128i test_mm256_cvtepi64_epi16(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepi64_epi16
+  // CHECK-LABEL: test_mm256_cvtepi64_epi16
   // CHECK: trunc <4 x i64> %{{.*}} to <4 x i16>
   // CHECK: shufflevector <4 x i16> %{{.*}}, <4 x i16> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   return _mm256_cvtepi64_epi16(__A); 
 }
 
 __m128i test_mm256_mask_cvtepi64_epi16(__m128i __O, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi64_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.qw.256
   return _mm256_mask_cvtepi64_epi16(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtepi64_epi16(__mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi64_epi16
+  // CHECK-LABEL: test_mm256_maskz_cvtepi64_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.qw.256
   return _mm256_maskz_cvtepi64_epi16(__M, __A); 
 }
 
 void test_mm256_mask_cvtepi64_storeu_epi16(void * __P, __mmask8 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi64_storeu_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtepi64_storeu_epi16
   // CHECK: @llvm.x86.avx512.mask.pmov.qw.mem.256
   return _mm256_mask_cvtepi64_storeu_epi16(__P, __M, __A); 
 }
 
 __m128 test_mm256_extractf32x4_ps(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_extractf32x4_ps
+  // CHECK-LABEL: test_mm256_extractf32x4_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   return _mm256_extractf32x4_ps(__A, 1); 
 }
 
 __m128 test_mm256_mask_extractf32x4_ps(__m128 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_extractf32x4_ps
+  // CHECK-LABEL: test_mm256_mask_extractf32x4_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm256_mask_extractf32x4_ps(__W, __U, __A, 1); 
 }
 
 __m128 test_mm256_maskz_extractf32x4_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_extractf32x4_ps
+  // CHECK-LABEL: test_mm256_maskz_extractf32x4_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm256_maskz_extractf32x4_ps(__U, __A, 1); 
 }
 
 __m128i test_mm256_extracti32x4_epi32(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_extracti32x4_epi32
+  // CHECK-LABEL: test_mm256_extracti32x4_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   return _mm256_extracti32x4_epi32(__A, 1); 
 }
 
 __m128i test_mm256_mask_extracti32x4_epi32(__m128i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_extracti32x4_epi32
+  // CHECK-LABEL: test_mm256_mask_extracti32x4_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm256_mask_extracti32x4_epi32(__W, __U, __A, 1); 
 }
 
 __m128i test_mm256_maskz_extracti32x4_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_extracti32x4_epi32
+  // CHECK-LABEL: test_mm256_maskz_extracti32x4_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm256_maskz_extracti32x4_epi32(__U, __A, 1); 
 }
 
 __m256 test_mm256_insertf32x4(__m256 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm256_insertf32x4
+  // CHECK-LABEL: test_mm256_insertf32x4
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   return _mm256_insertf32x4(__A, __B, 1); 
 }
 
 __m256 test_mm256_mask_insertf32x4(__m256 __W, __mmask8 __U, __m256 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm256_mask_insertf32x4
+  // CHECK-LABEL: test_mm256_mask_insertf32x4
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_insertf32x4(__W, __U, __A, __B, 1); 
 }
 
 __m256 test_mm256_maskz_insertf32x4(__mmask8 __U, __m256 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_insertf32x4
+  // CHECK-LABEL: test_mm256_maskz_insertf32x4
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_insertf32x4(__U, __A, __B, 1); 
 }
 
 __m256i test_mm256_inserti32x4(__m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_inserti32x4
+  // CHECK-LABEL: test_mm256_inserti32x4
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   return _mm256_inserti32x4(__A, __B, 1); 
 }
 
 __m256i test_mm256_mask_inserti32x4(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_inserti32x4
+  // CHECK-LABEL: test_mm256_mask_inserti32x4
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_inserti32x4(__W, __U, __A, __B, 1); 
 }
 
 __m256i test_mm256_maskz_inserti32x4(__mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_inserti32x4
+  // CHECK-LABEL: test_mm256_maskz_inserti32x4
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_inserti32x4(__U, __A, __B, 1); 
 }
 
 __m128d test_mm_getmant_pd(__m128d __A) {
-  // CHECK-LABEL: @test_mm_getmant_pd
+  // CHECK-LABEL: test_mm_getmant_pd
   // CHECK: @llvm.x86.avx512.mask.getmant.pd.128
   return _mm_getmant_pd(__A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m128d test_mm_mask_getmant_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_getmant_pd
+  // CHECK-LABEL: test_mm_mask_getmant_pd
   // CHECK: @llvm.x86.avx512.mask.getmant.pd.128
   return _mm_mask_getmant_pd(__W, __U, __A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m128d test_mm_maskz_getmant_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_getmant_pd
+  // CHECK-LABEL: test_mm_maskz_getmant_pd
   // CHECK: @llvm.x86.avx512.mask.getmant.pd.128
   return _mm_maskz_getmant_pd(__U, __A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m256d test_mm256_getmant_pd(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_getmant_pd
+  // CHECK-LABEL: test_mm256_getmant_pd
   // CHECK: @llvm.x86.avx512.mask.getmant.pd.256
   return _mm256_getmant_pd(__A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m256d test_mm256_mask_getmant_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_getmant_pd
+  // CHECK-LABEL: test_mm256_mask_getmant_pd
   // CHECK: @llvm.x86.avx512.mask.getmant.pd.256
   return _mm256_mask_getmant_pd(__W, __U, __A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m256d test_mm256_maskz_getmant_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_getmant_pd
+  // CHECK-LABEL: test_mm256_maskz_getmant_pd
   // CHECK: @llvm.x86.avx512.mask.getmant.pd.256
   return _mm256_maskz_getmant_pd(__U, __A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m128 test_mm_getmant_ps(__m128 __A) {
-  // CHECK-LABEL: @test_mm_getmant_ps
+  // CHECK-LABEL: test_mm_getmant_ps
   // CHECK: @llvm.x86.avx512.mask.getmant.ps.128
   return _mm_getmant_ps(__A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m128 test_mm_mask_getmant_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_getmant_ps
+  // CHECK-LABEL: test_mm_mask_getmant_ps
   // CHECK: @llvm.x86.avx512.mask.getmant.ps.128
   return _mm_mask_getmant_ps(__W, __U, __A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m128 test_mm_maskz_getmant_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_getmant_ps
+  // CHECK-LABEL: test_mm_maskz_getmant_ps
   // CHECK: @llvm.x86.avx512.mask.getmant.ps.128
   return _mm_maskz_getmant_ps(__U, __A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m256 test_mm256_getmant_ps(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_getmant_ps
+  // CHECK-LABEL: test_mm256_getmant_ps
   // CHECK: @llvm.x86.avx512.mask.getmant.ps.256
   return _mm256_getmant_ps(__A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m256 test_mm256_mask_getmant_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_getmant_ps
+  // CHECK-LABEL: test_mm256_mask_getmant_ps
   // CHECK: @llvm.x86.avx512.mask.getmant.ps.256
   return _mm256_mask_getmant_ps(__W, __U, __A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m256 test_mm256_maskz_getmant_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_getmant_ps
+  // CHECK-LABEL: test_mm256_maskz_getmant_ps
   // CHECK: @llvm.x86.avx512.mask.getmant.ps.256
   return _mm256_maskz_getmant_ps(__U, __A,_MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan); 
 }
 
 __m128d test_mm_mmask_i64gather_pd(__m128d __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm_mmask_i64gather_pd
+  // CHECK-LABEL: test_mm_mmask_i64gather_pd
   // CHECK: @llvm.x86.avx512.mask.gather3div2.df
   return _mm_mmask_i64gather_pd(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m128i test_mm_mmask_i64gather_epi64(__m128i __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm_mmask_i64gather_epi64
+  // CHECK-LABEL: test_mm_mmask_i64gather_epi64
   // CHECK: @llvm.x86.avx512.mask.gather3div2.di
   return _mm_mmask_i64gather_epi64(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m256d test_mm256_mmask_i64gather_pd(__m256d __v1_old, __mmask8 __mask, __m256i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm256_mmask_i64gather_pd
+  // CHECK-LABEL: test_mm256_mmask_i64gather_pd
   // CHECK: @llvm.x86.avx512.mask.gather3div4.df
   return _mm256_mmask_i64gather_pd(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m256i test_mm256_mmask_i64gather_epi64(__m256i __v1_old, __mmask8 __mask, __m256i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm256_mmask_i64gather_epi64
+  // CHECK-LABEL: test_mm256_mmask_i64gather_epi64
   // CHECK: @llvm.x86.avx512.mask.gather3div4.di
   return _mm256_mmask_i64gather_epi64(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m128 test_mm_mmask_i64gather_ps(__m128 __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm_mmask_i64gather_ps
+  // CHECK-LABEL: test_mm_mmask_i64gather_ps
   // CHECK: @llvm.x86.avx512.mask.gather3div4.sf
   return _mm_mmask_i64gather_ps(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m128i test_mm_mmask_i64gather_epi32(__m128i __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm_mmask_i64gather_epi32
+  // CHECK-LABEL: test_mm_mmask_i64gather_epi32
   // CHECK: @llvm.x86.avx512.mask.gather3div4.si
   return _mm_mmask_i64gather_epi32(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m128 test_mm256_mmask_i64gather_ps(__m128 __v1_old, __mmask8 __mask, __m256i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm256_mmask_i64gather_ps
+  // CHECK-LABEL: test_mm256_mmask_i64gather_ps
   // CHECK: @llvm.x86.avx512.mask.gather3div8.sf
   return _mm256_mmask_i64gather_ps(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m128i test_mm256_mmask_i64gather_epi32(__m128i __v1_old, __mmask8 __mask, __m256i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm256_mmask_i64gather_epi32
+  // CHECK-LABEL: test_mm256_mmask_i64gather_epi32
   // CHECK: @llvm.x86.avx512.mask.gather3div8.si
   return _mm256_mmask_i64gather_epi32(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m128d test_mm_mask_i32gather_pd(__m128d __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm_mask_i32gather_pd
+  // CHECK-LABEL: test_mm_mask_i32gather_pd
   // CHECK: @llvm.x86.avx512.mask.gather3siv2.df
   return _mm_mmask_i32gather_pd(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m128i test_mm_mask_i32gather_epi64(__m128i __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm_mask_i32gather_epi64
+  // CHECK-LABEL: test_mm_mask_i32gather_epi64
   // CHECK: @llvm.x86.avx512.mask.gather3siv2.di
   return _mm_mmask_i32gather_epi64(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m256d test_mm256_mask_i32gather_pd(__m256d __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm256_mask_i32gather_pd
+  // CHECK-LABEL: test_mm256_mask_i32gather_pd
   // CHECK: @llvm.x86.avx512.mask.gather3siv4.df
   return _mm256_mmask_i32gather_pd(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m256i test_mm256_mask_i32gather_epi64(__m256i __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm256_mask_i32gather_epi64
+  // CHECK-LABEL: test_mm256_mask_i32gather_epi64
   // CHECK: @llvm.x86.avx512.mask.gather3siv4.di
   return _mm256_mmask_i32gather_epi64(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m128 test_mm_mask_i32gather_ps(__m128 __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm_mask_i32gather_ps
+  // CHECK-LABEL: test_mm_mask_i32gather_ps
   // CHECK: @llvm.x86.avx512.mask.gather3siv4.sf
   return _mm_mmask_i32gather_ps(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m128i test_mm_mask_i32gather_epi32(__m128i __v1_old, __mmask8 __mask, __m128i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm_mask_i32gather_epi32
+  // CHECK-LABEL: test_mm_mask_i32gather_epi32
   // CHECK: @llvm.x86.avx512.mask.gather3siv4.si
   return _mm_mmask_i32gather_epi32(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m256 test_mm256_mask_i32gather_ps(__m256 __v1_old, __mmask8 __mask, __m256i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm256_mask_i32gather_ps
+  // CHECK-LABEL: test_mm256_mask_i32gather_ps
   // CHECK: @llvm.x86.avx512.mask.gather3siv8.sf
   return _mm256_mmask_i32gather_ps(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m256i test_mm256_mask_i32gather_epi32(__m256i __v1_old, __mmask8 __mask, __m256i __index, void const *__addr) {
-  // CHECK-LABEL: @test_mm256_mask_i32gather_epi32
+  // CHECK-LABEL: test_mm256_mask_i32gather_epi32
   // CHECK: @llvm.x86.avx512.mask.gather3siv8.si
   return _mm256_mmask_i32gather_epi32(__v1_old, __mask, __index, __addr, 2); 
 }
 
 __m256d test_mm256_permutex_pd(__m256d __X) {
-  // CHECK-LABEL: @test_mm256_permutex_pd
+  // CHECK-LABEL: test_mm256_permutex_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <4 x i32> <i32 3, i32 0, i32 0, i32 0>
   return _mm256_permutex_pd(__X, 3);
 }
 
 __m256d test_mm256_mask_permutex_pd(__m256d __W, __mmask8 __U, __m256d __X) {
-  // CHECK-LABEL: @test_mm256_mask_permutex_pd
+  // CHECK-LABEL: test_mm256_mask_permutex_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 0>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_permutex_pd(__W, __U, __X, 1);
 }
 
 __m256d test_mm256_maskz_permutex_pd(__mmask8 __U, __m256d __X) {
-  // CHECK-LABEL: @test_mm256_maskz_permutex_pd
+  // CHECK-LABEL: test_mm256_maskz_permutex_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 0>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_permutex_pd(__U, __X, 1);
 }
 
 __m256i test_mm256_permutex_epi64(__m256i __X) {
-  // CHECK-LABEL: @test_mm256_permutex_epi64
+  // CHECK-LABEL: test_mm256_permutex_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> poison, <4 x i32> <i32 3, i32 0, i32 0, i32 0>
   return _mm256_permutex_epi64(__X, 3);
 }
 
 __m256i test_mm256_mask_permutex_epi64(__m256i __W, __mmask8 __M, __m256i __X) {
-  // CHECK-LABEL: @test_mm256_mask_permutex_epi64
+  // CHECK-LABEL: test_mm256_mask_permutex_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> poison, <4 x i32> <i32 3, i32 0, i32 0, i32 0>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_permutex_epi64(__W, __M, __X, 3);
 }
 
 __m256i test_mm256_maskz_permutex_epi64(__mmask8 __M, __m256i __X) {
-  // CHECK-LABEL: @test_mm256_maskz_permutex_epi64
+  // CHECK-LABEL: test_mm256_maskz_permutex_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> poison, <4 x i32> <i32 3, i32 0, i32 0, i32 0>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_permutex_epi64(__M, __X, 3);
 }
 
 __m256d test_mm256_permutexvar_pd(__m256i __X, __m256d __Y) {
-  // CHECK-LABEL: @test_mm256_permutexvar_pd
+  // CHECK-LABEL: test_mm256_permutexvar_pd
   // CHECK: @llvm.x86.avx512.permvar.df.256
   return _mm256_permutexvar_pd(__X, __Y);
 }
 
 __m256d test_mm256_mask_permutexvar_pd(__m256d __W, __mmask8 __U, __m256i __X, __m256d __Y) {
-  // CHECK-LABEL: @test_mm256_mask_permutexvar_pd
+  // CHECK-LABEL: test_mm256_mask_permutexvar_pd
   // CHECK: @llvm.x86.avx512.permvar.df.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_permutexvar_pd(__W, __U, __X, __Y);
 }
 
 __m256d test_mm256_maskz_permutexvar_pd(__mmask8 __U, __m256i __X, __m256d __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_permutexvar_pd
+  // CHECK-LABEL: test_mm256_maskz_permutexvar_pd
   // CHECK: @llvm.x86.avx512.permvar.df.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_permutexvar_pd(__U, __X, __Y);
 }
 
 __m256i test_mm256_maskz_permutexvar_epi64(__mmask8 __M, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_permutexvar_epi64
+  // CHECK-LABEL: test_mm256_maskz_permutexvar_epi64
   // CHECK: @llvm.x86.avx512.permvar.di.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_permutexvar_epi64(__M, __X, __Y);
 }
 
 __m256i test_mm256_mask_permutexvar_epi64(__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_permutexvar_epi64
+  // CHECK-LABEL: test_mm256_mask_permutexvar_epi64
   // CHECK: @llvm.x86.avx512.permvar.di.256
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_permutexvar_epi64(__W, __M, __X, __Y);
 }
 
 __m256 test_mm256_mask_permutexvar_ps(__m256 __W, __mmask8 __U, __m256i __X, __m256 __Y) {
-  // CHECK-LABEL: @test_mm256_mask_permutexvar_ps
+  // CHECK-LABEL: test_mm256_mask_permutexvar_ps
   // CHECK: @llvm.x86.avx2.permps
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_permutexvar_ps(__W, __U, __X, __Y);
 }
 
 __m256 test_mm256_maskz_permutexvar_ps(__mmask8 __U, __m256i __X, __m256 __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_permutexvar_ps
+  // CHECK-LABEL: test_mm256_maskz_permutexvar_ps
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_permutexvar_ps(__U, __X, __Y);
 }
 
 __m256 test_mm256_permutexvar_ps(__m256i __X, __m256 __Y) {
-  // CHECK-LABEL: @test_mm256_permutexvar_ps
+  // CHECK-LABEL: test_mm256_permutexvar_ps
   // CHECK: @llvm.x86.avx2.permps
   return _mm256_permutexvar_ps( __X, __Y);
 }
 
 __m256i test_mm256_maskz_permutexvar_epi32(__mmask8 __M, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_permutexvar_epi32
+  // CHECK-LABEL: test_mm256_maskz_permutexvar_epi32
   // CHECK: @llvm.x86.avx2.permd
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_permutexvar_epi32(__M, __X, __Y);
 }
 
 __m256i test_mm256_permutexvar_epi32(__m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_permutexvar_epi32
+  // CHECK-LABEL: test_mm256_permutexvar_epi32
   // CHECK: @llvm.x86.avx2.permd
   return _mm256_permutexvar_epi32(__X, __Y);
 }
 
 __m256i test_mm256_mask_permutexvar_epi32(__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_permutexvar_epi32
+  // CHECK-LABEL: test_mm256_mask_permutexvar_epi32
   // CHECK: @llvm.x86.avx2.permd
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_permutexvar_epi32(__W, __M, __X, __Y);
 }
 
 __m128i test_mm_alignr_epi32(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_alignr_epi32
+  // CHECK-LABEL: test_mm_alignr_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   return _mm_alignr_epi32(__A, __B, 1);
 }
 
 __m128i test_mm_mask_alignr_epi32(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_alignr_epi32
+  // CHECK-LABEL: test_mm_mask_alignr_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_alignr_epi32(__W, __U, __A, __B, 5);
 }
 
 __m128i test_mm_maskz_alignr_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_alignr_epi32
+  // CHECK-LABEL: test_mm_maskz_alignr_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_alignr_epi32(__U, __A, __B, 1);
 }
 
 __m256i test_mm256_alignr_epi32(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_alignr_epi32
+  // CHECK-LABEL: test_mm256_alignr_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   return _mm256_alignr_epi32(__A, __B, 1);
 }
 
 __m256i test_mm256_mask_alignr_epi32(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_alignr_epi32
+  // CHECK-LABEL: test_mm256_mask_alignr_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_alignr_epi32(__W, __U, __A, __B, 9);
 }
 
 __m256i test_mm256_maskz_alignr_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_alignr_epi32
+  // CHECK-LABEL: test_mm256_maskz_alignr_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_alignr_epi32(__U, __A, __B, 1);
 }
 
 __m128i test_mm_alignr_epi64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_alignr_epi64
+  // CHECK-LABEL: test_mm_alignr_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> <i32 1, i32 2>
   return _mm_alignr_epi64(__A, __B, 1);
 }
 
 __m128i test_mm_mask_alignr_epi64(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_alignr_epi64
+  // CHECK-LABEL: test_mm_mask_alignr_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> <i32 1, i32 2>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_alignr_epi64(__W, __U, __A, __B, 3);
 }
 
 __m128i test_mm_maskz_alignr_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_alignr_epi64
+  // CHECK-LABEL: test_mm_maskz_alignr_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> <i32 1, i32 2>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_alignr_epi64(__U, __A, __B, 1);
 }
 
 __m256i test_mm256_alignr_epi64(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_alignr_epi64
+  // CHECK-LABEL: test_mm256_alignr_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   return _mm256_alignr_epi64(__A, __B, 1);
 }
 
 __m256i test_mm256_mask_alignr_epi64(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_alignr_epi64
+  // CHECK-LABEL: test_mm256_mask_alignr_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_alignr_epi64(__W, __U, __A, __B, 5);
 }
 
 __m256i test_mm256_maskz_alignr_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_alignr_epi64
+  // CHECK-LABEL: test_mm256_maskz_alignr_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_alignr_epi64(__U, __A, __B, 1);
 }
 
 __m128 test_mm_mask_movehdup_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_movehdup_ps
+  // CHECK-LABEL: test_mm_mask_movehdup_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
   // CHECK: select <4 x i1> %{{.*}} <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_movehdup_ps(__W, __U, __A);
 }
 
 __m128 test_mm_maskz_movehdup_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_movehdup_ps
+  // CHECK-LABEL: test_mm_maskz_movehdup_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
   // CHECK: select <4 x i1> %{{.*}} <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_movehdup_ps(__U, __A);
 }
 
 __m256 test_mm256_mask_movehdup_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_movehdup_ps
+  // CHECK-LABEL: test_mm256_mask_movehdup_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
   // CHECK: select <8 x i1> %{{.*}} <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_movehdup_ps(__W, __U, __A);
 }
 
 __m256 test_mm256_maskz_movehdup_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_movehdup_ps
+  // CHECK-LABEL: test_mm256_maskz_movehdup_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
   // CHECK: select <8 x i1> %{{.*}} <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_movehdup_ps(__U, __A);
 }
 
 __m128 test_mm_mask_moveldup_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_moveldup_ps
+  // CHECK-LABEL: test_mm_mask_moveldup_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   // CHECK: select <4 x i1> %{{.*}} <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_moveldup_ps(__W, __U, __A);
 }
 
 __m128 test_mm_maskz_moveldup_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_moveldup_ps
+  // CHECK-LABEL: test_mm_maskz_moveldup_ps
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
   // CHECK: select <4 x i1> %{{.*}} <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_moveldup_ps(__U, __A);
 }
 
 __m256 test_mm256_mask_moveldup_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_moveldup_ps
+  // CHECK-LABEL: test_mm256_mask_moveldup_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   // CHECK: select <8 x i1> %{{.*}} <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_moveldup_ps(__W, __U, __A);
 }
 
 __m256 test_mm256_maskz_moveldup_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_moveldup_ps
+  // CHECK-LABEL: test_mm256_maskz_moveldup_ps
   // CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
   // CHECK: select <8 x i1> %{{.*}} <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_moveldup_ps(__U, __A);
 }
 
 __m128i test_mm_mask_shuffle_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_shuffle_epi32
+  // CHECK-LABEL: test_mm_mask_shuffle_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> <i32 1, i32 0, i32 0, i32 0>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_shuffle_epi32(__W, __U, __A, 1);
 }
 
 __m128i test_mm_maskz_shuffle_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_shuffle_epi32
+  // CHECK-LABEL: test_mm_maskz_shuffle_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> poison, <4 x i32> <i32 2, i32 0, i32 0, i32 0>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_shuffle_epi32(__U, __A, 2);
 }
 
 __m256i test_mm256_mask_shuffle_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_shuffle_epi32
+  // CHECK-LABEL: test_mm256_mask_shuffle_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> poison, <8 x i32> <i32 2, i32 0, i32 0, i32 0, i32 6, i32 4, i32 4, i32 4>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_shuffle_epi32(__W, __U, __A, 2);
 }
 
 __m256i test_mm256_maskz_shuffle_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_shuffle_epi32
+  // CHECK-LABEL: test_mm256_maskz_shuffle_epi32
   // CHECK: shufflevector <8 x i32> %{{.*}}, <8 x i32> poison, <8 x i32> <i32 2, i32 0, i32 0, i32 0, i32 6, i32 4, i32 4, i32 4>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_shuffle_epi32(__U, __A, 2);
 }
 
 __m128d test_mm_mask_mov_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_mov_pd
+  // CHECK-LABEL: test_mm_mask_mov_pd
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_mov_pd(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_m128d(_mm_mask_mov_pd((__m128d){-2.0, -1.0}, 0x2, (__m128d){+1.0, +2.0}), -2.0, +2.0));
 
 __m128d test_mm_maskz_mov_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_mov_pd
+  // CHECK-LABEL: test_mm_maskz_mov_pd
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_mov_pd(__U, __A);
 }
+TEST_CONSTEXPR(match_m128d(_mm_maskz_mov_pd(0x1, (__m128d){+1.0, +2.0}), +1.0, +0.0));
 
 __m256d test_mm256_mask_mov_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_mov_pd
+  // CHECK-LABEL: test_mm256_mask_mov_pd
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_mov_pd(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_mask_mov_pd((__m256d){-4.0, -3.0, -2.0, -1.0}, 0x3, (__m256d){+1.0, +2.0, +3.0, +4.0}), +1.0, +2.0, -2.0, -1.0));
 
 __m256d test_mm256_maskz_mov_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_mov_pd
+  // CHECK-LABEL: test_mm256_maskz_mov_pd
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_mov_pd(__U, __A);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_maskz_mov_pd(0xC, (__m256d){+1.0, +2.0, +3.0, +4.0}), 0.0, 0.0, +3.0, +4.0));
 
 __m128 test_mm_mask_mov_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_mov_ps
+  // CHECK-LABEL: test_mm_mask_mov_ps
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask_mov_ps(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_m128(_mm_mask_mov_ps((__m128){-4.0f, -3.0f, -2.0f, -1.0f}, 0x3, (__m128){+1.0f, +2.0f, +3.0f, +4.0f}), +1.0f, +2.0f, -2.0f, -1.0f));
 
 __m128 test_mm_maskz_mov_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_mov_ps
+  // CHECK-LABEL: test_mm_maskz_mov_ps
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_maskz_mov_ps(__U, __A);
 }
+TEST_CONSTEXPR(match_m128(_mm_maskz_mov_ps(0xC, (__m128){+1.0f, +2.0f, +3.0f, +4.0f}), 0.0f, 0.0f, +3.0f, +4.0f));
 
 __m256 test_mm256_mask_mov_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_mov_ps
+  // CHECK-LABEL: test_mm256_mask_mov_ps
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_mov_ps(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_m256(_mm256_mask_mov_ps((__m256){-8.0f, -7.0f, -6.0f, -5.0f, -4.0f, -3.0f, -2.0f, -1.0f}, 0xC3, (__m256){+1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}), +1.0f, +2.0f, -6.0f, -5.0f, -4.0f, -3.0f, +7.0f, +8.0f));
 
 __m256 test_mm256_maskz_mov_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_mov_ps
+  // CHECK-LABEL: test_mm256_maskz_mov_ps
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_mov_ps(__U, __A);
 }
+TEST_CONSTEXPR(match_m256(_mm256_maskz_mov_ps(0xC3, (__m256){+1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f, +8.0f}), +1.0f, +2.0f, 0.0f, 0.0f, 0.0f, 0.0f, +7.0f, +8.0f));
 
 __m128 test_mm_mask_cvtph_ps(__m128 __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtph_ps
+  // CHECK-LABEL: test_mm_mask_cvtph_ps
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: fpext <4 x half> %{{.*}} to <4 x float>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -9565,7 +9577,7 @@ __m128 test_mm_mask_cvtph_ps(__m128 __W, __mmask8 __U, __m128i __A) {
 }
 
 __m128 test_mm_maskz_cvtph_ps(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtph_ps
+  // CHECK-LABEL: test_mm_maskz_cvtph_ps
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: fpext <4 x half> %{{.*}} to <4 x float>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -9573,167 +9585,167 @@ __m128 test_mm_maskz_cvtph_ps(__mmask8 __U, __m128i __A) {
 }
 
 __m256 test_mm256_mask_cvtph_ps(__m256 __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtph_ps
+  // CHECK-LABEL: test_mm256_mask_cvtph_ps
   // CHECK: fpext <8 x half> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_cvtph_ps(__W, __U, __A);
 }
 
 __m256 test_mm256_maskz_cvtph_ps(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtph_ps
+  // CHECK-LABEL: test_mm256_maskz_cvtph_ps
   // CHECK: fpext <8 x half> %{{.*}} to <8 x float>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_cvtph_ps(__U, __A);
 }
 
 __m128i test_mm_mask_cvtps_ph(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtps_ph
+  // CHECK-LABEL: test_mm_mask_cvtps_ph
   // CHECK: @llvm.x86.avx512.mask.vcvtps2ph.128
   return _mm_mask_cvtps_ph(__W, __U, __A, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128i test_mm_maskz_cvtps_ph(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtps_ph
+  // CHECK-LABEL: test_mm_maskz_cvtps_ph
   // CHECK: @llvm.x86.avx512.mask.vcvtps2ph.128
   return _mm_maskz_cvtps_ph(__U, __A, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128i test_mm256_mask_cvtps_ph(__m128i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtps_ph
+  // CHECK-LABEL: test_mm256_mask_cvtps_ph
   // CHECK: @llvm.x86.avx512.mask.vcvtps2ph.256
   return _mm256_mask_cvtps_ph(__W, __U, __A, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128i test_mm256_maskz_cvtps_ph(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtps_ph
+  // CHECK-LABEL: test_mm256_maskz_cvtps_ph
   // CHECK: @llvm.x86.avx512.mask.vcvtps2ph.256
   return _mm256_maskz_cvtps_ph(__U, __A, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC);
 }
 
 __m128i test_mm_mask_cvt_roundps_ph(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvt_roundps_ph
+  // CHECK-LABEL: test_mm_mask_cvt_roundps_ph
   // CHECK: @llvm.x86.avx512.mask.vcvtps2ph.128
   return _mm_mask_cvt_roundps_ph(__W, __U, __A, _MM_FROUND_TO_ZERO);
 }
 
 __m128i test_mm_maskz_cvt_roundps_ph(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvt_roundps_ph
+  // CHECK-LABEL: test_mm_maskz_cvt_roundps_ph
   // CHECK: @llvm.x86.avx512.mask.vcvtps2ph.128
   return _mm_maskz_cvt_roundps_ph(__U, __A, _MM_FROUND_TO_ZERO);
 }
 
 __m128i test_mm256_mask_cvt_roundps_ph(__m128i __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvt_roundps_ph
+  // CHECK-LABEL: test_mm256_mask_cvt_roundps_ph
   // CHECK: @llvm.x86.avx512.mask.vcvtps2ph.256
   return _mm256_mask_cvt_roundps_ph(__W, __U, __A, _MM_FROUND_TO_ZERO);
 }
 
 __m128i test_mm256_maskz_cvt_roundps_ph(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvt_roundps_ph
+  // CHECK-LABEL: test_mm256_maskz_cvt_roundps_ph
   // CHECK: @llvm.x86.avx512.mask.vcvtps2ph.256
   return _mm256_maskz_cvt_roundps_ph(__U, __A, _MM_FROUND_TO_ZERO);
 }
 
 __mmask8 test_mm_cmpeq_epi32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpeq_epi32_mask
+  // CHECK-LABEL: test_mm_cmpeq_epi32_mask
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpeq_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpeq_epi32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpeq_epi32_mask
+  // CHECK-LABEL: test_mm_mask_cmpeq_epi32_mask
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpeq_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_mask_cmpeq_epi64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpeq_epi64_mask
+  // CHECK-LABEL: test_mm_mask_cmpeq_epi64_mask
   // CHECK: icmp eq <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpeq_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpeq_epi64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpeq_epi64_mask
+  // CHECK-LABEL: test_mm_cmpeq_epi64_mask
   // CHECK: icmp eq <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpeq_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm_cmpgt_epi32_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpgt_epi32_mask
+  // CHECK-LABEL: test_mm_cmpgt_epi32_mask
   // CHECK: icmp sgt <4 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpgt_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpgt_epi32_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpgt_epi32_mask
+  // CHECK-LABEL: test_mm_mask_cmpgt_epi32_mask
   // CHECK: icmp sgt <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpgt_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_mask_cmpgt_epi64_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpgt_epi64_mask
+  // CHECK-LABEL: test_mm_mask_cmpgt_epi64_mask
   // CHECK: icmp sgt <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <2 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpgt_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpgt_epi64_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpgt_epi64_mask
+  // CHECK-LABEL: test_mm_cmpgt_epi64_mask
   // CHECK: icmp sgt <2 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpgt_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_cmpeq_epi32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpeq_epi32_mask
+  // CHECK-LABEL: test_mm256_cmpeq_epi32_mask
   // CHECK: icmp eq <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpeq_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpeq_epi32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpeq_epi32_mask
+  // CHECK-LABEL: test_mm256_mask_cmpeq_epi32_mask
   // CHECK: icmp eq <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpeq_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpeq_epi64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpeq_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_cmpeq_epi64_mask
   // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpeq_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpeq_epi64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpeq_epi64_mask
+  // CHECK-LABEL: test_mm256_cmpeq_epi64_mask
   // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpeq_epi64_mask(__a, __b);
 }
 
 __mmask8 test_mm256_cmpgt_epi32_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpgt_epi32_mask
+  // CHECK-LABEL: test_mm256_cmpgt_epi32_mask
   // CHECK: icmp sgt <8 x i32> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpgt_epi32_mask(__a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpgt_epi32_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpgt_epi32_mask
+  // CHECK-LABEL: test_mm256_mask_cmpgt_epi32_mask
   // CHECK: icmp sgt <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpgt_epi32_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_mask_cmpgt_epi64_mask(__mmask8 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpgt_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_cmpgt_epi64_mask
   // CHECK: icmp sgt <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: and <4 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_mask_cmpgt_epi64_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm256_cmpgt_epi64_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpgt_epi64_mask
+  // CHECK-LABEL: test_mm256_cmpgt_epi64_mask
   // CHECK: icmp sgt <4 x i64> %{{.*}}, %{{.*}}
   return (__mmask8)_mm256_cmpgt_epi64_mask(__a, __b);
 }
diff --git a/clang/test/CodeGen/X86/avx512vlbf16-builtins.c b/clang/test/CodeGen/X86/avx512vlbf16-builtins.c
index f62ba46cfd95c..5e37b4d502ad1 100644
--- a/clang/test/CodeGen/X86/avx512vlbf16-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vlbf16-builtins.c
@@ -1,222 +1,196 @@
-//  RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin \
-//  RUN:            -target-feature +avx512bf16 -target-feature \
-//  RUN:            +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bf16 -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512bf16 -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bf16 -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512bf16 -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 __m128bh test_mm_cvtne2ps2bf16(__m128 A, __m128 B) {
-  // CHECK-LABEL: @test_mm_cvtne2ps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.128
-  // CHECK: ret <8 x bfloat> %{{.*}}
+  // CHECK-LABEL: test_mm_cvtne2ps2bf16
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.128(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_cvtne2ps_pbh(A, B);
 }
 
 __m128bh test_mm_maskz_cvtne2ps2bf16(__m128 A, __m128 B, __mmask8 U) {
-  // CHECK-LABEL: @test_mm_maskz_cvtne2ps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.128
+  // CHECK-LABEL: test_mm_maskz_cvtne2ps2bf16
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.128(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x bfloat> %{{.*}}, <8 x bfloat> %{{.*}}
-  // CHECK: ret <8 x bfloat> %{{.*}}
   return _mm_maskz_cvtne2ps_pbh(U, A, B);
 }
 
 __m128bh test_mm_mask_cvtne2ps2bf16(__m128bh C, __mmask8 U, __m128 A, __m128 B) {
-  // CHECK-LABEL: @test_mm_mask_cvtne2ps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.128
+  // CHECK-LABEL: test_mm_mask_cvtne2ps2bf16
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.128(<4 x float> %{{.*}}, <4 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x bfloat> %{{.*}}, <8 x bfloat> %{{.*}}
-  // CHECK: ret <8 x bfloat> %{{.*}}
   return _mm_mask_cvtne2ps_pbh(C, U, A, B);
 }
 
 __m256bh test_mm256_cvtne2ps2bf16(__m256 A, __m256 B) {
-  // CHECK-LABEL: @test_mm256_cvtne2ps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.256
-  // CHECK: ret <16 x bfloat> %{{.*}}
+  // CHECK-LABEL: test_mm256_cvtne2ps2bf16
+  // CHECK: call {{.*}}<16 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.256(<8 x float> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_cvtne2ps_pbh(A, B);
 }
 
 __m256bh test_mm256_maskz_cvtne2ps2bf16(__m256 A, __m256 B, __mmask16 U) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtne2ps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.256
+  // CHECK-LABEL: test_mm256_maskz_cvtne2ps2bf16
+  // CHECK: call {{.*}}<16 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.256(<8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}}
-  // CHECK: ret <16 x bfloat> %{{.*}}
   return _mm256_maskz_cvtne2ps_pbh(U, A, B);
 }
 
 __m256bh test_mm256_mask_cvtne2ps2bf16(__m256bh C, __mmask16 U, __m256 A, __m256 B) {
-  // CHECK-LABEL: @test_mm256_mask_cvtne2ps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.256
+  // CHECK-LABEL: test_mm256_mask_cvtne2ps2bf16
+  // CHECK: call {{.*}}<16 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.256(<8 x float> %{{.*}}, <8 x float> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}}
-  // CHECK: ret <16 x bfloat> %{{.*}}
   return _mm256_mask_cvtne2ps_pbh(C, U, A, B);
 }
 
 __m512bh test_mm512_cvtne2ps2bf16(__m512 A, __m512 B) {
-  // CHECK-LABEL: @test_mm512_cvtne2ps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512
-  // CHECK: ret <32 x bfloat> %{{.*}}
+  // CHECK-LABEL: test_mm512_cvtne2ps2bf16
+  // CHECK: call {{.*}}<32 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float> %{{.*}}, <16 x float> %{{.*}})
   return _mm512_cvtne2ps_pbh(A, B);
 }
 
 __m512bh test_mm512_maskz_cvtne2ps2bf16(__m512 A, __m512 B, __mmask32 U) {
-  // CHECK-LABEL: @test_mm512_maskz_cvtne2ps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512
+  // CHECK-LABEL: test_mm512_maskz_cvtne2ps2bf16
+  // CHECK: call {{.*}}<32 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float> %{{.*}}, <16 x float> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}}
-  // CHECK: ret <32 x bfloat> %{{.*}}
   return _mm512_maskz_cvtne2ps_pbh(U, A, B);
 }
 
 __m512bh test_mm512_mask_cvtne2ps2bf16(__m512bh C, __mmask32 U, __m512 A, __m512 B) {
-  // CHECK-LABEL: @test_mm512_mask_cvtne2ps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512
+  // CHECK-LABEL: test_mm512_mask_cvtne2ps2bf16
+  // CHECK: call {{.*}}<32 x bfloat> @llvm.x86.avx512bf16.cvtne2ps2bf16.512(<16 x float> %{{.*}}, <16 x float> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x bfloat> %{{.*}}, <32 x bfloat> %{{.*}}
-  // CHECK: ret <32 x bfloat> %{{.*}}
   return _mm512_mask_cvtne2ps_pbh(C, U, A, B);
 }
 
 __m128bh test_mm_cvtneps2bf16(__m128 A) {
-  // CHECK-LABEL: @test_mm_cvtneps2bf16
-  // CHECK: @llvm.x86.vcvtneps2bf16128
-  // CHECK: ret <8 x bfloat> %{{.*}}
+  // CHECK-LABEL: test_mm_cvtneps2bf16
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.vcvtneps2bf16128(<4 x float> %{{.*}})
   return _mm_cvtneps_pbh(A);
 }
 
 __m128bh test_mm_mask_cvtneps2bf16(__m128bh C, __mmask8 U, __m128 A) {
-  // CHECK-LABEL: @test_mm_mask_cvtneps2bf16
-  // CHECK: @llvm.x86.avx512bf16.mask.cvtneps2bf16.
-  // CHECK: ret <8 x bfloat> %{{.*}}
+  // CHECK-LABEL: test_mm_mask_cvtneps2bf16
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.avx512bf16.mask.cvtneps2bf16.128(<4 x float> %{{.*}}, <8 x bfloat> %{{.*}}, <4 x i1> %{{.*}})
   return _mm_mask_cvtneps_pbh(C, U, A);
 }
 
 __m128bh test_mm_maskz_cvtneps2bf16(__m128 A, __mmask8 U) {
-  // CHECK-LABEL: @test_mm_maskz_cvtneps2bf16
-  // CHECK: @llvm.x86.avx512bf16.mask.cvtneps2bf16.128
-  // CHECK: ret <8 x bfloat> %{{.*}}
+  // CHECK-LABEL: test_mm_maskz_cvtneps2bf16
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.avx512bf16.mask.cvtneps2bf16.128(<4 x float> %{{.*}}, <8 x bfloat> %{{.*}}, <4 x i1> %{{.*}})
   return _mm_maskz_cvtneps_pbh(U, A);
 }
 
 __m128bh test_mm256_cvtneps2bf16(__m256 A) {
-  // CHECK-LABEL: @test_mm256_cvtneps2bf16
-  // CHECK: @llvm.x86.vcvtneps2bf16256
-  // CHECK: ret <8 x bfloat> %{{.*}}
+  // CHECK-LABEL: test_mm256_cvtneps2bf16
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.vcvtneps2bf16256(<8 x float> %{{.*}})
   return _mm256_cvtneps_pbh(A);
 }
 
 __m128bh test_mm256_mask_cvtneps2bf16(__m128bh C, __mmask8 U, __m256 A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtneps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.256
+  // CHECK-LABEL: test_mm256_mask_cvtneps2bf16
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.avx512bf16.cvtneps2bf16.256(<8 x float> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x bfloat> %{{.*}}, <8 x bfloat> %{{.*}}
-  // CHECK: ret <8 x bfloat> %{{.*}}
   return _mm256_mask_cvtneps_pbh(C, U, A);
 }
 
 __m128bh test_mm256_maskz_cvtneps2bf16(__m256 A, __mmask8 U) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtneps2bf16
-  // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.256
+  // CHECK-LABEL: test_mm256_maskz_cvtneps2bf16
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.avx512bf16.cvtneps2bf16.256
   // CHECK: select <8 x i1> %{{.*}}, <8 x bfloat> %{{.*}}, <8 x bfloat> %{{.*}}
-  // CHECK: ret <8 x bfloat> %{{.*}}
   return _mm256_maskz_cvtneps_pbh(U, A);
 }
 
 __m128 test_mm_dpbf16_ps(__m128 D, __m128bh A, __m128bh B) {
-  // CHECK-LABEL: @test_mm_dpbf16_ps
-  // CHECK: @llvm.x86.avx512bf16.dpbf16ps.128
-  // CHECK: ret <4 x float> %{{.*}}
+  // CHECK-LABEL: test_mm_dpbf16_ps
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.avx512bf16.dpbf16ps.128(<4 x float> %{{.*}}, <8 x bfloat> %{{.*}}, <8 x bfloat> %{{.*}})
   return _mm_dpbf16_ps(D, A, B);
 }
 
 __m128 test_mm_maskz_dpbf16_ps(__m128 D, __m128bh A, __m128bh B, __mmask8 U) {
-  // CHECK-LABEL: @test_mm_maskz_dpbf16_ps
-  // CHECK: @llvm.x86.avx512bf16.dpbf16ps.128
+  // CHECK-LABEL: test_mm_maskz_dpbf16_ps
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.avx512bf16.dpbf16ps.128(<4 x float> %{{.*}}, <8 x bfloat> %{{.*}}, <8 x bfloat> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
-  // CHECK: ret <4 x float> %{{.*}}
   return _mm_maskz_dpbf16_ps(U, D, A, B);
 }
 
 __m128 test_mm_mask_dpbf16_ps(__m128 D, __m128bh A, __m128bh B, __mmask8 U) {
-  // CHECK-LABEL: @test_mm_mask_dpbf16_ps
-  // CHECK: @llvm.x86.avx512bf16.dpbf16ps.128
+  // CHECK-LABEL: test_mm_mask_dpbf16_ps
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.avx512bf16.dpbf16ps.128(<4 x float> %{{.*}}, <8 x bfloat> %{{.*}}, <8 x bfloat> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
-  // CHECK: ret <4 x float> %{{.*}}
   return _mm_mask_dpbf16_ps(D, U, A, B);
 }
+
 __m256 test_mm256_dpbf16_ps(__m256 D, __m256bh A, __m256bh B) {
-  // CHECK-LABEL: @test_mm256_dpbf16_ps
-  // CHECK: @llvm.x86.avx512bf16.dpbf16ps.256
-  // CHECK: ret <8 x float> %{{.*}}
+  // CHECK-LABEL: test_mm256_dpbf16_ps
+  // CHECK: call {{.*}}<8 x float> @llvm.x86.avx512bf16.dpbf16ps.256(<8 x float> %{{.*}}, <16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}})
   return _mm256_dpbf16_ps(D, A, B);
 }
 
 __m256 test_mm256_maskz_dpbf16_ps(__m256 D, __m256bh A, __m256bh B, __mmask8 U) {
-  // CHECK-LABEL: @test_mm256_maskz_dpbf16_ps
-  // CHECK: @llvm.x86.avx512bf16.dpbf16ps.256
+  // CHECK-LABEL: test_mm256_maskz_dpbf16_ps
+  // CHECK: call {{.*}}<8 x float> @llvm.x86.avx512bf16.dpbf16ps.256(<8 x float> %{{.*}}, <16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
-  // CHECK: ret <8 x float> %{{.*}}
   return _mm256_maskz_dpbf16_ps(U, D, A, B);
 }
 
 __m256 test_mm256_mask_dpbf16_ps(__m256 D, __m256bh A, __m256bh B, __mmask8 U) {
-  // CHECK-LABEL: @test_mm256_mask_dpbf16_ps
-  // CHECK: @llvm.x86.avx512bf16.dpbf16ps.256
+  // CHECK-LABEL: test_mm256_mask_dpbf16_ps
+  // CHECK: call {{.*}}<8 x float> @llvm.x86.avx512bf16.dpbf16ps.256(<8 x float> %{{.*}}, <16 x bfloat> %{{.*}}, <16 x bfloat> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
-  // CHECK: ret <8 x float> %{{.*}}
   return _mm256_mask_dpbf16_ps(D, U, A, B);
 }
 
 __bf16 test_mm_cvtness_sbh(float A) {
-  // CHECK-LABEL: @test_mm_cvtness_sbh
-  // CHECK: @llvm.x86.avx512bf16.mask.cvtneps2bf16.128
-  // CHECK: ret bfloat %{{.*}}
+  // CHECK-LABEL: test_mm_cvtness_sbh
+  // CHECK: call {{.*}}<8 x bfloat> @llvm.x86.avx512bf16.mask.cvtneps2bf16.128(<4 x float> %{{.*}}, <8 x bfloat> %{{.*}}, <4 x i1> splat (i1 true))
   return _mm_cvtness_sbh(A);
 }
 
 __m128 test_mm_cvtpbh_ps(__m128bh A) {
-  // CHECK-LABEL: @test_mm_cvtpbh_ps
+  // CHECK-LABEL: test_mm_cvtpbh_ps
   // CHECK: sext <4 x i16> %{{.*}} to <4 x i32>
-  // CHECK: @llvm.x86.sse2.pslli.d
-  // CHECK: ret <4 x float> %{{.*}}
+  // CHECK: call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %{{.*}}, i32 %{{.*}})
   return _mm_cvtpbh_ps(A);
 }
 
 __m256 test_mm256_cvtpbh_ps(__m128bh A) {
-  // CHECK-LABEL: @test_mm256_cvtpbh_ps
+  // CHECK-LABEL: test_mm256_cvtpbh_ps
   // CHECK: sext <8 x i16> %{{.*}} to <8 x i32>
-  // CHECK: @llvm.x86.avx2.pslli.d
-  // CHECK: ret <8 x float> %{{.*}}
+  // CHECK: call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %{{.*}}, i32 %{{.*}})
   return _mm256_cvtpbh_ps(A);
 }
 
 __m128 test_mm_maskz_cvtpbh_ps(__mmask8 M, __m128bh A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtpbh_ps
+  // CHECK-LABEL: test_mm_maskz_cvtpbh_ps
   // CHECK: sext <4 x i16> %{{.*}} to <4 x i32>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
-  // CHECK: @llvm.x86.sse2.pslli.d
-  // CHECK: ret <4 x float> %{{.*}}
+  // CHECK: call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %{{.*}}, i32 %{{.*}})
   return _mm_maskz_cvtpbh_ps(M, A);
 }
 
 __m256 test_mm256_maskz_cvtpbh_ps(__mmask8 M, __m128bh A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtpbh_ps
+  // CHECK-LABEL: test_mm256_maskz_cvtpbh_ps
   // CHECK: sext <8 x i16> %{{.*}} to <8 x i32>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
-  // CHECK: @llvm.x86.avx2.pslli.d
-  // CHECK: ret <8 x float> %{{.*}}
+  // CHECK: call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %{{.*}}, i32 %{{.*}})
   return _mm256_maskz_cvtpbh_ps(M, A);
 }
 
 __m128 test_mm_mask_cvtpbh_ps(__m128 S, __mmask8 M, __m128bh A) {
-  // CHECK-LABEL: @test_mm_mask_cvtpbh_ps
+  // CHECK-LABEL: test_mm_mask_cvtpbh_ps
   // CHECK: sext <4 x i16> %{{.*}} to <4 x i32>
-  // CHECK: @llvm.x86.sse2.pslli.d
+  // CHECK: call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %{{.*}}, i32 %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
-  // CHECK: ret <4 x float> %{{.*}}
   return _mm_mask_cvtpbh_ps(S, M, A);
 }
 
 __m256 test_mm256_mask_cvtpbh_ps(__m256 S, __mmask8 M, __m128bh A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtpbh_ps
+  // CHECK-LABEL: test_mm256_mask_cvtpbh_ps
   // CHECK: sext <8 x i16> %{{.*}} to <8 x i32>
-  // CHECK: @llvm.x86.avx2.pslli.d
+  // CHECK: call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %{{.*}}, i32 %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
-  // CHECK: ret <8 x float> %{{.*}}
   return _mm256_mask_cvtpbh_ps(S, M, A);
 }
diff --git a/clang/test/CodeGen/X86/avx512vlbitalg-builtins.c b/clang/test/CodeGen/X86/avx512vlbitalg-builtins.c
index 4e65da02c137f..e2cfb3a348a93 100644
--- a/clang/test/CodeGen/X86/avx512vlbitalg-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vlbitalg-builtins.c
@@ -1,105 +1,125 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bitalg -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bitalg -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512bitalg -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bitalg -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512bitalg -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __m256i test_mm256_popcnt_epi16(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_popcnt_epi16
+  // CHECK-LABEL: test_mm256_popcnt_epi16
   // CHECK: @llvm.ctpop.v16i16
   return _mm256_popcnt_epi16(__A);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_popcnt_epi16((__m256i)(__v16hi){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), 2, 15, 14, 1, 0, 8, 1, 9, 2, 2, 4, 2, 6, 2, 9, 2));
 
 __m256i test_mm256_mask_popcnt_epi16(__m256i __A, __mmask16 __U, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_popcnt_epi16
+  // CHECK-LABEL: test_mm256_mask_popcnt_epi16
   // CHECK: @llvm.ctpop.v16i16
   // CHECK: select <16 x i1> %{{[0-9]+}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_popcnt_epi16(__A, __U, __B);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_mask_popcnt_epi16(_mm256_set1_epi16(-1), 0xF0F0, (__m256i)(__v16hi){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), -1, -1, -1, -1, 0, 8, 1, 9, -1, -1, -1, -1, 6, 2, 9, 2));
+
 __m256i test_mm256_maskz_popcnt_epi16(__mmask16 __U, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_popcnt_epi16
+  // CHECK-LABEL: test_mm256_maskz_popcnt_epi16
   // CHECK: @llvm.ctpop.v16i16
   // CHECK: select <16 x i1> %{{[0-9]+}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_popcnt_epi16(__U, __B);
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_maskz_popcnt_epi16(0x0F0F, (__m256i)(__v16hi){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), 2, 15, 14, 1, 0, 0, 0, 0, 2, 2, 4, 2, 0, 0, 0, 0));
 
 __m128i test_mm_popcnt_epi16(__m128i __A) {
-  // CHECK-LABEL: @test_mm_popcnt_epi16
+  // CHECK-LABEL: test_mm_popcnt_epi16
   // CHECK: @llvm.ctpop.v8i16
   return _mm_popcnt_epi16(__A);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_popcnt_epi16((__m128i)(__v8hi){+5, -3, -10, +8, 0, -256, +256, -128}), 2, 15, 14, 1, 0, 8, 1, 9));
 
 __m128i test_mm_mask_popcnt_epi16(__m128i __A, __mmask8 __U, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_popcnt_epi16
+  // CHECK-LABEL: test_mm_mask_popcnt_epi16
   // CHECK: @llvm.ctpop.v8i16
   // CHECK: select <8 x i1> %{{[0-9]+}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_popcnt_epi16(__A, __U, __B);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_mask_popcnt_epi16(_mm_set1_epi16(-1), 0xF0, (__m128i)(__v8hi){+5, -3, -10, +8, 0, -256, +256, -128}), -1, -1, -1, -1, 0, 8, 1, 9));
+
 __m128i test_mm_maskz_popcnt_epi16(__mmask8 __U, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_popcnt_epi16
+  // CHECK-LABEL: test_mm_maskz_popcnt_epi16
   // CHECK: @llvm.ctpop.v8i16
   // CHECK: select <8 x i1> %{{[0-9]+}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_popcnt_epi16(__U, __B);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_maskz_popcnt_epi16(0x0F, (__m128i)(__v8hi){+5, -3, -10, +8, 0, -256, +256, -128}), 2, 15, 14, 1, 0, 0, 0, 0));
 
 __m256i test_mm256_popcnt_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_popcnt_epi8
+  // CHECK-LABEL: test_mm256_popcnt_epi8
   // CHECK: @llvm.ctpop.v32i8
   return _mm256_popcnt_epi8(__A);
 }
+TEST_CONSTEXPR(match_v32qi(_mm256_popcnt_epi8((__m256i)(__v32qi){+5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73}), 2, 7, 6, 1, 0, 4, 1, 4, 2, 2, 4, 2, 6, 2, 4, 3, 2, 7, 6, 1, 0, 4, 1, 4, 2, 2, 4, 2, 6, 2, 4, 3));
 
 __m256i test_mm256_mask_popcnt_epi8(__m256i __A, __mmask32 __U, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_popcnt_epi8
+  // CHECK-LABEL: test_mm256_mask_popcnt_epi8
   // CHECK: @llvm.ctpop.v32i8
   // CHECK: select <32 x i1> %{{[0-9]+}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_popcnt_epi8(__A, __U, __B);
 }
+TEST_CONSTEXPR(match_v32qi(_mm256_mask_popcnt_epi8(_mm256_set1_epi8(-1), 0xF00F, (__m256i)(__v32qi){+5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73}), 2, 7, 6, 1, -1, -1, -1, -1, -1, -1, -1, -1, 6, 2, 4, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1));
+
 __m256i test_mm256_maskz_popcnt_epi8(__mmask32 __U, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_popcnt_epi8
+  // CHECK-LABEL: test_mm256_maskz_popcnt_epi8
   // CHECK: @llvm.ctpop.v32i8
   // CHECK: select <32 x i1> %{{[0-9]+}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_popcnt_epi8(__U, __B);
 }
+TEST_CONSTEXPR(match_v32qi(_mm256_maskz_popcnt_epi8(0x0FF0, (__m256i)(__v32qi){+5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73, +5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73}), 0, 0, 0, 0, 0, 4, 1, 4, 2, 2, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0));
 
 __m128i test_mm_popcnt_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_popcnt_epi8
+  // CHECK-LABEL: test_mm_popcnt_epi8
   // CHECK: @llvm.ctpop.v16i8
   return _mm_popcnt_epi8(__A);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_popcnt_epi8((__m128i)(__v16qi){+5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73}), 2, 7, 6, 1, 0, 4, 1, 4, 2, 2, 4, 2, 6, 2, 4, 3));
 
 __m128i test_mm_mask_popcnt_epi8(__m128i __A, __mmask16 __U, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_popcnt_epi8
+  // CHECK-LABEL: test_mm_mask_popcnt_epi8
   // CHECK: @llvm.ctpop.v16i8
   // CHECK: select <16 x i1> %{{[0-9]+}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_popcnt_epi8(__A, __U, __B);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_mask_popcnt_epi8(_mm_set1_epi8(-1), 0xF00F, (__m128i)(__v16qi){+5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73}), 2, 7, 6, 1, -1, -1, -1, -1, -1, -1, -1, -1, 6, 2, 4, 3));
+
 __m128i test_mm_maskz_popcnt_epi8(__mmask16 __U, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_popcnt_epi8
+  // CHECK-LABEL: test_mm_maskz_popcnt_epi8
   // CHECK: @llvm.ctpop.v16i8
   // CHECK: select <16 x i1> %{{[0-9]+}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_popcnt_epi8(__U, __B);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_maskz_popcnt_epi8(0x0FF0, (__m128i)(__v16qi){+5, -3, -10, +8, 0, -16, +16, -16, +3, +9, +15, +33, +63, +33, +53, +73}), 0, 0, 0, 0, 0, 4, 1, 4, 2, 2, 4, 2, 0, 0, 0, 0));
 
 __mmask32 test_mm256_mask_bitshuffle_epi64_mask(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_bitshuffle_epi64_mask
+  // CHECK-LABEL: test_mm256_mask_bitshuffle_epi64_mask
   // CHECK: @llvm.x86.avx512.vpshufbitqmb.256
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return _mm256_mask_bitshuffle_epi64_mask(__U, __A, __B);
 }
 
 __mmask32 test_mm256_bitshuffle_epi64_mask(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_bitshuffle_epi64_mask
+  // CHECK-LABEL: test_mm256_bitshuffle_epi64_mask
   // CHECK: @llvm.x86.avx512.vpshufbitqmb.256
   return _mm256_bitshuffle_epi64_mask(__A, __B);
 }
 
 __mmask16 test_mm_mask_bitshuffle_epi64_mask(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_bitshuffle_epi64_mask
+  // CHECK-LABEL: test_mm_mask_bitshuffle_epi64_mask
   // CHECK: @llvm.x86.avx512.vpshufbitqmb.128
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return _mm_mask_bitshuffle_epi64_mask(__U, __A, __B);
 }
 
 __mmask16 test_mm_bitshuffle_epi64_mask(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_bitshuffle_epi64_mask
+  // CHECK-LABEL: test_mm_bitshuffle_epi64_mask
   // CHECK: @llvm.x86.avx512.vpshufbitqmb.128
   return _mm_bitshuffle_epi64_mask(__A, __B);
 }
diff --git a/clang/test/CodeGen/X86/avx512vlbw-builtins.c b/clang/test/CodeGen/X86/avx512vlbw-builtins.c
index 4ec499caabf04..d8a2d1edf8afb 100644
--- a/clang/test/CodeGen/X86/avx512vlbw-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vlbw-builtins.c
@@ -1,732 +1,736 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -target-feature +avx512vl -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx10.1-512 -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -target-feature +avx512vl -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx10.1-512 -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512bw -target-feature +avx512vl -fno-signed-char -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx10.1-512 -emit-llvm -o - -Wall -Werror -Wsign-conversion | FileCheck %s
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __mmask32 test_mm256_cmpeq_epi8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpeq_epi8_mask
+  // CHECK-LABEL: test_mm256_cmpeq_epi8_mask
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmpeq_epi8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmpeq_epi8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpeq_epi8_mask
+  // CHECK-LABEL: test_mm256_mask_cmpeq_epi8_mask
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmpeq_epi8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmpeq_epi8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpeq_epi8_mask
+  // CHECK-LABEL: test_mm_cmpeq_epi8_mask
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmpeq_epi8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmpeq_epi8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpeq_epi8_mask
+  // CHECK-LABEL: test_mm_mask_cmpeq_epi8_mask
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmpeq_epi8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmpeq_epi16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpeq_epi16_mask
+  // CHECK-LABEL: test_mm256_cmpeq_epi16_mask
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmpeq_epi16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmpeq_epi16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpeq_epi16_mask
+  // CHECK-LABEL: test_mm256_mask_cmpeq_epi16_mask
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmpeq_epi16_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpeq_epi16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpeq_epi16_mask
+  // CHECK-LABEL: test_mm_cmpeq_epi16_mask
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpeq_epi16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpeq_epi16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpeq_epi16_mask
+  // CHECK-LABEL: test_mm_mask_cmpeq_epi16_mask
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpeq_epi16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmpgt_epi8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpgt_epi8_mask
+  // CHECK-LABEL: test_mm256_cmpgt_epi8_mask
   // CHECK: icmp sgt <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmpgt_epi8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmpgt_epi8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpgt_epi8_mask
+  // CHECK-LABEL: test_mm256_mask_cmpgt_epi8_mask
   // CHECK: icmp sgt <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmpgt_epi8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmpgt_epi8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpgt_epi8_mask
+  // CHECK-LABEL: test_mm_cmpgt_epi8_mask
   // CHECK: icmp sgt <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmpgt_epi8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmpgt_epi8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpgt_epi8_mask
+  // CHECK-LABEL: test_mm_mask_cmpgt_epi8_mask
   // CHECK: icmp sgt <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmpgt_epi8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmpgt_epi16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpgt_epi16_mask
+  // CHECK-LABEL: test_mm256_cmpgt_epi16_mask
   // CHECK: icmp sgt <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmpgt_epi16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmpgt_epi16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpgt_epi16_mask
+  // CHECK-LABEL: test_mm256_mask_cmpgt_epi16_mask
   // CHECK: icmp sgt <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmpgt_epi16_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpgt_epi16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpgt_epi16_mask
+  // CHECK-LABEL: test_mm_cmpgt_epi16_mask
   // CHECK: icmp sgt <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpgt_epi16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpgt_epi16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpgt_epi16_mask
+  // CHECK-LABEL: test_mm_mask_cmpgt_epi16_mask
   // CHECK: icmp sgt <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpgt_epi16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmpeq_epu8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpeq_epu8_mask
+  // CHECK-LABEL: test_mm_cmpeq_epu8_mask
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmpeq_epu8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmpeq_epu8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpeq_epu8_mask
+  // CHECK-LABEL: test_mm_mask_cmpeq_epu8_mask
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmpeq_epu8_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpeq_epu16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpeq_epu16_mask
+  // CHECK-LABEL: test_mm_cmpeq_epu16_mask
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpeq_epu16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpeq_epu16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpeq_epu16_mask
+  // CHECK-LABEL: test_mm_mask_cmpeq_epu16_mask
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpeq_epu16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmpeq_epu8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpeq_epu8_mask
+  // CHECK-LABEL: test_mm256_cmpeq_epu8_mask
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmpeq_epu8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmpeq_epu8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpeq_epu8_mask
+  // CHECK-LABEL: test_mm256_mask_cmpeq_epu8_mask
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmpeq_epu8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmpeq_epu16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpeq_epu16_mask
+  // CHECK-LABEL: test_mm256_cmpeq_epu16_mask
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmpeq_epu16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmpeq_epu16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpeq_epu16_mask
+  // CHECK-LABEL: test_mm256_mask_cmpeq_epu16_mask
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmpeq_epu16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmpgt_epu8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpgt_epu8_mask
+  // CHECK-LABEL: test_mm_cmpgt_epu8_mask
   // CHECK: icmp ugt <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmpgt_epu8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmpgt_epu8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpgt_epu8_mask
+  // CHECK-LABEL: test_mm_mask_cmpgt_epu8_mask
   // CHECK: icmp ugt <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmpgt_epu8_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpgt_epu16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpgt_epu16_mask
+  // CHECK-LABEL: test_mm_cmpgt_epu16_mask
   // CHECK: icmp ugt <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpgt_epu16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpgt_epu16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpgt_epu16_mask
+  // CHECK-LABEL: test_mm_mask_cmpgt_epu16_mask
   // CHECK: icmp ugt <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpgt_epu16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmpgt_epu8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpgt_epu8_mask
+  // CHECK-LABEL: test_mm256_cmpgt_epu8_mask
   // CHECK: icmp ugt <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmpgt_epu8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmpgt_epu8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpgt_epu8_mask
+  // CHECK-LABEL: test_mm256_mask_cmpgt_epu8_mask
   // CHECK: icmp ugt <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmpgt_epu8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmpgt_epu16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpgt_epu16_mask
+  // CHECK-LABEL: test_mm256_cmpgt_epu16_mask
   // CHECK: icmp ugt <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmpgt_epu16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmpgt_epu16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpgt_epu16_mask
+  // CHECK-LABEL: test_mm256_mask_cmpgt_epu16_mask
   // CHECK: icmp ugt <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmpgt_epu16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmpge_epi8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpge_epi8_mask
+  // CHECK-LABEL: test_mm_cmpge_epi8_mask
   // CHECK: icmp sge <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmpge_epi8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmpge_epi8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpge_epi8_mask
+  // CHECK-LABEL: test_mm_mask_cmpge_epi8_mask
   // CHECK: icmp sge <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmpge_epi8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmpge_epu8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpge_epu8_mask
+  // CHECK-LABEL: test_mm_cmpge_epu8_mask
   // CHECK: icmp uge <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmpge_epu8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmpge_epu8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpge_epu8_mask
+  // CHECK-LABEL: test_mm_mask_cmpge_epu8_mask
   // CHECK: icmp uge <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmpge_epu8_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpge_epi16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpge_epi16_mask
+  // CHECK-LABEL: test_mm_cmpge_epi16_mask
   // CHECK: icmp sge <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpge_epi16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpge_epi16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpge_epi16_mask
+  // CHECK-LABEL: test_mm_mask_cmpge_epi16_mask
   // CHECK: icmp sge <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpge_epi16_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpge_epu16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpge_epu16_mask
+  // CHECK-LABEL: test_mm_cmpge_epu16_mask
   // CHECK: icmp uge <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpge_epu16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpge_epu16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpge_epu16_mask
+  // CHECK-LABEL: test_mm_mask_cmpge_epu16_mask
   // CHECK: icmp uge <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpge_epu16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmpge_epi8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpge_epi8_mask
+  // CHECK-LABEL: test_mm256_cmpge_epi8_mask
   // CHECK: icmp sge <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmpge_epi8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmpge_epi8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpge_epi8_mask
+  // CHECK-LABEL: test_mm256_mask_cmpge_epi8_mask
   // CHECK: icmp sge <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmpge_epi8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmpge_epu8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpge_epu8_mask
+  // CHECK-LABEL: test_mm256_cmpge_epu8_mask
   // CHECK: icmp uge <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmpge_epu8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmpge_epu8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpge_epu8_mask
+  // CHECK-LABEL: test_mm256_mask_cmpge_epu8_mask
   // CHECK: icmp uge <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmpge_epu8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmpge_epi16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpge_epi16_mask
+  // CHECK-LABEL: test_mm256_cmpge_epi16_mask
   // CHECK: icmp sge <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmpge_epi16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmpge_epi16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpge_epi16_mask
+  // CHECK-LABEL: test_mm256_mask_cmpge_epi16_mask
   // CHECK: icmp sge <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmpge_epi16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmpge_epu16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpge_epu16_mask
+  // CHECK-LABEL: test_mm256_cmpge_epu16_mask
   // CHECK: icmp uge <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmpge_epu16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmpge_epu16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpge_epu16_mask
+  // CHECK-LABEL: test_mm256_mask_cmpge_epu16_mask
   // CHECK: icmp uge <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmpge_epu16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmple_epi8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmple_epi8_mask
+  // CHECK-LABEL: test_mm_cmple_epi8_mask
   // CHECK: icmp sle <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmple_epi8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmple_epi8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmple_epi8_mask
+  // CHECK-LABEL: test_mm_mask_cmple_epi8_mask
   // CHECK: icmp sle <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmple_epi8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmple_epu8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmple_epu8_mask
+  // CHECK-LABEL: test_mm_cmple_epu8_mask
   // CHECK: icmp ule <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmple_epu8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmple_epu8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmple_epu8_mask
+  // CHECK-LABEL: test_mm_mask_cmple_epu8_mask
   // CHECK: icmp ule <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmple_epu8_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmple_epi16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmple_epi16_mask
+  // CHECK-LABEL: test_mm_cmple_epi16_mask
   // CHECK: icmp sle <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmple_epi16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmple_epi16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmple_epi16_mask
+  // CHECK-LABEL: test_mm_mask_cmple_epi16_mask
   // CHECK: icmp sle <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmple_epi16_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmple_epu16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmple_epu16_mask
+  // CHECK-LABEL: test_mm_cmple_epu16_mask
   // CHECK: icmp ule <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmple_epu16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmple_epu16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmple_epu16_mask
+  // CHECK-LABEL: test_mm_mask_cmple_epu16_mask
   // CHECK: icmp ule <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmple_epu16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmple_epi8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmple_epi8_mask
+  // CHECK-LABEL: test_mm256_cmple_epi8_mask
   // CHECK: icmp sle <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmple_epi8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmple_epi8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmple_epi8_mask
+  // CHECK-LABEL: test_mm256_mask_cmple_epi8_mask
   // CHECK: icmp sle <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmple_epi8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmple_epu8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmple_epu8_mask
+  // CHECK-LABEL: test_mm256_cmple_epu8_mask
   // CHECK: icmp ule <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmple_epu8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmple_epu8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmple_epu8_mask
+  // CHECK-LABEL: test_mm256_mask_cmple_epu8_mask
   // CHECK: icmp ule <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmple_epu8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmple_epi16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmple_epi16_mask
+  // CHECK-LABEL: test_mm256_cmple_epi16_mask
   // CHECK: icmp sle <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmple_epi16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmple_epi16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmple_epi16_mask
+  // CHECK-LABEL: test_mm256_mask_cmple_epi16_mask
   // CHECK: icmp sle <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmple_epi16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmple_epu16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmple_epu16_mask
+  // CHECK-LABEL: test_mm256_cmple_epu16_mask
   // CHECK: icmp ule <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmple_epu16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmple_epu16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmple_epu16_mask
+  // CHECK-LABEL: test_mm256_mask_cmple_epu16_mask
   // CHECK: icmp ule <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmple_epu16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmplt_epi8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmplt_epi8_mask
+  // CHECK-LABEL: test_mm_cmplt_epi8_mask
   // CHECK: icmp slt <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmplt_epi8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmplt_epi8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmplt_epi8_mask
+  // CHECK-LABEL: test_mm_mask_cmplt_epi8_mask
   // CHECK: icmp slt <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmplt_epi8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmplt_epu8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmplt_epu8_mask
+  // CHECK-LABEL: test_mm_cmplt_epu8_mask
   // CHECK: icmp ult <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmplt_epu8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmplt_epu8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmplt_epu8_mask
+  // CHECK-LABEL: test_mm_mask_cmplt_epu8_mask
   // CHECK: icmp ult <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmplt_epu8_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmplt_epi16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmplt_epi16_mask
+  // CHECK-LABEL: test_mm_cmplt_epi16_mask
   // CHECK: icmp slt <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmplt_epi16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmplt_epi16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmplt_epi16_mask
+  // CHECK-LABEL: test_mm_mask_cmplt_epi16_mask
   // CHECK: icmp slt <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmplt_epi16_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmplt_epu16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmplt_epu16_mask
+  // CHECK-LABEL: test_mm_cmplt_epu16_mask
   // CHECK: icmp ult <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmplt_epu16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmplt_epu16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmplt_epu16_mask
+  // CHECK-LABEL: test_mm_mask_cmplt_epu16_mask
   // CHECK: icmp ult <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmplt_epu16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmplt_epi8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmplt_epi8_mask
+  // CHECK-LABEL: test_mm256_cmplt_epi8_mask
   // CHECK: icmp slt <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmplt_epi8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmplt_epi8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmplt_epi8_mask
+  // CHECK-LABEL: test_mm256_mask_cmplt_epi8_mask
   // CHECK: icmp slt <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmplt_epi8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmplt_epu8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmplt_epu8_mask
+  // CHECK-LABEL: test_mm256_cmplt_epu8_mask
   // CHECK: icmp ult <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmplt_epu8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmplt_epu8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmplt_epu8_mask
+  // CHECK-LABEL: test_mm256_mask_cmplt_epu8_mask
   // CHECK: icmp ult <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmplt_epu8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmplt_epi16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmplt_epi16_mask
+  // CHECK-LABEL: test_mm256_cmplt_epi16_mask
   // CHECK: icmp slt <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmplt_epi16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmplt_epi16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmplt_epi16_mask
+  // CHECK-LABEL: test_mm256_mask_cmplt_epi16_mask
   // CHECK: icmp slt <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmplt_epi16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmplt_epu16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmplt_epu16_mask
+  // CHECK-LABEL: test_mm256_cmplt_epu16_mask
   // CHECK: icmp ult <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmplt_epu16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmplt_epu16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmplt_epu16_mask
+  // CHECK-LABEL: test_mm256_mask_cmplt_epu16_mask
   // CHECK: icmp ult <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmplt_epu16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmpneq_epi8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpneq_epi8_mask
+  // CHECK-LABEL: test_mm_cmpneq_epi8_mask
   // CHECK: icmp ne <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmpneq_epi8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmpneq_epi8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpneq_epi8_mask
+  // CHECK-LABEL: test_mm_mask_cmpneq_epi8_mask
   // CHECK: icmp ne <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmpneq_epi8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmpneq_epu8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpneq_epu8_mask
+  // CHECK-LABEL: test_mm_cmpneq_epu8_mask
   // CHECK: icmp ne <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmpneq_epu8_mask(__a, __b);
 }
 
 __mmask16 test_mm_mask_cmpneq_epu8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpneq_epu8_mask
+  // CHECK-LABEL: test_mm_mask_cmpneq_epu8_mask
   // CHECK: icmp ne <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmpneq_epu8_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpneq_epi16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpneq_epi16_mask
+  // CHECK-LABEL: test_mm_cmpneq_epi16_mask
   // CHECK: icmp ne <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpneq_epi16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpneq_epi16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpneq_epi16_mask
+  // CHECK-LABEL: test_mm_mask_cmpneq_epi16_mask
   // CHECK: icmp ne <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpneq_epi16_mask(__u, __a, __b);
 }
 
 __mmask8 test_mm_cmpneq_epu16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmpneq_epu16_mask
+  // CHECK-LABEL: test_mm_cmpneq_epu16_mask
   // CHECK: icmp ne <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmpneq_epu16_mask(__a, __b);
 }
 
 __mmask8 test_mm_mask_cmpneq_epu16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmpneq_epu16_mask
+  // CHECK-LABEL: test_mm_mask_cmpneq_epu16_mask
   // CHECK: icmp ne <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmpneq_epu16_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmpneq_epi8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpneq_epi8_mask
+  // CHECK-LABEL: test_mm256_cmpneq_epi8_mask
   // CHECK: icmp ne <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmpneq_epi8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmpneq_epi8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpneq_epi8_mask
+  // CHECK-LABEL: test_mm256_mask_cmpneq_epi8_mask
   // CHECK: icmp ne <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmpneq_epi8_mask(__u, __a, __b);
 }
 
 __mmask32 test_mm256_cmpneq_epu8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpneq_epu8_mask
+  // CHECK-LABEL: test_mm256_cmpneq_epu8_mask
   // CHECK: icmp ne <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmpneq_epu8_mask(__a, __b);
 }
 
 __mmask32 test_mm256_mask_cmpneq_epu8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpneq_epu8_mask
+  // CHECK-LABEL: test_mm256_mask_cmpneq_epu8_mask
   // CHECK: icmp ne <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmpneq_epu8_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmpneq_epi16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpneq_epi16_mask
+  // CHECK-LABEL: test_mm256_cmpneq_epi16_mask
   // CHECK: icmp ne <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmpneq_epi16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmpneq_epi16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpneq_epi16_mask
+  // CHECK-LABEL: test_mm256_mask_cmpneq_epi16_mask
   // CHECK: icmp ne <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmpneq_epi16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm256_cmpneq_epu16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmpneq_epu16_mask
+  // CHECK-LABEL: test_mm256_cmpneq_epu16_mask
   // CHECK: icmp ne <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmpneq_epu16_mask(__a, __b);
 }
 
 __mmask16 test_mm256_mask_cmpneq_epu16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmpneq_epu16_mask
+  // CHECK-LABEL: test_mm256_mask_cmpneq_epu16_mask
   // CHECK: icmp ne <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmpneq_epu16_mask(__u, __a, __b);
 }
 
 __mmask16 test_mm_cmp_epi8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmp_epi8_mask
+  // CHECK-LABEL: test_mm_cmp_epi8_mask
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmp_epi8_mask(__a, __b, 0);
 }
 
 __mmask16 test_mm_mask_cmp_epi8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_epi8_mask
+  // CHECK-LABEL: test_mm_mask_cmp_epi8_mask
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmp_epi8_mask(__u, __a, __b, 0);
 }
 
 __mmask16 test_mm_cmp_epu8_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmp_epu8_mask
+  // CHECK-LABEL: test_mm_cmp_epu8_mask
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_cmp_epu8_mask(__a, __b, 0);
 }
 
 __mmask16 test_mm_mask_cmp_epu8_mask(__mmask16 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_epu8_mask
+  // CHECK-LABEL: test_mm_mask_cmp_epu8_mask
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm_mask_cmp_epu8_mask(__u, __a, __b, 0);
 }
 
 __mmask8 test_mm_cmp_epi16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmp_epi16_mask
+  // CHECK-LABEL: test_mm_cmp_epi16_mask
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmp_epi16_mask(__a, __b, 0);
 }
 
 __mmask8 test_mm_mask_cmp_epi16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_epi16_mask
+  // CHECK-LABEL: test_mm_mask_cmp_epi16_mask
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmp_epi16_mask(__u, __a, __b, 0);
 }
 
 __mmask8 test_mm_cmp_epu16_mask(__m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_cmp_epu16_mask
+  // CHECK-LABEL: test_mm_cmp_epu16_mask
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_cmp_epu16_mask(__a, __b, 0);
 }
 
 __mmask8 test_mm_mask_cmp_epu16_mask(__mmask8 __u, __m128i __a, __m128i __b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_epu16_mask
+  // CHECK-LABEL: test_mm_mask_cmp_epu16_mask
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
   return (__mmask8)_mm_mask_cmp_epu16_mask(__u, __a, __b, 0);
 }
 
 __mmask32 test_mm256_cmp_epi8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmp_epi8_mask
+  // CHECK-LABEL: test_mm256_cmp_epi8_mask
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmp_epi8_mask(__a, __b, 0);
 }
 
 __mmask32 test_mm256_mask_cmp_epi8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_epi8_mask
+  // CHECK-LABEL: test_mm256_mask_cmp_epi8_mask
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmp_epi8_mask(__u, __a, __b, 0);
 }
 
 __mmask32 test_mm256_cmp_epu8_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmp_epu8_mask
+  // CHECK-LABEL: test_mm256_cmp_epu8_mask
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_cmp_epu8_mask(__a, __b, 0);
 }
 
 __mmask32 test_mm256_mask_cmp_epu8_mask(__mmask32 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_epu8_mask
+  // CHECK-LABEL: test_mm256_mask_cmp_epu8_mask
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
   return (__mmask32)_mm256_mask_cmp_epu8_mask(__u, __a, __b, 0);
 }
 
 __mmask16 test_mm256_cmp_epi16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmp_epi16_mask
+  // CHECK-LABEL: test_mm256_cmp_epi16_mask
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmp_epi16_mask(__a, __b, 0);
 }
 
 __mmask16 test_mm256_mask_cmp_epi16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_epi16_mask
+  // CHECK-LABEL: test_mm256_mask_cmp_epi16_mask
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmp_epi16_mask(__u, __a, __b, 0);
 }
 
 __mmask16 test_mm256_cmp_epu16_mask(__m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_cmp_epu16_mask
+  // CHECK-LABEL: test_mm256_cmp_epu16_mask
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_cmp_epu16_mask(__a, __b, 0);
 }
 
 __mmask16 test_mm256_mask_cmp_epu16_mask(__mmask16 __u, __m256i __a, __m256i __b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_epu16_mask
+  // CHECK-LABEL: test_mm256_mask_cmp_epu16_mask
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
   return (__mmask16)_mm256_mask_cmp_epu16_mask(__u, __a, __b, 0);
@@ -734,139 +738,139 @@ __mmask16 test_mm256_mask_cmp_epu16_mask(__mmask16 __u, __m256i __a, __m256i __b
 
 
 __m256i test_mm256_mask_add_epi8 (__m256i __W, __mmask32 __U, __m256i __A, __m256i __B){
-  //CHECK-LABEL: @test_mm256_mask_add_epi8
+  //CHECK-LABEL: test_mm256_mask_add_epi8
   //CHECK: add <32 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_add_epi8(__W, __U , __A, __B);
 }
 
 __m256i test_mm256_maskz_add_epi8 (__mmask32 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_add_epi8
+  //CHECK-LABEL: test_mm256_maskz_add_epi8
   //CHECK: add <32 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_add_epi8(__U , __A, __B);
 }
 __m256i test_mm256_mask_add_epi16 (__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_add_epi16
+  //CHECK-LABEL: test_mm256_mask_add_epi16
   //CHECK: add <16 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_add_epi16(__W, __U , __A, __B);
 }
 
 __m256i test_mm256_maskz_add_epi16 (__mmask16 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_add_epi16
+  //CHECK-LABEL: test_mm256_maskz_add_epi16
   //CHECK: add <16 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_add_epi16(__U , __A, __B);
 }
 
 __m256i test_mm256_mask_sub_epi8 (__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_sub_epi8
+  //CHECK-LABEL: test_mm256_mask_sub_epi8
   //CHECK: sub <32 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_sub_epi8(__W, __U , __A, __B);
 }
 
 __m256i test_mm256_maskz_sub_epi8 (__mmask32 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_sub_epi8
+  //CHECK-LABEL: test_mm256_maskz_sub_epi8
   //CHECK: sub <32 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_sub_epi8(__U , __A, __B);
 }
 
 __m256i test_mm256_mask_sub_epi16 (__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_sub_epi16
+  //CHECK-LABEL: test_mm256_mask_sub_epi16
   //CHECK: sub <16 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_sub_epi16(__W, __U , __A, __B);
 }
 
 __m256i test_mm256_maskz_sub_epi16 (__mmask16 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_sub_epi16
+  //CHECK-LABEL: test_mm256_maskz_sub_epi16
   //CHECK: sub <16 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_sub_epi16(__U , __A, __B);
 }
 
 __m128i test_mm_mask_add_epi8 (__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_add_epi8
+  //CHECK-LABEL: test_mm_mask_add_epi8
   //CHECK: add <16 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_add_epi8(__W, __U , __A, __B);
 }
 
 __m128i test_mm_maskz_add_epi8 (__mmask16 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_add_epi8
+  //CHECK-LABEL: test_mm_maskz_add_epi8
   //CHECK: add <16 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_add_epi8(__U , __A, __B);
 }
 
 __m128i test_mm_mask_add_epi16 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_add_epi16
+  //CHECK-LABEL: test_mm_mask_add_epi16
   //CHECK: add <8 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_add_epi16(__W, __U , __A, __B);
 }
 
 __m128i test_mm_maskz_add_epi16 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_add_epi16
+  //CHECK-LABEL: test_mm_maskz_add_epi16
   //CHECK: add <8 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_add_epi16(__U , __A, __B);
 }
 
 __m128i test_mm_mask_sub_epi8 (__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_sub_epi8
+  //CHECK-LABEL: test_mm_mask_sub_epi8
   //CHECK: sub <16 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_sub_epi8(__W, __U , __A, __B);
 }
 
 __m128i test_mm_maskz_sub_epi8 (__mmask16 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_sub_epi8
+  //CHECK-LABEL: test_mm_maskz_sub_epi8
   //CHECK: sub <16 x i8> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_sub_epi8(__U , __A, __B);
 }
 
 __m128i test_mm_mask_sub_epi16 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_sub_epi16
+  //CHECK-LABEL: test_mm_mask_sub_epi16
   //CHECK: sub <8 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_sub_epi16(__W, __U , __A, __B);
 }
 
 __m128i test_mm_maskz_sub_epi16 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_sub_epi16
+  //CHECK-LABEL: test_mm_maskz_sub_epi16
   //CHECK: sub <8 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_sub_epi16(__U , __A, __B);
 }
 
 __m256i test_mm256_mask_mullo_epi16 (__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_mask_mullo_epi16
+  //CHECK-LABEL: test_mm256_mask_mullo_epi16
   //CHECK: mul <16 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_mullo_epi16(__W, __U , __A, __B);
 }
 
 __m256i test_mm256_maskz_mullo_epi16 (__mmask16 __U, __m256i __A, __m256i __B) {
-  //CHECK-LABEL: @test_mm256_maskz_mullo_epi16
+  //CHECK-LABEL: test_mm256_maskz_mullo_epi16
   //CHECK: mul <16 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_mullo_epi16(__U , __A, __B);
 }
 
 __m128i test_mm_mask_mullo_epi16 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_mask_mullo_epi16
+  //CHECK-LABEL: test_mm_mask_mullo_epi16
   //CHECK: mul <8 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_mullo_epi16(__W, __U , __A, __B);
 }
 
 __m128i test_mm_maskz_mullo_epi16 (__mmask8 __U, __m128i __A, __m128i __B) {
-  //CHECK-LABEL: @test_mm_maskz_mullo_epi16
+  //CHECK-LABEL: test_mm_maskz_mullo_epi16
   //CHECK: mul <8 x i16> %{{.*}}, %{{.*}}
   //CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_mullo_epi16(__U , __A, __B);
@@ -874,30 +878,30 @@ __m128i test_mm_maskz_mullo_epi16 (__mmask8 __U, __m128i __A, __m128i __B) {
 
 
 __m128i test_mm_mask_blend_epi8(__mmask16 __U, __m128i __A, __m128i __W) {
-  // CHECK-LABEL: @test_mm_mask_blend_epi8
+  // CHECK-LABEL: test_mm_mask_blend_epi8
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_blend_epi8(__U,__A,__W); 
 }
 __m256i test_mm256_mask_blend_epi8(__mmask32 __U, __m256i __A, __m256i __W) {
-  // CHECK-LABEL: @test_mm256_mask_blend_epi8
+  // CHECK-LABEL: test_mm256_mask_blend_epi8
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_blend_epi8(__U,__A,__W); 
 }
 
 __m128i test_mm_mask_blend_epi16(__mmask8 __U, __m128i __A, __m128i __W) {
-  // CHECK-LABEL: @test_mm_mask_blend_epi16
+  // CHECK-LABEL: test_mm_mask_blend_epi16
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_blend_epi16(__U,__A,__W); 
 }
 
 __m256i test_mm256_mask_blend_epi16(__mmask16 __U, __m256i __A, __m256i __W) {
-  // CHECK-LABEL: @test_mm256_mask_blend_epi16
+  // CHECK-LABEL: test_mm256_mask_blend_epi16
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_blend_epi16(__U,__A,__W); 
 }
 
 __m128i test_mm_mask_abs_epi8(__m128i __W, __mmask16 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_abs_epi8
+  // CHECK-LABEL: test_mm_mask_abs_epi8
   // CHECK: [[ABS:%.*]] = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <16 x i8> [[ABS]] to <2 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <2 x i64> [[TMP]] to <16 x i8>
@@ -906,7 +910,7 @@ __m128i test_mm_mask_abs_epi8(__m128i __W, __mmask16 __U, __m128i __A) {
 }
 
 __m128i test_mm_maskz_abs_epi8(__mmask16 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_abs_epi8
+  // CHECK-LABEL: test_mm_maskz_abs_epi8
   // CHECK: [[ABS:%.*]] = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <16 x i8> [[ABS]] to <2 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <2 x i64> [[TMP]] to <16 x i8>
@@ -915,7 +919,7 @@ __m128i test_mm_maskz_abs_epi8(__mmask16 __U, __m128i __A) {
 }
 
 __m256i test_mm256_mask_abs_epi8(__m256i __W, __mmask32 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_abs_epi8
+  // CHECK-LABEL: test_mm256_mask_abs_epi8
   // CHECK: [[ABS:%.*]] = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <32 x i8> [[ABS]] to <4 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <4 x i64> [[TMP]] to <32 x i8>
@@ -924,7 +928,7 @@ __m256i test_mm256_mask_abs_epi8(__m256i __W, __mmask32 __U, __m256i __A) {
 }
 
 __m256i test_mm256_maskz_abs_epi8(__mmask32 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_abs_epi8
+  // CHECK-LABEL: test_mm256_maskz_abs_epi8
   // CHECK: [[ABS:%.*]] = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <32 x i8> [[ABS]] to <4 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <4 x i64> [[TMP]] to <32 x i8>
@@ -933,7 +937,7 @@ __m256i test_mm256_maskz_abs_epi8(__mmask32 __U, __m256i __A) {
 }
 
 __m128i test_mm_mask_abs_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_abs_epi16
+  // CHECK-LABEL: test_mm_mask_abs_epi16
   // CHECK: [[ABS:%.*]] = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <8 x i16> [[ABS]] to <2 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <2 x i64> [[TMP]] to <8 x i16>
@@ -942,7 +946,7 @@ __m128i test_mm_mask_abs_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
 }
 
 __m128i test_mm_maskz_abs_epi16(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_abs_epi16
+  // CHECK-LABEL: test_mm_maskz_abs_epi16
   // CHECK: [[ABS:%.*]] = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <8 x i16> [[ABS]] to <2 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <2 x i64> [[TMP]] to <8 x i16>
@@ -951,7 +955,7 @@ __m128i test_mm_maskz_abs_epi16(__mmask8 __U, __m128i __A) {
 }
 
 __m256i test_mm256_mask_abs_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_abs_epi16
+  // CHECK-LABEL: test_mm256_mask_abs_epi16
   // CHECK: [[ABS:%.*]] = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <16 x i16> [[ABS]] to <4 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <4 x i64> [[TMP]] to <16 x i16>
@@ -960,7 +964,7 @@ __m256i test_mm256_mask_abs_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
 }
 
 __m256i test_mm256_maskz_abs_epi16(__mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_abs_epi16
+  // CHECK-LABEL: test_mm256_maskz_abs_epi16
   // CHECK: [[ABS:%.*]] = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %{{.*}}, i1 false)
   // CHECK: [[TMP:%.*]] = bitcast <16 x i16> [[ABS]] to <4 x i64>
   // CHECK: [[ABS:%.*]] = bitcast <4 x i64> [[TMP]] to <16 x i16>
@@ -969,264 +973,264 @@ __m256i test_mm256_maskz_abs_epi16(__mmask16 __U, __m256i __A) {
 }
 
 __m128i test_mm_maskz_packs_epi32(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_packs_epi32
+  // CHECK-LABEL: test_mm_maskz_packs_epi32
   // CHECK: @llvm.x86.sse2.packssdw
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_packs_epi32(__M,__A,__B); 
 }
 __m128i test_mm_mask_packs_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_packs_epi32
+  // CHECK-LABEL: test_mm_mask_packs_epi32
   // CHECK: @llvm.x86.sse2.packssdw
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_packs_epi32(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_packs_epi32(__mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_packs_epi32
+  // CHECK-LABEL: test_mm256_maskz_packs_epi32
   // CHECK: @llvm.x86.avx2.packssdw
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_packs_epi32(__M,__A,__B); 
 }
 __m256i test_mm256_mask_packs_epi32(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_packs_epi32
+  // CHECK-LABEL: test_mm256_mask_packs_epi32
   // CHECK: @llvm.x86.avx2.packssdw
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_packs_epi32(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_packs_epi16(__mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_packs_epi16
+  // CHECK-LABEL: test_mm_maskz_packs_epi16
   // CHECK: @llvm.x86.sse2.packsswb
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_packs_epi16(__M,__A,__B); 
 }
 __m128i test_mm_mask_packs_epi16(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_packs_epi16
+  // CHECK-LABEL: test_mm_mask_packs_epi16
   // CHECK: @llvm.x86.sse2.packsswb
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_packs_epi16(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_packs_epi16(__mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_packs_epi16
+  // CHECK-LABEL: test_mm256_maskz_packs_epi16
   // CHECK: @llvm.x86.avx2.packsswb
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_packs_epi16(__M,__A,__B); 
 }
 __m256i test_mm256_mask_packs_epi16(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_packs_epi16
+  // CHECK-LABEL: test_mm256_mask_packs_epi16
   // CHECK: @llvm.x86.avx2.packsswb
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_packs_epi16(__W,__M,__A,__B); 
 }
 
 __m128i test_mm_mask_packus_epi32(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_packus_epi32
+  // CHECK-LABEL: test_mm_mask_packus_epi32
   // CHECK: @llvm.x86.sse41.packusdw
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_packus_epi32(__W,__M,__A,__B); 
 }
 
 __m128i test_mm_maskz_packus_epi32(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_packus_epi32
+  // CHECK-LABEL: test_mm_maskz_packus_epi32
   // CHECK: @llvm.x86.sse41.packusdw
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_packus_epi32(__M,__A,__B); 
 }
 
 __m256i test_mm256_maskz_packus_epi32(__mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_packus_epi32
+  // CHECK-LABEL: test_mm256_maskz_packus_epi32
   // CHECK: @llvm.x86.avx2.packusdw
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_packus_epi32(__M,__A,__B); 
 }
 
 __m256i test_mm256_mask_packus_epi32(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_packus_epi32
+  // CHECK-LABEL: test_mm256_mask_packus_epi32
   // CHECK: @llvm.x86.avx2.packusdw
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_packus_epi32(__W,__M,__A,__B); 
 }
 
 __m128i test_mm_maskz_packus_epi16(__mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_packus_epi16
+  // CHECK-LABEL: test_mm_maskz_packus_epi16
   // CHECK: @llvm.x86.sse2.packuswb
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_packus_epi16(__M,__A,__B); 
 }
 
 __m128i test_mm_mask_packus_epi16(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_packus_epi16
+  // CHECK-LABEL: test_mm_mask_packus_epi16
   // CHECK: @llvm.x86.sse2.packuswb
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_packus_epi16(__W,__M,__A,__B); 
 }
 
 __m256i test_mm256_maskz_packus_epi16(__mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_packus_epi16
+  // CHECK-LABEL: test_mm256_maskz_packus_epi16
   // CHECK: @llvm.x86.avx2.packuswb
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_packus_epi16(__M,__A,__B); 
 }
 
 __m256i test_mm256_mask_packus_epi16(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_packus_epi16
+  // CHECK-LABEL: test_mm256_mask_packus_epi16
   // CHECK: @llvm.x86.avx2.packuswb
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_packus_epi16(__W,__M,__A,__B); 
 }
 
 __m128i test_mm_mask_adds_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_adds_epi8
+  // CHECK-LABEL: test_mm_mask_adds_epi8
   // CHECK: @llvm.sadd.sat.v16i8
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_adds_epi8(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_adds_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_adds_epi8
+  // CHECK-LABEL: test_mm_maskz_adds_epi8
   // CHECK: @llvm.sadd.sat.v16i8
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_adds_epi8(__U,__A,__B); 
 }
 __m256i test_mm256_mask_adds_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_adds_epi8
+  // CHECK-LABEL: test_mm256_mask_adds_epi8
   // CHECK: @llvm.sadd.sat.v32i8
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_adds_epi8(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_adds_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_adds_epi8
+  // CHECK-LABEL: test_mm256_maskz_adds_epi8
   // CHECK: @llvm.sadd.sat.v32i8
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_adds_epi8(__U,__A,__B); 
 }
 __m128i test_mm_mask_adds_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_adds_epi16
+  // CHECK-LABEL: test_mm_mask_adds_epi16
   // CHECK: @llvm.sadd.sat.v8i16
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_adds_epi16(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_adds_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_adds_epi16
+  // CHECK-LABEL: test_mm_maskz_adds_epi16
   // CHECK: @llvm.sadd.sat.v8i16
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_adds_epi16(__U,__A,__B); 
 }
 __m256i test_mm256_mask_adds_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_adds_epi16
+  // CHECK-LABEL: test_mm256_mask_adds_epi16
   // CHECK: @llvm.sadd.sat.v16i16
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_adds_epi16(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_adds_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_adds_epi16
+  // CHECK-LABEL: test_mm256_maskz_adds_epi16
   // CHECK: @llvm.sadd.sat.v16i16
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_adds_epi16(__U,__A,__B); 
 }
 __m128i test_mm_mask_adds_epu8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_adds_epu8
+  // CHECK-LABEL: test_mm_mask_adds_epu8
   // CHECK-NOT: @llvm.x86.sse2.paddus.b
   // CHECK: call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_adds_epu8(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_adds_epu8(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_adds_epu8
+  // CHECK-LABEL: test_mm_maskz_adds_epu8
   // CHECK-NOT: @llvm.x86.sse2.paddus.b
   // CHECK: call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_adds_epu8(__U,__A,__B); 
 }
 __m256i test_mm256_mask_adds_epu8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_adds_epu8
+  // CHECK-LABEL: test_mm256_mask_adds_epu8
   // CHECK-NOT: @llvm.x86.avx2.paddus.b
   // CHECK: call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_adds_epu8(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_adds_epu8(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_adds_epu8
+  // CHECK-LABEL: test_mm256_maskz_adds_epu8
   // CHECK-NOT: @llvm.x86.avx2.paddus.b
   // CHECK: call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_adds_epu8(__U,__A,__B); 
 }
 __m128i test_mm_mask_adds_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_adds_epu16
+  // CHECK-LABEL: test_mm_mask_adds_epu16
   // CHECK-NOT: @llvm.x86.sse2.paddus.w
   // CHECK: call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_adds_epu16(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_adds_epu16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_adds_epu16
+  // CHECK-LABEL: test_mm_maskz_adds_epu16
   // CHECK-NOT: @llvm.x86.sse2.paddus.w
   // CHECK: call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_adds_epu16(__U,__A,__B); 
 }
 __m256i test_mm256_mask_adds_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_adds_epu16
+  // CHECK-LABEL: test_mm256_mask_adds_epu16
   // CHECK-NOT: @llvm.x86.avx2.paddus.w
   // CHECK: call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_adds_epu16(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_adds_epu16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_adds_epu16
+  // CHECK-LABEL: test_mm256_maskz_adds_epu16
   // CHECK-NOT: @llvm.x86.avx2.paddus.w
   // CHECK: call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_adds_epu16(__U,__A,__B); 
 }
 __m128i test_mm_mask_avg_epu8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_avg_epu8
+  // CHECK-LABEL: test_mm_mask_avg_epu8
   // CHECK: @llvm.x86.sse2.pavg.b
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_avg_epu8(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_avg_epu8(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_avg_epu8
+  // CHECK-LABEL: test_mm_maskz_avg_epu8
   // CHECK: @llvm.x86.sse2.pavg.b
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_avg_epu8(__U,__A,__B); 
 }
 __m256i test_mm256_mask_avg_epu8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_avg_epu8
+  // CHECK-LABEL: test_mm256_mask_avg_epu8
   // CHECK: @llvm.x86.avx2.pavg.b
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_avg_epu8(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_avg_epu8(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_avg_epu8
+  // CHECK-LABEL: test_mm256_maskz_avg_epu8
   // CHECK: @llvm.x86.avx2.pavg.b
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_avg_epu8(__U,__A,__B); 
 }
 __m128i test_mm_mask_avg_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_avg_epu16
+  // CHECK-LABEL: test_mm_mask_avg_epu16
   // CHECK: @llvm.x86.sse2.pavg.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_avg_epu16(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_avg_epu16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_avg_epu16
+  // CHECK-LABEL: test_mm_maskz_avg_epu16
   // CHECK: @llvm.x86.sse2.pavg.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_avg_epu16(__U,__A,__B); 
 }
 __m256i test_mm256_mask_avg_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_avg_epu16
+  // CHECK-LABEL: test_mm256_mask_avg_epu16
   // CHECK: @llvm.x86.avx2.pavg.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_avg_epu16(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_avg_epu16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_avg_epu16
+  // CHECK-LABEL: test_mm256_maskz_avg_epu16
   // CHECK: @llvm.x86.avx2.pavg.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_avg_epu16(__U,__A,__B); 
 }
 __m128i test_mm_maskz_max_epi8(__mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_epi8
+  // CHECK-LABEL: test_mm_maskz_max_epi8
   // CHECK: [[RES:%.*]] = call <16 x i8> @llvm.smax.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i8>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1234,7 +1238,7 @@ __m128i test_mm_maskz_max_epi8(__mmask16 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_max_epi8(__M,__A,__B); 
 }
 __m128i test_mm_mask_max_epi8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_max_epi8
+  // CHECK-LABEL: test_mm_mask_max_epi8
   // CHECK: [[RES:%.*]] = call <16 x i8> @llvm.smax.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i8>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1242,7 +1246,7 @@ __m128i test_mm_mask_max_epi8(__m128i __W, __mmask16 __M, __m128i __A, __m128i _
   return _mm_mask_max_epi8(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_max_epi8(__mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_epi8
+  // CHECK-LABEL: test_mm256_maskz_max_epi8
   // CHECK: [[RES:%.*]] = call <32 x i8> @llvm.smax.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<32 x i8>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1250,7 +1254,7 @@ __m256i test_mm256_maskz_max_epi8(__mmask32 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_max_epi8(__M,__A,__B); 
 }
 __m256i test_mm256_mask_max_epi8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_epi8
+  // CHECK-LABEL: test_mm256_mask_max_epi8
   // CHECK: [[RES:%.*]] = call <32 x i8> @llvm.smax.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<32 x i8>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1258,7 +1262,7 @@ __m256i test_mm256_mask_max_epi8(__m256i __W, __mmask32 __M, __m256i __A, __m256
   return _mm256_mask_max_epi8(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_max_epi16(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_epi16
+  // CHECK-LABEL: test_mm_maskz_max_epi16
   // CHECK: [[RES:%.*]] = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<8 x i16>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1266,7 +1270,7 @@ __m128i test_mm_maskz_max_epi16(__mmask8 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_max_epi16(__M,__A,__B); 
 }
 __m128i test_mm_mask_max_epi16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_max_epi16
+  // CHECK-LABEL: test_mm_mask_max_epi16
   // CHECK: [[RES:%.*]] = call <8 x i16> @llvm.smax.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<8 x i16>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1274,7 +1278,7 @@ __m128i test_mm_mask_max_epi16(__m128i __W, __mmask8 __M, __m128i __A, __m128i _
   return _mm_mask_max_epi16(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_max_epi16(__mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_epi16
+  // CHECK-LABEL: test_mm256_maskz_max_epi16
   // CHECK: [[RES:%.*]] = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i16>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1282,7 +1286,7 @@ __m256i test_mm256_maskz_max_epi16(__mmask16 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_max_epi16(__M,__A,__B); 
 }
 __m256i test_mm256_mask_max_epi16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_epi16
+  // CHECK-LABEL: test_mm256_mask_max_epi16
   // CHECK: [[RES:%.*]] = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i16>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1290,7 +1294,7 @@ __m256i test_mm256_mask_max_epi16(__m256i __W, __mmask16 __M, __m256i __A, __m25
   return _mm256_mask_max_epi16(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_max_epu8(__mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_epu8
+  // CHECK-LABEL: test_mm_maskz_max_epu8
   // CHECK: [[RES:%.*]] = call <16 x i8> @llvm.umax.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i8>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1298,7 +1302,7 @@ __m128i test_mm_maskz_max_epu8(__mmask16 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_max_epu8(__M,__A,__B); 
 }
 __m128i test_mm_mask_max_epu8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_max_epu8
+  // CHECK-LABEL: test_mm_mask_max_epu8
   // CHECK: [[RES:%.*]] = call <16 x i8> @llvm.umax.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i8>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1306,7 +1310,7 @@ __m128i test_mm_mask_max_epu8(__m128i __W, __mmask16 __M, __m128i __A, __m128i _
   return _mm_mask_max_epu8(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_max_epu8(__mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_epu8
+  // CHECK-LABEL: test_mm256_maskz_max_epu8
   // CHECK: [[RES:%.*]] = call <32 x i8> @llvm.umax.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<32 x i8>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1314,7 +1318,7 @@ __m256i test_mm256_maskz_max_epu8(__mmask32 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_max_epu8(__M,__A,__B); 
 }
 __m256i test_mm256_mask_max_epu8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_epu8
+  // CHECK-LABEL: test_mm256_mask_max_epu8
   // CHECK: [[RES:%.*]] = call <32 x i8> @llvm.umax.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<32 x i8>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1322,7 +1326,7 @@ __m256i test_mm256_mask_max_epu8(__m256i __W, __mmask32 __M, __m256i __A, __m256
   return _mm256_mask_max_epu8(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_max_epu16(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_epu16
+  // CHECK-LABEL: test_mm_maskz_max_epu16
   // CHECK: [[RES:%.*]] = call <8 x i16> @llvm.umax.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<8 x i16>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1330,7 +1334,7 @@ __m128i test_mm_maskz_max_epu16(__mmask8 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_max_epu16(__M,__A,__B); 
 }
 __m128i test_mm_mask_max_epu16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_max_epu16
+  // CHECK-LABEL: test_mm_mask_max_epu16
   // CHECK: [[RES:%.*]] = call <8 x i16> @llvm.umax.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<8 x i16>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1338,7 +1342,7 @@ __m128i test_mm_mask_max_epu16(__m128i __W, __mmask8 __M, __m128i __A, __m128i _
   return _mm_mask_max_epu16(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_max_epu16(__mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_epu16
+  // CHECK-LABEL: test_mm256_maskz_max_epu16
   // CHECK: [[RES:%.*]] = call <16 x i16> @llvm.umax.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i16>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1346,7 +1350,7 @@ __m256i test_mm256_maskz_max_epu16(__mmask16 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_max_epu16(__M,__A,__B); 
 }
 __m256i test_mm256_mask_max_epu16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_epu16
+  // CHECK-LABEL: test_mm256_mask_max_epu16
   // CHECK: [[RES:%.*]] = call <16 x i16> @llvm.umax.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i16>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1354,7 +1358,7 @@ __m256i test_mm256_mask_max_epu16(__m256i __W, __mmask16 __M, __m256i __A, __m25
   return _mm256_mask_max_epu16(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_min_epi8(__mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_epi8
+  // CHECK-LABEL: test_mm_maskz_min_epi8
   // CHECK: [[RES:%.*]] = call <16 x i8> @llvm.smin.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i8>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1362,7 +1366,7 @@ __m128i test_mm_maskz_min_epi8(__mmask16 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_min_epi8(__M,__A,__B); 
 }
 __m128i test_mm_mask_min_epi8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_min_epi8
+  // CHECK-LABEL: test_mm_mask_min_epi8
   // CHECK: [[RES:%.*]] = call <16 x i8> @llvm.smin.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i8>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1370,7 +1374,7 @@ __m128i test_mm_mask_min_epi8(__m128i __W, __mmask16 __M, __m128i __A, __m128i _
   return _mm_mask_min_epi8(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_min_epi8(__mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_epi8
+  // CHECK-LABEL: test_mm256_maskz_min_epi8
   // CHECK: [[RES:%.*]] = call <32 x i8> @llvm.smin.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<32 x i8>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1378,7 +1382,7 @@ __m256i test_mm256_maskz_min_epi8(__mmask32 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_min_epi8(__M,__A,__B); 
 }
 __m256i test_mm256_mask_min_epi8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_epi8
+  // CHECK-LABEL: test_mm256_mask_min_epi8
   // CHECK: [[RES:%.*]] = call <32 x i8> @llvm.smin.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<32 x i8>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1386,7 +1390,7 @@ __m256i test_mm256_mask_min_epi8(__m256i __W, __mmask32 __M, __m256i __A, __m256
   return _mm256_mask_min_epi8(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_min_epi16(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_epi16
+  // CHECK-LABEL: test_mm_maskz_min_epi16
   // CHECK: [[RES:%.*]] = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<8 x i16>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1394,7 +1398,7 @@ __m128i test_mm_maskz_min_epi16(__mmask8 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_min_epi16(__M,__A,__B); 
 }
 __m128i test_mm_mask_min_epi16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_min_epi16
+  // CHECK-LABEL: test_mm_mask_min_epi16
   // CHECK: [[RES:%.*]] = call <8 x i16> @llvm.smin.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<8 x i16>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1402,7 +1406,7 @@ __m128i test_mm_mask_min_epi16(__m128i __W, __mmask8 __M, __m128i __A, __m128i _
   return _mm_mask_min_epi16(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_min_epi16(__mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_epi16
+  // CHECK-LABEL: test_mm256_maskz_min_epi16
   // CHECK: [[RES:%.*]] = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i16>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1410,7 +1414,7 @@ __m256i test_mm256_maskz_min_epi16(__mmask16 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_min_epi16(__M,__A,__B); 
 }
 __m256i test_mm256_mask_min_epi16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_epi16
+  // CHECK-LABEL: test_mm256_mask_min_epi16
   // CHECK: [[RES:%.*]] = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i16>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1418,7 +1422,7 @@ __m256i test_mm256_mask_min_epi16(__m256i __W, __mmask16 __M, __m256i __A, __m25
   return _mm256_mask_min_epi16(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_min_epu8(__mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_epu8
+  // CHECK-LABEL: test_mm_maskz_min_epu8
   // CHECK: [[RES:%.*]] = call <16 x i8> @llvm.umin.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i8>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1426,7 +1430,7 @@ __m128i test_mm_maskz_min_epu8(__mmask16 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_min_epu8(__M,__A,__B); 
 }
 __m128i test_mm_mask_min_epu8(__m128i __W, __mmask16 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_min_epu8
+  // CHECK-LABEL: test_mm_mask_min_epu8
   // CHECK: [[RES:%.*]] = call <16 x i8> @llvm.umin.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i8>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1434,7 +1438,7 @@ __m128i test_mm_mask_min_epu8(__m128i __W, __mmask16 __M, __m128i __A, __m128i _
   return _mm_mask_min_epu8(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_min_epu8(__mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_epu8
+  // CHECK-LABEL: test_mm256_maskz_min_epu8
   // CHECK: [[RES:%.*]] = call <32 x i8> @llvm.umin.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<32 x i8>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1442,7 +1446,7 @@ __m256i test_mm256_maskz_min_epu8(__mmask32 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_min_epu8(__M,__A,__B); 
 }
 __m256i test_mm256_mask_min_epu8(__m256i __W, __mmask32 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_epu8
+  // CHECK-LABEL: test_mm256_mask_min_epu8
   // CHECK: [[RES:%.*]] = call <32 x i8> @llvm.umin.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<32 x i8>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1450,7 +1454,7 @@ __m256i test_mm256_mask_min_epu8(__m256i __W, __mmask32 __M, __m256i __A, __m256
   return _mm256_mask_min_epu8(__W,__M,__A,__B); 
 }
 __m128i test_mm_maskz_min_epu16(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_epu16
+  // CHECK-LABEL: test_mm_maskz_min_epu16
   // CHECK: [[RES:%.*]] = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<8 x i16>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1458,7 +1462,7 @@ __m128i test_mm_maskz_min_epu16(__mmask8 __M, __m128i __A, __m128i __B) {
   return _mm_maskz_min_epu16(__M,__A,__B); 
 }
 __m128i test_mm_mask_min_epu16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_min_epu16
+  // CHECK-LABEL: test_mm_mask_min_epu16
   // CHECK: [[RES:%.*]] = call <8 x i16> @llvm.umin.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<8 x i16>]] [[RES]] to [[DSTTY:<2 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1466,7 +1470,7 @@ __m128i test_mm_mask_min_epu16(__m128i __W, __mmask8 __M, __m128i __A, __m128i _
   return _mm_mask_min_epu16(__W,__M,__A,__B); 
 }
 __m256i test_mm256_maskz_min_epu16(__mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_epu16
+  // CHECK-LABEL: test_mm256_maskz_min_epu16
   // CHECK: [[RES:%.*]] = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i16>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1474,7 +1478,7 @@ __m256i test_mm256_maskz_min_epu16(__mmask16 __M, __m256i __A, __m256i __B) {
   return _mm256_maskz_min_epu16(__M,__A,__B); 
 }
 __m256i test_mm256_mask_min_epu16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_epu16
+  // CHECK-LABEL: test_mm256_mask_min_epu16
   // CHECK: [[RES:%.*]] = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: [[TMP:%.*]] = bitcast [[SRCTY:<16 x i16>]] [[RES]] to [[DSTTY:<4 x i64>]]
   // CHECK: [[RES:%.*]] = bitcast [[DSTTY]] [[TMP]] to [[SRCTY]]
@@ -1482,128 +1486,128 @@ __m256i test_mm256_mask_min_epu16(__m256i __W, __mmask16 __M, __m256i __A, __m25
   return _mm256_mask_min_epu16(__W,__M,__A,__B); 
 }
 __m128i test_mm_mask_shuffle_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shuffle_epi8
+  // CHECK-LABEL: test_mm_mask_shuffle_epi8
   // CHECK: @llvm.x86.ssse3.pshuf.b
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_shuffle_epi8(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_shuffle_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shuffle_epi8
+  // CHECK-LABEL: test_mm_maskz_shuffle_epi8
   // CHECK: @llvm.x86.ssse3.pshuf.b
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_shuffle_epi8(__U,__A,__B); 
 }
 __m256i test_mm256_mask_shuffle_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shuffle_epi8
+  // CHECK-LABEL: test_mm256_mask_shuffle_epi8
   // CHECK: @llvm.x86.avx2.pshuf.b
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_shuffle_epi8(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_shuffle_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shuffle_epi8
+  // CHECK-LABEL: test_mm256_maskz_shuffle_epi8
   // CHECK: @llvm.x86.avx2.pshuf.b
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_shuffle_epi8(__U,__A,__B); 
 }
 __m128i test_mm_mask_subs_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_subs_epi8
+  // CHECK-LABEL: test_mm_mask_subs_epi8
   // CHECK: @llvm.ssub.sat.v16i8
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_subs_epi8(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_subs_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_subs_epi8
+  // CHECK-LABEL: test_mm_maskz_subs_epi8
   // CHECK: @llvm.ssub.sat.v16i8
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_subs_epi8(__U,__A,__B); 
 }
 __m256i test_mm256_mask_subs_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_subs_epi8
+  // CHECK-LABEL: test_mm256_mask_subs_epi8
   // CHECK: @llvm.ssub.sat.v32i8
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_subs_epi8(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_subs_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_subs_epi8
+  // CHECK-LABEL: test_mm256_maskz_subs_epi8
   // CHECK: @llvm.ssub.sat.v32i8
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_subs_epi8(__U,__A,__B); 
 }
 __m128i test_mm_mask_subs_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_subs_epi16
+  // CHECK-LABEL: test_mm_mask_subs_epi16
   // CHECK: @llvm.ssub.sat.v8i16
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_subs_epi16(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_subs_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_subs_epi16
+  // CHECK-LABEL: test_mm_maskz_subs_epi16
   // CHECK: @llvm.ssub.sat.v8i16
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_subs_epi16(__U,__A,__B); 
 }
 __m256i test_mm256_mask_subs_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_subs_epi16
+  // CHECK-LABEL: test_mm256_mask_subs_epi16
   // CHECK: @llvm.ssub.sat.v16i16
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_subs_epi16(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_subs_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_subs_epi16
+  // CHECK-LABEL: test_mm256_maskz_subs_epi16
   // CHECK: @llvm.ssub.sat.v16i16
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_subs_epi16(__U,__A,__B); 
 }
 __m128i test_mm_mask_subs_epu8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_subs_epu8
+  // CHECK-LABEL: test_mm_mask_subs_epu8
   // CHECK-NOT: @llvm.x86.sse2.psubus.b
   // CHECK: call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_subs_epu8(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_subs_epu8(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_subs_epu8
+  // CHECK-LABEL: test_mm_maskz_subs_epu8
   // CHECK-NOT: @llvm.x86.sse2.psubus.b
   // CHECK: call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_subs_epu8(__U,__A,__B); 
 }
 __m256i test_mm256_mask_subs_epu8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_subs_epu8
+  // CHECK-LABEL: test_mm256_mask_subs_epu8
   // CHECK-NOT: @llvm.x86.avx2.psubus.b
   // CHECK: call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_subs_epu8(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_subs_epu8(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_subs_epu8
+  // CHECK-LABEL: test_mm256_maskz_subs_epu8
   // CHECK-NOT: @llvm.x86.avx2.psubus.b
   // CHECK: call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}})
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_subs_epu8(__U,__A,__B); 
 }
 __m128i test_mm_mask_subs_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_subs_epu16
+  // CHECK-LABEL: test_mm_mask_subs_epu16
   // CHECK-NOT: @llvm.x86.sse2.psubus.w
   // CHECK: call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_subs_epu16(__W,__U,__A,__B); 
 }
 __m128i test_mm_maskz_subs_epu16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_subs_epu16
+  // CHECK-LABEL: test_mm_maskz_subs_epu16
   // CHECK-NOT: @llvm.x86.sse2.psubus.w
   // CHECK: call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_subs_epu16(__U,__A,__B); 
 }
 __m256i test_mm256_mask_subs_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_subs_epu16
+  // CHECK-LABEL: test_mm256_mask_subs_epu16
   // CHECK-NOT: @llvm.x86.avx2.psubus.w
   // CHECK: call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_subs_epu16(__W,__U,__A,__B); 
 }
 __m256i test_mm256_maskz_subs_epu16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_subs_epu16
+  // CHECK-LABEL: test_mm256_maskz_subs_epu16
   // CHECK-NOT: @llvm.x86.avx2.psubus.w
   // CHECK: call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
@@ -1612,1019 +1616,1027 @@ __m256i test_mm256_maskz_subs_epu16(__mmask16 __U, __m256i __A, __m256i __B) {
 
 
 __m128i test_mm_mask2_permutex2var_epi16(__m128i __A, __m128i __I, __mmask8 __U, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask2_permutex2var_epi16
+  // CHECK-LABEL: test_mm_mask2_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.128
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask2_permutex2var_epi16(__A,__I,__U,__B); 
 }
 __m256i test_mm256_mask2_permutex2var_epi16(__m256i __A, __m256i __I, __mmask16 __U, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask2_permutex2var_epi16
+  // CHECK-LABEL: test_mm256_mask2_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.256
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask2_permutex2var_epi16(__A,__I,__U,__B); 
 }
 __m128i test_mm_permutex2var_epi16(__m128i __A, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_permutex2var_epi16
+  // CHECK-LABEL: test_mm_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.128
   return _mm_permutex2var_epi16(__A,__I,__B); 
 }
 __m128i test_mm_mask_permutex2var_epi16(__m128i __A, __mmask8 __U, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_permutex2var_epi16
+  // CHECK-LABEL: test_mm_mask_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.128
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_permutex2var_epi16(__A,__U,__I,__B); 
 }
 __m128i test_mm_maskz_permutex2var_epi16(__mmask8 __U, __m128i __A, __m128i __I, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_permutex2var_epi16
+  // CHECK-LABEL: test_mm_maskz_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.128
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_permutex2var_epi16(__U,__A,__I,__B); 
 }
 
 __m256i test_mm256_permutex2var_epi16(__m256i __A, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_permutex2var_epi16
+  // CHECK-LABEL: test_mm256_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.256
   return _mm256_permutex2var_epi16(__A,__I,__B); 
 }
 __m256i test_mm256_mask_permutex2var_epi16(__m256i __A, __mmask16 __U, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_permutex2var_epi16
+  // CHECK-LABEL: test_mm256_mask_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.256
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_permutex2var_epi16(__A,__U,__I,__B); 
 }
 __m256i test_mm256_maskz_permutex2var_epi16(__mmask16 __U, __m256i __A, __m256i __I, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_permutex2var_epi16
+  // CHECK-LABEL: test_mm256_maskz_permutex2var_epi16
   // CHECK: @llvm.x86.avx512.vpermi2var.hi.256
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_permutex2var_epi16(__U,__A,__I,__B); 
 }
 __m128i test_mm_mask_maddubs_epi16(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_maddubs_epi16
+  // CHECK-LABEL: test_mm_mask_maddubs_epi16
   // CHECK: @llvm.x86.ssse3.pmadd.ub.sw
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_maddubs_epi16(__W, __U, __X, __Y); 
 }
 
 __m128i test_mm_maskz_maddubs_epi16(__mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_maskz_maddubs_epi16
+  // CHECK-LABEL: test_mm_maskz_maddubs_epi16
   // CHECK: @llvm.x86.ssse3.pmadd.ub.sw
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_maddubs_epi16(__U, __X, __Y); 
 }
 
 __m256i test_mm256_mask_maddubs_epi16(__m256i __W, __mmask16 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_maddubs_epi16
+  // CHECK-LABEL: test_mm256_mask_maddubs_epi16
   // CHECK: @llvm.x86.avx2.pmadd.ub.sw
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_maddubs_epi16(__W, __U, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_maddubs_epi16(__mmask16 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_maddubs_epi16
+  // CHECK-LABEL: test_mm256_maskz_maddubs_epi16
   // CHECK: @llvm.x86.avx2.pmadd.ub.sw
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_maddubs_epi16(__U, __X, __Y); 
 }
 
 __m128i test_mm_mask_madd_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_madd_epi16
+  // CHECK-LABEL: test_mm_mask_madd_epi16
   // CHECK: @llvm.x86.sse2.pmadd.wd
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_madd_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_madd_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_madd_epi16
+  // CHECK-LABEL: test_mm_maskz_madd_epi16
   // CHECK: @llvm.x86.sse2.pmadd.wd
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_madd_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_madd_epi16(__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_madd_epi16
+  // CHECK-LABEL: test_mm256_mask_madd_epi16
   // CHECK: @llvm.x86.avx2.pmadd.wd
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_madd_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_madd_epi16(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_madd_epi16
+  // CHECK-LABEL: test_mm256_maskz_madd_epi16
   // CHECK: @llvm.x86.avx2.pmadd.wd
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_madd_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_cvtsepi16_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtsepi16_epi8
+  // CHECK-LABEL: test_mm_cvtsepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.wb.128
   return _mm_cvtsepi16_epi8(__A); 
 }
 
 __m128i test_mm_mask_cvtsepi16_epi8(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtsepi16_epi8
+  // CHECK-LABEL: test_mm_mask_cvtsepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.wb.128
   return _mm_mask_cvtsepi16_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtsepi16_epi8(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtsepi16_epi8
+  // CHECK-LABEL: test_mm_maskz_cvtsepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.wb.128
   return _mm_maskz_cvtsepi16_epi8(__M, __A); 
 }
 
 __m128i test_mm256_cvtsepi16_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtsepi16_epi8
+  // CHECK-LABEL: test_mm256_cvtsepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.wb.256
   return _mm256_cvtsepi16_epi8(__A); 
 }
 
 __m128i test_mm256_mask_cvtsepi16_epi8(__m128i __O, __mmask16 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtsepi16_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtsepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.wb.256
   return _mm256_mask_cvtsepi16_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtsepi16_epi8(__mmask16 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtsepi16_epi8
+  // CHECK-LABEL: test_mm256_maskz_cvtsepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovs.wb.256
   return _mm256_maskz_cvtsepi16_epi8(__M, __A); 
 }
 
 __m128i test_mm_cvtusepi16_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtusepi16_epi8
+  // CHECK-LABEL: test_mm_cvtusepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.wb.128
   return _mm_cvtusepi16_epi8(__A); 
 }
 
 __m128i test_mm_mask_cvtusepi16_epi8(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtusepi16_epi8
+  // CHECK-LABEL: test_mm_mask_cvtusepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.wb.128
   return _mm_mask_cvtusepi16_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtusepi16_epi8(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtusepi16_epi8
+  // CHECK-LABEL: test_mm_maskz_cvtusepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.wb.128
   return _mm_maskz_cvtusepi16_epi8(__M, __A); 
 }
 
 __m128i test_mm256_cvtusepi16_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtusepi16_epi8
+  // CHECK-LABEL: test_mm256_cvtusepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.wb.256
   return _mm256_cvtusepi16_epi8(__A); 
 }
 
 __m128i test_mm256_mask_cvtusepi16_epi8(__m128i __O, __mmask16 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtusepi16_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtusepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.wb.256
   return _mm256_mask_cvtusepi16_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtusepi16_epi8(__mmask16 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtusepi16_epi8
+  // CHECK-LABEL: test_mm256_maskz_cvtusepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmovus.wb.256
   return _mm256_maskz_cvtusepi16_epi8(__M, __A); 
 }
 
 __m128i test_mm_cvtepi16_epi8(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepi16_epi8
+  // CHECK-LABEL: test_mm_cvtepi16_epi8
   // CHECK: trunc <8 x i16> %{{.*}} to <8 x i8>
   // CHECK: shufflevector <8 x i8> %{{.*}}, <8 x i8> %{{.*}}, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   return _mm_cvtepi16_epi8(__A); 
 }
 
 __m128i test_mm_mask_cvtepi16_epi8(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi16_epi8
+  // CHECK-LABEL: test_mm_mask_cvtepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.wb.128
   return _mm_mask_cvtepi16_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi16_epi8(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi16_epi8
+  // CHECK-LABEL: test_mm_maskz_cvtepi16_epi8
   // CHECK: @llvm.x86.avx512.mask.pmov.wb.128
   return _mm_maskz_cvtepi16_epi8(__M, __A); 
 }
 
 __m128i test_mm256_cvtepi16_epi8(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepi16_epi8
+  // CHECK-LABEL: test_mm256_cvtepi16_epi8
   // CHECK: trunc <16 x i16> %{{.*}} to <16 x i8>
   return _mm256_cvtepi16_epi8(__A); 
 }
 
 __m128i test_mm256_mask_cvtepi16_epi8(__m128i __O, __mmask16 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi16_epi8
+  // CHECK-LABEL: test_mm256_mask_cvtepi16_epi8
   // CHECK: trunc <16 x i16> %{{.*}} to <16 x i8>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm256_mask_cvtepi16_epi8(__O, __M, __A); 
 }
 
 __m128i test_mm256_maskz_cvtepi16_epi8(__mmask16 __M, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi16_epi8
+  // CHECK-LABEL: test_mm256_maskz_cvtepi16_epi8
   // CHECK: trunc <16 x i16> %{{.*}} to <16 x i8>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm256_maskz_cvtepi16_epi8(__M, __A); 
 }
 
 __m128i test_mm_mask_mulhrs_epi16(__m128i __W, __mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_mask_mulhrs_epi16
+  // CHECK-LABEL: test_mm_mask_mulhrs_epi16
   // CHECK: @llvm.x86.ssse3.pmul.hr.sw
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_mulhrs_epi16(__W, __U, __X, __Y); 
 }
 
 __m128i test_mm_maskz_mulhrs_epi16(__mmask8 __U, __m128i __X, __m128i __Y) {
-  // CHECK-LABEL: @test_mm_maskz_mulhrs_epi16
+  // CHECK-LABEL: test_mm_maskz_mulhrs_epi16
   // CHECK: @llvm.x86.ssse3.pmul.hr.sw
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_mulhrs_epi16(__U, __X, __Y); 
 }
 
 __m256i test_mm256_mask_mulhrs_epi16(__m256i __W, __mmask16 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_mask_mulhrs_epi16
+  // CHECK-LABEL: test_mm256_mask_mulhrs_epi16
   // CHECK: @llvm.x86.avx2.pmul.hr.sw
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_mulhrs_epi16(__W, __U, __X, __Y); 
 }
 
 __m256i test_mm256_maskz_mulhrs_epi16(__mmask16 __U, __m256i __X, __m256i __Y) {
-  // CHECK-LABEL: @test_mm256_maskz_mulhrs_epi16
+  // CHECK-LABEL: test_mm256_maskz_mulhrs_epi16
   // CHECK: @llvm.x86.avx2.pmul.hr.sw
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_mulhrs_epi16(__U, __X, __Y); 
 }
 
 __m128i test_mm_mask_mulhi_epu16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_mulhi_epu16
+  // CHECK-LABEL: test_mm_mask_mulhi_epu16
   // CHECK: @llvm.x86.sse2.pmulhu.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_mulhi_epu16(__W, __U, __A, __B); 
 }
+TEST_CONSTEXPR(match_v8hi(_mm_mask_mulhi_epu16(_mm_set1_epi16(1), 0x3C, (__m128i)(__v8hi){+1, -2, +3, -4, +5, -6, +7, -8}, (__m128i)(__v8hi){-16, -14, +12, +10, -8, +6, -4, +2}), 1, 1, 0, 9, 4, 5, 1, 1));
 
 __m128i test_mm_maskz_mulhi_epu16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_mulhi_epu16
+  // CHECK-LABEL: test_mm_maskz_mulhi_epu16
   // CHECK: @llvm.x86.sse2.pmulhu.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_mulhi_epu16(__U, __A, __B); 
 }
+TEST_CONSTEXPR(match_v8hi(_mm_maskz_mulhi_epu16(0x87, (__m128i)(__v8hi){+1, -2, +3, -4, +5, -6, +7, -8}, (__m128i)(__v8hi){-16, -14, +12, +10, -8, +6, -4, +2}), 0, -16, 0, 0, 0, 0, 0, 1));
 
 __m256i test_mm256_mask_mulhi_epu16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_mulhi_epu16
+  // CHECK-LABEL: test_mm256_mask_mulhi_epu16
   // CHECK: @llvm.x86.avx2.pmulhu.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_mulhi_epu16(__W, __U, __A, __B); 
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_mask_mulhi_epu16(_mm256_set1_epi16(1), 0xF00F, (__m256i)(__v16hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16}, (__m256i)(__v16hi){-32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), 0, -32, 0, 25, 1, 1, 1, 1, 1, 1, 1, 1, 12, 5, 14, 1));
 
 __m256i test_mm256_maskz_mulhi_epu16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_mulhi_epu16
+  // CHECK-LABEL: test_mm256_maskz_mulhi_epu16
   // CHECK: @llvm.x86.avx2.pmulhu.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_mulhi_epu16(__U, __A, __B); 
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_maskz_mulhi_epu16(0x0FF0, (__m256i)(__v16hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16}, (__m256i)(__v16hi){-32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), 0, 0, 0, 0, 4, -28, 0, 17, 8, -24, 0, 9, 0, 0, 0, 0));
 
 __m128i test_mm_mask_mulhi_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_mulhi_epi16
+  // CHECK-LABEL: test_mm_mask_mulhi_epi16
   // CHECK: @llvm.x86.sse2.pmulh.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_mulhi_epi16(__W, __U, __A, __B); 
 }
+TEST_CONSTEXPR(match_v8hi(_mm_mask_mulhi_epu16(_mm_set1_epi16(1), 0x78, (__m128i)(__v8hi){+1, -2, +3, -4, +5, -6, +7, -8}, (__m128i)(__v8hi){-16, -14, +12, +10, -8, +6, -4, +2}), 1, 1, 1, 9, 4, 5, 6, 1));
 
 __m128i test_mm_maskz_mulhi_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_mulhi_epi16
+  // CHECK-LABEL: test_mm_maskz_mulhi_epi16
   // CHECK: @llvm.x86.sse2.pmulh.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_mulhi_epi16(__U, __A, __B); 
 }
+TEST_CONSTEXPR(match_v8hi(_mm_maskz_mulhi_epi16(0xC3, (__m128i)(__v8hi){+1, -2, +3, -4, +5, -6, +7, -8}, (__m128i)(__v8hi){-16, -14, +12, +10, -8, +6, -4, +2}), -1, 0, 0, 0, 0, 0, -1, -1));
 
 __m256i test_mm256_mask_mulhi_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_mulhi_epi16
+  // CHECK-LABEL: test_mm256_mask_mulhi_epi16
   // CHECK: @llvm.x86.avx2.pmulh.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_mulhi_epi16(__W, __U, __A, __B); 
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_mask_mulhi_epi16(_mm256_set1_epi16(1), 0x0FF0, (__m256i)(__v16hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16}, (__m256i)(__v16hi){-32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), 1, 1, 1, 1, -1, 0, 0, -1, -1, 0, 0, -1, 1, 1, 1, 1));
 
 __m256i test_mm256_maskz_mulhi_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_mulhi_epi16
+  // CHECK-LABEL: test_mm256_maskz_mulhi_epi16
   // CHECK: @llvm.x86.avx2.pmulh.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_mulhi_epi16(__U, __A, __B); 
 }
+TEST_CONSTEXPR(match_v16hi(_mm256_maskz_mulhi_epi16(0xF00F, (__m256i)(__v16hi){+1, -2, +3, -4, +5, -6, +7, -8, +9, -10, +11, -12, +13, -14, +15, -16}, (__m256i)(__v16hi){-32, -30, +28, +26, -24, -22, +20, +18, -16, -14, +12, +10, -8, +6, -4, +2}), -1, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1));
 
 __m128i test_mm_mask_unpackhi_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_unpackhi_epi8
+  // CHECK-LABEL: test_mm_mask_unpackhi_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_unpackhi_epi8(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_unpackhi_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpackhi_epi8
+  // CHECK-LABEL: test_mm_maskz_unpackhi_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_unpackhi_epi8(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_unpackhi_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpackhi_epi8
+  // CHECK-LABEL: test_mm256_mask_unpackhi_epi8
   // CHECK: shufflevector <32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_unpackhi_epi8(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_unpackhi_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpackhi_epi8
+  // CHECK-LABEL: test_mm256_maskz_unpackhi_epi8
   // CHECK: shufflevector <32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_unpackhi_epi8(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_unpackhi_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_unpackhi_epi16
+  // CHECK-LABEL: test_mm_mask_unpackhi_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_unpackhi_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_unpackhi_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpackhi_epi16
+  // CHECK-LABEL: test_mm_maskz_unpackhi_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_unpackhi_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_unpackhi_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpackhi_epi16
+  // CHECK-LABEL: test_mm256_mask_unpackhi_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_unpackhi_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_unpackhi_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpackhi_epi16
+  // CHECK-LABEL: test_mm256_maskz_unpackhi_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_unpackhi_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_unpacklo_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_unpacklo_epi8
+  // CHECK-LABEL: test_mm_mask_unpacklo_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_unpacklo_epi8(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_unpacklo_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpacklo_epi8
+  // CHECK-LABEL: test_mm_maskz_unpacklo_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_unpacklo_epi8(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_unpacklo_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpacklo_epi8
+  // CHECK-LABEL: test_mm256_mask_unpacklo_epi8
   // CHECK: shufflevector <32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_unpacklo_epi8(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_unpacklo_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpacklo_epi8
+  // CHECK-LABEL: test_mm256_maskz_unpacklo_epi8
   // CHECK: shufflevector <32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_unpacklo_epi8(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_unpacklo_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_unpacklo_epi16
+  // CHECK-LABEL: test_mm_mask_unpacklo_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_unpacklo_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_unpacklo_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_unpacklo_epi16
+  // CHECK-LABEL: test_mm_maskz_unpacklo_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_unpacklo_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_unpacklo_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_unpacklo_epi16
+  // CHECK-LABEL: test_mm256_mask_unpacklo_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_unpacklo_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_unpacklo_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_unpacklo_epi16
+  // CHECK-LABEL: test_mm256_maskz_unpacklo_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_unpacklo_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_cvtepi8_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi8_epi16
+  // CHECK-LABEL: test_mm_mask_cvtepi8_epi16
   // CHECK: sext <8 x i8> %{{.*}} to <8 x i16>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_cvtepi8_epi16(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepi8_epi16(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi8_epi16
+  // CHECK-LABEL: test_mm_maskz_cvtepi8_epi16
   // CHECK: sext <8 x i8> %{{.*}} to <8 x i16>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_cvtepi8_epi16(__U, __A); 
 }
 
 __m256i test_mm256_mask_cvtepi8_epi16(__m256i __W, __mmask16 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi8_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtepi8_epi16
   // CHECK: sext <16 x i8> %{{.*}} to <16 x i16>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_cvtepi8_epi16(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepi8_epi16(__mmask16 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi8_epi16
+  // CHECK-LABEL: test_mm256_maskz_cvtepi8_epi16
   // CHECK: sext <16 x i8> %{{.*}} to <16 x i16>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_cvtepi8_epi16(__U, __A); 
 }
 
 __m128i test_mm_mask_cvtepu8_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu8_epi16
+  // CHECK-LABEL: test_mm_mask_cvtepu8_epi16
   // CHECK: zext <8 x i8> %{{.*}} to <8 x i16>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_cvtepu8_epi16(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtepu8_epi16(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu8_epi16
+  // CHECK-LABEL: test_mm_maskz_cvtepu8_epi16
   // CHECK: zext <8 x i8> %{{.*}} to <8 x i16>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_cvtepu8_epi16(__U, __A); 
 }
 
 __m256i test_mm256_mask_cvtepu8_epi16(__m256i __W, __mmask16 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu8_epi16
+  // CHECK-LABEL: test_mm256_mask_cvtepu8_epi16
   // CHECK: zext <16 x i8> %{{.*}} to <16 x i16>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_cvtepu8_epi16(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtepu8_epi16(__mmask16 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu8_epi16
+  // CHECK-LABEL: test_mm256_maskz_cvtepu8_epi16
   // CHECK: zext <16 x i8> %{{.*}} to <16 x i16>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_cvtepu8_epi16(__U, __A); 
 }
 
 __m256i test_mm256_sllv_epi16(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_sllv_epi16
+  // CHECK-LABEL: test_mm256_sllv_epi16
   // CHECK: @llvm.x86.avx512.psllv.w.256(
   return _mm256_sllv_epi16(__A, __B); 
 }
 
 __m256i test_mm256_mask_sllv_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_sllv_epi16
+  // CHECK-LABEL: test_mm256_mask_sllv_epi16
   // CHECK: @llvm.x86.avx512.psllv.w.256(
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_sllv_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_sllv_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sllv_epi16
+  // CHECK-LABEL: test_mm256_maskz_sllv_epi16
   // CHECK: @llvm.x86.avx512.psllv.w.256(
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_sllv_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_sllv_epi16(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_sllv_epi16
+  // CHECK-LABEL: test_mm_sllv_epi16
   // CHECK: @llvm.x86.avx512.psllv.w.128(
   return _mm_sllv_epi16(__A, __B); 
 }
 
 __m128i test_mm_mask_sllv_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_sllv_epi16
+  // CHECK-LABEL: test_mm_mask_sllv_epi16
   // CHECK: @llvm.x86.avx512.psllv.w.128(
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_sllv_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_sllv_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_sllv_epi16
+  // CHECK-LABEL: test_mm_maskz_sllv_epi16
   // CHECK: @llvm.x86.avx512.psllv.w.128(
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_sllv_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_sll_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_sll_epi16
+  // CHECK-LABEL: test_mm_mask_sll_epi16
   // CHECK: @llvm.x86.sse2.psll.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_sll_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_sll_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_sll_epi16
+  // CHECK-LABEL: test_mm_maskz_sll_epi16
   // CHECK: @llvm.x86.sse2.psll.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_sll_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_sll_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_sll_epi16
+  // CHECK-LABEL: test_mm256_mask_sll_epi16
   // CHECK: @llvm.x86.avx2.psll.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_sll_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_sll_epi16(__mmask16 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sll_epi16
+  // CHECK-LABEL: test_mm256_maskz_sll_epi16
   // CHECK: @llvm.x86.avx2.psll.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_sll_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_slli_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_slli_epi16
+  // CHECK-LABEL: test_mm_mask_slli_epi16
   // CHECK: @llvm.x86.sse2.pslli.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_slli_epi16(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_mask_slli_epi16_2(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_mask_slli_epi16_2
+  // CHECK-LABEL: test_mm_mask_slli_epi16_2
   // CHECK: @llvm.x86.sse2.pslli.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_slli_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_slli_epi16(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_slli_epi16
+  // CHECK-LABEL: test_mm_maskz_slli_epi16
   // CHECK: @llvm.x86.sse2.pslli.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_slli_epi16(__U, __A, 5); 
 }
 
 __m128i test_mm_maskz_slli_epi16_2(__mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_maskz_slli_epi16_2
+  // CHECK-LABEL: test_mm_maskz_slli_epi16_2
   // CHECK: @llvm.x86.sse2.pslli.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_slli_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_slli_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_slli_epi16
+  // CHECK-LABEL: test_mm256_mask_slli_epi16
   // CHECK: @llvm.x86.avx2.pslli.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_slli_epi16(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_mask_slli_epi16_2(__m256i __W, __mmask16 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_mask_slli_epi16_2
+  // CHECK-LABEL: test_mm256_mask_slli_epi16_2
   // CHECK: @llvm.x86.avx2.pslli.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_slli_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_slli_epi16(__mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_slli_epi16
+  // CHECK-LABEL: test_mm256_maskz_slli_epi16
   // CHECK: @llvm.x86.avx2.pslli.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_slli_epi16(__U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_slli_epi16_2(__mmask16 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_maskz_slli_epi16_2
+  // CHECK-LABEL: test_mm256_maskz_slli_epi16_2
   // CHECK: @llvm.x86.avx2.pslli.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_slli_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_srlv_epi16(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_srlv_epi16
+  // CHECK-LABEL: test_mm256_srlv_epi16
   // CHECK: @llvm.x86.avx512.psrlv.w.256(
   return _mm256_srlv_epi16(__A, __B); 
 }
 
 __m256i test_mm256_mask_srlv_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_srlv_epi16
+  // CHECK-LABEL: test_mm256_mask_srlv_epi16
   // CHECK: @llvm.x86.avx512.psrlv.w.256(
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_srlv_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srlv_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srlv_epi16
+  // CHECK-LABEL: test_mm256_maskz_srlv_epi16
   // CHECK: @llvm.x86.avx512.psrlv.w.256(
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_srlv_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_srlv_epi16(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_srlv_epi16
+  // CHECK-LABEL: test_mm_srlv_epi16
   // CHECK: @llvm.x86.avx512.psrlv.w.128(
   return _mm_srlv_epi16(__A, __B); 
 }
 
 __m128i test_mm_mask_srlv_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_srlv_epi16
+  // CHECK-LABEL: test_mm_mask_srlv_epi16
   // CHECK: @llvm.x86.avx512.psrlv.w.128(
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_srlv_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srlv_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_srlv_epi16
+  // CHECK-LABEL: test_mm_maskz_srlv_epi16
   // CHECK: @llvm.x86.avx512.psrlv.w.128(
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_srlv_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_srl_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_srl_epi16
+  // CHECK-LABEL: test_mm_mask_srl_epi16
   // CHECK: @llvm.x86.sse2.psrl.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_srl_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srl_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_srl_epi16
+  // CHECK-LABEL: test_mm_maskz_srl_epi16
   // CHECK: @llvm.x86.sse2.psrl.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_srl_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_srl_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_srl_epi16
+  // CHECK-LABEL: test_mm256_mask_srl_epi16
   // CHECK: @llvm.x86.avx2.psrl.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_srl_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srl_epi16(__mmask16 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srl_epi16
+  // CHECK-LABEL: test_mm256_maskz_srl_epi16
   // CHECK: @llvm.x86.avx2.psrl.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_srl_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_srli_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_srli_epi16
+  // CHECK-LABEL: test_mm_mask_srli_epi16
   // CHECK: @llvm.x86.sse2.psrli.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_srli_epi16(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_mask_srli_epi16_2(__m128i __W, __mmask8 __U, __m128i __A, int __B) {
-  // CHECK-LABEL: @test_mm_mask_srli_epi16_2
+  // CHECK-LABEL: test_mm_mask_srli_epi16_2
   // CHECK: @llvm.x86.sse2.psrli.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_srli_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srli_epi16(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_srli_epi16
+  // CHECK-LABEL: test_mm_maskz_srli_epi16
   // CHECK: @llvm.x86.sse2.psrli.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_srli_epi16(__U, __A, 5); 
 }
 
 __m128i test_mm_maskz_srli_epi16_2(__mmask8 __U, __m128i __A, int __B) {
-  // CHECK-LABEL: @test_mm_maskz_srli_epi16_2
+  // CHECK-LABEL: test_mm_maskz_srli_epi16_2
   // CHECK: @llvm.x86.sse2.psrli.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_srli_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_srli_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_srli_epi16
+  // CHECK-LABEL: test_mm256_mask_srli_epi16
   // CHECK: @llvm.x86.avx2.psrli.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_srli_epi16(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_mask_srli_epi16_2(__m256i __W, __mmask16 __U, __m256i __A, int __B) {
-  // CHECK-LABEL: @test_mm256_mask_srli_epi16_2
+  // CHECK-LABEL: test_mm256_mask_srli_epi16_2
   // CHECK: @llvm.x86.avx2.psrli.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_srli_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srli_epi16(__mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_srli_epi16
+  // CHECK-LABEL: test_mm256_maskz_srli_epi16
   // CHECK: @llvm.x86.avx2.psrli.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_srli_epi16(__U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_srli_epi16_2(__mmask16 __U, __m256i __A, int __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srli_epi16_2
+  // CHECK-LABEL: test_mm256_maskz_srli_epi16_2
   // CHECK: @llvm.x86.avx2.psrli.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_srli_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_srav_epi16(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_srav_epi16
+  // CHECK-LABEL: test_mm256_srav_epi16
   // CHECK: @llvm.x86.avx512.psrav.w.256(
   return _mm256_srav_epi16(__A, __B); 
 }
 
 __m256i test_mm256_mask_srav_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_srav_epi16
+  // CHECK-LABEL: test_mm256_mask_srav_epi16
   // CHECK: @llvm.x86.avx512.psrav.w.256(
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_srav_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srav_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srav_epi16
+  // CHECK-LABEL: test_mm256_maskz_srav_epi16
   // CHECK: @llvm.x86.avx512.psrav.w.256(
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_srav_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_srav_epi16(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_srav_epi16
+  // CHECK-LABEL: test_mm_srav_epi16
   // CHECK: @llvm.x86.avx512.psrav.w.128(
   return _mm_srav_epi16(__A, __B); 
 }
 
 __m128i test_mm_mask_srav_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_srav_epi16
+  // CHECK-LABEL: test_mm_mask_srav_epi16
   // CHECK: @llvm.x86.avx512.psrav.w.128(
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_srav_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srav_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_srav_epi16
+  // CHECK-LABEL: test_mm_maskz_srav_epi16
   // CHECK: @llvm.x86.avx512.psrav.w.128(
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_srav_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_sra_epi16(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_sra_epi16
+  // CHECK-LABEL: test_mm_mask_sra_epi16
   // CHECK: @llvm.x86.sse2.psra.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_sra_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_sra_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_sra_epi16
+  // CHECK-LABEL: test_mm_maskz_sra_epi16
   // CHECK: @llvm.x86.sse2.psra.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_sra_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_sra_epi16(__m256i __W, __mmask16 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_sra_epi16
+  // CHECK-LABEL: test_mm256_mask_sra_epi16
   // CHECK: @llvm.x86.avx2.psra.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_sra_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_sra_epi16(__mmask16 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sra_epi16
+  // CHECK-LABEL: test_mm256_maskz_sra_epi16
   // CHECK: @llvm.x86.avx2.psra.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_sra_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_srai_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_srai_epi16
+  // CHECK-LABEL: test_mm_mask_srai_epi16
   // CHECK: @llvm.x86.sse2.psrai.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_srai_epi16(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_mask_srai_epi16_2(__m128i __W, __mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_mask_srai_epi16_2
+  // CHECK-LABEL: test_mm_mask_srai_epi16_2
   // CHECK: @llvm.x86.sse2.psrai.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_srai_epi16(__W, __U, __A, __B); 
 }
 
 __m128i test_mm_maskz_srai_epi16(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_srai_epi16
+  // CHECK-LABEL: test_mm_maskz_srai_epi16
   // CHECK: @llvm.x86.sse2.psrai.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_srai_epi16(__U, __A, 5); 
 }
 
 __m128i test_mm_maskz_srai_epi16_2(__mmask8 __U, __m128i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm_maskz_srai_epi16_2
+  // CHECK-LABEL: test_mm_maskz_srai_epi16_2
   // CHECK: @llvm.x86.sse2.psrai.w
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_srai_epi16(__U, __A, __B); 
 }
 
 __m256i test_mm256_mask_srai_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_srai_epi16
+  // CHECK-LABEL: test_mm256_mask_srai_epi16
   // CHECK: @llvm.x86.avx2.psrai.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_srai_epi16(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_mask_srai_epi16_2(__m256i __W, __mmask16 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_mask_srai_epi16_2
+  // CHECK-LABEL: test_mm256_mask_srai_epi16_2
   // CHECK: @llvm.x86.avx2.psrai.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_srai_epi16(__W, __U, __A, __B); 
 }
 
 __m256i test_mm256_maskz_srai_epi16(__mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_srai_epi16
+  // CHECK-LABEL: test_mm256_maskz_srai_epi16
   // CHECK: @llvm.x86.avx2.psrai.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_srai_epi16(__U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_srai_epi16_2(__mmask16 __U, __m256i __A, unsigned int __B) {
-  // CHECK-LABEL: @test_mm256_maskz_srai_epi16_2
+  // CHECK-LABEL: test_mm256_maskz_srai_epi16_2
   // CHECK: @llvm.x86.avx2.psrai.w
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_srai_epi16(__U, __A, __B); 
 }
 
 __m128i test_mm_mask_mov_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_mov_epi16
+  // CHECK-LABEL: test_mm_mask_mov_epi16
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_mov_epi16(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_mov_epi16(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_mov_epi16
+  // CHECK-LABEL: test_mm_maskz_mov_epi16
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_mov_epi16(__U, __A); 
 }
 
 __m256i test_mm256_mask_mov_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_mov_epi16
+  // CHECK-LABEL: test_mm256_mask_mov_epi16
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_mov_epi16(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_mov_epi16(__mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_mov_epi16
+  // CHECK-LABEL: test_mm256_maskz_mov_epi16
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_mov_epi16(__U, __A); 
 }
 
 __m128i test_mm_mask_mov_epi8(__m128i __W, __mmask16 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_mov_epi8
+  // CHECK-LABEL: test_mm_mask_mov_epi8
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_mov_epi8(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_mov_epi8(__mmask16 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_mov_epi8
+  // CHECK-LABEL: test_mm_maskz_mov_epi8
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_mov_epi8(__U, __A); 
 }
 
 __m256i test_mm256_mask_mov_epi8(__m256i __W, __mmask32 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_mov_epi8
+  // CHECK-LABEL: test_mm256_mask_mov_epi8
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_mov_epi8(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_mov_epi8(__mmask32 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_mov_epi8
+  // CHECK-LABEL: test_mm256_maskz_mov_epi8
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_mov_epi8(__U, __A); 
 }
 
 __m128i test_mm_loadu_epi16(void const *__P) {
-  // CHECK-LABEL: @test_mm_loadu_epi16
+  // CHECK-LABEL: test_mm_loadu_epi16
   // CHECK: load <2 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm_loadu_epi16(__P);
 }
 
 __m128i test_mm_mask_loadu_epi16(__m128i __W, __mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_loadu_epi16
+  // CHECK-LABEL: test_mm_mask_loadu_epi16
   // CHECK: @llvm.masked.load.v8i16.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_mask_loadu_epi16(__W, __U, __P); 
 }
 
 __m128i test_mm_maskz_loadu_epi16(__mmask8 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_loadu_epi16
+  // CHECK-LABEL: test_mm_maskz_loadu_epi16
   // CHECK: @llvm.masked.load.v8i16.p0(ptr %{{.*}}, i32 1, <8 x i1> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_maskz_loadu_epi16(__U, __P); 
 }
 
 __m256i test_mm256_loadu_epi16(void const *__P) {
-  // CHECK-LABEL: @test_mm256_loadu_epi16
+  // CHECK-LABEL: test_mm256_loadu_epi16
   // CHECK: load <4 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm256_loadu_epi16(__P);
 }
 
 __m256i test_mm256_mask_loadu_epi16(__m256i __W, __mmask16 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_loadu_epi16
+  // CHECK-LABEL: test_mm256_mask_loadu_epi16
   // CHECK: @llvm.masked.load.v16i16.p0(ptr %{{.*}}, i32 1, <16 x i1> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_mask_loadu_epi16(__W, __U, __P); 
 }
 
 __m256i test_mm256_maskz_loadu_epi16(__mmask16 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_loadu_epi16
+  // CHECK-LABEL: test_mm256_maskz_loadu_epi16
   // CHECK: @llvm.masked.load.v16i16.p0(ptr %{{.*}}, i32 1, <16 x i1> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_maskz_loadu_epi16(__U, __P); 
 }
 
 __m128i test_mm_loadu_epi8(void const *__P) {
-  // CHECK-LABEL: @test_mm_loadu_epi8
+  // CHECK-LABEL: test_mm_loadu_epi8
   // CHECK: load <2 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm_loadu_epi8(__P);
 }
 
 __m128i test_mm_mask_loadu_epi8(__m128i __W, __mmask16 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_mask_loadu_epi8
+  // CHECK-LABEL: test_mm_mask_loadu_epi8
   // CHECK: @llvm.masked.load.v16i8.p0(ptr %{{.*}}, i32 1, <16 x i1> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_mask_loadu_epi8(__W, __U, __P); 
 }
 
 __m128i test_mm_maskz_loadu_epi8(__mmask16 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm_maskz_loadu_epi8
+  // CHECK-LABEL: test_mm_maskz_loadu_epi8
   // CHECK: @llvm.masked.load.v16i8.p0(ptr %{{.*}}, i32 1, <16 x i1> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_maskz_loadu_epi8(__U, __P); 
 }
 
 __m256i test_mm256_loadu_epi8(void const *__P) {
-  // CHECK-LABEL: @test_mm256_loadu_epi8
+  // CHECK-LABEL: test_mm256_loadu_epi8
   // CHECK: load <4 x i64>, ptr %{{.*}}, align 1{{$}}
   return _mm256_loadu_epi8(__P);
 }
 
 __m256i test_mm256_mask_loadu_epi8(__m256i __W, __mmask32 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_mask_loadu_epi8
+  // CHECK-LABEL: test_mm256_mask_loadu_epi8
   // CHECK: @llvm.masked.load.v32i8.p0(ptr %{{.*}}, i32 1, <32 x i1> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_mask_loadu_epi8(__W, __U, __P); 
 }
 
 __m256i test_mm256_maskz_loadu_epi8(__mmask32 __U, void const *__P) {
-  // CHECK-LABEL: @test_mm256_maskz_loadu_epi8
+  // CHECK-LABEL: test_mm256_maskz_loadu_epi8
   // CHECK: @llvm.masked.load.v32i8.p0(ptr %{{.*}}, i32 1, <32 x i1> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_maskz_loadu_epi8(__U, __P); 
 }
 
 void test_mm_storeu_epi16(void *__p, __m128i __a) {
-  // check-label: @test_mm_storeu_epi16
+  // CHECK-LABEL: test_mm_storeu_epi16
   // check: store <2 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm_storeu_epi16(__p, __a);
 }
 
 void test_mm_mask_storeu_epi16(void *__P, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_storeu_epi16
+  // CHECK-LABEL: test_mm_mask_storeu_epi16
   // CHECK: @llvm.masked.store.v8i16.p0(<8 x i16> %{{.*}}, ptr %{{.*}}, i32 1, <8 x i1> %{{.*}})
   return _mm_mask_storeu_epi16(__P, __U, __A); 
 }
 
 void test_mm256_storeu_epi16(void *__P, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_storeu_epi16
+  // CHECK-LABEL: test_mm256_storeu_epi16
   // CHECK: store <4 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm256_storeu_epi16(__P, __A);
 }
 
 void test_mm256_mask_storeu_epi16(void *__P, __mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_storeu_epi16
+  // CHECK-LABEL: test_mm256_mask_storeu_epi16
   // CHECK: @llvm.masked.store.v16i16.p0(<16 x i16> %{{.*}}, ptr %{{.*}}, i32 1, <16 x i1> %{{.*}})
   return _mm256_mask_storeu_epi16(__P, __U, __A); 
 }
 
 void test_mm_storeu_epi8(void *__p, __m128i __a) {
-  // check-label: @test_mm_storeu_epi8
+  // CHECK-LABEL: test_mm_storeu_epi8
   // check: store <2 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm_storeu_epi8(__p, __a);
 }
 
 void test_mm_mask_storeu_epi8(void *__P, __mmask16 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_storeu_epi8
+  // CHECK-LABEL: test_mm_mask_storeu_epi8
   // CHECK: @llvm.masked.store.v16i8.p0(<16 x i8> %{{.*}}, ptr %{{.*}}, i32 1, <16 x i1> %{{.*}})
   return _mm_mask_storeu_epi8(__P, __U, __A); 
 }
 
 void test_mm256_storeu_epi8(void *__P, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_storeu_epi8
+  // CHECK-LABEL: test_mm256_storeu_epi8
   // CHECK: store <4 x i64> %{{.*}}, ptr %{{.*}}, align 1{{$}}
   return _mm256_storeu_epi8(__P, __A);
 }
 
 void test_mm256_mask_storeu_epi8(void *__P, __mmask32 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_storeu_epi8
+  // CHECK-LABEL: test_mm256_mask_storeu_epi8
   // CHECK: @llvm.masked.store.v32i8.p0(<32 x i8> %{{.*}}, ptr %{{.*}}, i32 1, <32 x i1> %{{.*}})
   return _mm256_mask_storeu_epi8(__P, __U, __A); 
 }
 __mmask16 test_mm_test_epi8_mask(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_test_epi8_mask
+  // CHECK-LABEL: test_mm_test_epi8_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <16 x i8> %{{.*}}, %{{.*}}
   return _mm_test_epi8_mask(__A, __B); 
 }
 
 __mmask16 test_mm_mask_test_epi8_mask(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_test_epi8_mask
+  // CHECK-LABEL: test_mm_mask_test_epi8_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
@@ -2632,14 +2644,14 @@ __mmask16 test_mm_mask_test_epi8_mask(__mmask16 __U, __m128i __A, __m128i __B) {
 }
 
 __mmask32 test_mm256_test_epi8_mask(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_test_epi8_mask
+  // CHECK-LABEL: test_mm256_test_epi8_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <32 x i8> %{{.*}}, %{{.*}}
   return _mm256_test_epi8_mask(__A, __B); 
 }
 
 __mmask32 test_mm256_mask_test_epi8_mask(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_test_epi8_mask
+  // CHECK-LABEL: test_mm256_mask_test_epi8_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
@@ -2647,14 +2659,14 @@ __mmask32 test_mm256_mask_test_epi8_mask(__mmask32 __U, __m256i __A, __m256i __B
 }
 
 __mmask8 test_mm_test_epi16_mask(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_test_epi16_mask
+  // CHECK-LABEL: test_mm_test_epi16_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <8 x i16> %{{.*}}, %{{.*}}
   return _mm_test_epi16_mask(__A, __B); 
 }
 
 __mmask8 test_mm_mask_test_epi16_mask(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_test_epi16_mask
+  // CHECK-LABEL: test_mm_mask_test_epi16_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
@@ -2662,14 +2674,14 @@ __mmask8 test_mm_mask_test_epi16_mask(__mmask8 __U, __m128i __A, __m128i __B) {
 }
 
 __mmask16 test_mm256_test_epi16_mask(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_test_epi16_mask
+  // CHECK-LABEL: test_mm256_test_epi16_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <16 x i16> %{{.*}}, %{{.*}}
   return _mm256_test_epi16_mask(__A, __B); 
 }
 
 __mmask16 test_mm256_mask_test_epi16_mask(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_test_epi16_mask
+  // CHECK-LABEL: test_mm256_mask_test_epi16_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp ne <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
@@ -2677,14 +2689,14 @@ __mmask16 test_mm256_mask_test_epi16_mask(__mmask16 __U, __m256i __A, __m256i __
 }
 
 __mmask16 test_mm_testn_epi8_mask(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_testn_epi8_mask
+  // CHECK-LABEL: test_mm_testn_epi8_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   return _mm_testn_epi8_mask(__A, __B); 
 }
 
 __mmask16 test_mm_mask_testn_epi8_mask(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_testn_epi8_mask
+  // CHECK-LABEL: test_mm_mask_testn_epi8_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <16 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
@@ -2692,14 +2704,14 @@ __mmask16 test_mm_mask_testn_epi8_mask(__mmask16 __U, __m128i __A, __m128i __B)
 }
 
 __mmask32 test_mm256_testn_epi8_mask(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_testn_epi8_mask
+  // CHECK-LABEL: test_mm256_testn_epi8_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   return _mm256_testn_epi8_mask(__A, __B); 
 }
 
 __mmask32 test_mm256_mask_testn_epi8_mask(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_testn_epi8_mask
+  // CHECK-LABEL: test_mm256_mask_testn_epi8_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <32 x i8> %{{.*}}, %{{.*}}
   // CHECK: and <32 x i1> %{{.*}}, %{{.*}}
@@ -2707,14 +2719,14 @@ __mmask32 test_mm256_mask_testn_epi8_mask(__mmask32 __U, __m256i __A, __m256i __
 }
 
 __mmask8 test_mm_testn_epi16_mask(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_testn_epi16_mask
+  // CHECK-LABEL: test_mm_testn_epi16_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   return _mm_testn_epi16_mask(__A, __B); 
 }
 
 __mmask8 test_mm_mask_testn_epi16_mask(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_testn_epi16_mask
+  // CHECK-LABEL: test_mm_mask_testn_epi16_mask
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <8 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> %{{.*}}, %{{.*}}
@@ -2722,14 +2734,14 @@ __mmask8 test_mm_mask_testn_epi16_mask(__mmask8 __U, __m128i __A, __m128i __B) {
 }
 
 __mmask16 test_mm256_testn_epi16_mask(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_testn_epi16_mask
+  // CHECK-LABEL: test_mm256_testn_epi16_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   return _mm256_testn_epi16_mask(__A, __B); 
 }
 
 __mmask16 test_mm256_mask_testn_epi16_mask(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_testn_epi16_mask
+  // CHECK-LABEL: test_mm256_mask_testn_epi16_mask
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: icmp eq <16 x i16> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> %{{.*}}, %{{.*}}
@@ -2737,102 +2749,102 @@ __mmask16 test_mm256_mask_testn_epi16_mask(__mmask16 __U, __m256i __A, __m256i _
 }
 
 __mmask16 test_mm_movepi8_mask(__m128i __A) {
-  // CHECK-LABEL: @test_mm_movepi8_mask
+  // CHECK-LABEL: test_mm_movepi8_mask
   // CHECK: [[CMP:%.*]] = icmp slt <16 x i8> %{{.*}}, zeroinitializer
   return _mm_movepi8_mask(__A); 
 }
 
 __mmask32 test_mm256_movepi8_mask(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_movepi8_mask
+  // CHECK-LABEL: test_mm256_movepi8_mask
   // CHECK: [[CMP:%.*]] = icmp slt <32 x i8> %{{.*}}, zeroinitializer
   return _mm256_movepi8_mask(__A); 
 }
 
 __m128i test_mm_movm_epi8(__mmask16 __A) {
-  // CHECK-LABEL: @test_mm_movm_epi8
+  // CHECK-LABEL: test_mm_movm_epi8
   // CHECK: %{{.*}} = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: %vpmovm2.i = sext <16 x i1> %{{.*}} to <16 x i8>
   return _mm_movm_epi8(__A); 
 }
 
 __m256i test_mm256_movm_epi8(__mmask32 __A) {
-  // CHECK-LABEL: @test_mm256_movm_epi8
+  // CHECK-LABEL: test_mm256_movm_epi8
   // CHECK: %{{.*}} = bitcast i32 %{{.*}} to <32 x i1>
   // CHECK: %vpmovm2.i = sext <32 x i1> %{{.*}} to <32 x i8>
   return _mm256_movm_epi8(__A); 
 }
 
 __m128i test_mm_movm_epi16(__mmask8 __A) {
-  // CHECK-LABEL: @test_mm_movm_epi16
+  // CHECK-LABEL: test_mm_movm_epi16
   // CHECK: %{{.*}} = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: %vpmovm2.i = sext <8 x i1> %{{.*}} to <8 x i16>
   return _mm_movm_epi16(__A); 
 }
 
 __m256i test_mm256_movm_epi16(__mmask16 __A) {
-  // CHECK-LABEL: @test_mm256_movm_epi16
+  // CHECK-LABEL: test_mm256_movm_epi16
   // CHECK: %{{.*}} = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK: %vpmovm2.i = sext <16 x i1> %{{.*}} to <16 x i16>
   return _mm256_movm_epi16(__A); 
 }
 
 __m128i test_mm_mask_broadcastb_epi8(__m128i __O, __mmask16 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_broadcastb_epi8
+  // CHECK-LABEL: test_mm_mask_broadcastb_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> zeroinitializer
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_broadcastb_epi8(__O, __M, __A);
 }
 
 __m128i test_mm_maskz_broadcastb_epi8(__mmask16 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_broadcastb_epi8
+  // CHECK-LABEL: test_mm_maskz_broadcastb_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> zeroinitializer
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_broadcastb_epi8(__M, __A);
 }
 
 __m256i test_mm256_mask_broadcastb_epi8(__m256i __O, __mmask32 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcastb_epi8
+  // CHECK-LABEL: test_mm256_mask_broadcastb_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <32 x i32> zeroinitializer
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_broadcastb_epi8(__O, __M, __A);
 }
 
 __m256i test_mm256_maskz_broadcastb_epi8(__mmask32 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcastb_epi8
+  // CHECK-LABEL: test_mm256_maskz_broadcastb_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <32 x i32> zeroinitializer
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_broadcastb_epi8(__M, __A);
 }
 
 __m128i test_mm_mask_broadcastw_epi16(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_broadcastw_epi16
+  // CHECK-LABEL: test_mm_mask_broadcastw_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i32> zeroinitializer
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_broadcastw_epi16(__O, __M, __A);
 }
 
 __m128i test_mm_maskz_broadcastw_epi16(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_broadcastw_epi16
+  // CHECK-LABEL: test_mm_maskz_broadcastw_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i32> zeroinitializer
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_broadcastw_epi16(__M, __A);
 }
 
 __m256i test_mm256_mask_broadcastw_epi16(__m256i __O, __mmask16 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcastw_epi16
+  // CHECK-LABEL: test_mm256_mask_broadcastw_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <16 x i32> zeroinitializer
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_broadcastw_epi16(__O, __M, __A);
 }
 
 __m256i test_mm256_maskz_broadcastw_epi16(__mmask16 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcastw_epi16
+  // CHECK-LABEL: test_mm256_maskz_broadcastw_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <16 x i32> zeroinitializer
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_broadcastw_epi16(__M, __A);
 }
 __m128i test_mm_mask_set1_epi8 (__m128i __O, __mmask16 __M, char __A){
-  // CHECK-LABEL: @test_mm_mask_set1_epi8
+  // CHECK-LABEL: test_mm_mask_set1_epi8
   // CHECK: insertelement <16 x i8> poison, i8 %{{.*}}, i32 0
   // CHECK: insertelement <16 x i8> %{{.*}}, i8 %{{.*}}, i32 1
   // CHECK: insertelement <16 x i8> %{{.*}}, i8 %{{.*}}, i32 2
@@ -2853,7 +2865,7 @@ __m128i test_mm_mask_set1_epi8 (__m128i __O, __mmask16 __M, char __A){
   return _mm_mask_set1_epi8(__O, __M, __A);
 }
 __m128i test_mm_maskz_set1_epi8 ( __mmask16 __M, char __A){
-  // CHECK-LABEL: @test_mm_maskz_set1_epi8
+  // CHECK-LABEL: test_mm_maskz_set1_epi8
   // CHECK: insertelement <16 x i8> poison, i8 %{{.*}}, i32 0
   // CHECK: insertelement <16 x i8> %{{.*}}, i8 %{{.*}}, i32 1
   // CHECK: insertelement <16 x i8> %{{.*}}, i8 %{{.*}}, i32 2
@@ -2875,7 +2887,7 @@ __m128i test_mm_maskz_set1_epi8 ( __mmask16 __M, char __A){
 }
 
 __m256i test_mm256_mask_set1_epi8(__m256i __O, __mmask32 __M, char __A) {
-  // CHECK-LABEL: @test_mm256_mask_set1_epi8
+  // CHECK-LABEL: test_mm256_mask_set1_epi8
   // CHECK: insertelement <32 x i8> poison, i8 %{{.*}}, i32 0
   // CHECK: insertelement <32 x i8> %{{.*}}, i8 %{{.*}}, i32 1
   // CHECK: insertelement <32 x i8> %{{.*}}, i8 %{{.*}}, i32 2
@@ -2913,7 +2925,7 @@ __m256i test_mm256_mask_set1_epi8(__m256i __O, __mmask32 __M, char __A) {
 }
 
 __m256i test_mm256_maskz_set1_epi8( __mmask32 __M, char __A) {
-  // CHECK-LABEL: @test_mm256_maskz_set1_epi8
+  // CHECK-LABEL: test_mm256_maskz_set1_epi8
   // CHECK: insertelement <32 x i8> poison, i8 %{{.*}}, i32 0
   // CHECK: insertelement <32 x i8> %{{.*}}, i8 %{{.*}}, i32 1
   // CHECK: insertelement <32 x i8> %{{.*}}, i8 %{{.*}}, i32 2
@@ -2952,7 +2964,7 @@ __m256i test_mm256_maskz_set1_epi8( __mmask32 __M, char __A) {
 
 
 __m256i test_mm256_mask_set1_epi16(__m256i __O, __mmask16 __M, short __A) {
-  // CHECK-LABEL: @test_mm256_mask_set1_epi16
+  // CHECK-LABEL: test_mm256_mask_set1_epi16
   // CHECK: insertelement <16 x i16> poison, i16 %{{.*}}, i32 0
   // CHECK: insertelement <16 x i16> %{{.*}}, i16 %{{.*}}, i32 1
   // CHECK: insertelement <16 x i16> %{{.*}}, i16 %{{.*}}, i32 2
@@ -2974,7 +2986,7 @@ __m256i test_mm256_mask_set1_epi16(__m256i __O, __mmask16 __M, short __A) {
 }
 
 __m256i test_mm256_maskz_set1_epi16(__mmask16 __M, short __A) {
-  // CHECK-LABEL: @test_mm256_maskz_set1_epi16
+  // CHECK-LABEL: test_mm256_maskz_set1_epi16
   // CHECK: insertelement <16 x i16> poison, i16 %{{.*}}, i32 0
   // CHECK: insertelement <16 x i16> %{{.*}}, i16 %{{.*}}, i32 1
   // CHECK: insertelement <16 x i16> %{{.*}}, i16 %{{.*}}, i32 2
@@ -2996,7 +3008,7 @@ __m256i test_mm256_maskz_set1_epi16(__mmask16 __M, short __A) {
 }
 
 __m128i test_mm_mask_set1_epi16(__m128i __O, __mmask8 __M, short __A) {
-  // CHECK-LABEL: @test_mm_mask_set1_epi16
+  // CHECK-LABEL: test_mm_mask_set1_epi16
   // CHECK: insertelement <8 x i16> poison, i16 %{{.*}}, i32 0
   // CHECK: insertelement <8 x i16> %{{.*}}, i16 %{{.*}}, i32 1
   // CHECK: insertelement <8 x i16> %{{.*}}, i16 %{{.*}}, i32 2
@@ -3010,7 +3022,7 @@ __m128i test_mm_mask_set1_epi16(__m128i __O, __mmask8 __M, short __A) {
 }
 
 __m128i test_mm_maskz_set1_epi16(__mmask8 __M, short __A) {
-  // CHECK-LABEL: @test_mm_maskz_set1_epi16
+  // CHECK-LABEL: test_mm_maskz_set1_epi16
   // CHECK: insertelement <8 x i16> poison, i16 %{{.*}}, i32 0
   // CHECK: insertelement <8 x i16> %{{.*}}, i16 %{{.*}}, i32 1
   // CHECK: insertelement <8 x i16> %{{.*}}, i16 %{{.*}}, i32 2
@@ -3023,174 +3035,174 @@ __m128i test_mm_maskz_set1_epi16(__mmask8 __M, short __A) {
   return _mm_maskz_set1_epi16(__M, __A); 
 }
 __m128i test_mm_permutexvar_epi16(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_permutexvar_epi16
+  // CHECK-LABEL: test_mm_permutexvar_epi16
   // CHECK: @llvm.x86.avx512.permvar.hi.128
   return _mm_permutexvar_epi16(__A, __B); 
 }
 
 __m128i test_mm_maskz_permutexvar_epi16(__mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_permutexvar_epi16
+  // CHECK-LABEL: test_mm_maskz_permutexvar_epi16
   // CHECK: @llvm.x86.avx512.permvar.hi.128
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_permutexvar_epi16(__M, __A, __B); 
 }
 
 __m128i test_mm_mask_permutexvar_epi16(__m128i __W, __mmask8 __M, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_permutexvar_epi16
+  // CHECK-LABEL: test_mm_mask_permutexvar_epi16
   // CHECK: @llvm.x86.avx512.permvar.hi.128
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_permutexvar_epi16(__W, __M, __A, __B); 
 }
 
 __m256i test_mm256_permutexvar_epi16(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_permutexvar_epi16
+  // CHECK-LABEL: test_mm256_permutexvar_epi16
   // CHECK: @llvm.x86.avx512.permvar.hi.256
   return _mm256_permutexvar_epi16(__A, __B); 
 }
 
 __m256i test_mm256_maskz_permutexvar_epi16(__mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_permutexvar_epi16
+  // CHECK-LABEL: test_mm256_maskz_permutexvar_epi16
   // CHECK: @llvm.x86.avx512.permvar.hi.256
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_permutexvar_epi16(__M, __A, __B); 
 }
 
 __m256i test_mm256_mask_permutexvar_epi16(__m256i __W, __mmask16 __M, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_permutexvar_epi16
+  // CHECK-LABEL: test_mm256_mask_permutexvar_epi16
   // CHECK: @llvm.x86.avx512.permvar.hi.256
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_permutexvar_epi16(__W, __M, __A, __B); 
 }
 __m128i test_mm_mask_alignr_epi8(__m128i __W, __mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_alignr_epi8
+  // CHECK-LABEL: test_mm_mask_alignr_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_mask_alignr_epi8(__W, __U, __A, __B, 2); 
 }
 
 __m128i test_mm_maskz_alignr_epi8(__mmask16 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_alignr_epi8
+  // CHECK-LABEL: test_mm_maskz_alignr_epi8
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
   return _mm_maskz_alignr_epi8(__U, __A, __B, 2); 
 }
 
 __m256i test_mm256_mask_alignr_epi8(__m256i __W, __mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_alignr_epi8
+  // CHECK-LABEL: test_mm256_mask_alignr_epi8
   // CHECK: shufflevector <32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_mask_alignr_epi8(__W, __U, __A, __B, 2); 
 }
 
 __m256i test_mm256_maskz_alignr_epi8(__mmask32 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_alignr_epi8
+  // CHECK-LABEL: test_mm256_maskz_alignr_epi8
   // CHECK: shufflevector <32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49>
   // CHECK: select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
   return _mm256_maskz_alignr_epi8(__U, __A, __B, 2); 
 }
 
 __m128i test_mm_dbsad_epu8(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dbsad_epu8
+  // CHECK-LABEL: test_mm_dbsad_epu8
   // CHECK: @llvm.x86.avx512.dbpsadbw.128
   return _mm_dbsad_epu8(__A, __B, 170); 
 }
 
 __m128i test_mm_mask_dbsad_epu8(__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_dbsad_epu8
+  // CHECK-LABEL: test_mm_mask_dbsad_epu8
   // CHECK: @llvm.x86.avx512.dbpsadbw.128
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_dbsad_epu8(__W, __U, __A, __B, 170); 
 }
 
 __m128i test_mm_maskz_dbsad_epu8(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_dbsad_epu8
+  // CHECK-LABEL: test_mm_maskz_dbsad_epu8
   // CHECK: @llvm.x86.avx512.dbpsadbw.128
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_dbsad_epu8(__U, __A, __B, 170); 
 }
 
 __m256i test_mm256_dbsad_epu8(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dbsad_epu8
+  // CHECK-LABEL: test_mm256_dbsad_epu8
   // CHECK: @llvm.x86.avx512.dbpsadbw.256
   return _mm256_dbsad_epu8(__A, __B, 170); 
 }
 
 __m256i test_mm256_mask_dbsad_epu8(__m256i __W, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_dbsad_epu8
+  // CHECK-LABEL: test_mm256_mask_dbsad_epu8
   // CHECK: @llvm.x86.avx512.dbpsadbw.256
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_dbsad_epu8(__W, __U, __A, __B, 170); 
 }
 
 __m256i test_mm256_maskz_dbsad_epu8(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_dbsad_epu8
+  // CHECK-LABEL: test_mm256_maskz_dbsad_epu8
   // CHECK: @llvm.x86.avx512.dbpsadbw.256
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_dbsad_epu8(__U, __A, __B, 170); 
 }
 __mmask8 test_mm_movepi16_mask(__m128i __A) {
-  // CHECK-LABEL: @test_mm_movepi16_mask
+  // CHECK-LABEL: test_mm_movepi16_mask
   // CHECK: [[CMP:%.*]] = icmp slt <8 x i16> %{{.*}}, zeroinitializer
   return _mm_movepi16_mask(__A); 
 }
 
 __mmask16 test_mm256_movepi16_mask(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_movepi16_mask
+  // CHECK-LABEL: test_mm256_movepi16_mask
   // CHECK: [[CMP:%.*]] = icmp slt <16 x i16> %{{.*}}, zeroinitializer
   return _mm256_movepi16_mask(__A); 
 }
 
 __m128i test_mm_mask_shufflehi_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_shufflehi_epi16
+  // CHECK-LABEL: test_mm_mask_shufflehi_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 4, i32 4>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_shufflehi_epi16(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_maskz_shufflehi_epi16(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_shufflehi_epi16
+  // CHECK-LABEL: test_mm_maskz_shufflehi_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 4, i32 4>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_shufflehi_epi16(__U, __A, 5); 
 }
 
 __m128i test_mm_mask_shufflelo_epi16(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_shufflelo_epi16
+  // CHECK-LABEL: test_mm_mask_shufflelo_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_shufflelo_epi16(__W, __U, __A, 5); 
 }
 
 __m128i test_mm_maskz_shufflelo_epi16(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_shufflelo_epi16
+  // CHECK-LABEL: test_mm_maskz_shufflelo_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_shufflelo_epi16(__U, __A, 5); 
 }
 
 __m256i test_mm256_mask_shufflehi_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_shufflehi_epi16
+  // CHECK-LABEL: test_mm256_mask_shufflehi_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 13, i32 13, i32 12, i32 12>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_shufflehi_epi16(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_shufflehi_epi16(__mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_shufflehi_epi16
+  // CHECK-LABEL: test_mm256_maskz_shufflehi_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 4, i32 4, i32 8, i32 9, i32 10, i32 11, i32 13, i32 13, i32 12, i32 12>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_shufflehi_epi16(__U, __A, 5); 
 }
 
 __m256i test_mm256_mask_shufflelo_epi16(__m256i __W, __mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_shufflelo_epi16
+  // CHECK-LABEL: test_mm256_mask_shufflelo_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> poison, <16 x i32> <i32 1, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 9, i32 9, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_shufflelo_epi16(__W, __U, __A, 5); 
 }
 
 __m256i test_mm256_maskz_shufflelo_epi16(__mmask16 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_shufflelo_epi16
+  // CHECK-LABEL: test_mm256_maskz_shufflelo_epi16
   // CHECK: shufflevector <16 x i16> %{{.*}}, <16 x i16> poison, <16 x i32> <i32 1, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 9, i32 9, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15>
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_shufflelo_epi16(__U, __A, 5); 
@@ -3198,42 +3210,42 @@ __m256i test_mm256_maskz_shufflelo_epi16(__mmask16 __U, __m256i __A) {
 
 void test_mm_mask_cvtepi16_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)
 {
- // CHECK-LABEL:@test_mm_mask_cvtepi16_storeu_epi8
+ // CHECK-LABEL: test_mm_mask_cvtepi16_storeu_epi8
  // CHECK: @llvm.x86.avx512.mask.pmov.wb.mem.128
  _mm_mask_cvtepi16_storeu_epi8 (__P, __M, __A);
 }
 
 void test_mm_mask_cvtsepi16_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)
 {
- // CHECK-LABEL:@test_mm_mask_cvtsepi16_storeu_epi8
+ // CHECK-LABEL: test_mm_mask_cvtsepi16_storeu_epi8
  // CHECK: @llvm.x86.avx512.mask.pmovs.wb.mem.128
   _mm_mask_cvtsepi16_storeu_epi8 ( __P,  __M, __A);
 }
 
 void test_mm_mask_cvtusepi16_storeu_epi8 (void * __P, __mmask8 __M, __m128i __A)
 {
- // CHECK-LABEL:@test_mm_mask_cvtusepi16_storeu_epi8
+ // CHECK-LABEL: test_mm_mask_cvtusepi16_storeu_epi8
  // CHECK: @llvm.x86.avx512.mask.pmovus.wb.mem.128
   _mm_mask_cvtusepi16_storeu_epi8 (__P, __M, __A);
 }
 
 void test_mm256_mask_cvtusepi16_storeu_epi8 (void * __P, __mmask16 __M, __m256i __A)
 {
- // CHECK-LABEL:@test_mm256_mask_cvtusepi16_storeu_epi8
+ // CHECK-LABEL: test_mm256_mask_cvtusepi16_storeu_epi8
  // CHECK: @llvm.x86.avx512.mask.pmovus.wb.mem.256
   _mm256_mask_cvtusepi16_storeu_epi8 ( __P, __M, __A);
 }
 
 void test_mm256_mask_cvtepi16_storeu_epi8 (void * __P, __mmask16 __M, __m256i __A)
 {
- // CHECK-LABEL:@test_mm256_mask_cvtepi16_storeu_epi8
+ // CHECK-LABEL: test_mm256_mask_cvtepi16_storeu_epi8
  // CHECK: @llvm.x86.avx512.mask.pmov.wb.mem.256
   _mm256_mask_cvtepi16_storeu_epi8 ( __P,  __M, __A);
 }
 
 void test_mm256_mask_cvtsepi16_storeu_epi8 (void * __P, __mmask16 __M, __m256i __A)
 {
- // CHECK-LABEL:@test_mm256_mask_cvtsepi16_storeu_epi8
+ // CHECK-LABEL: test_mm256_mask_cvtsepi16_storeu_epi8
  // CHECK: @llvm.x86.avx512.mask.pmovs.wb.mem.256
  _mm256_mask_cvtsepi16_storeu_epi8 ( __P, __M, __A);
 }
diff --git a/clang/test/CodeGen/X86/avx512vlbw-reduceIntrin.c b/clang/test/CodeGen/X86/avx512vlbw-reduceIntrin.c
index 0a1692001efa6..faa3b54624a77 100644
--- a/clang/test/CodeGen/X86/avx512vlbw-reduceIntrin.c
+++ b/clang/test/CodeGen/X86/avx512vlbw-reduceIntrin.c
@@ -1,420 +1,426 @@
-// RUN: %clang_cc1 -ffreestanding %s -O0 -triple=x86_64 -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -ffreestanding %s -O0 -triple=i386 -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -O0 -triple=x86_64 -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -O0 -triple=x86_64 -target-feature +avx512bw -target-feature +avx512vl -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -O0 -triple=i386 -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -O0 -triple=i386 -target-feature +avx512bw -target-feature +avx512vl -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -O0 -triple=x86_64 -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -O0 -triple=x86_64 -target-feature +avx512bw -target-feature +avx512vl -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -O0 -triple=i386 -target-feature +avx512bw -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -O0 -triple=i386 -target-feature +avx512bw -target-feature +avx512vl -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 short test_mm_reduce_add_epi16(__m128i __W){
-// CHECK-LABEL: @test_mm_reduce_add_epi16(
-// CHECK: call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %{{.*}})
+// CHECK-LABEL: test_mm_reduce_add_epi16
+// CHECK: call {{.*}}i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %{{.*}})
   return _mm_reduce_add_epi16(__W);
 }
 
 short test_mm_reduce_mul_epi16(__m128i __W){
-// CHECK-LABEL: @test_mm_reduce_mul_epi16(
-// CHECK:    call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %{{.*}})
+// CHECK-LABEL: test_mm_reduce_mul_epi16
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %{{.*}})
   return _mm_reduce_mul_epi16(__W);
 }
 
 short test_mm_reduce_or_epi16(__m128i __W){
-// CHECK-LABEL: @test_mm_reduce_or_epi16(
-// CHECK:    call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> %{{.*}})
+// CHECK-LABEL: test_mm_reduce_or_epi16
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.or.v8i16(<8 x i16> %{{.*}})
   return _mm_reduce_or_epi16(__W);
 }
 
 short test_mm_reduce_and_epi16(__m128i __W){
-// CHECK-LABEL: @test_mm_reduce_and_epi16(
-// CHECK:    call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> %{{.*}})
+// CHECK-LABEL: test_mm_reduce_and_epi16
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.and.v8i16(<8 x i16> %{{.*}})
   return _mm_reduce_and_epi16(__W);
 }
 
 short test_mm_mask_reduce_add_epi16(__mmask8 __M, __m128i __W){
-// CHECK-LABEL: @test_mm_mask_reduce_add_epi16(
+// CHECK-LABEL: test_mm_mask_reduce_add_epi16
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %{{.*}})
   return _mm_mask_reduce_add_epi16(__M, __W);
 }
 
 short test_mm_mask_reduce_mul_epi16(__mmask8 __M, __m128i __W){
-// CHECK-LABEL: @test_mm_mask_reduce_mul_epi16(
+// CHECK-LABEL: test_mm_mask_reduce_mul_epi16
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %{{.*}})
   return _mm_mask_reduce_mul_epi16(__M, __W);
 }
 
 short test_mm_mask_reduce_and_epi16(__mmask8 __M, __m128i __W){
-// CHECK-LABEL: @test_mm_mask_reduce_and_epi16(
+// CHECK-LABEL: test_mm_mask_reduce_and_epi16
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> %{{.*}}
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.and.v8i16(<8 x i16> %{{.*}}
   return _mm_mask_reduce_and_epi16(__M, __W);
 }
 
 short test_mm_mask_reduce_or_epi16(__mmask8 __M, __m128i __W){
-// CHECK-LABEL: @test_mm_mask_reduce_or_epi16(
+// CHECK-LABEL: test_mm_mask_reduce_or_epi16
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.or.v8i16(<8 x i16> %{{.*}})
   return _mm_mask_reduce_or_epi16(__M, __W);
 }
 
 short test_mm256_reduce_add_epi16(__m256i __W){
-// CHECK-LABEL: @test_mm256_reduce_add_epi16(
-// CHECK:    call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %{{.*}})
+// CHECK-LABEL: test_mm256_reduce_add_epi16
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %{{.*}})
   return _mm256_reduce_add_epi16(__W);
 }
 
 short test_mm256_reduce_mul_epi16(__m256i __W){
-// CHECK-LABEL: @test_mm256_reduce_mul_epi16(
-// CHECK:    call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %{{.*}})
+// CHECK-LABEL: test_mm256_reduce_mul_epi16
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %{{.*}})
   return _mm256_reduce_mul_epi16(__W);
 }
 
 short test_mm256_reduce_or_epi16(__m256i __W){
-// CHECK-LABEL: @test_mm256_reduce_or_epi16(
-// CHECK:    call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> %{{.*}})
+// CHECK-LABEL: test_mm256_reduce_or_epi16
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.or.v16i16(<16 x i16> %{{.*}})
   return _mm256_reduce_or_epi16(__W);
 }
 
 short test_mm256_reduce_and_epi16(__m256i __W){
-// CHECK-LABEL: @test_mm256_reduce_and_epi16(
-// CHECK:    call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> %{{.*}})
+// CHECK-LABEL: test_mm256_reduce_and_epi16
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.and.v16i16(<16 x i16> %{{.*}})
   return _mm256_reduce_and_epi16(__W);
 }
 
 short test_mm256_mask_reduce_add_epi16(__mmask16 __M, __m256i __W){
-// CHECK-LABEL: @test_mm256_mask_reduce_add_epi16(
+// CHECK-LABEL: test_mm256_mask_reduce_add_epi16
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %{{.*}})
   return _mm256_mask_reduce_add_epi16(__M, __W);
 }
 
 short test_mm256_mask_reduce_mul_epi16(__mmask16 __M, __m256i __W){
-// CHECK-LABEL: @test_mm256_mask_reduce_mul_epi16(
+// CHECK-LABEL: test_mm256_mask_reduce_mul_epi16
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %{{.*}})
   return _mm256_mask_reduce_mul_epi16(__M, __W);
 }
 
 short test_mm256_mask_reduce_and_epi16(__mmask16 __M, __m256i __W){
-// CHECK-LABEL: @test_mm256_mask_reduce_and_epi16(
+// CHECK-LABEL: test_mm256_mask_reduce_and_epi16
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.and.v16i16(<16 x i16> %{{.*}})
   return _mm256_mask_reduce_and_epi16(__M, __W);
 }
 
 short test_mm256_mask_reduce_or_epi16(__mmask16 __M, __m256i __W){
-// CHECK-LABEL: @test_mm256_mask_reduce_or_epi16(
+// CHECK-LABEL: test_mm256_mask_reduce_or_epi16
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.or.v16i16(<16 x i16> %{{.*}})
   return _mm256_mask_reduce_or_epi16(__M, __W);
 }
 
 signed char test_mm_reduce_add_epi8(__m128i __W){
-// CHECK-LABEL: @test_mm_reduce_add_epi8(
-// CHECK:    call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %{{.*}})
+// CHECK-LABEL: test_mm_reduce_add_epi8
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %{{.*}})
   return _mm_reduce_add_epi8(__W);
 }
 
 signed char test_mm_reduce_mul_epi8(__m128i __W){
-// CHECK-LABEL: @test_mm_reduce_mul_epi8(
-// CHECK:    call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %{{.*}})
+// CHECK-LABEL: test_mm_reduce_mul_epi8
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %{{.*}})
   return _mm_reduce_mul_epi8(__W);
 }
 
 signed char test_mm_reduce_and_epi8(__m128i __W){
-// CHECK-LABEL: @test_mm_reduce_and_epi8(
-// CHECK:    call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> %{{.*}})
+// CHECK-LABEL: test_mm_reduce_and_epi8
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.and.v16i8(<16 x i8> %{{.*}})
   return _mm_reduce_and_epi8(__W);
 }
 
 signed char test_mm_reduce_or_epi8(__m128i __W){
-// CHECK-LABEL: @test_mm_reduce_or_epi8(
-// CHECK:    call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %{{.*}})
+// CHECK-LABEL: test_mm_reduce_or_epi8
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %{{.*}})
   return _mm_reduce_or_epi8(__W);
 }
 
 signed char test_mm_mask_reduce_add_epi8(__mmask16 __M, __m128i __W){
-// CHECK-LABEL: @test_mm_mask_reduce_add_epi8(
+// CHECK-LABEL: test_mm_mask_reduce_add_epi8
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %{{.*}})
   return _mm_mask_reduce_add_epi8(__M, __W);
 }
 
 signed char test_mm_mask_reduce_mul_epi8(__mmask16 __M, __m128i __W){
-// CHECK-LABEL: @test_mm_mask_reduce_mul_epi8(
+// CHECK-LABEL: test_mm_mask_reduce_mul_epi8
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %{{.*}})
   return _mm_mask_reduce_mul_epi8(__M, __W);
 }
 
 signed char test_mm_mask_reduce_and_epi8(__mmask16 __M, __m128i __W){
-// CHECK-LABEL: @test_mm_mask_reduce_and_epi8(
+// CHECK-LABEL: test_mm_mask_reduce_and_epi8
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.and.v16i8(<16 x i8> %{{.*}})
   return _mm_mask_reduce_and_epi8(__M, __W);
 }
 
 signed char test_mm_mask_reduce_or_epi8(__mmask16 __M, __m128i __W){
-// CHECK-LABEL: @test_mm_mask_reduce_or_epi8(
+// CHECK-LABEL: test_mm_mask_reduce_or_epi8
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %{{.*}})
   return _mm_mask_reduce_or_epi8(__M, __W);
 }
 
 signed char test_mm256_reduce_add_epi8(__m256i __W){
-// CHECK-LABEL: @test_mm256_reduce_add_epi8(
-// CHECK:    call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %{{.*}})
+// CHECK-LABEL: test_mm256_reduce_add_epi8
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %{{.*}})
   return _mm256_reduce_add_epi8(__W);
 }
 
 signed char test_mm256_reduce_mul_epi8(__m256i __W){
-// CHECK-LABEL: @test_mm256_reduce_mul_epi8(
-// CHECK:    call i8 @llvm.vector.reduce.mul.v32i8(<32 x i8> %{{.*}})
+// CHECK-LABEL: test_mm256_reduce_mul_epi8
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.mul.v32i8(<32 x i8> %{{.*}})
   return _mm256_reduce_mul_epi8(__W);
 }
 
 signed char test_mm256_reduce_and_epi8(__m256i __W){
-// CHECK-LABEL: @test_mm256_reduce_and_epi8(
-// CHECK:    call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> %{{.*}})
+// CHECK-LABEL: test_mm256_reduce_and_epi8
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.and.v32i8(<32 x i8> %{{.*}})
   return _mm256_reduce_and_epi8(__W);
 }
 
 signed char test_mm256_reduce_or_epi8(__m256i __W){
-// CHECK-LABEL: @test_mm256_reduce_or_epi8(
-// CHECK:    call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> %{{.*}})
+// CHECK-LABEL: test_mm256_reduce_or_epi8
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.or.v32i8(<32 x i8> %{{.*}})
   return _mm256_reduce_or_epi8(__W);
 }
 
 signed char test_mm256_mask_reduce_add_epi8(__mmask32 __M, __m256i __W){
-// CHECK-LABEL: @test_mm256_mask_reduce_add_epi8(
+// CHECK-LABEL: test_mm256_mask_reduce_add_epi8
 // CHECK:    select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %{{.*}})
   return _mm256_mask_reduce_add_epi8(__M, __W);
 }
 
 signed char test_mm256_mask_reduce_mul_epi8(__mmask32 __M, __m256i __W){
-// CHECK-LABEL: @test_mm256_mask_reduce_mul_epi8(
+// CHECK-LABEL: test_mm256_mask_reduce_mul_epi8
 // CHECK:    select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.mul.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.mul.v32i8(<32 x i8> %{{.*}})
   return _mm256_mask_reduce_mul_epi8(__M, __W);
 }
 
 signed char test_mm256_mask_reduce_and_epi8(__mmask32 __M, __m256i __W){
-// CHECK-LABEL: @test_mm256_mask_reduce_and_epi8(
+// CHECK-LABEL: test_mm256_mask_reduce_and_epi8
 // CHECK:    select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.and.v32i8(<32 x i8> %{{.*}})
   return _mm256_mask_reduce_and_epi8(__M, __W);
 }
 
 signed char test_mm256_mask_reduce_or_epi8(__mmask32 __M, __m256i __W){
-// CHECK-LABEL: @test_mm256_mask_reduce_or_epi8(
+// CHECK-LABEL: test_mm256_mask_reduce_or_epi8
 // CHECK:    select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.or.v32i8(<32 x i8> %{{.*}})
   return _mm256_mask_reduce_or_epi8(__M, __W);
 }
 
 short test_mm_reduce_max_epi16(__m128i __W){
 // CHECK-LABEL: test_mm_reduce_max_epi16
-// CHECK:    call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %{{.*}})
   return _mm_reduce_max_epi16(__W);
 }
 
 short test_mm_reduce_min_epi16(__m128i __W){
 // CHECK-LABEL: test_mm_reduce_min_epi16
-// CHECK:    call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> %{{.*}})
   return _mm_reduce_min_epi16(__W);
 }
 
 unsigned short test_mm_reduce_max_epu16(__m128i __W){
 // CHECK-LABEL: test_mm_reduce_max_epu16
-// CHECK:    call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> %{{.*}})
   return _mm_reduce_max_epu16(__W);
 }
 
 unsigned short test_mm_reduce_min_epu16(__m128i __W){
 // CHECK-LABEL: test_mm_reduce_min_epu16
-// CHECK:    call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %{{.*}})
   return _mm_reduce_min_epu16(__W);
 }
 
 short test_mm_mask_reduce_max_epi16(__mmask8 __M, __m128i __W){
 // CHECK-LABEL: test_mm_mask_reduce_max_epi16
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %{{.*}})
   return _mm_mask_reduce_max_epi16(__M, __W);
 }
 
 short test_mm_mask_reduce_min_epi16(__mmask8 __M, __m128i __W){
 // CHECK-LABEL: test_mm_mask_reduce_min_epi16
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> %{{.*}})
   return _mm_mask_reduce_min_epi16(__M, __W);
 }
 
 unsigned short test_mm_mask_reduce_max_epu16(__mmask8 __M, __m128i __W){
 // CHECK-LABEL: test_mm_mask_reduce_max_epu16
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> %{{.*}})
   return _mm_mask_reduce_max_epu16(__M, __W);
 }
 
 unsigned short test_mm_mask_reduce_min_epu16(__mmask8 __M, __m128i __W){
 // CHECK-LABEL: test_mm_mask_reduce_min_epu16
 // CHECK:    select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %{{.*}})
   return _mm_mask_reduce_min_epu16(__M, __W);
 }
 
 short test_mm256_reduce_max_epi16(__m256i __W){
 // CHECK-LABEL: test_mm256_reduce_max_epi16
-// CHECK:    call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> %{{.*}})
   return _mm256_reduce_max_epi16(__W);
 }
 
 short test_mm256_reduce_min_epi16(__m256i __W){
 // CHECK-LABEL: test_mm256_reduce_min_epi16
-// CHECK:    call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> %{{.*}})
   return _mm256_reduce_min_epi16(__W);
 }
 
 unsigned short test_mm256_reduce_max_epu16(__m256i __W){
 // CHECK-LABEL: test_mm256_reduce_max_epu16
-// CHECK:    call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> %{{.*}})
   return _mm256_reduce_max_epu16(__W);
 }
 
 unsigned short test_mm256_reduce_min_epu16(__m256i __W){
 // CHECK-LABEL: test_mm256_reduce_min_epu16
-// CHECK:    call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %{{.*}})
   return _mm256_reduce_min_epu16(__W);
 }
 
 short test_mm256_mask_reduce_max_epi16(__mmask16 __M, __m256i __W){
 // CHECK-LABEL: test_mm256_mask_reduce_max_epi16
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> %{{.*}})
   return _mm256_mask_reduce_max_epi16(__M, __W);
 }
 
 short test_mm256_mask_reduce_min_epi16(__mmask16 __M, __m256i __W){
 // CHECK-LABEL: test_mm256_mask_reduce_min_epi16
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> %{{.*}})
   return _mm256_mask_reduce_min_epi16(__M, __W);
 }
 
 unsigned short test_mm256_mask_reduce_max_epu16(__mmask16 __M, __m256i __W){
 // CHECK-LABEL: test_mm256_mask_reduce_max_epu16
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> %{{.*}})
   return _mm256_mask_reduce_max_epu16(__M, __W);
 }
 
 unsigned short test_mm256_mask_reduce_min_epu16(__mmask16 __M, __m256i __W){
 // CHECK-LABEL: test_mm256_mask_reduce_min_epu16
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
-// CHECK:    call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %{{.*}})
+// CHECK:    call {{.*}}i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %{{.*}})
   return _mm256_mask_reduce_min_epu16(__M, __W);
 }
 
 signed char test_mm_reduce_max_epi8(__m128i __W){
 // CHECK-LABEL: test_mm_reduce_max_epi8
-// CHECK:    call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> %{{.*}})
   return _mm_reduce_max_epi8(__W);
 }
 
 signed char test_mm_reduce_min_epi8(__m128i __W){
 // CHECK-LABEL: test_mm_reduce_min_epi8
-// CHECK:    call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %{{.*}})
   return _mm_reduce_min_epi8(__W);
 }
 
 unsigned char test_mm_reduce_max_epu8(__m128i __W){
 // CHECK-LABEL: test_mm_reduce_max_epu8
-// CHECK:    call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %{{.*}})
   return _mm_reduce_max_epu8(__W);
 }
 
 unsigned char test_mm_reduce_min_epu8(__m128i __W){
 // CHECK-LABEL: test_mm_reduce_min_epu8
-// CHECK:    call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> %{{.*}})
   return _mm_reduce_min_epu8(__W);
 }
 
 signed char test_mm_mask_reduce_max_epi8(__mmask16 __M, __m128i __W){
 // CHECK-LABEL: test_mm_mask_reduce_max_epi8
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> %{{.*}})
   return _mm_mask_reduce_max_epi8(__M, __W);
 }
 
 signed char test_mm_mask_reduce_min_epi8(__mmask16 __M, __m128i __W){
 // CHECK-LABEL: test_mm_mask_reduce_min_epi8
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %{{.*}})
   return _mm_mask_reduce_min_epi8(__M, __W);
 }
 
 unsigned char test_mm_mask_reduce_max_epu8(__mmask16 __M, __m128i __W){
 // CHECK-LABEL: test_mm_mask_reduce_max_epu8
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %{{.*}})
   return _mm_mask_reduce_max_epu8(__M, __W);
 }
 
 unsigned char test_mm_mask_reduce_min_epu8(__mmask16 __M, __m128i __W){
 // CHECK-LABEL: test_mm_mask_reduce_min_epu8
 // CHECK:    select <16 x i1> %{{.*}}, <16 x i8> %{{.*}}, <16 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> %{{.*}})
   return _mm_mask_reduce_min_epu8(__M, __W);
 }
 
 signed char test_mm256_reduce_max_epi8(__m256i __W){
 // CHECK-LABEL: test_mm256_reduce_max_epi8
-// CHECK:    call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> %{{.*}})
   return _mm256_reduce_max_epi8(__W);
 }
 
 signed char test_mm256_reduce_min_epi8(__m256i __W){
 // CHECK-LABEL: test_mm256_reduce_min_epi8
-// CHECK:    call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> %{{.*}})
   return _mm256_reduce_min_epi8(__W);
 }
 
 unsigned char test_mm256_reduce_max_epu8(__m256i __W){
 // CHECK-LABEL: test_mm256_reduce_max_epu8
-// CHECK:    call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> %{{.*}})
   return _mm256_reduce_max_epu8(__W);
 }
 
 unsigned char test_mm256_reduce_min_epu8(__m256i __W){
 // CHECK-LABEL: test_mm256_reduce_min_epu8
-// CHECK:    call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> %{{.*}})
   return _mm256_reduce_min_epu8(__W);
 }
 
 signed char test_mm256_mask_reduce_max_epi8(__mmask32 __M, __m256i __W){
 // CHECK-LABEL: test_mm256_mask_reduce_max_epi8
 // CHECK:    select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> %{{.*}})
   return _mm256_mask_reduce_max_epi8(__M, __W);
 }
 
 signed char test_mm256_mask_reduce_min_epi8(__mmask32 __M, __m256i __W){
 // CHECK-LABEL: test_mm256_mask_reduce_min_epi8
 // CHECK:    select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> %{{.*}})
   return _mm256_mask_reduce_min_epi8(__M, __W);
 }
 
 unsigned char test_mm256_mask_reduce_max_epu8(__mmask32 __M, __m256i __W){
 // CHECK-LABEL: test_mm256_mask_reduce_max_epu8
 // CHECK:    select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> %{{.*}})
   return _mm256_mask_reduce_max_epu8(__M, __W);
 }
 
 unsigned char test_mm256_mask_reduce_min_epu8(__mmask32 __M, __m256i __W){
 // CHECK-LABEL: test_mm256_mask_reduce_min_epu8
 // CHECK:    select <32 x i1> %{{.*}}, <32 x i8> %{{.*}}, <32 x i8> %{{.*}}
-// CHECK:    call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> %{{.*}})
+// CHECK:    call {{.*}}i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> %{{.*}})
   return _mm256_mask_reduce_min_epu8(__M, __W);
 }
diff --git a/clang/test/CodeGen/X86/avx512vlcd-builtins.c b/clang/test/CodeGen/X86/avx512vlcd-builtins.c
index b784809810e3d..939fd6460e6a6 100644
--- a/clang/test/CodeGen/X86/avx512vlcd-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vlcd-builtins.c
@@ -1,10 +1,13 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vl -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vl -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vl -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vl -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vl -target-feature +avx512cd -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
 
 __m128i test_mm_broadcastmb_epi64(__m128i a,__m128i b) {
-  // CHECK-LABEL: @test_mm_broadcastmb_epi64
+  // CHECK-LABEL: test_mm_broadcastmb_epi64
   // CHECK: icmp eq <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: shufflevector <4 x i1> %{{.*}}, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   // CHECK: zext i8 %{{.*}} to i64
@@ -14,7 +17,7 @@ __m128i test_mm_broadcastmb_epi64(__m128i a,__m128i b) {
 }
 
 __m256i test_mm256_broadcastmb_epi64(__m256i a, __m256i b) {
-  // CHECK-LABEL: @test_mm256_broadcastmb_epi64
+  // CHECK-LABEL: test_mm256_broadcastmb_epi64
   // CHECK: icmp eq <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: shufflevector <4 x i1> %{{.*}}, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   // CHECK: zext i8 %{{.*}} to i64
@@ -26,7 +29,7 @@ __m256i test_mm256_broadcastmb_epi64(__m256i a, __m256i b) {
 }
 
 __m128i test_mm_broadcastmw_epi32(__m512i a, __m512i b) {
-  // CHECK-LABEL: @test_mm_broadcastmw_epi32
+  // CHECK-LABEL: test_mm_broadcastmw_epi32
   // CHECK: icmp eq <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: zext i16 %{{.*}} to i32
   // CHECK: insertelement <4 x i32> poison, i32 %{{.*}}, i32 0
@@ -37,7 +40,7 @@ __m128i test_mm_broadcastmw_epi32(__m512i a, __m512i b) {
 }
 
 __m256i test_mm256_broadcastmw_epi32(__m512i a, __m512i b) {
-  // CHECK-LABEL: @test_mm256_broadcastmw_epi32
+  // CHECK-LABEL: test_mm256_broadcastmw_epi32
   // CHECK: icmp eq <16 x i32> %{{.*}}, %{{.*}}
   // CHECK: zext i16 %{{.*}} to i32
   // CHECK: insertelement <8 x i32> poison, i32 %{{.*}}, i32 0
@@ -52,161 +55,161 @@ __m256i test_mm256_broadcastmw_epi32(__m512i a, __m512i b) {
 }
 
 __m128i test_mm_conflict_epi64(__m128i __A) {
-  // CHECK-LABEL: @test_mm_conflict_epi64
-  // CHECK: @llvm.x86.avx512.conflict.q.128
+  // CHECK-LABEL: test_mm_conflict_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %{{.*}})
   return _mm_conflict_epi64(__A); 
 }
 
 __m128i test_mm_mask_conflict_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_conflict_epi64
-  // CHECK: @llvm.x86.avx512.conflict.q.128
+  // CHECK-LABEL: test_mm_mask_conflict_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_conflict_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_conflict_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_conflict_epi64
-  // CHECK: @llvm.x86.avx512.conflict.q.128
+  // CHECK-LABEL: test_mm_maskz_conflict_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_conflict_epi64(__U, __A); 
 }
 
 __m256i test_mm256_conflict_epi64(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_conflict_epi64
-  // CHECK: @llvm.x86.avx512.conflict.q.256
+  // CHECK-LABEL: test_mm256_conflict_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %{{.*}})
   return _mm256_conflict_epi64(__A); 
 }
 
 __m256i test_mm256_mask_conflict_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_conflict_epi64
-  // CHECK: @llvm.x86.avx512.conflict.q.256
+  // CHECK-LABEL: test_mm256_mask_conflict_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_conflict_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_conflict_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_conflict_epi64
-  // CHECK: @llvm.x86.avx512.conflict.q.256
+  // CHECK-LABEL: test_mm256_maskz_conflict_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_conflict_epi64(__U, __A); 
 }
 
 __m128i test_mm_conflict_epi32(__m128i __A) {
-  // CHECK-LABEL: @test_mm_conflict_epi32
-  // CHECK: @llvm.x86.avx512.conflict.d.128
+  // CHECK-LABEL: test_mm_conflict_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %{{.*}})
   return _mm_conflict_epi32(__A); 
 }
 
 __m128i test_mm_mask_conflict_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_conflict_epi32
-  // CHECK: @llvm.x86.avx512.conflict.d.128
+  // CHECK-LABEL: test_mm_mask_conflict_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_conflict_epi32(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_conflict_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_conflict_epi32
-  // CHECK: @llvm.x86.avx512.conflict.d.128
+  // CHECK-LABEL: test_mm_maskz_conflict_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_conflict_epi32(__U, __A); 
 }
 
 __m256i test_mm256_conflict_epi32(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_conflict_epi32
-  // CHECK: @llvm.x86.avx512.conflict.d.256
+  // CHECK-LABEL: test_mm256_conflict_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %{{.*}})
   return _mm256_conflict_epi32(__A); 
 }
 
 __m256i test_mm256_mask_conflict_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_conflict_epi32
-  // CHECK: @llvm.x86.avx512.conflict.d.256
+  // CHECK-LABEL: test_mm256_mask_conflict_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_conflict_epi32(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_conflict_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_conflict_epi32
-  // CHECK: @llvm.x86.avx512.conflict.d.256
+  // CHECK-LABEL: test_mm256_maskz_conflict_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_conflict_epi32(__U, __A); 
 }
 
 __m128i test_mm_lzcnt_epi32(__m128i __A) {
-  // CHECK-LABEL: @test_mm_lzcnt_epi32
+  // CHECK-LABEL: test_mm_lzcnt_epi32
   // CHECK: call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %{{.*}}, i1 false)
   return _mm_lzcnt_epi32(__A); 
 }
 
 __m128i test_mm_mask_lzcnt_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_lzcnt_epi32
+  // CHECK-LABEL: test_mm_mask_lzcnt_epi32
   // CHECK: call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %{{.*}}, i1 false)
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_lzcnt_epi32(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_lzcnt_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_lzcnt_epi32
+  // CHECK-LABEL: test_mm_maskz_lzcnt_epi32
   // CHECK: call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %{{.*}}, i1 false)
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_lzcnt_epi32(__U, __A); 
 }
 
 __m256i test_mm256_lzcnt_epi32(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_lzcnt_epi32
+  // CHECK-LABEL: test_mm256_lzcnt_epi32
   // CHECK: call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %{{.*}}, i1 false)
   return _mm256_lzcnt_epi32(__A); 
 }
 
 __m256i test_mm256_mask_lzcnt_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_lzcnt_epi32
+  // CHECK-LABEL: test_mm256_mask_lzcnt_epi32
   // CHECK: call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %{{.*}}, i1 false)
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_lzcnt_epi32(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_lzcnt_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_lzcnt_epi32
+  // CHECK-LABEL: test_mm256_maskz_lzcnt_epi32
   // CHECK: call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %{{.*}}, i1 false)
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_lzcnt_epi32(__U, __A); 
 }
 
 __m128i test_mm_lzcnt_epi64(__m128i __A) {
-  // CHECK-LABEL: @test_mm_lzcnt_epi64
-  // CHECK: call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm_lzcnt_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.ctlz.v2i64(<2 x i64> %{{.*}}, i1 false)
   return _mm_lzcnt_epi64(__A); 
 }
 
 __m128i test_mm_mask_lzcnt_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_lzcnt_epi64
-  // CHECK: call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm_mask_lzcnt_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.ctlz.v2i64(<2 x i64> %{{.*}}, i1 false)
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_lzcnt_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_lzcnt_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_lzcnt_epi64
-  // CHECK: call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm_maskz_lzcnt_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.ctlz.v2i64(<2 x i64> %{{.*}}, i1 false)
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_lzcnt_epi64(__U, __A); 
 }
 
 __m256i test_mm256_lzcnt_epi64(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_lzcnt_epi64
-  // CHECK: call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm256_lzcnt_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.ctlz.v4i64(<4 x i64> %{{.*}}, i1 false)
   return _mm256_lzcnt_epi64(__A); 
 }
 
 __m256i test_mm256_mask_lzcnt_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_lzcnt_epi64
-  // CHECK: call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm256_mask_lzcnt_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.ctlz.v4i64(<4 x i64> %{{.*}}, i1 false)
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_lzcnt_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_lzcnt_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_lzcnt_epi64
-  // CHECK: call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %{{.*}}, i1 false)
+  // CHECK-LABEL: test_mm256_maskz_lzcnt_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.ctlz.v4i64(<4 x i64> %{{.*}}, i1 false)
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_lzcnt_epi64(__U, __A); 
 }
diff --git a/clang/test/CodeGen/X86/avx512vldq-builtins.c b/clang/test/CodeGen/X86/avx512vldq-builtins.c
index cdbd19a91211b..66ba0c704668c 100644
--- a/clang/test/CodeGen/X86/avx512vldq-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vldq-builtins.c
@@ -1,50 +1,54 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512dq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512dq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512dq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512dq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512dq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __m256i test_mm256_mullo_epi64 (__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mullo_epi64
+  // CHECK-LABEL: test_mm256_mullo_epi64
   // CHECK: mul <4 x i64>
   return _mm256_mullo_epi64(__A, __B);
 }
 
 __m256i test_mm256_mask_mullo_epi64 (__m256i __W, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_mullo_epi64
+  // CHECK-LABEL: test_mm256_mask_mullo_epi64
   // CHECK: mul <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return (__m256i) _mm256_mask_mullo_epi64 ( __W, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_mullo_epi64 (__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_mullo_epi64
+  // CHECK-LABEL: test_mm256_maskz_mullo_epi64
   // CHECK: mul <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return (__m256i) _mm256_maskz_mullo_epi64 (__U, __A, __B);
 }
 
 __m128i test_mm_mullo_epi64 (__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mullo_epi64
+  // CHECK-LABEL: test_mm_mullo_epi64
   // CHECK: mul <2 x i64>
   return (__m128i) _mm_mullo_epi64(__A, __B);
 }
 
 __m128i test_mm_mask_mullo_epi64 (__m128i __W, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_mullo_epi64
+  // CHECK-LABEL: test_mm_mask_mullo_epi64
   // CHECK: mul <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return (__m128i) _mm_mask_mullo_epi64 ( __W, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_mullo_epi64 (__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_mullo_epi64
+  // CHECK-LABEL: test_mm_maskz_mullo_epi64
   // CHECK: mul <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return (__m128i) _mm_maskz_mullo_epi64 (__U, __A, __B);
 }
 
 __m256d test_mm256_mask_andnot_pd (__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_andnot_pd
+  // CHECK-LABEL: test_mm256_mask_andnot_pd
   // CHECK: xor <4 x i64> %{{.*}}, splat (i64 -1)
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
@@ -52,7 +56,7 @@ __m256d test_mm256_mask_andnot_pd (__m256d __W, __mmask8 __U, __m256d __A, __m25
 }
 
 __m256d test_mm256_maskz_andnot_pd (__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_andnot_pd
+  // CHECK-LABEL: test_mm256_maskz_andnot_pd
   // CHECK: xor <4 x i64> %{{.*}}, splat (i64 -1)
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
@@ -60,7 +64,7 @@ __m256d test_mm256_maskz_andnot_pd (__mmask8 __U, __m256d __A, __m256d __B) {
 }
 
 __m128d test_mm_mask_andnot_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_andnot_pd
+  // CHECK-LABEL: test_mm_mask_andnot_pd
   // CHECK: xor <2 x i64> %{{.*}}, splat (i64 -1)
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
@@ -68,7 +72,7 @@ __m128d test_mm_mask_andnot_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d
 }
 
 __m128d test_mm_maskz_andnot_pd (__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_andnot_pd
+  // CHECK-LABEL: test_mm_maskz_andnot_pd
   // CHECK: xor <2 x i64> %{{.*}}, splat (i64 -1)
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
@@ -76,7 +80,7 @@ __m128d test_mm_maskz_andnot_pd (__mmask8 __U, __m128d __A, __m128d __B) {
 }
 
 __m256 test_mm256_mask_andnot_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_andnot_ps
+  // CHECK-LABEL: test_mm256_mask_andnot_ps
   // CHECK: xor <8 x i32> %{{.*}}, splat (i32 -1)
   // CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -84,7 +88,7 @@ __m256 test_mm256_mask_andnot_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 _
 }
 
 __m256 test_mm256_maskz_andnot_ps (__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_andnot_ps
+  // CHECK-LABEL: test_mm256_maskz_andnot_ps
   // CHECK: xor <8 x i32> %{{.*}}, splat (i32 -1)
   // CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
@@ -92,7 +96,7 @@ __m256 test_mm256_maskz_andnot_ps (__mmask8 __U, __m256 __A, __m256 __B) {
 }
 
 __m128 test_mm_mask_andnot_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_andnot_ps
+  // CHECK-LABEL: test_mm_mask_andnot_ps
   // CHECK: xor <4 x i32> %{{.*}}, splat (i32 -1)
   // CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -100,7 +104,7 @@ __m128 test_mm_mask_andnot_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
 }
 
 __m128 test_mm_maskz_andnot_ps (__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_andnot_ps
+  // CHECK-LABEL: test_mm_maskz_andnot_ps
   // CHECK: xor <4 x i32> %{{.*}}, splat (i32 -1)
   // CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -108,776 +112,776 @@ __m128 test_mm_maskz_andnot_ps (__mmask8 __U, __m128 __A, __m128 __B) {
 }
 
 __m256d test_mm256_mask_and_pd (__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_and_pd
+  // CHECK-LABEL: test_mm256_mask_and_pd
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return (__m256d) _mm256_mask_and_pd ( __W, __U, __A, __B);
 }
 
 __m256d test_mm256_maskz_and_pd (__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_and_pd
+  // CHECK-LABEL: test_mm256_maskz_and_pd
   // CHECK: and <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return (__m256d) _mm256_maskz_and_pd (__U, __A, __B);
 }
 
 __m128d test_mm_mask_and_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_and_pd
+  // CHECK-LABEL: test_mm_mask_and_pd
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return (__m128d) _mm_mask_and_pd ( __W, __U, __A, __B);
 }
 
 __m128d test_mm_maskz_and_pd (__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_and_pd
+  // CHECK-LABEL: test_mm_maskz_and_pd
   // CHECK: and <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return (__m128d) _mm_maskz_and_pd (__U, __A, __B);
 }
 
 __m256 test_mm256_mask_and_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_and_ps
+  // CHECK-LABEL: test_mm256_mask_and_ps
   // CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return (__m256) _mm256_mask_and_ps ( __W, __U, __A, __B);
 }
 
 __m256 test_mm256_maskz_and_ps (__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_and_ps
+  // CHECK-LABEL: test_mm256_maskz_and_ps
   // CHECK: and <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return (__m256) _mm256_maskz_and_ps (__U, __A, __B);
 }
 
 __m128 test_mm_mask_and_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_and_ps
+  // CHECK-LABEL: test_mm_mask_and_ps
   // CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return (__m128) _mm_mask_and_ps ( __W, __U, __A, __B);
 }
 
 __m128 test_mm_maskz_and_ps (__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_and_ps
+  // CHECK-LABEL: test_mm_maskz_and_ps
   // CHECK: and <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return (__m128) _mm_maskz_and_ps (__U, __A, __B);
 }
 
 __m256d test_mm256_mask_xor_pd (__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_xor_pd
+  // CHECK-LABEL: test_mm256_mask_xor_pd
   // CHECK: xor <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return (__m256d) _mm256_mask_xor_pd ( __W, __U, __A, __B);
 }
 
 __m256d test_mm256_maskz_xor_pd (__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_xor_pd
+  // CHECK-LABEL: test_mm256_maskz_xor_pd
   // CHECK: xor <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return (__m256d) _mm256_maskz_xor_pd (__U, __A, __B);
 }
 
 __m128d test_mm_mask_xor_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_xor_pd
+  // CHECK-LABEL: test_mm_mask_xor_pd
   // CHECK: xor <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return (__m128d) _mm_mask_xor_pd ( __W, __U, __A, __B);
 }
 
 __m128d test_mm_maskz_xor_pd (__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_xor_pd
+  // CHECK-LABEL: test_mm_maskz_xor_pd
   // CHECK: xor <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return (__m128d) _mm_maskz_xor_pd (__U, __A, __B);
 }
 
 __m256 test_mm256_mask_xor_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_xor_ps
+  // CHECK-LABEL: test_mm256_mask_xor_ps
   // CHECK: xor <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return (__m256) _mm256_mask_xor_ps ( __W, __U, __A, __B);
 }
 
 __m256 test_mm256_maskz_xor_ps (__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_xor_ps
+  // CHECK-LABEL: test_mm256_maskz_xor_ps
   // CHECK: xor <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return (__m256) _mm256_maskz_xor_ps (__U, __A, __B);
 }
 
 __m128 test_mm_mask_xor_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_xor_ps
+  // CHECK-LABEL: test_mm_mask_xor_ps
   // CHECK: xor <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return (__m128) _mm_mask_xor_ps ( __W, __U, __A, __B);
 }
 
 __m128 test_mm_maskz_xor_ps (__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_xor_ps
+  // CHECK-LABEL: test_mm_maskz_xor_ps
   // CHECK: xor <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return (__m128) _mm_maskz_xor_ps (__U, __A, __B);
 }
 
 __m256d test_mm256_mask_or_pd (__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_or_pd
+  // CHECK-LABEL: test_mm256_mask_or_pd
   // CHECK: or <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return (__m256d) _mm256_mask_or_pd ( __W, __U, __A, __B);
 }
 
 __m256d test_mm256_maskz_or_pd (__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_or_pd
+  // CHECK-LABEL: test_mm256_maskz_or_pd
   // CHECK: or <4 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return (__m256d) _mm256_maskz_or_pd (__U, __A, __B);
 }
 
 __m128d test_mm_mask_or_pd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_or_pd
+  // CHECK-LABEL: test_mm_mask_or_pd
   // CHECK: or <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return (__m128d) _mm_mask_or_pd ( __W, __U, __A, __B);
 }
 
 __m128d test_mm_maskz_or_pd (__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_or_pd
+  // CHECK-LABEL: test_mm_maskz_or_pd
   // CHECK: or <2 x i64> %{{.*}}, %{{.*}}
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return (__m128d) _mm_maskz_or_pd (__U, __A, __B);
 }
 
 __m256 test_mm256_mask_or_ps (__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_or_ps
+  // CHECK-LABEL: test_mm256_mask_or_ps
   // CHECK: or <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return (__m256) _mm256_mask_or_ps ( __W, __U, __A, __B);
 }
 
 __m256 test_mm256_maskz_or_ps (__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_or_ps
+  // CHECK-LABEL: test_mm256_maskz_or_ps
   // CHECK: or <8 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return (__m256) _mm256_maskz_or_ps (__U, __A, __B);
 }
 
 __m128 test_mm_mask_or_ps (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_or_ps
+  // CHECK-LABEL: test_mm_mask_or_ps
   // CHECK: or <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return (__m128) _mm_mask_or_ps ( __W, __U, __A, __B);
 }
 
 __m128 test_mm_maskz_or_ps (__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_or_ps
+  // CHECK-LABEL: test_mm_maskz_or_ps
   // CHECK: or <4 x i32> %{{.*}}, %{{.*}}
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return (__m128) _mm_maskz_or_ps(__U, __A, __B);
 }
 
 __m128i test_mm_cvtpd_epi64(__m128d __A) {
-  // CHECK-LABEL: @test_mm_cvtpd_epi64
+  // CHECK-LABEL: test_mm_cvtpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.128
   return _mm_cvtpd_epi64(__A); 
 }
 
 __m128i test_mm_mask_cvtpd_epi64(__m128i __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtpd_epi64
+  // CHECK-LABEL: test_mm_mask_cvtpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.128
   return _mm_mask_cvtpd_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtpd_epi64(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtpd_epi64
+  // CHECK-LABEL: test_mm_maskz_cvtpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.128
   return _mm_maskz_cvtpd_epi64(__U, __A); 
 }
 
 __m256i test_mm256_cvtpd_epi64(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_cvtpd_epi64
+  // CHECK-LABEL: test_mm256_cvtpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.256
   return _mm256_cvtpd_epi64(__A); 
 }
 
 __m256i test_mm256_mask_cvtpd_epi64(__m256i __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtpd_epi64
+  // CHECK-LABEL: test_mm256_mask_cvtpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.256
   return _mm256_mask_cvtpd_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtpd_epi64(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtpd_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvtpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2qq.256
   return _mm256_maskz_cvtpd_epi64(__U, __A); 
 }
 
 __m128i test_mm_cvtpd_epu64(__m128d __A) {
-  // CHECK-LABEL: @test_mm_cvtpd_epu64
+  // CHECK-LABEL: test_mm_cvtpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.128
   return _mm_cvtpd_epu64(__A); 
 }
 
 __m128i test_mm_mask_cvtpd_epu64(__m128i __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtpd_epu64
+  // CHECK-LABEL: test_mm_mask_cvtpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.128
   return _mm_mask_cvtpd_epu64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtpd_epu64(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtpd_epu64
+  // CHECK-LABEL: test_mm_maskz_cvtpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.128
   return _mm_maskz_cvtpd_epu64(__U, __A); 
 }
 
 __m256i test_mm256_cvtpd_epu64(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_cvtpd_epu64
+  // CHECK-LABEL: test_mm256_cvtpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.256
   return _mm256_cvtpd_epu64(__A); 
 }
 
 __m256i test_mm256_mask_cvtpd_epu64(__m256i __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtpd_epu64
+  // CHECK-LABEL: test_mm256_mask_cvtpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.256
   return _mm256_mask_cvtpd_epu64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtpd_epu64(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtpd_epu64
+  // CHECK-LABEL: test_mm256_maskz_cvtpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtpd2uqq.256
   return _mm256_maskz_cvtpd_epu64(__U, __A); 
 }
 
 __m128i test_mm_cvtps_epi64(__m128 __A) {
-  // CHECK-LABEL: @test_mm_cvtps_epi64
+  // CHECK-LABEL: test_mm_cvtps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.128
   return _mm_cvtps_epi64(__A); 
 }
 
 __m128i test_mm_mask_cvtps_epi64(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtps_epi64
+  // CHECK-LABEL: test_mm_mask_cvtps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.128
   return _mm_mask_cvtps_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtps_epi64(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtps_epi64
+  // CHECK-LABEL: test_mm_maskz_cvtps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.128
   return _mm_maskz_cvtps_epi64(__U, __A); 
 }
 
 __m256i test_mm256_cvtps_epi64(__m128 __A) {
-  // CHECK-LABEL: @test_mm256_cvtps_epi64
+  // CHECK-LABEL: test_mm256_cvtps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.256
   return _mm256_cvtps_epi64(__A); 
 }
 
 __m256i test_mm256_mask_cvtps_epi64(__m256i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtps_epi64
+  // CHECK-LABEL: test_mm256_mask_cvtps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.256
   return _mm256_mask_cvtps_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtps_epi64(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtps_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvtps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvtps2qq.256
   return _mm256_maskz_cvtps_epi64(__U, __A); 
 }
 
 __m128i test_mm_cvtps_epu64(__m128 __A) {
-  // CHECK-LABEL: @test_mm_cvtps_epu64
+  // CHECK-LABEL: test_mm_cvtps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.128
   return _mm_cvtps_epu64(__A); 
 }
 
 __m128i test_mm_mask_cvtps_epu64(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtps_epu64
+  // CHECK-LABEL: test_mm_mask_cvtps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.128
   return _mm_mask_cvtps_epu64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvtps_epu64(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtps_epu64
+  // CHECK-LABEL: test_mm_maskz_cvtps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.128
   return _mm_maskz_cvtps_epu64(__U, __A); 
 }
 
 __m256i test_mm256_cvtps_epu64(__m128 __A) {
-  // CHECK-LABEL: @test_mm256_cvtps_epu64
+  // CHECK-LABEL: test_mm256_cvtps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.256
   return _mm256_cvtps_epu64(__A); 
 }
 
 __m256i test_mm256_mask_cvtps_epu64(__m256i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtps_epu64
+  // CHECK-LABEL: test_mm256_mask_cvtps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.256
   return _mm256_mask_cvtps_epu64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvtps_epu64(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtps_epu64
+  // CHECK-LABEL: test_mm256_maskz_cvtps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvtps2uqq.256
   return _mm256_maskz_cvtps_epu64(__U, __A); 
 }
 
 __m128d test_mm_cvtepi64_pd(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepi64_pd
+  // CHECK-LABEL: test_mm_cvtepi64_pd
   // CHECK: sitofp <2 x i64> %{{.*}} to <2 x double>
   return _mm_cvtepi64_pd(__A); 
 }
 
 __m128d test_mm_mask_cvtepi64_pd(__m128d __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi64_pd
+  // CHECK-LABEL: test_mm_mask_cvtepi64_pd
   // CHECK: sitofp <2 x i64> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_cvtepi64_pd(__W, __U, __A); 
 }
 
 __m128d test_mm_maskz_cvtepi64_pd(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi64_pd
+  // CHECK-LABEL: test_mm_maskz_cvtepi64_pd
   // CHECK: sitofp <2 x i64> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_cvtepi64_pd(__U, __A); 
 }
 
 __m256d test_mm256_cvtepi64_pd(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepi64_pd
+  // CHECK-LABEL: test_mm256_cvtepi64_pd
   // CHECK: sitofp <4 x i64> %{{.*}} to <4 x double>
   return _mm256_cvtepi64_pd(__A); 
 }
 
 __m256d test_mm256_mask_cvtepi64_pd(__m256d __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi64_pd
+  // CHECK-LABEL: test_mm256_mask_cvtepi64_pd
   // CHECK: sitofp <4 x i64> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_cvtepi64_pd(__W, __U, __A); 
 }
 
 __m256d test_mm256_maskz_cvtepi64_pd(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi64_pd
+  // CHECK-LABEL: test_mm256_maskz_cvtepi64_pd
   // CHECK: sitofp <4 x i64> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_cvtepi64_pd(__U, __A); 
 }
 
 __m128 test_mm_cvtepi64_ps(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepi64_ps
+  // CHECK-LABEL: test_mm_cvtepi64_ps
   // CHECK: @llvm.x86.avx512.mask.cvtqq2ps.128
   return _mm_cvtepi64_ps(__A); 
 }
 
 __m128 test_mm_mask_cvtepi64_ps(__m128 __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepi64_ps
+  // CHECK-LABEL: test_mm_mask_cvtepi64_ps
   // CHECK: @llvm.x86.avx512.mask.cvtqq2ps.128
   return _mm_mask_cvtepi64_ps(__W, __U, __A); 
 }
 
 __m128 test_mm_maskz_cvtepi64_ps(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepi64_ps
+  // CHECK-LABEL: test_mm_maskz_cvtepi64_ps
   // CHECK: @llvm.x86.avx512.mask.cvtqq2ps.128
   return _mm_maskz_cvtepi64_ps(__U, __A); 
 }
 
 __m128 test_mm256_cvtepi64_ps(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepi64_ps
+  // CHECK-LABEL: test_mm256_cvtepi64_ps
   // CHECK: sitofp <4 x i64> %{{.*}} to <4 x float>
   return _mm256_cvtepi64_ps(__A); 
 }
 
 __m128 test_mm256_mask_cvtepi64_ps(__m128 __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepi64_ps
+  // CHECK-LABEL: test_mm256_mask_cvtepi64_ps
   // CHECK: sitofp <4 x i64> %{{.*}} to <4 x float>
   // select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm256_mask_cvtepi64_ps(__W, __U, __A); 
 }
 
 __m128 test_mm256_maskz_cvtepi64_ps(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepi64_ps
+  // CHECK-LABEL: test_mm256_maskz_cvtepi64_ps
   // CHECK: sitofp <4 x i64> %{{.*}} to <4 x float>
   // select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm256_maskz_cvtepi64_ps(__U, __A); 
 }
 
 __m128i test_mm_cvttpd_epi64(__m128d __A) {
-  // CHECK-LABEL: @test_mm_cvttpd_epi64
+  // CHECK-LABEL: test_mm_cvttpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.128
   return _mm_cvttpd_epi64(__A); 
 }
 
 __m128i test_mm_mask_cvttpd_epi64(__m128i __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_cvttpd_epi64
+  // CHECK-LABEL: test_mm_mask_cvttpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.128
   return _mm_mask_cvttpd_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvttpd_epi64(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvttpd_epi64
+  // CHECK-LABEL: test_mm_maskz_cvttpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.128
   return _mm_maskz_cvttpd_epi64(__U, __A); 
 }
 
 __m256i test_mm256_cvttpd_epi64(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_cvttpd_epi64
+  // CHECK-LABEL: test_mm256_cvttpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.256
   return _mm256_cvttpd_epi64(__A); 
 }
 
 __m256i test_mm256_mask_cvttpd_epi64(__m256i __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvttpd_epi64
+  // CHECK-LABEL: test_mm256_mask_cvttpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.256
   return _mm256_mask_cvttpd_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvttpd_epi64(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvttpd_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvttpd_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2qq.256
   return _mm256_maskz_cvttpd_epi64(__U, __A); 
 }
 
 __m128i test_mm_cvttpd_epu64(__m128d __A) {
-  // CHECK-LABEL: @test_mm_cvttpd_epu64
+  // CHECK-LABEL: test_mm_cvttpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.128
   return _mm_cvttpd_epu64(__A); 
 }
 
 __m128i test_mm_mask_cvttpd_epu64(__m128i __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_cvttpd_epu64
+  // CHECK-LABEL: test_mm_mask_cvttpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.128
   return _mm_mask_cvttpd_epu64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvttpd_epu64(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvttpd_epu64
+  // CHECK-LABEL: test_mm_maskz_cvttpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.128
   return _mm_maskz_cvttpd_epu64(__U, __A); 
 }
 
 __m256i test_mm256_cvttpd_epu64(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_cvttpd_epu64
+  // CHECK-LABEL: test_mm256_cvttpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.256
   return _mm256_cvttpd_epu64(__A); 
 }
 
 __m256i test_mm256_mask_cvttpd_epu64(__m256i __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvttpd_epu64
+  // CHECK-LABEL: test_mm256_mask_cvttpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.256
   return _mm256_mask_cvttpd_epu64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvttpd_epu64(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvttpd_epu64
+  // CHECK-LABEL: test_mm256_maskz_cvttpd_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttpd2uqq.256
   return _mm256_maskz_cvttpd_epu64(__U, __A); 
 }
 
 __m128i test_mm_cvttps_epi64(__m128 __A) {
-  // CHECK-LABEL: @test_mm_cvttps_epi64
+  // CHECK-LABEL: test_mm_cvttps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.128
   return _mm_cvttps_epi64(__A); 
 }
 
 __m128i test_mm_mask_cvttps_epi64(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvttps_epi64
+  // CHECK-LABEL: test_mm_mask_cvttps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.128
   return _mm_mask_cvttps_epi64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvttps_epi64(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvttps_epi64
+  // CHECK-LABEL: test_mm_maskz_cvttps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.128
   return _mm_maskz_cvttps_epi64(__U, __A); 
 }
 
 __m256i test_mm256_cvttps_epi64(__m128 __A) {
-  // CHECK-LABEL: @test_mm256_cvttps_epi64
+  // CHECK-LABEL: test_mm256_cvttps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.256
   return _mm256_cvttps_epi64(__A); 
 }
 
 __m256i test_mm256_mask_cvttps_epi64(__m256i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvttps_epi64
+  // CHECK-LABEL: test_mm256_mask_cvttps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.256
   return _mm256_mask_cvttps_epi64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvttps_epi64(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvttps_epi64
+  // CHECK-LABEL: test_mm256_maskz_cvttps_epi64
   // CHECK: @llvm.x86.avx512.mask.cvttps2qq.256
   return _mm256_maskz_cvttps_epi64(__U, __A); 
 }
 
 __m128i test_mm_cvttps_epu64(__m128 __A) {
-  // CHECK-LABEL: @test_mm_cvttps_epu64
+  // CHECK-LABEL: test_mm_cvttps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.128
   return _mm_cvttps_epu64(__A); 
 }
 
 __m128i test_mm_mask_cvttps_epu64(__m128i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_cvttps_epu64
+  // CHECK-LABEL: test_mm_mask_cvttps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.128
   return _mm_mask_cvttps_epu64(__W, __U, __A); 
 }
 
 __m128i test_mm_maskz_cvttps_epu64(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvttps_epu64
+  // CHECK-LABEL: test_mm_maskz_cvttps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.128
   return _mm_maskz_cvttps_epu64(__U, __A); 
 }
 
 __m256i test_mm256_cvttps_epu64(__m128 __A) {
-  // CHECK-LABEL: @test_mm256_cvttps_epu64
+  // CHECK-LABEL: test_mm256_cvttps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.256
   return _mm256_cvttps_epu64(__A); 
 }
 
 __m256i test_mm256_mask_cvttps_epu64(__m256i __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvttps_epu64
+  // CHECK-LABEL: test_mm256_mask_cvttps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.256
   return _mm256_mask_cvttps_epu64(__W, __U, __A); 
 }
 
 __m256i test_mm256_maskz_cvttps_epu64(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvttps_epu64
+  // CHECK-LABEL: test_mm256_maskz_cvttps_epu64
   // CHECK: @llvm.x86.avx512.mask.cvttps2uqq.256
   return _mm256_maskz_cvttps_epu64(__U, __A); 
 }
 
 __m128d test_mm_cvtepu64_pd(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepu64_pd
+  // CHECK-LABEL: test_mm_cvtepu64_pd
   // CHECK: uitofp <2 x i64> %{{.*}} to <2 x double>
   return _mm_cvtepu64_pd(__A); 
 }
 
 __m128d test_mm_mask_cvtepu64_pd(__m128d __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu64_pd
+  // CHECK-LABEL: test_mm_mask_cvtepu64_pd
   // CHECK: uitofp <2 x i64> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_mask_cvtepu64_pd(__W, __U, __A); 
 }
 
 __m128d test_mm_maskz_cvtepu64_pd(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu64_pd
+  // CHECK-LABEL: test_mm_maskz_cvtepu64_pd
   // CHECK: uitofp <2 x i64> %{{.*}} to <2 x double>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm_maskz_cvtepu64_pd(__U, __A); 
 }
 
 __m256d test_mm256_cvtepu64_pd(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepu64_pd
+  // CHECK-LABEL: test_mm256_cvtepu64_pd
   // CHECK: uitofp <4 x i64> %{{.*}} to <4 x double>
   return _mm256_cvtepu64_pd(__A); 
 }
 
 __m256d test_mm256_mask_cvtepu64_pd(__m256d __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu64_pd
+  // CHECK-LABEL: test_mm256_mask_cvtepu64_pd
   // CHECK: uitofp <4 x i64> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_cvtepu64_pd(__W, __U, __A); 
 }
 
 __m256d test_mm256_maskz_cvtepu64_pd(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu64_pd
+  // CHECK-LABEL: test_mm256_maskz_cvtepu64_pd
   // CHECK: uitofp <4 x i64> %{{.*}} to <4 x double>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_cvtepu64_pd(__U, __A); 
 }
 
 __m128 test_mm_cvtepu64_ps(__m128i __A) {
-  // CHECK-LABEL: @test_mm_cvtepu64_ps
+  // CHECK-LABEL: test_mm_cvtepu64_ps
   // CHECK: @llvm.x86.avx512.mask.cvtuqq2ps.128
   return _mm_cvtepu64_ps(__A); 
 }
 
 __m128 test_mm_mask_cvtepu64_ps(__m128 __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_cvtepu64_ps
+  // CHECK-LABEL: test_mm_mask_cvtepu64_ps
   // CHECK: @llvm.x86.avx512.mask.cvtuqq2ps.128
   return _mm_mask_cvtepu64_ps(__W, __U, __A); 
 }
 
 __m128 test_mm_maskz_cvtepu64_ps(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_cvtepu64_ps
+  // CHECK-LABEL: test_mm_maskz_cvtepu64_ps
   // CHECK: @llvm.x86.avx512.mask.cvtuqq2ps.128
   return _mm_maskz_cvtepu64_ps(__U, __A); 
 }
 
 __m128 test_mm256_cvtepu64_ps(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_cvtepu64_ps
+  // CHECK-LABEL: test_mm256_cvtepu64_ps
   // CHECK: uitofp <4 x i64> %{{.*}} to <4 x float>
   return _mm256_cvtepu64_ps(__A); 
 }
 
 __m128 test_mm256_mask_cvtepu64_ps(__m128 __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_cvtepu64_ps
+  // CHECK-LABEL: test_mm256_mask_cvtepu64_ps
   // CHECK: uitofp <4 x i64> %{{.*}} to <4 x float>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm256_mask_cvtepu64_ps(__W, __U, __A); 
 }
 
 __m128 test_mm256_maskz_cvtepu64_ps(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_cvtepu64_ps
+  // CHECK-LABEL: test_mm256_maskz_cvtepu64_ps
   // CHECK: uitofp <4 x i64> %{{.*}} to <4 x float>
   // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm256_maskz_cvtepu64_ps(__U, __A); 
 }
 
 __m128d test_mm_range_pd(__m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_range_pd
+  // CHECK-LABEL: test_mm_range_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.128
   return _mm_range_pd(__A, __B, 4); 
 }
 
 __m128d test_mm_mask_range_pd(__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_mask_range_pd
+  // CHECK-LABEL: test_mm_mask_range_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.128
   return _mm_mask_range_pd(__W, __U, __A, __B, 4); 
 }
 
 __m128d test_mm_maskz_range_pd(__mmask8 __U, __m128d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm_maskz_range_pd
+  // CHECK-LABEL: test_mm_maskz_range_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.128
   return _mm_maskz_range_pd(__U, __A, __B, 4); 
 }
 
 __m256d test_mm256_range_pd(__m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_range_pd
+  // CHECK-LABEL: test_mm256_range_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.256
   return _mm256_range_pd(__A, __B, 4); 
 }
 
 __m256d test_mm256_mask_range_pd(__m256d __W, __mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_mask_range_pd
+  // CHECK-LABEL: test_mm256_mask_range_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.256
   return _mm256_mask_range_pd(__W, __U, __A, __B, 4); 
 }
 
 __m256d test_mm256_maskz_range_pd(__mmask8 __U, __m256d __A, __m256d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_range_pd
+  // CHECK-LABEL: test_mm256_maskz_range_pd
   // CHECK: @llvm.x86.avx512.mask.range.pd.256
   return _mm256_maskz_range_pd(__U, __A, __B, 4); 
 }
 
 __m128 test_mm_range_ps(__m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_range_ps
+  // CHECK-LABEL: test_mm_range_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.128
   return _mm_range_ps(__A, __B, 4); 
 }
 
 __m128 test_mm_mask_range_ps(__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_mask_range_ps
+  // CHECK-LABEL: test_mm_mask_range_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.128
   return _mm_mask_range_ps(__W, __U, __A, __B, 4); 
 }
 
 __m128 test_mm_maskz_range_ps(__mmask8 __U, __m128 __A, __m128 __B) {
-  // CHECK-LABEL: @test_mm_maskz_range_ps
+  // CHECK-LABEL: test_mm_maskz_range_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.128
   return _mm_maskz_range_ps(__U, __A, __B, 4); 
 }
 
 __m256 test_mm256_range_ps(__m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_range_ps
+  // CHECK-LABEL: test_mm256_range_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.256
   return _mm256_range_ps(__A, __B, 4); 
 }
 
 __m256 test_mm256_mask_range_ps(__m256 __W, __mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_mask_range_ps
+  // CHECK-LABEL: test_mm256_mask_range_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.256
   return _mm256_mask_range_ps(__W, __U, __A, __B, 4); 
 }
 
 __m256 test_mm256_maskz_range_ps(__mmask8 __U, __m256 __A, __m256 __B) {
-  // CHECK-LABEL: @test_mm256_maskz_range_ps
+  // CHECK-LABEL: test_mm256_maskz_range_ps
   // CHECK: @llvm.x86.avx512.mask.range.ps.256
   return _mm256_maskz_range_ps(__U, __A, __B, 4); 
 }
 
 __m128d test_mm_reduce_pd(__m128d __A) {
-  // CHECK-LABEL: @test_mm_reduce_pd
+  // CHECK-LABEL: test_mm_reduce_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.128
   return _mm_reduce_pd(__A, 4); 
 }
 
 __m128d test_mm_mask_reduce_pd(__m128d __W, __mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_reduce_pd
+  // CHECK-LABEL: test_mm_mask_reduce_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.128
   return _mm_mask_reduce_pd(__W, __U, __A, 4); 
 }
 
 __m128d test_mm_maskz_reduce_pd(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_maskz_reduce_pd
+  // CHECK-LABEL: test_mm_maskz_reduce_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.128
   return _mm_maskz_reduce_pd(__U, __A, 4); 
 }
 
 __m256d test_mm256_reduce_pd(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_reduce_pd
+  // CHECK-LABEL: test_mm256_reduce_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.256
   return _mm256_reduce_pd(__A, 4); 
 }
 
 __m256d test_mm256_mask_reduce_pd(__m256d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_reduce_pd
+  // CHECK-LABEL: test_mm256_mask_reduce_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.256
   return _mm256_mask_reduce_pd(__W, __U, __A, 4); 
 }
 
 __m256d test_mm256_maskz_reduce_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_reduce_pd
+  // CHECK-LABEL: test_mm256_maskz_reduce_pd
   // CHECK: @llvm.x86.avx512.mask.reduce.pd.256
   return _mm256_maskz_reduce_pd(__U, __A, 4); 
 }
 
 __m128 test_mm_reduce_ps(__m128 __A) {
-  // CHECK-LABEL: @test_mm_reduce_ps
+  // CHECK-LABEL: test_mm_reduce_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.128
   return _mm_reduce_ps(__A, 4); 
 }
 
 __m128 test_mm_mask_reduce_ps(__m128 __W, __mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_reduce_ps
+  // CHECK-LABEL: test_mm_mask_reduce_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.128
   return _mm_mask_reduce_ps(__W, __U, __A, 4); 
 }
 
 __m128 test_mm_maskz_reduce_ps(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_maskz_reduce_ps
+  // CHECK-LABEL: test_mm_maskz_reduce_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.128
   return _mm_maskz_reduce_ps(__U, __A, 4); 
 }
 
 __m256 test_mm256_reduce_ps(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_reduce_ps
+  // CHECK-LABEL: test_mm256_reduce_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.256
   return _mm256_reduce_ps(__A, 4); 
 }
 
 __m256 test_mm256_mask_reduce_ps(__m256 __W, __mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_reduce_ps
+  // CHECK-LABEL: test_mm256_mask_reduce_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.256
   return _mm256_mask_reduce_ps(__W, __U, __A, 4); 
 }
 
 __m256 test_mm256_maskz_reduce_ps(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_reduce_ps
+  // CHECK-LABEL: test_mm256_maskz_reduce_ps
   // CHECK: @llvm.x86.avx512.mask.reduce.ps.256
   return _mm256_maskz_reduce_ps(__U, __A, 4); 
 }
 
 __mmask8 test_mm_movepi32_mask(__m128i __A) {
-  // CHECK-LABEL: @test_mm_movepi32_mask
+  // CHECK-LABEL: test_mm_movepi32_mask
   // CHECK: [[CMP:%.*]] = icmp slt <4 x i32> %{{.*}}, zeroinitializer
   // CHECK: [[SHUF:%.*]] = shufflevector <4 x i1> [[CMP]], <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   return _mm_movepi32_mask(__A); 
 }
 
 __mmask8 test_mm256_movepi32_mask(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_movepi32_mask
+  // CHECK-LABEL: test_mm256_movepi32_mask
   // CHECK: [[CMP:%.*]] = icmp slt <8 x i32> %{{.*}}, zeroinitializer
   return _mm256_movepi32_mask(__A); 
 }
 
 __m128i test_mm_movm_epi32(__mmask8 __A) {
-  // CHECK-LABEL: @test_mm_movm_epi32
+  // CHECK-LABEL: test_mm_movm_epi32
   // CHECK: %{{.*}} = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: %extract.i = shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: %vpmovm2.i = sext <4 x i1> %extract.i to <4 x i32>
@@ -885,14 +889,14 @@ __m128i test_mm_movm_epi32(__mmask8 __A) {
 }
 
 __m256i test_mm256_movm_epi32(__mmask8 __A) {
-  // CHECK-LABEL: @test_mm256_movm_epi32
+  // CHECK-LABEL: test_mm256_movm_epi32
   // CHECK: %{{.*}} = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: %vpmovm2.i = sext <8 x i1> %{{.*}} to <8 x i32>
   return _mm256_movm_epi32(__A); 
 }
 
 __m128i test_mm_movm_epi64(__mmask8 __A) {
-  // CHECK-LABEL: @test_mm_movm_epi64
+  // CHECK-LABEL: test_mm_movm_epi64
   // CHECK: %{{.*}} = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: %extract.i = shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <2 x i32> <i32 0, i32 1>
   // CHECK: %vpmovm2.i = sext <2 x i1> %extract.i to <2 x i64>
@@ -900,7 +904,7 @@ __m128i test_mm_movm_epi64(__mmask8 __A) {
 }
 
 __m256i test_mm256_movm_epi64(__mmask8 __A) {
-  // CHECK-LABEL: @test_mm256_movm_epi64
+  // CHECK-LABEL: test_mm256_movm_epi64
   // CHECK: %{{.*}} = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK: %extract.i = shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: %vpmovm2.i = sext <4 x i1> %extract.i to <4 x i64>
@@ -908,14 +912,14 @@ __m256i test_mm256_movm_epi64(__mmask8 __A) {
 }
 
 __mmask8 test_mm_movepi64_mask(__m128i __A) {
-  // CHECK-LABEL: @test_mm_movepi64_mask
+  // CHECK-LABEL: test_mm_movepi64_mask
   // CHECK: [[CMP:%.*]] = icmp slt <2 x i64> %{{.*}}, zeroinitializer
   // CHECK: [[SHUF:%.*]] = shufflevector <2 x i1> [[CMP]], <2 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 2, i32 3>
   return _mm_movepi64_mask(__A); 
 }
 
 __mmask8 test_mm256_movepi64_mask(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_movepi64_mask
+  // CHECK-LABEL: test_mm256_movepi64_mask
   // CHECK: [[CMP:%.*]] = icmp slt <4 x i64> %{{.*}}, zeroinitializer
   // CHECK: [[SHUF:%.*]] = shufflevector <4 x i1> [[CMP]], <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   return _mm256_movepi64_mask(__A); 
@@ -923,229 +927,234 @@ __mmask8 test_mm256_movepi64_mask(__m256i __A) {
 
 
 __m256 test_mm256_broadcast_f32x2(__m128 __A) {
-  // CHECK-LABEL: @test_mm256_broadcast_f32x2
+  // CHECK-LABEL: test_mm256_broadcast_f32x2
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   return _mm256_broadcast_f32x2(__A); 
 }
+TEST_CONSTEXPR(match_m256(_mm256_broadcast_f32x2((__m128)(__v4sf){1.0f, -2.0f, 3.0f, -4.0f}), 1.0f, -2.0f, 1.0f, -2.0f, 1.0f, -2.0f, 1.0f, -2.0f));
 
 __m256 test_mm256_mask_broadcast_f32x2(__m256 __O, __mmask8 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcast_f32x2
+  // CHECK-LABEL: test_mm256_mask_broadcast_f32x2
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_broadcast_f32x2(__O, __M, __A); 
 }
 
 __m256 test_mm256_maskz_broadcast_f32x2(__mmask8 __M, __m128 __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcast_f32x2
+  // CHECK-LABEL: test_mm256_maskz_broadcast_f32x2
   // CHECK: shufflevector <4 x float> %{{.*}}, <4 x float> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_maskz_broadcast_f32x2(__M, __A); 
 }
 
 __m256d test_mm256_broadcast_f64x2(double const* __A) {
-  // CHECK-LABEL: @test_mm256_broadcast_f64x2
+  // CHECK-LABEL: test_mm256_broadcast_f64x2
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   return _mm256_broadcast_f64x2(_mm_loadu_pd(__A)); 
 }
+TEST_CONSTEXPR(match_m256d(_mm256_broadcast_f64x2((__m128d)(__v2df){1.0, -2.0}), 1.0, -2.0, 1.0, -2.0));
 
 __m256d test_mm256_mask_broadcast_f64x2(__m256d __O, __mmask8 __M, double const* __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcast_f64x2
+  // CHECK-LABEL: test_mm256_mask_broadcast_f64x2
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_broadcast_f64x2(__O, __M, _mm_loadu_pd(__A)); 
 }
 
 __m256d test_mm256_maskz_broadcast_f64x2(__mmask8 __M, double const* __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcast_f64x2
+  // CHECK-LABEL: test_mm256_maskz_broadcast_f64x2
   // CHECK: shufflevector <2 x double> %{{.*}}, <2 x double> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_broadcast_f64x2(__M, _mm_loadu_pd(__A)); 
 }
 
 __m128i test_mm_broadcast_i32x2(__m128i __A) {
-  // CHECK-LABEL: @test_mm_broadcast_i32x2
+  // CHECK-LABEL: test_mm_broadcast_i32x2
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   return _mm_broadcast_i32x2(__A); 
 }
+TEST_CONSTEXPR(match_v4si(_mm_broadcast_i32x2((__m128i)(__v4si){1, -2, 3, -4}), 1, -2, 1, -2));
 
 __m128i test_mm_mask_broadcast_i32x2(__m128i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_broadcast_i32x2
+  // CHECK-LABEL: test_mm_mask_broadcast_i32x2
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_broadcast_i32x2(__O, __M, __A); 
 }
 
 __m128i test_mm_maskz_broadcast_i32x2(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_broadcast_i32x2
+  // CHECK-LABEL: test_mm_maskz_broadcast_i32x2
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_broadcast_i32x2(__M, __A); 
 }
 
 __m256i test_mm256_broadcast_i32x2(__m128i __A) {
-  // CHECK-LABEL: @test_mm256_broadcast_i32x2
+  // CHECK-LABEL: test_mm256_broadcast_i32x2
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   return _mm256_broadcast_i32x2(__A); 
 }
+TEST_CONSTEXPR(match_v8si(_mm256_broadcast_i32x2((__m128i)(__v4si){1, -2, 3, -4}), 1, -2, 1, -2, 1, -2, 1, -2));
 
 __m256i test_mm256_mask_broadcast_i32x2(__m256i __O, __mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcast_i32x2
+  // CHECK-LABEL: test_mm256_mask_broadcast_i32x2
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_broadcast_i32x2(__O, __M, __A); 
 }
 
 __m256i test_mm256_maskz_broadcast_i32x2(__mmask8 __M, __m128i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcast_i32x2
+  // CHECK-LABEL: test_mm256_maskz_broadcast_i32x2
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_broadcast_i32x2(__M, __A); 
 }
 
 __m256i test_mm256_broadcast_i64x2(__m128i const* __A) {
-  // CHECK-LABEL: @test_mm256_broadcast_i64x2
+  // CHECK-LABEL: test_mm256_broadcast_i64x2
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   return _mm256_broadcast_i64x2(_mm_loadu_si128(__A)); 
 }
+TEST_CONSTEXPR(match_v4di(_mm256_broadcast_i64x2((__m128i)(__v2di){1, -2}), 1, -2, 1, -2));
 
 __m256i test_mm256_mask_broadcast_i64x2(__m256i __O, __mmask8 __M, __m128i const* __A) {
-  // CHECK-LABEL: @test_mm256_mask_broadcast_i64x2
+  // CHECK-LABEL: test_mm256_mask_broadcast_i64x2
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_broadcast_i64x2(__O, __M, _mm_loadu_si128(__A)); 
 }
 
 __m256i test_mm256_maskz_broadcast_i64x2(__mmask8 __M, __m128i const* __A) {
-  // CHECK-LABEL: @test_mm256_maskz_broadcast_i64x2
+  // CHECK-LABEL: test_mm256_maskz_broadcast_i64x2
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_broadcast_i64x2(__M, _mm_loadu_si128(__A)); 
 }
 
 __m128d test_mm256_extractf64x2_pd(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_extractf64x2_pd
+  // CHECK-LABEL: test_mm256_extractf64x2_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <2 x i32> <i32 2, i32 3>
   return _mm256_extractf64x2_pd(__A, 1); 
 }
 
 __m128d test_mm256_mask_extractf64x2_pd(__m128d __W, __mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_extractf64x2_pd
+  // CHECK-LABEL: test_mm256_mask_extractf64x2_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <2 x i32> <i32 2, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm256_mask_extractf64x2_pd(__W, __U, __A, 1); 
 }
 
 __m128d test_mm256_maskz_extractf64x2_pd(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_maskz_extractf64x2_pd
+  // CHECK-LABEL: test_mm256_maskz_extractf64x2_pd
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> poison, <2 x i32> <i32 2, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}}
   return _mm256_maskz_extractf64x2_pd(__U, __A, 1); 
 }
 
 __m128i test_mm256_extracti64x2_epi64(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_extracti64x2_epi64
+  // CHECK-LABEL: test_mm256_extracti64x2_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
   return _mm256_extracti64x2_epi64(__A, 1); 
 }
 
 __m128i test_mm256_mask_extracti64x2_epi64(__m128i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_extracti64x2_epi64
+  // CHECK-LABEL: test_mm256_mask_extracti64x2_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm256_mask_extracti64x2_epi64(__W, __U, __A, 1); 
 }
 
 __m128i test_mm256_maskz_extracti64x2_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_extracti64x2_epi64
+  // CHECK-LABEL: test_mm256_maskz_extracti64x2_epi64
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm256_maskz_extracti64x2_epi64(__U, __A, 1); 
 }
 
 __m256d test_mm256_insertf64x2(__m256d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm256_insertf64x2
+  // CHECK-LABEL: test_mm256_insertf64x2
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   return _mm256_insertf64x2(__A, __B, 1); 
 }
 
 __m256d test_mm256_mask_insertf64x2(__m256d __W, __mmask8 __U, __m256d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm256_mask_insertf64x2
+  // CHECK-LABEL: test_mm256_mask_insertf64x2
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_mask_insertf64x2(__W, __U, __A, __B, 1); 
 }
 
 __m256d test_mm256_maskz_insertf64x2(__mmask8 __U, __m256d __A, __m128d __B) {
-  // CHECK-LABEL: @test_mm256_maskz_insertf64x2
+  // CHECK-LABEL: test_mm256_maskz_insertf64x2
   // CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   // CHECK: select <4 x i1> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}}
   return _mm256_maskz_insertf64x2(__U, __A, __B, 1); 
 }
 
 __m256i test_mm256_inserti64x2(__m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_inserti64x2
+  // CHECK-LABEL: test_mm256_inserti64x2
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   return _mm256_inserti64x2(__A, __B, 1); 
 }
 
 __m256i test_mm256_mask_inserti64x2(__m256i __W, __mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_mask_inserti64x2
+  // CHECK-LABEL: test_mm256_mask_inserti64x2
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_inserti64x2(__W, __U, __A, __B, 1); 
 }
 
 __m256i test_mm256_maskz_inserti64x2(__mmask8 __U, __m256i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_inserti64x2
+  // CHECK-LABEL: test_mm256_maskz_inserti64x2
   // CHECK: shufflevector <4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_inserti64x2(__U, __A, __B, 1); 
 }
 
 __mmask8 test_mm_mask_fpclass_pd_mask(__mmask8 __U, __m128d __A) {
-  // CHECK-LABEL: @test_mm_mask_fpclass_pd_mask
+  // CHECK-LABEL: test_mm_mask_fpclass_pd_mask
   // CHECK: @llvm.x86.avx512.fpclass.pd.128
   return _mm_mask_fpclass_pd_mask(__U, __A, 2); 
 }
 
 __mmask8 test_mm_fpclass_pd_mask(__m128d __A) {
-  // CHECK-LABEL: @test_mm_fpclass_pd_mask
+  // CHECK-LABEL: test_mm_fpclass_pd_mask
   // CHECK: @llvm.x86.avx512.fpclass.pd.128
   return _mm_fpclass_pd_mask(__A, 2); 
 }
 
 __mmask8 test_mm256_mask_fpclass_pd_mask(__mmask8 __U, __m256d __A) {
-  // CHECK-LABEL: @test_mm256_mask_fpclass_pd_mask
+  // CHECK-LABEL: test_mm256_mask_fpclass_pd_mask
   // CHECK: @llvm.x86.avx512.fpclass.pd.256
   return _mm256_mask_fpclass_pd_mask(__U, __A, 2); 
 }
 
 __mmask8 test_mm256_fpclass_pd_mask(__m256d __A) {
-  // CHECK-LABEL: @test_mm256_fpclass_pd_mask
+  // CHECK-LABEL: test_mm256_fpclass_pd_mask
   // CHECK: @llvm.x86.avx512.fpclass.pd.256
   return _mm256_fpclass_pd_mask(__A, 2); 
 }
 
 __mmask8 test_mm_mask_fpclass_ps_mask(__mmask8 __U, __m128 __A) {
-  // CHECK-LABEL: @test_mm_mask_fpclass_ps_mask
+  // CHECK-LABEL: test_mm_mask_fpclass_ps_mask
   // CHECK: @llvm.x86.avx512.fpclass.ps.128
   return _mm_mask_fpclass_ps_mask(__U, __A, 2); 
 }
 
 __mmask8 test_mm_fpclass_ps_mask(__m128 __A) {
-  // CHECK-LABEL: @test_mm_fpclass_ps_mask
+  // CHECK-LABEL: test_mm_fpclass_ps_mask
   // CHECK: @llvm.x86.avx512.fpclass.ps.128
   return _mm_fpclass_ps_mask(__A, 2); 
 }
 
 __mmask8 test_mm256_mask_fpclass_ps_mask(__mmask8 __U, __m256 __A) {
-  // CHECK-LABEL: @test_mm256_mask_fpclass_ps_mask
+  // CHECK-LABEL: test_mm256_mask_fpclass_ps_mask
   // CHECK: @llvm.x86.avx512.fpclass.ps.256
   return _mm256_mask_fpclass_ps_mask(__U, __A, 2); 
 }
 
 __mmask8 test_mm256_fpclass_ps_mask(__m256 __A) {
-  // CHECK-LABEL: @test_mm256_fpclass_ps_mask
+  // CHECK-LABEL: test_mm256_fpclass_ps_mask
   // CHECK: @llvm.x86.avx512.fpclass.ps.256
   return _mm256_fpclass_ps_mask(__A, 2); 
 }
diff --git a/clang/test/CodeGen/X86/avx512vlfp16-builtins.c b/clang/test/CodeGen/X86/avx512vlfp16-builtins.c
index 3a212ed683437..4e8bffc12cf4e 100644
--- a/clang/test/CodeGen/X86/avx512vlfp16-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vlfp16-builtins.c
@@ -1,21 +1,24 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-unknown -target-feature +avx512vl -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-unknown -target-feature +avx512vl -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-unknown-unknown -target-feature +avx512vl -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-unknown-unknown -target-feature +avx512vl -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-unknown-unknown -target-feature +avx512vl -target-feature +avx512fp16 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 _Float16 test_mm_cvtsh_h(__m128h __A) {
-  // CHECK-LABEL: @test_mm_cvtsh_h
+  // CHECK-LABEL: test_mm_cvtsh_h
   // CHECK: extractelement <8 x half> %{{.*}}, i32 0
   return _mm_cvtsh_h(__A);
 }
 
 _Float16 test_mm256_cvtsh_h(__m256h __A) {
-  // CHECK-LABEL: @test_mm256_cvtsh_h
+  // CHECK-LABEL: test_mm256_cvtsh_h
   // CHECK: extractelement <16 x half> %{{.*}}, i32 0
   return _mm256_cvtsh_h(__A);
 }
 
 __m128h test_mm_set_sh(_Float16 __h) {
-  // CHECK-LABEL: @test_mm_set_sh
+  // CHECK-LABEL: test_mm_set_sh
   // CHECK: insertelement <8 x half> {{.*}}, i32 0
   // CHECK: insertelement <8 x half> %{{.*}}, half 0xH0000, i32 1
   // CHECK: insertelement <8 x half> %{{.*}}, half 0xH0000, i32 2
@@ -28,7 +31,7 @@ __m128h test_mm_set_sh(_Float16 __h) {
 }
 
 __m128h test_mm_set1_ph(_Float16 h) {
-  // CHECK-LABEL: @test_mm_set1_ph
+  // CHECK-LABEL: test_mm_set1_ph
   // CHECK: insertelement <8 x half> {{.*}}, i32 0
   // CHECK: insertelement <8 x half> {{.*}}, i32 1
   // CHECK: insertelement <8 x half> {{.*}}, i32 2
@@ -41,7 +44,7 @@ __m128h test_mm_set1_ph(_Float16 h) {
 }
 
 __m256h test_mm256_set1_ph(_Float16 h) {
-  // CHECK-LABEL: @test_mm256_set1_ph
+  // CHECK-LABEL: test_mm256_set1_ph
   // CHECK: insertelement <16 x half> {{.*}}, i32 0
   // CHECK: insertelement <16 x half> {{.*}}, i32 1
   // CHECK: insertelement <16 x half> {{.*}}, i32 2
@@ -62,7 +65,7 @@ __m256h test_mm256_set1_ph(_Float16 h) {
 }
 
 __m128h test_mm_set1_pch(_Float16 _Complex h) {
-  // CHECK-LABEL: @test_mm_set1_pch
+  // CHECK-LABEL: test_mm_set1_pch
   // CHECK: insertelement <4 x float> {{.*}}, i32 0
   // CHECK: insertelement <4 x float> {{.*}}, i32 1
   // CHECK: insertelement <4 x float> {{.*}}, i32 2
@@ -71,7 +74,7 @@ __m128h test_mm_set1_pch(_Float16 _Complex h) {
 }
 
 __m256h test_mm256_set1_pch(_Float16 _Complex h) {
-  // CHECK-LABEL: @test_mm256_set1_pch
+  // CHECK-LABEL: test_mm256_set1_pch
   // CHECK: insertelement <8 x float> {{.*}}, i32 0
   // CHECK: insertelement <8 x float> {{.*}}, i32 1
   // CHECK: insertelement <8 x float> {{.*}}, i32 2
@@ -85,7 +88,7 @@ __m256h test_mm256_set1_pch(_Float16 _Complex h) {
 
 __m128h test_mm_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16 __h4,
                        _Float16 __h5, _Float16 __h6, _Float16 __h7, _Float16 __h8) {
-  // CHECK-LABEL: @test_mm_set_ph
+  // CHECK-LABEL: test_mm_set_ph
   // CHECK: insertelement <8 x half> {{.*}}, i32 0
   // CHECK: insertelement <8 x half> {{.*}}, i32 1
   // CHECK: insertelement <8 x half> {{.*}}, i32 2
@@ -101,7 +104,7 @@ __m256h test_mm256_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16
                           _Float16 __h5, _Float16 __h6, _Float16 __h7, _Float16 __h8,
                           _Float16 __h9, _Float16 __h10, _Float16 __h11, _Float16 __h12,
                           _Float16 __h13, _Float16 __h14, _Float16 __h15, _Float16 __h16) {
-  // CHECK-LABEL: @test_mm256_set_ph
+  // CHECK-LABEL: test_mm256_set_ph
   // CHECK: insertelement <16 x half> {{.*}}, i32 0
   // CHECK: insertelement <16 x half> {{.*}}, i32 1
   // CHECK: insertelement <16 x half> {{.*}}, i32 2
@@ -124,7 +127,7 @@ __m256h test_mm256_set_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16
 
 __m128h test_mm_setr_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16 __h4,
                         _Float16 __h5, _Float16 __h6, _Float16 __h7, _Float16 __h8) {
-  // CHECK-LABEL: @test_mm_setr_ph
+  // CHECK-LABEL: test_mm_setr_ph
   // CHECK: insertelement <8 x half> {{.*}}, i32 0
   // CHECK: insertelement <8 x half> {{.*}}, i32 1
   // CHECK: insertelement <8 x half> {{.*}}, i32 2
@@ -140,7 +143,7 @@ __m256h test_mm256_setr_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16
                            _Float16 __h5, _Float16 __h6, _Float16 __h7, _Float16 __h8,
                            _Float16 __h9, _Float16 __h10, _Float16 __h11, _Float16 __h12,
                            _Float16 __h13, _Float16 __h14, _Float16 __h15, _Float16 __h16) {
-  // CHECK-LABEL: @test_mm256_setr_ph
+  // CHECK-LABEL: test_mm256_setr_ph
   // CHECK: insertelement <16 x half> {{.*}}, i32 0
   // CHECK: insertelement <16 x half> {{.*}}, i32 1
   // CHECK: insertelement <16 x half> {{.*}}, i32 2
@@ -162,251 +165,251 @@ __m256h test_mm256_setr_ph(_Float16 __h1, _Float16 __h2, _Float16 __h3, _Float16
 }
 
 __m256h test_mm256_add_ph(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_add_ph
+  // CHECK-LABEL: test_mm256_add_ph
   // CHECK: %{{.*}} = fadd <16 x half> %{{.*}}, %{{.*}}
   return _mm256_add_ph(__A, __B);
 }
 
 __m256h test_mm256_mask_add_ph(__m256h __W, __mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_add_ph
+  // CHECK-LABEL: test_mm256_mask_add_ph
   // CHECK: %{{.*}} = fadd <16 x half> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return (__m256h)_mm256_mask_add_ph(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_add_ph(__mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_add_ph
+  // CHECK-LABEL: test_mm256_maskz_add_ph
   // CHECK: %{{.*}} = fadd <16 x half> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_add_ph(__U, __A, __B);
 }
 
 __m128h test_mm_add_ph(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_add_ph
+  // CHECK-LABEL: test_mm_add_ph
   // CHECK: %{{.*}} = fadd <8 x half> %{{.*}}, %{{.*}}
   return _mm_add_ph(__A, __B);
 }
 
 __m128h test_mm_mask_add_ph(__m128h __W, __mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_add_ph
+  // CHECK-LABEL: test_mm_mask_add_ph
   // CHECK: %{{.*}} = fadd <8 x half> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return (__m128h)_mm_mask_add_ph(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_add_ph(__mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_add_ph
+  // CHECK-LABEL: test_mm_maskz_add_ph
   // CHECK: %{{.*}} = fadd <8 x half> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_add_ph(__U, __A, __B);
 }
 
 __m256h test_mm256_sub_ph(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_sub_ph
+  // CHECK-LABEL: test_mm256_sub_ph
   // CHECK: %{{.*}} = fsub <16 x half> %{{.*}}, %{{.*}}
   return _mm256_sub_ph(__A, __B);
 }
 
 __m256h test_mm256_mask_sub_ph(__m256h __W, __mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_sub_ph
+  // CHECK-LABEL: test_mm256_mask_sub_ph
   // CHECK: %{{.*}} = fsub <16 x half> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return (__m256h)_mm256_mask_sub_ph(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_sub_ph(__mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_sub_ph
+  // CHECK-LABEL: test_mm256_maskz_sub_ph
   // CHECK: %{{.*}} = fsub <16 x half> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_sub_ph(__U, __A, __B);
 }
 
 __m128h test_mm_sub_ph(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_sub_ph
+  // CHECK-LABEL: test_mm_sub_ph
   // CHECK: %{{.*}} = fsub <8 x half> %{{.*}}, %{{.*}}
   return _mm_sub_ph(__A, __B);
 }
 
 __m128h test_mm_mask_sub_ph(__m128h __W, __mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_sub_ph
+  // CHECK-LABEL: test_mm_mask_sub_ph
   // CHECK: %{{.*}} = fsub <8 x half> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return (__m128h)_mm_mask_sub_ph(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_sub_ph(__mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_sub_ph
+  // CHECK-LABEL: test_mm_maskz_sub_ph
   // CHECK: %{{.*}} = fsub <8 x half> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_sub_ph(__U, __A, __B);
 }
 
 __m256h test_mm256_mul_ph(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mul_ph
+  // CHECK-LABEL: test_mm256_mul_ph
   // CHECK: %{{.*}} = fmul <16 x half> %{{.*}}, %{{.*}}
   return _mm256_mul_ph(__A, __B);
 }
 
 __m256h test_mm256_mask_mul_ph(__m256h __W, __mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_mul_ph
+  // CHECK-LABEL: test_mm256_mask_mul_ph
   // CHECK: %{{.*}} = fmul <16 x half> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return (__m256h)_mm256_mask_mul_ph(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_mul_ph(__mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_mul_ph
+  // CHECK-LABEL: test_mm256_maskz_mul_ph
   // CHECK: %{{.*}} = fmul <16 x half> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_mul_ph(__U, __A, __B);
 }
 
 __m128h test_mm_mul_ph(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mul_ph
+  // CHECK-LABEL: test_mm_mul_ph
   // CHECK: %{{.*}} = fmul <8 x half> %{{.*}}, %{{.*}}
   return _mm_mul_ph(__A, __B);
 }
 
 __m128h test_mm_mask_mul_ph(__m128h __W, __mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_mul_ph
+  // CHECK-LABEL: test_mm_mask_mul_ph
   // CHECK: %{{.*}} = fmul <8 x half> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return (__m128h)_mm_mask_mul_ph(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_mul_ph(__mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_mul_ph
+  // CHECK-LABEL: test_mm_maskz_mul_ph
   // CHECK: %{{.*}} = fmul <8 x half> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_mul_ph(__U, __A, __B);
 }
 
 __m256h test_mm256_div_ph(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_div_ph
+  // CHECK-LABEL: test_mm256_div_ph
   // CHECK: %{{.*}} = fdiv <16 x half> %{{.*}}, %{{.*}}
   return _mm256_div_ph(__A, __B);
 }
 
 __m256h test_mm256_mask_div_ph(__m256h __W, __mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_div_ph
+  // CHECK-LABEL: test_mm256_mask_div_ph
   // CHECK: %{{.*}} = fdiv <16 x half> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return (__m256h)_mm256_mask_div_ph(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_div_ph(__mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_div_ph
+  // CHECK-LABEL: test_mm256_maskz_div_ph
   // CHECK: %{{.*}} = fdiv <16 x half> %{{.*}}, %{{.*}}
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_div_ph(__U, __A, __B);
 }
 
 __m128h test_mm_div_ph(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_div_ph
+  // CHECK-LABEL: test_mm_div_ph
   // CHECK: %{{.*}} = fdiv <8 x half> %{{.*}}, %{{.*}}
   return _mm_div_ph(__A, __B);
 }
 
 __m128h test_mm_mask_div_ph(__m128h __W, __mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_div_ph
+  // CHECK-LABEL: test_mm_mask_div_ph
   // CHECK: %{{.*}} = fdiv <8 x half> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return (__m128h)_mm_mask_div_ph(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_div_ph(__mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_div_ph
+  // CHECK-LABEL: test_mm_maskz_div_ph
   // CHECK: %{{.*}} = fdiv <8 x half> %{{.*}}, %{{.*}}
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_div_ph(__U, __A, __B);
 }
 
 __m256h test_mm256_min_ph(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_min_ph
+  // CHECK-LABEL: test_mm256_min_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.256
   return _mm256_min_ph(__A, __B);
 }
 
 __m256h test_mm256_mask_min_ph(__m256h __W, __mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_min_ph
+  // CHECK-LABEL: test_mm256_mask_min_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.256
   return (__m256h)_mm256_mask_min_ph(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_min_ph(__mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_min_ph
+  // CHECK-LABEL: test_mm256_maskz_min_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.256
   return _mm256_maskz_min_ph(__U, __A, __B);
 }
 
 __m128h test_mm_min_ph(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_min_ph
+  // CHECK-LABEL: test_mm_min_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.128
   return _mm_min_ph(__A, __B);
 }
 
 __m128h test_mm_mask_min_ph(__m128h __W, __mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_min_ph
+  // CHECK-LABEL: test_mm_mask_min_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.128
   return (__m128h)_mm_mask_min_ph(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_min_ph(__mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_min_ph
+  // CHECK-LABEL: test_mm_maskz_min_ph
   // CHECK: @llvm.x86.avx512fp16.min.ph.128
   return _mm_maskz_min_ph(__U, __A, __B);
 }
 
 __m256h test_mm256_max_ph(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_max_ph
+  // CHECK-LABEL: test_mm256_max_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.256
   return _mm256_max_ph(__A, __B);
 }
 
 __m256h test_mm256_mask_max_ph(__m256h __W, __mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_max_ph
+  // CHECK-LABEL: test_mm256_mask_max_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.256
   return (__m256h)_mm256_mask_max_ph(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_max_ph(__mmask32 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_max_ph
+  // CHECK-LABEL: test_mm256_maskz_max_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.256
   return _mm256_maskz_max_ph(__U, __A, __B);
 }
 
 __m128h test_mm_max_ph(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_max_ph
+  // CHECK-LABEL: test_mm_max_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.128
   return _mm_max_ph(__A, __B);
 }
 
 __m128h test_mm_mask_max_ph(__m128h __W, __mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_max_ph
+  // CHECK-LABEL: test_mm_mask_max_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.128
   return (__m128h)_mm_mask_max_ph(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_max_ph(__mmask32 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_max_ph
+  // CHECK-LABEL: test_mm_maskz_max_ph
   // CHECK: @llvm.x86.avx512fp16.max.ph.128
   return _mm_maskz_max_ph(__U, __A, __B);
 }
 
 __m128h test_mm_abs_ph(__m128h a) {
-  // CHECK-LABEL: @test_mm_abs_ph
+  // CHECK-LABEL: test_mm_abs_ph
   // CHECK: and <4 x i32>
   return _mm_abs_ph(a);
 }
 
 __m256h test_mm256_abs_ph(__m256h a) {
-  // CHECK-LABEL: @test_mm256_abs_ph
+  // CHECK-LABEL: test_mm256_abs_ph
   // CHECK: and <8 x i32>
   return _mm256_abs_ph(a);
 }
 
 __m256h test_mm256_conj_pch(__m256h __A) {
-  // CHECK-LABEL: @test_mm256_conj_pch
+  // CHECK-LABEL: test_mm256_conj_pch
   // CHECK:  %{{.*}} = bitcast <16 x half> %{{.*}} to <8 x float>
   // CHECK:  %{{.*}} = bitcast <8 x float> %{{.*}} to <8 x i32>
   // CHECK:  %{{.*}} = bitcast <8 x float> %{{.*}} to <8 x i32>
@@ -417,7 +420,7 @@ __m256h test_mm256_conj_pch(__m256h __A) {
 }
 
 __m256h test_mm256_mask_conj_pch(__m256h __W, __mmask32 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_mask_conj_pch
+  // CHECK-LABEL: test_mm256_mask_conj_pch
   // CHECK:  %{{.*}} = trunc i32 %{{.*}} to i8
   // CHECK:  %{{.*}} = bitcast <16 x half> %{{.*}} to <8 x float>
   // CHECK:  %{{.*}} = bitcast <8 x float> %{{.*}} to <8 x i32>
@@ -433,7 +436,7 @@ __m256h test_mm256_mask_conj_pch(__m256h __W, __mmask32 __U, __m256h __A) {
 }
 
 __m256h test_mm256_maskz_conj_pch(__mmask32 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_maskz_conj_pch
+  // CHECK-LABEL: test_mm256_maskz_conj_pch
   // CHECK:  %{{.*}} = trunc i32 %{{.*}} to i8
   // CHECK:  %{{.*}} = bitcast <16 x half> %{{.*}} to <8 x float>
   // CHECK:  %{{.*}} = bitcast <8 x float> %{{.*}} to <8 x i32>
@@ -448,7 +451,7 @@ __m256h test_mm256_maskz_conj_pch(__mmask32 __U, __m256h __A) {
 }
 
 __m128h test_mm_conj_pch(__m128h __A) {
-  // CHECK-LABEL: @test_mm_conj_pch
+  // CHECK-LABEL: test_mm_conj_pch
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <4 x float> %{{.*}} to <4 x i32>
   // CHECK:  %{{.*}} = bitcast <4 x float> %{{.*}} to <4 x i32>
@@ -459,7 +462,7 @@ __m128h test_mm_conj_pch(__m128h __A) {
 }
 
 __m128h test_mm_mask_conj_pch(__m128h __W, __mmask32 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_conj_pch
+  // CHECK-LABEL: test_mm_mask_conj_pch
   // CHECK:  %{{.*}} = trunc i32 %{{.*}} to i8
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <4 x float> %{{.*}} to <4 x i32>
@@ -476,7 +479,7 @@ __m128h test_mm_mask_conj_pch(__m128h __W, __mmask32 __U, __m128h __A) {
 }
 
 __m128h test_mm_maskz_conj_pch(__mmask32 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_maskz_conj_pch
+  // CHECK-LABEL: test_mm_maskz_conj_pch
   // CHECK:  %{{.*}} = trunc i32 %{{.*}} to i8
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <4 x float>
   // CHECK:  %{{.*}} = bitcast <4 x float> %{{.*}} to <4 x i32>
@@ -492,7 +495,7 @@ __m128h test_mm_maskz_conj_pch(__mmask32 __U, __m128h __A) {
 }
 
 __mmask16 test_mm256_cmp_ph_mask_eq_oq(__m256h a, __m256h b) {
-  // CHECK-LABEL: @test_mm256_cmp_ph_mask_eq_oq
+  // CHECK-LABEL: test_mm256_cmp_ph_mask_eq_oq
   // CHECK: fcmp oeq <16 x half> %{{.*}}, %{{.*}}
   return _mm256_cmp_ph_mask(a, b, _CMP_EQ_OQ);
 }
@@ -684,7 +687,7 @@ __mmask16 test_mm256_cmp_ph_mask_true_us(__m256h a, __m256h b) {
 }
 
 __mmask16 test_mm256_mask_cmp_ph_mask_eq_oq(__mmask16 m, __m256h a, __m256h b) {
-  // CHECK-LABEL: @test_mm256_mask_cmp_ph_mask_eq_oq
+  // CHECK-LABEL: test_mm256_mask_cmp_ph_mask_eq_oq
   // CHECK: [[CMP:%.*]] = fcmp oeq <16 x half> %{{.*}}, %{{.*}}
   // CHECK: and <16 x i1> [[CMP]], {{.*}}
   return _mm256_mask_cmp_ph_mask(m, a, b, _CMP_EQ_OQ);
@@ -908,7 +911,7 @@ __mmask16 test_mm256_mask_cmp_ph_mask_true_us(__mmask16 m, __m256h a, __m256h b)
 }
 
 __mmask8 test_mm_cmp_ph_mask_eq_oq(__m128h a, __m128h b) {
-  // CHECK-LABEL: @test_mm_cmp_ph_mask_eq_oq
+  // CHECK-LABEL: test_mm_cmp_ph_mask_eq_oq
   // CHECK: fcmp oeq <8 x half> %{{.*}}, %{{.*}}
   return _mm_cmp_ph_mask(a, b, _CMP_EQ_OQ);
 }
@@ -1100,7 +1103,7 @@ __mmask8 test_mm_cmp_ph_mask_true_us(__m128h a, __m128h b) {
 }
 
 __mmask8 test_mm_mask_cmp_ph_mask_eq_oq(__mmask8 m, __m128h a, __m128h b) {
-  // CHECK-LABEL: @test_mm_mask_cmp_ph_mask_eq_oq
+  // CHECK-LABEL: test_mm_mask_cmp_ph_mask_eq_oq
   // CHECK: [[CMP:%.*]] = fcmp oeq <8 x half> %{{.*}}, %{{.*}}
   // CHECK: and <8 x i1> [[CMP]], {{.*}}
   return _mm_mask_cmp_ph_mask(m, a, b, _CMP_EQ_OQ);
@@ -1324,315 +1327,315 @@ __mmask8 test_mm_mask_cmp_ph_mask_true_us(__mmask8 m, __m128h a, __m128h b) {
 }
 
 __m256h test_mm256_rcp_ph(__m256h __A) {
-  // CHECK-LABEL: @test_mm256_rcp_ph
+  // CHECK-LABEL: test_mm256_rcp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.ph.256
   return _mm256_rcp_ph(__A);
 }
 
 __m256h test_mm256_mask_rcp_ph(__m256h __W, __mmask32 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_mask_rcp_ph
+  // CHECK-LABEL: test_mm256_mask_rcp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.ph.256
   return (__m256h)_mm256_mask_rcp_ph(__W, __U, __A);
 }
 
 __m256h test_mm256_maskz_rcp_ph(__mmask32 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_maskz_rcp_ph
+  // CHECK-LABEL: test_mm256_maskz_rcp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.ph.256
   return _mm256_maskz_rcp_ph(__U, __A);
 }
 
 __m128h test_mm_rcp_ph(__m128h __A) {
-  // CHECK-LABEL: @test_mm_rcp_ph
+  // CHECK-LABEL: test_mm_rcp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.ph.128
   return _mm_rcp_ph(__A);
 }
 
 __m128h test_mm_mask_rcp_ph(__m128h __W, __mmask32 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_rcp_ph
+  // CHECK-LABEL: test_mm_mask_rcp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.ph.128
   return (__m128h)_mm_mask_rcp_ph(__W, __U, __A);
 }
 
 __m128h test_mm_maskz_rcp_ph(__mmask32 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_maskz_rcp_ph
+  // CHECK-LABEL: test_mm_maskz_rcp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rcp.ph.128
   return _mm_maskz_rcp_ph(__U, __A);
 }
 
 __m256h test_mm256_rsqrt_ph(__m256h __A) {
-  // CHECK-LABEL: @test_mm256_rsqrt_ph
+  // CHECK-LABEL: test_mm256_rsqrt_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.ph.256
   return _mm256_rsqrt_ph(__A);
 }
 
 __m256h test_mm256_mask_rsqrt_ph(__m256h __W, __mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_mask_rsqrt_ph
+  // CHECK-LABEL: test_mm256_mask_rsqrt_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.ph.256
   return (__m256h)_mm256_mask_rsqrt_ph(__W, __U, __A);
 }
 
 __m256h test_mm256_maskz_rsqrt_ph(__mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_maskz_rsqrt_ph
+  // CHECK-LABEL: test_mm256_maskz_rsqrt_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.ph.256
   return _mm256_maskz_rsqrt_ph(__U, __A);
 }
 
 __m128h test_mm_rsqrt_ph(__m128h __A) {
-  // CHECK-LABEL: @test_mm_rsqrt_ph
+  // CHECK-LABEL: test_mm_rsqrt_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.ph.128
   return _mm_rsqrt_ph(__A);
 }
 
 __m128h test_mm_mask_rsqrt_ph(__m128h __W, __mmask32 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_rsqrt_ph
+  // CHECK-LABEL: test_mm_mask_rsqrt_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.ph.128
   return (__m128h)_mm_mask_rsqrt_ph(__W, __U, __A);
 }
 
 __m128h test_mm_maskz_rsqrt_ph(__mmask32 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_maskz_rsqrt_ph
+  // CHECK-LABEL: test_mm_maskz_rsqrt_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rsqrt.ph.128
   return _mm_maskz_rsqrt_ph(__U, __A);
 }
 
 __m128h test_mm_getmant_ph(__m128h __A) {
-  // CHECK-LABEL: @test_mm_getmant_ph
+  // CHECK-LABEL: test_mm_getmant_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.128
   return _mm_getmant_ph(__A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan);
 }
 
 __m128h test_mm_mask_getmant_ph(__m128h __W, __mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_getmant_ph
+  // CHECK-LABEL: test_mm_mask_getmant_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.128
   return _mm_mask_getmant_ph(__W, __U, __A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan);
 }
 
 __m128h test_mm_maskz_getmant_ph(__mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_maskz_getmant_ph
+  // CHECK-LABEL: test_mm_maskz_getmant_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.128
   return _mm_maskz_getmant_ph(__U, __A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan);
 }
 
 __m256h test_mm256_getmant_ph(__m256h __A) {
-  // CHECK-LABEL: @test_mm256_getmant_ph
+  // CHECK-LABEL: test_mm256_getmant_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.256
   return _mm256_getmant_ph(__A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan);
 }
 
 __m256h test_mm256_mask_getmant_ph(__m256h __W, __mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_mask_getmant_ph
+  // CHECK-LABEL: test_mm256_mask_getmant_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.256
   return _mm256_mask_getmant_ph(__W, __U, __A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan);
 }
 
 __m256h test_mm256_maskz_getmant_ph(__mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_maskz_getmant_ph
+  // CHECK-LABEL: test_mm256_maskz_getmant_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getmant.ph.256
   return _mm256_maskz_getmant_ph(__U, __A, _MM_MANT_NORM_p5_2, _MM_MANT_SIGN_nan);
 }
 
 __m128h test_mm_getexp_ph(__m128h __A) {
-  // CHECK-LABEL: @test_mm_getexp_ph
+  // CHECK-LABEL: test_mm_getexp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.128
   return _mm_getexp_ph(__A);
 }
 
 __m128h test_mm_mask_getexp_ph(__m128h __W, __mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_getexp_ph
+  // CHECK-LABEL: test_mm_mask_getexp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.128
   return _mm_mask_getexp_ph(__W, __U, __A);
 }
 
 __m128h test_mm_maskz_getexp_ph(__mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_maskz_getexp_ph
+  // CHECK-LABEL: test_mm_maskz_getexp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.128
   return _mm_maskz_getexp_ph(__U, __A);
 }
 
 __m256h test_mm256_getexp_ph(__m256h __A) {
-  // CHECK-LABEL: @test_mm256_getexp_ph
+  // CHECK-LABEL: test_mm256_getexp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.256
   return _mm256_getexp_ph(__A);
 }
 
 __m256h test_mm256_mask_getexp_ph(__m256h __W, __mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_mask_getexp_ph
+  // CHECK-LABEL: test_mm256_mask_getexp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.256
   return _mm256_mask_getexp_ph(__W, __U, __A);
 }
 
 __m256h test_mm256_maskz_getexp_ph(__mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_maskz_getexp_ph
+  // CHECK-LABEL: test_mm256_maskz_getexp_ph
   // CHECK: @llvm.x86.avx512fp16.mask.getexp.ph.256
   return _mm256_maskz_getexp_ph(__U, __A);
 }
 
 __m128h test_mm_scalef_ph(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_scalef_ph
+  // CHECK-LABEL: test_mm_scalef_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.128
   return _mm_scalef_ph(__A, __B);
 }
 
 __m128h test_mm_mask_scalef_ph(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_scalef_ph
+  // CHECK-LABEL: test_mm_mask_scalef_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.128
   return _mm_mask_scalef_ph(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_scalef_ph(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_scalef_ph
+  // CHECK-LABEL: test_mm_maskz_scalef_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.128
   return _mm_maskz_scalef_ph(__U, __A, __B);
 }
 
 __m256h test_mm256_scalef_ph(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_scalef_ph
+  // CHECK-LABEL: test_mm256_scalef_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.256
   return _mm256_scalef_ph(__A, __B);
 }
 
 __m256h test_mm256_mask_scalef_ph(__m256h __W, __mmask16 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_scalef_ph
+  // CHECK-LABEL: test_mm256_mask_scalef_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.256
   return _mm256_mask_scalef_ph(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_scalef_ph(__mmask16 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_scalef_ph
+  // CHECK-LABEL: test_mm256_maskz_scalef_ph
   // CHECK: @llvm.x86.avx512fp16.mask.scalef.ph.256
   return _mm256_maskz_scalef_ph(__U, __A, __B);
 }
 
 __m128h test_mm_roundscale_ph(__m128h __A) {
-  // CHECK-LABEL: @test_mm_roundscale_ph
+  // CHECK-LABEL: test_mm_roundscale_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.128
   return _mm_roundscale_ph(__A, 4);
 }
 
 __m128h test_mm_mask_roundscale_ph(__m128h __W, __mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_roundscale_ph
+  // CHECK-LABEL: test_mm_mask_roundscale_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.128
   return _mm_mask_roundscale_ph(__W, __U, __A, 4);
 }
 
 __m128h test_mm_maskz_roundscale_ph(__mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_maskz_roundscale_ph
+  // CHECK-LABEL: test_mm_maskz_roundscale_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.128
   return _mm_maskz_roundscale_ph(__U, __A, 4);
 }
 
 __m256h test_mm256_roundscale_ph(__m256h __A) {
-  // CHECK-LABEL: @test_mm256_roundscale_ph
+  // CHECK-LABEL: test_mm256_roundscale_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.256
   return _mm256_roundscale_ph(__A, 4);
 }
 
 __m256h test_mm256_mask_roundscale_ph(__m256h __W, __mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_mask_roundscale_ph
+  // CHECK-LABEL: test_mm256_mask_roundscale_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.256
   return _mm256_mask_roundscale_ph(__W, __U, __A, 4);
 }
 
 __m256h test_mm256_maskz_roundscale_ph(__mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_maskz_roundscale_ph
+  // CHECK-LABEL: test_mm256_maskz_roundscale_ph
   // CHECK: @llvm.x86.avx512fp16.mask.rndscale.ph.256
   return _mm256_maskz_roundscale_ph(__U, __A, 4);
 }
 
 __m128h test_mm_reduce_ph(__m128h __A) {
-  // CHECK-LABEL: @test_mm_reduce_ph
+  // CHECK-LABEL: test_mm_reduce_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.128
   return _mm_reduce_ph(__A, 4);
 }
 
 __m128h test_mm_mask_reduce_ph(__m128h __W, __mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_reduce_ph
+  // CHECK-LABEL: test_mm_mask_reduce_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.128
   return _mm_mask_reduce_ph(__W, __U, __A, 4);
 }
 
 __m128h test_mm_maskz_reduce_ph(__mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_maskz_reduce_ph
+  // CHECK-LABEL: test_mm_maskz_reduce_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.128
   return _mm_maskz_reduce_ph(__U, __A, 4);
 }
 
 __m256h test_mm256_reduce_ph(__m256h __A) {
-  // CHECK-LABEL: @test_mm256_reduce_ph
+  // CHECK-LABEL: test_mm256_reduce_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.256
   return _mm256_reduce_ph(__A, 4);
 }
 
 __m256h test_mm256_mask_reduce_ph(__m256h __W, __mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_mask_reduce_ph
+  // CHECK-LABEL: test_mm256_mask_reduce_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.256
   return _mm256_mask_reduce_ph(__W, __U, __A, 4);
 }
 
 __m256h test_mm256_maskz_reduce_ph(__mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_maskz_reduce_ph
+  // CHECK-LABEL: test_mm256_maskz_reduce_ph
   // CHECK: @llvm.x86.avx512fp16.mask.reduce.ph.256
   return _mm256_maskz_reduce_ph(__U, __A, 4);
 }
 __m128h test_mm_sqrt_ph(__m128h x) {
   // CHECK-LABEL: test_mm_sqrt_ph
-  // CHECK: call <8 x half> @llvm.sqrt.v8f16(<8 x half> {{.*}})
+  // CHECK: @llvm.sqrt.v8f16(<8 x half> {{.*}})
   return _mm_sqrt_ph(x);
 }
 
 __m256h test_mm256_sqrt_ph(__m256h A) {
   // CHECK-LABEL: test_mm256_sqrt_ph
-  // CHECK: call <16 x half> @llvm.sqrt.v16f16(<16 x half> %{{.*}})
+  // CHECK: @llvm.sqrt.v16f16(<16 x half> %{{.*}})
   return _mm256_sqrt_ph(A);
 }
 
 __m128h test_mm_mask_sqrt_ph(__m128h __W, __mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_sqrt_ph
+  // CHECK-LABEL: test_mm_mask_sqrt_ph
   // CHECK: @llvm.sqrt.v8f16
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask_sqrt_ph(__W, __U, __A);
 }
 
 __m128h test_mm_maskz_sqrt_ph(__mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_maskz_sqrt_ph
+  // CHECK-LABEL: test_mm_maskz_sqrt_ph
   // CHECK: @llvm.sqrt.v8f16
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_sqrt_ph(__U, __A);
 }
 
 __m256h test_mm256_mask_sqrt_ph(__m256h __W, __mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_mask_sqrt_ph
+  // CHECK-LABEL: test_mm256_mask_sqrt_ph
   // CHECK: @llvm.sqrt.v16f16
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask_sqrt_ph(__W, __U, __A);
 }
 
 __m256h test_mm256_maskz_sqrt_ph(__mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_maskz_sqrt_ph
+  // CHECK-LABEL: test_mm256_maskz_sqrt_ph
   // CHECK: @llvm.sqrt.v16f16
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_sqrt_ph(__U, __A);
 }
 __mmask8 test_mm_mask_fpclass_ph_mask(__mmask8 __U, __m128h __A) {
-  // CHECK-LABEL: @test_mm_mask_fpclass_ph_mask
+  // CHECK-LABEL: test_mm_mask_fpclass_ph_mask
   // CHECK: @llvm.x86.avx512fp16.fpclass.ph.128
   return _mm_mask_fpclass_ph_mask(__U, __A, 2);
 }
 
 __mmask8 test_mm_fpclass_ph_mask(__m128h __A) {
-  // CHECK-LABEL: @test_mm_fpclass_ph_mask
+  // CHECK-LABEL: test_mm_fpclass_ph_mask
   // CHECK: @llvm.x86.avx512fp16.fpclass.ph.128
   return _mm_fpclass_ph_mask(__A, 2);
 }
 
 __mmask16 test_mm256_mask_fpclass_ph_mask(__mmask16 __U, __m256h __A) {
-  // CHECK-LABEL: @test_mm256_mask_fpclass_ph_mask
+  // CHECK-LABEL: test_mm256_mask_fpclass_ph_mask
   // CHECK: @llvm.x86.avx512fp16.fpclass.ph.256
   return _mm256_mask_fpclass_ph_mask(__U, __A, 2);
 }
 
 __mmask16 test_mm256_fpclass_ph_mask(__m256h __A) {
-  // CHECK-LABEL: @test_mm256_fpclass_ph_mask
+  // CHECK-LABEL: test_mm256_fpclass_ph_mask
   // CHECK: @llvm.x86.avx512fp16.fpclass.ph.256
   return _mm256_fpclass_ph_mask(__A, 2);
 }
@@ -2430,420 +2433,420 @@ __m128h test_mm256_maskz_cvtxps_ph(__mmask8 A, __m256 B) {
 }
 
 __m128h test_mm_fmadd_ph(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fmadd_ph
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_fmadd_ph
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   return _mm_fmadd_ph(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fmadd_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fmadd_ph
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_fmadd_ph
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask_fmadd_ph(__A, __U, __B, __C);
 }
 
 __m128h test_mm_fmsub_ph(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fmsub_ph
+  // CHECK-LABEL: test_mm_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   return _mm_fmsub_ph(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fmsub_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fmsub_ph
+  // CHECK-LABEL: test_mm_mask_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask_fmsub_ph(__A, __U, __B, __C);
 }
 
 __m128h test_mm_mask3_fmadd_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmadd_ph
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_mask3_fmadd_ph
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask3_fmadd_ph(__A, __B, __C, __U);
 }
 
 __m128h test_mm_mask3_fnmadd_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmadd_ph
+  // CHECK-LABEL: test_mm_mask3_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask3_fnmadd_ph(__A, __B, __C, __U);
 }
 
 __m128h test_mm_maskz_fmadd_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmadd_ph
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_fmadd_ph
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_fmadd_ph(__U, __A, __B, __C);
 }
 
 __m128h test_mm_maskz_fmsub_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmsub_ph
+  // CHECK-LABEL: test_mm_maskz_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_fmsub_ph(__U, __A, __B, __C);
 }
 
 __m128h test_mm_maskz_fnmadd_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmadd_ph
+  // CHECK-LABEL: test_mm_maskz_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_fnmadd_ph(__U, __A, __B, __C);
 }
 
 __m128h test_mm_maskz_fnmsub_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fnmsub_ph
+  // CHECK-LABEL: test_mm_maskz_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_fnmsub_ph(__U, __A, __B, __C);
 }
 
 __m256h test_mm256_fmadd_ph(__m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_fmadd_ph
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK-LABEL: test_mm256_fmadd_ph
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   return _mm256_fmadd_ph(__A, __B, __C);
 }
 
 __m256h test_mm256_mask_fmadd_ph(__m256h __A, __mmask8 __U, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmadd_ph
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_fmadd_ph
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask_fmadd_ph(__A, __U, __B, __C);
 }
 
 __m256h test_mm256_fmsub_ph(__m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_fmsub_ph
+  // CHECK-LABEL: test_mm256_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   return _mm256_fmsub_ph(__A, __B, __C);
 }
 
 __m256h test_mm256_mask_fmsub_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmsub_ph
+  // CHECK-LABEL: test_mm256_mask_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask_fmsub_ph(__A, __U, __B, __C);
 }
 
 __m256h test_mm256_mask3_fmadd_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmadd_ph
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask3_fmadd_ph
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask3_fmadd_ph(__A, __B, __C, __U);
 }
 
 __m256h test_mm256_mask3_fnmadd_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fnmadd_ph
+  // CHECK-LABEL: test_mm256_mask3_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask3_fnmadd_ph(__A, __B, __C, __U);
 }
 
 __m256h test_mm256_maskz_fmadd_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmadd_ph
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_fmadd_ph
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_fmadd_ph(__U, __A, __B, __C);
 }
 
 __m256h test_mm256_maskz_fmsub_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmsub_ph
+  // CHECK-LABEL: test_mm256_maskz_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_fmsub_ph(__U, __A, __B, __C);
 }
 
 __m256h test_mm256_maskz_fnmadd_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fnmadd_ph
+  // CHECK-LABEL: test_mm256_maskz_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_fnmadd_ph(__U, __A, __B, __C);
 }
 
 __m256h test_mm256_maskz_fnmsub_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fnmsub_ph
+  // CHECK-LABEL: test_mm256_maskz_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_fnmsub_ph(__U, __A, __B, __C);
 }
 
 __m128h test_mm_fmaddsub_ph(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fmaddsub_ph
+  // CHECK-LABEL: test_mm_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <8 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   return _mm_fmaddsub_ph(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fmaddsub_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fmaddsub_ph
+  // CHECK-LABEL: test_mm_mask_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <8 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask_fmaddsub_ph(__A, __U, __B, __C);
 }
 
 __m128h test_mm_fmsubadd_ph(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fmsubadd_ph
+  // CHECK-LABEL: test_mm_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <8 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> [[NEG]])
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> [[NEG]])
   return _mm_fmsubadd_ph(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fmsubadd_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fmsubadd_ph
+  // CHECK-LABEL: test_mm_mask_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <8 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> [[NEG]])
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> [[NEG]])
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask_fmsubadd_ph(__A, __U, __B, __C);
 }
 
 __m128h test_mm_mask3_fmaddsub_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmaddsub_ph
+  // CHECK-LABEL: test_mm_mask3_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <8 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask3_fmaddsub_ph(__A, __B, __C, __U);
 }
 
 __m128h test_mm_maskz_fmaddsub_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmaddsub_ph
+  // CHECK-LABEL: test_mm_maskz_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <8 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_fmaddsub_ph(__U, __A, __B, __C);
 }
 
 __m128h test_mm_maskz_fmsubadd_ph(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmsubadd_ph
+  // CHECK-LABEL: test_mm_maskz_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <8 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> [[NEG]])
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> [[NEG]])
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_maskz_fmsubadd_ph(__U, __A, __B, __C);
 }
 
 __m256h test_mm256_fmaddsub_ph(__m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_fmaddsub_ph
+  // CHECK-LABEL: test_mm256_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <16 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   return _mm256_fmaddsub_ph(__A, __B, __C);
 }
 
 __m256h test_mm256_mask_fmaddsub_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmaddsub_ph
+  // CHECK-LABEL: test_mm256_mask_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <16 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask_fmaddsub_ph(__A, __U, __B, __C);
 }
 
 __m256h test_mm256_fmsubadd_ph(__m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_fmsubadd_ph
+  // CHECK-LABEL: test_mm256_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <16 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> [[NEG]])
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> [[NEG]])
   return _mm256_fmsubadd_ph(__A, __B, __C);
 }
 
 __m256h test_mm256_mask_fmsubadd_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmsubadd_ph
+  // CHECK-LABEL: test_mm256_mask_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <16 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> [[NEG]])
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> [[NEG]])
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask_fmsubadd_ph(__A, __U, __B, __C);
 }
 
 __m256h test_mm256_mask3_fmaddsub_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmaddsub_ph
+  // CHECK-LABEL: test_mm256_mask3_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <16 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask3_fmaddsub_ph(__A, __B, __C, __U);
 }
 
 __m256h test_mm256_maskz_fmaddsub_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmaddsub_ph
+  // CHECK-LABEL: test_mm256_maskz_fmaddsub_ph
   // CHECK-NOT: fneg
-  // CHECK: call <16 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_fmaddsub_ph(__U, __A, __B, __C);
 }
 
 __m256h test_mm256_maskz_fmsubadd_ph(__mmask16 __U, __m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmsubadd_ph
+  // CHECK-LABEL: test_mm256_maskz_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <16 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> [[NEG]])
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> [[NEG]])
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_maskz_fmsubadd_ph(__U, __A, __B, __C);
 }
 
 __m128h test_mm_mask3_fmsub_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmsub_ph
+  // CHECK-LABEL: test_mm_mask3_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask3_fmsub_ph(__A, __B, __C, __U);
 }
 
 __m256h test_mm256_mask3_fmsub_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmsub_ph
+  // CHECK-LABEL: test_mm256_mask3_fmsub_ph
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask3_fmsub_ph(__A, __B, __C, __U);
 }
 
 __m128h test_mm_mask3_fmsubadd_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmsubadd_ph
+  // CHECK-LABEL: test_mm_mask3_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <8 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> [[NEG]])
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.128(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> [[NEG]])
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask3_fmsubadd_ph(__A, __B, __C, __U);
 }
 
 __m256h test_mm256_mask3_fmsubadd_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmsubadd_ph
+  // CHECK-LABEL: test_mm256_mask3_fmsubadd_ph
   // CHECK: [[NEG:%.+]] = fneg
-  // CHECK: call <16 x half> @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> [[NEG]])
+  // CHECK: @llvm.x86.avx512fp16.vfmaddsub.ph.256(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> [[NEG]])
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask3_fmsubadd_ph(__A, __B, __C, __U);
 }
 
 __m128h test_mm_fnmadd_ph(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fnmadd_ph
+  // CHECK-LABEL: test_mm_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   return _mm_fnmadd_ph(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fnmadd_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fnmadd_ph
+  // CHECK-LABEL: test_mm_mask_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask_fnmadd_ph(__A, __U, __B, __C);
 }
 
 __m256h test_mm256_fnmadd_ph(__m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_fnmadd_ph
+  // CHECK-LABEL: test_mm256_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   return _mm256_fnmadd_ph(__A, __B, __C);
 }
 
 __m256h test_mm256_mask_fnmadd_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_mask_fnmadd_ph
+  // CHECK-LABEL: test_mm256_mask_fnmadd_ph
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask_fnmadd_ph(__A, __U, __B, __C);
 }
 
 __m128h test_mm_fnmsub_ph(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fnmsub_ph
+  // CHECK-LABEL: test_mm_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   return _mm_fnmsub_ph(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fnmsub_ph(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fnmsub_ph
+  // CHECK-LABEL: test_mm_mask_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask_fnmsub_ph(__A, __U, __B, __C);
 }
 
 __m128h test_mm_mask3_fnmsub_ph(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fnmsub_ph
+  // CHECK-LABEL: test_mm_mask3_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <8 x half> @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
+  // CHECK: @llvm.fma.v8f16(<8 x half> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask3_fnmsub_ph(__A, __B, __C, __U);
 }
 
 __m256h test_mm256_fnmsub_ph(__m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_fnmsub_ph
+  // CHECK-LABEL: test_mm256_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   return _mm256_fnmsub_ph(__A, __B, __C);
 }
 
 __m256h test_mm256_mask_fnmsub_ph(__m256h __A, __mmask16 __U, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_mask_fnmsub_ph
+  // CHECK-LABEL: test_mm256_mask_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask_fnmsub_ph(__A, __U, __B, __C);
 }
 
 __m256h test_mm256_mask3_fnmsub_ph(__m256h __A, __m256h __B, __m256h __C, __mmask16 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fnmsub_ph
+  // CHECK-LABEL: test_mm256_mask3_fnmsub_ph
   // CHECK: fneg
   // CHECK: fneg
-  // CHECK: call <16 x half> @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
+  // CHECK: @llvm.fma.v16f16(<16 x half> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}})
   return _mm256_mask3_fnmsub_ph(__A, __B, __C, __U);
 }
 
 __m128h test_mm_fcmul_pch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fcmul_pch
+  // CHECK-LABEL: test_mm_fcmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.128
   return _mm_fcmul_pch(__A, __B);
 }
 
 __m128h test_mm_mask_fcmul_pch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fcmul_pch
+  // CHECK-LABEL: test_mm_mask_fcmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.128
   return _mm_mask_fcmul_pch(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_fcmul_pch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_fcmul_pch
+  // CHECK-LABEL: test_mm_maskz_fcmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.128
   return _mm_maskz_fcmul_pch(__U, __A, __B);
 }
 
 __m256h test_mm256_fcmul_pch(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_fcmul_pch
+  // CHECK-LABEL: test_mm256_fcmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.256
   return _mm256_fcmul_pch(__A, __B);
 }
 
 __m256h test_mm256_mask_fcmul_pch(__m256h __W, __mmask8 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_fcmul_pch
+  // CHECK-LABEL: test_mm256_mask_fcmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.256
   return _mm256_mask_fcmul_pch(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_fcmul_pch(__mmask8 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_fcmul_pch
+  // CHECK-LABEL: test_mm256_maskz_fcmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.256
   return _mm256_maskz_fcmul_pch(__U, __A, __B);
 }
 
 __m128h test_mm_fcmadd_pch(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fcmadd_pch
+  // CHECK-LABEL: test_mm_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.128
   return _mm_fcmadd_pch(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fcmadd_pch(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fcmadd_pch
+  // CHECK-LABEL: test_mm_mask_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.128
   // CHECK:  %{{.*}} = shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK:  %{{.*}} = select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -2851,88 +2854,88 @@ __m128h test_mm_mask_fcmadd_pch(__m128h __A, __mmask8 __U, __m128h __B, __m128h
 }
 
 __m128h test_mm_mask3_fcmadd_pch(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fcmadd_pch
+  // CHECK-LABEL: test_mm_mask3_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.128
   // CHECK-NOT:  %{{.*}} = select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
   return _mm_mask3_fcmadd_pch(__A, __B, __C, __U);
 }
 
 __m128h test_mm_maskz_fcmadd_pch(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fcmadd_pch
+  // CHECK-LABEL: test_mm_maskz_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfcmadd.cph.128
   return _mm_maskz_fcmadd_pch(__U, __A, __B, __C);
 }
 
 __m256h test_mm256_fcmadd_pch(__m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_fcmadd_pch
+  // CHECK-LABEL: test_mm256_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.256
   return _mm256_fcmadd_pch(__A, __B, __C);
 }
 
 __m256h test_mm256_mask_fcmadd_pch(__m256h __A, __mmask8 __U, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_mask_fcmadd_pch
+  // CHECK-LABEL: test_mm256_mask_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.256
   // CHECK:  %{{.*}} = select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_fcmadd_pch(__A, __U, __B, __C);
 }
 
 __m256h test_mm256_mask3_fcmadd_pch(__m256h __A, __m256h __B, __m256h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fcmadd_pch
+  // CHECK-LABEL: test_mm256_mask3_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmadd.cph.256
   // CHECK-NOT:  %{{.*}} = select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask3_fcmadd_pch(__A, __B, __C, __U);
 }
 
 __m256h test_mm256_maskz_fcmadd_pch(__mmask8 __U, __m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fcmadd_pch
+  // CHECK-LABEL: test_mm256_maskz_fcmadd_pch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfcmadd.cph.256
   return _mm256_maskz_fcmadd_pch(__U, __A, __B, __C);
 }
 
 __m128h test_mm_fmul_pch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_fmul_pch
+  // CHECK-LABEL: test_mm_fmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.128
   return _mm_fmul_pch(__A, __B);
 }
 
 __m128h test_mm_mask_fmul_pch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_fmul_pch
+  // CHECK-LABEL: test_mm_mask_fmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.128
   return _mm_mask_fmul_pch(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_fmul_pch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_fmul_pch
+  // CHECK-LABEL: test_mm_maskz_fmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.128
   return _mm_maskz_fmul_pch(__U, __A, __B);
 }
 
 __m256h test_mm256_fmul_pch(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_fmul_pch
+  // CHECK-LABEL: test_mm256_fmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.256
   return _mm256_fmul_pch(__A, __B);
 }
 
 __m256h test_mm256_mask_fmul_pch(__m256h __W, __mmask8 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_fmul_pch
+  // CHECK-LABEL: test_mm256_mask_fmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.256
   return _mm256_mask_fmul_pch(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_fmul_pch(__mmask8 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_fmul_pch
+  // CHECK-LABEL: test_mm256_maskz_fmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.256
   return _mm256_maskz_fmul_pch(__U, __A, __B);
 }
 
 __m128h test_mm_fmadd_pch(__m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_fmadd_pch
+  // CHECK-LABEL: test_mm_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.128
   return _mm_fmadd_pch(__A, __B, __C);
 }
 
 __m128h test_mm_mask_fmadd_pch(__m128h __A, __mmask8 __U, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_mask_fmadd_pch
+  // CHECK-LABEL: test_mm_mask_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.128
   // CHECK:  %{{.*}} = shufflevector <8 x i1> %{{.*}}, <8 x i1> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK:  %{{.*}} = select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}}
@@ -2940,58 +2943,58 @@ __m128h test_mm_mask_fmadd_pch(__m128h __A, __mmask8 __U, __m128h __B, __m128h _
 }
 
 __m128h test_mm_mask3_fmadd_pch(__m128h __A, __m128h __B, __m128h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm_mask3_fmadd_pch
+  // CHECK-LABEL: test_mm_mask3_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.128
   return _mm_mask3_fmadd_pch(__A, __B, __C, __U);
 }
 
 __m128h test_mm_maskz_fmadd_pch(__mmask8 __U, __m128h __A, __m128h __B, __m128h __C) {
-  // CHECK-LABEL: @test_mm_maskz_fmadd_pch
+  // CHECK-LABEL: test_mm_maskz_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfmadd.cph.128
   return _mm_maskz_fmadd_pch(__U, __A, __B, __C);
 }
 
 __m256h test_mm256_fmadd_pch(__m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_fmadd_pch
+  // CHECK-LABEL: test_mm256_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.256
   return _mm256_fmadd_pch(__A, __B, __C);
 }
 
 __m256h test_mm256_mask_fmadd_pch(__m256h __A, __mmask8 __U, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_mask_fmadd_pch
+  // CHECK-LABEL: test_mm256_mask_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.256
   // CHECK:  %{{.*}} = select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}}
   return _mm256_mask_fmadd_pch(__A, __U, __B, __C);
 }
 
 __m256h test_mm256_mask3_fmadd_pch(__m256h __A, __m256h __B, __m256h __C, __mmask8 __U) {
-  // CHECK-LABEL: @test_mm256_mask3_fmadd_pch
+  // CHECK-LABEL: test_mm256_mask3_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmadd.cph.256
   return _mm256_mask3_fmadd_pch(__A, __B, __C, __U);
 }
 
 __m256h test_mm256_maskz_fmadd_pch(__mmask8 __U, __m256h __A, __m256h __B, __m256h __C) {
-  // CHECK-LABEL: @test_mm256_maskz_fmadd_pch
+  // CHECK-LABEL: test_mm256_maskz_fmadd_pch
   // CHECK: @llvm.x86.avx512fp16.maskz.vfmadd.cph.256
   return _mm256_maskz_fmadd_pch(__U, __A, __B, __C);
 }
 
 __m128h test_mm_mask_blend_ph(__mmask8 __U, __m128h __A, __m128h __W) {
-  // CHECK-LABEL: @test_mm_mask_blend_ph
+  // CHECK-LABEL: test_mm_mask_blend_ph
   // CHECK:  %{{.*}} = bitcast i8 %{{.*}} to <8 x i1>
   // CHECK:  %{{.*}} = select <8 x i1> %{{.*}}, <8 x half> %{{.*}}, <8 x half> %{{.*}}
   return _mm_mask_blend_ph(__U, __A, __W);
 }
 
 __m256h test_mm256_mask_blend_ph(__mmask16 __U, __m256h __A, __m256h __W) {
-  // CHECK-LABEL: @test_mm256_mask_blend_ph
+  // CHECK-LABEL: test_mm256_mask_blend_ph
   // CHECK:  %{{.*}} = bitcast i16 %{{.*}} to <16 x i1>
   // CHECK:  %{{.*}} = select <16 x i1> %{{.*}}, <16 x half> %{{.*}}, <16 x half> %{{.*}}
   return _mm256_mask_blend_ph(__U, __A, __W);
 }
 
 __m128h test_mm_permutex2var_ph(__m128h __A, __m128i __I, __m128h __B) {
-  // CHECK-LABEL: @test_mm_permutex2var_ph
+  // CHECK-LABEL: test_mm_permutex2var_ph
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <8 x i16>
   // CHECK:  %{{.*}} = bitcast <2 x i64> %{{.*}} to <8 x i16>
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <8 x i16>
@@ -3001,7 +3004,7 @@ __m128h test_mm_permutex2var_ph(__m128h __A, __m128i __I, __m128h __B) {
 }
 
 __m256h test_mm256_permutex2var_ph(__m256h __A, __m256i __I, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_permutex2var_ph
+  // CHECK-LABEL: test_mm256_permutex2var_ph
   // CHECK:  %{{.*}} = bitcast <16 x half> %{{.*}} to <16 x i16>
   // CHECK:  %{{.*}} = bitcast <4 x i64> %{{.*}} to <16 x i16>
   // CHECK:  %{{.*}} = bitcast <16 x half> %{{.*}} to <16 x i16>
@@ -3011,7 +3014,7 @@ __m256h test_mm256_permutex2var_ph(__m256h __A, __m256i __I, __m256h __B) {
 }
 
 __m128h test_mm_permutexvar_ph(__m128i __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_permutexvar_ph
+  // CHECK-LABEL: test_mm_permutexvar_ph
   // CHECK:  %{{.*}} = bitcast <8 x half> %{{.*}} to <8 x i16>
   // CHECK:  %{{.*}} = bitcast <2 x i64> %{{.*}} to <8 x i16>
   // CHECK:  %{{.*}} = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
@@ -3020,7 +3023,7 @@ __m128h test_mm_permutexvar_ph(__m128i __A, __m128h __B) {
 }
 
 __m256h test_mm256_permutexvar_ph(__m256i __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_permutexvar_ph
+  // CHECK-LABEL: test_mm256_permutexvar_ph
   // CHECK:  %{{.*}} = bitcast <16 x half> %{{.*}} to <16 x i16>
   // CHECK:  %{{.*}} = bitcast <4 x i64> %{{.*}} to <16 x i16>
   // CHECK:  %{{.*}} = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %{{.*}}, <16 x i16> %{{.*}})
@@ -3029,122 +3032,122 @@ __m256h test_mm256_permutexvar_ph(__m256i __A, __m256h __B) {
 }
 
 _Float16 test_mm256_reduce_add_ph(__m256h __W) {
-  // CHECK-LABEL: @test_mm256_reduce_add_ph
-  // CHECK: call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH8000, <16 x half> %{{.*}})
+  // CHECK-LABEL: test_mm256_reduce_add_ph
+  // CHECK: @llvm.vector.reduce.fadd.v16f16(half 0xH8000, <16 x half> %{{.*}})
   return _mm256_reduce_add_ph(__W);
 }
 
 _Float16 test_mm256_reduce_mul_ph(__m256h __W) {
-  // CHECK-LABEL: @test_mm256_reduce_mul_ph
-  // CHECK: call reassoc half @llvm.vector.reduce.fmul.v16f16(half 0xH3C00, <16 x half> %{{.*}})
+  // CHECK-LABEL: test_mm256_reduce_mul_ph
+  // CHECK: @llvm.vector.reduce.fmul.v16f16(half 0xH3C00, <16 x half> %{{.*}})
   return _mm256_reduce_mul_ph(__W);
 }
 
 _Float16 test_mm256_reduce_max_ph(__m256h __W) {
-  // CHECK-LABEL: @test_mm256_reduce_max_ph
-  // CHECK: call nnan half @llvm.vector.reduce.fmax.v16f16(<16 x half> %{{.*}})
+  // CHECK-LABEL: test_mm256_reduce_max_ph
+  // CHECK: @llvm.vector.reduce.fmax.v16f16(<16 x half> %{{.*}})
   return _mm256_reduce_max_ph(__W);
 }
 
 _Float16 test_mm256_reduce_min_ph(__m256h __W) {
-  // CHECK-LABEL: @test_mm256_reduce_min_ph
-  // CHECK: call nnan half @llvm.vector.reduce.fmin.v16f16(<16 x half> %{{.*}})
+  // CHECK-LABEL: test_mm256_reduce_min_ph
+  // CHECK: @llvm.vector.reduce.fmin.v16f16(<16 x half> %{{.*}})
   return _mm256_reduce_min_ph(__W);
 }
 
 _Float16 test_mm_reduce_add_ph(__m128h __W) {
-  // CHECK-LABEL: @test_mm_reduce_add_ph
-  // CHECK: call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH8000, <8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_reduce_add_ph
+  // CHECK: @llvm.vector.reduce.fadd.v8f16(half 0xH8000, <8 x half> %{{.*}})
   return _mm_reduce_add_ph(__W);
 }
 
 _Float16 test_mm_reduce_mul_ph(__m128h __W) {
-  // CHECK-LABEL: @test_mm_reduce_mul_ph
-  // CHECK: call reassoc half @llvm.vector.reduce.fmul.v8f16(half 0xH3C00, <8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_reduce_mul_ph
+  // CHECK: @llvm.vector.reduce.fmul.v8f16(half 0xH3C00, <8 x half> %{{.*}})
   return _mm_reduce_mul_ph(__W);
 }
 
 _Float16 test_mm_reduce_min_ph(__m128h __W) {
-  // CHECK-LABEL: @test_mm_reduce_min_ph
-  // CHECK: call nnan half @llvm.vector.reduce.fmin.v8f16(<8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_reduce_min_ph
+  // CHECK: @llvm.vector.reduce.fmin.v8f16(<8 x half> %{{.*}})
   return _mm_reduce_min_ph(__W);
 }
 
 _Float16 test_mm_reduce_max_ph(__m128h __W) {
-  // CHECK-LABEL: @test_mm_reduce_max_ph
-  // CHECK: call nnan half @llvm.vector.reduce.fmax.v8f16(<8 x half> %{{.*}})
+  // CHECK-LABEL: test_mm_reduce_max_ph
+  // CHECK: @llvm.vector.reduce.fmax.v8f16(<8 x half> %{{.*}})
   return _mm_reduce_max_ph(__W);
 }
 
 // tests below are for alias intrinsics.
 __m128h test_mm_mul_pch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mul_pch
+  // CHECK-LABEL: test_mm_mul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.128
   return _mm_mul_pch(__A, __B);
 }
 
 __m128h test_mm_mask_mul_pch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_mul_pch
+  // CHECK-LABEL: test_mm_mask_mul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.128
   return _mm_mask_mul_pch(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_mul_pch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_mul_pch
+  // CHECK-LABEL: test_mm_maskz_mul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.128
   return _mm_maskz_mul_pch(__U, __A, __B);
 }
 
 __m256h test_mm256_mul_pch(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mul_pch
+  // CHECK-LABEL: test_mm256_mul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.256
   return _mm256_mul_pch(__A, __B);
 }
 
 __m256h test_mm256_mask_mul_pch(__m256h __W, __mmask8 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_mul_pch
+  // CHECK-LABEL: test_mm256_mask_mul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.256
   return _mm256_mask_mul_pch(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_mul_pch(__mmask8 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_mul_pch
+  // CHECK-LABEL: test_mm256_maskz_mul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfmul.cph.256
   return _mm256_maskz_mul_pch(__U, __A, __B);
 }
 
 __m128h test_mm_cmul_pch(__m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_cmul_pch
+  // CHECK-LABEL: test_mm_cmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.128
   return _mm_cmul_pch(__A, __B);
 }
 
 __m128h test_mm_mask_cmul_pch(__m128h __W, __mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_mask_cmul_pch
+  // CHECK-LABEL: test_mm_mask_cmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.128
   return _mm_mask_fcmul_pch(__W, __U, __A, __B);
 }
 
 __m128h test_mm_maskz_cmul_pch(__mmask8 __U, __m128h __A, __m128h __B) {
-  // CHECK-LABEL: @test_mm_maskz_cmul_pch
+  // CHECK-LABEL: test_mm_maskz_cmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.128
   return _mm_maskz_cmul_pch(__U, __A, __B);
 }
 
 __m256h test_mm256_cmul_pch(__m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_cmul_pch
+  // CHECK-LABEL: test_mm256_cmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.256
   return _mm256_cmul_pch(__A, __B);
 }
 
 __m256h test_mm256_mask_cmul_pch(__m256h __W, __mmask8 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_mask_cmul_pch
+  // CHECK-LABEL: test_mm256_mask_cmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.256
   return _mm256_mask_cmul_pch(__W, __U, __A, __B);
 }
 
 __m256h test_mm256_maskz_cmul_pch(__mmask8 __U, __m256h __A, __m256h __B) {
-  // CHECK-LABEL: @test_mm256_maskz_cmul_pch
+  // CHECK-LABEL: test_mm256_maskz_cmul_pch
   // CHECK: @llvm.x86.avx512fp16.mask.vfcmul.cph.256
   return _mm256_maskz_cmul_pch(__U, __A, __B);
 }
diff --git a/clang/test/CodeGen/X86/avx512vlvbmi2-builtins.c b/clang/test/CodeGen/X86/avx512vlvbmi2-builtins.c
index 5760c71790deb..7259325dd2e3d 100644
--- a/clang/test/CodeGen/X86/avx512vlvbmi2-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vlvbmi2-builtins.c
@@ -1,652 +1,655 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vl -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vl -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vl -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vl -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vl -target-feature +avx512vbmi2 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 __m128i test_mm_mask_compress_epi16(__m128i __S, __mmask8 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_mask_compress_epi16
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm_mask_compress_epi16
+  // CHECK: call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i1> %{{.*}})
   return _mm_mask_compress_epi16(__S, __U, __D);
 }
 
 __m128i test_mm_maskz_compress_epi16(__mmask8 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_maskz_compress_epi16
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm_maskz_compress_epi16
+  // CHECK: call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i1> %{{.*}})
   return _mm_maskz_compress_epi16(__U, __D);
 }
 
 __m128i test_mm_mask_compress_epi8(__m128i __S, __mmask16 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_mask_compress_epi8
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm_mask_compress_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i1> %{{.*}})
   return _mm_mask_compress_epi8(__S, __U, __D);
 }
 
 __m128i test_mm_maskz_compress_epi8(__mmask16 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_maskz_compress_epi8
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm_maskz_compress_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i1> %{{.*}})
   return _mm_maskz_compress_epi8(__U, __D);
 }
 
 void test_mm_mask_compressstoreu_epi16(void *__P, __mmask8 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_mask_compressstoreu_epi16
-  // CHECK: @llvm.masked.compressstore.v8i16(<8 x i16> %{{.*}}, ptr %{{.*}}, <8 x i1> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_compressstoreu_epi16
+  // CHECK: call void @llvm.masked.compressstore.v8i16(<8 x i16> %{{.*}}, ptr %{{.*}}, <8 x i1> %{{.*}})
   _mm_mask_compressstoreu_epi16(__P, __U, __D);
 }
 
 void test_mm_mask_compressstoreu_epi8(void *__P, __mmask16 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_mask_compressstoreu_epi8
-  // CHECK: @llvm.masked.compressstore.v16i8(<16 x i8> %{{.*}}, ptr %{{.*}}, <16 x i1> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_compressstoreu_epi8
+  // CHECK: call void @llvm.masked.compressstore.v16i8(<16 x i8> %{{.*}}, ptr %{{.*}}, <16 x i1> %{{.*}})
   _mm_mask_compressstoreu_epi8(__P, __U, __D);
 }
 
 __m128i test_mm_mask_expand_epi16(__m128i __S, __mmask8 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_mask_expand_epi16
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm_mask_expand_epi16
+  // CHECK: call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i1> %{{.*}})
   return _mm_mask_expand_epi16(__S, __U, __D);
 }
 
 __m128i test_mm_maskz_expand_epi16(__mmask8 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_maskz_expand_epi16
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm_maskz_expand_epi16
+  // CHECK: call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i1> %{{.*}})
   return _mm_maskz_expand_epi16(__U, __D);
 }
 
 __m128i test_mm_mask_expand_epi8(__m128i __S, __mmask16 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_mask_expand_epi8
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm_mask_expand_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i1> %{{.*}})
   return _mm_mask_expand_epi8(__S, __U, __D);
 }
 
 __m128i test_mm_maskz_expand_epi8(__mmask16 __U, __m128i __D) {
-  // CHECK-LABEL: @test_mm_maskz_expand_epi8
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm_maskz_expand_epi8
+  // CHECK: call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i1> %{{.*}})
   return _mm_maskz_expand_epi8(__U, __D);
 }
 
 __m128i test_mm_mask_expandloadu_epi16(__m128i __S, __mmask8 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm_mask_expandloadu_epi16
-  // CHECK: @llvm.masked.expandload.v8i16(ptr %{{.*}}, <8 x i1> %{{.*}}, <8 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_expandloadu_epi16
+  // CHECK: call <8 x i16> @llvm.masked.expandload.v8i16(ptr %{{.*}}, <8 x i1> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_mask_expandloadu_epi16(__S, __U, __P);
 }
 
 __m128i test_mm_maskz_expandloadu_epi16(__mmask8 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm_maskz_expandloadu_epi16
-  // CHECK: @llvm.masked.expandload.v8i16(ptr %{{.*}}, <8 x i1> %{{.*}}, <8 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_expandloadu_epi16
+  // CHECK: call <8 x i16> @llvm.masked.expandload.v8i16(ptr %{{.*}}, <8 x i1> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_maskz_expandloadu_epi16(__U, __P);
 }
 
 __m128i test_mm_mask_expandloadu_epi8(__m128i __S, __mmask16 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm_mask_expandloadu_epi8
-  // CHECK: @llvm.masked.expandload.v16i8(ptr %{{.*}}, <16 x i1> %{{.*}}, <16 x i8> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_expandloadu_epi8
+  // CHECK: call <16 x i8> @llvm.masked.expandload.v16i8(ptr %{{.*}}, <16 x i1> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_mask_expandloadu_epi8(__S, __U, __P);
 }
 
 __m128i test_mm_maskz_expandloadu_epi8(__mmask16 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm_maskz_expandloadu_epi8
-  // CHECK: @llvm.masked.expandload.v16i8(ptr %{{.*}}, <16 x i1> %{{.*}}, <16 x i8> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_expandloadu_epi8
+  // CHECK: call <16 x i8> @llvm.masked.expandload.v16i8(ptr %{{.*}}, <16 x i1> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_maskz_expandloadu_epi8(__U, __P);
 }
 
 __m256i test_mm256_mask_compress_epi16(__m256i __S, __mmask16 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_mask_compress_epi16
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm256_mask_compress_epi16
+  // CHECK: call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i1> %{{.*}})
   return _mm256_mask_compress_epi16(__S, __U, __D);
 }
 
 __m256i test_mm256_maskz_compress_epi16(__mmask16 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_maskz_compress_epi16
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm256_maskz_compress_epi16
+  // CHECK: call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i1> %{{.*}})
   return _mm256_maskz_compress_epi16(__U, __D);
 }
 
 __m256i test_mm256_mask_compress_epi8(__m256i __S, __mmask32 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_mask_compress_epi8
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm256_mask_compress_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i1> %{{.*}})
   return _mm256_mask_compress_epi8(__S, __U, __D);
 }
 
 __m256i test_mm256_maskz_compress_epi8(__mmask32 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_maskz_compress_epi8
-  // CHECK: @llvm.x86.avx512.mask.compress
+  // CHECK-LABEL: test_mm256_maskz_compress_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i1> %{{.*}})
   return _mm256_maskz_compress_epi8(__U, __D);
 }
 
 void test_mm256_mask_compressstoreu_epi16(void *__P, __mmask16 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_mask_compressstoreu_epi16
-  // CHECK: @llvm.masked.compressstore.v16i16(<16 x i16> %{{.*}}, ptr %{{.*}}, <16 x i1> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_compressstoreu_epi16
+  // CHECK: call void @llvm.masked.compressstore.v16i16(<16 x i16> %{{.*}}, ptr %{{.*}}, <16 x i1> %{{.*}})
   _mm256_mask_compressstoreu_epi16(__P, __U, __D);
 }
 
 void test_mm256_mask_compressstoreu_epi8(void *__P, __mmask32 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_mask_compressstoreu_epi8
-  // CHECK: @llvm.masked.compressstore.v32i8(<32 x i8> %{{.*}}, ptr %{{.*}}, <32 x i1> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_compressstoreu_epi8
+  // CHECK: call void @llvm.masked.compressstore.v32i8(<32 x i8> %{{.*}}, ptr %{{.*}}, <32 x i1> %{{.*}})
   _mm256_mask_compressstoreu_epi8(__P, __U, __D);
 }
 
 __m256i test_mm256_mask_expand_epi16(__m256i __S, __mmask16 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_mask_expand_epi16
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm256_mask_expand_epi16
+  // CHECK: call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i1> %{{.*}})
   return _mm256_mask_expand_epi16(__S, __U, __D);
 }
 
 __m256i test_mm256_maskz_expand_epi16(__mmask16 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_maskz_expand_epi16
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm256_maskz_expand_epi16
+  // CHECK: call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i1> %{{.*}})
   return _mm256_maskz_expand_epi16(__U, __D);
 }
 
 __m256i test_mm256_mask_expand_epi8(__m256i __S, __mmask32 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_mask_expand_epi8
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm256_mask_expand_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i1> %{{.*}})
   return _mm256_mask_expand_epi8(__S, __U, __D);
 }
 
 __m256i test_mm256_maskz_expand_epi8(__mmask32 __U, __m256i __D) {
-  // CHECK-LABEL: @test_mm256_maskz_expand_epi8
-  // CHECK: @llvm.x86.avx512.mask.expand
+  // CHECK-LABEL: test_mm256_maskz_expand_epi8
+  // CHECK: call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %{{.*}}, <32 x i8> %{{.*}}, <32 x i1> %{{.*}})
   return _mm256_maskz_expand_epi8(__U, __D);
 }
 
 __m256i test_mm256_mask_expandloadu_epi16(__m256i __S, __mmask16 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm256_mask_expandloadu_epi16
-  // CHECK: @llvm.masked.expandload.v16i16(ptr %{{.*}}, <16 x i1> %{{.*}}, <16 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_expandloadu_epi16
+  // CHECK: call <16 x i16> @llvm.masked.expandload.v16i16(ptr %{{.*}}, <16 x i1> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_mask_expandloadu_epi16(__S, __U, __P);
 }
 
 __m256i test_mm256_maskz_expandloadu_epi16(__mmask16 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm256_maskz_expandloadu_epi16
-  // CHECK: @llvm.masked.expandload.v16i16(ptr %{{.*}}, <16 x i1> %{{.*}}, <16 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_expandloadu_epi16
+  // CHECK: call <16 x i16> @llvm.masked.expandload.v16i16(ptr %{{.*}}, <16 x i1> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_maskz_expandloadu_epi16(__U, __P);
 }
 
 __m256i test_mm256_mask_expandloadu_epi8(__m256i __S, __mmask32 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm256_mask_expandloadu_epi8
-  // CHECK: @llvm.masked.expandload.v32i8(ptr %{{.*}}, <32 x i1> %{{.*}}, <32 x i8> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_expandloadu_epi8
+  // CHECK: call <32 x  i8> @llvm.masked.expandload.v32i8(ptr %{{.*}}, <32 x i1> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_mask_expandloadu_epi8(__S, __U, __P);
 }
 
 __m256i test_mm256_maskz_expandloadu_epi8(__mmask32 __U, void const* __P) {
-  // CHECK-LABEL: @test_mm256_maskz_expandloadu_epi8
-  // CHECK: @llvm.masked.expandload.v32i8(ptr %{{.*}}, <32 x i1> %{{.*}}, <32 x i8> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_expandloadu_epi8
+  // CHECK: call <32 x i8> @llvm.masked.expandload.v32i8(ptr %{{.*}}, <32 x i1> %{{.*}}, <32 x i8> %{{.*}})
   return _mm256_maskz_expandloadu_epi8(__U, __P);
 }
 
 __m256i test_mm256_mask_shldi_epi64(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shldi_epi64
-  // CHECK: @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 47))
+  // CHECK-LABEL: test_mm256_mask_shldi_epi64
+  // CHECK: call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 47))
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_shldi_epi64(__S, __U, __A, __B, 47);
 }
 
 __m256i test_mm256_maskz_shldi_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shldi_epi64
-  // CHECK: @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 63))
+  // CHECK-LABEL: test_mm256_maskz_shldi_epi64
+  // CHECK: call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 63))
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_shldi_epi64(__U, __A, __B, 63);
 }
 
 __m256i test_mm256_shldi_epi64(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shldi_epi64
-  // CHECK: @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 31))
+  // CHECK-LABEL: test_mm256_shldi_epi64
+  // CHECK: call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 31))
   return _mm256_shldi_epi64(__A, __B, 31);
 }
 
 __m128i test_mm_mask_shldi_epi64(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shldi_epi64
-  // CHECK: @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 47))
+  // CHECK-LABEL: test_mm_mask_shldi_epi64
+  // CHECK: call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 47))
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_shldi_epi64(__S, __U, __A, __B, 47);
 }
 
 __m128i test_mm_maskz_shldi_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shldi_epi64
-  // CHECK: @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 63))
+  // CHECK-LABEL: test_mm_maskz_shldi_epi64
+  // CHECK: call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 63))
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_shldi_epi64(__U, __A, __B, 63);
 }
 
 __m128i test_mm_shldi_epi64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shldi_epi64
-  // CHECK: @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 31))
+  // CHECK-LABEL: test_mm_shldi_epi64
+  // CHECK: call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 31))
   return _mm_shldi_epi64(__A, __B, 31);
 }
 
 __m256i test_mm256_mask_shldi_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shldi_epi32
-  // CHECK: @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 7))
+  // CHECK-LABEL: test_mm256_mask_shldi_epi32
+  // CHECK: call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 7))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_shldi_epi32(__S, __U, __A, __B, 7);
 }
 
 __m256i test_mm256_maskz_shldi_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shldi_epi32
-  // CHECK: @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 15))
+  // CHECK-LABEL: test_mm256_maskz_shldi_epi32
+  // CHECK: call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 15))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_shldi_epi32(__U, __A, __B, 15);
 }
 
 __m256i test_mm256_shldi_epi32(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shldi_epi32
-  // CHECK: @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 31))
+  // CHECK-LABEL: test_mm256_shldi_epi32
+  // CHECK: call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 31))
   return _mm256_shldi_epi32(__A, __B, 31);
 }
 
 __m128i test_mm_mask_shldi_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shldi_epi32
-  // CHECK: @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 7))
+  // CHECK-LABEL: test_mm_mask_shldi_epi32
+  // CHECK: call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 7))
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_shldi_epi32(__S, __U, __A, __B, 7);
 }
 
 __m128i test_mm_maskz_shldi_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shldi_epi32
-  // CHECK: @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 15))
+  // CHECK-LABEL: test_mm_maskz_shldi_epi32
+  // CHECK: call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 15))
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_shldi_epi32(__U, __A, __B, 15);
 }
 
 __m128i test_mm_shldi_epi32(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shldi_epi32
-  // CHECK: @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 31))
+  // CHECK-LABEL: test_mm_shldi_epi32
+  // CHECK: call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 31))
   return _mm_shldi_epi32(__A, __B, 31);
 }
 
 __m256i test_mm256_mask_shldi_epi16(__m256i __S, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shldi_epi16
-  // CHECK: @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 3))
+  // CHECK-LABEL: test_mm256_mask_shldi_epi16
+  // CHECK: call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 3))
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_shldi_epi16(__S, __U, __A, __B, 3);
 }
 
 __m256i test_mm256_maskz_shldi_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shldi_epi16
-  // CHECK: @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 7))
+  // CHECK-LABEL: test_mm256_maskz_shldi_epi16
+  // CHECK: call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 7))
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_shldi_epi16(__U, __A, __B, 7);
 }
 
 __m256i test_mm256_shldi_epi16(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shldi_epi16
-  // CHECK: @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 31))
+  // CHECK-LABEL: test_mm256_shldi_epi16
+  // CHECK: call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 31))
   return _mm256_shldi_epi16(__A, __B, 31);
 }
 
 __m128i test_mm_mask_shldi_epi16(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shldi_epi16
-  // CHECK: @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 3))
+  // CHECK-LABEL: test_mm_mask_shldi_epi16
+  // CHECK: call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 3))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_shldi_epi16(__S, __U, __A, __B, 3);
 }
 
 __m128i test_mm_maskz_shldi_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shldi_epi16
-  // CHECK: @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 7))
+  // CHECK-LABEL: test_mm_maskz_shldi_epi16
+  // CHECK: call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 7))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_shldi_epi16(__U, __A, __B, 7);
 }
 
 __m128i test_mm_shldi_epi16(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shldi_epi16
-  // CHECK: @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 31))
+  // CHECK-LABEL: test_mm_shldi_epi16
+  // CHECK: call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 31))
   return _mm_shldi_epi16(__A, __B, 31);
 }
 
 __m256i test_mm256_mask_shrdi_epi64(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shrdi_epi64
-  // CHECK: @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 47))
+  // CHECK-LABEL: test_mm256_mask_shrdi_epi64
+  // CHECK: call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 47))
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_shrdi_epi64(__S, __U, __A, __B, 47);
 }
 
 __m256i test_mm256_maskz_shrdi_epi64(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shrdi_epi64
-  // CHECK: @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 63))
+  // CHECK-LABEL: test_mm256_maskz_shrdi_epi64
+  // CHECK: call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 63))
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_shrdi_epi64(__U, __A, __B, 63);
 }
 
 __m256i test_mm256_shrdi_epi64(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shrdi_epi64
-  // CHECK: @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 31)
+  // CHECK-LABEL: test_mm256_shrdi_epi64
+  // CHECK: call <4 x i64>  @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> splat (i64 31)
   return _mm256_shrdi_epi64(__A, __B, 31);
 }
 
 __m128i test_mm_mask_shrdi_epi64(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shrdi_epi64
-  // CHECK: @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 47))
+  // CHECK-LABEL: test_mm_mask_shrdi_epi64
+  // CHECK: call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 47))
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_shrdi_epi64(__S, __U, __A, __B, 47);
 }
 
 __m128i test_mm_maskz_shrdi_epi64(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shrdi_epi64
-  // CHECK: @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 63))
+  // CHECK-LABEL: test_mm_maskz_shrdi_epi64
+  // CHECK: call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 63))
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_shrdi_epi64(__U, __A, __B, 63);
 }
 
 __m128i test_mm_shrdi_epi64(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shrdi_epi64
-  // CHECK: @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 31))
+  // CHECK-LABEL: test_mm_shrdi_epi64
+  // CHECK: call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 31))
   return _mm_shrdi_epi64(__A, __B, 31);
 }
 
 __m256i test_mm256_mask_shrdi_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shrdi_epi32
-  // CHECK: @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 7))
+  // CHECK-LABEL: test_mm256_mask_shrdi_epi32
+  // CHECK: call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 7))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_shrdi_epi32(__S, __U, __A, __B, 7);
 }
 
 __m256i test_mm256_maskz_shrdi_epi32(__mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shrdi_epi32
-  // CHECK: @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 15))
+  // CHECK-LABEL: test_mm256_maskz_shrdi_epi32
+  // CHECK: call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 15))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_shrdi_epi32(__U, __A, __B, 15);
 }
 
 __m256i test_mm256_shrdi_epi32(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shrdi_epi32
-  // CHECK: @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 31)
+  // CHECK-LABEL: test_mm256_shrdi_epi32
+  // CHECK: call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> splat (i32 31)
   return _mm256_shrdi_epi32(__A, __B, 31);
 }
 
 __m128i test_mm_mask_shrdi_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shrdi_epi32
-  // CHECK: @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 7))
+  // CHECK-LABEL: test_mm_mask_shrdi_epi32
+  // CHECK: call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 7))
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_shrdi_epi32(__S, __U, __A, __B, 7);
 }
 
 __m128i test_mm_maskz_shrdi_epi32(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shrdi_epi32
-  // CHECK: @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 15))
+  // CHECK-LABEL: test_mm_maskz_shrdi_epi32
+  // CHECK: call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 15))
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_shrdi_epi32(__U, __A, __B, 15);
 }
 
 __m128i test_mm_shrdi_epi32(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shrdi_epi32
-  // CHECK: @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 31))
+  // CHECK-LABEL: test_mm_shrdi_epi32
+  // CHECK: call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> splat (i32 31))
   return _mm_shrdi_epi32(__A, __B, 31);
 }
 
 __m256i test_mm256_mask_shrdi_epi16(__m256i __S, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shrdi_epi16
-  // CHECK: @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 3))
+  // CHECK-LABEL: test_mm256_mask_shrdi_epi16
+  // CHECK: call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 3))
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_shrdi_epi16(__S, __U, __A, __B, 3);
 }
 
 __m256i test_mm256_maskz_shrdi_epi16(__mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shrdi_epi16
-  // CHECK: @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 7))
+  // CHECK-LABEL: test_mm256_maskz_shrdi_epi16
+  // CHECK: call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 7))
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_shrdi_epi16(__U, __A, __B, 7);
 }
 
 __m256i test_mm256_shrdi_epi16(__m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shrdi_epi16
-  // CHECK: @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 31))
+  // CHECK-LABEL: test_mm256_shrdi_epi16
+  // CHECK: call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> splat (i16 31))
   return _mm256_shrdi_epi16(__A, __B, 31);
 }
 
 __m128i test_mm_mask_shrdi_epi16(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shrdi_epi16
-  // CHECK: @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 3))
+  // CHECK-LABEL: test_mm_mask_shrdi_epi16
+  // CHECK: call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 3))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_shrdi_epi16(__S, __U, __A, __B, 3);
 }
 
 __m128i test_mm_maskz_shrdi_epi16(__mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shrdi_epi16
-  // CHECK: @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 7))
+  // CHECK-LABEL: test_mm_maskz_shrdi_epi16
+  // CHECK: call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 7))
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_shrdi_epi16(__U, __A, __B, 7);
 }
 
 __m128i test_mm_shrdi_epi16(__m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shrdi_epi16
-  // CHECK: @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 31))
+  // CHECK-LABEL: test_mm_shrdi_epi16
+  // CHECK: call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> splat (i16 31))
   return _mm_shrdi_epi16(__A, __B, 31);
 }
 
 __m256i test_mm256_mask_shldv_epi64(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shldv_epi64
-  // CHECK: @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_shldv_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_shldv_epi64(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_shldv_epi64(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shldv_epi64
-  // CHECK: @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_shldv_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_shldv_epi64(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_shldv_epi64(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shldv_epi64
-  // CHECK: @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_shldv_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.fshl.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_shldv_epi64(__S, __A, __B);
 }
 
 __m128i test_mm_mask_shldv_epi64(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shldv_epi64
-  // CHECK: @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_shldv_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_shldv_epi64(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_shldv_epi64(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shldv_epi64
-  // CHECK: @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_shldv_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_shldv_epi64(__U, __S, __A, __B);
 }
 
 __m128i test_mm_shldv_epi64(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shldv_epi64
-  // CHECK: @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_shldv_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_shldv_epi64(__S, __A, __B);
 }
 
 __m256i test_mm256_mask_shldv_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shldv_epi32
-  // CHECK: @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_shldv_epi32
+  // CHECK: call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_shldv_epi32(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_shldv_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shldv_epi32
-  // CHECK: @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_shldv_epi32
+  // CHECK: call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_shldv_epi32(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_shldv_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shldv_epi32
-  // CHECK: @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm256_shldv_epi32
+  // CHECK: call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_shldv_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_mask_shldv_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shldv_epi32
-  // CHECK: @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_shldv_epi32
+  // CHECK: call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_shldv_epi32(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_shldv_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shldv_epi32
-  // CHECK: @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_shldv_epi32
+  // CHECK: call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_shldv_epi32(__U, __S, __A, __B);
 }
 
 __m128i test_mm_shldv_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shldv_epi32
-  // CHECK: @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm_shldv_epi32
+  // CHECK: call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_shldv_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_mask_shldv_epi16(__m256i __S, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shldv_epi16
-  // CHECK: @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_shldv_epi16
+  // CHECK: call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_shldv_epi16(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_shldv_epi16(__mmask16 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shldv_epi16
-  // CHECK: @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_shldv_epi16
+  // CHECK: call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_shldv_epi16(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_shldv_epi16(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shldv_epi16
-  // CHECK: @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm256_shldv_epi16
+  // CHECK: call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_shldv_epi16(__S, __A, __B);
 }
 
 __m128i test_mm_mask_shldv_epi16(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shldv_epi16
-  // CHECK: @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_shldv_epi16
+  // CHECK: call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_shldv_epi16(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_shldv_epi16(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shldv_epi16
-  // CHECK: @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_shldv_epi16
+  // CHECK: call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_shldv_epi16(__U, __S, __A, __B);
 }
 
 __m128i test_mm_shldv_epi16(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shldv_epi16
-  // CHECK: @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm_shldv_epi16
+  // CHECK: call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_shldv_epi16(__S, __A, __B);
 }
 
 __m256i test_mm256_mask_shrdv_epi64(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shrdv_epi64
-  // CHECK: @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_shrdv_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_shrdv_epi64(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_shrdv_epi64(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shrdv_epi64
-  // CHECK: @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_shrdv_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_shrdv_epi64(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_shrdv_epi64(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shrdv_epi64
-  // CHECK: @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm256_shrdv_epi64
+  // CHECK: call {{.*}}<4 x i64> @llvm.fshr.v4i64(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_shrdv_epi64(__S, __A, __B);
 }
 
 __m128i test_mm_mask_shrdv_epi64(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shrdv_epi64
-  // CHECK: @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_shrdv_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_shrdv_epi64(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_shrdv_epi64(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shrdv_epi64
-  // CHECK: @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_shrdv_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   // CHECK: select <2 x i1> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_shrdv_epi64(__U, __S, __A, __B);
 }
 
 __m128i test_mm_shrdv_epi64(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shrdv_epi64
-  // CHECK: @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK-LABEL: test_mm_shrdv_epi64
+  // CHECK: call {{.*}}<2 x i64> @llvm.fshr.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_shrdv_epi64(__S, __A, __B);
 }
 
 __m256i test_mm256_mask_shrdv_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shrdv_epi32
-  // CHECK: @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_shrdv_epi32
+  // CHECK: call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_shrdv_epi32(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_shrdv_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shrdv_epi32
-  // CHECK: @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_shrdv_epi32
+  // CHECK: call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_shrdv_epi32(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_shrdv_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shrdv_epi32
-  // CHECK: @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm256_shrdv_epi32
+  // CHECK: call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_shrdv_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_mask_shrdv_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shrdv_epi32
-  // CHECK: @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_shrdv_epi32
+  // CHECK: call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_shrdv_epi32(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_shrdv_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shrdv_epi32
-  // CHECK: @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_shrdv_epi32
+  // CHECK: call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_shrdv_epi32(__U, __S, __A, __B);
 }
 
 __m128i test_mm_shrdv_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shrdv_epi32
-  // CHECK: @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+  // CHECK-LABEL: test_mm_shrdv_epi32
+  // CHECK: call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_shrdv_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_mask_shrdv_epi16(__m256i __S, __mmask16 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_shrdv_epi16
-  // CHECK: @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm256_mask_shrdv_epi16
+  // CHECK: call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_mask_shrdv_epi16(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_shrdv_epi16(__mmask16 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_shrdv_epi16
-  // CHECK: @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm256_maskz_shrdv_epi16
+  // CHECK: call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}}
   return _mm256_maskz_shrdv_epi16(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_shrdv_epi16(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_shrdv_epi16
-  // CHECK: @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm256_shrdv_epi16
+  // CHECK: call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}})
   return _mm256_shrdv_epi16(__S, __A, __B);
 }
 
 __m128i test_mm_mask_shrdv_epi16(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_shrdv_epi16
-  // CHECK: @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm_mask_shrdv_epi16
+  // CHECK: call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_mask_shrdv_epi16(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_shrdv_epi16(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_shrdv_epi16
-  // CHECK: @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm_maskz_shrdv_epi16
+  // CHECK: call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}}
   return _mm_maskz_shrdv_epi16(__U, __S, __A, __B);
 }
 
 __m128i test_mm_shrdv_epi16(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_shrdv_epi16
-  // CHECK: @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
+  // CHECK-LABEL: test_mm_shrdv_epi16
+  // CHECK: call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_shrdv_epi16(__S, __A, __B);
 }
 
diff --git a/clang/test/CodeGen/X86/avx512vlvnni-builtins.c b/clang/test/CodeGen/X86/avx512vlvnni-builtins.c
index a69412dbe492e..3de4cca1a7e23 100644
--- a/clang/test/CodeGen/X86/avx512vlvnni-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vlvnni-builtins.c
@@ -1,164 +1,167 @@
-//  RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vnni -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vnni -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vnni -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vnni -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vnni -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 __m256i test_mm256_mask_dpbusd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.256
+  // CHECK-LABEL: test_mm256_mask_dpbusd_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_dpbusd_epi32(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_dpbusd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.256
+  // CHECK-LABEL: test_mm256_maskz_dpbusd_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_dpbusd_epi32(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_dpbusd_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.256
+  // CHECK-LABEL: test_mm256_dpbusd_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpbusd_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_mask_dpbusds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.256
+  // CHECK-LABEL: test_mm256_mask_dpbusds_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_dpbusds_epi32(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_dpbusds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.256
+  // CHECK-LABEL: test_mm256_maskz_dpbusds_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_dpbusds_epi32(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_dpbusds_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.256
+  // CHECK-LABEL: test_mm256_dpbusds_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpbusds_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_mask_dpwssd_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.256
+  // CHECK-LABEL: test_mm256_mask_dpwssd_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_dpwssd_epi32(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_dpwssd_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.256
+  // CHECK-LABEL: test_mm256_maskz_dpwssd_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_dpwssd_epi32(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_dpwssd_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.256
+  // CHECK-LABEL: test_mm256_dpwssd_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwssd_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_mask_dpwssds_epi32(__m256i __S, __mmask8 __U, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_mask_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.256
+  // CHECK-LABEL: test_mm256_mask_dpwssds_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_dpwssds_epi32(__S, __U, __A, __B);
 }
 
 __m256i test_mm256_maskz_dpwssds_epi32(__mmask8 __U, __m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_maskz_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.256
+  // CHECK-LABEL: test_mm256_maskz_dpwssds_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   // CHECK: select <8 x i1> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_dpwssds_epi32(__U, __S, __A, __B);
 }
 
 __m256i test_mm256_dpwssds_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.256
+  // CHECK-LABEL: test_mm256_dpwssds_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwssds_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_mask_dpbusd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.128
+  // CHECK-LABEL: test_mm_mask_dpbusd_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_dpbusd_epi32(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_dpbusd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.128
+  // CHECK-LABEL: test_mm_maskz_dpbusd_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_dpbusd_epi32(__U, __S, __A, __B);
 }
 
 __m128i test_mm_dpbusd_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.128
+  // CHECK-LABEL: test_mm_dpbusd_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpbusd_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_mask_dpbusds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.128
+  // CHECK-LABEL: test_mm_mask_dpbusds_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_dpbusds_epi32(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_dpbusds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.128
+  // CHECK-LABEL: test_mm_maskz_dpbusds_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_dpbusds_epi32(__U, __S, __A, __B);
 }
 
 __m128i test_mm_dpbusds_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.128
+  // CHECK-LABEL: test_mm_dpbusds_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpbusds_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_mask_dpwssd_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.128
+  // CHECK-LABEL: test_mm_mask_dpwssd_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_dpwssd_epi32(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_dpwssd_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.128
+  // CHECK-LABEL: test_mm_maskz_dpwssd_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_dpwssd_epi32(__U, __S, __A, __B);
 }
 
 __m128i test_mm_dpwssd_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.128
+  // CHECK-LABEL: test_mm_dpwssd_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwssd_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_mask_dpwssds_epi32(__m128i __S, __mmask8 __U, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_mask_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.128
+  // CHECK-LABEL: test_mm_mask_dpwssds_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_dpwssds_epi32(__S, __U, __A, __B);
 }
 
 __m128i test_mm_maskz_dpwssds_epi32(__mmask8 __U, __m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_maskz_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.128
+  // CHECK-LABEL: test_mm_maskz_dpwssds_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   // CHECK: select <4 x i1> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_dpwssds_epi32(__U, __S, __A, __B);
 }
 
 __m128i test_mm_dpwssds_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.128
+  // CHECK-LABEL: test_mm_dpwssds_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwssds_epi32(__S, __A, __B);
 }
 
diff --git a/clang/test/CodeGen/X86/avx512vlvp2intersect-builtins.c b/clang/test/CodeGen/X86/avx512vlvp2intersect-builtins.c
index 1b360d1d7c5ba..37ce638b02df1 100644
--- a/clang/test/CodeGen/X86/avx512vlvp2intersect-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vlvp2intersect-builtins.c
@@ -1,5 +1,7 @@
-// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -flax-vector-conversions=none -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -flax-vector-conversions=none -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/X86/avx512vnni-builtins.c b/clang/test/CodeGen/X86/avx512vnni-builtins.c
index db39fb096b3c4..a0177b3ba0a2c 100644
--- a/clang/test/CodeGen/X86/avx512vnni-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vnni-builtins.c
@@ -1,84 +1,87 @@
-//  RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vnni -emit-llvm -o - -Wall -Werror | FileCheck %s
+//  RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vnni -emit-llvm -o - -Wall -Werror | FileCheck %s
+//  RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vnni -emit-llvm -o - -Wall -Werror | FileCheck %s
+//  RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vnni -emit-llvm -o - -Wall -Werror | FileCheck %s
+//  RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vnni -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 __m512i test_mm512_mask_dpbusd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.512
+  // CHECK-LABEL: test_mm512_mask_dpbusd_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_dpbusd_epi32(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_dpbusd_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.512
+  // CHECK-LABEL: test_mm512_maskz_dpbusd_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_dpbusd_epi32(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_dpbusd_epi32(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.512
+  // CHECK-LABEL: test_mm512_dpbusd_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   return _mm512_dpbusd_epi32(__S, __A, __B);
 }
 
 __m512i test_mm512_mask_dpbusds_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.51
+  // CHECK-LABEL: test_mm512_mask_dpbusds_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_dpbusds_epi32(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_dpbusds_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.512
+  // CHECK-LABEL: test_mm512_maskz_dpbusds_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_dpbusds_epi32(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_dpbusds_epi32(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.512
+  // CHECK-LABEL: test_mm512_dpbusds_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   return _mm512_dpbusds_epi32(__S, __A, __B);
 }
 
 __m512i test_mm512_mask_dpwssd_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.512
+  // CHECK-LABEL: test_mm512_mask_dpwssd_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_dpwssd_epi32(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_dpwssd_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.512
+  // CHECK-LABEL: test_mm512_maskz_dpwssd_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_dpwssd_epi32(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_dpwssd_epi32(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.512
+  // CHECK-LABEL: test_mm512_dpwssd_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   return _mm512_dpwssd_epi32(__S, __A, __B);
 }
 
 __m512i test_mm512_mask_dpwssds_epi32(__m512i __S, __mmask16 __U, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_mask_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.512
+  // CHECK-LABEL: test_mm512_mask_dpwssds_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_dpwssds_epi32(__S, __U, __A, __B);
 }
 
 __m512i test_mm512_maskz_dpwssds_epi32(__mmask16 __U, __m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_maskz_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.512
+  // CHECK-LABEL: test_mm512_maskz_dpwssds_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   // CHECK: select <16 x i1> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_dpwssds_epi32(__U, __S, __A, __B);
 }
 
 __m512i test_mm512_dpwssds_epi32(__m512i __S, __m512i __A, __m512i __B) {
-  // CHECK-LABEL: @test_mm512_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.512
+  // CHECK-LABEL: test_mm512_dpwssds_epi32
+  // CHECK: call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %{{.*}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
   return _mm512_dpwssds_epi32(__S, __A, __B);
 }
 
diff --git a/clang/test/CodeGen/X86/avx512vp2intersect-builtins.c b/clang/test/CodeGen/X86/avx512vp2intersect-builtins.c
index 2a3d38acfbf2c..16704e8913322 100644
--- a/clang/test/CodeGen/X86/avx512vp2intersect-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vp2intersect-builtins.c
@@ -1,5 +1,7 @@
-// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -flax-vector-conversions=none -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -flax-vector-conversions=none -ffreestanding -triple=i386-unknown-unknown -target-feature +avx512vp2intersect -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
diff --git a/clang/test/CodeGen/X86/avx512vpopcntdq-builtins.c b/clang/test/CodeGen/X86/avx512vpopcntdq-builtins.c
index ca8f5e482cc56..b80ffdf0dcc7c 100644
--- a/clang/test/CodeGen/X86/avx512vpopcntdq-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vpopcntdq-builtins.c
@@ -1,46 +1,53 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vpopcntdq -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vpopcntdq -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 #include "builtin_test_helpers.h"
 
 __m512i test_mm512_popcnt_epi64(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_popcnt_epi64
+  // CHECK-LABEL: test_mm512_popcnt_epi64
   // CHECK: @llvm.ctpop.v8i64
   return _mm512_popcnt_epi64(__A);
 }
-TEST_CONSTEXPR(match_v8di(_mm512_popcnt_epi64((__m512i)(__v8di){+5, -3, -10, +8, 0, -256, +256, -128}), 2, 31, 30, 1, 0, 24, 1, 25));
+TEST_CONSTEXPR(match_v8di(_mm512_popcnt_epi64((__m512i)(__v8di){+5, -3, -10, +8, 0, -256, +256, -128}), 2, 63, 62, 1, 0, 56, 1, 57));
 
 __m512i test_mm512_mask_popcnt_epi64(__m512i __W, __mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_popcnt_epi64
+  // CHECK-LABEL: test_mm512_mask_popcnt_epi64
   // CHECK: @llvm.ctpop.v8i64
   // CHECK: select <8 x i1> %{{[0-9]+}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_mask_popcnt_epi64(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_mask_popcnt_epi64(_mm512_set1_epi64(-1), 0x81, (__m512i)(__v8di){+5, -3, -10, +8, 0, -256, +256, -128}), 2, -1, -1, -1, -1, -1, -1, 57));
 
 __m512i test_mm512_maskz_popcnt_epi64(__mmask8 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_popcnt_epi64
+  // CHECK-LABEL: test_mm512_maskz_popcnt_epi64
   // CHECK: @llvm.ctpop.v8i64
   // CHECK: select <8 x i1> %{{[0-9]+}}, <8 x i64> %{{.*}}, <8 x i64> %{{.*}}
   return _mm512_maskz_popcnt_epi64(__U, __A);
 }
+TEST_CONSTEXPR(match_v8di(_mm512_maskz_popcnt_epi64(0x42, (__m512i)(__v8di){+5, -3, -10, +8, 0, -256, +256, -128}), 0, 63, 0, 0, 0, 0, 1, 0));
 
 __m512i test_mm512_popcnt_epi32(__m512i __A) {
-  // CHECK-LABEL: @test_mm512_popcnt_epi32
+  // CHECK-LABEL: test_mm512_popcnt_epi32
   // CHECK: @llvm.ctpop.v16i32
   return _mm512_popcnt_epi32(__A);
 }
 TEST_CONSTEXPR(match_v16si(_mm512_popcnt_epi32((__m512i)(__v16si){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), 2, 31, 30, 1, 0, 24, 1, 25, 2, 2, 4, 2, 6, 2, 9, 2));
 
 __m512i test_mm512_mask_popcnt_epi32(__m512i __W, __mmask16 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_mask_popcnt_epi32
+  // CHECK-LABEL: test_mm512_mask_popcnt_epi32
   // CHECK: @llvm.ctpop.v16i32
   // CHECK: select <16 x i1> %{{[0-9]+}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_mask_popcnt_epi32(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_mask_popcnt_epi32(_mm512_set1_epi32(-1), 0x0F81, (__m512i)(__v16si){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), 2, -1, -1, -1, -1, -1, -1, 25, 2, 2, 4, 2, -1, -1, -1, -1));
 
 __m512i test_mm512_maskz_popcnt_epi32(__mmask16 __U, __m512i __A) {
-  // CHECK-LABEL: @test_mm512_maskz_popcnt_epi32
+  // CHECK-LABEL: test_mm512_maskz_popcnt_epi32
   // CHECK: @llvm.ctpop.v16i32
   // CHECK: select <16 x i1> %{{[0-9]+}}, <16 x i32> %{{.*}}, <16 x i32> %{{.*}}
   return _mm512_maskz_popcnt_epi32(__U, __A);
 }
+TEST_CONSTEXPR(match_v16si(_mm512_maskz_popcnt_epi32(0xF042, (__m512i)(__v16si){+5, -3, -10, +8, 0, -256, +256, -128, +3, +9, +15, +33, +63, +129, +511, +1025}), 0, 31, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 2, 9, 2));
diff --git a/clang/test/CodeGen/X86/avx512vpopcntdqvl-builtins.c b/clang/test/CodeGen/X86/avx512vpopcntdqvl-builtins.c
index 5d18b68e46905..7258034c57adc 100644
--- a/clang/test/CodeGen/X86/avx512vpopcntdqvl-builtins.c
+++ b/clang/test/CodeGen/X86/avx512vpopcntdqvl-builtins.c
@@ -1,88 +1,99 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vpopcntdq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512vpopcntdq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +avx512vpopcntdq -target-feature +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 #include "builtin_test_helpers.h"
 
 __m128i test_mm_popcnt_epi64(__m128i __A) {
-  // CHECK-LABEL: @test_mm_popcnt_epi64
+  // CHECK-LABEL: test_mm_popcnt_epi64
   // CHECK: @llvm.ctpop.v2i64
   return _mm_popcnt_epi64(__A);
 }
 TEST_CONSTEXPR(match_v2di(_mm_popcnt_epi64((__m128i)(__v2di){+5, -3}), 2, 63));
 
 __m128i test_mm_mask_popcnt_epi64(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_popcnt_epi64
+  // CHECK-LABEL: test_mm_mask_popcnt_epi64
   // CHECK: @llvm.ctpop.v2i64
   // CHECK: select <2 x i1> %{{.+}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_mask_popcnt_epi64(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_v2di(_mm_mask_popcnt_epi64(_mm_set1_epi64x(-1), 0x2, (__m128i)(__v2di){+5, -3}), -1, 63));
 
 __m128i test_mm_maskz_popcnt_epi64(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_popcnt_epi64
+  // CHECK-LABEL: test_mm_maskz_popcnt_epi64
   // CHECK: @llvm.ctpop.v2i64
   // CHECK: select <2 x i1> %{{.+}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}}
   return _mm_maskz_popcnt_epi64(__U, __A);
 }
+TEST_CONSTEXPR(match_v2di(_mm_maskz_popcnt_epi64(0x1, (__m128i)(__v2di){+5, -3}), 2, 0));
 
 __m128i test_mm_popcnt_epi32(__m128i __A) {
-  // CHECK-LABEL: @test_mm_popcnt_epi32
+  // CHECK-LABEL: test_mm_popcnt_epi32
   // CHECK: @llvm.ctpop.v4i32
   return _mm_popcnt_epi32(__A);
 }
 TEST_CONSTEXPR(match_v4si(_mm_popcnt_epi32((__m128i)(__v4si){+5, -3, -10, +8}), 2, 31, 30, 1));
 
 __m128i test_mm_mask_popcnt_epi32(__m128i __W, __mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_mask_popcnt_epi32
+  // CHECK-LABEL: test_mm_mask_popcnt_epi32
   // CHECK: @llvm.ctpop.v4i32
   // CHECK: select <4 x i1> %{{.+}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_mask_popcnt_epi32(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_v4si(_mm_mask_popcnt_epi32(_mm_set1_epi32(-1), 0x3, (__m128i)(__v4si){+5, -3, -10, +8}), 2, 31, -1, -1));
 
 __m128i test_mm_maskz_popcnt_epi32(__mmask8 __U, __m128i __A) {
-  // CHECK-LABEL: @test_mm_maskz_popcnt_epi32
+  // CHECK-LABEL: test_mm_maskz_popcnt_epi32
   // CHECK: @llvm.ctpop.v4i32
   // CHECK: select <4 x i1> %{{.+}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}}
   return _mm_maskz_popcnt_epi32(__U, __A);
 }
+TEST_CONSTEXPR(match_v4si(_mm_maskz_popcnt_epi32(0x5, (__m128i)(__v4si){+5, -3, -10, +8}), 2, 0, 30, 0));
 
 __m256i test_mm256_popcnt_epi64(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_popcnt_epi64
+  // CHECK-LABEL: test_mm256_popcnt_epi64
   // CHECK: @llvm.ctpop.v4i64
   return _mm256_popcnt_epi64(__A);
 }
 TEST_CONSTEXPR(match_v4di(_mm256_popcnt_epi64((__m256i)(__v4di){+5, -3, -10, +8}), 2, 63, 62, 1));
 
 __m256i test_mm256_mask_popcnt_epi64(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_popcnt_epi64
+  // CHECK-LABEL: test_mm256_mask_popcnt_epi64
   // CHECK: @llvm.ctpop.v4i64
   // CHECK: select <4 x i1> %{{.+}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_mask_popcnt_epi64(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_mask_popcnt_epi64(_mm256_set1_epi64x(-1), 0x3, (__m256i)(__v4di){+5, -3, -10, +8}), 2, 63, -1, -1));
 
 __m256i test_mm256_maskz_popcnt_epi64(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_popcnt_epi64
+  // CHECK-LABEL: test_mm256_maskz_popcnt_epi64
   // CHECK: @llvm.ctpop.v4i64
   // CHECK: select <4 x i1> %{{.+}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}}
   return _mm256_maskz_popcnt_epi64(__U, __A);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_maskz_popcnt_epi64(0x5, (__m256i)(__v4di){+5, -3, -10, +8}), 2, 0, 62, 0));
 
 __m256i test_mm256_popcnt_epi32(__m256i __A) {
-  // CHECK-LABEL: @test_mm256_popcnt_epi32
+  // CHECK-LABEL: test_mm256_popcnt_epi32
   // CHECK: @llvm.ctpop.v8i32
   return _mm256_popcnt_epi32(__A);
 }
 TEST_CONSTEXPR(match_v8si(_mm256_popcnt_epi32((__m256i)(__v8si){+5, -3, -10, +8, 0, -256, +256, -128}), 2, 31, 30, 1, 0, 24, 1, 25));
 
 __m256i test_mm256_mask_popcnt_epi32(__m256i __W, __mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_mask_popcnt_epi32
+  // CHECK-LABEL: test_mm256_mask_popcnt_epi32
   // CHECK: @llvm.ctpop.v8i32
   // CHECK: select <8 x i1> %{{.+}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_mask_popcnt_epi32(__W, __U, __A);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_mask_popcnt_epi32(_mm256_set1_epi32(-1), 0x37, (__m256i)(__v8si){+5, -3, -10, +8, 0, -256, +256, -128}), 2, 31, 30, -1, 0, 24, -1, -1));
 
 __m256i test_mm256_maskz_popcnt_epi32(__mmask8 __U, __m256i __A) {
-  // CHECK-LABEL: @test_mm256_maskz_popcnt_epi32
+  // CHECK-LABEL: test_mm256_maskz_popcnt_epi32
   // CHECK: @llvm.ctpop.v8i32
   // CHECK: select <8 x i1> %{{.+}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}}
   return _mm256_maskz_popcnt_epi32(__U, __A);
 }
+TEST_CONSTEXPR(match_v8si(_mm256_maskz_popcnt_epi32(0x8C, (__m256i)(__v8si){+5, -3, -10, +8, 0, -256, +256, -128}), 0, 0, 30, 1, 0, 0, 0, 25));
diff --git a/clang/test/CodeGen/X86/avxifma-builtins.c b/clang/test/CodeGen/X86/avxifma-builtins.c
index 56e434cd7b164..dd0f220b378b4 100644
--- a/clang/test/CodeGen/X86/avxifma-builtins.c
+++ b/clang/test/CodeGen/X86/avxifma-builtins.c
@@ -1,52 +1,54 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avxifma -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -ffreestanding %s -triple=i386-apple-darwin -target-feature +avxifma -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avxifma -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=i386-apple-darwin -target-feature +avxifma -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avxifma -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=i386-apple-darwin -target-feature +avxifma -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 __m128i test_mm_madd52hi_epu64(__m128i __X, __m128i __Y, __m128i __Z) {
-// CHECK-LABEL: @test_mm_madd52hi_epu64
-// CHECK:    call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128
+// CHECK-LABEL: test_mm_madd52hi_epu64
+// CHECK:    call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_madd52hi_epu64(__X, __Y, __Z);
 }
 
 __m256i test_mm256_madd52hi_epu64(__m256i __X, __m256i __Y, __m256i __Z) {
-// CHECK-LABEL: @test_mm256_madd52hi_epu64
-// CHECK:    call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256
+// CHECK-LABEL: test_mm256_madd52hi_epu64
+// CHECK:    call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_madd52hi_epu64(__X, __Y, __Z);
 }
 
 __m128i test_mm_madd52lo_epu64(__m128i __X, __m128i __Y, __m128i __Z) {
-// CHECK-LABEL: @test_mm_madd52lo_epu64
-// CHECK:    call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128
+// CHECK-LABEL: test_mm_madd52lo_epu64
+// CHECK:    call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_madd52lo_epu64(__X, __Y, __Z);
 }
 
 __m256i test_mm256_madd52lo_epu64(__m256i __X, __m256i __Y, __m256i __Z) {
-// CHECK-LABEL: @test_mm256_madd52lo_epu64
-// CHECK:    call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256
+// CHECK-LABEL: test_mm256_madd52lo_epu64
+// CHECK:    call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_madd52lo_epu64(__X, __Y, __Z);
 }
 
 __m128i test_mm_madd52hi_avx_epu64(__m128i __X, __m128i __Y, __m128i __Z) {
-// CHECK-LABEL: @test_mm_madd52hi_avx_epu64
-// CHECK:    call <2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128
+// CHECK-LABEL: test_mm_madd52hi_avx_epu64
+// CHECK:    call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52h.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_madd52hi_avx_epu64(__X, __Y, __Z);
 }
 
 __m256i test_mm256_madd52hi_avx_epu64(__m256i __X, __m256i __Y, __m256i __Z) {
-// CHECK-LABEL: @test_mm256_madd52hi_avx_epu64
-// CHECK:    call <4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256
+// CHECK-LABEL: test_mm256_madd52hi_avx_epu64
+// CHECK:    call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52h.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_madd52hi_avx_epu64(__X, __Y, __Z);
 }
 
 __m128i test_mm_madd52lo_avx_epu64(__m128i __X, __m128i __Y, __m128i __Z) {
-// CHECK-LABEL: @test_mm_madd52lo_avx_epu64
-// CHECK:    call <2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128
+// CHECK-LABEL: test_mm_madd52lo_avx_epu64
+// CHECK:    call {{.*}}<2 x i64> @llvm.x86.avx512.vpmadd52l.uq.128(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_madd52lo_avx_epu64(__X, __Y, __Z);
 }
 
 __m256i test_mm256_madd52lo_avx_epu64(__m256i __X, __m256i __Y, __m256i __Z) {
-// CHECK-LABEL: @test_mm256_madd52lo_avx_epu64
-// CHECK:    call <4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256
+// CHECK-LABEL: test_mm256_madd52lo_avx_epu64
+// CHECK:    call {{.*}}<4 x i64> @llvm.x86.avx512.vpmadd52l.uq.256(<4 x i64> %{{.*}}, <4 x i64> %{{.*}}, <4 x i64> %{{.*}})
   return _mm256_madd52lo_avx_epu64(__X, __Y, __Z);
 }
diff --git a/clang/test/CodeGen/X86/avxvnni-builtins.c b/clang/test/CodeGen/X86/avxvnni-builtins.c
index 089578df78e0c..bb28a359424c8 100644
--- a/clang/test/CodeGen/X86/avxvnni-builtins.c
+++ b/clang/test/CodeGen/X86/avxvnni-builtins.c
@@ -1,100 +1,102 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avxvnni -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -ffreestanding %s -triple=i386-apple-darwin -target-feature +avxvnni -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avxvnni -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=i386-apple-darwin -target-feature +avxvnni -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avxvnni -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=i386-apple-darwin -target-feature +avxvnni -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 __m256i test_mm256_dpbusd_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.256
+  // CHECK-LABEL: test_mm256_dpbusd_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpbusd_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_dpbusds_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.256
+  // CHECK-LABEL: test_mm256_dpbusds_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpbusds_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_dpwssd_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.256
+  // CHECK-LABEL: test_mm256_dpwssd_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwssd_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_dpwssds_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.256
+  // CHECK-LABEL: test_mm256_dpwssds_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwssds_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_dpbusd_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpbusd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.128
+  // CHECK-LABEL: test_mm_dpbusd_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpbusd_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_dpbusds_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpbusds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.128
+  // CHECK-LABEL: test_mm_dpbusds_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpbusds_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_dpwssd_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpwssd_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.128
+  // CHECK-LABEL: test_mm_dpwssd_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwssd_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_dpwssds_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpwssds_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.128
+  // CHECK-LABEL: test_mm_dpwssds_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwssds_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_dpbusd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpbusd_avx_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.256
+  // CHECK-LABEL: test_mm256_dpbusd_avx_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpbusd_avx_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_dpbusds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpbusds_avx_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.256
+  // CHECK-LABEL: test_mm256_dpbusds_avx_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpbusds_avx_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_dpwssd_avx_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpwssd_avx_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.256
+  // CHECK-LABEL: test_mm256_dpwssd_avx_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwssd_avx_epi32(__S, __A, __B);
 }
 
 __m256i test_mm256_dpwssds_avx_epi32(__m256i __S, __m256i __A, __m256i __B) {
-  // CHECK-LABEL: @test_mm256_dpwssds_avx_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.256
+  // CHECK-LABEL: test_mm256_dpwssds_avx_epi32
+  // CHECK: call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwssds_avx_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_dpbusd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpbusd_avx_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusd.128
+  // CHECK-LABEL: test_mm_dpbusd_avx_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpbusd_avx_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_dpbusds_avx_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpbusds_avx_epi32
-  // CHECK: @llvm.x86.avx512.vpdpbusds.128
+  // CHECK-LABEL: test_mm_dpbusds_avx_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpbusds_avx_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_dpwssd_avx_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpwssd_avx_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssd.128
+  // CHECK-LABEL: test_mm_dpwssd_avx_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwssd_avx_epi32(__S, __A, __B);
 }
 
 __m128i test_mm_dpwssds_avx_epi32(__m128i __S, __m128i __A, __m128i __B) {
-  // CHECK-LABEL: @test_mm_dpwssds_avx_epi32
-  // CHECK: @llvm.x86.avx512.vpdpwssds.128
+  // CHECK-LABEL: test_mm_dpwssds_avx_epi32
+  // CHECK: call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwssds_avx_epi32(__S, __A, __B);
 }
diff --git a/clang/test/CodeGen/X86/avxvnniint16-builtins.c b/clang/test/CodeGen/X86/avxvnniint16-builtins.c
index f9feaea1e244d..c25367bdd8502 100644
--- a/clang/test/CodeGen/X86/avxvnniint16-builtins.c
+++ b/clang/test/CodeGen/X86/avxvnniint16-builtins.c
@@ -1,78 +1,82 @@
-// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avxvnniint16 -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 %s -ffreestanding -triple=i386-unknown-unknown -target-feature +avxvnniint16 -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 %s -ffreestanding -triple=i386-unknown-unknown -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avxvnniint16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -ffreestanding -triple=i386-unknown-unknown -target-feature +avxvnniint16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c %s -ffreestanding -triple=i386-unknown-unknown -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avxvnniint16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -ffreestanding -triple=i386-unknown-unknown -target-feature +avxvnniint16 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ %s -ffreestanding -triple=i386-unknown-unknown -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
 __m128i test_mm_dpwsud_epi32(__m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_dpwsud_epi32(
+  // CHECK-LABEL: test_mm_dpwsud_epi32
   // CHECK: call <4 x i32> @llvm.x86.avx2.vpdpwsud.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwsud_epi32(__A, __B, __C);
 }
 
 __m256i test_mm256_dpwsud_epi32(__m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_dpwsud_epi32(
+  // CHECK-LABEL: test_mm256_dpwsud_epi32
   // CHECK: call <8 x i32> @llvm.x86.avx2.vpdpwsud.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwsud_epi32(__A, __B, __C);
 }
 
 __m128i test_mm_dpwsuds_epi32(__m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_dpwsuds_epi32(
+  // CHECK-LABEL: test_mm_dpwsuds_epi32
   // CHECK: call <4 x i32> @llvm.x86.avx2.vpdpwsuds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwsuds_epi32(__A, __B, __C);
 }
 
 __m256i test_mm256_dpwsuds_epi32(__m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_dpwsuds_epi32(
+  // CHECK-LABEL: test_mm256_dpwsuds_epi32
   // CHECK: call <8 x i32> @llvm.x86.avx2.vpdpwsuds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwsuds_epi32(__A, __B, __C);
 }
 
 __m128i test_mm_dpwusd_epi32(__m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_dpwusd_epi32(
+  // CHECK-LABEL: test_mm_dpwusd_epi32
   // CHECK: call <4 x i32> @llvm.x86.avx2.vpdpwusd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwusd_epi32(__A, __B, __C);
 }
 
 __m256i test_mm256_dpwusd_epi32(__m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_dpwusd_epi32(
+  // CHECK-LABEL: test_mm256_dpwusd_epi32
   // CHECK: call <8 x i32> @llvm.x86.avx2.vpdpwusd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwusd_epi32(__A, __B, __C);
 }
 
 __m128i test_mm_dpwusds_epi32(__m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_dpwusds_epi32(
+  // CHECK-LABEL: test_mm_dpwusds_epi32
   // CHECK: call <4 x i32> @llvm.x86.avx2.vpdpwusds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwusds_epi32(__A, __B, __C);
 }
 
 __m256i test_mm256_dpwusds_epi32(__m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_dpwusds_epi32(
+  // CHECK-LABEL: test_mm256_dpwusds_epi32
   // CHECK: call <8 x i32> @llvm.x86.avx2.vpdpwusds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwusds_epi32(__A, __B, __C);
 }
 
 __m128i test_mm_dpwuud_epi32(__m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_dpwuud_epi32(
+  // CHECK-LABEL: test_mm_dpwuud_epi32
   // CHECK: call <4 x i32> @llvm.x86.avx2.vpdpwuud.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwuud_epi32(__A, __B, __C);
 }
 
 __m256i test_mm256_dpwuud_epi32(__m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_dpwuud_epi32(
+  // CHECK-LABEL: test_mm256_dpwuud_epi32
   // CHECK: call <8 x i32> @llvm.x86.avx2.vpdpwuud.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwuud_epi32(__A, __B, __C);
 }
 
 __m128i test_mm_dpwuuds_epi32(__m128i __A, __m128i __B, __m128i __C) {
-  // CHECK-LABEL: @test_mm_dpwuuds_epi32(
+  // CHECK-LABEL: test_mm_dpwuuds_epi32
   // CHECK: call <4 x i32> @llvm.x86.avx2.vpdpwuuds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
   return _mm_dpwuuds_epi32(__A, __B, __C);
 }
 
 __m256i test_mm256_dpwuuds_epi32(__m256i __A, __m256i __B, __m256i __C) {
-  // CHECK-LABEL: @test_mm256_dpwuuds_epi32(
+  // CHECK-LABEL: test_mm256_dpwuuds_epi32
   // CHECK: call <8 x i32> @llvm.x86.avx2.vpdpwuuds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_dpwuuds_epi32(__A, __B, __C);
 }
diff --git a/clang/test/CodeGen/X86/avxvnniint8-builtins.c b/clang/test/CodeGen/X86/avxvnniint8-builtins.c
index 80d005c16d387..f808dee40519c 100644
--- a/clang/test/CodeGen/X86/avxvnniint8-builtins.c
+++ b/clang/test/CodeGen/X86/avxvnniint8-builtins.c
@@ -1,78 +1,82 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64- -target-feature +avxvnniint8 -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -ffreestanding %s -triple=i386-   -target-feature +avxvnniint8 -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64- -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -ffreestanding %s -triple=i386-   -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64- -target-feature +avxvnniint8 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=i386-   -target-feature +avxvnniint8 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=x86_64- -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -ffreestanding %s -triple=i386-   -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64- -target-feature +avxvnniint8 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=i386-   -target-feature +avxvnniint8 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=x86_64- -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -ffreestanding %s -triple=i386-   -target-feature +avx10.2-256 -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
-// CHECK-LABEL: @test_mm_dpbssd_epi32(
-// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbssd.128
+// CHECK-LABEL: test_mm_dpbssd_epi32
+// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbssd.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
 __m128i test_mm_dpbssd_epi32(__m128i __W, __m128i __A, __m128i __B) {
   return _mm_dpbssd_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm_dpbssds_epi32(
-// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbssds.128
+// CHECK-LABEL: test_mm_dpbssds_epi32
+// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbssds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
 __m128i test_mm_dpbssds_epi32(__m128i __W, __m128i __A, __m128i __B) {
   return _mm_dpbssds_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm_dpbsud_epi32(
-// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbsud.128
+// CHECK-LABEL: test_mm_dpbsud_epi32
+// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbsud.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
 __m128i test_mm_dpbsud_epi32(__m128i __W, __m128i __A, __m128i __B) {
   return _mm_dpbsud_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm_dpbsuds_epi32(
-// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbsuds.128
+// CHECK-LABEL: test_mm_dpbsuds_epi32
+// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbsuds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
 __m128i test_mm_dpbsuds_epi32(__m128i __W, __m128i __A, __m128i __B) {
   return _mm_dpbsuds_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm_dpbuud_epi32(
-// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbuud.128
+// CHECK-LABEL: test_mm_dpbuud_epi32
+// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbuud.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
 __m128i test_mm_dpbuud_epi32(__m128i __W, __m128i __A, __m128i __B) {
   return _mm_dpbuud_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm_dpbuuds_epi32(
-// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbuuds.128
+// CHECK-LABEL: test_mm_dpbuuds_epi32
+// CHECK:     call <4 x i32> @llvm.x86.avx2.vpdpbuuds.128(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
 __m128i test_mm_dpbuuds_epi32(__m128i __W, __m128i __A, __m128i __B) {
   return _mm_dpbuuds_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm256_dpbssd_epi32(
-// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbssd.256
+// CHECK-LABEL: test_mm256_dpbssd_epi32
+// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbssd.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
 __m256i test_mm256_dpbssd_epi32(__m256i __W, __m256i __A, __m256i __B) {
   return _mm256_dpbssd_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm256_dpbssds_epi32(
-// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbssds.256
+// CHECK-LABEL: test_mm256_dpbssds_epi32
+// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbssds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
 __m256i test_mm256_dpbssds_epi32(__m256i __W, __m256i __A, __m256i __B) {
   return _mm256_dpbssds_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm256_dpbsud_epi32(
-// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbsud.256
+// CHECK-LABEL: test_mm256_dpbsud_epi32
+// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbsud.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
 __m256i test_mm256_dpbsud_epi32(__m256i __W, __m256i __A, __m256i __B) {
   return _mm256_dpbsud_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm256_dpbsuds_epi32(
-// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbsuds.256
+// CHECK-LABEL: test_mm256_dpbsuds_epi32
+// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbsuds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
 __m256i test_mm256_dpbsuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
   return _mm256_dpbsuds_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm256_dpbuud_epi32(
-// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbuud.256
+// CHECK-LABEL: test_mm256_dpbuud_epi32
+// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbuud.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
 __m256i test_mm256_dpbuud_epi32(__m256i __W, __m256i __A, __m256i __B) {
   return _mm256_dpbuud_epi32(__W, __A, __B);
 }
 
-// CHECK-LABEL: @test_mm256_dpbuuds_epi32(
-// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbuuds.256
+// CHECK-LABEL: test_mm256_dpbuuds_epi32
+// CHECK:     call <8 x i32> @llvm.x86.avx2.vpdpbuuds.256(<8 x i32> %{{.*}}, <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
 __m256i test_mm256_dpbuuds_epi32(__m256i __W, __m256i __A, __m256i __B) {
   return _mm256_dpbuuds_epi32(__W, __A, __B);
 }
diff --git a/clang/test/CodeGen/X86/builtin_test_helpers.h b/clang/test/CodeGen/X86/builtin_test_helpers.h
index f719694d41e25..6541ca4fd922d 100644
--- a/clang/test/CodeGen/X86/builtin_test_helpers.h
+++ b/clang/test/CodeGen/X86/builtin_test_helpers.h
@@ -23,17 +23,35 @@ constexpr bool match_v4hi(__m64 _v, short a, short b, short c, short d) {
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
 }
 
-constexpr bool match_v8qi(__m64 _v, char a, char b, char c, char d, char e, char f, char g, char h) {
-  __v8qi v = (__v8qi)_v;
+constexpr bool match_v4hu(__m64 _v, unsigned short a, unsigned short b, unsigned short c, unsigned short d) {
+  __v4hu v = (__v4hu)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
+}
+
+constexpr bool match_v8qi(__m64 _v, signed char a, signed char b, signed char c, signed char d, signed char e, signed char f, signed char g, signed char h) {
+  __v8qs v = (__v8qs)_v;
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
 }
 
-constexpr bool match_m128(__m128 v, float a, float b, float c, float d) {
-  return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
+constexpr bool match_v8qu(__m64 _v, unsigned char a, unsigned char b, unsigned char c, unsigned char d, unsigned char e, unsigned char f, unsigned char g, unsigned char h) {
+  __v8qu v = (__v8qu)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
 }
 
-constexpr bool match_m128d(__m128d v, double a, double b) {
-  return v[0] == a && v[1] == b;
+constexpr bool match_m128(__m128 _v, float a, float b, float c, float d) {
+  __v4su v = (__v4su)_v;
+  return v[0] == __builtin_bit_cast(unsigned, a) && v[1] == __builtin_bit_cast(unsigned, b) && v[2] == __builtin_bit_cast(unsigned, c) && v[3] == __builtin_bit_cast(unsigned, d);
+}
+
+constexpr bool match_m128d(__m128d _v, double a, double b) {
+  __v2du v = (__v2du)_v;
+  return v[0] == __builtin_bit_cast(unsigned long long, a) && v[1] == __builtin_bit_cast(unsigned long long, b);
+}
+
+constexpr bool match_m128h(__m128h _v, _Float16 __e00, _Float16 __e01, _Float16 __e02, _Float16 __e03, _Float16 __e04, _Float16 __e05, _Float16 __e06, _Float16 __e07) {
+  __v8hu v = (__v8hu)_v;
+  return v[ 0] == __builtin_bit_cast(unsigned short, __e00) && v[ 1] == __builtin_bit_cast(unsigned short, __e01) && v[ 2] == __builtin_bit_cast(unsigned short, __e02) && v[ 3] == __builtin_bit_cast(unsigned short, __e03) &&
+         v[ 4] == __builtin_bit_cast(unsigned short, __e04) && v[ 5] == __builtin_bit_cast(unsigned short, __e05) && v[ 6] == __builtin_bit_cast(unsigned short, __e06) && v[ 7] == __builtin_bit_cast(unsigned short, __e07);
 }
 
 constexpr bool match_m128i(__m128i _v, unsigned long long a, unsigned long long b) {
@@ -55,17 +73,39 @@ constexpr bool match_v8hi(__m128i _v, short a, short b, short c, short d, short
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
 }
 
-constexpr bool match_v16qi(__m128i _v, char a, char b, char c, char d, char e, char f, char g, char h, char i, char j, char k, char l, char m, char n, char o, char p) {
-  __v16qi v = (__v16qi)_v;
+constexpr bool match_v8hu(__m128i _v, unsigned short a, unsigned short b, unsigned short c, unsigned short d, unsigned short e, unsigned short f, unsigned short g, unsigned short h) {
+  __v8hu v = (__v8hu)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
+}
+
+constexpr bool match_v16qi(__m128i _v, signed char a, signed char b, signed char c, signed char d, signed char e, signed char f, signed char g, signed char h, signed char i, signed char j, signed char k, signed char l, signed char m, signed char n, signed char o, signed char p) {
+  __v16qs v = (__v16qs)_v;
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
 }
 
-constexpr bool match_m256(__m256 v, float a, float b, float c, float d, float e, float f, float g, float h) {
-  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
+constexpr bool match_v16qu(__m128i _v, unsigned char a, unsigned char b, unsigned char c, unsigned char d, unsigned char e, unsigned char f, unsigned char g, unsigned char h, unsigned char i, unsigned char j, unsigned char k, unsigned char l, unsigned char m, unsigned char n, unsigned char o, unsigned char p) {
+  __v16qu v = (__v16qu)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
 }
 
-constexpr bool match_m256d(__m256d v, double a, double b, double c, double d) {
-  return v[0] == a && v[1] == b && v[2] == c && v[3] == d;
+constexpr bool match_m256(__m256 _v, float __e00, float __e01, float __e02, float __e03, float __e04, float __e05, float __e06, float __e07) {
+  __v8su v = (__v8su)_v;
+  return v[ 0] == __builtin_bit_cast(unsigned, __e00) && v[ 1] == __builtin_bit_cast(unsigned, __e01) && v[ 2] == __builtin_bit_cast(unsigned, __e02) && v[ 3] == __builtin_bit_cast(unsigned, __e03) &&
+         v[ 4] == __builtin_bit_cast(unsigned, __e04) && v[ 5] == __builtin_bit_cast(unsigned, __e05) && v[ 6] == __builtin_bit_cast(unsigned, __e06) && v[ 7] == __builtin_bit_cast(unsigned, __e07);
+}
+
+constexpr bool match_m256d(__m256d _v, double a, double b, double c, double d) {
+  __v4du v = (__v4du)_v;
+  return v[0] == __builtin_bit_cast(unsigned long long, a) && v[1] == __builtin_bit_cast(unsigned long long, b) && v[2] == __builtin_bit_cast(unsigned long long, c) && v[3] == __builtin_bit_cast(unsigned long long, d);
+}
+
+constexpr bool match_m256h(__m256h _v, _Float16 __e00, _Float16 __e01, _Float16 __e02, _Float16 __e03, _Float16 __e04, _Float16 __e05, _Float16 __e06, _Float16 __e07,
+                                       _Float16 __e08, _Float16 __e09, _Float16 __e10, _Float16 __e11, _Float16 __e12, _Float16 __e13, _Float16 __e14, _Float16 __e15) {
+  __v16hu v = (__v16hu)_v;
+  return v[ 0] == __builtin_bit_cast(unsigned short, __e00) && v[ 1] == __builtin_bit_cast(unsigned short, __e01) && v[ 2] == __builtin_bit_cast(unsigned short, __e02) && v[ 3] == __builtin_bit_cast(unsigned short, __e03) &&
+         v[ 4] == __builtin_bit_cast(unsigned short, __e04) && v[ 5] == __builtin_bit_cast(unsigned short, __e05) && v[ 6] == __builtin_bit_cast(unsigned short, __e06) && v[ 7] == __builtin_bit_cast(unsigned short, __e07) &&
+         v[ 8] == __builtin_bit_cast(unsigned short, __e08) && v[ 9] == __builtin_bit_cast(unsigned short, __e09) && v[10] == __builtin_bit_cast(unsigned short, __e10) && v[11] == __builtin_bit_cast(unsigned short, __e11) &&
+         v[12] == __builtin_bit_cast(unsigned short, __e12) && v[13] == __builtin_bit_cast(unsigned short, __e13) && v[14] == __builtin_bit_cast(unsigned short, __e14) && v[15] == __builtin_bit_cast(unsigned short, __e15);
 }
 
 constexpr bool match_m256i(__m256i _v, unsigned long long a, unsigned long long b, unsigned long long c, unsigned long long d) {
@@ -88,23 +128,60 @@ constexpr bool match_v16hi(__m256i _v, short a, short b, short c, short d, short
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
 }
 
-constexpr bool match_v32qi(__m256i _v, char __b00, char __b01, char __b02, char __b03, char __b04, char __b05, char __b06, char __b07,
-                                       char __b08, char __b09, char __b10, char __b11, char __b12, char __b13, char __b14, char __b15,
-                                       char __b16, char __b17, char __b18, char __b19, char __b20, char __b21, char __b22, char __b23,
-                                       char __b24, char __b25, char __b26, char __b27, char __b28, char __b29, char __b30, char __b31) {
-  __v32qi v = (__v32qi)_v;
+constexpr bool match_v16hu(__m256i _v, unsigned short a, unsigned short b, unsigned short c, unsigned short d, unsigned short e, unsigned short f, unsigned short g, unsigned short h, unsigned short i, unsigned short j, unsigned short k, unsigned short l, unsigned short m, unsigned short n, unsigned short o, unsigned short p) {
+  __v16hu v = (__v16hu)_v;
+  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
+}
+
+constexpr bool match_v32qi(__m256i _v, signed char __b00, signed char __b01, signed char __b02, signed char __b03, signed char __b04, signed char __b05, signed char __b06, signed char __b07,
+                                       signed char __b08, signed char __b09, signed char __b10, signed char __b11, signed char __b12, signed char __b13, signed char __b14, signed char __b15,
+                                       signed char __b16, signed char __b17, signed char __b18, signed char __b19, signed char __b20, signed char __b21, signed char __b22, signed char __b23,
+                                       signed char __b24, signed char __b25, signed char __b26, signed char __b27, signed char __b28, signed char __b29, signed char __b30, signed char __b31) {
+  __v32qs v = (__v32qs)_v;
   return v[ 0] == __b00 && v[ 1] == __b01 && v[ 2] == __b02 && v[ 3] == __b03 && v[ 4] == __b04 && v[ 5] == __b05 && v[ 6] == __b06 && v[ 7] ==  __b07 &&
          v[ 8] == __b08 && v[ 9] == __b09 && v[10] == __b10 && v[11] == __b11 && v[12] == __b12 && v[13] == __b13 && v[14] == __b14 && v[15] ==  __b15 &&
          v[16] == __b16 && v[17] == __b17 && v[18] == __b18 && v[19] == __b19 && v[20] == __b20 && v[21] == __b21 && v[22] == __b22 && v[23] ==  __b23 &&
          v[24] == __b24 && v[25] == __b25 && v[26] == __b26 && v[27] == __b27 && v[28] == __b28 && v[29] == __b29 && v[30] == __b30 && v[31] ==  __b31;
 }
 
-constexpr bool match_m512(__m512 v, float a, float b, float c, float d, float e, float f, float g, float h, float i, float j, float k, float l, float m, float n, float o, float p) {
-  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
+constexpr bool match_v32qu(__m256i _v, unsigned char __b00, unsigned char __b01, unsigned char __b02, unsigned char __b03, unsigned char __b04, unsigned char __b05, unsigned char __b06, unsigned char __b07,
+                                       unsigned char __b08, unsigned char __b09, unsigned char __b10, unsigned char __b11, unsigned char __b12, unsigned char __b13, unsigned char __b14, unsigned char __b15,
+                                       unsigned char __b16, unsigned char __b17, unsigned char __b18, unsigned char __b19, unsigned char __b20, unsigned char __b21, unsigned char __b22, unsigned char __b23,
+                                       unsigned char __b24, unsigned char __b25, unsigned char __b26, unsigned char __b27, unsigned char __b28, unsigned char __b29, unsigned char __b30, unsigned char __b31) {
+  __v32qu v = (__v32qu)_v;
+  return v[ 0] == __b00 && v[ 1] == __b01 && v[ 2] == __b02 && v[ 3] == __b03 && v[ 4] == __b04 && v[ 5] == __b05 && v[ 6] == __b06 && v[ 7] ==  __b07 &&
+         v[ 8] == __b08 && v[ 9] == __b09 && v[10] == __b10 && v[11] == __b11 && v[12] == __b12 && v[13] == __b13 && v[14] == __b14 && v[15] ==  __b15 &&
+         v[16] == __b16 && v[17] == __b17 && v[18] == __b18 && v[19] == __b19 && v[20] == __b20 && v[21] == __b21 && v[22] == __b22 && v[23] ==  __b23 &&
+         v[24] == __b24 && v[25] == __b25 && v[26] == __b26 && v[27] == __b27 && v[28] == __b28 && v[29] == __b29 && v[30] == __b30 && v[31] ==  __b31;
 }
 
-constexpr bool match_m512d(__m512d v, double a, double b, double c, double d, double e, double f, double g, double h) {
-  return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h;
+constexpr bool match_m512(__m512 _v, float __e00, float __e01, float __e02, float __e03, float __e04, float __e05, float __e06, float __e07, float __e08, float __e09, float __e10, float __e11, float __e12, float __e13, float __e14, float __e15) {
+  __v16su v = (__v16su)_v;
+  return v[ 0] == __builtin_bit_cast(unsigned, __e00) && v[ 1] == __builtin_bit_cast(unsigned, __e01) && v[ 2] == __builtin_bit_cast(unsigned, __e02) && v[ 3] == __builtin_bit_cast(unsigned, __e03) &&
+         v[ 4] == __builtin_bit_cast(unsigned, __e04) && v[ 5] == __builtin_bit_cast(unsigned, __e05) && v[ 6] == __builtin_bit_cast(unsigned, __e06) && v[ 7] == __builtin_bit_cast(unsigned, __e07) &&
+         v[ 8] == __builtin_bit_cast(unsigned, __e08) && v[ 9] == __builtin_bit_cast(unsigned, __e09) && v[10] == __builtin_bit_cast(unsigned, __e10) && v[11] == __builtin_bit_cast(unsigned, __e11) &&
+         v[12] == __builtin_bit_cast(unsigned, __e12) && v[13] == __builtin_bit_cast(unsigned, __e13) && v[14] == __builtin_bit_cast(unsigned, __e14) && v[15] == __builtin_bit_cast(unsigned, __e15);
+}
+
+constexpr bool match_m512d(__m512d _v, double __e00, double __e01, double __e02, double __e03, double __e04, double __e05, double __e06, double __e07) {
+  __v8du v = (__v8du)_v;
+  return v[ 0] == __builtin_bit_cast(unsigned long long, __e00) && v[ 1] == __builtin_bit_cast(unsigned long long, __e01) && v[ 2] == __builtin_bit_cast(unsigned long long, __e02) && v[ 3] == __builtin_bit_cast(unsigned long long, __e03) &&
+         v[ 4] == __builtin_bit_cast(unsigned long long, __e04) && v[ 5] == __builtin_bit_cast(unsigned long long, __e05) && v[ 6] == __builtin_bit_cast(unsigned long long, __e06) && v[ 7] == __builtin_bit_cast(unsigned long long, __e07);
+}
+
+constexpr bool match_m512h(__m512h _v, _Float16 __e00, _Float16 __e01, _Float16 __e02, _Float16 __e03, _Float16 __e04, _Float16 __e05, _Float16 __e06, _Float16 __e07,
+                                       _Float16 __e08, _Float16 __e09, _Float16 __e10, _Float16 __e11, _Float16 __e12, _Float16 __e13, _Float16 __e14, _Float16 __e15,
+                                       _Float16 __e16, _Float16 __e17, _Float16 __e18, _Float16 __e19, _Float16 __e20, _Float16 __e21, _Float16 __e22, _Float16 __e23,
+                                       _Float16 __e24, _Float16 __e25, _Float16 __e26, _Float16 __e27, _Float16 __e28, _Float16 __e29, _Float16 __e30, _Float16 __e31) {
+  __v32hu v = (__v32hu)_v;
+  return v[ 0] == __builtin_bit_cast(unsigned short, __e00) && v[ 1] == __builtin_bit_cast(unsigned short, __e01) && v[ 2] == __builtin_bit_cast(unsigned short, __e02) && v[ 3] == __builtin_bit_cast(unsigned short, __e03) &&
+         v[ 4] == __builtin_bit_cast(unsigned short, __e04) && v[ 5] == __builtin_bit_cast(unsigned short, __e05) && v[ 6] == __builtin_bit_cast(unsigned short, __e06) && v[ 7] == __builtin_bit_cast(unsigned short, __e07) &&
+         v[ 8] == __builtin_bit_cast(unsigned short, __e08) && v[ 9] == __builtin_bit_cast(unsigned short, __e09) && v[10] == __builtin_bit_cast(unsigned short, __e10) && v[11] == __builtin_bit_cast(unsigned short, __e11) &&
+         v[12] == __builtin_bit_cast(unsigned short, __e12) && v[13] == __builtin_bit_cast(unsigned short, __e13) && v[14] == __builtin_bit_cast(unsigned short, __e14) && v[15] == __builtin_bit_cast(unsigned short, __e15) &&
+         v[16] == __builtin_bit_cast(unsigned short, __e16) && v[17] == __builtin_bit_cast(unsigned short, __e17) && v[18] == __builtin_bit_cast(unsigned short, __e18) && v[19] == __builtin_bit_cast(unsigned short, __e19) &&
+         v[20] == __builtin_bit_cast(unsigned short, __e20) && v[21] == __builtin_bit_cast(unsigned short, __e21) && v[22] == __builtin_bit_cast(unsigned short, __e22) && v[23] == __builtin_bit_cast(unsigned short, __e23) &&
+         v[24] == __builtin_bit_cast(unsigned short, __e24) && v[25] == __builtin_bit_cast(unsigned short, __e25) && v[26] == __builtin_bit_cast(unsigned short, __e26) && v[27] == __builtin_bit_cast(unsigned short, __e27) &&
+         v[28] == __builtin_bit_cast(unsigned short, __e28) && v[29] == __builtin_bit_cast(unsigned short, __e29) && v[30] == __builtin_bit_cast(unsigned short, __e30) && v[31] == __builtin_bit_cast(unsigned short, __e31);
 }
 
 constexpr bool match_m512i(__m512i _v, unsigned long long a, unsigned long long b, unsigned long long c, unsigned long long d, unsigned long long e, unsigned long long f, unsigned long long g, unsigned long long h) {
@@ -122,6 +199,66 @@ constexpr bool match_v16si(__m512i _v, int a, int b, int c, int d, int e, int f,
   return v[0] == a && v[1] == b && v[2] == c && v[3] == d && v[4] == e && v[5] == f && v[6] == g && v[7] == h && v[8] == i && v[9] == j && v[10] == k && v[11] == l && v[12] == m && v[13] == n && v[14] == o && v[15] == p;
 }
 
+constexpr bool match_v32hi(__m512i _v, short __e00, short __e01, short __e02, short __e03, short __e04, short __e05, short __e06, short __e07,
+                                       short __e08, short __e09, short __e10, short __e11, short __e12, short __e13, short __e14, short __e15,
+                                       short __e16, short __e17, short __e18, short __e19, short __e20, short __e21, short __e22, short __e23,
+                                       short __e24, short __e25, short __e26, short __e27, short __e28, short __e29, short __e30, short __e31) {
+  __v32hi v = (__v32hi)_v;
+  return v[ 0] == __e00 && v[ 1] == __e01 && v[ 2] == __e02 && v[ 3] == __e03 && v[ 4] == __e04 && v[ 5] == __e05 && v[ 6] == __e06 && v[ 7] ==  __e07 &&
+         v[ 8] == __e08 && v[ 9] == __e09 && v[10] == __e10 && v[11] == __e11 && v[12] == __e12 && v[13] == __e13 && v[14] == __e14 && v[15] ==  __e15 &&
+         v[16] == __e16 && v[17] == __e17 && v[18] == __e18 && v[19] == __e19 && v[20] == __e20 && v[21] == __e21 && v[22] == __e22 && v[23] ==  __e23 &&
+         v[24] == __e24 && v[25] == __e25 && v[26] == __e26 && v[27] == __e27 && v[28] == __e28 && v[29] == __e29 && v[30] == __e30 && v[31] ==  __e31;
+}
+
+constexpr bool match_v32hu(__m512i _v, unsigned short __e00, unsigned short __e01, unsigned short __e02, unsigned short __e03, unsigned short __e04, unsigned short __e05, unsigned short __e06, unsigned short __e07,
+                                       unsigned short __e08, unsigned short __e09, unsigned short __e10, unsigned short __e11, unsigned short __e12, unsigned short __e13, unsigned short __e14, unsigned short __e15,
+                                       unsigned short __e16, unsigned short __e17, unsigned short __e18, unsigned short __e19, unsigned short __e20, unsigned short __e21, unsigned short __e22, unsigned short __e23,
+                                       unsigned short __e24, unsigned short __e25, unsigned short __e26, unsigned short __e27, unsigned short __e28, unsigned short __e29, unsigned short __e30, unsigned short __e31) {
+  __v32hu v = (__v32hu)_v;
+  return v[ 0] == __e00 && v[ 1] == __e01 && v[ 2] == __e02 && v[ 3] == __e03 && v[ 4] == __e04 && v[ 5] == __e05 && v[ 6] == __e06 && v[ 7] ==  __e07 &&
+         v[ 8] == __e08 && v[ 9] == __e09 && v[10] == __e10 && v[11] == __e11 && v[12] == __e12 && v[13] == __e13 && v[14] == __e14 && v[15] ==  __e15 &&
+         v[16] == __e16 && v[17] == __e17 && v[18] == __e18 && v[19] == __e19 && v[20] == __e20 && v[21] == __e21 && v[22] == __e22 && v[23] ==  __e23 &&
+         v[24] == __e24 && v[25] == __e25 && v[26] == __e26 && v[27] == __e27 && v[28] == __e28 && v[29] == __e29 && v[30] == __e30 && v[31] ==  __e31;
+}
+
+constexpr bool match_v64qi(__m512i _v, signed char __e00, signed char __e01, signed char __e02, signed char __e03, signed char __e04, signed char __e05, signed char __e06, signed char __e07,
+                                       signed char __e08, signed char __e09, signed char __e10, signed char __e11, signed char __e12, signed char __e13, signed char __e14, signed char __e15,
+                                       signed char __e16, signed char __e17, signed char __e18, signed char __e19, signed char __e20, signed char __e21, signed char __e22, signed char __e23,
+                                       signed char __e24, signed char __e25, signed char __e26, signed char __e27, signed char __e28, signed char __e29, signed char __e30, signed char __e31,
+                                       signed char __e32, signed char __e33, signed char __e34, signed char __e35, signed char __e36, signed char __e37, signed char __e38, signed char __e39,
+                                       signed char __e40, signed char __e41, signed char __e42, signed char __e43, signed char __e44, signed char __e45, signed char __e46, signed char __e47,
+                                       signed char __e48, signed char __e49, signed char __e50, signed char __e51, signed char __e52, signed char __e53, signed char __e54, signed char __e55,
+                                       signed char __e56, signed char __e57, signed char __e58, signed char __e59, signed char __e60, signed char __e61, signed char __e62, signed char __e63) {
+  __v64qs v = (__v64qs)_v;
+  return v[ 0] == __e00 && v[ 1] == __e01 && v[ 2] == __e02 && v[ 3] == __e03 && v[ 4] == __e04 && v[ 5] == __e05 && v[ 6] == __e06 && v[ 7] == __e07 &&
+         v[ 8] == __e08 && v[ 9] == __e09 && v[10] == __e10 && v[11] == __e11 && v[12] == __e12 && v[13] == __e13 && v[14] == __e14 && v[15] == __e15 &&
+         v[16] == __e16 && v[17] == __e17 && v[18] == __e18 && v[19] == __e19 && v[20] == __e20 && v[21] == __e21 && v[22] == __e22 && v[23] == __e23 &&
+         v[24] == __e24 && v[25] == __e25 && v[26] == __e26 && v[27] == __e27 && v[28] == __e28 && v[29] == __e29 && v[30] == __e30 && v[31] == __e31 &&
+         v[32] == __e32 && v[33] == __e33 && v[34] == __e34 && v[35] == __e35 && v[36] == __e36 && v[37] == __e37 && v[38] == __e38 && v[39] == __e39 &&
+         v[40] == __e40 && v[41] == __e41 && v[42] == __e42 && v[43] == __e43 && v[44] == __e44 && v[45] == __e45 && v[46] == __e46 && v[47] == __e47 &&
+         v[48] == __e48 && v[49] == __e49 && v[50] == __e50 && v[51] == __e51 && v[52] == __e52 && v[53] == __e53 && v[54] == __e54 && v[55] == __e55 &&
+         v[56] == __e56 && v[57] == __e57 && v[58] == __e58 && v[59] == __e59 && v[60] == __e60 && v[61] == __e61 && v[62] == __e62 && v[63] == __e63;
+}
+
+constexpr bool match_v64qu(__m512i _v, unsigned char __e00, unsigned char __e01, unsigned char __e02, unsigned char __e03, unsigned char __e04, unsigned char __e05, unsigned char __e06, unsigned char __e07,
+                                       unsigned char __e08, unsigned char __e09, unsigned char __e10, unsigned char __e11, unsigned char __e12, unsigned char __e13, unsigned char __e14, unsigned char __e15,
+                                       unsigned char __e16, unsigned char __e17, unsigned char __e18, unsigned char __e19, unsigned char __e20, unsigned char __e21, unsigned char __e22, unsigned char __e23,
+                                       unsigned char __e24, unsigned char __e25, unsigned char __e26, unsigned char __e27, unsigned char __e28, unsigned char __e29, unsigned char __e30, unsigned char __e31,
+                                       unsigned char __e32, unsigned char __e33, unsigned char __e34, unsigned char __e35, unsigned char __e36, unsigned char __e37, unsigned char __e38, unsigned char __e39,
+                                       unsigned char __e40, unsigned char __e41, unsigned char __e42, unsigned char __e43, unsigned char __e44, unsigned char __e45, unsigned char __e46, unsigned char __e47,
+                                       unsigned char __e48, unsigned char __e49, unsigned char __e50, unsigned char __e51, unsigned char __e52, unsigned char __e53, unsigned char __e54, unsigned char __e55,
+                                       unsigned char __e56, unsigned char __e57, unsigned char __e58, unsigned char __e59, unsigned char __e60, unsigned char __e61, unsigned char __e62, unsigned char __e63) {
+  __v64qu v = (__v64qu)_v;
+  return v[ 0] == __e00 && v[ 1] == __e01 && v[ 2] == __e02 && v[ 3] == __e03 && v[ 4] == __e04 && v[ 5] == __e05 && v[ 6] == __e06 && v[ 7] == __e07 &&
+         v[ 8] == __e08 && v[ 9] == __e09 && v[10] == __e10 && v[11] == __e11 && v[12] == __e12 && v[13] == __e13 && v[14] == __e14 && v[15] == __e15 &&
+         v[16] == __e16 && v[17] == __e17 && v[18] == __e18 && v[19] == __e19 && v[20] == __e20 && v[21] == __e21 && v[22] == __e22 && v[23] == __e23 &&
+         v[24] == __e24 && v[25] == __e25 && v[26] == __e26 && v[27] == __e27 && v[28] == __e28 && v[29] == __e29 && v[30] == __e30 && v[31] == __e31 &&
+         v[32] == __e32 && v[33] == __e33 && v[34] == __e34 && v[35] == __e35 && v[36] == __e36 && v[37] == __e37 && v[38] == __e38 && v[39] == __e39 &&
+         v[40] == __e40 && v[41] == __e41 && v[42] == __e42 && v[43] == __e43 && v[44] == __e44 && v[45] == __e45 && v[46] == __e46 && v[47] == __e47 &&
+         v[48] == __e48 && v[49] == __e49 && v[50] == __e50 && v[51] == __e51 && v[52] == __e52 && v[53] == __e53 && v[54] == __e54 && v[55] == __e55 &&
+         v[56] == __e56 && v[57] == __e57 && v[58] == __e58 && v[59] == __e59 && v[60] == __e60 && v[61] == __e61 && v[62] == __e62 && v[63] == __e63;
+}
+
 #define TEST_CONSTEXPR(...) static_assert(__VA_ARGS__)
 
 #else
diff --git a/clang/test/CodeGen/X86/f16c-builtins-constrained.c b/clang/test/CodeGen/X86/f16c-builtins-constrained.c
index bbd4d8f83b53a..50afea8e5fc1d 100644
--- a/clang/test/CodeGen/X86/f16c-builtins-constrained.c
+++ b/clang/test/CodeGen/X86/f16c-builtins-constrained.c
@@ -8,17 +8,8 @@
 
 float test_cvtsh_ss(unsigned short a) {
   // CHECK-LABEL: test_cvtsh_ss
-  // CHECK: insertelement <8 x i16> poison, i16 %{{.*}}, i32 0
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 1
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 2
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 3
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 4
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 5
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 6
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 7
-  // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  // CHECK: call <4 x float> @llvm.experimental.constrained.fpext.v4f32.v4f16(<4 x half> %{{.*}}, metadata !"fpexcept.strict")
-  // CHECK: extractelement <4 x float> %{{.*}}, i32 0
+  // CHECK: [[CONV:%.*]] = call {{.*}}float @llvm.experimental.constrained.fpext.f32.f16(half %{{.*}}, metadata !"fpexcept.strict")
+  // CHECK: ret float [[CONV]]
   return _cvtsh_ss(a);
 }
 
@@ -38,7 +29,7 @@ unsigned short test_cvtss_sh(float a) {
 
 __m128 test_mm_cvtph_ps(__m128i a) {
   // CHECK-LABEL: test_mm_cvtph_ps
-  // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: call {{.*}}<4 x float> @llvm.experimental.constrained.fpext.v4f32.v4f16(<4 x half> %{{.*}}, metadata !"fpexcept.strict")
   return _mm_cvtph_ps(a);
 }
diff --git a/clang/test/CodeGen/X86/f16c-builtins.c b/clang/test/CodeGen/X86/f16c-builtins.c
index 3c6d64c225b32..1bee8364f2ef8 100644
--- a/clang/test/CodeGen/X86/f16c-builtins.c
+++ b/clang/test/CodeGen/X86/f16c-builtins.c
@@ -8,17 +8,8 @@
 
 float test_cvtsh_ss(unsigned short a) {
   // CHECK-LABEL: test_cvtsh_ss
-  // CHECK: insertelement <8 x i16> poison, i16 %{{.*}}, i32 0
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 1
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 2
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 3
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 4
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 5
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 6
-  // CHECK: insertelement <8 x i16> %{{.*}}, i16 0, i32 7
-  // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  // CHECK: fpext <4 x half> %{{.*}} to <4 x float>
-  // CHECK: extractelement <4 x float> %{{.*}}, i32 0
+  // CHECK: [[CONV:%.*]] = fpext half %{{.*}} to float
+  // CHECK: ret float [[CONV]]
   return _cvtsh_ss(a);
 }
 
@@ -35,7 +26,7 @@ unsigned short test_cvtss_sh(float a) {
 
 __m128 test_mm_cvtph_ps(__m128i a) {
   // CHECK-LABEL: test_mm_cvtph_ps
-  // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   // CHECK: fpext <4 x half> %{{.*}} to <4 x float>
   return _mm_cvtph_ps(a);
 }
diff --git a/clang/test/CodeGen/X86/fma-builtins.c b/clang/test/CodeGen/X86/fma-builtins.c
index aa17dcc62fbc0..8e9822ec6ad2f 100644
--- a/clang/test/CodeGen/X86/fma-builtins.c
+++ b/clang/test/CodeGen/X86/fma-builtins.c
@@ -5,18 +5,21 @@
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 __m128 test_mm_fmadd_ps(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_fmadd_ps
   // CHECK: call {{.*}}<4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_fmadd_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m128(_mm_fmadd_ps((__m128){ 0.0f, 1.0f, -2.0f, -4.0f }, (__m128){ -0.0f, 4.0f, 2.0f, 1.0f }, (__m128){ -0.0f, -4.0f, 2.0f, 1.0f }), -0.0f, 0.0f, -2.0f, -3.0f));
 
 __m128d test_mm_fmadd_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK-LABEL: test_mm_fmadd_pd
   // CHECK: call {{.*}}<2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_fmadd_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m128d(_mm_fmadd_pd((__m128d){ 0.0, -4.0 }, (__m128d){ -0.0, 1.0 }, (__m128d){ -0.0, 1.0 }), -0.0, -3.0));
 
 __m128 test_mm_fmadd_ss(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_fmadd_ss
@@ -44,6 +47,7 @@ __m128 test_mm_fmsub_ps(__m128 a, __m128 b, __m128 c) {
   // CHECK: call {{.*}}<4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_fmsub_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m128(_mm_fmsub_ps((__m128){ 0.0f, 1.0f, -2.0f, -4.0f }, (__m128){ -0.0f, 4.0f, 2.0f, 1.0f }, (__m128){ -0.0f, -4.0f, 2.0f, 1.0f }), 0.0f, 8.0f, -6.0f, -5.0f));
 
 __m128d test_mm_fmsub_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK-LABEL: test_mm_fmsub_pd
@@ -51,6 +55,7 @@ __m128d test_mm_fmsub_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK: call {{.*}}<2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_fmsub_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m128d(_mm_fmsub_pd((__m128d){ 0.0, -4.0 }, (__m128d){ -0.0, 1.0 }, (__m128d){ -0.0, 1.0 }), 0.0, -5.0));
 
 __m128 test_mm_fmsub_ss(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_fmsub_ss
@@ -80,6 +85,7 @@ __m128 test_mm_fnmadd_ps(__m128 a, __m128 b, __m128 c) {
   // CHECK: call {{.*}}<4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_fnmadd_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m128(_mm_fnmadd_ps((__m128){ 0.0f, 1.0f, -2.0f, -4.0f }, (__m128){ -0.0f, 4.0f, 2.0f, 1.0f }, (__m128){ -0.0f, -4.0f, 2.0f, 1.0f }), 0.0f, -8.0f, 6.0f, 5.0f));
 
 __m128d test_mm_fnmadd_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK-LABEL: test_mm_fnmadd_pd
@@ -87,6 +93,7 @@ __m128d test_mm_fnmadd_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK: call {{.*}}<2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_fnmadd_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m128d(_mm_fnmadd_pd((__m128d){ 0.0, -4.0 }, (__m128d){ -0.0, 1.0 }, (__m128d){ -0.0, 1.0 }), 0.0, 5.0));
 
 __m128 test_mm_fnmadd_ss(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_fnmadd_ss
@@ -117,6 +124,7 @@ __m128 test_mm_fnmsub_ps(__m128 a, __m128 b, __m128 c) {
   // CHECK: call {{.*}}<4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_fnmsub_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m128(_mm_fnmsub_ps((__m128){ 0.0f, 1.0f, -2.0f, -4.0f }, (__m128){ -0.0f, 4.0f, 2.0f, 1.0f }, (__m128){ -0.0f, -4.0f, 2.0f, 1.0f }), 0.0f, 0.0f, 2.0f, 3.0f));
 
 __m128d test_mm_fnmsub_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK-LABEL: test_mm_fnmsub_pd
@@ -125,6 +133,7 @@ __m128d test_mm_fnmsub_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK: call {{.*}}<2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_fnmsub_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m128d(_mm_fnmsub_pd((__m128d){ 0.0, -4.0 }, (__m128d){ -0.0, 1.0 }, (__m128d){ -0.0, 1.0 }), 0.0, 3.0));
 
 __m128 test_mm_fnmsub_ss(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_fnmsub_ss
@@ -183,12 +192,14 @@ __m256 test_mm256_fmadd_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK: call {{.*}}<8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_fmadd_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m256(_mm256_fmadd_ps((__m256){ 0.0f, 1.0f, -2.0f, -4.0f, -8.0f, -16.0f, 32.0f, 64.0f }, (__m256){ -0.0f, 64.0f, 32.0f, 16.0f, -8.0f, -4.0f, -2.0f, -1.0f }, (__m256){ -0.0f, -4.0f, 2.0f, 1.0f, -8.0f, -16.0f, 32.0f, 64.0f }), -0.0f, 60.0f, -62.0f, -63.0f, +56.0f, +48.0f, -32.0f, 0.0f));
 
 __m256d test_mm256_fmadd_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK-LABEL: test_mm256_fmadd_pd
   // CHECK: call {{.*}}<4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_fmadd_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_fmadd_pd((__m256d){ 0.0, 1.0, -2.0, -4.0 }, (__m256d){ -0.0, 4.0, 2.0, 1.0 }, (__m256d){ -0.0, -4.0, 2.0, 1.0 }), -0.0, 0.0, -2.0, -3.0));
 
 __m256 test_mm256_fmsub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK-LABEL: test_mm256_fmsub_ps
@@ -196,6 +207,7 @@ __m256 test_mm256_fmsub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK: call {{.*}}<8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_fmsub_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m256(_mm256_fmsub_ps((__m256){ 0.0f, 1.0f, -2.0f, -4.0f, -8.0f, -16.0f, 32.0f, 64.0f }, (__m256){ -0.0f, 64.0f, 32.0f, 16.0f, -8.0f, -4.0f, -2.0f, -1.0f }, (__m256){ -0.0f, -4.0f, 2.0f, 1.0f, -8.0f, -16.0f, 32.0f, 64.0f }), 0.0f, 68.0f, -66.0f, -65.0f, 72.0f, 80.0f, -96.0f, -128.0f));
 
 __m256d test_mm256_fmsub_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK-LABEL: test_mm256_fmsub_pd
@@ -203,6 +215,7 @@ __m256d test_mm256_fmsub_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK: call {{.*}}<4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_fmsub_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_fmsub_pd((__m256d){ 0.0, 1.0, -2.0, -4.0 }, (__m256d){ -0.0, 4.0, 2.0, 1.0 }, (__m256d){ -0.0, -4.0, 2.0, 1.0 }), 0.0, 8.0, -6.0, -5.0));
 
 __m256 test_mm256_fnmadd_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK-LABEL: test_mm256_fnmadd_ps
@@ -210,6 +223,7 @@ __m256 test_mm256_fnmadd_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK: call {{.*}}<8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_fnmadd_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m256(_mm256_fnmadd_ps((__m256){ 0.0f, 1.0f, -2.0f, -4.0f, -8.0f, -16.0f, 32.0f, 64.0f }, (__m256){ -0.0f, 64.0f, 32.0f, 16.0f, -8.0f, -4.0f, -2.0f, -1.0f }, (__m256){ -0.0f, -4.0f, 2.0f, 1.0f, -8.0f, -16.0f, 32.0f, 64.0f }), 0.0f, -68.0f, 66.0f, 65.0f, -72.0f, -80.0f, 96.0f, 128.0f));
 
 __m256d test_mm256_fnmadd_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK-LABEL: test_mm256_fnmadd_pd
@@ -217,6 +231,7 @@ __m256d test_mm256_fnmadd_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK: call {{.*}}<4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_fnmadd_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_fnmadd_pd((__m256d){ 0.0, 1.0, -2.0, -4.0 }, (__m256d){ -0.0, 4.0, 2.0, 1.0 }, (__m256d){ -0.0, -4.0, 2.0, 1.0 }), 0.0, -8.0, 6.0, 5.0));
 
 __m256 test_mm256_fnmsub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK-LABEL: test_mm256_fnmsub_ps
@@ -225,6 +240,7 @@ __m256 test_mm256_fnmsub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK: call {{.*}}<8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_fnmsub_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m256(_mm256_fnmsub_ps((__m256){ 0.0f, 1.0f, -2.0f, -4.0f, -8.0f, -16.0f, 32.0f, 64.0f }, (__m256){ -0.0f, 64.0f, 32.0f, 16.0f, -8.0f, -4.0f, -2.0f, -1.0f }, (__m256){ -0.0f, -4.0f, 2.0f, 1.0f, -8.0f, -16.0f, 32.0f, 64.0f }), 0.0f, -60.0f, 62.0f, 63.0f, -56.0f, -48.0f, 32.0f, 0.0f));
 
 __m256d test_mm256_fnmsub_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK-LABEL: test_mm256_fnmsub_pd
@@ -233,6 +249,7 @@ __m256d test_mm256_fnmsub_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK: call {{.*}}<4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_fnmsub_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_fnmsub_pd((__m256d){ 0.0, 1.0, -2.0, -4.0 }, (__m256d){ -0.0, 4.0, 2.0, 1.0 }, (__m256d){ -0.0, -4.0, 2.0, 1.0 }), 0.0, 0.0, 2.0, 3.0));
 
 __m256 test_mm256_fmaddsub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK-LABEL: test_mm256_fmaddsub_ps
diff --git a/clang/test/CodeGen/X86/fma4-builtins.c b/clang/test/CodeGen/X86/fma4-builtins.c
index ccdba8fea87b5..dcfd48a220e11 100644
--- a/clang/test/CodeGen/X86/fma4-builtins.c
+++ b/clang/test/CodeGen/X86/fma4-builtins.c
@@ -5,18 +5,21 @@
 
 
 #include <x86intrin.h>
+#include "builtin_test_helpers.h"
 
 __m128 test_mm_macc_ps(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_macc_ps
   // CHECK: call {{.*}}<4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_macc_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m128(_mm_macc_ps((__m128){ 0.0f, 1.0f, -2.0f, -4.0f }, (__m128){ -0.0f, 4.0f, 2.0f, 1.0f }, (__m128){ -0.0f, -4.0f, 2.0f, 1.0f }), -0.0f, 0.0f, -2.0f, -3.0f));
 
 __m128d test_mm_macc_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK-LABEL: test_mm_macc_pd
   // CHECK: call {{.*}}<2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_macc_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m128d(_mm_macc_pd((__m128d){ 0.0, -4.0 }, (__m128d){ -0.0, 1.0 }, (__m128d){ -0.0, 1.0 }), -0.0, -3.0));
 
 __m128 test_mm_macc_ss(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_macc_ss
@@ -44,6 +47,7 @@ __m128 test_mm_msub_ps(__m128 a, __m128 b, __m128 c) {
   // CHECK: call {{.*}}<4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_msub_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m128(_mm_msub_ps((__m128){ 0.0f, 1.0f, -2.0f, -4.0f }, (__m128){ -0.0f, 4.0f, 2.0f, 1.0f }, (__m128){ -0.0f, -4.0f, 2.0f, 1.0f }), 0.0f, 8.0f, -6.0f, -5.0f));
 
 __m128d test_mm_msub_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK-LABEL: test_mm_msub_pd
@@ -51,6 +55,7 @@ __m128d test_mm_msub_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK: call {{.*}}<2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_msub_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m128d(_mm_msub_pd((__m128d){ 0.0, -4.0 }, (__m128d){ -0.0, 1.0 }, (__m128d){ -0.0, 1.0 }), 0.0, -5.0));
 
 __m128 test_mm_msub_ss(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_msub_ss
@@ -80,6 +85,7 @@ __m128 test_mm_nmacc_ps(__m128 a, __m128 b, __m128 c) {
   // CHECK: call {{.*}}<4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_nmacc_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m128(_mm_nmacc_ps((__m128){ 0.0f, 1.0f, -2.0f, -4.0f }, (__m128){ -0.0f, 4.0f, 2.0f, 1.0f }, (__m128){ -0.0f, -4.0f, 2.0f, 1.0f }), 0.0f, -8.0f, 6.0f, 5.0f));
 
 __m128d test_mm_nmacc_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK-LABEL: test_mm_nmacc_pd
@@ -87,6 +93,7 @@ __m128d test_mm_nmacc_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK: call {{.*}}<2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_nmacc_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m128d(_mm_nmacc_pd((__m128d){ 0.0, -4.0 }, (__m128d){ -0.0, 1.0 }, (__m128d){ -0.0, 1.0 }), 0.0, 5.0));
 
 __m128 test_mm_nmacc_ss(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_nmacc_ss
@@ -117,6 +124,7 @@ __m128 test_mm_nmsub_ps(__m128 a, __m128 b, __m128 c) {
   // CHECK: call {{.*}}<4 x float> @llvm.fma.v4f32(<4 x float> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}})
   return _mm_nmsub_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m128(_mm_nmsub_ps((__m128){ 0.0f, 1.0f, -2.0f, -4.0f }, (__m128){ -0.0f, 4.0f, 2.0f, 1.0f }, (__m128){ -0.0f, -4.0f, 2.0f, 1.0f }), 0.0f, 0.0f, 2.0f, 3.0f));
 
 __m128d test_mm_nmsub_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK-LABEL: test_mm_nmsub_pd
@@ -125,6 +133,7 @@ __m128d test_mm_nmsub_pd(__m128d a, __m128d b, __m128d c) {
   // CHECK: call {{.*}}<2 x double> @llvm.fma.v2f64(<2 x double> %{{.*}}, <2 x double> %{{.*}}, <2 x double> %{{.*}})
   return _mm_nmsub_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m128d(_mm_nmsub_pd((__m128d){ 0.0, -4.0 }, (__m128d){ -0.0, 1.0 }, (__m128d){ -0.0, 1.0 }), 0.0, 3.0));
 
 __m128 test_mm_nmsub_ss(__m128 a, __m128 b, __m128 c) {
   // CHECK-LABEL: test_mm_nmsub_ss
@@ -183,12 +192,14 @@ __m256 test_mm256_macc_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK: call {{.*}}<8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_macc_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m256(_mm256_macc_ps((__m256){ 0.0f, 1.0f, -2.0f, -4.0f, -8.0f, -16.0f, 32.0f, 64.0f }, (__m256){ -0.0f, 64.0f, 32.0f, 16.0f, -8.0f, -4.0f, -2.0f, -1.0f }, (__m256){ -0.0f, -4.0f, 2.0f, 1.0f, -8.0f, -16.0f, 32.0f, 64.0f }), -0.0f, 60.0f, -62.0f, -63.0f, +56.0f, +48.0f, -32.0f, 0.0f));
 
 __m256d test_mm256_macc_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK-LABEL: test_mm256_macc_pd
   // CHECK: call {{.*}}<4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_macc_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_macc_pd((__m256d){ 0.0, 1.0, -2.0, -4.0 }, (__m256d){ -0.0, 4.0, 2.0, 1.0 }, (__m256d){ -0.0, -4.0, 2.0, 1.0 }), -0.0, 0.0, -2.0, -3.0));
 
 __m256 test_mm256_msub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK-LABEL: test_mm256_msub_ps
@@ -196,6 +207,7 @@ __m256 test_mm256_msub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK: call {{.*}}<8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_msub_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m256(_mm256_msub_ps((__m256){ 0.0f, 1.0f, -2.0f, -4.0f, -8.0f, -16.0f, 32.0f, 64.0f }, (__m256){ -0.0f, 64.0f, 32.0f, 16.0f, -8.0f, -4.0f, -2.0f, -1.0f }, (__m256){ -0.0f, -4.0f, 2.0f, 1.0f, -8.0f, -16.0f, 32.0f, 64.0f }), 0.0f, 68.0f, -66.0f, -65.0f, 72.0f, 80.0f, -96.0f, -128.0f));
 
 __m256d test_mm256_msub_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK-LABEL: test_mm256_msub_pd
@@ -203,6 +215,7 @@ __m256d test_mm256_msub_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK: call {{.*}}<4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_msub_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_msub_pd((__m256d){ 0.0, 1.0, -2.0, -4.0 }, (__m256d){ -0.0, 4.0, 2.0, 1.0 }, (__m256d){ -0.0, -4.0, 2.0, 1.0 }), 0.0, 8.0, -6.0, -5.0));
 
 __m256 test_mm256_nmacc_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK-LABEL: test_mm256_nmacc_ps
@@ -210,6 +223,7 @@ __m256 test_mm256_nmacc_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK: call {{.*}}<8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_nmacc_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m256(_mm256_nmacc_ps((__m256){ 0.0f, 1.0f, -2.0f, -4.0f, -8.0f, -16.0f, 32.0f, 64.0f }, (__m256){ -0.0f, 64.0f, 32.0f, 16.0f, -8.0f, -4.0f, -2.0f, -1.0f }, (__m256){ -0.0f, -4.0f, 2.0f, 1.0f, -8.0f, -16.0f, 32.0f, 64.0f }), 0.0f, -68.0f, 66.0f, 65.0f, -72.0f, -80.0f, 96.0f, 128.0f));
 
 __m256d test_mm256_nmacc_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK-LABEL: test_mm256_nmacc_pd
@@ -217,6 +231,7 @@ __m256d test_mm256_nmacc_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK: call {{.*}}<4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_nmacc_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_nmacc_pd((__m256d){ 0.0, 1.0, -2.0, -4.0 }, (__m256d){ -0.0, 4.0, 2.0, 1.0 }, (__m256d){ -0.0, -4.0, 2.0, 1.0 }), 0.0, -8.0, 6.0, 5.0));
 
 __m256 test_mm256_nmsub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK-LABEL: test_mm256_nmsub_ps
@@ -225,6 +240,7 @@ __m256 test_mm256_nmsub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK: call {{.*}}<8 x float> @llvm.fma.v8f32(<8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}})
   return _mm256_nmsub_ps(a, b, c);
 }
+TEST_CONSTEXPR(match_m256(_mm256_nmsub_ps((__m256){ 0.0f, 1.0f, -2.0f, -4.0f, -8.0f, -16.0f, 32.0f, 64.0f }, (__m256){ -0.0f, 64.0f, 32.0f, 16.0f, -8.0f, -4.0f, -2.0f, -1.0f }, (__m256){ -0.0f, -4.0f, 2.0f, 1.0f, -8.0f, -16.0f, 32.0f, 64.0f }), 0.0f, -60.0f, 62.0f, 63.0f, -56.0f, -48.0f, 32.0f, 0.0f));
 
 __m256d test_mm256_nmsub_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK-LABEL: test_mm256_nmsub_pd
@@ -233,6 +249,7 @@ __m256d test_mm256_nmsub_pd(__m256d a, __m256d b, __m256d c) {
   // CHECK: call {{.*}}<4 x double> @llvm.fma.v4f64(<4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x double> %{{.*}})
   return _mm256_nmsub_pd(a, b, c);
 }
+TEST_CONSTEXPR(match_m256d(_mm256_nmsub_pd((__m256d){ 0.0, 1.0, -2.0, -4.0 }, (__m256d){ -0.0, 4.0, 2.0, 1.0 }, (__m256d){ -0.0, -4.0, 2.0, 1.0 }), 0.0, 0.0, 2.0, 3.0));
 
 __m256 test_mm256_maddsub_ps(__m256 a, __m256 b, __m256 c) {
   // CHECK-LABEL: test_mm256_maddsub_ps
diff --git a/clang/test/CodeGen/X86/mmx-builtins.c b/clang/test/CodeGen/X86/mmx-builtins.c
index 52cbe45ca238b..35f0d6c9b43e8 100644
--- a/clang/test/CodeGen/X86/mmx-builtins.c
+++ b/clang/test/CodeGen/X86/mmx-builtins.c
@@ -34,48 +34,56 @@ __m64 test_mm_add_pi8(__m64 a, __m64 b) {
   // CHECK: add <8 x i8> {{%.*}}, {{%.*}}
   return _mm_add_pi8(a, b);
 }
+TEST_CONSTEXPR(match_v8qi(_mm_add_pi8((__m64)(__v8qs){-3, +2, -1, 0, +1, -2, +3, -4}, (__m64)(__v8qs){-18, +16, -14, +12, -10, +8, +6, -4}), -21, +18, -15, +12, -9, +6, +9, -8));
 
 __m64 test_mm_add_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_add_pi16
   // CHECK: add <4 x i16> {{%.*}}, {{%.*}}
   return _mm_add_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_add_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-10, +8, +6, -4}), -9, +6, +9, -8));
 
 __m64 test_mm_add_pi32(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_add_pi32
   // CHECK: add <2 x i32> {{%.*}}, {{%.*}}
   return _mm_add_pi32(a, b);
 }
+TEST_CONSTEXPR(match_v2si(_mm_add_pi32((__m64)(__v2si){+5, -3}, (__m64)(__v2si){-9, +8}), -4, +5));
 
 __m64 test_mm_add_si64(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_add_si64
   // CHECK: add i64 {{%.*}}, {{%.*}}
   return _mm_add_si64(a, b);
 }
+TEST_CONSTEXPR(match_v1di(_mm_add_si64((__m64)(__v1di){+42}, (__m64)(__v1di){-100}), -58));
 
 __m64 test_mm_adds_pi8(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_adds_pi8
   // CHECK: call <8 x i8> @llvm.sadd.sat.v8i8(
   return _mm_adds_pi8(a, b);
 }
+TEST_CONSTEXPR(match_v8qi(_mm_adds_pi8((__m64)(__v8qs){+100, +50, -100, +20, +80, -50, +120, -20}, (__m64)(__v8qs){+50, +80, -50, +110, +60, -30, +20, -10}), +127, +127, -128, +127, +127, -80, +127, -30));
 
 __m64 test_mm_adds_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_adds_pi16
   // CHECK: call <4 x i16> @llvm.sadd.sat.v4i16(
   return _mm_adds_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_adds_pi16((__m64)(__v4hi){+32000, -32000, +32000, -32000}, (__m64)(__v4hi){+800, -800, -800, +800}), +32767, -32768, +31200, -31200));
 
 __m64 test_mm_adds_pu8(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_adds_pu8
   // CHECK: call <8 x i8> @llvm.uadd.sat.v8i8(
   return _mm_adds_pu8(a, b);
 }
+TEST_CONSTEXPR(match_v8qu(_mm_adds_pu8((__m64)(__v8qu){0, +1, +2, +3, +180, +150, +120, +200}, (__m64)(__v8qu){0, +1, +2, +3, +160, +30, +200, +10}), 0, +2, +4, +6, +255, +180, +255, +210));
 
 __m64 test_mm_adds_pu16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_adds_pu16
   // CHECK: call <4 x i16> @llvm.uadd.sat.v4i16(
   return _mm_adds_pu16(a, b);
 }
+TEST_CONSTEXPR(match_v4hu(_mm_adds_pu16((__m64)(__v4hu){+0, +1, +32000, +33000}, (__m64)(__v4hu){0, +1, +800, +33000}), 0, +2, +32800, +65535));
 
 __m64 test_mm_alignr_pi8(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_alignr_pi8
@@ -88,6 +96,7 @@ __m64 test_mm_and_si64(__m64 a, __m64 b) {
   // CHECK: and <1 x i64> {{%.*}}, {{%.*}}
   return _mm_and_si64(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_and_si64((__m64)(__v4hi){0, -1, 0, -1}, (__m64)(__v4hi){0, 0, -1, -1}), 0, 0, 0, -1));
 
 __m64 test_mm_andnot_si64(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_andnot_si64
@@ -95,6 +104,7 @@ __m64 test_mm_andnot_si64(__m64 a, __m64 b) {
   // CHECK: and <1 x i64> [[TMP]], {{%.*}}
   return _mm_andnot_si64(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_andnot_si64((__m64)(__v4hi){0, -1, 0, -1}, (__m64)(__v4hi){0, 0, -1, -1}), 0, 0, -1, 0));
 
 __m64 test_mm_avg_pu8(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_avg_pu8
@@ -114,6 +124,7 @@ __m64 test_mm_cmpeq_pi8(__m64 a, __m64 b) {
   // CHECK-NEXT: {{%.*}} = sext <8 x i1> [[CMP]] to <8 x i8>
   return _mm_cmpeq_pi8(a, b);
 }
+TEST_CONSTEXPR(match_v8qi(_mm_cmpeq_pi8((__m64)(__v8qs){-3, +2, -1, 0, +1, -2, +3, -4}, (__m64)(__v8qs){-3, -2, +1, 0, -1, -2, -3, -4}), -1, 0, 0, -1, 0, -1, 0, -1));
 
 __m64 test_mm_cmpeq_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_cmpeq_pi16
@@ -121,6 +132,7 @@ __m64 test_mm_cmpeq_pi16(__m64 a, __m64 b) {
   // CHECK-NEXT: {{%.*}} = sext <4 x i1> [[CMP]] to <4 x i16>
   return _mm_cmpeq_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_cmpeq_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-1, -1, +3, +4}), 0, 0, -1, 0));
 
 __m64 test_mm_cmpeq_pi32(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_cmpeq_pi32
@@ -128,6 +140,7 @@ __m64 test_mm_cmpeq_pi32(__m64 a, __m64 b) {
   // CHECK-NEXT: {{%.*}} = sext <2 x i1> [[CMP]] to <2 x i32>
   return _mm_cmpeq_pi32(a, b);
 }
+TEST_CONSTEXPR(match_v2si(_mm_cmpeq_pi32((__m64)(__v2si){+5, -3}, (__m64)(__v2si){-5, -3}), 0, -1));
 
 __m64 test_mm_cmpgt_pi8(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_cmpgt_pi8
@@ -135,6 +148,7 @@ __m64 test_mm_cmpgt_pi8(__m64 a, __m64 b) {
   // CHECK-NEXT: {{%.*}} = sext <8 x i1> [[CMP]] to <8 x i8>
   return _mm_cmpgt_pi8(a, b);
 }
+TEST_CONSTEXPR(match_v8qi(_mm_cmpgt_pi8((__m64)(__v8qs){-3, +2, -1, 0, +1, -2, +3, -4}, (__m64)(__v8qs){-3, -2, +1, 0, -1, -2, -3, -4}), 0, -1, 0, 0, -1, 0, -1, 0));
 
 __m64 test_mm_cmpgt_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_cmpgt_pi16
@@ -142,6 +156,7 @@ __m64 test_mm_cmpgt_pi16(__m64 a, __m64 b) {
   // CHECK-NEXT: {{%.*}} = sext <4 x i1> [[CMP]] to <4 x i16>
   return _mm_cmpgt_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_cmpgt_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-1, -1, +3, +4}), -1, 0, 0, 0));
 
 __m64 test_mm_cmpgt_pi32(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_cmpgt_pi32
@@ -149,12 +164,14 @@ __m64 test_mm_cmpgt_pi32(__m64 a, __m64 b) {
   // CHECK-NEXT: {{%.*}} = sext <2 x i1> [[CMP]] to <2 x i32>
   return _mm_cmpgt_pi32(a, b);
 }
+TEST_CONSTEXPR(match_v2si(_mm_cmpgt_pi32((__m64)(__v2si){+5, -3}, (__m64)(__v2si){-5, -3}), -1, 0));
 
 __m128 test_mm_cvt_pi2ps(__m128 a, __m64 b) {
   // CHECK-LABEL: test_mm_cvt_pi2ps
   // CHECK: sitofp <4 x i32> {{%.*}} to <4 x float>
   return _mm_cvt_pi2ps(a, b);
 }
+TEST_CONSTEXPR(match_m128(_mm_cvt_pi2ps((__m128){-5.0f, +7.0f, -9.0f, +11.0f}, (__m64)(__v2si){-2,+4}), -2.0f, +4.0f, -9.0f, +11.0f));
 
 __m64 test_mm_cvt_ps2pi(__m128 a) {
   // CHECK-LABEL: test_mm_cvt_ps2pi
@@ -168,29 +185,40 @@ __m64 test_mm_cvtpd_pi32(__m128d a) {
   return _mm_cvtpd_pi32(a);
 }
 
+__m128 test_mm_cvtpi8_ps(__m64 a) {
+  // CHECK-LABEL: test_mm_cvtpi8_ps
+  // CHECK: sitofp <4 x i8> {{%.*}} to <4 x float>
+  return _mm_cvtpi8_ps(a);
+}
+TEST_CONSTEXPR(match_m128(_mm_cvtpi8_ps((__m64)(__v8qi){1, 2, 3, 4, 5, 6, 7, 8}), +1.0f, +2.0f, +3.0f, +4.0f));
+
 __m128 test_mm_cvtpi16_ps(__m64 a) {
   // CHECK-LABEL: test_mm_cvtpi16_ps
   // CHECK: sitofp <4 x i16> {{%.*}} to <4 x float>
   return _mm_cvtpi16_ps(a);
 }
+TEST_CONSTEXPR(match_m128(_mm_cvtpi16_ps((__m64)(__v4hi){-3, +9, -8, +256}), -3.0f, +9.0f, -8.0f, +256.0f));
 
 __m128d test_mm_cvtpi32_pd(__m64 a) {
   // CHECK-LABEL: test_mm_cvtpi32_pd
   // CHECK: sitofp <2 x i32> {{%.*}} to <2 x double>
   return _mm_cvtpi32_pd(a);
 }
+TEST_CONSTEXPR(match_m128d(_mm_cvtpi32_pd((__m64)(__v2si){-10,+17}), -10.0, +17.0));
 
 __m128 test_mm_cvtpi32_ps(__m128 a, __m64 b) {
   // CHECK-LABEL: test_mm_cvtpi32_ps
   // CHECK: sitofp <4 x i32> {{%.*}} to <4 x float>
   return _mm_cvtpi32_ps(a, b);
 }
+TEST_CONSTEXPR(match_m128(_mm_cvtpi32_ps((__m128){+1.0f, -2.0f, +3.0f, +5.0f}, (__m64)(__v2si){+100,-200}), +100.0f, -200.0f, +3.0f, +5.0f));
 
 __m128 test_mm_cvtpi32x2_ps(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_cvtpi32x2_ps
   // CHECK: sitofp <4 x i32> {{%.*}} to <4 x float>
   return _mm_cvtpi32x2_ps(a, b);
 }
+TEST_CONSTEXPR(match_m128(_mm_cvtpi32x2_ps((__m64)(__v2si){-8,+7}, (__m64)(__v2si){+100,-200}), -8.0f, +7.0f, +100.0f, -200.0f));
 
 __m64 test_mm_cvtps_pi16(__m128 a) {
   // CHECK-LABEL: test_mm_cvtps_pi16
@@ -205,17 +233,33 @@ __m64 test_mm_cvtps_pi32(__m128 a) {
   return _mm_cvtps_pi32(a);
 }
 
+__m128 test_mm_cvtpu8_ps(__m64 a) {
+  // CHECK-LABEL: test_mm_cvtpu8_ps
+  // CHECK: uitofp <4 x i8> {{%.*}} to <4 x float>
+  return _mm_cvtpu8_ps(a);
+}
+TEST_CONSTEXPR(match_m128(_mm_cvtpu8_ps((__m64)(__v8qi){8, 7, 6, 5, 4, 3, 2, 1}), 8.0f, 7.0f, 6.0f, 5.0f));
+
+__m128 test_mm_cvtpu16_ps(__m64 a) {
+  // CHECK-LABEL: test_mm_cvtpu16_ps
+  // CHECK: uitofp <4 x i16> {{%.*}} to <4 x float>
+  return _mm_cvtpu16_ps(a);
+}
+TEST_CONSTEXPR(match_m128(_mm_cvtpu16_ps((__m64)(__v4hi){-3, +9, -8, +256}), 65533.0f, 9.0f, 65528.0f, 256.0f));
+
 __m64 test_mm_cvtsi32_si64(int a) {
   // CHECK-LABEL: test_mm_cvtsi32_si64
   // CHECK: insertelement <2 x i32>
   return _mm_cvtsi32_si64(a);
 }
+TEST_CONSTEXPR(match_v2si(_mm_cvtsi32_si64(-127), -127, 0));
 
 int test_mm_cvtsi64_si32(__m64 a) {
   // CHECK-LABEL: test_mm_cvtsi64_si32
   // CHECK: extractelement <2 x i32>
   return _mm_cvtsi64_si32(a);
 }
+TEST_CONSTEXPR(_mm_cvtsi64_si32((__m64)(__v4hi){-2, 0, -1, -1}) == 65534);
 
 __m64 test_mm_cvttpd_pi32(__m128d a) {
   // CHECK-LABEL: test_mm_cvttpd_pi32
@@ -240,11 +284,13 @@ __m64 test_m_from_int(int a) {
   // CHECK: insertelement <2 x i32>
   return _m_from_int(a);
 }
+TEST_CONSTEXPR(match_v2si(_m_from_int(255), 255, 0));
 
 __m64 test_m_from_int64(long long a) {
   // CHECK-LABEL: test_m_from_int64
   return _m_from_int64(a);
 }
+TEST_CONSTEXPR(match_v1di(_m_from_int64(-65536), -65536LL));
 
 __m64 test_mm_hadd_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_hadd_pi16
@@ -343,18 +389,21 @@ __m64 test_mm_mul_su32(__m64 a, __m64 b) {
   // CHECK: mul <2 x i64> %{{.*}}, %{{.*}}
   return _mm_mul_su32(a, b);
 }
+TEST_CONSTEXPR(match_m64(_mm_mul_su32((__m64)(__v2si){+1, -2}, (__m64)(__v2si){-10, +8}), 4294967286));
 
 __m64 test_mm_mulhi_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_mulhi_pi16
   // CHECK: call <8 x i16> @llvm.x86.sse2.pmulh.w(
   return _mm_mulhi_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_mulhi_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-10, +8, +6, -4}), -1, -1, 0, 0));
 
 __m64 test_mm_mulhi_pu16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_mulhi_pu16
   // CHECK: call <8 x i16> @llvm.x86.sse2.pmulhu.w(
   return _mm_mulhi_pu16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_mulhi_pu16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-10, +8, +6, -4}), 0, 7, 0, -8));
 
 __m64 test_mm_mulhrs_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_mulhrs_pi16
@@ -367,12 +416,14 @@ __m64 test_mm_mullo_pi16(__m64 a, __m64 b) {
   // CHECK: mul <4 x i16> {{%.*}}, {{%.*}}
   return _mm_mullo_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_mullo_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-10, +8, +6, -4}), -10, -16, +18, +16));
 
 __m64 test_mm_or_si64(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_or_si64
   // CHECK: or <1 x i64> {{%.*}}, {{%.*}}
   return _mm_or_si64(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_or_si64((__m64)(__v4hi){0, -1, 0, -1}, (__m64)(__v4hi){0, 0, -1, -1}), 0, -1, -1, -1));
 
 __m64 test_mm_packs_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_packs_pi16
@@ -644,98 +695,115 @@ __m64 test_mm_sub_pi8(__m64 a, __m64 b) {
   // CHECK: sub <8 x i8> {{%.*}}, {{%.*}}
   return _mm_sub_pi8(a, b);
 }
+TEST_CONSTEXPR(match_v8qi(_mm_sub_pi8((__m64)(__v8qs){-3, +2, -1, 0, +1, -2, +3, -4}, (__m64)(__v8qs){-18, +16, -14, +12, -10, +8, +6, -4}), +15, -14, +13, -12, +11, -10, -3, 0));
 
 __m64 test_mm_sub_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_sub_pi16
   // CHECK: sub <4 x i16> {{%.*}}, {{%.*}}
   return _mm_sub_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_sub_pi16((__m64)(__v4hi){+1, -2, +3, -4}, (__m64)(__v4hi){-10, +8, +6, -4}), +11, -10, -3, 0));
 
 __m64 test_mm_sub_pi32(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_sub_pi32
   // CHECK: sub <2 x i32> {{%.*}}, {{%.*}}
   return _mm_sub_pi32(a, b);
 }
+TEST_CONSTEXPR(match_v2si(_mm_sub_pi32((__m64)(__v2si){+5, -3}, (__m64)(__v2si){-9, +8}), +14, -11));
 
 __m64 test_mm_sub_si64(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_sub_si64
   // CHECK: sub i64 {{%.*}}, {{%.*}}
   return _mm_sub_si64(a, b);
 }
+TEST_CONSTEXPR(match_v1di(_mm_sub_si64((__m64)(__v1di){+42}, (__m64)(__v1di){-100}), +142));
 
 __m64 test_mm_subs_pi8(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_subs_pi8
   // CHECK: call <8 x i8> @llvm.ssub.sat.v8i8(
   return _mm_subs_pi8(a, b);
 }
+TEST_CONSTEXPR(match_v8qi(_mm_subs_pi8((__m64)(__v8qs){+100, +50, -100, +20, +80, -50, +120, -20}, (__m64)(__v8qs){-50, -80, +50, -110, -60, +30, -20, +10}), +127, +127, -128, +127, +127, -80, +127, -30));
 
 __m64 test_mm_subs_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_subs_pi16
   // CHECK: call <4 x i16> @llvm.ssub.sat.v4i16(
   return _mm_subs_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_subs_pi16((__m64)(__v4hi){+32000, -32000, +32000, -32000}, (__m64)(__v4hi){-800, +800, +800, -800}), +32767, -32768, +31200, -31200));
 
 __m64 test_mm_subs_pu8(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_subs_pu8
   // CHECK: call <8 x i8> @llvm.usub.sat.v8i8(
   return _mm_subs_pu8(a, b);
 }
+TEST_CONSTEXPR(match_v8qu(_mm_subs_pu8((__m64)(__v8qu){0, +1, +2, +3, +180, +250, +120, +200}, (__m64)(__v8qu){0, +1, +1, +255, +200, +30, +200, +10}), 0, 0, +1, 0, 0, +220, 0, +190));
 
 __m64 test_mm_subs_pu16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_subs_pu16
   // CHECK: call <4 x i16> @llvm.usub.sat.v4i16(
   return _mm_subs_pu16(a, b);
 }
+TEST_CONSTEXPR(match_v4hu(_mm_subs_pu16((__m64)(__v4hu){+0, +1, +32000, +33000}, (__m64)(__v4hu){0, +65535, +800, +34000}), 0, 0, +31200, 0));
 
 int test_m_to_int(__m64 a) {
   // CHECK-LABEL: test_m_to_int
   // CHECK: extractelement <2 x i32>
   return _m_to_int(a);
 }
+TEST_CONSTEXPR(_m_to_int((__m64)(__v4hi){0, -2, -1, -1}) == -131072);
 
 long long test_m_to_int64(__m64 a) {
   // CHECK-LABEL: test_m_to_int64
   return _m_to_int64(a);
 }
+TEST_CONSTEXPR(_m_to_int64((__m64)(__v4hi){0, -2, 0, -1}) == -281470681874432LL);
 
 __m64 test_mm_unpackhi_pi8(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_unpackhi_pi8
   // CHECK: shufflevector <8 x i8> {{%.*}}, <8 x i8> {{%.*}}, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   return _mm_unpackhi_pi8(a, b);
 }
+TEST_CONSTEXPR(match_v8qi(_mm_unpackhi_pi8((__m64)(__v8qi){0, 1, 2, 3, 4, 5, 6, 7}, (__m64)(__v8qi){8, 9, 10, 11, 12, 13, 14, 15}), 4, 12, 5, 13, 6, 14, 7, 15));
 
 __m64 test_mm_unpackhi_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_unpackhi_pi16
   // CHECK: shufflevector <4 x i16> {{%.*}}, <4 x i16> {{%.*}}, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   return _mm_unpackhi_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_unpackhi_pi16((__m64)(__v4hi){0, 1, 2, 3}, (__m64)(__v4hi){ 4, 5, 6, 7}), 2, 6, 3, 7));
 
 __m64 test_mm_unpackhi_pi32(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_unpackhi_pi32
   // CHECK: shufflevector <2 x i32> {{%.*}}, <2 x i32> {{%.*}}, <2 x i32> <i32 1, i32 3>
   return _mm_unpackhi_pi32(a, b);
 }
+TEST_CONSTEXPR(match_v2si(_mm_unpackhi_pi32((__m64)(__v2si){0, 1}, (__m64)(__v2si){2, 3}), 1, 3));
 
 __m64 test_mm_unpacklo_pi8(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_unpacklo_pi8
   // CHECK: shufflevector <8 x i8> {{%.*}}, <8 x i8> {{%.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   return _mm_unpacklo_pi8(a, b);
 }
+TEST_CONSTEXPR(match_v8qi(_mm_unpacklo_pi8((__m64)(__v8qi){0, 1, 2, 3, 4, 5, 6, 7}, (__m64)(__v8qi){8, 9, 10, 11, 12, 13, 14, 15}), 0, 8, 1, 9, 2, 10, 3, 11));
 
 __m64 test_mm_unpacklo_pi16(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_unpacklo_pi16
   // CHECK: shufflevector <4 x i16> {{%.*}}, <4 x i16> {{%.*}}, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   return _mm_unpacklo_pi16(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_unpacklo_pi16((__m64)(__v4hi){0, 1, 2, 3}, (__m64)(__v4hi){ 4, 5, 6, 7}), 0, 4, 1, 5));
 
 __m64 test_mm_unpacklo_pi32(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_unpacklo_pi32
   // CHECK: shufflevector <2 x i32> {{%.*}}, <2 x i32> {{%.*}}, <2 x i32> <i32 0, i32 2>
   return _mm_unpacklo_pi32(a, b);
 }
+TEST_CONSTEXPR(match_v2si(_mm_unpacklo_pi32((__m64)(__v2si){0, 1}, (__m64)(__v2si){2, 3}), 0, 2));
 
 __m64 test_mm_xor_si64(__m64 a, __m64 b) {
   // CHECK-LABEL: test_mm_xor_si64
   // CHECK: xor <1 x i64> {{%.*}}, {{%.*}}
   return _mm_xor_si64(a, b);
 }
+TEST_CONSTEXPR(match_v4hi(_mm_xor_si64((__m64)(__v4hi){0, -1, 0, -1}, (__m64)(__v4hi){0, 0, -1, -1}), 0, -1, -1, 0));
diff --git a/clang/test/CodeGen/X86/sse-builtins.c b/clang/test/CodeGen/X86/sse-builtins.c
index 104bfea05469e..12d9abdfb582b 100644
--- a/clang/test/CodeGen/X86/sse-builtins.c
+++ b/clang/test/CodeGen/X86/sse-builtins.c
@@ -31,7 +31,7 @@ __m128 test_mm_and_ps(__m128 A, __m128 B) {
   // CHECK: and <4 x i32>
   return _mm_and_ps(A, B);
 }
-TEST_CONSTEXPR(match_m128(_mm_and_ps((__m128){-4.0f, -5.0f, +6.0f, +7.0f}, (__m128){+0.0f, -0.0f, -0.0f, +7.0f}), -0.0f, -0.0f, +0.0f, +7.0f));
+TEST_CONSTEXPR(match_m128(_mm_and_ps((__m128){-4.0f, -5.0f, +6.0f, +7.0f}, (__m128){+0.0f, -0.0f, -0.0f, +7.0f}), +0.0f, -0.0f, +0.0f, +7.0f));
 
 __m128 test_mm_andnot_ps(__m128 A, __m128 B) {
   // CHECK-LABEL: test_mm_andnot_ps
@@ -39,7 +39,7 @@ __m128 test_mm_andnot_ps(__m128 A, __m128 B) {
   // CHECK: and <4 x i32>
   return _mm_andnot_ps(A, B);
 }
-TEST_CONSTEXPR(match_m128(_mm_andnot_ps((__m128){-4.0f, -5.0f, +6.0f, +7.0f}, (__m128){+0.0f, -0.0f, -0.0f, +7.0f}), +0.0f, +0.0f, +0.0f, +0.0f));
+TEST_CONSTEXPR(match_m128(_mm_andnot_ps((__m128){-4.0f, -5.0f, +6.0f, +7.0f}, (__m128){+0.0f, -0.0f, -0.0f, +7.0f}), +0.0f, +0.0f, -0.0f, +0.0f));
 
 __m128 test_mm_cmp_ps_eq_oq(__m128 a, __m128 b) {
   // CHECK-LABEL: test_mm_cmp_ps_eq_oq
diff --git a/clang/test/CodeGen/X86/sse2-builtins.c b/clang/test/CodeGen/X86/sse2-builtins.c
index 04df59ef331a5..38d5e877a5036 100644
--- a/clang/test/CodeGen/X86/sse2-builtins.c
+++ b/clang/test/CodeGen/X86/sse2-builtins.c
@@ -63,12 +63,14 @@ __m128i test_mm_adds_epi8(__m128i A, __m128i B) {
   // CHECK: call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_adds_epi8(A, B);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_adds_epi8((__m128i)(__v16qs){+100, +50, -100, +20, +80, -50, +120, -20, -100, -50, +100, -20, -80, +50, -120, +20}, (__m128i)(__v16qs){+50, +80, -50, +110, +60, -30, +20, -10, +50, +80, -50, +110, +60, -30, +20, -10}), +127, +127, -128, +127, +127, -80, +127, -30, -50, +30, +50, +90, -20, +20, -100, +10));
 
 __m128i test_mm_adds_epi16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_adds_epi16
   // CHECK: call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_adds_epi16(A, B);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_adds_epi16((__m128i)(__v8hi){+32000, -32000, +32000, -32000, +80, -50, +120, -20}, (__m128i)(__v8hi){+800, -800, -800, +800, +60, -30, +20, -10}), +32767, -32768, +31200, -31200, +140, -80, +140, -30));
 
 __m128i test_mm_adds_epu8(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_adds_epu8
@@ -76,6 +78,7 @@ __m128i test_mm_adds_epu8(__m128i A, __m128i B) {
   // CHECK: call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_adds_epu8(A, B);
 }
+TEST_CONSTEXPR(match_v16qu(_mm_adds_epu8((__m128i)(__v16qu){0, 0, 0, 0, +127, +127, +127, +127, +128, +128, +128, +128, +255, +255, +255, +255}, (__m128i)(__v16qu){0, +127, +128, +255, 0, +127, +128, +255, 0, +127, +128, +255, 0, +127, +128, +255}), 0, +127, +128, +255, +127, +254, +255, +255, +128, +255, +255, +255, +255, +255, +255, +255));
 
 __m128i test_mm_adds_epu16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_adds_epu16
@@ -83,6 +86,7 @@ __m128i test_mm_adds_epu16(__m128i A, __m128i B) {
   // CHECK: call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_adds_epu16(A, B);
 }
+TEST_CONSTEXPR(match_v8hu(_mm_adds_epu16((__m128i)(__v8hu){0, 0, 0, +32767, +32767, +32767, +65535, +65535}, (__m128i)(__v8hu){0, +32767, +65535, 0, +32767, +65535, 0, +32767}), 0, +32767, +65535, +32767, +65534, +65535, +65535, +65535));
 
 __m128d test_mm_and_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_and_pd
@@ -96,6 +100,7 @@ __m128i test_mm_and_si128(__m128i A, __m128i B) {
   // CHECK: and <2 x i64>
   return _mm_and_si128(A, B);
 }
+TEST_CONSTEXPR(match_v4si(_mm_and_si128((__m128i)(__v4si){0, -1, 0, -1}, (__m128i)(__v4si){0, 0, -1, -1}), 0, 0, 0, -1));
 
 __m128d test_mm_andnot_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_andnot_pd
@@ -103,7 +108,7 @@ __m128d test_mm_andnot_pd(__m128d A, __m128d B) {
   // CHECK: and <2 x i64>
   return _mm_andnot_pd(A, B);
 }
-TEST_CONSTEXPR(match_m128d(_mm_andnot_pd((__m128d){+1.0, -3.0}, (__m128d){+0.0, -0.0}), +0.0, -0.0));
+TEST_CONSTEXPR(match_m128d(_mm_andnot_pd((__m128d){+1.0, -3.0}, (__m128d){-0.0, +0.0}), -0.0, +0.0));
 
 __m128i test_mm_andnot_si128(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_andnot_si128
@@ -111,6 +116,7 @@ __m128i test_mm_andnot_si128(__m128i A, __m128i B) {
   // CHECK: and <2 x i64>
   return _mm_andnot_si128(A, B);
 }
+TEST_CONSTEXPR(match_v4si(_mm_andnot_si128((__m128i)(__v4si){0, -1, 0, -1}, (__m128i)(__v4si){0, 0, -1, -1}), 0, 0, -1, 0));
 
 __m128i test_mm_avg_epu8(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_avg_epu8
@@ -579,12 +585,14 @@ int test_mm_cvtsi128_si32(__m128i A) {
   // CHECK: extractelement <4 x i32> %{{.*}}, i32 0
   return _mm_cvtsi128_si32(A);
 }
+TEST_CONSTEXPR(_mm_cvtsi128_si32((__m128i)(__v4si){+1, -2, -3, +4}) == +1);
 
 long long test_mm_cvtsi128_si64(__m128i A) {
   // CHECK-LABEL: test_mm_cvtsi128_si64
   // CHECK: extractelement <2 x i64> %{{.*}}, i32 0
   return _mm_cvtsi128_si64(A);
 }
+TEST_CONSTEXPR(_mm_cvtsi128_si64((__m128i)(__v2di){42LL, -42LL}) == 42LL);
 
 __m128d test_mm_cvtsi32_sd(__m128d A, int B) {
   // CHECK-LABEL: test_mm_cvtsi32_sd
@@ -602,6 +610,7 @@ __m128i test_mm_cvtsi32_si128(int A) {
   // CHECK: insertelement <4 x i32> %{{.*}}, i32 0, i32 3
   return _mm_cvtsi32_si128(A);
 }
+TEST_CONSTEXPR(match_v4si(_mm_cvtsi32_si128(55), 55, 0, 0, 0));
 
 #ifdef __x86_64__
 __m128d test_mm_cvtsi64_sd(__m128d A, long long B) {
@@ -619,6 +628,7 @@ __m128i test_mm_cvtsi64_si128(long long A) {
   // CHECK: insertelement <2 x i64> %{{.*}}, i64 0, i32 1
   return _mm_cvtsi64_si128(A);
 }
+TEST_CONSTEXPR(match_v2di(_mm_cvtsi64_si128(-99LL), -99LL, 0LL));
 
 __m128d test_mm_cvtss_sd(__m128d A, __m128 B) {
   // CHECK-LABEL: test_mm_cvtss_sd
@@ -915,6 +925,7 @@ __m128i test_mm_mul_epu32(__m128i A, __m128i B) {
   // CHECK: mul <2 x i64> %{{.*}}, %{{.*}}
   return _mm_mul_epu32(A, B);
 }
+TEST_CONSTEXPR(match_m128i(_mm_mul_epu32((__m128i)(__v4si){+1, -2, +3, -4}, (__m128i)(__v4si){-16, -14, +12, +10}), 4294967280, 36));
 
 __m128d test_mm_mul_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_mul_pd
@@ -938,18 +949,21 @@ __m128i test_mm_mulhi_epi16(__m128i A, __m128i B) {
   // CHECK: call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_mulhi_epi16(A, B);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_mulhi_epi16((__m128i)(__v8hi){+1, -2, +3, -4, +5, -6, +7, -8}, (__m128i)(__v8hi){-16, -14, +12, +10, -8, +6, -4, +2}), -1, 0, 0, -1, -1, -1, -1, -1));
 
 __m128i test_mm_mulhi_epu16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_mulhi_epu16
   // CHECK: call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_mulhi_epu16(A, B);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_mulhi_epu16((__m128i)(__v8hi){+1, -2, +3, -4, +5, -6, +7, -8}, (__m128i)(__v8hi){-16, -14, +12, +10, -8, +6, -4, +2}), 0, -16, 0, 9, 4, 5, 6, 1));
 
 __m128i test_mm_mullo_epi16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_mullo_epi16
   // CHECK: mul <8 x i16> %{{.*}}, %{{.*}}
   return _mm_mullo_epi16(A, B);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_mullo_epi16((__m128i)(__v8hi){+1, -2, +3, -4, +5, -6, +7, -8}, (__m128i)(__v8hi){-16, -14, +12, +10, -8, +6, -4, +2}), -16, 28, 36, -40, -40, -36, -28, -16));
 
 __m128d test_mm_or_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_or_pd
@@ -963,6 +977,7 @@ __m128i test_mm_or_si128(__m128i A, __m128i B) {
   // CHECK: or <2 x i64> %{{.*}}, %{{.*}}
   return _mm_or_si128(A, B);
 }
+TEST_CONSTEXPR(match_v4si(_mm_or_si128((__m128i)(__v4si){0, -1, 0, -1}, (__m128i)(__v4si){0, 0, -1, -1}), 0, -1, -1, -1));
 
 __m128i test_mm_packs_epi16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_packs_epi16
@@ -1685,12 +1700,14 @@ __m128i test_mm_subs_epi8(__m128i A, __m128i B) {
   // CHECK: call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_subs_epi8(A, B);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_subs_epi8((__m128i)(__v16qs){+100, +50, -100, +20, +80, -50, +120, -20, -100, -50, +100, -20, -80, +50, -120, +20}, (__m128i)(__v16qs){-50, -80, +50, -110, -60, +30, -20, +10, -50, -80, +50, -110, -60, +30, -20, +10}), +127, +127, -128, +127, +127, -80, +127, -30, -50, +30, +50, +90, -20, +20, -100, +10));
 
 __m128i test_mm_subs_epi16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_subs_epi16
   // CHECK: call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_subs_epi16(A, B);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_subs_epi16((__m128i)(__v8hi){+32000, -32000, +32000, -32000, +80, -50, +120, -20}, (__m128i)(__v8hi){-800, +800, +800, -800, -60, +30, -20, +10}), +32767, -32768, +31200, -31200, +140, -80, +140, -30));
 
 __m128i test_mm_subs_epu8(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_subs_epu8
@@ -1698,6 +1715,7 @@ __m128i test_mm_subs_epu8(__m128i A, __m128i B) {
   // CHECK: call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %{{.*}}, <16 x i8> %{{.*}})
   return _mm_subs_epu8(A, B);
 }
+TEST_CONSTEXPR(match_v16qu(_mm_subs_epu8((__m128i)(__v16qu){0, 0, 0, 0, +127, +127, +127, +127, +128, +128, +128, +128, +255, +255, +255, +255}, (__m128i)(__v16qu){0, +127, +128, +255, 0, +127, +128, +255, 0, +127, +128, +255, 0, +127, +128, +255}), 0, 0, 0, 0, +127, 0, 0, 0, +128, +1, 0, 0, +255, +128, +127, 0));
 
 __m128i test_mm_subs_epu16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_subs_epu16
@@ -1705,6 +1723,7 @@ __m128i test_mm_subs_epu16(__m128i A, __m128i B) {
   // CHECK: call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %{{.*}}, <8 x i16> %{{.*}})
   return _mm_subs_epu16(A, B);
 }
+TEST_CONSTEXPR(match_v8hu(_mm_subs_epu16((__m128i)(__v8hu){0, 0, 0, +32767, +32767, +32767, +65535, +65535}, (__m128i)(__v8hu){0, +32767, +65535, 0, +32767, +65535, 0, +32767}), 0, 0, 0, +32767, 0, 0, +65535, +32768));
 
 int test_mm_ucomieq_sd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_ucomieq_sd
@@ -1762,24 +1781,28 @@ __m128i test_mm_unpackhi_epi8(__m128i A, __m128i B) {
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
   return _mm_unpackhi_epi8(A, B);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_unpackhi_epi8((__m128i)(__v16qi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, (__m128i)(__v16qi){16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}), 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31));
 
 __m128i test_mm_unpackhi_epi16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_unpackhi_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
   return _mm_unpackhi_epi16(A, B);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_unpackhi_epi16((__m128i)(__v8hi){0, 1, 2, 3, 4, 5, 6, 7}, (__m128i)(__v8hi){8, 9, 10, 11, 12, 13, 14, 15}), 4, 12, 5, 13, 6, 14, 7, 15));
 
 __m128i test_mm_unpackhi_epi32(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_unpackhi_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
   return _mm_unpackhi_epi32(A, B);
 }
+TEST_CONSTEXPR(match_v4si(_mm_unpackhi_epi32((__m128i)(__v4si){0, 1, 2, 3}, (__m128i)(__v4si){ 4, 5, 6, 7}), 2, 6, 3, 7));
 
 __m128i test_mm_unpackhi_epi64(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_unpackhi_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> <i32 1, i32 3>
   return _mm_unpackhi_epi64(A, B);
 }
+TEST_CONSTEXPR(match_v2di(_mm_unpackhi_epi64((__m128i)(__v2di){0, 1}, (__m128i)(__v2di){2, 3}), 1, 3));
 
 __m128d test_mm_unpackhi_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_unpackhi_pd
@@ -1793,24 +1816,28 @@ __m128i test_mm_unpacklo_epi8(__m128i A, __m128i B) {
   // CHECK: shufflevector <16 x i8> %{{.*}}, <16 x i8> %{{.*}}, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
   return _mm_unpacklo_epi8(A, B);
 }
+TEST_CONSTEXPR(match_v16qi(_mm_unpacklo_epi8((__m128i)(__v16qi){0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, (__m128i)(__v16qi){16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}), 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23));
 
 __m128i test_mm_unpacklo_epi16(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_unpacklo_epi16
   // CHECK: shufflevector <8 x i16> %{{.*}}, <8 x i16> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   return _mm_unpacklo_epi16(A, B);
 }
+TEST_CONSTEXPR(match_v8hi(_mm_unpacklo_epi16((__m128i)(__v8hi){0, 1, 2, 3, 4, 5, 6, 7}, (__m128i)(__v8hi){8, 9, 10, 11, 12, 13, 14, 15}), 0, 8, 1, 9, 2, 10, 3, 11));
 
 __m128i test_mm_unpacklo_epi32(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_unpacklo_epi32
   // CHECK: shufflevector <4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   return _mm_unpacklo_epi32(A, B);
 }
+TEST_CONSTEXPR(match_v4si(_mm_unpacklo_epi32((__m128i)(__v4si){0, 1, 2, 3}, (__m128i)(__v4si){ 4, 5, 6, 7}), 0, 4, 1, 5));
 
 __m128i test_mm_unpacklo_epi64(__m128i A, __m128i B) {
   // CHECK-LABEL: test_mm_unpacklo_epi64
   // CHECK: shufflevector <2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i32> <i32 0, i32 2>
   return _mm_unpacklo_epi64(A, B);
 }
+TEST_CONSTEXPR(match_v2di(_mm_unpacklo_epi64((__m128i)(__v2di){0, 1}, (__m128i)(__v2di){2, 3}), 0, 2));
 
 __m128d test_mm_unpacklo_pd(__m128d A, __m128d B) {
   // CHECK-LABEL: test_mm_unpacklo_pd
@@ -1831,3 +1858,4 @@ __m128i test_mm_xor_si128(__m128i A, __m128i B) {
   // CHECK: xor <2 x i64> %{{.*}}, %{{.*}}
   return _mm_xor_si128(A, B);
 }
+TEST_CONSTEXPR(match_v4si(_mm_xor_si128((__m128i)(__v4si){0, -1, 0, -1}, (__m128i)(__v4si){0, 0, -1, -1}), 0, -1, -1, 0));
diff --git a/clang/test/CodeGen/X86/sse41-builtins.c b/clang/test/CodeGen/X86/sse41-builtins.c
index d71a4b7e789ef..500b780d49057 100644
--- a/clang/test/CodeGen/X86/sse41-builtins.c
+++ b/clang/test/CodeGen/X86/sse41-builtins.c
@@ -9,6 +9,7 @@
 
 
 #include <immintrin.h>
+#include "builtin_test_helpers.h"
 
 // NOTE: This should match the tests in llvm/test/CodeGen/X86/sse41-intrinsics-fast-isel.ll
 
@@ -86,6 +87,8 @@ __m128i test_mm_cvtepi8_epi16(__m128i a) {
   return _mm_cvtepi8_epi16(a);
 }
 
+TEST_CONSTEXPR(match_v8hi(_mm_cvtepi8_epi16(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 0, 0, 0, 0, 0, 0, 0, 0)), -3, 2, -1, 0, 1, -2, 3, -4));
+
 __m128i test_mm_cvtepi8_epi32(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepi8_epi32
   // CHECK: shufflevector <16 x i8> {{.*}}, <16 x i8> {{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -93,6 +96,8 @@ __m128i test_mm_cvtepi8_epi32(__m128i a) {
   return _mm_cvtepi8_epi32(a);
 }
 
+TEST_CONSTEXPR(match_v4si(_mm_cvtepi8_epi32(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 0, 0, 0, 0, 0, 0, 0, 0)), -3, 2, -1, 0));
+
 __m128i test_mm_cvtepi8_epi64(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepi8_epi64
   // CHECK: shufflevector <16 x i8> {{.*}}, <16 x i8> {{.*}}, <2 x i32> <i32 0, i32 1>
@@ -100,6 +105,8 @@ __m128i test_mm_cvtepi8_epi64(__m128i a) {
   return _mm_cvtepi8_epi64(a);
 }
 
+TEST_CONSTEXPR(match_v2di(_mm_cvtepi8_epi64(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 0, 0, 0, 0, 0, 0, 0, 0)), -3, 2));
+
 __m128i test_mm_cvtepi16_epi32(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepi16_epi32
   // CHECK: shufflevector <8 x i16> {{.*}}, <8 x i16> {{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -107,6 +114,8 @@ __m128i test_mm_cvtepi16_epi32(__m128i a) {
   return _mm_cvtepi16_epi32(a);
 }
 
+TEST_CONSTEXPR(match_v4si(_mm_cvtepi16_epi32(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), -300, 2, -1, 0));
+
 __m128i test_mm_cvtepi16_epi64(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepi16_epi64
   // CHECK: shufflevector <8 x i16> {{.*}}, <8 x i16> {{.*}}, <2 x i32> <i32 0, i32 1>
@@ -114,6 +123,8 @@ __m128i test_mm_cvtepi16_epi64(__m128i a) {
   return _mm_cvtepi16_epi64(a);
 }
 
+TEST_CONSTEXPR(match_v2di(_mm_cvtepi16_epi64(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), -300, 2));
+
 __m128i test_mm_cvtepi32_epi64(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepi32_epi64
   // CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> {{.*}}, <2 x i32> <i32 0, i32 1>
@@ -121,6 +132,8 @@ __m128i test_mm_cvtepi32_epi64(__m128i a) {
   return _mm_cvtepi32_epi64(a);
 }
 
+TEST_CONSTEXPR(match_v2di(_mm_cvtepi32_epi64(_mm_setr_epi32(-70000, 2, -1, 0)), -70000, 2));
+
 __m128i test_mm_cvtepu8_epi16(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepu8_epi16
   // CHECK: shufflevector <16 x i8> {{.*}}, <16 x i8> {{.*}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -128,6 +141,8 @@ __m128i test_mm_cvtepu8_epi16(__m128i a) {
   return _mm_cvtepu8_epi16(a);
 }
 
+TEST_CONSTEXPR(match_v8hi(_mm_cvtepu8_epi16(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 0, 0, 0, 0, 0, 0, 0, 0)), 253, 2, 255, 0, 1, 254, 3, 252));
+
 __m128i test_mm_cvtepu8_epi32(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepu8_epi32
   // CHECK: shufflevector <16 x i8> {{.*}}, <16 x i8> {{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -135,6 +150,8 @@ __m128i test_mm_cvtepu8_epi32(__m128i a) {
   return _mm_cvtepu8_epi32(a);
 }
 
+TEST_CONSTEXPR(match_v4si(_mm_cvtepu8_epi32(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 0, 0, 0, 0, 0, 0, 0, 0)), 253, 2, 255, 0));
+
 __m128i test_mm_cvtepu8_epi64(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepu8_epi64
   // CHECK: shufflevector <16 x i8> {{.*}}, <16 x i8> {{.*}}, <2 x i32> <i32 0, i32 1>
@@ -142,6 +159,8 @@ __m128i test_mm_cvtepu8_epi64(__m128i a) {
   return _mm_cvtepu8_epi64(a);
 }
 
+TEST_CONSTEXPR(match_v2di(_mm_cvtepu8_epi64(_mm_setr_epi8(-3, 2, -1, 0, 1, -2, 3, -4, 0, 0, 0, 0, 0, 0, 0, 0)), 253, 2));
+
 __m128i test_mm_cvtepu16_epi32(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepu16_epi32
   // CHECK: shufflevector <8 x i16> {{.*}}, <8 x i16> {{.*}}, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -149,6 +168,8 @@ __m128i test_mm_cvtepu16_epi32(__m128i a) {
   return _mm_cvtepu16_epi32(a);
 }
 
+TEST_CONSTEXPR(match_v4si(_mm_cvtepu16_epi32(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), 65236, 2, 65535, 0));
+
 __m128i test_mm_cvtepu16_epi64(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepu16_epi64
   // CHECK: shufflevector <8 x i16> {{.*}}, <8 x i16> {{.*}}, <2 x i32> <i32 0, i32 1>
@@ -156,6 +177,8 @@ __m128i test_mm_cvtepu16_epi64(__m128i a) {
   return _mm_cvtepu16_epi64(a);
 }
 
+TEST_CONSTEXPR(match_v2di(_mm_cvtepu16_epi64(_mm_setr_epi16(-300, 2, -1, 0, 1, -2, 3, -4)), 65236, 2));
+
 __m128i test_mm_cvtepu32_epi64(__m128i a) {
   // CHECK-LABEL: test_mm_cvtepu32_epi64
   // CHECK: shufflevector <4 x i32> {{.*}}, <4 x i32> {{.*}}, <2 x i32> <i32 0, i32 1>
@@ -163,6 +186,8 @@ __m128i test_mm_cvtepu32_epi64(__m128i a) {
   return _mm_cvtepu32_epi64(a);
 }
 
+TEST_CONSTEXPR(match_v2di(_mm_cvtepu32_epi64(_mm_setr_epi32(-70000, 2, -1, 0)), 4294897296, 2));
+
 __m128d test_mm_dp_pd(__m128d x, __m128d y) {
   // CHECK-LABEL: test_mm_dp_pd
   // CHECK: call {{.*}}<2 x double> @llvm.x86.sse41.dppd(<2 x double> {{.*}}, <2 x double> {{.*}}, i8 7)
@@ -319,6 +344,7 @@ __m128i test_mm_mul_epi32(__m128i x, __m128i y) {
   // CHECK: mul <2 x i64> %{{.*}}, %{{.*}}
   return _mm_mul_epi32(x, y);
 }
+TEST_CONSTEXPR(match_m128i(_mm_mul_epi32((__m128i)(__v4si){+1, -2, +3, -4}, (__m128i)(__v4si){-16, -14, +12, +10}), -16, 36));
 
 __m128i test_mm_mullo_epi32(__m128i x, __m128i y) {
   // CHECK-LABEL: test_mm_mullo_epi32
diff --git a/clang/test/CodeGen/X86/ssse3-builtins.c b/clang/test/CodeGen/X86/ssse3-builtins.c
index 982c74c5fc3b4..ee44868fa56f7 100644
--- a/clang/test/CodeGen/X86/ssse3-builtins.c
+++ b/clang/test/CodeGen/X86/ssse3-builtins.c
@@ -1,7 +1,11 @@
 // RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +ssse3 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
 // RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +ssse3 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
 // RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +ssse3 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
 // RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +ssse3 -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +ssse3 -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
diff --git a/clang/test/CodeGen/X86/xop-builtins.c b/clang/test/CodeGen/X86/xop-builtins.c
index 8ba6b8bc5915b..eb9f832ac9b0b 100644
--- a/clang/test/CodeGen/X86/xop-builtins.c
+++ b/clang/test/CodeGen/X86/xop-builtins.c
@@ -1,10 +1,15 @@
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +xop -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +xop -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +xop -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +xop -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +xop -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +xop -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -x c++ -flax-vector-conversions=none -ffreestanding %s -triple=i386-apple-darwin -target-feature +xop -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <x86intrin.h>
+#include "builtin_test_helpers.h"
 
 // NOTE: This should match the tests in llvm/test/CodeGen/X86/xop-intrinsics-fast-isel.ll
 
@@ -46,25 +51,25 @@ __m128i test_mm_macc_epi32(__m128i a, __m128i b, __m128i c) {
 
 __m128i test_mm_maccslo_epi32(__m128i a, __m128i b, __m128i c) {
   // CHECK-LABEL: test_mm_maccslo_epi32
-  // CHECK: call <2 x i64> @llvm.x86.xop.vpmacssdql(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vpmacssdql(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_maccslo_epi32(a, b, c);
 }
 
 __m128i test_mm_macclo_epi32(__m128i a, __m128i b, __m128i c) {
   // CHECK-LABEL: test_mm_macclo_epi32
-  // CHECK: call <2 x i64> @llvm.x86.xop.vpmacsdql(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vpmacsdql(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_macclo_epi32(a, b, c);
 }
 
 __m128i test_mm_maccshi_epi32(__m128i a, __m128i b, __m128i c) {
   // CHECK-LABEL: test_mm_maccshi_epi32
-  // CHECK: call <2 x i64> @llvm.x86.xop.vpmacssdqh(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vpmacssdqh(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_maccshi_epi32(a, b, c);
 }
 
 __m128i test_mm_macchi_epi32(__m128i a, __m128i b, __m128i c) {
   // CHECK-LABEL: test_mm_macchi_epi32
-  // CHECK: call <2 x i64> @llvm.x86.xop.vpmacsdqh(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vpmacsdqh(<4 x i32> %{{.*}}, <4 x i32> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_macchi_epi32(a, b, c);
 }
 
@@ -94,7 +99,7 @@ __m128i test_mm_haddd_epi8(__m128i a) {
 
 __m128i test_mm_haddq_epi8(__m128i a) {
   // CHECK-LABEL: test_mm_haddq_epi8
-  // CHECK: call <2 x i64> @llvm.x86.xop.vphaddbq(<16 x i8> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vphaddbq(<16 x i8> %{{.*}})
   return _mm_haddq_epi8(a);
 }
 
@@ -106,13 +111,13 @@ __m128i test_mm_haddd_epi16(__m128i a) {
 
 __m128i test_mm_haddq_epi16(__m128i a) {
   // CHECK-LABEL: test_mm_haddq_epi16
-  // CHECK: call <2 x i64> @llvm.x86.xop.vphaddwq(<8 x i16> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vphaddwq(<8 x i16> %{{.*}})
   return _mm_haddq_epi16(a);
 }
 
 __m128i test_mm_haddq_epi32(__m128i a) {
   // CHECK-LABEL: test_mm_haddq_epi32
-  // CHECK: call <2 x i64> @llvm.x86.xop.vphadddq(<4 x i32> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vphadddq(<4 x i32> %{{.*}})
   return _mm_haddq_epi32(a);
 }
 
@@ -130,7 +135,7 @@ __m128i test_mm_haddd_epu8(__m128i a) {
 
 __m128i test_mm_haddq_epu8(__m128i a) {
   // CHECK-LABEL: test_mm_haddq_epu8
-  // CHECK: call <2 x i64> @llvm.x86.xop.vphaddubq(<16 x i8> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vphaddubq(<16 x i8> %{{.*}})
   return _mm_haddq_epu8(a);
 }
 
@@ -142,13 +147,13 @@ __m128i test_mm_haddd_epu16(__m128i a) {
 
 __m128i test_mm_haddq_epu16(__m128i a) {
   // CHECK-LABEL: test_mm_haddq_epu16
-  // CHECK: call <2 x i64> @llvm.x86.xop.vphadduwq(<8 x i16> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vphadduwq(<8 x i16> %{{.*}})
   return _mm_haddq_epu16(a);
 }
 
 __m128i test_mm_haddq_epu32(__m128i a) {
   // CHECK-LABEL: test_mm_haddq_epu32
-  // CHECK: call <2 x i64> @llvm.x86.xop.vphaddudq(<4 x i32> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vphaddudq(<4 x i32> %{{.*}})
   return _mm_haddq_epu32(a);
 }
 
@@ -166,7 +171,7 @@ __m128i test_mm_hsubd_epi16(__m128i a) {
 
 __m128i test_mm_hsubq_epi32(__m128i a) {
   // CHECK-LABEL: test_mm_hsubq_epi32
-  // CHECK: call <2 x i64> @llvm.x86.xop.vphsubdq(<4 x i32> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vphsubdq(<4 x i32> %{{.*}})
   return _mm_hsubq_epi32(a);
 }
 
@@ -178,6 +183,7 @@ __m128i test_mm_cmov_si128(__m128i a, __m128i b, __m128i c) {
   // CHECK-NEXT: %{{.*}} = or <2 x i64> [[AND]], [[ANDN]]
   return _mm_cmov_si128(a, b, c);
 }
+TEST_CONSTEXPR(match_v4si(_mm_cmov_si128((__m128i)(__v4si){+1,+2,+3,+4}, (__m128i)(__v4si){-4,-3,-2,-1}, (__m128i)(__v4si){-1,0,0,-1}), +1, -3, -2, +4));
 
 __m256i test_mm256_cmov_si256(__m256i a, __m256i b, __m256i c) {
   // CHECK-LABEL: test_mm256_cmov_si256
@@ -187,6 +193,7 @@ __m256i test_mm256_cmov_si256(__m256i a, __m256i b, __m256i c) {
   // CHECK-NEXT: %{{.*}} = or <4 x i64> [[AND]], [[ANDN]]
   return _mm256_cmov_si256(a, b, c);
 }
+TEST_CONSTEXPR(match_v4di(_mm256_cmov_si256((__m256i)(__v4di){+1,+2,+3,+4}, (__m256i)(__v4di){-4,-3,-2,-1}, (__m256i)(__v4di){0,-1,0,-1}), -4, +2, -2, +4));
 
 __m128i test_mm_perm_epi8(__m128i a, __m128i b, __m128i c) {
   // CHECK-LABEL: test_mm_perm_epi8
@@ -214,7 +221,7 @@ __m128i test_mm_rot_epi32(__m128i a, __m128i b) {
 
 __m128i test_mm_rot_epi64(__m128i a, __m128i b) {
   // CHECK-LABEL: test_mm_rot_epi64
-  // CHECK: call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_rot_epi64(a, b);
 }
 
@@ -238,7 +245,7 @@ __m128i test_mm_roti_epi32(__m128i a) {
 
 __m128i test_mm_roti_epi64(__m128i a) {
   // CHECK-LABEL: test_mm_roti_epi64
-  // CHECK: call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 100))
+  // CHECK: call {{.*}}<2 x i64> @llvm.fshl.v2i64(<2 x i64> %{{.*}}, <2 x i64> %{{.*}}, <2 x i64> splat (i64 100))
   return _mm_roti_epi64(a, 100);
 }
 
@@ -262,7 +269,7 @@ __m128i test_mm_shl_epi32(__m128i a, __m128i b) {
 
 __m128i test_mm_shl_epi64(__m128i a, __m128i b) {
   // CHECK-LABEL: test_mm_shl_epi64
-  // CHECK: call <2 x i64> @llvm.x86.xop.vpshlq(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vpshlq(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_shl_epi64(a, b);
 }
 
@@ -286,7 +293,7 @@ __m128i test_mm_sha_epi32(__m128i a, __m128i b) {
 
 __m128i test_mm_sha_epi64(__m128i a, __m128i b) {
   // CHECK-LABEL: test_mm_sha_epi64
-  // CHECK: call <2 x i64> @llvm.x86.xop.vpshaq(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+  // CHECK: call {{.*}}<2 x i64> @llvm.x86.xop.vpshaq(<2 x i64> %{{.*}}, <2 x i64> %{{.*}})
   return _mm_sha_epi64(a, b);
 }
 
@@ -372,36 +379,36 @@ __m256 test_mm256_permute2_ps(__m256 a, __m256 b, __m256i c) {
 
 __m128 test_mm_frcz_ss(__m128 a) {
   // CHECK-LABEL: test_mm_frcz_ss
-  // CHECK: call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %{{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %{{.*}})
   return _mm_frcz_ss(a);
 }
 
 __m128d test_mm_frcz_sd(__m128d a) {
   // CHECK-LABEL: test_mm_frcz_sd
-  // CHECK: call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %{{.*}})
   return _mm_frcz_sd(a);
 }
 
 __m128 test_mm_frcz_ps(__m128 a) {
   // CHECK-LABEL: test_mm_frcz_ps
-  // CHECK: call <4 x float> @llvm.x86.xop.vfrcz.ps(<4 x float> %{{.*}})
+  // CHECK: call {{.*}}<4 x float> @llvm.x86.xop.vfrcz.ps(<4 x float> %{{.*}})
   return _mm_frcz_ps(a);
 }
 
 __m128d test_mm_frcz_pd(__m128d a) {
   // CHECK-LABEL: test_mm_frcz_pd
-  // CHECK: call <2 x double> @llvm.x86.xop.vfrcz.pd(<2 x double> %{{.*}})
+  // CHECK: call {{.*}}<2 x double> @llvm.x86.xop.vfrcz.pd(<2 x double> %{{.*}})
   return _mm_frcz_pd(a);
 }
 
 __m256 test_mm256_frcz_ps(__m256 a) {
   // CHECK-LABEL: test_mm256_frcz_ps
-  // CHECK: call <8 x float> @llvm.x86.xop.vfrcz.ps.256(<8 x float> %{{.*}})
+  // CHECK: call {{.*}}<8 x float> @llvm.x86.xop.vfrcz.ps.256(<8 x float> %{{.*}})
   return _mm256_frcz_ps(a);
 }
 
 __m256d test_mm256_frcz_pd(__m256d a) {
   // CHECK-LABEL: test_mm256_frcz_pd
-  // CHECK: call <4 x double> @llvm.x86.xop.vfrcz.pd.256(<4 x double> %{{.*}})
+  // CHECK: call {{.*}}<4 x double> @llvm.x86.xop.vfrcz.pd.256(<4 x double> %{{.*}})
   return _mm256_frcz_pd(a);
 }
diff --git a/clang/test/CodeGen/aggregate-assign-call.c b/clang/test/CodeGen/aggregate-assign-call.c
index 7d972397ffaac..f09e770321656 100644
--- a/clang/test/CodeGen/aggregate-assign-call.c
+++ b/clang/test/CodeGen/aggregate-assign-call.c
@@ -24,23 +24,23 @@ struct S bar(void) {
   // O1: %[[TMP2_ALLOCA:[^ ]+]] = alloca %struct.S
   // O1: %[[TMP3_ALLOCA:[^ ]+]] = alloca %struct.S
 
-  // O1: call void @llvm.lifetime.start.p0({{[^,]*}}, ptr %[[TMP1_ALLOCA]])
+  // O1: call void @llvm.lifetime.start.p0(ptr %[[TMP1_ALLOCA]])
   // O1: call void @foo
   r = foo();
   // O1: memcpy
-  // O1: call void @llvm.lifetime.end.p0({{[^,]*}}, ptr %[[TMP1_ALLOCA]])
+  // O1: call void @llvm.lifetime.end.p0(ptr %[[TMP1_ALLOCA]])
 
-  // O1: call void @llvm.lifetime.start.p0({{[^,]*}}, ptr %[[TMP2_ALLOCA]])
+  // O1: call void @llvm.lifetime.start.p0(ptr %[[TMP2_ALLOCA]])
   // O1: call void @foo
   r = foo();
   // O1: memcpy
-  // O1: call void @llvm.lifetime.end.p0({{[^,]*}}, ptr %[[TMP2_ALLOCA]])
+  // O1: call void @llvm.lifetime.end.p0(ptr %[[TMP2_ALLOCA]])
 
-  // O1: call void @llvm.lifetime.start.p0({{[^,]*}}, ptr %[[TMP3_ALLOCA]])
+  // O1: call void @llvm.lifetime.start.p0(ptr %[[TMP3_ALLOCA]])
   // O1: call void @foo
   r = foo();
   // O1: memcpy
-  // O1: call void @llvm.lifetime.end.p0({{[^,]*}}, ptr %[[TMP3_ALLOCA]])
+  // O1: call void @llvm.lifetime.end.p0(ptr %[[TMP3_ALLOCA]])
 
   return r;
 }
@@ -59,17 +59,17 @@ struct S baz(int i, volatile int *j) {
   // O1: %[[TMP2_ALLOCA:[^ ]+]] = alloca %struct.S
 
   do {
-    // O1: call void @llvm.lifetime.start.p0({{[^,]*}}, ptr %[[TMP1_ALLOCA]])
+    // O1: call void @llvm.lifetime.start.p0(ptr %[[TMP1_ALLOCA]])
     //
-    // O1: call void @llvm.lifetime.end.p0({{[^,]*}}, ptr %[[TMP1_ALLOCA]])
+    // O1: call void @llvm.lifetime.end.p0(ptr %[[TMP1_ALLOCA]])
     //
     // O1: call void @foo_int(ptr dead_on_unwind writable sret(%struct.S) align 4 %[[TMP1_ALLOCA]],
     // O1: call void @llvm.memcpy
-    // O1: call void @llvm.lifetime.end.p0({{[^,]*}}, ptr %[[TMP1_ALLOCA]])
-    // O1: call void @llvm.lifetime.start.p0({{[^,]*}}, ptr %[[TMP2_ALLOCA]])
+    // O1: call void @llvm.lifetime.end.p0(ptr %[[TMP1_ALLOCA]])
+    // O1: call void @llvm.lifetime.start.p0(ptr %[[TMP2_ALLOCA]])
     // O1: call void @foo_int(ptr dead_on_unwind writable sret(%struct.S) align 4 %[[TMP2_ALLOCA]],
     // O1: call void @llvm.memcpy
-    // O1: call void @llvm.lifetime.end.p0({{[^,]*}}, ptr %[[TMP2_ALLOCA]])
+    // O1: call void @llvm.lifetime.end.p0(ptr %[[TMP2_ALLOCA]])
     r = foo_int(({
       if (*j)
         break;
diff --git a/clang/test/CodeGen/attr-counted-by-for-pointers.c b/clang/test/CodeGen/attr-counted-by-for-pointers.c
index e939e49a61d4d..0d72b58c78fd1 100644
--- a/clang/test/CodeGen/attr-counted-by-for-pointers.c
+++ b/clang/test/CodeGen/attr-counted-by-for-pointers.c
@@ -32,7 +32,7 @@ struct annotated_ptr {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 8
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT10:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3:![0-9]+]], !nosanitize [[META2]]
@@ -85,7 +85,7 @@ void test1(struct annotated_ptr *p, int index, struct foo *value) {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 8
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT10:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
@@ -138,7 +138,7 @@ void test2(struct annotated_ptr *p, int index, struct foo *value) {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 8
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTNOT:%.*]] = icmp ugt i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[DOTNOT]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], label [[CONT10:%.*]], !prof [[PROF15:![0-9]+]], !nosanitize [[META2]]
@@ -311,7 +311,7 @@ size_t test6(struct annotated_ptr *p, int index) {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[P]], i64 16
-// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 4
+// SANITIZE-WITH-ATTR-NEXT:    [[DOTCOUNTED_BY_LOAD:%.*]] = load i32, ptr [[DOTCOUNTED_BY_GEP]], align 8
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = zext i32 [[DOTCOUNTED_BY_LOAD]] to i64, !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[IDXPROM]], [[TMP0]], !nosanitize [[META2]]
 // SANITIZE-WITH-ATTR-NEXT:    br i1 [[TMP1]], label [[CONT10:%.*]], label [[HANDLER_OUT_OF_BOUNDS:%.*]], !prof [[PROF3]], !nosanitize [[META2]]
diff --git a/clang/test/CodeGen/attr-counted-by.c b/clang/test/CodeGen/attr-counted-by.c
index 9fb50c6ef0fff..59e1b134850a9 100644
--- a/clang/test/CodeGen/attr-counted-by.c
+++ b/clang/test/CodeGen/attr-counted-by.c
@@ -1209,7 +1209,7 @@ int test12_a, test12_b;
 // SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // SANITIZE-WITH-ATTR-NEXT:  entry:
 // SANITIZE-WITH-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR9:[0-9]+]]
+// SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BAZ]]) #[[ATTR9:[0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT10:![0-9]+]]
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITH-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
@@ -1235,7 +1235,7 @@ int test12_a, test12_b;
 // NO-SANITIZE-WITH-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR4:[0-9]+]] {
 // NO-SANITIZE-WITH-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// NO-SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR12:[0-9]+]]
+// NO-SANITIZE-WITH-ATTR-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BAZ]]) #[[ATTR12:[0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITH-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [6 x i32], ptr [[BAZ]], i64 0, i64 [[IDXPROM]]
@@ -1251,7 +1251,7 @@ int test12_a, test12_b;
 // SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR3:[0-9]+]] {
 // SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR7:[0-9]+]]
+// SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BAZ]]) #[[ATTR7:[0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[INDEX]], 6
 // SANITIZE-WITHOUT-ATTR-NEXT:    [[TMP1:%.*]] = zext i32 [[INDEX]] to i64
@@ -1277,7 +1277,7 @@ int test12_a, test12_b;
 // NO-SANITIZE-WITHOUT-ATTR-SAME: i32 noundef [[INDEX:%.*]]) local_unnamed_addr #[[ATTR2:[0-9]+]] {
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:  entry:
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[BAZ:%.*]] = alloca [[STRUCT_HANG:%.*]], align 4
-// NO-SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[BAZ]]) #[[ATTR10:[0-9]+]]
+// NO-SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BAZ]]) #[[ATTR10:[0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 4 dereferenceable(24) [[BAZ]], ptr noundef nonnull align 4 dereferenceable(24) @test12_bar, i64 24, i1 false), !tbaa.struct [[TBAA_STRUCT7:![0-9]+]]
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[IDXPROM:%.*]] = sext i32 [[INDEX]] to i64
 // NO-SANITIZE-WITHOUT-ATTR-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [6 x i32], ptr [[BAZ]], i64 0, i64 [[IDXPROM]]
diff --git a/clang/test/CodeGen/builtin-bpf-btf-type-id.c b/clang/test/CodeGen/builtin-bpf-btf-type-id.c
index 4c6efd69eb956..c8f29ee5fe4ab 100644
--- a/clang/test/CodeGen/builtin-bpf-btf-type-id.c
+++ b/clang/test/CodeGen/builtin-bpf-btf-type-id.c
@@ -21,5 +21,5 @@ unsigned test3() {
 //
 // CHECK: ![[INT]] = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed
 // CHECK: ![[INT_POINTER]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[INT]], size: 64
-// CHECK: ![[STRUCT_T1]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1"
 // CHECK: ![[TYPEDEF_T1]] = !DIDerivedType(tag: DW_TAG_typedef, name: "__t1"
+// CHECK: ![[STRUCT_T1]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1"
diff --git a/clang/test/CodeGen/builtin-masked.c b/clang/test/CodeGen/builtin-masked.c
new file mode 100644
index 0000000000000..67071ba19bd25
--- /dev/null
+++ b/clang/test/CodeGen/builtin-masked.c
@@ -0,0 +1,53 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -emit-llvm -o - %s | FileCheck %s
+
+typedef int v8i __attribute__((ext_vector_type(8)));
+typedef _Bool v8b __attribute__((ext_vector_type(8)));
+
+// CHECK-LABEL: define dso_local <8 x i32> @test_load(
+// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    [[P_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store i8 [[M_COERCE]], ptr [[M]], align 1
+// CHECK-NEXT:    [[LOAD_BITS:%.*]] = load i8, ptr [[M]], align 1
+// CHECK-NEXT:    [[M1:%.*]] = bitcast i8 [[LOAD_BITS]] to <8 x i1>
+// CHECK-NEXT:    [[TMP0:%.*]] = bitcast <8 x i1> [[M1]] to i8
+// CHECK-NEXT:    store i8 [[TMP0]], ptr [[M_ADDR]], align 1
+// CHECK-NEXT:    store ptr [[P]], ptr [[P_ADDR]], align 8
+// CHECK-NEXT:    [[LOAD_BITS2:%.*]] = load i8, ptr [[M_ADDR]], align 1
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8 [[LOAD_BITS2]] to <8 x i1>
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[P_ADDR]], align 8
+// CHECK-NEXT:    [[MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr [[TMP2]], i32 32, <8 x i1> [[TMP1]], <8 x i32> poison)
+// CHECK-NEXT:    ret <8 x i32> [[MASKED_LOAD]]
+//
+v8i test_load(v8b m, v8i *p) {
+  return __builtin_masked_load(m, p);
+}
+
+// CHECK-LABEL: define dso_local void @test_store(
+// CHECK-SAME: i8 noundef [[M_COERCE:%.*]], ptr noundef byval(<8 x i32>) align 32 [[TMP0:%.*]], ptr noundef [[P:%.*]]) #[[ATTR2:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[M:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    [[M_ADDR:%.*]] = alloca i8, align 1
+// CHECK-NEXT:    [[V_ADDR:%.*]] = alloca <8 x i32>, align 32
+// CHECK-NEXT:    [[P_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store i8 [[M_COERCE]], ptr [[M]], align 1
+// CHECK-NEXT:    [[LOAD_BITS:%.*]] = load i8, ptr [[M]], align 1
+// CHECK-NEXT:    [[M1:%.*]] = bitcast i8 [[LOAD_BITS]] to <8 x i1>
+// CHECK-NEXT:    [[V:%.*]] = load <8 x i32>, ptr [[TMP0]], align 32
+// CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i1> [[M1]] to i8
+// CHECK-NEXT:    store i8 [[TMP1]], ptr [[M_ADDR]], align 1
+// CHECK-NEXT:    store <8 x i32> [[V]], ptr [[V_ADDR]], align 32
+// CHECK-NEXT:    store ptr [[P]], ptr [[P_ADDR]], align 8
+// CHECK-NEXT:    [[LOAD_BITS2:%.*]] = load i8, ptr [[M_ADDR]], align 1
+// CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8 [[LOAD_BITS2]] to <8 x i1>
+// CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr [[V_ADDR]], align 32
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[P_ADDR]], align 8
+// CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0(<8 x i32> [[TMP3]], ptr [[TMP4]], i32 32, <8 x i1> [[TMP2]])
+// CHECK-NEXT:    ret void
+//
+void test_store(v8b m, v8i v, v8i *p) {
+  __builtin_masked_store(m, v, p);
+}
diff --git a/clang/test/CodeGen/builtins-elementwise-math.c b/clang/test/CodeGen/builtins-elementwise-math.c
index ee8345ff51e5e..188a6c3a30f0e 100644
--- a/clang/test/CodeGen/builtins-elementwise-math.c
+++ b/clang/test/CodeGen/builtins-elementwise-math.c
@@ -66,7 +66,7 @@ void test_builtin_elementwise_abs(float f1, float f2, double d1, double d2,
   // CHECK-NEXT: call i32 @llvm.abs.i32(i32 [[IA1]], i1 false)
   b = __builtin_elementwise_abs(int_as_one);
 
-  // CHECK:   call i32 @llvm.abs.i32(i32 -10, i1 false)
+  // CHECK:   store i32 %elt.abs11, ptr @b, align 4
   b = __builtin_elementwise_abs(-10);
 
   // CHECK:      [[SI:%.+]] = load i16, ptr %si.addr, align 2
@@ -418,7 +418,7 @@ void test_builtin_elementwise_max(float f1, float f2, double d1, double d2,
   // CHECK-NEXT: call i32 @llvm.smax.i32(i32 [[IAS1]], i32 [[B]])
   int_as_one = __builtin_elementwise_max(int_as_one, b);
 
-  // CHECK: call i32 @llvm.smax.i32(i32 1, i32 97)
+  // CHECK: store i64 97, ptr [[I1:%.+]], align 8
   i1 = __builtin_elementwise_max(1, 'a');
 }
 
@@ -508,6 +508,9 @@ void test_builtin_elementwise_min(float f1, float f2, double d1, double d2,
   // CHECK-NEXT: [[B:%.+]] = load i32, ptr @b, align 4
   // CHECK-NEXT: call i32 @llvm.smin.i32(i32 [[IAS1]], i32 [[B]])
   int_as_one = __builtin_elementwise_min(int_as_one, b);
+
+  // CHECK: store i64 2, ptr [[I1:%.+]], align 8
+  i1 = __builtin_elementwise_min(2, 'b');
 }
 
 void test_builtin_elementwise_bitreverse(si8 vi1, si8 vi2,
@@ -1176,3 +1179,185 @@ void test_builtin_elementwise_fma(float f32, double f64,
   half2 tmp2_v2f16 = __builtin_elementwise_fma(v2f16, v2f16, (half2)4.0);
 
 }
+
+void test_builtin_elementwise_fshl(long long int i1, long long int i2,
+				   long long int i3, unsigned short us1,
+                                   unsigned short us2, unsigned short us3,
+                                   char c1, char c2, char c3,
+                                   unsigned char uc1, unsigned char uc2,
+                                   unsigned char uc3,  si8 vi1, si8 vi2,
+                                   si8 vi3, u4 vu1, u4 vu2, u4 vu3) {
+  // CHECK:      [[I1:%.+]] = load i64, ptr %i1.addr
+  // CHECK-NEXT: [[I2:%.+]] = load i64, ptr %i2.addr
+  // CHECK-NEXT: [[I3:%.+]] = load i64, ptr %i3.addr
+  // CHECK-NEXT: [[I4:%.+]] = call i64 @llvm.fshl.i64(i64 [[I1]], i64 [[I2]], i64 [[I3]])
+  // CHECK-NEXT:              store i64 [[I4]], ptr %tmp_lli_l
+  // CHECK-NEXT: [[I5:%.+]] = load i64, ptr %i1.addr
+  // CHECK-NEXT: [[I6:%.+]] = load i64, ptr %i2.addr
+  // CHECK-NEXT: [[I7:%.+]] = load i64, ptr %i3.addr
+  // CHECK-NEXT: [[I8:%.+]] = call i64 @llvm.fshr.i64(i64 [[I5]], i64 [[I6]], i64 [[I7]])
+  // CHECK-NEXT:              store i64 [[I8]], ptr %tmp_lli_r
+  long long int tmp_lli_l = __builtin_elementwise_fshl(i1, i2, i3);
+  long long int tmp_lli_r = __builtin_elementwise_fshr(i1, i2, i3);
+
+  // CHECK:      [[US1:%.+]] = load i16, ptr %us1.addr
+  // CHECK-NEXT: [[US2:%.+]] = load i16, ptr %us2.addr
+  // CHECK-NEXT: [[US3:%.+]] = load i16, ptr %us3.addr
+  // CHECK-NEXT: [[US4:%.+]] = call i16 @llvm.fshl.i16(i16 [[US1]], i16 [[US2]], i16 [[US3]])
+  // CHECK-NEXT:               store i16 [[US4]], ptr %tmp_usi_l
+  // CHECK-NEXT: [[US5:%.+]] = load i16, ptr %us1.addr
+  // CHECK-NEXT: [[US6:%.+]] = load i16, ptr %us2.addr
+  // CHECK-NEXT: [[US7:%.+]] = load i16, ptr %us3.addr
+  // CHECK-NEXT: [[US8:%.+]] = call i16 @llvm.fshr.i16(i16 [[US5]], i16 [[US6]], i16 [[US7]])
+  // CHECK-NEXT:               store i16 [[US8]], ptr %tmp_usi_r
+  unsigned short tmp_usi_l = __builtin_elementwise_fshl(us1, us2, us3);
+  unsigned short tmp_usi_r = __builtin_elementwise_fshr(us1, us2, us3);
+
+  // CHECK:      [[C1:%.+]] = load i8, ptr %c1.addr
+  // CHECK-NEXT: [[C2:%.+]] = load i8, ptr %c2.addr
+  // CHECK-NEXT: [[C3:%.+]] = load i8, ptr %c3.addr
+  // CHECK-NEXT: [[C4:%.+]] = call i8 @llvm.fshl.i8(i8 [[C1]], i8 [[C2]], i8 [[C3]])
+  // CHECK-NEXT:              store i8 [[C4]], ptr %tmp_c_l
+  // CHECK-NEXT: [[C5:%.+]] = load i8, ptr %c1.addr
+  // CHECK-NEXT: [[C6:%.+]] = load i8, ptr %c2.addr
+  // CHECK-NEXT: [[C7:%.+]] = load i8, ptr %c3.addr
+  // CHECK-NEXT: [[C8:%.+]] = call i8 @llvm.fshr.i8(i8 [[C5]], i8 [[C6]], i8 [[C7]])
+  // CHECK-NEXT:              store i8 [[C8]], ptr %tmp_c_r
+  char tmp_c_l = __builtin_elementwise_fshl(c1, c2, c3);
+  char tmp_c_r = __builtin_elementwise_fshr(c1, c2, c3);
+
+  // CHECK:      [[UC1:%.+]] = load i8, ptr %uc1.addr
+  // CHECK-NEXT: [[UC2:%.+]] = load i8, ptr %uc2.addr
+  // CHECK-NEXT: [[UC3:%.+]] = load i8, ptr %uc3.addr
+  // CHECK-NEXT: [[UC4:%.+]] = call i8 @llvm.fshl.i8(i8 [[UC1]], i8 [[UC2]], i8 [[UC3]])
+  // CHECK-NEXT:               store i8 [[UC4]], ptr %tmp_uc_l
+  // CHECK-NEXT: [[UC5:%.+]] = load i8, ptr %uc1.addr
+  // CHECK-NEXT: [[UC6:%.+]] = load i8, ptr %uc2.addr
+  // CHECK-NEXT: [[UC7:%.+]] = load i8, ptr %uc3.addr
+  // CHECK-NEXT: [[UC8:%.+]] = call i8 @llvm.fshr.i8(i8 [[UC5]], i8 [[UC6]], i8 [[UC7]])
+  // CHECK-NEXT:               store i8 [[UC8]], ptr %tmp_uc_r
+  unsigned char tmp_uc_l = __builtin_elementwise_fshl(uc1, uc2, uc3);
+  unsigned char tmp_uc_r = __builtin_elementwise_fshr(uc1, uc2, uc3);
+
+  // CHECK:      [[VI1:%.+]] = load <8 x i16>, ptr %vi1.addr
+  // CHECK-NEXT: [[VI2:%.+]] = load <8 x i16>, ptr %vi2.addr
+  // CHECK-NEXT: [[VI3:%.+]] = load <8 x i16>, ptr %vi3.addr
+  // CHECK-NEXT: [[VI4:%.+]] = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> [[VI1]], <8 x i16> [[VI2]], <8 x i16> [[VI3]])
+  // CHECK-NEXT:               store <8 x i16> [[VI4]], ptr %tmp_vi_l
+  // CHECK-NEXT: [[VI5:%.+]] = load <8 x i16>, ptr %vi1.addr
+  // CHECK-NEXT: [[VI6:%.+]] = load <8 x i16>, ptr %vi2.addr
+  // CHECK-NEXT: [[VI7:%.+]] = load <8 x i16>, ptr %vi3.addr
+  // CHECK-NEXT: [[VI8:%.+]] = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> [[VI5]], <8 x i16> [[VI6]], <8 x i16> [[VI7]])
+  // CHECK-NEXT:               store <8 x i16> [[VI8]], ptr %tmp_vi_r
+  si8 tmp_vi_l = __builtin_elementwise_fshl(vi1, vi2, vi3);
+  si8 tmp_vi_r = __builtin_elementwise_fshr(vi1, vi2, vi3);
+
+  // CHECK:      [[VU1:%.+]] = load <4 x i32>, ptr %vu1.addr
+  // CHECK-NEXT: [[VU2:%.+]] = load <4 x i32>, ptr %vu2.addr
+  // CHECK-NEXT: [[VU3:%.+]] = load <4 x i32>, ptr %vu3.addr
+  // CHECK-NEXT: [[VU4:%.+]] = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> [[VU1]], <4 x i32> [[VU2]], <4 x i32> [[VU3]])
+  // CHECK-NEXT:               store <4 x i32> [[VU4]], ptr %tmp_vu_l
+  // CHECK-NEXT: [[VU5:%.+]] = load <4 x i32>, ptr %vu1.addr
+  // CHECK-NEXT: [[VU6:%.+]] = load <4 x i32>, ptr %vu2.addr
+  // CHECK-NEXT: [[VU7:%.+]] = load <4 x i32>, ptr %vu3.addr
+  // CHECK-NEXT: [[VU8:%.+]] = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> [[VU5]], <4 x i32> [[VU6]], <4 x i32> [[VU7]])
+  // CHECK-NEXT:               store <4 x i32> [[VU8]], ptr %tmp_vu_r
+  u4 tmp_vu_l = __builtin_elementwise_fshl(vu1, vu2, vu3);
+  u4 tmp_vu_r = __builtin_elementwise_fshr(vu1, vu2, vu3);
+}
+
+void test_builtin_elementwise_ctlz(si8 vs1, si8 vs2, u4 vu1,
+                                   long long int lli, short si,
+                                   _BitInt(31) bi, int i,
+                                   char ci) {
+  // CHECK:      [[V8S1:%.+]] = load <8 x i16>, ptr %vs1.addr
+  // CHECK-NEXT: call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[V8S1]], i1 true)
+  vs1 = __builtin_elementwise_ctlz(vs1);
+
+  // CHECK:      [[V8S1:%.+]] = load <8 x i16>, ptr %vs1.addr
+  // CHECK-NEXT: [[CLZ:%.+]] = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> [[V8S1]], i1 true)
+  // CHECK-NEXT: [[ISZERO:%.+]] = icmp eq <8 x i16> [[V8S1]], zeroinitializer
+  // CHECK-NEXT: [[V8S2:%.+]] = load <8 x i16>, ptr %vs2.addr
+  // select <8 x i1> [[ISZERO]], <8 x i16> [[CLZ]], <8 x i16> [[V8S2]]
+  vs1 = __builtin_elementwise_ctlz(vs1, vs2);
+
+  // CHECK:      [[V4U1:%.+]] = load <4 x i32>, ptr %vu1.addr
+  // CHECK-NEXT: call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[V4U1]], i1 true)
+  vu1 = __builtin_elementwise_ctlz(vu1);
+
+  // CHECK:      [[LLI:%.+]] = load i64, ptr %lli.addr
+  // CHECK-NEXT: call i64 @llvm.ctlz.i64(i64 [[LLI]], i1 true)
+  lli = __builtin_elementwise_ctlz(lli);
+
+  // CHECK:      [[SI:%.+]] = load i16, ptr %si.addr
+  // CHECK-NEXT: call i16 @llvm.ctlz.i16(i16 [[SI]], i1 true)
+  si = __builtin_elementwise_ctlz(si);
+
+  // CHECK:      [[BI1:%.+]] = load i32, ptr %bi.addr
+  // CHECK-NEXT: [[BI2:%.+]] = trunc i32 [[BI1]] to i31
+  // CHECK-NEXT: call i31 @llvm.ctlz.i31(i31 [[BI2]], i1 true)
+  bi = __builtin_elementwise_ctlz(bi);
+
+  // CHECK:      [[BI1:%.+]] = load i32, ptr %bi.addr
+  // CHECK-NEXT: [[BI2:%.+]] = trunc i32 [[BI1]] to i31
+  // CHECK-NEXT: [[CLZ:%.+]] = call i31 @llvm.ctlz.i31(i31 [[BI2]], i1 true)
+  // CHECK-NEXT: [[ISZERO:%.+]] = icmp eq i31 [[BI2]], 0
+  // CHECK-NEXT: select i1 [[ISZERO]], i31 1, i31 [[CLZ]]
+  bi = __builtin_elementwise_ctlz(bi, (_BitInt(31))1);
+
+  // CHECK:      [[I:%.+]] = load i32, ptr %i.addr
+  // CHECK-NEXT: call i32 @llvm.ctlz.i32(i32 [[I]], i1 true)
+  i = __builtin_elementwise_ctlz(i);
+
+  // CHECK:      [[CI:%.+]] = load i8, ptr %ci.addr
+  // CHECK-NEXT: call i8 @llvm.ctlz.i8(i8 [[CI]], i1 true)
+  ci = __builtin_elementwise_ctlz(ci);
+}
+
+void test_builtin_elementwise_cttz(si8 vs1, si8 vs2, u4 vu1,
+                                   long long int lli, short si,
+                                   _BitInt(31) bi, int i,
+                                   char ci) {
+  // CHECK:      [[V8S1:%.+]] = load <8 x i16>, ptr %vs1.addr
+  // CHECK-NEXT: call <8 x i16> @llvm.cttz.v8i16(<8 x i16> [[V8S1]], i1 true)
+  vs1 = __builtin_elementwise_cttz(vs1);
+
+  // CHECK:      [[V8S1:%.+]] = load <8 x i16>, ptr %vs1.addr
+  // CHECK-NEXT: [[ctz:%.+]] = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> [[V8S1]], i1 true)
+  // CHECK-NEXT: [[ISZERO:%.+]] = icmp eq <8 x i16> [[V8S1]], zeroinitializer
+  // CHECK-NEXT: [[V8S2:%.+]] = load <8 x i16>, ptr %vs2.addr
+  // select <8 x i1> [[ISZERO]], <8 x i16> [[ctz]], <8 x i16> [[V8S2]]
+  vs1 = __builtin_elementwise_cttz(vs1, vs2);
+
+  // CHECK:      [[V4U1:%.+]] = load <4 x i32>, ptr %vu1.addr
+  // CHECK-NEXT: call <4 x i32> @llvm.cttz.v4i32(<4 x i32> [[V4U1]], i1 true)
+  vu1 = __builtin_elementwise_cttz(vu1);
+
+  // CHECK:      [[LLI:%.+]] = load i64, ptr %lli.addr
+  // CHECK-NEXT: call i64 @llvm.cttz.i64(i64 [[LLI]], i1 true)
+  lli = __builtin_elementwise_cttz(lli);
+
+  // CHECK:      [[SI:%.+]] = load i16, ptr %si.addr
+  // CHECK-NEXT: call i16 @llvm.cttz.i16(i16 [[SI]], i1 true)
+  si = __builtin_elementwise_cttz(si);
+
+  // CHECK:      [[BI1:%.+]] = load i32, ptr %bi.addr
+  // CHECK-NEXT: [[BI2:%.+]] = trunc i32 [[BI1]] to i31
+  // CHECK-NEXT: call i31 @llvm.cttz.i31(i31 [[BI2]], i1 true)
+  bi = __builtin_elementwise_cttz(bi);
+
+  // CHECK:      [[BI1:%.+]] = load i32, ptr %bi.addr
+  // CHECK-NEXT: [[BI2:%.+]] = trunc i32 [[BI1]] to i31
+  // CHECK-NEXT: [[ctz:%.+]] = call i31 @llvm.cttz.i31(i31 [[BI2]], i1 true)
+  // CHECK-NEXT: [[ISZERO:%.+]] = icmp eq i31 [[BI2]], 0
+  // CHECK-NEXT: select i1 [[ISZERO]], i31 1, i31 [[ctz]]
+  bi = __builtin_elementwise_cttz(bi, (_BitInt(31))1);
+
+  // CHECK:      [[I:%.+]] = load i32, ptr %i.addr
+  // CHECK-NEXT: call i32 @llvm.cttz.i32(i32 [[I]], i1 true)
+  i = __builtin_elementwise_cttz(i);
+
+  // CHECK:      [[CI:%.+]] = load i8, ptr %ci.addr
+  // CHECK-NEXT: call i8 @llvm.cttz.i8(i8 [[CI]], i1 true)
+  ci = __builtin_elementwise_cttz(ci);
+}
diff --git a/clang/test/CodeGen/builtins-wasm.c b/clang/test/CodeGen/builtins-wasm.c
index f201dfe704e7e..375664b852636 100644
--- a/clang/test/CodeGen/builtins-wasm.c
+++ b/clang/test/CodeGen/builtins-wasm.c
@@ -751,24 +751,3 @@ void *tp (void) {
   return __builtin_thread_pointer ();
   // WEBASSEMBLY: call {{.*}} @llvm.thread.pointer.p0()
 }
-
-typedef void (*Fvoid)(void);
-typedef float (*Ffloats)(float, double, int);
-typedef void (*Fpointers)(Fvoid, Ffloats, void*, int*, int***, char[5]);
-
-void use(int);
-
-void test_function_pointer_signature_void(Fvoid func) {
-  // WEBASSEMBLY:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, token poison)
-  use(__builtin_wasm_test_function_pointer_signature(func));
-}
-
-void test_function_pointer_signature_floats(Ffloats func) {
-  // WEBASSEMBLY:  tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, float 0.000000e+00, token poison, float 0.000000e+00, double 0.000000e+00, i32 0)
-  use(__builtin_wasm_test_function_pointer_signature(func));
-}
-
-void test_function_pointer_signature_pointers(Fpointers func) {
-  // WEBASSEMBLY:  %0 = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, token poison, ptr null, ptr null, ptr null, ptr null, ptr null, ptr null)
-  use(__builtin_wasm_test_function_pointer_signature(func));
-}
diff --git a/clang/test/CodeGen/builtins-x86.c b/clang/test/CodeGen/builtins-x86.c
index c42c3216ec53c..31f309791c9f7 100644
--- a/clang/test/CodeGen/builtins-x86.c
+++ b/clang/test/CodeGen/builtins-x86.c
@@ -22,6 +22,7 @@ typedef float V2f __attribute__((vector_size(8)));
 // 128-bit
 typedef char V16c __attribute__((vector_size(16)));
 typedef signed short V8s __attribute__((vector_size(16)));
+typedef unsigned short V8u __attribute__((vector_size(16)));
 typedef signed int V4i __attribute__((vector_size(16)));
 #ifndef OPENCL
 typedef signed long long V2LLi __attribute__((vector_size(16)));
@@ -99,6 +100,7 @@ void f0(void) {
   // 128-bit
   V16c   tmp_V16c;
   V8s    tmp_V8s;
+  V8u    tmp_V8u;
   V4i    tmp_V4i;
   V2LLi  tmp_V2LLi;
   V4f    tmp_V4f;
@@ -192,7 +194,7 @@ void f0(void) {
   tmp_V16c = __builtin_ia32_packsswb128(tmp_V8s, tmp_V8s);
   tmp_V8s = __builtin_ia32_packssdw128(tmp_V4i, tmp_V4i);
   tmp_V16c = __builtin_ia32_packuswb128(tmp_V8s, tmp_V8s);
-  tmp_V8s = __builtin_ia32_pmulhuw128(tmp_V8s, tmp_V8s);
+  tmp_V8u = __builtin_ia32_pmulhuw128(tmp_V8u, tmp_V8u);
   tmp_V4f = __builtin_ia32_addsubps(tmp_V4f, tmp_V4f);
   tmp_V2d = __builtin_ia32_addsubpd(tmp_V2d, tmp_V2d);
   tmp_V4f = __builtin_ia32_haddps(tmp_V4f, tmp_V4f);
diff --git a/clang/test/CodeGen/c-strings.c b/clang/test/CodeGen/c-strings.c
index 988deee317d4c..31c438fd8ff2e 100644
--- a/clang/test/CodeGen/c-strings.c
+++ b/clang/test/CodeGen/c-strings.c
@@ -1,4 +1,3 @@
-// XFAIL: target=aarch64-pc-windows-{{.*}}
 // RUN: %clang_cc1 -triple %itanium_abi_triple -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK --check-prefix=ITANIUM
 // RUN: %clang_cc1 -triple %ms_abi_triple -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK --check-prefix=MSABI
 
@@ -16,6 +15,11 @@
 // MSABI: @f4.x = internal global %struct.s { ptr @"??_C@_05CJBACGMB@hello?$AA@" }
 // CHECK: @x = {{(dso_local )?}}global [3 x i8] c"ola", align [[ALIGN]]
 
+// XFAIL: target=aarch64-{{.*}}-windows-msvc, target=arm64ec-{{.*}}-windows-msvc
+// Arm64 in MSVC mode aligns arrays to either 32-bit or 64-bit boundaries, which fails
+// various checks above, since ALIGN is derived from the alignment of a single
+// i8, which is still 1.
+
 // XFAIL: target=hexagon-{{.*}}
 // Hexagon aligns arrays of size 8+ bytes to a 64-bit boundary, which
 // fails the check for "@f3.x = ... align [ALIGN]", since ALIGN is derived
diff --git a/clang/test/CodeGen/cfi-icall-generalize.c b/clang/test/CodeGen/cfi-icall-generalize.c
index 2011889fcb400..0af17e5760cc6 100644
--- a/clang/test/CodeGen/cfi-icall-generalize.c
+++ b/clang/test/CodeGen/cfi-icall-generalize.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -triple x86_64-unknown-linux -fsanitize=cfi-icall -fsanitize-trap=cfi-icall -emit-llvm -o - %s | FileCheck --check-prefix=CHECK --check-prefix=UNGENERALIZED %s
 // RUN: %clang_cc1 -triple x86_64-unknown-linux -fsanitize=cfi-icall -fsanitize-trap=cfi-icall -fsanitize-cfi-icall-generalize-pointers -emit-llvm -o - %s | FileCheck --check-prefix=CHECK --check-prefix=GENERALIZED %s
 
-// Test that const char* is generalized to const ptr and that const char** is
+// Test that const char* is generalized to const ptr and that char** is
 // generalized to ptr
 
 // CHECK: define{{.*}} ptr @f({{.*}} !type [[TYPE:![0-9]+]] !type [[TYPE_GENERALIZED:![0-9]+]]
diff --git a/clang/test/CodeGen/cfi-salt.c b/clang/test/CodeGen/cfi-salt.c
new file mode 100644
index 0000000000000..7ba1e2fc14daa
--- /dev/null
+++ b/clang/test/CodeGen/cfi-salt.c
@@ -0,0 +1,188 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -DORIG_ATTR_SYN -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -fpatchable-function-entry-offset=3 -DORIG_ATTR_SYN -o - %s | FileCheck %s --check-prefixes=CHECK,OFFSET
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -fpatchable-function-entry-offset=3 -o - %s | FileCheck %s --check-prefixes=CHECK,OFFSET
+
+// Note that the interleving of functions, which normally would be in sequence,
+// is due to the fact that Clang outputs them in a non-sequential order.
+
+#if !__has_feature(kcfi)
+#error Missing kcfi?
+#endif
+
+#ifdef ORIG_ATTR_SYN
+#define __cfi_salt       __attribute__((cfi_salt("pepper")))
+#define __cfi_salt_empty __attribute__((cfi_salt("")))
+#else
+#define __cfi_salt       [[clang::cfi_salt("pepper")]]
+#define __cfi_salt_empty [[clang::cfi_salt("")]]
+#endif
+
+typedef int (*fn_t)(void);
+typedef int (* __cfi_salt fn_salt_t)(void);
+typedef int (* __cfi_salt_empty fn_salt_empty_t)(void);
+
+typedef unsigned int (*ufn_t)(void);
+typedef unsigned int (* __cfi_salt ufn_salt_t)(void);
+
+/// Must emit __kcfi_typeid symbols for address-taken function declarations
+// CHECK: module asm ".weak __kcfi_typeid_[[F4:[a-zA-Z0-9_]+]]"
+// CHECK: module asm ".set __kcfi_typeid_[[F4]], [[#%d,LOW_SODIUM_HASH:]]"
+// CHECK: module asm ".weak __kcfi_typeid_[[F4_SALT:[a-zA-Z0-9_]+]]"
+// CHECK: module asm ".set __kcfi_typeid_[[F4_SALT]], [[#%d,ASM_SALTY_HASH:]]"
+
+/// Must not __kcfi_typeid symbols for non-address-taken declarations
+// CHECK-NOT: module asm ".weak __kcfi_typeid_f6"
+
+int f1(void);
+int f1_salt(void) __cfi_salt;
+
+unsigned int f2(void);
+unsigned int f2_salt(void) __cfi_salt;
+
+static int f3(void);
+static int f3_salt(void) __cfi_salt;
+
+extern int f4(void);
+extern int f4_salt(void) __cfi_salt;
+
+static int f5(void);
+static int f5_salt(void) __cfi_salt;
+
+extern int f6(void);
+extern int f6_salt(void) __cfi_salt;
+
+int f8(void);
+int f8_salt_empty(void) __cfi_salt_empty;
+
+struct cfi_struct {
+  fn_t __cfi_salt fptr;
+  fn_salt_t td_fptr;
+  fn_salt_empty_t td_empty_fptr;
+};
+
+int f7_salt(struct cfi_struct *ptr);
+int f7_typedef_salt(struct cfi_struct *ptr);
+
+// CHECK-LABEL: @__call
+// CHECK:         call{{.*}} i32
+// CHECK-NOT:     "kcfi"
+// CHECK-SAME:    ()
+__attribute__((__no_sanitize__("kcfi")))
+int __call(fn_t f) {
+  return f();
+}
+
+// CHECK-LABEL: @call
+// CHECK:         call{{.*}} i32 %{{.}}(){{.*}} [ "kcfi"(i32 [[#LOW_SODIUM_HASH]]) ]
+// CHECK-LABEL: @call_salt
+// CHECK:         call{{.*}} i32 %{{.}}(){{.*}} [ "kcfi"(i32 [[#%d,SALTY_HASH:]]) ]
+// CHECK-LABEL: @call_salt_ty
+// CHECK:         call{{.*}} i32 %{{.}}(){{.*}} [ "kcfi"(i32 [[#SALTY_HASH]]) ]
+int call(fn_t f) { return f(); }
+int call_salt(fn_t __cfi_salt f) { return f(); }
+int call_salt_ty(fn_salt_t f) { return f(); }
+int call_salt_empty_ty(fn_salt_empty_t f) { return f(); }
+
+// CHECK-LABEL: @ucall
+// CHECK:         call{{.*}} i32 %{{.}}(){{.*}} [ "kcfi"(i32 [[#%d,LOW_SODIUM_UHASH:]]) ]
+// CHECK-LABEL: @ucall_salt
+// CHECK:         call{{.*}} i32 %{{.}}(){{.*}} [ "kcfi"(i32 [[#%d,SALTY_UHASH:]]) ]
+// CHECK-LABEL: @ucall_salt_ty
+// CHECK:         call{{.*}} i32 %{{.}}(){{.*}} [ "kcfi"(i32 [[#SALTY_UHASH]]) ]
+unsigned int ucall(ufn_t f) { return f(); }
+unsigned int ucall_salt(ufn_t __cfi_salt f) { return f(); }
+unsigned int ucall_salt_ty(ufn_salt_t f) { return f(); }
+
+int test1(struct cfi_struct *ptr) {
+  return call(f1) +
+         call_salt(f1_salt) +
+         call_salt_ty(f1_salt) +
+
+         __call((fn_t)f2) +
+         __call((fn_t)f2_salt) +
+
+         ucall(f2) +
+         ucall_salt(f2_salt) +
+         ucall_salt_ty(f2_salt) +
+
+         call(f3) +
+         call_salt(f3_salt) +
+         call_salt_ty(f3_salt) +
+
+         call(f4) +
+         call_salt(f4_salt) +
+         call_salt_ty(f4_salt) +
+
+         f5() +
+         f5_salt() +
+
+         f6() +
+         f6_salt() +
+
+         f7_salt(ptr) +
+         f7_typedef_salt(ptr) +
+
+         f8() +
+         f8_salt_empty();
+}
+
+// CHECK-LABEL: define dso_local{{.*}} i32 @f1(){{.*}} !kcfi_type
+// CHECK-SAME:  ![[#LOW_SODIUM_TYPE:]]
+// CHECK-LABEL: define dso_local{{.*}} i32 @f1_salt(){{.*}} !kcfi_type
+// CHECK-SAME:  ![[#SALTY_TYPE:]]
+int f1(void) { return 0; }
+int f1_salt(void) __cfi_salt { return 0; }
+
+// CHECK-LABEL: define dso_local{{.*}} i32 @f2(){{.*}} !kcfi_type
+// CHECK-SAME:  ![[#LOW_SODIUM_UTYPE:]]
+// CHECK: define dso_local{{.*}} i32 @f2_salt(){{.*}} !kcfi_type
+// CHECK-SAME:  ![[#SALTY_UTYPE:]]
+unsigned int f2(void) { return 2; }
+unsigned int f2_salt(void) __cfi_salt { return 2; }
+
+// CHECK-LABEL: define internal{{.*}} i32 @f3(){{.*}} !kcfi_type
+// CHECK-SAME:  ![[#LOW_SODIUM_TYPE]]
+// CHECK-LABEL: define internal{{.*}} i32 @f3_salt(){{.*}} !kcfi_type
+// CHECK-SAME:  ![[#SALTY_TYPE]]
+static int f3(void) { return 1; }
+static int f3_salt(void) __cfi_salt { return 1; }
+
+// CHECK: declare !kcfi_type ![[#LOW_SODIUM_TYPE]]{{.*}} i32 @[[F4]]()
+// CHECK: declare !kcfi_type ![[#SALTY_TYPE]]{{.*}} i32 @[[F4_SALT]]()
+
+/// Must not emit !kcfi_type for non-address-taken local functions
+// CHECK-LABEL: define internal{{.*}} i32 @f5()
+// CHECK-NOT:   !kcfi_type
+// CHECK-SAME:  {
+// CHECK-LABEL: define internal{{.*}} i32 @f5_salt()
+// CHECK-NOT:   !kcfi_type
+// CHECK-SAME:  {
+static int f5(void) { return 2; }
+static int f5_salt(void) __cfi_salt { return 2; }
+
+// CHECK: declare !kcfi_type ![[#LOW_SODIUM_TYPE]]{{.*}} i32 @f6()
+// CHECK: declare !kcfi_type ![[#SALTY_TYPE]]{{.*}} i32 @f6_salt()
+
+// CHECK-LABEL: @f7_salt
+// CHECK:         call{{.*}} i32 %{{.*}}() [ "kcfi"(i32 [[#SALTY_HASH]]) ]
+// CHECK-LABEL: @f7_typedef_salt
+// CHECK:         call{{.*}} i32 %{{.*}}() [ "kcfi"(i32 [[#SALTY_HASH]]) ]
+int f7_salt(struct cfi_struct *ptr) { return ptr->fptr(); }
+int f7_typedef_salt(struct cfi_struct *ptr) { return ptr->td_fptr(); }
+
+// CHECK-LABEL: define dso_local{{.*}} i32 @f8(){{.*}} !kcfi_type
+// CHECK-SAME:  ![[#LOW_SODIUM_TYPE:]]
+// CHECK-LABEL: define dso_local{{.*}} i32 @f8_salt_empty(){{.*}} !kcfi_type
+// CHECK-SAME:  ![[#LOW_SODIUM_TYPE:]]
+int f8(void) { return 0; }
+int f8_salt_empty(void) __cfi_salt_empty { return 0; }
+
+// CHECK:  ![[#]] = !{i32 4, !"kcfi", i32 1}
+// OFFSET: ![[#]] = !{i32 4, !"kcfi-offset", i32 3}
+//
+// CHECK:  ![[#LOW_SODIUM_TYPE]] = !{i32 [[#LOW_SODIUM_HASH]]}
+// CHECK:  ![[#SALTY_TYPE]] = !{i32 [[#SALTY_HASH]]}
+//
+// CHECK:  ![[#LOW_SODIUM_UTYPE]] = !{i32 [[#LOW_SODIUM_UHASH]]}
+// CHECK:  ![[#SALTY_UTYPE]] = !{i32 [[#SALTY_UHASH]]}
diff --git a/clang/test/CodeGen/cleanup-destslot-simple.c b/clang/test/CodeGen/cleanup-destslot-simple.c
index a02841a5e2e2e..8ace33254723c 100644
--- a/clang/test/CodeGen/cleanup-destslot-simple.c
+++ b/clang/test/CodeGen/cleanup-destslot-simple.c
@@ -13,14 +13,14 @@
 // CHECK-LIFETIME-NEXT:  entry:
 // CHECK-LIFETIME-NEXT:    [[X:%.*]] = alloca i32, align 4
 // CHECK-LIFETIME-NEXT:    [[P:%.*]] = alloca ptr, align 8
-// CHECK-LIFETIME-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X]]) #[[ATTR2:[0-9]+]], !dbg [[DBG9:![0-9]+]]
+// CHECK-LIFETIME-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]]) #[[ATTR2:[0-9]+]], !dbg [[DBG9:![0-9]+]]
 // CHECK-LIFETIME-NEXT:    store i32 3, ptr [[X]], align 4, !dbg [[DBG10:![0-9]+]], !tbaa [[TBAA11:![0-9]+]]
-// CHECK-LIFETIME-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[P]]), !dbg [[DBG15:![0-9]+]]
+// CHECK-LIFETIME-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[P]]), !dbg [[DBG15:![0-9]+]]
 // CHECK-LIFETIME-NEXT:    store volatile ptr [[X]], ptr [[P]], align 8, !dbg [[DBG16:![0-9]+]], !tbaa [[TBAA17:![0-9]+]]
 // CHECK-LIFETIME-NEXT:    [[P_0_P_0_P_0_P_0_:%.*]] = load volatile ptr, ptr [[P]], align 8, !dbg [[DBG19:![0-9]+]], !tbaa [[TBAA17]]
 // CHECK-LIFETIME-NEXT:    [[TMP0:%.*]] = load i32, ptr [[P_0_P_0_P_0_P_0_]], align 4, !dbg [[DBG20:![0-9]+]], !tbaa [[TBAA11]]
-// CHECK-LIFETIME-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[P]]), !dbg [[DBG21:![0-9]+]]
-// CHECK-LIFETIME-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X]]) #[[ATTR2]], !dbg [[DBG21]]
+// CHECK-LIFETIME-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[P]]), !dbg [[DBG21:![0-9]+]]
+// CHECK-LIFETIME-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]]) #[[ATTR2]], !dbg [[DBG21]]
 // CHECK-LIFETIME-NEXT:    ret i32 [[TMP0]], !dbg [[DBG22:![0-9]+]]
 //
 // CHECK-OPTNONE-LABEL: @test(
@@ -37,13 +37,13 @@
 // CHECK-MSAN-NEXT:  entry:
 // CHECK-MSAN-NEXT:    [[X:%.*]] = alloca i32, align 4
 // CHECK-MSAN-NEXT:    [[P:%.*]] = alloca ptr, align 8
-// CHECK-MSAN-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X]]) #[[ATTR2:[0-9]+]], !dbg [[DBG9:![0-9]+]]
+// CHECK-MSAN-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]]) #[[ATTR2:[0-9]+]], !dbg [[DBG9:![0-9]+]]
 // CHECK-MSAN-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG9]]
 // CHECK-MSAN-NEXT:    [[TMP1:%.*]] = xor i64 [[TMP0]], 87960930222080, !dbg [[DBG9]]
 // CHECK-MSAN-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr, !dbg [[DBG9]]
 // CHECK-MSAN-NEXT:    store i32 0, ptr [[TMP2]], align 4, !dbg [[DBG10:![0-9]+]]
 // CHECK-MSAN-NEXT:    store i32 3, ptr [[X]], align 4, !dbg [[DBG10]], !tbaa [[TBAA11:![0-9]+]]
-// CHECK-MSAN-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[P]]), !dbg [[DBG15:![0-9]+]]
+// CHECK-MSAN-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[P]]), !dbg [[DBG15:![0-9]+]]
 // CHECK-MSAN-NEXT:    [[TMP3:%.*]] = ptrtoint ptr [[P]] to i64, !dbg [[DBG15]]
 // CHECK-MSAN-NEXT:    [[TMP4:%.*]] = xor i64 [[TMP3]], 87960930222080, !dbg [[DBG15]]
 // CHECK-MSAN-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr, !dbg [[DBG15]]
@@ -62,8 +62,8 @@
 // CHECK-MSAN-NEXT:    [[TMP10:%.*]] = xor i64 [[TMP9]], 87960930222080, !dbg [[DBG20]]
 // CHECK-MSAN-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr, !dbg [[DBG20]]
 // CHECK-MSAN-NEXT:    [[_MSLD1:%.*]] = load i32, ptr [[TMP11]], align 4, !dbg [[DBG20]]
-// CHECK-MSAN-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[P]]), !dbg [[DBG22:![0-9]+]]
-// CHECK-MSAN-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X]]) #[[ATTR2]], !dbg [[DBG22]]
+// CHECK-MSAN-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[P]]), !dbg [[DBG22:![0-9]+]]
+// CHECK-MSAN-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]]) #[[ATTR2]], !dbg [[DBG22]]
 // CHECK-MSAN-NEXT:    [[_MSCMP2_NOT:%.*]] = icmp eq i32 [[_MSLD1]], 0, !dbg [[DBG23:![0-9]+]]
 // CHECK-MSAN-NEXT:    br i1 [[_MSCMP2_NOT]], label [[TMP13:%.*]], label [[TMP12:%.*]], !dbg [[DBG23]], !prof [[PROF21]]
 // CHECK-MSAN:       12:
@@ -77,13 +77,13 @@
 // CHECK-KMSAN-NEXT:    [[TMP0:%.*]] = call ptr @__msan_get_context_state() #[[ATTR2:[0-9]+]]
 // CHECK-KMSAN-NEXT:    [[X:%.*]] = alloca i32, align 4
 // CHECK-KMSAN-NEXT:    [[P:%.*]] = alloca ptr, align 8
-// CHECK-KMSAN-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X]]) #[[ATTR2]], !dbg [[DBG9:![0-9]+]]
+// CHECK-KMSAN-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]]) #[[ATTR2]], !dbg [[DBG9:![0-9]+]]
 // CHECK-KMSAN-NEXT:    call void @__msan_poison_alloca(ptr nonnull [[X]], i64 4, ptr nonnull @[[GLOB0:[0-9]+]]) #[[ATTR2]], !dbg [[DBG9]]
 // CHECK-KMSAN-NEXT:    [[TMP1:%.*]] = call { ptr, ptr } @__msan_metadata_ptr_for_store_4(ptr nonnull [[X]]) #[[ATTR2]], !dbg [[DBG10:![0-9]+]]
 // CHECK-KMSAN-NEXT:    [[TMP2:%.*]] = extractvalue { ptr, ptr } [[TMP1]], 0, !dbg [[DBG10]]
 // CHECK-KMSAN-NEXT:    store i32 0, ptr [[TMP2]], align 4, !dbg [[DBG10]]
 // CHECK-KMSAN-NEXT:    store i32 3, ptr [[X]], align 4, !dbg [[DBG10]], !tbaa [[TBAA11:![0-9]+]]
-// CHECK-KMSAN-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[P]]), !dbg [[DBG15:![0-9]+]]
+// CHECK-KMSAN-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[P]]), !dbg [[DBG15:![0-9]+]]
 // CHECK-KMSAN-NEXT:    call void @__msan_poison_alloca(ptr nonnull [[P]], i64 8, ptr nonnull @[[GLOB1:[0-9]+]]) #[[ATTR2]], !dbg [[DBG15]]
 // CHECK-KMSAN-NEXT:    [[TMP3:%.*]] = call { ptr, ptr } @__msan_metadata_ptr_for_store_8(ptr nonnull [[P]]) #[[ATTR2]], !dbg [[DBG16:![0-9]+]]
 // CHECK-KMSAN-NEXT:    [[TMP4:%.*]] = extractvalue { ptr, ptr } [[TMP3]], 0, !dbg [[DBG16]]
@@ -109,8 +109,8 @@
 // CHECK-KMSAN-NEXT:    [[TMP14:%.*]] = extractvalue { ptr, ptr } [[TMP12]], 1, !dbg [[DBG20]]
 // CHECK-KMSAN-NEXT:    [[_MSLD1:%.*]] = load i32, ptr [[TMP13]], align 4, !dbg [[DBG20]]
 // CHECK-KMSAN-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4, !dbg [[DBG20]]
-// CHECK-KMSAN-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[P]]), !dbg [[DBG22:![0-9]+]]
-// CHECK-KMSAN-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X]]) #[[ATTR2]], !dbg [[DBG22]]
+// CHECK-KMSAN-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[P]]), !dbg [[DBG22:![0-9]+]]
+// CHECK-KMSAN-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]]) #[[ATTR2]], !dbg [[DBG22]]
 // CHECK-KMSAN-NEXT:    store i32 [[_MSLD1]], ptr [[RETVAL_SHADOW]], align 8, !dbg [[DBG23:![0-9]+]]
 // CHECK-KMSAN-NEXT:    store i32 [[TMP15]], ptr [[RETVAL_ORIGIN]], align 4, !dbg [[DBG23]]
 // CHECK-KMSAN-NEXT:    ret i32 [[TMP11]], !dbg [[DBG23]]
diff --git a/clang/test/CodeGen/dominating-store-to-return.c b/clang/test/CodeGen/dominating-store-to-return.c
index 1c53e358cf146..d095f3ffa0019 100644
--- a/clang/test/CodeGen/dominating-store-to-return.c
+++ b/clang/test/CodeGen/dominating-store-to-return.c
@@ -16,10 +16,10 @@
 // LIFETIME-NEXT:    [[FOO:%.*]] = alloca i32, align 4
 // LIFETIME-NEXT:    [[FOO2:%.*]] = alloca i32, align 4
 // LIFETIME-NEXT:    store i32 0, ptr [[RETVAL]], align 4
-// LIFETIME-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[FOO]]) #[[ATTR2:[0-9]+]]
-// LIFETIME-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[FOO2]]) #[[ATTR2]]
-// LIFETIME-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[FOO2]]) #[[ATTR2]]
-// LIFETIME-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[FOO]]) #[[ATTR2]]
+// LIFETIME-NEXT:    call void @llvm.lifetime.start.p0(ptr [[FOO]]) #[[ATTR2:[0-9]+]]
+// LIFETIME-NEXT:    call void @llvm.lifetime.start.p0(ptr [[FOO2]]) #[[ATTR2]]
+// LIFETIME-NEXT:    call void @llvm.lifetime.end.p0(ptr [[FOO2]]) #[[ATTR2]]
+// LIFETIME-NEXT:    call void @llvm.lifetime.end.p0(ptr [[FOO]]) #[[ATTR2]]
 // LIFETIME-NEXT:    ret i32 0
 //
 int main() {
diff --git a/clang/test/CodeGen/kcfi-generalize.c b/clang/test/CodeGen/kcfi-generalize.c
new file mode 100644
index 0000000000000..4e32f4f35057c
--- /dev/null
+++ b/clang/test/CodeGen/kcfi-generalize.c
@@ -0,0 +1,33 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux -fsanitize=kcfi -fsanitize-trap=kcfi -emit-llvm -o - %s | FileCheck --check-prefix=CHECK --check-prefix=UNGENERALIZED %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux -fsanitize=kcfi -fsanitize-trap=kcfi -fsanitize-cfi-icall-generalize-pointers -emit-llvm -o - %s | FileCheck --check-prefix=CHECK --check-prefix=GENERALIZED %s
+
+// Test that const char* is generalized to const ptr and that char** is
+// generalized to ptr
+
+// CHECK: define{{.*}} ptr @f({{.*}} !kcfi_type [[TYPE:![0-9]+]]
+int** f(const char *a, const char **b) {
+  return (int**)0;
+}
+
+// GENERALIZED: define{{.*}} ptr @f2({{.*}} !kcfi_type [[TYPE]]
+// UNGENERALIZED: define{{.*}} ptr @f2({{.*}} !kcfi_type [[TYPE2:![0-9]+]]
+int** f2(const int *a, const int **b) {
+  return (int**)0;
+}
+
+// CHECK: define{{.*}} ptr @f3({{.*}} !kcfi_type [[TYPE3:![0-9]+]]
+int** f3(char *a, char **b) {
+  return (int**)0;
+}
+
+void g(int** (*fp)(const char *, const char **)) {
+  // UNGENERALIZED: call {{.*}} [ "kcfi"(i32 1296635908) ]
+  // GENERALIZED: call {{.*}} [ "kcfi"(i32 -49168686) ]
+  fp(0, 0);
+}
+
+// UNGENERALIZED: [[TYPE]] = !{i32 1296635908}
+// GENERALIZED: [[TYPE]] = !{i32 -49168686}
+
+// UNGENERALIZED: [[TYPE3]] = !{i32 874141567}
+// GENERALIZED: [[TYPE3]] = !{i32 954385378}
diff --git a/clang/test/CodeGen/lifetime-sanitizer.c b/clang/test/CodeGen/lifetime-sanitizer.c
index 68879fda1e1a5..4d36bd7a9fee5 100644
--- a/clang/test/CodeGen/lifetime-sanitizer.c
+++ b/clang/test/CodeGen/lifetime-sanitizer.c
@@ -18,14 +18,14 @@ extern int bar(char *A, int n);
 // CHECK-O0-NOT: @llvm.lifetime.start
 int foo(int n) {
   if (n) {
-    // LIFETIME: @llvm.lifetime.start.p0(i64 10, ptr {{.*}})
+    // LIFETIME: @llvm.lifetime.start.p0(ptr {{.*}})
     char A[10];
     return bar(A, 1);
-    // LIFETIME: @llvm.lifetime.end.p0(i64 10, ptr {{.*}})
+    // LIFETIME: @llvm.lifetime.end.p0(ptr {{.*}})
   } else {
-    // LIFETIME: @llvm.lifetime.start.p0(i64 20, ptr {{.*}})
+    // LIFETIME: @llvm.lifetime.start.p0(ptr {{.*}})
     char A[20];
     return bar(A, 2);
-    // LIFETIME: @llvm.lifetime.end.p0(i64 20, ptr {{.*}})
+    // LIFETIME: @llvm.lifetime.end.p0(ptr {{.*}})
   }
 }
diff --git a/clang/test/CodeGen/lifetime.c b/clang/test/CodeGen/lifetime.c
index 748a496a6e9ea..edb6867465d99 100644
--- a/clang/test/CodeGen/lifetime.c
+++ b/clang/test/CodeGen/lifetime.c
@@ -8,14 +8,14 @@ extern void use(char *a);
 // CHECK-LABEL: @helper_no_markers
 __attribute__((always_inline)) void helper_no_markers(void) {
   char a;
-  // LIFETIME: call void @llvm.lifetime.start.p0(i64 1,
+  // LIFETIME: call void @llvm.lifetime.start.p0(
   use(&a);
-  // LIFETIME: call void @llvm.lifetime.end.p0(i64 1,
+  // LIFETIME: call void @llvm.lifetime.end.p0(
 }
 
 // CHECK-LABEL: @lifetime_test
 void lifetime_test(void) {
-// LIFETIME: call void @llvm.lifetime.start.p0(i64 1,
+// LIFETIME: call void @llvm.lifetime.start.p0(
   helper_no_markers();
-// LIFETIME: call void @llvm.lifetime.end.p0(i64 1,
+// LIFETIME: call void @llvm.lifetime.end.p0(
 }
diff --git a/clang/test/CodeGen/lifetime2.c b/clang/test/CodeGen/lifetime2.c
index 88c35fca029da..560389f912510 100644
--- a/clang/test/CodeGen/lifetime2.c
+++ b/clang/test/CodeGen/lifetime2.c
@@ -7,21 +7,21 @@ extern int bar(char *A, int n);
 // CHECK-LABEL: @foo
 int foo (int n) {
   if (n) {
-// O2: call void @llvm.lifetime.start.p0(i64 100,
+// O2: call void @llvm.lifetime.start.p0(
     char A[100];
     return bar(A, 1);
-// O2: call void @llvm.lifetime.end.p0(i64 100,
+// O2: call void @llvm.lifetime.end.p0(
   } else {
-// O2: call void @llvm.lifetime.start.p0(i64 100,
+// O2: call void @llvm.lifetime.start.p0(
     char A[100];
     return bar(A, 2);
-// O2: call void @llvm.lifetime.end.p0(i64 100,
+// O2: call void @llvm.lifetime.end.p0(
   }
 }
 
 // CHECK-LABEL: @no_goto_bypass
 void no_goto_bypass(void) {
-  // O2: call void @llvm.lifetime.start.p0(i64 1,
+  // O2: call void @llvm.lifetime.start.p0(
   char x;
 l1:
   bar(&x, 1);
@@ -45,16 +45,16 @@ void goto_bypass(void) {
 void no_switch_bypass(int n) {
   switch (n) {
   case 1: {
-    // O2: call void @llvm.lifetime.start.p0(i64 1,
-    // O2: call void @llvm.lifetime.end.p0(i64 1,
+    // O2: call void @llvm.lifetime.start.p0(
+    // O2: call void @llvm.lifetime.end.p0(
     char x;
     bar(&x, 1);
     break;
   }
   case 2:
     n = n;
-    // O2: call void @llvm.lifetime.start.p0(i64 5,
-    // O2: call void @llvm.lifetime.end.p0(i64 5,
+    // O2: call void @llvm.lifetime.start.p0(
+    // O2: call void @llvm.lifetime.end.p0(
     char y[5];
     bar(y, 5);
     break;
diff --git a/clang/test/CodeGen/lifetime3.cpp b/clang/test/CodeGen/lifetime3.cpp
index 64a097cfb859d..476ca0d2cde3a 100644
--- a/clang/test/CodeGen/lifetime3.cpp
+++ b/clang/test/CodeGen/lifetime3.cpp
@@ -6,30 +6,30 @@ extern int bar(char *A, int n);
 
 // CHECK-LABEL: @no_switch_bypass
 extern "C" void no_switch_bypass(int n) {
-  // O2: call void @llvm.lifetime.start.p0(i64 4,
+  // O2: call void @llvm.lifetime.start.p0(
   switch (n += 1; int b=n) {
   case 1: {
-    // O2: call void @llvm.lifetime.start.p0(i64 1,
-    // O2: call void @llvm.lifetime.end.p0(i64 1,
+    // O2: call void @llvm.lifetime.start.p0(
+    // O2: call void @llvm.lifetime.end.p0(
     char x;
     bar(&x, 1);
     break;
   }
   case 2:
     n = n;
-    // O2: call void @llvm.lifetime.start.p0(i64 5,
-    // O2: call void @llvm.lifetime.end.p0(i64 5,
+    // O2: call void @llvm.lifetime.start.p0(
+    // O2: call void @llvm.lifetime.end.p0(
     char y[5];
     bar(y, 5);
     break;
   }
-  // O2: call void @llvm.lifetime.end.p0(i64 4,
+  // O2: call void @llvm.lifetime.end.p0(
 }
 
 // CHECK-LABEL: @switch_bypass
 extern "C" void switch_bypass(int n) {
-  // O2: call void @llvm.lifetime.start.p0(i64 4,
-  // O2: call void @llvm.lifetime.end.p0(i64 4,
+  // O2: call void @llvm.lifetime.start.p0(
+  // O2: call void @llvm.lifetime.end.p0(
   switch (n += 1; int b=n) {
   case 1:
     n = n;
diff --git a/clang/test/CodeGen/math-libcalls-tbaa.c b/clang/test/CodeGen/math-libcalls-tbaa.c
index f4e81ea6dff17..b2f502e5b4729 100644
--- a/clang/test/CodeGen/math-libcalls-tbaa.c
+++ b/clang/test/CodeGen/math-libcalls-tbaa.c
@@ -82,12 +82,12 @@ double test_remainder (double num[], double a) {
 // CHECK-SAME: ptr noundef readonly captures(none) [[NUM:%.*]]) local_unnamed_addr #[[ATTR5:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[E:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[E]]) #[[ATTR9]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[E]]) #[[ATTR9]]
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 16
 // CHECK-NEXT:    [[TMP0:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA8]]
 // CHECK-NEXT:    [[CALL:%.*]] = call double @frexp(double noundef [[TMP0]], ptr noundef nonnull [[E]]) #[[ATTR9]]
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP0]], [[CALL]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[E]]) #[[ATTR9]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[E]]) #[[ATTR9]]
 // CHECK-NEXT:    ret double [[MUL]]
 //
 double test_frexp (double num[]) {
@@ -105,8 +105,8 @@ double test_frexp (double num[]) {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[SIN:%.*]] = alloca float, align 4
 // CHECK-NEXT:    [[COS:%.*]] = alloca float, align 4
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[SIN]]) #[[ATTR9]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[COS]]) #[[ATTR9]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[SIN]]) #[[ATTR9]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[COS]]) #[[ATTR9]]
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[NUM]], i64 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    call void @sincos(float noundef [[TMP0]], ptr noundef nonnull [[SIN]], ptr noundef nonnull [[COS]]) #[[ATTR9]]
@@ -115,8 +115,8 @@ double test_frexp (double num[]) {
 // CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP1]], [[TMP2]]
 // CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    [[ADD:%.*]] = fadd float [[MUL]], [[TMP3]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[COS]]) #[[ATTR9]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[SIN]]) #[[ATTR9]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[COS]]) #[[ATTR9]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[SIN]]) #[[ATTR9]]
 // CHECK-NEXT:    ret float [[ADD]]
 //
 float test_sincos (float num[]) {
diff --git a/clang/test/CodeGen/nofpclass.c b/clang/test/CodeGen/nofpclass.c
index 8e61ae22ef87f..6cd7ec12a2aea 100644
--- a/clang/test/CodeGen/nofpclass.c
+++ b/clang/test/CodeGen/nofpclass.c
@@ -927,14 +927,14 @@ _Complex _Float16 defined_complex_func_f16_ret(_Complex _Float16 c) {
 // CLFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16]], align 8
 // CLFINITEONLY-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[CF16]], i64 2
 // CLFINITEONLY-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
-// CLFINITEONLY-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12:[0-9]+]]
+// CLFINITEONLY-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12:[0-9]+]]
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[BYVAL_TEMP]], i64 8
 // CLFINITEONLY-NEXT:    store double [[CF64_COERCE0]], ptr [[BYVAL_TEMP]], align 8
 // CLFINITEONLY-NEXT:    store double [[CF64_COERCE1]], ptr [[BYVAL_TEMP_IMAGP]], align 8
 // CLFINITEONLY-NEXT:    [[COERCE5_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> poison, half [[CF16_REAL]], i64 0
 // CLFINITEONLY-NEXT:    [[COERCE5_SROA_0_2_VEC_INSERT:%.*]] = insertelement <2 x half> [[COERCE5_SROA_0_0_VEC_INSERT]], half [[CF16_IMAG]], i64 1
 // CLFINITEONLY-NEXT:    [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) float (float, ...) @variadic(float noundef nofpclass(nan inf) [[F32]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[F64]], half noundef nofpclass(nan inf) [[F16]], double noundef nofpclass(nan inf) [[V2F32_COERCE]], <2 x double> noundef nofpclass(nan inf) [[V2F64]], i32 noundef [[V2F16_COERCE]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE]], ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(nan inf) [[COERCE5_SROA_0_2_VEC_INSERT]]) #[[ATTR11]]
-// CLFINITEONLY-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12]]
+// CLFINITEONLY-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12]]
 // CLFINITEONLY-NEXT:    ret float [[CALL]]
 //
 // NONANS: Function Attrs: noinline nounwind optnone
@@ -1180,14 +1180,14 @@ float call_variadic(float f32, double f64, _Float16 f16,
 // CLFINITEONLY-NEXT:    [[CF16_REAL:%.*]] = load half, ptr [[CF16]], align 8
 // CLFINITEONLY-NEXT:    [[CF16_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[CF16]], i64 2
 // CLFINITEONLY-NEXT:    [[CF16_IMAG:%.*]] = load half, ptr [[CF16_IMAGP]], align 2
-// CLFINITEONLY-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12]]
+// CLFINITEONLY-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12]]
 // CLFINITEONLY-NEXT:    [[BYVAL_TEMP_IMAGP:%.*]] = getelementptr inbounds nuw i8, ptr [[BYVAL_TEMP]], i64 8
 // CLFINITEONLY-NEXT:    store double [[CF64_COERCE0]], ptr [[BYVAL_TEMP]], align 8
 // CLFINITEONLY-NEXT:    store double [[CF64_COERCE1]], ptr [[BYVAL_TEMP_IMAGP]], align 8
 // CLFINITEONLY-NEXT:    [[COERCE5_SROA_0_0_VEC_INSERT:%.*]] = insertelement <2 x half> poison, half [[CF16_REAL]], i64 0
 // CLFINITEONLY-NEXT:    [[COERCE5_SROA_0_2_VEC_INSERT:%.*]] = insertelement <2 x half> [[COERCE5_SROA_0_0_VEC_INSERT]], half [[CF16_IMAG]], i64 1
 // CLFINITEONLY-NEXT:    [[CALL:%.*]] = tail call nnan ninf nofpclass(nan inf) float (float, ...) [[FPTR]](float noundef nofpclass(nan inf) [[F32]], double noundef nofpclass(nan inf) [[CONV]], double noundef nofpclass(nan inf) [[F64]], half noundef nofpclass(nan inf) [[F16]], double noundef nofpclass(nan inf) [[V2F32_COERCE]], <2 x double> noundef nofpclass(nan inf) [[V2F64]], i32 noundef [[V2F16_COERCE]], <2 x float> noundef nofpclass(nan inf) [[CF32_COERCE]], ptr noundef nonnull byval({ double, double }) align 8 [[BYVAL_TEMP]], <2 x half> noundef nofpclass(nan inf) [[COERCE5_SROA_0_2_VEC_INSERT]]) #[[ATTR11]]
-// CLFINITEONLY-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12]]
+// CLFINITEONLY-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BYVAL_TEMP]]) #[[ATTR12]]
 // CLFINITEONLY-NEXT:    ret float [[CALL]]
 //
 // NONANS: Function Attrs: noinline nounwind optnone
diff --git a/clang/test/CodeGen/object-size.cpp b/clang/test/CodeGen/object-size.cpp
index e6ae3aefac2f8..39c0f35e79e22 100644
--- a/clang/test/CodeGen/object-size.cpp
+++ b/clang/test/CodeGen/object-size.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-llvm -o - %s -fexperimental-new-constant-interpreter | FileCheck %s
 
 // C++-specific tests for __builtin_object_size
 
diff --git a/clang/test/CodeGen/packed-arrays.c b/clang/test/CodeGen/packed-arrays.c
index 097fa7fc0feb7..51629b66d0687 100644
--- a/clang/test/CodeGen/packed-arrays.c
+++ b/clang/test/CodeGen/packed-arrays.c
@@ -55,7 +55,7 @@ int align3_x0 = __alignof(((struct s3*) 0)->x[0]);
 // CHECK:   load i32, ptr %{{.*}}, align 1
 // CHECK: }
 // CHECK-LABEL: define{{.*}} i32 @f0_b
-// CHECK:   load i32, ptr %{{.*}}, align 4
+// CHECK:   load i32, ptr %{{.*}}, align 1
 // CHECK: }
 int f0_a(struct s0 *a) {
   return a->x[1];
@@ -100,7 +100,7 @@ int f1_d(struct s1 *a) {
 // CHECK:   load i32, ptr %{{.*}}, align 1
 // CHECK: }
 // CHECK-LABEL: define{{.*}} i32 @f2_b
-// CHECK:   load i32, ptr %{{.*}}, align 4
+// CHECK:   load i32, ptr %{{.*}}, align 1
 // CHECK: }
 // CHECK-LABEL: define{{.*}} i32 @f2_c
 // CHECK:   load i32, ptr %{{.*}}, align 1
@@ -125,7 +125,7 @@ int f2_d(struct s2 *a) {
 // CHECK:   load i32, ptr %{{.*}}, align 1
 // CHECK: }
 // CHECK-LABEL: define{{.*}} i32 @f3_b
-// CHECK:   load i32, ptr %{{.*}}, align 4
+// CHECK:   load i32, ptr %{{.*}}, align 1
 // CHECK: }
 // CHECK-LABEL: define{{.*}} i32 @f3_c
 // CHECK:   load i32, ptr %{{.*}}, align 1
diff --git a/clang/test/CodeGen/palignr.c b/clang/test/CodeGen/palignr.c
index 092937ac115de..9b80cf992cd1b 100644
--- a/clang/test/CodeGen/palignr.c
+++ b/clang/test/CodeGen/palignr.c
@@ -1,16 +1,16 @@
 // REQUIRES: x86-registered-target
-// RUN: %clang_cc1 %s -triple=i686-apple-darwin -target-feature +ssse3 -O1 -S -o - | FileCheck %s
+// RUN: %clang_cc1 %s -triple=i686-apple-darwin -target-feature +ssse3 -O1 -S -flax-vector-conversions=none -o - | FileCheck %s
 
 #define _mm_alignr_epi8(a, b, n) (__builtin_ia32_palignr128((a), (b), (n)))
-typedef __attribute__((vector_size(16))) int int4;
+typedef char __v16qi __attribute__((__vector_size__(16)));
 
 // CHECK: palignr $15, %xmm1, %xmm0
-int4 align1(int4 a, int4 b) { return _mm_alignr_epi8(a, b, 15); }
+__v16qi align1(__v16qi a, __v16qi b) { return _mm_alignr_epi8(a, b, 15); }
 // CHECK: ret
 // CHECK: ret
 // CHECK-NOT: palignr
-int4 align2(int4 a, int4 b) { return _mm_alignr_epi8(a, b, 16); }
+__v16qi align2(__v16qi a, __v16qi b) { return _mm_alignr_epi8(a, b, 16); }
 // CHECK: psrldq $1, %xmm0
-int4 align3(int4 a, int4 b) { return _mm_alignr_epi8(a, b, 17); }
+__v16qi align3(__v16qi a, __v16qi b) { return _mm_alignr_epi8(a, b, 17); }
 // CHECK: xor
-int4 align4(int4 a, int4 b) { return _mm_alignr_epi8(a, b, 32); }
+__v16qi align4(__v16qi a, __v16qi b) { return _mm_alignr_epi8(a, b, 32); }
diff --git a/clang/test/CodeGen/pointer-arithmetic-align.c b/clang/test/CodeGen/pointer-arithmetic-align.c
new file mode 100644
index 0000000000000..745ab84635c1b
--- /dev/null
+++ b/clang/test/CodeGen/pointer-arithmetic-align.c
@@ -0,0 +1,83 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
+// RUN: %clang_cc1 -O1 -triple=x86_64-unknown-linux %s -emit-llvm -o - | FileCheck %s
+
+typedef unsigned char uint8_t;
+typedef unsigned long long uint64_t;
+
+struct a {
+  uint64_t b;
+  uint8_t block[16];
+};
+
+// CHECK-LABEL: define dso_local void @ptradd_0(
+// CHECK-SAME: ptr noundef writeonly captures(none) initializes((8, 9)) [[CTX:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[BLOCK:%.*]] = getelementptr inbounds nuw i8, ptr [[CTX]], i64 8
+// CHECK-NEXT:    store i8 0, ptr [[BLOCK]], align 8, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    ret void
+//
+void ptradd_0(struct a *ctx) {
+  *(ctx->block + 0) = 0;
+}
+
+// CHECK-LABEL: define dso_local void @ptradd_4(
+// CHECK-SAME: ptr noundef writeonly captures(none) initializes((12, 13)) [[CTX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[CTX]], i64 12
+// CHECK-NEXT:    store i8 0, ptr [[ADD_PTR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+void ptradd_4(struct a *ctx) {
+  *(ctx->block + 4) = 0;
+}
+
+// CHECK-LABEL: define dso_local void @ptradd_8(
+// CHECK-SAME: ptr noundef writeonly captures(none) initializes((16, 17)) [[CTX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[CTX]], i64 16
+// CHECK-NEXT:    store i8 0, ptr [[ADD_PTR]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+void ptradd_8(struct a *ctx) {
+  *(ctx->block + 8) = 0;
+}
+
+// CHECK-LABEL: define dso_local void @ptradd_8_commuted(
+// CHECK-SAME: ptr noundef writeonly captures(none) initializes((16, 17)) [[CTX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[CTX]], i64 16
+// CHECK-NEXT:    store i8 0, ptr [[ADD_PTR]], align 8, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+void ptradd_8_commuted(struct a *ctx) {
+  *(8 + ctx->block) = 0;
+}
+
+// CHECK-LABEL: define dso_local void @ptrsub_4(
+// CHECK-SAME: ptr noundef writeonly captures(none) initializes((8, 9)) [[CTX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[CTX]], i64 8
+// CHECK-NEXT:    store i8 0, ptr [[ADD_PTR]], align 4, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+void ptrsub_4(struct a *ctx) {
+  *(&ctx->block[4] - 4) = 0;
+}
+
+// CHECK-LABEL: define dso_local void @neg_ptradd_var_index(
+// CHECK-SAME: ptr noundef writeonly captures(none) [[CTX:%.*]], i8 noundef zeroext [[IDX:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[BLOCK:%.*]] = getelementptr inbounds nuw i8, ptr [[CTX]], i64 8
+// CHECK-NEXT:    [[IDX_EXT:%.*]] = zext i8 [[IDX]] to i64
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BLOCK]], i64 [[IDX_EXT]]
+// CHECK-NEXT:    store i8 0, ptr [[ADD_PTR]], align 1, !tbaa [[TBAA2]]
+// CHECK-NEXT:    ret void
+//
+void neg_ptradd_var_index(struct a *ctx, uint8_t idx) {
+  *(ctx->block + idx) = 0;
+}
+//.
+// CHECK: [[TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0}
+// CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+// CHECK: [[META4]] = !{!"Simple C/C++ TBAA"}
+//.
diff --git a/clang/test/CodeGen/ptrauth-qualifier-blocks.c b/clang/test/CodeGen/ptrauth-qualifier-blocks.c
index 62da59cf327f6..f460da205cac7 100644
--- a/clang/test/CodeGen/ptrauth-qualifier-blocks.c
+++ b/clang/test/CodeGen/ptrauth-qualifier-blocks.c
@@ -82,9 +82,15 @@ void test_block_address_byref_capture() {
   // CHECK: store i32 33554432,
   // CHECK: store i32 48,
   // CHECK: [[COPY_HELPER_FIELD:%.*]] = getelementptr inbounds nuw [[BYREF_T]], ptr [[BYREF]], i32 0, i32 4
-  // CHECK: store ptr @__Block_byref_object_copy_, ptr [[COPY_HELPER_FIELD]], align
+  // CHECK: [[T0:%.*]] = ptrtoint ptr [[COPY_HELPER_FIELD]] to i64
+  // CHECK: [[T1:%.*]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr @__Block_byref_object_copy_ to i64), i32 0, i64 [[T0]])
+  // CHECK: [[T2:%.*]] = inttoptr i64 [[T1]] to ptr
+  // CHECK: store ptr [[T2]], ptr [[COPY_HELPER_FIELD]], align
   // CHECK: [[DISPOSE_HELPER_FIELD:%.*]] = getelementptr inbounds nuw [[BYREF_T]], ptr [[BYREF]], i32 0, i32 5
-  // CHECK: store ptr @__Block_byref_object_dispose_, ptr [[DISPOSE_HELPER_FIELD]], align
+  // CHECK: [[T0:%.*]] = ptrtoint ptr [[DISPOSE_HELPER_FIELD]] to i64
+  // CHECK: [[T1:%.*]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr @__Block_byref_object_dispose_ to i64), i32 0, i64 [[T0]])
+  // CHECK: [[T2:%.*]] = inttoptr i64 [[T1]] to ptr
+  // CHECK: store ptr [[T2]], ptr [[DISPOSE_HELPER_FIELD]], align
   //   flags - copy/dispose required
   // CHECK: store i32 1107296256, ptr
   __block struct A * __ptrauth(1, 1, 60) ptr = createA();
diff --git a/clang/test/CodeGen/rounding-math.cpp b/clang/test/CodeGen/rounding-math.cpp
index 264031dc9daa9..5c44fd31242c6 100644
--- a/clang/test/CodeGen/rounding-math.cpp
+++ b/clang/test/CodeGen/rounding-math.cpp
@@ -11,3 +11,55 @@ float V3 = func_01(1.0F, 2.0F);
 // CHECK: @V1 = {{.*}}global float 1.000000e+00, align 4
 // CHECK: @V2 = {{.*}}global float 1.000000e+00, align 4
 // CHECK: @V3 = {{.*}}global float 3.000000e+00, align 4
+
+void test_builtin_elementwise_fma_round_upward() {
+  #pragma STDC FENV_ACCESS ON
+  #pragma STDC FENV_ROUND FE_UPWARD
+
+  // CHECK: store float 0x4018000100000000, ptr %f1
+  // CHECK: store float 0x4018000100000000, ptr %f2
+  constexpr float f1 = __builtin_elementwise_fma(2.0F, 3.000001F, 0.000001F);
+  constexpr float f2 = 2.0F * 3.000001F + 0.000001F;
+  static_assert(f1 == f2);
+  static_assert(f1 == 6.00000381F);
+  // CHECK: store double 0x40180000C9539B89, ptr %d1
+  // CHECK: store double 0x40180000C9539B89, ptr %d2
+  constexpr double d1 = __builtin_elementwise_fma(2.0, 3.000001, 0.000001);
+  constexpr double d2 = 2.0 * 3.000001 + 0.000001;
+  static_assert(d1 == d2);
+  static_assert(d1 == 6.0000030000000004);
+}
+
+void test_builtin_elementwise_fma_round_downward() {
+  #pragma STDC FENV_ACCESS ON
+  #pragma STDC FENV_ROUND FE_DOWNWARD
+
+  // CHECK: store float 0x40180000C0000000, ptr %f3
+  // CHECK: store float 0x40180000C0000000, ptr %f4
+  constexpr float f3 = __builtin_elementwise_fma(2.0F, 3.000001F, 0.000001F);
+  constexpr float f4 = 2.0F * 3.000001F + 0.000001F;
+  static_assert(f3 == f4);
+  // CHECK: store double 0x40180000C9539B87, ptr %d3
+  // CHECK: store double 0x40180000C9539B87, ptr %d4
+  constexpr double d3 = __builtin_elementwise_fma(2.0, 3.000001, 0.000001);
+  constexpr double d4 = 2.0 * 3.000001 + 0.000001;
+  static_assert(d3 == d4);
+}
+
+void test_builtin_elementwise_fma_round_nearest() {
+  #pragma STDC FENV_ACCESS ON
+  #pragma STDC FENV_ROUND FE_TONEAREST
+
+  // CHECK: store float 0x40180000C0000000, ptr %f5
+  // CHECK: store float 0x40180000C0000000, ptr %f6
+  constexpr float f5 = __builtin_elementwise_fma(2.0F, 3.000001F, 0.000001F);
+  constexpr float f6 = 2.0F * 3.000001F + 0.000001F;
+  static_assert(f5 == f6);
+  static_assert(f5 == 6.00000286F);
+  // CHECK: store double 0x40180000C9539B89, ptr %d5
+  // CHECK: store double 0x40180000C9539B89, ptr %d6
+  constexpr double d5 = __builtin_elementwise_fma(2.0, 3.000001, 0.000001);
+  constexpr double d6 = 2.0 * 3.000001 + 0.000001;
+  static_assert(d5 == d6);
+  static_assert(d5 == 6.0000030000000004);
+}
diff --git a/clang/test/CodeGen/target-builtin-noerror.c b/clang/test/CodeGen/target-builtin-noerror.c
index 0bbd8c3e5ddd8..5cf53b2d2fc8f 100644
--- a/clang/test/CodeGen/target-builtin-noerror.c
+++ b/clang/test/CodeGen/target-builtin-noerror.c
@@ -32,15 +32,15 @@ int qq(void) {
 
 // Test that fma and fma4 are both separately and combined valid for an fma intrinsic.
 __m128 __attribute__((target("fma"))) fma_1(__m128 a, __m128 b, __m128 c) {
-  return __builtin_ia32_vfmaddps(a, b, c);
+  return __builtin_ia32_vfmaddsubps(a, b, c);
 }
 
 __m128 __attribute__((target("fma4"))) fma_2(__m128 a, __m128 b, __m128 c) {
-  return __builtin_ia32_vfmaddps(a, b, c);
+  return __builtin_ia32_vfmaddsubps(a, b, c);
 }
 
 __m128 __attribute__((target("fma,fma4"))) fma_3(__m128 a, __m128 b, __m128 c) {
-  return __builtin_ia32_vfmaddps(a, b, c);
+  return __builtin_ia32_vfmaddsubps(a, b, c);
 }
 
 void verifyfeaturestrings(void) {
diff --git a/clang/test/CodeGen/target-features-error-3.c b/clang/test/CodeGen/target-features-error-3.c
new file mode 100644
index 0000000000000..ff4866ac69ca0
--- /dev/null
+++ b/clang/test/CodeGen/target-features-error-3.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -emit-llvm -verify -o /dev/null
+
+typedef double __v2df __attribute__((__vector_size__(16)));
+
+__v2df __attribute__((target("sse4.1"))) foo() {
+    __v2df v = {0.0, 0.0};
+    return __builtin_ia32_roundpd(v, 2);
+}
+
+__v2df __attribute__((flatten)) bar() {
+    return foo(); // expected-error {{flatten function 'bar' calls 'foo' which requires target feature 'sse4.1', but the caller is compiled without support for 'sse4.1'}}
+}
diff --git a/clang/test/CodeGen/target-features-error-4.c b/clang/test/CodeGen/target-features-error-4.c
new file mode 100644
index 0000000000000..fe4879e03b660
--- /dev/null
+++ b/clang/test/CodeGen/target-features-error-4.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -emit-llvm -verify -o /dev/null
+
+typedef double __v2df __attribute__((__vector_size__(16)));
+
+__v2df __attribute__((target("sse4.1"))) foo() {
+    __v2df v = {0.0, 0.0};
+    return __builtin_ia32_roundpd(v, 2);
+}
+
+__v2df __attribute__((target("no-sse4.1"), flatten)) bar() {
+    return foo(); // expected-error {{flatten function 'bar' calls 'foo' which requires target feature 'sse4.1', but the caller is compiled without support for 'sse4.1'}}
+}
diff --git a/clang/test/CodeGen/target-features-error-5.c b/clang/test/CodeGen/target-features-error-5.c
new file mode 100644
index 0000000000000..8bc3ba193463d
--- /dev/null
+++ b/clang/test/CodeGen/target-features-error-5.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -target-feature +sse4.1 -emit-llvm -verify -o /dev/null
+
+typedef double __v2df __attribute__((__vector_size__(16)));
+
+__v2df foo() {
+    __v2df v = {0.0, 0.0};
+    return __builtin_ia32_roundpd(v, 2);
+}
+
+__v2df __attribute__((target("no-sse4.1"), flatten)) bar() {
+    return foo(); // expected-error {{flatten function 'bar' calls 'foo' which requires target feature 'sse4.1', but the caller is compiled without support for 'sse4.1'}}
+}
diff --git a/clang/test/CodeGen/target-features-no-error-2.c b/clang/test/CodeGen/target-features-no-error-2.c
new file mode 100644
index 0000000000000..eeec3f3d05d7b
--- /dev/null
+++ b/clang/test/CodeGen/target-features-no-error-2.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 %s -triple=x86_64-linux-gnu -emit-llvm -verify -o /dev/null
+
+typedef double __v2df __attribute__((__vector_size__(16)));
+
+__v2df __attribute__((target("sse4.1"))) foo() {
+    __v2df v = {0.0, 0.0};
+    return __builtin_ia32_roundpd(v, 2);
+}
+
+__v2df __attribute__((target("sse4.1"), flatten)) bar() {
+    return foo(); // expected-no-diagnostics
+}
diff --git a/clang/test/CodeGen/temporary-lifetime-exceptions.cpp b/clang/test/CodeGen/temporary-lifetime-exceptions.cpp
index 50e4a0f56cb2d..45400c20d847b 100644
--- a/clang/test/CodeGen/temporary-lifetime-exceptions.cpp
+++ b/clang/test/CodeGen/temporary-lifetime-exceptions.cpp
@@ -8,15 +8,15 @@ A Baz(const A&);
 
 void Test1() {
   // CHECK-LABEL: @_Z5Test1v(
-  // CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[TMP:[^ ]+]])
-  // CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[TMP1:[^ ]+]])
+  // CHECK: call void @llvm.lifetime.start.p0(ptr nonnull [[TMP:[^ ]+]])
+  // CHECK: call void @llvm.lifetime.start.p0(ptr nonnull [[TMP1:[^ ]+]])
 
   // Normal exit
-  // CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[TMP1]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[TMP]])
+  // CHECK: call void @llvm.lifetime.end.p0(ptr nonnull [[TMP1]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[TMP]])
 
   // Exception exit
-  // CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[TMP1]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[TMP]])
+  // CHECK: call void @llvm.lifetime.end.p0(ptr nonnull [[TMP1]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[TMP]])
   Baz(Baz(A()));
 }
diff --git a/clang/test/CodeGen/temporary-lifetime.cpp b/clang/test/CodeGen/temporary-lifetime.cpp
index 9f085d41d1464..04087292b2c70 100644
--- a/clang/test/CodeGen/temporary-lifetime.cpp
+++ b/clang/test/CodeGen/temporary-lifetime.cpp
@@ -21,27 +21,27 @@ T Baz();
 
 void Test1() {
   // CHECK-DTOR-LABEL: Test1
-  // CHECK-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR:.+]])
+  // CHECK-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR:.+]])
   // CHECK-DTOR: call void @_ZN1AC1Ev(ptr nonnull {{[^,]*}} %[[VAR:[^ ]+]])
   // CHECK-DTOR: call void @_Z3FooIRK1AEvOT_
   // CHECK-DTOR: call void @_ZN1AD1Ev(ptr nonnull {{[^,]*}} %[[VAR]])
-  // CHECK-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR]])
-  // CHECK-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR:.+]])
+  // CHECK-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR]])
+  // CHECK-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR:.+]])
   // CHECK-DTOR: call void @_ZN1AC1Ev(ptr nonnull {{[^,]*}} %[[VAR:[^ ]+]])
   // CHECK-DTOR: call void @_Z3FooIRK1AEvOT_
   // CHECK-DTOR: call void @_ZN1AD1Ev(ptr nonnull {{[^,]*}} %[[VAR]])
-  // CHECK-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR]])
+  // CHECK-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR]])
   // CHECK-DTOR: }
 
   // CHECK-NO-DTOR-LABEL: Test1
-  // CHECK-NO-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR:.+]])
+  // CHECK-NO-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR:.+]])
   // CHECK-NO-DTOR: call void @_ZN1AC1Ev(ptr nonnull {{[^,]*}} %[[VAR:[^ ]+]])
   // CHECK-NO-DTOR: call void @_Z3FooIRK1AEvOT_
-  // CHECK-NO-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR]])
-  // CHECK-NO-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR:.+]])
+  // CHECK-NO-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR]])
+  // CHECK-NO-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR:.+]])
   // CHECK-NO-DTOR: call void @_ZN1AC1Ev(ptr nonnull {{[^,]*}} %[[VAR:[^ ]+]])
   // CHECK-NO-DTOR: call void @_Z3FooIRK1AEvOT_
-  // CHECK-NO-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR]])
+  // CHECK-NO-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR]])
   // CHECK-NO-DTOR: }
   {
     const A &a = A{};
@@ -55,27 +55,27 @@ void Test1() {
 
 void Test2() {
   // CHECK-DTOR-LABEL: Test2
-  // CHECK-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR1:.+]])
+  // CHECK-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR1:.+]])
   // CHECK-DTOR: call void @_ZN1AC1Ev(ptr nonnull {{[^,]*}} %[[VAR1:[^ ]+]])
   // CHECK-DTOR: call void @_Z3FooIRK1AEvOT_
-  // CHECK-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR2:.+]])
+  // CHECK-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR2:.+]])
   // CHECK-DTOR: call void @_ZN1AC1Ev(ptr nonnull {{[^,]*}} %[[VAR2:[^ ]+]])
   // CHECK-DTOR: call void @_Z3FooIRK1AEvOT_
   // CHECK-DTOR: call void @_ZN1AD1Ev(ptr nonnull {{[^,]*}} %[[VAR2]])
-  // CHECK-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR2]])
+  // CHECK-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR2]])
   // CHECK-DTOR: call void @_ZN1AD1Ev(ptr nonnull {{[^,]*}} %[[VAR1]])
-  // CHECK-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR1]])
+  // CHECK-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR1]])
   // CHECK-DTOR: }
 
   // CHECK-NO-DTOR-LABEL: Test2
-  // CHECK-NO-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR1:.+]])
+  // CHECK-NO-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR1:.+]])
   // CHECK-NO-DTOR: call void @_ZN1AC1Ev(ptr nonnull {{[^,]*}} %[[VAR1:[^ ]+]])
   // CHECK-NO-DTOR: call void @_Z3FooIRK1AEvOT_
-  // CHECK-NO-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR2:.+]])
+  // CHECK-NO-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR2:.+]])
   // CHECK-NO-DTOR: call void @_ZN1AC1Ev(ptr nonnull {{[^,]*}} %[[VAR2:[^ ]+]])
   // CHECK-NO-DTOR: call void @_Z3FooIRK1AEvOT_
-  // CHECK-NO-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR2]])
-  // CHECK-NO-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR1]])
+  // CHECK-NO-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR2]])
+  // CHECK-NO-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR1]])
   // CHECK-NO-DTOR: }
   const A &a = A{};
   Foo(a);
@@ -135,16 +135,16 @@ int Test5() {
 
 void Test6() {
   // CHECK-DTOR-LABEL: Test6
-  // CHECK-DTOR: call void @llvm.lifetime.start.p0(i64 {{[0-9]+}}, ptr nonnull %[[ADDR:.+]])
+  // CHECK-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR:.+]])
   // CHECK-DTOR: call i32 @_Z3BazIiET_v()
   // CHECK-DTOR: store
   // CHECK-DTOR: call void @_Z3FooIiEvOT_
-  // CHECK-DTOR: call void @llvm.lifetime.end.p0(i64 {{[0-9]+}}, ptr nonnull %[[ADDR]])
-  // CHECK-DTOR: call void @llvm.lifetime.start.p0(i64 {{[0-9]+}}, ptr nonnull %[[ADDR:.+]])
+  // CHECK-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR]])
+  // CHECK-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR:.+]])
   // CHECK-DTOR: call i32 @_Z3BazIiET_v()
   // CHECK-DTOR: store
   // CHECK-DTOR: call void @_Z3FooIiEvOT_
-  // CHECK-DTOR: call void @llvm.lifetime.end.p0(i64 {{[0-9]+}}, ptr nonnull %[[ADDR]])
+  // CHECK-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR]])
   // CHECK-DTOR: }
   Foo(Baz<int>());
   Foo(Baz<int>());
@@ -152,16 +152,16 @@ void Test6() {
 
 void Test7() {
   // CHECK-DTOR-LABEL: Test7
-  // CHECK-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR:.+]])
+  // CHECK-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR:.+]])
   // CHECK-DTOR: call void @_Z3BazI1AET_v({{.*}} %[[SLOT:[^ ]+]])
   // CHECK-DTOR: call void @_Z3FooI1AEvOT_({{.*}} %[[SLOT]])
   // CHECK-DTOR: call void @_ZN1AD1Ev(ptr nonnull {{[^,]*}} %[[SLOT]])
-  // CHECK-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR]])
-  // CHECK-DTOR: call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %[[ADDR:.+]])
+  // CHECK-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR]])
+  // CHECK-DTOR: call void @llvm.lifetime.start.p0(ptr nonnull %[[ADDR:.+]])
   // CHECK-DTOR: call void @_Z3BazI1AET_v({{.*}} %[[SLOT:[^ ]+]])
   // CHECK-DTOR: call void @_Z3FooI1AEvOT_({{.*}} %[[SLOT]])
   // CHECK-DTOR: call void @_ZN1AD1Ev(ptr nonnull {{[^,]*}} %[[SLOT]])
-  // CHECK-DTOR: call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %[[ADDR]])
+  // CHECK-DTOR: call void @llvm.lifetime.end.p0(ptr nonnull %[[ADDR]])
   // CHECK-DTOR: }
   Foo(Baz<A>());
   Foo(Baz<A>());
diff --git a/clang/test/CodeGen/union-tbaa1.c b/clang/test/CodeGen/union-tbaa1.c
index 1e2f384c29a5b..3322aaa7adf14 100644
--- a/clang/test/CodeGen/union-tbaa1.c
+++ b/clang/test/CodeGen/union-tbaa1.c
@@ -11,7 +11,7 @@ void bar(vect32 p[][2]);
 // CHECK-SAME: (i32 noundef [[NUM:%.*]], ptr noundef writeonly captures(none) initializes((0, 8)) [[VEC:%.*]], ptr noundef readonly captures(none) [[INDEX:%.*]], ptr noundef readonly captures(none) [[ARR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[TMP:%.*]] = alloca [4 x [2 x %union.vect32]], align 8
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[TMP]]) #[[ATTR3:[0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP]]) #[[ATTR3:[0-9]+]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[INDEX]], align 4, !tbaa [[TBAA2:![0-9]+]]
 // CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[ARR]], i32 [[TMP0]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4, !tbaa [[TBAA2]]
@@ -33,7 +33,7 @@ void bar(vect32 p[][2]);
 // CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds nuw i8, ptr [[VEC]], i32 4
 // CHECK-NEXT:    store i32 [[CONV16]], ptr [[ARRAYIDX17]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    call void @bar(ptr noundef nonnull [[TMP]]) #[[ATTR3]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[TMP]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP]]) #[[ATTR3]]
 // CHECK-NEXT:    ret void
 //
 void fred(unsigned Num, int Vec[2], int *Index, int Arr[4][2]) {
diff --git a/clang/test/CodeGen/volatile-1.c b/clang/test/CodeGen/volatile-1.c
index 7116a4e1d03da..9d97d898400fe 100644
--- a/clang/test/CodeGen/volatile-1.c
+++ b/clang/test/CodeGen/volatile-1.c
@@ -1,4 +1,3 @@
-// XFAIL: target=aarch64-pc-windows-msvc
 // RUN: %clang_cc1 -Wno-return-type -Wno-unused-value -emit-llvm %s -w -o - | FileCheck %s
 
 // CHECK: @i = {{(dso_local )?}}global [[INT:i[0-9]+]] 0
@@ -27,44 +26,44 @@ int printf(const char *, ...);
 void test(void) {
   // CHECK: load volatile [[INT]], ptr @i
   i;
-  // CHECK-NEXT: load volatile [[INT]], ptr @ci, align 4
-  // CHECK-NEXT: load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
+  // CHECK-NEXT: load volatile [[INT]], ptr @ci, align [[ALIGN:[0-9]+]]
+  // CHECK-NEXT: load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP:[0-9]+]]
   // CHECK-NEXT: sitofp [[INT]]
   (float)(ci);
-  // CHECK-NEXT: load volatile [[INT]], ptr @ci, align 4
-  // CHECK-NEXT: load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
+  // CHECK-NEXT: load volatile [[INT]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
   (void)ci;
   // CHECK-NEXT: memcpy
   (void)a;
-  // CHECK-NEXT: [[R:%.*]] = load volatile [[INT]], ptr @ci, align 4
-  // CHECK-NEXT: [[I:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
-  // CHECK-NEXT: store volatile [[INT]] [[R]], ptr @ci, align 4
-  // CHECK-NEXT: store volatile [[INT]] [[I]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
+  // CHECK-NEXT: [[R:%.*]] = load volatile [[INT]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: [[I:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
+  // CHECK-NEXT: store volatile [[INT]] [[R]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: store volatile [[INT]] [[I]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
   (void)(ci=ci);
   // CHECK-NEXT: [[T:%.*]] = load volatile [[INT]], ptr @j
   // CHECK-NEXT: store volatile [[INT]] [[T]], ptr @i
   (void)(i=j);
-  // CHECK-NEXT: [[R1:%.*]] = load volatile [[INT]], ptr @ci, align 4
-  // CHECK-NEXT: [[I1:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
-  // CHECK-NEXT: [[R2:%.*]] = load volatile [[INT]], ptr @ci, align 4
-  // CHECK-NEXT: [[I2:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
+  // CHECK-NEXT: [[R1:%.*]] = load volatile [[INT]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: [[I1:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
+  // CHECK-NEXT: [[R2:%.*]] = load volatile [[INT]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: [[I2:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
   // Not sure why they're ordered this way.
   // CHECK-NEXT: [[R:%.*]] = add [[INT]] [[R2]], [[R1]]
   // CHECK-NEXT: [[I:%.*]] = add [[INT]] [[I2]], [[I1]]
-  // CHECK-NEXT: store volatile [[INT]] [[R]], ptr @ci, align 4
-  // CHECK-NEXT: store volatile [[INT]] [[I]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
+  // CHECK-NEXT: store volatile [[INT]] [[R]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: store volatile [[INT]] [[I]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
   ci+=ci;
 
-  // CHECK-NEXT: [[R1:%.*]] = load volatile [[INT]], ptr @ci, align 4
-  // CHECK-NEXT: [[I1:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
-  // CHECK-NEXT: [[R2:%.*]] = load volatile [[INT]], ptr @ci, align 4
-  // CHECK-NEXT: [[I2:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
+  // CHECK-NEXT: [[R1:%.*]] = load volatile [[INT]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: [[I1:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
+  // CHECK-NEXT: [[R2:%.*]] = load volatile [[INT]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: [[I2:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
   // CHECK-NEXT: [[R:%.*]] = add [[INT]] [[R2]], [[R1]]
   // CHECK-NEXT: [[I:%.*]] = add [[INT]] [[I2]], [[I1]]
-  // CHECK-NEXT: store volatile [[INT]] [[R]], ptr @ci, align 4
-  // CHECK-NEXT: store volatile [[INT]] [[I]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
-  // CHECK-NEXT: [[R2:%.*]] = load volatile [[INT]], ptr @ci, align 4
-  // CHECK-NEXT: [[I2:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align 4
+  // CHECK-NEXT: store volatile [[INT]] [[R]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: store volatile [[INT]] [[I]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
+  // CHECK-NEXT: [[R2:%.*]] = load volatile [[INT]], ptr @ci, align [[ALIGN]]
+  // CHECK-NEXT: [[I2:%.*]] = load volatile [[INT]], ptr getelementptr inbounds nuw ([[CINT]], ptr @ci, i32 0, i32 1), align [[ALIGN_GEP]]
   // These additions can be elided
   // CHECK-NEXT: add [[INT]] [[R]], [[R2]]
   // CHECK-NEXT: add [[INT]] [[I]], [[I2]]
diff --git a/clang/test/CodeGenCXX/aarch64-sve-vector-conditional-op.cpp b/clang/test/CodeGenCXX/aarch64-sve-vector-conditional-op.cpp
index d6fa26bd34099..5c99393357425 100644
--- a/clang/test/CodeGenCXX/aarch64-sve-vector-conditional-op.cpp
+++ b/clang/test/CodeGenCXX/aarch64-sve-vector-conditional-op.cpp
@@ -10,8 +10,7 @@
 // CHECK-LABEL: @_Z9cond_boolu10__SVBool_tS_(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[CMP:%.*]] = icmp ult <vscale x 16 x i1> [[A:%.*]], [[B:%.*]]
-// CHECK-NEXT:    [[VECTOR_COND:%.*]] = icmp ne <vscale x 16 x i1> [[CMP]], zeroinitializer
-// CHECK-NEXT:    [[VECTOR_SELECT:%.*]] = select <vscale x 16 x i1> [[VECTOR_COND]], <vscale x 16 x i1> [[A]], <vscale x 16 x i1> [[B]]
+// CHECK-NEXT:    [[VECTOR_SELECT:%.*]] = select <vscale x 16 x i1> [[CMP]], <vscale x 16 x i1> [[A]], <vscale x 16 x i1> [[B]]
 // CHECK-NEXT:    ret <vscale x 16 x i1> [[VECTOR_SELECT]]
 //
 svbool_t cond_bool(svbool_t a, svbool_t b) {
diff --git a/clang/test/CodeGenCXX/amdgcn_declspec_get.cpp b/clang/test/CodeGenCXX/amdgcn_declspec_get.cpp
index 0ce88014eafd2..c784e6053edd0 100644
--- a/clang/test/CodeGenCXX/amdgcn_declspec_get.cpp
+++ b/clang/test/CodeGenCXX/amdgcn_declspec_get.cpp
@@ -15,10 +15,10 @@ extern const A a;
 // CHECK-LABEL: define{{.*}} void @_Z4testv()
 // CHECK:  %i = alloca i32, align 4, addrspace(5)
 // CHECK:  %[[ii:.*]] = addrspacecast ptr addrspace(5) %i to ptr
-// CHECK:  call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %i)
+// CHECK:  call void @llvm.lifetime.start.p5(ptr addrspace(5) %i)
 // CHECK:  %call = call noundef i32 @_ZN1A6_get_xEv()
 // CHECK:  store i32 %call, ptr %[[ii]]
-// CHECK:  call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %i)
+// CHECK:  call void @llvm.lifetime.end.p5(ptr addrspace(5) %i)
 void test()
 {
   int i = a.x;
diff --git a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
index fd9786de3a949..151b77ac1007b 100644
--- a/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
+++ b/clang/test/CodeGenCXX/attr-likelihood-iteration-stmt.cpp
@@ -68,7 +68,7 @@ void w_branch_elided(unsigned e){
 // CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store i32 [[E:%.*]], ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR3:[0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR3:[0-9]+]]
 // CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK:       for.cond:
@@ -78,7 +78,7 @@ void w_branch_elided(unsigned e){
 // CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
 // CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 // CHECK:       for.cond.cleanup:
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_END:%.*]]
 // CHECK:       for.body:
 // CHECK-NEXT:    br label [[FOR_INC:%.*]]
@@ -100,7 +100,7 @@ void fl(unsigned e)
 // CHECK-NEXT:    [[E_ADDR:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store i32 [[E:%.*]], ptr [[E_ADDR]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK:       for.cond:
@@ -110,7 +110,7 @@ void fl(unsigned e)
 // CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
 // CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 // CHECK:       for.cond.cleanup:
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_END:%.*]]
 // CHECK:       for.body:
 // CHECK-NEXT:    br label [[FOR_INC:%.*]]
@@ -146,14 +146,14 @@ void f_branch_elided()
 // CHECK-NEXT:    [[__END1:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[E:%.*]], ptr [[E_ADDR]], align 8, !tbaa [[TBAA14:![0-9]+]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[__RANGE1]]) #[[ATTR3]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !tbaa [[TBAA14]]
 // CHECK-NEXT:    store ptr [[TMP0]], ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[__BEGIN1]]) #[[ATTR3]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP1]], i64 0, i64 0
 // CHECK-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__END1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[__END1]]) #[[ATTR3]]
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
 // CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP2]], i64 0, i64 0
 // CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, ptr [[ARRAYDECAY1]], i64 4
@@ -166,16 +166,16 @@ void f_branch_elided()
 // CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 true)
 // CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 // CHECK:       for.cond.cleanup:
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__END1]]) #[[ATTR3]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR3]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[__END1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[__BEGIN1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[__RANGE1]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_END:%.*]]
 // CHECK:       for.body:
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
 // CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    store i32 [[TMP6]], ptr [[I]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK:       for.inc:
 // CHECK-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
@@ -198,14 +198,14 @@ void frl(int (&&e) [4])
 // CHECK-NEXT:    [[__END1:%.*]] = alloca ptr, align 8
 // CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    store ptr [[E:%.*]], ptr [[E_ADDR]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[__RANGE1]]) #[[ATTR3]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[E_ADDR]], align 8, !tbaa [[TBAA14]]
 // CHECK-NEXT:    store ptr [[TMP0]], ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[__BEGIN1]]) #[[ATTR3]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP1]], i64 0, i64 0
 // CHECK-NEXT:    store ptr [[ARRAYDECAY]], ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[__END1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[__END1]]) #[[ATTR3]]
 // CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE1]], align 8, !tbaa [[TBAA14]]
 // CHECK-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [4 x i32], ptr [[TMP2]], i64 0, i64 0
 // CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, ptr [[ARRAYDECAY1]], i64 4
@@ -218,16 +218,16 @@ void frl(int (&&e) [4])
 // CHECK-NEXT:    [[CMP_EXPVAL:%.*]] = call i1 @llvm.expect.i1(i1 [[CMP]], i1 false)
 // CHECK-NEXT:    br i1 [[CMP_EXPVAL]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 // CHECK:       for.cond.cleanup:
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__END1]]) #[[ATTR3]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__BEGIN1]]) #[[ATTR3]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[__RANGE1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[__END1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[__BEGIN1]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[__RANGE1]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_END:%.*]]
 // CHECK:       for.body:
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
 // CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    store i32 [[TMP6]], ptr [[I]], align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR3]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR3]]
 // CHECK-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK:       for.inc:
 // CHECK-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__BEGIN1]], align 8, !tbaa [[TBAA14]]
diff --git a/clang/test/CodeGenCXX/cxx2b-deducing-this.cpp b/clang/test/CodeGenCXX/cxx2b-deducing-this.cpp
index 8a78463d3a495..9664a866376ae 100644
--- a/clang/test/CodeGenCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/CodeGenCXX/cxx2b-deducing-this.cpp
@@ -264,3 +264,55 @@ void test() {
 // CHECK: call void @_ZNH5P27971C1cERKS0_
 // CHECK: call void @_ZN5P27971C1cEi
 }
+
+// This used to crash because we weren’t instantiating a dependent 'this'.
+namespace GH154054 {
+struct S {
+  int x;
+  auto byval() {
+    return [*this](this auto) { return this->x; };
+  }
+};
+
+// CHECK-LABEL: define {{.*}} void @_ZN8GH1540544mainEv
+void main() {
+  S s{ 42 };
+
+  // CHECK: call {{.*}} i32 @_ZZN8GH1540541S5byvalEvENHUlT_E_clIS2_EEDaS1_
+  if ( s.byval()() != 42)
+    __builtin_abort();
+}
+
+// CHECK-LABEL: define {{.*}} i32 @_ZZN8GH1540541S5byvalEvENHUlT_E_clIS2_EEDaS1_(i32 %.coerce)
+// CHECK: entry:
+// CHECK:   %0 = alloca %class.anon.11, align 4
+// CHECK:   %coerce.dive = getelementptr inbounds nuw %class.anon.11, ptr %0, i32 0, i32 0
+// CHECK:   %coerce.dive1 = getelementptr inbounds nuw %"struct.GH154054::S", ptr %coerce.dive, i32 0, i32 0
+// CHECK:   store i32 %.coerce, ptr %coerce.dive1, align 4
+// CHECK:   %1 = getelementptr inbounds nuw %class.anon.11, ptr %0, i32 0, i32 0
+// CHECK:   %x = getelementptr inbounds nuw %"struct.GH154054::S", ptr %1, i32 0, i32 0
+// CHECK:   %2 = load i32, ptr %x, align 4
+// CHECK:   ret i32 %2
+
+struct s {
+  int q;
+  auto f() {
+    return [*this](this auto) { return this; };
+  }
+};
+
+// CHECK-LABEL: define {{.*}} void @_ZN8GH1540541fEv
+void f() {
+  // CHECK: call {{.*}} ptr @_ZZN8GH1540541s1fEvENHUlT_E_clIS2_EEDaS1_
+  s{}.f()();
+}
+
+// CHECK-LABEL: define {{.*}} ptr @_ZZN8GH1540541s1fEvENHUlT_E_clIS2_EEDaS1_(i32 %.coerce)
+// CHECK: entry:
+// CHECK:   %0 = alloca %class.anon.12, align 4
+// CHECK:   %coerce.dive = getelementptr inbounds nuw %class.anon.12, ptr %0, i32 0, i32 0
+// CHECK:   %coerce.dive1 = getelementptr inbounds nuw %"struct.GH154054::s", ptr %coerce.dive, i32 0, i32 0
+// CHECK:   store i32 %.coerce, ptr %coerce.dive1, align 4
+// CHECK:   %1 = getelementptr inbounds nuw %class.anon.12, ptr %0, i32 0, i32 0
+// CHECK:   ret ptr %1
+}
diff --git a/clang/test/CodeGenCXX/debug-info-class-limited.test b/clang/test/CodeGenCXX/debug-info-class-limited.test
deleted file mode 100644
index c2e332866f9fa..0000000000000
--- a/clang/test/CodeGenCXX/debug-info-class-limited.test
+++ /dev/null
@@ -1 +0,0 @@
-RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited %S/Inputs/debug-info-class-limited.cpp -o - | FileCheck %S/Inputs/debug-info-class-limited.cpp
diff --git a/clang/test/CodeGenCXX/destructors.cpp b/clang/test/CodeGenCXX/destructors.cpp
index 99c82ec76989a..1bf8b163b4a09 100644
--- a/clang/test/CodeGenCXX/destructors.cpp
+++ b/clang/test/CodeGenCXX/destructors.cpp
@@ -308,7 +308,7 @@ namespace test5 {
   // CHECK5:      [[ELEMS:%.*]] = alloca [5 x [[A:%.*]]], align
   // CHECK5v03-NEXT: [[EXN:%.*]] = alloca ptr
   // CHECK5v03-NEXT: [[SEL:%.*]] = alloca i32
-  // CHECK5-NEXT: call void @llvm.lifetime.start.p0(i64 5, ptr [[ELEMS]])
+  // CHECK5-NEXT: call void @llvm.lifetime.start.p0(ptr [[ELEMS]])
   // CHECK5-NEXT: [[BEGIN:%.*]] = getelementptr inbounds [5 x [[A]]], ptr [[ELEMS]], i32 0, i32 0
   // CHECK5-NEXT: [[END:%.*]] = getelementptr inbounds [[A]], ptr [[BEGIN]], i64 5
   // CHECK5-NEXT: br label
@@ -482,24 +482,24 @@ namespace test11 {
 
 // CHECK6: {{^}}invoke.cont
 // CHECK6: call void @_ZN6test112S1D1Ev(ptr {{[^,]*}} [[T1]])
-// CHECK6: call void @llvm.lifetime.end.p0(i64 32, ptr [[T1]])
+// CHECK6: call void @llvm.lifetime.end.p0(ptr [[T1]])
 // CHECK6: {{^}}lpad
 // CHECK6: call void @_ZN6test112S1D1Ev(ptr {{[^,]*}} [[T1]])
-// CHECK6: call void @llvm.lifetime.end.p0(i64 32, ptr [[T1]])
+// CHECK6: call void @llvm.lifetime.end.p0(ptr [[T1]])
 
 // CHECK6: {{^}}invoke.cont
 // CHECK6: call void @_ZN6test112S1D1Ev(ptr {{[^,]*}} [[T2]])
-// CHECK6: call void @llvm.lifetime.end.p0(i64 32, ptr [[T2]])
+// CHECK6: call void @llvm.lifetime.end.p0(ptr [[T2]])
 // CHECK6: {{^}}lpad
 // CHECK6: call void @_ZN6test112S1D1Ev(ptr {{[^,]*}} [[T2]])
-// CHECK6: call void @llvm.lifetime.end.p0(i64 32, ptr [[T2]])
+// CHECK6: call void @llvm.lifetime.end.p0(ptr [[T2]])
 
 // CHECK6: {{^}}invoke.cont
 // CHECK6: call void @_ZN6test112S1D1Ev(ptr {{[^,]*}} [[T3]])
-// CHECK6: call void @llvm.lifetime.end.p0(i64 32, ptr [[T3]])
+// CHECK6: call void @llvm.lifetime.end.p0(ptr [[T3]])
 // CHECK6: {{^}}lpad
 // CHECK6: call void @_ZN6test112S1D1Ev(ptr {{[^,]*}} [[T3]])
-// CHECK6: call void @llvm.lifetime.end.p0(i64 32, ptr [[T3]])
+// CHECK6: call void @llvm.lifetime.end.p0(ptr [[T3]])
 
   struct S1 {
     ~S1();
diff --git a/clang/test/CodeGenCXX/dynamic-cast-exact-disabled.cpp b/clang/test/CodeGenCXX/dynamic-cast-exact-disabled.cpp
index 19c2a9bd0497e..3156e1bb21b1e 100644
--- a/clang/test/CodeGenCXX/dynamic-cast-exact-disabled.cpp
+++ b/clang/test/CodeGenCXX/dynamic-cast-exact-disabled.cpp
@@ -14,3 +14,19 @@ B *exact(A *a) {
   // EXACT-NOT: call {{.*}} @__dynamic_cast
   return dynamic_cast<B*>(a);
 }
+
+struct C {
+  virtual ~C();
+};
+
+struct D final : private C {
+
+};
+
+// CHECK-LABEL: @_Z5exactP1C
+D *exact(C *a) {
+  // INEXACT: call {{.*}} @__dynamic_cast
+  // EXACT: entry:
+  // EXACT-NEXT: ret ptr null
+  return dynamic_cast<D*>(a);
+}
diff --git a/clang/test/CodeGenCXX/dynamic-cast-exact.cpp b/clang/test/CodeGenCXX/dynamic-cast-exact.cpp
index 86e1965b4ba68..588d80844a2fa 100644
--- a/clang/test/CodeGenCXX/dynamic-cast-exact.cpp
+++ b/clang/test/CodeGenCXX/dynamic-cast-exact.cpp
@@ -9,6 +9,7 @@ struct E : A { int e; };
 struct F : virtual A { int f; };
 struct G : virtual A { int g; };
 struct H final : C, D, E, F, G { int h; };
+struct H1 final: C, private D { int h1; };
 
 // CHECK-LABEL: @_Z7inexactP1A
 C *inexact(A *a) {
@@ -77,10 +78,49 @@ H *exact_multi(A *a) {
   return dynamic_cast<H*>(a);
 }
 
+// CHECK-LABEL: @_Z19exact_invalid_multiP1D
+H1 *exact_invalid_multi(D* d) {
+  // CHECK:      entry:
+  // CHECK-NEXT:   %d.addr = alloca ptr
+  // CHECK-NEXT:   store ptr %d, ptr %d.addr
+  // CHECK-NEXT:   load ptr, ptr %d.addr
+  // CHECK-NEXT:   ret ptr null
+  return dynamic_cast<H1*>(d);
+}
+
+// CHECK-LABEL: @_Z19exact_invalid_multiR1D
+H1 &exact_invalid_multi(D& d) {
+  // CHECK:      entry:
+  // CHECK-NEXT:   %d.addr = alloca ptr
+  // CHECK-NEXT:   store ptr %d, ptr %d.addr
+  // CHECK-NEXT:   load ptr, ptr %d.addr
+  // CHECK-NEXT:   call void @__cxa_bad_cast()
+  // CHECK-NEXT:   unreachable
+  // CHECK:      dynamic_cast.unreachable:
+  // CHECK-NEXT:   ret ptr poison
+  return dynamic_cast<H1&>(d);
+}
+
+namespace GH137518 {
+  class base { virtual void fn() = 0; };
+  class test final : base { virtual void fn() { } };
+  test* new_test() { return new test(); }
+
+  // CHECK-LABEL: @_ZN8GH1375184castEPNS_4baseE(
+  test* cast(base* b) {
+    // CHECK:      entry:
+    // CHECK-NEXT:   %b.addr = alloca ptr
+    // CHECK-NEXT:   store ptr %b, ptr %b.addr
+    // CHECK-NEXT:   load ptr, ptr %b.addr
+    // CHECK-NEXT:   ret ptr null
+    return dynamic_cast<test*>(b);
+  }
+}
+
 namespace GH64088 {
   // Ensure we mark the B vtable as used here, because we're going to emit a
   // reference to it.
-  // CHECK: define {{.*}} @_ZN7GH640881BD0
+  // CHECK: define {{.*}} void @_ZN7GH640881BD0Ev(
   struct A { virtual ~A(); };
   struct B final : A { virtual ~B() = default; };
   B *cast(A *p) { return dynamic_cast<B*>(p); }
diff --git a/clang/test/CodeGenCXX/ext-vector-type-conditional.cpp b/clang/test/CodeGenCXX/ext-vector-type-conditional.cpp
index 4504000856f23..8ef3fbb5378d7 100644
--- a/clang/test/CodeGenCXX/ext-vector-type-conditional.cpp
+++ b/clang/test/CodeGenCXX/ext-vector-type-conditional.cpp
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 5
 // RUN: %clang_cc1 %s -triple x86_64-linux-gnu -Wno-unused -std=c++11 -emit-llvm -o - | FileCheck %s
 
 using FourShorts = short __attribute__((ext_vector_type(4)));
@@ -11,6 +12,7 @@ using TwoFloats = float __attribute__((ext_vector_type(2)));
 using FourFloats = float __attribute__((ext_vector_type(4)));
 using TwoDoubles = double __attribute__((ext_vector_type(2)));
 using FourDoubles = double __attribute__((ext_vector_type(4)));
+using TwoBools = bool __attribute__((ext_vector_type(2)));
 
 FourShorts four_shorts;
 TwoInts two_ints;
@@ -23,6 +25,7 @@ TwoFloats two_floats;
 FourFloats four_floats;
 TwoDoubles two_doubles;
 FourDoubles four_doubles;
+TwoBools two_bools;
 
 short some_short;
 unsigned short some_ushort;
@@ -33,235 +36,186 @@ long long some_ll;
 unsigned long long some_ull;
 double some_double;
 
-// CHECK: TwoVectorOps
+// CHECK-LABEL: define dso_local void @_Z12TwoVectorOpsv(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr @two_ints, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr @two_ints, align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr @two_ints, align 8
+// CHECK-NEXT:    [[VECTOR_COND:%.*]] = icmp slt <2 x i32> [[TMP0]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT:%.*]] = select <2 x i1> [[VECTOR_COND]], <2 x i32> [[TMP1]], <2 x i32> [[TMP2]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr @two_ints, align 8
+// CHECK-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr @two_floats, align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = load <2 x float>, ptr @two_floats, align 8
+// CHECK-NEXT:    [[VECTOR_COND1:%.*]] = icmp slt <2 x i32> [[TMP3]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT2:%.*]] = select <2 x i1> [[VECTOR_COND1]], <2 x float> [[TMP4]], <2 x float> [[TMP5]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr @two_ll, align 16
+// CHECK-NEXT:    [[TMP7:%.*]] = load <2 x double>, ptr @two_doubles, align 16
+// CHECK-NEXT:    [[TMP8:%.*]] = load <2 x double>, ptr @two_doubles, align 16
+// CHECK-NEXT:    [[VECTOR_COND3:%.*]] = icmp slt <2 x i64> [[TMP6]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT4:%.*]] = select <2 x i1> [[VECTOR_COND3]], <2 x double> [[TMP7]], <2 x double> [[TMP8]]
+// CHECK-NEXT:    [[LOAD_BITS:%.*]] = load i8, ptr @two_bools, align 1
+// CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[LOAD_BITS]] to <8 x i1>
+// CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <8 x i1> [[TMP9]], <8 x i1> poison, <2 x i32> <i32 0, i32 1>
+// CHECK-NEXT:    [[TMP10:%.*]] = load <2 x i32>, ptr @two_ints, align 8
+// CHECK-NEXT:    [[TMP11:%.*]] = load <2 x i32>, ptr @two_ints, align 8
+// CHECK-NEXT:    [[VECTOR_SELECT5:%.*]] = select <2 x i1> [[EXTRACTVEC]], <2 x i32> [[TMP10]], <2 x i32> [[TMP11]]
+// CHECK-NEXT:    [[LOAD_BITS6:%.*]] = load i8, ptr @two_bools, align 1
+// CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[LOAD_BITS6]] to <8 x i1>
+// CHECK-NEXT:    [[EXTRACTVEC7:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> poison, <2 x i32> <i32 0, i32 1>
+// CHECK-NEXT:    [[TMP13:%.*]] = load <2 x double>, ptr @two_doubles, align 16
+// CHECK-NEXT:    [[TMP14:%.*]] = load <2 x double>, ptr @two_doubles, align 16
+// CHECK-NEXT:    [[VECTOR_SELECT8:%.*]] = select <2 x i1> [[EXTRACTVEC7]], <2 x double> [[TMP13]], <2 x double> [[TMP14]]
+// CHECK-NEXT:    ret void
+//
 void TwoVectorOps() {
   two_ints ? two_ints : two_ints;
-  // CHECK: [[COND:%.+]] = load <2 x i32>
-  // CHECK: [[LHS:%.+]] = load <2 x i32>
-  // CHECK: [[RHS:%.+]] = load <2 x i32>
-  // CHECK: [[NEG:%.+]] = icmp slt <2 x i32> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <2 x i1> [[NEG]] to <2 x i32>
-  // CHECK: [[XOR:%.+]] = xor <2 x i32> [[SEXT]], splat (i32 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <2 x i32> [[RHS]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <2 x i32> [[LHS]], [[SEXT]]
-  // CHECK: = or <2 x i32> [[RHS_AND]], [[LHS_AND]]
 
   two_ints ? two_floats : two_floats;
-  // CHECK: [[COND:%.+]] = load <2 x i32>
-  // CHECK: [[LHS:%.+]] = load <2 x float>
-  // CHECK: [[RHS:%.+]] = load <2 x float>
-  // CHECK: [[NEG:%.+]] = icmp slt <2 x i32> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <2 x i1> [[NEG]] to <2 x i32>
-  // CHECK: [[XOR:%.+]] = xor <2 x i32> [[SEXT]], splat (i32 -1)
-  // CHECK: [[RHS_EXT:%.+]] = bitcast <2 x float> [[RHS]] to <2 x i32>
-  // CHECK: [[LHS_EXT:%.+]] = bitcast <2 x float> [[LHS]] to <2 x i32>
-  // CHECK: [[RHS_AND:%.+]] = and <2 x i32> [[RHS_EXT]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <2 x i32> [[LHS_EXT]], [[SEXT]]
-  // CHECK: [[OR:%.+]] = or <2 x i32> [[RHS_AND]], [[LHS_AND]]
-  // CHECK: = bitcast <2 x i32> [[OR]] to <2 x float>
 
   two_ll ? two_doubles : two_doubles;
-  // CHECK: [[COND:%.+]] = load <2 x i64>
-  // CHECK: [[LHS:%.+]] = load <2 x double>
-  // CHECK: [[RHS:%.+]] = load <2 x double>
-  // CHECK: [[NEG:%.+]] = icmp slt <2 x i64> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <2 x i1> [[NEG]] to <2 x i64>
-  // CHECK: [[XOR:%.+]] = xor <2 x i64> [[SEXT]], splat (i64 -1)
-  // CHECK: [[RHS_EXT:%.+]] = bitcast <2 x double> [[RHS]] to <2 x i64>
-  // CHECK: [[LHS_EXT:%.+]] = bitcast <2 x double> [[LHS]] to <2 x i64>
-  // CHECK: [[RHS_AND:%.+]] = and <2 x i64> [[RHS_EXT]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <2 x i64> [[LHS_EXT]], [[SEXT]]
-  // CHECK: [[OR:%.+]] = or <2 x i64> [[RHS_AND]], [[LHS_AND]]
-  // CHECK: = bitcast <2 x i64> [[OR]] to <2 x double>
+
+  two_bools ? two_ints : two_ints;
+
+  two_bools ? two_doubles : two_doubles;
 }
 
-// CHECK: TwoScalarOps
+// CHECK-LABEL: define dso_local void @_Z12TwoScalarOpsv(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr @four_shorts, align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr @some_short, align 2
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP1]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i16> [[SPLAT_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr @some_short, align 2
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT1:%.*]] = insertelement <4 x i16> poison, i16 [[TMP2]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT2:%.*]] = shufflevector <4 x i16> [[SPLAT_SPLATINSERT1]], <4 x i16> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND:%.*]] = icmp slt <4 x i16> [[TMP0]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT:%.*]] = select <4 x i1> [[VECTOR_COND]], <4 x i16> [[SPLAT_SPLAT]], <4 x i16> [[SPLAT_SPLAT2]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, ptr @four_shorts, align 8
+// CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr @some_ushort, align 2
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT3:%.*]] = insertelement <4 x i16> poison, i16 [[TMP4]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT4:%.*]] = shufflevector <4 x i16> [[SPLAT_SPLATINSERT3]], <4 x i16> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr @some_ushort, align 2
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT5:%.*]] = insertelement <4 x i16> poison, i16 [[TMP5]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT6:%.*]] = shufflevector <4 x i16> [[SPLAT_SPLATINSERT5]], <4 x i16> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND7:%.*]] = icmp slt <4 x i16> [[TMP3]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT8:%.*]] = select <4 x i1> [[VECTOR_COND7]], <4 x i16> [[SPLAT_SPLAT4]], <4 x i16> [[SPLAT_SPLAT6]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load <4 x i32>, ptr @four_ints, align 16
+// CHECK-NEXT:    [[TMP7:%.*]] = load i16, ptr @some_ushort, align 2
+// CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[TMP7]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT9:%.*]] = insertelement <4 x i32> poison, i32 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT10:%.*]] = shufflevector <4 x i32> [[SPLAT_SPLATINSERT9]], <4 x i32> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[TMP8:%.*]] = load i16, ptr @some_short, align 2
+// CHECK-NEXT:    [[CONV11:%.*]] = sext i16 [[TMP8]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT12:%.*]] = insertelement <4 x i32> poison, i32 [[CONV11]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT13:%.*]] = shufflevector <4 x i32> [[SPLAT_SPLATINSERT12]], <4 x i32> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND14:%.*]] = icmp slt <4 x i32> [[TMP6]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT15:%.*]] = select <4 x i1> [[VECTOR_COND14]], <4 x i32> [[SPLAT_SPLAT10]], <4 x i32> [[SPLAT_SPLAT13]]
+// CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, ptr @four_ints, align 16
+// CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr @some_int, align 4
+// CHECK-NEXT:    [[CONV16:%.*]] = sitofp i32 [[TMP10]] to float
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT17:%.*]] = insertelement <4 x float> poison, float [[CONV16]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT18:%.*]] = shufflevector <4 x float> [[SPLAT_SPLATINSERT17]], <4 x float> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr @some_float, align 4
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT19:%.*]] = insertelement <4 x float> poison, float [[TMP11]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <4 x float> [[SPLAT_SPLATINSERT19]], <4 x float> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND21:%.*]] = icmp slt <4 x i32> [[TMP9]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT22:%.*]] = select <4 x i1> [[VECTOR_COND21]], <4 x float> [[SPLAT_SPLAT18]], <4 x float> [[SPLAT_SPLAT20]]
+// CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i64>, ptr @four_ll, align 32
+// CHECK-NEXT:    [[TMP13:%.*]] = load double, ptr @some_double, align 8
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT23:%.*]] = insertelement <4 x double> poison, double [[TMP13]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT24:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT23]], <4 x double> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr @some_ll, align 8
+// CHECK-NEXT:    [[CONV25:%.*]] = sitofp i64 [[TMP14]] to double
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT26:%.*]] = insertelement <4 x double> poison, double [[CONV25]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT27:%.*]] = shufflevector <4 x double> [[SPLAT_SPLATINSERT26]], <4 x double> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND28:%.*]] = icmp slt <4 x i64> [[TMP12]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT29:%.*]] = select <4 x i1> [[VECTOR_COND28]], <4 x double> [[SPLAT_SPLAT24]], <4 x double> [[SPLAT_SPLAT27]]
+// CHECK-NEXT:    [[TMP15:%.*]] = load <4 x i32>, ptr @four_ints, align 16
+// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr @some_int, align 4
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP16]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT31:%.*]] = shufflevector <4 x i32> [[SPLAT_SPLATINSERT30]], <4 x i32> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[TMP17:%.*]] = load i16, ptr @some_short, align 2
+// CHECK-NEXT:    [[CONV32:%.*]] = sext i16 [[TMP17]] to i32
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT33:%.*]] = insertelement <4 x i32> poison, i32 [[CONV32]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT34:%.*]] = shufflevector <4 x i32> [[SPLAT_SPLATINSERT33]], <4 x i32> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND35:%.*]] = icmp slt <4 x i32> [[TMP15]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT36:%.*]] = select <4 x i1> [[VECTOR_COND35]], <4 x i32> [[SPLAT_SPLAT31]], <4 x i32> [[SPLAT_SPLAT34]]
+// CHECK-NEXT:    ret void
+//
 void TwoScalarOps() {
   four_shorts ? some_short : some_short;
-  // CHECK: [[COND:%.+]] = load <4 x i16>
-  // CHECK: [[LHS:%.+]] = load i16
-  // CHECK: [[LHS_SPLAT_INSERT:%.+]] = insertelement <4 x i16> poison, i16 [[LHS]], i64 0
-  // CHECK: [[LHS_SPLAT:%.+]] = shufflevector <4 x i16> [[LHS_SPLAT_INSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-  // CHECK: [[RHS:%.+]] = load i16
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x i16> poison, i16 [[RHS]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x i16> [[RHS_SPLAT_INSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i16> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i16>
-  // CHECK: [[XOR:%.+]] = xor <4 x i16> [[SEXT]], splat (i16 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i16> [[RHS_SPLAT]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i16> [[LHS_SPLAT]], [[SEXT]]
-  // CHECK: = or <4 x i16> [[RHS_AND]], [[LHS_AND]]
 
   four_shorts ? some_ushort : some_ushort;
-  // CHECK: [[COND:%.+]] = load <4 x i16>
-  // CHECK: [[LHS:%.+]] = load i16
-  // CHECK: [[LHS_SPLAT_INSERT:%.+]] = insertelement <4 x i16> poison, i16 [[LHS]], i64 0
-  // CHECK: [[LHS_SPLAT:%.+]] = shufflevector <4 x i16> [[LHS_SPLAT_INSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-  // CHECK: [[RHS:%.+]] = load i16
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x i16> poison, i16 [[RHS]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x i16> [[RHS_SPLAT_INSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i16> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i16>
-  // CHECK: [[XOR:%.+]] = xor <4 x i16> [[SEXT]], splat (i16 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i16> [[RHS_SPLAT]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i16> [[LHS_SPLAT]], [[SEXT]]
-  // CHECK: = or <4 x i16> [[RHS_AND]], [[LHS_AND]]
 
   four_ints ? some_ushort : some_short;
-  // CHECK: [[COND:%.+]] = load <4 x i32>
-  // CHECK: [[LHS:%.+]] = load i16
-  // CHECK: [[LHS_ZEXT:%.+]] = zext i16 [[LHS]] to i32
-  // CHECK: [[LHS_SPLAT_INSERT:%.+]] = insertelement <4 x i32> poison, i32 [[LHS_ZEXT]], i64 0
-  // CHECK: [[LHS_SPLAT:%.+]] = shufflevector <4 x i32> [[LHS_SPLAT_INSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-  // CHECK: [[RHS:%.+]] = load i16
-  // CHECK: [[RHS_SEXT:%.+]] = sext i16 [[RHS]] to i32
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x i32> poison, i32 [[RHS_SEXT]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x i32> [[RHS_SPLAT_INSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i32> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i32>
-  // CHECK: [[XOR:%.+]] = xor <4 x i32> [[SEXT]], splat (i32 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i32> [[RHS_SPLAT]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i32> [[LHS_SPLAT]], [[SEXT]]
-  // CHECK: = or <4 x i32> [[RHS_AND]], [[LHS_AND]]
 
   four_ints ? some_int : some_float;
-  // CHECK: [[COND:%.+]] = load <4 x i32>
-  // CHECK: [[LHS:%.+]] = load i32
-  // CHECK: [[LHS_CONV:%.+]] = sitofp i32 [[LHS]] to float
-  // CHECK: [[LHS_SPLAT_INSERT:%.+]] = insertelement <4 x float> poison, float [[LHS_CONV]], i64 0
-  // CHECK: [[LHS_SPLAT:%.+]] = shufflevector <4 x float> [[LHS_SPLAT_INSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-  // CHECK: [[RHS:%.+]] = load float
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x float> poison, float [[RHS]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x float> [[RHS_SPLAT_INSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i32> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i32>
-  // CHECK: [[XOR:%.+]] = xor <4 x i32> [[SEXT]], splat (i32 -1)
-  // CHECK: [[RHS_CAST:%.+]] = bitcast <4 x float> [[RHS_SPLAT]] to <4 x i32>
-  // CHECK: [[LHS_CAST:%.+]] = bitcast <4 x float> [[LHS_SPLAT]] to <4 x i32>
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i32> [[RHS_CAST]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i32> [[LHS_CAST]], [[SEXT]]
-  // CHECK: = or <4 x i32> [[RHS_AND]], [[LHS_AND]]
 
   four_ll ? some_double : some_ll;
-  // CHECK: [[COND:%.+]] = load <4 x i64>
-  // CHECK: [[LHS:%.+]] = load double
-  // CHECK: [[LHS_SPLAT_INSERT:%.+]] = insertelement <4 x double> poison, double [[LHS]], i64 0
-  // CHECK: [[LHS_SPLAT:%.+]] = shufflevector <4 x double> [[LHS_SPLAT_INSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-  // CHECK: [[RHS:%.+]] = load i64
-  // CHECK: [[RHS_CONV:%.+]] = sitofp i64 [[RHS]] to double
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x double> poison, double [[RHS_CONV]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x double> [[RHS_SPLAT_INSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i64> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i64>
-  // CHECK: [[XOR:%.+]] = xor <4 x i64> [[SEXT]], splat (i64 -1)
-  // CHECK: [[RHS_CAST:%.+]] = bitcast <4 x double> [[RHS_SPLAT]] to <4 x i64>
-  // CHECK: [[LHS_CAST:%.+]] = bitcast <4 x double> [[LHS_SPLAT]] to <4 x i64>
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i64> [[RHS_CAST]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i64> [[LHS_CAST]], [[SEXT]]
-  // CHECK: = or <4 x i64> [[RHS_AND]], [[LHS_AND]]
 
   four_ints ? some_int : some_short;
-  // CHECK: [[COND:%.+]] = load <4 x i32>
-  // CHECK: [[LHS:%.+]] = load i32
-  // CHECK: [[LHS_SPLAT_INSERT:%.+]] = insertelement <4 x i32> poison, i32 [[LHS]], i64 0
-  // CHECK: [[LHS_SPLAT:%.+]] = shufflevector <4 x i32> [[LHS_SPLAT_INSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-  // CHECK: [[RHS:%.+]] = load i16
-  // CHECK: [[RHS_SEXT:%.+]] = sext i16 [[RHS]] to i32
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x i32> poison, i32 [[RHS_SEXT]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x i32> [[RHS_SPLAT_INSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i32> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i32>
-  // CHECK: [[XOR:%.+]] = xor <4 x i32> [[SEXT]], splat (i32 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i32> [[RHS_SPLAT]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i32> [[LHS_SPLAT]], [[SEXT]]
-  // CHECK: = or <4 x i32> [[RHS_AND]], [[LHS_AND]]
 }
 
-// CHECK: OneScalarOp
+// CHECK-LABEL: define dso_local void @_Z11OneScalarOpv(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr @four_ints, align 16
+// CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @four_ints, align 16
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @some_int, align 4
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <4 x i32> [[SPLAT_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND:%.*]] = icmp slt <4 x i32> [[TMP0]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT:%.*]] = select <4 x i1> [[VECTOR_COND]], <4 x i32> [[TMP1]], <4 x i32> [[SPLAT_SPLAT]]
+// CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr @four_ints, align 16
+// CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr @four_ints, align 16
+// CHECK-NEXT:    [[VECTOR_COND1:%.*]] = icmp slt <4 x i32> [[TMP3]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT2:%.*]] = select <4 x i1> [[VECTOR_COND1]], <4 x i32> [[TMP4]], <4 x i32> splat (i32 5)
+// CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @four_ints, align 16
+// CHECK-NEXT:    [[TMP6:%.*]] = load <4 x float>, ptr @four_floats, align 16
+// CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr @some_float, align 4
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT3:%.*]] = insertelement <4 x float> poison, float [[TMP7]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT4:%.*]] = shufflevector <4 x float> [[SPLAT_SPLATINSERT3]], <4 x float> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND5:%.*]] = icmp slt <4 x i32> [[TMP5]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT6:%.*]] = select <4 x i1> [[VECTOR_COND5]], <4 x float> [[TMP6]], <4 x float> [[SPLAT_SPLAT4]]
+// CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i64>, ptr @four_ll, align 32
+// CHECK-NEXT:    [[TMP9:%.*]] = load <4 x double>, ptr @four_doubles, align 32
+// CHECK-NEXT:    [[VECTOR_COND7:%.*]] = icmp slt <4 x i64> [[TMP8]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT8:%.*]] = select <4 x i1> [[VECTOR_COND7]], <4 x double> [[TMP9]], <4 x double> splat (double 6.000000e+00)
+// CHECK-NEXT:    [[TMP10:%.*]] = load <4 x i64>, ptr @four_ll, align 32
+// CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i64>, ptr @four_ll, align 32
+// CHECK-NEXT:    [[VECTOR_COND9:%.*]] = icmp slt <4 x i64> [[TMP10]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT10:%.*]] = select <4 x i1> [[VECTOR_COND9]], <4 x i64> [[TMP11]], <4 x i64> splat (i64 6)
+// CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i64>, ptr @four_ll, align 32
+// CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i64>, ptr @four_ll, align 32
+// CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr @some_int, align 4
+// CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[TMP14]] to i64
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT11:%.*]] = insertelement <4 x i64> poison, i64 [[CONV]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT12:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT11]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND13:%.*]] = icmp slt <4 x i64> [[TMP12]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT14:%.*]] = select <4 x i1> [[VECTOR_COND13]], <4 x i64> [[TMP13]], <4 x i64> [[SPLAT_SPLAT12]]
+// CHECK-NEXT:    [[TMP15:%.*]] = load <4 x i64>, ptr @four_ll, align 32
+// CHECK-NEXT:    [[TMP16:%.*]] = load <4 x i64>, ptr @four_ll, align 32
+// CHECK-NEXT:    [[TMP17:%.*]] = load i64, ptr @some_ll, align 8
+// CHECK-NEXT:    [[SPLAT_SPLATINSERT15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP17]], i64 0
+// CHECK-NEXT:    [[SPLAT_SPLAT16:%.*]] = shufflevector <4 x i64> [[SPLAT_SPLATINSERT15]], <4 x i64> poison, <4 x i32> zeroinitializer
+// CHECK-NEXT:    [[VECTOR_COND17:%.*]] = icmp slt <4 x i64> [[TMP15]], zeroinitializer
+// CHECK-NEXT:    [[VECTOR_SELECT18:%.*]] = select <4 x i1> [[VECTOR_COND17]], <4 x i64> [[TMP16]], <4 x i64> [[SPLAT_SPLAT16]]
+// CHECK-NEXT:    ret void
+//
 void OneScalarOp() {
   four_ints ? four_ints : some_int;
-  // CHECK: [[COND:%.+]] = load <4 x i32>
-  // CHECK: [[LHS:%.+]] = load <4 x i32>
-  // CHECK: [[RHS:%.+]] = load i32
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x i32> poison, i32 [[RHS]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x i32> [[RHS_SPLAT_INSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i32> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i32>
-  // CHECK: [[XOR:%.+]] = xor <4 x i32> [[SEXT]], splat (i32 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i32> [[RHS_SPLAT]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i32> [[LHS]], [[SEXT]]
-  // CHECK: = or <4 x i32> [[RHS_AND]], [[LHS_AND]]
 
   four_ints ? four_ints : 5;
-  // CHECK: [[COND:%.+]] = load <4 x i32>
-  // CHECK: [[LHS:%.+]] = load <4 x i32>
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i32> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i32>
-  // CHECK: [[XOR:%.+]] = xor <4 x i32> [[SEXT]], splat (i32 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i32> splat (i32 5), [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i32> [[LHS]], [[SEXT]]
-  // CHECK: = or <4 x i32> [[RHS_AND]], [[LHS_AND]]
 
   four_ints ? four_floats : some_float;
-  // CHECK: [[COND:%.+]] = load <4 x i32>
-  // CHECK: [[LHS:%.+]] = load <4 x float>
-  // CHECK: [[RHS:%.+]] = load float
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x float> poison, float [[RHS]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x float> [[RHS_SPLAT_INSERT]], <4 x float> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i32> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i32>
-  // CHECK: [[XOR:%.+]] = xor <4 x i32> [[SEXT]], splat (i32 -1)
-  // CHECK: [[RHS_CAST:%.+]] = bitcast <4 x float> [[RHS_SPLAT]] to <4 x i32>
-  // CHECK: [[LHS_CAST:%.+]] = bitcast <4 x float> [[LHS]] to <4 x i32>
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i32> [[RHS_CAST]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i32> [[LHS_CAST]], [[SEXT]]
-  // CHECK: = or <4 x i32> [[RHS_AND]], [[LHS_AND]]
 
   four_ll ? four_doubles : 6.0;
-  // CHECK: [[COND:%.+]] = load <4 x i64>
-  // CHECK: [[LHS:%.+]] = load <4 x double>
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i64> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i64>
-  // CHECK: [[XOR:%.+]] = xor <4 x i64> [[SEXT]], splat (i64 -1)
-  // CHECK: [[LHS_CAST:%.+]] = bitcast <4 x double> [[LHS]] to <4 x i64>
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i64> splat (i64 4618441417868443648), [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i64> [[LHS_CAST]], [[SEXT]]
-  // CHECK: = or <4 x i64> [[RHS_AND]], [[LHS_AND]]
 
   four_ll ? four_ll : 6;
-  // CHECK: [[COND:%.+]] = load <4 x i64>
-  // CHECK: [[LHS:%.+]] = load <4 x i64>
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i64> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i64>
-  // CHECK: [[XOR:%.+]] = xor <4 x i64> [[SEXT]], splat (i64 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i64> splat (i64 6), [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i64> [[LHS]], [[SEXT]]
-  // CHECK: [[OR:%.+]] = or <4 x i64> [[RHS_AND]], [[LHS_AND]]
 
   four_ll ? four_ll : some_int;
-  // CHECK: [[COND:%.+]] = load <4 x i64>
-  // CHECK: [[LHS:%.+]] = load <4 x i64>
-  // CHECK: [[RHS:%.+]] = load i32
-  // CHECK: [[RHS_CONV:%.+]] = sext i32 [[RHS]] to i64
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x i64> poison, i64 [[RHS_CONV]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x i64> [[RHS_SPLAT_INSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i64> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i64>
-  // CHECK: [[XOR:%.+]] = xor <4 x i64> [[SEXT]], splat (i64 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i64> [[RHS_SPLAT]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i64> [[LHS]], [[SEXT]]
-  // CHECK: [[OR:%.+]] = or <4 x i64> [[RHS_AND]], [[LHS_AND]]
 
   four_ll ? four_ll : some_ll;
-  // CHECK: [[COND:%.+]] = load <4 x i64>
-  // CHECK: [[LHS:%.+]] = load <4 x i64>
-  // CHECK: [[RHS:%.+]] = load i64
-  // CHECK: [[RHS_SPLAT_INSERT:%.+]] = insertelement <4 x i64> poison, i64 [[RHS]], i64 0
-  // CHECK: [[RHS_SPLAT:%.+]] = shufflevector <4 x i64> [[RHS_SPLAT_INSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-  // CHECK: [[NEG:%.+]] = icmp slt <4 x i64> [[COND]], zeroinitializer
-  // CHECK: [[SEXT:%.+]] = sext <4 x i1> [[NEG]] to <4 x i64>
-  // CHECK: [[XOR:%.+]] = xor <4 x i64> [[SEXT]], splat (i64 -1)
-  // CHECK: [[RHS_AND:%.+]] = and <4 x i64> [[RHS_SPLAT]], [[XOR]]
-  // CHECK: [[LHS_AND:%.+]] = and <4 x i64> [[LHS]], [[SEXT]]
-  // CHECK: [[OR:%.+]] = or <4 x i64> [[RHS_AND]], [[LHS_AND]]
 }
diff --git a/clang/test/CodeGenCXX/debug-info-lambda-this.cpp b/clang/test/CodeGenCXX/lambda-this-2.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-lambda-this.cpp
rename to clang/test/CodeGenCXX/lambda-this-2.cpp
diff --git a/clang/test/CodeGenCXX/mangle-class-nttp.cpp b/clang/test/CodeGenCXX/mangle-class-nttp.cpp
index 12c81f2ba0514..536592c6a9308 100644
--- a/clang/test/CodeGenCXX/mangle-class-nttp.cpp
+++ b/clang/test/CodeGenCXX/mangle-class-nttp.cpp
@@ -27,12 +27,14 @@ template void f<B{nullptr}>();
 // CHECK: define weak_odr void @_Z1fIXtl1BLPKi32EEEEvv(
 // MSABI: define {{.*}} @"??$f@$2UB@@PEBH0CA@H0A@@@@YAXXZ"
 template void f<B{fold((int*)32)}>();
-#ifndef _WIN32
-// FIXME: On MS ABI, we mangle this the same as nullptr, despite considering a
-// null pointer and zero bitcast to a pointer to be distinct pointer values.
-// CHECK: define weak_odr void @_Z1fIXtl1BrcPKiLi0EEEEvv(
-template void f<B{fold(reinterpret_cast<int*>(0))}>();
-#endif
+
+// CHECK: define weak_odr void @_Z1fIXtl1BLPKi0ELi2EEEEvv(
+// MSABI: define {{.*}} @"??$f@$2UB@@PEBH0A@H01@@@YAXXZ"(
+template void f<B{fold(reinterpret_cast<int*>(0)), 2}>();
+
+// CHECK: define weak_odr void @_Z1fIXtl1BLPKi12EEEEvv(
+// MSABI: define {{.*}} @"??$f@$2UB@@PEBH0M@H0A@@@@YAXXZ"(
+template void f<B{fold(reinterpret_cast<int*>(12))}>();
 
 // Pointers to subobjects.
 struct Nested { union { int k; int arr[2]; }; } nested[2];
diff --git a/clang/test/CodeGenCXX/microsoft-abi-eh-cleanups.cpp b/clang/test/CodeGenCXX/microsoft-abi-eh-cleanups.cpp
index 5e2403b242fcb..a60781c164785 100644
--- a/clang/test/CodeGenCXX/microsoft-abi-eh-cleanups.cpp
+++ b/clang/test/CodeGenCXX/microsoft-abi-eh-cleanups.cpp
@@ -277,13 +277,13 @@ void f() {
 
 // WIN32-LIFETIME-LABEL: define dso_local void @"?f@lifetime_marker@@YAXXZ"()
 // WIN32-LIFETIME: %[[c:.*]] = alloca %"struct.lifetime_marker::C"
-// WIN32-LIFETIME: call void @llvm.lifetime.start.p0(i64 1, ptr %c)
+// WIN32-LIFETIME: call void @llvm.lifetime.start.p0(ptr %c)
 // WIN32-LIFETIME: invoke void @"?g@lifetime_marker@@YAXXZ"()
 // WIN32-LIFETIME-NEXT: to label %[[cont:[^ ]*]] unwind label %[[lpad0:[^ ]*]]
 //
 // WIN32-LIFETIME: [[cont]]
 // WIN32-LIFETIME: call x86_thiscallcc void @"??1C@lifetime_marker@@QAE@XZ"({{.*}})
-// WIN32-LIFETIME: call void @llvm.lifetime.end.p0(i64 1, ptr %[[c]])
+// WIN32-LIFETIME: call void @llvm.lifetime.end.p0(ptr %[[c]])
 //
 // WIN32-LIFETIME: [[lpad0]]
 // WIN32-LIFETIME-NEXT: cleanuppad
@@ -292,7 +292,7 @@ void f() {
 //
 // WIN32-LIFETIME: [[lpad1]]
 // WIN32-LIFETIME-NEXT: cleanuppad
-// WIN32-LIFETIME: call void @llvm.lifetime.end.p0(i64 1, ptr %[[c]])
+// WIN32-LIFETIME: call void @llvm.lifetime.end.p0(ptr %[[c]])
 }
 
 struct class_2 {
diff --git a/clang/test/CodeGenCXX/modules-vtable.cppm b/clang/test/CodeGenCXX/modules-vtable.cppm
index 6589b9f3c5d64..75f7598a63b06 100644
--- a/clang/test/CodeGenCXX/modules-vtable.cppm
+++ b/clang/test/CodeGenCXX/modules-vtable.cppm
@@ -1,4 +1,4 @@
-// REQUIRES: !system-windows
+// REQUIRES: !system-windows, !system-cygwin
 
 // RUN: rm -rf %t
 // RUN: split-file %s %t
diff --git a/clang/test/CodeGenCXX/pr70585.cppm b/clang/test/CodeGenCXX/pr70585.cppm
index ad4e13589d86e..d44a4f497285a 100644
--- a/clang/test/CodeGenCXX/pr70585.cppm
+++ b/clang/test/CodeGenCXX/pr70585.cppm
@@ -1,4 +1,4 @@
-// REQUIRES: !system-windows
+// REQUIRES: !system-windows, !system-cygwin
 
 // RUN: rm -rf %t
 // RUN: split-file %s %t
diff --git a/clang/test/CodeGenCXX/sret_cast_with_nonzero_alloca_as.cpp b/clang/test/CodeGenCXX/sret_cast_with_nonzero_alloca_as.cpp
index 320c712b665de..a1a6ada4f3fd6 100644
--- a/clang/test/CodeGenCXX/sret_cast_with_nonzero_alloca_as.cpp
+++ b/clang/test/CodeGenCXX/sret_cast_with_nonzero_alloca_as.cpp
@@ -10,16 +10,15 @@ struct X { int z[17]; };
 // CHECK-NEXT:    [[Y_ADDR:%.*]] = alloca i8, align 1, addrspace(5)
 // CHECK-NEXT:    [[X_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[X_ADDR]] to ptr
 // CHECK-NEXT:    [[Y_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[Y_ADDR]] to ptr
+// CHECK-NEXT:    [[AGG_RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[AGG_RESULT]] to ptr
 // CHECK-NEXT:    store i8 [[X]], ptr [[X_ADDR_ASCAST]], align 1
 // CHECK-NEXT:    store i8 [[Y]], ptr [[Y_ADDR_ASCAST]], align 1
 // CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[X_ADDR_ASCAST]], align 1
-// CHECK-NEXT:    [[AGG_RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[AGG_RESULT]] to ptr
 // CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[AGG_RESULT_ASCAST]], i64 1
 // CHECK-NEXT:    store i8 [[TMP0]], ptr [[ADD_PTR]], align 1
 // CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[Y_ADDR_ASCAST]], align 1
-// CHECK-NEXT:    [[AGG_RESULT_ASCAST1:%.*]] = addrspacecast ptr addrspace(5) [[AGG_RESULT]] to ptr
-// CHECK-NEXT:    [[ADD_PTR2:%.*]] = getelementptr inbounds i8, ptr [[AGG_RESULT_ASCAST1]], i64 2
-// CHECK-NEXT:    store i8 [[TMP1]], ptr [[ADD_PTR2]], align 1
+// CHECK-NEXT:    [[ADD_PTR1:%.*]] = getelementptr inbounds i8, ptr [[AGG_RESULT_ASCAST]], i64 2
+// CHECK-NEXT:    store i8 [[TMP1]], ptr [[ADD_PTR1]], align 2
 // CHECK-NEXT:    ret void
 //
 X foo(char x, char y) {
diff --git a/clang/test/CodeGenCXX/stack-reuse-exceptions.cpp b/clang/test/CodeGenCXX/stack-reuse-exceptions.cpp
index e036bcaf729a6..27123a689c303 100644
--- a/clang/test/CodeGenCXX/stack-reuse-exceptions.cpp
+++ b/clang/test/CodeGenCXX/stack-reuse-exceptions.cpp
@@ -20,32 +20,32 @@ struct NontrivialDtor {
 
 // CHECK-LABEL: define{{.*}} void @_Z33cleanupsAreEmittedWithoutTryCatchv
 void cleanupsAreEmittedWithoutTryCatch() {
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[CLEAN:.*]])
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[T1:.*]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[CLEAN:.*]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[T1:.*]])
 // CHECK-NEXT: invoke void @_Z8getLargev
 // CHECK-NEXT:     to label %[[CONT:[^ ]+]] unwind label %[[LPAD:[^ ]+]]
 //
 // CHECK: [[CONT]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T1]])
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[T2:.*]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T1]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[T2:.*]])
 // CHECK-NEXT: invoke void @_Z8getLargev
 // CHECK-NEXT:     to label %[[CONT2:[^ ]+]] unwind label %[[LPAD2:.+]]
 //
 // CHECK: [[CONT2]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T2]])
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[CLEAN]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T2]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[CLEAN]])
 // CHECK: ret void
 //
 // CHECK: [[LPAD]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T1]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T1]])
 // CHECK: br label %[[EHCLEANUP:.+]]
 //
 // CHECK: [[LPAD2]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T2]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T2]])
 // CHECK: br label %[[EHCLEANUP]]
 //
 // CHECK: [[EHCLEANUP]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[CLEAN]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[CLEAN]])
 
   NontrivialDtor clean;
 
@@ -55,27 +55,27 @@ void cleanupsAreEmittedWithoutTryCatch() {
 
 // CHECK-LABEL: define{{.*}} void @_Z30cleanupsAreEmittedWithTryCatchv
 void cleanupsAreEmittedWithTryCatch() {
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[CLEAN:.*]])
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[T1:.*]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[CLEAN:.*]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[T1:.*]])
 // CHECK-NEXT: invoke void @_Z8getLargev
 // CHECK-NEXT:     to label %[[CONT:[^ ]+]] unwind label %[[LPAD:[^ ]+]]
 //
 // CHECK: [[CONT]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T1]])
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[T2:.*]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T1]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[T2:.*]])
 // CHECK-NEXT: invoke void @_Z8getLargev
 // CHECK-NEXT:     to label %[[CONT2:[^ ]+]] unwind label %[[LPAD2:.+]]
 //
 // CHECK: [[CONT2]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T2]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T2]])
 // CHECK: br label %[[TRY_CONT:.+]]
 //
 // CHECK: [[LPAD]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T1]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T1]])
 // CHECK: br label %[[CATCH:.+]]
 //
 // CHECK: [[LPAD2]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T2]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T2]])
 // CHECK: br label %[[CATCH]]
 //
 // CHECK: [[CATCH]]:
@@ -84,13 +84,13 @@ void cleanupsAreEmittedWithTryCatch() {
 // CHECK-NEXT: to label %[[TRY_CONT]] unwind label %[[OUTER_LPAD:.+]]
 //
 // CHECK: [[TRY_CONT]]:
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[T_OUTER:.*]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[T_OUTER:.*]])
 // CHECK-NEXT: invoke void @_Z8getLargev
 // CHECK-NEXT:     to label %[[OUTER_CONT:[^ ]+]] unwind label %[[OUTER_LPAD2:.+]]
 //
 // CHECK: [[OUTER_CONT]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T_OUTER]])
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[CLEAN]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T_OUTER]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[CLEAN]])
 // CHECK: ret void
 //
 // CHECK: [[OUTER_LPAD]]:
@@ -98,11 +98,11 @@ void cleanupsAreEmittedWithTryCatch() {
 // CHECK: br label %[[EHCLEANUP:.+]]
 //
 // CHECK: [[OUTER_LPAD2]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T_OUTER]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T_OUTER]])
 // CHECK: br label %[[EHCLEANUP]]
 //
 // CHECK: [[EHCLEANUP]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[CLEAN]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[CLEAN]])
 
   NontrivialDtor clean;
 
@@ -116,44 +116,44 @@ void cleanupsAreEmittedWithTryCatch() {
 
 // CHECK-LABEL: define{{.*}} void @_Z39cleanupInTryHappensBeforeCleanupInCatchv
 void cleanupInTryHappensBeforeCleanupInCatch() {
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[T1:.*]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[T1:.*]])
 // CHECK-NEXT: invoke void @_Z8getLargev
 // CHECK-NEXT:     to label %[[CONT:[^ ]+]] unwind label %[[LPAD:[^ ]+]]
 //
 // CHECK: [[CONT]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T1]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T1]])
 // CHECK: br label %[[TRY_CONT]]
 //
 // CHECK: [[LPAD]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T1]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T1]])
 // CHECK: br i1 {{[^,]+}}, label %[[CATCH_INT_MATCH:[^,]+]], label %[[CATCH_ALL:.+]]
 //
 // CHECK: [[CATCH_INT_MATCH]]:
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[T2:.*]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[T2:.*]])
 // CHECK-NEXT: invoke void @_Z8getLargev
 // CHECK-NEXT:     to label %[[CATCH_INT_CONT:[^ ]+]] unwind label %[[CATCH_INT_LPAD:[^ ]+]]
 //
 // CHECK: [[CATCH_INT_CONT]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T2]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T2]])
 // CHECK: br label %[[TRY_CONT]]
 //
 // CHECK: [[TRY_CONT]]:
 // CHECK: ret void
 //
 // CHECK: [[CATCH_ALL]]:
-// CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr nonnull %[[T3:.*]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %[[T3:.*]])
 // CHECK-NEXT: invoke void @_Z8getLargev
 // CHECK-NEXT:     to label %[[CATCH_ALL_CONT:[^ ]+]] unwind label %[[CATCH_ALL_LPAD:[^ ]+]]
 //
 // CHECK: [[CATCH_ALL_CONT]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T3]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T3]])
 // CHECK: br label %[[TRY_CONT]]
 //
 // CHECK: [[CATCH_ALL_LPAD]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T3]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T3]])
 //
 // CHECK: [[CATCH_INT_LPAD]]:
-// CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr nonnull %[[T2]])
+// CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %[[T2]])
 // CHECK-NOT: call void @llvm.lifetime
 
   try {
diff --git a/clang/test/CodeGenCXX/stack-reuse-miscompile.cpp b/clang/test/CodeGenCXX/stack-reuse-miscompile.cpp
index 50a8d167f5b77..67fa9f9c9cd98 100644
--- a/clang/test/CodeGenCXX/stack-reuse-miscompile.cpp
+++ b/clang/test/CodeGenCXX/stack-reuse-miscompile.cpp
@@ -28,12 +28,12 @@ const char * f(S s)
 //
 // FIXME: We could defer starting the lifetime of the return object of concat
 // until the call.
-// CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr [[T1]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr [[T1]])
 //
-// CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr [[T2]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr [[T2]])
 // CHECK: [[T4:%.*]] = call noundef ptr @_ZN1TC1EPKc(ptr {{[^,]*}} [[T2]], ptr noundef @.str)
 //
-// CHECK: call void @llvm.lifetime.start.p0(i64 16, ptr [[T3]])
+// CHECK: call void @llvm.lifetime.start.p0(ptr [[T3]])
 // CHECK: [[T5:%.*]] = call noundef ptr @_ZN1TC1E1S(ptr {{[^,]*}} [[T3]], [2 x i32] %{{.*}})
 //
 // CHECK: call void @_ZNK1T6concatERKS_(ptr dead_on_unwind writable sret(%class.T) align 4 [[T1]], ptr {{[^,]*}} [[T2]], ptr noundef nonnull align 4 dereferenceable(16) [[T3]])
diff --git a/clang/test/CodeGenHIP/hip-cumode.hip b/clang/test/CodeGenHIP/hip-cumode.hip
index 1aa1ca7a1a7ee..61fd53c644e8c 100644
--- a/clang/test/CodeGenHIP/hip-cumode.hip
+++ b/clang/test/CodeGenHIP/hip-cumode.hip
@@ -5,14 +5,20 @@
 // RUN: %clang -S -o - --offload-arch=gfx906 --cuda-device-only -nogpuinc -nogpulib -mcumode \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=NOWGP %s
 // RUN: %clang -S -o - --offload-arch=gfx906 --cuda-device-only -nogpuinc -nogpulib -mno-cumode \
-// RUN:   %s 2>&1 | FileCheck --check-prefixes=NOWGP,WARN-CUMODE %s
+// RUN:   %s 2>&1 | FileCheck -DOFFLOAD_ARCH=gfx906 --check-prefixes=NOWGP,WARN-CUMODE %s
 // RUN: %clang -S -o - --offload-arch=gfx1030 --cuda-device-only -nogpuinc -nogpulib \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-OFF %s
 // RUN: %clang -S -o - --offload-arch=gfx1030 --cuda-device-only -nogpuinc -nogpulib -mcumode \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-ON %s
 // RUN: %clang -S -o - --offload-arch=gfx1030 --cuda-device-only -nogpuinc -nogpulib -mno-cumode \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-OFF %s
-// WARN-CUMODE: warning: ignoring '-mno-cumode' option as it is not currently supported for processor 'gfx906' [-Woption-ignored]
+// RUN: %clang -S -o - --offload-arch=gfx1250 --cuda-device-only -nogpuinc -nogpulib \
+// RUN:   %s 2>&1 | FileCheck --check-prefix=NOWGP %s
+// RUN: %clang -S -o - --offload-arch=gfx1250 --cuda-device-only -nogpuinc -nogpulib -mcumode \
+// RUN:   %s 2>&1 | FileCheck --check-prefix=NOWGP %s
+// RUN: %clang -S -o - --offload-arch=gfx1250 --cuda-device-only -nogpuinc -nogpulib -mno-cumode \
+// RUN:   %s 2>&1 | FileCheck -DOFFLOAD_ARCH=gfx1250 --check-prefixes=NOWGP,WARN-CUMODE %s
+// WARN-CUMODE: warning: ignoring '-mno-cumode' option as it is not currently supported for processor '[[OFFLOAD_ARCH]]' [-Woption-ignored]
 // NOWGP-NOT: .amdhsa_workgroup_processor_mode
 // CUMODE-ON: .amdhsa_workgroup_processor_mode 0
 // CUMODE-OFF: .amdhsa_workgroup_processor_mode 1
diff --git a/clang/test/CodeGenHIP/store-addr-space.hip b/clang/test/CodeGenHIP/store-addr-space.hip
new file mode 100644
index 0000000000000..6103edba46274
--- /dev/null
+++ b/clang/test/CodeGenHIP/store-addr-space.hip
@@ -0,0 +1,46 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --functions "bar" --version 5
+// REQUIRES: amdgpu-registered-target
+// RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -x hip -emit-llvm -fcuda-is-device \
+// RUN:   -o - %s | FileCheck --check-prefix=AMDGCN --enable-var-scope %s
+
+struct Foo {
+  unsigned long long val;
+//
+  __attribute__((device)) inline Foo() { val = 0; }
+  __attribute__((device)) inline Foo(const Foo &src) { val = src.val; }
+  __attribute__((device)) inline Foo(const volatile Foo &src) { val = src.val; }
+};
+
+// AMDGCN-LABEL: define dso_local void @_Z3barPK3Foo(
+// AMDGCN-SAME: ptr addrspace(5) dead_on_unwind noalias writable sret([[STRUCT_FOO:%.*]]) align 8 [[AGG_RESULT:%.*]], ptr noundef [[SRC_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+// AMDGCN-NEXT:  [[ENTRY:.*:]]
+// AMDGCN-NEXT:    [[RESULT_PTR:%.*]] = alloca ptr addrspace(5), align 4, addrspace(5)
+// AMDGCN-NEXT:    [[SRC_PTR_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// AMDGCN-NEXT:    [[DST:%.*]] = alloca [[UNION_ANON:%.*]], align 8, addrspace(5)
+// AMDGCN-NEXT:    [[RESULT_PTR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RESULT_PTR]] to ptr
+// AMDGCN-NEXT:    [[SRC_PTR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[SRC_PTR_ADDR]] to ptr
+// AMDGCN-NEXT:    [[AGG_RESULT_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[AGG_RESULT]] to ptr
+// AMDGCN-NEXT:    [[DST_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[DST]] to ptr
+// AMDGCN-NEXT:    store ptr addrspace(5) [[AGG_RESULT]], ptr [[RESULT_PTR_ASCAST]], align 4
+// AMDGCN-NEXT:    store ptr [[SRC_PTR]], ptr [[SRC_PTR_ADDR_ASCAST]], align 8
+// AMDGCN-NEXT:    call void @_ZN3FooC1Ev(ptr noundef nonnull align 8 dereferenceable(8) [[AGG_RESULT_ASCAST]]) #[[ATTR1:[0-9]+]]
+// AMDGCN-NEXT:    store ptr [[AGG_RESULT_ASCAST]], ptr [[DST_ASCAST]], align 8
+// AMDGCN-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[SRC_PTR_ADDR_ASCAST]], align 8
+// AMDGCN-NEXT:    [[VAL:%.*]] = getelementptr inbounds nuw [[STRUCT_FOO]], ptr [[TMP0]], i32 0, i32 0
+// AMDGCN-NEXT:    [[TMP1:%.*]] = load i64, ptr [[VAL]], align 8
+// AMDGCN-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[DST_ASCAST]], align 8
+// AMDGCN-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i64 0
+// AMDGCN-NEXT:    store i64 [[TMP1]], ptr [[ARRAYIDX]], align 8
+// AMDGCN-NEXT:    ret void
+//
+__attribute__((device)) Foo bar(const Foo *const src_ptr) {
+  Foo result;
+
+  union {
+    Foo* const ptr;
+    unsigned long long * const ptr64;
+  } dst = {&result};
+
+  dst.ptr64[0] = src_ptr->val;
+  return result;
+}
diff --git a/clang/test/CodeGenHLSL/builtins/dot2add.hlsl b/clang/test/CodeGenHLSL/builtins/dot2add.hlsl
index c345e17476e08..7c3a48e476306 100644
--- a/clang/test/CodeGenHLSL/builtins/dot2add.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/dot2add.hlsl
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -finclude-default-header -fnative-half-type -triple \
-// RUN:   dxil-pc-shadermodel6.3-compute %s -emit-llvm -o - | \
+// RUN:   dxil-pc-shadermodel6.4-compute %s -emit-llvm -o - | \
 // RUN:   FileCheck %s --check-prefixes=CHECK,CHECK-DXIL
 // RUN: %clang_cc1 -finclude-default-header -fnative-half-type -triple \
 // RUN:   spirv-pc-vulkan-compute %s -emit-llvm -o - | \
diff --git a/clang/test/CodeGenHLSL/convergence/global_array.hlsl b/clang/test/CodeGenHLSL/convergence/global_array.hlsl
index 030ba48a5f170..c594e3a3e62ae 100644
--- a/clang/test/CodeGenHLSL/convergence/global_array.hlsl
+++ b/clang/test/CodeGenHLSL/convergence/global_array.hlsl
@@ -6,9 +6,15 @@
 
 // CHECK: [[loop_entry]]:
 // CHECK: [[loop_token:%.*]] = call token @llvm.experimental.convergence.loop() [ "convergencectrl"(token [[entry_token]]) ]
-// CHECK: call void {{.*}} [ "convergencectrl"(token [[loop_token]]) ]
+// CHECK: call spir_func void {{.*}} [ "convergencectrl"(token [[loop_token]]) ]
 // CHECK: br i1 {{%.*}} label {{%.*}} label %[[loop_entry]]
-RWBuffer<float> e[2];
+
+struct S {
+    int i;
+    S() { i = 10; }
+};
+
+static S s[2];
 
 [numthreads(4,1,1)]
 void main() {
diff --git a/clang/test/CodeGenHLSL/resources/res-array-global-dyn-index.hlsl b/clang/test/CodeGenHLSL/resources/res-array-global-dyn-index.hlsl
new file mode 100644
index 0000000000000..5b62452ef9844
--- /dev/null
+++ b/clang/test/CodeGenHLSL/resources/res-array-global-dyn-index.hlsl
@@ -0,0 +1,29 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-compute -finclude-default-header \
+// RUN:   -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+// CHECK: @[[BufA:.*]] = private unnamed_addr constant [2 x i8] c"A\00", align 1
+
+RWBuffer<float> A[4][3] : register(u2);
+RWStructuredBuffer<float> Out;
+
+// Make sure A[GI.x][GI.y] is translated to a RWBuffer<float> constructor call with range 12 and dynamically calculated index
+
+// NOTE:
+// Constructor call for explicit binding has "jjij" in the mangled name and the arguments are (register, space, range_size, index, name).
+
+// CHECK: define internal void @_Z4mainDv3_j(<3 x i32> noundef %GI)
+// CHECK: %[[GI_alloca:.*]] = alloca <3 x i32>, align 16
+// CHECK: %[[Tmp0:.*]] = alloca %"class.hlsl::RWBuffer
+// CHECK: store <3 x i32> %GI, ptr %[[GI_alloca]]
+
+// CHECK: %[[GI:.*]] = load <3 x i32>, ptr %[[GI_alloca]], align 16
+// CHECK: %[[GI_y:.*]] = extractelement <3 x i32> %[[GI]], i32 1
+// CHECK: %[[GI:.*]] = load <3 x i32>, ptr %[[GI_alloca]], align 16
+// CHECK: %[[GI_x:.*]] = extractelement <3 x i32> %[[GI]], i32 0
+// CHECK: %[[Tmp1:.*]] = mul i32 %[[GI_x]], 3
+// CHECK: %[[Index:.*]] = add i32 %[[GI_y]], %[[Tmp1]]
+// CHECK: call void @_ZN4hlsl8RWBufferIfEC1EjjijPKc(ptr {{.*}} %[[Tmp0]], i32 noundef 2, i32 noundef 0, i32 noundef 12, i32 noundef %[[Index]], ptr noundef @A.str)
+[numthreads(4,1,1)]
+void main(uint3 GI : SV_GroupThreadID) {
+  Out[0] = A[GI.x][GI.y][0];
+}
diff --git a/clang/test/CodeGenHLSL/resources/res-array-global-multi-dim.hlsl b/clang/test/CodeGenHLSL/resources/res-array-global-multi-dim.hlsl
new file mode 100644
index 0000000000000..8d664dad58153
--- /dev/null
+++ b/clang/test/CodeGenHLSL/resources/res-array-global-multi-dim.hlsl
@@ -0,0 +1,46 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-compute -finclude-default-header \
+// RUN:   -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+// RUN: %clang_cc1 -finclude-default-header -triple spirv-unknown-vulkan-compute \
+// RUN:   -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+// CHECK: @[[BufB:.*]] = private unnamed_addr constant [2 x i8] c"B\00", align 1
+// CHECK: @[[BufC:.*]] = private unnamed_addr constant [2 x i8] c"C\00", align 1
+// CHECK: @[[BufD:.*]] = private unnamed_addr constant [2 x i8] c"D\00", align 1
+
+RWBuffer<float> B[4][4] : register(u2);
+RWBuffer<int> C[2][2][5] : register(u10, space1);
+
+typedef RWBuffer<uint> RWBufferArrayTenByFive[10][5]; // test typedef for the resource array type
+RWBufferArrayTenByFive D; // implicit binding -> u18, space0
+
+RWStructuredBuffer<float> Out;
+
+[numthreads(4,1,1)]
+void main() {
+  // CHECK: define internal{{.*}} void @_Z4mainv()
+  // CHECK: %[[Tmp0:.*]] = alloca %"class.hlsl::RWBuffer
+  // CHECK: %[[Tmp1:.*]] = alloca %"class.hlsl::RWBuffer
+  // CHECK: %[[Tmp2:.*]] = alloca %"class.hlsl::RWBuffer
+  // CHECK: %[[Tmp3:.*]] = alloca %"class.hlsl::RWBuffer
+
+  // NOTE:
+  // Constructor call for explicit binding has "jjij" in the mangled name and the arguments are (register, space, range_size, index, name).
+  // For implicit binding the constructor has "jijj" in the mangled name and the arguments are (space, range_size, index, order_id, name).
+  // The range_size can be -1 for unbounded arrays, and that is the only signed int in the signature.
+  // The order_id argument is a sequential number that is assigned to resources with implicit binding and corresponds to the order in which 
+  // the resources were declared. It is needed because implicit bindings are assigned later on in an LLVM pass that needs to know the order
+  // of the resource declarations.
+
+  // Make sure that B[3][2] is translated to a RWBuffer<float> constructor call for explicit binding (u2, space0) with range 16 and index 14
+  // CHECK: call void @_ZN4hlsl8RWBufferIfEC1EjjijPKc(ptr {{.*}} %[[Tmp0]], i32 noundef 2, i32 noundef 0, i32 noundef 16, i32 noundef 14, ptr noundef @[[BufB]])
+
+  // Make sure that C[1][0][3] is translated to a RWBuffer<int> constructor call for explicit binding (u10, space1) with range 20 and index 13
+  // CHECK: call void @_ZN4hlsl8RWBufferIiEC1EjjijPKc(ptr {{.*}} %[[Tmp1]], i32 noundef 10, i32 noundef 1, i32 noundef 20, i32 noundef 13, ptr noundef @[[BufC]])
+
+  // Make sure that D[9][2] is translated to a RWBuffer<uint> constructor call for implicit binding (u18, space0) with range 50 and index 47
+  // CHECK: call void @_ZN4hlsl8RWBufferIjEC1EjijjPKc(ptr {{.*}} %[[Tmp2]], i32 noundef 0, i32 noundef 50, i32 noundef 47, i32 noundef 0, ptr noundef @[[BufD]])
+
+  // Make sure that the second B[3][2] is translated to the same a RWBuffer<float> constructor call as the first B[3][2] subscript
+  // CHECK: call void @_ZN4hlsl8RWBufferIfEC1EjjijPKc(ptr {{.*}} %[[Tmp3]], i32 noundef 2, i32 noundef 0, i32 noundef 16, i32 noundef 14, ptr noundef @[[BufB]])
+  Out[0] =  B[3][2][0] + (float)C[1][0][3][0] + (float)D[9][2][0] + B[3][2][1];
+}
diff --git a/clang/test/CodeGenHLSL/resources/res-array-global.hlsl b/clang/test/CodeGenHLSL/resources/res-array-global.hlsl
new file mode 100644
index 0000000000000..595ea818e913b
--- /dev/null
+++ b/clang/test/CodeGenHLSL/resources/res-array-global.hlsl
@@ -0,0 +1,75 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-compute -finclude-default-header \
+// RUN:   -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s -check-prefixes=CHECK,DXIL
+// RUN: %clang_cc1 -finclude-default-header -triple spirv-unknown-vulkan-compute \
+// RUN:   -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s -check-prefixes=CHECK,SPV
+
+// CHECK: @[[BufA:.*]] = private unnamed_addr constant [2 x i8] c"A\00", align 1
+// CHECK: @[[BufB:.*]] = private unnamed_addr constant [2 x i8] c"B\00", align 1
+// CHECK: @[[BufC:.*]] = private unnamed_addr constant [2 x i8] c"C\00", align 1
+// CHECK: @[[BufD:.*]] = private unnamed_addr constant [2 x i8] c"D\00", align 1
+// CHECK: @[[BufE:.*]] = private unnamed_addr constant [2 x i8] c"E\00", align 1
+
+// different explicit binding for DXIL and SPIR-V
+[[vk::binding(12, 2)]]
+RWBuffer<float> A[4] : register(u10, space1);
+
+[[vk::binding(13)]] // SPIR-V explicit binding 13, set 0
+RWBuffer<int> B[5]; // DXIL implicit binding in space0
+
+// same explicit binding for both DXIL and SPIR-V
+// (SPIR-V takes the binding from register annotation if there is no vk::binding attribute))
+RWBuffer<int> C[3] : register(u2);
+
+// implicit binding for both DXIL and SPIR-V in space/set 0 
+RWBuffer<double> D[10];
+
+// implicit binding for both DXIL and SPIR-V with specified space/set 0 
+RWBuffer<uint> E[15] : register(space2);
+
+RWStructuredBuffer<float> Out;
+
+[numthreads(4,1,1)]
+void main() {
+  // CHECK: define internal{{.*}} void @_Z4mainv()
+  // CHECK: %[[Tmp0:.*]] = alloca %"class.hlsl::RWBuffer
+  // CHECK: %[[Tmp1:.*]] = alloca %"class.hlsl::RWBuffer
+  // CHECK: %[[Tmp2:.*]] = alloca %"class.hlsl::RWBuffer
+  // CHECK: %[[Tmp3:.*]] = alloca %"class.hlsl::RWBuffer
+  // CHECK: %[[Tmp4:.*]] = alloca %"class.hlsl::RWBuffer
+
+  // NOTE:
+  // Constructor call for explicit binding has "jjij" in the mangled name and the arguments are (register, space, range_size, index, name).
+  // For implicit binding the constructor has "jijj" in the mangled name and the arguments are (space, range_size, index, order_id, name).
+  // The range_size can be -1 for unbounded arrays, and that is the only signed int in the signature.
+  // The order_id argument is a sequential number that is assigned to resources with implicit binding and corresponds to the order in which 
+  // the resources were declared. It is needed because implicit bindings are assigned later on in an LLVM pass that needs to know the order
+  // of the resource declarations.
+
+  // Make sure A[2] is translated to a RWBuffer<float> constructor call with range 4 and index 2
+  // and DXIL explicit binding (u10, space1)
+  // and SPIR-V explicit binding (binding 12, set 2) 
+  // DXIL: call void @_ZN4hlsl8RWBufferIfEC1EjjijPKc(ptr {{.*}} %[[Tmp0]], i32 noundef 10, i32 noundef 1, i32 noundef 4, i32 noundef 2, ptr noundef @[[BufA]])
+  // SPV: call void @_ZN4hlsl8RWBufferIfEC1EjjijPKc(ptr {{.*}} %[[Tmp0]], i32 noundef 12, i32 noundef 2, i32 noundef 4, i32 noundef 2, ptr noundef @[[BufA]])
+
+  // Make sure B[3] is translated to a RWBuffer<int> constructor call with range 5 and index 3
+  // and DXIL for implicit binding in space0, order id 0
+  // and SPIR-V explicit binding (binding 13, set 0)
+  // DXIL: call void @_ZN4hlsl8RWBufferIiEC1EjijjPKc(ptr {{.*}} %[[Tmp1]], i32 noundef 0, i32 noundef 5, i32 noundef 3, i32 noundef 0, ptr noundef @[[BufB]])
+  // SPV: call void @_ZN4hlsl8RWBufferIiEC1EjjijPKc(ptr {{.*}} %[[Tmp1]], i32 noundef 13, i32 noundef 0, i32 noundef 5, i32 noundef 3, ptr noundef @[[BufB]])
+
+  // Make sure C[1] is translated to a RWBuffer<int> constructor call with range 3 and index 1
+  // and DXIL explicit binding (u2, space0) 
+  // and SPIR-V explicit binding (binding 2, set 0)
+  // DXIL: call void @_ZN4hlsl8RWBufferIiEC1EjjijPKc(ptr {{.*}} %[[Tmp2]], i32 noundef 2, i32 noundef 0, i32 noundef 3, i32 noundef 1, ptr noundef @[[BufC]])
+  // SPV: call void @_ZN4hlsl8RWBufferIiEC1EjjijPKc(ptr {{.*}} %[[Tmp2]], i32 noundef 2, i32 noundef 0, i32 noundef 3, i32 noundef 1, ptr noundef @[[BufC]])
+
+  // Make sure D[7] is translated to a RWBuffer<double> constructor call with implicit binding
+  // for both DXIL and SPIR-V
+  // DXIL: call void @_ZN4hlsl8RWBufferIdEC1EjijjPKc(ptr {{.*}} %[[Tmp3]], i32 noundef 0, i32 noundef 10, i32 noundef 7, i32 noundef 1, ptr noundef @[[BufD]])
+  // SPV: call void @_ZN4hlsl8RWBufferIdEC1EjijjPKc(ptr {{.*}} %[[Tmp3]], i32 noundef 0, i32 noundef 10, i32 noundef 7, i32 noundef 0, ptr noundef @[[BufD]])
+
+  // Make sure E[5][0] is translated to RWBuffer<uint> constructor call with implicit binding and specified space/set 2
+  // DXIL: call void @_ZN4hlsl8RWBufferIjEC1EjijjPKc(ptr {{.*}} %[[Tmp4]], i32 noundef 2, i32 noundef 15, i32 noundef 5, i32 noundef 2, ptr noundef @[[BufE]])
+  // SPV: call void @_ZN4hlsl8RWBufferIjEC1EjijjPKc(ptr {{.*}} %[[Tmp4]], i32 noundef 2, i32 noundef 15, i32 noundef 5, i32 noundef 1, ptr noundef @[[BufE]])
+  Out[0] = A[2][0] + (float)B[3][0] + (float)C[1][0] + (float)D[7][0] + (float)E[5][0];
+}
diff --git a/clang/test/CodeGenHLSL/resources/res-array-local-multi-dim.hlsl b/clang/test/CodeGenHLSL/resources/res-array-local-multi-dim.hlsl
new file mode 100644
index 0000000000000..d803882fd2f72
--- /dev/null
+++ b/clang/test/CodeGenHLSL/resources/res-array-local-multi-dim.hlsl
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-compute -finclude-default-header \
+// RUN:   -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+// This test verifies handling of multi-dimensional local arrays of resources
+// when used as a function argument and local variable.
+
+// CHECK: @_ZL1A = internal global %"class.hlsl::RWBuffer" poison, align 4
+// CHECK: @_ZL1B = internal global %"class.hlsl::RWBuffer" poison, align 4
+
+RWBuffer<float> A : register(u10);
+RWBuffer<float> B : register(u20);
+RWStructuredBuffer<float> Out;
+
+// NOTE: _ZN4hlsl8RWBufferIfEixEj is the subscript operator for RWBuffer<float> and
+//       _ZN4hlsl18RWStructuredBufferIfEixEj is the subscript operator for RWStructuredBuffer<float>
+
+// CHECK: define {{.*}} float @_Z3fooA2_A2_N4hlsl8RWBufferIfEE(ptr noundef byval([2 x [2 x %"class.hlsl::RWBuffer"]]) align 4 %Arr)
+// CHECK-NEXT: entry:
+float foo(RWBuffer<float> Arr[2][2]) {
+// CHECK-NEXT: %[[Arr_1_Ptr:.*]] = getelementptr inbounds [2 x [2 x %"class.hlsl::RWBuffer"]], ptr %Arr, i32 0, i32 1
+// CHECK-NEXT: %[[Arr_1_1_Ptr:.*]] = getelementptr inbounds [2 x %"class.hlsl::RWBuffer"], ptr %[[Arr_1_Ptr]], i32 0, i32 1
+// CHECK-NEXT: %[[BufPtr:.*]] = call {{.*}} ptr @_ZN4hlsl8RWBufferIfEixEj(ptr {{.*}} %[[Arr_1_1_Ptr]], i32 noundef 0)
+// CHECK-NEXT: %[[Value:.*]] = load float, ptr %[[BufPtr]], align 4
+// CHECK-NEXT: ret float %[[Value]]
+  return Arr[1][1][0];
+}
+
+// CHECK: define internal void @_Z4mainv()
+// CHECK-NEXT: entry:
+[numthreads(4,1,1)]
+void main() {
+// CHECK-NEXT: %L = alloca [2 x [2 x %"class.hlsl::RWBuffer"]], align 4
+// CHECK-NEXT: %[[Tmp:.*]] = alloca [2 x [2 x %"class.hlsl::RWBuffer"]], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %L, ptr align 4 @_ZL1A, i32 4, i1 false)
+// CHECK-NEXT: %[[Ptr1:.*]] = getelementptr inbounds %"class.hlsl::RWBuffer", ptr %L, i32 1
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[Ptr1]], ptr align 4 @_ZL1B, i32 4, i1 false)
+// CHECK-NEXT: %[[Ptr2:.*]] = getelementptr inbounds [2 x %"class.hlsl::RWBuffer"], ptr %L, i32 1
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[Ptr2]], ptr align 4 @_ZL1A, i32 4, i1 false)
+// CHECK-NEXT: %[[Ptr3:.*]] = getelementptr inbounds %"class.hlsl::RWBuffer", ptr %[[Ptr2]], i32 1
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[Ptr3]], ptr align 4 @_ZL1B, i32 4, i1 false)
+  RWBuffer<float> L[2][2] = { { A, B }, { A, B } };
+
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[Tmp]], ptr align 4 %L, i32 16, i1 false)
+// CHECK-NEXT: %[[ReturnedValue:.*]] = call {{.*}}float @_Z3fooA2_A2_N4hlsl8RWBufferIfEE(ptr noundef byval([2 x [2 x %"class.hlsl::RWBuffer"]]) align 4 %[[Tmp]])
+// CHECK-NEXT: %[[OutBufPtr:.*]] = call {{.*}} ptr @_ZN4hlsl18RWStructuredBufferIfEixEj(ptr {{.*}} @_ZL3Out, i32 noundef 0)
+// CHECK-NEXT: store float %[[ReturnedValue]], ptr %[[OutBufPtr]], align 4
+// CHECK-NEXT: ret void
+  Out[0] = foo(L);
+}
diff --git a/clang/test/CodeGenHLSL/resources/res-array-local1.hlsl b/clang/test/CodeGenHLSL/resources/res-array-local1.hlsl
new file mode 100644
index 0000000000000..c0d508b1395c3
--- /dev/null
+++ b/clang/test/CodeGenHLSL/resources/res-array-local1.hlsl
@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-compute -finclude-default-header \
+// RUN:   -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+// This test verifies local arrays of resources in HLSL.
+
+// CHECK: @_ZL1A = internal global %"class.hlsl::RWBuffer" poison, align 4
+// CHECK: @_ZL1B = internal global %"class.hlsl::RWBuffer" poison, align 4
+// CHECK: @_ZL1C = internal global %"class.hlsl::RWBuffer" poison, align 4
+
+RWBuffer<float> A : register(u1);
+RWBuffer<float> B : register(u2);
+RWBuffer<float> C : register(u3);
+RWStructuredBuffer<float> Out : register(u0);
+
+// CHECK: define internal void @_Z4mainv()
+// CHECK-NEXT: entry:
+[numthreads(4,1,1)]
+void main() {
+// CHECK-NEXT:  %First = alloca [3 x %"class.hlsl::RWBuffer"], align 4
+// CHECK-NEXT:  %Second = alloca [4 x %"class.hlsl::RWBuffer"], align 4
+  RWBuffer<float> First[3] = { A, B, C };
+  RWBuffer<float> Second[4];
+
+// Verify initialization of First array from an initialization list
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %First, ptr align 4 @_ZL1A, i32 4, i1 false)
+// CHECK-NEXT: %[[Ptr1:.*]] = getelementptr inbounds %"class.hlsl::RWBuffer", ptr %First, i32 1
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[Ptr1]], ptr align 4 @_ZL1B, i32 4, i1 false)
+// CHECK-NEXT: %[[Ptr2:.*]] = getelementptr inbounds %"class.hlsl::RWBuffer", ptr %First, i32 2
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[Ptr2]], ptr align 4 @_ZL1C, i32 4, i1 false)
+
+// Verify default initialization of Second array, which means there is a loop iterating
+// over the array elements and calling the default constructor for each
+// CHECK-NEXT: %[[ArrayBeginPtr:.*]] = getelementptr inbounds [4 x %"class.hlsl::RWBuffer"], ptr %Second, i32 0, i32 0
+// CHECK-NEXT: %[[ArrayEndPtr:.*]] = getelementptr inbounds %"class.hlsl::RWBuffer", ptr %[[ArrayBeginPtr]], i32 4
+// CHECK-NEXT: br label %[[ArrayInitLoop:.*]]
+// CHECK: [[ArrayInitLoop]]:
+// CHECK-NEXT: %[[ArrayCurPtr:.*]] = phi ptr [ %[[ArrayBeginPtr]], %entry ], [ %[[ArrayNextPtr:.*]], %[[ArrayInitLoop]] ]
+// CHECK-NEXT: call void @_ZN4hlsl8RWBufferIfEC1Ev(ptr {{.*}} %[[ArrayCurPtr]])
+// CHECK-NEXT: %[[ArrayNextPtr]] = getelementptr inbounds %"class.hlsl::RWBuffer", ptr %[[ArrayCurPtr]], i32 1
+// CHECK-NEXT: %[[ArrayInitDone:.*]] = icmp eq ptr %[[ArrayNextPtr]], %[[ArrayEndPtr]]
+// CHECK-NEXT: br i1 %[[ArrayInitDone]], label %[[AfterArrayInit:.*]], label %[[ArrayInitLoop]]
+// CHECK: [[AfterArrayInit]]:
+
+// Initialize First[2] with C
+// CHECK: %[[Ptr3:.*]] = getelementptr inbounds [4 x %"class.hlsl::RWBuffer"], ptr %Second, i32 0, i32 2
+// CHECK: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[Ptr3]], ptr align 4 @_ZL1C, i32 4, i1 false)
+  Second[2] = C;
+
+  // NOTE: _ZN4hlsl8RWBufferIfEixEj is the subscript operator for RWBuffer<float>
+
+// get First[1][0] value
+// CHECK: %[[First_1_Ptr:.*]] = getelementptr inbounds [3 x %"class.hlsl::RWBuffer"], ptr %First, i32 0, i32 1
+// CHECK: %[[BufPtr1:.*]] = call {{.*}} ptr @_ZN4hlsl8RWBufferIfEixEj(ptr {{.*}} %[[First_1_Ptr]], i32 noundef 0)
+// CHECK: %[[Value1:.*]] = load float, ptr %[[BufPtr1]], align 4
+
+// get Second[2][0] value
+// CHECK: %[[Second_2_Ptr:.*]] = getelementptr inbounds [4 x %"class.hlsl::RWBuffer"], ptr %Second, i32 0, i32 2
+// CHECK: %[[BufPtr2:.*]] = call {{.*}} ptr @_ZN4hlsl8RWBufferIfEixEj(ptr {{.*}} %[[Second_2_Ptr]], i32 noundef 0)
+// CHECK: %[[Value2:.*]] = load float, ptr %[[BufPtr2]], align 4
+
+// add them
+// CHECK: %{{.*}} = fadd {{.*}} float %[[Value1]], %[[Value2]]
+  Out[0] = First[1][0] + Second[2][0];
+}
diff --git a/clang/test/CodeGenHLSL/resources/res-array-local2.hlsl b/clang/test/CodeGenHLSL/resources/res-array-local2.hlsl
new file mode 100644
index 0000000000000..39f3aeb66ceb5
--- /dev/null
+++ b/clang/test/CodeGenHLSL/resources/res-array-local2.hlsl
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-compute -finclude-default-header \
+// RUN:   -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+// This test verifies handling of local arrays of resources when used as a function argument.
+
+// CHECK: @_ZL1A = internal global [3 x %"class.hlsl::RWBuffer"] poison, align 4
+
+RWBuffer<float> A[3] : register(u0);
+RWStructuredBuffer<float> Out : register(u0);
+
+// NOTE: _ZN4hlsl8RWBufferIfEixEj is the subscript operator for RWBuffer<float> and
+//       _ZN4hlsl18RWStructuredBufferIfEixEj is the subscript operator for RWStructuredBuffer<float>
+
+// CHECK: define {{.*}} float @_Z3fooA3_N4hlsl8RWBufferIfEE(ptr noundef byval([3 x %"class.hlsl::RWBuffer"]) align 4 %LocalA)
+// CHECK-NEXT: entry:
+float foo(RWBuffer<float> LocalA[3]) {
+// CHECK-NEXT: %[[LocalA_2_Ptr:.*]] = getelementptr inbounds [3 x %"class.hlsl::RWBuffer"], ptr %LocalA, i32 0, i32 2
+// CHECK-NEXT: %[[BufPtr:.*]] = call {{.*}} ptr @_ZN4hlsl8RWBufferIfEixEj(ptr {{.*}} %[[LocalA_2_Ptr]], i32 noundef 0)
+// CHECK-NEXT: %[[Value:.*]] = load float, ptr %[[BufPtr]], align 4
+// CHECK-NEXT: ret float %[[Value]]
+  return LocalA[2][0];
+}
+
+// CHECK: define internal void @_Z4mainv()
+// CHECK-NEXT: entry:
+[numthreads(4,1,1)]
+void main() {
+// Check that the `main` function calls `foo` with a local copy of the array
+// CHECK-NEXT: %[[Tmp:.*]] = alloca [3 x %"class.hlsl::RWBuffer"], align 4
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[Tmp]], ptr align 4 @_ZL1A, i32 12, i1 false)
+
+// CHECK-NEXT: %[[ReturnedValue:.*]] = call {{.*}} float @_Z3fooA3_N4hlsl8RWBufferIfEE(ptr noundef byval([3 x %"class.hlsl::RWBuffer"]) align 4 %[[Tmp]])
+// CHECK-NEXT: %[[OutBufPtr:.*]] = call {{.*}} ptr @_ZN4hlsl18RWStructuredBufferIfEixEj(ptr {{.*}} @_ZL3Out, i32 noundef 0)
+// CHECK-NEXT: store float %[[ReturnedValue]], ptr %[[OutBufPtr]], align 4
+// CHECK-NEXT: ret void
+  Out[0] = foo(A);
+}
diff --git a/clang/test/CodeGenHLSL/resources/res-array-local3.hlsl b/clang/test/CodeGenHLSL/resources/res-array-local3.hlsl
new file mode 100644
index 0000000000000..e5bcdc651254f
--- /dev/null
+++ b/clang/test/CodeGenHLSL/resources/res-array-local3.hlsl
@@ -0,0 +1,62 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-compute -finclude-default-header \
+// RUN:   -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s
+
+// This test verifies handling of local arrays of resources when used
+// as a function argument that is modified inside the function.
+
+// CHECK: @_ZL1X = internal global %"class.hlsl::RWBuffer" poison, align 4
+// CHECK: @_ZL1Y = internal global %"class.hlsl::RWBuffer" poison, align 4
+
+RWBuffer<int> X : register(u0);
+RWBuffer<int> Y : register(u1);
+
+// CHECK: define {{.*}} @_Z6SomeFnA2_N4hlsl8RWBufferIiEEji(
+// CHECK-SAME: ptr noundef byval([2 x %"class.hlsl::RWBuffer"]) align 4 %B, i32 noundef %Idx, i32 noundef %Val0)
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %[[Idx_addr:.*]] = alloca i32, align 4
+// CHECK-NEXT: %[[Val0_addr:.*]] = alloca i32, align 4
+// CHECK-NEXT: store i32 %Idx, ptr %[[Idx_addr]], align 4
+// CHECK-NEXT: store i32 %Val0, ptr %[[Val0_addr]], align 4
+void SomeFn(RWBuffer<int> B[2], uint Idx, int Val0) {
+
+// CHECK-NEXT: %[[B_0_Ptr:.*]] = getelementptr inbounds [2 x %"class.hlsl::RWBuffer"], ptr %B, i32 0, i32 0
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[B_0_Ptr]], ptr align 4 @_ZL1Y, i32 4, i1 false)
+  B[0] = Y;
+
+// NOTE: _ZN4hlsl8RWBufferIiEixEj is the subscript operator for RWBuffer<int>
+
+// CHECK-NEXT: %[[Val0:.*]] = load i32, ptr %[[Val0_addr]], align 4
+// CHECK-NEXT: %[[B_0_Ptr:.*]] = getelementptr inbounds [2 x %"class.hlsl::RWBuffer"], ptr %B, i32 0, i32 0
+// CHECK-NEXT: %[[Idx:.*]] = load i32, ptr %[[Idx_addr]], align 4
+// CHECK-NEXT: %[[BufPtr:.*]] = call {{.*}} ptr @_ZN4hlsl8RWBufferIiEixEj(ptr {{.*}} %[[B_0_Ptr]], i32 noundef %[[Idx]])
+// CHECK-NEXT: store i32 %[[Val0]], ptr %[[BufPtr]], align 4
+  B[0][Idx] = Val0;
+}
+
+// CHECK: define {{.*}} void @_Z4mainj(i32 noundef %GI)
+// CHECK-NEXT: entry:
+// CHECK-NEXT: %[[GI_addr:.*]] = alloca i32, align 4
+[numthreads(4,1,1)]
+void main(uint GI : SV_GroupIndex) {
+// CHECK-NEXT: %A = alloca [2 x %"class.hlsl::RWBuffer"], align 4
+// CHECK-NEXT: %[[Tmp:.*]] = alloca [2 x %"class.hlsl::RWBuffer"], align 4
+// CHECK-NEXT: store i32 %GI, ptr %GI.addr, align 4
+
+// Initialization of array A with resources X and Y
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %A, ptr align 4 @_ZL1X, i32 4, i1 false)
+// CHECK-NEXT: %[[A_1_Ptr:.*]] = getelementptr inbounds %"class.hlsl::RWBuffer", ptr %A, i32 1
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[A_1_Ptr]], ptr align 4 @_ZL1Y, i32 4, i1 false)
+  RWBuffer<int> A[2] = {X, Y};
+
+// Verify that SomeFn is called with a local copy of the array A
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr align 4 %[[Tmp]], ptr align 4 %A, i32 8, i1 false)
+// CHECK-NEXT: %[[GI:.*]] = load i32, ptr %[[GI_addr]], align 4
+// CHECK-NEXT: call void @_Z6SomeFnA2_N4hlsl8RWBufferIiEEji(ptr noundef byval([2 x %"class.hlsl::RWBuffer"]) align 4 %[[Tmp]], i32 noundef %[[GI]], i32 noundef 1)
+  SomeFn(A, GI, 1);
+
+// CHECK-NEXT: %[[A_0_Ptr:.*]] = getelementptr inbounds [2 x %"class.hlsl::RWBuffer"], ptr %A, i32 0, i32 0
+// CHECK-NEXT: %[[GI:.*]] = load i32, ptr %[[GI_addr]], align 4
+// CHECK-NEXT: %[[BufPtr:.*]] = call {{.*}} ptr @_ZN4hlsl8RWBufferIiEixEj(ptr {{.*}} %[[A_0_Ptr]], i32 noundef %[[GI]])
+// CHECK-NEXT: store i32 2, ptr %[[BufPtr]], align 4
+  A[0][GI] = 2;
+}
diff --git a/clang/test/CodeGenHLSL/resources/resource-bindings.hlsl b/clang/test/CodeGenHLSL/resources/resource-bindings.hlsl
index 0a301346e55be..2a6e51d765bd1 100644
--- a/clang/test/CodeGenHLSL/resources/resource-bindings.hlsl
+++ b/clang/test/CodeGenHLSL/resources/resource-bindings.hlsl
@@ -4,11 +4,13 @@
 // CHECK: %"class.hlsl::RWBuffer.0" = type { target("dx.TypedBuffer", float, 1, 0, 0) }
 // CHECK: %"class.hlsl::StructuredBuffer" = type { target("dx.RawBuffer", i32, 0, 0) }
 // CHECK: %"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", %struct.S, 1, 0) }
+// CHECK: %"class.hlsl::RWBuffer.1" = type { target("dx.TypedBuffer", double, 1, 0, 0) }
 
 // CHECK: @_ZL4U0S0 = internal global %"class.hlsl::RWBuffer" poison, align 4
 // CHECK: @_ZL4U5S3 = internal global %"class.hlsl::RWBuffer.0" poison, align 4
 // CHECK: @_ZL4T2S2 = internal global %"class.hlsl::StructuredBuffer" poison, align 4
 // CHECK: @_ZL4T3S0 = internal global %"class.hlsl::RWStructuredBuffer" poison, align 4
+// CHECK: @_ZL5Array = internal global [10 x %"class.hlsl::RWBuffer.1"] poison, align 4
 
 // CHECK: %[[HANDLE:.*]] = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
 // CHECK-SAME: @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_v4f32_1_0_0t(
@@ -42,5 +44,15 @@ struct S {
 };
 RWStructuredBuffer<S> T3S0 : register(u3);
 
+// Resource array elements are initialized on access; make sure there is no call
+// to initialize RWBuffer<double>.
+// CHECK-NOT: call target("dx.TypedBuffer", double, 1, 0, 0) @llvm.dx.resource.handlefrombinding.tdx.TypedBuffer_f64_1_0_0t(
+RWBuffer<double> Array[10] : register(u4, space0);
+
 [numthreads(4,1,1)]
-void main() {}
+void main() {
+  // Reference Array to ensure it is emitted and we can test that it is initialized
+  // to poison, but do not index it.
+  // Non-array resources are always emitted because they have a constructor initializer.
+  (void)Array;
+}
diff --git a/clang/test/CodeGenHLSL/static-local-ctor.hlsl b/clang/test/CodeGenHLSL/static-local-ctor.hlsl
index 87f49b8bf7ac7..9a4bf66f030ed 100644
--- a/clang/test/CodeGenHLSL/static-local-ctor.hlsl
+++ b/clang/test/CodeGenHLSL/static-local-ctor.hlsl
@@ -2,7 +2,7 @@
 
 // Verify that no per variable _Init_thread instructions are emitted for non-trivial static locals
 // These would normally be emitted by the MicrosoftCXXABI, but the DirectX backend should exlude them
-// Instead, check for the guardvar oparations that should protect the constructor initialization should
+// Instead, check for the guardvar operations that should protect the constructor initialization should
 // only take place once.
 
 RWBuffer<int> buf[10];
@@ -15,13 +15,14 @@ void InitBuf(RWBuffer<int> buf) {
 // CHECK-NOT: _Init_thread_epoch
 // CHECK: define internal void @_Z4mainv
 // CHECK-NEXT: entry:
+// CHECK-NEXT: [[Tmp0:%.*]] = alloca %"class.hlsl::RWBuffer"
 // CHECK-NEXT: [[Tmp1:%.*]] = alloca %"class.hlsl::RWBuffer"
 // CHECK-NEXT: [[Tmp2:%.*]] = load i8, ptr @_ZGVZ4mainvE5mybuf
 // CHECK-NEXT: [[Tmp3:%.*]] = icmp eq i8 [[Tmp2]], 0
 // CHECK-NEXT: br i1 [[Tmp3]]
 // CHECK-NOT: _Init_thread_header
 // CHECK: init.check:
-// CHECK-NEXT: call void @_ZN4hlsl8RWBufferIiEC1Ejijj
+// CHECK-NEXT: call void @_ZN4hlsl8RWBufferIiEC1EjijjPKc(
 // CHECK-NEXT: store i8 1, ptr @_ZGVZ4mainvE5mybuf
 // CHECK-NOT: _Init_thread_footer
 
diff --git a/clang/test/CodeGenObjC/arc-blocks.m b/clang/test/CodeGenObjC/arc-blocks.m
index 72bf35c2e117e..605dda7664bf4 100644
--- a/clang/test/CodeGenObjC/arc-blocks.m
+++ b/clang/test/CodeGenObjC/arc-blocks.m
@@ -73,7 +73,7 @@ void test3(void (^sink)(id*)) {
   // CHECK-NEXT: [[TEMP:%.*]] = alloca ptr
   // CHECK-NEXT: call ptr @llvm.objc.retain(
   // CHECK-NEXT: store ptr {{%.*}}, ptr [[SINK]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[STRONG]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[STRONG]])
   // CHECK-NEXT: store ptr null, ptr [[STRONG]]
 
   // CHECK-NEXT: [[BLOCK:%.*]] = load ptr, ptr [[SINK]]
@@ -91,7 +91,7 @@ void test3(void (^sink)(id*)) {
 
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[STRONG]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[STRONG]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[STRONG]])
 
   // CHECK-NEXT: load ptr, ptr [[SINK]]
   // CHECK-NEXT: call void @llvm.objc.release
@@ -161,7 +161,7 @@ void test5(void) {
   // CHECK-LABEL:    define{{.*}} void @test5()
   // CHECK:      [[VAR:%.*]] = alloca ptr
   // CHECK-NEXT: [[BLOCK:%.*]] = alloca [[BLOCK_T:<{.*}>]],
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VAR]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[VAR]])
   // CHECK: [[T1:%.*]] = call ptr @test5_source() [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T1]])
   // CHECK-NEXT: store ptr [[T1]], ptr [[VAR]],
@@ -172,7 +172,7 @@ void test5(void) {
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[VAR]]
   // CHECK-NEXT: store ptr [[T0]], ptr [[CAPTURE]]
   // CHECK: call void @test5_helper
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VAR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[VAR]])
   // CHECK-NEXT: ret void
 }
 
@@ -185,7 +185,7 @@ void test6(void) {
   // CHECK-LABEL:    define{{.*}} void @test6()
   // CHECK:      [[VAR:%.*]] = alloca [[BYREF_T:%.*]],
   // CHECK-NEXT: [[BLOCK:%.*]] = alloca [[BLOCK_T:<{.*}>]],
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 48, ptr [[VAR]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[VAR]])
   // CHECK:      [[T0:%.*]] = getelementptr inbounds nuw [[BYREF_T]], ptr [[VAR]], i32 0, i32 2
   // 0x02000000 - has copy/dispose helpers weak
   // CHECK-NEXT: store i32 1107296256, ptr [[T0]]
@@ -203,7 +203,7 @@ void test6(void) {
   // CHECK:      call void @test6_helper(
   // CHECK: call void @_Block_object_dispose(ptr [[VAR]], i32 8)
   // CHECK-NEXT: call void @llvm.objc.destroyWeak(ptr [[SLOT]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 48, ptr [[VAR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[VAR]])
   // CHECK-NEXT: ret void
 
   // CHECK-LABEL:    define internal void @__Block_byref_object_copy_.{{[0-9]+}}(ptr noundef %0, ptr noundef %1) #{{[0-9]+}} {
@@ -449,7 +449,7 @@ void test13(id x) {
   // CHECK-NEXT: [[CLEANUP_ACTIVE:%.*]] = alloca i1
   // CHECK-NEXT: [[T0:%.*]] = call ptr @llvm.objc.retain(ptr {{%.*}})
   // CHECK-NEXT: store ptr [[T0]], ptr [[X]], align 8
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[B]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[B]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[X]], align 8
   // CHECK-NEXT: [[T1:%.*]] = icmp ne ptr [[T0]], null
   // CHECK-NEXT: store i1 false, ptr [[CLEANUP_ACTIVE]]
@@ -479,7 +479,7 @@ void test13(id x) {
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
   // CHECK-NEXT: br label
 
-  // CHECK: call void @llvm.lifetime.end.p0(i64 8, ptr [[B]])
+  // CHECK: call void @llvm.lifetime.end.p0(ptr [[B]])
   // CHECK-NEXT:      [[T0:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
   // CHECK-NEXT: ret void
@@ -501,7 +501,7 @@ void test16(void) {
   // CHECK-LABEL: define{{.*}} void @test16(
   // CHECK: [[BLKVAR:%.*]]  = alloca ptr, align 8
   // CHECK-NEXT:  [[BLOCK:%.*]] = alloca [[BLOCK_T:<{.*}>]],
-  // CHECK-NEXT:  call void @llvm.lifetime.start.p0(i64 8, ptr [[BLKVAR]])
+  // CHECK-NEXT:  call void @llvm.lifetime.start.p0(ptr [[BLKVAR]])
   // CHECK-NEXT:  store ptr null, ptr [[BLKVAR]], align 8
 }
 
diff --git a/clang/test/CodeGenObjC/arc-precise-lifetime.m b/clang/test/CodeGenObjC/arc-precise-lifetime.m
index 473c0b031f9f9..ac761ba60e864 100644
--- a/clang/test/CodeGenObjC/arc-precise-lifetime.m
+++ b/clang/test/CodeGenObjC/arc-precise-lifetime.m
@@ -7,7 +7,7 @@ void test0(void) {
   PRECISE_LIFETIME id x = test0_helper();
   x = 0;
   // CHECK:      [[X:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: [[CALL:%.*]] = call ptr @test0_helper()
   // CHECK-NEXT: store ptr [[CALL]], ptr [[X]]
 
@@ -20,7 +20,7 @@ void test0(void) {
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T1]]) [[NUW:#[0-9]+]]
   // CHECK-NOT:  clang.imprecise_release
 
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
@@ -37,20 +37,20 @@ - (char*) interior __attribute__((objc_returns_inner_pointer));
 void test1a_message(void) {
   // CHECK:      [[PTR:%.*]] = alloca ptr, align 8
   // CHECK:      [[C:%.*]] = alloca ptr, align 8
-  // CHECK:      call void @llvm.lifetime.start.p0(i64 8, ptr [[PTR]])
+  // CHECK:      call void @llvm.lifetime.start.p0(ptr [[PTR]])
   // CHECK:      [[T0:%.*]] = call ptr @test1_helper() [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
   // CHECK-NEXT: store ptr [[T0]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[C]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[C]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: [[T2:%.*]] = call ptr @llvm.objc.retainAutorelease(ptr [[T0]])
   // CHECK-NEXT: [[T4:%.*]] = load ptr, ptr @OBJC_SELECTOR_REFERENCES_
   // CHECK-NEXT: [[T6:%.*]] = call ptr
   // CHECK-NEXT: store ptr [[T6]], ptr
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[C]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[C]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]], !clang.imprecise_release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PTR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PTR]])
   // CHECK-NEXT: ret void
   Test1 *ptr = test1_helper();
   char *c = [(ptr) interior];
@@ -61,20 +61,20 @@ void test1a_message(void) {
 void test1a_property(void) {
   // CHECK:      [[PTR:%.*]] = alloca ptr, align 8
   // CHECK:      [[C:%.*]] = alloca ptr, align 8
-  // CHECK:      call void @llvm.lifetime.start.p0(i64 8, ptr [[PTR]])
+  // CHECK:      call void @llvm.lifetime.start.p0(ptr [[PTR]])
   // CHECK:      [[T0:%.*]] = call ptr @test1_helper() [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
   // CHECK-NEXT: store ptr [[T0]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[C]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[C]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: [[T2:%.*]] = call ptr @llvm.objc.retainAutorelease(ptr [[T0]])
   // CHECK-NEXT: [[T4:%.*]] = load ptr, ptr @OBJC_SELECTOR_REFERENCES_
   // CHECK-NEXT: [[T6:%.*]] = call ptr
   // CHECK-NEXT: store ptr [[T6]], ptr
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[C]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[C]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]], !clang.imprecise_release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PTR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PTR]])
   // CHECK-NEXT: ret void
   Test1 *ptr = test1_helper();
   char *c = ptr.interior;
@@ -85,20 +85,20 @@ void test1a_property(void) {
 void test1b_message(void) {
   // CHECK:      [[PTR:%.*]] = alloca ptr, align 8
   // CHECK:      [[C:%.*]] = alloca ptr, align 8
-  // CHECK:      call void @llvm.lifetime.start.p0(i64 8, ptr [[PTR]])
+  // CHECK:      call void @llvm.lifetime.start.p0(ptr [[PTR]])
   // CHECK:      [[T0:%.*]] = call ptr @test1_helper() [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
   // CHECK-NEXT: store ptr [[T0]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[C]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[C]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: [[T1:%.*]] = load ptr, ptr @OBJC_SELECTOR_REFERENCES_
   // CHECK-NEXT: [[T3:%.*]] = call ptr
   // CHECK-NEXT: store ptr [[T3]], ptr
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[C]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[C]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]]
   // CHECK-NOT:  clang.imprecise_release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PTR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PTR]])
   // CHECK-NEXT: ret void
   PRECISE_LIFETIME Test1 *ptr = test1_helper();
   char *c = [ptr interior];
@@ -108,20 +108,20 @@ void test1b_message(void) {
 void test1b_property(void) {
   // CHECK:      [[PTR:%.*]] = alloca ptr, align 8
   // CHECK:      [[C:%.*]] = alloca ptr, align 8
-  // CHECK:      call void @llvm.lifetime.start.p0(i64 8, ptr [[PTR]])
+  // CHECK:      call void @llvm.lifetime.start.p0(ptr [[PTR]])
   // CHECK:      [[T0:%.*]] = call ptr @test1_helper() [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
   // CHECK-NEXT: store ptr [[T0]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[C]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[C]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: [[T1:%.*]] = load ptr, ptr @OBJC_SELECTOR_REFERENCES_
   // CHECK-NEXT: [[T3:%.*]] = call ptr
   // CHECK-NEXT: store ptr [[T3]], ptr
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[C]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[C]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]]
   // CHECK-NOT:  clang.imprecise_release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PTR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PTR]])
   // CHECK-NEXT: ret void
   PRECISE_LIFETIME Test1 *ptr = test1_helper();
   char *c = ptr.interior;
@@ -131,20 +131,20 @@ void test1b_property(void) {
 void test1c_message(void) {
   // CHECK:      [[PTR:%.*]] = alloca ptr, align 8
   // CHECK:      [[PC:%.*]] = alloca ptr, align 8
-  // CHECK:      call void @llvm.lifetime.start.p0(i64 8, ptr [[PTR]])
+  // CHECK:      call void @llvm.lifetime.start.p0(ptr [[PTR]])
   // CHECK:      [[T0:%.*]] = call ptr @test1_helper() [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
   // CHECK-NEXT: store ptr [[T0]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[PC]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[PC]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: [[T2:%.*]] = call ptr @llvm.objc.retainAutorelease(ptr [[T0]])
   // CHECK-NEXT: [[T4:%.*]] = load ptr, ptr @OBJC_SELECTOR_REFERENCES_
   // CHECK-NEXT: [[T6:%.*]] = call ptr
   // CHECK-NEXT: store ptr [[T6]], ptr
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PC]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PC]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]], !clang.imprecise_release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PTR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PTR]])
   // CHECK-NEXT: ret void
   Test1 *ptr = test1_helper();
   char *pc = [ptr PropertyReturnsInnerPointer];
@@ -154,20 +154,20 @@ void test1c_message(void) {
 void test1c_property(void) {
   // CHECK:      [[PTR:%.*]] = alloca ptr, align 8
   // CHECK:      [[PC:%.*]] = alloca ptr, align 8
-  // CHECK:      call void @llvm.lifetime.start.p0(i64 8, ptr [[PTR]])
+  // CHECK:      call void @llvm.lifetime.start.p0(ptr [[PTR]])
   // CHECK:      [[T0:%.*]] = call ptr @test1_helper() [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
   // CHECK-NEXT: store ptr [[T0]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[PC]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[PC]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: [[T2:%.*]] = call ptr @llvm.objc.retainAutorelease(ptr [[T0]])
   // CHECK-NEXT: [[T4:%.*]] = load ptr, ptr @OBJC_SELECTOR_REFERENCES_
   // CHECK-NEXT: [[T6:%.*]] = call ptr
   // CHECK-NEXT: store ptr [[T6]], ptr
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PC]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PC]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]], !clang.imprecise_release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PTR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PTR]])
   // CHECK-NEXT: ret void
   Test1 *ptr = test1_helper();
   char *pc = ptr.PropertyReturnsInnerPointer;
@@ -177,19 +177,19 @@ void test1c_property(void) {
 void test1d_message(void) {
   // CHECK:      [[PTR:%.*]] = alloca ptr, align 8
   // CHECK:      [[PC:%.*]] = alloca ptr, align 8
-  // CHECK:      call void @llvm.lifetime.start.p0(i64 8, ptr [[PTR]])
+  // CHECK:      call void @llvm.lifetime.start.p0(ptr [[PTR]])
   // CHECK:      [[T0:%.*]] = call ptr @test1_helper() [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
   // CHECK-NEXT: store ptr [[T0]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[PC]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[PC]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: [[SEL:%.*]] = load ptr, ptr @OBJC_SELECTOR_REFERENCES_
   // CHECK-NEXT: [[CALL1:%.*]] = call ptr @objc_msgSend(ptr noundef [[T0]], ptr noundef [[SEL]])
   // CHECK-NEXT: store ptr [[CALL1]], ptr
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PC]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PC]])
   // CHECK-NEXT: [[NINE:%.*]] = load ptr, ptr
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[NINE]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PTR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PTR]])
   // CHECK-NEXT: ret void
   PRECISE_LIFETIME Test1 *ptr = test1_helper();
   char *pc = [ptr PropertyReturnsInnerPointer];
@@ -199,19 +199,19 @@ void test1d_message(void) {
 void test1d_property(void) {
   // CHECK:      [[PTR:%.*]] = alloca ptr, align 8
   // CHECK:      [[PC:%.*]] = alloca ptr, align 8
-  // CHECK:      call void @llvm.lifetime.start.p0(i64 8, ptr [[PTR]])
+  // CHECK:      call void @llvm.lifetime.start.p0(ptr [[PTR]])
   // CHECK:      [[T0:%.*]] = call ptr @test1_helper() [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
   // CHECK-NEXT: store ptr [[T0]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[PC]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[PC]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr
   // CHECK-NEXT: [[SEL:%.*]] = load ptr, ptr @OBJC_SELECTOR_REFERENCES_
   // CHECK-NEXT: [[CALL1:%.*]] = call ptr @objc_msgSend(ptr noundef [[T0]], ptr noundef [[SEL]])
   // CHECK-NEXT: store ptr [[CALL1]], ptr
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PC]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PC]])
   // CHECK-NEXT: [[NINE:%.*]] = load ptr, ptr
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[NINE]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[PTR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[PTR]])
   // CHECK-NEXT: ret void
   PRECISE_LIFETIME Test1 *ptr = test1_helper();
   char *pc = ptr.PropertyReturnsInnerPointer;
diff --git a/clang/test/CodeGenObjC/arc-ternary-op.m b/clang/test/CodeGenObjC/arc-ternary-op.m
index 4a3c00c9807a9..46529b411f113 100644
--- a/clang/test/CodeGenObjC/arc-ternary-op.m
+++ b/clang/test/CodeGenObjC/arc-ternary-op.m
@@ -12,7 +12,7 @@ void test0(_Bool cond) {
   // CHECK-NEXT: [[RELCOND:%.*]] = alloca i1
   // CHECK-NEXT: zext
   // CHECK-NEXT: store
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: [[T0:%.*]] = load i8, ptr [[COND]]
   // CHECK-NEXT: [[T1:%.*]] = trunc i8 [[T0]] to i1
   // CHECK-NEXT: store i1 false, ptr [[RELCOND]]
@@ -32,7 +32,7 @@ void test0(_Bool cond) {
   // CHECK-NEXT: br label
   // CHECK:      [[T0:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]]
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
   id x = (cond ? 0 : test0_helper());
 }
@@ -53,9 +53,9 @@ void test1(int cond) {
   // CHECK-NEXT: [[CONDCLEANUPSAVE:%.*]] = alloca ptr
   // CHECK-NEXT: [[CONDCLEANUP:%.*]] = alloca i1
   // CHECK-NEXT: store i32
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[STRONG]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[STRONG]])
   // CHECK-NEXT: store ptr null, ptr [[STRONG]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[WEAK]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[WEAK]])
   // CHECK-NEXT: call ptr @llvm.objc.initWeak(ptr [[WEAK]], ptr null)
 
   // CHECK-NEXT: [[T0:%.*]] = load i32, ptr [[COND]]
@@ -99,8 +99,8 @@ void test1(int cond) {
   // CHECK-NEXT: br label
 
   // CHECK:      call void @llvm.objc.destroyWeak(ptr [[WEAK]])
-  // CHECK:      call void @llvm.lifetime.end.p0(i64 8, ptr [[WEAK]])
-  // CHECK:      call void @llvm.lifetime.end.p0(i64 8, ptr [[STRONG]])
+  // CHECK:      call void @llvm.lifetime.end.p0(ptr [[WEAK]])
+  // CHECK:      call void @llvm.lifetime.end.p0(ptr [[STRONG]])
   // CHECK:      ret void
 }
 
diff --git a/clang/test/CodeGenObjC/arc.m b/clang/test/CodeGenObjC/arc.m
index 57afe9c245e77..2d860c1116789 100644
--- a/clang/test/CodeGenObjC/arc.m
+++ b/clang/test/CodeGenObjC/arc.m
@@ -48,13 +48,13 @@ id test1(id x) {
   // CHECK-NEXT: [[Y:%.*]] = alloca ptr
   // CHECK-NEXT: [[PARM:%.*]] = call ptr @llvm.objc.retain(ptr {{%.*}})
   // CHECK-NEXT: store ptr [[PARM]], ptr [[X]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[Y]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[Y]])
   // CHECK-NEXT: store ptr null, ptr [[Y]]
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[Y]]
   // CHECK-NEXT: [[RET:%.*]] = call ptr @llvm.objc.retain(ptr [[T0]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[Y]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[Y]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[Y]])
   // CHECK-NEXT: [[T1:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T1]])
   // CHECK-NEXT: [[T1:%.*]] = tail call ptr @llvm.objc.autoreleaseReturnValue(ptr [[RET]])
@@ -99,7 +99,7 @@ void test3_unelided(void) {
   extern void test3_helper(void);
 
   // CHECK:      [[X:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: store ptr null, ptr [[X]], align
   Test3 *x;
 
@@ -118,14 +118,14 @@ void test3_unelided(void) {
 
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]]
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
 // CHECK-LABEL: define{{.*}} void @test3()
 void test3(void) {
   // CHECK:      [[X:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
 
   id x = [[Test3 alloc] initWith: 5];
 
@@ -155,7 +155,7 @@ void test3(void) {
   // Cleanup for x.
   // CHECK-NEXT: [[TMP:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[TMP]]) [[NUW]]
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
@@ -225,12 +225,12 @@ void test5(Test5 *x, id y) {
 // CHECK-LABEL: define{{.*}} void @test6()
 void test6(void) {
   // CHECK:      [[X:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: [[CALL:%.*]] = call ptr @test6_helper()
   // CHECK-NEXT: store ptr [[CALL]], ptr [[X]]
   // CHECK-NEXT: [[T1:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T1]]) [[NUW]], !clang.imprecise_release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
   id x = test6_helper();
 }
@@ -239,14 +239,14 @@ void test6(void) {
 // CHECK-LABEL: define{{.*}} void @test7()
 void test7(void) {
   // CHECK:      [[X:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: store ptr null, ptr [[X]]
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: [[T1:%.*]] = call ptr @llvm.objc.retain(ptr [[T0]]) [[NUW]]
   // CHECK-NEXT: call void @test7_helper(ptr noundef [[T1]])
   // CHECK-NEXT: [[T1:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T1]]) [[NUW]], !clang.imprecise_release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
   id x;
   test7_helper(x);
@@ -256,11 +256,11 @@ void test7(void) {
 void test8(void) {
   __unsafe_unretained id x = test8_helper();
   // CHECK:      [[X:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: [[T0:%.*]] = call ptr @test8_helper()
   // CHECK-NEXT: store ptr [[T0]], ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]], !clang.imprecise_release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
@@ -274,9 +274,9 @@ void test10(void) {
   // CHECK-LABEL:      define{{.*}} void @test10()
   // CHECK:      [[X:%.*]] = alloca ptr, align
   // CHECK-NEXT: [[Y:%.*]] = alloca ptr, align
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: store ptr null, ptr [[X]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[Y]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[Y]])
   // CHECK-NEXT: load ptr, ptr [[X]], align
   // CHECK-NEXT: load ptr, ptr @OBJC_SELECTOR_REFERENCES_{{[0-9]*}}
   // CHECK-NEXT: [[V:%.*]] = call ptr @objc_msgSend{{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
@@ -288,10 +288,10 @@ void test10(void) {
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[V]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[Y]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
-  // CHECK-NEXT: void @llvm.lifetime.end.p0(i64 8, ptr [[Y]])
+  // CHECK-NEXT: void @llvm.lifetime.end.p0(ptr [[Y]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
-  // CHECK-NEXT: void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
@@ -300,13 +300,13 @@ void test11(id (*f)(void) __attribute__((ns_returns_retained))) {
   // CHECK:      [[F:%.*]] = alloca ptr, align
   // CHECK-NEXT: [[X:%.*]] = alloca ptr, align
   // CHECK-NEXT: store ptr {{%.*}}, ptr [[F]], align
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[F]], align
   // CHECK-NEXT: [[T1:%.*]] = call ptr [[T0]]()
   // CHECK-NEXT: store ptr [[T1]], ptr [[X]], align
   // CHECK-NEXT: [[T3:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T3]]) [[NUW]], !clang.imprecise_release
-  // CHECK-NEXT: void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
   id x = f();
 }
@@ -319,7 +319,7 @@ void test12(void) {
   // CHECK-NEXT: [[Y:%.*]] = alloca ptr, align
 
   __weak id x = test12_helper();
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: [[T1:%.*]] = call ptr @test12_helper(){{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T1]])
   // CHECK-NEXT: call ptr @llvm.objc.initWeak(ptr [[X]], ptr [[T1]])
@@ -332,15 +332,15 @@ void test12(void) {
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T1]])
 
   id y = x;
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[Y]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[Y]])
   // CHECK-NEXT: [[T2:%.*]] = call ptr @llvm.objc.loadWeakRetained(ptr [[X]])
   // CHECK-NEXT: store ptr [[T2]], ptr [[Y]], align
 
   // CHECK-NEXT: [[T4:%.*]] = load ptr, ptr [[Y]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T4]]) [[NUW]], !clang.imprecise_release
-  // CHECK-NEXT: void @llvm.lifetime.end.p0(i64 8, ptr [[Y]])
+  // CHECK-NEXT: void @llvm.lifetime.end.p0(ptr [[Y]])
   // CHECK-NEXT: call void @llvm.objc.destroyWeak(ptr [[X]])
-  // CHECK-NEXT: void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK: ret void
 }
 
@@ -348,7 +348,7 @@ void test12(void) {
 void test13(void) {
   // CHECK-LABEL:      define{{.*}} void @test13()
   // CHECK:      [[X:%.*]] = alloca ptr, align
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: store ptr null, ptr [[X]], align
   id x;
 
@@ -371,7 +371,7 @@ void test13(void) {
 
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]]) [[NUW]]
-  // CHECK-NEXT: void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
@@ -886,7 +886,7 @@ void test37(void) {
   // CHECK-LABEL:    define{{.*}} void @test37()
   // CHECK:      [[VAR:%.*]] = alloca ptr,
   // CHECK-NEXT: [[TEMP:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[VAR]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[VAR]])
   // CHECK-NEXT: store ptr null, ptr [[VAR]]
 
   // CHECK-NEXT: [[W0:%.*]] = load ptr, ptr [[VAR]]
@@ -901,7 +901,7 @@ void test37(void) {
 
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[VAR]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[VAR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[VAR]])
   // CHECK-NEXT: ret void
 }
 
@@ -956,7 +956,7 @@ void test47(void) {
 
   // CHECK-LABEL:    define{{.*}} void @test47()
   // CHECK:      [[X:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: store ptr null, ptr [[X]]
   // CHECK-NEXT: [[T0:%.*]] = call ptr @test47_helper(){{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
@@ -969,7 +969,7 @@ void test47(void) {
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T3]])
   // CHECK-NEXT: [[T4:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T4]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
@@ -978,7 +978,7 @@ void test48(void) {
   __weak id x = x = test48_helper();
   // CHECK-LABEL:    define{{.*}} void @test48()
   // CHECK:      [[X:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: [[T0:%.*]] = call ptr @llvm.objc.initWeak(ptr [[X]], ptr null)
   // CHECK-NEXT: [[T2:%.*]] = call ptr @test48_helper(){{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T2]])
@@ -986,7 +986,7 @@ void test48(void) {
   // CHECK-NEXT: [[T4:%.*]] = call ptr @llvm.objc.storeWeak(ptr [[X]], ptr [[T3]])
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T2]])
   // CHECK-NEXT: call void @llvm.objc.destroyWeak(ptr [[X]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
@@ -995,7 +995,7 @@ void test49(void) {
   __autoreleasing id x = x = test49_helper();
   // CHECK-LABEL:    define{{.*}} void @test49()
   // CHECK:      [[X:%.*]] = alloca ptr
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK-NEXT: store ptr null, ptr [[X]]
   // CHECK-NEXT: [[T0:%.*]] = call ptr @test49_helper(){{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T0]])
@@ -1003,7 +1003,7 @@ void test49(void) {
   // CHECK-NEXT: store ptr [[T1]], ptr [[X]]
   // CHECK-NEXT: [[T3:%.*]] = call ptr @llvm.objc.retainAutorelease(ptr [[T1]])
   // CHECK-NEXT: store ptr [[T3]], ptr [[X]]
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
@@ -1035,12 +1035,12 @@ id test52(void) {
 // CHECK-LABEL:    define{{.*}} ptr @test52()
 // CHECK:      [[X:%.*]] = alloca i32
 // CHECK-NEXT: [[TMPALLOCA:%.*]] = alloca ptr
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[X]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
 // CHECK-NEXT: store i32 5, ptr [[X]],
 // CHECK-NEXT: [[T0:%.*]] = load i32, ptr [[X]],
 // CHECK-NEXT: [[T1:%.*]] = call ptr @test52_helper(i32 noundef [[T0]])
 // CHECK-NEXT: store ptr [[T1]], ptr [[TMPALLOCA]]
-// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[X]])
+// CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
 // CHECK-NEXT: [[T2:%.*]] = load ptr, ptr [[TMPALLOCA]]
 // CHECK-NEXT: [[T3:%.*]] = tail call ptr @llvm.objc.autoreleaseReturnValue(ptr [[T2]])
 // CHECK-NEXT: ret ptr [[T3]]
@@ -1054,8 +1054,8 @@ void test53(void) {
 // CHECK:      [[X:%.*]] = alloca ptr,
 // CHECK-NEXT: [[Y:%.*]] = alloca ptr,
 // CHECK-NEXT: [[TMPALLOCA:%.*]] = alloca ptr,
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[Y]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[Y]])
 // CHECK-NEXT: [[T1:%.*]] = call ptr @test53_helper(){{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
 // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T1]])
 // CHECK-NEXT: store ptr [[T1]], ptr [[Y]],
@@ -1064,13 +1064,13 @@ void test53(void) {
 // CHECK-NEXT: store ptr [[T1]], ptr [[TMPALLOCA]]
 // CHECK-NEXT: [[T2:%.*]] = load ptr, ptr [[Y]]
 // CHECK-NEXT: call void @llvm.objc.release(ptr [[T2]])
-// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[Y]])
+// CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[Y]])
 // CHECK-NEXT: [[T3:%.*]] = load ptr, ptr [[TMPALLOCA]]
 // CHECK-NEXT: store ptr [[T3]], ptr [[X]],
 // CHECK-NEXT: load ptr, ptr [[X]],
 // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[X]]
 // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
-// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+// CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
 // CHECK-NEXT: ret void
 }
 
@@ -1115,12 +1115,12 @@ void test56_test(void) {
   id x = [Test56 make];
   // CHECK-LABEL: define{{.*}} void @test56_test()
   // CHECK:      [[X:%.*]] = alloca ptr, align 8
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK:      [[T0:%.*]] = call ptr @objc_msgSend(
   // CHECK-NEXT: store ptr [[T0]], ptr [[X]]
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
@@ -1188,7 +1188,7 @@ void test61(void) {
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T1]])
   [test61_make() performSelector: @selector(test61_void)];
 
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[Y]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[Y]])
   // CHECK-NEXT: [[T1:%.*]] = call ptr @test61_make(){{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T1]])
   // CHECK-NEXT: [[T2:%.*]] = load ptr, ptr @OBJC_SELECTOR_REFERENCES_
@@ -1201,7 +1201,7 @@ void test61(void) {
 
   // CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[Y]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[T0]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[Y]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[Y]])
   // CHECK-NEXT: ret void
 }
 
@@ -1213,7 +1213,7 @@ void test62(void) {
   extern id test62_make(void);
   extern void test62_body(void);
 
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[I]])
   // CHECK-NEXT: store i32 0, ptr [[I]], align 4
   // CHECK-NEXT: br label
 
@@ -1300,10 +1300,10 @@ void test67(void) {
 }
 // CHECK-LABEL:    define{{.*}} void @test67()
 // CHECK:      [[CL:%.*]] = alloca ptr, align 8
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[CL]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[CL]])
 // CHECK-NEXT: [[T0:%.*]] = call ptr @test67_helper()
 // CHECK-NEXT: store ptr [[T0]], ptr [[CL]], align 8
-// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[CL]])
+// CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[CL]])
 // CHECK-NEXT: ret void
 
 Class test68_helper(void);
@@ -1312,13 +1312,13 @@ void test68(void) {
 }
 // CHECK-LABEL:    define{{.*}} void @test68()
 // CHECK:      [[CL:%.*]] = alloca ptr, align 8
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[CL]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[CL]])
 // CHECK-NEXT: [[T1:%.*]] = call ptr @test67_helper(){{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
 // CHECK-NEXT: call void (...) @llvm.objc.clang.arc.noop.use(ptr [[T1]])
 // CHECK-NEXT: store ptr [[T1]], ptr [[CL]], align 8
 // CHECK-NEXT: [[T2:%.*]] = load ptr, ptr [[CL]]
 // CHECK-NEXT: call void @llvm.objc.release(ptr [[T2]])
-// CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[CL]])
+// CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[CL]])
 // CHECK-NEXT: ret void
 
 @interface Test69 @end
@@ -1351,14 +1351,14 @@ void test70(id i) {
 
 // CHECK-LABEL: define{{.*}} void @test71
 void test71(void) {
-  // CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr %[[T:.*]])
+  // CHECK: call void @llvm.lifetime.start.p0(ptr %[[T:.*]])
   // CHECK: call void @getAggDtor(ptr dead_on_unwind writable sret(%struct.AggDtor) align 8 %[[T]])
   // CHECK: call void @__destructor_8_s40(ptr %[[T]])
-  // CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr %[[T]])
-  // CHECK: call void @llvm.lifetime.start.p0({{[^,]+}}, ptr %[[T2:.*]])
+  // CHECK: call void @llvm.lifetime.end.p0(ptr %[[T]])
+  // CHECK: call void @llvm.lifetime.start.p0(ptr %[[T2:.*]])
   // CHECK: call void @getAggDtor(ptr dead_on_unwind writable sret(%struct.AggDtor) align 8 %[[T2]])
   // CHECK: call void @__destructor_8_s40(ptr %[[T2]])
-  // CHECK: call void @llvm.lifetime.end.p0({{[^,]+}}, ptr %[[T2]])
+  // CHECK: call void @llvm.lifetime.end.p0(ptr %[[T2]])
   getAggDtor();
   getAggDtor();
 }
diff --git a/clang/test/CodeGenObjC/exceptions.m b/clang/test/CodeGenObjC/exceptions.m
index 832d3a458050c..66dc0515af03e 100644
--- a/clang/test/CodeGenObjC/exceptions.m
+++ b/clang/test/CodeGenObjC/exceptions.m
@@ -79,7 +79,7 @@ void f3(void) {
   extern void f3_helper(int, int*);
 
   // CHECK:      [[X:%.*]] = alloca i32
-  // CHECK:      call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X]])
+  // CHECK:      call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
   // CHECK:      store i32 0, ptr [[X]]
   int x = 0;
 
@@ -120,7 +120,7 @@ void f3(void) {
   }
 
   // CHECK:      call void @f3_helper(i32 noundef 4, ptr noundef nonnull [[X]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
   // CHECK-NEXT: ret void
   f3_helper(4, &x);
 }
diff --git a/clang/test/CodeGenObjC/ptrauth-block-descriptor-pointer.m b/clang/test/CodeGenObjC/ptrauth-block-descriptor-pointer.m
new file mode 100644
index 0000000000000..b51670fd6459a
--- /dev/null
+++ b/clang/test/CodeGenObjC/ptrauth-block-descriptor-pointer.m
@@ -0,0 +1,39 @@
+// RUN: %clang_cc1 -fobjc-arc -fblocks -fptrauth-calls -fptrauth-block-descriptor-pointers -triple arm64e-apple-ios  -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -fobjc-arc -fblocks -fptrauth-calls -triple arm64e-apple-ios -DNO_BLOCK_DESC_AUTH -emit-llvm -o - %s | FileCheck %s --check-prefix=NODESCRIPTORAUTH
+
+#ifndef NO_BLOCK_DESC_AUTH
+_Static_assert(__has_feature(ptrauth_signed_block_descriptors), "-fptrauth-block-descriptor-pointers should set ptrauth_signed_block_descriptors");
+#else
+_Static_assert(!__has_feature(ptrauth_signed_block_descriptors), "-fptrauth-block-descriptor-pointers should not be enabled by default");
+#endif
+
+void a() {
+  // Test out a global block.
+  void (^blk)(void) = ^{};
+}
+
+// CHECK: [[BLOCK_DESCRIPTOR_NAME:@"__block_descriptor_.*"]] = linkonce_odr hidden unnamed_addr constant { i64, i64, ptr, ptr } { i64 0, i64 32, ptr @.str, ptr null }
+// CHECK: @__block_literal_global = internal constant { ptr, i32, i32, ptr, ptr } { ptr @_NSConcreteGlobalBlock, i32 1342177280, i32 0, ptr ptrauth (ptr @__a_block_invoke, i32 0, i64 0, ptr getelementptr inbounds ({ ptr, i32, i32, ptr, ptr }, ptr @__block_literal_global, i32 0, i32 3)), ptr ptrauth (ptr [[BLOCK_DESCRIPTOR_NAME]], i32 2, i64 49339, ptr getelementptr inbounds ({ ptr, i32, i32, ptr, ptr }, ptr @__block_literal_global, i32 0, i32 4)) }
+
+// NODESCRIPTORAUTH: [[BLOCK_DESCRIPTOR_NAME:@"__block_descriptor_.*"]] = linkonce_odr hidden unnamed_addr constant { i64, i64, ptr, ptr } { i64 0, i64 32, ptr @.str, ptr null }
+// NODESCRIPTORAUTH: @__block_literal_global = internal constant { ptr, i32, i32, ptr, ptr } { ptr @_NSConcreteGlobalBlock, i32 1342177280, i32 0, ptr ptrauth (ptr @__a_block_invoke, i32 0, i64 0, ptr getelementptr inbounds ({ ptr, i32, i32, ptr, ptr }, ptr @__block_literal_global, i32 0, i32 3)), ptr [[BLOCK_DESCRIPTOR_NAME]] }
+
+
+void b(int p) {
+  // CHECK-LABEL: define void @b
+
+  // Test out a stack block.
+  void (^blk)(void) = ^{(void)p;};
+
+  // CHECK: [[BLOCK:%.*]] = alloca <{ ptr, i32, i32, ptr, ptr, i32 }>
+  // CHECK: [[BLOCK_DESCRIPTOR_REF:%.*]] = getelementptr inbounds nuw <{ {{.*}} }>, ptr [[BLOCK]], i32 0, i32 4
+  // CHECK: [[BLOCK_DESCRIPTOR_REF_INT:%.*]] = ptrtoint ptr [[BLOCK_DESCRIPTOR_REF]] to i64
+  // CHECK: [[BLENDED:%.*]] = call i64 @llvm.ptrauth.blend(i64 [[BLOCK_DESCRIPTOR_REF_INT]], i64 49339)
+  // CHECK: [[SIGNED_REF:%.*]] = call i64 @llvm.ptrauth.sign(i64 ptrtoint (ptr @"__block_descriptor_36_e5_v8\01?0l" to i64), i32 2, i64 [[BLENDED]])
+  // CHECK: [[SIGNED_REF_PTR:%.*]] = inttoptr i64 [[SIGNED_REF]] to ptr
+  // CHECK: store ptr [[SIGNED_REF_PTR]], ptr [[BLOCK_DESCRIPTOR_REF]]
+
+  // NODESCRIPTORAUTH: [[BLOCK:%.*]] = alloca <{ ptr, i32, i32, ptr, ptr, i32 }>
+  // NODESCRIPTORAUTH: [[BLOCK_DESCRIPTOR_REF:%.*]] = getelementptr inbounds nuw <{ {{.*}} }>, ptr [[BLOCK]], i32 0, i32 4
+  // NODESCRIPTORAUTH: store ptr @"__block_descriptor_36_e5_v8\01?0l", ptr [[BLOCK_DESCRIPTOR_REF]]
+}
diff --git a/clang/test/CodeGenObjC/ptrauth-block-isa.m b/clang/test/CodeGenObjC/ptrauth-block-isa.m
index c1e98c6fd9d3e..248e57769ba1e 100644
--- a/clang/test/CodeGenObjC/ptrauth-block-isa.m
+++ b/clang/test/CodeGenObjC/ptrauth-block-isa.m
@@ -1,7 +1,8 @@
-// RUN: %clang_cc1 -fptrauth-calls -fptrauth-objc-isa -fobjc-arc -fblocks -triple arm64e -emit-llvm %s  -o - | FileCheck %s
+// RUN: %clang_cc1 -fptrauth-calls -fptrauth-objc-isa -fobjc-arc -fblocks -triple arm64e -emit-llvm %s -o - | FileCheck %s
 
 void (^globalblock)(void) = ^{};
-// CHECK: [[GLOBAL_BLOCK:@.*]] = internal constant { ptr, i32, i32, ptr, ptr } { ptr ptrauth (ptr @_NSConcreteGlobalBlock, i32 2, i64 27361, ptr [[GLOBAL_BLOCK]]), i32 1342177280, i32 0, ptr @globalblock_block_invoke, ptr @"__block_descriptor_32_e5_v8\01?0l" }, align 8 #0
+// CHECK: [[BLOCK_DESCRIPTOR_NAME:@"__block_descriptor_.*"]] = linkonce_odr hidden unnamed_addr constant { i64, i64, ptr, ptr } { i64 0, i64 32, ptr @.str, ptr null }, comdat, align 8
+// CHECK: @__block_literal_global = internal constant { ptr, i32, i32, ptr, ptr } { ptr ptrauth (ptr @_NSConcreteGlobalBlock, i32 2, i64 27361, ptr @__block_literal_global), i32 1342177280, i32 0, ptr ptrauth (ptr @globalblock_block_invoke, i32 0, i64 0, ptr getelementptr inbounds ({ ptr, i32, i32, ptr, ptr }, ptr @__block_literal_global, i32 0, i32 3)), ptr [[BLOCK_DESCRIPTOR_NAME]] }
 
 @interface A
 - (int) count;
diff --git a/clang/test/CodeGenObjCXX/arc-move.mm b/clang/test/CodeGenObjCXX/arc-move.mm
index 9d25a6ac31939..bda03c169f97e 100644
--- a/clang/test/CodeGenObjCXX/arc-move.mm
+++ b/clang/test/CodeGenObjCXX/arc-move.mm
@@ -48,7 +48,7 @@ void library_move(__strong id &x, __strong id &y) {
 void library_move(__strong id &y) {
   // CHECK: [[X:%x]] = alloca ptr, align 8
   // CHECK: [[I:%.*]] = alloca i32, align 4
-  // CHECK: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+  // CHECK: call void @llvm.lifetime.start.p0(ptr [[X]])
   // CHECK: [[Y:%[a-zA-Z0-9]+]] = call noundef nonnull align {{[0-9]+}} dereferenceable({{[0-9]+}}) ptr @_Z4moveIRU8__strongP11objc_objectEON16remove_referenceIT_E4typeEOS5_
   // Load the object
   // CHECK-NEXT: [[OBJ:%[a-zA-Z0-9]+]] = load ptr, ptr [[Y]]
@@ -58,13 +58,13 @@ void library_move(__strong id &y) {
   // CHECK-NEXT: store ptr [[OBJ]], ptr [[X:%[a-zA-Z0-9]+]]
   id x = move(y);
 
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[I]])
   // CHECK-NEXT: store i32 17
   int i = 17;
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[I]])
   // CHECK-NEXT: [[OBJ:%[a-zA-Z0-9]+]] = load ptr, ptr [[X]]
   // CHECK-NEXT: call void @llvm.objc.release(ptr [[OBJ]])
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[X]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[X]])
   // CHECK-NEXT: ret void
 }
 
diff --git a/clang/test/CodeGenObjCXX/arc-references.mm b/clang/test/CodeGenObjCXX/arc-references.mm
index 273e339bec509..5fad327c0789e 100644
--- a/clang/test/CodeGenObjCXX/arc-references.mm
+++ b/clang/test/CodeGenObjCXX/arc-references.mm
@@ -45,7 +45,7 @@ void test3() {
   // CHECK-NEXT: call void @_Z6calleev()
   callee();
   // CHECK-NEXT: call void @llvm.objc.destroyWeak
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[REF]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[REF]])
   // CHECK-NEXT: ret void
 }
 
@@ -71,10 +71,10 @@ void test5(__strong id &x) {
   sink(x);  
   // CHECK-NEXT: [[OBJ_A:%[a-zA-Z0-9]+]] = load ptr, ptr [[REFTMP]]
   // CHECK-NEXT: call void @llvm.objc.release
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr [[I]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[I]])
   // CHECK-NEXT: store i32 17, ptr
   int i = 17;
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr [[I]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[I]])
   // CHECK-NEXT: ret void
 }
 
diff --git a/clang/test/CodeGenObjCXX/arc.mm b/clang/test/CodeGenObjCXX/arc.mm
index 7883378b33828..20f1e375d272e 100644
--- a/clang/test/CodeGenObjCXX/arc.mm
+++ b/clang/test/CodeGenObjCXX/arc.mm
@@ -61,9 +61,9 @@ void test34(int cond) {
   // CHECK-NEXT: [[CONDCLEANUPSAVE:%.*]] = alloca ptr
   // CHECK-NEXT: [[CONDCLEANUP:%.*]] = alloca i1
   // CHECK-NEXT: store i32
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[STRONG]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[STRONG]])
   // CHECK-NEXT: store ptr null, ptr [[STRONG]]
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[WEAK]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[WEAK]])
   // CHECK-NEXT: call ptr @llvm.objc.initWeak(ptr [[WEAK]], ptr null)
 
   // CHECK-NEXT: [[T0:%.*]] = load i32, ptr [[COND]]
@@ -293,7 +293,7 @@ + (void) foo:(id *)errorPtr;
 // CHECK-LABEL:    define weak_odr void @_Z13test40_helperIiEvv()
 // CHECK:      [[X:%.*]] = alloca ptr
 // CHECK-NEXT: [[TEMP:%.*]] = alloca ptr
-// CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr [[X]])
+// CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[X]])
 // CHECK-NEXT: store ptr null, ptr [[X]]
 // CHECK:      [[T0:%.*]] = load ptr, ptr [[X]]
 // CHECK-NEXT: store ptr [[T0]], ptr [[TEMP]]
diff --git a/clang/test/CodeGenObjCXX/literals.mm b/clang/test/CodeGenObjCXX/literals.mm
index 737aa9e6c5130..b7938ccc5867a 100644
--- a/clang/test/CodeGenObjCXX/literals.mm
+++ b/clang/test/CodeGenObjCXX/literals.mm
@@ -22,16 +22,16 @@ void test_array() {
   // CHECK: [[TMPY:%[a-zA-Z0-9.]+]] = alloca %
 
   // Initializing first element
-  // CHECK: call void @llvm.lifetime.start.p0(i64 8, ptr [[ARR]])
+  // CHECK: call void @llvm.lifetime.start.p0(ptr [[ARR]])
   // CHECK: [[ELEMENT0:%[a-zA-Z0-9.]+]] = getelementptr inbounds [2 x ptr], ptr [[OBJECTS]], i64 0, i64 0
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 1, ptr [[TMPX]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMPX]])
   // CHECK-NEXT: call void @_ZN1XC1Ev({{.*}} [[TMPX]])
   // CHECK-NEXT: [[OBJECT0:%[a-zA-Z0-9.]+]] = invoke noundef ptr @_ZNK1XcvP11objc_objectEv{{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK: store ptr [[OBJECT0]], ptr [[ELEMENT0]]
   
   // Initializing the second element
   // CHECK: [[ELEMENT1:%[a-zA-Z0-9.]+]] = getelementptr inbounds [2 x ptr], ptr [[OBJECTS]], i64 0, i64 1
-  // CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 1, ptr [[TMPY]])
+  // CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[TMPY]])
   // CHECK-NEXT: invoke void @_ZN1YC1Ev({{.*}} [[TMPY]])
   // CHECK: [[OBJECT1:%[a-zA-Z0-9.]+]] = invoke noundef ptr @_ZNK1YcvP11objc_objectEv{{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
   // CHECK: store ptr [[OBJECT1]], ptr [[ELEMENT1]]
@@ -50,7 +50,7 @@ void test_array() {
   // CHECK-NEXT: call void @_ZN1XD1Ev
   // CHECK-NOT: ret void
   // CHECK: call void @llvm.objc.release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[ARR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ARR]])
   // CHECK-NEXT: ret void
 
   // Check cleanups
@@ -71,7 +71,7 @@ void test_array_instantiation() {
   // CHECK: [[OBJECTS:%[a-zA-Z0-9.]+]] = alloca [2 x ptr]
 
   // Initializing first element
-  // CHECK: call void @llvm.lifetime.start.p0(i64 8, ptr [[ARR]])
+  // CHECK: call void @llvm.lifetime.start.p0(ptr [[ARR]])
   // CHECK: [[ELEMENT0:%[a-zA-Z0-9.]+]] = getelementptr inbounds [2 x ptr], ptr [[OBJECTS]], i64 0, i64 0
   // CHECK: call void @_ZN1XC1Ev
   // CHECK-NEXT: [[OBJECT0:%[a-zA-Z0-9.]+]] = invoke noundef ptr @_ZNK1XcvP11objc_objectEv{{.*}} [ "clang.arc.attachedcall"(ptr @llvm.objc.retainAutoreleasedReturnValue) ]
@@ -97,7 +97,7 @@ void test_array_instantiation() {
   // CHECK-NEXT: call void @_ZN1XD1Ev
   // CHECK-NOT: ret void
   // CHECK: call void @llvm.objc.release
-  // CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr [[ARR]])
+  // CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[ARR]])
   // CHECK-NEXT: ret void
 
   // Check cleanups
diff --git a/clang/test/CodeGenOpenCL/addr-space-struct-arg.cl b/clang/test/CodeGenOpenCL/addr-space-struct-arg.cl
index a70e9af75fa38..85157bdcf43f9 100644
--- a/clang/test/CodeGenOpenCL/addr-space-struct-arg.cl
+++ b/clang/test/CodeGenOpenCL/addr-space-struct-arg.cl
@@ -647,6 +647,7 @@ kernel void KernelLargeTwoMember(struct LargeStructTwoMember u) {
 // AMDGCN20-NEXT:    [[IN:%.*]] = alloca [[STRUCT_MAT3X3:%.*]], align 4, addrspace(5)
 // AMDGCN20-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
 // AMDGCN20-NEXT:    [[IN1:%.*]] = addrspacecast ptr addrspace(5) [[IN]] to ptr
+// AMDGCN20-NEXT:    [[RETVAL_ASCAST_ASCAST:%.*]] = addrspacecast ptr [[RETVAL_ASCAST]] to ptr addrspace(5)
 // AMDGCN20-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_MAT3X3]], ptr [[IN1]], i32 0, i32 0
 // AMDGCN20-NEXT:    store [9 x i32] [[IN_COERCE]], ptr [[COERCE_DIVE]], align 4
 // AMDGCN20-NEXT:    [[TMP0:%.*]] = load [[STRUCT_MAT4X4]], ptr [[RETVAL_ASCAST]], align 4
diff --git a/clang/test/CodeGenOpenCL/amdgpu-abi-struct-arg-byref.cl b/clang/test/CodeGenOpenCL/amdgpu-abi-struct-arg-byref.cl
index a1a114ef129a1..bc65788c17352 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-abi-struct-arg-byref.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-abi-struct-arg-byref.cl
@@ -121,6 +121,7 @@ kernel void KernelLargeTwoMember(struct LargeStructTwoMember u) {
 // AMDGCN-NEXT:    [[IN:%.*]] = alloca [[STRUCT_MAT3X3:%.*]], align 4, addrspace(5)
 // AMDGCN-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
 // AMDGCN-NEXT:    [[IN1:%.*]] = addrspacecast ptr addrspace(5) [[IN]] to ptr
+// AMDGCN-NEXT:    [[RETVAL_ASCAST_ASCAST:%.*]] = addrspacecast ptr [[RETVAL_ASCAST]] to ptr addrspace(5)
 // AMDGCN-NEXT:    [[COERCE_DIVE:%.*]] = getelementptr inbounds nuw [[STRUCT_MAT3X3]], ptr [[IN1]], i32 0, i32 0
 // AMDGCN-NEXT:    store [9 x i32] [[IN_COERCE]], ptr [[COERCE_DIVE]], align 4
 // AMDGCN-NEXT:    [[TMP0:%.*]] = load [[STRUCT_MAT4X4]], ptr [[RETVAL_ASCAST]], align 4
diff --git a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
index bfbed79dc7f16..d71c89811f04b 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-enqueue-kernel.cl
@@ -523,10 +523,10 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    store i8 [[B]], ptr [[B_ADDR_ASCAST]], align 1, !tbaa [[TBAA16]]
 // GFX900-NEXT:    store ptr addrspace(1) [[C]], ptr [[C_ADDR_ASCAST]], align 8, !tbaa [[TBAA7]]
 // GFX900-NEXT:    store i64 [[D]], ptr [[D_ADDR_ASCAST]], align 8, !tbaa [[TBAA3]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR9:[0-9]+]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR9:[0-9]+]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[FLAGS]]) #[[ATTR9]]
 // GFX900-NEXT:    store i32 0, ptr addrspace(5) [[FLAGS]], align 4, !tbaa [[TBAA17:![0-9]+]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[NDRANGE]]) #[[ATTR9]]
 // GFX900-NEXT:    [[TMP0:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[DEFAULT_QUEUE]], align 8, !tbaa [[TBAA19:![0-9]+]]
 // GFX900-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(5) [[FLAGS]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    call void @llvm.memcpy.p0.p5.i64(ptr align 4 [[TMP_ASCAST]], ptr addrspace(5) align 4 [[NDRANGE]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21:![0-9]+]]
@@ -586,12 +586,12 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[BLOCK_CAPTURED19:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, ptr addrspace(1), ptr addrspace(1), i64, i8 }>, ptr [[BLOCK12_ASCAST]], i32 0, i32 5
 // GFX900-NEXT:    [[TMP17:%.*]] = load i64, ptr [[D_ADDR_ASCAST]], align 8, !tbaa [[TBAA3]]
 // GFX900-NEXT:    store i64 [[TMP17]], ptr [[BLOCK_CAPTURED19]], align 8, !tbaa [[TBAA3]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[BLOCK_SIZES]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[BLOCK_SIZES]]) #[[ATTR9]]
 // GFX900-NEXT:    [[TMP18:%.*]] = getelementptr [1 x i64], ptr addrspace(5) [[BLOCK_SIZES]], i32 0, i32 0
 // GFX900-NEXT:    store i64 100, ptr addrspace(5) [[TMP18]], align 8
 // GFX900-NEXT:    [[TMP19:%.*]] = call i32 @__enqueue_kernel_varargs(ptr addrspace(1) [[TMP12]], i32 [[TMP13]], ptr addrspace(5) [[VARTMP11]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_3_kernel.runtime.handle to ptr), ptr [[BLOCK12_ASCAST]], i32 1, ptr addrspace(5) [[TMP18]])
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[BLOCK_SIZES]]) #[[ATTR9]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[BLOCK20]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[BLOCK_SIZES]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[BLOCK20]]) #[[ATTR9]]
 // GFX900-NEXT:    [[BLOCK_SIZE22:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, i64, ptr addrspace(1) }>, ptr [[BLOCK21_ASCAST]], i32 0, i32 0
 // GFX900-NEXT:    store i32 32, ptr [[BLOCK_SIZE22]], align 8
 // GFX900-NEXT:    [[BLOCK_ALIGN23:%.*]] = getelementptr inbounds nuw <{ i32, i32, ptr, i64, ptr addrspace(1) }>, ptr [[BLOCK21_ASCAST]], i32 0, i32 1
@@ -610,10 +610,10 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    call void @llvm.memcpy.p0.p5.i64(ptr align 4 [[TMP27_ASCAST]], ptr addrspace(5) align 4 [[NDRANGE]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
 // GFX900-NEXT:    [[TMP24:%.*]] = load ptr, ptr addrspace(5) [[BLOCK20]], align 8, !tbaa [[TBAA16]]
 // GFX900-NEXT:    [[TMP25:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP22]], i32 [[TMP23]], ptr addrspace(5) [[VARTMP27]], ptr addrspacecast (ptr addrspace(1) @__test_block_invoke_4_kernel.runtime.handle to ptr), ptr [[BLOCK21_ASCAST]])
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[BLOCK20]]) #[[ATTR9]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR9]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR9]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[BLOCK20]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[NDRANGE]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[FLAGS]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR9]]
 // GFX900-NEXT:    ret void
 //
 //
@@ -641,18 +641,18 @@ kernel void test_target_features_kernel(global int *i) {
 // GFX900-NEXT:    [[I_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[I_ADDR]] to ptr
 // GFX900-NEXT:    [[TMP_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[TMP]] to ptr
 // GFX900-NEXT:    store ptr addrspace(1) [[I]], ptr [[I_ADDR_ASCAST]], align 8, !tbaa [[TBAA26]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR9]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[FLAGS]]) #[[ATTR9]]
 // GFX900-NEXT:    store i32 0, ptr addrspace(5) [[FLAGS]], align 4, !tbaa [[TBAA17]]
-// GFX900-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[NDRANGE]]) #[[ATTR9]]
 // GFX900-NEXT:    [[TMP0:%.*]] = call i64 @llvm.amdgcn.s.memtime()
 // GFX900-NEXT:    [[TMP1:%.*]] = load ptr addrspace(1), ptr addrspace(5) [[DEFAULT_QUEUE]], align 8, !tbaa [[TBAA19]]
 // GFX900-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(5) [[FLAGS]], align 4, !tbaa [[TBAA17]]
 // GFX900-NEXT:    call void @llvm.memcpy.p0.p5.i64(ptr align 4 [[TMP_ASCAST]], ptr addrspace(5) align 4 [[NDRANGE]], i64 4, i1 false), !tbaa.struct [[TBAA_STRUCT21]]
 // GFX900-NEXT:    [[TMP3:%.*]] = call i32 @__enqueue_kernel_basic(ptr addrspace(1) [[TMP1]], i32 [[TMP2]], ptr addrspace(5) [[TMP]], ptr addrspacecast (ptr addrspace(1) @__test_target_features_kernel_block_invoke_kernel.runtime.handle to ptr), ptr addrspacecast (ptr addrspace(1) @__block_literal_global to ptr))
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[NDRANGE]]) #[[ATTR9]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[FLAGS]]) #[[ATTR9]]
-// GFX900-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[NDRANGE]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[FLAGS]]) #[[ATTR9]]
+// GFX900-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[DEFAULT_QUEUE]]) #[[ATTR9]]
 // GFX900-NEXT:    ret void
 //
 //
diff --git a/clang/test/CodeGenOpenCL/amdgpu-printf.cl b/clang/test/CodeGenOpenCL/amdgpu-printf.cl
index 33fee66d6e9fc..b9e25172a56af 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-printf.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-printf.cl
@@ -65,12 +65,12 @@ __kernel void test_printf_str_int(int i) {
 // CHECK-NEXT:    [[I_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
 // CHECK-NEXT:    [[S:%.*]] = alloca [4 x i8], align 1, addrspace(5)
 // CHECK-NEXT:    store i32 [[I]], ptr addrspace(5) [[I_ADDR]], align 4, !tbaa [[TBAA9]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[S]]) #[[ATTR7:[0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[S]]) #[[ATTR7:[0-9]+]]
 // CHECK-NEXT:    call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) align 1 [[S]], ptr addrspace(4) align 1 @__const.test_printf_str_int.s, i64 4, i1 false)
 // CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [4 x i8], ptr addrspace(5) [[S]], i64 0, i64 0
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[I_ADDR]], align 4, !tbaa [[TBAA9]]
 // CHECK-NEXT:    [[CALL:%.*]] = call i32 (ptr addrspace(4), ...) @printf(ptr addrspace(4) noundef @.str.2, ptr addrspace(5) noundef [[ARRAYDECAY]], i32 noundef [[TMP0]]) #[[ATTR6]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[S]]) #[[ATTR7]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[S]]) #[[ATTR7]]
 // CHECK-NEXT:    ret void
 //
 //.
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12-param-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12-param-err.cl
index 5d86a9b369429..1a5043328895a 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12-param-err.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12-param-err.cl
@@ -23,6 +23,13 @@ kernel void builtins_amdgcn_s_barrier_signal_isfirst_err(global int* in, global
   *out = *in;
 }
 
+kernel void builtins_amdgcn_s_barrier_leave_err(global int* in, global int* out, int barrier) {
+
+  __builtin_amdgcn_s_barrier_signal(-1);
+  __builtin_amdgcn_s_barrier_leave(barrier); // expected-error {{'__builtin_amdgcn_s_barrier_leave' must be a constant integer}}
+  *out = *in;
+}
+
 void test_s_buffer_prefetch_data(__amdgpu_buffer_rsrc_t rsrc, unsigned int off)
 {
   __builtin_amdgcn_s_buffer_prefetch_data(rsrc, off, 31); // expected-error {{'__builtin_amdgcn_s_buffer_prefetch_data' must be a constant integer}}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
index f7641280715c8..8c02616780182 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
@@ -139,6 +139,50 @@ void test_s_barrier_signal_isfirst(int* a, int* b, int *c)
   __builtin_amdgcn_s_barrier_wait(1);
 }
 
+// CHECK-LABEL: @test_s_barrier_init(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[BAR_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[BAR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[BAR_ADDR]] to ptr
+// CHECK-NEXT:    [[A_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A_ADDR]] to ptr
+// CHECK-NEXT:    store ptr [[BAR:%.*]], ptr [[BAR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BAR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[TMP0]] to ptr addrspace(3)
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR_ASCAST]], align 4
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.init(ptr addrspace(3) [[TMP1]], i32 [[TMP2]])
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_init(void *bar, int a)
+{
+  __builtin_amdgcn_s_barrier_init(bar, a);
+}
+
+// CHECK-LABEL: @test_s_barrier_join(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[BAR_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[BAR_ADDR_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[BAR_ADDR]] to ptr
+// CHECK-NEXT:    store ptr [[BAR:%.*]], ptr [[BAR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[BAR_ADDR_ASCAST]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[TMP0]] to ptr addrspace(3)
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) [[TMP1]])
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_join(void *bar)
+{
+  __builtin_amdgcn_s_barrier_join(bar);
+}
+
+// CHECK-LABEL: @test_s_barrier_leave(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.leave(i16 1)
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_leave()
+{
+  __builtin_amdgcn_s_barrier_leave(1);
+}
+
 // CHECK-LABEL: @test_s_get_barrier_state(
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
diff --git a/clang/test/CodeGenOpenCL/cl20-device-side-enqueue.cl b/clang/test/CodeGenOpenCL/cl20-device-side-enqueue.cl
index 6c85e734c0eb4..b0dbf601eec39 100644
--- a/clang/test/CodeGenOpenCL/cl20-device-side-enqueue.cl
+++ b/clang/test/CodeGenOpenCL/cl20-device-side-enqueue.cl
@@ -128,9 +128,9 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // SPIR: [[DEF_Q:%[0-9]+]] = load target("spirv.Queue"), ptr %default_queue
   // X86: [[DEF_Q:%[0-9]+]] = load ptr, ptr %default_queue
   // COMMON: [[FLAGS:%[0-9]+]] = load i32, ptr %flags
-  // CHECK-LIFETIMES: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %[[BLOCK_SIZES1]])
+  // CHECK-LIFETIMES: call void @llvm.lifetime.start.p0(ptr nonnull %[[BLOCK_SIZES1]])
   // CHECK-LIFETIMES-LABEL: call {{(spir_func )?}}i32 @__enqueue_kernel_varargs(
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %[[BLOCK_SIZES1]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull %[[BLOCK_SIZES1]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], ptr %[[BLOCK_SIZES1]], i32 0, i32 0
   // B32: store i32 256, ptr %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], ptr %[[BLOCK_SIZES1]], i32 0, i32 0
@@ -153,9 +153,9 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // SPIR: [[DEF_Q:%[0-9]+]] = load target("spirv.Queue"), ptr %default_queue
   // X86: [[DEF_Q:%[0-9]+]] = load ptr, ptr %default_queue
   // COMMON: [[FLAGS:%[0-9]+]] = load i32, ptr %flags
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %[[BLOCK_SIZES2]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull %[[BLOCK_SIZES2]])
   // CHECK-LIFETIMES-LABEL: call {{(spir_func )?}}i32 @__enqueue_kernel_varargs(
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %[[BLOCK_SIZES2]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull %[[BLOCK_SIZES2]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], ptr %[[BLOCK_SIZES2]], i32 0, i32 0
   // B32: store i32 %{{.*}}, ptr %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], ptr %[[BLOCK_SIZES2]], i32 0, i32 0
@@ -181,9 +181,9 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // X86: [[AD:%arraydecay[0-9]*]] = getelementptr inbounds [1 x ptr], ptr %event_wait_list2, i{{32|64}} 0, i{{32|64}} 0
   // COMMON: [[WAIT_EVNT:%[0-9]+]] ={{.*}} addrspacecast ptr [[AD]] to ptr addrspace(4)
   // COMMON: [[EVNT:%[0-9]+]]  ={{.*}} addrspacecast ptr %clk_event to ptr addrspace(4)
-  // CHECK-LIFETIMES: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %[[BLOCK_SIZES3]])
+  // CHECK-LIFETIMES: call void @llvm.lifetime.start.p0(ptr nonnull %[[BLOCK_SIZES3]])
   // CHECK-LIFETIMES-LABEL: call {{(spir_func )?}}i32 @__enqueue_kernel_events_varargs(
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %[[BLOCK_SIZES3]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull %[[BLOCK_SIZES3]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], ptr %[[BLOCK_SIZES3]], i32 0, i32 0
   // B32: store i32 256, ptr %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], ptr %[[BLOCK_SIZES3]], i32 0, i32 0
@@ -209,9 +209,9 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // X86: [[AD:%arraydecay[0-9]*]] = getelementptr inbounds [1 x ptr], ptr %event_wait_list2, i{{32|64}} 0, i{{32|64}} 0
   // COMMON: [[WAIT_EVNT:%[0-9]+]] ={{.*}} addrspacecast ptr [[AD]] to ptr addrspace(4)
   // COMMON: [[EVNT:%[0-9]+]]  ={{.*}} addrspacecast ptr %clk_event to ptr addrspace(4)
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %[[BLOCK_SIZES4]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull %[[BLOCK_SIZES4]])
   // CHECK-LIFETIMES-LABEL: call {{(spir_func )?}}i32 @__enqueue_kernel_events_varargs(
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %[[BLOCK_SIZES4]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull %[[BLOCK_SIZES4]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], ptr %[[BLOCK_SIZES4]], i32 0, i32 0
   // B32: store i32 %{{.*}}, ptr %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], ptr %[[BLOCK_SIZES4]], i32 0, i32 0
@@ -234,9 +234,9 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // SPIR: [[DEF_Q:%[0-9]+]] = load target("spirv.Queue"), ptr %default_queue
   // X86: [[DEF_Q:%[0-9]+]] = load ptr, ptr %default_queue
   // COMMON: [[FLAGS:%[0-9]+]] = load i32, ptr %flags
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %[[BLOCK_SIZES5]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull %[[BLOCK_SIZES5]])
   // CHECK-LIFETIMES-LABEL: call {{(spir_func )?}}i32 @__enqueue_kernel_varargs(
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %[[BLOCK_SIZES5]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull %[[BLOCK_SIZES5]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], ptr %[[BLOCK_SIZES5]], i32 0, i32 0
   // B32: store i32 %{{.*}}, ptr %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], ptr %[[BLOCK_SIZES5]], i32 0, i32 0
@@ -258,9 +258,9 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // SPIR: [[DEF_Q:%[0-9]+]] = load target("spirv.Queue"), ptr %default_queue
   // X86: [[DEF_Q:%[0-9]+]] = load ptr, ptr %default_queue
   // COMMON: [[FLAGS:%[0-9]+]] = load i32, ptr %flags
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %[[BLOCK_SIZES6]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull %[[BLOCK_SIZES6]])
   // CHECK-LIFETIMES-LABEL: call {{(spir_func )?}}i32 @__enqueue_kernel_varargs(
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %[[BLOCK_SIZES6]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull %[[BLOCK_SIZES6]])
   // B32: %[[TMP:.*]] = getelementptr [3 x i32], ptr %[[BLOCK_SIZES6]], i32 0, i32 0
   // B32: store i32 1, ptr %[[TMP]], align 4
   // B32: %[[BLOCK_SIZES62:.*]] = getelementptr [3 x i32], ptr %[[BLOCK_SIZES6]], i32 0, i32 1
@@ -290,9 +290,9 @@ kernel void device_side_enqueue(global int *a, global int *b, int i) {
   // SPIR: [[DEF_Q:%[0-9]+]] = load target("spirv.Queue"), ptr %default_queue
   // X86: [[DEF_Q:%[0-9]+]] = load ptr, ptr %default_queue
   // COMMON: [[FLAGS:%[0-9]+]] = load i32, ptr %flags
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %[[BLOCK_SIZES7]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.start.p0(ptr nonnull %[[BLOCK_SIZES7]])
   // CHECK-LIFETIMES-LABEL: call {{(spir_func )?}}i32 @__enqueue_kernel_varargs(
-  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %[[BLOCK_SIZES7]])
+  // CHECK-LIFETIMES-NEXT: call void @llvm.lifetime.end.p0(ptr nonnull %[[BLOCK_SIZES7]])
   // B32: %[[TMP:.*]] = getelementptr [1 x i32], ptr %[[BLOCK_SIZES7]], i32 0, i32 0
   // B32: store i32 0, ptr %[[TMP]], align 4
   // B64: %[[TMP:.*]] = getelementptr [1 x i64], ptr %[[BLOCK_SIZES7]], i32 0, i32 0
diff --git a/clang/test/CodeGenOpenCL/implicit-addrspacecast-function-parameter.cl b/clang/test/CodeGenOpenCL/implicit-addrspacecast-function-parameter.cl
index 8845ffe499fde..4e40073c7e27a 100644
--- a/clang/test/CodeGenOpenCL/implicit-addrspacecast-function-parameter.cl
+++ b/clang/test/CodeGenOpenCL/implicit-addrspacecast-function-parameter.cl
@@ -32,12 +32,12 @@ __kernel void use_of_local_var()
 // CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[X:%.*]] = alloca i32, align 4, addrspace(5)
-// CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[X]]) #[[ATTR5:[0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[X]]) #[[ATTR5:[0-9]+]]
 // CHECK-NEXT:    store i32 0, ptr addrspace(5) [[X]], align 4, !tbaa [[TBAA4:![0-9]+]]
 // CHECK-NEXT:    call void @private_ptr(ptr addrspace(5) noundef [[X]]) #[[ATTR6:[0-9]+]]
 // CHECK-NEXT:    [[X_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[X]] to ptr
 // CHECK-NEXT:    call void @generic_ptr(ptr noundef [[X_ASCAST]]) #[[ATTR6]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[X]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[X]]) #[[ATTR5]]
 // CHECK-NEXT:    ret void
 //
 //
diff --git a/clang/test/CodeGenOpenCL/preserve_vec3.cl b/clang/test/CodeGenOpenCL/preserve_vec3.cl
index 49ebae6fc7013..e73657e30d884 100644
--- a/clang/test/CodeGenOpenCL/preserve_vec3.cl
+++ b/clang/test/CodeGenOpenCL/preserve_vec3.cl
@@ -11,8 +11,8 @@ typedef float float4 __attribute__((ext_vector_type(4)));
 // CHECK-LABEL: define dso_local spir_kernel void @foo(
 // CHECK-SAME: ptr addrspace(1) noundef readonly align 16 captures(none) [[A:%.*]], ptr addrspace(1) noundef writeonly align 16 captures(none) initializes((0, 16)) [[B:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] !kernel_arg_addr_space [[META3:![0-9]+]] !kernel_arg_access_qual [[META4:![0-9]+]] !kernel_arg_type [[META5:![0-9]+]] !kernel_arg_base_type [[META6:![0-9]+]] !kernel_arg_type_qual [[META7:![0-9]+]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[LOADVECN:%.*]] = load <4 x float>, ptr addrspace(1) [[A]], align 16
-// CHECK-NEXT:    [[EXTRACTVEC1:%.*]] = shufflevector <4 x float> [[LOADVECN]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+// CHECK-NEXT:    [[TMP0:%.*]] = load <3 x float>, ptr addrspace(1) [[A]], align 16
+// CHECK-NEXT:    [[EXTRACTVEC1:%.*]] = shufflevector <3 x float> [[TMP0]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 // CHECK-NEXT:    store <4 x float> [[EXTRACTVEC1]], ptr addrspace(1) [[B]], align 16, !tbaa [[TBAA8:![0-9]+]]
 // CHECK-NEXT:    ret void
 //
@@ -23,8 +23,8 @@ void kernel foo(global float3 *a, global float3 *b) {
 // CHECK-LABEL: define dso_local spir_kernel void @float4_to_float3(
 // CHECK-SAME: ptr addrspace(1) noundef writeonly align 16 captures(none) initializes((0, 16)) [[A:%.*]], ptr addrspace(1) noundef readonly align 16 captures(none) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META11:![0-9]+]] !kernel_arg_base_type [[META12:![0-9]+]] !kernel_arg_type_qual [[META7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr addrspace(1) [[B]], align 16, !tbaa [[TBAA8]]
-// CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+// CHECK-NEXT:    [[TMP0:%.*]] = load <3 x float>, ptr addrspace(1) [[B]], align 16, !tbaa [[TBAA8]]
+// CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x float> [[TMP0]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 // CHECK-NEXT:    store <4 x float> [[EXTRACTVEC]], ptr addrspace(1) [[A]], align 16, !tbaa [[TBAA8]]
 // CHECK-NEXT:    ret void
 //
@@ -35,8 +35,8 @@ void kernel float4_to_float3(global float3 *a, global float4 *b) {
 // CHECK-LABEL: define dso_local spir_kernel void @float3_to_float4(
 // CHECK-SAME: ptr addrspace(1) noundef readonly align 16 captures(none) [[A:%.*]], ptr addrspace(1) noundef writeonly align 16 captures(none) initializes((0, 16)) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META11]] !kernel_arg_base_type [[META12]] !kernel_arg_type_qual [[META7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[LOADVECN:%.*]] = load <4 x float>, ptr addrspace(1) [[A]], align 16
-// CHECK-NEXT:    [[ASTYPE:%.*]] = shufflevector <4 x float> [[LOADVECN]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+// CHECK-NEXT:    [[TMP0:%.*]] = load <3 x float>, ptr addrspace(1) [[A]], align 16
+// CHECK-NEXT:    [[ASTYPE:%.*]] = shufflevector <3 x float> [[TMP0]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 // CHECK-NEXT:    store <4 x float> [[ASTYPE]], ptr addrspace(1) [[B]], align 16, !tbaa [[TBAA8]]
 // CHECK-NEXT:    ret void
 //
@@ -47,9 +47,9 @@ void kernel float3_to_float4(global float3 *a, global float4 *b) {
 // CHECK-LABEL: define dso_local spir_kernel void @float3_to_double2(
 // CHECK-SAME: ptr addrspace(1) noundef readonly align 16 captures(none) [[A:%.*]], ptr addrspace(1) noundef writeonly align 16 captures(none) initializes((0, 16)) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META13:![0-9]+]] !kernel_arg_base_type [[META14:![0-9]+]] !kernel_arg_type_qual [[META7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[LOADVECN:%.*]] = load <4 x float>, ptr addrspace(1) [[A]], align 16
-// CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x float> [[LOADVECN]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
-// CHECK-NEXT:    store <4 x float> [[TMP0]], ptr addrspace(1) [[B]], align 16, !tbaa [[TBAA8]]
+// CHECK-NEXT:    [[TMP0:%.*]] = load <3 x float>, ptr addrspace(1) [[A]], align 16
+// CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x float> [[TMP0]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+// CHECK-NEXT:    store <4 x float> [[TMP1]], ptr addrspace(1) [[B]], align 16, !tbaa [[TBAA8]]
 // CHECK-NEXT:    ret void
 //
 void kernel float3_to_double2(global float3 *a, global double2 *b) {
@@ -59,8 +59,8 @@ void kernel float3_to_double2(global float3 *a, global double2 *b) {
 // CHECK-LABEL: define dso_local spir_kernel void @char8_to_short3(
 // CHECK-SAME: ptr addrspace(1) noundef writeonly align 8 captures(none) initializes((0, 8)) [[A:%.*]], ptr addrspace(1) noundef readonly align 8 captures(none) [[B:%.*]]) local_unnamed_addr #[[ATTR0]] !kernel_arg_addr_space [[META3]] !kernel_arg_access_qual [[META4]] !kernel_arg_type [[META15:![0-9]+]] !kernel_arg_base_type [[META16:![0-9]+]] !kernel_arg_type_qual [[META7]] {
 // CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[B]], align 8, !tbaa [[TBAA8]]
-// CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
+// CHECK-NEXT:    [[TMP0:%.*]] = load <3 x i16>, ptr addrspace(1) [[B]], align 8, !tbaa [[TBAA8]]
+// CHECK-NEXT:    [[EXTRACTVEC:%.*]] = shufflevector <3 x i16> [[TMP0]], <3 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 poison>
 // CHECK-NEXT:    store <4 x i16> [[EXTRACTVEC]], ptr addrspace(1) [[A]], align 8, !tbaa [[TBAA8]]
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/CodeGenSPIRV/Builtins/generic_cast_to_ptr_explicit.c b/clang/test/CodeGenSPIRV/Builtins/generic_cast_to_ptr_explicit.c
index 8cfe650f4db10..30f4ecb589a5c 100644
--- a/clang/test/CodeGenSPIRV/Builtins/generic_cast_to_ptr_explicit.c
+++ b/clang/test/CodeGenSPIRV/Builtins/generic_cast_to_ptr_explicit.c
@@ -1,33 +1,36 @@
-// RUN: %clang_cc1 -O1 -triple spirv64 -fsycl-is-device %s -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -O1 -triple spirv64 -fsycl-is-device -x c++ %s -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -O1 -triple spirv64 -cl-std=CL3.0 -x cl %s -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -O1 -triple spirv32 -cl-std=CL3.0 -x cl %s -emit-llvm -o - | FileCheck %s
 
-// CHECK: spir_func noundef ptr @test_cast_to_private(
-// CHECK-SAME: ptr addrspace(4) noundef readnone [[P:%.*]]
+#ifdef __SYCL_DEVICE_ONLY__
+#define SYCL_EXTERNAL [[clang::sycl_external]]
+#else
+#define SYCL_EXTERNAL
+#endif
+
+// CHECK: spir_func noundef ptr @{{.*}}test_cast_to_private{{.*}}(ptr addrspace(4) noundef readnone [[P:%.*]]
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[SPV_CAST:%.*]] = tail call noundef ptr @llvm.spv.generic.cast.to.ptr.explicit.p0(ptr addrspace(4) %p)
 // CHECK-NEXT:    ret ptr [[SPV_CAST]]
 //
-__attribute__((opencl_private)) int* test_cast_to_private(int* p) {
+SYCL_EXTERNAL __attribute__((opencl_private)) int* test_cast_to_private(int* p) {
     return __builtin_spirv_generic_cast_to_ptr_explicit(p, 7);
 }
 
-// CHECK: spir_func noundef ptr addrspace(1) @test_cast_to_global(
-// CHECK-SAME: ptr addrspace(4) noundef readnone [[P:%.*]]
+// CHECK: spir_func noundef ptr addrspace(1) @{{.*}}test_cast_to_global{{.*}}(ptr addrspace(4) noundef readnone [[P:%.*]]
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[SPV_CAST:%.*]] = tail call noundef ptr addrspace(1) @llvm.spv.generic.cast.to.ptr.explicit.p1(ptr addrspace(4) %p)
 // CHECK-NEXT:    ret ptr addrspace(1) [[SPV_CAST]]
 //
-__attribute__((opencl_global)) int* test_cast_to_global(int* p) {
+SYCL_EXTERNAL __attribute__((opencl_global)) int* test_cast_to_global(int* p) {
     return __builtin_spirv_generic_cast_to_ptr_explicit(p, 5);
 }
 
-// CHECK: spir_func noundef ptr addrspace(3) @test_cast_to_local(
-// CHECK-SAME: ptr addrspace(4) noundef readnone [[P:%.*]]
+// CHECK: spir_func noundef ptr addrspace(3) @{{.*}}test_cast_to_local{{.*}}(ptr addrspace(4) noundef readnone [[P:%.*]]
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    [[SPV_CAST:%.*]] = tail call noundef ptr addrspace(3) @llvm.spv.generic.cast.to.ptr.explicit.p3(ptr addrspace(4) %p)
 // CHECK-NEXT:    ret ptr addrspace(3) [[SPV_CAST]]
 //
-__attribute__((opencl_local)) int* test_cast_to_local(int* p) {
+SYCL_EXTERNAL __attribute__((opencl_local)) int* test_cast_to_local(int* p) {
     return __builtin_spirv_generic_cast_to_ptr_explicit(p, 4);
 }
diff --git a/clang/test/CodeGenSPIRV/Builtins/ids_and_ranges.c b/clang/test/CodeGenSPIRV/Builtins/ids_and_ranges.c
index f71af779ec358..bff850b3622b9 100644
--- a/clang/test/CodeGenSPIRV/Builtins/ids_and_ranges.c
+++ b/clang/test/CodeGenSPIRV/Builtins/ids_and_ranges.c
@@ -1,106 +1,106 @@
-// RUN: %clang_cc1 -O1 -triple spirv64 -fsycl-is-device %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK64
+// RUN: %clang_cc1 -O1 -triple spirv64 -fsycl-is-device -x c++ %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK64
 // RUN: %clang_cc1 -O1 -triple spirv64 -cl-std=CL3.0 -x cl %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK64
 // RUN: %clang_cc1 -O1 -triple spirv32 -cl-std=CL3.0 -x cl %s -emit-llvm -o - | FileCheck %s --check-prefixes=CHECK,CHECK32
 
-// CHECK: @test_num_workgroups(
+// CHECK: @{{.*}}test_num_workgroups{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK64-NEXT:    tail call i64 @llvm.spv.num.workgroups.i64(i32 0)
 // CHECK32-NEXT:    tail call i32 @llvm.spv.num.workgroups.i32(i32 0)
 //
-unsigned int test_num_workgroups() {
+[[clang::sycl_external]] unsigned int test_num_workgroups() {
     return __builtin_spirv_num_workgroups(0);
 }
 
-// CHECK: @test_workgroup_size(
+// CHECK: @{{.*}}test_workgroup_size{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK64-NEXT:    tail call i64 @llvm.spv.workgroup.size.i64(i32 0)
 // CHECK32-NEXT:    tail call i32 @llvm.spv.workgroup.size.i32(i32 0)
 //
-unsigned int test_workgroup_size() {
+[[clang::sycl_external]] unsigned int test_workgroup_size() {
     return __builtin_spirv_workgroup_size(0);
 }
 
-// CHECK: @test_workgroup_id(
+// CHECK: @{{.*}}test_workgroup_id{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK64-NEXT:    tail call i64 @llvm.spv.group.id.i64(i32 0)
 // CHECK32-NEXT:    tail call i32 @llvm.spv.group.id.i32(i32 0)
 //
-unsigned int test_workgroup_id() {
+[[clang::sycl_external]] unsigned int test_workgroup_id() {
     return __builtin_spirv_workgroup_id(0);
 }
 
-// CHECK: @test_local_invocation_id(
+// CHECK: @{{.*}}test_local_invocation_id{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK64-NEXT:    tail call i64 @llvm.spv.thread.id.in.group.i64(i32 0)
 // CHECK32-NEXT:    tail call i32 @llvm.spv.thread.id.in.group.i32(i32 0)
 //
-unsigned int test_local_invocation_id() {
+[[clang::sycl_external]] unsigned int test_local_invocation_id() {
     return __builtin_spirv_local_invocation_id(0);
 }
 
-// CHECK: @test_global_invocation_id(
+// CHECK: @{{.*}}test_global_invocation_id{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK64-NEXT:    tail call i64 @llvm.spv.thread.id.i64(i32 0)
 // CHECK32-NEXT:    tail call i32 @llvm.spv.thread.id.i32(i32 0)
 //
-unsigned int test_global_invocation_id() {
+[[clang::sycl_external]] unsigned int test_global_invocation_id() {
     return __builtin_spirv_global_invocation_id(0);
 }
 
-// CHECK: @test_global_size(
+// CHECK: @{{.*}}test_global_size{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK64-NEXT:    tail call i64 @llvm.spv.global.size.i64(i32 0)
 // CHECK32-NEXT:    tail call i32 @llvm.spv.global.size.i32(i32 0)
 //
-unsigned int test_global_size() {
+[[clang::sycl_external]] unsigned int test_global_size() {
     return __builtin_spirv_global_size(0);
 }
 
-// CHECK: @test_global_offset(
+// CHECK: @{{.*}}test_global_offset{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK64-NEXT:    tail call i64 @llvm.spv.global.offset.i64(i32 0)
 // CHECK32-NEXT:    tail call i32 @llvm.spv.global.offset.i32(i32 0)
 //
-unsigned int test_global_offset() {
+[[clang::sycl_external]] unsigned int test_global_offset() {
     return __builtin_spirv_global_offset(0);
 }
 
-// CHECK: @test_subgroup_size(
+// CHECK: @{{.*}}test_subgroup_size{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    tail call i32 @llvm.spv.subgroup.size()
 //
-unsigned int test_subgroup_size() {
+[[clang::sycl_external]] unsigned int test_subgroup_size() {
     return __builtin_spirv_subgroup_size();
 }
 
-// CHECK: @test_subgroup_max_size(
+// CHECK: @{{.*}}test_subgroup_max_size{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    tail call i32 @llvm.spv.subgroup.max.size()
 //
-unsigned int test_subgroup_max_size() {
+[[clang::sycl_external]] unsigned int test_subgroup_max_size() {
     return __builtin_spirv_subgroup_max_size();
 }
 
-// CHECK: @test_num_subgroups(
+// CHECK: @{{.*}}test_num_subgroups{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    tail call i32 @llvm.spv.num.subgroups()
 //
-unsigned int test_num_subgroups() {
+[[clang::sycl_external]] unsigned int test_num_subgroups() {
     return __builtin_spirv_num_subgroups();
 }
 
-// CHECK: @test_subgroup_id(
+// CHECK: @{{.*}}test_subgroup_id{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    tail call i32 @llvm.spv.subgroup.id()
 //
-unsigned int test_subgroup_id() {
+[[clang::sycl_external]] unsigned int test_subgroup_id() {
     return __builtin_spirv_subgroup_id();
 }
 
-// CHECK: @test_subgroup_local_invocation_id(
+// CHECK: @{{.*}}test_subgroup_local_invocation_id{{.*}}(
 // CHECK-NEXT:  [[ENTRY:.*:]]
 // CHECK-NEXT:    tail call i32 @llvm.spv.subgroup.local.invocation.id()
 //
-unsigned int test_subgroup_local_invocation_id() {
+[[clang::sycl_external]] unsigned int test_subgroup_local_invocation_id() {
     return __builtin_spirv_subgroup_local_invocation_id();
 }
diff --git a/clang/test/CodeGenSYCL/address-space-conversions.cpp b/clang/test/CodeGenSYCL/address-space-conversions.cpp
index ee3183b74e038..fa7acb0d99433 100644
--- a/clang/test/CodeGenSYCL/address-space-conversions.cpp
+++ b/clang/test/CodeGenSYCL/address-space-conversions.cpp
@@ -1,143 +1,143 @@
 // RUN: %clang_cc1 -triple spir64 -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
 void bar(int &Data) {}
-// CHECK: define{{.*}} spir_func void @[[RAW_REF:[a-zA-Z0-9_]+]](ptr addrspace(4) noundef align 4 dereferenceable(4) %
+// CHECK-DAG: define{{.*}} spir_func void @[[RAW_REF:[a-zA-Z0-9_]+]](ptr addrspace(4) noundef align 4 dereferenceable(4) %
 void bar2(int &Data) {}
-// CHECK: define{{.*}} spir_func void @[[RAW_REF2:[a-zA-Z0-9_]+]](ptr addrspace(4) noundef align 4 dereferenceable(4) %
+// CHECK-DAG: define{{.*}} spir_func void @[[RAW_REF2:[a-zA-Z0-9_]+]](ptr addrspace(4) noundef align 4 dereferenceable(4) %
 void bar(__attribute__((opencl_local)) int &Data) {}
-// CHECK: define{{.*}} spir_func void [[LOC_REF:@[a-zA-Z0-9_]+]](ptr addrspace(3) noundef align 4 dereferenceable(4) %
+// CHECK-DAG: define{{.*}} spir_func void [[LOC_REF:@[a-zA-Z0-9_]+]](ptr addrspace(3) noundef align 4 dereferenceable(4) %
 void foo(int *Data) {}
-// CHECK: define{{.*}} spir_func void @[[RAW_PTR:[a-zA-Z0-9_]+]](ptr addrspace(4) noundef %
+// CHECK-DAG: define{{.*}} spir_func void @[[RAW_PTR:[a-zA-Z0-9_]+]](ptr addrspace(4) noundef %
 void foo2(int *Data) {}
-// CHECK: define{{.*}} spir_func void @[[RAW_PTR2:[a-zA-Z0-9_]+]](ptr addrspace(4) noundef %
+// CHECK-DAG: define{{.*}} spir_func void @[[RAW_PTR2:[a-zA-Z0-9_]+]](ptr addrspace(4) noundef %
 void foo(__attribute__((opencl_local)) int *Data) {}
-// CHECK: define{{.*}} spir_func void [[LOC_PTR:@[a-zA-Z0-9_]+]](ptr addrspace(3) noundef %
+// CHECK-DAG: define{{.*}} spir_func void [[LOC_PTR:@[a-zA-Z0-9_]+]](ptr addrspace(3) noundef %
 
 template <typename T>
 void tmpl(T t) {}
 // See Check Lines below.
 
-void usages() {
+[[clang::sycl_external]] void usages() {
   int *NoAS;
-  // CHECK: [[NoAS:%[a-zA-Z0-9]+]] = alloca ptr addrspace(4)
+  // CHECK-DAG: [[NoAS:%[a-zA-Z0-9]+]] = alloca ptr addrspace(4)
   __attribute__((opencl_global)) int *GLOB;
-  // CHECK: [[GLOB:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1)
+  // CHECK-DAG: [[GLOB:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1)
   __attribute__((opencl_local)) int *LOC;
-  // CHECK: [[LOC:%[a-zA-Z0-9]+]] = alloca ptr addrspace(3)
+  // CHECK-DAG: [[LOC:%[a-zA-Z0-9]+]] = alloca ptr addrspace(3)
   __attribute__((opencl_private)) int *PRIV;
-  // CHECK: [[PRIV:%[a-zA-Z0-9]+]] = alloca ptr
+  // CHECK-DAG: [[PRIV:%[a-zA-Z0-9]+]] = alloca ptr
   __attribute__((opencl_global_device)) int *GLOBDEVICE;
-  // CHECK: [[GLOB_DEVICE:%[a-zA-Z0-9]+]] = alloca ptr addrspace(5)
+  // CHECK-DAG: [[GLOB_DEVICE:%[a-zA-Z0-9]+]] = alloca ptr addrspace(5)
   __attribute__((opencl_global_host)) int *GLOBHOST;
-  // CHECK: [[GLOB_HOST:%[a-zA-Z0-9]+]] = alloca ptr addrspace(6)
+  // CHECK-DAG: [[GLOB_HOST:%[a-zA-Z0-9]+]] = alloca ptr addrspace(6)
 
-  // CHECK: [[NoAS]].ascast = addrspacecast ptr [[NoAS]] to ptr addrspace(4)
-  // CHECK: [[GLOB]].ascast = addrspacecast ptr [[GLOB]] to ptr addrspace(4)
-  // CHECK: [[LOC]].ascast = addrspacecast ptr [[LOC]] to ptr addrspace(4)
-  // CHECK: [[PRIV]].ascast = addrspacecast ptr [[PRIV]] to ptr addrspace(4)
+  // CHECK-DAG: [[NoAS]].ascast = addrspacecast ptr [[NoAS]] to ptr addrspace(4)
+  // CHECK-DAG: [[GLOB]].ascast = addrspacecast ptr [[GLOB]] to ptr addrspace(4)
+  // CHECK-DAG: [[LOC]].ascast = addrspacecast ptr [[LOC]] to ptr addrspace(4)
+  // CHECK-DAG: [[PRIV]].ascast = addrspacecast ptr [[PRIV]] to ptr addrspace(4)
   LOC = nullptr;
-  // CHECK: store ptr addrspace(3) null, ptr addrspace(4) [[LOC]].ascast, align 8
+  // CHECK-DAG: store ptr addrspace(3) null, ptr addrspace(4) [[LOC]].ascast, align 8
   GLOB = nullptr;
-  // CHECK: store ptr addrspace(1) null, ptr addrspace(4) [[GLOB]].ascast, align 8
+  // CHECK-DAG: store ptr addrspace(1) null, ptr addrspace(4) [[GLOB]].ascast, align 8
 
   // Explicit conversions
   // From named address spaces to default address space
-  // CHECK: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
-  // CHECK: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr addrspace(4)
-  // CHECK: store ptr addrspace(4) [[GLOB_CAST]], ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
+  // CHECK-DAG: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr addrspace(4)
+  // CHECK-DAG: store ptr addrspace(4) [[GLOB_CAST]], ptr addrspace(4) [[NoAS]].ascast
   NoAS = (int *)GLOB;
-  // CHECK: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
-  // CHECK: [[LOC_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD]] to ptr addrspace(4)
-  // CHECK: store ptr addrspace(4) [[LOC_CAST]], ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
+  // CHECK-DAG: [[LOC_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD]] to ptr addrspace(4)
+  // CHECK-DAG: store ptr addrspace(4) [[LOC_CAST]], ptr addrspace(4) [[NoAS]].ascast
   NoAS = (int *)LOC;
-  // CHECK: [[PRIV_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr addrspace(4) [[PRIV]].ascast
-  // CHECK: [[PRIV_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[PRIV_LOAD]] to ptr addrspace(4)
-  // CHECK: store ptr addrspace(4) [[PRIV_CAST]], ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: [[PRIV_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr addrspace(4) [[PRIV]].ascast
+  // CHECK-DAG: [[PRIV_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[PRIV_LOAD]] to ptr addrspace(4)
+  // CHECK-DAG: store ptr addrspace(4) [[PRIV_CAST]], ptr addrspace(4) [[NoAS]].ascast
   NoAS = (int *)PRIV;
   // From default address space to named address space
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
-  // CHECK: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(4) [[NoAS_LOAD]] to ptr addrspace(1)
-  // CHECK: store ptr addrspace(1) [[NoAS_CAST]], ptr addrspace(4) [[GLOB]].ascast
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(4) [[NoAS_LOAD]] to ptr addrspace(1)
+  // CHECK-DAG: store ptr addrspace(1) [[NoAS_CAST]], ptr addrspace(4) [[GLOB]].ascast
   GLOB = (__attribute__((opencl_global)) int *)NoAS;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
-  // CHECK: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(4) [[NoAS_LOAD]] to ptr addrspace(3)
-  // CHECK: store ptr addrspace(3) [[NoAS_CAST]], ptr addrspace(4) [[LOC]].ascast
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(4) [[NoAS_LOAD]] to ptr addrspace(3)
+  // CHECK-DAG: store ptr addrspace(3) [[NoAS_CAST]], ptr addrspace(4) [[LOC]].ascast
   LOC = (__attribute__((opencl_local)) int *)NoAS;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
-  // CHECK: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(4) [[NoAS_LOAD]] to ptr
-  // CHECK: store ptr [[NoAS_CAST]], ptr addrspace(4) [[PRIV]].ascast
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(4) [[NoAS_LOAD]] to ptr
+  // CHECK-DAG: store ptr [[NoAS_CAST]], ptr addrspace(4) [[PRIV]].ascast
   PRIV = (__attribute__((opencl_private)) int *)NoAS;
   // From opencl_global_[host/device] address spaces to opencl_global
-  // CHECK: [[GLOBDEVICE_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(5), ptr addrspace(4) [[GLOB_DEVICE]].ascast
-  // CHECK: [[GLOBDEVICE_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(5) [[GLOBDEVICE_LOAD]] to ptr addrspace(1)
-  // CHECK: store ptr addrspace(1) [[GLOBDEVICE_CAST]], ptr addrspace(4) [[GLOB]].ascast
+  // CHECK-DAG: [[GLOBDEVICE_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(5), ptr addrspace(4) [[GLOB_DEVICE]].ascast
+  // CHECK-DAG: [[GLOBDEVICE_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(5) [[GLOBDEVICE_LOAD]] to ptr addrspace(1)
+  // CHECK-DAG: store ptr addrspace(1) [[GLOBDEVICE_CAST]], ptr addrspace(4) [[GLOB]].ascast
   GLOB = (__attribute__((opencl_global)) int *)GLOBDEVICE;
-  // CHECK: [[GLOBHOST_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(6), ptr addrspace(4) [[GLOB_HOST]].ascast
-  // CHECK: [[GLOBHOST_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(6) [[GLOBHOST_LOAD]] to ptr addrspace(1)
-  // CHECK: store ptr addrspace(1) [[GLOBHOST_CAST]], ptr addrspace(4) [[GLOB]].ascast
+  // CHECK-DAG: [[GLOBHOST_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(6), ptr addrspace(4) [[GLOB_HOST]].ascast
+  // CHECK-DAG: [[GLOBHOST_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(6) [[GLOBHOST_LOAD]] to ptr addrspace(1)
+  // CHECK-DAG: store ptr addrspace(1) [[GLOBHOST_CAST]], ptr addrspace(4) [[GLOB]].ascast
   GLOB = (__attribute__((opencl_global)) int *)GLOBHOST;
 
   bar(*GLOB);
-  // CHECK: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
-  // CHECK: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr addrspace(4)
-  // CHECK: call spir_func void @[[RAW_REF]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[GLOB_CAST]])
+  // CHECK-DAG: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
+  // CHECK-DAG: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr addrspace(4)
+  // CHECK-DAG: call spir_func void @[[RAW_REF]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[GLOB_CAST]])
   bar2(*GLOB);
-  // CHECK: [[GLOB_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
-  // CHECK: [[GLOB_CAST2:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD2]] to ptr addrspace(4)
-  // CHECK: call spir_func void @[[RAW_REF2]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[GLOB_CAST2]])
+  // CHECK-DAG: [[GLOB_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
+  // CHECK-DAG: [[GLOB_CAST2:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD2]] to ptr addrspace(4)
+  // CHECK-DAG: call spir_func void @[[RAW_REF2]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[GLOB_CAST2]])
 
   bar(*LOC);
-  // CHECK: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
-  // CHECK: call spir_func void [[LOC_REF]](ptr addrspace(3) noundef align 4 dereferenceable(4) [[LOC_LOAD]])
+  // CHECK-DAG: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
+  // CHECK-DAG: call spir_func void [[LOC_REF]](ptr addrspace(3) noundef align 4 dereferenceable(4) [[LOC_LOAD]])
   bar2(*LOC);
-  // CHECK: [[LOC_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
-  // CHECK: [[LOC_CAST2:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD2]] to ptr addrspace(4)
-  // CHECK: call spir_func void @[[RAW_REF2]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[LOC_CAST2]])
+  // CHECK-DAG: [[LOC_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
+  // CHECK-DAG: [[LOC_CAST2:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD2]] to ptr addrspace(4)
+  // CHECK-DAG: call spir_func void @[[RAW_REF2]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[LOC_CAST2]])
 
   bar(*NoAS);
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
-  // CHECK: call spir_func void @[[RAW_REF]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[NoAS_LOAD]])
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: call spir_func void @[[RAW_REF]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[NoAS_LOAD]])
   bar2(*NoAS);
-  // CHECK: [[NoAS_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
-  // CHECK: call spir_func void @[[RAW_REF2]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[NoAS_LOAD2]])
+  // CHECK-DAG: [[NoAS_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: call spir_func void @[[RAW_REF2]](ptr addrspace(4) noundef align 4 dereferenceable(4) [[NoAS_LOAD2]])
 
   foo(GLOB);
-  // CHECK: [[GLOB_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
-  // CHECK: [[GLOB_CAST3:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD3]] to ptr addrspace(4)
-  // CHECK: call spir_func void @[[RAW_PTR]](ptr addrspace(4) noundef [[GLOB_CAST3]])
+  // CHECK-DAG: [[GLOB_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
+  // CHECK-DAG: [[GLOB_CAST3:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD3]] to ptr addrspace(4)
+  // CHECK-DAG: call spir_func void @[[RAW_PTR]](ptr addrspace(4) noundef [[GLOB_CAST3]])
   foo2(GLOB);
-  // CHECK: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
-  // CHECK: [[GLOB_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD4]] to ptr addrspace(4)
-  // CHECK: call spir_func void @[[RAW_PTR2]](ptr addrspace(4) noundef [[GLOB_CAST4]])
+  // CHECK-DAG: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
+  // CHECK-DAG: [[GLOB_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD4]] to ptr addrspace(4)
+  // CHECK-DAG: call spir_func void @[[RAW_PTR2]](ptr addrspace(4) noundef [[GLOB_CAST4]])
   foo(LOC);
-  // CHECK: [[LOC_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
-  // CHECK: call spir_func void [[LOC_PTR]](ptr addrspace(3) noundef [[LOC_LOAD3]])
+  // CHECK-DAG: [[LOC_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
+  // CHECK-DAG: call spir_func void [[LOC_PTR]](ptr addrspace(3) noundef [[LOC_LOAD3]])
   foo2(LOC);
-  // CHECK: [[LOC_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
-  // CHECK: [[LOC_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD4]] to ptr addrspace(4)
-  // CHECK: call spir_func void @[[RAW_PTR2]](ptr addrspace(4) noundef [[LOC_CAST4]])
+  // CHECK-DAG: [[LOC_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
+  // CHECK-DAG: [[LOC_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD4]] to ptr addrspace(4)
+  // CHECK-DAG: call spir_func void @[[RAW_PTR2]](ptr addrspace(4) noundef [[LOC_CAST4]])
   foo(NoAS);
-  // CHECK: [[NoAS_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
-  // CHECK: call spir_func void @[[RAW_PTR]](ptr addrspace(4) noundef [[NoAS_LOAD3]])
+  // CHECK-DAG: [[NoAS_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: call spir_func void @[[RAW_PTR]](ptr addrspace(4) noundef [[NoAS_LOAD3]])
   foo2(NoAS);
-  // CHECK: [[NoAS_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
-  // CHECK: call spir_func void @[[RAW_PTR2]](ptr addrspace(4) noundef [[NoAS_LOAD4]])
+  // CHECK-DAG: [[NoAS_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: call spir_func void @[[RAW_PTR2]](ptr addrspace(4) noundef [[NoAS_LOAD4]])
 
   // Ensure that we still get 3 different template instantiations.
   tmpl(GLOB);
-  // CHECK: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
-  // CHECK: call spir_func void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef [[GLOB_LOAD4]])
+  // CHECK-DAG: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr addrspace(4) [[GLOB]].ascast
+  // CHECK-DAG: call spir_func void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef [[GLOB_LOAD4]])
   tmpl(LOC);
-  // CHECK: [[LOC_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
-  // CHECK: call spir_func void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef [[LOC_LOAD5]])
+  // CHECK-DAG: [[LOC_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr addrspace(4) [[LOC]].ascast
+  // CHECK-DAG: call spir_func void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef [[LOC_LOAD5]])
   tmpl(PRIV);
-  // CHECK: [[PRIV_LOAD5:%[a-zA-Z0-9]+]] = load ptr, ptr addrspace(4) [[PRIV]].ascast
-  // CHECK: call spir_func void @_Z4tmplIPU3AS0iEvT_(ptr noundef [[PRIV_LOAD5]])
+  // CHECK-DAG: [[PRIV_LOAD5:%[a-zA-Z0-9]+]] = load ptr, ptr addrspace(4) [[PRIV]].ascast
+  // CHECK-DAG: call spir_func void @_Z4tmplIPU3AS0iEvT_(ptr noundef [[PRIV_LOAD5]])
   tmpl(NoAS);
-  // CHECK: [[NoAS_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
-  // CHECK: call spir_func void @_Z4tmplIPiEvT_(ptr addrspace(4) noundef [[NoAS_LOAD5]])
+  // CHECK-DAG: [[NoAS_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(4), ptr addrspace(4) [[NoAS]].ascast
+  // CHECK-DAG: call spir_func void @_Z4tmplIPiEvT_(ptr addrspace(4) noundef [[NoAS_LOAD5]])
 }
 
-// CHECK: define linkonce_odr spir_func void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef %
-// CHECK: define linkonce_odr spir_func void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef %
-// CHECK: define linkonce_odr spir_func void @_Z4tmplIPU3AS0iEvT_(ptr noundef %
-// CHECK: define linkonce_odr spir_func void @_Z4tmplIPiEvT_(ptr addrspace(4) noundef %
+// CHECK-DAG: define linkonce_odr spir_func void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef %
+// CHECK-DAG: define linkonce_odr spir_func void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef %
+// CHECK-DAG: define linkonce_odr spir_func void @_Z4tmplIPU3AS0iEvT_(ptr noundef %
+// CHECK-DAG: define linkonce_odr spir_func void @_Z4tmplIPiEvT_(ptr addrspace(4) noundef %
diff --git a/clang/test/CodeGenSYCL/address-space-deduction.cpp b/clang/test/CodeGenSYCL/address-space-deduction.cpp
index 5910ec3bfc305..0fb5c41d630cc 100644
--- a/clang/test/CodeGenSYCL/address-space-deduction.cpp
+++ b/clang/test/CodeGenSYCL/address-space-deduction.cpp
@@ -85,7 +85,7 @@
 // CHECK-NEXT:    store ptr addrspace(4) addrspacecast (ptr addrspace(1) @.str.1 to ptr addrspace(4)), ptr addrspace(4) [[SELECT_STR_TRIVIAL2_ASCAST]], align 8
 // CHECK-NEXT:    ret void
 //
-void test() {
+[[clang::sycl_external]] void test() {
   static const int foo = 0x42;
 
 
diff --git a/clang/test/CodeGenSYCL/address-space-mangling.cpp b/clang/test/CodeGenSYCL/address-space-mangling.cpp
index 868bf8ccbdcf8..ecc2d4b43a159 100644
--- a/clang/test/CodeGenSYCL/address-space-mangling.cpp
+++ b/clang/test/CodeGenSYCL/address-space-mangling.cpp
@@ -18,7 +18,7 @@ void foo(int *);
 // X86: declare void @_Z3fooPU9SYprivatei(ptr noundef) #1
 // X86: declare void @_Z3fooPi(ptr noundef) #1
 
-void test() {
+[[clang::sycl_external]] void test() {
   __attribute__((opencl_global)) int *glob;
   __attribute__((opencl_local)) int *loc;
   __attribute__((opencl_private)) int *priv;
diff --git a/clang/test/CodeGenSYCL/amd-address-space-conversions.cpp b/clang/test/CodeGenSYCL/amd-address-space-conversions.cpp
index d316f22096d3d..17a98195318ad 100644
--- a/clang/test/CodeGenSYCL/amd-address-space-conversions.cpp
+++ b/clang/test/CodeGenSYCL/amd-address-space-conversions.cpp
@@ -1,128 +1,128 @@
 // RUN: %clang_cc1 -triple amdgcn-amd-amdhsa -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
 void bar(int &Data) {}
-// CHECK: define dso_local void @[[RAW_REF:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
+// CHECK-DAG: define {{.*}} void @[[RAW_REF:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
 void bar2(int &Data) {}
-// CHECK: define dso_local void @[[RAW_REF2:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
+// CHECK-DAG: define {{.*}} void @[[RAW_REF2:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
 void bar(__attribute__((opencl_local)) int &Data) {}
-// CHECK: define dso_local void @[[LOCAL_REF:[a-zA-Z0-9_]+]](ptr addrspace(3) noundef align 4 dereferenceable(4) %
+// CHECK-DAG: define {{.*}} void @[[LOCAL_REF:[a-zA-Z0-9_]+]](ptr addrspace(3) noundef align 4 dereferenceable(4) %
 void foo(int *Data) {}
-// CHECK: define dso_local void @[[RAW_PTR:[a-zA-Z0-9_]+]](ptr noundef %
+// CHECK-DAG: define {{.*}} void @[[RAW_PTR:[a-zA-Z0-9_]+]](ptr noundef %
 void foo2(int *Data) {}
-// CHECK: define dso_local void @[[RAW_PTR2:[a-zA-Z0-9_]+]](ptr noundef %
+// CHECK-DAG: define {{.*}} void @[[RAW_PTR2:[a-zA-Z0-9_]+]](ptr noundef %
 void foo(__attribute__((opencl_local)) int *Data) {}
-// CHECK: define dso_local void @[[LOC_PTR:[a-zA-Z0-9_]+]](ptr addrspace(3) noundef %
+// CHECK-DAG: define {{.*}} void @[[LOC_PTR:[a-zA-Z0-9_]+]](ptr addrspace(3) noundef %
 
 template <typename T>
-void tmpl(T t);
+void tmpl(T t) {}
 // See Check Lines below.
 
-void usages() {
+[[clang::sycl_external]] void usages() {
   int *NoAS;
-  // CHECK: [[NoAS:%[a-zA-Z0-9]+]] = alloca ptr, align 8, addrspace(5)
+  // CHECK-DAG: [[NoAS:%[a-zA-Z0-9]+]] = alloca ptr, align 8, addrspace(5)
   __attribute__((opencl_global)) int *GLOB;
-  // CHECK: [[GLOB:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8, addrspace(5)
+  // CHECK-DAG: [[GLOB:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8, addrspace(5)
   __attribute__((opencl_local)) int *LOC;
-  // CHECK: [[LOC:%[a-zA-Z0-9]+]] = alloca ptr addrspace(3), align 4, addrspace(5)
+  // CHECK-DAG: [[LOC:%[a-zA-Z0-9]+]] = alloca ptr addrspace(3), align 4, addrspace(5)
   __attribute__((opencl_private)) int *PRIV;
-  // CHECK: [[PRIV:%[a-zA-Z0-9]+]] = alloca ptr addrspace(5), align 4, addrspace(5)
+  // CHECK-DAG: [[PRIV:%[a-zA-Z0-9]+]] = alloca ptr addrspace(5), align 4, addrspace(5)
   __attribute__((opencl_global_device)) int *GLOBDEVICE;
-  // CHECK: [[GLOB_DEVICE:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8, addrspace(5)
+  // CHECK-DAG: [[GLOB_DEVICE:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8, addrspace(5)
   __attribute__((opencl_global_host)) int *GLOBHOST;
-  // CHECK: [[GLOB_HOST:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8, addrspace(5)
+  // CHECK-DAG: [[GLOB_HOST:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8, addrspace(5)
   LOC = nullptr;
-  // CHECK: store ptr addrspace(3) addrspacecast (ptr null to ptr addrspace(3)), ptr [[LOC]].ascast, align 4
+  // CHECK-DAG: store ptr addrspace(3) addrspacecast (ptr null to ptr addrspace(3)), ptr [[LOC]].ascast, align 4
   GLOB = nullptr;
-  // CHECK: store ptr addrspace(1) null, ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: store ptr addrspace(1) null, ptr [[GLOB]].ascast, align 8
   NoAS = (int *)GLOB;
-  // CHECK: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
-  // CHECK: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
-  // CHECK: store ptr [[GLOB_CAST]], ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
+  // CHECK-DAG: store ptr [[GLOB_CAST]], ptr [[NoAS]].ascast, align 8
   NoAS = (int *)LOC;
-  // CHECK: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
-  // CHECK: [[LOC_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD]] to ptr
-  // CHECK: store ptr [[LOC_CAST]], ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
+  // CHECK-DAG: [[LOC_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD]] to ptr
+  // CHECK-DAG: store ptr [[LOC_CAST]], ptr [[NoAS]].ascast, align 8
   NoAS = (int *)PRIV;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(5), ptr [[PRIV]].ascast, align 4
-  // CHECK: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(5) [[NoAS_LOAD]] to ptr
-  // CHECK: store ptr %5, ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(5), ptr [[PRIV]].ascast, align 4
+  // CHECK-DAG: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(5) [[NoAS_LOAD]] to ptr
+  // CHECK-DAG: store ptr %5, ptr [[NoAS]].ascast, align 8
   GLOB = (__attribute__((opencl_global)) int *)NoAS;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
-  // CHECK: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr %6 to ptr addrspace(1)
-  // CHECK: store ptr addrspace(1) %7, ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr %6 to ptr addrspace(1)
+  // CHECK-DAG: store ptr addrspace(1) %7, ptr [[GLOB]].ascast, align 8
   LOC = (__attribute__((opencl_local)) int *)NoAS;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
-  // CHECK: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[NoAS_LOAD]] to ptr addrspace(3)
-  // CHECK: store ptr addrspace(3) %9, ptr [[LOC]].ascast, align 4
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[NoAS_LOAD]] to ptr addrspace(3)
+  // CHECK-DAG: store ptr addrspace(3) %9, ptr [[LOC]].ascast, align 4
   PRIV = (__attribute__((opencl_private)) int *)NoAS;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
-  // CHECK: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[NoAS_LOAD]] to ptr addrspace(5)
-  // CHECK: store ptr addrspace(5) [[NoAS_CAST]], ptr [[PRIV]].ascast, align 4
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[NoAS_LOAD]] to ptr addrspace(5)
+  // CHECK-DAG: store ptr addrspace(5) [[NoAS_CAST]], ptr [[PRIV]].ascast, align 4
   GLOB = (__attribute__((opencl_global)) int *)GLOBDEVICE;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]]DEVICE.ascast, align 8
-  // CHECK: store ptr addrspace(1) [[NoAS_LOAD]], ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]]DEVICE.ascast, align 8
+  // CHECK-DAG: store ptr addrspace(1) [[NoAS_LOAD]], ptr [[GLOB]].ascast, align 8
   GLOB = (__attribute__((opencl_global)) int *)GLOBHOST;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]]HOST.ascast, align 8
-  // CHECK: tore ptr addrspace(1) [[NoAS_LOAD]], ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]]HOST.ascast, align 8
+  // CHECK-DAG: tore ptr addrspace(1) [[NoAS_LOAD]], ptr [[GLOB]].ascast, align 8
   bar(*GLOB);
-  // CHECK: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
-  // CHECK: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
-  // CHECK: call void @[[RAW_REF]](ptr noundef nonnull align 4 dereferenceable(4) [[GLOB_CAST]])
+  // CHECK-DAG: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
+  // CHECK-DAG: call void @[[RAW_REF]](ptr noundef nonnull align 4 dereferenceable(4) [[GLOB_CAST]])
   bar2(*GLOB);
-  // CHECK: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
-  // CHECK: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
-  // CHECK: call void @[[RAW_REF2]](ptr noundef nonnull align 4 dereferenceable(4) [[GLOB_CAST]])
+  // CHECK-DAG: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
+  // CHECK-DAG: call void @[[RAW_REF2]](ptr noundef nonnull align 4 dereferenceable(4) [[GLOB_CAST]])
   bar(*LOC);
-  // CHECK: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
-  // CHECK: call void @_Z3barRU3AS3i(ptr addrspace(3) noundef align 4 dereferenceable(4) [[LOC_LOAD]])
+  // CHECK-DAG: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
+  // CHECK-DAG: call void @_Z3barRU3AS3i(ptr addrspace(3) noundef align 4 dereferenceable(4) [[LOC_LOAD]])
   bar2(*LOC);
-  // CHECK: [[LOC_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
-  // CHECK: [[LOC_CAST2:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD2]] to ptr
-  // CHECK: call void @_Z4bar2Ri(ptr noundef nonnull align 4 dereferenceable(4) [[LOC_CAST2]])
+  // CHECK-DAG: [[LOC_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
+  // CHECK-DAG: [[LOC_CAST2:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD2]] to ptr
+  // CHECK-DAG: call void @_Z4bar2Ri(ptr noundef nonnull align 4 dereferenceable(4) [[LOC_CAST2]])
   bar(*NoAS);
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
-  // CHECK: call void @_Z3barRi(ptr noundef nonnull align 4 dereferenceable(4) [[NoAS_LOAD]])
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: call void @_Z3barRi(ptr noundef nonnull align 4 dereferenceable(4) [[NoAS_LOAD]])
   bar2(*NoAS);
-  // CHECK: [[NoAS_LOAD2:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
-  // CHECK: call void @_Z4bar2Ri(ptr noundef nonnull align 4 dereferenceable(4) [[NoAS_LOAD2]])
+  // CHECK-DAG: [[NoAS_LOAD2:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: call void @_Z4bar2Ri(ptr noundef nonnull align 4 dereferenceable(4) [[NoAS_LOAD2]])
   foo(GLOB);
-  // CHECK: [[GLOB_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
-  // CHECK: [[GLOB_CAST3:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD3]] to ptr
-  // CHECK: call void @[[RAW_PTR]](ptr noundef [[GLOB_CAST3]])
+  // CHECK-DAG: [[GLOB_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: [[GLOB_CAST3:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD3]] to ptr
+  // CHECK-DAG: call void @[[RAW_PTR]](ptr noundef [[GLOB_CAST3]])
    foo2(GLOB);
-  // CHECK: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
-  // CHECK: [[GLOB_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD4]] to ptr
-  // CHECK: call void @[[RAW_PTR2]](ptr noundef [[GLOB_CAST4]])
+  // CHECK-DAG: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: [[GLOB_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD4]] to ptr
+  // CHECK-DAG: call void @[[RAW_PTR2]](ptr noundef [[GLOB_CAST4]])
   foo(LOC);
-  // CHECK: [[LOC_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
-  // CHECK: call void @[[LOC_PTR]](ptr addrspace(3) noundef [[LOC_LOAD3]])
+  // CHECK-DAG: [[LOC_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
+  // CHECK-DAG: call void @[[LOC_PTR]](ptr addrspace(3) noundef [[LOC_LOAD3]])
   foo2(LOC);
-  // CHECK: [[LOC_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
-  // CHECK: [[LOC_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD4]] to ptr
-  // CHECK: call void @[[RAW_PTR2]](ptr noundef [[LOC_CAST4]])
+  // CHECK-DAG: [[LOC_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
+  // CHECK-DAG: [[LOC_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD4]] to ptr
+  // CHECK-DAG: call void @[[RAW_PTR2]](ptr noundef [[LOC_CAST4]])
   foo(NoAS);
-  // CHECK: [[NoAS_LOAD3:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
-  // CHECK: call void @[[RAW_PTR]](ptr noundef [[NoAS_LOAD3]])
+  // CHECK-DAG: [[NoAS_LOAD3:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: call void @[[RAW_PTR]](ptr noundef [[NoAS_LOAD3]])
   foo2(NoAS);
-  // CHECK: [[NoAS_LOAD4:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
-  // CHECK: call void @[[RAW_PTR2]](ptr noundef [[NoAS_LOAD4]])
+  // CHECK-DAG: [[NoAS_LOAD4:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: call void @[[RAW_PTR2]](ptr noundef [[NoAS_LOAD4]])
 
   // Ensure that we still get 3 different template instantiations.
   tmpl(GLOB);
-  // CHECK: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
-  // CHECK: call void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef [[GLOB_LOAD4]])
+  // CHECK-DAG: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]].ascast, align 8
+  // CHECK-DAG: call void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef [[GLOB_LOAD4]])
   tmpl(LOC);
-  // CHECK: [[LOC_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
-  // CHECK: call void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef [[LOC_LOAD5]])
+  // CHECK-DAG: [[LOC_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]].ascast, align 4
+  // CHECK-DAG: call void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef [[LOC_LOAD5]])
   tmpl(PRIV);
-  // CHECK: [[PRIV_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(5), ptr [[PRIV]].ascast, align 4
-  // CHECK: call void @_Z4tmplIPU3AS5iEvT_(ptr addrspace(5) noundef [[PRIV_LOAD5]])
+  // CHECK-DAG: [[PRIV_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(5), ptr [[PRIV]].ascast, align 4
+  // CHECK-DAG: call void @_Z4tmplIPU3AS5iEvT_(ptr addrspace(5) noundef [[PRIV_LOAD5]])
   tmpl(NoAS);
-  // CHECK: [[NoAS_LOAD5:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
-  // CHECK: call void @_Z4tmplIPiEvT_(ptr noundef [[NoAS_LOAD5]])
+  // CHECK-DAG: [[NoAS_LOAD5:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]].ascast, align 8
+  // CHECK-DAG: call void @_Z4tmplIPiEvT_(ptr noundef [[NoAS_LOAD5]])
 }
 
-// CHECK: declare void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef)
-// CHECK: declare void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef)
-// CHECK: declare void @_Z4tmplIPU3AS5iEvT_(ptr addrspace(5) noundef)
-// CHECK: declare void @_Z4tmplIPiEvT_(ptr noundef)
+// CHECK-DAG: define linkonce_odr void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef %
+// CHECK-DAG: define linkonce_odr void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef %
+// CHECK-DAG: define linkonce_odr void @_Z4tmplIPU3AS5iEvT_(ptr addrspace(5) noundef %
+// CHECK-DAG: define linkonce_odr void @_Z4tmplIPiEvT_(ptr noundef %
 
diff --git a/clang/test/CodeGenSYCL/cuda-address-space-conversions.cpp b/clang/test/CodeGenSYCL/cuda-address-space-conversions.cpp
index 1875029de0856..ffb601e62c118 100644
--- a/clang/test/CodeGenSYCL/cuda-address-space-conversions.cpp
+++ b/clang/test/CodeGenSYCL/cuda-address-space-conversions.cpp
@@ -1,122 +1,122 @@
 // RUN: %clang_cc1 -triple nvptx64-nvidia-cuda -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
 void bar(int &Data) {}
-// CHECK: define dso_local void @[[RAW_REF:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
+// CHECK-DAG: define {{.*}} void @[[RAW_REF:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
 void bar2(int &Data) {}
-// CHECK: define dso_local void @[[RAW_REF2:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
+// CHECK-DAG: define {{.*}} void @[[RAW_REF2:[a-zA-Z0-9_]+]](ptr noundef nonnull align 4 dereferenceable(4) %
 void bar(__attribute__((opencl_local)) int &Data) {}
-// CHECK: define dso_local void @[[LOCAL_REF:[a-zA-Z0-9_]+]](ptr addrspace(3) noundef align 4 dereferenceable(4) %
+// CHECK-DAG: define {{.*}} void @[[LOCAL_REF:[a-zA-Z0-9_]+]](ptr addrspace(3) noundef align 4 dereferenceable(4) %
 void foo(int *Data) {}
-// CHECK: define dso_local void @[[RAW_PTR:[a-zA-Z0-9_]+]](ptr noundef %
+// CHECK-DAG: define {{.*}} void @[[RAW_PTR:[a-zA-Z0-9_]+]](ptr noundef %
 void foo2(int *Data) {}
-// CHECK: define dso_local void @[[RAW_PTR2:[a-zA-Z0-9_]+]](ptr noundef %
+// CHECK-DAG: define {{.*}} void @[[RAW_PTR2:[a-zA-Z0-9_]+]](ptr noundef %
 void foo(__attribute__((opencl_local)) int *Data) {}
-// CHECK: define dso_local void @[[LOC_PTR:[a-zA-Z0-9_]+]](ptr addrspace(3) noundef %
+// CHECK-DAG: define {{.*}} void @[[LOC_PTR:[a-zA-Z0-9_]+]](ptr addrspace(3) noundef %
 
 template <typename T>
 void tmpl(T t);
 // See Check Lines below.
 
-void usages() {
+[[clang::sycl_external]] void usages() {
   int *NoAS;
-  // CHECK: [[NoAS:%[a-zA-Z0-9]+]] = alloca ptr, align 8
+  // CHECK-DAG: [[NoAS:%[a-zA-Z0-9]+]] = alloca ptr, align 8
   __attribute__((opencl_global)) int *GLOB;
-  // CHECK: [[GLOB:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8
+  // CHECK-DAG: [[GLOB:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8
   __attribute__((opencl_local)) int *LOC;
-  // CHECK: [[LOC:%[a-zA-Z0-9]+]] = alloca ptr addrspace(3), align 8
+  // CHECK-DAG: [[LOC:%[a-zA-Z0-9]+]] = alloca ptr addrspace(3), align 8
   __attribute__((opencl_private)) int *PRIV;
-  // CHECK: [[PRIV:%[a-zA-Z0-9]+]] = alloca ptr, align 8
+  // CHECK-DAG: [[PRIV:%[a-zA-Z0-9]+]] = alloca ptr, align 8
   __attribute__((opencl_global_device)) int *GLOBDEVICE;
-  // CHECK: [[GLOB_DEVICE:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8
+  // CHECK-DAG: [[GLOB_DEVICE:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8
   __attribute__((opencl_global_host)) int *GLOBHOST;
-  // CHECK: [[GLOB_HOST:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8
+  // CHECK-DAG: [[GLOB_HOST:%[a-zA-Z0-9]+]] = alloca ptr addrspace(1), align 8
   LOC = nullptr;
-  // CHECK: store ptr addrspace(3) addrspacecast (ptr null to ptr addrspace(3)), ptr [[LOC]], align 8
+  // CHECK-DAG: store ptr addrspace(3) addrspacecast (ptr null to ptr addrspace(3)), ptr [[LOC]], align 8
   GLOB = nullptr;
-  // CHECK: store ptr addrspace(1) null, ptr [[GLOB]], align 8
+  // CHECK-DAG: store ptr addrspace(1) null, ptr [[GLOB]], align 8
   NoAS = (int *)GLOB;
-  // CHECK: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
-  // CHECK: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
-  // CHECK: store ptr [[GLOB_CAST]], ptr [[NoAS]], align 8
+  // CHECK-DAG: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
+  // CHECK-DAG: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
+  // CHECK-DAG: store ptr [[GLOB_CAST]], ptr [[NoAS]], align 8
   NoAS = (int *)LOC;
-  // CHECK: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
-  // CHECK: [[LOC_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD]] to ptr
-  // CHECK: store ptr [[LOC_CAST]], ptr [[NoAS]], align 8
+  // CHECK-DAG: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
+  // CHECK-DAG: [[LOC_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD]] to ptr
+  // CHECK-DAG: store ptr [[LOC_CAST]], ptr [[NoAS]], align 8
   NoAS = (int *)PRIV;
-  // CHECK: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[PRIV]], align 8
-  // CHECK: store ptr [[LOC_LOAD]], ptr [[NoAS]], align 8
+  // CHECK-DAG: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[PRIV]], align 8
+  // CHECK-DAG: store ptr [[LOC_LOAD]], ptr [[NoAS]], align 8
   GLOB = (__attribute__((opencl_global)) int *)NoAS;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
-  // CHECK: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[NoAS_LOAD]] to ptr addrspace(1)
-  // CHECK: store ptr addrspace(1) [[NoAS_CAST]], ptr [[GLOB]], align 8
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
+  // CHECK-DAG: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[NoAS_LOAD]] to ptr addrspace(1)
+  // CHECK-DAG: store ptr addrspace(1) [[NoAS_CAST]], ptr [[GLOB]], align 8
   LOC = (__attribute__((opencl_local)) int *)NoAS;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
-  // CHECK: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[NoAS_LOAD]] to ptr addrspace(3)
-  // CHECK: store ptr addrspace(3) [[NoAS_CAST]], ptr [[LOC]], align 8
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
+  // CHECK-DAG: [[NoAS_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr [[NoAS_LOAD]] to ptr addrspace(3)
+  // CHECK-DAG: store ptr addrspace(3) [[NoAS_CAST]], ptr [[LOC]], align 8
   PRIV = (__attribute__((opencl_private)) int *)NoAS;
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
-  // CHECK: store ptr [[NoAS_LOAD]], ptr [[PRIV]], align 8
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
+  // CHECK-DAG: store ptr [[NoAS_LOAD]], ptr [[PRIV]], align 8
   GLOB = (__attribute__((opencl_global)) int *)GLOBDEVICE;
-  // CHECK: [[GLOBDEVICE_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB_DEVICE]], align 8
-  // CHECK: store ptr addrspace(1) [[GLOBDEVICE_LOAD]], ptr %GLOB, align 8
+  // CHECK-DAG: [[GLOBDEVICE_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB_DEVICE]], align 8
+  // CHECK-DAG: store ptr addrspace(1) [[GLOBDEVICE_LOAD]], ptr %GLOB, align 8
   GLOB = (__attribute__((opencl_global)) int *)GLOBHOST;
-  // CHECK: [[GLOB_HOST_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB_HOST]], align 8
-  // CHECK: store ptr addrspace(1) [[GLOB_HOST_LOAD]], ptr [[GLOB]], align 8
+  // CHECK-DAG: [[GLOB_HOST_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB_HOST]], align 8
+  // CHECK-DAG: store ptr addrspace(1) [[GLOB_HOST_LOAD]], ptr [[GLOB]], align 8
   bar(*GLOB);
-  // CHECK: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
-  // CHECK: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
-  // CHECK: call void @[[RAW_REF]](ptr noundef nonnull align 4 dereferenceable(4) [[GLOB_CAST]])
+  // CHECK-DAG: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
+  // CHECK-DAG: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
+  // CHECK-DAG: call void @[[RAW_REF]](ptr noundef nonnull align 4 dereferenceable(4) [[GLOB_CAST]])
   bar2(*GLOB);
-  // CHECK: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
-  // CHECK: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
-  // CHECK: call void @[[RAW_REF2]](ptr noundef nonnull align 4 dereferenceable(4) [[GLOB_CAST]])
+  // CHECK-DAG: [[GLOB_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
+  // CHECK-DAG: [[GLOB_CAST:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD]] to ptr
+  // CHECK-DAG: call void @[[RAW_REF2]](ptr noundef nonnull align 4 dereferenceable(4) [[GLOB_CAST]])
   bar(*LOC);
-  // CHECK: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
-  // CHECK: call void @[[LOCAL_REF]](ptr addrspace(3) noundef align 4 dereferenceable(4) [[LOC_LOAD]])
+  // CHECK-DAG: [[LOC_LOAD:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
+  // CHECK-DAG: call void @[[LOCAL_REF]](ptr addrspace(3) noundef align 4 dereferenceable(4) [[LOC_LOAD]])
   bar2(*LOC);
-  // CHECK: [[LOC_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
-  // CHECK: [[LOC_CAST2:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD2]] to ptr
-  // CHECK: call void @[[RAW_REF2]](ptr noundef nonnull align 4 dereferenceable(4) [[LOC_CAST2]])
+  // CHECK-DAG: [[LOC_LOAD2:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
+  // CHECK-DAG: [[LOC_CAST2:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD2]] to ptr
+  // CHECK-DAG: call void @[[RAW_REF2]](ptr noundef nonnull align 4 dereferenceable(4) [[LOC_CAST2]])
   bar(*NoAS);
-  // CHECK: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
-  // CHECK: call void @[[RAW_REF]](ptr noundef nonnull align 4 dereferenceable(4) [[NoAS_LOAD]])
+  // CHECK-DAG: [[NoAS_LOAD:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
+  // CHECK-DAG: call void @[[RAW_REF]](ptr noundef nonnull align 4 dereferenceable(4) [[NoAS_LOAD]])
   bar2(*NoAS);
-  // CHECK: [[NoAS_LOAD2:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
-  // CHECK: call void @[[RAW_REF2]](ptr noundef nonnull align 4 dereferenceable(4) [[NoAS_LOAD2]])
+  // CHECK-DAG: [[NoAS_LOAD2:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
+  // CHECK-DAG: call void @[[RAW_REF2]](ptr noundef nonnull align 4 dereferenceable(4) [[NoAS_LOAD2]])
   foo(GLOB);
-  // CHECK: [[GLOB_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
-  // CHECK: [[GLOB_CAST3:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD3]] to ptr
-  // CHECK: call void @[[RAW_PTR]](ptr noundef [[GLOB_CAST3]])
+  // CHECK-DAG: [[GLOB_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
+  // CHECK-DAG: [[GLOB_CAST3:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD3]] to ptr
+  // CHECK-DAG: call void @[[RAW_PTR]](ptr noundef [[GLOB_CAST3]])
   foo2(GLOB);
-  // CHECK: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
-  // CHECK: [[GLOB_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD4]] to ptr
-  // CHECK: call void @[[RAW_PTR2]](ptr noundef [[GLOB_CAST4]])
+  // CHECK-DAG: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
+  // CHECK-DAG: [[GLOB_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(1) [[GLOB_LOAD4]] to ptr
+  // CHECK-DAG: call void @[[RAW_PTR2]](ptr noundef [[GLOB_CAST4]])
   foo(LOC);
-  // CHECK: [[LOC_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
-  // CHECK: call void @[[LOC_PTR]](ptr addrspace(3) noundef [[LOC_LOAD3]])
+  // CHECK-DAG: [[LOC_LOAD3:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
+  // CHECK-DAG: call void @[[LOC_PTR]](ptr addrspace(3) noundef [[LOC_LOAD3]])
   foo2(LOC);
-  // CHECK: [[LOC_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
-  // CHECK: [[LOC_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD4]] to ptr
-  // CHECK: call void @[[RAW_PTR2]](ptr noundef [[LOC_CAST4]])
+  // CHECK-DAG: [[LOC_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
+  // CHECK-DAG: [[LOC_CAST4:%[a-zA-Z0-9]+]] = addrspacecast ptr addrspace(3) [[LOC_LOAD4]] to ptr
+  // CHECK-DAG: call void @[[RAW_PTR2]](ptr noundef [[LOC_CAST4]])
   foo(NoAS);
-  // CHECK: [[NoAS_LOAD3:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
-  // CHECK: call void @[[RAW_PTR]](ptr noundef [[NoAS_LOAD3]])
+  // CHECK-DAG: [[NoAS_LOAD3:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
+  // CHECK-DAG: call void @[[RAW_PTR]](ptr noundef [[NoAS_LOAD3]])
   foo2(NoAS);
-  // CHECK: [[NoAS_LOAD4:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
-  // CHECK: call void @[[RAW_PTR2]](ptr noundef [[NoAS_LOAD4]])
+  // CHECK-DAG: [[NoAS_LOAD4:%[a-zA-Z0-9]+]] = load ptr, ptr [[NoAS]], align 8
+  // CHECK-DAG: call void @[[RAW_PTR2]](ptr noundef [[NoAS_LOAD4]])
   tmpl(GLOB);
-  // CHECK: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
-  // CHECK: call void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef [[GLOB_LOAD4]])
+  // CHECK-DAG: [[GLOB_LOAD4:%[a-zA-Z0-9]+]] = load ptr addrspace(1), ptr [[GLOB]], align 8
+  // CHECK-DAG: call void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef [[GLOB_LOAD4]])
   tmpl(LOC);
-  // CHECK: [[LOC_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
-  // CHECK: call void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef [[LOC_LOAD5]])
+  // CHECK-DAG: [[LOC_LOAD5:%[a-zA-Z0-9]+]] = load ptr addrspace(3), ptr [[LOC]], align 8
+  // CHECK-DAG: call void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef [[LOC_LOAD5]])
   tmpl(PRIV);
-  // CHECK: [[PRIV_LOAD5:%[a-zA-Z0-9]+]] = load ptr, ptr [[PRIV]], align 8
-  // CHECK: call void @_Z4tmplIPiEvT_(ptr noundef [[PRIV_LOAD5]])
+  // CHECK-DAG: [[PRIV_LOAD5:%[a-zA-Z0-9]+]] = load ptr, ptr [[PRIV]], align 8
+  // CHECK-DAG: call void @_Z4tmplIPiEvT_(ptr noundef [[PRIV_LOAD5]])
   tmpl(NoAS);
-// CHECK: %33 = load ptr, ptr %NoAS, align 8
-// CHECK: call void @_Z4tmplIPiEvT_(ptr noundef %33)
+// CHECK-DAG: %33 = load ptr, ptr %NoAS, align 8
+// CHECK-DAG: call void @_Z4tmplIPiEvT_(ptr noundef %33)
 }
 
-// CHECK: declare void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef)
-// CHECK: declare void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef)
-// CHECK: declare void @_Z4tmplIPiEvT_(ptr noundef)
+// CHECK-DAG: void @_Z4tmplIPU3AS1iEvT_(ptr addrspace(1) noundef
+// CHECK-DAG: void @_Z4tmplIPU3AS3iEvT_(ptr addrspace(3) noundef
+// CHECK-DAG: void @_Z4tmplIPiEvT_(ptr noundef
diff --git a/clang/test/CodeGenSYCL/debug-info-kernel-variables.cpp b/clang/test/CodeGenSYCL/debug-info-kernel-variables.cpp
index 96c0dcfdb75b6..551c4e7e2b8b4 100644
--- a/clang/test/CodeGenSYCL/debug-info-kernel-variables.cpp
+++ b/clang/test/CodeGenSYCL/debug-info-kernel-variables.cpp
@@ -18,7 +18,7 @@ KERNEL void parallel_for(const KernelType &KernelFunc) {
   KernelFunc();
 }
 
-void my_kernel(int my_param) {
+[[clang::sycl_external]] void my_kernel(int my_param) {
   int my_local = 0;
   my_local = my_param;
 }
diff --git a/clang/test/CodeGenSYCL/field-annotate-addr-space.cpp b/clang/test/CodeGenSYCL/field-annotate-addr-space.cpp
index 26bfda8185112..fe7a160900a54 100644
--- a/clang/test/CodeGenSYCL/field-annotate-addr-space.cpp
+++ b/clang/test/CodeGenSYCL/field-annotate-addr-space.cpp
@@ -9,7 +9,7 @@ struct HasField {
   int *a;
 };
 
-void foo(int *b) {
+[[clang::sycl_external]] void foo(int *b) {
   struct HasField f;
   // CHECK: %[[A:.+]] = getelementptr inbounds nuw %struct.HasField, ptr addrspace(4) %{{.+}}
   // CHECK: %[[CALL:.+]] = call ptr addrspace(4) @llvm.ptr.annotation.p4.p1(ptr addrspace(4) %[[A]], ptr addrspace(1) [[ANNOT]]
diff --git a/clang/test/CodeGenSYCL/function-attrs.cpp b/clang/test/CodeGenSYCL/function-attrs.cpp
index 83a77a617240a..81f893644bc7c 100644
--- a/clang/test/CodeGenSYCL/function-attrs.cpp
+++ b/clang/test/CodeGenSYCL/function-attrs.cpp
@@ -5,11 +5,11 @@
 int foo();
 
 // CHECK-LABEL: define dso_local spir_func void @_Z3barv(
-// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-SAME: ) #[[ATTR2:[0-9]+]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[A_ASCAST:%.*]] = addrspacecast ptr [[A]] to ptr addrspace(4)
-// CHECK-NEXT:    [[CALL:%.*]] = call spir_func noundef i32 @_Z3foov() #[[ATTR1:[0-9]+]]
+// CHECK-NEXT:    [[CALL:%.*]] = call spir_func noundef i32 @_Z3foov() #[[ATTR3:[0-9]+]]
 // CHECK-NEXT:    store i32 [[CALL]], ptr addrspace(4) [[A_ASCAST]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -18,7 +18,7 @@ void bar() {
 }
 
 // CHECK-LABEL: define dso_local spir_func noundef i32 @_Z3foov(
-// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-SAME: ) #[[ATTR2]] {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4)
@@ -29,21 +29,10 @@ int foo() {
 }
 
 template <typename Name, typename Func>
-__attribute__((sycl_kernel)) void kernel_single_task(const Func &kernelFunc) {
+[[clang::sycl_kernel_entry_point(Name)]] void kernel_single_task(const Func &kernelFunc) {
   kernelFunc();
 }
 
-// CHECK-LABEL: define dso_local noundef i32 @main(
-// CHECK-SAME: ) #[[ATTR0]] {
-// CHECK-NEXT:  entry:
-// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[REF_TMP:%.*]] = alloca [[CLASS_ANON:%.*]], align 1
-// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr [[RETVAL]] to ptr addrspace(4)
-// CHECK-NEXT:    [[REF_TMP_ASCAST:%.*]] = addrspacecast ptr [[REF_TMP]] to ptr addrspace(4)
-// CHECK-NEXT:    store i32 0, ptr addrspace(4) [[RETVAL_ASCAST]], align 4
-// CHECK-NEXT:    call spir_func void @_Z18kernel_single_taskIZ4mainE11fake_kernelZ4mainEUlvE_EvRKT0_(ptr addrspace(4) noundef align 1 dereferenceable(1) [[REF_TMP_ASCAST]]) #[[ATTR1]]
-// CHECK-NEXT:    ret i32 0
-//
 int main() {
   kernel_single_task<class fake_kernel>([] { bar(); });
   return 0;
@@ -52,5 +41,5 @@ int main() {
 // CHECK: attributes #0 = { convergent mustprogress noinline norecurse nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 // CHECK: attributes #1 = { convergent nounwind }
 //.
-// CHECK: !0 = !{i32 1, !"wchar_size", i32 4}
+// CHECK: !{{[0-9]+}} = !{i32 1, !"wchar_size", i32 4}
 //.
diff --git a/clang/test/CodeGenSYCL/functionptr-addrspace.cpp b/clang/test/CodeGenSYCL/functionptr-addrspace.cpp
index a477b4c7d03ab..060104a29b60a 100644
--- a/clang/test/CodeGenSYCL/functionptr-addrspace.cpp
+++ b/clang/test/CodeGenSYCL/functionptr-addrspace.cpp
@@ -8,7 +8,7 @@ __attribute__((sycl_kernel)) void kernel_single_task(const Func &kernelFunc) {
 }
 
 // CHECK: define dso_local spir_func{{.*}}invoke_function{{.*}}(ptr noundef %fptr, ptr addrspace(4) noundef %ptr)
-void invoke_function(int (*fptr)(), int *ptr) {}
+[[clang::sycl_external]] void invoke_function(int (*fptr)(), int *ptr) {}
 
 int f() { return 0; }
 
diff --git a/clang/test/CodeGenSYCL/kernel-caller-entry-point.cpp b/clang/test/CodeGenSYCL/kernel-caller-entry-point.cpp
index b5687523aee36..cd1d4d801951d 100644
--- a/clang/test/CodeGenSYCL/kernel-caller-entry-point.cpp
+++ b/clang/test/CodeGenSYCL/kernel-caller-entry-point.cpp
@@ -100,11 +100,8 @@ int main() {
 
 // Verify that SYCL kernel caller functions are emitted for each device target.
 //
-// FIXME: The following set of matches are used to skip over the declaration of
-// main(). main() shouldn't be emitted in device code, but that pruning isn't
-// performed yet.
-// CHECK-DEVICE:      Function Attrs: convergent mustprogress noinline norecurse nounwind optnone
-// CHECK-DEVICE-NEXT: define {{[a-z_ ]*}}noundef i32 @main() #0
+// main() shouldn't be emitted in device code.
+// CHECK-NOT: @main()
 
 // IR for the SYCL kernel caller function generated for
 // single_purpose_kernel_task with single_purpose_kernel_name as the SYCL kernel
diff --git a/clang/test/CodeGenSYCL/sycl-external-attr.cpp b/clang/test/CodeGenSYCL/sycl-external-attr.cpp
new file mode 100644
index 0000000000000..2c22e65874713
--- /dev/null
+++ b/clang/test/CodeGenSYCL/sycl-external-attr.cpp
@@ -0,0 +1,85 @@
+// RUN: %clang_cc1 -fsycl-is-device -triple spir64-unknown-unknown -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
+
+// This test code generation when sycl_external attribute is used
+
+// Function defined and not used - symbols emitted
+[[clang::sycl_external]] int square(int x) { return x*x; }
+// CHECK: define dso_local spir_func noundef i32 @_Z6squarei
+
+// Function defined and used - symbols emitted
+[[clang::sycl_external]] int squareUsed(int x) { return x*x; }
+// CHECK: define dso_local spir_func noundef i32 @_Z10squareUsedi
+
+// FIXME: Constexpr function defined and not used - symbols emitted
+[[clang::sycl_external]] constexpr int squareInlined(int x) { return x*x; }
+// CHECK: define linkonce_odr spir_func noundef i32 @_Z13squareInlinedi
+
+// Function declared but not defined or used - no symbols emitted
+[[clang::sycl_external]] int declOnly();
+// CHECK-NOT: define {{.*}} i32 @_Z8declOnlyv
+// CHECK-NOT: declare {{.*}} i32 @_Z8declOnlyv
+
+// Function declared and used in host but not defined - no symbols emitted
+[[clang::sycl_external]] void declUsedInHost(int y);
+
+// Function declared and used in device but not defined - emit external reference
+[[clang::sycl_external]] void declUsedInDevice(int y);
+// CHECK: define dso_local spir_func void @_Z9deviceUsev
+[[clang::sycl_external]] void deviceUse() { declUsedInDevice(3); }
+// CHECK: declare spir_func void @_Z16declUsedInDevicei
+
+// Function declared with the attribute and later defined - definition emitted
+[[clang::sycl_external]] int func1(int arg);
+int func1(int arg) { return arg; }
+// CHECK: define dso_local spir_func noundef i32 @_Z5func1i
+
+class A {
+// Unused defaulted special member functions - no symbols emitted
+  [[clang::sycl_external]] A& operator=(A& a) = default;
+};
+
+class B {
+  [[clang::sycl_external]] virtual void BFunc1WithAttr() { int i = 1; }
+// CHECK: define linkonce_odr spir_func void @_ZN1B14BFunc1WithAttrEv
+  virtual void BFunc2NoAttr() { int i = 2; }
+};
+
+class C {
+// Special member function defined - definition emitted
+  [[clang::sycl_external]] ~C() {}
+// CHECK: define linkonce_odr spir_func void @_ZN1CD1Ev
+};
+
+// Function reachable from an unused function - definition emitted
+int ret1() { return 1; }
+[[clang::sycl_external]] int withAttr() { return ret1(); }
+// CHECK: define dso_local spir_func noundef i32 @_Z8withAttrv
+// CHECK: define dso_local spir_func noundef i32 @_Z4ret1v
+
+template <typename T>
+[[clang::sycl_external]] void tFunc1(T arg) {}
+// Explicit specialization defined - symbols emitted
+template<>
+[[clang::sycl_external]] void tFunc1<int>(int arg) {}
+// CHECK: define dso_local spir_func void @_Z6tFunc1IiEvT_
+
+template <typename T>
+[[clang::sycl_external]] void tFunc2(T arg) {}
+template void tFunc2<int>(int arg);
+// CHECK: define weak_odr spir_func void @_Z6tFunc2IiEvT_
+template<> void tFunc2<char>(char arg) {}
+// CHECK: define dso_local spir_func void @_Z6tFunc2IcEvT_
+template<> [[clang::sycl_external]] void  tFunc2<long>(long arg) {}
+// CHECK: define dso_local spir_func void @_Z6tFunc2IlEvT_
+
+// Functions defined without the sycl_external attribute that are used
+// in host code, but not in device code are not emitted.
+int squareNoAttr(int x) { return x*x; }
+// CHECK-NOT: define {{.*}} i32 @_Z12squareNoAttri
+
+int main() {
+  declUsedInHost(4);
+  int i = squareUsed(5);
+  int j = squareNoAttr(6);
+  return 0;
+}
diff --git a/clang/test/CodeGenSYCL/unique_stable_name.cpp b/clang/test/CodeGenSYCL/unique_stable_name.cpp
index cc9dd61f435d7..3ab7e3b8f2e7a 100644
--- a/clang/test/CodeGenSYCL/unique_stable_name.cpp
+++ b/clang/test/CodeGenSYCL/unique_stable_name.cpp
@@ -1,22 +1,22 @@
-// RUN: %clang_cc1 -triple spir64-unknown-unknown-sycldevice -fsycl-is-device -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
-// CHECK: @[[LAMBDA_KERNEL3:[^\w]+]] = private unnamed_addr addrspace(1) constant [[LAMBDA_K3_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ4mainEUlPZ4mainEUlvE_E_\00"
-// CHECK: @[[INT1:[^\w]+]] = private unnamed_addr addrspace(1) constant [[INT_SIZE:\[[0-9]+ x i8\]]] c"_ZTSi\00"
-// CHECK: @[[STRING:[^\w]+]] = private unnamed_addr addrspace(1) constant [[STRING_SIZE:\[[0-9]+ x i8\]]] c"_ZTSAppL_ZZ4mainE1jE_i\00",
-// CHECK: @[[INT2:[^\w]+]] = private unnamed_addr addrspace(1) constant [[INT_SIZE]] c"_ZTSi\00"
-// CHECK: @[[LAMBDA_X:[^\w]+]] = private unnamed_addr addrspace(1) constant [[LAMBDA_X_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE_\00"
-// CHECK: @[[MACRO_X:[^\w]+]] = private unnamed_addr addrspace(1) constant [[MACRO_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE0_\00"
-// CHECK: @[[MACRO_Y:[^\w]+]] =  private unnamed_addr addrspace(1) constant [[MACRO_SIZE]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE1_\00"
-// CHECK: @{{.*}} = private unnamed_addr addrspace(1) constant [32 x i8] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE2_\00", align 1
-// CHECK: @{{.*}} = private unnamed_addr addrspace(1) constant [32 x i8] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE3_\00", align 1
-// CHECK: @[[MACRO_MACRO_X:[^\w]+]] = private unnamed_addr addrspace(1) constant [[MACRO_MACRO_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE4_\00"
-// CHECK: @[[MACRO_MACRO_Y:[^\w]+]] = private unnamed_addr addrspace(1) constant [[MACRO_MACRO_SIZE]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE5_\00"
-// CHECK: @[[INT3:[^\w]+]] = private unnamed_addr addrspace(1) constant [[INT_SIZE]] c"_ZTSi\00"
-// CHECK: @[[LAMBDA:[^\w]+]] = private unnamed_addr addrspace(1) constant [[LAMBDA_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE_\00"
-// CHECK: @[[LAMBDA_IN_DEP_INT:[^\w]+]] = private unnamed_addr addrspace(1) constant [[DEP_INT_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ28lambda_in_dependent_functionIiEvvEUlvE_\00",
-// CHECK: @[[LAMBDA_IN_DEP_X:[^\w]+]] = private unnamed_addr addrspace(1) constant [[DEP_LAMBDA_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ28lambda_in_dependent_functionIZZ4mainENKUlvE0_clEvEUlvE_EvvEUlvE_\00",
-// CHECK: @[[LAMBDA_NO_DEP:[^\w]+]] = private unnamed_addr addrspace(1) constant [[NO_DEP_LAMBDA_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ13lambda_no_depIidEvT_T0_EUlidE_\00",
-// CHECK: @[[LAMBDA_TWO_DEP:[^\w]+]] = private unnamed_addr addrspace(1) constant [[DEP_LAMBDA1_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ14lambda_two_depIZZ4mainENKUlvE0_clEvEUliE_ZZ4mainENKS0_clEvEUldE_EvvEUlvE_\00",
-// CHECK: @[[LAMBDA_TWO_DEP2:[^\w]+]] = private unnamed_addr addrspace(1) constant [[DEP_LAMBDA2_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ14lambda_two_depIZZ4mainENKUlvE0_clEvEUldE_ZZ4mainENKS0_clEvEUliE_EvvEUlvE_\00",
+// RUN: %clang_cc1 -triple x86_64-linux-pc -fsycl-is-host -disable-llvm-passes -emit-llvm %s -o - | FileCheck %s
+// CHECK: @[[LAMBDA_KERNEL3:[^\w]+]] = private unnamed_addr constant [[LAMBDA_K3_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ4mainEUlPZ4mainEUlvE_E_\00"
+// CHECK: @[[INT1:[^\w]+]] = private unnamed_addr constant [[INT_SIZE:\[[0-9]+ x i8\]]] c"_ZTSi\00"
+// CHECK: @[[STRING:[^\w]+]] = private unnamed_addr constant [[STRING_SIZE:\[[0-9]+ x i8\]]] c"_ZTSAppL_ZZ4mainE1jE_i\00",
+// CHECK: @[[INT2:[^\w]+]] = private unnamed_addr constant [[INT_SIZE]] c"_ZTSi\00"
+// CHECK: @[[LAMBDA_X:[^\w]+]] = private unnamed_addr constant [[LAMBDA_X_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE_\00"
+// CHECK: @[[MACRO_X:[^\w]+]] = private unnamed_addr constant [[MACRO_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE0_\00"
+// CHECK: @[[MACRO_Y:[^\w]+]] =  private unnamed_addr constant [[MACRO_SIZE]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE1_\00"
+// CHECK: @{{.*}} = private unnamed_addr constant [32 x i8] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE2_\00", align 1
+// CHECK: @{{.*}} = private unnamed_addr constant [32 x i8] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE3_\00", align 1
+// CHECK: @[[MACRO_MACRO_X:[^\w]+]] = private unnamed_addr constant [[MACRO_MACRO_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE4_\00"
+// CHECK: @[[MACRO_MACRO_Y:[^\w]+]] = private unnamed_addr constant [[MACRO_MACRO_SIZE]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE5_\00"
+// CHECK: @[[INT3:[^\w]+]] = private unnamed_addr constant [[INT_SIZE]] c"_ZTSi\00"
+// CHECK: @[[LAMBDA:[^\w]+]] = private unnamed_addr constant [[LAMBDA_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZZ4mainENKUlvE0_clEvEUlvE_\00"
+// CHECK: @[[LAMBDA_IN_DEP_INT:[^\w]+]] = private unnamed_addr constant [[DEP_INT_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ28lambda_in_dependent_functionIiEvvEUlvE_\00",
+// CHECK: @[[LAMBDA_IN_DEP_X:[^\w]+]] = private unnamed_addr constant [[DEP_LAMBDA_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ28lambda_in_dependent_functionIZZ4mainENKUlvE0_clEvEUlvE_EvvEUlvE_\00",
+// CHECK: @[[LAMBDA_NO_DEP:[^\w]+]] = private unnamed_addr constant [[NO_DEP_LAMBDA_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ13lambda_no_depIidEvT_T0_EUlidE_\00",
+// CHECK: @[[LAMBDA_TWO_DEP:[^\w]+]] = private unnamed_addr constant [[DEP_LAMBDA1_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ14lambda_two_depIZZ4mainENKUlvE0_clEvEUliE_ZZ4mainENKS0_clEvEUldE_EvvEUlvE_\00",
+// CHECK: @[[LAMBDA_TWO_DEP2:[^\w]+]] = private unnamed_addr constant [[DEP_LAMBDA2_SIZE:\[[0-9]+ x i8\]]] c"_ZTSZ14lambda_two_depIZZ4mainENKUlvE0_clEvEUldE_ZZ4mainENKS0_clEvEUliE_EvvEUlvE_\00",
 
 extern "C" void puts(const char *) {}
 
@@ -65,95 +65,105 @@ template <typename KernelName, typename KernelType>
   kernelFunc();
 }
 
+template<typename KernelType>
+void unnamed_kernel_single_task(KernelType kernelFunc) {
+  kernel_single_task<KernelType>(kernelFunc);
+}
+
+template <typename KernelName, typename KernelType>
+void not_kernel_single_task(KernelType kernelFunc) {
+  kernelFunc();
+}
+
 int main() {
-  kernel_single_task<class kernel2>(func<Derp>);
-  // CHECK: call spir_func void @_Z18kernel_single_taskIZ4mainE7kernel2PFPKcvEEvT0_(ptr noundef @_Z4funcI4DerpEDTu33__builtin_sycl_unique_stable_nameDtsrT_3strEEEv)
+  not_kernel_single_task<class kernel2>(func<Derp>);
+  // CHECK: call void @_Z22not_kernel_single_taskIZ4mainE7kernel2PFPKcvEEvT0_(ptr noundef @_Z4funcI4DerpEDTu33__builtin_sycl_unique_stable_nameDtsrT_3strEEEv)
 
   auto l1 = []() { return 1; };
   auto l2 = [](decltype(l1) *l = nullptr) { return 2; };
-  kernel_single_task<class kernel3>(l2);
+  kernel_single_task<decltype(l2)>(l2);
   puts(__builtin_sycl_unique_stable_name(decltype(l2)));
-  // CHECK: call spir_func void @_Z18kernel_single_taskIZ4mainE7kernel3Z4mainEUlPZ4mainEUlvE_E_EvT0_
-  // CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[LAMBDA_KERNEL3]] to ptr addrspace(4)))
+  // CHECK: call void @_Z18kernel_single_taskIZ4mainEUlPZ4mainEUlvE_E_S2_EvT0_
+  // CHECK: call void @puts(ptr noundef @[[LAMBDA_KERNEL3]])
 
   constexpr const char str[] = "lalala";
   static_assert(__builtin_strcmp(__builtin_sycl_unique_stable_name(decltype(str)), "_ZTSA7_Kc\0") == 0, "unexpected mangling");
 
   int i = 0;
   puts(__builtin_sycl_unique_stable_name(decltype(i++)));
-  // CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[INT1]] to ptr addrspace(4)))
+  // CHECK: call void @puts(ptr noundef @[[INT1]])
 
   // FIXME: Ensure that j is incremented because VLAs are terrible.
   int j = 55;
   puts(__builtin_sycl_unique_stable_name(int[++j]));
-  // CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[STRING]] to ptr addrspace(4)))
+  // CHECK: call void @puts(ptr noundef @[[STRING]])
 
-  // CHECK: define internal spir_func void @_Z18kernel_single_taskIZ4mainE7kernel2PFPKcvEEvT0_
-  // CHECK: declare spir_func noundef ptr addrspace(4) @_Z4funcI4DerpEDTu33__builtin_sycl_unique_stable_nameDtsrT_3strEEEv
-  // CHECK: define internal spir_func void @_Z18kernel_single_taskIZ4mainE7kernel3Z4mainEUlPZ4mainEUlvE_E_EvT0_
-  // CHECK: define internal spir_func void @_Z18kernel_single_taskIZ4mainE6kernelZ4mainEUlvE0_EvT0_
+  // CHECK: define internal void @_Z22not_kernel_single_taskIZ4mainE7kernel2PFPKcvEEvT0_
+  // CHECK: declare noundef ptr @_Z4funcI4DerpEDTu33__builtin_sycl_unique_stable_nameDtsrT_3strEEEv
+  // CHECK: define internal void @_Z18kernel_single_taskIZ4mainEUlPZ4mainEUlvE_E_S2_EvT0_
+  // CHECK: define internal void @_Z18kernel_single_taskIZ4mainEUlvE0_S0_EvT0_
 
-  kernel_single_task<class kernel>(
+  unnamed_kernel_single_task(
       []() {
         puts(__builtin_sycl_unique_stable_name(int));
-        // CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[INT2]] to ptr addrspace(4)))
+        // CHECK: call void @puts(ptr noundef @[[INT2]])
 
         auto x = []() {};
         puts(__builtin_sycl_unique_stable_name(decltype(x)));
-        // CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[LAMBDA_X]] to ptr addrspace(4)))
+        // CHECK: call void @puts(ptr noundef @[[LAMBDA_X]])
 
         DEF_IN_MACRO();
-        // CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[MACRO_X]] to ptr addrspace(4)))
-        // CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[MACRO_Y]] to ptr addrspace(4)))
+        // CHECK: call void @puts(ptr noundef @[[MACRO_X]])
+        // CHECK: call void @puts(ptr noundef @[[MACRO_Y]])
 
         MACRO_CALLS_MACRO();
-        // CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[MACRO_MACRO_X]] to ptr addrspace(4)))
-        // CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[MACRO_MACRO_Y]] to ptr addrspace(4)))
+        // CHECK: call void @puts(ptr noundef @[[MACRO_MACRO_X]])
+        // CHECK: call void @puts(ptr noundef @[[MACRO_MACRO_Y]])
 
         template_param<int>();
-        // CHECK: call spir_func void @_Z14template_paramIiEvv
+        // CHECK: call void @_Z14template_paramIiEvv
 
         template_param<decltype(x)>();
-        // CHECK: call spir_func void @_Z14template_paramIZZ4mainENKUlvE0_clEvEUlvE_Evv
+        // CHECK: call void @_Z14template_paramIZZ4mainENKUlvE0_clEvEUlvE_Evv
 
         lambda_in_dependent_function<int>();
-        // CHECK: call spir_func void @_Z28lambda_in_dependent_functionIiEvv
+        // CHECK: call void @_Z28lambda_in_dependent_functionIiEvv
 
         lambda_in_dependent_function<decltype(x)>();
-        // CHECK: call spir_func void @_Z28lambda_in_dependent_functionIZZ4mainENKUlvE0_clEvEUlvE_Evv
+        // CHECK: call void @_Z28lambda_in_dependent_functionIZZ4mainENKUlvE0_clEvEUlvE_Evv
 
         lambda_no_dep<int, double>(3, 5.5);
-        // CHECK: call spir_func void @_Z13lambda_no_depIidEvT_T0_(i32 noundef 3, double noundef 5.500000e+00)
+        // CHECK: call void @_Z13lambda_no_depIidEvT_T0_(i32 noundef 3, double noundef 5.500000e+00)
 
         int a = 5;
         double b = 10.7;
         auto y = [](int a) { return a; };
         auto z = [](double b) { return b; };
         lambda_two_dep<decltype(y), decltype(z)>();
-        // CHECK: call spir_func void @_Z14lambda_two_depIZZ4mainENKUlvE0_clEvEUliE_ZZ4mainENKS0_clEvEUldE_Evv
+        // CHECK: call void @_Z14lambda_two_depIZZ4mainENKUlvE0_clEvEUliE_ZZ4mainENKS0_clEvEUldE_Evv
 
         lambda_two_dep<decltype(z), decltype(y)>();
-        // CHECK: call spir_func void @_Z14lambda_two_depIZZ4mainENKUlvE0_clEvEUldE_ZZ4mainENKS0_clEvEUliE_Evv
+        // CHECK: call void @_Z14lambda_two_depIZZ4mainENKUlvE0_clEvEUldE_ZZ4mainENKS0_clEvEUliE_Evv
       });
 }
 
-// CHECK: define linkonce_odr spir_func void @_Z14template_paramIiEvv
-// CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[INT3]] to ptr addrspace(4)))
+// CHECK: define linkonce_odr void @_Z14template_paramIiEvv
+// CHECK: call void @puts(ptr noundef @[[INT3]])
 
-// CHECK: define internal spir_func void @_Z14template_paramIZZ4mainENKUlvE0_clEvEUlvE_Evv
-// CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[LAMBDA]] to ptr addrspace(4)))
+// CHECK: define internal void @_Z14template_paramIZZ4mainENKUlvE0_clEvEUlvE_Evv
+// CHECK: call void @puts(ptr noundef @[[LAMBDA]])
 
-// CHECK: define linkonce_odr spir_func void @_Z28lambda_in_dependent_functionIiEvv
-// CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[LAMBDA_IN_DEP_INT]] to ptr addrspace(4)))
+// CHECK: define linkonce_odr void @_Z28lambda_in_dependent_functionIiEvv
+// CHECK: call void @puts(ptr noundef @[[LAMBDA_IN_DEP_INT]])
 
-// CHECK: define internal spir_func void @_Z28lambda_in_dependent_functionIZZ4mainENKUlvE0_clEvEUlvE_Evv
-// CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[LAMBDA_IN_DEP_X]] to ptr addrspace(4)))
+// CHECK: define internal void @_Z28lambda_in_dependent_functionIZZ4mainENKUlvE0_clEvEUlvE_Evv
+// CHECK: call void @puts(ptr noundef @[[LAMBDA_IN_DEP_X]])
 
-// CHECK: define linkonce_odr spir_func void @_Z13lambda_no_depIidEvT_T0_(i32 noundef %a, double noundef %b)
-// CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[LAMBDA_NO_DEP]] to ptr addrspace(4)))
+// CHECK: define linkonce_odr void @_Z13lambda_no_depIidEvT_T0_(i32 noundef %a, double noundef %b)
+// CHECK: call void @puts(ptr noundef @[[LAMBDA_NO_DEP]])
 
-// CHECK: define internal spir_func void @_Z14lambda_two_depIZZ4mainENKUlvE0_clEvEUliE_ZZ4mainENKS0_clEvEUldE_Evv
-// CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[LAMBDA_TWO_DEP]] to ptr addrspace(4)))
+// CHECK: define internal void @_Z14lambda_two_depIZZ4mainENKUlvE0_clEvEUliE_ZZ4mainENKS0_clEvEUldE_Evv
+// CHECK: call void @puts(ptr noundef @[[LAMBDA_TWO_DEP]])
 
-// CHECK: define internal spir_func void @_Z14lambda_two_depIZZ4mainENKUlvE0_clEvEUldE_ZZ4mainENKS0_clEvEUliE_Evv
-// CHECK: call spir_func void @puts(ptr addrspace(4) noundef addrspacecast (ptr addrspace(1) @[[LAMBDA_TWO_DEP2]] to ptr addrspace(4)))
+// CHECK: define internal void @_Z14lambda_two_depIZZ4mainENKUlvE0_clEvEUldE_ZZ4mainENKS0_clEvEUliE_Evv
+// CHECK: call void @puts(ptr noundef @[[LAMBDA_TWO_DEP2]])
diff --git a/clang/test/CoverageMapping/logical.cpp b/clang/test/CoverageMapping/logical.cpp
index 2a22d6cca4518..caa773cfec3a5 100644
--- a/clang/test/CoverageMapping/logical.cpp
+++ b/clang/test/CoverageMapping/logical.cpp
@@ -1,27 +1,31 @@
 // RUN: %clang_cc1 -mllvm -emptyline-comment-coverage=false -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name logical.cpp %s | FileCheck %s
 // RUN: %clang_cc1 -mllvm -emptyline-comment-coverage=false -fcoverage-mcdc -fprofile-instrument=clang -fcoverage-mapping -dump-coverage-mapping -emit-llvm-only -main-file-name logical.cpp %s | FileCheck %s -check-prefix=MCDC
 
-int main() {                        // CHECK: File 0, [[@LINE]]:12 -> [[@LINE+23]]:2 = #0
+int main() {                        // CHECK: File 0, [[@LINE]]:12 -> [[@LINE+27]]:2 = #0
   bool bt = true;
   bool bf = false;                  // MCDC: Decision,File 0, [[@LINE+1]]:12 -> [[@LINE+1]]:20 = M:3, C:2
   bool a = bt && bf;                // CHECK-NEXT: File 0, [[@LINE]]:12 -> [[@LINE]]:14 = #0
                                     // CHECK-NEXT: Branch,File 0, [[@LINE-1]]:12 -> [[@LINE-1]]:14 = #1, (#0 - #1)
-                                    // CHECK-NEXT: File 0, [[@LINE-2]]:18 -> [[@LINE-2]]:20 = #1
-                                    // CHECK-NEXT: Branch,File 0, [[@LINE-3]]:18 -> [[@LINE-3]]:20 = #2, (#1 - #2)
+                                    // CHECK-NEXT: Gap,File 0, [[@LINE-2]]:14 -> [[@LINE-2]]:18 = #1
+                                    // CHECK-NEXT: File 0, [[@LINE-3]]:18 -> [[@LINE-3]]:20 = #1
+                                    // CHECK-NEXT: Branch,File 0, [[@LINE-4]]:18 -> [[@LINE-4]]:20 = #2, (#1 - #2)
                                     // MCDC: Decision,File 0, [[@LINE+1]]:7 -> [[@LINE+2]]:9 = M:6, C:2
   a = bt &&                         // CHECK-NEXT: File 0, [[@LINE]]:7 -> [[@LINE]]:9 = #0
       bf;                           // CHECK-NEXT: Branch,File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = #3, (#0 - #3)
-                                    // CHECK-NEXT: File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = #3
-                                    // CHECK-NEXT: Branch,File 0, [[@LINE-2]]:7 -> [[@LINE-2]]:9 = #4, (#3 - #4)
+                                    // CHECK-NEXT: Gap,File 0, [[@LINE-2]]:9 -> [[@LINE-1]]:7 = #3
+                                    // CHECK-NEXT: File 0, [[@LINE-2]]:7 -> [[@LINE-2]]:9 = #3
+                                    // CHECK-NEXT: Branch,File 0, [[@LINE-3]]:7 -> [[@LINE-3]]:9 = #4, (#3 - #4)
                                     // MCDC: Decision,File 0, [[@LINE+1]]:7 -> [[@LINE+1]]:15 = M:9, C:2
   a = bf || bt;                     // CHECK-NEXT: File 0, [[@LINE]]:7 -> [[@LINE]]:9 = #0
                                     // CHECK-NEXT: Branch,File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = (#0 - #5), #5
-                                    // CHECK-NEXT: File 0, [[@LINE-2]]:13 -> [[@LINE-2]]:15 = #5
-                                    // CHECK-NEXT: Branch,File 0, [[@LINE-3]]:13 -> [[@LINE-3]]:15 = (#5 - #6), #6
+                                    // CHECK-NEXT: Gap,File 0, [[@LINE-2]]:9 -> [[@LINE-2]]:13 = #5
+                                    // CHECK-NEXT: File 0, [[@LINE-3]]:13 -> [[@LINE-3]]:15 = #5
+                                    // CHECK-NEXT: Branch,File 0, [[@LINE-4]]:13 -> [[@LINE-4]]:15 = (#5 - #6), #6
                                     // MCDC: Decision,File 0, [[@LINE+1]]:7 -> [[@LINE+2]]:9 = M:12, C:2
   a = bf ||                         // CHECK-NEXT: File 0, [[@LINE]]:7 -> [[@LINE]]:9 = #0
       bt;                           // CHECK-NEXT: Branch,File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = (#0 - #7), #7
-                                    // CHECK-NEXT: File 0, [[@LINE-1]]:7 -> [[@LINE-1]]:9 = #7
-                                    // CHECK-NEXT: Branch,File 0, [[@LINE-2]]:7 -> [[@LINE-2]]:9 = (#7 - #8), #8
+                                    // CHECK-NEXT: Gap,File 0, [[@LINE-2]]:9 -> [[@LINE-1]]:7 = #7
+                                    // CHECK-NEXT: File 0, [[@LINE-2]]:7 -> [[@LINE-2]]:9 = #7
+                                    // CHECK-NEXT: Branch,File 0, [[@LINE-3]]:7 -> [[@LINE-3]]:9 = (#7 - #8), #8
   return 0;
 }
diff --git a/clang/test/CodeGen/AArch64/debug-sve-vector-types.c b/clang/test/DebugInfo/AArch64/sve-vector-types.c
similarity index 100%
rename from clang/test/CodeGen/AArch64/debug-sve-vector-types.c
rename to clang/test/DebugInfo/AArch64/sve-vector-types.c
diff --git a/clang/test/CodeGen/AArch64/debug-sve-vectorx2-types.c b/clang/test/DebugInfo/AArch64/sve-vectorx2-types.c
similarity index 100%
rename from clang/test/CodeGen/AArch64/debug-sve-vectorx2-types.c
rename to clang/test/DebugInfo/AArch64/sve-vectorx2-types.c
diff --git a/clang/test/CodeGen/AArch64/debug-sve-vectorx3-types.c b/clang/test/DebugInfo/AArch64/sve-vectorx3-types.c
similarity index 100%
rename from clang/test/CodeGen/AArch64/debug-sve-vectorx3-types.c
rename to clang/test/DebugInfo/AArch64/sve-vectorx3-types.c
diff --git a/clang/test/CodeGen/AArch64/debug-sve-vectorx4-types.c b/clang/test/DebugInfo/AArch64/sve-vectorx4-types.c
similarity index 100%
rename from clang/test/CodeGen/AArch64/debug-sve-vectorx4-types.c
rename to clang/test/DebugInfo/AArch64/sve-vectorx4-types.c
diff --git a/clang/test/CodeGen/AArch64/debug-types.c b/clang/test/DebugInfo/AArch64/types.c
similarity index 100%
rename from clang/test/CodeGen/AArch64/debug-types.c
rename to clang/test/DebugInfo/AArch64/types.c
diff --git a/clang/test/CodeGen/assignment-tracking/assignment-tracking.cpp b/clang/test/DebugInfo/AssignmentTracking/assignment-tracking.cpp
similarity index 100%
rename from clang/test/CodeGen/assignment-tracking/assignment-tracking.cpp
rename to clang/test/DebugInfo/AssignmentTracking/assignment-tracking.cpp
diff --git a/clang/test/CodeGen/assignment-tracking/flag.cpp b/clang/test/DebugInfo/AssignmentTracking/flag.cpp
similarity index 100%
rename from clang/test/CodeGen/assignment-tracking/flag.cpp
rename to clang/test/DebugInfo/AssignmentTracking/flag.cpp
diff --git a/clang/test/CodeGen/assignment-tracking/memcpy-fragment.cpp b/clang/test/DebugInfo/AssignmentTracking/memcpy-fragment.cpp
similarity index 100%
rename from clang/test/CodeGen/assignment-tracking/memcpy-fragment.cpp
rename to clang/test/DebugInfo/AssignmentTracking/memcpy-fragment.cpp
diff --git a/clang/test/CodeGen/assignment-tracking/nested-scope.cpp b/clang/test/DebugInfo/AssignmentTracking/nested-scope.cpp
similarity index 100%
rename from clang/test/CodeGen/assignment-tracking/nested-scope.cpp
rename to clang/test/DebugInfo/AssignmentTracking/nested-scope.cpp
diff --git a/clang/test/CodeGen/attr-btf_tag-typedef.c b/clang/test/DebugInfo/BPF/attr-btf_tag-typedef.c
similarity index 100%
rename from clang/test/CodeGen/attr-btf_tag-typedef.c
rename to clang/test/DebugInfo/BPF/attr-btf_tag-typedef.c
diff --git a/clang/test/CodeGen/attr-btf_type_tag-func.c b/clang/test/DebugInfo/BPF/attr-btf_type_tag-func.c
similarity index 100%
rename from clang/test/CodeGen/attr-btf_type_tag-func.c
rename to clang/test/DebugInfo/BPF/attr-btf_type_tag-func.c
diff --git a/clang/test/CodeGen/bpf-attr-type-tag-atomic.c b/clang/test/DebugInfo/BPF/bpf-attr-type-tag-atomic.c
similarity index 100%
rename from clang/test/CodeGen/bpf-attr-type-tag-atomic.c
rename to clang/test/DebugInfo/BPF/bpf-attr-type-tag-atomic.c
diff --git a/clang/test/CodeGen/bpf-debug-info-extern-func.c b/clang/test/DebugInfo/BPF/bpf-debug-info-extern-func.c
similarity index 100%
rename from clang/test/CodeGen/bpf-debug-info-extern-func.c
rename to clang/test/DebugInfo/BPF/bpf-debug-info-extern-func.c
diff --git a/clang/test/CodeGen/builtin-preserve-access-index-nonptr.c b/clang/test/DebugInfo/BPF/builtin-preserve-access-index-nonptr.c
similarity index 100%
rename from clang/test/CodeGen/builtin-preserve-access-index-nonptr.c
rename to clang/test/DebugInfo/BPF/builtin-preserve-access-index-nonptr.c
diff --git a/clang/test/CodeGen/builtins-bpf-preserve-field-info-1.c b/clang/test/DebugInfo/BPF/builtins-bpf-preserve-field-info-1.c
similarity index 100%
rename from clang/test/CodeGen/builtins-bpf-preserve-field-info-1.c
rename to clang/test/DebugInfo/BPF/builtins-bpf-preserve-field-info-1.c
diff --git a/clang/test/CodeGen/builtins-bpf-preserve-field-info-2.c b/clang/test/DebugInfo/BPF/builtins-bpf-preserve-field-info-2.c
similarity index 100%
rename from clang/test/CodeGen/builtins-bpf-preserve-field-info-2.c
rename to clang/test/DebugInfo/BPF/builtins-bpf-preserve-field-info-2.c
diff --git a/clang/test/CodeGen/builtins-bpf-preserve-field-info-3.c b/clang/test/DebugInfo/BPF/builtins-bpf-preserve-field-info-3.c
similarity index 100%
rename from clang/test/CodeGen/builtins-bpf-preserve-field-info-3.c
rename to clang/test/DebugInfo/BPF/builtins-bpf-preserve-field-info-3.c
index 53b448908a0a6..a8cc073974604 100644
--- a/clang/test/CodeGen/builtins-bpf-preserve-field-info-3.c
+++ b/clang/test/DebugInfo/BPF/builtins-bpf-preserve-field-info-3.c
@@ -37,5 +37,5 @@ unsigned unit3() {
 // CHECK: call i32 @llvm.bpf.preserve.type.info(i32 5, i64 1), !dbg !{{[0-9]+}}, !llvm.preserve.access.index ![[ENUM_AA]]
 
 // CHECK: ![[ENUM_AA]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "AA"
-// CHECK: ![[STRUCT_S]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s"
 // CHECK: ![[TYPEDEF_INT]] = !DIDerivedType(tag: DW_TAG_typedef, name: "__int"
+// CHECK: ![[STRUCT_S]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s"
diff --git a/clang/test/CodeGen/builtins-bpf-preserve-field-info-4.c b/clang/test/DebugInfo/BPF/builtins-bpf-preserve-field-info-4.c
similarity index 100%
rename from clang/test/CodeGen/builtins-bpf-preserve-field-info-4.c
rename to clang/test/DebugInfo/BPF/builtins-bpf-preserve-field-info-4.c
diff --git a/clang/test/CodeGenCXX/2006-11-20-GlobalSymbols.cpp b/clang/test/DebugInfo/CXX/2006-11-20-GlobalSymbols.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/2006-11-20-GlobalSymbols.cpp
rename to clang/test/DebugInfo/CXX/2006-11-20-GlobalSymbols.cpp
diff --git a/clang/test/CodeGenCXX/2007-01-02-UnboundedArray.cpp b/clang/test/DebugInfo/CXX/2007-01-02-UnboundedArray.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/2007-01-02-UnboundedArray.cpp
rename to clang/test/DebugInfo/CXX/2007-01-02-UnboundedArray.cpp
diff --git a/clang/test/CodeGenCXX/2009-03-17-dbg.cpp b/clang/test/DebugInfo/CXX/2009-03-17.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/2009-03-17-dbg.cpp
rename to clang/test/DebugInfo/CXX/2009-03-17.cpp
diff --git a/clang/test/CodeGenCXX/2009-06-16-DebugInfoCrash.cpp b/clang/test/DebugInfo/CXX/2009-06-16-Crash.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/2009-06-16-DebugInfoCrash.cpp
rename to clang/test/DebugInfo/CXX/2009-06-16-Crash.cpp
diff --git a/clang/test/CodeGenCXX/2010-05-10-Var-DbgInfo.cpp b/clang/test/DebugInfo/CXX/2010-05-10-Var.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/2010-05-10-Var-DbgInfo.cpp
rename to clang/test/DebugInfo/CXX/2010-05-10-Var.cpp
diff --git a/clang/test/CodeGenCXX/2010-05-12-PtrToMember-Dbg.cpp b/clang/test/DebugInfo/CXX/2010-05-12-PtrToMember.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/2010-05-12-PtrToMember-Dbg.cpp
rename to clang/test/DebugInfo/CXX/2010-05-12-PtrToMember.cpp
diff --git a/clang/test/CodeGenCXX/2010-06-21-LocalVarDbg.cpp b/clang/test/DebugInfo/CXX/2010-06-21-LocalVarDbg.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/2010-06-21-LocalVarDbg.cpp
rename to clang/test/DebugInfo/CXX/2010-06-21-LocalVarDbg.cpp
diff --git a/clang/test/CodeGenCXX/2010-07-23-DeclLoc.cpp b/clang/test/DebugInfo/CXX/2010-07-23-DeclLoc.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/2010-07-23-DeclLoc.cpp
rename to clang/test/DebugInfo/CXX/2010-07-23-DeclLoc.cpp
diff --git a/clang/test/CodeGenCXX/Inputs/debug-info-class-limited.cpp b/clang/test/DebugInfo/CXX/Inputs/class-limited.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/Inputs/debug-info-class-limited.cpp
rename to clang/test/DebugInfo/CXX/Inputs/class-limited.cpp
diff --git a/clang/test/CodeGenCXX/PR20038.cpp b/clang/test/DebugInfo/CXX/PR20038.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/PR20038.cpp
rename to clang/test/DebugInfo/CXX/PR20038.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-access.cpp b/clang/test/DebugInfo/CXX/access.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-access.cpp
rename to clang/test/DebugInfo/CXX/access.cpp
diff --git a/clang/test/CodeGenCXX/aix-static-init-debug-info.cpp b/clang/test/DebugInfo/CXX/aix-static-init.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/aix-static-init-debug-info.cpp
rename to clang/test/DebugInfo/CXX/aix-static-init.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-alias.cpp b/clang/test/DebugInfo/CXX/alias.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-alias.cpp
rename to clang/test/DebugInfo/CXX/alias.cpp
diff --git a/clang/test/CodeGenCXX/dbg-info-all-calls-described.cpp b/clang/test/DebugInfo/CXX/all-calls-described.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/dbg-info-all-calls-described.cpp
rename to clang/test/DebugInfo/CXX/all-calls-described.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-anon-namespace.cpp b/clang/test/DebugInfo/CXX/anon-namespace.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-anon-namespace.cpp
rename to clang/test/DebugInfo/CXX/anon-namespace.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp b/clang/test/DebugInfo/CXX/anon-union-vars.cpp
similarity index 95%
rename from clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
rename to clang/test/DebugInfo/CXX/anon-union-vars.cpp
index 61b3c7c0526c8..3aca4e199ab8d 100644
--- a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
+++ b/clang/test/DebugInfo/CXX/anon-union-vars.cpp
@@ -46,7 +46,7 @@ void instantiate(int x) {
 
 // CHECK: !DIGlobalVariable(name: "c",{{.*}} file: [[FILE:.*]], line: 6,{{.*}} isLocal: true, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "d",{{.*}} file: [[FILE]], line: 6,{{.*}} isLocal: true, isDefinition: true
-// CHECK: [[FILE]] = !DIFile(filename: "{{.*}}debug-info-anon-union-vars.cpp",
+// CHECK: [[FILE]] = !DIFile(filename: "{{.*}}anon-union-vars.cpp",
 // CHECK: !DIGlobalVariable(name: "a",{{.*}} file: [[FILE]], line: 6,{{.*}} isLocal: true, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "b",{{.*}} file: [[FILE]], line: 6,{{.*}} isLocal: true, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "result", {{.*}} isLocal: false, isDefinition: true
diff --git a/clang/test/CodeGenCXX/debug-info-artificial-arg.cpp b/clang/test/DebugInfo/CXX/artificial-arg.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-artificial-arg.cpp
rename to clang/test/DebugInfo/CXX/artificial-arg.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-atexit-stub.cpp b/clang/test/DebugInfo/CXX/atexit-stub.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-atexit-stub.cpp
rename to clang/test/DebugInfo/CXX/atexit-stub.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-auto-return.cpp b/clang/test/DebugInfo/CXX/auto-return.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-auto-return.cpp
rename to clang/test/DebugInfo/CXX/auto-return.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-block-invocation-linkage-name.cpp b/clang/test/DebugInfo/CXX/block-invocation-linkage-name.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-block-invocation-linkage-name.cpp
rename to clang/test/DebugInfo/CXX/block-invocation-linkage-name.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-blocks.cpp b/clang/test/DebugInfo/CXX/blocks.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-blocks.cpp
rename to clang/test/DebugInfo/CXX/blocks.cpp
diff --git a/clang/test/CodeGenCXX/bpf-debug-structors.cpp b/clang/test/DebugInfo/CXX/bpf-structors.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/bpf-debug-structors.cpp
rename to clang/test/DebugInfo/CXX/bpf-structors.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-byval.cpp b/clang/test/DebugInfo/CXX/byval.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-byval.cpp
rename to clang/test/DebugInfo/CXX/byval.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-calling-conventions.cpp b/clang/test/DebugInfo/CXX/calling-conventions.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-calling-conventions.cpp
rename to clang/test/DebugInfo/CXX/calling-conventions.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-char16.cpp b/clang/test/DebugInfo/CXX/char16.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-char16.cpp
rename to clang/test/DebugInfo/CXX/char16.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-class-limited-plugin.test b/clang/test/DebugInfo/CXX/class-limited-plugin.test
similarity index 62%
rename from clang/test/CodeGenCXX/debug-info-class-limited-plugin.test
rename to clang/test/DebugInfo/CXX/class-limited-plugin.test
index 17248d5743dac..1a2cc223a3829 100644
--- a/clang/test/CodeGenCXX/debug-info-class-limited-plugin.test
+++ b/clang/test/DebugInfo/CXX/class-limited-plugin.test
@@ -1,2 +1,2 @@
-RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -o - -load %llvmshlibdir/PrintFunctionNames%pluginext -add-plugin print-fns %S/Inputs/debug-info-class-limited.cpp 2>&1 | FileCheck %S/Inputs/debug-info-class-limited.cpp
+RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -o - -load %llvmshlibdir/PrintFunctionNames%pluginext -add-plugin print-fns %S/Inputs/class-limited.cpp 2>&1 | FileCheck %S/Inputs/class-limited.cpp
 REQUIRES: plugins, examples
diff --git a/clang/test/DebugInfo/CXX/class-limited.test b/clang/test/DebugInfo/CXX/class-limited.test
new file mode 100644
index 0000000000000..b1279d9855bf7
--- /dev/null
+++ b/clang/test/DebugInfo/CXX/class-limited.test
@@ -0,0 +1 @@
+RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited %S/Inputs/class-limited.cpp -o - | FileCheck %S/Inputs/class-limited.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-class-nolimit.cpp b/clang/test/DebugInfo/CXX/class-nolimit.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-class-nolimit.cpp
rename to clang/test/DebugInfo/CXX/class-nolimit.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-class-optzns.cpp b/clang/test/DebugInfo/CXX/class-optzns.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-class-optzns.cpp
rename to clang/test/DebugInfo/CXX/class-optzns.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-class.cpp b/clang/test/DebugInfo/CXX/class.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-class.cpp
rename to clang/test/DebugInfo/CXX/class.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-display-name.cpp b/clang/test/DebugInfo/CXX/codeview-display-name.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-codeview-display-name.cpp
rename to clang/test/DebugInfo/CXX/codeview-display-name.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-heapallocsite.cpp b/clang/test/DebugInfo/CXX/codeview-heapallocsite.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-codeview-heapallocsite.cpp
rename to clang/test/DebugInfo/CXX/codeview-heapallocsite.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-injected-class.cpp b/clang/test/DebugInfo/CXX/codeview-injected-class.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-codeview-injected-class.cpp
rename to clang/test/DebugInfo/CXX/codeview-injected-class.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-nested-types.cpp b/clang/test/DebugInfo/CXX/codeview-nested-types.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-codeview-nested-types.cpp
rename to clang/test/DebugInfo/CXX/codeview-nested-types.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-nodebug.cpp b/clang/test/DebugInfo/CXX/codeview-nodebug.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-codeview-nodebug.cpp
rename to clang/test/DebugInfo/CXX/codeview-nodebug.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-template-literal.cpp b/clang/test/DebugInfo/CXX/codeview-template-literal.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-codeview-template-literal.cpp
rename to clang/test/DebugInfo/CXX/codeview-template-literal.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-template-type.cpp b/clang/test/DebugInfo/CXX/codeview-template-type.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-codeview-template-type.cpp
rename to clang/test/DebugInfo/CXX/codeview-template-type.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp b/clang/test/DebugInfo/CXX/codeview-unnamed.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
rename to clang/test/DebugInfo/CXX/codeview-unnamed.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-var-templates.cpp b/clang/test/DebugInfo/CXX/codeview-var-templates.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-codeview-var-templates.cpp
rename to clang/test/DebugInfo/CXX/codeview-var-templates.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-coff.cpp b/clang/test/DebugInfo/CXX/coff.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-coff.cpp
rename to clang/test/DebugInfo/CXX/coff.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-composite-cc.cpp b/clang/test/DebugInfo/CXX/composite-cc.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-composite-cc.cpp
rename to clang/test/DebugInfo/CXX/composite-cc.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-composite-triviality-fwd-decl.cpp b/clang/test/DebugInfo/CXX/composite-triviality-fwd-decl.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-composite-triviality-fwd-decl.cpp
rename to clang/test/DebugInfo/CXX/composite-triviality-fwd-decl.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-composite-triviality.cpp b/clang/test/DebugInfo/CXX/composite-triviality.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-composite-triviality.cpp
rename to clang/test/DebugInfo/CXX/composite-triviality.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-context.cpp b/clang/test/DebugInfo/CXX/context.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-context.cpp
rename to clang/test/DebugInfo/CXX/context.cpp
diff --git a/clang/test/CodeGenCXX/cp-blocks-linetables.cpp b/clang/test/DebugInfo/CXX/cp-blocks-linetables.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/cp-blocks-linetables.cpp
rename to clang/test/DebugInfo/CXX/cp-blocks-linetables.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ctor-homing-flag.cpp b/clang/test/DebugInfo/CXX/ctor-homing-flag.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ctor-homing-flag.cpp
rename to clang/test/DebugInfo/CXX/ctor-homing-flag.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ctor.cpp b/clang/test/DebugInfo/CXX/ctor.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ctor.cpp
rename to clang/test/DebugInfo/CXX/ctor.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ctor2.cpp b/clang/test/DebugInfo/CXX/ctor2.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ctor2.cpp
rename to clang/test/DebugInfo/CXX/ctor2.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-cxx0x.cpp b/clang/test/DebugInfo/CXX/cxx0x.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-cxx0x.cpp
rename to clang/test/DebugInfo/CXX/cxx0x.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-cxx1y.cpp b/clang/test/DebugInfo/CXX/cxx1y.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-cxx1y.cpp
rename to clang/test/DebugInfo/CXX/cxx1y.cpp
diff --git a/clang/test/CodeGenCXX/debug-info.cpp b/clang/test/DebugInfo/CXX/debug-info.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info.cpp
rename to clang/test/DebugInfo/CXX/debug-info.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-decl-nested.cpp b/clang/test/DebugInfo/CXX/decl-nested.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-decl-nested.cpp
rename to clang/test/DebugInfo/CXX/decl-nested.cpp
diff --git a/clang/test/CodeGenCXX/defaulted-template-alias.cpp b/clang/test/DebugInfo/CXX/defaulted-template-alias.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/defaulted-template-alias.cpp
rename to clang/test/DebugInfo/CXX/defaulted-template-alias.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-deleted.cpp b/clang/test/DebugInfo/CXX/deleted.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-deleted.cpp
rename to clang/test/DebugInfo/CXX/deleted.cpp
diff --git a/clang/test/CodeGenCXX/dependent-template-alias.cpp b/clang/test/DebugInfo/CXX/dependent-template-alias.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/dependent-template-alias.cpp
rename to clang/test/DebugInfo/CXX/dependent-template-alias.cpp
diff --git a/clang/test/DebugInfo/CXX/dependent-template-type-scope.cpp b/clang/test/DebugInfo/CXX/dependent-template-type-scope.cpp
new file mode 100644
index 0000000000000..25a4d8741b01e
--- /dev/null
+++ b/clang/test/DebugInfo/CXX/dependent-template-type-scope.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -debug-info-kind=standalone  -o - %s | FileCheck %s
+
+template <typename T = int>
+struct Y {
+  typedef int outside;
+  outside o;
+};
+
+Y<> y;
+
+// CHECK: ![[Y:.*]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Y<int>", {{.*}}identifier: "_ZTS1YIiE")
+// CHECK: !DIDerivedType(tag: DW_TAG_typedef, name: "outside", scope: ![[Y]],
diff --git a/clang/test/CodeGenCXX/debug-info-destroy-helper.cpp b/clang/test/DebugInfo/CXX/destroy-helper.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-destroy-helper.cpp
rename to clang/test/DebugInfo/CXX/destroy-helper.cpp
diff --git a/clang/test/CodeGenCXX/destructor-debug-info.cpp b/clang/test/DebugInfo/CXX/destructor.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/destructor-debug-info.cpp
rename to clang/test/DebugInfo/CXX/destructor.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-determinism.cpp b/clang/test/DebugInfo/CXX/determinism.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-determinism.cpp
rename to clang/test/DebugInfo/CXX/determinism.cpp
diff --git a/clang/test/CodeGenCXX/difile_entry.cpp b/clang/test/DebugInfo/CXX/difile_entry.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/difile_entry.cpp
rename to clang/test/DebugInfo/CXX/difile_entry.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-dllimport-base-class.cpp b/clang/test/DebugInfo/CXX/dllimport-base-class.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-dllimport-base-class.cpp
rename to clang/test/DebugInfo/CXX/dllimport-base-class.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp b/clang/test/DebugInfo/CXX/dtor-implicit-args.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-dtor-implicit-args.cpp
rename to clang/test/DebugInfo/CXX/dtor-implicit-args.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-dup-fwd-decl.cpp b/clang/test/DebugInfo/CXX/dup-fwd-decl.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-dup-fwd-decl.cpp
rename to clang/test/DebugInfo/CXX/dup-fwd-decl.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-enum-class.cpp b/clang/test/DebugInfo/CXX/enum-class.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-enum-class.cpp
rename to clang/test/DebugInfo/CXX/enum-class.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-enum-i128.cpp b/clang/test/DebugInfo/CXX/enum-i128.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-enum-i128.cpp
rename to clang/test/DebugInfo/CXX/enum-i128.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-enum-metadata-collision.cpp b/clang/test/DebugInfo/CXX/enum-metadata-collision.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-enum-metadata-collision.cpp
rename to clang/test/DebugInfo/CXX/enum-metadata-collision.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-enum.cpp b/clang/test/DebugInfo/CXX/enum.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-enum.cpp
rename to clang/test/DebugInfo/CXX/enum.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-explicit-cast.cpp b/clang/test/DebugInfo/CXX/explicit-cast.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-explicit-cast.cpp
rename to clang/test/DebugInfo/CXX/explicit-cast.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-explicit-this.cpp b/clang/test/DebugInfo/CXX/explicit-this.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-explicit-this.cpp
rename to clang/test/DebugInfo/CXX/explicit-this.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-export_symbols.cpp b/clang/test/DebugInfo/CXX/export_symbols.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-export_symbols.cpp
rename to clang/test/DebugInfo/CXX/export_symbols.cpp
diff --git a/clang/test/CodeGenCXX/fdebug-info-for-profiling.cpp b/clang/test/DebugInfo/CXX/fdebug-info-for-profiling.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/fdebug-info-for-profiling.cpp
rename to clang/test/DebugInfo/CXX/fdebug-info-for-profiling.cpp
diff --git a/clang/test/CodeGenCXX/field-access-debug-info.cpp b/clang/test/DebugInfo/CXX/field-access.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/field-access-debug-info.cpp
rename to clang/test/DebugInfo/CXX/field-access.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-fn-template.cpp b/clang/test/DebugInfo/CXX/fn-template.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-fn-template.cpp
rename to clang/test/DebugInfo/CXX/fn-template.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-friend.cpp b/clang/test/DebugInfo/CXX/friend.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-friend.cpp
rename to clang/test/DebugInfo/CXX/friend.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-function-context.cpp b/clang/test/DebugInfo/CXX/function-context.cpp
similarity index 95%
rename from clang/test/CodeGenCXX/debug-info-function-context.cpp
rename to clang/test/DebugInfo/CXX/function-context.cpp
index 29c87b6ec0055..da5fa5a79a723 100644
--- a/clang/test/CodeGenCXX/debug-info-function-context.cpp
+++ b/clang/test/DebugInfo/CXX/function-context.cpp
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited -triple x86_64-pc-linux-gnu %s -fdebug-compilation-dir=%S \
-// RUN:     -dwarf-version=5 -main-file-name debug-info-function-context.cpp  -o - | FileCheck %s
+// RUN:     -dwarf-version=5 -main-file-name function-context.cpp  -o - | FileCheck %s
 
 struct C {
   void member_function();
diff --git a/clang/test/CodeGenCXX/debug-info-fwd-ref.cpp b/clang/test/DebugInfo/CXX/fwd-ref.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-fwd-ref.cpp
rename to clang/test/DebugInfo/CXX/fwd-ref.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-fwd-template-param.cpp b/clang/test/DebugInfo/CXX/fwd-template-param.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-fwd-template-param.cpp
rename to clang/test/DebugInfo/CXX/fwd-template-param.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp b/clang/test/DebugInfo/CXX/gline-tables-only-codeview.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
rename to clang/test/DebugInfo/CXX/gline-tables-only-codeview.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-gline-tables-only.cpp b/clang/test/DebugInfo/CXX/gline-tables-only.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-gline-tables-only.cpp
rename to clang/test/DebugInfo/CXX/gline-tables-only.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-global-ctor-dtor.cpp b/clang/test/DebugInfo/CXX/global-ctor-dtor.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-global-ctor-dtor.cpp
rename to clang/test/DebugInfo/CXX/global-ctor-dtor.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-global.cpp b/clang/test/DebugInfo/CXX/global.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-global.cpp
rename to clang/test/DebugInfo/CXX/global.cpp
diff --git a/clang/test/CodeGenCXX/globalinit-loc.cpp b/clang/test/DebugInfo/CXX/globalinit-loc.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/globalinit-loc.cpp
rename to clang/test/DebugInfo/CXX/globalinit-loc.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-globalinit.cpp b/clang/test/DebugInfo/CXX/globalinit.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-globalinit.cpp
rename to clang/test/DebugInfo/CXX/globalinit.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-hotpatch-aarch64.cpp b/clang/test/DebugInfo/CXX/hotpatch-aarch64.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-hotpatch-aarch64.cpp
rename to clang/test/DebugInfo/CXX/hotpatch-aarch64.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-hotpatch-arm.cpp b/clang/test/DebugInfo/CXX/hotpatch-arm.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-hotpatch-arm.cpp
rename to clang/test/DebugInfo/CXX/hotpatch-arm.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-hotpatch.cpp b/clang/test/DebugInfo/CXX/hotpatch.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-hotpatch.cpp
rename to clang/test/DebugInfo/CXX/hotpatch.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-incomplete-types.cpp b/clang/test/DebugInfo/CXX/incomplete-types.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-incomplete-types.cpp
rename to clang/test/DebugInfo/CXX/incomplete-types.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-indirect-field-decl.cpp b/clang/test/DebugInfo/CXX/indirect-field-decl.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-indirect-field-decl.cpp
rename to clang/test/DebugInfo/CXX/indirect-field-decl.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-inheriting-constructor.cpp b/clang/test/DebugInfo/CXX/inheriting-constructor.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-inheriting-constructor.cpp
rename to clang/test/DebugInfo/CXX/inheriting-constructor.cpp
diff --git a/clang/test/CodeGenCXX/inline-dllexport-member.cpp b/clang/test/DebugInfo/CXX/inline-dllexport-member.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/inline-dllexport-member.cpp
rename to clang/test/DebugInfo/CXX/inline-dllexport-member.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-inlined.cpp b/clang/test/DebugInfo/CXX/inlined.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-inlined.cpp
rename to clang/test/DebugInfo/CXX/inlined.cpp
diff --git a/clang/test/CodeGenCXX/debug-lambda-expressions.cpp b/clang/test/DebugInfo/CXX/lambda-expressions.cpp
similarity index 98%
rename from clang/test/CodeGenCXX/debug-lambda-expressions.cpp
rename to clang/test/DebugInfo/CXX/lambda-expressions.cpp
index 324c0926fb9f2..1604b908ab8bf 100644
--- a/clang/test/CodeGenCXX/debug-lambda-expressions.cpp
+++ b/clang/test/DebugInfo/CXX/lambda-expressions.cpp
@@ -21,7 +21,7 @@ int d(int x) { D y[10]; return [x,y] { return y[x].x; }(); }
 // CHECK-SAME:              line: [[VAR_LINE:[0-9]+]]
 // CHECK-SAME:              type: ![[VAR_T:[0-9]+]]
 
-// CHECK: [[FILE:.*]] = !DIFile(filename: "{{.*}}debug-lambda-expressions.cpp",
+// CHECK: [[FILE:.*]] = !DIFile(filename: "{{.*}}lambda-expressions.cpp",
 
 // CVAR:
 // CHECK: !DIGlobalVariable(name: "cvar"
diff --git a/clang/test/CodeGenCXX/debug-lambda-this.cpp b/clang/test/DebugInfo/CXX/lambda-this.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-lambda-this.cpp
rename to clang/test/DebugInfo/CXX/lambda-this.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-lambda.cpp b/clang/test/DebugInfo/CXX/lambda.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-lambda.cpp
rename to clang/test/DebugInfo/CXX/lambda.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-large-constant.cpp b/clang/test/DebugInfo/CXX/large-constant.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-large-constant.cpp
rename to clang/test/DebugInfo/CXX/large-constant.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-limited-ctor.cpp b/clang/test/DebugInfo/CXX/limited-ctor.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-limited-ctor.cpp
rename to clang/test/DebugInfo/CXX/limited-ctor.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-limited.cpp b/clang/test/DebugInfo/CXX/limited.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-limited.cpp
rename to clang/test/DebugInfo/CXX/limited.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-line-if-2.cpp b/clang/test/DebugInfo/CXX/line-if-2.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-line-if-2.cpp
rename to clang/test/DebugInfo/CXX/line-if-2.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-line-if.cpp b/clang/test/DebugInfo/CXX/line-if.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-line-if.cpp
rename to clang/test/DebugInfo/CXX/line-if.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-line.cpp b/clang/test/DebugInfo/CXX/line.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-line.cpp
rename to clang/test/DebugInfo/CXX/line.cpp
diff --git a/clang/test/CodeGenCXX/linetable-cleanup.cpp b/clang/test/DebugInfo/CXX/linetable-cleanup.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/linetable-cleanup.cpp
rename to clang/test/DebugInfo/CXX/linetable-cleanup.cpp
diff --git a/clang/test/CodeGenCXX/linetable-eh.cpp b/clang/test/DebugInfo/CXX/linetable-eh.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/linetable-eh.cpp
rename to clang/test/DebugInfo/CXX/linetable-eh.cpp
diff --git a/clang/test/CodeGenCXX/linetable-fnbegin.cpp b/clang/test/DebugInfo/CXX/linetable-fnbegin.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/linetable-fnbegin.cpp
rename to clang/test/DebugInfo/CXX/linetable-fnbegin.cpp
diff --git a/clang/test/CodeGenCXX/linetable-virtual-variadic.cpp b/clang/test/DebugInfo/CXX/linetable-virtual-variadic.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/linetable-virtual-variadic.cpp
rename to clang/test/DebugInfo/CXX/linetable-virtual-variadic.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-loops.cpp b/clang/test/DebugInfo/CXX/loops.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-loops.cpp
rename to clang/test/DebugInfo/CXX/loops.cpp
diff --git a/clang/test/CodeGenCXX/lpad-linetable.cpp b/clang/test/DebugInfo/CXX/lpad-linetable.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/lpad-linetable.cpp
rename to clang/test/DebugInfo/CXX/lpad-linetable.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-member-call.cpp b/clang/test/DebugInfo/CXX/member-call.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-member-call.cpp
rename to clang/test/DebugInfo/CXX/member-call.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-member.cpp b/clang/test/DebugInfo/CXX/member.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-member.cpp
rename to clang/test/DebugInfo/CXX/member.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-method-nodebug.cpp b/clang/test/DebugInfo/CXX/method-nodebug.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-method-nodebug.cpp
rename to clang/test/DebugInfo/CXX/method-nodebug.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-method-spec.cpp b/clang/test/DebugInfo/CXX/method-spec.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-method-spec.cpp
rename to clang/test/DebugInfo/CXX/method-spec.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-method.cpp b/clang/test/DebugInfo/CXX/method.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-method.cpp
rename to clang/test/DebugInfo/CXX/method.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-method2.cpp b/clang/test/DebugInfo/CXX/method2.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-method2.cpp
rename to clang/test/DebugInfo/CXX/method2.cpp
diff --git a/clang/test/CodeGenCXX/microsoft-abi-member-pointers-debug-info.cpp b/clang/test/DebugInfo/CXX/microsoft-abi-member-pointers.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/microsoft-abi-member-pointers-debug-info.cpp
rename to clang/test/DebugInfo/CXX/microsoft-abi-member-pointers.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ms-abi.cpp b/clang/test/DebugInfo/CXX/ms-abi.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ms-abi.cpp
rename to clang/test/DebugInfo/CXX/ms-abi.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ms-anonymous-tag.cpp b/clang/test/DebugInfo/CXX/ms-anonymous-tag.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ms-anonymous-tag.cpp
rename to clang/test/DebugInfo/CXX/ms-anonymous-tag.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ms-bitfields.cpp b/clang/test/DebugInfo/CXX/ms-bitfields.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ms-bitfields.cpp
rename to clang/test/DebugInfo/CXX/ms-bitfields.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ms-dtor-thunks.cpp b/clang/test/DebugInfo/CXX/ms-dtor-thunks.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ms-dtor-thunks.cpp
rename to clang/test/DebugInfo/CXX/ms-dtor-thunks.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ms-novtable.cpp b/clang/test/DebugInfo/CXX/ms-novtable.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ms-novtable.cpp
rename to clang/test/DebugInfo/CXX/ms-novtable.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ms-ptr-to-member.cpp b/clang/test/DebugInfo/CXX/ms-ptr-to-member.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ms-ptr-to-member.cpp
rename to clang/test/DebugInfo/CXX/ms-ptr-to-member.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ms-vbase.cpp b/clang/test/DebugInfo/CXX/ms-vbase.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ms-vbase.cpp
rename to clang/test/DebugInfo/CXX/ms-vbase.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-namespace.cpp b/clang/test/DebugInfo/CXX/namespace.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-namespace.cpp
rename to clang/test/DebugInfo/CXX/namespace.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-nested-exprs.cpp b/clang/test/DebugInfo/CXX/nested-exprs.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-nested-exprs.cpp
rename to clang/test/DebugInfo/CXX/nested-exprs.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-nodebug.cpp b/clang/test/DebugInfo/CXX/nodebug.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-nodebug.cpp
rename to clang/test/DebugInfo/CXX/nodebug.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-noreturn.cpp b/clang/test/DebugInfo/CXX/noreturn.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-noreturn.cpp
rename to clang/test/DebugInfo/CXX/noreturn.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-nrvo.cpp b/clang/test/DebugInfo/CXX/nrvo.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-nrvo.cpp
rename to clang/test/DebugInfo/CXX/nrvo.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-nullptr.cpp b/clang/test/DebugInfo/CXX/nullptr.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-nullptr.cpp
rename to clang/test/DebugInfo/CXX/nullptr.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-object-pointer.cpp b/clang/test/DebugInfo/CXX/object-pointer.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-object-pointer.cpp
rename to clang/test/DebugInfo/CXX/object-pointer.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-objname.cpp b/clang/test/DebugInfo/CXX/objname.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-objname.cpp
rename to clang/test/DebugInfo/CXX/objname.cpp
diff --git a/clang/test/CodeGenCXX/debug-prefix-map-lambda.cpp b/clang/test/DebugInfo/CXX/prefix-map-lambda.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-prefix-map-lambda.cpp
rename to clang/test/DebugInfo/CXX/prefix-map-lambda.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-programming-language.cpp b/clang/test/DebugInfo/CXX/programming-language.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-programming-language.cpp
rename to clang/test/DebugInfo/CXX/programming-language.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ptr-to-member-function.cpp b/clang/test/DebugInfo/CXX/ptr-to-member-function.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ptr-to-member-function.cpp
rename to clang/test/DebugInfo/CXX/ptr-to-member-function.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-ptr-to-ptr.cpp b/clang/test/DebugInfo/CXX/ptr-to-ptr.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-ptr-to-ptr.cpp
rename to clang/test/DebugInfo/CXX/ptr-to-ptr.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-qualifiers.cpp b/clang/test/DebugInfo/CXX/qualifiers.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-qualifiers.cpp
rename to clang/test/DebugInfo/CXX/qualifiers.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-range-for-var-names.cpp b/clang/test/DebugInfo/CXX/range-for-var-names.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-range-for-var-names.cpp
rename to clang/test/DebugInfo/CXX/range-for-var-names.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-rvalue-ref.cpp b/clang/test/DebugInfo/CXX/rvalue-ref.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-rvalue-ref.cpp
rename to clang/test/DebugInfo/CXX/rvalue-ref.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-scope.cpp b/clang/test/DebugInfo/CXX/scope.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-scope.cpp
rename to clang/test/DebugInfo/CXX/scope.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-scoped-class.cpp b/clang/test/DebugInfo/CXX/scoped-class.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-scoped-class.cpp
rename to clang/test/DebugInfo/CXX/scoped-class.cpp
diff --git a/clang/test/CodeGenCXX/scoped-enums-debug-info.cpp b/clang/test/DebugInfo/CXX/scoped-enums.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/scoped-enums-debug-info.cpp
rename to clang/test/DebugInfo/CXX/scoped-enums.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-simple-template-names.cpp b/clang/test/DebugInfo/CXX/simple-template-names.cpp
similarity index 92%
rename from clang/test/CodeGenCXX/debug-info-simple-template-names.cpp
rename to clang/test/DebugInfo/CXX/simple-template-names.cpp
index 98faa0fc6f0bb..5a5d706e81972 100644
--- a/clang/test/CodeGenCXX/debug-info-simple-template-names.cpp
+++ b/clang/test/DebugInfo/CXX/simple-template-names.cpp
@@ -70,18 +70,18 @@ void f() {
   // anything other than another unnamed class/struct.
   auto Lambda = [] {};
   f1<decltype(Lambda)>();
-  // CHECK: !DISubprogram(name: "f1<(lambda at {{.*}}debug-info-simple-template-names.cpp:[[# @LINE - 2]]:17)>",
+  // CHECK: !DISubprogram(name: "f1<(lambda at {{.*}}simple-template-names.cpp:[[# @LINE - 2]]:17)>",
   f1<t1<t1<decltype(Lambda)>>>();
   // CHECK: !DISubprogram(name: "f1<t1<t1<(lambda at {{.*}}> > >",
   struct {
   } unnamed_struct;
   f1<decltype(unnamed_struct)>();
-  // CHECK: !DISubprogram(name: "f1<(unnamed struct at {{.*}}debug-info-simple-template-names.cpp:[[# @LINE - 3]]:3)>",
+  // CHECK: !DISubprogram(name: "f1<(unnamed struct at {{.*}}simple-template-names.cpp:[[# @LINE - 3]]:3)>",
   f1<void (decltype(unnamed_struct))>();
-  // CHECK: !DISubprogram(name: "f1<void ((unnamed struct at {{.*}}debug-info-simple-template-names.cpp:[[# @LINE - 5]]:3))>",
+  // CHECK: !DISubprogram(name: "f1<void ((unnamed struct at {{.*}}simple-template-names.cpp:[[# @LINE - 5]]:3))>",
   enum {} unnamed_enum;
   f1<decltype(unnamed_enum)>();
-  // CHECK: !DISubprogram(name: "f1<(unnamed enum at {{.*}}debug-info-simple-template-names.cpp:[[# @LINE - 2]]:3)>",
+  // CHECK: !DISubprogram(name: "f1<(unnamed enum at {{.*}}simple-template-names.cpp:[[# @LINE - 2]]:3)>",
 
   // Declarations can't readily be reversed as the value in the DWARF only
   // contains the address of the value - we'd have to do symbol lookup to find
diff --git a/clang/test/CodeGenCXX/standalone-debug-attribute.cpp b/clang/test/DebugInfo/CXX/standalone-debug-attribute.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/standalone-debug-attribute.cpp
rename to clang/test/DebugInfo/CXX/standalone-debug-attribute.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-static-fns.cpp b/clang/test/DebugInfo/CXX/static-fns.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-static-fns.cpp
rename to clang/test/DebugInfo/CXX/static-fns.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-static-member.cpp b/clang/test/DebugInfo/CXX/static-member.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-static-member.cpp
rename to clang/test/DebugInfo/CXX/static-member.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-struct-align.cpp b/clang/test/DebugInfo/CXX/struct-align.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-struct-align.cpp
rename to clang/test/DebugInfo/CXX/struct-align.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-structured-binding-bitfield.cpp b/clang/test/DebugInfo/CXX/structured-binding-bitfield.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-structured-binding-bitfield.cpp
rename to clang/test/DebugInfo/CXX/structured-binding-bitfield.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-structured-binding.cpp b/clang/test/DebugInfo/CXX/structured-binding.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-structured-binding.cpp
rename to clang/test/DebugInfo/CXX/structured-binding.cpp
diff --git a/clang/test/CodeGenCXX/template-alias.cpp b/clang/test/DebugInfo/CXX/template-alias.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/template-alias.cpp
rename to clang/test/DebugInfo/CXX/template-alias.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-align.cpp b/clang/test/DebugInfo/CXX/template-align.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-align.cpp
rename to clang/test/DebugInfo/CXX/template-align.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-array.cpp b/clang/test/DebugInfo/CXX/template-array.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-array.cpp
rename to clang/test/DebugInfo/CXX/template-array.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-deduction-guide.cpp b/clang/test/DebugInfo/CXX/template-deduction-guide.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-deduction-guide.cpp
rename to clang/test/DebugInfo/CXX/template-deduction-guide.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-explicit-specialization.cpp b/clang/test/DebugInfo/CXX/template-explicit-specialization.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-explicit-specialization.cpp
rename to clang/test/DebugInfo/CXX/template-explicit-specialization.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-fwd.cpp b/clang/test/DebugInfo/CXX/template-fwd.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-fwd.cpp
rename to clang/test/DebugInfo/CXX/template-fwd.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-limit.cpp b/clang/test/DebugInfo/CXX/template-limit.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-limit.cpp
rename to clang/test/DebugInfo/CXX/template-limit.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-member.cpp b/clang/test/DebugInfo/CXX/template-member.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-member.cpp
rename to clang/test/DebugInfo/CXX/template-member.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-parameter.cpp b/clang/test/DebugInfo/CXX/template-parameter.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-parameter.cpp
rename to clang/test/DebugInfo/CXX/template-parameter.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-partial-specialization.cpp b/clang/test/DebugInfo/CXX/template-partial-specialization.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-partial-specialization.cpp
rename to clang/test/DebugInfo/CXX/template-partial-specialization.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-quals.cpp b/clang/test/DebugInfo/CXX/template-quals.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-quals.cpp
rename to clang/test/DebugInfo/CXX/template-quals.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template-recursive.cpp b/clang/test/DebugInfo/CXX/template-recursive.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template-recursive.cpp
rename to clang/test/DebugInfo/CXX/template-recursive.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-template.cpp b/clang/test/DebugInfo/CXX/template.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-template.cpp
rename to clang/test/DebugInfo/CXX/template.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-this.cpp b/clang/test/DebugInfo/CXX/this.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-this.cpp
rename to clang/test/DebugInfo/CXX/this.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-thunk-msabi.cpp b/clang/test/DebugInfo/CXX/thunk-msabi.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-thunk-msabi.cpp
rename to clang/test/DebugInfo/CXX/thunk-msabi.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-thunk.cpp b/clang/test/DebugInfo/CXX/thunk.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-thunk.cpp
rename to clang/test/DebugInfo/CXX/thunk.cpp
diff --git a/clang/test/CodeGenCXX/trivial_abi_debuginfo.cpp b/clang/test/DebugInfo/CXX/trivial_abi.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/trivial_abi_debuginfo.cpp
rename to clang/test/DebugInfo/CXX/trivial_abi.cpp
diff --git a/clang/test/CodeGenCXX/ubsan-check-debuglocs.cpp b/clang/test/DebugInfo/CXX/ubsan-check-debuglocs.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/ubsan-check-debuglocs.cpp
rename to clang/test/DebugInfo/CXX/ubsan-check-debuglocs.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-union-template.cpp b/clang/test/DebugInfo/CXX/union-template.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-union-template.cpp
rename to clang/test/DebugInfo/CXX/union-template.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-union.cpp b/clang/test/DebugInfo/CXX/union.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-union.cpp
rename to clang/test/DebugInfo/CXX/union.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-use-after-free.cpp b/clang/test/DebugInfo/CXX/use-after-free.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-use-after-free.cpp
rename to clang/test/DebugInfo/CXX/use-after-free.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-uuid.cpp b/clang/test/DebugInfo/CXX/uuid.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-uuid.cpp
rename to clang/test/DebugInfo/CXX/uuid.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-var-template-partial-spec.cpp b/clang/test/DebugInfo/CXX/var-template-partial-spec.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-var-template-partial-spec.cpp
rename to clang/test/DebugInfo/CXX/var-template-partial-spec.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-var-template-partial.cpp b/clang/test/DebugInfo/CXX/var-template-partial.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-var-template-partial.cpp
rename to clang/test/DebugInfo/CXX/var-template-partial.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-varargs.cpp b/clang/test/DebugInfo/CXX/varargs.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-varargs.cpp
rename to clang/test/DebugInfo/CXX/varargs.cpp
diff --git a/clang/test/CodeGenCXX/variadic-template-alias.cpp b/clang/test/DebugInfo/CXX/variadic-template-alias.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/variadic-template-alias.cpp
rename to clang/test/DebugInfo/CXX/variadic-template-alias.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-verbose-trap.cpp b/clang/test/DebugInfo/CXX/verbose-trap.cpp
similarity index 96%
rename from clang/test/CodeGenCXX/debug-info-verbose-trap.cpp
rename to clang/test/DebugInfo/CXX/verbose-trap.cpp
index f492698ccab83..4a88df934ff7c 100644
--- a/clang/test/CodeGenCXX/debug-info-verbose-trap.cpp
+++ b/clang/test/DebugInfo/CXX/verbose-trap.cpp
@@ -17,7 +17,7 @@
 
 // CHECK: attributes #[[ATTR1]] = { cold {{.*}}}
 
-// CHECK: ![[FILESCOPE:.*]] = !DIFile(filename: "{{.*}}debug-info-verbose-trap.cpp"
+// CHECK: ![[FILESCOPE:.*]] = !DIFile(filename: "{{.*}}verbose-trap.cpp"
 
 char const constCat[] = "category2";
 char const constMsg[] = "hello";
diff --git a/clang/test/CodeGenCXX/debug-info-vla.cpp b/clang/test/DebugInfo/CXX/vla.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-vla.cpp
rename to clang/test/DebugInfo/CXX/vla.cpp
diff --git a/clang/test/CodeGenCXX/vtable-holder-self-reference.cpp b/clang/test/DebugInfo/CXX/vtable-holder-self-reference.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/vtable-holder-self-reference.cpp
rename to clang/test/DebugInfo/CXX/vtable-holder-self-reference.cpp
diff --git a/clang/test/CodeGenCXX/vtable-debug-info-inheritance-diamond.cpp b/clang/test/DebugInfo/CXX/vtable-inheritance-diamond.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/vtable-debug-info-inheritance-diamond.cpp
rename to clang/test/DebugInfo/CXX/vtable-inheritance-diamond.cpp
diff --git a/clang/test/CodeGenCXX/vtable-debug-info-inheritance-multiple.cpp b/clang/test/DebugInfo/CXX/vtable-inheritance-multiple.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/vtable-debug-info-inheritance-multiple.cpp
rename to clang/test/DebugInfo/CXX/vtable-inheritance-multiple.cpp
diff --git a/clang/test/CodeGenCXX/vtable-debug-info-inheritance-simple-main.cpp b/clang/test/DebugInfo/CXX/vtable-inheritance-simple-main.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/vtable-debug-info-inheritance-simple-main.cpp
rename to clang/test/DebugInfo/CXX/vtable-inheritance-simple-main.cpp
diff --git a/clang/test/CodeGenCXX/vtable-debug-info-inheritance-simple.cpp b/clang/test/DebugInfo/CXX/vtable-inheritance-simple.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/vtable-debug-info-inheritance-simple.cpp
rename to clang/test/DebugInfo/CXX/vtable-inheritance-simple.cpp
diff --git a/clang/test/CodeGenCXX/vtable-debug-info-inheritance-virtual.cpp b/clang/test/DebugInfo/CXX/vtable-inheritance-virtual.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/vtable-debug-info-inheritance-virtual.cpp
rename to clang/test/DebugInfo/CXX/vtable-inheritance-virtual.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-vtable-optzn.cpp b/clang/test/DebugInfo/CXX/vtable-optzn.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-vtable-optzn.cpp
rename to clang/test/DebugInfo/CXX/vtable-optzn.cpp
diff --git a/clang/test/CodeGenCXX/vtable-debug-info.cpp b/clang/test/DebugInfo/CXX/vtable.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/vtable-debug-info.cpp
rename to clang/test/DebugInfo/CXX/vtable.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-wchar.cpp b/clang/test/DebugInfo/CXX/wchar.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-wchar.cpp
rename to clang/test/DebugInfo/CXX/wchar.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-windows-dtor.cpp b/clang/test/DebugInfo/CXX/windows-dtor.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-windows-dtor.cpp
rename to clang/test/DebugInfo/CXX/windows-dtor.cpp
diff --git a/clang/test/CodeGenCXX/debug-info-zero-length-arrays.cpp b/clang/test/DebugInfo/CXX/zero-length-arrays.cpp
similarity index 100%
rename from clang/test/CodeGenCXX/debug-info-zero-length-arrays.cpp
rename to clang/test/DebugInfo/CXX/zero-length-arrays.cpp
diff --git a/clang/test/CodeGen/2007-05-11-str-const.c b/clang/test/DebugInfo/Generic/2007-05-11-str-const.c
similarity index 100%
rename from clang/test/CodeGen/2007-05-11-str-const.c
rename to clang/test/DebugInfo/Generic/2007-05-11-str-const.c
diff --git a/clang/test/CodeGen/2009-03-13-dbg.c b/clang/test/DebugInfo/Generic/2009-03-13-dbg.c
similarity index 100%
rename from clang/test/CodeGen/2009-03-13-dbg.c
rename to clang/test/DebugInfo/Generic/2009-03-13-dbg.c
diff --git a/clang/test/CodeGen/2009-04-23-dbg.c b/clang/test/DebugInfo/Generic/2009-04-23-dbg.c
similarity index 100%
rename from clang/test/CodeGen/2009-04-23-dbg.c
rename to clang/test/DebugInfo/Generic/2009-04-23-dbg.c
diff --git a/clang/test/CodeGen/2009-07-31-DbgDeclare.c b/clang/test/DebugInfo/Generic/2009-07-31-DbgDeclare.c
similarity index 100%
rename from clang/test/CodeGen/2009-07-31-DbgDeclare.c
rename to clang/test/DebugInfo/Generic/2009-07-31-DbgDeclare.c
diff --git a/clang/test/CodeGen/2010-01-14-FnType-DebugInfo.c b/clang/test/DebugInfo/Generic/2010-01-14-FnType-DebugInfo.c
similarity index 100%
rename from clang/test/CodeGen/2010-01-14-FnType-DebugInfo.c
rename to clang/test/DebugInfo/Generic/2010-01-14-FnType-DebugInfo.c
diff --git a/clang/test/CodeGen/2010-01-18-Inlined-Debug.c b/clang/test/DebugInfo/Generic/2010-01-18-Inlined-Debug.c
similarity index 100%
rename from clang/test/CodeGen/2010-01-18-Inlined-Debug.c
rename to clang/test/DebugInfo/Generic/2010-01-18-Inlined-Debug.c
diff --git a/clang/test/CodeGen/2010-02-10-PointerName.c b/clang/test/DebugInfo/Generic/2010-02-10-PointerName.c
similarity index 100%
rename from clang/test/CodeGen/2010-02-10-PointerName.c
rename to clang/test/DebugInfo/Generic/2010-02-10-PointerName.c
diff --git a/clang/test/CodeGen/2010-02-15-DbgStaticVar.c b/clang/test/DebugInfo/Generic/2010-02-15-DbgStaticVar.c
similarity index 100%
rename from clang/test/CodeGen/2010-02-15-DbgStaticVar.c
rename to clang/test/DebugInfo/Generic/2010-02-15-DbgStaticVar.c
diff --git a/clang/test/CodeGen/2010-02-16-DbgScopes.c b/clang/test/DebugInfo/Generic/2010-02-16-DbgScopes.c
similarity index 100%
rename from clang/test/CodeGen/2010-02-16-DbgScopes.c
rename to clang/test/DebugInfo/Generic/2010-02-16-DbgScopes.c
diff --git a/clang/test/CodeGen/2010-03-5-LexicalScope.c b/clang/test/DebugInfo/Generic/2010-03-5-LexicalScope.c
similarity index 100%
rename from clang/test/CodeGen/2010-03-5-LexicalScope.c
rename to clang/test/DebugInfo/Generic/2010-03-5-LexicalScope.c
diff --git a/clang/test/CodeGen/2010-07-08-DeclDebugLineNo.c b/clang/test/DebugInfo/Generic/2010-07-08-DeclDebugLineNo.c
similarity index 100%
rename from clang/test/CodeGen/2010-07-08-DeclDebugLineNo.c
rename to clang/test/DebugInfo/Generic/2010-07-08-DeclDebugLineNo.c
diff --git a/clang/test/CodeGen/2010-08-10-DbgConstant.c b/clang/test/DebugInfo/Generic/2010-08-10-DbgConstant.c
similarity index 100%
rename from clang/test/CodeGen/2010-08-10-DbgConstant.c
rename to clang/test/DebugInfo/Generic/2010-08-10-DbgConstant.c
diff --git a/clang/test/CodeGen/debug-info-257-args.c b/clang/test/DebugInfo/Generic/257-args.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-257-args.c
rename to clang/test/DebugInfo/Generic/257-args.c
diff --git a/clang/test/CodeGen/Inputs/debug-info-embed-source.c b/clang/test/DebugInfo/Generic/Inputs/debug-info-embed-source.c
similarity index 100%
rename from clang/test/CodeGen/Inputs/debug-info-embed-source.c
rename to clang/test/DebugInfo/Generic/Inputs/debug-info-embed-source.c
diff --git a/clang/test/CodeGen/Inputs/debug-info-file-checksum-line.cpp b/clang/test/DebugInfo/Generic/Inputs/debug-info-file-checksum-line.cpp
similarity index 100%
rename from clang/test/CodeGen/Inputs/debug-info-file-checksum-line.cpp
rename to clang/test/DebugInfo/Generic/Inputs/debug-info-file-checksum-line.cpp
diff --git a/clang/test/CodeGen/Inputs/debug-info-file-checksum-pre.cpp b/clang/test/DebugInfo/Generic/Inputs/debug-info-file-checksum-pre.cpp
similarity index 100%
rename from clang/test/CodeGen/Inputs/debug-info-file-checksum-pre.cpp
rename to clang/test/DebugInfo/Generic/Inputs/debug-info-file-checksum-pre.cpp
diff --git a/clang/test/CodeGen/Inputs/debug-info-file-checksum.c b/clang/test/DebugInfo/Generic/Inputs/debug-info-file-checksum.c
similarity index 100%
rename from clang/test/CodeGen/Inputs/debug-info-file-checksum.c
rename to clang/test/DebugInfo/Generic/Inputs/debug-info-file-checksum.c
diff --git a/clang/test/CodeGen/Inputs/debug-info-macro.h b/clang/test/DebugInfo/Generic/Inputs/debug-info-macro.h
similarity index 100%
rename from clang/test/CodeGen/Inputs/debug-info-macro.h
rename to clang/test/DebugInfo/Generic/Inputs/debug-info-macro.h
diff --git a/clang/test/CodeGen/Inputs/debug-info-slash.cpp b/clang/test/DebugInfo/Generic/Inputs/debug-info-slash.cpp
similarity index 100%
rename from clang/test/CodeGen/Inputs/debug-info-slash.cpp
rename to clang/test/DebugInfo/Generic/Inputs/debug-info-slash.cpp
diff --git a/clang/test/CodeGen/Inputs/debug-info-slash.h b/clang/test/DebugInfo/Generic/Inputs/debug-info-slash.h
similarity index 100%
rename from clang/test/CodeGen/Inputs/debug-info-slash.h
rename to clang/test/DebugInfo/Generic/Inputs/debug-info-slash.h
diff --git a/clang/test/DebugInfo/Generic/Inputs/stdio.h b/clang/test/DebugInfo/Generic/Inputs/stdio.h
new file mode 100644
index 0000000000000..fc49fd8247038
--- /dev/null
+++ b/clang/test/DebugInfo/Generic/Inputs/stdio.h
@@ -0,0 +1,9 @@
+struct FILE;
+extern int vfprintf(struct FILE *s, const char *format, __builtin_va_list arg);
+extern int vprintf(const char *format, __builtin_va_list arg);
+
+extern __inline __attribute__((gnu_inline,always_inline)) int
+vprintf(const char *x, __builtin_va_list y)
+{
+  return vfprintf (0, 0, y);
+}
diff --git a/clang/test/CodeGen/debug-info-abspath.c b/clang/test/DebugInfo/Generic/abspath.c
similarity index 70%
rename from clang/test/CodeGen/debug-info-abspath.c
rename to clang/test/DebugInfo/Generic/abspath.c
index 193a72c0a66fd..95bed45518f2b 100644
--- a/clang/test/CodeGen/debug-info-abspath.c
+++ b/clang/test/DebugInfo/Generic/abspath.c
@@ -1,9 +1,9 @@
 // RUN: mkdir -p %t/UNIQUEISH_SENTINEL
-// RUN: cp %s %t/UNIQUEISH_SENTINEL/debug-info-abspath.c
+// RUN: cp %s %t/UNIQUEISH_SENTINEL/abspath.c
 
 // RUN: %clang_cc1 -debug-info-kind=limited -triple %itanium_abi_triple \
-// RUN:   -fdebug-compilation-dir=%t/UNIQUEISH_SENTINEL/debug-info-abspath.c \
-// RUN:   %t/UNIQUEISH_SENTINEL/debug-info-abspath.c -emit-llvm -o - \
+// RUN:   -fdebug-compilation-dir=%t/UNIQUEISH_SENTINEL/abspath.c \
+// RUN:   %t/UNIQUEISH_SENTINEL/abspath.c -emit-llvm -o - \
 // RUN:   | FileCheck %s
 
 // RUN: cp %s %t.c
@@ -18,8 +18,8 @@ void foo(void) {}
 
 // CHECK: = distinct !DISubprogram({{.*}}file: ![[SPFILE:[0-9]+]]
 // CHECK: ![[SPFILE]] = !DIFile(filename: "{{.*}}UNIQUEISH_SENTINEL
-// CHECK-SAME:                  debug-info-abspath.c"
+// CHECK-SAME:                  abspath.c"
 // CHECK-NOT:                   directory: "{{.*}}UNIQUEISH_SENTINEL
 
 // INTREE: = distinct !DISubprogram({{.*}}![[SPFILE:[0-9]+]]
-// INTREE: DIFile({{.*}}directory: "{{.+}}CodeGen{{.*}}")
+// INTREE: DIFile({{.*}}directory: "{{.+}}Generic{{.*}}")
diff --git a/clang/test/CodeGen/debug-info-alias-pointer.c b/clang/test/DebugInfo/Generic/alias-pointer.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-alias-pointer.c
rename to clang/test/DebugInfo/Generic/alias-pointer.c
diff --git a/clang/test/CodeGen/debug-info-alias.c b/clang/test/DebugInfo/Generic/alias.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-alias.c
rename to clang/test/DebugInfo/Generic/alias.c
diff --git a/clang/test/CodeGen/amdgpu-barrier-type-debug-info.c b/clang/test/DebugInfo/Generic/amdgpu-barrier-type-debug-info.c
similarity index 100%
rename from clang/test/CodeGen/amdgpu-barrier-type-debug-info.c
rename to clang/test/DebugInfo/Generic/amdgpu-barrier-type-debug-info.c
diff --git a/clang/test/CodeGen/amdgpu-buffer-rsrc-type-debug-info.c b/clang/test/DebugInfo/Generic/amdgpu-buffer-rsrc-type-debug-info.c
similarity index 100%
rename from clang/test/CodeGen/amdgpu-buffer-rsrc-type-debug-info.c
rename to clang/test/DebugInfo/Generic/amdgpu-buffer-rsrc-type-debug-info.c
diff --git a/clang/test/CodeGen/debug-info-args.c b/clang/test/DebugInfo/Generic/args.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-args.c
rename to clang/test/DebugInfo/Generic/args.c
diff --git a/clang/test/CodeGen/artificial.c b/clang/test/DebugInfo/Generic/artificial.c
similarity index 100%
rename from clang/test/CodeGen/artificial.c
rename to clang/test/DebugInfo/Generic/artificial.c
diff --git a/clang/test/CodeGen/debug-info-atomic.c b/clang/test/DebugInfo/Generic/atomic.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-atomic.c
rename to clang/test/DebugInfo/Generic/atomic.c
diff --git a/clang/test/CodeGen/attr-btf_type_tag-func-ptr.c b/clang/test/DebugInfo/Generic/attr-btf_type_tag-func-ptr.c
similarity index 100%
rename from clang/test/CodeGen/attr-btf_type_tag-func-ptr.c
rename to clang/test/DebugInfo/Generic/attr-btf_type_tag-func-ptr.c
diff --git a/clang/test/CodeGen/attr-btf_type_tag-similar-type.c b/clang/test/DebugInfo/Generic/attr-btf_type_tag-similar-type.c
similarity index 100%
rename from clang/test/CodeGen/attr-btf_type_tag-similar-type.c
rename to clang/test/DebugInfo/Generic/attr-btf_type_tag-similar-type.c
diff --git a/clang/test/CodeGen/attr-btf_type_tag-typedef-field.c b/clang/test/DebugInfo/Generic/attr-btf_type_tag-typedef-field.c
similarity index 100%
rename from clang/test/CodeGen/attr-btf_type_tag-typedef-field.c
rename to clang/test/DebugInfo/Generic/attr-btf_type_tag-typedef-field.c
diff --git a/clang/test/CodeGen/attr-btf_type_tag-var.c b/clang/test/DebugInfo/Generic/attr-btf_type_tag-var.c
similarity index 100%
rename from clang/test/CodeGen/attr-btf_type_tag-var.c
rename to clang/test/DebugInfo/Generic/attr-btf_type_tag-var.c
diff --git a/clang/test/CodeGen/attr-counted-by-debug-info.c b/clang/test/DebugInfo/Generic/attr-counted-by-debug-info.c
similarity index 100%
rename from clang/test/CodeGen/attr-counted-by-debug-info.c
rename to clang/test/DebugInfo/Generic/attr-counted-by-debug-info.c
diff --git a/clang/test/CodeGen/attr-nodebug.c b/clang/test/DebugInfo/Generic/attr-nodebug.c
similarity index 100%
rename from clang/test/CodeGen/attr-nodebug.c
rename to clang/test/DebugInfo/Generic/attr-nodebug.c
diff --git a/clang/test/CodeGen/attr-nodebug2.c b/clang/test/DebugInfo/Generic/attr-nodebug2.c
similarity index 100%
rename from clang/test/CodeGen/attr-nodebug2.c
rename to clang/test/DebugInfo/Generic/attr-nodebug2.c
diff --git a/clang/test/CodeGen/debug-info-attributed-stmt.c b/clang/test/DebugInfo/Generic/attributed-stmt.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-attributed-stmt.c
rename to clang/test/DebugInfo/Generic/attributed-stmt.c
diff --git a/clang/test/CodeGen/debug-info-bitfield-0-struct.c b/clang/test/DebugInfo/Generic/bitfield-0-struct.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-bitfield-0-struct.c
rename to clang/test/DebugInfo/Generic/bitfield-0-struct.c
diff --git a/clang/test/CodeGen/debug-info-block-decl.c b/clang/test/DebugInfo/Generic/block-decl.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-block-decl.c
rename to clang/test/DebugInfo/Generic/block-decl.c
diff --git a/clang/test/CodeGen/debug-info-block-expr.c b/clang/test/DebugInfo/Generic/block-expr.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-block-expr.c
rename to clang/test/DebugInfo/Generic/block-expr.c
diff --git a/clang/test/CodeGen/debug-info-block-out-return.c b/clang/test/DebugInfo/Generic/block-out-return.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-block-out-return.c
rename to clang/test/DebugInfo/Generic/block-out-return.c
diff --git a/clang/test/CodeGen/debug-info-block-vars.c b/clang/test/DebugInfo/Generic/block-vars.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-block-vars.c
rename to clang/test/DebugInfo/Generic/block-vars.c
diff --git a/clang/test/CodeGen/debug-info-block.c b/clang/test/DebugInfo/Generic/block.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-block.c
rename to clang/test/DebugInfo/Generic/block.c
diff --git a/clang/test/CodeGen/bounds-checking-debuginfo.c b/clang/test/DebugInfo/Generic/bounds-checking-debuginfo.c
similarity index 100%
rename from clang/test/CodeGen/bounds-checking-debuginfo.c
rename to clang/test/DebugInfo/Generic/bounds-checking-debuginfo.c
diff --git a/clang/test/CodeGen/bpf-debug-info-unref.c b/clang/test/DebugInfo/Generic/bpf-debug-info-unref.c
similarity index 100%
rename from clang/test/CodeGen/bpf-debug-info-unref.c
rename to clang/test/DebugInfo/Generic/bpf-debug-info-unref.c
diff --git a/clang/test/CodeGen/builtin-preserve-access-index-array.c b/clang/test/DebugInfo/Generic/builtin-preserve-access-index-array.c
similarity index 100%
rename from clang/test/CodeGen/builtin-preserve-access-index-array.c
rename to clang/test/DebugInfo/Generic/builtin-preserve-access-index-array.c
diff --git a/clang/test/CodeGen/debug-info-cc.c b/clang/test/DebugInfo/Generic/cc.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-cc.c
rename to clang/test/DebugInfo/Generic/cc.c
diff --git a/clang/test/CodeGen/cfi-check-fail-debuginfo.c b/clang/test/DebugInfo/Generic/cfi-check-fail-debuginfo.c
similarity index 100%
rename from clang/test/CodeGen/cfi-check-fail-debuginfo.c
rename to clang/test/DebugInfo/Generic/cfi-check-fail-debuginfo.c
diff --git a/clang/test/CodeGen/cfi-icall-generalize-debuginfo.c b/clang/test/DebugInfo/Generic/cfi-icall-generalize-debuginfo.c
similarity index 100%
rename from clang/test/CodeGen/cfi-icall-generalize-debuginfo.c
rename to clang/test/DebugInfo/Generic/cfi-icall-generalize-debuginfo.c
diff --git a/clang/test/CodeGen/cfi-icall-normalize2-debuginfo.c b/clang/test/DebugInfo/Generic/cfi-icall-normalize2-debuginfo.c
similarity index 100%
rename from clang/test/CodeGen/cfi-icall-normalize2-debuginfo.c
rename to clang/test/DebugInfo/Generic/cfi-icall-normalize2-debuginfo.c
diff --git a/clang/test/CodeGen/debug-info-codeview-buildinfo.c b/clang/test/DebugInfo/Generic/codeview-buildinfo.c
similarity index 97%
rename from clang/test/CodeGen/debug-info-codeview-buildinfo.c
rename to clang/test/DebugInfo/Generic/codeview-buildinfo.c
index 98d249be8c883..ee6f772944d84 100644
--- a/clang/test/CodeGen/debug-info-codeview-buildinfo.c
+++ b/clang/test/DebugInfo/Generic/codeview-buildinfo.c
@@ -17,7 +17,7 @@ int main(void) { return 42; }
 // CHECK:                       Types (.debug$T)
 // CHECK: ============================================================
 // CHECK: 0x[[PWD:.+]] | LF_STRING_ID [size = {{.+}}] ID: <no type>, String: [[PWDVAL:.+]]
-// CHECK: 0x[[FILEPATH:.+]] | LF_STRING_ID [size = {{.+}}] ID: <no type>, String: [[FILEPATHVAL:.+[\\/]debug-info-codeview-buildinfo.c]]
+// CHECK: 0x[[FILEPATH:.+]] | LF_STRING_ID [size = {{.+}}] ID: <no type>, String: [[FILEPATHVAL:.+[\\/]codeview-buildinfo.c]]
 // CHECK: 0x[[ZIPDB:.+]] | LF_STRING_ID [size = {{.+}}] ID: <no type>, String:
 // CHECK: 0x[[TOOL:.+]] | LF_STRING_ID [size = {{.+}}] ID: <no type>, String: [[TOOLVAL:.+[\\/][clang|llvm].*]]
 // CHECK: 0x[[CMDLINE:.+]] | LF_STRING_ID [size = {{.+}}] ID: <no type>, String: "-cc1
diff --git a/clang/test/CodeGen/debug-info-codeview-heapallocsite.c b/clang/test/DebugInfo/Generic/codeview-heapallocsite.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-codeview-heapallocsite.c
rename to clang/test/DebugInfo/Generic/codeview-heapallocsite.c
diff --git a/clang/test/CodeGen/debug-info-codeview-unnamed.c b/clang/test/DebugInfo/Generic/codeview-unnamed.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-codeview-unnamed.c
rename to clang/test/DebugInfo/Generic/codeview-unnamed.c
diff --git a/clang/test/CodeGen/debug-info-compilation-dir.c b/clang/test/DebugInfo/Generic/compilation-dir.c
similarity index 79%
rename from clang/test/CodeGen/debug-info-compilation-dir.c
rename to clang/test/DebugInfo/Generic/compilation-dir.c
index 5f5542c525495..d2c19f298ec72 100644
--- a/clang/test/CodeGen/debug-info-compilation-dir.c
+++ b/clang/test/DebugInfo/Generic/compilation-dir.c
@@ -5,12 +5,12 @@
 // CHECK-NONSENSE: nonsense
 
 // RUN: %clang_cc1 -emit-llvm -debug-info-kind=limited %s -o - | FileCheck -check-prefix=CHECK-DIR %s
-// CHECK-DIR: CodeGen
+// CHECK-DIR: Generic
 
 /// Test path remapping.
 // RUN: %clang_cc1 -fdebug-compilation-dir=%S -main-file-name %s -emit-llvm -debug-info-kind=limited %s -o - | FileCheck -check-prefix=CHECK-ABS %s
-// CHECK-ABS: DIFile(filename: "{{.*}}debug-info-compilation-dir.c", directory: "{{.*}}CodeGen")
+// CHECK-ABS: DIFile(filename: "{{.*}}compilation-dir.c", directory: "{{.*}}Generic")
 
 // RUN: %clang_cc1 -main-file-name %s -emit-llvm -debug-info-kind=limited %s -o - | FileCheck -check-prefix=CHECK-NOMAP %s
-// CHECK-NOMAP: DIFile(filename: "{{.*}}debug-info-compilation-dir.c", directory: "")
+// CHECK-NOMAP: DIFile(filename: "{{.*}}compilation-dir.c", directory: "")
 
diff --git a/clang/test/CodeGen/debug-info-crash.c b/clang/test/DebugInfo/Generic/crash.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-crash.c
rename to clang/test/DebugInfo/Generic/crash.c
diff --git a/clang/test/CodeGen/dbg-const-int128.c b/clang/test/DebugInfo/Generic/dbg-const-int128.c
similarity index 100%
rename from clang/test/CodeGen/dbg-const-int128.c
rename to clang/test/DebugInfo/Generic/dbg-const-int128.c
diff --git a/clang/test/CodeGen/dbg-info-all-calls-described.c b/clang/test/DebugInfo/Generic/dbg-info-all-calls-described.c
similarity index 100%
rename from clang/test/CodeGen/dbg-info-all-calls-described.c
rename to clang/test/DebugInfo/Generic/dbg-info-all-calls-described.c
diff --git a/clang/test/CodeGen/debug-info.c b/clang/test/DebugInfo/Generic/debug-info.c
similarity index 100%
rename from clang/test/CodeGen/debug-info.c
rename to clang/test/DebugInfo/Generic/debug-info.c
diff --git a/clang/test/CodeGen/debug-label-inline.c b/clang/test/DebugInfo/Generic/debug-label-inline.c
similarity index 100%
rename from clang/test/CodeGen/debug-label-inline.c
rename to clang/test/DebugInfo/Generic/debug-label-inline.c
diff --git a/clang/test/CodeGen/debug-line-1.c b/clang/test/DebugInfo/Generic/debug-line-1.c
similarity index 100%
rename from clang/test/CodeGen/debug-line-1.c
rename to clang/test/DebugInfo/Generic/debug-line-1.c
diff --git a/clang/test/CodeGen/debug-prefix-map.c b/clang/test/DebugInfo/Generic/debug-prefix-map.c
similarity index 100%
rename from clang/test/CodeGen/debug-prefix-map.c
rename to clang/test/DebugInfo/Generic/debug-prefix-map.c
diff --git a/clang/test/CodeGen/debug-prefix-map.cpp b/clang/test/DebugInfo/Generic/debug-prefix-map.cpp
similarity index 100%
rename from clang/test/CodeGen/debug-prefix-map.cpp
rename to clang/test/DebugInfo/Generic/debug-prefix-map.cpp
diff --git a/clang/test/CodeGen/debug-info-embed-source.c b/clang/test/DebugInfo/Generic/embed-source.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-embed-source.c
rename to clang/test/DebugInfo/Generic/embed-source.c
diff --git a/clang/test/CodeGen/debug-info-enum-case-val.c b/clang/test/DebugInfo/Generic/enum-case-val.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-enum-case-val.c
rename to clang/test/DebugInfo/Generic/enum-case-val.c
diff --git a/clang/test/CodeGen/debug-info-enum-extensibility.c b/clang/test/DebugInfo/Generic/enum-extensibility.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-enum-extensibility.c
rename to clang/test/DebugInfo/Generic/enum-extensibility.c
diff --git a/clang/test/CodeGen/debug-info-enum.c b/clang/test/DebugInfo/Generic/enum.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-enum.c
rename to clang/test/DebugInfo/Generic/enum.c
diff --git a/clang/test/CodeGen/debug-info-enum.cpp b/clang/test/DebugInfo/Generic/enum.cpp
similarity index 100%
rename from clang/test/CodeGen/debug-info-enum.cpp
rename to clang/test/DebugInfo/Generic/enum.cpp
diff --git a/clang/test/CodeGen/enum2.c b/clang/test/DebugInfo/Generic/enum2.c
similarity index 100%
rename from clang/test/CodeGen/enum2.c
rename to clang/test/DebugInfo/Generic/enum2.c
diff --git a/clang/test/CodeGen/debug-info-extern-basic.c b/clang/test/DebugInfo/Generic/extern-basic.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-extern-basic.c
rename to clang/test/DebugInfo/Generic/extern-basic.c
diff --git a/clang/test/CodeGen/debug-info-extern-basic.cpp b/clang/test/DebugInfo/Generic/extern-basic.cpp
similarity index 100%
rename from clang/test/CodeGen/debug-info-extern-basic.cpp
rename to clang/test/DebugInfo/Generic/extern-basic.cpp
diff --git a/clang/test/CodeGen/debug-info-extern-call.c b/clang/test/DebugInfo/Generic/extern-call.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-extern-call.c
rename to clang/test/DebugInfo/Generic/extern-call.c
diff --git a/clang/test/CodeGen/debug-info-extern-callback.c b/clang/test/DebugInfo/Generic/extern-callback.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-extern-callback.c
rename to clang/test/DebugInfo/Generic/extern-callback.c
diff --git a/clang/test/CodeGen/debug-info-extern-duplicate.c b/clang/test/DebugInfo/Generic/extern-duplicate.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-extern-duplicate.c
rename to clang/test/DebugInfo/Generic/extern-duplicate.c
diff --git a/clang/test/CodeGen/debug-info-extern-multi.c b/clang/test/DebugInfo/Generic/extern-multi.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-extern-multi.c
rename to clang/test/DebugInfo/Generic/extern-multi.c
diff --git a/clang/test/CodeGen/debug-info-extern-unused.c b/clang/test/DebugInfo/Generic/extern-unused.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-extern-unused.c
rename to clang/test/DebugInfo/Generic/extern-unused.c
diff --git a/clang/test/CodeGen/fake-use-return-line.c b/clang/test/DebugInfo/Generic/fake-use-return-line.c
similarity index 100%
rename from clang/test/CodeGen/fake-use-return-line.c
rename to clang/test/DebugInfo/Generic/fake-use-return-line.c
diff --git a/clang/test/CodeGen/debug-info-file-change.c b/clang/test/DebugInfo/Generic/file-change.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-file-change.c
rename to clang/test/DebugInfo/Generic/file-change.c
diff --git a/clang/test/CodeGen/debug-info-file-checksum.c b/clang/test/DebugInfo/Generic/file-checksum.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-file-checksum.c
rename to clang/test/DebugInfo/Generic/file-checksum.c
diff --git a/clang/test/CodeGen/debug-info-gline-tables-only.c b/clang/test/DebugInfo/Generic/gline-tables-only.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-gline-tables-only.c
rename to clang/test/DebugInfo/Generic/gline-tables-only.c
diff --git a/clang/test/CodeGen/debug-info-gline-tables-only2.c b/clang/test/DebugInfo/Generic/gline-tables-only2.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-gline-tables-only2.c
rename to clang/test/DebugInfo/Generic/gline-tables-only2.c
diff --git a/clang/test/CodeGen/global-blocks-lines.c b/clang/test/DebugInfo/Generic/global-blocks-lines.c
similarity index 100%
rename from clang/test/CodeGen/global-blocks-lines.c
rename to clang/test/DebugInfo/Generic/global-blocks-lines.c
diff --git a/clang/test/CodeGen/debug-info-global-constant.c b/clang/test/DebugInfo/Generic/global-constant.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-global-constant.c
rename to clang/test/DebugInfo/Generic/global-constant.c
diff --git a/clang/test/CodeGen/debug-info-imported-entity.cpp b/clang/test/DebugInfo/Generic/imported-entity.cpp
similarity index 87%
rename from clang/test/CodeGen/debug-info-imported-entity.cpp
rename to clang/test/DebugInfo/Generic/imported-entity.cpp
index 398536e581aea..f4763c14bbc7b 100644
--- a/clang/test/CodeGen/debug-info-imported-entity.cpp
+++ b/clang/test/DebugInfo/Generic/imported-entity.cpp
@@ -7,4 +7,4 @@ using std::A; using ::A;
 // CHECK: [[Imports]] = !{[[ImportedEntity:![0-9]+]]}
 // CHECK: [[ImportedEntity]] = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: [[CompileUnit]], entity: [[STDA:![0-9]+]], file: [[FILE:![0-9]+]], line: 4)
 // CHECK: [[STDA]] = !DICompositeType(tag: DW_TAG_class_type, name: "A",
-// CHECK: [[FILE]] = !DIFile(filename: {{.*}}debug-info-imported-entity.cpp
+// CHECK: [[FILE]] = !DIFile(filename: {{.*}}imported-entity.cpp
diff --git a/clang/test/CodeGen/debug-info-inline-for.c b/clang/test/DebugInfo/Generic/inline-for.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-inline-for.c
rename to clang/test/DebugInfo/Generic/inline-for.c
diff --git a/clang/test/CodeGen/debug-label.c b/clang/test/DebugInfo/Generic/label.c
similarity index 100%
rename from clang/test/CodeGen/debug-label.c
rename to clang/test/DebugInfo/Generic/label.c
diff --git a/clang/test/CodeGen/lifetime-debuginfo-1.c b/clang/test/DebugInfo/Generic/lifetime-debuginfo-1.c
similarity index 100%
rename from clang/test/CodeGen/lifetime-debuginfo-1.c
rename to clang/test/DebugInfo/Generic/lifetime-debuginfo-1.c
diff --git a/clang/test/CodeGen/lifetime-debuginfo-2.c b/clang/test/DebugInfo/Generic/lifetime-debuginfo-2.c
similarity index 100%
rename from clang/test/CodeGen/lifetime-debuginfo-2.c
rename to clang/test/DebugInfo/Generic/lifetime-debuginfo-2.c
diff --git a/clang/test/CodeGen/debug-info-limited.c b/clang/test/DebugInfo/Generic/limited.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-limited.c
rename to clang/test/DebugInfo/Generic/limited.c
diff --git a/clang/test/CodeGen/debug-info-line.c b/clang/test/DebugInfo/Generic/line.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-line.c
rename to clang/test/DebugInfo/Generic/line.c
diff --git a/clang/test/CodeGen/debug-info-line2.c b/clang/test/DebugInfo/Generic/line2.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-line2.c
rename to clang/test/DebugInfo/Generic/line2.c
diff --git a/clang/test/CodeGen/debug-info-line3.c b/clang/test/DebugInfo/Generic/line3.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-line3.c
rename to clang/test/DebugInfo/Generic/line3.c
diff --git a/clang/test/CodeGen/debug-info-line4.c b/clang/test/DebugInfo/Generic/line4.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-line4.c
rename to clang/test/DebugInfo/Generic/line4.c
diff --git a/clang/test/CodeGen/lineno-dbginfo.c b/clang/test/DebugInfo/Generic/lineno-dbginfo.c
similarity index 100%
rename from clang/test/CodeGen/lineno-dbginfo.c
rename to clang/test/DebugInfo/Generic/lineno-dbginfo.c
diff --git a/clang/test/CodeGen/linetable-endscope.c b/clang/test/DebugInfo/Generic/linetable-endscope.c
similarity index 100%
rename from clang/test/CodeGen/linetable-endscope.c
rename to clang/test/DebugInfo/Generic/linetable-endscope.c
diff --git a/clang/test/CodeGen/debug-info-lto.c b/clang/test/DebugInfo/Generic/lto.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-lto.c
rename to clang/test/DebugInfo/Generic/lto.c
diff --git a/clang/test/CodeGen/debug-info-macro.c b/clang/test/DebugInfo/Generic/macro.c
similarity index 98%
rename from clang/test/CodeGen/debug-info-macro.c
rename to clang/test/DebugInfo/Generic/macro.c
index 23fd67515e845..de8d1ab915d7e 100644
--- a/clang/test/CodeGen/debug-info-macro.c
+++ b/clang/test/DebugInfo/Generic/macro.c
@@ -30,7 +30,7 @@
 // PCH:    [[Macros]] = !{[[MainMacroFile:![0-9]+]], [[DefineC1:![0-9]+]], [[DefineA:![0-9]+]], [[UndefC1:![0-9]+]]}
 
 // CHECK:  [[MainMacroFile]] = !DIMacroFile(file: [[MainFile:![0-9]+]], nodes: [[N1:![0-9]+]])
-// CHECK:  [[MainFile]] = !DIFile(filename: "{{.*}}debug-info-macro.c"
+// CHECK:  [[MainFile]] = !DIFile(filename: "{{.*}}macro.c"
 // CHECK:  [[N1]] = !{[[CommandLineInclude:![0-9]+]], [[DefineD1:![0-9]+]], [[FileInclude1:![0-9]+]], [[UndefD1:![0-9]+]], [[DefineD2:![0-9]+]], [[FileInclude2:![0-9]+]], [[UndefD2:![0-9]+]]}
 
 // CHECK:  [[CommandLineInclude]] = !DIMacroFile(file: [[HeaderFile:![0-9]+]], nodes: [[N2:![0-9]+]])
diff --git a/clang/test/CodeGen/debug-info-matrix-types.c b/clang/test/DebugInfo/Generic/matrix-types.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-matrix-types.c
rename to clang/test/DebugInfo/Generic/matrix-types.c
diff --git a/clang/test/CodeGen/debug-info-member.c b/clang/test/DebugInfo/Generic/member.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-member.c
rename to clang/test/DebugInfo/Generic/member.c
diff --git a/clang/test/CodeGen/mips-debug-info-bitfield.c b/clang/test/DebugInfo/Generic/mips-debug-info-bitfield.c
similarity index 100%
rename from clang/test/CodeGen/mips-debug-info-bitfield.c
rename to clang/test/DebugInfo/Generic/mips-debug-info-bitfield.c
diff --git a/clang/test/CodeGen/debug-info-names.c b/clang/test/DebugInfo/Generic/names.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-names.c
rename to clang/test/DebugInfo/Generic/names.c
diff --git a/clang/test/CodeGen/debug-info-no-inline-line-tables.c b/clang/test/DebugInfo/Generic/no-inline-line-tables.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-no-inline-line-tables.c
rename to clang/test/DebugInfo/Generic/no-inline-line-tables.c
diff --git a/clang/test/CodeGen/nodebug-attr.c b/clang/test/DebugInfo/Generic/nodebug-attr.c
similarity index 100%
rename from clang/test/CodeGen/nodebug-attr.c
rename to clang/test/DebugInfo/Generic/nodebug-attr.c
diff --git a/clang/test/CodeGen/null-sanitizer-debug-info-regression.cpp b/clang/test/DebugInfo/Generic/null-sanitizer-debug-info-regression.cpp
similarity index 100%
rename from clang/test/CodeGen/null-sanitizer-debug-info-regression.cpp
rename to clang/test/DebugInfo/Generic/null-sanitizer-debug-info-regression.cpp
diff --git a/clang/test/CodeGen/debug-nvptx.c b/clang/test/DebugInfo/Generic/nvptx.c
similarity index 100%
rename from clang/test/CodeGen/debug-nvptx.c
rename to clang/test/DebugInfo/Generic/nvptx.c
diff --git a/clang/test/CodeGen/openmp-prefix-map.c b/clang/test/DebugInfo/Generic/openmp-prefix-map.c
similarity index 100%
rename from clang/test/CodeGen/openmp-prefix-map.c
rename to clang/test/DebugInfo/Generic/openmp-prefix-map.c
diff --git a/clang/test/CodeGen/debug-info-oslog.c b/clang/test/DebugInfo/Generic/oslog.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-oslog.c
rename to clang/test/DebugInfo/Generic/oslog.c
diff --git a/clang/test/CodeGen/overloadable-debug.c b/clang/test/DebugInfo/Generic/overloadable-debug.c
similarity index 100%
rename from clang/test/CodeGen/overloadable-debug.c
rename to clang/test/DebugInfo/Generic/overloadable-debug.c
diff --git a/clang/test/CodeGen/debug-info-packed-struct.c b/clang/test/DebugInfo/Generic/packed-struct.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-packed-struct.c
rename to clang/test/DebugInfo/Generic/packed-struct.c
diff --git a/clang/test/CodeGen/pr52782-stdcall-func-decl.cpp b/clang/test/DebugInfo/Generic/pr52782-stdcall-func-decl.cpp
similarity index 100%
rename from clang/test/CodeGen/pr52782-stdcall-func-decl.cpp
rename to clang/test/DebugInfo/Generic/pr52782-stdcall-func-decl.cpp
diff --git a/clang/test/CodeGen/debug-info-preferred-type.cpp b/clang/test/DebugInfo/Generic/preferred-type.cpp
similarity index 100%
rename from clang/test/CodeGen/debug-info-preferred-type.cpp
rename to clang/test/DebugInfo/Generic/preferred-type.cpp
diff --git a/clang/test/CodeGen/preferred_name-chain.cpp b/clang/test/DebugInfo/Generic/preferred_name-chain.cpp
similarity index 100%
rename from clang/test/CodeGen/preferred_name-chain.cpp
rename to clang/test/DebugInfo/Generic/preferred_name-chain.cpp
diff --git a/clang/test/CodeGen/preferred_name.cpp b/clang/test/DebugInfo/Generic/preferred_name.cpp
similarity index 100%
rename from clang/test/CodeGen/preferred_name.cpp
rename to clang/test/DebugInfo/Generic/preferred_name.cpp
diff --git a/clang/test/CodeGen/debug-info-preprocessed-file.i b/clang/test/DebugInfo/Generic/preprocessed-file.i
similarity index 100%
rename from clang/test/CodeGen/debug-info-preprocessed-file.i
rename to clang/test/DebugInfo/Generic/preprocessed-file.i
diff --git a/clang/test/CodeGen/debug-info-programming-language.c b/clang/test/DebugInfo/Generic/programming-language.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-programming-language.c
rename to clang/test/DebugInfo/Generic/programming-language.c
diff --git a/clang/test/CodeGen/debug-info-pseudo-probe.cpp b/clang/test/DebugInfo/Generic/pseudo-probe.cpp
similarity index 100%
rename from clang/test/CodeGen/debug-info-pseudo-probe.cpp
rename to clang/test/DebugInfo/Generic/pseudo-probe.cpp
diff --git a/clang/test/CodeGen/ptrauth-debuginfo.c b/clang/test/DebugInfo/Generic/ptrauth-debuginfo.c
similarity index 100%
rename from clang/test/CodeGen/ptrauth-debuginfo.c
rename to clang/test/DebugInfo/Generic/ptrauth-debuginfo.c
diff --git a/clang/test/CodeGen/debug-info-ranges-base-address.c b/clang/test/DebugInfo/Generic/ranges-base-address.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-ranges-base-address.c
rename to clang/test/DebugInfo/Generic/ranges-base-address.c
diff --git a/clang/test/CodeGen/debug-info-same-line.c b/clang/test/DebugInfo/Generic/same-line.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-same-line.c
rename to clang/test/DebugInfo/Generic/same-line.c
diff --git a/clang/test/CodeGen/debug-info-scope-file.c b/clang/test/DebugInfo/Generic/scope-file.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-scope-file.c
rename to clang/test/DebugInfo/Generic/scope-file.c
diff --git a/clang/test/CodeGen/debug-info-scope.c b/clang/test/DebugInfo/Generic/scope.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-scope.c
rename to clang/test/DebugInfo/Generic/scope.c
diff --git a/clang/test/CodeGen/debug-info-slash.c b/clang/test/DebugInfo/Generic/slash.c
similarity index 70%
rename from clang/test/CodeGen/debug-info-slash.c
rename to clang/test/DebugInfo/Generic/slash.c
index 56e51c09e5b0e..02187d077d45b 100644
--- a/clang/test/CodeGen/debug-info-slash.c
+++ b/clang/test/DebugInfo/Generic/slash.c
@@ -2,5 +2,5 @@
 // RUN: %clang -target x86_64-linux-gnu  -ffile-reproducible -emit-llvm -S -g %s -o - | FileCheck --check-prefix=LINUX %s
 int main() { return 0; }
 
-// WIN:   !DIFile(filename: "{{.*}}\\debug-info-slash.c"
-// LINUX: !DIFile(filename: "{{.*}}/debug-info-slash.c"
+// WIN:   !DIFile(filename: "{{.*}}\\slash.c"
+// LINUX: !DIFile(filename: "{{.*}}/slash.c"
diff --git a/clang/test/CodeGen/debug-info-slash.test b/clang/test/DebugInfo/Generic/slash.test
similarity index 100%
rename from clang/test/CodeGen/debug-info-slash.test
rename to clang/test/DebugInfo/Generic/slash.test
diff --git a/clang/test/CodeGen/split-debug-filename.c b/clang/test/DebugInfo/Generic/split-debug-filename.c
similarity index 100%
rename from clang/test/CodeGen/split-debug-filename.c
rename to clang/test/DebugInfo/Generic/split-debug-filename.c
diff --git a/clang/test/CodeGen/split-debug-inlining.c b/clang/test/DebugInfo/Generic/split-debug-inlining.c
similarity index 100%
rename from clang/test/CodeGen/split-debug-inlining.c
rename to clang/test/DebugInfo/Generic/split-debug-inlining.c
diff --git a/clang/test/CodeGen/split-debug-output.c b/clang/test/DebugInfo/Generic/split-debug-output.c
similarity index 100%
rename from clang/test/CodeGen/split-debug-output.c
rename to clang/test/DebugInfo/Generic/split-debug-output.c
diff --git a/clang/test/CodeGen/split-debug-single-file.c b/clang/test/DebugInfo/Generic/split-debug-single-file.c
similarity index 100%
rename from clang/test/CodeGen/split-debug-single-file.c
rename to clang/test/DebugInfo/Generic/split-debug-single-file.c
diff --git a/clang/test/CodeGen/debug-info-static-const-fp.c b/clang/test/DebugInfo/Generic/static-const-fp.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-static-const-fp.c
rename to clang/test/DebugInfo/Generic/static-const-fp.c
diff --git a/clang/test/CodeGen/debug-info-static.c b/clang/test/DebugInfo/Generic/static.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-static.c
rename to clang/test/DebugInfo/Generic/static.c
diff --git a/clang/test/CodeGen/debug-info-switch-fallthrough.c b/clang/test/DebugInfo/Generic/switch-fallthrough.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-switch-fallthrough.c
rename to clang/test/DebugInfo/Generic/switch-fallthrough.c
diff --git a/clang/test/CodeGen/debug-info-sysroot-sdk.c b/clang/test/DebugInfo/Generic/sysroot-sdk.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-sysroot-sdk.c
rename to clang/test/DebugInfo/Generic/sysroot-sdk.c
diff --git a/clang/test/CodeGen/thinlto-split-dwarf.c b/clang/test/DebugInfo/Generic/thinlto-split-dwarf.c
similarity index 100%
rename from clang/test/CodeGen/thinlto-split-dwarf.c
rename to clang/test/DebugInfo/Generic/thinlto-split-dwarf.c
diff --git a/clang/test/CodeGen/debug-info-typedef.c b/clang/test/DebugInfo/Generic/typedef.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-typedef.c
rename to clang/test/DebugInfo/Generic/typedef.c
diff --git a/clang/test/CodeGen/ubsan-debuglog-return.c b/clang/test/DebugInfo/Generic/ubsan-debuglog-return.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-debuglog-return.c
rename to clang/test/DebugInfo/Generic/ubsan-debuglog-return.c
diff --git a/clang/test/CodeGen/ubsan-function-debuginfo.c b/clang/test/DebugInfo/Generic/ubsan-function-debuginfo.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-function-debuginfo.c
rename to clang/test/DebugInfo/Generic/ubsan-function-debuginfo.c
diff --git a/clang/test/CodeGen/ubsan-trap-debugloc.c b/clang/test/DebugInfo/Generic/ubsan-trap-debugloc.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-debugloc.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-debugloc.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-add-overflow.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-add-overflow.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-add-overflow.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-add-overflow.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-alignment-assumption.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-alignment-assumption.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-alignment-assumption.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-alignment-assumption.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-builtin-unreachable.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-builtin-unreachable.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-builtin-unreachable.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-builtin-unreachable.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-cfi-check-fail.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-cfi-check-fail.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-cfi-check-fail.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-cfi-check-fail.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-crash.cpp b/clang/test/DebugInfo/Generic/ubsan-trap-reason-crash.cpp
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-crash.cpp
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-crash.cpp
diff --git a/clang/test/CodeGen/ubsan-trap-reason-div-rem-overflow.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-div-rem-overflow.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-div-rem-overflow.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-div-rem-overflow.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-dynamic-type-cache-miss.cpp b/clang/test/DebugInfo/Generic/ubsan-trap-reason-dynamic-type-cache-miss.cpp
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-dynamic-type-cache-miss.cpp
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-dynamic-type-cache-miss.cpp
diff --git a/clang/test/CodeGen/ubsan-trap-reason-flag.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-flag.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-flag.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-flag.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-float-cast-overflow.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-float-cast-overflow.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-float-cast-overflow.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-float-cast-overflow.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-function-type-mismatch.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-function-type-mismatch.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-function-type-mismatch.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-function-type-mismatch.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-implicit-conversion.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-implicit-conversion.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-implicit-conversion.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-implicit-conversion.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-invalid-builtin.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-invalid-builtin.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-invalid-builtin.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-invalid-builtin.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-invalid-objc-cast.m b/clang/test/DebugInfo/Generic/ubsan-trap-reason-invalid-objc-cast.m
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-invalid-objc-cast.m
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-invalid-objc-cast.m
diff --git a/clang/test/CodeGen/ubsan-trap-reason-load-invalid-value.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-load-invalid-value.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-load-invalid-value.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-load-invalid-value.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-missing-return.cpp b/clang/test/DebugInfo/Generic/ubsan-trap-reason-missing-return.cpp
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-missing-return.cpp
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-missing-return.cpp
diff --git a/clang/test/CodeGen/ubsan-trap-reason-mul-overflow.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-mul-overflow.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-mul-overflow.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-mul-overflow.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-negate-overflow.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-negate-overflow.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-negate-overflow.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-negate-overflow.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-nonnull-arg.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-nonnull-arg.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-nonnull-arg.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-nonnull-arg.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-nonnull-return.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-nonnull-return.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-nonnull-return.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-nonnull-return.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-nullability-arg.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-nullability-arg.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-nullability-arg.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-nullability-arg.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-nullability-return.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-nullability-return.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-nullability-return.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-nullability-return.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-out-of-bounds.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-out-of-bounds.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-out-of-bounds.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-out-of-bounds.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-pointer-overflow.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-pointer-overflow.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-pointer-overflow.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-pointer-overflow.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-shift-out-of-bounds.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-shift-out-of-bounds.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-shift-out-of-bounds.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-shift-out-of-bounds.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-sub-overflow.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-sub-overflow.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-sub-overflow.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-sub-overflow.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-type-mismatch.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-type-mismatch.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-type-mismatch.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-type-mismatch.c
diff --git a/clang/test/CodeGen/ubsan-trap-reason-vla-bound-not-positive.c b/clang/test/DebugInfo/Generic/ubsan-trap-reason-vla-bound-not-positive.c
similarity index 100%
rename from clang/test/CodeGen/ubsan-trap-reason-vla-bound-not-positive.c
rename to clang/test/DebugInfo/Generic/ubsan-trap-reason-vla-bound-not-positive.c
diff --git a/clang/test/CodeGen/unique-internal-linkage-names-dwarf.c b/clang/test/DebugInfo/Generic/unique-internal-linkage-names-dwarf.c
similarity index 100%
rename from clang/test/CodeGen/unique-internal-linkage-names-dwarf.c
rename to clang/test/DebugInfo/Generic/unique-internal-linkage-names-dwarf.c
diff --git a/clang/test/CodeGen/unique-internal-linkage-names-dwarf.cpp b/clang/test/DebugInfo/Generic/unique-internal-linkage-names-dwarf.cpp
similarity index 100%
rename from clang/test/CodeGen/unique-internal-linkage-names-dwarf.cpp
rename to clang/test/DebugInfo/Generic/unique-internal-linkage-names-dwarf.cpp
diff --git a/clang/test/CodeGen/unsigned-promotion-debuginfo.c b/clang/test/DebugInfo/Generic/unsigned-promotion-debuginfo.c
similarity index 100%
rename from clang/test/CodeGen/unsigned-promotion-debuginfo.c
rename to clang/test/DebugInfo/Generic/unsigned-promotion-debuginfo.c
diff --git a/clang/test/CodeGen/debug-info-unused-types.c b/clang/test/DebugInfo/Generic/unused-types.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-unused-types.c
rename to clang/test/DebugInfo/Generic/unused-types.c
diff --git a/clang/test/CodeGen/debug-info-unused-types.cpp b/clang/test/DebugInfo/Generic/unused-types.cpp
similarity index 100%
rename from clang/test/CodeGen/debug-info-unused-types.cpp
rename to clang/test/DebugInfo/Generic/unused-types.cpp
diff --git a/clang/test/CodeGen/unused_nested_enump.cpp b/clang/test/DebugInfo/Generic/unused_nested_enump.cpp
similarity index 100%
rename from clang/test/CodeGen/unused_nested_enump.cpp
rename to clang/test/DebugInfo/Generic/unused_nested_enump.cpp
diff --git a/clang/test/CodeGen/debug-info-variables.c b/clang/test/DebugInfo/Generic/variables.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-variables.c
rename to clang/test/DebugInfo/Generic/variables.c
diff --git a/clang/test/CodeGen/debug-info-vector-bool.c b/clang/test/DebugInfo/Generic/vector-bool.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-vector-bool.c
rename to clang/test/DebugInfo/Generic/vector-bool.c
diff --git a/clang/test/CodeGen/debug-info-vector.c b/clang/test/DebugInfo/Generic/vector.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-vector.c
rename to clang/test/DebugInfo/Generic/vector.c
diff --git a/clang/test/CodeGen/debug-info-version-coff.c b/clang/test/DebugInfo/Generic/version-coff.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-version-coff.c
rename to clang/test/DebugInfo/Generic/version-coff.c
diff --git a/clang/test/CodeGen/debug-info-version.c b/clang/test/DebugInfo/Generic/version.c
similarity index 84%
rename from clang/test/CodeGen/debug-info-version.c
rename to clang/test/DebugInfo/Generic/version.c
index c7c2bb95017a2..485b80e2a5b10 100644
--- a/clang/test/CodeGen/debug-info-version.c
+++ b/clang/test/DebugInfo/Generic/version.c
@@ -1,4 +1,4 @@
-// REQUIRES: !system-windows
+// REQUIRES: !system-windows, !system-cygwin
 // RUN: %clang -g -S -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang -S -emit-llvm -o - %s | FileCheck %s --check-prefix=NO_DEBUG
 int main (void) {
diff --git a/clang/test/CodeGen/debug-info-vla.c b/clang/test/DebugInfo/Generic/vla.c
similarity index 100%
rename from clang/test/CodeGen/debug-info-vla.c
rename to clang/test/DebugInfo/Generic/vla.c
diff --git a/clang/test/DebugInfo/KeyInstructions/lit.local.cfg b/clang/test/DebugInfo/KeyInstructions/lit.local.cfg
deleted file mode 100644
index 482bd5c8ac251..0000000000000
--- a/clang/test/DebugInfo/KeyInstructions/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not config.has_key_instructions:
-    config.unsupported = True
diff --git a/clang/test/CodeGenObjC/2009-01-21-invalid-debug-info.m b/clang/test/DebugInfo/ObjC/2009-01-21-invalid.m
similarity index 100%
rename from clang/test/CodeGenObjC/2009-01-21-invalid-debug-info.m
rename to clang/test/DebugInfo/ObjC/2009-01-21-invalid.m
diff --git a/clang/test/CodeGenObjC/2010-02-09-DbgSelf.m b/clang/test/DebugInfo/ObjC/2010-02-09-Self.m
similarity index 100%
rename from clang/test/CodeGenObjC/2010-02-09-DbgSelf.m
rename to clang/test/DebugInfo/ObjC/2010-02-09-Self.m
diff --git a/clang/test/CodeGenObjC/2010-02-15-Dbg-MethodStart.m b/clang/test/DebugInfo/ObjC/2010-02-15-MethodStart.m
similarity index 100%
rename from clang/test/CodeGenObjC/2010-02-15-Dbg-MethodStart.m
rename to clang/test/DebugInfo/ObjC/2010-02-15-MethodStart.m
diff --git a/clang/test/CodeGenObjC/2010-02-23-DbgInheritance.m b/clang/test/DebugInfo/ObjC/2010-02-23-Inheritance.m
similarity index 100%
rename from clang/test/CodeGenObjC/2010-02-23-DbgInheritance.m
rename to clang/test/DebugInfo/ObjC/2010-02-23-Inheritance.m
diff --git a/clang/test/CodeGenObjC/arc-linetable-autorelease.m b/clang/test/DebugInfo/ObjC/arc-linetable-autorelease.m
similarity index 100%
rename from clang/test/CodeGenObjC/arc-linetable-autorelease.m
rename to clang/test/DebugInfo/ObjC/arc-linetable-autorelease.m
diff --git a/clang/test/CodeGenObjC/arc-linetable.m b/clang/test/DebugInfo/ObjC/arc-linetable.m
similarity index 100%
rename from clang/test/CodeGenObjC/arc-linetable.m
rename to clang/test/DebugInfo/ObjC/arc-linetable.m
diff --git a/clang/test/CodeGenObjC/block-byref-debuginfo.m b/clang/test/DebugInfo/ObjC/block-byref.m
similarity index 100%
rename from clang/test/CodeGenObjC/block-byref-debuginfo.m
rename to clang/test/DebugInfo/ObjC/block-byref.m
diff --git a/clang/test/CodeGenObjC/debug-info-block-helper.m b/clang/test/DebugInfo/ObjC/block-helper.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-block-helper.m
rename to clang/test/DebugInfo/ObjC/block-helper.m
diff --git a/clang/test/CodeGenObjC/debug-info-block-line.m b/clang/test/DebugInfo/ObjC/block-line.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-block-line.m
rename to clang/test/DebugInfo/ObjC/block-line.m
diff --git a/clang/test/CodeGenObjC/debug-info-block-type.m b/clang/test/DebugInfo/ObjC/block-type.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-block-type.m
rename to clang/test/DebugInfo/ObjC/block-type.m
diff --git a/clang/test/CodeGenObjC/blocks-ivar-debug.m b/clang/test/DebugInfo/ObjC/blocks-ivar.m
similarity index 100%
rename from clang/test/CodeGenObjC/blocks-ivar-debug.m
rename to clang/test/DebugInfo/ObjC/blocks-ivar.m
diff --git a/clang/test/CodeGenObjC/debug-info-blocks.m b/clang/test/DebugInfo/ObjC/blocks.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-blocks.m
rename to clang/test/DebugInfo/ObjC/blocks.m
diff --git a/clang/test/CodeGenObjC/catch-lexical-block.m b/clang/test/DebugInfo/ObjC/catch-lexical-block.m
similarity index 100%
rename from clang/test/CodeGenObjC/catch-lexical-block.m
rename to clang/test/DebugInfo/ObjC/catch-lexical-block.m
diff --git a/clang/test/CodeGenObjC/debug-info-category.m b/clang/test/DebugInfo/ObjC/category.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-category.m
rename to clang/test/DebugInfo/ObjC/category.m
diff --git a/clang/test/CodeGenObjC/debug-info-class-extension.m b/clang/test/DebugInfo/ObjC/class-extension.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-class-extension.m
rename to clang/test/DebugInfo/ObjC/class-extension.m
diff --git a/clang/test/CodeGenObjC/debug-info-class-extension2.m b/clang/test/DebugInfo/ObjC/class-extension2.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-class-extension2.m
rename to clang/test/DebugInfo/ObjC/class-extension2.m
diff --git a/clang/test/CodeGenObjC/debug-info-class-extension3.m b/clang/test/DebugInfo/ObjC/class-extension3.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-class-extension3.m
rename to clang/test/DebugInfo/ObjC/class-extension3.m
diff --git a/clang/test/CodeGenObjC/debug-info-crash-2.m b/clang/test/DebugInfo/ObjC/crash-2.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-crash-2.m
rename to clang/test/DebugInfo/ObjC/crash-2.m
diff --git a/clang/test/CodeGenObjC/debug-info-crash.m b/clang/test/DebugInfo/ObjC/crash.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-crash.m
rename to clang/test/DebugInfo/ObjC/crash.m
diff --git a/clang/test/CodeGenObjC/debug-info-default-synth-ivar.m b/clang/test/DebugInfo/ObjC/default-synth-ivar.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-default-synth-ivar.m
rename to clang/test/DebugInfo/ObjC/default-synth-ivar.m
diff --git a/clang/test/CodeGenObjC/debug-info-direct-method.m b/clang/test/DebugInfo/ObjC/direct-method.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-direct-method.m
rename to clang/test/DebugInfo/ObjC/direct-method.m
diff --git a/clang/test/CodeGenObjC/debug-info-fwddecl.m b/clang/test/DebugInfo/ObjC/fwddecl.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-fwddecl.m
rename to clang/test/DebugInfo/ObjC/fwddecl.m
diff --git a/clang/test/CodeGenObjC/debug-info-getter-name.m b/clang/test/DebugInfo/ObjC/getter-name.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-getter-name.m
rename to clang/test/DebugInfo/ObjC/getter-name.m
diff --git a/clang/test/CodeGenObjC/debug-info-id-with-protocol.m b/clang/test/DebugInfo/ObjC/id-with-protocol.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-id-with-protocol.m
rename to clang/test/DebugInfo/ObjC/id-with-protocol.m
diff --git a/clang/test/CodeGenObjC/debug-info-impl.m b/clang/test/DebugInfo/ObjC/impl.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-impl.m
rename to clang/test/DebugInfo/ObjC/impl.m
diff --git a/clang/test/CodeGenObjC/debug-info-instancetype.m b/clang/test/DebugInfo/ObjC/instancetype.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-instancetype.m
rename to clang/test/DebugInfo/ObjC/instancetype.m
diff --git a/clang/test/CodeGenObjC/debug-info-ivars-extension.m b/clang/test/DebugInfo/ObjC/ivars-extension.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-ivars-extension.m
rename to clang/test/DebugInfo/ObjC/ivars-extension.m
diff --git a/clang/test/CodeGenObjC/debug-info-ivars-indirect.m b/clang/test/DebugInfo/ObjC/ivars-indirect.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-ivars-indirect.m
rename to clang/test/DebugInfo/ObjC/ivars-indirect.m
diff --git a/clang/test/CodeGenObjC/debug-info-ivars-private.m b/clang/test/DebugInfo/ObjC/ivars-private.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-ivars-private.m
rename to clang/test/DebugInfo/ObjC/ivars-private.m
diff --git a/clang/test/CodeGenObjC/debug-info-ivars.m b/clang/test/DebugInfo/ObjC/ivars.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-ivars.m
rename to clang/test/DebugInfo/ObjC/ivars.m
diff --git a/clang/test/CodeGenObjC/layout-bitfield-crash.m b/clang/test/DebugInfo/ObjC/layout-bitfield-crash.m
similarity index 100%
rename from clang/test/CodeGenObjC/layout-bitfield-crash.m
rename to clang/test/DebugInfo/ObjC/layout-bitfield-crash.m
diff --git a/clang/test/CodeGenObjC/debug-info-lifetime-crash.m b/clang/test/DebugInfo/ObjC/lifetime-crash.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-lifetime-crash.m
rename to clang/test/DebugInfo/ObjC/lifetime-crash.m
diff --git a/clang/test/CodeGenObjC/debug-info-linkagename.m b/clang/test/DebugInfo/ObjC/linkagename.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-linkagename.m
rename to clang/test/DebugInfo/ObjC/linkagename.m
diff --git a/clang/test/DebugInfo/ObjC/lit.local.cfg b/clang/test/DebugInfo/ObjC/lit.local.cfg
new file mode 100644
index 0000000000000..b48f68b01581d
--- /dev/null
+++ b/clang/test/DebugInfo/ObjC/lit.local.cfg
@@ -0,0 +1,5 @@
+# objective-C is not supported on AIX and zOS
+unsupported_platforms = [ "system-aix", "system-zos" ]
+
+if any(up in config.available_features for up in unsupported_platforms):
+    config.unsupported = True
diff --git a/clang/test/CodeGenObjC/debug-info-nested-blocks.m b/clang/test/DebugInfo/ObjC/nested-blocks.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-nested-blocks.m
rename to clang/test/DebugInfo/ObjC/nested-blocks.m
diff --git a/clang/test/CodeGenObjC/debug-info-nodebug.m b/clang/test/DebugInfo/ObjC/nodebug.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-nodebug.m
rename to clang/test/DebugInfo/ObjC/nodebug.m
diff --git a/clang/test/CodeGenObjC/nontrivial-c-struct-exception.m b/clang/test/DebugInfo/ObjC/nontrivial-c-struct-exception.m
similarity index 100%
rename from clang/test/CodeGenObjC/nontrivial-c-struct-exception.m
rename to clang/test/DebugInfo/ObjC/nontrivial-c-struct-exception.m
diff --git a/clang/test/CodeGenObjC/objc-fixed-enum.m b/clang/test/DebugInfo/ObjC/objc-fixed-enum.m
similarity index 100%
rename from clang/test/CodeGenObjC/objc-fixed-enum.m
rename to clang/test/DebugInfo/ObjC/objc-fixed-enum.m
diff --git a/clang/test/CodeGenObjC/debug-info-objc-property-dwarf5.m b/clang/test/DebugInfo/ObjC/objc-property-dwarf5.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-objc-property-dwarf5.m
rename to clang/test/DebugInfo/ObjC/objc-property-dwarf5.m
diff --git a/clang/test/CodeGenObjC/objc2-weak-ivar-debug.m b/clang/test/DebugInfo/ObjC/objc2-weak-ivar.m
similarity index 100%
rename from clang/test/CodeGenObjC/objc2-weak-ivar-debug.m
rename to clang/test/DebugInfo/ObjC/objc2-weak-ivar.m
diff --git a/clang/test/CodeGenObjC/debuginfo-properties.m b/clang/test/DebugInfo/ObjC/properties.m
similarity index 100%
rename from clang/test/CodeGenObjC/debuginfo-properties.m
rename to clang/test/DebugInfo/ObjC/properties.m
diff --git a/clang/test/CodeGenObjC/property-dbg.m b/clang/test/DebugInfo/ObjC/property-2.m
similarity index 100%
rename from clang/test/CodeGenObjC/property-dbg.m
rename to clang/test/DebugInfo/ObjC/property-2.m
diff --git a/clang/test/CodeGenObjC/debug-info-property-accessors.m b/clang/test/DebugInfo/ObjC/property-accessors.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-property-accessors.m
rename to clang/test/DebugInfo/ObjC/property-accessors.m
diff --git a/clang/test/CodeGenObjC/debug-info-property-class-extension.m b/clang/test/DebugInfo/ObjC/property-class-extension.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-property-class-extension.m
rename to clang/test/DebugInfo/ObjC/property-class-extension.m
diff --git a/clang/test/CodeGenObjC/debug-info-property-class-instance-same-name.m b/clang/test/DebugInfo/ObjC/property-class-instance-same-name.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-property-class-instance-same-name.m
rename to clang/test/DebugInfo/ObjC/property-class-instance-same-name.m
diff --git a/clang/test/CodeGenObjC/debug-property-synth.m b/clang/test/DebugInfo/ObjC/property-synth.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-property-synth.m
rename to clang/test/DebugInfo/ObjC/property-synth.m
diff --git a/clang/test/CodeGenObjC/debug-info-property.m b/clang/test/DebugInfo/ObjC/property.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-property.m
rename to clang/test/DebugInfo/ObjC/property.m
diff --git a/clang/test/CodeGenObjC/debug-info-property2.m b/clang/test/DebugInfo/ObjC/property2.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-property2.m
rename to clang/test/DebugInfo/ObjC/property2.m
diff --git a/clang/test/CodeGenObjC/debug-info-property3.m b/clang/test/DebugInfo/ObjC/property3.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-property3.m
rename to clang/test/DebugInfo/ObjC/property3.m
diff --git a/clang/test/CodeGenObjC/debug-info-property4.m b/clang/test/DebugInfo/ObjC/property4.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-property4.m
rename to clang/test/DebugInfo/ObjC/property4.m
diff --git a/clang/test/CodeGenObjC/debug-info-property5.m b/clang/test/DebugInfo/ObjC/property5.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-property5.m
rename to clang/test/DebugInfo/ObjC/property5.m
diff --git a/clang/test/CodeGenObjC/debug-info-pubtypes.m b/clang/test/DebugInfo/ObjC/pubtypes.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-pubtypes.m
rename to clang/test/DebugInfo/ObjC/pubtypes.m
diff --git a/clang/test/CodeGenObjC/debug-info-selector.m b/clang/test/DebugInfo/ObjC/selector.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-selector.m
rename to clang/test/DebugInfo/ObjC/selector.m
diff --git a/clang/test/CodeGenObjC/debug-info-self.m b/clang/test/DebugInfo/ObjC/self.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-self.m
rename to clang/test/DebugInfo/ObjC/self.m
diff --git a/clang/test/CodeGenObjC/debug-info-static-var.m b/clang/test/DebugInfo/ObjC/static-var.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-static-var.m
rename to clang/test/DebugInfo/ObjC/static-var.m
diff --git a/clang/test/CodeGenObjC/debug-info-synthesis.m b/clang/test/DebugInfo/ObjC/synthesis.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-synthesis.m
rename to clang/test/DebugInfo/ObjC/synthesis.m
diff --git a/clang/test/CodeGenObjC/ubsan-check-debuglocs.m b/clang/test/DebugInfo/ObjC/ubsan-check-debuglocs.m
similarity index 100%
rename from clang/test/CodeGenObjC/ubsan-check-debuglocs.m
rename to clang/test/DebugInfo/ObjC/ubsan-check-debuglocs.m
diff --git a/clang/test/CodeGenObjC/debug-info-variadic-method.m b/clang/test/DebugInfo/ObjC/variadic-method.m
similarity index 100%
rename from clang/test/CodeGenObjC/debug-info-variadic-method.m
rename to clang/test/DebugInfo/ObjC/variadic-method.m
diff --git a/clang/test/CodeGen/RISCV/riscv-v-debuginfo.c b/clang/test/DebugInfo/RISCV/riscv-v-debuginfo.c
similarity index 100%
rename from clang/test/CodeGen/RISCV/riscv-v-debuginfo.c
rename to clang/test/DebugInfo/RISCV/riscv-v-debuginfo.c
diff --git a/clang/test/CodeGen/X86/i128-debuginfo.c b/clang/test/DebugInfo/X86/i128-debuginfo.c
similarity index 100%
rename from clang/test/CodeGen/X86/i128-debuginfo.c
rename to clang/test/DebugInfo/X86/i128-debuginfo.c
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/10/crtbegin.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/10/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/10/crtend.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/10/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/10/include/c++/.keep b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/10/include/c++/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/11/crtbegin.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/11/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/11/crtend.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/11/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/11/include/.keep b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/11/include/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/12/crtbegin.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/12/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/12/crtend.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/12/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/12/include/.keep b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu/12/include/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/10/crtbegin.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/10/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/10/crtend.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/10/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/10/include/.keep b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/10/include/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/11/crtbegin.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/11/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/11/crtend.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/11/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/11/include/c++/.keep b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/11/include/c++/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/12/crtbegin.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/12/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/12/crtend.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/12/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/12/include/.keep b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu/12/include/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/10/crtbegin.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/10/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/10/crtend.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/10/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/10/include/.keep b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/10/include/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/11/crtbegin.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/11/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/11/crtend.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/11/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/11/include/c++/.keep b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/11/include/c++/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/12/crtbegin.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/12/crtbegin.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/12/crtend.o b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/12/crtend.o
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/12/include/c++/.keep b/clang/test/Driver/Inputs/gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu/12/include/c++/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Driver/aarch64-cpu-defaults-appleos26.c b/clang/test/Driver/aarch64-cpu-defaults-appleos26.c
new file mode 100644
index 0000000000000..99176050bf5ae
--- /dev/null
+++ b/clang/test/Driver/aarch64-cpu-defaults-appleos26.c
@@ -0,0 +1,22 @@
+/// iOS 26 and watchOS 26 bump the default arm64 CPU targets.
+
+/// arm64 iOS 26 defaults to apple-a12.  arm64e already did.
+// RUN: %clang -target arm64-apple-ios26  -### -c %s 2>&1 | FileCheck %s --check-prefix=A12
+// RUN: %clang -target arm64e-apple-ios26 -### -c %s 2>&1 | FileCheck %s --check-prefix=A12
+
+/// arm64e/arm64_32 watchOS 26 default to apple-s6.
+// RUN: %clang -target arm64e-apple-watchos26   -### -c %s 2>&1 | FileCheck %s --check-prefix=S6
+// RUN: %clang -target arm64_32-apple-watchos26 -### -c %s 2>&1 | FileCheck %s --check-prefix=S6
+
+/// arm64 is new in watchOS 26, and defaults to apple-s6.
+// RUN: %clang -target arm64-apple-watchos26  -### -c %s 2>&1 | FileCheck %s --check-prefix=S6
+
+/// llvm usually treats tvOS like iOS, but it runs on different hardware.
+// RUN: %clang -target arm64-apple-tvos26  -### -c %s 2>&1 | FileCheck %s --check-prefix=A7
+// RUN: %clang -target arm64e-apple-tvos26 -### -c %s 2>&1 | FileCheck %s --check-prefix=A12
+
+/// Simulators are tested with other Mac-like targets in aarch64-mac-cpus.c.
+
+// A12: "-target-cpu" "apple-a12"
+// S6:  "-target-cpu" "apple-s6"
+// A7:  "-target-cpu" "apple-a7"
diff --git a/clang/test/Driver/aarch64-mac-cpus.c b/clang/test/Driver/aarch64-mac-cpus.c
index 8d23ad8c956fd..1aef1ae285e86 100644
--- a/clang/test/Driver/aarch64-mac-cpus.c
+++ b/clang/test/Driver/aarch64-mac-cpus.c
@@ -1,4 +1,4 @@
-// arm64 Mac-based targets default to Apple A13.
+// arm64/arm64e Mac-based targets default to Apple M1.
 
 // RUN: %clang --target=arm64-apple-macos             -### -c %s 2>&1 | FileCheck %s
 // RUN: %clang --target=arm64-apple-ios-macabi        -### -c %s 2>&1 | FileCheck %s
diff --git a/clang/test/Driver/aarch64-toolchain.c b/clang/test/Driver/aarch64-toolchain.c
index 512b5a883cd5b..4fddcce240c13 100644
--- a/clang/test/Driver/aarch64-toolchain.c
+++ b/clang/test/Driver/aarch64-toolchain.c
@@ -157,3 +157,15 @@
 // AARCH64-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtbegin.o"
 // AARCH64-BAREMETAL-UNWINDLIB: "--start-group" "{{.*}}libclang_rt.builtins{{.*}}.a" "--as-needed" "-lunwind" "--no-as-needed" "-lc" "-lgloss" "--end-group"
 // AARCH64-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtend.o"
+
+// RUN: %clang -static-pie -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-ARM-STATIC-PIE %s
+
+// C-ARM-STATIC-PIE: "-Bstatic" "-pie" "--no-dynamic-linker" "-z" "text" "-m" "aarch64elf" "-EL"
+// C-ARM-STATIC-PIE: "{{.*}}rcrt1.o"
+// C-ARM-STATIC-PIE: "{{.*}}crtbeginS.o"
+// C-ARM-STATIC-PIE: "--start-group" "-lgcc" "-lgcc_eh" "-lc" "-lgloss" "--end-group"
+// C-ARM-STATIC-PIE: "{{.*}}crtendS.o"
diff --git a/clang/test/Driver/arch-arm64e.c b/clang/test/Driver/arch-arm64e.c
index 0fb12d4dcc5e3..39006d203989b 100644
--- a/clang/test/Driver/arch-arm64e.c
+++ b/clang/test/Driver/arch-arm64e.c
@@ -2,11 +2,20 @@
 
 // RUN: %clang -target arm64-apple-darwin -c %s -### 2>&1 | FileCheck %s --check-prefix NONE
 // NONE: "-cc1"
-// NONE-NOT: "-fptrauth-intrinsics"
+
 // NONE-NOT: "-fptrauth-calls"
 // NONE-NOT: "-fptrauth-returns"
+// NONE-NOT: "-fptrauth-intrinsics"
 // NONE-NOT: "-fptrauth-indirect-gotos"
 // NONE-NOT: "-fptrauth-auth-traps"
+// NONE-NOT: "-fptrauth-vtable-pointer-address-discrimination"
+// NONE-NOT: "-fptrauth-vtable-pointer-type-discrimination"
+// NONE-NOT: "-fptrauth-objc-isa"
+// NONE-NOT: "-fptrauth-objc-class-ro"
+// NONE-NOT: "-fptrauth-objc-interface-sel"
+
+// Final catch all if any new flags are added
+// NONE-NOT: "-fptrauth"
 
 // RUN: %clang -target arm64-apple-darwin -fptrauth-calls -c %s -### 2>&1 | FileCheck %s --check-prefix CALL
 // CALL: "-cc1"{{.*}} {{.*}} "-fptrauth-calls"
@@ -23,39 +32,39 @@
 // RUN: %clang -target arm64-apple-darwin -fptrauth-auth-traps -c %s -### 2>&1 | FileCheck %s --check-prefix TRAPS
 // TRAPS: "-cc1"{{.*}} {{.*}} "-fptrauth-auth-traps"
 
-
 // Check the arm64e defaults.
 
 // RUN: %clang -target arm64e-apple-ios -c %s -### 2>&1 | FileCheck %s --check-prefix DEFAULT
+// RUN: %clang -target arm64e-apple-macos -c %s -### 2>&1 | FileCheck %s --check-prefix DEFAULTMAC
+// RUN: %if system-darwin && target={{.*}}-{{darwin|macos}}{{.*}} %{ %clang -target arm64e-apple-macos -c %s -### 2>&1 | FileCheck %s --check-prefix DEFAULTARCH %}
 // RUN: %clang -mkernel -target arm64e-apple-ios -c %s -### 2>&1 | FileCheck %s --check-prefix DEFAULT
 // RUN: %clang -fapple-kext -target arm64e-apple-ios -c %s -### 2>&1 | FileCheck %s --check-prefix DEFAULT
-// DEFAULT: "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-calls" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" {{.*}}"-target-cpu" "apple-a12"{{.*}}
+// DEFAULT: "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-objc-isa" "-fptrauth-objc-class-ro" "-fptrauth-objc-interface-sel" {{.*}}"-target-cpu" "apple-a12"{{.*}}
+// DEFAULTMAC: "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-objc-isa" "-fptrauth-objc-class-ro" "-fptrauth-objc-interface-sel" {{.*}}"-target-cpu" "apple-m1"{{.*}}
+// DEFAULTARCH: "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-objc-isa" "-fptrauth-objc-class-ro" "-fptrauth-objc-interface-sel"
 
 // RUN: %clang -target arm64e-apple-none-macho -c %s -### 2>&1 | FileCheck %s --check-prefix DEFAULT-MACHO
-// DEFAULT-MACHO: "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-calls" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" {{.*}}"-target-cpu" "apple-a12"{{.*}}
+// DEFAULT-MACHO: "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-objc-isa" "-fptrauth-objc-class-ro" "-fptrauth-objc-interface-sel" {{.*}}"-target-cpu" "apple-a12"{{.*}}
 
 
 // RUN: %clang -target arm64e-apple-ios -fno-ptrauth-calls -c %s -### 2>&1 | FileCheck %s --check-prefix DEFAULT-NOCALL
-// RUN: %clang -mkernel -target arm64e-apple-ios -fno-ptrauth-calls -c %s -### 2>&1 | FileCheck %s --check-prefix DEFAULT-NOCALL
-// RUN: %clang -fapple-kext -target arm64e-apple-ios -fno-ptrauth-calls -c %s -### 2>&1 | FileCheck %s --check-prefix DEFAULT-NOCALL
 // DEFAULT-NOCALL-NOT: "-fptrauth-calls"
-// DEFAULT-NOCALL: "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" {{.*}}"-target-cpu" "apple-a12"
+// DEFAULT-NOCALL: "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-objc-isa" "-fptrauth-objc-class-ro" "-fptrauth-objc-interface-sel" {{.*}}"-target-cpu" "apple-a12"
 
 
 // RUN: %clang -target arm64e-apple-ios -fno-ptrauth-returns -c %s -### 2>&1 | FileCheck %s --check-prefix NORET
 
 // NORET-NOT: "-fptrauth-returns"
-// NORET: "-fptrauth-intrinsics" "-fptrauth-calls" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" {{.*}}"-target-cpu" "apple-a12"
+// NORET: "-fptrauth-calls" "-fptrauth-intrinsics" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-objc-isa" "-fptrauth-objc-class-ro" "-fptrauth-objc-interface-sel" {{.*}}"-target-cpu" "apple-a12"
 
 // RUN: %clang -target arm64e-apple-ios -fno-ptrauth-intrinsics -c %s -### 2>&1 | FileCheck %s --check-prefix NOINTRIN
 
-// NOINTRIN: "-fptrauth-returns"
 // NOINTRIN-NOT: "-fptrauth-intrinsics"
-// NOINTRIN: "-fptrauth-calls" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" {{.*}}"-target-cpu" "apple-a12"{{.*}}
+// NOINTRIN: "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-indirect-gotos" "-fptrauth-auth-traps" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-objc-isa" "-fptrauth-objc-class-ro" "-fptrauth-objc-interface-sel" {{.*}}"-target-cpu" "apple-a12"
 
 
 // RUN: %clang -target arm64e-apple-ios -fno-ptrauth-auth-traps -c %s -### 2>&1 | FileCheck %s --check-prefix NOTRAP
-// NOTRAP: "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-calls" "-fptrauth-indirect-gotos" {{.*}}"-target-cpu" "apple-a12"
+// NOTRAP: "-fptrauth-calls" "-fptrauth-returns" "-fptrauth-intrinsics" "-fptrauth-indirect-gotos" "-fptrauth-vtable-pointer-address-discrimination" "-fptrauth-vtable-pointer-type-discrimination" "-fptrauth-objc-isa" "-fptrauth-objc-class-ro" "-fptrauth-objc-interface-sel" {{.*}}"-target-cpu" "apple-a12"
 
 
 // Check the CPU defaults and overrides.
diff --git a/clang/test/Driver/arm-toolchain.c b/clang/test/Driver/arm-toolchain.c
index 9005992f2b759..7ee65a4782e21 100644
--- a/clang/test/Driver/arm-toolchain.c
+++ b/clang/test/Driver/arm-toolchain.c
@@ -158,3 +158,15 @@
 // ARM-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtbegin.o"
 // ARM-BAREMETAL-UNWINDLIB: "--start-group" "{{.*}}libclang_rt.builtins.a" "--as-needed" "-lunwind" "--no-as-needed" "-lc" "-lgloss" "--end-group"
 // ARM-BAREMETAL-UNWINDLIB: "{{.*}}clang_rt.crtend.o"
+
+// RUN: %clang -static-pie -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=libgcc --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-ARM-STATIC-PIE %s
+
+// C-ARM-STATIC-PIE: "-Bstatic" "-pie" "--no-dynamic-linker" "-z" "text" "-m" "armelf" "-EL"
+// C-ARM-STATIC-PIE: "{{.*}}rcrt1.o"
+// C-ARM-STATIC-PIE: "{{.*}}crtbeginS.o"
+// C-ARM-STATIC-PIE: "--start-group" "-lgcc" "-lgcc_eh" "-lc" "-lgloss" "--end-group"
+// C-ARM-STATIC-PIE: "{{.*}}crtendS.o"
diff --git a/clang/test/Driver/cl-x86-flags.c b/clang/test/Driver/cl-x86-flags.c
index 1e6418207e642..89526744c0a49 100644
--- a/clang/test/Driver/cl-x86-flags.c
+++ b/clang/test/Driver/cl-x86-flags.c
@@ -145,6 +145,7 @@ void f(void) {
 
 // RUN: %clang_cl --target=x86_64-pc-windows -mapxf -### -- 2>&1 %s | FileCheck -check-prefix=APXF %s
 // RUN: %clang_cl --target=x86_64-pc-windows -mapxf -mno-apxf -### -- 2>&1 %s | FileCheck -check-prefix=NO-APXF %s
-// RUN: %clang_cl --target=x86_64-pc-windows -mapx-features=egpr,push2pop2,ppx,ndd,ccmp,nf,cf,zu -### -- 2>&1 %s | FileCheck -check-prefix=APXF %s
-// APXF: "-target-feature" "+egpr" "-target-feature" "+push2pop2" "-target-feature" "+ppx" "-target-feature" "+ndd" "-target-feature" "+ccmp" "-target-feature" "+nf" "-target-feature" "+cf" "-target-feature" "+zu"
-// NO-APXF: "-target-feature" "-egpr" "-target-feature" "-push2pop2" "-target-feature" "-ppx" "-target-feature" "-ndd" "-target-feature" "-ccmp" "-target-feature" "-nf" "-target-feature" "-cf" "-target-feature" "-zu"
+// RUN: %clang_cl --target=x86_64-pc-windows -mapx-features=egpr,push2pop2,ppx,ndd,ccmp,nf,cf,zu -### -- 2>&1 %s | FileCheck -check-prefix=APXALL %s
+// APXF: "-target-feature" "+egpr" "-target-feature" "+push2pop2" "-target-feature" "+ppx" "-target-feature" "+ndd" "-target-feature" "+ccmp" "-target-feature" "+nf" "-target-feature" "+zu"
+// NO-APXF: "-target-feature" "-egpr" "-target-feature" "-push2pop2" "-target-feature" "-ppx" "-target-feature" "-ndd" "-target-feature" "-ccmp" "-target-feature" "-nf" "-target-feature" "-zu"
+// APXALL: "-target-feature" "+egpr" "-target-feature" "+push2pop2" "-target-feature" "+ppx" "-target-feature" "+ndd" "-target-feature" "+ccmp" "-target-feature" "+nf" "-target-feature" "+cf" "-target-feature" "+zu"
diff --git a/clang/test/Driver/clang-offload-bundler-zlib.c b/clang/test/Driver/clang-offload-bundler-zlib.c
index b026e2ec99877..211601c2c7fbb 100644
--- a/clang/test/Driver/clang-offload-bundler-zlib.c
+++ b/clang/test/Driver/clang-offload-bundler-zlib.c
@@ -66,6 +66,30 @@
 // NOHOST-V3-DAG: hip-amdgcn-amd-amdhsa--gfx900
 // NOHOST-V3-DAG: hip-amdgcn-amd-amdhsa--gfx906
 
+// Check compression/decompression of offload bundle using version 2 format.
+//
+// RUN: env OFFLOAD_BUNDLER_COMPRESS=1 OFFLOAD_BUNDLER_VERBOSE=1 COMPRESSED_BUNDLE_FORMAT_VERSION=2 \
+// RUN:   clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \
+// RUN:   -input=%t.tgt1 -input=%t.tgt2 -output=%t.hip.bundle.bc 2>&1 | \
+// RUN:   FileCheck -check-prefix=COMPRESS-V2 %s
+// RUN: clang-offload-bundler -type=bc -list -input=%t.hip.bundle.bc | FileCheck -check-prefix=NOHOST-V2 %s
+// RUN: env OFFLOAD_BUNDLER_VERBOSE=1 \
+// RUN:   clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \
+// RUN:   -output=%t.res.tgt1 -output=%t.res.tgt2 -input=%t.hip.bundle.bc -unbundle 2>&1 | \
+// RUN:   FileCheck -check-prefix=DECOMPRESS-V2 %s
+// RUN: diff %t.tgt1 %t.res.tgt1
+// RUN: diff %t.tgt2 %t.res.tgt2
+//
+// COMPRESS-V2: Compressed bundle format version: 2
+// COMPRESS-V2: Compression method used: zlib
+// COMPRESS-V2: Compression level: 6
+// DECOMPRESS-V2: Compressed bundle format version: 2
+// DECOMPRESS-V2: Decompression method: zlib
+// DECOMPRESS-V2: Hashes match: Yes
+// NOHOST-V2-NOT: host-
+// NOHOST-V2-DAG: hip-amdgcn-amd-amdhsa--gfx900
+// NOHOST-V2-DAG: hip-amdgcn-amd-amdhsa--gfx906
+
 // Check -compression-level= option
 
 // RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \
diff --git a/clang/test/Driver/clang-offload-bundler-zstd.c b/clang/test/Driver/clang-offload-bundler-zstd.c
index 667d9554daec7..c1123ae5acb38 100644
--- a/clang/test/Driver/clang-offload-bundler-zstd.c
+++ b/clang/test/Driver/clang-offload-bundler-zstd.c
@@ -29,11 +29,11 @@
 // RUN: diff %t.tgt1 %t.res.tgt1
 // RUN: diff %t.tgt2 %t.res.tgt2
 //
-// CHECK: Compressed bundle format version: 2
+// CHECK: Compressed bundle format version: 3
 // CHECK: Total file size (including headers): [[SIZE:[0-9]*]] bytes
 // CHECK: Compression method used: zstd
 // CHECK: Compression level: 3
-// CHECK: Compressed bundle format version: 2
+// CHECK: Compressed bundle format version: 3
 // CHECK: Total file size (from header): [[SIZE]] bytes
 // CHECK: Decompression method: zstd
 // CHECK: Hashes match: Yes
diff --git a/clang/test/Driver/cuda-cross-compiling.c b/clang/test/Driver/cuda-cross-compiling.c
index 1df231ecb4479..6717ae82d0e70 100644
--- a/clang/test/Driver/cuda-cross-compiling.c
+++ b/clang/test/Driver/cuda-cross-compiling.c
@@ -104,3 +104,11 @@
 // RUN:   -nogpulib -nogpuinc -### %s 2>&1 | FileCheck -check-prefix=STARTUP %s
 
 // STARTUP: clang-nvlink-wrapper{{.*}}"-lc" "-lm" "{{.*}}crt1.o"
+
+//
+// Test cuda path handling
+//
+// RUN: %clang -target nvptx64-nvidia-cuda -march=sm_52 --cuda-path=%S/Inputs/CUDA/usr/local/cuda \
+// RUN:   -nogpulib -nogpuinc -### %s 2>&1 | FileCheck -check-prefix=PATH %s
+
+// PATH: clang-nvlink-wrapper{{.*}}"--cuda-path={{.*}}/Inputs/CUDA/usr/local/cuda"
diff --git a/clang/test/Driver/cuda-detect-path.cu b/clang/test/Driver/cuda-detect-path.cu
index 8d249bd4c96d5..ce42ed75f7a8d 100644
--- a/clang/test/Driver/cuda-detect-path.cu
+++ b/clang/test/Driver/cuda-detect-path.cu
@@ -1,5 +1,5 @@
 // This tests uses the PATH environment variable.
-// REQUIRES: !system-windows
+// REQUIRES: !system-windows, !system-cygwin
 
 // RUN: env PATH=%S/Inputs/CUDA/usr/local/cuda/bin \
 // RUN:    %clang -v --target=i386-unknown-linux --sysroot=%S/no-cuda-there \
diff --git a/clang/test/Driver/dxc_fspv_extension.hlsl b/clang/test/Driver/dxc_fspv_extension.hlsl
index f09f21e8a72c9..aeba9f56463f8 100644
--- a/clang/test/Driver/dxc_fspv_extension.hlsl
+++ b/clang/test/Driver/dxc_fspv_extension.hlsl
@@ -14,6 +14,14 @@
 // RUN: %clang_dxc -spirv -Tlib_6_7 -### %s -fspv-extension=SPV_TEST1 -fspv-extension=KHR -fspv-extension=SPV_TEST2 2>&1 | FileCheck %s -check-prefix=TEST3
 // TEST3: "-spirv-ext=+SPV_TEST1,KHR,+SPV_TEST2"
 
+// Merge KHR with other extensions.
+// RUN: %clang_dxc -spirv -Tlib_6_7 -### %s -fspv-extension=KHR -fspv-extension=SPV_TEST2 2>&1 | FileCheck %s -check-prefix=TEST4
+// TEST4: "-spirv-ext=KHR,+SPV_TEST2"
+
+// Convert DXC to a list of SPV extensions.
+// RUN: %clang_dxc -spirv -Tlib_6_7 -### %s -fspv-extension=DXC 2>&1 | FileCheck %s -check-prefix=TEST5
+// TEST5: "-spirv-ext={{(\+SPV_[a-zA-Z0-9_]+,?)+}}"
+
 // Check for the error message if the extension name is not properly formed.
 // RUN: not %clang_dxc -spirv -Tlib_6_7 -### %s -fspv-extension=KHR_BAD -fspv-extension=TEST1 -fspv-extension=SPV_GOOD -fspv-extension=TEST2 2>&1 | FileCheck %s -check-prefix=FAIL
 // FAIL: invalid value 'KHR_BAD' in '-fspv-extension'
diff --git a/clang/test/Driver/fsanitize.c b/clang/test/Driver/fsanitize.c
index fbe1fd72c84c6..263301ad4466a 100644
--- a/clang/test/Driver/fsanitize.c
+++ b/clang/test/Driver/fsanitize.c
@@ -794,6 +794,11 @@
 // RUN: not %clang --target=x86_64-linux-gnu -fsanitize=cfi-icall -fsanitize-cfi-icall-generalize-pointers -fsanitize-cfi-cross-dso -fvisibility=hidden -flto -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CFI-GENERALIZE-AND-CROSS-DSO
 // CHECK-CFI-GENERALIZE-AND-CROSS-DSO: error: invalid argument '-fsanitize-cfi-cross-dso' not allowed with '-fsanitize-cfi-icall-generalize-pointers'
 
+// RUN: %clang --target=x86_64-linux-gnu -fsanitize=kcfi -fsanitize-cfi-icall-generalize-pointers -fvisibility=hidden -flto -c -resource-dir=%S/Inputs/resource_dir %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-GENERALIZE-POINTERS
+// RUN: %clang --target=x86_64-linux-gnu -fsanitize=kcfi -fvisibility=hidden -flto -c -resource-dir=%S/Inputs/resource_dir %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-KCFI-GENERALIZE-POINTERS
+// CHECK-KCFI-GENERALIZE-POINTERS: -fsanitize-cfi-icall-generalize-pointers
+// CHECK-NO-KCFI-GENERALIZE-POINTERS-NOT: -fsanitize-cfi-icall-generalize-pointers
+
 // RUN: %clang --target=x86_64-linux-gnu -fsanitize=cfi-icall -fsanitize-cfi-canonical-jump-tables -fvisibility=hidden -flto -c -resource-dir=%S/Inputs/resource_dir %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CFI-CANONICAL-JUMP-TABLES
 // RUN: %clang --target=x86_64-linux-gnu -fsanitize=cfi-icall -fno-sanitize-cfi-canonical-jump-tables -fvisibility=hidden -flto -c %s -resource-dir=%S/Inputs/resource_dir -### 2>&1 | FileCheck %s --check-prefix=CHECK-NO-CFI-CANONICAL-JUMP-TABLES
 // RUN: %clang --target=x86_64-linux-gnu -fsanitize=cfi-icall -fvisibility=hidden -flto -c -resource-dir=%S/Inputs/resource_dir %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CFI-CANONICAL-JUMP-TABLES
diff --git a/clang/test/Driver/gcc-toolchain-libstdcxx.cpp b/clang/test/Driver/gcc-toolchain-libstdcxx.cpp
new file mode 100644
index 0000000000000..bd5b0af53ac92
--- /dev/null
+++ b/clang/test/Driver/gcc-toolchain-libstdcxx.cpp
@@ -0,0 +1,28 @@
+// UNSUPPORTED: system-windows
+
+// This file tests that the GCC installation directory detection takes
+// the libstdc++ includes into account.  In each directory
+// Inputs/gcc_toolchain_libstdcxx/gccX, the installation directory for
+// gcc X should be picked in the future since it is the directory with
+// the largest version number which also contains the libstdc++
+// include directory.
+
+// RUN: %clang --target=x86_64-linux-gnu --gcc-toolchain=%S/Inputs/gcc_toolchain_libstdcxx/gcc10/usr -v 2>&1 | FileCheck %s --check-prefix=GCC10
+// GCC10: clang: warning: future releases of the clang compiler will prefer GCC installations containing libstdc++ include directories; '[[PREFIX:.*gcc_toolchain_libstdcxx/gcc10/usr/lib/gcc/x86_64-linux-gnu]]/10' would be chosen over '[[PREFIX]]/12' [-Wgcc-install-dir-libstdcxx]
+// GCC10: Found candidate GCC installation: [[PREFIX]]/10
+// GCC10: Found candidate GCC installation: [[PREFIX]]/11
+// GCC10: Found candidate GCC installation: [[PREFIX]]/12
+// GCC10: Selected GCC installation: [[PREFIX]]/12
+
+// RUN: %clang --target=x86_64-linux-gnu --gcc-toolchain=%S/Inputs/gcc_toolchain_libstdcxx/gcc11/usr -v 2>&1 | FileCheck %s --check-prefix=ONLY_GCC11
+// ONLY_GCC11: clang: warning: future releases of the clang compiler will prefer GCC installations containing libstdc++ include directories; '[[PREFIX:.*gcc_toolchain_libstdcxx/gcc11/usr/lib/gcc/x86_64-linux-gnu]]/11' would be chosen over '[[PREFIX]]/12' [-Wgcc-install-dir-libstdcxx]
+// ONLY_GCC11: Found candidate GCC installation: [[PREFIX]]/10
+// ONLY_GCC11: Found candidate GCC installation: [[PREFIX]]/11
+// ONLY_GCC11: Found candidate GCC installation: [[PREFIX]]/12
+// ONLY_GCC11: Selected GCC installation: [[PREFIX]]/12
+
+// RUN: %clang --target=x86_64-linux-gnu --gcc-toolchain=%S/Inputs/gcc_toolchain_libstdcxx/gcc12/usr -v 2>&1 | FileCheck %s --check-prefix=GCC12
+// GCC12: Found candidate GCC installation: [[PREFIX:.*gcc_toolchain_libstdcxx/gcc12/usr/lib/gcc/x86_64-linux-gnu]]/10
+// GCC12: Found candidate GCC installation: [[PREFIX]]/11
+// GCC12: Found candidate GCC installation: [[PREFIX]]/12
+// GCC12: Selected GCC installation: [[PREFIX]]/12
diff --git a/clang/test/Driver/hip-macros.hip b/clang/test/Driver/hip-macros.hip
index bd93f9985a774..516e01a6c4743 100644
--- a/clang/test/Driver/hip-macros.hip
+++ b/clang/test/Driver/hip-macros.hip
@@ -27,21 +27,27 @@
 // RUN: %clang -E -dM --offload-arch=gfx906 --cuda-device-only -nogpuinc -nogpulib -mcumode \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-ON %s
 // RUN: %clang -E -dM --offload-arch=gfx906 --cuda-device-only -nogpuinc -nogpulib -mno-cumode \
-// RUN:   %s 2>&1 | FileCheck --check-prefixes=CUMODE-ON,WARN-CUMODE %s
+// RUN:   %s 2>&1 | FileCheck -DOFFLOAD_ARCH=gfx906 --check-prefixes=CUMODE-ON,WARN-CUMODE %s
 // RUN: %clang -E -dM --offload-arch=gfx1030 --cuda-device-only -nogpuinc -nogpulib \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-OFF %s
 // RUN: %clang -E -dM --offload-arch=gfx1030 --cuda-device-only -nogpuinc -nogpulib -mcumode \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-ON %s
 // RUN: %clang -E -dM --offload-arch=gfx1030 --cuda-device-only -nogpuinc -nogpulib -mno-cumode \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-OFF %s
+// RUN: %clang -E -dM --offload-arch=gfx1250 --cuda-device-only -nogpuinc -nogpulib \
+// RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-ON %s
+// RUN: %clang -E -dM --offload-arch=gfx1250 --cuda-device-only -nogpuinc -nogpulib -mcumode \
+// RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-ON %s
+// RUN: %clang -E -dM --offload-arch=gfx1250 --cuda-device-only -nogpuinc -nogpulib -mno-cumode \
+// RUN:   %s 2>&1 | FileCheck -DOFFLOAD_ARCH=gfx1250 --check-prefixes=CUMODE-ON,WARN-CUMODE %s
 
 // Check no duplicate warnings.
 // RUN: %clang -E -dM --offload-arch=gfx906 --cuda-device-only -nogpuinc -nogpulib -mcumode \
 // RUN:   -mno-cumode -mno-cumode \
-// RUN:   %s 2>&1 | FileCheck --check-prefixes=CUMODE-ON,WARN-CUMODE %s
+// RUN:   %s 2>&1 | FileCheck -DOFFLOAD_ARCH=gfx906 --check-prefixes=CUMODE-ON,WARN-CUMODE %s
 
-// WARN-CUMODE-DAG: warning: ignoring '-mno-cumode' option as it is not currently supported for processor 'gfx906' [-Woption-ignored]
-// WARN-CUMODE-NOT: warning: ignoring '-mno-cumode' option as it is not currently supported for processor 'gfx906' [-Woption-ignored]
+// WARN-CUMODE-DAG: warning: ignoring '-mno-cumode' option as it is not currently supported for processor '[[OFFLOAD_ARCH]]' [-Woption-ignored]
+// WARN-CUMODE-NOT: warning: ignoring '-mno-cumode' option as it is not currently supported for processor '[[OFFLOAD_ARCH]]' [-Woption-ignored]
 // CUMODE-ON-DAG: #define __AMDGCN_CUMODE__ 1
 // CUMODE-OFF-DAG: #define __AMDGCN_CUMODE__ 0
 
diff --git a/clang/test/Driver/hip-runtime-libs-msvc.hip b/clang/test/Driver/hip-runtime-libs-msvc.hip
index 943cd0569f4fd..d282a2646342a 100644
--- a/clang/test/Driver/hip-runtime-libs-msvc.hip
+++ b/clang/test/Driver/hip-runtime-libs-msvc.hip
@@ -10,4 +10,11 @@
 // RUN:   --rocm-path=%S/Inputs/rocm %s 2>&1 \
 // RUN:   | FileCheck %s
 
+// Test HIP runtime lib is linked even if -nostdlib is specified when the input
+// is a HIP file. This is important when composing with e.g. the UCRT or other
+// non glibc-like implementations of the C standard library.
+// RUN: %clang -### --target=x86_64-pc-windows-msvc -nogpuinc -nogpulib \
+// RUN:   -nostdlib --rocm-path=%S/Inputs/rocm %s 2>&1 \
+// RUN:   | FileCheck %s
+
 // CHECK: "-libpath:{{.*Inputs.*rocm.*lib}}" "amdhip64.lib"
diff --git a/clang/test/Driver/hipspv-toolchain.hip b/clang/test/Driver/hipspv-toolchain.hip
index b2187acbcd5ab..3c175ebf433cc 100644
--- a/clang/test/Driver/hipspv-toolchain.hip
+++ b/clang/test/Driver/hipspv-toolchain.hip
@@ -1,4 +1,4 @@
-// UNSUPPORTED: system-windows
+// UNSUPPORTED: system-windows, system-cygwin
 
 // RUN: %clang -### -target x86_64-linux-gnu --offload=spirv64 \
 // RUN:   --no-offload-new-driver --hip-path=%S/Inputs/hipspv -nohipwrapperinc %s \
diff --git a/clang/test/Driver/lanai-mcmodel.c b/clang/test/Driver/lanai-mcmodel.c
new file mode 100644
index 0000000000000..d62d7e1c37f24
--- /dev/null
+++ b/clang/test/Driver/lanai-mcmodel.c
@@ -0,0 +1,10 @@
+// RUN: %clang --target=lanai -### -c -mcmodel=small %s 2>&1 | FileCheck --check-prefix=SMALL %s
+// RUN: %clang --target=lanai -### -c -mcmodel=medium %s 2>&1 | FileCheck --check-prefix=MEDIUM %s
+// RUN: %clang --target=lanai -### -c -mcmodel=large %s 2>&1 | FileCheck --check-prefix=LARGE %s
+// RUN: not %clang --target=lanai -### -c -mcmodel=something %s 2>&1 | FileCheck --check-prefix=ERR-MCMODEL %s
+
+// SMALL: "-mcmodel=small"
+// MEDIUM: "-mcmodel=medium"
+// LARGE: "-mcmodel=large"
+
+// ERR-MCMODEL:  error: unsupported argument 'something' to option '-mcmodel=' for target 'lanai'
diff --git a/clang/test/Driver/ld-path.c b/clang/test/Driver/ld-path.c
index bc10b9e12e25a..e00b63d4eff5c 100644
--- a/clang/test/Driver/ld-path.c
+++ b/clang/test/Driver/ld-path.c
@@ -1,5 +1,5 @@
 /// This tests uses the PATH environment variable.
-// UNSUPPORTED: system-windows
+// UNSUPPORTED: system-windows, system-cygwin
 
 // RUN: cd %S
 
diff --git a/clang/test/Driver/linker-wrapper-libs.c b/clang/test/Driver/linker-wrapper-libs.c
deleted file mode 100644
index 1404fe3fc7a3e..0000000000000
--- a/clang/test/Driver/linker-wrapper-libs.c
+++ /dev/null
@@ -1,191 +0,0 @@
-// REQUIRES: x86-registered-target
-// REQUIRES: nvptx-registered-target
-// REQUIRES: amdgpu-registered-target
-
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.elf.o
-
-#if defined(RESOLVES)
-int __attribute__((visibility("hidden"))) sym;
-#elif defined(GLOBAL)
-int __attribute__((visibility("protected"))) global;
-#elif defined(WEAK)
-int __attribute__((visibility("hidden"))) weak;
-#elif defined(HIDDEN)
-int __attribute__((visibility("hidden"))) hidden;
-#elif defined(UNDEFINED)
-extern int sym;
-int baz() { return sym; }
-#else
-extern int sym;
-
-extern int __attribute__((weak)) weak;
-
-int foo() { return sym; }
-int bar() { return weak; }
-#endif
-
-//
-// Check that we extract a static library defining an undefined symbol.
-//
-// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -DRESOLVES -o %t.nvptx.resolves.bc
-// RUN: %clang -cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm-bc -DRESOLVES -o %t.amdgpu.resolves.bc
-// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -DUNDEFINED -o %t.nvptx.undefined.bc
-// RUN: %clang -cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm-bc -DUNDEFINED -o %t.amdgpu.undefined.bc
-// RUN: clang-offload-packager -o %t-lib.out \
-// RUN:   --image=file=%t.nvptx.undefined.bc,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.amdgpu.undefined.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 \
-// RUN:   --image=file=%t.nvptx.resolves.bc,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.amdgpu.resolves.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t-lib.out
-// RUN: llvm-ar rcs %t.a %t.o
-// RUN: clang-offload-packager -o %t.out \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
-// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
-// RUN:   --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \
-// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
-// RUN:   --linker-path=/usr/bin/ld %t.a %t.o -o a.out 2>&1 \
-// RUN: | FileCheck %s --check-prefix=LIBRARY-RESOLVES
-
-// LIBRARY-RESOLVES: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=nvptx64-nvidia-cuda -march=sm_70 {{.*}}.o {{.*}}.o
-// LIBRARY-RESOLVES: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o
-
-//
-// Check that we extract a static library that defines a global visibile to the
-// host.
-//
-// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -DGLOBAL -o %t.nvptx.global.bc
-// RUN: %clang -cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm-bc -DGLOBAL -o %t.amdgpu.global.bc
-// RUN: clang-offload-packager -o %t-lib.out \
-// RUN:   --image=file=%t.nvptx.global.bc,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.amdgpu.global.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t-lib.out
-// RUN: llvm-ar rcs %t.a %t.o
-// RUN: clang-offload-packager -o %t.out \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
-// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
-// RUN:   --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \
-// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
-// RUN:   --linker-path=/usr/bin/ld %t.a %t.o -o a.out 2>&1 \
-// RUN: | FileCheck %s --check-prefix=LIBRARY-GLOBAL
-
-// LIBRARY-GLOBAL: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=nvptx64-nvidia-cuda -march=sm_70 {{.*}}.o {{.*}}.o
-// LIBRARY-GLOBAL: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o
-
-//
-// Check that we do not extract a global symbol if the source file was not
-// created by an offloading language that expects there to be a host version of
-// the symbol.
-//
-// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -DGLOBAL -o %t.nvptx.global.bc
-// RUN: %clang -cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm-bc -DGLOBAL -o %t.amdgpu.global.bc
-// RUN: clang-offload-packager -o %t-lib.out \
-// RUN:   --image=file=%t.nvptx.global.bc,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.amdgpu.global.bc,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t-lib.out
-// RUN: llvm-ar rcs %t.a %t.o
-// RUN: clang-offload-packager -o %t.out \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
-// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
-// RUN:   --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \
-// RUN: | FileCheck %s --check-prefix=LIBRARY-GLOBAL-NONE
-
-// LIBRARY-GLOBAL-NONE-NOT: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o
-// LIBRARY-GLOBAL-NONE-NOT: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=nvptx64-nvidia-cuda -march=sm_70 {{.*}}.o {{.*}}.o
-
-//
-// Check that we do not extract an external weak symbol.
-//
-// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -DWEAK -o %t.nvptx.weak.bc
-// RUN: %clang -cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm-bc -DWEAK -o %t.amdgpu.weak.bc
-// RUN: clang-offload-packager -o %t-lib.out \
-// RUN:   --image=file=%t.nvptx.weak.bc,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.amdgpu.weak.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t-lib.out
-// RUN: llvm-ar rcs %t.a %t.o
-// RUN: clang-offload-packager -o %t.out \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
-// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
-// RUN:   --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \
-// RUN: | FileCheck %s --check-prefix=LIBRARY-WEAK
-
-// LIBRARY-WEAK: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=nvptx64-nvidia-cuda -march=sm_70
-// LIBRARY-WEAK-NOT: {{.*}}.o {{.*}}.o
-// LIBRARY-WEAK: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=amdgcn-amd-amdhsa -mcpu=gfx1030
-
-//
-// Check that we do not extract an unneeded hidden symbol.
-//
-// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -DHIDDEN -o %t.nvptx.hidden.bc
-// RUN: %clang -cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm-bc -DHIDDEN -o %t.amdgpu.hidden.bc
-// RUN: clang-offload-packager -o %t-lib.out \
-// RUN:   --image=file=%t.nvptx.hidden.bc,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.amdgpu.hidden.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t-lib.out
-// RUN: llvm-ar rcs %t.a %t.o
-// RUN: clang-offload-packager -o %t.out \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
-// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
-// RUN:   --linker-path=/usr/bin/ld %t.o %t.a -o a.out 2>&1 \
-// RUN: | FileCheck %s --check-prefix=LIBRARY-HIDDEN
-
-// LIBRARY-HIDDEN: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=nvptx64-nvidia-cuda -march=sm_70
-// LIBRARY-HIDDEN-NOT: {{.*}}.o {{.*}}.o
-// LIBRARY-HIDDEN: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=amdgcn-amd-amdhsa -mcpu=gfx1030
-
-//
-// Check that we do not extract a static library that defines a global visibile
-// to the host that is already defined.
-//
-// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -DGLOBAL -o %t.nvptx.global.bc
-// RUN: %clang -cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm-bc -DGLOBAL -o %t.amdgpu.global.bc
-// RUN: clang-offload-packager -o %t-lib.out \
-// RUN:   --image=file=%t.nvptx.global.bc,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.amdgpu.global.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t-lib.out
-// RUN: llvm-ar rcs %t.a %t.o
-// RUN: clang-offload-packager -o %t.out \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
-// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
-// RUN:   --linker-path=/usr/bin/ld %t.o %t.a %t.a -o a.out 2>&1 \
-// RUN: | FileCheck %s --check-prefix=LIBRARY-GLOBAL-DEFINED
-
-// LIBRARY-GLOBAL-DEFINED: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=nvptx64-nvidia-cuda -march=sm_70 {{.*}}.o {{.*}}.o
-// LIBRARY-GLOBAL-DEFINED-NOT: {{.*}}gfx1030{{.*}}.o
-// LIBRARY-GLOBAL-DEFINED: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o
-
-//
-// Check that we can use --[no-]whole-archive to control extraction.
-//
-// RUN: %clang -cc1 %s -triple nvptx64-nvidia-cuda -emit-llvm-bc -DGLOBAL -o %t.nvptx.global.bc
-// RUN: %clang -cc1 %s -triple amdgcn-amd-amdhsa -emit-llvm-bc -DGLOBAL -o %t.amdgpu.global.bc
-// RUN: clang-offload-packager -o %t-lib.out \
-// RUN:   --image=file=%t.nvptx.global.bc,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.nvptx.global.bc,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_52 \
-// RUN:   --image=file=%t.amdgpu.global.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 \
-// RUN:   --image=file=%t.amdgpu.global.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx90a
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t-lib.out
-// RUN: llvm-ar rcs %t.a %t.o
-// RUN: clang-offload-packager -o %t.out \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=nvptx64-nvidia-cuda,arch=sm_70 \
-// RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
-// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
-// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
-// RUN:   --linker-path=/usr/bin/ld %t.o --whole-archive %t.a -o a.out 2>&1 \
-// RUN: | FileCheck %s --check-prefix=LIBRARY-WHOLE-ARCHIVE
-
-// LIBRARY-WHOLE-ARCHIVE: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=nvptx64-nvidia-cuda -march=sm_70 {{.*}}.o {{.*}}.o
-// LIBRARY-WHOLE-ARCHIVE: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=amdgcn-amd-amdhsa -mcpu=gfx1030 {{.*}}.o {{.*}}.o
-// LIBRARY-WHOLE-ARCHIVE: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=nvptx64-nvidia-cuda -march=sm_52 {{.*}}.o
-// LIBRARY-WHOLE-ARCHIVE: clang{{.*}} -o {{.*}}.img -dumpdir {{.*}}.img. --target=amdgcn-amd-amdhsa -mcpu=gfx90a {{.*}}.o
diff --git a/clang/test/Driver/modules-driver-cxx20-module-usage-scanner.cpp b/clang/test/Driver/modules-driver-cxx20-module-usage-scanner.cpp
new file mode 100644
index 0000000000000..a434587a78759
--- /dev/null
+++ b/clang/test/Driver/modules-driver-cxx20-module-usage-scanner.cpp
@@ -0,0 +1,192 @@
+// The driver never checks to implicitly enable the explicit module build 
+// support unless at least two input files are provided.
+// To trigger the C++20 module usage check, we always pass a second dummy file
+// as input.
+// TODO: Remove -fmodules everywhere once implicitly enabled explicit module 
+// builds are supported.
+
+// RUN: split-file %s %t
+//--- empty.cpp
+// Nothing here
+
+//--- only-global.cpp
+// RUN: %clang -std=c++20 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/only-global.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK1
+// CHECK1: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+module;
+
+//--- only-import.cpp
+// RUN: %clang -std=c++20 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/only-import.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK2
+// CHECK2: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+import A;
+
+//--- only-export.cpp
+// RUN: %clang -std=c++20 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/only-export.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK3
+// CHECK3: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+export module A;
+
+//--- leading-line-comment.cpp
+// RUN: %clang -std=c++20 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/leading-line-comment.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK4
+// CHECK4: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+// My line comment
+import A;
+
+//--- leading-block-comment1.cpp
+// RUN: %clang -std=c++20 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/leading-block-comment1.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK5
+// CHECK5: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+/*My block comment */
+import A;
+
+//--- leading-block-comment2.cpp
+// RUN: %clang -std=c++20 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/leading-block-comment2.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK6
+// CHECK6: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+/*My line comment */ import A;
+
+//--- inline-block-comment1.cpp
+// RUN: %clang -std=c++20 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/leading-block-comment1.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK7
+// CHECK7: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+export/*a comment*/module/*another comment*/A;
+
+//--- inline-block-comment2.cpp
+// RUN: %clang -std=c++20 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/leading-block-comment2.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK8
+// CHECK8: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+module/*a comment*/;
+
+//--- leading-directives.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/leading-directives.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK9
+// CHECK9: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+#define A
+#undef A
+#if A
+#ifdef A
+#elifdef A
+#elifndef A
+#endif
+#ifndef A
+#elif A
+#else
+#endif
+#endif
+#pragma once;
+#include <iostream>
+import m;
+
+//--- multiline-directive.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/multiline-directive.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK10
+// CHECK10: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+#define MACRO(a,  \
+              b)  \
+        call((a), \
+             (b)
+import a;
+
+//--- leading-line-splice.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/leading-line-splice.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK11
+// CHECK11: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+\
+module;
+
+//--- leading-line-splice-trailing-whitespace.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/leading-line-splice-trailing-whitespace.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK12
+// CHECK12: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+// v This backslash has trailing whitespace.
+   \      
+export module A;
+
+//--- comment-line-splice.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/comment-line-splice.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s  --allow-empty --check-prefix=CHECK13
+// CHECK13-NOT: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+// My comment continues next-line!\
+import A;
+
+//--- comment-line-splice-trailing-whitespace.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/comment-line-splice-trailing-whitespace.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --allow-empty --check-prefix=CHECK14
+// CHECK14-NOT: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+// My comment continues next-line! This backslash has trailing whitespace. -> \   
+module;
+
+//--- line-splice-in-directive1.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/line-splice-in-directive1.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK15
+// CHECK15: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+
+module\
+;
+
+//--- line-splice-in-directive2.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/line-splice-in-directive2.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK16
+// CHECK16: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+
+export\
+  module\
+  A;
+
+//--- no-module-usage1.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/no-module-usage1.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s  --allow-empty --check-prefix=CHECK17
+// CHECK17-NOT: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+auto main() -> int {}
+
+//--- no-module-usage2.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/no-module-usage2.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s  --allow-empty --check-prefix=CHECK18
+// CHECK18-NOT: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+moduleStruct{};
+
+//--- no-module-usage3.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/no-module-usage3.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s  --allow-empty --check-prefix=CHECK19
+// CHECK19-NOT: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+export_struct{};
+
+//--- no-module-usage-namespace-import.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/no-module-usage-namespace-import.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s  --allow-empty --check-prefix=CHECK20
+// CHECK20-NOT: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+import::inner xi = {};
+
+//--- no-module-usage-namespace-module.cpp
+// RUN: %clang -std=c++23 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   %t/no-module-usage-namespace-module.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s  --allow-empty --check-prefix=CHECK21
+// CHECK21-NOT: remark: found C++20 module usage in file '{{.*}}' [-Rmodules-driver]
+module::inner yi = {};
+
+// RUN: not %clang -std=c++20 -ccc-print-phases -fmodules-driver -Rmodules-driver \
+// RUN:   imaginary-file.cpp %t/empty.cpp 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-NON-EXISTING-FILE-ERR
+// CHECK-NON-EXISTING-FILE-ERR: clang: error: no such file or directory: 'imaginary-file.cpp'
diff --git a/clang/test/Driver/print-supported-extensions-riscv.c b/clang/test/Driver/print-supported-extensions-riscv.c
index 2503f2473d64a..f9a4929017d0e 100644
--- a/clang/test/Driver/print-supported-extensions-riscv.c
+++ b/clang/test/Driver/print-supported-extensions-riscv.c
@@ -128,6 +128,7 @@
 // CHECK-NEXT:     smcdeleg             1.0       'Smcdeleg' (Counter Delegation Machine Level)
 // CHECK-NEXT:     smcntrpmf            1.0       'Smcntrpmf' (Cycle and Instret Privilege Mode Filtering)
 // CHECK-NEXT:     smcsrind             1.0       'Smcsrind' (Indirect CSR Access Machine Level)
+// CHECK-NEXT:     smctr                1.0       'Smctr' (Control Transfer Records Machine Level)
 // CHECK-NEXT:     smdbltrp             1.0       'Smdbltrp' (Double Trap Machine Level)
 // CHECK-NEXT:     smepmp               1.0       'Smepmp' (Enhanced Physical Memory Protection)
 // CHECK-NEXT:     smmpm                1.0       'Smmpm' (Machine-level Pointer Masking for M-mode)
@@ -140,6 +141,7 @@
 // CHECK-NEXT:     sscofpmf             1.0       'Sscofpmf' (Count Overflow and Mode-Based Filtering)
 // CHECK-NEXT:     sscounterenw         1.0       'Sscounterenw' (Support writeable scounteren enable bit for any hpmcounter that is not read-only zero)
 // CHECK-NEXT:     sscsrind             1.0       'Sscsrind' (Indirect CSR Access Supervisor Level)
+// CHECK-NEXT:     ssctr                1.0       'Ssctr' (Control Transfer Records Supervisor Level)
 // CHECK-NEXT:     ssdbltrp             1.0       'Ssdbltrp' (Double Trap Supervisor Level)
 // CHECK-NEXT:     ssnpm                1.0       'Ssnpm' (Supervisor-level Pointer Masking for next lower privilege mode)
 // CHECK-NEXT:     sspm                 1.0       'Sspm' (Indicates Supervisor-mode Pointer Masking)
@@ -192,6 +194,7 @@
 // CHECK-NEXT:     xsfvqmaccqoq         1.0       'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4))
 // CHECK-NEXT:     xsifivecdiscarddlone 1.0       'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction)
 // CHECK-NEXT:     xsifivecflushdlone   1.0       'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction)
+// CHECK-NEXT:     xsmtvdot             1.0       'XSMTVDot' (SpacemiT Vector Dot Product Extension)
 // CHECK-NEXT:     xtheadba             1.0       'XTHeadBa' (T-Head address calculation instructions)
 // CHECK-NEXT:     xtheadbb             1.0       'XTHeadBb' (T-Head basic bit-manipulation instructions)
 // CHECK-NEXT:     xtheadbs             1.0       'XTHeadBs' (T-Head single-bit instructions)
@@ -207,15 +210,13 @@
 // CHECK-NEXT:     xwchc                2.2       'Xwchc' (WCH/QingKe additional compressed opcodes)
 // CHECK-EMPTY:
 // CHECK-NEXT: Experimental extensions
-// CHECK-NEXT:     p                    0.14      'P' ('Base P' (Packed SIMD))
+// CHECK-NEXT:     p                    0.15      'P' ('Base P' (Packed SIMD))
 // CHECK-NEXT:     zicfilp              1.0       'Zicfilp' (Landing pad)
 // CHECK-NEXT:     zicfiss              1.0       'Zicfiss' (Shadow stack)
 // CHECK-NEXT:     zalasr               0.1       'Zalasr' (Load-Acquire and Store-Release Instructions)
 // CHECK-NEXT:     zvbc32e              0.7       'Zvbc32e' (Vector Carryless Multiplication with 32-bits elements)
 // CHECK-NEXT:     zvkgs                0.7       'Zvkgs' (Vector-Scalar GCM instructions for Cryptography)
 // CHECK-NEXT:     zvqdotq              0.0       'Zvqdotq' (Vector quad widening 4D Dot Product)
-// CHECK-NEXT:     smctr                1.0       'Smctr' (Control Transfer Records Machine Level)
-// CHECK-NEXT:     ssctr                1.0       'Ssctr' (Control Transfer Records Supervisor Level)
 // CHECK-NEXT:     svukte               0.3       'Svukte' (Address-Independent Latency of User-Mode Faults to Supervisor Addresses)
 // CHECK-NEXT:     xqccmp               0.3       'Xqccmp' (Qualcomm 16-bit Push/Pop and Double Moves)
 // CHECK-NEXT:     xqcia                0.7       'Xqcia' (Qualcomm uC Arithmetic Extension)
diff --git a/clang/test/Driver/program-path-priority.c b/clang/test/Driver/program-path-priority.c
index c940c4ced9442..b88c0f29f7f33 100644
--- a/clang/test/Driver/program-path-priority.c
+++ b/clang/test/Driver/program-path-priority.c
@@ -1,5 +1,5 @@
 /// Don't create symlinks on Windows
-// UNSUPPORTED: system-windows
+// UNSUPPORTED: system-windows, system-cygwin
 
 /// Check the priority used when searching for tools
 /// Names and locations are usually in this order:
diff --git a/clang/test/Driver/riscv-cpus.c b/clang/test/Driver/riscv-cpus.c
index 2698612f815d3..ea0821cc39c45 100644
--- a/clang/test/Driver/riscv-cpus.c
+++ b/clang/test/Driver/riscv-cpus.c
@@ -157,6 +157,7 @@
 // MCPU-SPACEMIT-X60-SAME: "-target-feature" "+svinval"
 // MCPU-SPACEMIT-X60-SAME: "-target-feature" "+svnapot"
 // MCPU-SPACEMIT-X60-SAME: "-target-feature" "+svpbmt"
+// MCPU-SPACEMIT-X60-SAME: "-target-feature" "+xsmtvdot"
 // MCPU-SPACEMIT-X60-SAME: "-target-abi" "lp64d"
 
 // We cannot check much for -mcpu=native, but it should be replaced by a valid CPU string.
@@ -403,6 +404,11 @@
 // MCPU-MARCH: "-nostdsysteminc" "-target-cpu" "sifive-e31" "-target-feature" "+m" "-target-feature" "+c"
 // MCPU-MARCH: "-target-abi" "ilp32"
 
+// -march=unset erases previous march
+// RUN: %clang --target=riscv32 -### -c %s 2>&1 -march=rv32imc -march=unset -mcpu=sifive-e31 | FileCheck -check-prefix=MARCH-UNSET %s
+// MARCH-UNSET: "-nostdsysteminc" "-target-cpu" "sifive-e31" "-target-feature" "+m" "-target-feature" "+a" "-target-feature" "+c"
+// MARCH-UNSET-SAME: "-target-abi" "ilp32"
+
 // Check interaction between -mcpu and mtune, -mtune won't affect arch related
 // target feature, but -mcpu will.
 //
diff --git a/clang/test/Driver/riscv32-toolchain.c b/clang/test/Driver/riscv32-toolchain.c
index 8cf20aa592a3a..04acf1e7edbe0 100644
--- a/clang/test/Driver/riscv32-toolchain.c
+++ b/clang/test/Driver/riscv32-toolchain.c
@@ -215,7 +215,7 @@
 
 // RUN: %clang --target=riscv32 %s -emit-llvm -S -o - | FileCheck %s
 
-// Check that "--no-relax" is forwarded to the linker for RISC-V.
+/// Check that "--no-relax" is forwarded to the linker for RISC-V.
 // RUN: env "PATH=" %clang %s -### 2>&1 -mno-relax \
 // RUN:   --target=riscv32-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv32imac -mabi=lp32\
@@ -223,7 +223,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-RV32-NORELAX %s
 // CHECK-RV32-NORELAX: "--no-relax"
 
-// Check that "--no-relax" is not forwarded to the linker for RISC-V.
+/// Check that "--no-relax" is not forwarded to the linker for RISC-V.
 // RUN:env "PATH=" %clang %s -### 2>&1 \
 // RUN:   --target=riscv32-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv32imac -mabi=lp32\
@@ -231,7 +231,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-RV32-RELAX %s
 // CHECK-RV32-RELAX-NOT: "--no-relax"
 
-// Check that "--no-relax" is forwarded to the linker for RISC-V (Gnu.cpp).
+/// Check that "--no-relax" is forwarded to the linker for RISC-V (Gnu.cpp).
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld -no-pie -mno-relax \
 // RUN:   --target=riscv32-unknown-linux-gnu --rtlib=platform --unwindlib=platform -mabi=ilp32 \
 // RUN:   --gcc-toolchain=%S/Inputs/multilib_riscv_linux_sdk \
@@ -239,7 +239,7 @@
 // RUN:   | FileCheck -check-prefix=CHECK-RV32-GNU-NORELAX %s
 // CHECK-RV32-GNU-NORELAX: "--no-relax"
 
-// Check that "--no-relax" is not forwarded to the linker for RISC-V (Gnu.cpp).
+/// Check that "--no-relax" is not forwarded to the linker for RISC-V (Gnu.cpp).
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld -no-pie \
 // RUN:   --target=riscv32-unknown-linux-gnu --rtlib=platform --unwindlib=platform -mabi=ilp32 \
 // RUN:   --gcc-toolchain=%S/Inputs/multilib_riscv_linux_sdk \
@@ -247,6 +247,19 @@
 // RUN:   | FileCheck -check-prefix=CHECK-RV32-GNU-RELAX %s
 // CHECK-RV32-GNU-RELAX-NOT: "--no-relax"
 
+/// Check that "-static -pie" is forwarded to linker when "-static-pie" is used
+// RUN: %clang -static-pie -### %s -fuse-ld= \
+// RUN:   --target=riscv32-unknown-elf -rtlib=platform --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_riscv32_tree \
+// RUN:   --sysroot=%S/Inputs/basic_riscv32_tree/riscv32-unknown-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-RV32-STATIC-PIE %s
+
+// C-RV32-STATIC-PIE: "-Bstatic" "-pie" "--no-dynamic-linker" "-z" "text" "-m" "elf32lriscv" "-X"
+// C-RV32-STATIC-PIE: "{{.*}}rcrt1.o"
+// C-RV32-STATIC-PIE: "{{.*}}crtbeginS.o"
+// C-RV32-STATIC-PIE: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
+// C-RV32-STATIC-PIE: "{{.*}}crtendS.o"
+
 typedef __builtin_va_list va_list;
 typedef __SIZE_TYPE__ size_t;
 typedef __PTRDIFF_TYPE__ ptrdiff_t;
diff --git a/clang/test/Driver/riscv64-toolchain.c b/clang/test/Driver/riscv64-toolchain.c
index 1550f46af8c9c..378f3d9db7bad 100644
--- a/clang/test/Driver/riscv64-toolchain.c
+++ b/clang/test/Driver/riscv64-toolchain.c
@@ -171,7 +171,7 @@
 
 // RUN: %clang --target=riscv64 %s -emit-llvm -S -o - | FileCheck %s
 
-// Check that "--no-relax" is forwarded to the linker for RISC-V.
+/// Check that "--no-relax" is forwarded to the linker for RISC-V.
 // RUN: env "PATH=" %clang %s -### 2>&1 -mno-relax \
 // RUN:   --target=riscv64-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv64imac -mabi=lp64\
@@ -179,7 +179,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-RV64-NORELAX %s
 // CHECK-RV64-NORELAX: "--no-relax"
 
-// Check that "--no-relax" is not forwarded to the linker for RISC-V.
+/// Check that "--no-relax" is not forwarded to the linker for RISC-V.
 // RUN:env "PATH=" %clang %s -### 2>&1 \
 // RUN:   --target=riscv64-unknown-elf --rtlib=platform --unwindlib=platform --sysroot= \
 // RUN:   -march=rv64imac -mabi=lp64\
@@ -187,7 +187,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-RV64-RELAX %s
 // CHECK-RV64-RELAX-NOT: "--no-relax"
 
-// Check that "--no-relax" is forwarded to the linker for RISC-V (Gnu.cpp).
+/// Check that "--no-relax" is forwarded to the linker for RISC-V (Gnu.cpp).
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld -no-pie -mno-relax \
 // RUN:   --target=riscv64-unknown-linux-gnu --rtlib=platform --unwindlib=platform -mabi=lp64 \
 // RUN:   --gcc-toolchain=%S/Inputs/multilib_riscv_linux_sdk \
@@ -195,7 +195,7 @@
 // RUN:   | FileCheck -check-prefix=CHECK-RV64-GNU-NORELAX %s
 // CHECK-RV64-GNU-NORELAX: "--no-relax"
 
-// Check that "--no-relax" is not forwarded to the linker for RISC-V (Gnu.cpp).
+/// Check that "--no-relax" is not forwarded to the linker for RISC-V (Gnu.cpp).
 // RUN: env "PATH=" %clang -### %s -fuse-ld=ld -no-pie \
 // RUN:   --target=riscv64-unknown-linux-gnu --rtlib=platform --unwindlib=platform -mabi=lp64 \
 // RUN:   --gcc-toolchain=%S/Inputs/multilib_riscv_linux_sdk \
@@ -203,6 +203,19 @@
 // RUN:   | FileCheck -check-prefix=CHECK-RV64-GNU-RELAX %s
 // CHECK-RV64-GNU-RELAX-NOT: "--no-relax"
 
+/// Check that "-static -pie" is forwarded to linker when "-static-pie" is used
+// RUN: %clang -static-pie -### %s -fuse-ld= \
+// RUN:   --target=riscv64-unknown-elf -rtlib=platform --unwindlib=platform \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_riscv64_tree \
+// RUN:   --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-RV64-STATIC-PIE %s
+
+// C-RV64-STATIC-PIE: "-Bstatic" "-pie" "--no-dynamic-linker" "-z" "text" "-m" "elf64lriscv" "-X"
+// C-RV64-STATIC-PIE: "{{.*}}rcrt1.o"
+// C-RV64-STATIC-PIE: "{{.*}}crtbeginS.o"
+// C-RV64-STATIC-PIE: "--start-group" "-lgcc" "-lc" "-lgloss" "--end-group"
+// C-RV64-STATIC-PIE: "{{.*}}crtendS.o"
+
 typedef __builtin_va_list va_list;
 typedef __SIZE_TYPE__ size_t;
 typedef __PTRDIFF_TYPE__ ptrdiff_t;
diff --git a/clang/test/Driver/spirv-openmp-toolchain.c b/clang/test/Driver/spirv-openmp-toolchain.c
index a61f3bc2399eb..1542f500971fa 100644
--- a/clang/test/Driver/spirv-openmp-toolchain.c
+++ b/clang/test/Driver/spirv-openmp-toolchain.c
@@ -60,3 +60,12 @@
 // RUN:        --libomptarget-spirv-bc-path=%t/ -nogpulib %s 2>&1 \
 // RUN: | FileCheck %s --check-prefix=CHECK-OFFLOAD-ARCH-ERROR
 // CHECK-OFFLOAD-ARCH-ERROR: error: failed to deduce triple for target architecture 'spirv64-intel'; specify the triple using '-fopenmp-targets' and '-Xopenmp-target' instead
+
+// RUN: %clang -mllvm --spirv-ext=+SPV_INTEL_function_pointers -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=spirv64-intel \
+// RUN:        -nogpulib %s 2>&1 \
+// RUN: | FileCheck %s --check-prefix=CHECK-LINKER-ARG
+// CHECK-LINKER-ARG: clang-linker-wrapper
+// CHECK-LINKER-ARG-NOT: --device-linker=spirv64
+// CHECK-LINKER-ARG-NOT: -mllvm
+// CHECK-LINKER-ARG-NOT: --spirv-ext=+SPV_INTEL_function_pointers
+// CHECK-LINKER-ARG: --linker-path
diff --git a/clang/test/Driver/spirv-toolchain.cl b/clang/test/Driver/spirv-toolchain.cl
index 53e8455e3d3dd..54c794c77ea04 100644
--- a/clang/test/Driver/spirv-toolchain.cl
+++ b/clang/test/Driver/spirv-toolchain.cl
@@ -92,7 +92,7 @@
 // RUN: mkdir -p %t/versioned
 // RUN: touch %t/versioned/spirv-as-%llvm-version-major \
 // RUN:   && chmod +x %t/versioned/spirv-as-%llvm-version-major
-// RUN: %if !system-windows %{ env "PATH=%t/versioned" %clang -### --target=spirv64 -x cl -c --save-temps %s 2>&1 \
+// RUN: %if !system-windows && !system-cygwin %{ env "PATH=%t/versioned" %clang -### --target=spirv64 -x cl -c --save-temps %s 2>&1 \
 // RUN:   | FileCheck -DVERSION=%llvm-version-major --check-prefix=VERSIONED %s %}
 
 // VERSIONED: {{.*}}spirv-as-[[VERSION]]
diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c
index e83b4f3ee78e1..7ec453108b264 100644
--- a/clang/test/Driver/x86-target-features.c
+++ b/clang/test/Driver/x86-target-features.c
@@ -476,8 +476,8 @@
 // RUN: %clang --target=x86_64-unknown-linux-gnu -mno-apxf -mapxf %s -### -o %t.o 2>&1 | FileCheck -check-prefix=APXF %s
 // RUN: %clang --target=x86_64-unknown-linux-gnu -mapxf -mno-apxf %s -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-APXF %s
 //
-// APXF: "-target-feature" "+egpr" "-target-feature" "+push2pop2" "-target-feature" "+ppx" "-target-feature" "+ndd" "-target-feature" "+ccmp" "-target-feature" "+nf" "-target-feature" "+cf" "-target-feature" "+zu"
-// NO-APXF: "-target-feature" "-egpr" "-target-feature" "-push2pop2" "-target-feature" "-ppx" "-target-feature" "-ndd" "-target-feature" "-ccmp" "-target-feature" "-nf" "-target-feature" "-cf" "-target-feature" "-zu"
+// APXF: "-target-feature" "+egpr" "-target-feature" "+push2pop2" "-target-feature" "+ppx" "-target-feature" "+ndd" "-target-feature" "+ccmp" "-target-feature" "+nf" "-target-feature" "+zu"
+// NO-APXF: "-target-feature" "-egpr" "-target-feature" "-push2pop2" "-target-feature" "-ppx" "-target-feature" "-ndd" "-target-feature" "-ccmp" "-target-feature" "-nf" "-target-feature" "-zu"
 
 // RUN: %clang --target=x86_64-unknown-linux-gnu -mapx-features=egpr %s -### -o %t.o 2>&1 | FileCheck -check-prefix=EGPR %s
 // RUN: %clang --target=x86_64-unknown-linux-gnu -mapx-features=push2pop2 %s -### -o %t.o 2>&1 | FileCheck -check-prefix=PUSH2POP2 %s
diff --git a/clang/test/FixIt/fixit-enum-scoped.cpp b/clang/test/FixIt/fixit-enum-scoped.cpp
new file mode 100644
index 0000000000000..b7e4138b7d216
--- /dev/null
+++ b/clang/test/FixIt/fixit-enum-scoped.cpp
@@ -0,0 +1,95 @@
+// RUN: not %clang_cc1 -fsyntax-only -fdiagnostics-parseable-fixits -std=c++20 -triple x86_64-apple-darwin %s 2>&1 | FileCheck %s
+
+namespace GH24265 {
+  enum class E_int { e };
+  enum class E_long : long { e };
+
+  void f() {
+    E_int::e + E_long::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:5-[[@LINE]]:5}:"static_cast<int>("
+                          // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:12-[[@LINE-1]]:12}:")"
+                          // CHECK: fix-it:{{.*}}:{[[@LINE-2]]:16-[[@LINE-2]]:16}:"static_cast<long>("
+                          // CHECK: fix-it:{{.*}}:{[[@LINE-3]]:24-[[@LINE-3]]:24}:")"
+    E_int::e + 0; // CHECK: fix-it:{{.*}}:{[[@LINE]]:5-[[@LINE]]:5}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:12-[[@LINE-1]]:12}:")"
+
+    0 * E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 / E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 % E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 + E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 - E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 << E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    0 >> E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    0 <=> E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:11-[[@LINE]]:11}:"static_cast<int>("
+                    // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:18-[[@LINE-1]]:18}:")"
+    0 < E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 > E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 <= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    0 >= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    0 == E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    0 != E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    0 & E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 ^ E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 | E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:9-[[@LINE]]:9}:"static_cast<int>("
+                  // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:16-[[@LINE-1]]:16}:")"
+    0 && E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    0 || E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+
+    int a;
+    a *= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    a /= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    a %= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    a += E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    a -= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    a <<= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:11-[[@LINE]]:11}:"static_cast<int>("
+                    // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:18-[[@LINE-1]]:18}:")"
+    a >>= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:11-[[@LINE]]:11}:"static_cast<int>("
+                    // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:18-[[@LINE-1]]:18}:")"
+    a &= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    a ^= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+    a |= E_int::e; // CHECK: fix-it:{{.*}}:{[[@LINE]]:10-[[@LINE]]:10}:"static_cast<int>("
+                   // CHECK: fix-it:{{.*}}:{[[@LINE-1]]:17-[[@LINE-1]]:17}:")"
+
+    // TODO: These do not have the diagnostic yet
+    E_int b;
+    b *= 0;
+    b /= 0;
+    b %= 0;
+    b += 0;
+    b -= 0;
+    b <<= 0;
+    b >>= 0;
+    b &= 0;
+    b ^= 0;
+    b |= 0;
+
+    a = E_int::e;
+    b = 0;
+
+    E_int c = 0;
+    int d = E_int::e;
+  }
+}
diff --git a/clang/test/Frontend/skip-function-bodies.cpp b/clang/test/Frontend/skip-function-bodies.cpp
index d0593b474bda2..4cfc4c5bfff3c 100644
--- a/clang/test/Frontend/skip-function-bodies.cpp
+++ b/clang/test/Frontend/skip-function-bodies.cpp
@@ -1,13 +1,15 @@
 // Trivial check to ensure skip-function-bodies flag is propagated.
 //
-// RUN: %clang_cc1 -verify -skip-function-bodies -pedantic-errors %s
-// expected-no-diagnostics
+// RUN: %clang_cc1 -verify -skip-function-bodies %s
 
 int f() {
   // normally this should emit some diags, but we're skipping it!
   this is garbage;
 }
 
+void g() __attribute__((__diagnose_if__(baz))) {}
+// expected-error@-1 {{use of undeclared identifier 'baz'}}
+
 // Make sure we only accept it as a cc1 arg.
 // RUN: not %clang -skip-function-bodies %s 2>&1 | FileCheck %s
 // CHECK: clang: error: unknown argument '-skip-function-bodies'; did you mean '-Xclang -skip-function-bodies'?
diff --git a/clang/test/Headers/__clang_hip_cmath.hip b/clang/test/Headers/__clang_hip_cmath.hip
index fcd74996e5fa4..2e0b776c4bfe5 100644
--- a/clang/test/Headers/__clang_hip_cmath.hip
+++ b/clang/test/Headers/__clang_hip_cmath.hip
@@ -124,12 +124,12 @@ namespace user_namespace {
 // DEFAULT-NEXT:    [[B:%.*]] = alloca [[STRUCT_USER_BFLOAT16]], align 1, addrspace(5)
 // DEFAULT-NEXT:    [[A_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
 // DEFAULT-NEXT:    [[B_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[B]] to ptr
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 1, ptr addrspace(5) [[A]]) #[[ATTR11:[0-9]+]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[A]]) #[[ATTR11:[0-9]+]]
 // DEFAULT-NEXT:    call void @_ZN13user_bfloat16C1Ef(ptr noundef nonnull align 1 dereferenceable(1) [[A_ASCAST]], float noundef 1.000000e+00) #[[ATTR10]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 1, ptr addrspace(5) [[B]]) #[[ATTR11]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[B]]) #[[ATTR11]]
 // DEFAULT-NEXT:    call void @_ZN13user_bfloat16C1Ef(ptr noundef nonnull align 1 dereferenceable(1) [[B_ASCAST]], float noundef 2.000000e+00) #[[ATTR10]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 1, ptr addrspace(5) [[B]]) #[[ATTR11]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 1, ptr addrspace(5) [[A]]) #[[ATTR11]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[B]]) #[[ATTR11]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[A]]) #[[ATTR11]]
 // DEFAULT-NEXT:    ret void
 //
 // FINITEONLY-LABEL: @_ZN14user_namespace8test_fmaEv(
@@ -138,12 +138,12 @@ namespace user_namespace {
 // FINITEONLY-NEXT:    [[B:%.*]] = alloca [[STRUCT_USER_BFLOAT16]], align 1, addrspace(5)
 // FINITEONLY-NEXT:    [[A_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[A]] to ptr
 // FINITEONLY-NEXT:    [[B_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[B]] to ptr
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 1, ptr addrspace(5) [[A]]) #[[ATTR11:[0-9]+]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[A]]) #[[ATTR11:[0-9]+]]
 // FINITEONLY-NEXT:    call void @_ZN13user_bfloat16C1Ef(ptr noundef nonnull align 1 dereferenceable(1) [[A_ASCAST]], float noundef nofpclass(nan inf) 1.000000e+00) #[[ATTR10]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 1, ptr addrspace(5) [[B]]) #[[ATTR11]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[B]]) #[[ATTR11]]
 // FINITEONLY-NEXT:    call void @_ZN13user_bfloat16C1Ef(ptr noundef nonnull align 1 dereferenceable(1) [[B_ASCAST]], float noundef nofpclass(nan inf) 2.000000e+00) #[[ATTR10]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 1, ptr addrspace(5) [[B]]) #[[ATTR11]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 1, ptr addrspace(5) [[A]]) #[[ATTR11]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[B]]) #[[ATTR11]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[A]]) #[[ATTR11]]
 // FINITEONLY-NEXT:    ret void
 //
   __global__ void test_fma() {
diff --git a/clang/test/Headers/__clang_hip_math.hip b/clang/test/Headers/__clang_hip_math.hip
index 81c5f43a006e7..15bdb7589bf45 100644
--- a/clang/test/Headers/__clang_hip_math.hip
+++ b/clang/test/Headers/__clang_hip_math.hip
@@ -3636,52 +3636,52 @@ extern "C" __device__ long int test_lround(double x) {
 // DEFAULT-LABEL: @test_modff(
 // DEFAULT-NEXT:  entry:
 // DEFAULT-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17:[0-9]+]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17:[0-9]+]]
 // DEFAULT-NEXT:    [[CALL_I:%.*]] = call contract noundef float @__ocml_modf_f32(float noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // DEFAULT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA16:![0-9]+]]
 // DEFAULT-NEXT:    store float [[TMP0]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA16]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    ret float [[CALL_I]]
 //
 // FINITEONLY-LABEL: @test_modff(
 // FINITEONLY-NEXT:  entry:
 // FINITEONLY-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17:[0-9]+]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17:[0-9]+]]
 // FINITEONLY-NEXT:    [[CALL_I:%.*]] = call nnan ninf contract noundef nofpclass(nan inf) float @__ocml_modf_f32(float noundef nofpclass(nan inf) [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // FINITEONLY-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA16:![0-9]+]]
 // FINITEONLY-NEXT:    store float [[TMP0]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA16]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    ret float [[CALL_I]]
 //
 // APPROX-LABEL: @test_modff(
 // APPROX-NEXT:  entry:
 // APPROX-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// APPROX-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17:[0-9]+]]
+// APPROX-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17:[0-9]+]]
 // APPROX-NEXT:    [[CALL_I:%.*]] = call contract noundef float @__ocml_modf_f32(float noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // APPROX-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA16:![0-9]+]]
 // APPROX-NEXT:    store float [[TMP0]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA16]]
-// APPROX-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    ret float [[CALL_I]]
 //
 // NCRDIV-LABEL: @test_modff(
 // NCRDIV-NEXT:  entry:
 // NCRDIV-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17:[0-9]+]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17:[0-9]+]]
 // NCRDIV-NEXT:    [[CALL_I:%.*]] = call contract noundef float @__ocml_modf_f32(float noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // NCRDIV-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA17:![0-9]+]]
 // NCRDIV-NEXT:    store float [[TMP0]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA17]]
-// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    ret float [[CALL_I]]
 //
 // AMDGCNSPIRV-LABEL: @test_modff(
 // AMDGCNSPIRV-NEXT:  entry:
 // AMDGCNSPIRV-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4
 // AMDGCNSPIRV-NEXT:    [[__TMP_ASCAST_I:%.*]] = addrspacecast ptr [[__TMP_I]] to ptr addrspace(4)
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15:[0-9]+]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15:[0-9]+]]
 // AMDGCNSPIRV-NEXT:    [[CALL_I:%.*]] = call contract spir_func noundef addrspace(4) float @__ocml_modf_f32(float noundef [[X:%.*]], ptr noundef nonnull [[__TMP_I]]) #[[ATTR14]]
 // AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[__TMP_ASCAST_I]], align 4, !tbaa [[TBAA17:![0-9]+]]
 // AMDGCNSPIRV-NEXT:    store float [[TMP0]], ptr addrspace(4) [[Y:%.*]], align 4, !tbaa [[TBAA17]]
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    ret float [[CALL_I]]
 //
 extern "C" __device__ float test_modff(float x, float* y) {
@@ -3691,52 +3691,52 @@ extern "C" __device__ float test_modff(float x, float* y) {
 // DEFAULT-LABEL: @test_modf(
 // DEFAULT-NEXT:  entry:
 // DEFAULT-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    [[CALL_I:%.*]] = call contract noundef double @__ocml_modf_f64(double noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // DEFAULT-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA18:![0-9]+]]
 // DEFAULT-NEXT:    store double [[TMP0]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA18]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    ret double [[CALL_I]]
 //
 // FINITEONLY-LABEL: @test_modf(
 // FINITEONLY-NEXT:  entry:
 // FINITEONLY-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    [[CALL_I:%.*]] = call nnan ninf contract noundef nofpclass(nan inf) double @__ocml_modf_f64(double noundef nofpclass(nan inf) [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // FINITEONLY-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA18:![0-9]+]]
 // FINITEONLY-NEXT:    store double [[TMP0]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA18]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    ret double [[CALL_I]]
 //
 // APPROX-LABEL: @test_modf(
 // APPROX-NEXT:  entry:
 // APPROX-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// APPROX-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    [[CALL_I:%.*]] = call contract noundef double @__ocml_modf_f64(double noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // APPROX-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA18:![0-9]+]]
 // APPROX-NEXT:    store double [[TMP0]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA18]]
-// APPROX-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    ret double [[CALL_I]]
 //
 // NCRDIV-LABEL: @test_modf(
 // NCRDIV-NEXT:  entry:
 // NCRDIV-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    [[CALL_I:%.*]] = call contract noundef double @__ocml_modf_f64(double noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // NCRDIV-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA19:![0-9]+]]
 // NCRDIV-NEXT:    store double [[TMP0]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA19]]
-// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    ret double [[CALL_I]]
 //
 // AMDGCNSPIRV-LABEL: @test_modf(
 // AMDGCNSPIRV-NEXT:  entry:
 // AMDGCNSPIRV-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8
 // AMDGCNSPIRV-NEXT:    [[__TMP_ASCAST_I:%.*]] = addrspacecast ptr [[__TMP_I]] to ptr addrspace(4)
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    [[CALL_I:%.*]] = call contract spir_func noundef addrspace(4) double @__ocml_modf_f64(double noundef [[X:%.*]], ptr noundef nonnull [[__TMP_I]]) #[[ATTR14]]
 // AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[__TMP_ASCAST_I]], align 8, !tbaa [[TBAA19:![0-9]+]]
 // AMDGCNSPIRV-NEXT:    store double [[TMP0]], ptr addrspace(4) [[Y:%.*]], align 8, !tbaa [[TBAA19]]
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    ret double [[CALL_I]]
 //
 extern "C" __device__ double test_modf(double x, double* y) {
@@ -5362,52 +5362,52 @@ extern "C" __device__ double test_remainder(double x, double y) {
 // DEFAULT-LABEL: @test_remquof(
 // DEFAULT-NEXT:  entry:
 // DEFAULT-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4, addrspace(5)
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    [[CALL_I:%.*]] = call contract noundef float @__ocml_remquo_f32(float noundef [[X:%.*]], float noundef [[Y:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // DEFAULT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA12]]
 // DEFAULT-NEXT:    store i32 [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA12]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    ret float [[CALL_I]]
 //
 // FINITEONLY-LABEL: @test_remquof(
 // FINITEONLY-NEXT:  entry:
 // FINITEONLY-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4, addrspace(5)
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    [[CALL_I:%.*]] = call nnan ninf contract noundef nofpclass(nan inf) float @__ocml_remquo_f32(float noundef nofpclass(nan inf) [[X:%.*]], float noundef nofpclass(nan inf) [[Y:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // FINITEONLY-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA12]]
 // FINITEONLY-NEXT:    store i32 [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA12]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    ret float [[CALL_I]]
 //
 // APPROX-LABEL: @test_remquof(
 // APPROX-NEXT:  entry:
 // APPROX-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4, addrspace(5)
-// APPROX-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    [[CALL_I:%.*]] = call contract noundef float @__ocml_remquo_f32(float noundef [[X:%.*]], float noundef [[Y:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // APPROX-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA12]]
 // APPROX-NEXT:    store i32 [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA12]]
-// APPROX-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    ret float [[CALL_I]]
 //
 // NCRDIV-LABEL: @test_remquof(
 // NCRDIV-NEXT:  entry:
 // NCRDIV-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4, addrspace(5)
-// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    [[CALL_I:%.*]] = call contract noundef float @__ocml_remquo_f32(float noundef [[X:%.*]], float noundef [[Y:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // NCRDIV-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA13]]
 // NCRDIV-NEXT:    store i32 [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA13]]
-// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    ret float [[CALL_I]]
 //
 // AMDGCNSPIRV-LABEL: @test_remquof(
 // AMDGCNSPIRV-NEXT:  entry:
 // AMDGCNSPIRV-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4
 // AMDGCNSPIRV-NEXT:    [[__TMP_ASCAST_I:%.*]] = addrspacecast ptr [[__TMP_I]] to ptr addrspace(4)
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    [[CALL_I:%.*]] = call contract spir_func noundef addrspace(4) float @__ocml_remquo_f32(float noundef [[X:%.*]], float noundef [[Y:%.*]], ptr noundef nonnull [[__TMP_I]]) #[[ATTR14]]
 // AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[__TMP_ASCAST_I]], align 4, !tbaa [[TBAA13]]
 // AMDGCNSPIRV-NEXT:    store i32 [[TMP0]], ptr addrspace(4) [[Z:%.*]], align 4, !tbaa [[TBAA13]]
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    ret float [[CALL_I]]
 //
 extern "C" __device__ float test_remquof(float x, float y, int* z) {
@@ -5417,52 +5417,52 @@ extern "C" __device__ float test_remquof(float x, float y, int* z) {
 // DEFAULT-LABEL: @test_remquo(
 // DEFAULT-NEXT:  entry:
 // DEFAULT-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4, addrspace(5)
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    [[CALL_I:%.*]] = call contract noundef double @__ocml_remquo_f64(double noundef [[X:%.*]], double noundef [[Y:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // DEFAULT-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA12]]
 // DEFAULT-NEXT:    store i32 [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA12]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    ret double [[CALL_I]]
 //
 // FINITEONLY-LABEL: @test_remquo(
 // FINITEONLY-NEXT:  entry:
 // FINITEONLY-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4, addrspace(5)
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    [[CALL_I:%.*]] = call nnan ninf contract noundef nofpclass(nan inf) double @__ocml_remquo_f64(double noundef nofpclass(nan inf) [[X:%.*]], double noundef nofpclass(nan inf) [[Y:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // FINITEONLY-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA12]]
 // FINITEONLY-NEXT:    store i32 [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA12]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    ret double [[CALL_I]]
 //
 // APPROX-LABEL: @test_remquo(
 // APPROX-NEXT:  entry:
 // APPROX-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4, addrspace(5)
-// APPROX-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    [[CALL_I:%.*]] = call contract noundef double @__ocml_remquo_f64(double noundef [[X:%.*]], double noundef [[Y:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // APPROX-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA12]]
 // APPROX-NEXT:    store i32 [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA12]]
-// APPROX-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    ret double [[CALL_I]]
 //
 // NCRDIV-LABEL: @test_remquo(
 // NCRDIV-NEXT:  entry:
 // NCRDIV-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4, addrspace(5)
-// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    [[CALL_I:%.*]] = call contract noundef double @__ocml_remquo_f64(double noundef [[X:%.*]], double noundef [[Y:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // NCRDIV-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA13]]
 // NCRDIV-NEXT:    store i32 [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA13]]
-// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    ret double [[CALL_I]]
 //
 // AMDGCNSPIRV-LABEL: @test_remquo(
 // AMDGCNSPIRV-NEXT:  entry:
 // AMDGCNSPIRV-NEXT:    [[__TMP_I:%.*]] = alloca i32, align 4
 // AMDGCNSPIRV-NEXT:    [[__TMP_ASCAST_I:%.*]] = addrspacecast ptr [[__TMP_I]] to ptr addrspace(4)
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    [[CALL_I:%.*]] = call contract spir_func noundef addrspace(4) double @__ocml_remquo_f64(double noundef [[X:%.*]], double noundef [[Y:%.*]], ptr noundef nonnull [[__TMP_I]]) #[[ATTR14]]
 // AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(4) [[__TMP_ASCAST_I]], align 4, !tbaa [[TBAA13]]
 // AMDGCNSPIRV-NEXT:    store i32 [[TMP0]], ptr addrspace(4) [[Z:%.*]], align 4, !tbaa [[TBAA13]]
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    ret double [[CALL_I]]
 //
 extern "C" __device__ double test_remquo(double x, double y, int* z) {
@@ -6198,57 +6198,57 @@ extern "C" __device__ BOOL_TYPE test___signbit(double x) {
 // DEFAULT-LABEL: @test_sincosf(
 // DEFAULT-NEXT:  entry:
 // DEFAULT-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    [[CALL_I:%.*]] = call contract float @__ocml_sincos_f32(float noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // DEFAULT-NEXT:    store float [[CALL_I]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA16]]
 // DEFAULT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA16]]
 // DEFAULT-NEXT:    store float [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA16]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    ret void
 //
 // FINITEONLY-LABEL: @test_sincosf(
 // FINITEONLY-NEXT:  entry:
 // FINITEONLY-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    [[CALL_I:%.*]] = call nnan ninf contract nofpclass(nan inf) float @__ocml_sincos_f32(float noundef nofpclass(nan inf) [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // FINITEONLY-NEXT:    store float [[CALL_I]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA16]]
 // FINITEONLY-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA16]]
 // FINITEONLY-NEXT:    store float [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA16]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    ret void
 //
 // APPROX-LABEL: @test_sincosf(
 // APPROX-NEXT:  entry:
 // APPROX-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// APPROX-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    [[CALL_I:%.*]] = call contract float @__ocml_sincos_f32(float noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // APPROX-NEXT:    store float [[CALL_I]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA16]]
 // APPROX-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA16]]
 // APPROX-NEXT:    store float [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA16]]
-// APPROX-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    ret void
 //
 // NCRDIV-LABEL: @test_sincosf(
 // NCRDIV-NEXT:  entry:
 // NCRDIV-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    [[CALL_I:%.*]] = call contract float @__ocml_sincos_f32(float noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // NCRDIV-NEXT:    store float [[CALL_I]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA17]]
 // NCRDIV-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA17]]
 // NCRDIV-NEXT:    store float [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA17]]
-// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    ret void
 //
 // AMDGCNSPIRV-LABEL: @test_sincosf(
 // AMDGCNSPIRV-NEXT:  entry:
 // AMDGCNSPIRV-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4
 // AMDGCNSPIRV-NEXT:    [[__TMP_ASCAST_I:%.*]] = addrspacecast ptr [[__TMP_I]] to ptr addrspace(4)
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    [[CALL_I:%.*]] = call contract spir_func addrspace(4) float @__ocml_sincos_f32(float noundef [[X:%.*]], ptr noundef nonnull [[__TMP_I]]) #[[ATTR14]]
 // AMDGCNSPIRV-NEXT:    store float [[CALL_I]], ptr addrspace(4) [[Y:%.*]], align 4, !tbaa [[TBAA17]]
 // AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[__TMP_ASCAST_I]], align 4, !tbaa [[TBAA17]]
 // AMDGCNSPIRV-NEXT:    store float [[TMP0]], ptr addrspace(4) [[Z:%.*]], align 4, !tbaa [[TBAA17]]
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    ret void
 //
 extern "C" __device__ void test_sincosf(float x, float *y, float *z) {
@@ -6258,57 +6258,57 @@ extern "C" __device__ void test_sincosf(float x, float *y, float *z) {
 // DEFAULT-LABEL: @test_sincos(
 // DEFAULT-NEXT:  entry:
 // DEFAULT-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    [[CALL_I:%.*]] = call contract double @__ocml_sincos_f64(double noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // DEFAULT-NEXT:    store double [[CALL_I]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA18]]
 // DEFAULT-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA18]]
 // DEFAULT-NEXT:    store double [[TMP0]], ptr [[Z:%.*]], align 8, !tbaa [[TBAA18]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    ret void
 //
 // FINITEONLY-LABEL: @test_sincos(
 // FINITEONLY-NEXT:  entry:
 // FINITEONLY-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    [[CALL_I:%.*]] = call nnan ninf contract nofpclass(nan inf) double @__ocml_sincos_f64(double noundef nofpclass(nan inf) [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // FINITEONLY-NEXT:    store double [[CALL_I]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA18]]
 // FINITEONLY-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA18]]
 // FINITEONLY-NEXT:    store double [[TMP0]], ptr [[Z:%.*]], align 8, !tbaa [[TBAA18]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    ret void
 //
 // APPROX-LABEL: @test_sincos(
 // APPROX-NEXT:  entry:
 // APPROX-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// APPROX-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    [[CALL_I:%.*]] = call contract double @__ocml_sincos_f64(double noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // APPROX-NEXT:    store double [[CALL_I]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA18]]
 // APPROX-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA18]]
 // APPROX-NEXT:    store double [[TMP0]], ptr [[Z:%.*]], align 8, !tbaa [[TBAA18]]
-// APPROX-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    ret void
 //
 // NCRDIV-LABEL: @test_sincos(
 // NCRDIV-NEXT:  entry:
 // NCRDIV-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    [[CALL_I:%.*]] = call contract double @__ocml_sincos_f64(double noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // NCRDIV-NEXT:    store double [[CALL_I]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA19]]
 // NCRDIV-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA19]]
 // NCRDIV-NEXT:    store double [[TMP0]], ptr [[Z:%.*]], align 8, !tbaa [[TBAA19]]
-// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    ret void
 //
 // AMDGCNSPIRV-LABEL: @test_sincos(
 // AMDGCNSPIRV-NEXT:  entry:
 // AMDGCNSPIRV-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8
 // AMDGCNSPIRV-NEXT:    [[__TMP_ASCAST_I:%.*]] = addrspacecast ptr [[__TMP_I]] to ptr addrspace(4)
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    [[CALL_I:%.*]] = call contract spir_func addrspace(4) double @__ocml_sincos_f64(double noundef [[X:%.*]], ptr noundef nonnull [[__TMP_I]]) #[[ATTR14]]
 // AMDGCNSPIRV-NEXT:    store double [[CALL_I]], ptr addrspace(4) [[Y:%.*]], align 8, !tbaa [[TBAA19]]
 // AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[__TMP_ASCAST_I]], align 8, !tbaa [[TBAA19]]
 // AMDGCNSPIRV-NEXT:    store double [[TMP0]], ptr addrspace(4) [[Z:%.*]], align 8, !tbaa [[TBAA19]]
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    ret void
 //
 extern "C" __device__ void test_sincos(double x, double *y, double *z) {
@@ -6318,57 +6318,57 @@ extern "C" __device__ void test_sincos(double x, double *y, double *z) {
 // DEFAULT-LABEL: @test_sincospif(
 // DEFAULT-NEXT:  entry:
 // DEFAULT-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    [[CALL_I:%.*]] = call contract float @__ocml_sincospi_f32(float noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // DEFAULT-NEXT:    store float [[CALL_I]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA16]]
 // DEFAULT-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA16]]
 // DEFAULT-NEXT:    store float [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA16]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    ret void
 //
 // FINITEONLY-LABEL: @test_sincospif(
 // FINITEONLY-NEXT:  entry:
 // FINITEONLY-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    [[CALL_I:%.*]] = call nnan ninf contract nofpclass(nan inf) float @__ocml_sincospi_f32(float noundef nofpclass(nan inf) [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // FINITEONLY-NEXT:    store float [[CALL_I]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA16]]
 // FINITEONLY-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA16]]
 // FINITEONLY-NEXT:    store float [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA16]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    ret void
 //
 // APPROX-LABEL: @test_sincospif(
 // APPROX-NEXT:  entry:
 // APPROX-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// APPROX-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    [[CALL_I:%.*]] = call contract float @__ocml_sincospi_f32(float noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // APPROX-NEXT:    store float [[CALL_I]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA16]]
 // APPROX-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA16]]
 // APPROX-NEXT:    store float [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA16]]
-// APPROX-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    ret void
 //
 // NCRDIV-LABEL: @test_sincospif(
 // NCRDIV-NEXT:  entry:
 // NCRDIV-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4, addrspace(5)
-// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    [[CALL_I:%.*]] = call contract float @__ocml_sincospi_f32(float noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // NCRDIV-NEXT:    store float [[CALL_I]], ptr [[Y:%.*]], align 4, !tbaa [[TBAA17]]
 // NCRDIV-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(5) [[__TMP_I]], align 4, !tbaa [[TBAA17]]
 // NCRDIV-NEXT:    store float [[TMP0]], ptr [[Z:%.*]], align 4, !tbaa [[TBAA17]]
-// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    ret void
 //
 // AMDGCNSPIRV-LABEL: @test_sincospif(
 // AMDGCNSPIRV-NEXT:  entry:
 // AMDGCNSPIRV-NEXT:    [[__TMP_I:%.*]] = alloca float, align 4
 // AMDGCNSPIRV-NEXT:    [[__TMP_ASCAST_I:%.*]] = addrspacecast ptr [[__TMP_I]] to ptr addrspace(4)
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    [[CALL_I:%.*]] = call contract spir_func addrspace(4) float @__ocml_sincospi_f32(float noundef [[X:%.*]], ptr noundef nonnull [[__TMP_I]]) #[[ATTR14]]
 // AMDGCNSPIRV-NEXT:    store float [[CALL_I]], ptr addrspace(4) [[Y:%.*]], align 4, !tbaa [[TBAA17]]
 // AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load float, ptr addrspace(4) [[__TMP_ASCAST_I]], align 4, !tbaa [[TBAA17]]
 // AMDGCNSPIRV-NEXT:    store float [[TMP0]], ptr addrspace(4) [[Z:%.*]], align 4, !tbaa [[TBAA17]]
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    ret void
 //
 extern "C" __device__ void test_sincospif(float x, float *y, float *z) {
@@ -6378,57 +6378,57 @@ extern "C" __device__ void test_sincospif(float x, float *y, float *z) {
 // DEFAULT-LABEL: @test_sincospi(
 // DEFAULT-NEXT:  entry:
 // DEFAULT-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    [[CALL_I:%.*]] = call contract double @__ocml_sincospi_f64(double noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // DEFAULT-NEXT:    store double [[CALL_I]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA18]]
 // DEFAULT-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA18]]
 // DEFAULT-NEXT:    store double [[TMP0]], ptr [[Z:%.*]], align 8, !tbaa [[TBAA18]]
-// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// DEFAULT-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // DEFAULT-NEXT:    ret void
 //
 // FINITEONLY-LABEL: @test_sincospi(
 // FINITEONLY-NEXT:  entry:
 // FINITEONLY-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    [[CALL_I:%.*]] = call nnan ninf contract nofpclass(nan inf) double @__ocml_sincospi_f64(double noundef nofpclass(nan inf) [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // FINITEONLY-NEXT:    store double [[CALL_I]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA18]]
 // FINITEONLY-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA18]]
 // FINITEONLY-NEXT:    store double [[TMP0]], ptr [[Z:%.*]], align 8, !tbaa [[TBAA18]]
-// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// FINITEONLY-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // FINITEONLY-NEXT:    ret void
 //
 // APPROX-LABEL: @test_sincospi(
 // APPROX-NEXT:  entry:
 // APPROX-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// APPROX-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    [[CALL_I:%.*]] = call contract double @__ocml_sincospi_f64(double noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // APPROX-NEXT:    store double [[CALL_I]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA18]]
 // APPROX-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA18]]
 // APPROX-NEXT:    store double [[TMP0]], ptr [[Z:%.*]], align 8, !tbaa [[TBAA18]]
-// APPROX-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// APPROX-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // APPROX-NEXT:    ret void
 //
 // NCRDIV-LABEL: @test_sincospi(
 // NCRDIV-NEXT:  entry:
 // NCRDIV-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8, addrspace(5)
-// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    [[CALL_I:%.*]] = call contract double @__ocml_sincospi_f64(double noundef [[X:%.*]], ptr addrspace(5) noundef [[__TMP_I]]) #[[ATTR16]]
 // NCRDIV-NEXT:    store double [[CALL_I]], ptr [[Y:%.*]], align 8, !tbaa [[TBAA19]]
 // NCRDIV-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(5) [[__TMP_I]], align 8, !tbaa [[TBAA19]]
 // NCRDIV-NEXT:    store double [[TMP0]], ptr [[Z:%.*]], align 8, !tbaa [[TBAA19]]
-// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
+// NCRDIV-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[__TMP_I]]) #[[ATTR17]]
 // NCRDIV-NEXT:    ret void
 //
 // AMDGCNSPIRV-LABEL: @test_sincospi(
 // AMDGCNSPIRV-NEXT:  entry:
 // AMDGCNSPIRV-NEXT:    [[__TMP_I:%.*]] = alloca double, align 8
 // AMDGCNSPIRV-NEXT:    [[__TMP_ASCAST_I:%.*]] = addrspacecast ptr [[__TMP_I]] to ptr addrspace(4)
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.start.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    [[CALL_I:%.*]] = call contract spir_func addrspace(4) double @__ocml_sincospi_f64(double noundef [[X:%.*]], ptr noundef nonnull [[__TMP_I]]) #[[ATTR14]]
 // AMDGCNSPIRV-NEXT:    store double [[CALL_I]], ptr addrspace(4) [[Y:%.*]], align 8, !tbaa [[TBAA19]]
 // AMDGCNSPIRV-NEXT:    [[TMP0:%.*]] = load double, ptr addrspace(4) [[__TMP_ASCAST_I]], align 8, !tbaa [[TBAA19]]
 // AMDGCNSPIRV-NEXT:    store double [[TMP0]], ptr addrspace(4) [[Z:%.*]], align 8, !tbaa [[TBAA19]]
-// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[__TMP_I]]) #[[ATTR15]]
+// AMDGCNSPIRV-NEXT:    call addrspace(4) void @llvm.lifetime.end.p0(ptr nonnull [[__TMP_I]]) #[[ATTR15]]
 // AMDGCNSPIRV-NEXT:    ret void
 //
 extern "C" __device__ void test_sincospi(double x, double *y, double *z) {
diff --git a/clang/test/Headers/__cpuidex_conflict.c b/clang/test/Headers/__cpuidex_conflict.c
index 74f45327de2bb..67f2a0cf908e5 100644
--- a/clang/test/Headers/__cpuidex_conflict.c
+++ b/clang/test/Headers/__cpuidex_conflict.c
@@ -5,6 +5,7 @@
 
 // Ensure that we do not run into conflicts when offloading.
 // RUN: %clang_cc1 %s -DIS_STATIC=static -ffreestanding -fopenmp -fopenmp-is-target-device -aux-triple x86_64-unknown-linux-gnu
+// RUN: %clang_cc1 -DIS_STATIC="" -triple nvptx64-nvidia-cuda -aux-triple x86_64-unknown-linux-gnu -aux-target-cpu x86-64 -fcuda-is-device -x cuda %s -o -
 
 typedef __SIZE_TYPE__ size_t;
 
diff --git a/clang/test/Headers/mm3dnow.c b/clang/test/Headers/mm3dnow.c
index a9b6dd88f8034..e45acb12ddbde 100644
--- a/clang/test/Headers/mm3dnow.c
+++ b/clang/test/Headers/mm3dnow.c
@@ -2,6 +2,9 @@
 // RUN: %clang_cc1 -fsyntax-only -D_CLANG_DISABLE_CRT_DEPRECATION_WARNINGS -ffreestanding %s -verify
 // RUN: %clang_cc1 -fsyntax-only -ffreestanding -x c++ %s -verify
 
+// XFAIL: target=arm64ec-pc-windows-msvc
+// These intrinsics are not yet implemented for Arm64EC.
+
 #if defined(i386) || defined(__x86_64__)
 #ifndef _CLANG_DISABLE_CRT_DEPRECATION_WARNINGS
 // expected-warning@mm3dnow.h:*{{The <mm3dnow.h> header is deprecated}}
diff --git a/clang/test/Headers/pmmintrin.c b/clang/test/Headers/pmmintrin.c
index 5b7a3a4ef6b9b..776ef75d70c54 100644
--- a/clang/test/Headers/pmmintrin.c
+++ b/clang/test/Headers/pmmintrin.c
@@ -2,6 +2,9 @@
 // RUN: %clang_cc1 -fsyntax-only -ffreestanding -x c++ %s -verify
 // expected-no-diagnostics
 
+// XFAIL: target=arm64ec-pc-windows-msvc
+// These intrinsics are not yet implemented for Arm64EC.
+
 #if defined(i386) || defined(__x86_64__)
 #include <pmmintrin.h>
 
diff --git a/clang/test/Headers/spirv_functions.cpp b/clang/test/Headers/spirv_functions.cpp
index ff036b75faf02..fa9e2cea51079 100644
--- a/clang/test/Headers/spirv_functions.cpp
+++ b/clang/test/Headers/spirv_functions.cpp
@@ -15,7 +15,7 @@
 // NV: call noundef ptr @_Z42__spirv_GenericCastToPtrExplicit_ToPrivatePvi
 // NV: addrspacecast ptr %{{.*}} to ptr addrspace(1)
 // NV: addrspacecast ptr %{{.*}} to ptr addrspace(3)
-void test_cast(int* p) {
+[[clang::sycl_external]] void test_cast(int* p) {
   __spirv_GenericCastToPtrExplicit_ToGlobal(p, 5);
   __spirv_GenericCastToPtrExplicit_ToLocal(p, 4);
   __spirv_GenericCastToPtrExplicit_ToPrivate(p, 7);
diff --git a/clang/test/Headers/spirv_ids.cpp b/clang/test/Headers/spirv_ids.cpp
index 466be5deee87a..a1c953eff7771 100644
--- a/clang/test/Headers/spirv_ids.cpp
+++ b/clang/test/Headers/spirv_ids.cpp
@@ -80,7 +80,7 @@
 // NV: call noundef i32 @_Z25__spirv_BuiltInSubgroupIdv() #2
 // NV: call noundef i32 @_Z40__spirv_BuiltInSubgroupLocalInvocationIdv() #2
 
-void test_id_and_range() {
+[[clang::sycl_external]] void test_id_and_range() {
   __spirv_BuiltInNumWorkgroups(0);
   __spirv_BuiltInNumWorkgroups(1);
   __spirv_BuiltInNumWorkgroups(2);
diff --git a/clang/test/Headers/x86-intrinsics-headers.c b/clang/test/Headers/x86-intrinsics-headers.c
index 59ca354e1160b..dc06cbde0f587 100644
--- a/clang/test/Headers/x86-intrinsics-headers.c
+++ b/clang/test/Headers/x86-intrinsics-headers.c
@@ -2,6 +2,9 @@
 // RUN: %clang_cc1 -fsyntax-only -ffreestanding -flax-vector-conversions=none %s
 // RUN: %clang_cc1 -fsyntax-only -ffreestanding -x c++ %s
 
+// XFAIL: target=arm64ec-pc-windows-msvc
+// These intrinsics are not yet implemented for Arm64EC.
+
 #if defined(i386) || defined(__x86_64__)
 
 #ifdef __SSE4_2__
diff --git a/clang/test/Headers/x86intrin.c b/clang/test/Headers/x86intrin.c
index 53e369559f408..c01af1a43d1fa 100644
--- a/clang/test/Headers/x86intrin.c
+++ b/clang/test/Headers/x86intrin.c
@@ -3,6 +3,9 @@
 // RUN: %clang_cc1 -fsyntax-only -ffreestanding -x c++ %s -verify
 // expected-no-diagnostics
 
+// XFAIL: target=arm64ec-pc-windows-msvc
+// These intrinsics are not yet implemented for Arm64EC.
+
 #if defined(i386) || defined(__x86_64__)
 
 // Include the metaheader that includes all x86 intrinsic headers.
diff --git a/clang/test/Headers/x86intrin.cpp b/clang/test/Headers/x86intrin.cpp
index 11d442db3d2c4..6c9baa6fff635 100644
--- a/clang/test/Headers/x86intrin.cpp
+++ b/clang/test/Headers/x86intrin.cpp
@@ -1,6 +1,9 @@
 // RUN: %clang_cc1 -fsyntax-only -ffreestanding %s -verify
 // expected-no-diagnostics
 
+// XFAIL: target=arm64ec-pc-windows-msvc
+// These intrinsics are not yet implemented for Arm64EC.
+
 #if defined(i386) || defined(__x86_64__)
 
 // Include the metaheader that includes all x86 intrinsic headers.
diff --git a/clang/test/Import/builtin-template/Inputs/S.cpp b/clang/test/Import/builtin-template/Inputs/S.cpp
index d5c9a9ae0309d..85c71f61b0220 100644
--- a/clang/test/Import/builtin-template/Inputs/S.cpp
+++ b/clang/test/Import/builtin-template/Inputs/S.cpp
@@ -14,3 +14,13 @@ using TypePackElement = __type_pack_element<i, T...>;
 
 template <int i>
 struct X;
+
+using X0 = X<0>;
+template <int I>
+using SameAsX = X<I>;
+
+template <template <class...> class Templ, class...Types>
+using TypePackDedup = Templ<__builtin_dedup_pack<Types...>...>;
+
+template <class ...Ts>
+struct TypeList {};
diff --git a/clang/test/Import/builtin-template/test.cpp b/clang/test/Import/builtin-template/test.cpp
index 590efad0c71dc..a9afbd1b316bf 100644
--- a/clang/test/Import/builtin-template/test.cpp
+++ b/clang/test/Import/builtin-template/test.cpp
@@ -1,9 +1,11 @@
 // RUN: clang-import-test -dump-ast -import %S/Inputs/S.cpp -expression %s -Xcc -DSEQ | FileCheck --check-prefix=CHECK-SEQ %s
 // RUN: clang-import-test -dump-ast -import %S/Inputs/S.cpp -expression %s -Xcc -DPACK | FileCheck --check-prefix=CHECK-PACK %s
-// RUN: clang-import-test -dump-ast -import %S/Inputs/S.cpp -expression %s -Xcc -DPACK -Xcc -DSEQ | FileCheck --check-prefixes=CHECK-SEQ,CHECK-PACK %s
+// RUN: clang-import-test -dump-ast -import %S/Inputs/S.cpp -expression %s -Xcc -DDEDUP | FileCheck --check-prefix=CHECK-DEDUP %s
+// RUN: clang-import-test -dump-ast -import %S/Inputs/S.cpp -expression %s -Xcc -DPACK -Xcc -DSEQ -Xcc -DDEDUP | FileCheck --check-prefixes=CHECK-SEQ,CHECK-PACK,CHECK-DEDUP %s
 
 // CHECK-SEQ:  BuiltinTemplateDecl {{.+}} <<invalid sloc>> <invalid sloc> implicit __make_integer_seq{{$}}
 // CHECK-PACK: BuiltinTemplateDecl {{.+}} <<invalid sloc>> <invalid sloc> implicit __type_pack_element{{$}}
+// CHECK-DEDUP: BuiltinTemplateDecl {{.+}} <<invalid sloc>> <invalid sloc> implicit __builtin_dedup_pack{{$}}
 
 void expr() {
 #ifdef SEQ
@@ -20,4 +22,12 @@ void expr() {
   static_assert(__is_same(TypePackElement<0, X<0>, X<1>>, X<0>), "");
   static_assert(__is_same(TypePackElement<1, X<0>, X<1>>, X<1>), "");
 #endif
+
+#ifdef DEDUP
+  static_assert(__is_same(TypePackDedup<TypeList>, TypeList<>), "");
+  static_assert(__is_same(TypePackDedup<TypeList, int, double, int>, TypeList<int, double>), "");
+  static_assert(!__is_same(TypePackDedup<TypeList, int, double, int>, TypeList<double, int>), "");
+  static_assert(__is_same(TypePackDedup<TypeList, X<0>, X<1>, X<1>, X<2>, X<0>>, TypeList<X<0>, X<1>, X<2>>), "");
+  static_assert(__is_same(TypePackDedup<TypeList, X0, SameAsX<1>, X<1>, X<0>>, TypeList<X<0>,X<1>>), "");
+#endif
 }
diff --git a/clang/test/Index/Core/index-instantiated-source.cpp b/clang/test/Index/Core/index-instantiated-source.cpp
index 2a67a3a3c7938..91be4a4138f24 100644
--- a/clang/test/Index/Core/index-instantiated-source.cpp
+++ b/clang/test/Index/Core/index-instantiated-source.cpp
@@ -73,8 +73,8 @@ void canonicalizeInstaniationReferences(TemplateClass<int, float> &object) {
 
   typedef TemplateClass<int, float> TT;
   TT::NestedType::SubNestedType subNestedType(0);
-// CHECK: [[@LINE-1]]:7 | struct/C++ | NestedType | c:@ST>2#T#T@TemplateClass@S@NestedType |
-// CHECK: [[@LINE-2]]:19 | class/C++ | SubNestedType | c:@ST>2#T#T@TemplateClass@S@NestedType@S@SubNestedType |
+// CHECK: [[@LINE-1]]:19 | class/C++ | SubNestedType | c:@ST>2#T#T@TemplateClass@S@NestedType@S@SubNestedType |
+// CHECK: [[@LINE-2]]:7 | struct/C++ | NestedType | c:@ST>2#T#T@TemplateClass@S@NestedType |
 
   TT::NestedType::TypeAlias nestedTypeAlias;
 // CHECK: [[@LINE-1]]:19 | type-alias/C++ | TypeAlias | c:@ST>2#T#T@TemplateClass@S@NestedType@TypeAlias |
diff --git a/clang/test/Index/Core/index-source.cpp b/clang/test/Index/Core/index-source.cpp
index 043e616a1d366..36bc663b89684 100644
--- a/clang/test/Index/Core/index-source.cpp
+++ b/clang/test/Index/Core/index-source.cpp
@@ -525,9 +525,9 @@ struct Outer {
 
 template<>
 struct rd33122110::Outer::Nested<int>;
-// CHECK: [[@LINE-1]]:8 | namespace/C++ | rd33122110 | c:@N@rd33122110 | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK: [[@LINE-1]]:20 | struct/C++ | Outer | c:@N@rd33122110@S@Outer | <no-cgname> | Ref,RelCont | rel: 1
 // CHECK-NEXT: RelCont | Nested | c:@N@rd33122110@S@Outer@S@Nested>#I
-// CHECK: [[@LINE-3]]:20 | struct/C++ | Outer | c:@N@rd33122110@S@Outer | <no-cgname> | Ref,RelCont | rel: 1
+// CHECK: [[@LINE-3]]:8 | namespace/C++ | rd33122110 | c:@N@rd33122110 | <no-cgname> | Ref,RelCont | rel: 1
 // CHECK-NEXT: RelCont | Nested | c:@N@rd33122110@S@Outer@S@Nested>#I
 
 namespace index_offsetof {
diff --git a/clang/test/Index/c-index-api-loadTU-test.m b/clang/test/Index/c-index-api-loadTU-test.m
index eb3fde08a36fd..8ddb193e42428 100644
--- a/clang/test/Index/c-index-api-loadTU-test.m
+++ b/clang/test/Index/c-index-api-loadTU-test.m
@@ -164,7 +164,7 @@ @interface TestAttributes()
 // CHECK: c-index-api-loadTU-test.m:66:28: TypeRef=id:0:0 Extent=[66:28 - 66:30]
 // CHECK: c-index-api-loadTU-test.m:69:16: StructDecl=X0:69:16 Extent=[69:9 - 69:18]
 // CHECK: c-index-api-loadTU-test.m:69:19: TypedefDecl=X1:69:19 (Definition) Extent=[69:1 - 69:21]
-// CHECK: c-index-api-loadTU-test.m:69:16: TypeRef=struct X0:71:8 Extent=[69:16 - 69:18]
+// CHECK: c-index-api-loadTU-test.m:69:16: TypeRef=struct X0:69:16 Extent=[69:16 - 69:18]
 // CHECK: c-index-api-loadTU-test.m:70:8: StructDecl=X0:70:8 Extent=[70:1 - 70:10]
 // CHECK: c-index-api-loadTU-test.m:71:8: StructDecl=X0:71:8 (Definition) Extent=[71:1 - 71:14]
 // CHECK: c-index-api-loadTU-test.m:73:12: ObjCCategoryDecl=:73:12 Extent=[73:1 - 76:5]
diff --git a/clang/test/Index/copy-assignment-operator.cpp b/clang/test/Index/copy-assignment-operator.cpp
index 7f741931f9cef..46902c04e21c8 100644
--- a/clang/test/Index/copy-assignment-operator.cpp
+++ b/clang/test/Index/copy-assignment-operator.cpp
@@ -32,7 +32,7 @@ class Bar {
 // CHECK: CXXMethod=operator=:4:10 (copy-assignment operator) [type=bool (Foo &)] [typekind=FunctionProto] [canonicaltype=bool (Foo &)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [Foo &] [LValueReference]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:5:10 (copy-assignment operator) [type=bool (volatile Foo &)] [typekind=FunctionProto] [canonicaltype=bool (volatile Foo &)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [volatile Foo &] [LValueReference]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:6:10 (copy-assignment operator) [type=bool (const volatile Foo &)] [typekind=FunctionProto] [canonicaltype=bool (const volatile Foo &)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [const volatile Foo &] [LValueReference]] [isPOD=0] [isAnonRecDecl=0]
-// CHECK: CXXMethod=operator=:7:10 (copy-assignment operator) [type=bool (Foo)] [typekind=FunctionProto] [canonicaltype=bool (Foo)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [Foo] [Elaborated]] [isPOD=0] [isAnonRecDecl=0]
+// CHECK: CXXMethod=operator=:7:10 (copy-assignment operator) [type=bool (Foo)] [typekind=FunctionProto] [canonicaltype=bool (Foo)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [Foo] [Record]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: FunctionTemplate=operator=:11:10 [type=bool (const T &)] [typekind=FunctionProto] [canonicaltype=bool (const type-parameter-0-0 &)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:12:10 [type=bool (const bool &)] [typekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [const bool &] [LValueReference]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:13:10 [type=bool (char &)] [typekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [char &] [LValueReference]] [isPOD=0] [isAnonRecDecl=0]
@@ -44,4 +44,4 @@ class Bar {
 // CHECK: CXXMethod=operator=:23:10 (copy-assignment operator) [type=bool (Bar<T> &)] [typekind=FunctionProto] [canonicaltype=bool (Bar<T> &)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [Bar<T> &] [LValueReference]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:24:10 (copy-assignment operator) [type=bool (volatile Bar<T> &)] [typekind=FunctionProto] [canonicaltype=bool (volatile Bar<T> &)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [volatile Bar<T> &] [LValueReference]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:25:10 (copy-assignment operator) [type=bool (const volatile Bar<T> &)] [typekind=FunctionProto] [canonicaltype=bool (const volatile Bar<T> &)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [const volatile Bar<T> &] [LValueReference]] [isPOD=0] [isAnonRecDecl=0]
-// CHECK: CXXMethod=operator=:26:10 (copy-assignment operator) [type=bool (Bar<T>)] [typekind=FunctionProto] [canonicaltype=bool (Bar<T>)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [Bar<T>] [Elaborated]] [isPOD=0] [isAnonRecDecl=0]
+// CHECK: CXXMethod=operator=:26:10 (copy-assignment operator) [type=bool (Bar<T>)] [typekind=FunctionProto] [canonicaltype=bool (Bar<T>)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [Bar<T>] [Unexposed]] [isPOD=0] [isAnonRecDecl=0]
diff --git a/clang/test/Index/index-refs.cpp b/clang/test/Index/index-refs.cpp
index 14946849777d4..22eb7532ffad6 100644
--- a/clang/test/Index/index-refs.cpp
+++ b/clang/test/Index/index-refs.cpp
@@ -81,13 +81,13 @@ int ginitlist[] = {EnumVal};
 // CHECK-NEXT: [indexDeclaration]: kind: enum
 // CHECK-NEXT: [indexDeclaration]: kind: enumerator | name: EnumVal
 // CHECK-NEXT: [indexDeclaration]: kind: variable | name: gx
-// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS
 // CHECK-NEXT: [indexEntityReference]: kind: typedef | name: MyInt
 // CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS
+// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS
 // CHECK-NEXT: [indexEntityReference]: kind: enumerator | name: EnumVal
 // CHECK-NEXT: [indexDeclaration]: kind: function | name: foo
-// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS
 // CHECK-NEXT: [indexEntityReference]: kind: typedef | name: MyInt
+// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS
 // CHECK-NEXT: [indexDeclaration]: kind: enum
 // CHECK-NEXT: [indexDeclaration]: kind: enumerator | name: SecondVal
 // CHECK-NEXT: [indexEntityReference]: kind: enumerator | name: EnumVal
diff --git a/clang/test/Index/keep-going.cpp b/clang/test/Index/keep-going.cpp
index 635415178cf09..0b2df725a5a73 100644
--- a/clang/test/Index/keep-going.cpp
+++ b/clang/test/Index/keep-going.cpp
@@ -26,10 +26,10 @@ class C : public A<float> { };
 // CHECK: FieldDecl=a:4:13 (Definition) [type=T] [typekind=Unexposed] [canonicaltype=type-parameter-0-0] [canonicaltypekind=Unexposed] [isPOD=0]
 // CHECK: TypeRef=T:3:16 [type=T] [typekind=Unexposed] [canonicaltype=type-parameter-0-0] [canonicaltypekind=Unexposed] [isPOD=0]
 // CHECK: ClassDecl=B:6:7 (Definition) [type=B] [typekind=Record] [isPOD=0]
-// CHECK: C++ base class specifier=A<int>:4:7 [access=public isVirtual=false] [type=A<int>] [typekind=Elaborated] [templateargs/1= [type=int] [typekind=Int]] [canonicaltype=A<int>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=int] [typekind=Int]] [isPOD=0] [nbFields=1]
+// CHECK: C++ base class specifier=A<int>:4:7 [access=public isVirtual=false] [type=A<int>] [typekind=Unexposed] [templateargs/1= [type=int] [typekind=Int]] [canonicaltype=A<int>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=int] [typekind=Int]] [isPOD=0] [nbFields=1]
 // CHECK: TemplateRef=A:4:7 [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: ClassDecl=C:10:7 (Definition) [type=C] [typekind=Record] [isPOD=0]
-// CHECK: C++ base class specifier=A<float>:4:7 [access=public isVirtual=false] [type=A<float>] [typekind=Elaborated] [templateargs/1= [type=float] [typekind=Float]] [canonicaltype=A<float>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=float] [typekind=Float]] [isPOD=0] [nbFields=1]
+// CHECK: C++ base class specifier=A<float>:4:7 [access=public isVirtual=false] [type=A<float>] [typekind=Unexposed] [templateargs/1= [type=float] [typekind=Float]] [canonicaltype=A<float>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=float] [typekind=Float]] [isPOD=0] [nbFields=1]
 // CHECK: TemplateRef=A:4:7 [type=] [typekind=Invalid] [isPOD=0]
 
 // CHECK-KEEP-GOING-ONLY: VarDecl=global_var:1:12 [type=int] [typekind=Int] [isPOD=1]
diff --git a/clang/test/Index/move-assignment-operator.cpp b/clang/test/Index/move-assignment-operator.cpp
index a2c71e679e0c6..d6c4e9a64789f 100644
--- a/clang/test/Index/move-assignment-operator.cpp
+++ b/clang/test/Index/move-assignment-operator.cpp
@@ -37,7 +37,7 @@ class Bar {
 // CHECK: CXXMethod=operator=:13:10 [type=bool (volatile unsigned int &&)] [typekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [volatile unsigned int &&] [RValueReference]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:14:10 [type=bool (const volatile unsigned char &&)] [typekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [const volatile unsigned char &&] [RValueReference]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:15:10 [type=bool (int)] [typekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [int] [Int]] [isPOD=0] [isAnonRecDecl=0]
-// CHECK: CXXMethod=operator=:16:10 (copy-assignment operator) [type=bool (Foo)] [typekind=FunctionProto] [canonicaltype=bool (Foo)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [Foo] [Elaborated]] [isPOD=0] [isAnonRecDecl=0]
+// CHECK: CXXMethod=operator=:16:10 (copy-assignment operator) [type=bool (Foo)] [typekind=FunctionProto] [canonicaltype=bool (Foo)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [Foo] [Record]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: ClassTemplate=Bar:21:7 (Definition) [type=] [typekind=Invalid] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:22:10 (move-assignment operator) [type=bool (const Bar<T> &&)] [typekind=FunctionProto] [canonicaltype=bool (const Bar<T> &&)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [const Bar<T> &&] [RValueReference]] [isPOD=0] [isAnonRecDecl=0]
 // CHECK: CXXMethod=operator=:23:10 (move-assignment operator) [type=bool (Bar<T> &&)] [typekind=FunctionProto] [canonicaltype=bool (Bar<T> &&)] [canonicaltypekind=FunctionProto] [resulttype=bool] [resulttypekind=Bool] [args= [Bar<T> &&] [RValueReference]] [isPOD=0] [isAnonRecDecl=0]
diff --git a/clang/test/Index/opencl-types.cl b/clang/test/Index/opencl-types.cl
index 13d7937b85447..485060167d218 100644
--- a/clang/test/Index/opencl-types.cl
+++ b/clang/test/Index/opencl-types.cl
@@ -17,11 +17,11 @@ void kernel testFloatTypes() {
 }
 
 // CHECK: VarDecl=scalarHalf:11:8 (Definition){{( \(invalid\))?}} [type=__private half] [typekind=Half] [isPOD=1]
-// CHECK: VarDecl=vectorHalf:12:9 (Definition) [type=__private half4] [typekind=Elaborated] [canonicaltype=half __private __attribute__((ext_vector_type(4)))] [canonicaltypekind=ExtVector] [isPOD=1]
+// CHECK: VarDecl=vectorHalf:12:9 (Definition) [type=__private half4] [typekind=Typedef] [canonicaltype=half __private __attribute__((ext_vector_type(4)))] [canonicaltypekind=ExtVector] [isPOD=1]
 // CHECK: VarDecl=scalarFloat:13:9 (Definition) [type=__private float] [typekind=Float] [isPOD=1]
-// CHECK: VarDecl=vectorFloat:14:10 (Definition) [type=__private float4] [typekind=Elaborated] [canonicaltype=float __private __attribute__((ext_vector_type(4)))] [canonicaltypekind=ExtVector] [isPOD=1]
+// CHECK: VarDecl=vectorFloat:14:10 (Definition) [type=__private float4] [typekind=Typedef] [canonicaltype=float __private __attribute__((ext_vector_type(4)))] [canonicaltypekind=ExtVector] [isPOD=1]
 // CHECK: VarDecl=scalarDouble:15:10 (Definition){{( \(invalid\))?}} [type=__private double] [typekind=Double] [isPOD=1]
-// CHECK: VarDecl=vectorDouble:16:11 (Definition){{( \(invalid\))?}} [type=__private double4] [typekind=Elaborated] [canonicaltype=double __private __attribute__((ext_vector_type(4)))] [canonicaltypekind=ExtVector] [isPOD=1]
+// CHECK: VarDecl=vectorDouble:16:11 (Definition){{( \(invalid\))?}} [type=__private double4] [typekind=Typedef] [canonicaltype=double __private __attribute__((ext_vector_type(4)))] [canonicaltypekind=ExtVector] [isPOD=1]
 
 #pragma OPENCL EXTENSION cl_khr_gl_msaa_sharing : enable
 
@@ -120,10 +120,10 @@ void kernel testMiscOpenCLTypes() {
   reserve_id_t scalarOCLReserveID;
 }
 
-// CHECK: VarDecl=scalarOCLSampler:117:19 (Definition) [type=const sampler_t] [typekind=Elaborated] const [canonicaltype=const sampler_t] [canonicaltypekind=OCLSampler] [isPOD=1]
-// CHECK: VarDecl=scalarOCLEvent:118:15 (Definition) [type=__private clk_event_t] [typekind=Elaborated] [canonicaltype=__private clk_event_t] [canonicaltypekind=Unexposed] [isPOD=1]
-// CHECK: VarDecl=scalarOCLQueue:119:11 (Definition) [type=__private queue_t] [typekind=Elaborated] [canonicaltype=__private queue_t] [canonicaltypekind=OCLQueue] [isPOD=1]
-// CHECK: VarDecl=scalarOCLReserveID:120:16 (Definition) [type=__private reserve_id_t] [typekind=Elaborated] [canonicaltype=__private reserve_id_t] [canonicaltypekind=OCLReserveID] [isPOD=1]
+// CHECK: VarDecl=scalarOCLSampler:117:19 (Definition) [type=const sampler_t] [typekind=Typedef] const [canonicaltype=const sampler_t] [canonicaltypekind=OCLSampler] [isPOD=1]
+// CHECK: VarDecl=scalarOCLEvent:118:15 (Definition) [type=__private clk_event_t] [typekind=Typedef] [canonicaltype=__private clk_event_t] [canonicaltypekind=Unexposed] [isPOD=1]
+// CHECK: VarDecl=scalarOCLQueue:119:11 (Definition) [type=__private queue_t] [typekind=Typedef] [canonicaltype=__private queue_t] [canonicaltypekind=OCLQueue] [isPOD=1]
+// CHECK: VarDecl=scalarOCLReserveID:120:16 (Definition) [type=__private reserve_id_t] [typekind=Typedef] [canonicaltype=__private reserve_id_t] [canonicaltypekind=OCLReserveID] [isPOD=1]
 
 #pragma OPENCL EXTENSION cl_intel_device_side_avc_motion_estimation : enable
 
@@ -131,4 +131,4 @@ void kernel testExtOpenCLTypes() {
   intel_sub_group_avc_mce_payload_t mce_payload;
 }
 
-// CHECK: VarDecl=mce_payload:131:37 (Definition){{( \(invalid\))?}} [type=__private intel_sub_group_avc_mce_payload_t] [typekind=Elaborated] [canonicaltype=__private intel_sub_group_avc_mce_payload_t] [canonicaltypekind=OCLIntelSubgroupAVCMcePayload] [isPOD=1]
+// CHECK: VarDecl=mce_payload:131:37 (Definition){{( \(invalid\))?}} [type=__private intel_sub_group_avc_mce_payload_t] [typekind=Typedef] [canonicaltype=__private intel_sub_group_avc_mce_payload_t] [canonicaltypekind=OCLIntelSubgroupAVCMcePayload] [isPOD=1]
diff --git a/clang/test/Index/paren-type.c b/clang/test/Index/paren-type.c
index 09751917a7f44..14a7785e0580d 100644
--- a/clang/test/Index/paren-type.c
+++ b/clang/test/Index/paren-type.c
@@ -9,7 +9,7 @@ extern int (VariableWithParentheses);
 
 typedef int MyTypedef;
 // CHECK-TYPE: VarDecl=VariableWithParentheses2:
-// CHECK-TYPE-SAME: [type=MyTypedef] [typekind=Elaborated]
+// CHECK-TYPE-SAME: [type=MyTypedef] [typekind=Typedef]
 // CHECK-TYPE-SAME: [canonicaltype=int] [canonicaltypekind=Int]
 // CHECK-TYPEDECL: VarDecl=VariableWithParentheses2
 // CHECK-TYPEDECL-SAME: [typedeclaration=MyTypedef] [typekind=Typedef]
diff --git a/clang/test/Index/print-type-size.cpp b/clang/test/Index/print-type-size.cpp
index a365528f5bc9f..6a5a02a085693 100644
--- a/clang/test/Index/print-type-size.cpp
+++ b/clang/test/Index/print-type-size.cpp
@@ -45,8 +45,8 @@ union u {
   struct simple s1;
 };
 
-// CHECK64: VarDecl=s1:[[@LINE+2]]:8 (Definition) [type=simple] [typekind=Elaborated] [sizeof=48] [alignof=8]
-// CHECK32: VarDecl=s1:[[@LINE+1]]:8 (Definition) [type=simple] [typekind=Elaborated] [sizeof=36] [alignof=4]
+// CHECK64: VarDecl=s1:[[@LINE+2]]:8 (Definition) [type=simple] [typekind=Record] [sizeof=48] [alignof=8]
+// CHECK32: VarDecl=s1:[[@LINE+1]]:8 (Definition) [type=simple] [typekind=Record] [sizeof=36] [alignof=4]
 simple s1;
 
 struct Test {
@@ -354,8 +354,8 @@ struct BaseStruct
     BaseStruct(){}
     double v0;
     float v1;
-// CHECK64: FieldDecl=fg:[[@LINE+2]]:7 (Definition) [type=C] [typekind=Elaborated] [sizeof=88] [alignof=8] [offsetof=128]
-// CHECK32: FieldDecl=fg:[[@LINE+1]]:7 (Definition) [type=C] [typekind=Elaborated] [sizeof=60] [alignof=4] [offsetof=96]
+// CHECK64: FieldDecl=fg:[[@LINE+2]]:7 (Definition) [type=C] [typekind=Record] [sizeof=88] [alignof=8] [offsetof=128]
+// CHECK32: FieldDecl=fg:[[@LINE+1]]:7 (Definition) [type=C] [typekind=Record] [sizeof=60] [alignof=4] [offsetof=96]
     C fg;
 // CHECK64: FieldDecl=rg:[[@LINE+2]]:8 (Definition) [type=C &] [typekind=LValueReference] [sizeof=88] [alignof=8] [offsetof=832]
 // CHECK32: FieldDecl=rg:[[@LINE+1]]:8 (Definition) [type=C &] [typekind=LValueReference] [sizeof=60] [alignof=4] [offsetof=576]
diff --git a/clang/test/Index/print-type.c b/clang/test/Index/print-type.c
index 7375644f10059..d30f4bed246c5 100644
--- a/clang/test/Index/print-type.c
+++ b/clang/test/Index/print-type.c
@@ -32,10 +32,10 @@ void fun(struct { int x; int y; } *param);
 _Atomic(unsigned long) aul;
 
 // RUN: c-index-test -test-print-type %s | FileCheck %s
-// CHECK: FunctionDecl=f:3:6 (Definition) [type=int *(int *, char *, FooType, int *, void (*)(int))] [typekind=FunctionProto] [canonicaltype=int *(int *, char *, int, int *, void (*)(int))] [canonicaltypekind=FunctionProto] [resulttype=int *] [resulttypekind=Pointer] [args= [int *] [Pointer] [char *] [Pointer] [FooType] [Elaborated] [int[5]] [ConstantArray] [void (*)(int)] [Pointer]] [isPOD=0]
+// CHECK: FunctionDecl=f:3:6 (Definition) [type=int *(int *, char *, FooType, int *, void (*)(int))] [typekind=FunctionProto] [canonicaltype=int *(int *, char *, int, int *, void (*)(int))] [canonicaltypekind=FunctionProto] [resulttype=int *] [resulttypekind=Pointer] [args= [int *] [Pointer] [char *] [Pointer] [FooType] [Typedef] [int[5]] [ConstantArray] [void (*)(int)] [Pointer]] [isPOD=0]
 // CHECK: ParmDecl=p:3:13 (Definition) [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
 // CHECK: ParmDecl=x:3:22 (Definition) [type=char *] [typekind=Pointer] [isPOD=1] [pointeetype=char] [pointeekind=Char_{{[US]}}]
-// CHECK: ParmDecl=z:3:33 (Definition) [type=FooType] [typekind=Elaborated] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: ParmDecl=z:3:33 (Definition) [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
 // CHECK: TypeRef=FooType:1:13 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
 // CHECK: ParmDecl=arr:3:40 (Definition) [type=int[5]] [typekind=ConstantArray] [isPOD=1]
 // CHECK: IntegerLiteral= [type=int] [typekind=Int] [isPOD=1]
@@ -47,14 +47,14 @@ _Atomic(unsigned long) aul;
 // CHECK: UnaryOperator= [type=int] [typekind=Int] [isPOD=1]
 // CHECK: DeclRefExpr=p:3:13 [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
 // CHECK: DeclStmt= [type=] [typekind=Invalid] [isPOD=0]
-// CHECK: VarDecl=w:5:17 (Definition) [type=const FooType] [typekind=Elaborated] const [canonicaltype=const int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: VarDecl=w:5:17 (Definition) [type=const FooType] [typekind=Typedef] const [canonicaltype=const int] [canonicaltypekind=Int] [isPOD=1]
 // CHECK: TypeRef=FooType:1:13 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
-// CHECK: DeclRefExpr=z:3:33 [type=FooType] [typekind=Elaborated] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: DeclRefExpr=z:3:33 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
 // CHECK: ReturnStmt= [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: BinaryOperator=+ [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
 // CHECK: BinaryOperator=+ [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
 // CHECK: DeclRefExpr=p:3:13 [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
-// CHECK: DeclRefExpr=z:3:33 [type=FooType] [typekind=Elaborated] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: DeclRefExpr=z:3:33 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
 // CHECK: ArraySubscriptExpr= [type=int] [typekind=Int] [isPOD=1]
 // CHECK: UnexposedExpr=arr:3:40 [type=int[5]] [typekind=ConstantArray] [isPOD=1]
 // CHECK: IntegerLiteral= [type=int] [typekind=Int] [isPOD=1]
@@ -64,10 +64,10 @@ _Atomic(unsigned long) aul;
 // CHECK: VarDecl=x:10:38 [type=__attribute__((__vector_size__(4 * sizeof(int)))) int] [typekind=Vector] [isPOD=1]
 // CHECK: TypedefDecl=int4_t:11:46 (Definition) [type=int4_t] [typekind=Typedef] [canonicaltype=__attribute__((__vector_size__(4 * sizeof(int)))) int] [canonicaltypekind=Vector] [isPOD=1]
 // CHECK: ParmDecl=incompletearray:13:12 (Definition) [type=int[]] [typekind=IncompleteArray] [isPOD=1]
-// CHECK: FunctionDecl=elaboratedEnumType:15:25 [type=enum Enum ()] [typekind=FunctionNoProto] [canonicaltype=enum Enum ()] [canonicaltypekind=FunctionNoProto] [resulttype=enum Enum] [resulttypekind=Elaborated] [isPOD=0]
+// CHECK: FunctionDecl=elaboratedEnumType:15:25 [type=enum Enum ()] [typekind=FunctionNoProto] [canonicaltype=enum Enum ()] [canonicaltypekind=FunctionNoProto] [resulttype=enum Enum] [resulttypekind=Enum] [isPOD=0]
 // CHECK: TypeRef=enum Enum:15:6 [type=enum Enum] [typekind=Enum] [isPOD=1]
 // CHECK: StructDecl=Struct:16:8 (Definition) [type=struct Struct] [typekind=Record] [isPOD=1]
-// CHECK: FunctionDecl=elaboratedStructType:16:32 [type=struct Struct ()] [typekind=FunctionNoProto] [canonicaltype=struct Struct ()] [canonicaltypekind=FunctionNoProto] [resulttype=struct Struct] [resulttypekind=Elaborated] [isPOD=0]
+// CHECK: FunctionDecl=elaboratedStructType:16:32 [type=struct Struct ()] [typekind=FunctionNoProto] [canonicaltype=struct Struct ()] [canonicaltypekind=FunctionNoProto] [resulttype=struct Struct] [resulttypekind=Record] [isPOD=0]
 // CHECK: TypeRef=struct Struct:16:8 [type=struct Struct] [typekind=Record] [isPOD=1]
 // CHECK: StructDecl=struct (unnamed at {{.*}}):18:1 (Definition) [type=struct (unnamed at {{.*}}print-type.c:18:1)] [typekind=Record] [isPOD=1] [nbFields=2] [isAnon=1] [isAnonRecDecl=0]
 // CHECK: StructDecl=struct (unnamed at {{.*}}):23:1 (Definition) [type=struct (unnamed at {{.*}}print-type.c:23:1)] [typekind=Record] [isPOD=1] [nbFields=1] [isAnon=1] [isAnonRecDecl=0]
diff --git a/clang/test/Index/print-type.cpp b/clang/test/Index/print-type.cpp
index 141895d1dd2b2..ef9805f6a1c62 100644
--- a/clang/test/Index/print-type.cpp
+++ b/clang/test/Index/print-type.cpp
@@ -105,38 +105,38 @@ inline namespace InlineNS {}
 // CHECK: Namespace=inner:14:11 (Definition) [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: StructDecl=Bar:16:8 (Definition) [type=outer::inner::Bar] [typekind=Record] [isPOD=0] [nbFields=3]
 // CHECK: CXXConstructor=Bar:17:3 (Definition) (converting constructor) [type=void (outer::Foo<bool> *){{.*}}] [typekind=FunctionProto] [canonicaltype=void (outer::Foo<bool> *){{.*}}] [canonicaltypekind=FunctionProto] [resulttype=void] [resulttypekind=Void] [args= [outer::Foo<bool> *] [Pointer]] [isPOD=0]
-// CHECK: ParmDecl=foo:17:25 (Definition) [type=outer::Foo<bool> *] [typekind=Pointer] [canonicaltype=outer::Foo<bool> *] [canonicaltypekind=Pointer] [isPOD=1] [pointeetype=outer::Foo<bool>] [pointeekind=Elaborated]
+// CHECK: ParmDecl=foo:17:25 (Definition) [type=outer::Foo<bool> *] [typekind=Pointer] [canonicaltype=outer::Foo<bool> *] [canonicaltypekind=Pointer] [isPOD=1] [pointeetype=outer::Foo<bool>] [pointeekind=Unexposed]
 // CHECK: NamespaceRef=outer:1:11 [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: TemplateRef=Foo:4:8 [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: CompoundStmt= [type=] [typekind=Invalid] [isPOD=0]
-// CHECK: TypedefDecl=FooType:19:15 (Definition) [type=outer::inner::Bar::FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
-// CHECK: TypeAliasDecl=AliasType:20:9 (Definition) [type=outer::inner::Bar::AliasType] [typekind=Typedef] [canonicaltype=double] [canonicaltypekind=Double] [isPOD=1]
+// CHECK: TypedefDecl=FooType:19:15 (Definition) [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: TypeAliasDecl=AliasType:20:9 (Definition) [type=AliasType] [typekind=Typedef] [canonicaltype=double] [canonicaltypekind=Double] [isPOD=1]
 // CHECK: FieldDecl=p:21:8 (Definition) [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
-// CHECK: CXXMethod=f:22:8 (Definition) [type=int *(int *, char *, FooType){{.*}}] [typekind=FunctionProto] [canonicaltype=int *(int *, char *, int){{.*}}] [canonicaltypekind=FunctionProto] [resulttype=int *] [resulttypekind=Pointer] [args= [int *] [Pointer] [char *] [Pointer] [FooType] [Elaborated]] [isPOD=0]
+// CHECK: CXXMethod=f:22:8 (Definition) [type=int *(int *, char *, FooType){{.*}}] [typekind=FunctionProto] [canonicaltype=int *(int *, char *, int){{.*}}] [canonicaltypekind=FunctionProto] [resulttype=int *] [resulttypekind=Pointer] [args= [int *] [Pointer] [char *] [Pointer] [FooType] [Typedef]] [isPOD=0]
 // CHECK: ParmDecl=p:22:15 (Definition) [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
 // CHECK: ParmDecl=x:22:24 (Definition) [type=char *] [typekind=Pointer] [isPOD=1] [pointeetype=char] [pointeekind=Char_{{[US]}}]
-// CHECK: ParmDecl=z:22:35 (Definition) [type=FooType] [typekind=Elaborated] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
-// CHECK: TypeRef=outer::inner::Bar::FooType:19:15 [type=outer::inner::Bar::FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: ParmDecl=z:22:35 (Definition) [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: TypeRef=outer::inner::Bar::FooType:19:15 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
 // CHECK: CompoundStmt= [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: DeclStmt= [type=] [typekind=Invalid] [isPOD=0]
-// CHECK: VarDecl=w:23:19 (Definition) [type=const FooType] [typekind=Elaborated] const [canonicaltype=const int] [canonicaltypekind=Int] [isPOD=1]
-// CHECK: TypeRef=outer::inner::Bar::FooType:19:15 [type=outer::inner::Bar::FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
-// CHECK: UnexposedExpr=z:22:35 [type=FooType] [typekind=Elaborated] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
-// CHECK: DeclRefExpr=z:22:35 [type=FooType] [typekind=Elaborated] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: VarDecl=w:23:19 (Definition) [type=const FooType] [typekind=Typedef] const [canonicaltype=const int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: TypeRef=outer::inner::Bar::FooType:19:15 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: UnexposedExpr=z:22:35 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: DeclRefExpr=z:22:35 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
 // CHECK: ReturnStmt= [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: BinaryOperator=+ [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
 // CHECK: UnexposedExpr=p:22:15 [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
 // CHECK: DeclRefExpr=p:22:15 [type=int *] [typekind=Pointer] [isPOD=1] [pointeetype=int] [pointeekind=Int]
-// CHECK: UnexposedExpr=z:22:35 [type=FooType] [typekind=Elaborated] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
-// CHECK: DeclRefExpr=z:22:35 [type=FooType] [typekind=Elaborated] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
-// CHECK: TypedefDecl=OtherType:26:18 (Definition) [type=outer::inner::Bar::OtherType] [typekind=Typedef] [canonicaltype=double] [canonicaltypekind=Double] [isPOD=1]
-// CHECK: TypedefDecl=ArrayType:27:15 (Definition) [type=outer::inner::Bar::ArrayType] [typekind=Typedef] [canonicaltype=int[5]] [canonicaltypekind=ConstantArray] [isPOD=1]
+// CHECK: UnexposedExpr=z:22:35 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: DeclRefExpr=z:22:35 [type=FooType] [typekind=Typedef] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
+// CHECK: TypedefDecl=OtherType:26:18 (Definition) [type=OtherType] [typekind=Typedef] [canonicaltype=double] [canonicaltypekind=Double] [isPOD=1]
+// CHECK: TypedefDecl=ArrayType:27:15 (Definition) [type=ArrayType] [typekind=Typedef] [canonicaltype=int[5]] [canonicaltypekind=ConstantArray] [isPOD=1]
 // CHECK: IntegerLiteral= [type=int] [typekind=Int] [isPOD=1]
-// CHECK: FieldDecl=baz:28:20 (Definition) [type=Baz<int, 1, Foo>] [typekind=Elaborated] [templateargs/3= [type=int] [typekind=Int]] [canonicaltype=outer::Baz<int, 1, outer::Foo>] [canonicaltypekind=Record] [canonicaltemplateargs/3= [type=int] [typekind=Int]] [isPOD=1]
+// CHECK: FieldDecl=baz:28:20 (Definition) [type=Baz<int, 1, Foo>] [typekind=Unexposed] [templateargs/3= [type=int] [typekind=Int]] [canonicaltype=outer::Baz<int, 1, outer::Foo>] [canonicaltypekind=Record] [canonicaltemplateargs/3= [type=int] [typekind=Int]] [isPOD=1]
 // CHECK: TemplateRef=Baz:9:8 [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: IntegerLiteral= [type=int] [typekind=Int] [isPOD=1]
 // CHECK: TemplateRef=Foo:4:8 [type=] [typekind=Invalid] [isPOD=0]
-// CHECK: FieldDecl=qux:29:38 (Definition) [type=Qux<int, char *, Foo<int>, FooType>] [typekind=Elaborated] [templateargs/4= [type=int] [typekind=Int] [type=char *] [typekind=Pointer] [type=Foo<int>] [typekind=Elaborated] [type=FooType] [typekind=Elaborated]] [canonicaltype=outer::Qux<int, char *, outer::Foo<int>, int>] [canonicaltypekind=Record] [canonicaltemplateargs/4= [type=int] [typekind=Int] [type=char *] [typekind=Pointer] [type=outer::Foo<int>] [typekind=Record] [type=int] [typekind=Int]] [isPOD=1]
+// CHECK: FieldDecl=qux:29:38 (Definition) [type=Qux<int, char *, Foo<int>, FooType>] [typekind=Unexposed] [templateargs/4= [type=int] [typekind=Int] [type=char *] [typekind=Pointer] [type=Foo<int>] [typekind=Unexposed] [type=FooType] [typekind=Typedef]] [canonicaltype=outer::Qux<int, char *, outer::Foo<int>, int>] [canonicaltypekind=Record] [canonicaltemplateargs/4= [type=int] [typekind=Int] [type=char *] [typekind=Pointer] [type=outer::Foo<int>] [typekind=Record] [type=int] [typekind=Int]] [isPOD=1]
 // CHECK: TemplateRef=Qux:12:8 [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: TemplateRef=Foo:4:8 [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: FunctionTemplate=tbar:36:3 [type=T (int)] [typekind=FunctionProto] [canonicaltype=type-parameter-0-0 (int)] [canonicaltypekind=FunctionProto] [resulttype=T] [resulttypekind=Unexposed] [isPOD=0]
@@ -163,11 +163,11 @@ inline namespace InlineNS {}
 // CHECK: DeclRefExpr=i:44:14 [type=int] [typekind=Int] [isPOD=1]
 // CHECK: StructDecl=Blob:46:8 (Definition) [type=Blob] [typekind=Record] [isPOD=1] [nbFields=2]
 // CHECK: FieldDecl=i:47:7 (Definition) [type=int] [typekind=Int] [isPOD=1]
-// CHECK: VarDecl=member_pointer:50:12 (Definition) [type=int Blob::*] [typekind=MemberPointer] [isPOD=1] [pointeetype=int] [pointeekind=Int] [isAnonRecDecl=0]
-// CHECK: FunctionDecl=elaboratedNamespaceType:52:42 [type=NS::Type (const NS::Type)] [typekind=FunctionProto] [canonicaltype=NS::Type (NS::Type)] [canonicaltypekind=FunctionProto] [resulttype=NS::Type] [resulttypekind=Elaborated] [args= [const NS::Type] [Elaborated]] [isPOD=0]
+// CHECK: VarDecl=member_pointer:50:12 (Definition) [type=int Blob::*] [typekind=MemberPointer] [canonicaltype=int Blob::*] [canonicaltypekind=MemberPointer] [isPOD=1] [pointeetype=int] [pointeekind=Int] [isAnonRecDecl=0]
+// CHECK: FunctionDecl=elaboratedNamespaceType:52:42 [type=NS::Type (const NS::Type)] [typekind=FunctionProto] [canonicaltype=NS::Type (NS::Type)] [canonicaltypekind=FunctionProto] [resulttype=NS::Type] [resulttypekind=Record] [args= [const NS::Type] [Record]] [isPOD=0]
 // CHECK: NamespaceRef=NS:52:11 [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: TypeRef=struct NS::Type:52:23 [type=NS::Type] [typekind=Record] [isPOD=1]
-// CHECK: ParmDecl=t:52:81 (Definition) [type=const NS::Type] [typekind=Elaborated] const [canonicaltype=const NS::Type] [canonicaltypekind=Record] [isPOD=1]
+// CHECK: ParmDecl=t:52:81 (Definition) [type=const NS::Type] [typekind=Record] const [canonicaltype=const NS::Type] [canonicaltypekind=Record] [isPOD=1]
 // CHECK: VarDecl=autoI:54:6 (Definition) [type=int] [typekind=Auto] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
 // CHECK: IntegerLiteral= [type=int] [typekind=Int] [isPOD=1]
 // CHECK: VarDecl=autoTbar:55:6 (Definition) [type=int] [typekind=Auto] [canonicaltype=int] [canonicaltypekind=Int] [isPOD=1]
@@ -176,9 +176,9 @@ inline namespace InlineNS {}
 // CHECK: DeclRefExpr=tbar:36:3 RefName=[55:17 - 55:21] RefName=[55:21 - 55:26] [type=int (int)] [typekind=FunctionProto] [canonicaltype=int (int)] [canonicaltypekind=FunctionProto] [isPOD=0]
 // CHECK: IntegerLiteral= [type=int] [typekind=Int] [isPOD=1]
 // CHECK: VarDecl=autoBlob:56:6 (Definition) [type=Blob *] [typekind=Auto] [canonicaltype=Blob *] [canonicaltypekind=Pointer] [isPOD=1]
-// CHECK: CXXNewExpr= [type=Blob *] [typekind=Pointer] [canonicaltype=Blob *] [canonicaltypekind=Pointer] [isPOD=1] [pointeetype=Blob] [pointeekind=Elaborated]
+// CHECK: CXXNewExpr= [type=Blob *] [typekind=Pointer] [canonicaltype=Blob *] [canonicaltypekind=Pointer] [isPOD=1] [pointeetype=Blob] [pointeekind=Record]
 // CHECK: TypeRef=struct Blob:46:8 [type=Blob] [typekind=Record] [isPOD=1] [nbFields=2]
-// CHECK: CallExpr=Blob:46:8 [type=Blob] [typekind=Elaborated] [canonicaltype=Blob] [canonicaltypekind=Record] [isPOD=1] [nbFields=2]
+// CHECK: CallExpr=Blob:46:8 [type=Blob] [typekind=Record] [canonicaltype=Blob] [canonicaltypekind=Record] [isPOD=1] [nbFields=2]
 // CHECK: FunctionDecl=autoFunction:57:6 (Definition) [type=int ()] [typekind=FunctionProto] [canonicaltype=int ()] [canonicaltypekind=FunctionProto] [resulttype=int] [resulttypekind=Auto] [isPOD=0]
 // CHECK: CompoundStmt= [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: ReturnStmt= [type=] [typekind=Invalid] [isPOD=0]
@@ -187,20 +187,20 @@ inline namespace InlineNS {}
 // CHECK: IntegerLiteral= [type=int] [typekind=Int] [isPOD=1]
 // CHECK: TypeAliasTemplateDecl=TypeAlias:61:1 (Definition) [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: TemplateTypeParameter=T:60:20 (Definition) [type=T] [typekind=Unexposed] [canonicaltype=type-parameter-0-0] [canonicaltypekind=Unexposed] [isPOD=0]
-// CHECK: FieldDecl=foo:63:39 (Definition) [type=TypeAlias<int>] [typekind=Elaborated] [templateargs/1= [type=int] [typekind=Int]] [canonicaltype=outer::Qux<int>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=int] [typekind=Int]] [isPOD=1]
+// CHECK: FieldDecl=foo:63:39 (Definition) [type=TypeAlias<int>] [typekind=Unexposed] [templateargs/1= [type=int] [typekind=Int]] [canonicaltype=outer::Qux<int>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=int] [typekind=Int]] [isPOD=1]
 // CHECK: TemplateRef=TypeAlias:61:1 [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: ClassTemplate=Specialization:66:8 (Definition) [type=] [typekind=Invalid] [isPOD=0]
 // CHECK: TemplateTypeParameter=T:65:19 (Definition) [type=T] [typekind=Unexposed] [canonicaltype=type-parameter-0-0] [canonicaltypekind=Unexposed] [isPOD=0]
 // CHECK: StructDecl=Specialization:69:8 [Specialization of Specialization:66:8] [Template arg 0: kind: 1, type: int] [type=Specialization<int>] [typekind=Record] [templateargs/1= [type=int] [typekind=Int]] [isPOD=0]
-// CHECK: VarDecl=templRefParam:71:40 (Definition) [type=Specialization<Specialization<bool> &>] [typekind=Elaborated] [templateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [canonicaltype=Specialization<Specialization<bool> &>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [isPOD=1]
+// CHECK: VarDecl=templRefParam:71:40 (Definition) [type=Specialization<Specialization<bool> &>] [typekind=Unexposed] [templateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [canonicaltype=Specialization<Specialization<bool> &>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [isPOD=1]
 // CHECK: TemplateRef=Specialization:66:8 [type=] [typekind=Invalid] [isPOD=0]
-// CHECK: CallExpr=Specialization:66:8 [type=Specialization<Specialization<bool> &>] [typekind=Elaborated] [templateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [canonicaltype=Specialization<Specialization<bool> &>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [isPOD=1]
+// CHECK: CallExpr=Specialization:66:8 [type=Specialization<Specialization<bool> &>] [typekind=Unexposed] [templateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [canonicaltype=Specialization<Specialization<bool> &>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [isPOD=1]
 // CHECK: VarDecl=autoTemplRefParam:72:6 (Definition) [type=Specialization<Specialization<bool> &>] [typekind=Auto] [templateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [canonicaltype=Specialization<Specialization<bool> &>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [isPOD=1]
-// CHECK: UnexposedExpr=templRefParam:71:40 [type=const Specialization<Specialization<bool> &>] [typekind=Elaborated] const [templateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [canonicaltype=const Specialization<Specialization<bool> &>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [isPOD=1] [isAnonRecDecl=0]
-// CHECK: DeclRefExpr=templRefParam:71:40 [type=Specialization<Specialization<bool> &>] [typekind=Elaborated] [templateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [canonicaltype=Specialization<Specialization<bool> &>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [isPOD=1]
-// CHECK: TypeAliasDecl=baz:76:7 (Definition) [type=baz] [typekind=Typedef] [templateargs/1= [type=A<void>] [typekind=Elaborated]] [canonicaltype=A<void>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=void] [typekind=Void]] [isPOD=0]
+// CHECK: UnexposedExpr=templRefParam:71:40 [type=const Specialization<Specialization<bool> &>] [typekind=Record] const [templateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [canonicaltype=const Specialization<Specialization<bool> &>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [isPOD=1] [isAnonRecDecl=0]
+// CHECK: DeclRefExpr=templRefParam:71:40 [type=Specialization<Specialization<bool> &>] [typekind=Unexposed] [templateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [canonicaltype=Specialization<Specialization<bool> &>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=Specialization<bool> &] [typekind=LValueReference]] [isPOD=1]
+// CHECK: TypeAliasDecl=baz:76:7 (Definition) [type=baz] [typekind=Typedef] [templateargs/1= [type=A<void>] [typekind=Unexposed]] [canonicaltype=A<void>] [canonicaltypekind=Record] [canonicaltemplateargs/1= [type=void] [typekind=Void]] [isPOD=0]
 // CHECK: VarDecl=autoTemplPointer:78:6 (Definition) [type=Specialization<Specialization<bool> &> *] [typekind=Auto] [canonicaltype=Specialization<Specialization<bool> &> *] [canonicaltypekind=Pointer] [isPOD=1] [pointeetype=Specialization<Specialization<bool> &>] [pointeekind=Auto]
-// CHECK: CallExpr=Bar:17:3 [type=outer::inner::Bar] [typekind=Elaborated] [canonicaltype=outer::inner::Bar] [canonicaltypekind=Record] [args= [outer::Foo<bool> *] [Pointer]] [isPOD=0] [nbFields=3]
+// CHECK: CallExpr=Bar:17:3 [type=outer::inner::Bar] [typekind=Record] [canonicaltype=outer::inner::Bar] [canonicaltypekind=Record] [args= [outer::Foo<bool> *] [Pointer]] [isPOD=0] [nbFields=3]
 // CHECK: StructDecl=(anonymous struct at {{.*}}):84:3 (Definition) [type=X::(anonymous struct at {{.*}}print-type.cpp:84:3)] [typekind=Record] [isPOD=1] [nbFields=1] [isAnon=1]
 // CHECK: ClassDecl=(anonymous class at {{.*}}:85:3 (Definition) [type=X::(anonymous class at {{.*}}print-type.cpp:85:3)] [typekind=Record] [isPOD=1] [nbFields=1] [isAnon=1]
 // CHECK: UnionDecl=(anonymous union at {{.*}}:86:3 (Definition) [type=X::(anonymous union at {{.*}}print-type.cpp:86:3)] [typekind=Record] [isPOD=1] [nbFields=2] [isAnon=1]
diff --git a/clang/test/Index/recursive-cxx-member-calls.cpp b/clang/test/Index/recursive-cxx-member-calls.cpp
index 11c011a432cd4..b378975b5a763 100644
--- a/clang/test/Index/recursive-cxx-member-calls.cpp
+++ b/clang/test/Index/recursive-cxx-member-calls.cpp
@@ -402,7 +402,7 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo * Name) {
 // CHECK-tokens: Identifier: "getKind" [33:17 - 33:24] CXXMethod=getKind:33:17 (static)
 // CHECK-tokens: Punctuation: "(" [33:24 - 33:25] CXXMethod=getKind:33:17 (static)
 // CHECK-tokens: Keyword: "const" [33:25 - 33:30] ParmDecl=Name:33:48 (Definition)
-// CHECK-tokens: Identifier: "IdentifierInfo" [33:31 - 33:45] TypeRef=class clang::IdentifierInfo:66:7
+// CHECK-tokens: Identifier: "IdentifierInfo" [33:31 - 33:45] TypeRef=class clang::IdentifierInfo:11:9
 // CHECK-tokens: Punctuation: "*" [33:46 - 33:47] ParmDecl=Name:33:48 (Definition)
 // CHECK-tokens: Identifier: "Name" [33:48 - 33:52] ParmDecl=Name:33:48 (Definition)
 // CHECK-tokens: Punctuation: ")" [33:52 - 33:53] CXXMethod=getKind:33:17 (static)
@@ -823,18 +823,18 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo * Name) {
 // CHECK-tokens: Punctuation: ";" [85:18 - 85:19] ClassTemplate=StringSwitch:83:47 (Definition)
 // CHECK-tokens: Keyword: "public" [86:1 - 86:7] CXXAccessSpecifier=:86:1 (Definition)
 // CHECK-tokens: Punctuation: ":" [86:7 - 86:8] CXXAccessSpecifier=:86:1 (Definition)
-// CHECK-tokens: Keyword: "explicit" [87:3 - 87:11] CXXConstructor=StringSwitch<T, R>:87:12 (Definition)
-// CHECK-tokens: Identifier: "StringSwitch" [87:12 - 87:24] CXXConstructor=StringSwitch<T, R>:87:12 (Definition) (explicit)
-// CHECK-tokens: Punctuation: "(" [87:24 - 87:25] CXXConstructor=StringSwitch<T, R>:87:12 (Definition)
+// CHECK-tokens: Keyword: "explicit" [87:3 - 87:11] CXXConstructor=llvm::StringSwitch<T, R>:87:12 (Definition)
+// CHECK-tokens: Identifier: "StringSwitch" [87:12 - 87:24] CXXConstructor=llvm::StringSwitch<T, R>:87:12 (Definition) (explicit)
+// CHECK-tokens: Punctuation: "(" [87:24 - 87:25] CXXConstructor=llvm::StringSwitch<T, R>:87:12 (Definition)
 // CHECK-tokens: Identifier: "StringRef" [87:25 - 87:34] TypeRef=class llvm::StringRef:38:7
 // CHECK-tokens: Identifier: "Str" [87:35 - 87:38] ParmDecl=Str:87:35 (Definition)
-// CHECK-tokens: Punctuation: ")" [87:38 - 87:39] CXXConstructor=StringSwitch<T, R>:87:12 (Definition)
-// CHECK-tokens: Punctuation: ":" [87:40 - 87:41] CXXConstructor=StringSwitch<T, R>:87:12 (Definition)
+// CHECK-tokens: Punctuation: ")" [87:38 - 87:39] CXXConstructor=llvm::StringSwitch<T, R>:87:12 (Definition)
+// CHECK-tokens: Punctuation: ":" [87:40 - 87:41] CXXConstructor=llvm::StringSwitch<T, R>:87:12 (Definition)
 // CHECK-tokens: Identifier: "Str" [87:42 - 87:45] MemberRef=Str:84:13
 // CHECK-tokens: Punctuation: "(" [87:45 - 87:46] CallExpr=StringRef:38:7
 // CHECK-tokens: Identifier: "Str" [87:46 - 87:49] DeclRefExpr=Str:87:35
 // CHECK-tokens: Punctuation: ")" [87:49 - 87:50] CallExpr=StringRef:38:7
-// CHECK-tokens: Punctuation: "," [87:50 - 87:51] CXXConstructor=StringSwitch<T, R>:87:12 (Definition)
+// CHECK-tokens: Punctuation: "," [87:50 - 87:51] CXXConstructor=llvm::StringSwitch<T, R>:87:12 (Definition)
 // CHECK-tokens: Identifier: "Result" [87:52 - 87:58] MemberRef=Result:85:12
 // CHECK-tokens: Punctuation: "(" [87:58 - 87:59] UnexposedExpr=
 // CHECK-tokens: Literal: "0" [87:59 - 87:60] IntegerLiteral=
@@ -846,7 +846,7 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo * Name) {
 // CHECK-tokens: Keyword: "unsigned" [88:14 - 88:22] NonTypeTemplateParameter=N:88:23 (Definition)
 // CHECK-tokens: Identifier: "N" [88:23 - 88:24] NonTypeTemplateParameter=N:88:23 (Definition)
 // CHECK-tokens: Punctuation: ">" [88:25 - 88:26] FunctionTemplate=Case:88:42 (Definition)
-// CHECK-tokens: Identifier: "StringSwitch" [88:27 - 88:39] TypeRef=StringSwitch<T, R>:83:47
+// CHECK-tokens: Identifier: "StringSwitch" [88:27 - 88:39] TypeRef=llvm::StringSwitch<T, R>:83:47
 // CHECK-tokens: Punctuation: "&" [88:40 - 88:41] FunctionTemplate=Case:88:42 (Definition)
 // CHECK-tokens: Identifier: "Case" [88:42 - 88:46] FunctionTemplate=Case:88:42 (Definition)
 // CHECK-tokens: Punctuation: "(" [88:46 - 88:47] FunctionTemplate=Case:88:42 (Definition)
@@ -1619,7 +1619,7 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo * Name) {
 // CHECK: 33:17: CXXMethod=getKind:33:17 (static) Extent=[33:5 - 33:53]
 // CHECK: 33:12: TypeRef=enum clang::AttributeList::Kind:13:10 Extent=[33:12 - 33:16]
 // CHECK: 33:48: ParmDecl=Name:33:48 (Definition) Extent=[33:25 - 33:52]
-// CHECK: 33:31: TypeRef=class clang::IdentifierInfo:66:7 Extent=[33:31 - 33:45]
+// CHECK: 33:31: TypeRef=class clang::IdentifierInfo:11:9 Extent=[33:31 - 33:45]
 // CHECK: 36:8: FunctionDecl=magic_length:36:8 Extent=[36:1 - 36:35]
 // CHECK: 36:1: TypeRef=size_t:2:25 Extent=[36:1 - 36:7]
 // CHECK: 36:33: ParmDecl=s:36:33 (Definition) Extent=[36:21 - 36:34]
@@ -1839,7 +1839,7 @@ AttributeList::Kind AttributeList::getKind(const IdentifierInfo * Name) {
 // CHECK: 84:3: TypeRef=class llvm::StringRef:38:7 Extent=[84:3 - 84:12]
 // CHECK: 85:12: FieldDecl=Result:85:12 (Definition) Extent=[85:3 - 85:18]
 // CHECK: 86:1: CXXAccessSpecifier=:86:1 (Definition) Extent=[86:1 - 86:8]
-// CHECK: 87:12: CXXConstructor=StringSwitch<T, R>:87:12 (Definition) (explicit) Extent=[87:3 - 87:64]
+// CHECK: 87:12: CXXConstructor=llvm::StringSwitch<T, R>:87:12 (Definition) (explicit) Extent=[87:3 - 87:64]
 // CHECK: 87:35: ParmDecl=Str:87:35 (Definition) Extent=[87:25 - 87:38]
 // CHECK: 87:25: TypeRef=class llvm::StringRef:38:7 Extent=[87:25 - 87:34]
 // CHECK: 87:42: MemberRef=Str:84:13 Extent=[87:42 - 87:45]
diff --git a/clang/test/Index/redeclarations.cpp b/clang/test/Index/redeclarations.cpp
index 11dc9320430c1..ea5e9706cbcbf 100644
--- a/clang/test/Index/redeclarations.cpp
+++ b/clang/test/Index/redeclarations.cpp
@@ -17,5 +17,5 @@ class A
 // CHECK: redeclarations.h:19:19: FieldDecl=x:19:19 (Definition) Extent=[19:5 - 19:20]
 // CHECK: redeclarations.h:19:5: TemplateRef=B:8:7 Extent=[19:5 - 19:6]
 // CHECK: redeclarations.h:19:7: TypeRef=class D:17:7 Extent=[19:7 - 19:8]
-// CHECK: redeclarations.h:19:16: TypeRef=class A:3:7 Extent=[19:16 - 19:17]
+// CHECK: redeclarations.h:19:16: TypeRef=class A:19:16 Extent=[19:16 - 19:17]
 // CHECK: redeclarations.cpp:3:7: ClassDecl=A:3:7 (Definition) Extent=[3:1 - 5:2]
diff --git a/clang/test/Index/skip-parsed-bodies/compile_commands.json b/clang/test/Index/skip-parsed-bodies/compile_commands.json
index 991227a843b1d..e087a28b1cf2c 100644
--- a/clang/test/Index/skip-parsed-bodies/compile_commands.json
+++ b/clang/test/Index/skip-parsed-bodies/compile_commands.json
@@ -25,8 +25,8 @@
 // CHECK-NEXT: [indexEntityReference]: kind: variable | name: some_val | {{.*}} | loc: .{{/|\\\\?}}t.h:9:27
 // CHECK-NEXT: [indexDeclaration]: kind: c++-instance-method | name: method_def2 | {{.*}} | isRedecl: 0 | isDef: 0 | isContainer: 0
 // CHECK-NEXT: [indexDeclaration]: kind: c++-instance-method | name: method_def2 | {{.*}} | isRedecl: 1 | isDef: 1 | isContainer: 1
-// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS |
 // CHECK-NEXT: [indexEntityReference]: kind: c++-class | name: C |
+// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS |
 // CHECK-NEXT: [indexEntityReference]: kind: variable | name: some_val | {{.*}} | loc: .{{/|\\\\?}}t.h:15:5
 // CHECK-NEXT: [indexDeclaration]: kind: function | name: foo1 | {{.*}} | isRedecl: 0 | isDef: 1 | isContainer: 1
 // CHECK-NEXT: [indexEntityReference]: kind: variable | name: some_val | {{.*}} | loc: .{{/|\\\\?}}t.h:19:5
@@ -40,8 +40,8 @@
 // CHECK-NEXT: [indexDeclaration]: kind: c++-instance-method | name: method_def1 | {{.*}} | isRedecl: 0 | isDef: 1 | isContainer: skipped
 // CHECK-NEXT: [indexDeclaration]: kind: c++-instance-method | name: method_def2 | {{.*}} | isRedecl: 0 | isDef: 0 | isContainer: 0
 // CHECK-NEXT: [indexDeclaration]: kind: c++-instance-method | name: method_def2 | {{.*}} | isContainer: skipped
-// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS |
 // CHECK-NEXT: [indexEntityReference]: kind: c++-class | name: C |
+// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS |
 // CHECK-NEXT: [indexDeclaration]: kind: function | name: foo1 | {{.*}} | isRedecl: 0 | isDef: 1 | isContainer: skipped
 // CHECK-NEXT: [ppIncludedFile]: .{{/|\\\\?}}pragma_once.h
 // CHECK-NEXT: [indexDeclaration]: kind: function | name: foo2 | {{.*}} | isRedecl: 0 | isDef: 1 | isContainer: 1
@@ -60,8 +60,8 @@
 // CHECK-NEXT: [indexDeclaration]: kind: c++-instance-method | name: method_def1 | {{.*}} | isRedecl: 0 | isDef: 1 | isContainer: skipped
 // CHECK-NEXT: [indexDeclaration]: kind: c++-instance-method | name: method_def2 | {{.*}} | isRedecl: 0 | isDef: 0 | isContainer: 0
 // CHECK-NEXT: [indexDeclaration]: kind: c++-instance-method | name: method_def2 | {{.*}} | isRedecl: 1 | isDef: 1 | isContainer: skipped
-// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS |
 // CHECK-NEXT: [indexEntityReference]: kind: c++-class | name: C |
+// CHECK-NEXT: [indexEntityReference]: kind: namespace | name: NS |
 // CHECK-NEXT: [indexDeclaration]: kind: function | name: foo1 | {{.*}} | isRedecl: 0 | isDef: 1 | isContainer: skipped
 // CHECK-NEXT: [ppIncludedFile]: .{{/|\\\\?}}pragma_once.h
 // CHECK-NEXT: [indexDeclaration]: kind: function | name: foo2 | {{.*}} | isRedecl: 0 | isDef: 1 | isContainer: skipped
diff --git a/clang/test/Interpreter/assignment-with-implicit-ctor.cpp b/clang/test/Interpreter/assignment-with-implicit-ctor.cpp
index 24cea8ec1a4b2..cef568c78663e 100644
--- a/clang/test/Interpreter/assignment-with-implicit-ctor.cpp
+++ b/clang/test/Interpreter/assignment-with-implicit-ctor.cpp
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit
-// UNSUPPORTED: system-aix
 //
 // RUN: cat %s | clang-repl | FileCheck %s
 // RUN: cat %s | clang-repl -Xcc -O2 | FileCheck %s
diff --git a/clang/test/Interpreter/code-undo.cpp b/clang/test/Interpreter/code-undo.cpp
index 83ade0ec9158b..4516910ca3b4f 100644
--- a/clang/test/Interpreter/code-undo.cpp
+++ b/clang/test/Interpreter/code-undo.cpp
@@ -1,4 +1,3 @@
-// UNSUPPORTED: system-aix
 // RUN: cat %s | clang-repl | FileCheck %s
 extern "C" int printf(const char *, ...);
 int x1 = 0;
diff --git a/clang/test/Interpreter/const.cpp b/clang/test/Interpreter/const.cpp
index 52be75e09ade7..cadd446b7504f 100644
--- a/clang/test/Interpreter/const.cpp
+++ b/clang/test/Interpreter/const.cpp
@@ -1,4 +1,3 @@
-// UNSUPPORTED: system-aix, system-zos
 // see https://github.com/llvm/llvm-project/issues/68092
 // XFAIL: host={{.*}}-windows-msvc
 
diff --git a/clang/test/Interpreter/cxx20-modules.cppm b/clang/test/Interpreter/cxx20-modules.cppm
index 4e56e2fc1528a..97744e3b25f77 100644
--- a/clang/test/Interpreter/cxx20-modules.cppm
+++ b/clang/test/Interpreter/cxx20-modules.cppm
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit, x86_64-linux
-// UNSUPPORTED: system-aix
 //
 // RUN: rm -rf %t
 // RUN: mkdir -p %t
diff --git a/clang/test/Interpreter/execute-stmts.cpp b/clang/test/Interpreter/execute-stmts.cpp
index 433c6811777da..cc27fa615f716 100644
--- a/clang/test/Interpreter/execute-stmts.cpp
+++ b/clang/test/Interpreter/execute-stmts.cpp
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit
-// UNSUPPORTED: system-aix
 // RUN: cat %s | clang-repl -Xcc -Xclang -Xcc  -verify | FileCheck %s
 // RUN: %clang_cc1 -verify -fincremental-extensions -emit-llvm -o -  %s \
 // RUN:           | FileCheck --check-prefix=CODEGEN-CHECK %s
diff --git a/clang/test/Interpreter/execute-weak.cpp b/clang/test/Interpreter/execute-weak.cpp
index 85fa5d276f5f4..f469451f5e5e4 100644
--- a/clang/test/Interpreter/execute-weak.cpp
+++ b/clang/test/Interpreter/execute-weak.cpp
@@ -1,4 +1,4 @@
-// UNSUPPORTED: system-aix, system-windows
+// UNSUPPORTED: system-windows
 // RUN: cat %s | clang-repl | FileCheck %s
 
 extern "C" int printf(const char *, ...);
diff --git a/clang/test/Interpreter/execute.c b/clang/test/Interpreter/execute.c
index 44a3a32c93011..ca8f83cf6e374 100644
--- a/clang/test/Interpreter/execute.c
+++ b/clang/test/Interpreter/execute.c
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit
-// UNSUPPORTED: system-aix
 
 // RUN: cat %s | clang-repl -Xcc -xc -Xcc -Xclang -Xcc -verify | FileCheck %s
 // RUN: cat %s | clang-repl -Xcc -xc -Xcc -O2 -Xcc -Xclang -Xcc -verify| FileCheck %s
diff --git a/clang/test/Interpreter/execute.cpp b/clang/test/Interpreter/execute.cpp
index 534a54ed94fba..82cd70a93980a 100644
--- a/clang/test/Interpreter/execute.cpp
+++ b/clang/test/Interpreter/execute.cpp
@@ -1,5 +1,3 @@
-// UNSUPPORTED: system-aix
-
 // clang-format off
 // RUN: clang-repl "int i = 10;" 'extern "C" int printf(const char*,...);' \
 // RUN:            'auto r1 = printf("i = %d\n", i);' | FileCheck --check-prefix=CHECK-DRIVER %s
diff --git a/clang/test/Interpreter/fail.cpp b/clang/test/Interpreter/fail.cpp
index 4963df8c54a49..d92debc253542 100644
--- a/clang/test/Interpreter/fail.cpp
+++ b/clang/test/Interpreter/fail.cpp
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit
-// UNSUPPORTED: system-aix
 // clang-repl can be called from the prompt in non-interactive mode as a
 // calculator in shell scripts, for example. In that case if there is an error
 // we should set the exit code as failure.
diff --git a/clang/test/Interpreter/global-dtor.cpp b/clang/test/Interpreter/global-dtor.cpp
index 1f241d9f19317..9cb454b06c171 100644
--- a/clang/test/Interpreter/global-dtor.cpp
+++ b/clang/test/Interpreter/global-dtor.cpp
@@ -1,5 +1,4 @@
 // clang-format off
-// UNSUPPORTED: system-aix
 //
 // Tests that a global destructor is ran on platforms with gnu exception support.
 //
@@ -10,4 +9,4 @@ extern "C" int printf(const char *, ...);
 struct D { float f = 1.0; D *m = nullptr; D(){} ~D() { printf("D[f=%f, m=0x%llx]\n", f, reinterpret_cast<unsigned long long>(m)); }} d;
 // CHECK: D[f=1.000000, m=0x0]
 
-%quit
\ No newline at end of file
+%quit
diff --git a/clang/test/Interpreter/incremental-mode.cpp b/clang/test/Interpreter/incremental-mode.cpp
index 71ff794872b21..d63cee0dd6d15 100644
--- a/clang/test/Interpreter/incremental-mode.cpp
+++ b/clang/test/Interpreter/incremental-mode.cpp
@@ -1,5 +1,3 @@
-// UNSUPPORTED: system-aix
-//
 // RUN: clang-repl -Xcc -E
 // RUN: clang-repl -Xcc -emit-llvm
 // RUN: clang-repl -Xcc -xc
diff --git a/clang/test/Interpreter/inline-asm.cpp b/clang/test/Interpreter/inline-asm.cpp
index f94f14df72f80..6d071b1ef207d 100644
--- a/clang/test/Interpreter/inline-asm.cpp
+++ b/clang/test/Interpreter/inline-asm.cpp
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit, x86_64-linux
-// UNSUPPORTED: system-aix
 //
 // RUN: rm -rf %t
 // RUN: mkdir -p %t
diff --git a/clang/test/Interpreter/inline-virtual.cpp b/clang/test/Interpreter/inline-virtual.cpp
index c9e85683d3cd0..3790c110af09d 100644
--- a/clang/test/Interpreter/inline-virtual.cpp
+++ b/clang/test/Interpreter/inline-virtual.cpp
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit
-// UNSUPPORTED: system-aix
 //
 // This test is flaky with ASan: https://github.com/llvm/llvm-project/issues/135401
 // UNSUPPORTED: asan
diff --git a/clang/test/Interpreter/lambda.cpp b/clang/test/Interpreter/lambda.cpp
index db8c9db9b3442..e6941087e9470 100644
--- a/clang/test/Interpreter/lambda.cpp
+++ b/clang/test/Interpreter/lambda.cpp
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit
-// UNSUPPORTED: system-aix
 // RUN: cat %s | clang-repl | FileCheck %s
 // At -O2, somehow "x = 42" appears first when piped into FileCheck,
 // see https://github.com/llvm/llvm-project/issues/143547.
@@ -27,4 +26,4 @@ auto capture = [&]() { return x * 2; };
 printf("x = %d\n", x);
 // CHECK: x = 42
 
-%quit
\ No newline at end of file
+%quit
diff --git a/clang/test/Interpreter/lit.local.cfg b/clang/test/Interpreter/lit.local.cfg
index ac6d2205e9fcd..37af5129d4eee 100644
--- a/clang/test/Interpreter/lit.local.cfg
+++ b/clang/test/Interpreter/lit.local.cfg
@@ -1,2 +1,6 @@
-if "host-supports-jit" not in config.available_features:
+# clang-repl is not supported on AIX and zOS
+unsupported_platforms = [ "system-aix", "system-zos" ]
+
+if "host-supports-jit" not in config.available_features or \
+  any(up in config.available_features for up in unsupported_platforms):
     config.unsupported = True
diff --git a/clang/test/Interpreter/multiline.cpp b/clang/test/Interpreter/multiline.cpp
index 0f5ef48417f13..a9f1455fc94a5 100644
--- a/clang/test/Interpreter/multiline.cpp
+++ b/clang/test/Interpreter/multiline.cpp
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit
-// UNSUPPORTED: system-aix
 // RUN: cat %s | clang-repl -Xcc -Xclang -Xcc -verify | FileCheck %s
 
 // expected-no-diagnostics
diff --git a/clang/test/Interpreter/pretty-print.c b/clang/test/Interpreter/pretty-print.c
index e1408c035a164..588df70e33e84 100644
--- a/clang/test/Interpreter/pretty-print.c
+++ b/clang/test/Interpreter/pretty-print.c
@@ -1,5 +1,4 @@
 // REQUIRES: host-supports-jit
-// UNSUPPORTED: system-aix
 // RUN: cat %s | clang-repl -Xcc -xc  | FileCheck %s
 // RUN: cat %s | clang-repl -Xcc -std=c++11 | FileCheck %s
 
diff --git a/clang/test/Interpreter/pretty-print.cpp b/clang/test/Interpreter/pretty-print.cpp
index e1036ab87df95..bad71cdd48f0b 100644
--- a/clang/test/Interpreter/pretty-print.cpp
+++ b/clang/test/Interpreter/pretty-print.cpp
@@ -1,7 +1,7 @@
 // RUN: clang-repl "int i = 10;" 'extern "C" int printf(const char*,...);' \
 // RUN:            'auto r1 = printf("i = %d\n", i);' | FileCheck --check-prefix=CHECK-DRIVER %s
 // The test is flaky with asan https://github.com/llvm/llvm-project/pull/148701.
-// UNSUPPORTED: system-aix, asan
+// UNSUPPORTED: asan
 // CHECK-DRIVER: i = 10
 // RUN: cat %s | clang-repl -Xcc -std=c++11 -Xcc -fno-delayed-template-parsing | FileCheck %s
 extern "C" int printf(const char*,...);
diff --git a/clang/test/Interpreter/simple-exception.cpp b/clang/test/Interpreter/simple-exception.cpp
index 651e8d9402f89..2d43f807d7b92 100644
--- a/clang/test/Interpreter/simple-exception.cpp
+++ b/clang/test/Interpreter/simple-exception.cpp
@@ -1,7 +1,6 @@
 // clang-format off
-// UNSUPPORTED: system-aix
 // XFAIL for arm, or running on Windows.
-// XFAIL: target=arm-{{.*}}, target=armv{{.*}}, system-windows
+// XFAIL: target=arm-{{.*}}, target=armv{{.*}}, system-windows, system-cygwin
 // RUN: cat %s | clang-repl | FileCheck %s
 
 // Incompatible with msan. It passes with -O3 but fail -Oz. Interpreter
diff --git a/clang/test/Lexer/cross-windows-on-linux.cpp b/clang/test/Lexer/cross-windows-on-linux.cpp
index 3932ffcb33c82..ece16b1ac8550 100644
--- a/clang/test/Lexer/cross-windows-on-linux.cpp
+++ b/clang/test/Lexer/cross-windows-on-linux.cpp
@@ -10,4 +10,4 @@
 // on non-Windows unless -fms-extensions is passed. It won't fail in this way on
 // Windows because the filesystem will interpret the backslash as a directory
 // separator.
-// UNSUPPORTED: system-windows
+// UNSUPPORTED: system-windows, system-cygwin
diff --git a/clang/test/Lexer/has_feature_cfi.c b/clang/test/Lexer/has_feature_cfi.c
new file mode 100644
index 0000000000000..a4e5803f68ce7
--- /dev/null
+++ b/clang/test/Lexer/has_feature_cfi.c
@@ -0,0 +1,87 @@
+// REQUIRES: target={{x86_64.*-linux.*}}
+
+// RUN: %clang -E -fvisibility=hidden -flto -fno-sanitize-ignorelist -fsanitize=cfi -c %s -o - | FileCheck %s --check-prefix=CHECK-CFI
+// RUN: %clang -E -fvisibility=hidden -flto -fno-sanitize-ignorelist -fsanitize=cfi -fsanitize-cfi-cross-dso -c %s -o - | FileCheck %s --check-prefix=CHECK-CFI
+// RUN: %clang -E -fvisibility=hidden -flto -fno-sanitize-ignorelist -fsanitize=cfi -fno-sanitize=cfi-nvcall,cfi-vcall,cfi-mfcall,cfi-icall -c %s -o - | FileCheck %s --check-prefix=CHECK-CFI
+// CHECK-CFI: CFISanitizerEnabled
+
+// RUN: %clang -E -c %s -o - | FileCheck %s --check-prefix=CHECK-NO-CFI
+// CHECK-NO-CFI: CFISanitizerDisabled
+
+// RUN: %clang -E -fsanitize=kcfi -c %s -o - | FileCheck %s --check-prefixes=CHECK-KCFI,CHECK-NO-CFI
+// CHECK-KCFI: KCFISanitizerEnabled
+
+// RUN: %clang -E -fsanitize=cfi-cast-strict -c %s -o - | FileCheck %s --check-prefix=CHECK-CFI-CAST-STRICT
+// CHECK-CFI-CAST-STRICT: CFICastStrictSanitizerEnabled
+
+// RUN: %clang -E -fvisibility=hidden -flto -fno-sanitize-ignorelist -fsanitize=cfi-derived-cast -c %s -o - | FileCheck %s --check-prefixes=CHECK-CFI,CHECK-CFI-DERIVED-CAST
+// CHECK-CFI-DERIVED-CAST: CFIDerivedCastSanitizerEnabled
+
+// RUN: %clang -E -fvisibility=hidden -flto -fno-sanitize-ignorelist -fsanitize=cfi-icall -c %s -o - | FileCheck %s --check-prefixes=CHECK-CFI,CHECK-CFI-ICALL
+// CHECK-CFI-ICALL: CFIICallSanitizerEnabled
+
+// RUN: %clang -E -fvisibility=hidden -flto -fno-sanitize-ignorelist -fsanitize=cfi-mfcall -c %s -o - | FileCheck %s --check-prefixes=CHECK-CFI,CHECK-CFI-MFCALL
+// CHECK-CFI-MFCALL: CFIMFCallSanitizerEnabled
+
+// RUN: %clang -E -fvisibility=hidden -flto -fno-sanitize-ignorelist -fsanitize=cfi-unrelated-cast -c %s -o - | FileCheck %s --check-prefixes=CHECK-CFI,CHECK-CFI-UNRELATED-CAST
+// CHECK-CFI-UNRELATED-CAST: CFIUnrelatedCastSanitizerEnabled
+
+// RUN: %clang -E -fvisibility=hidden -flto -fno-sanitize-ignorelist -fsanitize=cfi-nvcall -c %s -o - | FileCheck %s --check-prefixes=CHECK-CFI,CHECK-CFI-NVCALL
+// CHECK-CFI-NVCALL: CFINVCallSanitizerEnabled
+
+// RUN: %clang -E -fvisibility=hidden -flto -fno-sanitize-ignorelist -fsanitize=cfi-vcall -c %s -o - | FileCheck %s --check-prefixes=CHECK-CFI,CHECK-CFI-VCALL
+// CHECK-CFI-VCALL: CFIVCallSanitizerEnabled
+
+#if __has_feature(cfi_sanitizer)
+int CFISanitizerEnabled();
+#else
+int CFISanitizerDisabled();
+#endif
+
+#if __has_feature(kcfi)
+int KCFISanitizerEnabled();
+#else
+int KCFISanitizerDisabled();
+#endif
+
+#if __has_feature(cfi_cast_strict_sanitizer)
+int CFICastStrictSanitizerEnabled();
+#else
+int CFICastStrictSanitizerDisabled();
+#endif
+
+#if __has_feature(cfi_derived_cast_sanitizer)
+int CFIDerivedCastSanitizerEnabled();
+#else
+int CFIDerivedCastSanitizerDisabled();
+#endif
+
+#if __has_feature(cfi_icall_sanitizer)
+int CFIICallSanitizerEnabled();
+#else
+int CFIICallSanitizerDisabled();
+#endif
+
+#if __has_feature(cfi_mfcall_sanitizer)
+int CFIMFCallSanitizerEnabled();
+#else
+int CFIMFCallSanitizerDisabled();
+#endif
+
+#if __has_feature(cfi_unrelated_cast_sanitizer)
+int CFIUnrelatedCastSanitizerEnabled();
+#else
+int CFIUnrelatedCastSanitizerDisabled();
+#endif
+
+#if __has_feature(cfi_nvcall_sanitizer)
+int CFINVCallSanitizerEnabled();
+#else
+int CFINVCallSanitizerDisabled();
+#endif
+
+#if __has_feature(cfi_vcall_sanitizer)
+int CFIVCallSanitizerEnabled();
+#else
+int CFIVCallSanitizerDisabled();
+#endif
diff --git a/clang/test/Misc/diag-template-diffing-cxx11.cpp b/clang/test/Misc/diag-template-diffing-cxx11.cpp
index c62bffe2b458d..0b145475fe191 100644
--- a/clang/test/Misc/diag-template-diffing-cxx11.cpp
+++ b/clang/test/Misc/diag-template-diffing-cxx11.cpp
@@ -24,17 +24,17 @@ namespace std {
   }
 } // end namespace std
 // CHECK-ELIDE-NOTREE: no matching function for call to 'f'
-// CHECK-ELIDE-NOTREE: candidate function not viable: no known conversion from 'vector<std::string>' to 'vector<string>' for 1st argument
+// CHECK-ELIDE-NOTREE: candidate function not viable: no known conversion from 'vector<std::basic_string>' to 'vector<versa_string>' for 1st argument
 // CHECK-NOELIDE-NOTREE: no matching function for call to 'f'
-// CHECK-NOELIDE-NOTREE: candidate function not viable: no known conversion from 'vector<std::string>' to 'vector<string>' for 1st argument
+// CHECK-NOELIDE-NOTREE: candidate function not viable: no known conversion from 'vector<std::basic_string>' to 'vector<versa_string>' for 1st argument
 // CHECK-ELIDE-TREE: no matching function for call to 'f'
 // CHECK-ELIDE-TREE: candidate function not viable: no known conversion from argument type to parameter type for 1st argument
 // CHECK-ELIDE-TREE:   vector<
-// CHECK-ELIDE-TREE:     [std::string != string]>
+// CHECK-ELIDE-TREE:     [std::basic_string != versa_string]>
 // CHECK-NOELIDE-TREE: no matching function for call to 'f'
 // CHECK-NOELIDE-TREE: candidate function not viable: no known conversion from argument type to parameter type for 1st argument
 // CHECK-NOELIDE-TREE:   vector<
-// CHECK-NOELIDE-TREE:     [std::string != string]>
+// CHECK-NOELIDE-TREE:     [std::basic_string != versa_string]>
 
 template <int... A>
 class I1{};
diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
index 05693538252aa..37ff33e5a1523 100644
--- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test
+++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test
@@ -31,6 +31,7 @@
 // CHECK-NEXT: CFConsumed (SubjectMatchRule_variable_is_parameter)
 // CHECK-NEXT: CFGuard (SubjectMatchRule_function)
 // CHECK-NEXT: CFICanonicalJumpTable (SubjectMatchRule_function)
+// CHECK-NEXT: CFISalt (SubjectMatchRule_hasType_functionType)
 // CHECK-NEXT: CFUnknownTransfer (SubjectMatchRule_function)
 // CHECK-NEXT: CPUDispatch (SubjectMatchRule_function)
 // CHECK-NEXT: CPUSpecific (SubjectMatchRule_function)
@@ -181,6 +182,7 @@
 // CHECK-NEXT: ReturnTypestate (SubjectMatchRule_function, SubjectMatchRule_variable_is_parameter)
 // CHECK-NEXT: ReturnsNonNull (SubjectMatchRule_objc_method, SubjectMatchRule_function)
 // CHECK-NEXT: ReturnsTwice (SubjectMatchRule_function)
+// CHECK-NEXT: SYCLExternal (SubjectMatchRule_function)
 // CHECK-NEXT: SYCLKernelEntryPoint (SubjectMatchRule_function)
 // CHECK-NEXT: SYCLSpecialClass (SubjectMatchRule_record)
 // CHECK-NEXT: ScopedLockable (SubjectMatchRule_record)
diff --git a/clang/test/Modules/GH153933.cpp b/clang/test/Modules/GH153933.cpp
new file mode 100644
index 0000000000000..41184c6b00607
--- /dev/null
+++ b/clang/test/Modules/GH153933.cpp
@@ -0,0 +1,23 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/B.cppm -emit-module-interface -o %t/B.pcm
+// RUN: %clang_cc1 -std=c++20 -fsyntax-only -fprebuilt-module-path=%t %t/C.cpp
+
+//--- A.hpp
+template<class> struct A {};
+template<class T> struct B {
+  virtual A<T> v() { return {}; }
+};
+B<void> x;
+
+//--- B.cppm
+module;
+#include "A.hpp"
+export module B;
+using ::x;
+
+//--- C.cpp
+#include "A.hpp"
+import B;
diff --git a/clang/test/Modules/befriend-2.cppm b/clang/test/Modules/befriend-2.cppm
new file mode 100644
index 0000000000000..9d0baf849cad4
--- /dev/null
+++ b/clang/test/Modules/befriend-2.cppm
@@ -0,0 +1,65 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/test-A.pcm
+// RUN: %clang_cc1 -std=c++20 %t/N.cppm -emit-reduced-module-interface -o %t/test-N.pcm
+// RUN: %clang_cc1 -std=c++20 %t/B.cppm -verify -fsyntax-only -fprebuilt-module-path=%t
+
+//--- a.h
+namespace N {
+
+    template <typename>
+    class C {
+    template <typename> friend void foo();
+    };
+
+    template <typename> void foo() {}
+} // namespace N
+
+//--- a.cppm
+// This is some unrelated file. It also #includes system headers, but
+// here does not even export anything.
+module;
+#include "a.h"
+export module test:A;
+export {
+    using N::C;
+    using N::foo;
+}
+
+//--- std.h
+// Declarations typically #included from C++ header files:
+namespace N {               // In practice, this would be namespace std
+    inline namespace impl {   // In practice, this would be namespace __1
+        template <typename>
+        class C {
+        template <typename> friend void foo();
+        };
+    
+        template <typename> void foo() {}
+    } // namespace impl
+    } // namespace N
+
+//--- N.cppm
+module;
+#include "std.h"
+export module test:N;
+
+// Now wrap these names into a module and export them:
+export {
+    namespace N   {
+        using N::C;
+        using N::foo;
+    }
+}
+
+//--- B.cppm
+// expected-no-diagnostics
+// A file that consumes the partitions from the other two files,
+// including the exported N::C name.
+module test:B;
+import :N;
+import :A;
+
+N::C<int> x;
diff --git a/clang/test/Modules/befriend-3.cppm b/clang/test/Modules/befriend-3.cppm
new file mode 100644
index 0000000000000..f8dbc423be2ca
--- /dev/null
+++ b/clang/test/Modules/befriend-3.cppm
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -std=c++20 %s -fsyntax-only -verify
+export module m;
+
+namespace test {
+namespace ns1 {
+    namespace ns2 {
+    template<class T> void f(T t); // expected-note {{target of using declaration}}
+    }
+    using ns2::f; // expected-note {{using declaration}}
+}
+struct A { void f(); }; // expected-note 2{{target of using declaration}}
+struct B : public A { using A::f; }; // expected-note {{using declaration}}
+template<typename T> struct C : A { using A::f; }; // expected-note {{using declaration}}
+struct X {
+    template<class T> friend void ns1::f(T t); // expected-error {{cannot befriend target of using declaration}}
+    friend void B::f(); // expected-error {{cannot befriend target of using declaration}}
+    friend void C<int>::f(); // expected-error {{cannot befriend target of using declaration}}
+};
+}
diff --git a/clang/test/Modules/modules-merge-enum.m b/clang/test/Modules/modules-merge-enum.m
index f1010c1decc24..fc07c46a5117e 100644
--- a/clang/test/Modules/modules-merge-enum.m
+++ b/clang/test/Modules/modules-merge-enum.m
@@ -18,9 +18,8 @@
 // CHECK-NEXT: | |-also in ModB
 // CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} imported in ModA.ModAFile1 referenced MyVal_A 'int'
 // CHECK-NEXT: |-TypedefDecl 0x{{.*}} imported in ModA.ModAFile1 hidden MyEnum1 'enum MyEnum1'
-// CHECK-NEXT: | `-ElaboratedType 0x{{.*}} 'enum MyEnum1' sugar imported
-// CHECK-NEXT: |   `-EnumType 0x{{.*}} 'enum MyEnum1' imported
-// CHECK-NEXT: |     `-Enum 0x{{.*}} 'MyEnum1'
+// CHECK-NEXT: | `-EnumType 0x{{.*}} 'enum MyEnum1' imported
+// CHECK-NEXT: |   `-Enum 0x{{.*}} 'MyEnum1'
 
 
 enum MyEnum2 { MyVal_B };
@@ -33,10 +32,9 @@
 // CHECK:      |-EnumDecl 0x{{.*}} imported in ModA.ModAFile1 <undeserialized declarations>
 // CHECK-NEXT: | |-also in ModB
 // CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} imported in ModA.ModAFile1 referenced MyVal_C 'int'
-// CHECK-NEXT: |-TypedefDecl 0x{{.*}} imported in ModA.ModAFile1 hidden MyEnum3 'enum MyEnum3':'MyEnum3'
-// CHECK-NEXT: | `-ElaboratedType 0x{{.*}} 'enum MyEnum3' sugar imported
-// CHECK-NEXT: |   `-EnumType 0x{{.*}} 'MyEnum3' imported
-// CHECK-NEXT: |     `-Enum 0x{{.*}}
+// CHECK-NEXT: |-TypedefDecl 0x{{.*}} imported in ModA.ModAFile1 hidden MyEnum3 'enum MyEnum3'
+// CHECK-NEXT: | `-EnumType 0x{{.*}} 'enum MyEnum3' imported
+// CHECK-NEXT: |   `-Enum 0x{{.*}}
 
 struct MyStruct {
   enum MyEnum5 { MyVal_D } Field;
@@ -57,17 +55,15 @@
 
 // Redeclarations coming from ModB.
 // CHECK:      |-TypedefDecl 0x{{.*}} prev 0x{{.*}} imported in ModB MyEnum1 'enum MyEnum1'
-// CHECK-NEXT: | `-ElaboratedType 0x{{.*}} 'enum MyEnum1' sugar imported
-// CHECK-NEXT: |   `-EnumType 0x{{.*}} 'enum MyEnum1' imported
-// CHECK-NEXT: |     `-Enum 0x{{.*}} 'MyEnum1'
+// CHECK-NEXT: | `-EnumType 0x{{.*}} 'enum MyEnum1' imported
+// CHECK-NEXT: |   `-Enum 0x{{.*}} 'MyEnum1'
 
 // CHECK:      |-EnumDecl 0x{{.*}} prev 0x{{.*}} imported in ModB <undeserialized declarations>
 // CHECK-NEXT: | |-also in ModB
 // CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} imported in ModB MyVal_C 'int'
-// CHECK-NEXT: |-TypedefDecl 0x{{.*}} prev 0x{{.*}} imported in ModB MyEnum3 'enum MyEnum3':'MyEnum3'
-// CHECK-NEXT: | `-ElaboratedType 0x{{.*}} 'enum MyEnum3' sugar imported
-// CHECK-NEXT: |   `-EnumType 0x{{.*}} 'MyEnum3' imported
-// CHECK-NEXT: |     `-Enum 0x{{.*}}
+// CHECK-NEXT: |-TypedefDecl 0x{{.*}} prev 0x{{.*}} imported in ModB MyEnum3 'enum MyEnum3'
+// CHECK-NEXT: | `-EnumType 0x{{.*}} 'enum MyEnum3' imported
+// CHECK-NEXT: |   `-Enum 0x{{.*}}
 
 // CHECK:      |-EnumDecl 0x{{.*}} imported in ModB <undeserialized declarations>
 // CHECK-NEXT: | `-EnumConstantDecl 0x{{.*}} first 0x{{.*}} imported in ModB referenced MyVal_E 'int'
diff --git a/clang/test/Modules/odr_hash.cpp b/clang/test/Modules/odr_hash.cpp
index 8ef53e32f2e95..f22f3c71f44d2 100644
--- a/clang/test/Modules/odr_hash.cpp
+++ b/clang/test/Modules/odr_hash.cpp
@@ -1314,7 +1314,7 @@ class S1 {
 #else
 template<class T>
 using U1 = S1<T>;
-// expected-error@first.h:* {{'DependentType::S1::x' from module 'FirstModule' is not present in definition of 'S1<T>' in module 'SecondModule'}}
+// expected-error@first.h:* {{'DependentType::S1::x' from module 'FirstModule' is not present in definition of 'DependentType::S1<T>' in module 'SecondModule'}}
 // expected-note@second.h:* {{declaration of 'x' does not match}}
 #endif
 
@@ -2343,7 +2343,7 @@ struct S1 {
 };
 #else
 using TemplateTypeParmType::S1;
-// expected-error@first.h:* {{'TemplateTypeParmType::S1::x' from module 'FirstModule' is not present in definition of 'S1<T1, T2>' in module 'SecondModule'}}
+// expected-error@first.h:* {{'TemplateTypeParmType::S1::x' from module 'FirstModule' is not present in definition of 'TemplateTypeParmType::S1<T1, T2>' in module 'SecondModule'}}
 // expected-note@second.h:* {{declaration of 'x' does not match}}
 #endif
 
@@ -2365,9 +2365,9 @@ class S2 {
 };
 #else
 using TemplateTypeParmType::S2;
-// expected-error@first.h:* {{'TemplateTypeParmType::S2::x' from module 'FirstModule' is not present in definition of 'S2<T, U>' in module 'SecondModule'}}
+// expected-error@first.h:* {{'TemplateTypeParmType::S2::x' from module 'FirstModule' is not present in definition of 'TemplateTypeParmType::S2<T, U>' in module 'SecondModule'}}
 // expected-note@second.h:* {{declaration of 'x' does not match}}
-// expected-error@first.h:* {{'TemplateTypeParmType::S2::type' from module 'FirstModule' is not present in definition of 'S2<T, U>' in module 'SecondModule'}}
+// expected-error@first.h:* {{'TemplateTypeParmType::S2::type' from module 'FirstModule' is not present in definition of 'TemplateTypeParmType::S2<T, U>' in module 'SecondModule'}}
 // expected-note@second.h:* {{declaration of 'type' does not match}}
 #endif
 
@@ -4020,7 +4020,7 @@ struct Valid {
 };
 #else
 Invalid::L2<1>::L3<1> invalid;
-// expected-error@second.h:* {{'Types::InjectedClassName::Invalid::L2::L3::x' from module 'SecondModule' is not present in definition of 'L3<value-parameter-1-0>' in module 'FirstModule'}}
+// expected-error@second.h:* {{'Types::InjectedClassName::Invalid::L2::L3::x' from module 'SecondModule' is not present in definition of 'Types::InjectedClassName::Invalid::L2::L3<value-parameter-1-0>' in module 'FirstModule'}}
 // expected-note@first.h:* {{declaration of 'x' does not match}}
 Valid::L2<1>::L3<1> valid;
 #endif
@@ -4291,7 +4291,7 @@ struct Valid {
 };
 #else
 template <class T> using I = Invalid<T>;
-// expected-error@first.h:* {{'Types::UnresolvedUsing::Invalid::x' from module 'FirstModule' is not present in definition of 'Invalid<T>' in module 'SecondModule'}}
+// expected-error@first.h:* {{'Types::UnresolvedUsing::Invalid::x' from module 'FirstModule' is not present in definition of 'Types::UnresolvedUsing::Invalid<T>' in module 'SecondModule'}}
 // expected-note@second.h:* {{declaration of 'x' does not match}}
 
 template <class T> using V = Valid<T>;
diff --git a/clang/test/Modules/pr138558.cppm b/clang/test/Modules/pr138558.cppm
new file mode 100644
index 0000000000000..c637ce2f266eb
--- /dev/null
+++ b/clang/test/Modules/pr138558.cppm
@@ -0,0 +1,54 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/test-A.pcm
+// RUN: %clang_cc1 -std=c++20 %t/N.cppm -emit-reduced-module-interface -o %t/test-N.pcm
+// RUN: %clang_cc1 -std=c++20 %t/B.cppm -verify -fsyntax-only -fprebuilt-module-path=%t
+
+//--- a.h
+namespace N {
+inline namespace impl   {
+    template <typename>
+    class C {
+    template <typename> friend void foo();
+    };
+
+    template <typename> void foo() {}
+} // namespace impl
+} // namespace N
+
+//--- a.cppm
+// This is some unrelated file. It also #includes system headers, but
+// here does not even export anything.
+module;
+#include "a.h"
+export module test:A;
+// To make sure they won't elided.
+using N::C;
+using N::foo;
+
+//--- N.cppm
+module;
+#include "a.h"
+export module test:N;
+
+// Now wrap these names into a module and export them:
+export {
+  namespace N   {
+    inline namespace impl    {
+      using N::impl::C;
+      using N::impl::foo;
+    }
+  }
+}
+
+//--- B.cppm
+// expected-no-diagnostics
+// A file that consumes the partitions from the other two files,
+// including the exported N::C name.
+module test:B;
+import :N;
+import :A;
+
+N::C<int> x;
diff --git a/clang/test/Modules/pr97313.cppm b/clang/test/Modules/pr97313.cppm
index 32c7112da091e..99795d6e43030 100644
--- a/clang/test/Modules/pr97313.cppm
+++ b/clang/test/Modules/pr97313.cppm
@@ -1,4 +1,4 @@
-// REQUIRES: !system-windows
+// REQUIRES: !system-windows, !system-cygwin
 //
 // RUN: rm -rf %t
 // RUN: mkdir -p %t
diff --git a/clang/test/Modules/redundant-template-default-arg2.cpp b/clang/test/Modules/redundant-template-default-arg2.cpp
index ae1f0c7e69cc0..6e22d823a18dc 100644
--- a/clang/test/Modules/redundant-template-default-arg2.cpp
+++ b/clang/test/Modules/redundant-template-default-arg2.cpp
@@ -33,8 +33,8 @@ int v2; // expected-error {{declaration of 'v2' in the global module follows dec
         // expected-note@foo.cppm:6 {{previous declaration is here}}
 
 template <typename T>
-class my_array {}; // expected-error {{redefinition of 'my_array'}}
-                   // expected-note@foo.cppm:9 {{previous definition is here}}
+class my_array {}; // expected-error {{declaration of 'my_array' in the global module follows declaration in module foo}}
+                   // expected-note@foo.cppm:9 {{previous declaration is here}}
 
 template <template <typename> typename C = my_array>
 int v3; // expected-error {{declaration of 'v3' in the global module follows declaration in module foo}}
diff --git a/clang/test/Modules/skip-body-2.cppm b/clang/test/Modules/skip-body-2.cppm
new file mode 100644
index 0000000000000..781f4fefb72b1
--- /dev/null
+++ b/clang/test/Modules/skip-body-2.cppm
@@ -0,0 +1,58 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.cpp -fmodule-file=a=%t/a.pcm -ast-dump | FileCheck %s
+
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.cpp -fmodule-file=a=%t/a.pcm -ast-dump | FileCheck %s
+
+//--- a.h
+namespace a {
+class A {
+public:
+    int aaaa;
+
+    int get() {
+        return aaaa;
+    }
+};
+
+
+template <class T>
+class B {
+public:
+    B(T t): t(t) {}
+    T t;
+};
+
+using BI = B<int>;
+
+inline int get(A a, BI b) {
+    return a.get() + b.t;
+}
+
+}
+
+//--- a.cppm
+export module a;
+
+export extern "C++" {
+#include "a.h"
+}
+
+//--- a.cpp
+import a;
+#include "a.h"
+
+int test() {
+    a::A aa;
+    a::BI bb(43);
+    return get(aa, bb);
+}
+
+// CHECK-NOT: DefinitionData
+// CHECK: FunctionDecl {{.*}} get 'int (A, BI)' {{.*}}
+// CHECK-NOT: CompoundStmt
+// CHECK: FunctionDecl {{.*}} test {{.*}}
diff --git a/clang/test/Modules/skip-body.cppm b/clang/test/Modules/skip-body.cppm
new file mode 100644
index 0000000000000..cbb466fce07f4
--- /dev/null
+++ b/clang/test/Modules/skip-body.cppm
@@ -0,0 +1,63 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.cpp -fmodule-file=a=%t/a.pcm -ast-dump | FileCheck %s
+
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.cpp -fmodule-file=a=%t/a.pcm -ast-dump | FileCheck %s
+
+//--- a.h
+namespace a {
+class A {
+public:
+    int aaaa;
+
+    int get() {
+        return aaaa;
+    }
+};
+
+
+template <class T>
+class B {
+public:
+    B(T t): t(t) {}
+    T t;
+};
+
+using BI = B<int>;
+
+inline int get(A a, BI b) {
+    return a.get() + b.t;
+}
+
+}
+
+//--- a.cppm
+module;
+#include "a.h"
+
+export module a;
+
+namespace a {
+    export using ::a::A;
+    export using ::a::get;
+    export using ::a::BI;
+}
+
+//--- a.cpp
+import a;
+#include "a.h"
+
+int test() {
+    a::A aa;
+    a::BI bb(43);
+    return get(aa, bb);
+}
+
+// CHECK-NOT: DefinitionData
+// CHECK: FunctionDecl {{.*}} get 'int (A, BI)' {{.*}}
+// CHECK-NOT: CompoundStmt
+// CHECK: FunctionDecl {{.*}} test {{.*}}
diff --git a/clang/test/OpenMP/allocate_modifiers_messages.cpp b/clang/test/OpenMP/allocate_modifiers_messages.cpp
index 6867e78a89ee9..83314a5b61e6b 100644
--- a/clang/test/OpenMP/allocate_modifiers_messages.cpp
+++ b/clang/test/OpenMP/allocate_modifiers_messages.cpp
@@ -88,7 +88,7 @@ int main() {
   // expected-warning@+2 {{extra tokens at the end of '#pragma omp scope' are ignored}}
   // expected-note@+1 {{to match this '('}}
   #pragma omp scope private(a,b,c) allocate(allocator(omp_const_mem_alloc):c:b;a)
-  // expected-error@+1 {{initializing 'const omp_allocator_handle_t' with an expression of incompatible type 'int'}}
+  // expected-error@+1 {{initializing 'const omp_allocator_handle_t' (aka 'const enum omp_allocator_handle_t') with an expression of incompatible type 'int'}}
   #pragma omp scope private(c,a,b) allocate(allocator(myAlloc()):a,b,c)
   // expected-error@+2 {{missing ':' after allocate clause modifier}}
   // expected-error@+1 {{expected expression}}
diff --git a/clang/test/OpenMP/amdgcn_debug_nowait.c b/clang/test/OpenMP/amdgcn_debug_nowait.c
new file mode 100644
index 0000000000000..d691327512ff7
--- /dev/null
+++ b/clang/test/OpenMP/amdgcn_debug_nowait.c
@@ -0,0 +1,16 @@
+// REQUIRES: amdgpu-registered-target
+
+// RUN: %clang_cc1 -debug-info-kind=line-tables-only -fopenmp -triple x86_64-unknown-unknown -fopenmp-targets=amdgcn-amd-amdhsa -emit-llvm-bc %s -o %t-host.bc
+
+int test() {
+  int c;
+
+#pragma omp target data map(tofrom: c)
+{
+  #pragma omp target nowait
+  {
+      c = 2;
+  }
+}
+  return c;
+}
diff --git a/clang/test/OpenMP/bug54082.c b/clang/test/OpenMP/bug54082.c
index da32be2867411..bda4bd29b9e66 100644
--- a/clang/test/OpenMP/bug54082.c
+++ b/clang/test/OpenMP/bug54082.c
@@ -68,14 +68,14 @@ void foo() {
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    [[X_TRAITS:%.*]] = alloca [1 x %struct.omp_alloctrait_t], align 16
 // CHECK-NEXT:    [[X_ALLOC:%.*]] = alloca i64, align 8
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[X_TRAITS]]) #[[ATTR5:[0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X_TRAITS]]) #[[ATTR5:[0-9]+]]
 // CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 16 dereferenceable(16) [[X_TRAITS]], ptr noundef nonnull align 16 dereferenceable(16) @__const.foo.x_traits, i64 16, i1 false)
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[X_ALLOC]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X_ALLOC]]) #[[ATTR5]]
 // CHECK-NEXT:    [[CALL:%.*]] = call i64 @omp_init_allocator(i64 noundef 0, i32 noundef 1, ptr noundef nonnull [[X_TRAITS]]) #[[ATTR5]]
 // CHECK-NEXT:    store i64 [[CALL]], ptr [[X_ALLOC]], align 8, !tbaa [[TBAA3:![0-9]+]]
 // CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr nonnull @[[GLOB2:[0-9]+]], i32 1, ptr nonnull @foo.omp_outlined, ptr nonnull [[X_ALLOC]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[X_ALLOC]]) #[[ATTR5]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[X_TRAITS]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X_ALLOC]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X_TRAITS]]) #[[ATTR5]]
 // CHECK-NEXT:    ret void
 //
 //
@@ -86,13 +86,13 @@ void foo() {
 // CHECK-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[DOTOMP_LB]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[DOTOMP_LB]]) #[[ATTR5]]
 // CHECK-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA7:![0-9]+]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[DOTOMP_UB]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[DOTOMP_UB]]) #[[ATTR5]]
 // CHECK-NEXT:    store i32 1023, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA7]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[DOTOMP_STRIDE]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[DOTOMP_STRIDE]]) #[[ATTR5]]
 // CHECK-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA7]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[DOTOMP_IS_LAST]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[DOTOMP_IS_LAST]]) #[[ATTR5]]
 // CHECK-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA7]]
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4, !tbaa [[TBAA7]]
 // CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[X_ALLOC]], align 8, !tbaa [[TBAA3]]
@@ -106,9 +106,9 @@ void foo() {
 // CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[X_ALLOC]], align 8, !tbaa [[TBAA3]]
 // CHECK-NEXT:    [[CONV5:%.*]] = inttoptr i64 [[TMP3]] to ptr
 // CHECK-NEXT:    call void @__kmpc_free(i32 [[TMP0]], ptr [[DOTX__VOID_ADDR]], ptr [[CONV5]])
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[DOTOMP_IS_LAST]]) #[[ATTR5]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[DOTOMP_STRIDE]]) #[[ATTR5]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[DOTOMP_UB]]) #[[ATTR5]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[DOTOMP_LB]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[DOTOMP_IS_LAST]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[DOTOMP_STRIDE]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[DOTOMP_UB]]) #[[ATTR5]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[DOTOMP_LB]]) #[[ATTR5]]
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/OpenMP/bug56913.c b/clang/test/OpenMP/bug56913.c
index cc7231626878c..fad9e17ac4dd8 100644
--- a/clang/test/OpenMP/bug56913.c
+++ b/clang/test/OpenMP/bug56913.c
@@ -20,12 +20,12 @@ void loop(int n) {
 // CHECK-NEXT:    br i1 [[CMP]], label [[SIMD_IF_THEN:%.*]], label [[SIMD_IF_END:%.*]]
 // CHECK:       simd.if.then:
 // CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr @j, align 4, !tbaa [[TBAA2:![0-9]+]]
-// CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[J]]) #[[ATTR2:[0-9]+]]
+// CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[J]]) #[[ATTR2:[0-9]+]]
 // CHECK-NEXT:    store ptr [[J]], ptr @u, align 8, !tbaa [[TBAA6:![0-9]+]], !llvm.access.group [[ACC_GRP8:![0-9]+]]
 // CHECK-NEXT:    [[INC_LE:%.*]] = add i32 [[TMP0]], [[N]]
 // CHECK-NEXT:    store i32 [[INC_LE]], ptr [[J]], align 4, !tbaa [[TBAA2]]
 // CHECK-NEXT:    store i32 [[INC_LE]], ptr @j, align 4, !tbaa [[TBAA2]]
-// CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[J]]) #[[ATTR2]]
+// CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[J]]) #[[ATTR2]]
 // CHECK-NEXT:    br label [[SIMD_IF_END]]
 // CHECK:       simd.if.end:
 // CHECK-NEXT:    ret void
diff --git a/clang/test/OpenMP/bug57757.cpp b/clang/test/OpenMP/bug57757.cpp
index eabf233dde247..caf53a5b62c1c 100644
--- a/clang/test/OpenMP/bug57757.cpp
+++ b/clang/test/OpenMP/bug57757.cpp
@@ -46,7 +46,7 @@ void foo() {
 // CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 52
 // CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 48
 // CHECK-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[TMP5]], align 8, !tbaa [[TBAA19:![0-9]+]], !noalias [[META13]]
-// CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 4, !tbaa [[TBAA16]], !noalias [[META13]]
+// CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP7]], align 8, !tbaa [[TBAA16]], !noalias [[META13]]
 // CHECK-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP6]], align 4, !tbaa [[TBAA20:![0-9]+]], !noalias [[META13]]
 // CHECK-NEXT:    tail call void [[TMP8]](i32 noundef [[TMP9]], float noundef [[TMP10]]) #[[ATTR2:[0-9]+]], !noalias [[META13]]
 // CHECK-NEXT:    br label [[DOTOMP_OUTLINED__EXIT]]
diff --git a/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp b/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
index 232cfdad043f0..20e344f0a34a0 100644
--- a/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
+++ b/clang/test/OpenMP/nvptx_target_parallel_reduction_codegen_tbaa_PR46146.cpp
@@ -71,16 +71,16 @@ void test() {
 // CHECK1-NEXT:    [[ISTART:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[IEND:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[PARTIAL_SUM:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 8)
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_IV]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_LB]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_UB]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[IB]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_distribute_static_init_4(ptr @[[GLOB2:[0-9]+]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -110,13 +110,13 @@ void test() {
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA19:![0-9]+]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP2]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP2]]) #[[ATTR4]]
 // CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP2]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP2]]) #[[ATTR11:[0-9]+]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP2]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP2]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL3:%.*]] = mul nsw i32 [[TMP8]], 4
 // CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA15]]
@@ -143,12 +143,12 @@ void test() {
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
 // CHECK1:       omp.loop.exit:
 // CHECK1-NEXT:    call void @__kmpc_distribute_static_fini(ptr @[[GLOB2]], i32 [[TMP1]])
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[IB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_UB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_LB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @__kmpc_free_shared(ptr [[PARTIAL_SUM]], i64 8)
 // CHECK1-NEXT:    call void @__kmpc_free_shared(ptr [[IEND]], i64 4)
 // CHECK1-NEXT:    call void @__kmpc_free_shared(ptr [[ISTART]], i64 4)
@@ -205,14 +205,14 @@ void test() {
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
 // CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
 // CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA23]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_IV]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
@@ -221,33 +221,33 @@ void test() {
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], 1
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       omp.precond.then:
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_LB]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_UB]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP]], align 4, !tbaa [[TBAA19]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP6]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP6]]) #[[ATTR4]]
 // CHECK1-NEXT:    store float 0.000000e+00, ptr [[REF_TMP6]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP6]]) #[[ATTR11]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP6]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP6]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I7]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB3:[0-9]+]], i32 [[TMP12]], i32 33, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -291,20 +291,20 @@ void test() {
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP23]], 1
 // CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP22]], [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP14]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP15]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP14]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP15]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP24]] to float
 // CHECK1-NEXT:    store float [[CONV]], ptr [[REF_TMP15]], align 4, !tbaa [[TBAA19]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[REF_TMP16]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP16]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV17:%.*]] = sitofp i32 [[TMP25]] to float
 // CHECK1-NEXT:    store float [[CONV17]], ptr [[REF_TMP16]], align 4, !tbaa [[TBAA19]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIfEC1ERKfS2_(ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP15]], ptr nonnull align 4 dereferenceable(4) [[REF_TMP16]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[CALL:%.*]] = call nonnull align 4 dereferenceable(8) ptr @_ZNSt7complexIfEpLIfEERS0_RKS_IT_E(ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]], ptr nonnull align 4 dereferenceable(8) [[REF_TMP14]]) #[[ATTR11]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP16]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[REF_TMP15]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP14]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP16]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP15]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP14]]) #[[ATTR4]]
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
@@ -338,18 +338,18 @@ void test() {
 // CHECK1-NEXT:    [[CALL21:%.*]] = call nonnull align 4 dereferenceable(8) ptr @_ZNSt7complexIfEpLIfEERS0_RKS_IT_E(ptr nonnull align 4 dereferenceable(8) [[TMP2]], ptr nonnull align 4 dereferenceable(8) [[PARTIAL_SUM5]]) #[[ATTR11]]
 // CHECK1-NEXT:    br label [[DOTOMP_REDUCTION_DONE]]
 // CHECK1:       .omp.reduction.done:
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I7]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_UB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_LB]]) #[[ATTR4]]
 // CHECK1-NEXT:    br label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.end:
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    ret void
 //
 //
@@ -563,16 +563,16 @@ void test() {
 // CHECK1-NEXT:    [[ISTART:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[IEND:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 4)
 // CHECK1-NEXT:    [[PARTIAL_SUM:%.*]] = call align 16 ptr @__kmpc_alloc_shared(i64 16)
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_IV]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_LB]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_UB]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 99, ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[IB]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_distribute_static_init_4(ptr @[[GLOB2]], i32 [[TMP1]], i32 92, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -602,13 +602,13 @@ void test() {
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 1
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[IB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA36:![0-9]+]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP2]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP2]]) #[[ATTR4]]
 // CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP2]], align 8, !tbaa [[TBAA36]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP2]]) #[[ATTR11]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP2]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP2]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[IB]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[MUL3:%.*]] = mul nsw i32 [[TMP8]], 4
 // CHECK1-NEXT:    store i32 [[MUL3]], ptr [[ISTART]], align 4, !tbaa [[TBAA15]]
@@ -635,12 +635,12 @@ void test() {
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
 // CHECK1:       omp.loop.exit:
 // CHECK1-NEXT:    call void @__kmpc_distribute_static_fini(ptr @[[GLOB2]], i32 [[TMP1]])
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[IB]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[IB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_UB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_LB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    call void @__kmpc_free_shared(ptr [[PARTIAL_SUM]], i64 16)
 // CHECK1-NEXT:    call void @__kmpc_free_shared(ptr [[IEND]], i64 4)
 // CHECK1-NEXT:    call void @__kmpc_free_shared(ptr [[ISTART]], i64 4)
@@ -697,14 +697,14 @@ void test() {
 // CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[ISTART_ADDR]], align 8, !tbaa [[TBAA17]]
 // CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[IEND_ADDR]], align 8, !tbaa [[TBAA17]]
 // CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PARTIAL_SUM_ADDR]], align 8, !tbaa [[TBAA38]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_IV]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP0]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 [[TMP4]], ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
@@ -713,33 +713,33 @@ void test() {
 // CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], 1
 // CHECK1-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
 // CHECK1-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 [[TMP7]], ptr [[I]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP8]], [[TMP9]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       omp.precond.then:
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_LB]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_UB]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTOMP_UB]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP]]) #[[ATTR4]]
 // CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP]], align 8, !tbaa [[TBAA36]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP6]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP6]]) #[[ATTR4]]
 // CHECK1-NEXT:    store double 0.000000e+00, ptr [[REF_TMP6]], align 8, !tbaa [[TBAA36]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP6]]) #[[ATTR11]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP6]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP6]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I7]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB3]], i32 [[TMP12]], i32 33, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
@@ -783,20 +783,20 @@ void test() {
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP23]], 1
 // CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP22]], [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I7]], align 4, !tbaa [[TBAA15]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[REF_TMP14]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP15]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP14]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP15]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP24]] to double
 // CHECK1-NEXT:    store double [[CONV]], ptr [[REF_TMP15]], align 8, !tbaa [[TBAA36]]
-// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[REF_TMP16]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr [[REF_TMP16]]) #[[ATTR4]]
 // CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I7]], align 4, !tbaa [[TBAA15]]
 // CHECK1-NEXT:    [[CONV17:%.*]] = sitofp i32 [[TMP25]] to double
 // CHECK1-NEXT:    store double [[CONV17]], ptr [[REF_TMP16]], align 8, !tbaa [[TBAA36]]
 // CHECK1-NEXT:    call void @_ZNSt7complexIdEC1ERKdS2_(ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP15]], ptr nonnull align 8 dereferenceable(8) [[REF_TMP16]]) #[[ATTR11]]
 // CHECK1-NEXT:    [[CALL:%.*]] = call nonnull align 8 dereferenceable(16) ptr @_ZNSt7complexIdEpLIdEERS0_RKS_IT_E(ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]], ptr nonnull align 8 dereferenceable(16) [[REF_TMP14]]) #[[ATTR11]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP16]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[REF_TMP15]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[REF_TMP14]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP16]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP15]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[REF_TMP14]]) #[[ATTR4]]
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
@@ -830,18 +830,18 @@ void test() {
 // CHECK1-NEXT:    [[CALL21:%.*]] = call nonnull align 8 dereferenceable(16) ptr @_ZNSt7complexIdEpLIdEERS0_RKS_IT_E(ptr nonnull align 8 dereferenceable(16) [[TMP2]], ptr nonnull align 8 dereferenceable(16) [[PARTIAL_SUM5]]) #[[ATTR11]]
 // CHECK1-NEXT:    br label [[DOTOMP_REDUCTION_DONE]]
 // CHECK1:       .omp.reduction.done:
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I7]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_UB]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_LB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I7]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PARTIAL_SUM5]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_IS_LAST]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_STRIDE]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_UB]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_LB]]) #[[ATTR4]]
 // CHECK1-NEXT:    br label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.end:
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
-// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DOTOMP_IV]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCAPTURE_EXPR_2]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCAPTURE_EXPR_1]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCAPTURE_EXPR_]]) #[[ATTR4]]
+// CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTOMP_IV]]) #[[ATTR4]]
 // CHECK1-NEXT:    ret void
 //
 //
diff --git a/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_ast_dump.cpp b/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_ast_dump.cpp
index a5847709d3e76..15b976f0d7d1d 100644
--- a/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_ast_dump.cpp
+++ b/clang/test/OpenMP/target_map_array_of_structs_with_nested_mapper_ast_dump.cpp
@@ -27,7 +27,7 @@ void foo() {
 // DUM-NEXT:  |-OMPMapClause {{.*}}<<invalid sloc>> <implicit>
 // DUM-NEXT:  | |-MemberExpr {{.*}}<line:9:3> 'int' lvalue .e
 // DUM-NEXT:  | | `-DeclRefExpr {{.*}}<<invalid sloc>> 'D' lvalue Var {{.*}} '_s' 'D'
-// DUM-NEXT:  | |-MemberExpr {{.*}}<line:10:3> 'C' lvalue .f {{.*}}
+// DUM-NEXT:  | |-MemberExpr {{.*}}<line:10:3> 'C':'struct C' lvalue .f {{.*}}
 // DUM-NEXT:  | | `-DeclRefExpr {{.*}}<<invalid sloc>> 'D' lvalue Var {{.*}} '_s' 'D'
 // DUM-NEXT:  | `-MemberExpr {{.*}}<line:11:3> 'int' lvalue .h {{.*}}
 // DUM-NEXT:  |   `-DeclRefExpr {{.*}}<<invalid sloc>> 'D' lvalue Var {{.*}} '_s' 'D'
diff --git a/clang/test/OpenMP/target_map_array_section_of_structs_with_nested_mapper_ast_dump.cpp b/clang/test/OpenMP/target_map_array_section_of_structs_with_nested_mapper_ast_dump.cpp
index b2fb8fbaa50b4..bdf3dd0d4fbfe 100644
--- a/clang/test/OpenMP/target_map_array_section_of_structs_with_nested_mapper_ast_dump.cpp
+++ b/clang/test/OpenMP/target_map_array_section_of_structs_with_nested_mapper_ast_dump.cpp
@@ -27,7 +27,7 @@ void foo() {
 // DUM-NEXT:  |-OMPMapClause {{.*}}<<invalid sloc>> <implicit>
 // DUM-NEXT:  | |-MemberExpr {{.*}}<line:9:3> 'int' lvalue .e
 // DUM-NEXT:  | | `-DeclRefExpr {{.*}}<<invalid sloc>> 'D' lvalue Var {{.*}} '_s' 'D'
-// DUM-NEXT:  | |-MemberExpr {{.*}}<line:10:3> 'C' lvalue .f {{.*}}
+// DUM-NEXT:  | |-MemberExpr {{.*}}<line:10:3> 'C':'struct C' lvalue .f {{.*}}
 // DUM-NEXT:  | | `-DeclRefExpr {{.*}}<<invalid sloc>> 'D' lvalue Var {{.*}} '_s' 'D'
 // DUM-NEXT:  | `-MemberExpr {{.*}}<line:11:3> 'int' lvalue .h {{.*}}
 // DUM-NEXT:  |   `-DeclRefExpr {{.*}}<<invalid sloc>> 'D' lvalue Var {{.*}} '_s' 'D'
diff --git a/clang/test/OpenMP/target_update_strided_messages.c b/clang/test/OpenMP/target_update_strided_messages.c
new file mode 100644
index 0000000000000..1f50af4e52805
--- /dev/null
+++ b/clang/test/OpenMP/target_update_strided_messages.c
@@ -0,0 +1,38 @@
+// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wuninitialized
+
+void foo(void) {}
+
+int main(int argc, char **argv) {
+  int len = 8;
+  double data[len];
+  
+  // Valid strided array sections
+  #pragma omp target update from(data[0:4:2]) // OK
+  {}
+  
+  #pragma omp target update to(data[0:len/2:2]) // OK
+  {}
+  
+  #pragma omp target update from(data[1:3:2]) // OK
+  {}
+  
+  // Missing stride (default = 1)
+  #pragma omp target update from(data[0:4]) // OK
+  {}
+  
+  // Invalid stride expressions
+  #pragma omp target update from(data[0:4:0]) // expected-error {{section stride is evaluated to a non-positive value 0}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  
+  #pragma omp target update from(data[0:4:-1]) // expected-error {{section stride is evaluated to a non-positive value -1}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  
+  // Missing colon 
+  #pragma omp target update from(data[0:4 2]) // expected-error {{expected ']'}} expected-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  {}
+  
+  // Too many colons
+  #pragma omp target update from(data[0:4:2:1]) // expected-error {{expected ']'}} expected-note {{to match this '['}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  {}
+  
+  return 0;
+}
\ No newline at end of file
diff --git a/clang/test/OpenMP/target_update_strided_multiple_messages.c b/clang/test/OpenMP/target_update_strided_multiple_messages.c
new file mode 100644
index 0000000000000..361d4c66c362b
--- /dev/null
+++ b/clang/test/OpenMP/target_update_strided_multiple_messages.c
@@ -0,0 +1,46 @@
+// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wuninitialized
+
+void foo(void) {}
+
+typedef struct {
+  int len;
+  double data[12];
+} S;
+
+int main(int argc, char **argv) {
+  int len = 12;
+  double data1[len], data2[len];
+  S s;
+  
+  // Valid multiple strided array sections
+  #pragma omp target update from(data1[0:4:2], data2[0:2:5]) // OK
+  {}
+  
+  #pragma omp target update to(data1[1:2:3], data2[2:3:2]) // OK
+  {}
+  
+  // Mixed strided and regular array sections
+  #pragma omp target update from(data1[0:len], data2[0:4:2]) // OK
+  {}
+  
+  // Struct member arrays with strides
+  #pragma omp target update from(s.data[0:4:2]) // OK
+  {}
+  
+  #pragma omp target update from(s.data[0:s.len/2:2]) // OK
+  {}
+  
+  // Invalid stride in one of multiple sections
+  #pragma omp target update from(data1[0:3:4], data2[0:2:0]) // expected-error {{section stride is evaluated to a non-positive value 0}}
+  
+  // Complex expressions in multiple arrays
+  int stride1 = 2, stride2 = 3;
+  #pragma omp target update from(data1[0:len/2:stride1], data2[1:len/3:stride2]) // OK
+  {}
+  
+  // Missing colon 
+  #pragma omp target update from(data1[0:4:2], data2[0:3 4]) // expected-error {{expected ']'}} expected-note {{to match this '['}}
+  
+  return 0;
+}
\ No newline at end of file
diff --git a/clang/test/OpenMP/target_update_strided_partial_messages.c b/clang/test/OpenMP/target_update_strided_partial_messages.c
new file mode 100644
index 0000000000000..6dc286c8a1161
--- /dev/null
+++ b/clang/test/OpenMP/target_update_strided_partial_messages.c
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -verify -fopenmp -ferror-limit 100 %s -Wuninitialized
+// RUN: %clang_cc1 -verify -fopenmp-simd -ferror-limit 100 %s -Wuninitialized
+
+void foo(void) {}
+
+int main(int argc, char **argv) {
+  int len = 11;
+  double data[len];
+  
+  // Valid partial strided updates
+  #pragma omp target update from(data[0:4:3]) // OK
+  {}
+  
+  // Stride larger than length
+  #pragma omp target update from(data[0:2:10]) // OK
+  {}
+  
+  // Valid: complex expressions
+  int offset = 1;
+  int count = 3;
+  int stride = 2;
+  #pragma omp target update from(data[offset:count:stride]) // OK
+  {}
+  
+  // Invalid stride expressions
+  #pragma omp target update from(data[0:4:offset-1]) // OK if offset > 1
+  {}
+  
+  #pragma omp target update from(data[0:count:0]) // expected-error {{section stride is evaluated to a non-positive value 0}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+  
+  return 0;
+}
\ No newline at end of file
diff --git a/clang/test/PCH/cxx-explicit-specifier.cpp b/clang/test/PCH/cxx-explicit-specifier.cpp
index 84548faf48810..a30c669fddb47 100644
--- a/clang/test/PCH/cxx-explicit-specifier.cpp
+++ b/clang/test/PCH/cxx-explicit-specifier.cpp
@@ -79,13 +79,13 @@ struct A {
 B<true> b_true;
 B<false> b_false;
 #else
-//expected-note@-8 {{candidate template ignored}} expected-note@-8 {{implicit deduction guide declared as 'template <bool b> A(A<b>) -> A<b>'}}
-//expected-note@-8 {{explicit constructor declared here}} expected-note@-8 {{implicit deduction guide declared as 'template <bool b> explicit(b) A(B<b>) -> A<b>'}}
+//expected-note@-8 {{candidate template ignored}} expected-note@-8 {{implicit deduction guide declared as 'template <bool b> A(templ::A<b>) -> templ::A<b>'}}
+//expected-note@-8 {{explicit constructor declared here}} expected-note@-8 {{implicit deduction guide declared as 'template <bool b> explicit(b) A(B<b>) -> templ::A<b>'}}
 //expected-note@-15+ {{candidate constructor}}
 //expected-note@-8+ {{explicit conversion function is not a candidate (explicit specifier}}
 //expected-note@-11 {{explicit constructor is not a candidate (explicit specifier}}
 
-//CHECK: explicit(b){{ +}}A
+//CHECK: explicit(b){{ +}}templ::A<b>(B<b>)
 //CHECK: explicit(b{{ +}}^{{ +}}T::value){{ +}}operator
 
 A a = { b_true }; //expected-error {{class template argument deduction}}
diff --git a/clang/test/PCH/dedup_types.cpp b/clang/test/PCH/dedup_types.cpp
new file mode 100644
index 0000000000000..d4b19b4411169
--- /dev/null
+++ b/clang/test/PCH/dedup_types.cpp
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -std=c++14 -x c++-header %s -emit-pch -o %t.pch
+// RUN: %clang_cc1 -std=c++14 -x c++ /dev/null -include-pch %t.pch
+
+// RUN: %clang_cc1 -std=c++14 -x c++-header %s -emit-pch -fpch-instantiate-templates -o %t.pch
+// RUN: %clang_cc1 -std=c++14 -x c++ /dev/null -include-pch %t.pch
+
+template <template <class...> class Templ, class...Types>
+using TypePackDedup = Templ<__builtin_dedup_pack<Types...>...>;
+
+template <class ...Ts>
+struct TypeList {};
+
+template <int i>
+struct X {};
+
+void fn1() {
+  TypeList<int, double> l1 = TypePackDedup<TypeList, int, double, int>{};
+  TypeList<> l2 = TypePackDedup<TypeList>{};
+  TypeList<X<0>, X<1>> x1 = TypePackDedup<TypeList, X<0>, X<1>, X<0>, X<1>>{};
+}
diff --git a/clang/test/Parser/MicrosoftExtensions.cpp b/clang/test/Parser/MicrosoftExtensions.cpp
index 9102bca8f6bb2..e32d7fae6af39 100644
--- a/clang/test/Parser/MicrosoftExtensions.cpp
+++ b/clang/test/Parser/MicrosoftExtensions.cpp
@@ -145,7 +145,7 @@ typedef COM_CLASS_TEMPLATE_REF<struct_with_uuid, __uuidof(struct_with_uuid)> COM
 
 COM_CLASS_TEMPLATE_REF<int, __uuidof(struct_with_uuid)> good_template_arg;
 
-COM_CLASS_TEMPLATE<int, __uuidof(struct_with_uuid)> bad_template_arg; // expected-error {{non-type template argument for template parameter of pointer type 'const GUID *' (aka 'const _GUID *') must have its address taken}}
+COM_CLASS_TEMPLATE<int, __uuidof(struct_with_uuid)> bad_template_arg; // expected-error {{non-type template argument for template parameter of pointer type 'const GUID *' (aka 'const struct _GUID *') must have its address taken}}
 
 namespace PR16911 {
 struct __declspec(uuid("{12345678-1234-1234-1234-1234567890aB}")) uuid;
diff --git a/clang/test/Parser/brackets.cpp b/clang/test/Parser/brackets.cpp
index 927b66a7ebcb8..91d4f9b507c8f 100644
--- a/clang/test/Parser/brackets.cpp
+++ b/clang/test/Parser/brackets.cpp
@@ -158,4 +158,53 @@ struct A {
 const char[] A::f = "f";
 // expected-error@-1{{brackets are not allowed here; to declare an array, place the brackets after the name}}
 }
-// CHECK: 15 errors generated.
+
+namespace gh147333 {
+    template<class T, char fmt>
+    constexpr inline auto& to_print_fmt = "";
+    template<> constexpr inline char[] to_print_fmt<unsigned, 'x'> = "0x%x";
+    // expected-error@-1{{brackets are not allowed here; to declare an array, place the brackets after the name}}
+
+#ifndef FIXIT
+    // Further related test cases.
+
+    int[1] operator+();
+    // expected-error@-1{{brackets are not allowed here; to declare an array, place the brackets after the name}}
+    // expected-error@-2{{function cannot return array type}}
+    
+    int[1] operator ""_x(unsigned long long);
+    // expected-error@-1{{brackets are not allowed here; to declare an array, place the brackets after the name}}
+    // expected-error@-2{{function cannot return array type}}
+       
+    struct A {
+        int[1] operator int();
+        // expected-error@-1{{brackets are not allowed here; to declare an array, place the brackets after the name}}
+        // TODO: The following is too noisy and redundant.
+        // expected-error@-3{{conversion function cannot have a return type}}
+        // expected-error@-4{{cannot specify any part of a return type in the declaration of a conversion function}}
+        // expected-error@-5{{conversion function cannot convert to an array type}}
+
+        int[1] A();
+        // expected-error@-1{{brackets are not allowed here; to declare an array, place the brackets after the name}}
+        // TODO: The following is too noisy and redundant.
+        // expected-error@-3{{function cannot return array type}}
+        // expected-error@-4{{constructor cannot have a return type}}
+        
+        int[1] ~A();
+        // expected-error@-1{{brackets are not allowed here; to declare an array, place the brackets after the name}}
+        // TODO: This isn't helpful.
+        // expected-error@-3{{array has incomplete element type 'void'}}
+    };
+    
+    template<typename T>
+    struct B {
+        int[1] B<T>();
+        // expected-error@-1{{brackets are not allowed here; to declare an array, place the brackets after the name}}
+        // TODO: The following is too noisy and redundant.
+        // expected-error@-3{{function cannot return array type}}
+        // expected-error@-4{{constructor cannot have a return type}}
+    };
+#endif
+}
+
+// CHECK: 32 errors generated.
diff --git a/clang/test/Parser/cxx-variadic-func.cpp b/clang/test/Parser/cxx-variadic-func.cpp
index 98a34d3e1bf67..73124b8b1b05b 100644
--- a/clang/test/Parser/cxx-variadic-func.cpp
+++ b/clang/test/Parser/cxx-variadic-func.cpp
@@ -6,3 +6,24 @@ void f(...) {
 }
 
 void h(int n..., int m); // expected-error {{expected ')'}} expected-note {{to match}}
+
+
+namespace GH153445 {
+void f(int = {}...);
+
+struct S {
+  void f(int = {}...);
+  void g(int...);
+};
+
+void S::g(int = {}...) {}
+}
+
+
+template <typename ...T>
+constexpr int a() {return 1;}
+
+struct S2 {
+  template <typename ...Ts>
+  void f(int = a<Ts...>()...);
+};
diff --git a/clang/test/Parser/cxx0x-attributes-preprocessor-tokens.cpp b/clang/test/Parser/cxx0x-attributes-preprocessor-tokens.cpp
new file mode 100644
index 0000000000000..6605d2486aa03
--- /dev/null
+++ b/clang/test/Parser/cxx0x-attributes-preprocessor-tokens.cpp
@@ -0,0 +1,58 @@
+// RUN: %clang_cc1 -fsyntax-only -Wattribute-preprocessor-tokens -verify %s
+// RUN: %clang_cc1 -Wattribute-preprocessor-tokens -E %s | FileCheck %s
+// RUN: %clang_cc1 -x c -fsyntax-only -verify=c %s
+// RUN: %clang_cc1 -x c -E %s | FileCheck %s
+
+#define ATTR_STR(X) [[clang::annotate(#X)]]
+#define ATTR_PASTE(X, Y) [[clang::annotate("test", X ## Y)]]
+
+[[clang::assume(#)]] void f1();           // c-error {{expected expression}} \
+                                          // expected-warning {{'#' is not allowed in an attribute argument list}}
+
+[[clang::assume(##)]] void f2();          // c-error {{expected expression}} \
+                                          // expected-warning {{'##' is not allowed in an attribute argument list}}
+
+[[clang::assume(1#2#3)]] void f3();       // c-error {{use of this expression in an 'assume' attribute requires parentheses}} \
+                                          // c-error {{expected ')'}} \
+                                          // c-note {{to match this '('}} \
+                                          // expected-warning {{'#' is not allowed in an attribute argument list}} \
+                                          // expected-warning {{'#' is not allowed in an attribute argument list}}
+
+[[unknown::unknown(#)]] void f4();        // c-warning {{unknown attribute 'unknown::unknown' ignored}} \
+                                          // expected-warning {{'#' is not allowed in an attribute argument list}}
+
+[[unknown::unknown(##)]] void f5();       // c-warning {{unknown attribute 'unknown::unknown' ignored}} \
+                                          // expected-warning {{'##' is not allowed in an attribute argument list}}
+
+[[unknown::unknown(1#2#3)]] void f6();    // c-warning {{unknown attribute 'unknown::unknown' ignored}} \
+                                          // expected-warning {{'#' is not allowed in an attribute argument list}} \
+                                          // expected-warning {{'#' is not allowed in an attribute argument list}}
+
+[[clang::assume(%:)]] void f7();          // c-error {{expected expression}} \
+                                          // expected-warning {{'%:' is not allowed in an attribute argument list}}
+
+
+[[clang::assume(%:%:)]] void f8();        // c-error {{expected expression}} \
+                                          // expected-warning {{'%:%:' is not allowed in an attribute argument list}}
+
+[[clang::assume(1%:2%:3)]] void f9();     // c-error {{use of this expression in an 'assume' attribute requires parentheses}} \
+                                          // c-error {{expected ')'}} \
+                                          // c-note {{to match this '('}} \
+                                          // expected-warning {{'%:' is not allowed in an attribute argument list}} \
+                                          // expected-warning {{'%:' is not allowed in an attribute argument list}}
+
+[[unknown::unknown(%:)]] void f10();      // c-warning {{unknown attribute 'unknown::unknown' ignored}} \
+                                          // expected-warning {{'%:' is not allowed in an attribute argument list}}
+
+[[unknown::unknown(%:%:)]] void f11();    // c-warning {{unknown attribute 'unknown::unknown' ignored}} \
+                                          // expected-warning {{'%:%:' is not allowed in an attribute argument list}}
+
+[[unknown::unknown(1%:2%:3)]] void f12(); // c-warning {{unknown attribute 'unknown::unknown' ignored}} \
+                                          // expected-warning {{'%:' is not allowed in an attribute argument list}} \
+                                          // expected-warning {{'%:' is not allowed in an attribute argument list}}
+
+ATTR_STR(stringify) void f13();
+// CHECK: {{\[\[}}clang{{::}}annotate("stringify"){{\]\]}} void f13();
+
+ATTR_PASTE(1, 2) void f14();
+// CHECK: {{\[\[}}clang{{::}}annotate("test", 12){{\]\]}} void f14();
diff --git a/clang/test/Parser/cxx1z-class-template-argument-deduction.cpp b/clang/test/Parser/cxx1z-class-template-argument-deduction.cpp
index d29eed40b1864..9d27f83698e00 100644
--- a/clang/test/Parser/cxx1z-class-template-argument-deduction.cpp
+++ b/clang/test/Parser/cxx1z-class-template-argument-deduction.cpp
@@ -196,8 +196,8 @@ namespace typename_specifier {
     new typename T::A{0};
     typename T::A a = 0;
     const typename T::A b = 0;
-    if (typename T::A a = 0) {} // expected-error {{value of type 'typename X::A<int>' (aka 'typename_specifier::X::A<int>') is not contextually convertible to 'bool'}}
-    for (typename T::A a = 0; typename T::A b = 0; /**/) {} // expected-error {{value of type 'typename X::A<int>' (aka 'typename_specifier::X::A<int>') is not contextually convertible to 'bool'}}
+    if (typename T::A a = 0) {} // expected-error {{value of type 'typename typename_specifier::X::A<int>' (aka 'typename_specifier::X::A<int>') is not contextually convertible to 'bool'}}
+    for (typename T::A a = 0; typename T::A b = 0; /**/) {} // expected-error {{value of type 'typename typename_specifier::X::A<int>' (aka 'typename_specifier::X::A<int>') is not contextually convertible to 'bool'}}
 
     {(void)(typename T::A)(0);} // expected-error{{refers to class template member}}
     {(void)(typename T::A){0};} // expected-error{{refers to class template member}}
@@ -208,7 +208,7 @@ namespace typename_specifier {
     {typename T::A arr[3] = 0;} // expected-error {{refers to class template member}}
     {typename T::A F::*pm = 0;} // expected-error {{refers to class template member}}
     {typename T::A (*fp)() = 0;} // expected-error {{refers to class template member}}
-    {typename T::A [x, y] = 0;} // expected-error {{cannot be declared with type 'typename T::A'}} expected-error {{type 'typename X::A<int>' (aka 'typename_specifier::X::A<int>') decomposes into 0}}
+    {typename T::A [x, y] = 0;} // expected-error {{cannot be declared with type 'typename T::A'}} expected-error {{type 'typename typename_specifier::X::A<int>' (aka 'typename_specifier::X::A<int>') decomposes into 0}}
   }
   template void f<X>(); // expected-note {{instantiation of}}
 
diff --git a/clang/test/Parser/cxx2c-oxford-variadic-comma.cpp b/clang/test/Parser/cxx2c-oxford-variadic-comma.cpp
index b8015b4815b2e..18ce770851a36 100644
--- a/clang/test/Parser/cxx2c-oxford-variadic-comma.cpp
+++ b/clang/test/Parser/cxx2c-oxford-variadic-comma.cpp
@@ -36,6 +36,7 @@ void o(int x, ...);
 
 struct S {
   void p(this S...) {} // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
+  void f(int = {}...); // expected-warning {{declaration of a variadic function without a comma before '...' is deprecated}}
 };
 
 template<class ...Ts>
diff --git a/clang/test/Parser/diagnose_if.cpp b/clang/test/Parser/diagnose_if.cpp
new file mode 100644
index 0000000000000..52059804c9ef9
--- /dev/null
+++ b/clang/test/Parser/diagnose_if.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 %s -fsyntax-only -fcxx-exceptions -verify
+
+void t1() __attribute__((__diagnose_if__(baz))) try {} catch(...) {}
+// expected-error@-1 {{use of undeclared identifier 'baz'}}
+
+struct A {
+  A();
+};
+
+A::A() __attribute__((__diagnose_if__(baz))) :;
+// expected-error@-1 {{expected class member or base class name}}
+// expected-error@-2 {{use of undeclared identifier 'baz'}}
diff --git a/clang/test/Parser/explicit-bool-pre-cxx17.cpp b/clang/test/Parser/explicit-bool-pre-cxx17.cpp
new file mode 100644
index 0000000000000..fee0889737a89
--- /dev/null
+++ b/clang/test/Parser/explicit-bool-pre-cxx17.cpp
@@ -0,0 +1,15 @@
+// Regression test for assertion failure when explicit(bool) is used in pre-C++20
+// Fixes GitHub issue #152729
+// RUN: %clang_cc1 -std=c++98 -verify %s
+// RUN: %clang_cc1 -std=c++03 -verify %s
+// RUN: %clang_cc1 -std=c++11 -verify %s
+// RUN: %clang_cc1 -std=c++14 -verify %s
+// RUN: %clang_cc1 -std=c++17 -verify %s
+
+struct S {
+  explicit(true) S(int);
+  // expected-warning@-1 {{explicit(bool) is a C++20 extension}}
+  
+  explicit(false) S(float);
+  // expected-warning@-1 {{explicit(bool) is a C++20 extension}}
+};
diff --git a/clang/test/Preprocessor/embed_constexpr.c b/clang/test/Preprocessor/embed_constexpr.c
index e444dfec158b5..e4c85cce459e8 100644
--- a/clang/test/Preprocessor/embed_constexpr.c
+++ b/clang/test/Preprocessor/embed_constexpr.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 %s -fsyntax-only --embed-dir=%S/Inputs -verify -std=c23
+// RUN: %clang_cc1 %s -fsyntax-only --embed-dir=%S/Inputs -verify -std=c23 -fexperimental-new-constant-interpreter
 
 static constexpr unsigned char data[] = {
 #embed "big_char.txt"
@@ -19,3 +20,7 @@ static constexpr unsigned data3[] = {
 static constexpr int data4[] = {
 #embed "big_char.txt" suffix(, -1)
 };
+
+static constexpr float data5[] = {
+#embed "big_char.txt" suffix(, -1)
+};
diff --git a/clang/test/Preprocessor/embed_parsing_errors.c b/clang/test/Preprocessor/embed_parsing_errors.c
index 490ec6d4ded2c..a8bbdea91eb16 100644
--- a/clang/test/Preprocessor/embed_parsing_errors.c
+++ b/clang/test/Preprocessor/embed_parsing_errors.c
@@ -94,6 +94,9 @@ char buffer[] = {
 #embed "embed_parsing_errors.c" prefix() // OK: tokens within parens are optional
 #embed "embed_parsing_errors.c" prefix)
 // expected-error@-1 {{expected '('}}
+#embed "embed_parsing_errors.c" prefix()) // expected-error {{expected identifier}}
+#embed "embed_parsing_errors.c" prefix(]) // expected-error {{expected ')'}}
+#embed "embed_parsing_errors.c" prefix(}) // expected-error {{expected ')'}}
 
 #embed "embed_parsing_errors.c" suffix
 // expected-error@-1 {{expected '('}}
@@ -115,6 +118,9 @@ char buffer[] = {
 #embed "embed_parsing_errors.c" suffix() // OK: tokens within parens are optional
 #embed "embed_parsing_errors.c" suffix)
 // expected-error@-1 {{expected '('}}
+#embed "embed_parsing_errors.c" suffix()) // expected-error {{expected identifier}}
+#embed "embed_parsing_errors.c" suffix(]) // expected-error {{expected ')'}}
+#embed "embed_parsing_errors.c" suffix(}) // expected-error {{expected ')'}}
 
 #embed "embed_parsing_errors.c" if_empty(1/0) // OK: emitted as tokens, not evaluated yet.
 #embed "embed_parsing_errors.c" if_empty(([{}])) // OK: delimiters balanced
@@ -128,3 +134,6 @@ char buffer[] = {
 #embed "embed_parsing_errors.c" if_empty)
 // expected-error@-1 {{expected '('}}
 };
+#embed "embed_parsing_errors.c" if_empty()) // expected-error {{expected identifier}}
+#embed "embed_parsing_errors.c" if_empty(]) // expected-error {{expected ')'}}
+#embed "embed_parsing_errors.c" if_empty(}) // expected-error {{expected ')'}}
diff --git a/clang/test/Preprocessor/file_test.c b/clang/test/Preprocessor/file_test.c
index 945882ded7a6a..1e7e1dfe96dd9 100644
--- a/clang/test/Preprocessor/file_test.c
+++ b/clang/test/Preprocessor/file_test.c
@@ -1,4 +1,4 @@
-// UNSUPPORTED: system-windows
+// UNSUPPORTED: system-windows, system-cygwin
 // RUN: %clang -E -ffile-prefix-map=%p=/UNLIKELY_PATH/empty -c -o - %s | FileCheck %s
 // RUN: %clang -E -fmacro-prefix-map=%p=/UNLIKELY_PATH/empty -c -o - %s | FileCheck %s
 // RUN: %clang -E -fmacro-prefix-map=%p=/UNLIKELY_PATH=empty -c -o - %s | FileCheck %s -check-prefix CHECK-EVIL
diff --git a/clang/test/Preprocessor/init.c b/clang/test/Preprocessor/init.c
index bed39dc3e34dc..7e0df96141364 100644
--- a/clang/test/Preprocessor/init.c
+++ b/clang/test/Preprocessor/init.c
@@ -1622,6 +1622,14 @@
 // RUN: %clang_cc1 -x c -std=c99 -E -dM -ffreestanding -triple=amd64-unknown-openbsd < /dev/null | FileCheck -match-full-lines -check-prefix OPENBSD-STDC-N %s
 // OPENBSD-STDC-N-NOT:#define __STDC_NO_THREADS__ 1
 //
+// RUN: %clang_cc1 -x c -std=c11 -E -dM -ffreestanding -triple=x86_64-unknown-dragonfly < /dev/null | FileCheck -match-full-lines -check-prefix DRAGONFLY-STDC %s
+// RUN: %clang_cc1 -x c -std=gnu11 -E -dM -ffreestanding -triple=x86_64-unknown-dragonfly < /dev/null | FileCheck -match-full-lines -check-prefix DRAGONFLY-STDC %s
+// RUN: %clang_cc1 -x c -std=c17 -E -dM -ffreestanding -triple=x86_64-unknown-dragonfly < /dev/null | FileCheck -match-full-lines -check-prefix DRAGONFLY-STDC %s
+// DRAGONFLY-STDC:#define __STDC_NO_THREADS__ 1
+//
+// RUN: %clang_cc1 -x c -std=c99 -E -dM -ffreestanding -triple=x86_64-unknown-dragonfly < /dev/null | FileCheck -match-full-lines -check-prefix DRAGONFLY-STDC-N %s
+// DRAGONFLY-STDC-N-NOT:#define __STDC_NO_THREADS__ 1
+//
 // RUN: %clang_cc1 -triple=aarch64-unknown-managarm-mlibc -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix MANAGARM %s
 // RUN: %clang_cc1 -triple=riscv64-unknown-managarm-mlibc -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix MANAGARM %s
 // RUN: %clang_cc1 -triple=x86_64-unknown-managarm-mlibc -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix MANAGARM %s
diff --git a/clang/test/Preprocessor/predefined-arch-macros.c b/clang/test/Preprocessor/predefined-arch-macros.c
index e82d825704439..2ea2d511f5e4e 100644
--- a/clang/test/Preprocessor/predefined-arch-macros.c
+++ b/clang/test/Preprocessor/predefined-arch-macros.c
@@ -1907,7 +1907,6 @@
 // CHECK_GNR_M32: #define __BMI2__ 1
 // CHECK_GNR_M32: #define __BMI__ 1
 // CHECK_DMR_M32: #define __CCMP__ 1
-// CHECK_DMR_M32: #define __CF__ 1
 // CHECK_GNR_M32: #define __CLDEMOTE__ 1
 // CHECK_GNR_M32: #define __CLFLUSHOPT__ 1
 // CHECK_GNR_M32: #define __CLWB__ 1
@@ -2017,7 +2016,6 @@
 // CHECK_GNR_M64: #define __BMI2__ 1
 // CHECK_GNR_M64: #define __BMI__ 1
 // CHECK_DMR_M64: #define __CCMP__ 1
-// CHECK_DMR_M64: #define __CF__ 1
 // CHECK_GNR_M64: #define __CLDEMOTE__ 1
 // CHECK_GNR_M64: #define __CLFLUSHOPT__ 1
 // CHECK_GNR_M64: #define __CLWB__ 1
diff --git a/clang/test/Preprocessor/ptrauth_extension.c b/clang/test/Preprocessor/ptrauth_extension.c
index d6b79187ba62d..3267b0786c28f 100644
--- a/clang/test/Preprocessor/ptrauth_extension.c
+++ b/clang/test/Preprocessor/ptrauth_extension.c
@@ -4,10 +4,32 @@
 // RUN: %clang_cc1 -E %s -triple=aarch64 -fptrauth-calls | \
 // RUN:   FileCheck %s --check-prefixes=NOINTRIN
 
-#if __has_extension(ptrauth_qualifier)
-// INTRIN: has_ptrauth_qualifier
-void has_ptrauth_qualifier() {}
-#else
+// RUN: %clang_cc1 -E %s -DIS_DARWIN -triple=arm64e-apple-darwin -fptrauth-intrinsics | \
+// RUN:   FileCheck %s --check-prefixes=INTRIN,INTRIN_MAC
+
+// RUN: %clang_cc1 -E %s -DIS_DARWIN -triple=arm64e-apple-darwin -fptrauth-calls | \
+// RUN:   FileCheck %s --check-prefixes=NOINTRIN
+
+#if defined(IS_DARWIN) && __has_extension(ptrauth_qualifier)
+// INTRIN_MAC: has_ptrauth_qualifier1
+void has_ptrauth_qualifier1() {}
+#ifndef __PTRAUTH__
+#error ptrauth_qualifier extension present without predefined test macro
+#endif
+#endif
+#if defined(IS_DARWIN) && __has_feature(ptrauth_qualifier)
+// INTRIN_MAC: has_ptrauth_qualifier2
+void has_ptrauth_qualifier2() {}
+#ifndef __PTRAUTH__
+#error ptrauth_qualifier extension present without predefined test macro
+#endif
+#endif
+#if defined(__PTRAUTH__)
+// INTRIN: has_ptrauth_qualifier3
+void has_ptrauth_qualifier3() {}
+#endif
+
+#if !defined(__PTRAUTH__) && !__has_feature(ptrauth_qualifier) && !__has_extension(ptrauth_qualifier)
 // NOINTRIN: no_ptrauth_qualifier
 void no_ptrauth_qualifier() {}
 #endif
diff --git a/clang/test/Preprocessor/ptrauth_feature.c b/clang/test/Preprocessor/ptrauth_feature.c
index a440791d6cc69..cebea4188415f 100644
--- a/clang/test/Preprocessor/ptrauth_feature.c
+++ b/clang/test/Preprocessor/ptrauth_feature.c
@@ -34,7 +34,7 @@
 // RUN: %clang_cc1 -E %s -triple=aarch64 -fptrauth-elf-got | \
 // RUN:   FileCheck %s --check-prefixes=NOINTRIN,NOCALLS,NORETS,NOVPTR_ADDR_DISCR,NOVPTR_TYPE_DISCR,NOTYPE_INFO_DISCR,NOFUNC,NOINITFINI,NOINITFINI_ADDR_DISCR,NOGOTOS,ELFGOT
 
-#if __has_feature(ptrauth_intrinsics)
+#if defined(__PTRAUTH__)
 // INTRIN: has_ptrauth_intrinsics
 void has_ptrauth_intrinsics() {}
 #else
@@ -130,3 +130,11 @@ void has_ptrauth_elf_got() {}
 // NOELFGOT: no_ptrauth_elf_got
 void no_ptrauth_elf_got() {}
 #endif
+
+#if __has_feature(ptrauth_objc_signable_class)
+// INTRIN: has_ptrauth_objc_signable_class
+void has_ptrauth_objc_signable_class(){}
+#else
+// NOINTRIN: no_ptrauth_objc_signable_class
+void no_ptrauth_objc_signable_class(){}
+#endif
diff --git a/clang/test/Preprocessor/sanitizer-predefines.c b/clang/test/Preprocessor/sanitizer-predefines.c
new file mode 100644
index 0000000000000..9d2f6bf2517a2
--- /dev/null
+++ b/clang/test/Preprocessor/sanitizer-predefines.c
@@ -0,0 +1,8 @@
+// RUN: %clang_cc1 -E -dM -triple aarch64-unknown-linux -fsanitize=address %s | FileCheck %s --check-prefix=ASAN
+// ASAN: #define __SANITIZE_ADDRESS__ 1
+
+// RUN: %clang_cc1 -E -dM -triple aarch64-unknown-linux -fsanitize=hwaddress %s | FileCheck %s --check-prefix=HWASAN
+// HWASAN: #define __SANITIZE_HWADDRESS__ 1
+
+// RUN: %clang_cc1 -E -dM -triple aarch64-unknown-linux -fsanitize=thread %s | FileCheck %s --check-prefix=TSAN
+// TSAN: #define __SANITIZE_THREAD__ 1
diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c
index 3edc92c75303a..43bf177c689ca 100644
--- a/clang/test/Preprocessor/x86_target_features.c
+++ b/clang/test/Preprocessor/x86_target_features.c
@@ -795,7 +795,7 @@
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=nf -x c -E -dM -o - %s | FileCheck --check-prefix=NF %s
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=cf -x c -E -dM -o - %s | FileCheck --check-prefix=CF %s
 // RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapx-features=zu -x c -E -dM -o - %s | FileCheck --check-prefix=ZU %s
-// RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapxf -x c -E -dM -o - %s | FileCheck --check-prefixes=EGPR,PUSH2POP2,PPX,NDD,CCMP,NF,CF,ZU,APXF %s
+// RUN: %clang -target x86_64-unknown-unknown -march=x86-64 -mapxf -x c -E -dM -o - %s | FileCheck --check-prefixes=EGPR,PUSH2POP2,PPX,NDD,CCMP,NF,ZU,APXF %s
 // APXF: #define __APX_F__ 1
 // CCMP: #define __CCMP__ 1
 // CF: #define __CF__ 1
diff --git a/clang/test/Sema/aarch64-sve-intrinsics/acle_sve_compact.cpp b/clang/test/Sema/aarch64-sve-intrinsics/acle_sve_compact.cpp
new file mode 100644
index 0000000000000..4de3f39928f16
--- /dev/null
+++ b/clang/test/Sema/aarch64-sve-intrinsics/acle_sve_compact.cpp
@@ -0,0 +1,18 @@
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve \
+// RUN: -verify -verify-ignore-unexpected=error,note -emit-llvm -o - %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme \
+// RUN: -verify -verify-ignore-unexpected=error,note -emit-llvm -o - %s
+// REQUIRES: aarch64-registered-target
+// expected-no-diagnostics
+
+#include <arm_sve.h>
+
+__attribute__((target("sme2p2")))
+void test_svcompact(svbool_t pg, svfloat32_t op) __arm_streaming{
+  svcompact(pg, op);
+}
+
+void test_svcompact_nofeature(svbool_t pg, svfloat32_t op) __arm_streaming{
+  // expected-error@+1 {{'svcompact' needs target feature (sve)|(sme, sme2p2)}}
+  svcompact(pg, op);
+}
\ No newline at end of file
diff --git a/clang/test/Sema/address-packed.c b/clang/test/Sema/address-packed.c
index 29f12490e9fab..f826b7d57d91c 100644
--- a/clang/test/Sema/address-packed.c
+++ b/clang/test/Sema/address-packed.c
@@ -338,3 +338,11 @@ struct Invalid0 {
 void *g14(struct Invalid0 *ivl) {
   return &(ivl->x);
 }
+
+void to_void_with_expr(void *ptr, int expr);
+
+void g15(void) {
+  struct Arguable arguable;
+  to_void_with_expr(&arguable.x, 3); // no-warning
+  to_void_with_expr(&arguable.x, ({3;})); // no-warning
+}
diff --git a/clang/test/Sema/attr-cfi-salt.c b/clang/test/Sema/attr-cfi-salt.c
new file mode 100644
index 0000000000000..bccdfc49ec5bb
--- /dev/null
+++ b/clang/test/Sema/attr-cfi-salt.c
@@ -0,0 +1,60 @@
+// RUN: %clang_cc1 -std=c11 -fsyntax-only -fsanitize=kcfi -verify %s
+// RUN: %clang_cc1 -std=c89 -DKNR -fsyntax-only -fsanitize=kcfi -verify %s
+
+#define __cfi_salt(S) __attribute__((cfi_salt(S)))
+
+int bad1(void) __cfi_salt(); // expected-error{{'cfi_salt' attribute takes one argument}}
+int bad2(void) __cfi_salt(42); // expected-error{{expected string literal as argument of 'cfi_salt' attribute}}
+int bad3(void) __attribute__((cfi_salt("a", "b", "c"))); // expected-error{{'cfi_salt' attribute takes one argument}}
+
+
+int foo(int a, int b) __cfi_salt("pepper"); // ok
+int foo(int a, int b) __cfi_salt("pepper"); // ok
+
+#ifndef KNR
+typedef int (*bar_t)(void) __cfi_salt("pepper"); // ok
+typedef int (*bar_t)(void) __cfi_salt("pepper"); // ok
+#endif
+
+// FIXME: Should we allow this?
+// int b(void) __cfi_salt("salt 'n") __cfi_salt("pepper");
+// bar_t bar_fn __cfi_salt("salt 'n");
+
+int baz __cfi_salt("salt"); // expected-warning{{'cfi_salt' only applies to function types}}
+
+int baz_fn(int a, int b) __cfi_salt("salt 'n"); // expected-note{{previous declaration is here}}
+int baz_fn(int a, int b) __cfi_salt("pepper"); // expected-error{{conflicting types for 'baz_fn'}}
+
+int mux_fn(int a, int b) __cfi_salt("salt 'n"); // expected-note{{previous declaration is here}}
+int mux_fn(int a, int b) __cfi_salt("pepper") { // expected-error{{conflicting types for 'mux_fn'}}
+  return a * b;
+}
+
+typedef int qux_t __cfi_salt("salt"); // expected-warning{{'cfi_salt' only applies to function types}}
+
+typedef int (*quux_t)(void) __cfi_salt("salt 'n"); // expected-note{{previous definition is here}}
+typedef int (*quux_t)(void) __cfi_salt("pepper"); // expected-error{{typedef redefinition with different type}}
+
+void func1(int a) __cfi_salt("pepper"); // expected-note{{previous declaration is here}}
+void func1(int a) { } // expected-error{{conflicting types for 'func1'}}
+void (*fp1)(int) = func1; // expected-error{{incompatible function pointer types initializing 'void (*)(int)' with an expression of type 'void (int)'}}
+
+void func2(int) [[clang::cfi_salt("test")]]; // expected-note{{previous declaration is here}}
+void func2(int a) { } // expected-error{{conflicting types for 'func2'}}
+void (*fp2)(int) = func2; // expected-error{{incompatible function pointer types initializing 'void (*)(int)' with an expression of type 'void (int)'}}
+
+void func3(int) __cfi_salt("pepper"); // ok
+void func3(int a) __cfi_salt("pepper") { } // ok
+void (* __cfi_salt("pepper") fp3)(int) = func3; // ok
+void (*fp3_noattr)(int) = func3; // expected-error{{incompatible function pointer types initializing 'void (*)(int)' with an expression of type 'void (int)'}}
+
+void func4(int) [[clang::cfi_salt("test")]]; // ok
+void func4(int a) [[clang::cfi_salt("test")]] { } // ok
+void (* [[clang::cfi_salt("test")]] fp4)(int) = func4; // ok
+void (*fp4_noattr)(int) = func4; // expected-error{{incompatible function pointer types initializing 'void (*)(int)' with an expression of type 'void (int)'}}
+
+#ifdef KNR
+// K&R C function without a prototype
+void func() __attribute__((cfi_salt("pepper"))); // expected-error {{attribute only applies to non-K&R-style functions}}
+void (*fp)() __attribute__((cfi_salt("pepper")));  // expected-error {{attribute only applies to non-K&R-style functions}}
+#endif
diff --git a/clang/test/Sema/builtin-masked.c b/clang/test/Sema/builtin-masked.c
new file mode 100644
index 0000000000000..81f5323bbe260
--- /dev/null
+++ b/clang/test/Sema/builtin-masked.c
@@ -0,0 +1,25 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fsyntax-only -verify %s
+
+typedef int v8i __attribute__((ext_vector_type(8)));
+typedef _Bool v8b __attribute__((ext_vector_type(8)));
+typedef _Bool v2b __attribute__((ext_vector_type(2)));
+typedef float v8f __attribute__((ext_vector_type(8)));
+
+void test_masked_load(v8i *pf, v8b mask, v2b mask2) {
+  (void)__builtin_masked_load(mask); // expected-error {{too few arguments to function call, expected 2, have 1}}
+  (void)__builtin_masked_load(mask, pf, pf); // expected-error {{too many arguments to function call, expected 2, have 3}}
+  (void)__builtin_masked_load(mask2, pf); // expected-error {{all arguments to __builtin_masked_load must have the same number of elements}}
+  (void)__builtin_masked_load(mask, mask); // expected-error {{2nd argument must be a pointer to vector}}
+  (void)__builtin_masked_load(mask, (void *)0); // expected-error {{2nd argument must be a pointer to vector}}
+  (void)__builtin_masked_load(mask2, pf); // expected-error {{all arguments to __builtin_masked_load must have the same number of elements}}
+}
+
+void test_masked_store(v8i *pf, v8f *pf2, v8b mask, v2b mask2) {
+  __builtin_masked_store(mask); // expected-error {{too few arguments to function call, expected 3, have 1}}
+  __builtin_masked_store(mask, 0, 0, 0); // expected-error {{too many arguments to function call, expected 3, have 4}}
+  __builtin_masked_store(0, 0, pf); // expected-error {{1st argument must be a vector of boolean types (was 'int')}}
+  __builtin_masked_store(mask, 0, pf); // expected-error {{2nd argument must be a vector}}
+  __builtin_masked_store(mask, *pf, 0); // expected-error {{3rd argument must be a pointer to vector}}
+  __builtin_masked_store(mask2, *pf, pf); // expected-error {{all arguments to __builtin_masked_store must have the same number of elements}}
+  __builtin_masked_store(mask, *pf, pf2); // expected-error {{last two arguments to '__builtin_masked_store' must have the same type}}
+}
diff --git a/clang/test/Sema/builtin-object-size.c b/clang/test/Sema/builtin-object-size.c
index 20d4e2ab6da79..a763c24fd6620 100644
--- a/clang/test/Sema/builtin-object-size.c
+++ b/clang/test/Sema/builtin-object-size.c
@@ -2,6 +2,10 @@
 // RUN: %clang_cc1 -fsyntax-only -triple x86_64-apple-darwin9 -verify %s
 // RUN: %clang_cc1 -DDYNAMIC -fsyntax-only -triple x86_64-apple-darwin9 -verify %s
 
+// RUN: %clang_cc1 -fsyntax-only -verify %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -fsyntax-only -triple x86_64-apple-darwin9 -verify %s -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -DDYNAMIC -fsyntax-only -triple x86_64-apple-darwin9 -verify %s -fexperimental-new-constant-interpreter
+
 #ifndef DYNAMIC
 #define OBJECT_SIZE_BUILTIN __builtin_object_size
 #else
diff --git a/clang/test/Sema/builtins-elementwise-math.c b/clang/test/Sema/builtins-elementwise-math.c
index 8548d3be8c44a..02dd44211e96a 100644
--- a/clang/test/Sema/builtins-elementwise-math.c
+++ b/clang/test/Sema/builtins-elementwise-math.c
@@ -1294,6 +1294,42 @@ void test_builtin_elementwise_fma(int i32, int2 v2i32, short i16,
   // expected-error@-1 {{3rd argument must be a scalar or vector of floating-point types (was '_Complex float')}}
 }
 
+void test_builtin_elementwise_fsh(int i32, int2 v2i32, short i16, int3 v3i32,
+				  double f64, float f32, float2 v2f32) {
+    i32 = __builtin_elementwise_fshl();
+    // expected-error@-1 {{too few arguments to function call, expected 3, have 0}}
+
+    i32 = __builtin_elementwise_fshr();
+    // expected-error@-1 {{too few arguments to function call, expected 3, have 0}}
+
+    i32 = __builtin_elementwise_fshl(i32, i32);
+    // expected-error@-1 {{too few arguments to function call, expected 3, have 2}}
+
+    i32 = __builtin_elementwise_fshr(i32, i32);
+    // expected-error@-1 {{too few arguments to function call, expected 3, have 2}}
+
+    i32 = __builtin_elementwise_fshl(i32, i32, i16);
+    // expected-error@-1 {{arguments are of different types ('int' vs 'short')}}
+
+    i16 = __builtin_elementwise_fshr(i16, i32, i16);
+    // expected-error@-1 {{arguments are of different types ('short' vs 'int')}}
+
+    f32 = __builtin_elementwise_fshl(f32, f32, f32);
+    // expected-error@-1 {{argument must be a scalar or vector of integer types (was 'float')}}
+
+    f64 = __builtin_elementwise_fshr(f64, f64, f64);
+    // expected-error@-1 {{argument must be a scalar or vector of integer types (was 'double')}}
+
+    v2i32 = __builtin_elementwise_fshl(v2i32, v2i32, v2f32);
+    // expected-error@-1 {{argument must be a scalar or vector of integer types (was 'float2' (vector of 2 'float' values))}}
+
+    v2i32 = __builtin_elementwise_fshr(v2i32, v2i32, v3i32);
+    // expected-error@-1 {{arguments are of different types ('int2' (vector of 2 'int' values) vs 'int3' (vector of 3 'int' values))}}
+
+    v3i32 = __builtin_elementwise_fshl(v3i32, v3i32, v2i32);
+    // expected-error@-1 {{arguments are of different types ('int3' (vector of 3 'int' values) vs 'int2' (vector of 2 'int' values))}}
+}
+
 typedef struct {
   float3 b;
 } struct_float3;
@@ -1303,3 +1339,47 @@ float3 foo(float3 a,const struct_float3* hi) {
   float3 b = __builtin_elementwise_max((float3)(0.0f), a);
   return __builtin_elementwise_pow(b, hi->b.yyy);
 }
+
+void test_builtin_elementwise_ctlz(int i32, int2 v2i32, short i16,
+                                   double f64, double2 v2f64) {
+  f64 = __builtin_elementwise_ctlz(f64);
+  // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'double')}}
+
+  _Complex float c1;
+  c1 = __builtin_elementwise_ctlz(c1);
+  // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was '_Complex float')}}
+
+  v2i32 = __builtin_elementwise_ctlz(v2i32, i32);
+  // expected-error@-1 {{arguments are of different types ('int2' (vector of 2 'int' values) vs 'int')}}
+
+  v2i32 = __builtin_elementwise_ctlz(v2i32, f64);
+  // expected-error@-1 {{arguments are of different types ('int2' (vector of 2 'int' values) vs 'double')}}
+
+  v2i32 = __builtin_elementwise_ctlz();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+
+  v2i32 = __builtin_elementwise_ctlz(v2i32, v2i32, f64);
+  // expected-error@-1 {{too many arguments to function call, expected 2, have 3}}
+}
+
+void test_builtin_elementwise_cttz(int i32, int2 v2i32, short i16,
+                                   double f64, double2 v2f64) {
+  f64 = __builtin_elementwise_cttz(f64);
+  // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was 'double')}}
+
+  _Complex float c1;
+  c1 = __builtin_elementwise_cttz(c1);
+  // expected-error@-1 {{1st argument must be a scalar or vector of integer types (was '_Complex float')}}
+
+  v2i32 = __builtin_elementwise_cttz(v2i32, i32);
+  // expected-error@-1 {{arguments are of different types ('int2' (vector of 2 'int' values) vs 'int')}}
+
+  v2i32 = __builtin_elementwise_cttz(v2i32, f64);
+  // expected-error@-1 {{arguments are of different types ('int2' (vector of 2 'int' values) vs 'double')}}
+
+  v2i32 = __builtin_elementwise_cttz();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+
+  v2i32 = __builtin_elementwise_cttz(v2i32, v2i32, f64);
+  // expected-error@-1 {{too many arguments to function call, expected 2, have 3}}
+}
diff --git a/clang/test/Sema/builtins-wasm.c b/clang/test/Sema/builtins-wasm.c
index a3486b1aedb13..9075e9eaa5230 100644
--- a/clang/test/Sema/builtins-wasm.c
+++ b/clang/test/Sema/builtins-wasm.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -triple wasm32 -target-feature +reference-types %s
+// RUN: %clang_cc1 -fsyntax-only -verify -triple wasm32 -target-abi experimental-mv -DMULTIVALUE -target-feature +reference-types %s
 
 #define EXPR_HAS_TYPE(expr, type) _Generic((expr), type : 1, default : 0)
 
@@ -57,8 +58,8 @@ void test_table_copy(int dst_idx, int src_idx, int nelem) {
 
 typedef void (*F1)(void);
 typedef int (*F2)(int);
-typedef int (*F3)(__externref_t);
-typedef __externref_t (*F4)(int);
+typedef void (*F3)(struct {int x; double y;});
+typedef struct {int x; double y;} (*F4)(void);
 
 void test_function_pointer_signature() {
   // Test argument count validation
@@ -68,8 +69,6 @@ void test_function_pointer_signature() {
   // // Test argument type validation - should require function pointer
   (void)__builtin_wasm_test_function_pointer_signature((void*)0); // expected-error {{used type 'void *' where function pointer is required}}
   (void)__builtin_wasm_test_function_pointer_signature((int)0);   // expected-error {{used type 'int' where function pointer is required}}
-  (void)__builtin_wasm_test_function_pointer_signature((F3)0);   // expected-error {{not supported for function pointers with a reference type parameter}}
-  (void)__builtin_wasm_test_function_pointer_signature((F4)0);   // expected-error {{not supported for function pointers with a reference type return value}}
 
   // // Test valid usage
   int res = __builtin_wasm_test_function_pointer_signature((F1)0);
@@ -77,4 +76,14 @@ void test_function_pointer_signature() {
 
   // Test return type
   _Static_assert(EXPR_HAS_TYPE(__builtin_wasm_test_function_pointer_signature((F1)0), int), "");
+
+#ifdef MULTIVALUE
+  // Test that struct arguments and returns are rejected with multivalue abi
+  (void)__builtin_wasm_test_function_pointer_signature((F3)0); // expected-error {{not supported with the multivalue ABI for function pointers with a struct/union as parameter}}
+  (void)__builtin_wasm_test_function_pointer_signature((F4)0); // expected-error {{not supported with the multivalue ABI for function pointers with a struct/union as return value}}
+#else
+  // with default abi they are fine
+  (void)__builtin_wasm_test_function_pointer_signature((F3)0);
+  (void)__builtin_wasm_test_function_pointer_signature((F4)0);
+#endif
 }
diff --git a/clang/test/Sema/c2x-nodiscard.c b/clang/test/Sema/c2x-nodiscard.c
index e2537bcf1d29d..852c74721693b 100644
--- a/clang/test/Sema/c2x-nodiscard.c
+++ b/clang/test/Sema/c2x-nodiscard.c
@@ -41,6 +41,10 @@ void f2(void) {
   (void)get_s3();
   (void)get_i();
   (void)get_e();
+
+  One; // expected-warning {{expression result unused}}
+  (enum E2)(0); // expected-warning {{expression result unused}}
+  (struct S4){1}; // expected-warning {{expression result unused}}
 }
 
 struct [[nodiscard]] error_info{
@@ -60,3 +64,16 @@ void GH104391() {
 #define M (unsigned int) f3()
   M; // expected-warning {{ignoring return value of function declared with 'nodiscard' attribute}}
 }
+
+[[nodiscard]] typedef int NoDInt; // expected-warning {{'[[nodiscard]]' attribute ignored when applied to a typedef}}
+typedef __attribute__((warn_unused)) int WUInt; // expected-warning {{'warn_unused' attribute only applies to structs, unions, and classes}}
+typedef __attribute__((warn_unused_result)) int WURInt;
+NoDInt get_nodint();
+WUInt get_wuint();
+WURInt get_wurint();
+
+void f4(void) {
+  get_nodint(); // no warning because attribute is ignored
+  get_wuint();  // no warning because attribute is ignored
+  get_wurint(); // expected-warning {{ignoring return value of type 'WURInt' declared with 'warn_unused_result' attribute}}
+}
diff --git a/clang/test/Sema/constant-builtins-vector.cpp b/clang/test/Sema/constant-builtins-vector.cpp
index bde5c478b2b6f..17fa95871740d 100644
--- a/clang/test/Sema/constant-builtins-vector.cpp
+++ b/clang/test/Sema/constant-builtins-vector.cpp
@@ -860,3 +860,104 @@ static_assert(__builtin_elementwise_sub_sat(0U, 1U) == 0U);
 static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_sub_sat((vector4char){5, 4, 3, 2}, (vector4char){1, 1, 1, 1})) == (LITTLE_END ? 0x01020304 : 0x04030201));
 static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_sub_sat((vector4uchar){5, 4, 3, 2}, (vector4uchar){1, 1, 1, 1})) == (LITTLE_END ? 0x01020304U : 0x04030201U));
 static_assert(__builtin_bit_cast(unsigned long long, __builtin_elementwise_sub_sat((vector4short){(short)0x8000, (short)0x8001, (short)0x8002, (short)0x8003}, (vector4short){7, 8, 9, 10}) == (LITTLE_END ? 0x8000800080008000 : 0x8000800080008000)));
+
+static_assert(__builtin_elementwise_max(1, 2) == 2);
+static_assert(__builtin_elementwise_max(-1, 1) == 1);
+static_assert(__builtin_elementwise_max(1U, 2U) == 2U);
+static_assert(__builtin_elementwise_max(~0U, 0U) == ~0U);
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_max((vector4char){1, -2, 3, -4}, (vector4char){4, -3, 2, -1})) == (LITTLE_END ? 0xFF03FE04 : 0x04FE03FF ));
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_max((vector4uchar){1, 2, 3, 4}, (vector4uchar){4, 3, 2, 1})) == 0x04030304U);
+static_assert(__builtin_bit_cast(unsigned long long, __builtin_elementwise_max((vector4short){1, -2, 3, -4}, (vector4short){4, -3, 2, -1})) == (LITTLE_END ? 0xFFFF0003FFFE0004 : 0x0004FFFE0003FFFF));
+
+static_assert(__builtin_elementwise_min(1, 2) == 1);
+static_assert(__builtin_elementwise_min(-1, 1) == -1);
+static_assert(__builtin_elementwise_min(1U, 2U) == 1U);
+static_assert(__builtin_elementwise_min(~0U, 0U) == 0U);
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_min((vector4char){1, -2, 3, -4}, (vector4char){4, -3, 2, -1})) == (LITTLE_END ? 0xFC02FD01 : 0x01FD02FC));
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_min((vector4uchar){1, 2, 3, 4}, (vector4uchar){4, 3, 2, 1})) == 0x01020201U);
+static_assert(__builtin_bit_cast(unsigned long long, __builtin_elementwise_min((vector4short){1, -2, 3, -4}, (vector4short){4, -3, 2, -1})) == (LITTLE_END ? 0xFFFC0002FFFD0001 : 0x0001FFFD0002FFFC));
+
+static_assert(__builtin_elementwise_abs(10) == 10);
+static_assert(__builtin_elementwise_abs(-10) == 10);
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_abs((vector4char){-1, -2, -3, 4})) == (LITTLE_END ? 0x04030201 : 0x01020304));
+static_assert(__builtin_elementwise_abs((int)(-2147483648)) == (int)(-2147483648)); // the absolute value of the most negative integer remains the most negative integer
+
+// check floating point for elementwise abs
+#define CHECK_FOUR_FLOAT_VEC(vec1, vec2) \
+    static_assert(__builtin_fabs(vec1[0] - vec2[0]) < 1e-6); \
+    static_assert(__builtin_fabs(vec1[1] - vec2[1]) < 1e-6); \
+    static_assert(__builtin_fabs(vec1[2] - vec2[2]) < 1e-6); \
+    static_assert(__builtin_fabs(vec1[3] - vec2[3]) < 1e-6);
+
+// checking floating point vector
+CHECK_FOUR_FLOAT_VEC(__builtin_elementwise_abs((vector4float){-1.123, 2.123, -3.123, 4.123}), ((vector4float){1.123, 2.123, 3.123, 4.123}))
+CHECK_FOUR_FLOAT_VEC(__builtin_elementwise_abs((vector4double){-1.123, 2.123, -3.123, 4.123}), ((vector4double){1.123, 2.123, 3.123, 4.123}))
+static_assert(__builtin_elementwise_abs((float)-1.123) - (float)1.123 < 1e-6); // making sure one element works
+#undef CHECK_FOUR_FLOAT_VEC
+
+static_assert(__builtin_elementwise_ctlz(2) == 30);
+static_assert(__builtin_elementwise_ctlz(2, 8) == 30);
+static_assert(__builtin_elementwise_ctlz(0, 8) == 8);
+static_assert(__builtin_elementwise_ctlz(0, 0) == 0);
+static_assert(__builtin_elementwise_ctlz((char)2) == 6);
+static_assert(__builtin_elementwise_ctlz((short)2) == 14);
+static_assert(__builtin_elementwise_ctlz((char)1) == 0x7);
+static_assert(__builtin_elementwise_ctlz((char)4) == 0x5);
+static_assert(__builtin_elementwise_ctlz((char)127) == 0x1);
+static_assert(__builtin_elementwise_ctlz((char)128) == 0x0);
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_ctlz((vector4char){1, 4, 127, (char)128})) == (LITTLE_END ? 0x00010507 : 0x07050100));
+
+constexpr int clz0 = __builtin_elementwise_ctlz(0);
+// expected-error@-1 {{must be initialized by a constant expression}} \
+// expected-note@-1 {{evaluation of __builtin_elementwise_ctlz with a zero value is undefined}}
+constexpr vector4char clz1 = __builtin_elementwise_ctlz((vector4char){1, 0, 3, 4});
+// expected-error@-1 {{must be initialized by a constant expression}} \
+// expected-note@-1 {{evaluation of __builtin_elementwise_ctlz with a zero value is undefined}}
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_ctlz((vector4char){1, 0, 127, 0}, (vector4char){9, -1, 9, -2})) == (LITTLE_END ? 0xFE01FF07 : 0x07FF01FE));
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_ctlz((vector4char){0, 0, 0, 0}, (vector4char){0, 0, 0, 0})) == 0);
+
+static_assert(__builtin_elementwise_cttz(2) == 1);
+static_assert(__builtin_elementwise_cttz(2, 8) == 1);
+static_assert(__builtin_elementwise_cttz(0, 8) == 8);
+static_assert(__builtin_elementwise_cttz(0, 0) == 0);
+static_assert(__builtin_elementwise_cttz((char)2) == 1);
+static_assert(__builtin_elementwise_cttz((short)2) == 1);
+static_assert(__builtin_elementwise_cttz((char)8) == 0x3);
+static_assert(__builtin_elementwise_cttz((char)32) == 0x5);
+static_assert(__builtin_elementwise_cttz((char)127) == 0x0);
+static_assert(__builtin_elementwise_cttz((char)128) == 0x7);
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_cttz((vector4char){8, 32, 127, (char)128})) == (LITTLE_END ? 0x07000503 : 0x03050007));
+
+constexpr int ctz0 = __builtin_elementwise_cttz(0);
+// expected-error@-1 {{must be initialized by a constant expression}} \
+// expected-note@-1 {{evaluation of __builtin_elementwise_cttz with a zero value is undefined}}
+constexpr vector4char ctz1 = __builtin_elementwise_cttz((vector4char){1, 0, 3, 4});
+// expected-error@-1 {{must be initialized by a constant expression}} \
+// expected-note@-1 {{evaluation of __builtin_elementwise_cttz with a zero value is undefined}}
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_cttz((vector4char){8, 0, 127, 0}, (vector4char){9, -1, 9, -2})) == (LITTLE_END ? 0xFE00FF03 : 0x03FF00FE));
+static_assert(__builtin_bit_cast(unsigned, __builtin_elementwise_cttz((vector4char){0, 0, 0, 0}, (vector4char){0, 0, 0, 0})) == 0);
+
+// Non-vector floating point types.
+static_assert(__builtin_elementwise_fma(2.0, 3.0, 4.0) == 10.0);
+static_assert(__builtin_elementwise_fma(200.0, 300.0, 400.0) == 60400.0);
+// Vector type.
+constexpr vector4float fmaFloat1 =
+  __builtin_elementwise_fma((vector4float){1.0, 2.0, 3.0, 4.0},
+                            (vector4float){2.0, 3.0, 4.0, 5.0},
+                            (vector4float){3.0, 4.0, 5.0, 6.0});
+static_assert(fmaFloat1[0] == 5.0);
+static_assert(fmaFloat1[1] == 10.0);
+static_assert(fmaFloat1[2] == 17.0);
+static_assert(fmaFloat1[3] == 26.0);
+constexpr vector4double fmaDouble1 =
+  __builtin_elementwise_fma((vector4double){1.0, 2.0, 3.0, 4.0},
+                            (vector4double){2.0, 3.0, 4.0, 5.0},
+                            (vector4double){3.0, 4.0, 5.0, 6.0});
+static_assert(fmaDouble1[0] == 5.0);
+static_assert(fmaDouble1[1] == 10.0);
+static_assert(fmaDouble1[2] == 17.0);
+static_assert(fmaDouble1[3] == 26.0);
+
+constexpr float fmaArray[] = {2.0f, 2.0f, 2.0f, 2.0f};
+constexpr float fmaResult = __builtin_elementwise_fma(fmaArray[1], fmaArray[2], fmaArray[3]);
+static_assert(fmaResult == 6.0f, "");
diff --git a/clang/test/Sema/designated-initializers.c b/clang/test/Sema/designated-initializers.c
index 31a3380b5db7d..11dc3a2308dee 100644
--- a/clang/test/Sema/designated-initializers.c
+++ b/clang/test/Sema/designated-initializers.c
@@ -368,3 +368,10 @@ struct {
     .b = 0, // expected-warning {{initializer overrides prior initialization of this subobject}}
   },
 };
+
+void gh154046(void) {
+  (void)(const char[]) {
+    [0] = "", // expected-error {{incompatible pointer to integer conversion initializing 'const char' with an expression of type 'char[1]'}}
+    [1] = ""  // expected-error {{incompatible pointer to integer conversion initializing 'const char' with an expression of type 'char[1]'}}
+  }[1];
+}
diff --git a/clang/test/Sema/format-strings-signedness.c b/clang/test/Sema/format-strings-signedness.c
index d5a8140d9ef8a..773ff412ac177 100644
--- a/clang/test/Sema/format-strings-signedness.c
+++ b/clang/test/Sema/format-strings-signedness.c
@@ -39,13 +39,13 @@ void test_printf_unsigned_char(unsigned char x)
 void test_printf_int(int x)
 {
     printf("%d", x); // no-warning
-    printf("%u", x); // expected-warning{{format specifies type 'unsigned int' but the argument has type 'int'}}
-    printf("%x", x); // expected-warning{{format specifies type 'unsigned int' but the argument has type 'int'}}
+    printf("%u", x); // expected-warning{{format specifies type 'unsigned int' but the argument has type 'int', which differs in signedness}}
+    printf("%x", x); // expected-warning{{format specifies type 'unsigned int' but the argument has type 'int', which differs in signedness}}
 }
 
 void test_printf_unsigned(unsigned x)
 {
-    printf("%d", x); // expected-warning{{format specifies type 'int' but the argument has type 'unsigned int'}}
+    printf("%d", x); // expected-warning{{format specifies type 'int' but the argument has type 'unsigned int', which differs in signedness}}
     printf("%u", x); // no-warning
     printf("%x", x); // no-warning
 }
@@ -53,13 +53,13 @@ void test_printf_unsigned(unsigned x)
 void test_printf_long(long x)
 {
     printf("%ld", x); // no-warning
-    printf("%lu", x); // expected-warning{{format specifies type 'unsigned long' but the argument has type 'long'}}
-    printf("%lx", x); // expected-warning{{format specifies type 'unsigned long' but the argument has type 'long'}}
+    printf("%lu", x); // expected-warning{{format specifies type 'unsigned long' but the argument has type 'long', which differs in signedness}}
+    printf("%lx", x); // expected-warning{{format specifies type 'unsigned long' but the argument has type 'long', which differs in signedness}}
 }
 
 void test_printf_unsigned_long(unsigned long x)
 {
-    printf("%ld", x); // expected-warning{{format specifies type 'long' but the argument has type 'unsigned long'}}
+    printf("%ld", x); // expected-warning{{format specifies type 'long' but the argument has type 'unsigned long', which differs in signedness}}
     printf("%lu", x); // no-warning
     printf("%lx", x); // no-warning
 }
@@ -67,13 +67,13 @@ void test_printf_unsigned_long(unsigned long x)
 void test_printf_long_long(long long x)
 {
     printf("%lld", x); // no-warning
-    printf("%llu", x); // expected-warning{{format specifies type 'unsigned long long' but the argument has type 'long long'}}
-    printf("%llx", x); // expected-warning{{format specifies type 'unsigned long long' but the argument has type 'long long'}}
+    printf("%llu", x); // expected-warning{{format specifies type 'unsigned long long' but the argument has type 'long long', which differs in signedness}}
+    printf("%llx", x); // expected-warning{{format specifies type 'unsigned long long' but the argument has type 'long long', which differs in signedness}}
 }
 
 void test_printf_unsigned_long_long(unsigned long long x)
 {
-    printf("%lld", x); // expected-warning{{format specifies type 'long long' but the argument has type 'unsigned long long'}}
+    printf("%lld", x); // expected-warning{{format specifies type 'long long' but the argument has type 'unsigned long long', which differs in signedness}}
     printf("%llu", x); // no-warning
     printf("%llx", x); // no-warning
 }
@@ -85,8 +85,8 @@ enum enum_int {
 void test_printf_enum_int(enum enum_int x)
 {
     printf("%d", x); // no-warning
-    printf("%u", x); // expected-warning{{format specifies type 'unsigned int' but the argument has underlying type 'int'}}
-    printf("%x", x); // expected-warning{{format specifies type 'unsigned int' but the argument has underlying type 'int'}}
+    printf("%u", x); // expected-warning{{format specifies type 'unsigned int' but the argument has underlying type 'int', which differs in signedness}}
+    printf("%x", x); // expected-warning{{format specifies type 'unsigned int' but the argument has underlying type 'int', which differs in signedness}}
 }
 
 #ifndef _WIN32 // Disabled due to enums have different underlying type on _WIN32
@@ -96,7 +96,7 @@ enum enum_unsigned {
 
 void test_printf_enum_unsigned(enum enum_unsigned x)
 {
-    printf("%d", x); // expected-warning{{format specifies type 'int' but the argument has underlying type 'unsigned int'}}
+    printf("%d", x); // expected-warning{{format specifies type 'int' but the argument has underlying type 'unsigned int', which differs in signedness}}
     printf("%u", x); // no-warning
     printf("%x", x); // no-warning
 }
@@ -110,8 +110,8 @@ enum enum_long {
 void test_printf_enum_long(enum enum_long x)
 {
     printf("%ld", x); // no-warning
-    printf("%lu", x); // expected-warning{{format specifies type 'unsigned long' but the argument has underlying type 'long'}}
-    printf("%lx", x); // expected-warning{{format specifies type 'unsigned long' but the argument has underlying type 'long'}}
+    printf("%lu", x); // expected-warning{{format specifies type 'unsigned long' but the argument has underlying type 'long', which differs in signedness}}
+    printf("%lx", x); // expected-warning{{format specifies type 'unsigned long' but the argument has underlying type 'long', which differs in signedness}}
 }
 
 enum enum_unsigned_long {
@@ -120,7 +120,7 @@ enum enum_unsigned_long {
 
 void test_printf_enum_unsigned_long(enum enum_unsigned_long x)
 {
-    printf("%ld", x); // expected-warning{{format specifies type 'long' but the argument has underlying type 'unsigned long'}}
+    printf("%ld", x); // expected-warning{{format specifies type 'long' but the argument has underlying type 'unsigned long', which differs in signedness}}
     printf("%lu", x); // no-warning
     printf("%lx", x); // no-warning
 }
@@ -136,61 +136,61 @@ void test_scanf_unsigned_char(unsigned char *y) {
 
 void test_scanf_int(int *x) {
   scanf("%d", x); // no-warning
-  scanf("%u", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'int *'}}
-  scanf("%x", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'int *'}}
+  scanf("%u", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'int *', which differs in signedness}}
+  scanf("%x", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'int *', which differs in signedness}}
 }
 
 void test_scanf_unsigned(unsigned *x) {
-  scanf("%d", x); // expected-warning{{format specifies type 'int *' but the argument has type 'unsigned int *'}}
+  scanf("%d", x); // expected-warning{{format specifies type 'int *' but the argument has type 'unsigned int *', which differs in signedness}}
   scanf("%u", x); // no-warning
   scanf("%x", x); // no-warning
 }
 
 void test_scanf_long(long *x) {
   scanf("%ld", x); // no-warning
-  scanf("%lu", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'long *'}}
-  scanf("%lx", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'long *'}}
+  scanf("%lu", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'long *', which differs in signedness}}
+  scanf("%lx", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'long *', which differs in signedness}}
 }
 
 void test_scanf_unsigned_long(unsigned long *x) {
-  scanf("%ld", x); // expected-warning{{format specifies type 'long *' but the argument has type 'unsigned long *'}}
+  scanf("%ld", x); // expected-warning{{format specifies type 'long *' but the argument has type 'unsigned long *', which differs in signedness}}
   scanf("%lu", x); // no-warning
   scanf("%lx", x); // no-warning
 }
 
 void test_scanf_longlong(long long *x) {
   scanf("%lld", x); // no-warning
-  scanf("%llu", x); // expected-warning{{format specifies type 'unsigned long long *' but the argument has type 'long long *'}}
-  scanf("%llx", x); // expected-warning{{format specifies type 'unsigned long long *' but the argument has type 'long long *'}}
+  scanf("%llu", x); // expected-warning{{format specifies type 'unsigned long long *' but the argument has type 'long long *', which differs in signedness}}
+  scanf("%llx", x); // expected-warning{{format specifies type 'unsigned long long *' but the argument has type 'long long *', which differs in signedness}}
 }
 
 void test_scanf_unsigned_longlong(unsigned long long *x) {
-  scanf("%lld", x); // expected-warning{{format specifies type 'long long *' but the argument has type 'unsigned long long *'}}
+  scanf("%lld", x); // expected-warning{{format specifies type 'long long *' but the argument has type 'unsigned long long *', which differs in signedness}}
   scanf("%llu", x); // no-warning
   scanf("%llx", x); // no-warning
 }
 
 void test_scanf_enum_int(enum enum_int *x) {
   scanf("%d", x); // no-warning
-  scanf("%u", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'enum enum_int *'}}
-  scanf("%x", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'enum enum_int *'}}
+  scanf("%u", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'enum enum_int *', which differs in signedness}}
+  scanf("%x", x); // expected-warning{{format specifies type 'unsigned int *' but the argument has type 'enum enum_int *', which differs in signedness}}
 }
 
 #ifndef _WIN32 // Disabled due to enums have different underlying type on _WIN32
 void test_scanf_enum_unsigned(enum enum_unsigned *x) {
-  scanf("%d", x); // expected-warning{{format specifies type 'int *' but the argument has type 'enum enum_unsigned *'}}
+  scanf("%d", x); // expected-warning{{format specifies type 'int *' but the argument has type 'enum enum_unsigned *', which differs in signedness}}
   scanf("%u", x); // no-warning
   scanf("%x", x); // no-warning
 }
 
 void test_scanf_enum_long(enum enum_long *x) {
   scanf("%ld", x); // no-warning
-  scanf("%lu", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'enum enum_long *'}}
-  scanf("%lx", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'enum enum_long *'}}
+  scanf("%lu", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'enum enum_long *', which differs in signedness}}
+  scanf("%lx", x); // expected-warning{{format specifies type 'unsigned long *' but the argument has type 'enum enum_long *', which differs in signedness}}
 }
 
 void test_scanf_enum_unsigned_long(enum enum_unsigned_long *x) {
-  scanf("%ld", x); // expected-warning{{format specifies type 'long *' but the argument has type 'enum enum_unsigned_long *'}}
+  scanf("%ld", x); // expected-warning{{format specifies type 'long *' but the argument has type 'enum enum_unsigned_long *', which differs in signedness}}
   scanf("%lu", x); // no-warning
   scanf("%lx", x); // no-warning
 }
diff --git a/clang/test/Sema/implicit-void-ptr-cast.c b/clang/test/Sema/implicit-void-ptr-cast.c
index 3c3e153d1dbda..f083cfb77329f 100644
--- a/clang/test/Sema/implicit-void-ptr-cast.c
+++ b/clang/test/Sema/implicit-void-ptr-cast.c
@@ -82,3 +82,15 @@ void more(void) {
   ptr3 = SOMETHING_THAT_IS_NOT_NULL; // c-warning {{implicit conversion when assigning to 'char *' from type 'void *' is not permitted in C++}} \
                                         cxx-error {{assigning to 'char *' from incompatible type 'void *'}}
 }
+
+void gh154157(void) {
+  #define ATOMIC_VAR_INIT(value) (value)
+
+  typedef const struct T * T_Ref;
+  static T_Ref _Atomic x = ATOMIC_VAR_INIT((void*)NULL); // c-warning {{implicit conversion when initializing '_Atomic(T_Ref)' with an expression of type 'void *' is not permitted in C++}} \
+                                                            cxx-error {{cannot initialize a variable of type '_Atomic(T_Ref)' with an rvalue of type 'void *'}}
+  static T_Ref const y = ATOMIC_VAR_INIT((void*)NULL);   // c-warning {{implicit conversion when initializing 'const T_Ref' (aka 'const struct T *const') with an expression of type 'void *' is not permitted in C++}} \
+                                                            cxx-error {{cannot initialize a variable of type 'const T_Ref' (aka 'const struct T *const') with an rvalue of type 'void *'}}
+  static T_Ref z = ATOMIC_VAR_INIT((void*)NULL);         // c-warning {{implicit conversion when initializing 'T_Ref' (aka 'const struct T *') with an expression of type 'void *' is not permitted in C++}} \
+                                                            cxx-error {{cannot initialize a variable of type 'T_Ref' (aka 'const struct T *') with an rvalue of type 'void *'}}
+}
diff --git a/clang/test/Sema/ptrauth-qualifier.c b/clang/test/Sema/ptrauth-qualifier.c
index 5d932b724f07a..3e568ce9f37e3 100644
--- a/clang/test/Sema/ptrauth-qualifier.c
+++ b/clang/test/Sema/ptrauth-qualifier.c
@@ -1,13 +1,25 @@
-// RUN: %clang_cc1 -triple arm64-apple-ios -std=c23 -fsyntax-only -verify -fptrauth-intrinsics %s
+// RUN: %clang_cc1 -triple arm64-apple-ios -DIS_DARWIN -std=c23 -fsyntax-only -verify -fptrauth-intrinsics %s
 // RUN: %clang_cc1 -triple aarch64-linux-gnu -std=c23 -fsyntax-only -verify -fptrauth-intrinsics %s
 
-#if !__has_extension(ptrauth_qualifier)
+#if defined(IS_DARWIN) && !__has_extension(ptrauth_qualifier)
 // This error means that the __ptrauth qualifier availability test says  that it
 // is not available. This error is not expected in the output, if it is seen
 // there is a feature detection regression.
 #error __ptrauth qualifier not enabled
 #endif
 
+#if defined(IS_DARWIN) && !__has_feature(ptrauth_qualifier)
+// This error means that the __has_feature test for ptrauth_qualifier has
+// failed, despite it being expected on darwin.
+#error __ptrauth qualifier not enabled
+#elif !defined(IS_DARWIN) && (__has_feature(ptrauth_qualifier) || __has_extension(ptrauth_qualifier))
+#error ptrauth_qualifier labeled a feature on a non-darwin platform
+#endif
+
+#if !defined (__PTRAUTH__)
+#error __PTRAUTH__ test macro not defined when ptrauth is enabled
+#endif
+
 #if __aarch64__
 #define VALID_CODE_KEY 0
 #define VALID_DATA_KEY 2
diff --git a/clang/test/Sema/warn-lifetime-safety-dataflow.cpp b/clang/test/Sema/warn-lifetime-safety-dataflow.cpp
index 2b934ac23b92d..bcde9adf25ca5 100644
--- a/clang/test/Sema/warn-lifetime-safety-dataflow.cpp
+++ b/clang/test/Sema/warn-lifetime-safety-dataflow.cpp
@@ -12,11 +12,11 @@ MyObj* return_local_addr() {
   MyObj x {10};
   MyObj* p = &x;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_X:[0-9]+]], OriginID: [[O_ADDR_X:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_ADDR_X]])
+// CHECK:   Issue (LoanID: [[L_X:[0-9]+]], ToOrigin: [[O_ADDR_X:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_ADDR_X]] (Expr: UnaryOperator))
   return p;
-// CHECK:   AssignOrigin (DestID: [[O_RET_VAL:[0-9]+]], SrcID: [[O_P]])
-// CHECK:   ReturnOfOrigin (OriginID: [[O_RET_VAL]])
+// CHECK:   AssignOrigin (Dest: [[O_RET_VAL:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_P]] (Decl: p))
+// CHECK:   ReturnOfOrigin ([[O_RET_VAL]] (Expr: ImplicitCastExpr))
 // CHECK:   Expire (LoanID: [[L_X]])
 }
 
@@ -27,20 +27,20 @@ MyObj* return_local_addr() {
 MyObj* assign_and_return_local_addr() {
   MyObj y{20};
   MyObj* ptr1 = &y;
-// CHECK: Issue (LoanID: [[L_Y:[0-9]+]], OriginID: [[O_ADDR_Y:[0-9]+]])
-// CHECK: AssignOrigin (DestID: [[O_PTR1:[0-9]+]], SrcID: [[O_ADDR_Y]])
+// CHECK: Issue (LoanID: [[L_Y:[0-9]+]], ToOrigin: [[O_ADDR_Y:[0-9]+]] (Expr: UnaryOperator))
+// CHECK: AssignOrigin (Dest: [[O_PTR1:[0-9]+]] (Decl: ptr1), Src: [[O_ADDR_Y]] (Expr: UnaryOperator))
   MyObj* ptr2 = ptr1;
-// CHECK: AssignOrigin (DestID: [[O_PTR1_RVAL:[0-9]+]], SrcID: [[O_PTR1]])
-// CHECK: AssignOrigin (DestID: [[O_PTR2:[0-9]+]], SrcID: [[O_PTR1_RVAL]])
+// CHECK: AssignOrigin (Dest: [[O_PTR1_RVAL:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_PTR1]] (Decl: ptr1))
+// CHECK: AssignOrigin (Dest: [[O_PTR2:[0-9]+]] (Decl: ptr2), Src: [[O_PTR1_RVAL]] (Expr: ImplicitCastExpr))
   ptr2 = ptr1;
-// CHECK: AssignOrigin (DestID: [[O_PTR1_RVAL_2:[0-9]+]], SrcID: [[O_PTR1]])
-// CHECK: AssignOrigin (DestID: [[O_PTR2]], SrcID: [[O_PTR1_RVAL_2]])
+// CHECK: AssignOrigin (Dest: [[O_PTR1_RVAL_2:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_PTR1]] (Decl: ptr1))
+// CHECK: AssignOrigin (Dest: [[O_PTR2]] (Decl: ptr2), Src: [[O_PTR1_RVAL_2]] (Expr: ImplicitCastExpr))
   ptr2 = ptr2; // Self assignment.
-// CHECK: AssignOrigin (DestID: [[O_PTR2_RVAL:[0-9]+]], SrcID: [[O_PTR2]])
-// CHECK: AssignOrigin (DestID: [[O_PTR2]], SrcID: [[O_PTR2_RVAL]])
+// CHECK: AssignOrigin (Dest: [[O_PTR2_RVAL:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_PTR2]] (Decl: ptr2))
+// CHECK: AssignOrigin (Dest: [[O_PTR2]] (Decl: ptr2), Src: [[O_PTR2_RVAL]] (Expr: ImplicitCastExpr))
   return ptr2;
-// CHECK: AssignOrigin (DestID: [[O_PTR2_RVAL_2:[0-9]+]], SrcID: [[O_PTR2]])
-// CHECK: ReturnOfOrigin (OriginID: [[O_PTR2_RVAL_2]])
+// CHECK: AssignOrigin (Dest: [[O_PTR2_RVAL_2:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_PTR2]] (Decl: ptr2))
+// CHECK: ReturnOfOrigin ([[O_PTR2_RVAL_2]] (Expr: ImplicitCastExpr))
 // CHECK: Expire (LoanID: [[L_Y]])
 }
 
@@ -60,8 +60,8 @@ int return_int_val() {
 void loan_expires_cpp() {
   MyObj obj{1};
   MyObj* pObj = &obj;
-// CHECK: Issue (LoanID: [[L_OBJ:[0-9]+]], OriginID: [[O_ADDR_OBJ:[0-9]+]])
-// CHECK: AssignOrigin (DestID: [[O_POBJ:[0-9]+]], SrcID: [[O_ADDR_OBJ]])
+// CHECK: Issue (LoanID: [[L_OBJ:[0-9]+]], ToOrigin: [[O_ADDR_OBJ:[0-9]+]] (Expr: UnaryOperator))
+// CHECK: AssignOrigin (Dest: [[O_POBJ:[0-9]+]] (Decl: pObj), Src: [[O_ADDR_OBJ]] (Expr: UnaryOperator))
 // CHECK: Expire (LoanID: [[L_OBJ]])
 }
 
@@ -72,8 +72,8 @@ void loan_expires_cpp() {
 void loan_expires_trivial() {
   int trivial_obj = 1;
   int* pTrivialObj = &trivial_obj;
-// CHECK: Issue (LoanID: [[L_TRIVIAL_OBJ:[0-9]+]], OriginID: [[O_ADDR_TRIVIAL_OBJ:[0-9]+]])
-// CHECK: AssignOrigin (DestID: [[O_PTOBJ:[0-9]+]], SrcID: [[O_ADDR_TRIVIAL_OBJ]])
+// CHECK: Issue (LoanID: [[L_TRIVIAL_OBJ:[0-9]+]], ToOrigin: [[O_ADDR_TRIVIAL_OBJ:[0-9]+]] (Expr: UnaryOperator))
+// CHECK: AssignOrigin (Dest: [[O_PTOBJ:[0-9]+]] (Decl: pTrivialObj), Src: [[O_ADDR_TRIVIAL_OBJ]] (Expr: UnaryOperator))
 // CHECK-NOT: Expire (LoanID: [[L_TRIVIAL_OBJ]])
 // CHECK-NEXT: End of Block
   // FIXME: Add check for Expire once trivial destructors are handled for expiration.
@@ -87,15 +87,15 @@ void conditional(bool condition) {
 
   if (condition)
     p = &a;
-  // CHECK: Issue (LoanID: [[L_A:[0-9]+]], OriginID: [[O_ADDR_A:[0-9]+]])
-  // CHECK: AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_ADDR_A]])
+// CHECK: Issue (LoanID: [[L_A:[0-9]+]], ToOrigin: [[O_ADDR_A:[0-9]+]] (Expr: UnaryOperator))
+// CHECK: AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_ADDR_A]] (Expr: UnaryOperator))
   else
     p = &b;
-  // CHECK: Issue (LoanID: [[L_B:[0-9]+]], OriginID: [[O_ADDR_B:[0-9]+]])
-  // CHECK: AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_B]])
+// CHECK: Issue (LoanID: [[L_B:[0-9]+]], ToOrigin: [[O_ADDR_B:[0-9]+]] (Expr: UnaryOperator))
+// CHECK: AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_B]] (Expr: UnaryOperator))
   int *q = p;
-  // CHECK: AssignOrigin (DestID: [[O_P_RVAL:[0-9]+]], SrcID: [[O_P]])
-  // CHECK: AssignOrigin (DestID: [[O_Q:[0-9]+]], SrcID: [[O_P_RVAL]])
+// CHECK: AssignOrigin (Dest: [[O_P_RVAL:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_P]] (Decl: p))
+// CHECK: AssignOrigin (Dest: [[O_Q:[0-9]+]] (Decl: q), Src: [[O_P_RVAL]] (Expr: ImplicitCastExpr))
 }
 
 
@@ -109,12 +109,12 @@ void pointers_in_a_cycle(bool condition) {
   MyObj* p2 = &v2;
   MyObj* p3 = &v3;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_V1:[0-9]+]], OriginID: [[O_ADDR_V1:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P1:[0-9]+]], SrcID: [[O_ADDR_V1]])
-// CHECK:   Issue (LoanID: [[L_V2:[0-9]+]], OriginID: [[O_ADDR_V2:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P2:[0-9]+]], SrcID: [[O_ADDR_V2]])
-// CHECK:   Issue (LoanID: [[L_V3:[0-9]+]], OriginID: [[O_ADDR_V3:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P3:[0-9]+]], SrcID: [[O_ADDR_V3]])
+// CHECK:   Issue (LoanID: [[L_V1:[0-9]+]], ToOrigin: [[O_ADDR_V1:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P1:[0-9]+]] (Decl: p1), Src: [[O_ADDR_V1]] (Expr: UnaryOperator))
+// CHECK:   Issue (LoanID: [[L_V2:[0-9]+]], ToOrigin: [[O_ADDR_V2:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P2:[0-9]+]] (Decl: p2), Src: [[O_ADDR_V2]] (Expr: UnaryOperator))
+// CHECK:   Issue (LoanID: [[L_V3:[0-9]+]], ToOrigin: [[O_ADDR_V3:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P3:[0-9]+]] (Decl: p3), Src: [[O_ADDR_V3]] (Expr: UnaryOperator))
 
   while (condition) {
     MyObj* temp = p1;
@@ -122,14 +122,14 @@ void pointers_in_a_cycle(bool condition) {
     p2 = p3;
     p3 = temp;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   AssignOrigin (DestID: [[O_P1_RVAL:[0-9]+]], SrcID: [[O_P1]])
-// CHECK:   AssignOrigin (DestID: [[O_TEMP:[0-9]+]], SrcID: [[O_P1_RVAL]])
-// CHECK:   AssignOrigin (DestID: [[O_P2_RVAL:[0-9]+]], SrcID: [[O_P2]])
-// CHECK:   AssignOrigin (DestID: [[O_P1]], SrcID: [[O_P2_RVAL]])
-// CHECK:   AssignOrigin (DestID: [[O_P3_RVAL:[0-9]+]], SrcID: [[O_P3]])
-// CHECK:   AssignOrigin (DestID: [[O_P2]], SrcID: [[O_P3_RVAL]])
-// CHECK:   AssignOrigin (DestID: [[O_TEMP_RVAL:[0-9]+]], SrcID: [[O_TEMP]])
-// CHECK:   AssignOrigin (DestID: [[O_P3]], SrcID: [[O_TEMP_RVAL]])
+// CHECK:   AssignOrigin (Dest: [[O_P1_RVAL:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_P1]] (Decl: p1))
+// CHECK:   AssignOrigin (Dest: [[O_TEMP:[0-9]+]] (Decl: temp), Src: [[O_P1_RVAL]] (Expr: ImplicitCastExpr))
+// CHECK:   AssignOrigin (Dest: [[O_P2_RVAL:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_P2]] (Decl: p2))
+// CHECK:   AssignOrigin (Dest: [[O_P1]] (Decl: p1), Src: [[O_P2_RVAL]] (Expr: ImplicitCastExpr))
+// CHECK:   AssignOrigin (Dest: [[O_P3_RVAL:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_P3]] (Decl: p3))
+// CHECK:   AssignOrigin (Dest: [[O_P2]] (Decl: p2), Src: [[O_P3_RVAL]] (Expr: ImplicitCastExpr))
+// CHECK:   AssignOrigin (Dest: [[O_TEMP_RVAL:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_TEMP]] (Decl: temp))
+// CHECK:   AssignOrigin (Dest: [[O_P3]] (Decl: p3), Src: [[O_TEMP_RVAL]] (Expr: ImplicitCastExpr))
   }
 }
 
@@ -139,11 +139,11 @@ void overwrite_origin() {
   MyObj s2;
   MyObj* p = &s1;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], OriginID: [[O_ADDR_S1:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_ADDR_S1]])
+// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], ToOrigin: [[O_ADDR_S1:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_ADDR_S1]] (Expr: UnaryOperator))
   p = &s2;
-// CHECK:   Issue (LoanID: [[L_S2:[0-9]+]], OriginID: [[O_ADDR_S2:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_S2]])
+// CHECK:   Issue (LoanID: [[L_S2:[0-9]+]], ToOrigin: [[O_ADDR_S2:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_S2]] (Expr: UnaryOperator))
 // CHECK:   Expire (LoanID: [[L_S2]])
 // CHECK:   Expire (LoanID: [[L_S1]])
 }
@@ -153,10 +153,11 @@ void reassign_to_null() {
   MyObj s1;
   MyObj* p = &s1;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], OriginID: [[O_ADDR_S1:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_ADDR_S1]])
+// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], ToOrigin: [[O_ADDR_S1:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_ADDR_S1]] (Expr: UnaryOperator))
   p = nullptr;
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_NULLPTR:[0-9]+]])
+// CHECK:   AssignOrigin (Dest: [[O_NULLPTR_CAST:[0-9]+]] (Expr: ImplicitCastExpr), Src: {{[0-9]+}} (Expr: CXXNullPtrLiteralExpr))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_NULLPTR_CAST]] (Expr: ImplicitCastExpr))
 // CHECK:   Expire (LoanID: [[L_S1]])
 }
 // FIXME: Have a better representation for nullptr than just an empty origin. 
@@ -169,13 +170,13 @@ void reassign_in_if(bool condition) {
   MyObj s2;
   MyObj* p = &s1;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], OriginID: [[O_ADDR_S1:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_ADDR_S1]])
+// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], ToOrigin: [[O_ADDR_S1:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_ADDR_S1]] (Expr: UnaryOperator))
   if (condition) {
     p = &s2;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_S2:[0-9]+]], OriginID: [[O_ADDR_S2:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_S2]])
+// CHECK:   Issue (LoanID: [[L_S2:[0-9]+]], ToOrigin: [[O_ADDR_S2:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_S2]] (Expr: UnaryOperator))
   }
 // CHECK: Block B{{[0-9]+}}:
 // CHECK:   Expire (LoanID: [[L_S2]])
@@ -190,26 +191,26 @@ void assign_in_switch(int mode) {
   MyObj s3;
   MyObj* p = nullptr;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   AssignOrigin (DestID: [[O_NULLPTR_CAST:[0-9]+]], SrcID: [[O_NULLPTR:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_NULLPTR_CAST]])
+// CHECK:   AssignOrigin (Dest: [[O_NULLPTR_CAST:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_NULLPTR:[0-9]+]] (Expr: CXXNullPtrLiteralExpr))
+// CHECK:   AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_NULLPTR_CAST]] (Expr: ImplicitCastExpr))
   switch (mode) {
     case 1:
       p = &s1;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], OriginID: [[O_ADDR_S1:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_S1]])
+// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], ToOrigin: [[O_ADDR_S1:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_S1]] (Expr: UnaryOperator))
       break;
     case 2:
       p = &s2;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_S2:[0-9]+]], OriginID: [[O_ADDR_S2:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_S2]])
+// CHECK:   Issue (LoanID: [[L_S2:[0-9]+]], ToOrigin: [[O_ADDR_S2:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_S2]] (Expr: UnaryOperator))
       break;
     default:
       p = &s3;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_S3:[0-9]+]], OriginID: [[O_ADDR_S3:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_S3]])
+// CHECK:   Issue (LoanID: [[L_S3:[0-9]+]], ToOrigin: [[O_ADDR_S3:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_S3]] (Expr: UnaryOperator))
       break;
   }
 // CHECK: Block B{{[0-9]+}}:
@@ -221,14 +222,14 @@ void assign_in_switch(int mode) {
 // CHECK-LABEL: Function: loan_in_loop
 void loan_in_loop(bool condition) {
   MyObj* p = nullptr;
-  // CHECK:   AssignOrigin (DestID: [[O_NULLPTR_CAST:[0-9]+]], SrcID: [[O_NULLPTR:[0-9]+]])
-  // CHECK:   AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_NULLPTR_CAST]])
+  // CHECK:   AssignOrigin (Dest: [[O_NULLPTR_CAST:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_NULLPTR:[0-9]+]] (Expr: CXXNullPtrLiteralExpr))
+  // CHECK:   AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_NULLPTR_CAST]] (Expr: ImplicitCastExpr))
   while (condition) {
     MyObj inner;
     p = &inner;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_INNER:[0-9]+]], OriginID: [[O_ADDR_INNER:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_INNER]])
+// CHECK:   Issue (LoanID: [[L_INNER:[0-9]+]], ToOrigin: [[O_ADDR_INNER:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_INNER]] (Expr: UnaryOperator))
 // CHECK:   Expire (LoanID: [[L_INNER]])
   }
 }
@@ -239,14 +240,14 @@ void loop_with_break(int count) {
   MyObj s2;
   MyObj* p = &s1;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], OriginID: [[O_ADDR_S1:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_ADDR_S1]])
+// CHECK:   Issue (LoanID: [[L_S1:[0-9]+]], ToOrigin: [[O_ADDR_S1:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_ADDR_S1]] (Expr: UnaryOperator))
   for (int i = 0; i < count; ++i) {
     if (i == 5) {
       p = &s2;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   Issue (LoanID: [[L_S2:[0-9]+]], OriginID: [[O_ADDR_S2:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_S2]])
+// CHECK:   Issue (LoanID: [[L_S2:[0-9]+]], ToOrigin: [[O_ADDR_S2:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_S2]] (Expr: UnaryOperator))
       break;
     }
   }
@@ -259,18 +260,18 @@ void loop_with_break(int count) {
 void nested_scopes() {
   MyObj* p = nullptr;
 // CHECK: Block B{{[0-9]+}}:
-// CHECK:   AssignOrigin (DestID: [[O_NULLPTR_CAST:[0-9]+]], SrcID: [[O_NULLPTR:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_NULLPTR_CAST]])
+// CHECK:   AssignOrigin (Dest: [[O_NULLPTR_CAST:[0-9]+]] (Expr: ImplicitCastExpr), Src: [[O_NULLPTR:[0-9]+]] (Expr: CXXNullPtrLiteralExpr))
+// CHECK:   AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_NULLPTR_CAST]] (Expr: ImplicitCastExpr))
   {
     MyObj outer;
     p = &outer;
-// CHECK:   Issue (LoanID: [[L_OUTER:[0-9]+]], OriginID: [[O_ADDR_OUTER:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_OUTER]])
+// CHECK:   Issue (LoanID: [[L_OUTER:[0-9]+]], ToOrigin: [[O_ADDR_OUTER:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_OUTER]] (Expr: UnaryOperator))
     {
       MyObj inner;
       p = &inner;
-// CHECK:   Issue (LoanID: [[L_INNER:[0-9]+]], OriginID: [[O_ADDR_INNER:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P]], SrcID: [[O_ADDR_INNER]])
+// CHECK:   Issue (LoanID: [[L_INNER:[0-9]+]], ToOrigin: [[O_ADDR_INNER:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P]] (Decl: p), Src: [[O_ADDR_INNER]] (Expr: UnaryOperator))
     }
 // CHECK:   Expire (LoanID: [[L_INNER]])
   }
@@ -282,13 +283,13 @@ void pointer_indirection() {
   int a;
   int *p = &a;
 // CHECK: Block B1:
-// CHECK:   Issue (LoanID: [[L_A:[0-9]+]], OriginID: [[O_ADDR_A:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_P:[0-9]+]], SrcID: [[O_ADDR_A]])
+// CHECK:   Issue (LoanID: [[L_A:[0-9]+]], ToOrigin: [[O_ADDR_A:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_P:[0-9]+]] (Decl: p), Src: [[O_ADDR_A]] (Expr: UnaryOperator))
   int **pp = &p;
-// CHECK:   Issue (LoanID: [[L_P:[0-9]+]], OriginID: [[O_ADDR_P:[0-9]+]])
-// CHECK:   AssignOrigin (DestID: [[O_PP:[0-9]+]], SrcID: [[O_ADDR_P]])
+// CHECK:   Issue (LoanID: [[L_P:[0-g]+]], ToOrigin: [[O_ADDR_P:[0-9]+]] (Expr: UnaryOperator))
+// CHECK:   AssignOrigin (Dest: [[O_PP:[0-9]+]] (Decl: pp), Src: [[O_ADDR_P]] (Expr: UnaryOperator))
 
 // FIXME: The Origin for the RHS is broken
   int *q = *pp;
-// CHECK:   AssignOrigin (DestID: [[O_Q:[0-9]+]], SrcID: {{[0-9]+}})
+// CHECK:   AssignOrigin (Dest: {{[0-9]+}} (Decl: q), Src: {{[0-9]+}} (Expr: ImplicitCastExpr))
 }
diff --git a/clang/test/Sema/warn-lifetime-safety.cpp b/clang/test/Sema/warn-lifetime-safety.cpp
new file mode 100644
index 0000000000000..660b9c9d5e243
--- /dev/null
+++ b/clang/test/Sema/warn-lifetime-safety.cpp
@@ -0,0 +1,273 @@
+// RUN: %clang_cc1 -fsyntax-only -fexperimental-lifetime-safety -Wexperimental-lifetime-safety -verify %s
+
+struct MyObj {
+  int id;
+  ~MyObj() {}  // Non-trivial destructor
+  MyObj operator+(MyObj);
+};
+
+//===----------------------------------------------------------------------===//
+// Basic Definite Use-After-Free (-W...permissive)
+// These are cases where the pointer is guaranteed to be dangling at the use site.
+//===----------------------------------------------------------------------===//
+
+void definite_simple_case() {
+  MyObj* p;
+  {
+    MyObj s;
+    p = &s;     // expected-warning {{object whose reference is captured does not live long enough}}
+  }             // expected-note {{destroyed here}}
+  (void)*p;     // expected-note {{later used here}}
+}
+
+void no_use_no_error() {
+  MyObj* p;
+  {
+    MyObj s;
+    p = &s;
+  }
+}
+
+void definite_pointer_chain() {
+  MyObj* p;
+  MyObj* q;
+  {
+    MyObj s;
+    p = &s;     // expected-warning {{does not live long enough}}
+    q = p;
+  }             // expected-note {{destroyed here}}
+  (void)*q;     // expected-note {{later used here}}
+}
+
+void definite_multiple_uses_one_warning() {
+  MyObj* p;
+  {
+    MyObj s;
+    p = &s;     // expected-warning {{does not live long enough}}
+  }             // expected-note {{destroyed here}}
+  (void)*p;     // expected-note {{later used here}}
+  // No second warning for the same loan.
+  p->id = 1;
+  MyObj* q = p;
+  (void)*q;
+}
+
+void definite_multiple_pointers() {
+  MyObj *p, *q, *r;
+  {
+    MyObj s;
+    p = &s;     // expected-warning {{does not live long enough}}
+    q = &s;     // expected-warning {{does not live long enough}}
+    r = &s;     // expected-warning {{does not live long enough}}
+  }             // expected-note 3 {{destroyed here}}
+  (void)*p;     // expected-note {{later used here}}
+  (void)*q;     // expected-note {{later used here}}
+  (void)*r;     // expected-note {{later used here}}
+}
+
+void definite_single_pointer_multiple_loans(bool cond) {
+  MyObj *p;
+  if (cond){
+    MyObj s;
+    p = &s;     // expected-warning {{does not live long enough}}
+  }             // expected-note {{destroyed here}}
+  else {
+    MyObj t;
+    p = &t;     // expected-warning {{does not live long enough}}
+  }             // expected-note {{destroyed here}}
+  (void)*p;     // expected-note 2  {{later used here}}
+}
+
+
+//===----------------------------------------------------------------------===//
+// Potential (Maybe) Use-After-Free (-W...strict)
+// These are cases where the pointer *may* become dangling, depending on the path taken.
+//===----------------------------------------------------------------------===//
+
+void potential_if_branch(bool cond) {
+  MyObj safe;
+  MyObj* p = &safe;
+  if (cond) {
+    MyObj temp;
+    p = &temp;  // expected-warning {{object whose reference is captured may not live long enough}}
+  }             // expected-note {{destroyed here}}
+  (void)*p;     // expected-note {{later used here}}
+}
+
+// If all paths lead to a dangle, it becomes a definite error.
+void potential_becomes_definite(bool cond) {
+  MyObj* p;
+  if (cond) {
+    MyObj temp1;
+    p = &temp1; // expected-warning {{does not live long enough}}
+  }             // expected-note {{destroyed here}}
+  else {      
+    MyObj temp2;
+    p = &temp2; // expected-warning {{does not live long enough}}
+  }             // expected-note {{destroyed here}}
+  (void)*p;     // expected-note 2 {{later used here}}
+}
+
+void definite_potential_together(bool cond) {
+  MyObj safe;
+  MyObj* p_maybe = &safe;
+  MyObj* p_definite = nullptr;
+
+  {
+    MyObj s;
+    p_definite = &s;  // expected-warning {{does not live long enough}}
+    if (cond) {
+      p_maybe = &s;   // expected-warning {{may not live long enough}}
+    }                 
+  }                   // expected-note 2 {{destroyed here}}
+  (void)*p_definite;  // expected-note {{later used here}}
+  (void)*p_maybe;     // expected-note {{later used here}}
+}
+
+void definite_overrides_potential(bool cond) {
+  MyObj safe;
+  MyObj* p;
+  MyObj* q;
+  {
+    MyObj s;
+    q = &s;       // expected-warning {{does not live long enough}}
+    p = q;
+  }               // expected-note {{destroyed here}}
+
+  if (cond) {
+    // 'q' is conditionally "rescued". 'p' is not.
+    q = &safe;
+  }
+
+  // The use of 'p' is a definite error because it was never rescued.
+  (void)*q;
+  (void)*p;       // expected-note {{later used here}}
+  (void)*q;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Control Flow Tests
+//===----------------------------------------------------------------------===//
+
+void potential_for_loop_use_after_loop_body(MyObj safe) {
+  MyObj* p = &safe;
+  for (int i = 0; i < 1; ++i) {
+    MyObj s;
+    p = &s;     // expected-warning {{may not live long enough}}
+  }             // expected-note {{destroyed here}}
+  (void)*p;     // expected-note {{later used here}}
+}
+
+void potential_for_loop_use_before_loop_body(MyObj safe) {
+  MyObj* p = &safe;
+  for (int i = 0; i < 1; ++i) {
+    (void)*p;   // expected-note {{later used here}}
+    MyObj s;
+    p = &s;     // expected-warning {{may not live long enough}}
+  }             // expected-note {{destroyed here}}
+  (void)*p;
+}
+
+void potential_loop_with_break(bool cond) {
+  MyObj safe;
+  MyObj* p = &safe;
+  for (int i = 0; i < 10; ++i) {
+    if (cond) {
+      MyObj temp;
+      p = &temp; // expected-warning {{may not live long enough}}
+      break;     // expected-note {{destroyed here}}
+    }           
+  } 
+  (void)*p;     // expected-note {{later used here}}
+}
+
+void potential_multiple_expiry_of_same_loan(bool cond) {
+  // Choose the last expiry location for the loan.
+  MyObj safe;
+  MyObj* p = &safe;
+  for (int i = 0; i < 10; ++i) {
+    MyObj unsafe;
+    if (cond) {
+      p = &unsafe; // expected-warning {{may not live long enough}}
+      break;
+    }
+  }               // expected-note {{destroyed here}} 
+  (void)*p;       // expected-note {{later used here}}
+
+  p = &safe;
+  for (int i = 0; i < 10; ++i) {
+    MyObj unsafe;
+    if (cond) {
+      p = &unsafe;    // expected-warning {{may not live long enough}}
+      if (cond)
+        break;
+    }
+  }                   // expected-note {{destroyed here}}
+  (void)*p;           // expected-note {{later used here}}
+
+  p = &safe;
+  for (int i = 0; i < 10; ++i) {
+    if (cond) {
+      MyObj unsafe2;
+      p = &unsafe2;   // expected-warning {{may not live long enough}}
+      break;          // expected-note {{destroyed here}}
+    }
+  }
+  (void)*p;           // expected-note {{later used here}}
+}
+
+void potential_switch(int mode) {
+  MyObj safe;
+  MyObj* p = &safe;
+  switch (mode) {
+  case 1: {
+    MyObj temp;
+    p = &temp;  // expected-warning {{object whose reference is captured may not live long enough}}
+    break;      // expected-note {{destroyed here}}
+  }
+  case 2: {
+    p = &safe;  // This path is okay.
+    break;
+  }
+  }
+  (void)*p;     // expected-note {{later used here}}
+}
+
+void definite_switch(int mode) {
+  MyObj safe;
+  MyObj* p = &safe;
+  // All cases are UaF --> Definite error.
+  switch (mode) {
+  case 1: {
+    MyObj temp1;
+    p = &temp1; // expected-warning {{does not live long enough}}
+    break;      // expected-note {{destroyed here}}
+  }
+  case 2: {
+    MyObj temp2;
+    p = &temp2; // expected-warning {{does not live long enough}}
+    break;      // expected-note {{destroyed here}}
+  }
+  default: {
+    MyObj temp2;
+    p = &temp2; // expected-warning {{does not live long enough}}
+    break;      // expected-note {{destroyed here}}
+  }
+  }
+  (void)*p;     // expected-note 3 {{later used here}}
+}
+
+//===----------------------------------------------------------------------===//
+// No-Error Cases
+//===----------------------------------------------------------------------===//
+void no_error_if_dangle_then_rescue() {
+  MyObj safe;
+  MyObj* p;
+  {
+    MyObj temp;
+    p = &temp;  // p is temporarily dangling.
+  }
+  p = &safe;    // p is "rescued" before use.
+  (void)*p;     // This is safe.
+}
diff --git a/clang/test/SemaCXX/MicrosoftExtensions.cpp b/clang/test/SemaCXX/MicrosoftExtensions.cpp
index 4dff2b1c362a7..ca072d8bd2f6a 100644
--- a/clang/test/SemaCXX/MicrosoftExtensions.cpp
+++ b/clang/test/SemaCXX/MicrosoftExtensions.cpp
@@ -126,17 +126,17 @@ __inline void FreeIDListArray(LPITEMIDLIST *ppidls) {
 typedef struct in_addr {
 public:
   in_addr(in_addr &a) {} // precxx17-note {{candidate constructor not viable: expects an lvalue for 1st argument}}
-  in_addr(in_addr *a) {} // precxx17-note {{candidate constructor not viable: no known conversion from 'IN_ADDR' (aka 'in_addr') to 'in_addr *' for 1st argument}}
+  in_addr(in_addr *a) {} // precxx17-note {{candidate constructor not viable: no known conversion from 'IN_ADDR' (aka 'struct in_addr') to 'in_addr *' for 1st argument}}
 } IN_ADDR;
 
 void f(IN_ADDR __unaligned *a) {
   IN_ADDR local_addr = *a;
   // FIXME: MSVC accepts the following; not sure why clang tries to
   // copy-construct an in_addr.
-  IN_ADDR local_addr2 = a; // precxx17-error {{no viable constructor copying variable of type 'IN_ADDR' (aka 'in_addr')}}
-  // expected-warning@-1 {{implicit cast from type '__unaligned IN_ADDR *' (aka '__unaligned in_addr *') to type 'in_addr *' drops __unaligned qualifier}}
+  IN_ADDR local_addr2 = a; // precxx17-error {{no viable constructor copying variable of type 'IN_ADDR' (aka 'struct in_addr')}}
+  // expected-warning@-1 {{implicit cast from type '__unaligned IN_ADDR *' (aka '__unaligned struct in_addr *') to type 'in_addr *' drops __unaligned qualifier}}
   IN_ADDR local_addr3(a);
-  // expected-warning@-1 {{implicit cast from type '__unaligned IN_ADDR *' (aka '__unaligned in_addr *') to type 'in_addr *' drops __unaligned qualifier}}
+  // expected-warning@-1 {{implicit cast from type '__unaligned IN_ADDR *' (aka '__unaligned struct in_addr *') to type 'in_addr *' drops __unaligned qualifier}}
 }
 
 template<typename T> void h1(T (__stdcall M::* const )()) { }
diff --git a/clang/test/SemaCXX/builtin-get-vtable-pointer.cpp b/clang/test/SemaCXX/builtin-get-vtable-pointer.cpp
index 273f9c3b4c667..b04b38d7996eb 100644
--- a/clang/test/SemaCXX/builtin-get-vtable-pointer.cpp
+++ b/clang/test/SemaCXX/builtin-get-vtable-pointer.cpp
@@ -53,7 +53,7 @@ const void *getThing(const Bar<T> *b = nullptr) {
   return __builtin_get_vtable_pointer(b->ty()); // expected-error{{__builtin_get_vtable_pointer requires an argument of class pointer type, but 'SubType *' (aka 'int *') was provided}}
                                                 // expected-error@-1{{__builtin_get_vtable_pointer requires an argument of polymorphic class pointer type, but 'Thing1' has no virtual methods}}
                                                 // expected-error@-2{{__builtin_get_vtable_pointer requires an argument of polymorphic class pointer type, but 'NonPolymorphic' has no virtual methods}}
-                                                // expected-error@-3{{__builtin_get_vtable_pointer requires an argument with a complete type, but 'SubType' (aka 'basic::ForwardDeclaration') is incomplete}}
+                                                // expected-error@-3{{__builtin_get_vtable_pointer requires an argument with a complete type, but 'SubType' (aka 'ForwardDeclaration') is incomplete}}
 }
 template <typename>
 struct IncompleteTemplate; // expected-note{{template is declared here}}
diff --git a/clang/test/SemaCXX/class-base-member-init.cpp b/clang/test/SemaCXX/class-base-member-init.cpp
index f5489e2da3e93..29b4545e4fb16 100644
--- a/clang/test/SemaCXX/class-base-member-init.cpp
+++ b/clang/test/SemaCXX/class-base-member-init.cpp
@@ -83,7 +83,7 @@ namespace test5 {
     A() : decltype(Base(1))(3) {
     }
     A(int) : Base(3), // expected-note {{previous initialization is here}}
-             decltype(Base(1))(2), // expected-error {{multiple initializations given for base 'decltype(Base(1))' (aka 'test5::Base')}}
+             decltype(Base(1))(2), // expected-error {{multiple initializations given for base 'decltype(Base(1))' (aka 'Base')}}
              decltype(int())() { // expected-error {{constructor initializer 'decltype(int())' (aka 'int') does not name a class}}
     }
     A(float) : decltype(A())(3) {
diff --git a/clang/test/SemaCXX/co_await-ast.cpp b/clang/test/SemaCXX/co_await-ast.cpp
index f792a2cb7b1a1..5be2004eb1072 100644
--- a/clang/test/SemaCXX/co_await-ast.cpp
+++ b/clang/test/SemaCXX/co_await-ast.cpp
@@ -48,9 +48,9 @@ awaitable foo() {
 // CHECK:   | `-ExprWithCleanups {{.*}} 'void'
 // CHECK:   |   `-CoawaitExpr {{.*}} 'void'
 // CHECK:   |     |-CXXTemporaryObjectExpr {{.*}} 'executor' 'void (){{.*}} noexcept' zeroing
-// CHECK:   |     |-MaterializeTemporaryExpr {{.*}} 'result_t':'awaitable_frame::result_t' lvalue
-// CHECK:   |     | `-CXXBindTemporaryExpr {{.*}} 'result_t':'awaitable_frame::result_t' (CXXTemporary {{.*}})
-// CHECK:   |     |   `-CXXMemberCallExpr {{.*}} 'result_t':'awaitable_frame::result_t'
+// CHECK:   |     |-MaterializeTemporaryExpr {{.*}} 'result_t' lvalue
+// CHECK:   |     | `-CXXBindTemporaryExpr {{.*}} 'result_t' (CXXTemporary {{.*}})
+// CHECK:   |     |   `-CXXMemberCallExpr {{.*}} 'result_t'
 // CHECK:   |     |     |-MemberExpr {{.*}} '<bound member function type>' .await_transform {{.*}}
 // CHECK:   |     |     | `-DeclRefExpr {{.*}} 'std::coroutine_traits<awaitable>::promise_type':'awaitable_frame' lvalue Var {{.*}} '__promise' 'std::coroutine_traits<awaitable>::promise_type':'awaitable_frame'
 // CHECK:   |     |     `-CXXTemporaryObjectExpr {{.*}} 'executor' 'void (){{.*}} noexcept' zeroing
@@ -58,27 +58,27 @@ awaitable foo() {
 // CHECK:   |     | `-CXXMemberCallExpr {{.*}} 'bool'
 // CHECK:   |     |   `-MemberExpr {{.*}} '<bound member function type>' .await_ready {{.*}}
 // CHECK:   |     |     `-ImplicitCastExpr {{.*}} 'const awaitable_frame::result_t' lvalue <NoOp>
-// CHECK:   |     |       `-OpaqueValueExpr {{.*}} 'result_t':'awaitable_frame::result_t' lvalue
-// CHECK:   |     |         `-MaterializeTemporaryExpr {{.*}} 'result_t':'awaitable_frame::result_t' lvalue
-// CHECK:   |     |           `-CXXBindTemporaryExpr {{.*}} 'result_t':'awaitable_frame::result_t' (CXXTemporary {{.*}})
-// CHECK:   |     |             `-CXXMemberCallExpr {{.*}} 'result_t':'awaitable_frame::result_t'
+// CHECK:   |     |       `-OpaqueValueExpr {{.*}} 'result_t' lvalue
+// CHECK:   |     |         `-MaterializeTemporaryExpr {{.*}} 'result_t' lvalue
+// CHECK:   |     |           `-CXXBindTemporaryExpr {{.*}} 'result_t' (CXXTemporary {{.*}})
+// CHECK:   |     |             `-CXXMemberCallExpr {{.*}} 'result_t'
 // CHECK:   |     |               |-MemberExpr {{.*}} '<bound member function type>' .await_transform {{.*}}
 // CHECK:   |     |               | `-DeclRefExpr {{.*}} 'std::coroutine_traits<awaitable>::promise_type':'awaitable_frame' lvalue Var {{.*}} '__promise' 'std::coroutine_traits<awaitable>::promise_type':'awaitable_frame'
 // CHECK:   |     |               `-CXXTemporaryObjectExpr {{.*}} 'executor' 'void (){{.*}} noexcept' zeroing
 // CHECK:   |     |-ExprWithCleanups {{.*}} 'void'
 // CHECK:   |     | `-CXXMemberCallExpr {{.*}} 'void'
 // CHECK:   |     |   |-MemberExpr {{.*}} '<bound member function type>' .await_suspend {{.*}}
-// CHECK:   |     |   | `-OpaqueValueExpr {{.*}} 'result_t':'awaitable_frame::result_t' lvalue
-// CHECK:   |     |   |   `-MaterializeTemporaryExpr {{.*}} 'result_t':'awaitable_frame::result_t' lvalue
-// CHECK:   |     |   |     `-CXXBindTemporaryExpr {{.*}} 'result_t':'awaitable_frame::result_t' (CXXTemporary {{.*}})
-// CHECK:   |     |   |       `-CXXMemberCallExpr {{.*}} 'result_t':'awaitable_frame::result_t'
+// CHECK:   |     |   | `-OpaqueValueExpr {{.*}} 'result_t' lvalue
+// CHECK:   |     |   |   `-MaterializeTemporaryExpr {{.*}} 'result_t' lvalue
+// CHECK:   |     |   |     `-CXXBindTemporaryExpr {{.*}} 'result_t' (CXXTemporary {{.*}})
+// CHECK:   |     |   |       `-CXXMemberCallExpr {{.*}} 'result_t'
 // CHECK:   |     |   |         |-MemberExpr {{.*}} '<bound member function type>' .await_transform {{.*}}
 // CHECK:   |     |   |         | `-DeclRefExpr {{.*}} 'std::coroutine_traits<awaitable>::promise_type':'awaitable_frame' lvalue Var {{.*}} '__promise' 'std::coroutine_traits<awaitable>::promise_type':'awaitable_frame'
 // CHECK:   |     |   |         `-CXXTemporaryObjectExpr {{.*}} 'executor' 'void (){{.*}} noexcept' zeroing
 // CHECK:   |     |   `-ImplicitCastExpr {{.*}} 'std::coroutine_handle<void>' <ConstructorConversion>
 // CHECK:   |     |     `-CXXConstructExpr {{.*}} 'std::coroutine_handle<void>' 'void (coroutine_handle<awaitable_frame> &&){{.*}} noexcept'
-// CHECK:   |     |       `-MaterializeTemporaryExpr {{.*}} 'coroutine_handle<awaitable_frame>':'std::coroutine_handle<awaitable_frame>' xvalue
-// CHECK:   |     |         `-CallExpr {{.*}} 'coroutine_handle<awaitable_frame>':'std::coroutine_handle<awaitable_frame>'
+// CHECK:   |     |       `-MaterializeTemporaryExpr {{.*}} 'coroutine_handle<awaitable_frame>' xvalue
+// CHECK:   |     |         `-CallExpr {{.*}} 'coroutine_handle<awaitable_frame>'
 // CHECK:   |     |           |-ImplicitCastExpr {{.*}} 'coroutine_handle<awaitable_frame> (*)(void *) noexcept' <FunctionToPointerDecay>
 // CHECK:   |     |           | `-DeclRefExpr {{.*}} 'coroutine_handle<awaitable_frame> (void *) noexcept' lvalue CXXMethod {{.*}} 'from_address' 'coroutine_handle<awaitable_frame> (void *) noexcept'
 // CHECK:   |     |           `-CallExpr {{.*}} 'void *'
@@ -87,10 +87,10 @@ awaitable foo() {
 // CHECK:   |     `-CXXMemberCallExpr {{.*}} 'void'
 // CHECK:   |       `-MemberExpr {{.*}} '<bound member function type>' .await_resume {{.*}}
 // CHECK:   |         `-ImplicitCastExpr {{.*}} 'const awaitable_frame::result_t' lvalue <NoOp>
-// CHECK:   |           `-OpaqueValueExpr {{.*}} 'result_t':'awaitable_frame::result_t' lvalue
-// CHECK:   |             `-MaterializeTemporaryExpr {{.*}} 'result_t':'awaitable_frame::result_t' lvalue
-// CHECK:   |               `-CXXBindTemporaryExpr {{.*}} 'result_t':'awaitable_frame::result_t' (CXXTemporary {{.*}})
-// CHECK:   |                 `-CXXMemberCallExpr {{.*}} 'result_t':'awaitable_frame::result_t'
+// CHECK:   |           `-OpaqueValueExpr {{.*}} 'result_t' lvalue
+// CHECK:   |             `-MaterializeTemporaryExpr {{.*}} 'result_t' lvalue
+// CHECK:   |               `-CXXBindTemporaryExpr {{.*}} 'result_t' (CXXTemporary {{.*}})
+// CHECK:   |                 `-CXXMemberCallExpr {{.*}} 'result_t'
 // CHECK:   |                   |-MemberExpr {{.*}} '<bound member function type>' .await_transform {{.*}}
 // CHECK:   |                   | `-DeclRefExpr {{.*}} 'std::coroutine_traits<awaitable>::promise_type':'awaitable_frame' lvalue Var {{.*}} '__promise' 'std::coroutine_traits<awaitable>::promise_type':'awaitable_frame'
 // CHECK:   |                   `-CXXTemporaryObjectExpr {{.*}} <col:12, col:21> 'executor' 'void (){{.*}} noexcept' zeroing
diff --git a/clang/test/SemaCXX/compound-literal.cpp b/clang/test/SemaCXX/compound-literal.cpp
index 9c7c606838de9..4b975a0d73074 100644
--- a/clang/test/SemaCXX/compound-literal.cpp
+++ b/clang/test/SemaCXX/compound-literal.cpp
@@ -37,8 +37,8 @@ namespace brace_initializers {
 
   POD p = (POD){1, 2};
   // CHECK-NOT: CXXBindTemporaryExpr {{.*}} 'brace_initializers::POD'
-  // CHECK: CompoundLiteralExpr {{.*}} 'POD':'brace_initializers::POD'
-  // CHECK-NEXT: InitListExpr {{.*}} 'POD':'brace_initializers::POD'
+  // CHECK: CompoundLiteralExpr {{.*}} 'POD'{{$}}
+  // CHECK-NEXT: InitListExpr {{.*}} 'POD'{{$}}
   // CHECK-NEXT: ConstantExpr {{.*}}
   // CHECK-NEXT: IntegerLiteral {{.*}} 1{{$}}
   // CHECK-NEXT: ConstantExpr {{.*}}
@@ -46,34 +46,34 @@ namespace brace_initializers {
 
   void test() {
     (void)(POD){1, 2};
-    // CHECK-NOT: CXXBindTemporaryExpr {{.*}} 'POD':'brace_initializers::POD'
-    // CHECK-NOT: ConstantExpr {{.*}} 'POD':'brace_initializers::POD'
-    // CHECK: CompoundLiteralExpr {{.*}} 'POD':'brace_initializers::POD'
-    // CHECK-NEXT: InitListExpr {{.*}} 'POD':'brace_initializers::POD'
+    // CHECK-NOT: CXXBindTemporaryExpr {{.*}} 'POD'
+    // CHECK-NOT: ConstantExpr {{.*}} 'POD'
+    // CHECK: CompoundLiteralExpr {{.*}} 'POD'{{$}}
+    // CHECK-NEXT: InitListExpr {{.*}} 'POD'{{$}}
     // CHECK-NEXT: IntegerLiteral {{.*}} 1{{$}}
     // CHECK-NEXT: IntegerLiteral {{.*}} 2{{$}}
 
     (void)(HasDtor){1, 2};
-    // CHECK: CXXBindTemporaryExpr {{.*}} 'HasDtor':'brace_initializers::HasDtor'
-    // CHECK-NEXT: CompoundLiteralExpr {{.*}} 'HasDtor':'brace_initializers::HasDtor'
-    // CHECK-NEXT: InitListExpr {{.*}} 'HasDtor':'brace_initializers::HasDtor'
+    // CHECK: CXXBindTemporaryExpr {{.*}} 'HasDtor'
+    // CHECK-NEXT: CompoundLiteralExpr {{.*}} 'HasDtor'{{$}}
+    // CHECK-NEXT: InitListExpr {{.*}} 'HasDtor'{{$}}
     // CHECK-NEXT: IntegerLiteral {{.*}} 1{{$}}
     // CHECK-NEXT: IntegerLiteral {{.*}} 2{{$}}
 
 #if __cplusplus >= 201103L
     (void)(HasCtor){1, 2};
-    // CHECK-CXX11-NOT: CXXBindTemporaryExpr {{.*}} 'HasCtor':'brace_initializers::HasCtor'
-    // CHECK-CXX11-NOT: ConstantExpr {{.*}} 'HasCtor':'brace_initializers::HasCtor'
-    // CHECK-CXX11: CompoundLiteralExpr {{.*}} 'HasCtor':'brace_initializers::HasCtor'
-    // CHECK-CXX11-NEXT: CXXTemporaryObjectExpr {{.*}} 'HasCtor':'brace_initializers::HasCtor'
+    // CHECK-CXX11-NOT: CXXBindTemporaryExpr {{.*}} 'HasCtor'
+    // CHECK-CXX11-NOT: ConstantExpr {{.*}} 'HasCtor'
+    // CHECK-CXX11: CompoundLiteralExpr {{.*}} 'HasCtor'{{$}}
+    // CHECK-CXX11-NEXT: CXXTemporaryObjectExpr {{.*}} 'HasCtor'
     // CHECK-CXX11-NEXT: IntegerLiteral {{.*}} 1{{$}}
     // CHECK-CXX11-NEXT: IntegerLiteral {{.*}} 2{{$}}
 
     (void)(HasCtorDtor){1, 2};
-    // CHECK-CXX11: CXXBindTemporaryExpr {{.*}} 'HasCtorDtor':'brace_initializers::HasCtorDtor'
-    // CHECK-CXX11-NOT: ConstantExpr {{.*}} 'HasCtorDtor':'brace_initializers::HasCtorDtor'
-    // CHECK-CXX11: CompoundLiteralExpr {{.*}} 'HasCtorDtor':'brace_initializers::HasCtorDtor'
-    // CHECK-CXX11-NEXT: CXXTemporaryObjectExpr {{.*}} 'HasCtorDtor':'brace_initializers::HasCtorDtor'
+    // CHECK-CXX11: CXXBindTemporaryExpr {{.*}} 'HasCtorDtor'
+    // CHECK-CXX11-NOT: ConstantExpr {{.*}} 'HasCtorDtor'
+    // CHECK-CXX11: CompoundLiteralExpr {{.*}} 'HasCtorDtor'{{$}}
+    // CHECK-CXX11-NEXT: CXXTemporaryObjectExpr {{.*}} 'HasCtorDtor'
     // CHECK-CXX11-NEXT: IntegerLiteral {{.*}} 1{{$}}
     // CHECK-CXX11-NEXT: IntegerLiteral {{.*}} 2{{$}}
 #endif
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index 2423a77e6e7d2..91c4ff1cb520d 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -2645,3 +2645,19 @@ namespace GH150709 {
   static_assert((e2[0].*mp)() == 1, ""); // expected-error {{constant expression}}
   static_assert((g.*mp)() == 1, ""); // expected-error {{constant expression}}
 }
+
+namespace GH154567 {
+  struct T {
+    int i;
+  };
+
+  struct S {
+    struct { // expected-warning {{GNU extension}}
+      T val;
+    };
+    constexpr S() : val() {}
+  };
+
+  constexpr S s{};
+  static_assert(s.val.i == 0, "");
+}
diff --git a/clang/test/SemaCXX/constant-expression.cpp b/clang/test/SemaCXX/constant-expression.cpp
index cc041a4acd18c..ef48ee50be288 100644
--- a/clang/test/SemaCXX/constant-expression.cpp
+++ b/clang/test/SemaCXX/constant-expression.cpp
@@ -149,7 +149,7 @@ namespace PR31701 {
   };
   template <int M> class D;
   template <int M>
-  template<int i> void D<M>::set() { // expected-error {{from class 'D<M>' without definition}}
+  template<int i> void D<M>::set() { // expected-error {{from class 'PR31701::D<M>' without definition}}
     const C c = C::n<i>;
   }
 }
diff --git a/clang/test/SemaCXX/constructor.cpp b/clang/test/SemaCXX/constructor.cpp
index b0b580b6aac9b..b278fd5cff204 100644
--- a/clang/test/SemaCXX/constructor.cpp
+++ b/clang/test/SemaCXX/constructor.cpp
@@ -92,7 +92,7 @@ namespace PR38286 {
   template<typename T> A<T>::A() {} // expected-error {{incomplete type 'A' named in nested name specifier}}
   /*FIXME: needed to recover properly from previous error*/;
   template<typename> struct B;
-  template<typename T> void B<T>::f() {} // expected-error {{out-of-line definition of 'f' from class 'B<type-parameter-0-0>'}}
+  template<typename T> void B<T>::f() {} // expected-error {{out-of-line definition of 'f' from class 'PR38286::B<type-parameter-0-0>'}}
   template<typename> struct C; // expected-note {{non-type declaration found}}
   template<typename T> C<T>::~C() {} // expected-error {{identifier 'C' after '~' in destructor name does not name a type}}
 }
diff --git a/clang/test/SemaCXX/coroutine-allocs.cpp b/clang/test/SemaCXX/coroutine-allocs.cpp
index cce56deaa5f10..e6b086bd1c720 100644
--- a/clang/test/SemaCXX/coroutine-allocs.cpp
+++ b/clang/test/SemaCXX/coroutine-allocs.cpp
@@ -19,7 +19,7 @@ struct resumable {
   };
 };
 
-resumable f1() { // expected-error {{'operator new' provided by 'std::coroutine_traits<resumable>::promise_type' (aka 'resumable::promise_type') is not usable with the function signature of 'f1'}}
+resumable f1() { // expected-error {{'operator new' provided by 'std::coroutine_traits<resumable>::promise_type' (aka 'typename resumable::promise_type') is not usable with the function signature of 'f1'}}
   co_return;
 }
 
@@ -52,11 +52,11 @@ resumable f4(Allocator) {
   co_return;
 }
 
-resumable f5(const Allocator) { // expected-error {{operator new' provided by 'std::coroutine_traits<resumable, const Allocator>::promise_type' (aka 'resumable::promise_type') is not usable}}
+resumable f5(const Allocator) { // expected-error {{operator new' provided by 'std::coroutine_traits<resumable, const Allocator>::promise_type' (aka 'typename resumable::promise_type') is not usable}}
   co_return;
 }
 
-resumable f6(const Allocator &) { // expected-error {{operator new' provided by 'std::coroutine_traits<resumable, const Allocator &>::promise_type' (aka 'resumable::promise_type') is not usable}}
+resumable f6(const Allocator &) { // expected-error {{operator new' provided by 'std::coroutine_traits<resumable, const Allocator &>::promise_type' (aka 'typename resumable::promise_type') is not usable}}
   co_return;
 }
 
diff --git a/clang/test/SemaCXX/coroutine-traits-undefined-template.cpp b/clang/test/SemaCXX/coroutine-traits-undefined-template.cpp
index ea25cea365b94..e7de0c80d7343 100644
--- a/clang/test/SemaCXX/coroutine-traits-undefined-template.cpp
+++ b/clang/test/SemaCXX/coroutine-traits-undefined-template.cpp
@@ -14,5 +14,5 @@ template <> struct coroutine_traits<void>; // expected-note {{forward declaratio
 } // namespace std
 
 void uses_forward_declaration() {
-  co_return; // expected-error {{this function cannot be a coroutine: missing definition of specialization 'coroutine_traits<void>'}}
+  co_return; // expected-error {{this function cannot be a coroutine: missing definition of specialization 'std::coroutine_traits<void>'}}
 }
diff --git a/clang/test/SemaCXX/coroutines.cpp b/clang/test/SemaCXX/coroutines.cpp
index c9cefeb30c15a..098c1c21a5962 100644
--- a/clang/test/SemaCXX/coroutines.cpp
+++ b/clang/test/SemaCXX/coroutines.cpp
@@ -1396,7 +1396,7 @@ struct bad_promise_deleted_constructor {
 
 coro<bad_promise_deleted_constructor>
 bad_coroutine_calls_deleted_promise_constructor() {
-  // expected-error@-1 {{call to deleted constructor of 'std::coroutine_traits<coro<CoroHandleMemberFunctionTest::bad_promise_deleted_constructor>>::promise_type' (aka 'CoroHandleMemberFunctionTest::bad_promise_deleted_constructor')}}
+  // expected-error@-1 {{call to deleted constructor of 'std::coroutine_traits<coro<bad_promise_deleted_constructor>>::promise_type' (aka 'CoroHandleMemberFunctionTest::bad_promise_deleted_constructor')}}
   co_return;
 }
 
@@ -1463,7 +1463,7 @@ struct bad_promise_no_matching_constructor {
 
 coro<bad_promise_no_matching_constructor>
 bad_coroutine_calls_with_no_matching_constructor(int, int) {
-  // expected-error@-1 {{call to deleted constructor of 'std::coroutine_traits<coro<CoroHandleMemberFunctionTest::bad_promise_no_matching_constructor>, int, int>::promise_type' (aka 'CoroHandleMemberFunctionTest::bad_promise_no_matching_constructor')}}
+  // expected-error@-1 {{call to deleted constructor of 'std::coroutine_traits<coro<bad_promise_no_matching_constructor>, int, int>::promise_type' (aka 'CoroHandleMemberFunctionTest::bad_promise_no_matching_constructor')}}
   co_return;
 }
 
diff --git a/clang/test/SemaCXX/ctad.cpp b/clang/test/SemaCXX/ctad.cpp
index 50b64e39c3f92..8380b564bdcdd 100644
--- a/clang/test/SemaCXX/ctad.cpp
+++ b/clang/test/SemaCXX/ctad.cpp
@@ -186,7 +186,7 @@ namespace GH136624 {
     template<class Y> using Alias = A<Y>;
   }
 
-  // FIXME: This diagnostic prints incorrect qualification for `A<int>`.
+  // FIXME: This diagnostic is missing 'foo::Alias', as written.
   foo::Alias t = 0;
-  // expected-error@-1 {{no viable conversion from 'int' to 'foo::A<int>' (aka 'A<int>')}}
+  // expected-error@-1 {{no viable conversion from 'int' to 'GH136624::A<int>' (aka 'A<int>')}}
 } // namespace GH136624
diff --git a/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp b/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp
index 57a48fac56cd6..e18223d6a6df7 100644
--- a/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp
+++ b/clang/test/SemaCXX/cxx1y-variable-templates_in_class.cpp
@@ -302,17 +302,17 @@ namespace in_class_template {
     };
 
     template<typename T> void f() {
-      typename T::template A<int> a; // expected-error {{template name refers to non-type template 'S::template A'}}
+      typename T::template A<int> a; // expected-error {{template name refers to non-type template 'in_class_template::bad_reference::S::template A'}}
     }
     template<typename T> void g() {
-      T::template A<int>::B = 0; // expected-error {{template name refers to non-type template 'S::template A'}}
+      T::template A<int>::B = 0; // expected-error {{template name refers to non-type template 'in_class_template::bad_reference::S::template A'}}
     }
     template<typename T> void h() {
-      class T::template A<int> c; // expected-error {{template name refers to non-type template 'S::template A'}}
+      class T::template A<int> c; // expected-error {{template name refers to non-type template 'in_class_template::bad_reference::S::template A'}}
     }
 
     template<typename T>
-    struct X : T::template A<int> {}; // expected-error {{template name refers to non-type template 'S::template A'}}
+    struct X : T::template A<int> {}; // expected-error {{template name refers to non-type template 'in_class_template::bad_reference::S::template A'}}
 
     template void f<S>(); // expected-note {{in instantiation of}}
     template void g<S>(); // expected-note {{in instantiation of}}
@@ -393,7 +393,7 @@ namespace dependent_static_var_template {
   int &r = A::template n; // expected-error {{use of variable template 'A::template n' requires template arguments}} expected-error {{a template argument list is expected after a name prefixed by the template keyword}}
 
   template<typename T>
-  int &f() { return T::template n; } // expected-error {{use of variable template 'A::template n' requires template arguments}} expected-error {{a template argument list is expected after a name prefixed by the template keyword}}
+  int &f() { return T::template n; } // expected-error {{use of variable template 'dependent_static_var_template::A::template n' requires template arguments}} expected-error {{a template argument list is expected after a name prefixed by the template keyword}}
   int &s = f<A>(); // expected-note {{instantiation of}}
 
   namespace B {
diff --git a/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp b/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
index 8745185a1135c..204dd9b077b93 100644
--- a/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
+++ b/clang/test/SemaCXX/cxx1z-class-template-argument-deduction.cpp
@@ -142,13 +142,13 @@ namespace look_into_current_instantiation {
                     // templates, and members of the current instantiation
   A<float> &r = a;
 
-  template<typename T> struct B { // expected-note {{could not match 'B<T>' against 'int'}} \
-                                  // expected-note {{implicit deduction guide declared as 'template <typename T> B(B<T>) -> B<T>'}}
+  template<typename T> struct B { // expected-note {{could not match 'look_into_current_instantiation::B<T>' against 'int'}} \
+                                  // expected-note {{implicit deduction guide declared as 'template <typename T> B(look_into_current_instantiation::B<T>) -> look_into_current_instantiation::B<T>'}}
     struct X {
       typedef T type;
     };
     B(typename X::type); // expected-note {{couldn't infer template argument 'T'}} \
-                         // expected-note {{implicit deduction guide declared as 'template <typename T> B(typename X::type) -> B<T>'}}
+                         // expected-note {{implicit deduction guide declared as 'template <typename T> B(typename X::type) -> look_into_current_instantiation::B<T>'}}
   };
   B b = 0; // expected-error {{no viable}}
 
diff --git a/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp b/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp
index 3229a91fbcefb..1c832e51c2cbc 100644
--- a/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp
+++ b/clang/test/SemaCXX/cxx23-invalid-constexpr.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify=expected -std=c++23 %s
+// RUN: %clang_cc1 -fsyntax-only -verify=expected -std=c++23 %s -fexperimental-new-constant-interpreter
 
 // This test covers modifications made by P2448R2.
 
diff --git a/clang/test/SemaCXX/cxx2a-destroying-delete.cpp b/clang/test/SemaCXX/cxx2a-destroying-delete.cpp
index 27ea6663bae6a..812fd789f0c5e 100644
--- a/clang/test/SemaCXX/cxx2a-destroying-delete.cpp
+++ b/clang/test/SemaCXX/cxx2a-destroying-delete.cpp
@@ -138,7 +138,7 @@ namespace templated {
     void operator delete(id_alias<C> *, std::destroying_delete_t);
   };
   template<typename T> struct D {
-    void operator delete(typename id_struct<D>::type *, std::destroying_delete_t); // expected-error {{use 'D<T> *'}}
+    void operator delete(typename id_struct<D>::type *, std::destroying_delete_t); // expected-error {{use 'templated::D<T> *'}}
   };
 }
 
diff --git a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
index c4cfd9398920a..6cf0e0251ab62 100644
--- a/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
+++ b/clang/test/SemaCXX/cxx2b-consteval-propagate.cpp
@@ -610,3 +610,19 @@ namespace GH135281 {
   void (*ff)() = f2<B>; // expected-note {{instantiation of function template specialization}}
 }
 #endif
+
+namespace GH145776 {
+
+void runtime_only() {}
+consteval void comptime_only() {}
+
+void fn() {
+  []() {
+    runtime_only();
+    []() {
+      &comptime_only;
+    }();
+  }();
+}
+
+}
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index fcbe0f62e6d4f..74b3573a0dcaa 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -96,12 +96,12 @@ struct Test {
 void test() {
 
     [i = 0](this Test) { }();
-    // expected-error@-1 {{invalid explicit object parameter type 'ThisInLambdaWithCaptures::Test' in lambda with capture; the type must be the same as, or derived from, the lambda}}
+    // expected-error@-1 {{invalid explicit object parameter type 'Test' in lambda with capture; the type must be the same as, or derived from, the lambda}}
 
     struct Derived;
     auto ok = [i = 0](this const Derived&) {};
     auto ko = [i = 0](this const Test&) {};
-    // expected-error@-1 {{invalid explicit object parameter type 'ThisInLambdaWithCaptures::Test' in lambda with capture; the type must be the same as, or derived from, the lambda}}
+    // expected-error@-1 {{invalid explicit object parameter type 'Test' in lambda with capture; the type must be the same as, or derived from, the lambda}}
 
     struct Derived : decltype(ok){};
     Derived dok{ok};
diff --git a/clang/test/SemaCXX/cxx2c-variadic-friends.cpp b/clang/test/SemaCXX/cxx2c-variadic-friends.cpp
index a4d7c8078338d..0b01907c8c722 100644
--- a/clang/test/SemaCXX/cxx2c-variadic-friends.cpp
+++ b/clang/test/SemaCXX/cxx2c-variadic-friends.cpp
@@ -145,7 +145,7 @@ class S {
 template <typename U>
 struct T {
   static_assert(S<U, T>::a == 42);
-  static_assert(S<U, T>::a == 43); // expected-error {{static assertion failed due to requirement 'S<int, template_template::T>::a == 43'}} \
+  static_assert(S<U, T>::a == 43); // expected-error {{static assertion failed due to requirement 'template_template::S<int, template_template::T>::a == 43'}} \
                                    // expected-note {{expression evaluates to '42 == 43'}}
 };
 
diff --git a/clang/test/SemaCXX/destructor.cpp b/clang/test/SemaCXX/destructor.cpp
index b9e0b17d510ab..bc47d873c4e28 100644
--- a/clang/test/SemaCXX/destructor.cpp
+++ b/clang/test/SemaCXX/destructor.cpp
@@ -431,15 +431,15 @@ namespace PR9238 {
 }
 
 namespace PR7900 {
-  struct A { // expected-note 2{{type 'PR7900::A' found by destructor name lookup}}
+  struct A { // expected-note 2{{type 'A' found by destructor name lookup}}
   };
   struct B : public A {
   };
   void foo() {
     B b;
     b.~B();
-    b.~A(); // expected-error{{destructor type 'PR7900::A' in object destruction expression does not match the type 'B' of the object being destroyed}}
-    (&b)->~A(); // expected-error{{destructor type 'PR7900::A' in object destruction expression does not match the type 'B' of the object being destroyed}}
+    b.~A(); // expected-error{{destructor type 'A' in object destruction expression does not match the type 'B' of the object being destroyed}}
+    (&b)->~A(); // expected-error{{destructor type 'A' in object destruction expression does not match the type 'B' of the object being destroyed}}
   }
 }
 
diff --git a/clang/test/SemaCXX/elaborated-type-specifier.cpp b/clang/test/SemaCXX/elaborated-type-specifier.cpp
index a96e696edcc6c..a80ba0794bf12 100644
--- a/clang/test/SemaCXX/elaborated-type-specifier.cpp
+++ b/clang/test/SemaCXX/elaborated-type-specifier.cpp
@@ -27,7 +27,7 @@ namespace NS {
 
 void test_X_elab(NS::X x) {
   struct S4 *s4 = 0; // expected-note{{'S4' is not defined, but forward declared here; conversion would be valid if it was derived from 'NS::S4'}}
-  x.test_elab2(s4); // expected-error{{cannot initialize a parameter of type 'S4 *' (aka 'NS::S4 *') with an lvalue of type 'struct S4 *'}}
+  x.test_elab2(s4); // expected-error{{cannot initialize a parameter of type 'S4 *' with an lvalue of type 'struct S4 *'}}
 }
 
 namespace NS {
diff --git a/clang/test/SemaCXX/enum-scoped.cpp b/clang/test/SemaCXX/enum-scoped.cpp
index 2d7b3c9557ebd..09f3206c84ccb 100644
--- a/clang/test/SemaCXX/enum-scoped.cpp
+++ b/clang/test/SemaCXX/enum-scoped.cpp
@@ -1,5 +1,6 @@
-// RUN: %clang_cc1 -fsyntax-only -pedantic -std=c++11 -verify -triple x86_64-apple-darwin %s
-// RUN: %clang_cc1 -fsyntax-only -pedantic -std=c++17 -verify -triple x86_64-apple-darwin %s
+// RUN: %clang_cc1 -fsyntax-only -pedantic -std=c++11 -verify=expected,cxx11-17 -triple x86_64-apple-darwin %s
+// RUN: %clang_cc1 -fsyntax-only -pedantic -std=c++17 -verify=expected,cxx11-17 -triple x86_64-apple-darwin %s
+// RUN: %clang_cc1 -fsyntax-only -pedantic -std=c++20 -verify -triple x86_64-apple-darwin %s
 
 enum class E1 {
   Val1 = 1L
@@ -128,7 +129,10 @@ namespace rdar9366066 {
 
   void f(X x) {
     x % X::value; // expected-error{{invalid operands to binary expression ('X' and 'rdar9366066::X')}}
+                  // expected-note@-1{{no implicit conversion for scoped enum; consider casting to underlying type}}
+                  // expected-note@-2{{no implicit conversion for scoped enum; consider casting to underlying type}}
     x % 8; // expected-error{{invalid operands to binary expression ('X' and 'int')}}
+           // expected-note@-1{{no implicit conversion for scoped enum; consider casting to underlying type}}
   }
 }
 
@@ -325,7 +329,7 @@ namespace PR18044 {
   int E::*p; // expected-error {{does not point into a class}}
   using E::f; // expected-error {{no member named 'f'}}
 
-  using E::a; // expected-warning {{using declaration naming a scoped enumerator is a C++20 extension}}
+  using E::a; // cxx11-17-warning {{using declaration naming a scoped enumerator is a C++20 extension}}
   E b = a;
 }
 
@@ -334,7 +338,7 @@ namespace test11 {
   typedef E E2;
   E2 f1() { return E::a; }
 
-  bool f() { return !f1(); } // expected-error {{invalid argument type 'E2' (aka 'test11::E') to unary expression}}
+  bool f() { return !f1(); } // expected-error {{invalid argument type 'E2' (aka 'E') to unary expression}}
 }
 
 namespace PR35586 {
@@ -364,3 +368,102 @@ S<_Atomic(int)> s; // expected-warning {{'_Atomic' is a C11 extension}}
 static_assert(__is_same(__underlying_type(S<_Atomic(long long)>::OhBoy), long long), ""); // expected-warning {{'_Atomic' is a C11 extension}}
                                                                                           // expected-note@-1 {{in instantiation of template class 'GH147736::S<_Atomic(long long)>' requested here}}
 }
+
+namespace GH24265 {
+  enum class E_int { e };
+  enum class E_long : long { e };
+
+  void f() {
+    E_int::e + E_long::e; // expected-error {{invalid operands to binary expression ('GH24265::E_int' and 'GH24265::E_long')}}
+                          // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+                          // expected-note@-2 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    E_int::e + 0; // expected-error {{invalid operands to binary expression ('GH24265::E_int' and 'int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+
+    0 * E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 / E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 % E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 + E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 - E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 << E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 >> E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+
+    #if __cplusplus >= 202002L
+    0 <=> E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                    // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    #endif
+
+    0 < E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 > E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 <= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 >= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 == E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 != E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 & E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 ^ E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 | E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                  // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 && E_int::e; // expected-error {{value of type 'GH24265::E_int' is not contextually convertible to 'bool'}}
+                   // expected-error@-1 {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-2 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    0 || E_int::e; // expected-error {{value of type 'GH24265::E_int' is not contextually convertible to 'bool'}}
+                   // expected-error@-1 {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-2 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+
+    int a;
+    a *= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    a /= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    a %= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    a += E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    a -= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    a <<= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                    // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    a >>= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                    // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    a &= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    a ^= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+    a |= E_int::e; // expected-error {{invalid operands to binary expression ('int' and 'GH24265::E_int')}}
+                   // expected-note@-1 {{no implicit conversion for scoped enum; consider casting to underlying type}}
+
+    // TODO: These do not have the diagnostic yet
+    E_int b;
+    b *= 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+    b /= 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+    b %= 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+    b += 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+    b -= 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+    b <<= 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+    b >>= 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+    b &= 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+    b ^= 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+    b |= 0; // expected-error {{invalid operands to binary expression ('E_int' and 'int')}}
+
+    a = E_int::e; // expected-error {{assigning to 'int' from incompatible type 'GH24265::E_int'}}
+    b = 0; // expected-error {{assigning to 'E_int' from incompatible type 'int'}}
+
+    E_int c = 0; // expected-error {{cannot initialize a variable of type 'E_int' with an rvalue of type 'int'}}
+    int d = E_int::e; // expected-error {{cannot initialize a variable of type 'int' with an rvalue of type 'GH24265::E_int'}}
+  }
+}
diff --git a/clang/test/SemaCXX/err_init_conversion_failed.cpp b/clang/test/SemaCXX/err_init_conversion_failed.cpp
index f1949c079ecb6..e2617a1061ed7 100644
--- a/clang/test/SemaCXX/err_init_conversion_failed.cpp
+++ b/clang/test/SemaCXX/err_init_conversion_failed.cpp
@@ -56,6 +56,6 @@ template <class P> struct S2 {
 
 void test_15() {
   S2<S> X = {&S::foo};
-  // expected-error-re@-1{{cannot initialize a member subobject of type 'void (S::*)(const int &){{( __attribute__\(\(thiscall\)\))?}}' with an rvalue of type 'void (S::*)(int){{( __attribute__\(\(thiscall\)\))?}}': type mismatch at 1st parameter ('const int &' vs 'int')}}
+  // expected-error-re@-1{{cannot initialize a member subobject of type 'void (template_test::S::*)(const int &){{( __attribute__\(\(thiscall\)\))?}}' with an rvalue of type 'void (S::*)(int){{( __attribute__\(\(thiscall\)\))?}}': type mismatch at 1st parameter ('const int &' vs 'int')}}
 }
 }
diff --git a/clang/test/SemaCXX/gh102293.cpp b/clang/test/SemaCXX/gh102293.cpp
index fe417e697841b..37b674d98a282 100644
--- a/clang/test/SemaCXX/gh102293.cpp
+++ b/clang/test/SemaCXX/gh102293.cpp
@@ -35,7 +35,7 @@ class bar {       // expected-note {{definition of 'GH104802::bar' is not comple
 
 class baz {       // expected-note {{definition of 'GH104802::baz' is not complete until the closing '}'}}
   typedef class baz blech;
-  blech a;        // expected-error {{field has incomplete type 'blech' (aka 'GH104802::baz')}}
+  blech a;        // expected-error {{field has incomplete type 'blech' (aka 'class baz')}}
 
   virtual int c();
 };
diff --git a/clang/test/SemaCXX/incomplete-call.cpp b/clang/test/SemaCXX/incomplete-call.cpp
index 0846c88cd68f4..8f24d455c4110 100644
--- a/clang/test/SemaCXX/incomplete-call.cpp
+++ b/clang/test/SemaCXX/incomplete-call.cpp
@@ -59,7 +59,7 @@ namespace pr18542 {
     int count;
     template<typename CharT> class basic_istream;
     template<typename CharT>
-      void basic_istream<CharT>::read() { // expected-error{{out-of-line definition of 'read' from class 'basic_istream<CharT>' without definition}}
+      void basic_istream<CharT>::read() { // expected-error{{out-of-line definition of 'read' from class 'pr18542::X::basic_istream<CharT>' without definition}}
         count = 0;
       }
   };
diff --git a/clang/test/SemaCXX/matrix-casts.cpp b/clang/test/SemaCXX/matrix-casts.cpp
index c0f3df1ae8106..708b3fa3f3d16 100644
--- a/clang/test/SemaCXX/matrix-casts.cpp
+++ b/clang/test/SemaCXX/matrix-casts.cpp
@@ -35,7 +35,7 @@ void f1() {
   (matrix_4_4<char>)v; // expected-error {{C-style cast from 'vec' (vector of 1 'int' value) to 'matrix_4_4<char>' (aka 'char __attribute__((matrix_type(4, 4)))') is not allowed}}
 
   (test_struct *)m1;    // expected-error {{cannot cast from type 'matrix_4_4<char>' (aka 'char __attribute__((matrix_type(4, 4)))') to pointer type 'test_struct *'}}
-  (matrix_5_5<float>)s; // expected-error {{C-style cast from 'test_struct *' to 'matrix_5_5<float>' (aka 'float __attribute__((matrix_type(5, 5)))') is not allowed}}
+  (matrix_5_5<float>)s; // expected-error {{C-style cast from 'test_struct *' (aka 'struct test_struct *') to 'matrix_5_5<float>' (aka 'float __attribute__((matrix_type(5, 5)))') is not allowed}}
 }
 
 void f2() {
@@ -59,7 +59,7 @@ void f2() {
   static_cast<matrix_4_4<char>>(v); // expected-error {{static_cast from 'vec' (vector of 1 'int' value) to 'matrix_4_4<char>' (aka 'char __attribute__((matrix_type(4, 4)))') is not allowed}}
 
   static_cast<test_struct *>(m1);    // expected-error {{cannot cast from type 'matrix_4_4<char>' (aka 'char __attribute__((matrix_type(4, 4)))') to pointer type 'test_struct *'}}
-  static_cast<matrix_5_5<float>>(s); // expected-error {{static_cast from 'test_struct *' to 'matrix_5_5<float>' (aka 'float __attribute__((matrix_type(5, 5)))') is not allowed}}
+  static_cast<matrix_5_5<float>>(s); // expected-error {{static_cast from 'test_struct *' (aka 'struct test_struct *') to 'matrix_5_5<float>' (aka 'float __attribute__((matrix_type(5, 5)))') is not allowed}}
 }
 
 void f3() {
diff --git a/clang/test/SemaCXX/nested-name-spec.cpp b/clang/test/SemaCXX/nested-name-spec.cpp
index fedbb3070070b..c60275b709ddb 100644
--- a/clang/test/SemaCXX/nested-name-spec.cpp
+++ b/clang/test/SemaCXX/nested-name-spec.cpp
@@ -15,7 +15,9 @@ namespace A {
     static int Ag1();
     static int Ag2();
   };
-  int ax; // expected-note {{'ax' declared here}}
+  int ax;
+  // expected-note@-1 {{'ax' declared here}}
+  // expected-note@-2 {{'::A::ax' declared here}}
   void Af();
 }
 
@@ -100,7 +102,7 @@ void f3() {
   N::x = 0; // expected-error {{'N' is not a class, namespace, or enumeration}}
   { int A;           A::ax = 0; }
   { typedef int A;   A::ax = 0; } // expected-error{{'A' (aka 'int') is not a class, namespace, or enumeration}}
-  { typedef A::C A;  A::ax = 0; } // expected-error {{no member named 'ax'}}
+  { typedef A::C A;  A::ax = 0; } // expected-error {{no member named 'ax' in 'A::C'; did you mean '::A::ax'?}}
   { typedef A::C A;  A::cx = 0; }
 }
 
@@ -474,7 +476,7 @@ namespace A {
 class B {
   typedef C D; // expected-error{{unknown type name 'C'}}
   A::D::F;
-  // expected-error@-1{{'PR30619::A::B::D' (aka 'int') is not a class, namespace, or enumeration}}
+  // expected-error@-1{{'A::D' (aka 'int') is not a class, namespace, or enumeration}}
 };
 }
 }
diff --git a/clang/test/SemaCXX/new-delete.cpp b/clang/test/SemaCXX/new-delete.cpp
index c05130bb30729..1adb993f46c8f 100644
--- a/clang/test/SemaCXX/new-delete.cpp
+++ b/clang/test/SemaCXX/new-delete.cpp
@@ -721,19 +721,7 @@ int (*const_fold)[12] = new int[3][&const_fold + 12 - &const_fold];
 #if __cplusplus >= 201402L && !defined(NEW_INTERP)
 // expected-error@-2 {{array size is not a constant expression}}
 // expected-note@-3 {{cannot refer to element 12 of non-array}}
-#elif __cplusplus < 201103L && !defined(NEW_INTERP)
+#elif __cplusplus < 201103L
 // expected-error@-5 {{cannot allocate object of variably modified type}}
 // expected-warning@-6 {{variable length arrays in C++ are a Clang extension}}
 #endif
-#ifdef NEW_INTERP
-#if __cplusplus >= 201402L
-// expected-error@-10 {{array size is not a constant expression}}
-// expected-note@-11 {{cannot refer to element 12 of non-array}}
-#elif __cplusplus >= 201103L
-// expected-error@-13 {{only the first dimension of an allocated array may have dynamic size}}
-// expected-note@-14 {{cannot refer to element 12 of non-array}}
-#else
-// expected-error@-16 {{only the first dimension of an allocated array may have dynamic size}}
-// expected-note@-17 {{cannot refer to element 12 of non-array}}
-#endif
-#endif
diff --git a/clang/test/SemaCXX/opaque-enum-declaration-in-class-template.cpp b/clang/test/SemaCXX/opaque-enum-declaration-in-class-template.cpp
index 7101a153c6ebb..4257e53ce9af0 100644
--- a/clang/test/SemaCXX/opaque-enum-declaration-in-class-template.cpp
+++ b/clang/test/SemaCXX/opaque-enum-declaration-in-class-template.cpp
@@ -94,7 +94,9 @@ struct S5 {
 };
 
 int X5 = S5<char>::E1{} + '\0'; // expected-error{{invalid operands to binary expression}}
+                                // expected-note@-1{{no implicit conversion for scoped enum; consider casting to underlying type}}
 int Y5 = S5<char>::E2{} + '\0'; // expected-error{{invalid operands to binary expression}}
+                                // expected-note@-1{{no implicit conversion for scoped enum; consider casting to underlying type}}
 
 
 template <typename T>
diff --git a/clang/test/SemaCXX/pointer-forward-declared-class-conversion.cpp b/clang/test/SemaCXX/pointer-forward-declared-class-conversion.cpp
index b0a101e0fdcd5..8d0cdcdc22139 100644
--- a/clang/test/SemaCXX/pointer-forward-declared-class-conversion.cpp
+++ b/clang/test/SemaCXX/pointer-forward-declared-class-conversion.cpp
@@ -12,6 +12,6 @@ A2<int> *a2 = b2; // expected-error{{cannot initialize a variable of type 'A2<in
 
 typedef struct S s;
 const s *f();
-s *g1() { return f(); } // expected-error{{cannot initialize return object of type 's *' (aka 'S *') with an rvalue of type 'const s *' (aka 'const S *')}}
+s *g1() { return f(); } // expected-error{{cannot initialize return object of type 's *' (aka 'struct S *') with an rvalue of type 'const s *' (aka 'const struct S *')}}
 
-B1 *g2() { return f(); } // expected-error{{cannot initialize return object of type 'B1 *' with an rvalue of type 'const s *' (aka 'const S *')}}
+B1 *g2() { return f(); } // expected-error{{cannot initialize return object of type 'B1 *' with an rvalue of type 'const s *' (aka 'const struct S *')}}
diff --git a/clang/test/SemaCXX/pr100095.cpp b/clang/test/SemaCXX/pr100095.cpp
index 15913fec9d5ae..9b8c09cb82977 100644
--- a/clang/test/SemaCXX/pr100095.cpp
+++ b/clang/test/SemaCXX/pr100095.cpp
@@ -1,5 +1,4 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++11 %s
-// XFAIL: asserts
 
 template <class> struct Pair;
 template <class...> struct Tuple {
diff --git a/clang/test/SemaCXX/pseudo-destructors.cpp b/clang/test/SemaCXX/pseudo-destructors.cpp
index 812029285c4e0..55849d69b9b38 100644
--- a/clang/test/SemaCXX/pseudo-destructors.cpp
+++ b/clang/test/SemaCXX/pseudo-destructors.cpp
@@ -116,7 +116,7 @@ typedef Derived *Foo;
 
 void test2(Foo d) {
   d.~Foo(); // This is ok
-  d.~Derived(); // expected-error {{member reference type 'Foo' (aka 'dotPointerAccess::Derived *') is a pointer; did you mean to use '->'}}
+  d.~Derived(); // expected-error {{member reference type 'Foo' (aka 'Derived *') is a pointer; did you mean to use '->'}}
 }
 }
 
diff --git a/clang/test/SemaCXX/return.cpp b/clang/test/SemaCXX/return.cpp
index 796c9ae91dedc..60accbde9d701 100644
--- a/clang/test/SemaCXX/return.cpp
+++ b/clang/test/SemaCXX/return.cpp
@@ -115,9 +115,9 @@ namespace ctor_returns_void {
   };
 
   template <typename T> struct ST {
-    ST() { return f(); } // expected-error {{constructor 'ST<T>' must not return void expression}}
+    ST() { return f(); } // expected-error {{constructor 'ctor_returns_void::ST<T>' must not return void expression}}
                          // expected-error@-1 {{constructor 'ST' must not return void expression}}
-    ~ST() { return f(); } // expected-error {{destructor '~ST<T>' must not return void expression}}
+    ~ST() { return f(); } // expected-error {{destructor '~ctor_returns_void::ST<T>' must not return void expression}}
                           // expected-error@-1 {{destructor '~ST' must not return void expression}}
   };
 
diff --git a/clang/test/SemaCXX/static-assert.cpp b/clang/test/SemaCXX/static-assert.cpp
index bf6a2eeb432a3..354016db36432 100644
--- a/clang/test/SemaCXX/static-assert.cpp
+++ b/clang/test/SemaCXX/static-assert.cpp
@@ -199,7 +199,7 @@ void foo2() {
   // FIXME: Here the template keyword is dropped because the failed condition
   // for a static assert is always printed with canonical types.
   static_assert(::ns::NestedTemplates1<T, a>::NestedTemplates2::template NestedTemplates3<U>::value, "message");
-  // expected-error@-1{{static assertion failed due to requirement '::ns::NestedTemplates1<int, 3>::NestedTemplates2::NestedTemplates3<float>::value': message}}
+  // expected-error@-1{{static assertion failed due to requirement 'ns::NestedTemplates1<int, 3>::NestedTemplates2::NestedTemplates3<float>::value': message}}
 }
 template void foo2<int, float, 3>();
 // expected-note@-1{{in instantiation of function template specialization 'foo2<int, float, 3>' requested here}}
diff --git a/clang/test/SemaCXX/sugar-common-types.cpp b/clang/test/SemaCXX/sugar-common-types.cpp
index d58f6cdd900fc..dd5fc4a654795 100644
--- a/clang/test/SemaCXX/sugar-common-types.cpp
+++ b/clang/test/SemaCXX/sugar-common-types.cpp
@@ -44,7 +44,8 @@ template <class T> struct S1 {
 };
 
 N t10 = 0 ? S1<X1>() : S1<Y1>(); // expected-error {{from 'S1<B1>' (aka 'S1<int>')}}
-N t11 = 0 ? S1<X1>::S2<X2>() : S1<Y1>::S2<Y2>(); // expected-error {{from 'S1<B1>::S2<B2>' (aka 'S2<void>')}}
+// FIXME: needs to compute common sugar for qualified template names
+N t11 = 0 ? S1<X1>::S2<X2>() : S1<Y1>::S2<Y2>(); // expected-error {{from 'S1<int>::S2<B2>' (aka 'S1<int>::S2<void>')}}
 
 template <class T> using Al = S1<T>;
 
@@ -200,5 +201,5 @@ namespace member_pointers {
 
   // FIXME: adjusted MemberPointer does not preserve qualifier
   N t3 = 0 ? &W1::a : &W2::b;
-  // expected-error@-1 {{rvalue of type 'B1 W<void>::*'}}
+  // expected-error@-1 {{rvalue of type 'B1 member_pointers::W<void>::*'}}
 } // namespace member_pointers
diff --git a/clang/test/SemaCXX/sugared-auto.cpp b/clang/test/SemaCXX/sugared-auto.cpp
index b5bb4f0f85a77..cf879efd393d0 100644
--- a/clang/test/SemaCXX/sugared-auto.cpp
+++ b/clang/test/SemaCXX/sugared-auto.cpp
@@ -54,7 +54,7 @@ N t4 = x4; // expected-error {{lvalue of type 'Man' (aka 'int')}}
 N t5 = x5; // expected-error {{lvalue of type 'Dog' (aka 'int')}}
 
 auto x6 = { Man(), Dog() };
-N t6 = x6; // expected-error {{from 'std::initializer_list<Animal>' (aka 'initializer_list<int>')}}
+N t6 = x6; // expected-error {{from 'std::initializer_list<Animal>' (aka 'std::initializer_list<int>')}}
 
 } // namespace variable
 
diff --git a/clang/test/SemaCXX/type-aware-coroutines.cpp b/clang/test/SemaCXX/type-aware-coroutines.cpp
index 742e5f02b6dab..e41d07b9bccf5 100644
--- a/clang/test/SemaCXX/type-aware-coroutines.cpp
+++ b/clang/test/SemaCXX/type-aware-coroutines.cpp
@@ -93,7 +93,7 @@ struct resumable5 {
 };
 
 resumable f1(int) {
-  // expected-error@-1 {{'operator new' provided by 'std::coroutine_traits<resumable, int>::promise_type' (aka 'resumable::promise_type') is not usable with the function signature of 'f1'}}
+  // expected-error@-1 {{'operator new' provided by 'std::coroutine_traits<resumable, int>::promise_type' (aka 'typename resumable::promise_type') is not usable with the function signature of 'f1'}}
   // expected-note@-2 {{type aware 'operator new' will not be used for coroutine allocation}}
   // expected-note@#resumable_tan1 {{type aware 'operator new' declared here}}
   // expected-note@#resumable_tan2 {{type aware 'operator new' declared here}}
@@ -101,7 +101,7 @@ resumable f1(int) {
 }
 
 resumable f2(float) {
-  // expected-error@-1 {{'operator new' provided by 'std::coroutine_traits<resumable, float>::promise_type' (aka 'resumable::promise_type') is not usable with the function signature of 'f2'}}
+  // expected-error@-1 {{'operator new' provided by 'std::coroutine_traits<resumable, float>::promise_type' (aka 'typename resumable::promise_type') is not usable with the function signature of 'f2'}}
   // expected-note@-2 {{type aware 'operator new' will not be used for coroutine allocation}}
   // expected-note@#resumable_tan1 {{type aware 'operator new' declared here}}
   // expected-note@#resumable_tan2 {{type aware 'operator new' declared here}}
@@ -109,7 +109,7 @@ resumable f2(float) {
 }
 
 resumable2 f3(int, float, const char*, Allocator) {
-  // expected-error@-1 {{'operator new' provided by 'std::coroutine_traits<resumable2, int, float, const char *, Allocator>::promise_type' (aka 'resumable2::promise_type') is not usable with the function signature of 'f3'}}
+  // expected-error@-1 {{'operator new' provided by 'std::coroutine_traits<resumable2, int, float, const char *, Allocator>::promise_type' (aka 'typename resumable2::promise_type') is not usable with the function signature of 'f3'}}
   // expected-note@-2 {{type aware 'operator new' will not be used for coroutine allocation}}
   // expected-note@#resumable2_tan1 {{type aware 'operator new' declared here}}
   co_yield 1;
@@ -117,7 +117,7 @@ resumable2 f3(int, float, const char*, Allocator) {
 }
 
 resumable f4(int n = 10) {
-  // expected-error@-1 {{'operator new' provided by 'std::coroutine_traits<resumable, int>::promise_type' (aka 'resumable::promise_type') is not usable with the function signature of 'f4'}}
+  // expected-error@-1 {{'operator new' provided by 'std::coroutine_traits<resumable, int>::promise_type' (aka 'typename resumable::promise_type') is not usable with the function signature of 'f4'}}
   // expected-note@-2 {{type aware 'operator new' will not be used for coroutine allocation}}
   // expected-note@#resumable_tan1 {{type aware 'operator new' declared here}}
   // expected-note@#resumable_tan2 {{type aware 'operator new' declared here}}
diff --git a/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp b/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
index f3ddbbfe15bdc..7c6c9ea4dde80 100644
--- a/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
+++ b/clang/test/SemaCXX/type-traits-unsatisfied-diags-std.cpp
@@ -50,6 +50,14 @@ struct is_constructible {
 
 template <typename... Args>
 constexpr bool is_constructible_v = __is_constructible(Args...);
+
+template <typename T>
+struct is_final {
+    static constexpr bool value = __is_final(T);
+};
+template <typename T>
+constexpr bool is_final_v = __is_final(T);
+
 #endif
 
 #ifdef STD2
@@ -116,6 +124,16 @@ using is_constructible  = __details_is_constructible<Args...>;
 
 template <typename... Args>
 constexpr bool is_constructible_v = __is_constructible(Args...);
+
+template <typename T>
+struct __details_is_final {
+    static constexpr bool value = __is_final(T);
+};
+template <typename T>
+using is_final = __details_is_final<T>;
+template <typename T>
+constexpr bool is_final_v = __is_final(T);
+
 #endif
 
 
@@ -177,6 +195,14 @@ using is_constructible  = __details_is_constructible<Args...>;
 
 template <typename... Args>
 constexpr bool is_constructible_v = is_constructible<Args...>::value;
+
+template <typename T>
+struct __details_is_final : bool_constant<__is_final(T)> {};
+template <typename T>
+using is_final = __details_is_final<T>;
+template <typename T>
+constexpr bool is_final_v = is_final<T>::value;
+
 #endif
 
 }
@@ -248,6 +274,31 @@ static_assert(std::is_constructible_v<void>);
 // expected-error@-1 {{static assertion failed due to requirement 'std::is_constructible_v<void>'}} \
 // expected-note@-1 {{because it is a cv void type}}
 
+static_assert(!std::is_final<int>::value);
+
+static_assert(std::is_final<int&>::value);
+// expected-error-re@-1 {{static assertion failed due to requirement 'std::{{.*}}is_final<int &>::value'}} \
+// expected-note@-1 {{'int &' is not final}} \
+// expected-note@-1 {{because it is a reference type}} \
+// expected-note@-1 {{because it is not a class or union type}}
+
+static_assert(std::is_final_v<int&>);
+// expected-error@-1 {{static assertion failed due to requirement 'std::is_final_v<int &>'}} \
+// expected-note@-1 {{'int &' is not final}} \
+// expected-note@-1 {{because it is a reference type}} \
+// expected-note@-1 {{because it is not a class or union type}}
+
+using Arr = int[3];
+static_assert(std::is_final<Arr>::value);
+// expected-error-re@-1 {{static assertion failed due to requirement 'std::{{.*}}is_final<int[3]>::value'}} \
+// expected-note@-1 {{'Arr' (aka 'int[3]') is not final}} \
+// expected-note@-1 {{because it is not a class or union type}}
+
+static_assert(std::is_final_v<Arr>);
+// expected-error@-1 {{static assertion failed due to requirement 'std::is_final_v<int[3]>'}} \
+// expected-note@-1 {{'int[3]' is not final}} \
+// expected-note@-1 {{because it is not a class or union type}}
+
 namespace test_namespace {
     using namespace std;
     static_assert(is_trivially_relocatable<int&>::value);
@@ -300,6 +351,31 @@ namespace test_namespace {
     static_assert(is_constructible_v<void>);
     // expected-error@-1 {{static assertion failed due to requirement 'is_constructible_v<void>'}} \
     // expected-note@-1 {{because it is a cv void type}}
+
+    static_assert(is_final<int&>::value);
+    // expected-error-re@-1 {{static assertion failed due to requirement '{{.*}}is_final<int &>::value'}} \
+    // expected-note@-1 {{'int &' is not final}} \
+    // expected-note@-1 {{because it is a reference type}} \
+    // expected-note@-1 {{because it is not a class or union type}}
+
+    static_assert(is_final_v<int&>);
+    // expected-error@-1 {{static assertion failed due to requirement 'is_final_v<int &>'}} \
+    // expected-note@-1 {{'int &' is not final}} \
+    // expected-note@-1 {{because it is a reference type}} \
+    // expected-note@-1 {{because it is not a class or union type}}
+
+    using A = int[2];
+    static_assert(is_final<A>::value);
+    // expected-error-re@-1 {{static assertion failed due to requirement '{{.*}}is_final<int[2]>::value'}} \
+    // expected-note@-1 {{'A' (aka 'int[2]') is not final}} \
+    // expected-note@-1 {{because it is not a class or union type}}
+
+    using Fn = void();
+    static_assert(is_final<Fn>::value);
+    // expected-error-re@-1 {{static assertion failed due to requirement '{{.*}}is_final<void ()>::value'}} \
+    // expected-note@-1 {{'Fn' (aka 'void ()') is not final}} \
+    // expected-note@-1 {{because it is a function type}} \
+    // expected-note@-1 {{because it is not a class or union type}}
 }
 
 
diff --git a/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp b/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
index 54806a93ddf80..1619b0b22b85f 100644
--- a/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
+++ b/clang/test/SemaCXX/type-traits-unsatisfied-diags.cpp
@@ -829,3 +829,46 @@ static_assert(__is_standard_layout(H)); // no diagnostics
 static_assert(__is_standard_layout(I)); // no diagnostics
 }
 
+namespace is_final_tests {
+    struct C {}; // #e-C
+    static_assert(__is_final(C));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_final(is_final_tests::C)'}} \
+    // expected-note@-1 {{'C' is not final}} \
+    // expected-note@-1 {{because it is not marked 'final'}} \
+    // expected-note@#e-C {{'C' defined here}}
+
+    union U {}; // #e-U
+    static_assert(__is_final(U));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_final(is_final_tests::U)'}} \
+    // expected-note@-1 {{'U' is not final}} \
+    // expected-note@-1 {{because it is not marked 'final'}} \
+    // expected-note@#e-U {{'U' defined here}}
+
+    // ----- non-class/union types -----
+    using I = int;
+    static_assert(__is_final(I));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_final(int)'}} \
+    // expected-note@-1 {{'I' (aka 'int') is not final}} \
+    // expected-note@-1 {{because it is not a class or union type}}
+
+    using Fty = void(); // function type
+    static_assert(__is_final(Fty));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_final(void ())'}} \
+    // expected-note@-1 {{'Fty' (aka 'void ()') is not final}} \
+    // expected-note@-1 {{because it is a function type}} \
+    // expected-note@-1 {{because it is not a class or union type}}
+
+    using Arr = int[3];
+    static_assert(__is_final(Arr));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_final(int[3])'}} \
+    // expected-note@-1 {{'Arr' (aka 'int[3]') is not final}} \
+    // expected-note@-1 {{because it is not a class or union type}}
+
+    using Ref = int&;
+    static_assert(__is_final(Ref));
+    // expected-error@-1 {{static assertion failed due to requirement '__is_final(int &)'}} \
+    // expected-note@-1 {{'Ref' (aka 'int &') is not final}} \
+    // expected-note@-1 {{because it is a reference type}} \
+    // expected-note@-1 {{because it is not a class or union type}}
+
+}
diff --git a/clang/test/SemaCXX/undefined-partial-specialization.cpp b/clang/test/SemaCXX/undefined-partial-specialization.cpp
index 0f776a6145fef..3992d02f479f0 100644
--- a/clang/test/SemaCXX/undefined-partial-specialization.cpp
+++ b/clang/test/SemaCXX/undefined-partial-specialization.cpp
@@ -10,6 +10,6 @@ template <typename T>
 class boo<T, true>;
 
 template<typename T>
-void boo<T, true>::foo(){} // expected-error{{out-of-line definition of 'foo' from class 'boo<T, true>' without definition}}
+void boo<T, true>::foo(){} // expected-error{{out-of-line definition of 'foo' from class 'GH61356::boo<T, true>' without definition}}
 
 }
diff --git a/clang/test/SemaCXX/using-decl-templates.cpp b/clang/test/SemaCXX/using-decl-templates.cpp
index c96c4879cb68c..58b30595b148c 100644
--- a/clang/test/SemaCXX/using-decl-templates.cpp
+++ b/clang/test/SemaCXX/using-decl-templates.cpp
@@ -153,3 +153,11 @@ T foo(T t) { // OK
 }
 } // namespace sss
 } // namespace func_templ
+
+namespace DependentName {
+  template <typename T> struct S {
+    using typename T::Ty;
+    static Ty Val;
+  };
+  template <typename T> typename S<T>::Ty S<T>::Val;
+} // DependentName
diff --git a/clang/test/SemaCXX/vector-bool.cpp b/clang/test/SemaCXX/vector-bool.cpp
index cd638056f348b..0db04dcdb014f 100644
--- a/clang/test/SemaCXX/vector-bool.cpp
+++ b/clang/test/SemaCXX/vector-bool.cpp
@@ -112,3 +112,7 @@ void Sizeof() {
   static_assert(sizeof(Bool195) == 32);
   static_assert(sizeof(Bool257) == 64);
 }
+
+#if !__has_feature(ext_vector_type_boolean)
+#error "FAIL"
+#endif
diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-libc-functions.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-libc-functions.cpp
index 6ae329a736d91..765dcbcc07df5 100644
--- a/clang/test/SemaCXX/warn-unsafe-buffer-usage-libc-functions.cpp
+++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-libc-functions.cpp
@@ -4,8 +4,16 @@
 // RUN:            -verify %s -x objective-c++
 // RUN: %clang_cc1 -std=c++20 -Wno-all -Wunsafe-buffer-usage-in-libc-call \
 // RUN:            -verify %s
+// RUN: %clang_cc1 -std=c++20 -Wno-all -Wunsafe-buffer-usage-in-libc-call \
+// RUN:            -verify %s -DTEST_STD_NS
 
 typedef struct {} FILE;
+typedef unsigned int size_t;
+
+#ifdef TEST_STD_NS
+namespace std {
+#endif
+
 void memcpy();
 void __asan_memcpy();
 void strcpy();
@@ -25,6 +33,11 @@ int sscanf(const char * buffer, const char * format, ... );
 int wprintf(const wchar_t* format, ... );
 int __asan_printf();
 
+#ifdef TEST_STD_NS
+} //namespace std
+using namespace std;
+#endif
+
 namespace std {
   template< class InputIt, class OutputIt >
   OutputIt copy( InputIt first, InputIt last,
@@ -64,10 +77,6 @@ namespace std {
 
   typedef basic_string_view<char> string_view;
   typedef basic_string_view<wchar_t> wstring_view;
-
-  // C function under std:
-  void memcpy();
-  void strcpy();
 }
 
 void f(char * p, char * q, std::span<char> s, std::span<char> s2) {
@@ -77,14 +86,16 @@ void f(char * p, char * q, std::span<char> s, std::span<char> s2) {
   aligned_char_ptr_t cp;
 
   memcpy();                   // expected-warning{{function 'memcpy' is unsafe}}
-  std::memcpy();              // expected-warning{{function 'memcpy' is unsafe}}
   __builtin_memcpy(p, q, 64); // expected-warning{{function '__builtin_memcpy' is unsafe}}
   __builtin___memcpy_chk(p, q, 8, 64);  // expected-warning{{function '__builtin___memcpy_chk' is unsafe}}
   __asan_memcpy();                      // expected-warning{{function '__asan_memcpy' is unsafe}}
   strcpy();                   // expected-warning{{function 'strcpy' is unsafe}}
-  std::strcpy();              // expected-warning{{function 'strcpy' is unsafe}}
   strcpy_s();                 // expected-warning{{function 'strcpy_s' is unsafe}}
   wcscpy_s();                 // expected-warning{{function 'wcscpy_s' is unsafe}}
+#ifdef TEST_STD_NS
+  std::strcpy();              // expected-warning{{function 'strcpy' is unsafe}}
+  std::memcpy();              // expected-warning{{function 'memcpy' is unsafe}}
+#endif
 
   /* Test printfs */
   fprintf((FILE*)p, "%s%d", p, *p);  // expected-warning{{function 'fprintf' is unsafe}} expected-note{{string argument is not guaranteed to be null-terminated}}
@@ -181,13 +192,59 @@ void ff(char * p, char * q, std::span<char> s, std::span<char> s2) {
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wunsafe-buffer-usage-in-libc-call"
   memcpy();
-  std::memcpy();
   __builtin_memcpy(p, q, 64);
   __builtin___memcpy_chk(p, q, 8, 64);
   __asan_memcpy();
   strcpy();
+#ifdef TEST_STD_NS
   std::strcpy();
+  std::memcpy();
+#endif
   strcpy_s();
   wcscpy_s();
 #pragma clang diagnostic pop
 }
+
+
+
+// functions not in global scope or std:: namespace are not libc
+// functions regardless of their names:
+struct StrBuff
+{
+  void strcpy();
+  void strcpy(char* dst);
+  void memcpy(void *dst, const void *src, size_t size);
+};
+
+namespace NS {
+  void strcpy();
+  void strcpy(char* dst);
+  void memcpy(void *dst, const void *src, size_t size);
+}
+
+namespace std {
+  // class methods even in std namespace cannot be libc functions:
+  struct LibC
+  {
+    void strcpy();
+    void strcpy(char* dst);
+    void memcpy(void *dst, const void *src, size_t size);
+  };
+}
+
+void test(StrBuff& str)
+{
+  char buff[64];
+  str.strcpy();
+  str.strcpy(buff);
+  str.memcpy(buff, buff, 64);
+  NS::strcpy();
+  NS::strcpy(buff);
+  NS::memcpy(buff, buff, 64);
+
+  std::LibC LibC;
+
+  LibC.strcpy();
+  LibC.strcpy(buff);
+  LibC.memcpy(buff, buff, 64);
+}
diff --git a/clang/test/SemaCXX/warn-unused-result.cpp b/clang/test/SemaCXX/warn-unused-result.cpp
index 447654eccd563..098817729efb1 100644
--- a/clang/test/SemaCXX/warn-unused-result.cpp
+++ b/clang/test/SemaCXX/warn-unused-result.cpp
@@ -309,7 +309,7 @@ void use() {
 
   S<double>(2);     // no warning
   S<int>(2);        // expected-warning {{ignoring temporary of type 'S<int>' declared with 'nodiscard'}}
-  S<const char>(2); // no warning (warn_unused_result does not diagnose constructor temporaries)
+  S<const char>(2); // expected-warning {{ignoring temporary of type 'S<const char>' declared with 'clang::warn_unused_result' attribute}}
 
   // function should take precedence over type
   obtain2(1.0);             // expected-warning {{ignoring return value of function declared with 'nodiscard'}}
@@ -336,7 +336,7 @@ struct [[nodiscard]] G {
 void use2() {
   H{2};       // no warning
   H(2.0);     // expected-warning {{ignoring temporary created by a constructor declared with 'nodiscard'}}
-  H("Hello"); // no warning (warn_unused_result does not diagnose constructor temporaries)
+  H("Hello"); // expected-warning {{ignoring temporary created by a constructor declared with 'warn_unused_result' attribute}}
 
   // no warning for explicit cast to void
   (void)H(2);
@@ -407,3 +407,88 @@ void doGccThings() {
 }
 
 } // namespace BuildStringOnClangScope
+
+namespace candiscard {
+
+struct [[nodiscard]] NoDiscard {
+  [[nodiscard]] NoDiscard(int);
+  NoDiscard(const char *);
+};
+
+struct [[gnu::warn_unused]] WarnUnused {
+  [[gnu::warn_unused]] WarnUnused(int); // expected-warning {{'gnu::warn_unused' attribute only applies to structs, unions, and classes}}
+  WarnUnused(const char*);
+};
+
+struct [[gnu::warn_unused_result]] WarnUnusedResult {
+  [[gnu::warn_unused_result]] WarnUnusedResult(int);
+  WarnUnusedResult(const char*);
+};
+
+NoDiscard return_nodiscard();
+WarnUnused return_warnunused();
+WarnUnusedResult return_warnunusedresult();
+
+NoDiscard (*p_return_nodiscard)();
+WarnUnused (*p_return_warnunused)();
+WarnUnusedResult (*p_return_warnunusedresult)();
+
+NoDiscard (*(*pp_return_nodiscard)())();
+WarnUnused (*(*pp_return_warnunused)())();
+WarnUnusedResult (*(*pp_return_warnunusedresult)())();
+
+template <class T> T from_a_template();
+
+void test() {
+  // Unused but named variables
+  NoDiscard unused_variable1(1);         // no warning
+  NoDiscard unused_variable2("");        // no warning
+  WarnUnused unused_variable3(1);        // no warning
+  WarnUnused unused_variable4("");       // no warning
+  WarnUnusedResult unused_variable5(1);  // no warning
+  WarnUnusedResult unused_variable6(""); // no warning
+
+  // Constructor return values
+  NoDiscard(1);         // expected-warning {{ignoring temporary created by a constructor declared with 'nodiscard' attribute}}
+  NoDiscard("");        // expected-warning {{ignoring temporary of type 'NoDiscard' declared with 'nodiscard' attribute}}
+  WarnUnused(1);        // expected-warning {{expression result unused}}
+  WarnUnused("");       // expected-warning {{expression result unused}}
+  WarnUnusedResult(1);  // expected-warning {{ignoring temporary created by a constructor declared with 'gnu::warn_unused_result' attribute}}
+  WarnUnusedResult(""); // expected-warning {{ignoring temporary of type 'WarnUnusedResult' declared with 'gnu::warn_unused_result' attribute}}
+
+  NoDiscard{1};         // expected-warning {{ignoring temporary created by a constructor declared with 'nodiscard' attribute}}
+  NoDiscard{""};        // expected-warning {{ignoring temporary of type 'NoDiscard' declared with 'nodiscard' attribute}}
+  WarnUnused{1};        // expected-warning {{expression result unused}}
+  WarnUnused{""};       // expected-warning {{expression result unused}}
+  WarnUnusedResult{1};  // expected-warning {{ignoring temporary created by a constructor declared with 'gnu::warn_unused_result' attribute}}
+  WarnUnusedResult{""}; // expected-warning {{ignoring temporary of type 'WarnUnusedResult' declared with 'gnu::warn_unused_result' attribute}}
+
+  static_cast<NoDiscard>(1);         // expected-warning {{ignoring temporary created by a constructor declared with 'nodiscard' attribute}}
+  static_cast<NoDiscard>("");        // expected-warning {{ignoring temporary of type 'NoDiscard' declared with 'nodiscard' attribute}}
+  static_cast<WarnUnused>(1);        // expected-warning {{expression result unused}}
+  static_cast<WarnUnused>("");       // expected-warning {{expression result unused}}
+  static_cast<WarnUnusedResult>(1);  // expected-warning {{ignoring temporary created by a constructor declared with 'gnu::warn_unused_result' attribute}}
+  static_cast<WarnUnusedResult>(""); // expected-warning {{ignoring temporary of type 'WarnUnusedResult' declared with 'gnu::warn_unused_result' attribute}}
+
+  // Function return values
+  return_nodiscard(); // expected-warning {{ignoring return value of type 'NoDiscard' declared with 'nodiscard' attribute}}
+  return_warnunused(); // no warning
+  return_warnunusedresult(); // expected-warning {{ignoring return value of type 'WarnUnusedResult' declared with 'gnu::warn_unused_result' attribute}}
+
+  // Function pointer return values
+  p_return_nodiscard(); // expected-warning {{ignoring return value of type 'NoDiscard' declared with 'nodiscard' attribute}}
+  p_return_warnunused(); // no warning
+  p_return_warnunusedresult(); // expected-warning {{ignoring return value of type 'WarnUnusedResult' declared with 'gnu::warn_unused_result' attribute}}
+
+  // Function pointer expression return values
+  pp_return_nodiscard()(); // expected-warning {{ignoring return value of type 'NoDiscard' declared with 'nodiscard' attribute}}
+  pp_return_warnunused()(); // no warning
+  pp_return_warnunusedresult()(); // expected-warning {{ignoring return value of type 'WarnUnusedResult' declared with 'gnu::warn_unused_result' attribute}}
+
+  // From a template
+  from_a_template<NoDiscard>(); // expected-warning {{ignoring return value of type 'NoDiscard' declared with 'nodiscard' attribute}}
+  from_a_template<WarnUnused>(); // no warning
+  from_a_template<WarnUnusedResult>(); // expected-warning {{ignoring return value of type 'WarnUnusedResult' declared with 'gnu::warn_unused_result' attribute}}
+}
+
+} // namespace candiscard
diff --git a/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp b/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp
index 8beffcd39e85c..06db972118d1c 100644
--- a/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp
+++ b/clang/test/SemaCXX/wmissing-noreturn-suggestion.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fsyntax-only -fcxx-exceptions -fexceptions -Wreturn-type -Wmissing-noreturn -verify %s
+// RUN: %clang_cc1 -fsyntax-only -fcxx-exceptions -fexceptions -Wreturn-type -Wmissing-noreturn -verify=expected,cxx17 -std=c++17 %s
+// RUN: %clang_cc1 -fsyntax-only -fcxx-exceptions -fexceptions -Wreturn-type -Wmissing-noreturn -verify=expected,cxx23 -std=c++23 %s
 
 namespace std {
   class string {
@@ -16,6 +17,15 @@ void throwError(const std::string& msg) { // expected-warning {{function 'throwE
   throw std::runtime_error(msg);
 }
 
+// Using the [[noreturn]] attribute on lambdas is not available until C++23,
+// so we should not emit the -Wmissing-noreturn warning on earlier standards.
+// Clang supports the attribute on earlier standards as an extension, and emits
+// the c++23-lambda-attributes warning.
+void lambda() {
+  auto l1 = []              () { throw std::runtime_error("ERROR"); }; // cxx23-warning {{function 'operator()' could be declared with attribute 'noreturn'}}
+  auto l2 = [] [[noreturn]] () { throw std::runtime_error("ERROR"); }; // cxx17-warning {{an attribute specifier sequence in this position is a C++23 extension}}
+}
+
 // The non-void caller should not warn about missing return.
 int ensureZero(int i) {
   if (i == 0) return 0;
diff --git a/clang/test/SemaHIP/amdgcnspirv-implicit-alloc-function-calling-conv.hip b/clang/test/SemaHIP/amdgcnspirv-implicit-alloc-function-calling-conv.hip
new file mode 100644
index 0000000000000..c3e7e1a5cd59e
--- /dev/null
+++ b/clang/test/SemaHIP/amdgcnspirv-implicit-alloc-function-calling-conv.hip
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 %s -fcuda-is-device -std=c++17 -triple spirv32 -verify
+// RUN: %clang_cc1 %s -fcuda-is-device -std=c++17 -triple spirv64 -verify
+// RUN: %clang_cc1 %s -fcuda-is-device -std=c++17 -triple spirv64-amd-amdhsa -verify
+// RUN: %clang_cc1 %s -fcuda-is-device -std=c++17 -triple spirv32 -aux-triple x86_64-unknown-linux-gnu -verify
+// RUN: %clang_cc1 %s -fcuda-is-device -std=c++17 -triple spirv64 -aux-triple x86_64-unknown-linux-gnu -verify
+// RUN: %clang_cc1 %s -fcuda-is-device -std=c++17 -triple spirv64-amd-amdhsa -aux-triple x86_64-unknown-linux-gnu -verify
+// RUN: %clang_cc1 %s -fcuda-is-device -std=c++17 -triple spirv32 -aux-triple x86_64-pc-windows-msvc -verify
+// RUN: %clang_cc1 %s -fcuda-is-device -std=c++17 -triple spirv64 -aux-triple x86_64-pc-windows-msvc -verify
+// RUN: %clang_cc1 %s -fcuda-is-device -std=c++17 -triple spirv64-amd-amdhsa -aux-triple x86_64-pc-windows-msvc -verify
+
+// expected-no-diagnostics
+
+namespace std
+{
+    enum class align_val_t : __SIZE_TYPE__ {};
+    struct nothrow_t { explicit nothrow_t() = default; };
+    extern nothrow_t const nothrow;
+}
+
+void* __attribute__((cdecl)) operator new(__SIZE_TYPE__);
+void* __attribute__((cdecl)) operator new[](__SIZE_TYPE__);
+void* __attribute__((cdecl)) operator new(__SIZE_TYPE__, ::std::align_val_t);
+void* __attribute__((cdecl)) operator new[](__SIZE_TYPE__, ::std::align_val_t);
+
+void __attribute__((cdecl)) operator delete(void*) noexcept;
+void __attribute__((cdecl)) operator delete[](void*) noexcept;
+void __attribute__((cdecl)) operator delete(void*, __SIZE_TYPE__) noexcept;
+void __attribute__((cdecl)) operator delete[](void*, __SIZE_TYPE__) noexcept;
+void __attribute__((cdecl)) operator delete(void*, ::std::align_val_t) noexcept;
+void __attribute__((cdecl)) operator delete[](void*, ::std::align_val_t) noexcept;
+void __attribute__((cdecl)) operator delete(void*, __SIZE_TYPE__, ::std::align_val_t) noexcept;
+void __attribute__((cdecl)) operator delete[](void*, __SIZE_TYPE__, ::std::align_val_t) noexcept;
diff --git a/clang/test/SemaObjC/attr-nodiscard.m b/clang/test/SemaObjC/attr-nodiscard.m
index 6d04665da25ca..26bbd247d4a3d 100644
--- a/clang/test/SemaObjC/attr-nodiscard.m
+++ b/clang/test/SemaObjC/attr-nodiscard.m
@@ -4,6 +4,9 @@
 
 typedef struct expected E;
 
+[[nodiscard]] typedef int NI; // expected-warning {{'[[nodiscard]]' attribute ignored when applied to a typedef}}
+typedef __attribute__((warn_unused_result)) int WUR;
+
 @interface INTF
 - (int) a [[nodiscard]];
 + (int) b [[nodiscard]];
@@ -12,6 +15,8 @@ + (struct expected) d;
 - (E) e;
 + (E) f;
 - (void) g [[nodiscard]]; // expected-warning {{attribute 'nodiscard' cannot be applied to Objective-C method without return value}}
+- (NI) h;
+- (WUR) i;
 @end
 
 void foo(INTF *a) {
@@ -21,5 +26,7 @@ void foo(INTF *a) {
   [INTF d]; // expected-warning {{ignoring return value of type 'expected' declared with 'nodiscard' attribute}}
   [a e]; // expected-warning {{ignoring return value of type 'expected' declared with 'nodiscard' attribute}}
   [INTF f]; // expected-warning {{ignoring return value of type 'expected' declared with 'nodiscard' attribute}}
-  [a g];
+  [a g]; // no warning because g returns void
+  [a h]; // no warning because attribute is ignored when applied to a typedef
+  [a i]; // expected-warning {{ignoring return value of type 'WUR' declared with 'warn_unused_result' attribute}}
 }
diff --git a/clang/test/SemaObjC/exprs.m b/clang/test/SemaObjC/exprs.m
index dcf46d3cdbfbc..c42d270657c10 100644
--- a/clang/test/SemaObjC/exprs.m
+++ b/clang/test/SemaObjC/exprs.m
@@ -36,3 +36,10 @@ void test_encode(void) {
   (void)@encode(Incomplete_ObjC_class*);
   (void)@encode(id);
 }
+
+void gh154046(void) {
+  (void)(const char[]) {
+    [0] = @encode(int),  // expected-error {{incompatible pointer to integer conversion initializing 'const char' with an expression of type 'char[2]'}}
+    [1] = @encode(float) // expected-error {{incompatible pointer to integer conversion initializing 'const char' with an expression of type 'char[2]'}}
+  }[1];
+}
diff --git a/clang/test/SemaObjC/ptrauth-qualifier.m b/clang/test/SemaObjC/ptrauth-qualifier.m
index 74bbe6f09899b..67a73bbe45777 100644
--- a/clang/test/SemaObjC/ptrauth-qualifier.m
+++ b/clang/test/SemaObjC/ptrauth-qualifier.m
@@ -1,13 +1,25 @@
-// RUN: %clang_cc1 -triple arm64-apple-ios -fsyntax-only -verify -fptrauth-intrinsics %s
+// RUN: %clang_cc1 -triple arm64-apple-ios -DIS_DARWIN -fsyntax-only -verify -fptrauth-intrinsics %s
 // RUN: %clang_cc1 -triple aarch64-linux-gnu -fsyntax-only -verify -fptrauth-intrinsics %s
 
-#if !__has_extension(ptrauth_qualifier)
+#if defined(IS_DARWIN) && !__has_extension(ptrauth_qualifier)
 // This error means that the __ptrauth qualifier availability test says  that it
 // is not available. This error is not expected in the output, if it is seen
 // there is a feature detection regression.
 #error __ptrauth qualifier not enabled
 #endif
 
+#if defined(IS_DARWIN) && !__has_feature(ptrauth_qualifier)
+// This error means that the __has_feature test for ptrauth_qualifier has
+// failed, despite it being expected on darwin.
+#error __ptrauth qualifier not enabled
+#elif !defined(IS_DARWIN) && (__has_feature(ptrauth_qualifier) || __has_extension(ptrauth_qualifier))
+#error ptrauth_qualifier labeled a feature on a non-darwin platform
+#endif
+
+#if !defined (__PTRAUTH__)
+#error __PTRAUTH__ test macro not defined when ptrauth is enabled
+#endif
+
 @interface Foo
 // expected-warning@-1 {{class 'Foo' defined without specifying a base class}}
 // expected-note@-2 {{add a super class to fix this problem}}
diff --git a/clang/test/SemaObjCXX/Inputs/nullability-pragmas-1.h b/clang/test/SemaObjCXX/Inputs/nullability-pragmas-1.h
index 91753ca7f3ca5..9d5329c9d03c4 100644
--- a/clang/test/SemaObjCXX/Inputs/nullability-pragmas-1.h
+++ b/clang/test/SemaObjCXX/Inputs/nullability-pragmas-1.h
@@ -39,7 +39,7 @@ A *f14(void);
 
 int * _Null_unspecified f15(void);
 A * _Null_unspecified f16(void);
-void f17(CFErrorRef *error); // expected-note{{no known conversion from 'A * _Nonnull' to 'CFErrorRef  _Nullable * _Nullable' (aka '__CFError **') for 1st argument}}
+void f17(CFErrorRef *error); // expected-note{{no known conversion from 'A * _Nonnull' to 'CFErrorRef  _Nullable * _Nullable' (aka 'struct __CFError **') for 1st argument}}
 void f18(A **); // expected-warning 2{{pointer is missing a nullability type specifier}}
 // expected-note@-1 2 {{insert '_Nullable' if the pointer may be null}}
 // expected-note@-2 2 {{insert '_Nonnull' if the pointer should never be null}}
diff --git a/clang/test/SemaObjCXX/arc-bridged-cast.mm b/clang/test/SemaObjCXX/arc-bridged-cast.mm
index 1f68897dede57..67769ebabad33 100644
--- a/clang/test/SemaObjCXX/arc-bridged-cast.mm
+++ b/clang/test/SemaObjCXX/arc-bridged-cast.mm
@@ -63,7 +63,7 @@ void testObjCBridgeId() {
 void testExternC() {
   id obj;
   obj = (id)r0;
-  obj = (id)r1; // expected-error{{cast of C pointer type 'CFAnnotatedObjectRef' (aka 'const __CFAnnotatedObject *') to Objective-C pointer type 'id' requires a bridged cast}} expected-note{{use __bridge to convert directly}} expected-note{{use __bridge_transfer to transfer ownership of a +1 'CFAnnotatedObjectRef'}}
+  obj = (id)r1; // expected-error{{cast of C pointer type 'CFAnnotatedObjectRef' (aka 'const struct __CFAnnotatedObject *') to Objective-C pointer type 'id' requires a bridged cast}} expected-note{{use __bridge to convert directly}} expected-note{{use __bridge_transfer to transfer ownership of a +1 'CFAnnotatedObjectRef'}}
   obj = (id)r2;
-  obj = (id)r3; // expected-error{{cast of C pointer type 'CFAnnotatedObjectRef' (aka 'const __CFAnnotatedObject *') to Objective-C pointer type 'id' requires a bridged cast}} expected-note{{use __bridge to convert directly}} expected-note{{use __bridge_transfer to transfer ownership of a +1 'CFAnnotatedObjectRef'}}
+  obj = (id)r3; // expected-error{{cast of C pointer type 'CFAnnotatedObjectRef' (aka 'const struct __CFAnnotatedObject *') to Objective-C pointer type 'id' requires a bridged cast}} expected-note{{use __bridge to convert directly}} expected-note{{use __bridge_transfer to transfer ownership of a +1 'CFAnnotatedObjectRef'}}
 }
diff --git a/clang/test/SemaObjCXX/attr-nodiscard.mm b/clang/test/SemaObjCXX/attr-nodiscard.mm
index e1eefb74d3961..18d829632e428 100644
--- a/clang/test/SemaObjCXX/attr-nodiscard.mm
+++ b/clang/test/SemaObjCXX/attr-nodiscard.mm
@@ -5,6 +5,9 @@
 
 using E = expected<int>;
 
+using NI [[nodiscard]] = int; // expected-warning {{'[[nodiscard]]' attribute ignored when applied to a typedef}}
+using WURI [[clang::warn_unused_result]] = int;
+
 @interface INTF
 - (int) a [[nodiscard]];
 + (int) b [[nodiscard]];
@@ -13,6 +16,8 @@ + (int) b [[nodiscard]];
 - (E) e;
 + (E) f;
 - (void) g [[nodiscard]]; // expected-warning {{attribute 'nodiscard' cannot be applied to Objective-C method without return value}}
+- (NI) h;
+- (WURI) i;
 @end
 
 void foo(INTF *a) {
@@ -22,5 +27,7 @@ void foo(INTF *a) {
   [INTF d]; // expected-warning {{ignoring return value of type 'expected<int>' declared with 'nodiscard' attribute}}
   [a e]; // expected-warning {{ignoring return value of type 'expected<int>' declared with 'nodiscard' attribute}}
   [INTF f]; // expected-warning {{ignoring return value of type 'expected<int>' declared with 'nodiscard' attribute}}
-  [a g];
+  [a g]; // no warning because g returns void
+  [a h]; // no warning because attribute is ignored
+  [a i]; // expected-warning {{ignoring return value of type 'WURI' declared with 'clang::warn_unused_result' attribute}}
 }
diff --git a/clang/test/SemaObjCXX/discarded-block-type-inference.mm b/clang/test/SemaObjCXX/discarded-block-type-inference.mm
new file mode 100644
index 0000000000000..8e2587724a7f6
--- /dev/null
+++ b/clang/test/SemaObjCXX/discarded-block-type-inference.mm
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -fobjc-arc -fblocks %s
+
+void  block_receiver(int (^)() );
+
+int f1() {
+  if constexpr (0)
+    (block_receiver)(^{ return 2; });
+  return 1;
+}
+
+int f2() {
+  if constexpr (0)
+    return (^{ return 2; })();
+  return 1;
+}
diff --git a/clang/test/SemaObjCXX/objc-boxed-expressions-nsvalue.mm b/clang/test/SemaObjCXX/objc-boxed-expressions-nsvalue.mm
index 3b590c46f004b..57aaa62221b65 100644
--- a/clang/test/SemaObjCXX/objc-boxed-expressions-nsvalue.mm
+++ b/clang/test/SemaObjCXX/objc-boxed-expressions-nsvalue.mm
@@ -81,7 +81,7 @@ int main() {
   id edge_insets_object = @(edge_insets);
 
   SomeStruct s;
-  id err = @(s); // expected-error{{illegal type 'SomeStruct' (aka '_SomeStruct') used in a boxed expression}}
+  id err = @(s); // expected-error{{illegal type 'SomeStruct' (aka 'struct _SomeStruct') used in a boxed expression}}
 
   NonTriviallyCopyable ntc;
   id ntcErr = @(ntc); // expected-error{{non-trivially copyable type 'NonTriviallyCopyable' cannot be used in a boxed expression}}
@@ -99,7 +99,7 @@ SomeStruct getSomeStruct() {
 
 void rvalue() {
   id rv_rect = @(getRect());
-  id rv_some_struct = @(getSomeStruct()); // expected-error {{illegal type 'SomeStruct' (aka '_SomeStruct') used in a boxed expression}}
+  id rv_some_struct = @(getSomeStruct()); // expected-error {{illegal type 'SomeStruct' (aka 'struct _SomeStruct') used in a boxed expression}}
 }
 
 template <class T> id box(T value) { return @(value); } // expected-error{{non-trivially copyable type 'NonTriviallyCopyable' cannot be used in a boxed expression}}
diff --git a/clang/test/SemaObjCXX/objcbridge-attribute-arc.mm b/clang/test/SemaObjCXX/objcbridge-attribute-arc.mm
index 0a86ee9f329ff..0d3751f3fd121 100644
--- a/clang/test/SemaObjCXX/objcbridge-attribute-arc.mm
+++ b/clang/test/SemaObjCXX/objcbridge-attribute-arc.mm
@@ -26,10 +26,10 @@ @protocol NSTesting @end
 typedef struct __attribute__((objc_bridge(NSTesting))) __CFError *CFTestingRef; // expected-note {{declared here}}
 
 id Test1(CFTestingRef cf) {
-  return (NSString *)cf; // expected-error {{CF object of type 'CFTestingRef' (aka '__CFError *') is bridged to 'NSTesting', which is not an Objective-C class}} \
-                         // expected-error {{cast of C pointer type 'CFTestingRef' (aka '__CFError *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
+  return (NSString *)cf; // expected-error {{CF object of type 'CFTestingRef' (aka 'struct __CFError *') is bridged to 'NSTesting', which is not an Objective-C class}} \
+                         // expected-error {{cast of C pointer type 'CFTestingRef' (aka 'struct __CFError *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
                          // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-                         // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFTestingRef' (aka '__CFError *') into ARC}}
+                         // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFTestingRef' (aka 'struct __CFError *') into ARC}}
 }
 
 typedef CFErrorRef CFErrorRef1;
@@ -52,105 +52,105 @@ @interface NSUColor @end
 @class NSString;
 
 void Test2(CFErrorRef2 cf, NSError *ns, NSString *str, Class c, CFUColor2Ref cf2) {
-  (void)(NSString *)cf; // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'NSString'}} \
-                        // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
+  (void)(NSString *)cf; // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'NSString'}} \
+                        // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
                         // expected-note {{__bridge to convert directly (no change in ownership)}} \
-                        // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}}
-  (void)(NSError *)cf; // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'NSError *' requires a bridged cast}} \
+                        // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}}
+  (void)(NSError *)cf; // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'NSError *' requires a bridged cast}} \
                        // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-                       // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}}
-  (void)(MyError*)cf; // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'MyError *' requires a bridged cast}} \
+                       // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}}
+  (void)(MyError*)cf; // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'MyError *' requires a bridged cast}} \
                         // expected-note {{__bridge to convert directly (no change in ownership)}} \
-                        // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}} \
-			// expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'MyError'}}
-  (void)(NSUColor *)cf2; // expected-error {{cast of C pointer type 'CFUColor2Ref' (aka '__CFUPrimeColor *') to Objective-C pointer type 'NSUColor *' requires a bridged cast}} \
+                        // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}} \
+			// expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'MyError'}}
+  (void)(NSUColor *)cf2; // expected-error {{cast of C pointer type 'CFUColor2Ref' (aka 'union __CFUPrimeColor *') to Objective-C pointer type 'NSUColor *' requires a bridged cast}} \
                          // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-                         // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFUColor2Ref' (aka '__CFUPrimeColor *') into ARC}}
-  (void)(CFErrorRef)ns; // expected-error {{cast of Objective-C pointer type 'NSError *' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+                         // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFUColor2Ref' (aka 'union __CFUPrimeColor *') into ARC}}
+  (void)(CFErrorRef)ns; // expected-error {{cast of Objective-C pointer type 'NSError *' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
                         // expected-note {{use __bridge to convert directly (no change in ownership)}} \
- 			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
-  (void)(CFErrorRef)str;  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *')}} \\
-                          // expected-error {{cast of Objective-C pointer type 'NSString *' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+ 			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+  (void)(CFErrorRef)str;  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *')}} \\
+                          // expected-error {{cast of Objective-C pointer type 'NSString *' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
                         // expected-note {{use __bridge to convert directly (no change in ownership)}} \
- 			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
-  (void)(Class)cf; // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'Class'}} \\
-                   // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'Class' requires a bridged cast}} \
+ 			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+  (void)(Class)cf; // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'Class'}} \\
+                   // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'Class' requires a bridged cast}} \
  			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}}
-  (void)(CFErrorRef)c; // expected-warning {{'__unsafe_unretained Class' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *}} \\
-                       // expected-error {{cast of Objective-C pointer type 'Class' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}}
+  (void)(CFErrorRef)c; // expected-warning {{'__unsafe_unretained Class' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *'}} \\
+                       // expected-error {{cast of Objective-C pointer type 'Class' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
 }
 
 
 void Test3(CFErrorRef cf, NSError *ns) {
-  (void)(id)cf; // expected-error {{cast of C pointer type 'CFErrorRef' (aka '__CFErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
+  (void)(id)cf; // expected-error {{cast of C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka '__CFErrorRef *') into ARC}}
- (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFErrorRef' (aka '__CFErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka 'struct __CFErrorRef *') into ARC}}
+ (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka '__CFErrorRef *') into ARC}}
- (void)(id<P1, P2, P4>)cf; // expected-warning {{'CFErrorRef' (aka '__CFErrorRef *') bridges to NSError, not 'id<P1,P2,P4>'}} \
-                           // expected-error {{cast of C pointer type 'CFErrorRef' (aka '__CFErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka 'struct __CFErrorRef *') into ARC}}
+ (void)(id<P1, P2, P4>)cf; // expected-warning {{'CFErrorRef' (aka 'struct __CFErrorRef *') bridges to NSError, not 'id<P1,P2,P4>'}} \
+                           // expected-error {{cast of C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka '__CFErrorRef *') into ARC}}
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka 'struct __CFErrorRef *') into ARC}}
 }
 
 void Test4(CFMyErrorRef cf) {
-   (void)(id)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
+   (void)(id)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)(id<P1, P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)(id<P1, P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)(id<P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P2,P3>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)(id<P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P2,P3>' requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)(id<P1, P2, P4>)cf; // expected-warning {{'CFMyErrorRef' (aka '__CFMyErrorRef *') bridges to MyError, not 'id<P1,P2,P4>'}} \
-                           // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)(id<P1, P2, P4>)cf; // expected-warning {{'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') bridges to MyError, not 'id<P1,P2,P4>'}} \
+                           // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
+				// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
 }
 
 void Test5(id<P1, P2, P3> P123, id ID, id<P1, P2, P3, P4> P1234, id<P1, P2> P12, id<P2, P3> P23) {
- (void)(CFErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+ (void)(CFErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)(CFErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)(CFErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)(CFErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)(CFErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)(CFErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)(CFErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)(CFErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)(CFErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
 }
 
 void Test6(id<P1, P2, P3> P123, id ID, id<P1, P2, P3, P4> P1234, id<P1, P2> P12, id<P2, P3> P23) {
 
- (void)(CFMyErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+ (void)(CFMyErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
- (void)(CFMyErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
+ (void)(CFMyErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
- (void)(CFMyErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
+ (void)(CFMyErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
- (void)(CFMyErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
+ (void)(CFMyErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
- (void)(CFMyErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
+ (void)(CFMyErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
 }
 
 typedef struct __attribute__ ((objc_bridge(MyPersonalError))) __CFMyPersonalErrorRef * CFMyPersonalErrorRef;  // expected-note 1 {{declared here}}
@@ -159,51 +159,51 @@ @interface MyPersonalError : NSError <P4> // expected-note 1 {{declared here}}
 @end
 
 void Test7(id<P1, P2, P3> P123, id ID, id<P1, P2, P3, P4> P1234, id<P1, P2> P12, id<P2, P3> P23) {
- (void)(CFMyPersonalErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+ (void)(CFMyPersonalErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
- (void)(CFMyPersonalErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
+ (void)(CFMyPersonalErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
- (void)(CFMyPersonalErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
+ (void)(CFMyPersonalErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
- (void)(CFMyPersonalErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
+ (void)(CFMyPersonalErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
- (void)(CFMyPersonalErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
+ (void)(CFMyPersonalErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
 }
 
 void Test8(CFMyPersonalErrorRef cf) {
-  (void)(id)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
+  (void)(id)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1, P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1, P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1, P2, P3, P4>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3,P4>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1, P2, P3, P4>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3,P4>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1, P2, P3, P4, P5>)cf; // expected-warning {{'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') bridges to MyPersonalError, not 'id<P1,P2,P3,P4,P5>'}} \
-                                    // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3,P4,P5>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1, P2, P3, P4, P5>)cf; // expected-warning {{'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') bridges to MyPersonalError, not 'id<P1,P2,P3,P4,P5>'}} \
+                                    // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3,P4,P5>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
 }
 
 void Test9(CFErrorRef2 cf, NSError *ns, NSString *str, Class c, CFUColor2Ref cf2) {
-  (void)(__bridge NSString *)cf; // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'NSString'}}
+  (void)(__bridge NSString *)cf; // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'NSString'}}
   (void)(__bridge NSError *)cf; // okay
   (void)(__bridge NSUColor *)cf2; // okay
   (void)(__bridge CFErrorRef)ns; // okay
-  (void)(__bridge CFErrorRef)str;  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *')}}
-  (void)(__bridge Class)cf; // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'Class'}}
-  (void)(__bridge CFErrorRef)c; // expected-warning {{'__unsafe_unretained Class' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *')}}
+  (void)(__bridge CFErrorRef)str;  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+  (void)(__bridge Class)cf; // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'Class'}}
+  (void)(__bridge CFErrorRef)c; // expected-warning {{'__unsafe_unretained Class' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
 }
diff --git a/clang/test/SemaObjCXX/objcbridge-attribute.mm b/clang/test/SemaObjCXX/objcbridge-attribute.mm
index 907163b7115b3..c04fba1c1d84f 100644
--- a/clang/test/SemaObjCXX/objcbridge-attribute.mm
+++ b/clang/test/SemaObjCXX/objcbridge-attribute.mm
@@ -26,10 +26,10 @@ @protocol NSTesting @end
 typedef struct __attribute__((objc_bridge(NSTesting))) __CFError *CFTestingRef; // expected-note {{declared here}}
 
 id Test1(CFTestingRef cf) {
-  return (NSString *)cf; // expected-error {{CF object of type 'CFTestingRef' (aka '__CFError *') is bridged to 'NSTesting', which is not an Objective-C class}} \
-                         // expected-error {{cast of C pointer type 'CFTestingRef' (aka '__CFError *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
+  return (NSString *)cf; // expected-error {{CF object of type 'CFTestingRef' (aka 'struct __CFError *') is bridged to 'NSTesting', which is not an Objective-C class}} \
+                         // expected-error {{cast of C pointer type 'CFTestingRef' (aka 'struct __CFError *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
                          // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-                         // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFTestingRef' (aka '__CFError *') into ARC}}
+                         // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFTestingRef' (aka 'struct __CFError *') into ARC}}
 }
 
 typedef CFErrorRef CFErrorRef1;
@@ -52,105 +52,105 @@ @interface NSUColor @end
 @class NSString;
 
 void Test2(CFErrorRef2 cf, NSError *ns, NSString *str, Class c, CFUColor2Ref cf2) {
-  (void)(NSString *)cf; // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'NSString'}} \
-                        // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
+  (void)(NSString *)cf; // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'NSString'}} \
+                        // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
                         // expected-note {{__bridge to convert directly (no change in ownership)}} \
-                        // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}}
-  (void)(NSError *)cf; // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'NSError *' requires a bridged cast}} \
+                        // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}}
+  (void)(NSError *)cf; // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'NSError *' requires a bridged cast}} \
                        // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-                       // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}}
-  (void)(MyError*)cf; // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'MyError *' requires a bridged cast}} \
+                       // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}}
+  (void)(MyError*)cf; // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'MyError *' requires a bridged cast}} \
                         // expected-note {{__bridge to convert directly (no change in ownership)}} \
-                        // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}} \
-			// expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'MyError'}}
-  (void)(NSUColor *)cf2; // expected-error {{cast of C pointer type 'CFUColor2Ref' (aka '__CFUPrimeColor *') to Objective-C pointer type 'NSUColor *' requires a bridged cast}} \
+                        // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}} \
+			// expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'MyError'}}
+  (void)(NSUColor *)cf2; // expected-error {{cast of C pointer type 'CFUColor2Ref' (aka 'union __CFUPrimeColor *') to Objective-C pointer type 'NSUColor *' requires a bridged cast}} \
                          // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-                         // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFUColor2Ref' (aka '__CFUPrimeColor *') into ARC}}
-  (void)(CFErrorRef)ns; // expected-error {{cast of Objective-C pointer type 'NSError *' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+                         // expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFUColor2Ref' (aka 'union __CFUPrimeColor *') into ARC}}
+  (void)(CFErrorRef)ns; // expected-error {{cast of Objective-C pointer type 'NSError *' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
                         // expected-note {{use __bridge to convert directly (no change in ownership)}} \
- 			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
-  (void)(CFErrorRef)str;  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *')}} \\
-                          // expected-error {{cast of Objective-C pointer type 'NSString *' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+ 			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+  (void)(CFErrorRef)str;  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *')}} \\
+                          // expected-error {{cast of Objective-C pointer type 'NSString *' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
                         // expected-note {{use __bridge to convert directly (no change in ownership)}} \
- 			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
-  (void)(Class)cf; // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'Class'}} \\
-                   // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'Class' requires a bridged cast}} \
+ 			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+  (void)(Class)cf; // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'Class'}} \\
+                   // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'Class' requires a bridged cast}} \
  			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}}
-  (void)(CFErrorRef)c; // expected-warning {{'__unsafe_unretained Class' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *')}} \
-                       // expected-error {{cast of Objective-C pointer type 'Class' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}}
+  (void)(CFErrorRef)c; // expected-warning {{'__unsafe_unretained Class' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *')}} \
+                       // expected-error {{cast of Objective-C pointer type 'Class' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
 }
 
 
 void Test3(CFErrorRef cf, NSError *ns) {
-  (void)(id)cf; // expected-error {{cast of C pointer type 'CFErrorRef' (aka '__CFErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
+  (void)(id)cf; // expected-error {{cast of C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka '__CFErrorRef *') into ARC}}
- (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFErrorRef' (aka '__CFErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka 'struct __CFErrorRef *') into ARC}}
+ (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka '__CFErrorRef *') into ARC}}
- (void)(id<P1, P2, P4>)cf; // expected-warning {{'CFErrorRef' (aka '__CFErrorRef *') bridges to NSError, not 'id<P1,P2,P4>'}} \
-                           // expected-error {{cast of C pointer type 'CFErrorRef' (aka '__CFErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka 'struct __CFErrorRef *') into ARC}}
+ (void)(id<P1, P2, P4>)cf; // expected-warning {{'CFErrorRef' (aka 'struct __CFErrorRef *') bridges to NSError, not 'id<P1,P2,P4>'}} \
+                           // expected-error {{cast of C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka '__CFErrorRef *') into ARC}}
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFErrorRef' (aka 'struct __CFErrorRef *') into ARC}}
 }
 
 void Test4(CFMyErrorRef cf) {
-   (void)(id)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
+   (void)(id)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)(id<P1, P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)(id<P1, P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)(id<P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P2,P3>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)(id<P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P2,P3>' requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)(id<P1, P2, P4>)cf; // expected-warning {{'CFMyErrorRef' (aka '__CFMyErrorRef *') bridges to MyError, not 'id<P1,P2,P4>'}} \
-                           // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)(id<P1, P2, P4>)cf; // expected-warning {{'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') bridges to MyError, not 'id<P1,P2,P4>'}} \
+                           // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
+				// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
 }
 
 void Test5(id<P1, P2, P3> P123, id ID, id<P1, P2, P3, P4> P1234, id<P1, P2> P12, id<P2, P3> P23) {
- (void)(CFErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+ (void)(CFErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)(CFErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)(CFErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)(CFErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)(CFErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)(CFErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)(CFErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)(CFErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)(CFErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
+			// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
 }
 
 void Test6(id<P1, P2, P3> P123, id ID, id<P1, P2, P3, P4> P1234, id<P1, P2> P12, id<P2, P3> P23) {
 
- (void)(CFMyErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+ (void)(CFMyErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
- (void)(CFMyErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
+ (void)(CFMyErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
- (void)(CFMyErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
+ (void)(CFMyErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
- (void)(CFMyErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
+ (void)(CFMyErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
- (void)(CFMyErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') requires a bridged cast}} \
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
+ (void)(CFMyErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') requires a bridged cast}} \
 				// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *')}}
+				// expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *')}}
 }
 
 typedef struct __attribute__ ((objc_bridge(MyPersonalError))) __CFMyPersonalErrorRef * CFMyPersonalErrorRef;  // expected-note 1 {{declared here}}
@@ -159,52 +159,52 @@ @interface MyPersonalError : NSError <P4> // expected-note 1 {{declared here}}
 @end
 
 void Test7(id<P1, P2, P3> P123, id ID, id<P1, P2, P3, P4> P1234, id<P1, P2> P12, id<P2, P3> P23) {
- (void)(CFMyPersonalErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+ (void)(CFMyPersonalErrorRef)ID; // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
- (void)(CFMyPersonalErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
+ (void)(CFMyPersonalErrorRef)P123; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
- (void)(CFMyPersonalErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
+ (void)(CFMyPersonalErrorRef)P1234; // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
- (void)(CFMyPersonalErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
+ (void)(CFMyPersonalErrorRef)P12; // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
- (void)(CFMyPersonalErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') requires a bridged cast}} \
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
+ (void)(CFMyPersonalErrorRef)P23; // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') requires a bridged cast}} \
 				 // expected-note {{use __bridge to convert directly (no change in ownership)}} \
-				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *')}}
+				 // expected-note {{use __bridge_retained to make an ARC object available as a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *')}}
 }
 
 void Test8(CFMyPersonalErrorRef cf) {
-  (void)(id)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
+  (void)(id)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1, P2>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1, P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1, P2, P3>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1, P2, P3, P4>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3,P4>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1, P2, P3, P4>)cf; // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3,P4>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
-  (void)(id<P1, P2, P3, P4, P5>)cf; // expected-warning {{'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') bridges to MyPersonalError, not 'id<P1,P2,P3,P4,P5>'}} \
-                                    // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3,P4,P5>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
+  (void)(id<P1, P2, P3, P4, P5>)cf; // expected-warning {{'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') bridges to MyPersonalError, not 'id<P1,P2,P3,P4,P5>'}} \
+                                    // expected-error {{cast of C pointer type 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') to Objective-C pointer type 'id<P1,P2,P3,P4,P5>' requires a bridged cast}} \
 		// expected-note {{use __bridge to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka '__CFMyPersonalErrorRef *') into ARC}}
+		// expected-note {{use __bridge_transfer to transfer ownership of a +1 'CFMyPersonalErrorRef' (aka 'struct __CFMyPersonalErrorRef *') into ARC}}
 }
 
 void Test9(CFErrorRef2 cf, NSError *ns, NSString *str, Class c, CFUColor2Ref cf2) {
-  (void)(__bridge NSString *)cf; // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'NSString'}}
+  (void)(__bridge NSString *)cf; // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'NSString'}}
   (void)(__bridge NSError *)cf; // okay
-  (void)(__bridge MyError*)cf; // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'MyError'}} 
+  (void)(__bridge MyError*)cf; // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'MyError'}}
   (void)(__bridge NSUColor *)cf2; // okay
   (void)(__bridge CFErrorRef)ns; // okay
-  (void)(__bridge CFErrorRef)str;  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *')}}
-  (void)(__bridge Class)cf; // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'Class'}}
-  (void)(__bridge CFErrorRef)c; // expected-warning {{'__unsafe_unretained Class' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *')}}
+  (void)(__bridge CFErrorRef)str;  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+  (void)(__bridge Class)cf; // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'Class'}}
+  (void)(__bridge CFErrorRef)c; // expected-warning {{'__unsafe_unretained Class' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
 }
diff --git a/clang/test/SemaObjCXX/objcbridge-related-attribute.mm b/clang/test/SemaObjCXX/objcbridge-related-attribute.mm
index 66755f3d413fd..b835a104e47cf 100644
--- a/clang/test/SemaObjCXX/objcbridge-related-attribute.mm
+++ b/clang/test/SemaObjCXX/objcbridge-related-attribute.mm
@@ -13,14 +13,14 @@ - (NSColor *)backgroundColor;
 
 
 NSColor *Test1(NSColor *nsColor, CGColorRef newColor) {
-  nsColor = newColor; // expected-error {{'CGColorRef' (aka 'CGColor *') must be explicitly converted to 'NSColor *'; use '+colorWithCGColor:' method for this conversion}}
-  NSColor *ns = newColor; // expected-error {{'CGColorRef' (aka 'CGColor *') must be explicitly converted to 'NSColor *'; use '+colorWithCGColor:' method for this conversion}} 
-  return newColor; // expected-error {{'CGColorRef' (aka 'CGColor *') must be explicitly converted to 'NSColor *'; use '+colorWithCGColor:' method for this conversion}} 
+  nsColor = newColor; // expected-error {{'CGColorRef' (aka 'struct CGColor *') must be explicitly converted to 'NSColor *'; use '+colorWithCGColor:' method for this conversion}}
+  NSColor *ns = newColor; // expected-error {{'CGColorRef' (aka 'struct CGColor *') must be explicitly converted to 'NSColor *'; use '+colorWithCGColor:' method for this conversion}}
+  return newColor; // expected-error {{'CGColorRef' (aka 'struct CGColor *') must be explicitly converted to 'NSColor *'; use '+colorWithCGColor:' method for this conversion}}
 }
 
 CGColorRef Test2(NSColor *newColor, CGColorRef cgColor) {
-  cgColor = newColor; // expected-error {{'NSColor *' must be explicitly converted to 'CGColorRef' (aka 'CGColor *'); use '-CGColor' method for this conversion}}
-  CGColorRef cg = newColor; // expected-error {{'NSColor *' must be explicitly converted to 'CGColorRef' (aka 'CGColor *'); use '-CGColor' method for this conversion}} 
-  return newColor; // expected-error {{'NSColor *' must be explicitly converted to 'CGColorRef' (aka 'CGColor *'); use '-CGColor' method for this conversion}}
+  cgColor = newColor; // expected-error {{'NSColor *' must be explicitly converted to 'CGColorRef' (aka 'struct CGColor *'); use '-CGColor' method for this conversion}}
+  CGColorRef cg = newColor; // expected-error {{'NSColor *' must be explicitly converted to 'CGColorRef' (aka 'struct CGColor *'); use '-CGColor' method for this conversion}}
+  return newColor; // expected-error {{'NSColor *' must be explicitly converted to 'CGColorRef' (aka 'struct CGColor *'); use '-CGColor' method for this conversion}}
 }
 
diff --git a/clang/test/SemaObjCXX/objcbridge-static-cast.mm b/clang/test/SemaObjCXX/objcbridge-static-cast.mm
index 6cb913792a6eb..ad939da9762a4 100644
--- a/clang/test/SemaObjCXX/objcbridge-static-cast.mm
+++ b/clang/test/SemaObjCXX/objcbridge-static-cast.mm
@@ -26,10 +26,10 @@ @protocol NSTesting @end
 typedef struct __attribute__((objc_bridge(NSTesting))) __CFError *CFTestingRef; // expected-note {{declared here}}
 
 id Test1(CFTestingRef cf) {
-  return static_cast<NSString *>(cf); // expected-error {{CF object of type 'CFTestingRef' (aka '__CFError *') is bridged to 'NSTesting', which is not an Objective-C class}} \
-                         // expected-error {{cast of C pointer type 'CFTestingRef' (aka '__CFError *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
+  return static_cast<NSString *>(cf); // expected-error {{CF object of type 'CFTestingRef' (aka 'struct __CFError *') is bridged to 'NSTesting', which is not an Objective-C class}} \
+                         // expected-error {{cast of C pointer type 'CFTestingRef' (aka 'struct __CFError *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
 			 // expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-                         // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFTestingRef' (aka '__CFError *') into ARC}}
+                         // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFTestingRef' (aka 'struct __CFError *') into ARC}}
 }
 
 typedef CFErrorRef CFErrorRef1;
@@ -52,84 +52,84 @@ @interface NSUColor @end
 @class NSString;
 
 void Test2(CFErrorRef2 cf, NSError *ns, NSString *str, Class c, CFUColor2Ref cf2) {
-  (void)static_cast<NSString *>(cf); // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'NSString'}} \
-                        // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
+  (void)static_cast<NSString *>(cf); // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'NSString'}} \
+                        // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'NSString *' requires a bridged cast}} \
                         // expected-note {{__bridge with C-style cast to convert directly (no change in ownership)}} \
-                        // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}}
-  (void)static_cast<NSError *>(cf); // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'NSError *' requires a bridged cast}} \
+                        // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}}
+  (void)static_cast<NSError *>(cf); // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'NSError *' requires a bridged cast}} \
                        // expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-                       // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}}
-  (void)static_cast<MyError*>(cf); // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'MyError *' requires a bridged cast}} \
+                       // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}}
+  (void)static_cast<MyError*>(cf); // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'MyError *' requires a bridged cast}} \
                         // expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-                        // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}} \
-			// expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'MyError'}}
-  (void)static_cast<NSUColor *>(cf2); // expected-error {{cast of C pointer type 'CFUColor2Ref' (aka '__CFUPrimeColor *') to Objective-C pointer type 'NSUColor *' requires a bridged cast}} \
+                        // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}} \
+			// expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'MyError'}}
+  (void)static_cast<NSUColor *>(cf2); // expected-error {{cast of C pointer type 'CFUColor2Ref' (aka 'union __CFUPrimeColor *') to Objective-C pointer type 'NSUColor *' requires a bridged cast}} \
                          // expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-                         // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFUColor2Ref' (aka '__CFUPrimeColor *') into ARC}}
-  (void)static_cast<CFErrorRef>(ns); // expected-error {{cast of Objective-C pointer type 'NSError *' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+                         // expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFUColor2Ref' (aka 'union __CFUPrimeColor *') into ARC}}
+  (void)static_cast<CFErrorRef>(ns); // expected-error {{cast of Objective-C pointer type 'NSError *' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
                         // expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
- 			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
-  (void)static_cast<CFErrorRef>(str);  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *')}} \\
-                          // expected-error {{cast of Objective-C pointer type 'NSString *' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+ 			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+  (void)static_cast<CFErrorRef>(str);  // expected-warning {{'NSString' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *')}} \\
+                          // expected-error {{cast of Objective-C pointer type 'NSString *' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
                         // expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
- 			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
-  (void)static_cast<Class>(cf); // expected-warning {{'CFErrorRef2' (aka '__CFErrorRef *') bridges to NSError, not 'Class'}} \\
-                   // expected-error {{cast of C pointer type 'CFErrorRef2' (aka '__CFErrorRef *') to Objective-C pointer type 'Class' requires a bridged cast}} \
+ 			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+  (void)static_cast<Class>(cf); // expected-warning {{'CFErrorRef2' (aka 'struct __CFErrorRef *') bridges to NSError, not 'Class'}} \\
+                   // expected-error {{cast of C pointer type 'CFErrorRef2' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'Class' requires a bridged cast}} \
  			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef2' (aka '__CFErrorRef *') into ARC}}
-  (void)static_cast<CFErrorRef>(c); // expected-warning {{'Class' cannot bridge to 'CFErrorRef' (aka '__CFErrorRef *}} \\
-                       // expected-error {{cast of Objective-C pointer type 'Class' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef2' (aka 'struct __CFErrorRef *') into ARC}}
+  (void)static_cast<CFErrorRef>(c); // expected-warning {{'Class' cannot bridge to 'CFErrorRef' (aka 'struct __CFErrorRef *'}} \\
+                       // expected-error {{cast of Objective-C pointer type 'Class' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
+			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
 }
 
 
 void Test3(CFErrorRef cf, NSError *ns) {
-  (void)static_cast<id>(cf); // expected-error {{cast of C pointer type 'CFErrorRef' (aka '__CFErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
+  (void)static_cast<id>(cf); // expected-error {{cast of C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
 		// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef' (aka '__CFErrorRef *') into ARC}}
- (void)static_cast< id<P1, P2> >(cf); // expected-error {{cast of C pointer type 'CFErrorRef' (aka '__CFErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef' (aka 'struct __CFErrorRef *') into ARC}}
+ (void)static_cast< id<P1, P2> >(cf); // expected-error {{cast of C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
 		// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef' (aka '__CFErrorRef *') into ARC}}
- (void)static_cast< id<P1, P2, P4> >(cf); // expected-warning {{'CFErrorRef' (aka '__CFErrorRef *') bridges to NSError, not 'id<P1,P2,P4>'}} \
-                           // expected-error {{cast of C pointer type 'CFErrorRef' (aka '__CFErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
+		// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef' (aka 'struct __CFErrorRef *') into ARC}}
+ (void)static_cast< id<P1, P2, P4> >(cf); // expected-warning {{'CFErrorRef' (aka 'struct __CFErrorRef *') bridges to NSError, not 'id<P1,P2,P4>'}} \
+                           // expected-error {{cast of C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
 		// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-		// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef' (aka '__CFErrorRef *') into ARC}}
+		// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFErrorRef' (aka 'struct __CFErrorRef *') into ARC}}
 }
 
 void Test4(CFMyErrorRef cf) {
-   (void)static_cast<id>(cf); // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
+   (void)static_cast<id>(cf); // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id' requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)static_cast< id<P1, P2> >(cf); // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)static_cast< id<P1, P2> >(cf); // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2>' requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)static_cast< id<P1, P2, P3> >(cf); // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)static_cast< id<P1, P2, P3> >(cf); // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P3>' requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)static_cast< id<P2, P3> >(cf); // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P2,P3>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)static_cast< id<P2, P3> >(cf); // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P2,P3>' requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
- (void)static_cast< id<P1, P2, P4> >(cf); // expected-warning {{'CFMyErrorRef' (aka '__CFMyErrorRef *') bridges to MyError, not 'id<P1,P2,P4>'}} \
-                           // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka '__CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
+			// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
+ (void)static_cast< id<P1, P2, P4> >(cf); // expected-warning {{'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') bridges to MyError, not 'id<P1,P2,P4>'}} \
+                           // expected-error {{cast of C pointer type 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') to Objective-C pointer type 'id<P1,P2,P4>' requires a bridged cast}} \
 				// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-				// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka '__CFMyErrorRef *') into ARC}}
+				// expected-note {{use __bridge_transfer with C-style cast to transfer ownership of a +1 'CFMyErrorRef' (aka 'struct __CFMyErrorRef *') into ARC}}
 }
 
 void Test5(id<P1, P2, P3> P123, id ID, id<P1, P2, P3, P4> P1234, id<P1, P2> P12, id<P2, P3> P23) {
- (void)static_cast<CFErrorRef>(ID); // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+ (void)static_cast<CFErrorRef>(ID); // expected-error {{cast of Objective-C pointer type 'id' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)static_cast<CFErrorRef>(P123); // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)static_cast<CFErrorRef>(P123); // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)static_cast<CFErrorRef>(P1234); // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)static_cast<CFErrorRef>(P1234); // expected-error {{cast of Objective-C pointer type 'id<P1,P2,P3,P4>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)static_cast<CFErrorRef>(P12); // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)static_cast<CFErrorRef>(P12); // expected-error {{cast of Objective-C pointer type 'id<P1,P2>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
- (void)static_cast<CFErrorRef>(P23); // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFErrorRef' (aka '__CFErrorRef *') requires a bridged cast}} \
+			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
+ (void)static_cast<CFErrorRef>(P23); // expected-error {{cast of Objective-C pointer type 'id<P2,P3>' to C pointer type 'CFErrorRef' (aka 'struct __CFErrorRef *') requires a bridged cast}} \
 			// expected-note {{use __bridge with C-style cast to convert directly (no change in ownership)}} \
-			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka '__CFErrorRef *')}}
+			// expected-note {{use __bridge_retained with C-style cast to make an ARC object available as a +1 'CFErrorRef' (aka 'struct __CFErrorRef *')}}
 }
diff --git a/clang/test/SemaOpenACC/combined-construct-num_workers-ast.cpp b/clang/test/SemaOpenACC/combined-construct-num_workers-ast.cpp
index a04fcdd3903df..cdd6bc77c765f 100644
--- a/clang/test/SemaOpenACC/combined-construct-num_workers-ast.cpp
+++ b/clang/test/SemaOpenACC/combined-construct-num_workers-ast.cpp
@@ -66,7 +66,7 @@ void NormalUses() {
   // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <UserDefinedConversion>
   // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
   // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator int
-  // CHECK-NEXT: DeclRefExpr{{.*}} 'struct CorrectConvert':'CorrectConvert' lvalue Var
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'struct CorrectConvert' lvalue Var
   // CHECK-NEXT: ForStmt
   // CHECK: NullStmt
 }
diff --git a/clang/test/SemaOpenACC/combined-construct-reduction-clause.cpp b/clang/test/SemaOpenACC/combined-construct-reduction-clause.cpp
index 7b1a61e526871..1a5490766e77a 100644
--- a/clang/test/SemaOpenACC/combined-construct-reduction-clause.cpp
+++ b/clang/test/SemaOpenACC/combined-construct-reduction-clause.cpp
@@ -39,8 +39,9 @@ void uses(unsigned Parm) {
 #pragma acc kernels loop reduction(min: CoS, Array[I], Array[0:I])
   for(int i = 0; i < 5; ++i);
 
-  // expected-error@+2{{OpenACC 'reduction' composite variable must not have non-scalar field}}
-  // expected-note@#COS_FIELD{{invalid field is here}}
+  // expected-error@+3{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
 #pragma acc parallel loop reduction(&: ChC)
   for(int i = 0; i < 5; ++i);
 
@@ -166,4 +167,24 @@ void uses(unsigned Parm) {
 #pragma acc parallel loop reduction(&:I)
     for(int i = 0; i < 5; ++i);
   }
+
+  CompositeHasComposite CoCArr[5];
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of array type 'CompositeHasComposite'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
+#pragma acc parallel loop reduction(+:CoCArr)
+    for(int i = 0; i < 5; ++i);
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of array type 'CompositeHasComposite[5]'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
+#pragma acc parallel loop reduction(+:CoCArr[3])
+    for(int i = 0; i < 5; ++i);
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of sub-array type 'CompositeHasComposite'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
+#pragma acc parallel loop reduction(+:CoCArr[1:1])
+    for(int i = 0; i < 5; ++i);
 }
diff --git a/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp b/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp
index 58476dfcee06d..63c5cdee9c966 100644
--- a/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-intexpr-clause-ast.cpp
@@ -73,7 +73,7 @@ void NormalUses() {
   // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <UserDefinedConversion>
   // CHECK-NEXT: CXXMemberCallExpr{{.*}}'int'
   // CHECK-NEXT: MemberExpr{{.*}} '<bound member function type>' .operator int
-  // CHECK-NEXT: DeclRefExpr{{.*}} 'struct CorrectConvert':'CorrectConvert' lvalue Var
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'struct CorrectConvert' lvalue Var
   // CHECK-NEXT: WhileStmt
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: CompoundStmt
diff --git a/clang/test/SemaOpenACC/compute-construct-reduction-clause.c b/clang/test/SemaOpenACC/compute-construct-reduction-clause.c
index 995b6d3d95eb0..ebced2b92feb8 100644
--- a/clang/test/SemaOpenACC/compute-construct-reduction-clause.c
+++ b/clang/test/SemaOpenACC/compute-construct-reduction-clause.c
@@ -63,8 +63,9 @@ void uses(unsigned Parm) {
   // Vars in a reduction must be a scalar or a composite of scalars.
 #pragma acc parallel reduction(&: CoS, I, F)
   while (1);
-  // expected-error@+2{{OpenACC 'reduction' composite variable must not have non-scalar field}}
-  // expected-note@#COS_FIELD{{invalid field is here}}
+  // expected-error@+3{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
 #pragma acc parallel reduction(&: ChC)
   while (1);
 
@@ -75,7 +76,10 @@ void uses(unsigned Parm) {
   while (1);
 
   struct CompositeHasComposite ChCArray[5];
-  // expected-error@+1{{OpenACC 'reduction' variable must be of scalar type, aggregate, sub-array, or a composite of scalar types; sub-array base type is 'struct CompositeHasComposite'}}
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of sub-array type 'struct CompositeHasComposite'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
 #pragma acc parallel reduction(&: CoS, Array[I], ChCArray[0:I])
   while (1);
 
diff --git a/clang/test/SemaOpenACC/compute-construct-reduction-clause.cpp b/clang/test/SemaOpenACC/compute-construct-reduction-clause.cpp
index b40268c9b597d..924d19939d278 100644
--- a/clang/test/SemaOpenACC/compute-construct-reduction-clause.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-reduction-clause.cpp
@@ -66,8 +66,9 @@ void uses(unsigned Parm) {
   // Vars in a reduction must be a scalar or a composite of scalars.
 #pragma acc parallel reduction(&: CoS, I, F)
   while (1);
-  // expected-error@+2{{OpenACC 'reduction' composite variable must not have non-scalar field}}
-  // expected-note@#COS_FIELD{{invalid field is here}}
+  // expected-error@+3{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
 #pragma acc parallel reduction(&: ChC)
   while (1);
 #pragma acc parallel reduction(&: Array)
@@ -91,6 +92,26 @@ void uses(unsigned Parm) {
   // expected-error@+1{{OpenACC variable is not a valid variable name, sub-array, array element, or composite variable member}}
 #pragma acc parallel reduction(&: ChCPtr->COS)
   while (1);
+
+  CompositeHasComposite CoCArr[5];
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of array type 'CompositeHasComposite'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
+#pragma acc parallel reduction(+:CoCArr)
+  while (1);
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of array type 'CompositeHasComposite[5]'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
+#pragma acc parallel reduction(+:CoCArr[3])
+  while (1);
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of sub-array type 'CompositeHasComposite'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
+#pragma acc parallel reduction(+:CoCArr[1:1])
+  while (1);
 }
 
 template<typename T, typename U, typename V>
@@ -135,8 +156,9 @@ void TemplUses(T Parm, U CoS, V ChC) {
   // Vars in a reduction must be a scalar or a composite of scalars.
 #pragma acc parallel reduction(&: CoS, Var, Parm)
   while (1);
-  // expected-error@+2{{OpenACC 'reduction' composite variable must not have non-scalar field}}
-  // expected-note@#COS_FIELD{{invalid field is here}}
+  // expected-error@+3{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
 #pragma acc parallel reduction(&: ChC)
   while (1);
 #pragma acc parallel reduction(&: Array)
diff --git a/clang/test/SemaOpenACC/compute-construct-varlist-ast.cpp b/clang/test/SemaOpenACC/compute-construct-varlist-ast.cpp
index e3c41849eba81..2d2b20f1b77f3 100644
--- a/clang/test/SemaOpenACC/compute-construct-varlist-ast.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-varlist-ast.cpp
@@ -102,7 +102,7 @@ void NormalUses(float *PointerParam) {
 #pragma acc parallel copy(GlobalArray) pcopy(always: PointerParam[Global]) present_or_copy(alwaysin, alwaysout: Global)
   while(true);
   // CHECK-NEXT: OpenACCComputeConstruct{{.*}} parallel
-  // CHECK-NEXT: copy clause 
+  // CHECK-NEXT: copy clause
   // CHECK-NEXT: DeclRefExpr{{.*}}'short[5]' lvalue Var{{.*}}'GlobalArray' 'short[5]'
   // CHECK-NEXT: pcopy clause modifiers: always
   // CHECK-NEXT: ArraySubscriptExpr{{.*}}'float' lvalue
@@ -649,7 +649,7 @@ void TemplUses(T t, U u, T*PointerParam) {
 
 struct S {
   // CHECK-NEXT: CXXRecordDecl{{.*}} struct S definition
-  // CHECK: CXXRecordDecl{{.*}} implicit struct S
+  // CHECK: CXXRecordDecl{{.*}} implicit {{.*}}struct S{{$}}
   int ThisMember;
   // CHECK-NEXT: FieldDecl{{.*}} ThisMember 'int'
   int *ThisMemberPtr;
diff --git a/clang/test/SemaOpenACC/gh154008.cpp b/clang/test/SemaOpenACC/gh154008.cpp
new file mode 100644
index 0000000000000..1ec114c000b3f
--- /dev/null
+++ b/clang/test/SemaOpenACC/gh154008.cpp
@@ -0,0 +1,5 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+// expected-error@+2{{expected ';'}}
+// expected-error@+1{{blocks support disabled}}
+void *a = ^ { static int b };
diff --git a/clang/test/SemaOpenACC/init-construct.cpp b/clang/test/SemaOpenACC/init-construct.cpp
index abc7f741e3f47..d55358990518a 100644
--- a/clang/test/SemaOpenACC/init-construct.cpp
+++ b/clang/test/SemaOpenACC/init-construct.cpp
@@ -34,6 +34,12 @@ void uses() {
   // expected-error@+2{{OpenACC integer expression requires explicit conversion from 'struct ExplicitConvertOnly' to 'int'}}
   // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
 #pragma acc init device_num(Explicit)
+
+  // expected-error@+1{{OpenACC 'device_type' clause on a 'init' construct only permits one architecture}}
+#pragma acc init device_type(nvidia, radeon)
+
+  // expected-error@+1{{OpenACC 'device_type' clause on a 'init' construct only permits one architecture}}
+#pragma acc init  device_type(nonsense, nvidia, radeon)
 }
 
 template<typename T>
diff --git a/clang/test/SemaOpenACC/loop-construct-reduction-clause.cpp b/clang/test/SemaOpenACC/loop-construct-reduction-clause.cpp
index 00bcd744b9e79..4e885166be318 100644
--- a/clang/test/SemaOpenACC/loop-construct-reduction-clause.cpp
+++ b/clang/test/SemaOpenACC/loop-construct-reduction-clause.cpp
@@ -42,8 +42,9 @@ void uses() {
 
 #pragma acc serial
   {
-  // expected-error@+2{{OpenACC 'reduction' composite variable must not have non-scalar field}}
-  // expected-note@#COS_FIELD{{invalid field is here}}
+  // expected-error@+3{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
 #pragma acc loop reduction(+:ChC)
     for(int i = 0; i < 5; ++i){}
   }
@@ -153,6 +154,26 @@ void uses() {
     for(int i = 0; i < 5; ++i) {
     }
   }
+
+  CompositeHasComposite CoCArr[5];
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of array type 'CompositeHasComposite'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
+#pragma acc loop reduction(+:CoCArr)
+    for(int i = 0; i < 5; ++i);
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of array type 'CompositeHasComposite[5]'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
+#pragma acc loop reduction(+:CoCArr[3])
+    for(int i = 0; i < 5; ++i);
+  // expected-error@+4{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@+3{{used as element type of sub-array type 'CompositeHasComposite'}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
+#pragma acc loop reduction(+:CoCArr[1:1])
+    for(int i = 0; i < 5; ++i);
 }
 
 template<typename IntTy, typename CoSTy, typename ChCTy, unsigned One,
@@ -177,8 +198,9 @@ void templ_uses() {
 
 #pragma acc serial
   {
-  // expected-error@+2{{OpenACC 'reduction' composite variable must not have non-scalar field}}
-  // expected-note@#COS_FIELD{{invalid field is here}}
+  // expected-error@+3{{invalid type 'struct CompositeOfScalars' used in OpenACC 'reduction' variable reference; type is not a scalar value}}
+  // expected-note@#COS_FIELD{{used as field 'COS' of composite 'CompositeHasComposite'}}
+  // expected-note@+1{{OpenACC 'reduction' variable reference must be a scalar variable or a composite of scalars, or an array, sub-array, or element of scalar types}}
 #pragma acc loop reduction(+:ChC)
     for(int i = 0; i < 5; ++i){}
   }
diff --git a/clang/test/SemaOpenACC/private_firstprivate_reduction_required_ops.cpp b/clang/test/SemaOpenACC/private_firstprivate_reduction_required_ops.cpp
index 4644bdeeacc0e..38eb4e43813a8 100644
--- a/clang/test/SemaOpenACC/private_firstprivate_reduction_required_ops.cpp
+++ b/clang/test/SemaOpenACC/private_firstprivate_reduction_required_ops.cpp
@@ -37,10 +37,12 @@ struct ImplicitDelDtor {
 };
 
 struct DeletedCopy {
+  DeletedCopy();
   DeletedCopy(const DeletedCopy&) = delete;
 };
 
 struct DefaultedCopy {
+  DefaultedCopy();
   DefaultedCopy(const DefaultedCopy&) = default;
 };
 struct UserCopy {
@@ -273,3 +275,20 @@ void firstprivate_arrays() {
 #pragma acc parallel firstprivate(UDCopyArr)
   ;
 }
+
+template<unsigned I>
+void non_const_array_templ() {
+  int CArr[I];
+
+#pragma acc parallel firstprivate(CArr)
+  ;
+}
+
+void non_const_arrays(int I) {
+  non_const_array_templ<5>();
+
+  int NCArr[I];
+  // expected-warning@+1{{variable of array type 'int[I]' referenced in OpenACC 'firstprivate' clause does not have constant bounds; initialization will happen after decay to pointer}}
+#pragma acc parallel firstprivate(NCArr)
+  ;
+}
diff --git a/clang/test/SemaOpenACC/shutdown-construct.cpp b/clang/test/SemaOpenACC/shutdown-construct.cpp
index 95cea90ddb74a..e08a9680d71f5 100644
--- a/clang/test/SemaOpenACC/shutdown-construct.cpp
+++ b/clang/test/SemaOpenACC/shutdown-construct.cpp
@@ -34,6 +34,12 @@ void uses() {
   // expected-error@+2{{OpenACC integer expression requires explicit conversion from 'struct ExplicitConvertOnly' to 'int'}}
   // expected-note@#EXPL_CONV{{conversion to integral type 'int'}}
 #pragma acc shutdown device_num(Explicit)
+
+  // expected-error@+1{{OpenACC 'device_type' clause on a 'shutdown' construct only permits one architecture}}
+#pragma acc shutdown device_type(nvidia, radeon)
+
+  // expected-error@+1{{OpenACC 'device_type' clause on a 'shutdown' construct only permits one architecture}}
+#pragma acc shutdown device_type(nonsense, nvidia, radeon)
 }
 
 template<typename T>
diff --git a/clang/test/SemaOpenACC/sub-array.cpp b/clang/test/SemaOpenACC/sub-array.cpp
index 355ac5ef1d3ce..ec929759b6d56 100644
--- a/clang/test/SemaOpenACC/sub-array.cpp
+++ b/clang/test/SemaOpenACC/sub-array.cpp
@@ -61,13 +61,15 @@ void Func(int i, int j) {
 #pragma acc parallel private(ptr[3:])
   while (true);
 
-  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is an array of unknown bound}}
+  // expected-error@+2{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is an array of unknown bound}}
+  // expected-warning@+1{{variable of array type 'int[i]' referenced in OpenACC 'private' clause does not have constant bounds; initialization will happen after decay to pointer}}
 #pragma acc parallel private(VLA[3:])
   while (true);
 
 #pragma acc parallel private(ptr[:3])
   while (true);
 
+  // expected-warning@+1{{variable of array type 'int[i]' referenced in OpenACC 'private' clause does not have constant bounds; initialization will happen after decay to pointer}}
 #pragma acc parallel private(VLA[:3])
   while (true);
 
@@ -159,11 +161,13 @@ void Templ(int i){
   // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is not an array}}
 #pragma acc parallel private(ptr[Conv:])
   while (true);
-  // expected-error@+1{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is an array of unknown bound}}
+  // expected-error@+2{{OpenACC sub-array length is unspecified and cannot be inferred because the subscripted value is an array of unknown bound}}
+  // expected-warning@+1{{variable of array type 'int[i]' referenced in OpenACC 'private' clause does not have constant bounds; initialization will happen after decay to pointer}}
 #pragma acc parallel private(VLA[Conv:])
   while (true);
 #pragma acc parallel private(ptr[:Conv])
   while (true);
+  // expected-warning@+1{{variable of array type 'int[i]' referenced in OpenACC 'private' clause does not have constant bounds; initialization will happen after decay to pointer}}
 #pragma acc parallel private(VLA[:Conv])
   while (true);
 
diff --git a/clang/test/SemaOpenCLCXX/amdgpu-nullptr.clcpp b/clang/test/SemaOpenCLCXX/amdgpu-nullptr.clcpp
new file mode 100644
index 0000000000000..5bc147c2deba7
--- /dev/null
+++ b/clang/test/SemaOpenCLCXX/amdgpu-nullptr.clcpp
@@ -0,0 +1,9 @@
+// RUN: %clang_cc1 -triple amdgcn -cl-std=clc++ -verify %s
+
+// expected-no-diagnostics
+
+#define fold(x) (__builtin_constant_p(x) ? (x) : (x))
+static_assert(nullptr != fold(reinterpret_cast<private int*>(0)));
+
+static_assert(nullptr == (private int *)0);
+
diff --git a/clang/test/SemaSYCL/sycl-external-attr-appertainment.cpp b/clang/test/SemaSYCL/sycl-external-attr-appertainment.cpp
new file mode 100644
index 0000000000000..d06c9c9d53580
--- /dev/null
+++ b/clang/test/SemaSYCL/sycl-external-attr-appertainment.cpp
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -fsycl-is-host -fsyntax-only -std=c++17 -verify %s
+// RUN: %clang_cc1 -fsycl-is-device -fsyntax-only -std=c++17 -verify %s
+// RUN: %clang_cc1 -fsycl-is-host -fsyntax-only -std=c++20 -verify %s
+// RUN: %clang_cc1 -fsycl-is-device -fsyntax-only -std=c++20 -verify %s
+// RUN: %clang_cc1 -fsycl-is-host -fsyntax-only -std=c++23 -verify %s
+// RUN: %clang_cc1 -fsycl-is-device -fsyntax-only -std=c++23 -verify %s
+
+// expected-error@+1{{'clang::sycl_external' attribute only applies to functions}}
+[[clang::sycl_external]] int bad1;
+
+
+// expected-error@+2{{'clang::sycl_external' attribute only applies to functions}}
+struct s {
+[[clang::sycl_external]] int bad2;
+};
+
+// expected-error@+1{{'clang::sycl_external' attribute only applies to functions}}
+namespace [[clang::sycl_external]] bad3 {}
+
+// expected-error@+1{{'clang::sycl_external' attribute only applies to functions}}
+struct [[clang::sycl_external]]  bad4;
+
+// expected-error@+1{{'clang::sycl_external' attribute only applies to functions}}
+enum [[clang::sycl_external]] bad5 {};
+
+// expected-error@+1{{'clang::sycl_external' attribute only applies to functions}}
+int bad6(void (fp [[clang::sycl_external]])());
+
+// expected-error@+1{{'clang::sycl_external' attribute only applies to functions}}
+[[clang::sycl_external]];
+
+#if __cplusplus >= 202002L
+// expected-error@+2{{'clang::sycl_external' attribute only applies to functions}}
+template<typename>
+concept bad8 [[clang::sycl_external]] = true;
+#endif
diff --git a/clang/test/SemaSYCL/sycl-external-attr-grammar.cpp b/clang/test/SemaSYCL/sycl-external-attr-grammar.cpp
new file mode 100644
index 0000000000000..a0169851cdaf0
--- /dev/null
+++ b/clang/test/SemaSYCL/sycl-external-attr-grammar.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -fsycl-is-host -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsycl-is-device -fsyntax-only -verify %s
+
+// FIXME-expected-error@+1{{'clang::sycl_external' attribute takes no arguments}}
+[[clang::sycl_external()]] void bad1();
+
+// expected-error@+1{{expected expression}}
+[[clang::sycl_external(,)]] void bad2();
+
+// expected-error@+1{{'clang::sycl_external' attribute takes no arguments}}
+[[clang::sycl_external(3)]] void bad3();
+
+// expected-error@+1{{expected expression}}
+[[clang::sycl_external(4,)]] void bad4();
diff --git a/clang/test/SemaSYCL/sycl-external-attr-ignored.cpp b/clang/test/SemaSYCL/sycl-external-attr-ignored.cpp
new file mode 100644
index 0000000000000..211b3f1989f11
--- /dev/null
+++ b/clang/test/SemaSYCL/sycl-external-attr-ignored.cpp
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+// These tests validate that the sycl_external attribute is ignored when SYCL
+// support is not enabled.
+
+// expected-warning@+1{{'clang::sycl_external' attribute ignored}}
+[[clang::sycl_external]] void bar() {}
+
+// expected-warning@+1{{'clang::sycl_external' attribute ignored}}
+[[clang::sycl_external]] int a;
+
+// expected-warning@+2{{'clang::sycl_external' attribute ignored}}
+template<typename T>
+[[clang::sycl_external]] void ft(T) {}
+template void ft(int);
diff --git a/clang/test/SemaSYCL/sycl-external-attr.cpp b/clang/test/SemaSYCL/sycl-external-attr.cpp
new file mode 100644
index 0000000000000..ebda94e7d5030
--- /dev/null
+++ b/clang/test/SemaSYCL/sycl-external-attr.cpp
@@ -0,0 +1,154 @@
+// RUN: %clang_cc1 -fsycl-is-host -std=c++17 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsycl-is-device -std=c++17 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsycl-is-host -std=c++20 -fsyntax-only -verify -DCPP20 %s
+// RUN: %clang_cc1 -fsycl-is-device -std=c++20 -fsyntax-only -verify -DCPP20 %s
+
+// Semantic tests for the sycl_external attribute.
+
+// expected-error@+1{{'clang::sycl_external' can only be applied to functions with external linkage}}
+[[clang::sycl_external]]
+static void func1() {}
+
+// expected-error@+2{{'clang::sycl_external' can only be applied to functions with external linkage}}
+namespace {
+  [[clang::sycl_external]]
+  void func2() {}
+}
+
+// expected-error@+2{{'clang::sycl_external' can only be applied to functions with external linkage}}
+namespace { struct S4 {}; }
+[[clang::sycl_external]] void func4(S4) {}
+
+// expected-error@+3{{'clang::sycl_external' can only be applied to functions with external linkage}}
+namespace { struct S5 {}; }
+template<typename> [[clang::sycl_external]] void func5();
+template<> [[clang::sycl_external]] void func5<S5>() {}
+
+namespace { struct S6 {}; }
+template<typename>
+[[clang::sycl_external]] void func6() {}
+template void func6<S6>();
+
+// FIXME: C++23 [temp.expl.spec]p12 states:
+//   ... Similarly, attributes appearing in the declaration of a template
+//   have no effect on an explicit specialization of that template.
+// Clang currently instantiates and propagates attributes from a function
+// template to its explicit specializations resulting in the following
+// spurious error.
+// expected-error@+3{{'clang::sycl_external' can only be applied to functions with external linkage}}
+namespace { struct S7 {}; }
+template<typename>
+[[clang::sycl_external]] void func7();
+template<> void func7<S7>() {}
+
+// FIXME: The explicit function template specialization appears to trigger
+// instantiation of a declaration from the primary template without the
+// attribute leading to a spurious diagnostic that the sycl_external
+// attribute is not present on the first declaration.
+namespace { struct S8 {}; }
+template<typename>
+void func8();
+template<> [[clang::sycl_external]] void func8<S8>() {}
+// expected-warning@-1{{'clang::sycl_external' attribute does not appear on the first declaration}}
+// expected-error@-2{{'clang::sycl_external' can only be applied to functions with external linkage}}
+// expected-note@-3{{previous declaration is here}}
+
+namespace { struct S9 {}; }
+struct T9 {
+  using type = S9;
+};
+template<typename>
+[[clang::sycl_external]] void func9() {}
+template<typename T>
+[[clang::sycl_external]] void test_func9() {
+  func9<typename T::type>();
+}
+template void test_func9<T9>();
+
+// The first declaration of a SYCL external function is required to have this attribute.
+// expected-note@+1{{previous declaration is here}}
+int foo();
+// expected-warning@+1{{'clang::sycl_external' attribute does not appear on the first declaration}}
+[[clang::sycl_external]] int foo();
+
+// expected-note@+1{{previous declaration is here}}
+void goo();
+// expected-warning@+1{{'clang::sycl_external' attribute does not appear on the first declaration}}
+[[clang::sycl_external]] void goo();
+void goo() {}
+
+// expected-note@+1{{previous declaration is here}}
+void hoo() {}
+// expected-warning@+1{{'clang::sycl_external' attribute does not appear on the first declaration}}
+[[clang::sycl_external]] void hoo();
+
+// expected-note@+1{{previous declaration is here}}
+void joo();
+void use_joo() {
+  joo();
+}
+// expected-warning@+1{{'clang::sycl_external' attribute does not appear on the first declaration}}
+[[clang::sycl_external]] void joo();
+
+// Subsequent declarations of a SYCL external function may optionally specify this attribute.
+[[clang::sycl_external]] int boo();
+[[clang::sycl_external]] int boo(); // OK
+int boo(); // OK
+
+class C {
+  [[clang::sycl_external]] void member();
+};
+
+// expected-error@+1{{'clang::sycl_external' cannot be applied to the 'main' function}}
+[[clang::sycl_external]] int main()
+{
+  return 0;
+}
+
+// expected-error@+2{{'clang::sycl_external' cannot be applied to an explicitly deleted function}}
+class D {
+  [[clang::sycl_external]] void mdel() = delete;
+};
+
+// expected-error@+1{{'clang::sycl_external' cannot be applied to an explicitly deleted function}}
+[[clang::sycl_external]] void del() = delete;
+
+struct NonCopyable {
+  ~NonCopyable() = delete;
+  [[clang::sycl_external]] NonCopyable(const NonCopyable&) = default;
+};
+
+class A {
+  [[clang::sycl_external]]
+  A() {}
+
+  [[clang::sycl_external]] void mf() {}
+  [[clang::sycl_external]] static void smf();
+};
+
+class B {
+public:
+  [[clang::sycl_external]] virtual void foo() {}
+
+  [[clang::sycl_external]] virtual void bar() = 0;
+};
+[[clang::sycl_external]] void B::bar() {}
+
+[[clang::sycl_external]] constexpr int square(int x);
+
+// Devices that do not support the generic address space shall not specify
+// a raw pointer or reference type as the return type or as a parameter type.
+[[clang::sycl_external]] int *fun0();
+[[clang::sycl_external]] int &fun1();
+[[clang::sycl_external]] int &&fun2();
+[[clang::sycl_external]] void fun3(int *);
+[[clang::sycl_external]] void fun4(int &);
+[[clang::sycl_external]] void fun5(int &&);
+template<typename T>
+[[clang::sycl_external]] void fun6(T) {}
+template void fun6(int *);
+template<> [[clang::sycl_external]] void fun6<long*>(long *) {}
+
+#if CPP20
+[[clang::sycl_external]] consteval int func();
+#endif
diff --git a/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-appertainment.cpp b/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-appertainment.cpp
index 4774c8ef545f8..9aba284145fcb 100644
--- a/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-appertainment.cpp
+++ b/clang/test/SemaSYCL/sycl-kernel-entry-point-attr-appertainment.cpp
@@ -360,3 +360,16 @@ void bad38() try {} catch(...) {}
 template<typename>
 [[clang::sycl_kernel_entry_point(BADKN<39>)]]
 void bad39() try {} catch(...) {}
+
+// expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute only applies to functions}}
+[[clang::sycl_kernel_entry_point(BADKN<40>)]];
+
+void bad41() {
+  // expected-error@+1 {{'clang::sycl_kernel_entry_point' attribute cannot be applied to a statement}}
+  [[clang::sycl_kernel_entry_point(BADKN<41>)]];
+}
+
+struct B42 {
+  // expected-warning@+1 {{declaration does not declare anything}}
+  [[clang::sycl_kernel_entry_point(BADKN<42>)]];
+};
diff --git a/clang/test/SemaTemplate/aggregate-deduction-candidate.cpp b/clang/test/SemaTemplate/aggregate-deduction-candidate.cpp
index 2ecd2694ce09d..bf153b2ec3708 100644
--- a/clang/test/SemaTemplate/aggregate-deduction-candidate.cpp
+++ b/clang/test/SemaTemplate/aggregate-deduction-candidate.cpp
@@ -15,7 +15,7 @@ namespace Basic {
   // CHECK-LABEL: Dumping Basic::<deduction guide for A>:
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for A>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced class depth 0 index 0 T
-  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (T, T) -> A<T>'
+  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (T, T) -> Basic::A<T>'
   // CHECK: | |-ParmVarDecl {{.*}} 'T'
   // CHECK: | `-ParmVarDecl {{.*}} 'T'
   // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for A> 'auto (double, double) -> Basic::A<double>'
@@ -23,8 +23,8 @@ namespace Basic {
   // CHECK:   | `-BuiltinType {{.*}} 'double'
   // CHECK:   |-ParmVarDecl {{.*}} 'double'
   // CHECK:   `-ParmVarDecl {{.*}} 'double'
-  // CHECK: FunctionProtoType {{.*}} 'auto (T, T) -> A<T>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'A<T>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (T, T) -> Basic::A<T>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'Basic::A<T>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'A'
   // CHECK: |-TemplateTypeParmType {{.*}} 'T' dependent depth 0 index 0
   // CHECK: | `-TemplateTypeParm {{.*}} 'T'
@@ -66,7 +66,7 @@ namespace Basic {
   // CHECK-LABEL: Dumping Basic::<deduction guide for C>:
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for C>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
-  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for C> 'auto (S<T>, T) -> C<T>'
+  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for C> 'auto (S<T>, T) -> Basic::C<T>'
   // CHECK: | |-ParmVarDecl {{.*}} 'S<T>'
   // CHECK: | `-ParmVarDecl {{.*}} 'T'
   // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for C> 'auto (S<int>, int) -> Basic::C<int>'
@@ -74,25 +74,24 @@ namespace Basic {
   // CHECK:   | `-BuiltinType {{.*}} 'int'
   // CHECK:   |-ParmVarDecl {{.*}} 'S<int>':'Basic::S<int>'
   // CHECK:   `-ParmVarDecl {{.*}} 'int'
-  // CHECK: FunctionProtoType {{.*}} 'auto (S<T>, T) -> C<T>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'C<T>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (S<T>, T) -> Basic::C<T>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'Basic::C<T>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'C'
-  // CHECK: |-ElaboratedType {{.*}} 'S<T>' sugar dependent
-  // CHECK: | `-TemplateSpecializationType {{.*}} 'S<T>' dependent
-  // CHECK: |   `-TemplateArgument type 'T'
-  // CHECK: |     `-TemplateTypeParmType {{.*}} 'T' dependent depth 0 index 0
-  // CHECK: |       `-TemplateTypeParm {{.*}} 'T'
+  // CHECK: |-TemplateSpecializationType {{.*}} 'S<T>' dependent
+  // CHECK: | `-TemplateArgument type 'T'
+  // CHECK: |   `-TemplateTypeParmType {{.*}} 'T' dependent depth 0 index 0
+  // CHECK: |     `-TemplateTypeParm {{.*}} 'T'
   // CHECK: `-TemplateTypeParmType {{.*}} 'T' dependent depth 0 index 0
   // CHECK:   `-TemplateTypeParm {{.*}} 'T'
 
   // CHECK-LABEL: Dumping Basic::<deduction guide for D>:
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for D>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
-  // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for D> 'auto (int, int) -> D<T>'
+  // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for D> 'auto (int, int) -> Basic::D<T>'
   // CHECK:   |-ParmVarDecl {{.*}} 'int'
   // CHECK:   `-ParmVarDecl {{.*}} 'int'
-  // CHECK: FunctionProtoType {{.*}} 'auto (int, int) -> D<T>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'D<T>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (int, int) -> Basic::D<T>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'Basic::D<T>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'D'
   // CHECK: |-SubstTemplateTypeParmType {{.*}} 'int' sugar typename depth 0 index 0 T
   // CHECK: | |-ClassTemplateSpecialization {{.*}} 'S'
@@ -113,7 +112,7 @@ namespace Basic {
   // CHECK-LABEL: Dumping Basic::<deduction guide for E>:
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for E>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
-  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for E> 'auto (T, decltype(t)) -> E<T>'
+  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for E> 'auto (T, decltype(t)) -> Basic::E<T>'
   // CHECK: | |-ParmVarDecl {{.*}} 'T'
   // CHECK: | `-ParmVarDecl {{.*}} 'decltype(t)'
   // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for E> 'auto (int, decltype(t)) -> Basic::E<int>'
@@ -121,8 +120,8 @@ namespace Basic {
   // CHECK:   | `-BuiltinType {{.*}} 'int'
   // CHECK:   |-ParmVarDecl {{.*}} 'int'
   // CHECK:   `-ParmVarDecl {{.*}} 'decltype(t)':'int'
-  // CHECK: FunctionProtoType {{.*}} 'auto (T, decltype(t)) -> E<T>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'E<T>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (T, decltype(t)) -> Basic::E<T>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'Basic::E<T>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'E'
   // CHECK: |-TemplateTypeParmType {{.*}} 'T' dependent depth 0 index 0
   // CHECK: | `-TemplateTypeParm {{.*}} 'T'
@@ -145,7 +144,7 @@ namespace Basic {
   // CHECK-LABEL: Dumping Basic::<deduction guide for F>:
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for F>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
-  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (typename I<T>::type, T) -> F<T>'
+  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (typename I<T>::type, T) -> Basic::F<T>'
   // CHECK: | |-ParmVarDecl {{.*}} 'typename I<T>::type'
   // CHECK: | `-ParmVarDecl {{.*}} 'T'
   // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for F> 'auto (typename I<int>::type, int) -> Basic::F<int>'
@@ -153,8 +152,8 @@ namespace Basic {
   // CHECK:   | `-BuiltinType {{.*}} 'int'
   // CHECK:   |-ParmVarDecl {{.*}} 'typename I<int>::type':'int'
   // CHECK:   `-ParmVarDecl {{.*}} 'int'
-  // CHECK: FunctionProtoType {{.*}} 'auto (typename I<T>::type, T) -> F<T>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'F<T>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (typename I<T>::type, T) -> Basic::F<T>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'Basic::F<T>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'F'
   // CHECK: |-DependentNameType {{.*}} 'typename I<T>::type' dependent
   // CHECK: `-TemplateTypeParmType {{.*}} 'T' dependent depth 0 index 0
@@ -181,15 +180,15 @@ namespace Array {
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for A>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
   // CHECK: |-NonTypeTemplateParmDecl {{.*}} 'size_t':'unsigned {{.*}}' depth 0 index 1 N
-  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (T (&&)[N]) -> A<T, N>'
+  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (T (&&)[N]) -> Array::A<T, N>'
   // CHECK: | `-ParmVarDecl {{.*}} 'T (&&)[N]'
   // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for A> 'auto (int (&&)[3]) -> Array::A<int, 3>'
   // CHECK:   |-TemplateArgument type 'int'
   // CHECK:   | `-BuiltinType {{.*}} 'int'
   // CHECK:   |-TemplateArgument integral '3UL'
   // CHECK:   `-ParmVarDecl {{.*}} 'int (&&)[3]'
-  // CHECK: FunctionProtoType {{.*}} 'auto (T (&&)[N]) -> A<T, N>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'A<T, N>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (T (&&)[N]) -> Array::A<T, N>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'Array::A<T, N>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'A'
   // CHECK: `-RValueReferenceType {{.*}} 'T (&&)[N]' dependent
   // CHECK:   `-DependentSizedArrayType {{.*}} 'T[N]' dependent
@@ -201,15 +200,15 @@ namespace Array {
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for A>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
   // CHECK: |-NonTypeTemplateParmDecl {{.*}} 'size_t':'unsigned {{.*}}' depth 0 index 1 N
-  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (const T (&)[N]) -> A<T, N>'
+  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (const T (&)[N]) -> Array::A<T, N>'
   // CHECK: | `-ParmVarDecl {{.*}} 'const T (&)[N]'
   // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for A> 'auto (const char (&)[5]) -> Array::A<char, 5>'
   // CHECK:   |-TemplateArgument type 'char'
   // CHECK:   | `-BuiltinType {{.*}} 'char'
   // CHECK:   |-TemplateArgument integral '5UL'
   // CHECK:   `-ParmVarDecl {{.*}} 'const char (&)[5]'
-  // CHECK: FunctionProtoType {{.*}} 'auto (const T (&)[N]) -> A<T, N>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'A<T, N>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (const T (&)[N]) -> Array::A<T, N>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'Array::A<T, N>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'A'
   // CHECK: `-LValueReferenceType {{.*}} 'const T (&)[N]' dependent
   // CHECK:   `-QualType {{.*}} 'const T[N]' const
@@ -232,7 +231,7 @@ namespace BraceElision {
   // CHECK-LABEL: Dumping BraceElision::<deduction guide for A>:
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for A>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 T
-  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (T, T) -> A<T>'
+  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (T, T) -> BraceElision::A<T>'
   // CHECK: | |-ParmVarDecl {{.*}} 'T'
   // CHECK: | `-ParmVarDecl {{.*}} 'T'
   // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for A> 'auto (int, int) -> BraceElision::A<int>'
@@ -240,8 +239,8 @@ namespace BraceElision {
   // CHECK:   | `-BuiltinType {{.*}} 'int'
   // CHECK:   |-ParmVarDecl {{.*}} 'int'
   // CHECK:   `-ParmVarDecl {{.*}} 'int'
-  // CHECK: FunctionProtoType {{.*}} 'auto (T, T) -> A<T>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'A<T>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (T, T) -> BraceElision::A<T>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'BraceElision::A<T>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'A'
   // CHECK: |-TemplateTypeParmType {{.*}} 'T' dependent depth 0 index 0
   // CHECK: | `-TemplateTypeParm {{.*}} 'T'
@@ -266,7 +265,7 @@ namespace TrailingPack {
   // CHECK-LABEL: Dumping TrailingPack::<deduction guide for A>:
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for A>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 ... T
-  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (T...) -> A<T...>'
+  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for A> 'auto (T...) -> TrailingPack::A<T...>'
   // CHECK: | `-ParmVarDecl {{.*}} 'T...' pack
   // CHECK: `-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for A>
   // CHECK-SAME: 'auto (TrailingPack::(lambda at {{.*}}), TrailingPack::(lambda at {{.*}})) ->
@@ -274,14 +273,14 @@ namespace TrailingPack {
   // CHECK: |-TemplateArgument pack
   // CHECK: | |-TemplateArgument type 'TrailingPack::(lambda at {{.*}})'
   // CHECK: | | `-RecordType {{.*}} 'TrailingPack::(lambda at {{.*}})'
-  // CHECK: | |   `-CXXRecord {{.*}} <line:262:5>
+  // CHECK: | |   `-CXXRecord {{.*}} <line:261:5>
   // CHECK: | `-TemplateArgument type 'TrailingPack::(lambda at {{.*}})'
   // CHECK: |   `-RecordType {{.*}} 'TrailingPack::(lambda at {{.*}})'
-  // CHECK: |     `-CXXRecord {{.*}} <line:263:5>
+  // CHECK: |     `-CXXRecord {{.*}} <line:262:5>
   // CHECK: |-ParmVarDecl {{.*}} 'TrailingPack::(lambda at {{.*}})'
   // CHECK: `-ParmVarDecl {{.*}} 'TrailingPack::(lambda at {{.*}})'
-  // CHECK: FunctionProtoType {{.*}} 'auto (T...) -> A<T...>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'A<T...>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (T...) -> TrailingPack::A<T...>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'TrailingPack::A<T...>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'A'
   // CHECK: `-PackExpansionType {{.*}} 'T...' dependent
   // CHECK:   `-TemplateTypeParmType {{.*}} 'T' dependent contains_unexpanded_pack depth 0 index 0 pack
@@ -327,7 +326,7 @@ namespace DeduceArity {
   // CHECK-LABEL: Dumping DeduceArity::<deduction guide for F>:
   // CHECK: FunctionTemplateDecl {{.*}} implicit <deduction guide for F>
   // CHECK: |-TemplateTypeParmDecl {{.*}} referenced typename depth 0 index 0 ... T
-  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (Types<T...>, T...) -> F<T...>'
+  // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit <deduction guide for F> 'auto (Types<T...>, T...) -> DeduceArity::F<T...>'
   // CHECK: | |-ParmVarDecl {{.*}} 'Types<T...>'
   // CHECK: | `-ParmVarDecl {{.*}} 'T...' pack
   // CHECK: |-CXXDeductionGuideDecl {{.*}} implicit used <deduction guide for F>
@@ -354,15 +353,14 @@ namespace DeduceArity {
   // CHECK:   |     `-CXXRecord {{.*}} 'X'
   // CHECK:   |-ParmVarDecl {{.*}} 'Types<DeduceArity::X>':'DeduceArity::Types<DeduceArity::X>'
   // CHECK:   `-ParmVarDecl {{.*}} 'DeduceArity::X'
-  // CHECK: FunctionProtoType {{.*}} 'auto (Types<T...>, T...) -> F<T...>' dependent trailing_return cdecl
-  // CHECK: |-InjectedClassNameType {{.*}} 'F<T...>' dependent
+  // CHECK: FunctionProtoType {{.*}} 'auto (Types<T...>, T...) -> DeduceArity::F<T...>' dependent trailing_return cdecl
+  // CHECK: |-InjectedClassNameType {{.*}} 'DeduceArity::F<T...>' dependent
   // CHECK: | `-CXXRecord {{.*}} 'F'
-  // CHECK: |-ElaboratedType {{.*}} 'Types<T...>' sugar dependent
-  // CHECK: | `-TemplateSpecializationType {{.*}} 'Types<T...>' dependent
-  // CHECK: |   `-TemplateArgument type 'T...'
-  // CHECK: |     `-PackExpansionType {{.*}} 'T...' dependent
-  // CHECK: |       `-TemplateTypeParmType {{.*}} 'T' dependent contains_unexpanded_pack depth 0 index 0 pack
-  // CHECK: |         `-TemplateTypeParm {{.*}} 'T'
+  // CHECK: |-TemplateSpecializationType {{.*}} 'Types<T...>' dependent
+  // CHECK: | `-TemplateArgument type 'T...'
+  // CHECK: |   `-PackExpansionType {{.*}} 'T...' dependent
+  // CHECK: |     `-TemplateTypeParmType {{.*}} 'T' dependent contains_unexpanded_pack depth 0 index 0 pack
+  // CHECK: |       `-TemplateTypeParm {{.*}} 'T'
   // CHECK: `-PackExpansionType {{.*}} 'T...' dependent
   // CHECK:   `-TemplateTypeParmType {{.*}} 'T' dependent contains_unexpanded_pack depth 0 index 0 pack
   // CHECK:     `-TemplateTypeParm {{.*}} 'T'
diff --git a/clang/test/SemaTemplate/class-template-ctor-initializer.cpp b/clang/test/SemaTemplate/class-template-ctor-initializer.cpp
index 6dae20774585e..43a3986fea845 100644
--- a/clang/test/SemaTemplate/class-template-ctor-initializer.cpp
+++ b/clang/test/SemaTemplate/class-template-ctor-initializer.cpp
@@ -4,8 +4,8 @@
 
 template<class X> struct A {};
 
-template<class X> struct B : A<X> { 
-  B() : A<X>() {} 
+template<class X> struct B : A<X> {
+  B() : A<X>() {}
 };
 B<int> x;
 
@@ -76,3 +76,12 @@ namespace NonDependentError {
   Derived1<void> d1;
   Derived2<void> d2;
 }
+
+namespace UnresolvedUsing {
+  template <class T> class A {
+    using typename T::B;
+    struct C : B {
+      C() : B() {}
+    };
+  };
+} // namespace UnresolvedUsing
diff --git a/clang/test/SemaTemplate/ctad.cpp b/clang/test/SemaTemplate/ctad.cpp
index 1bf605f823bbe..1a575ea527006 100644
--- a/clang/test/SemaTemplate/ctad.cpp
+++ b/clang/test/SemaTemplate/ctad.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -std=c++17 -verify %s
+// RUN: %clang_cc1 -std=c++17 -verify=expected,cxx17 %s
+// RUN: %clang_cc1 -std=c++20 -verify=expected,cxx20 %s
 
 namespace pr41427 {
   template <typename T> class A {
@@ -21,9 +22,9 @@ namespace Access {
     struct type {};
   };
   template<typename T> struct D : B { // expected-note {{not viable}} \
-                                         expected-note {{implicit deduction guide declared as 'template <typename T> D(D<T>) -> D<T>'}}
+                                         expected-note {{implicit deduction guide declared as 'template <typename T> D(Access::D<T>) -> Access::D<T>'}}
     D(T, typename T::type); // expected-note {{private member}} \
-                            // expected-note {{implicit deduction guide declared as 'template <typename T> D(T, typename T::type) -> D<T>'}}
+                            // expected-note {{implicit deduction guide declared as 'template <typename T> D(T, typename T::type) -> Access::D<T>'}}
   };
   D b = {B(), {}};
 
@@ -61,15 +62,45 @@ namespace NoCrashOnGettingDefaultArgLoc {
 template <typename>
 class A {
   A(int = 1); // expected-note {{candidate template ignored: couldn't infer template argumen}} \
-              // expected-note {{implicit deduction guide declared as 'template <typename> D(int = <null expr>) -> D<type-parameter-0-0>'}}
+              // expected-note {{implicit deduction guide declared as 'template <typename> D(int = <null expr>) -> NoCrashOnGettingDefaultArgLoc::D<type-parameter-0-0>'}}
 };
 class C : A<int> {
   using A::A;
 };
 template <typename>
 class D : C { // expected-note {{candidate function template not viable: requires 1 argument}} \
-                 expected-note {{implicit deduction guide declared as 'template <typename> D(D<type-parameter-0-0>) -> D<type-parameter-0-0>'}}
+                 expected-note {{implicit deduction guide declared as 'template <typename> D(NoCrashOnGettingDefaultArgLoc::D<type-parameter-0-0>) -> NoCrashOnGettingDefaultArgLoc::D<type-parameter-0-0>'}}
   using C::C;
 };
 D abc; // expected-error {{no viable constructor or deduction guide}}
 }
+
+namespace AsValueParameter {
+  namespace foo {
+    // cxx17-note@+2 {{template is declared here}}
+    // cxx20-note@+1 {{'A<int>' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}}
+    template <class> struct A {
+      A();
+    };
+  }
+  template <foo::A> struct B {}; // expected-note {{template parameter is declared here}}
+  // cxx17-error@-1 {{use of class template 'foo::A' requires template arguments; argument deduction not allowed in template parameter}}
+
+  template struct B<foo::A<int>{}>;
+  // cxx17-error@-1 {{value of type 'foo::A<int>' is not implicitly convertible to 'int'}}
+  // cxx20-error@-2 {{non-type template parameter has non-literal type 'foo::A<int>' (aka 'AsValueParameter::foo::A<int>')}}
+} // namespace AsValueParameter
+
+namespace ConvertDeducedTemplateArgument {
+  namespace A {
+    template <class> struct B {};
+  }
+
+  template <template <class> class TT1> struct C {
+    C(TT1<int>);
+  };
+
+  template <template <class> class TT2> using D = TT2<int>;
+
+  auto x = C(D<A::B>());
+}
diff --git a/clang/test/SemaTemplate/current-instantiation.cpp b/clang/test/SemaTemplate/current-instantiation.cpp
index 9030768846b28..9214bbeb973d6 100644
--- a/clang/test/SemaTemplate/current-instantiation.cpp
+++ b/clang/test/SemaTemplate/current-instantiation.cpp
@@ -245,5 +245,5 @@ namespace RebuildDependentScopeDeclRefExpr {
   };
   template<typename T> N<X<T>::thing> X<T>::data() {}
   // FIXME: We should issue a typo-correction here.
-  template<typename T> N<X<T>::think> X<T>::foo() {} // expected-error {{no member named 'think' in 'X<T>'}}
+  template<typename T> N<X<T>::think> X<T>::foo() {} // expected-error {{no member named 'think' in 'RebuildDependentScopeDeclRefExpr::X<T>'}}
 }
diff --git a/clang/test/SemaTemplate/deduction-crash.cpp b/clang/test/SemaTemplate/deduction-crash.cpp
index 287c61aef00d6..99ca0b365ff6f 100644
--- a/clang/test/SemaTemplate/deduction-crash.cpp
+++ b/clang/test/SemaTemplate/deduction-crash.cpp
@@ -166,9 +166,9 @@ namespace PR51872_part1 {
   template<int> class T1 { template <struct U1> T1(); };
   // expected-error@-1 {{non-type template parameter has incomplete type 'struct U1'}}
   // expected-note@-2  {{forward declaration of 'PR51872_part1::U1'}}
-  // expected-note@-3  {{implicit deduction guide declared as 'template <int> T1(T1<value-parameter-0-0>) -> T1<value-parameter-0-0>'}}
+  // expected-note@-3  {{implicit deduction guide declared as 'template <int> T1(PR51872_part1::T1<value-parameter-0-0>) -> PR51872_part1::T1<value-parameter-0-0>'}}
 
   T1 t1 = 0;
   // expected-error@-1 {{no viable constructor or deduction guide for deduction of template arguments of 'T1'}}
-  // expected-note@-7  {{candidate template ignored: could not match 'T1<value-parameter-0-0>' against 'int'}}
+  // expected-note@-7  {{candidate template ignored: could not match 'PR51872_part1::T1<value-parameter-0-0>' against 'int'}}
 }
diff --git a/clang/test/SemaTemplate/deduction-guide.cpp b/clang/test/SemaTemplate/deduction-guide.cpp
index f6bc6ee3673c2..ef9b91ca0b18e 100644
--- a/clang/test/SemaTemplate/deduction-guide.cpp
+++ b/clang/test/SemaTemplate/deduction-guide.cpp
@@ -43,11 +43,10 @@ using AT = A<int[3], int, int, short>;
 // CHECK:   `-ParmVarDecl {{.*}} 'short (*)[4]'
 // CHECK: FunctionProtoType {{.*}} 'auto (X<Ps...>, Ts (*)[Ns]...) -> A<T, Ts...>' dependent trailing_return
 // CHECK: |-InjectedClassNameType {{.*}} 'A<T, Ts...>' dependent
-// CHECK: |-ElaboratedType {{.*}} 'X<Ps...>' sugar dependent
-// CHECK: | `-TemplateSpecializationType {{.*}} 'X<Ps...>' dependent
-// CHECK: |   `-TemplateArgument expr
-// CHECK: |     `-PackExpansionExpr {{.*}} 'T *'
-// CHECK: |       `-DeclRefExpr {{.*}} 'T *' NonTypeTemplateParm {{.*}} 'Ps' 'T *'
+// CHECK: |-TemplateSpecializationType {{.*}} 'X<Ps...>' dependent
+// CHECK: | `-TemplateArgument expr
+// CHECK: |   `-PackExpansionExpr {{.*}} 'T *'
+// CHECK: |     `-DeclRefExpr {{.*}} 'T *' NonTypeTemplateParm {{.*}} 'Ps' 'T *'
 // CHECK: `-PackExpansionType {{.*}} 'Ts (*)[Ns]...' dependent
 // CHECK:   `-PointerType {{.*}} 'Ts (*)[Ns]' dependent contains_unexpanded_pack
 // CHECK:     `-ParenType {{.*}} 'Ts[Ns]' sugar dependent contains_unexpanded_pack
@@ -118,9 +117,8 @@ using CT = C<int>;
 // CHECK: |-InjectedClassNameType {{.*}} 'C<A>' dependent
 // CHECK: |-TemplateTypeParmType {{.*}} 'A' dependent depth 0 index 0
 // CHECK: | `-TemplateTypeParm {{.*}} 'A'
-// CHECK: |-ElaboratedType {{.*}} 'Y<T>' sugar dependent
-// CHECK: | `-TemplateSpecializationType {{.*}} 'Y<T>' dependent
-// CHECK: |   `-TemplateArgument template
+// CHECK: |-TemplateSpecializationType {{.*}} 'Y<T>' dependent
+// CHECK: | `-TemplateArgument template
 // CHECK: `-TemplateTypeParmType {{.*}} 'U' dependent depth 0 index 2
 
 template<typename ...T> struct D { // expected-note {{candidate}} \
@@ -321,24 +319,23 @@ namespace TTP {
 // CHECK-NEXT:  |-TemplateTypeParmDecl {{.+}} class depth 0 index 0 T{{$}}
 // CHECK-NEXT:  |-TemplateTemplateParmDecl {{.+}} depth 0 index 1 TT{{$}}
 // CHECK-NEXT:  | `-TemplateTypeParmDecl {{.+}} class depth 1 index 0{{$}}
-// CHECK-NEXT:  |-CXXDeductionGuideDecl {{.+}} 'auto (TT<T>) -> B<T>'{{$}}
+// CHECK-NEXT:  |-CXXDeductionGuideDecl {{.+}} 'auto (TT<T>) -> TTP::B<T>'{{$}}
 // CHECK-NEXT:  | `-ParmVarDecl {{.+}} 'TT<T>'{{$}}
-// CHECK-NEXT:  `-CXXDeductionGuideDecl {{.+}} 'auto (A<int>) -> TTP::B<int>'
+// CHECK-NEXT:  `-CXXDeductionGuideDecl {{.+}} 'auto (TTP::A<int>) -> TTP::B<int>'
 // CHECK-NEXT:    |-TemplateArgument type 'int'
 // CHECK-NEXT:    | `-BuiltinType {{.+}} 'int'{{$}}
 // CHECK-NEXT:    |-TemplateArgument template 'TTP::A'{{$}}
 // CHECK-NEXT:    | `-ClassTemplateDecl {{.+}} A{{$}}
-// CHECK-NEXT:    `-ParmVarDecl {{.+}} 'A<int>':'TTP::A<int>'{{$}}
-// CHECK-NEXT:  FunctionProtoType {{.+}} 'auto (TT<T>) -> B<T>' dependent trailing_return cdecl{{$}}
-// CHECK-NEXT:  |-InjectedClassNameType {{.+}} 'B<T>' dependent{{$}}
+// CHECK-NEXT:    `-ParmVarDecl {{.+}} 'TTP::A<int>'{{$}}
+// CHECK-NEXT:  FunctionProtoType {{.+}} 'auto (TT<T>) -> TTP::B<T>' dependent trailing_return cdecl{{$}}
+// CHECK-NEXT:  |-InjectedClassNameType {{.+}} 'TTP::B<T>' dependent{{$}}
 // CHECK-NEXT:  | `-CXXRecord {{.+}} 'B'{{$}}
-// CHECK-NEXT:  `-ElaboratedType {{.+}} 'TT<T>' sugar dependent{{$}}
-// CHECK-NEXT:    `-TemplateSpecializationType {{.+}} 'TT<T>' dependent{{$}}
-// CHECK-NEXT:      |-name: 'TT':'template-parameter-0-1' qualified
-// CHECK-NEXT:      | `-TemplateTemplateParmDecl {{.+}} depth 0 index 1
-// CHECK-NEXT:      `-TemplateArgument type 'T':'type-parameter-0-0'{{$}}
-// CHECK-NEXT:        `-TemplateTypeParmType {{.+}} 'T' dependent depth 0 index 0{{$}}
-// CHECK-NEXT:          `-TemplateTypeParm {{.+}} 'T'{{$}}
+// CHECK-NEXT:  `-TemplateSpecializationType {{.+}} 'TT<T>' dependent{{$}}
+// CHECK-NEXT:    |-name: 'TT':'template-parameter-0-1' qualified
+// CHECK-NEXT:    | `-TemplateTemplateParmDecl {{.+}} depth 0 index 1
+// CHECK-NEXT:    `-TemplateArgument type 'T':'type-parameter-0-0'{{$}}
+// CHECK-NEXT:      `-TemplateTypeParmType {{.+}} 'T' dependent depth 0 index 0{{$}}
+// CHECK-NEXT:        `-TemplateTypeParm {{.+}} 'T'{{$}}
 
 namespace GH64625 {
 
@@ -351,14 +348,14 @@ X x = {{1, 2}};
 // CHECK-LABEL: Dumping GH64625::<deduction guide for X>:
 // CHECK-NEXT: FunctionTemplateDecl {{.+}} <{{.+}}:[[#@LINE - 7]]:1, col:27> col:27 implicit <deduction guide for X>
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.+}} <col:11, col:17> col:17 referenced class depth 0 index 0 T
-// CHECK:      |-CXXDeductionGuideDecl {{.+}} <col:27> col:27 implicit <deduction guide for X> 'auto (T (&&)[2]) -> X<T>' aggregate
+// CHECK:      |-CXXDeductionGuideDecl {{.+}} <col:27> col:27 implicit <deduction guide for X> 'auto (T (&&)[2]) -> GH64625::X<T>' aggregate
 // CHECK-NEXT: | `-ParmVarDecl {{.+}} <col:27> col:27 'T (&&)[2]'
 // CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} <col:27> col:27 implicit used <deduction guide for X> 'auto (int (&&)[2]) -> GH64625::X<int>' implicit_instantiation aggregate
 // CHECK-NEXT:  |-TemplateArgument type 'int'
 // CHECK-NEXT:  | `-BuiltinType {{.+}} 'int'
 // CHECK-NEXT:  `-ParmVarDecl {{.+}} <col:27> col:27 'int (&&)[2]'
-// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (T (&&)[2]) -> X<T>' dependent trailing_return
-// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'X<T>' dependent
+// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (T (&&)[2]) -> GH64625::X<T>' dependent trailing_return
+// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'GH64625::X<T>' dependent
 // CHECK-NEXT: | `-CXXRecord {{.+}} 'X'
 // CHECK-NEXT: `-RValueReferenceType {{.+}} 'T (&&)[2]' dependent
 // CHECK-NEXT:  `-ConstantArrayType {{.+}} 'T[2]' dependent 2
@@ -375,7 +372,7 @@ TwoArrays ta = {{1, 2}, {3, 4, 5}};
 // CHECK-NEXT: FunctionTemplateDecl {{.+}} <{{.+}}:[[#@LINE - 7]]:1, col:36> col:36 implicit <deduction guide for TwoArrays>
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.+}} <col:11, col:17> col:17 referenced class depth 0 index 0 T
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.+}} <col:20, col:26> col:26 referenced class depth 0 index 1 U
-// CHECK:      |-CXXDeductionGuideDecl {{.+}} <col:36> col:36 implicit <deduction guide for TwoArrays> 'auto (T (&&)[2], U (&&)[3]) -> TwoArrays<T, U>' aggregate
+// CHECK:      |-CXXDeductionGuideDecl {{.+}} <col:36> col:36 implicit <deduction guide for TwoArrays> 'auto (T (&&)[2], U (&&)[3]) -> GH64625::TwoArrays<T, U>' aggregate
 // CHECK-NEXT: | |-ParmVarDecl {{.+}} <col:36> col:36 'T (&&)[2]'
 // CHECK-NEXT: | `-ParmVarDecl {{.+}} <col:36> col:36 'U (&&)[3]'
 // CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} <col:36> col:36 implicit used <deduction guide for TwoArrays> 'auto (int (&&)[2], int (&&)[3]) -> GH64625::TwoArrays<int, int>' implicit_instantiation aggregate
@@ -385,8 +382,8 @@ TwoArrays ta = {{1, 2}, {3, 4, 5}};
 // CHECK-NEXT:   | `-BuiltinType {{.+}} 'int'
 // CHECK-NEXT:   |-ParmVarDecl {{.+}} <col:36> col:36 'int (&&)[2]'
 // CHECK-NEXT:   `-ParmVarDecl {{.+}} <col:36> col:36 'int (&&)[3]'
-// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (T (&&)[2], U (&&)[3]) -> TwoArrays<T, U>' dependent trailing_return
-// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'TwoArrays<T, U>' dependent
+// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (T (&&)[2], U (&&)[3]) -> GH64625::TwoArrays<T, U>' dependent trailing_return
+// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'GH64625::TwoArrays<T, U>' dependent
 // CHECK-NEXT: | `-CXXRecord {{.+}} 'TwoArrays'
 // CHECK-NEXT: |-RValueReferenceType {{.+}} 'T (&&)[2]' dependent
 // CHECK-NEXT: | `-ConstantArrayType {{.+}} 'T[2]' dependent 2
@@ -398,7 +395,7 @@ TwoArrays ta = {{1, 2}, {3, 4, 5}};
 // CHECK-NEXT:       `-TemplateTypeParm {{.+}} 'U'
 
 TwoArrays tb = {1, 2, {3, 4, 5}};
-// CHECK:   |-CXXDeductionGuideDecl {{.+}} <col:36> col:36 implicit <deduction guide for TwoArrays> 'auto (T, T, U (&&)[3]) -> TwoArrays<T, U>' aggregate
+// CHECK:   |-CXXDeductionGuideDecl {{.+}} <col:36> col:36 implicit <deduction guide for TwoArrays> 'auto (T, T, U (&&)[3]) -> GH64625::TwoArrays<T, U>' aggregate
 // CHECK-NEXT: | |-ParmVarDecl {{.+}} <col:36> col:36 'T'
 // CHECK-NEXT: | |-ParmVarDecl {{.+}} <col:36> col:36 'T'
 // CHECK-NEXT: | `-ParmVarDecl {{.+}} <col:36> col:36 'U (&&)[3]'
@@ -410,8 +407,8 @@ TwoArrays tb = {1, 2, {3, 4, 5}};
 // CHECK-NEXT:   |-ParmVarDecl {{.+}} <col:36> col:36 'int'
 // CHECK-NEXT:   |-ParmVarDecl {{.+}} <col:36> col:36 'int'
 // CHECK-NEXT:   `-ParmVarDecl {{.+}} <col:36> col:36 'int (&&)[3]'
-// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (T, T, U (&&)[3]) -> TwoArrays<T, U>' dependent trailing_return
-// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'TwoArrays<T, U>' dependent
+// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (T, T, U (&&)[3]) -> GH64625::TwoArrays<T, U>' dependent trailing_return
+// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'GH64625::TwoArrays<T, U>' dependent
 // CHECK-NEXT: | `-CXXRecord {{.+}} 'TwoArrays'
 // CHECK-NEXT: |-TemplateTypeParmType {{.+}} 'T' dependent depth 0 index 0
 // CHECK-NEXT: | `-TemplateTypeParm {{.+}} 'T'
@@ -423,7 +420,7 @@ TwoArrays tb = {1, 2, {3, 4, 5}};
 // CHECK-NEXT:       `-TemplateTypeParm {{.+}} 'U'
 
 TwoArrays tc = {{1, 2}, 3, 4, 5};
-// CHECK: |-CXXDeductionGuideDecl {{.+}} <col:36> col:36 implicit <deduction guide for TwoArrays> 'auto (T (&&)[2], U, U, U) -> TwoArrays<T, U>' aggregate
+// CHECK: |-CXXDeductionGuideDecl {{.+}} <col:36> col:36 implicit <deduction guide for TwoArrays> 'auto (T (&&)[2], U, U, U) -> GH64625::TwoArrays<T, U>' aggregate
 // CHECK-NEXT: | |-ParmVarDecl {{.+}} <col:36> col:36 'T (&&)[2]'
 // CHECK-NEXT: | |-ParmVarDecl {{.+}} <col:36> col:36 'U'
 // CHECK-NEXT: | |-ParmVarDecl {{.+}} <col:36> col:36 'U'
@@ -437,8 +434,8 @@ TwoArrays tc = {{1, 2}, 3, 4, 5};
 // CHECK-NEXT:   |-ParmVarDecl {{.+}} <col:36> col:36 'int'
 // CHECK-NEXT:   |-ParmVarDecl {{.+}} <col:36> col:36 'int'
 // CHECK-NEXT:   `-ParmVarDecl {{.+}} <col:36> col:36 'int'
-// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (T (&&)[2], U, U, U) -> TwoArrays<T, U>' dependent trailing_return
-// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'TwoArrays<T, U>' dependent
+// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (T (&&)[2], U, U, U) -> GH64625::TwoArrays<T, U>' dependent trailing_return
+// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'GH64625::TwoArrays<T, U>' dependent
 // CHECK-NEXT: | `-CXXRecord {{.+}} 'TwoArrays'
 // CHECK-NEXT: |-RValueReferenceType {{.+}} 'T (&&)[2]' dependent
 // CHECK-NEXT: | `-ConstantArrayType {{.+}} 'T[2]' dependent 2
@@ -464,13 +461,13 @@ A a{.f1 = {1}};
 // CHECK-LABEL: Dumping GH83368::<deduction guide for A>:
 // CHECK-NEXT: FunctionTemplateDecl 0x{{.+}} <{{.+}}:[[#@LINE - 7]]:1, col:25> col:25 implicit <deduction guide for A>
 // CHECK-NEXT: |-NonTypeTemplateParmDecl {{.+}} <col:11, col:15> col:15 referenced 'int' depth 0 index 0 N
-// CHECK:      |-CXXDeductionGuideDecl {{.+}} <col:25> col:25 implicit <deduction guide for A> 'auto (int (&&)[N]) -> A<N>' aggregate
+// CHECK:      |-CXXDeductionGuideDecl {{.+}} <col:25> col:25 implicit <deduction guide for A> 'auto (int (&&)[N]) -> GH83368::A<N>' aggregate
 // CHECK-NEXT: | `-ParmVarDecl {{.+}} <col:25> col:25 'int (&&)[N]'
 // CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} <col:25> col:25 implicit used <deduction guide for A> 'auto (int (&&)[1]) -> GH83368::A<1>' implicit_instantiation aggregate
 // CHECK-NEXT:   |-TemplateArgument integral '1'
 // CHECK-NEXT:   `-ParmVarDecl {{.+}} <col:25> col:25 'int (&&)[1]'
-// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (int (&&)[N]) -> A<N>' dependent trailing_return
-// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'A<N>' dependent
+// CHECK-NEXT: FunctionProtoType {{.+}} 'auto (int (&&)[N]) -> GH83368::A<N>' dependent trailing_return
+// CHECK-NEXT: |-InjectedClassNameType {{.+}} 'GH83368::A<N>' dependent
 // CHECK-NEXT: | `-CXXRecord {{.+}} 'A'
 // CHECK-NEXT: `-RValueReferenceType {{.+}} 'int (&&)[N]' dependent
 // CHECK-NEXT:   `-DependentSizedArrayType {{.+}} 'int[N]' dependent
@@ -512,7 +509,7 @@ A a(42);
 // CHECK-NEXT: |   |       `-TemplateTypeParm 0x{{.+}} 'Ts'
 // CHECK-NEXT: |   `-ImplicitCastExpr {{.+}} <IntegralCast>
 // CHECK-NEXT: |     `-IntegerLiteral 0x{{.+}} <{{.+}}> 'int' 0
-// CHECK-NEXT: |-CXXDeductionGuideDecl 0x{{.+}} <{{.+}}> line:{{.+}} implicit <deduction guide for A> 'auto (T, Ts...) -> A<T>'
+// CHECK-NEXT: |-CXXDeductionGuideDecl 0x{{.+}} <{{.+}}> line:{{.+}} implicit <deduction guide for A> 'auto (T, Ts...) -> GH60777::A<T>'
 // CHECK-NEXT: | |-ParmVarDecl 0x{{.+}} <{{.+}}> col:{{.+}} val 'T'
 // CHECK-NEXT: | |-ParmVarDecl 0x{{.+}} <{{.+}}> col:{{.+}} tail 'Ts...' pack
 // CHECK-NEXT: | `-BinaryOperator 0x{{.+}} <{{.+}}> 'bool' '&&'
@@ -541,9 +538,9 @@ B b(42, 43);
 // expected-error@-1 {{no viable constructor}} \
 //   expected-note@-6 {{constraints not satisfied}} \
 //   expected-note@-5 {{because substituted constraint expression is ill-formed}} \
-//   expected-note@-6 {{implicit deduction guide declared as 'template <typename T, typename ...Ts> B(T val, Ts ...tail) -> B<T> requires (True<tail...>())'}} \
+//   expected-note@-6 {{implicit deduction guide declared as 'template <typename T, typename ...Ts> B(T val, Ts ...tail) -> GH60777::B<T> requires (True<tail...>())'}} \
 //   expected-note@-8 {{function template not viable}} \
-//   expected-note@-8 {{implicit deduction guide declared as 'template <typename T> B(B<T>) -> B<T>'}}
+//   expected-note@-8 {{implicit deduction guide declared as 'template <typename T> B(GH60777::B<T>) -> GH60777::B<T>'}}
 
 } // namespace GH60777
 
@@ -572,7 +569,7 @@ static_assert(x.size == 4);
 // CHECK-NEXT: FunctionTemplateDecl 0x{{.+}} <{{.+}}> col:13 implicit <deduction guide for X>
 // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{.+}} <{{.+}}> col:17 referenced class depth 0 index 0 T
 // CHECK-NEXT: |-TemplateTypeParmDecl 0x{{.+}} <{{.+}}> col:19 class depth 0 index 1 U
-// CHECK-NEXT: |-CXXDeductionGuideDecl 0x{{.+}} <{{.+}}> col:13 implicit <deduction guide for X> 'auto (T, U (&)[3]) -> X<T>'
+// CHECK-NEXT: |-CXXDeductionGuideDecl 0x{{.+}} <{{.+}}> col:13 implicit <deduction guide for X> 'auto (T, U (&)[3]) -> GH98592::X<T>'
 // CHECK-NEXT: | |-ParmVarDecl 0x{{.+}} <col:15> col:16 'T'
 // CHECK-NEXT: | |-ParmVarDecl 0x{{.+}} <col:18, col:24> col:21 'U (&)[3]'
 // CHECK-NEXT: | `-ConceptSpecializationExpr 0x{{.+}} <col:36, col:42> 'bool' Concept 0x{{.+}} 'True'
@@ -719,7 +716,7 @@ void test() { NewDeleteAllocator abc(42); } // expected-error {{no viable constr
 // CHECK-NEXT: |   |-inherited from TemplateTypeParm {{.+}} depth 0 index 0
 // CHECK-NEXT: |   `-BuiltinType {{.+}} 'void'
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.+}} typename depth 0 index 1 T
-// CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} <deduction guide for NewDeleteAllocator> 'auto (T) -> NewDeleteAllocator<type-parameter-0-0>'
+// CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} <deduction guide for NewDeleteAllocator> 'auto (T) -> GH128691::NewDeleteAllocator<type-parameter-0-0>'
 // CHECK-NEXT:  `-ParmVarDecl {{.+}} 'T'
 
 } // namespace GH128691
@@ -745,7 +742,7 @@ B b(24);
 // CHECK-NEXT: FunctionTemplateDecl {{.+}} implicit <deduction guide for B>
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.+}} typename depth 0 index 0
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.+}} class depth 0 index 1 U
-// CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} implicit <deduction guide for B> 'auto (U) -> B<type-parameter-0-0>'
+// CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} implicit <deduction guide for B> 'auto (U) -> GH132616_DeductionGuide::B<type-parameter-0-0>'
 // CHECK-NEXT:  `-ParmVarDecl {{.+}} 'U'
 
 struct C {
@@ -767,7 +764,7 @@ D d(24);
 // CHECK-NEXT: FunctionTemplateDecl {{.+}} implicit <deduction guide for D>
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.+}} typename depth 0 index 0
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.+}} class depth 0 index 1 U
-// CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} implicit <deduction guide for D> 'auto (U) -> D<type-parameter-0-0>'
+// CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} implicit <deduction guide for D> 'auto (U) -> GH132616_DeductionGuide::D<type-parameter-0-0>'
 // CHECK-NEXT:  `-ParmVarDecl {{.+}} 'U'
 
 } // namespace GH132616_DeductionGuide
@@ -859,13 +856,12 @@ CC c{};
 // CHECK-NEXT:  |   `-IntegerLiteral {{.+}} 'int' 42
 // CHECK-NEXT:  |-TemplateTypeParmDecl {{.+}} class depth 0 index 1 U
 // CHECK-NEXT:  | `-TemplateArgument type 'A<decltype(N)>'
-// CHECK-NEXT:  |   `-ElaboratedType {{.+}} 'A<decltype(N)>' sugar dependent
-// CHECK-NEXT:  |     `-TemplateSpecializationType {{.+}} 'A<decltype(N)>' dependent
-// CHECK-NEXT:  |       |-name: 'A':'GH133132::A' qualified
-// CHECK-NEXT:  |       | `-ClassTemplateDecl {{.+}} A
-// CHECK-NEXT:  |       `-TemplateArgument type 'decltype(N)'
-// CHECK-NEXT:  |         `-DecltypeType {{.+}} 'decltype(N)' dependent
-// CHECK-NEXT:  |           `-DeclRefExpr {{.+}} 'int' NonTypeTemplateParm {{.+}} 'N' 'int'
+// CHECK-NEXT:  |   `-TemplateSpecializationType {{.+}} 'A<decltype(N)>' dependent
+// CHECK-NEXT:  |     |-name: 'A':'GH133132::A' qualified
+// CHECK-NEXT:  |     | `-ClassTemplateDecl {{.+}} A
+// CHECK-NEXT:  |     `-TemplateArgument type 'decltype(N)'
+// CHECK-NEXT:  |       `-DecltypeType {{.+}} 'decltype(N)' dependent
+// CHECK-NEXT:  |         `-DeclRefExpr {{.+}} 'int' NonTypeTemplateParm {{.+}} 'N' 'int'
 // CHECK-NEXT:  |-TypeTraitExpr {{.+}} 'bool' __is_deducible
 // CHECK-NEXT:  | |-DeducedTemplateSpecializationType {{.+}} 'GH133132::CC' dependent
 // CHECK-NEXT:  | | `-name: 'GH133132::CC'
@@ -903,7 +899,7 @@ void f() {
 // CHECK-LABEL: Dumping GH67173::<deduction guide for Vec2d>:
 // CHECK-NEXT: FunctionTemplateDecl {{.+}} implicit <deduction guide for Vec2d>
 // CHECK-NEXT: |-TemplateTypeParmDecl {{.+}} referenced class depth 0 index 0 T
-// CHECK:      |-CXXDeductionGuideDecl {{.+}} implicit <deduction guide for Vec2d> 'auto (T, T) -> Vec2d<T>' aggregate
+// CHECK:      |-CXXDeductionGuideDecl {{.+}} implicit <deduction guide for Vec2d> 'auto (T, T) -> GH67173::Vec2d<T>' aggregate
 // CHECK-NEXT: | |-ParmVarDecl {{.+}} col:27 'T'
 // CHECK-NEXT: | `-ParmVarDecl {{.+}} col:27 'T'
 // CHECK-NEXT: `-CXXDeductionGuideDecl {{.+}} implicit used <deduction guide for Vec2d> 'auto (int, int) -> GH67173::Vec2d<int>' implicit_instantiation aggregate
diff --git a/clang/test/SemaTemplate/dedup-types-builtin.cpp b/clang/test/SemaTemplate/dedup-types-builtin.cpp
new file mode 100644
index 0000000000000..fe6efd7901695
--- /dev/null
+++ b/clang/test/SemaTemplate/dedup-types-builtin.cpp
@@ -0,0 +1,225 @@
+// RUN: %clang_cc1 %s -verify -Wno-c++20-extensions
+template <typename...> struct TypeList;
+
+// === Check results of the builtin.
+template <class>
+struct TemplateWrapper {
+  static_assert(__is_same( // expected-error {{static assertion contains an unexpanded parameter pack}}
+    TypeList<__builtin_dedup_pack<int, int*, int, double, float>>,
+    TypeList<int, int*, double, float>));
+};
+
+template <template<typename ...> typename Templ, typename ...Types>
+struct Dependent {
+  using empty_list = Templ<__builtin_dedup_pack<>...>;
+  using same = Templ<__builtin_dedup_pack<Types...>...>;
+  using twice = Templ<__builtin_dedup_pack<Types..., Types...>...>;
+  using dep_only_types = TypeList<__builtin_dedup_pack<Types...>...>;
+  using dep_only_template = Templ<__builtin_dedup_pack<int, double, int>...>;
+};
+
+// Check the reverse condition to make sure we see an error and not accidentally produced dependent expression.
+static_assert(!__is_same(Dependent<TypeList>::empty_list, TypeList<>)); // expected-error {{static assertion failed}}
+static_assert(!__is_same(Dependent<TypeList>::same, TypeList<>)); // expected-error {{static assertion failed}}
+static_assert(!__is_same(Dependent<TypeList>::twice, TypeList<>)); // expected-error {{static assertion failed}}
+static_assert(!__is_same(Dependent<TypeList>::dep_only_types, TypeList<>)); // expected-error {{static assertion failed}}
+static_assert(!__is_same(Dependent<TypeList>::dep_only_template, TypeList<int, double>)); // expected-error {{static assertion failed}}
+static_assert(!__is_same(Dependent<TypeList, int*, double*, int*>::empty_list, TypeList<>)); // expected-error {{static assertion failed}}
+static_assert(!__is_same(Dependent<TypeList, int*, double*, int*>::same, TypeList<int*, double*>)); // expected-error {{static assertion failed}}
+static_assert(!__is_same(Dependent<TypeList, int*, double*, int*>::twice, TypeList<int*, double*>)); // expected-error {{static assertion failed}}
+static_assert(!__is_same(Dependent<TypeList, int*, double*, int*>::dep_only_types, TypeList<int*, double*>)); // expected-error {{static assertion failed}}
+static_assert(!__is_same(Dependent<TypeList, int*, double*, int*>::dep_only_template, TypeList<int, double>)); // expected-error {{static assertion failed}}
+
+
+template <class ...T>
+using Twice = TypeList<T..., T...>;
+
+template <class>
+struct TwiceTemplateWrapper {
+  static_assert(!__is_same(Twice<__builtin_dedup_pack<int, double, int>...>, TypeList<int, double, int, double>)); // expected-error {{static assertion failed}}
+
+};
+template struct TwiceTemplateWrapper<int>; // expected-note {{in instantiation of template class 'TwiceTemplateWrapper<int>' requested here}}
+
+template <int...> struct IntList;
+// Wrong kinds of template arguments.
+template <class> struct IntListTemplateWrapper {
+  IntList<__builtin_dedup_pack<int>...>* wrong_template; // expected-error {{template argument for non-type template parameter must be an expression}}
+                                                         // expected-note@-4 {{template parameter is declared here}}
+  TypeList<__builtin_dedup_pack<1, 2, 3>...>* wrong_template_args; // expected-error  {{template argument for template type parameter must be a type}}
+                                                                    // expected-note@* {{template parameter from hidden source}}
+  __builtin_dedup_pack<> not_enough_args; // expected-error {{data member type contains an unexpanded parameter pack}}
+                                          // expected-note@* {{template declaration from hidden source}}
+  __builtin_dedup_pack missing_template_args; // expected-error {{use of template '__builtin_dedup_pack' requires template arguments}}
+};
+
+// Make sure various canonical / non-canonical type representations do not affect results
+// of the deduplication and the qualifiers do end up creating different types when C++ requires it.
+using Int = int;
+using CInt = const Int;
+using IntArray = Int[10];
+using CIntArray = Int[10];
+using IntPtr = int*;
+using CIntPtr = const int*;
+
+template <class>
+struct Foo {
+  static_assert(
+    !__is_same( // expected-error {{static assertion failed}}
+                // expected-note@* {{in instantiation of template class 'Foo<int>'}}
+      TypeList<__builtin_dedup_pack<
+        Int, int,
+        const int, const Int, CInt, const CInt,
+        IntArray, Int[10], int[10],
+        const IntArray, const int[10], CIntArray, const CIntArray,
+        IntPtr, int*,
+        const IntPtr, int* const,
+        CIntPtr, const int*,
+        const IntPtr*, int*const*,
+        CIntPtr*, const int**,
+        const CIntPtr*, const int* const*
+      >...>,
+      TypeList<int, const int, int[10], const int [10], int*, int* const, const int*, int*const *, const int**, const int*const*>),
+    "");
+};
+
+template struct Foo<int>;
+
+// === Show an error when packs are used in non-template contexts.
+static_assert(!__is_same(TypeList<__builtin_dedup_pack<int>...>, TypeList<int>)); // expected-error {{outside}}
+// Non-dependent uses in template are fine, though.
+template <class T>
+struct NonDepInTemplate {
+  static_assert(!__is_same(TypeList<__builtin_dedup_pack<int>...>, TypeList<int>)); // expected-error {{static assertion failed}}
+};
+template struct NonDepInTemplate<int>; // expected-note {{requested here}}
+
+template <template<class...> class T = __builtin_dedup_pack> // expected-error {{use of template '__builtin_dedup_pack' requires template arguments}}
+                                                             // expected-note@* {{template declaration from hidden source}}
+struct UseAsTemplate;
+template <template<class...> class>
+struct AcceptsTemplateArg;
+template <class>
+struct UseAsTemplateWrapper {
+  AcceptsTemplateArg<__builtin_dedup_pack>* a; // expected-error {{use of template '__builtin_dedup_pack' requires template arguments}}
+                                               // expected-note@* {{template declaration from hidden source}}
+};
+
+// === Check how expansions in various contexts behave.
+// The following cases are not supported yet, should produce an error.
+template <class... T>
+struct DedupBases : __builtin_dedup_pack<T...>... {};
+struct Base1 {
+   int a1;
+};
+struct Base2 {
+   int a2;
+};
+static_assert(DedupBases<Base1, Base1, Base2, Base1, Base2, Base2>{1, 2}.a1 != 1); // expected-error {{static assertion failed}} \
+                                                                                   // expected-note {{}}
+static_assert(DedupBases<Base1, Base1, Base2, Base1, Base2, Base2>{1, 2}.a2 != 2); // expected-error {{static assertion failed}} \
+                                                                                   // expected-note {{}}
+
+template <class ...T>
+constexpr int dedup_params(__builtin_dedup_pack<T...>... as) {
+ return (as + ...);
+}
+static_assert(dedup_params<int, int, short, int, short, short>(1, 2)); // expected-error {{no matching function}} \
+                                                                       // expected-note@-3 {{expansions of '__builtin_dedup_pack' are not supported here}}
+
+template <class ...T>
+constexpr int dedup_params_into_type_list(TypeList<__builtin_dedup_pack<T...>...> *, T... as) {
+ return (as + ...);
+}
+static_assert(dedup_params_into_type_list(static_cast<TypeList<int,short,long>*>(nullptr), 1, short(1), 1, 1l, 1l) != 5); // expected-error {{static assertion failed}} \
+                                                                                                                          // expected-note {{expression evaluates}}
+
+template <class T, __builtin_dedup_pack<T, int>...> // expected-error 2{{expansions of '__builtin_dedup_pack' are not supported here}}
+struct InTemplateParams {};
+InTemplateParams<int> itp1;
+InTemplateParams<int, 1, 2, 3, 4, 5> itp2;
+
+template <class T>
+struct DeepTemplateParams {
+  template <__builtin_dedup_pack<T, int>...> // expected-error {{expansions of '__builtin_dedup_pack' are not supported here}}
+  struct Templ {};
+};
+DeepTemplateParams<int>::Templ<> dtp1; // expected-note {{requested here}} \
+                                       // expected-error {{no template named 'Templ'}}
+
+
+template <class ...T>
+struct MemInitializers : T... {
+  MemInitializers() : __builtin_dedup_pack<T...>()... {} // expected-error 2{{expansions of '__builtin_dedup_pack' are not supported here.}}
+};
+MemInitializers<> mi1; // expected-note {{in instantiation of member function}}
+MemInitializers<Base1, Base2> mi2; // expected-note {{in instantiation of member function}}
+
+template <class ...T>
+constexpr int dedup_in_expressions() {
+ // counts the number of unique Ts.
+ return ((1 + __builtin_dedup_pack<T...>()) + ...); // expected-error {{expansions of '__builtin_dedup_pack' are not supported here.}} \
+                                                    // expected-note@+3 {{in instantiation of function template specialization}}
+}
+static_assert(dedup_in_expressions<int, int, short, double, int, short, double, int>() == 3); // expected-error {{not an integral constant expression}}
+
+template <class ...T>
+void in_exception_spec() throw(__builtin_dedup_pack<T...>...); // expected-error{{C++17 does not allow dynamic exception specifications}} \
+                                                               // expected-note {{use 'noexcept}} \
+                                                               // expected-error{{expansions of '__builtin_dedup_pack' are not supported here.}}
+
+void test_in_exception_spec() {
+  in_exception_spec<int, double, int>(); // expected-note {{instantiation of exception specification}}
+}
+
+template <class ...T>
+constexpr bool in_type_trait = __is_trivially_constructible(int, __builtin_dedup_pack<T...>...);  // expected-error{{expansions of '__builtin_dedup_pack' are not supported here.}}
+
+static_assert(in_type_trait<int, int, int>); // expected-note{{in instantiation of variable template specialization}}
+
+template <class ...T>
+struct InFriends {
+  friend __builtin_dedup_pack<T>...; // expected-warning {{variadic 'friend' declarations are a C++2c extension}} \
+                                     // expected-error  2 {{expansions of '__builtin_dedup_pack' are not supported here.}} \
+                                     // expected-note@* 2 {{in instantiation of template class}}
+
+};
+struct Friend1 {};
+struct Friend2 {};
+InFriends<> if1;
+InFriends<Friend1, Friend2> if2;
+
+template <class ...T>
+struct InUsingDecl {
+  using __builtin_dedup_pack<T...>::func...; // expected-error  2 {{expansions of '__builtin_dedup_pack' are not supported here.}}
+};
+struct WithFunc1 { void func(); };
+struct WithFunc2 { void func(int); };
+InUsingDecl<> iu1; // expected-note {{in instantiation of template class}}
+InUsingDecl<WithFunc1, WithFunc2> iu2; // expected-note {{in instantiation of template class}}
+
+// Note: produces parsing errors and does not construct pack indexing.
+// Keep this commented out until the parser supports this.
+//
+// template <class ...T>
+// struct InPackIndexing {
+//
+//   using type = __builtin_dedup_pack<T...>...[0];
+// };
+// static_assert(__is_same(InPackIndexing<int, int>, int));
+
+template <class ...T>
+struct LambdaInitCaptures {
+  static constexpr int test() {
+    [...foos=__builtin_dedup_pack<T...>()]{}; // expected-error 2{{expansions of '__builtin_dedup_pack' are not supported here.}}
+    return 3;
+  }
+};
+static_assert(LambdaInitCaptures<>::test() == 3); // expected-note {{in instantiation of member function}}
+static_assert(LambdaInitCaptures<int, int, int>::test() == 3); // expected-note {{in instantiation of member function}}
+
+template <class ...T>
+struct alignas(__builtin_dedup_pack<T...>...) AlignAs {}; // expected-error 2{{expansions of '__builtin_dedup_pack' are not supported here.}}
+AlignAs<> aa1; // expected-note {{in instantiation of template class}}
+AlignAs<int, double> aa2; // expected-note {{in instantiation of template class}}
+
diff --git a/clang/test/SemaTemplate/dependent-base-classes.cpp b/clang/test/SemaTemplate/dependent-base-classes.cpp
index b4b26698ebf9c..b1c6102ea5666 100644
--- a/clang/test/SemaTemplate/dependent-base-classes.cpp
+++ b/clang/test/SemaTemplate/dependent-base-classes.cpp
@@ -32,7 +32,7 @@ namespace PR6031 {
   template <class TT>
   struct FI2
   {
-    C<typename FI2::type> a; // expected-error{{no type named 'type' in 'FI2<TT>'}}
+    C<typename FI2::type> a; // expected-error{{no type named 'type' in 'PR6031::FI2<TT>'}}
   };
 
   template<typename T>
@@ -54,9 +54,9 @@ namespace PR6031 {
   template<typename T>
   struct NoDepBase {
     int foo() {
-      class NoDepBase::Nested nested; // expected-error{{no class named 'Nested' in 'NoDepBase<T>'}}
-      typedef typename NoDepBase::template MemberTemplate<T>::type type; // expected-error{{no member named 'MemberTemplate' in 'NoDepBase<T>'}}
-      return NoDepBase::a; // expected-error{{no member named 'a' in 'NoDepBase<T>'}}
+      class NoDepBase::Nested nested; // expected-error{{no class named 'Nested' in 'PR6031::NoDepBase<T>'}}
+      typedef typename NoDepBase::template MemberTemplate<T>::type type; // expected-error{{no member named 'MemberTemplate' in 'PR6031::NoDepBase<T>'}}
+      return NoDepBase::a; // expected-error{{no member named 'a' in 'PR6031::NoDepBase<T>'}}
     }
   };
 }
@@ -102,7 +102,7 @@ namespace PR6081 {
     template< class X >
     void f0(const X & k)
     {
-      this->template f1<int>()(k); // expected-error{{no member named 'f1' in 'C<T>'}}
+      this->template f1<int>()(k); // expected-error{{no member named 'f1' in 'PR6081::C<T>'}}
     }
   };
 }
diff --git a/clang/test/SemaTemplate/dependent-names.cpp b/clang/test/SemaTemplate/dependent-names.cpp
index 54ce3760d2d01..d6bd670841c08 100644
--- a/clang/test/SemaTemplate/dependent-names.cpp
+++ b/clang/test/SemaTemplate/dependent-names.cpp
@@ -336,7 +336,7 @@ template < unsigned > struct X {
   static const unsigned dimension = 3;
   template<unsigned dim=dimension>
   struct Y: Y<dim> { }; // expected-error{{base class has incomplete type}}
-                        // expected-note@-1{{definition of 'Y<dim>' is not complete until the closing '}'}}
+                        // expected-note@-1{{definition of 'PR11421::X::Y<dim>' is not complete until the closing '}'}}
 };
 typedef X<3> X3;
 X3::Y<>::iterator it; // expected-error {{no type named 'iterator' in 'PR11421::X<3>::Y<>'}}
diff --git a/clang/test/SemaTemplate/elaborated-type-specifier.cpp b/clang/test/SemaTemplate/elaborated-type-specifier.cpp
index 95c2aa9f60a39..ce6e2588a02b7 100644
--- a/clang/test/SemaTemplate/elaborated-type-specifier.cpp
+++ b/clang/test/SemaTemplate/elaborated-type-specifier.cpp
@@ -79,21 +79,18 @@ namespace canon {
   // expected-note@#canon-t3-3 {{candidate function}}
 
   template <class T> constexpr int t4(typename T::template X<int>* = 0) { return 0; }
-  // expected-note@-1 3{{candidate function}}
+  // expected-note@-1 2{{candidate function}}
   template <class T> constexpr int t4(struct   T::template X<int>* = 0) { return 1; }
-  // expected-note@-1 3{{candidate function}}
+  // expected-note@-1 2{{candidate function}}
   template <class T> constexpr int t4(union    T::template X<int>* = 0) { return 2; }
-  // expected-note@-1 3{{candidate function}}
+  // expected-note@-1 2{{candidate function}}
 
-  // FIXME: This should work.
   struct E { template <class T> using X = T; };
-  static_assert(t4<E>() == 0); // expected-error {{call to 't4' is ambiguous}}
+  static_assert(t4<E>() == 0);
 
-  // FIXME: Should not match the union overload.
   struct F { template <class> struct X {}; };
   static_assert(t4<F>() == 1); // expected-error {{call to 't4' is ambiguous}}
 
-  // FIXME: Should not match the struct overload.
   struct G { template <class> union X {}; };
   static_assert(t4<G>() == 2); // expected-error {{call to 't4' is ambiguous}}
 } // namespace canon
diff --git a/clang/test/SemaTemplate/instantiate-requires-expr.cpp b/clang/test/SemaTemplate/instantiate-requires-expr.cpp
index 47689b93db50f..e60f79230aa00 100644
--- a/clang/test/SemaTemplate/instantiate-requires-expr.cpp
+++ b/clang/test/SemaTemplate/instantiate-requires-expr.cpp
@@ -72,8 +72,8 @@ namespace type_requirement {
 
   template<typename T> requires
   false_v<requires { typename T::template temp<T>; }>
-  // expected-note@-1 {{because 'false_v<requires { typename contains_template<int>::template temp<type_requirement::contains_template<int>>; }>' evaluated to false}}
-  // expected-note@-2 {{because 'false_v<requires { typename contains_template<short>::template temp<type_requirement::contains_template<short>>; }>' evaluated to false}}
+  // expected-note@-1 {{because 'false_v<requires { typename type_requirement::contains_template<int>::template temp<type_requirement::contains_template<int>>; }>' evaluated to false}}
+  // expected-note@-2 {{because 'false_v<requires { typename type_requirement::contains_template<short>::template temp<type_requirement::contains_template<short>>; }>' evaluated to false}}
   struct r2 {};
 
   using r2i1 = r2<contains_template<int>>; // expected-error{{constraints not satisfied for class template 'r2' [with T = type_requirement::contains_template<int>]}}
diff --git a/clang/test/SemaTemplate/make_integer_seq.cpp b/clang/test/SemaTemplate/make_integer_seq.cpp
index cd36d1e2c8e09..1203a5812b991 100644
--- a/clang/test/SemaTemplate/make_integer_seq.cpp
+++ b/clang/test/SemaTemplate/make_integer_seq.cpp
@@ -4,116 +4,110 @@ template <class A1, A1... A2> struct A {};
 
 using test1 = __make_integer_seq<A, int, 1>;
 //      CHECK: |-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <line:5:1, col:43> col:7 test1 '__make_integer_seq<A, int, 1>':'A<int, 0>'
-// CHECK-NEXT:   `-ElaboratedType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, int, 1>' sugar
+// CHECK-NEXT:   `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, int, 1>' sugar alias
+// CHECK-NEXT:     |-name: '__make_integer_seq' qualified
+// CHECK-NEXT:     | `-BuiltinTemplateDecl {{.+}} __make_integer_seq
+// CHECK-NEXT:     |-TemplateArgument template 'A'
+// CHECK-NEXT:     | `-ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:1, col:41> col:38 A
+// CHECK-NEXT:     |-TemplateArgument type 'int'
+// CHECK-NEXT:     | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT:     |-TemplateArgument expr '1'
+// CHECK-NEXT:     | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:42> 'int'
+// CHECK-NEXT:     |   |-value: Int 1
+// CHECK-NEXT:     |   `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:42> 'int' 1
+// CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} 'A<int, 0>' sugar
+// CHECK-NEXT:       |-name: 'A' qualified
+// CHECK-NEXT:       | `-ClassTemplateDecl {{.+}} A
+// CHECK-NEXT:       |-TemplateArgument type 'int'
+// CHECK-NEXT:       | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT:       |-TemplateArgument expr '0'
+// CHECK-NEXT:       | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:42> 'int'
+// CHECK-NEXT:       |   |-value: Int 0
+// CHECK-NEXT:       |   `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:42> 'int' 0
+// CHECK-NEXT:       `-RecordType 0x{{[0-9A-Fa-f]+}} 'A<int, 0>'
+// CHECK-NEXT:         `-ClassTemplateSpecialization 0x{{[0-9A-Fa-f]+}} 'A'
+
+template <class B1, B1 B2> using B = __make_integer_seq<A, B1, B2>;
+using test2 = B<int, 1>;
+//      CHECK: |-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:1, col:23> col:7 test2 'B<int, 1>':'A<int, 0>'
+// CHECK-NEXT:   `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} 'B<int, 1>' sugar alias
+// CHECK-NEXT:     |-name: 'B' qualified
+// CHECK-NEXT:     | `-TypeAliasTemplateDecl {{.+}} B
+// CHECK-NEXT:     |-TemplateArgument type 'int'
+// CHECK-NEXT:     | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT:     |-TemplateArgument expr '1'
+// CHECK-NEXT:     | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:22> 'int'
+// CHECK-NEXT:     |   |-value: Int 1
+// CHECK-NEXT:     |   `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:22> 'int' 1
 // CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, int, 1>' sugar alias
 // CHECK-NEXT:       |-name: '__make_integer_seq' qualified
 // CHECK-NEXT:       | `-BuiltinTemplateDecl {{.+}} __make_integer_seq
 // CHECK-NEXT:       |-TemplateArgument template 'A'
 // CHECK-NEXT:       | `-ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:1, col:41> col:38 A
 // CHECK-NEXT:       |-TemplateArgument type 'int'
-// CHECK-NEXT:       | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT:       | `-SubstTemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'int' sugar class depth 0 index 0 B1 final
+// CHECK-NEXT:       |   |-TypeAliasTemplate 0x{{[0-9A-Fa-f]+}} 'B'
+// CHECK-NEXT:       |   `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
 // CHECK-NEXT:       |-TemplateArgument expr '1'
-// CHECK-NEXT:       | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:42> 'int'
+// CHECK-NEXT:       | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:64> 'int'
 // CHECK-NEXT:       |   |-value: Int 1
-// CHECK-NEXT:       |   `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:42> 'int' 1
+// CHECK-NEXT:       |   `-SubstNonTypeTemplateParmExpr 0x{{[0-9A-Fa-f]+}} <col:64> 'int'
+// CHECK-NEXT:       |     |-NonTypeTemplateParmDecl 0x{{[0-9A-Fa-f]+}} <col:21, col:24> col:24 referenced 'B1' depth 0 index 1 B2
+// CHECK-NEXT:       |     `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:64> 'int' 1
 // CHECK-NEXT:       `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} 'A<int, 0>' sugar
 // CHECK-NEXT:         |-name: 'A' qualified
 // CHECK-NEXT:         | `-ClassTemplateDecl {{.+}} A
 // CHECK-NEXT:         |-TemplateArgument type 'int'
-// CHECK-NEXT:         | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT:         | `-SubstTemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'int' sugar class depth 0 index 0 B1 final
+// CHECK-NEXT:         |   |-TypeAliasTemplate 0x{{[0-9A-Fa-f]+}} 'B'
+// CHECK-NEXT:         |   `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
 // CHECK-NEXT:         |-TemplateArgument expr '0'
-// CHECK-NEXT:         | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:42> 'int'
+// CHECK-NEXT:         | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:64> 'int'
 // CHECK-NEXT:         |   |-value: Int 0
-// CHECK-NEXT:         |   `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:42> 'int' 0
+// CHECK-NEXT:         |   `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:64> 'int' 0
 // CHECK-NEXT:         `-RecordType 0x{{[0-9A-Fa-f]+}} 'A<int, 0>'
 // CHECK-NEXT:           `-ClassTemplateSpecialization 0x{{[0-9A-Fa-f]+}} 'A'
 
-template <class B1, B1 B2> using B = __make_integer_seq<A, B1, B2>;
-using test2 = B<int, 1>;
-//      CHECK: |-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:1, col:23> col:7 test2 'B<int, 1>':'A<int, 0>'
-// CHECK-NEXT:   `-ElaboratedType 0x{{[0-9A-Fa-f]+}} 'B<int, 1>' sugar
-// CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} 'B<int, 1>' sugar alias
-// CHECK-NEXT:       |-name: 'B' qualified
-// CHECK-NEXT:       | `-TypeAliasTemplateDecl {{.+}} B
-// CHECK-NEXT:       |-TemplateArgument type 'int'
-// CHECK-NEXT:       | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
-// CHECK-NEXT:       |-TemplateArgument expr '1'
-// CHECK-NEXT:       | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:22> 'int'
-// CHECK-NEXT:       |   |-value: Int 1
-// CHECK-NEXT:       |   `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:22> 'int' 1
-// CHECK-NEXT:       `-ElaboratedType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, int, 1>' sugar
-// CHECK-NEXT:         `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, int, 1>' sugar alias
-// CHECK-NEXT:           |-name: '__make_integer_seq' qualified
-// CHECK-NEXT:           | `-BuiltinTemplateDecl {{.+}} __make_integer_seq
-// CHECK-NEXT:           |-TemplateArgument template 'A'
-// CHECK-NEXT:           | `-ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:1, col:41> col:38 A
-// CHECK-NEXT:           |-TemplateArgument type 'int'
-// CHECK-NEXT:           | `-SubstTemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'int' sugar class depth 0 index 0 B1 final
-// CHECK-NEXT:           |   |-TypeAliasTemplate 0x{{[0-9A-Fa-f]+}} 'B'
-// CHECK-NEXT:           |   `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
-// CHECK-NEXT:           |-TemplateArgument expr '1'
-// CHECK-NEXT:           | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:64> 'int'
-// CHECK-NEXT:           |   |-value: Int 1
-// CHECK-NEXT:           |   `-SubstNonTypeTemplateParmExpr 0x{{[0-9A-Fa-f]+}} <col:64> 'int'
-// CHECK-NEXT:           |     |-NonTypeTemplateParmDecl 0x{{[0-9A-Fa-f]+}} <col:21, col:24> col:24 referenced 'B1' depth 0 index 1 B2
-// CHECK-NEXT:           |     `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:64> 'int' 1
-// CHECK-NEXT:           `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} 'A<int, 0>' sugar
-// CHECK-NEXT:             |-name: 'A' qualified
-// CHECK-NEXT:             | `-ClassTemplateDecl {{.+}} A
-// CHECK-NEXT:             |-TemplateArgument type 'int'
-// CHECK-NEXT:             | `-SubstTemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'int' sugar class depth 0 index 0 B1 final
-// CHECK-NEXT:             |   |-TypeAliasTemplate 0x{{[0-9A-Fa-f]+}} 'B'
-// CHECK-NEXT:             |   `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
-// CHECK-NEXT:             |-TemplateArgument expr '0'
-// CHECK-NEXT:             | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:64> 'int'
-// CHECK-NEXT:             |   |-value: Int 0
-// CHECK-NEXT:             |   `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:64> 'int' 0
-// CHECK-NEXT:             `-RecordType 0x{{[0-9A-Fa-f]+}} 'A<int, 0>'
-// CHECK-NEXT:               `-ClassTemplateSpecialization 0x{{[0-9A-Fa-f]+}} 'A'
-
 template <template <class T, T...> class S, class T, int N> struct C {
   using test3 = __make_integer_seq<S, T, N>;
 //      CHECK: |-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:3, col:43> col:9 test3 '__make_integer_seq<S, T, N>'
-// CHECK-NEXT:   `-ElaboratedType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<S, T, N>' sugar dependent
-// CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<S, T, N>' dependent
-// CHECK-NEXT:       |-name: '__make_integer_seq'
-// CHECK-NEXT:       | `-BuiltinTemplateDecl {{.+}} __make_integer_seq
-// CHECK-NEXT:       |-TemplateArgument template 'S'
-// CHECK-NEXT:       | | `-TemplateTemplateParmDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:11, col:42> col:42 depth 0 index 0 S
-// CHECK-NEXT:       |-TemplateArgument type 'T'
-// CHECK-NEXT:       | `-TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'T' dependent depth 0 index 1
-// CHECK-NEXT:       |   `-TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'T'
-// CHECK-NEXT:       `-TemplateArgument expr 'N'
-// CHECK-NEXT:         `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:42> 'T' <Dependent>
-// CHECK-NEXT:           `-DeclRefExpr 0x{{[0-9A-Fa-f]+}} <col:42> 'int' NonTypeTemplateParm 0x{{[0-9A-Fa-f]+}} 'N' 'int'
+// CHECK-NEXT:   `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<S, T, N>' dependent
+// CHECK-NEXT:     |-name: '__make_integer_seq'
+// CHECK-NEXT:     | `-BuiltinTemplateDecl {{.+}} __make_integer_seq
+// CHECK-NEXT:     |-TemplateArgument template 'S'
+// CHECK-NEXT:     | | `-TemplateTemplateParmDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:11, col:42> col:42 depth 0 index 0 S
+// CHECK-NEXT:     |-TemplateArgument type 'T'
+// CHECK-NEXT:     | `-TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'T' dependent depth 0 index 1
+// CHECK-NEXT:     |   `-TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'T'
+// CHECK-NEXT:     `-TemplateArgument expr 'N'
+// CHECK-NEXT:       `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:42> 'T' <Dependent>
+// CHECK-NEXT:         `-DeclRefExpr 0x{{[0-9A-Fa-f]+}} <col:42> 'int' NonTypeTemplateParm 0x{{[0-9A-Fa-f]+}} 'N' 'int'
 
   using test4 = __make_integer_seq<A, T, 1>;
 //      CHECK: |-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:3, col:43> col:9 test4 '__make_integer_seq<A, T, 1>'
-// CHECK-NEXT:   `-ElaboratedType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, T, 1>' sugar dependent
-// CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, T, 1>' dependent
-// CHECK-NEXT:       |-name: '__make_integer_seq'
-// CHECK-NEXT:       | `-BuiltinTemplateDecl {{.+}} __make_integer_seq
-// CHECK-NEXT:       |-TemplateArgument template 'A'
-// CHECK-NEXT:       | `-ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:1, col:41> col:38 A
-// CHECK-NEXT:       |-TemplateArgument type 'T'
-// CHECK-NEXT:       | `-TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'T' dependent depth 0 index 1
-// CHECK-NEXT:       |   `-TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'T'
-// CHECK-NEXT:       `-TemplateArgument expr '1'
-// CHECK-NEXT:         `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:42> 'T' <Dependent>
-// CHECK-NEXT:           `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:42> 'int' 1
+// CHECK-NEXT:   `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, T, 1>' dependent
+// CHECK-NEXT:     |-name: '__make_integer_seq'
+// CHECK-NEXT:     | `-BuiltinTemplateDecl {{.+}} __make_integer_seq
+// CHECK-NEXT:     |-TemplateArgument template 'A'
+// CHECK-NEXT:     | `-ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:1, col:41> col:38 A
+// CHECK-NEXT:     |-TemplateArgument type 'T'
+// CHECK-NEXT:     | `-TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'T' dependent depth 0 index 1
+// CHECK-NEXT:     |   `-TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'T'
+// CHECK-NEXT:     `-TemplateArgument expr '1'
+// CHECK-NEXT:       `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:42> 'T' <Dependent>
+// CHECK-NEXT:         `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:42> 'int' 1
 
   using test5 = __make_integer_seq<A, int, N>;
 //      CHECK: `-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:3, col:45> col:9 test5 '__make_integer_seq<A, int, N>'
-// CHECK-NEXT:   `-ElaboratedType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, int, N>' sugar dependent
-// CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, int, N>' dependent
-// CHECK-NEXT:       |-name: '__make_integer_seq'
-// CHECK-NEXT:       | `-BuiltinTemplateDecl {{.+}} __make_integer_seq
-// CHECK-NEXT:       |-TemplateArgument template 'A'
-// CHECK-NEXT:       | `-ClassTemplateDecl 0x{{.+}} <line:{{.+}}:1, col:41> col:38 A
-// CHECK-NEXT:       |-TemplateArgument type 'int'
-// CHECK-NEXT:       | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
-// CHECK-NEXT:       `-TemplateArgument expr 'N'
-// CHECK-NEXT:         `-DeclRefExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:44> 'int' NonTypeTemplateParm 0x{{[0-9A-Fa-f]+}} 'N' 'int'
+// CHECK-NEXT:   `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__make_integer_seq<A, int, N>' dependent
+// CHECK-NEXT:     |-name: '__make_integer_seq'
+// CHECK-NEXT:     | `-BuiltinTemplateDecl {{.+}} __make_integer_seq
+// CHECK-NEXT:     |-TemplateArgument template 'A'
+// CHECK-NEXT:     | `-ClassTemplateDecl 0x{{.+}} <line:{{.+}}:1, col:41> col:38 A
+// CHECK-NEXT:     |-TemplateArgument type 'int'
+// CHECK-NEXT:     | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT:     `-TemplateArgument expr 'N'
+// CHECK-NEXT:       `-DeclRefExpr 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:44> 'int' NonTypeTemplateParm 0x{{[0-9A-Fa-f]+}} 'N' 'int'
 };
 
 // expected-no-diagnostics
diff --git a/clang/test/SemaTemplate/ms-lookup-template-base-classes.cpp b/clang/test/SemaTemplate/ms-lookup-template-base-classes.cpp
index 547e5945ac6bc..16fe8d3c78897 100644
--- a/clang/test/SemaTemplate/ms-lookup-template-base-classes.cpp
+++ b/clang/test/SemaTemplate/ms-lookup-template-base-classes.cpp
@@ -76,7 +76,7 @@ template<typename T> struct C;
 // Test lookup with incomplete lookup context
 template<typename T>
 auto C<T>::f() -> decltype(x) { } // expected-error {{use of undeclared identifier 'x'}}
-                                  // expected-error@-1 {{out-of-line definition of 'f' from class 'C<T>' without definition}}
+                                  // expected-error@-1 {{out-of-line definition of 'f' from class 'lookup_dependent_bases_id_expr::C<T>' without definition}}
 
 }
 
diff --git a/clang/test/SemaTemplate/ms-sizeof-missing-typename.cpp b/clang/test/SemaTemplate/ms-sizeof-missing-typename.cpp
index 463d86f0d3b94..ea723324212a2 100644
--- a/clang/test/SemaTemplate/ms-sizeof-missing-typename.cpp
+++ b/clang/test/SemaTemplate/ms-sizeof-missing-typename.cpp
@@ -4,9 +4,9 @@
 // get the size of this type, so they don't get errors after inserting typename.
 
 namespace basic {
-template <typename T> int type_f() { return sizeof T::type; }  // expected-error {{missing 'typename' prior to dependent type name 'X::type'}}
-template <typename T> int type_g() { return sizeof(T::type); }  // expected-warning {{missing 'typename' prior to dependent type name 'X::type'}}
-template <typename T> int type_h() { return sizeof((T::type)); }  // expected-error {{missing 'typename' prior to dependent type name 'X::type'}}
+template <typename T> int type_f() { return sizeof T::type; }  // expected-error {{missing 'typename' prior to dependent type name 'basic::X::type'}}
+template <typename T> int type_g() { return sizeof(T::type); }  // expected-warning {{missing 'typename' prior to dependent type name 'basic::X::type'}}
+template <typename T> int type_h() { return sizeof((T::type)); }  // expected-error {{missing 'typename' prior to dependent type name 'basic::X::type'}}
 template <typename T> int value_f() { return sizeof T::not_a_type; }
 template <typename T> int value_g() { return sizeof(T::not_a_type); }
 template <typename T> int value_h() { return sizeof((T::not_a_type)); }
@@ -30,9 +30,9 @@ template <typename T>
 struct Foo {
   enum {
     // expected-warning@+2 {{use 'template' keyword to treat 'InnerTemplate' as a dependent template name}}
-    // expected-warning@+1 {{missing 'typename' prior to dependent type name 'Bar::InnerType'}}
+    // expected-warning@+1 {{missing 'typename' prior to dependent type name 'nested_sizeof::Bar::InnerType'}}
     x1 = sizeof(typename T::/*template*/ InnerTemplate<sizeof(/*typename*/ T::InnerType)>),
-    // expected-warning@+1 {{missing 'typename' prior to dependent type name 'Bar::InnerType'}}
+    // expected-warning@+1 {{missing 'typename' prior to dependent type name 'nested_sizeof::Bar::InnerType'}}
     x2 = sizeof(typename T::template InnerTemplate<sizeof(/*typename*/ T::InnerType)>),
     // expected-warning@+1 {{use 'template' keyword to treat 'InnerTemplate' as a dependent template name}}
     y1 = sizeof(typename T::/*template*/ InnerTemplate<sizeof(T::InnerVar)>),
@@ -50,7 +50,7 @@ template struct Foo<Bar>; // expected-note-re {{in instantiation {{.*}} requeste
 }
 
 namespace ambiguous_missing_parens {
-// expected-error@+1 {{'Q::template U' is expected to be a non-type template, but instantiated to a class template}}
+// expected-error@+1 {{'ambiguous_missing_parens::Q::template U' is expected to be a non-type template, but instantiated to a class template}}
 template <typename T> void f() { int a = sizeof T::template U<0> + 4; }
 struct Q {
   // expected-note@+1 {{class template declared here}}
diff --git a/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp b/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
index 5c7a90273d0e0..8e3b1b6b6ca77 100644
--- a/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
+++ b/clang/test/SemaTemplate/nested-implicit-deduction-guides.cpp
@@ -84,11 +84,11 @@ nested_init_list<int>::concept_fail nil_invalid{1, ""};
 // expected-note@#INIT_LIST_INNER_INVALID {{candidate template ignored: constraints not satisfied [with F = const char *]}}
 // expected-note@#INIT_LIST_INNER_INVALID_HEADER {{because 'const char *' does not satisfy 'False'}}
 // expected-note@#False {{because 'false' evaluated to false}}
-// expected-note@#INIT_LIST_INNER_INVALID {{implicit deduction guide declared as 'template <False F> concept_fail(int, F) -> concept_fail<F>'}}
+// expected-note@#INIT_LIST_INNER_INVALID {{implicit deduction guide declared as 'template <False F> concept_fail(int, F) -> nested_init_list<int>::concept_fail<F>'}}
 // expected-note@#INIT_LIST_INNER_INVALID {{candidate function template not viable: requires 1 argument, but 2 were provided}}
-// expected-note@#INIT_LIST_INNER_INVALID {{implicit deduction guide declared as 'template <False F> concept_fail(concept_fail<F>) -> concept_fail<F>'}}
+// expected-note@#INIT_LIST_INNER_INVALID {{implicit deduction guide declared as 'template <False F> concept_fail(nested_init_list<int>::concept_fail<F>) -> nested_init_list<int>::concept_fail<F>'}}
 // expected-note@#INIT_LIST_INNER_INVALID {{candidate function template not viable: requires 0 arguments, but 2 were provided}}
-// expected-note@#INIT_LIST_INNER_INVALID {{implicit deduction guide declared as 'template <False F> concept_fail() -> concept_fail<F>'}}
+// expected-note@#INIT_LIST_INNER_INVALID {{implicit deduction guide declared as 'template <False F> concept_fail() -> nested_init_list<int>::concept_fail<F>'}}
 
 namespace GH88142 {
 
diff --git a/clang/test/SemaTemplate/nested-name-spec-template.cpp b/clang/test/SemaTemplate/nested-name-spec-template.cpp
index d36692570c0c2..153f4e2555e3a 100644
--- a/clang/test/SemaTemplate/nested-name-spec-template.cpp
+++ b/clang/test/SemaTemplate/nested-name-spec-template.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsyntax-only -verify %s -Wno-c++20-extensions
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++98 %s
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
 
@@ -130,7 +130,7 @@ namespace PR9226 {
 
   template<typename T, typename U>
   struct Y {
-    typedef typename T::template f<U> type; // expected-error{{template name refers to non-type template 'X::template f'}}
+    typedef typename T::template f<U> type; // expected-error{{template name refers to non-type template 'PR9226::X::template f'}}
   };
 
   Y<X, int> yxi; // expected-note{{in instantiation of template class 'PR9226::Y<PR9226::X, int>' requested here}}
@@ -154,3 +154,30 @@ namespace sugared_template_instantiation {
   struct B { typedef int type1; };
   typedef A<const B> type2;
 } // namespace sugated_template_instantiation
+
+namespace unresolved_using {
+  template <class> struct A {
+    struct B {
+      typedef int X;
+    };
+  };
+  template <class T> struct C : A<T> {
+    using typename A<T>::B;
+    typedef typename B::X Y;
+  };
+  template struct C<int>;
+} // namespace unresolved_using
+
+#if __cplusplus >= 201703L
+namespace SubstTemplateTypeParmPackType {
+  template <int...> struct A {};
+
+  template <class... Ts> void f() {
+    []<int ... Is>(A<Is...>) { (Ts::g(Is) && ...); }(A<0>{});
+  };
+
+  struct B { static void g(int); };
+
+  template void f<B>();
+} // namespace SubstTemplateTypeParmPackType
+#endif
diff --git a/clang/test/SemaTemplate/nested-template.cpp b/clang/test/SemaTemplate/nested-template.cpp
index 7911cf555f4b4..b5da1b1c3ccd4 100644
--- a/clang/test/SemaTemplate/nested-template.cpp
+++ b/clang/test/SemaTemplate/nested-template.cpp
@@ -152,14 +152,14 @@ namespace PR10924 {
 
   template< class Topology, class ctype >
   template< int codim >
-  class ReferenceElement< Topology, ctype > :: BaryCenterArray // expected-error{{out-of-line definition of 'BaryCenterArray' does not match any declaration in 'ReferenceElement<Topology, ctype>'}}
+  class ReferenceElement< Topology, ctype > :: BaryCenterArray // expected-error{{out-of-line definition of 'BaryCenterArray' does not match any declaration in 'PR10924::ReferenceElement<Topology, ctype>'}}
   {
   };
 }
 
 class Outer1 {
     template <typename T> struct X;
-    template <typename T> int X<T>::func() {} //  expected-error{{out-of-line definition of 'func' from class 'X<T>' without definition}}
+    template <typename T> int X<T>::func() {} //  expected-error{{out-of-line definition of 'func' from class 'Outer1::X<T>' without definition}}
 };
 
 namespace RefPack {
diff --git a/clang/test/SemaTemplate/overload-candidates.cpp b/clang/test/SemaTemplate/overload-candidates.cpp
index de998d74f9af6..a9c86b2118fbb 100644
--- a/clang/test/SemaTemplate/overload-candidates.cpp
+++ b/clang/test/SemaTemplate/overload-candidates.cpp
@@ -16,9 +16,9 @@ void test_dyn_cast(int* ptr) {
   (void)dyn_cast(ptr); // expected-error{{no matching function for call to 'dyn_cast'}}
 }
 
-template<int I, typename T> 
+template<int I, typename T>
   void get(const T&); // expected-note{{candidate template ignored: invalid explicitly-specified argument for template parameter 'I'}}
-template<template<class T> class, typename T> 
+template<template<class T> class, typename T>
   void get(const T&); // expected-note{{candidate template ignored: invalid explicitly-specified argument for 1st template parameter}}
 
 void test_get(void *ptr) {
@@ -100,7 +100,7 @@ namespace PR15673 {
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{default template arguments for a function template are a C++11 extension}}
 #endif
-  // expected-note@+1 {{candidate template ignored: requirement 'a_trait<int>::value' was not satisfied [with T = int]}}
+  // expected-note@+1 {{candidate template ignored: requirement 'PR15673::a_trait<int>::value' was not satisfied [with T = int]}}
   void foo() {}
   void bar() { foo<int>(); } // expected-error {{no matching function for call to 'foo'}}
 
@@ -128,7 +128,7 @@ namespace PR15673 {
 #if __cplusplus <= 199711L
   // expected-warning@-2 {{alias declarations are a C++11 extension}}
 #endif
-  // expected-note@+7 {{candidate template ignored: requirement 'some_trait<int>::value' was not satisfied [with T = int]}}
+  // expected-note@+7 {{candidate template ignored: requirement 'PR15673::some_trait<int>::value' was not satisfied [with T = int]}}
 
   template<typename T,
            typename Requires = unicorns<T> >
@@ -148,7 +148,7 @@ namespace PR15673 {
   template<typename T,
            int n = 42,
            typename std::enable_if<n == 43 || (some_passing_trait<T>::value && some_trait<T>::value), int>::type = 0>
-  void almost_rangesv3(); // expected-note{{candidate template ignored: requirement '42 == 43 || (some_passing_trait<int>::value && some_trait<int>::value)' was not satisfied}}
+  void almost_rangesv3(); // expected-note{{candidate template ignored: requirement '42 == 43 || (PR15673::some_passing_trait<int>::value && PR15673::some_trait<int>::value)' was not satisfied}}
   void test_almost_rangesv3() { almost_rangesv3<int>(); } // expected-error{{no matching function for call to 'almost_rangesv3'}}
 
   #define CONCEPT_REQUIRES_(...)                                        \
@@ -161,6 +161,6 @@ namespace PR15673 {
 #endif
   template<typename T,
            CONCEPT_REQUIRES_(some_passing_trait<T>::value && some_trait<T>::value)>
-  void rangesv3(); // expected-note{{candidate template ignored: requirement 'some_trait<int>::value' was not satisfied [with T = int, x = 42]}}
+  void rangesv3(); // expected-note{{candidate template ignored: requirement 'PR15673::some_trait<int>::value' was not satisfied [with T = int, x = 42]}}
   void test_rangesv3() { rangesv3<int>(); } // expected-error{{no matching function for call to 'rangesv3'}}
 }
diff --git a/clang/test/SemaTemplate/temp_arg_nontype.cpp b/clang/test/SemaTemplate/temp_arg_nontype.cpp
index 9363e748c7028..7d2a010295b47 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype.cpp
@@ -438,7 +438,7 @@ namespace dependent_nested_partial_specialization {
 
   template<template<typename> class X> struct A {
     template<typename T, X<T> N> struct B; // expected-note 2{{here}}
-    template <typename T> struct B<T, 0> {}; // expected-error {{non-type template argument specializes a template parameter with dependent type 'Y<T>' (aka 'T *')}}
+    template <typename T> struct B<T, 0> {}; // expected-error {{non-type template argument specializes a template parameter with dependent type 'dependent_nested_partial_specialization::Y<T>' (aka 'T *')}}
   };
   A<X>::B<int, 0> ax;
   A<Y>::B<int, &n> ay; // expected-error {{undefined}} expected-note {{instantiation of}}
diff --git a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
index c35743b87adbc..9c25e26f43c36 100644
--- a/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
+++ b/clang/test/SemaTemplate/temp_arg_nontype_cxx1z.cpp
@@ -621,3 +621,8 @@ namespace GH73460 {
   int j;
   template struct A<int&, j, j>;
 } // namespace GH73460
+
+namespace GH118190 {
+  template <auto> int x;
+  template <int i> int x<i>;
+}
diff --git a/clang/test/SemaTemplate/template-id-expr.cpp b/clang/test/SemaTemplate/template-id-expr.cpp
index a13013a1ffd47..0527af4cfd887 100644
--- a/clang/test/SemaTemplate/template-id-expr.cpp
+++ b/clang/test/SemaTemplate/template-id-expr.cpp
@@ -202,7 +202,7 @@ struct P {
 struct Q {
   template <typename T> int foo() {
     return T::template I<int>;
-    // expected-error@-1 {{'P::template I' is expected to be a non-type template, but instantiated to a type alias template}}
+    // expected-error@-1 {{'non_functions::PR88832::P::template I' is expected to be a non-type template, but instantiated to a type alias template}}
     // expected-note@#TypeAlias {{type alias template declared here}}
   }
 };
diff --git a/clang/test/SemaTemplate/type_pack_element.cpp b/clang/test/SemaTemplate/type_pack_element.cpp
index 5ff010c7db29c..e0776381823ee 100644
--- a/clang/test/SemaTemplate/type_pack_element.cpp
+++ b/clang/test/SemaTemplate/type_pack_element.cpp
@@ -2,61 +2,57 @@
 
 using test1 = __type_pack_element<0, int>;
 //      CHECK: |-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <<stdin>:3:1, col:41> col:7 test1 '__type_pack_element<0, int>':'int'
-// CHECK-NEXT:   `-ElaboratedType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<0, int>' sugar
-// CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<0, int>' sugar alias
-// CHECK-NEXT:       |-name: '__type_pack_element' qualified
-// CHECK-NEXT:       | `-BuiltinTemplateDecl {{.+}} __type_pack_element
-// CHECK-NEXT:       |-TemplateArgument expr '0'
-// CHECK-NEXT:       | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <col:35> '__size_t':'unsigned long'
-// CHECK-NEXT:       |   |-value: Int 0
-// CHECK-NEXT:       |   `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <col:35> '__size_t':'unsigned long' <IntegralCast>
-// CHECK-NEXT:       |     `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:35> 'int' 0
-// CHECK-NEXT:       |-TemplateArgument type 'int'
-// CHECK-NEXT:       | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
-// CHECK-NEXT:       `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT:   `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<0, int>' sugar alias
+// CHECK-NEXT:     |-name: '__type_pack_element' qualified
+// CHECK-NEXT:     | `-BuiltinTemplateDecl {{.+}} __type_pack_element
+// CHECK-NEXT:     |-TemplateArgument expr '0'
+// CHECK-NEXT:     | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <col:35> '__size_t':'unsigned long'
+// CHECK-NEXT:     |   |-value: Int 0
+// CHECK-NEXT:     |   `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <col:35> '__size_t':'unsigned long' <IntegralCast>
+// CHECK-NEXT:     |     `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:35> 'int' 0
+// CHECK-NEXT:     |-TemplateArgument type 'int'
+// CHECK-NEXT:     | `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT:     `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
 
 template<int N, class ...Ts> struct A {
   using test2 = __type_pack_element<N, Ts...>;
 //      CHECK: |-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:3, col:45> col:9 test2 '__type_pack_element<N, Ts...>'
-// CHECK-NEXT:   `-ElaboratedType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<N, Ts...>' sugar dependent
-// CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<N, Ts...>' dependent
-// CHECK-NEXT:       |-name: '__type_pack_element' qualified
-// CHECK-NEXT:       | `-BuiltinTemplateDecl {{.+}} __type_pack_element
-// CHECK-NEXT:       |-TemplateArgument expr 'N'
-// CHECK-NEXT:       | `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <col:37> '__size_t':'unsigned long' <IntegralCast>
-// CHECK-NEXT:       |   `-DeclRefExpr 0x{{[0-9A-Fa-f]+}} <col:37> 'int' NonTypeTemplateParm 0x{{[0-9A-Fa-f]+}} 'N' 'int'
-// CHECK-NEXT:       `-TemplateArgument type 'Ts...'
-// CHECK-NEXT:         `-PackExpansionType 0x{{[0-9A-Fa-f]+}} 'Ts...' dependent
-// CHECK-NEXT:           `-TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'Ts' dependent contains_unexpanded_pack depth 0 index 1 pack
-// CHECK-NEXT:             `-TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'Ts'
+// CHECK-NEXT:   `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<N, Ts...>' dependent
+// CHECK-NEXT:     |-name: '__type_pack_element' qualified
+// CHECK-NEXT:     | `-BuiltinTemplateDecl {{.+}} __type_pack_element
+// CHECK-NEXT:     |-TemplateArgument expr 'N'
+// CHECK-NEXT:     | `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <col:37> '__size_t':'unsigned long' <IntegralCast>
+// CHECK-NEXT:     |   `-DeclRefExpr 0x{{[0-9A-Fa-f]+}} <col:37> 'int' NonTypeTemplateParm 0x{{[0-9A-Fa-f]+}} 'N' 'int'
+// CHECK-NEXT:     `-TemplateArgument type 'Ts...'
+// CHECK-NEXT:       `-PackExpansionType 0x{{[0-9A-Fa-f]+}} 'Ts...' dependent
+// CHECK-NEXT:         `-TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'Ts' dependent contains_unexpanded_pack depth 0 index 1 pack
+// CHECK-NEXT:           `-TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'Ts'
 
   using test3 = __type_pack_element<0, Ts...>;
 //      CHECK: |-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:3, col:45> col:9 test3 '__type_pack_element<0, Ts...>'
-// CHECK-NEXT:   `-ElaboratedType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<0, Ts...>' sugar dependent
-// CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<0, Ts...>' dependent
-// CHECK-NEXT:       |-name: '__type_pack_element' qualified
-// CHECK-NEXT:       | `-BuiltinTemplateDecl {{.+}} __type_pack_element
-// CHECK-NEXT:       |-TemplateArgument expr '0'
-// CHECK-NEXT:       | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <col:37> '__size_t':'unsigned long'
-// CHECK-NEXT:       |   |-value: Int 0
-// CHECK-NEXT:       |   `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <col:37> '__size_t':'unsigned long' <IntegralCast>
-// CHECK-NEXT:       |     `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:37> 'int' 0
-// CHECK-NEXT:       `-TemplateArgument type 'Ts...'
-// CHECK-NEXT:         `-PackExpansionType 0x{{[0-9A-Fa-f]+}} 'Ts...' dependent
-// CHECK-NEXT:           `-TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'Ts' dependent contains_unexpanded_pack depth 0 index 1 pack
-// CHECK-NEXT:             `-TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'Ts'
+// CHECK-NEXT:   `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<0, Ts...>' dependent
+// CHECK-NEXT:     |-name: '__type_pack_element' qualified
+// CHECK-NEXT:     | `-BuiltinTemplateDecl {{.+}} __type_pack_element
+// CHECK-NEXT:     |-TemplateArgument expr '0'
+// CHECK-NEXT:     | `-ConstantExpr 0x{{[0-9A-Fa-f]+}} <col:37> '__size_t':'unsigned long'
+// CHECK-NEXT:     |   |-value: Int 0
+// CHECK-NEXT:     |   `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <col:37> '__size_t':'unsigned long' <IntegralCast>
+// CHECK-NEXT:     |     `-IntegerLiteral 0x{{[0-9A-Fa-f]+}} <col:37> 'int' 0
+// CHECK-NEXT:     `-TemplateArgument type 'Ts...'
+// CHECK-NEXT:       `-PackExpansionType 0x{{[0-9A-Fa-f]+}} 'Ts...' dependent
+// CHECK-NEXT:         `-TemplateTypeParmType 0x{{[0-9A-Fa-f]+}} 'Ts' dependent contains_unexpanded_pack depth 0 index 1 pack
+// CHECK-NEXT:           `-TemplateTypeParm 0x{{[0-9A-Fa-f]+}} 'Ts'
 
   using test4 = __type_pack_element<N, int>;
 //      CHECK: `-TypeAliasDecl 0x{{[0-9A-Fa-f]+}} <line:{{.+}}:3, col:43> col:9 test4 '__type_pack_element<N, int>'
-// CHECK-NEXT:   `-ElaboratedType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<N, int>' sugar dependent
-// CHECK-NEXT:     `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<N, int>' dependent
-// CHECK-NEXT:       |-name: '__type_pack_element' qualified
-// CHECK-NEXT:       | `-BuiltinTemplateDecl {{.+}} __type_pack_element
-// CHECK-NEXT:       |-TemplateArgument expr 'N'
-// CHECK-NEXT:       | `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <col:37> '__size_t':'unsigned long' <IntegralCast>
-// CHECK-NEXT:       |   `-DeclRefExpr 0x{{[0-9A-Fa-f]+}} <col:37> 'int' NonTypeTemplateParm 0x{{[0-9A-Fa-f]+}} 'N' 'int'
-// CHECK-NEXT:       `-TemplateArgument type 'int'
-// CHECK-NEXT:         `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
+// CHECK-NEXT:   `-TemplateSpecializationType 0x{{[0-9A-Fa-f]+}} '__type_pack_element<N, int>' dependent
+// CHECK-NEXT:     |-name: '__type_pack_element' qualified
+// CHECK-NEXT:     | `-BuiltinTemplateDecl {{.+}} __type_pack_element
+// CHECK-NEXT:     |-TemplateArgument expr 'N'
+// CHECK-NEXT:     | `-ImplicitCastExpr 0x{{[0-9A-Fa-f]+}} <col:37> '__size_t':'unsigned long' <IntegralCast>
+// CHECK-NEXT:     |   `-DeclRefExpr 0x{{[0-9A-Fa-f]+}} <col:37> 'int' NonTypeTemplateParm 0x{{[0-9A-Fa-f]+}} 'N' 'int'
+// CHECK-NEXT:     `-TemplateArgument type 'int'
+// CHECK-NEXT:       `-BuiltinType 0x{{[0-9A-Fa-f]+}} 'int'
 };
 
 // expected-no-diagnostics
diff --git a/clang/test/SemaTemplate/typename-specifier-4.cpp b/clang/test/SemaTemplate/typename-specifier-4.cpp
index 8d9962eafbcd1..8f9dce6e69fa0 100644
--- a/clang/test/SemaTemplate/typename-specifier-4.cpp
+++ b/clang/test/SemaTemplate/typename-specifier-4.cpp
@@ -147,7 +147,7 @@ namespace rdar8740998 {
     // expected-error{{dependent using declaration resolved to type without 'typename'}}
 
     void f() {
-      typename X<T>::iterator i; // expected-error{{typename specifier refers to a dependent using declaration for a value 'iterator' in 'X<T>'}}
+      typename X<T>::iterator i; // expected-error{{typename specifier refers to a dependent using declaration for a value 'iterator' in 'rdar8740998::X<T>'}}
     }
   };
 
diff --git a/clang/test/SemaTemplate/typename-specifier.cpp b/clang/test/SemaTemplate/typename-specifier.cpp
index 01acc34f3bb37..27175c5b0b568 100644
--- a/clang/test/SemaTemplate/typename-specifier.cpp
+++ b/clang/test/SemaTemplate/typename-specifier.cpp
@@ -40,7 +40,7 @@ void test(double d) {
   // expected-warning@-3 2{{'typename' outside of a template is a C++11 extension}}
 #endif
   int five = f(5);
-  
+
   using namespace N;
   for (typename A::type i = 0; i < 10; ++i)
 #if __cplusplus <= 199711L
@@ -102,7 +102,7 @@ D<long> struct_D;  // expected-note {{in instantiation of template class 'D<long
 
 template<typename T> struct E {
   typedef typename T::foo foo;
-  typedef typename foo::bar bar;  // expected-error {{type 'E<F>::foo' (aka 'double') cannot be used prior to '::' because it has no members}}
+  typedef typename foo::bar bar;  // expected-error {{type 'foo' (aka 'double') cannot be used prior to '::' because it has no members}}
 };
 
 struct F {
@@ -245,7 +245,7 @@ void j() {
 
 namespace pointer_vs_multiply {
 int x;
-// expected-error@+1 {{missing 'typename' prior to dependent type name 'B::type_or_int'}}
+// expected-error@+1 {{missing 'typename' prior to dependent type name 'pointer_vs_multiply::B::type_or_int'}}
 template <typename T> void g() { T::type_or_int * x; }
 // expected-error@+1 {{typename specifier refers to non-type member 'type_or_int' in 'pointer_vs_multiply::A'}}
 template <typename T> void h() { typename T::type_or_int * x; }
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index 1957bb1715eb6..d34319131ab6d 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -316,7 +316,7 @@ def is_filesystem_case_insensitive():
 
 # [PR8833] LLP64-incompatible tests
 if not re.match(
-    r"^(aarch64|x86_64).*-(windows-msvc|windows-gnu)$", config.target_triple
+    r"^(aarch64|arm64ec|x86_64).*-(windows-msvc|windows-gnu)$", config.target_triple
 ):
     config.available_features.add("LP64")
 
@@ -410,3 +410,13 @@ def calculate_arch_features(arch_string):
 # possibly be present in system and user configuration files, so disable
 # default configs for the test runs.
 config.environment["CLANG_NO_DEFAULT_CONFIG"] = "1"
+
+if lit_config.update_tests:
+    import sys
+    import os
+
+    utilspath = os.path.join(config.llvm_src_root, "utils")
+    sys.path.append(utilspath)
+    from update_any_test_checks import utc_lit_plugin
+
+    lit_config.test_updaters.append(utc_lit_plugin)
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index 176cf644badcc..f50953a93a412 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -46,7 +46,6 @@ config.ppc_linux_default_ieeelongdouble = @PPC_LINUX_DEFAULT_IEEELONGDOUBLE@
 config.have_llvm_driver = @LLVM_TOOL_LLVM_DRIVER_BUILD@
 config.spirv_tools_tests = @LLVM_INCLUDE_SPIRV_TOOLS_TESTS@
 config.substitutions.append(("%llvm-version-major", "@LLVM_VERSION_MAJOR@"))
-config.has_key_instructions = @LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS@
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 1d91f5f255f18..a56e758fb75d8 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -911,9 +911,9 @@ Error handleOverrideImages(
 
 /// Transforms all the extracted offloading input files into an image that can
 /// be registered by the runtime.
-Expected<SmallVector<StringRef>> linkAndWrapDeviceFiles(
-    SmallVectorImpl<SmallVector<OffloadFile>> &LinkerInputFiles,
-    const InputArgList &Args, char **Argv, int Argc) {
+Expected<SmallVector<StringRef>>
+linkAndWrapDeviceFiles(ArrayRef<SmallVector<OffloadFile>> LinkerInputFiles,
+                       const InputArgList &Args, char **Argv, int Argc) {
   llvm::TimeTraceScope TimeScope("Handle all device input");
 
   std::mutex ImageMtx;
@@ -1069,147 +1069,6 @@ std::optional<std::string> searchLibrary(StringRef Input, StringRef Root,
   return searchLibraryBaseName(Input, Root, SearchPaths);
 }
 
-/// Common redeclaration of needed symbol flags.
-enum Symbol : uint32_t {
-  Sym_None = 0,
-  Sym_Undefined = 1U << 1,
-  Sym_Weak = 1U << 2,
-};
-
-/// Scan the symbols from a BitcodeFile \p Buffer and record if we need to
-/// extract any symbols from it.
-Expected<bool> getSymbolsFromBitcode(MemoryBufferRef Buffer, OffloadKind Kind,
-                                     bool IsArchive, StringSaver &Saver,
-                                     DenseMap<StringRef, Symbol> &Syms) {
-  Expected<IRSymtabFile> IRSymtabOrErr = readIRSymtab(Buffer);
-  if (!IRSymtabOrErr)
-    return IRSymtabOrErr.takeError();
-
-  bool ShouldExtract = !IsArchive;
-  DenseMap<StringRef, Symbol> TmpSyms;
-  for (unsigned I = 0; I != IRSymtabOrErr->Mods.size(); ++I) {
-    for (const auto &Sym : IRSymtabOrErr->TheReader.module_symbols(I)) {
-      if (Sym.isFormatSpecific() || !Sym.isGlobal())
-        continue;
-
-      auto It = Syms.find(Sym.getName());
-      bool NewSymbol = It == Syms.end();
-      auto OldSym = NewSymbol ? Sym_None : It->second;
-
-      // We will extract if it defines a currenlty undefined non-weak
-      // symbol.
-      bool ResolvesStrongReference =
-          ((OldSym & Sym_Undefined && !(OldSym & Sym_Weak)) &&
-           !Sym.isUndefined());
-      // We will extract if it defines a new global symbol visible to the
-      // host. This is only necessary for code targeting an offloading
-      // language.
-      bool NewGlobalSymbol =
-          ((NewSymbol || (OldSym & Sym_Undefined)) && !Sym.isUndefined() &&
-           !Sym.canBeOmittedFromSymbolTable() && Kind != object::OFK_None &&
-           (Sym.getVisibility() != GlobalValue::HiddenVisibility));
-      ShouldExtract |= ResolvesStrongReference | NewGlobalSymbol;
-
-      // Update this symbol in the "table" with the new information.
-      if (OldSym & Sym_Undefined && !Sym.isUndefined())
-        TmpSyms[Saver.save(Sym.getName())] =
-            static_cast<Symbol>(OldSym & ~Sym_Undefined);
-      if (Sym.isUndefined() && NewSymbol)
-        TmpSyms[Saver.save(Sym.getName())] =
-            static_cast<Symbol>(OldSym | Sym_Undefined);
-      if (Sym.isWeak())
-        TmpSyms[Saver.save(Sym.getName())] =
-            static_cast<Symbol>(OldSym | Sym_Weak);
-    }
-  }
-
-  // If the file gets extracted we update the table with the new symbols.
-  if (ShouldExtract)
-    Syms.insert_range(TmpSyms);
-
-  return ShouldExtract;
-}
-
-/// Scan the symbols from an ObjectFile \p Obj and record if we need to extract
-/// any symbols from it.
-Expected<bool> getSymbolsFromObject(const ObjectFile &Obj, OffloadKind Kind,
-                                    bool IsArchive, StringSaver &Saver,
-                                    DenseMap<StringRef, Symbol> &Syms) {
-  bool ShouldExtract = !IsArchive;
-  DenseMap<StringRef, Symbol> TmpSyms;
-  for (SymbolRef Sym : Obj.symbols()) {
-    auto FlagsOrErr = Sym.getFlags();
-    if (!FlagsOrErr)
-      return FlagsOrErr.takeError();
-
-    if (!(*FlagsOrErr & SymbolRef::SF_Global) ||
-        (*FlagsOrErr & SymbolRef::SF_FormatSpecific))
-      continue;
-
-    auto NameOrErr = Sym.getName();
-    if (!NameOrErr)
-      return NameOrErr.takeError();
-
-    bool NewSymbol = Syms.count(*NameOrErr) == 0;
-    auto OldSym = NewSymbol ? Sym_None : Syms[*NameOrErr];
-
-    // We will extract if it defines a currenlty undefined non-weak symbol.
-    bool ResolvesStrongReference = (OldSym & Sym_Undefined) &&
-                                   !(OldSym & Sym_Weak) &&
-                                   !(*FlagsOrErr & SymbolRef::SF_Undefined);
-
-    // We will extract if it defines a new global symbol visible to the
-    // host. This is only necessary for code targeting an offloading
-    // language.
-    bool NewGlobalSymbol =
-        ((NewSymbol || (OldSym & Sym_Undefined)) &&
-         !(*FlagsOrErr & SymbolRef::SF_Undefined) && Kind != object::OFK_None &&
-         !(*FlagsOrErr & SymbolRef::SF_Hidden));
-    ShouldExtract |= ResolvesStrongReference | NewGlobalSymbol;
-
-    // Update this symbol in the "table" with the new information.
-    if (OldSym & Sym_Undefined && !(*FlagsOrErr & SymbolRef::SF_Undefined))
-      TmpSyms[Saver.save(*NameOrErr)] =
-          static_cast<Symbol>(OldSym & ~Sym_Undefined);
-    if (*FlagsOrErr & SymbolRef::SF_Undefined && NewSymbol)
-      TmpSyms[Saver.save(*NameOrErr)] =
-          static_cast<Symbol>(OldSym | Sym_Undefined);
-    if (*FlagsOrErr & SymbolRef::SF_Weak)
-      TmpSyms[Saver.save(*NameOrErr)] = static_cast<Symbol>(OldSym | Sym_Weak);
-  }
-
-  // If the file gets extracted we update the table with the new symbols.
-  if (ShouldExtract)
-    Syms.insert_range(TmpSyms);
-
-  return ShouldExtract;
-}
-
-/// Attempt to 'resolve' symbols found in input files. We use this to
-/// determine if an archive member needs to be extracted. An archive member
-/// will be extracted if any of the following is true.
-///   1) It defines an undefined symbol in a regular object filie.
-///   2) It defines a global symbol without hidden visibility that has not
-///      yet been defined.
-Expected<bool> getSymbols(StringRef Image, OffloadKind Kind, bool IsArchive,
-                          StringSaver &Saver,
-                          DenseMap<StringRef, Symbol> &Syms) {
-  MemoryBufferRef Buffer = MemoryBufferRef(Image, "");
-  switch (identify_magic(Image)) {
-  case file_magic::bitcode:
-    return getSymbolsFromBitcode(Buffer, Kind, IsArchive, Saver, Syms);
-  case file_magic::elf_relocatable: {
-    Expected<std::unique_ptr<ObjectFile>> ObjFile =
-        ObjectFile::createObjectFile(Buffer);
-    if (!ObjFile)
-      return ObjFile.takeError();
-    return getSymbolsFromObject(**ObjFile, Kind, IsArchive, Saver, Syms);
-  }
-  default:
-    return false;
-  }
-}
-
 /// Search the input files and libraries for embedded device offloading code
 /// and add it to the list of files to be linked. Files coming from static
 /// libraries are only added to the input if they are used by an existing
@@ -1279,7 +1138,6 @@ getDeviceInput(const ArgList &Args) {
 
   // Link all standard input files and update the list of symbols.
   MapVector<OffloadFile::TargetID, SmallVector<OffloadFile, 0>> InputFiles;
-  DenseMap<OffloadFile::TargetID, DenseMap<StringRef, Symbol>> Syms;
   for (OffloadFile &Binary : ObjectFilesToExtract) {
     if (!Binary.getBinary())
       continue;
@@ -1290,12 +1148,6 @@ getDeviceInput(const ArgList &Args) {
         CompatibleTargets.emplace_back(ID);
 
     for (const auto &[Index, ID] : llvm::enumerate(CompatibleTargets)) {
-      Expected<bool> ExtractOrErr = getSymbols(
-          Binary.getBinary()->getImage(), Binary.getBinary()->getOffloadKind(),
-          /*IsArchive=*/false, Saver, Syms[ID]);
-      if (!ExtractOrErr)
-        return ExtractOrErr.takeError();
-
       // If another target needs this binary it must be copied instead.
       if (Index == CompatibleTargets.size() - 1)
         InputFiles[ID].emplace_back(std::move(Binary));
@@ -1304,55 +1156,33 @@ getDeviceInput(const ArgList &Args) {
     }
   }
 
-  // Archive members only extract if they define needed symbols. We do this
-  // after every regular input file so that libraries may be included out of
-  // order. This follows 'ld.lld' semantics which are more lenient.
-  bool Extracted = true;
   llvm::DenseSet<StringRef> ShouldExtract;
   for (auto &Arg : Args.getAllArgValues(OPT_should_extract))
     ShouldExtract.insert(Arg);
-  while (Extracted) {
-    Extracted = false;
-    for (OffloadFile &Binary : ArchiveFilesToExtract) {
-      // If the binary was previously extracted it will be set to null.
-      if (!Binary.getBinary())
-        continue;
-
-      SmallVector<OffloadFile::TargetID> CompatibleTargets = {Binary};
-      for (const auto &[ID, Input] : InputFiles)
-        if (object::areTargetsCompatible(Binary, ID))
-          CompatibleTargets.emplace_back(ID);
-
-      for (const auto &[Index, ID] : llvm::enumerate(CompatibleTargets)) {
-        // Only extract an if we have an an object matching this target or it
-        // was specifically requested.
-        if (!InputFiles.count(ID) && !ShouldExtract.contains(ID.second))
-          continue;
-
-        Expected<bool> ExtractOrErr =
-            getSymbols(Binary.getBinary()->getImage(),
-                       Binary.getBinary()->getOffloadKind(),
-                       /*IsArchive=*/true, Saver, Syms[ID]);
-        if (!ExtractOrErr)
-          return ExtractOrErr.takeError();
 
-        Extracted = *ExtractOrErr;
+  // We only extract archive members from the fat binary if we find a used or
+  // requested target. Unlike normal static archive handling, we just extract
+  // every object file contained in the archive.
+  for (OffloadFile &Binary : ArchiveFilesToExtract) {
+    if (!Binary.getBinary())
+      continue;
 
-        // Skip including the file if it is an archive that does not resolve
-        // any symbols.
-        if (!Extracted && !ShouldExtract.contains(ID.second))
-          continue;
+    SmallVector<OffloadFile::TargetID> CompatibleTargets = {Binary};
+    for (const auto &[ID, Input] : InputFiles)
+      if (object::areTargetsCompatible(Binary, ID))
+        CompatibleTargets.emplace_back(ID);
 
-        // If another target needs this binary it must be copied instead.
-        if (Index == CompatibleTargets.size() - 1)
-          InputFiles[ID].emplace_back(std::move(Binary));
-        else
-          InputFiles[ID].emplace_back(Binary.copy());
-      }
+    for (const auto &[Index, ID] : llvm::enumerate(CompatibleTargets)) {
+      // Only extract an if we have an an object matching this target or it
+      // was specifically requested.
+      if (!InputFiles.count(ID) && !ShouldExtract.contains(ID.second))
+        continue;
 
-      // If we extracted any files we need to check all the symbols again.
-      if (Extracted)
-        break;
+      // If another target needs this binary it must be copied instead.
+      if (Index == CompatibleTargets.size() - 1)
+        InputFiles[ID].emplace_back(std::move(Binary));
+      else
+        InputFiles[ID].emplace_back(Binary.copy());
     }
   }
 
diff --git a/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp b/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp
index 4b63971214f81..58eb671c61989 100644
--- a/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp
+++ b/clang/tools/clang-nvlink-wrapper/ClangNVLinkWrapper.cpp
@@ -286,12 +286,16 @@ struct Symbol {
 };
 
 Expected<StringRef> runPTXAs(StringRef File, const ArgList &Args) {
-  std::string CudaPath = Args.getLastArgValue(OPT_cuda_path_EQ).str();
-  std::string GivenPath = Args.getLastArgValue(OPT_ptxas_path_EQ).str();
-  Expected<std::string> PTXAsPath =
-      findProgram(Args, "ptxas", {CudaPath + "/bin", GivenPath});
+  SmallVector<StringRef, 1> SearchPaths;
+  if (Arg *A = Args.getLastArg(OPT_cuda_path_EQ))
+    SearchPaths.push_back(Args.MakeArgString(A->getValue() + Twine("/bin")));
+  if (Arg *A = Args.getLastArg(OPT_ptxas_path_EQ))
+    SearchPaths.push_back(Args.MakeArgString(A->getValue()));
+
+  Expected<std::string> PTXAsPath = findProgram(Args, "ptxas", SearchPaths);
   if (!PTXAsPath)
     return PTXAsPath.takeError();
+
   if (!Args.hasArg(OPT_arch))
     return createStringError(
         "must pass in an explicit nvptx64 gpu architecture to 'ptxas'");
@@ -691,9 +695,11 @@ Error runNVLink(ArrayRef<StringRef> Files, const ArgList &Args) {
   if (Args.hasArg(OPT_lto_emit_asm) || Args.hasArg(OPT_lto_emit_llvm))
     return Error::success();
 
-  std::string CudaPath = Args.getLastArgValue(OPT_cuda_path_EQ).str();
-  Expected<std::string> NVLinkPath =
-      findProgram(Args, "nvlink", {CudaPath + "/bin"});
+  SmallVector<StringRef, 1> SearchPaths;
+  if (Arg *A = Args.getLastArg(OPT_cuda_path_EQ))
+    SearchPaths.push_back(Args.MakeArgString(A->getValue() + Twine("/bin")));
+
+  Expected<std::string> NVLinkPath = findProgram(Args, "nvlink", SearchPaths);
   if (!NVLinkPath)
     return NVLinkPath.takeError();
 
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 95cda674a5895..e1aa9bb0721f8 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -1420,79 +1420,25 @@ bool CursorVisitor::VisitDeclarationNameInfo(DeclarationNameInfo Name) {
   llvm_unreachable("Invalid DeclarationName::Kind!");
 }
 
-bool CursorVisitor::VisitNestedNameSpecifier(NestedNameSpecifier *NNS,
-                                             SourceRange Range) {
-  // FIXME: This whole routine is a hack to work around the lack of proper
-  // source information in nested-name-specifiers (PR5791). Since we do have
-  // a beginning source location, we can visit the first component of the
-  // nested-name-specifier, if it's a single-token component.
-  if (!NNS)
-    return false;
-
-  // Get the first component in the nested-name-specifier.
-  while (NestedNameSpecifier *Prefix = NNS->getPrefix())
-    NNS = Prefix;
-
-  switch (NNS->getKind()) {
-  case NestedNameSpecifier::Namespace:
-    return Visit(
-        MakeCursorNamespaceRef(NNS->getAsNamespace(), Range.getBegin(), TU));
-
-  case NestedNameSpecifier::TypeSpec: {
-    // If the type has a form where we know that the beginning of the source
-    // range matches up with a reference cursor. Visit the appropriate reference
-    // cursor.
-    const Type *T = NNS->getAsType();
-    if (const TypedefType *Typedef = dyn_cast<TypedefType>(T))
-      return Visit(MakeCursorTypeRef(Typedef->getDecl(), Range.getBegin(), TU));
-    if (const TagType *Tag = dyn_cast<TagType>(T))
-      return Visit(MakeCursorTypeRef(Tag->getDecl(), Range.getBegin(), TU));
-    if (const TemplateSpecializationType *TST =
-            dyn_cast<TemplateSpecializationType>(T))
-      return VisitTemplateName(TST->getTemplateName(), Range.getBegin());
-    break;
-  }
-
-  case NestedNameSpecifier::Global:
-  case NestedNameSpecifier::Identifier:
-  case NestedNameSpecifier::Super:
-    break;
-  }
-
-  return false;
-}
-
 bool CursorVisitor::VisitNestedNameSpecifierLoc(
     NestedNameSpecifierLoc Qualifier) {
-  SmallVector<NestedNameSpecifierLoc, 4> Qualifiers;
-  for (; Qualifier; Qualifier = Qualifier.getPrefix())
-    Qualifiers.push_back(Qualifier);
-
-  while (!Qualifiers.empty()) {
-    NestedNameSpecifierLoc Q = Qualifiers.pop_back_val();
-    NestedNameSpecifier *NNS = Q.getNestedNameSpecifier();
-    switch (NNS->getKind()) {
-    case NestedNameSpecifier::Namespace:
-      if (Visit(MakeCursorNamespaceRef(NNS->getAsNamespace(),
-                                       Q.getLocalBeginLoc(), TU)))
-        return true;
-
-      break;
-
-    case NestedNameSpecifier::TypeSpec:
-      if (Visit(Q.getTypeLoc()))
-        return true;
-
-      break;
-
-    case NestedNameSpecifier::Global:
-    case NestedNameSpecifier::Identifier:
-    case NestedNameSpecifier::Super:
-      break;
-    }
+  NestedNameSpecifier NNS = Qualifier.getNestedNameSpecifier();
+  switch (NNS.getKind()) {
+  case NestedNameSpecifier::Kind::Namespace: {
+    auto [Namespace, Prefix] = Qualifier.castAsNamespaceAndPrefix();
+    if (VisitNestedNameSpecifierLoc(Prefix))
+      return true;
+    return Visit(
+        MakeCursorNamespaceRef(Namespace, Qualifier.getLocalBeginLoc(), TU));
   }
-
-  return false;
+  case NestedNameSpecifier::Kind::Type:
+    return Visit(Qualifier.castAsTypeLoc());
+  case NestedNameSpecifier::Kind::Null:
+  case NestedNameSpecifier::Kind::Global:
+  case NestedNameSpecifier::Kind::MicrosoftSuper:
+    return false;
+  }
+  llvm_unreachable("unexpected nested name specifier kind");
 }
 
 bool CursorVisitor::VisitTemplateParameters(
@@ -1515,16 +1461,23 @@ bool CursorVisitor::VisitTemplateParameters(
   return false;
 }
 
-bool CursorVisitor::VisitTemplateName(TemplateName Name, SourceLocation Loc) {
+bool CursorVisitor::VisitTemplateName(TemplateName Name, SourceLocation NameLoc,
+                                      NestedNameSpecifierLoc NNS) {
   switch (Name.getKind()) {
+  case TemplateName::QualifiedTemplate: {
+    const QualifiedTemplateName *QTN = Name.getAsQualifiedTemplateName();
+    assert(QTN->getQualifier() == NNS.getNestedNameSpecifier());
+    if (VisitNestedNameSpecifierLoc(NNS))
+      return true;
+    return VisitTemplateName(QTN->getUnderlyingTemplate(), NameLoc, /*NNS=*/{});
+  }
   case TemplateName::Template:
   case TemplateName::UsingTemplate:
-  case TemplateName::QualifiedTemplate: // FIXME: Visit nested-name-specifier.
-    return Visit(MakeCursorTemplateRef(Name.getAsTemplateDecl(), Loc, TU));
+    return Visit(MakeCursorTemplateRef(Name.getAsTemplateDecl(), NameLoc, TU));
 
   case TemplateName::OverloadedTemplate:
     // Visit the overloaded template set.
-    if (Visit(MakeCursorOverloadedDeclRef(Name, Loc, TU)))
+    if (Visit(MakeCursorOverloadedDeclRef(Name, NameLoc, TU)))
       return true;
 
     return false;
@@ -1533,17 +1486,19 @@ bool CursorVisitor::VisitTemplateName(TemplateName Name, SourceLocation Loc) {
     // FIXME: Visit DeclarationName?
     return false;
 
-  case TemplateName::DependentTemplate:
-    // FIXME: Visit nested-name-specifier.
-    return false;
+  case TemplateName::DependentTemplate: {
+    assert(Name.getAsDependentTemplateName()->getQualifier() ==
+           NNS.getNestedNameSpecifier());
+    return VisitNestedNameSpecifierLoc(NNS);
+  }
 
   case TemplateName::SubstTemplateTemplateParm:
     return Visit(MakeCursorTemplateRef(
-        Name.getAsSubstTemplateTemplateParm()->getParameter(), Loc, TU));
+        Name.getAsSubstTemplateTemplateParm()->getParameter(), NameLoc, TU));
 
   case TemplateName::SubstTemplateTemplateParmPack:
     return Visit(MakeCursorTemplateRef(
-        Name.getAsSubstTemplateTemplateParmPack()->getParameterPack(), Loc,
+        Name.getAsSubstTemplateTemplateParmPack()->getParameterPack(), NameLoc,
         TU));
 
   case TemplateName::DeducedTemplate:
@@ -1587,11 +1542,9 @@ bool CursorVisitor::VisitTemplateArgumentLoc(const TemplateArgumentLoc &TAL) {
 
   case TemplateArgument::Template:
   case TemplateArgument::TemplateExpansion:
-    if (VisitNestedNameSpecifierLoc(TAL.getTemplateQualifierLoc()))
-      return true;
-
     return VisitTemplateName(TAL.getArgument().getAsTemplateOrTemplatePattern(),
-                             TAL.getTemplateNameLoc());
+                             TAL.getTemplateNameLoc(),
+                             TAL.getTemplateQualifierLoc());
   }
 
   llvm_unreachable("Invalid TemplateArgument::Kind!");
@@ -1669,7 +1622,10 @@ bool CursorVisitor::VisitBuiltinTypeLoc(BuiltinTypeLoc TL) {
 }
 
 bool CursorVisitor::VisitTypedefTypeLoc(TypedefTypeLoc TL) {
-  return Visit(MakeCursorTypeRef(TL.getTypedefNameDecl(), TL.getNameLoc(), TU));
+  if (VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
+    return true;
+
+  return Visit(MakeCursorTypeRef(TL.getDecl(), TL.getNameLoc(), TU));
 }
 
 bool CursorVisitor::VisitPredefinedSugarTypeLoc(PredefinedSugarTypeLoc TL) {
@@ -1677,14 +1633,20 @@ bool CursorVisitor::VisitPredefinedSugarTypeLoc(PredefinedSugarTypeLoc TL) {
 }
 
 bool CursorVisitor::VisitUnresolvedUsingTypeLoc(UnresolvedUsingTypeLoc TL) {
+  if (VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
+    return true;
+
   return Visit(MakeCursorTypeRef(TL.getDecl(), TL.getNameLoc(), TU));
 }
 
 bool CursorVisitor::VisitTagTypeLoc(TagTypeLoc TL) {
+  if (VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
+    return true;
+
   if (TL.isDefinition())
-    return Visit(MakeCXCursor(TL.getDecl(), TU, RegionOfInterest));
+    return Visit(MakeCXCursor(TL.getOriginalDecl(), TU, RegionOfInterest));
 
-  return Visit(MakeCursorTypeRef(TL.getDecl(), TL.getNameLoc(), TU));
+  return Visit(MakeCursorTypeRef(TL.getOriginalDecl(), TL.getNameLoc(), TU));
 }
 
 bool CursorVisitor::VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc TL) {
@@ -1763,7 +1725,10 @@ bool CursorVisitor::VisitRValueReferenceTypeLoc(RValueReferenceTypeLoc TL) {
 }
 
 bool CursorVisitor::VisitUsingTypeLoc(UsingTypeLoc TL) {
-  auto *underlyingDecl = TL.getUnderlyingType()->getAsTagDecl();
+  if (VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
+    return true;
+
+  auto *underlyingDecl = TL.getTypePtr()->desugar()->getAsTagDecl();
   if (underlyingDecl) {
     return Visit(MakeCursorTypeRef(underlyingDecl, TL.getNameLoc(), TU));
   }
@@ -1826,7 +1791,7 @@ bool CursorVisitor::VisitAdjustedTypeLoc(AdjustedTypeLoc TL) {
 bool CursorVisitor::VisitDeducedTemplateSpecializationTypeLoc(
     DeducedTemplateSpecializationTypeLoc TL) {
   if (VisitTemplateName(TL.getTypePtr()->getTemplateName(),
-                        TL.getTemplateNameLoc()))
+                        TL.getTemplateNameLoc(), TL.getQualifierLoc()))
     return true;
 
   return false;
@@ -1836,7 +1801,7 @@ bool CursorVisitor::VisitTemplateSpecializationTypeLoc(
     TemplateSpecializationTypeLoc TL) {
   // Visit the template name.
   if (VisitTemplateName(TL.getTypePtr()->getTemplateName(),
-                        TL.getTemplateNameLoc()))
+                        TL.getTemplateNameLoc(), TL.getQualifierLoc()))
     return true;
 
   // Visit the template arguments.
@@ -1871,8 +1836,7 @@ bool CursorVisitor::VisitDependentNameTypeLoc(DependentNameTypeLoc TL) {
 
 bool CursorVisitor::VisitDependentTemplateSpecializationTypeLoc(
     DependentTemplateSpecializationTypeLoc TL) {
-  // Visit the nested-name-specifier, if there is one.
-  if (TL.getQualifierLoc() && VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
+  if (VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
     return true;
 
   // Visit the template arguments.
@@ -1883,13 +1847,6 @@ bool CursorVisitor::VisitDependentTemplateSpecializationTypeLoc(
   return false;
 }
 
-bool CursorVisitor::VisitElaboratedTypeLoc(ElaboratedTypeLoc TL) {
-  if (VisitNestedNameSpecifierLoc(TL.getQualifierLoc()))
-    return true;
-
-  return Visit(TL.getNamedTypeLoc());
-}
-
 bool CursorVisitor::VisitPackExpansionTypeLoc(PackExpansionTypeLoc TL) {
   return Visit(TL.getPatternLoc());
 }
@@ -1908,7 +1865,7 @@ bool CursorVisitor::VisitPackIndexingTypeLoc(PackIndexingTypeLoc TL) {
 }
 
 bool CursorVisitor::VisitInjectedClassNameTypeLoc(InjectedClassNameTypeLoc TL) {
-  return Visit(MakeCursorTypeRef(TL.getDecl(), TL.getNameLoc(), TU));
+  return Visit(MakeCursorTypeRef(TL.getOriginalDecl(), TL.getNameLoc(), TU));
 }
 
 bool CursorVisitor::VisitAtomicTypeLoc(AtomicTypeLoc TL) {
@@ -1943,6 +1900,7 @@ DEFAULT_TYPELOC_IMPL(Record, TagType)
 DEFAULT_TYPELOC_IMPL(Enum, TagType)
 DEFAULT_TYPELOC_IMPL(SubstTemplateTypeParm, Type)
 DEFAULT_TYPELOC_IMPL(SubstTemplateTypeParmPack, Type)
+DEFAULT_TYPELOC_IMPL(SubstBuiltinTemplatePack, Type)
 DEFAULT_TYPELOC_IMPL(Auto, Type)
 DEFAULT_TYPELOC_IMPL(BitInt, Type)
 DEFAULT_TYPELOC_IMPL(DependentBitInt, Type)
@@ -2065,7 +2023,7 @@ class NestedNameSpecifierLocVisit : public VisitorJob {
 public:
   NestedNameSpecifierLocVisit(NestedNameSpecifierLoc Qualifier, CXCursor parent)
       : VisitorJob(parent, VisitorJob::NestedNameSpecifierLocVisitKind,
-                   Qualifier.getNestedNameSpecifier(),
+                   Qualifier.getNestedNameSpecifier().getAsVoidPointer(),
                    Qualifier.getOpaqueData()) {}
 
   static bool classof(const VisitorJob *VJ) {
@@ -2074,8 +2032,7 @@ class NestedNameSpecifierLocVisit : public VisitorJob {
 
   NestedNameSpecifierLoc get() const {
     return NestedNameSpecifierLoc(
-        const_cast<NestedNameSpecifier *>(
-            static_cast<const NestedNameSpecifier *>(data[0])),
+        NestedNameSpecifier::getFromVoidPointer(data[0]),
         const_cast<void *>(data[1]));
   }
 };
@@ -2895,8 +2852,10 @@ void OpenACCClauseEnqueue::VisitDeviceClause(const OpenACCDeviceClause &C) {
 void OpenACCClauseEnqueue::VisitFirstPrivateClause(
     const OpenACCFirstPrivateClause &C) {
   VisitVarList(C);
-  for (VarDecl *V : C.getInitRecipes())
-    Visitor.AddDecl(V);
+  for (const OpenACCFirstPrivateRecipe &R : C.getInitRecipes()) {
+    Visitor.AddDecl(R.RecipeDecl);
+    Visitor.AddDecl(R.InitFromTemporary);
+  }
 }
 
 void OpenACCClauseEnqueue::VisitPresentClause(const OpenACCPresentClause &C) {
@@ -2970,15 +2929,17 @@ void OpenACCClauseEnqueue::VisitDeviceTypeClause(
 void OpenACCClauseEnqueue::VisitReductionClause(
     const OpenACCReductionClause &C) {
   VisitVarList(C);
+  for (const OpenACCReductionRecipe &R : C.getRecipes()) {
+    static_assert(sizeof(OpenACCReductionRecipe) == sizeof(int *));
+    Visitor.AddDecl(R.RecipeDecl);
+  }
 }
 void OpenACCClauseEnqueue::VisitAutoClause(const OpenACCAutoClause &C) {}
 void OpenACCClauseEnqueue::VisitIndependentClause(
     const OpenACCIndependentClause &C) {}
 void OpenACCClauseEnqueue::VisitSeqClause(const OpenACCSeqClause &C) {}
 void OpenACCClauseEnqueue::VisitNoHostClause(const OpenACCNoHostClause &C) {}
-void OpenACCClauseEnqueue::VisitBindClause(const OpenACCBindClause &C) {
-  assert(false && "TODO ERICH");
-}
+void OpenACCClauseEnqueue::VisitBindClause(const OpenACCBindClause &C) { }
 void OpenACCClauseEnqueue::VisitFinalizeClause(const OpenACCFinalizeClause &C) {
 }
 void OpenACCClauseEnqueue::VisitIfPresentClause(
@@ -4720,8 +4681,7 @@ static const ExprEvalResult *evaluateExpr(Expr *expr, CXCursor C) {
 
       std::string strRef(StrE->getString().str());
       result->EvalData.stringVal = new char[strRef.size() + 1];
-      strncpy((char *)result->EvalData.stringVal, strRef.c_str(),
-              strRef.size());
+      strncpy(result->EvalData.stringVal, strRef.c_str(), strRef.size());
       result->EvalData.stringVal[strRef.size()] = '\0';
       return result.release();
     }
@@ -4741,7 +4701,7 @@ static const ExprEvalResult *evaluateExpr(Expr *expr, CXCursor C) {
 
     std::string strRef(StrE->getString().str());
     result->EvalData.stringVal = new char[strRef.size() + 1];
-    strncpy((char *)result->EvalData.stringVal, strRef.c_str(), strRef.size());
+    strncpy(result->EvalData.stringVal, strRef.c_str(), strRef.size());
     result->EvalData.stringVal[strRef.size()] = '\0';
     return result.release();
   }
@@ -4760,7 +4720,7 @@ static const ExprEvalResult *evaluateExpr(Expr *expr, CXCursor C) {
         result->EvalType = CXEval_CFStr;
 
         result->EvalData.stringVal = new char[strLiteral.size() + 1];
-        strncpy((char *)result->EvalData.stringVal, strLiteral.c_str(),
+        strncpy(result->EvalData.stringVal, strLiteral.c_str(),
                 strLiteral.size());
         result->EvalData.stringVal[strLiteral.size()] = '\0';
         return result.release();
@@ -4785,7 +4745,7 @@ static const ExprEvalResult *evaluateExpr(Expr *expr, CXCursor C) {
         std::string strLiteral(S->getString().str());
         result->EvalType = CXEval_CFStr;
         result->EvalData.stringVal = new char[strLiteral.size() + 1];
-        strncpy((char *)result->EvalData.stringVal, strLiteral.c_str(),
+        strncpy(result->EvalData.stringVal, strLiteral.c_str(),
                 strLiteral.size());
         result->EvalData.stringVal[strLiteral.size()] = '\0';
         return result.release();
@@ -5364,9 +5324,13 @@ CXString clang_getCursorSpelling(CXCursor C) {
     case CXCursor_TypeRef: {
       const TypeDecl *Type = getCursorTypeRef(C).first;
       assert(Type && "Missing type decl");
+      const ASTContext &Ctx = getCursorContext(C);
+      QualType T = Ctx.getTypeDeclType(Type);
 
-      return cxstring::createDup(
-          getCursorContext(C).getTypeDeclType(Type).getAsString());
+      PrintingPolicy Policy = Ctx.getPrintingPolicy();
+      Policy.FullyQualifiedName = true;
+      Policy.SuppressTagKeyword = false;
+      return cxstring::createDup(T.getAsString(Policy));
     }
     case CXCursor_TemplateRef: {
       const TemplateDecl *Template = getCursorTemplateRef(C).first;
diff --git a/clang/tools/libclang/CIndexCodeCompletion.cpp b/clang/tools/libclang/CIndexCodeCompletion.cpp
index 81448b4d11342..6d14f286b973a 100644
--- a/clang/tools/libclang/CIndexCodeCompletion.cpp
+++ b/clang/tools/libclang/CIndexCodeCompletion.cpp
@@ -617,7 +617,7 @@ namespace {
       if (!baseType.isNull()) {
         // Get the declaration for a class/struct/union/enum type
         if (const TagType *Tag = baseType->getAs<TagType>())
-          D = Tag->getDecl();
+          D = Tag->getOriginalDecl();
         // Get the @interface declaration for a (possibly-qualified) Objective-C
         // object pointer type, e.g., NSString*
         else if (const ObjCObjectPointerType *ObjPtr = 
@@ -629,7 +629,7 @@ namespace {
         // Get the class for a C++ injected-class-name
         else if (const InjectedClassNameType *Injected =
                  baseType->getAs<InjectedClassNameType>())
-          D = Injected->getDecl();
+          D = Injected->getOriginalDecl();
       }
 
       if (D != nullptr) {
diff --git a/clang/tools/libclang/CMakeLists.txt b/clang/tools/libclang/CMakeLists.txt
index 2b1e266f07392..e0ff7605b68b8 100644
--- a/clang/tools/libclang/CMakeLists.txt
+++ b/clang/tools/libclang/CMakeLists.txt
@@ -93,7 +93,7 @@ if(MSVC)
   set(LLVM_EXPORTED_SYMBOL_FILE)
 endif()
 
-if (UNIX AND NOT APPLE AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "AIX" AND NOT CYGWIN)
+if (UNIX AND NOT APPLE AND NOT "${CMAKE_SYSTEM_NAME}" MATCHES "AIX" AND NOT CYGWIN)
   set(LLVM_EXPORTED_SYMBOL_FILE)
   set(USE_VERSION_SCRIPT ${LLVM_HAVE_LINK_VERSION_SCRIPT})
 endif()
@@ -125,7 +125,7 @@ else()
   set(output_name "clang")
 endif()
 
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
     set(CMAKE_AIX_EXPORT_ALL_SYMBOLS OFF)
     # libclang requires headers which need _ALL_SOURCE to build on AIX
     remove_definitions("-D_XOPEN_SOURCE=700")
@@ -186,7 +186,7 @@ if(ENABLE_SHARED)
     endif()
   endif()
   if (USE_VERSION_SCRIPT)
-    if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+    if ("${CMAKE_SYSTEM_NAME}" MATCHES "SunOS")
       include(CheckLinkerFlag)
       # The Solaris 11.4 linker supports a subset of GNU ld version scripts,
       # but requires a special option to enable it.
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index a6301daa672c3..3c4062410eac1 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -1332,16 +1332,10 @@ CXCursor cxcursor::getTypeRefCursor(CXCursor cursor) {
   TypeLoc TL = Type->getTypeLoc();
   SourceLocation Loc = TL.getBeginLoc();
 
-  if (const ElaboratedType *ElabT = Ty->getAs<ElaboratedType>()) {
-    Ty = ElabT->getNamedType();
-    ElaboratedTypeLoc ElabTL = TL.castAs<ElaboratedTypeLoc>();
-    Loc = ElabTL.getNamedTypeLoc().getBeginLoc();
-  }
-
   if (const TypedefType *Typedef = Ty->getAs<TypedefType>())
     return MakeCursorTypeRef(Typedef->getDecl(), Loc, TU);
   if (const TagType *Tag = Ty->getAs<TagType>())
-    return MakeCursorTypeRef(Tag->getDecl(), Loc, TU);
+    return MakeCursorTypeRef(Tag->getOriginalDecl(), Loc, TU);
   if (const TemplateTypeParmType *TemplP = Ty->getAs<TemplateTypeParmType>())
     return MakeCursorTypeRef(TemplP->getDecl(), Loc, TU);
 
diff --git a/clang/tools/libclang/CXIndexDataConsumer.cpp b/clang/tools/libclang/CXIndexDataConsumer.cpp
index 73d04b85d72db..2623f6023c784 100644
--- a/clang/tools/libclang/CXIndexDataConsumer.cpp
+++ b/clang/tools/libclang/CXIndexDataConsumer.cpp
@@ -357,7 +357,7 @@ CXIndexDataConsumer::CXXBasesListInfo::CXXBasesListInfo(const CXXRecordDecl *D,
           TST = T->getAs<TemplateSpecializationType>()) {
       BaseD = TST->getTemplateName().getAsTemplateDecl();
     } else if (const RecordType *RT = T->getAs<RecordType>()) {
-      BaseD = RT->getDecl();
+      BaseD = RT->getOriginalDecl();
     }
 
     if (BaseD)
@@ -389,13 +389,22 @@ SourceLocation CXIndexDataConsumer::CXXBasesListInfo::getBaseLoc(
   if (QualifiedTypeLoc QL = TL.getAs<QualifiedTypeLoc>())
     TL = QL.getUnqualifiedLoc();
 
-  if (ElaboratedTypeLoc EL = TL.getAs<ElaboratedTypeLoc>())
-    return EL.getNamedTypeLoc().getBeginLoc();
-  if (DependentNameTypeLoc DL = TL.getAs<DependentNameTypeLoc>())
-    return DL.getNameLoc();
-  if (DependentTemplateSpecializationTypeLoc DTL =
-          TL.getAs<DependentTemplateSpecializationTypeLoc>())
-    return DTL.getTemplateNameLoc();
+  // FIXME: Factor this out, a lot of TypeLoc users seem to need a generic
+  // TypeLoc::getNameLoc()
+  if (auto TTL = TL.getAs<DependentNameTypeLoc>())
+    return TTL.getNameLoc();
+  if (auto TTL = TL.getAs<DependentTemplateSpecializationTypeLoc>())
+    return TTL.getTemplateNameLoc();
+  if (auto TTL = TL.getAs<TemplateSpecializationTypeLoc>())
+    return TTL.getTemplateNameLoc();
+  if (auto TTL = TL.getAs<TagTypeLoc>())
+    return TTL.getNameLoc();
+  if (auto TTL = TL.getAs<TypedefTypeLoc>())
+    return TTL.getNameLoc();
+  if (auto TTL = TL.getAs<UnresolvedUsingTypeLoc>())
+    return TTL.getNameLoc();
+  if (auto TTL = TL.getAs<UsingTypeLoc>())
+    return TTL.getNameLoc();
 
   return Loc;
 }
diff --git a/clang/tools/libclang/CXType.cpp b/clang/tools/libclang/CXType.cpp
index e7864e6d62e4d..d21ac7cceed9a 100644
--- a/clang/tools/libclang/CXType.cpp
+++ b/clang/tools/libclang/CXType.cpp
@@ -118,7 +118,6 @@ static CXTypeKind GetTypeKind(QualType T) {
     TKCASE(ExtVector);
     TKCASE(MemberPointer);
     TKCASE(Auto);
-    TKCASE(Elaborated);
     TKCASE(Pipe);
     TKCASE(Attributed);
     TKCASE(BTFTagAttributed);
@@ -225,6 +224,11 @@ FindTemplateArgumentTypeAt(ArrayRef<TemplateArgument> TA, unsigned index) {
   return std::nullopt;
 }
 
+static CXType getTypeDeclType(const ASTContext &Context, CXTranslationUnit TU,
+                              const TypeDecl *TD) {
+  return MakeCXType(Context.getTypeDeclType(TD), TU);
+}
+
 CXType clang_getCursorType(CXCursor C) {
   using namespace cxcursor;
 
@@ -244,7 +248,7 @@ CXType clang_getCursorType(CXCursor C) {
       return MakeCXType(QualType(), TU);
 
     if (const TypeDecl *TD = dyn_cast<TypeDecl>(D))
-      return MakeCXType(Context.getTypeDeclType(TD), TU);
+      return getTypeDeclType(Context, TU, TD);
     if (const ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(D))
       return MakeCXType(Context.getObjCInterfaceType(ID), TU);
     if (const DeclaratorDecl *DD = dyn_cast<DeclaratorDecl>(D))
@@ -271,11 +275,8 @@ CXType clang_getCursorType(CXCursor C) {
       return MakeCXType(T, TU);
     }
 
-    case CXCursor_TypeRef: {
-      QualType T = Context.getTypeDeclType(getCursorTypeRef(C).first);
-      return MakeCXType(T, TU);
-
-    }
+    case CXCursor_TypeRef:
+      return getTypeDeclType(Context, TU, getCursorTypeRef(C).first);
 
     case CXCursor_CXXBaseSpecifier:
       return cxtype::MakeCXType(getCursorCXXBaseSpecifier(C)->getType(), TU);
@@ -545,11 +546,11 @@ CXCursor clang_getTypeDeclaration(CXType CT) {
     break;
   case Type::Record:
   case Type::Enum:
-    D = cast<TagType>(TP)->getDecl();
+    D = cast<TagType>(TP)->getOriginalDecl();
     break;
   case Type::TemplateSpecialization:
     if (const RecordType *Record = TP->getAs<RecordType>())
-      D = Record->getDecl();
+      D = Record->getOriginalDecl();
     else
       D = cast<TemplateSpecializationType>(TP)->getTemplateName()
                                                          .getAsTemplateDecl();
@@ -563,14 +564,10 @@ CXCursor clang_getTypeDeclaration(CXType CT) {
     break;
 
   case Type::InjectedClassName:
-    D = cast<InjectedClassNameType>(TP)->getDecl();
+    D = cast<InjectedClassNameType>(TP)->getOriginalDecl();
     break;
 
-  // FIXME: Template type parameters!      
-
-  case Type::Elaborated:
-    TP = cast<ElaboratedType>(TP)->getNamedType().getTypePtrOrNull();
-    goto try_again;
+    // FIXME: Template type parameters!
 
   default:
     break;
@@ -990,7 +987,7 @@ CXType clang_Type_getClassType(CXType CT) {
   const Type *TP = T.getTypePtrOrNull();
 
   if (TP && TP->getTypeClass() == Type::MemberPointer) {
-    ET = Ctx.getTypeDeclType(
+    ET = Ctx.getCanonicalTagType(
         cast<MemberPointerType>(TP)->getMostRecentCXXRecordDecl());
   }
   return MakeCXType(ET, GetTU(CT));
@@ -1040,7 +1037,7 @@ static long long visitRecordForValidation(const RecordDecl *RD) {
       return CXTypeLayoutError_Dependent;
     // recurse
     if (const RecordType *ChildType = I->getType()->getAs<RecordType>()) {
-      if (const RecordDecl *Child = ChildType->getDecl()) {
+      if (const RecordDecl *Child = ChildType->getOriginalDecl()) {
         long long ret = visitRecordForValidation(Child);
         if (ret < 0)
           return ret;
@@ -1390,10 +1387,9 @@ unsigned clang_Cursor_isInlineNamespace(CXCursor C) {
 
 CXType clang_Type_getNamedType(CXType CT){
   QualType T = GetQualType(CT);
-  const Type *TP = T.getTypePtrOrNull();
 
-  if (TP && TP->getTypeClass() == Type::Elaborated)
-    return MakeCXType(cast<ElaboratedType>(TP)->getNamedType(), GetTU(CT));
+  if (!T.isNull() && !T.isCanonical())
+    return MakeCXType(T, GetTU(CT));
 
   return MakeCXType(QualType(), GetTU(CT));
 }
diff --git a/clang/tools/libclang/CursorVisitor.h b/clang/tools/libclang/CursorVisitor.h
index 949b73908c315..d5ab699756988 100644
--- a/clang/tools/libclang/CursorVisitor.h
+++ b/clang/tools/libclang/CursorVisitor.h
@@ -255,12 +255,12 @@ class CursorVisitor : public DeclVisitor<CursorVisitor, bool>,
 
   // Name visitor
   bool VisitDeclarationNameInfo(DeclarationNameInfo Name);
-  bool VisitNestedNameSpecifier(NestedNameSpecifier *NNS, SourceRange Range);
   bool VisitNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS);
 
   // Template visitors
   bool VisitTemplateParameters(const TemplateParameterList *Params);
-  bool VisitTemplateName(TemplateName Name, SourceLocation Loc);
+  bool VisitTemplateName(TemplateName Name, SourceLocation NameLoc,
+                         NestedNameSpecifierLoc NNS);
   bool VisitTemplateArgumentLoc(const TemplateArgumentLoc &TAL);
 
   // Type visitors
diff --git a/clang/unittests/AST/ASTContextParentMapTest.cpp b/clang/unittests/AST/ASTContextParentMapTest.cpp
index 9af0a46817a25..4a8aa488cb70e 100644
--- a/clang/unittests/AST/ASTContextParentMapTest.cpp
+++ b/clang/unittests/AST/ASTContextParentMapTest.cpp
@@ -131,10 +131,10 @@ TEST(GetParents, FriendTypeLoc) {
   auto &FrB = *cast<FriendDecl>(*++(cast<CXXRecordDecl>(B).decls_begin()));
   TypeLoc FrALoc = FrA.getFriendType()->getTypeLoc();
   TypeLoc FrBLoc = FrB.getFriendType()->getTypeLoc();
+  bool FrAOwnsTag = FrALoc.getTypePtr()->getAs<TagType>()->isTagOwned();
   TagDecl *FrATagDecl =
-      FrALoc.getTypePtr()->getAs<ElaboratedType>()->getOwnedTagDecl();
-  TagDecl *FrBTagDecl =
-      FrBLoc.getTypePtr()->getAs<ElaboratedType>()->getOwnedTagDecl();
+      FrALoc.getTypePtr()->getAs<TagType>()->getOriginalDecl();
+  bool FrBOwnsTag = FrBLoc.getTypePtr()->getAs<TagType>()->isTagOwned();
 
   EXPECT_THAT(Ctx.getParents(A), ElementsAre(DynTypedNode::create(TU)));
   EXPECT_THAT(Ctx.getParents(B), ElementsAre(DynTypedNode::create(TU)));
@@ -142,8 +142,8 @@ TEST(GetParents, FriendTypeLoc) {
   EXPECT_THAT(Ctx.getParents(FrB), ElementsAre(DynTypedNode::create(B)));
   EXPECT_THAT(Ctx.getParents(FrALoc), ElementsAre(DynTypedNode::create(FrA)));
   EXPECT_THAT(Ctx.getParents(FrBLoc), ElementsAre(DynTypedNode::create(FrB)));
-  EXPECT_TRUE(FrATagDecl);
-  EXPECT_FALSE(FrBTagDecl);
+  EXPECT_TRUE(FrAOwnsTag);
+  EXPECT_FALSE(FrBOwnsTag);
   EXPECT_THAT(Ctx.getParents(*FrATagDecl),
               ElementsAre(DynTypedNode::create(FrA)));
 }
diff --git a/clang/unittests/AST/ASTExprTest.cpp b/clang/unittests/AST/ASTExprTest.cpp
index 5ec6aea8edba3..adaceb76de8b4 100644
--- a/clang/unittests/AST/ASTExprTest.cpp
+++ b/clang/unittests/AST/ASTExprTest.cpp
@@ -89,14 +89,14 @@ TEST(ASTExpr, InitListIsConstantInitialized) {
 
   SourceLocation Loc{};
   InitListExpr *BaseInit = new (Ctx) InitListExpr(Ctx, Loc, {}, Loc);
-  BaseInit->setType(Ctx.getRecordType(Empty));
+  BaseInit->setType(Ctx.getCanonicalTagType(Empty));
   Expr *Exprs[3] = {
       BaseInit,
       createIntLiteral(Ctx, 13),
       createIntLiteral(Ctx, 42),
   };
   InitListExpr *FooInit = new (Ctx) InitListExpr(Ctx, Loc, Exprs, Loc);
-  FooInit->setType(Ctx.getRecordType(Foo));
+  FooInit->setType(Ctx.getCanonicalTagType(Foo));
   EXPECT_TRUE(FooInit->isConstantInitializer(Ctx, false));
 
   // Replace the last initializer with something non-constant and make sure
diff --git a/clang/unittests/AST/ASTImporterFixtures.h b/clang/unittests/AST/ASTImporterFixtures.h
index 87e62cbda422a..2af69c62bc62d 100644
--- a/clang/unittests/AST/ASTImporterFixtures.h
+++ b/clang/unittests/AST/ASTImporterFixtures.h
@@ -425,8 +425,7 @@ class TestImportBase
 };
 
 template <typename T> RecordDecl *getRecordDecl(T *D) {
-  auto *ET = cast<ElaboratedType>(D->getType().getTypePtr());
-  return cast<RecordType>(ET->getNamedType().getTypePtr())->getDecl();
+  return D->getType()->getAsRecordDecl();
 }
 
 template <class T>
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index ac38300142e34..15df9af889386 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -29,7 +29,7 @@ using internal::Matcher;
 
 static const RecordDecl *getRecordDeclOfFriend(FriendDecl *FD) {
   QualType Ty = FD->getFriendType()->getType().getCanonicalType();
-  return cast<RecordType>(Ty)->getDecl();
+  return cast<RecordType>(Ty)->getOriginalDecl();
 }
 
 struct ImportExpr : TestImportBase {};
@@ -427,15 +427,14 @@ TEST_P(ImportExpr, ImportParenListExpr) {
       "typedef dummy<int> declToImport;"
       "template class dummy<int>;",
       Lang_CXX03, "", Lang_CXX03, Verifier,
-      typedefDecl(hasType(elaboratedType(namesType(templateSpecializationType(
+      typedefDecl(hasType(templateSpecializationType(
           hasDeclaration(classTemplateSpecializationDecl(hasSpecializedTemplate(
-              classTemplateDecl(hasTemplateDecl(cxxRecordDecl(hasMethod(
-                  allOf(hasName("f"),
-                        hasBody(compoundStmt(has(declStmt(hasSingleDecl(varDecl(
-                            hasInitializer(parenListExpr(has(unaryOperator(
-                                hasOperatorName("*"),
-                                hasUnaryOperand(
-                                    cxxThisExpr())))))))))))))))))))))))));
+              classTemplateDecl(hasTemplateDecl(cxxRecordDecl(hasMethod(allOf(
+                  hasName("f"),
+                  hasBody(compoundStmt(has(declStmt(hasSingleDecl(
+                      varDecl(hasInitializer(parenListExpr(has(unaryOperator(
+                          hasOperatorName("*"),
+                          hasUnaryOperand(cxxThisExpr())))))))))))))))))))))));
 }
 
 TEST_P(ImportExpr, ImportSwitch) {
@@ -691,8 +690,8 @@ TEST_P(ImportType, ImportUsingType) {
   testImport("struct C {};"
              "void declToImport() { using ::C; new C{}; }",
              Lang_CXX11, "", Lang_CXX11, Verifier,
-             functionDecl(hasDescendant(cxxNewExpr(hasType(pointerType(
-                 pointee(elaboratedType(namesType(usingType())))))))));
+             functionDecl(hasDescendant(
+                 cxxNewExpr(hasType(pointerType(pointee(usingType())))))));
 }
 
 TEST_P(ImportDecl, ImportFunctionTemplateDecl) {
@@ -785,8 +784,7 @@ TEST_P(ImportType, ImportDeducedTemplateSpecialization) {
              "class C { public: C(T); };"
              "C declToImport(123);",
              Lang_CXX17, "", Lang_CXX17, Verifier,
-             varDecl(hasType(elaboratedType(
-                 namesType(deducedTemplateSpecializationType())))));
+             varDecl(hasType(deducedTemplateSpecializationType())));
 }
 
 const internal::VariadicDynCastAllOfMatcher<Stmt, SizeOfPackExpr>
@@ -996,9 +994,9 @@ TEST_P(ImportDecl, ImportUsingTemplate) {
              "void declToImport() {"
              "using ns::S;  X<S> xi; }",
              Lang_CXX11, "", Lang_CXX11, Verifier,
-             functionDecl(hasDescendant(varDecl(hasTypeLoc(elaboratedTypeLoc(
-                 hasNamedTypeLoc(templateSpecializationTypeLoc(
-                     hasAnyTemplateArgumentLoc(templateArgumentLoc())))))))));
+             functionDecl(
+                 hasDescendant(varDecl(hasTypeLoc(templateSpecializationTypeLoc(
+                     hasAnyTemplateArgumentLoc(templateArgumentLoc())))))));
 }
 
 TEST_P(ImportDecl, ImportUsingEnumDecl) {
@@ -1019,24 +1017,10 @@ TEST_P(ImportDecl, ImportUsingPackDecl) {
       "template<typename ...T> struct C : T... { using T::operator()...; };"
       "C<A, B> declToImport;",
       Lang_CXX20, "", Lang_CXX20, Verifier,
-      varDecl(hasType(elaboratedType(namesType(templateSpecializationType(
-          hasDeclaration(classTemplateSpecializationDecl(
-              hasDescendant(usingPackDecl())))))))));
+      varDecl(hasType(templateSpecializationType(hasDeclaration(
+          classTemplateSpecializationDecl(hasDescendant(usingPackDecl())))))));
 }
 
-/// \brief Matches shadow declarations introduced into a scope by a
-///        (resolved) using declaration.
-///
-/// Given
-/// \code
-///   namespace n { int f; }
-///   namespace declToImport { using n::f; }
-/// \endcode
-/// usingShadowDecl()
-///   matches \code f \endcode
-const internal::VariadicDynCastAllOfMatcher<Decl,
-                                            UsingShadowDecl> usingShadowDecl;
-
 TEST_P(ImportDecl, ImportUsingShadowDecl) {
   MatchVerifier<Decl> Verifier;
   // from using-decl
@@ -2855,8 +2839,9 @@ TEST_P(ImportFriendFunctions, ImportFriendFunctionRedeclChainDefWithClass) {
   EXPECT_FALSE(InClassFD->doesThisDeclarationHaveABody());
   EXPECT_EQ(InClassFD->getPreviousDecl(), ImportedD);
   // The parameters must refer the same type
-  EXPECT_EQ((*InClassFD->param_begin())->getOriginalType(),
-            (*ImportedD->param_begin())->getOriginalType());
+  EXPECT_TRUE(ToTU->getASTContext().hasSameType(
+      (*InClassFD->param_begin())->getOriginalType(),
+      (*ImportedD->param_begin())->getOriginalType()));
 }
 
 TEST_P(ImportFriendFunctions,
@@ -2884,8 +2869,9 @@ TEST_P(ImportFriendFunctions,
   EXPECT_TRUE(OutOfClassFD->doesThisDeclarationHaveABody());
   EXPECT_EQ(ImportedD->getPreviousDecl(), OutOfClassFD);
   // The parameters must refer the same type
-  EXPECT_EQ((*OutOfClassFD->param_begin())->getOriginalType(),
-            (*ImportedD->param_begin())->getOriginalType());
+  EXPECT_TRUE(ToTU->getASTContext().hasSameType(
+      (*OutOfClassFD->param_begin())->getOriginalType(),
+      (*ImportedD->param_begin())->getOriginalType()));
 }
 
 TEST_P(ImportFriendFunctions, ImportFriendFunctionFromMultipleTU) {
@@ -4486,8 +4472,7 @@ TEST_P(ImportFriendClasses, TypeForDeclShouldBeSetInTemplated) {
   auto *Definition = FirstDeclMatcher<ClassTemplateDecl>().match(
       FromTU1, classTemplateDecl(hasName("F")));
   auto *Imported1 = cast<ClassTemplateDecl>(Import(Definition, Lang_CXX03));
-  EXPECT_EQ(Imported0->getTemplatedDecl()->getTypeForDecl(),
-            Imported1->getTemplatedDecl()->getTypeForDecl());
+  EXPECT_TRUE(declaresSameEntity(Imported0, Imported1));
 }
 
 TEST_P(ImportFriendClasses, DeclsFromFriendsShouldBeInRedeclChains) {
@@ -4548,8 +4533,10 @@ TEST_P(ImportFriendClasses, SkipComparingFriendTemplateDepth) {
       classTemplateDecl(has(cxxRecordDecl(hasDefinition(), hasName("A")))));
   auto *ToA = Import(FromA, Lang_CXX11);
   EXPECT_TRUE(ToA);
-  EXPECT_EQ(Fwd->getTemplatedDecl()->getTypeForDecl(),
-            ToA->getTemplatedDecl()->getTypeForDecl());
+  const ASTContext &Ctx = ToTU->getASTContext();
+  EXPECT_TRUE(
+      Ctx.hasSameType(Ctx.getCanonicalTagType(Fwd->getTemplatedDecl()),
+                      Ctx.getCanonicalTagType(ToA->getTemplatedDecl())));
 }
 
 TEST_P(ImportFriendClasses,
@@ -4627,7 +4614,7 @@ TEST_P(ImportFriendClasses, ImportOfClassDefinitionAndFwdFriendShouldBeLinked) {
   auto *Friend = FirstDeclMatcher<FriendDecl>().match(FromTU0, friendDecl());
   QualType FT = Friend->getFriendType()->getType();
   FT = FromTU0->getASTContext().getCanonicalType(FT);
-  auto *Fwd = cast<TagType>(FT)->getDecl();
+  auto *Fwd = cast<TagType>(FT)->getOriginalDecl();
   auto *ImportedFwd = Import(Fwd, Lang_CXX03);
   Decl *FromTU1 = getTuDecl(
       R"(
@@ -7543,7 +7530,8 @@ TEST_P(ImportAutoFunctions, ReturnWithTypedefDeclaredInside) {
   ASTContext &Ctx = From->getASTContext();
   TypeAliasDecl *FromTA =
       FirstDeclMatcher<TypeAliasDecl>().match(FromTU, typeAliasDecl());
-  QualType TT = Ctx.getTypedefType(FromTA);
+  QualType TT = Ctx.getTypedefType(ElaboratedTypeKeyword::None,
+                                   /*Qualifier=*/std::nullopt, FromTA);
   const FunctionProtoType *FPT = cast<FunctionProtoType>(From->getType());
   QualType NewFunType =
       Ctx.getFunctionType(TT, FPT->getParamTypes(), FPT->getExtProtoInfo());
@@ -9193,41 +9181,39 @@ TEST_P(ASTImporterOptionSpecificTestBase, isNewDecl) {
 
 struct ImportInjectedClassNameType : public ASTImporterOptionSpecificTestBase {
 protected:
-  const CXXRecordDecl *findInjected(const CXXRecordDecl *Parent) {
-    for (Decl *Found : Parent->decls()) {
-      const auto *Record = dyn_cast<CXXRecordDecl>(Found);
-      if (Record && Record->isInjectedClassName())
-        return Record;
-    }
-    return nullptr;
-  }
-
-  void checkInjType(const CXXRecordDecl *D) {
-    // The whole redecl chain should have the same InjectedClassNameType
-    // instance. The injected record declaration is a separate chain, this
-    // should contain the same type too.
-    const Type *Ty = nullptr;
-    for (const Decl *ReD : D->redecls()) {
-      const auto *ReRD = cast<CXXRecordDecl>(ReD);
-      EXPECT_TRUE(ReRD->getTypeForDecl());
-      EXPECT_TRUE(!Ty || Ty == ReRD->getTypeForDecl());
-      Ty = ReRD->getTypeForDecl();
-    }
+  void checkInjType(const ASTContext &Ctx, const CXXRecordDecl *D) {
+    ASSERT_TRUE(D->hasInjectedClassType());
+    const Type *Ty = Ctx.getTagType(ElaboratedTypeKeyword::None,
+                                    /*Qualifier=*/std::nullopt, D,
+                                    /*OwnsTag=*/false)
+                         .getTypePtr();
     ASSERT_TRUE(Ty);
+    EXPECT_FALSE(Ty->isCanonicalUnqualified());
     const auto *InjTy = Ty->castAs<InjectedClassNameType>();
     EXPECT_TRUE(InjTy);
-    if (CXXRecordDecl *Def = D->getDefinition()) {
-      const CXXRecordDecl *InjRD = findInjected(Def);
-      EXPECT_TRUE(InjRD);
-      EXPECT_EQ(InjRD->getTypeForDecl(), InjTy);
+    for (const Decl *ReD : D->redecls()) {
+      if (ReD == D)
+        continue;
+      const auto *ReRD = cast<CXXRecordDecl>(ReD);
+      ASSERT_TRUE(ReRD->hasInjectedClassType());
+      const Type *ReTy = Ctx.getTagType(ElaboratedTypeKeyword::None,
+                                        /*Qualifier=*/std::nullopt, ReRD,
+                                        /*OwnsTag=*/false)
+                             .getTypePtr();
+      ASSERT_TRUE(ReTy);
+      EXPECT_FALSE(ReTy->isCanonicalUnqualified());
+      EXPECT_NE(ReTy, Ty);
+      EXPECT_TRUE(Ctx.hasSameType(ReTy, Ty));
+      const auto *ReInjTy = Ty->castAs<InjectedClassNameType>();
+      EXPECT_TRUE(ReInjTy);
     }
   }
 
   void testImport(Decl *ToTU, Decl *FromTU, Decl *FromD) {
-    checkInjType(cast<CXXRecordDecl>(FromD));
+    checkInjType(FromTU->getASTContext(), cast<CXXRecordDecl>(FromD));
     Decl *ToD = Import(FromD, Lang_CXX11);
     if (auto *ToRD = dyn_cast<CXXRecordDecl>(ToD))
-      checkInjType(ToRD);
+      checkInjType(ToTU->getASTContext(), ToRD);
   }
 
   const char *ToCodeA =
@@ -9679,8 +9665,7 @@ TEST_P(ASTImporterOptionSpecificTestBase,
 
   auto *ToX = Import(FromX, Lang_CXX11);
   auto *ToXType = ToX->getType()->getAs<TypedefType>();
-  // FIXME: This should be false.
-  EXPECT_TRUE(ToXType->typeMatchesDecl());
+  EXPECT_FALSE(ToXType->typeMatchesDecl());
 }
 
 TEST_P(ASTImporterOptionSpecificTestBase,
@@ -10038,7 +10023,12 @@ struct ImportTemplateParmDeclDefaultValue
                   .getInheritedFrom(),
               GetTemplateParm(ToD));
 
-    EXPECT_EQ(ToD->getPreviousDecl(), ToDInherited);
+    if (FromD->getPreviousDecl() == FromDInherited) {
+      EXPECT_EQ(ToD->getPreviousDecl(), ToDInherited);
+    } else {
+      EXPECT_EQ(FromD, FromDInherited->getPreviousDecl());
+      EXPECT_EQ(ToD, ToDInherited->getPreviousDecl());
+    }
   }
 
   const char *CodeFunction =
diff --git a/clang/unittests/AST/ByteCode/Descriptor.cpp b/clang/unittests/AST/ByteCode/Descriptor.cpp
index b3517d8731c85..37e6f24f6b409 100644
--- a/clang/unittests/AST/ByteCode/Descriptor.cpp
+++ b/clang/unittests/AST/ByteCode/Descriptor.cpp
@@ -399,7 +399,12 @@ TEST(Descriptor, Primitives) {
     const Pointer &PF5 = GlobalPtr.atField(F5->Offset);
 
     ASSERT_TRUE(PF5.isZeroSizeArray());
-    ASSERT_FALSE(PF5.isOnePastEnd());
+    ASSERT_TRUE(PF5.isOnePastEnd());
+    ASSERT_FALSE(PF5.isElementPastEnd());
+
+    const Pointer &E1 = PF5.atIndex(0);
+    (void)E1;
+    ASSERT_TRUE(PF5.isOnePastEnd());
     ASSERT_FALSE(PF5.isElementPastEnd());
   }
 }
diff --git a/clang/unittests/AST/DeclPrinterTest.cpp b/clang/unittests/AST/DeclPrinterTest.cpp
index 124b1a166cb18..28750c41796d4 100644
--- a/clang/unittests/AST/DeclPrinterTest.cpp
+++ b/clang/unittests/AST/DeclPrinterTest.cpp
@@ -356,42 +356,40 @@ TEST(DeclPrinter, TestCXXRecordDecl11) {
 }
 
 TEST(DeclPrinter, TestCXXRecordDecl12) {
-  ASSERT_TRUE(
-      PrintedDeclCXX98Matches("struct S { int x; };"
-                              "namespace NS { class C {};}"
-                              "void foo() {using namespace NS; C c;}",
-                              "foo",
-                              "void foo() {\nusing namespace NS;\nclass "
-                              "NS::C c;\n}\n",
-                              [](PrintingPolicy &Policy) {
-                                Policy.SuppressTagKeyword = false;
-                                Policy.SuppressScope = true;
-                                Policy.TerseOutput = false;
-                              }));
+  ASSERT_TRUE(PrintedDeclCXX98Matches("struct S { int x; };"
+                                      "namespace NS { class C {};}"
+                                      "void foo() {using namespace NS; C c;}",
+                                      "foo",
+                                      "void foo() {\nusing namespace NS;\n"
+                                      "C c;\n}\n",
+                                      [](PrintingPolicy &Policy) {
+                                        Policy.SuppressTagKeyword = false;
+                                        Policy.SuppressScope = true;
+                                        Policy.TerseOutput = false;
+                                      }));
 }
 
 TEST(DeclPrinter, TestCXXRecordDecl13) {
-  ASSERT_TRUE(PrintedDeclCXX98Matches(
-      "struct S { int x; };"
-      "S s1;"
-      "S foo() {return s1;}",
-      "foo", "struct S foo() {\nreturn s1;\n}\n", [](PrintingPolicy &Policy) {
-        Policy.SuppressTagKeyword = false;
-        Policy.SuppressScope = true;
-        Policy.TerseOutput = false;
-      }));
+  ASSERT_TRUE(PrintedDeclCXX98Matches("struct S { int x; };"
+                                      "S s1;"
+                                      "S foo() {return s1;}",
+                                      "foo", "S foo() {\nreturn s1;\n}\n",
+                                      [](PrintingPolicy &Policy) {
+                                        Policy.SuppressTagKeyword = false;
+                                        Policy.SuppressScope = true;
+                                        Policy.TerseOutput = false;
+                                      }));
 }
 
 TEST(DeclPrinter, TestCXXRecordDecl14) {
-  ASSERT_TRUE(PrintedDeclCXX98Matches(
-      "struct S { int x; };"
-      "S foo(S s1) {return s1;}",
-      "foo", "struct S foo(struct S s1) {\nreturn s1;\n}\n",
-      [](PrintingPolicy &Policy) {
-        Policy.SuppressTagKeyword = false;
-        Policy.SuppressScope = true;
-        Policy.TerseOutput = false;
-      }));
+  ASSERT_TRUE(PrintedDeclCXX98Matches("struct S { int x; };"
+                                      "S foo(S s1) {return s1;}",
+                                      "foo", "S foo(S s1) {\nreturn s1;\n}\n",
+                                      [](PrintingPolicy &Policy) {
+                                        Policy.SuppressTagKeyword = false;
+                                        Policy.SuppressScope = true;
+                                        Policy.TerseOutput = false;
+                                      }));
 }
 TEST(DeclPrinter, TestCXXRecordDecl15) {
   ASSERT_TRUE(PrintedDeclCXX98Matches(
@@ -399,8 +397,8 @@ TEST(DeclPrinter, TestCXXRecordDecl15) {
       "namespace NS { class C {};}"
       "S foo(S s1, NS::C c1) {using namespace NS; C c; return s1;}",
       "foo",
-      "struct S foo(struct S s1, class NS::C c1) {\nusing namespace NS;\nclass "
-      "NS::C c;\nreturn s1;\n}\n",
+      "S foo(S s1, NS::C c1) {\nusing namespace NS;\n"
+      "C c;\nreturn s1;\n}\n",
       [](PrintingPolicy &Policy) {
         Policy.SuppressTagKeyword = false;
         Policy.SuppressScope = true;
diff --git a/clang/unittests/AST/DeclTest.cpp b/clang/unittests/AST/DeclTest.cpp
index afaf413493299..6b443918ec137 100644
--- a/clang/unittests/AST/DeclTest.cpp
+++ b/clang/unittests/AST/DeclTest.cpp
@@ -90,7 +90,7 @@ TEST(Decl, AsmLabelAttr) {
   DeclF->addAttr(AsmLabelAttr::Create(Ctx, "foo"));
 
   // Mangle the decl names.
-  std::string MangleF, MangleG;
+  std::string MangleF;
   std::unique_ptr<ItaniumMangleContext> MC(
       ItaniumMangleContext::create(Ctx, Diags));
   {
diff --git a/clang/unittests/AST/ProfilingTest.cpp b/clang/unittests/AST/ProfilingTest.cpp
index 27a4a197f1cbf..b46bade186301 100644
--- a/clang/unittests/AST/ProfilingTest.cpp
+++ b/clang/unittests/AST/ProfilingTest.cpp
@@ -63,11 +63,11 @@ TEST(Profiling, DeducedTemplateSpecializationType_Name) {
   ASTContext &Ctx = AST->getASTContext();
 
   auto *T1 = cast<DeducedTemplateSpecializationType>(
-      Ctx.getDeducedTemplateSpecializationType(TemplateName(CTD1), QualType(),
-                                               false));
+      Ctx.getDeducedTemplateSpecializationType(
+          ElaboratedTypeKeyword::None, TemplateName(CTD1), QualType(), false));
   auto *T2 = cast<DeducedTemplateSpecializationType>(
-      Ctx.getDeducedTemplateSpecializationType(TemplateName(CTD2), QualType(),
-                                               false));
+      Ctx.getDeducedTemplateSpecializationType(
+          ElaboratedTypeKeyword::None, TemplateName(CTD2), QualType(), false));
   testTypeNode(T1, T2);
 }
 
diff --git a/clang/unittests/AST/RandstructTest.cpp b/clang/unittests/AST/RandstructTest.cpp
index c22d866ed4fad..cba446fc529e1 100644
--- a/clang/unittests/AST/RandstructTest.cpp
+++ b/clang/unittests/AST/RandstructTest.cpp
@@ -532,7 +532,7 @@ TEST(RANDSTRUCT_TEST, AnonymousStructsAndUnionsRetainFieldOrder) {
   for (const Decl *D : RD->decls())
     if (const FieldDecl *FD = dyn_cast<FieldDecl>(D)) {
       if (const auto *Record = FD->getType()->getAs<RecordType>()) {
-        RD = Record->getDecl();
+        RD = Record->getOriginalDecl()->getDefinitionOrSelf();
         if (RD->isAnonymousStructOrUnion()) {
           // These field orders shouldn't change.
           if (RD->isUnion()) {
diff --git a/clang/unittests/AST/RecursiveASTVisitorTest.cpp b/clang/unittests/AST/RecursiveASTVisitorTest.cpp
index 9d7ff5947fe53..c5ad29a77d211 100644
--- a/clang/unittests/AST/RecursiveASTVisitorTest.cpp
+++ b/clang/unittests/AST/RecursiveASTVisitorTest.cpp
@@ -95,9 +95,10 @@ class CollectInterestingEvents
     return Ret;
   }
 
-  bool TraverseTypedefTypeLoc(TypedefTypeLoc TL) {
+  bool TraverseTypedefTypeLoc(TypedefTypeLoc TL, bool TraverseQualifier) {
     Events.push_back(VisitEvent::StartTraverseTypedefType);
-    bool Ret = RecursiveASTVisitor::TraverseTypedefTypeLoc(TL);
+    bool Ret =
+        RecursiveASTVisitor::TraverseTypedefTypeLoc(TL, TraverseQualifier);
     Events.push_back(VisitEvent::EndTraverseTypedefType);
 
     return Ret;
diff --git a/clang/unittests/AST/SizelessTypesTest.cpp b/clang/unittests/AST/SizelessTypesTest.cpp
index 0b984b6922934..4a945a439492d 100644
--- a/clang/unittests/AST/SizelessTypesTest.cpp
+++ b/clang/unittests/AST/SizelessTypesTest.cpp
@@ -24,7 +24,7 @@ struct SizelessTypeTester : public ::testing::Test {
   ASTContext &Ctx = AST->getASTContext();
   TranslationUnitDecl &TU = *Ctx.getTranslationUnitDecl();
   TypeDecl *Foo = cast<TypeDecl>(TU.lookup(&Ctx.Idents.get("foo")).front());
-  const Type *FooTy = Foo->getTypeForDecl();
+  const Type *FooTy = Ctx.getTypeDeclType(Foo).getTypePtr();
 };
 
 TEST_F(SizelessTypeTester, TestSizelessBuiltin) {
diff --git a/clang/unittests/AST/StructuralEquivalenceTest.cpp b/clang/unittests/AST/StructuralEquivalenceTest.cpp
index ef82afaf3f8dc..bee288dbfe436 100644
--- a/clang/unittests/AST/StructuralEquivalenceTest.cpp
+++ b/clang/unittests/AST/StructuralEquivalenceTest.cpp
@@ -617,8 +617,7 @@ TEST_F(StructuralEquivalenceCXXMethodTest, OutOfClass2) {
 struct StructuralEquivalenceRecordTest : StructuralEquivalenceTest {
   // FIXME Use a common getRecordDecl with ASTImporterTest.cpp!
   RecordDecl *getRecordDecl(FieldDecl *FD) {
-    auto *ET = cast<ElaboratedType>(FD->getType().getTypePtr());
-    return cast<RecordType>(ET->getNamedType().getTypePtr())->getDecl();
+    return FD->getType()->getAsRecordDecl();
   };
 };
 
@@ -720,11 +719,13 @@ TEST_F(StructuralEquivalenceRecordTest, AnonymousRecordsShouldBeInequivalent) {
   auto *A = FirstDeclMatcher<IndirectFieldDecl>().match(
       TU, indirectFieldDecl(hasName("a")));
   auto *FA = cast<FieldDecl>(A->chain().front());
-  RecordDecl *RA = cast<RecordType>(FA->getType().getTypePtr())->getDecl();
+  RecordDecl *RA =
+      cast<RecordType>(FA->getType().getTypePtr())->getOriginalDecl();
   auto *B = FirstDeclMatcher<IndirectFieldDecl>().match(
       TU, indirectFieldDecl(hasName("b")));
   auto *FB = cast<FieldDecl>(B->chain().front());
-  RecordDecl *RB = cast<RecordType>(FB->getType().getTypePtr())->getDecl();
+  RecordDecl *RB =
+      cast<RecordType>(FB->getType().getTypePtr())->getOriginalDecl();
 
   ASSERT_NE(RA, RB);
   EXPECT_TRUE(testStructuralMatch(RA, RA));
@@ -753,13 +754,15 @@ TEST_F(StructuralEquivalenceRecordTest,
   auto *A = FirstDeclMatcher<IndirectFieldDecl>().match(
       TU, indirectFieldDecl(hasName("a")));
   auto *FA = cast<FieldDecl>(A->chain().front());
-  RecordDecl *RA = cast<RecordType>(FA->getType().getTypePtr())->getDecl();
+  RecordDecl *RA =
+      cast<RecordType>(FA->getType().getTypePtr())->getOriginalDecl();
 
   auto *TU1 = get<1>(t);
   auto *A1 = FirstDeclMatcher<IndirectFieldDecl>().match(
       TU1, indirectFieldDecl(hasName("a")));
   auto *FA1 = cast<FieldDecl>(A1->chain().front());
-  RecordDecl *RA1 = cast<RecordType>(FA1->getType().getTypePtr())->getDecl();
+  RecordDecl *RA1 =
+      cast<RecordType>(FA1->getType().getTypePtr())->getOriginalDecl();
 
   RecordDecl *X =
       FirstDeclMatcher<RecordDecl>().match(TU, recordDecl(hasName("X")));
diff --git a/clang/unittests/AST/TemplateNameTest.cpp b/clang/unittests/AST/TemplateNameTest.cpp
index 2eac5c508d059..31655e2fd6be9 100644
--- a/clang/unittests/AST/TemplateNameTest.cpp
+++ b/clang/unittests/AST/TemplateNameTest.cpp
@@ -120,14 +120,14 @@ TEST(TemplateName, UsingTemplate) {
     // are rather part of the ElaboratedType)!
     absl::vector<int> v(123);
   )cpp");
-  auto Matcher = elaboratedTypeLoc(
-      hasNamedTypeLoc(loc(templateSpecializationType().bind("id"))));
+  auto Matcher = templateSpecializationTypeLoc().bind("id");
   auto MatchResults = match(Matcher, AST->getASTContext());
   const auto *TST =
-      MatchResults.front().getNodeAs<TemplateSpecializationType>("id");
+      MatchResults.front().getNodeAs<TemplateSpecializationTypeLoc>("id");
   ASSERT_TRUE(TST);
-  EXPECT_EQ(TST->getTemplateName().getKind(), TemplateName::QualifiedTemplate);
-  EXPECT_TRUE(TST->getTemplateName().getAsUsingShadowDecl() != nullptr);
+  TemplateName TN = TST->getTypePtr()->getTemplateName();
+  EXPECT_EQ(TN.getKind(), TemplateName::QualifiedTemplate);
+  EXPECT_TRUE(TN.getAsUsingShadowDecl() != nullptr);
 
   AST = tooling::buildASTFromCodeWithArgs(R"cpp(
     namespace std {
@@ -139,8 +139,7 @@ TEST(TemplateName, UsingTemplate) {
     absl::vector DTST(123);
     )cpp",
                                           {"-std=c++17"});
-  Matcher = elaboratedTypeLoc(
-      hasNamedTypeLoc(loc(deducedTemplateSpecializationType().bind("id"))));
+  Matcher = loc(deducedTemplateSpecializationType().bind("id"));
   MatchResults = match(Matcher, AST->getASTContext());
   const auto *DTST =
       MatchResults.front().getNodeAs<DeducedTemplateSpecializationType>("id");
diff --git a/clang/unittests/AST/TypePrinterTest.cpp b/clang/unittests/AST/TypePrinterTest.cpp
index 2b37add007086..ca0380b3c9dd3 100644
--- a/clang/unittests/AST/TypePrinterTest.cpp
+++ b/clang/unittests/AST/TypePrinterTest.cpp
@@ -60,7 +60,7 @@ TEST(TypePrinter, TemplateId) {
       [](PrintingPolicy &Policy) { Policy.FullyQualifiedName = false; }));
 
   ASSERT_TRUE(PrintedTypeMatches(
-      Code, {}, Matcher, "const Type<T> &",
+      Code, {}, Matcher, "const N::Type<T> &",
       [](PrintingPolicy &Policy) { Policy.FullyQualifiedName = true; }));
 }
 
@@ -97,7 +97,7 @@ TEST(TypePrinter, ParamsUglified) {
                                  "const f<Tp &> *", Clean));
 }
 
-TEST(TypePrinter, SuppressElaboration) {
+TEST(TypePrinter, TemplateSpecializationFullyQualified) {
   llvm::StringLiteral Code = R"cpp(
     namespace shared {
     namespace a {
@@ -115,13 +115,10 @@ TEST(TypePrinter, SuppressElaboration) {
                                  hasType(qualType().bind("id")));
   ASSERT_TRUE(PrintedTypeMatches(
       Code, {}, Matcher, "a::S<b::Foo>",
+      [](PrintingPolicy &Policy) { Policy.FullyQualifiedName = false; }));
+  ASSERT_TRUE(PrintedTypeMatches(
+      Code, {}, Matcher, "shared::a::S<shared::b::Foo>",
       [](PrintingPolicy &Policy) { Policy.FullyQualifiedName = true; }));
-  ASSERT_TRUE(PrintedTypeMatches(Code, {}, Matcher,
-                                 "shared::a::S<shared::b::Foo>",
-                                 [](PrintingPolicy &Policy) {
-                                   Policy.SuppressElaboration = true;
-                                   Policy.FullyQualifiedName = true;
-                                 }));
 }
 
 TEST(TypePrinter, TemplateIdWithNTTP) {
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
index 287122393446d..8a957864cdd12 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNarrowingTest.cpp
@@ -1172,8 +1172,8 @@ TEST_P(ASTMatchersTest, IsDerivedFrom_ElaboratedType) {
     return;
   }
 
-  DeclarationMatcher IsDerivenFromBase =
-      cxxRecordDecl(isDerivedFrom(decl().bind("typedef")));
+  DeclarationMatcher IsDerivenFromBase = cxxRecordDecl(
+      isDerivedFrom(decl().bind("typedef")), unless(isImplicit()));
 
   EXPECT_TRUE(matchAndVerifyResultTrue(
       "struct AnInterface {};"
@@ -2302,10 +2302,9 @@ TEST(ASTMatchersTest, NamesMember_CXXDependentScopeMemberExpr) {
     EXPECT_TRUE(matches(
         Code,
         cxxDependentScopeMemberExpr(
-            hasObjectExpression(declRefExpr(hasType(elaboratedType(namesType(
-                templateSpecializationType(hasDeclaration(classTemplateDecl(
-                    has(cxxRecordDecl(has(cxxMethodDecl(hasName("mem"))
-                                              .bind("templMem")))))))))))),
+            hasObjectExpression(declRefExpr(hasType(templateSpecializationType(
+                hasDeclaration(classTemplateDecl(has(cxxRecordDecl(
+                    has(cxxMethodDecl(hasName("mem")).bind("templMem")))))))))),
             memberHasSameNameAsBoundNode("templMem"))));
 
     EXPECT_TRUE(
@@ -2323,10 +2322,9 @@ TEST(ASTMatchersTest, NamesMember_CXXDependentScopeMemberExpr) {
     EXPECT_TRUE(matches(
         Code,
         cxxDependentScopeMemberExpr(
-            hasObjectExpression(declRefExpr(
-                hasType(elaboratedType(namesType(templateSpecializationType(
-                    hasDeclaration(classTemplateDecl(has(cxxRecordDecl(has(
-                        fieldDecl(hasName("mem")).bind("templMem")))))))))))),
+            hasObjectExpression(declRefExpr(hasType(templateSpecializationType(
+                hasDeclaration(classTemplateDecl(has(cxxRecordDecl(
+                    has(fieldDecl(hasName("mem")).bind("templMem")))))))))),
             memberHasSameNameAsBoundNode("templMem"))));
   }
 
@@ -2341,10 +2339,9 @@ TEST(ASTMatchersTest, NamesMember_CXXDependentScopeMemberExpr) {
     EXPECT_TRUE(matches(
         Code,
         cxxDependentScopeMemberExpr(
-            hasObjectExpression(declRefExpr(
-                hasType(elaboratedType(namesType(templateSpecializationType(
-                    hasDeclaration(classTemplateDecl(has(cxxRecordDecl(
-                        has(varDecl(hasName("mem")).bind("templMem")))))))))))),
+            hasObjectExpression(declRefExpr(hasType(templateSpecializationType(
+                hasDeclaration(classTemplateDecl(has(cxxRecordDecl(
+                    has(varDecl(hasName("mem")).bind("templMem")))))))))),
             memberHasSameNameAsBoundNode("templMem"))));
   }
   {
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index 07450a0c59ec6..d7df9cae01f33 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -1183,6 +1183,39 @@ TEST_P(ASTMatchersTest, AsmStatement) {
   EXPECT_TRUE(matches("void foo() { __asm(\"mov al, 2\"); }", asmStmt()));
 }
 
+TEST_P(ASTMatchersTest, HasConditionVariableStatement) {
+  if (!GetParam().isCXX()) {
+    // FIXME: Add a test for `hasConditionVariableStatement()` that does not
+    // depend on C++.
+    return;
+  }
+
+  StatementMatcher IfCondition =
+      ifStmt(hasConditionVariableStatement(declStmt()));
+
+  EXPECT_TRUE(matches("void x() { if (int* a = 0) {} }", IfCondition));
+  EXPECT_TRUE(notMatches("void x() { if (true) {} }", IfCondition));
+  EXPECT_TRUE(notMatches("void x() { int x; if ((x = 42)) {} }", IfCondition));
+
+  StatementMatcher SwitchCondition =
+      switchStmt(hasConditionVariableStatement(declStmt()));
+
+  EXPECT_TRUE(matches("void x() { switch (int a = 0) {} }", SwitchCondition));
+  if (GetParam().isCXX17OrLater()) {
+    EXPECT_TRUE(
+        notMatches("void x() { switch (int a = 0; a) {} }", SwitchCondition));
+  }
+
+  StatementMatcher ForCondition =
+      forStmt(hasConditionVariableStatement(declStmt()));
+
+  EXPECT_TRUE(matches("void x() { for (; int a = 0; ) {} }", ForCondition));
+  EXPECT_TRUE(notMatches("void x() { for (int a = 0; ; ) {} }", ForCondition));
+
+  EXPECT_TRUE(matches("void x() { while (int a = 0) {} }",
+                      whileStmt(hasConditionVariableStatement(declStmt()))));
+}
+
 TEST_P(ASTMatchersTest, HasCondition) {
   if (!GetParam().isCXX()) {
     // FIXME: Add a test for `hasCondition()` that does not depend on C++.
@@ -1938,8 +1971,7 @@ TEST_P(ASTMatchersTest, PointerType_MatchesPointersToConstTypes) {
 
 TEST_P(ASTMatchersTest, TypedefType) {
   EXPECT_TRUE(matches("typedef int X; X a;",
-                      varDecl(hasName("a"), hasType(elaboratedType(
-                                                namesType(typedefType()))))));
+                      varDecl(hasName("a"), hasType(typedefType()))));
 }
 
 TEST_P(ASTMatchersTest, MacroQualifiedType) {
@@ -2018,22 +2050,6 @@ TEST_P(ASTMatchersTest, RecordType_CXX) {
                       recordType(hasDeclaration(recordDecl(hasName("S"))))));
 }
 
-TEST_P(ASTMatchersTest, ElaboratedType) {
-  if (!GetParam().isCXX()) {
-    // FIXME: Add a test for `elaboratedType()` that does not depend on C++.
-    return;
-  }
-  EXPECT_TRUE(matches("namespace N {"
-                      "  namespace M {"
-                      "    class D {};"
-                      "  }"
-                      "}"
-                      "N::M::D d;",
-                      elaboratedType()));
-  EXPECT_TRUE(matches("class C {} c;", elaboratedType()));
-  EXPECT_TRUE(matches("class C {}; C c;", elaboratedType()));
-}
-
 TEST_P(ASTMatchersTest, SubstTemplateTypeParmType) {
   if (!GetParam().isCXX()) {
     return;
@@ -2133,16 +2149,16 @@ TEST_P(ASTMatchersTest,
   if (!GetParam().isCXX()) {
     return;
   }
-  EXPECT_TRUE(matches(
-      "struct A { struct B { struct C {}; }; }; A::B::C c;",
-      nestedNameSpecifier(hasPrefix(specifiesType(asString("struct A"))))));
+  EXPECT_TRUE(
+      matches("struct A { struct B { struct C {}; }; }; A::B::C c;",
+              nestedNameSpecifier(hasPrefix(specifiesType(asString("A"))))));
   EXPECT_TRUE(matches("struct A { struct B { struct C {}; }; }; A::B::C c;",
-                      nestedNameSpecifierLoc(hasPrefix(specifiesTypeLoc(
-                          loc(qualType(asString("struct A"))))))));
+                      nestedNameSpecifierLoc(hasPrefix(
+                          specifiesTypeLoc(loc(qualType(asString("A"))))))));
   EXPECT_TRUE(matches(
       "namespace N { struct A { struct B { struct C {}; }; }; } N::A::B::C c;",
-      nestedNameSpecifierLoc(hasPrefix(
-          specifiesTypeLoc(loc(qualType(asString("struct N::A"))))))));
+      nestedNameSpecifierLoc(
+          hasPrefix(specifiesTypeLoc(loc(qualType(asString("N::A"))))))));
 }
 
 template <typename T>
@@ -2338,8 +2354,7 @@ TEST_P(ASTMatchersTest,
   }
   EXPECT_TRUE(matches(
       "template <typename T> class C {}; C<char> var;",
-      varDecl(hasName("var"), hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
-                                  templateSpecializationTypeLoc()))))));
+      varDecl(hasName("var"), hasTypeLoc(templateSpecializationTypeLoc()))));
 }
 
 TEST_P(
@@ -2353,58 +2368,6 @@ TEST_P(
       varDecl(hasName("var"), hasTypeLoc(templateSpecializationTypeLoc()))));
 }
 
-TEST_P(ASTMatchersTest,
-       ElaboratedTypeLocTest_BindsToElaboratedObjectDeclaration) {
-  if (!GetParam().isCXX()) {
-    return;
-  }
-  EXPECT_TRUE(matches("class C {}; class C c;",
-                      varDecl(hasName("c"), hasTypeLoc(elaboratedTypeLoc()))));
-}
-
-TEST_P(ASTMatchersTest,
-       ElaboratedTypeLocTest_BindsToNamespaceElaboratedObjectDeclaration) {
-  if (!GetParam().isCXX()) {
-    return;
-  }
-  EXPECT_TRUE(matches("namespace N { class D {}; } N::D d;",
-                      varDecl(hasName("d"), hasTypeLoc(elaboratedTypeLoc()))));
-}
-
-TEST_P(ASTMatchersTest,
-       ElaboratedTypeLocTest_BindsToElaboratedStructDeclaration) {
-  EXPECT_TRUE(matches("struct s {}; struct s ss;",
-                      varDecl(hasName("ss"), hasTypeLoc(elaboratedTypeLoc()))));
-}
-
-TEST_P(ASTMatchersTest,
-       ElaboratedTypeLocTest_BindsToBareElaboratedObjectDeclaration) {
-  if (!GetParam().isCXX()) {
-    return;
-  }
-  EXPECT_TRUE(matches("class C {}; C c;",
-                      varDecl(hasName("c"), hasTypeLoc(elaboratedTypeLoc()))));
-}
-
-TEST_P(
-    ASTMatchersTest,
-    ElaboratedTypeLocTest_DoesNotBindToNamespaceNonElaboratedObjectDeclaration) {
-  if (!GetParam().isCXX()) {
-    return;
-  }
-  EXPECT_TRUE(matches("namespace N { class D {}; } using N::D; D d;",
-                      varDecl(hasName("d"), hasTypeLoc(elaboratedTypeLoc()))));
-}
-
-TEST_P(ASTMatchersTest,
-       ElaboratedTypeLocTest_BindsToBareElaboratedStructDeclaration) {
-  if (!GetParam().isCXX()) {
-    return;
-  }
-  EXPECT_TRUE(matches("struct s {}; s ss;",
-                      varDecl(hasName("ss"), hasTypeLoc(elaboratedTypeLoc()))));
-}
-
 TEST_P(ASTMatchersTest, LambdaCaptureTest) {
   if (!GetParam().isCXX11OrLater()) {
     return;
diff --git a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
index 8ddae4e47004b..c0a03deb5b543 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersTraversalTest.cpp
@@ -190,13 +190,11 @@ TEST(TypeMatcher, MatchesDeclTypes) {
   EXPECT_TRUE(matches("template <typename T> struct S {"
                       "  void f(S s);"
                       "};",
-                      parmVarDecl(hasType(elaboratedType(
-                          namesType(injectedClassNameType()))))));
+                      parmVarDecl(hasType(injectedClassNameType()))));
   EXPECT_TRUE(notMatches("template <typename T> struct S {"
                          "  void g(S<T> s);"
                          "};",
-                         parmVarDecl(hasType(elaboratedType(
-                             namesType(injectedClassNameType()))))));
+                         parmVarDecl(hasType(injectedClassNameType()))));
   // InjectedClassNameType -> CXXRecordDecl
   EXPECT_TRUE(matches("template <typename T> struct S {"
                         "  void f(S s);"
@@ -228,46 +226,39 @@ TEST(HasDeclaration, HasDeclarationOfEnumType) {
 TEST(HasDeclaration, HasGetDeclTraitTest) {
   static_assert(internal::has_getDecl<TypedefType>,
                 "Expected TypedefType to have a getDecl.");
-  static_assert(internal::has_getDecl<RecordType>,
-                "Expected RecordType to have a getDecl.");
   static_assert(!internal::has_getDecl<TemplateSpecializationType>,
                 "Expected TemplateSpecializationType to *not* have a getDecl.");
 }
 
 TEST(HasDeclaration, ElaboratedType) {
-  EXPECT_TRUE(matches(
-      "namespace n { template <typename T> struct X {}; }"
-      "void f(n::X<int>);",
-      parmVarDecl(hasType(qualType(hasDeclaration(cxxRecordDecl()))))));
-  EXPECT_TRUE(matches(
-      "namespace n { template <typename T> struct X {}; }"
-      "void f(n::X<int>);",
-      parmVarDecl(hasType(elaboratedType(hasDeclaration(cxxRecordDecl()))))));
+  static const char Elaborated[] = "namespace n { struct X {}; }"
+                                   "void f(n::X);";
+  EXPECT_TRUE(
+      matches(Elaborated,
+              parmVarDecl(hasType(qualType(hasDeclaration(cxxRecordDecl()))))));
+  EXPECT_TRUE(matches(Elaborated, parmVarDecl(hasType(recordType(
+                                      hasDeclaration(cxxRecordDecl()))))));
 }
 
 TEST(HasDeclaration, HasDeclarationOfTypeWithDecl) {
   EXPECT_TRUE(matches(
       "typedef int X; X a;",
-      varDecl(hasName("a"), hasType(elaboratedType(namesType(
-                                typedefType(hasDeclaration(decl()))))))));
+      varDecl(hasName("a"), hasType(typedefType(hasDeclaration(decl()))))));
 
   // FIXME: Add tests for other types with getDecl() (e.g. RecordType)
 }
 
 TEST(HasDeclaration, HasDeclarationOfTemplateSpecializationType) {
-  EXPECT_TRUE(matches(
-      "template <typename T> class A {}; A<int> a;",
-      varDecl(hasType(elaboratedType(namesType(templateSpecializationType(
-          hasDeclaration(namedDecl(hasName("A"))))))))));
-  EXPECT_TRUE(matches(
-      "template <typename T> class A {};"
-      "template <typename T> class B { A<T> a; };",
-      fieldDecl(hasType(elaboratedType(namesType(templateSpecializationType(
-          hasDeclaration(namedDecl(hasName("A"))))))))));
-  EXPECT_TRUE(matches(
-      "template <typename T> class A {}; A<int> a;",
-      varDecl(hasType(elaboratedType(namesType(
-          templateSpecializationType(hasDeclaration(cxxRecordDecl()))))))));
+  EXPECT_TRUE(matches("template <typename T> class A {}; A<int> a;",
+                      varDecl(hasType(templateSpecializationType(
+                          hasDeclaration(namedDecl(hasName("A"))))))));
+  EXPECT_TRUE(matches("template <typename T> class A {};"
+                      "template <typename T> class B { A<T> a; };",
+                      fieldDecl(hasType(templateSpecializationType(
+                          hasDeclaration(namedDecl(hasName("A"))))))));
+  EXPECT_TRUE(matches("template <typename T> class A {}; A<int> a;",
+                      varDecl(hasType(templateSpecializationType(
+                          hasDeclaration(cxxRecordDecl()))))));
 }
 
 TEST(HasDeclaration, HasDeclarationOfCXXNewExpr) {
@@ -277,10 +268,9 @@ TEST(HasDeclaration, HasDeclarationOfCXXNewExpr) {
 }
 
 TEST(HasDeclaration, HasDeclarationOfTypeAlias) {
-  EXPECT_TRUE(matches(
-      "template <typename T> using C = T; C<int> c;",
-      varDecl(hasType(elaboratedType(namesType(templateSpecializationType(
-          hasDeclaration(typeAliasTemplateDecl()))))))));
+  EXPECT_TRUE(matches("template <typename T> using C = T; C<int> c;",
+                      varDecl(hasType(templateSpecializationType(
+                          hasDeclaration(typeAliasTemplateDecl()))))));
 }
 
 TEST(HasDeclaration, HasDeclarationOfObjCInterface) {
@@ -5152,21 +5142,6 @@ TEST(ForEachLambdaCapture, MatchExplicitCapturesOnly) {
       matcher, std::make_unique<VerifyIdIsBoundTo<LambdaCapture>>("LC", 1)));
 }
 
-TEST(HasConditionVariableStatement, DoesNotMatchCondition) {
-  EXPECT_TRUE(notMatches(
-    "void x() { if(true) {} }",
-    ifStmt(hasConditionVariableStatement(declStmt()))));
-  EXPECT_TRUE(notMatches(
-    "void x() { int x; if((x = 42)) {} }",
-    ifStmt(hasConditionVariableStatement(declStmt()))));
-}
-
-TEST(HasConditionVariableStatement, MatchesConditionVariables) {
-  EXPECT_TRUE(matches(
-    "void x() { if(int* a = 0) {} }",
-    ifStmt(hasConditionVariableStatement(declStmt()))));
-}
-
 TEST(ForEach, BindsOneNode) {
   EXPECT_TRUE(matchAndVerifyResultTrue("class C { int x; };",
                                        recordDecl(hasName("C"), forEach(fieldDecl(hasName("x")).bind("x"))),
@@ -5410,14 +5385,15 @@ TEST(LoopingMatchers, DoNotOverwritePreviousMatchResultOnFailure) {
       functionDecl(parameterCountIs(1))))),
     std::make_unique<VerifyIdIsBoundTo<Decl>>("x", 1)));
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "class A{}; class B{}; class C : B, A {};",
-    cxxRecordDecl(decl().bind("x"), isDerivedFrom("::A")),
-    std::make_unique<VerifyIdIsBoundTo<Decl>>("x", 1)));
+      "class A{}; class B{}; class C : B, A {};",
+      cxxRecordDecl(decl().bind("x"), isDerivedFrom("::A"),
+                    unless(isImplicit())),
+      std::make_unique<VerifyIdIsBoundTo<Decl>>("x", 1)));
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    "class A{}; typedef A B; typedef A C; typedef A D;"
+      "class A{}; typedef A B; typedef A C; typedef A D;"
       "class E : A {};",
-    cxxRecordDecl(decl().bind("x"), isDerivedFrom("C")),
-    std::make_unique<VerifyIdIsBoundTo<Decl>>("x", 1)));
+      cxxRecordDecl(decl().bind("x"), isDerivedFrom("C"), unless(isImplicit())),
+      std::make_unique<VerifyIdIsBoundTo<Decl>>("x", 1)));
   EXPECT_TRUE(matchAndVerifyResultTrue(
     "class A { class B { void f() {} }; };",
     functionDecl(decl().bind("x"), hasAncestor(recordDecl(hasName("::A")))),
@@ -5710,7 +5686,7 @@ TEST(HasAnyBase, BindsInnerBoundNodes) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
       "struct Inner {}; struct Proxy : Inner {}; struct Main : public "
       "Proxy {};",
-      cxxRecordDecl(hasName("Main"),
+      cxxRecordDecl(hasName("Main"), unless(isImplicit()),
                     hasAnyBase(cxxBaseSpecifier(hasType(
                         cxxRecordDecl(hasName("Inner")).bind("base-class")))))
           .bind("class"),
@@ -5736,47 +5712,28 @@ TEST(TypeMatching, PointeeTypes) {
 
 TEST(ElaboratedTypeNarrowing, hasQualifier) {
   EXPECT_TRUE(matches(
-    "namespace N {"
-      "  namespace M {"
-      "    class D {};"
-      "  }"
-      "}"
-      "N::M::D d;",
-    elaboratedType(hasQualifier(hasPrefix(specifiesNamespace(hasName("N")))))));
-  EXPECT_TRUE(notMatches(
-    "namespace M {"
-      "  class D {};"
-      "}"
-      "M::D d;",
-    elaboratedType(hasQualifier(hasPrefix(specifiesNamespace(hasName("N")))))));
-  EXPECT_TRUE(notMatches(
-    "struct D {"
-      "} d;",
-    elaboratedType(hasQualifier(nestedNameSpecifier()))));
-}
-
-TEST(ElaboratedTypeNarrowing, namesType) {
-  EXPECT_TRUE(matches(
-    "namespace N {"
+      "namespace N {"
       "  namespace M {"
       "    class D {};"
       "  }"
       "}"
       "N::M::D d;",
-    elaboratedType(elaboratedType(namesType(recordType(
-      hasDeclaration(namedDecl(hasName("D")))))))));
+      recordType(hasQualifier(hasPrefix(specifiesNamespace(hasName("N")))))));
   EXPECT_TRUE(notMatches(
-    "namespace M {"
+      "namespace M {"
       "  class D {};"
       "}"
       "M::D d;",
-    elaboratedType(elaboratedType(namesType(typedefType())))));
+      recordType(hasQualifier(hasPrefix(specifiesNamespace(hasName("N")))))));
+  EXPECT_TRUE(notMatches("struct D {"
+                         "} d;",
+                         recordType(hasQualifier(nestedNameSpecifier()))));
 }
 
 TEST(NNS, BindsNestedNameSpecifiers) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
       "namespace ns { struct E { struct B {}; }; } ns::E::B b;",
-      nestedNameSpecifier(specifiesType(asString("struct ns::E"))).bind("nns"),
+      nestedNameSpecifier(specifiesType(asString("ns::E"))).bind("nns"),
       std::make_unique<VerifyIdIsBoundTo<NestedNameSpecifier>>("nns",
                                                                "ns::E::")));
 }
@@ -5793,38 +5750,33 @@ TEST(NNS, DescendantsOfNestedNameSpecifiers) {
       "namespace a { struct A { struct B { struct C {}; }; }; };"
       "void f() { a::A::B::C c; }";
   EXPECT_TRUE(matches(
-    Fragment,
-    nestedNameSpecifier(specifiesType(asString("struct a::A::B")),
-                        hasDescendant(nestedNameSpecifier(
-                          specifiesNamespace(hasName("a")))))));
+      Fragment, nestedNameSpecifier(specifiesType(asString("a::A::B")),
+                                    hasDescendant(nestedNameSpecifier(
+                                        specifiesNamespace(hasName("a")))))));
   EXPECT_TRUE(notMatches(
-    Fragment,
-    nestedNameSpecifier(specifiesType(asString("struct a::A::B")),
-                        has(nestedNameSpecifier(
-                          specifiesNamespace(hasName("a")))))));
+      Fragment, nestedNameSpecifier(specifiesType(asString("a::A::B")),
+                                    has(nestedNameSpecifier(
+                                        specifiesNamespace(hasName("a")))))));
   EXPECT_TRUE(matches(
-    Fragment,
-    nestedNameSpecifier(specifiesType(asString("struct a::A")),
-                        has(nestedNameSpecifier(
-                          specifiesNamespace(hasName("a")))))));
+      Fragment, nestedNameSpecifier(specifiesType(asString("a::A")),
+                                    has(nestedNameSpecifier(
+                                        specifiesNamespace(hasName("a")))))));
 
   // Not really useful because a NestedNameSpecifier can af at most one child,
   // but to complete the interface.
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    Fragment,
-    nestedNameSpecifier(specifiesType(asString("struct a::A::B")),
-                        forEach(nestedNameSpecifier().bind("x"))),
-    std::make_unique<VerifyIdIsBoundTo<NestedNameSpecifier>>("x", 1)));
+      Fragment,
+      nestedNameSpecifier(specifiesType(asString("a::A::B")),
+                          forEach(nestedNameSpecifier().bind("x"))),
+      std::make_unique<VerifyIdIsBoundTo<NestedNameSpecifier>>("x", 1)));
 }
 
 TEST(NNS, NestedNameSpecifiersAsDescendants) {
   StringRef Fragment =
       "namespace a { struct A { struct B { struct C {}; }; }; };"
       "void f() { a::A::B::C c; }";
-  EXPECT_TRUE(matches(
-    Fragment,
-    decl(hasDescendant(nestedNameSpecifier(specifiesType(
-      asString("struct a::A")))))));
+  EXPECT_TRUE(matches(Fragment, decl(hasDescendant(nestedNameSpecifier(
+                                    specifiesType(asString("a::A")))))));
   EXPECT_TRUE(matchAndVerifyResultTrue(
     Fragment,
     functionDecl(hasName("f"),
@@ -5837,37 +5789,34 @@ TEST(NNSLoc, DescendantsOfNestedNameSpecifierLocs) {
   StringRef Fragment =
       "namespace a { struct A { struct B { struct C {}; }; }; };"
       "void f() { a::A::B::C c; }";
-  EXPECT_TRUE(matches(
-    Fragment,
-    nestedNameSpecifierLoc(loc(specifiesType(asString("struct a::A::B"))),
-                           hasDescendant(loc(nestedNameSpecifier(
-                             specifiesNamespace(hasName("a"))))))));
+  EXPECT_TRUE(matches(Fragment, nestedNameSpecifierLoc(
+                                    loc(specifiesType(asString("a::A::B"))),
+                                    hasDescendant(loc(nestedNameSpecifier(
+                                        specifiesNamespace(hasName("a"))))))));
   EXPECT_TRUE(notMatches(
-    Fragment,
-    nestedNameSpecifierLoc(loc(specifiesType(asString("struct a::A::B"))),
-                           has(loc(nestedNameSpecifier(
-                             specifiesNamespace(hasName("a"))))))));
+      Fragment,
+      nestedNameSpecifierLoc(
+          loc(specifiesType(asString("a::A::B"))),
+          has(loc(nestedNameSpecifier(specifiesNamespace(hasName("a"))))))));
   EXPECT_TRUE(matches(
-    Fragment,
-    nestedNameSpecifierLoc(loc(specifiesType(asString("struct a::A"))),
-                           has(loc(nestedNameSpecifier(
-                             specifiesNamespace(hasName("a"))))))));
+      Fragment,
+      nestedNameSpecifierLoc(
+          loc(specifiesType(asString("a::A"))),
+          has(loc(nestedNameSpecifier(specifiesNamespace(hasName("a"))))))));
 
   EXPECT_TRUE(matchAndVerifyResultTrue(
-    Fragment,
-    nestedNameSpecifierLoc(loc(specifiesType(asString("struct a::A::B"))),
-                           forEach(nestedNameSpecifierLoc().bind("x"))),
-    std::make_unique<VerifyIdIsBoundTo<NestedNameSpecifierLoc>>("x", 1)));
+      Fragment,
+      nestedNameSpecifierLoc(loc(specifiesType(asString("a::A::B"))),
+                             forEach(nestedNameSpecifierLoc().bind("x"))),
+      std::make_unique<VerifyIdIsBoundTo<NestedNameSpecifierLoc>>("x", 1)));
 }
 
 TEST(NNSLoc, NestedNameSpecifierLocsAsDescendants) {
   StringRef Fragment =
       "namespace a { struct A { struct B { struct C {}; }; }; };"
       "void f() { a::A::B::C c; }";
-  EXPECT_TRUE(matches(
-    Fragment,
-    decl(hasDescendant(loc(nestedNameSpecifier(specifiesType(
-      asString("struct a::A"))))))));
+  EXPECT_TRUE(matches(Fragment, decl(hasDescendant(loc(nestedNameSpecifier(
+                                    specifiesType(asString("a::A"))))))));
   EXPECT_TRUE(matchAndVerifyResultTrue(
     Fragment,
     functionDecl(hasName("f"),
@@ -5883,7 +5832,9 @@ TEST(Attr, AttrsAsDescendants) {
   EXPECT_TRUE(matchAndVerifyResultTrue(
       Fragment,
       namespaceDecl(hasName("a"),
-                    forEachDescendant(attr(unless(isImplicit())).bind("x"))),
+                    forEachDescendant(decl(
+                        hasDescendant(attr(unless(isImplicit())).bind("x")),
+                        unless(isImplicit())))),
       std::make_unique<VerifyIdIsBoundTo<Attr>>("x", 2)));
 }
 
@@ -6531,21 +6482,19 @@ TEST(HasReferentLoc, DoesNotBindToParameterWithoutIntReferenceTypeLoc) {
 }
 
 TEST(HasAnyTemplateArgumentLoc, BindsToSpecializationWithIntArgument) {
-  EXPECT_TRUE(matches(
-      "template<typename T> class A {}; A<int> a;",
-      varDecl(hasName("a"),
-              hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
-                  templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-                      hasTypeLoc(loc(asString("int")))))))))));
+  EXPECT_TRUE(
+      matches("template<typename T> class A {}; A<int> a;",
+              varDecl(hasName("a"), hasTypeLoc(templateSpecializationTypeLoc(
+                                        hasAnyTemplateArgumentLoc(hasTypeLoc(
+                                            loc(asString("int")))))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc, BindsToSpecializationWithDoubleArgument) {
-  EXPECT_TRUE(matches(
-      "template<typename T> class A {}; A<double> a;",
-      varDecl(hasName("a"),
-              hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
-                  templateSpecializationTypeLoc(hasAnyTemplateArgumentLoc(
-                      hasTypeLoc(loc(asString("double")))))))))));
+  EXPECT_TRUE(
+      matches("template<typename T> class A {}; A<double> a;",
+              varDecl(hasName("a"), hasTypeLoc(templateSpecializationTypeLoc(
+                                        hasAnyTemplateArgumentLoc(hasTypeLoc(
+                                            loc(asString("double")))))))));
 }
 
 TEST(HasAnyTemplateArgumentLoc, BindsToExplicitSpecializationWithIntArgument) {
@@ -6598,30 +6547,27 @@ TEST(HasAnyTemplateArgumentLoc,
 }
 
 TEST(HasTemplateArgumentLoc, BindsToSpecializationWithIntArgument) {
-  EXPECT_TRUE(
-      matches("template<typename T> class A {}; A<int> a;",
-              varDecl(hasName("a"),
-                      hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
-                          templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                              0, hasTypeLoc(loc(asString("int")))))))))));
+  EXPECT_TRUE(matches(
+      "template<typename T> class A {}; A<int> a;",
+      varDecl(hasName("a"),
+              hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                  0, hasTypeLoc(loc(asString("int")))))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToSpecializationWithDoubleArgument) {
-  EXPECT_TRUE(
-      matches("template<typename T> class A {}; A<double> a;",
-              varDecl(hasName("a"),
-                      hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
-                          templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                              0, hasTypeLoc(loc(asString("double")))))))))));
+  EXPECT_TRUE(matches(
+      "template<typename T> class A {}; A<double> a;",
+      varDecl(hasName("a"),
+              hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                  0, hasTypeLoc(loc(asString("double")))))))));
 }
 
 TEST(HasTemplateArgumentLoc, DoesNotBindToSpecializationWithIntArgument) {
   EXPECT_TRUE(notMatches(
       "template<typename T> class A {}; A<int> a;",
       varDecl(hasName("a"),
-              hasTypeLoc(elaboratedTypeLoc(hasNamedTypeLoc(
-                  templateSpecializationTypeLoc(hasTemplateArgumentLoc(
-                      0, hasTypeLoc(loc(asString("double")))))))))));
+              hasTypeLoc(templateSpecializationTypeLoc(hasTemplateArgumentLoc(
+                  0, hasTypeLoc(loc(asString("double")))))))));
 }
 
 TEST(HasTemplateArgumentLoc, BindsToExplicitSpecializationWithIntArgument) {
@@ -6735,12 +6681,11 @@ TEST(HasNamedTypeLoc, BindsToElaboratedObjectDeclaration) {
       class C<int> c;
       )",
       varDecl(hasName("c"),
-              hasTypeLoc(elaboratedTypeLoc(
-                  hasNamedTypeLoc(templateSpecializationTypeLoc(
-                      hasAnyTemplateArgumentLoc(templateArgumentLoc()))))))));
+              hasTypeLoc(templateSpecializationTypeLoc(
+                  hasAnyTemplateArgumentLoc(templateArgumentLoc()))))));
 }
 
-TEST(HasNamedTypeLoc, DoesNotBindToNonElaboratedObjectDeclaration) {
+TEST(HasNamedTypeLoc, BindsToNonElaboratedObjectDeclaration) {
   EXPECT_TRUE(matches(
       R"(
       template <typename T>
@@ -6748,9 +6693,8 @@ TEST(HasNamedTypeLoc, DoesNotBindToNonElaboratedObjectDeclaration) {
       C<int> c;
       )",
       varDecl(hasName("c"),
-              hasTypeLoc(elaboratedTypeLoc(
-                  hasNamedTypeLoc(templateSpecializationTypeLoc(
-                      hasAnyTemplateArgumentLoc(templateArgumentLoc()))))))));
+              hasTypeLoc(templateSpecializationTypeLoc(
+                  hasAnyTemplateArgumentLoc(templateArgumentLoc()))))));
 }
 
 } // namespace ast_matchers
diff --git a/clang/unittests/Analysis/FlowSensitive/CMakeLists.txt b/clang/unittests/Analysis/FlowSensitive/CMakeLists.txt
index 4ac563143cd68..3bd4a6e21bee7 100644
--- a/clang/unittests/Analysis/FlowSensitive/CMakeLists.txt
+++ b/clang/unittests/Analysis/FlowSensitive/CMakeLists.txt
@@ -8,6 +8,7 @@ add_clang_unittest(ClangAnalysisFlowSensitiveTests
   DataflowEnvironmentTest.cpp
   DebugSupportTest.cpp
   DeterminismTest.cpp
+  FormulaTest.cpp
   LoggerTest.cpp
   MapLatticeTest.cpp
   MatchSwitchTest.cpp
diff --git a/clang/unittests/Analysis/FlowSensitive/CachedConstAccessorsLatticeTest.cpp b/clang/unittests/Analysis/FlowSensitive/CachedConstAccessorsLatticeTest.cpp
index ffc50fbb65523..fb3ab7c0dc9b4 100644
--- a/clang/unittests/Analysis/FlowSensitive/CachedConstAccessorsLatticeTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/CachedConstAccessorsLatticeTest.cpp
@@ -80,7 +80,7 @@ struct CommonTestInputs {
   )cpp") {
     auto *SDecl = cast<CXXRecordDecl>(
         lookup("S", *AST.context().getTranslationUnitDecl()));
-    SType = AST.context().getRecordType(SDecl);
+    SType = AST.context().getCanonicalTagType(SDecl);
     CallVal = selectFirst<CallExpr>(
         "call",
         match(cxxMemberCallExpr(callee(functionDecl(hasName("valProperty"))))
@@ -163,7 +163,7 @@ TEST_F(CachedConstAccessorsLatticeTest,
   )cpp");
   auto *SDecl =
       cast<CXXRecordDecl>(lookup("S", *AST.context().getTranslationUnitDecl()));
-  QualType SType = AST.context().getRecordType(SDecl);
+  CanQualType SType = AST.context().getCanonicalTagType(SDecl);
   const CallExpr *CE = selectFirst<CallExpr>(
       "call", match(cxxMemberCallExpr(
                         callee(functionDecl(hasName("structValProperty"))))
@@ -237,7 +237,7 @@ TEST_F(CachedConstAccessorsLatticeTest, DifferentValsFromDifferentLocs) {
   )cpp");
   auto *SDecl =
       cast<CXXRecordDecl>(lookup("S", *AST.context().getTranslationUnitDecl()));
-  QualType SType = AST.context().getRecordType(SDecl);
+  CanQualType SType = AST.context().getCanonicalTagType(SDecl);
   SmallVector<BoundNodes, 1> valPropertyCalls =
       match(cxxMemberCallExpr(callee(functionDecl(hasName("valProperty"))))
                 .bind("call"),
diff --git a/clang/unittests/Analysis/FlowSensitive/DataflowAnalysisContextTest.cpp b/clang/unittests/Analysis/FlowSensitive/DataflowAnalysisContextTest.cpp
index 4f7a72c502ccf..92b687a5a18a4 100644
--- a/clang/unittests/Analysis/FlowSensitive/DataflowAnalysisContextTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/DataflowAnalysisContextTest.cpp
@@ -17,6 +17,9 @@ namespace {
 using namespace clang;
 using namespace dataflow;
 
+using ::testing::IsEmpty;
+using ::testing::UnorderedElementsAre;
+
 class DataflowAnalysisContextTest : public ::testing::Test {
 protected:
   DataflowAnalysisContextTest()
@@ -171,4 +174,97 @@ TEST_F(DataflowAnalysisContextTest, EquivBoolVals) {
                                          A.makeAnd(X, A.makeAnd(Y, Z))));
 }
 
+using ExportLogicalContextTest = DataflowAnalysisContextTest;
+
+TEST_F(ExportLogicalContextTest, EmptySet) {
+  EXPECT_THAT(Context.exportLogicalContext({}).TokenDefs, IsEmpty());
+}
+
+// Only constrainted tokens are included in the output.
+TEST_F(ExportLogicalContextTest, UnconstrainedIgnored) {
+  Atom FC1 = A.makeFlowConditionToken();
+  EXPECT_THAT(Context.exportLogicalContext({FC1}).TokenDefs, IsEmpty());
+}
+
+TEST_F(ExportLogicalContextTest, SingletonSet) {
+  Atom FC1 = A.makeFlowConditionToken();
+  auto &C1 = A.makeAtomRef(A.makeAtom());
+  Context.addFlowConditionConstraint(FC1, C1);
+  EXPECT_THAT(Context.exportLogicalContext({FC1}).TokenDefs.keys(),
+              UnorderedElementsAre(FC1));
+}
+
+TEST_F(ExportLogicalContextTest, NoDependency) {
+  Atom FC1 = A.makeFlowConditionToken();
+  Atom FC2 = A.makeFlowConditionToken();
+  Atom FC3 = A.makeFlowConditionToken();
+  auto &C1 = A.makeAtomRef(A.makeAtom());
+  auto &C2 = A.makeAtomRef(A.makeAtom());
+  auto &C3 = A.makeAtomRef(A.makeAtom());
+
+  Context.addFlowConditionConstraint(FC1, C1);
+  Context.addFlowConditionConstraint(FC2, C2);
+  Context.addFlowConditionConstraint(FC3, C3);
+
+  // FCs are independent.
+  EXPECT_THAT(Context.exportLogicalContext({FC1}).TokenDefs.keys(),
+              UnorderedElementsAre(FC1));
+  EXPECT_THAT(Context.exportLogicalContext({FC2}).TokenDefs.keys(),
+              UnorderedElementsAre(FC2));
+  EXPECT_THAT(Context.exportLogicalContext({FC3}).TokenDefs.keys(),
+              UnorderedElementsAre(FC3));
+}
+
+TEST_F(ExportLogicalContextTest, SimpleDependencyChain) {
+  Atom FC1 = A.makeFlowConditionToken();
+  const Formula &C = A.makeAtomRef(A.makeAtom());
+  Context.addFlowConditionConstraint(FC1, C);
+  Atom FC2 = Context.forkFlowCondition(FC1);
+  Atom FC3 = Context.forkFlowCondition(FC2);
+
+  EXPECT_THAT(Context.exportLogicalContext({FC3}).TokenDefs.keys(),
+              UnorderedElementsAre(FC1, FC2, FC3));
+}
+
+TEST_F(ExportLogicalContextTest, DependencyTree) {
+  Atom FC1 = A.makeFlowConditionToken();
+  const Formula &C = A.makeAtomRef(A.makeAtom());
+  Context.addFlowConditionConstraint(FC1, C);
+  Atom FC2 = Context.forkFlowCondition(FC1);
+  Atom FC3 = A.makeFlowConditionToken();
+  Context.addFlowConditionConstraint(FC3, C);
+  Atom FC4 = Context.joinFlowConditions(FC2, FC3);
+
+  EXPECT_THAT(Context.exportLogicalContext({FC4}).TokenDefs.keys(),
+              UnorderedElementsAre(FC1, FC2, FC3, FC4));
+}
+
+TEST_F(ExportLogicalContextTest, DependencyDAG) {
+  Atom FC1 = A.makeFlowConditionToken();
+  const Formula &C = A.makeAtomRef(A.makeAtom());
+  Context.addFlowConditionConstraint(FC1, C);
+
+  Atom FC2 = Context.forkFlowCondition(FC1);
+  Atom FC3 = Context.forkFlowCondition(FC1);
+  Atom FC4 = Context.joinFlowConditions(FC2, FC3);
+
+  EXPECT_THAT(Context.exportLogicalContext({FC4}).TokenDefs.keys(),
+              UnorderedElementsAre(FC1, FC2, FC3, FC4));
+}
+
+TEST_F(ExportLogicalContextTest, MixedDependencies) {
+  Atom FC1 = A.makeFlowConditionToken();
+  const Formula &C = A.makeAtomRef(A.makeAtom());
+  Context.addFlowConditionConstraint(FC1, C);
+
+  Atom FC2 = Context.forkFlowCondition(FC1);
+  Atom FC3 = Context.forkFlowCondition(FC2);
+  (void)FC3; // unused, and we test below that it is not included.
+
+  Atom FC4 = A.makeFlowConditionToken();
+  Context.addFlowConditionConstraint(FC4, C);
+
+  EXPECT_THAT(Context.exportLogicalContext({FC2, FC4}).TokenDefs.keys(),
+              UnorderedElementsAre(FC1, FC2, FC4));
+}
 } // namespace
diff --git a/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp b/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
index 737277e167edd..0780db921652c 100644
--- a/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/DataflowEnvironmentTest.cpp
@@ -402,7 +402,7 @@ TEST_F(EnvironmentTest,
   // `Target<&S::accessor>`.
   Environment Env(DAContext, *Fun);
   Env.initialize();
-  EXPECT_THAT(DAContext.getModeledFields(QualType(Struct->getTypeForDecl(), 0)),
+  EXPECT_THAT(DAContext.getModeledFields(Context.getCanonicalTagType(Struct)),
               Contains(Member));
 }
 
diff --git a/clang/unittests/Analysis/FlowSensitive/FormulaTest.cpp b/clang/unittests/Analysis/FlowSensitive/FormulaTest.cpp
new file mode 100644
index 0000000000000..db9c02854c7fa
--- /dev/null
+++ b/clang/unittests/Analysis/FlowSensitive/FormulaTest.cpp
@@ -0,0 +1,199 @@
+//===- unittests/Analysis/FlowSensitive/FormulaTest.cpp -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Analysis/FlowSensitive/Formula.h"
+#include "clang/Analysis/FlowSensitive/Arena.h"
+#include "clang/Analysis/FlowSensitive/FormulaSerialization.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace {
+
+using namespace clang;
+using namespace dataflow;
+
+using ::llvm::Failed;
+using ::llvm::HasValue;
+using ::llvm::Succeeded;
+
+class SerializeFormulaTest : public ::testing::Test {
+protected:
+  Arena A;
+  std::string Out;
+  llvm::raw_string_ostream OS{Out};
+
+  const Formula &A1 = A.makeAtomRef(A.makeAtom());
+  const Formula &A2 = A.makeAtomRef(A.makeAtom());
+};
+
+TEST_F(SerializeFormulaTest, Atom) {
+  serializeFormula(A1, OS);
+  EXPECT_EQ(Out, "V0");
+  Out = "";
+
+  serializeFormula(A2, OS);
+  EXPECT_EQ(Out, "V1");
+}
+
+TEST_F(SerializeFormulaTest, LiteralTrue) {
+  serializeFormula(A.makeLiteral(true), OS);
+  EXPECT_EQ(Out, "T");
+}
+
+TEST_F(SerializeFormulaTest, LiteralFalse) {
+  serializeFormula(A.makeLiteral(false), OS);
+  EXPECT_EQ(Out, "F");
+}
+
+TEST_F(SerializeFormulaTest, Not) {
+  serializeFormula(A.makeNot(A1), OS);
+  EXPECT_EQ(Out, "!V0");
+}
+
+TEST_F(SerializeFormulaTest, Or) {
+  serializeFormula(A.makeOr(A1, A2), OS);
+  EXPECT_EQ(Out, "|V0V1");
+}
+
+TEST_F(SerializeFormulaTest, And) {
+  serializeFormula(A.makeAnd(A1, A2), OS);
+  EXPECT_EQ(Out, "&V0V1");
+}
+
+TEST_F(SerializeFormulaTest, Implies) {
+  serializeFormula(A.makeImplies(A1, A2), OS);
+  EXPECT_EQ(Out, ">V0V1");
+}
+
+TEST_F(SerializeFormulaTest, Equal) {
+  serializeFormula(A.makeEquals(A1, A2), OS);
+  EXPECT_EQ(Out, "=V0V1");
+}
+
+TEST_F(SerializeFormulaTest, NestedBinaryUnary) {
+  serializeFormula(A.makeEquals(A.makeOr(A1, A2), A2), OS);
+  EXPECT_EQ(Out, "=|V0V1V1");
+}
+
+TEST_F(SerializeFormulaTest, NestedBinaryBinary) {
+  serializeFormula(A.makeEquals(A.makeOr(A1, A2), A.makeAnd(A1, A2)), OS);
+  EXPECT_EQ(Out, "=|V0V1&V0V1");
+}
+
+class ParseFormulaTest : public ::testing::Test {
+protected:
+  void SetUp() override {
+    AtomMap[0] = Atom1;
+    AtomMap[1] = Atom2;
+  }
+
+  // Convenience wrapper for `testParseFormula`.
+  llvm::Expected<const Formula *> testParseFormula(llvm::StringRef Str) {
+    return parseFormula(Str, A, AtomMap);
+  }
+
+  Arena A;
+  std::string Out;
+  llvm::raw_string_ostream OS{Out};
+
+  Atom Atom1 = A.makeAtom();
+  Atom Atom2 = A.makeAtom();
+  const Formula &A1 = A.makeAtomRef(Atom1);
+  const Formula &A2 = A.makeAtomRef(Atom2);
+  llvm::DenseMap<unsigned, Atom> AtomMap;
+};
+
+TEST_F(ParseFormulaTest, Atom) {
+  EXPECT_THAT_EXPECTED(testParseFormula("V0"), HasValue(&A1));
+  EXPECT_THAT_EXPECTED(testParseFormula("V1"), HasValue(&A2));
+}
+
+TEST_F(ParseFormulaTest, LiteralTrue) {
+  EXPECT_THAT_EXPECTED(testParseFormula("T"), HasValue(&A.makeLiteral(true)));
+}
+
+TEST_F(ParseFormulaTest, LiteralFalse) {
+  EXPECT_THAT_EXPECTED(testParseFormula("F"), HasValue(&A.makeLiteral(false)));
+}
+
+TEST_F(ParseFormulaTest, Not) {
+  EXPECT_THAT_EXPECTED(testParseFormula("!V0"), HasValue(&A.makeNot(A1)));
+}
+
+TEST_F(ParseFormulaTest, Or) {
+  EXPECT_THAT_EXPECTED(testParseFormula("|V0V1"), HasValue(&A.makeOr(A1, A2)));
+}
+
+TEST_F(ParseFormulaTest, And) {
+  EXPECT_THAT_EXPECTED(testParseFormula("&V0V1"), HasValue(&A.makeAnd(A1, A2)));
+}
+
+TEST_F(ParseFormulaTest, OutOfNumericOrder) {
+  EXPECT_THAT_EXPECTED(testParseFormula("&V1V0"), HasValue(&A.makeAnd(A2, A1)));
+}
+
+TEST_F(ParseFormulaTest, Implies) {
+  EXPECT_THAT_EXPECTED(testParseFormula(">V0V1"),
+                       HasValue(&A.makeImplies(A1, A2)));
+}
+
+TEST_F(ParseFormulaTest, Equal) {
+  EXPECT_THAT_EXPECTED(testParseFormula("=V0V1"),
+                       HasValue(&A.makeEquals(A1, A2)));
+}
+
+TEST_F(ParseFormulaTest, NestedBinaryUnary) {
+  EXPECT_THAT_EXPECTED(testParseFormula("=|V0V1V1"),
+                       HasValue(&A.makeEquals(A.makeOr(A1, A2), A2)));
+}
+
+TEST_F(ParseFormulaTest, NestedBinaryBinary) {
+  EXPECT_THAT_EXPECTED(
+      testParseFormula("=|V0V1&V0V1"),
+      HasValue(&A.makeEquals(A.makeOr(A1, A2), A.makeAnd(A1, A2))));
+}
+
+// Verifies that parsing generates fresh atoms, if they are not already in the
+// map.
+TEST_F(ParseFormulaTest, GeneratesAtoms) {
+  llvm::DenseMap<unsigned, Atom> FreshAtomMap;
+  ASSERT_THAT_EXPECTED(parseFormula("=V0V1", A, FreshAtomMap), Succeeded());
+  // The map contains two, unique elements.
+  ASSERT_EQ(FreshAtomMap.size(), 2U);
+  EXPECT_NE(FreshAtomMap[0], FreshAtomMap[1]);
+}
+
+TEST_F(ParseFormulaTest, MalformedFormulaFails) {
+  // Arbitrary string.
+  EXPECT_THAT_EXPECTED(testParseFormula("Hello"), Failed());
+  // Empty string.
+  EXPECT_THAT_EXPECTED(testParseFormula(""), Failed());
+  // Malformed atom.
+  EXPECT_THAT_EXPECTED(testParseFormula("Vabc"), Failed());
+  // Irrelevant suffix.
+  EXPECT_THAT_EXPECTED(testParseFormula("V0Hello"), Failed());
+  EXPECT_THAT_EXPECTED(testParseFormula("=V0V1Hello"), Failed());
+  // Sequence without operator.
+  EXPECT_THAT_EXPECTED(testParseFormula("TF"), Failed());
+  // Bad subformula.
+  EXPECT_THAT_EXPECTED(testParseFormula("!G"), Failed());
+  // Incomplete formulas.
+  EXPECT_THAT_EXPECTED(testParseFormula("V"), Failed());
+  EXPECT_THAT_EXPECTED(testParseFormula("&"), Failed());
+  EXPECT_THAT_EXPECTED(testParseFormula("|"), Failed());
+  EXPECT_THAT_EXPECTED(testParseFormula(">"), Failed());
+  EXPECT_THAT_EXPECTED(testParseFormula("="), Failed());
+  EXPECT_THAT_EXPECTED(testParseFormula("&V0"), Failed());
+  EXPECT_THAT_EXPECTED(testParseFormula("|V0"), Failed());
+  EXPECT_THAT_EXPECTED(testParseFormula(">V0"), Failed());
+  EXPECT_THAT_EXPECTED(testParseFormula("=V0"), Failed());
+}
+
+} // namespace
diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
index 214aaee9f97f6..96e759e73c154 100644
--- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
@@ -9,17 +9,25 @@
 #include "TestingSupport.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/OperationKinds.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Analysis/FlowSensitive/DataflowAnalysis.h"
 #include "clang/Analysis/FlowSensitive/DataflowAnalysisContext.h"
 #include "clang/Analysis/FlowSensitive/DataflowEnvironment.h"
 #include "clang/Analysis/FlowSensitive/NoopAnalysis.h"
+#include "clang/Analysis/FlowSensitive/NoopLattice.h"
 #include "clang/Analysis/FlowSensitive/RecordOps.h"
 #include "clang/Analysis/FlowSensitive/StorageLocation.h"
 #include "clang/Analysis/FlowSensitive/Value.h"
 #include "clang/Basic/LangStandard.h"
 #include "clang/Testing/TestAST.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
@@ -27,6 +35,7 @@
 #include <string>
 #include <string_view>
 #include <utility>
+#include <vector>
 
 namespace clang {
 namespace dataflow {
@@ -3541,7 +3550,7 @@ TEST(TransferTest, ResultObjectLocationDontVisitUnevaluatedContexts) {
   testFunction(Code, "noexceptTarget");
 }
 
-TEST(TransferTest, StaticCast) {
+TEST(TransferTest, StaticCastNoOp) {
   std::string Code = R"(
     void target(int Foo) {
       int Bar = static_cast<int>(Foo);
@@ -3561,6 +3570,13 @@ TEST(TransferTest, StaticCast) {
         const ValueDecl *BarDecl = findValueDecl(ASTCtx, "Bar");
         ASSERT_THAT(BarDecl, NotNull());
 
+        const auto *Cast = ast_matchers::selectFirst<CXXStaticCastExpr>(
+            "cast",
+            ast_matchers::match(ast_matchers::cxxStaticCastExpr().bind("cast"),
+                                ASTCtx));
+        ASSERT_THAT(Cast, NotNull());
+        ASSERT_EQ(Cast->getCastKind(), CK_NoOp);
+
         const auto *FooVal = Env.getValue(*FooDecl);
         const auto *BarVal = Env.getValue(*BarDecl);
         EXPECT_TRUE(isa<IntegerValue>(FooVal));
@@ -3569,6 +3585,268 @@ TEST(TransferTest, StaticCast) {
       });
 }
 
+TEST(TransferTest, StaticCastBaseToDerived) {
+  std::string Code = R"cc(
+    struct Base {
+      char C;
+    };
+    struct Intermediate : public Base {
+      bool B;
+    };
+    struct Derived : public Intermediate {
+      int I;
+    };
+    Base& getBaseRef();
+    void target(Base* BPtr) {
+      Derived* DPtr = static_cast<Derived*>(BPtr);
+      DPtr->C;
+      DPtr->B;
+      DPtr->I;
+      Derived& DRef = static_cast<Derived&>(*BPtr);
+      DRef.C;
+      DRef.B;
+      DRef.I;
+      Derived& DRefFromFunc = static_cast<Derived&>(getBaseRef());
+      DRefFromFunc.C;
+      DRefFromFunc.B;
+      DRefFromFunc.I;
+      // [[p]]
+    }
+  )cc";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        const ValueDecl *BPtrDecl = findValueDecl(ASTCtx, "BPtr");
+        ASSERT_THAT(BPtrDecl, NotNull());
+
+        const ValueDecl *DPtrDecl = findValueDecl(ASTCtx, "DPtr");
+        ASSERT_THAT(DPtrDecl, NotNull());
+
+        const ValueDecl *DRefDecl = findValueDecl(ASTCtx, "DRef");
+        ASSERT_THAT(DRefDecl, NotNull());
+
+        const ValueDecl *DRefFromFuncDecl =
+            findValueDecl(ASTCtx, "DRefFromFunc");
+        ASSERT_THAT(DRefFromFuncDecl, NotNull());
+
+        const auto *Cast = ast_matchers::selectFirst<CXXStaticCastExpr>(
+            "cast",
+            ast_matchers::match(ast_matchers::cxxStaticCastExpr().bind("cast"),
+                                ASTCtx));
+        ASSERT_THAT(Cast, NotNull());
+        ASSERT_EQ(Cast->getCastKind(), CK_BaseToDerived);
+
+        EXPECT_EQ(Env.getValue(*BPtrDecl), Env.getValue(*DPtrDecl));
+        EXPECT_EQ(&Env.get<PointerValue>(*BPtrDecl)->getPointeeLoc(),
+                  Env.getStorageLocation(*DRefDecl));
+        // For DRefFromFunc, not crashing when analyzing the field accesses is
+        // enough.
+      });
+}
+
+TEST(TransferTest, ExplicitDerivedToBaseCast) {
+  std::string Code = R"cc(
+    struct Base {};
+    struct Derived : public Base {};
+    void target(Derived D) {
+      (Base*)&D;
+      // [[p]]
+    }
+)cc";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        auto *Cast = ast_matchers::selectFirst<ImplicitCastExpr>(
+            "cast", ast_matchers::match(
+                        ast_matchers::implicitCastExpr().bind("cast"), ASTCtx));
+        ASSERT_THAT(Cast, NotNull());
+        ASSERT_EQ(Cast->getCastKind(), CK_DerivedToBase);
+
+        auto *AddressOf = ast_matchers::selectFirst<UnaryOperator>(
+            "addressof",
+            ast_matchers::match(ast_matchers::unaryOperator().bind("addressof"),
+                                ASTCtx));
+        ASSERT_THAT(AddressOf, NotNull());
+        ASSERT_EQ(AddressOf->getOpcode(), UO_AddrOf);
+
+        EXPECT_EQ(Env.getValue(*Cast), Env.getValue(*AddressOf));
+      });
+}
+
+TEST(TransferTest, ConstructorConversion) {
+  std::string Code = R"cc(
+    struct Base {};
+    struct Derived : public Base {};
+    void target(Derived D) {
+      Base B = (Base)D;
+      // [[p]]
+    }
+)cc";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        auto *Cast = ast_matchers::selectFirst<CStyleCastExpr>(
+            "cast", ast_matchers::match(
+                        ast_matchers::cStyleCastExpr().bind("cast"), ASTCtx));
+        ASSERT_THAT(Cast, NotNull());
+        ASSERT_EQ(Cast->getCastKind(), CK_ConstructorConversion);
+
+        auto &DLoc = getLocForDecl<StorageLocation>(ASTCtx, Env, "D");
+        auto &BLoc = getLocForDecl<StorageLocation>(ASTCtx, Env, "B");
+        EXPECT_NE(&BLoc, &DLoc);
+      });
+}
+
+TEST(TransferTest, UserDefinedConversion) {
+  std::string Code = R"cc(
+    struct To {};
+    struct From {
+        operator To();
+    };
+    void target(From F) {
+        To T = (To)F;
+        // [[p]]
+    }
+)cc";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        auto *Cast = ast_matchers::selectFirst<ImplicitCastExpr>(
+            "cast", ast_matchers::match(
+                        ast_matchers::implicitCastExpr().bind("cast"), ASTCtx));
+        ASSERT_THAT(Cast, NotNull());
+        ASSERT_EQ(Cast->getCastKind(), CK_UserDefinedConversion);
+
+        auto &FLoc = getLocForDecl<StorageLocation>(ASTCtx, Env, "F");
+        auto &TLoc = getLocForDecl<StorageLocation>(ASTCtx, Env, "T");
+        EXPECT_NE(&TLoc, &FLoc);
+      });
+}
+
+TEST(TransferTest, ImplicitUncheckedDerivedToBaseCast) {
+  std::string Code = R"cc(
+    struct Base {
+      void method();
+    };
+    struct Derived : public Base {};
+    void target(Derived D) {
+      D.method();
+      // [[p]]
+    }
+)cc";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        auto *Cast = ast_matchers::selectFirst<ImplicitCastExpr>(
+            "cast", ast_matchers::match(
+                        ast_matchers::implicitCastExpr().bind("cast"), ASTCtx));
+        ASSERT_THAT(Cast, NotNull());
+        ASSERT_EQ(Cast->getCastKind(), CK_UncheckedDerivedToBase);
+
+        auto &DLoc = getLocForDecl<StorageLocation>(ASTCtx, Env, "D");
+        EXPECT_EQ(Env.getStorageLocation(*Cast), &DLoc);
+      });
+}
+
+TEST(TransferTest, ImplicitDerivedToBaseCast) {
+  std::string Code = R"cc(
+    struct Base {};
+    struct Derived : public Base {};
+    void target() {
+      Base* B = new Derived();
+      // [[p]]
+    }
+)cc";
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        auto *Cast = ast_matchers::selectFirst<ImplicitCastExpr>(
+            "cast", ast_matchers::match(
+                        ast_matchers::implicitCastExpr().bind("cast"), ASTCtx));
+        ASSERT_THAT(Cast, NotNull());
+        ASSERT_EQ(Cast->getCastKind(), CK_DerivedToBase);
+
+        auto *New = ast_matchers::selectFirst<CXXNewExpr>(
+            "new", ast_matchers::match(ast_matchers::cxxNewExpr().bind("new"),
+                                       ASTCtx));
+        ASSERT_THAT(New, NotNull());
+
+        EXPECT_EQ(Env.getValue(*Cast), Env.getValue(*New));
+      });
+}
+
+TEST(TransferTest, ReinterpretCast) {
+  std::string Code = R"cc(
+    struct S {
+        int I;
+    };
+
+    void target(unsigned char* Bytes) {
+        S& SRef = reinterpret_cast<S&>(Bytes);
+        SRef.I;
+        S* SPtr = reinterpret_cast<S*>(Bytes);
+        SPtr->I;
+        // [[p]]
+    }
+  )cc";
+  runDataflow(Code, [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>>
+                           &Results,
+                       ASTContext &ASTCtx) {
+    ASSERT_THAT(Results.keys(), UnorderedElementsAre("p"));
+    const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+    const ValueDecl *I = findValueDecl(ASTCtx, "I");
+    ASSERT_THAT(I, NotNull());
+
+    // No particular knowledge of I's value is modeled, but for both casts,
+    // the fields of S are modeled.
+
+    {
+      auto &Loc = getLocForDecl<RecordStorageLocation>(ASTCtx, Env, "SRef");
+      std::vector<const ValueDecl *> Children;
+      for (const auto &Entry : Loc.children()) {
+        Children.push_back(Entry.getFirst());
+      }
+
+      EXPECT_THAT(Children, UnorderedElementsAre(I));
+    }
+
+    {
+      auto &Loc = cast<RecordStorageLocation>(
+          getValueForDecl<PointerValue>(ASTCtx, Env, "SPtr").getPointeeLoc());
+      std::vector<const ValueDecl *> Children;
+      for (const auto &Entry : Loc.children()) {
+        Children.push_back(Entry.getFirst());
+      }
+
+      EXPECT_THAT(Children, UnorderedElementsAre(I));
+    }
+  });
+}
+
 TEST(TransferTest, IntegralCast) {
   std::string Code = R"(
     void target(int Foo) {
diff --git a/clang/unittests/Analysis/LifetimeSafetyTest.cpp b/clang/unittests/Analysis/LifetimeSafetyTest.cpp
index 7cd679e184f6c..c8d88b4ea2277 100644
--- a/clang/unittests/Analysis/LifetimeSafetyTest.cpp
+++ b/clang/unittests/Analysis/LifetimeSafetyTest.cpp
@@ -33,7 +33,9 @@ class LifetimeTestRunner {
     )";
     FullCode += Code.str();
 
-    AST = std::make_unique<clang::TestAST>(FullCode);
+    Inputs = TestInputs(FullCode);
+    Inputs.Language = TestLanguage::Lang_CXX20;
+    AST = std::make_unique<clang::TestAST>(Inputs);
     ASTCtx = &AST->context();
 
     // Find the target function using AST matchers.
@@ -51,7 +53,7 @@ class LifetimeTestRunner {
     BuildOptions.AddTemporaryDtors = true;
 
     // Run the main analysis.
-    Analysis = std::make_unique<LifetimeSafetyAnalysis>(*AnalysisCtx);
+    Analysis = std::make_unique<LifetimeSafetyAnalysis>(*AnalysisCtx, nullptr);
     Analysis->run();
 
     AnnotationToPointMap = Analysis->getTestPoints();
@@ -70,6 +72,7 @@ class LifetimeTestRunner {
   }
 
 private:
+  TestInputs Inputs;
   std::unique_ptr<TestAST> AST;
   ASTContext *ASTCtx = nullptr;
   std::unique_ptr<AnalysisDeclContext> AnalysisCtx;
@@ -118,11 +121,13 @@ class LifetimeTestHelper {
     return Analysis.getLoansAtPoint(OID, PP);
   }
 
-  std::optional<LoanSet> getExpiredLoansAtPoint(llvm::StringRef Annotation) {
+  std::optional<llvm::DenseSet<LoanID>>
+  getExpiredLoansAtPoint(llvm::StringRef Annotation) {
     ProgramPoint PP = Runner.getProgramPoint(Annotation);
     if (!PP)
       return std::nullopt;
-    return Analysis.getExpiredLoansAtPoint(PP);
+    auto Expired = Analysis.getExpiredLoansAtPoint(PP);
+    return llvm::DenseSet<LoanID>{Expired.begin(), Expired.end()};
   }
 
 private:
diff --git a/clang/unittests/CodeGen/CodeGenExternalTest.cpp b/clang/unittests/CodeGen/CodeGenExternalTest.cpp
index 00815889d7796..be3be147460f3 100644
--- a/clang/unittests/CodeGen/CodeGenExternalTest.cpp
+++ b/clang/unittests/CodeGen/CodeGenExternalTest.cpp
@@ -172,6 +172,7 @@ static void test_codegen_fns(MyASTConsumer *my) {
   bool mytest_struct_ok = false;
 
   CodeGen::CodeGenModule &CGM = my->Builder->CGM();
+  const ASTContext &Ctx = my->toplevel_decls.front()->getASTContext();
 
   for (auto decl : my->toplevel_decls ) {
     if (FunctionDecl *fd = dyn_cast<FunctionDecl>(decl)) {
@@ -189,9 +190,7 @@ static void test_codegen_fns(MyASTConsumer *my) {
       if (rd->getName() == "mytest_struct") {
         RecordDecl *def = rd->getDefinition();
         ASSERT_TRUE(def != NULL);
-        const clang::Type *clangTy = rd->getCanonicalDecl()->getTypeForDecl();
-        ASSERT_TRUE(clangTy != NULL);
-        QualType qType = clangTy->getCanonicalTypeInternal();
+        CanQualType qType = Ctx.getCanonicalTagType(rd);
 
         // Check convertTypeForMemory
         llvm::Type *llvmTy = CodeGen::convertTypeForMemory(CGM, qType);
diff --git a/clang/unittests/Format/CMakeLists.txt b/clang/unittests/Format/CMakeLists.txt
index edfc8d7a5beaa..c4c7b483ba68e 100644
--- a/clang/unittests/Format/CMakeLists.txt
+++ b/clang/unittests/Format/CMakeLists.txt
@@ -27,6 +27,7 @@ add_distinct_clang_unittest(FormatTests
   MacroExpanderTest.cpp
   MatchFilePathTest.cpp
   NamespaceEndCommentsFixerTest.cpp
+  NumericLiteralInfoTest.cpp
   ObjCPropertyAttributeOrderFixerTest.cpp
   QualifierFixerTest.cpp
   SortImportsTestJS.cpp
diff --git a/clang/unittests/Format/ConfigParseTest.cpp b/clang/unittests/Format/ConfigParseTest.cpp
index 9de3cca71630d..7c993c0f8fd33 100644
--- a/clang/unittests/Format/ConfigParseTest.cpp
+++ b/clang/unittests/Format/ConfigParseTest.cpp
@@ -200,7 +200,6 @@ TEST(ConfigParseTest, ParsesConfigurationBools) {
   CHECK_PARSE_BOOL(RemoveSemicolon);
   CHECK_PARSE_BOOL(SkipMacroDefinitionBody);
   CHECK_PARSE_BOOL(SpacesInSquareBrackets);
-  CHECK_PARSE_BOOL(SpaceInEmptyBlock);
   CHECK_PARSE_BOOL(SpacesInContainerLiterals);
   CHECK_PARSE_BOOL(SpaceAfterCStyleCast);
   CHECK_PARSE_BOOL(SpaceAfterTemplateKeyword);
@@ -688,6 +687,17 @@ TEST(ConfigParseTest, ParsesConfiguration) {
               SpaceBeforeParens,
               FormatStyle::SBPO_ControlStatementsExceptControlMacros);
 
+  Style.SpaceInEmptyBraces = FormatStyle::SIEB_Never;
+  CHECK_PARSE("SpaceInEmptyBraces: Always", SpaceInEmptyBraces,
+              FormatStyle::SIEB_Always);
+  CHECK_PARSE("SpaceInEmptyBraces: Block", SpaceInEmptyBraces,
+              FormatStyle::SIEB_Block);
+  CHECK_PARSE("SpaceInEmptyBraces: Never", SpaceInEmptyBraces,
+              FormatStyle::SIEB_Never);
+  // For backward compatibility:
+  CHECK_PARSE("SpaceInEmptyBlock: true", SpaceInEmptyBraces,
+              FormatStyle::SIEB_Block);
+
   // For backward compatibility:
   Style.SpacesInParens = FormatStyle::SIPO_Never;
   Style.SpacesInParensOptions = {};
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index 96cc650f52a5d..136039d255393 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -4050,6 +4050,10 @@ TEST_F(FormatTest, FormatsBitfields) {
                "  uchar : 8;\n"
                "  uchar other;\n"
                "};");
+  verifyFormat("struct foo {\n"
+               "  uint8_t i_am_a_bit_field_this_long\n"
+               "      : struct_with_constexpr::i_am_a_constexpr_lengthhhhh;\n"
+               "};");
   FormatStyle Style = getLLVMStyle();
   Style.BitFieldColonSpacing = FormatStyle::BFCS_None;
   verifyFormat("struct Bitfields {\n"
@@ -7055,7 +7059,7 @@ TEST_F(FormatTest, PutEmptyBlocksIntoOneLine) {
   verifyFormat("enum E {};");
   verifyFormat("enum E {}");
   FormatStyle Style = getLLVMStyle();
-  Style.SpaceInEmptyBlock = true;
+  Style.SpaceInEmptyBraces = FormatStyle::SIEB_Block;
   verifyFormat("void f() { }", "void f() {}", Style);
   Style.AllowShortBlocksOnASingleLine = FormatStyle::SBS_Empty;
   verifyFormat("{ }", Style);
@@ -7083,7 +7087,7 @@ TEST_F(FormatTest, PutEmptyBlocksIntoOneLine) {
                Style);
 
   Style = getLLVMStyle(FormatStyle::LK_CSharp);
-  Style.SpaceInEmptyBlock = true;
+  Style.SpaceInEmptyBraces = FormatStyle::SIEB_Block;
   verifyFormat("Event += () => { };", Style);
 }
 
@@ -8614,7 +8618,7 @@ TEST_F(FormatTest, BreaksFunctionDeclarations) {
   verifyFormat("extern \"C\" //\n"
                "    void f();");
 
-  FormatStyle Style = getLLVMStyle();
+  auto Style = getLLVMStyle();
   Style.PointerAlignment = FormatStyle::PAS_Left;
   verifyFormat("void aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n"
                "    aaaaaaaaaaaaaaaaaaaaaaaaa* const aaaaaaaaaaaa) {}",
@@ -8622,6 +8626,14 @@ TEST_F(FormatTest, BreaksFunctionDeclarations) {
   verifyFormat("void aaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa*\n"
                "                 aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) {}",
                Style);
+
+  Style = getLLVMStyleWithColumns(45);
+  Style.PenaltyReturnTypeOnItsOwnLine = 400;
+  verifyFormat("template <bool abool, // a comment\n"
+               "          bool anotherbool>\n"
+               "static inline std::pair<size_t, MyCustomType>\n"
+               "myfunc(const char *buf, const char *&err);",
+               Style);
 }
 
 TEST_F(FormatTest, DontBreakBeforeQualifiedOperator) {
@@ -25584,6 +25596,30 @@ TEST_F(FormatTest, SpacesInConditionalStatement) {
   verifyFormat("MYIF( a )\n  return;\nelse\n  return;", Spaces);
 }
 
+TEST_F(FormatTest, SpaceInEmptyBraces) {
+  constexpr StringRef Code("void f() {}\n"
+                           "class Unit {};\n"
+                           "auto a = [] {};\n"
+                           "int x{};");
+  verifyFormat(Code);
+
+  auto Style = getWebKitStyle();
+  EXPECT_EQ(Style.SpaceInEmptyBraces, FormatStyle::SIEB_Always);
+
+  verifyFormat("void f() { }\n"
+               "class Unit { };\n"
+               "auto a = [] { };\n"
+               "int x { };",
+               Code, Style);
+
+  Style.SpaceInEmptyBraces = FormatStyle::SIEB_Block;
+  verifyFormat("void f() { }\n"
+               "class Unit { };\n"
+               "auto a = [] { };\n"
+               "int x {};",
+               Code, Style);
+}
+
 TEST_F(FormatTest, AlternativeOperators) {
   // Test case for ensuring alternate operators are not
   // combined with their right most neighbour.
@@ -25986,6 +26022,10 @@ TEST_F(FormatTest, SkipMacroDefinitionBody) {
                  "  A  a \\\n "
                  " A  a",
                  Style);
+  verifyNoChange("#define MY_MACRO  \\\n"
+                 " /* comment */  \\\n"
+                 "   1",
+                 Style);
 }
 
 TEST_F(FormatTest, VeryLongNamespaceCommentSplit) {
diff --git a/clang/unittests/Format/NumericLiteralInfoTest.cpp b/clang/unittests/Format/NumericLiteralInfoTest.cpp
new file mode 100644
index 0000000000000..a892cfff531e3
--- /dev/null
+++ b/clang/unittests/Format/NumericLiteralInfoTest.cpp
@@ -0,0 +1,71 @@
+//===- unittest/Format/NumericLiteralInfoTest.cpp -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../../lib/Format/NumericLiteralInfo.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace format {
+namespace {
+
+static constexpr auto npos = llvm::StringRef::npos;
+
+class NumericLiteralInfoTest : public testing::Test {
+protected:
+  bool verifyInfo(const NumericLiteralInfo &Info, size_t BaseLetterPos = npos,
+                  size_t DotPos = npos, size_t ExponentLetterPos = npos,
+                  size_t SuffixPos = npos) {
+    return Info.BaseLetterPos == BaseLetterPos && Info.DotPos == DotPos &&
+           Info.ExponentLetterPos == ExponentLetterPos &&
+           Info.SuffixPos == SuffixPos;
+  }
+};
+
+TEST_F(NumericLiteralInfoTest, IntegerLiteral) {
+  // Decimal.
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("90")));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("9L"), npos, npos, npos, 1));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("9'0U"), npos, npos, npos, 3));
+
+  // Octal.
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0")));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("07")));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0z"), npos, npos, npos, 1));
+  // JavaScript.
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0o7"), 1));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0O7_0", '_'), 1));
+
+  // Binary.
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0b1"), 1));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0B1ul"), 1, npos, npos, 3));
+
+  // Hexadecimal.
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0xF"), 1));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0XfZ"), 1, npos, npos, 3));
+}
+
+TEST_F(NumericLiteralInfoTest, FloatingPointLiteral) {
+  // Decimal.
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo(".9"), npos, 0));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("9."), npos, 1));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("9.F"), npos, 1, npos, 2));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("9e9"), npos, npos, 1));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("9E-9f"), npos, npos, 1, 4));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("9.9e+9bf16"), npos, 1, 3, 6));
+
+  // Hexadecimal.
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0X.Fp9"), 1, 2, 4));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0xF.P9"), 1, 3, 4));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0xFp9"), 1, npos, 3));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0xFp+9F128"), 1, npos, 3, 6));
+  EXPECT_TRUE(verifyInfo(NumericLiteralInfo("0xF.Fp-9_Pa"), 1, 3, 5, 8));
+}
+
+} // namespace
+} // namespace format
+} // namespace clang
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 7f99655b1fa49..85ccba38ac8ca 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -618,6 +618,13 @@ TEST_F(TokenAnnotatorTest, UnderstandsStructs) {
   EXPECT_TOKEN(Tokens[19], tok::l_brace, TT_StructLBrace);
   EXPECT_TOKEN(Tokens[20], tok::r_brace, TT_StructRBrace);
 
+  Tokens = annotate("class Outer {\n"
+                    "  struct Inner final : Base {};\n"
+                    "};");
+  ASSERT_EQ(Tokens.size(), 14u) << Tokens;
+  EXPECT_TOKEN(Tokens[5], tok::identifier, TT_Unknown); // Not TT_StartOfName
+  EXPECT_TOKEN(Tokens[6], tok::colon, TT_InheritanceColon);
+
   constexpr StringRef Code("struct EXPORT StructName {};");
 
   Tokens = annotate(Code);
diff --git a/clang/unittests/Index/IndexTests.cpp b/clang/unittests/Index/IndexTests.cpp
index 05ce448237fdf..6df4b577d98a0 100644
--- a/clang/unittests/Index/IndexTests.cpp
+++ b/clang/unittests/Index/IndexTests.cpp
@@ -347,7 +347,7 @@ TEST(IndexTest, Constructors) {
                 WrittenAt(Position(4, 8)))));
 }
 
-TEST(IndexTest, InjecatedNameClass) {
+TEST(IndexTest, InjectedNameClass) {
   std::string Code = R"cpp(
     template <typename T>
     class Foo {
diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt
index 1dda9024075a1..db9f80d9f53fe 100644
--- a/clang/unittests/Interpreter/CMakeLists.txt
+++ b/clang/unittests/Interpreter/CMakeLists.txt
@@ -1,3 +1,34 @@
+if(EMSCRIPTEN)
+set(LLVM_COMPONENTS_TO_LINK
+  ""
+  )
+set(LLVM_LIBS_TO_LINK
+  ""
+  )
+set(CLANG_LIBS_TO_LINK
+  clangInterpreter
+  )
+else()
+set(LLVM_COMPONENTS_TO_LINK
+  ${LLVM_TARGETS_TO_BUILD}
+  Core
+  MC
+  OrcJIT
+  Support
+  TargetParser
+  )
+set(LLVM_LIBS_TO_LINK
+  LLVMTestingSupport
+  )
+set(CLANG_LIBS_TO_LINK
+  clangAST
+  clangBasic
+  clangInterpreter
+  clangFrontend
+  clangSema
+  )
+endif()
+
 add_distinct_clang_unittest(ClangReplInterpreterTests
   IncrementalCompilerBuilderTest.cpp
   IncrementalProcessingTest.cpp
@@ -8,24 +39,38 @@ add_distinct_clang_unittest(ClangReplInterpreterTests
   EXPORT_SYMBOLS
 
   CLANG_LIBS
-  clangAST
-  clangBasic
-  clangInterpreter
-  clangFrontend
-  clangSema
+  ${CLANG_LIBS_TO_LINK}
 
   LINK_LIBS
-  LLVMTestingSupport
+  ${LLVM_LIBS_TO_LINK}
 
   LLVM_COMPONENTS
-  ${LLVM_TARGETS_TO_BUILD}
-  Core
-  MC
-  OrcJIT
-  Support
-  TargetParser
+  ${LLVM_COMPONENTS_TO_LINK}
   )
 
+if(EMSCRIPTEN)
+# Without the above you try to link to LLVMSupport twice, and end
+# up with a duplicate symbol error when creating the main module
+get_target_property(LINKED_LIBS ClangReplInterpreterTests LINK_LIBRARIES)
+list(REMOVE_ITEM LINKED_LIBS LLVMSupport)
+set_target_properties(ClangReplInterpreterTests PROPERTIES LINK_LIBRARIES "${LINKED_LIBS}")
+target_link_options(ClangReplInterpreterTests
+  PUBLIC "SHELL: -s MAIN_MODULE=1"
+  PUBLIC "SHELL: -s ALLOW_MEMORY_GROWTH=1"
+  PUBLIC "SHELL: -s STACK_SIZE=32mb"
+  PUBLIC "SHELL: -s INITIAL_MEMORY=128mb"
+  PUBLIC "SHELL: --emrun"
+  PUBLIC "SHELL: -Wl,--export=__clang_Interpreter_SetValueWithAlloc"
+  PUBLIC "SHELL: -Wl,--export=__clang_Interpreter_SetValueNoAlloc"
+  PUBLIC "SHELL: -Wl,--export=_ZnwmPv26__clang_Interpreter_NewTag"
+  PUBLIC "SHELL: -Wl,--export=_Z9getGlobalv"
+  PUBLIC "SHELL: -Wl,--export=_Z9setGlobali"
+)
+set_target_properties(ClangReplInterpreterTests PROPERTIES
+  SUFFIX ".html"
+)
+endif()
+
 # Exceptions on Windows are not yet supported.
 if(NOT WIN32)
   add_subdirectory(ExceptionTests)
diff --git a/clang/unittests/Interpreter/CodeCompletionTest.cpp b/clang/unittests/Interpreter/CodeCompletionTest.cpp
index 23cfc469695d2..ceb683497ac74 100644
--- a/clang/unittests/Interpreter/CodeCompletionTest.cpp
+++ b/clang/unittests/Interpreter/CodeCompletionTest.cpp
@@ -29,8 +29,14 @@ class CodeCompletionTest : public InterpreterTestBase {
   std::unique_ptr<clang::Interpreter> Interp;
 
   void SetUp() override {
+// FIXME : WebAssembly doesn't currently support Jit (see
+// https: // github.com/llvm/llvm-project/pull/150977#discussion_r2237521095).
+// so this check of HostSupportsJIT has been skipped
+// over until support is added, and HostSupportsJIT can return true.
+#ifndef __EMSCRIPTEN__
     if (!HostSupportsJIT())
       GTEST_SKIP();
+#endif
     std::unique_ptr<CompilerInstance> CI = cantFail(CB.CreateCpp());
     this->Interp = cantFail(clang::Interpreter::create(std::move(CI)));
   }
diff --git a/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp b/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp
index c4a40071f55cf..7b4633bfc9e7a 100644
--- a/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp
+++ b/clang/unittests/Interpreter/IncrementalCompilerBuilderTest.cpp
@@ -37,6 +37,14 @@ TEST(IncrementalCompilerBuilder, SetCompilerArgs) {
 }
 
 TEST(IncrementalCompilerBuilder, SetTargetTriple) {
+// FIXME : This test doesn't current work for Emscripten builds.
+// It should be possible to make it work.For details on how it fails and
+// the current progress to enable this test see
+// the following Github issue https: //
+// github.com/llvm/llvm-project/issues/153461
+#ifdef __EMSCRIPTEN__
+  GTEST_SKIP() << "Test fails for Emscipten builds";
+#endif
   auto CB = clang::IncrementalCompilerBuilder();
   CB.SetTargetTriple("armv6-none-eabi");
   auto CI = cantFail(CB.CreateCpp());
diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp
index 1c27cfb2c48fa..f50f6e320776d 100644
--- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp
+++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp
@@ -75,9 +75,14 @@ struct OutOfProcInterpreter : public Interpreter {
 };
 
 TEST_F(InterpreterExtensionsTest, FindRuntimeInterface) {
+// FIXME : WebAssembly doesn't currently support Jit (see
+// https: // github.com/llvm/llvm-project/pull/150977#discussion_r2237521095).
+// so this check of HostSupportsJIT has been skipped
+// over until support is added, and HostSupportsJIT can return true.
+#ifndef __EMSCRIPTEN__
   if (!HostSupportsJIT())
     GTEST_SKIP();
-
+#endif
   clang::IncrementalCompilerBuilder CB;
   llvm::Error ErrOut = llvm::Error::success();
   auto CI = cantFail(CB.CreateCpp());
diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp
index 768058b954d49..9ff9092524d21 100644
--- a/clang/unittests/Interpreter/InterpreterTest.cpp
+++ b/clang/unittests/Interpreter/InterpreterTest.cpp
@@ -147,6 +147,14 @@ TEST_F(InterpreterTest, DeclsAndStatements) {
 }
 
 TEST_F(InterpreterTest, UndoCommand) {
+// FIXME : This test doesn't current work for Emscripten builds.
+// It should be possible to make it work.For details on how it fails and
+// the current progress to enable this test see
+// the following Github issue https: //
+// github.com/llvm/llvm-project/issues/153461
+#ifdef __EMSCRIPTEN__
+  GTEST_SKIP() << "Test fails for Emscipten builds";
+#endif
   Args ExtraArgs = {"-Xclang", "-diagnostic-log-file", "-Xclang", "-"};
 
   // Create the diagnostic engine with unowned consumer.
diff --git a/clang/unittests/Interpreter/InterpreterTestFixture.h b/clang/unittests/Interpreter/InterpreterTestFixture.h
index 113599ff98894..b088fa4a5f896 100644
--- a/clang/unittests/Interpreter/InterpreterTestFixture.h
+++ b/clang/unittests/Interpreter/InterpreterTestFixture.h
@@ -38,8 +38,14 @@ class InterpreterTestBase : public ::testing::Test {
   }
 
   void SetUp() override {
+// FIXME : WebAssembly doesn't currently support Jit (see
+// https: // github.com/llvm/llvm-project/pull/150977#discussion_r2237521095).
+// so this check of HostSupportsJIT has been skipped
+// over until support is added, and HostSupportsJIT can return true.
+#ifndef __EMSCRIPTEN__
     if (!HostSupportsJIT())
       GTEST_SKIP();
+#endif
   }
 
   void TearDown() override {}
diff --git a/clang/unittests/Lex/CMakeLists.txt b/clang/unittests/Lex/CMakeLists.txt
index 96ca6dda9cd85..fa5e58f5a8932 100644
--- a/clang/unittests/Lex/CMakeLists.txt
+++ b/clang/unittests/Lex/CMakeLists.txt
@@ -5,6 +5,7 @@ add_clang_unittest(LexTests
   LexerTest.cpp
   LexHLSLRootSignatureTest.cpp
   ModuleDeclStateTest.cpp
+  NoTrivialPPDirectiveTracerTest.cpp
   PPCallbacksTest.cpp
   PPConditionalDirectiveRecordTest.cpp
   PPDependencyDirectivesTest.cpp
diff --git a/clang/unittests/Lex/LexerTest.cpp b/clang/unittests/Lex/LexerTest.cpp
index 56d73cec1363f..c51cd0d2bfdaa 100644
--- a/clang/unittests/Lex/LexerTest.cpp
+++ b/clang/unittests/Lex/LexerTest.cpp
@@ -795,7 +795,7 @@ TEST_F(LexerTest, CheckFirstPPToken) {
     EXPECT_FALSE(Lexer::getRawToken(PP->getMainFileFirstPPTokenLoc(), Tok,
                                     PP->getSourceManager(), PP->getLangOpts(),
                                     /*IgnoreWhiteSpace=*/false));
-    EXPECT_TRUE(Tok.isFirstPPToken());
+    EXPECT_TRUE(PP->getMainFileFirstPPTokenLoc() == Tok.getLocation());
     EXPECT_TRUE(Tok.is(tok::hash));
   }
 
@@ -811,7 +811,7 @@ TEST_F(LexerTest, CheckFirstPPToken) {
     EXPECT_FALSE(Lexer::getRawToken(PP->getMainFileFirstPPTokenLoc(), Tok,
                                     PP->getSourceManager(), PP->getLangOpts(),
                                     /*IgnoreWhiteSpace=*/false));
-    EXPECT_TRUE(Tok.isFirstPPToken());
+    EXPECT_TRUE(PP->getMainFileFirstPPTokenLoc() == Tok.getLocation());
     EXPECT_TRUE(Tok.is(tok::raw_identifier));
     EXPECT_TRUE(Tok.getRawIdentifier() == "FOO");
   }
diff --git a/clang/unittests/Lex/ModuleDeclStateTest.cpp b/clang/unittests/Lex/ModuleDeclStateTest.cpp
index adc6cf1d2e598..ac2ddfaf52cd0 100644
--- a/clang/unittests/Lex/ModuleDeclStateTest.cpp
+++ b/clang/unittests/Lex/ModuleDeclStateTest.cpp
@@ -61,14 +61,15 @@ class ModuleDeclStateTest : public ::testing::Test {
     Target = TargetInfo::CreateTargetInfo(Diags, *TargetOpts);
   }
 
-  std::unique_ptr<Preprocessor>
-  getPreprocessor(const char *source, Language Lang) {
+  std::unique_ptr<Preprocessor> getPreprocessor(const char *source,
+                                                Language Lang) {
     std::unique_ptr<llvm::MemoryBuffer> Buf =
         llvm::MemoryBuffer::getMemBuffer(source);
     SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
 
     std::vector<std::string> Includes;
-    LangOptions::setLangDefaults(LangOpts, Lang, Target->getTriple(), Includes, LangStandard::lang_cxx20);
+    LangOptions::setLangDefaults(LangOpts, Lang, Target->getTriple(), Includes,
+                                 LangStandard::lang_cxx20);
     LangOpts.CPlusPlusModules = true;
     if (Lang != Language::CXX) {
       LangOpts.Modules = true;
@@ -112,12 +113,11 @@ export module foo;
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
 
   std::initializer_list<bool> ImportKinds = {};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)0);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)0);
   EXPECT_TRUE(PP->isInNamedModule());
   EXPECT_TRUE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
@@ -131,12 +131,11 @@ module foo;
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
 
   std::initializer_list<bool> ImportKinds = {};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)0);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)0);
   EXPECT_TRUE(PP->isInNamedModule());
   EXPECT_FALSE(PP->isInNamedInterfaceUnit());
   EXPECT_TRUE(PP->isInImplementationUnit());
@@ -150,12 +149,11 @@ module foo:part;
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
 
   std::initializer_list<bool> ImportKinds = {};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)0);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)0);
   EXPECT_TRUE(PP->isInNamedModule());
   EXPECT_FALSE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
@@ -169,12 +167,11 @@ export module foo:part;
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
 
   std::initializer_list<bool> ImportKinds = {};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)0);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)0);
   EXPECT_TRUE(PP->isInNamedModule());
   EXPECT_TRUE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
@@ -188,12 +185,11 @@ export module foo.dot:part.dot;
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
 
   std::initializer_list<bool> ImportKinds = {};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)0);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)0);
   EXPECT_TRUE(PP->isInNamedModule());
   EXPECT_TRUE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
@@ -207,12 +203,11 @@ TEST_F(ModuleDeclStateTest, NotModule) {
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
 
   std::initializer_list<bool> ImportKinds = {};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)0);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)0);
   EXPECT_FALSE(PP->isInNamedModule());
   EXPECT_FALSE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
@@ -233,12 +228,11 @@ import :another;
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
 
   std::initializer_list<bool> ImportKinds = {true, true};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)2);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)2);
   EXPECT_TRUE(PP->isInNamedModule());
   EXPECT_TRUE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
@@ -260,12 +254,11 @@ import :another;
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
 
   std::initializer_list<bool> ImportKinds = {true, true};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)2);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)2);
   EXPECT_TRUE(PP->isInNamedModule());
   EXPECT_TRUE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
@@ -286,12 +279,11 @@ import :another;
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
 
   std::initializer_list<bool> ImportKinds = {true};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)1);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)1);
   EXPECT_FALSE(PP->isInNamedModule());
   EXPECT_FALSE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
@@ -304,12 +296,11 @@ TEST_F(ModuleDeclStateTest, ImportAClangNamedModule) {
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::ObjCXX);
 
   std::initializer_list<bool> ImportKinds = {false};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)1);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)1);
   EXPECT_FALSE(PP->isInNamedModule());
   EXPECT_FALSE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
@@ -326,12 +317,11 @@ import M2;
   std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::ObjCXX);
 
   std::initializer_list<bool> ImportKinds = {false, true, false, true};
-  preprocess(*PP,
-             std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds));
-
-  auto *Callback =
-      static_cast<CheckNamedModuleImportingCB *>(PP->getPPCallbacks());
-  EXPECT_EQ(Callback->importNamedModuleNum(), (size_t)4);
+  auto Callback =
+      std::make_unique<CheckNamedModuleImportingCB>(*PP, ImportKinds);
+  CheckNamedModuleImportingCB *CallbackPtr = Callback.get();
+  preprocess(*PP, std::move(Callback));
+  EXPECT_EQ(CallbackPtr->importNamedModuleNum(), (size_t)4);
   EXPECT_FALSE(PP->isInNamedModule());
   EXPECT_FALSE(PP->isInNamedInterfaceUnit());
   EXPECT_FALSE(PP->isInImplementationUnit());
diff --git a/clang/unittests/Lex/NoTrivialPPDirectiveTracerTest.cpp b/clang/unittests/Lex/NoTrivialPPDirectiveTracerTest.cpp
new file mode 100644
index 0000000000000..d79c1428e55bb
--- /dev/null
+++ b/clang/unittests/Lex/NoTrivialPPDirectiveTracerTest.cpp
@@ -0,0 +1,182 @@
+//===- unittests/Lex/NoTrivialPPDirectiveTracerTest.cpp -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticOptions.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/LangOptions.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/ModuleLoader.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
+#include "gtest/gtest.h"
+#include <cstddef>
+#include <initializer_list>
+
+using namespace clang;
+
+namespace {
+class NoTrivialPPDirectiveTracerTest : public ::testing::Test {
+protected:
+  NoTrivialPPDirectiveTracerTest()
+      : VFS(llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>()),
+        FileMgr(FileMgrOpts, VFS),
+        Diags(DiagnosticIDs::create(), DiagOpts, new IgnoringDiagConsumer()),
+        SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions) {
+    TargetOpts->Triple = "x86_64-unknown-linux-gnu";
+    Target = TargetInfo::CreateTargetInfo(Diags, *TargetOpts);
+  }
+
+  void addFile(const char *source, StringRef Filename) {
+    VFS->addFile(Filename, 0, llvm::MemoryBuffer::getMemBuffer(source),
+                 /*User=*/std::nullopt,
+                 /*Group=*/std::nullopt,
+                 llvm::sys::fs::file_type::regular_file);
+  }
+
+  std::unique_ptr<Preprocessor> getPreprocessor(const char *source,
+                                                Language Lang) {
+    std::unique_ptr<llvm::MemoryBuffer> Buf =
+        llvm::MemoryBuffer::getMemBuffer(source);
+    SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
+
+    std::vector<std::string> Includes;
+    LangOptions::setLangDefaults(LangOpts, Lang, Target->getTriple(), Includes,
+                                 LangStandard::lang_cxx20);
+    LangOpts.CPlusPlusModules = true;
+    if (Lang != Language::CXX) {
+      LangOpts.Modules = true;
+      LangOpts.ImplicitModules = true;
+    }
+
+    HeaderInfo.emplace(HSOpts, SourceMgr, Diags, LangOpts, Target.get());
+
+    auto DE = FileMgr.getOptionalDirectoryRef(".");
+    assert(DE);
+    auto DL = DirectoryLookup(*DE, SrcMgr::C_User, /*isFramework=*/false);
+    HeaderInfo->AddSearchPath(DL, /*isAngled=*/false);
+
+    return std::make_unique<Preprocessor>(PPOpts, Diags, LangOpts, SourceMgr,
+                                          *HeaderInfo, ModLoader,
+                                          /*IILookup=*/nullptr,
+                                          /*OwnsHeaderSearch=*/false);
+  }
+
+  IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> VFS;
+  FileSystemOptions FileMgrOpts;
+  FileManager FileMgr;
+  DiagnosticOptions DiagOpts;
+  DiagnosticsEngine Diags;
+  SourceManager SourceMgr;
+  std::shared_ptr<TargetOptions> TargetOpts;
+  IntrusiveRefCntPtr<TargetInfo> Target;
+  LangOptions LangOpts;
+  TrivialModuleLoader ModLoader;
+  HeaderSearchOptions HSOpts;
+  std::optional<HeaderSearch> HeaderInfo;
+  PreprocessorOptions PPOpts;
+};
+
+TEST_F(NoTrivialPPDirectiveTracerTest, TrivialDirective) {
+  const char *source = R"(
+    #line 7
+    # 1 __FILE__ 1 3
+    #ident "$Header:$"
+    #pragma comment(lib, "msvcrt.lib")
+    #pragma mark LLVM's world
+    #pragma detect_mismatch("test", "1")
+    #pragma clang __debug dump Test
+    #pragma message "test"
+    #pragma GCC warning "Foo"
+    #pragma GCC error "Foo"
+    #pragma gcc diagnostic push
+    #pragma gcc diagnostic pop
+    #pragma GCC diagnostic ignored "-Wframe-larger-than"
+    #pragma OPENCL EXTENSION __cl_clang_variadic_functions : enable
+    #pragma warning(push)
+    #pragma warning(pop)
+    #pragma execution_character_set(push, "UTF-8")
+    #pragma execution_character_set(pop)
+    #pragma clang assume_nonnull begin
+    #pragma clang assume_nonnull end
+    int foo;
+  )";
+  std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
+  PP->Initialize(*Target);
+  PP->EnterMainSourceFile();
+  Token Tok;
+  PP->Lex(Tok);
+  EXPECT_FALSE(PP->hasSeenNoTrivialPPDirective());
+}
+
+TEST_F(NoTrivialPPDirectiveTracerTest, IncludeDirective) {
+  const char *source = R"(
+    #include "header.h"
+    int foo;
+  )";
+  const char *header = R"(
+    #ifndef HEADER_H
+    #define HEADER_H
+    #endif // HEADER_H
+  )";
+  std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
+  addFile(header, "header.h");
+  PP->Initialize(*Target);
+  PP->EnterMainSourceFile();
+  Token Tok;
+  PP->Lex(Tok);
+  EXPECT_TRUE(PP->hasSeenNoTrivialPPDirective());
+}
+
+TEST_F(NoTrivialPPDirectiveTracerTest, DefineDirective) {
+  const char *source = R"(
+    #define FOO
+    int foo;
+  )";
+  std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
+  PP->Initialize(*Target);
+  PP->EnterMainSourceFile();
+  Token Tok;
+  PP->Lex(Tok);
+  EXPECT_TRUE(PP->hasSeenNoTrivialPPDirective());
+}
+
+TEST_F(NoTrivialPPDirectiveTracerTest, UnDefineDirective) {
+  const char *source = R"(
+    #undef FOO
+    int foo;
+  )";
+  std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
+  PP->Initialize(*Target);
+  PP->setPredefines("#define FOO");
+  PP->EnterMainSourceFile();
+  Token Tok;
+  PP->Lex(Tok);
+  EXPECT_TRUE(PP->hasSeenNoTrivialPPDirective());
+}
+
+TEST_F(NoTrivialPPDirectiveTracerTest, IfDefinedDirective) {
+  const char *source = R"(
+    #if defined(FOO)
+    #endif
+    int foo;
+  )";
+  std::unique_ptr<Preprocessor> PP = getPreprocessor(source, Language::CXX);
+  PP->Initialize(*Target);
+  PP->setPredefines("#define FOO");
+  PP->EnterMainSourceFile();
+  Token Tok;
+  PP->Lex(Tok);
+  EXPECT_TRUE(PP->hasSeenNoTrivialPPDirective());
+}
+
+} // namespace
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index 44f6b0469f38e..44c0978a243bc 100644
--- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -180,7 +180,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
   // First Descriptor Table with 4 elements
   RootElement Elem = Elements[0].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::CBuffer);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::CBuffer);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Reg.ViewType,
             RegisterType::BReg);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Reg.Number, 0u);
@@ -193,7 +193,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
 
   Elem = Elements[1].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::SRV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::SRV);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Reg.ViewType,
             RegisterType::TReg);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Reg.Number, 42u);
@@ -205,7 +205,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
 
   Elem = Elements[2].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::Sampler);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Reg.ViewType,
             RegisterType::SReg);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Reg.Number, 987u);
@@ -218,7 +218,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseDTClausesTest) {
 
   Elem = Elements[3].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::UAV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::UAV);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Reg.ViewType,
             RegisterType::UReg);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Reg.Number, 4294967294u);
@@ -445,7 +445,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidSamplerFlagsTest) {
   auto Elements = Parser.getElements();
   RootElement Elem = Elements[0].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::Sampler);
   auto ValidSamplerFlags =
       llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile;
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidSamplerFlags);
@@ -591,7 +591,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
 
   RootElement Elem = Elements[0].getElement();
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::CBuffer);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::CBuffer);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 0u);
@@ -602,7 +602,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
 
   Elem = Elements[1].getElement();
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::SRV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::SRV);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::TReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 42u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 4u);
@@ -616,7 +616,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
 
   Elem = Elements[2].getElement();
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::UAV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::UAV);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::UReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 34893247u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 0u);
@@ -628,7 +628,7 @@ TEST_F(ParseHLSLRootSignatureTest, ValidParseRootDescriptorsTest) {
             RootDescriptorFlags::DataVolatile);
 
   Elem = Elements[3].getElement();
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::CBuffer);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::CBuffer);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.ViewType, RegisterType::BReg);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Reg.Number, 0u);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Space, 0u);
@@ -696,17 +696,17 @@ TEST_F(ParseHLSLRootSignatureTest, ValidVersion10Test) {
   auto DefRootDescriptorFlag = llvm::dxbc::RootDescriptorFlags::DataVolatile;
   RootElement Elem = Elements[0].getElement();
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::CBuffer);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::CBuffer);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, DefRootDescriptorFlag);
 
   Elem = Elements[1].getElement();
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::SRV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::SRV);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, DefRootDescriptorFlag);
 
   Elem = Elements[2].getElement();
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::UAV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::UAV);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags, DefRootDescriptorFlag);
 
   auto ValidNonSamplerFlags =
@@ -714,22 +714,22 @@ TEST_F(ParseHLSLRootSignatureTest, ValidVersion10Test) {
       llvm::dxbc::DescriptorRangeFlags::DataVolatile;
   Elem = Elements[3].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::CBuffer);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::CBuffer);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidNonSamplerFlags);
 
   Elem = Elements[4].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::SRV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::SRV);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidNonSamplerFlags);
 
   Elem = Elements[5].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::UAV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::UAV);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags, ValidNonSamplerFlags);
 
   Elem = Elements[6].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::Sampler);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
             llvm::dxbc::DescriptorRangeFlags::DescriptorsVolatile);
 
@@ -767,43 +767,43 @@ TEST_F(ParseHLSLRootSignatureTest, ValidVersion11Test) {
   auto Elements = Parser.getElements();
   RootElement Elem = Elements[0].getElement();
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::CBuffer);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::CBuffer);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
             llvm::dxbc::RootDescriptorFlags::DataStaticWhileSetAtExecute);
 
   Elem = Elements[1].getElement();
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::SRV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::SRV);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
             llvm::dxbc::RootDescriptorFlags::DataStaticWhileSetAtExecute);
 
   Elem = Elements[2].getElement();
   ASSERT_TRUE(std::holds_alternative<RootDescriptor>(Elem));
-  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, DescriptorType::UAV);
+  ASSERT_EQ(std::get<RootDescriptor>(Elem).Type, ResourceClass::UAV);
   ASSERT_EQ(std::get<RootDescriptor>(Elem).Flags,
             llvm::dxbc::RootDescriptorFlags::DataVolatile);
 
   Elem = Elements[3].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::CBuffer);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::CBuffer);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
             llvm::dxbc::DescriptorRangeFlags::DataStaticWhileSetAtExecute);
 
   Elem = Elements[4].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::SRV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::SRV);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
             llvm::dxbc::DescriptorRangeFlags::DataStaticWhileSetAtExecute);
 
   Elem = Elements[5].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::UAV);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::UAV);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
             llvm::dxbc::DescriptorRangeFlags::DataVolatile);
 
   Elem = Elements[6].getElement();
   ASSERT_TRUE(std::holds_alternative<DescriptorTableClause>(Elem));
-  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ClauseType::Sampler);
+  ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Type, ResourceClass::Sampler);
   ASSERT_EQ(std::get<DescriptorTableClause>(Elem).Flags,
             llvm::dxbc::DescriptorRangeFlags::None);
 
diff --git a/clang/unittests/Sema/HeuristicResolverTest.cpp b/clang/unittests/Sema/HeuristicResolverTest.cpp
index ee434f7a1d43a..7df25e01e66d4 100644
--- a/clang/unittests/Sema/HeuristicResolverTest.cpp
+++ b/clang/unittests/Sema/HeuristicResolverTest.cpp
@@ -29,7 +29,7 @@ MATCHER_P2(matchAdapter, MatcherForElement, MatchFunction, "matchAdapter") {
 
 template <typename InputNode>
 using ResolveFnT = std::function<std::vector<const NamedDecl *>(
-    const HeuristicResolver *, const InputNode *)>;
+    const HeuristicResolver *, InputNode)>;
 
 // Test heuristic resolution on `Code` using the resolution procedure
 // `ResolveFn`, which takes a `HeuristicResolver` and an input AST node of type
@@ -37,8 +37,9 @@ using ResolveFnT = std::function<std::vector<const NamedDecl *>(
 // `InputMatcher` should be an AST matcher that matches a single node to pass as
 // input to `ResolveFn`, bound to the ID "input". `OutputMatchers` should be AST
 // matchers that each match a single node, bound to the ID "output".
-template <typename InputNode, typename InputMatcher, typename... OutputMatchers>
-void expectResolution(llvm::StringRef Code, ResolveFnT<InputNode> ResolveFn,
+template <typename InputNode, typename ParamT, typename InputMatcher,
+          typename... OutputMatchers>
+void expectResolution(llvm::StringRef Code, ResolveFnT<ParamT> ResolveFn,
                       const InputMatcher &IM, const OutputMatchers &...OMS) {
   auto TU = tooling::buildASTFromCodeWithArgs(Code, {"-std=c++20"});
   auto &Ctx = TU->getASTContext();
@@ -59,7 +60,11 @@ void expectResolution(llvm::StringRef Code, ResolveFnT<InputNode> ResolveFn,
   };
 
   HeuristicResolver H(Ctx);
-  auto Results = ResolveFn(&H, Input);
+  std::vector<const NamedDecl *> Results;
+  if constexpr (std::is_pointer_v<ParamT>)
+    Results = ResolveFn(&H, Input);
+  else
+    Results = ResolveFn(&H, *Input);
   EXPECT_THAT(Results, ElementsAre(matchAdapter(OMS, OutputNodeMatches)...));
 }
 
@@ -71,8 +76,8 @@ void expectResolution(llvm::StringRef Code,
                           HeuristicResolver::*ResolveFn)(const InputNode *)
                           const,
                       const InputMatcher &IM, const OutputMatchers &...OMS) {
-  expectResolution(Code, ResolveFnT<InputNode>(std::mem_fn(ResolveFn)), IM,
-                   OMS...);
+  expectResolution<InputNode>(
+      Code, ResolveFnT<const InputNode *>(std::mem_fn(ResolveFn)), IM, OMS...);
 }
 
 TEST(HeuristicResolver, MemberExpr) {
@@ -643,15 +648,16 @@ TEST(HeuristicResolver, NestedNameSpecifier) {
   // expected by expectResolution() (returning a vector of decls).
   ResolveFnT<NestedNameSpecifier> ResolveFn =
       [](const HeuristicResolver *H,
-         const NestedNameSpecifier *NNS) -> std::vector<const NamedDecl *> {
+         NestedNameSpecifier NNS) -> std::vector<const NamedDecl *> {
     return {H->resolveNestedNameSpecifierToType(NNS)->getAsCXXRecordDecl()};
   };
-  expectResolution(Code, ResolveFn,
-                   nestedNameSpecifier(hasPrefix(specifiesType(hasDeclaration(
-                                           classTemplateDecl(hasName("A"))))))
-                       .bind("input"),
-                   classTemplateDecl(has(cxxRecordDecl(
-                       has(cxxRecordDecl(hasName("B")).bind("output"))))));
+  expectResolution<NestedNameSpecifier>(
+      Code, ResolveFn,
+      nestedNameSpecifier(hasPrefix(specifiesType(
+                              hasDeclaration(classTemplateDecl(hasName("A"))))))
+          .bind("input"),
+      classTemplateDecl(
+          has(cxxRecordDecl(has(cxxRecordDecl(hasName("B")).bind("output"))))));
 }
 
 TEST(HeuristicResolver, TemplateSpecializationType) {
diff --git a/clang/unittests/StaticAnalyzer/ExprEngineVisitTest.cpp b/clang/unittests/StaticAnalyzer/ExprEngineVisitTest.cpp
index 12be2289c3174..ab4b8c79fcd1a 100644
--- a/clang/unittests/StaticAnalyzer/ExprEngineVisitTest.cpp
+++ b/clang/unittests/StaticAnalyzer/ExprEngineVisitTest.cpp
@@ -55,11 +55,13 @@ class MemAccessChecker : public Checker<check::Location, check::Bind> {
                         ", Stmt = " + S->getStmtClassName());
   }
 
-  void checkBind(SVal Loc, SVal Val, const Stmt *S, CheckerContext &C) const {
+  void checkBind(SVal Loc, SVal Val, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const {
     emitErrorReport(C, Bug,
                     "checkBind: Loc = " + dumpToString(Loc) +
                         ", Val = " + dumpToString(Val) +
-                        ", Stmt = " + S->getStmtClassName());
+                        ", Stmt = " + S->getStmtClassName() +
+                        ", AtDeclInit = " + (AtDeclInit ? "true" : "false"));
   }
 
 private:
@@ -140,7 +142,7 @@ TEST(ExprEngineVisitTest, checkLocationAndBind) {
                        "Stmt = ImplicitCastExpr";
   std::string BindMsg =
       "checkBind: Loc = &MyClassWrite, Val = lazyCompoundVal{0x0,MyClassRead}, "
-      "Stmt = CXXOperatorCallExpr";
+      "Stmt = CXXOperatorCallExpr, AtDeclInit = false";
   std::size_t LocPos = Diags.find(LocMsg);
   std::size_t BindPos = Diags.find(BindMsg);
   EXPECT_NE(LocPos, std::string::npos);
@@ -150,4 +152,20 @@ TEST(ExprEngineVisitTest, checkLocationAndBind) {
   EXPECT_TRUE(LocPos > BindPos);
 }
 
+TEST(ExprEngineVisitTest, checkLocationAndBindInitialization) {
+  std::string Diags;
+  EXPECT_TRUE(runCheckerOnCode<addMemAccessChecker>(R"(
+    class MyClass{
+    public:
+      int Value;
+    };
+    void top(MyClass param) {
+      MyClass MyClassWrite = param;
+    }
+  )",
+                                                    Diags));
+
+  EXPECT_TRUE(StringRef(Diags).contains("AtDeclInit = true"));
+}
+
 } // namespace
diff --git a/clang/unittests/StaticAnalyzer/SValTest.cpp b/clang/unittests/StaticAnalyzer/SValTest.cpp
index 58e9a8da0e99d..1f4a18b6b8294 100644
--- a/clang/unittests/StaticAnalyzer/SValTest.cpp
+++ b/clang/unittests/StaticAnalyzer/SValTest.cpp
@@ -61,7 +61,8 @@ using SVals = llvm::StringMap<SVal>;
 /// can test whatever we gathered.
 class SValCollector : public Checker<check::Bind, check::EndAnalysis> {
 public:
-  void checkBind(SVal Loc, SVal Val, const Stmt *S, CheckerContext &C) const {
+  void checkBind(SVal Loc, SVal Val, const Stmt *S, bool AtDeclInit,
+                 CheckerContext &C) const {
     // Skip instantly if we finished testing.
     // Also, we care only for binds happening in variable initializations.
     if (Tested || !isa<DeclStmt>(S))
@@ -301,13 +302,13 @@ void foo(int x) {
   ASSERT_FALSE(B.getType(Context).isNull());
   const auto *BRecordType = dyn_cast<RecordType>(B.getType(Context));
   ASSERT_NE(BRecordType, nullptr);
-  EXPECT_EQ("TestStruct", BRecordType->getDecl()->getName());
+  EXPECT_EQ("TestStruct", BRecordType->getOriginalDecl()->getName());
 
   SVal C = getByName("c");
   ASSERT_FALSE(C.getType(Context).isNull());
   const auto *CRecordType = dyn_cast<RecordType>(C.getType(Context));
   ASSERT_NE(CRecordType, nullptr);
-  EXPECT_EQ("TestUnion", CRecordType->getDecl()->getName());
+  EXPECT_EQ("TestUnion", CRecordType->getOriginalDecl()->getName());
 
   auto D = getByName("d").getAs<nonloc::CompoundVal>();
   ASSERT_TRUE(D.has_value());
@@ -319,12 +320,9 @@ void foo(int x) {
   ASSERT_TRUE(LD.has_value());
   auto LDT = LD->getType(Context);
   ASSERT_FALSE(LDT.isNull());
-  const auto *DElaboratedType = dyn_cast<ElaboratedType>(LDT);
-  ASSERT_NE(DElaboratedType, nullptr);
-  const auto *DRecordType =
-      dyn_cast<RecordType>(DElaboratedType->getNamedType());
+  const auto *DRecordType = dyn_cast<RecordType>(LDT);
   ASSERT_NE(DRecordType, nullptr);
-  EXPECT_EQ("TestStruct", DRecordType->getDecl()->getName());
+  EXPECT_EQ("TestStruct", DRecordType->getOriginalDecl()->getName());
 }
 
 SVAL_TEST(GetStringType, R"(
@@ -353,7 +351,7 @@ void TestClass::foo() {
   ASSERT_NE(APtrTy, nullptr);
   const auto *ARecordType = dyn_cast<RecordType>(APtrTy->getPointeeType());
   ASSERT_NE(ARecordType, nullptr);
-  EXPECT_EQ("TestClass", ARecordType->getDecl()->getName());
+  EXPECT_EQ("TestClass", ARecordType->getOriginalDecl()->getName());
 }
 
 SVAL_TEST(GetFunctionPtrType, R"(
diff --git a/clang/unittests/Tooling/LookupTest.cpp b/clang/unittests/Tooling/LookupTest.cpp
index acd1714a26e07..4c49ebe47acb9 100644
--- a/clang/unittests/Tooling/LookupTest.cpp
+++ b/clang/unittests/Tooling/LookupTest.cpp
@@ -193,16 +193,16 @@ TEST(LookupTest, replaceNestedClassName) {
   auto replaceTypeLoc = [&](const NamedDecl *ND, SourceLocation Loc,
                             StringRef ReplacementString) {
     return tooling::replaceNestedName(
-        nullptr, Loc, Visitor.DeclStack.back()->getDeclContext(), ND,
-        ReplacementString);
+        /*Use=*/std::nullopt, Loc, Visitor.DeclStack.back()->getDeclContext(),
+        ND, ReplacementString);
   };
 
   Visitor.OnRecordTypeLoc = [&](RecordTypeLoc Type) {
     // Filter Types by name since there are other `RecordTypeLoc` in the test
     // file.
-    if (Type.getDecl()->getQualifiedNameAsString() == "a::b::Foo") {
-      EXPECT_EQ("x::Bar", replaceTypeLoc(Type.getDecl(), Type.getBeginLoc(),
-                                         "::a::x::Bar"));
+    if (Type.getOriginalDecl()->getQualifiedNameAsString() == "a::b::Foo") {
+      EXPECT_EQ("x::Bar", replaceTypeLoc(Type.getOriginalDecl(),
+                                         Type.getBeginLoc(), "::a::x::Bar"));
     }
   };
   Visitor.runOver("namespace a { namespace b {\n"
@@ -214,7 +214,7 @@ TEST(LookupTest, replaceNestedClassName) {
     // Filter Types by name since there are other `RecordTypeLoc` in the test
     // file.
     // `a::b::Foo` in using shadow decl is not `TypeLoc`.
-    auto *TD = Type.getFoundDecl()->getTargetDecl();
+    auto *TD = Type.getDecl()->getTargetDecl();
     if (TD->getQualifiedNameAsString() == "a::b::Foo") {
       EXPECT_EQ("Bar", replaceTypeLoc(TD, Type.getBeginLoc(), "::a::x::Bar"));
     }
@@ -227,9 +227,9 @@ TEST(LookupTest, replaceNestedClassName) {
   // `x::y::Foo` in c.cc [1], it should not make "Foo" at [0] ambiguous because
   // it's not visible at [0].
   Visitor.OnRecordTypeLoc = [&](RecordTypeLoc Type) {
-    if (Type.getDecl()->getQualifiedNameAsString() == "x::y::Old") {
-      EXPECT_EQ("Foo",
-                replaceTypeLoc(Type.getDecl(), Type.getBeginLoc(), "::x::Foo"));
+    if (Type.getOriginalDecl()->getQualifiedNameAsString() == "x::y::Old") {
+      EXPECT_EQ("Foo", replaceTypeLoc(Type.getOriginalDecl(),
+                                      Type.getBeginLoc(), "::x::Foo"));
     }
   };
   Visitor.runOver(R"(
diff --git a/clang/unittests/Tooling/QualTypeNamesTest.cpp b/clang/unittests/Tooling/QualTypeNamesTest.cpp
index bcf7d8935f792..1139392983fce 100644
--- a/clang/unittests/Tooling/QualTypeNamesTest.cpp
+++ b/clang/unittests/Tooling/QualTypeNamesTest.cpp
@@ -295,9 +295,9 @@ TEST(QualTypeNameTest, TemplatedClass) {
   auto *A2 = *ASpec;
 
   // Their type names follow the records.
-  QualType A1RecordTy = Context.getRecordType(A1);
+  CanQualType A1RecordTy = Context.getCanonicalTagType(A1);
   EXPECT_EQ(getFullyQualifiedName(A1RecordTy), "A<1>");
-  QualType A2RecordTy = Context.getRecordType(A2);
+  CanQualType A2RecordTy = Context.getCanonicalTagType(A2);
   EXPECT_EQ(getFullyQualifiedName(A2RecordTy), "A<2U>");
 
   // getTemplateSpecializationType() gives types that print the integral
@@ -305,13 +305,13 @@ TEST(QualTypeNameTest, TemplatedClass) {
   TemplateArgument Args1[] = {
       {Context, llvm::APSInt::getUnsigned(1u), Context.UnsignedIntTy}};
   QualType A1TemplateSpecTy = Context.getTemplateSpecializationType(
-      TemplateName(A), Args1, Args1, A1RecordTy);
+      ElaboratedTypeKeyword::None, TemplateName(A), Args1, Args1, A1RecordTy);
   EXPECT_EQ(A1TemplateSpecTy.getAsString(), "A<1>");
 
   TemplateArgument Args2[] = {
       {Context, llvm::APSInt::getUnsigned(2u), Context.UnsignedIntTy}};
   QualType A2TemplateSpecTy = Context.getTemplateSpecializationType(
-      TemplateName(A), Args2, Args2, A2RecordTy);
+      ElaboratedTypeKeyword::None, TemplateName(A), Args2, Args2, A2RecordTy);
   EXPECT_EQ(A2TemplateSpecTy.getAsString(), "A<2>");
 
   // Find A<1>::B and its specialization B<3>.
@@ -321,21 +321,19 @@ TEST(QualTypeNameTest, TemplatedClass) {
   auto A1BSpec = A1B->spec_begin();
   ASSERT_NE(A1BSpec, A1B->spec_end());
   auto *A1B3 = *A1BSpec;
-  QualType A1B3RecordTy = Context.getRecordType(A1B3);
+  CanQualType A1B3RecordTy = Context.getCanonicalTagType(A1B3);
   EXPECT_EQ(getFullyQualifiedName(A1B3RecordTy), "A<1>::B<3>");
 
   // Construct A<1>::B<3> and check name.
+  NestedNameSpecifier A1Nested(A1TemplateSpecTy.getTypePtr());
+  TemplateName A1B3Name = Context.getQualifiedTemplateName(
+      A1Nested, /*TemplateKeyword=*/false, TemplateName(A1B));
+
   TemplateArgument Args3[] = {
       {Context, llvm::APSInt::getUnsigned(3u), Context.UnsignedIntTy}};
   QualType A1B3TemplateSpecTy = Context.getTemplateSpecializationType(
-      TemplateName(A1B), Args3, Args3, A1B3RecordTy);
-  EXPECT_EQ(A1B3TemplateSpecTy.getAsString(), "B<3>");
-
-  NestedNameSpecifier *A1Nested = NestedNameSpecifier::Create(
-      Context, nullptr, A1TemplateSpecTy.getTypePtr());
-  QualType A1B3ElaboratedTy = Context.getElaboratedType(
-      ElaboratedTypeKeyword::None, A1Nested, A1B3TemplateSpecTy);
-  EXPECT_EQ(A1B3ElaboratedTy.getAsString(), "A<1>::B<3>");
+      ElaboratedTypeKeyword::None, A1B3Name, Args3, Args3, A1B3RecordTy);
+  EXPECT_EQ(A1B3TemplateSpecTy.getAsString(), "A<1>::B<3>");
 
   // Find A<2u>::B and its specialization B<4u>.
   auto *A2B =
@@ -344,21 +342,19 @@ TEST(QualTypeNameTest, TemplatedClass) {
   auto A2BSpec = A2B->spec_begin();
   ASSERT_NE(A2BSpec, A2B->spec_end());
   auto *A2B4 = *A2BSpec;
-  QualType A2B4RecordTy = Context.getRecordType(A2B4);
+  CanQualType A2B4RecordTy = Context.getCanonicalTagType(A2B4);
   EXPECT_EQ(getFullyQualifiedName(A2B4RecordTy), "A<2U>::B<4U>");
 
   // Construct A<2>::B<4> and check name.
+  NestedNameSpecifier A2Nested(A2TemplateSpecTy.getTypePtr());
+  TemplateName A2B4Name = Context.getQualifiedTemplateName(
+      A2Nested, /*TemplateKeyword=*/false, TemplateName(A2B));
+
   TemplateArgument Args4[] = {
       {Context, llvm::APSInt::getUnsigned(4u), Context.UnsignedIntTy}};
   QualType A2B4TemplateSpecTy = Context.getTemplateSpecializationType(
-      TemplateName(A2B), Args4, Args4, A2B4RecordTy);
-  EXPECT_EQ(A2B4TemplateSpecTy.getAsString(), "B<4>");
-
-  NestedNameSpecifier *A2Nested = NestedNameSpecifier::Create(
-      Context, nullptr, A2TemplateSpecTy.getTypePtr());
-  QualType A2B4ElaboratedTy = Context.getElaboratedType(
-      ElaboratedTypeKeyword::None, A2Nested, A2B4TemplateSpecTy);
-  EXPECT_EQ(A2B4ElaboratedTy.getAsString(), "A<2>::B<4>");
+      ElaboratedTypeKeyword::None, A2B4Name, Args4, Args4, A2B4RecordTy);
+  EXPECT_EQ(A2B4TemplateSpecTy.getAsString(), "A<2>::B<4>");
 }
 
 TEST(QualTypeNameTest, AnonStrucs) {
diff --git a/clang/unittests/Tooling/RangeSelectorTest.cpp b/clang/unittests/Tooling/RangeSelectorTest.cpp
index 12f7a8c5223e7..adf5e74ea3192 100644
--- a/clang/unittests/Tooling/RangeSelectorTest.cpp
+++ b/clang/unittests/Tooling/RangeSelectorTest.cpp
@@ -474,15 +474,15 @@ TEST(RangeSelectorTest, NameOpTypeLoc) {
   // Matches declaration of `a`
   TestMatch MatchA = matchCode(
       Code, varDecl(hasName("a"), hasTypeLoc(typeLoc().bind(CtorTy))));
-  EXPECT_THAT_EXPECTED(select(name(CtorTy), MatchA), HasValue("Foo"));
+  EXPECT_THAT_EXPECTED(select(name(CtorTy), MatchA), HasValue("ns::Foo"));
   // Matches call of Foo(int)
   TestMatch MatchB = matchCode(
       Code, cxxFunctionalCastExpr(hasTypeLoc(typeLoc().bind(CtorTy))));
-  EXPECT_THAT_EXPECTED(select(name(CtorTy), MatchB), HasValue("Foo"));
+  EXPECT_THAT_EXPECTED(select(name(CtorTy), MatchB), HasValue("ns::Foo"));
   // Matches call of Foo(int, int)
   TestMatch MatchC = matchCode(
       Code, cxxTemporaryObjectExpr(hasTypeLoc(typeLoc().bind(CtorTy))));
-  EXPECT_THAT_EXPECTED(select(name(CtorTy), MatchC), HasValue("Foo"));
+  EXPECT_THAT_EXPECTED(select(name(CtorTy), MatchC), HasValue("ns::Foo"));
 }
 
 TEST(RangeSelectorTest, NameOpTemplateSpecializationTypeLoc) {
diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTestTypeLocVisitor.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTestTypeLocVisitor.cpp
index eec628ca39641..7a3566114fbea 100644
--- a/clang/unittests/Tooling/RecursiveASTVisitorTestTypeLocVisitor.cpp
+++ b/clang/unittests/Tooling/RecursiveASTVisitorTestTypeLocVisitor.cpp
@@ -22,13 +22,13 @@ class TypeLocVisitor : public ExpectedLocationVisitor {
 
 TEST(RecursiveASTVisitor, VisitsBaseClassDeclarations) {
   TypeLocVisitor Visitor;
-  Visitor.ExpectMatch("class X", 1, 30);
+  Visitor.ExpectMatch("X", 1, 30);
   EXPECT_TRUE(Visitor.runOver("class X {}; class Y : public X {};"));
 }
 
 TEST(RecursiveASTVisitor, VisitsCXXBaseSpecifiersOfForwardDeclaredClass) {
   TypeLocVisitor Visitor;
-  Visitor.ExpectMatch("class X", 3, 18);
+  Visitor.ExpectMatch("X", 3, 18);
   EXPECT_TRUE(Visitor.runOver(
     "class Y;\n"
     "class X {};\n"
@@ -37,7 +37,7 @@ TEST(RecursiveASTVisitor, VisitsCXXBaseSpecifiersOfForwardDeclaredClass) {
 
 TEST(RecursiveASTVisitor, VisitsCXXBaseSpecifiersWithIncompleteInnerClass) {
   TypeLocVisitor Visitor;
-  Visitor.ExpectMatch("class X", 2, 18);
+  Visitor.ExpectMatch("X", 2, 18);
   EXPECT_TRUE(Visitor.runOver(
     "class X {};\n"
     "class Y : public X { class Z; };"));
@@ -45,7 +45,7 @@ TEST(RecursiveASTVisitor, VisitsCXXBaseSpecifiersWithIncompleteInnerClass) {
 
 TEST(RecursiveASTVisitor, VisitsCXXBaseSpecifiersOfSelfReferentialType) {
   TypeLocVisitor Visitor;
-  Visitor.ExpectMatch("X<Y>", 2, 18, 2);
+  Visitor.ExpectMatch("X<Y>", 2, 18);
   EXPECT_TRUE(Visitor.runOver(
     "template<typename T> class X {};\n"
     "class Y : public X<Y> {};"));
@@ -53,7 +53,7 @@ TEST(RecursiveASTVisitor, VisitsCXXBaseSpecifiersOfSelfReferentialType) {
 
 TEST(RecursiveASTVisitor, VisitsClassTemplateTypeParmDefaultArgument) {
   TypeLocVisitor Visitor;
-  Visitor.ExpectMatch("class X", 2, 23);
+  Visitor.ExpectMatch("X", 2, 23);
   EXPECT_TRUE(Visitor.runOver(
     "class X;\n"
     "template<typename T = X> class Y;\n"
@@ -62,7 +62,7 @@ TEST(RecursiveASTVisitor, VisitsClassTemplateTypeParmDefaultArgument) {
 
 TEST(RecursiveASTVisitor, VisitsCompoundLiteralType) {
   TypeLocVisitor Visitor;
-  Visitor.ExpectMatch("struct S", 1, 26);
+  Visitor.ExpectMatch("struct S", 1, 19);
   EXPECT_TRUE(Visitor.runOver(
       "int f() { return (struct S { int a; }){.a = 0}.a; }",
       TypeLocVisitor::Lang_C));
diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/MemberPointerTypeLoc.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/MemberPointerTypeLoc.cpp
index 587a00dd27051..88cebb7c28033 100644
--- a/clang/unittests/Tooling/RecursiveASTVisitorTests/MemberPointerTypeLoc.cpp
+++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/MemberPointerTypeLoc.cpp
@@ -24,7 +24,7 @@ class MemberPointerTypeLocVisitor : public ExpectedLocationVisitor {
   bool VisitRecordTypeLoc(RecordTypeLoc RTL) override {
     if (!RTL)
       return true;
-    Match(RTL.getDecl()->getName(), RTL.getNameLoc());
+    Match(RTL.getOriginalDecl()->getName(), RTL.getNameLoc());
     return true;
   }
 };
diff --git a/clang/unittests/Tooling/RecursiveASTVisitorTests/NestedNameSpecifiers.cpp b/clang/unittests/Tooling/RecursiveASTVisitorTests/NestedNameSpecifiers.cpp
index 23a2df42ff08c..4181cd2881054 100644
--- a/clang/unittests/Tooling/RecursiveASTVisitorTests/NestedNameSpecifiers.cpp
+++ b/clang/unittests/Tooling/RecursiveASTVisitorTests/NestedNameSpecifiers.cpp
@@ -18,17 +18,20 @@ class NestedNameSpecifiersVisitor : public ExpectedLocationVisitor {
   bool VisitRecordTypeLoc(RecordTypeLoc RTL) override {
     if (!RTL)
       return true;
-    Match(RTL.getDecl()->getName(), RTL.getNameLoc());
+    Match(RTL.getOriginalDecl()->getName(), RTL.getNameLoc());
     return true;
   }
 
-  bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS) override {
-    if (!NNS)
-      return true;
-    if (const auto *ND = dyn_cast_if_present<NamespaceDecl>(
-            NNS.getNestedNameSpecifier()->getAsNamespace()))
-      Match(ND->getName(), NNS.getLocalBeginLoc());
-    return ExpectedLocationVisitor::TraverseNestedNameSpecifierLoc(NNS);
+  bool
+  TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc QualifierLoc) override {
+    NestedNameSpecifier Qualifier = QualifierLoc.getNestedNameSpecifier();
+    if (Qualifier.getKind() == NestedNameSpecifier::Kind::Namespace) {
+      if (const auto *ND = dyn_cast<NamespaceDecl>(
+              Qualifier.getAsNamespaceAndPrefix().Namespace))
+        Match(ND->getName(), QualifierLoc.getLocalBeginLoc());
+    }
+    return ExpectedLocationVisitor::TraverseNestedNameSpecifierLoc(
+        QualifierLoc);
   }
 };
 
diff --git a/clang/unittests/Tooling/RefactoringTest.cpp b/clang/unittests/Tooling/RefactoringTest.cpp
index aff7523e7b340..171dc6de2cae7 100644
--- a/clang/unittests/Tooling/RefactoringTest.cpp
+++ b/clang/unittests/Tooling/RefactoringTest.cpp
@@ -747,9 +747,12 @@ TEST(Replacement, TemplatedFunctionCall) {
 class NestedNameSpecifierAVisitor : public TestVisitor {
 public:
   bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNSLoc) override {
-    if (NNSLoc.getNestedNameSpecifier()) {
-      if (const auto *NS = dyn_cast_if_present<NamespaceDecl>(
-              NNSLoc.getNestedNameSpecifier()->getAsNamespace())) {
+    if (NestedNameSpecifier NNS = NNSLoc.getNestedNameSpecifier();
+        NNS.getKind() == NestedNameSpecifier::Kind::Namespace) {
+      if (const auto *NS =
+              dyn_cast<NamespaceDecl>(NNSLoc.getNestedNameSpecifier()
+                                          .getAsNamespaceAndPrefix()
+                                          .Namespace)) {
         if (NS->getName() == "a") {
           Replace = Replacement(*SM, &NNSLoc, "", Context->getLangOpts());
         }
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index e2da20efaba5a..af2dcf6bfa982 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -972,10 +972,10 @@ Intrinsic::Intrinsic(StringRef Name, StringRef Proto, uint64_t MergeTy,
       BaseType(BT, 'd'), Flags(Flags), ImmChecks(Checks) {
 
   auto FormatGuard = [](StringRef Guard, StringRef Base) -> std::string {
+    if (Guard.empty() || Guard == Base)
+      return Guard.str();
     if (Guard.contains('|'))
       return Base.str() + ",(" + Guard.str() + ")";
-    if (Guard.empty() || Guard == Base || Guard.starts_with(Base.str() + ","))
-      return Guard.str();
     return Base.str() + "," + Guard.str();
   };
 
diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index dcff2fc2b1a3e..5b31f97e7a2e5 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -1302,15 +1302,10 @@ <h2 id="c99">C99 implementation status</h2>
       <td class="full" align="center">Yes</td>
     </tr>
     <tr>
-      <td>mixed declarations and code</td>
+      <td>mixed declarations and code<br \>new block scopes for selection and iteration statements</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n740.htm">N740</a></td>
       <td class="full" align="center">Yes</td>
     </tr>
-    <tr>
-      <td>new block scopes for selection and iteration statements</td>
-      <td>Unknown</td>
-      <td class="full" align="center">Yes</td>
-    </tr>
     <tr>
       <td>integer constant type rules</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n629.htm">N629</a></td>
diff --git a/cmake/Modules/FindLibcCommonUtils.cmake b/cmake/Modules/FindLibcCommonUtils.cmake
index 95426c51a6041..81cf74fbd0d41 100644
--- a/cmake/Modules/FindLibcCommonUtils.cmake
+++ b/cmake/Modules/FindLibcCommonUtils.cmake
@@ -12,6 +12,9 @@ if(NOT TARGET llvm-libc-common-utilities)
     add_library(llvm-libc-common-utilities INTERFACE)
     # TODO: Reorganize the libc shared section so that it can be included without
     # adding the root "libc" directory to the include path.
+    if (NOT(LIBCXX_ENABLE_THREADS))
+      target_compile_definitions(llvm-libc-common-utilities INTERFACE LIBC_THREAD_MODE=LIBC_THREAD_MODE_SINGLE)
+    endif()
     target_include_directories(llvm-libc-common-utilities INTERFACE ${libc_path})
     target_compile_definitions(llvm-libc-common-utilities INTERFACE LIBC_NAMESPACE=__llvm_libc_common_utils)
     target_compile_features(llvm-libc-common-utilities INTERFACE cxx_std_17)
diff --git a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
index fa8c6be6c5d46..c6777bda9f0d1 100644
--- a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
@@ -194,13 +194,15 @@ function(darwin_filter_host_archs input output)
     endif()
   endif()
 
+  # arm64e isn't usually supported out of the box on darwin, because of ABI.
+  list(REMOVE_ITEM tmp_var arm64e)
+
   if(ARM_HOST)
     list(REMOVE_ITEM tmp_var i386)
     list(REMOVE_ITEM tmp_var x86_64)
     list(REMOVE_ITEM tmp_var x86_64h)
   else()
     list(REMOVE_ITEM tmp_var arm64)
-    list(REMOVE_ITEM tmp_var arm64e)
     execute_process(
       COMMAND sysctl hw.cpusubtype
       OUTPUT_VARIABLE SUBTYPE)
diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index e3310b1ff0e2c..67db4383ec3dc 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -204,7 +204,7 @@ check_library_exists(stdc++ __cxa_throw "" COMPILER_RT_HAS_LIBSTDCXX)
 llvm_check_compiler_linker_flag(C "-Wl,-z,text" COMPILER_RT_HAS_Z_TEXT)
 llvm_check_compiler_linker_flag(C "-fuse-ld=lld" COMPILER_RT_HAS_FUSE_LD_LLD_FLAG)
 
-if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS" AND LLVM_LINKER_IS_SOLARISLD)
+if("${CMAKE_SYSTEM_NAME}" MATCHES "SunOS" AND LLVM_LINKER_IS_SOLARISLD)
   set(VERS_COMPAT_OPTION "-Wl,-z,gnu-version-script-compat")
   llvm_check_compiler_linker_flag(C "${VERS_COMPAT_OPTION}" COMPILER_RT_HAS_GNU_VERSION_SCRIPT_COMPAT)
 endif()
diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt
index 97cc5c85703e1..7d07ec765c005 100644
--- a/compiler-rt/lib/asan/CMakeLists.txt
+++ b/compiler-rt/lib/asan/CMakeLists.txt
@@ -106,6 +106,12 @@ if(MSVC)
 endif()
 set(ASAN_CFLAGS ${SANITIZER_COMMON_CFLAGS})
 
+# Win/ASan relies on the runtime functions being hotpatchable. See
+# https://github.com/llvm/llvm-project/pull/149444
+if(MSVC)
+  list(APPEND ASAN_CFLAGS /hotpatch)
+endif()
+
 append_list_if(MSVC /Zl ASAN_CFLAGS)
 
 set(ASAN_COMMON_DEFINITIONS "")
@@ -282,7 +288,7 @@ else()
   endif()
 
   # On AIX, we only need the static libraries.
-  if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  if (NOT "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   foreach(arch ${ASAN_SUPPORTED_ARCH})
     if (COMPILER_RT_HAS_VERSION_SCRIPT)
       if(WIN32)
@@ -392,7 +398,7 @@ add_compiler_rt_resource_file(asan_ignorelist asan_ignorelist.txt asan)
 # On AIX, static sanitizer libraries are not added to the DSO, so we need to put 
 # asan.link_with_main_exec.txt and asan_cxx.link_with_main_exec.txt to the build
 # and install dir for use in resolving undefined sanitizer symbols at runtime.
-if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   foreach(arch ${ASAN_SUPPORTED_ARCH})
     add_compiler_rt_cfg(asan_symbols_${arch} asan.link_with_main_exec.txt asan ${arch})
     add_compiler_rt_cfg(asan_cxx_symbols_${arch} asan_cxx.link_with_main_exec.txt asan ${arch})
diff --git a/compiler-rt/lib/asan/asan_fake_stack.cpp b/compiler-rt/lib/asan/asan_fake_stack.cpp
index 0f696075fb78d..c3ed2526f0ed4 100644
--- a/compiler-rt/lib/asan/asan_fake_stack.cpp
+++ b/compiler-rt/lib/asan/asan_fake_stack.cpp
@@ -54,18 +54,34 @@ FakeStack *FakeStack::Create(uptr stack_size_log) {
     stack_size_log = kMinStackSizeLog;
   if (stack_size_log > kMaxStackSizeLog)
     stack_size_log = kMaxStackSizeLog;
+  CHECK_LE(kMaxStackFrameSizeLog, stack_size_log);
   uptr size = RequiredSize(stack_size_log);
+  uptr padded_size = size + kMaxStackFrameSize;
+  void *true_res = reinterpret_cast<void *>(
+      flags()->uar_noreserve ? MmapNoReserveOrDie(padded_size, "FakeStack")
+                             : MmapOrDie(padded_size, "FakeStack"));
+  // GetFrame() requires the property that
+  // (res + kFlagsOffset + SizeRequiredForFlags(stack_size_log)) is aligned to
+  // kMaxStackFrameSize.
+  // We didn't use MmapAlignedOrDieOnFatalError, because it requires that the
+  // *size* is a power of 2, which is an overly strong condition.
+  static_assert(alignof(FakeStack) <= kMaxStackFrameSize);
   FakeStack *res = reinterpret_cast<FakeStack *>(
-      flags()->uar_noreserve ? MmapNoReserveOrDie(size, "FakeStack")
-                             : MmapOrDie(size, "FakeStack"));
+      RoundUpTo(
+          (uptr)true_res + kFlagsOffset + SizeRequiredForFlags(stack_size_log),
+          kMaxStackFrameSize) -
+      kFlagsOffset - SizeRequiredForFlags(stack_size_log));
+  res->true_start = true_res;
   res->stack_size_log_ = stack_size_log;
   u8 *p = reinterpret_cast<u8 *>(res);
   VReport(1,
           "T%d: FakeStack created: %p -- %p stack_size_log: %zd; "
-          "mmapped %zdK, noreserve=%d \n",
+          "mmapped %zdK, noreserve=%d, true_start: %p, start of first frame: "
+          "0x%zx\n",
           GetCurrentTidOrInvalid(), (void *)p,
           (void *)(p + FakeStack::RequiredSize(stack_size_log)), stack_size_log,
-          size >> 10, flags()->uar_noreserve);
+          size >> 10, flags()->uar_noreserve, res->true_start,
+          res->GetFrame(stack_size_log, /*class_id*/ 0, /*pos*/ 0));
   return res;
 }
 
@@ -79,8 +95,10 @@ void FakeStack::Destroy(int tid) {
     Report("T%d: FakeStack destroyed: %s\n", tid, str.data());
   }
   uptr size = RequiredSize(stack_size_log_);
-  FlushUnneededASanShadowMemory(reinterpret_cast<uptr>(this), size);
-  UnmapOrDie(this, size);
+  uptr padded_size = size + kMaxStackFrameSize;
+  FlushUnneededASanShadowMemory(reinterpret_cast<uptr>(true_start),
+                                padded_size);
+  UnmapOrDie(true_start, padded_size);
 }
 
 void FakeStack::PoisonAll(u8 magic) {
diff --git a/compiler-rt/lib/asan/asan_fake_stack.h b/compiler-rt/lib/asan/asan_fake_stack.h
index 270a19816d6e2..50706e6e5876c 100644
--- a/compiler-rt/lib/asan/asan_fake_stack.h
+++ b/compiler-rt/lib/asan/asan_fake_stack.h
@@ -32,12 +32,12 @@ struct FakeFrame {
 // is not popped but remains there for quite some time until gets used again.
 // So, we poison the objects on the fake stack when function returns.
 // It helps us find use-after-return bugs.
-//
 // The FakeStack objects is allocated by a single mmap call and has no other
 // pointers. The size of the fake stack depends on the actual thread stack size
 // and thus can not be a constant.
 // stack_size is a power of two greater or equal to the thread's stack size;
 // we store it as its logarithm (stack_size_log).
+// FakeStack is padded such that GetFrame() is aligned to BytesInSizeClass().
 // FakeStack has kNumberOfSizeClasses (11) size classes, each size class
 // is a power of two, starting from 64 bytes. Each size class occupies
 // stack_size bytes and thus can allocate
@@ -56,6 +56,9 @@ struct FakeFrame {
 class FakeStack {
   static const uptr kMinStackFrameSizeLog = 6;  // Min frame is 64B.
   static const uptr kMaxStackFrameSizeLog = 16;  // Max stack frame is 64K.
+  static_assert(kMaxStackFrameSizeLog >= kMinStackFrameSizeLog);
+
+  static const u64 kMaxStackFrameSize = 1 << kMaxStackFrameSizeLog;
 
  public:
   static const uptr kNumberOfSizeClasses =
@@ -66,7 +69,7 @@ class FakeStack {
 
   void Destroy(int tid);
 
-  // stack_size_log is at least 15 (stack_size >= 32K).
+  // min_uar_stack_size_log is 16 (stack_size >= 64KB)
   static uptr SizeRequiredForFlags(uptr stack_size_log) {
     return ((uptr)1) << (stack_size_log + 1 - kMinStackFrameSizeLog);
   }
@@ -110,6 +113,28 @@ class FakeStack {
   }
 
   // Get frame by class_id and pos.
+  // Return values are guaranteed to be aligned to BytesInSizeClass(class_id),
+  // which is useful in combination with
+  // ASanStackFrameLayout::ComputeASanStackFrameLayout().
+  //
+  // Note that alignment to 1<<kMaxStackFrameSizeLog (aka
+  // BytesInSizeClass(max_class_id)) implies alignment to BytesInSizeClass()
+  // for any class_id, since the class sizes are increasing powers of 2.
+  //
+  // 1) (this + kFlagsOffset + SizeRequiredForFlags())) is aligned to
+  //    1<<kMaxStackFrameSizeLog (see FakeStack::Create)
+  //
+  //    Note that SizeRequiredForFlags(16) == 2048. If FakeStack::Create() had
+  //    merely returned an address from mmap (4K-aligned), the addition would
+  //    not be 4K-aligned.
+  // 2) We know that stack_size_log >= kMaxStackFrameSizeLog (otherwise you
+  //    couldn't store a single frame of that size in the entire stack)
+  //    hence (1<<stack_size_log) is aligned to 1<<kMaxStackFrameSizeLog
+  //    and   ((1<<stack_size_log) * class_id) is aligned to
+  //          1<<kMaxStackFrameSizeLog
+  // 3) BytesInSizeClass(class_id) * pos is aligned to
+  //    BytesInSizeClass(class_id)
+  // The sum of these is aligned to BytesInSizeClass(class_id).
   u8 *GetFrame(uptr stack_size_log, uptr class_id, uptr pos) {
     return reinterpret_cast<u8 *>(this) + kFlagsOffset +
            SizeRequiredForFlags(stack_size_log) +
@@ -156,15 +181,18 @@ class FakeStack {
 
  private:
   FakeStack() { }
-  static const uptr kFlagsOffset = 4096;  // This is were the flags begin.
+  static const uptr kFlagsOffset = 4096;  // This is where the flags begin.
   // Must match the number of uses of DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID
   COMPILER_CHECK(kNumberOfSizeClasses == 11);
   static const uptr kMaxStackMallocSize = ((uptr)1) << kMaxStackFrameSizeLog;
 
   uptr hint_position_[kNumberOfSizeClasses];
   uptr stack_size_log_;
-  // a bit is set if something was allocated from the corresponding size class.
   bool needs_gc_;
+  // We allocated more memory than needed to ensure the FakeStack (and, by
+  // extension, each of the fake stack frames) is aligned. We keep track of the
+  // true start so that we can unmap it.
+  void *true_start;
 };
 
 FakeStack *GetTLSFakeStack();
diff --git a/compiler-rt/lib/asan/asan_fuchsia.cpp b/compiler-rt/lib/asan/asan_fuchsia.cpp
index 6876be1dca535..a9e5dad91b973 100644
--- a/compiler-rt/lib/asan/asan_fuchsia.cpp
+++ b/compiler-rt/lib/asan/asan_fuchsia.cpp
@@ -32,11 +32,11 @@ void EarlySanitizerInit() {}
 
 namespace __asan {
 
-// The system already set up the shadow memory for us.
-// __sanitizer::GetMaxUserVirtualAddress has already been called by
-// AsanInitInternal->InitializeHighMemEnd (asan_rtl.cpp).
-// Just do some additional sanity checks here.
 void InitializeShadowMemory() {
+  // Explicitly setup shadow here right beforer any of the ShadowBounds members
+  // are used.
+  InitShadowBounds();
+
   if (Verbosity())
     PrintAddressSpaceLayout();
 
diff --git a/compiler-rt/lib/asan/asan_mapping.h b/compiler-rt/lib/asan/asan_mapping.h
index 91fe60db6329a..bddae9a074056 100644
--- a/compiler-rt/lib/asan/asan_mapping.h
+++ b/compiler-rt/lib/asan/asan_mapping.h
@@ -83,18 +83,29 @@
 // || `[0x0000000000, 0x0d5554ffff]` || LowMem     ||
 //
 // Default Linux/AArch64 (39-bit VMA) mapping:
-// || `[0x2000000000, 0x7fffffffff]` || highmem    ||
-// || `[0x1400000000, 0x1fffffffff]` || highshadow ||
-// || `[0x1200000000, 0x13ffffffff]` || shadowgap  ||
-// || `[0x1000000000, 0x11ffffffff]` || lowshadow  ||
-// || `[0x0000000000, 0x0fffffffff]` || lowmem     ||
+// TODO: this mapping is ok, but the allocator size is too large on non-Android
+//       AArch64 platforms (see asan_allocator.h)
+// || `[0x2000000000, 0x7fffffffff]` || highmem    || 384GB
+// || `[0x1400000000, 0x1fffffffff]` || highshadow || 48GB
+// || `[0x1200000000, 0x13ffffffff]` || shadowgap  || 8GB
+// || `[0x1000000000, 0x11ffffffff]` || lowshadow  || 8GB
+// || `[0x0000000000, 0x0fffffffff]` || lowmem     || 64GB
 //
 // Default Linux/AArch64 (42-bit VMA) mapping:
-// || `[0x10000000000, 0x3ffffffffff]` || highmem    ||
-// || `[0x0a000000000, 0x0ffffffffff]` || highshadow ||
-// || `[0x09000000000, 0x09fffffffff]` || shadowgap  ||
-// || `[0x08000000000, 0x08fffffffff]` || lowshadow  ||
-// || `[0x00000000000, 0x07fffffffff]` || lowmem     ||
+// TODO: this mapping is ok, but the allocator size is too large on non-Android
+//       AArch64 platforms (see asan_allocator.h)
+// || `[0x09000000000, 0x03ffffffffff]` || highmem    || 3520GB
+// || `[0x02200000000, 0x008fffffffff]` || highshadow || 440GB
+// || `[0x01200000000, 0x0021ffffffff]` || shadowgap  || 64GB
+// || `[0x01000000000, 0x0011ffffffff]` || lowshadow  || 8GB
+// || `[0x00000000000, 0x000fffffffff]` || lowmem     || 64GB
+//
+// Default Linux/AArch64 (48-bit VMA) mapping:
+// || `[0x201000000000, 0xffffffffffff]` || HighMem    || 229312GB
+// || `[0x041200000000, 0x200fffffffff]` || HighShadow || 28664GB
+// || `[0x001200000000, 0x0411ffffffff]` || ShadowGap  || 4096GB
+// || `[0x001000000000, 0x0011ffffffff]` || LowShadow  || 8GB
+// || `[0x000000000000, 0x000fffffffff]` || LowMem     || 64GB
 //
 // Default Linux/S390 mapping:
 // || `[0x30000000, 0x7fffffff]` || HighMem    ||
diff --git a/compiler-rt/lib/asan/scripts/asan_symbolize.py b/compiler-rt/lib/asan/scripts/asan_symbolize.py
index 058a1614b55e6..8ecd66c745119 100755
--- a/compiler-rt/lib/asan/scripts/asan_symbolize.py
+++ b/compiler-rt/lib/asan/scripts/asan_symbolize.py
@@ -507,20 +507,29 @@ def symbolize_address(self, addr, binary, offset, arch):
         assert result
         return result
 
-    def get_symbolized_lines(self, symbolized_lines, inc_frame_counter=True):
+    def get_symbolized_lines(self, symbolized_lines):
         if not symbolized_lines:
-            if inc_frame_counter:
-                self.frame_no += 1
-            return [self.current_line]
-        else:
-            assert inc_frame_counter
-            result = []
-            for symbolized_frame in symbolized_lines:
-                result.append(
-                    "    #%s %s" % (str(self.frame_no), symbolized_frame.rstrip())
+            # If it is an unparsable frame, but contains a frame counter and address
+            # replace the frame counter so the stack is still consistent.
+            unknown_stack_frame_format = r"^( *#([0-9]+) +)(0x[0-9a-f]+) +.*"
+            match = re.match(unknown_stack_frame_format, self.current_line)
+            if match:
+                rewritten_line = (
+                    self.current_line[: match.start(2)]
+                    + str(self.frame_no)
+                    + self.current_line[match.end(2) :]
                 )
                 self.frame_no += 1
-            return result
+                return [rewritten_line]
+            # Not a frame line so don't increment the frame counter.
+            return [self.current_line]
+        result = []
+        for symbolized_frame in symbolized_lines:
+            result.append(
+                "    #%s %s" % (str(self.frame_no), symbolized_frame.rstrip())
+            )
+            self.frame_no += 1
+        return result
 
     def process_logfile(self):
         self.frame_no = 0
@@ -546,8 +555,7 @@ def process_line_posix(self, line):
         match = re.match(stack_trace_line_format, line)
         if not match:
             logging.debug('Line "{}" does not match regex'.format(line))
-            # Not a frame line so don't increment the frame counter.
-            return self.get_symbolized_lines(None, inc_frame_counter=False)
+            return self.get_symbolized_lines(None)
         logging.debug(line)
         _, frameno_str, addr, binary, offset = match.groups()
 
diff --git a/compiler-rt/lib/asan/tests/asan_fake_stack_test.cpp b/compiler-rt/lib/asan/tests/asan_fake_stack_test.cpp
index 504b0aaf30143..c60e2eadc35db 100644
--- a/compiler-rt/lib/asan/tests/asan_fake_stack_test.cpp
+++ b/compiler-rt/lib/asan/tests/asan_fake_stack_test.cpp
@@ -113,6 +113,7 @@ TEST(FakeStack, Allocate) {
       uptr bytes_in_class = FakeStack::BytesInSizeClass(cid);
       for (uptr j = 0; j < n; j++) {
         FakeFrame *ff = fs->Allocate(stack_size_log, cid, 0);
+        EXPECT_EQ(reinterpret_cast<uptr>(ff) % bytes_in_class, 0U);
         uptr x = reinterpret_cast<uptr>(ff);
         EXPECT_TRUE(s.insert(std::make_pair(ff, cid)).second);
         EXPECT_EQ(x, fs->AddrIsInFakeStack(x));
diff --git a/compiler-rt/lib/builtins/aarch64/lse.S b/compiler-rt/lib/builtins/aarch64/lse.S
index d7c1db7243ef8..a444d82892c36 100644
--- a/compiler-rt/lib/builtins/aarch64/lse.S
+++ b/compiler-rt/lib/builtins/aarch64/lse.S
@@ -264,7 +264,7 @@ END_COMPILERRT_OUTLINE_FUNCTION(NAME(LDNM))
 
 NO_EXEC_STACK_DIRECTIVE
 
-// GNU property note for BTI and PAC
-GNU_PROPERTY_BTI_PAC
+// GNU property note for BTI, PAC, and GCS
+GNU_PROPERTY_BTI_PAC_GCS
 
 #endif // defined(__aarch64__) || defined(__arm64ec__)
diff --git a/compiler-rt/lib/builtins/aarch64/sme-abi.S b/compiler-rt/lib/builtins/aarch64/sme-abi.S
index 7c47336cfc579..d5510ac0cfa50 100644
--- a/compiler-rt/lib/builtins/aarch64/sme-abi.S
+++ b/compiler-rt/lib/builtins/aarch64/sme-abi.S
@@ -371,5 +371,5 @@ END_COMPILERRT_FUNCTION(__arm_sme_restore)
 
 NO_EXEC_STACK_DIRECTIVE
 
-// GNU property note for BTI and PAC
-GNU_PROPERTY_BTI_PAC
+// GNU property note for BTI, PAC, and GCS
+GNU_PROPERTY_BTI_PAC_GCS
diff --git a/compiler-rt/lib/builtins/assembly.h b/compiler-rt/lib/builtins/assembly.h
index 89372f18c84b2..d7db7d8189457 100644
--- a/compiler-rt/lib/builtins/assembly.h
+++ b/compiler-rt/lib/builtins/assembly.h
@@ -79,11 +79,12 @@
 #define FUNC_ALIGN
 #endif
 
-// BTI and PAC gnu property note
+// BTI, PAC, and GCS gnu property note
 #define NT_GNU_PROPERTY_TYPE_0 5
 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI 1
 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC 2
+#define GNU_PROPERTY_AARCH64_FEATURE_1_GCS 4
 
 #if defined(__ARM_FEATURE_BTI_DEFAULT)
 #define BTI_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_BTI
@@ -97,6 +98,12 @@
 #define PAC_FLAG 0
 #endif
 
+#if defined(__ARM_FEATURE_GCS_DEFAULT)
+#define GCS_FLAG GNU_PROPERTY_AARCH64_FEATURE_1_GCS
+#else
+#define GCS_FLAG 0
+#endif
+
 #define GNU_PROPERTY(type, value)                                              \
   .pushsection .note.gnu.property, "a" SEPARATOR                               \
   .p2align 3 SEPARATOR                                                         \
@@ -118,11 +125,12 @@
 #define BTI_J
 #endif
 
-#if (BTI_FLAG | PAC_FLAG) != 0
-#define GNU_PROPERTY_BTI_PAC                                                   \
-  GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND, BTI_FLAG | PAC_FLAG)
+#if (BTI_FLAG | PAC_FLAG | GCS_FLAG) != 0
+#define GNU_PROPERTY_BTI_PAC_GCS                                               \
+  GNU_PROPERTY(GNU_PROPERTY_AARCH64_FEATURE_1_AND,                             \
+               BTI_FLAG | PAC_FLAG | GCS_FLAG)
 #else
-#define GNU_PROPERTY_BTI_PAC
+#define GNU_PROPERTY_BTI_PAC_GCS
 #endif
 
 #if defined(__clang__) || defined(__GCC_HAVE_DWARF2_CFI_ASM)
diff --git a/compiler-rt/lib/builtins/cpu_model/x86.c b/compiler-rt/lib/builtins/cpu_model/x86.c
index 606571d527501..79705ca0886f8 100644
--- a/compiler-rt/lib/builtins/cpu_model/x86.c
+++ b/compiler-rt/lib/builtins/cpu_model/x86.c
@@ -328,7 +328,7 @@ static const char *getIntelProcessorTypeAndSubtype(unsigned Family,
   const char *CPU = 0;
 
   switch (Family) {
-  case 6:
+  case 0x6:
     switch (Model) {
     case 0x0f: // Intel Core 2 Duo processor, Intel Core 2 Duo mobile
                // processor, Intel Core 2 Quad processor, Intel Core 2 Quad
@@ -626,7 +626,7 @@ static const char *getIntelProcessorTypeAndSubtype(unsigned Family,
       break;
     }
     break;
-  case 19:
+  case 0x13:
     switch (Model) {
     // Diamond Rapids:
     case 0x01:
diff --git a/compiler-rt/lib/dfsan/dfsan.cpp b/compiler-rt/lib/dfsan/dfsan.cpp
index 886e93e5fa813..d09a9a70fd83b 100644
--- a/compiler-rt/lib/dfsan/dfsan.cpp
+++ b/compiler-rt/lib/dfsan/dfsan.cpp
@@ -792,7 +792,7 @@ static void PrintNoOriginTrackingWarning() {
 
 static void PrintNoTaintWarning(const void *address) {
   Decorator d;
-  Printf("  %sDFSan: no tainted value at %x%s\n", d.Warning(), address,
+  Printf("  %sDFSan: no tainted value at %zx%s\n", d.Warning(), (uptr)address,
          d.Default());
 }
 
diff --git a/compiler-rt/lib/gwp_asan/tests/basic.cpp b/compiler-rt/lib/gwp_asan/tests/basic.cpp
index 88e7ed14a5c2f..7d36a2ee1f947 100644
--- a/compiler-rt/lib/gwp_asan/tests/basic.cpp
+++ b/compiler-rt/lib/gwp_asan/tests/basic.cpp
@@ -65,11 +65,12 @@ TEST_F(DefaultGuardedPoolAllocator, NonPowerOfTwoAlignment) {
 
 // Added multi-page slots? You'll need to expand this test.
 TEST_F(DefaultGuardedPoolAllocator, TooBigForSinglePageSlots) {
-  EXPECT_EQ(nullptr, GPA.allocate(0x1001, 0));
-  EXPECT_EQ(nullptr, GPA.allocate(0x1001, 1));
-  EXPECT_EQ(nullptr, GPA.allocate(0x1001, 0x1000));
-  EXPECT_EQ(nullptr, GPA.allocate(1, 0x2000));
-  EXPECT_EQ(nullptr, GPA.allocate(0, 0x2000));
+  size_t PageSize = sysconf(_SC_PAGESIZE);
+  EXPECT_EQ(nullptr, GPA.allocate(PageSize + 1, 0));
+  EXPECT_EQ(nullptr, GPA.allocate(PageSize + 1, 1));
+  EXPECT_EQ(nullptr, GPA.allocate(PageSize + 1, PageSize));
+  EXPECT_EQ(nullptr, GPA.allocate(1, 2 * PageSize));
+  EXPECT_EQ(nullptr, GPA.allocate(0, 2 * PageSize));
 }
 
 TEST_F(CustomGuardedPoolAllocator, AllocAllSlots) {
diff --git a/compiler-rt/lib/gwp_asan/tests/never_allocated.cpp b/compiler-rt/lib/gwp_asan/tests/never_allocated.cpp
index 2f695b4379861..37a4b384e4ac0 100644
--- a/compiler-rt/lib/gwp_asan/tests/never_allocated.cpp
+++ b/compiler-rt/lib/gwp_asan/tests/never_allocated.cpp
@@ -13,8 +13,10 @@
 #include "gwp_asan/tests/harness.h"
 
 TEST_P(BacktraceGuardedPoolAllocatorDeathTest, NeverAllocated) {
+  size_t PageSize = sysconf(_SC_PAGESIZE);
+
   SCOPED_TRACE("");
-  void *Ptr = GPA.allocate(0x1000);
+  void *Ptr = GPA.allocate(PageSize);
   GPA.deallocate(Ptr);
 
   std::string DeathNeedle =
@@ -23,7 +25,7 @@ TEST_P(BacktraceGuardedPoolAllocatorDeathTest, NeverAllocated) {
   // Trigger a guard page in a completely different slot that's never allocated.
   // Previously, there was a bug that this would result in nullptr-dereference
   // in the posix crash handler.
-  char *volatile NeverAllocatedPtr = static_cast<char *>(Ptr) + 0x3000;
+  char *volatile NeverAllocatedPtr = static_cast<char *>(Ptr) + 3 * PageSize;
   if (!Recoverable) {
     EXPECT_DEATH(*NeverAllocatedPtr = 0, DeathNeedle);
     return;
@@ -37,8 +39,8 @@ TEST_P(BacktraceGuardedPoolAllocatorDeathTest, NeverAllocated) {
   GetOutputBuffer().clear();
   for (size_t i = 0; i < 100; ++i) {
     *NeverAllocatedPtr = 0;
-    *(NeverAllocatedPtr + 0x2000) = 0;
-    *(NeverAllocatedPtr + 0x3000) = 0;
+    *(NeverAllocatedPtr + 2 * PageSize) = 0;
+    *(NeverAllocatedPtr + 3 * PageSize) = 0;
     ASSERT_TRUE(GetOutputBuffer().empty());
   }
 
diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp
index 24384d8b4d2cf..615bae4b3a3fc 100644
--- a/compiler-rt/lib/hwasan/hwasan.cpp
+++ b/compiler-rt/lib/hwasan/hwasan.cpp
@@ -176,7 +176,7 @@ static void HwasanFormatMemoryUsage(InternalScopedString &s) {
       "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd"
       " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd"
       " heap: %zd",
-      internal_getpid(), GetRSS(), thread_stats.n_live_threads,
+      (int)internal_getpid(), GetRSS(), thread_stats.n_live_threads,
       thread_stats.total_stack_size,
       thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(),
       sds.allocated, sds.n_uniq_ids, asc[AllocatorStatMapped]);
@@ -692,7 +692,7 @@ void __hwasan_handle_longjmp(const void *sp_dst) {
         "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: "
         "stack top: %p; target %p; distance: %p (%zd)\n"
         "False positive error reports may follow\n",
-        (void *)sp, (void *)dst, dst - sp, dst - sp);
+        (void *)sp, (void *)dst, (void *)(dst - sp), dst - sp);
     return;
   }
   TagMemory(sp, dst - sp, 0);
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S
index fd20825e3dacc..825f411565098 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S
@@ -11,4 +11,4 @@
 
 NO_EXEC_STACK_DIRECTIVE
 
-GNU_PROPERTY_BTI_PAC
+GNU_PROPERTY_BTI_PAC_GCS
diff --git a/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/compiler-rt/lib/hwasan/hwasan_interface_internal.h
index 8f2f77dad917d..86ddfea5bf82a 100644
--- a/compiler-rt/lib/hwasan/hwasan_interface_internal.h
+++ b/compiler-rt/lib/hwasan/hwasan_interface_internal.h
@@ -247,6 +247,13 @@ void *__hwasan_memmove_match_all(void *dest, const void *src, uptr n, u8);
 
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_set_error_report_callback(void (*callback)(const char *));
+
+// hwasan does not need fake stack, so we leave it empty here.
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_start_switch_fiber(void **, const void *bottom, uptr size);
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_finish_switch_fiber(void *, const void **bottom_old,
+                                     uptr *size_old);
 }  // extern "C"
 
 #endif  // HWASAN_INTERFACE_INTERNAL_H
diff --git a/compiler-rt/lib/hwasan/hwasan_malloc_bisect.h b/compiler-rt/lib/hwasan/hwasan_malloc_bisect.h
index 7d134e8c4b7fa..52a28438f3a9b 100644
--- a/compiler-rt/lib/hwasan/hwasan_malloc_bisect.h
+++ b/compiler-rt/lib/hwasan/hwasan_malloc_bisect.h
@@ -41,7 +41,7 @@ static inline bool malloc_bisect(StackTrace *stack, uptr orig_size) {
   if (h < left || h > right)
     return false;
   if (flags()->malloc_bisect_dump) {
-    Printf("[alloc] %u %zu\n", h, orig_size);
+    Printf("[alloc] %u %zu\n", (u32)h, orig_size);
     stack->Print();
   }
   return true;
diff --git a/compiler-rt/lib/hwasan/hwasan_report.cpp b/compiler-rt/lib/hwasan/hwasan_report.cpp
index bc66e6e805c91..6eafcf9163afa 100644
--- a/compiler-rt/lib/hwasan/hwasan_report.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_report.cpp
@@ -306,8 +306,9 @@ static void PrintStackAllocations(const StackAllocationsRingBuffer *sa,
           "%p is located %zd bytes %s a %zd-byte local variable %s "
           "[%p,%p) "
           "in %s %s\n",
-          untagged_addr, offset, whence, local.size, local.name, best_beg,
-          best_beg + local.size, local.function_name, location.data());
+          (void *)untagged_addr, offset, whence, local.size, local.name,
+          (void *)best_beg, (void *)(best_beg + local.size),
+          local.function_name, location.data());
       location.clear();
       Printf("%s\n", d.Default());
     }
@@ -738,8 +739,8 @@ void BaseReport::PrintHeapOrGlobalCandidate() const {
     Printf("%s", d.Location());
     Printf("%p is located %zd bytes %s a %zd-byte region [%p,%p)\n",
            untagged_addr, offset, whence,
-           candidate.heap.end - candidate.heap.begin, candidate.heap.begin,
-           candidate.heap.end);
+           candidate.heap.end - candidate.heap.begin,
+           (void *)candidate.heap.begin, (void *)candidate.heap.end);
     Printf("%s", d.Allocation());
     Printf("allocated by thread T%u here:\n", candidate.heap.thread_id);
     Printf("%s", d.Default());
@@ -762,11 +763,11 @@ void BaseReport::PrintHeapOrGlobalCandidate() const {
       Printf(
           "%p is located %zd bytes %s a %zd-byte global variable "
           "%s [%p,%p) in %s\n",
-          untagged_addr,
+          (void *)untagged_addr,
           candidate.after ? untagged_addr - (info.start + info.size)
                           : info.start - untagged_addr,
           candidate.after ? "after" : "before", info.size, info.name,
-          info.start, info.start + info.size, module_name);
+          (void *)info.start, (void *)(info.start + info.size), module_name);
     } else {
       uptr size = GetGlobalSizeFromDescriptor(candidate.untagged_addr);
       if (size == 0)
@@ -774,14 +775,14 @@ void BaseReport::PrintHeapOrGlobalCandidate() const {
         Printf(
             "%p is located %s a global variable in "
             "\n    #0 0x%x (%s+0x%x)\n",
-            untagged_addr, candidate.after ? "after" : "before",
-            candidate.untagged_addr, module_name, module_address);
+            (void *)untagged_addr, candidate.after ? "after" : "before",
+            (void *)candidate.untagged_addr, module_name, (u32)module_address);
       else
         Printf(
             "%p is located %s a %zd-byte global variable in "
             "\n    #0 0x%x (%s+0x%x)\n",
-            untagged_addr, candidate.after ? "after" : "before", size,
-            candidate.untagged_addr, module_name, module_address);
+            (void *)untagged_addr, candidate.after ? "after" : "before", size,
+            (void *)candidate.untagged_addr, module_name, (u32)module_address);
     }
     Printf("%s", d.Default());
   }
@@ -792,8 +793,8 @@ void BaseReport::PrintAddressDescription() const {
   int num_descriptions_printed = 0;
 
   if (MemIsShadow(untagged_addr)) {
-    Printf("%s%p is HWAsan shadow memory.\n%s", d.Location(), untagged_addr,
-           d.Default());
+    Printf("%s%p is HWAsan shadow memory.\n%s", d.Location(),
+           (void *)untagged_addr, d.Default());
     return;
   }
 
@@ -802,7 +803,7 @@ void BaseReport::PrintAddressDescription() const {
     Printf(
         "%s[%p,%p) is a %s %s heap chunk; "
         "size: %zd offset: %zd\n%s",
-        d.Location(), heap.begin, heap.begin + heap.size,
+        d.Location(), (void *)heap.begin, (void *)(heap.begin + heap.size),
         heap.from_small_heap ? "small" : "large",
         heap.is_allocated ? "allocated" : "unallocated", heap.size,
         untagged_addr - heap.begin, d.Default());
@@ -821,8 +822,8 @@ void BaseReport::PrintAddressDescription() const {
     Printf("%s", d.Error());
     Printf("\nCause: stack tag-mismatch\n");
     Printf("%s", d.Location());
-    Printf("Address %p is located in stack of thread T%zd\n", untagged_addr,
-           sa.thread_id());
+    Printf("Address %p is located in stack of thread T%zd\n",
+           (void *)untagged_addr, (ssize)sa.thread_id());
     Printf("%s", d.Default());
     announce_by_id(sa.thread_id());
     PrintStackAllocations(sa.get(), ptr_tag, untagged_addr);
@@ -842,9 +843,9 @@ void BaseReport::PrintAddressDescription() const {
     Printf("\nCause: use-after-free\n");
     Printf("%s", d.Location());
     Printf("%p is located %zd bytes inside a %zd-byte region [%p,%p)\n",
-           untagged_addr, untagged_addr - UntagAddr(har.tagged_addr),
-           har.requested_size, UntagAddr(har.tagged_addr),
-           UntagAddr(har.tagged_addr) + har.requested_size);
+           (void *)untagged_addr, untagged_addr - UntagAddr(har.tagged_addr),
+           (ssize)har.requested_size, UntagAddr(har.tagged_addr),
+           (void *)(UntagAddr(har.tagged_addr) + har.requested_size));
     Printf("%s", d.Allocation());
     Printf("freed by thread T%u here:\n", ha.free_thread_id);
     Printf("%s", d.Default());
@@ -858,7 +859,7 @@ void BaseReport::PrintAddressDescription() const {
     // Print a developer note: the index of this heap object
     // in the thread's deallocation ring buffer.
     Printf("hwasan_dev_note_heap_rb_distance: %zd %zd\n", ha.ring_index + 1,
-           flags()->heap_history_size);
+           (ssize)flags()->heap_history_size);
     Printf("hwasan_dev_note_num_matching_addrs: %zd\n", ha.num_matching_addrs);
     Printf("hwasan_dev_note_num_matching_addrs_4b: %zd\n",
            ha.num_matching_addrs_4b);
@@ -915,10 +916,11 @@ InvalidFreeReport::~InvalidFreeReport() {
   const Thread *thread = GetCurrentThread();
   if (thread) {
     Report("ERROR: %s: %s on address %p at pc %p on thread T%zd\n",
-           SanitizerToolName, bug_type, untagged_addr, pc, thread->unique_id());
+           SanitizerToolName, bug_type, (void *)untagged_addr, (void *)pc,
+           (ssize)thread->unique_id());
   } else {
     Report("ERROR: %s: %s on address %p at pc %p on unknown thread\n",
-           SanitizerToolName, bug_type, untagged_addr, pc);
+           SanitizerToolName, bug_type, (void *)untagged_addr, (void *)pc);
   }
   Printf("%s", d.Access());
   if (shadow.addr) {
@@ -967,7 +969,8 @@ TailOverwrittenReport::~TailOverwrittenReport() {
   Printf("%s", d.Error());
   const char *bug_type = "allocation-tail-overwritten";
   Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName,
-         bug_type, untagged_addr, untagged_addr + orig_size, orig_size);
+         bug_type, (void *)untagged_addr, (void *)(untagged_addr + orig_size),
+         orig_size);
   Printf("\n%s", d.Default());
   Printf(
       "Stack of invalid access unknown. Issue detected at deallocation "
@@ -1037,7 +1040,7 @@ TagMismatchReport::~TagMismatchReport() {
   uptr pc = GetTopPc(stack);
   Printf("%s", d.Error());
   Report("ERROR: %s: %s on address %p at pc %p\n", SanitizerToolName, bug_type,
-         untagged_addr, pc);
+         (void *)untagged_addr, (void *)pc);
 
   Thread *t = GetCurrentThread();
 
@@ -1049,12 +1052,12 @@ TagMismatchReport::~TagMismatchReport() {
         GetShortTagCopy(MemToShadow(untagged_addr + mismatch_offset));
     Printf(
         "%s of size %zu at %p tags: %02x/%02x(%02x) (ptr/mem) in thread T%zd\n",
-        is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag,
-        mem_tag, short_tag, t->unique_id());
+        is_store ? "WRITE" : "READ", access_size, (void *)untagged_addr,
+        ptr_tag, mem_tag, short_tag, (ssize)t->unique_id());
   } else {
     Printf("%s of size %zu at %p tags: %02x/%02x (ptr/mem) in thread T%zd\n",
-           is_store ? "WRITE" : "READ", access_size, untagged_addr, ptr_tag,
-           mem_tag, t->unique_id());
+           is_store ? "WRITE" : "READ", access_size, (void *)untagged_addr,
+           ptr_tag, mem_tag, (ssize)t->unique_id());
   }
   if (mismatch_offset)
     Printf("Invalid access starting at offset %zu\n", mismatch_offset);
@@ -1093,7 +1096,7 @@ void ReportTagMismatch(StackTrace *stack, uptr tagged_addr, uptr access_size,
 // See the frame breakdown defined in __hwasan_tag_mismatch (from
 // hwasan_tag_mismatch_{aarch64,riscv64}.S).
 void ReportRegisters(const uptr *frame, uptr pc) {
-  Printf("\nRegisters where the failure occurred (pc %p):\n", pc);
+  Printf("\nRegisters where the failure occurred (pc %p):\n", (void *)pc);
 
   // We explicitly print a single line (4 registers/line) each iteration to
   // reduce the amount of logcat error messages printed. Each Printf() will
diff --git a/compiler-rt/lib/hwasan/hwasan_setjmp_aarch64.S b/compiler-rt/lib/hwasan/hwasan_setjmp_aarch64.S
index 0c0abb6de861f..b8d98b09ada25 100644
--- a/compiler-rt/lib/hwasan/hwasan_setjmp_aarch64.S
+++ b/compiler-rt/lib/hwasan/hwasan_setjmp_aarch64.S
@@ -99,4 +99,4 @@ ASM_TRAMPOLINE_ALIAS(_setjmp, setjmp)
 // We do not need executable stack.
 NO_EXEC_STACK_DIRECTIVE
 
-GNU_PROPERTY_BTI_PAC
+GNU_PROPERTY_BTI_PAC_GCS
diff --git a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
index fd060c51cd8e2..be82475101c8c 100644
--- a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
+++ b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
@@ -157,4 +157,4 @@ mismatch:
 // We do not need executable stack.
 NO_EXEC_STACK_DIRECTIVE
 
-GNU_PROPERTY_BTI_PAC
+GNU_PROPERTY_BTI_PAC_GCS
diff --git a/compiler-rt/lib/hwasan/hwasan_thread.cpp b/compiler-rt/lib/hwasan/hwasan_thread.cpp
index 5c07522d42796..bebc11d082cab 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_thread.cpp
@@ -119,10 +119,64 @@ void Thread::Destroy() {
   *GetCurrentThreadLongPtr() = 0;
 }
 
+void Thread::StartSwitchFiber(uptr bottom, uptr size) {
+  if (atomic_load(&stack_switching_, memory_order_acquire)) {
+    Report("ERROR: starting fiber switch while in fiber switch\n");
+    Die();
+  }
+
+  next_stack_bottom_ = bottom;
+  next_stack_top_ = bottom + size;
+  atomic_store(&stack_switching_, 1, memory_order_release);
+}
+
+void Thread::FinishSwitchFiber(uptr *bottom_old, uptr *size_old) {
+  if (!atomic_load(&stack_switching_, memory_order_acquire)) {
+    Report("ERROR: finishing a fiber switch that has not started\n");
+    Die();
+  }
+
+  if (bottom_old)
+    *bottom_old = stack_bottom_;
+  if (size_old)
+    *size_old = stack_top_ - stack_bottom_;
+  stack_bottom_ = next_stack_bottom_;
+  stack_top_ = next_stack_top_;
+  atomic_store(&stack_switching_, 0, memory_order_release);
+  next_stack_top_ = 0;
+  next_stack_bottom_ = 0;
+}
+
+inline Thread::StackBounds Thread::GetStackBounds() const {
+  if (!atomic_load(&stack_switching_, memory_order_acquire)) {
+    // Make sure the stack bounds are fully initialized.
+    if (stack_bottom_ >= stack_top_)
+      return {0, 0};
+    return {stack_bottom_, stack_top_};
+  }
+  const uptr cur_stack = (uptr)__builtin_frame_address(0);
+  // Note: need to check next stack first, because FinishSwitchFiber
+  // may be in process of overwriting stack_top_/bottom_. But in such case
+  // we are already on the next stack.
+  if (cur_stack >= next_stack_bottom_ && cur_stack < next_stack_top_)
+    return {next_stack_bottom_, next_stack_top_};
+  return {stack_bottom_, stack_top_};
+}
+
+uptr Thread::stack_top() { return GetStackBounds().top; }
+
+uptr Thread::stack_bottom() { return GetStackBounds().bottom; }
+
+uptr Thread::stack_size() {
+  const auto bounds = GetStackBounds();
+  return bounds.top - bounds.bottom;
+}
+
 void Thread::Print(const char *Prefix) {
-  Printf("%sT%zd %p stack: [%p,%p) sz: %zd tls: [%p,%p)\n", Prefix, unique_id_,
-         (void *)this, stack_bottom(), stack_top(),
-         stack_top() - stack_bottom(), tls_begin(), tls_end());
+  Printf("%sT%zd %p stack: [%p,%p) sz: %zd tls: [%p,%p)\n", Prefix,
+         (ssize)unique_id_, (void *)this, (void *)stack_bottom(),
+         (void *)stack_top(), stack_top() - stack_bottom(), (void *)tls_begin(),
+         (void *)tls_end());
 }
 
 static u32 xorshift(u32 state) {
@@ -226,3 +280,25 @@ void PrintThreads() {
 }
 
 }  // namespace __lsan
+
+// ---------------------- Interface ---------------- {{{1
+using namespace __hwasan;
+
+extern "C" {
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_start_switch_fiber(void **, const void *bottom, uptr size) {
+  if (auto *t = GetCurrentThread())
+    t->StartSwitchFiber((uptr)bottom, size);
+  else
+    VReport(1, "__hwasan_start_switch_fiber called from unknown thread\n");
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __sanitizer_finish_switch_fiber(void *, const void **bottom_old,
+                                     uptr *size_old) {
+  if (auto *t = GetCurrentThread())
+    t->FinishSwitchFiber((uptr *)bottom_old, size_old);
+  else
+    VReport(1, "__hwasan_finish_switch_fiber called from unknown thread\n");
+}
+}
diff --git a/compiler-rt/lib/hwasan/hwasan_thread.h b/compiler-rt/lib/hwasan/hwasan_thread.h
index 62d6157f98b87..8ef282fd7b10f 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread.h
+++ b/compiler-rt/lib/hwasan/hwasan_thread.h
@@ -41,9 +41,9 @@ class Thread {
 
   void Destroy();
 
-  uptr stack_top() { return stack_top_; }
-  uptr stack_bottom() { return stack_bottom_; }
-  uptr stack_size() { return stack_top() - stack_bottom(); }
+  uptr stack_top();
+  uptr stack_bottom();
+  uptr stack_size();
   uptr tls_begin() { return tls_begin_; }
   uptr tls_end() { return tls_end_; }
   DTLS *dtls() { return dtls_; }
@@ -53,6 +53,9 @@ class Thread {
     return addr >= stack_bottom_ && addr < stack_top_;
   }
 
+  void StartSwitchFiber(uptr bottom, uptr size);
+  void FinishSwitchFiber(uptr *bottom_old, uptr *size_old);
+
   AllocatorCache *allocator_cache() { return &allocator_cache_; }
   HeapAllocationsRingBuffer *heap_allocations() { return heap_allocations_; }
   StackAllocationsRingBuffer *stack_allocations() { return stack_allocations_; }
@@ -80,9 +83,22 @@ class Thread {
   void ClearShadowForThreadStackAndTLS();
   void Print(const char *prefix);
   void InitRandomState();
+
+  struct StackBounds {
+    uptr bottom;
+    uptr top;
+  };
+  StackBounds GetStackBounds() const;
+
   uptr vfork_spill_;
   uptr stack_top_;
   uptr stack_bottom_;
+  // these variables are used when the thread is about to switch stack
+  uptr next_stack_top_;
+  uptr next_stack_bottom_;
+  // true if switching is in progress
+  atomic_uint8_t stack_switching_;
+
   uptr tls_begin_;
   uptr tls_end_;
   DTLS *dtls_;
diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp
index d9afa45a3ad74..39537b67b681d 100644
--- a/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/compiler-rt/lib/lsan/lsan_common.cpp
@@ -806,7 +806,7 @@ static bool ReportUnsuspendedThreads(
       succeded = false;
       Report(
           "Running thread %zu was not suspended. False leaks are possible.\n",
-          os_id);
+          (usize)os_id);
     }
   }
   return succeded;
diff --git a/compiler-rt/lib/memprof/memprof_malloc_linux.cpp b/compiler-rt/lib/memprof/memprof_malloc_linux.cpp
index 2a028c7d0b489..68fe65475889a 100644
--- a/compiler-rt/lib/memprof/memprof_malloc_linux.cpp
+++ b/compiler-rt/lib/memprof/memprof_malloc_linux.cpp
@@ -50,6 +50,24 @@ INTERCEPTOR(void, cfree, void *ptr) {
 }
 #endif // SANITIZER_INTERCEPT_CFREE
 
+#if SANITIZER_INTERCEPT_FREE_SIZED
+INTERCEPTOR(void, free_sized, void *ptr, uptr size) {
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
+  GET_STACK_TRACE_FREE;
+  memprof_delete(ptr, size, 0, &stack, FROM_MALLOC);
+}
+#endif // SANITIZER_INTERCEPT_FREE_SIZED
+
+#if SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED
+INTERCEPTOR(void, free_aligned_sized, void *ptr, uptr alignment, uptr size) {
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
+  GET_STACK_TRACE_FREE;
+  memprof_delete(ptr, size, alignment, &stack, FROM_MALLOC);
+}
+#endif // SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED
+
 INTERCEPTOR(void *, malloc, uptr size) {
   if (DlsymAlloc::Use())
     return DlsymAlloc::Allocate(size);
diff --git a/compiler-rt/lib/memprof/memprof_shadow_setup.cpp b/compiler-rt/lib/memprof/memprof_shadow_setup.cpp
index e7832f656ee8e..7712a94fde3d6 100644
--- a/compiler-rt/lib/memprof/memprof_shadow_setup.cpp
+++ b/compiler-rt/lib/memprof/memprof_shadow_setup.cpp
@@ -29,7 +29,7 @@ static void ProtectGap(uptr addr, uptr size) {
       Printf("protect_shadow_gap=0:"
              " not protecting shadow gap, allocating gap's shadow\n"
              "|| `[%p, %p]` || ShadowGap's shadow ||\n",
-             GapShadowBeg, GapShadowEnd);
+             (void *)GapShadowBeg, (void *)GapShadowEnd);
     ReserveShadowMemoryRange(GapShadowBeg, GapShadowEnd,
                              "unprotected gap shadow");
     return;
diff --git a/compiler-rt/lib/msan/tests/msan_test.cpp b/compiler-rt/lib/msan/tests/msan_test.cpp
index d1c481483dfad..b0d8409d97ffc 100644
--- a/compiler-rt/lib/msan/tests/msan_test.cpp
+++ b/compiler-rt/lib/msan/tests/msan_test.cpp
@@ -4271,14 +4271,39 @@ TEST(VectorSadTest, sse2_psad_bw) {
 }
 
 TEST(VectorMaddTest, mmx_pmadd_wd) {
-  V4x16 a = {Poisoned<U2>(), 1, 2, 3};
+  V4x16 a = {Poisoned<U2>(0), 1, 2, 3};
   V4x16 b = {100, 101, 102, 103};
   V2x32 c = _mm_madd_pi16(a, b);
+  // Multiply step:
+  //    {Poison * 100, 1 * 101, 2 * 102, 3 * 103}
+  // == {Poison,       1 * 101, 2 * 102, 3 * 103}
+  //    Notice that for the poisoned value, we ignored the concrete zero value.
+  //
+  // Horizontal add step:
+  //    {Poison + 1 * 101, 2 * 102 + 3 * 103}
+  // == {Poison,           2 * 102 + 3 * 103}
 
   EXPECT_POISONED(c[0]);
   EXPECT_NOT_POISONED(c[1]);
 
   EXPECT_EQ((unsigned)(2 * 102 + 3 * 103), c[1]);
+
+  V4x16 d = {Poisoned<U2>(0), 1, 0, 3};
+  V4x16 e = {100, 101, Poisoned<U2>(102), 103};
+  V2x32 f = _mm_madd_pi16(d, e);
+  // Multiply step:
+  //    {Poison * 100, 1 * 101, 0 * Poison, 3 * 103}
+  // == {Poison,       1 * 101, 0         , 3 * 103}
+  //    Notice that 0 * Poison == 0.
+  //
+  // Horizontal add step:
+  //    {Poison + 1 * 101, 0 + 3 * 103}
+  // == {Poison,           3 * 103}
+
+  EXPECT_POISONED(f[0]);
+  EXPECT_NOT_POISONED(f[1]);
+
+  EXPECT_EQ((unsigned)(3 * 103), f[1]);
 }
 
 TEST(VectorCmpTest, mm_cmpneq_ps) {
diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
index a9d864e9fe926..d89cec88cc430 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors_posix.cpp
@@ -15,6 +15,7 @@
 
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_allocator_dlsym.h"
+#include "sanitizer_common/sanitizer_glibc_version.h"
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
 
 #include "interception/interception.h"
@@ -46,12 +47,41 @@
 
 using namespace __sanitizer;
 
+DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, usize size)
+DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr)
+
 namespace {
 struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
   static bool UseImpl() { return !__rtsan_is_initialized(); }
 };
 } // namespace
 
+// See note in tsan as to why this is necessary
+static pthread_cond_t *init_cond(pthread_cond_t *c, bool force = false) {
+  if (!common_flags()->legacy_pthread_cond)
+    return c;
+
+  atomic_uintptr_t *p = (atomic_uintptr_t *)c;
+  uptr cond = atomic_load(p, memory_order_acquire);
+  if (!force && cond != 0)
+    return (pthread_cond_t *)cond;
+  void *newcond = WRAP(malloc)(sizeof(pthread_cond_t));
+  internal_memset(newcond, 0, sizeof(pthread_cond_t));
+  if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond,
+                                     memory_order_acq_rel))
+    return (pthread_cond_t *)newcond;
+  WRAP(free)(newcond);
+  return (pthread_cond_t *)cond;
+}
+
+static void destroy_cond(pthread_cond_t *cond) {
+  if (common_flags()->legacy_pthread_cond) {
+    // Free our aux cond and zero the pointer to not leave dangling pointers.
+    WRAP(free)(cond);
+    atomic_store((atomic_uintptr_t *)cond, 0, memory_order_relaxed);
+  }
+}
+
 // Filesystem
 
 INTERCEPTOR(int, open, const char *path, int oflag, ...) {
@@ -766,26 +796,45 @@ INTERCEPTOR(int, pthread_join, pthread_t thread, void **value_ptr) {
   return REAL(pthread_join)(thread, value_ptr);
 }
 
+INTERCEPTOR(int, pthread_cond_init, pthread_cond_t *cond,
+            const pthread_condattr_t *a) {
+  __rtsan_notify_intercepted_call("pthread_cond_init");
+  pthread_cond_t *c = init_cond(cond, true);
+  return REAL(pthread_cond_init)(c, a);
+}
+
 INTERCEPTOR(int, pthread_cond_signal, pthread_cond_t *cond) {
   __rtsan_notify_intercepted_call("pthread_cond_signal");
-  return REAL(pthread_cond_signal)(cond);
+  pthread_cond_t *c = init_cond(cond);
+  return REAL(pthread_cond_signal)(c);
 }
 
 INTERCEPTOR(int, pthread_cond_broadcast, pthread_cond_t *cond) {
   __rtsan_notify_intercepted_call("pthread_cond_broadcast");
-  return REAL(pthread_cond_broadcast)(cond);
+  pthread_cond_t *c = init_cond(cond);
+  return REAL(pthread_cond_broadcast)(c);
 }
 
 INTERCEPTOR(int, pthread_cond_wait, pthread_cond_t *cond,
             pthread_mutex_t *mutex) {
   __rtsan_notify_intercepted_call("pthread_cond_wait");
-  return REAL(pthread_cond_wait)(cond, mutex);
+  pthread_cond_t *c = init_cond(cond);
+  return REAL(pthread_cond_wait)(c, mutex);
 }
 
 INTERCEPTOR(int, pthread_cond_timedwait, pthread_cond_t *cond,
             pthread_mutex_t *mutex, const timespec *ts) {
   __rtsan_notify_intercepted_call("pthread_cond_timedwait");
-  return REAL(pthread_cond_timedwait)(cond, mutex, ts);
+  pthread_cond_t *c = init_cond(cond);
+  return REAL(pthread_cond_timedwait)(c, mutex, ts);
+}
+
+INTERCEPTOR(int, pthread_cond_destroy, pthread_cond_t *cond) {
+  __rtsan_notify_intercepted_call("pthread_cond_destroy");
+  pthread_cond_t *c = init_cond(cond);
+  int res = REAL(pthread_cond_destroy)(c);
+  destroy_cond(c);
+  return res;
 }
 
 INTERCEPTOR(int, pthread_rwlock_rdlock, pthread_rwlock_t *lock) {
@@ -1641,10 +1690,26 @@ void __rtsan::InitializeInterceptors() {
   INTERCEPT_FUNCTION(pthread_mutex_lock);
   INTERCEPT_FUNCTION(pthread_mutex_unlock);
   INTERCEPT_FUNCTION(pthread_join);
+
+  // See the comment in tsan_interceptors_posix.cpp.
+#if SANITIZER_GLIBC && !__GLIBC_PREREQ(2, 36) &&                               \
+    (defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 ||          \
+     defined(__s390x__))
+  INTERCEPT_FUNCTION_VER(pthread_cond_init, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_signal, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_broadcast, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_wait, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_timedwait, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_destroy, "GLIBC_2.3.2");
+#else
+  INTERCEPT_FUNCTION(pthread_cond_init);
   INTERCEPT_FUNCTION(pthread_cond_signal);
   INTERCEPT_FUNCTION(pthread_cond_broadcast);
   INTERCEPT_FUNCTION(pthread_cond_wait);
   INTERCEPT_FUNCTION(pthread_cond_timedwait);
+  INTERCEPT_FUNCTION(pthread_cond_destroy);
+#endif
+
   INTERCEPT_FUNCTION(pthread_rwlock_rdlock);
   INTERCEPT_FUNCTION(pthread_rwlock_unlock);
   INTERCEPT_FUNCTION(pthread_rwlock_wrlock);
diff --git a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
index 9b684e36e3b35..20e3b485f3be0 100644
--- a/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
+++ b/compiler-rt/lib/rtsan/tests/rtsan_test_interceptors_posix.cpp
@@ -753,15 +753,24 @@ TEST_F(RtsanOpenedFileTest, RewindDieWhenRealtime) {
 #endif
 
 TEST_F(RtsanOpenedFileTest, IoctlDiesWhenRealtime) {
-  auto Func = [this]() { ioctl(GetOpenFd(), FIONREAD); };
+  auto Func = [this]() {
+    int arg{};
+    ioctl(GetOpenFd(), FIONREAD, &arg);
+    EXPECT_THAT(arg, Ge(0));
+  };
   ExpectRealtimeDeath(Func, "ioctl");
   ExpectNonRealtimeSurvival(Func);
 }
 
+TEST_F(RtsanOpenedFileTest, IoctlBehavesWithoutOutputArg) {
+  const int result = ioctl(GetOpenFd(), FIONCLEX);
+  EXPECT_THAT(result, Ne(-1));
+}
+
 TEST_F(RtsanOpenedFileTest, IoctlBehavesWithOutputArg) {
   int arg{};
-  ioctl(GetOpenFd(), FIONREAD, &arg);
-
+  const int result = ioctl(GetOpenFd(), FIONREAD, &arg);
+  ASSERT_THAT(result, Ne(-1));
   EXPECT_THAT(arg, Ge(0));
 }
 
@@ -1232,6 +1241,24 @@ TEST(TestRtsanInterceptors, SpinLockLockDiesWhenRealtime) {
 }
 #endif
 
+TEST(TestRtsanInterceptors, PthreadCondInitDiesWhenRealtime) {
+  pthread_cond_t cond{};
+  auto Func = [&cond]() { pthread_cond_init(&cond, nullptr); };
+  ExpectRealtimeDeath(Func, "pthread_cond_init");
+  ExpectNonRealtimeSurvival(Func);
+}
+
+TEST(TestRtsanInterceptors, PthreadCondDestroyDiesWhenRealtime) {
+  pthread_cond_t cond{};
+  ASSERT_EQ(0, pthread_cond_init(&cond, nullptr));
+
+  auto Func = [&cond]() { pthread_cond_destroy(&cond); };
+  ExpectRealtimeDeath(Func, "pthread_cond_destroy");
+  ExpectNonRealtimeSurvival(Func);
+
+  pthread_cond_destroy(&cond);
+}
+
 TEST(TestRtsanInterceptors, PthreadCondSignalDiesWhenRealtime) {
   pthread_cond_t cond{};
   ASSERT_EQ(0, pthread_cond_init(&cond, nullptr));
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
index 51ac1b6ae4975..b39eb1538cbcd 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h
@@ -113,6 +113,24 @@ class SizeClassAllocator64 {
   // ~(uptr)0.
   void Init(s32 release_to_os_interval_ms, uptr heap_start = 0) {
     uptr TotalSpaceSize = kSpaceSize + AdditionalSize();
+
+    uptr MaxAddr = GetMaxUserVirtualAddress();
+    // VReport does not call the sanitizer allocator.
+    VReport(3, "Max user virtual address: 0x%zx\n", MaxAddr);
+    VReport(3, "Total space size for primary allocator: 0x%zx\n",
+            TotalSpaceSize);
+    // TODO: revise the check if we ever configure sanitizers to deliberately
+    //       map beyond the 2**48 barrier (note that Linux pretends the VMA is
+    //       limited to 48-bit for backwards compatibility, but allows apps to
+    //       explicitly specify an address beyond that).
+    if (heap_start + TotalSpaceSize >= MaxAddr) {
+      // We can't easily adjust the requested heap size, because kSpaceSize is
+      // const (for optimization) and used throughout the code.
+      VReport(0, "Error: heap size %zx exceeds max user virtual address %zx\n",
+              TotalSpaceSize, MaxAddr);
+      VReport(
+          0, "Try using a kernel that allows a larger virtual address space\n");
+    }
     PremappedHeap = heap_start != 0;
     if (PremappedHeap) {
       CHECK(!kUsingConstantSpaceBeg);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S
index cdfa6f1d7f53b..5066953980af7 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S
@@ -43,6 +43,6 @@ ASM_SIZE(vfork)
 ASM_INTERCEPTOR_TRAMPOLINE(vfork)
 ASM_TRAMPOLINE_ALIAS(vfork, vfork)
 
-GNU_PROPERTY_BTI_PAC
+GNU_PROPERTY_BTI_PAC_GCS
 
 #endif
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
index 625f30ceca1c2..3c61b60802996 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp
@@ -14,6 +14,7 @@
 #include "sanitizer_fuchsia.h"
 #if SANITIZER_FUCHSIA
 
+#  include <limits.h>
 #  include <pthread.h>
 #  include <stdlib.h>
 #  include <unistd.h>
@@ -117,11 +118,37 @@ uptr GetMmapGranularity() { return _zx_system_get_page_size(); }
 
 sanitizer_shadow_bounds_t ShadowBounds;
 
+// Any sanitizer that utilizes shadow should explicitly call whenever it's
+// appropriate for that sanitizer to reference shadow bounds. For ASan, this is
+// done in `InitializeShadowMemory` and for HWASan, this is done in
+// `InitShadow`.
 void InitShadowBounds() { ShadowBounds = __sanitizer_shadow_bounds(); }
 
+// TODO(leonardchan): It's not immediately clear from a user perspective if
+// `GetMaxUserVirtualAddress` should be called exatly once on runtime startup
+// or can be called multiple times. Currently it looks like most instances of
+// `GetMaxUserVirtualAddress` are meant to be called once, but if someone
+// decides to call this multiple times in the future, we should have a separate
+// function that's ok to call multiple times. Ideally we would just invoke this
+// syscall once. Also for Fuchsia, this syscall technically gets invoked twice
+// since `__sanitizer_shadow_bounds` also invokes this syscall under the hood.
 uptr GetMaxUserVirtualAddress() {
-  InitShadowBounds();
-  return ShadowBounds.memory_limit - 1;
+  zx_info_vmar_t info;
+  zx_status_t status = _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR,
+                                           &info, sizeof(info), NULL, NULL);
+  CHECK_EQ(status, ZX_OK);
+
+  // Find the top of the accessible address space.
+  uintptr_t top = info.base + info.len;
+
+  // Round it up to a power-of-two size.  There may be some pages at
+  // the top that can't actually be mapped, but for purposes of the
+  // the shadow, we'll pretend they could be.
+  int bit = (sizeof(uintptr_t) * CHAR_BIT) - __builtin_clzl(top);
+  if (top != (uintptr_t)1 << bit)
+    top = (uintptr_t)1 << (bit + 1);
+
+  return top - 1;
 }
 
 uptr GetMaxVirtualAddress() { return GetMaxUserVirtualAddress(); }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
index 3bc241525d069..d4811ff4ed217 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp
@@ -769,11 +769,17 @@ void internal_join_thread(void *th) { pthread_join((pthread_t)th, 0); }
 static Mutex syslog_lock;
 #  endif
 
+#  if SANITIZER_DRIVERKIT
+#    define SANITIZER_OS_LOG os_log
+#  else
+#    define SANITIZER_OS_LOG os_log_error
+#  endif
+
 void WriteOneLineToSyslog(const char *s) {
 #if !SANITIZER_GO
   syslog_lock.CheckLocked();
   if (GetMacosAlignedVersion() >= MacosVersion(10, 12)) {
-    os_log_error(OS_LOG_DEFAULT, "%{public}s", s);
+    SANITIZER_OS_LOG(OS_LOG_DEFAULT, "%{public}s", s);
   } else {
 #pragma clang diagnostic push
 // as_log is deprecated.
@@ -837,22 +843,22 @@ void LogMessageOnPrintf(const char *str) {
 
 void LogFullErrorReport(const char *buffer) {
 #  if !SANITIZER_GO
-  // Log with os_log_error. This will make it into the crash log.
+  // When logging with os_log_error this will make it into the crash log.
   if (internal_strncmp(SanitizerToolName, "AddressSanitizer",
                        sizeof("AddressSanitizer") - 1) == 0)
-    os_log_error(OS_LOG_DEFAULT, "Address Sanitizer reported a failure.");
+    SANITIZER_OS_LOG(OS_LOG_DEFAULT, "Address Sanitizer reported a failure.");
   else if (internal_strncmp(SanitizerToolName, "UndefinedBehaviorSanitizer",
                             sizeof("UndefinedBehaviorSanitizer") - 1) == 0)
-    os_log_error(OS_LOG_DEFAULT,
-                 "Undefined Behavior Sanitizer reported a failure.");
+    SANITIZER_OS_LOG(OS_LOG_DEFAULT,
+                     "Undefined Behavior Sanitizer reported a failure.");
   else if (internal_strncmp(SanitizerToolName, "ThreadSanitizer",
                             sizeof("ThreadSanitizer") - 1) == 0)
-    os_log_error(OS_LOG_DEFAULT, "Thread Sanitizer reported a failure.");
+    SANITIZER_OS_LOG(OS_LOG_DEFAULT, "Thread Sanitizer reported a failure.");
   else
-    os_log_error(OS_LOG_DEFAULT, "Sanitizer tool reported a failure.");
+    SANITIZER_OS_LOG(OS_LOG_DEFAULT, "Sanitizer tool reported a failure.");
 
   if (common_flags()->log_to_syslog)
-    os_log_error(OS_LOG_DEFAULT, "Consult syslog for more information.");
+    SANITIZER_OS_LOG(OS_LOG_DEFAULT, "Consult syslog for more information.");
 
   // Log to syslog.
   // The logging on OS X may call pthread_create so we need the threading
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp
index 4c1e005289230..c4fa1e3c1f6fa 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp
@@ -71,14 +71,8 @@
 #include <semaphore.h>
 #include <signal.h>
 #include <stddef.h>
-#include <md5.h>
-#include <sha224.h>
-#include <sha256.h>
-#include <sha384.h>
-#include <sha512.h>
 #include <stdio.h>
 #include <stringlist.h>
-#include <term.h>
 #include <termios.h>
 #include <time.h>
 #include <ttyent.h>
@@ -370,22 +364,6 @@ const int si_SEGV_MAPERR = SEGV_MAPERR;
 const int si_SEGV_ACCERR = SEGV_ACCERR;
 const int unvis_valid = UNVIS_VALID;
 const int unvis_validpush = UNVIS_VALIDPUSH;
-
-const unsigned MD5_CTX_sz = sizeof(MD5_CTX);
-const unsigned MD5_return_length = MD5_DIGEST_STRING_LENGTH;
-
-#define SHA2_CONST(LEN)                                                      \
-  const unsigned SHA##LEN##_CTX_sz = sizeof(SHA##LEN##_CTX);                 \
-  const unsigned SHA##LEN##_return_length = SHA##LEN##_DIGEST_STRING_LENGTH; \
-  const unsigned SHA##LEN##_block_length = SHA##LEN##_BLOCK_LENGTH;          \
-  const unsigned SHA##LEN##_digest_length = SHA##LEN##_DIGEST_LENGTH
-
-SHA2_CONST(224);
-SHA2_CONST(256);
-SHA2_CONST(384);
-SHA2_CONST(512);
-
-#undef SHA2_CONST
 }  // namespace __sanitizer
 
 using namespace __sanitizer;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h
index 382b67ce78ebd..1cbb40e0b2ffc 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h
@@ -710,22 +710,6 @@ extern unsigned IOCTL_KDSKBMODE;
 extern const int si_SEGV_MAPERR;
 extern const int si_SEGV_ACCERR;
 
-extern const unsigned MD5_CTX_sz;
-extern const unsigned MD5_return_length;
-
-#define SHA2_EXTERN(LEN)                          \
-  extern const unsigned SHA##LEN##_CTX_sz;        \
-  extern const unsigned SHA##LEN##_return_length; \
-  extern const unsigned SHA##LEN##_block_length;  \
-  extern const unsigned SHA##LEN##_digest_length
-
-SHA2_EXTERN(224);
-SHA2_EXTERN(256);
-SHA2_EXTERN(384);
-SHA2_EXTERN(512);
-
-#undef SHA2_EXTERN
-
 struct __sanitizer_cap_rights {
   u64 cr_rights[2];
 };
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp
index aacd28c55ceaa..435f3b2861dc9 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp
@@ -498,7 +498,6 @@ struct urio_command {
 #include <md5.h>
 #include <rmd160.h>
 #include <soundcard.h>
-#include <term.h>
 #include <termios.h>
 #include <time.h>
 #include <ttyent.h>
@@ -515,7 +514,7 @@ struct urio_command {
 #include <stringlist.h>
 
 #if defined(__x86_64__)
-#include <nvmm.h>
+#include <dev/nvmm/nvmm_ioctl.h>
 #endif
 // clang-format on
 
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h b/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h
index bda0f04687693..7d88911176577 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h
@@ -15,7 +15,7 @@
 #    define SANITIZER_REDEFINE_BUILTINS_H
 
 // The asm hack only works with GCC and Clang.
-#    if !defined(_WIN32) && !defined(_AIX)
+#    if !defined(_WIN32) && !defined(_AIX) && !defined(__APPLE__)
 
 asm(R"(
     .set memcpy, __sanitizer_internal_memcpy
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
index 5fde65ea48c42..2bf547f4a7216 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cpp
@@ -413,11 +413,12 @@ static void TestPTrace() {
   // internal_fork() on SPARC actually calls __fork(). We can't safely fork,
   // because it's possible seccomp has been configured to disallow fork() but
   // allow clone().
-  Report("WARNING: skipping TestPTrace() because this is SPARC\n");
-  Report(
-      "If seccomp blocks ptrace, LeakSanitizer may hang without further "
-      "notice\n");
-  Report(
+  VReport(1, "WARNING: skipping TestPTrace() because this is SPARC\n");
+  VReport(1,
+          "If seccomp blocks ptrace, LeakSanitizer may hang without further "
+          "notice\n");
+  VReport(
+      1,
       "If seccomp does not block ptrace, you can safely ignore this warning\n");
 #  else
   // Heuristic: only check the first time this is called. This is not always
@@ -438,7 +439,7 @@ static void TestPTrace() {
   if (pid < 0) {
     int rverrno;
     if (internal_iserror(pid, &rverrno))
-      Report("WARNING: TestPTrace() failed to fork (errno %d)\n", rverrno);
+      VReport(0, "WARNING: TestPTrace() failed to fork (errno %d)\n", rverrno);
 
     // We don't abort the sanitizer - it's still worth letting the sanitizer
     // try.
diff --git a/compiler-rt/lib/tsan/dd/dd_interceptors.cpp b/compiler-rt/lib/tsan/dd/dd_interceptors.cpp
index 2c36f691ec5b4..50531ea759a7b 100644
--- a/compiler-rt/lib/tsan/dd/dd_interceptors.cpp
+++ b/compiler-rt/lib/tsan/dd/dd_interceptors.cpp
@@ -11,6 +11,7 @@
 #include "dd_rtl.h"
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_allocator_internal.h"
+#include "sanitizer_common/sanitizer_glibc_version.h"
 #include "sanitizer_common/sanitizer_procmaps.h"
 
 using namespace __dsan;
@@ -312,12 +313,24 @@ void InitializeInterceptors() {
   INTERCEPT_FUNCTION(pthread_rwlock_timedwrlock);
   INTERCEPT_FUNCTION(pthread_rwlock_unlock);
 
+  // See the comment in tsan_interceptors_posix.cpp.
+#if SANITIZER_GLIBC && !__GLIBC_PREREQ(2, 36) &&                      \
+    (defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \
+     defined(__s390x__))
   INTERCEPT_FUNCTION_VER(pthread_cond_init, "GLIBC_2.3.2");
   INTERCEPT_FUNCTION_VER(pthread_cond_signal, "GLIBC_2.3.2");
   INTERCEPT_FUNCTION_VER(pthread_cond_broadcast, "GLIBC_2.3.2");
   INTERCEPT_FUNCTION_VER(pthread_cond_wait, "GLIBC_2.3.2");
   INTERCEPT_FUNCTION_VER(pthread_cond_timedwait, "GLIBC_2.3.2");
   INTERCEPT_FUNCTION_VER(pthread_cond_destroy, "GLIBC_2.3.2");
+#else
+  INTERCEPT_FUNCTION(pthread_cond_init);
+  INTERCEPT_FUNCTION(pthread_cond_signal);
+  INTERCEPT_FUNCTION(pthread_cond_broadcast);
+  INTERCEPT_FUNCTION(pthread_cond_wait);
+  INTERCEPT_FUNCTION(pthread_cond_timedwait);
+  INTERCEPT_FUNCTION(pthread_cond_destroy);
+#endif
 
   // for symbolizer
   INTERCEPT_FUNCTION(realpath);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 795e05394d71a..b46a81031258c 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -22,6 +22,7 @@
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_placement_new.h"
 #include "sanitizer_common/sanitizer_platform_interceptors.h"
 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
@@ -78,17 +79,6 @@ struct ucontext_t {
 };
 #endif
 
-#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \
-    defined(__s390x__)
-#define PTHREAD_ABI_BASE  "GLIBC_2.3.2"
-#elif defined(__aarch64__) || SANITIZER_PPC64V2
-#define PTHREAD_ABI_BASE  "GLIBC_2.17"
-#elif SANITIZER_LOONGARCH64
-#define PTHREAD_ABI_BASE  "GLIBC_2.36"
-#elif SANITIZER_RISCV64
-#  define PTHREAD_ABI_BASE "GLIBC_2.27"
-#endif
-
 extern "C" int pthread_attr_init(void *attr);
 extern "C" int pthread_attr_destroy(void *attr);
 DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *)
@@ -340,11 +330,6 @@ void ScopedInterceptor::DisableIgnoresImpl() {
 }
 
 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
-#if SANITIZER_FREEBSD || SANITIZER_NETBSD
-#  define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
-#else
-#  define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
-#endif
 #if SANITIZER_FREEBSD
 #  define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) \
     INTERCEPT_FUNCTION(_pthread_##func)
@@ -2141,13 +2126,29 @@ static void ReportErrnoSpoiling(ThreadState *thr, uptr pc, int sig) {
   // StackTrace::GetNestInstructionPc(pc) is used because return address is
   // expected, OutputReport() will undo this.
   ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack);
-  ThreadRegistryLock l(&ctx->thread_registry);
-  ScopedReport rep(ReportTypeErrnoInSignal);
-  rep.SetSigNum(sig);
-  if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) {
-    rep.AddStack(stack, true);
-    OutputReport(thr, rep);
+  // Use alloca, because malloc during signal handling deadlocks
+  ScopedReport *rep = (ScopedReport *)__builtin_alloca(sizeof(ScopedReport));
+  bool suppressed;
+  // Take a new scope as Apple platforms require the below locks released
+  // before symbolizing in order to avoid a deadlock
+  {
+    ThreadRegistryLock l(&ctx->thread_registry);
+    new (rep) ScopedReport(ReportTypeErrnoInSignal);
+    rep->SetSigNum(sig);
+    suppressed = IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack);
+    if (!suppressed)
+      rep->AddStack(stack, true);
+#if SANITIZER_APPLE
+  }  // Close this scope to release the locks before writing report
+#endif
+    if (!suppressed)
+      OutputReport(thr, *rep);
+
+    // Need to manually destroy this because we used placement new to allocate
+    rep->~ScopedReport();
+#if !SANITIZER_APPLE
   }
+#endif
 }
 
 static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire,
@@ -3024,12 +3025,26 @@ void InitializeInterceptors() {
   TSAN_INTERCEPT(pthread_timedjoin_np);
   #endif
 
-  TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE);
-  TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE);
-  TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE);
-  TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE);
-  TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE);
-  TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE);
+  // In glibc versions older than 2.36, dlsym(RTLD_NEXT, "pthread_cond_init")
+  // may return an outdated symbol (max(2.2,base_version)) if the port was
+  // introduced before 2.3.2 (when the new pthread_cond_t was introduced).
+#if SANITIZER_GLIBC && !__GLIBC_PREREQ(2, 36) &&                      \
+    (defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \
+     defined(__s390x__))
+  INTERCEPT_FUNCTION_VER(pthread_cond_init, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_signal, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_broadcast, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_wait, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_timedwait, "GLIBC_2.3.2");
+  INTERCEPT_FUNCTION_VER(pthread_cond_destroy, "GLIBC_2.3.2");
+#else
+  INTERCEPT_FUNCTION(pthread_cond_init);
+  INTERCEPT_FUNCTION(pthread_cond_signal);
+  INTERCEPT_FUNCTION(pthread_cond_broadcast);
+  INTERCEPT_FUNCTION(pthread_cond_wait);
+  INTERCEPT_FUNCTION(pthread_cond_timedwait);
+  INTERCEPT_FUNCTION(pthread_cond_destroy);
+#endif
 
   TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT;
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cpp b/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cpp
index befd6a369026d..02ca82369aae1 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cpp
@@ -437,16 +437,30 @@ void __tsan_mutex_post_divert(void *addr, unsigned flagz) {
 }
 
 static void ReportMutexHeldWrongContext(ThreadState *thr, uptr pc) {
-  ThreadRegistryLock l(&ctx->thread_registry);
-  ScopedReport rep(ReportTypeMutexHeldWrongContext);
-  for (uptr i = 0; i < thr->mset.Size(); ++i) {
-    MutexSet::Desc desc = thr->mset.Get(i);
-    rep.AddMutex(desc.addr, desc.stack_id);
+  // Use alloca, because malloc during signal handling deadlocks
+  ScopedReport *rep = (ScopedReport *)__builtin_alloca(sizeof(ScopedReport));
+  // Take a new scope as Apple platforms require the below locks released
+  // before symbolizing in order to avoid a deadlock
+  {
+    ThreadRegistryLock l(&ctx->thread_registry);
+    new (rep) ScopedReport(ReportTypeMutexHeldWrongContext);
+    for (uptr i = 0; i < thr->mset.Size(); ++i) {
+      MutexSet::Desc desc = thr->mset.Get(i);
+      rep->AddMutex(desc.addr, desc.stack_id);
+    }
+    VarSizeStackTrace trace;
+    ObtainCurrentStack(thr, pc, &trace);
+    rep->AddStack(trace, true);
+#if SANITIZER_APPLE
+  }  // Close this scope to release the locks
+#endif
+    OutputReport(thr, *rep);
+
+    // Need to manually destroy this because we used placement new to allocate
+    rep->~ScopedReport();
+#if !SANITIZER_APPLE
   }
-  VarSizeStackTrace trace;
-  ObtainCurrentStack(thr, pc, &trace);
-  rep.AddStack(trace, true);
-  OutputReport(thr, rep);
+#endif
 }
 
 INTERFACE_ATTRIBUTE
diff --git a/compiler-rt/lib/tsan/rtl/tsan_mman.cpp b/compiler-rt/lib/tsan/rtl/tsan_mman.cpp
index 0ea83fb3b5982..caacb3675825f 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_mman.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_mman.cpp
@@ -182,10 +182,24 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
   ObtainCurrentStack(thr, pc, &stack);
   if (IsFiredSuppression(ctx, ReportTypeSignalUnsafe, stack))
     return;
-  ThreadRegistryLock l(&ctx->thread_registry);
-  ScopedReport rep(ReportTypeSignalUnsafe);
-  rep.AddStack(stack, true);
-  OutputReport(thr, rep);
+  // Use alloca, because malloc during signal handling deadlocks
+  ScopedReport *rep = (ScopedReport *)__builtin_alloca(sizeof(ScopedReport));
+  // Take a new scope as Apple platforms require the below locks released
+  // before symbolizing in order to avoid a deadlock
+  {
+    ThreadRegistryLock l(&ctx->thread_registry);
+    new (rep) ScopedReport(ReportTypeSignalUnsafe);
+    rep->AddStack(stack, true);
+#if SANITIZER_APPLE
+  }  // Close this scope to release the locks
+#endif
+    OutputReport(thr, *rep);
+
+    // Need to manually destroy this because we used placement new to allocate
+    rep->~ScopedReport();
+#if !SANITIZER_APPLE
+  }
+#endif
 }
 
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S b/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S
index 7d920bee4a2db..f1d11a3e7f54f 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_aarch64.S
@@ -222,6 +222,6 @@ ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(__sigsetjmp))
 
 NO_EXEC_STACK_DIRECTIVE
 
-GNU_PROPERTY_BTI_PAC
+GNU_PROPERTY_BTI_PAC_GCS
 
 #endif
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
index 77cba5f81e1b8..b2e70475e0b73 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
@@ -419,6 +419,11 @@ NOINLINE void TraceRestartMemoryAccess(ThreadState* thr, uptr pc, uptr addr,
 
 ALWAYS_INLINE USED void MemoryAccess(ThreadState* thr, uptr pc, uptr addr,
                                      uptr size, AccessType typ) {
+#if SANITIZER_APPLE && !SANITIZER_GO
+  // Swift symbolizer can be intercepted and deadlock without this
+  if (thr->in_symbolizer)
+    return;
+#endif
   RawShadow* shadow_mem = MemToShadow(addr);
   UNUSED char memBuf[4][64];
   DPrintf2("#%d: Access: %d@%d %p/%zd typ=0x%x {%s, %s, %s, %s}\n", thr->tid,
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp
index 2a2bf42c92ecb..30f5e964939d0 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_mutex.cpp
@@ -11,14 +11,15 @@
 //===----------------------------------------------------------------------===//
 
 #include <sanitizer_common/sanitizer_deadlock_detector_interface.h>
+#include <sanitizer_common/sanitizer_placement_new.h>
 #include <sanitizer_common/sanitizer_stackdepot.h>
 
-#include "tsan_rtl.h"
 #include "tsan_flags.h"
-#include "tsan_sync.h"
+#include "tsan_platform.h"
 #include "tsan_report.h"
+#include "tsan_rtl.h"
 #include "tsan_symbolize.h"
-#include "tsan_platform.h"
+#include "tsan_sync.h"
 
 namespace __tsan {
 
@@ -55,14 +56,28 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ,
     return;
   if (!ShouldReport(thr, typ))
     return;
-  ThreadRegistryLock l(&ctx->thread_registry);
-  ScopedReport rep(typ);
-  rep.AddMutex(addr, creation_stack_id);
-  VarSizeStackTrace trace;
-  ObtainCurrentStack(thr, pc, &trace);
-  rep.AddStack(trace, true);
-  rep.AddLocation(addr, 1);
-  OutputReport(thr, rep);
+  // Use alloca, because malloc during signal handling deadlocks
+  ScopedReport *rep = (ScopedReport *)__builtin_alloca(sizeof(ScopedReport));
+  // Take a new scope as Apple platforms require the below locks released
+  // before symbolizing in order to avoid a deadlock
+  {
+    ThreadRegistryLock l(&ctx->thread_registry);
+    new (rep) ScopedReport(typ);
+    rep->AddMutex(addr, creation_stack_id);
+    VarSizeStackTrace trace;
+    ObtainCurrentStack(thr, pc, &trace);
+    rep->AddStack(trace, true);
+    rep->AddLocation(addr, 1);
+#if SANITIZER_APPLE
+  }  // Close this scope to release the locks
+#endif
+    OutputReport(thr, *rep);
+
+    // Need to manually destroy this because we used placement new to allocate
+    rep->~ScopedReport();
+#if !SANITIZER_APPLE
+  }
+#endif
 }
 
 static void RecordMutexLock(ThreadState *thr, uptr pc, uptr addr,
@@ -528,53 +543,81 @@ void AfterSleep(ThreadState *thr, uptr pc) {
 void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) {
   if (r == 0 || !ShouldReport(thr, ReportTypeDeadlock))
     return;
-  ThreadRegistryLock l(&ctx->thread_registry);
-  ScopedReport rep(ReportTypeDeadlock);
-  for (int i = 0; i < r->n; i++) {
-    rep.AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]);
-    rep.AddUniqueTid((int)r->loop[i].thr_ctx);
-    rep.AddThread((int)r->loop[i].thr_ctx);
-  }
-  uptr dummy_pc = 0x42;
-  for (int i = 0; i < r->n; i++) {
-    for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
-      u32 stk = r->loop[i].stk[j];
-      StackTrace stack;
-      if (stk && stk != kInvalidStackID) {
-        stack = StackDepotGet(stk);
-      } else {
-        // Sometimes we fail to extract the stack trace (FIXME: investigate),
-        // but we should still produce some stack trace in the report.
-        stack = StackTrace(&dummy_pc, 1);
+  // Use alloca, because malloc during signal handling deadlocks
+  ScopedReport *rep = (ScopedReport *)__builtin_alloca(sizeof(ScopedReport));
+  // Take a new scope as Apple platforms require the below locks released
+  // before symbolizing in order to avoid a deadlock
+  {
+    ThreadRegistryLock l(&ctx->thread_registry);
+    new (rep) ScopedReport(ReportTypeDeadlock);
+    for (int i = 0; i < r->n; i++) {
+      rep->AddMutex(r->loop[i].mtx_ctx0, r->loop[i].stk[0]);
+      rep->AddUniqueTid((int)r->loop[i].thr_ctx);
+      rep->AddThread((int)r->loop[i].thr_ctx);
+    }
+    uptr dummy_pc = 0x42;
+    for (int i = 0; i < r->n; i++) {
+      for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) {
+        u32 stk = r->loop[i].stk[j];
+        StackTrace stack;
+        if (stk && stk != kInvalidStackID) {
+          stack = StackDepotGet(stk);
+        } else {
+          // Sometimes we fail to extract the stack trace (FIXME: investigate),
+          // but we should still produce some stack trace in the report.
+          stack = StackTrace(&dummy_pc, 1);
+        }
+        rep->AddStack(stack, true);
       }
-      rep.AddStack(stack, true);
     }
+#if SANITIZER_APPLE
+  }  // Close this scope to release the locks
+#endif
+    OutputReport(thr, *rep);
+
+    // Need to manually destroy this because we used placement new to allocate
+    rep->~ScopedReport();
+#if !SANITIZER_APPLE
   }
-  OutputReport(thr, rep);
+#endif
 }
 
 void ReportDestroyLocked(ThreadState *thr, uptr pc, uptr addr,
                          FastState last_lock, StackID creation_stack_id) {
-  // We need to lock the slot during RestoreStack because it protects
-  // the slot journal.
-  Lock slot_lock(&ctx->slots[static_cast<uptr>(last_lock.sid())].mtx);
-  ThreadRegistryLock l0(&ctx->thread_registry);
-  Lock slots_lock(&ctx->slot_mtx);
-  ScopedReport rep(ReportTypeMutexDestroyLocked);
-  rep.AddMutex(addr, creation_stack_id);
-  VarSizeStackTrace trace;
-  ObtainCurrentStack(thr, pc, &trace);
-  rep.AddStack(trace, true);
-
-  Tid tid;
-  DynamicMutexSet mset;
-  uptr tag;
-  if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(), addr,
-                    0, kAccessWrite, &tid, &trace, mset, &tag))
-    return;
-  rep.AddStack(trace, true);
-  rep.AddLocation(addr, 1);
-  OutputReport(thr, rep);
+  // Use alloca, because malloc during signal handling deadlocks
+  ScopedReport *rep = (ScopedReport *)__builtin_alloca(sizeof(ScopedReport));
+  // Take a new scope as Apple platforms require the below locks released
+  // before symbolizing in order to avoid a deadlock
+  {
+    // We need to lock the slot during RestoreStack because it protects
+    // the slot journal.
+    Lock slot_lock(&ctx->slots[static_cast<uptr>(last_lock.sid())].mtx);
+    ThreadRegistryLock l0(&ctx->thread_registry);
+    Lock slots_lock(&ctx->slot_mtx);
+    new (rep) ScopedReport(ReportTypeMutexDestroyLocked);
+    rep->AddMutex(addr, creation_stack_id);
+    VarSizeStackTrace trace;
+    ObtainCurrentStack(thr, pc, &trace);
+    rep->AddStack(trace, true);
+
+    Tid tid;
+    DynamicMutexSet mset;
+    uptr tag;
+    if (!RestoreStack(EventType::kLock, last_lock.sid(), last_lock.epoch(),
+                      addr, 0, kAccessWrite, &tid, &trace, mset, &tag))
+      return;
+    rep->AddStack(trace, true);
+    rep->AddLocation(addr, 1);
+#if SANITIZER_APPLE
+  }  // Close this scope to release the locks
+#endif
+    OutputReport(thr, *rep);
+
+    // Need to manually destroy this because we used placement new to allocate
+    rep->~ScopedReport();
+#if !SANITIZER_APPLE
+  }
+#endif
 }
 
 }  // namespace __tsan
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp
index e6f0fda9c72af..43aef30d2f3b3 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_report.cpp
@@ -16,6 +16,7 @@
 #include "sanitizer_common/sanitizer_placement_new.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 #include "sanitizer_common/sanitizer_stacktrace.h"
+#include "tsan_defs.h"
 #include "tsan_fd.h"
 #include "tsan_flags.h"
 #include "tsan_mman.h"
@@ -806,65 +807,80 @@ void ReportRace(ThreadState *thr, RawShadow *shadow_mem, Shadow cur, Shadow old,
   DynamicMutexSet mset1;
   MutexSet *mset[kMop] = {&thr->mset, mset1};
 
-  // We need to lock the slot during RestoreStack because it protects
-  // the slot journal.
-  Lock slot_lock(&ctx->slots[static_cast<uptr>(s[1].sid())].mtx);
-  ThreadRegistryLock l0(&ctx->thread_registry);
-  Lock slots_lock(&ctx->slot_mtx);
-  if (SpuriousRace(old))
-    return;
-  if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1,
-                    size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) {
-    StoreShadow(&ctx->last_spurious_race, old.raw());
-    return;
-  }
+  // Use alloca, because malloc during signal handling deadlocks
+  ScopedReport *rep = (ScopedReport *)__builtin_alloca(sizeof(ScopedReport));
+  // Take a new scope as Apple platforms require the below locks released
+  // before symbolizing in order to avoid a deadlock
+  {
+    // We need to lock the slot during RestoreStack because it protects
+    // the slot journal.
+    Lock slot_lock(&ctx->slots[static_cast<uptr>(s[1].sid())].mtx);
+    ThreadRegistryLock l0(&ctx->thread_registry);
+    Lock slots_lock(&ctx->slot_mtx);
+    if (SpuriousRace(old))
+      return;
+    if (!RestoreStack(EventType::kAccessExt, s[1].sid(), s[1].epoch(), addr1,
+                      size1, typ1, &tids[1], &traces[1], mset[1], &tags[1])) {
+      StoreShadow(&ctx->last_spurious_race, old.raw());
+      return;
+    }
 
-  if (IsFiredSuppression(ctx, rep_typ, traces[1]))
-    return;
+    if (IsFiredSuppression(ctx, rep_typ, traces[1]))
+      return;
 
-  if (HandleRacyStacks(thr, traces))
-    return;
+    if (HandleRacyStacks(thr, traces))
+      return;
 
-  // If any of the accesses has a tag, treat this as an "external" race.
-  uptr tag = kExternalTagNone;
-  for (uptr i = 0; i < kMop; i++) {
-    if (tags[i] != kExternalTagNone) {
-      rep_typ = ReportTypeExternalRace;
-      tag = tags[i];
-      break;
+    // If any of the accesses has a tag, treat this as an "external" race.
+    uptr tag = kExternalTagNone;
+    for (uptr i = 0; i < kMop; i++) {
+      if (tags[i] != kExternalTagNone) {
+        rep_typ = ReportTypeExternalRace;
+        tag = tags[i];
+        break;
+      }
     }
-  }
 
-  ScopedReport rep(rep_typ, tag);
-  for (uptr i = 0; i < kMop; i++)
-    rep.AddMemoryAccess(addr, tags[i], s[i], tids[i], traces[i], mset[i]);
+    new (rep) ScopedReport(rep_typ, tag);
+    for (uptr i = 0; i < kMop; i++)
+      rep->AddMemoryAccess(addr, tags[i], s[i], tids[i], traces[i], mset[i]);
 
-  for (uptr i = 0; i < kMop; i++) {
-    ThreadContext *tctx = static_cast<ThreadContext *>(
-        ctx->thread_registry.GetThreadLocked(tids[i]));
-    rep.AddThread(tctx);
-  }
+    for (uptr i = 0; i < kMop; i++) {
+      ThreadContext *tctx = static_cast<ThreadContext *>(
+          ctx->thread_registry.GetThreadLocked(tids[i]));
+      rep->AddThread(tctx);
+    }
 
-  rep.AddLocation(addr_min, addr_max - addr_min);
-
-  if (flags()->print_full_thread_history) {
-    const ReportDesc *rep_desc = rep.GetReport();
-    for (uptr i = 0; i < rep_desc->threads.Size(); i++) {
-      Tid parent_tid = rep_desc->threads[i]->parent_tid;
-      if (parent_tid == kMainTid || parent_tid == kInvalidTid)
-        continue;
-      ThreadContext *parent_tctx = static_cast<ThreadContext *>(
-          ctx->thread_registry.GetThreadLocked(parent_tid));
-      rep.AddThread(parent_tctx);
+    rep->AddLocation(addr_min, addr_max - addr_min);
+
+    if (flags()->print_full_thread_history) {
+      const ReportDesc *rep_desc = rep->GetReport();
+      for (uptr i = 0; i < rep_desc->threads.Size(); i++) {
+        Tid parent_tid = rep_desc->threads[i]->parent_tid;
+        if (parent_tid == kMainTid || parent_tid == kInvalidTid)
+          continue;
+        ThreadContext *parent_tctx = static_cast<ThreadContext *>(
+            ctx->thread_registry.GetThreadLocked(parent_tid));
+        rep->AddThread(parent_tctx);
+      }
     }
-  }
 
 #if !SANITIZER_GO
-  if (!((typ0 | typ1) & kAccessFree) &&
-      s[1].epoch() <= thr->last_sleep_clock.Get(s[1].sid()))
-    rep.AddSleep(thr->last_sleep_stack_id);
+    if (!((typ0 | typ1) & kAccessFree) &&
+        s[1].epoch() <= thr->last_sleep_clock.Get(s[1].sid()))
+      rep->AddSleep(thr->last_sleep_stack_id);
+#endif
+
+#if SANITIZER_APPLE
+  }  // Close this scope to release the locks
+#endif
+    OutputReport(thr, *rep);
+
+    // Need to manually destroy this because we used placement new to allocate
+    rep->~ScopedReport();
+#if !SANITIZER_APPLE
+  }
 #endif
-  OutputReport(thr, rep);
 }
 
 void PrintCurrentStack(ThreadState *thr, uptr pc) {
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
index c6a8fd2acb6a8..b1464ccfddeb4 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cpp
@@ -88,15 +88,33 @@ void ThreadFinalize(ThreadState *thr) {
 #if !SANITIZER_GO
   if (!ShouldReport(thr, ReportTypeThreadLeak))
     return;
-  ThreadRegistryLock l(&ctx->thread_registry);
   Vector<ThreadLeak> leaks;
-  ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks,
-                                                      &leaks);
+  {
+    ThreadRegistryLock l(&ctx->thread_registry);
+    ctx->thread_registry.RunCallbackForEachThreadLocked(CollectThreadLeaks,
+                                                        &leaks);
+  }
+
   for (uptr i = 0; i < leaks.Size(); i++) {
-    ScopedReport rep(ReportTypeThreadLeak);
-    rep.AddThread(leaks[i].tctx, true);
-    rep.SetCount(leaks[i].count);
-    OutputReport(thr, rep);
+    // Use alloca, because malloc during signal handling deadlocks
+    ScopedReport *rep = (ScopedReport *)__builtin_alloca(sizeof(ScopedReport));
+    // Take a new scope as Apple platforms require the below locks released
+    // before symbolizing in order to avoid a deadlock
+    {
+      ThreadRegistryLock l(&ctx->thread_registry);
+      new (rep) ScopedReport(ReportTypeThreadLeak);
+      rep->AddThread(leaks[i].tctx, true);
+      rep->SetCount(leaks[i].count);
+#  if SANITIZER_APPLE
+    }  // Close this scope to release the locks
+#  endif
+      OutputReport(thr, *rep);
+
+      // Need to manually destroy this because we used placement new to allocate
+      rep->~ScopedReport();
+#  if !SANITIZER_APPLE
+    }
+#  endif
   }
 #endif
 }
diff --git a/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cpp b/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cpp
index ebc36a8583e05..1e143075f1bee 100644
--- a/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cpp
+++ b/compiler-rt/lib/ubsan_minimal/ubsan_minimal_handlers.cpp
@@ -34,12 +34,16 @@ static char *append_hex(uintptr_t d, char *buf, const char *end) {
   return buf;
 }
 
-static void format_msg(const char *kind, uintptr_t caller, char *buf,
-                       const char *end) {
+static void format_msg(const char *kind, uintptr_t caller,
+                       const uintptr_t *address, char *buf, const char *end) {
   buf = append_str("ubsan: ", buf, end);
   buf = append_str(kind, buf, end);
   buf = append_str(" by 0x", buf, end);
   buf = append_hex(caller, buf, end);
+  if (address) {
+    buf = append_str(" address 0x", buf, end);
+    buf = append_hex(*address, buf, end);
+  }
   buf = append_str("\n", buf, end);
   if (buf == end)
     --buf; // Make sure we don't cause a buffer overflow.
@@ -47,7 +51,7 @@ static void format_msg(const char *kind, uintptr_t caller, char *buf,
 }
 
 SANITIZER_INTERFACE_WEAK_DEF(void, __ubsan_report_error, const char *kind,
-                             uintptr_t caller) {
+                             uintptr_t caller, const uintptr_t *address) {
   if (caller == 0)
     return;
   while (true) {
@@ -80,28 +84,32 @@ SANITIZER_INTERFACE_WEAK_DEF(void, __ubsan_report_error, const char *kind,
     __sanitizer::atomic_store_relaxed(&caller_pcs[sz], caller);
 
     char msg_buf[128];
-    format_msg(kind, caller, msg_buf, msg_buf + sizeof(msg_buf));
+    format_msg(kind, caller, address, msg_buf, msg_buf + sizeof(msg_buf));
     message(msg_buf);
   }
 }
 
 SANITIZER_INTERFACE_WEAK_DEF(void, __ubsan_report_error_fatal, const char *kind,
-                             uintptr_t caller) {
+                             uintptr_t caller, const uintptr_t *address) {
   // Use another handlers, in case it's already overriden.
-  __ubsan_report_error(kind, caller);
+  __ubsan_report_error(kind, caller, address);
 }
 
 #if defined(__ANDROID__)
 extern "C" __attribute__((weak)) void android_set_abort_message(const char *);
-static void abort_with_message(const char *kind, uintptr_t caller) {
+static void abort_with_message(const char *kind, uintptr_t caller,
+                               const uintptr_t *address) {
   char msg_buf[128];
-  format_msg(kind, caller, msg_buf, msg_buf + sizeof(msg_buf));
+  format_msg(kind, caller, address, msg_buf, msg_buf + sizeof(msg_buf));
   if (&android_set_abort_message)
     android_set_abort_message(msg_buf);
   abort();
 }
 #else
-static void abort_with_message(const char *kind, uintptr_t caller) { abort(); }
+static void abort_with_message(const char *kind, uintptr_t caller,
+                               const uintptr_t *address) {
+  abort();
+}
 #endif
 
 #if SANITIZER_DEBUG
@@ -117,25 +125,42 @@ void NORETURN CheckFailed(const char *file, int, const char *cond, u64, u64) {
 } // namespace __sanitizer
 #endif
 
-#define INTERFACE extern "C" __attribute__((visibility("default")))
-
 #define HANDLER_RECOVER(name, kind)                                            \
-  INTERFACE void __ubsan_handle_##name##_minimal() {                           \
-    __ubsan_report_error(kind, GET_CALLER_PC());                               \
+  SANITIZER_INTERFACE_WEAK_DEF(void, __ubsan_handle_##name##_minimal) {        \
+    __ubsan_report_error(kind, GET_CALLER_PC(), nullptr);                      \
   }
 
 #define HANDLER_NORECOVER(name, kind)                                          \
-  INTERFACE void __ubsan_handle_##name##_minimal_abort() {                     \
+  SANITIZER_INTERFACE_WEAK_DEF(void, __ubsan_handle_##name##_minimal_abort) {  \
     uintptr_t caller = GET_CALLER_PC();                                        \
-    __ubsan_report_error_fatal(kind, caller);                                  \
-    abort_with_message(kind, caller);                                          \
+    __ubsan_report_error_fatal(kind, caller, nullptr);                         \
+    abort_with_message(kind, caller, nullptr);                                 \
   }
 
 #define HANDLER(name, kind)                                                    \
   HANDLER_RECOVER(name, kind)                                                  \
   HANDLER_NORECOVER(name, kind)
 
-HANDLER(type_mismatch, "type-mismatch")
+#define HANDLER_RECOVER_PTR(name, kind)                                        \
+  SANITIZER_INTERFACE_WEAK_DEF(void, __ubsan_handle_##name##_minimal,          \
+                               const uintptr_t address) {                      \
+    __ubsan_report_error(kind, GET_CALLER_PC(), &address);                     \
+  }
+
+#define HANDLER_NORECOVER_PTR(name, kind)                                      \
+  SANITIZER_INTERFACE_WEAK_DEF(void, __ubsan_handle_##name##_minimal_abort,    \
+                               const uintptr_t address) {                      \
+    uintptr_t caller = GET_CALLER_PC();                                        \
+    __ubsan_report_error_fatal(kind, caller, &address);                        \
+    abort_with_message(kind, caller, &address);                                \
+  }
+
+// A version of a handler that takes a pointer to a value.
+#define HANDLER_PTR(name, kind)                                                \
+  HANDLER_RECOVER_PTR(name, kind)                                              \
+  HANDLER_NORECOVER_PTR(name, kind)
+
+HANDLER_PTR(type_mismatch, "type-mismatch")
 HANDLER(alignment_assumption, "alignment-assumption")
 HANDLER(add_overflow, "add-overflow")
 HANDLER(sub_overflow, "sub-overflow")
diff --git a/compiler-rt/lib/xray/xray_fdr_logging.cpp b/compiler-rt/lib/xray/xray_fdr_logging.cpp
index 7def3565d56ab..977a0b9b87f4e 100644
--- a/compiler-rt/lib/xray/xray_fdr_logging.cpp
+++ b/compiler-rt/lib/xray/xray_fdr_logging.cpp
@@ -73,7 +73,7 @@ static_assert(std::is_trivially_destructible<ThreadLocalData>::value,
 static pthread_key_t Key;
 
 // Global BufferQueue.
-static std::byte BufferQueueStorage[sizeof(BufferQueue)];
+alignas(BufferQueue) static std::byte BufferQueueStorage[sizeof(BufferQueue)];
 static BufferQueue *BQ = nullptr;
 
 // Global thresholds for function durations.
diff --git a/compiler-rt/lib/xray/xray_init.cpp b/compiler-rt/lib/xray/xray_init.cpp
index 020bfe52b5320..9cc6d5fcc4c1d 100644
--- a/compiler-rt/lib/xray/xray_init.cpp
+++ b/compiler-rt/lib/xray/xray_init.cpp
@@ -105,7 +105,7 @@ __xray_register_sleds(const XRaySledEntry *SledsBegin,
   }
 
   if (Verbosity())
-    Report("Registering %d new functions!\n", SledMap.Functions);
+    Report("Registering %d new functions!\n", (int)SledMap.Functions);
 
   {
     SpinMutexLock Guard(&XRayInstrMapMutex);
diff --git a/compiler-rt/lib/xray/xray_interface.cpp b/compiler-rt/lib/xray/xray_interface.cpp
index 3f97827874a70..9bf0c56c4521a 100644
--- a/compiler-rt/lib/xray/xray_interface.cpp
+++ b/compiler-rt/lib/xray/xray_interface.cpp
@@ -308,7 +308,8 @@ XRayPatchingStatus controlPatchingObjectUnchecked(bool Enable, int32_t ObjId) {
     return XRayPatchingStatus::NOT_INITIALIZED;
 
   if (Verbosity())
-    Report("Patching object %d with %d functions.\n", ObjId, InstrMap.Entries);
+    Report("Patching object %d with %d functions.\n", ObjId,
+           (int)InstrMap.Entries);
 
   // Check if the corresponding DSO has been unloaded.
   if (!InstrMap.Loaded) {
diff --git a/compiler-rt/test/asan/TestCases/Darwin/duplicate_os_log_reports.cpp b/compiler-rt/test/asan/TestCases/Darwin/duplicate_os_log_reports.cpp
index dd5a257e39855..43ca027c970cd 100644
--- a/compiler-rt/test/asan/TestCases/Darwin/duplicate_os_log_reports.cpp
+++ b/compiler-rt/test/asan/TestCases/Darwin/duplicate_os_log_reports.cpp
@@ -1,6 +1,4 @@
 // UNSUPPORTED: ios
-// Don't re-enable until rdar://problem/62141527 is fixed.
-// REQUIRES: rdar_62141527
 // REQUIRES: shell
 // REQUIRES: darwin_log_cmd
 // RUN: %clangxx_asan -fsanitize-recover=address %s -o %t
@@ -16,6 +14,7 @@
 // RUN: FileCheck %s -input-file=%t.process_syslog_output.txt
 #include <cassert>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <sanitizer/asan_interface.h>
 
diff --git a/compiler-rt/test/asan/TestCases/Darwin/suppressions-sandbox.cpp b/compiler-rt/test/asan/TestCases/Darwin/suppressions-sandbox.cpp
index 651d0c5d05b07..f12e2b2ada50d 100644
--- a/compiler-rt/test/asan/TestCases/Darwin/suppressions-sandbox.cpp
+++ b/compiler-rt/test/asan/TestCases/Darwin/suppressions-sandbox.cpp
@@ -15,6 +15,9 @@
 // sandbox-exec isn't available on iOS
 // UNSUPPORTED: ios
 
+// Symbolizer fails to find test functions on current macOS bot version
+// XFAIL: system-darwin && target=arm{{.*}}
+
 #include <CoreFoundation/CoreFoundation.h>
 
 #if defined(SHARED_LIB)
diff --git a/compiler-rt/test/asan/TestCases/Linux/release_to_os_test.cpp b/compiler-rt/test/asan/TestCases/Linux/release_to_os_test.cpp
index 3e28ffde46ab6..dc3ead9e8436c 100644
--- a/compiler-rt/test/asan/TestCases/Linux/release_to_os_test.cpp
+++ b/compiler-rt/test/asan/TestCases/Linux/release_to_os_test.cpp
@@ -6,6 +6,7 @@
 // RUN: %env_asan_opts=allocator_release_to_os_interval_ms=-1 %run %t force 2>&1 | FileCheck %s --check-prefix=FORCE_RELEASE
 
 // REQUIRES: x86_64-target-arch
+// REQUIRES: page-size-4096
 
 #include <algorithm>
 #include <assert.h>
diff --git a/compiler-rt/test/asan/TestCases/Posix/asan_symbolize_script/anon_wrong_frame_number.cpp b/compiler-rt/test/asan/TestCases/Posix/asan_symbolize_script/anon_wrong_frame_number.cpp
new file mode 100644
index 0000000000000..aa931f61e2211
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Posix/asan_symbolize_script/anon_wrong_frame_number.cpp
@@ -0,0 +1,69 @@
+// This test case checks for a bug where anonymous code results in
+// out-of-order stack frame numbers.
+
+// UNSUPPORTED: android
+// UNSUPPORTED: aarch64
+// UNSUPPORTED: darwin
+// UNSUPPORTED: ios
+// REQUIRES: x86-64-registered-target
+
+// RUN: %clangxx_asan -O0 -g %s -o %t
+// RUN: %env_asan_opts=symbolize=0 not %run %t DUMMY_ARG > %t.asan_report 2>&1
+// RUN: %asan_symbolize -d --log-level debug --log-dest %t_debug_log_output.txt -l %t.asan_report > %t.asan_report_sym
+// RUN: FileCheck --input-file=%t.asan_report_sym %s
+
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <sys/mman.h>
+#include <unistd.h>
+
+static void call_via_anon_page(void (*fn)()) {
+  const size_t pagesz = static_cast<size_t>(sysconf(_SC_PAGESIZE));
+  uint8_t *mem =
+      static_cast<uint8_t *>(mmap(nullptr, pagesz, PROT_READ | PROT_WRITE,
+                                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
+  if (mem == MAP_FAILED)
+    perror("mmap");
+
+#if defined(__x86_64__)
+  // x86_64: mov rax, imm64; call rax; ret
+  // 48 B8 <imm64> FF D0 C3
+  uint8_t stub[2 + 8 + 2 + 1] = {0x48, 0xB8};
+  std::memcpy(stub + 2, &fn, sizeof(fn)); // imm64
+  stub[2 + 8] = 0xFF;
+  stub[2 + 9] = 0xD0;  // call rax
+  stub[2 + 10] = 0xC3; // ret
+#else
+#  error "unsupported platform"
+#endif
+
+  std::memcpy(mem, stub, sizeof(stub));
+  mprotect(mem, pagesz, PROT_READ | PROT_EXEC);
+
+  using Thunk = void (*)();
+  reinterpret_cast<Thunk>(mem)();
+
+  munmap(mem, pagesz);
+}
+
+static void crash() {
+  char p[8], *s = p;
+
+  // out-of-bounds write to trigger ASan
+  s[16] = 42;
+}
+
+int main() {
+  call_via_anon_page(crash);
+  return 0;
+}
+
+// Check that the numbering of the stackframes is correct.
+
+// CHECK: AddressSanitizer: stack-buffer-overflow
+// CHECK-NEXT: WRITE of size
+// CHECK-NEXT: #0 0x{{[0-9a-fA-F]+}} in crash
+// CHECK-NEXT: #1 0x{{[0-9a-fA-F]+}}
+// CHECK-NEXT: #2 0x{{[0-9a-fA-F]+}} in main
diff --git a/compiler-rt/test/asan/TestCases/Posix/fakestack_alignment.cpp b/compiler-rt/test/asan/TestCases/Posix/fakestack_alignment.cpp
new file mode 100644
index 0000000000000..b7ce6bc7a7673
--- /dev/null
+++ b/compiler-rt/test/asan/TestCases/Posix/fakestack_alignment.cpp
@@ -0,0 +1,69 @@
+// Regression test 1:
+// When the stack size is 1<<16, SizeRequiredForFlags(16) == 2KB. This forces
+// FakeStack's GetFrame() out of alignment if the FakeStack isn't padded.
+// RUN: %clangxx_asan -pthread -fsanitize-address-use-after-return=always -O0 -DALIGNMENT=4096  -DTHREAD_COUNT=1 -DTHREAD_STACK_SIZE=65536 %s -o %t && %run %t 2>&1
+
+// Regression test 2:
+// Check that the FakeStack frame is aligned, beyond the typical 4KB page
+// alignment. Alignment can happen by chance, so try this on many threads.
+// RUN: %clangxx_asan -pthread -fsanitize-address-use-after-return=always -O0 -DALIGNMENT=8192  -DTHREAD_COUNT=32 -DTHREAD_STACK_SIZE=131072 %s -o %t && %run %t 2>&1
+// RUN: %clangxx_asan -pthread -fsanitize-address-use-after-return=always -O0 -DALIGNMENT=16384 -DTHREAD_COUNT=32 -DTHREAD_STACK_SIZE=131072 %s -o %t && %run %t 2>&1
+
+// Extra tests:
+// RUN: %clangxx_asan -pthread -fsanitize-address-use-after-return=always -O0 -DALIGNMENT=4096  -DTHREAD_COUNT=32 -DTHREAD_STACK_SIZE=65536 %s -o %t && %run %t 2>&1
+// RUN: %clangxx_asan -pthread -fsanitize-address-use-after-return=always -O0 -DALIGNMENT=8192  -DTHREAD_COUNT=32 -DTHREAD_STACK_SIZE=65536 %s -o %t && %run %t 2>&1
+// RUN: %clangxx_asan -pthread -fsanitize-address-use-after-return=always -O0 -DALIGNMENT=16384 -DTHREAD_COUNT=32 -DTHREAD_STACK_SIZE=65536 %s -o %t && %run %t 2>&1
+// RUN: %clangxx_asan -pthread -fsanitize-address-use-after-return=always -O0 -DALIGNMENT=4096  -DTHREAD_COUNT=32 -DTHREAD_STACK_SIZE=131072 %s -o %t && %run %t 2>&1
+// RUN: %clangxx_asan -pthread -fsanitize-address-use-after-return=always -O0 -DALIGNMENT=8192  -DTHREAD_COUNT=32 -DTHREAD_STACK_SIZE=131072 %s -o %t && %run %t 2>&1
+// RUN: %clangxx_asan -pthread -fsanitize-address-use-after-return=always -O0 -DALIGNMENT=16384 -DTHREAD_COUNT=32 -DTHREAD_STACK_SIZE=131072 %s -o %t && %run %t 2>&1
+
+// UNSUPPORTED: android
+
+#include <assert.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+struct alignas(ALIGNMENT) big_object {
+  int x;
+};
+
+bool misaligned = false;
+
+// Check whether the FakeStack frame is sufficiently aligned. Alignment can
+// happen by chance, so try this on many threads.
+void *Thread(void *unused) {
+  big_object x;
+  uintptr_t alignment = (uintptr_t)&x % alignof(big_object);
+
+  if (alignment != 0)
+    misaligned = true;
+
+  return nullptr;
+}
+
+int main(int argc, char **argv) {
+  pthread_attr_t attr;
+  pthread_attr_init(&attr);
+#ifdef THREAD_STACK_SIZE
+  pthread_attr_setstacksize(&attr, THREAD_STACK_SIZE);
+#endif
+
+  pthread_t threads[THREAD_COUNT];
+  for (pthread_t &t : threads)
+    pthread_create(&t, &attr, Thread, 0);
+
+  pthread_attr_destroy(&attr);
+
+  for (pthread_t &t : threads)
+    pthread_join(t, 0);
+
+  if (misaligned) {
+    printf("Test failed: not perfectly aligned\n");
+    exit(1);
+  }
+
+  return 0;
+}
diff --git a/compiler-rt/test/asan/TestCases/Posix/fread_fwrite.cpp b/compiler-rt/test/asan/TestCases/Posix/fread_fwrite.cpp
index c0629260418a3..c7b9280ea7d8e 100644
--- a/compiler-rt/test/asan/TestCases/Posix/fread_fwrite.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/fread_fwrite.cpp
@@ -2,6 +2,9 @@
 // RUN: not %run %t 2>&1 | FileCheck %s --check-prefix=CHECK-FWRITE
 // RUN: not %run %t 1 2>&1 | FileCheck %s --check-prefix=CHECK-FREAD
 
+// Symbolizer fails to find test functions on current macOS bot version
+// XFAIL: system-darwin && target=arm{{.*}}
+
 #include <stdio.h>
 #include <stdlib.h>
 
diff --git a/compiler-rt/test/asan/TestCases/Posix/new_array_cookie_test.cpp b/compiler-rt/test/asan/TestCases/Posix/new_array_cookie_test.cpp
index edbdb4016d86e..fa896128cdb72 100644
--- a/compiler-rt/test/asan/TestCases/Posix/new_array_cookie_test.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/new_array_cookie_test.cpp
@@ -6,6 +6,9 @@
 
 // UNSUPPORTED: ios
 
+// Poisoning C++ array redzones is not implemented on arm
+// XFAIL: target=arm{{.*}}
+
 #include <stdio.h>
 #include <stdlib.h>
 struct C {
diff --git a/compiler-rt/test/asan/TestCases/Posix/new_array_cookie_uaf_test.cpp b/compiler-rt/test/asan/TestCases/Posix/new_array_cookie_uaf_test.cpp
index 335a56757d296..63b1171df0aef 100644
--- a/compiler-rt/test/asan/TestCases/Posix/new_array_cookie_uaf_test.cpp
+++ b/compiler-rt/test/asan/TestCases/Posix/new_array_cookie_uaf_test.cpp
@@ -5,6 +5,9 @@
 
 // UNSUPPORTED: ios
 
+// Poisoning C++ array redzones is not implemented on arm
+// XFAIL: target=arm{{.*}}
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <assert.h>
diff --git a/compiler-rt/test/asan/TestCases/Windows/fuse-lld-globals.cpp b/compiler-rt/test/asan/TestCases/Windows/fuse-lld-globals.cpp
index 3ee413b7dfd4e..97f3783cb58c8 100644
--- a/compiler-rt/test/asan/TestCases/Windows/fuse-lld-globals.cpp
+++ b/compiler-rt/test/asan/TestCases/Windows/fuse-lld-globals.cpp
@@ -1,3 +1,5 @@
+// REQUIRES: lld-available
+
 // RUN: %clangxx_asan -fuse-ld=lld -O3 %s -o %t && not %run %t 2>&1 | FileCheck %s
 
 #include <string.h>
diff --git a/compiler-rt/test/cfi/cross-dso/lit.local.cfg.py b/compiler-rt/test/cfi/cross-dso/lit.local.cfg.py
index 2778d8c995fd1..bd0fabd1f26df 100644
--- a/compiler-rt/test/cfi/cross-dso/lit.local.cfg.py
+++ b/compiler-rt/test/cfi/cross-dso/lit.local.cfg.py
@@ -12,3 +12,7 @@ def getRoot(config):
 # Android O (API level 26) has support for cross-dso cfi in libdl.so.
 if config.android and "android-26" not in config.available_features:
     config.unsupported = True
+
+# The runtime library only supports 4K pages.
+if "page-size-4096" not in config.available_features:
+    config.unsupported = True
diff --git a/compiler-rt/test/dfsan/atomic.cpp b/compiler-rt/test/dfsan/atomic.cpp
index 22ee323c752f8..73e1cbd17a7cd 100644
--- a/compiler-rt/test/dfsan/atomic.cpp
+++ b/compiler-rt/test/dfsan/atomic.cpp
@@ -1,9 +1,12 @@
-// RUN: %clangxx_dfsan %s -fno-exceptions -o %t && %run %t
-// RUN: %clangxx_dfsan -DORIGIN_TRACKING -mllvm -dfsan-track-origins=1 %s -fno-exceptions -o %t && %run %t
+// RUN: %clangxx_dfsan %s -fno-exceptions -D_GLIBCXX_NO_ASSERTIONS -o %t && %run %t
+// RUN: %clangxx_dfsan -DORIGIN_TRACKING -mllvm -dfsan-track-origins=1 %s -fno-exceptions -D_GLIBCXX_NO_ASSERTIONS -o %t && %run %t
 //
 // Use -fno-exceptions to turn off exceptions to avoid instrumenting
 // __cxa_begin_catch, std::terminate and __gxx_personality_v0.
 //
+// Use -D_GLIBCXX_NO_ASSERTIONS to avoid depending on
+// std::__glibcxx_assert_fail with gcc >= 15.
+//
 // TODO: Support builtin atomics. For example, https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html
 // DFSan instrumentation pass cannot identify builtin callsites yet.
 
diff --git a/compiler-rt/test/fuzzer/focus-function.test b/compiler-rt/test/fuzzer/focus-function.test
index 64fd5eebb2389..aa15692db358d 100644
--- a/compiler-rt/test/fuzzer/focus-function.test
+++ b/compiler-rt/test/fuzzer/focus-function.test
@@ -7,15 +7,15 @@ UNSUPPORTED: target=aarch64{{.*}}
 
 RUN: %cpp_compiler %S/OnlySomeBytesTest.cpp -o %t-exe
 
-RUN: %t-exe -runs=100 2>&1 | FileCheck %s --check-prefix=FOCUS_NONE
+RUN: %run %t-exe -runs=100 2>&1 | FileCheck %s --check-prefix=FOCUS_NONE
 FOCUS_NONE-NOT: INFO: Focus function is set to
 FOCUS_NONE-NOT: INFO: {{.*}} inputs touch the focus function
 
-RUN: not %t-exe -runs=100 -focus_function=WRONG 2>&1 | FileCheck %s --check-prefix=FOCUS_WRONG
+RUN: not %run %t-exe -runs=100 -focus_function=WRONG 2>&1 | FileCheck %s --check-prefix=FOCUS_WRONG
 FOCUS_WRONG-NOT: INFO: Focus function is set to
 FOCUS_WRONG: ERROR: Failed to set focus function
 
-RUN: %t-exe -runs=100 -focus_function=f0 2>&1 | FileCheck %s --check-prefix=FOCUS_F0
+RUN: %run %t-exe -runs=100 -focus_function=f0 2>&1 | FileCheck %s --check-prefix=FOCUS_F0
 FOCUS_F0: INFO: Focus function is set to 'f0'
 FOCUS_F0: INFO: 0/1 inputs touch the focus function
 
@@ -26,6 +26,6 @@ RUN: echo ABC$(for((i=0;i<2048;i++)); do echo -n x; done) > %t-corpus/ABC
 RUN: echo AXY$(for((i=0;i<2048;i++)); do echo -n x; done) > %t-corpus/AXY
 RUN: echo ABX$(for((i=0;i<2048;i++)); do echo -n x; done) > %t-corpus/ABX
 
-RUN: %t-exe -runs=10000 -focus_function=f0 %t-corpus 2>&1 | FileCheck %s --check-prefix=CORPUS_1_3
+RUN: %run %t-exe -runs=10000 -focus_function=f0 %t-corpus 2>&1 | FileCheck %s --check-prefix=CORPUS_1_3
 CORPUS_1_3: INFO: 1/3 inputs touch the focus function
 CORPUS_1_3: DONE {{.*}} focus:
diff --git a/compiler-rt/test/fuzzer/fuzzer-ubsan.test b/compiler-rt/test/fuzzer/fuzzer-ubsan.test
index 6bc2c38636688..d22339d72e261 100644
--- a/compiler-rt/test/fuzzer/fuzzer-ubsan.test
+++ b/compiler-rt/test/fuzzer/fuzzer-ubsan.test
@@ -1,3 +1,6 @@
+// This test currently fails to compile on green.lab.llvm.org (arm)
+// XFAIL: system-darwin && target=arm{{.*}}
+
 RUN: %cpp_compiler -fsanitize=undefined -fno-sanitize-recover=all %S/SignedIntOverflowTest.cpp -o %t-SignedIntOverflowTest-Ubsan
 RUN: not %run %t-SignedIntOverflowTest-Ubsan 2>&1 | FileCheck %s
 CHECK: runtime error: signed integer overflow: 2147483647 + 1 cannot be represented in type 'int'
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/release-shadow.c b/compiler-rt/test/hwasan/TestCases/Linux/release-shadow.c
index c17dc8453f82c..db9df0ac9f6a6 100644
--- a/compiler-rt/test/hwasan/TestCases/Linux/release-shadow.c
+++ b/compiler-rt/test/hwasan/TestCases/Linux/release-shadow.c
@@ -57,7 +57,7 @@ int test_rss_difference(void *p) {
   size_t diff = rss_before - rss_after;
   fprintf(stderr, "diff %zu\n", diff);
   // Check that the difference is at least close to kNumShadowPages.
-  return diff > kNumShadowPages / 2;
+  return diff >= kNumShadowPages / 2;
 }
 
 int main() {
diff --git a/compiler-rt/test/hwasan/TestCases/Linux/swapcontext_annotation.cpp b/compiler-rt/test/hwasan/TestCases/Linux/swapcontext_annotation.cpp
new file mode 100644
index 0000000000000..d02e7dcb84047
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/Linux/swapcontext_annotation.cpp
@@ -0,0 +1,200 @@
+// Test hwasan __sanitizer_start_switch_fiber and
+// __sanitizer_finish_switch_fiber interface.
+
+// RUN: %clangxx_hwasan -std=c++11 -lpthread -O0 %s -o %t && %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_hwasan -std=c++11 -lpthread -O1 %s -o %t && %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_hwasan -std=c++11 -lpthread -O2 %s -o %t && %run %t 2>&1 | FileCheck %s
+// RUN: %clangxx_hwasan -std=c++11 -lpthread -O3 %s -o %t && %run %t 2>&1 | FileCheck %s
+// RUN: seq 30 | xargs -i -- grep LOOPCHECK %s > %t.checks
+// RUN: %clangxx_hwasan -std=c++11 -lpthread -O0 %s -o %t && %run %t 2>&1 | FileCheck %t.checks --check-prefix LOOPCHECK
+// RUN: %clangxx_hwasan -std=c++11 -lpthread -O1 %s -o %t && %run %t 2>&1 | FileCheck %t.checks --check-prefix LOOPCHECK
+// RUN: %clangxx_hwasan -std=c++11 -lpthread -O2 %s -o %t && %run %t 2>&1 | FileCheck %t.checks --check-prefix LOOPCHECK
+// RUN: %clangxx_hwasan -std=c++11 -lpthread -O3 %s -o %t && %run %t 2>&1 | FileCheck %t.checks --check-prefix LOOPCHECK
+
+//
+// Android and musl do not support swapcontext.
+// REQUIRES: glibc-2.27
+
+#include <pthread.h>
+#include <setjmp.h>
+#include <signal.h>
+#include <stdio.h>
+#include <sys/time.h>
+#include <ucontext.h>
+#include <unistd.h>
+
+#include <sanitizer/common_interface_defs.h>
+
+ucontext_t orig_context;
+ucontext_t child_context;
+ucontext_t next_child_context;
+
+char *next_child_stack;
+
+const int kStackSize = 1 << 20;
+
+const void *main_thread_stack;
+size_t main_thread_stacksize;
+
+const void *from_stack;
+size_t from_stacksize;
+
+// hwasan does not support longjmp with tagged stack pointer, make sure it is
+// not tagged.
+char __attribute__((no_sanitize("hwaddress"))) allocated_stack[kStackSize + 1];
+char __attribute__((
+    no_sanitize("hwaddress"))) allocated_child_stack[kStackSize + 1];
+
+__attribute__((noinline, noreturn)) void LongJump(jmp_buf env) {
+  longjmp(env, 1);
+  _exit(1);
+}
+
+// Simulate __asan_handle_no_return().
+__attribute__((noinline)) void CallNoReturn() {
+  jmp_buf env;
+  if (setjmp(env) != 0)
+    return;
+
+  LongJump(env);
+  _exit(1);
+}
+
+void NextChild() {
+  CallNoReturn();
+  __sanitizer_finish_switch_fiber(nullptr, &from_stack, &from_stacksize);
+
+  printf("NextChild from: %p %zu\n", from_stack, from_stacksize);
+
+  char x[32] = {0}; // Stack gets poisoned.
+  printf("NextChild: %p\n", x);
+
+  CallNoReturn();
+
+  __sanitizer_start_switch_fiber(nullptr, main_thread_stack,
+                                 main_thread_stacksize);
+  CallNoReturn();
+  if (swapcontext(&next_child_context, &orig_context) < 0) {
+    perror("swapcontext");
+    _exit(1);
+  }
+}
+
+void Child(int mode) {
+  CallNoReturn();
+  __sanitizer_finish_switch_fiber(nullptr, &main_thread_stack,
+                                  &main_thread_stacksize);
+  char x[32] = {0}; // Stack gets poisoned.
+  printf("Child: %p\n", x);
+  CallNoReturn();
+  // (a) Do nothing, just return to parent function.
+  // (b) Jump into the original function. Stack remains poisoned unless we do
+  //     something.
+  // (c) Jump to another function which will then jump back to the main function
+  if (mode == 0) {
+    __sanitizer_start_switch_fiber(nullptr, main_thread_stack,
+                                   main_thread_stacksize);
+    CallNoReturn();
+  } else if (mode == 1) {
+    __sanitizer_start_switch_fiber(nullptr, main_thread_stack,
+                                   main_thread_stacksize);
+    CallNoReturn();
+    if (swapcontext(&child_context, &orig_context) < 0) {
+      perror("swapcontext");
+      _exit(1);
+    }
+  } else if (mode == 2) {
+    printf("NextChild stack: %p\n", next_child_stack);
+
+    getcontext(&next_child_context);
+    next_child_context.uc_stack.ss_sp = next_child_stack;
+    next_child_context.uc_stack.ss_size = kStackSize / 2;
+    makecontext(&next_child_context, (void (*)())NextChild, 0);
+    __sanitizer_start_switch_fiber(nullptr, next_child_context.uc_stack.ss_sp,
+                                   next_child_context.uc_stack.ss_size);
+    CallNoReturn();
+    if (swapcontext(&child_context, &next_child_context) < 0) {
+      perror("swapcontext");
+      _exit(1);
+    }
+  }
+}
+
+int Run(int arg, int mode, char *child_stack) {
+  printf("Child stack: %p\n", child_stack);
+  // Setup child context.
+  getcontext(&child_context);
+  child_context.uc_stack.ss_sp = child_stack;
+  child_context.uc_stack.ss_size = kStackSize / 2;
+  if (mode == 0) {
+    child_context.uc_link = &orig_context;
+  }
+  makecontext(&child_context, (void (*)())Child, 1, mode);
+  CallNoReturn();
+  __sanitizer_start_switch_fiber(nullptr, child_context.uc_stack.ss_sp,
+                                 child_context.uc_stack.ss_size);
+  CallNoReturn();
+  if (swapcontext(&orig_context, &child_context) < 0) {
+    perror("swapcontext");
+    _exit(1);
+  }
+  CallNoReturn();
+  __sanitizer_finish_switch_fiber(nullptr, &from_stack, &from_stacksize);
+  CallNoReturn();
+  printf("Main context from: %p %zu\n", from_stack, from_stacksize);
+
+  return child_stack[arg];
+}
+
+void handler(int sig) { CallNoReturn(); }
+
+int main(int argc, char **argv) {
+  // This testcase is copied from ASan's swapcontext_annotation.cpp testcase
+  // and adapted to HWASan:
+  // 1. removed huge stack test since hwasan has no huge stack limitations
+  // 2. stack allocations are now done with original malloc/free instead of
+  //   hwasan interceptor, since HWASan does not support tagged stack pointer
+  //   in longjmp (see __hwasan_handle_longjmp)
+
+  // set up a signal that will spam and trigger __hwasan_handle_vfork at
+  // tricky moments
+  struct sigaction act = {};
+  act.sa_handler = &handler;
+  if (sigaction(SIGPROF, &act, 0)) {
+    perror("sigaction");
+    _exit(1);
+  }
+
+  itimerval t;
+  t.it_interval.tv_sec = 0;
+  t.it_interval.tv_usec = 10;
+  t.it_value = t.it_interval;
+  if (setitimer(ITIMER_PROF, &t, 0)) {
+    perror("setitimer");
+    _exit(1);
+  }
+
+  char *heap = allocated_stack;
+  next_child_stack = allocated_child_stack;
+  int ret = 0;
+  // CHECK-NOT: WARNING: HWASan is ignoring requested __hwasan_handle_vfork
+  for (unsigned int i = 0; i < 30; ++i) {
+    // LOOPCHECK: Child stack: [[CHILD_STACK:0x[0-9a-f]*]]
+    // LOOPCHECK: Main context from: [[CHILD_STACK]] 524288
+    ret += Run(argc - 1, 0, heap);
+    // LOOPCHECK: Child stack: [[CHILD_STACK:0x[0-9a-f]*]]
+    // LOOPCHECK: Main context from: [[CHILD_STACK]] 524288
+    ret += Run(argc - 1, 1, heap);
+    // LOOPCHECK: Child stack: [[CHILD_STACK:0x[0-9a-f]*]]
+    // LOOPCHECK: NextChild stack: [[NEXT_CHILD_STACK:0x[0-9a-f]*]]
+    // LOOPCHECK: NextChild from: [[CHILD_STACK]] 524288
+    // LOOPCHECK: Main context from: [[NEXT_CHILD_STACK]] 524288
+    ret += Run(argc - 1, 2, heap);
+    printf("Iteration %d passed\n", i);
+  }
+
+  // CHECK: Test passed
+  printf("Test passed\n");
+
+  return ret;
+}
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index 8328b407dcc36..e2e815444dcf9 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -965,6 +965,23 @@ def is_windows_lto_supported():
 else:
     config.available_features.add("memprof-shadow-scale-3")
 
+
+def target_page_size():
+    try:
+        proc = subprocess.Popen(
+            f"{emulator or ''} python3",
+            shell=True,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        out, err = proc.communicate(b'import os; print(os.sysconf("SC_PAGESIZE"))')
+        return int(out)
+    except:
+        return 4096
+
+
+config.available_features.add(f"page-size-{target_page_size()}")
+
 if config.expensive_checks:
     config.available_features.add("expensive_checks")
 
diff --git a/compiler-rt/test/memprof/TestCases/free_sized.cpp b/compiler-rt/test/memprof/TestCases/free_sized.cpp
new file mode 100644
index 0000000000000..c8ce8464ef895
--- /dev/null
+++ b/compiler-rt/test/memprof/TestCases/free_sized.cpp
@@ -0,0 +1,21 @@
+// RUN: %clangxx_memprof %s -o %t
+
+// RUN: %env_memprof_opts=print_text=true:log_path=stdout %run %t | FileCheck %s
+
+#include <stdlib.h>
+#include <unistd.h>
+
+extern "C" {
+void free_sized(void *ptr, size_t size);
+void free_aligned_sized(void *ptr, size_t alignement, size_t size);
+}
+
+int main() {
+  void *p = aligned_alloc(16, 32);
+  free_aligned_sized(p, 16, 32);
+  p = malloc(10);
+  free_sized(p, 10);
+  return 0;
+}
+// CHECK: Memory allocation stack id
+// CHECK: Memory allocation stack id
diff --git a/compiler-rt/test/msan/dtls_test.c b/compiler-rt/test/msan/dtls_test.c
index 3c384256147a0..6daaab0ae0b8d 100644
--- a/compiler-rt/test/msan/dtls_test.c
+++ b/compiler-rt/test/msan/dtls_test.c
@@ -11,6 +11,7 @@
 
    // Reports use-of-uninitialized-value, not analyzed
    XFAIL: target={{.*netbsd.*}}
+   UNSUPPORTED: aarch64-target-arch
 
 */
 
diff --git a/compiler-rt/test/profile/Linux/coverage_short_circuit.cpp b/compiler-rt/test/profile/Linux/coverage_short_circuit.cpp
new file mode 100644
index 0000000000000..2abf68380a412
--- /dev/null
+++ b/compiler-rt/test/profile/Linux/coverage_short_circuit.cpp
@@ -0,0 +1,37 @@
+// REQUIRES: lld-available
+// RUN: %clangxx_profgen -std=c++17 -fuse-ld=lld -fcoverage-mapping -o %t %s
+// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t
+// RUN: llvm-profdata merge -o %t.profdata %t.profraw
+// RUN: llvm-cov show %t -instr-profile=%t.profdata 2>&1 | FileCheck %s
+
+void foo() {          // CHECK:       [[@LINE]]| 1|void foo() {
+  bool cond1 = false; // CHECK-NEXT:  [[@LINE]]| 1|  bool cond1 = false;
+  bool cond2 = true;  // CHECK-NEXT:  [[@LINE]]| 1|  bool cond2 = true;
+  if (cond1 &&        // CHECK-NEXT:  [[@LINE]]| 1|  if (cond1 &&
+      cond2) {        // CHECK-NEXT:  [[@LINE]]| 0|      cond2) {
+  } // CHECK-NEXT:  [[@LINE]]| 0|  }
+} // CHECK-NEXT:  [[@LINE]]| 1|}
+
+void bar() {          // CHECK:       [[@LINE]]| 1|void bar() {
+  bool cond1 = true;  // CHECK-NEXT:  [[@LINE]]| 1|  bool cond1 = true;
+  bool cond2 = false; // CHECK-NEXT:  [[@LINE]]| 1|  bool cond2 = false;
+  if (cond1 &&        // CHECK-NEXT:  [[@LINE]]| 1|  if (cond1 &&
+      cond2) {        // CHECK-NEXT:  [[@LINE]]| 1|      cond2) {
+  } // CHECK-NEXT:  [[@LINE]]| 0|  }
+} // CHECK-NEXT:  [[@LINE]]| 1|}
+
+void baz() {          // CHECK:       [[@LINE]]| 1|void baz() {
+  bool cond1 = false; // CHECK-NEXT:  [[@LINE]]| 1|  bool cond1 = false;
+  bool cond2 = true;  // CHECK-NEXT:  [[@LINE]]| 1|  bool cond2 = true;
+  if (cond1           // CHECK-NEXT:  [[@LINE]]| 1|  if (cond1
+      &&              // CHECK-NEXT:  [[@LINE]]| 0|      &&
+      cond2) {        // CHECK-NEXT:  [[@LINE]]| 0|      cond2) {
+  } // CHECK-NEXT:  [[@LINE]]| 0|  }
+} // CHECK-NEXT:  [[@LINE]]| 1|}
+
+int main() {
+  foo();
+  bar();
+  baz();
+  return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/odd_stack_size.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/odd_stack_size.cpp
index 9d7d46b462a88..cc76804aed210 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/odd_stack_size.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/odd_stack_size.cpp
@@ -1,4 +1,5 @@
 // RUN: %clangxx -O1 %s -o %t && %run %t
+// REQUIRES: page-size-4096
 // UNSUPPORTED: android
 
 // Fail on powerpc64 bots with:
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/release_to_os_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/release_to_os_test.cpp
index 0fa77200bf1cc..c7a5534696361 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/release_to_os_test.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/release_to_os_test.cpp
@@ -11,6 +11,9 @@
 // FIXME: This mode uses 32bit allocator without purge.
 // UNSUPPORTED: hwasan-aliasing
 
+// Page size is hardcoded below, but test still fails even if not hardcoded.
+// REQUIRES: page-size-4096
+
 #include <algorithm>
 #include <assert.h>
 #include <fcntl.h>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
index c288e1d69baf9..3e9ff924a3c4a 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/resize_tls_dynamic.cpp
@@ -11,6 +11,9 @@
 // FIXME: Investigate
 // UNSUPPORTED: target=powerpc64{{.*}}
 
+// Fails because AArch64 uses TLSDESC instead of __tls_get_addr.
+// UNSUPPORTED: aarch64-target-arch
+
 #include <string.h>
 
 #ifndef BUILD_DSO
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/tls_get_addr.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/tls_get_addr.c
index 0aff6039ac4e8..a4a4f64ed3706 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/tls_get_addr.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/tls_get_addr.c
@@ -13,6 +13,9 @@
 // FIXME: Fails for unknown reasons.
 // UNSUPPORTED: powerpc64le-target-arch
 
+// Fails because AArch64 uses TLSDESC instead of __tls_get_addr.
+// UNSUPPORTED: aarch64-target-arch
+
 #ifndef BUILD_SO
 #  include <assert.h>
 #  include <dlfcn.h>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/dedup_token_length_test.cpp b/compiler-rt/test/sanitizer_common/TestCases/Posix/dedup_token_length_test.cpp
index deedbba76cdeb..37bfee4806173 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Posix/dedup_token_length_test.cpp
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/dedup_token_length_test.cpp
@@ -10,6 +10,10 @@
 
 // REQUIRES: stable-runtime
 
+// rdar://158303080 top few frames are at times inaccurate in ubsan fast stack
+// unwind on darwin
+// XFAIL: (darwin && ubsan && (arm64-target-arch || arm64e-target-arch))
+
 // XFAIL: target={{.*netbsd.*}} && !asan
 
 volatile int *null = 0;
diff --git a/compiler-rt/test/ubsan_minimal/TestCases/misalignment.cpp b/compiler-rt/test/ubsan_minimal/TestCases/misalignment.cpp
new file mode 100644
index 0000000000000..cf828792a324f
--- /dev/null
+++ b/compiler-rt/test/ubsan_minimal/TestCases/misalignment.cpp
@@ -0,0 +1,13 @@
+// RUN: %clang_min_runtime -fsanitize=alignment %s -o %t && %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK
+
+void f(int &n) {}
+
+int *t;
+
+int main() {
+  int r;
+  t = (int *)(((char *)&r) + 1);
+  // CHECK: ubsan: type-mismatch by 0x{{[[:xdigit:]]+}} address 0x{{[[:xdigit:]]+$}}
+  // CHECK-NOT: type-mismatch
+  f(*t);
+}
diff --git a/compiler-rt/test/ubsan_minimal/TestCases/null.cpp b/compiler-rt/test/ubsan_minimal/TestCases/null.cpp
new file mode 100644
index 0000000000000..c97527b72cd0c
--- /dev/null
+++ b/compiler-rt/test/ubsan_minimal/TestCases/null.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang_min_runtime -fsanitize=null %s -o %t && %run %t 2>&1 | FileCheck %s --check-prefixes=CHECK
+
+void f(int &n) {}
+
+int *t;
+
+int main() {
+  // CHECK: ubsan: type-mismatch by 0x{{[[:xdigit:]]+}} address 0x{{[[:xdigit:]]+$}}
+  // CHECK-NOT: type-mismatch
+  f(*t);
+}
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DAP.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DAP.py
new file mode 100644
index 0000000000000..06515dd8e0b10
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DAP.py
@@ -0,0 +1,838 @@
+# DExTer : Debugging Experience Tester
+# ~~~~~~   ~         ~~         ~   ~~
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Interface for communicating with a debugger via the DAP.
+"""
+
+import abc
+from collections import defaultdict
+import copy
+import json
+import os
+import shlex
+import subprocess
+import sys
+import threading
+import time
+from enum import Enum
+
+from dex.debugger.DebuggerBase import DebuggerBase, watch_is_active
+from dex.dextIR import FrameIR, LocIR, StepIR, StopReason, ValueIR
+from dex.dextIR import StackFrame, SourceLocation, ProgramState
+from dex.utils.Exceptions import DebuggerException, LoadDebuggerException
+from dex.utils.ReturnCode import ReturnCode
+from dex.utils.Logging import Logger
+from dex.utils.Timeout import Timeout
+
+
+# Helper enum used for colorizing DAP Message Log output.
+class Color(Enum):
+    CYAN = 36
+    GREEN = 32
+    YELLOW = 33
+    RED = 31
+    MAGENTA = 35
+
+    def apply(self, text: str) -> str:
+        return f"\033[{self.value}m{text}\033[0m"
+
+
+class DAPMessageLogger:
+    def __init__(self, context):
+        self.dexter_logger = context.logger
+        self.log_file: str = context.options.dap_message_log
+        self.colorized: bool = context.options.colorize_dap_log
+        self.indent = 2 if context.options.format_dap_log == "pretty" else None
+        self.prefix_send: str = "->"
+        self.prefix_recv: str = "<-"
+        self.out_handle = None
+        self.open = False
+        self.lock = threading.Lock()
+
+    def _custom_enter(self):
+        self.open = True
+        if self.log_file is None:
+            return
+        if self.log_file == "-":
+            self.out_handle = sys.stdout
+            return
+        self.out_handle = open(self.log_file, "w+", encoding="utf-8")
+
+    def _custom_exit(self):
+        if self.out_handle is not None and self.log_file != "-":
+            self.out_handle.close()
+        self.open = False
+
+    def _colorize_dap_message(self, message: dict) -> dict:
+        if not self.colorized:
+            return message
+        colorized_message = copy.deepcopy(message)
+        if colorized_message["type"] == "event":
+            colorized_message["type"] = Color.YELLOW.apply("event")
+            colorized_message["event"] = Color.YELLOW.apply(colorized_message["event"])
+        elif colorized_message["type"] == "response":
+            colorized_message["type"] = Color.GREEN.apply("response")
+            colorized_message["command"] = Color.YELLOW.apply(
+                colorized_message["command"]
+            )
+        elif colorized_message["type"] == "request":
+            colorized_message["type"] = Color.CYAN.apply("request")
+            colorized_message["command"] = Color.YELLOW.apply(
+                colorized_message["command"]
+            )
+        return colorized_message
+
+    def write_message(self, message: dict, incoming: bool):
+        prefix = self.prefix_recv if incoming else self.prefix_send
+        # ANSI escape codes get butchered by json.dumps(), so we fix them up here.
+        message_str = json.dumps(
+            self._colorize_dap_message(message), indent=self.indent
+        ).replace("\\u001b", "\033")
+        if self.out_handle is not None and self.open:
+            with self.lock:
+                self.out_handle.write(f"{prefix} {message_str}\n")
+        elif not self.open:
+            self.dexter_logger.warning(
+                f'Attempted to write message after program closed: "{prefix} {message_str}"'
+            )
+
+
+# Debuggers communicate optional feature support.
+class DAPDebuggerCapabilities:
+    def __init__(self):
+        self.supportsConfigurationDoneRequest: bool = False
+        self.supportsFunctionBreakpoints: bool = False
+        self.supportsConditionalBreakpoints: bool = False
+        self.supportsHitConditionalBreakpoints: bool = False
+        self.supportsEvaluateForHovers: bool = False
+        self.supportsSetVariable: bool = False
+        self.supportsStepInTargetsRequest: bool = False
+        self.supportsModulesRequest: bool = False
+        self.supportsValueFormattingOptions: bool = False
+        self.supportsLogPoints: bool = False
+        self.supportsSetExpression: bool = False
+        self.supportsDataBreakpoints: bool = False
+        self.supportsReadMemoryRequest: bool = False
+        self.supportsWriteMemoryRequest: bool = False
+        self.supportsDisassembleRequest: bool = False
+        self.supportsCancelRequest: bool = False
+        self.supportsSteppingGranularity: bool = False
+        self.supportsInstructionBreakpoints: bool = False
+
+    def update(self, logger: Logger, feature_dict: dict):
+        for k, v in feature_dict.items():
+            if hasattr(self, k):
+                setattr(self, k, v)
+            else:
+                logger.warning(f"DAP: Unknown support flag: {k}")
+
+
+# As DAP does not give us a trivially query-able process, we are responsible for maintaining our own state information,
+# including what breakpoints are currently set, and whether the debugger is running or stopped.
+# This class holds all state that is set based on events sent by the debug adapter; most responses are forwarded through
+# to the main DAP class, though in a few cases where it is convenient for bookkeeping the DAPDebuggerState may read some
+# information from the responses before forwarding them onwards.
+class DAPDebuggerState:
+    def __init__(self):
+        ## Overall debugger state information.
+        #
+        # Whether we have received the initialize update yet.
+        self.initialized: bool = False
+        # Whether the debugger has successfully launched yet.
+        self.launched: bool = False
+        # The thread that we are debugging.
+        # TODO: This is primitively handled right now, assuming that we only ever have one thread; if we want
+        # support for debugging any multi-threaded program then we will need to track some more complex state.
+        self.thread = None
+        # True if the debuggee is currently executing.
+        self.is_running: bool = False
+        # True if the debuggee has finished executing.
+        self.is_finished: bool = False
+
+        ## Information for the program at a particular stopped point, which will be invalidated when execution resumes.
+        #
+        # Either None if the debuggee is currently running, or a string specifying the reason why the
+        # debuggee is currently stopped otherwise.
+        self.stopped_reason = None
+        # If we were stopped for the reason 'breakpoint', this will contain a list of the DAP breakpoint IDs
+        # responsible for stopping us.
+        self.stopped_bps = []
+        # For a currently stopped process, stores the mapping of frame indices (top of stack=0) to frameIds returned
+        # from the debug adapter.
+        self.frame_map = []
+
+        # We use responses[idx] to refer to the response for the request sent with seq=idx, where the value
+        # is either the response payload, or None if the response hasn't arrived yet.
+        # Since requests are indexed from 1, we insert a 'None' at the front to ensure that the first real
+        # entry is indexed correctly.
+        self.responses = [None]
+        # Map of DAP breakpoint IDs to resolved instruction addresses.
+        self.bp_addr_map = {}
+
+        # DAP features supported by the debugger.
+        self.capabilities = DAPDebuggerCapabilities()
+
+    def set_response(self, req_id: int, response: dict):
+        if len(self.responses) > req_id:
+            self.responses[req_id] = response
+            return
+        while len(self.responses) < req_id:
+            self.responses.append(None)
+        self.responses.append(response)
+
+    # As the receiver thread does not know when a request has been sent, and only the receiver thread should write to the DebuggerState object,
+    # the responses list may not have been populated with a None for a pending request at the time that the main thread expects it. Therefore,
+    # we use this getter to account for requests that the receiver thread is unaware of.
+    def get_response(self, req_id: int):
+        if len(self.responses) <= req_id:
+            return None
+        return self.responses[req_id]
+
+
+# DAP Communication model:
+# - Communication is message-based, not stateful - we cannot simply query information from the debugger as we can with
+#   other debugger implementations, we need to maintain local state.
+# - All messages are utf-encoded JSON, which we convert to/from python dicts via methods above; some amount of
+#   bookkeeping is performed automatically in the DAP class.
+# - Commands and queries are sent via 'request' messages, for which a corresponding 'response' will always be sent back
+#   by the adapter indicating success/failure, containing data related to the request.
+# - The adapter will also send 'event' messages, indicating state changes in the debugger - for example, when the
+#   debugger has stopped at a breakpoint.
+# In order to handle this, we run a separate thread that will continuously insert any messages received
+# from the adapter into a queue, which the main thread will read; generally, our response to any read message
+# is to update our state, which Dexter's DebuggerController will then read.
+class DAP(DebuggerBase, metaclass=abc.ABCMeta):
+    def __init__(self, context, *args):
+        self._debugger_state = DAPDebuggerState()
+        self._proc = None
+        self._receiver_thread = None
+        self._err_thread = None
+        self.seq = 0
+        self.target_proc_id = -1
+        self.max_bp_id = 0
+        # Mapping of active breakpoints per-file - intentionally excludes breakpoints that we have deleted.
+        # { file -> [dex_breakpoint_id]}
+        self.file_to_bp = defaultdict(list)
+        # { dex_breakpoint_id -> (file, line, condition) }
+        self.bp_info = {}
+        # We don't rely on IDs returned directly from the debug adapter. Instead, we use dexter breakpoint IDs, and
+        # maintain a two-way-mapping of dex_bp_id<->dap_bp_id. This also allows us to defer the setting of breakpoints
+        # in the debug adapter itself until necessary.
+        # NB: The debug adapter may merge dexter-side breakpoints into a single debugger-side breakpoint; therefore, the
+        # DAP->Dex mapping is one-to-many.
+        self.dex_id_to_dap_id = {}
+        self.dap_id_to_dex_ids = {}
+        self.pending_breakpoints: bool = False
+        # List of breakpoints, indexed by BP ID
+        # Each entry has the source file (for use in referencing desired_bps), and the DA-assigned
+        # ID for that breakpoint if it has one (if it has been removed or not yet created then it will be None).
+        # self.bp_source_list: list[(str, int)]
+        self.message_logger = None
+        super(DAP, self).__init__(context, *args)
+
+    @property
+    @abc.abstractmethod
+    def _debug_adapter_name(self) -> str:
+        pass
+
+    @property
+    @abc.abstractmethod
+    def _debug_adapter_executable(self) -> str:
+        pass
+
+    @property
+    def _debug_adapter_launch_args(self) -> list:
+        return []
+
+    @staticmethod
+    def make_request(command: str, arguments=None) -> dict:
+        request = {"type": "request", "command": command}
+        if arguments is not None:
+            request["arguments"] = arguments
+        return request
+
+    @staticmethod
+    def make_initialize_request(adapterID: str) -> dict:
+        return DAP.make_request(
+            "initialize",
+            {
+                "clientID": "dexter",
+                "adapterID": adapterID,
+                "pathFormat": "path",
+                "linesStartAt1": True,
+                "columnsStartAt1": True,
+                "supportsVariableType": True,
+                "supportsVariablePaging": True,
+                "supportsRunInTerminalRequest": False,
+            },
+        )
+
+    class BreakpointRequest:
+        def __init__(self, line: int, condition=None):
+            self.line = line
+            self.condition = condition
+
+        def toDict(self) -> dict:
+            result = {"line": self.line}
+            if self.condition is not None:
+                result["condition"] = self.condition
+            return result
+
+    @staticmethod
+    def make_set_breakpoint_request(source: str, bps) -> dict:
+        return DAP.make_request(
+            "setBreakpoints",
+            {"source": {"path": source}, "breakpoints": [bp.toDict() for bp in bps]},
+        )
+
+    ############################################################################
+    ## DAP communication & state-handling functions
+
+    # Sends a request to the adapter, returning the seq value of the request.
+    def send_message(self, payload: dict) -> int:
+        self.seq = self.seq + 1
+        payload["seq"] = self.seq
+        self.message_logger.write_message(payload, False)
+        body = json.dumps(payload)
+        message = f"Content-Length: {len(body)}\r\n\r\n{body}".encode("utf-8")
+        self._proc.stdin.write(message)
+        self._proc.stdin.flush()
+        return self.seq
+
+    def _handle_message(
+        message: dict, debugger_state: DAPDebuggerState, logger: Logger
+    ):
+        # We only support events and responses, we do not implement any reverse-requests.
+        # TODO: If we find cases where 'seq' becomes important, we need to read it here and process
+        # pending messages in order.
+        if message["type"] == "event":
+            event_type = message["event"]
+            event_details = message.get("body")
+            if event_type == "initialized":
+                debugger_state.initialized = True
+            elif event_type == "process":
+                debugger_state.launched = True
+                debugger_state.is_running = True
+            # The debugger has stopped for some reason.
+            elif event_type == "stopped":
+                stop_reason = event_details["reason"]
+                debugger_state.is_running = False
+                debugger_state.stopped_reason = stop_reason
+                debugger_state.stopped_bps = event_details.get("hitBreakpointIds", [])
+                debugger_state.thread = event_details["threadId"]
+            elif event_type == "breakpoint":
+                # We handle most BP information in the main DAP thread by reading responses to breakpoint requests;
+                # some information is only passed via event, however, which we store here.
+                breakpoint_details = event_details["breakpoint"]
+                if "instructionReference" in breakpoint_details:
+                    debugger_state.bp_addr_map[
+                        breakpoint_details["id"]
+                    ] = breakpoint_details["instructionReference"]
+            elif event_type == "exited" or event_type == "terminated":
+                debugger_state.stopped_reason = event_type
+                debugger_state.is_running = False
+                debugger_state.is_finished = True
+            # We may receive this event before or after the response to the corresponding "continue" request.
+            elif event_type == "continued":
+                debugger_state.is_running = True
+                # Reset all state that is invalidated upon program continue.
+                debugger_state.stopped_reason = None
+                debugger_state.stopped_bps = []
+                debugger_state.frame_map = []
+            elif event_type == "thread":
+                if (
+                    event_details["reason"] == "started"
+                    and debugger_state.thread is None
+                ):
+                    debugger_state.thread = event_details["threadId"]
+            elif event_type == "capabilities":
+                # Unchanged capabilites may not be included.
+                debugger_state.capabilities.update(logger, event_details)
+            # There are many events we do not care about, just skip processing them.
+            else:
+                pass
+        elif message["type"] == "response":
+            request_seq = message["request_seq"]
+            debugger_state.set_response(request_seq, message)
+            # TODO: We also receive a "continued" event, but it seems reasonable to set state based on either the
+            # response or the event, since the DAP does not specify an order in which they are sent. May need revisiting
+            # if there turns out to be some odd ordering issues, e.g. if we can receive messages in the order
+            # ["response: continued", "event: stopped", "event: continued"].
+            if (
+                message["command"] in ["continue", "stepIn", "next", "stepOut"]
+                and message["success"] == True
+            ):
+                debugger_state.is_running = True
+                # Reset all state that is invalidated upon program continue.
+                debugger_state.stopped_reason = None
+                debugger_state.stopped_bps = []
+                debugger_state.frame_map = []
+            # It is useful to cache a mapping of frames; since this is invalidated when we continue, and only this
+            # message-handling thread should write to debugger_state, we do so while handling the response for
+            # convenience.
+            if message["command"] == "stackTrace" and message["success"] == True:
+                debugger_state.frame_map = [
+                    stackframe["id"] for stackframe in message["body"]["stackFrames"]
+                ]
+            # The debugger communicates which optional DAP features are
+            # supported in its initalize response.
+            if message["command"] == "initialize" and message["success"] == True:
+                body = message.get("body")
+                if body:
+                    debugger_state.capabilities.update(logger, body)
+
+    def _colorize_dap_message(message: dict) -> dict:
+        colorized_message = copy.deepcopy(message)
+        if colorized_message["type"] == "event":
+            colorized_message["type"] = "<y>event</>"
+            colorized_message["event"] = f"<y>{colorized_message['event']}</>"
+        elif colorized_message["type"] == "response":
+            colorized_message["type"] = "<g>response</>"
+            colorized_message["command"] = f"<y>{colorized_message['command']}</>"
+        elif colorized_message["type"] == "request":
+            colorized_message["type"] = "<b>request</>"
+            colorized_message["command"] = f"<y>{colorized_message['command']}</>"
+        return colorized_message
+
+    def _read_dap_output(
+        proc: subprocess.Popen,
+        debugger_state: DAPDebuggerState,
+        message_logger: DAPMessageLogger,
+        logger: Logger,
+    ):
+        buffer: bytes = b""
+        while True:
+            chunk: bytes = proc.stdout.read(1)
+            if not chunk:
+                break
+            buffer += chunk
+            if b"\r\n\r\n" in buffer:
+                header, rest = buffer.split(b"\r\n\r\n", 1)
+                content_length = int(header.decode().split(":")[1].strip())
+                while len(rest) < content_length:
+                    rest += proc.stdout.read(content_length - len(rest))
+                message = json.loads(rest[:content_length])
+                message_logger.write_message(message, True)
+                DAP._handle_message(message, debugger_state, logger)
+                buffer = rest[content_length:]
+
+    def _read_dap_err(proc: subprocess.Popen, logger: Logger):
+        while True:
+            err: bytes = proc.stderr.readline()
+            if len(err) > 0:
+                logger.error(f"DAP server: {err.decode().strip()}")
+
+    def _custom_init(self):
+        self.context.logger.note(
+            f"Opening DAP server: {shlex.join([self._debug_adapter_executable] + self._debug_adapter_launch_args)}"
+        )
+        self.message_logger = DAPMessageLogger(self.context)
+        self.message_logger._custom_enter()
+        self._proc = subprocess.Popen(
+            [self._debug_adapter_executable] + self._debug_adapter_launch_args,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            bufsize=0,
+        )
+        self._receiver_thread = threading.Thread(
+            target=DAP._read_dap_output,
+            args=(
+                self._proc,
+                self._debugger_state,
+                self.message_logger,
+                self.context.logger,
+            ),
+            daemon=True,
+        )
+        self._err_thread = threading.Thread(
+            target=DAP._read_dap_err,
+            args=(self._proc, self.context.logger),
+            daemon=True,
+        )
+        self._receiver_thread.start()
+        self._err_thread.start()
+        init_req = self.send_message(
+            self.make_initialize_request(self._debug_adapter_name)
+        )
+        assert self._proc.poll() is None, "Process has closed unexpectedly early?"
+        self._await_response(init_req)
+
+    def _custom_exit(self):
+        if self._proc is not None:
+            dc_req = self.send_message(self.make_request("disconnect"))
+            dc_req_timeout = 3
+            try:
+                result = self._await_response(dc_req, dc_req_timeout)
+                if not result["success"]:
+                    self.context.logger.warning(
+                        "The disconnect request sent to the DAP server failed; forcibly shutting down DAP server."
+                    )
+                else:
+                    self.context.logger.note(
+                        "Successfully disconnected from DAP server."
+                    )
+            except:
+                # We're going to kill the process regardless, we just want to give the target a chance to shut down
+                # gracefully first.
+                self.context.logger.warning(
+                    f"The disconnect request sent to the DAP server timed out after {dc_req_timeout}s; forcibly shutting down DAP server."
+                )
+                pass
+            self._proc.kill()
+            self._proc = None
+        self.message_logger._custom_exit()
+
+    # Waits for a response to the request with the given seq, optionally raising an error
+    # if the response takes too long (blocks forever by default/if timeout=0).
+    def _await_response(self, seq: int, timeout: float = 0.0) -> dict:
+        timeout_check = Timeout(timeout)
+        while self._debugger_state.get_response(seq) is None:
+            if timeout_check.timed_out():
+                if self._proc.poll() is not None:
+                    self.context.logger.error(
+                        f"Debug adapter exited while Dexter is awaiting response? Result: {self._proc.poll()}"
+                    )
+                raise TimeoutError(
+                    f"Timed out while waiting for response to DAP request {seq}"
+                )
+            time.sleep(0.001)
+        return self._debugger_state.get_response(seq)
+
+    ## End of DAP communication methods
+    ############################################################################
+
+    def _translate_stop_reason(self, reason):
+        if reason is None:
+            return None
+        if "breakpoint" in reason:
+            return StopReason.BREAKPOINT
+        if reason == "step":
+            return StopReason.STEP
+        if reason == "exited" or reason == "terminated":
+            return StopReason.PROGRAM_EXIT
+        if reason == "exception":
+            return StopReason.ERROR
+        return StopReason.OTHER
+
+    def _load_interface(self):
+        if not os.path.isfile(self._debug_adapter_executable):
+            raise LoadDebuggerException(
+                f'debug adapter "{self._debug_adapter_executable}" does not exist',
+                sys.exc_info(),
+            )
+        # We don't make use of _interface, so return nothing.
+
+    @property
+    @abc.abstractmethod
+    def version(self):
+        """The version of this DAP debugger."""
+
+    ############################################################################
+    ## Breakpoint Methods
+
+    def get_next_bp_id(self):
+        new_id = self.max_bp_id
+        self.max_bp_id += 1
+        return new_id
+
+    def get_current_bps(self, source):
+        if source in self.file_to_bp:
+            return self.file_to_bp[source]
+        return []
+
+    def _update_requested_bp_list(self, bp_list):
+        """Can be overridden for any specific implementations that need further processing before sending breakpoints to
+        the debug adapter, e.g. in LLDB we cannot store multiple breakpoints at a single location, and therefore must
+        combine conditions for breakpoints at the same location."""
+        return bp_list
+
+    # For a source file, returns the list of BreakpointRequests for the breakpoints in that file, which can be sent to
+    # the debug adapter.
+    def _get_desired_bps(self, file: str):
+        bp_list = [
+            DAP.BreakpointRequest(line, cond)
+            for (_, line, cond) in map(
+                lambda dex_bp_id: self.bp_info[dex_bp_id], self.get_current_bps(file)
+            )
+        ]
+        return self._update_requested_bp_list(bp_list)
+
+    def clear_breakpoints(self):
+        # We don't actually need to do anything here - even if breakpoints were preserved between runs, we will
+        # automatically clear old breakpoints on the first 'setBreakpoints' message.
+        pass
+
+    def _add_breakpoint(self, file, line):
+        return self._add_conditional_breakpoint(file, line, None)
+
+    def _add_conditional_breakpoint(self, file, line, condition):
+        new_id = self.get_next_bp_id()
+        self.file_to_bp[file].append(new_id)
+        self.bp_info[new_id] = (file, line, condition)
+        self.pending_breakpoints = True
+        return new_id
+
+    def _flush_breakpoints(self):
+        if not self.pending_breakpoints:
+            return
+        for file in self.file_to_bp.keys():
+            desired_bps = self._get_desired_bps(file)
+            request_id = self.send_message(
+                self.make_set_breakpoint_request(file, desired_bps)
+            )
+            result = self._await_response(request_id, 10)
+            if not result["success"]:
+                raise DebuggerException(f"could not set breakpoints for '{file}'")
+            # The debug adapter may have chosen to merge our breakpoints. From here we need to identify such cases and
+            # handle them so that our internal bookkeeping is correct.
+            dex_bp_ids = self.get_current_bps(file)
+            dap_bp_ids = [bp["id"] for bp in result["body"]["breakpoints"]]
+            if len(dex_bp_ids) != len(dap_bp_ids):
+                self.context.logger.error(
+                    f"Sent request to set {len(dex_bp_ids)} breakpoints, but received {len(dap_bp_ids)} in response."
+                )
+            visited_dap_ids = set()
+            for i, dex_bp_id in enumerate(dex_bp_ids):
+                dap_bp_id = dap_bp_ids[i]
+                self.dex_id_to_dap_id[dex_bp_id] = dap_bp_id
+                # We take the mappings in the response as the canonical mapping, meaning that if the debug server has
+                # simply *changed* the DAP ID for a breakpoint we overwrite the existing mapping rather than adding to
+                # it, but if we receive the same DAP ID for multiple Dex IDs *then* we store a one-to-many mapping.
+                if dap_bp_id in visited_dap_ids:
+                    self.dap_id_to_dex_ids[dap_bp_id].append(dex_bp_id)
+                else:
+                    self.dap_id_to_dex_ids[dap_bp_id] = [dex_bp_id]
+                    visited_dap_ids.add(dap_bp_id)
+        self.pending_breakpoints = False
+
+    def _confirm_triggered_breakpoint_ids(self, dex_bp_ids):
+        """Can be overridden for any specific implementations that need further processing from the debug server's
+        reported 'hitBreakpointIds', e.g. in LLDB where we the ID for every breakpoint at the current PC, even if some
+        are conditional and their condition is not met."""
+        return dex_bp_ids
+
+    def get_triggered_breakpoint_ids(self):
+        # Breakpoints can only have been triggered if we've hit one.
+        stop_reason = self._translate_stop_reason(self._debugger_state.stopped_reason)
+        if stop_reason != StopReason.BREAKPOINT:
+            return []
+        breakpoint_ids = set(
+            [
+                dex_id
+                for dap_id in self._debugger_state.stopped_bps
+                for dex_id in self.dap_id_to_dex_ids[dap_id]
+            ]
+        )
+        return self._confirm_triggered_breakpoint_ids(breakpoint_ids)
+
+    def delete_breakpoints(self, ids):
+        per_file_deletions = defaultdict(list)
+        for dex_bp_id in ids:
+            source, _, _ = self.bp_info[dex_bp_id]
+            per_file_deletions[source].append(dex_bp_id)
+        for file, deleted_ids in per_file_deletions.items():
+            old_len = len(self.file_to_bp[file])
+            self.file_to_bp[file] = [
+                bp_id for bp_id in self.file_to_bp[file] if bp_id not in deleted_ids
+            ]
+            if len(self.file_to_bp[file]) != old_len:
+                self.pending_breakpoints = True
+
+    ## End of breakpoint methods
+    ############################################################################
+
+    @classmethod
+    @abc.abstractmethod
+    def _get_launch_params(self, cmdline):
+        """ "Set the debugger-specific params used in a launch request."""
+
+    def launch(self, cmdline):
+        assert len(self.file_to_bp.keys()) > 0
+
+        if self.context.options.target_run_args:
+            cmdline += shlex.split(self.context.options.target_run_args)
+
+        launch_request = self._get_launch_params(cmdline)
+
+        # For some reason, we *must* submit in the order launch->configurationDone, and then we will receive responses
+        # in the order configurationDone->launch.
+        self._flush_breakpoints()
+        launch_req_id = self.send_message(self.make_request("launch", launch_request))
+        config_done_req_id = self.send_message(self.make_request("configurationDone"))
+        config_done_response = self._await_response(config_done_req_id)
+        assert config_done_response["success"], "Should simply receive an affirmative?"
+        launch_response = self._await_response(launch_req_id)
+        if not launch_response["success"]:
+            raise DebuggerException(
+                f"failure launching debugger: \"{launch_response['body']['error']['format']}\""
+            )
+        # We can't interact meaningfully with the process until we have the thread ID and confirmation that the process
+        # has finished launching.
+        while self._debugger_state.thread is None or not self._debugger_state.launched:
+            time.sleep(0.001)
+
+    # LLDB has unique stepping behaviour w.r.t. breakpoints that needs to be handled after completing a step, so we use
+    # an overridable hook to enable debugger-specific behaviour.
+    def _post_step_hook(self):
+        """Hook to be executed after completing a step request."""
+
+    def _step(self, step_request_string):
+        self._flush_breakpoints()
+        step_req_id = self.send_message(
+            self.make_request(
+                step_request_string, {"threadId": self._debugger_state.thread}
+            )
+        )
+        response = self._await_response(step_req_id)
+        if not response["success"]:
+            raise DebuggerException(
+                f"failed to perform debugger action: '{step_request_string}'"
+            )
+        # If we've "stepped" to a breakpoint, then continue to hit the breakpoint properly.
+        # NB: This is an issue that only seems relevant to LLDB, but is also harmless outside of LLDB; if it turns out
+        #     to cause issues for other debuggers, we can move it to a post-step hook.
+        while self._debugger_state.is_running:
+            time.sleep(0.001)
+        self._post_step_hook()
+
+    def step_in(self):
+        self._step("stepIn")
+
+    def step_next(self):
+        self._step("next")
+
+    def step_out(self):
+        self._step("stepOut")
+
+    def go(self) -> ReturnCode:
+        self._flush_breakpoints()
+        continue_req_id = self.send_message(
+            self.make_request("continue", {"threadId": self._debugger_state.thread})
+        )
+        response = self._await_response(continue_req_id)
+        if not response["success"]:
+            raise DebuggerException("failed to continue")
+        # Assuming the request to continue succeeded, we still need to wait to receive an event back from the debugger
+        # indicating that we have successfully resumed.
+
+    def _get_step_info(self, watches, step_index):
+        assert (
+            not self._debugger_state.is_running
+        ), "Cannot get step info while debugger is running!"
+        trace_req_id = self.send_message(
+            self.make_request("stackTrace", {"threadId": self._debugger_state.thread})
+        )
+        trace_response = self._await_response(trace_req_id)
+        if not trace_response["success"]:
+            raise DebuggerException("failed to get stack frames")
+        stackframes = trace_response["body"]["stackFrames"]
+
+        frames = []
+        state_frames = []
+
+        for idx, stackframe in enumerate(stackframes):
+            # FIXME: No source, skip the frame! Currently I've only observed this for frames below main, so we break
+            # here; if it happens elsewhere, then this will break more stuff and we'll come up with a better solution.
+            if (
+                stackframe.get("source") is None
+                or stackframe["source"].get("path") is None
+            ):
+                break
+            loc_dict = {
+                "path": stackframe["source"]["path"],
+                "lineno": stackframe["line"],
+                "column": stackframe["column"],
+            }
+            loc = LocIR(**loc_dict)
+            valid_loc_for_watch = loc.path and os.path.exists(loc.path)
+            frame = FrameIR(
+                function=self._sanitize_function_name(stackframe["name"]),
+                is_inlined=stackframe["name"].startswith("[Inline Frame]"),
+                loc=loc,
+            )
+
+            # We skip frames that are below "main", since we do not expect those to be user code.
+            fname = frame.function or ""  # pylint: disable=no-member
+            if any(name in fname for name in self.frames_below_main):
+                break
+
+            frames.append(frame)
+
+            state_frame = StackFrame(
+                function=frame.function,
+                is_inlined=frame.is_inlined,
+                location=SourceLocation(**loc_dict),
+                watches={},
+            )
+            if valid_loc_for_watch:
+                for expr in map(
+                    # Filter out watches that are not active in the current frame,
+                    # and then evaluate all the active watches.
+                    lambda watch_info, idx=idx: self.evaluate_expression(
+                        watch_info.expression, idx
+                    ),
+                    filter(
+                        lambda watch_info, idx=idx, line_no=loc.lineno, loc_path=loc.path: watch_is_active(
+                            watch_info, loc_path, idx, line_no
+                        ),
+                        watches,
+                    ),
+                ):
+                    state_frame.watches[expr.expression] = expr
+            state_frames.append(state_frame)
+
+        if len(frames) == 1 and frames[0].function is None:
+            frames = []
+            state_frames = []
+
+        reason = self._translate_stop_reason(self._debugger_state.stopped_reason)
+
+        return StepIR(
+            step_index=step_index,
+            frames=frames,
+            stop_reason=reason,
+            program_state=ProgramState(state_frames),
+        )
+
+    @property
+    def is_running(self):
+        return self._debugger_state.is_running
+
+    @property
+    def is_finished(self):
+        return self._debugger_state.is_finished
+
+    @property
+    def frames_below_main(self):
+        pass
+
+    @staticmethod
+    @abc.abstractmethod
+    def _evaluate_result_value(expression: str, result_string: str) -> ValueIR:
+        """For the result of an "evaluate" message, return a ValueIR. Implementation must be debugger-specific."""
+
+    def evaluate_expression(self, expression, frame_idx=0) -> ValueIR:
+        # The frame_idx passed in here needs to be translated to the debug adapter's internal frame ID.
+        dap_frame_id = self._debugger_state.frame_map[frame_idx]
+        eval_req_id = self.send_message(
+            self.make_request(
+                "evaluate",
+                {
+                    "expression": expression,
+                    "frameId": dap_frame_id,
+                    "context": "watch",
+                },
+            )
+        )
+        eval_response = self._await_response(eval_req_id)
+        if not eval_response["success"]:
+            result: str = eval_response["message"]
+        else:
+            result: str = eval_response["body"]["result"]
+        type_str = eval_response["body"].get("type")
+
+        return self._evaluate_result_value(expression, result, type_str)
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerBase.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerBase.py
index d4a555085face..bf7552bd5fe3a 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerBase.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerBase.py
@@ -184,7 +184,7 @@ def launch(self):
         pass
 
     @abc.abstractmethod
-    def step(self):
+    def step_in(self):
         pass
 
     @abc.abstractmethod
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ConditionalController.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ConditionalController.py
index ac3054c3a0edf..c53f1419ee138 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ConditionalController.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/ConditionalController.py
@@ -181,7 +181,7 @@ def _run_debugger_custom(self, cmdline):
                     )
                     timed_out = True
 
-            if timed_out:
+            if timed_out or self.debugger.is_finished:
                 break
 
             step_info = self.debugger.get_step_info(self._watches, self._step_index)
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/DefaultController.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/DefaultController.py
index 709bd76e2050c..14270843822f3 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/DefaultController.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/DebuggerControllers/DefaultController.py
@@ -128,7 +128,7 @@ def _run_debugger_custom(self, cmdline):
                     break
 
             if in_source_file(self.source_files, step_info):
-                self.debugger.step()
+                self.debugger.step_in()
             else:
                 self.debugger.go()
 
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/Debuggers.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/Debuggers.py
index 67b715af78698..ef23dcf6bebbc 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/Debuggers.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/Debuggers.py
@@ -24,7 +24,7 @@
 from dex.debugger.DebuggerControllers.DefaultController import DefaultController
 
 from dex.debugger.dbgeng.dbgeng import DbgEng
-from dex.debugger.lldb.LLDB import LLDB
+from dex.debugger.lldb.LLDB import LLDB, LLDBDAP
 from dex.debugger.visualstudio.VisualStudio2015 import VisualStudio2015
 from dex.debugger.visualstudio.VisualStudio2017 import VisualStudio2017
 from dex.debugger.visualstudio.VisualStudio2019 import VisualStudio2019
@@ -39,6 +39,7 @@ def _get_potential_debuggers():  # noqa
     return {
         DbgEng.get_option_name(): DbgEng,
         LLDB.get_option_name(): LLDB,
+        LLDBDAP.get_option_name(): LLDBDAP,
         VisualStudio2015.get_option_name(): VisualStudio2015,
         VisualStudio2017.get_option_name(): VisualStudio2017,
         VisualStudio2019.get_option_name(): VisualStudio2019,
@@ -65,7 +66,27 @@ def add_debugger_tool_base_arguments(parser, defaults):
         metavar="<file>",
         default=None,
         display_default=defaults.lldb_executable,
-        help="location of LLDB executable",
+        help="location of `lldb` executable for --debugger=lldb, or `lldb-dap` for --debugger=lldb-dap",
+    )
+    dap_group = parser.add_argument_group("DAP Debugger arguments")
+    dap_group.add_argument(
+        "--dap-message-log",
+        type=str,
+        metavar="<filepath>",
+        default=None,
+        help="log file for messages between Dexter and the debug adapter; set to '-' to log to stdout",
+    )
+    dap_group.add_argument(
+        "--colorize-dap-log",
+        action="store_true",
+        default=False,
+        help="apply colors to the logged DAP messages",
+    )
+    dap_group.add_argument(
+        "--format-dap-log",
+        type=str,
+        default="pretty",
+        choices=["oneline", "pretty"],
     )
 
 
@@ -156,7 +177,7 @@ def handle_debugger_tool_base_options(context, defaults):  # noqa
     if options.lldb_executable is None:
         options.lldb_executable = defaults.lldb_executable
     else:
-        if getattr(options, "debugger", "lldb") != "lldb":
+        if getattr(options, "debugger", "lldb") not in ("lldb", "lldb-dap"):
             _warn_meaningless_option(context, "--lldb-executable")
 
         options.lldb_executable = os.path.abspath(options.lldb_executable)
@@ -165,6 +186,9 @@ def handle_debugger_tool_base_options(context, defaults):  # noqa
                 '<d>could not find</> <r>"{}"</>'.format(options.lldb_executable)
             )
 
+    if options.dap_message_log is not None and options.dap_message_log != "-":
+        options.dap_message_log = os.path.abspath(options.dap_message_log)
+
 
 def handle_debugger_tool_options(context, defaults):  # noqa
     options = context.options
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/__init__.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/__init__.py
index 6bd389a1dc906..bb292b5b7ba19 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/__init__.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/__init__.py
@@ -5,6 +5,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+from dex.debugger.DAP import DAP
 from dex.debugger.Debuggers import Debuggers
 from dex.debugger.DebuggerControllers.DebuggerControllerBase import (
     DebuggerControllerBase,
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/dbgeng/dbgeng.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/dbgeng/dbgeng.py
index 03aa548495ce1..bf0a0f1986721 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/dbgeng/dbgeng.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/dbgeng/dbgeng.py
@@ -111,7 +111,7 @@ def launch(self, cmdline):
         # We are, by this point, already launched.
         self.step_info = probe_process.probe_state(self.client)
 
-    def step(self):
+    def step_in(self):
         res = setup.step_once(self.client)
         if not res:
             self.finished = True
@@ -122,7 +122,7 @@ def go(self):
         # relevant source file -- this is likely to be a problem when setting
         # breakpoints. Until that's fixed, single step instead of running
         # freely. This isn't very efficient, but at least makes progress.
-        self.step()
+        self.step_in()
 
     def _get_step_info(self, watches, step_index):
         frames = self.step_info
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py
index e8bc65cd3fbe8..4263fdb5aa019 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py
@@ -13,6 +13,7 @@
 import sys
 
 from dex.debugger.DebuggerBase import DebuggerBase, watch_is_active
+from dex.debugger.DAP import DAP
 from dex.dextIR import FrameIR, LocIR, StepIR, StopReason, ValueIR
 from dex.dextIR import StackFrame, SourceLocation, ProgramState
 from dex.utils.Exceptions import DebuggerException, LoadDebuggerException
@@ -203,7 +204,7 @@ def launch(self, cmdline):
         assert num_stopped_threads > 0
         assert self._thread, (self._process, self._thread)
 
-    def step(self):
+    def step_in(self):
         self._thread.StepInto()
         stop_reason = self._thread.GetStopReason()
         # If we (1) completed a step and (2) are sitting at a breakpoint,
@@ -382,3 +383,156 @@ def evaluate_expression(self, expression, frame_idx=0) -> ValueIR:
             is_optimized_away=is_optimized_away,
             is_irretrievable=is_irretrievable,
         )
+
+
+class LLDBDAP(DAP):
+    def __init__(self, context, *args):
+        self.lldb_dap_executable = context.options.lldb_executable
+        super(LLDBDAP, self).__init__(context, *args)
+
+    @classmethod
+    def get_name(cls):
+        return "lldb-dap"
+
+    @classmethod
+    def get_option_name(cls):
+        return "lldb-dap"
+
+    @property
+    def version(self):
+        return 1
+
+    @property
+    def _debug_adapter_name(self) -> str:
+        return "lldb-dap"
+
+    @property
+    def _debug_adapter_executable(self) -> str:
+        return self.lldb_dap_executable
+
+    @property
+    def frames_below_main(self):
+        return [
+            "__scrt_common_main_seh",
+            "__libc_start_main",
+            "__libc_start_call_main",
+            "_start",
+        ]
+
+    def _post_step_hook(self):
+        """Hook to be executed after completing a step request."""
+        if self._debugger_state.stopped_reason == "step":
+            trace_req_id = self.send_message(
+                self.make_request(
+                    "stackTrace", {"threadId": self._debugger_state.thread, "levels": 1}
+                )
+            )
+            trace_response = self._await_response(trace_req_id)
+            if not trace_response["success"]:
+                raise DebuggerException("failed to get stack frames")
+            stackframes = trace_response["body"]["stackFrames"]
+            path = stackframes[0]["source"]["path"]
+            addr = stackframes[0]["instructionPointerReference"]
+            if any(
+                self._debugger_state.bp_addr_map.get(self.dex_id_to_dap_id[dex_bp_id])
+                == addr
+                for dex_bp_id in self.file_to_bp.get(path, [])
+            ):
+                # Step again now to get to the breakpoint.
+                step_req_id = self.send_message(
+                    self.make_request(
+                        "stepIn", {"threadId": self._debugger_state.thread}
+                    )
+                )
+                response = self._await_response(step_req_id)
+                if not response["success"]:
+                    raise DebuggerException("failed to step")
+
+    def _get_launch_params(self, cmdline):
+        cwd = os.getcwd()
+        return {
+            "cwd": cwd,
+            "args": cmdline,
+            "program": self.context.options.executable,
+        }
+
+    @staticmethod
+    def _evaluate_result_value(
+        expression: str, result_string: str, type_string
+    ) -> ValueIR:
+        could_evaluate = not any(
+            s in result_string
+            for s in [
+                "Can't run the expression locally",
+                "use of undeclared identifier",
+                "no member named",
+                "Couldn't lookup symbols",
+                "Couldn't look up symbols",
+                "reference to local variable",
+                "invalid use of 'this' outside of a non-static member function",
+            ]
+        )
+
+        is_optimized_away = any(
+            s in result_string
+            for s in [
+                "value may have been optimized out",
+            ]
+        )
+
+        is_irretrievable = any(
+            s in result_string
+            for s in [
+                "couldn't get the value of variable",
+                "couldn't read its memory",
+                "couldn't read from memory",
+                "Cannot access memory at address",
+                "invalid address (fault address:",
+            ]
+        )
+
+        if could_evaluate and not is_irretrievable and not is_optimized_away:
+            error_string = None
+        else:
+            error_string = result_string
+
+        return ValueIR(
+            expression=expression,
+            value=result_string,
+            type_name=type_string,
+            error_string=error_string,
+            could_evaluate=could_evaluate,
+            is_optimized_away=is_optimized_away,
+            is_irretrievable=is_irretrievable,
+        )
+
+    def _update_requested_bp_list(self, bp_list):
+        """ "As lldb-dap cannot have multiple breakpoints at the same location with different conditions, we must
+        manually merge conditions here."""
+        line_to_cond = {}
+        for bp in bp_list:
+            if bp.condition is None:
+                line_to_cond[bp.line] = None
+                continue
+            # If we have a condition, we merge it with the existing condition if one exists, unless the known condition
+            # is None in which case we preserve the None condition (as the underlying breakpoint should always be hit).
+            if bp.line not in line_to_cond:
+                line_to_cond[bp.line] = f"({bp.condition})"
+            elif line_to_cond[bp.line] is not None:
+                line_to_cond[bp.line] = f"{line_to_cond[bp.line]} || ({bp.condition})"
+            bp.condition = line_to_cond[bp.line]
+        return bp_list
+
+    def _confirm_triggered_breakpoint_ids(self, dex_bp_ids):
+        """ "As lldb returns every breakpoint at the current PC regardless of whether their condition was met, we must
+        manually check conditions here."""
+        confirmed_breakpoint_ids = set()
+        for dex_bp_id in dex_bp_ids:
+            _, _, cond = self.bp_info[dex_bp_id]
+            if cond is None:
+                confirmed_breakpoint_ids.add(dex_bp_id)
+                continue
+            valueIR = self.evaluate_expression(cond)
+            if valueIR.type_name == "bool" and valueIR.value == "true":
+                confirmed_breakpoint_ids.add(dex_bp_id)
+        return confirmed_breakpoint_ids
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
index a7f12cde1f047..1a1c8e9a8d9aa 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
@@ -308,7 +308,7 @@ def launch(self, cmdline):
         self.context.logger.note("Launching VS debugger...")
         self._fn_go(False)
 
-    def step(self):
+    def step_in(self):
         self._fn_step(False)
 
     def go(self) -> ReturnCode:
diff --git a/flang-rt/include/flang-rt/runtime/file.h b/flang-rt/include/flang-rt/runtime/file.h
index 3bba29722b3b8..6e35fe89b5341 100644
--- a/flang-rt/include/flang-rt/runtime/file.h
+++ b/flang-rt/include/flang-rt/runtime/file.h
@@ -68,7 +68,7 @@ class OpenFile {
   void WaitAll(IoErrorHandler &);
 
   // INQUIRE(POSITION=)
-  Position InquirePosition() const;
+  Position InquirePosition(FileOffset offset) const;
 
 private:
   struct Pending {
@@ -80,7 +80,7 @@ class OpenFile {
   void CheckOpen(const Terminator &);
   bool Seek(FileOffset, IoErrorHandler &);
   bool RawSeek(FileOffset);
-  bool RawSeekToEnd();
+  bool SeekToEnd(IoErrorHandler &);
   int PendingResult(const Terminator &, int);
   void SetPosition(FileOffset pos) {
     position_ = pos;
diff --git a/flang-rt/include/flang-rt/runtime/io-error.h b/flang-rt/include/flang-rt/runtime/io-error.h
index 1cef6a208f374..3e8401036f289 100644
--- a/flang-rt/include/flang-rt/runtime/io-error.h
+++ b/flang-rt/include/flang-rt/runtime/io-error.h
@@ -38,6 +38,9 @@ class IoErrorHandler : public Terminator {
   RT_API_ATTRS bool InError() const {
     return ioStat_ != IostatOk || pendingError_ != IostatOk;
   }
+  RT_API_ATTRS bool HasErrorRecovery() const {
+    return (flags_ & (hasIoStat | hasErr)) != 0;
+  }
 
   // For I/O statements that detect fatal errors in their
   // Begin...() API routines before it is known whether they
diff --git a/flang-rt/include/flang-rt/runtime/io-stmt.h b/flang-rt/include/flang-rt/runtime/io-stmt.h
index 0b3194da0ab4c..3d1ca5091e923 100644
--- a/flang-rt/include/flang-rt/runtime/io-stmt.h
+++ b/flang-rt/include/flang-rt/runtime/io-stmt.h
@@ -438,7 +438,9 @@ template <>
 class ListDirectedStatementState<Direction::Input>
     : public FormattedIoStatementState<Direction::Input> {
 public:
-  RT_API_ATTRS bool inNamelistSequence() const { return inNamelistSequence_; }
+  RT_API_ATTRS const NamelistGroup *namelistGroup() const {
+    return namelistGroup_;
+  }
   RT_API_ATTRS int EndIoStatement();
 
   // Skips value separators, handles repetition and null values.
@@ -451,18 +453,19 @@ class ListDirectedStatementState<Direction::Input>
   // input statement.  This member function resets some state so that
   // repetition and null values work correctly for each successive
   // NAMELIST input item.
-  RT_API_ATTRS void ResetForNextNamelistItem(bool inNamelistSequence) {
+  RT_API_ATTRS void ResetForNextNamelistItem(
+      const NamelistGroup *namelistGroup) {
     remaining_ = 0;
     if (repeatPosition_) {
       repeatPosition_->Cancel();
     }
     eatComma_ = false;
     realPart_ = imaginaryPart_ = false;
-    inNamelistSequence_ = inNamelistSequence;
+    namelistGroup_ = namelistGroup;
   }
 
 protected:
-  bool inNamelistSequence_{false};
+  const NamelistGroup *namelistGroup_{nullptr};
 
 private:
   int remaining_{0}; // for "r*" repetition
@@ -693,6 +696,7 @@ class ChildListIoStatementState : public ChildIoStatementState<DIR>,
   RT_API_ATTRS ChildListIoStatementState(
       ChildIo &, const char *sourceFile = nullptr, int sourceLine = 0);
   using ListDirectedStatementState<DIR>::GetNextDataEdit;
+  RT_API_ATTRS bool AdvanceRecord(int = 1);
   RT_API_ATTRS int EndIoStatement();
 };
 
@@ -726,6 +730,9 @@ class OpenStatementState : public ExternalIoStatementBase {
   RT_API_ATTRS void set_isUnformatted(bool yes = true) {
     isUnformatted_ = yes;
   } // FORM=
+  RT_API_ATTRS void set_mustBeFormatted(bool yes = true) {
+    mustBeFormatted_ = yes;
+  }
 
   RT_API_ATTRS void CompleteOperation();
   RT_API_ATTRS int EndIoStatement();
@@ -740,6 +747,7 @@ class OpenStatementState : public ExternalIoStatementBase {
   OwningPtr<char> path_;
   std::size_t pathLength_{};
   Fortran::common::optional<bool> isUnformatted_;
+  Fortran::common::optional<bool> mustBeFormatted_;
   Fortran::common::optional<Access> access_;
 };
 
diff --git a/flang-rt/lib/cuda/allocatable.cpp b/flang-rt/lib/cuda/allocatable.cpp
index c77819e9440d7..ff1a225d66ce9 100644
--- a/flang-rt/lib/cuda/allocatable.cpp
+++ b/flang-rt/lib/cuda/allocatable.cpp
@@ -44,13 +44,6 @@ int RTDEF(CUFAllocatableAllocateSync)(Descriptor &desc, int64_t *stream,
 int RTDEF(CUFAllocatableAllocate)(Descriptor &desc, int64_t *stream,
     bool *pinned, bool hasStat, const Descriptor *errMsg,
     const char *sourceFile, int sourceLine) {
-  if (desc.HasAddendum()) {
-    Terminator terminator{sourceFile, sourceLine};
-    // TODO: This require a bit more work to set the correct type descriptor
-    // address
-    terminator.Crash(
-        "not yet implemented: CUDA descriptor allocation with addendum");
-  }
   // Perform the standard allocation.
   int stat{RTNAME(AllocatableAllocate)(
       desc, stream, hasStat, errMsg, sourceFile, sourceLine)};
diff --git a/flang-rt/lib/cuda/pointer.cpp b/flang-rt/lib/cuda/pointer.cpp
index 0ed2b0a2b751f..d3f5cfe8e96a1 100644
--- a/flang-rt/lib/cuda/pointer.cpp
+++ b/flang-rt/lib/cuda/pointer.cpp
@@ -25,13 +25,6 @@ RT_EXT_API_GROUP_BEGIN
 int RTDEF(CUFPointerAllocate)(Descriptor &desc, int64_t *stream, bool *pinned,
     bool hasStat, const Descriptor *errMsg, const char *sourceFile,
     int sourceLine) {
-  if (desc.HasAddendum()) {
-    Terminator terminator{sourceFile, sourceLine};
-    // TODO: This require a bit more work to set the correct type descriptor
-    // address
-    terminator.Crash(
-        "not yet implemented: CUDA descriptor allocation with addendum");
-  }
   // Perform the standard allocation.
   int stat{
       RTNAME(PointerAllocate)(desc, hasStat, errMsg, sourceFile, sourceLine)};
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index 332c0872e065f..dc2db1d9902cb 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -251,19 +251,33 @@ else()
   add_win_flangrt_runtime(STATIC dynamic     MultiThreadedDLL      INSTALL_WITH_TOOLCHAIN)
   add_win_flangrt_runtime(STATIC dynamic_dbg MultiThreadedDebugDLL INSTALL_WITH_TOOLCHAIN)
 
-  # Unittests link against LLVMSupport which is using CMake's default runtime
-  # library selection, which is either MultiThreadedDLL or MultiThreadedDebugDLL
-  # depending on the configuration. They have to match or linking will fail.
+  # Unittests link against LLVMSupport. If CMAKE_MSVC_RUNTIME_LIBRARY is set,
+  # that will have been used for LLVMSupport so it must also be used here.
+  # Otherwise this will use CMake's default runtime library selection, which
+  # is either MultiThreadedDLL or MultiThreadedDebugDLL depending on the configuration.
+  # They have to match or linking will fail.
   if (GENERATOR_IS_MULTI_CONFIG)
     # We cannot select an ALIAS library because it may be different
     # per configuration. Fallback to CMake's default.
     add_win_flangrt_runtime(STATIC unittest "" EXCLUDE_FROM_ALL)
   else ()
-    string(TOLOWER ${CMAKE_BUILD_TYPE} build_type)
-    if (build_type STREQUAL "debug")
-      add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic_dbg)
-    else ()
-      add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic)
-    endif ()
+    # Check if CMAKE_MSVC_RUNTIME_LIBRARY was set.
+    if (CMAKE_MSVC_RUNTIME_LIBRARY STREQUAL "MultiThreaded")
+        add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.static)
+    elseif (CMAKE_MSVC_RUNTIME_LIBRARY STREQUAL "MultiThreadedDLL")
+        add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic)
+    elseif (CMAKE_MSVC_RUNTIME_LIBRARY STREQUAL "MultiThreadedDebug")
+        add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.static_dbg)
+    elseif (CMAKE_MSVC_RUNTIME_LIBRARY STREQUAL "MultiThreadedDebugDLL")
+        add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic_dbg)
+    else()
+      # Default based on the build type.
+      string(TOLOWER ${CMAKE_BUILD_TYPE} build_type)
+      if (build_type STREQUAL "debug")
+          add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic_dbg)
+      else ()
+          add_library(flang_rt.runtime.unittest ALIAS flang_rt.runtime.dynamic)
+      endif ()
+    endif()
   endif ()
 endif()
diff --git a/flang-rt/lib/runtime/assign.cpp b/flang-rt/lib/runtime/assign.cpp
index 7cf4147a94a95..2c29a98d5a5cb 100644
--- a/flang-rt/lib/runtime/assign.cpp
+++ b/flang-rt/lib/runtime/assign.cpp
@@ -88,23 +88,32 @@ static inline RT_API_ATTRS bool MustDeallocateLHS(
 // originally deallocated or because it required reallocation
 static RT_API_ATTRS int AllocateAssignmentLHS(
     Descriptor &to, const Descriptor &from, Terminator &terminator, int flags) {
-  to.raw().type = from.raw().type;
-  if (!(flags & ExplicitLengthCharacterLHS)) {
-    to.raw().elem_len = from.ElementBytes();
-  }
-  const typeInfo::DerivedType *derived{nullptr};
   DescriptorAddendum *toAddendum{to.Addendum()};
+  const typeInfo::DerivedType *derived{nullptr};
+  if (toAddendum) {
+    derived = toAddendum->derivedType();
+  }
   if (const DescriptorAddendum * fromAddendum{from.Addendum()}) {
-    derived = fromAddendum->derivedType();
-    if (toAddendum) {
-      toAddendum->set_derivedType(derived);
-      std::size_t lenParms{derived ? derived->LenParameters() : 0};
+    if (!derived || (flags & PolymorphicLHS)) {
+      derived = fromAddendum->derivedType();
+    }
+    if (toAddendum && derived) {
+      std::size_t lenParms{derived->LenParameters()};
       for (std::size_t j{0}; j < lenParms; ++j) {
         toAddendum->SetLenParameterValue(j, fromAddendum->LenParameterValue(j));
       }
     }
-  } else if (toAddendum) {
-    toAddendum->set_derivedType(nullptr);
+  } else {
+    derived = nullptr;
+  }
+  if (toAddendum) {
+    toAddendum->set_derivedType(derived);
+  }
+  to.raw().type = from.raw().type;
+  if (derived) {
+    to.raw().elem_len = derived->sizeInBytes();
+  } else if (!(flags & ExplicitLengthCharacterLHS)) {
+    to.raw().elem_len = from.ElementBytes();
   }
   // subtle: leave bounds in place when "from" is scalar (10.2.1.3(3))
   int rank{from.rank()};
@@ -369,6 +378,9 @@ RT_API_ATTRS int AssignTicket::Begin(WorkQueue &workQueue) {
         return status;
       }
     } else if (!toDerived_->noDestructionNeeded()) {
+      // F'2023 9.7.3.2 p7: "When an intrinsic assignment statement (10.2.1.3)
+      // is executed, any noncoarray allocated allocatable subobject of the
+      // variable is deallocated before the assignment takes place."
       if (int status{
               workQueue.BeginDestroy(to_, *toDerived_, /*finalize=*/false)};
           status != StatOk && status != StatContinue) {
diff --git a/flang-rt/lib/runtime/edit-input.cpp b/flang-rt/lib/runtime/edit-input.cpp
index 80cc085d99541..1bfc16cbc966d 100644
--- a/flang-rt/lib/runtime/edit-input.cpp
+++ b/flang-rt/lib/runtime/edit-input.cpp
@@ -534,7 +534,7 @@ static RT_API_ATTRS ScannedRealInput ScanRealInput(
       next = io.NextInField(remaining, edit);
     }
     if (!next || *next == ')') { // NextInField fails on separators like ')'
-      std::size_t byteCount{0};
+      std::size_t byteCount{1};
       if (!next) {
         next = io.GetCurrentChar(byteCount);
       }
@@ -663,6 +663,7 @@ static RT_API_ATTRS bool TryFastPathRealDecimalInput(
   *reinterpret_cast<decimal::BinaryFloatingPointNumber<PRECISION> *>(n) =
       converted.binary;
   io.HandleRelativePosition(p - str);
+  io.GotChar(p - str);
   // Set FP exception flags
   if (converted.flags != decimal::ConversionResultFlags::Exact) {
     RaiseFPExceptions(converted.flags);
diff --git a/flang-rt/lib/runtime/extensions.cpp b/flang-rt/lib/runtime/extensions.cpp
index f6c39468d5655..a24810b4f344a 100644
--- a/flang-rt/lib/runtime/extensions.cpp
+++ b/flang-rt/lib/runtime/extensions.cpp
@@ -18,6 +18,7 @@
 #include "flang/Runtime/entry-names.h"
 #include "flang/Runtime/io-api.h"
 #include "flang/Runtime/iostat-consts.h"
+#include <atomic>
 #include <chrono>
 #include <cstdio>
 #include <cstring>
@@ -57,10 +58,76 @@ inline void CtimeBuffer(char *buffer, size_t bufsize, const time_t cur_time,
 #include <direct.h>
 #endif
 
-extern "C" {
-
 namespace Fortran::runtime {
 
+// Common implementation that could be used for either SECNDS() or SECNDSD(),
+// which are defined for float or double.
+template <typename T> T SecndsImpl(T *refTime) {
+  static_assert(std::is_same<T, float>::value || std::is_same<T, double>::value,
+      "T must be float or double");
+  constexpr T FAIL_SECNDS{T{-1.0}}; // Failure code for this function
+  // Failure code for time functions that return std::time_t
+  constexpr std::time_t FAIL_TIME{std::time_t{-1}};
+  constexpr std::time_t TIME_UNINITIALIZED{std::time_t{0}};
+  if (!refTime) {
+    return FAIL_SECNDS;
+  }
+  std::time_t now{std::time(nullptr)};
+  if (now == FAIL_TIME) {
+    return FAIL_SECNDS;
+  }
+  // In case we are using a float result, we can only precisely store
+  // 2^24 seconds, which comes out to about 194 days. Thus, need to pick
+  // a starting point, which will allow us to keep the time diffs as precise
+  // as possible. Given the description of this function, midnight of the
+  // current day is the best starting point.
+  static std::atomic<std::time_t> startingPoint{TIME_UNINITIALIZED};
+  // "Acquire" will give us writes from other threads.
+  std::time_t localStartingPoint{startingPoint.load(std::memory_order_acquire)};
+  // Initialize startingPoint if we haven't initialized it yet or
+  // if we were passed 0.0, which indicates to compute seconds from
+  // current day's midnight.
+  if (localStartingPoint == TIME_UNINITIALIZED || *refTime == 0.0) {
+    // Compute midnight in the current timezone and try to initialize
+    // startingPoint with it. If there are any errors during computation,
+    // exit with error and hope that the other threads have better luck
+    // (or the user retries the call).
+    struct tm timeInfo;
+#ifdef _WIN32
+    if (localtime_s(&timeInfo, &now)) {
+#else
+    if (!localtime_r(&now, &timeInfo)) {
+#endif
+      return FAIL_SECNDS;
+    }
+    // Back to midnight
+    timeInfo.tm_hour = 0;
+    timeInfo.tm_min = 0;
+    timeInfo.tm_sec = 0;
+    localStartingPoint = std::mktime(&timeInfo);
+    if (localStartingPoint == FAIL_TIME) {
+      return FAIL_SECNDS;
+    }
+    INTERNAL_CHECK(localStartingPoint > TIME_UNINITIALIZED);
+    // Attempt to atomically set startingPoint to localStartingPoint
+    std::time_t expected{TIME_UNINITIALIZED};
+    if (startingPoint.compare_exchange_strong(expected, localStartingPoint,
+            std::memory_order_acq_rel, // "Acquire and release" on success
+            std::memory_order_acquire)) { // "Acquire" on failure
+      // startingPoint was set to localStartingPoint
+    } else {
+      // startingPoint was already initialized and its value was loaded
+      // into `expected`. Discard our precomputed midnight value in favor
+      // of the one from startingPoint.
+      localStartingPoint = expected;
+    }
+  }
+  double diffStartingPoint{std::difftime(now, localStartingPoint)};
+  return static_cast<T>(diffStartingPoint) - *refTime;
+}
+
+extern "C" {
+
 gid_t RTNAME(GetGID)() {
 #ifdef _WIN32
   // Group IDs don't exist on Windows, return 1 to avoid errors
@@ -303,6 +370,17 @@ void FORTRAN_PROCEDURE_NAME(qsort)(int *array, int *len, int *isize,
 // PERROR(STRING)
 void RTNAME(Perror)(const char *str) { perror(str); }
 
+// GNU extension function SECNDS(refTime)
+float FORTRAN_PROCEDURE_NAME(secnds)(float *refTime) {
+  return SecndsImpl(refTime);
+}
+
+float RTNAME(Secnds)(float *refTime, const char *sourceFile, int line) {
+  Terminator terminator{sourceFile, line};
+  RUNTIME_CHECK(terminator, refTime != nullptr);
+  return FORTRAN_PROCEDURE_NAME(secnds)(refTime);
+}
+
 // GNU extension function TIME()
 std::int64_t RTNAME(time)() { return time(nullptr); }
 
@@ -337,5 +415,6 @@ std::int64_t RTNAME(Ftell)(int unitNumber) {
 }
 } // namespace io
 
-} // namespace Fortran::runtime
 } // extern "C"
+
+} // namespace Fortran::runtime
diff --git a/flang-rt/lib/runtime/external-unit.cpp b/flang-rt/lib/runtime/external-unit.cpp
index b8004d6315994..42441e59d9bb6 100644
--- a/flang-rt/lib/runtime/external-unit.cpp
+++ b/flang-rt/lib/runtime/external-unit.cpp
@@ -57,7 +57,7 @@ ExternalFileUnit *ExternalFileUnit::LookUpOrCreate(
 }
 
 ExternalFileUnit *ExternalFileUnit::LookUpOrCreateAnonymous(int unit,
-    Direction dir, Fortran::common::optional<bool> isUnformatted,
+    Direction dir, common::optional<bool> isUnformatted,
     IoErrorHandler &handler) {
   // Make sure that the returned anonymous unit has been opened,
   // not just created in the unitMap.
@@ -109,8 +109,8 @@ ExternalFileUnit &ExternalFileUnit::NewUnit(
   return unit;
 }
 
-bool ExternalFileUnit::OpenUnit(Fortran::common::optional<OpenStatus> status,
-    Fortran::common::optional<Action> action, Position position,
+bool ExternalFileUnit::OpenUnit(common::optional<OpenStatus> status,
+    common::optional<Action> action, Position position,
     OwningPtr<char> &&newPath, std::size_t newPathLength, Convert convert,
     IoErrorHandler &handler) {
   if (convert == Convert::Unknown) {
@@ -131,6 +131,7 @@ bool ExternalFileUnit::OpenUnit(Fortran::common::optional<OpenStatus> status,
     if (!newPath.get() || isSamePath) {
       // OPEN of existing unit, STATUS='OLD' or unspecified, not new FILE=
       newPath.reset();
+      Open(status.value_or(OpenStatus::Old), action, position, handler);
       return impliedClose;
     }
     // Otherwise, OPEN on open unit with new FILE= implies CLOSE
@@ -194,10 +195,9 @@ bool ExternalFileUnit::OpenUnit(Fortran::common::optional<OpenStatus> status,
   return impliedClose;
 }
 
-bool ExternalFileUnit::OpenAnonymousUnit(
-    Fortran::common::optional<OpenStatus> status,
-    Fortran::common::optional<Action> action, Position position,
-    Convert convert, IoErrorHandler &handler) {
+bool ExternalFileUnit::OpenAnonymousUnit(common::optional<OpenStatus> status,
+    common::optional<Action> action, Position position, Convert convert,
+    IoErrorHandler &handler) {
   // I/O to an unconnected unit reads/creates a local file, e.g. fort.7
   std::size_t pathMaxLen{32};
   auto path{SizedNew<char>{handler}(pathMaxLen)};
diff --git a/flang-rt/lib/runtime/file.cpp b/flang-rt/lib/runtime/file.cpp
index 16e73db488727..8255ec8691886 100644
--- a/flang-rt/lib/runtime/file.cpp
+++ b/flang-rt/lib/runtime/file.cpp
@@ -60,10 +60,16 @@ static int openfile_mkstemp(IoErrorHandler &handler) {
   return fd;
 }
 
-void OpenFile::Open(OpenStatus status, Fortran::common::optional<Action> action,
+void OpenFile::Open(OpenStatus status, common::optional<Action> action,
     Position position, IoErrorHandler &handler) {
   if (fd_ >= 0 &&
       (status == OpenStatus::Old || status == OpenStatus::Unknown)) {
+    if (position == Position::Rewind) {
+      Seek(0, handler);
+    } else if (position == Position::Append) {
+      SeekToEnd(handler);
+    }
+    openPosition_ = position; // for INQUIRE(POSITION=)
     return;
   }
   CloseFd(handler);
@@ -131,8 +137,8 @@ void OpenFile::Open(OpenStatus status, Fortran::common::optional<Action> action,
   }
   RUNTIME_CHECK(handler, action.has_value());
   pending_.reset();
-  if (fd_ >= 0 && position == Position::Append && !RawSeekToEnd()) {
-    handler.SignalError(IostatOpenBadAppend);
+  if (fd_ >= 0 && position == Position::Append) {
+    SeekToEnd(handler);
   }
   isTerminal_ = fd_ >= 0 && IsATerminal(fd_);
   mayRead_ = *action != Action::Write;
@@ -322,7 +328,7 @@ int OpenFile::WriteAsynchronously(FileOffset at, const char *buffer,
 }
 
 void OpenFile::Wait(int id, IoErrorHandler &handler) {
-  Fortran::common::optional<int> ioStat;
+  common::optional<int> ioStat;
   Pending *prev{nullptr};
   for (Pending *p{pending_.get()}; p; p = (prev = p)->next.get()) {
     if (p->id == id) {
@@ -353,13 +359,13 @@ void OpenFile::WaitAll(IoErrorHandler &handler) {
   }
 }
 
-Position OpenFile::InquirePosition() const {
+Position OpenFile::InquirePosition(FileOffset offset) const {
   if (openPosition_) { // from OPEN statement
     return *openPosition_;
   } else { // unit has been repositioned since opening
-    if (position_ == knownSize_.value_or(position_ + 1)) {
+    if (offset == knownSize_.value_or(offset + 1)) {
       return Position::Append;
-    } else if (position_ == 0 && mayPosition_) {
+    } else if (offset == 0 && mayPosition_) {
       return Position::Rewind;
     } else {
       return Position::AsIs; // processor-dependent & no common behavior
@@ -391,7 +397,7 @@ bool OpenFile::RawSeek(FileOffset at) {
 #endif
 }
 
-bool OpenFile::RawSeekToEnd() {
+bool OpenFile::SeekToEnd(IoErrorHandler &handler) {
 #ifdef _LARGEFILE64_SOURCE
   std::int64_t at{::lseek64(fd_, 0, SEEK_END)};
 #else
@@ -399,8 +405,10 @@ bool OpenFile::RawSeekToEnd() {
 #endif
   if (at >= 0) {
     knownSize_ = at;
+    SetPosition(at);
     return true;
   } else {
+    handler.SignalError(IostatOpenBadAppend);
     return false;
   }
 }
diff --git a/flang-rt/lib/runtime/io-api.cpp b/flang-rt/lib/runtime/io-api.cpp
index 6af0121437cd5..c7c15e77c0770 100644
--- a/flang-rt/lib/runtime/io-api.cpp
+++ b/flang-rt/lib/runtime/io-api.cpp
@@ -528,6 +528,9 @@ bool IODEF(SetAdvance)(Cookie cookie, const char *keyword, std::size_t length) {
 
 bool IODEF(SetBlank)(Cookie cookie, const char *keyword, std::size_t length) {
   IoStatementState &io{*cookie};
+  if (auto *open{io.get_if<OpenStatementState>()}) {
+    open->set_mustBeFormatted();
+  }
   static const char *keywords[]{"NULL", "ZERO", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
   case 0:
@@ -545,6 +548,9 @@ bool IODEF(SetBlank)(Cookie cookie, const char *keyword, std::size_t length) {
 
 bool IODEF(SetDecimal)(Cookie cookie, const char *keyword, std::size_t length) {
   IoStatementState &io{*cookie};
+  if (auto *open{io.get_if<OpenStatementState>()}) {
+    open->set_mustBeFormatted();
+  }
   static const char *keywords[]{"COMMA", "POINT", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
   case 0:
@@ -562,6 +568,9 @@ bool IODEF(SetDecimal)(Cookie cookie, const char *keyword, std::size_t length) {
 
 bool IODEF(SetDelim)(Cookie cookie, const char *keyword, std::size_t length) {
   IoStatementState &io{*cookie};
+  if (auto *open{io.get_if<OpenStatementState>()}) {
+    open->set_mustBeFormatted();
+  }
   static const char *keywords[]{"APOSTROPHE", "QUOTE", "NONE", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
   case 0:
@@ -583,6 +592,9 @@ bool IODEF(SetDelim)(Cookie cookie, const char *keyword, std::size_t length) {
 bool IODEF(SetPad)(Cookie cookie, const char *keyword, std::size_t length) {
   IoStatementState &io{*cookie};
   IoErrorHandler &handler{io.GetIoErrorHandler()};
+  if (auto *open{io.get_if<OpenStatementState>()}) {
+    open->set_mustBeFormatted();
+  }
   io.mutableModes().pad = YesOrNo(keyword, length, "PAD", handler);
   return !handler.InError();
 }
@@ -617,6 +629,9 @@ bool IODEF(SetRec)(Cookie cookie, std::int64_t rec) {
 
 bool IODEF(SetRound)(Cookie cookie, const char *keyword, std::size_t length) {
   IoStatementState &io{*cookie};
+  if (auto *open{io.get_if<OpenStatementState>()}) {
+    open->set_mustBeFormatted();
+  }
   static const char *keywords[]{"UP", "DOWN", "ZERO", "NEAREST", "COMPATIBLE",
       "PROCESSOR_DEFINED", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
@@ -647,6 +662,9 @@ bool IODEF(SetRound)(Cookie cookie, const char *keyword, std::size_t length) {
 
 bool IODEF(SetSign)(Cookie cookie, const char *keyword, std::size_t length) {
   IoStatementState &io{*cookie};
+  if (auto *open{io.get_if<OpenStatementState>()}) {
+    open->set_mustBeFormatted();
+  }
   static const char *keywords[]{
       "PLUS", "SUPPRESS", "PROCESSOR_DEFINED", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
@@ -784,6 +802,7 @@ bool IODEF(SetCarriagecontrol)(
     io.GetIoErrorHandler().Crash(
         "SetCarriageControl() called after GetNewUnit() for an OPEN statement");
   }
+  open->set_mustBeFormatted();
   static const char *keywords[]{"LIST", "FORTRAN", "NONE", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
   case 0:
@@ -840,6 +859,7 @@ bool IODEF(SetEncoding)(
     io.GetIoErrorHandler().Crash(
         "SetEncoding() called after GetNewUnit() for an OPEN statement");
   }
+  open->set_mustBeFormatted();
   // Allow the encoding to be changed on an open unit -- it's
   // useful and safe.
   static const char *keywords[]{"UTF-8", "DEFAULT", nullptr};
@@ -872,10 +892,10 @@ bool IODEF(SetForm)(Cookie cookie, const char *keyword, std::size_t length) {
   }
   static const char *keywords[]{"FORMATTED", "UNFORMATTED", "BINARY", nullptr};
   switch (IdentifyValue(keyword, length, keywords)) {
-  case 0:
+  case 0: // FORM='FORMATTED'
     open->set_isUnformatted(false);
     break;
-  case 1:
+  case 1: // FORM='UNFORMATTED'
     open->set_isUnformatted(true);
     break;
   case 2: // legacy FORM='BINARY' means an unformatted stream
diff --git a/flang-rt/lib/runtime/io-stmt.cpp b/flang-rt/lib/runtime/io-stmt.cpp
index af44a9d6f2722..28149090eb169 100644
--- a/flang-rt/lib/runtime/io-stmt.cpp
+++ b/flang-rt/lib/runtime/io-stmt.cpp
@@ -352,6 +352,17 @@ void OpenStatementState::CompleteOperation() {
     // Set default format (C.7.4 point 2).
     unit().isUnformatted = unit().access != Access::Sequential;
   }
+  if (unit().isUnformatted.value_or(false) && mustBeFormatted_) {
+    // This is an unformatted unit, but the OPEN statement contained at least
+    // one specifier that is not permitted unless the unit is formatted
+    // (e.g., BLANK=).  Programs that want to detect this error (i.e., tests)
+    // should be informed about it, but don't crash the program otherwise
+    // since most other compilers let it slide.
+    if (HasErrorRecovery()) {
+      SignalError("FORM='UNFORMATTED' is not allowed with OPEN specifiers that "
+                  "apply only to formatted units");
+    }
+  }
   if (!wasExtant_ && InError()) {
     // Release the new unit on failure
     set_destroy();
@@ -1086,7 +1097,7 @@ ChildListIoStatementState<DIR>::ChildListIoStatementState(
   if constexpr (DIR == Direction::Input) {
     if (auto *listInput{child.parent()
                 .get_if<ListDirectedStatementState<Direction::Input>>()}) {
-      this->inNamelistSequence_ = listInput->inNamelistSequence();
+      this->namelistGroup_ = listInput->namelistGroup();
     }
   }
 #else
@@ -1095,10 +1106,14 @@ ChildListIoStatementState<DIR>::ChildListIoStatementState(
 }
 
 template <Direction DIR>
-bool ChildUnformattedIoStatementState<DIR>::Receive(
-    char *data, std::size_t bytes, std::size_t elementBytes) {
+bool ChildListIoStatementState<DIR>::AdvanceRecord(int n) {
 #if !defined(RT_DEVICE_AVOID_RECURSION)
-  return this->child().parent().Receive(data, bytes, elementBytes);
+  // Allow child NAMELIST input to advance
+  if (DIR == Direction::Input && this->mutableModes().inNamelist) {
+    return this->child().parent().AdvanceRecord(n);
+  } else {
+    return false;
+  }
 #else
   this->ReportUnsupportedChildIo();
 #endif
@@ -1114,6 +1129,16 @@ template <Direction DIR> int ChildListIoStatementState<DIR>::EndIoStatement() {
   return ChildIoStatementState<DIR>::EndIoStatement();
 }
 
+template <Direction DIR>
+bool ChildUnformattedIoStatementState<DIR>::Receive(
+    char *data, std::size_t bytes, std::size_t elementBytes) {
+#if !defined(RT_DEVICE_AVOID_RECURSION)
+  return this->child().parent().Receive(data, bytes, elementBytes);
+#else
+  this->ReportUnsupportedChildIo();
+#endif
+}
+
 template class InternalIoStatementState<Direction::Output>;
 template class InternalIoStatementState<Direction::Input>;
 template class InternalFormattedIoStatementState<Direction::Output>;
diff --git a/flang-rt/lib/runtime/namelist.cpp b/flang-rt/lib/runtime/namelist.cpp
index cbc32262ffc22..44a8fe2de3cc0 100644
--- a/flang-rt/lib/runtime/namelist.cpp
+++ b/flang-rt/lib/runtime/namelist.cpp
@@ -44,8 +44,7 @@ bool IODEF(OutputNamelist)(Cookie cookie, const NamelistGroup &group) {
     if ((connection.NeedAdvance(prefixLen) &&
             !(io.AdvanceRecord() && EmitAscii(io, " ", 1))) ||
         !EmitAscii(io, prefix, prefixLen) ||
-        (connection.NeedAdvance(
-             Fortran::runtime::strlen(str) + (suffix != ' ')) &&
+        (connection.NeedAdvance(runtime::strlen(str) + (suffix != ' ')) &&
             !(io.AdvanceRecord() && EmitAscii(io, " ", 1)))) {
       return false;
     }
@@ -102,8 +101,8 @@ static constexpr RT_API_ATTRS char NormalizeIdChar(char32_t ch) {
   return static_cast<char>(ch >= 'A' && ch <= 'Z' ? ch - 'A' + 'a' : ch);
 }
 
-static RT_API_ATTRS bool GetLowerCaseName(
-    IoStatementState &io, char buffer[], std::size_t maxLength) {
+static RT_API_ATTRS bool GetLowerCaseName(IoStatementState &io, char buffer[],
+    std::size_t maxLength, bool crashIfTooLong = true) {
   std::size_t byteLength{0};
   if (auto ch{io.GetNextNonBlank(byteLength)}) {
     if (IsLegalIdStart(*ch)) {
@@ -117,8 +116,10 @@ static RT_API_ATTRS bool GetLowerCaseName(
       if (j <= maxLength) {
         return true;
       }
-      io.GetIoErrorHandler().SignalError(
-          "Identifier '%s...' in NAMELIST input group is too long", buffer);
+      if (crashIfTooLong) {
+        io.GetIoErrorHandler().SignalError(
+            "Identifier '%s...' in NAMELIST input group is too long", buffer);
+      }
     }
   }
   return false;
@@ -356,9 +357,8 @@ static RT_API_ATTRS bool HandleComponent(IoStatementState &io, Descriptor &desc,
     const DescriptorAddendum *addendum{source.Addendum()};
     if (const typeInfo::DerivedType *
         type{addendum ? addendum->derivedType() : nullptr}) {
-      if (const typeInfo::Component *
-          comp{type->FindDataComponent(
-              compName, Fortran::runtime::strlen(compName))}) {
+      if (const typeInfo::Component *comp{
+              type->FindDataComponent(compName, runtime::strlen(compName))}) {
         bool createdDesc{false};
         if (comp->rank() > 0 && source.rank() > 0) {
           // If base and component are both arrays, the component name
@@ -484,7 +484,7 @@ bool IODEF(InputNamelist)(Cookie cookie, const NamelistGroup &group) {
       handler.SignalError("NAMELIST input group has no name");
       return false;
     }
-    if (Fortran::runtime::strcmp(group.groupName, name) == 0) {
+    if (runtime::strcmp(group.groupName, name) == 0) {
       break; // found it
     }
     SkipNamelistGroup(io);
@@ -503,7 +503,7 @@ bool IODEF(InputNamelist)(Cookie cookie, const NamelistGroup &group) {
     }
     std::size_t itemIndex{0};
     for (; itemIndex < group.items; ++itemIndex) {
-      if (Fortran::runtime::strcmp(name, group.item[itemIndex].name) == 0) {
+      if (runtime::strcmp(name, group.item[itemIndex].name) == 0) {
         break;
       }
     }
@@ -577,13 +577,14 @@ bool IODEF(InputNamelist)(Cookie cookie, const NamelistGroup &group) {
     if (const auto *addendum{useDescriptor->Addendum()};
         addendum && addendum->derivedType()) {
       const NonTbpDefinedIoTable *table{group.nonTbpDefinedIo};
-      listInput->ResetForNextNamelistItem(/*inNamelistSequence=*/true);
+      listInput->ResetForNextNamelistItem(&group);
       if (!IONAME(InputDerivedType)(cookie, *useDescriptor, table) &&
           handler.InError()) {
         return false;
       }
     } else {
-      listInput->ResetForNextNamelistItem(useDescriptor->rank() > 0);
+      listInput->ResetForNextNamelistItem(
+          useDescriptor->rank() > 0 ? &group : nullptr);
       if (!descr::DescriptorIO<Direction::Input>(io, *useDescriptor) &&
           handler.InError()) {
         return false;
@@ -607,27 +608,51 @@ bool IODEF(InputNamelist)(Cookie cookie, const NamelistGroup &group) {
 }
 
 RT_API_ATTRS bool IsNamelistNameOrSlash(IoStatementState &io) {
-  if (auto *listInput{
-          io.get_if<ListDirectedStatementState<Direction::Input>>()}) {
-    if (listInput->inNamelistSequence()) {
-      SavedPosition savedPosition{io};
-      std::size_t byteCount{0};
-      if (auto ch{io.GetNextNonBlank(byteCount)}) {
-        if (IsLegalIdStart(*ch)) {
-          do {
-            io.HandleRelativePosition(byteCount);
-            ch = io.GetCurrentChar(byteCount);
-          } while (ch && IsLegalIdChar(*ch));
-          ch = io.GetNextNonBlank(byteCount);
-          // TODO: how to deal with NaN(...) ambiguity?
-          return ch && (*ch == '=' || *ch == '(' || *ch == '%');
-        } else {
-          return *ch == '/' || *ch == '&' || *ch == '$';
-        }
-      }
+  auto *listInput{io.get_if<ListDirectedStatementState<Direction::Input>>()};
+  if (!listInput || !listInput->namelistGroup()) {
+    return false; // not namelist
+  }
+  SavedPosition savedPosition{io};
+  std::size_t byteCount{0};
+  auto ch{io.GetNextNonBlank(byteCount)};
+  if (!ch) {
+    return false;
+  } else if (!IsLegalIdStart(*ch)) {
+    return *ch == '/' || *ch == '&' || *ch == '$';
+  }
+  char id[nameBufferSize];
+  if (!GetLowerCaseName(io, id, sizeof id, /*crashIfTooLong=*/false)) {
+    return true; // long name
+  }
+  // It looks like a name, but might be "inf" or "nan".  Check what
+  // follows.
+  ch = io.GetNextNonBlank(byteCount);
+  if (!ch) {
+    return false;
+  } else if (*ch == '=' || *ch == '%') {
+    return true;
+  } else if (*ch != '(') {
+    return false;
+  } else if (runtime::strcmp(id, "nan") != 0) {
+    return true;
+  }
+  // "nan(" ambiguity
+  int depth{1};
+  while (true) {
+    io.HandleRelativePosition(byteCount);
+    ch = io.GetNextNonBlank(byteCount);
+    if (depth == 0) {
+      // nan(...) followed by '=', '%', or '('?
+      break;
+    } else if (!ch) {
+      return true; // not a valid NaN(...)
+    } else if (*ch == '(') {
+      ++depth;
+    } else if (*ch == ')') {
+      --depth;
     }
   }
-  return false;
+  return ch && (*ch == '=' || *ch == '%' || *ch == '(');
 }
 
 RT_OFFLOAD_API_GROUP_END
diff --git a/flang-rt/lib/runtime/pseudo-unit.cpp b/flang-rt/lib/runtime/pseudo-unit.cpp
index 7e1f3bc86b294..74ce101e25fe4 100644
--- a/flang-rt/lib/runtime/pseudo-unit.cpp
+++ b/flang-rt/lib/runtime/pseudo-unit.cpp
@@ -160,7 +160,7 @@ void PseudoOpenFile::WaitAll(IoErrorHandler &handler) {
   handler.Crash("%s: unsupported", RT_PRETTY_FUNCTION);
 }
 
-Position PseudoOpenFile::InquirePosition() const {
+Position PseudoOpenFile::InquirePosition(FileOffset) const {
   Terminator{__FILE__, __LINE__}.Crash("%s: unsupported", RT_PRETTY_FUNCTION);
 }
 
diff --git a/flang-rt/lib/runtime/unit.h b/flang-rt/lib/runtime/unit.h
index f266a486bb708..db7cdc315184a 100644
--- a/flang-rt/lib/runtime/unit.h
+++ b/flang-rt/lib/runtime/unit.h
@@ -88,7 +88,7 @@ class PseudoOpenFile {
       FileOffset, const char *, std::size_t, IoErrorHandler &);
   RT_API_ATTRS void Wait(int id, IoErrorHandler &);
   RT_API_ATTRS void WaitAll(IoErrorHandler &);
-  RT_API_ATTRS Position InquirePosition() const;
+  RT_API_ATTRS Position InquirePosition(FileOffset) const;
 };
 #endif // defined(RT_USE_PSEUDO_FILE_UNIT)
 
@@ -197,6 +197,10 @@ class ExternalFileUnit : public ConnectionState,
 
   RT_API_ATTRS int GetAsynchronousId(IoErrorHandler &);
   RT_API_ATTRS bool Wait(int);
+  RT_API_ATTRS Position InquirePosition() const {
+    return OpenFileClass::InquirePosition(
+        static_cast<std::int64_t>(frameOffsetInFile_ + recordOffsetInFrame_));
+  }
 
 private:
   static RT_API_ATTRS UnitMap &CreateUnitMap();
diff --git a/flang-rt/test/Driver/ctofortran.f90 b/flang-rt/test/Driver/ctofortran.f90
index e385e7974cdc1..4979a4e68da59 100644
--- a/flang-rt/test/Driver/ctofortran.f90
+++ b/flang-rt/test/Driver/ctofortran.f90
@@ -2,8 +2,8 @@
 ! UNSUPPORTED: offload-cuda
 
 ! RUN: split-file %s %t
-! RUN: %clang -I"%include/flang" -c %t/cfile.c -o %t/cfile.o
-! RUN: %flang -L"%libdir" %t/ffile.f90 %t/cfile.o -o %t/ctofortran
+! RUN: %clang %isysroot -I"%include/flang" -c %t/cfile.c -o %t/cfile.o
+! RUN: %flang %isysroot -L"%libdir" %t/ffile.f90 %t/cfile.o -o %t/ctofortran
 ! RUN: env LD_LIBRARY_PATH="$LD_LIBRARY_PATH:%libdir" %t/ctofortran | FileCheck %s
 
 !--- ffile.f90
diff --git a/flang-rt/test/Runtime/no-cpp-dep.c b/flang-rt/test/Runtime/no-cpp-dep.c
index c98678b69aa14..006f44b4aff7a 100644
--- a/flang-rt/test/Runtime/no-cpp-dep.c
+++ b/flang-rt/test/Runtime/no-cpp-dep.c
@@ -7,7 +7,7 @@ UNSUPPORTED: system-windows
 UNSUPPORTED: offload-cuda
 
 RUN: %if system-aix %{ export OBJECT_MODE=64 %}
-RUN: %cc -std=c99 %s -I%include -L"%libdir" -lflang_rt.runtime -lm \
+RUN: %cc -std=c99 %s %isysroot -I%include -L"%libdir" -lflang_rt.runtime -lm \
 RUN: %if system-aix %{-lpthread %}
 RUN: rm a.out
 */
diff --git a/flang-rt/test/lit.cfg.py b/flang-rt/test/lit.cfg.py
index 032aeef2d5bf6..27f4666a4a42a 100644
--- a/flang-rt/test/lit.cfg.py
+++ b/flang-rt/test/lit.cfg.py
@@ -62,25 +62,24 @@ def shjoin(args, sep=" "):
 # lit writes a '.lit_test_times.txt' file into this directory.
 config.test_exec_root = config.flang_rt_binary_test_dir
 
-# On MacOS, -isysroot is needed to build binaries.
+# On MacOS, some tests need -isysroot to build binaries.
 isysroot_flag = []
 if config.osx_sysroot:
     isysroot_flag = ["-isysroot", config.osx_sysroot]
+config.substitutions.append(("%isysroot", " ".join(isysroot_flag)))
 
 tools = [
     ToolSubst(
         "%flang",
         command=config.flang,
-        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
     ToolSubst(
         "%clang",
         command=FindTool("clang"),
-        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
-    ToolSubst("%cc", command=config.cc, extra_args=isysroot_flag, unresolved="fatal"),
+    ToolSubst("%cc", command=config.cc, unresolved="fatal"),
 ]
 llvm_config.add_tool_substitutions(tools)
 
diff --git a/flang-rt/unittests/CMakeLists.txt b/flang-rt/unittests/CMakeLists.txt
index 831bc8a4c2906..fd63ad11dcf43 100644
--- a/flang-rt/unittests/CMakeLists.txt
+++ b/flang-rt/unittests/CMakeLists.txt
@@ -94,14 +94,6 @@ function(add_flangrt_unittest test_dirname)
   target_link_libraries(${test_dirname} PRIVATE ${ARG_LINK_LIBS})
   add_flangrt_unittest_offload_properties(${test_dirname})
   add_flangrt_dependent_libs(${test_dirname})
-
-  # Required because LLVMSupport is compiled with this option.
-  # FIXME: According to CMake documentation, this is the default. Why is it
-  #        needed? LLVM's add_unittest doesn't set it either.
-  set_target_properties(${test_dirname}
-      PROPERTIES
-        MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL"
-    )
 endfunction()
 
 function(add_flangrt_nongtest_unittest test_name)
diff --git a/flang-rt/unittests/Runtime/Namelist.cpp b/flang-rt/unittests/Runtime/Namelist.cpp
index ee4018e491c32..f190bea14acfe 100644
--- a/flang-rt/unittests/Runtime/Namelist.cpp
+++ b/flang-rt/unittests/Runtime/Namelist.cpp
@@ -334,4 +334,35 @@ TEST(NamelistTests, RealValueForInt) {
   EXPECT_EQ(got, expect);
 }
 
+TEST(NamelistTests, NanInputAmbiguity) {
+  OwningPtr<Descriptor> xDesc{// real :: x(5) = 0.
+      MakeArray<TypeCategory::Real, static_cast<int>(sizeof(float))>(
+          std::vector<int>{5}, std::vector<float>{{0, 0, 0, 0, 0}})};
+  OwningPtr<Descriptor> nanDesc{// real :: nan(2) = 0.
+      MakeArray<TypeCategory::Real, static_cast<int>(sizeof(float))>(
+          std::vector<int>{2}, std::vector<float>{{0, 0}})};
+  const NamelistGroup::Item items[]{{"x", *xDesc}, {"nan", *nanDesc}};
+  const NamelistGroup group{"nml", 2, items};
+  static char t1[]{"&nml x=1 2 nan(q) 4 nan(1)=5 nan(q)/"};
+  StaticDescriptor<1, true> statDesc;
+  Descriptor &internalDesc{statDesc.descriptor()};
+  internalDesc.Establish(TypeCode{CFI_type_char},
+      /*elementBytes=*/std::strlen(t1), t1, 0, nullptr, CFI_attribute_pointer);
+  auto inCookie{IONAME(BeginInternalArrayListInput)(
+      internalDesc, nullptr, 0, __FILE__, __LINE__)};
+  ASSERT_TRUE(IONAME(InputNamelist)(inCookie, group));
+  ASSERT_EQ(IONAME(EndIoStatement)(inCookie), IostatOk)
+      << "namelist real input for nans";
+  char out[40];
+  internalDesc.Establish(TypeCode{CFI_type_char}, /*elementBytes=*/sizeof out,
+      out, 0, nullptr, CFI_attribute_pointer);
+  auto outCookie{IONAME(BeginInternalArrayListOutput)(
+      internalDesc, nullptr, 0, __FILE__, __LINE__)};
+  ASSERT_TRUE(IONAME(OutputNamelist)(outCookie, group));
+  ASSERT_EQ(IONAME(EndIoStatement)(outCookie), IostatOk) << "namelist output";
+  std::string got{out, sizeof out};
+  static const std::string expect{" &NML X= 1. 2. NaN 4. 0.,NAN= 5. NaN/   "};
+  EXPECT_EQ(got, expect);
+}
+
 // TODO: Internal NAMELIST error tests
diff --git a/flang/CMakeLists.txt b/flang/CMakeLists.txt
index 0bfada476348a..c01eb56d5e496 100644
--- a/flang/CMakeLists.txt
+++ b/flang/CMakeLists.txt
@@ -317,7 +317,7 @@ if (NOT ENABLE_LINKER_BUILD_ID)
   set(ENABLE_LINKER_BUILD_ID OFF CACHE BOOL "pass --build-id to ld")
 endif()
 
-set(FLANG_DEFAULT_LINKER "" CACHE STRING
+set(FLANG_DEFAULT_LINKER "${CLANG_DEFAULT_LINKER}" CACHE STRING
   "Default linker to use (linker name or absolute path, empty for platform default)")
 
 set(FLANG_DEFAULT_RTLIB "" CACHE STRING
@@ -495,6 +495,9 @@ endif()
 
 include(AddFlang)
 include(FlangCommon)
+include(GetClangResourceDir)
+
+get_clang_resource_dir(HEADER_BINARY_DIR PREFIX ${LLVM_LIBRARY_OUTPUT_INTDIR}/.. SUBDIR include)
 
 if (FLANG_INCLUDE_TESTS)
   add_compile_definitions(FLANG_INCLUDE_TESTS=1)
@@ -575,8 +578,6 @@ endif()
 
 # Put ISO_Fortran_binding.h into the include files of the build area now
 # so that we can run tests before installing
-include(GetClangResourceDir)
-get_clang_resource_dir(HEADER_BINARY_DIR PREFIX ${LLVM_LIBRARY_OUTPUT_INTDIR}/.. SUBDIR include)
 configure_file(
   ${FLANG_SOURCE_DIR}/include/flang/ISO_Fortran_binding.h
   ${HEADER_BINARY_DIR}/ISO_Fortran_binding.h COPYONLY)
diff --git a/flang/docs/Extensions.md b/flang/docs/Extensions.md
index 11c67176df200..b20503e542fb8 100644
--- a/flang/docs/Extensions.md
+++ b/flang/docs/Extensions.md
@@ -900,6 +900,16 @@ print *, [(j,j=1,10)]
   since these default values need to be available to process incomplete
   structure constructors.
 
+* When an `ALLOCATE` or `DEALLOCATE` statement with multiple variables
+  has a `STAT=` specifier that allows the program to continue execution
+  after an error, the variables after the one with the error are left
+  deallocated (or allocated).  This interpretation allows the program to
+  identify the variable that encountered the problem while avoiding any
+  ambiguity in the case of multiple errors with distinct status codes.
+  Some compilers work differently; for maximum portability, avoid
+  `ALLOCATE` and `DEALLOCATE` statements with error recovery for
+  multiple variables.
+
 ## De Facto Standard Features
 
 * `EXTENDS_TYPE_OF()` returns `.TRUE.` if both of its arguments have the
diff --git a/flang/docs/ReleaseNotes.md b/flang/docs/ReleaseNotes.md
index 99dc41c907c36..c9623ea08c4e6 100644
--- a/flang/docs/ReleaseNotes.md
+++ b/flang/docs/ReleaseNotes.md
@@ -1,3 +1,6 @@
+<!-- If you want to modify sections/contents permanently, you should modify both
+ReleaseNotes.md and ReleaseNotesTemplate.txt. -->
+
 # Flang |version| (In-Progress) Release Notes
 
 > **warning**
diff --git a/flang/docs/ReleaseNotesTemplate.txt b/flang/docs/ReleaseNotesTemplate.txt
new file mode 100644
index 0000000000000..2ccf5472ee234
--- /dev/null
+++ b/flang/docs/ReleaseNotesTemplate.txt
@@ -0,0 +1,51 @@
+<!-- If you want to modify sections/contents permanently, you should modify both
+ReleaseNotes.md and ReleaseNotesTemplate.txt. -->
+
+# Flang |version| (In-Progress) Release Notes
+
+> **warning**
+>
+> These are in-progress notes for the upcoming LLVM |version| release.
+> Release notes for previous releases can be found on [the Download
+> Page](https://releases.llvm.org/download.html).
+
+## Introduction
+
+This document contains the release notes for the Flang Fortran frontend,
+part of the LLVM Compiler Infrastructure, release |version|. Here we
+describe the status of Flang in some detail, including major
+improvements from the previous release and new feature work. For the
+general LLVM release notes, see [the LLVM
+documentation](https://llvm.org/docs/ReleaseNotes.html). All LLVM
+releases may be downloaded from the [LLVM releases web
+site](https://llvm.org/releases/).
+
+Note that if you are reading this file from a Git checkout, this
+document applies to the *next* release, not the current one. To see the
+release notes for a specific release, please see the [releases
+page](https://llvm.org/releases/).
+
+## Major New Features
+
+## Bug Fixes
+
+## Non-comprehensive list of changes in this release
+
+## New Compiler Flags
+
+## Windows Support
+
+## Fortran Language Changes in Flang
+
+## Build System Changes
+
+## New Issues Found
+
+## Additional Information
+
+Flang's documentation is located in the `flang/docs/` directory in the
+LLVM monorepo.
+
+If you have any questions or comments about Flang, please feel free to
+contact us on the [Discourse
+forums](https://discourse.llvm.org/c/subprojects/flang/33).
diff --git a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
index 5c64870b74be2..cc058bf28329b 100644
--- a/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
+++ b/flang/examples/FlangOmpReport/FlangOmpReportVisitor.cpp
@@ -101,22 +101,8 @@ std::string OpenMPCounterVisitor::getName(const OmpWrapperType &w) {
   return getName(*std::get<const OpenMPDeclarativeConstruct *>(w));
 }
 std::string OpenMPCounterVisitor::getName(const OpenMPDeclarativeConstruct &c) {
-  return std::visit( //
-      Fortran::common::visitors{
-          [&](const OpenMPUtilityConstruct &o) -> std::string {
-            const CharBlock &source{o.source};
-            return normalize_construct_name(source.ToString());
-          },
-          [&](const OmpMetadirectiveDirective &o) -> std::string {
-            const CharBlock &source{o.source};
-            return normalize_construct_name(source.ToString());
-          },
-          [&](const auto &o) -> std::string {
-            const CharBlock &source{std::get<Verbatim>(o.t).source};
-            return normalize_construct_name(source.ToString());
-          },
-      },
-      c.u);
+  return normalize_construct_name(
+      omp::GetOmpDirectiveName(c).source.ToString());
 }
 std::string OpenMPCounterVisitor::getName(const OpenMPConstruct &c) {
   return normalize_construct_name(
diff --git a/flang/include/flang/Common/enum-set.h b/flang/include/flang/Common/enum-set.h
index 5290b76debee8..e048c66a393d0 100644
--- a/flang/include/flang/Common/enum-set.h
+++ b/flang/include/flang/Common/enum-set.h
@@ -175,10 +175,8 @@ template <typename ENUM, std::size_t BITS> class EnumSet {
   constexpr bool empty() const { return none(); }
   void clear() { reset(); }
   void insert(enumerationType x) { set(x); }
-  void insert(enumerationType &&x) { set(x); }
-  void emplace(enumerationType &&x) { set(x); }
+  void emplace(enumerationType x) { set(x); }
   void erase(enumerationType x) { reset(x); }
-  void erase(enumerationType &&x) { reset(x); }
 
   constexpr std::optional<enumerationType> LeastElement() const {
     if (empty()) {
diff --git a/flang/include/flang/Config/config.h.cmake b/flang/include/flang/Config/config.h.cmake
index fd34d3f403631..92fbd14c3d6db 100644
--- a/flang/include/flang/Config/config.h.cmake
+++ b/flang/include/flang/Config/config.h.cmake
@@ -1,10 +1,10 @@
-#===-- include/flang/Config/config.h.cmake ---------------------------------===#
-#
-# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-# See https://llvm.org/LICENSE.txt for license information.
-# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#
-#===------------------------------------------------------------------------===#
+//===-- include/flang/Config/config.h.cmake -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
 
 /* This generated file is for internal use. Do not include it from headers. */
 
@@ -16,6 +16,8 @@
 
 #define FLANG_VERSION            "${FLANG_VERSION}"
 
+#define FLANG_DEFAULT_LINKER     "${FLANG_DEFAULT_LINKER}"
+
 #endif
 
 
diff --git a/flang/include/flang/Evaluate/check-expression.h b/flang/include/flang/Evaluate/check-expression.h
index 0cf12f340ec5c..eb152652f88c3 100644
--- a/flang/include/flang/Evaluate/check-expression.h
+++ b/flang/include/flang/Evaluate/check-expression.h
@@ -64,6 +64,13 @@ bool IsInitialProcedureTarget(const Symbol &);
 bool IsInitialProcedureTarget(const ProcedureDesignator &);
 bool IsInitialProcedureTarget(const Expr<SomeType> &);
 
+// Emit warnings about default REAL literal constants in contexts that
+// will be converted to a higher precision REAL kind than the default.
+void CheckRealWidening(
+    const Expr<SomeType> &, const DynamicType &toType, FoldingContext &);
+void CheckRealWidening(const Expr<SomeType> &,
+    const std::optional<DynamicType> &, FoldingContext &);
+
 // Validate the value of a named constant, the static initial
 // value of a non-pointer non-allocatable non-dummy variable, or the
 // default initializer of a component of a derived type (or instantiation
diff --git a/flang/include/flang/Evaluate/constant.h b/flang/include/flang/Evaluate/constant.h
index d4c6601c37bca..9ae37cd999aa9 100644
--- a/flang/include/flang/Evaluate/constant.h
+++ b/flang/include/flang/Evaluate/constant.h
@@ -128,17 +128,19 @@ class ConstantBase : public ConstantBounds {
   bool empty() const { return values_.empty(); }
   std::size_t size() const { return values_.size(); }
   const std::vector<Element> &values() const { return values_; }
-  constexpr Result result() const { return result_; }
+  Result &result() { return result_; }
+  const Result &result() const { return result_; }
 
   constexpr DynamicType GetType() const { return result_.GetType(); }
   llvm::raw_ostream &AsFortran(llvm::raw_ostream &) const;
+  std::string AsFortran() const;
 
 protected:
   std::vector<Element> Reshape(const ConstantSubscripts &) const;
   std::size_t CopyFrom(const ConstantBase &source, std::size_t count,
       ConstantSubscripts &resultSubscripts, const std::vector<int> *dimOrder);
 
-  Result result_;
+  Result result_; // usually empty except for Real & Complex
   std::vector<Element> values_;
 };
 
@@ -209,6 +211,7 @@ class Constant<Type<TypeCategory::Character, KIND>> : public ConstantBounds {
 
   Constant Reshape(ConstantSubscripts &&) const;
   llvm::raw_ostream &AsFortran(llvm::raw_ostream &) const;
+  std::string AsFortran() const;
   DynamicType GetType() const { return {KIND, length_}; }
   std::size_t CopyFrom(const Constant &source, std::size_t count,
       ConstantSubscripts &resultSubscripts, const std::vector<int> *dimOrder);
diff --git a/flang/include/flang/Evaluate/match.h b/flang/include/flang/Evaluate/match.h
new file mode 100644
index 0000000000000..79da40f7c1338
--- /dev/null
+++ b/flang/include/flang/Evaluate/match.h
@@ -0,0 +1,211 @@
+//===-- include/flang/Evaluate/match.h --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef FORTRAN_EVALUATE_MATCH_H_
+#define FORTRAN_EVALUATE_MATCH_H_
+
+#include "flang/Common/visit.h"
+#include "flang/Evaluate/expression.h"
+#include "llvm/ADT/STLExtras.h"
+
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+namespace Fortran::evaluate {
+namespace match {
+namespace detail {
+template <typename, typename = void> //
+struct IsOperation {
+  static constexpr bool value{false};
+};
+
+template <typename T>
+struct IsOperation<T, std::void_t<decltype(T::operands)>> {
+  static constexpr bool value{true};
+};
+} // namespace detail
+
+template <typename T>
+constexpr bool is_operation_v{detail::IsOperation<T>::value};
+
+template <typename T>
+const evaluate::Expr<T> &deparen(const evaluate::Expr<T> &x) {
+  if (auto *parens{std::get_if<evaluate::Parentheses<T>>(&x.u)}) {
+    return deparen(parens->template operand<0>());
+  } else {
+    return x;
+  }
+}
+
+// Expr<T> matchers (patterns)
+//
+// Each pattern should implement
+//   bool match(const U &input) const
+// member function that returns `true` when the match was successful,
+// and `false` otherwise.
+//
+// Patterns are intended to be composable, i.e. a pattern can take operands
+// which themselves are patterns. This composition is expected to match if
+// the root pattern and all its operands match given input.
+
+/// Matches any input as long as it has the expected type `MatchType`.
+/// Additionally, it sets the member `ref` to the matched input.
+template <typename T> struct TypePattern {
+  using MatchType = llvm::remove_cvref_t<T>;
+
+  template <typename U> bool match(const U &input) const {
+    if constexpr (std::is_same_v<MatchType, U>) {
+      ref = &input;
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  mutable const MatchType *ref{nullptr};
+};
+
+/// Matches one of the patterns provided as template arguments. All of these
+/// patterns should have the same number of operands, i.e. they all should
+/// try to match input expression with the same number of children, i.e.
+/// AnyOfPattern<SomeBinaryOp, OtherBinaryOp> is ok, whereas
+/// AnyOfPattern<SomeBinaryOp, SomeTernaryOp> is not.
+template <typename... Patterns> struct AnyOfPattern {
+  static_assert(sizeof...(Patterns) != 0);
+
+private:
+  using PatternTuple = std::tuple<Patterns...>;
+
+  template <size_t I>
+  using Pattern = typename std::tuple_element<I, PatternTuple>::type;
+
+  template <size_t... Is, typename... Ops>
+  AnyOfPattern(std::index_sequence<Is...>, const Ops &...ops)
+      : patterns(std::make_tuple(Pattern<Is>(ops...)...)) {}
+
+  template <typename P, typename U>
+  bool matchOne(const P &pattern, const U &input) const {
+    if (pattern.match(input)) {
+      ref = &pattern;
+      return true;
+    }
+    return false;
+  }
+
+  template <typename U, size_t... Is>
+  bool matchImpl(const U &input, std::index_sequence<Is...>) const {
+    return (matchOne(std::get<Is>(patterns), input) || ...);
+  }
+
+  PatternTuple patterns;
+
+public:
+  using Indexes = std::index_sequence_for<Patterns...>;
+  using MatchTypes = std::tuple<typename Patterns::MatchType...>;
+
+  template <typename... Ops>
+  AnyOfPattern(const Ops &...ops) : AnyOfPattern(Indexes{}, ops...) {}
+
+  template <typename U> bool match(const U &input) const {
+    return matchImpl(input, Indexes{});
+  }
+
+  mutable std::variant<const Patterns *..., std::monostate> ref{
+      std::monostate{}};
+};
+
+/// Matches any input of type Expr<T>
+/// The indent if this pattern is to be a leaf in multi-operand patterns.
+template <typename T> //
+struct ExprPattern : public TypePattern<evaluate::Expr<T>> {};
+
+/// Matches evaluate::Expr<T> that contains evaluate::Opreration<OpType>.
+template <typename OpType, typename... Ops>
+struct OperationPattern : public TypePattern<OpType> {
+private:
+  using Indexes = std::index_sequence_for<Ops...>;
+
+  template <typename S, size_t... Is>
+  bool matchImpl(const S &op, std::index_sequence<Is...>) const {
+    using TypeS = llvm::remove_cvref_t<S>;
+    if constexpr (is_operation_v<TypeS>) {
+      if constexpr (TypeS::operands == Indexes::size()) {
+        return TypePattern<OpType>::match(op) &&
+            (std::get<Is>(operands).match(op.template operand<Is>()) && ...);
+      }
+    }
+    return false;
+  }
+
+  std::tuple<const Ops &...> operands;
+
+public:
+  using MatchType = OpType;
+
+  OperationPattern(const Ops &...ops, llvm::type_identity<OpType> = {})
+      : operands(ops...) {}
+
+  template <typename T> bool match(const evaluate::Expr<T> &input) const {
+    return common::visit(
+        [&](auto &&s) { return matchImpl(s, Indexes{}); }, deparen(input).u);
+  }
+
+  template <typename U> bool match(const U &input) const {
+    // Only match Expr<T>
+    return false;
+  }
+};
+
+template <typename OpType, typename... Ops>
+OperationPattern(const Ops &...ops, llvm::type_identity<OpType>)
+    -> OperationPattern<OpType, Ops...>;
+
+// Namespace-level definitions
+
+template <typename T> using Expr = ExprPattern<T>;
+
+template <typename OpType, typename... Ops>
+using Op = OperationPattern<OpType, Ops...>;
+
+template <typename Pattern, typename Input>
+bool match(const Pattern &pattern, const Input &input) {
+  return pattern.match(input);
+}
+
+// Specific operation patterns
+
+// -- Add
+template <typename Type, typename Op0, typename Op1>
+struct Add : public Op<evaluate::Add<Type>, Op0, Op1> {
+  using Base = Op<evaluate::Add<Type>, Op0, Op1>;
+
+  Add(const Op0 &op0, const Op1 &op1) : Base(op0, op1) {}
+};
+
+template <typename Type, typename Op0, typename Op1>
+Add<Type, Op0, Op1> add(const Op0 &op0, const Op1 &op1) {
+  return Add<Type, Op0, Op1>(op0, op1);
+}
+
+// -- Mul
+template <typename Type, typename Op0, typename Op1>
+struct Mul : public Op<evaluate::Multiply<Type>, Op0, Op1> {
+  using Base = Op<evaluate::Multiply<Type>, Op0, Op1>;
+
+  Mul(const Op0 &op0, const Op1 &op1) : Base(op0, op1) {}
+};
+
+template <typename Type, typename Op0, typename Op1>
+Mul<Type, Op0, Op1> mul(const Op0 &op0, const Op1 &op1) {
+  return Mul<Type, Op0, Op1>(op0, op1);
+}
+} // namespace match
+} // namespace Fortran::evaluate
+
+#endif // FORTRAN_EVALUATE_MATCH_H_
diff --git a/flang/include/flang/Evaluate/real.h b/flang/include/flang/Evaluate/real.h
index 76d25d9fe2670..dcd74073a4737 100644
--- a/flang/include/flang/Evaluate/real.h
+++ b/flang/include/flang/Evaluate/real.h
@@ -442,6 +442,7 @@ template <typename WORD, int PREC> class Real {
   // or parenthesized constant expression that produces this value.
   llvm::raw_ostream &AsFortran(
       llvm::raw_ostream &, int kind, bool minimal = false) const;
+  std::string AsFortran(int kind, bool minimal = false) const;
 
 private:
   using Significand = Integer<significandBits>; // no implicit bit
diff --git a/flang/include/flang/Evaluate/rewrite.h b/flang/include/flang/Evaluate/rewrite.h
new file mode 100644
index 0000000000000..50259cc0959f4
--- /dev/null
+++ b/flang/include/flang/Evaluate/rewrite.h
@@ -0,0 +1,160 @@
+//===-- include/flang/Evaluate/rewrite.h ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef FORTRAN_EVALUATE_REWRITE_H_
+#define FORTRAN_EVALUATE_REWRITE_H_
+
+#include "flang/Common/visit.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Support/Fortran.h"
+#include "llvm/ADT/STLExtras.h"
+
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+
+namespace Fortran::evaluate {
+namespace rewrite {
+namespace detail {
+template <typename, typename = void> //
+struct IsOperation {
+  static constexpr bool value{false};
+};
+
+template <typename T>
+struct IsOperation<T, std::void_t<decltype(T::operands)>> {
+  static constexpr bool value{true};
+};
+} // namespace detail
+
+template <typename T>
+constexpr bool is_operation_v{detail::IsOperation<T>::value};
+
+/// Individual Expr<T> rewriter that simply constructs an expression that is
+/// identical to the input. This is a suitable base class for all user-defined
+/// rewriters.
+struct Identity {
+  template <typename T, typename U>
+  Expr<T> operator()(Expr<T> &&x, const U &op) {
+    return std::move(x);
+  }
+};
+
+/// Bottom-up Expr<T> rewriter.
+///
+/// The Mutator traverses and reconstructs given Expr<T>. Going bottom-up,
+/// whenever the traversal visits a sub-node of type Expr<U> (for some U),
+/// it will invoke the user-provided rewriter via the () operator.
+///
+/// If x is of type Expr<U>, it will call (in pseudo-code):
+///   rewriter_(x, active_member_of(x.u))
+/// The second parameter is there to make it easier to overload the () operator
+/// for specific operations in Expr<...>.
+///
+/// The user rewriter is only invoked for Expr<U>, not for Operation, nor any
+/// other subobject.
+template <typename Rewriter> struct Mutator {
+  Mutator(Rewriter &rewriter) : rewriter_(rewriter) {}
+
+  template <typename T, typename U = llvm::remove_cvref_t<T>>
+  U operator()(T &&x) {
+    if constexpr (std::is_lvalue_reference_v<T>) {
+      return Mutate(U(x));
+    } else {
+      return Mutate(std::move(x));
+    }
+  }
+
+private:
+  template <typename T> struct LambdaWithRvalueCapture {
+    LambdaWithRvalueCapture(Rewriter &r, Expr<T> &&c)
+        : rewriter_(r), capture_(std::move(c)) {}
+    template <typename S> Expr<T> operator()(const S &s) {
+      return rewriter_(std::move(capture_), s);
+    }
+
+  private:
+    Rewriter &rewriter_;
+    Expr<T> &&capture_;
+  };
+
+  template <typename T, typename = std::enable_if_t<!is_operation_v<T>>>
+  T Mutate(T &&x) const {
+    return std::move(x);
+  }
+
+  template <typename D, typename = std::enable_if_t<is_operation_v<D>>>
+  D Mutate(D &&op, std::make_index_sequence<D::operands> t = {}) const {
+    return MutateOp(std::move(op), t);
+  }
+
+  template <typename T> //
+  Expr<T> Mutate(Expr<T> &&x) const {
+    // First construct the new expression with the rewritten op.
+    Expr<T> n{common::visit(
+        [&](auto &&s) { //
+          return Expr<T>(Mutate(std::move(s)));
+        },
+        std::move(x.u))};
+    // Return the rewritten expression. The second visit is to make sure
+    // that the second argument in the call to the rewriter is a part of
+    // the Expr<T> passed to it.
+    return common::visit(
+        LambdaWithRvalueCapture<T>(rewriter_, std::move(n)), std::move(n.u));
+  }
+
+  template <typename... Ts>
+  std::variant<Ts...> Mutate(std::variant<Ts...> &&u) const {
+    return common::visit(
+        [this](auto &&s) { return Mutate(std::move(s)); }, std::move(u));
+  }
+
+  template <typename... Ts>
+  std::tuple<Ts...> Mutate(std::tuple<Ts...> &&t) const {
+    return MutateTuple(std::move(t), std::index_sequence_for<Ts...>{});
+  }
+
+  template <typename... Ts, size_t... Is>
+  std::tuple<Ts...> MutateTuple(
+      std::tuple<Ts...> &&t, std::index_sequence<Is...>) const {
+    return std::make_tuple(Mutate(std::move(std::get<Is>(t))...));
+  }
+
+  template <typename D, size_t... Is>
+  D MutateOp(D &&op, std::index_sequence<Is...>) const {
+    return D(Mutate(std::move(op.template operand<Is>()))...);
+  }
+
+  template <typename T, size_t... Is>
+  Extremum<T> MutateOp(Extremum<T> &&op, std::index_sequence<Is...>) const {
+    return Extremum<T>(
+        op.ordering, Mutate(std::move(op.template operand<Is>()))...);
+  }
+
+  template <int K, size_t... Is>
+  ComplexComponent<K> MutateOp(
+      ComplexComponent<K> &&op, std::index_sequence<Is...>) const {
+    return ComplexComponent<K>(
+        op.isImaginaryPart, Mutate(std::move(op.template operand<Is>()))...);
+  }
+
+  template <int K, size_t... Is>
+  LogicalOperation<K> MutateOp(
+      LogicalOperation<K> &&op, std::index_sequence<Is...>) const {
+    return LogicalOperation<K>(
+        op.logicalOperator, Mutate(std::move(op.template operand<Is>()))...);
+  }
+
+  Rewriter &rewriter_;
+};
+
+template <typename Rewriter> Mutator(Rewriter &) -> Mutator<Rewriter>;
+} // namespace rewrite
+} // namespace Fortran::evaluate
+
+#endif // FORTRAN_EVALUATE_REWRITE_H_
diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 212356136d6ee..74c6acbcb1ed5 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1144,15 +1144,14 @@ std::optional<std::string> FindImpureCall(
 std::optional<std::string> FindImpureCall(
     FoldingContext &, const ProcedureRef &);
 
-// Predicate: is a scalar expression suitable for naive scalar expansion
-// in the flattening of an array expression?
-// TODO: capture such scalar expansions in temporaries, flatten everything
-class UnexpandabilityFindingVisitor
-    : public AnyTraverse<UnexpandabilityFindingVisitor> {
+// Predicate: does an expression contain anything that would prevent it from
+// being duplicated so that two instances of it then appear in the same
+// expression?
+class UnsafeToCopyVisitor : public AnyTraverse<UnsafeToCopyVisitor> {
 public:
-  using Base = AnyTraverse<UnexpandabilityFindingVisitor>;
+  using Base = AnyTraverse<UnsafeToCopyVisitor>;
   using Base::operator();
-  explicit UnexpandabilityFindingVisitor(bool admitPureCall)
+  explicit UnsafeToCopyVisitor(bool admitPureCall)
       : Base{*this}, admitPureCall_{admitPureCall} {}
   template <typename T> bool operator()(const FunctionRef<T> &procRef) {
     return !admitPureCall_ || !procRef.proc().IsPure();
@@ -1163,14 +1162,22 @@ class UnexpandabilityFindingVisitor
   bool admitPureCall_{false};
 };
 
+template <typename A>
+bool IsSafelyCopyable(const A &x, bool admitPureCall = false) {
+  return !UnsafeToCopyVisitor{admitPureCall}(x);
+}
+
+// Predicate: is a scalar expression suitable for naive scalar expansion
+// in the flattening of an array expression?
+// TODO: capture such scalar expansions in temporaries, flatten everything
 template <typename T>
 bool IsExpandableScalar(const Expr<T> &expr, FoldingContext &context,
     const Shape &shape, bool admitPureCall = false) {
-  if (UnexpandabilityFindingVisitor{admitPureCall}(expr)) {
+  if (IsSafelyCopyable(expr, admitPureCall)) {
+    return true;
+  } else {
     auto extents{AsConstantExtents(context, shape)};
     return extents && !HasNegativeExtent(*extents) && GetSize(*extents) == 1;
-  } else {
-    return true;
   }
 }
 
diff --git a/flang/include/flang/Evaluate/type.h b/flang/include/flang/Evaluate/type.h
index f3bba7790e1a2..222018bb452a0 100644
--- a/flang/include/flang/Evaluate/type.h
+++ b/flang/include/flang/Evaluate/type.h
@@ -274,9 +274,26 @@ class Type<TypeCategory::Unsigned, KIND>
   using Scalar = value::Integer<8 * KIND>;
 };
 
+// Records when a default REAL literal constant is inexactly converted to binary
+// (e.g., 0.1 but not 0.125) to enable a usage warning if the expression in
+// which it appears undergoes an implicit widening conversion.
+class TrackInexactLiteralConversion {
+public:
+  constexpr bool isFromInexactLiteralConversion() const {
+    return isFromInexactLiteralConversion_;
+  }
+  void set_isFromInexactLiteralConversion(bool yes = true) {
+    isFromInexactLiteralConversion_ = yes;
+  }
+
+private:
+  bool isFromInexactLiteralConversion_{false};
+};
+
 template <int KIND>
 class Type<TypeCategory::Real, KIND>
-    : public TypeBase<TypeCategory::Real, KIND> {
+    : public TypeBase<TypeCategory::Real, KIND>,
+      public TrackInexactLiteralConversion {
 public:
   static constexpr int precision{common::PrecisionOfRealKind(KIND)};
   static constexpr int bits{common::BitsForBinaryPrecision(precision)};
@@ -289,7 +306,8 @@ class Type<TypeCategory::Real, KIND>
 // The KIND type parameter on COMPLEX is the kind of each of its components.
 template <int KIND>
 class Type<TypeCategory::Complex, KIND>
-    : public TypeBase<TypeCategory::Complex, KIND> {
+    : public TypeBase<TypeCategory::Complex, KIND>,
+      public TrackInexactLiteralConversion {
 public:
   using Part = Type<TypeCategory::Real, KIND>;
   using Scalar = value::Complex<typename Part::Scalar>;
diff --git a/flang/include/flang/Lower/Cuda.h b/flang/include/flang/Lower/CUDA.h
similarity index 60%
rename from flang/include/flang/Lower/Cuda.h
rename to flang/include/flang/Lower/CUDA.h
index b6f849e3d63f0..4a831fd502af4 100644
--- a/flang/include/flang/Lower/Cuda.h
+++ b/flang/include/flang/Lower/CUDA.h
@@ -1,4 +1,4 @@
-//===-- Lower/Cuda.h -- Cuda Fortran utilities ------------------*- C++ -*-===//
+//===-- Lower/CUDA.h -- CUDA Fortran utilities ------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -14,13 +14,23 @@
 #define FORTRAN_LOWER_CUDA_H
 
 #include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/MutableBox.h"
 #include "flang/Optimizer/Dialect/CUF/CUFOps.h"
+#include "flang/Runtime/allocator-registry-consts.h"
 #include "flang/Semantics/tools.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/OpenACC/OpenACC.h"
 
+namespace mlir {
+class Value;
+class Location;
+class MLIRContext;
+} // namespace mlir
+
 namespace Fortran::lower {
 
+class AbstractConverter;
+
 static inline unsigned getAllocatorIdx(const Fortran::semantics::Symbol &sym) {
   std::optional<Fortran::common::CUDADataAttr> cudaAttr =
       Fortran::semantics::GetCUDADataAttr(&sym.GetUltimate());
@@ -37,6 +47,23 @@ static inline unsigned getAllocatorIdx(const Fortran::semantics::Symbol &sym) {
   return kDefaultAllocator;
 }
 
+void initializeDeviceComponentAllocator(
+    Fortran::lower::AbstractConverter &converter,
+    const Fortran::semantics::Symbol &sym, const fir::MutableBoxValue &box);
+
+mlir::Type gatherDeviceComponentCoordinatesAndType(
+    fir::FirOpBuilder &builder, mlir::Location loc,
+    const Fortran::semantics::Symbol &sym, fir::RecordType recTy,
+    llvm::SmallVector<mlir::Value> &coordinates);
+
+/// Translate the CUDA Fortran attributes of \p sym into the FIR CUDA attribute
+/// representation.
+cuf::DataAttributeAttr
+translateSymbolCUFDataAttribute(mlir::MLIRContext *mlirContext,
+                                const Fortran::semantics::Symbol &sym);
+
+bool isTransferWithConversion(mlir::Value rhs);
+
 } // end namespace Fortran::lower
 
 #endif // FORTRAN_LOWER_CUDA_H
diff --git a/flang/include/flang/Lower/ConvertVariable.h b/flang/include/flang/Lower/ConvertVariable.h
index e05625a229ac7..b938f6be196af 100644
--- a/flang/include/flang/Lower/ConvertVariable.h
+++ b/flang/include/flang/Lower/ConvertVariable.h
@@ -162,12 +162,6 @@ translateSymbolAttributes(mlir::MLIRContext *mlirContext,
                           fir::FortranVariableFlagsEnum extraFlags =
                               fir::FortranVariableFlagsEnum::None);
 
-/// Translate the CUDA Fortran attributes of \p sym into the FIR CUDA attribute
-/// representation.
-cuf::DataAttributeAttr
-translateSymbolCUFDataAttribute(mlir::MLIRContext *mlirContext,
-                                const Fortran::semantics::Symbol &sym);
-
 /// Map a symbol to a given fir::ExtendedValue. This will generate an
 /// hlfir.declare when lowering to HLFIR and map the hlfir.declare result to the
 /// symbol.
diff --git a/flang/include/flang/Lower/OpenMP.h b/flang/include/flang/Lower/OpenMP.h
index 6e150ef4e8e82..581c93f76d627 100644
--- a/flang/include/flang/Lower/OpenMP.h
+++ b/flang/include/flang/Lower/OpenMP.h
@@ -57,6 +57,7 @@ struct Variable;
 struct OMPDeferredDeclareTargetInfo {
   mlir::omp::DeclareTargetCaptureClause declareTargetCaptureClause;
   mlir::omp::DeclareTargetDeviceType declareTargetDeviceType;
+  bool automap = false;
   const Fortran::semantics::Symbol &sym;
 };
 
diff --git a/flang/include/flang/Lower/OpenMP/Clauses.h b/flang/include/flang/Lower/OpenMP/Clauses.h
index 7f317f05f67b7..1ab594ffcd209 100644
--- a/flang/include/flang/Lower/OpenMP/Clauses.h
+++ b/flang/include/flang/Lower/OpenMP/Clauses.h
@@ -219,6 +219,7 @@ using DistSchedule = tomp::clause::DistScheduleT<TypeTy, IdTy, ExprTy>;
 using Doacross = tomp::clause::DoacrossT<TypeTy, IdTy, ExprTy>;
 using DynamicAllocators =
     tomp::clause::DynamicAllocatorsT<TypeTy, IdTy, ExprTy>;
+using DynGroupprivate = tomp::clause::DynGroupprivateT<TypeTy, IdTy, ExprTy>;
 using Enter = tomp::clause::EnterT<TypeTy, IdTy, ExprTy>;
 using Exclusive = tomp::clause::ExclusiveT<TypeTy, IdTy, ExprTy>;
 using Fail = tomp::clause::FailT<TypeTy, IdTy, ExprTy>;
diff --git a/flang/include/flang/Lower/Support/Utils.h b/flang/include/flang/Lower/Support/Utils.h
index e544542e2ff71..697bb05ec8cee 100644
--- a/flang/include/flang/Lower/Support/Utils.h
+++ b/flang/include/flang/Lower/Support/Utils.h
@@ -101,7 +101,7 @@ void privatizeSymbol(
     lower::AbstractConverter &converter, fir::FirOpBuilder &firOpBuilder,
     lower::SymMap &symTable,
     llvm::SetVector<const semantics::Symbol *> &allPrivatizedSymbols,
-    llvm::SmallSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
+    llvm::SmallPtrSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
     const semantics::Symbol *symToPrivatize, OperandsStructType *clauseOps);
 
 } // end namespace Fortran::lower
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index d8b6a9f87ab19..e3a44f147b4cd 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -944,16 +944,15 @@ void genDimInfoFromBox(fir::FirOpBuilder &builder, mlir::Location loc,
                        llvm::SmallVectorImpl<mlir::Value> *strides);
 
 /// Generate an LLVM dialect lifetime start marker at the current insertion
-/// point given an fir.alloca and its constant size in bytes. Returns the value
-/// to be passed to the lifetime end marker.
+/// point given an fir.alloca. Returns the value to be passed to the lifetime
+/// end marker.
 mlir::Value genLifetimeStart(mlir::OpBuilder &builder, mlir::Location loc,
-                             fir::AllocaOp alloc, int64_t size,
-                             const mlir::DataLayout *dl);
+                             fir::AllocaOp alloc, const mlir::DataLayout *dl);
 
 /// Generate an LLVM dialect lifetime end marker at the current insertion point
-/// given an llvm.ptr value and the constant size in bytes of its storage.
+/// given an llvm.ptr value.
 void genLifetimeEnd(mlir::OpBuilder &builder, mlir::Location loc,
-                    mlir::Value mem, int64_t size);
+                    mlir::Value mem);
 
 } // namespace fir::factory
 
diff --git a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
index 2afd50410ae82..88c3ada3ff64f 100644
--- a/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
+++ b/flang/include/flang/Optimizer/Builder/IntrinsicCall.h
@@ -378,6 +378,8 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genNorm2(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genNot(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genNull(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
+  fir::ExtendedValue genNumImages(mlir::Type,
+                                  llvm::ArrayRef<fir::ExtendedValue>);
   template <typename OpTy>
   mlir::Value genNVVMTime(mlir::Type, llvm::ArrayRef<mlir::Value>);
   fir::ExtendedValue genPack(mlir::Type, llvm::ArrayRef<fir::ExtendedValue>);
@@ -449,6 +451,8 @@ struct IntrinsicLibrary {
   fir::ExtendedValue genTranspose(mlir::Type,
                                   llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genThisGrid(mlir::Type, llvm::ArrayRef<mlir::Value>);
+  fir::ExtendedValue genThisImage(mlir::Type,
+                                  llvm::ArrayRef<fir::ExtendedValue>);
   mlir::Value genThisThreadBlock(mlir::Type, llvm::ArrayRef<mlir::Value>);
   mlir::Value genThisWarp(mlir::Type, llvm::ArrayRef<mlir::Value>);
   void genThreadFence(llvm::ArrayRef<fir::ExtendedValue>);
@@ -563,6 +567,15 @@ struct IntrinsicLibrary {
 
   void setResultMustBeFreed() { resultMustBeFreed = true; }
 
+  // Check support of coarray features
+  void checkCoarrayEnabled() {
+    if (converter &&
+        !converter->getFoldingContext().languageFeatures().IsEnabled(
+            Fortran::common::LanguageFeature::Coarray))
+      fir::emitFatalError(loc, "Coarrays disabled, use '-fcoarray' to enable.",
+                          false);
+  }
+
   fir::FirOpBuilder &builder;
   mlir::Location loc;
   bool resultMustBeFreed = false;
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Coarray.h b/flang/include/flang/Optimizer/Builder/Runtime/Coarray.h
new file mode 100644
index 0000000000000..23bb378c30838
--- /dev/null
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Coarray.h
@@ -0,0 +1,53 @@
+//===-- Coarray.h -- generate Coarray intrinsics runtime calls --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FORTRAN_OPTIMIZER_BUILDER_RUNTIME_COARRAY_H
+#define FORTRAN_OPTIMIZER_BUILDER_RUNTIME_COARRAY_H
+
+#include "flang/Lower/AbstractConverter.h"
+#include "flang/Optimizer/Support/InternalNames.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+
+namespace fir {
+class ExtendedValue;
+class FirOpBuilder;
+} // namespace fir
+
+namespace fir::runtime {
+
+// Get the function type for a prif subroutine with a variable number of
+// arguments
+#define PRIF_FUNCTYPE(...)                                                     \
+  mlir::FunctionType::get(builder.getContext(), /*inputs*/ {__VA_ARGS__},      \
+                          /*result*/ {})
+
+// Default prefix for subroutines of PRIF compiled with LLVM
+#define PRIFNAME_SUB(fmt)                                                      \
+  []() {                                                                       \
+    std::ostringstream oss;                                                    \
+    oss << "prif_" << fmt;                                                     \
+    return fir::NameUniquer::doProcedure({"prif"}, {}, oss.str());             \
+  }()
+
+/// Generate Call to runtime prif_init
+mlir::Value genInitCoarray(fir::FirOpBuilder &builder, mlir::Location loc);
+
+/// Generate Call to runtime prif_num_images
+mlir::Value getNumImages(fir::FirOpBuilder &builder, mlir::Location loc);
+
+/// Generate Call to runtime prif_num_images_with_team or
+/// prif_num_images_with_team_number
+mlir::Value getNumImagesWithTeam(fir::FirOpBuilder &builder, mlir::Location loc,
+                                 mlir::Value team);
+
+/// Generate Call to runtime prif_this_image_no_coarray
+mlir::Value getThisImage(fir::FirOpBuilder &builder, mlir::Location loc,
+                         mlir::Value team = {});
+
+} // namespace fir::runtime
+#endif // FORTRAN_OPTIMIZER_BUILDER_RUNTIME_COARRAY_H
diff --git a/flang/include/flang/Optimizer/Builder/Runtime/Main.h b/flang/include/flang/Optimizer/Builder/Runtime/Main.h
index a0586deade42a..d4067b367f73e 100644
--- a/flang/include/flang/Optimizer/Builder/Runtime/Main.h
+++ b/flang/include/flang/Optimizer/Builder/Runtime/Main.h
@@ -25,7 +25,7 @@ namespace fir::runtime {
 
 void genMain(fir::FirOpBuilder &builder, mlir::Location loc,
              const std::vector<Fortran::lower::EnvironmentDefault> &defs,
-             bool initCuda = false);
+             bool initCuda = false, bool initCoarrayEnv = false);
 }
 
 #endif // FORTRAN_OPTIMIZER_BUILDER_RUNTIME_MAIN_H
diff --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
index b7fa8fc3848f2..7d816a8843371 100644
--- a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
+++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h
@@ -237,9 +237,7 @@ class FIROpConversion : public ConvertFIRToLLVMPattern {
   virtual llvm::LogicalResult
   matchAndRewrite(SourceOp op, OneToNOpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const {
-    llvm::SmallVector<mlir::Value> oneToOneOperands =
-        getOneToOneAdaptorOperands(adaptor.getOperands());
-    return matchAndRewrite(op, OpAdaptor(oneToOneOperands, adaptor), rewriter);
+    return dispatchTo1To1(*this, op, adaptor, rewriter);
   }
 
 private:
diff --git a/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h
index 85615a4d69157..4a250d1cc6c54 100644
--- a/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h
+++ b/flang/include/flang/Optimizer/Dialect/CUF/Attributes/CUFAttr.h
@@ -20,6 +20,10 @@ namespace llvm {
 class StringRef;
 }
 
+namespace mlir {
+class Operation;
+}
+
 #include "flang/Optimizer/Dialect/CUF/Attributes/CUFEnumAttr.h.inc"
 
 #define GET_ATTRDEF_CLASSES
@@ -28,6 +32,7 @@ class StringRef;
 namespace cuf {
 
 /// Attribute to mark Fortran entities with the CUDA attribute.
+static constexpr llvm::StringRef dataAttrName = "data_attr";
 static constexpr llvm::StringRef getDataAttrName() { return "cuf.data_attr"; }
 static constexpr llvm::StringRef getProcAttrName() { return "cuf.proc_attr"; }
 
@@ -101,6 +106,12 @@ getProcAttribute(mlir::MLIRContext *mlirContext,
   return {};
 }
 
+/// Returns the data attribute if the operation has one.
+cuf::DataAttributeAttr getDataAttr(mlir::Operation *op);
+
+/// Returns true if the operation has a data attribute with the given value.
+bool hasDataAttr(mlir::Operation *op, cuf::DataAttribute value);
+
 } // namespace cuf
 
 #endif // FORTRAN_OPTIMIZER_DIALECT_CUF_CUFATTR_H
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
index ee0b5aa9760b1..0bddfd85d436b 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROpBase.td
@@ -95,9 +95,9 @@ def IsFortranValuePred : CPred<"::hlfir::isFortranValueType($_self)">;
 def AnyFortranValue
         : TypeConstraint<IsFortranValuePred, "any Fortran value type">;
 
-
-def AnyFortranEntity : TypeConstraint<Or<[AnyFortranVariable.predicate,
-    AnyFortranValue.predicate]>, "any Fortran value or variable type">;
+def AnyFortranEntity
+    : Type<Or<[AnyFortranVariable.predicate, AnyFortranValue.predicate]>,
+           "any Fortran value or variable type">;
 
 def IsFortranScalarCharacterPred
         : CPred<"::hlfir::isFortranScalarCharacterType($_self)">;
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index 2f5da720fbe1d..db3fb0b90464d 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -721,6 +721,28 @@ def hlfir_CShiftOp
   let hasVerifier = 1;
 }
 
+def hlfir_EOShiftOp
+    : hlfir_Op<
+          "eoshift", [AttrSizedOperandSegments,
+                      DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
+  let summary = "EOSHIFT transformational intrinsic";
+  let description = [{
+    End-off shift of an array
+  }];
+
+  let arguments = (ins AnyFortranArrayObject:$array,
+      AnyFortranIntegerScalarOrArrayObject:$shift,
+      Optional<AnyFortranEntity>:$boundary, Optional<AnyIntegerType>:$dim);
+
+  let results = (outs hlfir_ExprType);
+
+  let assemblyFormat = [{
+    $array $shift (`boundary` $boundary^)? (`dim` $dim^)? attr-dict `:` functional-type(operands, results)
+  }];
+
+  let hasVerifier = 1;
+}
+
 def hlfir_ReshapeOp
     : hlfir_Op<
           "reshape", [AttrSizedOperandSegments,
diff --git a/flang/include/flang/Optimizer/OpenMP/Passes.td b/flang/include/flang/Optimizer/OpenMP/Passes.td
index 704faf0ccd856..99202f6ee81e7 100644
--- a/flang/include/flang/Optimizer/OpenMP/Passes.td
+++ b/flang/include/flang/Optimizer/OpenMP/Passes.td
@@ -112,4 +112,20 @@ def GenericLoopConversionPass
   ];
 }
 
+def SimdOnlyPass : Pass<"omp-simd-only", "mlir::ModuleOp"> {
+  let summary = "Filters out non-simd OpenMP constructs";
+  let dependentDialects = ["mlir::omp::OpenMPDialect"];
+}
+
+def AutomapToTargetDataPass
+    : Pass<"omp-automap-to-target-data", "::mlir::ModuleOp"> {
+  let summary = "Insert OpenMP target data operations for AUTOMAP variables";
+  let description = [{
+    Inserts `omp.target_enter_data` and `omp.target_exit_data` operations to
+    map variables marked with the `AUTOMAP` modifier when their allocation
+    or deallocation is detected in the FIR.
+  }];
+  let dependentDialects = ["mlir::omp::OpenMPDialect"];
+}
+
 #endif //FORTRAN_OPTIMIZER_OPENMP_PASSES
diff --git a/flang/include/flang/Optimizer/Passes/Pipelines.h b/flang/include/flang/Optimizer/Passes/Pipelines.h
index a3f59ee8dd013..fd8c43cc88a19 100644
--- a/flang/include/flang/Optimizer/Passes/Pipelines.h
+++ b/flang/include/flang/Optimizer/Passes/Pipelines.h
@@ -119,13 +119,16 @@ void registerDefaultInlinerPass(MLIRToLLVMPassPipelineConfig &config);
 void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
                                            MLIRToLLVMPassPipelineConfig &pc);
 
+/// Select which mode to enable OpenMP support in.
+enum class EnableOpenMP { None, Simd, Full };
+
 /// Create a pass pipeline for lowering from HLFIR to FIR
 ///
 /// \param pm - MLIR pass manager that will hold the pipeline definition
 /// \param optLevel - optimization level used for creating FIR optimization
 ///   passes pipeline
 void createHLFIRToFIRPassPipeline(
-    mlir::PassManager &pm, bool enableOpenMP,
+    mlir::PassManager &pm, EnableOpenMP enableOpenMP,
     llvm::OptimizationLevel optLevel = defaultOptLevel);
 
 struct OpenMPFIRPassPipelineOpts {
diff --git a/flang/include/flang/Optimizer/Support/Utils.h b/flang/include/flang/Optimizer/Support/Utils.h
index 83c936b7dcada..0b31cfea0430a 100644
--- a/flang/include/flang/Optimizer/Support/Utils.h
+++ b/flang/include/flang/Optimizer/Support/Utils.h
@@ -27,6 +27,8 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringRef.h"
 
+#include "flang/Optimizer/CodeGen/TypeConverter.h"
+
 namespace fir {
 /// Return the integer value of a arith::ConstantOp.
 inline std::int64_t toInt(mlir::arith::ConstantOp cop) {
@@ -198,6 +200,37 @@ std::optional<llvm::ArrayRef<int64_t>> getComponentLowerBoundsIfNonDefault(
     fir::RecordType recordType, llvm::StringRef component,
     mlir::ModuleOp module, const mlir::SymbolTable *symbolTable = nullptr);
 
+/// Generate a LLVM constant value of type `ity`, using the provided offset.
+mlir::LLVM::ConstantOp
+genConstantIndex(mlir::Location loc, mlir::Type ity,
+                 mlir::ConversionPatternRewriter &rewriter,
+                 std::int64_t offset);
+
+/// Helper function for generating the LLVM IR that computes the distance
+/// in bytes between adjacent elements pointed to by a pointer
+/// of type \p ptrTy. The result is returned as a value of \p idxTy integer
+/// type.
+mlir::Value computeElementDistance(mlir::Location loc,
+                                   mlir::Type llvmObjectType, mlir::Type idxTy,
+                                   mlir::ConversionPatternRewriter &rewriter,
+                                   const mlir::DataLayout &dataLayout);
+
+// Compute the alloc scale size (constant factors encoded in the array type).
+// We do this for arrays without a constant interior or arrays of character with
+// dynamic length arrays, since those are the only ones that get decayed to a
+// pointer to the element type.
+mlir::Value genAllocationScaleSize(mlir::Location loc, mlir::Type dataTy,
+                                   mlir::Type ity,
+                                   mlir::ConversionPatternRewriter &rewriter);
+
+/// Perform an extension or truncation as needed on an integer value. Lowering
+/// to the specific target may involve some sign-extending or truncation of
+/// values, particularly to fit them from abstract box types to the
+/// appropriate reified structures.
+mlir::Value integerCast(const fir::LLVMTypeConverter &converter,
+                        mlir::Location loc,
+                        mlir::ConversionPatternRewriter &rewriter,
+                        mlir::Type ty, mlir::Value val, bool fold = false);
 } // namespace fir
 
 #endif // FORTRAN_OPTIMIZER_SUPPORT_UTILS_H
diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 2c666a6d09a7b..7170dfb591fae 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -49,6 +49,7 @@ class ParseTreeDumper {
   NODE(std, uint64_t)
   NODE_ENUM(common, CUDADataAttr)
   NODE_ENUM(common, CUDASubprogramAttrs)
+  NODE_ENUM(common, OmpMemoryOrderType)
   NODE_ENUM(common, OpenACCDeviceType)
   NODE(format, ControlEditDesc)
   NODE(format::ControlEditDesc, Kind)
@@ -482,122 +483,129 @@ class ParseTreeDumper {
   NODE(parser, NullInit)
   NODE(parser, ObjectDecl)
   NODE(parser, OldParameterStmt)
-  NODE(parser, OmpTypeSpecifier)
-  NODE(parser, OmpTypeNameList)
+
+  static std::string GetNodeName(const llvm::omp::Directive &x) {
+    return llvm::Twine("llvm::omp::Directive = ",
+        llvm::omp::getOpenMPDirectiveName(x, llvm::omp::FallbackVersion))
+        .str();
+  }
+  static std::string GetNodeName(const llvm::omp::Clause &x) {
+    return llvm::Twine(
+        "llvm::omp::Clause = ", llvm::omp::getOpenMPClauseName(x))
+        .str();
+  }
+  NODE(parser, OmpAbsentClause)
+  NODE(parser, OmpAccessGroup)
+  NODE_ENUM(OmpAccessGroup, Value)
   NODE(parser, OmpAdjustArgsClause)
   NODE(OmpAdjustArgsClause, OmpAdjustOp)
   NODE_ENUM(OmpAdjustArgsClause::OmpAdjustOp, Value)
-  NODE(parser, OmpAppendArgsClause)
-  NODE(OmpAppendArgsClause, OmpAppendOp)
-  NODE(parser, OmpLocator)
-  NODE(parser, OmpLocatorList)
-  NODE(parser, OmpReductionSpecifier)
-  NODE(parser, OmpArgument)
-  NODE(parser, OmpArgumentList)
-  NODE(parser, OmpMetadirectiveDirective)
-  NODE(parser, OmpMatchClause)
-  NODE(parser, OmpOtherwiseClause)
-  NODE(parser, OmpWhenClause)
-  NODE(OmpWhenClause, Modifier)
-  NODE(parser, OmpDirectiveName)
-  NODE(parser, OmpDirectiveSpecification)
-  NODE_ENUM(OmpDirectiveSpecification, Flags)
-  NODE(parser, OmpTraitPropertyName)
-  NODE(parser, OmpTraitScore)
-  NODE(parser, OmpTraitPropertyExtension)
-  NODE(OmpTraitPropertyExtension, Complex)
-  NODE(parser, OmpTraitProperty)
-  NODE(parser, OmpTraitSelectorName)
-  NODE_ENUM(OmpTraitSelectorName, Value)
-  NODE(parser, OmpTraitSelector)
-  NODE(OmpTraitSelector, Properties)
-  NODE(parser, OmpTraitSetSelectorName)
-  NODE_ENUM(OmpTraitSetSelectorName, Value)
-  NODE(parser, OmpTraitSetSelector)
-  NODE(parser, OmpContextSelectorSpecification)
-  NODE(parser, OmpMapper)
-  NODE(parser, OmpMapType)
-  NODE_ENUM(OmpMapType, Value)
-  NODE(parser, OmpMapTypeModifier)
-  NODE_ENUM(OmpMapTypeModifier, Value)
-  NODE(parser, OmpIteratorSpecifier)
-  NODE(parser, OmpIterator)
-  NODE(parser, OmpAbsentClause)
   NODE(parser, OmpAffinityClause)
   NODE(OmpAffinityClause, Modifier)
-  NODE(parser, OmpAlignment)
   NODE(parser, OmpAlignClause)
   NODE(parser, OmpAlignedClause)
   NODE(OmpAlignedClause, Modifier)
+  NODE(parser, OmpAlignment)
+  NODE(parser, OmpAlignModifier)
+  NODE(parser, OmpAllocateClause)
+  NODE(OmpAllocateClause, Modifier)
+  NODE(parser, OmpAllocatorComplexModifier)
+  NODE(parser, OmpAllocatorSimpleModifier)
   NODE(parser, OmpAlwaysModifier)
   NODE_ENUM(OmpAlwaysModifier, Value)
+  NODE(parser, OmpAppendArgsClause)
+  NODE(OmpAppendArgsClause, OmpAppendOp)
+  NODE(parser, OmpArgument)
+  NODE(parser, OmpArgumentList)
+  NODE(parser, OmpAssumeDirective)
   NODE(parser, OmpAtClause)
   NODE_ENUM(OmpAtClause, ActionTime)
-  NODE_ENUM(OmpSeverityClause, Severity)
+  NODE(parser, OmpAtomicDefaultMemOrderClause)
+  NODE(parser, OmpAutomapModifier)
+  NODE_ENUM(OmpAutomapModifier, Value)
+  NODE(parser, OmpBeginDirective)
   NODE(parser, OmpBeginLoopDirective)
   NODE(parser, OmpBeginSectionsDirective)
-  static std::string GetNodeName(const llvm::omp::Directive &x) {
-    return llvm::Twine("llvm::omp::Directive = ",
-        llvm::omp::getOpenMPDirectiveName(x, llvm::omp::FallbackVersion))
-        .str();
-  }
+  NODE(parser, OmpBindClause)
+  NODE_ENUM(OmpBindClause, Binding)
+  NODE(parser, OmpCancellationConstructTypeClause)
+  NODE(parser, OmpChunkModifier)
+  NODE_ENUM(OmpChunkModifier, Value)
   NODE(parser, OmpClause)
-#define GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
-#include "llvm/Frontend/OpenMP/OMP.inc"
   NODE(parser, OmpClauseList)
-  NODE(parser, OmpCancellationConstructTypeClause)
   NODE(parser, OmpCloseModifier)
   NODE_ENUM(OmpCloseModifier, Value)
   NODE(parser, OmpContainsClause)
+  NODE(parser, OmpContextSelectorSpecification)
   NODE(parser, OmpCriticalDirective)
-  NODE(parser, OmpErrorDirective)
-  NODE(parser, OmpNothingDirective)
   NODE(parser, OmpDeclareTargetSpecifier)
   NODE(parser, OmpDeclareTargetWithClause)
   NODE(parser, OmpDeclareTargetWithList)
-  NODE(parser, OmpMapperSpecifier)
+  NODE(parser, OmpDeclareVariantDirective)
   NODE(parser, OmpDefaultClause)
   NODE_ENUM(OmpDefaultClause, DataSharingAttribute)
-  NODE(parser, OmpVariableCategory)
-  NODE_ENUM(OmpVariableCategory, Value)
   NODE(parser, OmpDefaultmapClause)
-  NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(OmpDefaultmapClause, Modifier)
+  NODE_ENUM(OmpDefaultmapClause, ImplicitBehavior)
   NODE(parser, OmpDeleteModifier)
   NODE_ENUM(OmpDeleteModifier, Value)
+  NODE(parser, OmpDependClause)
+  NODE(OmpDependClause, TaskDep)
+  NODE(OmpDependClause::TaskDep, Modifier)
   NODE(parser, OmpDependenceType)
   NODE_ENUM(OmpDependenceType, Value)
-  NODE(parser, OmpTaskDependenceType)
-  NODE_ENUM(OmpTaskDependenceType, Value)
-  NODE(parser, OmpIndirectClause)
-  NODE(parser, OmpIterationOffset)
-  NODE(parser, OmpIteration)
-  NODE(parser, OmpIterationVector)
+  NODE(parser, OmpDestroyClause)
+  NODE(parser, OmpDetachClause)
+  NODE(parser, OmpDeviceClause)
+  NODE(OmpDeviceClause, Modifier)
+  NODE(parser, OmpDeviceModifier)
+  NODE_ENUM(OmpDeviceModifier, Value)
+  NODE(parser, OmpDeviceTypeClause)
+  NODE_ENUM(OmpDeviceTypeClause, DeviceTypeDescription)
+  NODE(parser, OmpDirectiveName)
+  NODE(parser, OmpDirectiveSpecification)
+  NODE_ENUM(OmpDirectiveSpecification, Flags)
   NODE(parser, OmpDoacross)
   NODE(OmpDoacross, Sink)
   NODE(OmpDoacross, Source)
-  NODE(parser, OmpDependClause)
-  NODE(OmpDependClause, TaskDep)
-  NODE(OmpDependClause::TaskDep, Modifier)
-  NODE(parser, OmpAutomapModifier)
-  NODE_ENUM(OmpAutomapModifier, Value)
-  NODE(parser, OmpDetachClause)
   NODE(parser, OmpDoacrossClause)
-  NODE(parser, OmpDestroyClause)
+  NODE(parser, OmpDynGroupprivateClause)
+  NODE(OmpDynGroupprivateClause, Modifier)
+  NODE(parser, OmpEndAssumeDirective)
   NODE(parser, OmpEndCriticalDirective)
+  NODE(parser, OmpEndDirective)
   NODE(parser, OmpEndLoopDirective)
   NODE(parser, OmpEndSectionsDirective)
   NODE(parser, OmpEnterClause)
   NODE(OmpEnterClause, Modifier)
+  NODE(parser, OmpErrorDirective)
+  NODE(parser, OmpExpectation)
+  NODE_ENUM(OmpExpectation, Value)
   NODE(parser, OmpFailClause)
   NODE(parser, OmpFromClause)
   NODE(OmpFromClause, Modifier)
-  NODE(parser, OmpExpectation)
-  NODE_ENUM(OmpExpectation, Value)
+  NODE(parser, OmpGrainsizeClause)
+  NODE(OmpGrainsizeClause, Modifier)
   NODE(parser, OmpHintClause)
   NODE(parser, OmpHoldsClause)
   NODE(parser, OmpIfClause)
   NODE(OmpIfClause, Modifier)
+  NODE(parser, OmpIndirectClause)
+  NODE(parser, OmpInitClause)
+  NODE(OmpInitClause, Modifier)
+  NODE(parser, OmpInitializerClause)
+  NODE(parser, OmpInitializerProc)
+  NODE(parser, OmpInReductionClause)
+  NODE(OmpInReductionClause, Modifier)
+  NODE(parser, OmpInteropPreference)
+  NODE(parser, OmpInteropRuntimeIdentifier)
+  NODE(parser, OmpInteropType)
+  NODE_ENUM(OmpInteropType, Value)
+  NODE(parser, OmpIteration)
+  NODE(parser, OmpIterationOffset)
+  NODE(parser, OmpIterationVector)
+  NODE(parser, OmpIterator)
+  NODE(parser, OmpIteratorSpecifier)
   NODE(parser, OmpLastprivateClause)
   NODE(OmpLastprivateClause, Modifier)
   NODE(parser, OmpLastprivateModifier)
@@ -606,87 +614,92 @@ class ParseTreeDumper {
   NODE(OmpLinearClause, Modifier)
   NODE(parser, OmpLinearModifier)
   NODE_ENUM(OmpLinearModifier, Value)
-  NODE(parser, OmpStepComplexModifier)
-  NODE(parser, OmpStepSimpleModifier)
+  NODE(parser, OmpLocator)
+  NODE(parser, OmpLocatorList)
   NODE(parser, OmpLoopDirective)
   NODE(parser, OmpMapClause)
-  NODE(parser, OmpMessageClause)
   NODE(OmpMapClause, Modifier)
-  static std::string GetNodeName(const llvm::omp::Clause &x) {
-    return llvm::Twine(
-        "llvm::omp::Clause = ", llvm::omp::getOpenMPClauseName(x))
-        .str();
-  }
-  NODE(parser, OmpObject)
-  NODE(parser, OmpObjectList)
+  NODE(parser, OmpMapper)
+  NODE(parser, OmpMapperSpecifier)
+  NODE(parser, OmpMapType)
+  NODE_ENUM(OmpMapType, Value)
+  NODE(parser, OmpMapTypeModifier)
+  NODE_ENUM(OmpMapTypeModifier, Value)
+  NODE(parser, OmpMatchClause)
+  NODE(parser, OmpMessageClause)
+  NODE(parser, OmpMetadirectiveDirective)
   NODE(parser, OmpNoOpenMPClause)
   NODE(parser, OmpNoOpenMPRoutinesClause)
   NODE(parser, OmpNoParallelismClause)
+  NODE(parser, OmpNothingDirective)
+  NODE(parser, OmpNumTasksClause)
+  NODE(OmpNumTasksClause, Modifier)
+  NODE(parser, OmpObject)
+  NODE(parser, OmpObjectList)
   NODE(parser, OmpOrderClause)
   NODE(OmpOrderClause, Modifier)
   NODE_ENUM(OmpOrderClause, Ordering)
+  NODE(parser, OmpOrderingModifier)
+  NODE_ENUM(OmpOrderingModifier, Value)
   NODE(parser, OmpOrderModifier)
   NODE_ENUM(OmpOrderModifier, Value)
-  NODE(parser, OmpGrainsizeClause)
-  NODE(OmpGrainsizeClause, Modifier)
+  NODE(parser, OmpOtherwiseClause)
   NODE(parser, OmpPrescriptiveness)
   NODE_ENUM(OmpPrescriptiveness, Value)
-  NODE(parser, OmpNumTasksClause)
-  NODE(OmpNumTasksClause, Modifier)
-  NODE(parser, OmpBindClause)
-  NODE_ENUM(OmpBindClause, Binding)
   NODE(parser, OmpPresentModifier)
   NODE_ENUM(OmpPresentModifier, Value)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, AffinityPolicy)
-  NODE(parser, OmpReductionModifier)
-  NODE_ENUM(OmpReductionModifier, Value)
   NODE(parser, OmpReductionClause)
   NODE(OmpReductionClause, Modifier)
-  NODE(parser, OmpInReductionClause)
-  NODE(OmpInReductionClause, Modifier)
   NODE(parser, OmpReductionCombiner)
+  NODE(parser, OmpReductionIdentifier)
+  NODE(parser, OmpReductionModifier)
+  NODE_ENUM(OmpReductionModifier, Value)
+  NODE(parser, OmpReductionSpecifier)
   NODE(parser, OmpRefModifier)
   NODE_ENUM(OmpRefModifier, Value)
-  NODE(parser, OmpSelfModifier)
-  NODE_ENUM(OmpSelfModifier, Value)
-  NODE(parser, OmpTaskReductionClause)
-  NODE(OmpTaskReductionClause, Modifier)
-  NODE(parser, OmpInitializerProc)
-  NODE(parser, OmpInitializerClause)
-  NODE(parser, OmpReductionIdentifier)
-  NODE(parser, OmpAllocateClause)
-  NODE(OmpAllocateClause, Modifier)
-  NODE(parser, OmpAlignModifier)
-  NODE(parser, OmpAllocatorComplexModifier)
-  NODE(parser, OmpAllocatorSimpleModifier)
   NODE(parser, OmpScheduleClause)
   NODE(OmpScheduleClause, Modifier)
   NODE_ENUM(OmpScheduleClause, Kind)
-  NODE(parser, OmpSeverityClause)
-  NODE(parser, OmpDeviceClause)
-  NODE(OmpDeviceClause, Modifier)
-  NODE(parser, OmpDeviceModifier)
-  NODE_ENUM(OmpDeviceModifier, Value)
-  NODE(parser, OmpDeviceTypeClause)
-  NODE_ENUM(OmpDeviceTypeClause, DeviceTypeDescription)
-  NODE(parser, OmpInteropRuntimeIdentifier)
-  NODE(parser, OmpInteropPreference)
-  NODE(parser, OmpInteropType)
-  NODE_ENUM(OmpInteropType, Value)
-  NODE(parser, OmpInitClause)
-  NODE(OmpInitClause, Modifier)
-  NODE(parser, OmpUseClause)
-  NODE(parser, OmpUpdateClause)
-  NODE(parser, OmpChunkModifier)
-  NODE_ENUM(OmpChunkModifier, Value)
-  NODE(parser, OmpOrderingModifier)
-  NODE_ENUM(OmpOrderingModifier, Value)
   NODE(parser, OmpSectionsDirective)
+  NODE(parser, OmpSelfModifier)
+  NODE_ENUM(OmpSelfModifier, Value)
+  NODE(parser, OmpSeverityClause)
+  NODE_ENUM(OmpSeverityClause, Severity)
+  NODE(parser, OmpStepComplexModifier)
+  NODE(parser, OmpStepSimpleModifier)
+  NODE(parser, OmpTaskDependenceType)
+  NODE_ENUM(OmpTaskDependenceType, Value)
+  NODE(parser, OmpTaskReductionClause)
+  NODE(OmpTaskReductionClause, Modifier)
   NODE(parser, OmpToClause)
   NODE(OmpToClause, Modifier)
+  NODE(parser, OmpTraitProperty)
+  NODE(parser, OmpTraitPropertyExtension)
+  NODE(OmpTraitPropertyExtension, Complex)
+  NODE(parser, OmpTraitPropertyName)
+  NODE(parser, OmpTraitScore)
+  NODE(parser, OmpTraitSelector)
+  NODE(OmpTraitSelector, Properties)
+  NODE(parser, OmpTraitSelectorName)
+  NODE_ENUM(OmpTraitSelectorName, Value)
+  NODE(parser, OmpTraitSetSelector)
+  NODE(parser, OmpTraitSetSelectorName)
+  NODE_ENUM(OmpTraitSetSelectorName, Value)
+  NODE(parser, OmpTypeNameList)
+  NODE(parser, OmpTypeSpecifier)
+  NODE(parser, OmpUpdateClause)
+  NODE(parser, OmpUseClause)
+  NODE(parser, OmpVariableCategory)
+  NODE_ENUM(OmpVariableCategory, Value)
+  NODE(parser, OmpWhenClause)
+  NODE(OmpWhenClause, Modifier)
   NODE(parser, OmpxHoldModifier)
   NODE_ENUM(OmpxHoldModifier, Value)
+#define GEN_FLANG_DUMP_PARSE_TREE_CLAUSES
+#include "llvm/Frontend/OpenMP/OMP.inc"
+
   NODE(parser, Only)
   NODE(parser, OpenACCAtomicConstruct)
   NODE(parser, OpenACCBlockConstruct)
@@ -701,12 +714,9 @@ class ParseTreeDumper {
   NODE(parser, OpenACCStandaloneDeclarativeConstruct)
   NODE(parser, OpenACCStandaloneConstruct)
   NODE(parser, OpenACCWaitConstruct)
+
+  NODE(parser, OpenMPAllocatorsConstruct)
   NODE(parser, OpenMPAssumeConstruct)
-  NODE(parser, OpenMPDeclarativeAssumes)
-  NODE(parser, OmpAssumeDirective)
-  NODE(parser, OmpEndAssumeDirective)
-  NODE(parser, OmpBeginDirective)
-  NODE(parser, OmpEndDirective)
   NODE(parser, OpenMPAtomicConstruct)
   NODE(parser, OpenMPBlockConstruct)
   NODE(parser, OpenMPCancelConstruct)
@@ -714,27 +724,26 @@ class ParseTreeDumper {
   NODE(parser, OpenMPConstruct)
   NODE(parser, OpenMPCriticalConstruct)
   NODE(parser, OpenMPDeclarativeAllocate)
+  NODE(parser, OpenMPDeclarativeAssumes)
   NODE(parser, OpenMPDeclarativeConstruct)
-  NODE(parser, OmpDeclareVariantDirective)
+  NODE(parser, OpenMPDeclareMapperConstruct)
   NODE(parser, OpenMPDeclareReductionConstruct)
   NODE(parser, OpenMPDeclareSimdConstruct)
   NODE(parser, OpenMPDeclareTargetConstruct)
-  NODE(parser, OpenMPDeclareMapperConstruct)
-  NODE_ENUM(common, OmpMemoryOrderType)
-  NODE(parser, OmpAtomicDefaultMemOrderClause)
   NODE(parser, OpenMPDepobjConstruct)
-  NODE(parser, OpenMPUtilityConstruct)
   NODE(parser, OpenMPDispatchConstruct)
+  NODE(parser, OpenMPExecutableAllocate)
   NODE(parser, OpenMPFlushConstruct)
+  NODE(parser, OpenMPGroupprivate)
   NODE(parser, OpenMPLoopConstruct)
-  NODE(parser, OpenMPExecutableAllocate)
-  NODE(parser, OpenMPAllocatorsConstruct)
   NODE(parser, OpenMPRequiresConstruct)
-  NODE(parser, OpenMPSimpleStandaloneConstruct)
-  NODE(parser, OpenMPStandaloneConstruct)
   NODE(parser, OpenMPSectionConstruct)
   NODE(parser, OpenMPSectionsConstruct)
+  NODE(parser, OpenMPSimpleStandaloneConstruct)
+  NODE(parser, OpenMPStandaloneConstruct)
   NODE(parser, OpenMPThreadprivate)
+  NODE(parser, OpenMPUtilityConstruct)
+
   NODE(parser, OpenStmt)
   NODE(parser, Optional)
   NODE(parser, OptionalStmt)
diff --git a/flang/include/flang/Parser/openmp-utils.h b/flang/include/flang/Parser/openmp-utils.h
index fa0f7656cd5d8..6bd578b50dc3e 100644
--- a/flang/include/flang/Parser/openmp-utils.h
+++ b/flang/include/flang/Parser/openmp-utils.h
@@ -95,7 +95,8 @@ struct DirectiveNameScope {
           std::is_same_v<T, OpenMPDepobjConstruct> ||
           std::is_same_v<T, OpenMPFlushConstruct> ||
           std::is_same_v<T, OpenMPInteropConstruct> ||
-          std::is_same_v<T, OpenMPSimpleStandaloneConstruct>) {
+          std::is_same_v<T, OpenMPSimpleStandaloneConstruct> ||
+          std::is_same_v<T, OpenMPGroupprivate>) {
         return x.v.DirName();
       } else {
         return GetOmpDirectiveName(x.v);
@@ -157,6 +158,8 @@ template <typename T> OmpDirectiveName GetOmpDirectiveName(const T &x) {
   return detail::DirectiveNameScope::GetOmpDirectiveName(x);
 }
 
+const OmpObjectList *GetOmpObjectList(const OmpClause &clause);
+
 } // namespace Fortran::parser::omp
 
 #endif // FORTRAN_PARSER_OPENMP_UTILS_H
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 8302e40984af0..1d1a4a163084b 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3736,6 +3736,11 @@ inline namespace modifier {
 //   ENUM_CLASS(Value, Keyword1, Keyword2);
 // };
 
+struct OmpAccessGroup {
+  ENUM_CLASS(Value, Cgroup);
+  WRAPPER_CLASS_BOILERPLATE(OmpAccessGroup, Value);
+};
+
 // Ref: [4.5:72-81], [5.0:110-119], [5.1:134-143], [5.2:169-170]
 //
 // alignment ->
@@ -4019,8 +4024,9 @@ struct OmpOrderModifier {
 //
 // prescriptiveness ->
 //    STRICT                                        // since 5.1
+//    FALLBACK                                      // since 6.1
 struct OmpPrescriptiveness {
-  ENUM_CLASS(Value, Strict)
+  ENUM_CLASS(Value, Strict, Fallback)
   WRAPPER_CLASS_BOILERPLATE(OmpPrescriptiveness, Value);
 };
 
@@ -4375,6 +4381,12 @@ struct OmpDeviceTypeClause {
   WRAPPER_CLASS_BOILERPLATE(OmpDeviceTypeClause, DeviceTypeDescription);
 };
 
+struct OmpDynGroupprivateClause {
+  TUPLE_CLASS_BOILERPLATE(OmpDynGroupprivateClause);
+  MODIFIER_BOILERPLATE(OmpAccessGroup, OmpPrescriptiveness);
+  std::tuple<MODIFIERS(), ScalarIntExpr> t;
+};
+
 // Ref: [5.2:158-159], [6.0:289-290]
 //
 // enter-clause ->
@@ -4943,6 +4955,15 @@ struct OpenMPDeclareSimdConstruct {
   std::tuple<Verbatim, std::optional<Name>, OmpClauseList> t;
 };
 
+// ref: [6.0:301-303]
+//
+// groupprivate-directive ->
+//    GROUPPRIVATE (variable-list-item...)          // since 6.0
+struct OpenMPGroupprivate {
+  WRAPPER_CLASS_BOILERPLATE(OpenMPGroupprivate, OmpDirectiveSpecification);
+  CharBlock source;
+};
+
 // 2.4 requires -> REQUIRES requires-clause[ [ [,] requires-clause]...]
 struct OpenMPRequiresConstruct {
   TUPLE_CLASS_BOILERPLATE(OpenMPRequiresConstruct);
@@ -4970,8 +4991,9 @@ struct OpenMPDeclarativeConstruct {
   std::variant<OpenMPDeclarativeAllocate, OpenMPDeclarativeAssumes,
       OpenMPDeclareMapperConstruct, OpenMPDeclareReductionConstruct,
       OpenMPDeclareSimdConstruct, OpenMPDeclareTargetConstruct,
-      OmpDeclareVariantDirective, OpenMPThreadprivate, OpenMPRequiresConstruct,
-      OpenMPUtilityConstruct, OmpMetadirectiveDirective>
+      OmpDeclareVariantDirective, OpenMPGroupprivate, OpenMPThreadprivate,
+      OpenMPRequiresConstruct, OpenMPUtilityConstruct,
+      OmpMetadirectiveDirective>
       u;
 };
 
@@ -4986,9 +5008,9 @@ struct OmpEndCriticalDirective {
   CharBlock source;
   std::tuple<Verbatim, std::optional<Name>> t;
 };
-struct OpenMPCriticalConstruct {
-  TUPLE_CLASS_BOILERPLATE(OpenMPCriticalConstruct);
-  std::tuple<OmpCriticalDirective, Block, OmpEndCriticalDirective> t;
+
+struct OpenMPCriticalConstruct : public OmpBlockConstruct {
+  INHERITED_TUPLE_CLASS_BOILERPLATE(OpenMPCriticalConstruct, OmpBlockConstruct);
 };
 
 // 2.11.3 allocate -> ALLOCATE [(variable-name-list)] [clause]
diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index b350204714431..9a100cec9e6b9 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -90,5 +90,9 @@ void RTNAME(Perror)(const char *str);
 // MCLOCK -- returns accumulated time in ticks
 int FORTRAN_PROCEDURE_NAME(mclock)();
 
+// GNU extension subroutine SECNDS(refTime)
+float FORTRAN_PROCEDURE_NAME(secnds)(float *refTime);
+float RTNAME(Secnds)(float *refTime, const char *sourceFile, int line);
+
 } // extern "C"
 #endif // FORTRAN_RUNTIME_EXTENSIONS_H_
diff --git a/flang/include/flang/Semantics/openmp-directive-sets.h b/flang/include/flang/Semantics/openmp-directive-sets.h
index dd610c9702c28..01e8481e05721 100644
--- a/flang/include/flang/Semantics/openmp-directive-sets.h
+++ b/flang/include/flang/Semantics/openmp-directive-sets.h
@@ -143,6 +143,7 @@ static const OmpDirectiveSet topTargetSet{
     Directive::OMPD_target_teams_distribute_parallel_do_simd,
     Directive::OMPD_target_teams_distribute_simd,
     Directive::OMPD_target_teams_loop,
+    Directive::OMPD_target_teams_workdistribute,
 };
 
 static const OmpDirectiveSet allTargetSet{topTargetSet};
@@ -172,6 +173,7 @@ static const OmpDirectiveSet topTeamsSet{
     Directive::OMPD_teams_distribute_parallel_do_simd,
     Directive::OMPD_teams_distribute_simd,
     Directive::OMPD_teams_loop,
+    Directive::OMPD_teams_workdistribute,
 };
 
 static const OmpDirectiveSet bottomTeamsSet{
@@ -187,6 +189,7 @@ static const OmpDirectiveSet allTeamsSet{
         Directive::OMPD_target_teams_distribute_parallel_do_simd,
         Directive::OMPD_target_teams_distribute_simd,
         Directive::OMPD_target_teams_loop,
+        Directive::OMPD_target_teams_workdistribute,
     } | topTeamsSet,
 };
 
@@ -230,6 +233,9 @@ static const OmpDirectiveSet blockConstructSet{
     Directive::OMPD_taskgroup,
     Directive::OMPD_teams,
     Directive::OMPD_workshare,
+    Directive::OMPD_target_teams_workdistribute,
+    Directive::OMPD_teams_workdistribute,
+    Directive::OMPD_workdistribute,
 };
 
 static const OmpDirectiveSet loopConstructSet{
@@ -376,6 +382,7 @@ static const OmpDirectiveSet nestedReduceWorkshareAllowedSet{
 };
 
 static const OmpDirectiveSet nestedTeamsAllowedSet{
+    Directive::OMPD_workdistribute,
     Directive::OMPD_distribute,
     Directive::OMPD_distribute_parallel_do,
     Directive::OMPD_distribute_parallel_do_simd,
@@ -401,6 +408,22 @@ static const OmpDirectiveSet nestedWorkshareErrSet{
         Directive::OMPD_taskloop,
     } | workShareSet,
 };
+
+//===----------------------------------------------------------------------===//
+// Misc directive sets
+//===----------------------------------------------------------------------===//
+
+// Simple standalone directives than can be erased by -fopenmp-simd.
+static const OmpDirectiveSet simpleStandaloneNonSimdOnlySet{
+    Directive::OMPD_taskyield,
+    Directive::OMPD_barrier,
+    Directive::OMPD_ordered,
+    Directive::OMPD_target_enter_data,
+    Directive::OMPD_target_exit_data,
+    Directive::OMPD_target_update,
+    Directive::OMPD_taskwait,
+};
+
 } // namespace llvm::omp
 
 #endif // FORTRAN_SEMANTICS_OPENMP_DIRECTIVE_SETS_H_
diff --git a/flang/lib/Semantics/openmp-utils.h b/flang/include/flang/Semantics/openmp-utils.h
similarity index 93%
rename from flang/lib/Semantics/openmp-utils.h
rename to flang/include/flang/Semantics/openmp-utils.h
index b8ad9ed17c720..1c54124a5738a 100644
--- a/flang/lib/Semantics/openmp-utils.h
+++ b/flang/include/flang/Semantics/openmp-utils.h
@@ -22,6 +22,8 @@
 
 #include <optional>
 #include <string>
+#include <type_traits>
+#include <utility>
 
 namespace Fortran::semantics {
 class SemanticsContext;
@@ -29,6 +31,12 @@ class Symbol;
 
 // Add this namespace to avoid potential conflicts
 namespace omp {
+template <typename T, typename U = std::remove_const_t<T>> U AsRvalue(T &t) {
+  return U(t);
+}
+
+template <typename T> T &&AsRvalue(T &&t) { return std::move(t); }
+
 // There is no consistent way to get the source of an ActionStmt, but there
 // is "source" in Statement<T>. This structure keeps the ActionStmt with the
 // extracted source for further use.
diff --git a/flang/include/flang/Semantics/symbol.h b/flang/include/flang/Semantics/symbol.h
index 5bde9f39ca0b0..774fc9873f7bc 100644
--- a/flang/include/flang/Semantics/symbol.h
+++ b/flang/include/flang/Semantics/symbol.h
@@ -811,6 +811,7 @@ class Symbol {
       AccCommonBlock, AccThreadPrivate, AccReduction, AccNone, AccPreDetermined,
       // OpenMP data-sharing attribute
       OmpShared, OmpPrivate, OmpLinear, OmpFirstPrivate, OmpLastPrivate,
+      OmpGroupPrivate,
       // OpenMP data-mapping attribute
       OmpMapTo, OmpMapFrom, OmpMapToFrom, OmpMapStorage, OmpMapDelete,
       OmpUseDevicePtr, OmpUseDeviceAddr, OmpIsDevicePtr, OmpHasDeviceAddr,
diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index 743abf606ab5f..83a75b0efcb55 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -56,7 +56,7 @@ ENUM_CLASS(LanguageFeature, BackslashEscapes, OldDebugLines,
     IgnoreIrrelevantAttributes, Unsigned, AmbiguousStructureConstructor,
     ContiguousOkForSeqAssociation, ForwardRefExplicitTypeDummy,
     InaccessibleDeferredOverride, CudaWarpMatchFunction, DoConcurrentOffload,
-    TransferBOZ)
+    TransferBOZ, Coarray)
 
 // Portability and suspicious usage warnings
 ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
@@ -78,7 +78,8 @@ ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
     MismatchingDummyProcedure, SubscriptedEmptyArray, UnsignedLiteralTruncation,
     CompatibleDeclarationsFromDistinctModules,
     NullActualForDefaultIntentAllocatable, UseAssociationIntoSameNameSubprogram,
-    HostAssociatedIntentOutInSpecExpr, NonVolatilePointerToVolatile)
+    HostAssociatedIntentOutInSpecExpr, NonVolatilePointerToVolatile,
+    RealConstantWidening)
 
 using LanguageFeatures = EnumSet<LanguageFeature, LanguageFeature_enumSize>;
 using UsageWarnings = EnumSet<UsageWarning, UsageWarning_enumSize>;
diff --git a/flang/include/flang/Support/Fortran.h b/flang/include/flang/Support/Fortran.h
index 0b4fc1a608e9e..ea0344ecb0830 100644
--- a/flang/include/flang/Support/Fortran.h
+++ b/flang/include/flang/Support/Fortran.h
@@ -95,8 +95,8 @@ static constexpr IgnoreTKRSet ignoreTKRAll{IgnoreTKR::Type, IgnoreTKR::Kind,
 std::string AsFortran(IgnoreTKRSet);
 
 bool AreCompatibleCUDADataAttrs(std::optional<CUDADataAttr>,
-    std::optional<CUDADataAttr>, IgnoreTKRSet, std::optional<std::string> *,
-    bool allowUnifiedMatchingRule, bool isHostDeviceProcedure,
+    std::optional<CUDADataAttr>, IgnoreTKRSet, bool allowUnifiedMatchingRule,
+    bool isHostDeviceProcedure,
     const LanguageFeatureControl *features = nullptr);
 
 static constexpr char blankCommonObjectName[] = "__BLNK__";
diff --git a/flang/include/flang/Support/LangOptions.def b/flang/include/flang/Support/LangOptions.def
index d5bf7a2ecc036..ba72d7b4b7212 100644
--- a/flang/include/flang/Support/LangOptions.def
+++ b/flang/include/flang/Support/LangOptions.def
@@ -58,6 +58,8 @@ LANGOPT(OpenMPTeamSubscription, 1, 0)
 LANGOPT(OpenMPNoThreadState, 1, 0)
 /// Assume that no thread in a parallel region will encounter a parallel region
 LANGOPT(OpenMPNoNestedParallelism, 1, 0)
+/// Use SIMD only OpenMP support.
+LANGOPT(OpenMPSimd, 1, false)
 
 LANGOPT(VScaleMin, 32, 0)  ///< Minimum vscale range value
 LANGOPT(VScaleMax, 32, 0)  ///< Maximum vscale range value
diff --git a/flang/include/flang/Tools/CrossToolHelpers.h b/flang/include/flang/Tools/CrossToolHelpers.h
index df1da27058552..51958fa36c3ad 100644
--- a/flang/include/flang/Tools/CrossToolHelpers.h
+++ b/flang/include/flang/Tools/CrossToolHelpers.h
@@ -134,6 +134,7 @@ struct MLIRToLLVMPassPipelineConfig : public FlangEPCallBacks {
                                       ///< functions.
   bool NSWOnLoopVarInc = true; ///< Add nsw flag to loop variable increments.
   bool EnableOpenMP = false; ///< Enable OpenMP lowering.
+  bool EnableOpenMPSimd = false; ///< Enable OpenMP simd-only mode.
   std::string InstrumentFunctionEntry =
       ""; ///< Name of the instrument-function that is called on each
           ///< function-entry
diff --git a/flang/lib/Evaluate/characteristics.cpp b/flang/lib/Evaluate/characteristics.cpp
index 89547733ea33c..37c62c93a87df 100644
--- a/flang/lib/Evaluate/characteristics.cpp
+++ b/flang/lib/Evaluate/characteristics.cpp
@@ -400,7 +400,7 @@ bool DummyDataObject::IsCompatibleWith(const DummyDataObject &actual,
   }
   if (!attrs.test(Attr::Value) &&
       !common::AreCompatibleCUDADataAttrs(cudaDataAttr, actual.cudaDataAttr,
-          ignoreTKR, warning,
+          ignoreTKR,
           /*allowUnifiedMatchingRule=*/false,
           /*=isHostDeviceProcedure*/ false)) {
     if (whyNot) {
@@ -1816,7 +1816,7 @@ bool DistinguishUtils::Distinguishable(
       x.intent != common::Intent::In) {
     return true;
   } else if (!common::AreCompatibleCUDADataAttrs(x.cudaDataAttr, y.cudaDataAttr,
-                 x.ignoreTKR | y.ignoreTKR, nullptr,
+                 x.ignoreTKR | y.ignoreTKR,
                  /*allowUnifiedMatchingRule=*/false,
                  /*=isHostDeviceProcedure*/ false)) {
     return true;
diff --git a/flang/lib/Evaluate/check-expression.cpp b/flang/lib/Evaluate/check-expression.cpp
index 3d7f01d56c465..522ab1980f4ee 100644
--- a/flang/lib/Evaluate/check-expression.cpp
+++ b/flang/lib/Evaluate/check-expression.cpp
@@ -405,6 +405,88 @@ bool IsInitialProcedureTarget(const Expr<SomeType> &expr) {
   }
 }
 
+class SuspiciousRealLiteralFinder
+    : public AnyTraverse<SuspiciousRealLiteralFinder> {
+public:
+  using Base = AnyTraverse<SuspiciousRealLiteralFinder>;
+  SuspiciousRealLiteralFinder(int kind, FoldingContext &c)
+      : Base{*this}, kind_{kind}, context_{c} {}
+  using Base::operator();
+  template <int KIND>
+  bool operator()(const Constant<Type<TypeCategory::Real, KIND>> &x) const {
+    if (kind_ > KIND && x.result().isFromInexactLiteralConversion()) {
+      context_.messages().Say(common::UsageWarning::RealConstantWidening,
+          "Default real literal in REAL(%d) context might need a kind suffix, as its rounded value %s is inexact"_warn_en_US,
+          kind_, x.AsFortran());
+      return true;
+    } else {
+      return false;
+    }
+  }
+  template <int KIND>
+  bool operator()(const Constant<Type<TypeCategory::Complex, KIND>> &x) const {
+    if (kind_ > KIND && x.result().isFromInexactLiteralConversion()) {
+      context_.messages().Say(common::UsageWarning::RealConstantWidening,
+          "Default real literal in COMPLEX(%d) context might need a kind suffix, as its rounded value %s is inexact"_warn_en_US,
+          kind_, x.AsFortran());
+      return true;
+    } else {
+      return false;
+    }
+  }
+  template <TypeCategory TOCAT, int TOKIND, TypeCategory FROMCAT>
+  bool operator()(const Convert<Type<TOCAT, TOKIND>, FROMCAT> &x) const {
+    if constexpr ((TOCAT == TypeCategory::Real ||
+                      TOCAT == TypeCategory::Complex) &&
+        (FROMCAT == TypeCategory::Real || FROMCAT == TypeCategory::Complex)) {
+      auto fromType{x.left().GetType()};
+      if (!fromType || fromType->kind() < TOKIND) {
+        return false;
+      }
+    }
+    return (*this)(x.left());
+  }
+
+private:
+  int kind_;
+  FoldingContext &context_;
+};
+
+void CheckRealWidening(const Expr<SomeType> &expr, const DynamicType &toType,
+    FoldingContext &context) {
+  if (toType.category() == TypeCategory::Real ||
+      toType.category() == TypeCategory::Complex) {
+    if (auto fromType{expr.GetType()}) {
+      if ((fromType->category() == TypeCategory::Real ||
+              fromType->category() == TypeCategory::Complex) &&
+          toType.kind() > fromType->kind()) {
+        SuspiciousRealLiteralFinder{toType.kind(), context}(expr);
+      }
+    }
+  }
+}
+
+void CheckRealWidening(const Expr<SomeType> &expr,
+    const std::optional<DynamicType> &toType, FoldingContext &context) {
+  if (toType) {
+    CheckRealWidening(expr, *toType, context);
+  }
+}
+
+class InexactLiteralConversionFlagClearer
+    : public AnyTraverse<InexactLiteralConversionFlagClearer> {
+public:
+  using Base = AnyTraverse<InexactLiteralConversionFlagClearer>;
+  InexactLiteralConversionFlagClearer() : Base(*this) {}
+  using Base::operator();
+  template <int KIND>
+  bool operator()(const Constant<Type<TypeCategory::Real, KIND>> &x) const {
+    auto &mut{const_cast<Type<TypeCategory::Real, KIND> &>(x.result())};
+    mut.set_isFromInexactLiteralConversion(false);
+    return false;
+  }
+};
+
 // Converts, folds, and then checks type, rank, and shape of an
 // initialization expression for a named constant, a non-pointer
 // variable static initialization, a component default initializer,
@@ -416,6 +498,7 @@ std::optional<Expr<SomeType>> NonPointerInitializationExpr(const Symbol &symbol,
   if (auto symTS{
           characteristics::TypeAndShape::Characterize(symbol, context)}) {
     auto xType{x.GetType()};
+    CheckRealWidening(x, symTS->type(), context);
     auto converted{ConvertToType(symTS->type(), Expr<SomeType>{x})};
     if (!converted &&
         symbol.owner().context().IsEnabled(
@@ -433,6 +516,7 @@ std::optional<Expr<SomeType>> NonPointerInitializationExpr(const Symbol &symbol,
     if (converted) {
       auto folded{Fold(context, std::move(*converted))};
       if (IsActuallyConstant(folded)) {
+        InexactLiteralConversionFlagClearer{}(folded);
         int symRank{symTS->Rank()};
         if (IsImpliedShape(symbol)) {
           if (folded.Rank() == symRank) {
diff --git a/flang/lib/Evaluate/fold-complex.cpp b/flang/lib/Evaluate/fold-complex.cpp
index 3eb8e1f3f1fc2..bcaede5536260 100644
--- a/flang/lib/Evaluate/fold-complex.cpp
+++ b/flang/lib/Evaluate/fold-complex.cpp
@@ -83,12 +83,21 @@ Expr<Type<TypeCategory::Complex, KIND>> FoldOperation(
   if (auto array{ApplyElementwise(context, x)}) {
     return *array;
   }
-  using Result = Type<TypeCategory::Complex, KIND>;
+  using ComplexType = Type<TypeCategory::Complex, KIND>;
   if (auto folded{OperandsAreConstants(x)}) {
-    return Expr<Result>{
-        Constant<Result>{Scalar<Result>{folded->first, folded->second}}};
+    using RealType = typename ComplexType::Part;
+    Constant<ComplexType> result{
+        Scalar<ComplexType>{folded->first, folded->second}};
+    if (const auto *re{UnwrapConstantValue<RealType>(x.left())};
+        re && re->result().isFromInexactLiteralConversion()) {
+      result.result().set_isFromInexactLiteralConversion();
+    } else if (const auto *im{UnwrapConstantValue<RealType>(x.right())};
+        im && im->result().isFromInexactLiteralConversion()) {
+      result.result().set_isFromInexactLiteralConversion();
+    }
+    return Expr<ComplexType>{std::move(result)};
   }
-  return Expr<Result>{std::move(x)};
+  return Expr<ComplexType>{std::move(x)};
 }
 
 #ifdef _MSC_VER // disable bogus warning about missing definitions
diff --git a/flang/lib/Evaluate/fold-implementation.h b/flang/lib/Evaluate/fold-implementation.h
index 52e954d1cffbf..7c80d76784463 100644
--- a/flang/lib/Evaluate/fold-implementation.h
+++ b/flang/lib/Evaluate/fold-implementation.h
@@ -1321,8 +1321,8 @@ template <typename T> class ArrayConstructorFolder {
               *charLength_, std::move(elements_), ConstantSubscripts{n}}};
         }
       } else {
-        return Expr<T>{
-            Constant<T>{std::move(elements_), ConstantSubscripts{n}}};
+        return Expr<T>{Constant<T>{
+            std::move(elements_), ConstantSubscripts{n}, resultInfo_}};
       }
     }
     return Expr<T>{std::move(array)};
@@ -1343,6 +1343,11 @@ template <typename T> class ArrayConstructorFolder {
         if (!knownCharLength_) {
           charLength_ = std::max(c->LEN(), charLength_.value_or(-1));
         }
+      } else if constexpr (T::category == TypeCategory::Real ||
+          T::category == TypeCategory::Complex) {
+        if (c->result().isFromInexactLiteralConversion()) {
+          resultInfo_.set_isFromInexactLiteralConversion();
+        }
       }
       return true;
     } else {
@@ -1395,6 +1400,7 @@ template <typename T> class ArrayConstructorFolder {
   std::vector<Scalar<T>> elements_;
   std::optional<ConstantSubscript> charLength_;
   bool knownCharLength_{false};
+  typename Constant<T>::Result resultInfo_;
 };
 
 template <typename T>
diff --git a/flang/lib/Evaluate/formatting.cpp b/flang/lib/Evaluate/formatting.cpp
index 121afc6f0f8bf..ec5dc0baaa5cb 100644
--- a/flang/lib/Evaluate/formatting.cpp
+++ b/flang/lib/Evaluate/formatting.cpp
@@ -98,6 +98,14 @@ llvm::raw_ostream &ConstantBase<RESULT, VALUE>::AsFortran(
   return o;
 }
 
+template <typename RESULT, typename VALUE>
+std::string ConstantBase<RESULT, VALUE>::AsFortran() const {
+  std::string result;
+  llvm::raw_string_ostream sstream(result);
+  AsFortran(sstream);
+  return result;
+}
+
 template <int KIND>
 llvm::raw_ostream &Constant<Type<TypeCategory::Character, KIND>>::AsFortran(
     llvm::raw_ostream &o) const {
@@ -126,6 +134,14 @@ llvm::raw_ostream &Constant<Type<TypeCategory::Character, KIND>>::AsFortran(
   return o;
 }
 
+template <int KIND>
+std::string Constant<Type<TypeCategory::Character, KIND>>::AsFortran() const {
+  std::string result;
+  llvm::raw_string_ostream sstream(result);
+  AsFortran(sstream);
+  return result;
+}
+
 llvm::raw_ostream &EmitVar(llvm::raw_ostream &o, const Symbol &symbol,
     std::optional<parser::CharBlock> name = std::nullopt) {
   const auto &renamings{symbol.owner().context().moduleFileOutputRenamings()};
diff --git a/flang/lib/Evaluate/intrinsics.cpp b/flang/lib/Evaluate/intrinsics.cpp
index c37a7f908d4d1..0f79ba6ed62b6 100644
--- a/flang/lib/Evaluate/intrinsics.cpp
+++ b/flang/lib/Evaluate/intrinsics.cpp
@@ -666,7 +666,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
             {ArgFlag::canBeMoldNull, ArgFlag::onlyConstantInquiry}}},
         DefaultInt, Rank::elemental, IntrinsicClass::inquiryFunction},
     {"lbound",
-        {{"array", AnyData, Rank::anyOrAssumedRank}, RequiredDIM,
+        {{"array", AnyData, Rank::arrayOrAssumedRank}, RequiredDIM,
             SizeDefaultKIND},
         KINDInt, Rank::scalar, IntrinsicClass::inquiryFunction},
     {"lbound", {{"array", AnyData, Rank::arrayOrAssumedRank}, SizeDefaultKIND},
@@ -1034,7 +1034,7 @@ static const IntrinsicInterface genericIntrinsicFunction[]{
     {"trim", {{"string", SameCharNoLen, Rank::scalar}}, SameCharNoLen,
         Rank::scalar, IntrinsicClass::transformationalFunction},
     {"ubound",
-        {{"array", AnyData, Rank::anyOrAssumedRank}, RequiredDIM,
+        {{"array", AnyData, Rank::arrayOrAssumedRank}, RequiredDIM,
             SizeDefaultKIND},
         KINDInt, Rank::scalar, IntrinsicClass::inquiryFunction},
     {"ubound", {{"array", AnyData, Rank::arrayOrAssumedRank}, SizeDefaultKIND},
diff --git a/flang/lib/Evaluate/real.cpp b/flang/lib/Evaluate/real.cpp
index 2c0f2833f07dc..6e6b9f3ac77c2 100644
--- a/flang/lib/Evaluate/real.cpp
+++ b/flang/lib/Evaluate/real.cpp
@@ -750,6 +750,14 @@ llvm::raw_ostream &Real<W, P>::AsFortran(
   return o;
 }
 
+template <typename W, int P>
+std::string Real<W, P>::AsFortran(int kind, bool minimal) const {
+  std::string result;
+  llvm::raw_string_ostream sstream(result);
+  AsFortran(sstream, kind, minimal);
+  return result;
+}
+
 // 16.9.180
 template <typename W, int P> Real<W, P> Real<W, P>::RRSPACING() const {
   if (IsNotANumber()) {
diff --git a/flang/lib/Evaluate/shape.cpp b/flang/lib/Evaluate/shape.cpp
index 776866d1416d2..894049f32a6bf 100644
--- a/flang/lib/Evaluate/shape.cpp
+++ b/flang/lib/Evaluate/shape.cpp
@@ -623,7 +623,7 @@ MaybeExtentExpr GetRawUpperBound(
       } else if (semantics::IsAssumedSizeArray(symbol) &&
           dimension + 1 == symbol.Rank()) {
         return std::nullopt;
-      } else {
+      } else if (IsSafelyCopyable(base, /*admitPureCall=*/true)) {
         return ComputeUpperBound(
             GetRawLowerBound(base, dimension), GetExtent(base, dimension));
       }
@@ -678,9 +678,11 @@ static MaybeExtentExpr GetUBOUND(FoldingContext *context,
       } else if (semantics::IsAssumedSizeArray(symbol) &&
           dimension + 1 == symbol.Rank()) {
         return std::nullopt; // UBOUND() folding replaces with -1
-      } else if (auto lb{GetLBOUND(base, dimension, invariantOnly)}) {
-        return ComputeUpperBound(
-            std::move(*lb), GetExtent(base, dimension, invariantOnly));
+      } else if (IsSafelyCopyable(base, /*admitPureCall=*/true)) {
+        if (auto lb{GetLBOUND(base, dimension, invariantOnly)}) {
+          return ComputeUpperBound(
+              std::move(*lb), GetExtent(base, dimension, invariantOnly));
+        }
       }
     }
   } else if (const auto *assoc{
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 9c059b08dd41c..3b2c4f9f56016 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -1129,7 +1129,7 @@ struct CollectCudaSymbolsHelper : public SetTraverse<CollectCudaSymbolsHelper,
   CollectCudaSymbolsHelper() : Base{*this} {}
   using Base::operator();
   semantics::UnorderedSymbolSet operator()(const Symbol &symbol) const {
-    return {symbol};
+    return {symbol.GetUltimate()};
   }
   // Overload some of the operator() to filter out the symbols that are not
   // of interest for CUDA data transfer logic.
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 111c5aa48726f..4719a242035ed 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1152,6 +1152,17 @@ static bool parseDialectArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
       diags.Report(diagID);
     }
   }
+  // -fcoarray
+  if (args.hasArg(clang::driver::options::OPT_fcoarray)) {
+    res.getFrontendOpts().features.Enable(
+        Fortran::common::LanguageFeature::Coarray);
+    const unsigned diagID =
+        diags.getCustomDiagID(clang::DiagnosticsEngine::Warning,
+                              "Support for multi image Fortran features is "
+                              "still experimental and in development.");
+    diags.Report(diagID);
+  }
+
   return diags.getNumErrors() == numErrorsBefore;
 }
 
@@ -1162,13 +1173,21 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
                             clang::DiagnosticsEngine &diags) {
   llvm::opt::Arg *arg = args.getLastArg(clang::driver::options::OPT_fopenmp,
                                         clang::driver::options::OPT_fno_openmp);
-  if (!arg || arg->getOption().matches(clang::driver::options::OPT_fno_openmp))
-    return true;
+  if (!arg ||
+      arg->getOption().matches(clang::driver::options::OPT_fno_openmp)) {
+    bool isSimdSpecified = args.hasFlag(
+        clang::driver::options::OPT_fopenmp_simd,
+        clang::driver::options::OPT_fno_openmp_simd, /*Default=*/false);
+    if (!isSimdSpecified)
+      return true;
+    res.getLangOpts().OpenMPSimd = 1;
+  }
 
   unsigned numErrorsBefore = diags.getNumErrors();
   llvm::Triple t(res.getTargetOpts().triple);
 
   constexpr unsigned newestFullySupported = 31;
+  constexpr unsigned latestFinalized = 60;
   // By default OpenMP is set to the most recent fully supported version
   res.getLangOpts().OpenMPVersion = newestFullySupported;
   res.getFrontendOpts().features.Enable(
@@ -1191,12 +1210,26 @@ static bool parseOpenMPArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
       diags.Report(diagID) << value << arg->getAsString(args) << versions.str();
     };
 
+    auto reportFutureVersion = [&](llvm::StringRef value) {
+      const unsigned diagID = diags.getCustomDiagID(
+          clang::DiagnosticsEngine::Warning,
+          "The specification for OpenMP version %0 is still under development; "
+          "the syntax and semantics of new features may be subject to change");
+      std::string buffer;
+      llvm::raw_string_ostream versions(buffer);
+      llvm::interleaveComma(ompVersions, versions);
+
+      diags.Report(diagID) << value;
+    };
+
     llvm::StringRef value = arg->getValue();
     if (!value.getAsInteger(/*radix=*/10, version)) {
       if (llvm::is_contained(ompVersions, version)) {
         res.getLangOpts().OpenMPVersion = version;
 
-        if (version > newestFullySupported)
+        if (version > latestFinalized)
+          reportFutureVersion(value);
+        else if (version > newestFullySupported)
           diags.Report(clang::diag::warn_openmp_incomplete) << version;
       } else if (llvm::is_contained(oldVersions, version)) {
         const unsigned diagID =
@@ -1696,6 +1729,20 @@ void CompilerInvocation::setDefaultPredefinitions() {
   fortranOptions.predefinitions.emplace_back("__flang_patchlevel__",
                                              FLANG_VERSION_PATCHLEVEL_STRING);
 
+  // Add predefinitions based on the relocation model
+  if (unsigned PICLevel = getCodeGenOpts().PICLevel) {
+    fortranOptions.predefinitions.emplace_back("__PIC__",
+                                               std::to_string(PICLevel));
+    fortranOptions.predefinitions.emplace_back("__pic__",
+                                               std::to_string(PICLevel));
+    if (getCodeGenOpts().IsPIE) {
+      fortranOptions.predefinitions.emplace_back("__PIE__",
+                                                 std::to_string(PICLevel));
+      fortranOptions.predefinitions.emplace_back("__pie__",
+                                                 std::to_string(PICLevel));
+    }
+  }
+
   // Add predefinitions based on extensions enabled
   if (frontendOptions.features.IsEnabled(
           Fortran::common::LanguageFeature::OpenACC)) {
diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp
index 5c66ecf3043cd..3bef6b1c31825 100644
--- a/flang/lib/Frontend/FrontendActions.cpp
+++ b/flang/lib/Frontend/FrontendActions.cpp
@@ -298,6 +298,7 @@ bool CodeGenAction::beginSourceFileAction() {
   bool isOpenMPEnabled =
       ci.getInvocation().getFrontendOpts().features.IsEnabled(
           Fortran::common::LanguageFeature::OpenMP);
+  bool isOpenMPSimd = ci.getInvocation().getLangOpts().OpenMPSimd;
 
   fir::OpenMPFIRPassPipelineOpts opts;
 
@@ -329,12 +330,13 @@ bool CodeGenAction::beginSourceFileAction() {
     if (auto offloadMod = llvm::dyn_cast<mlir::omp::OffloadModuleInterface>(
             mlirModule->getOperation()))
       opts.isTargetDevice = offloadMod.getIsTargetDevice();
+  }
 
-    // WARNING: This pipeline must be run immediately after the lowering to
-    // ensure that the FIR is correct with respect to OpenMP operations/
-    // attributes.
+  // WARNING: This pipeline must be run immediately after the lowering to
+  // ensure that the FIR is correct with respect to OpenMP operations/
+  // attributes.
+  if (isOpenMPEnabled || isOpenMPSimd)
     fir::createOpenMPFIRPassPipeline(pm, opts);
-  }
 
   pm.enableVerifier(/*verifyPasses=*/true);
   pm.addPass(std::make_unique<Fortran::lower::VerifierPass>());
@@ -617,12 +619,14 @@ void CodeGenAction::lowerHLFIRToFIR() {
   pm.addPass(std::make_unique<Fortran::lower::VerifierPass>());
   pm.enableVerifier(/*verifyPasses=*/true);
 
+  fir::EnableOpenMP enableOpenMP = fir::EnableOpenMP::None;
+  if (ci.getInvocation().getFrontendOpts().features.IsEnabled(
+          Fortran::common::LanguageFeature::OpenMP))
+    enableOpenMP = fir::EnableOpenMP::Full;
+  if (ci.getInvocation().getLangOpts().OpenMPSimd)
+    enableOpenMP = fir::EnableOpenMP::Simd;
   // Create the pass pipeline
-  fir::createHLFIRToFIRPassPipeline(
-      pm,
-      ci.getInvocation().getFrontendOpts().features.IsEnabled(
-          Fortran::common::LanguageFeature::OpenMP),
-      level);
+  fir::createHLFIRToFIRPassPipeline(pm, enableOpenMP, level);
   (void)mlir::applyPassManagerCLOptions(pm);
 
   mlir::TimingScope timingScopeMLIRPasses = timingScopeRoot.nest(
@@ -748,6 +752,9 @@ void CodeGenAction::generateLLVMIR() {
           Fortran::common::LanguageFeature::OpenMP))
     config.EnableOpenMP = true;
 
+  if (ci.getInvocation().getLangOpts().OpenMPSimd)
+    config.EnableOpenMPSimd = true;
+
   if (ci.getInvocation().getLoweringOpts().getIntegerWrapAround())
     config.NSWOnLoopVarInc = false;
 
diff --git a/flang/lib/Lower/Allocatable.cpp b/flang/lib/Lower/Allocatable.cpp
index 219f9205f45d5..444b5b6c7c4b1 100644
--- a/flang/lib/Lower/Allocatable.cpp
+++ b/flang/lib/Lower/Allocatable.cpp
@@ -13,9 +13,9 @@
 #include "flang/Lower/Allocatable.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/CUDA.h"
 #include "flang/Lower/ConvertType.h"
 #include "flang/Lower/ConvertVariable.h"
-#include "flang/Lower/Cuda.h"
 #include "flang/Lower/IterationSpace.h"
 #include "flang/Lower/Mangler.h"
 #include "flang/Lower/OpenACC.h"
@@ -445,10 +445,14 @@ class AllocateStmtHelper {
                                        /*mustBeHeap=*/true);
   }
 
-  void postAllocationAction(const Allocation &alloc) {
+  void postAllocationAction(const Allocation &alloc,
+                            const fir::MutableBoxValue &box) {
     if (alloc.getSymbol().test(Fortran::semantics::Symbol::Flag::AccDeclare))
       Fortran::lower::attachDeclarePostAllocAction(converter, builder,
                                                    alloc.getSymbol());
+    if (Fortran::semantics::HasCUDAComponent(alloc.getSymbol()))
+      Fortran::lower::initializeDeviceComponentAllocator(
+          converter, alloc.getSymbol(), box);
   }
 
   void setPinnedToFalse() {
@@ -481,11 +485,21 @@ class AllocateStmtHelper {
       // Pointers must use PointerAllocate so that their deallocations
       // can be validated.
       genInlinedAllocation(alloc, box);
-      postAllocationAction(alloc);
+      postAllocationAction(alloc, box);
       setPinnedToFalse();
       return;
     }
 
+    // Preserve characters' dynamic length.
+    if (lenParams.empty() && box.isCharacter() &&
+        !box.hasNonDeferredLenParams()) {
+      auto charTy = mlir::dyn_cast<fir::CharacterType>(box.getEleTy());
+      if (charTy && charTy.hasDynamicLen()) {
+        fir::ExtendedValue exv{box};
+        lenParams.push_back(fir::factory::readCharLen(builder, loc, exv));
+      }
+    }
+
     // Generate a sequence of runtime calls.
     errorManager.genStatCheck(builder, loc);
     genAllocateObjectInit(box, allocatorIdx);
@@ -504,7 +518,7 @@ class AllocateStmtHelper {
           genCudaAllocate(builder, loc, box, errorManager, alloc.getSymbol());
     }
     fir::factory::syncMutableBoxFromIRBox(builder, loc, box);
-    postAllocationAction(alloc);
+    postAllocationAction(alloc, box);
     errorManager.assignStat(builder, loc, stat);
   }
 
@@ -647,7 +661,7 @@ class AllocateStmtHelper {
       setPinnedToFalse();
     }
     fir::factory::syncMutableBoxFromIRBox(builder, loc, box);
-    postAllocationAction(alloc);
+    postAllocationAction(alloc, box);
     errorManager.assignStat(builder, loc, stat);
   }
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 6b7efe6b57db3..c003a5b04ecd5 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -13,6 +13,7 @@
 #include "flang/Lower/Bridge.h"
 
 #include "flang/Lower/Allocatable.h"
+#include "flang/Lower/CUDA.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/Coarray.h"
 #include "flang/Lower/ConvertCall.h"
@@ -20,7 +21,6 @@
 #include "flang/Lower/ConvertExprToHLFIR.h"
 #include "flang/Lower/ConvertType.h"
 #include "flang/Lower/ConvertVariable.h"
-#include "flang/Lower/Cuda.h"
 #include "flang/Lower/DirectivesCommon.h"
 #include "flang/Lower/HostAssociations.h"
 #include "flang/Lower/IO.h"
@@ -475,7 +475,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         fir::runtime::genMain(*builder, toLocation(),
                               bridge.getEnvironmentDefaults(),
                               getFoldingContext().languageFeatures().IsEnabled(
-                                  Fortran::common::LanguageFeature::CUDA));
+                                  Fortran::common::LanguageFeature::CUDA),
+                              getFoldingContext().languageFeatures().IsEnabled(
+                                  Fortran::common::LanguageFeature::Coarray));
       });
 
     finalizeOpenMPLowering(globalOmpRequiresSymbol);
@@ -1400,21 +1402,23 @@ class FirConverter : public Fortran::lower::AbstractConverter {
   mlir::Value genLoopVariableAddress(mlir::Location loc,
                                      const Fortran::semantics::Symbol &sym,
                                      bool isUnordered) {
-    if (isUnordered || sym.has<Fortran::semantics::HostAssocDetails>() ||
-        sym.has<Fortran::semantics::UseDetails>()) {
-      if (!shallowLookupSymbol(sym) &&
-          !GetSymbolDSA(sym).test(
-              Fortran::semantics::Symbol::Flag::OmpShared)) {
-        // Do concurrent loop variables are not mapped yet since they are local
-        // to the Do concurrent scope (same for OpenMP loops).
-        mlir::OpBuilder::InsertPoint insPt = builder->saveInsertionPoint();
-        builder->setInsertionPointToStart(builder->getAllocaBlock());
-        mlir::Type tempTy = genType(sym);
-        mlir::Value temp =
-            builder->createTemporaryAlloc(loc, tempTy, toStringRef(sym.name()));
-        bindIfNewSymbol(sym, temp);
-        builder->restoreInsertionPoint(insPt);
-      }
+    if (!shallowLookupSymbol(sym) &&
+        (isUnordered ||
+         GetSymbolDSA(sym).test(Fortran::semantics::Symbol::Flag::OmpPrivate) ||
+         GetSymbolDSA(sym).test(
+             Fortran::semantics::Symbol::Flag::OmpFirstPrivate) ||
+         GetSymbolDSA(sym).test(
+             Fortran::semantics::Symbol::Flag::OmpLastPrivate) ||
+         GetSymbolDSA(sym).test(Fortran::semantics::Symbol::Flag::OmpLinear))) {
+      // Do concurrent loop variables are not mapped yet since they are
+      // local to the Do concurrent scope (same for OpenMP loops).
+      mlir::OpBuilder::InsertPoint insPt = builder->saveInsertionPoint();
+      builder->setInsertionPointToStart(builder->getAllocaBlock());
+      mlir::Type tempTy = genType(sym);
+      mlir::Value temp =
+          builder->createTemporaryAlloc(loc, tempTy, toStringRef(sym.name()));
+      bindIfNewSymbol(sym, temp);
+      builder->restoreInsertionPoint(insPt);
     }
     auto entry = lookupSymbol(sym);
     (void)entry;
@@ -2060,10 +2064,10 @@ class FirConverter : public Fortran::lower::AbstractConverter {
     // TODO Promote to using `enableDelayedPrivatization` (which is enabled by
     // default unlike the staging flag) once the implementation of this is more
     // complete.
-    bool useDelayedPriv =
-        enableDelayedPrivatizationStaging && doConcurrentLoopOp;
+    bool useDelayedPriv = enableDelayedPrivatization && doConcurrentLoopOp;
     llvm::SetVector<const Fortran::semantics::Symbol *> allPrivatizedSymbols;
-    llvm::SmallSet<const Fortran::semantics::Symbol *, 16> mightHaveReadHostSym;
+    llvm::SmallPtrSet<const Fortran::semantics::Symbol *, 16>
+        mightHaveReadHostSym;
 
     for (const Fortran::semantics::Symbol *symToPrivatize : info.localSymList) {
       if (useDelayedPriv) {
@@ -2122,6 +2126,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       }
     }
 
+    if (!doConcurrentLoopOp)
+      return;
+
     llvm::SmallVector<bool> reduceVarByRef;
     llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
     llvm::SmallVector<mlir::Attribute> nestReduceAttrs;
@@ -4824,7 +4831,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
   void genCUDADataTransfer(fir::FirOpBuilder &builder, mlir::Location loc,
                            const Fortran::evaluate::Assignment &assign,
-                           hlfir::Entity &lhs, hlfir::Entity &rhs) {
+                           hlfir::Entity &lhs, hlfir::Entity &rhs,
+                           bool isWholeAllocatableAssignment,
+                           bool keepLhsLengthInAllocatableAssignment) {
     bool lhsIsDevice = Fortran::evaluate::HasCUDADeviceAttrs(assign.lhs);
     bool rhsIsDevice = Fortran::evaluate::HasCUDADeviceAttrs(assign.rhs);
 
@@ -4889,6 +4898,28 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
     // host = device
     if (!lhsIsDevice && rhsIsDevice) {
+      if (Fortran::lower::isTransferWithConversion(rhs)) {
+        mlir::OpBuilder::InsertionGuard insertionGuard(builder);
+        auto elementalOp =
+            mlir::dyn_cast<hlfir::ElementalOp>(rhs.getDefiningOp());
+        assert(elementalOp && "expect elemental op");
+        auto designateOp =
+            *elementalOp.getBody()->getOps<hlfir::DesignateOp>().begin();
+        builder.setInsertionPoint(elementalOp);
+        // Create a temp to transfer the rhs before applying the conversion.
+        hlfir::Entity entity{designateOp.getMemref()};
+        auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, entity);
+        auto transferKindAttr = cuf::DataTransferKindAttr::get(
+            builder.getContext(), cuf::DataTransferKind::DeviceHost);
+        cuf::DataTransferOp::create(builder, loc, designateOp.getMemref(), temp,
+                                    /*shape=*/mlir::Value{}, transferKindAttr);
+        designateOp.getMemrefMutable().assign(temp);
+        builder.setInsertionPointAfter(elementalOp);
+        hlfir::AssignOp::create(builder, loc, elementalOp, lhs,
+                                isWholeAllocatableAssignment,
+                                keepLhsLengthInAllocatableAssignment);
+        return;
+      }
       auto transferKindAttr = cuf::DataTransferKindAttr::get(
           builder.getContext(), cuf::DataTransferKind::DeviceHost);
       cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
@@ -4898,7 +4929,6 @@ class FirConverter : public Fortran::lower::AbstractConverter {
 
     // device = device
     if (lhsIsDevice && rhsIsDevice) {
-      assert(rhs.isVariable() && "CUDA Fortran assignment rhs is not legal");
       auto transferKindAttr = cuf::DataTransferKindAttr::get(
           builder.getContext(), cuf::DataTransferKind::DeviceDevice);
       cuf::DataTransferOp::create(builder, loc, rhsVal, lhsVal, shape,
@@ -5037,7 +5067,9 @@ class FirConverter : public Fortran::lower::AbstractConverter {
       hlfir::Entity rhs = evaluateRhs(localStmtCtx);
       hlfir::Entity lhs = evaluateLhs(localStmtCtx);
       if (isCUDATransfer && !hasCUDAImplicitTransfer)
-        genCUDADataTransfer(builder, loc, assign, lhs, rhs);
+        genCUDADataTransfer(builder, loc, assign, lhs, rhs,
+                            isWholeAllocatableAssignment,
+                            keepLhsLengthInAllocatableAssignment);
       else
         hlfir::AssignOp::create(builder, loc, rhs, lhs,
                                 isWholeAllocatableAssignment,
diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 8e20abf0e9f2d..1d1c7ddda8e9b 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -15,6 +15,7 @@ add_flang_library(FortranLower
   ConvertProcedureDesignator.cpp
   ConvertType.cpp
   ConvertVariable.cpp
+  CUDA.cpp
   CustomIntrinsicCall.cpp
   HlfirIntrinsics.cpp
   HostAssociations.cpp
diff --git a/flang/lib/Lower/CUDA.cpp b/flang/lib/Lower/CUDA.cpp
new file mode 100644
index 0000000000000..1293d2c5bd3ae
--- /dev/null
+++ b/flang/lib/Lower/CUDA.cpp
@@ -0,0 +1,167 @@
+//===-- CUDA.cpp -- CUDA Fortran specific lowering ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Lower/CUDA.h"
+#include "flang/Lower/AbstractConverter.h"
+#include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
+
+#define DEBUG_TYPE "flang-lower-cuda"
+
+void Fortran::lower::initializeDeviceComponentAllocator(
+    Fortran::lower::AbstractConverter &converter,
+    const Fortran::semantics::Symbol &sym, const fir::MutableBoxValue &box) {
+  if (const auto *details{
+          sym.GetUltimate()
+              .detailsIf<Fortran::semantics::ObjectEntityDetails>()}) {
+    const Fortran::semantics::DeclTypeSpec *type{details->type()};
+    const Fortran::semantics::DerivedTypeSpec *derived{type ? type->AsDerived()
+                                                            : nullptr};
+    if (derived) {
+      if (!FindCUDADeviceAllocatableUltimateComponent(*derived))
+        return; // No device components.
+
+      fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+      mlir::Location loc = converter.getCurrentLocation();
+
+      mlir::Type baseTy = fir::unwrapRefType(box.getAddr().getType());
+
+      // Only pointer and allocatable needs post allocation initialization
+      // of components descriptors.
+      if (!fir::isAllocatableType(baseTy) && !fir::isPointerType(baseTy))
+        return;
+
+      // Extract the derived type.
+      mlir::Type ty = fir::getDerivedType(baseTy);
+      auto recTy = mlir::dyn_cast<fir::RecordType>(ty);
+      assert(recTy && "expected fir::RecordType");
+
+      if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(baseTy))
+        baseTy = boxTy.getEleTy();
+      baseTy = fir::unwrapRefType(baseTy);
+
+      Fortran::semantics::UltimateComponentIterator components{*derived};
+      mlir::Value loadedBox = fir::LoadOp::create(builder, loc, box.getAddr());
+      mlir::Value addr;
+      if (auto seqTy = mlir::dyn_cast<fir::SequenceType>(baseTy)) {
+        mlir::Type idxTy = builder.getIndexType();
+        mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1);
+        mlir::Value zero = builder.createIntegerConstant(loc, idxTy, 0);
+        llvm::SmallVector<fir::DoLoopOp> loops;
+        llvm::SmallVector<mlir::Value> indices;
+        llvm::SmallVector<mlir::Value> extents;
+        for (unsigned i = 0; i < seqTy.getDimension(); ++i) {
+          mlir::Value dim = builder.createIntegerConstant(loc, idxTy, i);
+          auto dimInfo = fir::BoxDimsOp::create(builder, loc, idxTy, idxTy,
+                                                idxTy, loadedBox, dim);
+          mlir::Value lbub = mlir::arith::AddIOp::create(
+              builder, loc, dimInfo.getResult(0), dimInfo.getResult(1));
+          mlir::Value ext =
+              mlir::arith::SubIOp::create(builder, loc, lbub, one);
+          mlir::Value cmp = mlir::arith::CmpIOp::create(
+              builder, loc, mlir::arith::CmpIPredicate::sgt, ext, zero);
+          ext = mlir::arith::SelectOp::create(builder, loc, cmp, ext, zero);
+          extents.push_back(ext);
+
+          auto loop = fir::DoLoopOp::create(
+              builder, loc, dimInfo.getResult(0), dimInfo.getResult(1),
+              dimInfo.getResult(2), /*isUnordered=*/true,
+              /*finalCount=*/false, mlir::ValueRange{});
+          loops.push_back(loop);
+          indices.push_back(loop.getInductionVar());
+          builder.setInsertionPointToStart(loop.getBody());
+        }
+        mlir::Value boxAddr = fir::BoxAddrOp::create(builder, loc, loadedBox);
+        auto shape = fir::ShapeOp::create(builder, loc, extents);
+        addr = fir::ArrayCoorOp::create(
+            builder, loc, fir::ReferenceType::get(recTy), boxAddr, shape,
+            /*slice=*/mlir::Value{}, indices, /*typeparms=*/mlir::ValueRange{});
+      } else {
+        addr = fir::BoxAddrOp::create(builder, loc, loadedBox);
+      }
+      for (const auto &compSym : components) {
+        if (Fortran::semantics::IsDeviceAllocatable(compSym)) {
+          llvm::SmallVector<mlir::Value> coord;
+          mlir::Type fieldTy = gatherDeviceComponentCoordinatesAndType(
+              builder, loc, compSym, recTy, coord);
+          assert(coord.size() == 1 && "expect one coordinate");
+          mlir::Value comp = fir::CoordinateOp::create(
+              builder, loc, builder.getRefType(fieldTy), addr, coord[0]);
+          cuf::DataAttributeAttr dataAttr =
+              Fortran::lower::translateSymbolCUFDataAttribute(
+                  builder.getContext(), compSym);
+          cuf::SetAllocatorIndexOp::create(builder, loc, comp, dataAttr);
+        }
+      }
+    }
+  }
+}
+
+mlir::Type Fortran::lower::gatherDeviceComponentCoordinatesAndType(
+    fir::FirOpBuilder &builder, mlir::Location loc,
+    const Fortran::semantics::Symbol &sym, fir::RecordType recTy,
+    llvm::SmallVector<mlir::Value> &coordinates) {
+  unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
+  mlir::Type fieldTy;
+  if (fieldIdx != std::numeric_limits<unsigned>::max()) {
+    // Field found in the base record type.
+    auto fieldName = recTy.getTypeList()[fieldIdx].first;
+    fieldTy = recTy.getTypeList()[fieldIdx].second;
+    mlir::Value fieldIndex = fir::FieldIndexOp::create(
+        builder, loc, fir::FieldType::get(fieldTy.getContext()), fieldName,
+        recTy,
+        /*typeParams=*/mlir::ValueRange{});
+    coordinates.push_back(fieldIndex);
+  } else {
+    // Field not found in base record type, search in potential
+    // record type components.
+    for (auto component : recTy.getTypeList()) {
+      if (auto childRecTy = mlir::dyn_cast<fir::RecordType>(component.second)) {
+        fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
+        if (fieldIdx != std::numeric_limits<unsigned>::max()) {
+          mlir::Value parentFieldIndex = fir::FieldIndexOp::create(
+              builder, loc, fir::FieldType::get(childRecTy.getContext()),
+              component.first, recTy,
+              /*typeParams=*/mlir::ValueRange{});
+          coordinates.push_back(parentFieldIndex);
+          auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
+          fieldTy = childRecTy.getTypeList()[fieldIdx].second;
+          mlir::Value childFieldIndex = fir::FieldIndexOp::create(
+              builder, loc, fir::FieldType::get(fieldTy.getContext()),
+              fieldName, childRecTy,
+              /*typeParams=*/mlir::ValueRange{});
+          coordinates.push_back(childFieldIndex);
+          break;
+        }
+      }
+    }
+  }
+  if (coordinates.empty())
+    TODO(loc, "device resident component in complex derived-type hierarchy");
+  return fieldTy;
+}
+
+cuf::DataAttributeAttr Fortran::lower::translateSymbolCUFDataAttribute(
+    mlir::MLIRContext *mlirContext, const Fortran::semantics::Symbol &sym) {
+  std::optional<Fortran::common::CUDADataAttr> cudaAttr =
+      Fortran::semantics::GetCUDADataAttr(&sym.GetUltimate());
+  return cuf::getDataAttribute(mlirContext, cudaAttr);
+}
+
+bool Fortran::lower::isTransferWithConversion(mlir::Value rhs) {
+  if (auto elOp = mlir::dyn_cast<hlfir::ElementalOp>(rhs.getDefiningOp()))
+    if (llvm::hasSingleElement(elOp.getBody()->getOps<hlfir::DesignateOp>()) &&
+        llvm::hasSingleElement(elOp.getBody()->getOps<fir::LoadOp>()) == 1 &&
+        llvm::hasSingleElement(elOp.getBody()->getOps<fir::ConvertOp>()) == 1)
+      return true;
+  return false;
+}
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index a4a8a697e02ae..fd66592bc285b 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -14,12 +14,12 @@
 #include "flang/Lower/AbstractConverter.h"
 #include "flang/Lower/Allocatable.h"
 #include "flang/Lower/BoxAnalyzer.h"
+#include "flang/Lower/CUDA.h"
 #include "flang/Lower/CallInterface.h"
 #include "flang/Lower/ConvertConstant.h"
 #include "flang/Lower/ConvertExpr.h"
 #include "flang/Lower/ConvertExprToHLFIR.h"
 #include "flang/Lower/ConvertProcedureDesignator.h"
-#include "flang/Lower/Cuda.h"
 #include "flang/Lower/Mangler.h"
 #include "flang/Lower/PFTBuilder.h"
 #include "flang/Lower/StatementContext.h"
@@ -814,81 +814,24 @@ initializeDeviceComponentAllocator(Fortran::lower::AbstractConverter &converter,
         baseTy = boxTy.getEleTy();
       baseTy = fir::unwrapRefType(baseTy);
 
-      if (mlir::isa<fir::SequenceType>(baseTy) &&
-          (fir::isAllocatableType(fir::getBase(exv).getType()) ||
-           fir::isPointerType(fir::getBase(exv).getType())))
+      if (fir::isAllocatableType(fir::getBase(exv).getType()) ||
+          fir::isPointerType(fir::getBase(exv).getType()))
         return; // Allocator index need to be set after allocation.
 
       auto recTy =
           mlir::dyn_cast<fir::RecordType>(fir::unwrapSequenceType(baseTy));
       assert(recTy && "expected fir::RecordType");
 
-      llvm::SmallVector<mlir::Value> coordinates;
       Fortran::semantics::UltimateComponentIterator components{*derived};
       for (const auto &sym : components) {
         if (Fortran::semantics::IsDeviceAllocatable(sym)) {
-          unsigned fieldIdx = recTy.getFieldIndex(sym.name().ToString());
-          mlir::Type fieldTy;
-          llvm::SmallVector<mlir::Value> coordinates;
-
-          if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-            // Field found in the base record type.
-            auto fieldName = recTy.getTypeList()[fieldIdx].first;
-            fieldTy = recTy.getTypeList()[fieldIdx].second;
-            mlir::Value fieldIndex = fir::FieldIndexOp::create(
-                builder, loc, fir::FieldType::get(fieldTy.getContext()),
-                fieldName, recTy,
-                /*typeParams=*/mlir::ValueRange{});
-            coordinates.push_back(fieldIndex);
-          } else {
-            // Field not found in base record type, search in potential
-            // record type components.
-            for (auto component : recTy.getTypeList()) {
-              if (auto childRecTy =
-                      mlir::dyn_cast<fir::RecordType>(component.second)) {
-                fieldIdx = childRecTy.getFieldIndex(sym.name().ToString());
-                if (fieldIdx != std::numeric_limits<unsigned>::max()) {
-                  mlir::Value parentFieldIndex = fir::FieldIndexOp::create(
-                      builder, loc,
-                      fir::FieldType::get(childRecTy.getContext()),
-                      component.first, recTy,
-                      /*typeParams=*/mlir::ValueRange{});
-                  coordinates.push_back(parentFieldIndex);
-                  auto fieldName = childRecTy.getTypeList()[fieldIdx].first;
-                  fieldTy = childRecTy.getTypeList()[fieldIdx].second;
-                  mlir::Value childFieldIndex = fir::FieldIndexOp::create(
-                      builder, loc, fir::FieldType::get(fieldTy.getContext()),
-                      fieldName, childRecTy,
-                      /*typeParams=*/mlir::ValueRange{});
-                  coordinates.push_back(childFieldIndex);
-                  break;
-                }
-              }
-            }
-          }
-
-          if (coordinates.empty())
-            TODO(loc, "device resident component in complex derived-type "
-                      "hierarchy");
-
+          llvm::SmallVector<mlir::Value> coord;
+          mlir::Type fieldTy =
+              Fortran::lower::gatherDeviceComponentCoordinatesAndType(
+                  builder, loc, sym, recTy, coord);
           mlir::Value base = fir::getBase(exv);
-          mlir::Value comp;
-          if (mlir::isa<fir::BaseBoxType>(fir::unwrapRefType(base.getType()))) {
-            mlir::Value box = fir::LoadOp::create(builder, loc, base);
-            mlir::Value addr = fir::BoxAddrOp::create(builder, loc, box);
-            llvm::SmallVector<mlir::Value> lenParams;
-            assert(coordinates.size() == 1 && "expect one coordinate");
-            auto field = mlir::dyn_cast<fir::FieldIndexOp>(
-                coordinates[0].getDefiningOp());
-            comp = hlfir::DesignateOp::create(
-                builder, loc, builder.getRefType(fieldTy), addr,
-                /*component=*/field.getFieldName(),
-                /*componentShape=*/mlir::Value{},
-                hlfir::DesignateOp::Subscripts{});
-          } else {
-            comp = fir::CoordinateOp::create(
-                builder, loc, builder.getRefType(fieldTy), base, coordinates);
-          }
+          mlir::Value comp = fir::CoordinateOp::create(
+              builder, loc, builder.getRefType(fieldTy), base, coord);
           cuf::DataAttributeAttr dataAttr =
               Fortran::lower::translateSymbolCUFDataAttribute(
                   builder.getContext(), sym);
@@ -1950,13 +1893,6 @@ fir::FortranVariableFlagsAttr Fortran::lower::translateSymbolAttributes(
   return fir::FortranVariableFlagsAttr::get(mlirContext, flags);
 }
 
-cuf::DataAttributeAttr Fortran::lower::translateSymbolCUFDataAttribute(
-    mlir::MLIRContext *mlirContext, const Fortran::semantics::Symbol &sym) {
-  std::optional<Fortran::common::CUDADataAttr> cudaAttr =
-      Fortran::semantics::GetCUDADataAttr(&sym.GetUltimate());
-  return cuf::getDataAttribute(mlirContext, cudaAttr);
-}
-
 static bool
 isCapturedInInternalProcedure(Fortran::lower::AbstractConverter &converter,
                               const Fortran::semantics::Symbol &sym) {
diff --git a/flang/lib/Lower/HlfirIntrinsics.cpp b/flang/lib/Lower/HlfirIntrinsics.cpp
index 6e1d06a25924b..39595d6f519ad 100644
--- a/flang/lib/Lower/HlfirIntrinsics.cpp
+++ b/flang/lib/Lower/HlfirIntrinsics.cpp
@@ -170,6 +170,17 @@ class HlfirCShiftLowering : public HlfirTransformationalIntrinsic {
             mlir::Type stmtResultType) override;
 };
 
+class HlfirEOShiftLowering : public HlfirTransformationalIntrinsic {
+public:
+  using HlfirTransformationalIntrinsic::HlfirTransformationalIntrinsic;
+
+protected:
+  mlir::Value
+  lowerImpl(const Fortran::lower::PreparedActualArguments &loweredActuals,
+            const fir::IntrinsicArgumentLoweringRules *argLowering,
+            mlir::Type stmtResultType) override;
+};
+
 class HlfirReshapeLowering : public HlfirTransformationalIntrinsic {
 public:
   using HlfirTransformationalIntrinsic::HlfirTransformationalIntrinsic;
@@ -430,6 +441,46 @@ mlir::Value HlfirCShiftLowering::lowerImpl(
   return createOp<hlfir::CShiftOp>(resultType, operands);
 }
 
+mlir::Value HlfirEOShiftLowering::lowerImpl(
+    const Fortran::lower::PreparedActualArguments &loweredActuals,
+    const fir::IntrinsicArgumentLoweringRules *argLowering,
+    mlir::Type stmtResultType) {
+  auto operands = getOperandVector(loweredActuals, argLowering);
+  assert(operands.size() == 4);
+  mlir::Value array = operands[0];
+  mlir::Value shift = operands[1];
+  mlir::Value boundary = operands[2];
+  mlir::Value dim = operands[3];
+  // If DIM is present, then dereference it if it is a ref.
+  if (dim)
+    dim = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{dim});
+
+  mlir::Type resultType = computeResultType(array, stmtResultType);
+
+  if (boundary && fir::isa_trivial(boundary.getType())) {
+    mlir::Type elementType = hlfir::getFortranElementType(resultType);
+    if (auto logicalTy = mlir::dyn_cast<fir::LogicalType>(elementType)) {
+      // Scalar logical constant boundary might be represented using i1, i2, ...
+      // type. We need to cast it to fir.logical type of the ARRAY/result.
+      if (boundary.getType() != logicalTy)
+        boundary = builder.createConvert(loc, logicalTy, boundary);
+    } else {
+      // When the boundary is a constant like '1u', the lowering converts
+      // it into a signless arith.constant value (which is a requirement
+      // of the Arith dialect). If the ARRAY/RESULT is also UNSIGNED,
+      // we have to cast the boundary to the same unsigned type.
+      auto resultIntTy = mlir::dyn_cast<mlir::IntegerType>(elementType);
+      auto boundaryIntTy =
+          mlir::dyn_cast<mlir::IntegerType>(boundary.getType());
+      if (resultIntTy && boundaryIntTy &&
+          resultIntTy.getSignedness() != boundaryIntTy.getSignedness())
+        boundary = builder.createConvert(loc, resultIntTy, boundary);
+    }
+  }
+
+  return createOp<hlfir::EOShiftOp>(resultType, array, shift, boundary, dim);
+}
+
 mlir::Value HlfirReshapeLowering::lowerImpl(
     const Fortran::lower::PreparedActualArguments &loweredActuals,
     const fir::IntrinsicArgumentLoweringRules *argLowering,
@@ -489,6 +540,9 @@ std::optional<hlfir::EntityWithAttributes> Fortran::lower::lowerHlfirIntrinsic(
   if (name == "cshift")
     return HlfirCShiftLowering{builder, loc}.lower(loweredActuals, argLowering,
                                                    stmtResultType);
+  if (name == "eoshift")
+    return HlfirEOShiftLowering{builder, loc}.lower(loweredActuals, argLowering,
+                                                    stmtResultType);
   if (name == "reshape")
     return HlfirReshapeLowering{builder, loc}.lower(loweredActuals, argLowering,
                                                     stmtResultType);
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
index ed0bff04ed889..ff82a36951bfa 100644
--- a/flang/lib/Lower/OpenMP/Atomic.cpp
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -43,179 +43,6 @@ namespace omp {
 using namespace Fortran::lower::omp;
 }
 
-namespace {
-// An example of a type that can be used to get the return value from
-// the visitor:
-//   visitor(type_identity<Xyz>) -> result_type
-using SomeArgType = evaluate::Type<common::TypeCategory::Integer, 4>;
-
-struct GetProc
-    : public evaluate::Traverse<GetProc, const evaluate::ProcedureDesignator *,
-                                false> {
-  using Result = const evaluate::ProcedureDesignator *;
-  using Base = evaluate::Traverse<GetProc, Result, false>;
-  GetProc() : Base(*this) {}
-
-  using Base::operator();
-
-  static Result Default() { return nullptr; }
-
-  Result operator()(const evaluate::ProcedureDesignator &p) const { return &p; }
-  static Result Combine(Result a, Result b) { return a != nullptr ? a : b; }
-};
-
-struct WithType {
-  WithType(const evaluate::DynamicType &t) : type(t) {
-    assert(type.category() != common::TypeCategory::Derived &&
-           "Type cannot be a derived type");
-  }
-
-  template <typename VisitorTy> //
-  auto visit(VisitorTy &&visitor) const
-      -> std::invoke_result_t<VisitorTy, SomeArgType> {
-    switch (type.category()) {
-    case common::TypeCategory::Integer:
-      switch (type.kind()) {
-      case 1:
-        return visitor(llvm::type_identity<evaluate::Type<Integer, 1>>{});
-      case 2:
-        return visitor(llvm::type_identity<evaluate::Type<Integer, 2>>{});
-      case 4:
-        return visitor(llvm::type_identity<evaluate::Type<Integer, 4>>{});
-      case 8:
-        return visitor(llvm::type_identity<evaluate::Type<Integer, 8>>{});
-      case 16:
-        return visitor(llvm::type_identity<evaluate::Type<Integer, 16>>{});
-      }
-      break;
-    case common::TypeCategory::Unsigned:
-      switch (type.kind()) {
-      case 1:
-        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 1>>{});
-      case 2:
-        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 2>>{});
-      case 4:
-        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 4>>{});
-      case 8:
-        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 8>>{});
-      case 16:
-        return visitor(llvm::type_identity<evaluate::Type<Unsigned, 16>>{});
-      }
-      break;
-    case common::TypeCategory::Real:
-      switch (type.kind()) {
-      case 2:
-        return visitor(llvm::type_identity<evaluate::Type<Real, 2>>{});
-      case 3:
-        return visitor(llvm::type_identity<evaluate::Type<Real, 3>>{});
-      case 4:
-        return visitor(llvm::type_identity<evaluate::Type<Real, 4>>{});
-      case 8:
-        return visitor(llvm::type_identity<evaluate::Type<Real, 8>>{});
-      case 10:
-        return visitor(llvm::type_identity<evaluate::Type<Real, 10>>{});
-      case 16:
-        return visitor(llvm::type_identity<evaluate::Type<Real, 16>>{});
-      }
-      break;
-    case common::TypeCategory::Complex:
-      switch (type.kind()) {
-      case 2:
-        return visitor(llvm::type_identity<evaluate::Type<Complex, 2>>{});
-      case 3:
-        return visitor(llvm::type_identity<evaluate::Type<Complex, 3>>{});
-      case 4:
-        return visitor(llvm::type_identity<evaluate::Type<Complex, 4>>{});
-      case 8:
-        return visitor(llvm::type_identity<evaluate::Type<Complex, 8>>{});
-      case 10:
-        return visitor(llvm::type_identity<evaluate::Type<Complex, 10>>{});
-      case 16:
-        return visitor(llvm::type_identity<evaluate::Type<Complex, 16>>{});
-      }
-      break;
-    case common::TypeCategory::Logical:
-      switch (type.kind()) {
-      case 1:
-        return visitor(llvm::type_identity<evaluate::Type<Logical, 1>>{});
-      case 2:
-        return visitor(llvm::type_identity<evaluate::Type<Logical, 2>>{});
-      case 4:
-        return visitor(llvm::type_identity<evaluate::Type<Logical, 4>>{});
-      case 8:
-        return visitor(llvm::type_identity<evaluate::Type<Logical, 8>>{});
-      }
-      break;
-    case common::TypeCategory::Character:
-      switch (type.kind()) {
-      case 1:
-        return visitor(llvm::type_identity<evaluate::Type<Character, 1>>{});
-      case 2:
-        return visitor(llvm::type_identity<evaluate::Type<Character, 2>>{});
-      case 4:
-        return visitor(llvm::type_identity<evaluate::Type<Character, 4>>{});
-      }
-      break;
-    case common::TypeCategory::Derived:
-      (void)Derived;
-      break;
-    }
-    llvm_unreachable("Unhandled type");
-  }
-
-  const evaluate::DynamicType &type;
-
-private:
-  // Shorter names.
-  static constexpr auto Character = common::TypeCategory::Character;
-  static constexpr auto Complex = common::TypeCategory::Complex;
-  static constexpr auto Derived = common::TypeCategory::Derived;
-  static constexpr auto Integer = common::TypeCategory::Integer;
-  static constexpr auto Logical = common::TypeCategory::Logical;
-  static constexpr auto Real = common::TypeCategory::Real;
-  static constexpr auto Unsigned = common::TypeCategory::Unsigned;
-};
-
-template <typename T, typename U = std::remove_const_t<T>>
-U AsRvalue(T &t) {
-  U copy{t};
-  return std::move(copy);
-}
-
-template <typename T>
-T &&AsRvalue(T &&t) {
-  return std::move(t);
-}
-
-struct ArgumentReplacer
-    : public evaluate::Traverse<ArgumentReplacer, bool, false> {
-  using Base = evaluate::Traverse<ArgumentReplacer, bool, false>;
-  using Result = bool;
-
-  Result Default() const { return false; }
-
-  ArgumentReplacer(evaluate::ActualArguments &&newArgs)
-      : Base(*this), args_(std::move(newArgs)) {}
-
-  using Base::operator();
-
-  template <typename T>
-  Result operator()(const evaluate::FunctionRef<T> &x) {
-    assert(!done_);
-    auto &mut = const_cast<evaluate::FunctionRef<T> &>(x);
-    mut.arguments() = args_;
-    done_ = true;
-    return true;
-  }
-
-  Result Combine(Result &&a, Result &&b) { return a || b; }
-
-private:
-  bool done_{false};
-  evaluate::ActualArguments &&args_;
-};
-} // namespace
-
 [[maybe_unused]] static void
 dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
   auto whatStr = [](int k) {
@@ -412,85 +239,6 @@ makeMemOrderAttr(lower::AbstractConverter &converter,
   return nullptr;
 }
 
-static bool replaceArgs(semantics::SomeExpr &expr,
-                        evaluate::ActualArguments &&newArgs) {
-  return ArgumentReplacer(std::move(newArgs))(expr);
-}
-
-static semantics::SomeExpr makeCall(const evaluate::DynamicType &type,
-                                    const evaluate::ProcedureDesignator &proc,
-                                    const evaluate::ActualArguments &args) {
-  return WithType(type).visit([&](auto &&s) -> semantics::SomeExpr {
-    using Type = typename llvm::remove_cvref_t<decltype(s)>::type;
-    return evaluate::AsGenericExpr(
-        evaluate::FunctionRef<Type>(AsRvalue(proc), AsRvalue(args)));
-  });
-}
-
-static const evaluate::ProcedureDesignator &
-getProcedureDesignator(const semantics::SomeExpr &call) {
-  const evaluate::ProcedureDesignator *proc = GetProc{}(call);
-  assert(proc && "Call has no procedure designator");
-  return *proc;
-}
-
-static semantics::SomeExpr //
-genReducedMinMax(const semantics::SomeExpr &orig,
-                 const semantics::SomeExpr *atomArg,
-                 const std::vector<semantics::SomeExpr> &args) {
-  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
-  // One of the a_i's, say a_t, must be atomArg.
-  // Generate tmp = min/max(a0, a1, ... [except a_t]). Then generate
-  // call = min/max(a_t, tmp).
-  // Return "call".
-
-  // The min/max intrinsics have 2 mandatory arguments, the rest is optional.
-  // Make sure that the "tmp = min/max(...)" doesn't promote an optional
-  // argument to a non-optional position. This could happen if a_t is at
-  // position 0 or 1.
-  if (args.size() <= 2)
-    return orig;
-
-  evaluate::ActualArguments nonAtoms;
-
-  auto AsActual = [](const semantics::SomeExpr &x) {
-    semantics::SomeExpr copy = x;
-    return evaluate::ActualArgument(std::move(copy));
-  };
-  // Semantic checks guarantee that the "atom" shows exactly once in the
-  // argument list (with potential conversions around it).
-  // For the first two (non-optional) arguments, if "atom" is among them,
-  // replace it with another occurrence of the other non-optional argument.
-  if (atomArg == &args[0]) {
-    // (atom, x, y...) -> (x, x, y...)
-    nonAtoms.push_back(AsActual(args[1]));
-    nonAtoms.push_back(AsActual(args[1]));
-  } else if (atomArg == &args[1]) {
-    // (x, atom, y...) -> (x, x, y...)
-    nonAtoms.push_back(AsActual(args[0]));
-    nonAtoms.push_back(AsActual(args[0]));
-  } else {
-    // (x, y, z...) -> unchanged
-    nonAtoms.push_back(AsActual(args[0]));
-    nonAtoms.push_back(AsActual(args[1]));
-  }
-
-  // The rest of arguments are optional, so we can just skip "atom".
-  for (size_t i = 2, e = args.size(); i != e; ++i) {
-    if (atomArg != &args[i])
-      nonAtoms.push_back(AsActual(args[i]));
-  }
-
-  // The type of the intermediate min/max is the same as the type of its
-  // arguments, which may be different from the type of the original
-  // expression. The original expression may have additional coverts.
-  auto tmp =
-      makeCall(*atomArg->GetType(), getProcedureDesignator(orig), nonAtoms);
-  semantics::SomeExpr call = orig;
-  replaceArgs(call, {AsActual(*atomArg), AsActual(tmp)});
-  return call;
-}
-
 static mlir::Operation * //
 genAtomicRead(lower::AbstractConverter &converter,
               semantics::SemanticsContext &semaCtx, mlir::Location loc,
@@ -610,25 +358,6 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   auto [opcode, args] = evaluate::GetTopLevelOperationIgnoreResizing(input);
   assert(!args.empty() && "Update operation without arguments");
 
-  // Pass args as an argument to avoid capturing a structured binding.
-  const semantics::SomeExpr *atomArg = [&](auto &args) {
-    for (const semantics::SomeExpr &e : args) {
-      if (evaluate::IsSameOrConvertOf(e, atom))
-        return &e;
-    }
-    llvm_unreachable("Atomic variable not in argument list");
-  }(args);
-
-  if (opcode == evaluate::operation::Operator::Min ||
-      opcode == evaluate::operation::Operator::Max) {
-    // Min and max operations are expanded inline, so reduce them to
-    // operations with exactly two (non-optional) arguments.
-    rhs = genReducedMinMax(rhs, atomArg, args);
-    input = *evaluate::GetConvertInput(rhs);
-    std::tie(opcode, args) =
-        evaluate::GetTopLevelOperationIgnoreResizing(input);
-    atomArg = nullptr; // No longer valid.
-  }
   for (auto &arg : args) {
     if (!evaluate::IsSameOrConvertOf(arg, atom)) {
       mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index b98ad3cf7b76a..5d19f589d79fc 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1179,12 +1179,13 @@ bool ClauseProcessor::processLinear(mlir::omp::LinearClauseOps &result) const {
 }
 
 bool ClauseProcessor::processLink(
-    llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const {
+    llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &result) const {
   return findRepeatableClause<omp::clause::Link>(
       [&](const omp::clause::Link &clause, const parser::CharBlock &) {
         // Case: declare target link(var1, var2)...
         gatherFuncAndVarSyms(
-            clause.v, mlir::omp::DeclareTargetCaptureClause::link, result);
+            clause.v, mlir::omp::DeclareTargetCaptureClause::link, result,
+            /*automap=*/false);
       });
 }
 
@@ -1507,26 +1508,27 @@ bool ClauseProcessor::processTaskReduction(
 }
 
 bool ClauseProcessor::processTo(
-    llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const {
+    llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &result) const {
   return findRepeatableClause<omp::clause::To>(
       [&](const omp::clause::To &clause, const parser::CharBlock &) {
         // Case: declare target to(func, var1, var2)...
         gatherFuncAndVarSyms(std::get<ObjectList>(clause.t),
-                             mlir::omp::DeclareTargetCaptureClause::to, result);
+                             mlir::omp::DeclareTargetCaptureClause::to, result,
+                             /*automap=*/false);
       });
 }
 
 bool ClauseProcessor::processEnter(
-    llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const {
+    llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &result) const {
   return findRepeatableClause<omp::clause::Enter>(
       [&](const omp::clause::Enter &clause, const parser::CharBlock &source) {
-        mlir::Location currentLocation = converter.genLocation(source);
-        if (std::get<std::optional<omp::clause::Enter::Modifier>>(clause.t))
-          TODO(currentLocation, "Declare target enter AUTOMAP modifier");
+        bool automap =
+            std::get<std::optional<omp::clause::Enter::Modifier>>(clause.t)
+                .has_value();
         // Case: declare target enter(func, var1, var2)...
         gatherFuncAndVarSyms(std::get<ObjectList>(clause.t),
                              mlir::omp::DeclareTargetCaptureClause::enter,
-                             result);
+                             result, automap);
       });
 }
 
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index f8a1f7983b79b..c46bdb348a3ef 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -118,7 +118,7 @@ class ClauseProcessor {
   bool processDepend(lower::SymMap &symMap, lower::StatementContext &stmtCtx,
                      mlir::omp::DependClauseOps &result) const;
   bool
-  processEnter(llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const;
+  processEnter(llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &result) const;
   bool processIf(omp::clause::If::DirectiveNameModifier directiveName,
                  mlir::omp::IfClauseOps &result) const;
   bool processInReduction(
@@ -129,7 +129,7 @@ class ClauseProcessor {
       llvm::SmallVectorImpl<const semantics::Symbol *> &isDeviceSyms) const;
   bool processLinear(mlir::omp::LinearClauseOps &result) const;
   bool
-  processLink(llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const;
+  processLink(llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &result) const;
 
   // This method is used to process a map clause.
   // The optional parameter mapSyms is used to store the original Fortran symbol
@@ -150,7 +150,7 @@ class ClauseProcessor {
   bool processTaskReduction(
       mlir::Location currentLocation, mlir::omp::TaskReductionClauseOps &result,
       llvm::SmallVectorImpl<const semantics::Symbol *> &outReductionSyms) const;
-  bool processTo(llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const;
+  bool processTo(llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &result) const;
   bool processUseDeviceAddr(
       lower::StatementContext &stmtCtx,
       mlir::omp::UseDeviceAddrClauseOps &result,
@@ -208,11 +208,15 @@ void ClauseProcessor::processTODO(mlir::Location currentLocation,
     if (!x)
       return;
     unsigned version = semaCtx.langOptions().OpenMPVersion;
-    TODO(currentLocation,
-         "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() +
-             " in " +
-             llvm::omp::getOpenMPDirectiveName(directive, version).upper() +
-             " construct");
+    bool isSimdDirective = llvm::omp::getOpenMPDirectiveName(directive, version)
+                               .upper()
+                               .find("SIMD") != llvm::StringRef::npos;
+    if (!semaCtx.langOptions().OpenMPSimd || isSimdDirective)
+      TODO(currentLocation,
+           "Unhandled clause " + llvm::omp::getOpenMPClauseName(id).upper() +
+               " in " +
+               llvm::omp::getOpenMPDirectiveName(directive, version).upper() +
+               " construct");
   };
 
   for (ClauseIterator it = clauses.begin(); it != clauses.end(); ++it)
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 7f75aae09def1..1a16e1c87e250 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -396,6 +396,8 @@ makePrescriptiveness(parser::OmpPrescriptiveness::Value v) {
   switch (v) {
   case parser::OmpPrescriptiveness::Value::Strict:
     return clause::Prescriptiveness::Strict;
+  case parser::OmpPrescriptiveness::Value::Fallback:
+    return clause::Prescriptiveness::Fallback;
   }
   llvm_unreachable("Unexpected prescriptiveness");
 }
@@ -770,6 +772,27 @@ Doacross make(const parser::OmpClause::Doacross &inp,
 
 // DynamicAllocators: empty
 
+DynGroupprivate make(const parser::OmpClause::DynGroupprivate &inp,
+                     semantics::SemanticsContext &semaCtx) {
+  // imp.v -> OmpDyngroupprivateClause
+  CLAUSET_ENUM_CONVERT( //
+      convert, parser::OmpAccessGroup::Value, DynGroupprivate::AccessGroup,
+      // clang-format off
+      MS(Cgroup,  Cgroup)
+      // clang-format on
+  );
+
+  auto &mods = semantics::OmpGetModifiers(inp.v);
+  auto *m0 = semantics::OmpGetUniqueModifier<parser::OmpAccessGroup>(mods);
+  auto *m1 = semantics::OmpGetUniqueModifier<parser::OmpPrescriptiveness>(mods);
+  auto &size = std::get<parser::ScalarIntExpr>(inp.v.t);
+
+  return DynGroupprivate{
+      {/*AccessGroup=*/maybeApplyToV(convert, m0),
+       /*Prescriptiveness=*/maybeApplyToV(makePrescriptiveness, m1),
+       /*Size=*/makeExpr(size, semaCtx)}};
+}
+
 Enter make(const parser::OmpClause::Enter &inp,
            semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpEnterClause
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 67a9a4675f115..9d1c730b38edd 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -30,18 +30,26 @@
 #include "flang/Semantics/tools.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallSet.h"
+#include <variant>
 
 namespace Fortran {
 namespace lower {
 namespace omp {
 bool DataSharingProcessor::OMPConstructSymbolVisitor::isSymbolDefineBy(
     const semantics::Symbol *symbol, lower::pft::Evaluation &eval) const {
-  return eval.visit(
-      common::visitors{[&](const parser::OpenMPConstruct &functionParserNode) {
-                         return symDefMap.count(symbol) &&
-                                symDefMap.at(symbol) == &functionParserNode;
-                       },
-                       [](const auto &functionParserNode) { return false; }});
+  return eval.visit(common::visitors{
+      [&](const parser::OpenMPConstruct &functionParserNode) {
+        return symDefMap.count(symbol) &&
+               symDefMap.at(symbol) == ConstructPtr(&functionParserNode);
+      },
+      [](const auto &functionParserNode) { return false; }});
+}
+
+bool DataSharingProcessor::OMPConstructSymbolVisitor::
+    isSymbolDefineByNestedDeclaration(const semantics::Symbol *symbol) const {
+  return symDefMap.count(symbol) &&
+         std::holds_alternative<const parser::DeclarationConstruct *>(
+             symDefMap.at(symbol));
 }
 
 static bool isConstructWithTopLevelTarget(lower::pft::Evaluation &eval) {
@@ -550,11 +558,23 @@ void DataSharingProcessor::collectSymbols(
         return false;
       }
 
-      return sym->test(semantics::Symbol::Flag::OmpImplicit);
+      // Collect implicit symbols only if they are not defined by a nested
+      // `DeclarationConstruct`. If `sym` is not defined by the current OpenMP
+      // evaluation then it is defined by a block nested within the OpenMP
+      // construct. This, in turn, means that the private allocation for the
+      // symbol will be emitted as part of the nested block and there is no need
+      // to privatize it within the OpenMP construct.
+      return !visitor.isSymbolDefineByNestedDeclaration(sym) &&
+             sym->test(semantics::Symbol::Flag::OmpImplicit);
     }
 
-    if (collectPreDetermined)
-      return sym->test(semantics::Symbol::Flag::OmpPreDetermined);
+    if (collectPreDetermined) {
+      // Similar to implicit symbols, collect pre-determined symbols only if
+      // they are not defined by a nested `DeclarationConstruct`
+      return visitor.isSymbolDefineBy(sym, eval) &&
+             !visitor.isSymbolDefineByNestedDeclaration(sym) &&
+             sym->test(semantics::Symbol::Flag::OmpPreDetermined);
+    }
 
     return !sym->test(semantics::Symbol::Flag::OmpImplicit) &&
            !sym->test(semantics::Symbol::Flag::OmpPreDetermined);
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index 96e7fa6d5d93c..00b2d95bab224 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -19,6 +19,7 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/symbol.h"
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include <variant>
 
 namespace mlir {
 namespace omp {
@@ -58,20 +59,35 @@ class DataSharingProcessor {
     }
 
     void Post(const parser::Name &name) {
-      auto *current = !constructs.empty() ? constructs.back() : nullptr;
+      auto current = !constructs.empty() ? constructs.back() : ConstructPtr();
       symDefMap.try_emplace(name.symbol, current);
     }
 
-    llvm::SmallVector<const parser::OpenMPConstruct *> constructs;
-    llvm::DenseMap<semantics::Symbol *, const parser::OpenMPConstruct *>
-        symDefMap;
+    bool Pre(const parser::DeclarationConstruct &decl) {
+      constructs.push_back(&decl);
+      return true;
+    }
+
+    void Post(const parser::DeclarationConstruct &decl) {
+      constructs.pop_back();
+    }
 
     /// Given a \p symbol and an \p eval, returns true if eval is the OMP
     /// construct that defines symbol.
     bool isSymbolDefineBy(const semantics::Symbol *symbol,
                           lower::pft::Evaluation &eval) const;
 
+    // Given a \p symbol, returns true if it is defined by a nested
+    // `DeclarationConstruct`.
+    bool
+    isSymbolDefineByNestedDeclaration(const semantics::Symbol *symbol) const;
+
   private:
+    using ConstructPtr = std::variant<const parser::OpenMPConstruct *,
+                                      const parser::DeclarationConstruct *>;
+    llvm::SmallVector<ConstructPtr> constructs;
+    llvm::DenseMap<semantics::Symbol *, ConstructPtr> symDefMap;
+
     unsigned version;
   };
 
@@ -91,7 +107,7 @@ class DataSharingProcessor {
   lower::pft::Evaluation &eval;
   bool shouldCollectPreDeterminedSymbols;
   bool useDelayedPrivatization;
-  llvm::SmallSet<const semantics::Symbol *, 16> mightHaveReadHostSym;
+  llvm::SmallPtrSet<const semantics::Symbol *, 16> mightHaveReadHostSym;
   lower::SymMap &symTable;
   bool isTargetPrivatization;
   OMPConstructSymbolVisitor visitor;
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index db6a0e2f39601..ec2ec37e623f8 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -34,6 +34,7 @@
 #include "flang/Parser/openmp-utils.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/openmp-directive-sets.h"
+#include "flang/Semantics/openmp-utils.h"
 #include "flang/Semantics/tools.h"
 #include "flang/Support/Flags.h"
 #include "flang/Support/OpenMP-utils.h"
@@ -764,14 +765,14 @@ static void getDeclareTargetInfo(
     lower::pft::Evaluation &eval,
     const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
     mlir::omp::DeclareTargetOperands &clauseOps,
-    llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause) {
+    llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &symbolAndClause) {
   const auto &spec =
       std::get<parser::OmpDeclareTargetSpecifier>(declareTargetConstruct.t);
   if (const auto *objectList{parser::Unwrap<parser::OmpObjectList>(spec.u)}) {
     ObjectList objects{makeObjects(*objectList, semaCtx)};
     // Case: declare target(func, var1, var2)
     gatherFuncAndVarSyms(objects, mlir::omp::DeclareTargetCaptureClause::to,
-                         symbolAndClause);
+                         symbolAndClause, /*automap=*/false);
   } else if (const auto *clauseList{
                  parser::Unwrap<parser::OmpClauseList>(spec.u)}) {
     List<Clause> clauses = makeClauses(*clauseList, semaCtx);
@@ -804,21 +805,20 @@ static void collectDeferredDeclareTargets(
     llvm::SmallVectorImpl<lower::OMPDeferredDeclareTargetInfo>
         &deferredDeclareTarget) {
   mlir::omp::DeclareTargetOperands clauseOps;
-  llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
+  llvm::SmallVector<DeclareTargetCaptureInfo> symbolAndClause;
   getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
                        clauseOps, symbolAndClause);
   // Return the device type only if at least one of the targets for the
   // directive is a function or subroutine
   mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
 
-  for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
-    mlir::Operation *op = mod.lookupSymbol(
-        converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
+  for (const DeclareTargetCaptureInfo &symClause : symbolAndClause) {
+    mlir::Operation *op =
+        mod.lookupSymbol(converter.mangleName(symClause.symbol));
 
     if (!op) {
-      deferredDeclareTarget.push_back({std::get<0>(symClause),
-                                       clauseOps.deviceType,
-                                       std::get<1>(symClause)});
+      deferredDeclareTarget.push_back({symClause.clause, clauseOps.deviceType,
+                                       symClause.automap, symClause.symbol});
     }
   }
 }
@@ -829,16 +829,16 @@ getDeclareTargetFunctionDevice(
     lower::pft::Evaluation &eval,
     const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {
   mlir::omp::DeclareTargetOperands clauseOps;
-  llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
+  llvm::SmallVector<DeclareTargetCaptureInfo> symbolAndClause;
   getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
                        clauseOps, symbolAndClause);
 
   // Return the device type only if at least one of the targets for the
   // directive is a function or subroutine
   mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
-  for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
-    mlir::Operation *op = mod.lookupSymbol(
-        converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
+  for (const DeclareTargetCaptureInfo &symClause : symbolAndClause) {
+    mlir::Operation *op =
+        mod.lookupSymbol(converter.mangleName(symClause.symbol));
 
     if (mlir::isa_and_nonnull<mlir::func::FuncOp>(op))
       return clauseOps.deviceType;
@@ -1055,7 +1055,7 @@ getImplicitMapTypeAndKind(fir::FirOpBuilder &firOpBuilder,
 static void
 markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
                   mlir::omp::DeclareTargetCaptureClause captureClause,
-                  mlir::omp::DeclareTargetDeviceType deviceType) {
+                  mlir::omp::DeclareTargetDeviceType deviceType, bool automap) {
   // TODO: Add support for program local variables with declare target applied
   auto declareTargetOp = llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(op);
   if (!declareTargetOp)
@@ -1070,11 +1070,11 @@ markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
   if (declareTargetOp.isDeclareTarget()) {
     if (declareTargetOp.getDeclareTargetDeviceType() != deviceType)
       declareTargetOp.setDeclareTarget(mlir::omp::DeclareTargetDeviceType::any,
-                                       captureClause);
+                                       captureClause, automap);
     return;
   }
 
-  declareTargetOp.setDeclareTarget(deviceType, captureClause);
+  declareTargetOp.setDeclareTarget(deviceType, captureClause, automap);
 }
 
 //===----------------------------------------------------------------------===//
@@ -2262,7 +2262,8 @@ genOrderedOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
              semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
              mlir::Location loc, const ConstructQueue &queue,
              ConstructQueue::const_iterator item) {
-  TODO(loc, "OMPD_ordered");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "OMPD_ordered");
   return nullptr;
 }
 
@@ -2449,7 +2450,8 @@ genScopeOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
            semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
            mlir::Location loc, const ConstructQueue &queue,
            ConstructQueue::const_iterator item) {
-  TODO(loc, "Scope construct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "Scope construct");
   return nullptr;
 }
 
@@ -3276,7 +3278,8 @@ static mlir::omp::TaskloopOp genCompositeTaskloopSimd(
     lower::pft::Evaluation &eval, mlir::Location loc,
     const ConstructQueue &queue, ConstructQueue::const_iterator item) {
   assert(std::distance(item, queue.end()) == 2 && "Invalid leaf constructs");
-  TODO(loc, "Composite TASKLOOP SIMD");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(loc, "Composite TASKLOOP SIMD");
   return nullptr;
 }
 
@@ -3448,8 +3451,10 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
     break;
   case llvm::omp::Directive::OMPD_tile: {
     unsigned version = semaCtx.langOptions().OpenMPVersion;
-    TODO(loc, "Unhandled loop directive (" +
-                  llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
+    if (!semaCtx.langOptions().OpenMPSimd)
+      TODO(loc, "Unhandled loop directive (" +
+                    llvm::omp::getOpenMPDirectiveName(dir, version) + ")");
+    break;
   }
   case llvm::omp::Directive::OMPD_unroll:
     genUnrollOp(converter, symTable, stmtCtx, semaCtx, eval, loc, queue, item);
@@ -3484,35 +3489,40 @@ static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OpenMPDeclarativeAllocate &declarativeAllocate) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclarativeAllocate");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPDeclarativeAssumes &assumesConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMP ASSUMES declaration");
 }
 
 static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OmpDeclareVariantDirective &declareVariantDirective) {
-  TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OmpDeclareVariantDirective");
 }
 
 static void genOMP(
     lower::AbstractConverter &converter, lower::SymMap &symTable,
     semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
     const parser::OpenMPDeclareReductionConstruct &declareReductionConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclareReductionConstruct");
 }
 
 static void
 genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OpenMPDeclareSimdConstruct &declareSimdConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDeclareSimdConstruct");
 }
 
 static void
@@ -3563,14 +3573,14 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
        semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
        const parser::OpenMPDeclareTargetConstruct &declareTargetConstruct) {
   mlir::omp::DeclareTargetOperands clauseOps;
-  llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
+  llvm::SmallVector<DeclareTargetCaptureInfo> symbolAndClause;
   mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
   getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
                        clauseOps, symbolAndClause);
 
-  for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
-    mlir::Operation *op = mod.lookupSymbol(
-        converter.mangleName(std::get<const semantics::Symbol &>(symClause)));
+  for (const DeclareTargetCaptureInfo &symClause : symbolAndClause) {
+    mlir::Operation *op =
+        mod.lookupSymbol(converter.mangleName(symClause.symbol));
 
     // Some symbols are deferred until later in the module, these are handled
     // upon finalization of the module for OpenMP inside of Bridge, so we simply
@@ -3578,13 +3588,18 @@ genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
     if (!op)
       continue;
 
-    markDeclareTarget(
-        op, converter,
-        std::get<mlir::omp::DeclareTargetCaptureClause>(symClause),
-        clauseOps.deviceType);
+    markDeclareTarget(op, converter, symClause.clause, clauseOps.deviceType,
+                      symClause.automap);
   }
 }
 
+static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
+                   semantics::SemanticsContext &semaCtx,
+                   lower::pft::Evaluation &eval,
+                   const parser::OpenMPGroupprivate &directive) {
+  TODO(converter.getCurrentLocation(), "GROUPPRIVATE");
+}
+
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
@@ -3708,14 +3723,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   (void)objects;
   (void)clauses;
 
-  TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDepobjConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPInteropConstruct &interopConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPInteropConstruct");
 }
 
 static void
@@ -3731,7 +3748,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAllocatorsConstruct &allocsConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPAllocatorsConstruct");
 }
 
 //===----------------------------------------------------------------------===//
@@ -3797,7 +3815,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
         !std::holds_alternative<clause::Detach>(clause.u)) {
       std::string name =
           parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(clause.id));
-      TODO(clauseLocation, name + " clause is not implemented yet");
+      if (!semaCtx.langOptions().OpenMPSimd)
+        TODO(clauseLocation, name + " clause is not implemented yet");
     }
   }
 
@@ -3813,46 +3832,61 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAssumeConstruct &assumeConstruct) {
   mlir::Location clauseLocation = converter.genLocation(assumeConstruct.source);
-  TODO(clauseLocation, "OpenMP ASSUME construct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(clauseLocation, "OpenMP ASSUME construct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPCriticalConstruct &criticalConstruct) {
-  const auto &cd = std::get<parser::OmpCriticalDirective>(criticalConstruct.t);
-  List<Clause> clauses =
-      makeClauses(std::get<parser::OmpClauseList>(cd.t), semaCtx);
+  const parser::OmpDirectiveSpecification &beginSpec =
+      criticalConstruct.BeginDir();
+  List<Clause> clauses = makeClauses(beginSpec.Clauses(), semaCtx);
 
   ConstructQueue queue{buildConstructQueue(
-      converter.getFirOpBuilder().getModule(), semaCtx, eval, cd.source,
+      converter.getFirOpBuilder().getModule(), semaCtx, eval, beginSpec.source,
       llvm::omp::Directive::OMPD_critical, clauses)};
 
-  const auto &name = std::get<std::optional<parser::Name>>(cd.t);
+  std::optional<parser::Name> critName;
+  const parser::OmpArgumentList &args = beginSpec.Arguments();
+  if (!args.v.empty()) {
+    // All of these things should be guaranteed to exist after semantic checks.
+    auto *object = parser::Unwrap<parser::OmpObject>(args.v.front());
+    assert(object && "Expecting object as argument");
+    auto *designator = semantics::omp::GetDesignatorFromObj(*object);
+    assert(designator && "Expecting desginator in argument");
+    auto *name = semantics::getDesignatorNameIfDataRef(*designator);
+    assert(name && "Expecting dataref in designator");
+    critName = *name;
+  }
   mlir::Location currentLocation = converter.getCurrentLocation();
   genCriticalOp(converter, symTable, semaCtx, eval, currentLocation, queue,
-                queue.begin(), name);
+                queue.begin(), critName);
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPUtilityConstruct &) {
-  TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPUtilityConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPDispatchConstruct &) {
-  TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPDispatchConstruct");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPExecutableAllocate &execAllocConstruct) {
-  TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
+  if (!semaCtx.langOptions().OpenMPSimd)
+    TODO(converter.getCurrentLocation(), "OpenMPExecutableAllocate");
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
@@ -4164,7 +4198,7 @@ bool Fortran::lower::markOpenMPDeferredDeclareTargetFunctions(
       deviceCodeFound = true;
 
     markDeclareTarget(op, converter, declTar.declareTargetCaptureClause,
-                      devType);
+                      devType, declTar.automap);
   }
 
   return deviceCodeFound;
diff --git a/flang/lib/Lower/OpenMP/Utils.cpp b/flang/lib/Lower/OpenMP/Utils.cpp
index 13fda978c5369..77b1e39083aa6 100644
--- a/flang/lib/Lower/OpenMP/Utils.cpp
+++ b/flang/lib/Lower/OpenMP/Utils.cpp
@@ -102,9 +102,10 @@ getIterationVariableSymbol(const lower::pft::Evaluation &eval) {
 
 void gatherFuncAndVarSyms(
     const ObjectList &objects, mlir::omp::DeclareTargetCaptureClause clause,
-    llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause) {
+    llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &symbolAndClause,
+    bool automap) {
   for (const Object &object : objects)
-    symbolAndClause.emplace_back(clause, *object.sym());
+    symbolAndClause.emplace_back(clause, *object.sym(), automap);
 }
 
 mlir::omp::MapInfoOp
diff --git a/flang/lib/Lower/OpenMP/Utils.h b/flang/lib/Lower/OpenMP/Utils.h
index 11641ba5e8606..60f44a7f0610c 100644
--- a/flang/lib/Lower/OpenMP/Utils.h
+++ b/flang/lib/Lower/OpenMP/Utils.h
@@ -42,8 +42,15 @@ class AbstractConverter;
 
 namespace omp {
 
-using DeclareTargetCapturePair =
-    std::pair<mlir::omp::DeclareTargetCaptureClause, const semantics::Symbol &>;
+struct DeclareTargetCaptureInfo {
+  mlir::omp::DeclareTargetCaptureClause clause;
+  bool automap = false;
+  const semantics::Symbol &symbol;
+
+  DeclareTargetCaptureInfo(mlir::omp::DeclareTargetCaptureClause c,
+                           const semantics::Symbol &s, bool a = false)
+      : clause(c), automap(a), symbol(s) {}
+};
 
 // A small helper structure for keeping track of a component members MapInfoOp
 // and index data when lowering OpenMP map clauses. Keeps track of the
@@ -150,7 +157,8 @@ getIterationVariableSymbol(const lower::pft::Evaluation &eval);
 
 void gatherFuncAndVarSyms(
     const ObjectList &objects, mlir::omp::DeclareTargetCaptureClause clause,
-    llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause);
+    llvm::SmallVectorImpl<DeclareTargetCaptureInfo> &symbolAndClause,
+    bool automap = false);
 
 int64_t getCollapseValue(const List<Clause> &clauses);
 
diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp
index a28cc019a7310..80f31c2be91a8 100644
--- a/flang/lib/Lower/PFTBuilder.cpp
+++ b/flang/lib/Lower/PFTBuilder.cpp
@@ -1742,11 +1742,11 @@ struct SymbolDependenceAnalysis {
                                layeredVarList[i].end());
   }
 
-  llvm::SmallSet<const semantics::Symbol *, 32> seen;
+  llvm::SmallPtrSet<const semantics::Symbol *, 32> seen;
   std::vector<Fortran::lower::pft::VariableList> layeredVarList;
-  llvm::SmallSet<const semantics::Symbol *, 32> aliasSyms;
+  llvm::SmallPtrSet<const semantics::Symbol *, 32> aliasSyms;
   /// Set of scopes that have been analyzed for aliases.
-  llvm::SmallSet<const semantics::Scope *, 4> analyzedScopes;
+  llvm::SmallPtrSet<const semantics::Scope *, 4> analyzedScopes;
   std::vector<Fortran::lower::pft::Variable::AggregateStore> stores;
 };
 } // namespace
diff --git a/flang/lib/Lower/Support/PrivateReductionUtils.cpp b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
index fff060b79c9fe..1b09801c607c6 100644
--- a/flang/lib/Lower/Support/PrivateReductionUtils.cpp
+++ b/flang/lib/Lower/Support/PrivateReductionUtils.cpp
@@ -616,6 +616,8 @@ void PopulateInitAndCleanupRegionsHelper::populateByRefInitAndCleanupRegions() {
     assert(sym && "Symbol information is required to privatize derived types");
     assert(!scalarInitValue && "ScalarInitvalue is unused for privatization");
   }
+  if (hlfir::Entity{moldArg}.isAssumedRank())
+    TODO(loc, "Privatization of assumed rank variable");
   mlir::Type valTy = fir::unwrapRefType(argType);
 
   if (fir::isa_trivial(valTy)) {
diff --git a/flang/lib/Lower/Support/Utils.cpp b/flang/lib/Lower/Support/Utils.cpp
index 881401e11fee4..0cdb03beb72a2 100644
--- a/flang/lib/Lower/Support/Utils.cpp
+++ b/flang/lib/Lower/Support/Utils.cpp
@@ -654,7 +654,7 @@ void privatizeSymbol(
     lower::AbstractConverter &converter, fir::FirOpBuilder &firOpBuilder,
     lower::SymMap &symTable,
     llvm::SetVector<const semantics::Symbol *> &allPrivatizedSymbols,
-    llvm::SmallSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
+    llvm::SmallPtrSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
     const semantics::Symbol *symToPrivatize, OperandsStructType *clauseOps) {
   constexpr bool isDoConcurrent =
       std::is_same_v<OpType, fir::LocalitySpecifierOp>;
@@ -846,7 +846,7 @@ privatizeSymbol<mlir::omp::PrivateClauseOp, mlir::omp::PrivateClauseOps>(
     lower::AbstractConverter &converter, fir::FirOpBuilder &firOpBuilder,
     lower::SymMap &symTable,
     llvm::SetVector<const semantics::Symbol *> &allPrivatizedSymbols,
-    llvm::SmallSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
+    llvm::SmallPtrSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
     const semantics::Symbol *symToPrivatize,
     mlir::omp::PrivateClauseOps *clauseOps);
 
@@ -855,7 +855,7 @@ privatizeSymbol<fir::LocalitySpecifierOp, fir::LocalitySpecifierOperands>(
     lower::AbstractConverter &converter, fir::FirOpBuilder &firOpBuilder,
     lower::SymMap &symTable,
     llvm::SetVector<const semantics::Symbol *> &allPrivatizedSymbols,
-    llvm::SmallSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
+    llvm::SmallPtrSet<const semantics::Symbol *, 16> &mightHaveReadHostSym,
     const semantics::Symbol *symToPrivatize,
     fir::LocalitySpecifierOperands *clauseOps);
 
diff --git a/flang/lib/Optimizer/Builder/CMakeLists.txt b/flang/lib/Optimizer/Builder/CMakeLists.txt
index 31ae395805faf..8fb36a750d433 100644
--- a/flang/lib/Optimizer/Builder/CMakeLists.txt
+++ b/flang/lib/Optimizer/Builder/CMakeLists.txt
@@ -16,6 +16,7 @@ add_flang_library(FIRBuilder
   Runtime/Allocatable.cpp
   Runtime/ArrayConstructor.cpp
   Runtime/Assign.cpp
+  Runtime/Coarray.cpp
   Runtime/Character.cpp
   Runtime/Command.cpp
   Runtime/CUDA/Descriptor.cpp
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 87a52ffc339f4..99533690018eb 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -147,8 +147,20 @@ mlir::Value fir::FirOpBuilder::createIntegerConstant(mlir::Location loc,
   assert((cst >= 0 || mlir::isa<mlir::IndexType>(ty) ||
           mlir::cast<mlir::IntegerType>(ty).getWidth() <= 64) &&
          "must use APint");
-  return mlir::arith::ConstantOp::create(*this, loc, ty,
-                                         getIntegerAttr(ty, cst));
+
+  mlir::Type cstType = ty;
+  if (auto intType = mlir::dyn_cast<mlir::IntegerType>(ty)) {
+    // Signed and unsigned constants must be encoded as signless
+    // arith.constant followed by fir.convert cast.
+    if (intType.isUnsigned())
+      cstType = mlir::IntegerType::get(getContext(), intType.getWidth());
+    else if (intType.isSigned())
+      TODO(loc, "signed integer constant");
+  }
+
+  mlir::Value cstValue = mlir::arith::ConstantOp::create(
+      *this, loc, cstType, getIntegerAttr(cstType, cst));
+  return createConvert(loc, ty, cstValue);
 }
 
 mlir::Value fir::FirOpBuilder::createAllOnesInteger(mlir::Location loc,
@@ -1947,17 +1959,17 @@ void fir::factory::genDimInfoFromBox(
 
 mlir::Value fir::factory::genLifetimeStart(mlir::OpBuilder &builder,
                                            mlir::Location loc,
-                                           fir::AllocaOp alloc, int64_t size,
+                                           fir::AllocaOp alloc,
                                            const mlir::DataLayout *dl) {
   mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(
       alloc.getContext(), getAllocaAddressSpace(dl));
   mlir::Value cast =
       fir::ConvertOp::create(builder, loc, ptrTy, alloc.getResult());
-  mlir::LLVM::LifetimeStartOp::create(builder, loc, size, cast);
+  mlir::LLVM::LifetimeStartOp::create(builder, loc, cast);
   return cast;
 }
 
 void fir::factory::genLifetimeEnd(mlir::OpBuilder &builder, mlir::Location loc,
-                                  mlir::Value cast, int64_t size) {
-  mlir::LLVM::LifetimeEndOp::create(builder, loc, size, cast);
+                                  mlir::Value cast) {
+  mlir::LLVM::LifetimeEndOp::create(builder, loc, cast);
 }
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index b6d692a0226cd..086dd66711602 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -416,7 +416,10 @@ hlfir::Entity hlfir::loadTrivialScalar(mlir::Location loc,
   entity = derefPointersAndAllocatables(loc, builder, entity);
   if (entity.isVariable() && entity.isScalar() &&
       fir::isa_trivial(entity.getFortranElementType())) {
-    return Entity{fir::LoadOp::create(builder, loc, entity)};
+    // Optional entities may be represented with !fir.box<i32/f32/...>.
+    // We need to take the data pointer before loading the scalar.
+    mlir::Value base = genVariableRawAddress(loc, builder, entity);
+    return Entity{fir::LoadOp::create(builder, loc, base)};
   }
   return entity;
 }
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index bfa470dd5690d..7e987de94c5d5 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -25,6 +25,7 @@
 #include "flang/Optimizer/Builder/Runtime/Allocatable.h"
 #include "flang/Optimizer/Builder/Runtime/CUDA/Descriptor.h"
 #include "flang/Optimizer/Builder/Runtime/Character.h"
+#include "flang/Optimizer/Builder/Runtime/Coarray.h"
 #include "flang/Optimizer/Builder/Runtime/Command.h"
 #include "flang/Optimizer/Builder/Runtime/Derived.h"
 #include "flang/Optimizer/Builder/Runtime/Exceptions.h"
@@ -778,6 +779,10 @@ static constexpr IntrinsicHandler handlers[]{
      /*isElemental=*/false},
     {"not", &I::genNot},
     {"null", &I::genNull, {{{"mold", asInquired}}}, /*isElemental=*/false},
+    {"num_images",
+     &I::genNumImages,
+     {{{"team", asAddr}, {"team_number", asAddr}}},
+     /*isElemental*/ false},
     {"pack",
      &I::genPack,
      {{{"array", asBox},
@@ -947,6 +952,12 @@ static constexpr IntrinsicHandler handlers[]{
     {"tand", &I::genTand},
     {"tanpi", &I::genTanpi},
     {"this_grid", &I::genThisGrid, {}, /*isElemental=*/false},
+    {"this_image",
+     &I::genThisImage,
+     {{{"coarray", asBox},
+       {"dim", asAddr},
+       {"team", asBox, handleDynamicOptional}}},
+     /*isElemental=*/false},
     {"this_thread_block", &I::genThisThreadBlock, {}, /*isElemental=*/false},
     {"this_warp", &I::genThisWarp, {}, /*isElemental=*/false},
     {"threadfence", &I::genThreadFence, {}, /*isElemental=*/false},
@@ -1276,6 +1287,26 @@ mlir::Value genComplexMathOp(fir::FirOpBuilder &builder, mlir::Location loc,
   return result;
 }
 
+mlir::Value genComplexPow(fir::FirOpBuilder &builder, mlir::Location loc,
+                          const MathOperation &mathOp,
+                          mlir::FunctionType mathLibFuncType,
+                          llvm::ArrayRef<mlir::Value> args) {
+  bool isAMDGPU = fir::getTargetTriple(builder.getModule()).isAMDGCN();
+  if (!isAMDGPU)
+    return genLibCall(builder, loc, mathOp, mathLibFuncType, args);
+
+  auto complexTy = mlir::cast<mlir::ComplexType>(mathLibFuncType.getInput(0));
+  auto realTy = complexTy.getElementType();
+  mlir::Value realExp = builder.createConvert(loc, realTy, args[1]);
+  mlir::Value zero = builder.createRealConstant(loc, realTy, 0);
+  mlir::Value complexExp =
+      builder.create<mlir::complex::CreateOp>(loc, complexTy, realExp, zero);
+  mlir::Value result =
+      builder.create<mlir::complex::PowOp>(loc, args[0], complexExp);
+  result = builder.createConvert(loc, mathLibFuncType.getResult(0), result);
+  return result;
+}
+
 /// Mapping between mathematical intrinsic operations and MLIR operations
 /// of some appropriate dialect (math, complex, etc.) or libm calls.
 /// TODO: support remaining Fortran math intrinsics.
@@ -1625,15 +1656,19 @@ static constexpr MathOperation mathOperations[] = {
      genFuncType<Ty::Real<16>, Ty::Real<16>, Ty::Integer<8>>,
      genMathOp<mlir::math::FPowIOp>},
     {"pow", RTNAME_STRING(cpowi),
-     genFuncType<Ty::Complex<4>, Ty::Complex<4>, Ty::Integer<4>>, genLibCall},
+     genFuncType<Ty::Complex<4>, Ty::Complex<4>, Ty::Integer<4>>,
+     genComplexPow},
     {"pow", RTNAME_STRING(zpowi),
-     genFuncType<Ty::Complex<8>, Ty::Complex<8>, Ty::Integer<4>>, genLibCall},
+     genFuncType<Ty::Complex<8>, Ty::Complex<8>, Ty::Integer<4>>,
+     genComplexPow},
     {"pow", RTNAME_STRING(cqpowi), FuncTypeComplex16Complex16Integer4,
      genLibF128Call},
     {"pow", RTNAME_STRING(cpowk),
-     genFuncType<Ty::Complex<4>, Ty::Complex<4>, Ty::Integer<8>>, genLibCall},
+     genFuncType<Ty::Complex<4>, Ty::Complex<4>, Ty::Integer<8>>,
+     genComplexPow},
     {"pow", RTNAME_STRING(zpowk),
-     genFuncType<Ty::Complex<8>, Ty::Complex<8>, Ty::Integer<8>>, genLibCall},
+     genFuncType<Ty::Complex<8>, Ty::Complex<8>, Ty::Integer<8>>,
+     genComplexPow},
     {"pow", RTNAME_STRING(cqpowk), FuncTypeComplex16Complex16Integer8,
      genLibF128Call},
     {"remainder", "remainderf",
@@ -2672,10 +2707,11 @@ mlir::Value IntrinsicLibrary::genAcosd(mlir::Type resultType,
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
   mlir::Value result =
       getRuntimeCallGenerator("acos", ftype)(builder, loc, {args[0]});
-  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
-  mlir::Value dfactor = builder.createRealConstant(
-      loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
-  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  const llvm::fltSemantics &fltSem =
+      llvm::cast<mlir::FloatType>(resultType).getFloatSemantics();
+  llvm::APFloat pi = llvm::APFloat(fltSem, llvm::numbers::pis);
+  mlir::Value factor = builder.createRealConstant(
+      loc, resultType, llvm::APFloat(fltSem, "180.0") / pi);
   return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
@@ -2687,10 +2723,10 @@ mlir::Value IntrinsicLibrary::genAcospi(mlir::Type resultType,
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
   mlir::Value acos = getRuntimeCallGenerator("acos", ftype)(builder, loc, args);
-  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
-  mlir::Value dfactor =
-      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
-  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  llvm::APFloat inv_pi =
+      llvm::APFloat(llvm::cast<mlir::FloatType>(resultType).getFloatSemantics(),
+                    llvm::numbers::inv_pis);
+  mlir::Value factor = builder.createRealConstant(loc, resultType, inv_pi);
   return mlir::arith::MulFOp::create(builder, loc, acos, factor);
 }
 
@@ -2840,10 +2876,11 @@ mlir::Value IntrinsicLibrary::genAsind(mlir::Type resultType,
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
   mlir::Value result =
       getRuntimeCallGenerator("asin", ftype)(builder, loc, {args[0]});
-  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
-  mlir::Value dfactor = builder.createRealConstant(
-      loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
-  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  const llvm::fltSemantics &fltSem =
+      llvm::cast<mlir::FloatType>(resultType).getFloatSemantics();
+  llvm::APFloat pi = llvm::APFloat(fltSem, llvm::numbers::pis);
+  mlir::Value factor = builder.createRealConstant(
+      loc, resultType, llvm::APFloat(fltSem, "180.0") / pi);
   return mlir::arith::MulFOp::create(builder, loc, result, factor);
 }
 
@@ -2855,10 +2892,10 @@ mlir::Value IntrinsicLibrary::genAsinpi(mlir::Type resultType,
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
   mlir::Value asin = getRuntimeCallGenerator("asin", ftype)(builder, loc, args);
-  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
-  mlir::Value dfactor =
-      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
-  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  llvm::APFloat inv_pi =
+      llvm::APFloat(llvm::cast<mlir::FloatType>(resultType).getFloatSemantics(),
+                    llvm::numbers::inv_pis);
+  mlir::Value factor = builder.createRealConstant(loc, resultType, inv_pi);
   return mlir::arith::MulFOp::create(builder, loc, asin, factor);
 }
 
@@ -2880,10 +2917,11 @@ mlir::Value IntrinsicLibrary::genAtand(mlir::Type resultType,
         mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
     atan = getRuntimeCallGenerator("atan", ftype)(builder, loc, args);
   }
-  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
-  mlir::Value dfactor = builder.createRealConstant(
-      loc, mlir::Float64Type::get(context), llvm::APFloat(180.0) / pi);
-  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  const llvm::fltSemantics &fltSem =
+      llvm::cast<mlir::FloatType>(resultType).getFloatSemantics();
+  llvm::APFloat pi = llvm::APFloat(fltSem, llvm::numbers::pis);
+  mlir::Value factor = builder.createRealConstant(
+      loc, resultType, llvm::APFloat(fltSem, "180.0") / pi);
   return mlir::arith::MulFOp::create(builder, loc, atan, factor);
 }
 
@@ -2905,10 +2943,10 @@ mlir::Value IntrinsicLibrary::genAtanpi(mlir::Type resultType,
         mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
     atan = getRuntimeCallGenerator("atan", ftype)(builder, loc, args);
   }
-  llvm::APFloat inv_pi = llvm::APFloat(llvm::numbers::inv_pi);
-  mlir::Value dfactor =
-      builder.createRealConstant(loc, mlir::Float64Type::get(context), inv_pi);
-  mlir::Value factor = builder.createConvert(loc, resultType, dfactor);
+  llvm::APFloat inv_pi =
+      llvm::APFloat(llvm::cast<mlir::FloatType>(resultType).getFloatSemantics(),
+                    llvm::numbers::inv_pis);
+  mlir::Value factor = builder.createRealConstant(loc, resultType, inv_pi);
   return mlir::arith::MulFOp::create(builder, loc, atan, factor);
 }
 
@@ -3684,10 +3722,10 @@ mlir::Value IntrinsicLibrary::genCospi(mlir::Type resultType,
   mlir::MLIRContext *context = builder.getContext();
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
-  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
-  mlir::Value dfactor =
-      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
-  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  llvm::APFloat pi =
+      llvm::APFloat(llvm::cast<mlir::FloatType>(resultType).getFloatSemantics(),
+                    llvm::numbers::pis);
+  mlir::Value factor = builder.createRealConstant(loc, resultType, pi);
   mlir::Value arg = mlir::arith::MulFOp::create(builder, loc, args[0], factor);
   return getRuntimeCallGenerator("cos", ftype)(builder, loc, {arg});
 }
@@ -4031,21 +4069,20 @@ void IntrinsicLibrary::genExecuteCommandLine(
     mlir::Value waitAddr = fir::getBase(wait);
     mlir::Value waitIsPresentAtRuntime =
         builder.genIsNotNullAddr(loc, waitAddr);
-    waitBool = builder
-                   .genIfOp(loc, {i1Ty}, waitIsPresentAtRuntime,
-                            /*withElseRegion=*/true)
-                   .genThen([&]() {
-                     auto waitLoad =
-                         fir::LoadOp::create(builder, loc, waitAddr);
-                     mlir::Value cast =
-                         builder.createConvert(loc, i1Ty, waitLoad);
-                     fir::ResultOp::create(builder, loc, cast);
-                   })
-                   .genElse([&]() {
-                     mlir::Value trueVal = builder.createBool(loc, true);
-                     fir::ResultOp::create(builder, loc, trueVal);
-                   })
-                   .getResults()[0];
+    waitBool =
+        builder
+            .genIfOp(loc, {i1Ty}, waitIsPresentAtRuntime,
+                     /*withElseRegion=*/true)
+            .genThen([&]() {
+              auto waitLoad = fir::LoadOp::create(builder, loc, waitAddr);
+              mlir::Value cast = builder.createConvert(loc, i1Ty, waitLoad);
+              fir::ResultOp::create(builder, loc, cast);
+            })
+            .genElse([&]() {
+              mlir::Value trueVal = builder.createBool(loc, true);
+              fir::ResultOp::create(builder, loc, trueVal);
+            })
+            .getResults()[0];
   }
 
   mlir::Value exitstatBox =
@@ -7277,6 +7314,19 @@ IntrinsicLibrary::genNull(mlir::Type, llvm::ArrayRef<fir::ExtendedValue> args) {
   return fir::MutableBoxValue(boxStorage, mold->nonDeferredLenParams(), {});
 }
 
+// NUM_IMAGES
+fir::ExtendedValue
+IntrinsicLibrary::genNumImages(mlir::Type resultType,
+                               llvm::ArrayRef<fir::ExtendedValue> args) {
+  checkCoarrayEnabled();
+  assert(args.size() == 0 || args.size() == 1);
+
+  if (args.size())
+    return fir::runtime::getNumImagesWithTeam(builder, loc,
+                                              fir::getBase(args[0]));
+  return fir::runtime::getNumImages(builder, loc);
+}
+
 // CLOCK, CLOCK64, GLOBALTIMER
 template <typename OpTy>
 mlir::Value IntrinsicLibrary::genNVVMTime(mlir::Type resultType,
@@ -8136,10 +8186,10 @@ mlir::Value IntrinsicLibrary::genSinpi(mlir::Type resultType,
   mlir::MLIRContext *context = builder.getContext();
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
-  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
-  mlir::Value dfactor =
-      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
-  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  llvm::APFloat pi =
+      llvm::APFloat(llvm::cast<mlir::FloatType>(resultType).getFloatSemantics(),
+                    llvm::numbers::pis);
+  mlir::Value factor = builder.createRealConstant(loc, resultType, pi);
   mlir::Value arg = mlir::arith::MulFOp::create(builder, loc, args[0], factor);
   return getRuntimeCallGenerator("sin", ftype)(builder, loc, {arg});
 }
@@ -8233,10 +8283,10 @@ mlir::Value IntrinsicLibrary::genTanpi(mlir::Type resultType,
   mlir::MLIRContext *context = builder.getContext();
   mlir::FunctionType ftype =
       mlir::FunctionType::get(context, {resultType}, {args[0].getType()});
-  llvm::APFloat pi = llvm::APFloat(llvm::numbers::pi);
-  mlir::Value dfactor =
-      builder.createRealConstant(loc, mlir::Float64Type::get(context), pi);
-  mlir::Value factor = builder.createConvert(loc, args[0].getType(), dfactor);
+  llvm::APFloat pi =
+      llvm::APFloat(llvm::cast<mlir::FloatType>(resultType).getFloatSemantics(),
+                    llvm::numbers::pis);
+  mlir::Value factor = builder.createRealConstant(loc, resultType, pi);
   mlir::Value arg = mlir::arith::MulFOp::create(builder, loc, args[0], factor);
   return getRuntimeCallGenerator("tan", ftype)(builder, loc, {arg});
 }
@@ -8327,6 +8377,27 @@ mlir::Value IntrinsicLibrary::genThisGrid(mlir::Type resultType,
   return res;
 }
 
+// THIS_IMAGE
+fir::ExtendedValue
+IntrinsicLibrary::genThisImage(mlir::Type resultType,
+                               llvm::ArrayRef<fir::ExtendedValue> args) {
+  checkCoarrayEnabled();
+  assert(args.size() >= 1 && args.size() <= 3);
+  const bool coarrayIsAbsent = args.size() == 1;
+  mlir::Value team =
+      !isStaticallyAbsent(args, args.size() - 1)
+          ? fir::getBase(args[args.size() - 1])
+          : builder
+                .create<fir::AbsentOp>(loc,
+                                       fir::BoxType::get(builder.getNoneType()))
+                .getResult();
+
+  if (!coarrayIsAbsent)
+    TODO(loc, "this_image with coarray argument.");
+  mlir::Value res = fir::runtime::getThisImage(builder, loc, team);
+  return builder.createConvert(loc, resultType, res);
+}
+
 // THIS_THREAD_BLOCK
 mlir::Value
 IntrinsicLibrary::genThisThreadBlock(mlir::Type resultType,
diff --git a/flang/lib/Optimizer/Builder/Runtime/Coarray.cpp b/flang/lib/Optimizer/Builder/Runtime/Coarray.cpp
new file mode 100644
index 0000000000000..fb72fc2089e23
--- /dev/null
+++ b/flang/lib/Optimizer/Builder/Runtime/Coarray.cpp
@@ -0,0 +1,86 @@
+//===-- Coarray.cpp -- runtime API for coarray intrinsics -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Builder/Runtime/Coarray.h"
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+
+using namespace Fortran::runtime;
+using namespace Fortran::semantics;
+
+/// Generate Call to runtime prif_init
+mlir::Value fir::runtime::genInitCoarray(fir::FirOpBuilder &builder,
+                                         mlir::Location loc) {
+  mlir::Type i32Ty = builder.getI32Type();
+  mlir::Value result = builder.createTemporary(loc, i32Ty);
+  mlir::FunctionType ftype = PRIF_FUNCTYPE(builder.getRefType(i32Ty));
+  mlir::func::FuncOp funcOp =
+      builder.createFunction(loc, PRIFNAME_SUB("init"), ftype);
+  llvm::SmallVector<mlir::Value> args =
+      fir::runtime::createArguments(builder, loc, ftype, result);
+  builder.create<fir::CallOp>(loc, funcOp, args);
+  return builder.create<fir::LoadOp>(loc, result);
+}
+
+/// Generate Call to runtime prif_num_images
+mlir::Value fir::runtime::getNumImages(fir::FirOpBuilder &builder,
+                                       mlir::Location loc) {
+  mlir::Value result = builder.createTemporary(loc, builder.getI32Type());
+  mlir::FunctionType ftype =
+      PRIF_FUNCTYPE(builder.getRefType(builder.getI32Type()));
+  mlir::func::FuncOp funcOp =
+      builder.createFunction(loc, PRIFNAME_SUB("num_images"), ftype);
+  llvm::SmallVector<mlir::Value> args =
+      fir::runtime::createArguments(builder, loc, ftype, result);
+  builder.create<fir::CallOp>(loc, funcOp, args);
+  return builder.create<fir::LoadOp>(loc, result);
+}
+
+/// Generate Call to runtime prif_num_images_with_{team|team_number}
+mlir::Value fir::runtime::getNumImagesWithTeam(fir::FirOpBuilder &builder,
+                                               mlir::Location loc,
+                                               mlir::Value team) {
+  bool isTeamNumber = fir::unwrapPassByRefType(team.getType()).isInteger();
+  std::string numImagesName = isTeamNumber
+                                  ? PRIFNAME_SUB("num_images_with_team_number")
+                                  : PRIFNAME_SUB("num_images_with_team");
+
+  mlir::Value result = builder.createTemporary(loc, builder.getI32Type());
+  mlir::Type refTy = builder.getRefType(builder.getI32Type());
+  mlir::FunctionType ftype =
+      isTeamNumber
+          ? PRIF_FUNCTYPE(builder.getRefType(builder.getI64Type()), refTy)
+          : PRIF_FUNCTYPE(fir::BoxType::get(builder.getNoneType()), refTy);
+  mlir::func::FuncOp funcOp = builder.createFunction(loc, numImagesName, ftype);
+
+  if (!isTeamNumber)
+    team = builder.createBox(loc, team);
+  llvm::SmallVector<mlir::Value> args =
+      fir::runtime::createArguments(builder, loc, ftype, team, result);
+  builder.create<fir::CallOp>(loc, funcOp, args);
+  return builder.create<fir::LoadOp>(loc, result);
+}
+
+/// Generate Call to runtime prif_this_image_no_coarray
+mlir::Value fir::runtime::getThisImage(fir::FirOpBuilder &builder,
+                                       mlir::Location loc, mlir::Value team) {
+  mlir::Type refTy = builder.getRefType(builder.getI32Type());
+  mlir::Type boxTy = fir::BoxType::get(builder.getNoneType());
+  mlir::FunctionType ftype = PRIF_FUNCTYPE(boxTy, refTy);
+  mlir::func::FuncOp funcOp =
+      builder.createFunction(loc, PRIFNAME_SUB("this_image_no_coarray"), ftype);
+
+  mlir::Value result = builder.createTemporary(loc, builder.getI32Type());
+  mlir::Value teamArg =
+      !team ? builder.create<fir::AbsentOp>(loc, boxTy) : team;
+  llvm::SmallVector<mlir::Value> args =
+      fir::runtime::createArguments(builder, loc, ftype, teamArg, result);
+  builder.create<fir::CallOp>(loc, funcOp, args);
+  return builder.create<fir::LoadOp>(loc, result);
+}
diff --git a/flang/lib/Optimizer/Builder/Runtime/Main.cpp b/flang/lib/Optimizer/Builder/Runtime/Main.cpp
index d35f687167b05..d303e0ad63842 100644
--- a/flang/lib/Optimizer/Builder/Runtime/Main.cpp
+++ b/flang/lib/Optimizer/Builder/Runtime/Main.cpp
@@ -10,6 +10,7 @@
 #include "flang/Lower/EnvironmentDefault.h"
 #include "flang/Optimizer/Builder/BoxValue.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/Runtime/Coarray.h"
 #include "flang/Optimizer/Builder/Runtime/EnvironmentDefaults.h"
 #include "flang/Optimizer/Builder/Runtime/RTBuilder.h"
 #include "flang/Optimizer/Dialect/FIROps.h"
@@ -23,8 +24,8 @@ using namespace Fortran::runtime;
 /// Create a `int main(...)` that calls the Fortran entry point
 void fir::runtime::genMain(
     fir::FirOpBuilder &builder, mlir::Location loc,
-    const std::vector<Fortran::lower::EnvironmentDefault> &defs,
-    bool initCuda) {
+    const std::vector<Fortran::lower::EnvironmentDefault> &defs, bool initCuda,
+    bool initCoarrayEnv) {
   auto *context = builder.getContext();
   auto argcTy = builder.getDefaultIntegerType();
   auto ptrTy = mlir::LLVM::LLVMPointerType::get(context);
@@ -69,6 +70,8 @@ void fir::runtime::genMain(
         loc, RTNAME_STRING(CUFInit), mlir::FunctionType::get(context, {}, {}));
     fir::CallOp::create(builder, loc, initFn);
   }
+  if (initCoarrayEnv)
+    fir::runtime::genInitCoarray(builder, loc);
 
   fir::CallOp::create(builder, loc, qqMainFn);
   fir::CallOp::create(builder, loc, stopFn);
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index 1b289ae690cbe..76f3cbd421cb9 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -87,14 +87,6 @@ static inline mlir::Type getI8Type(mlir::MLIRContext *context) {
   return mlir::IntegerType::get(context, 8);
 }
 
-static mlir::LLVM::ConstantOp
-genConstantIndex(mlir::Location loc, mlir::Type ity,
-                 mlir::ConversionPatternRewriter &rewriter,
-                 std::int64_t offset) {
-  auto cattr = rewriter.getI64IntegerAttr(offset);
-  return mlir::LLVM::ConstantOp::create(rewriter, loc, ity, cattr);
-}
-
 static mlir::Block *createBlock(mlir::ConversionPatternRewriter &rewriter,
                                 mlir::Block *insertBefore) {
   assert(insertBefore && "expected valid insertion block");
@@ -208,39 +200,6 @@ getDependentTypeMemSizeFn(fir::RecordType recTy, fir::AllocaOp op,
   TODO(op.getLoc(), "did not find allocation function");
 }
 
-// Compute the alloc scale size (constant factors encoded in the array type).
-// We do this for arrays without a constant interior or arrays of character with
-// dynamic length arrays, since those are the only ones that get decayed to a
-// pointer to the element type.
-template <typename OP>
-static mlir::Value
-genAllocationScaleSize(OP op, mlir::Type ity,
-                       mlir::ConversionPatternRewriter &rewriter) {
-  mlir::Location loc = op.getLoc();
-  mlir::Type dataTy = op.getInType();
-  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
-  fir::SequenceType::Extent constSize = 1;
-  if (seqTy) {
-    int constRows = seqTy.getConstantRows();
-    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
-    if (constRows != static_cast<int>(shape.size())) {
-      for (auto extent : shape) {
-        if (constRows-- > 0)
-          continue;
-        if (extent != fir::SequenceType::getUnknownExtent())
-          constSize *= extent;
-      }
-    }
-  }
-
-  if (constSize != 1) {
-    mlir::Value constVal{
-        genConstantIndex(loc, ity, rewriter, constSize).getResult()};
-    return constVal;
-  }
-  return nullptr;
-}
-
 namespace {
 struct DeclareOpConversion : public fir::FIROpConversion<fir::cg::XDeclareOp> {
 public:
@@ -275,7 +234,7 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
     auto loc = alloc.getLoc();
     mlir::Type ity = lowerTy().indexType();
     unsigned i = 0;
-    mlir::Value size = genConstantIndex(loc, ity, rewriter, 1).getResult();
+    mlir::Value size = fir::genConstantIndex(loc, ity, rewriter, 1).getResult();
     mlir::Type firObjType = fir::unwrapRefType(alloc.getType());
     mlir::Type llvmObjectType = convertObjectType(firObjType);
     if (alloc.hasLenParams()) {
@@ -307,7 +266,8 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
                << scalarType << " with type parameters";
       }
     }
-    if (auto scaleSize = genAllocationScaleSize(alloc, ity, rewriter))
+    if (auto scaleSize = fir::genAllocationScaleSize(
+            alloc.getLoc(), alloc.getInType(), ity, rewriter))
       size =
           rewriter.createOrFold<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     if (alloc.hasShapeOperands()) {
@@ -484,7 +444,7 @@ struct BoxIsArrayOpConversion : public fir::FIROpConversion<fir::BoxIsArrayOp> {
     auto loc = boxisarray.getLoc();
     TypePair boxTyPair = getBoxTypePair(boxisarray.getVal().getType());
     mlir::Value rank = getRankFromBox(loc, boxTyPair, a, rewriter);
-    mlir::Value c0 = genConstantIndex(loc, rank.getType(), rewriter, 0);
+    mlir::Value c0 = fir::genConstantIndex(loc, rank.getType(), rewriter, 0);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         boxisarray, mlir::LLVM::ICmpPredicate::ne, rank, c0);
     return mlir::success();
@@ -820,7 +780,7 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
       // Do folding for constant inputs.
       if (auto constVal = fir::getIntIfConstant(op0)) {
         mlir::Value normVal =
-            genConstantIndex(loc, toTy, rewriter, *constVal ? 1 : 0);
+            fir::genConstantIndex(loc, toTy, rewriter, *constVal ? 1 : 0);
         rewriter.replaceOp(convert, normVal);
         return mlir::success();
       }
@@ -833,7 +793,7 @@ struct ConvertOpConversion : public fir::FIROpConversion<fir::ConvertOp> {
       }
 
       // Compare the input with zero.
-      mlir::Value zero = genConstantIndex(loc, fromTy, rewriter, 0);
+      mlir::Value zero = fir::genConstantIndex(loc, fromTy, rewriter, 0);
       auto isTrue = mlir::LLVM::ICmpOp::create(
           rewriter, loc, mlir::LLVM::ICmpPredicate::ne, op0, zero);
 
@@ -1082,21 +1042,6 @@ static mlir::SymbolRefAttr getMalloc(fir::AllocMemOp op,
   return getMallocInModule(mod, op, rewriter, indexType);
 }
 
-/// Helper function for generating the LLVM IR that computes the distance
-/// in bytes between adjacent elements pointed to by a pointer
-/// of type \p ptrTy. The result is returned as a value of \p idxTy integer
-/// type.
-static mlir::Value
-computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
-                       mlir::Type idxTy,
-                       mlir::ConversionPatternRewriter &rewriter,
-                       const mlir::DataLayout &dataLayout) {
-  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
-  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
-  std::int64_t distance = llvm::alignTo(size, alignment);
-  return genConstantIndex(loc, idxTy, rewriter, distance);
-}
-
 /// Return value of the stride in bytes between adjacent elements
 /// of LLVM type \p llTy. The result is returned as a value of
 /// \p idxTy integer type.
@@ -1105,7 +1050,7 @@ genTypeStrideInBytes(mlir::Location loc, mlir::Type idxTy,
                      mlir::ConversionPatternRewriter &rewriter, mlir::Type llTy,
                      const mlir::DataLayout &dataLayout) {
   // Create a pointer type and use computeElementDistance().
-  return computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
+  return fir::computeElementDistance(loc, llTy, idxTy, rewriter, dataLayout);
 }
 
 namespace {
@@ -1124,8 +1069,9 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
       TODO(loc, "fir.allocmem codegen of derived type with length parameters");
     mlir::Value size = genTypeSizeInBytes(loc, ity, rewriter, llvmObjectTy);
-    if (auto scaleSize = genAllocationScaleSize(heap, ity, rewriter))
-      size = mlir::LLVM::MulOp::create(rewriter, loc, ity, size, scaleSize);
+    if (auto scaleSize =
+            fir::genAllocationScaleSize(loc, heap.getInType(), ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
     for (mlir::Value opnd : adaptor.getOperands())
       size = mlir::LLVM::MulOp::create(rewriter, loc, ity, size,
                                        integerCast(loc, rewriter, ity, opnd));
@@ -1133,8 +1079,8 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
     // As the return value of malloc(0) is implementation defined, allocate one
     // byte to ensure the allocation status being true. This behavior aligns to
     // what the runtime has.
-    mlir::Value zero = genConstantIndex(loc, ity, rewriter, 0);
-    mlir::Value one = genConstantIndex(loc, ity, rewriter, 1);
+    mlir::Value zero = fir::genConstantIndex(loc, ity, rewriter, 0);
+    mlir::Value one = fir::genConstantIndex(loc, ity, rewriter, 1);
     mlir::Value cmp = mlir::LLVM::ICmpOp::create(
         rewriter, loc, mlir::LLVM::ICmpPredicate::sgt, size, zero);
     size = mlir::LLVM::SelectOp::create(rewriter, loc, cmp, size, one);
@@ -1157,7 +1103,8 @@ struct AllocMemOpConversion : public fir::FIROpConversion<fir::AllocMemOp> {
   mlir::Value genTypeSizeInBytes(mlir::Location loc, mlir::Type idxTy,
                                  mlir::ConversionPatternRewriter &rewriter,
                                  mlir::Type llTy) const {
-    return computeElementDistance(loc, llTy, idxTy, rewriter, getDataLayout());
+    return fir::computeElementDistance(loc, llTy, idxTy, rewriter,
+                                       getDataLayout());
   }
 };
 } // namespace
@@ -1344,7 +1291,7 @@ genCUFAllocDescriptor(mlir::Location loc,
   mlir::Type structTy = typeConverter.convertBoxTypeAsStruct(boxTy);
   std::size_t boxSize = dl->getTypeSizeInBits(structTy) / 8;
   mlir::Value sizeInBytes =
-      genConstantIndex(loc, llvmIntPtrType, rewriter, boxSize);
+      fir::genConstantIndex(loc, llvmIntPtrType, rewriter, boxSize);
   llvm::SmallVector args = {sizeInBytes, sourceFile, sourceLine};
   return mlir::LLVM::CallOp::create(rewriter, loc, fctTy,
                                     RTNAME_STRING(CUFAllocDescriptor), args)
@@ -1599,7 +1546,7 @@ struct EmboxCommonConversion : public fir::FIROpConversion<OP> {
       // representation of derived types with pointer/allocatable components.
       // This has been seen in hashing algorithms using TRANSFER.
       mlir::Value zero =
-          genConstantIndex(loc, rewriter.getI64Type(), rewriter, 0);
+          fir::genConstantIndex(loc, rewriter.getI64Type(), rewriter, 0);
       descriptor = insertField(rewriter, loc, descriptor,
                                {getLenParamFieldId(boxTy), 0}, zero);
     }
@@ -1944,8 +1891,8 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
     bool hasSlice = !xbox.getSlice().empty();
     unsigned sliceOffset = xbox.getSliceOperandIndex();
     mlir::Location loc = xbox.getLoc();
-    mlir::Value zero = genConstantIndex(loc, i64Ty, rewriter, 0);
-    mlir::Value one = genConstantIndex(loc, i64Ty, rewriter, 1);
+    mlir::Value zero = fir::genConstantIndex(loc, i64Ty, rewriter, 0);
+    mlir::Value one = fir::genConstantIndex(loc, i64Ty, rewriter, 1);
     mlir::Value prevPtrOff = one;
     mlir::Type eleTy = boxTy.getEleTy();
     const unsigned rank = xbox.getRank();
@@ -1994,7 +1941,7 @@ struct XEmboxOpConversion : public EmboxCommonConversion<fir::cg::XEmboxOp> {
         prevDimByteStride =
             getCharacterByteSize(loc, rewriter, charTy, adaptor.getLenParams());
       } else {
-        prevDimByteStride = genConstantIndex(
+        prevDimByteStride = fir::genConstantIndex(
             loc, i64Ty, rewriter,
             charTy.getLen() * lowerTy().characterBitsize(charTy) / 8);
       }
@@ -2152,7 +2099,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     if (auto charTy = mlir::dyn_cast<fir::CharacterType>(inputEleTy)) {
       if (charTy.hasConstantLen()) {
         mlir::Value len =
-            genConstantIndex(loc, idxTy, rewriter, charTy.getLen());
+            fir::genConstantIndex(loc, idxTy, rewriter, charTy.getLen());
         lenParams.emplace_back(len);
       } else {
         mlir::Value len = getElementSizeFromBox(loc, idxTy, inputBoxTyPair,
@@ -2161,7 +2108,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
           assert(!isInGlobalOp(rewriter) &&
                  "character target in global op must have constant length");
           mlir::Value width =
-              genConstantIndex(loc, idxTy, rewriter, charTy.getFKind());
+              fir::genConstantIndex(loc, idxTy, rewriter, charTy.getFKind());
           len = mlir::LLVM::SDivOp::create(rewriter, loc, idxTy, len, width);
         }
         lenParams.emplace_back(len);
@@ -2215,8 +2162,9 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
                 mlir::ConversionPatternRewriter &rewriter) const {
     mlir::Location loc = rebox.getLoc();
     mlir::Value zero =
-        genConstantIndex(loc, lowerTy().indexType(), rewriter, 0);
-    mlir::Value one = genConstantIndex(loc, lowerTy().indexType(), rewriter, 1);
+        fir::genConstantIndex(loc, lowerTy().indexType(), rewriter, 0);
+    mlir::Value one =
+        fir::genConstantIndex(loc, lowerTy().indexType(), rewriter, 1);
     for (auto iter : llvm::enumerate(llvm::zip(extents, strides))) {
       mlir::Value extent = std::get<0>(iter.value());
       unsigned dim = iter.index();
@@ -2249,7 +2197,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     mlir::Location loc = rebox.getLoc();
     mlir::Type byteTy = ::getI8Type(rebox.getContext());
     mlir::Type idxTy = lowerTy().indexType();
-    mlir::Value zero = genConstantIndex(loc, idxTy, rewriter, 0);
+    mlir::Value zero = fir::genConstantIndex(loc, idxTy, rewriter, 0);
     // Apply subcomponent and substring shift on base address.
     if (!rebox.getSubcomponent().empty() || !rebox.getSubstr().empty()) {
       // Cast to inputEleTy* so that a GEP can be used.
@@ -2277,7 +2225,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     // and strides.
     llvm::SmallVector<mlir::Value> slicedExtents;
     llvm::SmallVector<mlir::Value> slicedStrides;
-    mlir::Value one = genConstantIndex(loc, idxTy, rewriter, 1);
+    mlir::Value one = fir::genConstantIndex(loc, idxTy, rewriter, 1);
     const bool sliceHasOrigins = !rebox.getShift().empty();
     unsigned sliceOps = rebox.getSliceOperandIndex();
     unsigned shiftOps = rebox.getShiftOperandIndex();
@@ -2350,7 +2298,7 @@ struct XReboxOpConversion : public EmboxCommonConversion<fir::cg::XReboxOp> {
     // which may be OK if all new extents are ones, the stride does not
     // matter, use one.
     mlir::Value stride = inputStrides.empty()
-                             ? genConstantIndex(loc, idxTy, rewriter, 1)
+                             ? fir::genConstantIndex(loc, idxTy, rewriter, 1)
                              : inputStrides[0];
     for (unsigned i = 0; i < rebox.getShape().size(); ++i) {
       mlir::Value rawExtent = operands[rebox.getShapeOperandIndex() + i];
@@ -2585,9 +2533,9 @@ struct XArrayCoorOpConversion
     unsigned shiftOffset = coor.getShiftOperandIndex();
     unsigned sliceOffset = coor.getSliceOperandIndex();
     auto sliceOps = coor.getSlice().begin();
-    mlir::Value one = genConstantIndex(loc, idxTy, rewriter, 1);
+    mlir::Value one = fir::genConstantIndex(loc, idxTy, rewriter, 1);
     mlir::Value prevExt = one;
-    mlir::Value offset = genConstantIndex(loc, idxTy, rewriter, 0);
+    mlir::Value offset = fir::genConstantIndex(loc, idxTy, rewriter, 0);
     const bool isShifted = !coor.getShift().empty();
     const bool isSliced = !coor.getSlice().empty();
     const bool baseIsBoxed =
@@ -2918,7 +2866,7 @@ struct CoordinateOpConversion
         // of lower bound aspects. This both accounts for dynamically sized
         // types and non contiguous arrays.
         auto idxTy = lowerTy().indexType();
-        mlir::Value off = genConstantIndex(loc, idxTy, rewriter, 0);
+        mlir::Value off = fir::genConstantIndex(loc, idxTy, rewriter, 0);
         unsigned arrayDim = arrTy.getDimension();
         for (unsigned dim = 0; dim < arrayDim && it != end; ++dim, ++it) {
           mlir::Value stride =
@@ -3525,114 +3473,123 @@ struct SelectCaseOpConversion : public fir::FIROpConversion<fir::SelectCaseOp> {
   }
 };
 
-/// Helper function for converting select ops. This function converts the
-/// signature of the given block. If the new block signature is different from
-/// `expectedTypes`, returns "failure".
-static llvm::FailureOr<mlir::Block *>
-getConvertedBlock(mlir::ConversionPatternRewriter &rewriter,
-                  const mlir::TypeConverter *converter,
-                  mlir::Operation *branchOp, mlir::Block *block,
-                  mlir::TypeRange expectedTypes) {
-  assert(converter && "expected non-null type converter");
-  assert(!block->isEntryBlock() && "entry blocks have no predecessors");
-
-  // There is nothing to do if the types already match.
-  if (block->getArgumentTypes() == expectedTypes)
-    return block;
-
-  // Compute the new block argument types and convert the block.
-  std::optional<mlir::TypeConverter::SignatureConversion> conversion =
-      converter->convertBlockSignature(block);
-  if (!conversion)
-    return rewriter.notifyMatchFailure(branchOp,
-                                       "could not compute block signature");
-  if (expectedTypes != conversion->getConvertedTypes())
-    return rewriter.notifyMatchFailure(
-        branchOp,
-        "mismatch between adaptor operand types and computed block signature");
-  return rewriter.applySignatureConversion(block, *conversion, converter);
-}
-
+/// Base class for SelectOpConversion and SelectRankOpConversion.
 template <typename OP>
-static llvm::LogicalResult
-selectMatchAndRewrite(const fir::LLVMTypeConverter &lowering, OP select,
-                      typename OP::Adaptor adaptor,
-                      mlir::ConversionPatternRewriter &rewriter,
-                      const mlir::TypeConverter *converter) {
-  unsigned conds = select.getNumConditions();
-  auto cases = select.getCases().getValue();
-  mlir::Value selector = adaptor.getSelector();
-  auto loc = select.getLoc();
-  assert(conds > 0 && "select must have cases");
-
-  llvm::SmallVector<mlir::Block *> destinations;
-  llvm::SmallVector<mlir::ValueRange> destinationsOperands;
-  mlir::Block *defaultDestination;
-  mlir::ValueRange defaultOperands;
-  llvm::SmallVector<int32_t> caseValues;
-
-  for (unsigned t = 0; t != conds; ++t) {
-    mlir::Block *dest = select.getSuccessor(t);
-    auto destOps = select.getSuccessorOperands(adaptor.getOperands(), t);
-    const mlir::Attribute &attr = cases[t];
-    if (auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(attr)) {
-      destinationsOperands.push_back(destOps ? *destOps : mlir::ValueRange{});
-      auto convertedBlock =
-          getConvertedBlock(rewriter, converter, select, dest,
-                            mlir::TypeRange(destinationsOperands.back()));
+struct SelectOpConversionBase : public fir::FIROpConversion<OP> {
+  using fir::FIROpConversion<OP>::FIROpConversion;
+
+private:
+  /// Helper function for converting select ops. This function converts the
+  /// signature of the given block. If the new block signature is different from
+  /// `expectedTypes`, returns "failure".
+  llvm::FailureOr<mlir::Block *>
+  getConvertedBlock(mlir::ConversionPatternRewriter &rewriter,
+                    mlir::Operation *branchOp, mlir::Block *block,
+                    mlir::TypeRange expectedTypes) const {
+    const mlir::TypeConverter *converter = this->getTypeConverter();
+    assert(converter && "expected non-null type converter");
+    assert(!block->isEntryBlock() && "entry blocks have no predecessors");
+
+    // There is nothing to do if the types already match.
+    if (block->getArgumentTypes() == expectedTypes)
+      return block;
+
+    // Compute the new block argument types and convert the block.
+    std::optional<mlir::TypeConverter::SignatureConversion> conversion =
+        converter->convertBlockSignature(block);
+    if (!conversion)
+      return rewriter.notifyMatchFailure(branchOp,
+                                         "could not compute block signature");
+    if (expectedTypes != conversion->getConvertedTypes())
+      return rewriter.notifyMatchFailure(branchOp,
+                                         "mismatch between adaptor operand "
+                                         "types and computed block signature");
+    return rewriter.applySignatureConversion(block, *conversion, converter);
+  }
+
+protected:
+  llvm::LogicalResult
+  selectMatchAndRewrite(OP select, typename OP::Adaptor adaptor,
+                        mlir::ConversionPatternRewriter &rewriter) const {
+    unsigned conds = select.getNumConditions();
+    auto cases = select.getCases().getValue();
+    mlir::Value selector = adaptor.getSelector();
+    auto loc = select.getLoc();
+    assert(conds > 0 && "select must have cases");
+
+    llvm::SmallVector<mlir::Block *> destinations;
+    llvm::SmallVector<mlir::ValueRange> destinationsOperands;
+    mlir::Block *defaultDestination;
+    mlir::ValueRange defaultOperands;
+    // LLVM::SwitchOp selector type and the case values types
+    // must have the same bit width, so cast the selector to i64,
+    // and use i64 for the case values. It is hard to imagine
+    // a computed GO TO with the number of labels in the label-list
+    // bigger than INT_MAX, but let's use i64 to be on the safe side.
+    // Moreover, fir.select operation is more relaxed than
+    // a Fortran computed GO TO, so it may specify such a case value
+    // even if there is just a single label/case.
+    llvm::SmallVector<int64_t> caseValues;
+
+    for (unsigned t = 0; t != conds; ++t) {
+      mlir::Block *dest = select.getSuccessor(t);
+      auto destOps = select.getSuccessorOperands(adaptor.getOperands(), t);
+      const mlir::Attribute &attr = cases[t];
+      if (auto intAttr = mlir::dyn_cast<mlir::IntegerAttr>(attr)) {
+        destinationsOperands.push_back(destOps ? *destOps : mlir::ValueRange{});
+        auto convertedBlock =
+            getConvertedBlock(rewriter, select, dest,
+                              mlir::TypeRange(destinationsOperands.back()));
+        if (mlir::failed(convertedBlock))
+          return mlir::failure();
+        destinations.push_back(*convertedBlock);
+        caseValues.push_back(intAttr.getInt());
+        continue;
+      }
+      assert(mlir::dyn_cast_or_null<mlir::UnitAttr>(attr));
+      assert((t + 1 == conds) && "unit must be last");
+      defaultOperands = destOps ? *destOps : mlir::ValueRange{};
+      auto convertedBlock = getConvertedBlock(rewriter, select, dest,
+                                              mlir::TypeRange(defaultOperands));
       if (mlir::failed(convertedBlock))
         return mlir::failure();
-      destinations.push_back(*convertedBlock);
-      caseValues.push_back(intAttr.getInt());
-      continue;
+      defaultDestination = *convertedBlock;
     }
-    assert(mlir::dyn_cast_or_null<mlir::UnitAttr>(attr));
-    assert((t + 1 == conds) && "unit must be last");
-    defaultOperands = destOps ? *destOps : mlir::ValueRange{};
-    auto convertedBlock = getConvertedBlock(rewriter, converter, select, dest,
-                                            mlir::TypeRange(defaultOperands));
-    if (mlir::failed(convertedBlock))
-      return mlir::failure();
-    defaultDestination = *convertedBlock;
-  }
-
-  // LLVM::SwitchOp takes a i32 type for the selector.
-  if (select.getSelector().getType() != rewriter.getI32Type())
-    selector = mlir::LLVM::TruncOp::create(rewriter, loc, rewriter.getI32Type(),
-                                           selector);
-
-  rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>(
-      select, selector,
-      /*defaultDestination=*/defaultDestination,
-      /*defaultOperands=*/defaultOperands,
-      /*caseValues=*/caseValues,
-      /*caseDestinations=*/destinations,
-      /*caseOperands=*/destinationsOperands,
-      /*branchWeights=*/llvm::ArrayRef<std::int32_t>());
-  return mlir::success();
-}
 
+    selector =
+        this->integerCast(loc, rewriter, rewriter.getI64Type(), selector);
+
+    rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>(
+        select, selector,
+        /*defaultDestination=*/defaultDestination,
+        /*defaultOperands=*/defaultOperands,
+        /*caseValues=*/rewriter.getI64VectorAttr(caseValues),
+        /*caseDestinations=*/destinations,
+        /*caseOperands=*/destinationsOperands,
+        /*branchWeights=*/llvm::ArrayRef<std::int32_t>());
+    return mlir::success();
+  }
+};
 /// conversion of fir::SelectOp to an if-then-else ladder
-struct SelectOpConversion : public fir::FIROpConversion<fir::SelectOp> {
-  using FIROpConversion::FIROpConversion;
+struct SelectOpConversion : public SelectOpConversionBase<fir::SelectOp> {
+  using SelectOpConversionBase::SelectOpConversionBase;
 
   llvm::LogicalResult
   matchAndRewrite(fir::SelectOp op, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
-    return selectMatchAndRewrite<fir::SelectOp>(lowerTy(), op, adaptor,
-                                                rewriter, getTypeConverter());
+    return this->selectMatchAndRewrite(op, adaptor, rewriter);
   }
 };
 
 /// conversion of fir::SelectRankOp to an if-then-else ladder
-struct SelectRankOpConversion : public fir::FIROpConversion<fir::SelectRankOp> {
-  using FIROpConversion::FIROpConversion;
+struct SelectRankOpConversion
+    : public SelectOpConversionBase<fir::SelectRankOp> {
+  using SelectOpConversionBase::SelectOpConversionBase;
 
   llvm::LogicalResult
   matchAndRewrite(fir::SelectRankOp op, OpAdaptor adaptor,
                   mlir::ConversionPatternRewriter &rewriter) const override {
-    return selectMatchAndRewrite<fir::SelectRankOp>(
-        lowerTy(), op, adaptor, rewriter, getTypeConverter());
+    return this->selectMatchAndRewrite(op, adaptor, rewriter);
   }
 };
 
@@ -3837,7 +3794,7 @@ struct IsPresentOpConversion : public fir::FIROpConversion<fir::IsPresentOp> {
       ptr = mlir::LLVM::ExtractValueOp::create(rewriter, loc, ptr, 0);
     }
     mlir::LLVM::ConstantOp c0 =
-        genConstantIndex(isPresent.getLoc(), idxTy, rewriter, 0);
+        fir::genConstantIndex(isPresent.getLoc(), idxTy, rewriter, 0);
     auto addr = mlir::LLVM::PtrToIntOp::create(rewriter, loc, idxTy, ptr);
     rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
         isPresent, mlir::LLVM::ICmpPredicate::ne, addr, c0);
diff --git a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
index 37f1c9f97e1ce..97912bda79b08 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGenOpenMP.cpp
@@ -21,6 +21,7 @@
 #include "flang/Optimizer/Dialect/Support/FIRContext.h"
 #include "flang/Optimizer/Support/FatalError.h"
 #include "flang/Optimizer/Support/InternalNames.h"
+#include "flang/Optimizer/Support/Utils.h"
 #include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
 #include "mlir/Conversion/LLVMCommon/Pattern.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
@@ -125,10 +126,58 @@ struct PrivateClauseOpConversion
     return mlir::success();
   }
 };
+
+// Convert FIR type to LLVM without turning fir.box<T> into memory
+// reference.
+static mlir::Type convertObjectType(const fir::LLVMTypeConverter &converter,
+                                    mlir::Type firType) {
+  if (auto boxTy = mlir::dyn_cast<fir::BaseBoxType>(firType))
+    return converter.convertBoxTypeAsStruct(boxTy);
+  return converter.convertType(firType);
+}
+
+// FIR Op specific conversion for TargetAllocMemOp
+struct TargetAllocMemOpConversion
+    : public OpenMPFIROpConversion<mlir::omp::TargetAllocMemOp> {
+  using OpenMPFIROpConversion::OpenMPFIROpConversion;
+
+  llvm::LogicalResult
+  matchAndRewrite(mlir::omp::TargetAllocMemOp allocmemOp, OpAdaptor adaptor,
+                  mlir::ConversionPatternRewriter &rewriter) const override {
+    mlir::Type heapTy = allocmemOp.getAllocatedType();
+    mlir::Location loc = allocmemOp.getLoc();
+    auto ity = lowerTy().indexType();
+    mlir::Type dataTy = fir::unwrapRefType(heapTy);
+    mlir::Type llvmObjectTy = convertObjectType(lowerTy(), dataTy);
+    if (fir::isRecordWithTypeParameters(fir::unwrapSequenceType(dataTy)))
+      TODO(loc, "omp.target_allocmem codegen of derived type with length "
+                "parameters");
+    mlir::Value size = fir::computeElementDistance(
+        loc, llvmObjectTy, ity, rewriter, lowerTy().getDataLayout());
+    if (auto scaleSize = fir::genAllocationScaleSize(
+            loc, allocmemOp.getInType(), ity, rewriter))
+      size = rewriter.create<mlir::LLVM::MulOp>(loc, ity, size, scaleSize);
+    for (mlir::Value opnd : adaptor.getOperands().drop_front())
+      size = rewriter.create<mlir::LLVM::MulOp>(
+          loc, ity, size, integerCast(lowerTy(), loc, rewriter, ity, opnd));
+    auto mallocTyWidth = lowerTy().getIndexTypeBitwidth();
+    auto mallocTy =
+        mlir::IntegerType::get(rewriter.getContext(), mallocTyWidth);
+    if (mallocTyWidth != ity.getIntOrFloatBitWidth())
+      size = integerCast(lowerTy(), loc, rewriter, mallocTy, size);
+    rewriter.modifyOpInPlace(allocmemOp, [&]() {
+      allocmemOp.setInType(rewriter.getI8Type());
+      allocmemOp.getTypeparamsMutable().clear();
+      allocmemOp.getTypeparamsMutable().append(size);
+    });
+    return mlir::success();
+  }
+};
 } // namespace
 
 void fir::populateOpenMPFIRToLLVMConversionPatterns(
     const LLVMTypeConverter &converter, mlir::RewritePatternSet &patterns) {
   patterns.add<MapInfoOpConversion>(converter);
   patterns.add<PrivateClauseOpConversion>(converter);
+  patterns.add<TargetAllocMemOpConversion>(converter);
 }
diff --git a/flang/lib/Optimizer/Dialect/CUF/Attributes/CUFAttr.cpp b/flang/lib/Optimizer/Dialect/CUF/Attributes/CUFAttr.cpp
index 52c733dcad6f0..bd0499f406c18 100644
--- a/flang/lib/Optimizer/Dialect/CUF/Attributes/CUFAttr.cpp
+++ b/flang/lib/Optimizer/Dialect/CUF/Attributes/CUFAttr.cpp
@@ -16,6 +16,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/Operation.h"
 #include "llvm/ADT/TypeSwitch.h"
 
 #include "flang/Optimizer/Dialect/CUF/Attributes/CUFEnumAttr.cpp.inc"
@@ -29,4 +30,26 @@ void CUFDialect::registerAttributes() {
                 LaunchBoundsAttr, ProcAttributeAttr>();
 }
 
+cuf::DataAttributeAttr getDataAttr(mlir::Operation *op) {
+  if (!op)
+    return {};
+
+  if (auto dataAttr =
+          op->getAttrOfType<cuf::DataAttributeAttr>(cuf::getDataAttrName()))
+    return dataAttr;
+
+  // When the attribute is declared on the operation, it doesn't have a prefix.
+  if (auto dataAttr =
+          op->getAttrOfType<cuf::DataAttributeAttr>(cuf::dataAttrName))
+    return dataAttr;
+
+  return {};
+}
+
+bool hasDataAttr(mlir::Operation *op, cuf::DataAttribute value) {
+  if (auto dataAttr = getDataAttr(op))
+    return dataAttr.getValue() == value;
+  return false;
+}
+
 } // namespace cuf
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 01975f357a8da..87f9899aa7879 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -107,7 +107,6 @@ static bool verifyTypeParamCount(mlir::Type inType, unsigned numParams) {
 }
 
 /// Parser shared by Alloca and Allocmem
-///
 /// operation ::= %res = (`fir.alloca` | `fir.allocmem`) $in_type
 ///                      ( `(` $typeparams `)` )? ( `,` $shape )?
 ///                      attr-dict-without-keyword
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index ed102db69dae3..3c5095da0145a 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -1440,44 +1440,46 @@ void hlfir::MatmulTransposeOp::getEffects(
 }
 
 //===----------------------------------------------------------------------===//
-// CShiftOp
+// Array shifts: CShiftOp/EOShiftOp
 //===----------------------------------------------------------------------===//
 
-llvm::LogicalResult hlfir::CShiftOp::verify() {
-  mlir::Value array = getArray();
+template <typename Op>
+static llvm::LogicalResult verifyArrayShift(Op op) {
+  mlir::Value array = op.getArray();
   fir::SequenceType arrayTy = mlir::cast<fir::SequenceType>(
       hlfir::getFortranElementOrSequenceType(array.getType()));
   llvm::ArrayRef<int64_t> inShape = arrayTy.getShape();
   std::size_t arrayRank = inShape.size();
   mlir::Type eleTy = arrayTy.getEleTy();
-  hlfir::ExprType resultTy = mlir::cast<hlfir::ExprType>(getResult().getType());
+  hlfir::ExprType resultTy =
+      mlir::cast<hlfir::ExprType>(op.getResult().getType());
   llvm::ArrayRef<int64_t> resultShape = resultTy.getShape();
   std::size_t resultRank = resultShape.size();
   mlir::Type resultEleTy = resultTy.getEleTy();
-  mlir::Value shift = getShift();
+  mlir::Value shift = op.getShift();
   mlir::Type shiftTy = hlfir::getFortranElementOrSequenceType(shift.getType());
 
-  // TODO: turn allowCharacterLenMismatch into true.
-  if (auto match = areMatchingTypes(*this, eleTy, resultEleTy,
-                                    /*allowCharacterLenMismatch=*/false);
+  if (auto match = areMatchingTypes(
+          op, eleTy, resultEleTy,
+          /*allowCharacterLenMismatch=*/!useStrictIntrinsicVerifier);
       match.failed())
-    return emitOpError(
+    return op.emitOpError(
         "input and output arrays should have the same element type");
 
   if (arrayRank != resultRank)
-    return emitOpError("input and output arrays should have the same rank");
+    return op.emitOpError("input and output arrays should have the same rank");
 
   constexpr int64_t unknownExtent = fir::SequenceType::getUnknownExtent();
   for (auto [inDim, resultDim] : llvm::zip(inShape, resultShape))
     if (inDim != unknownExtent && resultDim != unknownExtent &&
         inDim != resultDim)
-      return emitOpError(
+      return op.emitOpError(
           "output array's shape conflicts with the input array's shape");
 
   int64_t dimVal = -1;
-  if (!getDim())
+  if (!op.getDim())
     dimVal = 1;
-  else if (auto dim = fir::getIntIfConstant(getDim()))
+  else if (auto dim = fir::getIntIfConstant(op.getDim()))
     dimVal = *dim;
 
   // The DIM argument may be statically invalid (e.g. exceed the
@@ -1485,44 +1487,79 @@ llvm::LogicalResult hlfir::CShiftOp::verify() {
   // so avoid some checks unless useStrictIntrinsicVerifier is true.
   if (useStrictIntrinsicVerifier && dimVal != -1) {
     if (dimVal < 1)
-      return emitOpError("DIM must be >= 1");
+      return op.emitOpError("DIM must be >= 1");
     if (dimVal > static_cast<int64_t>(arrayRank))
-      return emitOpError("DIM must be <= input array's rank");
+      return op.emitOpError("DIM must be <= input array's rank");
   }
 
-  if (auto shiftSeqTy = mlir::dyn_cast<fir::SequenceType>(shiftTy)) {
-    // SHIFT is an array. Verify the rank and the shape (if DIM is constant).
-    llvm::ArrayRef<int64_t> shiftShape = shiftSeqTy.getShape();
-    std::size_t shiftRank = shiftShape.size();
-    if (shiftRank != arrayRank - 1)
-      return emitOpError(
-          "SHIFT's rank must be 1 less than the input array's rank");
-
-    if (useStrictIntrinsicVerifier && dimVal != -1) {
-      // SHIFT's shape must be [d(1), d(2), ..., d(DIM-1), d(DIM+1), ..., d(n)],
-      // where [d(1), d(2), ..., d(n)] is the shape of the ARRAY.
-      int64_t arrayDimIdx = 0;
-      int64_t shiftDimIdx = 0;
-      for (auto shiftDim : shiftShape) {
-        if (arrayDimIdx == dimVal - 1)
+  // A helper lambda to verify the shape of the array types of
+  // certain operands of the array shift (e.g. the SHIFT and BOUNDARY operands).
+  auto verifyOperandTypeShape = [&](mlir::Type type,
+                                    llvm::Twine name) -> llvm::LogicalResult {
+    if (auto opndSeqTy = mlir::dyn_cast<fir::SequenceType>(type)) {
+      // The operand is an array. Verify the rank and the shape (if DIM is
+      // constant).
+      llvm::ArrayRef<int64_t> opndShape = opndSeqTy.getShape();
+      std::size_t opndRank = opndShape.size();
+      if (opndRank != arrayRank - 1)
+        return op.emitOpError(
+            name + "'s rank must be 1 less than the input array's rank");
+
+      if (useStrictIntrinsicVerifier && dimVal != -1) {
+        // The operand's shape must be
+        // [d(1), d(2), ..., d(DIM-1), d(DIM+1), ..., d(n)],
+        // where [d(1), d(2), ..., d(n)] is the shape of the ARRAY.
+        int64_t arrayDimIdx = 0;
+        int64_t opndDimIdx = 0;
+        for (auto opndDim : opndShape) {
+          if (arrayDimIdx == dimVal - 1)
+            ++arrayDimIdx;
+
+          if (inShape[arrayDimIdx] != unknownExtent &&
+              opndDim != unknownExtent && inShape[arrayDimIdx] != opndDim)
+            return op.emitOpError("SHAPE(ARRAY)(" +
+                                  llvm::Twine(arrayDimIdx + 1) +
+                                  ") must be equal to SHAPE(" + name + ")(" +
+                                  llvm::Twine(opndDimIdx + 1) +
+                                  "): " + llvm::Twine(inShape[arrayDimIdx]) +
+                                  " != " + llvm::Twine(opndDim));
           ++arrayDimIdx;
-
-        if (inShape[arrayDimIdx] != unknownExtent &&
-            shiftDim != unknownExtent && inShape[arrayDimIdx] != shiftDim)
-          return emitOpError("SHAPE(ARRAY)(" + llvm::Twine(arrayDimIdx + 1) +
-                             ") must be equal to SHAPE(SHIFT)(" +
-                             llvm::Twine(shiftDimIdx + 1) +
-                             "): " + llvm::Twine(inShape[arrayDimIdx]) +
-                             " != " + llvm::Twine(shiftDim));
-        ++arrayDimIdx;
-        ++shiftDimIdx;
+          ++opndDimIdx;
+        }
       }
     }
+    return mlir::success();
+  };
+
+  if (failed(verifyOperandTypeShape(shiftTy, "SHIFT")))
+    return mlir::failure();
+
+  if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>) {
+    if (mlir::Value boundary = op.getBoundary()) {
+      mlir::Type boundaryTy =
+          hlfir::getFortranElementOrSequenceType(boundary.getType());
+      if (auto match = areMatchingTypes(
+              op, eleTy, hlfir::getFortranElementType(boundaryTy),
+              /*allowCharacterLenMismatch=*/!useStrictIntrinsicVerifier);
+          match.failed())
+        return op.emitOpError(
+            "ARRAY and BOUNDARY operands must have the same element type");
+      if (failed(verifyOperandTypeShape(boundaryTy, "BOUNDARY")))
+        return mlir::failure();
+    }
   }
 
   return mlir::success();
 }
 
+//===----------------------------------------------------------------------===//
+// CShiftOp
+//===----------------------------------------------------------------------===//
+
+llvm::LogicalResult hlfir::CShiftOp::verify() {
+  return verifyArrayShift(*this);
+}
+
 void hlfir::CShiftOp::getEffects(
     llvm::SmallVectorImpl<
         mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
@@ -1530,6 +1567,21 @@ void hlfir::CShiftOp::getEffects(
   getIntrinsicEffects(getOperation(), effects);
 }
 
+//===----------------------------------------------------------------------===//
+// EOShiftOp
+//===----------------------------------------------------------------------===//
+
+llvm::LogicalResult hlfir::EOShiftOp::verify() {
+  return verifyArrayShift(*this);
+}
+
+void hlfir::EOShiftOp::getEffects(
+    llvm::SmallVectorImpl<
+        mlir::SideEffects::EffectInstance<mlir::MemoryEffects::Effect>>
+        &effects) {
+  getIntrinsicEffects(getOperation(), effects);
+}
+
 //===----------------------------------------------------------------------===//
 // ReshapeOp
 //===----------------------------------------------------------------------===//
@@ -1543,7 +1595,8 @@ llvm::LogicalResult hlfir::ReshapeOp::verify() {
       hlfir::getFortranElementOrSequenceType(array.getType()));
   if (auto match = areMatchingTypes(
           *this, hlfir::getFortranElementType(resultType),
-          arrayType.getElementType(), /*allowCharacterLenMismatch=*/true);
+          arrayType.getElementType(),
+          /*allowCharacterLenMismatch=*/!useStrictIntrinsicVerifier);
       match.failed())
     return emitOpError("ARRAY and the result must have the same element type");
   if (hlfir::isPolymorphicType(resultType) !=
@@ -1565,9 +1618,9 @@ llvm::LogicalResult hlfir::ReshapeOp::verify() {
   if (mlir::Value pad = getPad()) {
     auto padArrayType = mlir::cast<fir::SequenceType>(
         hlfir::getFortranElementOrSequenceType(pad.getType()));
-    if (auto match = areMatchingTypes(*this, arrayType.getElementType(),
-                                      padArrayType.getElementType(),
-                                      /*allowCharacterLenMismatch=*/true);
+    if (auto match = areMatchingTypes(
+            *this, arrayType.getElementType(), padArrayType.getElementType(),
+            /*allowCharacterLenMismatch=*/!useStrictIntrinsicVerifier);
         match.failed())
       return emitOpError("ARRAY and PAD must be of the same type");
   }
@@ -1847,8 +1900,7 @@ hlfir::ShapeOfOp::canonicalize(ShapeOfOp shapeOf,
     // shape information is not available at compile time
     return llvm::LogicalResult::failure();
 
-  rewriter.replaceAllUsesWith(shapeOf.getResult(), shape);
-  rewriter.eraseOp(shapeOf);
+  rewriter.replaceOp(shapeOf, shape);
   return llvm::LogicalResult::success();
 }
 
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
index 9109f2b331567..886a8a59e7441 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/BufferizeHLFIR.cpp
@@ -455,12 +455,8 @@ struct AssociateOpConversion
 
       mlir::Type associateHlfirVarType = associate.getResultTypes()[0];
       hlfirVar = adjustVar(hlfirVar, associateHlfirVarType);
-      associate.getResult(0).replaceAllUsesWith(hlfirVar);
-
       mlir::Type associateFirVarType = associate.getResultTypes()[1];
       firVar = adjustVar(firVar, associateFirVarType);
-      associate.getResult(1).replaceAllUsesWith(firVar);
-      associate.getResult(2).replaceAllUsesWith(flag);
       // FIXME: note that the AssociateOp that is being erased
       // here will continue to be a user of the original Source
       // operand (e.g. a result of hlfir.elemental), because
@@ -472,7 +468,7 @@ struct AssociateOpConversion
       // the conversions, so that we can analyze HLFIR in its
       // original form and decide which of the AssociateOp
       // users of hlfir.expr can reuse the buffer (if it can).
-      rewriter.eraseOp(associate);
+      rewriter.replaceOp(associate, {hlfirVar, firVar, flag});
     };
 
     // If this is the last use of the expression value and this is an hlfir.expr
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index 2e273243eeb2e..4e7de4732357d 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -490,15 +490,18 @@ class DesignateOpConversion
         }
         baseEleTy = hlfir::getFortranElementType(componentType);
         shape = designate.getComponentShape();
-      } else {
-        // array%component[(indices) substring|complex part] cases.
-        // Component ref of array bases are dealt with below in embox/rebox.
-        assert(mlir::isa<fir::BaseBoxType>(designateResultType));
       }
     }
 
-    if (mlir::isa<fir::BaseBoxType>(designateResultType)) {
-      // Generate embox or rebox.
+    if (mlir::isa<fir::BaseBoxType>(designateResultType) ||
+        // Convert the component array slices using embox/rebox
+        // even if the result is a contiguous array section, e.g.:
+        //   hlfir.designate %base{"i"} shape %shape :
+        //       (!fir.box<!fir.array<2x!fir.type<_QMtypesTt{i:i32}>>>,
+        //        !fir.shape<1>) -> !fir.ref<!fir.array<2xi32>>
+        // fir.coordinate_of should probably be a better option, though.
+        (fieldIndex && baseEntity.isArray())) {
+      // Generate embox or rebox for slicing.
       mlir::Type eleTy = fir::unwrapPassByRefType(designateResultType);
       bool isScalarDesignator = !mlir::isa<fir::SequenceType>(eleTy);
       mlir::Value sourceBox;
@@ -575,8 +578,13 @@ class DesignateOpConversion
       else
         assert(sliceFields.empty() && substring.empty());
 
-      llvm::SmallVector<mlir::Type> resultType{
-          fir::updateTypeWithVolatility(designateResultType, isVolatile)};
+      // If the designate's result type is not a box, then create
+      // a box type to be used for the result of the embox/rebox.
+      mlir::Type resultType = designateResultType;
+      if (!mlir::isa<fir::BaseBoxType>(resultType))
+        resultType = fir::wrapInClassOrBoxType(resultType);
+
+      resultType = fir::updateTypeWithVolatility(resultType, isVolatile);
 
       mlir::Value resultBox;
       if (mlir::isa<fir::BaseBoxType>(base.getType())) {
@@ -587,6 +595,13 @@ class DesignateOpConversion
             fir::EmboxOp::create(builder, loc, resultType, base, shape, slice,
                                  firBaseTypeParameters, sourceBox);
       }
+
+      if (!mlir::isa<fir::BaseBoxType>(designateResultType)) {
+        // If the designate's result is not a box, use the raw address
+        // as the new result.
+        resultBox = fir::BoxAddrOp::create(rewriter, loc, resultBox);
+        resultBox = builder.createConvert(loc, designateResultType, resultBox);
+      }
       rewriter.replaceOp(designate, resultBox);
       return mlir::success();
     }
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
index c42b895946d19..ff84a3cff0afb 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineElementals.cpp
@@ -101,9 +101,8 @@ class InlineElementalConversion
         elemental.getLoc(), builder, elemental, apply.getIndices());
 
     // remove the old elemental and all of the bookkeeping
-    rewriter.replaceAllUsesWith(apply.getResult(), yield.getElementValue());
+    rewriter.replaceOp(apply, {yield.getElementValue()});
     rewriter.eraseOp(yield);
-    rewriter.eraseOp(apply);
     rewriter.eraseOp(destroy);
     rewriter.eraseOp(elemental);
 
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
index 3c29d6877e8de..e0167cc12b8a3 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
@@ -469,33 +469,49 @@ struct MatmulTransposeOpConversion
   }
 };
 
-class CShiftOpConversion : public HlfirIntrinsicConversion<hlfir::CShiftOp> {
-  using HlfirIntrinsicConversion<hlfir::CShiftOp>::HlfirIntrinsicConversion;
+// A converter for hlfir.cshift and hlfir.eoshift.
+template <typename T>
+class ArrayShiftOpConversion : public HlfirIntrinsicConversion<T> {
+  using HlfirIntrinsicConversion<T>::HlfirIntrinsicConversion;
+  using HlfirIntrinsicConversion<T>::lowerArguments;
+  using HlfirIntrinsicConversion<T>::processReturnValue;
+  using typename HlfirIntrinsicConversion<T>::IntrinsicArgument;
 
   llvm::LogicalResult
-  matchAndRewrite(hlfir::CShiftOp cshift,
-                  mlir::PatternRewriter &rewriter) const override {
-    fir::FirOpBuilder builder{rewriter, cshift.getOperation()};
-    const mlir::Location &loc = cshift->getLoc();
+  matchAndRewrite(T op, mlir::PatternRewriter &rewriter) const override {
+    fir::FirOpBuilder builder{rewriter, op.getOperation()};
+    const mlir::Location &loc = op->getLoc();
 
-    llvm::SmallVector<IntrinsicArgument, 3> inArgs;
-    mlir::Value array = cshift.getArray();
+    llvm::SmallVector<IntrinsicArgument, 4> inArgs;
+    llvm::StringRef intrinsicName{[]() {
+      if constexpr (std::is_same_v<T, hlfir::EOShiftOp>)
+        return "eoshift";
+      else if constexpr (std::is_same_v<T, hlfir::CShiftOp>)
+        return "cshift";
+      else
+        llvm_unreachable("unsupported array shift");
+    }()};
+
+    mlir::Value array = op.getArray();
     inArgs.push_back({array, array.getType()});
-    mlir::Value shift = cshift.getShift();
+    mlir::Value shift = op.getShift();
     inArgs.push_back({shift, shift.getType()});
-    inArgs.push_back({cshift.getDim(), builder.getI32Type()});
+    if constexpr (std::is_same_v<T, hlfir::EOShiftOp>) {
+      mlir::Value boundary = op.getBoundary();
+      inArgs.push_back({boundary, boundary ? boundary.getType() : nullptr});
+    }
+    inArgs.push_back({op.getDim(), builder.getI32Type()});
 
-    auto *argLowering = fir::getIntrinsicArgumentLowering("cshift");
+    auto *argLowering = fir::getIntrinsicArgumentLowering(intrinsicName);
     llvm::SmallVector<fir::ExtendedValue, 3> args =
-        lowerArguments(cshift, inArgs, rewriter, argLowering);
+        lowerArguments(op, inArgs, rewriter, argLowering);
 
-    mlir::Type scalarResultType =
-        hlfir::getFortranElementType(cshift.getType());
+    mlir::Type scalarResultType = hlfir::getFortranElementType(op.getType());
 
-    auto [resultExv, mustBeFreed] =
-        fir::genIntrinsicCall(builder, loc, "cshift", scalarResultType, args);
+    auto [resultExv, mustBeFreed] = fir::genIntrinsicCall(
+        builder, loc, intrinsicName, scalarResultType, args);
 
-    processReturnValue(cshift, resultExv, mustBeFreed, builder, rewriter);
+    processReturnValue(op, resultExv, mustBeFreed, builder, rewriter);
     return mlir::success();
   }
 };
@@ -547,7 +563,8 @@ class LowerHLFIRIntrinsics
         AnyOpConversion, SumOpConversion, ProductOpConversion,
         TransposeOpConversion, CountOpConversion, DotProductOpConversion,
         MaxvalOpConversion, MinvalOpConversion, MinlocOpConversion,
-        MaxlocOpConversion, CShiftOpConversion, ReshapeOpConversion>(context);
+        MaxlocOpConversion, ArrayShiftOpConversion<hlfir::CShiftOp>,
+        ArrayShiftOpConversion<hlfir::EOShiftOp>, ReshapeOpConversion>(context);
 
     // While conceptually this pass is performing dialect conversion, we use
     // pattern rewrites here instead of dialect conversion because this pass
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
index 8e25298122411..32998ab64cb8c 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIROrderedAssignments.cpp
@@ -96,7 +96,7 @@ struct MaskedArrayExpr {
   /// hlfir.elemental_addr that form the elemental tree producing
   /// the expression value. hlfir.elemental that produce values
   /// used inside transformational operations are not part of this set.
-  llvm::SmallSet<mlir::Operation *, 4> elementalParts{};
+  llvm::SmallPtrSet<mlir::Operation *, 4> elementalParts{};
   /// Was generateNoneElementalPart called?
   bool noneElementalPartWasGenerated = false;
   /// Is this expression the mask expression of the outer where statement?
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ScheduleOrderedAssignments.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ScheduleOrderedAssignments.cpp
index 722cd8a4488b1..a48b7ba8a77df 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ScheduleOrderedAssignments.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ScheduleOrderedAssignments.cpp
@@ -137,7 +137,7 @@ class Scheduler {
   // Schedule being built.
   hlfir::Schedule schedule;
   /// Leaf regions that have been saved so far.
-  llvm::SmallSet<mlir::Region *, 16> savedRegions;
+  llvm::SmallPtrSet<mlir::Region *, 16> savedRegions;
   /// Is schedule.back() a schedule that is only saving region with read
   /// effects?
   bool currentRunIsReadOnly = false;
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
index b27c3a8526945..fe12f49c655b8 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/SimplifyHLFIRIntrinsics.cpp
@@ -10,6 +10,7 @@
 // into the calling function.
 //===----------------------------------------------------------------------===//
 
+#include "flang/Optimizer/Builder/Character.h"
 #include "flang/Optimizer/Builder/Complex.h"
 #include "flang/Optimizer/Builder/FIRBuilder.h"
 #include "flang/Optimizer/Builder/HLFIRTools.h"
@@ -1269,64 +1270,91 @@ class ReductionConversion : public mlir::OpRewritePattern<Op> {
   }
 };
 
-class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
+template <typename Op>
+class ArrayShiftConversion : public mlir::OpRewritePattern<Op> {
 public:
-  using mlir::OpRewritePattern<hlfir::CShiftOp>::OpRewritePattern;
+  // The implementation below only support CShiftOp and EOShiftOp.
+  static_assert(std::is_same_v<Op, hlfir::CShiftOp> ||
+                std::is_same_v<Op, hlfir::EOShiftOp>);
+
+  using mlir::OpRewritePattern<Op>::OpRewritePattern;
 
   llvm::LogicalResult
-  matchAndRewrite(hlfir::CShiftOp cshift,
-                  mlir::PatternRewriter &rewriter) const override {
+  matchAndRewrite(Op op, mlir::PatternRewriter &rewriter) const override {
 
-    hlfir::ExprType expr = mlir::dyn_cast<hlfir::ExprType>(cshift.getType());
+    hlfir::ExprType expr = mlir::dyn_cast<hlfir::ExprType>(op.getType());
     assert(expr &&
-           "expected an expression type for the result of hlfir.cshift");
+           "expected an expression type for the result of the array shift");
     unsigned arrayRank = expr.getRank();
-    // When it is a 1D CSHIFT, we may assume that the DIM argument
+    // When it is a 1D CSHIFT/EOSHIFT, we may assume that the DIM argument
     // (whether it is present or absent) is equal to 1, otherwise,
     // the program is illegal.
     int64_t dimVal = 1;
     if (arrayRank != 1)
-      if (mlir::Value dim = cshift.getDim()) {
+      if (mlir::Value dim = op.getDim()) {
         auto constDim = fir::getIntIfConstant(dim);
         if (!constDim)
-          return rewriter.notifyMatchFailure(cshift,
-                                             "Nonconstant DIM for CSHIFT");
+          return rewriter.notifyMatchFailure(
+              op, "Nonconstant DIM for CSHIFT/EOSHIFT");
         dimVal = *constDim;
       }
 
     if (dimVal <= 0 || dimVal > arrayRank)
-      return rewriter.notifyMatchFailure(cshift, "Invalid DIM for CSHIFT");
+      return rewriter.notifyMatchFailure(op, "Invalid DIM for CSHIFT/EOSHIFT");
+
+    if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>) {
+      // TODO: the EOSHIFT inlining code is not ready to produce
+      // fir.if selecting between ARRAY and BOUNDARY (or the default
+      // boundary value), when they are expressions of type CHARACTER.
+      // This needs more work.
+      if (mlir::isa<fir::CharacterType>(expr.getEleTy())) {
+        if (!hlfir::Entity{op.getArray()}.isVariable())
+          return rewriter.notifyMatchFailure(
+              op, "EOSHIFT with ARRAY being CHARACTER expression");
+        if (op.getBoundary() && !hlfir::Entity{op.getBoundary()}.isVariable())
+          return rewriter.notifyMatchFailure(
+              op, "EOSHIFT with BOUNDARY being CHARACTER expression");
+      }
+      // TODO: selecting between ARRAY and BOUNDARY values with derived types
+      // need more work.
+      if (fir::isa_derived(expr.getEleTy()))
+        return rewriter.notifyMatchFailure(op, "EOSHIFT of derived type");
+    }
 
     // When DIM==1 and the contiguity of the input array is not statically
     // known, try to exploit the fact that the leading dimension might be
     // contiguous. We can do this now using hlfir.eval_in_mem with
     // a dynamic check for the leading dimension contiguity.
-    // Otherwise, convert hlfir.cshift to hlfir.elemental.
+    // Otherwise, convert hlfir.cshift/eoshift to hlfir.elemental.
     //
     // Note that the hlfir.elemental can be inlined into other hlfir.elemental,
     // while hlfir.eval_in_mem prevents this, and we will end up creating
     // a temporary array for the result. We may need to come up with
     // a more sophisticated logic for picking the most efficient
     // representation.
-    hlfir::Entity array = hlfir::Entity{cshift.getArray()};
+    hlfir::Entity array = hlfir::Entity{op.getArray()};
     mlir::Type elementType = array.getFortranElementType();
     if (dimVal == 1 && fir::isa_trivial(elementType) &&
-        // genInMemCShift() only works for variables currently.
+        // genInMemArrayShift() only works for variables currently.
         array.isVariable())
-      rewriter.replaceOp(cshift, genInMemCShift(rewriter, cshift, dimVal));
+      rewriter.replaceOp(op, genInMemArrayShift(rewriter, op, dimVal));
     else
-      rewriter.replaceOp(cshift, genElementalCShift(rewriter, cshift, dimVal));
+      rewriter.replaceOp(op, genElementalArrayShift(rewriter, op, dimVal));
     return mlir::success();
   }
 
 private:
-  /// Generate MODULO(\p shiftVal, \p extent).
+  /// For CSHIFT, generate MODULO(\p shiftVal, \p extent).
+  /// For EOSHIFT, return \p shiftVal casted to \p calcType.
   static mlir::Value normalizeShiftValue(mlir::Location loc,
                                          fir::FirOpBuilder &builder,
                                          mlir::Value shiftVal,
                                          mlir::Value extent,
                                          mlir::Type calcType) {
     shiftVal = builder.createConvert(loc, calcType, shiftVal);
+    if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)
+      return shiftVal;
+
     extent = builder.createConvert(loc, calcType, extent);
     // Make sure that we do not divide by zero. When the dimension
     // has zero size, turn the extent into 1. Note that the computed
@@ -1342,24 +1370,227 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
     return builder.createConvert(loc, calcType, shiftVal);
   }
 
-  /// Convert \p cshift into an hlfir.elemental using
+  /// The indices computations for the array shifts are done using I64 type.
+  /// For CSHIFT, all computations do not overflow signed and unsigned I64.
+  /// For EOSHIFT, some computations may involve negative shift values,
+  /// so using no-unsigned wrap flag would be incorrect.
+  static void setArithOverflowFlags(Op op, fir::FirOpBuilder &builder) {
+    if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)
+      builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw);
+    else
+      builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw |
+                                      mlir::arith::IntegerOverflowFlags::nuw);
+  }
+
+  /// Return the element type of the EOSHIFT boundary that may be omitted
+  /// statically or dynamically. This element type might be used
+  /// to generate MLIR where we have to select between the default
+  /// boundary value and the dynamically absent/present boundary value.
+  /// If the boundary has a type not defined in Table 16.4 in 16.9.77
+  /// of F2023, then the return value is nullptr.
+  static mlir::Type getDefaultBoundaryValueType(mlir::Type elementType) {
+    // To be able to generate a "select" between the default boundary value
+    // and the dynamic boundary value, use BoxCharType for the CHARACTER
+    // cases. This might be a little bit inefficient, because we may
+    // create unnecessary tuples, but it simplifies the inlining code.
+    if (auto charTy = mlir::dyn_cast<fir::CharacterType>(elementType))
+      return fir::BoxCharType::get(charTy.getContext(), charTy.getFKind());
+
+    if (mlir::isa<fir::LogicalType>(elementType) ||
+        fir::isa_integer(elementType) || fir::isa_real(elementType) ||
+        fir::isa_complex(elementType))
+      return elementType;
+
+    return nullptr;
+  }
+
+  /// Generate the default boundary value as defined in Table 16.4 in 16.9.77
+  /// of F2023.
+  static mlir::Value genDefaultBoundary(mlir::Location loc,
+                                        fir::FirOpBuilder &builder,
+                                        mlir::Type elementType) {
+    assert(getDefaultBoundaryValueType(elementType) &&
+           "default boundary value cannot be computed for the given type");
+    if (mlir::isa<fir::CharacterType>(elementType)) {
+      // Create an empty CHARACTER of the same kind. The assignment
+      // of this empty CHARACTER into the result will add the padding
+      // if necessary.
+      fir::factory::CharacterExprHelper charHelper{builder, loc};
+      mlir::Value zeroLen = builder.createIntegerConstant(
+          loc, builder.getCharacterLengthType(), 0);
+      fir::CharBoxValue emptyCharTemp =
+          charHelper.createCharacterTemp(elementType, zeroLen);
+      return charHelper.createEmbox(emptyCharTemp);
+    }
+
+    return fir::factory::createZeroValue(builder, loc, elementType);
+  }
+
+  /// \p entity represents the boundary operand of hlfir.eoshift.
+  /// This method generates a scalar boundary value fetched
+  /// from the boundary entity using \p indices (which may be empty,
+  /// if the boundary operand is scalar).
+  static mlir::Value loadEoshiftVal(mlir::Location loc,
+                                    fir::FirOpBuilder &builder,
+                                    hlfir::Entity entity,
+                                    mlir::ValueRange indices = {}) {
+    hlfir::Entity boundaryVal =
+        hlfir::loadElementAt(loc, builder, entity, indices);
+
+    mlir::Type boundaryValTy =
+        getDefaultBoundaryValueType(entity.getFortranElementType());
+
+    // Boxed !fir.char<KIND,LEN> with known LEN are loaded
+    // as raw references to !fir.char<KIND,LEN>.
+    // We need to wrap them into the !fir.boxchar.
+    if (boundaryVal.isVariable() && boundaryValTy &&
+        mlir::isa<fir::BoxCharType>(boundaryValTy))
+      return hlfir::genVariableBoxChar(loc, builder, boundaryVal);
+    return boundaryVal;
+  }
+
+  /// This method generates a scalar boundary value for the given hlfir.eoshift
+  /// \p op that can be used to initialize cells of the result
+  /// if the scalar/array boundary operand is statically or dynamically
+  /// absent. The first result is the scalar boundary value. The second result
+  /// is a dynamic predicate indicating whether the scalar boundary value
+  /// should actually be used.
+  [[maybe_unused]] static std::pair<mlir::Value, mlir::Value>
+  genScalarBoundaryForEOShift(mlir::Location loc, fir::FirOpBuilder &builder,
+                              hlfir::EOShiftOp op) {
+    hlfir::Entity array{op.getArray()};
+    mlir::Type elementType = array.getFortranElementType();
+
+    if (!op.getBoundary()) {
+      // Boundary operand is statically absent.
+      mlir::Value defaultVal = genDefaultBoundary(loc, builder, elementType);
+      mlir::Value boundaryIsScalarPred = builder.createBool(loc, true);
+      return {defaultVal, boundaryIsScalarPred};
+    }
+
+    hlfir::Entity boundary{op.getBoundary()};
+    mlir::Type boundaryValTy = getDefaultBoundaryValueType(elementType);
+
+    if (boundary.isScalar()) {
+      if (!boundaryValTy || !boundary.mayBeOptional()) {
+        // The boundary must be present.
+        mlir::Value boundaryVal = loadEoshiftVal(loc, builder, boundary);
+        mlir::Value boundaryIsScalarPred = builder.createBool(loc, true);
+        return {boundaryVal, boundaryIsScalarPred};
+      }
+
+      // Boundary is a scalar that may be dynamically absent.
+      // If boundary is not present dynamically, we must use the default
+      // value.
+      assert(mlir::isa<fir::BaseBoxType>(boundary.getType()));
+      mlir::Value isPresentPred =
+          fir::IsPresentOp::create(builder, loc, builder.getI1Type(), boundary);
+      mlir::Value boundaryVal =
+          builder
+              .genIfOp(loc, {boundaryValTy}, isPresentPred,
+                       /*withElseRegion=*/true)
+              .genThen([&]() {
+                mlir::Value boundaryVal =
+                    loadEoshiftVal(loc, builder, boundary);
+                fir::ResultOp::create(builder, loc, boundaryVal);
+              })
+              .genElse([&]() {
+                mlir::Value defaultVal =
+                    genDefaultBoundary(loc, builder, elementType);
+                fir::ResultOp::create(builder, loc, defaultVal);
+              })
+              .getResults()[0];
+      mlir::Value boundaryIsScalarPred = builder.createBool(loc, true);
+      return {boundaryVal, boundaryIsScalarPred};
+    }
+    if (!boundaryValTy || !boundary.mayBeOptional()) {
+      // The boundary must be present
+      mlir::Value boundaryIsScalarPred = builder.createBool(loc, false);
+      return {nullptr, boundaryIsScalarPred};
+    }
+
+    // Boundary is an array that may be dynamically absent.
+    mlir::Value defaultVal = genDefaultBoundary(loc, builder, elementType);
+    mlir::Value isPresentPred =
+        fir::IsPresentOp::create(builder, loc, builder.getI1Type(), boundary);
+    // If the array is present, then boundaryIsScalarPred must be equal
+    // to false, otherwise, it should be true.
+    mlir::Value trueVal = builder.createBool(loc, true);
+    mlir::Value falseVal = builder.createBool(loc, false);
+    mlir::Value boundaryIsScalarPred = mlir::arith::SelectOp::create(
+        builder, loc, isPresentPred, falseVal, trueVal);
+    return {defaultVal, boundaryIsScalarPred};
+  }
+
+  /// Generate code that produces the final boundary value to be assigned
+  /// to the result of hlfir.eoshift \p op. \p precomputedScalarBoundary
+  /// specifies the scalar boundary value pre-computed before the elemental
+  /// or the assignment loop. If it is nullptr, then the boundary operand
+  /// of \p op must be a present array. \p boundaryIsScalarPred is a dynamic
+  /// predicate that is true, when the pre-computed scalar value must be used.
+  /// \p oneBasedIndices specify the indices to address into the boundary
+  /// array - they may be empty, if the boundary is scalar.
+  [[maybe_unused]] static mlir::Value selectBoundaryValue(
+      mlir::Location loc, fir::FirOpBuilder &builder, hlfir::EOShiftOp op,
+      mlir::Value precomputedScalarBoundary, mlir::Value boundaryIsScalarPred,
+      mlir::ValueRange oneBasedIndices) {
+    // Boundary is statically absent: a default value has been precomputed.
+    if (!op.getBoundary())
+      return precomputedScalarBoundary;
+
+    // Boundary is statically present and is a scalar: boundary does not depend
+    // upon the indices and so it has been precomputed.
+    hlfir::Entity boundary{op.getBoundary()};
+    if (boundary.isScalar())
+      return precomputedScalarBoundary;
+
+    // Boundary is statically present and is an array: if the scalar
+    // boundary has not been precomputed, this means that the data type
+    // of the shifted values does not provide a way to compute
+    // the default boundary value, so the array boundary must be dynamically
+    // present, and we can load the boundary values from it.
+    bool mustBePresent = !precomputedScalarBoundary;
+    if (mustBePresent)
+      return loadEoshiftVal(loc, builder, boundary, oneBasedIndices);
+
+    // The array boundary may be dynamically absent.
+    // In this case, precomputedScalarBoundary is a pre-computed scalar
+    // boundary value that has to be used if boundaryIsScalarPred
+    // is true, otherwise, the boundary value has to be loaded
+    // from the boundary array.
+    mlir::Type boundaryValTy = precomputedScalarBoundary.getType();
+    mlir::Value newBoundaryVal =
+        builder
+            .genIfOp(loc, {boundaryValTy}, boundaryIsScalarPred,
+                     /*withElseRegion=*/true)
+            .genThen([&]() {
+              fir::ResultOp::create(builder, loc, precomputedScalarBoundary);
+            })
+            .genElse([&]() {
+              mlir::Value elem =
+                  loadEoshiftVal(loc, builder, boundary, oneBasedIndices);
+              fir::ResultOp::create(builder, loc, elem);
+            })
+            .getResults()[0];
+    return newBoundaryVal;
+  }
+
+  /// Convert \p op into an hlfir.elemental using
   /// the pre-computed constant \p dimVal.
-  static mlir::Operation *genElementalCShift(mlir::PatternRewriter &rewriter,
-                                             hlfir::CShiftOp cshift,
-                                             int64_t dimVal) {
+  static mlir::Operation *
+  genElementalArrayShift(mlir::PatternRewriter &rewriter, Op op,
+                         int64_t dimVal) {
     using Fortran::common::maxRank;
-    hlfir::Entity shift = hlfir::Entity{cshift.getShift()};
-    hlfir::Entity array = hlfir::Entity{cshift.getArray()};
+    hlfir::Entity shift = hlfir::Entity{op.getShift()};
+    hlfir::Entity array = hlfir::Entity{op.getArray()};
 
-    mlir::Location loc = cshift.getLoc();
-    fir::FirOpBuilder builder{rewriter, cshift.getOperation()};
+    mlir::Location loc = op.getLoc();
+    fir::FirOpBuilder builder{rewriter, op.getOperation()};
     // The new index computation involves MODULO, which is not implemented
     // for IndexType, so use I64 instead.
     mlir::Type calcType = builder.getI64Type();
-    // All the indices arithmetic used below does not overflow
-    // signed and unsigned I64.
-    builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw |
-                                    mlir::arith::IntegerOverflowFlags::nuw);
+    // Set the indices arithmetic overflow flags.
+    setArithOverflowFlags(op, builder);
 
     mlir::Value arrayShape = hlfir::genShape(loc, builder, array);
     llvm::SmallVector<mlir::Value, maxRank> arrayExtents =
@@ -1374,6 +1605,17 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
       shiftVal =
           normalizeShiftValue(loc, builder, shiftVal, shiftDimExtent, calcType);
     }
+    // The boundary operand of hlfir.eoshift may be statically or
+    // dynamically absent.
+    // In both cases, it is assumed to be a scalar with the value
+    // corresponding to the array element type.
+    // boundaryIsScalarPred is a dynamic predicate that identifies
+    // these cases. If boundaryIsScalarPred is dynamicaly false,
+    // then the boundary operand must be a present array.
+    mlir::Value boundaryVal, boundaryIsScalarPred;
+    if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)
+      std::tie(boundaryVal, boundaryIsScalarPred) =
+          genScalarBoundaryForEOShift(loc, builder, op);
 
     auto genKernel = [&](mlir::Location loc, fir::FirOpBuilder &builder,
                          mlir::ValueRange inputIndices) -> hlfir::Entity {
@@ -1394,34 +1636,84 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
         shiftVal = normalizeShiftValue(loc, builder, shiftVal, shiftDimExtent,
                                        calcType);
       }
+      if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>) {
+        llvm::SmallVector<mlir::Value, maxRank> boundaryIndices{indices};
+        boundaryIndices.erase(boundaryIndices.begin() + dimVal - 1);
+        boundaryVal =
+            selectBoundaryValue(loc, builder, op, boundaryVal,
+                                boundaryIsScalarPred, boundaryIndices);
+      }
 
-      // Element i of the result (1-based) is element
-      // 'MODULO(i + SH - 1, SIZE(ARRAY,DIM)) + 1' (1-based) of the original
-      // ARRAY (or its section, when ARRAY is not a vector).
-
-      // Compute the index into the original array using the normalized
-      // shift value, which satisfies (SH >= 0 && SH < SIZE(ARRAY,DIM)):
-      //   newIndex =
-      //     i + ((i <= SIZE(ARRAY,DIM) - SH) ? SH : SH - SIZE(ARRAY,DIM))
-      //
-      // Such index computation allows for further loop vectorization
-      // in LLVM.
-      mlir::Value wrapBound =
-          mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);
-      mlir::Value adjustedShiftVal =
-          mlir::arith::SubIOp::create(builder, loc, shiftVal, shiftDimExtent);
-      mlir::Value index =
-          builder.createConvert(loc, calcType, inputIndices[dimVal - 1]);
-      mlir::Value wrapCheck = mlir::arith::CmpIOp::create(
-          builder, loc, mlir::arith::CmpIPredicate::sle, index, wrapBound);
-      mlir::Value actualShift = mlir::arith::SelectOp::create(
-          builder, loc, wrapCheck, shiftVal, adjustedShiftVal);
-      mlir::Value newIndex =
-          mlir::arith::AddIOp::create(builder, loc, index, actualShift);
-      newIndex = builder.createConvert(loc, builder.getIndexType(), newIndex);
-      indices[dimVal - 1] = newIndex;
-      hlfir::Entity element = hlfir::getElementAt(loc, builder, array, indices);
-      return hlfir::loadTrivialScalar(loc, builder, element);
+      if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>) {
+        // EOSHIFT:
+        // Element i of the result (1-based) is the element of the original
+        // array (or its section, when ARRAY is not a vector) with index
+        // (i + SH), if (1 <= i + SH <= SIZE(ARRAY,DIM)), otherwise
+        // it is the BOUNDARY value.
+        mlir::Value index =
+            builder.createConvert(loc, calcType, inputIndices[dimVal - 1]);
+        mlir::arith::IntegerOverflowFlags savedFlags =
+            builder.getIntegerOverflowFlags();
+        builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw);
+        mlir::Value indexPlusShift =
+            mlir::arith::AddIOp::create(builder, loc, index, shiftVal);
+        builder.setIntegerOverflowFlags(savedFlags);
+        mlir::Value one = builder.createIntegerConstant(loc, calcType, 1);
+        mlir::Value cmp1 = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::sge, indexPlusShift, one);
+        mlir::Value cmp2 = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::sle, indexPlusShift,
+            shiftDimExtent);
+        mlir::Value loadFromArray =
+            mlir::arith::AndIOp::create(builder, loc, cmp1, cmp2);
+        mlir::Type boundaryValTy = boundaryVal.getType();
+        mlir::Value result =
+            builder
+                .genIfOp(loc, {boundaryValTy}, loadFromArray,
+                         /*withElseRegion=*/true)
+                .genThen([&]() {
+                  indices[dimVal - 1] = builder.createConvert(
+                      loc, builder.getIndexType(), indexPlusShift);
+                  ;
+                  mlir::Value elem =
+                      loadEoshiftVal(loc, builder, array, indices);
+                  fir::ResultOp::create(builder, loc, elem);
+                })
+                .genElse(
+                    [&]() { fir::ResultOp::create(builder, loc, boundaryVal); })
+                .getResults()[0];
+        return hlfir::Entity{result};
+      } else {
+        // CSHIFT:
+        // Element i of the result (1-based) is element
+        // 'MODULO(i + SH - 1, SIZE(ARRAY,DIM)) + 1' (1-based) of the original
+        // ARRAY (or its section, when ARRAY is not a vector).
+
+        // Compute the index into the original array using the normalized
+        // shift value, which satisfies (SH >= 0 && SH < SIZE(ARRAY,DIM)):
+        //   newIndex =
+        //     i + ((i <= SIZE(ARRAY,DIM) - SH) ? SH : SH - SIZE(ARRAY,DIM))
+        //
+        // Such index computation allows for further loop vectorization
+        // in LLVM.
+        mlir::Value wrapBound =
+            mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);
+        mlir::Value adjustedShiftVal =
+            mlir::arith::SubIOp::create(builder, loc, shiftVal, shiftDimExtent);
+        mlir::Value index =
+            builder.createConvert(loc, calcType, inputIndices[dimVal - 1]);
+        mlir::Value wrapCheck = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::sle, index, wrapBound);
+        mlir::Value actualShift = mlir::arith::SelectOp::create(
+            builder, loc, wrapCheck, shiftVal, adjustedShiftVal);
+        mlir::Value newIndex =
+            mlir::arith::AddIOp::create(builder, loc, index, actualShift);
+        newIndex = builder.createConvert(loc, builder.getIndexType(), newIndex);
+        indices[dimVal - 1] = newIndex;
+        hlfir::Entity element =
+            hlfir::getElementAt(loc, builder, array, indices);
+        return hlfir::loadTrivialScalar(loc, builder, element);
+      }
     };
 
     mlir::Type elementType = array.getFortranElementType();
@@ -1429,19 +1721,42 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
         loc, builder, elementType, arrayShape, typeParams, genKernel,
         /*isUnordered=*/true,
         array.isPolymorphic() ? static_cast<mlir::Value>(array) : nullptr,
-        cshift.getResult().getType());
+        op.getResult().getType());
     return elementalOp.getOperation();
   }
 
-  /// Convert \p cshift into an hlfir.eval_in_mem using the pre-computed
+  /// Convert \p op into an hlfir.eval_in_mem using the pre-computed
   /// constant \p dimVal.
-  /// The converted code looks like this:
-  ///   do i=1,SH
-  ///     result(i + (SIZE(ARRAY,DIM) - SH)) = array(i)
+  /// The converted code for CSHIFT looks like this:
+  ///   DEST_OFFSET = SIZE(ARRAY,DIM) - SH
+  ///   COPY_END1 = SH
+  ///   do i=1,COPY_END1
+  ///     result(i + DEST_OFFSET) = array(i)
+  ///   end
+  ///   SOURCE_OFFSET = SH
+  ///   COPY_END2 = SIZE(ARRAY,DIM) - SH
+  ///   do i=1,COPY_END2
+  ///     result(i) = array(i + SOURCE_OFFSET)
+  ///   end
+  /// Where SH is the normalized shift value, which satisfies
+  /// (SH >= 0 && SH < SIZE(ARRAY,DIM)).
+  ///
+  /// The converted code for EOSHIFT looks like this:
+  ///   EXTENT = SIZE(ARRAY,DIM)
+  ///   DEST_OFFSET = SH < 0 ? -SH : 0
+  ///   SOURCE_OFFSET = SH < 0 ? 0 : SH
+  ///   COPY_END = SH < 0 ?
+  ///       (-EXTENT > SH ? 0 : EXTENT + SH) :
+  ///       (EXTENT < SH ? 0 : EXTENT - SH)
+  ///   do i=1,COPY_END
+  ///     result(i + DEST_OFFSET) = array(i + SOURCE_OFFSET)
   ///   end
-  ///   do i=1,SIZE(ARRAY,DIM) - SH
-  ///     result(i) = array(i + SH)
+  ///   INIT_END = EXTENT - COPY_END
+  ///   INIT_OFFSET = SH < 0 ? 0 : COPY_END
+  ///   do i=1,INIT_END
+  ///     result(i + INIT_OFFSET) = BOUNDARY
   ///   end
+  /// Where SH is the original shift value.
   ///
   /// When \p dimVal is 1, we generate the same code twice
   /// under a dynamic check for the contiguity of the leading
@@ -1450,24 +1765,21 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
   /// as a contiguous slice of the original array.
   /// This allows recognizing the above two loops as memcpy
   /// loop idioms in LLVM.
-  static mlir::Operation *genInMemCShift(mlir::PatternRewriter &rewriter,
-                                         hlfir::CShiftOp cshift,
-                                         int64_t dimVal) {
+  static mlir::Operation *genInMemArrayShift(mlir::PatternRewriter &rewriter,
+                                             Op op, int64_t dimVal) {
     using Fortran::common::maxRank;
-    hlfir::Entity shift = hlfir::Entity{cshift.getShift()};
-    hlfir::Entity array = hlfir::Entity{cshift.getArray()};
+    hlfir::Entity shift = hlfir::Entity{op.getShift()};
+    hlfir::Entity array = hlfir::Entity{op.getArray()};
     assert(array.isVariable() && "array must be a variable");
     assert(!array.isPolymorphic() &&
-           "genInMemCShift does not support polymorphic types");
-    mlir::Location loc = cshift.getLoc();
-    fir::FirOpBuilder builder{rewriter, cshift.getOperation()};
+           "genInMemArrayShift does not support polymorphic types");
+    mlir::Location loc = op.getLoc();
+    fir::FirOpBuilder builder{rewriter, op.getOperation()};
     // The new index computation involves MODULO, which is not implemented
     // for IndexType, so use I64 instead.
     mlir::Type calcType = builder.getI64Type();
-    // All the indices arithmetic used below does not overflow
-    // signed and unsigned I64.
-    builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw |
-                                    mlir::arith::IntegerOverflowFlags::nuw);
+    // Set the indices arithmetic overflow flags.
+    setArithOverflowFlags(op, builder);
 
     mlir::Value arrayShape = hlfir::genShape(loc, builder, array);
     llvm::SmallVector<mlir::Value, maxRank> arrayExtents =
@@ -1482,10 +1794,20 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
       shiftVal =
           normalizeShiftValue(loc, builder, shiftVal, shiftDimExtent, calcType);
     }
+    // The boundary operand of hlfir.eoshift may be statically or
+    // dynamically absent.
+    // In both cases, it is assumed to be a scalar with the value
+    // corresponding to the array element type.
+    // boundaryIsScalarPred is a dynamic predicate that identifies
+    // these cases. If boundaryIsScalarPred is dynamicaly false,
+    // then the boundary operand must be a present array.
+    mlir::Value boundaryVal, boundaryIsScalarPred;
+    if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)
+      std::tie(boundaryVal, boundaryIsScalarPred) =
+          genScalarBoundaryForEOShift(loc, builder, op);
 
     hlfir::EvaluateInMemoryOp evalOp = hlfir::EvaluateInMemoryOp::create(
-        builder, loc, mlir::cast<hlfir::ExprType>(cshift.getType()),
-        arrayShape);
+        builder, loc, mlir::cast<hlfir::ExprType>(op.getType()), arrayShape);
     builder.setInsertionPointToStart(&evalOp.getBody().front());
 
     mlir::Value resultArray = evalOp.getMemory();
@@ -1499,11 +1821,12 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
     // (if any). If exposeContiguity is true, the array's section
     // array(s(1), ..., s(dim-1), :, s(dim+1), ..., s(n)) is represented
     // as a contiguous 1D array.
-    // shiftVal is the normalized shift value that satisfies (SH >= 0 && SH <
-    // SIZE(ARRAY,DIM)).
+    // For CSHIFT, shiftVal is the normalized shift value that satisfies
+    // (SH >= 0 && SH < SIZE(ARRAY,DIM)).
     //
     auto genDimensionShift = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                                 mlir::Value shiftVal, bool exposeContiguity,
+                                 mlir::Value shiftVal, mlir::Value boundary,
+                                 bool exposeContiguity,
                                  mlir::ValueRange oneBasedIndices)
         -> llvm::SmallVector<mlir::Value, 0> {
       // Create a vector of indices (s(1), ..., s(dim-1), nullptr, s(dim+1),
@@ -1536,63 +1859,143 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
         srcIndices.resize(1);
       }
 
-      // Copy first portion of the array:
-      // do i=1,SH
-      //   result(i + (SIZE(ARRAY,DIM) - SH)) = array(i)
-      // end
-      auto genAssign1 = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                            mlir::ValueRange index,
-                            mlir::ValueRange reductionArgs)
+      // genCopy labda generates the body of a generic copy loop.
+      //   do i=1,COPY_END
+      //     result(i + DEST_OFFSET) = array(i + SOURCE_OFFSET)
+      //   end
+      //
+      // It is parameterized by DEST_OFFSET and SOURCE_OFFSET.
+      mlir::Value dstOffset, srcOffset;
+      auto genCopy = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                         mlir::ValueRange index, mlir::ValueRange reductionArgs)
           -> llvm::SmallVector<mlir::Value, 0> {
         assert(index.size() == 1 && "expected single loop");
         mlir::Value srcIndex = builder.createConvert(loc, calcType, index[0]);
+        mlir::Value dstIndex = srcIndex;
+        if (srcOffset)
+          srcIndex =
+              mlir::arith::AddIOp::create(builder, loc, srcIndex, srcOffset);
         srcIndices[dimVal - 1] = srcIndex;
         hlfir::Entity srcElementValue =
             hlfir::loadElementAt(loc, builder, srcArray, srcIndices);
-        mlir::Value dstIndex = mlir::arith::AddIOp::create(
-            builder, loc, srcIndex,
-            mlir::arith::SubIOp::create(builder, loc, shiftDimExtent,
-                                        shiftVal));
-        dstIndices[dimVal - 1] = dstIndex;
-        hlfir::Entity dstElement = hlfir::getElementAt(
-            loc, builder, hlfir::Entity{resultArray}, dstIndices);
-        hlfir::AssignOp::create(builder, loc, srcElementValue, dstElement);
-        return {};
-      };
-
-      // Generate the first loop.
-      hlfir::genLoopNestWithReductions(loc, builder, {shiftVal},
-                                       /*reductionInits=*/{}, genAssign1,
-                                       /*isUnordered=*/true);
-
-      // Copy second portion of the array:
-      // do i=1,SIZE(ARRAY,DIM)-SH
-      //   result(i) = array(i + SH)
-      // end
-      auto genAssign2 = [&](mlir::Location loc, fir::FirOpBuilder &builder,
-                            mlir::ValueRange index,
-                            mlir::ValueRange reductionArgs)
-          -> llvm::SmallVector<mlir::Value, 0> {
-        assert(index.size() == 1 && "expected single loop");
-        mlir::Value dstIndex = builder.createConvert(loc, calcType, index[0]);
-        mlir::Value srcIndex =
-            mlir::arith::AddIOp::create(builder, loc, dstIndex, shiftVal);
-        srcIndices[dimVal - 1] = srcIndex;
-        hlfir::Entity srcElementValue =
-            hlfir::loadElementAt(loc, builder, srcArray, srcIndices);
+        if (dstOffset)
+          dstIndex =
+              mlir::arith::AddIOp::create(builder, loc, dstIndex, dstOffset);
         dstIndices[dimVal - 1] = dstIndex;
         hlfir::Entity dstElement = hlfir::getElementAt(
             loc, builder, hlfir::Entity{resultArray}, dstIndices);
         hlfir::AssignOp::create(builder, loc, srcElementValue, dstElement);
+        // Reset the external parameters' values to make sure
+        // they are properly updated between the labda calls.
+        // WARNING: if genLoopNestWithReductions() calls the lambda
+        // multiple times, this is going to be a problem.
+        dstOffset = nullptr;
+        srcOffset = nullptr;
         return {};
       };
 
-      // Generate the second loop.
-      mlir::Value bound =
-          mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);
-      hlfir::genLoopNestWithReductions(loc, builder, {bound},
-                                       /*reductionInits=*/{}, genAssign2,
-                                       /*isUnordered=*/true);
+      if constexpr (std::is_same_v<Op, hlfir::CShiftOp>) {
+        // Copy first portion of the array:
+        //   DEST_OFFSET = SIZE(ARRAY,DIM) - SH
+        //   COPY_END1 = SH
+        //   do i=1,COPY_END1
+        //     result(i + DEST_OFFSET) = array(i)
+        //   end
+        dstOffset =
+            mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);
+        srcOffset = nullptr;
+        hlfir::genLoopNestWithReductions(loc, builder, {shiftVal},
+                                         /*reductionInits=*/{}, genCopy,
+                                         /*isUnordered=*/true);
+
+        // Copy second portion of the array:
+        //   SOURCE_OFFSET = SH
+        //   COPY_END2 = SIZE(ARRAY,DIM) - SH
+        //   do i=1,COPY_END2
+        //     result(i) = array(i + SOURCE_OFFSET)
+        //   end
+        mlir::Value bound =
+            mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);
+        dstOffset = nullptr;
+        srcOffset = shiftVal;
+        hlfir::genLoopNestWithReductions(loc, builder, {bound},
+                                         /*reductionInits=*/{}, genCopy,
+                                         /*isUnordered=*/true);
+      } else {
+        // Do the copy:
+        //   EXTENT = SIZE(ARRAY,DIM)
+        //   DEST_OFFSET = SH < 0 ? -SH : 0
+        //   SOURCE_OFFSET = SH < 0 ? 0 : SH
+        //   COPY_END = SH < 0 ?
+        //       (-EXTENT > SH ? 0 : EXTENT + SH) :
+        //       (EXTENT < SH ? 0 : EXTENT - SH)
+        //   do i=1,COPY_END
+        //     result(i + DEST_OFFSET) = array(i + SOURCE_OFFSET)
+        //   end
+        mlir::arith::IntegerOverflowFlags savedFlags =
+            builder.getIntegerOverflowFlags();
+        builder.setIntegerOverflowFlags(mlir::arith::IntegerOverflowFlags::nsw);
+
+        mlir::Value zero = builder.createIntegerConstant(loc, calcType, 0);
+        mlir::Value isNegativeShift = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::slt, shiftVal, zero);
+        mlir::Value shiftNeg =
+            mlir::arith::SubIOp::create(builder, loc, zero, shiftVal);
+        dstOffset = mlir::arith::SelectOp::create(builder, loc, isNegativeShift,
+                                                  shiftNeg, zero);
+        srcOffset = mlir::arith::SelectOp::create(builder, loc, isNegativeShift,
+                                                  zero, shiftVal);
+        mlir::Value extentNeg =
+            mlir::arith::SubIOp::create(builder, loc, zero, shiftDimExtent);
+        mlir::Value extentPlusShift =
+            mlir::arith::AddIOp::create(builder, loc, shiftDimExtent, shiftVal);
+        mlir::Value extentNegShiftCmp = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::sgt, extentNeg, shiftVal);
+        mlir::Value negativeShiftBound = mlir::arith::SelectOp::create(
+            builder, loc, extentNegShiftCmp, zero, extentPlusShift);
+        mlir::Value extentMinusShift =
+            mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, shiftVal);
+        mlir::Value extentShiftCmp = mlir::arith::CmpIOp::create(
+            builder, loc, mlir::arith::CmpIPredicate::slt, shiftDimExtent,
+            shiftVal);
+        mlir::Value positiveShiftBound = mlir::arith::SelectOp::create(
+            builder, loc, extentShiftCmp, zero, extentMinusShift);
+        mlir::Value copyEnd = mlir::arith::SelectOp::create(
+            builder, loc, isNegativeShift, negativeShiftBound,
+            positiveShiftBound);
+        hlfir::genLoopNestWithReductions(loc, builder, {copyEnd},
+                                         /*reductionInits=*/{}, genCopy,
+                                         /*isUnordered=*/true);
+
+        // Do the init:
+        //   INIT_END = EXTENT - COPY_END
+        //   INIT_OFFSET = SH < 0 ? 0 : COPY_END
+        //   do i=1,INIT_END
+        //     result(i + INIT_OFFSET) = BOUNDARY
+        //   end
+        assert(boundary && "boundary cannot be null");
+        mlir::Value initEnd =
+            mlir::arith::SubIOp::create(builder, loc, shiftDimExtent, copyEnd);
+        mlir::Value initOffset = mlir::arith::SelectOp::create(
+            builder, loc, isNegativeShift, zero, copyEnd);
+        auto genInit = [&](mlir::Location loc, fir::FirOpBuilder &builder,
+                           mlir::ValueRange index,
+                           mlir::ValueRange reductionArgs)
+            -> llvm::SmallVector<mlir::Value, 0> {
+          mlir::Value dstIndex = builder.createConvert(loc, calcType, index[0]);
+          dstIndex =
+              mlir::arith::AddIOp::create(builder, loc, dstIndex, initOffset);
+          dstIndices[dimVal - 1] = dstIndex;
+          hlfir::Entity dstElement = hlfir::getElementAt(
+              loc, builder, hlfir::Entity{resultArray}, dstIndices);
+          hlfir::AssignOp::create(builder, loc, boundary, dstElement);
+          return {};
+        };
+        hlfir::genLoopNestWithReductions(loc, builder, {initEnd},
+                                         /*reductionInits=*/{}, genInit,
+                                         /*isUnordered=*/true);
+        builder.setIntegerOverflowFlags(savedFlags);
+      }
       return {};
     };
 
@@ -1614,6 +2017,10 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
         shiftVal = normalizeShiftValue(loc, builder, shiftVal, shiftDimExtent,
                                        calcType);
       }
+      if constexpr (std::is_same_v<Op, hlfir::EOShiftOp>)
+        boundaryVal =
+            selectBoundaryValue(loc, builder, op, boundaryVal,
+                                boundaryIsScalarPred, oneBasedIndices);
 
       // If we can fetch the byte stride of the leading dimension,
       // and the byte size of the element, then we can generate
@@ -1635,8 +2042,8 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
       }
 
       if (array.isSimplyContiguous() || !elemSize || !stride) {
-        genDimensionShift(loc, builder, shiftVal, /*exposeContiguity=*/false,
-                          oneBasedIndices);
+        genDimensionShift(loc, builder, shiftVal, boundaryVal,
+                          /*exposeContiguity=*/false, oneBasedIndices);
         return {};
       }
 
@@ -1644,11 +2051,11 @@ class CShiftConversion : public mlir::OpRewritePattern<hlfir::CShiftOp> {
           builder, loc, mlir::arith::CmpIPredicate::eq, elemSize, stride);
       builder.genIfOp(loc, {}, isContiguous, /*withElseRegion=*/true)
           .genThen([&]() {
-            genDimensionShift(loc, builder, shiftVal, /*exposeContiguity=*/true,
-                              oneBasedIndices);
+            genDimensionShift(loc, builder, shiftVal, boundaryVal,
+                              /*exposeContiguity=*/true, oneBasedIndices);
           })
           .genElse([&]() {
-            genDimensionShift(loc, builder, shiftVal,
+            genDimensionShift(loc, builder, shiftVal, boundaryVal,
                               /*exposeContiguity=*/false, oneBasedIndices);
           });
 
@@ -2339,7 +2746,8 @@ class SimplifyHLFIRIntrinsics
     mlir::RewritePatternSet patterns(context);
     patterns.insert<TransposeAsElementalConversion>(context);
     patterns.insert<ReductionConversion<hlfir::SumOp>>(context);
-    patterns.insert<CShiftConversion>(context);
+    patterns.insert<ArrayShiftConversion<hlfir::CShiftOp>>(context);
+    patterns.insert<ArrayShiftConversion<hlfir::EOShiftOp>>(context);
     patterns.insert<MatmulConversion<hlfir::MatmulTransposeOp>>(context);
 
     patterns.insert<ReductionConversion<hlfir::CountOp>>(context);
diff --git a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
index e5fd19d4622b6..5b6d904fb0d59 100644
--- a/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
+++ b/flang/lib/Optimizer/OpenACC/Support/FIROpenACCTypeInterfaces.cpp
@@ -591,7 +591,8 @@ mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
           hlfir::AssignOp::create(firBuilder, loc, initVal,
                                   declareOp.getBase());
         } else {
-          for (auto ext : seqTy.getShape()) {
+          // Generate loop nest from slowest to fastest running dimension
+          for (auto ext : llvm::reverse(seqTy.getShape())) {
             auto lb = firBuilder.createIntegerConstant(loc, idxTy, 0);
             auto ub = firBuilder.createIntegerConstant(loc, idxTy, ext - 1);
             auto step = firBuilder.createIntegerConstant(loc, idxTy, 1);
@@ -614,6 +615,11 @@ mlir::Value OpenACCMappableModel<Ty>::generatePrivateInit(
     mlir::Type innerTy = fir::unwrapRefType(boxTy.getEleTy());
     if (fir::isa_trivial(innerTy)) {
       retVal = getDeclareOpForType(unwrappedTy).getBase();
+      mlir::Value allocatedScalar =
+          fir::AllocMemOp::create(builder, loc, innerTy);
+      mlir::Value firClass =
+          fir::EmboxOp::create(builder, loc, boxTy, allocatedScalar);
+      fir::StoreOp::create(builder, loc, firClass, retVal);
     } else if (mlir::isa<fir::SequenceType>(innerTy)) {
       hlfir::Entity source = hlfir::Entity{var};
       auto [temp, cleanup] = hlfir::createTempFromMold(loc, firBuilder, source);
diff --git a/flang/lib/Optimizer/OpenMP/AutomapToTargetData.cpp b/flang/lib/Optimizer/OpenMP/AutomapToTargetData.cpp
new file mode 100644
index 0000000000000..8b9991301aae8
--- /dev/null
+++ b/flang/lib/Optimizer/OpenMP/AutomapToTargetData.cpp
@@ -0,0 +1,159 @@
+//===- AutomapToTargetData.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Builder/DirectivesCommon.h"
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/HLFIRTools.h"
+#include "flang/Optimizer/Dialect/FIROps.h"
+#include "flang/Optimizer/Dialect/FIRType.h"
+#include "flang/Optimizer/Dialect/Support/KindMapping.h"
+#include "flang/Optimizer/HLFIR/HLFIROps.h"
+
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/Dialect/OpenMP/OpenMPInterfaces.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/Pass/Pass.h"
+
+#include "llvm/Frontend/OpenMP/OMPConstants.h"
+
+namespace flangomp {
+#define GEN_PASS_DEF_AUTOMAPTOTARGETDATAPASS
+#include "flang/Optimizer/OpenMP/Passes.h.inc"
+} // namespace flangomp
+
+using namespace mlir;
+
+namespace {
+class AutomapToTargetDataPass
+    : public flangomp::impl::AutomapToTargetDataPassBase<
+          AutomapToTargetDataPass> {
+
+  // Returns true if the variable has a dynamic size and therefore requires
+  // bounds operations to describe its extents.
+  inline bool needsBoundsOps(mlir::Value var) {
+    assert(mlir::isa<mlir::omp::PointerLikeType>(var.getType()) &&
+           "only pointer like types expected");
+    mlir::Type t = fir::unwrapRefType(var.getType());
+    if (mlir::Type inner = fir::dyn_cast_ptrOrBoxEleTy(t))
+      return fir::hasDynamicSize(inner);
+    return fir::hasDynamicSize(t);
+  }
+
+  // Generate MapBoundsOp operations for the variable if required.
+  inline void genBoundsOps(fir::FirOpBuilder &builder, mlir::Value var,
+                           llvm::SmallVectorImpl<mlir::Value> &boundsOps) {
+    mlir::Location loc = var.getLoc();
+    fir::factory::AddrAndBoundsInfo info =
+        fir::factory::getDataOperandBaseAddr(builder, var,
+                                             /*isOptional=*/false, loc);
+    fir::ExtendedValue exv =
+        hlfir::translateToExtendedValue(loc, builder, hlfir::Entity{info.addr},
+                                        /*contiguousHint=*/true)
+            .first;
+    llvm::SmallVector<mlir::Value> tmp =
+        fir::factory::genImplicitBoundsOps<mlir::omp::MapBoundsOp,
+                                           mlir::omp::MapBoundsType>(
+            builder, info, exv, /*dataExvIsAssumedSize=*/false, loc);
+    llvm::append_range(boundsOps, tmp);
+  }
+
+  void findRelatedAllocmemFreemem(fir::AddrOfOp addressOfOp,
+                                  llvm::DenseSet<fir::StoreOp> &allocmems,
+                                  llvm::DenseSet<fir::LoadOp> &freemems) {
+    assert(addressOfOp->hasOneUse() && "op must have single use");
+
+    auto declaredRef =
+        cast<hlfir::DeclareOp>(*addressOfOp->getUsers().begin())->getResult(0);
+
+    for (Operation *refUser : declaredRef.getUsers()) {
+      if (auto storeOp = dyn_cast<fir::StoreOp>(refUser))
+        if (auto emboxOp = storeOp.getValue().getDefiningOp<fir::EmboxOp>())
+          if (auto allocmemOp =
+                  emboxOp.getOperand(0).getDefiningOp<fir::AllocMemOp>())
+            allocmems.insert(storeOp);
+
+      if (auto loadOp = dyn_cast<fir::LoadOp>(refUser))
+        for (Operation *loadUser : loadOp.getResult().getUsers())
+          if (auto boxAddrOp = dyn_cast<fir::BoxAddrOp>(loadUser))
+            for (Operation *boxAddrUser : boxAddrOp.getResult().getUsers())
+              if (auto freememOp = dyn_cast<fir::FreeMemOp>(boxAddrUser))
+                freemems.insert(loadOp);
+    }
+  }
+
+  void runOnOperation() override {
+    ModuleOp module = getOperation()->getParentOfType<ModuleOp>();
+    if (!module)
+      module = dyn_cast<ModuleOp>(getOperation());
+    if (!module)
+      return;
+
+    // Build FIR builder for helper utilities.
+    fir::KindMapping kindMap = fir::getKindMapping(module);
+    fir::FirOpBuilder builder{module, std::move(kindMap)};
+
+    // Collect global variables with AUTOMAP flag.
+    llvm::DenseSet<fir::GlobalOp> automapGlobals;
+    module.walk([&](fir::GlobalOp globalOp) {
+      if (auto iface =
+              dyn_cast<omp::DeclareTargetInterface>(globalOp.getOperation()))
+        if (iface.isDeclareTarget() && iface.getDeclareTargetAutomap() &&
+            iface.getDeclareTargetDeviceType() !=
+                omp::DeclareTargetDeviceType::host)
+          automapGlobals.insert(globalOp);
+    });
+
+    auto addMapInfo = [&](auto globalOp, auto memOp) {
+      builder.setInsertionPointAfter(memOp);
+      SmallVector<Value> bounds;
+      if (needsBoundsOps(memOp.getMemref()))
+        genBoundsOps(builder, memOp.getMemref(), bounds);
+
+      omp::TargetEnterExitUpdateDataOperands clauses;
+      mlir::omp::MapInfoOp mapInfo = mlir::omp::MapInfoOp::create(
+          builder, memOp.getLoc(), memOp.getMemref().getType(),
+          memOp.getMemref(),
+          TypeAttr::get(fir::unwrapRefType(memOp.getMemref().getType())),
+          builder.getIntegerAttr(
+              builder.getIntegerType(64, false),
+              static_cast<unsigned>(
+                  isa<fir::StoreOp>(memOp)
+                      ? llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO
+                      : llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE)),
+          builder.getAttr<omp::VariableCaptureKindAttr>(
+              omp::VariableCaptureKind::ByCopy),
+          /*var_ptr_ptr=*/mlir::Value{},
+          /*members=*/SmallVector<Value>{},
+          /*members_index=*/ArrayAttr{}, bounds,
+          /*mapperId=*/mlir::FlatSymbolRefAttr(), globalOp.getSymNameAttr(),
+          builder.getBoolAttr(false));
+      clauses.mapVars.push_back(mapInfo);
+      isa<fir::StoreOp>(memOp)
+          ? builder.create<omp::TargetEnterDataOp>(memOp.getLoc(), clauses)
+          : builder.create<omp::TargetExitDataOp>(memOp.getLoc(), clauses);
+    };
+
+    for (fir::GlobalOp globalOp : automapGlobals) {
+      if (auto uses = globalOp.getSymbolUses(module.getOperation())) {
+        llvm::DenseSet<fir::StoreOp> allocmemStores;
+        llvm::DenseSet<fir::LoadOp> freememLoads;
+        for (auto &x : *uses)
+          if (auto addrOp = dyn_cast<fir::AddrOfOp>(x.getUser()))
+            findRelatedAllocmemFreemem(addrOp, allocmemStores, freememLoads);
+
+        for (auto storeOp : allocmemStores)
+          addMapInfo(globalOp, storeOp);
+
+        for (auto loadOp : freememLoads)
+          addMapInfo(globalOp, loadOp);
+      }
+    }
+  }
+};
+} // namespace
diff --git a/flang/lib/Optimizer/OpenMP/CMakeLists.txt b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
index e31543328a9f9..e0aebd0714c8f 100644
--- a/flang/lib/Optimizer/OpenMP/CMakeLists.txt
+++ b/flang/lib/Optimizer/OpenMP/CMakeLists.txt
@@ -1,6 +1,7 @@
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 
 add_flang_library(FlangOpenMPTransforms
+  AutomapToTargetData.cpp
   DoConcurrentConversion.cpp
   FunctionFiltering.cpp
   GenericLoopConversion.cpp
@@ -9,6 +10,7 @@ add_flang_library(FlangOpenMPTransforms
   MarkDeclareTarget.cpp
   LowerWorkshare.cpp
   LowerNontemporal.cpp
+  SimdOnly.cpp
 
   DEPENDS
   FIRDialect
diff --git a/flang/lib/Optimizer/OpenMP/FunctionFiltering.cpp b/flang/lib/Optimizer/OpenMP/FunctionFiltering.cpp
index ae5c0ecc5b7f6..3031bb5da6919 100644
--- a/flang/lib/Optimizer/OpenMP/FunctionFiltering.cpp
+++ b/flang/lib/Optimizer/OpenMP/FunctionFiltering.cpp
@@ -95,8 +95,9 @@ class FunctionFilteringPass
           return WalkResult::skip();
         }
         if (declareTargetOp)
-          declareTargetOp.setDeclareTarget(declareType,
-                                           omp::DeclareTargetCaptureClause::to);
+          declareTargetOp.setDeclareTarget(
+              declareType, omp::DeclareTargetCaptureClause::to,
+              declareTargetOp.getDeclareTargetAutomap());
       }
       return WalkResult::advance();
     });
diff --git a/flang/lib/Optimizer/OpenMP/MarkDeclareTarget.cpp b/flang/lib/Optimizer/OpenMP/MarkDeclareTarget.cpp
index a7ffd5fda82b7..0b0e6bd9ecf34 100644
--- a/flang/lib/Optimizer/OpenMP/MarkDeclareTarget.cpp
+++ b/flang/lib/Optimizer/OpenMP/MarkDeclareTarget.cpp
@@ -33,7 +33,7 @@ class MarkDeclareTargetPass
 
   void markNestedFuncs(mlir::omp::DeclareTargetDeviceType parentDevTy,
                        mlir::omp::DeclareTargetCaptureClause parentCapClause,
-                       mlir::Operation *currOp,
+                       bool parentAutomap, mlir::Operation *currOp,
                        llvm::SmallPtrSet<mlir::Operation *, 16> visited) {
     if (visited.contains(currOp))
       return;
@@ -57,13 +57,16 @@ class MarkDeclareTargetPass
                   currentDt != mlir::omp::DeclareTargetDeviceType::any) {
                 current.setDeclareTarget(
                     mlir::omp::DeclareTargetDeviceType::any,
-                    current.getDeclareTargetCaptureClause());
+                    current.getDeclareTargetCaptureClause(),
+                    current.getDeclareTargetAutomap());
               }
             } else {
-              current.setDeclareTarget(parentDevTy, parentCapClause);
+              current.setDeclareTarget(parentDevTy, parentCapClause,
+                                       parentAutomap);
             }
 
-            markNestedFuncs(parentDevTy, parentCapClause, currFOp, visited);
+            markNestedFuncs(parentDevTy, parentCapClause, parentAutomap,
+                            currFOp, visited);
           }
         }
       }
@@ -81,7 +84,8 @@ class MarkDeclareTargetPass
         llvm::SmallPtrSet<mlir::Operation *, 16> visited;
         markNestedFuncs(declareTargetOp.getDeclareTargetDeviceType(),
                         declareTargetOp.getDeclareTargetCaptureClause(),
-                        functionOp, visited);
+                        declareTargetOp.getDeclareTargetAutomap(), functionOp,
+                        visited);
       }
     }
 
@@ -92,9 +96,10 @@ class MarkDeclareTargetPass
     // the contents of the device clause
     getOperation()->walk([&](mlir::omp::TargetOp tarOp) {
       llvm::SmallPtrSet<mlir::Operation *, 16> visited;
-      markNestedFuncs(mlir::omp::DeclareTargetDeviceType::nohost,
-                      mlir::omp::DeclareTargetCaptureClause::to, tarOp,
-                      visited);
+      markNestedFuncs(
+          /*parentDevTy=*/mlir::omp::DeclareTargetDeviceType::nohost,
+          /*parentCapClause=*/mlir::omp::DeclareTargetCaptureClause::to,
+          /*parentAutomap=*/false, tarOp, visited);
     });
   }
 };
diff --git a/flang/lib/Optimizer/OpenMP/SimdOnly.cpp b/flang/lib/Optimizer/OpenMP/SimdOnly.cpp
new file mode 100644
index 0000000000000..4a559d296f502
--- /dev/null
+++ b/flang/lib/Optimizer/OpenMP/SimdOnly.cpp
@@ -0,0 +1,209 @@
+//===-- SimdOnly.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Support/LLVM.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/Support/Debug.h"
+
+namespace flangomp {
+#define GEN_PASS_DEF_SIMDONLYPASS
+#include "flang/Optimizer/OpenMP/Passes.h.inc"
+} // namespace flangomp
+
+namespace {
+
+#define DEBUG_TYPE "omp-simd-only-pass"
+
+/// Rewrite and remove OpenMP operations left after the parse tree rewriting for
+/// -fopenmp-simd is done. If possible, OpenMP constructs should be rewritten at
+/// the parse tree stage. This pass is supposed to only handle complexities
+/// around untangling composite simd constructs, and perform the necessary
+/// cleanup.
+class SimdOnlyConversionPattern : public mlir::RewritePattern {
+public:
+  SimdOnlyConversionPattern(mlir::MLIRContext *ctx)
+      : mlir::RewritePattern(MatchAnyOpTypeTag{}, 1, ctx) {}
+
+  mlir::LogicalResult
+  matchAndRewrite(mlir::Operation *op,
+                  mlir::PatternRewriter &rewriter) const override {
+    if (op->getDialect()->getNamespace() !=
+        mlir::omp::OpenMPDialect::getDialectNamespace())
+      return rewriter.notifyMatchFailure(op, "Not an OpenMP op");
+
+    if (auto simdOp = mlir::dyn_cast<mlir::omp::SimdOp>(op)) {
+      // Remove the composite attr given that the op will no longer be composite
+      if (simdOp.isComposite()) {
+        simdOp.setComposite(false);
+        return mlir::success();
+      }
+
+      return rewriter.notifyMatchFailure(op, "Op is a plain SimdOp");
+    }
+
+    if (op->getParentOfType<mlir::omp::SimdOp>() &&
+        (mlir::isa<mlir::omp::YieldOp>(op) ||
+         mlir::isa<mlir::omp::ScanOp>(op) ||
+         mlir::isa<mlir::omp::LoopNestOp>(op) ||
+         mlir::isa<mlir::omp::TerminatorOp>(op)))
+      return rewriter.notifyMatchFailure(op, "Op is part of a simd construct");
+
+    if (!mlir::isa<mlir::func::FuncOp>(op->getParentOp()) &&
+        (mlir::isa<mlir::omp::TerminatorOp>(op) ||
+         mlir::isa<mlir::omp::YieldOp>(op)))
+      return rewriter.notifyMatchFailure(op,
+                                         "Non top-level yield or terminator");
+
+    LLVM_DEBUG(llvm::dbgs() << "SimdOnlyPass matched OpenMP op:\n");
+    LLVM_DEBUG(op->dump());
+
+    auto eraseUnlessUsedBySimd = [&](mlir::Operation *ompOp,
+                                     mlir::StringAttr name) {
+      if (auto uses =
+              mlir::SymbolTable::getSymbolUses(name, op->getParentOp())) {
+        for (auto &use : *uses)
+          if (mlir::isa<mlir::omp::SimdOp>(use.getUser()))
+            return rewriter.notifyMatchFailure(op,
+                                               "Op used by a simd construct");
+      }
+      rewriter.eraseOp(ompOp);
+      return mlir::success();
+    };
+
+    if (auto ompOp = mlir::dyn_cast<mlir::omp::PrivateClauseOp>(op))
+      return eraseUnlessUsedBySimd(ompOp, ompOp.getSymNameAttr());
+    if (auto ompOp = mlir::dyn_cast<mlir::omp::DeclareReductionOp>(op))
+      return eraseUnlessUsedBySimd(ompOp, ompOp.getSymNameAttr());
+
+    // Might be left over from rewriting composite simd with target map
+    if (mlir::isa<mlir::omp::MapBoundsOp>(op)) {
+      rewriter.eraseOp(op);
+      return mlir::success();
+    }
+    if (auto mapInfoOp = mlir::dyn_cast<mlir::omp::MapInfoOp>(op)) {
+      rewriter.replaceOp(mapInfoOp, {mapInfoOp.getVarPtr()});
+      return mlir::success();
+    }
+
+    // Might be leftover after parse tree rewriting
+    if (auto threadPrivateOp = mlir::dyn_cast<mlir::omp::ThreadprivateOp>(op)) {
+      rewriter.replaceOp(threadPrivateOp, {threadPrivateOp.getSymAddr()});
+      return mlir::success();
+    }
+
+    fir::FirOpBuilder builder(rewriter, op);
+    mlir::Location loc = op->getLoc();
+
+    auto inlineSimpleOp = [&](mlir::Operation *ompOp) -> bool {
+      if (!ompOp)
+        return false;
+
+      assert("OpenMP operation has one region" && ompOp->getNumRegions() == 1);
+
+      llvm::SmallVector<std::pair<mlir::Value, mlir::BlockArgument>>
+          blockArgsPairs;
+      if (auto iface =
+              mlir::dyn_cast<mlir::omp::BlockArgOpenMPOpInterface>(op)) {
+        iface.getBlockArgsPairs(blockArgsPairs);
+        for (auto [value, argument] : blockArgsPairs)
+          rewriter.replaceAllUsesWith(argument, value);
+      }
+
+      if (ompOp->getRegion(0).getBlocks().size() == 1) {
+        auto &block = *ompOp->getRegion(0).getBlocks().begin();
+        // This block is about to be removed so any arguments should have been
+        // replaced by now.
+        block.eraseArguments(0, block.getNumArguments());
+        if (auto terminatorOp =
+                mlir::dyn_cast<mlir::omp::TerminatorOp>(block.back())) {
+          rewriter.eraseOp(terminatorOp);
+        }
+        rewriter.inlineBlockBefore(&block, ompOp, {});
+      } else {
+        // When dealing with multi-block regions we need to fix up the control
+        // flow
+        auto *origBlock = ompOp->getBlock();
+        auto *newBlock = rewriter.splitBlock(origBlock, ompOp->getIterator());
+        auto *innerFrontBlock = &ompOp->getRegion(0).getBlocks().front();
+        builder.setInsertionPointToEnd(origBlock);
+        mlir::cf::BranchOp::create(builder, loc, innerFrontBlock);
+        // We are no longer passing any arguments to the first block in the
+        // region, so this should be safe to erase.
+        innerFrontBlock->eraseArguments(0, innerFrontBlock->getNumArguments());
+
+        for (auto &innerBlock : ompOp->getRegion(0).getBlocks()) {
+          // Remove now-unused block arguments
+          for (auto arg : innerBlock.getArguments()) {
+            if (arg.getUses().empty())
+              innerBlock.eraseArgument(arg.getArgNumber());
+          }
+          if (auto terminatorOp =
+                  mlir::dyn_cast<mlir::omp::TerminatorOp>(innerBlock.back())) {
+            builder.setInsertionPointToEnd(&innerBlock);
+            mlir::cf::BranchOp::create(builder, loc, newBlock);
+            rewriter.eraseOp(terminatorOp);
+          }
+        }
+
+        rewriter.inlineRegionBefore(ompOp->getRegion(0), newBlock);
+      }
+
+      rewriter.eraseOp(op);
+      return true;
+    };
+
+    // Remove ops that will be surrounding simd once a composite simd construct
+    // goes through the codegen stage. All of the other ones should have alredy
+    // been removed in the parse tree rewriting stage.
+    if (inlineSimpleOp(mlir::dyn_cast<mlir::omp::TeamsOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::ParallelOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::TargetOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::WsloopOp>(op)) ||
+        inlineSimpleOp(mlir::dyn_cast<mlir::omp::DistributeOp>(op)))
+      return mlir::success();
+
+    op->emitOpError("left unhandled after SimdOnly pass.");
+    return mlir::failure();
+  }
+};
+
+class SimdOnlyPass : public flangomp::impl::SimdOnlyPassBase<SimdOnlyPass> {
+
+public:
+  SimdOnlyPass() = default;
+
+  void runOnOperation() override {
+    mlir::ModuleOp module = getOperation();
+
+    mlir::MLIRContext *context = &getContext();
+    mlir::RewritePatternSet patterns(context);
+    patterns.insert<SimdOnlyConversionPattern>(context);
+
+    mlir::GreedyRewriteConfig config;
+    // Prevent the pattern driver from merging blocks.
+    config.setRegionSimplificationLevel(
+        mlir::GreedySimplifyRegionLevel::Disabled);
+
+    if (mlir::failed(
+            mlir::applyPatternsGreedily(module, std::move(patterns), config))) {
+      mlir::emitError(module.getLoc(), "Error in SimdOnly conversion pass");
+      signalPassFailure();
+    }
+  }
+};
+
+} // namespace
diff --git a/flang/lib/Optimizer/Passes/Pipelines.cpp b/flang/lib/Optimizer/Passes/Pipelines.cpp
index ca8e820608688..98f947a1f635d 100644
--- a/flang/lib/Optimizer/Passes/Pipelines.cpp
+++ b/flang/lib/Optimizer/Passes/Pipelines.cpp
@@ -242,7 +242,8 @@ void createDefaultFIROptimizerPassPipeline(mlir::PassManager &pm,
 /// \param pm - MLIR pass manager that will hold the pipeline definition
 /// \param optLevel - optimization level used for creating FIR optimization
 ///   passes pipeline
-void createHLFIRToFIRPassPipeline(mlir::PassManager &pm, bool enableOpenMP,
+void createHLFIRToFIRPassPipeline(mlir::PassManager &pm,
+                                  EnableOpenMP enableOpenMP,
                                   llvm::OptimizationLevel optLevel) {
   if (optLevel.isOptimizingForSpeed()) {
     addCanonicalizerPassWithoutRegionSimplification(pm);
@@ -294,8 +295,10 @@ void createHLFIRToFIRPassPipeline(mlir::PassManager &pm, bool enableOpenMP,
     addNestedPassToAllTopLevelOperations<PassConstructor>(
         pm, hlfir::createInlineHLFIRAssign);
   pm.addPass(hlfir::createConvertHLFIRtoFIR());
-  if (enableOpenMP)
+  if (enableOpenMP != EnableOpenMP::None)
     pm.addPass(flangomp::createLowerWorkshare());
+  if (enableOpenMP == EnableOpenMP::Simd)
+    pm.addPass(flangomp::createSimdOnlyPass());
 }
 
 /// Create a pass pipeline for handling certain OpenMP transformations needed
@@ -316,13 +319,13 @@ void createOpenMPFIRPassPipeline(mlir::PassManager &pm,
     pm.addPass(flangomp::createDoConcurrentConversionPass(
         opts.doConcurrentMappingKind == DoConcurrentMappingKind::DCMK_Device));
 
-  // The MapsForPrivatizedSymbols pass needs to run before
-  // MapInfoFinalizationPass because the former creates new
-  // MapInfoOp instances, typically for descriptors.
-  // MapInfoFinalizationPass adds MapInfoOp instances for the descriptors
-  // underlying data which is necessary to access the data on the offload
-  // target device.
+  // The MapsForPrivatizedSymbols and AutomapToTargetDataPass pass need to run
+  // before MapInfoFinalizationPass because they create new MapInfoOp
+  // instances, typically for descriptors. MapInfoFinalizationPass adds
+  // MapInfoOp instances for the descriptors underlying data which is necessary
+  // to access the data on the offload target device.
   pm.addPass(flangomp::createMapsForPrivatizedSymbolsPass());
+  pm.addPass(flangomp::createAutomapToTargetDataPass());
   pm.addPass(flangomp::createMapInfoFinalizationPass());
   pm.addPass(flangomp::createMarkDeclareTargetPass());
   pm.addPass(flangomp::createGenericLoopConversionPass());
@@ -396,7 +399,12 @@ void createDefaultFIRCodeGenPassPipeline(mlir::PassManager &pm,
 void createMLIRToLLVMPassPipeline(mlir::PassManager &pm,
                                   MLIRToLLVMPassPipelineConfig &config,
                                   llvm::StringRef inputFilename) {
-  fir::createHLFIRToFIRPassPipeline(pm, config.EnableOpenMP, config.OptLevel);
+  fir::EnableOpenMP enableOpenMP = fir::EnableOpenMP::None;
+  if (config.EnableOpenMP)
+    enableOpenMP = fir::EnableOpenMP::Full;
+  if (config.EnableOpenMPSimd)
+    enableOpenMP = fir::EnableOpenMP::Simd;
+  fir::createHLFIRToFIRPassPipeline(pm, enableOpenMP, config.OptLevel);
 
   // Add default optimizer pass pipeline.
   fir::createDefaultFIROptimizerPassPipeline(pm, config);
diff --git a/flang/lib/Optimizer/Support/Utils.cpp b/flang/lib/Optimizer/Support/Utils.cpp
index 5d663e28336c0..c71642ce4e806 100644
--- a/flang/lib/Optimizer/Support/Utils.cpp
+++ b/flang/lib/Optimizer/Support/Utils.cpp
@@ -50,3 +50,74 @@ std::optional<llvm::ArrayRef<int64_t>> fir::getComponentLowerBoundsIfNonDefault(
       return componentInfo.getLowerBounds();
   return std::nullopt;
 }
+
+mlir::LLVM::ConstantOp
+fir::genConstantIndex(mlir::Location loc, mlir::Type ity,
+                      mlir::ConversionPatternRewriter &rewriter,
+                      std::int64_t offset) {
+  auto cattr = rewriter.getI64IntegerAttr(offset);
+  return rewriter.create<mlir::LLVM::ConstantOp>(loc, ity, cattr);
+}
+
+mlir::Value
+fir::computeElementDistance(mlir::Location loc, mlir::Type llvmObjectType,
+                            mlir::Type idxTy,
+                            mlir::ConversionPatternRewriter &rewriter,
+                            const mlir::DataLayout &dataLayout) {
+  llvm::TypeSize size = dataLayout.getTypeSize(llvmObjectType);
+  unsigned short alignment = dataLayout.getTypeABIAlignment(llvmObjectType);
+  std::int64_t distance = llvm::alignTo(size, alignment);
+  return fir::genConstantIndex(loc, idxTy, rewriter, distance);
+}
+
+mlir::Value
+fir::genAllocationScaleSize(mlir::Location loc, mlir::Type dataTy,
+                            mlir::Type ity,
+                            mlir::ConversionPatternRewriter &rewriter) {
+  auto seqTy = mlir::dyn_cast<fir::SequenceType>(dataTy);
+  fir::SequenceType::Extent constSize = 1;
+  if (seqTy) {
+    int constRows = seqTy.getConstantRows();
+    const fir::SequenceType::ShapeRef &shape = seqTy.getShape();
+    if (constRows != static_cast<int>(shape.size())) {
+      for (auto extent : shape) {
+        if (constRows-- > 0)
+          continue;
+        if (extent != fir::SequenceType::getUnknownExtent())
+          constSize *= extent;
+      }
+    }
+  }
+
+  if (constSize != 1) {
+    mlir::Value constVal{
+        fir::genConstantIndex(loc, ity, rewriter, constSize).getResult()};
+    return constVal;
+  }
+  return nullptr;
+}
+
+mlir::Value fir::integerCast(const fir::LLVMTypeConverter &converter,
+                             mlir::Location loc,
+                             mlir::ConversionPatternRewriter &rewriter,
+                             mlir::Type ty, mlir::Value val, bool fold) {
+  auto valTy = val.getType();
+  // If the value was not yet lowered, lower its type so that it can
+  // be used in getPrimitiveTypeSizeInBits.
+  if (!mlir::isa<mlir::IntegerType>(valTy))
+    valTy = converter.convertType(valTy);
+  auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty);
+  auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy);
+  if (fold) {
+    if (toSize < fromSize)
+      return rewriter.createOrFold<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.createOrFold<mlir::LLVM::SExtOp>(loc, ty, val);
+  } else {
+    if (toSize < fromSize)
+      return rewriter.create<mlir::LLVM::TruncOp>(loc, ty, val);
+    if (toSize > fromSize)
+      return rewriter.create<mlir::LLVM::SExtOp>(loc, ty, val);
+  }
+  return val;
+}
diff --git a/flang/lib/Optimizer/Transforms/AffineDemotion.cpp b/flang/lib/Optimizer/Transforms/AffineDemotion.cpp
index f1c66a5bbcf8c..430ef62a3a55d 100644
--- a/flang/lib/Optimizer/Transforms/AffineDemotion.cpp
+++ b/flang/lib/Optimizer/Transforms/AffineDemotion.cpp
@@ -117,10 +117,7 @@ class ConvertConversion : public mlir::OpRewritePattern<fir::ConvertOp> {
                                                       op.getValue());
           return success();
         }
-      rewriter.startOpModification(op->getParentOp());
-      op.getResult().replaceAllUsesWith(op.getValue());
-      rewriter.finalizeOpModification(op->getParentOp());
-      rewriter.eraseOp(op);
+      rewriter.replaceOp(op, op.getValue());
     }
     return success();
   }
diff --git a/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp b/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp
index 247ba953f3265..ed9a2ae11f0d9 100644
--- a/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp
+++ b/flang/lib/Optimizer/Transforms/ArrayValueCopy.cpp
@@ -1264,7 +1264,6 @@ class ArrayUpdateConversion : public ArrayUpdateConversionBase<ArrayUpdateOp> {
     auto lhsEltRefType = toRefType(update.getMerge().getType());
     auto [_, lhsLoadResult] = materializeAssignment(
         loc, rewriter, update, assignElement, lhsEltRefType);
-    update.replaceAllUsesWith(lhsLoadResult);
     rewriter.replaceOp(update, lhsLoadResult);
     return mlir::success();
   }
@@ -1287,7 +1286,6 @@ class ArrayModifyConversion : public ArrayUpdateConversionBase<ArrayModifyOp> {
     auto lhsEltRefType = modify.getResult(0).getType();
     auto [lhsEltCoor, lhsLoadResult] = materializeAssignment(
         loc, rewriter, modify, assignElement, lhsEltRefType);
-    modify.replaceAllUsesWith(mlir::ValueRange{lhsEltCoor, lhsLoadResult});
     rewriter.replaceOp(modify, mlir::ValueRange{lhsEltCoor, lhsLoadResult});
     return mlir::success();
   }
@@ -1339,7 +1337,6 @@ class ArrayAccessConversion : public ArrayUpdateConversionBase<ArrayAccessOp> {
       // This array_access is associated with an array_amend and there is a
       // conflict. Make a copy to store into.
       auto result = referenceToClone(loc, rewriter, access);
-      access.replaceAllUsesWith(result);
       rewriter.replaceOp(access, result);
       return mlir::success();
     }
diff --git a/flang/lib/Optimizer/Transforms/CUFComputeSharedMemoryOffsetsAndSize.cpp b/flang/lib/Optimizer/Transforms/CUFComputeSharedMemoryOffsetsAndSize.cpp
index 5e910f7da6472..6e04c71c41606 100644
--- a/flang/lib/Optimizer/Transforms/CUFComputeSharedMemoryOffsetsAndSize.cpp
+++ b/flang/lib/Optimizer/Transforms/CUFComputeSharedMemoryOffsetsAndSize.cpp
@@ -38,6 +38,15 @@ using namespace Fortran::runtime::cuda;
 
 namespace {
 
+static bool isAssumedSize(mlir::ValueRange shape) {
+  if (shape.size() != 1)
+    return false;
+  std::optional<std::int64_t> val = fir::getIntIfConstant(shape[0]);
+  if (val && *val == -1)
+    return true;
+  return false;
+}
+
 struct CUFComputeSharedMemoryOffsetsAndSize
     : public fir::impl::CUFComputeSharedMemoryOffsetsAndSizeBase<
           CUFComputeSharedMemoryOffsetsAndSize> {
@@ -82,12 +91,12 @@ struct CUFComputeSharedMemoryOffsetsAndSize
           alignment = std::max(alignment, align);
           uint64_t tySize = dl->getTypeSize(ty);
           ++nbDynamicSharedVariables;
-          if (crtDynOffset) {
-            sharedOp.getOffsetMutable().assign(
-                builder.createConvert(loc, i32Ty, crtDynOffset));
-          } else {
+          if (isAssumedSize(sharedOp.getShape()) || !crtDynOffset) {
             mlir::Value zero = builder.createIntegerConstant(loc, i32Ty, 0);
             sharedOp.getOffsetMutable().assign(zero);
+          } else {
+            sharedOp.getOffsetMutable().assign(
+                builder.createConvert(loc, i32Ty, crtDynOffset));
           }
 
           mlir::Value dynSize =
diff --git a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
index 1902757e83bf3..70d6ebbcb039c 100644
--- a/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
+++ b/flang/lib/Optimizer/Transforms/FIRToSCF.cpp
@@ -9,36 +9,34 @@
 #include "flang/Optimizer/Dialect/FIRDialect.h"
 #include "flang/Optimizer/Transforms/Passes.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
-#include "mlir/Transforms/DialectConversion.h"
+#include "mlir/Transforms/WalkPatternRewriteDriver.h"
 
 namespace fir {
 #define GEN_PASS_DEF_FIRTOSCFPASS
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 } // namespace fir
 
-using namespace fir;
-using namespace mlir;
-
 namespace {
 class FIRToSCFPass : public fir::impl::FIRToSCFPassBase<FIRToSCFPass> {
 public:
   void runOnOperation() override;
 };
 
-struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
+struct DoLoopConversion : public mlir::OpRewritePattern<fir::DoLoopOp> {
   using OpRewritePattern<fir::DoLoopOp>::OpRewritePattern;
 
-  LogicalResult matchAndRewrite(fir::DoLoopOp doLoopOp,
-                                PatternRewriter &rewriter) const override {
-    auto loc = doLoopOp.getLoc();
+  mlir::LogicalResult
+  matchAndRewrite(fir::DoLoopOp doLoopOp,
+                  mlir::PatternRewriter &rewriter) const override {
+    mlir::Location loc = doLoopOp.getLoc();
     bool hasFinalValue = doLoopOp.getFinalValue().has_value();
 
     // Get loop values from the DoLoopOp
-    auto low = doLoopOp.getLowerBound();
-    auto high = doLoopOp.getUpperBound();
+    mlir::Value low = doLoopOp.getLowerBound();
+    mlir::Value high = doLoopOp.getUpperBound();
     assert(low && high && "must be a Value");
-    auto step = doLoopOp.getStep();
-    llvm::SmallVector<Value> iterArgs;
+    mlir::Value step = doLoopOp.getStep();
+    mlir::SmallVector<mlir::Value> iterArgs;
     if (hasFinalValue)
       iterArgs.push_back(low);
     iterArgs.append(doLoopOp.getIterOperands().begin(),
@@ -49,31 +47,33 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     // must be a positive value.
     // For easier conversion, we calculate the trip count and use a canonical
     // induction variable.
-    auto diff = arith::SubIOp::create(rewriter, loc, high, low);
-    auto distance = arith::AddIOp::create(rewriter, loc, diff, step);
-    auto tripCount = arith::DivSIOp::create(rewriter, loc, distance, step);
-    auto zero = arith::ConstantIndexOp::create(rewriter, loc, 0);
-    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
+    auto diff = mlir::arith::SubIOp::create(rewriter, loc, high, low);
+    auto distance = mlir::arith::AddIOp::create(rewriter, loc, diff, step);
+    auto tripCount =
+        mlir::arith::DivSIOp::create(rewriter, loc, distance, step);
+    auto zero = mlir::arith::ConstantIndexOp::create(rewriter, loc, 0);
+    auto one = mlir::arith::ConstantIndexOp::create(rewriter, loc, 1);
     auto scfForOp =
-        scf::ForOp::create(rewriter, loc, zero, tripCount, one, iterArgs);
+        mlir::scf::ForOp::create(rewriter, loc, zero, tripCount, one, iterArgs);
 
     auto &loopOps = doLoopOp.getBody()->getOperations();
-    auto resultOp = cast<fir::ResultOp>(doLoopOp.getBody()->getTerminator());
+    auto resultOp =
+        mlir::cast<fir::ResultOp>(doLoopOp.getBody()->getTerminator());
     auto results = resultOp.getOperands();
-    Block *loweredBody = scfForOp.getBody();
+    mlir::Block *loweredBody = scfForOp.getBody();
 
     loweredBody->getOperations().splice(loweredBody->begin(), loopOps,
                                         loopOps.begin(),
                                         std::prev(loopOps.end()));
 
     rewriter.setInsertionPointToStart(loweredBody);
-    Value iv =
-        arith::MulIOp::create(rewriter, loc, scfForOp.getInductionVar(), step);
-    iv = arith::AddIOp::create(rewriter, loc, low, iv);
+    mlir::Value iv = mlir::arith::MulIOp::create(
+        rewriter, loc, scfForOp.getInductionVar(), step);
+    iv = mlir::arith::AddIOp::create(rewriter, loc, low, iv);
 
     if (!results.empty()) {
       rewriter.setInsertionPointToEnd(loweredBody);
-      scf::YieldOp::create(rewriter, resultOp->getLoc(), results);
+      mlir::scf::YieldOp::create(rewriter, resultOp->getLoc(), results);
     }
     doLoopOp.getInductionVar().replaceAllUsesWith(iv);
     rewriter.replaceAllUsesWith(doLoopOp.getRegionIterArgs(),
@@ -84,34 +84,103 @@ struct DoLoopConversion : public OpRewritePattern<fir::DoLoopOp> {
     // Copy all the attributes from the old to new op.
     scfForOp->setAttrs(doLoopOp->getAttrs());
     rewriter.replaceOp(doLoopOp, scfForOp);
-    return success();
+    return mlir::success();
+  }
+};
+
+struct IterWhileConversion : public mlir::OpRewritePattern<fir::IterWhileOp> {
+  using OpRewritePattern<fir::IterWhileOp>::OpRewritePattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(fir::IterWhileOp iterWhileOp,
+                  mlir::PatternRewriter &rewriter) const override {
+
+    mlir::Location loc = iterWhileOp.getLoc();
+    mlir::Value lowerBound = iterWhileOp.getLowerBound();
+    mlir::Value upperBound = iterWhileOp.getUpperBound();
+    mlir::Value step = iterWhileOp.getStep();
+
+    mlir::Value okInit = iterWhileOp.getIterateIn();
+    mlir::ValueRange iterArgs = iterWhileOp.getInitArgs();
+
+    mlir::SmallVector<mlir::Value> initVals;
+    initVals.push_back(lowerBound);
+    initVals.push_back(okInit);
+    initVals.append(iterArgs.begin(), iterArgs.end());
+
+    mlir::SmallVector<mlir::Type> loopTypes;
+    loopTypes.push_back(lowerBound.getType());
+    loopTypes.push_back(okInit.getType());
+    for (auto val : iterArgs)
+      loopTypes.push_back(val.getType());
+
+    auto scfWhileOp =
+        mlir::scf::WhileOp::create(rewriter, loc, loopTypes, initVals);
+
+    auto &beforeBlock = *rewriter.createBlock(
+        &scfWhileOp.getBefore(), scfWhileOp.getBefore().end(), loopTypes,
+        mlir::SmallVector<mlir::Location>(loopTypes.size(), loc));
+
+    mlir::Region::BlockArgListType argsInBefore =
+        scfWhileOp.getBefore().getArguments();
+    auto ivInBefore = argsInBefore[0];
+    auto earlyExitInBefore = argsInBefore[1];
+
+    rewriter.setInsertionPointToStart(&beforeBlock);
+
+    mlir::Value inductionCmp = mlir::arith::CmpIOp::create(
+        rewriter, loc, mlir::arith::CmpIPredicate::sle, ivInBefore, upperBound);
+    mlir::Value cond = mlir::arith::AndIOp::create(rewriter, loc, inductionCmp,
+                                                   earlyExitInBefore);
+
+    mlir::scf::ConditionOp::create(rewriter, loc, cond, argsInBefore);
+
+    rewriter.moveBlockBefore(iterWhileOp.getBody(), &scfWhileOp.getAfter(),
+                             scfWhileOp.getAfter().begin());
+
+    auto *afterBody = scfWhileOp.getAfterBody();
+    auto resultOp = mlir::cast<fir::ResultOp>(afterBody->getTerminator());
+    mlir::SmallVector<mlir::Value> results(resultOp->getOperands());
+    mlir::Value ivInAfter = scfWhileOp.getAfterArguments()[0];
+
+    rewriter.setInsertionPointToStart(afterBody);
+    results[0] = mlir::arith::AddIOp::create(rewriter, loc, ivInAfter, step);
+
+    rewriter.setInsertionPointToEnd(afterBody);
+    rewriter.replaceOpWithNewOp<mlir::scf::YieldOp>(resultOp, results);
+
+    scfWhileOp->setAttrs(iterWhileOp->getAttrs());
+    rewriter.replaceOp(iterWhileOp, scfWhileOp);
+    return mlir::success();
   }
 };
 
-void copyBlockAndTransformResult(PatternRewriter &rewriter, Block &srcBlock,
-                                 Block &dstBlock) {
-  Operation *srcTerminator = srcBlock.getTerminator();
-  auto resultOp = cast<fir::ResultOp>(srcTerminator);
+void copyBlockAndTransformResult(mlir::PatternRewriter &rewriter,
+                                 mlir::Block &srcBlock, mlir::Block &dstBlock) {
+  mlir::Operation *srcTerminator = srcBlock.getTerminator();
+  auto resultOp = mlir::cast<fir::ResultOp>(srcTerminator);
 
   dstBlock.getOperations().splice(dstBlock.begin(), srcBlock.getOperations(),
                                   srcBlock.begin(), std::prev(srcBlock.end()));
 
   if (!resultOp->getOperands().empty()) {
     rewriter.setInsertionPointToEnd(&dstBlock);
-    scf::YieldOp::create(rewriter, resultOp->getLoc(), resultOp->getOperands());
+    mlir::scf::YieldOp::create(rewriter, resultOp->getLoc(),
+                               resultOp->getOperands());
   }
 
   rewriter.eraseOp(srcTerminator);
 }
 
-struct IfConversion : public OpRewritePattern<fir::IfOp> {
+struct IfConversion : public mlir::OpRewritePattern<fir::IfOp> {
   using OpRewritePattern<fir::IfOp>::OpRewritePattern;
-  LogicalResult matchAndRewrite(fir::IfOp ifOp,
-                                PatternRewriter &rewriter) const override {
+  mlir::LogicalResult
+  matchAndRewrite(fir::IfOp ifOp,
+                  mlir::PatternRewriter &rewriter) const override {
     bool hasElse = !ifOp.getElseRegion().empty();
     auto scfIfOp =
-        scf::IfOp::create(rewriter, ifOp.getLoc(), ifOp.getResultTypes(),
-                          ifOp.getCondition(), hasElse);
+        mlir::scf::IfOp::create(rewriter, ifOp.getLoc(), ifOp.getResultTypes(),
+                                ifOp.getCondition(), hasElse);
 
     copyBlockAndTransformResult(rewriter, ifOp.getThenRegion().front(),
                                 scfIfOp.getThenRegion().front());
@@ -123,22 +192,18 @@ struct IfConversion : public OpRewritePattern<fir::IfOp> {
 
     scfIfOp->setAttrs(ifOp->getAttrs());
     rewriter.replaceOp(ifOp, scfIfOp);
-    return success();
+    return mlir::success();
   }
 };
 } // namespace
 
 void FIRToSCFPass::runOnOperation() {
-  RewritePatternSet patterns(&getContext());
-  patterns.add<DoLoopConversion, IfConversion>(patterns.getContext());
-  ConversionTarget target(getContext());
-  target.addIllegalOp<fir::DoLoopOp, fir::IfOp>();
-  target.markUnknownOpDynamicallyLegal([](Operation *) { return true; });
-  if (failed(
-          applyPartialConversion(getOperation(), target, std::move(patterns))))
-    signalPassFailure();
+  mlir::RewritePatternSet patterns(&getContext());
+  patterns.add<DoLoopConversion, IterWhileConversion, IfConversion>(
+      patterns.getContext());
+  walkAndApplyPatterns(getOperation(), std::move(patterns));
 }
 
-std::unique_ptr<Pass> fir::createFIRToSCFPass() {
+std::unique_ptr<mlir::Pass> fir::createFIRToSCFPass() {
   return std::make_unique<FIRToSCFPass>();
 }
diff --git a/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
index 1688f2887a57a..68f5b5a4b5962 100644
--- a/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
+++ b/flang/lib/Optimizer/Transforms/OptimizeArrayRepacking.cpp
@@ -26,6 +26,8 @@ namespace fir {
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 } // namespace fir
 
+#define DEBUG_TYPE "optimize-array-repacking"
+
 namespace {
 class OptimizeArrayRepackingPass
     : public fir::impl::OptimizeArrayRepackingBase<OptimizeArrayRepackingPass> {
@@ -56,8 +58,7 @@ PackingOfContiguous::matchAndRewrite(fir::PackArrayOp op,
                                      mlir::PatternRewriter &rewriter) const {
   mlir::Value box = op.getArray();
   if (hlfir::isSimplyContiguous(box, !op.getInnermost())) {
-    rewriter.replaceAllUsesWith(op, box);
-    rewriter.eraseOp(op);
+    rewriter.replaceOp(op, box);
     return mlir::success();
   }
   return mlir::failure();
@@ -78,13 +79,19 @@ void OptimizeArrayRepackingPass::runOnOperation() {
   mlir::MLIRContext *context = &getContext();
   mlir::RewritePatternSet patterns(context);
   mlir::GreedyRewriteConfig config;
-  config.setRegionSimplificationLevel(
-      mlir::GreedySimplifyRegionLevel::Disabled);
+  config
+      .setRegionSimplificationLevel(mlir::GreedySimplifyRegionLevel::Disabled)
+      // Traverse the operations top-down, so that fir.pack_array
+      // operations are optimized before their using fir.pack_array
+      // operations. This way the rewrite may converge faster.
+      .setUseTopDownTraversal();
   patterns.insert<PackingOfContiguous>(context);
   patterns.insert<NoopUnpacking>(context);
   if (mlir::failed(
           mlir::applyPatternsGreedily(funcOp, std::move(patterns), config))) {
-    mlir::emitError(funcOp.getLoc(), "failure in array repacking optimization");
-    signalPassFailure();
+    // Failure may happen if the rewriter does not converge soon enough.
+    // That is not an error, so just report a diagnostic under debug.
+    LLVM_DEBUG(mlir::emitError(funcOp.getLoc(),
+                               "failure in array repacking optimization"));
   }
 }
diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp
index 0d131291feef3..80b3f689efa39 100644
--- a/flang/lib/Optimizer/Transforms/StackArrays.cpp
+++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp
@@ -600,10 +600,7 @@ AllocMemConversion::matchAndRewrite(fir::AllocMemOp allocmem,
   // replace references to heap allocation with references to stack allocation
   mlir::Value newValue = convertAllocationType(
       rewriter, allocmem.getLoc(), allocmem.getResult(), alloca->getResult());
-  rewriter.replaceAllUsesWith(allocmem.getResult(), newValue);
-
-  // remove allocmem operation
-  rewriter.eraseOp(allocmem.getOperation());
+  rewriter.replaceOp(allocmem, newValue);
 
   return mlir::success();
 }
@@ -813,10 +810,10 @@ void AllocMemConversion::insertLifetimeMarkers(
     mlir::OpBuilder::InsertionGuard insertGuard(rewriter);
     rewriter.setInsertionPoint(oldAlloc);
     mlir::Value ptr = fir::factory::genLifetimeStart(
-        rewriter, newAlloc.getLoc(), newAlloc, *size, &*dl);
+        rewriter, newAlloc.getLoc(), newAlloc, &*dl);
     visitFreeMemOp(oldAlloc, [&](mlir::Operation *op) {
       rewriter.setInsertionPoint(op);
-      fir::factory::genLifetimeEnd(rewriter, op->getLoc(), ptr, *size);
+      fir::factory::genLifetimeEnd(rewriter, op->getLoc(), ptr);
     });
     newAlloc->setAttr(attrName, rewriter.getUnitAttr());
   }
diff --git a/flang/lib/Parser/CMakeLists.txt b/flang/lib/Parser/CMakeLists.txt
index 1855b8a841ba7..20c6c2a7c8f80 100644
--- a/flang/lib/Parser/CMakeLists.txt
+++ b/flang/lib/Parser/CMakeLists.txt
@@ -12,6 +12,7 @@ add_flang_library(FortranParser
   message.cpp
   openacc-parsers.cpp
   openmp-parsers.cpp
+  openmp-utils.cpp
   parse-tree.cpp
   parsing.cpp
   preprocessor.cpp
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index 84d1e81bfd9be..51b49a591b02f 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -469,6 +469,9 @@ TYPE_PARSER(sourced(construct<OmpContextSelectorSpecification>(
 
 // --- Parsers for clause modifiers -----------------------------------
 
+TYPE_PARSER(construct<OmpAccessGroup>( //
+    "CGROUP" >> pure(OmpAccessGroup::Value::Cgroup)))
+
 TYPE_PARSER(construct<OmpAlignment>(scalarIntExpr))
 
 TYPE_PARSER(construct<OmpAlignModifier>( //
@@ -573,7 +576,8 @@ TYPE_PARSER(construct<OmpOrderingModifier>(
     "SIMD" >> pure(OmpOrderingModifier::Value::Simd)))
 
 TYPE_PARSER(construct<OmpPrescriptiveness>(
-    "STRICT" >> pure(OmpPrescriptiveness::Value::Strict)))
+    "STRICT" >> pure(OmpPrescriptiveness::Value::Strict) ||
+    "FALLBACK" >> pure(OmpPrescriptiveness::Value::Fallback)))
 
 TYPE_PARSER(construct<OmpPresentModifier>( //
     "PRESENT" >> pure(OmpPresentModifier::Value::Present)))
@@ -636,6 +640,12 @@ TYPE_PARSER(sourced(construct<OmpDependClause::TaskDep::Modifier>(sourced(
     construct<OmpDependClause::TaskDep::Modifier>(
         Parser<OmpTaskDependenceType>{})))))
 
+TYPE_PARSER( //
+    sourced(construct<OmpDynGroupprivateClause::Modifier>(
+        Parser<OmpAccessGroup>{})) ||
+    sourced(construct<OmpDynGroupprivateClause::Modifier>(
+        Parser<OmpPrescriptiveness>{})))
+
 TYPE_PARSER(
     sourced(construct<OmpDeviceClause::Modifier>(Parser<OmpDeviceModifier>{})))
 
@@ -777,6 +787,10 @@ TYPE_PARSER(construct<OmpDefaultClause>(
         Parser<OmpDefaultClause::DataSharingAttribute>{}) ||
     construct<OmpDefaultClause>(indirect(Parser<OmpDirectiveSpecification>{}))))
 
+TYPE_PARSER(construct<OmpDynGroupprivateClause>(
+    maybe(nonemptyList(Parser<OmpDynGroupprivateClause::Modifier>{}) / ":"),
+    scalarIntExpr))
+
 TYPE_PARSER(construct<OmpEnterClause>(
     maybe(nonemptyList(Parser<OmpEnterClause::Modifier>{}) / ":"),
     Parser<OmpObjectList>{}))
@@ -1068,6 +1082,9 @@ TYPE_PARSER( //
         construct<OmpClause>(parenthesized(Parser<OmpDoacrossClause>{})) ||
     "DYNAMIC_ALLOCATORS" >>
         construct<OmpClause>(construct<OmpClause::DynamicAllocators>()) ||
+    "DYN_GROUPPRIVATE" >>
+        construct<OmpClause>(construct<OmpClause::DynGroupprivate>(
+            parenthesized(Parser<OmpDynGroupprivateClause>{}))) ||
     "ENTER" >> construct<OmpClause>(construct<OmpClause::Enter>(
                    parenthesized(Parser<OmpEnterClause>{}))) ||
     "EXCLUSIVE" >> construct<OmpClause>(construct<OmpClause::Exclusive>(
@@ -1758,17 +1775,8 @@ TYPE_PARSER(sourced(construct<OpenMPDeclareMapperConstruct>(
 TYPE_PARSER(construct<OmpReductionCombiner>(Parser<AssignmentStmt>{}) ||
     construct<OmpReductionCombiner>(Parser<FunctionReference>{}))
 
-// 2.13.2 OMP CRITICAL
-TYPE_PARSER(startOmpLine >>
-    sourced(construct<OmpEndCriticalDirective>(
-        verbatim("END CRITICAL"_tok), maybe(parenthesized(name)))) /
-        endOmpLine)
-TYPE_PARSER(sourced(construct<OmpCriticalDirective>(verbatim("CRITICAL"_tok),
-                maybe(parenthesized(name)), Parser<OmpClauseList>{})) /
-    endOmpLine)
-
 TYPE_PARSER(construct<OpenMPCriticalConstruct>(
-    Parser<OmpCriticalDirective>{}, block, Parser<OmpEndCriticalDirective>{}))
+    OmpBlockConstructParser{llvm::omp::Directive::OMPD_critical}))
 
 // 2.11.3 Executable Allocate directive
 TYPE_PARSER(
@@ -1782,6 +1790,12 @@ TYPE_PARSER(sourced(construct<OpenMPDeclareSimdConstruct>(
     verbatim("DECLARE SIMD"_tok) || verbatim("DECLARE_SIMD"_tok),
     maybe(parenthesized(name)), Parser<OmpClauseList>{})))
 
+TYPE_PARSER(sourced( //
+    construct<OpenMPGroupprivate>(
+        predicated(OmpDirectiveNameParser{},
+            IsDirective(llvm::omp::Directive::OMPD_groupprivate)) >=
+        Parser<OmpDirectiveSpecification>{})))
+
 // 2.4 Requires construct
 TYPE_PARSER(sourced(construct<OpenMPRequiresConstruct>(
     verbatim("REQUIRES"_tok), Parser<OmpClauseList>{})))
@@ -1817,6 +1831,8 @@ TYPE_PARSER(
                                 Parser<OmpDeclareVariantDirective>{}) ||
                             construct<OpenMPDeclarativeConstruct>(
                                 Parser<OpenMPDeclarativeAllocate>{}) ||
+                            construct<OpenMPDeclarativeConstruct>(
+                                Parser<OpenMPGroupprivate>{}) ||
                             construct<OpenMPDeclarativeConstruct>(
                                 Parser<OpenMPRequiresConstruct>{}) ||
                             construct<OpenMPDeclarativeConstruct>(
@@ -1854,11 +1870,15 @@ TYPE_PARSER( //
     MakeBlockConstruct(llvm::omp::Directive::OMPD_target_data) ||
     MakeBlockConstruct(llvm::omp::Directive::OMPD_target_parallel) ||
     MakeBlockConstruct(llvm::omp::Directive::OMPD_target_teams) ||
+    MakeBlockConstruct(
+        llvm::omp::Directive::OMPD_target_teams_workdistribute) ||
     MakeBlockConstruct(llvm::omp::Directive::OMPD_target) ||
     MakeBlockConstruct(llvm::omp::Directive::OMPD_task) ||
     MakeBlockConstruct(llvm::omp::Directive::OMPD_taskgroup) ||
     MakeBlockConstruct(llvm::omp::Directive::OMPD_teams) ||
-    MakeBlockConstruct(llvm::omp::Directive::OMPD_workshare))
+    MakeBlockConstruct(llvm::omp::Directive::OMPD_teams_workdistribute) ||
+    MakeBlockConstruct(llvm::omp::Directive::OMPD_workshare) ||
+    MakeBlockConstruct(llvm::omp::Directive::OMPD_workdistribute))
 #undef MakeBlockConstruct
 
 // OMP SECTIONS Directive
diff --git a/flang/lib/Parser/openmp-utils.cpp b/flang/lib/Parser/openmp-utils.cpp
new file mode 100644
index 0000000000000..ef7e4fcdbbd07
--- /dev/null
+++ b/flang/lib/Parser/openmp-utils.cpp
@@ -0,0 +1,64 @@
+//===-- flang/Parser/openmp-utils.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common OpenMP utilities.
+//
+//===----------------------------------------------------------------------===//
+
+#include "flang/Parser/openmp-utils.h"
+
+#include "flang/Common/template.h"
+#include "flang/Common/visit.h"
+
+#include <tuple>
+#include <type_traits>
+#include <variant>
+
+namespace Fortran::parser::omp {
+
+const OmpObjectList *GetOmpObjectList(const OmpClause &clause) {
+  // Clauses with OmpObjectList as its data member
+  using MemberObjectListClauses = std::tuple<OmpClause::Copyin,
+      OmpClause::Copyprivate, OmpClause::Exclusive, OmpClause::Firstprivate,
+      OmpClause::HasDeviceAddr, OmpClause::Inclusive, OmpClause::IsDevicePtr,
+      OmpClause::Link, OmpClause::Private, OmpClause::Shared,
+      OmpClause::UseDeviceAddr, OmpClause::UseDevicePtr>;
+
+  // Clauses with OmpObjectList in the tuple
+  using TupleObjectListClauses = std::tuple<OmpClause::AdjustArgs,
+      OmpClause::Affinity, OmpClause::Aligned, OmpClause::Allocate,
+      OmpClause::Enter, OmpClause::From, OmpClause::InReduction,
+      OmpClause::Lastprivate, OmpClause::Linear, OmpClause::Map,
+      OmpClause::Reduction, OmpClause::TaskReduction, OmpClause::To>;
+
+  // TODO:: Generate the tuples using TableGen.
+  return common::visit(
+      common::visitors{
+          [&](const OmpClause::Depend &x) -> const OmpObjectList * {
+            if (auto *taskDep{std::get_if<OmpDependClause::TaskDep>(&x.v.u)}) {
+              return &std::get<OmpObjectList>(taskDep->t);
+            } else {
+              return nullptr;
+            }
+          },
+          [&](const auto &x) -> const OmpObjectList * {
+            using Ty = std::decay_t<decltype(x)>;
+            if constexpr (common::HasMember<Ty, MemberObjectListClauses>) {
+              return &x.v;
+            } else if constexpr (common::HasMember<Ty,
+                                     TupleObjectListClauses>) {
+              return &std::get<OmpObjectList>(x.v.t);
+            } else {
+              return nullptr;
+            }
+          },
+      },
+      clause.u);
+}
+
+} // namespace Fortran::parser::omp
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index 46141e2ccab56..09dcfe60a46bc 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2250,6 +2250,11 @@ class UnparseVisitor {
     Walk(std::get<OmpObjectList>(x.t));
     Walk(": ", std::get<std::optional<std::list<Modifier>>>(x.t));
   }
+  void Unparse(const OmpDynGroupprivateClause &x) {
+    using Modifier = OmpDynGroupprivateClause::Modifier;
+    Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
+    Walk(std::get<ScalarIntExpr>(x.t));
+  }
   void Unparse(const OmpEnterClause &x) {
     using Modifier = OmpEnterClause::Modifier;
     Walk(std::get<std::optional<std::list<Modifier>>>(x.t), ": ");
@@ -2606,9 +2611,7 @@ class UnparseVisitor {
     EndOpenMP();
   }
   void Unparse(const OpenMPCriticalConstruct &x) {
-    Walk(std::get<OmpCriticalDirective>(x.t));
-    Walk(std::get<Block>(x.t), "");
-    Walk(std::get<OmpEndCriticalDirective>(x.t));
+    Unparse(static_cast<const OmpBlockConstruct &>(x));
   }
   void Unparse(const OmpDeclareTargetWithList &x) {
     Put("("), Walk(x.v), Put(")");
@@ -2718,6 +2721,13 @@ class UnparseVisitor {
   void Unparse(const OpenMPDispatchConstruct &x) { //
     Unparse(static_cast<const OmpBlockConstruct &>(x));
   }
+  void Unparse(const OpenMPGroupprivate &x) {
+    BeginOpenMP();
+    Word("!$OMP ");
+    Walk(x.v);
+    Put("\n");
+    EndOpenMP();
+  }
   void Unparse(const OpenMPRequiresConstruct &y) {
     BeginOpenMP();
     Word("!$OMP REQUIRES ");
@@ -2943,6 +2953,7 @@ class UnparseVisitor {
   WALK_NESTED_ENUM(OmpTaskDependenceType, Value) // OMP task-dependence-type
   WALK_NESTED_ENUM(OmpScheduleClause, Kind) // OMP schedule-kind
   WALK_NESTED_ENUM(OmpSeverityClause, Severity) // OMP severity
+  WALK_NESTED_ENUM(OmpAccessGroup, Value)
   WALK_NESTED_ENUM(OmpDeviceModifier, Value) // OMP device modifier
   WALK_NESTED_ENUM(
       OmpDeviceTypeClause, DeviceTypeDescription) // OMP device_type
diff --git a/flang/lib/Semantics/check-acc-structure.cpp b/flang/lib/Semantics/check-acc-structure.cpp
index 051abdceba19d..6cb7e5e9e6e25 100644
--- a/flang/lib/Semantics/check-acc-structure.cpp
+++ b/flang/lib/Semantics/check-acc-structure.cpp
@@ -983,24 +983,26 @@ void AccStructureChecker::Enter(const parser::AccClause::Reduction &reduction) {
             [&](const parser::Designator &designator) {
               if (const auto *name = getDesignatorNameIfDataRef(designator)) {
                 if (name->symbol) {
-                  const auto *type{name->symbol->GetType()};
-                  if (type->IsNumeric(TypeCategory::Integer) &&
-                      !reductionIntegerSet.test(op.v)) {
-                    context_.Say(GetContext().clauseSource,
-                        "reduction operator not supported for integer type"_err_en_US);
-                  } else if (type->IsNumeric(TypeCategory::Real) &&
-                      !reductionRealSet.test(op.v)) {
-                    context_.Say(GetContext().clauseSource,
-                        "reduction operator not supported for real type"_err_en_US);
-                  } else if (type->IsNumeric(TypeCategory::Complex) &&
-                      !reductionComplexSet.test(op.v)) {
-                    context_.Say(GetContext().clauseSource,
-                        "reduction operator not supported for complex type"_err_en_US);
-                  } else if (type->category() ==
-                          Fortran::semantics::DeclTypeSpec::Category::Logical &&
-                      !reductionLogicalSet.test(op.v)) {
-                    context_.Say(GetContext().clauseSource,
-                        "reduction operator not supported for logical type"_err_en_US);
+                  if (const auto *type{name->symbol->GetType()}) {
+                    if (type->IsNumeric(TypeCategory::Integer) &&
+                        !reductionIntegerSet.test(op.v)) {
+                      context_.Say(GetContext().clauseSource,
+                          "reduction operator not supported for integer type"_err_en_US);
+                    } else if (type->IsNumeric(TypeCategory::Real) &&
+                        !reductionRealSet.test(op.v)) {
+                      context_.Say(GetContext().clauseSource,
+                          "reduction operator not supported for real type"_err_en_US);
+                    } else if (type->IsNumeric(TypeCategory::Complex) &&
+                        !reductionComplexSet.test(op.v)) {
+                      context_.Say(GetContext().clauseSource,
+                          "reduction operator not supported for complex type"_err_en_US);
+                    } else if (type->category() ==
+                            Fortran::semantics::DeclTypeSpec::Category::
+                                Logical &&
+                        !reductionLogicalSet.test(op.v)) {
+                      context_.Say(GetContext().clauseSource,
+                          "reduction operator not supported for logical type"_err_en_US);
+                    }
                   }
                   // TODO: check composite type.
                 }
diff --git a/flang/lib/Semantics/check-call.cpp b/flang/lib/Semantics/check-call.cpp
index 6f2503285013d..83f59f0cac3df 100644
--- a/flang/lib/Semantics/check-call.cpp
+++ b/flang/lib/Semantics/check-call.cpp
@@ -1058,12 +1058,11 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
             dummyName);
       }
     }
-    std::optional<std::string> warning;
     bool isHostDeviceProc{procedure.cudaSubprogramAttrs &&
         *procedure.cudaSubprogramAttrs ==
             common::CUDASubprogramAttrs::HostDevice};
     if (!common::AreCompatibleCUDADataAttrs(dummyDataAttr, actualDataAttr,
-            dummy.ignoreTKR, &warning, /*allowUnifiedMatchingRule=*/true,
+            dummy.ignoreTKR, /*allowUnifiedMatchingRule=*/true,
             isHostDeviceProc, &context.languageFeatures())) {
       auto toStr{[](std::optional<common::CUDADataAttr> x) {
         return x ? "ATTRIBUTES("s +
@@ -1074,10 +1073,6 @@ static void CheckExplicitDataArg(const characteristics::DummyDataObject &dummy,
           "%s has %s but its associated actual argument has %s"_err_en_US,
           dummyName, toStr(dummyDataAttr), toStr(actualDataAttr));
     }
-    if (warning && context.ShouldWarn(common::UsageWarning::CUDAUsage)) {
-      messages.Say(common::UsageWarning::CUDAUsage, "%s"_warn_en_US,
-          std::move(*warning));
-    }
   }
 
   // Warning for breaking F'2023 change with character allocatables
diff --git a/flang/lib/Semantics/check-omp-atomic.cpp b/flang/lib/Semantics/check-omp-atomic.cpp
index a5fdabf0b103c..50e63d356be02 100644
--- a/flang/lib/Semantics/check-omp-atomic.cpp
+++ b/flang/lib/Semantics/check-omp-atomic.cpp
@@ -11,13 +11,16 @@
 //===----------------------------------------------------------------------===//
 
 #include "check-omp-structure.h"
-#include "openmp-utils.h"
 
 #include "flang/Common/indirection.h"
+#include "flang/Common/template.h"
 #include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/match.h"
+#include "flang/Evaluate/rewrite.h"
 #include "flang/Evaluate/tools.h"
 #include "flang/Parser/char-block.h"
 #include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/openmp-utils.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
 #include "flang/Semantics/type.h"
@@ -42,11 +45,144 @@ using namespace Fortran::semantics::omp;
 
 namespace operation = Fortran::evaluate::operation;
 
+static MaybeExpr PostSemaRewrite(const SomeExpr &atom, const SomeExpr &expr);
+
 template <typename T, typename U>
 static bool operator!=(const evaluate::Expr<T> &e, const evaluate::Expr<U> &f) {
   return !(e == f);
 }
 
+namespace {
+template <typename...> struct IsIntegral {
+  static constexpr bool value{false};
+};
+
+template <common::TypeCategory C, int K>
+struct IsIntegral<evaluate::Type<C, K>> {
+  static constexpr bool value{//
+      C == common::TypeCategory::Integer ||
+      C == common::TypeCategory::Unsigned ||
+      C == common::TypeCategory::Logical};
+};
+
+template <typename T> constexpr bool is_integral_v{IsIntegral<T>::value};
+
+template <typename T, typename Op0, typename Op1>
+using ReassocOpBase = evaluate::match::AnyOfPattern< //
+    evaluate::match::Add<T, Op0, Op1>, //
+    evaluate::match::Mul<T, Op0, Op1>>;
+
+template <typename T, typename Op0, typename Op1>
+struct ReassocOp : public ReassocOpBase<T, Op0, Op1> {
+  using Base = ReassocOpBase<T, Op0, Op1>;
+  using Base::Base;
+};
+
+template <typename T, typename Op0, typename Op1>
+ReassocOp<T, Op0, Op1> reassocOp(const Op0 &op0, const Op1 &op1) {
+  return ReassocOp<T, Op0, Op1>(op0, op1);
+}
+} // namespace
+
+struct ReassocRewriter : public evaluate::rewrite::Identity {
+  using Id = evaluate::rewrite::Identity;
+  struct NonIntegralTag {};
+
+  ReassocRewriter(const SomeExpr &atom) : atom_(atom) {}
+
+  // Try to find cases where the input expression is of the form
+  // (1) (a . b) . c, or
+  // (2) a . (b . c),
+  // where . denotes an associative operation (currently + or *), and a, b, c
+  // are some subexpresions.
+  // If one of the operands in the nested operation is the atomic variable
+  // (with some possible type conversions applied to it), bring it to the
+  // top-level operation, and move the top-level operand into the nested
+  // operation.
+  // For example, assuming x is the atomic variable:
+  //   (a + x) + b  ->  (a + b) + x,  i.e. (conceptually) swap x and b.
+  template <typename T, typename U,
+      typename = std::enable_if_t<is_integral_v<T>>>
+  evaluate::Expr<T> operator()(evaluate::Expr<T> &&x, const U &u) {
+    // As per the above comment, there are 3 subexpressions involved in this
+    // transformation. A match::Expr<T> will match evaluate::Expr<U> when T is
+    // same as U, plus it will store a pointer (ref) to the matched expression.
+    // When the match is successful, the sub[i].ref will point to a, b, x (in
+    // some order) from the example above.
+    evaluate::match::Expr<T> sub[3];
+    auto inner{reassocOp<T>(sub[0], sub[1])};
+    auto outer1{reassocOp<T>(inner, sub[2])}; // inner + something
+    auto outer2{reassocOp<T>(sub[2], inner)}; // something + inner
+#if !defined(__clang__) && !defined(_MSC_VER) && \
+    (__GNUC__ < 8 || (__GNUC__ == 8 && __GNUC_MINOR__ < 5))
+    // If GCC version < 8.5, use this definition. For the other definition
+    // (which is equivalent), GCC 7.5 emits a somewhat cryptic error:
+    //    use of ‘outer1’ before deduction of ‘auto’
+    // inside of the visitor function in common::visit.
+    // Since this works with clang, MSVC and at least GCC 8.5, I'm assuming
+    // that this is some kind of a GCC issue.
+    using MatchTypes = std::tuple<evaluate::Add<T>, evaluate::Multiply<T>>;
+#else
+    using MatchTypes = typename decltype(outer1)::MatchTypes;
+#endif
+    // There is no way to ensure that the outer operation is the same as
+    // the inner one. They are matched independently, so we need to compare
+    // the index in the member variant that represents the matched type.
+    if ((match(outer1, x) && outer1.ref.index() == inner.ref.index()) ||
+        (match(outer2, x) && outer2.ref.index() == inner.ref.index())) {
+      size_t atomIdx{[&]() { // sub[atomIdx] will be the atom.
+        size_t idx;
+        for (idx = 0; idx != 3; ++idx) {
+          if (IsAtom(*sub[idx].ref)) {
+            break;
+          }
+        }
+        return idx;
+      }()};
+
+      if (atomIdx > 2) {
+        return Id::operator()(std::move(x), u);
+      }
+      return common::visit(
+          [&](auto &&s) {
+            using Expr = evaluate::Expr<T>;
+            using TypeS = llvm::remove_cvref_t<decltype(s)>;
+            // This visitor has to be semantically correct for all possible
+            // types of s even though at runtime s will only be one of the
+            // matched types.
+            // Limit the construction to the operation types that we tried
+            // to match (otherwise TypeS(op1, op2) would fail for non-binary
+            // operations).
+            if constexpr (common::HasMember<TypeS, MatchTypes>) {
+              Expr atom{*sub[atomIdx].ref};
+              Expr op1{*sub[(atomIdx + 1) % 3].ref};
+              Expr op2{*sub[(atomIdx + 2) % 3].ref};
+              return Expr(
+                  TypeS(atom, Expr(TypeS(std::move(op1), std::move(op2)))));
+            } else {
+              return Expr(TypeS(s));
+            }
+          },
+          evaluate::match::deparen(x).u);
+    }
+    return Id::operator()(std::move(x), u);
+  }
+
+  template <typename T, typename U,
+      typename = std::enable_if_t<!is_integral_v<T>>>
+  evaluate::Expr<T> operator()(
+      evaluate::Expr<T> &&x, const U &u, NonIntegralTag = {}) {
+    return Id::operator()(std::move(x), u);
+  }
+
+private:
+  template <typename T> bool IsAtom(const evaluate::Expr<T> &x) const {
+    return IsSameOrConvertOf(evaluate::AsGenericExpr(AsRvalue(x)), atom_);
+  }
+
+  const SomeExpr &atom_;
+};
+
 struct AnalyzedCondStmt {
   SomeExpr cond{evaluate::NullPointer{}}; // Default ctor is deleted
   parser::CharBlock source;
@@ -196,6 +332,26 @@ static std::pair<parser::CharBlock, parser::CharBlock> SplitAssignmentSource(
   llvm_unreachable("Could not find assignment operator");
 }
 
+static std::vector<SomeExpr> GetNonAtomExpressions(
+    const SomeExpr &atom, const std::vector<SomeExpr> &exprs) {
+  std::vector<SomeExpr> nonAtom;
+  for (const SomeExpr &e : exprs) {
+    if (!IsSameOrConvertOf(e, atom)) {
+      nonAtom.push_back(e);
+    }
+  }
+  return nonAtom;
+}
+
+static std::vector<SomeExpr> GetNonAtomArguments(
+    const SomeExpr &atom, const SomeExpr &expr) {
+  if (auto &&maybe{GetConvertInput(expr)}) {
+    return GetNonAtomExpressions(
+        atom, GetTopLevelOperationIgnoreResizing(*maybe).second);
+  }
+  return {};
+}
+
 static bool IsCheckForAssociated(const SomeExpr &cond) {
   return GetTopLevelOperationIgnoreResizing(cond).first ==
       operation::Operator::Associated;
@@ -222,47 +378,85 @@ static void SetAssignment(parser::AssignmentStmt::TypedAssignment &assign,
   }
 }
 
-static parser::OpenMPAtomicConstruct::Analysis::Op MakeAtomicAnalysisOp(
-    int what,
-    const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
-  parser::OpenMPAtomicConstruct::Analysis::Op operation;
-  operation.what = what;
-  SetAssignment(operation.assign, maybeAssign);
-  return operation;
-}
+namespace {
+struct AtomicAnalysis {
+  AtomicAnalysis(const SomeExpr &atom, const MaybeExpr &cond = std::nullopt)
+      : atom_(atom), cond_(cond) {}
 
-static parser::OpenMPAtomicConstruct::Analysis MakeAtomicAnalysis(
-    const SomeExpr &atom, const MaybeExpr &cond,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op0,
-    parser::OpenMPAtomicConstruct::Analysis::Op &&op1) {
-  // Defined in flang/include/flang/Parser/parse-tree.h
-  //
-  // struct Analysis {
-  //   struct Kind {
-  //     static constexpr int None = 0;
-  //     static constexpr int Read = 1;
-  //     static constexpr int Write = 2;
-  //     static constexpr int Update = Read | Write;
-  //     static constexpr int Action = 3; // Bits containing N, R, W, U
-  //     static constexpr int IfTrue = 4;
-  //     static constexpr int IfFalse = 8;
-  //     static constexpr int Condition = 12; // Bits containing IfTrue, IfFalse
-  //   };
-  //   struct Op {
-  //     int what;
-  //     TypedAssignment assign;
-  //   };
-  //   TypedExpr atom, cond;
-  //   Op op0, op1;
-  // };
-
-  parser::OpenMPAtomicConstruct::Analysis an;
-  SetExpr(an.atom, atom);
-  SetExpr(an.cond, cond);
-  an.op0 = std::move(op0);
-  an.op1 = std::move(op1);
-  return an;
-}
+  AtomicAnalysis &addOp0(int what,
+      const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
+    return addOp(op0_, what, maybeAssign);
+  }
+  AtomicAnalysis &addOp1(int what,
+      const std::optional<evaluate::Assignment> &maybeAssign = std::nullopt) {
+    return addOp(op1_, what, maybeAssign);
+  }
+
+  operator parser::OpenMPAtomicConstruct::Analysis() const {
+    // Defined in flang/include/flang/Parser/parse-tree.h
+    //
+    // struct Analysis {
+    //   struct Kind {
+    //     static constexpr int None = 0;
+    //     static constexpr int Read = 1;
+    //     static constexpr int Write = 2;
+    //     static constexpr int Update = Read | Write;
+    //     static constexpr int Action = 3; // Bits containing None, Read,
+    //                                      // Write, Update
+    //     static constexpr int IfTrue = 4;
+    //     static constexpr int IfFalse = 8;
+    //     static constexpr int Condition = 12; // Bits containing IfTrue,
+    //                                          // IfFalse
+    //   };
+    //   struct Op {
+    //     int what;
+    //     TypedAssignment assign;
+    //   };
+    //   TypedExpr atom, cond;
+    //   Op op0, op1;
+    // };
+
+    parser::OpenMPAtomicConstruct::Analysis an;
+    SetExpr(an.atom, atom_);
+    SetExpr(an.cond, cond_);
+    an.op0 = std::move(op0_);
+    an.op1 = std::move(op1_);
+    return an;
+  }
+
+private:
+  struct Op {
+    operator parser::OpenMPAtomicConstruct::Analysis::Op() const {
+      parser::OpenMPAtomicConstruct::Analysis::Op op;
+      op.what = what;
+      SetAssignment(op.assign, assign);
+      return op;
+    }
+
+    int what;
+    std::optional<evaluate::Assignment> assign;
+  };
+
+  AtomicAnalysis &addOp(Op &op, int what,
+      const std::optional<evaluate::Assignment> &maybeAssign) {
+    op.what = what;
+    if (maybeAssign) {
+      if (MaybeExpr rewritten{PostSemaRewrite(atom_, maybeAssign->rhs)}) {
+        op.assign = evaluate::Assignment(
+            AsRvalue(maybeAssign->lhs), std::move(*rewritten));
+        op.assign->u = std::move(maybeAssign->u);
+      } else {
+        op.assign = *maybeAssign;
+      }
+    }
+    return *this;
+  }
+
+  const SomeExpr &atom_;
+  const MaybeExpr &cond_;
+  Op op0_, op1_;
+};
+} // namespace
 
 /// Check if `expr` satisfies the following conditions for x and v:
 ///
@@ -535,6 +729,7 @@ void OmpStructureChecker::CheckAtomicCaptureAssignment(
     const evaluate::Assignment &capture, const SomeExpr &atom,
     parser::CharBlock source) {
   auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  (void)lsrc;
   const SomeExpr &cap{capture.lhs};
 
   if (!IsVarOrFunctionRef(atom)) {
@@ -551,6 +746,7 @@ void OmpStructureChecker::CheckAtomicCaptureAssignment(
 void OmpStructureChecker::CheckAtomicReadAssignment(
     const evaluate::Assignment &read, parser::CharBlock source) {
   auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  (void)lsrc;
 
   if (auto maybe{GetConvertInput(read.rhs)}) {
     const SomeExpr &atom{*maybe};
@@ -584,7 +780,8 @@ void OmpStructureChecker::CheckAtomicWriteAssignment(
   }
 }
 
-void OmpStructureChecker::CheckAtomicUpdateAssignment(
+std::optional<evaluate::Assignment>
+OmpStructureChecker::CheckAtomicUpdateAssignment(
     const evaluate::Assignment &update, parser::CharBlock source) {
   // [6.0:191:1-7]
   // An update structured block is update-statement, an update statement
@@ -600,14 +797,47 @@ void OmpStructureChecker::CheckAtomicUpdateAssignment(
   if (!IsVarOrFunctionRef(atom)) {
     ErrorShouldBeVariable(atom, rsrc);
     // Skip other checks.
-    return;
+    return std::nullopt;
   }
 
   CheckAtomicVariable(atom, lsrc);
 
+  auto [hasErrors, tryReassoc]{CheckAtomicUpdateAssignmentRhs(
+      atom, update.rhs, source, /*suppressDiagnostics=*/true)};
+
+  if (!hasErrors) {
+    CheckStorageOverlap(atom, GetNonAtomArguments(atom, update.rhs), source);
+    return std::nullopt;
+  } else if (tryReassoc) {
+    ReassocRewriter ra(atom);
+    SomeExpr raRhs{evaluate::rewrite::Mutator(ra)(update.rhs)};
+
+    std::tie(hasErrors, tryReassoc) = CheckAtomicUpdateAssignmentRhs(
+        atom, raRhs, source, /*suppressDiagnostics=*/true);
+    if (!hasErrors) {
+      CheckStorageOverlap(atom, GetNonAtomArguments(atom, raRhs), source);
+
+      evaluate::Assignment raAssign(update);
+      raAssign.rhs = raRhs;
+      return raAssign;
+    }
+  }
+
+  // This is guaranteed to report errors.
+  CheckAtomicUpdateAssignmentRhs(
+      atom, update.rhs, source, /*suppressDiagnostics=*/false);
+  return std::nullopt;
+}
+
+std::pair<bool, bool> OmpStructureChecker::CheckAtomicUpdateAssignmentRhs(
+    const SomeExpr &atom, const SomeExpr &rhs, parser::CharBlock source,
+    bool suppressDiagnostics) {
+  auto [lsrc, rsrc]{SplitAssignmentSource(source)};
+  (void)lsrc;
+
   std::pair<operation::Operator, std::vector<SomeExpr>> top{
       operation::Operator::Unknown, {}};
-  if (auto &&maybeInput{GetConvertInput(update.rhs)}) {
+  if (auto &&maybeInput{GetConvertInput(rhs)}) {
     top = GetTopLevelOperationIgnoreResizing(*maybeInput);
   }
   switch (top.first) {
@@ -624,29 +854,39 @@ void OmpStructureChecker::CheckAtomicUpdateAssignment(
   case operation::Operator::Identity:
     break;
   case operation::Operator::Call:
-    context_.Say(source,
-        "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
+    if (!suppressDiagnostics) {
+      context_.Say(source,
+          "A call to this function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    }
+    return std::make_pair(true, false);
   case operation::Operator::Convert:
-    context_.Say(source,
-        "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
+    if (!suppressDiagnostics) {
+      context_.Say(source,
+          "An implicit or explicit type conversion is not a valid ATOMIC UPDATE operation"_err_en_US);
+    }
+    return std::make_pair(true, false);
   case operation::Operator::Intrinsic:
-    context_.Say(source,
-        "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
+    if (!suppressDiagnostics) {
+      context_.Say(source,
+          "This intrinsic function is not a valid ATOMIC UPDATE operation"_err_en_US);
+    }
+    return std::make_pair(true, false);
   case operation::Operator::Constant:
   case operation::Operator::Unknown:
-    context_.Say(
-        source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
-    return;
+    if (!suppressDiagnostics) {
+      context_.Say(
+          source, "This is not a valid ATOMIC UPDATE operation"_err_en_US);
+    }
+    return std::make_pair(true, false);
   default:
     assert(
         top.first != operation::Operator::Identity && "Handle this separately");
-    context_.Say(source,
-        "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
-        operation::ToString(top.first));
-    return;
+    if (!suppressDiagnostics) {
+      context_.Say(source,
+          "The %s operator is not a valid ATOMIC UPDATE operation"_err_en_US,
+          operation::ToString(top.first));
+    }
+    return std::make_pair(true, false);
   }
   // Check how many times `atom` occurs as an argument, if it's a subexpression
   // of an argument, and collect the non-atom arguments.
@@ -667,39 +907,48 @@ void OmpStructureChecker::CheckAtomicUpdateAssignment(
     return count;
   }()};
 
-  bool hasError{false};
+  bool hasError{false}, tryReassoc{false};
   if (subExpr) {
-    context_.Say(rsrc,
-        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
-        atom.AsFortran(), subExpr->AsFortran());
+    if (!suppressDiagnostics) {
+      context_.Say(rsrc,
+          "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
+          atom.AsFortran(), subExpr->AsFortran());
+    }
     hasError = true;
   }
   if (top.first == operation::Operator::Identity) {
     // This is "x = y".
     assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
     if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
-          atom.AsFortran());
+      if (!suppressDiagnostics) {
+        context_.Say(rsrc,
+            "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
+            atom.AsFortran());
+      }
       hasError = true;
     }
   } else {
     if (atomCount == 0) {
-      context_.Say(rsrc,
-          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
+      if (!suppressDiagnostics) {
+        context_.Say(rsrc,
+            "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
+            atom.AsFortran(), operation::ToString(top.first));
+      }
+      // If `atom` is a proper subexpression, and it not present as an
+      // argument on its own, reassociation may be able to help.
+      tryReassoc = subExpr.has_value();
       hasError = true;
     } else if (atomCount > 1) {
-      context_.Say(rsrc,
-          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
-          atom.AsFortran(), operation::ToString(top.first));
+      if (!suppressDiagnostics) {
+        context_.Say(rsrc,
+            "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
+            atom.AsFortran(), operation::ToString(top.first));
+      }
       hasError = true;
     }
   }
 
-  if (!hasError) {
-    CheckStorageOverlap(atom, nonAtom, source);
-  }
+  return std::make_pair(hasError, tryReassoc);
 }
 
 void OmpStructureChecker::CheckAtomicConditionalUpdateAssignment(
@@ -802,12 +1051,14 @@ void OmpStructureChecker::CheckAtomicUpdateOnly(
     SourcedActionStmt action{GetActionStmt(&body.front())};
     if (auto maybeUpdate{GetEvaluateAssignment(action.stmt)}) {
       const SomeExpr &atom{maybeUpdate->lhs};
-      CheckAtomicUpdateAssignment(*maybeUpdate, action.source);
+      auto maybeAssign{
+          CheckAtomicUpdateAssignment(*maybeUpdate, action.source)};
+      auto &updateAssign{maybeAssign.has_value() ? maybeAssign : maybeUpdate};
 
       using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Update, maybeUpdate),
-          MakeAtomicAnalysisOp(Analysis::None));
+      x.analysis = AtomicAnalysis(atom)
+                       .addOp0(Analysis::Update, updateAssign)
+                       .addOp1(Analysis::None);
     } else if (!IsAssignment(action.stmt)) {
       context_.Say(
           source, "ATOMIC UPDATE operation should be an assignment"_err_en_US);
@@ -889,9 +1140,11 @@ void OmpStructureChecker::CheckAtomicConditionalUpdate(
   }
 
   using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-  x.analysis = MakeAtomicAnalysis(assign.lhs, update.cond,
-      MakeAtomicAnalysisOp(Analysis::Update | Analysis::IfTrue, assign),
-      MakeAtomicAnalysisOp(Analysis::None));
+  const SomeExpr &atom{assign.lhs};
+
+  x.analysis = AtomicAnalysis(atom, update.cond)
+                   .addOp0(Analysis::Update | Analysis::IfTrue, assign)
+                   .addOp1(Analysis::None);
 }
 
 void OmpStructureChecker::CheckAtomicUpdateCapture(
@@ -920,29 +1173,32 @@ void OmpStructureChecker::CheckAtomicUpdateCapture(
   using Analysis = parser::OpenMPAtomicConstruct::Analysis;
   int action;
 
+  std::optional<evaluate::Assignment> updateAssign{update};
   if (IsMaybeAtomicWrite(update)) {
     action = Analysis::Write;
     CheckAtomicWriteAssignment(update, uact.source);
   } else {
     action = Analysis::Update;
-    CheckAtomicUpdateAssignment(update, uact.source);
+    if (auto &&maybe{CheckAtomicUpdateAssignment(update, uact.source)}) {
+      updateAssign = maybe;
+    }
   }
   CheckAtomicCaptureAssignment(capture, atom, cact.source);
 
-  if (IsPointerAssignment(update) != IsPointerAssignment(capture)) {
+  if (IsPointerAssignment(*updateAssign) != IsPointerAssignment(capture)) {
     context_.Say(cact.source,
         "The update and capture assignments should both be pointer-assignments or both be non-pointer-assignments"_err_en_US);
     return;
   }
 
   if (GetActionStmt(&body.front()).stmt == uact.stmt) {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(action, update),
-        MakeAtomicAnalysisOp(Analysis::Read, capture));
+    x.analysis = AtomicAnalysis(atom)
+                     .addOp0(action, updateAssign)
+                     .addOp1(Analysis::Read, capture);
   } else {
-    x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-        MakeAtomicAnalysisOp(Analysis::Read, capture),
-        MakeAtomicAnalysisOp(action, update));
+    x.analysis = AtomicAnalysis(atom)
+                     .addOp0(Analysis::Read, capture)
+                     .addOp1(action, updateAssign);
   }
 }
 
@@ -1087,15 +1343,16 @@ void OmpStructureChecker::CheckAtomicConditionalUpdateCapture(
 
   evaluate::Assignment updAssign{*GetEvaluateAssignment(update.ift.stmt)};
   evaluate::Assignment capAssign{*GetEvaluateAssignment(capture.stmt)};
+  const SomeExpr &atom{updAssign.lhs};
 
   if (captureFirst) {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign),
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign));
+    x.analysis = AtomicAnalysis(atom, update.cond)
+                     .addOp0(Analysis::Read | captureWhen, capAssign)
+                     .addOp1(Analysis::Write | updateWhen, updAssign);
   } else {
-    x.analysis = MakeAtomicAnalysis(updAssign.lhs, update.cond,
-        MakeAtomicAnalysisOp(Analysis::Write | updateWhen, updAssign),
-        MakeAtomicAnalysisOp(Analysis::Read | captureWhen, capAssign));
+    x.analysis = AtomicAnalysis(atom, update.cond)
+                     .addOp0(Analysis::Write | updateWhen, updAssign)
+                     .addOp1(Analysis::Read | captureWhen, capAssign);
   }
 }
 
@@ -1125,9 +1382,9 @@ void OmpStructureChecker::CheckAtomicRead(
       if (auto maybe{GetConvertInput(maybeRead->rhs)}) {
         const SomeExpr &atom{*maybe};
         using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-        x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-            MakeAtomicAnalysisOp(Analysis::Read, maybeRead),
-            MakeAtomicAnalysisOp(Analysis::None));
+        x.analysis = AtomicAnalysis(atom)
+                         .addOp0(Analysis::Read, maybeRead)
+                         .addOp1(Analysis::None);
       }
     } else if (!IsAssignment(action.stmt)) {
       context_.Say(
@@ -1159,9 +1416,9 @@ void OmpStructureChecker::CheckAtomicWrite(
       CheckAtomicWriteAssignment(*maybeWrite, action.source);
 
       using Analysis = parser::OpenMPAtomicConstruct::Analysis;
-      x.analysis = MakeAtomicAnalysis(atom, std::nullopt,
-          MakeAtomicAnalysisOp(Analysis::Write, maybeWrite),
-          MakeAtomicAnalysisOp(Analysis::None));
+      x.analysis = AtomicAnalysis(atom)
+                       .addOp0(Analysis::Write, maybeWrite)
+                       .addOp1(Analysis::None);
     } else if (!IsAssignment(action.stmt)) {
       context_.Say(
           x.source, "ATOMIC WRITE operation should be an assignment"_err_en_US);
@@ -1260,4 +1517,118 @@ void OmpStructureChecker::Leave(const parser::OpenMPAtomicConstruct &) {
   dirContext_.pop_back();
 }
 
+// Rewrite min/max:
+// Min and max intrinsics in Fortran take an arbitrary number of arguments
+// (two or more). The first two are mandatory, the rest is optional. That
+// means that arguments beyond the first two may be optional dummy argument
+// from the caller. In that case, a reference to such an argument will
+// cause presence test to be emitted, which cannot go inside of the atomic
+// operation. Since the atom operand must be present, rewrite the min/max
+// operation in a way that avoid the presence tests in the atomic code.
+// For example, in
+//   subroutine f(atom, x, y, z)
+//     integer :: atom, x
+//     integer, optional :: y, z
+//     !$omp atomic update
+//     atom = min(atom, x, y, z)
+//   end
+// the min operation will become
+//   atom = min(atom, min(x, y, z))
+// and in the final code
+//   // Presence check is fine here.
+//   tmp = min(x, y, z)
+//   atomic update {
+//     // Both operands are mandatory, no presence check needed.
+//     atom = min(atom, tmp)
+//   }
+struct MinMaxRewriter : public evaluate::rewrite::Identity {
+  using Id = evaluate::rewrite::Identity;
+  using Id::operator();
+
+  MinMaxRewriter(const SomeExpr &atom) : atom_(atom) {}
+
+  static bool IsMinMax(const evaluate::ProcedureDesignator &p) {
+    if (auto *intrin{p.GetSpecificIntrinsic()}) {
+      return intrin->name == "min" || intrin->name == "max";
+    }
+    return false;
+  }
+
+  // Take a list of arguments to a min/max operation, e.g. [a0, a1, ...]
+  // One of the a_i's, say a_t, must be the atom.
+  // Generate
+  //   min/max(a_t, min/max(a0, a1, ... [except a_t]))
+  template <typename T>
+  evaluate::Expr<T> operator()(
+      evaluate::Expr<T> &&x, const evaluate::FunctionRef<T> &f) {
+    const evaluate::ProcedureDesignator &proc = f.proc();
+    if (!IsMinMax(proc) || f.arguments().size() <= 2) {
+      return Id::operator()(std::move(x), f);
+    }
+
+    // Collect arguments as SomeExpr's and find out which argument
+    // corresponds to atom.
+    const SomeExpr *atomArg{nullptr};
+    std::vector<const SomeExpr *> args;
+    for (const std::optional<evaluate::ActualArgument> &a : f.arguments()) {
+      if (!a) {
+        continue;
+      }
+      if (const SomeExpr *e{a->UnwrapExpr()}) {
+        if (evaluate::IsSameOrConvertOf(*e, atom_)) {
+          atomArg = e;
+        }
+        args.push_back(e);
+      }
+    }
+    if (!atomArg) {
+      return Id::operator()(std::move(x), f);
+    }
+
+    evaluate::ActualArguments nonAtoms;
+
+    auto AsActual = [](const SomeExpr &z) {
+      SomeExpr copy = z;
+      return evaluate::ActualArgument(std::move(copy));
+    };
+    // Semantic checks guarantee that the "atom" shows exactly once in the
+    // argument list (with potential conversions around it).
+    // For the first two (non-optional) arguments, if "atom" is among them,
+    // replace it with another occurrence of the other non-optional argument.
+    if (atomArg == args[0]) {
+      // (atom, x, y...) -> (x, x, y...)
+      nonAtoms.push_back(AsActual(*args[1]));
+      nonAtoms.push_back(AsActual(*args[1]));
+    } else if (atomArg == args[1]) {
+      // (x, atom, y...) -> (x, x, y...)
+      nonAtoms.push_back(AsActual(*args[0]));
+      nonAtoms.push_back(AsActual(*args[0]));
+    } else {
+      // (x, y, z...) -> unchanged
+      nonAtoms.push_back(AsActual(*args[0]));
+      nonAtoms.push_back(AsActual(*args[1]));
+    }
+
+    // The rest of arguments are optional, so we can just skip "atom".
+    for (size_t i = 2, e = args.size(); i != e; ++i) {
+      if (atomArg != args[i])
+        nonAtoms.push_back(AsActual(*args[i]));
+    }
+
+    SomeExpr tmp = evaluate::AsGenericExpr(
+        evaluate::FunctionRef<T>(AsRvalue(proc), AsRvalue(nonAtoms)));
+
+    return evaluate::Expr<T>(evaluate::FunctionRef<T>(
+        AsRvalue(proc), {AsActual(*atomArg), AsActual(tmp)}));
+  }
+
+private:
+  const SomeExpr &atom_;
+};
+
+static MaybeExpr PostSemaRewrite(const SomeExpr &atom, const SomeExpr &expr) {
+  MinMaxRewriter rewriter(atom);
+  return evaluate::rewrite::Mutator(rewriter)(expr);
+}
+
 } // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/check-omp-loop.cpp b/flang/lib/Semantics/check-omp-loop.cpp
index 59d57a2ec7cfb..8dad1f5d605e7 100644
--- a/flang/lib/Semantics/check-omp-loop.cpp
+++ b/flang/lib/Semantics/check-omp-loop.cpp
@@ -13,7 +13,6 @@
 #include "check-omp-structure.h"
 
 #include "check-directive-structure.h"
-#include "openmp-utils.h"
 
 #include "flang/Common/idioms.h"
 #include "flang/Common/visit.h"
@@ -23,6 +22,7 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Parser/tools.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/openmp-utils.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
diff --git a/flang/lib/Semantics/check-omp-metadirective.cpp b/flang/lib/Semantics/check-omp-metadirective.cpp
index 03487da64f1bf..cf5ea9028edc7 100644
--- a/flang/lib/Semantics/check-omp-metadirective.cpp
+++ b/flang/lib/Semantics/check-omp-metadirective.cpp
@@ -12,8 +12,6 @@
 
 #include "check-omp-structure.h"
 
-#include "openmp-utils.h"
-
 #include "flang/Common/idioms.h"
 #include "flang/Common/indirection.h"
 #include "flang/Common/visit.h"
@@ -21,6 +19,7 @@
 #include "flang/Parser/message.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/openmp-utils.h"
 #include "flang/Semantics/tools.h"
 
 #include "llvm/Frontend/OpenMP/OMP.h"
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index a9c56c347877f..6dad9a3d0711d 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -10,7 +10,6 @@
 
 #include "check-directive-structure.h"
 #include "definable.h"
-#include "openmp-utils.h"
 #include "resolve-names-utils.h"
 
 #include "flang/Common/idioms.h"
@@ -21,12 +20,14 @@
 #include "flang/Parser/char-block.h"
 #include "flang/Parser/characters.h"
 #include "flang/Parser/message.h"
+#include "flang/Parser/openmp-utils.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Parser/tools.h"
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/openmp-utils.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
@@ -57,6 +58,7 @@
 namespace Fortran::semantics {
 
 using namespace Fortran::semantics::omp;
+using namespace Fortran::parser::omp;
 
 // Use when clause falls under 'struct OmpClause' in 'parse-tree.h'.
 #define CHECK_SIMPLE_CLAUSE(X, Y) \
@@ -141,6 +143,64 @@ class OmpWorkshareBlockChecker {
   parser::CharBlock source_;
 };
 
+// 'OmpWorkdistributeBlockChecker' is used to check the validity of the
+// assignment statements and the expressions enclosed in an OpenMP
+// WORKDISTRIBUTE construct
+class OmpWorkdistributeBlockChecker {
+public:
+  OmpWorkdistributeBlockChecker(
+      SemanticsContext &context, parser::CharBlock source)
+      : context_{context}, source_{source} {}
+
+  template <typename T> bool Pre(const T &) { return true; }
+  template <typename T> void Post(const T &) {}
+
+  bool Pre(const parser::AssignmentStmt &assignment) {
+    const auto &var{std::get<parser::Variable>(assignment.t)};
+    const auto &expr{std::get<parser::Expr>(assignment.t)};
+    const auto *lhs{GetExpr(context_, var)};
+    const auto *rhs{GetExpr(context_, expr)};
+    if (lhs && rhs) {
+      Tristate isDefined{semantics::IsDefinedAssignment(
+          lhs->GetType(), lhs->Rank(), rhs->GetType(), rhs->Rank())};
+      if (isDefined == Tristate::Yes) {
+        context_.Say(expr.source,
+            "Defined assignment statement is not allowed in a WORKDISTRIBUTE construct"_err_en_US);
+      }
+    }
+    return true;
+  }
+
+  bool Pre(const parser::Expr &expr) {
+    if (const auto *e{GetExpr(context_, expr)}) {
+      if (!e)
+        return false;
+      for (const Symbol &symbol : evaluate::CollectSymbols(*e)) {
+        const Symbol &root{GetAssociationRoot(symbol)};
+        if (IsFunction(root)) {
+          std::vector<std::string> attrs;
+          if (!IsElementalProcedure(root)) {
+            attrs.push_back("non-ELEMENTAL");
+          }
+          if (root.attrs().test(Attr::IMPURE)) {
+            attrs.push_back("IMPURE");
+          }
+          std::string attrsStr =
+              attrs.empty() ? "" : " " + llvm::join(attrs, ", ");
+          context_.Say(expr.source,
+              "User defined%s function '%s' is not allowed in a WORKDISTRIBUTE construct"_err_en_US,
+              attrsStr, root.name());
+        }
+      }
+    }
+    return false;
+  }
+
+private:
+  SemanticsContext &context_;
+  parser::CharBlock source_;
+};
+
 // `OmpUnitedTaskDesignatorChecker` is used to check if the designator
 // can appear within the TASK construct
 class OmpUnitedTaskDesignatorChecker {
@@ -537,14 +597,6 @@ template <typename Checker> struct DirectiveSpellingVisitor {
     checker_(x.v.source, Directive::OMPD_assume);
     return false;
   }
-  bool Pre(const parser::OmpCriticalDirective &x) {
-    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_critical);
-    return false;
-  }
-  bool Pre(const parser::OmpEndCriticalDirective &x) {
-    checker_(std::get<parser::Verbatim>(x.t).source, Directive::OMPD_critical);
-    return false;
-  }
   bool Pre(const parser::OmpMetadirectiveDirective &x) {
     checker_(
         std::get<parser::Verbatim>(x.t).source, Directive::OMPD_metadirective);
@@ -579,6 +631,10 @@ template <typename Checker> struct DirectiveSpellingVisitor {
         Directive::OMPD_declare_variant);
     return false;
   }
+  bool Pre(const parser::OpenMPGroupprivate &x) {
+    checker_(x.v.DirName().source, Directive::OMPD_groupprivate);
+    return false;
+  }
   bool Pre(const parser::OpenMPThreadprivate &x) {
     checker_(
         std::get<parser::Verbatim>(x.t).source, Directive::OMPD_threadprivate);
@@ -817,6 +873,12 @@ void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
           "TARGET construct with nested TEAMS region contains statements or "
           "directives outside of the TEAMS construct"_err_en_US);
     }
+    if (GetContext().directive == llvm::omp::Directive::OMPD_workdistribute &&
+        GetContextParent().directive != llvm::omp::Directive::OMPD_teams) {
+      context_.Say(x.BeginDir().DirName().source,
+          "%s region can only be strictly nested within TEAMS region"_err_en_US,
+          ContextDirectiveAsFortran());
+    }
   }
 
   CheckNoBranching(block, beginSpec.DirId(), beginSpec.source);
@@ -900,6 +962,17 @@ void OmpStructureChecker::Enter(const parser::OpenMPBlockConstruct &x) {
     HasInvalidWorksharingNesting(
         beginSpec.source, llvm::omp::nestedWorkshareErrSet);
     break;
+  case llvm::omp::OMPD_workdistribute:
+    if (!CurrentDirectiveIsNested()) {
+      context_.Say(beginSpec.source,
+          "A WORKDISTRIBUTE region must be nested inside TEAMS region only."_err_en_US);
+    }
+    CheckWorkdistributeBlockStmts(block, beginSpec.source);
+    break;
+  case llvm::omp::OMPD_teams_workdistribute:
+  case llvm::omp::OMPD_target_teams_workdistribute:
+    CheckWorkdistributeBlockStmts(block, beginSpec.source);
+    break;
   case llvm::omp::Directive::OMPD_scope:
   case llvm::omp::Directive::OMPD_single:
     // TODO: This check needs to be extended while implementing nesting of
@@ -1089,114 +1162,156 @@ void OmpStructureChecker::Leave(const parser::OmpEndSectionsDirective &x) {
   }
 }
 
+void OmpStructureChecker::CheckThreadprivateOrDeclareTargetVar(
+    const parser::Designator &designator) {
+  auto *name{parser::Unwrap<parser::Name>(designator)};
+  // If the symbol is null, return early, CheckSymbolNames
+  // should have already reported the missing symbol as a
+  // diagnostic error
+  if (!name || !name->symbol) {
+    return;
+  }
+
+  llvm::omp::Directive directive{GetContext().directive};
+
+  if (name->symbol->GetUltimate().IsSubprogram()) {
+    if (directive == llvm::omp::Directive::OMPD_threadprivate)
+      context_.Say(name->source,
+          "The procedure name cannot be in a %s directive"_err_en_US,
+          ContextDirectiveAsFortran());
+    // TODO: Check for procedure name in declare target directive.
+  } else if (name->symbol->attrs().test(Attr::PARAMETER)) {
+    if (directive == llvm::omp::Directive::OMPD_threadprivate)
+      context_.Say(name->source,
+          "The entity with PARAMETER attribute cannot be in a %s directive"_err_en_US,
+          ContextDirectiveAsFortran());
+    else if (directive == llvm::omp::Directive::OMPD_declare_target)
+      context_.Warn(common::UsageWarning::OpenMPUsage, name->source,
+          "The entity with PARAMETER attribute is used in a %s directive"_warn_en_US,
+          ContextDirectiveAsFortran());
+  } else if (FindCommonBlockContaining(*name->symbol)) {
+    context_.Say(name->source,
+        "A variable in a %s directive cannot be an element of a common block"_err_en_US,
+        ContextDirectiveAsFortran());
+  } else if (FindEquivalenceSet(*name->symbol)) {
+    context_.Say(name->source,
+        "A variable in a %s directive cannot appear in an EQUIVALENCE statement"_err_en_US,
+        ContextDirectiveAsFortran());
+  } else if (name->symbol->test(Symbol::Flag::OmpThreadprivate) &&
+      directive == llvm::omp::Directive::OMPD_declare_target) {
+    context_.Say(name->source,
+        "A THREADPRIVATE variable cannot appear in a %s directive"_err_en_US,
+        ContextDirectiveAsFortran());
+  } else {
+    const semantics::Scope &useScope{
+        context_.FindScope(GetContext().directiveSource)};
+    const semantics::Scope &curScope = name->symbol->GetUltimate().owner();
+    if (!curScope.IsTopLevel()) {
+      const semantics::Scope &declScope =
+          GetProgramUnitOrBlockConstructContaining(curScope);
+      const semantics::Symbol *sym{
+          declScope.parent().FindSymbol(name->symbol->name())};
+      if (sym &&
+          (sym->has<MainProgramDetails>() || sym->has<ModuleDetails>())) {
+        context_.Say(name->source,
+            "The module name cannot be in a %s directive"_err_en_US,
+            ContextDirectiveAsFortran());
+      } else if (!IsSaved(*name->symbol) &&
+          declScope.kind() != Scope::Kind::MainProgram &&
+          declScope.kind() != Scope::Kind::Module) {
+        context_.Say(name->source,
+            "A variable that appears in a %s directive must be declared in the scope of a module or have the SAVE attribute, either explicitly or implicitly"_err_en_US,
+            ContextDirectiveAsFortran());
+      } else if (useScope != declScope) {
+        context_.Say(name->source,
+            "The %s directive and the common block or variable in it must appear in the same declaration section of a scoping unit"_err_en_US,
+            ContextDirectiveAsFortran());
+      }
+    }
+  }
+}
+
+void OmpStructureChecker::CheckThreadprivateOrDeclareTargetVar(
+    const parser::Name &name) {
+  if (!name.symbol) {
+    return;
+  }
+
+  if (auto *cb{name.symbol->detailsIf<CommonBlockDetails>()}) {
+    for (const auto &obj : cb->objects()) {
+      if (FindEquivalenceSet(*obj)) {
+        context_.Say(name.source,
+            "A variable in a %s directive cannot appear in an EQUIVALENCE statement (variable '%s' from common block '/%s/')"_err_en_US,
+            ContextDirectiveAsFortran(), obj->name(), name.symbol->name());
+      }
+    }
+  }
+}
+
 void OmpStructureChecker::CheckThreadprivateOrDeclareTargetVar(
     const parser::OmpObjectList &objList) {
   for (const auto &ompObject : objList.v) {
-    common::visit(
-        common::visitors{
-            [&](const parser::Designator &) {
-              if (const auto *name{parser::Unwrap<parser::Name>(ompObject)}) {
-                // The symbol is null, return early, CheckSymbolNames
-                // should have already reported the missing symbol as a
-                // diagnostic error
-                if (!name->symbol) {
-                  return;
-                }
-
-                if (name->symbol->GetUltimate().IsSubprogram()) {
-                  if (GetContext().directive ==
-                      llvm::omp::Directive::OMPD_threadprivate)
-                    context_.Say(name->source,
-                        "The procedure name cannot be in a %s "
-                        "directive"_err_en_US,
-                        ContextDirectiveAsFortran());
-                  // TODO: Check for procedure name in declare target directive.
-                } else if (name->symbol->attrs().test(Attr::PARAMETER)) {
-                  if (GetContext().directive ==
-                      llvm::omp::Directive::OMPD_threadprivate)
-                    context_.Say(name->source,
-                        "The entity with PARAMETER attribute cannot be in a %s "
-                        "directive"_err_en_US,
-                        ContextDirectiveAsFortran());
-                  else if (GetContext().directive ==
-                      llvm::omp::Directive::OMPD_declare_target)
-                    context_.Warn(common::UsageWarning::OpenMPUsage,
-                        name->source,
-                        "The entity with PARAMETER attribute is used in a %s directive"_warn_en_US,
-                        ContextDirectiveAsFortran());
-                } else if (FindCommonBlockContaining(*name->symbol)) {
-                  context_.Say(name->source,
-                      "A variable in a %s directive cannot be an element of a "
-                      "common block"_err_en_US,
-                      ContextDirectiveAsFortran());
-                } else if (FindEquivalenceSet(*name->symbol)) {
-                  context_.Say(name->source,
-                      "A variable in a %s directive cannot appear in an "
-                      "EQUIVALENCE statement"_err_en_US,
-                      ContextDirectiveAsFortran());
-                } else if (name->symbol->test(Symbol::Flag::OmpThreadprivate) &&
-                    GetContext().directive ==
-                        llvm::omp::Directive::OMPD_declare_target) {
-                  context_.Say(name->source,
-                      "A THREADPRIVATE variable cannot appear in a %s "
-                      "directive"_err_en_US,
-                      ContextDirectiveAsFortran());
-                } else {
-                  const semantics::Scope &useScope{
-                      context_.FindScope(GetContext().directiveSource)};
-                  const semantics::Scope &curScope =
-                      name->symbol->GetUltimate().owner();
-                  if (!curScope.IsTopLevel()) {
-                    const semantics::Scope &declScope =
-                        GetProgramUnitOrBlockConstructContaining(curScope);
-                    const semantics::Symbol *sym{
-                        declScope.parent().FindSymbol(name->symbol->name())};
-                    if (sym &&
-                        (sym->has<MainProgramDetails>() ||
-                            sym->has<ModuleDetails>())) {
-                      context_.Say(name->source,
-                          "The module name cannot be in a %s "
-                          "directive"_err_en_US,
-                          ContextDirectiveAsFortran());
-                    } else if (!IsSaved(*name->symbol) &&
-                        declScope.kind() != Scope::Kind::MainProgram &&
-                        declScope.kind() != Scope::Kind::Module) {
-                      context_.Say(name->source,
-                          "A variable that appears in a %s directive must be "
-                          "declared in the scope of a module or have the SAVE "
-                          "attribute, either explicitly or "
-                          "implicitly"_err_en_US,
-                          ContextDirectiveAsFortran());
-                    } else if (useScope != declScope) {
-                      context_.Say(name->source,
-                          "The %s directive and the common block or variable "
-                          "in it must appear in the same declaration section "
-                          "of a scoping unit"_err_en_US,
-                          ContextDirectiveAsFortran());
-                    }
-                  }
-                }
-              }
-            },
-            [&](const parser::Name &name) {
-              if (name.symbol) {
-                if (auto *cb{name.symbol->detailsIf<CommonBlockDetails>()}) {
-                  for (const auto &obj : cb->objects()) {
-                    if (FindEquivalenceSet(*obj)) {
-                      context_.Say(name.source,
-                          "A variable in a %s directive cannot appear in an EQUIVALENCE statement (variable '%s' from common block '/%s/')"_err_en_US,
-                          ContextDirectiveAsFortran(), obj->name(),
-                          name.symbol->name());
-                    }
-                  }
-                }
-              }
-            },
-        },
+    common::visit([&](auto &&s) { CheckThreadprivateOrDeclareTargetVar(s); },
         ompObject.u);
   }
 }
 
+void OmpStructureChecker::Enter(const parser::OpenMPGroupprivate &x) {
+  PushContextAndClauseSets(
+      x.v.DirName().source, llvm::omp::Directive::OMPD_groupprivate);
+
+  for (const parser::OmpArgument &arg : x.v.Arguments().v) {
+    auto *locator{std::get_if<parser::OmpLocator>(&arg.u)};
+    const Symbol *sym{GetArgumentSymbol(arg)};
+
+    if (!locator || !sym ||
+        (!IsVariableListItem(*sym) && !IsCommonBlock(*sym))) {
+      context_.Say(arg.source,
+          "GROUPPRIVATE argument should be a variable or a named common block"_err_en_US);
+      continue;
+    }
+
+    if (sym->has<AssocEntityDetails>()) {
+      context_.SayWithDecl(*sym, arg.source,
+          "GROUPPRIVATE argument cannot be an ASSOCIATE name"_err_en_US);
+      continue;
+    }
+    if (auto *obj{sym->detailsIf<ObjectEntityDetails>()}) {
+      if (obj->IsCoarray()) {
+        context_.Say(
+            arg.source, "GROUPPRIVATE argument cannot be a coarray"_err_en_US);
+        continue;
+      }
+      if (obj->init()) {
+        context_.SayWithDecl(*sym, arg.source,
+            "GROUPPRIVATE argument cannot be declared with an initializer"_err_en_US);
+        continue;
+      }
+    }
+    if (sym->test(Symbol::Flag::InCommonBlock)) {
+      context_.Say(arg.source,
+          "GROUPPRIVATE argument cannot be a member of a common block"_err_en_US);
+      continue;
+    }
+    if (!IsCommonBlock(*sym)) {
+      const Scope &thisScope{context_.FindScope(x.v.source)};
+      if (thisScope != sym->owner()) {
+        context_.SayWithDecl(*sym, arg.source,
+            "GROUPPRIVATE argument variable must be declared in the same scope as the construct on which it appears"_err_en_US);
+        continue;
+      } else if (!thisScope.IsModule() && !sym->attrs().test(Attr::SAVE)) {
+        context_.SayWithDecl(*sym, arg.source,
+            "GROUPPRIVATE argument variable must be declared in the module scope or have SAVE attribute"_err_en_US);
+        continue;
+      }
+    }
+  }
+}
+
+void OmpStructureChecker::Leave(const parser::OpenMPGroupprivate &x) {
+  dirContext_.pop_back();
+}
+
 void OmpStructureChecker::Enter(const parser::OpenMPThreadprivate &c) {
   const auto &dir{std::get<parser::Verbatim>(c.t)};
   PushContextAndClauseSets(
@@ -2034,41 +2149,87 @@ void OmpStructureChecker::Leave(const parser::OpenMPCancelConstruct &) {
 }
 
 void OmpStructureChecker::Enter(const parser::OpenMPCriticalConstruct &x) {
-  const auto &dir{std::get<parser::OmpCriticalDirective>(x.t)};
-  const auto &dirSource{std::get<parser::Verbatim>(dir.t).source};
-  const auto &endDir{std::get<parser::OmpEndCriticalDirective>(x.t)};
-  PushContextAndClauseSets(dirSource, llvm::omp::Directive::OMPD_critical);
+  const parser::OmpBeginDirective &beginSpec{x.BeginDir()};
+  const std::optional<parser::OmpEndDirective> &endSpec{x.EndDir()};
+  PushContextAndClauseSets(beginSpec.DirName().source, beginSpec.DirName().v);
+
   const auto &block{std::get<parser::Block>(x.t)};
-  CheckNoBranching(block, llvm::omp::Directive::OMPD_critical, dir.source);
-  const auto &dirName{std::get<std::optional<parser::Name>>(dir.t)};
-  const auto &endDirName{std::get<std::optional<parser::Name>>(endDir.t)};
-  const auto &ompClause{std::get<parser::OmpClauseList>(dir.t)};
-  if (dirName && endDirName &&
-      dirName->ToString().compare(endDirName->ToString())) {
-    context_
-        .Say(endDirName->source,
-            parser::MessageFormattedText{
-                "CRITICAL directive names do not match"_err_en_US})
-        .Attach(dirName->source, "should be "_en_US);
-  } else if (dirName && !endDirName) {
-    context_
-        .Say(dirName->source,
-            parser::MessageFormattedText{
-                "CRITICAL directive names do not match"_err_en_US})
-        .Attach(dirName->source, "should be NULL"_en_US);
-  } else if (!dirName && endDirName) {
-    context_
-        .Say(endDirName->source,
-            parser::MessageFormattedText{
-                "CRITICAL directive names do not match"_err_en_US})
-        .Attach(endDirName->source, "should be NULL"_en_US);
-  }
-  if (!dirName && !ompClause.source.empty() &&
-      ompClause.source.NULTerminatedToString() != "hint(omp_sync_hint_none)") {
-    context_.Say(dir.source,
-        parser::MessageFormattedText{
-            "Hint clause other than omp_sync_hint_none cannot be specified for "
-            "an unnamed CRITICAL directive"_err_en_US});
+  CheckNoBranching(
+      block, llvm::omp::Directive::OMPD_critical, beginSpec.DirName().source);
+
+  auto getNameFromArg{[](const parser::OmpArgument &arg) {
+    if (auto *object{parser::Unwrap<parser::OmpObject>(arg.u)}) {
+      if (auto *designator{omp::GetDesignatorFromObj(*object)}) {
+        return getDesignatorNameIfDataRef(*designator);
+      }
+    }
+    return static_cast<const parser::Name *>(nullptr);
+  }};
+
+  auto checkArgumentList{[&](const parser::OmpArgumentList &args) {
+    if (args.v.size() > 1) {
+      context_.Say(args.source,
+          "Only a single argument is allowed in CRITICAL directive"_err_en_US);
+    } else if (!args.v.empty()) {
+      if (!getNameFromArg(args.v.front())) {
+        context_.Say(args.v.front().source,
+            "CRITICAL argument should be a name"_err_en_US);
+      }
+    }
+  }};
+
+  const parser::Name *beginName{nullptr};
+  const parser::Name *endName{nullptr};
+
+  auto &beginArgs{beginSpec.Arguments()};
+  checkArgumentList(beginArgs);
+
+  if (!beginArgs.v.empty()) {
+    beginName = getNameFromArg(beginArgs.v.front());
+  }
+
+  if (endSpec) {
+    auto &endArgs{endSpec->Arguments()};
+    checkArgumentList(endArgs);
+
+    if (beginArgs.v.empty() != endArgs.v.empty()) {
+      parser::CharBlock source{
+          beginArgs.v.empty() ? endArgs.source : beginArgs.source};
+      context_.Say(source,
+          "Either both CRITICAL and END CRITICAL should have an argument, or none of them should"_err_en_US);
+    } else if (!beginArgs.v.empty()) {
+      endName = getNameFromArg(endArgs.v.front());
+      if (beginName && endName) {
+        if (beginName->ToString() != endName->ToString()) {
+          context_.Say(endName->source,
+              "The names on CRITICAL and END CRITICAL must match"_err_en_US);
+        }
+      }
+    }
+  }
+
+  for (auto &clause : beginSpec.Clauses().v) {
+    auto *hint{std::get_if<parser::OmpClause::Hint>(&clause.u)};
+    if (!hint) {
+      continue;
+    }
+    const int64_t OmpSyncHintNone = 0; // omp_sync_hint_none
+    std::optional<int64_t> hintValue{GetIntValue(hint->v.v)};
+    if (hintValue && *hintValue != OmpSyncHintNone) {
+      // Emit a diagnostic if the name is missing, and point to the directive
+      // with a missing name.
+      parser::CharBlock source;
+      if (!beginName) {
+        source = beginSpec.DirName().source;
+      } else if (endSpec && !endName) {
+        source = endSpec->DirName().source;
+      }
+
+      if (!source.empty()) {
+        context_.Say(source,
+            "When HINT other than 'omp_sync_hint_none' is present, CRITICAL directive should have a name"_err_en_US);
+      }
+    }
   }
 }
 
@@ -2523,11 +2684,24 @@ void OmpStructureChecker::Enter(const parser::OmpClause &x) {
     break;
   }
 
+  // Named constants are OK to be used within 'shared' and 'firstprivate'
+  // clauses.  The check for this happens a few lines below.
+  bool SharedOrFirstprivate = false;
+  switch (x.Id()) {
+  case llvm::omp::Clause::OMPC_shared:
+  case llvm::omp::Clause::OMPC_firstprivate:
+    SharedOrFirstprivate = true;
+    break;
+  default:
+    break;
+  }
+
   if (const parser::OmpObjectList *objList{GetOmpObjectList(x)}) {
     SymbolSourceMap symbols;
     GetSymbolsInObjectList(*objList, symbols);
     for (const auto &[symbol, source] : symbols) {
-      if (!IsVariableListItem(*symbol)) {
+      if (!IsVariableListItem(*symbol) &&
+          !(IsNamedConstant(*symbol) && SharedOrFirstprivate)) {
         deferredNonVariables_.insert({symbol, source});
       }
     }
@@ -2543,6 +2717,7 @@ CHECK_SIMPLE_CLAUSE(Default, OMPC_default)
 CHECK_SIMPLE_CLAUSE(Depobj, OMPC_depobj)
 CHECK_SIMPLE_CLAUSE(DeviceType, OMPC_device_type)
 CHECK_SIMPLE_CLAUSE(DistSchedule, OMPC_dist_schedule)
+CHECK_SIMPLE_CLAUSE(DynGroupprivate, OMPC_dyn_groupprivate)
 CHECK_SIMPLE_CLAUSE(Exclusive, OMPC_exclusive)
 CHECK_SIMPLE_CLAUSE(Final, OMPC_final)
 CHECK_SIMPLE_CLAUSE(Flush, OMPC_flush)
@@ -2853,7 +3028,8 @@ static bool CheckSymbolSupportsType(const Scope &scope,
 
 static bool IsReductionAllowedForType(
     const parser::OmpReductionIdentifier &ident, const DeclTypeSpec &type,
-    const Scope &scope, SemanticsContext &context) {
+    bool cannotBeBuiltinReduction, const Scope &scope,
+    SemanticsContext &context) {
   auto isLogical{[](const DeclTypeSpec &type) -> bool {
     return type.category() == DeclTypeSpec::Logical;
   }};
@@ -2864,6 +3040,10 @@ static bool IsReductionAllowedForType(
   auto checkOperator{[&](const parser::DefinedOperator &dOpr) {
     if (const auto *intrinsicOp{
             std::get_if<parser::DefinedOperator::IntrinsicOperator>(&dOpr.u)}) {
+      if (cannotBeBuiltinReduction) {
+        return false;
+      }
+
       // OMP5.2: The type [...] of a list item that appears in a
       // reduction clause must be valid for the combiner expression
       // See F2023: Table 10.2
@@ -2915,7 +3095,8 @@ static bool IsReductionAllowedForType(
         // IAND: arguments must be integers: F2023 16.9.100
         // IEOR: arguments must be integers: F2023 16.9.106
         // IOR: arguments must be integers: F2023 16.9.111
-        if (type.IsNumeric(TypeCategory::Integer)) {
+        if (type.IsNumeric(TypeCategory::Integer) &&
+            !cannotBeBuiltinReduction) {
           return true;
         }
       } else if (realName == "max" || realName == "min") {
@@ -2923,8 +3104,9 @@ static bool IsReductionAllowedForType(
         // F2023 16.9.135
         // MIN: arguments must be integer, real, or character:
         // F2023 16.9.141
-        if (type.IsNumeric(TypeCategory::Integer) ||
-            type.IsNumeric(TypeCategory::Real) || isCharacter(type)) {
+        if ((type.IsNumeric(TypeCategory::Integer) ||
+                type.IsNumeric(TypeCategory::Real) || isCharacter(type)) &&
+            !cannotBeBuiltinReduction) {
           return true;
         }
       }
@@ -2957,9 +3139,16 @@ void OmpStructureChecker::CheckReductionObjectTypes(
   GetSymbolsInObjectList(objects, symbols);
 
   for (auto &[symbol, source] : symbols) {
+    // Built in reductions require types which can be used in their initializer
+    // and combiner expressions. For example, for +:
+    // r = 0; r = r + r2
+    // But it might be valid to use these with DECLARE REDUCTION.
+    // Assumed size is already caught elsewhere.
+    bool cannotBeBuiltinReduction{evaluate::IsAssumedRank(*symbol)};
     if (auto *type{symbol->GetType()}) {
       const auto &scope{context_.FindScope(symbol->name())};
-      if (!IsReductionAllowedForType(ident, *type, scope, context_)) {
+      if (!IsReductionAllowedForType(
+              ident, *type, cannotBeBuiltinReduction, scope, context_)) {
         context_.Say(source,
             "The type of '%s' is incompatible with the reduction operator."_err_en_US,
             symbol->name());
@@ -4432,6 +4621,27 @@ void OmpStructureChecker::CheckWorkshareBlockStmts(
   }
 }
 
+void OmpStructureChecker::CheckWorkdistributeBlockStmts(
+    const parser::Block &block, parser::CharBlock source) {
+  unsigned version{context_.langOptions().OpenMPVersion};
+  unsigned since{60};
+  if (version < since)
+    context_.Say(source,
+        "WORKDISTRIBUTE construct is not allowed in %s, %s"_err_en_US,
+        ThisVersion(version), TryVersion(since));
+
+  OmpWorkdistributeBlockChecker ompWorkdistributeBlockChecker{context_, source};
+
+  for (auto it{block.begin()}; it != block.end(); ++it) {
+    if (parser::Unwrap<parser::AssignmentStmt>(*it)) {
+      parser::Walk(*it, ompWorkdistributeBlockChecker);
+    } else {
+      context_.Say(source,
+          "The structured block in a WORKDISTRIBUTE construct may consist of only SCALAR or ARRAY assignments"_err_en_US);
+    }
+  }
+}
+
 void OmpStructureChecker::CheckIfContiguous(const parser::OmpObject &object) {
   if (auto contig{IsContiguous(context_, object)}; contig && !*contig) {
     const parser::Name *name{GetObjectName(object)};
@@ -4475,42 +4685,6 @@ const parser::Name *OmpStructureChecker::GetObjectName(
   return NameHelper::Visit(object);
 }
 
-const parser::OmpObjectList *OmpStructureChecker::GetOmpObjectList(
-    const parser::OmpClause &clause) {
-
-  // Clauses with OmpObjectList as its data member
-  using MemberObjectListClauses =
-      std::tuple<parser::OmpClause::Copyprivate, parser::OmpClause::Copyin,
-          parser::OmpClause::Firstprivate, parser::OmpClause::Link,
-          parser::OmpClause::Private, parser::OmpClause::Shared,
-          parser::OmpClause::UseDevicePtr, parser::OmpClause::UseDeviceAddr>;
-
-  // Clauses with OmpObjectList in the tuple
-  using TupleObjectListClauses =
-      std::tuple<parser::OmpClause::Aligned, parser::OmpClause::Allocate,
-          parser::OmpClause::From, parser::OmpClause::Lastprivate,
-          parser::OmpClause::Map, parser::OmpClause::Reduction,
-          parser::OmpClause::To, parser::OmpClause::Enter>;
-
-  // TODO:: Generate the tuples using TableGen.
-  // Handle other constructs with OmpObjectList such as OpenMPThreadprivate.
-  return common::visit(
-      common::visitors{
-          [&](const auto &x) -> const parser::OmpObjectList * {
-            using Ty = std::decay_t<decltype(x)>;
-            if constexpr (common::HasMember<Ty, MemberObjectListClauses>) {
-              return &x.v;
-            } else if constexpr (common::HasMember<Ty,
-                                     TupleObjectListClauses>) {
-              return &(std::get<parser::OmpObjectList>(x.v.t));
-            } else {
-              return nullptr;
-            }
-          },
-      },
-      clause.u);
-}
-
 void OmpStructureChecker::Enter(
     const parser::OmpClause::AtomicDefaultMemOrder &x) {
   CheckAllowedRequiresClause(llvm::omp::Clause::OMPC_atomic_default_mem_order);
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 6b33ca6ab583f..8ac905242162f 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -126,6 +126,8 @@ class OmpStructureChecker
   void Leave(const parser::OpenMPAllocatorsConstruct &);
   void Enter(const parser::OpenMPRequiresConstruct &);
   void Leave(const parser::OpenMPRequiresConstruct &);
+  void Enter(const parser::OpenMPGroupprivate &);
+  void Leave(const parser::OpenMPGroupprivate &);
   void Enter(const parser::OpenMPThreadprivate &);
   void Leave(const parser::OpenMPThreadprivate &);
 
@@ -222,8 +224,9 @@ class OmpStructureChecker
       const parser::OmpObject &obj, llvm::StringRef clause = "");
   void CheckVarIsNotPartOfAnotherVar(const parser::CharBlock &source,
       const parser::OmpObjectList &objList, llvm::StringRef clause = "");
-  void CheckThreadprivateOrDeclareTargetVar(
-      const parser::OmpObjectList &objList);
+  void CheckThreadprivateOrDeclareTargetVar(const parser::Designator &);
+  void CheckThreadprivateOrDeclareTargetVar(const parser::Name &);
+  void CheckThreadprivateOrDeclareTargetVar(const parser::OmpObjectList &);
   void CheckSymbolNames(
       const parser::CharBlock &source, const parser::OmpObjectList &objList);
   void CheckIntentInPointer(SymbolSourceMap &, const llvm::omp::Clause);
@@ -242,6 +245,7 @@ class OmpStructureChecker
       llvmOmpClause clause, const parser::OmpObjectList &ompObjectList);
   bool CheckTargetBlockOnlyTeams(const parser::Block &);
   void CheckWorkshareBlockStmts(const parser::Block &, parser::CharBlock);
+  void CheckWorkdistributeBlockStmts(const parser::Block &, parser::CharBlock);
 
   void CheckIteratorRange(const parser::OmpIteratorSpecifier &x);
   void CheckIteratorModifier(const parser::OmpIterator &x);
@@ -267,8 +271,10 @@ class OmpStructureChecker
       const evaluate::Assignment &read, parser::CharBlock source);
   void CheckAtomicWriteAssignment(
       const evaluate::Assignment &write, parser::CharBlock source);
-  void CheckAtomicUpdateAssignment(
+  std::optional<evaluate::Assignment> CheckAtomicUpdateAssignment(
       const evaluate::Assignment &update, parser::CharBlock source);
+  std::pair<bool, bool> CheckAtomicUpdateAssignmentRhs(const SomeExpr &atom,
+      const SomeExpr &rhs, parser::CharBlock source, bool suppressDiagnostics);
   void CheckAtomicConditionalUpdateAssignment(const SomeExpr &cond,
       parser::CharBlock condSource, const evaluate::Assignment &assign,
       parser::CharBlock assignSource);
@@ -321,7 +327,6 @@ class OmpStructureChecker
       const parser::OmpObjectList &ompObjectList);
   void CheckIfContiguous(const parser::OmpObject &object);
   const parser::Name *GetObjectName(const parser::OmpObject &object);
-  const parser::OmpObjectList *GetOmpObjectList(const parser::OmpClause &);
   void CheckPredefinedAllocatorRestriction(const parser::CharBlock &source,
       const parser::OmpObjectList &ompObjectList);
   void CheckPredefinedAllocatorRestriction(
diff --git a/flang/lib/Semantics/data-to-inits.cpp b/flang/lib/Semantics/data-to-inits.cpp
index b4c83bab67088..1c454385e6989 100644
--- a/flang/lib/Semantics/data-to-inits.cpp
+++ b/flang/lib/Semantics/data-to-inits.cpp
@@ -285,21 +285,22 @@ template <typename DSV>
 std::optional<std::pair<SomeExpr, bool>>
 DataInitializationCompiler<DSV>::ConvertElement(
     const SomeExpr &expr, const evaluate::DynamicType &type) {
+  evaluate::FoldingContext &foldingContext{exprAnalyzer_.GetFoldingContext()};
+  evaluate::CheckRealWidening(expr, type, foldingContext);
   if (auto converted{evaluate::ConvertToType(type, SomeExpr{expr})}) {
     return {std::make_pair(std::move(*converted), false)};
   }
   // Allow DATA initialization with Hollerith and kind=1 CHARACTER like
   // (most) other Fortran compilers do.
-  if (auto converted{evaluate::HollerithToBOZ(
-          exprAnalyzer_.GetFoldingContext(), expr, type)}) {
+  if (auto converted{evaluate::HollerithToBOZ(foldingContext, expr, type)}) {
     return {std::make_pair(std::move(*converted), true)};
   }
   SemanticsContext &context{exprAnalyzer_.context()};
   if (context.IsEnabled(common::LanguageFeature::LogicalIntegerAssignment)) {
     if (MaybeExpr converted{evaluate::DataConstantConversionExtension(
-            exprAnalyzer_.GetFoldingContext(), type, expr)}) {
+            foldingContext, type, expr)}) {
       context.Warn(common::LanguageFeature::LogicalIntegerAssignment,
-          exprAnalyzer_.GetFoldingContext().messages().at(),
+          foldingContext.messages().at(),
           "nonstandard usage: initialization of %s with %s"_port_en_US,
           type.AsFortran(), expr.GetType().value().AsFortran());
       return {std::make_pair(std::move(*converted), false)};
diff --git a/flang/lib/Semantics/expression.cpp b/flang/lib/Semantics/expression.cpp
index 92dbe0e5da11c..d022378ce1455 100644
--- a/flang/lib/Semantics/expression.cpp
+++ b/flang/lib/Semantics/expression.cpp
@@ -828,7 +828,7 @@ MaybeExpr ExpressionAnalyzer::Analyze(
 
 template <typename TYPE>
 Constant<TYPE> ReadRealLiteral(
-    parser::CharBlock source, FoldingContext &context) {
+    parser::CharBlock source, FoldingContext &context, bool isDefaultKind) {
   const char *p{source.begin()};
   auto valWithFlags{
       Scalar<TYPE>::Read(p, context.targetCharacteristics().roundingMode())};
@@ -838,19 +838,24 @@ Constant<TYPE> ReadRealLiteral(
   if (context.targetCharacteristics().areSubnormalsFlushedToZero()) {
     value = value.FlushSubnormalToZero();
   }
-  return {value};
+  typename Constant<TYPE>::Result resultInfo;
+  resultInfo.set_isFromInexactLiteralConversion(
+      isDefaultKind && valWithFlags.flags.test(RealFlag::Inexact));
+  return {value, resultInfo};
 }
 
 struct RealTypeVisitor {
   using Result = std::optional<Expr<SomeReal>>;
   using Types = RealTypes;
 
-  RealTypeVisitor(int k, parser::CharBlock lit, FoldingContext &ctx)
-      : kind{k}, literal{lit}, context{ctx} {}
+  RealTypeVisitor(
+      int k, parser::CharBlock lit, FoldingContext &ctx, bool isDeftKind)
+      : kind{k}, literal{lit}, context{ctx}, isDefaultKind{isDeftKind} {}
 
   template <typename T> Result Test() {
     if (kind == T::kind) {
-      return {AsCategoryExpr(ReadRealLiteral<T>(literal, context))};
+      return {
+          AsCategoryExpr(ReadRealLiteral<T>(literal, context, isDefaultKind))};
     }
     return std::nullopt;
   }
@@ -858,6 +863,7 @@ struct RealTypeVisitor {
   int kind;
   parser::CharBlock literal;
   FoldingContext &context;
+  bool isDefaultKind;
 };
 
 // Reads a real literal constant and encodes it with the right kind.
@@ -909,8 +915,9 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::RealLiteralConstant &x) {
           "Explicit kind parameter together with non-'E' exponent letter is not standard"_port_en_US);
     }
   }
-  auto result{common::SearchTypes(
-      RealTypeVisitor{kind, x.real.source, GetFoldingContext()})};
+  bool isDefaultKind{!x.kind && letterKind.value_or('e') == 'e'};
+  auto result{common::SearchTypes(RealTypeVisitor{
+      kind, x.real.source, GetFoldingContext(), isDefaultKind})};
   if (!result) { // C717
     Say("Unsupported REAL(KIND=%d)"_err_en_US, kind);
   }
@@ -1841,8 +1848,7 @@ void ArrayConstructorContext::Push(MaybeExpr &&x) {
           if (*thisLen != *constantLength_ && !(messageDisplayedSet_ & 1)) {
             exprAnalyzer_.Warn(
                 common::LanguageFeature::DistinctArrayConstructorLengths,
-                "Character literal in array constructor without explicit "
-                "type has different length than earlier elements"_port_en_US);
+                "Character literal in array constructor without explicit type has different length than earlier elements"_port_en_US);
             messageDisplayedSet_ |= 1;
           }
           if (*thisLen > *constantLength_) {
@@ -1862,17 +1868,17 @@ void ArrayConstructorContext::Push(MaybeExpr &&x) {
     } else {
       if (!(messageDisplayedSet_ & 2)) {
         exprAnalyzer_.Say(
-            "Values in array constructor must have the same declared type "
-            "when no explicit type appears"_err_en_US); // C7110
+            "Values in array constructor must have the same declared type when no explicit type appears"_err_en_US); // C7110
         messageDisplayedSet_ |= 2;
       }
     }
   } else {
+    CheckRealWidening(*x, *type_, exprAnalyzer_.GetFoldingContext());
     if (auto cast{ConvertToType(*type_, std::move(*x))}) {
       values_.Push(std::move(*cast));
     } else if (!(messageDisplayedSet_ & 4)) {
-      exprAnalyzer_.Say("Value in array constructor of type '%s' could not "
-                        "be converted to the type of the array '%s'"_err_en_US,
+      exprAnalyzer_.Say(
+          "Value in array constructor of type '%s' could not be converted to the type of the array '%s'"_err_en_US,
           x->GetType()->AsFortran(), type_->AsFortran()); // C7111, C7112
       messageDisplayedSet_ |= 4;
     }
@@ -2065,8 +2071,9 @@ MaybeExpr ExpressionAnalyzer::Analyze(const parser::ArrayConstructor &array) {
 
 // Check if implicit conversion of expr to the symbol type is legal (if needed),
 // and make it explicit if requested.
-static MaybeExpr ImplicitConvertTo(const semantics::Symbol &sym,
-    Expr<SomeType> &&expr, bool keepConvertImplicit) {
+static MaybeExpr ImplicitConvertTo(const Symbol &sym, Expr<SomeType> &&expr,
+    bool keepConvertImplicit, FoldingContext &foldingContext) {
+  CheckRealWidening(expr, DynamicType::From(sym), foldingContext);
   if (!keepConvertImplicit) {
     return ConvertToType(sym, std::move(expr));
   } else {
@@ -2293,10 +2300,12 @@ MaybeExpr ExpressionAnalyzer::CheckStructureConstructor(
       // convert would cause a segfault. Lowering will deal with
       // conditionally converting and preserving the lower bounds in this
       // case.
-      if (MaybeExpr converted{ImplicitConvertTo(
-              *symbol, std::move(value), IsAllocatable(*symbol))}) {
-        if (auto componentShape{GetShape(GetFoldingContext(), *symbol)}) {
-          if (auto valueShape{GetShape(GetFoldingContext(), *converted)}) {
+      FoldingContext &foldingContext{GetFoldingContext()};
+      if (MaybeExpr converted{ImplicitConvertTo(*symbol, std::move(value),
+              /*keepConvertImplicit=*/IsAllocatable(*symbol),
+              foldingContext)}) {
+        if (auto componentShape{GetShape(foldingContext, *symbol)}) {
+          if (auto valueShape{GetShape(foldingContext, *converted)}) {
             if (GetRank(*componentShape) == 0 && GetRank(*valueShape) > 0) {
               AttachDeclaration(
                   Say(exprSource,
@@ -2310,7 +2319,7 @@ MaybeExpr ExpressionAnalyzer::CheckStructureConstructor(
               if (checked && *checked && GetRank(*componentShape) > 0 &&
                   GetRank(*valueShape) == 0 &&
                   (IsDeferredShape(*symbol) ||
-                      !IsExpandableScalar(*converted, GetFoldingContext(),
+                      !IsExpandableScalar(*converted, foldingContext,
                           *componentShape, true /*admit PURE call*/))) {
                 AttachDeclaration(
                     Say(exprSource,
@@ -4827,6 +4836,11 @@ std::optional<ProcedureRef> ArgumentAnalyzer::TryDefinedAssignment() {
     // conversion in this case.
     if (lhsType) {
       if (rhsType) {
+        FoldingContext &foldingContext{context_.GetFoldingContext()};
+        auto restorer{foldingContext.messages().SetLocation(
+            actuals_.at(1).value().sourceLocation().value_or(
+                foldingContext.messages().at()))};
+        CheckRealWidening(rhs, lhsType, foldingContext);
         if (!IsAllocatableDesignator(lhs) || context_.inWhereBody()) {
           AddAssignmentConversion(*lhsType, *rhsType);
         }
diff --git a/flang/lib/Semantics/openmp-utils.cpp b/flang/lib/Semantics/openmp-utils.cpp
index 7a492a4378907..e8df346ccdc3e 100644
--- a/flang/lib/Semantics/openmp-utils.cpp
+++ b/flang/lib/Semantics/openmp-utils.cpp
@@ -10,7 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "openmp-utils.h"
+#include "flang/Semantics/openmp-utils.h"
 
 #include "flang/Common/indirection.h"
 #include "flang/Common/reference.h"
diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index f08c77352f567..dbd4f512a0465 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -10,7 +10,6 @@
 
 #include "check-acc-structure.h"
 #include "check-omp-structure.h"
-#include "openmp-utils.h"
 #include "resolve-names-utils.h"
 #include "flang/Common/idioms.h"
 #include "flang/Evaluate/fold.h"
@@ -22,6 +21,7 @@
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/openmp-dsa.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/openmp-utils.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
 #include "flang/Support/Flags.h"
@@ -391,6 +391,9 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
     GetContext().withinConstruct = true;
   }
 
+  bool Pre(const parser::OpenMPGroupprivate &);
+  void Post(const parser::OpenMPGroupprivate &) { PopContext(); }
+
   bool Pre(const parser::OpenMPStandaloneConstruct &x) {
     common::visit(
         [&](auto &&s) {
@@ -793,7 +796,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
                   if (name->symbol) {
                     name->symbol->set(
                         ompFlag.value_or(Symbol::Flag::OmpMapStorage));
-                    AddToContextObjectWithDSA(*name->symbol, *ompFlag);
+                    AddToContextObjectWithDSA(*name->symbol,
+                        ompFlag.value_or(Symbol::Flag::OmpMapStorage));
                     if (semantics::IsAssumedSizeArray(*name->symbol)) {
                       context_.Say(designator.source,
                           "Assumed-size whole arrays may not appear on the %s "
@@ -841,7 +845,8 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
 
   Symbol::Flags ompFlagsRequireMark{Symbol::Flag::OmpThreadprivate,
       Symbol::Flag::OmpDeclareTarget, Symbol::Flag::OmpExclusiveScan,
-      Symbol::Flag::OmpInclusiveScan, Symbol::Flag::OmpInScanReduction};
+      Symbol::Flag::OmpInclusiveScan, Symbol::Flag::OmpInScanReduction,
+      Symbol::Flag::OmpGroupPrivate};
 
   Symbol::Flags dataCopyingAttributeFlags{
       Symbol::Flag::OmpCopyIn, Symbol::Flag::OmpCopyPrivate};
@@ -876,6 +881,9 @@ class OmpAttributeVisitor : DirectiveAttributeVisitor<llvm::omp::Directive> {
 
   bool IsNestedInDirective(llvm::omp::Directive directive);
   void ResolveOmpObjectList(const parser::OmpObjectList &, Symbol::Flag);
+  void ResolveOmpDesignator(
+      const parser::Designator &designator, Symbol::Flag ompFlag);
+  void ResolveOmpCommonBlock(const parser::Name &name, Symbol::Flag ompFlag);
   void ResolveOmpObject(const parser::OmpObject &, Symbol::Flag);
   Symbol *ResolveOmp(const parser::Name &, Symbol::Flag, Scope &);
   Symbol *ResolveOmp(Symbol &, Symbol::Flag, Scope &);
@@ -1732,10 +1740,13 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_taskgroup:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_workshare:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
   case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_teams_workdistribute:
     PushContext(dirSpec.source, dirId);
     break;
   default:
@@ -1765,9 +1776,12 @@ void OmpAttributeVisitor::Post(const parser::OpenMPBlockConstruct &x) {
   case llvm::omp::Directive::OMPD_target:
   case llvm::omp::Directive::OMPD_task:
   case llvm::omp::Directive::OMPD_teams:
+  case llvm::omp::Directive::OMPD_workdistribute:
   case llvm::omp::Directive::OMPD_parallel_workshare:
   case llvm::omp::Directive::OMPD_target_teams:
-  case llvm::omp::Directive::OMPD_target_parallel: {
+  case llvm::omp::Directive::OMPD_target_parallel:
+  case llvm::omp::Directive::OMPD_target_teams_workdistribute:
+  case llvm::omp::Directive::OMPD_teams_workdistribute: {
     bool hasPrivate;
     for (const auto *allocName : allocateNames_) {
       hasPrivate = false;
@@ -2114,6 +2128,18 @@ void OmpAttributeVisitor::CheckAssocLoopLevel(
   }
 }
 
+bool OmpAttributeVisitor::Pre(const parser::OpenMPGroupprivate &x) {
+  PushContext(x.source, llvm::omp::Directive::OMPD_groupprivate);
+  for (const parser::OmpArgument &arg : x.v.Arguments().v) {
+    if (auto *locator{std::get_if<parser::OmpLocator>(&arg.u)}) {
+      if (auto *object{std::get_if<parser::OmpObject>(&locator->u)}) {
+        ResolveOmpObject(*object, Symbol::Flag::OmpGroupPrivate);
+      }
+    }
+  }
+  return true;
+}
+
 bool OmpAttributeVisitor::Pre(const parser::OpenMPSectionsConstruct &x) {
   const auto &beginSectionsDir{
       std::get<parser::OmpBeginSectionsDirective>(x.t)};
@@ -2139,18 +2165,9 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPSectionConstruct &x) {
 }
 
 bool OmpAttributeVisitor::Pre(const parser::OpenMPCriticalConstruct &x) {
-  const auto &beginCriticalDir{std::get<parser::OmpCriticalDirective>(x.t)};
-  const auto &endCriticalDir{std::get<parser::OmpEndCriticalDirective>(x.t)};
-  PushContext(beginCriticalDir.source, llvm::omp::Directive::OMPD_critical);
+  const parser::OmpBeginDirective &beginSpec{x.BeginDir()};
+  PushContext(beginSpec.DirName().source, beginSpec.DirName().v);
   GetContext().withinConstruct = true;
-  if (const auto &criticalName{
-          std::get<std::optional<parser::Name>>(beginCriticalDir.t)}) {
-    ResolveOmpName(*criticalName, Symbol::Flag::OmpCriticalLock);
-  }
-  if (const auto &endCriticalName{
-          std::get<std::optional<parser::Name>>(endCriticalDir.t)}) {
-    ResolveOmpName(*endCriticalName, Symbol::Flag::OmpCriticalLock);
-  }
   return true;
 }
 
@@ -2795,196 +2812,182 @@ static bool SymbolOrEquivalentIsInNamelist(const Symbol &symbol) {
   });
 }
 
-void OmpAttributeVisitor::ResolveOmpObject(
-    const parser::OmpObject &ompObject, Symbol::Flag ompFlag) {
+void OmpAttributeVisitor::ResolveOmpDesignator(
+    const parser::Designator &designator, Symbol::Flag ompFlag) {
   unsigned version{context_.langOptions().OpenMPVersion};
-  common::visit(
-      common::visitors{
-          [&](const parser::Designator &designator) {
-            if (const auto *name{
-                    semantics::getDesignatorNameIfDataRef(designator)}) {
-              if (auto *symbol{ResolveOmp(*name, ompFlag, currScope())}) {
-                auto checkExclusivelists =
-                    [&](const Symbol *symbol1, Symbol::Flag firstOmpFlag,
-                        const Symbol *symbol2, Symbol::Flag secondOmpFlag) {
-                      if ((symbol1->test(firstOmpFlag) &&
-                              symbol2->test(secondOmpFlag)) ||
-                          (symbol1->test(secondOmpFlag) &&
-                              symbol2->test(firstOmpFlag))) {
-                        context_.Say(designator.source,
-                            "Variable '%s' may not "
-                            "appear on both %s and %s "
-                            "clauses on a %s construct"_err_en_US,
-                            symbol2->name(),
-                            Symbol::OmpFlagToClauseName(firstOmpFlag),
-                            Symbol::OmpFlagToClauseName(secondOmpFlag),
-                            parser::ToUpperCaseLetters(
-                                llvm::omp::getOpenMPDirectiveName(
-                                    GetContext().directive, version)
-                                    .str()));
-                      }
-                    };
-                if (dataCopyingAttributeFlags.test(ompFlag)) {
-                  CheckDataCopyingClause(*name, *symbol, ompFlag);
-                } else {
-                  AddToContextObjectWithExplicitDSA(*symbol, ompFlag);
-                  if (dataSharingAttributeFlags.test(ompFlag)) {
-                    CheckMultipleAppearances(*name, *symbol, ompFlag);
-                  }
-                  if (privateDataSharingAttributeFlags.test(ompFlag)) {
-                    CheckObjectIsPrivatizable(*name, *symbol, ompFlag);
-                  }
+  llvm::omp::Directive directive{GetContext().directive};
 
-                  if (ompFlag == Symbol::Flag::OmpAllocate) {
-                    AddAllocateName(name);
-                  }
-                }
-                if (ompFlag == Symbol::Flag::OmpDeclarativeAllocateDirective &&
-                    IsAllocatable(*symbol) &&
-                    !IsNestedInDirective(llvm::omp::Directive::OMPD_allocate)) {
-                  context_.Say(designator.source,
-                      "List items specified in the ALLOCATE directive must not "
-                      "have the ALLOCATABLE attribute unless the directive is "
-                      "associated with an ALLOCATE statement"_err_en_US);
-                }
-                if ((ompFlag == Symbol::Flag::OmpDeclarativeAllocateDirective ||
-                        ompFlag ==
-                            Symbol::Flag::OmpExecutableAllocateDirective) &&
-                    ResolveOmpObjectScope(name) == nullptr) {
-                  context_.Say(designator.source, // 2.15.3
-                      "List items must be declared in the same scoping unit "
-                      "in which the %s directive appears"_err_en_US,
-                      parser::ToUpperCaseLetters(
-                          llvm::omp::getOpenMPDirectiveName(
-                              GetContext().directive, version)
-                              .str()));
-                }
-                if (ompFlag == Symbol::Flag::OmpReduction) {
-                  // Using variables inside of a namelist in OpenMP reductions
-                  // is allowed by the standard, but is not allowed for
-                  // privatisation. This looks like an oversight. If the
-                  // namelist is hoisted to a global, we cannot apply the
-                  // mapping for the reduction variable: resulting in incorrect
-                  // results. Disabling this hoisting could make some real
-                  // production code go slower. See discussion in #109303
-                  if (SymbolOrEquivalentIsInNamelist(*symbol)) {
-                    context_.Say(name->source,
-                        "Variable '%s' in NAMELIST cannot be in a REDUCTION clause"_err_en_US,
-                        name->ToString());
-                  }
-                }
-                if (ompFlag == Symbol::Flag::OmpInclusiveScan ||
-                    ompFlag == Symbol::Flag::OmpExclusiveScan) {
-                  if (!symbol->test(Symbol::Flag::OmpInScanReduction)) {
-                    context_.Say(name->source,
-                        "List item %s must appear in REDUCTION clause "
-                        "with the INSCAN modifier of the parent "
-                        "directive"_err_en_US,
-                        name->ToString());
-                  }
-                }
-                if (ompFlag == Symbol::Flag::OmpDeclareTarget) {
-                  if (symbol->IsFuncResult()) {
-                    if (Symbol * func{currScope().symbol()}) {
-                      CHECK(func->IsSubprogram());
-                      func->set(ompFlag);
-                      name->symbol = func;
-                    }
-                  }
-                }
-                if (GetContext().directive ==
-                    llvm::omp::Directive::OMPD_target_data) {
-                  checkExclusivelists(symbol, Symbol::Flag::OmpUseDevicePtr,
-                      symbol, Symbol::Flag::OmpUseDeviceAddr);
-                }
-                if (llvm::omp::allDistributeSet.test(GetContext().directive)) {
-                  checkExclusivelists(symbol, Symbol::Flag::OmpFirstPrivate,
-                      symbol, Symbol::Flag::OmpLastPrivate);
-                }
-                if (llvm::omp::allTargetSet.test(GetContext().directive)) {
-                  checkExclusivelists(symbol, Symbol::Flag::OmpIsDevicePtr,
-                      symbol, Symbol::Flag::OmpHasDeviceAddr);
-                  const auto *hostAssocSym{symbol};
-                  if (!(symbol->test(Symbol::Flag::OmpIsDevicePtr) ||
-                          symbol->test(Symbol::Flag::OmpHasDeviceAddr))) {
-                    if (const auto *details{
-                            symbol->detailsIf<HostAssocDetails>()}) {
-                      hostAssocSym = &details->symbol();
-                    }
-                  }
-                  Symbol::Flag dataMappingAttributeFlags[] = {
-                      Symbol::Flag::OmpMapTo, Symbol::Flag::OmpMapFrom,
-                      Symbol::Flag::OmpMapToFrom, Symbol::Flag::OmpMapStorage,
-                      Symbol::Flag::OmpMapDelete, Symbol::Flag::OmpIsDevicePtr,
-                      Symbol::Flag::OmpHasDeviceAddr};
-
-                  Symbol::Flag dataSharingAttributeFlags[] = {
-                      Symbol::Flag::OmpPrivate, Symbol::Flag::OmpFirstPrivate,
-                      Symbol::Flag::OmpLastPrivate, Symbol::Flag::OmpShared,
-                      Symbol::Flag::OmpLinear};
-
-                  // For OMP TARGET TEAMS directive some sharing attribute
-                  // flags and mapping attribute flags can co-exist.
-                  if (!(llvm::omp::allTeamsSet.test(GetContext().directive) ||
-                          llvm::omp::allParallelSet.test(
-                              GetContext().directive))) {
-                    for (Symbol::Flag ompFlag1 : dataMappingAttributeFlags) {
-                      for (Symbol::Flag ompFlag2 : dataSharingAttributeFlags) {
-                        if ((hostAssocSym->test(ompFlag2) &&
-                                hostAssocSym->test(
-                                    Symbol::Flag::OmpExplicit)) ||
-                            (symbol->test(ompFlag2) &&
-                                symbol->test(Symbol::Flag::OmpExplicit))) {
-                          checkExclusivelists(
-                              hostAssocSym, ompFlag1, symbol, ompFlag2);
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            } else {
-              // Array sections to be changed to substrings as needed
-              if (AnalyzeExpr(context_, designator)) {
-                if (std::holds_alternative<parser::Substring>(designator.u)) {
-                  context_.Say(designator.source,
-                      "Substrings are not allowed on OpenMP "
-                      "directives or clauses"_err_en_US);
-                }
-              }
-              // other checks, more TBD
-            }
-          },
-          [&](const parser::Name &name) { // common block
-            if (auto *symbol{ResolveOmpCommonBlockName(&name)}) {
-              if (!dataCopyingAttributeFlags.test(ompFlag)) {
-                CheckMultipleAppearances(
-                    name, *symbol, Symbol::Flag::OmpCommonBlock);
-              }
-              // 2.15.3 When a named common block appears in a list, it has the
-              // same meaning as if every explicit member of the common block
-              // appeared in the list
-              auto &details{symbol->get<CommonBlockDetails>()};
-              unsigned index{0};
-              for (auto &object : details.objects()) {
-                if (auto *resolvedObject{
-                        ResolveOmp(*object, ompFlag, currScope())}) {
-                  if (dataCopyingAttributeFlags.test(ompFlag)) {
-                    CheckDataCopyingClause(name, *resolvedObject, ompFlag);
-                  } else {
-                    AddToContextObjectWithExplicitDSA(*resolvedObject, ompFlag);
-                  }
-                  details.replace_object(*resolvedObject, index);
-                }
-                index++;
-              }
-            } else {
-              context_.Say(name.source, // 2.15.3
-                  "COMMON block must be declared in the same scoping unit "
-                  "in which the OpenMP directive or clause appears"_err_en_US);
+  const auto *name{semantics::getDesignatorNameIfDataRef(designator)};
+  if (!name) {
+    // Array sections to be changed to substrings as needed
+    if (AnalyzeExpr(context_, designator)) {
+      if (std::holds_alternative<parser::Substring>(designator.u)) {
+        context_.Say(designator.source,
+            "Substrings are not allowed on OpenMP directives or clauses"_err_en_US);
+      }
+    }
+    // other checks, more TBD
+    return;
+  }
+
+  if (auto *symbol{ResolveOmp(*name, ompFlag, currScope())}) {
+    auto checkExclusivelists{//
+        [&](const Symbol *symbol1, Symbol::Flag firstOmpFlag,
+            const Symbol *symbol2, Symbol::Flag secondOmpFlag) {
+          if ((symbol1->test(firstOmpFlag) && symbol2->test(secondOmpFlag)) ||
+              (symbol1->test(secondOmpFlag) && symbol2->test(firstOmpFlag))) {
+            context_.Say(designator.source,
+                "Variable '%s' may not appear on both %s and %s clauses on a %s construct"_err_en_US,
+                symbol2->name(), Symbol::OmpFlagToClauseName(firstOmpFlag),
+                Symbol::OmpFlagToClauseName(secondOmpFlag),
+                parser::ToUpperCaseLetters(
+                    llvm::omp::getOpenMPDirectiveName(directive, version)));
+          }
+        }};
+    if (dataCopyingAttributeFlags.test(ompFlag)) {
+      CheckDataCopyingClause(*name, *symbol, ompFlag);
+    } else {
+      AddToContextObjectWithExplicitDSA(*symbol, ompFlag);
+      if (dataSharingAttributeFlags.test(ompFlag)) {
+        CheckMultipleAppearances(*name, *symbol, ompFlag);
+      }
+      if (privateDataSharingAttributeFlags.test(ompFlag)) {
+        CheckObjectIsPrivatizable(*name, *symbol, ompFlag);
+      }
+
+      if (ompFlag == Symbol::Flag::OmpAllocate) {
+        AddAllocateName(name);
+      }
+    }
+    if (ompFlag == Symbol::Flag::OmpDeclarativeAllocateDirective &&
+        IsAllocatable(*symbol) &&
+        !IsNestedInDirective(llvm::omp::Directive::OMPD_allocate)) {
+      context_.Say(designator.source,
+          "List items specified in the ALLOCATE directive must not have the ALLOCATABLE attribute unless the directive is associated with an ALLOCATE statement"_err_en_US);
+    }
+    if ((ompFlag == Symbol::Flag::OmpDeclarativeAllocateDirective ||
+            ompFlag == Symbol::Flag::OmpExecutableAllocateDirective) &&
+        ResolveOmpObjectScope(name) == nullptr) {
+      context_.Say(designator.source, // 2.15.3
+          "List items must be declared in the same scoping unit in which the %s directive appears"_err_en_US,
+          parser::ToUpperCaseLetters(
+              llvm::omp::getOpenMPDirectiveName(directive, version)));
+    }
+    if (ompFlag == Symbol::Flag::OmpReduction) {
+      // Using variables inside of a namelist in OpenMP reductions
+      // is allowed by the standard, but is not allowed for
+      // privatisation. This looks like an oversight. If the
+      // namelist is hoisted to a global, we cannot apply the
+      // mapping for the reduction variable: resulting in incorrect
+      // results. Disabling this hoisting could make some real
+      // production code go slower. See discussion in #109303
+      if (SymbolOrEquivalentIsInNamelist(*symbol)) {
+        context_.Say(name->source,
+            "Variable '%s' in NAMELIST cannot be in a REDUCTION clause"_err_en_US,
+            name->ToString());
+      }
+    }
+    if (ompFlag == Symbol::Flag::OmpInclusiveScan ||
+        ompFlag == Symbol::Flag::OmpExclusiveScan) {
+      if (!symbol->test(Symbol::Flag::OmpInScanReduction)) {
+        context_.Say(name->source,
+            "List item %s must appear in REDUCTION clause with the INSCAN modifier of the parent directive"_err_en_US,
+            name->ToString());
+      }
+    }
+    if (ompFlag == Symbol::Flag::OmpDeclareTarget) {
+      if (symbol->IsFuncResult()) {
+        if (Symbol * func{currScope().symbol()}) {
+          CHECK(func->IsSubprogram());
+          func->set(ompFlag);
+          name->symbol = func;
+        }
+      }
+    }
+    if (directive == llvm::omp::Directive::OMPD_target_data) {
+      checkExclusivelists(symbol, Symbol::Flag::OmpUseDevicePtr, symbol,
+          Symbol::Flag::OmpUseDeviceAddr);
+    }
+    if (llvm::omp::allDistributeSet.test(directive)) {
+      checkExclusivelists(symbol, Symbol::Flag::OmpFirstPrivate, symbol,
+          Symbol::Flag::OmpLastPrivate);
+    }
+    if (llvm::omp::allTargetSet.test(directive)) {
+      checkExclusivelists(symbol, Symbol::Flag::OmpIsDevicePtr, symbol,
+          Symbol::Flag::OmpHasDeviceAddr);
+      const auto *hostAssocSym{symbol};
+      if (!symbol->test(Symbol::Flag::OmpIsDevicePtr) &&
+          !symbol->test(Symbol::Flag::OmpHasDeviceAddr)) {
+        if (const auto *details{symbol->detailsIf<HostAssocDetails>()}) {
+          hostAssocSym = &details->symbol();
+        }
+      }
+      static Symbol::Flag dataMappingAttributeFlags[] = {//
+          Symbol::Flag::OmpMapTo, Symbol::Flag::OmpMapFrom,
+          Symbol::Flag::OmpMapToFrom, Symbol::Flag::OmpMapStorage,
+          Symbol::Flag::OmpMapDelete, Symbol::Flag::OmpIsDevicePtr,
+          Symbol::Flag::OmpHasDeviceAddr};
+
+      static Symbol::Flag dataSharingAttributeFlags[] = {//
+          Symbol::Flag::OmpPrivate, Symbol::Flag::OmpFirstPrivate,
+          Symbol::Flag::OmpLastPrivate, Symbol::Flag::OmpShared,
+          Symbol::Flag::OmpLinear};
+
+      // For OMP TARGET TEAMS directive some sharing attribute
+      // flags and mapping attribute flags can co-exist.
+      if (!llvm::omp::allTeamsSet.test(directive) &&
+          !llvm::omp::allParallelSet.test(directive)) {
+        for (Symbol::Flag ompFlag1 : dataMappingAttributeFlags) {
+          for (Symbol::Flag ompFlag2 : dataSharingAttributeFlags) {
+            if ((hostAssocSym->test(ompFlag2) &&
+                    hostAssocSym->test(Symbol::Flag::OmpExplicit)) ||
+                (symbol->test(ompFlag2) &&
+                    symbol->test(Symbol::Flag::OmpExplicit))) {
+              checkExclusivelists(hostAssocSym, ompFlag1, symbol, ompFlag2);
             }
-          },
-      },
+          }
+        }
+      }
+    }
+  }
+}
+
+void OmpAttributeVisitor::ResolveOmpCommonBlock(
+    const parser::Name &name, Symbol::Flag ompFlag) {
+  if (auto *symbol{ResolveOmpCommonBlockName(&name)}) {
+    if (!dataCopyingAttributeFlags.test(ompFlag)) {
+      CheckMultipleAppearances(name, *symbol, Symbol::Flag::OmpCommonBlock);
+    }
+    // 2.15.3 When a named common block appears in a list, it has the
+    // same meaning as if every explicit member of the common block
+    // appeared in the list
+    auto &details{symbol->get<CommonBlockDetails>()};
+    for (auto [index, object] : llvm::enumerate(details.objects())) {
+      if (auto *resolvedObject{ResolveOmp(*object, ompFlag, currScope())}) {
+        if (dataCopyingAttributeFlags.test(ompFlag)) {
+          CheckDataCopyingClause(name, *resolvedObject, ompFlag);
+        } else {
+          AddToContextObjectWithExplicitDSA(*resolvedObject, ompFlag);
+        }
+        details.replace_object(*resolvedObject, index);
+      }
+    }
+  } else {
+    context_.Say(name.source, // 2.15.3
+        "COMMON block must be declared in the same scoping unit in which the OpenMP directive or clause appears"_err_en_US);
+  }
+}
+
+void OmpAttributeVisitor::ResolveOmpObject(
+    const parser::OmpObject &ompObject, Symbol::Flag ompFlag) {
+  common::visit(common::visitors{
+                    [&](const parser::Designator &designator) {
+                      ResolveOmpDesignator(designator, ompFlag);
+                    },
+                    [&](const parser::Name &name) { // common block
+                      ResolveOmpCommonBlock(name, ompFlag);
+                    },
+                },
       ompObject.u);
 }
 
diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index 25b13700cd3ab..5808b4b3cc4fe 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -30,6 +30,7 @@
 #include "flang/Semantics/attr.h"
 #include "flang/Semantics/expression.h"
 #include "flang/Semantics/openmp-modifiers.h"
+#include "flang/Semantics/openmp-utils.h"
 #include "flang/Semantics/program-tree.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/semantics.h"
@@ -1486,6 +1487,16 @@ class OmpVisitor : public virtual DeclarationVisitor {
   void Post(const parser::OpenMPBlockConstruct &);
   bool Pre(const parser::OmpBeginDirective &x) {
     AddOmpSourceRange(x.source);
+    // Manually resolve names in CRITICAL directives. This is because these
+    // names do not denote Fortran objects, and the CRITICAL directive causes
+    // them to be "auto-declared", i.e. inserted into the global scope.
+    // More specifically, they are not expected to have explicit declarations,
+    // and if they do the behavior is unspeficied.
+    if (x.DirName().v == llvm::omp::Directive::OMPD_critical) {
+      for (const parser::OmpArgument &arg : x.Arguments().v) {
+        ResolveCriticalName(arg);
+      }
+    }
     return true;
   }
   void Post(const parser::OmpBeginDirective &) {
@@ -1493,6 +1504,12 @@ class OmpVisitor : public virtual DeclarationVisitor {
   }
   bool Pre(const parser::OmpEndDirective &x) {
     AddOmpSourceRange(x.source);
+    // Manually resolve names in CRITICAL directives.
+    if (x.DirName().v == llvm::omp::Directive::OMPD_critical) {
+      for (const parser::OmpArgument &arg : x.Arguments().v) {
+        ResolveCriticalName(arg);
+      }
+    }
     return true;
   }
   void Post(const parser::OmpEndDirective &) {
@@ -1591,20 +1608,6 @@ class OmpVisitor : public virtual DeclarationVisitor {
   void Post(const parser::OmpEndSectionsDirective &) {
     messageHandler().set_currStmtSource(std::nullopt);
   }
-  bool Pre(const parser::OmpCriticalDirective &x) {
-    AddOmpSourceRange(x.source);
-    return true;
-  }
-  void Post(const parser::OmpCriticalDirective &) {
-    messageHandler().set_currStmtSource(std::nullopt);
-  }
-  bool Pre(const parser::OmpEndCriticalDirective &x) {
-    AddOmpSourceRange(x.source);
-    return true;
-  }
-  void Post(const parser::OmpEndCriticalDirective &) {
-    messageHandler().set_currStmtSource(std::nullopt);
-  }
   bool Pre(const parser::OpenMPThreadprivate &) {
     SkipImplicitTyping(true);
     return true;
@@ -1720,6 +1723,8 @@ class OmpVisitor : public virtual DeclarationVisitor {
       const std::optional<parser::OmpClauseList> &clauses,
       const T &wholeConstruct);
 
+  void ResolveCriticalName(const parser::OmpArgument &arg);
+
   int metaLevel_{0};
   const parser::OmpMetadirectiveDirective *metaDirective_{nullptr};
 };
@@ -1947,6 +1952,34 @@ void OmpVisitor::ProcessReductionSpecifier(
   }
 }
 
+void OmpVisitor::ResolveCriticalName(const parser::OmpArgument &arg) {
+  auto &globalScope{[&]() -> Scope & {
+    for (Scope *s{&currScope()};; s = &s->parent()) {
+      if (s->IsTopLevel()) {
+        return *s;
+      }
+    }
+    llvm_unreachable("Cannot find global scope");
+  }()};
+
+  if (auto *object{parser::Unwrap<parser::OmpObject>(arg.u)}) {
+    if (auto *desg{omp::GetDesignatorFromObj(*object)}) {
+      if (auto *name{getDesignatorNameIfDataRef(*desg)}) {
+        if (auto *symbol{FindInScope(globalScope, *name)}) {
+          if (!symbol->test(Symbol::Flag::OmpCriticalLock)) {
+            SayWithDecl(*name, *symbol,
+                "CRITICAL construct name '%s' conflicts with a previous declaration"_warn_en_US,
+                name->ToString());
+          }
+        } else {
+          name->symbol = &MakeSymbol(globalScope, name->source, Attrs{});
+          name->symbol->set(Symbol::Flag::OmpCriticalLock);
+        }
+      }
+    }
+  }
+}
+
 bool OmpVisitor::Pre(const parser::OmpDirectiveSpecification &x) {
   AddOmpSourceRange(x.source);
   if (metaLevel_ == 0) {
diff --git a/flang/lib/Semantics/rewrite-parse-tree.cpp b/flang/lib/Semantics/rewrite-parse-tree.cpp
index 4eeb1b9ed3c1e..b3019762ead1f 100644
--- a/flang/lib/Semantics/rewrite-parse-tree.cpp
+++ b/flang/lib/Semantics/rewrite-parse-tree.cpp
@@ -12,6 +12,7 @@
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Parser/tools.h"
+#include "flang/Semantics/openmp-directive-sets.h"
 #include "flang/Semantics/scope.h"
 #include "flang/Semantics/semantics.h"
 #include "flang/Semantics/symbol.h"
@@ -41,11 +42,23 @@ class RewriteMutator {
 
   void Post(parser::Name &);
   bool Pre(parser::MainProgram &);
+  bool Pre(parser::Module &);
   bool Pre(parser::FunctionSubprogram &);
   bool Pre(parser::SubroutineSubprogram &);
   bool Pre(parser::SeparateModuleSubprogram &);
   bool Pre(parser::BlockConstruct &);
+  bool Pre(parser::Block &);
+  bool Pre(parser::DoConstruct &);
+  bool Pre(parser::IfConstruct &);
   bool Pre(parser::ActionStmt &);
+  void Post(parser::MainProgram &);
+  void Post(parser::FunctionSubprogram &);
+  void Post(parser::SubroutineSubprogram &);
+  void Post(parser::SeparateModuleSubprogram &);
+  void Post(parser::BlockConstruct &);
+  void Post(parser::Block &);
+  void Post(parser::DoConstruct &);
+  void Post(parser::IfConstruct &);
   void Post(parser::ReadStmt &);
   void Post(parser::WriteStmt &);
 
@@ -67,8 +80,15 @@ class RewriteMutator {
   bool Pre(parser::EndSubroutineStmt &) { return false; }
   bool Pre(parser::EndTypeStmt &) { return false; }
 
+  bool Pre(parser::OpenMPBlockConstruct &);
+  bool Pre(parser::OpenMPLoopConstruct &);
+  void Post(parser::OpenMPBlockConstruct &);
+  void Post(parser::OpenMPLoopConstruct &);
+
 private:
   void FixMisparsedStmtFuncs(parser::SpecificationPart &, parser::Block &);
+  void OpenMPSimdOnly(parser::Block &, bool);
+  void OpenMPSimdOnly(parser::SpecificationPart &);
 
   SemanticsContext &context_;
   bool errorOnUnresolvedName_{true};
@@ -96,6 +116,132 @@ static bool ReturnsDataPointer(const Symbol &symbol) {
   return false;
 }
 
+static bool LoopConstructIsSIMD(parser::OpenMPLoopConstruct *ompLoop) {
+  auto &begin = std::get<parser::OmpBeginLoopDirective>(ompLoop->t);
+  auto directive = std::get<parser::OmpLoopDirective>(begin.t).v;
+  return llvm::omp::allSimdSet.test(directive);
+}
+
+// Remove non-SIMD OpenMPConstructs once they are parsed.
+// This massively simplifies the logic inside the SimdOnlyPass for
+// -fopenmp-simd.
+void RewriteMutator::OpenMPSimdOnly(parser::SpecificationPart &specPart) {
+  auto &list{std::get<std::list<parser::DeclarationConstruct>>(specPart.t)};
+  for (auto it{list.begin()}; it != list.end();) {
+    if (auto *specConstr{std::get_if<parser::SpecificationConstruct>(&it->u)}) {
+      if (auto *ompDecl{std::get_if<
+              common::Indirection<parser::OpenMPDeclarativeConstruct>>(
+              &specConstr->u)}) {
+        if (std::holds_alternative<parser::OpenMPThreadprivate>(
+                ompDecl->value().u) ||
+            std::holds_alternative<parser::OpenMPDeclareMapperConstruct>(
+                ompDecl->value().u)) {
+          it = list.erase(it);
+          continue;
+        }
+      }
+    }
+    ++it;
+  }
+}
+
+// Remove non-SIMD OpenMPConstructs once they are parsed.
+// This massively simplifies the logic inside the SimdOnlyPass for
+// -fopenmp-simd. `isNonSimdLoopBody` should be set to true if `block` is the
+// body of a non-simd OpenMP loop. This is to indicate that scan constructs
+// should be removed from the body, where they would be kept if it were a simd
+// loop.
+void RewriteMutator::OpenMPSimdOnly(
+    parser::Block &block, bool isNonSimdLoopBody = false) {
+  auto replaceInlineBlock =
+      [&](std::list<parser::ExecutionPartConstruct> &innerBlock,
+          auto it) -> auto {
+    auto insertPos = std::next(it);
+    block.splice(insertPos, innerBlock);
+    block.erase(it);
+    return insertPos;
+  };
+
+  for (auto it{block.begin()}; it != block.end();) {
+    if (auto *stmt{std::get_if<parser::ExecutableConstruct>(&it->u)}) {
+      if (auto *omp{std::get_if<common::Indirection<parser::OpenMPConstruct>>(
+              &stmt->u)}) {
+        if (auto *ompStandalone{std::get_if<parser::OpenMPStandaloneConstruct>(
+                &omp->value().u)}) {
+          if (std::holds_alternative<parser::OpenMPCancelConstruct>(
+                  ompStandalone->u) ||
+              std::holds_alternative<parser::OpenMPFlushConstruct>(
+                  ompStandalone->u) ||
+              std::holds_alternative<parser::OpenMPCancellationPointConstruct>(
+                  ompStandalone->u)) {
+            it = block.erase(it);
+            continue;
+          }
+          if (auto *constr{std::get_if<parser::OpenMPSimpleStandaloneConstruct>(
+                  &ompStandalone->u)}) {
+            auto directive = constr->v.DirId();
+            // Scan should only be removed from non-simd loops
+            if (llvm::omp::simpleStandaloneNonSimdOnlySet.test(directive) ||
+                (isNonSimdLoopBody && directive == llvm::omp::OMPD_scan)) {
+              it = block.erase(it);
+              continue;
+            }
+          }
+        } else if (auto *ompBlock{std::get_if<parser::OpenMPBlockConstruct>(
+                       &omp->value().u)}) {
+          it = replaceInlineBlock(std::get<parser::Block>(ompBlock->t), it);
+          continue;
+        } else if (auto *ompLoop{std::get_if<parser::OpenMPLoopConstruct>(
+                       &omp->value().u)}) {
+          if (LoopConstructIsSIMD(ompLoop)) {
+            ++it;
+            continue;
+          }
+          auto &nest =
+              std::get<std::optional<parser::NestedConstruct>>(ompLoop->t);
+
+          if (auto *doConstruct =
+                  std::get_if<parser::DoConstruct>(&nest.value())) {
+            auto &loopBody = std::get<parser::Block>(doConstruct->t);
+            // We can only remove some constructs from a loop when it's _not_ a
+            // OpenMP simd loop
+            OpenMPSimdOnly(loopBody, /*isNonSimdLoopBody=*/true);
+            auto newDoConstruct = std::move(*doConstruct);
+            auto newLoop = parser::ExecutionPartConstruct{
+                parser::ExecutableConstruct{std::move(newDoConstruct)}};
+            it = block.erase(it);
+            block.insert(it, std::move(newLoop));
+            continue;
+          }
+        } else if (auto *ompCon{std::get_if<parser::OpenMPSectionsConstruct>(
+                       &omp->value().u)}) {
+          auto &sections =
+              std::get<std::list<parser::OpenMPConstruct>>(ompCon->t);
+          auto insertPos = std::next(it);
+          for (auto &sectionCon : sections) {
+            auto &section =
+                std::get<parser::OpenMPSectionConstruct>(sectionCon.u);
+            auto &innerBlock = std::get<parser::Block>(section.t);
+            block.splice(insertPos, innerBlock);
+          }
+          block.erase(it);
+          it = insertPos;
+          continue;
+        } else if (auto *atomic{std::get_if<parser::OpenMPAtomicConstruct>(
+                       &omp->value().u)}) {
+          it = replaceInlineBlock(std::get<parser::Block>(atomic->t), it);
+          continue;
+        } else if (auto *critical{std::get_if<parser::OpenMPCriticalConstruct>(
+                       &omp->value().u)}) {
+          it = replaceInlineBlock(std::get<parser::Block>(critical->t), it);
+          continue;
+        }
+      }
+    }
+    ++it;
+  }
+}
+
 // Finds misparsed statement functions in a specification part, rewrites
 // them into array element assignment statements, and moves them into the
 // beginning of the corresponding (execution part's) block.
@@ -133,33 +279,155 @@ void RewriteMutator::FixMisparsedStmtFuncs(
 bool RewriteMutator::Pre(parser::MainProgram &program) {
   FixMisparsedStmtFuncs(std::get<parser::SpecificationPart>(program.t),
       std::get<parser::ExecutionPart>(program.t).v);
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::ExecutionPart>(program.t).v);
+    OpenMPSimdOnly(std::get<parser::SpecificationPart>(program.t));
+  }
+  return true;
+}
+
+void RewriteMutator::Post(parser::MainProgram &program) {
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::ExecutionPart>(program.t).v);
+  }
+}
+
+bool RewriteMutator::Pre(parser::Module &module) {
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::SpecificationPart>(module.t));
+  }
   return true;
 }
 
 bool RewriteMutator::Pre(parser::FunctionSubprogram &func) {
   FixMisparsedStmtFuncs(std::get<parser::SpecificationPart>(func.t),
       std::get<parser::ExecutionPart>(func.t).v);
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::ExecutionPart>(func.t).v);
+  }
   return true;
 }
 
+void RewriteMutator::Post(parser::FunctionSubprogram &func) {
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::ExecutionPart>(func.t).v);
+  }
+}
+
 bool RewriteMutator::Pre(parser::SubroutineSubprogram &subr) {
   FixMisparsedStmtFuncs(std::get<parser::SpecificationPart>(subr.t),
       std::get<parser::ExecutionPart>(subr.t).v);
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::ExecutionPart>(subr.t).v);
+  }
   return true;
 }
 
+void RewriteMutator::Post(parser::SubroutineSubprogram &subr) {
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::ExecutionPart>(subr.t).v);
+  }
+}
+
 bool RewriteMutator::Pre(parser::SeparateModuleSubprogram &subp) {
   FixMisparsedStmtFuncs(std::get<parser::SpecificationPart>(subp.t),
       std::get<parser::ExecutionPart>(subp.t).v);
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::ExecutionPart>(subp.t).v);
+  }
   return true;
 }
 
+void RewriteMutator::Post(parser::SeparateModuleSubprogram &subp) {
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::ExecutionPart>(subp.t).v);
+  }
+}
+
 bool RewriteMutator::Pre(parser::BlockConstruct &block) {
   FixMisparsedStmtFuncs(std::get<parser::BlockSpecificationPart>(block.t).v,
       std::get<parser::Block>(block.t));
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::Block>(block.t));
+  }
+  return true;
+}
+
+void RewriteMutator::Post(parser::BlockConstruct &block) {
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(std::get<parser::Block>(block.t));
+  }
+}
+
+bool RewriteMutator::Pre(parser::Block &block) {
+  if (context_.langOptions().OpenMPSimd) {
+    OpenMPSimdOnly(block);
+  }
   return true;
 }
 
+void RewriteMutator::Post(parser::Block &block) { this->Pre(block); }
+
+bool RewriteMutator::Pre(parser::OpenMPBlockConstruct &block) {
+  if (context_.langOptions().OpenMPSimd) {
+    auto &innerBlock = std::get<parser::Block>(block.t);
+    OpenMPSimdOnly(innerBlock);
+  }
+  return true;
+}
+
+void RewriteMutator::Post(parser::OpenMPBlockConstruct &block) {
+  this->Pre(block);
+}
+
+bool RewriteMutator::Pre(parser::OpenMPLoopConstruct &ompLoop) {
+  if (context_.langOptions().OpenMPSimd) {
+    if (LoopConstructIsSIMD(&ompLoop)) {
+      return true;
+    }
+    // If we're looking at a non-simd OpenMP loop, we need to explicitly
+    // call OpenMPSimdOnly on the nested loop block while indicating where
+    // the block comes from.
+    auto &nest = std::get<std::optional<parser::NestedConstruct>>(ompLoop.t);
+    if (!nest.has_value()) {
+      return true;
+    }
+    if (auto *doConstruct = std::get_if<parser::DoConstruct>(&*nest)) {
+      auto &innerBlock = std::get<parser::Block>(doConstruct->t);
+      OpenMPSimdOnly(innerBlock, /*isNonSimdLoopBody=*/true);
+    }
+  }
+  return true;
+}
+
+void RewriteMutator::Post(parser::OpenMPLoopConstruct &ompLoop) {
+  this->Pre(ompLoop);
+}
+
+bool RewriteMutator::Pre(parser::DoConstruct &doConstruct) {
+  if (context_.langOptions().OpenMPSimd) {
+    auto &innerBlock = std::get<parser::Block>(doConstruct.t);
+    OpenMPSimdOnly(innerBlock);
+  }
+  return true;
+}
+
+void RewriteMutator::Post(parser::DoConstruct &doConstruct) {
+  this->Pre(doConstruct);
+}
+
+bool RewriteMutator::Pre(parser::IfConstruct &ifConstruct) {
+  if (context_.langOptions().OpenMPSimd) {
+    auto &innerBlock = std::get<parser::Block>(ifConstruct.t);
+    OpenMPSimdOnly(innerBlock);
+  }
+  return true;
+}
+
+void RewriteMutator::Post(parser::IfConstruct &ifConstruct) {
+  this->Pre(ifConstruct);
+}
+
 // Rewrite PRINT NML -> WRITE(*,NML=NML)
 bool RewriteMutator::Pre(parser::ActionStmt &x) {
   if (auto *print{std::get_if<common::Indirection<parser::PrintStmt>>(&x.u)};
diff --git a/flang/lib/Semantics/unparse-with-symbols.cpp b/flang/lib/Semantics/unparse-with-symbols.cpp
index 3093e39ba2411..b199481131065 100644
--- a/flang/lib/Semantics/unparse-with-symbols.cpp
+++ b/flang/lib/Semantics/unparse-with-symbols.cpp
@@ -47,6 +47,11 @@ class SymbolDumpVisitor {
     return true;
   }
   void Post(const parser::OmpClause &) { currStmt_ = std::nullopt; }
+  bool Pre(const parser::OpenMPGroupprivate &dir) {
+    currStmt_ = dir.source;
+    return true;
+  }
+  void Post(const parser::OpenMPGroupprivate &) { currStmt_ = std::nullopt; }
   bool Pre(const parser::OpenMPThreadprivate &dir) {
     currStmt_ = dir.source;
     return true;
@@ -70,20 +75,6 @@ class SymbolDumpVisitor {
     currStmt_ = std::nullopt;
   }
 
-  bool Pre(const parser::OmpCriticalDirective &x) {
-    currStmt_ = x.source;
-    return true;
-  }
-  void Post(const parser::OmpCriticalDirective &) { currStmt_ = std::nullopt; }
-
-  bool Pre(const parser::OmpEndCriticalDirective &x) {
-    currStmt_ = x.source;
-    return true;
-  }
-  void Post(const parser::OmpEndCriticalDirective &) {
-    currStmt_ = std::nullopt;
-  }
-
   // Directive arguments can be objects with symbols.
   bool Pre(const parser::OmpBeginDirective &x) {
     currStmt_ = x.source;
diff --git a/flang/lib/Support/Fortran-features.cpp b/flang/lib/Support/Fortran-features.cpp
index df51b3c577125..4a6fb8d75a135 100644
--- a/flang/lib/Support/Fortran-features.cpp
+++ b/flang/lib/Support/Fortran-features.cpp
@@ -90,6 +90,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
   disable_.set(LanguageFeature::OldStyleParameter);
   // Possibly an accidental "feature" of nvfortran.
   disable_.set(LanguageFeature::AssumedRankPassedToNonAssumedRank);
+  disable_.set(LanguageFeature::Coarray);
   // These warnings are enabled by default, but only because they used
   // to be unconditional.  TODO: prune this list
   warnLanguage_.set(LanguageFeature::ExponentMatchingKindParam);
@@ -147,6 +148,7 @@ LanguageFeatureControl::LanguageFeatureControl() {
   warnUsage_.set(UsageWarning::UseAssociationIntoSameNameSubprogram);
   warnUsage_.set(UsageWarning::HostAssociatedIntentOutInSpecExpr);
   warnUsage_.set(UsageWarning::NonVolatilePointerToVolatile);
+  warnUsage_.set(UsageWarning::RealConstantWidening);
   // New warnings, on by default
   warnLanguage_.set(LanguageFeature::SavedLocalInSpecExpr);
   warnLanguage_.set(LanguageFeature::NullActualForAllocatable);
diff --git a/flang/lib/Support/Fortran.cpp b/flang/lib/Support/Fortran.cpp
index 8e286be1624df..3a8ebbb7d61ef 100644
--- a/flang/lib/Support/Fortran.cpp
+++ b/flang/lib/Support/Fortran.cpp
@@ -103,8 +103,8 @@ std::string AsFortran(IgnoreTKRSet tkr) {
 /// dummy argument attribute while `y` represents the actual argument attribute.
 bool AreCompatibleCUDADataAttrs(std::optional<CUDADataAttr> x,
     std::optional<CUDADataAttr> y, IgnoreTKRSet ignoreTKR,
-    std::optional<std::string> *warning, bool allowUnifiedMatchingRule,
-    bool isHostDeviceProcedure, const LanguageFeatureControl *features) {
+    bool allowUnifiedMatchingRule, bool isHostDeviceProcedure,
+    const LanguageFeatureControl *features) {
   bool isCudaManaged{features
           ? features->IsEnabled(common::LanguageFeature::CudaManaged)
           : false};
@@ -145,9 +145,6 @@ bool AreCompatibleCUDADataAttrs(std::optional<CUDADataAttr> x,
                     *y == CUDADataAttr::Shared ||
                     *y == CUDADataAttr::Constant)) ||
             (!y && (isCudaUnified || isCudaManaged))) {
-          if (y && *y == CUDADataAttr::Shared && warning) {
-            *warning = "SHARED attribute ignored"s;
-          }
           return true;
         }
       } else if (*x == CUDADataAttr::Managed) {
diff --git a/flang/module/cudadevice.f90 b/flang/module/cudadevice.f90
index d0c312c09353f..1598c64db2cb5 100644
--- a/flang/module/cudadevice.f90
+++ b/flang/module/cudadevice.f90
@@ -324,6 +324,27 @@ attributes(device) real(8) function rsqrt(x) bind(c,name='__nv_rsqrt')
       real(8), value :: x
     end function
   end interface
+
+  interface saturate
+    attributes(device) real function __saturatef(r) bind(c, name='__nv_saturatef')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __sad
+    attributes(device) integer function __sad(i,j,k) bind(c, name='__nv_sad')
+      !dir$ ignore_tkr (d) i, (d) j, (d) k
+      integer, value :: i,j,k
+    end function
+  end interface
+
+  interface __usad
+    attributes(device) integer function __usad(i,j,k) bind(c, name='__nv_usad')
+      !dir$ ignore_tkr (d) i, (d) j, (d) k
+      integer, value :: i,j,k
+    end function
+  end interface
   
   interface signbit
     attributes(device) integer(4) function signbitf(x) bind(c,name='__nv_signbitf')
@@ -372,6 +393,83 @@ attributes(device) subroutine sincospi(x, y, z) bind(c,name='__nv_sincospi')
     procedure :: sincospi
   end interface
 
+  interface
+    attributes(device) real(4) function __cosf(x) bind(c, name='__nv_fast_cosf')
+      real(4), value :: x
+    end function
+  end interface
+
+  interface __exp10f
+    attributes(device) real function __exp10f(r) bind(c, name='__nv_fast_exp10f')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __expf
+    attributes(device) real function __expf(r) bind(c, name='__nv_fast_expf')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+  
+  interface __fdividef
+    attributes(device) real function __fdividef(r,d) bind(c, name='__nv_fast_fdividef')
+      !dir$ ignore_tkr (d) r, (d) d
+      real, value :: r,d
+    end function
+  end interface
+
+  interface __log10f
+    attributes(device) real function __log10f(r) bind(c, name='__nv_fast_log10f')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __log2f
+    attributes(device) real function __log2f(r) bind(c, name='__nv_fast_log2f')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __logf
+    attributes(device) real function __logf(r) bind(c, name='__nv_fast_logf')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface
+    attributes(device) real(4) function __powf(x,y) bind(c, name='__nv_fast_powf')
+      !dir$ ignore_tkr (d) x, y
+      real(4), value :: x, y
+    end function
+  end interface
+
+  interface __sincosf
+    attributes(device) subroutine __sincosf(r, s, c) bind(c, name='__nv_fast_sincosf')
+      !dir$ ignore_tkr (d) r, (d) s, (d) c
+      real, value :: r
+      real :: s, c
+    end subroutine
+  end interface
+
+  interface __sinf
+    attributes(device) real function __sinf(r) bind(c, name='__nv_fast_sinf')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __tanf
+    attributes(device) real function __tanf(r) bind(c, name='__nv_fast_tanf')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
   interface
     attributes(device) real(4) function cospif(x) bind(c,name='__nv_cospif')
       real(4), value :: x
@@ -430,346 +528,612 @@ attributes(device) real(8) function sinpi(x) bind(c,name='__nv_sinpi')
    end function
   end interface
 
+  interface int_as_float
+    attributes(device) real function __int_as_float(i) bind(c, name='__nv_int_as_float')
+      !dir$ ignore_tkr (d) i
+      integer, value :: i
+    end function
+  end interface
+
+  interface float_as_int
+    attributes(device) integer function __float_as_int(i) bind(c, name='__nv_float_as_int')
+      !dir$ ignore_tkr (d) i
+      real, value :: i
+    end function
+  end interface
+
   interface __float2half_rn
-    attributes(device) real(2) function __float2half_rn(r) bind(c)
+    attributes(device) real(2) function __float2half_rn(r) bind(c, name='__nv_float2half_rn')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2int_rd
+    attributes(device) integer function __float2int_rd(r) bind(c, name='__nv_float2int_rd')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2int_rn
+    attributes(device) integer function __float2int_rn(r) bind(c, name='__nv_float2int_rn')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2int_ru
+    attributes(device) integer function __float2int_ru(r) bind(c, name='__nv_float2int_ru')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2int_rz
+    attributes(device) integer function __float2int_rz(r) bind(c, name='__nv_float2int_rz')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2uint_rd
+    attributes(device) integer function __float2uint_rd(r) bind(c, name='__nv_float2uint_rd')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2uint_rn
+    attributes(device) integer function __float2uint_rn(r) bind(c, name='__nv_float2uint_rn')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2uint_ru
+    attributes(device) integer function __float2uint_ru(r) bind(c, name='__nv_float2uint_ru')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2uint_rz
+    attributes(device) integer function __float2uint_rz(r) bind(c, name='__nv_float2uint_rz')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2ll_rd
+    attributes(device) integer(8) function __float2ll_rd(r) bind(c, name='__nv_float2ll_rd')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2ll_rn
+    attributes(device) integer(8) function __float2ll_rn(r) bind(c, name='__nv_float2ll_rn')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2ll_ru
+    attributes(device) integer(8) function __float2ll_ru(r) bind(c, name='__nv_float2ll_ru')
+      !dir$ ignore_tkr (d) r
+      real, value :: r
+    end function
+  end interface
+
+  interface __float2ll_rz
+    attributes(device) integer(8) function __float2ll_rz(r) bind(c, name='__nv_float2ll_rz')
       !dir$ ignore_tkr (d) r
       real, value :: r
     end function
   end interface
 
   interface __half2float
-    attributes(device) real function __half2float(i) bind(c)
+    attributes(device) real function __half2float(i) bind(c, name='__nv_half2float')
       !dir$ ignore_tkr (d) i
       real(2), value :: i
     end function
   end interface
 
-  interface __double2int_rn
-    attributes(device) integer function __double2int_rn(r) bind(c)
+  interface double_as_longlong
+    attributes(device) integer(8) function __double_as_longlong(i) bind(c, name='__nv_double_as_longlong')
+      !dir$ ignore_tkr (d) i
+      real(8), value :: i
+    end function
+  end interface
+
+  interface longlong_as_double
+    attributes(device) real(8) function __longlong_as_double(i) bind(c, name='__nv_longlong_as_double')
+      !dir$ ignore_tkr (d) i
+      integer(8), value :: i
+    end function
+  end interface
+
+  interface __double2int_rd
+    attributes(device) integer function __double2int_rd(r) bind(c, name='__nv_double2int_rd')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2int_rz
-    attributes(device) integer function __double2int_rz(r) bind(c)
+  interface __double2int_rn
+    attributes(device) integer function __double2int_rn(r) bind(c, name='__nv_double2int_rn')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2int_ru
-    attributes(device) integer function __double2int_ru(r) bind(c)
+    attributes(device) integer function __double2int_ru(r) bind(c, name='__nv_double2int_ru')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2int_rd
-    attributes(device) integer function __double2int_rd(r) bind(c)
+  interface __double2int_rz
+    attributes(device) integer function __double2int_rz(r) bind(c, name='__nv_double2int_rz')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2uint_rn
-    attributes(device) integer function __double2uint_rn(r) bind(c)
+  interface __double2uint_rd
+    attributes(device) integer function __double2uint_rd(r) bind(c, name='__nv_double2uint_rd')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2uint_rz
-    attributes(device) integer function __double2uint_rz(r) bind(c)
+  interface __double2uint_rn
+    attributes(device) integer function __double2uint_rn(r) bind(c, name='__nv_double2uint_rn')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2uint_ru
-    attributes(device) integer function __double2uint_ru(r) bind(c)
+    attributes(device) integer function __double2uint_ru(r) bind(c, name='__nv_double2uint_ru')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2uint_rd
-    attributes(device) integer function __double2uint_rd(r) bind(c)
+  interface __double2uint_rz
+    attributes(device) integer function __double2uint_rz(r) bind(c, name='__nv_double2uint_rz')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2float_rn
-    attributes(device) real function __double2float_rn(r) bind(c)
+    attributes(device) real function __double2float_rn(r) bind(c, name='__nv_double2float_rn')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2float_rz
-    attributes(device) real function __double2float_rz(r) bind(c)
+    attributes(device) real function __double2float_rz(r) bind(c, name='__nv_double2float_rz')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2float_ru
-    attributes(device) real function __double2float_ru(r) bind(c)
+    attributes(device) real function __double2float_ru(r) bind(c, name='__nv_double2float_ru')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2float_rd
-    attributes(device) real function __double2float_rd(r) bind(c)
+    attributes(device) real function __double2float_rd(r) bind(c, name='__nv_double2float_rd')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2loint
-    attributes(device) integer function __double2loint(r) bind(c)
+    attributes(device) integer function __double2loint(r) bind(c, name='__nv_double2loint')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2hiint
-    attributes(device) integer function __double2hiint(r) bind(c)
+    attributes(device) integer function __double2hiint(r) bind(c, name='__nv_double2hiint')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __hiloint2double
-    attributes(device) double precision function __hiloint2double(i, j) bind(c)
+    attributes(device) double precision function __hiloint2double(i, j) bind(c, name='__nv_hiloint2double')
       !dir$ ignore_tkr (d) i, (d) j
       integer, value :: i, j
     end function
   end interface
 
+  interface __int2float_rd
+    attributes(device) real function __int2float_rd(i) bind(c, name='__nv_int2float_rd')
+      !dir$ ignore_tkr (d) i
+      integer, value :: i
+    end function
+  end interface
+
+  interface __int2float_rn
+    attributes(device) real function __int2float_rn(i) bind(c, name='__nv_int2float_rn')
+      !dir$ ignore_tkr (d) i
+      integer, value :: i
+    end function
+  end interface
+
+  interface __int2float_ru
+    attributes(device) real function __int2float_ru(i) bind(c, name='__nv_int2float_ru')
+      !dir$ ignore_tkr (d) i
+      integer, value :: i
+    end function
+  end interface
+
+  interface __int2float_rz
+    attributes(device) real function __int2float_rz(i) bind(c, name='__nv_int2float_rz')
+      !dir$ ignore_tkr (d) i
+      integer, value :: i
+    end function
+  end interface
+
   interface __int2double_rn
-    attributes(device) double precision function __int2double_rn(i) bind(c)
+    attributes(device) double precision function __int2double_rn(i) bind(c, name='__nv_int2double_rn')
+      !dir$ ignore_tkr (d) i
+      integer, value :: i
+    end function
+  end interface
+
+  interface __uint2float_rd
+    attributes(device) real function __uint2float_rd(i) bind(c, name='__nv_uint2float_rd')
+      !dir$ ignore_tkr (d) i
+      integer, value :: i
+    end function
+  end interface
+
+  interface __uint2float_rn
+    attributes(device) real function __uint2float_rn(i) bind(c, name='__nv_uint2float_rn')
+      !dir$ ignore_tkr (d) i
+      integer, value :: i
+    end function
+  end interface
+
+  interface __uint2float_ru
+    attributes(device) real function __uint2float_ru(i) bind(c, name='__nv_uint2float_ru')
+      !dir$ ignore_tkr (d) i
+      integer, value :: i
+    end function
+  end interface
+
+  interface __uint2float_rz
+    attributes(device) real function __uint2float_rz(i) bind(c, name='__nv_uint2float_rz')
       !dir$ ignore_tkr (d) i
       integer, value :: i
     end function
   end interface
 
   interface __uint2double_rn
-    attributes(device) double precision function __uint2double_rn(i) bind(c)
+    attributes(device) double precision function __uint2double_rn(i) bind(c, name='__nv_uint2double_rn')
       !dir$ ignore_tkr (d) i
       integer, value :: i
     end function
   end interface
 
-  interface __double2ll_rn
-    attributes(device) integer(8) function __double2ll_rn(r) bind(c)
+  interface __double2ll_rd
+    attributes(device) integer(8) function __double2ll_rd(r) bind(c, name='__nv_double2ll_rd')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2ll_rz
-    attributes(device) integer(8) function __double2ll_rz(r) bind(c)
+  interface __double2ll_rn
+    attributes(device) integer(8) function __double2ll_rn(r) bind(c, name='__nv_double2ll_rn')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2ll_ru
-    attributes(device) integer(8) function __double2ll_ru(r) bind(c)
+    attributes(device) integer(8) function __double2ll_ru(r) bind(c, name='__nv_double2ll_ru')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2ll_rd
-    attributes(device) integer(8) function __double2ll_rd(r) bind(c)
+  interface __double2ll_rz
+    attributes(device) integer(8) function __double2ll_rz(r) bind(c, name='__nv_double2ll_rz')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2ull_rn
-    attributes(device) integer(8) function __double2ull_rn(r) bind(c)
+  interface __double2ull_rd
+    attributes(device) integer(8) function __double2ull_rd(r) bind(c, name='__nv_double2ull_rd')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2ull_rz
-    attributes(device) integer(8) function __double2ull_rz(r) bind(c)
+  interface __double2ull_rn
+    attributes(device) integer(8) function __double2ull_rn(r) bind(c, name='__nv_double2ull_rn')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
   interface __double2ull_ru
-    attributes(device) integer(8) function __double2ull_ru(r) bind(c)
+    attributes(device) integer(8) function __double2ull_ru(r) bind(c, name='__nv_double2ull_ru')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __double2ull_rd
-    attributes(device) integer(8) function __double2ull_rd(r) bind(c)
+  interface __double2ull_rz
+    attributes(device) integer(8) function __double2ull_rz(r) bind(c, name='__nv_double2ull_rz')
       !dir$ ignore_tkr (d) r
       double precision, value :: r
     end function
   end interface
 
-  interface __ll2double_rn
-    attributes(device) double precision function __ll2double_rn(i) bind(c)
+  interface __ll2float_rd
+    attributes(device) real function __ll2float_rd(i) bind(c, name='__nv_ll2float_rd')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
-  interface __ll2double_rz
-    attributes(device) double precision function __ll2double_rz(i) bind(c)
+  interface __ll2float_rn
+    attributes(device) real function __ll2float_rn(i) bind(c, name='__nv_ll2float_rn')
+      !dir$ ignore_tkr (d) i
+      integer(8), value :: i
+    end function
+    end interface
+
+  interface __ll2float_ru
+    attributes(device) real function __ll2float_ru(i) bind(c, name='__nv_ll2float_ru')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
-  interface __ll2double_ru
-    attributes(device) double precision function __ll2double_ru(i) bind(c)
+  interface __ll2float_rz
+    attributes(device) real function __ll2float_rz(i) bind(c, name='__nv_ll2float_rz')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
   interface __ll2double_rd
-    attributes(device) double precision function __ll2double_rd(i) bind(c)
+    attributes(device) double precision function __ll2double_rd(i) bind(c, name='__nv_ll2double_rd')
+      !dir$ ignore_tkr (d) i
+      integer(8), value :: i
+    end function
+  end interface
+
+  interface __ll2double_rn
+    attributes(device) double precision function __ll2double_rn(i) bind(c, name='__nv_ll2double_rn')
+      !dir$ ignore_tkr (d) i
+      integer(8), value :: i
+    end function
+  end interface
+
+  interface __ll2double_ru
+    attributes(device) double precision function __ll2double_ru(i) bind(c, name='__nv_ll2double_ru')
+      !dir$ ignore_tkr (d) i
+      integer(8), value :: i
+    end function
+  end interface
+
+  interface __ll2double_rz
+    attributes(device) double precision function __ll2double_rz(i) bind(c, name='__nv_ll2double_rz')
+      !dir$ ignore_tkr (d) i
+      integer(8), value :: i
+    end function
+  end interface
+
+  interface __ull2double_rd
+    attributes(device) double precision function __ull2double_rd(i) bind(c, name='__nv_ull2double_rd')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
   interface __ull2double_rn
-    attributes(device) double precision function __ull2double_rn(i) bind(c)
+    attributes(device) double precision function __ull2double_rn(i) bind(c, name='__nv_ull2double_rn')
+      !dir$ ignore_tkr (d) i
+      integer(8), value :: i
+    end function
+  end interface
+
+  interface __ull2double_ru
+    attributes(device) double precision function __ull2double_ru(i) bind(c, name='__nv_ull2double_ru')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
   interface __ull2double_rz
-    attributes(device) double precision function __ull2double_rz(i) bind(c)
+    attributes(device) double precision function __ull2double_rz(i) bind(c, name='__nv_ull2double_rz')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
-  interface __ull2double_ru
-    attributes(device) double precision function __ull2double_ru(i) bind(c)
+  interface __ull2float_rd
+    attributes(device) real function __ull2float_rd(i) bind(c, name='__nv_ull2float_rd')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
-  interface __ull2double_rd
-    attributes(device) double precision function __ull2double_rd(i) bind(c)
+  interface __ull2float_rn
+    attributes(device) real function __ull2float_rn(i) bind(c, name='__nv_ull2float_rn')
+      !dir$ ignore_tkr (d) i
+      integer(8), value :: i
+    end function
+  end interface
+
+  interface __ull2float_ru
+    attributes(device) real function __ull2float_ru(i) bind(c, name='__nv_ull2float_ru')
+      !dir$ ignore_tkr (d) i
+      integer(8), value :: i
+    end function
+  end interface
+
+  interface __ull2float_rz
+    attributes(device) real function __ull2float_rz(i) bind(c, name='__nv_ull2float_rz')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
   interface __mul24
-    attributes(device) integer function __mul24(i,j) bind(c)
+    attributes(device) integer function __mul24(i,j) bind(c, name='__nv_mul24')
       !dir$ ignore_tkr (d) i, (d) j
       integer, value :: i,j
     end function
   end interface
 
   interface __umul24
-    attributes(device) integer function __umul24(i,j) bind(c)
+    attributes(device) integer function __umul24(i,j) bind(c, name='__nv_umul24')
       !dir$ ignore_tkr (d) i, (d) j
       integer, value :: i,j
     end function
   end interface
 
-  interface __dsqrt_ru
-    attributes(device) double precision function __dsqrt_ru(x) bind(c)
+  interface __drcp_rd
+    attributes(device) double precision function __drcp_rd(x) bind(c, name='__nv_drcp_rd')
+      !dir$ ignore_tkr (d) x
+      double precision, value :: x
+    end function
+  end interface
+
+  interface __drcp_rn
+    attributes(device) double precision function __drcp_rn(x) bind(c, name='__nv_drcp_rn')
+      !dir$ ignore_tkr (d) x
+      double precision, value :: x
+    end function
+  end interface
+
+  interface __drcp_ru
+    attributes(device) double precision function __drcp_ru(x) bind(c, name='__nv_drcp_ru')
+      !dir$ ignore_tkr (d) x
+      double precision, value :: x
+    end function
+  end interface
+
+  interface __drcp_rz
+    attributes(device) double precision function __drcp_rz(x) bind(c, name='__nv_drcp_rz')
       !dir$ ignore_tkr (d) x
       double precision, value :: x
     end function
   end interface
 
   interface __dsqrt_rd
-    attributes(device) double precision function __dsqrt_rd(x) bind(c)
+    attributes(device) double precision function __dsqrt_rd(x) bind(c, name='__nv_dsqrt_rd')
+      !dir$ ignore_tkr (d) x
+      double precision, value :: x
+    end function
+  end interface
+
+  interface __dsqrt_rn
+    attributes(device) double precision function __dsqrt_rn(x) bind(c, name='__nv_dsqrt_rn')
+      !dir$ ignore_tkr (d) x
+      double precision, value :: x
+    end function
+  end interface
+
+  interface __dsqrt_ru
+    attributes(device) double precision function __dsqrt_ru(x) bind(c, name='__nv_dsqrt_ru')
+      !dir$ ignore_tkr (d) x
+      double precision, value :: x
+    end function
+  end interface
+
+  interface __dsqrt_rz
+    attributes(device) double precision function __dsqrt_rz(x) bind(c, name='__nv_dsqrt_rz')
       !dir$ ignore_tkr (d) x
       double precision, value :: x
     end function
   end interface
 
   interface __ddiv_rn
-    attributes(device) double precision function __ddiv_rn(x,y) bind(c)
+    attributes(device) double precision function __ddiv_rn(x,y) bind(c, name='__nv_ddiv_rn')
       !dir$ ignore_tkr (d) x, (d) y
       double precision, value :: x, y
     end function
   end interface
 
   interface __ddiv_rz
-    attributes(device) double precision function __ddiv_rz(x,y) bind(c)
+    attributes(device) double precision function __ddiv_rz(x,y) bind(c, name='__nv_ddiv_rz')
       !dir$ ignore_tkr (d) x, (d) y
       double precision, value :: x, y
     end function
   end interface
 
   interface __ddiv_ru
-    attributes(device) double precision function __ddiv_ru(x,y) bind(c)
+    attributes(device) double precision function __ddiv_ru(x,y) bind(c, name='__nv_ddiv_ru')
       !dir$ ignore_tkr (d) x, (d) y
       double precision, value :: x, y
     end function
   end interface
 
   interface __ddiv_rd
-    attributes(device) double precision function __ddiv_rd(x,y) bind(c)
+    attributes(device) double precision function __ddiv_rd(x,y) bind(c, name='__nv_ddiv_rd')
       !dir$ ignore_tkr (d) x, (d) y
       double precision, value :: x, y
     end function
   end interface
 
   interface __clz
-    attributes(device) integer function __clz(i) bind(c)
+    attributes(device) integer function __clz(i) bind(c, name='__nv_clz')
       !dir$ ignore_tkr (d) i
       integer, value :: i
     end function
-    attributes(device) integer function __clzll(i) bind(c)
+    attributes(device) integer function __clzll(i) bind(c, name='__nv_clzll')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
   interface __ffs
-    attributes(device) integer function __ffs(i) bind(c)
+    attributes(device) integer function __ffs(i) bind(c, name='__nv_ffs')
       !dir$ ignore_tkr (d) i
       integer, value :: i
     end function
-    attributes(device) integer function __ffsll(i) bind(c)
+    attributes(device) integer function __ffsll(i) bind(c, name='__nv_ffsll')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
   interface __popc
-    attributes(device) integer function __popc(i) bind(c)
+    attributes(device) integer function __popc(i) bind(c, name='__nv_popc')
       !dir$ ignore_tkr (d) i
       integer, value :: i
     end function
-    attributes(device) integer function __popcll(i) bind(c)
+    attributes(device) integer function __popcll(i) bind(c, name='__nv_popcll')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
   end interface
 
   interface __brev
-    attributes(device) integer function __brev(i) bind(c)
+    attributes(device) integer function __brev(i) bind(c, name='__nv_brev')
       !dir$ ignore_tkr (d) i
       integer, value :: i
     end function
-    attributes(device) integer(8) function __brevll(i) bind(c)
+    attributes(device) integer(8) function __brevll(i) bind(c, name ='__nv_brevll')
       !dir$ ignore_tkr (d) i
       integer(8), value :: i
     end function
diff --git a/flang/test/Driver/atomic-control-options.f90 b/flang/test/Driver/atomic-control-options.f90
new file mode 100644
index 0000000000000..04ced31195c0e
--- /dev/null
+++ b/flang/test/Driver/atomic-control-options.f90
@@ -0,0 +1,24 @@
+! REQUIRES: amdgpu-registered-target
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -munsafe-fp-atomics %s -o -|FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang --offload-arch=gfx90a --offload-device-only -fopenmp -emit-llvm -S %s -munsafe-fp-atomics -nogpulib -o -|FileCheck -check-prefix=UNSAFE-FP-ATOMICS %s
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-ignore-denormal-mode %s -o -|FileCheck -check-prefix=IGNORE-DENORMAL-MODE %s
+! RUN: %flang --offload-arch=gfx90a --offload-device-only -fopenmp -emit-llvm -S %s -fatomic-ignore-denormal-mode -nogpulib -o -|FileCheck -check-prefix=IGNORE-DENORMAL-MODE %s
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-fine-grained-memory %s -o -|FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang --offload-arch=gfx90a --offload-device-only -fopenmp -emit-llvm -S %s -fatomic-fine-grained-memory -nogpulib -o -|FileCheck -check-prefix=FINE-GRAINED-MEMORY %s
+! RUN: %flang_fc1 -emit-llvm -triple amdgcn-amd-amdhsa -fopenmp -fopenmp-is-device -fatomic-remote-memory %s -o -|FileCheck -check-prefix=REMOTE-MEMORY %s
+! RUN: %flang --offload-arch=gfx90a --offload-device-only -fopenmp -emit-llvm -S %s -fatomic-remote-memory -nogpulib -o -|FileCheck -check-prefix=REMOTE-MEMORY %s
+program test
+    implicit none
+    integer :: A, threads
+    threads = 128
+    A = 0
+    !$omp target parallel num_threads(threads)
+    !$omp atomic
+    A =  A + 1
+    !$omp end target parallel
+end program test
+
+!UNSAFE-FP-ATOMICS: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.ignore.denormal.mode !{{.*}}, !amdgpu.no.fine.grained.memory !{{.*}}, !amdgpu.no.remote.memory !{{.*}}
+!IGNORE-DENORMAL-MODE: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.ignore.denormal.mode !{{.*}}, !amdgpu.no.fine.grained.memory !{{.*}}, !amdgpu.no.remote.memory !{{.*}}
+!FINE-GRAINED-MEMORY: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.no.remote.memory !{{.*}}
+!REMOTE-MEMORY: %{{.*}} = atomicrmw add ptr {{.*}}, i32 1 monotonic, align 4, !amdgpu.no.fine.grained.memory !{{.*}}
diff --git a/flang/test/Driver/fopenmp-simd.f90 b/flang/test/Driver/fopenmp-simd.f90
new file mode 100644
index 0000000000000..b25adee2779ee
--- /dev/null
+++ b/flang/test/Driver/fopenmp-simd.f90
@@ -0,0 +1,59 @@
+! RUN: %flang -target x86_64-linux-gnu -fopenmp-simd %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-OPENMP-SIMD-FLAG --check-prefix=CHECK-NO-LD-ANY
+! RUN: %flang -target x86_64-darwin -fopenmp-simd %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-OPENMP-SIMD-FLAG --check-prefix=CHECK-NO-LD-ANY
+! RUN: %flang -target x86_64-freebsd -fopenmp-simd %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-OPENMP-SIMD-FLAG --check-prefix=CHECK-NO-LD-ANY
+! RUN: %flang -target x86_64-windows-gnu -fopenmp-simd %s -o %t -### 2>&1 | FileCheck %s --check-prefix=CHECK-OPENMP-SIMD-FLAG --check-prefix=CHECK-NO-LD-ANY
+
+! CHECK-OPENMP-SIMD-FLAG: "-fopenmp-simd"
+! CHECK-NO-LD-ANY-NOT: "-l{{(omp|gomp|iomp5)}}"
+
+! -fopenmp-simd enables openmp support only for simd constructs
+! RUN: %flang_fc1 -fopenmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP-SIMD %s
+! RUN: %flang_fc1 -fno-openmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-NO-OMP-SIMD %s
+! RUN: %flang_fc1 -fopenmp-simd -fno-openmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-NO-OMP-SIMD %s
+! RUN: %flang_fc1 -fno-openmp-simd -fopenmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP-SIMD %s
+! -fopenmp-simd should have no effect if -fopenmp is already set
+! RUN: %flang_fc1 -fopenmp %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP %s
+! RUN: %flang_fc1 -fopenmp -fopenmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP %s
+! RUN: %flang_fc1 -fopenmp -fno-openmp-simd %s -emit-fir -o - | FileCheck --check-prefix=CHECK-OMP %s
+
+subroutine main
+  ! CHECK-OMP-SIMD-NOT: omp.parallel
+  ! CHECK-OMP-SIMD-NOT: omp.wsloop
+  ! CHECK-OMP-SIMD-NOT: omp.loop_nest
+  ! CHECK-OMP-SIMD: fir.do_loop
+  ! CHECK-NO-OMP-SIMD-NOT: omp.parallel
+  ! CHECK-NO-OMP-SIMD-NOT: omp.wsloop
+  ! CHECK-NO-OMP-SIMD-NOT: omp.loop_nest
+  ! CHECK-NO-OMP-SIMD: fir.do_loop
+  ! CHECK-OMP: omp.parallel
+  ! CHECK-OMP: omp.wsloop
+  ! CHECK-OMP: omp.loop_nest
+  ! CHECK-OMP-NOT: fir.do_loop
+  !$omp parallel do
+  do i = 1, 10
+    print *, "test"
+  end do
+  ! CHECK-NO-OMP-SIMD-NOT: omp.yield
+  ! CHECK-NO-OMP-SIMD-NOT: omp.terminator
+  ! CHECK-OMP-SIMD-NOT: omp.yield
+  ! CHECK-OMP-SIMD-NOT: omp.terminator
+  ! CHECK-OMP: omp.yield
+  ! CHECK-OMP: omp.terminator
+  !$omp end parallel do
+
+  ! CHECK-OMP-SIMD: omp.simd
+  ! CHECK-NO-OMP-SIMD-NOT: omp.simd
+  ! CHECK-OMP: omp.simd
+  !$omp simd
+  ! CHECK-OMP-SIMD: omp.loop_nest
+  ! CHECK-NO-OMP-SIMD-NOT: omp.loop_nest
+  ! CHECK-NO-OMP-SIMD: fir.do_loop
+  ! CHECK-OMP: omp.loop_nest
+  ! CHECK-OMP-NOT: fir.do_loop
+  do i = 1, 10
+    print *, "test"
+  ! CHECK-OMP-SIMD: omp.yield
+  ! CHECK-NO-OMP-SIMD-NOT: omp.yield
+  ! CHECK-OMP: omp.yield
+  end do
+end subroutine
diff --git a/flang/test/Driver/fopenmp-version.F90 b/flang/test/Driver/fopenmp-version.F90
index c2866561461b7..59406d3dd32c8 100644
--- a/flang/test/Driver/fopenmp-version.F90
+++ b/flang/test/Driver/fopenmp-version.F90
@@ -22,4 +22,8 @@
 
 !RUN: not %flang -c -fopenmp -fopenmp-version=29 %s 2>&1 | FileCheck --check-prefix=ERR-BAD %s
 
-!ERR-BAD: error: '29' is not a valid OpenMP version in '-fopenmp-version=29', valid versions are 31, 40, 45, 50, 51, 52, 60
+!ERR-BAD: error: '29' is not a valid OpenMP version in '-fopenmp-version=29', valid versions are 31, 40, 45, 50, 51, 52, 60, 61
+
+!RUN: %flang -c -fopenmp -fopenmp-version=61 %s 2>&1 | FileCheck --check-prefix=FUTURE %s
+
+!FUTURE: The specification for OpenMP version 61 is still under development; the syntax and semantics of new features may be subject to change
diff --git a/flang/test/Driver/linker-flags.f90 b/flang/test/Driver/linker-flags.f90
index ad48ea1b9e9b1..2b56fdfb8da05 100644
--- a/flang/test/Driver/linker-flags.f90
+++ b/flang/test/Driver/linker-flags.f90
@@ -77,7 +77,7 @@
 ! MINGW-SAME: -lflang_rt.runtime
 ! MINGW-STATIC-FLANGRT: "{{.*}}{{\\|/}}libflang_rt.runtime.a"
 
-! NOTE: This also matches lld-link (when CLANG_DEFAULT_LINKER=lld) and
+! NOTE: This also matches lld-link (when FLANG_DEFAULT_LINKER=lld) and
 !       any .exe suffix that is added when resolving to the full path of
 !       (lld-)link.exe on Windows platforms. The suffix may not be added
 !       when the executable is not found or on non-Windows platforms.
diff --git a/flang/test/Evaluate/bug153031.f90 b/flang/test/Evaluate/bug153031.f90
new file mode 100644
index 0000000000000..a717954ecaed1
--- /dev/null
+++ b/flang/test/Evaluate/bug153031.f90
@@ -0,0 +1,18 @@
+! RUN: %flang_fc1 -fdebug-unparse %s 2>&1 | FileCheck %s
+! Ensure that UBOUND() calculation from LBOUND()+SIZE() isn't applied to
+! variables containing references to impure functions.
+type t
+  real, allocatable :: a(:)
+end type
+interface
+  pure integer function pure(n)
+    integer, intent(in) :: n
+  end
+end interface
+type(t) :: x(10)
+allocate(x(1)%a(2))
+!CHECK: PRINT *, ubound(x(int(impure(1_4),kind=8))%a,dim=1_4)
+print *, ubound(x(impure(1))%a, dim=1)
+!CHECK: PRINT *, int(size(x(int(pure(1_4),kind=8))%a,dim=1,kind=8)+lbound(x(int(pure(1_4),kind=8))%a,dim=1,kind=8)-1_8,kind=4)
+print *, ubound(x(pure(1))%a, dim=1)
+end
diff --git a/flang/test/Evaluate/errors01.f90 b/flang/test/Evaluate/errors01.f90
index b20922237f240..90a0c300e3567 100644
--- a/flang/test/Evaluate/errors01.f90
+++ b/flang/test/Evaluate/errors01.f90
@@ -6,8 +6,8 @@ module m
     real x
   end type t
  contains
-  subroutine s1(a,b,c)
-    real :: a(*), b(:), c(..)
+  subroutine s1(a,b,c,d)
+    real :: a(*), b(:), c(..), d
     !CHECK: error: DIM=1 dimension is out of range for rank-1 assumed-size array
     integer :: ub1(ubound(a,1))
     !CHECK-NOT: error: DIM=1 dimension is out of range for rank-1 assumed-size array
@@ -23,7 +23,11 @@ subroutine s1(a,b,c)
     !CHECK: error: DIM=0 dimension must be positive
     integer :: lb4(lbound(c,0))
     !CHECK: error: DIM=666 dimension is too large for any array (maximum rank 15)
-    integer :: lb4(lbound(c,666))
+    integer :: lb5(lbound(c,666))
+    !CHECK: error: 'array=' argument has unacceptable rank 0
+    integer :: lb6(lbound(d,1))
+    !CHECK: error: 'array=' argument has unacceptable rank 0
+    integer :: ub4(ubound(d,1))
   end subroutine
   subroutine s2
     integer, parameter :: array(2,3) = reshape([(j, j=1, 6)], shape(array))
diff --git a/flang/test/Fir/CUDA/cuda-shared-offset.mlir b/flang/test/Fir/CUDA/cuda-shared-offset.mlir
index 8c377dbadd835..29316c90e5281 100644
--- a/flang/test/Fir/CUDA/cuda-shared-offset.mlir
+++ b/flang/test/Fir/CUDA/cuda-shared-offset.mlir
@@ -121,4 +121,40 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr, dense<
 // CHECK-LABEL: gpu.func @_QPnoshared()
 // CHECK-NOT: fir.global internal @_QPnoshared__shared_mem
 
+// -----
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<!llvm.ptr, dense<64> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<271>, dense<32> : vector<4xi64>>, #dlti.dl_entry<!llvm.ptr<270>, dense<32> : vector<4xi64>>, #dlti.dl_entry<f128, dense<128> : vector<2xi64>>, #dlti.dl_entry<f64, dense<64> : vector<2xi64>>, #dlti.dl_entry<f80, dense<128> : vector<2xi64>>, #dlti.dl_entry<f16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i32, dense<32> : vector<2xi64>>, #dlti.dl_entry<i16, dense<16> : vector<2xi64>>, #dlti.dl_entry<i128, dense<128> : vector<2xi64>>, #dlti.dl_entry<i8, dense<8> : vector<2xi64>>, #dlti.dl_entry<!llvm.ptr<272>, dense<64> : vector<4xi64>>, #dlti.dl_entry<i64, dense<64> : vector<2xi64>>, #dlti.dl_entry<i1, dense<8> : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (https://github.com/llvm/llvm-project.git cae351f3453a0a26ec8eb2ddaf773c24a29d929e)", llvm.target_triple = "x86_64-unknown-linux-gnu"} {
+  gpu.module @cuda_device_mod {
+    gpu.func @_QMmtestsPtestany(%arg0: !fir.ref<!fir.array<?xf32>> {cuf.data_attr = #cuf.cuda<device>, fir.bindc_name = "a"}) attributes {cuf.proc_attr = #cuf.cuda_proc<global>} {
+      %0 = fir.dummy_scope : !fir.dscope
+      %c-1 = arith.constant -1 : index
+      %1 = fir.shape %c-1 : (index) -> !fir.shape<1>
+      %2:2 = hlfir.declare %arg0(%1) dummy_scope %0 {data_attr = #cuf.cuda<device>, uniq_name = "_QMmtestsFtestanyEa"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+      %3 = fir.address_of(@_QM__fortran_builtinsE__builtin_blockdim) : !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>
+      %4:2 = hlfir.declare %3 {uniq_name = "_QM__fortran_builtinsE__builtin_blockdim"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>)
+      %5 = fir.address_of(@_QM__fortran_builtinsE__builtin_blockidx) : !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>
+      %6:2 = hlfir.declare %5 {uniq_name = "_QM__fortran_builtinsE__builtin_blockidx"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>)
+      %c-1_0 = arith.constant -1 : index
+      %7 = cuf.shared_memory !fir.array<?xf64>, %c-1_0 : index {bindc_name = "dmasks", uniq_name = "_QMmtestsFtestanyEdmasks"} -> !fir.ref<!fir.array<?xf64>>
+      %8 = fir.shape %c-1_0 : (index) -> !fir.shape<1>
+      %9:2 = hlfir.declare %7(%8) {data_attr = #cuf.cuda<shared>, uniq_name = "_QMmtestsFtestanyEdmasks"} : (!fir.ref<!fir.array<?xf64>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.array<?xf64>>)
+      %10 = fir.address_of(@_QM__fortran_builtinsE__builtin_griddim) : !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>
+      %11:2 = hlfir.declare %10 {uniq_name = "_QM__fortran_builtinsE__builtin_griddim"} : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>) -> (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>, !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_dim3{x:i32,y:i32,z:i32}>>)
+      %12 = fir.alloca i32 {bindc_name = "i", uniq_name = "_QMmtestsFtestanyEi"}
+      %13:2 = hlfir.declare %12 {uniq_name = "_QMmtestsFtestanyEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+      %14 = fir.alloca i32 {bindc_name = "iam", uniq_name = "_QMmtestsFtestanyEiam"}
+      %15:2 = hlfir.declare %14 {uniq_name = "_QMmtestsFtestanyEiam"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+      %16 = fir.alloca i32 {bindc_name = "j", uniq_name = "_QMmtestsFtestanyEj"}
+      %17:2 = hlfir.declare %16 {uniq_name = "_QMmtestsFtestanyEj"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+      %c-1_1 = arith.constant -1 : index
+      %18 = cuf.shared_memory !fir.array<?xf32>, %c-1_1 : index {bindc_name = "smasks", uniq_name = "_QMmtestsFtestanyEsmasks"} -> !fir.ref<!fir.array<?xf32>>
+      %19 = fir.shape %c-1_1 : (index) -> !fir.shape<1>
+      %20:2 = hlfir.declare %18(%19) {data_attr = #cuf.cuda<shared>, uniq_name = "_QMmtestsFtestanyEsmasks"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+      gpu.return
+    }
+  }
+}
 
+// CHECK-LABEL: gpu.func @_QMmtestsPtestany
+// CHECK: %{{.*}} = cuf.shared_memory[%c0{{.*}} : i32] !fir.array<?xf64>, %c-1{{.*}} : index {bindc_name = "dmasks", uniq_name = "_QMmtestsFtestanyEdmasks"} -> !fir.ref<!fir.array<?xf64>>
+// CHECK: %{{.*}} = cuf.shared_memory[%c0{{.*}} : i32] !fir.array<?xf32>, %c-1{{.*}} : index {bindc_name = "smasks", uniq_name = "_QMmtestsFtestanyEsmasks"} -> !fir.ref<!fir.array<?xf32>>
diff --git a/flang/test/Fir/FirToSCF/iter-while.fir b/flang/test/Fir/FirToSCF/iter-while.fir
new file mode 100644
index 0000000000000..0de7aabed120e
--- /dev/null
+++ b/flang/test/Fir/FirToSCF/iter-while.fir
@@ -0,0 +1,99 @@
+// RUN: fir-opt %s --fir-to-scf | FileCheck %s
+
+// CHECK-LABEL:   func.func @test_simple_iterate_while_1() -> (index, i1, i16, i32) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 11 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 22 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 123 : i16
+// CHECK:           %[[VAL_5:.*]] = arith.constant 456 : i32
+// CHECK:           %[[VAL_6:.*]]:4 = scf.while (%[[VAL_7:.*]] = %[[VAL_0]], %[[VAL_8:.*]] = %[[VAL_3]], %[[VAL_9:.*]] = %[[VAL_4]], %[[VAL_10:.*]] = %[[VAL_5]]) : (index, i1, i16, i32) -> (index, i1, i16, i32) {
+// CHECK:             %[[VAL_11:.*]] = arith.cmpi sle, %[[VAL_7]], %[[VAL_1]] : index
+// CHECK:             %[[VAL_12:.*]] = arith.andi %[[VAL_11]], %[[VAL_8]] : i1
+// CHECK:             scf.condition(%[[VAL_12]]) %[[VAL_7]], %[[VAL_8]], %[[VAL_9]], %[[VAL_10]] : index, i1, i16, i32
+// CHECK:           } do {
+// CHECK:           ^bb0(%[[VAL_13:.*]]: index, %[[VAL_14:.*]]: i1, %[[VAL_15:.*]]: i16, %[[VAL_16:.*]]: i32):
+// CHECK:             %[[VAL_17:.*]] = arith.addi %[[VAL_13]], %[[VAL_2]] : index
+// CHECK:             %[[VAL_18:.*]] = arith.constant true
+// CHECK:             %[[VAL_19:.*]] = arith.constant 22 : i16
+// CHECK:             %[[VAL_20:.*]] = arith.constant 33 : i32
+// CHECK:             scf.yield %[[VAL_17]], %[[VAL_18]], %[[VAL_19]], %[[VAL_20]] : index, i1, i16, i32
+// CHECK:           }
+// CHECK:           return %[[VAL_21:.*]]#0, %[[VAL_21]]#1, %[[VAL_21]]#2, %[[VAL_21]]#3 : index, i1, i16, i32
+// CHECK:         }
+func.func @test_simple_iterate_while_1() -> (index, i1, i16, i32) {
+  %lo = arith.constant 11 : index
+  %up = arith.constant 22 : index
+  %step = arith.constant 2 : index
+  %ok = arith.constant 1 : i1
+  %val1 = arith.constant 123 : i16
+  %val2 = arith.constant 456 : i32
+
+  %res:4 = fir.iterate_while (%i = %lo to %up step %step) and (%c = %ok) iter_args(%v1 = %val1, %v2 = %val2) -> (index, i1, i16, i32) {
+    %new_c = arith.constant 1 : i1
+    %new_v1 = arith.constant 22 : i16
+    %new_v2 = arith.constant 33 : i32
+    fir.result %i, %new_c, %new_v1, %new_v2 : index, i1, i16, i32
+  }
+
+  return %res#0, %res#1, %res#2, %res#3 : index, i1, i16, i32
+}
+
+// CHECK-LABEL:   func.func @test_simple_iterate_while_2(
+// CHECK-SAME:        %[[ARG0:.*]]: index, %[[ARG1:.*]]: index, %[[ARG2:.*]]: i1, %[[ARG3:.*]]: i32) -> (index, i1, i32) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_1:.*]]:3 = scf.while (%[[VAL_2:.*]] = %[[ARG0]], %[[VAL_3:.*]] = %[[ARG2]], %[[VAL_4:.*]] = %[[ARG3]]) : (index, i1, i32) -> (index, i1, i32) {
+// CHECK:             %[[VAL_5:.*]] = arith.cmpi sle, %[[VAL_2]], %[[ARG1]] : index
+// CHECK:             %[[VAL_6:.*]] = arith.andi %[[VAL_5]], %[[VAL_3]] : i1
+// CHECK:             scf.condition(%[[VAL_6]]) %[[VAL_2]], %[[VAL_3]], %[[VAL_4]] : index, i1, i32
+// CHECK:           } do {
+// CHECK:           ^bb0(%[[VAL_7:.*]]: index, %[[VAL_8:.*]]: i1, %[[VAL_9:.*]]: i32):
+// CHECK:             %[[VAL_10:.*]] = arith.addi %[[VAL_7]], %[[VAL_0]] : index
+// CHECK:             %[[VAL_11:.*]] = arith.constant 123 : i32
+// CHECK:             %[[VAL_12:.*]] = arith.constant true
+// CHECK:             scf.yield %[[VAL_10]], %[[VAL_12]], %[[VAL_11]] : index, i1, i32
+// CHECK:           }
+// CHECK:           return %[[VAL_13:.*]]#0, %[[VAL_13]]#1, %[[VAL_13]]#2 : index, i1, i32
+// CHECK:         }
+func.func @test_simple_iterate_while_2(%start: index, %stop: index, %cond: i1, %val: i32) -> (index, i1, i32) {
+  %step = arith.constant 1 : index
+
+  %res:3 = fir.iterate_while (%i = %start to %stop step %step) and (%ok = %cond) iter_args(%x = %val) -> (index, i1, i32) {
+    %new_x = arith.constant 123 : i32
+    %new_ok = arith.constant 1 : i1
+    fir.result %i, %new_ok, %new_x : index, i1, i32
+  }
+
+  return %res#0, %res#1, %res#2 : index, i1, i32
+}
+
+// CHECK-LABEL:   func.func @test_zero_iterations() -> (index, i1, i8) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 5 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 42 : i8
+// CHECK:           %[[VAL_5:.*]]:3 = scf.while (%[[VAL_6:.*]] = %[[VAL_0]], %[[VAL_7:.*]] = %[[VAL_3]], %[[VAL_8:.*]] = %[[VAL_4]]) : (index, i1, i8) -> (index, i1, i8) {
+// CHECK:             %[[VAL_9:.*]] = arith.cmpi sle, %[[VAL_6]], %[[VAL_1]] : index
+// CHECK:             %[[VAL_10:.*]] = arith.andi %[[VAL_9]], %[[VAL_7]] : i1
+// CHECK:             scf.condition(%[[VAL_10]]) %[[VAL_6]], %[[VAL_7]], %[[VAL_8]] : index, i1, i8
+// CHECK:           } do {
+// CHECK:           ^bb0(%[[VAL_11:.*]]: index, %[[VAL_12:.*]]: i1, %[[VAL_13:.*]]: i8):
+// CHECK:             %[[VAL_14:.*]] = arith.addi %[[VAL_11]], %[[VAL_2]] : index
+// CHECK:             scf.yield %[[VAL_14]], %[[VAL_12]], %[[VAL_13]] : index, i1, i8
+// CHECK:           }
+// CHECK:           return %[[VAL_15:.*]]#0, %[[VAL_15]]#1, %[[VAL_15]]#2 : index, i1, i8
+// CHECK:         }
+func.func @test_zero_iterations() -> (index, i1, i8) {
+  %lo = arith.constant 10 : index
+  %up = arith.constant 5 : index
+  %step = arith.constant 1 : index
+  %ok = arith.constant 1 : i1
+  %x = arith.constant 42 : i8
+
+  %res:3 = fir.iterate_while (%i = %lo to %up step %step) and (%c = %ok) iter_args(%xv = %x) -> (index, i1, i8) {
+    fir.result %i, %c, %xv : index, i1, i8
+  }
+
+  return %res#0, %res#1, %res#2 : index, i1, i8
+}
diff --git a/flang/test/Fir/convert-to-llvm.fir b/flang/test/Fir/convert-to-llvm.fir
index 50a98466f0d4b..cd87bf8d28ed5 100644
--- a/flang/test/Fir/convert-to-llvm.fir
+++ b/flang/test/Fir/convert-to-llvm.fir
@@ -338,8 +338,7 @@ func.func @select(%arg : index, %arg2 : i32) -> i32 {
 // CHECK:         %[[C0:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:         %[[C1:.*]] = llvm.mlir.constant(2 : i32) : i32
 // CHECK:         %[[C2:.*]] = llvm.mlir.constant(3 : i32) : i32
-// CHECK:         %[[SELECTOR:.*]] = llvm.trunc %[[SELECTVALUE]] : i{{.*}} to i32
-// CHECK:         llvm.switch %[[SELECTOR]] : i32, ^bb5 [
+// CHECK:         llvm.switch %[[SELECTVALUE]] : i64, ^bb5 [
 // CHECK:           1: ^bb1(%[[C0]] : i32),
 // CHECK:           2: ^bb2(%[[C2]], %[[SELECTVALUE]], %[[ARG1]] : i32, [[IDX]], i32),
 // CHECK:           3: ^bb3(%[[ARG1]], %[[C2]] : i32, i32),
@@ -384,7 +383,8 @@ func.func @select_rank(%arg : i32, %arg2 : i32) -> i32 {
 // CHECK:         %[[C0:.*]] = llvm.mlir.constant(1 : i32) : i32
 // CHECK:         %[[C1:.*]] = llvm.mlir.constant(2 : i32) : i32
 // CHECK:         %[[C2:.*]] = llvm.mlir.constant(3 : i32) : i32
-// CHECK:         llvm.switch %[[SELECTVALUE]] : i32, ^bb5 [
+// CHECK:         %[[SELECTOR:.*]] = llvm.sext %[[SELECTVALUE]] : i{{.*}} to i64
+// CHECK:         llvm.switch %[[SELECTOR]] : i64, ^bb5 [
 // CHECK:           1: ^bb1(%[[C0]] : i32),
 // CHECK:           2: ^bb2(%[[C2]], %[[SELECTVALUE]], %[[ARG1]] : i32, i32, i32),
 // CHECK:           3: ^bb3(%[[ARG1]], %[[C2]] : i32, i32),
@@ -2853,6 +2853,8 @@ func.func @test_call_arg_attrs_direct(%arg0: i32, %arg1: !fir.ref<i64>) {
   return
 }
 
+// -----
+
 // CHECK-LABEL: @test_call_arg_attrs_indirect
 func.func @test_call_arg_attrs_indirect(%arg0: i16, %arg1: (i16)-> i16) -> i16 {
   // CHECK: llvm.call %arg1(%{{.*}}) : !llvm.ptr, (i16 {llvm.noundef, llvm.signext}) -> (i16 {llvm.signext})
@@ -2860,6 +2862,8 @@ func.func @test_call_arg_attrs_indirect(%arg0: i16, %arg1: (i16)-> i16) -> i16 {
   return %0 : i16
 }
 
+// -----
+
 // CHECK-LABEL: @test_byval
 func.func @test_byval(%arg0: (!fir.ref<!fir.type<t{a:!fir.array<5xf64>}>>, f64) -> (), %arg1: !fir.ref<!fir.type<t{a:!fir.array<5xf64>}>>, %arg2: f64) {
   //  llvm.call %{{.*}}(%{{.*}}, %{{.*}}) : !llvm.ptr, (!llvm.ptr {llvm.byval = !llvm.struct<"t", (array<5 x f64>)>}, f64) -> ()
@@ -2867,9 +2871,56 @@ func.func @test_byval(%arg0: (!fir.ref<!fir.type<t{a:!fir.array<5xf64>}>>, f64)
   return
 }
 
+// -----
+
 // CHECK-LABEL: @test_sret
 func.func @test_sret(%arg0: (!fir.ref<!fir.type<t{a:!fir.array<5xf64>}>>, f64) -> (), %arg1: !fir.ref<!fir.type<t{a:!fir.array<5xf64>}>>, %arg2: f64) {
   //  llvm.call %{{.*}}(%{{.*}}, %{{.*}}) : !llvm.ptr, (!llvm.ptr {llvm.sret = !llvm.struct<"t", (array<5 x f64>)>}, f64) -> ()
   fir.call %arg0(%arg1, %arg2) : (!fir.ref<!fir.type<t{a:!fir.array<5xf64>}>> {llvm.sret = !fir.type<t{a:!fir.array<5xf64>}>}, f64) -> ()
   return
 }
+
+// -----
+
+func.func @select_with_cast(%arg1 : i8, %arg2 : i16, %arg3: i64, %arg4: index) -> () {
+  fir.select %arg1 : i8 [ 1, ^bb1, unit, ^bb1 ]
+  ^bb1:
+  fir.select %arg2 : i16 [ 1, ^bb2, unit, ^bb2 ]
+  ^bb2:
+  fir.select %arg3 : i64 [ 1, ^bb3, unit, ^bb3 ]
+  ^bb3:
+  fir.select %arg4 : index [ 1, ^bb4, unit, ^bb4 ]
+  ^bb4:
+  fir.select %arg3 : i64 [ 4294967296, ^bb5, unit, ^bb5 ]
+  ^bb5:
+  return
+}
+// CHECK-LABEL:   llvm.func @select_with_cast(
+// CHECK-SAME:      %[[ARG0:.*]]: i8,
+// CHECK-SAME:      %[[ARG1:.*]]: i16,
+// CHECK-SAME:      %[[ARG2:.*]]: i64,
+// CHECK-SAME:      %[[ARG3:.*]]: i64) {
+// CHECK:           %[[VAL_0:.*]] = llvm.sext %[[ARG0]] : i8 to i64
+// CHECK:           llvm.switch %[[VAL_0]] : i64, ^bb1 [
+// CHECK:             1: ^bb1
+// CHECK:           ]
+// CHECK:         ^bb1:
+// CHECK:           %[[VAL_1:.*]] = llvm.sext %[[ARG1]] : i16 to i64
+// CHECK:           llvm.switch %[[VAL_1]] : i64, ^bb2 [
+// CHECK:             1: ^bb2
+// CHECK:           ]
+// CHECK:         ^bb2:
+// CHECK:           llvm.switch %[[ARG2]] : i64, ^bb3 [
+// CHECK:             1: ^bb3
+// CHECK:           ]
+// CHECK:         ^bb3:
+// CHECK:           llvm.switch %[[ARG3]] : i64, ^bb4 [
+// CHECK:             1: ^bb4
+// CHECK:           ]
+// CHECK:         ^bb4:
+// CHECK:           llvm.switch %[[ARG2]] : i64, ^bb5 [
+// CHECK:             4294967296: ^bb5
+// CHECK:           ]
+// CHECK:         ^bb5:
+// CHECK:           llvm.return
+// CHECK:         }
diff --git a/flang/test/Fir/omp_target_allocmem_freemem.fir b/flang/test/Fir/omp_target_allocmem_freemem.fir
new file mode 100644
index 0000000000000..03eb94acb1ac7
--- /dev/null
+++ b/flang/test/Fir/omp_target_allocmem_freemem.fir
@@ -0,0 +1,294 @@
+// RUN: %flang_fc1 -emit-llvm  %s -o - | FileCheck %s
+
+// UNSUPPORTED: system-windows
+// Disabled on 32-bit targets due to the additional `trunc` opcodes required
+// UNSUPPORTED: target-x86
+// UNSUPPORTED: target=sparc-{{.*}}
+// UNSUPPORTED: target=sparcel-{{.*}}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_nonchar() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 4, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, i32
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalars_nonchar() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 400, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalars_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %0 = arith.constant 100 : index
+  %1 = omp.target_allocmem %device : i32, i32, %0
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_char() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 10, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_char() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<1,10>
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_char_kind() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 20, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_char_kind() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<2,10>
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_dynchar(
+// CHECK-SAME: i32 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_dynchar(%l : i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<1,?>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_scalar_dynchar_kind(
+// CHECK-SAME: i32 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 2, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_scalar_dynchar_kind(%l : i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.char<2,?>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_nonchar() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 36, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_of_nonchar() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3xi32>
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_char() {
+// CHECK-NEXT:    [[TMP1:%.*]] = call ptr @omp_target_alloc(i64 90, i32 0)
+// CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP2]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP3]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_of_char() -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,10>>
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_of_dynchar(
+// CHECK-SAME: i32 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 9, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_of_dynchar(%l: i32) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x3x!fir.char<1,?>>(%l : i32)
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_nonchar(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 12, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = call ptr @omp_target_alloc(i64 [[TMP3]], i32 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP6]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_nonchar(%e: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?xi32>, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_nonchar2(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 4, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_nonchar2(%e: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<?x?xi32>, %e, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_char(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 60, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = call ptr @omp_target_alloc(i64 [[TMP3]], i32 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP6]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_char(%e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x!fir.char<2,10>>, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_char2(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 20, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 1, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = call ptr @omp_target_alloc(i64 [[TMP4]], i32 0)
+// CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
+// CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP7]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_char2(%e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<?x?x!fir.char<2,10>>, %e, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_dynchar(
+// CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 6, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP1]]
+// CHECK-NEXT:    [[TMP6:%.*]] = mul i64 1, [[TMP5]]
+// CHECK-NEXT:    [[TMP7:%.*]] = call ptr @omp_target_alloc(i64 [[TMP6]], i32 0)
+// CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[TMP7]] to i64
+// CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP9]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_dynchar(%l: i32, %e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x!fir.char<2,?>>(%l : i32), %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_dynarray_of_dynchar2(
+// CHECK-SAME: i32 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[TMP0]] to i64
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 2, [[TMP3]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP1]]
+// CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP1]]
+// CHECK-NEXT:    [[TMP7:%.*]] = mul i64 1, [[TMP6]]
+// CHECK-NEXT:    [[TMP8:%.*]] = call ptr @omp_target_alloc(i64 [[TMP7]], i32 0)
+// CHECK-NEXT:    [[TMP9:%.*]] = ptrtoint ptr [[TMP8]] to i64
+// CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP10]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_dynarray_of_dynchar2(%l: i32, %e : index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<?x?x!fir.char<2,?>>(%l : i32), %e, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_with_holes_nonchar(
+// CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 240, [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 1, [[TMP4]]
+// CHECK-NEXT:    [[TMP6:%.*]] = call ptr @omp_target_alloc(i64 [[TMP5]], i32 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP6]] to i64
+// CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP8]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_with_holes_nonchar(%0 : index, %1 : index) -> () {
+  %device = arith.constant 0 : i32
+  %2 = omp.target_allocmem %device : i32, !fir.array<4x?x3x?x5xi32>, %0, %1
+  omp.target_freemem %device, %2 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_with_holes_char(
+// CHECK-SAME: i64 [[TMP0:%.*]]) {
+// CHECK-NEXT:    [[TMP2:%.*]] = mul i64 240, [[TMP0]]
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 1, [[TMP2]]
+// CHECK-NEXT:    [[TMP4:%.*]] = call ptr @omp_target_alloc(i64 [[TMP3]], i32 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint ptr [[TMP4]] to i64
+// CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP6]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_with_holes_char(%e: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x4x!fir.char<2,10>>, %e
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
+
+// CHECK-LABEL: define void @omp_target_allocmem_array_with_holes_dynchar(
+// CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]]) {
+// CHECK-NEXT:    [[TMP3:%.*]] = mul i64 24, [[TMP0]]
+// CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], [[TMP1]]
+// CHECK-NEXT:    [[TMP5:%.*]] = mul i64 1, [[TMP4]]
+// CHECK-NEXT:    [[TMP6:%.*]] = call ptr @omp_target_alloc(i64 [[TMP5]], i32 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[TMP6]] to i64
+// CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+// CHECK-NEXT:    call void @omp_target_free(ptr [[TMP8]], i32 0)
+// CHECK-NEXT:    ret void
+func.func @omp_target_allocmem_array_with_holes_dynchar(%arg0: index, %arg1: index) -> () {
+  %device = arith.constant 0 : i32
+  %1 = omp.target_allocmem %device : i32, !fir.array<3x?x4x!fir.char<2,?>>(%arg0 : index), %arg1
+  omp.target_freemem %device, %1 : i32, i64
+  return
+}
diff --git a/flang/test/Fir/select.fir b/flang/test/Fir/select.fir
index 47cc5e4122076..5e88048446407 100644
--- a/flang/test/Fir/select.fir
+++ b/flang/test/Fir/select.fir
@@ -7,8 +7,8 @@
 func.func @f(%a : i32) -> i32 {
    %1 = arith.constant 1 : i32
    %2 = arith.constant 42 : i32
-// CHECK: switch i32 %{{.*}}, label %{{.*}} [
-// CHECK:   i32 1, label %{{.*}}
+// CHECK: switch i64 %{{.*}}, label %{{.*}} [
+// CHECK:   i64 1, label %{{.*}}
 // CHECK: ]
    fir.select %a : i32 [1, ^bb2(%1:i32), unit, ^bb3(%2:i32)]
 ^bb2(%3 : i32) :
@@ -24,9 +24,9 @@ func.func @g(%a : i32) -> i32 {
    %1 = arith.constant 1 : i32
    %2 = arith.constant 42 : i32
 
-// CHECK: switch i32 %{{.*}}, label %{{.*}} [
-// CHECK:  i32 1, label %{{.*}}
-// CHECK:  i32 -1, label %{{.*}}
+// CHECK: switch i64 %{{.*}}, label %{{.*}} [
+// CHECK:  i64 1, label %{{.*}}
+// CHECK:  i64 -1, label %{{.*}}
 // CHECK: ]
    fir.select_rank %a : i32 [1, ^bb2(%1:i32), -1, ^bb4, unit, ^bb3(%2:i32)]
 ^bb2(%3 : i32) :
diff --git a/flang/test/HLFIR/designate-codegen-component-refs.fir b/flang/test/HLFIR/designate-codegen-component-refs.fir
index 278a7be0e2da1..5d90e325fdf98 100644
--- a/flang/test/HLFIR/designate-codegen-component-refs.fir
+++ b/flang/test/HLFIR/designate-codegen-component-refs.fir
@@ -220,3 +220,33 @@ func.func @test_array_comp_non_contiguous_slice(%arg0: !fir.ref<!fir.type<t_arra
 // CHECK:  %[[VAL_12:.*]] = fir.undefined index
 // CHECK:  %[[VAL_13:.*]] = fir.slice %[[VAL_5]], %[[VAL_6]], %[[VAL_5]], %[[VAL_7]], %[[VAL_3]], %[[VAL_5]] : (index, index, index, index, index, index) -> !fir.slice<2>
 // CHECK:  %[[VAL_14:.*]] = fir.embox %[[VAL_11]](%[[VAL_4]]) {{\[}}%[[VAL_13]]] : (!fir.ref<!fir.array<10x20xf32>>, !fir.shape<2>, !fir.slice<2>) -> !fir.box<!fir.array<6x17xf32>>
+
+func.func @test_array_comp_slice_contiguous(%arg0: !fir.box<!fir.array<2x!fir.type<_QMtypesTt{i:i32}>>, volatile>) {
+  %c2 = arith.constant 2 : index
+  %c0_i32 = arith.constant 0 : i32
+  %4 = fir.shape %c2 : (index) -> !fir.shape<1>
+  %5 = hlfir.designate %arg0{"i"} shape %4 : (!fir.box<!fir.array<2x!fir.type<_QMtypesTt{i:i32}>>, volatile>, !fir.shape<1>) -> !fir.ref<!fir.array<2xi32>, volatile>
+  hlfir.assign %c0_i32 to %5 : i32, !fir.ref<!fir.array<2xi32>, volatile>
+  return
+}
+// CHECK-LABEL:   func.func @test_array_comp_slice_contiguous(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.box<!fir.array<2x!fir.type<_QMtypesTt{i:i32}>>, volatile>) {
+// CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.box<!fir.array<2xi32>, volatile>
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_3:.*]] = fir.shape %[[VAL_1]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_4]] : (!fir.box<!fir.array<2x!fir.type<_QMtypesTt{i:i32}>>, volatile>, index) -> (index, index, index)
+// CHECK:           %[[VAL_6:.*]] = fir.shift %[[VAL_5]]#0 : (index) -> !fir.shift<1>
+// CHECK:           %[[VAL_7:.*]] = fir.field_index i, !fir.type<_QMtypesTt{i:i32}>
+// CHECK:           %[[VAL_8:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_9:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_10:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_9]] : (!fir.box<!fir.array<2x!fir.type<_QMtypesTt{i:i32}>>, volatile>, index) -> (index, index, index)
+// CHECK:           %[[VAL_11:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_12:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_13:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_12]] : (!fir.box<!fir.array<2x!fir.type<_QMtypesTt{i:i32}>>, volatile>, index) -> (index, index, index)
+// CHECK:           %[[VAL_14:.*]] = arith.addi %[[VAL_10]]#0, %[[VAL_13]]#1 : index
+// CHECK:           %[[VAL_15:.*]] = arith.subi %[[VAL_14]], %[[VAL_11]] : index
+// CHECK:           %[[VAL_16:.*]] = fir.slice %[[VAL_10]]#0, %[[VAL_15]], %[[VAL_8]] path %[[VAL_7]] : (index, index, index, !fir.field) -> !fir.slice<1>
+// CHECK:           %[[VAL_17:.*]] = fir.rebox %[[ARG0]](%[[VAL_6]]) {{\[}}%[[VAL_16]]] : (!fir.box<!fir.array<2x!fir.type<_QMtypesTt{i:i32}>>, volatile>, !fir.shift<1>, !fir.slice<1>) -> !fir.box<!fir.ref<!fir.array<2xi32>, volatile>, volatile>
+// CHECK:           %[[VAL_18:.*]] = fir.box_addr %[[VAL_17]] : (!fir.box<!fir.ref<!fir.array<2xi32>, volatile>, volatile>) -> !fir.ref<!fir.array<2xi32>, volatile>
diff --git a/flang/test/HLFIR/eoshift-lowering.fir b/flang/test/HLFIR/eoshift-lowering.fir
new file mode 100644
index 0000000000000..7bfc3e21f0527
--- /dev/null
+++ b/flang/test/HLFIR/eoshift-lowering.fir
@@ -0,0 +1,294 @@
+// Test hlfir.eoshift operation lowering to fir runtime call
+// RUN: fir-opt %s -lower-hlfir-intrinsics | FileCheck %s
+
+// 1d boxed vector shift by scalar
+func.func @eoshift1(%arg0: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "sh"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %1:2 = hlfir.declare %arg1 {uniq_name = "sh"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2 = hlfir.eoshift %0#0 %1#0 : (!fir.box<!fir.array<?xi32>>, !fir.ref<i32>) -> !hlfir.expr<?xi32>
+  hlfir.assign %2 to %0#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+  return
+}
+// CHECK-LABEL:   func.func @eoshift1(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "sh"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_8:.*]] = fir.zero_bits !fir.heap<!fir.array<?xi32>>
+// CHECK:           %[[VAL_9:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_10:.*]] = fir.embox %[[VAL_8]](%[[VAL_9]]) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK:           fir.store %[[VAL_10]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+// CHECK:           %[[BOUNDARY:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_6]]#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_11]] : (i32) -> i64
+// CHECK:           fir.call @_FortranAEoshiftVector(%[[VAL_13]], %[[VAL_14]], %[[VAL_15]], %[[BOUNDARY]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i64, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+
+// 2d boxed array shift by scalar
+func.func @eoshift2(%arg0: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"}, %arg1: i32 {fir.bindc_name = "sh"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+  %2 = hlfir.eoshift %0#0 %arg1 : (!fir.box<!fir.array<?x?xi32>>, i32) -> !hlfir.expr<?x?xi32>
+  hlfir.assign %2 to %0#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+  return
+}
+// CHECK-LABEL:   func.func @eoshift2(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: i32 {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_7:.*]] = fir.alloca i32
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+// CHECK:           fir.store %[[VAL_1]] to %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_11:.*]] = fir.embox %[[VAL_9]](%[[VAL_10]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_11]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[BOUNDARY:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_7]] : (!fir.ref<i32>) -> !fir.box<i32>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_8]]#1 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_12]] : (!fir.box<i32>) -> !fir.box<none>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// CHECK:           fir.call @_FortranAEoshift(%[[VAL_14]], %[[VAL_15]], %[[VAL_16]], %[[BOUNDARY]], %[[VAL_17]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
+
+// 2d boxed array shift by boxed array
+func.func @eoshift3(%arg0: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "sh"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+  %1:2 = hlfir.declare %arg1 {uniq_name = "sh"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %2 = hlfir.eoshift %0#0 %1#0 : (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?xi32>>) -> !hlfir.expr<?x?xi32>
+  hlfir.assign %2 to %0#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+  return
+}
+// CHECK-LABEL:   func.func @eoshift3(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_1]] {uniq_name = "sh"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+// CHECK:           %[[VAL_9:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_10:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_11:.*]] = fir.embox %[[VAL_9]](%[[VAL_10]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_11]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[BOUNDARY:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_7]]#1 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_8]]#1 : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// CHECK:           fir.call @_FortranAEoshift(%[[VAL_13]], %[[VAL_14]], %[[VAL_15]], %[[BOUNDARY]], %[[VAL_16]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
+
+// 2d boxed array shift by array expr
+func.func @eoshift4(%arg0: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"}, %arg1: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+  %2 = hlfir.eoshift %0#0 %arg1 : (!fir.box<!fir.array<?x?xi32>>, !hlfir.expr<?xi32>) -> !hlfir.expr<?x?xi32>
+  hlfir.assign %2 to %0#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+  return
+}
+// CHECK-LABEL:   func.func @eoshift4(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "a"} : (!fir.box<!fir.array<?x?xi32>>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?x?xi32>>)
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_1]](%[[VAL_8]]) {adapt.valuebyref} : (!hlfir.expr<?xi32>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, i1)
+// CHECK:           %[[VAL_10:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_11:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_13:.*]] = fir.embox %[[VAL_11]](%[[VAL_12]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_13]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[BOUNDARY:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_9]]#1(%[[VAL_14]]) : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_7]]#1 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_15]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// CHECK:           fir.call @_FortranAEoshift(%[[VAL_17]], %[[VAL_18]], %[[VAL_19]], %[[BOUNDARY]], %[[VAL_20]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
+
+// 2d array expr shift by array expr
+func.func @eoshift5(%arg0: !hlfir.expr<?x?xi32> {fir.bindc_name = "a"}, %arg1: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}) {
+  %2 = hlfir.eoshift %arg0 %arg1 : (!hlfir.expr<?x?xi32>, !hlfir.expr<?xi32>) -> !hlfir.expr<?x?xi32>
+  hlfir.destroy %2 : !hlfir.expr<?x?xi32>
+  return
+}
+// CHECK-LABEL:   func.func @eoshift5(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !hlfir.expr<?x?xi32> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_7:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_8:.*]]:3 = hlfir.associate %[[VAL_0]](%[[VAL_7]]) {adapt.valuebyref} : (!hlfir.expr<?x?xi32>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.ref<!fir.array<?x?xi32>>, i1)
+// CHECK:           %[[VAL_9:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_10:.*]] = hlfir.get_extent %[[VAL_7]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_9]], %[[VAL_10]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_8]]#1(%[[VAL_14]]) : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_11:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:3 = hlfir.associate %[[VAL_1]](%[[VAL_11]]) {adapt.valuebyref} : (!hlfir.expr<?xi32>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, i1)
+// CHECK:           %[[VAL_13:.*]] = hlfir.get_extent %[[VAL_11]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_16:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_5]], %[[VAL_5]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_18:.*]] = fir.embox %[[VAL_16]](%[[VAL_17]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_18]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[BOUNDARY:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_19:.*]] = fir.shape %[[VAL_13]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_20:.*]] = fir.embox %[[VAL_12]]#1(%[[VAL_19]]) : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_15]] : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_20]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_4]] : (index) -> i32
+// CHECK:           fir.call @_FortranAEoshift(%[[VAL_22]], %[[VAL_23]], %[[VAL_24]], %[[BOUNDARY]], %[[VAL_25]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
+
+// 2d array expr shift by array expr with explicit dim
+func.func @eoshift6(%arg0: !hlfir.expr<?x?xi32> {fir.bindc_name = "a"}, %arg1: !hlfir.expr<?xi32> {fir.bindc_name = "sh"}, %dim : i16) {
+  %2 = hlfir.eoshift %arg0 %arg1 dim %dim : (!hlfir.expr<?x?xi32>, !hlfir.expr<?xi32>, i16) -> !hlfir.expr<?x?xi32>
+  hlfir.destroy %2 : !hlfir.expr<?x?xi32>
+  return
+}
+// CHECK-LABEL:   func.func @eoshift6(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !hlfir.expr<?x?xi32> {fir.bindc_name = "a"},
+// CHECK-SAME:                       %[[VAL_1:.*]]: !hlfir.expr<?xi32> {fir.bindc_name = "sh"},
+// CHECK-SAME:                       %[[VAL_2:.*]]: i16) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_7:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           %[[VAL_8:.*]] = hlfir.shape_of %[[VAL_0]] : (!hlfir.expr<?x?xi32>) -> !fir.shape<2>
+// CHECK:           %[[VAL_9:.*]]:3 = hlfir.associate %[[VAL_0]](%[[VAL_8]]) {adapt.valuebyref} : (!hlfir.expr<?x?xi32>, !fir.shape<2>) -> (!fir.box<!fir.array<?x?xi32>>, !fir.ref<!fir.array<?x?xi32>>, i1)
+// CHECK:           %[[VAL_10:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 0 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_11:.*]] = hlfir.get_extent %[[VAL_8]] {dim = 1 : index} : (!fir.shape<2>) -> index
+// CHECK:           %[[VAL_16:.*]] = fir.shape %[[VAL_10]], %[[VAL_11]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_17:.*]] = fir.embox %[[VAL_9]]#1(%[[VAL_16]]) : (!fir.ref<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_12:.*]] = hlfir.shape_of %[[VAL_1]] : (!hlfir.expr<?xi32>) -> !fir.shape<1>
+// CHECK:           %[[VAL_13:.*]]:3 = hlfir.associate %[[VAL_1]](%[[VAL_12]]) {adapt.valuebyref} : (!hlfir.expr<?xi32>, !fir.shape<1>) -> (!fir.box<!fir.array<?xi32>>, !fir.ref<!fir.array<?xi32>>, i1)
+// CHECK:           %[[VAL_14:.*]] = hlfir.get_extent %[[VAL_12]] {dim = 0 : index} : (!fir.shape<1>) -> index
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_2]] : (i16) -> i32
+// CHECK:           %[[VAL_18:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xi32>>
+// CHECK:           %[[VAL_19:.*]] = fir.shape %[[VAL_6]], %[[VAL_6]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_20:.*]] = fir.embox %[[VAL_18]](%[[VAL_19]]) : (!fir.heap<!fir.array<?x?xi32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xi32>>>
+// CHECK:           fir.store %[[VAL_20]] to %[[VAL_7]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>
+// CHECK:           %[[BOUNDARY:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_21:.*]] = fir.shape %[[VAL_14]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_22:.*]] = fir.embox %[[VAL_13]]#1(%[[VAL_21]]) : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_7]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xi32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_17]] : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_22]] : (!fir.box<!fir.array<?xi32>>) -> !fir.box<none>
+// CHECK:           fir.call @_FortranAEoshift(%[[VAL_24]], %[[VAL_25]], %[[VAL_26]], %[[BOUNDARY]], %[[VAL_15]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
+
+// shift of polymorphic array
+func.func @eoshift7(%arg0: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, %arg1: !fir.ref<f32>) {
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "a"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>)
+  %2:2 = hlfir.declare %arg1 dummy_scope %0 {uniq_name = "sh"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+  %c2_i32 = arith.constant 2 : i32
+  %3 = fir.load %1#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+  %4 = hlfir.eoshift %3 %c2_i32 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>, i32) -> !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+  hlfir.assign %4 to %1#0 realloc : !hlfir.expr<?x!fir.type<_QMtypesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+  hlfir.destroy %4 : !hlfir.expr<?x!fir.type<_QMtypesTt>?>
+  return
+}
+// CHECK-LABEL:   func.func @eoshift7(
+// CHECK-SAME:                       %[[VAL_0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>,
+// CHECK-SAME:                       %[[VAL_1:.*]]: !fir.ref<f32>) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           %[[VAL_7:.*]] = fir.alloca i32
+// CHECK:           %[[VAL_8:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_8]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "a"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>)
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_8]] {uniq_name = "sh"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_9]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           fir.store %[[VAL_5]] to %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.zero_bits !fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>
+// CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_4]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_14:.*]] = fir.embox %[[VAL_12]](%[[VAL_13]]) source_box %[[VAL_11]] : (!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>, !fir.shape<1>, !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>
+// CHECK:           fir.store %[[VAL_14]] to %[[VAL_6]] : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>
+// CHECK:           %[[BOUNDARY:.*]] = fir.absent !fir.box<none>
+// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_7]] : !fir.ref<i32>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_11]] : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMtypesTt>>>>) -> !fir.box<none>
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_15]] : (i32) -> i64
+// CHECK:           fir.call @_FortranAEoshiftVector(%[[VAL_17]], %[[VAL_18]], %[[VAL_19]], %[[BOUNDARY]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i64, !fir.box<none>, !fir.ref<i8>, i32) -> ()
+
+// shift with the present scalar boundary and dim
+func.func @_QPeoshift8(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "array"}) {
+  %cst = arith.constant 3.000000e+00 : f32
+  %c2_i32 = arith.constant 2 : i32
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift8Earray"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+  %2 = hlfir.eoshift %1#0 %c2_i32 boundary %cst dim %c2_i32 : (!fir.box<!fir.array<?x?xf32>>, i32, f32, i32) -> !hlfir.expr<?x?xf32>
+  hlfir.assign %2 to %1#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+  hlfir.destroy %2 : !hlfir.expr<?x?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift8(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "array"}) {
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 3.000000e+00 : f32
+// CHECK:           %[[VAL_6:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+// CHECK:           %[[VAL_7:.*]] = fir.alloca f32
+// CHECK:           %[[VAL_8:.*]] = fir.alloca i32
+// CHECK:           %[[VAL_9:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_10:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_9]] {uniq_name = "_QFeoshift8Earray"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_8]] : !fir.ref<i32>
+// CHECK:           fir.store %[[VAL_5]] to %[[VAL_7]] : !fir.ref<f32>
+// CHECK:           %[[VAL_11:.*]] = fir.embox %[[VAL_7]] : (!fir.ref<f32>) -> !fir.box<f32>
+// CHECK:           %[[VAL_12:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xf32>>
+// CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_3]], %[[VAL_3]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_14:.*]] = fir.embox %[[VAL_12]](%[[VAL_13]]) : (!fir.heap<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+// CHECK:           fir.store %[[VAL_14]] to %[[VAL_6]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+// CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_8]] : (!fir.ref<i32>) -> !fir.box<i32>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_6]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_10]]#1 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_15]] : (!fir.box<i32>) -> !fir.box<none>
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_11]] : (!fir.box<f32>) -> !fir.box<none>
+// CHECK:           fir.call @_FortranAEoshift(%[[VAL_17]], %[[VAL_18]], %[[VAL_19]], %[[VAL_20]], %[[VAL_4]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
+
+// shift with the present array boundary
+func.func @_QPeoshift9(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "array"}, %arg1: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "boundary"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift9Earray"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+  %2:2 = hlfir.declare %arg1 dummy_scope %0 {uniq_name = "_QFeoshift9Eboundary"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %3 = hlfir.eoshift %1#0 %c2_i32 boundary %2#0 : (!fir.box<!fir.array<?x?xf32>>, i32, !fir.box<!fir.array<?xf32>>) -> !hlfir.expr<?x?xf32>
+  hlfir.assign %3 to %1#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+  hlfir.destroy %3 : !hlfir.expr<?x?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift9(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "boundary"}) {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_5:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+// CHECK:           %[[VAL_6:.*]] = fir.alloca i32
+// CHECK:           %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_7]] {uniq_name = "_QFeoshift9Earray"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+// CHECK:           %[[VAL_9:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %[[VAL_7]] {uniq_name = "_QFeoshift9Eboundary"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK:           fir.store %[[VAL_4]] to %[[VAL_6]] : !fir.ref<i32>
+// CHECK:           %[[VAL_10:.*]] = fir.zero_bits !fir.heap<!fir.array<?x?xf32>>
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_3]], %[[VAL_3]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_12:.*]] = fir.embox %[[VAL_10]](%[[VAL_11]]) : (!fir.heap<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.heap<!fir.array<?x?xf32>>>
+// CHECK:           fir.store %[[VAL_12]] to %[[VAL_5]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>
+// CHECK:           %[[VAL_13:.*]] = fir.embox %[[VAL_6]] : (!fir.ref<i32>) -> !fir.box<i32>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_5]] : (!fir.ref<!fir.box<!fir.heap<!fir.array<?x?xf32>>>>) -> !fir.ref<!fir.box<none>>
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_8]]#1 : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_13]] : (!fir.box<i32>) -> !fir.box<none>
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_9]]#1 : (!fir.box<!fir.array<?xf32>>) -> !fir.box<none>
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_2]] : (index) -> i32
+// CHECK:           fir.call @_FortranAEoshift(%[[VAL_15]], %[[VAL_16]], %[[VAL_17]], %[[VAL_18]], %[[VAL_19]], %{{.*}}, %{{.*}}) : (!fir.ref<!fir.box<none>>, !fir.box<none>, !fir.box<none>, !fir.box<none>, i32, !fir.ref<i8>, i32) -> ()
diff --git a/flang/test/HLFIR/invalid.fir b/flang/test/HLFIR/invalid.fir
index d61efe0062e69..0f54a0250294b 100644
--- a/flang/test/HLFIR/invalid.fir
+++ b/flang/test/HLFIR/invalid.fir
@@ -1555,3 +1555,96 @@ func.func @bad_reshape(%arg0: !hlfir.expr<1x!fir.char<1,2>>, %arg1: !hlfir.expr<
   %0 = hlfir.reshape %arg0 %arg1 pad %arg2 : (!hlfir.expr<1x!fir.char<1,2>>, !hlfir.expr<1xi32>, !hlfir.expr<1x!fir.char<2,?>>) -> !hlfir.expr<?x!fir.char<1,?>>
   return
 }
+
+// -----
+
+func.func @bad_eoshift1(%arg0: !hlfir.expr<?x?xi32>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.eoshift' op input and output arrays should have the same element type}}
+  %0 = hlfir.eoshift %arg0 %arg1 : (!hlfir.expr<?x?xi32>, i32) -> !hlfir.expr<?x?xf32>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift2(%arg0: !hlfir.expr<?x?xi32>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.eoshift' op input and output arrays should have the same rank}}
+  %0 = hlfir.eoshift %arg0 %arg1 : (!hlfir.expr<?x?xi32>, i32) -> !hlfir.expr<?xi32>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift3(%arg0: !hlfir.expr<2x2xi32>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.eoshift' op output array's shape conflicts with the input array's shape}}
+  %0 = hlfir.eoshift %arg0 %arg1 : (!hlfir.expr<2x2xi32>, i32) -> !hlfir.expr<2x3xi32>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift4(%arg0: !hlfir.expr<2x2xi32>, %arg1: i32) {
+  %c0 = arith.constant 0 : index
+  // expected-error@+1 {{'hlfir.eoshift' op DIM must be >= 1}}
+  %0 = hlfir.eoshift %arg0 %arg1 dim %c0 : (!hlfir.expr<2x2xi32>, i32, index) -> !hlfir.expr<2x2xi32>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift5(%arg0: !hlfir.expr<2x2xi32>, %arg1: i32) {
+  %c10 = arith.constant 10 : index
+  // expected-error@+1 {{'hlfir.eoshift' op DIM must be <= input array's rank}}
+  %0 = hlfir.eoshift %arg0 %arg1 dim %c10 : (!hlfir.expr<2x2xi32>, i32, index) -> !hlfir.expr<2x2xi32>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift6(%arg0: !hlfir.expr<2x2xi32>, %arg1: !hlfir.expr<2x2xi32>) {
+  // expected-error@+1 {{'hlfir.eoshift' op SHIFT's rank must be 1 less than the input array's rank}}
+  %0 = hlfir.eoshift %arg0 %arg1 : (!hlfir.expr<2x2xi32>, !hlfir.expr<2x2xi32>) -> !hlfir.expr<2x2xi32>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift7(%arg0: !hlfir.expr<?x2xi32>, %arg1: !hlfir.expr<3xi32>) {
+  %c1 = arith.constant 1 : index
+  // expected-error@+1 {{'hlfir.eoshift' op SHAPE(ARRAY)(2) must be equal to SHAPE(SHIFT)(1): 2 != 3}}
+  %0 = hlfir.eoshift %arg0 %arg1 dim %c1 : (!hlfir.expr<?x2xi32>, !hlfir.expr<3xi32>, index) -> !hlfir.expr<2x2xi32>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift8(%arg0: !hlfir.expr<?x!fir.char<1,?>>, %arg1: i32) {
+  // expected-error@+2 {{'hlfir.eoshift' op character KIND mismatch}}
+  // expected-error@+1 {{'hlfir.eoshift' op input and output arrays should have the same element type}}
+  %0 = hlfir.eoshift %arg0 %arg1 : (!hlfir.expr<?x!fir.char<1,?>>, i32) -> !hlfir.expr<?x!fir.char<2,?>>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift9(%arg0: !hlfir.expr<?x!fir.char<1,1>>, %arg1: i32) {
+  // expected-error@+2 {{'hlfir.eoshift' op character LEN mismatch}}
+  // expected-error@+1 {{'hlfir.eoshift' op input and output arrays should have the same element type}}
+  %0 = hlfir.eoshift %arg0 %arg1 : (!hlfir.expr<?x!fir.char<1,1>>, i32) -> !hlfir.expr<?x!fir.char<1,2>>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift10(%arg0: !hlfir.expr<2x2xi32>, %arg1: i32, %arg2: f32) {
+  // expected-error@+1 {{'hlfir.eoshift' op ARRAY and BOUNDARY operands must have the same element type}}
+  %0 = hlfir.eoshift %arg0 %arg1 boundary %arg2 : (!hlfir.expr<2x2xi32>, i32, f32) -> !hlfir.expr<2x2xi32>
+  return
+}
+
+// -----
+
+func.func @bad_eoshift11(%arg0: !hlfir.expr<2x2xi32>, %arg1: i32, %arg2: !hlfir.expr<2x2xi32>) {
+  // expected-error@+1 {{'hlfir.eoshift' op BOUNDARY's rank must be 1 less than the input array's rank}}
+  %0 = hlfir.eoshift %arg0 %arg1 boundary %arg2 : (!hlfir.expr<2x2xi32>, i32, !hlfir.expr<2x2xi32>) -> !hlfir.expr<2x2xi32>
+  return
+}
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
index 8684a429ea5b4..f5af990da194c 100644
--- a/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-cshift.fir
@@ -38,12 +38,12 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:               %[[VAL_24:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_2]]:%[[VAL_6]]#1:%[[VAL_2]])  shape %[[VAL_23]] : (!fir.box<!fir.array<?xi32>>, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
 // CHECK:               %[[VAL_25:.*]] = fir.box_addr %[[VAL_24]] : (!fir.box<!fir.array<?xi32>>) -> !fir.ref<!fir.array<?xi32>>
 // CHECK:               %[[VAL_26:.*]] = fir.embox %[[VAL_25]](%[[VAL_23]]) : (!fir.ref<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xi32>>
+// CHECK:               %[[VAL_36:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_17]] : (i64) -> index
 // CHECK:               fir.do_loop %[[VAL_28:.*]] = %[[VAL_2]] to %[[VAL_27]] step %[[VAL_2]] unordered {
 // CHECK:                 %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (index) -> i64
 // CHECK:                 %[[VAL_34:.*]] = hlfir.designate %[[VAL_26]] (%[[VAL_29]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_35:.*]] = fir.load %[[VAL_34]] : !fir.ref<i32>
-// CHECK:                 %[[VAL_36:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_37:.*]] = arith.addi %[[VAL_29]], %[[VAL_36]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_42:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_37]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_35]] to %[[VAL_42]] : i32, !fir.ref<i32>
@@ -59,6 +59,7 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:                 hlfir.assign %[[VAL_53]] to %[[VAL_58]] : i32, !fir.ref<i32>
 // CHECK:               }
 // CHECK:             } else {
+// CHECK:               %[[VAL_68:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:               %[[VAL_59:.*]] = fir.convert %[[VAL_17]] : (i64) -> index
 // CHECK:               fir.do_loop %[[VAL_60:.*]] = %[[VAL_2]] to %[[VAL_59]] step %[[VAL_2]] unordered {
 // CHECK:                 %[[VAL_61:.*]] = fir.convert %[[VAL_60]] : (index) -> i64
@@ -68,7 +69,6 @@ func.func @cshift_vector(%arg0: !fir.box<!fir.array<?xi32>>, %arg1: !fir.ref<i32
 // CHECK:                 %[[VAL_65:.*]] = arith.addi %[[VAL_63]], %[[VAL_64]] overflow<nsw, nuw> : index
 // CHECK:                 %[[VAL_66:.*]] = hlfir.designate %[[VAL_0]] (%[[VAL_65]])  : (!fir.box<!fir.array<?xi32>>, index) -> !fir.ref<i32>
 // CHECK:                 %[[VAL_67:.*]] = fir.load %[[VAL_66]] : !fir.ref<i32>
-// CHECK:                 %[[VAL_68:.*]] = arith.subi %[[VAL_8]], %[[VAL_17]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_69:.*]] = arith.addi %[[VAL_61]], %[[VAL_68]] overflow<nsw, nuw> : i64
 // CHECK:                 %[[VAL_74:.*]] = hlfir.designate %[[VAL_20]] (%[[VAL_69]])  : (!fir.box<!fir.array<?xi32>>, i64) -> !fir.ref<i32>
 // CHECK:                 hlfir.assign %[[VAL_67]] to %[[VAL_74]] : i32, !fir.ref<i32>
diff --git a/flang/test/HLFIR/simplify-hlfir-intrinsics-eoshift.fir b/flang/test/HLFIR/simplify-hlfir-intrinsics-eoshift.fir
new file mode 100644
index 0000000000000..d8975c9741287
--- /dev/null
+++ b/flang/test/HLFIR/simplify-hlfir-intrinsics-eoshift.fir
@@ -0,0 +1,2237 @@
+// Test hlfir.eoshift simplification to hlfir.elemental and hlfir.eval_in_mem:
+// RUN: fir-opt --simplify-hlfir-intrinsics %s | FileCheck %s
+
+// module eoshift_types
+//   type t
+//   end type t
+// end module eoshift_types
+//
+// ! Test contiguous 1D array with statically absent boundary.
+// subroutine eoshift1(n, array)
+//   integer :: n
+//   real(2) :: array(n)
+//   array = EOSHIFT(array, 2)
+// end subroutine
+func.func @_QPeoshift1(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!fir.array<?xf16>> {fir.bindc_name = "array"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift1En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2 = fir.load %1#0 : !fir.ref<i32>
+  %3 = fir.convert %2 : (i32) -> index
+  %4 = arith.cmpi sgt, %3, %c0 : index
+  %5 = arith.select %4, %3, %c0 : index
+  %6 = fir.shape %5 : (index) -> !fir.shape<1>
+  %7:2 = hlfir.declare %arg1(%6) dummy_scope %0 {uniq_name = "_QFeoshift1Earray"} : (!fir.ref<!fir.array<?xf16>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf16>>, !fir.ref<!fir.array<?xf16>>)
+  %8 = hlfir.eoshift %7#0 %c2_i32 : (!fir.box<!fir.array<?xf16>>, i32) -> !hlfir.expr<?xf16>
+  hlfir.assign %8 to %7#0 : !hlfir.expr<?xf16>, !fir.box<!fir.array<?xf16>>
+  hlfir.destroy %8 : !hlfir.expr<?xf16>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift1(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<!fir.array<?xf16>> {fir.bindc_name = "array"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f16
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_5]] {uniq_name = "_QFeoshift1En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i32) -> index
+// CHECK:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_8]], %[[VAL_4]] : index
+// CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_9]], %[[VAL_8]], %[[VAL_4]] : index
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[ARG1]](%[[VAL_11]]) dummy_scope %[[VAL_5]] {uniq_name = "_QFeoshift1Earray"} : (!fir.ref<!fir.array<?xf16>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf16>>, !fir.ref<!fir.array<?xf16>>)
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_3]] : (i32) -> i64
+// CHECK:           %[[VAL_15:.*]] = hlfir.eval_in_mem shape %[[VAL_11]] : (!fir.shape<1>) -> !hlfir.expr<?xf16> {
+// CHECK:           ^bb0(%[[VAL_16:.*]]: !fir.ref<!fir.array<?xf16>>):
+// CHECK:             %[[VAL_17:.*]] = fir.embox %[[VAL_16]](%[[VAL_11]]) : (!fir.ref<!fir.array<?xf16>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf16>>
+// CHECK:             %[[VAL_18:.*]] = arith.cmpi slt, %[[VAL_14]], %[[VAL_1]] : i64
+// CHECK:             %[[VAL_19:.*]] = arith.subi %[[VAL_1]], %[[VAL_14]] overflow<nsw> : i64
+// CHECK:             %[[VAL_20:.*]] = arith.select %[[VAL_18]], %[[VAL_19]], %[[VAL_1]] : i64
+// CHECK:             %[[VAL_21:.*]] = arith.select %[[VAL_18]], %[[VAL_1]], %[[VAL_14]] : i64
+// CHECK:             %[[VAL_22:.*]] = arith.subi %[[VAL_1]], %[[VAL_13]] overflow<nsw> : i64
+// CHECK:             %[[VAL_23:.*]] = arith.addi %[[VAL_13]], %[[VAL_14]] overflow<nsw> : i64
+// CHECK:             %[[VAL_24:.*]] = arith.cmpi sgt, %[[VAL_22]], %[[VAL_14]] : i64
+// CHECK:             %[[VAL_25:.*]] = arith.select %[[VAL_24]], %[[VAL_1]], %[[VAL_23]] : i64
+// CHECK:             %[[VAL_26:.*]] = arith.subi %[[VAL_13]], %[[VAL_14]] overflow<nsw> : i64
+// CHECK:             %[[VAL_27:.*]] = arith.cmpi slt, %[[VAL_13]], %[[VAL_14]] : i64
+// CHECK:             %[[VAL_28:.*]] = arith.select %[[VAL_27]], %[[VAL_1]], %[[VAL_26]] : i64
+// CHECK:             %[[VAL_29:.*]] = arith.select %[[VAL_18]], %[[VAL_25]], %[[VAL_28]] : i64
+// CHECK:             %[[VAL_30:.*]] = fir.convert %[[VAL_29]] : (i64) -> index
+// CHECK:             fir.do_loop %[[VAL_31:.*]] = %[[VAL_0]] to %[[VAL_30]] step %[[VAL_0]] unordered {
+// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (index) -> i64
+// CHECK:               %[[VAL_33:.*]] = arith.addi %[[VAL_32]], %[[VAL_21]] overflow<nsw> : i64
+// CHECK:               %[[VAL_34:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_33]])  : (!fir.box<!fir.array<?xf16>>, i64) -> !fir.ref<f16>
+// CHECK:               %[[VAL_35:.*]] = fir.load %[[VAL_34]] : !fir.ref<f16>
+// CHECK:               %[[VAL_36:.*]] = arith.addi %[[VAL_32]], %[[VAL_20]] overflow<nsw> : i64
+// CHECK:               %[[VAL_37:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_36]])  : (!fir.box<!fir.array<?xf16>>, i64) -> !fir.ref<f16>
+// CHECK:               hlfir.assign %[[VAL_35]] to %[[VAL_37]] : f16, !fir.ref<f16>
+// CHECK:             }
+// CHECK:             %[[VAL_38:.*]] = arith.subi %[[VAL_13]], %[[VAL_29]] overflow<nsw> : i64
+// CHECK:             %[[VAL_39:.*]] = arith.select %[[VAL_18]], %[[VAL_1]], %[[VAL_29]] : i64
+// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (i64) -> index
+// CHECK:             fir.do_loop %[[VAL_41:.*]] = %[[VAL_0]] to %[[VAL_40]] step %[[VAL_0]] unordered {
+// CHECK:               %[[VAL_42:.*]] = fir.convert %[[VAL_41]] : (index) -> i64
+// CHECK:               %[[VAL_43:.*]] = arith.addi %[[VAL_42]], %[[VAL_39]] overflow<nsw> : i64
+// CHECK:               %[[VAL_44:.*]] = hlfir.designate %[[VAL_17]] (%[[VAL_43]])  : (!fir.box<!fir.array<?xf16>>, i64) -> !fir.ref<f16>
+// CHECK:               hlfir.assign %[[VAL_2]] to %[[VAL_44]] : f16, !fir.ref<f16>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_15]] to %[[VAL_12]]#0 : !hlfir.expr<?xf16>, !fir.box<!fir.array<?xf16>>
+// CHECK:           hlfir.destroy %[[VAL_15]] : !hlfir.expr<?xf16>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the scalar constant boundary.
+// subroutine eoshift2(n, array)
+//   integer :: n
+//   logical(2) :: array(n)
+//   array = EOSHIFT(array, 2, boundary=.true._2, dim=1)
+// end subroutine
+func.func @_QPeoshift2(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!fir.array<?x!fir.logical<2>>> {fir.bindc_name = "array"}) {
+  %c1_i32 = arith.constant 1 : i32
+  %true = arith.constant true
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift2En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2 = fir.load %1#0 : !fir.ref<i32>
+  %3 = fir.convert %2 : (i32) -> index
+  %4 = arith.cmpi sgt, %3, %c0 : index
+  %5 = arith.select %4, %3, %c0 : index
+  %6 = fir.shape %5 : (index) -> !fir.shape<1>
+  %7:2 = hlfir.declare %arg1(%6) dummy_scope %0 {uniq_name = "_QFeoshift2Earray"} : (!fir.ref<!fir.array<?x!fir.logical<2>>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<2>>>, !fir.ref<!fir.array<?x!fir.logical<2>>>)
+  %8 = fir.convert %true : (i1) -> !fir.logical<2>
+  %9 = hlfir.eoshift %7#0 %c2_i32 boundary %8 dim %c1_i32 : (!fir.box<!fir.array<?x!fir.logical<2>>>, i32, !fir.logical<2>, i32) -> !hlfir.expr<?x!fir.logical<2>>
+  hlfir.assign %9 to %7#0 : !hlfir.expr<?x!fir.logical<2>>, !fir.box<!fir.array<?x!fir.logical<2>>>
+  hlfir.destroy %9 : !hlfir.expr<?x!fir.logical<2>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift2(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<!fir.array<?x!fir.logical<2>>> {fir.bindc_name = "array"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_2:.*]] = arith.constant true
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_5]] {uniq_name = "_QFeoshift2En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i32) -> index
+// CHECK:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_8]], %[[VAL_4]] : index
+// CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_9]], %[[VAL_8]], %[[VAL_4]] : index
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[ARG1]](%[[VAL_11]]) dummy_scope %[[VAL_5]] {uniq_name = "_QFeoshift2Earray"} : (!fir.ref<!fir.array<?x!fir.logical<2>>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<2>>>, !fir.ref<!fir.array<?x!fir.logical<2>>>)
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<2>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_3]] : (i32) -> i64
+// CHECK:           %[[VAL_16:.*]] = hlfir.eval_in_mem shape %[[VAL_11]] : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<2>> {
+// CHECK:           ^bb0(%[[VAL_17:.*]]: !fir.ref<!fir.array<?x!fir.logical<2>>>):
+// CHECK:             %[[VAL_18:.*]] = fir.embox %[[VAL_17]](%[[VAL_11]]) : (!fir.ref<!fir.array<?x!fir.logical<2>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.logical<2>>>
+// CHECK:             %[[VAL_19:.*]] = arith.cmpi slt, %[[VAL_15]], %[[VAL_1]] : i64
+// CHECK:             %[[VAL_20:.*]] = arith.subi %[[VAL_1]], %[[VAL_15]] overflow<nsw> : i64
+// CHECK:             %[[VAL_21:.*]] = arith.select %[[VAL_19]], %[[VAL_20]], %[[VAL_1]] : i64
+// CHECK:             %[[VAL_22:.*]] = arith.select %[[VAL_19]], %[[VAL_1]], %[[VAL_15]] : i64
+// CHECK:             %[[VAL_23:.*]] = arith.subi %[[VAL_1]], %[[VAL_14]] overflow<nsw> : i64
+// CHECK:             %[[VAL_24:.*]] = arith.addi %[[VAL_14]], %[[VAL_15]] overflow<nsw> : i64
+// CHECK:             %[[VAL_25:.*]] = arith.cmpi sgt, %[[VAL_23]], %[[VAL_15]] : i64
+// CHECK:             %[[VAL_26:.*]] = arith.select %[[VAL_25]], %[[VAL_1]], %[[VAL_24]] : i64
+// CHECK:             %[[VAL_27:.*]] = arith.subi %[[VAL_14]], %[[VAL_15]] overflow<nsw> : i64
+// CHECK:             %[[VAL_28:.*]] = arith.cmpi slt, %[[VAL_14]], %[[VAL_15]] : i64
+// CHECK:             %[[VAL_29:.*]] = arith.select %[[VAL_28]], %[[VAL_1]], %[[VAL_27]] : i64
+// CHECK:             %[[VAL_30:.*]] = arith.select %[[VAL_19]], %[[VAL_26]], %[[VAL_29]] : i64
+// CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (i64) -> index
+// CHECK:             fir.do_loop %[[VAL_32:.*]] = %[[VAL_0]] to %[[VAL_31]] step %[[VAL_0]] unordered {
+// CHECK:               %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (index) -> i64
+// CHECK:               %[[VAL_34:.*]] = arith.addi %[[VAL_33]], %[[VAL_22]] overflow<nsw> : i64
+// CHECK:               %[[VAL_35:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_34]])  : (!fir.box<!fir.array<?x!fir.logical<2>>>, i64) -> !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<!fir.logical<2>>
+// CHECK:               %[[VAL_37:.*]] = arith.addi %[[VAL_33]], %[[VAL_21]] overflow<nsw> : i64
+// CHECK:               %[[VAL_38:.*]] = hlfir.designate %[[VAL_18]] (%[[VAL_37]])  : (!fir.box<!fir.array<?x!fir.logical<2>>>, i64) -> !fir.ref<!fir.logical<2>>
+// CHECK:               hlfir.assign %[[VAL_36]] to %[[VAL_38]] : !fir.logical<2>, !fir.ref<!fir.logical<2>>
+// CHECK:             }
+// CHECK:             %[[VAL_39:.*]] = arith.subi %[[VAL_14]], %[[VAL_30]] overflow<nsw> : i64
+// CHECK:             %[[VAL_40:.*]] = arith.select %[[VAL_19]], %[[VAL_1]], %[[VAL_30]] : i64
+// CHECK:             %[[VAL_41:.*]] = fir.convert %[[VAL_39]] : (i64) -> index
+// CHECK:             fir.do_loop %[[VAL_42:.*]] = %[[VAL_0]] to %[[VAL_41]] step %[[VAL_0]] unordered {
+// CHECK:               %[[VAL_43:.*]] = fir.convert %[[VAL_42]] : (index) -> i64
+// CHECK:               %[[VAL_44:.*]] = arith.addi %[[VAL_43]], %[[VAL_40]] overflow<nsw> : i64
+// CHECK:               %[[VAL_45:.*]] = hlfir.designate %[[VAL_18]] (%[[VAL_44]])  : (!fir.box<!fir.array<?x!fir.logical<2>>>, i64) -> !fir.ref<!fir.logical<2>>
+// CHECK:               hlfir.assign %[[VAL_13]] to %[[VAL_45]] : !fir.logical<2>, !fir.ref<!fir.logical<2>>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : !hlfir.expr<?x!fir.logical<2>>, !fir.box<!fir.array<?x!fir.logical<2>>>
+// CHECK:           hlfir.destroy %[[VAL_16]] : !hlfir.expr<?x!fir.logical<2>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the scalar always present boundary.
+// subroutine eoshift3(n, array, boundary)
+//   integer :: n
+//   complex(2) :: array(n), boundary
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift3(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!fir.array<?xcomplex<f16>>> {fir.bindc_name = "array"}, %arg2: !fir.ref<complex<f16>> {fir.bindc_name = "boundary"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift3En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = hlfir.declare %arg2 dummy_scope %0 {uniq_name = "_QFeoshift3Eboundary"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+  %3 = fir.load %1#0 : !fir.ref<i32>
+  %4 = fir.convert %3 : (i32) -> index
+  %5 = arith.cmpi sgt, %4, %c0 : index
+  %6 = arith.select %5, %4, %c0 : index
+  %7 = fir.shape %6 : (index) -> !fir.shape<1>
+  %8:2 = hlfir.declare %arg1(%7) dummy_scope %0 {uniq_name = "_QFeoshift3Earray"} : (!fir.ref<!fir.array<?xcomplex<f16>>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xcomplex<f16>>>, !fir.ref<!fir.array<?xcomplex<f16>>>)
+  %9 = hlfir.eoshift %8#0 %c2_i32 boundary %2#0 : (!fir.box<!fir.array<?xcomplex<f16>>>, i32, !fir.ref<complex<f16>>) -> !hlfir.expr<?xcomplex<f16>>
+  hlfir.assign %9 to %8#0 : !hlfir.expr<?xcomplex<f16>>, !fir.box<!fir.array<?xcomplex<f16>>>
+  hlfir.destroy %9 : !hlfir.expr<?xcomplex<f16>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift3(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<!fir.array<?xcomplex<f16>>> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.ref<complex<f16>> {fir.bindc_name = "boundary"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift3En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift3Eboundary"} : (!fir.ref<complex<f16>>, !fir.dscope) -> (!fir.ref<complex<f16>>, !fir.ref<complex<f16>>)
+// CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i32) -> index
+// CHECK:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_8]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_9]], %[[VAL_8]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[ARG1]](%[[VAL_11]]) dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift3Earray"} : (!fir.ref<!fir.array<?xcomplex<f16>>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xcomplex<f16>>>, !fir.ref<!fir.array<?xcomplex<f16>>>)
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_2]] : (i32) -> i64
+// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<complex<f16>>
+// CHECK:           %[[VAL_16:.*]] = hlfir.eval_in_mem shape %[[VAL_11]] : (!fir.shape<1>) -> !hlfir.expr<?xcomplex<f16>> {
+// CHECK:           ^bb0(%[[VAL_17:.*]]: !fir.ref<!fir.array<?xcomplex<f16>>>):
+// CHECK:             %[[VAL_18:.*]] = fir.embox %[[VAL_17]](%[[VAL_11]]) : (!fir.ref<!fir.array<?xcomplex<f16>>>, !fir.shape<1>) -> !fir.box<!fir.array<?xcomplex<f16>>>
+// CHECK:             %[[VAL_19:.*]] = arith.cmpi slt, %[[VAL_14]], %[[VAL_1]] : i64
+// CHECK:             %[[VAL_20:.*]] = arith.subi %[[VAL_1]], %[[VAL_14]] overflow<nsw> : i64
+// CHECK:             %[[VAL_21:.*]] = arith.select %[[VAL_19]], %[[VAL_20]], %[[VAL_1]] : i64
+// CHECK:             %[[VAL_22:.*]] = arith.select %[[VAL_19]], %[[VAL_1]], %[[VAL_14]] : i64
+// CHECK:             %[[VAL_23:.*]] = arith.subi %[[VAL_1]], %[[VAL_13]] overflow<nsw> : i64
+// CHECK:             %[[VAL_24:.*]] = arith.addi %[[VAL_13]], %[[VAL_14]] overflow<nsw> : i64
+// CHECK:             %[[VAL_25:.*]] = arith.cmpi sgt, %[[VAL_23]], %[[VAL_14]] : i64
+// CHECK:             %[[VAL_26:.*]] = arith.select %[[VAL_25]], %[[VAL_1]], %[[VAL_24]] : i64
+// CHECK:             %[[VAL_27:.*]] = arith.subi %[[VAL_13]], %[[VAL_14]] overflow<nsw> : i64
+// CHECK:             %[[VAL_28:.*]] = arith.cmpi slt, %[[VAL_13]], %[[VAL_14]] : i64
+// CHECK:             %[[VAL_29:.*]] = arith.select %[[VAL_28]], %[[VAL_1]], %[[VAL_27]] : i64
+// CHECK:             %[[VAL_30:.*]] = arith.select %[[VAL_19]], %[[VAL_26]], %[[VAL_29]] : i64
+// CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (i64) -> index
+// CHECK:             fir.do_loop %[[VAL_32:.*]] = %[[VAL_0]] to %[[VAL_31]] step %[[VAL_0]] unordered {
+// CHECK:               %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (index) -> i64
+// CHECK:               %[[VAL_34:.*]] = arith.addi %[[VAL_33]], %[[VAL_22]] overflow<nsw> : i64
+// CHECK:               %[[VAL_35:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_34]])  : (!fir.box<!fir.array<?xcomplex<f16>>>, i64) -> !fir.ref<complex<f16>>
+// CHECK:               %[[VAL_36:.*]] = fir.load %[[VAL_35]] : !fir.ref<complex<f16>>
+// CHECK:               %[[VAL_37:.*]] = arith.addi %[[VAL_33]], %[[VAL_21]] overflow<nsw> : i64
+// CHECK:               %[[VAL_38:.*]] = hlfir.designate %[[VAL_18]] (%[[VAL_37]])  : (!fir.box<!fir.array<?xcomplex<f16>>>, i64) -> !fir.ref<complex<f16>>
+// CHECK:               hlfir.assign %[[VAL_36]] to %[[VAL_38]] : complex<f16>, !fir.ref<complex<f16>>
+// CHECK:             }
+// CHECK:             %[[VAL_39:.*]] = arith.subi %[[VAL_13]], %[[VAL_30]] overflow<nsw> : i64
+// CHECK:             %[[VAL_40:.*]] = arith.select %[[VAL_19]], %[[VAL_1]], %[[VAL_30]] : i64
+// CHECK:             %[[VAL_41:.*]] = fir.convert %[[VAL_39]] : (i64) -> index
+// CHECK:             fir.do_loop %[[VAL_42:.*]] = %[[VAL_0]] to %[[VAL_41]] step %[[VAL_0]] unordered {
+// CHECK:               %[[VAL_43:.*]] = fir.convert %[[VAL_42]] : (index) -> i64
+// CHECK:               %[[VAL_44:.*]] = arith.addi %[[VAL_43]], %[[VAL_40]] overflow<nsw> : i64
+// CHECK:               %[[VAL_45:.*]] = hlfir.designate %[[VAL_18]] (%[[VAL_44]])  : (!fir.box<!fir.array<?xcomplex<f16>>>, i64) -> !fir.ref<complex<f16>>
+// CHECK:               hlfir.assign %[[VAL_15]] to %[[VAL_45]] : complex<f16>, !fir.ref<complex<f16>>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_16]] to %[[VAL_12]]#0 : !hlfir.expr<?xcomplex<f16>>, !fir.box<!fir.array<?xcomplex<f16>>>
+// CHECK:           hlfir.destroy %[[VAL_16]] : !hlfir.expr<?xcomplex<f16>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the scalar optional boundary.
+// subroutine eoshift4(n, array, boundary)
+//   integer :: n
+//   logical :: array(n)
+//   logical, optional :: boundary
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift4(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "array"}, %arg2: !fir.ref<!fir.logical<4>> {fir.bindc_name = "boundary", fir.optional}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift4En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = hlfir.declare %arg2 dummy_scope %0 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift4Eboundary"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+  %3 = fir.load %1#0 : !fir.ref<i32>
+  %4 = fir.convert %3 : (i32) -> index
+  %5 = arith.cmpi sgt, %4, %c0 : index
+  %6 = arith.select %5, %4, %c0 : index
+  %7 = fir.shape %6 : (index) -> !fir.shape<1>
+  %8:2 = hlfir.declare %arg1(%7) dummy_scope %0 {uniq_name = "_QFeoshift4Earray"} : (!fir.ref<!fir.array<?x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.ref<!fir.array<?x!fir.logical<4>>>)
+  %9 = fir.is_present %2#0 : (!fir.ref<!fir.logical<4>>) -> i1
+  %10 = fir.embox %2#0 : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+  %11 = fir.absent !fir.box<!fir.logical<4>>
+  %12 = arith.select %9, %10, %11 : !fir.box<!fir.logical<4>>
+  %13 = hlfir.eoshift %8#0 %c2_i32 boundary %12 : (!fir.box<!fir.array<?x!fir.logical<4>>>, i32, !fir.box<!fir.logical<4>>) -> !hlfir.expr<?x!fir.logical<4>>
+  hlfir.assign %13 to %8#0 : !hlfir.expr<?x!fir.logical<4>>, !fir.box<!fir.array<?x!fir.logical<4>>>
+  hlfir.destroy %13 : !hlfir.expr<?x!fir.logical<4>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift4(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.ref<!fir.logical<4>> {fir.bindc_name = "boundary", fir.optional}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_1:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_5]] {uniq_name = "_QFeoshift4En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %[[VAL_5]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift4Eboundary"} : (!fir.ref<!fir.logical<4>>, !fir.dscope) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i32) -> index
+// CHECK:           %[[VAL_10:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_4]] : index
+// CHECK:           %[[VAL_11:.*]] = arith.select %[[VAL_10]], %[[VAL_9]], %[[VAL_4]] : index
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_11]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[ARG1]](%[[VAL_12]]) dummy_scope %[[VAL_5]] {uniq_name = "_QFeoshift4Earray"} : (!fir.ref<!fir.array<?x!fir.logical<4>>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.ref<!fir.array<?x!fir.logical<4>>>)
+// CHECK:           %[[VAL_14:.*]] = fir.is_present %[[VAL_7]]#0 : (!fir.ref<!fir.logical<4>>) -> i1
+// CHECK:           %[[VAL_15:.*]] = fir.embox %[[VAL_7]]#0 : (!fir.ref<!fir.logical<4>>) -> !fir.box<!fir.logical<4>>
+// CHECK:           %[[VAL_16:.*]] = fir.absent !fir.box<!fir.logical<4>>
+// CHECK:           %[[VAL_17:.*]] = arith.select %[[VAL_14]], %[[VAL_15]], %[[VAL_16]] : !fir.box<!fir.logical<4>>
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_11]] : (index) -> i64
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_3]] : (i32) -> i64
+// CHECK:           %[[VAL_20:.*]] = fir.is_present %[[VAL_17]] : (!fir.box<!fir.logical<4>>) -> i1
+// CHECK:           %[[VAL_21:.*]] = fir.if %[[VAL_20]] -> (!fir.logical<4>) {
+// CHECK:             %[[VAL_22:.*]] = fir.box_addr %[[VAL_17]] : (!fir.box<!fir.logical<4>>) -> !fir.ref<!fir.logical<4>>
+// CHECK:             %[[VAL_23:.*]] = fir.load %[[VAL_22]] : !fir.ref<!fir.logical<4>>
+// CHECK:             fir.result %[[VAL_23]] : !fir.logical<4>
+// CHECK:           } else {
+// CHECK:             %[[VAL_24:.*]] = fir.convert %[[VAL_2]] : (i1) -> !fir.logical<4>
+// CHECK:             fir.result %[[VAL_24]] : !fir.logical<4>
+// CHECK:           }
+// CHECK:           %[[VAL_25:.*]] = hlfir.eval_in_mem shape %[[VAL_12]] : (!fir.shape<1>) -> !hlfir.expr<?x!fir.logical<4>> {
+// CHECK:           ^bb0(%[[VAL_26:.*]]: !fir.ref<!fir.array<?x!fir.logical<4>>>):
+// CHECK:             %[[VAL_27:.*]] = fir.embox %[[VAL_26]](%[[VAL_12]]) : (!fir.ref<!fir.array<?x!fir.logical<4>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.logical<4>>>
+// CHECK:             %[[VAL_28:.*]] = arith.cmpi slt, %[[VAL_19]], %[[VAL_1]] : i64
+// CHECK:             %[[VAL_29:.*]] = arith.subi %[[VAL_1]], %[[VAL_19]] overflow<nsw> : i64
+// CHECK:             %[[VAL_30:.*]] = arith.select %[[VAL_28]], %[[VAL_29]], %[[VAL_1]] : i64
+// CHECK:             %[[VAL_31:.*]] = arith.select %[[VAL_28]], %[[VAL_1]], %[[VAL_19]] : i64
+// CHECK:             %[[VAL_32:.*]] = arith.subi %[[VAL_1]], %[[VAL_18]] overflow<nsw> : i64
+// CHECK:             %[[VAL_33:.*]] = arith.addi %[[VAL_18]], %[[VAL_19]] overflow<nsw> : i64
+// CHECK:             %[[VAL_34:.*]] = arith.cmpi sgt, %[[VAL_32]], %[[VAL_19]] : i64
+// CHECK:             %[[VAL_35:.*]] = arith.select %[[VAL_34]], %[[VAL_1]], %[[VAL_33]] : i64
+// CHECK:             %[[VAL_36:.*]] = arith.subi %[[VAL_18]], %[[VAL_19]] overflow<nsw> : i64
+// CHECK:             %[[VAL_37:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_19]] : i64
+// CHECK:             %[[VAL_38:.*]] = arith.select %[[VAL_37]], %[[VAL_1]], %[[VAL_36]] : i64
+// CHECK:             %[[VAL_39:.*]] = arith.select %[[VAL_28]], %[[VAL_35]], %[[VAL_38]] : i64
+// CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_39]] : (i64) -> index
+// CHECK:             fir.do_loop %[[VAL_41:.*]] = %[[VAL_0]] to %[[VAL_40]] step %[[VAL_0]] unordered {
+// CHECK:               %[[VAL_42:.*]] = fir.convert %[[VAL_41]] : (index) -> i64
+// CHECK:               %[[VAL_43:.*]] = arith.addi %[[VAL_42]], %[[VAL_31]] overflow<nsw> : i64
+// CHECK:               %[[VAL_44:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_43]])  : (!fir.box<!fir.array<?x!fir.logical<4>>>, i64) -> !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_45:.*]] = fir.load %[[VAL_44]] : !fir.ref<!fir.logical<4>>
+// CHECK:               %[[VAL_46:.*]] = arith.addi %[[VAL_42]], %[[VAL_30]] overflow<nsw> : i64
+// CHECK:               %[[VAL_47:.*]] = hlfir.designate %[[VAL_27]] (%[[VAL_46]])  : (!fir.box<!fir.array<?x!fir.logical<4>>>, i64) -> !fir.ref<!fir.logical<4>>
+// CHECK:               hlfir.assign %[[VAL_45]] to %[[VAL_47]] : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+// CHECK:             }
+// CHECK:             %[[VAL_48:.*]] = arith.subi %[[VAL_18]], %[[VAL_39]] overflow<nsw> : i64
+// CHECK:             %[[VAL_49:.*]] = arith.select %[[VAL_28]], %[[VAL_1]], %[[VAL_39]] : i64
+// CHECK:             %[[VAL_50:.*]] = fir.convert %[[VAL_48]] : (i64) -> index
+// CHECK:             fir.do_loop %[[VAL_51:.*]] = %[[VAL_0]] to %[[VAL_50]] step %[[VAL_0]] unordered {
+// CHECK:               %[[VAL_52:.*]] = fir.convert %[[VAL_51]] : (index) -> i64
+// CHECK:               %[[VAL_53:.*]] = arith.addi %[[VAL_52]], %[[VAL_49]] overflow<nsw> : i64
+// CHECK:               %[[VAL_54:.*]] = hlfir.designate %[[VAL_27]] (%[[VAL_53]])  : (!fir.box<!fir.array<?x!fir.logical<4>>>, i64) -> !fir.ref<!fir.logical<4>>
+// CHECK:               hlfir.assign %[[VAL_21]] to %[[VAL_54]] : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+// CHECK:             }
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_25]] to %[[VAL_13]]#0 : !hlfir.expr<?x!fir.logical<4>>, !fir.box<!fir.array<?x!fir.logical<4>>>
+// CHECK:           hlfir.destroy %[[VAL_25]] : !hlfir.expr<?x!fir.logical<4>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the array always present boundary.
+// subroutine eoshift5(n, array, boundary)
+//   integer :: n
+//   real :: array(n,n)
+//   real :: boundary(:)
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift5(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "array"}, %arg2: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "boundary"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift5En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = hlfir.declare %arg2 dummy_scope %0 {uniq_name = "_QFeoshift5Eboundary"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+  %3 = fir.load %1#0 : !fir.ref<i32>
+  %4 = fir.convert %3 : (i32) -> index
+  %5 = arith.cmpi sgt, %4, %c0 : index
+  %6 = arith.select %5, %4, %c0 : index
+  %7 = fir.load %1#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> index
+  %9 = arith.cmpi sgt, %8, %c0 : index
+  %10 = arith.select %9, %8, %c0 : index
+  %11 = fir.shape %6, %10 : (index, index) -> !fir.shape<2>
+  %12:2 = hlfir.declare %arg1(%11) dummy_scope %0 {uniq_name = "_QFeoshift5Earray"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+  %13 = hlfir.eoshift %12#0 %c2_i32 boundary %2#0 : (!fir.box<!fir.array<?x?xf32>>, i32, !fir.box<!fir.array<?xf32>>) -> !hlfir.expr<?x?xf32>
+  hlfir.assign %13 to %12#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+  hlfir.destroy %13 : !hlfir.expr<?x?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift5(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.box<!fir.array<?xf32>> {fir.bindc_name = "boundary"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift5En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift5Eboundary"} : (!fir.box<!fir.array<?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.box<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i32) -> index
+// CHECK:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_8]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_9]], %[[VAL_8]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+// CHECK:           %[[VAL_13:.*]] = arith.cmpi sgt, %[[VAL_12]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.select %[[VAL_13]], %[[VAL_12]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_15:.*]] = fir.shape %[[VAL_10]], %[[VAL_14]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[ARG1]](%[[VAL_15]]) dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift5Earray"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_2]] : (i32) -> i64
+// CHECK:           %[[VAL_19:.*]] = hlfir.eval_in_mem shape %[[VAL_15]] : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {
+// CHECK:           ^bb0(%[[VAL_20:.*]]: !fir.ref<!fir.array<?x?xf32>>):
+// CHECK:             %[[VAL_21:.*]] = fir.embox %[[VAL_20]](%[[VAL_15]]) : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+// CHECK:             fir.do_loop %[[VAL_22:.*]] = %[[VAL_1]] to %[[VAL_14]] step %[[VAL_1]] unordered {
+// CHECK:               %[[VAL_23:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_22]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:               %[[VAL_24:.*]] = fir.load %[[VAL_23]] : !fir.ref<f32>
+// CHECK:               %[[VAL_25:.*]] = arith.cmpi slt, %[[VAL_18]], %[[VAL_0]] : i64
+// CHECK:               %[[VAL_26:.*]] = arith.subi %[[VAL_0]], %[[VAL_18]] overflow<nsw> : i64
+// CHECK:               %[[VAL_27:.*]] = arith.select %[[VAL_25]], %[[VAL_26]], %[[VAL_0]] : i64
+// CHECK:               %[[VAL_28:.*]] = arith.select %[[VAL_25]], %[[VAL_0]], %[[VAL_18]] : i64
+// CHECK:               %[[VAL_29:.*]] = arith.subi %[[VAL_0]], %[[VAL_17]] overflow<nsw> : i64
+// CHECK:               %[[VAL_30:.*]] = arith.addi %[[VAL_17]], %[[VAL_18]] overflow<nsw> : i64
+// CHECK:               %[[VAL_31:.*]] = arith.cmpi sgt, %[[VAL_29]], %[[VAL_18]] : i64
+// CHECK:               %[[VAL_32:.*]] = arith.select %[[VAL_31]], %[[VAL_0]], %[[VAL_30]] : i64
+// CHECK:               %[[VAL_33:.*]] = arith.subi %[[VAL_17]], %[[VAL_18]] overflow<nsw> : i64
+// CHECK:               %[[VAL_34:.*]] = arith.cmpi slt, %[[VAL_17]], %[[VAL_18]] : i64
+// CHECK:               %[[VAL_35:.*]] = arith.select %[[VAL_34]], %[[VAL_0]], %[[VAL_33]] : i64
+// CHECK:               %[[VAL_36:.*]] = arith.select %[[VAL_25]], %[[VAL_32]], %[[VAL_35]] : i64
+// CHECK:               %[[VAL_37:.*]] = fir.convert %[[VAL_36]] : (i64) -> index
+// CHECK:               fir.do_loop %[[VAL_38:.*]] = %[[VAL_1]] to %[[VAL_37]] step %[[VAL_1]] unordered {
+// CHECK:                 %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
+// CHECK:                 %[[VAL_40:.*]] = arith.addi %[[VAL_39]], %[[VAL_28]] overflow<nsw> : i64
+// CHECK:                 %[[VAL_41:.*]] = hlfir.designate %[[VAL_16]]#0 (%[[VAL_40]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf32>>, i64, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_42:.*]] = fir.load %[[VAL_41]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_43:.*]] = arith.addi %[[VAL_39]], %[[VAL_27]] overflow<nsw> : i64
+// CHECK:                 %[[VAL_44:.*]] = hlfir.designate %[[VAL_21]] (%[[VAL_43]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf32>>, i64, index) -> !fir.ref<f32>
+// CHECK:                 hlfir.assign %[[VAL_42]] to %[[VAL_44]] : f32, !fir.ref<f32>
+// CHECK:               }
+// CHECK:               %[[VAL_45:.*]] = arith.subi %[[VAL_17]], %[[VAL_36]] overflow<nsw> : i64
+// CHECK:               %[[VAL_46:.*]] = arith.select %[[VAL_25]], %[[VAL_0]], %[[VAL_36]] : i64
+// CHECK:               %[[VAL_47:.*]] = fir.convert %[[VAL_45]] : (i64) -> index
+// CHECK:               fir.do_loop %[[VAL_48:.*]] = %[[VAL_1]] to %[[VAL_47]] step %[[VAL_1]] unordered {
+// CHECK:                 %[[VAL_49:.*]] = fir.convert %[[VAL_48]] : (index) -> i64
+// CHECK:                 %[[VAL_50:.*]] = arith.addi %[[VAL_49]], %[[VAL_46]] overflow<nsw> : i64
+// CHECK:                 %[[VAL_51:.*]] = hlfir.designate %[[VAL_21]] (%[[VAL_50]], %[[VAL_22]])  : (!fir.box<!fir.array<?x?xf32>>, i64, index) -> !fir.ref<f32>
+// CHECK:                 hlfir.assign %[[VAL_24]] to %[[VAL_51]] : f32, !fir.ref<f32>
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_19]] to %[[VAL_16]]#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+// CHECK:           hlfir.destroy %[[VAL_19]] : !hlfir.expr<?x?xf32>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the array optional boundary.
+// subroutine eoshift6(n, array, boundary)
+//   integer :: n
+//   real :: array(n,n)
+//   real, optional :: boundary(n)
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift6(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "array"}, %arg2: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "boundary", fir.optional}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift6En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2 = fir.load %1#0 : !fir.ref<i32>
+  %3 = fir.convert %2 : (i32) -> index
+  %4 = arith.cmpi sgt, %3, %c0 : index
+  %5 = arith.select %4, %3, %c0 : index
+  %6 = fir.load %1#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> index
+  %8 = arith.cmpi sgt, %7, %c0 : index
+  %9 = arith.select %8, %7, %c0 : index
+  %10 = fir.shape %5, %9 : (index, index) -> !fir.shape<2>
+  %11:2 = hlfir.declare %arg1(%10) dummy_scope %0 {uniq_name = "_QFeoshift6Earray"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+  %12 = fir.load %1#0 : !fir.ref<i32>
+  %13 = fir.convert %12 : (i32) -> index
+  %14 = arith.cmpi sgt, %13, %c0 : index
+  %15 = arith.select %14, %13, %c0 : index
+  %16 = fir.shape %15 : (index) -> !fir.shape<1>
+  %17:2 = hlfir.declare %arg2(%16) dummy_scope %0 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift6Eboundary"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+  %18 = fir.is_present %17#0 : (!fir.box<!fir.array<?xf32>>) -> i1
+  %19 = fir.shape %15 : (index) -> !fir.shape<1>
+  %20 = fir.embox %17#1(%19) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+  %21 = fir.absent !fir.box<!fir.array<?xf32>>
+  %22 = arith.select %18, %20, %21 : !fir.box<!fir.array<?xf32>>
+  %23 = hlfir.eoshift %11#0 %c2_i32 boundary %22 : (!fir.box<!fir.array<?x?xf32>>, i32, !fir.box<!fir.array<?xf32>>) -> !hlfir.expr<?x?xf32>
+  hlfir.assign %23 to %11#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+  hlfir.destroy %23 : !hlfir.expr<?x?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift6(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "boundary", fir.optional}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_7]] {uniq_name = "_QFeoshift6En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+// CHECK:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_10]], %[[VAL_6]] : index
+// CHECK:           %[[VAL_12:.*]] = arith.select %[[VAL_11]], %[[VAL_10]], %[[VAL_6]] : index
+// CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i32) -> index
+// CHECK:           %[[VAL_15:.*]] = arith.cmpi sgt, %[[VAL_14]], %[[VAL_6]] : index
+// CHECK:           %[[VAL_16:.*]] = arith.select %[[VAL_15]], %[[VAL_14]], %[[VAL_6]] : index
+// CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_12]], %[[VAL_16]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[ARG1]](%[[VAL_17]]) dummy_scope %[[VAL_7]] {uniq_name = "_QFeoshift6Earray"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+// CHECK:           %[[VAL_19:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (i32) -> index
+// CHECK:           %[[VAL_21:.*]] = arith.cmpi sgt, %[[VAL_20]], %[[VAL_6]] : index
+// CHECK:           %[[VAL_22:.*]] = arith.select %[[VAL_21]], %[[VAL_20]], %[[VAL_6]] : index
+// CHECK:           %[[VAL_23:.*]] = fir.shape %[[VAL_22]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_24:.*]]:2 = hlfir.declare %[[ARG2]](%[[VAL_23]]) dummy_scope %[[VAL_7]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift6Eboundary"} : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?xf32>>, !fir.ref<!fir.array<?xf32>>)
+// CHECK:           %[[VAL_25:.*]] = fir.is_present %[[VAL_24]]#0 : (!fir.box<!fir.array<?xf32>>) -> i1
+// CHECK:           %[[VAL_26:.*]] = fir.shape %[[VAL_22]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_27:.*]] = fir.embox %[[VAL_24]]#1(%[[VAL_26]]) : (!fir.ref<!fir.array<?xf32>>, !fir.shape<1>) -> !fir.box<!fir.array<?xf32>>
+// CHECK:           %[[VAL_28:.*]] = fir.absent !fir.box<!fir.array<?xf32>>
+// CHECK:           %[[VAL_29:.*]] = arith.select %[[VAL_25]], %[[VAL_27]], %[[VAL_28]] : !fir.box<!fir.array<?xf32>>
+// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_12]] : (index) -> i64
+// CHECK:           %[[VAL_31:.*]] = fir.convert %[[VAL_5]] : (i32) -> i64
+// CHECK:           %[[VAL_32:.*]] = fir.is_present %[[VAL_29]] : (!fir.box<!fir.array<?xf32>>) -> i1
+// CHECK:           %[[VAL_33:.*]] = arith.select %[[VAL_32]], %[[VAL_2]], %[[VAL_3]] : i1
+// CHECK:           %[[VAL_34:.*]] = hlfir.eval_in_mem shape %[[VAL_17]] : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {
+// CHECK:           ^bb0(%[[VAL_35:.*]]: !fir.ref<!fir.array<?x?xf32>>):
+// CHECK:             %[[VAL_36:.*]] = fir.embox %[[VAL_35]](%[[VAL_17]]) : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+// CHECK:             fir.do_loop %[[VAL_37:.*]] = %[[VAL_1]] to %[[VAL_16]] step %[[VAL_1]] unordered {
+// CHECK:               %[[VAL_38:.*]] = fir.if %[[VAL_33]] -> (f32) {
+// CHECK:                 fir.result %[[VAL_4]] : f32
+// CHECK:               } else {
+// CHECK:                 %[[VAL_39:.*]]:3 = fir.box_dims %[[VAL_29]], %[[VAL_6]] : (!fir.box<!fir.array<?xf32>>, index) -> (index, index, index)
+// CHECK:                 %[[VAL_40:.*]] = arith.subi %[[VAL_39]]#0, %[[VAL_1]] overflow<nsw> : index
+// CHECK:                 %[[VAL_41:.*]] = arith.addi %[[VAL_37]], %[[VAL_40]] overflow<nsw> : index
+// CHECK:                 %[[VAL_42:.*]] = hlfir.designate %[[VAL_29]] (%[[VAL_41]])  : (!fir.box<!fir.array<?xf32>>, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_43:.*]] = fir.load %[[VAL_42]] : !fir.ref<f32>
+// CHECK:                 fir.result %[[VAL_43]] : f32
+// CHECK:               }
+// CHECK:               %[[VAL_44:.*]] = arith.cmpi slt, %[[VAL_31]], %[[VAL_0]] : i64
+// CHECK:               %[[VAL_45:.*]] = arith.subi %[[VAL_0]], %[[VAL_31]] overflow<nsw> : i64
+// CHECK:               %[[VAL_46:.*]] = arith.select %[[VAL_44]], %[[VAL_45]], %[[VAL_0]] : i64
+// CHECK:               %[[VAL_47:.*]] = arith.select %[[VAL_44]], %[[VAL_0]], %[[VAL_31]] : i64
+// CHECK:               %[[VAL_48:.*]] = arith.subi %[[VAL_0]], %[[VAL_30]] overflow<nsw> : i64
+// CHECK:               %[[VAL_49:.*]] = arith.addi %[[VAL_30]], %[[VAL_31]] overflow<nsw> : i64
+// CHECK:               %[[VAL_50:.*]] = arith.cmpi sgt, %[[VAL_48]], %[[VAL_31]] : i64
+// CHECK:               %[[VAL_51:.*]] = arith.select %[[VAL_50]], %[[VAL_0]], %[[VAL_49]] : i64
+// CHECK:               %[[VAL_52:.*]] = arith.subi %[[VAL_30]], %[[VAL_31]] overflow<nsw> : i64
+// CHECK:               %[[VAL_53:.*]] = arith.cmpi slt, %[[VAL_30]], %[[VAL_31]] : i64
+// CHECK:               %[[VAL_54:.*]] = arith.select %[[VAL_53]], %[[VAL_0]], %[[VAL_52]] : i64
+// CHECK:               %[[VAL_55:.*]] = arith.select %[[VAL_44]], %[[VAL_51]], %[[VAL_54]] : i64
+// CHECK:               %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (i64) -> index
+// CHECK:               fir.do_loop %[[VAL_57:.*]] = %[[VAL_1]] to %[[VAL_56]] step %[[VAL_1]] unordered {
+// CHECK:                 %[[VAL_58:.*]] = fir.convert %[[VAL_57]] : (index) -> i64
+// CHECK:                 %[[VAL_59:.*]] = arith.addi %[[VAL_58]], %[[VAL_47]] overflow<nsw> : i64
+// CHECK:                 %[[VAL_60:.*]] = hlfir.designate %[[VAL_18]]#0 (%[[VAL_59]], %[[VAL_37]])  : (!fir.box<!fir.array<?x?xf32>>, i64, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_61:.*]] = fir.load %[[VAL_60]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_62:.*]] = arith.addi %[[VAL_58]], %[[VAL_46]] overflow<nsw> : i64
+// CHECK:                 %[[VAL_63:.*]] = hlfir.designate %[[VAL_36]] (%[[VAL_62]], %[[VAL_37]])  : (!fir.box<!fir.array<?x?xf32>>, i64, index) -> !fir.ref<f32>
+// CHECK:                 hlfir.assign %[[VAL_61]] to %[[VAL_63]] : f32, !fir.ref<f32>
+// CHECK:               }
+// CHECK:               %[[VAL_64:.*]] = arith.subi %[[VAL_30]], %[[VAL_55]] overflow<nsw> : i64
+// CHECK:               %[[VAL_65:.*]] = arith.select %[[VAL_44]], %[[VAL_0]], %[[VAL_55]] : i64
+// CHECK:               %[[VAL_66:.*]] = fir.convert %[[VAL_64]] : (i64) -> index
+// CHECK:               fir.do_loop %[[VAL_67:.*]] = %[[VAL_1]] to %[[VAL_66]] step %[[VAL_1]] unordered {
+// CHECK:                 %[[VAL_68:.*]] = fir.convert %[[VAL_67]] : (index) -> i64
+// CHECK:                 %[[VAL_69:.*]] = arith.addi %[[VAL_68]], %[[VAL_65]] overflow<nsw> : i64
+// CHECK:                 %[[VAL_70:.*]] = hlfir.designate %[[VAL_36]] (%[[VAL_69]], %[[VAL_37]])  : (!fir.box<!fir.array<?x?xf32>>, i64, index) -> !fir.ref<f32>
+// CHECK:                 hlfir.assign %[[VAL_38]] to %[[VAL_70]] : f32, !fir.ref<f32>
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_34]] to %[[VAL_18]]#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+// CHECK:           hlfir.destroy %[[VAL_34]] : !hlfir.expr<?x?xf32>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the array expression boundary.
+// subroutine eoshift7(n, array)
+//   interface
+//      function real_boundary(n)
+//        integer :: n
+//        real :: real_boundary(n)
+//      end function
+//   end interface
+//   integer :: n
+//   real :: array(n,n)
+//   array = EOSHIFT(array, 2, real_boundary(n))
+// end subroutine
+func.func @_QPeoshift7(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "array"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift7En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2 = fir.load %1#0 : !fir.ref<i32>
+  %3 = fir.convert %2 : (i32) -> index
+  %4 = arith.cmpi sgt, %3, %c0 : index
+  %5 = arith.select %4, %3, %c0 : index
+  %6 = fir.load %1#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> index
+  %8 = arith.cmpi sgt, %7, %c0 : index
+  %9 = arith.select %8, %7, %c0 : index
+  %10 = fir.shape %5, %9 : (index, index) -> !fir.shape<2>
+  %11:2 = hlfir.declare %arg1(%10) dummy_scope %0 {uniq_name = "_QFeoshift7Earray"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+  %12:2 = hlfir.declare %1#0 {uniq_name = "_QFeoshift7Freal_boundaryEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %13 = fir.load %12#0 : !fir.ref<i32>
+  %14 = fir.convert %13 : (i32) -> index
+  %15 = arith.cmpi sgt, %14, %c0 : index
+  %16 = arith.select %15, %14, %c0 : index
+  %17 = fir.shape %16 : (index) -> !fir.shape<1>
+  %18 = hlfir.eval_in_mem shape %17 : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+  ^bb0(%arg2: !fir.ref<!fir.array<?xf32>>):
+    %20 = fir.call @_QPreal_boundary(%1#0) fastmath<contract> : (!fir.ref<i32>) -> !fir.array<?xf32>
+    fir.save_result %20 to %arg2(%17) : !fir.array<?xf32>, !fir.ref<!fir.array<?xf32>>, !fir.shape<1>
+  }
+  %19 = hlfir.eoshift %11#0 %c2_i32 boundary %18 : (!fir.box<!fir.array<?x?xf32>>, i32, !hlfir.expr<?xf32>) -> !hlfir.expr<?x?xf32>
+  hlfir.assign %19 to %11#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+  hlfir.destroy %19 : !hlfir.expr<?x?xf32>
+  hlfir.destroy %18 : !hlfir.expr<?xf32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift7(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<!fir.array<?x?xf32>> {fir.bindc_name = "array"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 0 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift7En"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]] : (i32) -> index
+// CHECK:           %[[VAL_8:.*]] = arith.cmpi sgt, %[[VAL_7]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_9:.*]] = arith.select %[[VAL_8]], %[[VAL_7]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_10:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (i32) -> index
+// CHECK:           %[[VAL_12:.*]] = arith.cmpi sgt, %[[VAL_11]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_13:.*]] = arith.select %[[VAL_12]], %[[VAL_11]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_9]], %[[VAL_13]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[ARG1]](%[[VAL_14]]) dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift7Earray"} : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.ref<!fir.array<?x?xf32>>)
+// CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_5]]#0 {uniq_name = "_QFeoshift7Freal_boundaryEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_17:.*]] = fir.load %[[VAL_16]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_17]] : (i32) -> index
+// CHECK:           %[[VAL_19:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_20:.*]] = arith.select %[[VAL_19]], %[[VAL_18]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_21:.*]] = fir.shape %[[VAL_20]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_22:.*]] = hlfir.eval_in_mem shape %[[VAL_21]] : (!fir.shape<1>) -> !hlfir.expr<?xf32> {
+// CHECK:           ^bb0(%[[VAL_23:.*]]: !fir.ref<!fir.array<?xf32>>):
+// CHECK:             %[[VAL_24:.*]] = fir.call @_QPreal_boundary(%[[VAL_5]]#0) fastmath<contract> : (!fir.ref<i32>) -> !fir.array<?xf32>
+// CHECK:             fir.save_result %[[VAL_24]] to %[[VAL_23]](%[[VAL_21]]) : !fir.array<?xf32>, !fir.ref<!fir.array<?xf32>>, !fir.shape<1>
+// CHECK:           }
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_9]] : (index) -> i64
+// CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_2]] : (i32) -> i64
+// CHECK:           %[[VAL_27:.*]] = hlfir.eval_in_mem shape %[[VAL_14]] : (!fir.shape<2>) -> !hlfir.expr<?x?xf32> {
+// CHECK:           ^bb0(%[[VAL_28:.*]]: !fir.ref<!fir.array<?x?xf32>>):
+// CHECK:             %[[VAL_29:.*]] = fir.embox %[[VAL_28]](%[[VAL_14]]) : (!fir.ref<!fir.array<?x?xf32>>, !fir.shape<2>) -> !fir.box<!fir.array<?x?xf32>>
+// CHECK:             fir.do_loop %[[VAL_30:.*]] = %[[VAL_1]] to %[[VAL_13]] step %[[VAL_1]] unordered {
+// CHECK:               %[[VAL_31:.*]] = hlfir.apply %[[VAL_22]], %[[VAL_30]] : (!hlfir.expr<?xf32>, index) -> f32
+// CHECK:               %[[VAL_32:.*]] = arith.cmpi slt, %[[VAL_26]], %[[VAL_0]] : i64
+// CHECK:               %[[VAL_33:.*]] = arith.subi %[[VAL_0]], %[[VAL_26]] overflow<nsw> : i64
+// CHECK:               %[[VAL_34:.*]] = arith.select %[[VAL_32]], %[[VAL_33]], %[[VAL_0]] : i64
+// CHECK:               %[[VAL_35:.*]] = arith.select %[[VAL_32]], %[[VAL_0]], %[[VAL_26]] : i64
+// CHECK:               %[[VAL_36:.*]] = arith.subi %[[VAL_0]], %[[VAL_25]] overflow<nsw> : i64
+// CHECK:               %[[VAL_37:.*]] = arith.addi %[[VAL_25]], %[[VAL_26]] overflow<nsw> : i64
+// CHECK:               %[[VAL_38:.*]] = arith.cmpi sgt, %[[VAL_36]], %[[VAL_26]] : i64
+// CHECK:               %[[VAL_39:.*]] = arith.select %[[VAL_38]], %[[VAL_0]], %[[VAL_37]] : i64
+// CHECK:               %[[VAL_40:.*]] = arith.subi %[[VAL_25]], %[[VAL_26]] overflow<nsw> : i64
+// CHECK:               %[[VAL_41:.*]] = arith.cmpi slt, %[[VAL_25]], %[[VAL_26]] : i64
+// CHECK:               %[[VAL_42:.*]] = arith.select %[[VAL_41]], %[[VAL_0]], %[[VAL_40]] : i64
+// CHECK:               %[[VAL_43:.*]] = arith.select %[[VAL_32]], %[[VAL_39]], %[[VAL_42]] : i64
+// CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_43]] : (i64) -> index
+// CHECK:               fir.do_loop %[[VAL_45:.*]] = %[[VAL_1]] to %[[VAL_44]] step %[[VAL_1]] unordered {
+// CHECK:                 %[[VAL_46:.*]] = fir.convert %[[VAL_45]] : (index) -> i64
+// CHECK:                 %[[VAL_47:.*]] = arith.addi %[[VAL_46]], %[[VAL_35]] overflow<nsw> : i64
+// CHECK:                 %[[VAL_48:.*]] = hlfir.designate %[[VAL_15]]#0 (%[[VAL_47]], %[[VAL_30]])  : (!fir.box<!fir.array<?x?xf32>>, i64, index) -> !fir.ref<f32>
+// CHECK:                 %[[VAL_49:.*]] = fir.load %[[VAL_48]] : !fir.ref<f32>
+// CHECK:                 %[[VAL_50:.*]] = arith.addi %[[VAL_46]], %[[VAL_34]] overflow<nsw> : i64
+// CHECK:                 %[[VAL_51:.*]] = hlfir.designate %[[VAL_29]] (%[[VAL_50]], %[[VAL_30]])  : (!fir.box<!fir.array<?x?xf32>>, i64, index) -> !fir.ref<f32>
+// CHECK:                 hlfir.assign %[[VAL_49]] to %[[VAL_51]] : f32, !fir.ref<f32>
+// CHECK:               }
+// CHECK:               %[[VAL_52:.*]] = arith.subi %[[VAL_25]], %[[VAL_43]] overflow<nsw> : i64
+// CHECK:               %[[VAL_53:.*]] = arith.select %[[VAL_32]], %[[VAL_0]], %[[VAL_43]] : i64
+// CHECK:               %[[VAL_54:.*]] = fir.convert %[[VAL_52]] : (i64) -> index
+// CHECK:               fir.do_loop %[[VAL_55:.*]] = %[[VAL_1]] to %[[VAL_54]] step %[[VAL_1]] unordered {
+// CHECK:                 %[[VAL_56:.*]] = fir.convert %[[VAL_55]] : (index) -> i64
+// CHECK:                 %[[VAL_57:.*]] = arith.addi %[[VAL_56]], %[[VAL_53]] overflow<nsw> : i64
+// CHECK:                 %[[VAL_58:.*]] = hlfir.designate %[[VAL_29]] (%[[VAL_57]], %[[VAL_30]])  : (!fir.box<!fir.array<?x?xf32>>, i64, index) -> !fir.ref<f32>
+// CHECK:                 hlfir.assign %[[VAL_31]] to %[[VAL_58]] : f32, !fir.ref<f32>
+// CHECK:               }
+// CHECK:             }
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_15]]#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+// CHECK:           hlfir.destroy %[[VAL_27]] : !hlfir.expr<?x?xf32>
+// CHECK:           hlfir.destroy %[[VAL_22]] : !hlfir.expr<?xf32>
+// CHECK:           return
+// CHECK:         }
+
+// Test UNSIGNED data type.
+// The default value of the BOUNDARY must be an integer 0
+// converted to ui32 type.
+// subroutine eoshift8(array)
+//   unsigned :: array(:,:)
+//   array = EOSHIFT(array, shift=1, dim=2)
+// end subroutine
+func.func @_QPeoshift8(%arg0: !fir.box<!fir.array<?x?xui32>> {fir.bindc_name = "array"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c1_i32 = arith.constant 1 : i32
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift8Earray"} : (!fir.box<!fir.array<?x?xui32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xui32>>, !fir.box<!fir.array<?x?xui32>>)
+  %2 = hlfir.eoshift %1#0 %c1_i32 dim %c2_i32 : (!fir.box<!fir.array<?x?xui32>>, i32, i32) -> !hlfir.expr<?x?xui32>
+  hlfir.assign %2 to %1#0 : !hlfir.expr<?x?xui32>, !fir.box<!fir.array<?x?xui32>>
+  hlfir.destroy %2 : !hlfir.expr<?x?xui32>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift8(
+// CHECK-DAG:           hlfir.elemental %{{.*}} unordered : (!fir.shape<2>) -> !hlfir.expr<?x?xui32> {
+// CHECK-DAG:               %[[VAL_24:.*]] = fir.load %{{.*}} : !fir.ref<ui32>
+// CHECK-DAG:               fir.result %[[VAL_24]] : ui32
+// CHECK-DAG:             } else {
+// CHECK-DAG:               fir.result %[[VAL_12:.*]] : ui32
+// CHECK-DAG:             }
+// CHECK-DAG:           %[[VAL_12]] = fir.convert %[[VAL_1:.*]] : (i32) -> ui32
+// CHECK-DAG:           %[[VAL_1]] = arith.constant 0 : i32
+
+// ! Tests for CHARACTER type (lowered via hlfir.elemental).
+
+// ! Test contiguous 1D array with statically absent boundary.
+// ! CHARACTER with constant length.
+// subroutine eoshift1c(n, array)
+//   integer :: n
+//   character(10,1) :: array(n)
+//   array = EOSHIFT(array, 2)
+// end subroutine
+func.func @_QPeoshift1c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift1cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,10>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.shape %7 : (index) -> !fir.shape<1>
+  %9:2 = hlfir.declare %3(%8) typeparams %c10 dummy_scope %0 {uniq_name = "_QFeoshift1cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x!fir.char<1,10>>>)
+  %10 = hlfir.eoshift %9#0 %c2_i32 : (!fir.box<!fir.array<?x!fir.char<1,10>>>, i32) -> !hlfir.expr<?x!fir.char<1,10>>
+  hlfir.assign %10 to %9#0 : !hlfir.expr<?x!fir.char<1,10>>, !fir.box<!fir.array<?x!fir.char<1,10>>>
+  hlfir.destroy %10 : !hlfir.expr<?x!fir.char<1,10>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift1c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift1cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i32) -> index
+// CHECK:           %[[VAL_10:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_11:.*]] = arith.select %[[VAL_10]], %[[VAL_9]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_11]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_12]]) typeparams %[[VAL_3]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift1cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x!fir.char<1,10>>>)
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_11]] : (index) -> i64
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_16:.*]] = fir.alloca !fir.char<1,0> {bindc_name = ".chrtmp"}
+// CHECK:           %[[VAL_17:.*]] = fir.emboxchar %[[VAL_16]], %[[VAL_2]] : (!fir.ref<!fir.char<1,0>>, index) -> !fir.boxchar<1>
+// CHECK:           %[[VAL_18:.*]] = hlfir.elemental %[[VAL_12]] typeparams %[[VAL_3]] unordered : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<1,10>> {
+// CHECK:           ^bb0(%[[VAL_19:.*]]: index):
+// CHECK:             %[[VAL_20:.*]] = fir.convert %[[VAL_19]] : (index) -> i64
+// CHECK:             %[[VAL_21:.*]] = arith.addi %[[VAL_20]], %[[VAL_15]] overflow<nsw> : i64
+// CHECK:             %[[VAL_22:.*]] = arith.cmpi sge, %[[VAL_21]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_23:.*]] = arith.cmpi sle, %[[VAL_21]], %[[VAL_14]] : i64
+// CHECK:             %[[VAL_24:.*]] = arith.andi %[[VAL_22]], %[[VAL_23]] : i1
+// CHECK:             %[[VAL_25:.*]] = fir.if %[[VAL_24]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_26:.*]] = fir.convert %[[VAL_21]] : (i64) -> index
+// CHECK:               %[[VAL_27:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_26]])  typeparams %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.char<1,10>>>, index, index) -> !fir.ref<!fir.char<1,10>>
+// CHECK:               %[[VAL_28:.*]] = fir.emboxchar %[[VAL_27]], %[[VAL_3]] : (!fir.ref<!fir.char<1,10>>, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_28]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_17]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_25]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_18]] to %[[VAL_13]]#0 : !hlfir.expr<?x!fir.char<1,10>>, !fir.box<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           hlfir.destroy %[[VAL_18]] : !hlfir.expr<?x!fir.char<1,10>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with statically absent boundary.
+// ! CHARACTER with variable length.
+// subroutine eoshift2c(n, array)
+//   integer :: n
+//   character(n,1) :: array(n)
+//   array = EOSHIFT(array, 2)
+// end subroutine
+func.func @_QPeoshift2c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift2cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = arith.cmpi sgt, %4, %c0_i32 : i32
+  %6 = arith.select %5, %4, %c0_i32 : i32
+  %7 = fir.load %1#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> index
+  %9 = arith.cmpi sgt, %8, %c0 : index
+  %10 = arith.select %9, %8, %c0 : index
+  %11 = fir.shape %10 : (index) -> !fir.shape<1>
+  %12:2 = hlfir.declare %3(%11) typeparams %6 dummy_scope %0 {uniq_name = "_QFeoshift2cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+  %13 = hlfir.eoshift %12#0 %c2_i32 : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i32) -> !hlfir.expr<?x!fir.char<1,?>>
+  hlfir.assign %13 to %12#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+  hlfir.destroy %13 : !hlfir.expr<?x!fir.char<1,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift2c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift2cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_8]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_9]], %[[VAL_8]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+// CHECK:           %[[VAL_13:.*]] = arith.cmpi sgt, %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.select %[[VAL_13]], %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_15:.*]] = fir.shape %[[VAL_14]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_15]]) typeparams %[[VAL_10]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift2cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_14]] : (index) -> i64
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_19:.*]] = fir.alloca !fir.char<1,0> {bindc_name = ".chrtmp"}
+// CHECK:           %[[VAL_20:.*]] = fir.emboxchar %[[VAL_19]], %[[VAL_2]] : (!fir.ref<!fir.char<1,0>>, index) -> !fir.boxchar<1>
+// CHECK:           %[[VAL_21:.*]] = hlfir.elemental %[[VAL_15]] typeparams %[[VAL_10]] unordered : (!fir.shape<1>, i32) -> !hlfir.expr<?x!fir.char<1,?>> {
+// CHECK:           ^bb0(%[[VAL_22:.*]]: index):
+// CHECK:             %[[VAL_23:.*]] = fir.convert %[[VAL_22]] : (index) -> i64
+// CHECK:             %[[VAL_24:.*]] = arith.addi %[[VAL_23]], %[[VAL_18]] overflow<nsw> : i64
+// CHECK:             %[[VAL_25:.*]] = arith.cmpi sge, %[[VAL_24]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_26:.*]] = arith.cmpi sle, %[[VAL_24]], %[[VAL_17]] : i64
+// CHECK:             %[[VAL_27:.*]] = arith.andi %[[VAL_25]], %[[VAL_26]] : i1
+// CHECK:             %[[VAL_28:.*]] = fir.if %[[VAL_27]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_29:.*]] = fir.convert %[[VAL_24]] : (i64) -> index
+// CHECK:               %[[VAL_30:.*]] = hlfir.designate %[[VAL_16]]#0 (%[[VAL_29]])  typeparams %[[VAL_10]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index, i32) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_30]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_20]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_28]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_21]] to %[[VAL_16]]#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           hlfir.destroy %[[VAL_21]] : !hlfir.expr<?x!fir.char<1,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with statically absent boundary.
+// ! CHARACTER with assumed length.
+// subroutine eoshift3c(n, array)
+//   integer :: n
+//   character(*,1) :: array(n)
+//   array = EOSHIFT(array, 2)
+// end subroutine
+func.func @_QPeoshift3c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift3cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.shape %7 : (index) -> !fir.shape<1>
+  %9:2 = hlfir.declare %3(%8) typeparams %2#1 dummy_scope %0 {uniq_name = "_QFeoshift3cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+  %10 = hlfir.eoshift %9#0 %c2_i32 : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i32) -> !hlfir.expr<?x!fir.char<1,?>>
+  hlfir.assign %10 to %9#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+  hlfir.destroy %10 : !hlfir.expr<?x!fir.char<1,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift3c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFeoshift3cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_5:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_6:.*]] = fir.convert %[[VAL_5]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_7:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]] : (i32) -> index
+// CHECK:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_8]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_9]], %[[VAL_8]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_11:.*]] = fir.shape %[[VAL_10]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_12:.*]]:2 = hlfir.declare %[[VAL_6]](%[[VAL_11]]) typeparams %[[VAL_5]]#1 dummy_scope %[[VAL_3]] {uniq_name = "_QFeoshift3cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_10]] : (index) -> i64
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_15:.*]] = fir.alloca !fir.char<1,0> {bindc_name = ".chrtmp"}
+// CHECK:           %[[VAL_16:.*]] = fir.emboxchar %[[VAL_15]], %[[VAL_2]] : (!fir.ref<!fir.char<1,0>>, index) -> !fir.boxchar<1>
+// CHECK:           %[[VAL_17:.*]] = hlfir.elemental %[[VAL_11]] typeparams %[[VAL_5]]#1 unordered : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<1,?>> {
+// CHECK:           ^bb0(%[[VAL_18:.*]]: index):
+// CHECK:             %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (index) -> i64
+// CHECK:             %[[VAL_20:.*]] = arith.addi %[[VAL_19]], %[[VAL_14]] overflow<nsw> : i64
+// CHECK:             %[[VAL_21:.*]] = arith.cmpi sge, %[[VAL_20]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_22:.*]] = arith.cmpi sle, %[[VAL_20]], %[[VAL_13]] : i64
+// CHECK:             %[[VAL_23:.*]] = arith.andi %[[VAL_21]], %[[VAL_22]] : i1
+// CHECK:             %[[VAL_24:.*]] = fir.if %[[VAL_23]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_20]] : (i64) -> index
+// CHECK:               %[[VAL_26:.*]] = hlfir.designate %[[VAL_12]]#0 (%[[VAL_25]])  typeparams %[[VAL_5]]#1 : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_26]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_16]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_24]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_12]]#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           hlfir.destroy %[[VAL_17]] : !hlfir.expr<?x!fir.char<1,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with scalar constant boundary.
+// subroutine eoshift4c(n, array)
+//   integer :: n
+//   character(10,1) :: array(n)
+//   array = EOSHIFT(array, 2, '0123456789')
+// end subroutine
+func.func @_QPeoshift4c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift4cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,10>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.shape %7 : (index) -> !fir.shape<1>
+  %9:2 = hlfir.declare %3(%8) typeparams %c10 dummy_scope %0 {uniq_name = "_QFeoshift4cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x!fir.char<1,10>>>)
+  %10 = fir.address_of(@_QQclX30313233343536373839) : !fir.ref<!fir.char<1,10>>
+  %11:2 = hlfir.declare %10 typeparams %c10 {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQclX30313233343536373839"} : (!fir.ref<!fir.char<1,10>>, index) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
+  %12 = hlfir.eoshift %9#0 %c2_i32 boundary %11#0 : (!fir.box<!fir.array<?x!fir.char<1,10>>>, i32, !fir.ref<!fir.char<1,10>>) -> !hlfir.expr<?x!fir.char<1,10>>
+  hlfir.assign %12 to %9#0 : !hlfir.expr<?x!fir.char<1,10>>, !fir.box<!fir.array<?x!fir.char<1,10>>>
+  hlfir.destroy %12 : !hlfir.expr<?x!fir.char<1,10>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift4c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift4cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]] : (i32) -> index
+// CHECK:           %[[VAL_10:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_11:.*]] = arith.select %[[VAL_10]], %[[VAL_9]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_12:.*]] = fir.shape %[[VAL_11]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_12]]) typeparams %[[VAL_3]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift4cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x!fir.char<1,10>>>)
+// CHECK:           %[[VAL_14:.*]] = fir.address_of(@_QQclX30313233343536373839) : !fir.ref<!fir.char<1,10>>
+// CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_14]] typeparams %[[VAL_3]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQclX30313233343536373839"} : (!fir.ref<!fir.char<1,10>>, index) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_11]] : (index) -> i64
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_18:.*]] = fir.emboxchar %[[VAL_15]]#0, %[[VAL_3]] : (!fir.ref<!fir.char<1,10>>, index) -> !fir.boxchar<1>
+// CHECK:           %[[VAL_19:.*]] = hlfir.elemental %[[VAL_12]] typeparams %[[VAL_3]] unordered : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<1,10>> {
+// CHECK:           ^bb0(%[[VAL_20:.*]]: index):
+// CHECK:             %[[VAL_21:.*]] = fir.convert %[[VAL_20]] : (index) -> i64
+// CHECK:             %[[VAL_22:.*]] = arith.addi %[[VAL_21]], %[[VAL_17]] overflow<nsw> : i64
+// CHECK:             %[[VAL_23:.*]] = arith.cmpi sge, %[[VAL_22]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_24:.*]] = arith.cmpi sle, %[[VAL_22]], %[[VAL_16]] : i64
+// CHECK:             %[[VAL_25:.*]] = arith.andi %[[VAL_23]], %[[VAL_24]] : i1
+// CHECK:             %[[VAL_26:.*]] = fir.if %[[VAL_25]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_27:.*]] = fir.convert %[[VAL_22]] : (i64) -> index
+// CHECK:               %[[VAL_28:.*]] = hlfir.designate %[[VAL_13]]#0 (%[[VAL_27]])  typeparams %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.char<1,10>>>, index, index) -> !fir.ref<!fir.char<1,10>>
+// CHECK:               %[[VAL_29:.*]] = fir.emboxchar %[[VAL_28]], %[[VAL_3]] : (!fir.ref<!fir.char<1,10>>, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_29]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_18]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_26]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_19]] to %[[VAL_13]]#0 : !hlfir.expr<?x!fir.char<1,10>>, !fir.box<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           hlfir.destroy %[[VAL_19]] : !hlfir.expr<?x!fir.char<1,10>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with scalar always present boundary.
+// ! CHARACTER with constant length.
+// subroutine eoshift5c(n, array, boundary)
+//   integer :: n
+//   character(10,1) :: array(n), boundary
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift5c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}, %arg2: !fir.boxchar<1> {fir.bindc_name = "boundary"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift5cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,10>>
+  %4:2 = hlfir.declare %3 typeparams %c10 dummy_scope %0 {uniq_name = "_QFeoshift5cEboundary"} : (!fir.ref<!fir.char<1,10>>, index, !fir.dscope) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
+  %5:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %6 = fir.convert %5#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,10>>>
+  %7 = fir.load %1#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> index
+  %9 = arith.cmpi sgt, %8, %c0 : index
+  %10 = arith.select %9, %8, %c0 : index
+  %11 = fir.shape %10 : (index) -> !fir.shape<1>
+  %12:2 = hlfir.declare %6(%11) typeparams %c10 dummy_scope %0 {uniq_name = "_QFeoshift5cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x!fir.char<1,10>>>)
+  %13 = hlfir.eoshift %12#0 %c2_i32 boundary %4#0 : (!fir.box<!fir.array<?x!fir.char<1,10>>>, i32, !fir.ref<!fir.char<1,10>>) -> !hlfir.expr<?x!fir.char<1,10>>
+  hlfir.assign %13 to %12#0 : !hlfir.expr<?x!fir.char<1,10>>, !fir.box<!fir.array<?x!fir.char<1,10>>>
+  hlfir.destroy %13 : !hlfir.expr<?x!fir.char<1,10>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift5c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.boxchar<1> {fir.bindc_name = "boundary"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift5cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = fir.unboxchar %[[ARG2]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1,10>>
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] typeparams %[[VAL_3]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift5cEboundary"} : (!fir.ref<!fir.char<1,10>>, index, !fir.dscope) -> (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>)
+// CHECK:           %[[VAL_9:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+// CHECK:           %[[VAL_13:.*]] = arith.cmpi sgt, %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.select %[[VAL_13]], %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_15:.*]] = fir.shape %[[VAL_14]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_10]](%[[VAL_15]]) typeparams %[[VAL_3]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift5cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x!fir.char<1,10>>>)
+// CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_14]] : (index) -> i64
+// CHECK:           %[[VAL_18:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_19:.*]] = fir.emboxchar %[[VAL_8]]#0, %[[VAL_3]] : (!fir.ref<!fir.char<1,10>>, index) -> !fir.boxchar<1>
+// CHECK:           %[[VAL_20:.*]] = hlfir.elemental %[[VAL_15]] typeparams %[[VAL_3]] unordered : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<1,10>> {
+// CHECK:           ^bb0(%[[VAL_21:.*]]: index):
+// CHECK:             %[[VAL_22:.*]] = fir.convert %[[VAL_21]] : (index) -> i64
+// CHECK:             %[[VAL_23:.*]] = arith.addi %[[VAL_22]], %[[VAL_18]] overflow<nsw> : i64
+// CHECK:             %[[VAL_24:.*]] = arith.cmpi sge, %[[VAL_23]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_25:.*]] = arith.cmpi sle, %[[VAL_23]], %[[VAL_17]] : i64
+// CHECK:             %[[VAL_26:.*]] = arith.andi %[[VAL_24]], %[[VAL_25]] : i1
+// CHECK:             %[[VAL_27:.*]] = fir.if %[[VAL_26]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_28:.*]] = fir.convert %[[VAL_23]] : (i64) -> index
+// CHECK:               %[[VAL_29:.*]] = hlfir.designate %[[VAL_16]]#0 (%[[VAL_28]])  typeparams %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.char<1,10>>>, index, index) -> !fir.ref<!fir.char<1,10>>
+// CHECK:               %[[VAL_30:.*]] = fir.emboxchar %[[VAL_29]], %[[VAL_3]] : (!fir.ref<!fir.char<1,10>>, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_30]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_19]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_27]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_20]] to %[[VAL_16]]#0 : !hlfir.expr<?x!fir.char<1,10>>, !fir.box<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           hlfir.destroy %[[VAL_20]] : !hlfir.expr<?x!fir.char<1,10>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with scalar always present boundary.
+// ! CHARACTER with variable length.
+// subroutine eoshift6c(n, array, boundary)
+//   integer :: n
+//   character(n,1) :: array(n), boundary
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift6c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}, %arg2: !fir.boxchar<1> {fir.bindc_name = "boundary"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift6cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = arith.cmpi sgt, %4, %c0_i32 : i32
+  %6 = arith.select %5, %4, %c0_i32 : i32
+  %7 = fir.load %1#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> index
+  %9 = arith.cmpi sgt, %8, %c0 : index
+  %10 = arith.select %9, %8, %c0 : index
+  %11 = fir.shape %10 : (index) -> !fir.shape<1>
+  %12:2 = hlfir.declare %3(%11) typeparams %6 dummy_scope %0 {uniq_name = "_QFeoshift6cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+  %13:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %14 = fir.load %1#0 : !fir.ref<i32>
+  %15 = arith.cmpi sgt, %14, %c0_i32 : i32
+  %16 = arith.select %15, %14, %c0_i32 : i32
+  %17:2 = hlfir.declare %13#0 typeparams %16 dummy_scope %0 {uniq_name = "_QFeoshift6cEboundary"} : (!fir.ref<!fir.char<1,?>>, i32, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+  %18 = hlfir.eoshift %12#0 %c2_i32 boundary %17#0 : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i32, !fir.boxchar<1>) -> !hlfir.expr<?x!fir.char<1,?>>
+  hlfir.assign %18 to %12#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+  hlfir.destroy %18 : !hlfir.expr<?x!fir.char<1,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift6c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.boxchar<1> {fir.bindc_name = "boundary"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift6cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_8]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_9]], %[[VAL_8]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+// CHECK:           %[[VAL_13:.*]] = arith.cmpi sgt, %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.select %[[VAL_13]], %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_15:.*]] = fir.shape %[[VAL_14]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_15]]) typeparams %[[VAL_10]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift6cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+// CHECK:           %[[VAL_17:.*]]:2 = fir.unboxchar %[[ARG2]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_18:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_19:.*]] = arith.cmpi sgt, %[[VAL_18]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_20:.*]] = arith.select %[[VAL_19]], %[[VAL_18]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_21:.*]]:2 = hlfir.declare %[[VAL_17]]#0 typeparams %[[VAL_20]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift6cEboundary"} : (!fir.ref<!fir.char<1,?>>, i32, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_14]] : (index) -> i64
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_24:.*]] = hlfir.elemental %[[VAL_15]] typeparams %[[VAL_10]] unordered : (!fir.shape<1>, i32) -> !hlfir.expr<?x!fir.char<1,?>> {
+// CHECK:           ^bb0(%[[VAL_25:.*]]: index):
+// CHECK:             %[[VAL_26:.*]] = fir.convert %[[VAL_25]] : (index) -> i64
+// CHECK:             %[[VAL_27:.*]] = arith.addi %[[VAL_26]], %[[VAL_23]] overflow<nsw> : i64
+// CHECK:             %[[VAL_28:.*]] = arith.cmpi sge, %[[VAL_27]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_29:.*]] = arith.cmpi sle, %[[VAL_27]], %[[VAL_22]] : i64
+// CHECK:             %[[VAL_30:.*]] = arith.andi %[[VAL_28]], %[[VAL_29]] : i1
+// CHECK:             %[[VAL_31:.*]] = fir.if %[[VAL_30]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_27]] : (i64) -> index
+// CHECK:               %[[VAL_33:.*]] = hlfir.designate %[[VAL_16]]#0 (%[[VAL_32]])  typeparams %[[VAL_10]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index, i32) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_33]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_21]]#0 : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_31]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_24]] to %[[VAL_16]]#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           hlfir.destroy %[[VAL_24]] : !hlfir.expr<?x!fir.char<1,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with scalar always present boundary.
+// ! CHARACTER with assumed length.
+// subroutine eoshift7c(n, array, boundary)
+//   integer :: n
+//   character(*,1) :: array(n), boundary
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift7c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}, %arg2: !fir.boxchar<1> {fir.bindc_name = "boundary"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift7cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3:2 = hlfir.declare %2#0 typeparams %2#1 dummy_scope %0 {uniq_name = "_QFeoshift7cEboundary"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+  %4:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %5 = fir.convert %4#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+  %6 = fir.load %1#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> index
+  %8 = arith.cmpi sgt, %7, %c0 : index
+  %9 = arith.select %8, %7, %c0 : index
+  %10 = fir.shape %9 : (index) -> !fir.shape<1>
+  %11:2 = hlfir.declare %5(%10) typeparams %4#1 dummy_scope %0 {uniq_name = "_QFeoshift7cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+  %12 = hlfir.eoshift %11#0 %c2_i32 boundary %3#0 : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i32, !fir.boxchar<1>) -> !hlfir.expr<?x!fir.char<1,?>>
+  hlfir.assign %12 to %11#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+  hlfir.destroy %12 : !hlfir.expr<?x!fir.char<1,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift7c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.boxchar<1> {fir.bindc_name = "boundary"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFeoshift7cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_5:.*]]:2 = fir.unboxchar %[[ARG2]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_5]]#0 typeparams %[[VAL_5]]#1 dummy_scope %[[VAL_3]] {uniq_name = "_QFeoshift7cEboundary"} : (!fir.ref<!fir.char<1,?>>, index, !fir.dscope) -> (!fir.boxchar<1>, !fir.ref<!fir.char<1,?>>)
+// CHECK:           %[[VAL_7:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_4]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+// CHECK:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_10]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_12:.*]] = arith.select %[[VAL_11]], %[[VAL_10]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_13:.*]] = fir.shape %[[VAL_12]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_14:.*]]:2 = hlfir.declare %[[VAL_8]](%[[VAL_13]]) typeparams %[[VAL_7]]#1 dummy_scope %[[VAL_3]] {uniq_name = "_QFeoshift7cEarray"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_12]] : (index) -> i64
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_17:.*]] = hlfir.elemental %[[VAL_13]] typeparams %[[VAL_7]]#1 unordered : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<1,?>> {
+// CHECK:           ^bb0(%[[VAL_18:.*]]: index):
+// CHECK:             %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (index) -> i64
+// CHECK:             %[[VAL_20:.*]] = arith.addi %[[VAL_19]], %[[VAL_16]] overflow<nsw> : i64
+// CHECK:             %[[VAL_21:.*]] = arith.cmpi sge, %[[VAL_20]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_22:.*]] = arith.cmpi sle, %[[VAL_20]], %[[VAL_15]] : i64
+// CHECK:             %[[VAL_23:.*]] = arith.andi %[[VAL_21]], %[[VAL_22]] : i1
+// CHECK:             %[[VAL_24:.*]] = fir.if %[[VAL_23]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_25:.*]] = fir.convert %[[VAL_20]] : (i64) -> index
+// CHECK:               %[[VAL_26:.*]] = hlfir.designate %[[VAL_14]]#0 (%[[VAL_25]])  typeparams %[[VAL_7]]#1 : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_26]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_6]]#0 : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_24]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_17]] to %[[VAL_14]]#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           hlfir.destroy %[[VAL_17]] : !hlfir.expr<?x!fir.char<1,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the scalar optional boundary.
+// ! CHARACTER with constant length.
+// subroutine eoshift8c(n, array, boundary)
+//   integer :: n
+//   character(10,2) :: array(n)
+//   character(10,2), optional :: boundary
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift8c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<2> {fir.bindc_name = "array"}, %arg2: !fir.boxchar<2> {fir.bindc_name = "boundary", fir.optional}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift8cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg2 : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<2,?>>) -> !fir.ref<!fir.char<2,10>>
+  %4:2 = hlfir.declare %3 typeparams %c10 dummy_scope %0 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift8cEboundary"} : (!fir.ref<!fir.char<2,10>>, index, !fir.dscope) -> (!fir.ref<!fir.char<2,10>>, !fir.ref<!fir.char<2,10>>)
+  %5:2 = fir.unboxchar %arg1 : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+  %6 = fir.convert %5#0 : (!fir.ref<!fir.char<2,?>>) -> !fir.ref<!fir.array<?x!fir.char<2,10>>>
+  %7 = fir.load %1#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> index
+  %9 = arith.cmpi sgt, %8, %c0 : index
+  %10 = arith.select %9, %8, %c0 : index
+  %11 = fir.shape %10 : (index) -> !fir.shape<1>
+  %12:2 = hlfir.declare %6(%11) typeparams %c10 dummy_scope %0 {uniq_name = "_QFeoshift8cEarray"} : (!fir.ref<!fir.array<?x!fir.char<2,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<2,10>>>, !fir.ref<!fir.array<?x!fir.char<2,10>>>)
+  %13 = fir.is_present %4#0 : (!fir.ref<!fir.char<2,10>>) -> i1
+  %14 = fir.embox %4#0 : (!fir.ref<!fir.char<2,10>>) -> !fir.box<!fir.char<2,10>>
+  %15 = fir.absent !fir.box<!fir.char<2,10>>
+  %16 = arith.select %13, %14, %15 : !fir.box<!fir.char<2,10>>
+  %17 = hlfir.eoshift %12#0 %c2_i32 boundary %16 : (!fir.box<!fir.array<?x!fir.char<2,10>>>, i32, !fir.box<!fir.char<2,10>>) -> !hlfir.expr<?x!fir.char<2,10>>
+  hlfir.assign %17 to %12#0 : !hlfir.expr<?x!fir.char<2,10>>, !fir.box<!fir.array<?x!fir.char<2,10>>>
+  hlfir.destroy %17 : !hlfir.expr<?x!fir.char<2,10>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift8c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<2> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.boxchar<2> {fir.bindc_name = "boundary", fir.optional}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift8cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = fir.unboxchar %[[ARG2]] : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<2,?>>) -> !fir.ref<!fir.char<2,10>>
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[VAL_7]] typeparams %[[VAL_3]] dummy_scope %[[VAL_4]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift8cEboundary"} : (!fir.ref<!fir.char<2,10>>, index, !fir.dscope) -> (!fir.ref<!fir.char<2,10>>, !fir.ref<!fir.char<2,10>>)
+// CHECK:           %[[VAL_9:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref<!fir.char<2,?>>) -> !fir.ref<!fir.array<?x!fir.char<2,10>>>
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+// CHECK:           %[[VAL_13:.*]] = arith.cmpi sgt, %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.select %[[VAL_13]], %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_15:.*]] = fir.shape %[[VAL_14]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_16:.*]]:2 = hlfir.declare %[[VAL_10]](%[[VAL_15]]) typeparams %[[VAL_3]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift8cEarray"} : (!fir.ref<!fir.array<?x!fir.char<2,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<2,10>>>, !fir.ref<!fir.array<?x!fir.char<2,10>>>)
+// CHECK:           %[[VAL_17:.*]] = fir.is_present %[[VAL_8]]#0 : (!fir.ref<!fir.char<2,10>>) -> i1
+// CHECK:           %[[VAL_18:.*]] = fir.embox %[[VAL_8]]#0 : (!fir.ref<!fir.char<2,10>>) -> !fir.box<!fir.char<2,10>>
+// CHECK:           %[[VAL_19:.*]] = fir.absent !fir.box<!fir.char<2,10>>
+// CHECK:           %[[VAL_20:.*]] = arith.select %[[VAL_17]], %[[VAL_18]], %[[VAL_19]] : !fir.box<!fir.char<2,10>>
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_14]] : (index) -> i64
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_23:.*]] = fir.is_present %[[VAL_20]] : (!fir.box<!fir.char<2,10>>) -> i1
+// CHECK:           %[[VAL_24:.*]] = fir.if %[[VAL_23]] -> (!fir.boxchar<2>) {
+// CHECK:             %[[VAL_25:.*]] = fir.box_addr %[[VAL_20]] : (!fir.box<!fir.char<2,10>>) -> !fir.ref<!fir.char<2,10>>
+// CHECK:             %[[VAL_26:.*]] = fir.emboxchar %[[VAL_25]], %[[VAL_3]] : (!fir.ref<!fir.char<2,10>>, index) -> !fir.boxchar<2>
+// CHECK:             fir.result %[[VAL_26]] : !fir.boxchar<2>
+// CHECK:           } else {
+// CHECK:             %[[VAL_27:.*]] = fir.alloca !fir.char<2,0> {bindc_name = ".chrtmp"}
+// CHECK:             %[[VAL_28:.*]] = fir.emboxchar %[[VAL_27]], %[[VAL_2]] : (!fir.ref<!fir.char<2,0>>, index) -> !fir.boxchar<2>
+// CHECK:             fir.result %[[VAL_28]] : !fir.boxchar<2>
+// CHECK:           }
+// CHECK:           %[[VAL_29:.*]] = hlfir.elemental %[[VAL_15]] typeparams %[[VAL_3]] unordered : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<2,10>> {
+// CHECK:           ^bb0(%[[VAL_30:.*]]: index):
+// CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (index) -> i64
+// CHECK:             %[[VAL_32:.*]] = arith.addi %[[VAL_31]], %[[VAL_22]] overflow<nsw> : i64
+// CHECK:             %[[VAL_33:.*]] = arith.cmpi sge, %[[VAL_32]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_34:.*]] = arith.cmpi sle, %[[VAL_32]], %[[VAL_21]] : i64
+// CHECK:             %[[VAL_35:.*]] = arith.andi %[[VAL_33]], %[[VAL_34]] : i1
+// CHECK:             %[[VAL_36:.*]] = fir.if %[[VAL_35]] -> (!fir.boxchar<2>) {
+// CHECK:               %[[VAL_37:.*]] = fir.convert %[[VAL_32]] : (i64) -> index
+// CHECK:               %[[VAL_38:.*]] = hlfir.designate %[[VAL_16]]#0 (%[[VAL_37]])  typeparams %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.char<2,10>>>, index, index) -> !fir.ref<!fir.char<2,10>>
+// CHECK:               %[[VAL_39:.*]] = fir.emboxchar %[[VAL_38]], %[[VAL_3]] : (!fir.ref<!fir.char<2,10>>, index) -> !fir.boxchar<2>
+// CHECK:               fir.result %[[VAL_39]] : !fir.boxchar<2>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_24]] : !fir.boxchar<2>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_36]] : !fir.boxchar<2>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_29]] to %[[VAL_16]]#0 : !hlfir.expr<?x!fir.char<2,10>>, !fir.box<!fir.array<?x!fir.char<2,10>>>
+// CHECK:           hlfir.destroy %[[VAL_29]] : !hlfir.expr<?x!fir.char<2,10>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the scalar optional boundary.
+// ! CHARACTER with variable length.
+// subroutine eoshift9c(n, array, boundary)
+//   integer :: n
+//   character(n,2) :: array(n)
+//   character(n,2), optional :: boundary
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift9c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<2> {fir.bindc_name = "array"}, %arg2: !fir.boxchar<2> {fir.bindc_name = "boundary", fir.optional}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift9cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<2,?>>) -> !fir.ref<!fir.array<?x!fir.char<2,?>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = arith.cmpi sgt, %4, %c0_i32 : i32
+  %6 = arith.select %5, %4, %c0_i32 : i32
+  %7 = fir.load %1#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> index
+  %9 = arith.cmpi sgt, %8, %c0 : index
+  %10 = arith.select %9, %8, %c0 : index
+  %11 = fir.shape %10 : (index) -> !fir.shape<1>
+  %12:2 = hlfir.declare %3(%11) typeparams %6 dummy_scope %0 {uniq_name = "_QFeoshift9cEarray"} : (!fir.ref<!fir.array<?x!fir.char<2,?>>>, !fir.shape<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<2,?>>>, !fir.ref<!fir.array<?x!fir.char<2,?>>>)
+  %13:2 = fir.unboxchar %arg2 : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+  %14 = fir.load %1#0 : !fir.ref<i32>
+  %15 = arith.cmpi sgt, %14, %c0_i32 : i32
+  %16 = arith.select %15, %14, %c0_i32 : i32
+  %17:2 = hlfir.declare %13#0 typeparams %16 dummy_scope %0 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift9cEboundary"} : (!fir.ref<!fir.char<2,?>>, i32, !fir.dscope) -> (!fir.boxchar<2>, !fir.ref<!fir.char<2,?>>)
+  %18 = fir.is_present %17#0 : (!fir.boxchar<2>) -> i1
+  %19 = fir.embox %17#1 typeparams %16 : (!fir.ref<!fir.char<2,?>>, i32) -> !fir.box<!fir.char<2,?>>
+  %20 = fir.absent !fir.box<!fir.char<2,?>>
+  %21 = arith.select %18, %19, %20 : !fir.box<!fir.char<2,?>>
+  %22 = hlfir.eoshift %12#0 %c2_i32 boundary %21 : (!fir.box<!fir.array<?x!fir.char<2,?>>>, i32, !fir.box<!fir.char<2,?>>) -> !hlfir.expr<?x!fir.char<2,?>>
+  hlfir.assign %22 to %12#0 : !hlfir.expr<?x!fir.char<2,?>>, !fir.box<!fir.array<?x!fir.char<2,?>>>
+  hlfir.destroy %22 : !hlfir.expr<?x!fir.char<2,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift9c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<2> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.boxchar<2> {fir.bindc_name = "boundary", fir.optional}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_5:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_5]] {uniq_name = "_QFeoshift9cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_7:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.char<2,?>>) -> !fir.ref<!fir.array<?x!fir.char<2,?>>>
+// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_10:.*]] = arith.cmpi sgt, %[[VAL_9]], %[[VAL_4]] : i32
+// CHECK:           %[[VAL_11:.*]] = arith.select %[[VAL_10]], %[[VAL_9]], %[[VAL_4]] : i32
+// CHECK:           %[[VAL_12:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_13:.*]] = fir.convert %[[VAL_12]] : (i32) -> index
+// CHECK:           %[[VAL_14:.*]] = arith.cmpi sgt, %[[VAL_13]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_15:.*]] = arith.select %[[VAL_14]], %[[VAL_13]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_16:.*]] = fir.shape %[[VAL_15]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_17:.*]]:2 = hlfir.declare %[[VAL_8]](%[[VAL_16]]) typeparams %[[VAL_11]] dummy_scope %[[VAL_5]] {uniq_name = "_QFeoshift9cEarray"} : (!fir.ref<!fir.array<?x!fir.char<2,?>>>, !fir.shape<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<2,?>>>, !fir.ref<!fir.array<?x!fir.char<2,?>>>)
+// CHECK:           %[[VAL_18:.*]]:2 = fir.unboxchar %[[ARG2]] : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+// CHECK:           %[[VAL_19:.*]] = fir.load %[[VAL_6]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_20:.*]] = arith.cmpi sgt, %[[VAL_19]], %[[VAL_4]] : i32
+// CHECK:           %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_19]], %[[VAL_4]] : i32
+// CHECK:           %[[VAL_22:.*]]:2 = hlfir.declare %[[VAL_18]]#0 typeparams %[[VAL_21]] dummy_scope %[[VAL_5]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift9cEboundary"} : (!fir.ref<!fir.char<2,?>>, i32, !fir.dscope) -> (!fir.boxchar<2>, !fir.ref<!fir.char<2,?>>)
+// CHECK:           %[[VAL_23:.*]] = fir.is_present %[[VAL_22]]#0 : (!fir.boxchar<2>) -> i1
+// CHECK:           %[[VAL_24:.*]] = fir.embox %[[VAL_22]]#1 typeparams %[[VAL_21]] : (!fir.ref<!fir.char<2,?>>, i32) -> !fir.box<!fir.char<2,?>>
+// CHECK:           %[[VAL_25:.*]] = fir.absent !fir.box<!fir.char<2,?>>
+// CHECK:           %[[VAL_26:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_25]] : !fir.box<!fir.char<2,?>>
+// CHECK:           %[[VAL_27:.*]] = fir.convert %[[VAL_15]] : (index) -> i64
+// CHECK:           %[[VAL_28:.*]] = fir.convert %[[VAL_2]] : (i32) -> i64
+// CHECK:           %[[VAL_29:.*]] = fir.is_present %[[VAL_26]] : (!fir.box<!fir.char<2,?>>) -> i1
+// CHECK:           %[[VAL_30:.*]] = fir.if %[[VAL_29]] -> (!fir.boxchar<2>) {
+// CHECK:             %[[VAL_31:.*]] = fir.box_addr %[[VAL_26]] : (!fir.box<!fir.char<2,?>>) -> !fir.ref<!fir.char<2,?>>
+// CHECK:             %[[VAL_32:.*]] = fir.box_elesize %[[VAL_26]] : (!fir.box<!fir.char<2,?>>) -> index
+// CHECK:             %[[VAL_33:.*]] = arith.divsi %[[VAL_32]], %[[VAL_1]] : index
+// CHECK:             %[[VAL_34:.*]] = fir.emboxchar %[[VAL_31]], %[[VAL_33]] : (!fir.ref<!fir.char<2,?>>, index) -> !fir.boxchar<2>
+// CHECK:             fir.result %[[VAL_34]] : !fir.boxchar<2>
+// CHECK:           } else {
+// CHECK:             %[[VAL_35:.*]] = fir.alloca !fir.char<2,0> {bindc_name = ".chrtmp"}
+// CHECK:             %[[VAL_36:.*]] = fir.emboxchar %[[VAL_35]], %[[VAL_3]] : (!fir.ref<!fir.char<2,0>>, index) -> !fir.boxchar<2>
+// CHECK:             fir.result %[[VAL_36]] : !fir.boxchar<2>
+// CHECK:           }
+// CHECK:           %[[VAL_37:.*]] = hlfir.elemental %[[VAL_16]] typeparams %[[VAL_11]] unordered : (!fir.shape<1>, i32) -> !hlfir.expr<?x!fir.char<2,?>> {
+// CHECK:           ^bb0(%[[VAL_38:.*]]: index):
+// CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_38]] : (index) -> i64
+// CHECK:             %[[VAL_40:.*]] = arith.addi %[[VAL_39]], %[[VAL_28]] overflow<nsw> : i64
+// CHECK:             %[[VAL_41:.*]] = arith.cmpi sge, %[[VAL_40]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_42:.*]] = arith.cmpi sle, %[[VAL_40]], %[[VAL_27]] : i64
+// CHECK:             %[[VAL_43:.*]] = arith.andi %[[VAL_41]], %[[VAL_42]] : i1
+// CHECK:             %[[VAL_44:.*]] = fir.if %[[VAL_43]] -> (!fir.boxchar<2>) {
+// CHECK:               %[[VAL_45:.*]] = fir.convert %[[VAL_40]] : (i64) -> index
+// CHECK:               %[[VAL_46:.*]] = hlfir.designate %[[VAL_17]]#0 (%[[VAL_45]])  typeparams %[[VAL_11]] : (!fir.box<!fir.array<?x!fir.char<2,?>>>, index, i32) -> !fir.boxchar<2>
+// CHECK:               fir.result %[[VAL_46]] : !fir.boxchar<2>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_30]] : !fir.boxchar<2>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_44]] : !fir.boxchar<2>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_37]] to %[[VAL_17]]#0 : !hlfir.expr<?x!fir.char<2,?>>, !fir.box<!fir.array<?x!fir.char<2,?>>>
+// CHECK:           hlfir.destroy %[[VAL_37]] : !hlfir.expr<?x!fir.char<2,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the scalar optional boundary.
+// ! CHARACTER with assumed length.
+// subroutine eoshift10c(n, array, boundary)
+//   integer :: n
+//   character(*,2) :: array(n)
+//   character(*,2), optional :: boundary
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift10c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<2> {fir.bindc_name = "array"}, %arg2: !fir.boxchar<2> {fir.bindc_name = "boundary", fir.optional}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift10cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg2 : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+  %3:2 = hlfir.declare %2#0 typeparams %2#1 dummy_scope %0 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift10cEboundary"} : (!fir.ref<!fir.char<2,?>>, index, !fir.dscope) -> (!fir.boxchar<2>, !fir.ref<!fir.char<2,?>>)
+  %4:2 = fir.unboxchar %arg1 : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+  %5 = fir.convert %4#0 : (!fir.ref<!fir.char<2,?>>) -> !fir.ref<!fir.array<?x!fir.char<2,?>>>
+  %6 = fir.load %1#0 : !fir.ref<i32>
+  %7 = fir.convert %6 : (i32) -> index
+  %8 = arith.cmpi sgt, %7, %c0 : index
+  %9 = arith.select %8, %7, %c0 : index
+  %10 = fir.shape %9 : (index) -> !fir.shape<1>
+  %11:2 = hlfir.declare %5(%10) typeparams %4#1 dummy_scope %0 {uniq_name = "_QFeoshift10cEarray"} : (!fir.ref<!fir.array<?x!fir.char<2,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<2,?>>>, !fir.ref<!fir.array<?x!fir.char<2,?>>>)
+  %12 = fir.is_present %3#0 : (!fir.boxchar<2>) -> i1
+  %13 = fir.embox %3#1 typeparams %2#1 : (!fir.ref<!fir.char<2,?>>, index) -> !fir.box<!fir.char<2,?>>
+  %14 = fir.absent !fir.box<!fir.char<2,?>>
+  %15 = arith.select %12, %13, %14 : !fir.box<!fir.char<2,?>>
+  %16 = hlfir.eoshift %11#0 %c2_i32 boundary %15 : (!fir.box<!fir.array<?x!fir.char<2,?>>>, i32, !fir.box<!fir.char<2,?>>) -> !hlfir.expr<?x!fir.char<2,?>>
+  hlfir.assign %16 to %11#0 : !hlfir.expr<?x!fir.char<2,?>>, !fir.box<!fir.array<?x!fir.char<2,?>>>
+  hlfir.destroy %16 : !hlfir.expr<?x!fir.char<2,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift10c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<2> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.boxchar<2> {fir.bindc_name = "boundary", fir.optional}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift10cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = fir.unboxchar %[[ARG2]] : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[VAL_6]]#0 typeparams %[[VAL_6]]#1 dummy_scope %[[VAL_4]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift10cEboundary"} : (!fir.ref<!fir.char<2,?>>, index, !fir.dscope) -> (!fir.boxchar<2>, !fir.ref<!fir.char<2,?>>)
+// CHECK:           %[[VAL_8:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<2>) -> (!fir.ref<!fir.char<2,?>>, index)
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]]#0 : (!fir.ref<!fir.char<2,?>>) -> !fir.ref<!fir.array<?x!fir.char<2,?>>>
+// CHECK:           %[[VAL_10:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (i32) -> index
+// CHECK:           %[[VAL_12:.*]] = arith.cmpi sgt, %[[VAL_11]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_13:.*]] = arith.select %[[VAL_12]], %[[VAL_11]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_14:.*]] = fir.shape %[[VAL_13]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_15:.*]]:2 = hlfir.declare %[[VAL_9]](%[[VAL_14]]) typeparams %[[VAL_8]]#1 dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift10cEarray"} : (!fir.ref<!fir.array<?x!fir.char<2,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<2,?>>>, !fir.ref<!fir.array<?x!fir.char<2,?>>>)
+// CHECK:           %[[VAL_16:.*]] = fir.is_present %[[VAL_7]]#0 : (!fir.boxchar<2>) -> i1
+// CHECK:           %[[VAL_17:.*]] = fir.embox %[[VAL_7]]#1 typeparams %[[VAL_6]]#1 : (!fir.ref<!fir.char<2,?>>, index) -> !fir.box<!fir.char<2,?>>
+// CHECK:           %[[VAL_18:.*]] = fir.absent !fir.box<!fir.char<2,?>>
+// CHECK:           %[[VAL_19:.*]] = arith.select %[[VAL_16]], %[[VAL_17]], %[[VAL_18]] : !fir.box<!fir.char<2,?>>
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_13]] : (index) -> i64
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_2]] : (i32) -> i64
+// CHECK:           %[[VAL_22:.*]] = fir.is_present %[[VAL_19]] : (!fir.box<!fir.char<2,?>>) -> i1
+// CHECK:           %[[VAL_23:.*]] = fir.if %[[VAL_22]] -> (!fir.boxchar<2>) {
+// CHECK:             %[[VAL_24:.*]] = fir.box_addr %[[VAL_19]] : (!fir.box<!fir.char<2,?>>) -> !fir.ref<!fir.char<2,?>>
+// CHECK:             %[[VAL_25:.*]] = fir.box_elesize %[[VAL_19]] : (!fir.box<!fir.char<2,?>>) -> index
+// CHECK:             %[[VAL_26:.*]] = arith.divsi %[[VAL_25]], %[[VAL_1]] : index
+// CHECK:             %[[VAL_27:.*]] = fir.emboxchar %[[VAL_24]], %[[VAL_26]] : (!fir.ref<!fir.char<2,?>>, index) -> !fir.boxchar<2>
+// CHECK:             fir.result %[[VAL_27]] : !fir.boxchar<2>
+// CHECK:           } else {
+// CHECK:             %[[VAL_28:.*]] = fir.alloca !fir.char<2,0> {bindc_name = ".chrtmp"}
+// CHECK:             %[[VAL_29:.*]] = fir.emboxchar %[[VAL_28]], %[[VAL_3]] : (!fir.ref<!fir.char<2,0>>, index) -> !fir.boxchar<2>
+// CHECK:             fir.result %[[VAL_29]] : !fir.boxchar<2>
+// CHECK:           }
+// CHECK:           %[[VAL_30:.*]] = hlfir.elemental %[[VAL_14]] typeparams %[[VAL_8]]#1 unordered : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<2,?>> {
+// CHECK:           ^bb0(%[[VAL_31:.*]]: index):
+// CHECK:             %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (index) -> i64
+// CHECK:             %[[VAL_33:.*]] = arith.addi %[[VAL_32]], %[[VAL_21]] overflow<nsw> : i64
+// CHECK:             %[[VAL_34:.*]] = arith.cmpi sge, %[[VAL_33]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_35:.*]] = arith.cmpi sle, %[[VAL_33]], %[[VAL_20]] : i64
+// CHECK:             %[[VAL_36:.*]] = arith.andi %[[VAL_34]], %[[VAL_35]] : i1
+// CHECK:             %[[VAL_37:.*]] = fir.if %[[VAL_36]] -> (!fir.boxchar<2>) {
+// CHECK:               %[[VAL_38:.*]] = fir.convert %[[VAL_33]] : (i64) -> index
+// CHECK:               %[[VAL_39:.*]] = hlfir.designate %[[VAL_15]]#0 (%[[VAL_38]])  typeparams %[[VAL_8]]#1 : (!fir.box<!fir.array<?x!fir.char<2,?>>>, index, index) -> !fir.boxchar<2>
+// CHECK:               fir.result %[[VAL_39]] : !fir.boxchar<2>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_23]] : !fir.boxchar<2>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_37]] : !fir.boxchar<2>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_30]] to %[[VAL_15]]#0 : !hlfir.expr<?x!fir.char<2,?>>, !fir.box<!fir.array<?x!fir.char<2,?>>>
+// CHECK:           hlfir.destroy %[[VAL_30]] : !hlfir.expr<?x!fir.char<2,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the array always present boundary.
+// ! CHARACTER with constant length.
+// subroutine eoshift11c(n, array, boundary)
+//   integer :: n
+//   character(10,4) :: array(n,n), boundary(:)
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift11c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<4> {fir.bindc_name = "array"}, %arg2: !fir.box<!fir.array<?x!fir.char<4,10>>> {fir.bindc_name = "boundary"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift11cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = hlfir.declare %arg2 typeparams %c10 dummy_scope %0 {uniq_name = "_QFeoshift11cEboundary"} : (!fir.box<!fir.array<?x!fir.char<4,10>>>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<4,10>>>, !fir.box<!fir.array<?x!fir.char<4,10>>>)
+  %3:2 = fir.unboxchar %arg1 : (!fir.boxchar<4>) -> (!fir.ref<!fir.char<4,?>>, index)
+  %4 = fir.convert %3#0 : (!fir.ref<!fir.char<4,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<4,10>>>
+  %5 = fir.load %1#0 : !fir.ref<i32>
+  %6 = fir.convert %5 : (i32) -> index
+  %7 = arith.cmpi sgt, %6, %c0 : index
+  %8 = arith.select %7, %6, %c0 : index
+  %9 = fir.load %1#0 : !fir.ref<i32>
+  %10 = fir.convert %9 : (i32) -> index
+  %11 = arith.cmpi sgt, %10, %c0 : index
+  %12 = arith.select %11, %10, %c0 : index
+  %13 = fir.shape %8, %12 : (index, index) -> !fir.shape<2>
+  %14:2 = hlfir.declare %4(%13) typeparams %c10 dummy_scope %0 {uniq_name = "_QFeoshift11cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<4,10>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<4,10>>>, !fir.ref<!fir.array<?x?x!fir.char<4,10>>>)
+  %15 = hlfir.eoshift %14#0 %c2_i32 boundary %2#0 : (!fir.box<!fir.array<?x?x!fir.char<4,10>>>, i32, !fir.box<!fir.array<?x!fir.char<4,10>>>) -> !hlfir.expr<?x?x!fir.char<4,10>>
+  hlfir.assign %15 to %14#0 : !hlfir.expr<?x?x!fir.char<4,10>>, !fir.box<!fir.array<?x?x!fir.char<4,10>>>
+  hlfir.destroy %15 : !hlfir.expr<?x?x!fir.char<4,10>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift11c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<4> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.box<!fir.array<?x!fir.char<4,10>>> {fir.bindc_name = "boundary"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift11cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[ARG2]] typeparams %[[VAL_3]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift11cEboundary"} : (!fir.box<!fir.array<?x!fir.char<4,10>>>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<4,10>>>, !fir.box<!fir.array<?x!fir.char<4,10>>>)
+// CHECK:           %[[VAL_7:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<4>) -> (!fir.ref<!fir.char<4,?>>, index)
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.char<4,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<4,10>>>
+// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+// CHECK:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_10]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_12:.*]] = arith.select %[[VAL_11]], %[[VAL_10]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i32) -> index
+// CHECK:           %[[VAL_15:.*]] = arith.cmpi sgt, %[[VAL_14]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_16:.*]] = arith.select %[[VAL_15]], %[[VAL_14]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_12]], %[[VAL_16]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_8]](%[[VAL_17]]) typeparams %[[VAL_3]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift11cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<4,10>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<4,10>>>, !fir.ref<!fir.array<?x?x!fir.char<4,10>>>)
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_12]] : (index) -> i64
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_21:.*]] = hlfir.elemental %[[VAL_17]] typeparams %[[VAL_3]] unordered : (!fir.shape<2>, index) -> !hlfir.expr<?x?x!fir.char<4,10>> {
+// CHECK:           ^bb0(%[[VAL_22:.*]]: index, %[[VAL_23:.*]]: index):
+// CHECK:             %[[VAL_24:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_23]])  typeparams %[[VAL_3]] : (!fir.box<!fir.array<?x!fir.char<4,10>>>, index, index) -> !fir.ref<!fir.char<4,10>>
+// CHECK:             %[[VAL_25:.*]] = fir.emboxchar %[[VAL_24]], %[[VAL_3]] : (!fir.ref<!fir.char<4,10>>, index) -> !fir.boxchar<4>
+// CHECK:             %[[VAL_26:.*]] = fir.convert %[[VAL_22]] : (index) -> i64
+// CHECK:             %[[VAL_27:.*]] = arith.addi %[[VAL_26]], %[[VAL_20]] overflow<nsw> : i64
+// CHECK:             %[[VAL_28:.*]] = arith.cmpi sge, %[[VAL_27]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_29:.*]] = arith.cmpi sle, %[[VAL_27]], %[[VAL_19]] : i64
+// CHECK:             %[[VAL_30:.*]] = arith.andi %[[VAL_28]], %[[VAL_29]] : i1
+// CHECK:             %[[VAL_31:.*]] = fir.if %[[VAL_30]] -> (!fir.boxchar<4>) {
+// CHECK:               %[[VAL_32:.*]] = fir.convert %[[VAL_27]] : (i64) -> index
+// CHECK:               %[[VAL_33:.*]] = hlfir.designate %[[VAL_18]]#0 (%[[VAL_32]], %[[VAL_23]])  typeparams %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.char<4,10>>>, index, index, index) -> !fir.ref<!fir.char<4,10>>
+// CHECK:               %[[VAL_34:.*]] = fir.emboxchar %[[VAL_33]], %[[VAL_3]] : (!fir.ref<!fir.char<4,10>>, index) -> !fir.boxchar<4>
+// CHECK:               fir.result %[[VAL_34]] : !fir.boxchar<4>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_25]] : !fir.boxchar<4>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_31]] : !fir.boxchar<4>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_21]] to %[[VAL_18]]#0 : !hlfir.expr<?x?x!fir.char<4,10>>, !fir.box<!fir.array<?x?x!fir.char<4,10>>>
+// CHECK:           hlfir.destroy %[[VAL_21]] : !hlfir.expr<?x?x!fir.char<4,10>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the array always present boundary.
+// ! CHARACTER with variable length.
+// subroutine eoshift12c(n, array, boundary)
+//   integer :: n
+//   character(n,4) :: array(n,n), boundary(:)
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift12c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<4> {fir.bindc_name = "array"}, %arg2: !fir.box<!fir.array<?x!fir.char<4,?>>> {fir.bindc_name = "boundary"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift12cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<4>) -> (!fir.ref<!fir.char<4,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<4,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<4,?>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = arith.cmpi sgt, %4, %c0_i32 : i32
+  %6 = arith.select %5, %4, %c0_i32 : i32
+  %7 = fir.load %1#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> index
+  %9 = arith.cmpi sgt, %8, %c0 : index
+  %10 = arith.select %9, %8, %c0 : index
+  %11 = fir.load %1#0 : !fir.ref<i32>
+  %12 = fir.convert %11 : (i32) -> index
+  %13 = arith.cmpi sgt, %12, %c0 : index
+  %14 = arith.select %13, %12, %c0 : index
+  %15 = fir.shape %10, %14 : (index, index) -> !fir.shape<2>
+  %16:2 = hlfir.declare %3(%15) typeparams %6 dummy_scope %0 {uniq_name = "_QFeoshift12cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<4,?>>>, !fir.shape<2>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<4,?>>>, !fir.ref<!fir.array<?x?x!fir.char<4,?>>>)
+  %17 = fir.load %1#0 : !fir.ref<i32>
+  %18 = arith.cmpi sgt, %17, %c0_i32 : i32
+  %19 = arith.select %18, %17, %c0_i32 : i32
+  %20:2 = hlfir.declare %arg2 typeparams %19 dummy_scope %0 {uniq_name = "_QFeoshift12cEboundary"} : (!fir.box<!fir.array<?x!fir.char<4,?>>>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<4,?>>>, !fir.box<!fir.array<?x!fir.char<4,?>>>)
+  %21 = hlfir.eoshift %16#0 %c2_i32 boundary %20#0 : (!fir.box<!fir.array<?x?x!fir.char<4,?>>>, i32, !fir.box<!fir.array<?x!fir.char<4,?>>>) -> !hlfir.expr<?x?x!fir.char<4,?>>
+  hlfir.assign %21 to %16#0 : !hlfir.expr<?x?x!fir.char<4,?>>, !fir.box<!fir.array<?x?x!fir.char<4,?>>>
+  hlfir.destroy %21 : !hlfir.expr<?x?x!fir.char<4,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift12c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<4> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.box<!fir.array<?x!fir.char<4,?>>> {fir.bindc_name = "boundary"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift12cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<4>) -> (!fir.ref<!fir.char<4,?>>, index)
+// CHECK:           %[[VAL_7:.*]] = fir.convert %[[VAL_6]]#0 : (!fir.ref<!fir.char<4,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<4,?>>>
+// CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_9:.*]] = arith.cmpi sgt, %[[VAL_8]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_10:.*]] = arith.select %[[VAL_9]], %[[VAL_8]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+// CHECK:           %[[VAL_13:.*]] = arith.cmpi sgt, %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.select %[[VAL_13]], %[[VAL_12]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> index
+// CHECK:           %[[VAL_17:.*]] = arith.cmpi sgt, %[[VAL_16]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_18:.*]] = arith.select %[[VAL_17]], %[[VAL_16]], %[[VAL_2]] : index
+// CHECK:           %[[VAL_19:.*]] = fir.shape %[[VAL_14]], %[[VAL_18]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_7]](%[[VAL_19]]) typeparams %[[VAL_10]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift12cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<4,?>>>, !fir.shape<2>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<4,?>>>, !fir.ref<!fir.array<?x?x!fir.char<4,?>>>)
+// CHECK:           %[[VAL_21:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_22:.*]] = arith.cmpi sgt, %[[VAL_21]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_23:.*]] = arith.select %[[VAL_22]], %[[VAL_21]], %[[VAL_3]] : i32
+// CHECK:           %[[VAL_24:.*]]:2 = hlfir.declare %[[ARG2]] typeparams %[[VAL_23]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift12cEboundary"} : (!fir.box<!fir.array<?x!fir.char<4,?>>>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<4,?>>>, !fir.box<!fir.array<?x!fir.char<4,?>>>)
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_14]] : (index) -> i64
+// CHECK:           %[[VAL_26:.*]] = fir.convert %[[VAL_1]] : (i32) -> i64
+// CHECK:           %[[VAL_27:.*]] = hlfir.elemental %[[VAL_19]] typeparams %[[VAL_10]] unordered : (!fir.shape<2>, i32) -> !hlfir.expr<?x?x!fir.char<4,?>> {
+// CHECK:           ^bb0(%[[VAL_28:.*]]: index, %[[VAL_29:.*]]: index):
+// CHECK:             %[[VAL_30:.*]] = hlfir.designate %[[VAL_24]]#0 (%[[VAL_29]])  typeparams %[[VAL_23]] : (!fir.box<!fir.array<?x!fir.char<4,?>>>, index, i32) -> !fir.boxchar<4>
+// CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_28]] : (index) -> i64
+// CHECK:             %[[VAL_32:.*]] = arith.addi %[[VAL_31]], %[[VAL_26]] overflow<nsw> : i64
+// CHECK:             %[[VAL_33:.*]] = arith.cmpi sge, %[[VAL_32]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_34:.*]] = arith.cmpi sle, %[[VAL_32]], %[[VAL_25]] : i64
+// CHECK:             %[[VAL_35:.*]] = arith.andi %[[VAL_33]], %[[VAL_34]] : i1
+// CHECK:             %[[VAL_36:.*]] = fir.if %[[VAL_35]] -> (!fir.boxchar<4>) {
+// CHECK:               %[[VAL_37:.*]] = fir.convert %[[VAL_32]] : (i64) -> index
+// CHECK:               %[[VAL_38:.*]] = hlfir.designate %[[VAL_20]]#0 (%[[VAL_37]], %[[VAL_29]])  typeparams %[[VAL_10]] : (!fir.box<!fir.array<?x?x!fir.char<4,?>>>, index, index, i32) -> !fir.boxchar<4>
+// CHECK:               fir.result %[[VAL_38]] : !fir.boxchar<4>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_30]] : !fir.boxchar<4>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_36]] : !fir.boxchar<4>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_27]] to %[[VAL_20]]#0 : !hlfir.expr<?x?x!fir.char<4,?>>, !fir.box<!fir.array<?x?x!fir.char<4,?>>>
+// CHECK:           hlfir.destroy %[[VAL_27]] : !hlfir.expr<?x?x!fir.char<4,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the array always present boundary.
+// ! CHARACTER with assumed length.
+// subroutine eoshift13c(n, array, boundary)
+//   integer :: n
+//   character(*,4) :: array(n,n), boundary(:)
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift13c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<4> {fir.bindc_name = "array"}, %arg2: !fir.box<!fir.array<?x!fir.char<4,?>>> {fir.bindc_name = "boundary"}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift13cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = hlfir.declare %arg2 dummy_scope %0 {uniq_name = "_QFeoshift13cEboundary"} : (!fir.box<!fir.array<?x!fir.char<4,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<4,?>>>, !fir.box<!fir.array<?x!fir.char<4,?>>>)
+  %3:2 = fir.unboxchar %arg1 : (!fir.boxchar<4>) -> (!fir.ref<!fir.char<4,?>>, index)
+  %4 = fir.convert %3#0 : (!fir.ref<!fir.char<4,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<4,?>>>
+  %5 = fir.load %1#0 : !fir.ref<i32>
+  %6 = fir.convert %5 : (i32) -> index
+  %7 = arith.cmpi sgt, %6, %c0 : index
+  %8 = arith.select %7, %6, %c0 : index
+  %9 = fir.load %1#0 : !fir.ref<i32>
+  %10 = fir.convert %9 : (i32) -> index
+  %11 = arith.cmpi sgt, %10, %c0 : index
+  %12 = arith.select %11, %10, %c0 : index
+  %13 = fir.shape %8, %12 : (index, index) -> !fir.shape<2>
+  %14:2 = hlfir.declare %4(%13) typeparams %3#1 dummy_scope %0 {uniq_name = "_QFeoshift13cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<4,?>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<4,?>>>, !fir.ref<!fir.array<?x?x!fir.char<4,?>>>)
+  %15 = hlfir.eoshift %14#0 %c2_i32 boundary %2#0 : (!fir.box<!fir.array<?x?x!fir.char<4,?>>>, i32, !fir.box<!fir.array<?x!fir.char<4,?>>>) -> !hlfir.expr<?x?x!fir.char<4,?>>
+  hlfir.assign %15 to %14#0 : !hlfir.expr<?x?x!fir.char<4,?>>, !fir.box<!fir.array<?x?x!fir.char<4,?>>>
+  hlfir.destroy %15 : !hlfir.expr<?x?x!fir.char<4,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift13c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<4> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.box<!fir.array<?x!fir.char<4,?>>> {fir.bindc_name = "boundary"}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 4 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift13cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift13cEboundary"} : (!fir.box<!fir.array<?x!fir.char<4,?>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<4,?>>>, !fir.box<!fir.array<?x!fir.char<4,?>>>)
+// CHECK:           %[[VAL_7:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<4>) -> (!fir.ref<!fir.char<4,?>>, index)
+// CHECK:           %[[VAL_8:.*]] = fir.convert %[[VAL_7]]#0 : (!fir.ref<!fir.char<4,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<4,?>>>
+// CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]] : (i32) -> index
+// CHECK:           %[[VAL_11:.*]] = arith.cmpi sgt, %[[VAL_10]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_12:.*]] = arith.select %[[VAL_11]], %[[VAL_10]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_14:.*]] = fir.convert %[[VAL_13]] : (i32) -> index
+// CHECK:           %[[VAL_15:.*]] = arith.cmpi sgt, %[[VAL_14]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_16:.*]] = arith.select %[[VAL_15]], %[[VAL_14]], %[[VAL_3]] : index
+// CHECK:           %[[VAL_17:.*]] = fir.shape %[[VAL_12]], %[[VAL_16]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_18:.*]]:2 = hlfir.declare %[[VAL_8]](%[[VAL_17]]) typeparams %[[VAL_7]]#1 dummy_scope %[[VAL_4]] {uniq_name = "_QFeoshift13cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<4,?>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<4,?>>>, !fir.ref<!fir.array<?x?x!fir.char<4,?>>>)
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_12]] : (index) -> i64
+// CHECK:           %[[VAL_20:.*]] = fir.convert %[[VAL_2]] : (i32) -> i64
+// CHECK:           %[[VAL_21:.*]] = hlfir.elemental %[[VAL_17]] typeparams %[[VAL_7]]#1 unordered : (!fir.shape<2>, index) -> !hlfir.expr<?x?x!fir.char<4,?>> {
+// CHECK:           ^bb0(%[[VAL_22:.*]]: index, %[[VAL_23:.*]]: index):
+// CHECK:             %[[VAL_24:.*]] = fir.box_elesize %[[VAL_6]]#1 : (!fir.box<!fir.array<?x!fir.char<4,?>>>) -> index
+// CHECK:             %[[VAL_25:.*]] = arith.divsi %[[VAL_24]], %[[VAL_1]] : index
+// CHECK:             %[[VAL_26:.*]] = hlfir.designate %[[VAL_6]]#0 (%[[VAL_23]])  typeparams %[[VAL_25]] : (!fir.box<!fir.array<?x!fir.char<4,?>>>, index, index) -> !fir.boxchar<4>
+// CHECK:             %[[VAL_27:.*]] = fir.convert %[[VAL_22]] : (index) -> i64
+// CHECK:             %[[VAL_28:.*]] = arith.addi %[[VAL_27]], %[[VAL_20]] overflow<nsw> : i64
+// CHECK:             %[[VAL_29:.*]] = arith.cmpi sge, %[[VAL_28]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_30:.*]] = arith.cmpi sle, %[[VAL_28]], %[[VAL_19]] : i64
+// CHECK:             %[[VAL_31:.*]] = arith.andi %[[VAL_29]], %[[VAL_30]] : i1
+// CHECK:             %[[VAL_32:.*]] = fir.if %[[VAL_31]] -> (!fir.boxchar<4>) {
+// CHECK:               %[[VAL_33:.*]] = fir.convert %[[VAL_28]] : (i64) -> index
+// CHECK:               %[[VAL_34:.*]] = hlfir.designate %[[VAL_18]]#0 (%[[VAL_33]], %[[VAL_23]])  typeparams %[[VAL_7]]#1 : (!fir.box<!fir.array<?x?x!fir.char<4,?>>>, index, index, index) -> !fir.boxchar<4>
+// CHECK:               fir.result %[[VAL_34]] : !fir.boxchar<4>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_26]] : !fir.boxchar<4>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_32]] : !fir.boxchar<4>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_21]] to %[[VAL_18]]#0 : !hlfir.expr<?x?x!fir.char<4,?>>, !fir.box<!fir.array<?x?x!fir.char<4,?>>>
+// CHECK:           hlfir.destroy %[[VAL_21]] : !hlfir.expr<?x?x!fir.char<4,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the array optional boundary.
+// ! CHARACTER with constant length.
+// subroutine eoshift14c(n, array, boundary)
+//   integer :: n
+//   character(10,1) :: array(n,n)
+//   character(10,1), optional :: boundary(n)
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift14c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}, %arg2: !fir.boxchar<1> {fir.bindc_name = "boundary", fir.optional}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift14cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<1,10>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.load %1#0 : !fir.ref<i32>
+  %9 = fir.convert %8 : (i32) -> index
+  %10 = arith.cmpi sgt, %9, %c0 : index
+  %11 = arith.select %10, %9, %c0 : index
+  %12 = fir.shape %7, %11 : (index, index) -> !fir.shape<2>
+  %13:2 = hlfir.declare %3(%12) typeparams %c10 dummy_scope %0 {uniq_name = "_QFeoshift14cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<1,10>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x?x!fir.char<1,10>>>)
+  %14:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %15 = fir.convert %14#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,10>>>
+  %16 = fir.load %1#0 : !fir.ref<i32>
+  %17 = fir.convert %16 : (i32) -> index
+  %18 = arith.cmpi sgt, %17, %c0 : index
+  %19 = arith.select %18, %17, %c0 : index
+  %20 = fir.shape %19 : (index) -> !fir.shape<1>
+  %21:2 = hlfir.declare %15(%20) typeparams %c10 dummy_scope %0 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift14cEboundary"} : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x!fir.char<1,10>>>)
+  %22 = fir.is_present %21#0 : (!fir.box<!fir.array<?x!fir.char<1,10>>>) -> i1
+  %23 = fir.shape %19 : (index) -> !fir.shape<1>
+  %24 = fir.embox %21#1(%23) : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.char<1,10>>>
+  %25 = fir.absent !fir.box<!fir.array<?x!fir.char<1,10>>>
+  %26 = arith.select %22, %24, %25 : !fir.box<!fir.array<?x!fir.char<1,10>>>
+  %27 = hlfir.eoshift %13#0 %c2_i32 boundary %26 : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, i32, !fir.box<!fir.array<?x!fir.char<1,10>>>) -> !hlfir.expr<?x?x!fir.char<1,10>>
+  hlfir.assign %27 to %13#0 : !hlfir.expr<?x?x!fir.char<1,10>>, !fir.box<!fir.array<?x?x!fir.char<1,10>>>
+  hlfir.destroy %27 : !hlfir.expr<?x?x!fir.char<1,10>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift14c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.boxchar<1> {fir.bindc_name = "boundary", fir.optional}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant 10 : index
+// CHECK:           %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_7]] {uniq_name = "_QFeoshift14cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_9:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<1,10>>>
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = fir.convert %[[VAL_11]] : (i32) -> index
+// CHECK:           %[[VAL_13:.*]] = arith.cmpi sgt, %[[VAL_12]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_14:.*]] = arith.select %[[VAL_13]], %[[VAL_12]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_15:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_16:.*]] = fir.convert %[[VAL_15]] : (i32) -> index
+// CHECK:           %[[VAL_17:.*]] = arith.cmpi sgt, %[[VAL_16]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_18:.*]] = arith.select %[[VAL_17]], %[[VAL_16]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_19:.*]] = fir.shape %[[VAL_14]], %[[VAL_18]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_20:.*]]:2 = hlfir.declare %[[VAL_10]](%[[VAL_19]]) typeparams %[[VAL_6]] dummy_scope %[[VAL_7]] {uniq_name = "_QFeoshift14cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<1,10>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x?x!fir.char<1,10>>>)
+// CHECK:           %[[VAL_21:.*]]:2 = fir.unboxchar %[[ARG2]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_22:.*]] = fir.convert %[[VAL_21]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           %[[VAL_23:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_24:.*]] = fir.convert %[[VAL_23]] : (i32) -> index
+// CHECK:           %[[VAL_25:.*]] = arith.cmpi sgt, %[[VAL_24]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_26:.*]] = arith.select %[[VAL_25]], %[[VAL_24]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_27:.*]] = fir.shape %[[VAL_26]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_28:.*]]:2 = hlfir.declare %[[VAL_22]](%[[VAL_27]]) typeparams %[[VAL_6]] dummy_scope %[[VAL_7]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift14cEboundary"} : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x!fir.char<1,10>>>)
+// CHECK:           %[[VAL_29:.*]] = fir.is_present %[[VAL_28]]#0 : (!fir.box<!fir.array<?x!fir.char<1,10>>>) -> i1
+// CHECK:           %[[VAL_30:.*]] = fir.shape %[[VAL_26]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_31:.*]] = fir.embox %[[VAL_28]]#1(%[[VAL_30]]) : (!fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>) -> !fir.box<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           %[[VAL_32:.*]] = fir.absent !fir.box<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           %[[VAL_33:.*]] = arith.select %[[VAL_29]], %[[VAL_31]], %[[VAL_32]] : !fir.box<!fir.array<?x!fir.char<1,10>>>
+// CHECK:           %[[VAL_34:.*]] = fir.convert %[[VAL_14]] : (index) -> i64
+// CHECK:           %[[VAL_35:.*]] = fir.convert %[[VAL_4]] : (i32) -> i64
+// CHECK:           %[[VAL_36:.*]] = fir.alloca !fir.char<1,0> {bindc_name = ".chrtmp"}
+// CHECK:           %[[VAL_37:.*]] = fir.emboxchar %[[VAL_36]], %[[VAL_5]] : (!fir.ref<!fir.char<1,0>>, index) -> !fir.boxchar<1>
+// CHECK:           %[[VAL_38:.*]] = fir.is_present %[[VAL_33]] : (!fir.box<!fir.array<?x!fir.char<1,10>>>) -> i1
+// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_38]], %[[VAL_2]], %[[VAL_3]] : i1
+// CHECK:           %[[VAL_40:.*]] = hlfir.elemental %[[VAL_19]] typeparams %[[VAL_6]] unordered : (!fir.shape<2>, index) -> !hlfir.expr<?x?x!fir.char<1,10>> {
+// CHECK:           ^bb0(%[[VAL_41:.*]]: index, %[[VAL_42:.*]]: index):
+// CHECK:             %[[VAL_43:.*]] = fir.if %[[VAL_39]] -> (!fir.boxchar<1>) {
+// CHECK:               fir.result %[[VAL_37]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               %[[VAL_44:.*]]:3 = fir.box_dims %[[VAL_33]], %[[VAL_5]] : (!fir.box<!fir.array<?x!fir.char<1,10>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_45:.*]] = arith.subi %[[VAL_44]]#0, %[[VAL_1]] overflow<nsw> : index
+// CHECK:               %[[VAL_46:.*]] = arith.addi %[[VAL_42]], %[[VAL_45]] overflow<nsw> : index
+// CHECK:               %[[VAL_47:.*]] = hlfir.designate %[[VAL_33]] (%[[VAL_46]])  typeparams %[[VAL_6]] : (!fir.box<!fir.array<?x!fir.char<1,10>>>, index, index) -> !fir.ref<!fir.char<1,10>>
+// CHECK:               %[[VAL_48:.*]] = fir.emboxchar %[[VAL_47]], %[[VAL_6]] : (!fir.ref<!fir.char<1,10>>, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_48]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             %[[VAL_49:.*]] = fir.convert %[[VAL_41]] : (index) -> i64
+// CHECK:             %[[VAL_50:.*]] = arith.addi %[[VAL_49]], %[[VAL_35]] overflow<nsw> : i64
+// CHECK:             %[[VAL_51:.*]] = arith.cmpi sge, %[[VAL_50]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_52:.*]] = arith.cmpi sle, %[[VAL_50]], %[[VAL_34]] : i64
+// CHECK:             %[[VAL_53:.*]] = arith.andi %[[VAL_51]], %[[VAL_52]] : i1
+// CHECK:             %[[VAL_54:.*]] = fir.if %[[VAL_53]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_55:.*]] = fir.convert %[[VAL_50]] : (i64) -> index
+// CHECK:               %[[VAL_56:.*]] = hlfir.designate %[[VAL_20]]#0 (%[[VAL_55]], %[[VAL_42]])  typeparams %[[VAL_6]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, index, index, index) -> !fir.ref<!fir.char<1,10>>
+// CHECK:               %[[VAL_57:.*]] = fir.emboxchar %[[VAL_56]], %[[VAL_6]] : (!fir.ref<!fir.char<1,10>>, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_57]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_43]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_54]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_40]] to %[[VAL_20]]#0 : !hlfir.expr<?x?x!fir.char<1,10>>, !fir.box<!fir.array<?x?x!fir.char<1,10>>>
+// CHECK:           hlfir.destroy %[[VAL_40]] : !hlfir.expr<?x?x!fir.char<1,10>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the array optional boundary.
+// ! CHARACTER with variable length.
+// subroutine eoshift15c(n, array, boundary)
+//   integer :: n
+//   character(n,1) :: array(n,n)
+//   character(n,1), optional :: boundary(n)
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift15c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}, %arg2: !fir.boxchar<1> {fir.bindc_name = "boundary", fir.optional}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c0_i32 = arith.constant 0 : i32
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift15cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<1,?>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = arith.cmpi sgt, %4, %c0_i32 : i32
+  %6 = arith.select %5, %4, %c0_i32 : i32
+  %7 = fir.load %1#0 : !fir.ref<i32>
+  %8 = fir.convert %7 : (i32) -> index
+  %9 = arith.cmpi sgt, %8, %c0 : index
+  %10 = arith.select %9, %8, %c0 : index
+  %11 = fir.load %1#0 : !fir.ref<i32>
+  %12 = fir.convert %11 : (i32) -> index
+  %13 = arith.cmpi sgt, %12, %c0 : index
+  %14 = arith.select %13, %12, %c0 : index
+  %15 = fir.shape %10, %14 : (index, index) -> !fir.shape<2>
+  %16:2 = hlfir.declare %3(%15) typeparams %6 dummy_scope %0 {uniq_name = "_QFeoshift15cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x?x!fir.char<1,?>>>)
+  %17:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %18 = fir.convert %17#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+  %19 = fir.load %1#0 : !fir.ref<i32>
+  %20 = arith.cmpi sgt, %19, %c0_i32 : i32
+  %21 = arith.select %20, %19, %c0_i32 : i32
+  %22 = fir.load %1#0 : !fir.ref<i32>
+  %23 = fir.convert %22 : (i32) -> index
+  %24 = arith.cmpi sgt, %23, %c0 : index
+  %25 = arith.select %24, %23, %c0 : index
+  %26 = fir.shape %25 : (index) -> !fir.shape<1>
+  %27:2 = hlfir.declare %18(%26) typeparams %21 dummy_scope %0 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift15cEboundary"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+  %28 = fir.is_present %27#0 : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> i1
+  %29 = fir.shape %25 : (index) -> !fir.shape<1>
+  %30 = fir.embox %27#1(%29) typeparams %21 : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, i32) -> !fir.box<!fir.array<?x!fir.char<1,?>>>
+  %31 = fir.absent !fir.box<!fir.array<?x!fir.char<1,?>>>
+  %32 = arith.select %28, %30, %31 : !fir.box<!fir.array<?x!fir.char<1,?>>>
+  %33 = hlfir.eoshift %16#0 %c2_i32 boundary %32 : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, i32, !fir.box<!fir.array<?x!fir.char<1,?>>>) -> !hlfir.expr<?x?x!fir.char<1,?>>
+  hlfir.assign %33 to %16#0 : !hlfir.expr<?x?x!fir.char<1,?>>, !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+  hlfir.destroy %33 : !hlfir.expr<?x?x!fir.char<1,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift15c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.boxchar<1> {fir.bindc_name = "boundary", fir.optional}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = arith.constant 0 : i32
+// CHECK:           %[[VAL_7:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_8:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_7]] {uniq_name = "_QFeoshift15cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_9:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_10:.*]] = fir.convert %[[VAL_9]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_11:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_12:.*]] = arith.cmpi sgt, %[[VAL_11]], %[[VAL_6]] : i32
+// CHECK:           %[[VAL_13:.*]] = arith.select %[[VAL_12]], %[[VAL_11]], %[[VAL_6]] : i32
+// CHECK:           %[[VAL_14:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i32) -> index
+// CHECK:           %[[VAL_16:.*]] = arith.cmpi sgt, %[[VAL_15]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_17:.*]] = arith.select %[[VAL_16]], %[[VAL_15]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_18:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_19:.*]] = fir.convert %[[VAL_18]] : (i32) -> index
+// CHECK:           %[[VAL_20:.*]] = arith.cmpi sgt, %[[VAL_19]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_21:.*]] = arith.select %[[VAL_20]], %[[VAL_19]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_22:.*]] = fir.shape %[[VAL_17]], %[[VAL_21]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_23:.*]]:2 = hlfir.declare %[[VAL_10]](%[[VAL_22]]) typeparams %[[VAL_13]] dummy_scope %[[VAL_7]] {uniq_name = "_QFeoshift15cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x?x!fir.char<1,?>>>)
+// CHECK:           %[[VAL_24:.*]]:2 = fir.unboxchar %[[ARG2]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_25:.*]] = fir.convert %[[VAL_24]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_26:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_27:.*]] = arith.cmpi sgt, %[[VAL_26]], %[[VAL_6]] : i32
+// CHECK:           %[[VAL_28:.*]] = arith.select %[[VAL_27]], %[[VAL_26]], %[[VAL_6]] : i32
+// CHECK:           %[[VAL_29:.*]] = fir.load %[[VAL_8]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_30:.*]] = fir.convert %[[VAL_29]] : (i32) -> index
+// CHECK:           %[[VAL_31:.*]] = arith.cmpi sgt, %[[VAL_30]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_32:.*]] = arith.select %[[VAL_31]], %[[VAL_30]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_33:.*]] = fir.shape %[[VAL_32]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_34:.*]]:2 = hlfir.declare %[[VAL_25]](%[[VAL_33]]) typeparams %[[VAL_28]] dummy_scope %[[VAL_7]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift15cEboundary"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, i32, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+// CHECK:           %[[VAL_35:.*]] = fir.is_present %[[VAL_34]]#0 : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> i1
+// CHECK:           %[[VAL_36:.*]] = fir.shape %[[VAL_32]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_37:.*]] = fir.embox %[[VAL_34]]#1(%[[VAL_36]]) typeparams %[[VAL_28]] : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, i32) -> !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_38:.*]] = fir.absent !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_39:.*]] = arith.select %[[VAL_35]], %[[VAL_37]], %[[VAL_38]] : !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_40:.*]] = fir.convert %[[VAL_17]] : (index) -> i64
+// CHECK:           %[[VAL_41:.*]] = fir.convert %[[VAL_4]] : (i32) -> i64
+// CHECK:           %[[VAL_42:.*]] = fir.alloca !fir.char<1,0> {bindc_name = ".chrtmp"}
+// CHECK:           %[[VAL_43:.*]] = fir.emboxchar %[[VAL_42]], %[[VAL_5]] : (!fir.ref<!fir.char<1,0>>, index) -> !fir.boxchar<1>
+// CHECK:           %[[VAL_44:.*]] = fir.is_present %[[VAL_39]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> i1
+// CHECK:           %[[VAL_45:.*]] = arith.select %[[VAL_44]], %[[VAL_2]], %[[VAL_3]] : i1
+// CHECK:           %[[VAL_46:.*]] = hlfir.elemental %[[VAL_22]] typeparams %[[VAL_13]] unordered : (!fir.shape<2>, i32) -> !hlfir.expr<?x?x!fir.char<1,?>> {
+// CHECK:           ^bb0(%[[VAL_47:.*]]: index, %[[VAL_48:.*]]: index):
+// CHECK:             %[[VAL_49:.*]] = fir.if %[[VAL_45]] -> (!fir.boxchar<1>) {
+// CHECK:               fir.result %[[VAL_43]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               %[[VAL_50:.*]] = fir.box_elesize %[[VAL_39]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> index
+// CHECK:               %[[VAL_51:.*]]:3 = fir.box_dims %[[VAL_39]], %[[VAL_5]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_52:.*]] = arith.subi %[[VAL_51]]#0, %[[VAL_1]] overflow<nsw> : index
+// CHECK:               %[[VAL_53:.*]] = arith.addi %[[VAL_48]], %[[VAL_52]] overflow<nsw> : index
+// CHECK:               %[[VAL_54:.*]] = hlfir.designate %[[VAL_39]] (%[[VAL_53]])  typeparams %[[VAL_50]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_54]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             %[[VAL_55:.*]] = fir.convert %[[VAL_47]] : (index) -> i64
+// CHECK:             %[[VAL_56:.*]] = arith.addi %[[VAL_55]], %[[VAL_41]] overflow<nsw> : i64
+// CHECK:             %[[VAL_57:.*]] = arith.cmpi sge, %[[VAL_56]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_58:.*]] = arith.cmpi sle, %[[VAL_56]], %[[VAL_40]] : i64
+// CHECK:             %[[VAL_59:.*]] = arith.andi %[[VAL_57]], %[[VAL_58]] : i1
+// CHECK:             %[[VAL_60:.*]] = fir.if %[[VAL_59]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_61:.*]] = fir.convert %[[VAL_56]] : (i64) -> index
+// CHECK:               %[[VAL_62:.*]] = hlfir.designate %[[VAL_23]]#0 (%[[VAL_61]], %[[VAL_48]])  typeparams %[[VAL_13]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index, index, i32) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_62]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_49]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_60]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_46]] to %[[VAL_23]]#0 : !hlfir.expr<?x?x!fir.char<1,?>>, !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:           hlfir.destroy %[[VAL_46]] : !hlfir.expr<?x?x!fir.char<1,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! Test contiguous 1D array with the array optional boundary.
+// ! CHARACTER with assumed length.
+// subroutine eoshift16c(n, array, boundary)
+//   integer :: n
+//   character(*,1) :: array(n,n)
+//   character(*,1), optional :: boundary(n)
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift16c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}, %arg2: !fir.boxchar<1> {fir.bindc_name = "boundary", fir.optional}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift16cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<1,?>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.load %1#0 : !fir.ref<i32>
+  %9 = fir.convert %8 : (i32) -> index
+  %10 = arith.cmpi sgt, %9, %c0 : index
+  %11 = arith.select %10, %9, %c0 : index
+  %12 = fir.shape %7, %11 : (index, index) -> !fir.shape<2>
+  %13:2 = hlfir.declare %3(%12) typeparams %2#1 dummy_scope %0 {uniq_name = "_QFeoshift16cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x?x!fir.char<1,?>>>)
+  %14:2 = fir.unboxchar %arg2 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %15 = fir.convert %14#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+  %16 = fir.load %1#0 : !fir.ref<i32>
+  %17 = fir.convert %16 : (i32) -> index
+  %18 = arith.cmpi sgt, %17, %c0 : index
+  %19 = arith.select %18, %17, %c0 : index
+  %20 = fir.shape %19 : (index) -> !fir.shape<1>
+  %21:2 = hlfir.declare %15(%20) typeparams %14#1 dummy_scope %0 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift16cEboundary"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+  %22 = fir.is_present %21#0 : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> i1
+  %23 = fir.shape %19 : (index) -> !fir.shape<1>
+  %24 = fir.embox %21#1(%23) typeparams %14#1 : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index) -> !fir.box<!fir.array<?x!fir.char<1,?>>>
+  %25 = fir.absent !fir.box<!fir.array<?x!fir.char<1,?>>>
+  %26 = arith.select %22, %24, %25 : !fir.box<!fir.array<?x!fir.char<1,?>>>
+  %27 = hlfir.eoshift %13#0 %c2_i32 boundary %26 : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, i32, !fir.box<!fir.array<?x!fir.char<1,?>>>) -> !hlfir.expr<?x?x!fir.char<1,?>>
+  hlfir.assign %27 to %13#0 : !hlfir.expr<?x?x!fir.char<1,?>>, !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+  hlfir.destroy %27 : !hlfir.expr<?x?x!fir.char<1,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift16c(
+// CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<i32> {fir.bindc_name = "n"},
+// CHECK-SAME:      %[[ARG1:.*]]: !fir.boxchar<1> {fir.bindc_name = "array"},
+// CHECK-SAME:      %[[ARG2:.*]]: !fir.boxchar<1> {fir.bindc_name = "boundary", fir.optional}) {
+// CHECK:           %[[VAL_0:.*]] = arith.constant 1 : i64
+// CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VAL_2:.*]] = arith.constant false
+// CHECK:           %[[VAL_3:.*]] = arith.constant true
+// CHECK:           %[[VAL_4:.*]] = arith.constant 2 : i32
+// CHECK:           %[[VAL_5:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_6:.*]] = fir.dummy_scope : !fir.dscope
+// CHECK:           %[[VAL_7:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_6]] {uniq_name = "_QFeoshift16cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+// CHECK:           %[[VAL_8:.*]]:2 = fir.unboxchar %[[ARG1]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_8]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_10:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_11:.*]] = fir.convert %[[VAL_10]] : (i32) -> index
+// CHECK:           %[[VAL_12:.*]] = arith.cmpi sgt, %[[VAL_11]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_13:.*]] = arith.select %[[VAL_12]], %[[VAL_11]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_14:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_15:.*]] = fir.convert %[[VAL_14]] : (i32) -> index
+// CHECK:           %[[VAL_16:.*]] = arith.cmpi sgt, %[[VAL_15]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_17:.*]] = arith.select %[[VAL_16]], %[[VAL_15]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_18:.*]] = fir.shape %[[VAL_13]], %[[VAL_17]] : (index, index) -> !fir.shape<2>
+// CHECK:           %[[VAL_19:.*]]:2 = hlfir.declare %[[VAL_9]](%[[VAL_18]]) typeparams %[[VAL_8]]#1 dummy_scope %[[VAL_6]] {uniq_name = "_QFeoshift16cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<1,?>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x?x!fir.char<1,?>>>)
+// CHECK:           %[[VAL_20:.*]]:2 = fir.unboxchar %[[ARG2]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+// CHECK:           %[[VAL_21:.*]] = fir.convert %[[VAL_20]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_22:.*]] = fir.load %[[VAL_7]]#0 : !fir.ref<i32>
+// CHECK:           %[[VAL_23:.*]] = fir.convert %[[VAL_22]] : (i32) -> index
+// CHECK:           %[[VAL_24:.*]] = arith.cmpi sgt, %[[VAL_23]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_25:.*]] = arith.select %[[VAL_24]], %[[VAL_23]], %[[VAL_5]] : index
+// CHECK:           %[[VAL_26:.*]] = fir.shape %[[VAL_25]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_27:.*]]:2 = hlfir.declare %[[VAL_21]](%[[VAL_26]]) typeparams %[[VAL_20]]#1 dummy_scope %[[VAL_6]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift16cEboundary"} : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.char<1,?>>>, !fir.ref<!fir.array<?x!fir.char<1,?>>>)
+// CHECK:           %[[VAL_28:.*]] = fir.is_present %[[VAL_27]]#0 : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> i1
+// CHECK:           %[[VAL_29:.*]] = fir.shape %[[VAL_25]] : (index) -> !fir.shape<1>
+// CHECK:           %[[VAL_30:.*]] = fir.embox %[[VAL_27]]#1(%[[VAL_29]]) typeparams %[[VAL_20]]#1 : (!fir.ref<!fir.array<?x!fir.char<1,?>>>, !fir.shape<1>, index) -> !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_31:.*]] = fir.absent !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_32:.*]] = arith.select %[[VAL_28]], %[[VAL_30]], %[[VAL_31]] : !fir.box<!fir.array<?x!fir.char<1,?>>>
+// CHECK:           %[[VAL_33:.*]] = fir.convert %[[VAL_13]] : (index) -> i64
+// CHECK:           %[[VAL_34:.*]] = fir.convert %[[VAL_4]] : (i32) -> i64
+// CHECK:           %[[VAL_35:.*]] = fir.alloca !fir.char<1,0> {bindc_name = ".chrtmp"}
+// CHECK:           %[[VAL_36:.*]] = fir.emboxchar %[[VAL_35]], %[[VAL_5]] : (!fir.ref<!fir.char<1,0>>, index) -> !fir.boxchar<1>
+// CHECK:           %[[VAL_37:.*]] = fir.is_present %[[VAL_32]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> i1
+// CHECK:           %[[VAL_38:.*]] = arith.select %[[VAL_37]], %[[VAL_2]], %[[VAL_3]] : i1
+// CHECK:           %[[VAL_39:.*]] = hlfir.elemental %[[VAL_18]] typeparams %[[VAL_8]]#1 unordered : (!fir.shape<2>, index) -> !hlfir.expr<?x?x!fir.char<1,?>> {
+// CHECK:           ^bb0(%[[VAL_40:.*]]: index, %[[VAL_41:.*]]: index):
+// CHECK:             %[[VAL_42:.*]] = fir.if %[[VAL_38]] -> (!fir.boxchar<1>) {
+// CHECK:               fir.result %[[VAL_36]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               %[[VAL_43:.*]] = fir.box_elesize %[[VAL_32]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>) -> index
+// CHECK:               %[[VAL_44:.*]]:3 = fir.box_dims %[[VAL_32]], %[[VAL_5]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index) -> (index, index, index)
+// CHECK:               %[[VAL_45:.*]] = arith.subi %[[VAL_44]]#0, %[[VAL_1]] overflow<nsw> : index
+// CHECK:               %[[VAL_46:.*]] = arith.addi %[[VAL_41]], %[[VAL_45]] overflow<nsw> : index
+// CHECK:               %[[VAL_47:.*]] = hlfir.designate %[[VAL_32]] (%[[VAL_46]])  typeparams %[[VAL_43]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, index, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_47]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             %[[VAL_48:.*]] = fir.convert %[[VAL_40]] : (index) -> i64
+// CHECK:             %[[VAL_49:.*]] = arith.addi %[[VAL_48]], %[[VAL_34]] overflow<nsw> : i64
+// CHECK:             %[[VAL_50:.*]] = arith.cmpi sge, %[[VAL_49]], %[[VAL_0]] : i64
+// CHECK:             %[[VAL_51:.*]] = arith.cmpi sle, %[[VAL_49]], %[[VAL_33]] : i64
+// CHECK:             %[[VAL_52:.*]] = arith.andi %[[VAL_50]], %[[VAL_51]] : i1
+// CHECK:             %[[VAL_53:.*]] = fir.if %[[VAL_52]] -> (!fir.boxchar<1>) {
+// CHECK:               %[[VAL_54:.*]] = fir.convert %[[VAL_49]] : (i64) -> index
+// CHECK:               %[[VAL_55:.*]] = hlfir.designate %[[VAL_19]]#0 (%[[VAL_54]], %[[VAL_41]])  typeparams %[[VAL_8]]#1 : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index, index, index) -> !fir.boxchar<1>
+// CHECK:               fir.result %[[VAL_55]] : !fir.boxchar<1>
+// CHECK:             } else {
+// CHECK:               fir.result %[[VAL_42]] : !fir.boxchar<1>
+// CHECK:             }
+// CHECK:             hlfir.yield_element %[[VAL_53]] : !fir.boxchar<1>
+// CHECK:           }
+// CHECK:           hlfir.assign %[[VAL_39]] to %[[VAL_19]]#0 : !hlfir.expr<?x?x!fir.char<1,?>>, !fir.box<!fir.array<?x?x!fir.char<1,?>>>
+// CHECK:           hlfir.destroy %[[VAL_39]] : !hlfir.expr<?x?x!fir.char<1,?>>
+// CHECK:           return
+// CHECK:         }
+
+// ! TODO: ARRAY or/and BOUNDARY are expressions of CHARACTER type.
+// ! Test contiguous 1D array with the array expression boundary.
+// ! CHARACTER with constant length.
+// subroutine eoshift17c(n, array)
+//   interface
+//      function charc_boundary(n)
+//        integer :: n
+//        character(10,1) :: charc_boundary(n)
+//      end function
+//   end interface
+//   integer :: n
+//   character(10,1) :: array(n,n)
+//   array = EOSHIFT(array//array, 2, charc_boundary(n))
+// end subroutine
+func.func @_QPeoshift17c(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.boxchar<1> {fir.bindc_name = "array"}) {
+  %c20 = arith.constant 20 : index
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %c10 = arith.constant 10 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift17cEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = fir.unboxchar %arg1 : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+  %3 = fir.convert %2#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.array<?x?x!fir.char<1,10>>>
+  %4 = fir.load %1#0 : !fir.ref<i32>
+  %5 = fir.convert %4 : (i32) -> index
+  %6 = arith.cmpi sgt, %5, %c0 : index
+  %7 = arith.select %6, %5, %c0 : index
+  %8 = fir.load %1#0 : !fir.ref<i32>
+  %9 = fir.convert %8 : (i32) -> index
+  %10 = arith.cmpi sgt, %9, %c0 : index
+  %11 = arith.select %10, %9, %c0 : index
+  %12 = fir.shape %7, %11 : (index, index) -> !fir.shape<2>
+  %13:2 = hlfir.declare %3(%12) typeparams %c10 dummy_scope %0 {uniq_name = "_QFeoshift17cEarray"} : (!fir.ref<!fir.array<?x?x!fir.char<1,10>>>, !fir.shape<2>, index, !fir.dscope) -> (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, !fir.ref<!fir.array<?x?x!fir.char<1,10>>>)
+  %14 = hlfir.elemental %12 typeparams %c20 unordered : (!fir.shape<2>, index) -> !hlfir.expr<?x?x!fir.char<1,?>> {
+  ^bb0(%arg2: index, %arg3: index):
+    %23 = hlfir.designate %13#0 (%arg2, %arg3)  typeparams %c10 : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, index, index, index) -> !fir.ref<!fir.char<1,10>>
+    %24 = hlfir.designate %13#0 (%arg2, %arg3)  typeparams %c10 : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, index, index, index) -> !fir.ref<!fir.char<1,10>>
+    %25 = hlfir.concat %23, %24 len %c20 : (!fir.ref<!fir.char<1,10>>, !fir.ref<!fir.char<1,10>>, index) -> !hlfir.expr<!fir.char<1,20>>
+    hlfir.yield_element %25 : !hlfir.expr<!fir.char<1,20>>
+  }
+  %15:2 = hlfir.declare %1#0 {uniq_name = "_QFeoshift17cFcharc_boundaryEn"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %16 = fir.load %15#0 : !fir.ref<i32>
+  %17 = fir.convert %16 : (i32) -> index
+  %18 = arith.cmpi sgt, %17, %c0 : index
+  %19 = arith.select %18, %17, %c0 : index
+  %20 = fir.shape %19 : (index) -> !fir.shape<1>
+  %21 = hlfir.eval_in_mem shape %20 typeparams %c10 : (!fir.shape<1>, index) -> !hlfir.expr<?x!fir.char<1,10>> {
+  ^bb0(%arg2: !fir.ref<!fir.array<?x!fir.char<1,10>>>):
+    %23 = fir.call @_QPcharc_boundary(%1#0) fastmath<contract> : (!fir.ref<i32>) -> !fir.array<?x!fir.char<1,10>>
+    fir.save_result %23 to %arg2(%20) typeparams %c10 : !fir.array<?x!fir.char<1,10>>, !fir.ref<!fir.array<?x!fir.char<1,10>>>, !fir.shape<1>, index
+  }
+  %22 = hlfir.eoshift %14 %c2_i32 boundary %21 : (!hlfir.expr<?x?x!fir.char<1,?>>, i32, !hlfir.expr<?x!fir.char<1,10>>) -> !hlfir.expr<?x?x!fir.char<1,20>>
+  hlfir.assign %22 to %13#0 : !hlfir.expr<?x?x!fir.char<1,20>>, !fir.box<!fir.array<?x?x!fir.char<1,10>>>
+  hlfir.destroy %22 : !hlfir.expr<?x?x!fir.char<1,20>>
+  hlfir.destroy %21 : !hlfir.expr<?x!fir.char<1,10>>
+  hlfir.destroy %14 : !hlfir.expr<?x?x!fir.char<1,?>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift17c(
+// CHECK: hlfir.eoshift
+
+// ! Tests for derived types.
+
+// ! TODO: selecting between !fir.ref<!fir.type<>> and !fir.box<!fir.type<>>
+// ! is not implemented.
+// ! Test contiguous 1D array with the scalar optional boundary.
+// subroutine eoshift1d(n, array, boundary)
+//   use eoshift_types
+//   integer :: n
+//   type(t) :: array(n)
+//   type(t), optional :: boundary
+//   array = EOSHIFT(array, 2, boundary)
+// end subroutine
+func.func @_QPeoshift1d(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.ref<!fir.array<?x!fir.type<_QMeoshift_typesTt>>> {fir.bindc_name = "array"}, %arg2: !fir.ref<!fir.type<_QMeoshift_typesTt>> {fir.bindc_name = "boundary", fir.optional}) {
+  %c2_i32 = arith.constant 2 : i32
+  %c0 = arith.constant 0 : index
+  %0 = fir.dummy_scope : !fir.dscope
+  %1:2 = hlfir.declare %arg0 dummy_scope %0 {uniq_name = "_QFeoshift1dEn"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = hlfir.declare %arg2 dummy_scope %0 {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift1dEboundary"} : (!fir.ref<!fir.type<_QMeoshift_typesTt>>, !fir.dscope) -> (!fir.ref<!fir.type<_QMeoshift_typesTt>>, !fir.ref<!fir.type<_QMeoshift_typesTt>>)
+  %3 = fir.load %1#0 : !fir.ref<i32>
+  %4 = fir.convert %3 : (i32) -> index
+  %5 = arith.cmpi sgt, %4, %c0 : index
+  %6 = arith.select %5, %4, %c0 : index
+  %7 = fir.shape %6 : (index) -> !fir.shape<1>
+  %8:2 = hlfir.declare %arg1(%7) dummy_scope %0 {uniq_name = "_QFeoshift1dEarray"} : (!fir.ref<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>, !fir.shape<1>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>, !fir.ref<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>)
+  %9 = fir.is_present %2#0 : (!fir.ref<!fir.type<_QMeoshift_typesTt>>) -> i1
+  %10 = fir.embox %2#0 : (!fir.ref<!fir.type<_QMeoshift_typesTt>>) -> !fir.box<!fir.type<_QMeoshift_typesTt>>
+  %11 = fir.absent !fir.box<!fir.type<_QMeoshift_typesTt>>
+  %12 = arith.select %9, %10, %11 : !fir.box<!fir.type<_QMeoshift_typesTt>>
+  %13 = hlfir.eoshift %8#0 %c2_i32 boundary %12 : (!fir.box<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>, i32, !fir.box<!fir.type<_QMeoshift_typesTt>>) -> !hlfir.expr<?x!fir.type<_QMeoshift_typesTt>>
+  hlfir.assign %13 to %8#0 : !hlfir.expr<?x!fir.type<_QMeoshift_typesTt>>, !fir.box<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>
+  hlfir.destroy %13 : !hlfir.expr<?x!fir.type<_QMeoshift_typesTt>>
+  return
+}
+// CHECK-LABEL:   func.func @_QPeoshift1d(
+// CHECK: hlfir.eoshift
diff --git a/flang/test/Integration/iso-fortran-binding.cpp b/flang/test/Integration/iso-fortran-binding.cpp
index aaafd7cccd07d..11f5c33bf14f2 100644
--- a/flang/test/Integration/iso-fortran-binding.cpp
+++ b/flang/test/Integration/iso-fortran-binding.cpp
@@ -1,33 +1,9 @@
+// REQUIRES: clang
 // UNSUPPORTED: system-windows
-// RUN: split-file %s %t
-// RUN: chmod +x %t/runtest.sh
-// RUN: %t/runtest.sh %t %t/cppfile.cpp %flang | FileCheck %s
+// RUN: %clang_cc1 -fsyntax-only -I%flang_include %s -x c++
 
-//--- cppfile.cpp
 extern "C" {
 #include "ISO_Fortran_binding.h"
 }
-#include <iostream>
 
-int main() {
-  std::cout << "PASS\n";
-  return 0;
-}
-
-// CHECK: PASS
-// clang-format off
-//--- runtest.sh
-#!/bin/bash
-TMPDIR=$1
-CPPFILE=$2
-FLANG=$3
-BINDIR=`dirname $FLANG`
-CPPCOMP=$BINDIR/clang++
-if [ -x $CPPCOMP ]
-then
-  $CPPCOMP $CPPFILE -o $TMPDIR/a.out
-  $TMPDIR/a.out # should print "PASS"
-else
-  # No clang compiler, just pass by default
-  echo "PASS"
-fi
+int main() { return 0; }
diff --git a/flang/test/Lower/CUDA/cuda-data-transfer.cuf b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
index 3a4aff977b7a5..aef926b09a1ed 100644
--- a/flang/test/Lower/CUDA/cuda-data-transfer.cuf
+++ b/flang/test/Lower/CUDA/cuda-data-transfer.cuf
@@ -13,6 +13,8 @@ module mod1
 
   integer, device, dimension(11:20) :: cdev
 
+  real(kind=8), device, allocatable, dimension(:) :: p
+
 contains
   function dev1(a)
     integer, device :: a(:)
@@ -444,3 +446,79 @@ subroutine sub23(n)
 end subroutine
 
 ! CHECK-LABEL: func.func @_QPsub23
+
+subroutine sub24()
+  real, managed :: m
+  real, device :: d(4)
+  m = d(1)
+end
+
+! CHECK-LABEL: func.func @_QPsub24()
+! CHECK: %[[D:.*]]:2 = hlfir.declare %1(%2) {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub24Ed"} : (!fir.ref<!fir.array<4xf32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<4xf32>>, !fir.ref<!fir.array<4xf32>>)
+! CHECK: %[[M:.*]]:2 = hlfir.declare %4 {data_attr = #cuf.cuda<managed>, uniq_name = "_QFsub24Em"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK: %[[D1:.*]] = hlfir.designate %[[D]]#0 (%c1{{.*}})  : (!fir.ref<!fir.array<4xf32>>, index) -> !fir.ref<f32>
+! CHECK: cuf.data_transfer %[[D1]] to %[[M]]#0 {transfer_kind = #cuf.cuda_transfer<device_device>} : !fir.ref<f32>, !fir.ref<f32>
+
+subroutine sub25()
+  use mod1
+  integer :: i
+  real(8) :: c
+
+  do i = 1, 10
+    c = c + p(i)
+  end do
+end
+
+! CHECK-LABEL: func.func @_QPsub25()
+! CHECK: fir.allocmem !fir.array<?xf64>, %15#1 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK: cuf.data_transfer %{{.*}} to %{{.*}} {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<!fir.box<!fir.heap<!fir.array<?xf64>>>>, !fir.box<!fir.array<?xf64>>
+! CHECK: hlfir.assign %{{.*}} to %{{.*}} : f64, !fir.ref<f64>
+! CHECK: fir.freemem %{{.*}} : !fir.heap<!fir.array<?xf64>>
+
+subroutine sub26(i, j, k)
+  integer :: i, j, k
+  real(2), dimension(i,j,k), device :: d
+  real(4), dimension(i,j,k) :: hd
+
+  hd = d
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub26
+! CHECK: %[[ALLOC_D:.*]] = cuf.alloc !fir.array<?x?x?xf16>, %{{.*}}, %{{.*}}, %{{.*}} : index, index, index {bindc_name = "d", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub26Ed"} -> !fir.ref<!fir.array<?x?x?xf16>>
+! CHECK: %[[D:.*]]:2 = hlfir.declare %[[ALLOC_D]](%{{.*}}) {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub26Ed"} : (!fir.ref<!fir.array<?x?x?xf16>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x?xf16>>, !fir.ref<!fir.array<?x?x?xf16>>)
+! CHECK: %[[HD:.*]]:2 = hlfir.declare %{{.*}}(%{{.*}}) {uniq_name = "_QFsub26Ehd"} : (!fir.ref<!fir.array<?x?x?xf32>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x?xf32>>, !fir.ref<!fir.array<?x?x?xf32>>)
+! CHECK: %[[ALLOC:.*]] = fir.allocmem !fir.array<?x?x?xf16>, %8, %13, %18 {bindc_name = ".tmp", uniq_name = ""}
+! CHECK: %[[TEMP:.*]]:2 = hlfir.declare %[[ALLOC]](%{{.*}}) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<?x?x?xf16>>, !fir.shape<3>) -> (!fir.box<!fir.array<?x?x?xf16>>, !fir.heap<!fir.array<?x?x?xf16>>)
+! CHECK: cuf.data_transfer %[[D]]#0 to %[[TEMP]]#0 {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.box<!fir.array<?x?x?xf16>>, !fir.box<!fir.array<?x?x?xf16>>
+! CHECK: %[[ELE:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<3>) -> !hlfir.expr<?x?x?xf32> {
+! CHECK: ^bb0(%{{.*}}: index, %{{.*}}: index, %{{.*}}: index):
+! CHECK: %[[DESIGNATE:.*]] = hlfir.designate %[[TEMP]]#0 (%{{.*}}, %{{.*}}, %{{.*}})  : (!fir.box<!fir.array<?x?x?xf16>>, index, index, index) -> !fir.ref<f16>
+! CHECK: %[[LOAD:.*]] = fir.load %[[DESIGNATE]] : !fir.ref<f16>
+! CHECK: %[[CONV:.*]] = fir.convert %[[LOAD]] : (f16) -> f32
+! CHECK: hlfir.yield_element %[[CONV]] : f32
+! CHECK: }
+! CHECK: hlfir.assign %[[ELE]] to %[[HD]]#0 : !hlfir.expr<?x?x?xf32>, !fir.box<!fir.array<?x?x?xf32>> 
+
+subroutine sub27()
+  real(2), dimension(10, 20, 30), device :: d
+  real(4), dimension(10, 20, 30) :: hd
+
+  hd = d
+end subroutine
+
+! CHECK-LABEL: func.func @_QPsub27()
+! CHECK: %[[ALLOC_D:.*]] = cuf.alloc !fir.array<10x20x30xf16> {bindc_name = "d", data_attr = #cuf.cuda<device>, uniq_name = "_QFsub27Ed"} -> !fir.ref<!fir.array<10x20x30xf16>>
+! CHECK: %[[D:.*]]:2 = hlfir.declare %[[ALLOC_D]](%{{.*}}) {data_attr = #cuf.cuda<device>, uniq_name = "_QFsub27Ed"} : (!fir.ref<!fir.array<10x20x30xf16>>, !fir.shape<3>) -> (!fir.ref<!fir.array<10x20x30xf16>>, !fir.ref<!fir.array<10x20x30xf16>>)
+! CHECK: %[[ALLOC_HD:.*]] = fir.alloca !fir.array<10x20x30xf32> {bindc_name = "hd", uniq_name = "_QFsub27Ehd"}
+! CHECK: %[[HD:.*]]:2 = hlfir.declare %[[ALLOC_HD]](%{{.*}}) {uniq_name = "_QFsub27Ehd"} : (!fir.ref<!fir.array<10x20x30xf32>>, !fir.shape<3>) -> (!fir.ref<!fir.array<10x20x30xf32>>, !fir.ref<!fir.array<10x20x30xf32>>)
+! CHECK: %[[ALLOC_TEMP:.*]] = fir.allocmem !fir.array<10x20x30xf16> {bindc_name = ".tmp", uniq_name = ""}
+! CHECK: %[[TEMP:.*]]:2 = hlfir.declare %[[ALLOC_TEMP]](%{{.*}}) {uniq_name = ".tmp"} : (!fir.heap<!fir.array<10x20x30xf16>>, !fir.shape<3>) -> (!fir.heap<!fir.array<10x20x30xf16>>, !fir.heap<!fir.array<10x20x30xf16>>)
+! CHECK: cuf.data_transfer %[[D]]#0 to %[[TEMP]]#0 {transfer_kind = #cuf.cuda_transfer<device_host>} : !fir.ref<!fir.array<10x20x30xf16>>, !fir.heap<!fir.array<10x20x30xf16>>
+! CHECK: %[[ELE:.*]] = hlfir.elemental %{{.*}} unordered : (!fir.shape<3>) -> !hlfir.expr<10x20x30xf32> {
+! CHECK: ^bb0(%{{.*}}: index, %{{.*}}: index, %{{.*}}: index):
+! CHECK: %[[DESIGNATE:.*]] = hlfir.designate %[[TEMP]]#0 (%{{.*}}, %{{.*}}, %{{.*}})  : (!fir.heap<!fir.array<10x20x30xf16>>, index, index, index) -> !fir.ref<f16>
+! CHECK: %[[LOAD:.*]] = fir.load %[[DESIGNATE]] : !fir.ref<f16>
+! CHECK: %[[CONV:.*]] = fir.convert %[[LOAD]] : (f16) -> f32
+! CHECK: hlfir.yield_element %[[CONV]] : f32
+! CHECK: }
+! CHECKL: hlfir.assign %[[ELE]] to %[[HD]]#0 : !hlfir.expr<10x20x30xf32>, !fir.ref<!fir.array<10x20x30xf32>>
diff --git a/flang/test/Lower/CUDA/cuda-device-proc.cuf b/flang/test/Lower/CUDA/cuda-device-proc.cuf
index d5e614a83b354..5e1f6b66d1d53 100644
--- a/flang/test/Lower/CUDA/cuda-device-proc.cuf
+++ b/flang/test/Lower/CUDA/cuda-device-proc.cuf
@@ -5,12 +5,15 @@
 attributes(global) subroutine devsub()
   implicit none
   integer :: ret
+  real(2) :: r2
   real(4) :: af
   real(8) :: ad
   integer(4) :: ai
   integer(8) :: al
   integer(8) :: time
   integer :: smalltime
+  integer(4) :: res
+  integer(8) :: resl
 
   call syncthreads()
   call syncwarp(1)
@@ -49,6 +52,46 @@ attributes(global) subroutine devsub()
   smalltime = clock()
   time = clock64()
   time = globalTimer()
+
+  res = __popc(ai)
+  res = __popc(al)
+  res = __ffs(ai)
+  res = __ffs(al)
+  res = __brev(ai)
+  resl = __brev(al)
+
+  res = __clz(ai)
+  res = __clz(al)  
+  af = __cosf(af)
+  ad = __ddiv_rn(ad, ad)
+  ad = __ddiv_rz(ad, ad)
+  ad = __ddiv_ru(ad, ad)
+  ad = __ddiv_rd(ad, ad)
+  af = __double2float_rn(ad)
+  af = __double2float_rz(ad)
+  af = __double2float_ru(ad)
+  af = __double2float_rd(ad)
+  ai = __double2int_rd(ad)
+  ai = __double2int_rn(ad)
+  ai = __double2int_ru(ad)
+  ai = __double2int_rz(ad)
+  ai = __double2uint_rd(ad)
+  ai = __double2uint_rn(ad)
+  ai = __double2uint_ru(ad)
+  ai = __double2uint_rz(ad)
+  ai = __mul24(ai, ai)
+  ai = __umul24(ai, ai)
+  af = __powf(af, af)
+  ad = __ull2double_rd(al)
+  ad = __ull2double_rn(al)
+  ad = __ull2double_ru(al)
+  ad = __ull2double_rz(al)
+  r2 = __float2half_rn(af)
+  af = __half2float(r2)
+  ad = __ll2double_rd(al)
+  ad = __ll2double_rn(al)
+  ad = __ll2double_ru(al)
+  ad = __ll2double_rz(al)
 end
 
 ! CHECK-LABEL: func.func @_QPdevsub() attributes {cuf.proc_attr = #cuf.cuda_proc<global>}
@@ -89,6 +132,45 @@ end
 ! CHECK: %{{.*}} = nvvm.read.ptx.sreg.clock64 : i64
 ! CHECK: %{{.*}} = nvvm.read.ptx.sreg.globaltimer : i64
 
+! CHECK: %{{.*}} = fir.call @__nv_popc(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_popcll(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_ffs(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_ffsll(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_brev(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_brevll(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_clz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_clzll(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_fast_cosf(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_ddiv_rn(%{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64, f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_ddiv_rz(%{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64, f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_ddiv_ru(%{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64, f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_ddiv_rd(%{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64, f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_double2float_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_double2float_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_double2float_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_double2float_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_double2int_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_double2int_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_double2int_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_double2int_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_double2uint_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_double2uint_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_double2uint_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_double2uint_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_mul24(%{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32, i32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_umul24(%{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32, i32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_fast_powf(%{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32, f32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_ull2double_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_ull2double_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_ull2double_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_ull2double_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_float2half_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f16
+! CHECK: %{{.*}} = fir.call @__nv_half2float(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f16) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_ll2double_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_ll2double_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_ll2double_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_ll2double_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f64
+
 subroutine host1()
   integer, device :: a(32)
   integer, device :: ret
diff --git a/flang/test/Lower/CUDA/cuda-libdevice.cuf b/flang/test/Lower/CUDA/cuda-libdevice.cuf
new file mode 100644
index 0000000000000..d243c49f05160
--- /dev/null
+++ b/flang/test/Lower/CUDA/cuda-libdevice.cuf
@@ -0,0 +1,335 @@
+! RUN: bbc -emit-hlfir -fcuda %s -o - | FileCheck %s
+
+! Test CUDA Fortran procedures available in cudadevice module
+
+attributes(global) subroutine test_sad()
+  integer :: res
+  integer :: i, j, k
+  res = __sad(i, j, k)
+end subroutine
+
+! CHECK-LABEL: _QPtest_sad
+! CHECK: %{{.*}} = fir.call @__nv_sad(%{{.*}}, %{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32, i32, i32) -> i32
+
+attributes(global) subroutine test_usad()
+  integer :: res
+  integer :: i, j, k
+  res = __usad(i, j, k)
+end subroutine
+
+! CHECK-LABEL: _QPtest_usad
+! CHECK: %{{.*}} = fir.call @__nv_usad(%{{.*}}, %{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32, i32, i32) -> i32
+
+attributes(global) subroutine test_dsqrt_rX()
+  double precision :: res
+  double precision :: p
+  res = __dsqrt_rd(p)
+  res = __dsqrt_rn(p)
+  res = __dsqrt_ru(p)
+  res = __dsqrt_rz(p)
+end subroutine
+
+! CHECK-LABEL: _QPtest_dsqrt_rx
+! CHECK: %{{.*}} = fir.call @__nv_dsqrt_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_dsqrt_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_dsqrt_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_dsqrt_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f64
+
+attributes(global) subroutine test_uint2float_rX()
+  real :: res
+  integer :: i
+  res = __uint2float_rd(i)
+  res = __uint2float_rn(i)
+  res = __uint2float_ru(i)
+  res = __uint2float_rz(i)
+end subroutine
+
+! CHECK-LABEL: _QPtest_uint2float_rx
+! CHECK: %{{.*}} = fir.call @__nv_uint2float_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_uint2float_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_uint2float_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_uint2float_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f32
+
+attributes(global) subroutine test_uint2double_rn()
+  double precision :: res
+  integer :: i
+  res = __uint2double_rn(i)
+end subroutine
+
+! CHECK-LABEL: _QPtest_uint2double_rn
+! CHECK: %{{.*}} = fir.call @__nv_uint2double_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f64
+
+attributes(global) subroutine test_ull2dloat_rX()
+  real :: res
+  integer(8) :: i
+  res = __ull2float_rd(i)
+  res = __ull2float_rn(i)
+  res = __ull2float_ru(i)
+  res = __ull2float_rz(i)
+end subroutine
+
+! CHECK-LABEL: _QPtest_ull2dloat_rx
+! CHECK: %{{.*}} = fir.call @__nv_ull2float_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_ull2float_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_ull2float_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_ull2float_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f32
+
+attributes(global) subroutine test_log()
+  real :: res
+  real :: r
+  res = __logf(r)
+  res = __log2f(r)
+  res = __log10f(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_log
+! CHECK: %{{.*}} = fir.call @__nv_fast_logf(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_fast_log2f(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_fast_log10f(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f32
+
+attributes(global) subroutine test_sincosf()
+  real :: r, s, c
+  call __sincosf(r, s, c)
+end subroutine
+
+! CHECK-LABEL: _QPtest_sincosf
+! CHECK: fir.call @__nv_fast_sincosf(%{{.*}}, %{{.*}}#0, %{{.*}}#0) proc_attrs<bind_c> fastmath<contract> : (f32, !fir.ref<f32>, !fir.ref<f32>) -> () 
+
+attributes(global) subroutine test_sinf()
+  real :: res
+  real :: r
+  res = __sinf(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_sinf
+! CHECK: %{{.*}} = fir.call @__nv_fast_sinf(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f32
+
+attributes(global) subroutine test_tanf()
+  real :: res
+  real :: r
+  res = __tanf(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_tanf
+! CHECK: %{{.*}} = fir.call @__nv_fast_tanf(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f32
+
+attributes(global) subroutine test_exp()
+  real :: res
+  real :: r
+  res = __expf(r)
+  res = __exp10f(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_exp
+! CHECK: %{{.*}} = fir.call @__nv_fast_expf(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_fast_exp10f(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f32
+
+attributes(global) subroutine test_double2ll_rX()
+  integer(8) :: res
+  double precision :: r
+  res = __double2ll_rd(r)
+  res = __double2ll_rn(r)
+  res = __double2ll_ru(r)
+  res = __double2ll_rz(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_double2ll_rx
+! CHECK: %{{.*}} = fir.call @__nv_double2ll_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_double2ll_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_double2ll_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_double2ll_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i64
+
+attributes(global) subroutine test_drcp_rX()
+  double precision :: res
+  double precision :: r
+  res = __drcp_rd(r)
+  res = __drcp_rn(r)
+  res = __drcp_ru(r)
+  res = __drcp_rz(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_drcp_rx
+! CHECK: %{{.*}} = fir.call @__nv_drcp_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_drcp_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_drcp_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f64
+! CHECK: %{{.*}} = fir.call @__nv_drcp_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> f64
+
+attributes(global) subroutine test_double2ull_rX()
+  integer(8) :: res
+  double precision :: r
+  res = __double2ull_rd(r)
+  res = __double2ull_rn(r)
+  res = __double2ull_ru(r)
+  res = __double2ull_rz(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_double2ull_rx
+! CHECK: %{{.*}} = fir.call @__nv_double2ull_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_double2ull_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_double2ull_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_double2ull_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i64
+
+attributes(global) subroutine test_saturatef()
+  real :: res
+  real :: r
+  res = __saturatef(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_saturatef
+! CHECK: %{{.*}} = fir.call @__nv_saturatef(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> f32
+
+attributes(global) subroutine test_float2ll_rX()
+  integer(8) :: res
+  real :: r
+  res = __float2ll_rd(r)
+  res = __float2ll_rn(r)
+  res = __float2ll_ru(r)
+  res = __float2ll_rz(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_float2ll_rx
+! CHECK: %{{.*}} = fir.call @__nv_float2ll_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_float2ll_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_float2ll_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i64
+! CHECK: %{{.*}} = fir.call @__nv_float2ll_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i64
+
+attributes(global) subroutine test_ll2float_rX()
+  real :: res
+  integer(8) :: i
+  res = __ll2float_rd(i)
+  res = __ll2float_rn(i)
+  res = __ll2float_ru(i)
+  res = __ll2float_rz(i)
+end subroutine
+
+! CHECK-LABEL: _QPtest_ll2float_rx
+! CHECK: %{{.*}} = fir.call @__nv_ll2float_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_ll2float_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_ll2float_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_ll2float_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f32
+
+attributes(global) subroutine test_int2float_rX()
+  real :: res
+  integer :: i
+  res = __int2float_rd(i)
+  res = __int2float_rn(i)
+  res = __int2float_ru(i)
+  res = __int2float_rz(i)
+end subroutine
+
+! CHECK-LABEL: _QPtest_int2float_rx
+! CHECK: %{{.*}} = fir.call @__nv_int2float_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_int2float_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_int2float_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f32
+! CHECK: %{{.*}} = fir.call @__nv_int2float_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f32
+
+attributes(global) subroutine test_float2int_rX()
+  integer :: res
+  real :: r
+  res = __float2int_rd(r)
+  res = __float2int_rn(r)
+  res = __float2int_ru(r)
+  res = __float2int_rz(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_float2int_rx
+! CHECK: %{{.*}} = fir.call @__nv_float2int_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i32    
+! CHECK: %{{.*}} = fir.call @__nv_float2int_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_float2int_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_float2int_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i32
+
+attributes(global) subroutine test_float2uint_rX()
+  integer :: res
+  real :: r
+  res = __float2uint_rd(r)
+  res = __float2uint_rn(r)
+  res = __float2uint_ru(r)
+  res = __float2uint_rz(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_float2uint_rx
+! CHECK: %{{.*}} = fir.call @__nv_float2uint_rd(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_float2uint_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_float2uint_ru(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i32
+! CHECK: %{{.*}} = fir.call @__nv_float2uint_rz(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i32
+
+attributes(global) subroutine test_int2double_rn()
+  double precision :: res
+  integer :: r
+  res = __int2double_rn(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_int2double_rn
+! CHECK: %{{.*}} = fir.call @__nv_int2double_rn(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f64
+
+attributes(global) subroutine test_fdividef()
+  real :: res
+  real :: r
+  res = __fdividef(r, r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_fdividef
+! CHECK: %{{.*}} = fir.call @__nv_fast_fdividef(%{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32, f32) -> f32
+
+attributes(global) subroutine test_double_as_longlong()
+  integer(8) :: res
+  real(8) :: r
+  res = double_as_longlong(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_double_as_longlong
+! CHECK: %{{.*}} = fir.call @__nv_double_as_longlong(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i64
+
+attributes(global) subroutine test_longlong_as_double()
+  integer(8) :: i
+  real(8) :: res
+  res = longlong_as_double(i)
+end subroutine
+
+! CHECK-LABEL: _QPtest_longlong_as_double
+! CHECK: %{{.*}} = fir.call @__nv_longlong_as_double(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i64) -> f64
+
+attributes(global) subroutine test_int_as_float()
+  integer :: i
+  real :: res
+  res = int_as_float(i)
+end subroutine
+
+! CHECK-LABEL: _QPtest_int_as_float
+! CHECK: %{{.*}} = fir.call @__nv_int_as_float(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32) -> f32
+
+attributes(global) subroutine test_float_as_int()
+  integer :: res
+  real :: r
+  res = float_as_int(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_float_as_int
+! CHECK: %{{.*}} = fir.call @__nv_float_as_int(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f32) -> i32
+
+attributes(global) subroutine test_double2loint()
+  integer :: res
+  double precision :: r
+  res = __double2loint(r)
+end subroutine
+
+! CHECK-LABEL: _QPtest_double2loint
+! CHECK: %{{.*}} = fir.call @__nv_double2loint(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+
+attributes(global) subroutine test_double2hiint()
+  integer :: res
+  double precision :: r
+  res = __double2hiint(r)
+end subroutine
+  
+! CHECK-LABEL: _QPtest_double2hiint
+! CHECK: %{{.*}} = fir.call @__nv_double2hiint(%{{.*}}) proc_attrs<bind_c> fastmath<contract> : (f64) -> i32
+
+attributes(global) subroutine test_hiloint2double()
+  double precision :: res
+  integer :: i, j
+  res = __hiloint2double(i, j)
+end subroutine
+
+! CHECK-LABEL: _QPtest_hiloint2double
+! CHECK: %{{.*}} = fir.call @__nv_hiloint2double(%{{.*}}, %{{.*}}) proc_attrs<bind_c> fastmath<contract> : (i32, i32) -> f64
diff --git a/flang/test/Lower/CUDA/cuda-set-allocator.cuf b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
index e3bb181f65398..d783f340fe9a4 100644
--- a/flang/test/Lower/CUDA/cuda-set-allocator.cuf
+++ b/flang/test/Lower/CUDA/cuda-set-allocator.cuf
@@ -23,34 +23,44 @@ contains
 
   subroutine sub2()
     type(ty_device), pointer :: d1
+    allocate(d1)
   end subroutine
 
 ! CHECK-LABEL: func.func @_QMm1Psub2()
 ! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub2Ed1"} -> !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QMm1Fsub2Ed1"} : (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
-! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
-! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
-! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
-! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
-! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
-! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: cuf.allocate
+! CHECK: %[[LOAD:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[COORD1:.*]] = fir.coordinate_of %[[ADDR]], x : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[COORD1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[COORD2:.*]] = fir.coordinate_of %[[ADDR]], z : (!fir.ptr<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[COORD2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
 
   subroutine sub3()
     type(ty_device), allocatable :: d1
+    allocate(d1)
   end subroutine
 
 ! CHECK-LABEL: func.func @_QMm1Psub3()
 ! CHECK: %[[ALLOC:.*]] = cuf.alloc !fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>> {bindc_name = "d1", data_attr = #cuf.cuda<managed>, uniq_name = "_QMm1Fsub3Ed1"} -> !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
 ! CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[ALLOC]] {data_attr = #cuf.cuda<managed>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QMm1Fsub3Ed1"} : (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>, !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>)
-! CHECK: %[[LOAD1:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
-! CHECK: %[[ADDR1:.*]] = fir.box_addr %[[LOAD1]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
-! CHECK: %[[DESIGNATE1:.*]] = hlfir.designate %[[ADDR1]]{"x"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK: cuf.set_allocator_idx %[[DESIGNATE1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
-! CHECK: %[[LOAD2:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
-! CHECK: %[[ADDR2:.*]] = fir.box_addr %[[LOAD2]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
-! CHECK: %[[DESIGNATE2:.*]] = hlfir.designate %[[ADDR2]]{"z"} : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
-! CHECK: cuf.set_allocator_idx %[[DESIGNATE2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: cuf.allocate
+! CHECK: %[[LOAD:.*]] = fir.load %[[DECL]]#0 : !fir.ref<!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>>
+! CHECK: %[[ADDR:.*]] = fir.box_addr %[[LOAD]] : (!fir.box<!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>>) -> !fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>
+! CHECK: %[[COORD1:.*]] = fir.coordinate_of %[[ADDR]], x : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[COORD1]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+! CHECK: %[[COORD2:.*]] = fir.coordinate_of %[[ADDR]], z : (!fir.heap<!fir.type<_QMm1Tty_device{x:!fir.box<!fir.heap<!fir.array<?xi32>>>,y:i32,z:!fir.box<!fir.heap<!fir.array<?xi32>>>}>>) -> !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+! CHECK: cuf.set_allocator_idx %[[COORD2]] : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>> {data_attr = #cuf.cuda<device>}
+
+  subroutine sub4()
+    type(ty_device), allocatable :: d1(:,:)
+    allocate(d1(10, 10))
+  end subroutine
+
+! CHECK-LABEL: func.func @_QMm1Psub4()
+! CHECK: cuf.allocate
+! CHECK-COUNT-2: fir.do_loop
+! CHECK-COUNT-2: cuf.set_allocator_idx
 
 end module
diff --git a/flang/test/Lower/Coarray/coarray-init.f90 b/flang/test/Lower/Coarray/coarray-init.f90
new file mode 100644
index 0000000000000..055bc0fc4da72
--- /dev/null
+++ b/flang/test/Lower/Coarray/coarray-init.f90
@@ -0,0 +1,11 @@
+! RUN: %flang_fc1 -emit-hlfir -fcoarray %s -o - | FileCheck %s --check-prefixes=ALL,COARRAY
+! RUN: %flang_fc1 -emit-hlfir %s -o - | FileCheck %s --check-prefixes=ALL,NOCOARRAY
+
+program test_init
+
+end 
+
+! ALL-LABEL: func.func @main
+! ALL: fir.call @_FortranAProgramStart
+! COARRAY: fir.call @_QMprifPprif_init(%[[ARG:.*]]) fastmath<contract> : (!fir.ref<i32>) -> ()
+! NOCOARRAY-NOT: fir.call @_QMprifPprif_init(%[[ARG:.*]]) fastmath<contract> : (!fir.ref<i32>) -> ()
diff --git a/flang/test/Lower/Coarray/num_images.f90 b/flang/test/Lower/Coarray/num_images.f90
new file mode 100644
index 0000000000000..ebfce5db0dbfb
--- /dev/null
+++ b/flang/test/Lower/Coarray/num_images.f90
@@ -0,0 +1,18 @@
+! RUN: %flang_fc1 -emit-hlfir -fcoarray %s -o - | FileCheck %s
+
+program test
+  use iso_fortran_env
+  integer :: i
+  integer :: team_number 
+  type(team_type) :: team
+
+  ! CHECK: fir.call @_QMprifPprif_num_images
+  i = num_images()
+
+  ! CHECK: fir.call @_QMprifPprif_num_images_with_team_number
+  i = num_images(TEAM_NUMBER=team_number)
+
+  ! CHECK: fir.call @_QMprifPprif_num_images_with_team
+  i = num_images(TEAM=team)
+
+end program
diff --git a/flang/test/Lower/Coarray/this_image.f90 b/flang/test/Lower/Coarray/this_image.f90
new file mode 100644
index 0000000000000..143504b5f922c
--- /dev/null
+++ b/flang/test/Lower/Coarray/this_image.f90
@@ -0,0 +1,14 @@
+! RUN: %flang_fc1 -emit-hlfir -fcoarray %s -o - | FileCheck %s
+
+program test
+  use iso_fortran_env
+  integer :: i
+  type(team_type) :: team
+
+  ! CHECK: fir.call @_QMprifPprif_this_image_no_coarray
+  i = this_image()
+
+  ! CHECK: fir.call @_QMprifPprif_this_image_no_coarray
+  i = this_image(TEAM=team)
+
+end program
diff --git a/flang/test/Lower/HLFIR/eoshift.f90 b/flang/test/Lower/HLFIR/eoshift.f90
new file mode 100644
index 0000000000000..e7fb98c2b0401
--- /dev/null
+++ b/flang/test/Lower/HLFIR/eoshift.f90
@@ -0,0 +1,271 @@
+! Test lowering of EOSHIFT intrinsic to HLFIR
+! RUN: bbc -emit-hlfir -o - -I nowhere %s 2>&1 | FileCheck %s
+
+module eoshift_types
+  type t
+  end type t
+end module eoshift_types
+
+! 1d shift by scalar
+subroutine eoshift1(a, s)
+  integer :: a(:), s
+  a = EOSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift1(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = hlfir.eoshift %[[VAL_3]]#0 %[[VAL_5]] : (!fir.box<!fir.array<?xi32>>, i32) -> !hlfir.expr<?xi32>
+! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_3]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_6]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 1d shift by scalar with dim
+subroutine eoshift2(a, s)
+  integer :: a(:), s
+  a = EOSHIFT(a, 2, dim=1)
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift2(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_7:.*]] = hlfir.eoshift %[[VAL_3]]#0 %[[VAL_5]] dim %[[VAL_6]] : (!fir.box<!fir.array<?xi32>>, i32, i32) -> !hlfir.expr<?xi32>
+! CHECK:           hlfir.assign %[[VAL_7]] to %[[VAL_3]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_7]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 2d shift by scalar
+subroutine eoshift3(a, s)
+  integer :: a(:,:), s
+  a = EOSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift3(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = hlfir.eoshift %[[VAL_3]]#0 %[[VAL_5]] : (!fir.box<!fir.array<?x?xi32>>, i32) -> !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_3]]#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_6]] : !hlfir.expr<?x?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 2d shift by scalar with dim
+subroutine eoshift4(a, s)
+  integer :: a(:,:), s
+  a = EOSHIFT(a, 2, dim=2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift4(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_7:.*]] = hlfir.eoshift %[[VAL_3]]#0 %[[VAL_5]] dim %[[VAL_6]] : (!fir.box<!fir.array<?x?xi32>>, i32, i32) -> !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.assign %[[VAL_7]] to %[[VAL_3]]#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_7]] : !hlfir.expr<?x?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 2d shift by array
+subroutine eoshift5(a, s)
+  integer :: a(:,:), s(:)
+  a = EOSHIFT(a, s)
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift5(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = hlfir.eoshift %[[VAL_3]]#0 %[[VAL_4]]#0 : (!fir.box<!fir.array<?x?xi32>>, !fir.box<!fir.array<?xi32>>) -> !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_3]]#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_5]] : !hlfir.expr<?x?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 2d shift by array expr
+subroutine eoshift6(a, s)
+  integer :: a(:,:), s(:)
+  a = EOSHIFT(a, s + 1)
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift6(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_6:.*]] = arith.constant 0 : index
+! CHECK:           %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_4]]#0, %[[VAL_6]] : (!fir.box<!fir.array<?xi32>>, index) -> (index, index, index)
+! CHECK:           %[[VAL_8:.*]] = fir.shape %[[VAL_7]]#1 : (index) -> !fir.shape<1>
+! CHECK:           %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] unordered : (!fir.shape<1>) -> !hlfir.expr<?xi32>
+! CHECK:           %[[VAL_14:.*]] = hlfir.eoshift %[[VAL_3]]#0 %[[VAL_9]] : (!fir.box<!fir.array<?x?xi32>>, !hlfir.expr<?xi32>) -> !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.assign %[[VAL_14]] to %[[VAL_3]]#0 : !hlfir.expr<?x?xi32>, !fir.box<!fir.array<?x?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_14]] : !hlfir.expr<?x?xi32>
+! CHECK:           hlfir.destroy %[[VAL_9]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
+
+! 1d character(10,2) shift by scalar
+subroutine eoshift7(a, s)
+  character(10,2) :: a(:)
+  a = EOSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift7(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.char<2,10>>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<f32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]] = arith.constant 10 : index
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_6:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_7:.*]] = hlfir.eoshift %[[VAL_4]]#0 %[[VAL_6]] : (!fir.box<!fir.array<?x!fir.char<2,10>>>, i32) -> !hlfir.expr<?x!fir.char<2,10>>
+! CHECK:           hlfir.assign %[[VAL_7]] to %[[VAL_4]]#0 : !hlfir.expr<?x!fir.char<2,10>>, !fir.box<!fir.array<?x!fir.char<2,10>>>
+! CHECK:           hlfir.destroy %[[VAL_7]] : !hlfir.expr<?x!fir.char<2,10>>
+! CHECK:           return
+! CHECK:         }
+
+! 1d character(*) shift by scalar
+subroutine eoshift8(a, s)
+  character(*) :: a(:)
+  a = EOSHIFT(a, 2)
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift8(
+! CHECK-SAME:                          %[[VAL_0:.*]]: !fir.box<!fir.array<?x!fir.char<1,?>>> {fir.bindc_name = "a"},
+! CHECK-SAME:                          %[[VAL_1:.*]]: !fir.ref<f32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_2:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0]]
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_1]]
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_6:.*]] = hlfir.eoshift %[[VAL_3]]#0 %[[VAL_5]] : (!fir.box<!fir.array<?x!fir.char<1,?>>>, i32) -> !hlfir.expr<?x!fir.char<1,?>>
+! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_3]]#0 : !hlfir.expr<?x!fir.char<1,?>>, !fir.box<!fir.array<?x!fir.char<1,?>>>
+! CHECK:           hlfir.destroy %[[VAL_6]] : !hlfir.expr<?x!fir.char<1,?>>
+! CHECK:           return
+! CHECK:         }
+
+! 1d type(t) shift by scalar
+subroutine eoshift9(a, s)
+  use eoshift_types
+  type(t) :: a(:)
+  a = EOSHIFT(a, 2, boundary=t())
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift9(
+! CHECK-SAME:      %[[ARG0:.*]]: !fir.box<!fir.array<?x!fir.type<_QMeoshift_typesTt>>> {fir.bindc_name = "a"},
+! CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<f32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_0]] {uniq_name = "_QFeoshift9Ea"} : (!fir.box<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>, !fir.dscope) -> (!fir.box<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>, !fir.box<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %[[VAL_0]] {uniq_name = "_QFeoshift9Es"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:           %[[VAL_3:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_4:.*]] = fir.address_of(@_QQro._QMeoshift_typesTt.0) : !fir.ref<!fir.type<_QMeoshift_typesTt>>
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQro._QMeoshift_typesTt.0"} : (!fir.ref<!fir.type<_QMeoshift_typesTt>>) -> (!fir.ref<!fir.type<_QMeoshift_typesTt>>, !fir.ref<!fir.type<_QMeoshift_typesTt>>)
+! CHECK:           %[[VAL_6:.*]] = hlfir.eoshift %[[VAL_1]]#0 %[[VAL_3]] boundary %[[VAL_5]]#0 : (!fir.box<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>, i32, !fir.ref<!fir.type<_QMeoshift_typesTt>>) -> !hlfir.expr<?x!fir.type<_QMeoshift_typesTt>>
+! CHECK:           hlfir.assign %[[VAL_6]] to %[[VAL_1]]#0 : !hlfir.expr<?x!fir.type<_QMeoshift_typesTt>>, !fir.box<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>
+! CHECK:           hlfir.destroy %[[VAL_6]] : !hlfir.expr<?x!fir.type<_QMeoshift_typesTt>>
+! CHECK:           return
+! CHECK:         }
+
+! 1d class(t) shift by scalar
+subroutine eoshift10(a, s)
+  use eoshift_types
+  class(t), allocatable :: a(:)
+  a = EOSHIFT(a, 2, boundary=t())
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift10(
+! CHECK-SAME:      %[[ARG0:.*]]: !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>>> {fir.bindc_name = "a"},
+! CHECK-SAME:      %[[ARG1:.*]]: !fir.ref<f32> {fir.bindc_name = "s"}) {
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_0]] {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFeoshift10Ea"} : (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>>>, !fir.dscope) -> (!fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>>>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %[[VAL_0]] {uniq_name = "_QFeoshift10Es"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:           %[[VAL_3:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_4:.*]] = fir.address_of(@_QQro._QMeoshift_typesTt.1) : !fir.ref<!fir.type<_QMeoshift_typesTt>>
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_4]] {fortran_attrs = #fir.var_attrs<parameter>, uniq_name = "_QQro._QMeoshift_typesTt.1"} : (!fir.ref<!fir.type<_QMeoshift_typesTt>>) -> (!fir.ref<!fir.type<_QMeoshift_typesTt>>, !fir.ref<!fir.type<_QMeoshift_typesTt>>)
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_1]]#0 : !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>>>
+! CHECK:           %[[VAL_7:.*]] = hlfir.eoshift %[[VAL_6]] %[[VAL_3]] boundary %[[VAL_5]]#0 : (!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>>, i32, !fir.ref<!fir.type<_QMeoshift_typesTt>>) -> !hlfir.expr<?x!fir.type<_QMeoshift_typesTt>?>
+! CHECK:           hlfir.assign %[[VAL_7]] to %[[VAL_1]]#0 realloc : !hlfir.expr<?x!fir.type<_QMeoshift_typesTt>?>, !fir.ref<!fir.class<!fir.heap<!fir.array<?x!fir.type<_QMeoshift_typesTt>>>>>
+! CHECK:           hlfir.destroy %[[VAL_7]] : !hlfir.expr<?x!fir.type<_QMeoshift_typesTt>?>
+! CHECK:           return
+! CHECK:         }
+
+! 1d shift by scalar with variable dim
+subroutine eoshift11(a, s, d)
+  integer :: a(:), s, d
+  a = EOSHIFT(a, 2, dim=d)
+end subroutine
+! CHECK-LABEL:   func.func @_QPeoshift11(
+! CHECK-SAME:                           %[[VAL_0:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "a"},
+! CHECK-SAME:                           %[[VAL_1:.*]]: !fir.ref<i32> {fir.bindc_name = "s"},
+! CHECK-SAME:                           %[[VAL_2:.*]]: !fir.ref<i32> {fir.bindc_name = "d"}) {
+! CHECK:           %[[VAL_3:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[VAL_0]] dummy_scope %[[VAL_3]] {uniq_name = "_QFeoshift11Ea"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_2]] dummy_scope %[[VAL_3]] {uniq_name = "_QFeoshift11Ed"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_1]] dummy_scope %[[VAL_3]] {uniq_name = "_QFeoshift11Es"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_7:.*]] = arith.constant 2 : i32
+! CHECK:           %[[VAL_8:.*]] = fir.load %[[VAL_5]]#0 : !fir.ref<i32>
+! CHECK:           %[[VAL_9:.*]] = hlfir.eoshift %[[VAL_4]]#0 %[[VAL_7]] dim %[[VAL_8]] : (!fir.box<!fir.array<?xi32>>, i32, i32) -> !hlfir.expr<?xi32>
+! CHECK:           hlfir.assign %[[VAL_9]] to %[[VAL_4]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK:           hlfir.destroy %[[VAL_9]] : !hlfir.expr<?xi32>
+! CHECK:           return
+! CHECK:         }
+
+subroutine eoshift12(array, shift, boundary, dim)
+  real :: array(:,:)
+  real, optional :: boundary
+  integer :: shift(:), dim
+  array = EOSHIFT(array, shift, boundary, dim)
+end subroutine eoshift12
+! CHECK-LABEL:   func.func @_QPeoshift12(
+! CHECK-SAME:      %[[ARG0:.*]]: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "array"},
+! CHECK-SAME:      %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "shift"},
+! CHECK-SAME:      %[[ARG2:.*]]: !fir.ref<f32> {fir.bindc_name = "boundary", fir.optional},
+! CHECK-SAME:      %[[ARG3:.*]]: !fir.ref<i32> {fir.bindc_name = "dim"}) {
+! CHECK:           %[[VAL_0:.*]] = fir.dummy_scope : !fir.dscope
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[ARG0]] dummy_scope %[[VAL_0]] {uniq_name = "_QFeoshift12Earray"} : (!fir.box<!fir.array<?x?xf32>>, !fir.dscope) -> (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?x?xf32>>)
+! CHECK:           %[[VAL_2:.*]]:2 = hlfir.declare %[[ARG2]] dummy_scope %[[VAL_0]] {fortran_attrs = #fir.var_attrs<optional>, uniq_name = "_QFeoshift12Eboundary"} : (!fir.ref<f32>, !fir.dscope) -> (!fir.ref<f32>, !fir.ref<f32>)
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[ARG3]] dummy_scope %[[VAL_0]] {uniq_name = "_QFeoshift12Edim"} : (!fir.ref<i32>, !fir.dscope) -> (!fir.ref<i32>, !fir.ref<i32>)
+! CHECK:           %[[VAL_4:.*]]:2 = hlfir.declare %[[ARG1]] dummy_scope %[[VAL_0]] {uniq_name = "_QFeoshift12Eshift"} : (!fir.box<!fir.array<?xi32>>, !fir.dscope) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+! CHECK:           %[[VAL_5:.*]] = fir.is_present %[[VAL_2]]#0 : (!fir.ref<f32>) -> i1
+! CHECK:           %[[VAL_6:.*]] = fir.embox %[[VAL_2]]#0 : (!fir.ref<f32>) -> !fir.box<f32>
+! CHECK:           %[[VAL_7:.*]] = fir.absent !fir.box<f32>
+! CHECK:           %[[VAL_8:.*]] = arith.select %[[VAL_5]], %[[VAL_6]], %[[VAL_7]] : !fir.box<f32>
+! CHECK:           %[[VAL_9:.*]] = fir.load %[[VAL_3]]#0 : !fir.ref<i32>
+! CHECK:           %[[VAL_10:.*]] = hlfir.eoshift %[[VAL_1]]#0 %[[VAL_4]]#0 boundary %[[VAL_8]] dim %[[VAL_9]] : (!fir.box<!fir.array<?x?xf32>>, !fir.box<!fir.array<?xi32>>, !fir.box<f32>, i32) -> !hlfir.expr<?x?xf32>
+! CHECK:           hlfir.assign %[[VAL_10]] to %[[VAL_1]]#0 : !hlfir.expr<?x?xf32>, !fir.box<!fir.array<?x?xf32>>
+! CHECK:           hlfir.destroy %[[VAL_10]] : !hlfir.expr<?x?xf32>
+! CHECK:           return
+! CHECK:         }
+
+! Test scalar logical boundary.
+! CHECK-LABEL:   func.func @_QPeoshift13(
+subroutine eoshift13(array)
+  logical(1) :: array(:)
+  array = EOSHIFT(array, -1, .true._1)
+! CHECK:           %[[VAL_5:.*]] = hlfir.eoshift %{{.*}} %{{.*}} boundary %{{.*}} : (!fir.box<!fir.array<?x!fir.logical<1>>>, i32, !fir.logical<1>) -> !hlfir.expr<?x!fir.logical<1>>
+  array = EOSHIFT(array.EQV..false., -1, .true.)
+! CHECK:           %[[VAL_24:.*]] = hlfir.eoshift %{{.*}} %{{.*}} boundary %{{.*}} : (!hlfir.expr<?x!fir.logical<4>>, i32, !fir.logical<4>) -> !hlfir.expr<?x!fir.logical<4>>
+end subroutine eoshift13
+
+! Test scalar constant BOUNDARY value of UNSIGNED type.
+! The BOUNDARY operand of hlfir.eoshift must have ui32 type
+! (i.e. consistent with the array/result type).
+! CHECK-LABEL:   func.func @_QPeoshift14(
+subroutine eoshift14(array)
+  unsigned :: array(:)
+  array = EOSHIFT(array, shift=1, boundary=1u)
+! CHECK-DAG:           %[[VAL_4:.*]] = fir.convert %[[VAL_3:.*]] : (i32) -> ui32
+! CHECK-DAG:           %[[VAL_3]] = arith.constant 1 : i32
+! CHECK:           %[[VAL_5:.*]] = hlfir.eoshift{{.*}}boundary %[[VAL_4]] : (!fir.box<!fir.array<?xui32>>, i32, ui32) -> !hlfir.expr<?xui32>
+end subroutine eoshift14
diff --git a/flang/test/Lower/Intrinsics/acosd.f90 b/flang/test/Lower/Intrinsics/acosd.f90
index 7dfa28fd6494e..175a4902620b8 100644
--- a/flang/test/Lower/Intrinsics/acosd.f90
+++ b/flang/test/Lower/Intrinsics/acosd.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -6,9 +7,8 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[factor:.*]] = arith.constant 57.2957763 : f32
 ! CHECK: %[[result:.*]] = math.acos %{{.*}} fastmath<contract> : f32
-! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
 ! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f32
 
 function test_real8(x)
@@ -17,6 +17,16 @@ function test_real8(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
-! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[factor:.*]] = arith.constant 57.295779513082323 : f64
 ! CHECK: %[[result:.*]] = math.acos %{{.*}} fastmath<contract> : f64
-! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[dfactor]] fastmath<contract> : f64
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f64
+
+function test_real16(x)
+  real(16) :: x, test_real16
+  test_real16 = acosd(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[factor:.*]] = arith.constant 57.295779513082320876798154814105{{.*}} : f128
+! CHECK: %[[result:.*]] = fir.call @_FortranAAcosF128({{.*}}) fastmath<contract> : (f128) -> f128
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f128
diff --git a/flang/test/Lower/Intrinsics/acospi.f90 b/flang/test/Lower/Intrinsics/acospi.f90
index dcacd25bca480..38c547fd5dd13 100644
--- a/flang/test/Lower/Intrinsics/acospi.f90
+++ b/flang/test/Lower/Intrinsics/acospi.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
 ! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
@@ -10,8 +11,7 @@ function test_real4(x)
 ! CHECK-LABEL: @_QPtest_real4
 ! CHECK-PRECISE: %[[acos:.*]] = fir.call @acosf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
 ! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f32
-! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
-! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.318309873 : f32
 ! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f32
 
 function test_real8(x)
@@ -24,3 +24,13 @@ function test_real8(x)
 ! CHECK-FAST: %[[acos:.*]] = math.acos %{{.*}} : f64
 ! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
 ! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f64
+
+function test_real16(x)
+  real(16) :: x, test_real16
+  test_real16 = acospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[acos:.*]] = fir.call @_FortranAAcosF128({{.*}}) fastmath<contract> : (f128) -> f128
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.3183098861837906715377675267450{{.*}} : f128
+! CHECK: %{{.*}} = arith.mulf %[[acos]], %[[inv_pi]] fastmath<contract> : f128
diff --git a/flang/test/Lower/Intrinsics/asind.f90 b/flang/test/Lower/Intrinsics/asind.f90
index 564fa955fa969..8d6198f53e4b4 100644
--- a/flang/test/Lower/Intrinsics/asind.f90
+++ b/flang/test/Lower/Intrinsics/asind.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -6,9 +7,8 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[factor:.*]] = arith.constant 57.2957763 : f32
 ! CHECK: %[[result:.*]] = math.asin %{{.*}} fastmath<contract> : f32
-! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
 ! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f32
 
 function test_real8(x)
@@ -17,6 +17,16 @@ function test_real8(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
-! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
+! CHECK: %[[factor:.*]] = arith.constant 57.295779513082323 : f64
 ! CHECK: %[[result:.*]] = math.asin %{{.*}} fastmath<contract> : f64
-! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[dfactor]] fastmath<contract> : f64
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f64
+
+function test_real16(x)
+  real(16) :: x, test_real16
+  test_real16 = asind(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[factor:.*]] = arith.constant 57.295779513082320876798154814105{{.*}} : f128
+! CHECK: %[[result:.*]] = fir.call @_FortranAAsinF128({{.*}}) fastmath<contract> : (f128) -> f128
+! CHECK: %[[arg:.*]] = arith.mulf %[[result]], %[[factor]] fastmath<contract> : f128
diff --git a/flang/test/Lower/Intrinsics/asinpi.f90 b/flang/test/Lower/Intrinsics/asinpi.f90
index 1c1838c56ca27..bceba3cf1be5c 100644
--- a/flang/test/Lower/Intrinsics/asinpi.f90
+++ b/flang/test/Lower/Intrinsics/asinpi.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
 ! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
@@ -10,8 +11,7 @@ function test_real4(x)
 ! CHECK-LABEL: @_QPtest_real4
 ! CHECK-PRECISE: %[[asin:.*]] = fir.call @asinf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
 ! CHECK-FAST: %[[asin:.*]] = math.asin %{{.*}} : f32
-! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
-! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.318309873 : f32
 ! CHECK: %{{.*}} = arith.mulf %[[asin]], %[[inv_pi]] fastmath<contract> : f32
 
 function test_real8(x)
@@ -24,3 +24,13 @@ function test_real8(x)
 ! CHECK-FAST: %[[asin:.*]] = math.asin %{{.*}} : f64
 ! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
 ! CHECK: %{{.*}} = arith.mulf %[[asin]], %[[inv_pi]] fastmath<contract> : f64
+
+function test_real16(x)
+  real(16) :: x, test_real16
+  test_real16 = asinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[asin:.*]] = fir.call @_FortranAAsinF128({{.*}}) fastmath<contract> : (f128) -> f128
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.3183098861837906715377675267450{{.*}} : f128
+! CHECK: %{{.*}} = arith.mulf %[[asin]], %[[inv_pi]] fastmath<contract> : f128
diff --git a/flang/test/Lower/Intrinsics/atan2d.f90 b/flang/test/Lower/Intrinsics/atan2d.f90
index 6ebf2976266ab..ea9174243c457 100644
--- a/flang/test/Lower/Intrinsics/atan2d.f90
+++ b/flang/test/Lower/Intrinsics/atan2d.f90
@@ -1,21 +1,19 @@
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
 
-
-function test_real4(y,x)
+function test_real4(y, x)
   real(4) :: x, y, test_real4
-  test_real4 = atan2d(y,x)
+  test_real4 = atan2d(y, x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
 ! CHECK-FAST: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f32
-! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
-! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[factor:.*]] = arith.constant 57.2957763 : f32
 ! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[factor]] fastmath<contract> : f32
 
-function test_real8(y,x)
+function test_real8(y, x)
   real(8) :: x, y, test_real8
-  test_real8 = atan2d(y,x)
+  test_real8 = atan2d(y, x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
diff --git a/flang/test/Lower/Intrinsics/atan2pi.f90 b/flang/test/Lower/Intrinsics/atan2pi.f90
index df72237610609..83039c0b6431e 100644
--- a/flang/test/Lower/Intrinsics/atan2pi.f90
+++ b/flang/test/Lower/Intrinsics/atan2pi.f90
@@ -1,24 +1,34 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
 
 
-function test_real4(y,x)
+function test_real4(y, x)
   real(4) :: x, y, test_real4
-  test_real4 = atan2pi(y,x)
+  test_real4 = atan2pi(y, x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
 ! CHECK-FAST: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f32
-! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
-! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.318309873 : f32
 ! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[inv_pi]] fastmath<contract> : f32
 
-function test_real8(y,x)
+function test_real8(y, x)
   real(8) :: x, y, test_real8
-  test_real8 = atan2pi(y,x)
+  test_real8 = atan2pi(y, x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8
 ! CHECK-FAST: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f64
 ! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
 ! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[inv_pi]] fastmath<contract> : f64
+
+function test_real16(y, x)
+  real(16) :: x, y, test_real16
+  test_real16 = atan2pi(y, x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f128
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.3183098861837906715377675267450{{.*}} : f128
+! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[inv_pi]] fastmath<contract> : f128
diff --git a/flang/test/Lower/Intrinsics/atand.f90 b/flang/test/Lower/Intrinsics/atand.f90
index 07ea56eff1e49..c27de4b5afbe7 100644
--- a/flang/test/Lower/Intrinsics/atand.f90
+++ b/flang/test/Lower/Intrinsics/atand.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
 ! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
@@ -10,8 +11,7 @@ function test_real4(x)
 ! CHECK-LABEL: @_QPtest_real4
 ! CHECK-PRECISE: %[[atan:.*]] = fir.call @atanf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
 ! CHECK-FAST: %[[atan:.*]] = math.atan %{{.*}} : f32
-! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
-! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[factor:.*]] = arith.constant 57.2957763 : f32
 ! CHECK: %{{.*}} = arith.mulf %[[atan]], %[[factor]] fastmath<contract> : f32
 
 function test_real8(x)
@@ -25,23 +25,42 @@ function test_real8(x)
 ! CHECK: %[[factor:.*]] = arith.constant 57.295779513082323 : f64
 ! CHECK: %{{.*}} = arith.mulf %[[atan]], %[[factor]] fastmath<contract> : f64
 
-function test_real4_yx(y,x)
+function test_real16(x)
+  real(16) :: x, test_real16
+  test_real16 = atand(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[atan:.*]] = fir.call @_FortranAAtanF128({{.*}}) fastmath<contract> : (f128) -> f128
+! CHECK: %[[factor:.*]] = arith.constant 57.295779513082320876798154814105{{.*}} : f128
+! CHECK: %{{.*}} = arith.mulf %[[atan]], %[[factor]] fastmath<contract> : f128
+
+function test_real4_yx(y, x)
   real(4) :: x, y, test_real4
-  test_real4 = atand(y,x)
+  test_real4 = atand(y, x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4_yx
 ! CHECK: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f32
-! CHECK: %[[dfactor:.*]] = arith.constant 57.295779513082323 : f64
-! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[factor:.*]] = arith.constant 57.2957763 : f32
 ! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[factor]] fastmath<contract> : f32
 
-function test_real8_yx(y,x)
+function test_real8_yx(y, x)
   real(8) :: x, y, test_real8
-  test_real8 = atand(y,x)
+  test_real8 = atand(y, x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8_yx
 ! CHECK: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f64
 ! CHECK: %[[factor:.*]] = arith.constant 57.295779513082323 : f64
 ! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[factor]] fastmath<contract> : f64
+
+function test_real16_yx(y, x)
+  real(16) :: x, y, test_real16
+  test_real16 = atand(y, x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16_yx
+! CHECK: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f128
+! CHECK: %[[factor:.*]] = arith.constant 57.295779513082320876798154814105{{.*}} : f128
+! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[factor]] fastmath<contract> : f128
diff --git a/flang/test/Lower/Intrinsics/atanpi.f90 b/flang/test/Lower/Intrinsics/atanpi.f90
index 6382dbd1a30cf..ece42f9500ce8 100644
--- a/flang/test/Lower/Intrinsics/atanpi.f90
+++ b/flang/test/Lower/Intrinsics/atanpi.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
 ! RUN: bbc --math-runtime=precise -emit-fir -hlfir=false %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-PRECISE"
 ! RUN: %flang_fc1 -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s --check-prefixes="CHECK,CHECK-FAST"
@@ -10,8 +11,7 @@ function test_real4(x)
 ! CHECK-LABEL: @_QPtest_real4
 ! CHECK-PRECISE: %[[atan:.*]] = fir.call @atanf({{%[A-Za-z0-9._]+}}) fastmath<contract> : (f32) -> f32
 ! CHECK-FAST: %[[atan:.*]] = math.atan %{{.*}} : f32
-! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
-! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.318309873 : f32
 ! CHECK: %{{.*}} = arith.mulf %[[atan]], %[[inv_pi]] fastmath<contract> : f32
 
 function test_real8(x)
@@ -25,23 +25,42 @@ function test_real8(x)
 ! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
 ! CHECK: %{{.*}} = arith.mulf %[[atan]], %[[inv_pi]] fastmath<contract> : f64
 
-function test_real4_yx(y,x)
+function test_real16(x)
+  real(16) :: x, test_real16
+  test_real16 = atanpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[atan:.*]] = fir.call @_FortranAAtanF128({{.*}}) fastmath<contract> : (f128) -> f128
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.3183098861837906715377675267450{{.*}} : f128
+! CHECK: %{{.*}} = arith.mulf %[[atan]], %[[inv_pi]] fastmath<contract> : f128
+
+function test_real4_yx(y, x)
   real(4) :: x, y, test_real4
-  test_real4 = atanpi(y,x)
+  test_real4 = atanpi(y, x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4_yx
 ! CHECK: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f32
-! CHECK: %[[dpi:.*]] = arith.constant 0.31830988618379069 : f64
-! CHECK: %[[inv_pi:.*]] = fir.convert %[[dpi]] : (f64) -> f32
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.318309873 : f32
 ! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[inv_pi]] fastmath<contract> : f32
 
-function test_real8_yx(y,x)
+function test_real8_yx(y, x)
   real(8) :: x, y, test_real8
-  test_real8 = atanpi(y,x)
+  test_real8 = atanpi(y, x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real8_yx
 ! CHECK: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f64
 ! CHECK: %[[inv_pi:.*]] = arith.constant 0.31830988618379069 : f64
 ! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[inv_pi]] fastmath<contract> : f64
+
+function test_real16_yx(y, x)
+  real(16) :: x, y, test_real16
+  test_real16 = atanpi(y, x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16_yx
+! CHECK: %[[atan2:.*]] = math.atan2 %{{.*}}, %{{.*}}: f128
+! CHECK: %[[inv_pi:.*]] = arith.constant 0.3183098861837906715377675267450{{.*}} : f128
+! CHECK: %{{.*}} = arith.mulf %[[atan2]], %[[inv_pi]] fastmath<contract> : f128
diff --git a/flang/test/Lower/Intrinsics/cospi.f90 b/flang/test/Lower/Intrinsics/cospi.f90
index 894002566141b..5c612901d8302 100644
--- a/flang/test/Lower/Intrinsics/cospi.f90
+++ b/flang/test/Lower/Intrinsics/cospi.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -6,8 +7,7 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
-! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[factor:.*]] = arith.constant 3.14159274 : f32
 ! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f32
 ! CHECK: %[[cos:.*]] = math.cos %[[mul]] fastmath<contract> : f32
 
@@ -20,3 +20,13 @@ function test_real8(x)
 ! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
 ! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[dfactor]] fastmath<contract> : f64
 ! CHECK: %[[cos:.*]] = math.cos %[[mul]] fastmath<contract> : f64
+
+function test_real16(x)
+  real(16) :: x, test_real16
+  test_real16 = cospi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[factor:.*]] = arith.constant 3.141592653589793238462643383279{{.*}} : f128
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f128
+! CHECK: %[[cos:.*]] = fir.call @_FortranACosF128(%[[mul]]) fastmath<contract> : (f128) -> f128
diff --git a/flang/test/Lower/Intrinsics/sinpi.f90 b/flang/test/Lower/Intrinsics/sinpi.f90
index 38c2277892ec7..06699b7f8ea71 100644
--- a/flang/test/Lower/Intrinsics/sinpi.f90
+++ b/flang/test/Lower/Intrinsics/sinpi.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -6,8 +7,7 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
-! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[factor:.*]] = arith.constant 3.14159274 : f32
 ! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f32
 ! CHECK: %[[sin:.*]] = math.sin %[[mul]] fastmath<contract> : f32
 
@@ -20,3 +20,13 @@ function test_real8(x)
 ! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
 ! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[dfactor]] fastmath<contract> : f64
 ! CHECK: %[[sin:.*]] = math.sin %[[mul]] fastmath<contract> : f64
+
+function test_real16(x)
+  real(16) :: x, test_real16
+  test_real16 = sinpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[factor:.*]] = arith.constant 3.141592653589793238462643383279{{.*}} : f128
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f128
+! CHECK: %[[sin:.*]] = fir.call @_FortranASinF128(%[[mul]]) fastmath<contract> : (f128) -> f128
diff --git a/flang/test/Lower/Intrinsics/tanpi.f90 b/flang/test/Lower/Intrinsics/tanpi.f90
index 9cc3ae6ef1563..0a011047ce93e 100644
--- a/flang/test/Lower/Intrinsics/tanpi.f90
+++ b/flang/test/Lower/Intrinsics/tanpi.f90
@@ -1,3 +1,4 @@
+! REQUIRES: flang-supports-f128-math
 ! RUN: %flang_fc1 -emit-fir %s -o - | FileCheck %s --check-prefixes="CHECK"
 
 function test_real4(x)
@@ -6,8 +7,7 @@ function test_real4(x)
 end function
 
 ! CHECK-LABEL: @_QPtest_real4
-! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
-! CHECK: %[[factor:.*]] = fir.convert %[[dfactor]] : (f64) -> f32
+! CHECK: %[[factor:.*]] = arith.constant 3.14159274 : f32
 ! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f32
 ! CHECK: %[[tan:.*]] = math.tan %[[mul]] fastmath<contract> : f32
 
@@ -20,3 +20,13 @@ function test_real8(x)
 ! CHECK: %[[dfactor:.*]] = arith.constant 3.1415926535897931 : f64
 ! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[dfactor]] fastmath<contract> : f64
 ! CHECK: %[[tan:.*]] = math.tan %[[mul]] fastmath<contract> : f64
+
+function test_real16(x)
+  real(16) :: x, test_real16
+  test_real16 = tanpi(x)
+end function
+
+! CHECK-LABEL: @_QPtest_real16
+! CHECK: %[[factor:.*]] = arith.constant 3.141592653589793238462643383279{{.*}} : f128
+! CHECK: %[[mul:.*]] = arith.mulf %{{.*}}, %[[factor]] fastmath<contract> : f128
+! CHECK: %[[tan:.*]] = fir.call @_FortranATanF128(%[[mul]]) fastmath<contract> : (f128) -> f128
diff --git a/flang/test/Lower/OpenACC/acc-private.f90 b/flang/test/Lower/OpenACC/acc-private.f90
index b1bfb02439f03..5ca08a39a1e1c 100644
--- a/flang/test/Lower/OpenACC/acc-private.f90
+++ b/flang/test/Lower/OpenACC/acc-private.f90
@@ -95,6 +95,9 @@
 ! CHECK: ^bb0(%arg0: !fir.ref<!fir.box<!fir.heap<i32>>>):
 ! CHECK:   %[[ALLOCA:.*]] = fir.alloca !fir.box<!fir.heap<i32>>
 ! CHECK:   %[[DECLARE:.*]]:2 = hlfir.declare %[[ALLOCA]] {uniq_name = "acc.private.init"} : (!fir.ref<!fir.box<!fir.heap<i32>>>) -> (!fir.ref<!fir.box<!fir.heap<i32>>>, !fir.ref<!fir.box<!fir.heap<i32>>>)
+! CHECK:   %[[ALLOCMEM:.*]] = fir.allocmem i32
+! CHECK:   %[[BOX:.*]] = fir.embox %[[ALLOCMEM]] : (!fir.heap<i32>) -> !fir.box<!fir.heap<i32>>
+! CHECK:   fir.store %[[BOX]] to %[[DECLARE]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK:   acc.yield %[[DECLARE]]#0 : !fir.ref<!fir.box<!fir.heap<i32>>>
 ! CHECK: }
 
diff --git a/flang/test/Lower/OpenACC/acc-reduction.f90 b/flang/test/Lower/OpenACC/acc-reduction.f90
index 20b5ad28f78a1..035b38b8a4da4 100644
--- a/flang/test/Lower/OpenACC/acc-reduction.f90
+++ b/flang/test/Lower/OpenACC/acc-reduction.f90
@@ -189,6 +189,14 @@
 ! CHECK:   acc.yield %arg0 : !fir.box<!fir.array<?xi32>>
 ! CHECK: }
 
+! CHECK-LABEL: acc.reduction.recipe @reduction_add_section_lb0.ub9xlb0.ub19_ref_10x20xi32 : !fir.ref<!fir.array<10x20xi32>> reduction_operator <add> init {
+! CHECK:     fir.do_loop %arg1 = %c0 to %c19 step %c1 {
+! CHECK:       fir.do_loop %arg2 = %c0_0 to %c9 step %c1_1 {
+! CHECK: } combiner {
+! CHECK:     fir.do_loop %arg2 = %c0 to %c19 step %c1 {
+! CHECK:       fir.do_loop %arg3 = %c0_0 to %c9 step %c1_1 {
+! CHECK: }
+
 ! CHECK-LABEL: acc.reduction.recipe @reduction_mul_ref_z32 : !fir.ref<complex<f32>> reduction_operator <mul> init {
 ! CHECK: ^bb0(%{{.*}}: !fir.ref<complex<f32>>):
 ! CHECK:   %[[REAL:.*]] = arith.constant 1.000000e+00 : f32
@@ -1167,6 +1175,29 @@ subroutine acc_reduction_add_static_slice(a)
 ! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.array<100xi32>>) bounds(%[[BOUND]]) -> !fir.ref<!fir.array<100xi32>> {name = "a(11:20)"}
 ! CHECK: acc.parallel reduction(@reduction_add_section_lb10.ub19_ref_100xi32 -> %[[RED]] : !fir.ref<!fir.array<100xi32>>)
 
+subroutine acc_reduction_add_static_slice_2d(a)
+  integer :: a(10,20)
+  !$acc parallel reduction(+:a(:10,:20))
+  !$acc end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPacc_reduction_add_static_slice_2d(
+! CHECK-SAME: %[[ARG0:.*]]: !fir.ref<!fir.array<10x20xi32>> {fir.bindc_name = "a"})
+! CHECK: %[[C10:.*]] = arith.constant 10 : index
+! CHECK: %[[C20:.*]] = arith.constant 20 : index
+! CHECK: %[[DECLARG0:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK: %[[LB:.*]] = arith.constant 0 : index
+! CHECK: %[[C1:.*]] = arith.constant 1 : index
+! CHECK: %[[UB9:.*]] = arith.constant 9 : index
+! CHECK: %[[STRIDE1:.*]] = arith.constant 10 : index
+! CHECK: %[[BOUND0:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB9]] : index) extent(%[[C10]] : index) stride(%[[C1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[UB19:.*]] = arith.constant 19 : index
+! CHECK: %[[BOUND1:.*]] = acc.bounds lowerbound(%[[LB]] : index) upperbound(%[[UB19]] : index) extent(%[[C20]] : index)
+! stride(%[[STRIDE1]] : index) startIdx(%[[C1]] : index)
+! CHECK: %[[RED:.*]] = acc.reduction varPtr(%[[DECLARG0]]#0 : !fir.ref<!fir.array<10x20xi32>>) bounds(%[[BOUND0]], %[[BOUND1]]) ->
+! !fir.ref<!fir.array<10x20xi32>> {name = "a(:10,:20)"}
+! CHECK: acc.parallel reduction(@reduction_add_section_lb0.ub9xlb0.ub19_ref_10x20xi32 -> %[[RED]] : !fir.ref<!fir.array<10x20xi32>>)
+
 subroutine acc_reduction_add_dynamic_extent_add(a)
   integer :: a(:)
   !$acc parallel reduction(+:a)
diff --git a/flang/test/Lower/OpenMP/Todo/assumed-rank-privatization.f90 b/flang/test/Lower/OpenMP/Todo/assumed-rank-privatization.f90
new file mode 100644
index 0000000000000..e57833a1013b6
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/assumed-rank-privatization.f90
@@ -0,0 +1,9 @@
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Privatization of assumed rank variable
+subroutine assumedPriv(a)
+  integer :: a(..)
+
+  !$omp parallel private(a)
+  !$omp end parallel
+end
diff --git a/flang/test/Lower/OpenMP/Todo/dyn-groupprivate-clause.f90 b/flang/test/Lower/OpenMP/Todo/dyn-groupprivate-clause.f90
new file mode 100644
index 0000000000000..e06470f772bf8
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/dyn-groupprivate-clause.f90
@@ -0,0 +1,10 @@
+!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=61 -o - %s 2>&1 | FileCheck %s
+
+!CHECK: not yet implemented: DYN_GROUPPRIVATE clause is not implemented yet
+subroutine f00(n)
+  implicit none
+  integer :: n
+  !$omp target dyn_groupprivate(n)
+  !$omp end target
+end
+
diff --git a/flang/test/Lower/OpenMP/Todo/groupprivate.f90 b/flang/test/Lower/OpenMP/Todo/groupprivate.f90
new file mode 100644
index 0000000000000..9ad9b9382760c
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/groupprivate.f90
@@ -0,0 +1,9 @@
+!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=60 -o - %s 2>&1 | FileCheck %s
+
+!CHECK: not yet implemented: GROUPPRIVATE
+
+module m
+implicit none
+integer :: x
+!$omp groupprivate(x)
+end module
diff --git a/flang/test/Lower/OpenMP/atomic-update-reassoc.f90 b/flang/test/Lower/OpenMP/atomic-update-reassoc.f90
new file mode 100644
index 0000000000000..96ebb56b8d6ca
--- /dev/null
+++ b/flang/test/Lower/OpenMP/atomic-update-reassoc.f90
@@ -0,0 +1,75 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=60 %s -o - | FileCheck %s
+
+subroutine f00(x, y)
+  implicit none
+  integer :: x, y
+
+  !$omp atomic update
+  x = ((x + 1) + y) + 2
+end
+
+!CHECK-LABEL: func.func @_QPf00
+!CHECK: %[[X:[0-9]+]]:2 = hlfir.declare %arg0
+!CHECK: %[[Y:[0-9]+]]:2 = hlfir.declare %arg1
+!CHECK: %c1_i32 = arith.constant 1 : i32
+!CHECK: %[[LOAD_Y:[0-9]+]] = fir.load %[[Y]]#0 : !fir.ref<i32>
+!CHECK: %[[Y_1:[0-9]+]] = arith.addi %c1_i32, %[[LOAD_Y]] : i32
+!CHECK: %c2_i32 = arith.constant 2 : i32
+!CHECK: %[[Y_1_2:[0-9]+]] = arith.addi %[[Y_1]], %c2_i32 : i32
+!CHECK: omp.atomic.update memory_order(relaxed) %[[X]]#0 : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:arg[0-9]+]]: i32):
+!CHECK:   %[[ARG_P:[0-9]+]] = arith.addi %[[ARG]], %[[Y_1_2]] : i32
+!CHECK:   omp.yield(%[[ARG_P]] : i32)
+!CHECK: }
+
+
+subroutine f01(x, y)
+  implicit none
+  real :: x
+  integer :: y
+
+  !$omp atomic update
+  x = (int(x) + y) + 1
+end
+
+!CHECK-LABEL: func.func @_QPf01
+!CHECK: %[[X:[0-9]+]]:2 = hlfir.declare %arg0
+!CHECK: %[[Y:[0-9]+]]:2 = hlfir.declare %arg1
+!CHECK: %[[LOAD_Y:[0-9]+]] = fir.load %[[Y]]#0 : !fir.ref<i32>
+!CHECK: %c1_i32 = arith.constant 1 : i32
+!CHECK: %[[Y_1:[0-9]+]] = arith.addi %[[LOAD_Y]], %c1_i32 : i32
+!CHECK: omp.atomic.update memory_order(relaxed) %[[X]]#0 : !fir.ref<f32> {
+!CHECK: ^bb0(%[[ARG:arg[0-9]+]]: f32):
+!CHECK:   %[[ARG_I:[0-9]+]] = fir.convert %[[ARG]] : (f32) -> i32
+!CHECK:   %[[ARG_P:[0-9]+]] = arith.addi %[[ARG_I]], %[[Y_1]] : i32
+!CHECK:   %[[ARG_F:[0-9]+]] = fir.convert %[[ARG_P]] : (i32) -> f32
+!CHECK:   omp.yield(%[[ARG_F]] : f32)
+!CHECK: }
+
+
+subroutine f02(x, a, b, c)
+  implicit none
+  integer(kind=4) :: x
+  integer(kind=8) :: a, b, c
+
+  !$omp atomic update
+  x = ((b + a) + x) + c
+end
+
+!CHECK-LABEL: func.func @_QPf02
+!CHECK: %[[A:[0-9]+]]:2 = hlfir.declare %arg1
+!CHECK: %[[B:[0-9]+]]:2 = hlfir.declare %arg2
+!CHECK: %[[C:[0-9]+]]:2 = hlfir.declare %arg3
+!CHECK: %[[X:[0-9]+]]:2 = hlfir.declare %arg0
+!CHECK: %[[LOAD_B:[0-9]+]] = fir.load %[[B]]#0 : !fir.ref<i64>
+!CHECK: %[[LOAD_A:[0-9]+]] = fir.load %[[A]]#0 : !fir.ref<i64>
+!CHECK: %[[A_B:[0-9]+]] = arith.addi %[[LOAD_B]], %[[LOAD_A]] : i64
+!CHECK: %[[LOAD_C:[0-9]+]] = fir.load %[[C]]#0 : !fir.ref<i64>
+!CHECK: %[[A_B_C:[0-9]+]] = arith.addi %[[A_B]], %[[LOAD_C]] : i64
+!CHECK: omp.atomic.update memory_order(relaxed) %[[X]]#0 : !fir.ref<i32> {
+!CHECK: ^bb0(%[[ARG:arg[0-9]+]]: i32):
+!CHECK:   %[[ARG_8:[0-9]+]] = fir.convert %[[ARG]] : (i32) -> i64
+!CHECK:   %[[ARG_P:[0-9]+]] = arith.addi %[[ARG_8]], %[[A_B_C]] : i64
+!CHECK:   %[[ARG_4:[0-9]+]] = fir.convert %[[ARG_P]] : (i64) -> i32
+!CHECK:   omp.yield(%[[ARG_4]] : i32)
+!CHECK: }
diff --git a/flang/test/Lower/OpenMP/block_implicit_privatization.f90 b/flang/test/Lower/OpenMP/block_implicit_privatization.f90
new file mode 100644
index 0000000000000..32b26ac01f181
--- /dev/null
+++ b/flang/test/Lower/OpenMP/block_implicit_privatization.f90
@@ -0,0 +1,31 @@
+! When a block variable is marked as implicit private, we can simply ignore
+! privatizing that symbol within the context of the currrent OpenMP construct
+! since the "private" allocation for the symbol will be emitted within the nested
+! block anyway.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+subroutine block_implicit_privatization
+  implicit none
+  integer :: i
+
+  !$omp task
+  do i=1,10
+    block
+      integer :: j
+      j = 0
+    end block
+  end do
+  !$omp end task
+end subroutine
+
+! CHECK-LABEL: func.func @_QPblock_implicit_privatization() {
+! CHECK:         %[[I_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}Ei"}
+! CHECK:         omp.task private(@{{.*}}Ei_private_i32 %[[I_DECL]]#0 -> %{{.*}} : !fir.ref<i32>) {
+! CHECK:           fir.do_loop {{.*}} {
+! Verify that `j` is allocated whithin the same scope of its block (i.e. inside
+! the `task` loop).
+! CHECK:             fir.alloca i32 {bindc_name = "j", {{.*}}}
+! CHECK:           }
+! CHECK:         }
+! CHECK:       }
diff --git a/flang/test/Lower/OpenMP/block_predetermined_privatization.f90 b/flang/test/Lower/OpenMP/block_predetermined_privatization.f90
new file mode 100644
index 0000000000000..12346c14ef5d9
--- /dev/null
+++ b/flang/test/Lower/OpenMP/block_predetermined_privatization.f90
@@ -0,0 +1,32 @@
+! Fixes a bug when a block variable is marked as pre-determined private. In such
+! case, we can simply ignore privatizing that symbol within the context of the
+! currrent OpenMP construct since the "private" allocation for the symbol will
+! be emitted within the nested block anyway.
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+subroutine block_predetermined_privatization
+  implicit none
+  integer :: i
+
+  !$omp parallel
+  do i=1,10
+    block
+      integer :: j
+      do j=1,10
+      end do
+    end block
+  end do
+  !$omp end parallel
+end subroutine
+
+! CHECK-LABEL: func.func @_QPblock_predetermined_privatization() {
+! CHECK:         %[[I_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}Ei"}
+! CHECK:         omp.parallel private(@{{.*}}Ei_private_i32 %[[I_DECL]]#0 -> %{{.*}} : !fir.ref<i32>) {
+! CHECK:           fir.do_loop {{.*}} {
+! Verify that `j` is allocated whithin the same scope of its block (i.e. inside
+! the `parallel` loop).
+! CHECK:             fir.alloca i32 {bindc_name = "j", {{.*}}}
+! CHECK:           }
+! CHECK:         }
+! CHECK:       }
diff --git a/flang/test/Lower/OpenMP/common-block-map.f90 b/flang/test/Lower/OpenMP/common-block-map.f90
index 06df0d2d9fb18..743438593a3d5 100644
--- a/flang/test/Lower/OpenMP/common-block-map.f90
+++ b/flang/test/Lower/OpenMP/common-block-map.f90
@@ -1,7 +1,7 @@
 !RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
 
 !CHECK: fir.global common @var_common_(dense<0> : vector<8xi8>) {{.*}} : !fir.array<8xi8>
-!CHECK: fir.global common @var_common_link_(dense<0> : vector<8xi8>) {{{.*}} omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : !fir.array<8xi8>
+!CHECK: fir.global common @var_common_link_(dense<0> : vector<8xi8>) {{{.*}} omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} : !fir.array<8xi8>
 
 !CHECK-LABEL: func.func @_QPmap_full_block
 !CHECK: %[[CB_ADDR:.*]] = fir.address_of(@var_common_) : !fir.ref<!fir.array<8xi8>>
diff --git a/flang/test/Lower/OpenMP/declare-target-data.f90 b/flang/test/Lower/OpenMP/declare-target-data.f90
index 154853a0fa20c..474944d7c0bb0 100644
--- a/flang/test/Lower/OpenMP/declare-target-data.f90
+++ b/flang/test/Lower/OpenMP/declare-target-data.f90
@@ -1,86 +1,90 @@
-!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s 
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
 !RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-target-device %s -o - | FileCheck %s
 
 module test_0
     implicit none
 
-!CHECK-DAG: fir.global @_QMtest_0Edata_int {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : i32
+!CHECK-DAG: fir.global @_QMtest_0Edata_int {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} : i32
 INTEGER :: data_int = 10
 !$omp declare target link(data_int)
 
-!CHECK-DAG: fir.global @_QMtest_0Earray_1d({{.*}}) {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : !fir.array<3xi32>
+!CHECK-DAG: fir.global @_QMtest_0Earray_1d({{.*}}) {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} : !fir.array<3xi32>
 INTEGER :: array_1d(3) = (/1,2,3/)
 !$omp declare target link(array_1d)
 
-!CHECK-DAG: fir.global @_QMtest_0Earray_2d({{.*}}) {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : !fir.array<2x2xi32> 
+!CHECK-DAG: fir.global @_QMtest_0Earray_2d({{.*}}) {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} : !fir.array<2x2xi32>
 INTEGER :: array_2d(2,2) = reshape((/1,2,3,4/), (/2,2/))
 !$omp declare target link(array_2d)
 
-!CHECK-DAG: fir.global @_QMtest_0Ept1 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : !fir.box<!fir.ptr<i32>>
+!CHECK-DAG: fir.global @_QMtest_0Ept1 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} : !fir.box<!fir.ptr<i32>>
 INTEGER, POINTER :: pt1
 !$omp declare target link(pt1)
 
-!CHECK-DAG: fir.global @_QMtest_0Ept2_tar {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} target : i32
-INTEGER, TARGET :: pt2_tar = 5 
+!CHECK-DAG: fir.global @_QMtest_0Ept2_tar {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} target : i32
+INTEGER, TARGET :: pt2_tar = 5
 !$omp declare target link(pt2_tar)
 
-!CHECK-DAG: fir.global @_QMtest_0Ept2 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : !fir.box<!fir.ptr<i32>>
+!CHECK-DAG: fir.global @_QMtest_0Ept2 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} : !fir.box<!fir.ptr<i32>>
 INTEGER, POINTER :: pt2 => pt2_tar
 !$omp declare target link(pt2)
 
-!CHECK-DAG: fir.global @_QMtest_0Edata_int_to {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : i32
+!CHECK-DAG: fir.global @_QMtest_0Edata_int_to {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>} : i32
 INTEGER :: data_int_to = 5
 !$omp declare target to(data_int_to)
 
-!CHECK-DAG: fir.global @_QMtest_0Edata_int_enter {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>} : i32
+!CHECK-DAG: fir.global @_QMtest_0Edata_int_enter {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>} : i32
 INTEGER :: data_int_enter = 5
 !$omp declare target enter(data_int_enter)
 
-!CHECK-DAG: fir.global @_QMtest_0Edata_int_clauseless {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : i32
+!CHECK-DAG: fir.global @_QMtest_0Edata_int_clauseless {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>} : i32
 INTEGER :: data_int_clauseless = 1
 !$omp declare target(data_int_clauseless)
 
-!CHECK-DAG: fir.global @_QMtest_0Edata_extended_to_1 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : f32
-!CHECK-DAG: fir.global @_QMtest_0Edata_extended_to_2 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : f32
+!CHECK-DAG: fir.global @_QMtest_0Edata_extended_to_1 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>} : f32
+!CHECK-DAG: fir.global @_QMtest_0Edata_extended_to_2 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>} : f32
 REAL :: data_extended_to_1 = 2
 REAL :: data_extended_to_2 = 3
 !$omp declare target to(data_extended_to_1, data_extended_to_2)
 
-!CHECK-DAG: fir.global @_QMtest_0Edata_extended_enter_1 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>} : f32
-!CHECK-DAG: fir.global @_QMtest_0Edata_extended_enter_2 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>} : f32
+!CHECK-DAG: fir.global @_QMtest_0Edata_extended_enter_1 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>} : f32
+!CHECK-DAG: fir.global @_QMtest_0Edata_extended_enter_2 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>} : f32
 REAL :: data_extended_enter_1 = 2
 REAL :: data_extended_enter_2 = 3
 !$omp declare target enter(data_extended_enter_1, data_extended_enter_2)
 
-!CHECK-DAG: fir.global @_QMtest_0Edata_extended_link_1 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : f32
-!CHECK-DAG: fir.global @_QMtest_0Edata_extended_link_2 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : f32
+!CHECK-DAG: fir.global @_QMtest_0Edata_extended_link_1 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} : f32
+!CHECK-DAG: fir.global @_QMtest_0Edata_extended_link_2 {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} : f32
 REAL :: data_extended_link_1 = 2
 REAL :: data_extended_link_2 = 3
 !$omp declare target link(data_extended_link_1, data_extended_link_2)
 
+!CHECK-DAG: fir.global @_QMtest_0Eautomap_data {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = true>} target : !fir.box<!fir.heap<i32>>
+INTEGER, ALLOCATABLE, TARGET :: automap_data
+!$omp declare target enter(automap : automap_data)
+
 contains
 end module test_0
 
 PROGRAM commons
-    !CHECK-DAG: fir.global @numbers_ {alignment = 4 : i64, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : tuple<f32, f32> {
+    !CHECK-DAG: fir.global @numbers_ {alignment = 4 : i64, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>} : tuple<f32, f32> {
     REAL :: one = 1
     REAL :: two = 2
     COMMON /numbers/ one, two
     !$omp declare target(/numbers/)
-    
-    !CHECK-DAG: fir.global @numbers_link_ {alignment = 4 : i64, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link)>} : tuple<f32, f32> {
+
+    !CHECK-DAG: fir.global @numbers_link_ {alignment = 4 : i64, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (link), automap = false>} : tuple<f32, f32> {
     REAL :: one_link = 1
     REAL :: two_link = 2
     COMMON /numbers_link/ one_link, two_link
     !$omp declare target link(/numbers_link/)
 
-    !CHECK-DAG: fir.global @numbers_to_ {alignment = 4 : i64, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : tuple<f32, f32> {
+    !CHECK-DAG: fir.global @numbers_to_ {alignment = 4 : i64, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>} : tuple<f32, f32> {
     REAL :: one_to = 1
     REAL :: two_to = 2
     COMMON /numbers_to/ one_to, two_to
     !$omp declare target to(/numbers_to/)
 
-    !CHECK-DAG: fir.global @numbers_enter_ {alignment = 4 : i64, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>} : tuple<f32, f32> {
+    !CHECK-DAG: fir.global @numbers_enter_ {alignment = 4 : i64, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>} : tuple<f32, f32> {
     REAL :: one_enter = 1
     REAL :: two_enter = 2
     COMMON /numbers_enter/ one_enter, two_enter
diff --git a/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90 b/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90
index 079d43e309028..528563aba6962 100644
--- a/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90
+++ b/flang/test/Lower/OpenMP/declare-target-deferred-marking.f90
@@ -51,10 +51,10 @@ end subroutine unused_unemitted_interface
  end program main
 
 !HOST-LABEL: func.func {{.*}} @host_interface()
-!HOST-SAME: {{.*}}, omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter)>{{.*}}
+!HOST-SAME: {{.*}}, omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter), automap = false>{{.*}}
 !ALL-LABEL: func.func {{.*}} @called_from_target_interface(!fir.ref<i64>, !fir.ref<i64>)
-!ALL-SAME: {{.*}}, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}
+!ALL-SAME: {{.*}}, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}
 !ALL-LABEL: func.func {{.*}} @any_interface()
-!ALL-SAME: {{.*}}, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}
+!ALL-SAME: {{.*}}, omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}
 !ALL-LABEL: func.func {{.*}} @device_interface()
-!ALL-SAME: {{.*}}, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}
+!ALL-SAME: {{.*}}, omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}
diff --git a/flang/test/Lower/OpenMP/declare-target-func-and-subr.f90 b/flang/test/Lower/OpenMP/declare-target-func-and-subr.f90
index 1c43f1d09eddb..4abf750cf735a 100644
--- a/flang/test/Lower/OpenMP/declare-target-func-and-subr.f90
+++ b/flang/test/Lower/OpenMP/declare-target-func-and-subr.f90
@@ -6,7 +6,7 @@
 ! zero clause declare target
 
 ! DEVICE-LABEL: func.func @_QPfunc_t_device()
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}
 FUNCTION FUNC_T_DEVICE() RESULT(I)
 !$omp declare target to(FUNC_T_DEVICE) device_type(nohost)
     INTEGER :: I
@@ -14,7 +14,7 @@ FUNCTION FUNC_T_DEVICE() RESULT(I)
 END FUNCTION FUNC_T_DEVICE
 
 ! DEVICE-LABEL: func.func @_QPfunc_enter_device()
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}
 FUNCTION FUNC_ENTER_DEVICE() RESULT(I)
 !$omp declare target enter(FUNC_ENTER_DEVICE) device_type(nohost)
     INTEGER :: I
@@ -22,7 +22,7 @@ FUNCTION FUNC_ENTER_DEVICE() RESULT(I)
 END FUNCTION FUNC_ENTER_DEVICE
 
 ! HOST-LABEL: func.func @_QPfunc_t_host()
-! HOST-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to)>{{.*}}
+! HOST-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to), automap = false>{{.*}}
 FUNCTION FUNC_T_HOST() RESULT(I)
 !$omp declare target to(FUNC_T_HOST) device_type(host)
     INTEGER :: I
@@ -30,7 +30,7 @@ FUNCTION FUNC_T_HOST() RESULT(I)
 END FUNCTION FUNC_T_HOST
 
 ! HOST-LABEL: func.func @_QPfunc_enter_host()
-! HOST-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter)>{{.*}}
+! HOST-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter), automap = false>{{.*}}
 FUNCTION FUNC_ENTER_HOST() RESULT(I)
 !$omp declare target enter(FUNC_ENTER_HOST) device_type(host)
     INTEGER :: I
@@ -38,7 +38,7 @@ FUNCTION FUNC_ENTER_HOST() RESULT(I)
 END FUNCTION FUNC_ENTER_HOST
 
 ! ALL-LABEL: func.func @_QPfunc_t_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}
 FUNCTION FUNC_T_ANY() RESULT(I)
 !$omp declare target to(FUNC_T_ANY) device_type(any)
     INTEGER :: I
@@ -46,7 +46,7 @@ FUNCTION FUNC_T_ANY() RESULT(I)
 END FUNCTION FUNC_T_ANY
 
 ! ALL-LABEL: func.func @_QPfunc_enter_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}
 FUNCTION FUNC_ENTER_ANY() RESULT(I)
 !$omp declare target enter(FUNC_ENTER_ANY) device_type(any)
     INTEGER :: I
@@ -54,7 +54,7 @@ FUNCTION FUNC_ENTER_ANY() RESULT(I)
 END FUNCTION FUNC_ENTER_ANY
 
 ! ALL-LABEL: func.func @_QPfunc_default_t_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}
 FUNCTION FUNC_DEFAULT_T_ANY() RESULT(I)
 !$omp declare target to(FUNC_DEFAULT_T_ANY)
     INTEGER :: I
@@ -62,7 +62,7 @@ FUNCTION FUNC_DEFAULT_T_ANY() RESULT(I)
 END FUNCTION FUNC_DEFAULT_T_ANY
 
 ! ALL-LABEL: func.func @_QPfunc_default_enter_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}
 FUNCTION FUNC_DEFAULT_ENTER_ANY() RESULT(I)
 !$omp declare target enter(FUNC_DEFAULT_ENTER_ANY)
     INTEGER :: I
@@ -70,7 +70,7 @@ FUNCTION FUNC_DEFAULT_ENTER_ANY() RESULT(I)
 END FUNCTION FUNC_DEFAULT_ENTER_ANY
 
 ! ALL-LABEL: func.func @_QPfunc_default_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}
 FUNCTION FUNC_DEFAULT_ANY() RESULT(I)
 !$omp declare target
     INTEGER :: I
@@ -78,7 +78,7 @@ FUNCTION FUNC_DEFAULT_ANY() RESULT(I)
 END FUNCTION FUNC_DEFAULT_ANY
 
 ! ALL-LABEL: func.func @_QPfunc_default_extendedlist()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}
 FUNCTION FUNC_DEFAULT_EXTENDEDLIST() RESULT(I)
 !$omp declare target(FUNC_DEFAULT_EXTENDEDLIST)
     INTEGER :: I
@@ -86,7 +86,7 @@ FUNCTION FUNC_DEFAULT_EXTENDEDLIST() RESULT(I)
 END FUNCTION FUNC_DEFAULT_EXTENDEDLIST
 
 ! ALL-LABEL: func.func @_QPfunc_name_as_result()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}
 FUNCTION FUNC_NAME_AS_RESULT()
 !$omp declare target(FUNC_NAME_AS_RESULT)
   FUNC_NAME_AS_RESULT = 1.0
@@ -99,61 +99,61 @@ END FUNCTION FUNC_NAME_AS_RESULT
 ! zero clause declare target
 
 ! DEVICE-LABEL: func.func @_QPsubr_t_device()
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}
 SUBROUTINE SUBR_T_DEVICE()
 !$omp declare target to(SUBR_T_DEVICE) device_type(nohost)
 END
 
 ! DEVICE-LABEL: func.func @_QPsubr_enter_device()
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}
 SUBROUTINE SUBR_ENTER_DEVICE()
 !$omp declare target enter(SUBR_ENTER_DEVICE) device_type(nohost)
 END
 
 ! HOST-LABEL: func.func @_QPsubr_t_host()
-! HOST-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to)>{{.*}}
+! HOST-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to), automap = false>{{.*}}
 SUBROUTINE SUBR_T_HOST()
 !$omp declare target to(SUBR_T_HOST) device_type(host)
 END
 
 ! HOST-LABEL: func.func @_QPsubr_enter_host()
-! HOST-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter)>{{.*}}
+! HOST-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter), automap = false>{{.*}}
 SUBROUTINE SUBR_ENTER_HOST()
 !$omp declare target enter(SUBR_ENTER_HOST) device_type(host)
 END
 
 ! ALL-LABEL: func.func @_QPsubr_t_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}
 SUBROUTINE SUBR_T_ANY()
 !$omp declare target to(SUBR_T_ANY) device_type(any)
 END
 
 ! ALL-LABEL: func.func @_QPsubr_enter_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}
 SUBROUTINE SUBR_ENTER_ANY()
 !$omp declare target enter(SUBR_ENTER_ANY) device_type(any)
 END
 
 ! ALL-LABEL: func.func @_QPsubr_default_t_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}
 SUBROUTINE SUBR_DEFAULT_T_ANY()
 !$omp declare target to(SUBR_DEFAULT_T_ANY)
 END
 
 ! ALL-LABEL: func.func @_QPsubr_default_enter_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}
 SUBROUTINE SUBR_DEFAULT_ENTER_ANY()
 !$omp declare target enter(SUBR_DEFAULT_ENTER_ANY)
 END
 
 ! ALL-LABEL: func.func @_QPsubr_default_any()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}
 SUBROUTINE SUBR_DEFAULT_ANY()
 !$omp declare target
 END
 
 ! ALL-LABEL: func.func @_QPsubr_default_extendedlist()
-! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}
+! ALL-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}
 SUBROUTINE SUBR_DEFAULT_EXTENDEDLIST()
 !$omp declare target(SUBR_DEFAULT_EXTENDEDLIST)
 END
@@ -161,7 +161,7 @@ SUBROUTINE SUBR_DEFAULT_EXTENDEDLIST()
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPrecursive_declare_target
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}
 RECURSIVE FUNCTION RECURSIVE_DECLARE_TARGET(INCREMENT) RESULT(K)
 !$omp declare target to(RECURSIVE_DECLARE_TARGET) device_type(nohost)
     INTEGER :: INCREMENT, K
@@ -173,7 +173,7 @@ RECURSIVE FUNCTION RECURSIVE_DECLARE_TARGET(INCREMENT) RESULT(K)
 END FUNCTION RECURSIVE_DECLARE_TARGET
 
 ! DEVICE-LABEL: func.func @_QPrecursive_declare_target_enter
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}
 RECURSIVE FUNCTION RECURSIVE_DECLARE_TARGET_ENTER(INCREMENT) RESULT(K)
 !$omp declare target enter(RECURSIVE_DECLARE_TARGET_ENTER) device_type(nohost)
     INTEGER :: INCREMENT, K
diff --git a/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap-enter.f90 b/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap-enter.f90
index 941f1eecbaf56..e8709f23c5413 100644
--- a/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap-enter.f90
+++ b/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap-enter.f90
@@ -4,7 +4,7 @@
 !RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-target-device %s -o - | FileCheck %s --check-prefix=DEVICE
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}}
 function implicitly_captured_twice() result(k)
    integer :: i
    i = 10
@@ -12,7 +12,7 @@ function implicitly_captured_twice() result(k)
 end function implicitly_captured_twice
 
 ! CHECK-LABEL: func.func @_QPtarget_function_twice_host
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter), automap = false>{{.*}}}
 function target_function_twice_host() result(i)
 !$omp declare target enter(target_function_twice_host) device_type(host)
    integer :: i
@@ -20,7 +20,7 @@ function target_function_twice_host() result(i)
 end function target_function_twice_host
 
 ! DEVICE-LABEL: func.func @_QPtarget_function_twice_device
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
 function target_function_twice_device() result(i)
 !$omp declare target enter(target_function_twice_device) device_type(nohost)
    integer :: i
@@ -30,7 +30,7 @@ end function target_function_twice_device
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_nest
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
 function implicitly_captured_nest() result(k)
    integer :: i
    i = 10
@@ -44,7 +44,7 @@ function implicitly_captured_one() result(k)
 end function implicitly_captured_one
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_two
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
 function implicitly_captured_two() result(k)
    integer :: i
    i = 10
@@ -52,7 +52,7 @@ function implicitly_captured_two() result(k)
 end function implicitly_captured_two
 
 ! DEVICE-LABEL: func.func @_QPtarget_function_test
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
 function target_function_test() result(j)
 !$omp declare target enter(target_function_test) device_type(nohost)
    integer :: i, j
@@ -63,7 +63,7 @@ end function target_function_test
 !! -----
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_nest_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}}
 function implicitly_captured_nest_twice() result(k)
    integer :: i
    i = 10
@@ -71,13 +71,13 @@ function implicitly_captured_nest_twice() result(k)
 end function implicitly_captured_nest_twice
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_one_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}}
 function implicitly_captured_one_twice() result(k)
    k = implicitly_captured_nest_twice()
 end function implicitly_captured_one_twice
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_two_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}}
 function implicitly_captured_two_twice() result(k)
    integer :: i
    i = 10
@@ -85,7 +85,7 @@ function implicitly_captured_two_twice() result(k)
 end function implicitly_captured_two_twice
 
 ! DEVICE-LABEL: func.func @_QPtarget_function_test_device
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
 function target_function_test_device() result(j)
    !$omp declare target enter(target_function_test_device) device_type(nohost)
    integer :: i, j
@@ -94,7 +94,7 @@ function target_function_test_device() result(j)
 end function target_function_test_device
 
 ! CHECK-LABEL: func.func @_QPtarget_function_test_host
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter), automap = false>{{.*}}}
 function target_function_test_host() result(j)
    !$omp declare target enter(target_function_test_host) device_type(host)
    integer :: i, j
@@ -105,7 +105,7 @@ end function target_function_test_host
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_with_dev_type_recursive
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}}
 recursive function implicitly_captured_with_dev_type_recursive(increment) result(k)
 !$omp declare target enter(implicitly_captured_with_dev_type_recursive) device_type(host)
    integer :: increment, k
@@ -117,7 +117,7 @@ recursive function implicitly_captured_with_dev_type_recursive(increment) result
 end function implicitly_captured_with_dev_type_recursive
 
 ! DEVICE-LABEL: func.func @_QPtarget_function_with_dev_type_recurse
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
 function target_function_with_dev_type_recurse() result(i)
 !$omp declare target enter(target_function_with_dev_type_recurse) device_type(nohost)
    integer :: i
@@ -129,28 +129,28 @@ end function target_function_with_dev_type_recurse
 module test_module
 contains
 ! CHECK-LABEL: func.func @_QMtest_modulePimplicitly_captured_nest_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}}
    function implicitly_captured_nest_twice() result(i)
       integer :: i
       i = 10
    end function implicitly_captured_nest_twice
 
 ! CHECK-LABEL: func.func @_QMtest_modulePimplicitly_captured_one_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (enter), automap = false>{{.*}}}
    function implicitly_captured_one_twice() result(k)
       !$omp declare target enter(implicitly_captured_one_twice) device_type(host)
       k = implicitly_captured_nest_twice()
    end function implicitly_captured_one_twice
 
 ! DEVICE-LABEL: func.func @_QMtest_modulePimplicitly_captured_two_twice
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
    function implicitly_captured_two_twice() result(y)
       integer :: y
       y = 5
    end function implicitly_captured_two_twice
 
 ! DEVICE-LABEL: func.func @_QMtest_modulePtarget_function_test_device
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
    function target_function_test_device() result(j)
       !$omp declare target enter(target_function_test_device) device_type(nohost)
       integer :: i, j
@@ -174,7 +174,7 @@ recursive subroutine implicitly_captured_recursive(increment)
 end program
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_recursive
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
 recursive subroutine implicitly_captured_recursive(increment)
    integer :: increment
    if (increment == 10) then
@@ -185,7 +185,7 @@ recursive subroutine implicitly_captured_recursive(increment)
 end subroutine
 
 ! DEVICE-LABEL: func.func @_QPcaller_recursive
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>{{.*}}}
 subroutine caller_recursive
 !$omp declare target enter(caller_recursive) device_type(nohost)
    call implicitly_captured_recursive(0)
diff --git a/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap.f90 b/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap.f90
index 8140fcc5f4813..be1e5a0d31f4b 100644
--- a/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap.f90
+++ b/flang/test/Lower/OpenMP/declare-target-implicit-func-and-subr-cap.f90
@@ -4,7 +4,7 @@
 !RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=50 -fopenmp-is-target-device %s -o - | FileCheck %s --check-prefix=DEVICE
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured(toggle) result(k)
    integer :: i, j, k
    logical :: toggle
@@ -19,7 +19,7 @@ end function implicitly_captured
 
 
 ! CHECK-LABEL: func.func @_QPtarget_function
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 function target_function(toggle) result(i)
 !$omp declare target
    integer :: i
@@ -30,7 +30,7 @@ end function target_function
 !! -----
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured_twice() result(k)
    integer :: i
    i = 10
@@ -38,7 +38,7 @@ function implicitly_captured_twice() result(k)
 end function implicitly_captured_twice
 
 ! CHECK-LABEL: func.func @_QPtarget_function_twice_host
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to), automap = false>{{.*}}}
 function target_function_twice_host() result(i)
 !$omp declare target to(target_function_twice_host) device_type(host)
    integer :: i
@@ -46,7 +46,7 @@ function target_function_twice_host() result(i)
 end function target_function_twice_host
 
 ! DEVICE-LABEL: func.func @_QPtarget_function_twice_device
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 function target_function_twice_device() result(i)
 !$omp declare target to(target_function_twice_device) device_type(nohost)
    integer :: i
@@ -56,7 +56,7 @@ end function target_function_twice_device
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_nest
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured_nest() result(k)
    integer :: i
    i = 10
@@ -70,7 +70,7 @@ function implicitly_captured_one() result(k)
 end function implicitly_captured_one
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_two
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured_two() result(k)
    integer :: i
    i = 10
@@ -78,7 +78,7 @@ function implicitly_captured_two() result(k)
 end function implicitly_captured_two
 
 ! DEVICE-LABEL: func.func @_QPtarget_function_test
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 function target_function_test() result(j)
 !$omp declare target to(target_function_test) device_type(nohost)
    integer :: i, j
@@ -89,7 +89,7 @@ end function target_function_test
 !! -----
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_nest_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured_nest_twice() result(k)
    integer :: i
    i = 10
@@ -97,13 +97,13 @@ function implicitly_captured_nest_twice() result(k)
 end function implicitly_captured_nest_twice
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_one_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured_one_twice() result(k)
    k = implicitly_captured_nest_twice()
 end function implicitly_captured_one_twice
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_two_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured_two_twice() result(k)
    integer :: i
    i = 10
@@ -111,7 +111,7 @@ function implicitly_captured_two_twice() result(k)
 end function implicitly_captured_two_twice
 
 ! DEVICE-LABEL: func.func @_QPtarget_function_test_device
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 function target_function_test_device() result(j)
    !$omp declare target to(target_function_test_device) device_type(nohost)
    integer :: i, j
@@ -120,7 +120,7 @@ function target_function_test_device() result(j)
 end function target_function_test_device
 
 ! CHECK-LABEL: func.func @_QPtarget_function_test_host
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to), automap = false>{{.*}}}
 function target_function_test_host() result(j)
    !$omp declare target to(target_function_test_host) device_type(host)
    integer :: i, j
@@ -131,7 +131,7 @@ end function target_function_test_host
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_with_dev_type_recursive
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 recursive function implicitly_captured_with_dev_type_recursive(increment) result(k)
 !$omp declare target to(implicitly_captured_with_dev_type_recursive) device_type(host)
    integer :: increment, k
@@ -143,7 +143,7 @@ recursive function implicitly_captured_with_dev_type_recursive(increment) result
 end function implicitly_captured_with_dev_type_recursive
 
 ! DEVICE-LABEL: func.func @_QPtarget_function_with_dev_type_recurse
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 function target_function_with_dev_type_recurse() result(i)
 !$omp declare target to(target_function_with_dev_type_recurse) device_type(nohost)
    integer :: i
@@ -155,28 +155,28 @@ end function target_function_with_dev_type_recurse
 module test_module
 contains
 ! CHECK-LABEL: func.func @_QMtest_modulePimplicitly_captured_nest_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
    function implicitly_captured_nest_twice() result(i)
       integer :: i
       i = 10
    end function implicitly_captured_nest_twice
 
 ! CHECK-LABEL: func.func @_QMtest_modulePimplicitly_captured_one_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
    function implicitly_captured_one_twice() result(k)
       !$omp declare target to(implicitly_captured_one_twice) device_type(host)
       k = implicitly_captured_nest_twice()
    end function implicitly_captured_one_twice
 
 ! DEVICE-LABEL: func.func @_QMtest_modulePimplicitly_captured_two_twice
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
    function implicitly_captured_two_twice() result(y)
       integer :: y
       y = 5
    end function implicitly_captured_two_twice
 
 ! DEVICE-LABEL: func.func @_QMtest_modulePtarget_function_test_device
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
    function target_function_test_device() result(j)
       !$omp declare target to(target_function_test_device) device_type(nohost)
       integer :: i, j
@@ -200,7 +200,7 @@ recursive subroutine implicitly_captured_recursive(increment)
 end program
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_recursive
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 recursive subroutine implicitly_captured_recursive(increment)
    integer :: increment
    if (increment == 10) then
@@ -211,7 +211,7 @@ recursive subroutine implicitly_captured_recursive(increment)
 end subroutine
 
 ! DEVICE-LABEL: func.func @_QPcaller_recursive
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 subroutine caller_recursive
 !$omp declare target to(caller_recursive) device_type(nohost)
    call implicitly_captured_recursive(0)
diff --git a/flang/test/Lower/OpenMP/declare-target-implicit-tarop-cap.f90 b/flang/test/Lower/OpenMP/declare-target-implicit-tarop-cap.f90
index eca527ffffcec..c1c1ea37fe471 100644
--- a/flang/test/Lower/OpenMP/declare-target-implicit-tarop-cap.f90
+++ b/flang/test/Lower/OpenMP/declare-target-implicit-tarop-cap.f90
@@ -4,7 +4,7 @@
 !RUN: bbc -emit-hlfir -fopenmp -fopenmp-version=52 -fopenmp-is-target-device %s -o - | FileCheck %s --check-prefix=DEVICE
 
 ! DEVICE-LABEL: func.func @_QPimplicit_capture
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 function implicit_capture() result(i)
    implicit none
    integer :: i
@@ -21,35 +21,35 @@ subroutine subr_target()
 !! -----
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_nest_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured_nest_twice() result(i)
    integer :: i
    i = 10
 end function implicitly_captured_nest_twice
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_one_twice
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured_one_twice() result(k)
 !$omp declare target to(implicitly_captured_one_twice) device_type(host)
    k = implicitly_captured_nest_twice()
 end function implicitly_captured_one_twice
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_nest_twice_enter
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter), automap = false>{{.*}}}
 function implicitly_captured_nest_twice_enter() result(i)
    integer :: i
    i = 10
 end function implicitly_captured_nest_twice_enter
 
 ! CHECK-LABEL: func.func @_QPimplicitly_captured_one_twice_enter
-! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter)>{{.*}}}
+! CHECK-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (enter), automap = false>{{.*}}}
 function implicitly_captured_one_twice_enter() result(k)
 !$omp declare target enter(implicitly_captured_one_twice_enter) device_type(host)
    k = implicitly_captured_nest_twice_enter()
 end function implicitly_captured_one_twice_enter
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_two_twice
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 function implicitly_captured_two_twice() result(y)
    integer :: y
    y = 5
@@ -67,7 +67,7 @@ end function target_function_test_device
 !! -----
 
 ! DEVICE-LABEL: func.func @_QPimplicitly_captured_recursive
-! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>{{.*}}}
+! DEVICE-SAME: {{.*}}attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>{{.*}}}
 recursive function implicitly_captured_recursive(increment) result(k)
    integer :: increment, k
    if (increment == 10) then
diff --git a/flang/test/Lower/OpenMP/declare-target-unnamed-main.f90 b/flang/test/Lower/OpenMP/declare-target-unnamed-main.f90
index b7d6d2fa232ad..f54f7ed14d1d5 100644
--- a/flang/test/Lower/OpenMP/declare-target-unnamed-main.f90
+++ b/flang/test/Lower/OpenMP/declare-target-unnamed-main.f90
@@ -7,7 +7,7 @@
 ! appropriately mark the function as declare target, even when
 ! unused within the target region.
 
-!CHECK: func.func @_QPfoo(%{{.*}}: !fir.ref<f32>{{.*}}) -> f32 attributes {{{.*}}omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>{{.*}}}
+!CHECK: func.func @_QPfoo(%{{.*}}: !fir.ref<f32>{{.*}}) -> f32 attributes {{{.*}}omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>{{.*}}}
 
 interface
 real function foo (x)
diff --git a/flang/test/Lower/OpenMP/function-filtering-2.f90 b/flang/test/Lower/OpenMP/function-filtering-2.f90
index a94cbfffca5bf..34d910c53d6ea 100644
--- a/flang/test/Lower/OpenMP/function-filtering-2.f90
+++ b/flang/test/Lower/OpenMP/function-filtering-2.f90
@@ -5,13 +5,13 @@
 ! RUN: bbc -fopenmp -fopenmp-version=52 -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-HOST,MLIR-ALL %s
 ! RUN: %if amdgpu-registered-target %{ bbc -target amdgcn-amd-amdhsa -fopenmp -fopenmp-version=52 -fopenmp-is-target-device -emit-hlfir %s -o - | FileCheck --check-prefixes=MLIR-DEVICE,MLIR-ALL %s %}
 
-! MLIR: func.func @{{.*}}implicit_invocation() attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>}
+! MLIR: func.func @{{.*}}implicit_invocation() attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>}
 ! MLIR: return
 ! LLVM: define {{.*}} @{{.*}}implicit_invocation{{.*}}(
 subroutine implicit_invocation()
 end subroutine implicit_invocation
 
-! MLIR: func.func @{{.*}}declaretarget() attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>}
+! MLIR: func.func @{{.*}}declaretarget() attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>}
 ! MLIR: return
 ! LLVM: define {{.*}} @{{.*}}declaretarget{{.*}}(
 subroutine declaretarget()
@@ -19,7 +19,7 @@ subroutine declaretarget()
     call implicit_invocation()
 end subroutine declaretarget
 
-! MLIR: func.func @{{.*}}declaretarget_enter() attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter)>}
+! MLIR: func.func @{{.*}}declaretarget_enter() attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (enter), automap = false>}
 ! MLIR: return
 ! LLVM: define {{.*}} @{{.*}}declaretarget_enter{{.*}}(
 subroutine declaretarget_enter()
@@ -27,7 +27,7 @@ subroutine declaretarget_enter()
     call implicit_invocation()
 end subroutine declaretarget_enter
 
-! MLIR: func.func @{{.*}}no_declaretarget() attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to)>}
+! MLIR: func.func @{{.*}}no_declaretarget() attributes {omp.declare_target = #omp.declaretarget<device_type = (nohost), capture_clause = (to), automap = false>}
 ! MLIR: return
 ! LLVM: define {{.*}} @{{.*}}no_declaretarget{{.*}}(
 subroutine no_declaretarget()
diff --git a/flang/test/Lower/OpenMP/map-no-modifier-v60.f90 b/flang/test/Lower/OpenMP/map-no-modifier-v60.f90
new file mode 100644
index 0000000000000..bcc37e48f8c11
--- /dev/null
+++ b/flang/test/Lower/OpenMP/map-no-modifier-v60.f90
@@ -0,0 +1,12 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=60 %s -o - | FileCheck %s
+
+!This shouldn't crash. Check for a symptom of a successful compilation
+!CHECK: omp.map.info
+
+subroutine f00
+  implicit none
+  integer :: x
+  !$omp target map(x)
+  !$omp end target
+end
+  
diff --git a/flang/test/Lower/OpenMP/omp-declare-target-program-var.f90 b/flang/test/Lower/OpenMP/omp-declare-target-program-var.f90
index d18f42ae3ceb0..dc23a8190cb8d 100644
--- a/flang/test/Lower/OpenMP/omp-declare-target-program-var.f90
+++ b/flang/test/Lower/OpenMP/omp-declare-target-program-var.f90
@@ -5,7 +5,7 @@ PROGRAM main
     ! HOST-DAG: %[[I_REF:.*]] = fir.alloca f32 {bindc_name = "i", uniq_name = "_QFEi"}
     ! HOST-DAG: %[[I_DECL:.*]]:2 = hlfir.declare %[[I_REF]] {uniq_name = "_QFEi"} : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
     REAL :: I
-    ! ALL-DAG: fir.global internal @_QFEi {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to)>} : f32 {
+    ! ALL-DAG: fir.global internal @_QFEi {omp.declare_target = #omp.declaretarget<device_type = (any), capture_clause = (to), automap = false>} : f32 {
     ! ALL-DAG: %[[UNDEF:.*]] = fir.zero_bits f32
     ! ALL-DAG: fir.has_value %[[UNDEF]] : f32
     ! ALL-DAG: }
diff --git a/flang/test/Lower/OpenMP/private-character.f90 b/flang/test/Lower/OpenMP/private-character.f90
new file mode 100644
index 0000000000000..3f0a5bb81cc38
--- /dev/null
+++ b/flang/test/Lower/OpenMP/private-character.f90
@@ -0,0 +1,35 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+!CHECK-LABEL: func @_QPtest_dynlen_char_ptr
+!CHECK:         omp.parallel private(@{{.*}} %{{.*}}#0 -> %[[A:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>) {
+!CHECK:           %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_dynlen_char_ptrEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>)
+!CHECK:           %[[A_VAL:.*]] = fir.load %[[A_DECL]]#0 : !fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>
+!CHECK:           %[[LEN:.*]] = fir.box_elesize %[[A_VAL]] : (!fir.box<!fir.ptr<!fir.char<1,?>>>) -> index
+!CHECK:           %[[A_BOX_NONE:.*]] = fir.convert %[[A_DECL]]#0 : (!fir.ref<!fir.box<!fir.ptr<!fir.char<1,?>>>>) -> !fir.ref<!fir.box<none>>
+!CHECK:           %[[LEN_I64:.*]] = fir.convert %[[LEN]] : (index) -> i64
+!CHECK:           fir.call @_FortranAPointerNullifyCharacter(%[[A_BOX_NONE]], %[[LEN_I64]], {{.*}})
+subroutine test_dynlen_char_ptr(i)
+  character(i), pointer :: a
+
+  !$omp parallel private(a)
+    allocate(a)
+    a = "abc"
+  !$omp end parallel
+end subroutine
+
+!CHECK-LABEL: func @_QPtest_dynlen_char_ptr_array
+!CHECK:         omp.parallel private(@{{.*}} %{{.*}}#0 -> %[[A:.*]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>) {
+!CHECK:           %[[A_DECL:.*]]:2 = hlfir.declare %[[A]] {fortran_attrs = #fir.var_attrs<pointer>, uniq_name = "_QFtest_dynlen_char_ptr_arrayEa"} : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>) -> (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>)
+!CHECK:           %[[A_VAL:.*]] = fir.load %[[A_DECL]]#0
+!CHECK:           %[[LEN:.*]] = fir.box_elesize %[[A_VAL]]
+!CHECK:           %[[A_BOX_NONE:.*]] = fir.convert %[[A_DECL]]#0 : (!fir.ref<!fir.box<!fir.ptr<!fir.array<?x!fir.char<1,?>>>>>) -> !fir.ref<!fir.box<none>>
+!CHECK:           %[[LEN_I64:.*]] = fir.convert %[[LEN]] : (index) -> i64
+!CHECK:           fir.call @_FortranAPointerNullifyCharacter(%[[A_BOX_NONE]], %[[LEN_I64]], {{.*}})
+subroutine test_dynlen_char_ptr_array(i)
+  character(i), pointer :: a(:)
+
+  !$omp parallel private(a)
+    allocate(a(i))
+    a = "abc"
+  !$omp end parallel
+end subroutine
diff --git a/flang/test/Lower/OpenMP/privatize_predetermined_only_when_defined_by_eval.f90 b/flang/test/Lower/OpenMP/privatize_predetermined_only_when_defined_by_eval.f90
new file mode 100644
index 0000000000000..7671073c2598a
--- /dev/null
+++ b/flang/test/Lower/OpenMP/privatize_predetermined_only_when_defined_by_eval.f90
@@ -0,0 +1,35 @@
+! Fixes a regression uncovered by Fujitsu test 0686_0024.f90. In particular,
+! verifies that a pre-determined symbol is only privatized by its defining
+! evaluation (e.g. the loop for which the symbol was marked as pre-determined).
+
+! RUN: %flang_fc1 -emit-hlfir -fopenmp %s -o - | FileCheck %s
+
+subroutine privatize_predetermined_when_defined_by_eval
+  integer::i,ii
+  integer::j
+
+  !$omp parallel
+    !$omp do lastprivate(ii)
+    do i=1,10
+      do ii=1,10
+      enddo
+    enddo
+
+    !$omp do 
+    do j=1,ii
+    enddo
+  !$omp end parallel
+end subroutine
+
+! Verify that nothing is privatized by the `omp.parallel` op.
+! CHECK: omp.parallel {
+
+! Verify that `i` and `ii` are privatized by the first loop.
+! CHECK:   omp.wsloop private(@{{.*}}ii_private_i32 %{{.*}}#0 -> %{{.*}}, @{{.*}}i_private_i32 %2#0 -> %{{.*}} : {{.*}}) {
+! CHECK:   }
+
+! Verify that `j` is privatized by the second loop.
+! CHECK:   omp.wsloop private(@{{.*}}j_private_i32 %{{.*}}#0 -> %{{.*}} : {{.*}}) {
+! CHECK:   }
+
+! CHECK: }
diff --git a/flang/test/Lower/amdgcn-complex.f90 b/flang/test/Lower/amdgcn-complex.f90
index f15c7db2b7316..4ee5de4d2842e 100644
--- a/flang/test/Lower/amdgcn-complex.f90
+++ b/flang/test/Lower/amdgcn-complex.f90
@@ -1,21 +1,27 @@
 ! REQUIRES: amdgpu-registered-target
-! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-fir -flang-deprecated-no-hlfir %s -o - | FileCheck %s
+! RUN: %flang_fc1 -triple amdgcn-amd-amdhsa -emit-fir %s -o - | FileCheck %s
 
+! CHECK-LABEL: func @_QPcabsf_test(
+! CHECK: complex.abs
+! CHECK-NOT: fir.call @cabsf
 subroutine cabsf_test(a, b)
    complex :: a
    real :: b
    b = abs(a)
 end subroutine
 
-! CHECK-LABEL: func @_QPcabsf_test(
-! CHECK: complex.abs
-! CHECK-NOT: fir.call @cabsf
-
+! CHECK-LABEL: func @_QPcexpf_test(
+! CHECK: complex.exp
+! CHECK-NOT: fir.call @cexpf
 subroutine cexpf_test(a, b)
    complex :: a, b
    b = exp(a)
 end subroutine
 
-! CHECK-LABEL: func @_QPcexpf_test(
-! CHECK: complex.exp
-! CHECK-NOT: fir.call @cexpf
+! CHECK-LABEL: func @_QPpow_test(
+! CHECK: complex.pow
+! CHECK-NOT: fir.call @_FortranAcpowi
+subroutine pow_test(a, b, c)
+   complex :: a, b, c
+   a = b**c
+end subroutine pow_test
diff --git a/flang/test/Lower/do_concurrent_loop_in_nested_block.f90 b/flang/test/Lower/do_concurrent_loop_in_nested_block.f90
new file mode 100644
index 0000000000000..8c4f5048e87fa
--- /dev/null
+++ b/flang/test/Lower/do_concurrent_loop_in_nested_block.f90
@@ -0,0 +1,26 @@
+! RUN: %flang_fc1 -emit-hlfir -mmlir --enable-delayed-privatization-staging=true -o - %s | FileCheck %s
+
+subroutine loop_in_nested_block
+  implicit none
+  integer :: i, j
+
+  do concurrent (i=1:10) local(j)
+    block
+      do j=1,20
+      end do
+    end block
+  end do
+end subroutine
+
+! CHECK-LABEL: func.func @_QPloop_in_nested_block() {
+! CHECK:         %[[OUTER_J_DECL:.*]]:2 = hlfir.declare %{{.*}} {uniq_name = "{{.*}}Ej"}
+! CHECK:         fir.do_concurrent {
+! CHECK:           fir.do_concurrent.loop {{.*}} local(@{{.*}} %[[OUTER_J_DECL]]#0 -> %[[LOCAL_J_ARG:.*]] : !fir.ref<i32>) {
+! CHECK:             %[[LOCAL_J_DECL:.*]]:2 = hlfir.declare %[[LOCAL_J_ARG]]
+! CHECK:             fir.do_loop {{.*}} iter_args(%[[NESTED_LOOP_ARG:.*]] = {{.*}}) {
+! CHECK:               fir.store %[[NESTED_LOOP_ARG]] to %[[LOCAL_J_DECL]]#0
+! CHECK:             }
+! CHECK:           }
+! CHECK:         }
+! CHECK:       }
+
diff --git a/flang/test/Lower/do_loop_unstructured.f90 b/flang/test/Lower/do_loop_unstructured.f90
index d8890b2d0926e..176ea5ccee129 100644
--- a/flang/test/Lower/do_loop_unstructured.f90
+++ b/flang/test/Lower/do_loop_unstructured.f90
@@ -232,3 +232,22 @@ subroutine nested_structured_in_unstructured()
 ! CHECK:   cf.br ^[[HEADER]]
 ! CHECK: ^[[EXIT]]:
 ! CHECK:   return
+
+subroutine unstructured_do_concurrent
+  logical :: success
+  do concurrent (i=1:10) local(success)
+    error stop "fail"
+  enddo
+end
+! CHECK-LABEL: func.func @_QPunstructured_do_concurrent
+! CHECK:         %[[ITER_VAR:.*]] = fir.alloca i32
+
+! CHECK:       ^[[HEADER]]:
+! CHECK:         %{{.*}} = fir.load %[[ITER_VAR]] : !fir.ref<i32>
+! CHECK:         cf.cond_br %{{.*}}, ^[[BODY:.*]], ^[[EXIT:.*]]
+
+! CHECK:       ^[[BODY]]:
+! CHECK-NEXT:    %{{.*}} = fir.alloca !fir.logical<4> {bindc_name = "success", {{.*}}}
+
+! CHECK:       ^[[EXIT]]:
+! CHECK-NEXT:    return
diff --git a/flang/test/Parser/OpenMP/critical-unparse-with-symbols.f90 b/flang/test/Parser/OpenMP/critical-unparse-with-symbols.f90
index 4d0d93ac48740..e5e7561d4f63e 100644
--- a/flang/test/Parser/OpenMP/critical-unparse-with-symbols.f90
+++ b/flang/test/Parser/OpenMP/critical-unparse-with-symbols.f90
@@ -13,9 +13,9 @@ subroutine f
 !UNPARSE:  implicit none
 !UNPARSE:  !DEF: /f/x ObjectEntity INTEGER(4)
 !UNPARSE:  integer x
-!UNPARSE: !$omp critical (c)
+!UNPARSE: !$omp critical(c)
 !UNPARSE:  !REF: /f/x
 !UNPARSE:  x = 0
-!UNPARSE: !$omp end critical (c)
+!UNPARSE: !$omp end critical(c)
 !UNPARSE: end subroutine
 
diff --git a/flang/test/Parser/OpenMP/dyn-groupprivate-clause.f90 b/flang/test/Parser/OpenMP/dyn-groupprivate-clause.f90
new file mode 100644
index 0000000000000..7d41efd348e50
--- /dev/null
+++ b/flang/test/Parser/OpenMP/dyn-groupprivate-clause.f90
@@ -0,0 +1,70 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=61 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=61 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine f00(n)
+  implicit none
+  integer :: n
+  !$omp target dyn_groupprivate(n)
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f00 (n)
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER n
+!UNPARSE: !$OMP TARGET DYN_GROUPPRIVATE(n)
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginDirective
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> DynGroupprivate -> OmpDynGroupprivateClause
+!PARSE-TREE: | | Scalar -> Integer -> Expr = 'n'
+!PARSE-TREE: | | | Designator -> DataRef -> Name = 'n'
+!PARSE-TREE: | Flags = None
+
+
+subroutine f01(n)
+  implicit none
+  integer :: n
+  !$omp target dyn_groupprivate(strict: n)
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f01 (n)
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER n
+!UNPARSE: !$OMP TARGET DYN_GROUPPRIVATE(STRICT: n)
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginDirective
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> DynGroupprivate -> OmpDynGroupprivateClause
+!PARSE-TREE: | | Modifier -> OmpPrescriptiveness -> Value = Strict
+!PARSE-TREE: | | Scalar -> Integer -> Expr = 'n'
+!PARSE-TREE: | | | Designator -> DataRef -> Name = 'n'
+!PARSE-TREE: | Flags = None
+
+
+subroutine f02(n)
+  implicit none
+  integer :: n
+  !$omp target dyn_groupprivate(fallback, cgroup: n)
+  !$omp end target
+end
+
+!UNPARSE: SUBROUTINE f02 (n)
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER n
+!UNPARSE: !$OMP TARGET DYN_GROUPPRIVATE(FALLBACK, CGROUP: n)
+!UNPARSE: !$OMP END TARGET
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: OmpBeginDirective
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = target
+!PARSE-TREE: | OmpClauseList -> OmpClause -> DynGroupprivate -> OmpDynGroupprivateClause
+!PARSE-TREE: | | Modifier -> OmpPrescriptiveness -> Value = Fallback
+!PARSE-TREE: | | Modifier -> OmpAccessGroup -> Value = Cgroup
+!PARSE-TREE: | | Scalar -> Integer -> Expr = 'n'
+!PARSE-TREE: | | | Designator -> DataRef -> Name = 'n'
+!PARSE-TREE: | Flags = None
diff --git a/flang/test/Parser/OpenMP/groupprivate.f90 b/flang/test/Parser/OpenMP/groupprivate.f90
new file mode 100644
index 0000000000000..8bd840147a2dd
--- /dev/null
+++ b/flang/test/Parser/OpenMP/groupprivate.f90
@@ -0,0 +1,30 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+module m
+implicit none
+
+integer :: x, y(10), z
+!$omp groupprivate(x, y) device_type(nohost)
+!$omp groupprivate(z)
+
+end module
+
+!UNPARSE: MODULE m
+!UNPARSE:  IMPLICIT NONE
+!UNPARSE:  INTEGER x, y(10_4), z
+!UNPARSE: !$OMP GROUPPRIVATE(x, y) DEVICE_TYPE(NOHOST)
+!UNPARSE: !$OMP GROUPPRIVATE(z)
+!UNPARSE: END MODULE
+
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPGroupprivate -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = groupprivate
+!PARSE-TREE: | OmpArgumentList -> OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'x'
+!PARSE-TREE: | OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'y'
+!PARSE-TREE: | OmpClauseList -> OmpClause -> DeviceType -> OmpDeviceTypeClause -> DeviceTypeDescription = Nohost
+!PARSE-TREE: | Flags = None
+!PARSE-TREE: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPGroupprivate -> OmpDirectiveSpecification
+!PARSE-TREE: | OmpDirectiveName -> llvm::omp::Directive = groupprivate
+!PARSE-TREE: | OmpArgumentList -> OmpArgument -> OmpLocator -> OmpObject -> Designator -> DataRef -> Name = 'z'
+!PARSE-TREE: | OmpClauseList ->
+!PARSE-TREE: | Flags = None
diff --git a/flang/test/Parser/OpenMP/workdistribute.f90 b/flang/test/Parser/OpenMP/workdistribute.f90
new file mode 100644
index 0000000000000..09273ab0485cd
--- /dev/null
+++ b/flang/test/Parser/OpenMP/workdistribute.f90
@@ -0,0 +1,27 @@
+!RUN: %flang_fc1 -fdebug-unparse -fopenmp -fopenmp-version=60 %s | FileCheck --ignore-case --check-prefix="UNPARSE" %s
+!RUN: %flang_fc1 -fdebug-dump-parse-tree -fopenmp -fopenmp-version=60 %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+!UNPARSE: SUBROUTINE teams_workdistribute
+!UNPARSE:  USE :: iso_fortran_env
+!UNPARSE:  REAL(KIND=4_4) a
+!UNPARSE:  REAL(KIND=4_4), DIMENSION(10_4) :: x
+!UNPARSE:  REAL(KIND=4_4), DIMENSION(10_4) :: y
+!UNPARSE: !$OMP TEAMS WORKDISTRIBUTE
+!UNPARSE:   y=a*x+y
+!UNPARSE: !$OMP END TEAMS WORKDISTRIBUTE
+!UNPARSE: END SUBROUTINE teams_workdistribute
+
+!PARSE-TREE: | | | OmpBeginDirective
+!PARSE-TREE: | | | | OmpDirectiveName -> llvm::omp::Directive = teams workdistribute
+!PARSE-TREE: | | | OmpEndDirective
+!PARSE-TREE: | | | | OmpDirectiveName -> llvm::omp::Directive = teams workdistribute
+
+subroutine teams_workdistribute()
+  use iso_fortran_env
+  real(kind=real32) :: a
+  real(kind=real32), dimension(10) :: x
+  real(kind=real32), dimension(10) :: y
+  !$omp teams workdistribute
+  y = a * x + y
+  !$omp end teams workdistribute
+end subroutine teams_workdistribute
diff --git a/flang/test/Preprocessing/defines_pic_frontend.F90 b/flang/test/Preprocessing/defines_pic_frontend.F90
new file mode 100644
index 0000000000000..ad871e0acfab3
--- /dev/null
+++ b/flang/test/Preprocessing/defines_pic_frontend.F90
@@ -0,0 +1,38 @@
+! Check that the pie/pic/PIE/PIC macros are defined properly through the frontend driver
+
+! RUN: %flang_fc1 -dM -E -o - %s \
+! RUN:   | FileCheck %s
+! CHECK-NOT: #define __PIC__
+! CHECK-NOT: #define __PIE__
+! CHECK-NOT: #define __pic__
+! CHECK-NOT: #define __pie__
+!
+! RUN: %flang_fc1 -pic-level 1 -dM -E -o - %s \
+! RUN:   | FileCheck --check-prefix=CHECK-PIC1 %s
+! CHECK-PIC1: #define __PIC__ 1
+! CHECK-PIC1-NOT: #define __PIE__
+! CHECK-PIC1: #define __pic__ 1
+! CHECK-PIC1-NOT: #define __pie__
+!
+! RUN: %flang_fc1 -pic-level 2 -dM -E -o - %s \
+! RUN:   | FileCheck --check-prefix=CHECK-PIC2 %s
+! CHECK-PIC2: #define __PIC__ 2
+! CHECK-PIC2-NOT: #define __PIE__
+! CHECK-PIC2: #define __pic__ 2
+! CHECK-PIC2-NOT: #define __pie__
+!
+! RUN: %flang_fc1 -pic-level 1 -pic-is-pie -dM -E -o - %s \
+! RUN:   | FileCheck --check-prefix=CHECK-PIE1 %s
+! CHECK-PIE1: #define __PIC__ 1
+! CHECK-PIE1: #define __PIE__ 1
+! CHECK-PIE1: #define __pic__ 1
+! CHECK-PIE1: #define __pie__ 1
+!
+! RUN: %flang_fc1 -pic-level 2 -pic-is-pie -dM -E -o - %s \
+! RUN:   | FileCheck --check-prefix=CHECK-PIE2 %s
+! CHECK-PIE2: #define __PIC__ 2
+! CHECK-PIE2: #define __PIE__ 2
+! CHECK-PIE2: #define __pic__ 2
+! CHECK-PIE2: #define __pie__ 2
+
+integer, parameter :: pic_level = __pic__
diff --git a/flang/test/Semantics/OpenACC/acc-branch.f90 b/flang/test/Semantics/OpenACC/acc-branch.f90
index a2d7b583f6794..0a1bdc3afd65a 100644
--- a/flang/test/Semantics/OpenACC/acc-branch.f90
+++ b/flang/test/Semantics/OpenACC/acc-branch.f90
@@ -13,7 +13,7 @@ subroutine openacc_clause_validity
   !$acc parallel
   !$acc loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     !ERROR: RETURN statement is not allowed in a PARALLEL construct
     return
   end do
@@ -21,21 +21,21 @@ subroutine openacc_clause_validity
 
   !$acc parallel loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     !ERROR: RETURN statement is not allowed in a PARALLEL LOOP construct
     return
   end do
 
   !$acc serial loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     !ERROR: RETURN statement is not allowed in a SERIAL LOOP construct
     return
   end do
 
   !$acc kernels loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     !ERROR: RETURN statement is not allowed in a KERNELS LOOP construct
     return
   end do
@@ -43,7 +43,7 @@ subroutine openacc_clause_validity
   !$acc parallel
   !$acc loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     if(i == N-1) THEN
       exit
     end if
@@ -81,7 +81,7 @@ subroutine openacc_clause_validity
         exit fortname
         !$acc loop
           do i = 1, N
-            a(i) = 3.14
+            a(i) = 3.14d0
             if(i == N-1) THEN
               !ERROR: EXIT to construct 'name1' outside of PARALLEL construct is not allowed
               exit name1
@@ -89,7 +89,7 @@ subroutine openacc_clause_validity
           end do
 
           loop2: do i = 1, N
-            a(i) = 3.33
+            a(i) = 3.33d0
             !ERROR: EXIT to construct 'thisblk' outside of PARALLEL construct is not allowed
             exit thisblk
           end do loop2
@@ -102,7 +102,7 @@ subroutine openacc_clause_validity
   !$acc parallel
   !$acc loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     ifname: if (i == 2) then
       ! This is allowed.
       exit ifname
@@ -113,7 +113,7 @@ subroutine openacc_clause_validity
   !$acc parallel
   !$acc loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     if(i == N-1) THEN
       stop 999 ! no error
     end if
@@ -122,7 +122,7 @@ subroutine openacc_clause_validity
 
   !$acc kernels
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     !ERROR: RETURN statement is not allowed in a KERNELS construct
     return
   end do
@@ -130,7 +130,7 @@ subroutine openacc_clause_validity
 
   !$acc kernels
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     if(i == N-1) THEN
       exit
     end if
@@ -139,7 +139,7 @@ subroutine openacc_clause_validity
 
   !$acc kernels
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     if(i == N-1) THEN
       stop 999 ! no error
     end if
@@ -148,7 +148,7 @@ subroutine openacc_clause_validity
 
   !$acc serial
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     !ERROR: RETURN statement is not allowed in a SERIAL construct
     return
   end do
@@ -156,7 +156,7 @@ subroutine openacc_clause_validity
 
   !$acc serial
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     if(i == N-1) THEN
       exit
     end if
@@ -168,7 +168,7 @@ subroutine openacc_clause_validity
   do i = 1, N
     ifname: if (.true.) then
       print *, "LGTM"
-    a(i) = 3.14
+    a(i) = 3.14d0
     if(i == N-1) THEN
         !ERROR: EXIT to construct 'name2' outside of SERIAL construct is not allowed
         exit name2
@@ -181,7 +181,7 @@ subroutine openacc_clause_validity
 
   !$acc serial
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
     if(i == N-1) THEN
       stop 999 ! no error
     end if
diff --git a/flang/test/Semantics/OpenACC/acc-init-validity.f90 b/flang/test/Semantics/OpenACC/acc-init-validity.f90
index 083a2415a5d77..bede04dc9f38e 100644
--- a/flang/test/Semantics/OpenACC/acc-init-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-init-validity.f90
@@ -44,7 +44,7 @@ program openacc_init_validity
   do i = 1, N
     !ERROR: Directive INIT may not be called within a compute region
     !$acc init
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -53,7 +53,7 @@ program openacc_init_validity
   do i = 1, N
     !ERROR: Directive INIT may not be called within a compute region
     !$acc init
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial
 
@@ -62,7 +62,7 @@ program openacc_init_validity
   do i = 1, N
     !ERROR: Directive INIT may not be called within a compute region
     !$acc init
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end kernels
 
@@ -70,21 +70,21 @@ program openacc_init_validity
   do i = 1, N
     !ERROR: Directive INIT may not be called within a compute region
     !$acc init
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc serial loop
   do i = 1, N
     !ERROR: Directive INIT may not be called within a compute region
     !$acc init
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop
   do i = 1, N
     !ERROR: Directive INIT may not be called within a compute region
     !$acc init
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !ERROR: At most one IF clause can appear on the INIT directive
diff --git a/flang/test/Semantics/OpenACC/acc-kernels-loop.f90 b/flang/test/Semantics/OpenACC/acc-kernels-loop.f90
index cfe27e4f8fca1..65c6293485aeb 100644
--- a/flang/test/Semantics/OpenACC/acc-kernels-loop.f90
+++ b/flang/test/Semantics/OpenACC/acc-kernels-loop.f90
@@ -31,75 +31,75 @@ program openacc_kernels_loop_validity
 
   !$acc kernels loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end kernels loop
 
   !$acc kernels loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end kernels loop
 
   !$acc kernels loop num_gangs(8)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop num_gangs(gang_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop num_gangs(8)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop num_workers(worker_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop num_workers(8)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop vector_length(vector_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop vector_length(128)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop num_gangs(gang_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
 
   !$acc kernels loop if(.TRUE.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop if(ifCondition)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !ERROR: Unmatched END SERIAL LOOP directive
   !$acc end serial loop
@@ -107,194 +107,194 @@ program openacc_kernels_loop_validity
   !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the KERNELS LOOP directive
   !$acc kernels loop device_type(*) if(.TRUE.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end kernels loop
 
   !$acc kernels loop async
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop async(1)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop async(async1)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop wait(wait1)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop wait(wait1, wait2)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop wait(wait1) wait(wait2)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop wait(1, 2) async(3)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop wait(queues: 1, 2) async(3)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop wait(devnum: 1: 1, 2) async(3)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop wait(devnum: 1: queues: 1, 2) async(3)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop num_gangs(8)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop num_workers(8)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop vector_length(128)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop if(.true.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop if(ifCondition)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !ERROR: At most one IF clause can appear on the KERNELS LOOP directive
   !$acc kernels loop if(.true.) if(ifCondition)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop self
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop self(.true.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop self(ifCondition)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop copy(aa) copyin(bb) copyout(cc)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop copy(aa, bb) copyout(zero: cc)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop present(aa, bb) create(cc)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop copyin(readonly: aa, bb) create(zero: cc)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop deviceptr(aa, bb) no_create(cc)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !ERROR: Argument `aa` on the ATTACH clause must be a variable or array with the POINTER or ALLOCATABLE attribute
   !$acc kernels loop attach(aa, dd, p)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop private(aa, bb, cc)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop default(none) private(N, a)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop default(none)
   !ERROR: The DEFAULT(NONE) clause requires that 'n' must be listed in a data-mapping clause
   do i = 1, N
     !ERROR: The DEFAULT(NONE) clause requires that 'a' must be listed in a data-mapping clause
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop default(present)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !ERROR: At most one DEFAULT clause can appear on the KERNELS LOOP directive
   !$acc kernels loop default(none) default(present)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop device_type(*)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop device_type(multicore)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop device_type(host, multicore)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop device_type(*) async wait num_gangs(8) num_workers(8) vector_length(128)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop device_type(*) async
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the KERNELS LOOP directive
   !$acc kernels loop device_type(*) if(.TRUE.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc parallel loop
diff --git a/flang/test/Semantics/OpenACC/acc-kernels.f90 b/flang/test/Semantics/OpenACC/acc-kernels.f90
index 44e532a9012bf..9c3adfb233ff7 100644
--- a/flang/test/Semantics/OpenACC/acc-kernels.f90
+++ b/flang/test/Semantics/OpenACC/acc-kernels.f90
@@ -177,14 +177,14 @@ program openacc_kernels_validity
 
   !$acc kernels device_type(*) async
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end kernels
 
   !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the KERNELS directive
   !$acc kernels device_type(*) if(.TRUE.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end kernels
 
diff --git a/flang/test/Semantics/OpenACC/acc-loop.f90 b/flang/test/Semantics/OpenACC/acc-loop.f90
index 9301cf85305d4..77c427e0a85ae 100644
--- a/flang/test/Semantics/OpenACC/acc-loop.f90
+++ b/flang/test/Semantics/OpenACC/acc-loop.f90
@@ -31,35 +31,35 @@ program openacc_loop_validity
   !$acc parallel
   !$acc loop tile(2)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel device_type(*) num_gangs(2)
   !$acc loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop seq
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop independent
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop auto
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -67,35 +67,35 @@ program openacc_loop_validity
   !ERROR: At most one VECTOR clause can appear on the LOOP directive or in group separated by the DEVICE_TYPE clause
   !$acc loop vector vector(128)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop vector
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop vector(10)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop vector(vector_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop vector(length: vector_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -103,35 +103,35 @@ program openacc_loop_validity
   !ERROR: At most one WORKER clause can appear on the LOOP directive or in group separated by the DEVICE_TYPE clause
   !$acc loop worker worker(10)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop worker
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop worker(10)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop worker(worker_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop worker(num: worker_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -139,58 +139,58 @@ program openacc_loop_validity
   !ERROR: At most one GANG clause can appear on the LOOP directive or in group separated by the DEVICE_TYPE clause
   !$acc loop gang gang(gang_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc loop gang device_type(default) gang(gang_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !ERROR: At most one GANG clause can appear on the PARALLEL LOOP directive or in group separated by the DEVICE_TYPE clause
   !$acc parallel loop gang gang(gang_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc parallel loop gang device_type(default) gang(gang_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc parallel
   !$acc loop gang(gang_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop gang(num: gang_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop gang(gang_size, static:*)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop gang(num: gang_size, static:*)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel
   !$acc loop gang(num: gang_size, static: gang_size)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -222,7 +222,7 @@ program openacc_loop_validity
   !$acc loop collapse(-1)
   do i = 1, N
     do j = 1, N
-      a(i) = 3.14 + j
+      a(i) = 3.14d0 + j
     end do
   end do
   !$acc end parallel
@@ -231,7 +231,7 @@ program openacc_loop_validity
   !ERROR: Clause PRIVATE is not allowed after clause DEVICE_TYPE on the LOOP directive
   !$acc loop device_type(*) private(i)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -239,7 +239,7 @@ program openacc_loop_validity
   !ERROR: Clause GANG is not allowed if clause SEQ appears on the LOOP directive
   !$acc loop gang seq
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -247,7 +247,7 @@ program openacc_loop_validity
   !ERROR: Clause WORKER is not allowed if clause SEQ appears on the LOOP directive
   !$acc loop worker seq
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -255,7 +255,7 @@ program openacc_loop_validity
   !ERROR: Clause VECTOR is not allowed if clause SEQ appears on the LOOP directive
   !$acc loop vector seq
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -355,7 +355,7 @@ program openacc_loop_validity
   !$acc parallel device_type(*) if(.TRUE.)
   !$acc loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -363,7 +363,7 @@ program openacc_loop_validity
   do i = 1, N
     !ERROR: Loop control is not present in the DO LOOP
     do
-      a(i) = 3.14
+      a(i) = 3.14d0
     end do
   end do
 
diff --git a/flang/test/Semantics/OpenACC/acc-parallel-loop-validity.f90 b/flang/test/Semantics/OpenACC/acc-parallel-loop-validity.f90
index 78e1a7ad7c452..96962bb1aa58a 100644
--- a/flang/test/Semantics/OpenACC/acc-parallel-loop-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-parallel-loop-validity.f90
@@ -19,64 +19,64 @@ program openacc_parallel_loop_validity
 
   !$acc parallel loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc parallel loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel loop
 
   !$acc parallel loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
   !$acc parallel loop tile(2)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc parallel loop self
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !ERROR: SELF clause on the PARALLEL LOOP directive only accepts optional scalar logical expression
   !$acc parallel loop self(bb, cc(:,:))
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc parallel loop self(.true.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc parallel loop self(ifCondition)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc parallel loop tile(2, 2)
   do i = 1, N
     do j = 1, N
-      aa(i, j) = 3.14
+      aa(i, j) = 3.14d0
     end do
   end do
 
   !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the PARALLEL LOOP directive
   !$acc parallel loop device_type(*) if(.TRUE.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel loop
 
   !$acc kernels loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !ERROR: Unmatched END PARALLEL LOOP directive
   !$acc end parallel loop
diff --git a/flang/test/Semantics/OpenACC/acc-parallel.f90 b/flang/test/Semantics/OpenACC/acc-parallel.f90
index b9d989ec6fced..635c547f744c8 100644
--- a/flang/test/Semantics/OpenACC/acc-parallel.f90
+++ b/flang/test/Semantics/OpenACC/acc-parallel.f90
@@ -24,7 +24,7 @@ program openacc_parallel_validity
   !$acc parallel device_type(*) num_gangs(2)
   !$acc loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -149,7 +149,7 @@ program openacc_parallel_validity
   !$acc parallel device_type(*) if(.TRUE.)
   !$acc loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
diff --git a/flang/test/Semantics/OpenACC/acc-reduction-validity.f90 b/flang/test/Semantics/OpenACC/acc-reduction-validity.f90
index cecc7e0f507b2..0cdf33a2adb94 100644
--- a/flang/test/Semantics/OpenACC/acc-reduction-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-reduction-validity.f90
@@ -175,3 +175,15 @@ program openacc_reduction_validity
 
 
 end program
+
+subroutine sum()
+  ! ERROR: 'sum' is already declared in this scoping unit
+  integer :: i,sum 
+  sum = 0
+  !$acc parallel 
+  !$acc loop independent gang reduction(+:sum)
+  do i=1,10
+     sum = sum + i
+  enddo
+  !$acc end parallel
+end subroutine
diff --git a/flang/test/Semantics/OpenACC/acc-serial-loop.f90 b/flang/test/Semantics/OpenACC/acc-serial-loop.f90
index 5d2be7f7c6474..9f23a27a38b44 100644
--- a/flang/test/Semantics/OpenACC/acc-serial-loop.f90
+++ b/flang/test/Semantics/OpenACC/acc-serial-loop.f90
@@ -77,32 +77,32 @@ program openacc_serial_loop_validity
   !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the SERIAL LOOP directive
   !$acc serial loop device_type(*) if(.TRUE.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial loop
 
   !$acc serial loop if(ifCondition)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial loop
 
   !$acc serial loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !ERROR: Unmatched END PARALLEL LOOP directive
   !$acc end parallel loop
 
   !$acc serial loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial loop
 
   !$acc serial loop
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial
 
diff --git a/flang/test/Semantics/OpenACC/acc-serial.f90 b/flang/test/Semantics/OpenACC/acc-serial.f90
index f3b81c992e6ea..d50bdf90931aa 100644
--- a/flang/test/Semantics/OpenACC/acc-serial.f90
+++ b/flang/test/Semantics/OpenACC/acc-serial.f90
@@ -39,7 +39,7 @@ program openacc_serial_validity
   do i = 1, N
     !ERROR: Directive SET may not be called within a compute region
     !$acc set default_async(i)
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial
 
@@ -162,14 +162,14 @@ program openacc_serial_validity
 
   !$acc serial device_type(*) async
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial
 
   !ERROR: Clause IF is not allowed after clause DEVICE_TYPE on the SERIAL directive
   !$acc serial device_type(*) if(.TRUE.)
   do i = 1, N
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial
 
diff --git a/flang/test/Semantics/OpenACC/acc-set-validity.f90 b/flang/test/Semantics/OpenACC/acc-set-validity.f90
index 74522b30d11bc..3d514e1ec4ecc 100644
--- a/flang/test/Semantics/OpenACC/acc-set-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-set-validity.f90
@@ -31,7 +31,7 @@ program openacc_clause_validity
   do i = 1, N
     !ERROR: Directive SET may not be called within a compute region
     !$acc set default_async(i)
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -40,7 +40,7 @@ program openacc_clause_validity
   do i = 1, N
     !ERROR: Directive SET may not be called within a compute region
     !$acc set default_async(i)
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial
 
@@ -49,7 +49,7 @@ program openacc_clause_validity
   do i = 1, N
     !ERROR: Directive SET may not be called within a compute region
     !$acc set default_async(i)
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end kernels
 
@@ -57,21 +57,21 @@ program openacc_clause_validity
   do i = 1, N
     !ERROR: Directive SET may not be called within a compute region
     !$acc set default_async(i)
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc serial loop
   do i = 1, N
     !ERROR: Directive SET may not be called within a compute region
     !$acc set default_async(i)
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop
   do i = 1, N
     !ERROR: Directive SET may not be called within a compute region
     !$acc set default_async(i)
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !ERROR: At least one of DEFAULT_ASYNC, DEVICE_NUM, DEVICE_TYPE clause must appear on the SET directive
diff --git a/flang/test/Semantics/OpenACC/acc-shutdown-validity.f90 b/flang/test/Semantics/OpenACC/acc-shutdown-validity.f90
index 163130d41bdcc..fff630e8a9f83 100644
--- a/flang/test/Semantics/OpenACC/acc-shutdown-validity.f90
+++ b/flang/test/Semantics/OpenACC/acc-shutdown-validity.f90
@@ -32,7 +32,7 @@ program openacc_shutdown_validity
   do i = 1, N
     !ERROR: Directive SHUTDOWN may not be called within a compute region
     !$acc shutdown
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end parallel
 
@@ -41,7 +41,7 @@ program openacc_shutdown_validity
   do i = 1, N
     !ERROR: Directive SHUTDOWN may not be called within a compute region
     !$acc shutdown
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end serial
 
@@ -50,7 +50,7 @@ program openacc_shutdown_validity
   do i = 1, N
     !ERROR: Directive SHUTDOWN may not be called within a compute region
     !$acc shutdown
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
   !$acc end kernels
 
@@ -58,21 +58,21 @@ program openacc_shutdown_validity
   do i = 1, N
     !ERROR: Directive SHUTDOWN may not be called within a compute region
     !$acc shutdown
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc serial loop
   do i = 1, N
     !ERROR: Directive SHUTDOWN may not be called within a compute region
     !$acc shutdown
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc kernels loop
   do i = 1, N
     !ERROR: Directive SHUTDOWN may not be called within a compute region
     !$acc shutdown
-    a(i) = 3.14
+    a(i) = 3.14d0
   end do
 
   !$acc shutdown
diff --git a/flang/test/Semantics/OpenMP/atomic-update-only.f90 b/flang/test/Semantics/OpenMP/atomic-update-only.f90
index 3c027924a1423..8ae261c463b00 100644
--- a/flang/test/Semantics/OpenMP/atomic-update-only.f90
+++ b/flang/test/Semantics/OpenMP/atomic-update-only.f90
@@ -28,11 +28,18 @@ subroutine f02
 
 subroutine f03
   integer :: x, y
+  real :: xr, yr
 
+  !With integer type the reassociation should be able to bring the `x` to
+  !the top of the + operator. Expect no diagnostics.
   !$omp atomic update
-  !ERROR: The atomic variable x cannot be a proper subexpression of an argument (here: (x+y)) in the update operation
-  !ERROR: The atomic variable x should appear as an argument of the top-level + operator
   x = (x + y) + 1
+
+  !Real variables cannot be reassociated (unless fastmath options are present).
+  !$omp atomic update
+  !ERROR: The atomic variable xr cannot be a proper subexpression of an argument (here: (xr+yr)) in the update operation
+  !ERROR: The atomic variable xr should appear as an argument of the top-level + operator
+  xr = (xr + yr) + 1
 end
 
 subroutine f04
diff --git a/flang/test/Semantics/OpenMP/atomic04.f90 b/flang/test/Semantics/OpenMP/atomic04.f90
index 8f8af31245404..002e06be68656 100644
--- a/flang/test/Semantics/OpenMP/atomic04.f90
+++ b/flang/test/Semantics/OpenMP/atomic04.f90
@@ -205,9 +205,8 @@ subroutine more_invalid_atomic_update_stmts()
     !ERROR: The atomic variable a should appear as an argument of the top-level + operator
         a = a * b + c
 
+    !This is expected to work due to reassociation.
     !$omp atomic update
-    !ERROR: The atomic variable a cannot be a proper subexpression of an argument (here: a+b) in the update operation
-    !ERROR: The atomic variable a should appear as an argument of the top-level + operator
         a = a + b + c
 
     !$omp atomic
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index e725e2670e8dc..5f74978f38279 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -21,8 +21,8 @@
   integer(omp_allocator_handle_kind) :: xy_alloc
   xy_alloc = omp_init_allocator(xy_memspace, 1, xy_traits)
 
-  arrayA = 1.414
-  arrayB = 3.14
+  arrayA = 1.414d0
+  arrayB = 3.14d0
   N = 1024
 
 ! 2.5 parallel-clause -> if-clause |
diff --git a/flang/test/Semantics/OpenMP/combined-constructs.f90 b/flang/test/Semantics/OpenMP/combined-constructs.f90
index 2298d33ef33eb..49da56298d426 100644
--- a/flang/test/Semantics/OpenMP/combined-constructs.f90
+++ b/flang/test/Semantics/OpenMP/combined-constructs.f90
@@ -10,46 +10,46 @@ program main
   !ERROR: `DISTRIBUTE` region has to be strictly nested inside `TEAMS` region.
   !$omp distribute simd
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end distribute simd
 
   !$omp target parallel device(0)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel
 
   !ERROR: At most one DEVICE clause can appear on the TARGET PARALLEL directive
   !$omp target parallel device(0) device(1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel
 
   !$omp target parallel defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel
 
   !ERROR: 'variable-category' modifier is required
   !$omp target parallel defaultmap(tofrom)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel
 
   !ERROR: At most one DEFAULTMAP clause can appear on the TARGET PARALLEL directive
   !$omp target parallel defaultmap(tofrom:scalar) defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel
 
   !$omp target parallel map(tofrom:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel
 
@@ -57,46 +57,46 @@ program main
   !ERROR: Non-THREADPRIVATE object 'a' in COPYIN clause
   !$omp target parallel copyin(a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel
 
   !$omp target parallel do device(0)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel do
 
   !ERROR: At most one DEVICE clause can appear on the TARGET PARALLEL DO directive
   !$omp target parallel do device(0) device(1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel do
 
   !$omp target parallel do defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel do
 
   !ERROR: 'variable-category' modifier is required
   !$omp target parallel do defaultmap(tofrom)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel do
 
   !ERROR: At most one DEFAULTMAP clause can appear on the TARGET PARALLEL DO directive
   !$omp target parallel do defaultmap(tofrom:scalar) defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel do
 
   !$omp target parallel do map(tofrom:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel do
 
@@ -104,406 +104,406 @@ program main
   !ERROR: Non-THREADPRIVATE object 'a' in COPYIN clause
   !$omp target parallel do copyin(a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target parallel do
 
   !$omp target teams map(a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !$omp target teams device(0)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: At most one DEVICE clause can appear on the TARGET TEAMS directive
   !$omp target teams device(0) device(1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: SCHEDULE clause is not allowed on the TARGET TEAMS directive
   !$omp target teams schedule(static)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !$omp target teams defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: 'variable-category' modifier is required
   !$omp target teams defaultmap(tofrom)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: At most one DEFAULTMAP clause can appear on the TARGET TEAMS directive
   !$omp target teams defaultmap(tofrom:scalar) defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !$omp target teams num_teams(3) thread_limit(10) default(shared) private(i) shared(a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: At most one NUM_TEAMS clause can appear on the TARGET TEAMS directive
   !$omp target teams num_teams(2) num_teams(3)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: The parameter of the NUM_TEAMS clause must be a positive integer expression
   !$omp target teams num_teams(-1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: At most one THREAD_LIMIT clause can appear on the TARGET TEAMS directive
   !$omp target teams thread_limit(2) thread_limit(3)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: The parameter of the THREAD_LIMIT clause must be a positive integer expression
   !$omp target teams thread_limit(-1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: At most one DEFAULT clause can appear on the TARGET TEAMS directive
   !$omp target teams default(shared) default(private)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !$omp target teams num_teams(2) defaultmap(tofrom:scalar)
   do i = 1, N
-      a(i) = 3.14
+      a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !$omp target teams map(tofrom:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
   !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS directive
   !$omp target teams map(delete:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams
 
 
   !$omp target teams distribute map(a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !$omp target teams distribute device(0)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !ERROR: At most one DEVICE clause can appear on the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute device(0) device(1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !$omp target teams distribute defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !ERROR: 'variable-category' modifier is required
   !$omp target teams distribute defaultmap(tofrom)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !ERROR: At most one DEFAULTMAP clause can appear on the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute defaultmap(tofrom:scalar) defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !$omp target teams distribute num_teams(3) thread_limit(10) default(shared) private(i) shared(a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !ERROR: At most one NUM_TEAMS clause can appear on the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute num_teams(2) num_teams(3)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !ERROR: The parameter of the NUM_TEAMS clause must be a positive integer expression
   !$omp target teams distribute num_teams(-1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !ERROR: At most one THREAD_LIMIT clause can appear on the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute thread_limit(2) thread_limit(3)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !ERROR: The parameter of the THREAD_LIMIT clause must be a positive integer expression
   !$omp target teams distribute thread_limit(-1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !ERROR: At most one DEFAULT clause can appear on the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute default(shared) default(private)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !$omp target teams distribute num_teams(2) defaultmap(tofrom:scalar)
   do i = 1, N
-      a(i) = 3.14
+      a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !$omp target teams distribute map(tofrom:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE directive
   !$omp target teams distribute map(delete:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute
 
   !$omp target teams distribute parallel do device(0)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !ERROR: At most one DEVICE clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do device(0) device(1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !$omp target teams distribute parallel do defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !ERROR: 'variable-category' modifier is required
   !$omp target teams distribute parallel do defaultmap(tofrom)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !ERROR: At most one DEFAULTMAP clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do defaultmap(tofrom:scalar) defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !$omp target teams distribute parallel do num_teams(3) thread_limit(10) default(shared) private(i) shared(a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !ERROR: At most one NUM_TEAMS clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do num_teams(2) num_teams(3)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !ERROR: The parameter of the NUM_TEAMS clause must be a positive integer expression
   !$omp target teams distribute parallel do num_teams(-1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !ERROR: At most one THREAD_LIMIT clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do thread_limit(2) thread_limit(3)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !ERROR: The parameter of the THREAD_LIMIT clause must be a positive integer expression
   !$omp target teams distribute parallel do thread_limit(-1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !ERROR: At most one DEFAULT clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do default(shared) default(private)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !$omp target teams distribute parallel do num_teams(2) defaultmap(tofrom:scalar)
   do i = 1, N
-      a(i) = 3.14
+      a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !$omp target teams distribute parallel do map(tofrom:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
   !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO directive
   !$omp target teams distribute parallel do map(delete:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do
 
 
   !$omp target teams distribute parallel do simd map(a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !$omp target teams distribute parallel do simd device(0)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !ERROR: At most one DEVICE clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd device(0) device(1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !$omp target teams distribute parallel do simd defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !ERROR: 'variable-category' modifier is required
   !$omp target teams distribute parallel do simd defaultmap(tofrom)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !ERROR: At most one DEFAULTMAP clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd defaultmap(tofrom:scalar) defaultmap(tofrom:scalar)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !$omp target teams distribute parallel do simd num_teams(3) thread_limit(10) default(shared) private(i) shared(a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !ERROR: At most one NUM_TEAMS clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd num_teams(2) num_teams(3)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !ERROR: The parameter of the NUM_TEAMS clause must be a positive integer expression
   !$omp target teams distribute parallel do simd num_teams(-1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !ERROR: At most one THREAD_LIMIT clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd thread_limit(2) thread_limit(3)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !ERROR: The parameter of the THREAD_LIMIT clause must be a positive integer expression
   !$omp target teams distribute parallel do simd thread_limit(-1)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !ERROR: At most one DEFAULT clause can appear on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd default(shared) default(private)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !$omp target teams distribute parallel do simd num_teams(2) defaultmap(tofrom:scalar)
   do i = 1, N
-      a(i) = 3.14
+      a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !$omp target teams distribute parallel do simd map(tofrom:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
   !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET TEAMS DISTRIBUTE PARALLEL DO SIMD directive
   !$omp target teams distribute parallel do simd map(delete:a)
   do i = 1, N
-     a(i) = 3.14
+     a(i) = 3.14d0
   enddo
   !$omp end target teams distribute parallel do simd
 
diff --git a/flang/test/Semantics/OpenMP/critical-global-conflict.f90 b/flang/test/Semantics/OpenMP/critical-global-conflict.f90
new file mode 100644
index 0000000000000..2546b68748d93
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/critical-global-conflict.f90
@@ -0,0 +1,15 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -Werror
+
+subroutine g
+end
+
+subroutine f(x)
+  implicit none
+  integer :: x
+
+!ERROR: CRITICAL construct name 'g' conflicts with a previous declaration
+  !$omp critical(g)
+  x = 0
+!ERROR: CRITICAL construct name 'g' conflicts with a previous declaration
+  !$omp end critical(g)
+end
diff --git a/flang/test/Semantics/OpenMP/critical_within_default.f90 b/flang/test/Semantics/OpenMP/critical_within_default.f90
index a5fe30eeb7de0..70353e8e4b585 100644
--- a/flang/test/Semantics/OpenMP/critical_within_default.f90
+++ b/flang/test/Semantics/OpenMP/critical_within_default.f90
@@ -1,11 +1,16 @@
 ! RUN: %flang_fc1 -fopenmp -fdebug-dump-symbols %s | FileCheck %s
 ! Test that we do not make a private copy of the critical name
 
+!CHECK: Global scope:
+!CHECK-NEXT: MN: MainProgram
+!CHECK-NEXT: k2 (OmpCriticalLock): Unknown
+
 !CHECK:  MainProgram scope: MN
 !CHECK-NEXT:    j size=4 offset=0: ObjectEntity type: INTEGER(4)
 !CHECK-NEXT:    OtherConstruct scope:
 !CHECK-NEXT:      j (OmpPrivate): HostAssoc
-!CHECK-NEXT:      k2 (OmpCriticalLock): Unknown
+!CHECK-NOT:  k2
+
 program mn
   integer :: j
   j=2
diff --git a/flang/test/Semantics/OpenMP/device-constructs.f90 b/flang/test/Semantics/OpenMP/device-constructs.f90
index 431e0f88e3237..a41c461874b90 100644
--- a/flang/test/Semantics/OpenMP/device-constructs.f90
+++ b/flang/test/Semantics/OpenMP/device-constructs.f90
@@ -8,131 +8,131 @@ program main
   integer :: N
   type(c_ptr) :: cptr
 
-  arrayA = 1.414
-  arrayB = 3.14
+  arrayA = 1.414d0
+  arrayB = 3.14d0
   N = 256
 
   !$omp target map(arrayA)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !$omp target device(0)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !ERROR: At most one DEVICE clause can appear on the TARGET directive
   !$omp target device(0) device(1)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !ERROR: SCHEDULE clause is not allowed on the TARGET directive
   !$omp target schedule(static)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !$omp target defaultmap(tofrom:scalar)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !$omp target defaultmap(tofrom)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !ERROR: At most one DEFAULTMAP clause can appear on the TARGET directive
   !$omp target defaultmap(tofrom:scalar) defaultmap(tofrom:scalar)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !$omp target thread_limit(4)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !ERROR: At most one THREAD_LIMIT clause can appear on the TARGET directive
   !$omp target thread_limit(4) thread_limit(8)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !$omp teams num_teams(3) thread_limit(10) default(shared) private(i) shared(a)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end teams
 
   !ERROR: At most one NUM_TEAMS clause can appear on the TEAMS directive
   !$omp teams num_teams(2) num_teams(3)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end teams
 
   !ERROR: The parameter of the NUM_TEAMS clause must be a positive integer expression
   !$omp teams num_teams(-1)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end teams
 
   !ERROR: At most one THREAD_LIMIT clause can appear on the TEAMS directive
   !$omp teams thread_limit(2) thread_limit(3)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end teams
 
   !ERROR: The parameter of the THREAD_LIMIT clause must be a positive integer expression
   !$omp teams thread_limit(-1)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end teams
 
   !ERROR: At most one DEFAULT clause can appear on the TEAMS directive
   !$omp teams default(shared) default(private)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end teams
 
   !$omp target teams num_teams(2) defaultmap(tofrom:scalar)
   do i = 1, N
-      a = 3.14
+      a = 3.14d0
   enddo
   !$omp end target teams
 
   !$omp target map(tofrom:a)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !ERROR: Only the ALLOC, FROM, TO, TOFROM map types are permitted for MAP clauses on the TARGET directive
   !$omp target map(delete:a)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target
 
   !$omp target data device(0) map(to:a)
   do i = 1, N
-    a = 3.14
+    a = 3.14d0
   enddo
   !$omp end target data
 
@@ -147,7 +147,7 @@ program main
   !ERROR: At least one of MAP, USE_DEVICE_ADDR, USE_DEVICE_PTR clause must appear on the TARGET DATA directive
   !$omp target data device(0)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end target data
 
@@ -183,7 +183,7 @@ program main
   !ERROR: `DISTRIBUTE` region has to be strictly nested inside `TEAMS` region.
   !$omp distribute
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end distribute
   !$omp end target
@@ -192,7 +192,7 @@ program main
   !$omp teams
   !$omp distribute
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end distribute
   !$omp end teams
@@ -205,7 +205,7 @@ program main
   do i = 1, N
      do j = 1, N
         do k = 1, N
-           a = 3.14
+           a = 3.14d0
         enddo
      enddo
   enddo
@@ -219,7 +219,7 @@ program main
   do i = 1, N
      do j = 1, N
         do k = 1, N
-           a = 3.14
+           a = 3.14d0
         enddo
      enddo
   enddo
@@ -231,7 +231,7 @@ program main
   !ERROR: `DISTRIBUTE` region has to be strictly nested inside `TEAMS` region.
   !$omp distribute dist_schedule(static, 2)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end distribute
   !$omp end target
@@ -240,7 +240,7 @@ program main
   !$omp teams
   !$omp distribute dist_schedule(static, 2)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end distribute
   !$omp end teams
@@ -251,7 +251,7 @@ program main
   !ERROR: At most one DIST_SCHEDULE clause can appear on the DISTRIBUTE directive
   !$omp distribute dist_schedule(static, 2) dist_schedule(static, 3)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end distribute
   !$omp end target
@@ -261,7 +261,7 @@ program main
   !ERROR: At most one DIST_SCHEDULE clause can appear on the DISTRIBUTE directive
   !$omp distribute dist_schedule(static, 2) dist_schedule(static, 3)
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end distribute
   !$omp end teams
diff --git a/flang/test/Semantics/OpenMP/groupprivate.f90 b/flang/test/Semantics/OpenMP/groupprivate.f90
new file mode 100644
index 0000000000000..a875c466111a3
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/groupprivate.f90
@@ -0,0 +1,47 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=60
+
+module m00
+implicit none
+integer :: x = 1
+!ERROR: GROUPPRIVATE argument cannot be declared with an initializer
+!$omp groupprivate(x)
+!ERROR: GROUPPRIVATE argument should be a variable or a named common block
+!$omp groupprivate(f00)
+
+contains
+subroutine f00
+  implicit none
+  integer, save :: y
+  associate (z => y)
+  block
+    !ERROR: GROUPPRIVATE argument cannot be an ASSOCIATE name
+    !$omp groupprivate(z)
+  end block
+  end associate
+end
+end module
+
+module m01
+implicit none
+integer :: x, y
+common /some_block/ x
+!ERROR: GROUPPRIVATE argument cannot be a member of a common block
+!$omp groupprivate(x)
+
+contains
+subroutine f01
+  implicit none
+  integer :: z
+  !ERROR: GROUPPRIVATE argument variable must be declared in the same scope as the construct on which it appears
+  !$omp groupprivate(y)
+  !ERROR: GROUPPRIVATE argument variable must be declared in the module scope or have SAVE attribute
+  !$omp groupprivate(z)
+end
+end module
+
+module m02
+implicit none
+integer :: x(10)[*]
+!ERROR: GROUPPRIVATE argument cannot be a coarray
+!$omp groupprivate(x)
+end module
diff --git a/flang/test/Semantics/OpenMP/named-constants.f90 b/flang/test/Semantics/OpenMP/named-constants.f90
new file mode 100644
index 0000000000000..ac0850066cebe
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/named-constants.f90
@@ -0,0 +1,44 @@
+!RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+
+module named_constants
+    implicit none
+contains
+    subroutine shrd()
+        implicit none
+        integer, parameter :: n = 7
+        real, parameter :: m = 7.0
+        logical, parameter :: l = .false.
+        integer, dimension(3), parameter :: a = [1, 2, 3]
+        ! no error expected
+        !$omp parallel shared(n, m, l, a)
+            print *, n, m, l, a
+        !$omp end parallel
+    end subroutine shrd
+
+    subroutine frstprvt()
+        implicit none
+        integer, parameter :: n = 7
+        real, parameter :: m = 7.0
+        logical, parameter :: l = .false.
+        integer, dimension(3), parameter :: a = [1, 2, 3]
+        ! no error expected
+        !$omp parallel firstprivate(n, m, l, a)
+            print *, n, m, l, a
+        !$omp end parallel
+    end subroutine frstprvt
+
+    subroutine prvt()
+        implicit none
+        integer, parameter :: n = 7
+        real, parameter :: m = 7.0
+        logical, parameter :: l = .false.
+        integer, dimension(3), parameter :: a = [1, 2, 3]
+        !ERROR: 'n' must be a variable
+        !ERROR: 'm' must be a variable
+        !ERROR: 'l' must be a variable
+        !ERROR: 'a' must be a variable
+        !$omp parallel private(n, m, l, a)
+            print *, n, m, l, a
+        !$omp end parallel
+    end subroutine prvt
+end module named_constants
diff --git a/flang/test/Semantics/OpenMP/nested-distribute.f90 b/flang/test/Semantics/OpenMP/nested-distribute.f90
index c212763cba1df..cb4aea3bcdffa 100644
--- a/flang/test/Semantics/OpenMP/nested-distribute.f90
+++ b/flang/test/Semantics/OpenMP/nested-distribute.f90
@@ -6,15 +6,15 @@ program main
   real(8) :: arrayA(256), arrayB(256)
   integer :: N
 
-  arrayA = 1.414
-  arrayB = 3.14
+  arrayA = 1.414d0
+  arrayB = 3.14d0
   N = 256
 
   !$omp task
   !ERROR: `DISTRIBUTE` region has to be strictly nested inside `TEAMS` region.
   !$omp distribute 
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end distribute
   !$omp end task
@@ -24,7 +24,7 @@ program main
       !ERROR: Only `DISTRIBUTE`, `PARALLEL`, or `LOOP` regions are allowed to be strictly nested inside `TEAMS` region.
       !$omp task
       do k = 1, N
-         a = 3.14
+         a = 3.14d0
       enddo
       !$omp end task
    enddo
@@ -34,7 +34,7 @@ program main
    do i = 1, N
       !$omp parallel
       do k = 1, N
-         a = 3.14
+         a = 3.14d0
       enddo
       !$omp end parallel
    enddo
@@ -44,7 +44,7 @@ program main
   !ERROR: `DISTRIBUTE` region has to be strictly nested inside `TEAMS` region.
   !$omp distribute 
   do i = 1, N
-     a = 3.14
+     a = 3.14d0
   enddo
   !$omp end distribute
   !$omp end parallel
diff --git a/flang/test/Semantics/OpenMP/reduction-assumed.f90 b/flang/test/Semantics/OpenMP/reduction-assumed.f90
new file mode 100644
index 0000000000000..0bc8cd318a74d
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/reduction-assumed.f90
@@ -0,0 +1,53 @@
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp
+
+! Types for built in reductions must have types which are valid for the
+! initialization and combiner expressions in the spec. This implies assumed
+! rank and assumed size cannot be used.
+
+subroutine assumedRank1(a)
+  integer :: a(..)
+
+  ! ERROR: The type of 'a' is incompatible with the reduction operator.
+  !$omp parallel reduction(+:a)
+  !$omp end parallel
+end
+
+subroutine assumedRank2(a)
+  integer :: a(..)
+
+  ! ERROR: The type of 'a' is incompatible with the reduction operator.
+  !$omp parallel reduction(min:a)
+  !$omp end parallel
+end
+
+subroutine assumedRank3(a)
+  integer :: a(..)
+
+  ! ERROR: The type of 'a' is incompatible with the reduction operator.
+  !$omp parallel reduction(iand:a)
+  !$omp end parallel
+end
+
+subroutine assumedSize1(a)
+  integer :: a(*)
+
+  ! ERROR: Whole assumed-size array 'a' may not appear here without subscripts
+  !$omp parallel reduction(+:a)
+  !$omp end parallel
+end
+
+subroutine assumedSize2(a)
+  integer :: a(*)
+
+  ! ERROR: Whole assumed-size array 'a' may not appear here without subscripts
+  !$omp parallel reduction(max:a)
+  !$omp end parallel
+end
+
+subroutine assumedSize3(a)
+  integer :: a(*)
+
+  ! ERROR: Whole assumed-size array 'a' may not appear here without subscripts
+  !$omp parallel reduction(ior:a)
+  !$omp end parallel
+end
diff --git a/flang/test/Semantics/OpenMP/simd-only.f90 b/flang/test/Semantics/OpenMP/simd-only.f90
new file mode 100644
index 0000000000000..da42b10d73bed
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/simd-only.f90
@@ -0,0 +1,416 @@
+! RUN: %flang_fc1 -fopenmp-simd -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s
+
+! Test that non-SIMD OpenMPConstructs are removed on the parse tree level
+! when -fopenmp-simd is specified.
+! Tests the logic in lib/Semantics/rewrite-parse-tree.cpp
+
+! CHECK-LABEL: Name = 'test_simd'
+subroutine test_simd()
+  integer :: i
+
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK: OmpLoopDirective -> llvm::omp::Directive = simd
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp simd
+  do i = 1, 100
+  end do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_do_simd'
+subroutine test_do_simd()
+  integer :: i
+
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK: OmpLoopDirective -> llvm::omp::Directive = do simd
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp do simd
+  do i = 1, 100
+  end do
+end subroutine
+
+
+! CHECK-LABEL: Name = 'test_parallel_do_simd'
+subroutine test_parallel_do_simd()
+  integer :: i
+
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK: OmpLoopDirective -> llvm::omp::Directive = parallel do simd
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp parallel do simd
+  do i = 1, 100
+  end do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_simd_scan'
+subroutine test_simd_scan()
+  integer :: i
+  real :: sum
+
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK: OmpLoopDirective -> llvm::omp::Directive = simd
+  !$omp simd reduction(inscan,+:sum)
+  do i = 1, N
+    sum = sum + a(i)
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+    ! CHECK: OmpDirectiveName -> llvm::omp::Directive = scan
+    !$omp scan inclusive(sum)
+    sum       = sum + a(i)
+  end do
+
+end subroutine
+
+! CHECK-LABEL: Name = 'test_simd_atomic'
+subroutine test_simd_atomic()
+  integer :: i, x
+
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK: OmpLoopDirective -> llvm::omp::Directive = simd
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp simd
+  do i = 1, 100
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=i'
+  !$omp atomic write
+  x = i
+  end do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_do'
+subroutine test_do()
+  integer :: i
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = do
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp parallel do
+  do i = 1, 100
+  end do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_do_nested'
+subroutine test_do_nested()
+  integer :: i
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = parallel do
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp parallel do
+  do i = 1, 100
+    do j = 1, 100
+    end do
+  end do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_target'
+subroutine test_target()
+  integer :: i
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = target
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp target
+  do i = 1, 100
+  end do
+  !$omp end target
+end subroutine
+
+! CHECK-LABEL: Name = 'test_target_teams_distribute'
+subroutine test_target_teams_distribute()
+  integer :: i
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = target teams distribute
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp target teams distribute
+  do i = 1, 100
+  end do
+  !$omp end target teams distribute
+end subroutine
+
+
+! CHECK-LABEL: Name = 'test_target_data'
+subroutine test_target_data()
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = target data
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp target data map(to: A) map(tofrom: B)
+  do i = 1, 100
+  end do
+  !$omp end target data
+end subroutine
+
+! CHECK-LABEL: Name = 'test_loop'
+subroutine test_loop()
+  integer :: i
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = loop
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp loop bind(thread)
+  do i = 1, 100
+  end do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_unroll'
+subroutine test_unroll()
+  integer :: i
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = unroll
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp unroll
+  do i = 1, 100
+  end do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_do_ordered'
+subroutine test_do_ordered()
+  integer :: i, x
+  x = 0
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = do
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp do ordered
+  do i = 1, 100
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = ordered
+  !$omp ordered
+  x = x + 1
+  !$omp end ordered
+  end do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_cancel'
+subroutine test_cancel()
+  integer :: i, x
+  x = 0
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = parallel do
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp parallel do
+  do i = 1, 100
+  if (i == 10) then
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPCancelConstruct -> OmpDirectiveSpecification
+    ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = cancel
+    !$omp cancel do
+  end if
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPCancellationPointConstruct -> OmpDirectiveSpecification
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = cancellation point
+  !$omp cancellation point do
+  end do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_scan'
+subroutine test_scan()
+  integer :: i, sum
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = parallel do
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  !$omp parallel do reduction(inscan, +: sum)
+  do i = 1, n
+    sum = sum + i
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+    ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = scan
+    !$omp scan inclusive(sum)
+  end do
+  !$omp end parallel do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_target_map'
+subroutine test_target_map()
+  integer :: array(10)
+
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+  ! CHECK-NOT: OmpLoopDirective -> llvm::omp::Directive = target
+  !$omp target map(tofrom: array(2:10))
+    array(2) = array(2) * 2
+  !$omp end target
+end subroutine
+
+! CHECK-LABEL: Name = 'test_sections'
+subroutine test_sections()
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPSectionsConstruct
+  !$omp sections
+  ! CHECK-NOT: OpenMPConstruct -> OpenMPSectionConstruct
+  !$omp section
+  ! CHECK-NOT: OpenMPConstruct -> OpenMPSectionConstruct
+  !$omp section
+  !$omp end sections
+end subroutine
+
+! CHECK-LABEL: Name = 'test_threadprivate_mod'
+module test_threadprivate_mod
+  implicit none
+  ! CHECK: DeclarationConstruct -> SpecificationConstruct -> TypeDeclarationStmt
+  ! CHECK: Name = 'x'
+  ! CHECK: Name = 'y'
+  integer :: x, y
+  ! CHECK: DeclarationConstruct -> SpecificationConstruct -> OtherSpecificationStmt -> CommonStmt
+  ! CHECK: Name = 'x'
+  ! CHECK: Name = 'y'
+  common /vars/ x, y
+  ! CHECK-NOT: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPThreadprivate
+  !$omp threadprivate(/vars/)
+end module
+
+! CHECK-LABEL: Name = 'test_atomic'
+subroutine test_atomic()
+  real :: z, x, y
+  !$omp parallel private(tid, z)
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=y'
+    !$omp atomic write
+      x = y
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'z=x'
+    !$omp atomic read
+      z = x
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=x+1._4'
+    !$omp atomic update
+      x = x + 1
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'z=x'
+    !$omp atomic read
+      z = x
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPAtomicConstruct
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=x+y'
+    !$omp atomic capture
+      x   = x + y
+    !$omp end atomic
+  !$omp end parallel
+end subroutine
+
+! CHECK-LABEL: Name = 'test_task_single_taskwait'
+subroutine test_task_single_taskwait()
+  integer :: x
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+  ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = parallel
+  !$omp parallel
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+  ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = single
+  !$omp single
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+    do i = 1, 5
+      ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+      ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = task
+      ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=i'
+      !$omp task
+      x = i
+      !$omp end task
+    end do
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+    ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = taskwait
+    !$omp taskwait
+  !$omp end single
+  !$omp end parallel
+end subroutine
+
+! CHECK-LABEL: Name = 'test_task_taskyield_flush_barrier'
+subroutine test_task_taskyield_flush_barrier()
+  integer :: x, i
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+  ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = parallel
+  !$omp parallel
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+    ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = barrier
+    !$omp barrier
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+    ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = single
+    !$omp single
+      ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+      ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = task
+      !$omp task
+        ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+        ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = taskyield
+        !$omp taskyield
+        ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=i'
+        x = i
+        ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPFlushConstruct -> OmpDirectiveSpecification
+        !$omp flush
+      !$omp end task
+      ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+      ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = task
+      !$omp task
+        ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPFlushConstruct -> OmpDirectiveSpecification
+        !$omp flush
+      !$omp end task
+      ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+      ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = taskwait
+      !$omp taskwait
+    !$omp end single
+  !$omp end parallel
+end subroutine
+
+! CHECK-LABEL: Name = 'test_master_masked'
+subroutine test_master_masked()
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+  ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = parallel
+  !$omp parallel private(tid)
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+    ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = masked
+    !$omp masked
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=y'
+    x = y
+    !$omp end masked
+    ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+    ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = master
+    !$omp master
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'y=x'
+    y = x
+    !$omp end master
+  !$omp end parallel
+end subroutine
+
+! CHECK-LABEL: Name = 'test_critical'
+subroutine test_critical()
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+  ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = parallel
+  !$omp parallel do private(i)
+  do i = 1, 4
+    !$omp critical(mylock)
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=y'
+    x = y
+    !$omp end critical(mylock)
+  end do
+  !$omp end parallel do
+end subroutine
+
+! CHECK-LABEL: Name = 'test_target_enter_exit_update_data'
+subroutine test_target_enter_exit_update_data()
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+  ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = target enter data
+  !$omp target enter data map(to: A)
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPBlockConstruct
+  ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = target teams distribute parallel do
+  !$omp target teams distribute parallel do
+  ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> DoConstruct
+  do i = 1, n
+    ! CHECK: ExecutionPartConstruct -> ExecutableConstruct -> ActionStmt -> AssignmentStmt = 'x=y'
+    x = y
+  end do
+  !$omp end target teams distribute parallel do
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+  ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = target update
+  !$omp target update from(A)
+  ! CHECK-NOT: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPStandaloneConstruct -> OpenMPSimpleStandaloneConstruct -> OmpDirectiveSpecification
+  ! CHECK-NOT: OmpDirectiveName -> llvm::omp::Directive = target exit data
+  !$omp target exit data map(from: A)
+end subroutine
+
+! CHECK-LABEL: Name = 'test_declare_mapper'
+module test_declare_mapper
+  implicit none
+
+  type :: myvec_t
+    integer               :: len
+    real, allocatable     :: data(:)
+  end type myvec_t
+
+  ! CHECK-NOT: DeclarationConstruct -> SpecificationConstruct -> OpenMPDeclarativeConstruct -> OpenMPDeclareMapperConstruct
+  !$omp declare mapper(myvec_t :: v) map(v, v%data(1:v%len))
+end module
diff --git a/flang/test/Semantics/OpenMP/sync-critical01.f90 b/flang/test/Semantics/OpenMP/sync-critical01.f90
index b597eb17ea226..01cc0acf65936 100644
--- a/flang/test/Semantics/OpenMP/sync-critical01.f90
+++ b/flang/test/Semantics/OpenMP/sync-critical01.f90
@@ -17,22 +17,22 @@ integer function timer_tick_sec()
 
    !$OMP CRITICAL (foo)
    t = t + 1
-   !ERROR: CRITICAL directive names do not match
+   !ERROR: The names on CRITICAL and END CRITICAL must match
    !$OMP END CRITICAL (bar)
 
    !$OMP CRITICAL (bar)
    t = t + 1
-   !ERROR: CRITICAL directive names do not match
+   !ERROR: The names on CRITICAL and END CRITICAL must match
    !$OMP END CRITICAL (foo)
 
-   !ERROR: CRITICAL directive names do not match
+   !ERROR: Either both CRITICAL and END CRITICAL should have an argument, or none of them should
    !$OMP CRITICAL (bar)
    t = t + 1
    !$OMP END CRITICAL
 
    !$OMP CRITICAL
    t = t + 1
-   !ERROR: CRITICAL directive names do not match
+   !ERROR: Either both CRITICAL and END CRITICAL should have an argument, or none of them should
    !$OMP END CRITICAL (foo)
 
    timer_tick_sec = t
diff --git a/flang/test/Semantics/OpenMP/sync-critical02.f90 b/flang/test/Semantics/OpenMP/sync-critical02.f90
index 1fa9d6ad84f28..b77bd66aac5f8 100644
--- a/flang/test/Semantics/OpenMP/sync-critical02.f90
+++ b/flang/test/Semantics/OpenMP/sync-critical02.f90
@@ -8,7 +8,7 @@
 program sample
    use omp_lib
    integer i, j
-   !ERROR: Hint clause other than omp_sync_hint_none cannot be specified for an unnamed CRITICAL directive
+   !ERROR: When HINT other than 'omp_sync_hint_none' is present, CRITICAL directive should have a name
    !$omp critical hint(omp_lock_hint_speculative)
    j = j + 1
    !$omp end critical
@@ -17,7 +17,7 @@ program sample
    i = i - 1
    !$omp end critical (foo)
 
-   !ERROR: Hint clause other than omp_sync_hint_none cannot be specified for an unnamed CRITICAL directive
+   !ERROR: When HINT other than 'omp_sync_hint_none' is present, CRITICAL directive should have a name
    !$omp critical hint(omp_lock_hint_nonspeculative)
    j = j + 1
    !$omp end critical
@@ -26,7 +26,7 @@ program sample
    i = i - 1
    !$omp end critical (foo)
 
-   !ERROR: Hint clause other than omp_sync_hint_none cannot be specified for an unnamed CRITICAL directive
+   !ERROR: When HINT other than 'omp_sync_hint_none' is present, CRITICAL directive should have a name
    !$omp critical hint(omp_lock_hint_contended)
    j = j + 1
    !$omp end critical
@@ -35,7 +35,7 @@ program sample
    i = i - 1
    !$omp end critical (foo)
 
-   !ERROR: Hint clause other than omp_sync_hint_none cannot be specified for an unnamed CRITICAL directive
+   !ERROR: When HINT other than 'omp_sync_hint_none' is present, CRITICAL directive should have a name
    !$omp critical hint(omp_lock_hint_uncontended)
    j = j + 1
    !$omp end critical
diff --git a/flang/test/Semantics/OpenMP/workdistribute01.f90 b/flang/test/Semantics/OpenMP/workdistribute01.f90
new file mode 100644
index 0000000000000..f7e36976dfb65
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/workdistribute01.f90
@@ -0,0 +1,16 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=60
+! OpenMP Version 6.0
+! workdistribute Construct
+! Invalid do construct inside !$omp workdistribute
+
+subroutine workdistribute()
+  integer n, i
+  !ERROR: A WORKDISTRIBUTE region must be nested inside TEAMS region only.
+  !ERROR: The structured block in a WORKDISTRIBUTE construct may consist of only SCALAR or ARRAY assignments
+  !$omp workdistribute
+  do i = 1, n
+    print *, "omp workdistribute"
+  end do
+  !$omp end workdistribute
+
+end subroutine workdistribute
diff --git a/flang/test/Semantics/OpenMP/workdistribute02.f90 b/flang/test/Semantics/OpenMP/workdistribute02.f90
new file mode 100644
index 0000000000000..6de3a55f545b5
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/workdistribute02.f90
@@ -0,0 +1,34 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=60
+! OpenMP Version 6.0
+! workdistribute Construct
+! The !omp workdistribute construct must not contain any user defined
+! function calls unless the function is ELEMENTAL.
+
+module my_mod
+  contains
+  integer function my_func()
+    my_func = 10
+  end function my_func
+
+  impure integer function impure_my_func()
+    impure_my_func = 20
+  end function impure_my_func
+
+  impure elemental integer function impure_ele_my_func()
+    impure_ele_my_func = 20
+  end function impure_ele_my_func
+end module my_mod
+
+subroutine workdistribute(aa, bb, cc, n)
+  use my_mod
+  integer n
+  real aa(n), bb(n), cc(n)
+  !$omp teams
+  !$omp workdistribute
+  !ERROR: User defined non-ELEMENTAL function 'my_func' is not allowed in a WORKDISTRIBUTE construct
+  aa = my_func()
+  aa = bb * cc
+  !$omp end workdistribute
+  !$omp end teams
+
+end subroutine workdistribute
diff --git a/flang/test/Semantics/OpenMP/workdistribute03.f90 b/flang/test/Semantics/OpenMP/workdistribute03.f90
new file mode 100644
index 0000000000000..828170a016ed2
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/workdistribute03.f90
@@ -0,0 +1,34 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=60
+! OpenMP Version 6.0
+! workdistribute Construct
+! All array assignments, scalar assignments, and masked array assignments
+! must be intrinsic assignments.
+
+module defined_assign
+  interface assignment(=)
+    module procedure work_assign
+  end interface
+
+  contains
+    subroutine work_assign(a,b)
+      integer, intent(out) :: a
+      logical, intent(in) :: b(:)
+    end subroutine work_assign
+end module defined_assign
+
+program omp_workdistribute
+  use defined_assign
+
+  integer :: a, aa(10), bb(10)
+  logical :: l(10)
+  l = .TRUE.
+
+  !$omp teams
+  !$omp workdistribute
+  !ERROR: Defined assignment statement is not allowed in a WORKDISTRIBUTE construct
+  a = l
+  aa = bb
+  !$omp end workdistribute
+  !$omp end teams
+
+end program omp_workdistribute
diff --git a/flang/test/Semantics/OpenMP/workdistribute04.f90 b/flang/test/Semantics/OpenMP/workdistribute04.f90
new file mode 100644
index 0000000000000..d407e8a073ae4
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/workdistribute04.f90
@@ -0,0 +1,15 @@
+! RUN: %python %S/../test_errors.py %s %flang -fopenmp -fopenmp-version=50
+! OpenMP Version 6.0
+! workdistribute Construct
+! Unsuported OpenMP version
+
+subroutine teams_workdistribute()
+  use iso_fortran_env
+  real(kind=real32) :: a
+  real(kind=real32), dimension(10) :: x
+  real(kind=real32), dimension(10) :: y
+  !ERROR: WORKDISTRIBUTE construct is not allowed in OpenMP v5.0, try -fopenmp-version=60
+  !$omp teams workdistribute
+  y = a * x + y
+  !$omp end teams workdistribute
+end subroutine teams_workdistribute
diff --git a/flang/test/Semantics/cuf17.cuf b/flang/test/Semantics/cuf17.cuf
deleted file mode 100644
index daeb59033561c..0000000000000
--- a/flang/test/Semantics/cuf17.cuf
+++ /dev/null
@@ -1,18 +0,0 @@
-! RUN: bbc -emit-hlfir -fcuda %s 2>&1 | FileCheck %s
-
-module mod1
-contains
-
-attributes(device) subroutine sub1(adev)
-  real, device :: adev(10)
-end
-
-attributes(global) subroutine sub2()
-  real, shared :: adev(10)
-  !WARNING: SHARED attribute ignored
-  call sub1(adev)
-end subroutine
-
-end module
-
-! CHECK: warning: SHARED attribute ignored
diff --git a/flang/test/Semantics/widening.f90 b/flang/test/Semantics/widening.f90
new file mode 100644
index 0000000000000..52090c171de98
--- /dev/null
+++ b/flang/test/Semantics/widening.f90
@@ -0,0 +1,48 @@
+! RUN: %python %S/test_errors.py %s %flang_fc1 -Werror
+
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 1.00000001490116119384765625e-1_4 is inexact [-Wreal-constant-widening]
+real(8), parameter :: warning1 = 0.1
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 1.10000002384185791015625_4 is inexact [-Wreal-constant-widening]
+real(8) :: warning2 = 1.1
+real, parameter :: noWarning1 = 2.1
+real(8) :: noWarning2 = warning1
+real(8) :: noWarning3 = noWarning1
+real(8) :: noWarning4 = 3.125 ! exact
+real(8) :: noWarning5 = 4.1d0 ! explicit 'd'
+real(8) :: noWarning6 = 5.1_4 ! explicit suffix
+real(8) :: noWarning7 = real(6.1, 8) ! explicit conversion
+real(8) :: noWarning8 = real(7.1d0) ! explicit narrowing conversion
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 8.1000003814697265625_4 is inexact [-Wreal-constant-widening]
+real(8) :: warning3 = real(8.1) ! no-op conversion
+! WARNING: Default real literal in COMPLEX(8) context might need a kind suffix, as its rounded value (9.1000003814697265625_4,1.01000003814697265625e1_4) is inexact [-Wreal-constant-widening]
+complex(8), parameter :: warning4 = (9.1, 10.1)
+! WARNING: Default real literal in COMPLEX(8) context might need a kind suffix, as its rounded value (1.11000003814697265625e1_4,1.21000003814697265625e1_4) is inexact [-Wreal-constant-widening]
+complex(8) :: warning5 = (11.1, 12.1)
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value [REAL(4)::1.31000003814697265625e1_4] is inexact [-Wreal-constant-widening]
+real(8) :: warning6(1) = [ 13.1 ]
+real(8) warning7
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 1.41000003814697265625e1_4 is inexact [-Wreal-constant-widening]
+data warning7/14.1/
+type derived
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 1.51000003814697265625e1_4 is inexact [-Wreal-constant-widening]
+  real(8) :: warning8 = 15.1
+  real(8) :: noWarning9 = real(16.1, 8)
+  real :: noWarning10 = 17.1
+end type
+type(derived) dx
+real noWarning11
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 1.81000003814697265625e1_4 is inexact [-Wreal-constant-widening]
+warning7 = 18.1
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 1.91000003814697265625e1_4 is inexact [-Wreal-constant-widening]
+dx%warning8 = 19.1
+dx%noWarning10 = 20.1
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 2.11000003814697265625e1_4 is inexact [-Wreal-constant-widening]
+dx = derived(21.1)
+dx = derived(22.125)
+noWarning11 = 23.1
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 2.41000003814697265625e1_4 is inexact [-Wreal-constant-widening]
+print *, [real(8) :: 24.1]
+! WARNING: Default real literal in REAL(8) context might need a kind suffix, as its rounded value 2.51000003814697265625e1_4 is inexact [-Wreal-constant-widening]
+print *, [real(8) :: noWarning11, 25.1]
+print *, [real(8) :: noWarning1] ! ok
+end
diff --git a/flang/test/Transforms/OpenMP/simd-only.mlir b/flang/test/Transforms/OpenMP/simd-only.mlir
new file mode 100644
index 0000000000000..0025d10fbd21a
--- /dev/null
+++ b/flang/test/Transforms/OpenMP/simd-only.mlir
@@ -0,0 +1,196 @@
+// RUN: fir-opt --split-input-file --verify-diagnostics --omp-simd-only %s | FileCheck %s
+
+// Check that simd operations are not removed and rewritten, but all the other OpenMP ops are.
+// Tests the logic in flang/lib/Optimizer/OpenMP/SimdOnly.cpp
+
+// CHECK: omp.private
+// CHECK-LABEL: func.func @simd
+omp.private {type = private} @_QFEi_private_i32 : i32
+func.func @simd(%arg0: i32, %arg1: !fir.ref<i32>, %arg2: !fir.ref<i32>) {
+  %c1_i32 = arith.constant 1 : i32
+  %c100000_i32 = arith.constant 100000 : i32
+  // CHECK: omp.simd private
+  omp.simd private(@_QFEi_private_i32 %arg2 -> %arg3 : !fir.ref<i32>) {
+    // CHECK: omp.loop_nest
+    omp.loop_nest (%arg4) : i32 = (%c1_i32) to (%c100000_i32) inclusive step (%c1_i32) {
+      // CHECK: fir.store
+      fir.store %arg0 to %arg1 : !fir.ref<i32>
+      // CHECK: omp.yield
+      omp.yield
+    }
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @simd_composite
+func.func @simd_composite(%arg0: i32, %arg1: !fir.ref<i32>) {
+  %c1_i32 = arith.constant 1 : i32
+  %c100000_i32 = arith.constant 100000 : i32
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK-NOT: omp.wsloop
+    omp.wsloop {
+      // CHECK: omp.simd
+      omp.simd {
+        // CHECK: omp.loop_nest
+        omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100000_i32) inclusive step (%c1_i32) {
+          // CHECK: fir.store
+          fir.store %arg0 to %arg1 : !fir.ref<i32>
+          // CHECK: omp.yield
+          omp.yield
+        }
+      // CHECK-NOT: {omp.composite}
+      } {omp.composite}
+    } {omp.composite}
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-NOT: omp.private
+// CHECK-LABEL: func.func @parallel
+omp.private {type = private} @_QFEi_private_i32 : i32
+func.func @parallel(%arg0: i32, %arg1: !fir.ref<i32>) {
+  %c1 = arith.constant 1 : index
+  %c1_i32 = arith.constant 1 : i32
+  %c100000_i32 = arith.constant 100000 : i32
+  // CHECK-NOT: omp.parallel
+  omp.parallel private(@_QFEi_private_i32 %arg1 -> %arg3 : !fir.ref<i32>) {
+    // CHECK: fir.convert
+    %15 = fir.convert %c1_i32 : (i32) -> index
+    // CHECK: fir.convert
+    %16 = fir.convert %c100000_i32 : (i32) -> index
+    // CHECK: fir.do_loop
+    %18:2 = fir.do_loop %arg4 = %15 to %16 step %c1 iter_args(%arg2 = %arg0) -> (index, i32) {
+      // CHECK: fir.store
+      fir.store %arg0 to %arg1 : !fir.ref<i32>
+      fir.result %arg4, %arg2 : index, i32
+    }
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+    }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @target_map(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @target_map(%arg5: i32, %arg6: !fir.ref<i32>) {
+  // CHECK-NOT: omp.map.info
+  %3 = omp.map.info var_ptr(%arg6 : !fir.ref<i32>, i32) map_clauses(implicit) capture(ByCopy) -> !fir.ref<i32>
+  // CHECK-NOT: omp.target
+  omp.target map_entries(%3 -> %arg0 : !fir.ref<i32>) {
+    // CHECK: arith.constant
+    %c1_i32 = arith.constant 1 : i32
+    // CHECK: fir.store %c1_i32 to %[[ARG_1]]
+    fir.store %c1_i32 to %arg0 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @teams
+func.func @teams(%arg0: i32, %arg1: !fir.ref<i32>) {
+  // CHECK-NOT: omp.teams
+  omp.teams {
+    // CHECK: fir.store
+    fir.store %arg0 to %arg1 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @distribute_simd
+func.func @distribute_simd(%arg0: i32, %arg1: !fir.ref<i32>) {
+  %c1_i32 = arith.constant 1 : i32
+  %c100000_i32 = arith.constant 100000 : i32
+  // CHECK-NOT: omp.distribute
+  omp.distribute {
+    // CHECK: omp.simd
+    omp.simd {
+      // CHECK: omp.loop_nest
+      omp.loop_nest (%arg3) : i32 = (%c1_i32) to (%c100000_i32) inclusive step (%c1_i32) {
+        // CHECK: fir.store
+        fir.store %arg0 to %arg1 : !fir.ref<i32>
+        // CHECK: omp.yield
+        omp.yield
+      }
+    // CHECK-NOT: {omp.composite}
+    } {omp.composite}
+  // CHECK-NOT: {omp.composite}
+  } {omp.composite}
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @threadprivate(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @threadprivate(%arg0: i32, %arg1: !fir.ref<i32>) {
+  // CHECK-NOT: omp.threadprivate
+  %1 = omp.threadprivate %arg1 : !fir.ref<i32> -> !fir.ref<i32>
+  // CHECK: fir.store %[[ARG_0]] to %[[ARG_1]]
+  fir.store %arg0 to %1 : !fir.ref<i32>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @multi_block(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>, %[[ARG_3:.*]]: i1
+func.func @multi_block(%funcArg0: i32, %funcArg1: !fir.ref<i32>, %6: i1) {
+  %false = arith.constant false
+  %c0_i32 = arith.constant 0 : i32
+  // CHECK-NOT: omp.parallel
+  omp.parallel {
+    // CHECK: cf.cond_br %[[ARG_3]], ^[[BB1:.*]], ^[[BB2:.*]]
+    cf.cond_br %6, ^bb1, ^bb2
+  // CHECK: ^[[BB1]]
+  ^bb1:  // pred: ^bb0
+    // CHECK: fir.call
+    fir.call @_FortranAStopStatement(%c0_i32, %false, %false) fastmath<contract> : (i32, i1, i1) -> ()
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  // CHECK: ^[[BB2]]
+  ^bb2:  // pred: ^bb0
+    // CHECK: fir.store
+    fir.store %funcArg0 to %funcArg1 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  return
+}
+
+// -----
+
+// CHECK-LABEL: func.func @map_info(
+// CHECK-SAME: %[[ARG_0:.*]]: i32, %[[ARG_1:.*]]: !fir.ref<i32>
+func.func @map_info(%funcArg0: i32, %funcArg1: !fir.ref<i32>) {
+  %c1 = arith.constant 1 : index
+  // CHECK-NOT: omp.map.bounds
+  %1 = omp.map.bounds lower_bound(%c1 : index) upper_bound(%c1 : index) extent(%c1 : index) stride(%c1 : index) start_idx(%c1 : index)
+  // CHECK-NOT: omp.map.info
+  %13 = omp.map.info var_ptr(%funcArg1 : !fir.ref<i32>, i32) map_clauses(to) capture(ByRef) bounds(%1) -> !fir.ref<i32>
+  // CHECK-NOT: omp.target
+  omp.target map_entries(%13 -> %arg3 : !fir.ref<i32>) {
+    %c1_i32 = arith.constant 1 : i32
+    // CHECK: fir.store %c1_i32 to %[[ARG_1]]
+    fir.store %c1_i32 to %arg3 : !fir.ref<i32>
+    // CHECK-NOT: omp.terminator
+    omp.terminator
+  }
+  // CHECK-NOT: omp.map.info
+  %18 = omp.map.info var_ptr(%funcArg1 : !fir.ref<i32>, i32) map_clauses(from) capture(ByRef) bounds(%1) -> !fir.ref<i32>
+  return
+}
diff --git a/flang/test/Transforms/omp-automap-to-target-data.fir b/flang/test/Transforms/omp-automap-to-target-data.fir
new file mode 100644
index 0000000000000..7a19705a248b4
--- /dev/null
+++ b/flang/test/Transforms/omp-automap-to-target-data.fir
@@ -0,0 +1,58 @@
+// RUN: fir-opt --omp-automap-to-target-data %s | FileCheck %s
+// Test OMP AutomapToTargetData pass.
+
+module {
+  fir.global
+      @_QMtestEarr{omp.declare_target = #omp.declaretarget<device_type = (any),
+                       capture_clause = (enter), automap = true>} target
+                       : !fir.box<!fir.heap<!fir.array<?xi32>>>
+
+  func.func @automap() {
+    %c0 = arith.constant 0 : index
+    %c10 = arith.constant 10 : i32
+    %addr = fir.address_of(@_QMtestEarr) : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+    %decl:2 = hlfir.declare %addr {fortran_attrs = #fir.var_attrs<allocatable, target>, uniq_name = "_QMtestEarr"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>)
+    %idx = fir.convert %c10 : (i32) -> index
+    %cond = arith.cmpi sgt, %idx, %c0 : index
+    %n = arith.select %cond, %idx, %c0 : index
+    %mem = fir.allocmem !fir.array<?xi32>, %n {fir.must_be_heap = true}
+    %shape = fir.shape %n : (index) -> !fir.shape<1>
+    %box = fir.embox %mem(%shape) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+    fir.store %box to %decl#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+    %ld = fir.load %decl#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+    %base = fir.box_addr %ld : (!fir.box<!fir.heap<!fir.array<?xi32>>>) -> !fir.heap<!fir.array<?xi32>>
+    fir.freemem %base : !fir.heap<!fir.array<?xi32>>
+    %undef = fir.zero_bits !fir.heap<!fir.array<?xi32>>
+    %sh0 = fir.shape %c0 : (index) -> !fir.shape<1>
+    %empty = fir.embox %undef(%sh0) : (!fir.heap<!fir.array<?xi32>>, !fir.shape<1>) -> !fir.box<!fir.heap<!fir.array<?xi32>>>
+    fir.store %empty to %decl#0 : !fir.ref<!fir.box<!fir.heap<!fir.array<?xi32>>>>
+    return
+  }
+}
+
+// CHECK:         fir.global @[[AUTOMAP:.*]] {{{.*}} automap = true
+// CHECK-LABEL:   func.func @automap()
+// CHECK:           %[[AUTOMAP_ADDR:.*]] = fir.address_of(@[[AUTOMAP]])
+// CHECK:           %[[AUTOMAP_DECL:.*]]:2 = hlfir.declare %[[AUTOMAP_ADDR]]
+// CHECK:           %[[ALLOC_MEM:.*]] = fir.allocmem
+// CHECK-NEXT:      fir.shape
+// CHECK-NEXT:      %[[ARR_BOXED:.*]] = fir.embox %[[ALLOC_MEM]]
+// CHECK-NEXT:      fir.store %[[ARR_BOXED]]
+// CHECK-NEXT:      %[[ARR_BOXED_LOADED:.*]] = fir.load %[[AUTOMAP_DECL]]#0
+// CHECK-NEXT:      %[[ARR_HEAP_PTR:.*]] = fir.box_addr %[[ARR_BOXED_LOADED]]
+// CHECK-NEXT:      %[[DIM0:.*]] = arith.constant 0 : index
+// CHECK-NEXT:      %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[ARR_BOXED_LOADED]], %[[DIM0]]
+// CHECK-NEXT:      %[[ONE:.*]] = arith.constant 1 : index
+// CHECK-NEXT:      %[[ZERO:.*]] = arith.constant 0 : index
+// CHECK-NEXT:      %[[BOX_DIMS2:.*]]:3 = fir.box_dims %[[ARR_BOXED_LOADED]], %[[ZERO]]
+// CHECK-NEXT:      %[[LOWER_BOUND:.*]] = arith.constant 0 : index
+// CHECK-NEXT:      %[[UPPER_BOUND:.*]] = arith.subi %[[BOX_DIMS2]]#1, %[[ONE]] : index
+// CHECK-NEXT:      omp.map.bounds lower_bound(%[[LOWER_BOUND]] : index) upper_bound(%[[UPPER_BOUND]] : index) extent(%[[BOX_DIMS2]]#1 : index) stride(%[[BOX_DIMS2]]#2 : index) start_idx(%[[BOX_DIMS]]#0 : index) {stride_in_bytes = true}
+// CHECK-NEXT:      arith.muli %[[BOX_DIMS2]]#2, %[[BOX_DIMS2]]#1 : index
+// CHECK-NEXT:      %[[MAP_INFO:.*]] = omp.map.info var_ptr(%[[AUTOMAP_DECL]]#0 {{.*}} map_clauses(to) capture(ByCopy)
+// CHECK-NEXT:      omp.target_enter_data map_entries(%[[MAP_INFO]]
+// CHECK:           %[[LOAD:.*]] = fir.load %[[AUTOMAP_DECL]]#0
+// CHECK:           %[[EXIT_MAP:.*]] = omp.map.info var_ptr(%[[AUTOMAP_DECL]]#0 {{.*}} map_clauses(delete) capture(ByCopy)
+// CHECK-NEXT:      omp.target_exit_data map_entries(%[[EXIT_MAP]]
+// CHECK-NEXT:      %[[BOXADDR:.*]] = fir.box_addr %[[LOAD]]
+// CHECK-NEXT:      fir.freemem %[[BOXADDR]]
diff --git a/flang/test/Transforms/optimize-array-repacking.fir b/flang/test/Transforms/optimize-array-repacking.fir
index 6269fa441fe44..15a3e3941f44f 100644
--- a/flang/test/Transforms/optimize-array-repacking.fir
+++ b/flang/test/Transforms/optimize-array-repacking.fir
@@ -658,3 +658,136 @@ func.func @_QPneg_test_pointer(%arg0: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf3
   fir.unpack_array %9 to %7 heap : !fir.box<!fir.array<?xf32>>
   return
 }
+
+// Test a long chain of fir.pack_array operations.
+// The rewriter used to use a down-top traversal that optimized
+// fir.pack_array operations starting from the innermost one.
+// The rewriter did not converge in 10 (default) iterations
+// causing the pass to report a failure.
+// A top-down traversal should fix this an allow optimizing
+// all the repackings.
+// CHECK-LABEL:   func.func @test_long_chain(
+// CHECK-NOT: fir.pack_array
+// CHECK-NOT: fir.unpack_array
+func.func @test_long_chain(%pred: i1) {
+  %c10 = arith.constant 10 : index
+  %3 = fir.dummy_scope : !fir.dscope
+  %4 = fir.address_of(@aaa) : !fir.ref<!fir.array<10x10xi32>>
+  %5 = fir.shape %c10, %c10 : (index, index) -> !fir.shape<2>
+  %6 = fir.declare %4(%5) {uniq_name = "aaa"} : (!fir.ref<!fir.array<10x10xi32>>, !fir.shape<2>) -> !fir.ref<!fir.array<10x10xi32>>
+  %9 = fir.embox %6(%5) : (!fir.ref<!fir.array<10x10xi32>>, !fir.shape<2>) -> !fir.box<!fir.array<10x10xi32>>
+  %10 = fir.convert %9 : (!fir.box<!fir.array<10x10xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %11 = fir.dummy_scope : !fir.dscope
+  %12 = fir.pack_array %10 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %13 = fir.declare %12 dummy_scope %11 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %14 = fir.rebox %13 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  cf.cond_br %pred, ^bb17, ^bb1
+^bb1:  // pred: ^bb0
+  %20 = fir.dummy_scope : !fir.dscope
+  %21 = fir.pack_array %14 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %22 = fir.declare %21 dummy_scope %20 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %23 = fir.rebox %22 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %28 = fir.dummy_scope : !fir.dscope
+  %29 = fir.pack_array %23 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %30 = fir.declare %29 dummy_scope %28 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %31 = fir.rebox %30 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  cf.cond_br %pred, ^bb16, ^bb2
+^bb2:  // pred: ^bb1
+  %37 = fir.dummy_scope : !fir.dscope
+  %38 = fir.pack_array %31 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %39 = fir.declare %38 dummy_scope %37 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %40 = fir.rebox %39 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %45 = fir.dummy_scope : !fir.dscope
+  %46 = fir.pack_array %40 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %47 = fir.declare %46 dummy_scope %45 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %48 = fir.rebox %47 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  cf.cond_br %pred, ^bb15, ^bb3
+^bb3:  // pred: ^bb2
+  %54 = fir.dummy_scope : !fir.dscope
+  %55 = fir.pack_array %48 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %56 = fir.declare %55 dummy_scope %54 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %57 = fir.rebox %56 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %62 = fir.dummy_scope : !fir.dscope
+  %63 = fir.pack_array %57 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %64 = fir.declare %63 dummy_scope %62 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %65 = fir.rebox %64 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  cf.cond_br %pred, ^bb14, ^bb4
+^bb4:  // pred: ^bb3
+  %71 = fir.dummy_scope : !fir.dscope
+  %72 = fir.pack_array %65 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %73 = fir.declare %72 dummy_scope %71 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %74 = fir.rebox %73 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %79 = fir.dummy_scope : !fir.dscope
+  %80 = fir.pack_array %74 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %81 = fir.declare %80 dummy_scope %79 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %82 = fir.rebox %81 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  cf.cond_br %pred, ^bb13, ^bb5
+^bb5:  // pred: ^bb4
+  %88 = fir.dummy_scope : !fir.dscope
+  %89 = fir.pack_array %82 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %90 = fir.declare %89 dummy_scope %88 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %91 = fir.rebox %90 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %96 = fir.dummy_scope : !fir.dscope
+  %97 = fir.pack_array %91 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %98 = fir.declare %97 dummy_scope %96 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %99 = fir.rebox %98 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  cf.cond_br %pred, ^bb12, ^bb6
+^bb6:  // pred: ^bb5
+  %105 = fir.dummy_scope : !fir.dscope
+  %106 = fir.pack_array %99 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %107 = fir.declare %106 dummy_scope %105 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %108 = fir.rebox %107 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %113 = fir.dummy_scope : !fir.dscope
+  %114 = fir.pack_array %108 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %115 = fir.declare %114 dummy_scope %113 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %116 = fir.rebox %115 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  cf.cond_br %pred, ^bb11, ^bb7
+^bb7:  // pred: ^bb6
+  %122 = fir.dummy_scope : !fir.dscope
+  %123 = fir.pack_array %116 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %124 = fir.declare %123 dummy_scope %122 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %125 = fir.rebox %124 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %130 = fir.dummy_scope : !fir.dscope
+  %131 = fir.pack_array %125 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  %132 = fir.declare %131 dummy_scope %130 {uniq_name = "aaa"} : (!fir.box<!fir.array<?x?xi32>>, !fir.dscope) -> !fir.box<!fir.array<?x?xi32>>
+  %133 = fir.rebox %132 : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  cf.cond_br %pred, ^bb9, ^bb8
+^bb8:  // pred: ^bb7
+  %139 = fir.dummy_scope : !fir.dscope
+  %140 = fir.pack_array %133 heap innermost : (!fir.box<!fir.array<?x?xi32>>) -> !fir.box<!fir.array<?x?xi32>>
+  fir.unpack_array %140 to %133 heap : !fir.box<!fir.array<?x?xi32>>
+  cf.br ^bb9
+^bb9:  // 2 preds: ^bb7, ^bb8
+  fir.unpack_array %131 to %125 heap : !fir.box<!fir.array<?x?xi32>>
+  cf.br ^bb10
+^bb10:  // pred: ^bb9
+  fir.unpack_array %123 to %116 heap : !fir.box<!fir.array<?x?xi32>>
+  cf.br ^bb11
+^bb11:  // 2 preds: ^bb6, ^bb10
+  fir.unpack_array %114 to %108 heap : !fir.box<!fir.array<?x?xi32>>
+  fir.unpack_array %106 to %99 heap : !fir.box<!fir.array<?x?xi32>>
+  cf.br ^bb12
+^bb12:  // 2 preds: ^bb5, ^bb11
+  fir.unpack_array %97 to %91 heap : !fir.box<!fir.array<?x?xi32>>
+  fir.unpack_array %89 to %82 heap : !fir.box<!fir.array<?x?xi32>>
+  cf.br ^bb13
+^bb13:  // 2 preds: ^bb4, ^bb12
+  fir.unpack_array %80 to %74 heap : !fir.box<!fir.array<?x?xi32>>
+  fir.unpack_array %72 to %65 heap : !fir.box<!fir.array<?x?xi32>>
+  cf.br ^bb14
+^bb14:  // 2 preds: ^bb3, ^bb13
+  fir.unpack_array %63 to %57 heap : !fir.box<!fir.array<?x?xi32>>
+  fir.unpack_array %55 to %48 heap : !fir.box<!fir.array<?x?xi32>>
+  cf.br ^bb15
+^bb15:  // 2 preds: ^bb2, ^bb14
+  fir.unpack_array %46 to %40 heap : !fir.box<!fir.array<?x?xi32>>
+  fir.unpack_array %38 to %31 heap : !fir.box<!fir.array<?x?xi32>>
+  cf.br ^bb16
+^bb16:  // 2 preds: ^bb1, ^bb15
+  fir.unpack_array %29 to %23 heap : !fir.box<!fir.array<?x?xi32>>
+  fir.unpack_array %21 to %14 heap : !fir.box<!fir.array<?x?xi32>>
+  cf.br ^bb17
+^bb17:  // 2 preds: ^bb0, ^bb16
+  fir.unpack_array %12 to %10 heap : !fir.box<!fir.array<?x?xi32>>
+  return
+}
diff --git a/flang/test/Transforms/stack-arrays-lifetime.fir b/flang/test/Transforms/stack-arrays-lifetime.fir
index 5b2faeba132c3..960ce9fad7b2d 100644
--- a/flang/test/Transforms/stack-arrays-lifetime.fir
+++ b/flang/test/Transforms/stack-arrays-lifetime.fir
@@ -39,15 +39,15 @@ func.func @_QPcst_alloca(%arg0: !fir.ref<!fir.array<100000xf32>> {fir.bindc_name
 // CHECK-DAG:           %[[VAL_0:.*]] = fir.alloca !fir.array<100000xf32> {bindc_name = ".tmp.array", fir.has_lifetime}
 // CHECK-DAG:           %[[VAL_2:.*]] = fir.alloca !fir.array<100000xi32> {bindc_name = ".tmp.array", fir.has_lifetime}
 // CHECK:           %[[VAL_9:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<!fir.array<100000xf32>>) -> !llvm.ptr
-// CHECK:           llvm.intr.lifetime.start 400000, %[[VAL_9]] : !llvm.ptr
+// CHECK:           llvm.intr.lifetime.start %[[VAL_9]] : !llvm.ptr
 // CHECK:           fir.do_loop
 // CHECK:           fir.call @_QPbar(
-// CHECK:           llvm.intr.lifetime.end 400000, %[[VAL_9]] : !llvm.ptr
+// CHECK:           llvm.intr.lifetime.end %[[VAL_9]] : !llvm.ptr
 // CHECK:           %[[VAL_17:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.array<100000xi32>>) -> !llvm.ptr
-// CHECK:           llvm.intr.lifetime.start 400000, %[[VAL_17]] : !llvm.ptr
+// CHECK:           llvm.intr.lifetime.start %[[VAL_17]] : !llvm.ptr
 // CHECK:           fir.do_loop
 // CHECK:           fir.call @_QPibar(
-// CHECK:           llvm.intr.lifetime.end 400000, %[[VAL_17]] : !llvm.ptr
+// CHECK:           llvm.intr.lifetime.end %[[VAL_17]] : !llvm.ptr
 
 
 func.func @_QPdyn_alloca(%arg0: !fir.ref<!fir.array<?xf32>> {fir.bindc_name = "x"}, %arg1: !fir.ref<i64> {fir.bindc_name = "n"}) {
diff --git a/flang/test/lit.cfg.py b/flang/test/lit.cfg.py
index 7eb57670ac767..9d81aa41384ed 100644
--- a/flang/test/lit.cfg.py
+++ b/flang/test/lit.cfg.py
@@ -118,10 +118,11 @@
                 "PATH", config.flang_llvm_tools_dir, append_path=True
             )
 
-# On MacOS, -isysroot is needed to build binaries.
+# On MacOS, some tests need -isysroot to build binaries.
 isysroot_flag = []
 if config.osx_sysroot:
     isysroot_flag = ["-isysroot", config.osx_sysroot]
+config.substitutions.append(("%isysroot", " ".join(isysroot_flag)))
 
 # Check for DEFAULT_SYSROOT, because when it is set -isysroot has no effect.
 if config.default_sysroot:
@@ -133,7 +134,6 @@
     ToolSubst(
         "%flang",
         command=FindTool("flang"),
-        extra_args=isysroot_flag,
         unresolved="fatal",
     ),
     ToolSubst(
@@ -172,6 +172,11 @@
 else:
     llvm_config.add_tool_substitutions(tools, config.llvm_tools_dir)
 
+llvm_config.use_clang(required=False)
+
+# Clang may need the include path for ISO_fortran_binding.h.
+config.substitutions.append(("%flang_include", config.flang_headers_dir))
+
 # Enable libpgmath testing
 result = lit_config.params.get("LIBPGMATH")
 if result:
diff --git a/flang/test/lit.site.cfg.py.in b/flang/test/lit.site.cfg.py.in
index ae5144010bc8b..cc1f4fa6cc9c5 100644
--- a/flang/test/lit.site.cfg.py.in
+++ b/flang/test/lit.site.cfg.py.in
@@ -6,6 +6,7 @@ import lit.util
 config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_DIR@")
 config.llvm_shlib_dir = lit_config.substitute(path(r"@SHLIBDIR@"))
 config.llvm_plugin_ext = "@LLVM_PLUGIN_EXT@"
+config.host_triple = "@LLVM_HOST_TRIPLE@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.llvm_target_triple_env = "@LLVM_TARGET_TRIPLE_ENV@"
 config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
@@ -13,6 +14,7 @@ config.errc_messages = "@LLVM_LIT_ERRC_MESSAGES@"
 config.flang_obj_root = "@FLANG_BINARY_DIR@"
 config.flang_tools_dir = lit_config.substitute("@FLANG_TOOLS_DIR@")
 config.flang_intrinsic_modules_dir = "@FLANG_INTRINSIC_MODULES_DIR@"
+config.flang_headers_dir = "@HEADER_BINARY_DIR@"
 config.flang_llvm_tools_dir = "@CMAKE_BINARY_DIR@/bin"
 config.flang_test_triple = "@FLANG_TEST_TARGET_TRIPLE@"
 config.flang_examples = @LLVM_BUILD_EXAMPLES@
diff --git a/flang/tools/bbc/CMakeLists.txt b/flang/tools/bbc/CMakeLists.txt
index 469266cc81558..7516157731b5e 100644
--- a/flang/tools/bbc/CMakeLists.txt
+++ b/flang/tools/bbc/CMakeLists.txt
@@ -30,6 +30,11 @@ target_link_libraries(bbc PRIVATE
   flangFrontend
   flangPasses
   FlangOpenMPTransforms
+  FortranSupport
+  FortranParser
+  FortranEvaluate
+  FortranSemantics
+  FortranLower
 )
 
 mlir_target_link_libraries(bbc PRIVATE
@@ -37,9 +42,4 @@ mlir_target_link_libraries(bbc PRIVATE
   ${extension_libs}
   MLIRAffineToStandard
   MLIRSCFToControlFlow
-  FortranSupport
-  FortranParser
-  FortranEvaluate
-  FortranSemantics
-  FortranLower
 )
diff --git a/flang/tools/bbc/bbc.cpp b/flang/tools/bbc/bbc.cpp
index edfc878d17524..82dff2653ad09 100644
--- a/flang/tools/bbc/bbc.cpp
+++ b/flang/tools/bbc/bbc.cpp
@@ -520,7 +520,9 @@ static llvm::LogicalResult convertFortranSourceToMLIR(
 
     if (emitFIR && useHLFIR) {
       // lower HLFIR to FIR
-      fir::createHLFIRToFIRPassPipeline(pm, enableOpenMP,
+      fir::EnableOpenMP enableOmp =
+          enableOpenMP ? fir::EnableOpenMP::Full : fir::EnableOpenMP::None;
+      fir::createHLFIRToFIRPassPipeline(pm, enableOmp,
                                         llvm::OptimizationLevel::O2);
       if (mlir::failed(pm.run(mlirModule))) {
         llvm::errs() << "FATAL: lowering from HLFIR to FIR failed";
diff --git a/flang/tools/flang-driver/driver.cpp b/flang/tools/flang-driver/driver.cpp
index 8321b165ba5bb..bd878b7a642f1 100644
--- a/flang/tools/flang-driver/driver.cpp
+++ b/flang/tools/flang-driver/driver.cpp
@@ -16,6 +16,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Driver/Driver.h"
+#include "flang/Config/config.h"
 #include "flang/Frontend/CompilerInvocation.h"
 #include "flang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Basic/Diagnostic.h"
@@ -137,6 +138,7 @@ int main(int argc, const char **argv) {
                                   llvm::sys::getDefaultTargetTriple(), diags,
                                   "flang LLVM compiler");
   theDriver.setTargetAndMode(targetandMode);
+  theDriver.setPreferredLinker(FLANG_DEFAULT_LINKER);
 #ifdef FLANG_RUNTIME_F128_MATH_LIB
   theDriver.setFlangF128MathLibrary(FLANG_RUNTIME_F128_MATH_LIB);
 #endif
diff --git a/flang/tools/tco/tco.cpp b/flang/tools/tco/tco.cpp
index d8daf8769cb21..36939802f55a6 100644
--- a/flang/tools/tco/tco.cpp
+++ b/flang/tools/tco/tco.cpp
@@ -51,6 +51,12 @@ static cl::opt<bool> emitFir("emit-fir",
                              cl::desc("Parse and pretty-print the input"),
                              cl::init(false));
 
+static cl::opt<unsigned>
+    OptLevel("O",
+             cl::desc("Optimization level. [-O0, -O1, -O2, or -O3] "
+                      "(default = '-O2')"),
+             cl::Prefix, cl::init(2));
+
 static cl::opt<std::string> targetTriple("target",
                                          cl::desc("specify a target triple"),
                                          cl::init("native"));
@@ -96,6 +102,22 @@ static void printModule(mlir::ModuleOp mod, raw_ostream &output) {
   output << mod << '\n';
 }
 
+static std::optional<llvm::OptimizationLevel>
+getOptimizationLevel(unsigned level) {
+  switch (level) {
+  default:
+    return std::nullopt;
+  case 0:
+    return llvm::OptimizationLevel::O0;
+  case 1:
+    return llvm::OptimizationLevel::O1;
+  case 2:
+    return llvm::OptimizationLevel::O2;
+  case 3:
+    return llvm::OptimizationLevel::O3;
+  }
+}
+
 // compile a .fir file
 static llvm::LogicalResult
 compileFIR(const mlir::PassPipelineCLParser &passPipeline) {
@@ -157,9 +179,17 @@ compileFIR(const mlir::PassPipelineCLParser &passPipeline) {
     if (mlir::failed(passPipeline.addToPipeline(pm, errorHandler)))
       return mlir::failure();
   } else {
-    MLIRToLLVMPassPipelineConfig config(llvm::OptimizationLevel::O2);
+    std::optional<llvm::OptimizationLevel> level =
+        getOptimizationLevel(OptLevel);
+    if (!level) {
+      errs() << "Error invalid optimization level\n";
+      return mlir::failure();
+    }
+    MLIRToLLVMPassPipelineConfig config(*level);
+    // TODO: config.StackArrays should be set here?
     config.EnableOpenMP = true;  // assume the input contains OpenMP
     config.AliasAnalysis = enableAliasAnalysis && !testGeneratorMode;
+    config.LoopVersioning = OptLevel > 2;
     if (codeGenLLVM) {
       // Run only CodeGen passes.
       fir::createDefaultFIRCodeGenPassPipeline(pm, config);
diff --git a/libc/benchmarks/gpu/BenchmarkLogger.cpp b/libc/benchmarks/gpu/BenchmarkLogger.cpp
deleted file mode 100644
index d5996a74f6dd7..0000000000000
--- a/libc/benchmarks/gpu/BenchmarkLogger.cpp
+++ /dev/null
@@ -1,97 +0,0 @@
-#include "benchmarks/gpu/BenchmarkLogger.h"
-#include "hdr/stdint_proxy.h"
-#include "src/__support/CPP/string.h"
-#include "src/__support/CPP/string_view.h"
-#include "src/__support/OSUtil/io.h"               // write_to_stderr
-#include "src/__support/big_int.h"                 // is_big_int
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/properties/types.h" // LIBC_TYPES_HAS_INT128
-#include "src/__support/uint128.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace benchmarks {
-
-// cpp::string_view specialization
-template <>
-BenchmarkLogger &
-    BenchmarkLogger::operator<< <cpp::string_view>(cpp::string_view str) {
-  LIBC_NAMESPACE::write_to_stderr(str);
-  return *this;
-}
-
-// cpp::string specialization
-template <>
-BenchmarkLogger &BenchmarkLogger::operator<< <cpp::string>(cpp::string str) {
-  return *this << static_cast<cpp::string_view>(str);
-}
-
-// const char* specialization
-template <>
-BenchmarkLogger &BenchmarkLogger::operator<< <const char *>(const char *str) {
-  return *this << cpp::string_view(str);
-}
-
-// char* specialization
-template <> BenchmarkLogger &BenchmarkLogger::operator<< <char *>(char *str) {
-  return *this << cpp::string_view(str);
-}
-
-// char specialization
-template <> BenchmarkLogger &BenchmarkLogger::operator<<(char ch) {
-  return *this << cpp::string_view(&ch, 1);
-}
-
-// bool specialization
-template <> BenchmarkLogger &BenchmarkLogger::operator<<(bool cond) {
-  return *this << (cond ? "true" : "false");
-}
-
-// void * specialization
-template <> BenchmarkLogger &BenchmarkLogger::operator<<(void *addr) {
-  return *this << "0x" << cpp::to_string(reinterpret_cast<uintptr_t>(addr));
-}
-
-template <typename T> BenchmarkLogger &BenchmarkLogger::operator<<(T t) {
-  if constexpr (is_big_int_v<T> ||
-                (cpp::is_integral_v<T> && cpp::is_unsigned_v<T> &&
-                 (sizeof(T) > sizeof(uint64_t)))) {
-    static_assert(sizeof(T) % 8 == 0, "Unsupported size of UInt");
-    const IntegerToString<T, radix::Hex::WithPrefix> buffer(t);
-    return *this << buffer.view();
-  } else {
-    return *this << cpp::to_string(t);
-  }
-}
-
-// is_integral specializations
-// char is already specialized to handle character
-template BenchmarkLogger &BenchmarkLogger::operator<< <short>(short);
-template BenchmarkLogger &BenchmarkLogger::operator<< <int>(int);
-template BenchmarkLogger &BenchmarkLogger::operator<< <long>(long);
-template BenchmarkLogger &BenchmarkLogger::operator<< <long long>(long long);
-template BenchmarkLogger &
-    BenchmarkLogger::operator<< <unsigned char>(unsigned char);
-template BenchmarkLogger &
-    BenchmarkLogger::operator<< <unsigned short>(unsigned short);
-template BenchmarkLogger &
-    BenchmarkLogger::operator<< <unsigned int>(unsigned int);
-template BenchmarkLogger &
-    BenchmarkLogger::operator<< <unsigned long>(unsigned long);
-template BenchmarkLogger &
-    BenchmarkLogger::operator<< <unsigned long long>(unsigned long long);
-
-#ifdef LIBC_TYPES_HAS_INT128
-template BenchmarkLogger &
-    BenchmarkLogger::operator<< <__uint128_t>(__uint128_t);
-#endif // LIBC_TYPES_HAS_INT128
-template BenchmarkLogger &BenchmarkLogger::operator<< <UInt<128>>(UInt<128>);
-template BenchmarkLogger &BenchmarkLogger::operator<< <UInt<192>>(UInt<192>);
-template BenchmarkLogger &BenchmarkLogger::operator<< <UInt<256>>(UInt<256>);
-template BenchmarkLogger &BenchmarkLogger::operator<< <UInt<320>>(UInt<320>);
-
-// TODO: Add floating point formatting once it's supported by StringStream.
-
-BenchmarkLogger log;
-
-} // namespace benchmarks
-} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/benchmarks/gpu/BenchmarkLogger.h b/libc/benchmarks/gpu/BenchmarkLogger.h
deleted file mode 100644
index 2b22aba085f86..0000000000000
--- a/libc/benchmarks/gpu/BenchmarkLogger.h
+++ /dev/null
@@ -1,29 +0,0 @@
-//===-- Utilities to log to standard output during tests --------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_BENCHMARKS_GPU_BENCHMARKLOGGER_H
-#define LLVM_LIBC_BENCHMARKS_GPU_BENCHMARKLOGGER_H
-
-#include "src/__support/macros/config.h"
-
-namespace LIBC_NAMESPACE_DECL {
-namespace benchmarks {
-
-// A class to log to standard output in the context of hermetic tests.
-struct BenchmarkLogger {
-  constexpr BenchmarkLogger() = default;
-  template <typename T> BenchmarkLogger &operator<<(T);
-};
-
-// A global TestLogger instance to be used in tests.
-extern BenchmarkLogger log;
-
-} // namespace benchmarks
-} // namespace LIBC_NAMESPACE_DECL
-
-#endif /* LLVM_LIBC_BENCHMARKS_GPU_BENCHMARKLOGGER_H */
diff --git a/libc/benchmarks/gpu/CMakeLists.txt b/libc/benchmarks/gpu/CMakeLists.txt
index 6ec64bf270b53..6ca134b12a479 100644
--- a/libc/benchmarks/gpu/CMakeLists.txt
+++ b/libc/benchmarks/gpu/CMakeLists.txt
@@ -22,8 +22,6 @@ function(add_benchmark benchmark_name)
       ${BENCHMARK_LINK_LIBRARIES}
     DEPENDS
       libc.src.stdio.printf
-      libc.src.stdlib.srand
-      libc.src.stdlib.rand
       ${BENCHMARK_DEPENDS}
     ${BENCHMARK_UNPARSED_ARGUMENTS}
     COMPILE_OPTIONS
@@ -40,34 +38,25 @@ add_unittest_framework_library(
   SRCS
     LibcGpuBenchmark.cpp
     LibcGpuBenchmarkMain.cpp
-    BenchmarkLogger.cpp
   HDRS
     LibcGpuBenchmark.h
-    BenchmarkLogger.h
   DEPENDS
+    libc.benchmarks.gpu.timing.timing
     libc.hdr.stdint_proxy
-    libc.src.__support.big_int
-    libc.src.__support.c_string
     libc.src.__support.CPP.string
     libc.src.__support.CPP.string_view
     libc.src.__support.CPP.type_traits
-    libc.src.__support.CPP.functional
-    libc.src.__support.CPP.limits
     libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.atomic
     libc.src.__support.CPP.array
-    libc.src.__support.fixed_point.fx_rep
-    libc.src.__support.macros.properties.types
-    libc.src.__support.OSUtil.osutil
-    libc.src.__support.uint128
     libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.nearest_integer_operations
     libc.src.__support.FPUtil.sqrt
     libc.src.__support.fixedvector
-    libc.src.time.clock
-    libc.src.stdlib.rand
-    libc.src.stdlib.srand
-    libc.benchmarks.gpu.timing.timing
+    libc.src.__support.GPU.utils
+    libc.src.__support.time.gpu.time_utils
     libc.src.stdio.printf
+    libc.src.time.clock
 )
 
 add_subdirectory(src)
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
index 57ff5b9fdb846..a4a0ff4ec46e5 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.cpp
@@ -1,15 +1,18 @@
 #include "LibcGpuBenchmark.h"
+
+#include "hdr/stdint_proxy.h"
 #include "src/__support/CPP/algorithm.h"
-#include "src/__support/CPP/array.h"
 #include "src/__support/CPP/atomic.h"
 #include "src/__support/CPP/string.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
 #include "src/__support/FPUtil/sqrt.h"
 #include "src/__support/GPU/utils.h"
 #include "src/__support/fixedvector.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/time/gpu/time_utils.h"
 #include "src/stdio/printf.h"
-#include "src/stdlib/srand.h"
+#include "src/time/clock.h"
 
 namespace LIBC_NAMESPACE_DECL {
 namespace benchmarks {
@@ -20,37 +23,56 @@ void Benchmark::add_benchmark(Benchmark *benchmark) {
   benchmarks.push_back(benchmark);
 }
 
+static void atomic_add_double(cpp::Atomic<uint64_t> &atomic_bits,
+                              double value) {
+  using FPBits = LIBC_NAMESPACE::fputil::FPBits<double>;
+
+  uint64_t expected_bits = atomic_bits.load(cpp::MemoryOrder::RELAXED);
+
+  while (true) {
+    double current_value = FPBits(expected_bits).get_val();
+    double next_value = current_value + value;
+
+    uint64_t desired_bits = FPBits(next_value).uintval();
+    if (atomic_bits.compare_exchange_strong(expected_bits, desired_bits,
+                                            cpp::MemoryOrder::ACQUIRE,
+                                            cpp::MemoryOrder::RELAXED))
+      break;
+  }
+}
+
 struct AtomicBenchmarkSums {
-  cpp::Atomic<uint64_t> cycles_sum = 0;
-  cpp::Atomic<uint64_t> standard_deviation_sum = 0;
+  cpp::Atomic<uint32_t> active_threads = 0;
+  cpp::Atomic<uint64_t> iterations_sum = 0;
+  cpp::Atomic<uint64_t> weighted_cycles_sum_bits = 0;
+  cpp::Atomic<uint64_t> weighted_squared_cycles_sum_bits = 0;
   cpp::Atomic<uint64_t> min = UINT64_MAX;
   cpp::Atomic<uint64_t> max = 0;
-  cpp::Atomic<uint32_t> samples_sum = 0;
-  cpp::Atomic<uint32_t> iterations_sum = 0;
-  cpp::Atomic<clock_t> time_sum = 0;
-  cpp::Atomic<uint64_t> active_threads = 0;
 
   void reset() {
     cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
     active_threads.store(0, cpp::MemoryOrder::RELAXED);
-    cycles_sum.store(0, cpp::MemoryOrder::RELAXED);
-    standard_deviation_sum.store(0, cpp::MemoryOrder::RELAXED);
+    iterations_sum.store(0, cpp::MemoryOrder::RELAXED);
+    weighted_cycles_sum_bits.store(0, cpp::MemoryOrder::RELAXED);
+    weighted_squared_cycles_sum_bits.store(0, cpp::MemoryOrder::RELAXED);
     min.store(UINT64_MAX, cpp::MemoryOrder::RELAXED);
     max.store(0, cpp::MemoryOrder::RELAXED);
-    samples_sum.store(0, cpp::MemoryOrder::RELAXED);
-    iterations_sum.store(0, cpp::MemoryOrder::RELAXED);
-    time_sum.store(0, cpp::MemoryOrder::RELAXED);
     cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
   }
 
   void update(const BenchmarkResult &result) {
     cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
     active_threads.fetch_add(1, cpp::MemoryOrder::RELAXED);
+    iterations_sum.fetch_add(result.total_iterations,
+                             cpp::MemoryOrder::RELAXED);
 
-    cycles_sum.fetch_add(result.cycles, cpp::MemoryOrder::RELAXED);
-    standard_deviation_sum.fetch_add(
-        static_cast<uint64_t>(result.standard_deviation),
-        cpp::MemoryOrder::RELAXED);
+    const double n_i = static_cast<double>(result.total_iterations);
+    const double mean_i = result.cycles;
+    const double stddev_i = result.standard_deviation;
+    const double variance_i = stddev_i * stddev_i;
+    atomic_add_double(weighted_cycles_sum_bits, n_i * mean_i);
+    atomic_add_double(weighted_squared_cycles_sum_bits,
+                      n_i * (variance_i + mean_i * mean_i));
 
     // Perform a CAS loop to atomically update the min
     uint64_t orig_min = min.load(cpp::MemoryOrder::RELAXED);
@@ -66,10 +88,6 @@ struct AtomicBenchmarkSums {
         cpp::MemoryOrder::RELAXED))
       ;
 
-    samples_sum.fetch_add(result.samples, cpp::MemoryOrder::RELAXED);
-    iterations_sum.fetch_add(result.total_iterations,
-                             cpp::MemoryOrder::RELAXED);
-    time_sum.fetch_add(result.total_time, cpp::MemoryOrder::RELAXED);
     cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
   }
 };
@@ -79,46 +97,51 @@ constexpr auto GREEN = "\033[32m";
 constexpr auto RESET = "\033[0m";
 
 void print_results(Benchmark *b) {
-  BenchmarkResult result;
+  using FPBits = LIBC_NAMESPACE::fputil::FPBits<double>;
+
+  BenchmarkResult final_result;
   cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
-  int num_threads = all_results.active_threads.load(cpp::MemoryOrder::RELAXED);
-  result.cycles =
-      all_results.cycles_sum.load(cpp::MemoryOrder::RELAXED) / num_threads;
-  result.standard_deviation =
-      all_results.standard_deviation_sum.load(cpp::MemoryOrder::RELAXED) /
-      num_threads;
-  result.min = all_results.min.load(cpp::MemoryOrder::RELAXED);
-  result.max = all_results.max.load(cpp::MemoryOrder::RELAXED);
-  result.samples =
-      all_results.samples_sum.load(cpp::MemoryOrder::RELAXED) / num_threads;
-  result.total_iterations =
-      all_results.iterations_sum.load(cpp::MemoryOrder::RELAXED) / num_threads;
-  const uint64_t duration_ns =
-      all_results.time_sum.load(cpp::MemoryOrder::RELAXED) / num_threads;
-  const uint64_t duration_us = duration_ns / 1000;
-  const uint64_t duration_ms = duration_ns / (1000 * 1000);
-  uint64_t converted_duration = duration_ns;
-  const char *time_unit;
-  if (duration_ms != 0) {
-    converted_duration = duration_ms;
-    time_unit = "ms";
-  } else if (duration_us != 0) {
-    converted_duration = duration_us;
-    time_unit = "us";
+
+  const uint32_t num_threads =
+      all_results.active_threads.load(cpp::MemoryOrder::RELAXED);
+  final_result.total_iterations =
+      all_results.iterations_sum.load(cpp::MemoryOrder::RELAXED);
+
+  if (final_result.total_iterations > 0) {
+    const uint64_t s1_bits =
+        all_results.weighted_cycles_sum_bits.load(cpp::MemoryOrder::RELAXED);
+    const uint64_t s2_bits = all_results.weighted_squared_cycles_sum_bits.load(
+        cpp::MemoryOrder::RELAXED);
+
+    const double S1 = FPBits(s1_bits).get_val();
+    const double S2 = FPBits(s2_bits).get_val();
+    const double N = static_cast<double>(final_result.total_iterations);
+
+    const double global_mean = S1 / N;
+    const double global_mean_of_squares = S2 / N;
+    const double global_variance =
+        global_mean_of_squares - (global_mean * global_mean);
+
+    final_result.cycles = global_mean;
+    final_result.standard_deviation =
+        fputil::sqrt<double>(global_variance < 0.0 ? 0.0 : global_variance);
   } else {
-    converted_duration = duration_ns;
-    time_unit = "ns";
+    final_result.cycles = 0.0;
+    final_result.standard_deviation = 0.0;
   }
-  result.total_time = converted_duration;
-  // result.total_time =
-  //     all_results.time_sum.load(cpp::MemoryOrder::RELAXED) / num_threads;
+
+  final_result.min = all_results.min.load(cpp::MemoryOrder::RELAXED);
+  final_result.max = all_results.max.load(cpp::MemoryOrder::RELAXED);
   cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
 
   LIBC_NAMESPACE::printf(
-      "%-24s |%8ld |%8ld |%8ld |%11d |%14ld %2s |%9ld |%9d |\n",
-      b->get_test_name().data(), result.cycles, result.min, result.max,
-      result.total_iterations, result.total_time, time_unit,
-      static_cast<uint64_t>(result.standard_deviation), num_threads);
+      "%-24s |%15.0f |%9.0f |%8llu |%8llu |%15llu |%9u |\n",
+      b->get_test_name().data(), final_result.cycles,
+      final_result.standard_deviation,
+      static_cast<unsigned long long>(final_result.min),
+      static_cast<unsigned long long>(final_result.max),
+      static_cast<unsigned long long>(final_result.total_iterations),
+      static_cast<unsigned>(num_threads));
 }
 
 void print_header() {
@@ -126,9 +149,8 @@ void print_header() {
   LIBC_NAMESPACE::printf("Running Suite: %-10s\n",
                          benchmarks[0]->get_suite_name().data());
   LIBC_NAMESPACE::printf("%s", RESET);
-  cpp::string titles =
-      "Benchmark                |  Cycles |     Min |     Max | "
-      "Iterations | Time / Iteration |   Stddev |  Threads |\n";
+  cpp::string titles = "Benchmark                |  Cycles (Mean) |   Stddev | "
+                       "    Min |     Max |     Iterations |  Threads |\n";
   LIBC_NAMESPACE::printf(titles.data());
 
   cpp::string separator(titles.size(), '-');
@@ -139,10 +161,8 @@ void print_header() {
 void Benchmark::run_benchmarks() {
   uint64_t id = gpu::get_thread_id();
 
-  if (id == 0) {
+  if (id == 0)
     print_header();
-    LIBC_NAMESPACE::srand(gpu::processor_clock());
-  }
 
   gpu::sync_threads();
 
@@ -164,69 +184,64 @@ void Benchmark::run_benchmarks() {
 }
 
 BenchmarkResult benchmark(const BenchmarkOptions &options,
-                          cpp::function<uint64_t(void)> wrapper_func) {
+                          const BenchmarkTarget &target) {
   BenchmarkResult result;
   RuntimeEstimationProgression rep;
-  uint32_t total_iterations = 0;
   uint32_t iterations = options.initial_iterations;
+
   if (iterations < 1u)
     iterations = 1;
 
   uint32_t samples = 0;
   uint64_t total_time = 0;
-  uint64_t best_guess = 0;
-  uint64_t cycles_squared = 0;
   uint64_t min = UINT64_MAX;
   uint64_t max = 0;
 
-  uint64_t overhead = UINT64_MAX;
-  int overhead_iterations = 10;
-  for (int i = 0; i < overhead_iterations; i++)
-    overhead = cpp::min(overhead, LIBC_NAMESPACE::overhead());
+  uint32_t call_index = 0;
 
   for (int64_t time_budget = options.max_duration; time_budget >= 0;) {
-    uint64_t sample_cycles = 0;
-    const clock_t start = static_cast<double>(clock());
-    for (uint32_t i = 0; i < iterations; i++) {
-      auto wrapper_intermediate = wrapper_func();
-      uint64_t current_result = wrapper_intermediate - overhead;
+    RefinableRuntimeEstimator sample_estimator;
+
+    const clock_t start = clock();
+    while (sample_estimator.get_iterations() < iterations) {
+      auto current_result = target(call_index++);
       max = cpp::max(max, current_result);
       min = cpp::min(min, current_result);
-      sample_cycles += current_result;
+      sample_estimator.update(current_result);
     }
     const clock_t end = clock();
+
     const clock_t duration_ns =
         ((end - start) * 1000 * 1000 * 1000) / CLOCKS_PER_SEC;
     total_time += duration_ns;
     time_budget -= duration_ns;
     samples++;
-    cycles_squared += sample_cycles * sample_cycles;
 
-    total_iterations += iterations;
-    const double change_ratio =
-        rep.compute_improvement({iterations, sample_cycles});
-    best_guess = rep.current_estimation;
+    const double change_ratio = rep.compute_improvement(sample_estimator);
 
     if (samples >= options.max_samples || iterations >= options.max_iterations)
       break;
+
+    const auto total_iterations = rep.get_estimator().get_iterations();
+
     if (total_time >= options.min_duration && samples >= options.min_samples &&
         total_iterations >= options.min_iterations &&
         change_ratio < options.epsilon)
       break;
 
-    iterations *= options.scaling_factor;
+    iterations = static_cast<uint32_t>(
+        fputil::ceil(iterations * options.scaling_factor));
   }
-  result.cycles = best_guess;
-  result.standard_deviation = fputil::sqrt<double>(
-      static_cast<double>(cycles_squared) / total_iterations -
-      static_cast<double>(best_guess * best_guess));
+
+  const auto &estimator = rep.get_estimator();
+  result.total_iterations = estimator.get_iterations();
+  result.cycles = estimator.get_mean();
+  result.standard_deviation = estimator.get_stddev();
   result.min = min;
   result.max = max;
-  result.samples = samples;
-  result.total_iterations = total_iterations;
-  result.total_time = total_time / total_iterations;
+
   return result;
-};
+}
 
 } // namespace benchmarks
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/benchmarks/gpu/LibcGpuBenchmark.h b/libc/benchmarks/gpu/LibcGpuBenchmark.h
index a6cf62dd30ce5..e36e93c7efc18 100644
--- a/libc/benchmarks/gpu/LibcGpuBenchmark.h
+++ b/libc/benchmarks/gpu/LibcGpuBenchmark.h
@@ -1,18 +1,16 @@
 #ifndef LLVM_LIBC_BENCHMARKS_LIBC_GPU_BENCHMARK_H
 #define LLVM_LIBC_BENCHMARKS_LIBC_GPU_BENCHMARK_H
 
-#include "benchmarks/gpu/BenchmarkLogger.h"
 #include "benchmarks/gpu/timing/timing.h"
+
 #include "hdr/stdint_proxy.h"
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/CPP/array.h"
-#include "src/__support/CPP/functional.h"
-#include "src/__support/CPP/limits.h"
 #include "src/__support/CPP/string_view.h"
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/sqrt.h"
 #include "src/__support/macros/config.h"
-#include "src/stdlib/rand.h"
-#include "src/time/clock.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -30,68 +28,136 @@ struct BenchmarkOptions {
   double scaling_factor = 1.4;
 };
 
-struct Measurement {
+class RefinableRuntimeEstimator {
   uint32_t iterations = 0;
-  uint64_t elapsed_cycles = 0;
-};
-
-class RefinableRuntimeEstimation {
-  uint64_t total_cycles = 0;
-  uint32_t total_iterations = 0;
+  uint64_t sum_of_cycles = 0;
+  uint64_t sum_of_squared_cycles = 0;
 
 public:
-  uint64_t update(const Measurement &M) {
-    total_cycles += M.elapsed_cycles;
-    total_iterations += M.iterations;
-    return total_cycles / total_iterations;
+  void update(uint64_t cycles) noexcept {
+    iterations += 1;
+    sum_of_cycles += cycles;
+    sum_of_squared_cycles += cycles * cycles;
+  }
+
+  void update(const RefinableRuntimeEstimator &other) noexcept {
+    iterations += other.iterations;
+    sum_of_cycles += other.sum_of_cycles;
+    sum_of_squared_cycles += other.sum_of_squared_cycles;
+  }
+
+  double get_mean() const noexcept {
+    if (iterations == 0)
+      return 0.0;
+
+    return static_cast<double>(sum_of_cycles) / iterations;
   }
+
+  double get_variance() const noexcept {
+    if (iterations == 0)
+      return 0.0;
+
+    const double num = static_cast<double>(iterations);
+    const double sum_x = static_cast<double>(sum_of_cycles);
+    const double sum_x2 = static_cast<double>(sum_of_squared_cycles);
+
+    const double mean_of_squares = sum_x2 / num;
+    const double mean = sum_x / num;
+    const double mean_squared = mean * mean;
+    const double variance = mean_of_squares - mean_squared;
+
+    return variance < 0.0 ? 0.0 : variance;
+  }
+
+  double get_stddev() const noexcept {
+    return fputil::sqrt<double>(get_variance());
+  }
+
+  uint32_t get_iterations() const noexcept { return iterations; }
 };
 
 // Tracks the progression of the runtime estimation
 class RuntimeEstimationProgression {
-  RefinableRuntimeEstimation rre;
+  RefinableRuntimeEstimator estimator;
+  double current_mean = 0.0;
 
 public:
-  uint64_t current_estimation = 0;
+  const RefinableRuntimeEstimator &get_estimator() const noexcept {
+    return estimator;
+  }
+
+  double
+  compute_improvement(const RefinableRuntimeEstimator &sample_estimator) {
+    if (sample_estimator.get_iterations() == 0)
+      return 1.0;
 
-  double compute_improvement(const Measurement &M) {
-    const uint64_t new_estimation = rre.update(M);
-    double ratio =
-        (static_cast<double>(current_estimation) / new_estimation) - 1.0;
+    estimator.update(sample_estimator);
 
-    // Get absolute value
+    const double new_mean = estimator.get_mean();
+    if (current_mean == 0.0 || new_mean == 0.0) {
+      current_mean = new_mean;
+      return 1.0;
+    }
+
+    double ratio = (current_mean / new_mean) - 1.0;
     if (ratio < 0)
-      ratio *= -1;
+      ratio = -ratio;
 
-    current_estimation = new_estimation;
+    current_mean = new_mean;
     return ratio;
   }
 };
 
 struct BenchmarkResult {
-  uint64_t cycles = 0;
+  uint64_t total_iterations = 0;
+  double cycles = 0;
   double standard_deviation = 0;
   uint64_t min = UINT64_MAX;
   uint64_t max = 0;
-  uint32_t samples = 0;
-  uint32_t total_iterations = 0;
-  clock_t total_time = 0;
+};
+
+struct BenchmarkTarget {
+  using IndexedFnPtr = uint64_t (*)(uint32_t);
+  using IndexlessFnPtr = uint64_t (*)();
+
+  enum class Kind : uint8_t { Indexed, Indexless } kind;
+  union {
+    IndexedFnPtr indexed_fn_ptr;
+    IndexlessFnPtr indexless_fn_ptr;
+  };
+
+  LIBC_INLINE BenchmarkTarget(IndexedFnPtr func)
+      : kind(Kind::Indexed), indexed_fn_ptr(func) {}
+  LIBC_INLINE BenchmarkTarget(IndexlessFnPtr func)
+      : kind(Kind::Indexless), indexless_fn_ptr(func) {}
+
+  LIBC_INLINE uint64_t operator()([[maybe_unused]] uint32_t call_index) const {
+    return kind == Kind::Indexed ? indexed_fn_ptr(call_index)
+                                 : indexless_fn_ptr();
+  }
 };
 
 BenchmarkResult benchmark(const BenchmarkOptions &options,
-                          cpp::function<uint64_t(void)> wrapper_func);
+                          const BenchmarkTarget &target);
 
 class Benchmark {
-  const cpp::function<uint64_t(void)> func;
+  const BenchmarkTarget target;
   const cpp::string_view suite_name;
   const cpp::string_view test_name;
   const uint32_t num_threads;
 
 public:
-  Benchmark(cpp::function<uint64_t(void)> func, char const *suite_name,
+  Benchmark(uint64_t (*f)(), const char *suite, const char *test,
+            uint32_t threads)
+      : target(BenchmarkTarget(f)), suite_name(suite), test_name(test),
+        num_threads(threads) {
+    add_benchmark(this);
+  }
+
+  Benchmark(uint64_t (*f)(uint32_t), char const *suite_name,
             char const *test_name, uint32_t num_threads)
-      : func(func), suite_name(suite_name), test_name(test_name),
-        num_threads(num_threads) {
+      : target(BenchmarkTarget(f)), suite_name(suite_name),
+        test_name(test_name), num_threads(num_threads) {
     add_benchmark(this);
   }
 
@@ -105,67 +171,139 @@ class Benchmark {
 private:
   BenchmarkResult run() {
     BenchmarkOptions options;
-    return benchmark(options, func);
+    return benchmark(options, target);
   }
 };
 
-// We want our random values to be approximately
-// Output: a random number with the exponent field between min_exp and max_exp,
-// i.e. 2^min_exp <= |real_value| < 2^(max_exp + 1),
-// Caveats:
-//   -EXP_BIAS corresponding to denormal values,
-//   EXP_BIAS + 1 corresponding to inf or nan.
+class RandomGenerator {
+  uint64_t state;
+
+  static LIBC_INLINE uint64_t splitmix64(uint64_t x) noexcept {
+    x += 0x9E3779B97F4A7C15ULL;
+    x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ULL;
+    x = (x ^ (x >> 27)) * 0x94D049BB133111EBULL;
+    x = (x ^ (x >> 31));
+    return x ? x : 0x9E3779B97F4A7C15ULL;
+  }
+
+public:
+  explicit LIBC_INLINE RandomGenerator(uint64_t seed) noexcept
+      : state(splitmix64(seed)) {}
+
+  LIBC_INLINE uint64_t next64() noexcept {
+    uint64_t x = state;
+    x ^= x >> 12;
+    x ^= x << 25;
+    x ^= x >> 27;
+    state = x;
+    return x * 0x2545F4914F6CDD1DULL;
+  }
+
+  LIBC_INLINE uint32_t next32() noexcept {
+    return static_cast<uint32_t>(next64() >> 32);
+  }
+};
+
+// We want random floating-point values whose *unbiased* exponent e is
+// approximately uniform in [min_exp, max_exp]. That is,
+//   2^min_exp <= |value| < 2^(max_exp + 1).
+// Caveats / boundaries:
+// - e = -EXP_BIAS  ==> subnormal range (biased exponent = 0). We ensure a
+//                      non-zero mantissa so we don't accidentally produce 0.
+// - e in [1 - EXP_BIAS, EXP_BIAS] ==> normal numbers.
+// - e = EXP_BIAS + 1 ==> Inf/NaN. We do not include it by default; max_exp
+//                        defaults to EXP_BIAS.
 template <typename T>
 static T
-get_rand_input(int max_exp = LIBC_NAMESPACE::fputil::FPBits<T>::EXP_BIAS,
-               int min_exp = -LIBC_NAMESPACE::fputil::FPBits<T>::EXP_BIAS) {
+get_rand_input(RandomGenerator &rng,
+               int min_exp = -LIBC_NAMESPACE::fputil::FPBits<T>::EXP_BIAS,
+               int max_exp = LIBC_NAMESPACE::fputil::FPBits<T>::EXP_BIAS) {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
-
-  // Required to correctly instantiate FPBits for floats and doubles.
-  using RandType = typename cpp::conditional_t<(cpp::is_same_v<T, double>),
-                                               uint64_t, uint32_t>;
-  RandType bits;
-  if constexpr (cpp::is_same_v<T, uint64_t>)
-    bits = (static_cast<uint64_t>(LIBC_NAMESPACE::rand()) << 32) |
-           static_cast<uint64_t>(LIBC_NAMESPACE::rand());
-  else
-    bits = LIBC_NAMESPACE::rand();
-  double scale =
-      static_cast<double>(max_exp - min_exp + 1) / (2 * FPBits::EXP_BIAS + 1);
-  FPBits fp(bits);
-  fp.set_biased_exponent(
-      static_cast<uint32_t>(fp.get_biased_exponent() * scale + min_exp));
-  return fp.get_val();
+  using Storage = typename FPBits::StorageType;
+
+  // Sanitize and clamp requested range to what the format supports
+  if (min_exp > max_exp) {
+    auto tmp = min_exp;
+    min_exp = max_exp;
+    max_exp = tmp;
+  };
+  min_exp = cpp::max(min_exp, -FPBits::EXP_BIAS);
+  max_exp = cpp::min(max_exp, FPBits::EXP_BIAS);
+
+  // Sample unbiased exponent e uniformly in [min_exp, max_exp] without modulo
+  // bias
+  auto sample_in_range = [&](uint64_t r) -> int32_t {
+    const uint64_t range = static_cast<uint64_t>(
+        static_cast<int64_t>(max_exp) - static_cast<int64_t>(min_exp) + 1);
+    const uint64_t threshold = (-range) % range;
+    while (r < threshold)
+      r = rng.next64();
+    return static_cast<int32_t>(min_exp + static_cast<int64_t>(r % range));
+  };
+  const int32_t e = sample_in_range(rng.next64());
+
+  // Start from random bits to get random sign and mantissa
+  FPBits xbits([&] {
+    if constexpr (cpp::is_same_v<T, double>)
+      return FPBits(rng.next64());
+    else
+      return FPBits(rng.next32());
+  }());
+
+  if (e == -FPBits::EXP_BIAS) {
+    // Subnormal: biased exponent must be 0; ensure mantissa != 0 to avoid 0
+    xbits.set_biased_exponent(Storage(0));
+    if (xbits.get_mantissa() == Storage(0))
+      xbits.set_mantissa(Storage(1));
+  } else {
+    // Normal: biased exponent in [1, 2 * FPBits::EXP_BIAS]
+    const int32_t biased = e + FPBits::EXP_BIAS;
+    xbits.set_biased_exponent(static_cast<Storage>(biased));
+  }
+  return xbits.get_val();
 }
 
 template <typename T> class MathPerf {
-  using FPBits = fputil::FPBits<T>;
-  using StorageType = typename FPBits::StorageType;
-  static constexpr StorageType UIntMax =
-      cpp::numeric_limits<StorageType>::max();
+  static LIBC_INLINE uint64_t make_seed(uint64_t base_seed, uint64_t salt) {
+    const uint64_t tid = gpu::get_thread_id();
+    return base_seed ^ (salt << 32) ^ (tid * 0x9E3779B97F4A7C15ULL);
+  }
 
 public:
+  // Returns cycles-per-call (lower is better)
   template <size_t N = 1>
-  static uint64_t run_throughput_in_range(T f(T), int min_exp, int max_exp) {
+  static uint64_t run_throughput_in_range(T f(T), int min_exp, int max_exp,
+                                          uint32_t call_index) {
     cpp::array<T, N> inputs;
+
+    uint64_t base_seed = static_cast<uint64_t>(call_index);
+    uint64_t salt = static_cast<uint64_t>(N);
+    RandomGenerator rng(make_seed(base_seed, salt));
+
     for (size_t i = 0; i < N; ++i)
-      inputs[i] = get_rand_input<T>(min_exp, max_exp);
+      inputs[i] = get_rand_input<T>(rng, min_exp, max_exp);
 
     uint64_t total_time = LIBC_NAMESPACE::throughput(f, inputs);
 
     return total_time / N;
   }
 
-  // Throughput benchmarking for functions that take 2 inputs.
+  // Returns cycles-per-call (lower is better)
   template <size_t N = 1>
   static uint64_t run_throughput_in_range(T f(T, T), int arg1_min_exp,
                                           int arg1_max_exp, int arg2_min_exp,
-                                          int arg2_max_exp) {
+                                          int arg2_max_exp,
+                                          uint32_t call_index) {
     cpp::array<T, N> inputs1;
     cpp::array<T, N> inputs2;
+
+    uint64_t base_seed = static_cast<uint64_t>(call_index);
+    uint64_t salt = static_cast<uint64_t>(N);
+    RandomGenerator rng(make_seed(base_seed, salt));
+
     for (size_t i = 0; i < N; ++i) {
-      inputs1[i] = get_rand_input<T>(arg1_min_exp, arg1_max_exp);
-      inputs2[i] = get_rand_input<T>(arg2_min_exp, arg2_max_exp);
+      inputs1[i] = get_rand_input<T>(rng, arg1_min_exp, arg1_max_exp);
+      inputs2[i] = get_rand_input<T>(rng, arg2_min_exp, arg2_max_exp);
     }
 
     uint64_t total_time = LIBC_NAMESPACE::throughput(f, inputs1, inputs2);
@@ -193,4 +331,5 @@ template <typename T> class MathPerf {
 #define SINGLE_WAVE_BENCHMARK(SuiteName, TestName, Func)                       \
   BENCHMARK_N_THREADS(SuiteName, TestName, Func,                               \
                       LIBC_NAMESPACE::gpu::get_lane_size())
-#endif
+
+#endif // LLVM_LIBC_BENCHMARKS_LIBC_GPU_BENCHMARK_H
diff --git a/libc/benchmarks/gpu/src/math/CMakeLists.txt b/libc/benchmarks/gpu/src/math/CMakeLists.txt
index 7a12ce4e61c9e..8417f23c124a0 100644
--- a/libc/benchmarks/gpu/src/math/CMakeLists.txt
+++ b/libc/benchmarks/gpu/src/math/CMakeLists.txt
@@ -34,11 +34,6 @@ add_benchmark(
     libc.hdr.stdint_proxy
     libc.src.math.sin
     libc.src.math.sinf
-    libc.src.stdlib.srand
-    libc.src.stdlib.rand
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.CPP.bit
-    libc.src.__support.CPP.array
   COMPILE_OPTIONS
     ${math_benchmark_flags}
   LOADER_ARGS
@@ -54,11 +49,6 @@ add_benchmark(
   DEPENDS
     libc.hdr.stdint_proxy
     libc.src.math.atan2
-    libc.src.stdlib.srand
-    libc.src.stdlib.rand
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.CPP.bit
-    libc.src.__support.CPP.array
   COMPILE_OPTIONS
     ${math_benchmark_flags}
   LOADER_ARGS
diff --git a/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp b/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp
index 1f91a9a35c373..82bb0c5d7de49 100644
--- a/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp
+++ b/libc/benchmarks/gpu/src/math/atan2_benchmark.cpp
@@ -1,27 +1,27 @@
 #include "benchmarks/gpu/LibcGpuBenchmark.h"
 
+#include "hdr/stdint_proxy.h"
 #include "src/math/atan2.h"
-#include "src/stdlib/rand.h"
 
 #if defined(NVPTX_MATH_FOUND) || defined(AMDGPU_MATH_FOUND)
 #include "platform.h"
 #endif
 
-#define BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, N)                      \
-  []() {                                                                       \
+#define BM_RANDOM_INPUTS(T, Func, MinExp, MaxExp, N)                           \
+  [](uint32_t call_index) {                                                    \
     return LIBC_NAMESPACE::benchmarks::MathPerf<T>::run_throughput_in_range<   \
-        N>(Func, MIN_EXP, MAX_EXP, MIN_EXP, MAX_EXP);                          \
+        N>(Func, MinExp, MaxExp, MinExp, MaxExp, call_index);                  \
   }
 
-#define BENCH(T, Name, Func, MIN_EXP, MAX_EXP)                                 \
+#define BENCH(T, Name, Func, MinExp, MaxExp)                                   \
   SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_1,                   \
-                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 1));    \
+                        BM_RANDOM_INPUTS(T, Func, MinExp, MaxExp, 1));         \
   SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_128,                 \
-                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 128));  \
+                        BM_RANDOM_INPUTS(T, Func, MinExp, MaxExp, 128));       \
   SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_1024,                \
-                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 1024)); \
+                        BM_RANDOM_INPUTS(T, Func, MinExp, MaxExp, 1024));      \
   SINGLE_WAVE_BENCHMARK(LlvmLibcAtan2GpuBenchmark, Name##_4096,                \
-                        BM_TWO_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 4096))
+                        BM_RANDOM_INPUTS(T, Func, MinExp, MaxExp, 4096))
 
 BENCH(double, Atan2, LIBC_NAMESPACE::atan2, -1023, 1023);
 BENCH(double, Atan2TwoPi, LIBC_NAMESPACE::atan2, -10, 3);
diff --git a/libc/benchmarks/gpu/src/math/sin_benchmark.cpp b/libc/benchmarks/gpu/src/math/sin_benchmark.cpp
index a759db2e9d33f..5fe95c3f3b268 100644
--- a/libc/benchmarks/gpu/src/math/sin_benchmark.cpp
+++ b/libc/benchmarks/gpu/src/math/sin_benchmark.cpp
@@ -1,36 +1,28 @@
 #include "benchmarks/gpu/LibcGpuBenchmark.h"
 
-#include "src/__support/CPP/array.h"
-#include "src/__support/CPP/bit.h"
-#include "src/__support/CPP/functional.h"
-#include "src/__support/FPUtil/FPBits.h"
+#include "hdr/stdint_proxy.h"
 #include "src/math/sin.h"
 #include "src/math/sinf.h"
-#include "src/stdlib/rand.h"
 
 #if defined(NVPTX_MATH_FOUND) || defined(AMDGPU_MATH_FOUND)
 #include "platform.h"
 #endif
 
-// BENCHMARK() expects a function that with no parameters that returns a
-// uint64_t representing the latency. Defining each benchmark using macro that
-// expands to a lambda to allow us to switch the implementation of `sin()` to
-// easily register NVPTX benchmarks.
-#define BM_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, N)                          \
-  []() {                                                                       \
+#define BM_RANDOM_INPUT(T, Func, MinExp, MaxExp, N)                            \
+  [](uint32_t call_index) {                                                    \
     return LIBC_NAMESPACE::benchmarks::MathPerf<T>::run_throughput_in_range<   \
-        N>(Func, MIN_EXP, MAX_EXP);                                            \
+        N>(Func, MinExp, MaxExp, call_index);                                  \
   }
 
-#define BENCH(T, Name, Func, MIN_EXP, MAX_EXP)                                 \
+#define BENCH(T, Name, Func, MinExp, MaxExp)                                   \
   SINGLE_WAVE_BENCHMARK(LlvmLibcSinGpuBenchmark, Name##_1,                     \
-                        BM_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 1));        \
+                        BM_RANDOM_INPUT(T, Func, MinExp, MaxExp, 1));          \
   SINGLE_WAVE_BENCHMARK(LlvmLibcSinGpuBenchmark, Name##_128,                   \
-                        BM_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 128));      \
+                        BM_RANDOM_INPUT(T, Func, MinExp, MaxExp, 128));        \
   SINGLE_WAVE_BENCHMARK(LlvmLibcSinGpuBenchmark, Name##_1024,                  \
-                        BM_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 1024));     \
+                        BM_RANDOM_INPUT(T, Func, MinExp, MaxExp, 1024));       \
   SINGLE_WAVE_BENCHMARK(LlvmLibcSinGpuBenchmark, Name##_4096,                  \
-                        BM_RANDOM_INPUT(T, Func, MIN_EXP, MAX_EXP, 4096))
+                        BM_RANDOM_INPUT(T, Func, MinExp, MaxExp, 4096))
 
 BENCH(double, Sin, LIBC_NAMESPACE::sin, -1023, 1023);
 BENCH(double, SinTwoPi, LIBC_NAMESPACE::sin, -10, 3);
diff --git a/libc/benchmarks/gpu/timing/amdgpu/CMakeLists.txt b/libc/benchmarks/gpu/timing/amdgpu/CMakeLists.txt
index dd7c2d342f70f..f85152e69c346 100644
--- a/libc/benchmarks/gpu/timing/amdgpu/CMakeLists.txt
+++ b/libc/benchmarks/gpu/timing/amdgpu/CMakeLists.txt
@@ -4,9 +4,11 @@ add_header_library(
     timing.h
   DEPENDS
     libc.hdr.stdint_proxy
-    libc.src.__support.common
     libc.src.__support.macros.config
     libc.src.__support.macros.attributes
-    libc.src.__support.CPP.type_traits
+    libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.array
+    libc.src.__support.CPP.atomic
+    libc.src.__support.CPP.type_traits
+    libc.src.__support.GPU.utils
 )
diff --git a/libc/benchmarks/gpu/timing/amdgpu/timing.h b/libc/benchmarks/gpu/timing/amdgpu/timing.h
index 37dbb9af5976b..8b92584b39230 100644
--- a/libc/benchmarks/gpu/timing/amdgpu/timing.h
+++ b/libc/benchmarks/gpu/timing/amdgpu/timing.h
@@ -10,11 +10,11 @@
 #define LLVM_LIBC_UTILS_GPU_TIMING_AMDGPU
 
 #include "hdr/stdint_proxy.h"
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/CPP/array.h"
 #include "src/__support/CPP/atomic.h"
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/GPU/utils.h"
-#include "src/__support/common.h"
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/config.h"
 
@@ -105,20 +105,86 @@ template <typename F, typename T1, typename T2>
   return stop - start;
 }
 
-// Provides throughput benchmarking.
-template <typename F, typename T, size_t N>
-[[gnu::noinline]] static LIBC_INLINE uint64_t
-throughput(F f, const cpp::array<T, N> &inputs) {
+// Provides the *baseline* for throughput: measures loop and measurement costs
+// without calling the f function
+template <typename T, size_t N>
+static LIBC_INLINE uint64_t
+throughput_baseline(const cpp::array<T, N> &inputs) {
   asm("" ::"v"(&inputs));
 
   cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
   uint64_t start = gpu::processor_clock();
+  asm("" ::"s"(start));
+
+  T result{};
+
+#pragma clang loop unroll(disable)
+  for (auto input : inputs) {
+    asm("" ::"v"(input));
+    result = input;
+    asm("" ::"v"(result));
+  }
+
+  uint64_t stop = gpu::processor_clock();
+  asm("" ::"s"(stop));
+  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+
+  volatile auto output = result;
 
+  return stop - start;
+}
+
+// Provides throughput benchmarking
+template <typename F, typename T, size_t N>
+static LIBC_INLINE uint64_t throughput(F f, const cpp::array<T, N> &inputs) {
+  uint64_t baseline = UINT64_MAX;
+  for (int i = 0; i < 5; ++i)
+    baseline = cpp::min(baseline, throughput_baseline<T, N>(inputs));
+
+  asm("" ::"v"(&inputs));
+
+  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+  uint64_t start = gpu::processor_clock();
   asm("" ::"s"(start));
 
+  T result{};
+
+#pragma clang loop unroll(disable)
   for (auto input : inputs) {
-    auto result = f(input);
+    asm("" ::"v"(input));
+    result = f(input);
+    asm("" ::"v"(result));
+  }
+
+  uint64_t stop = gpu::processor_clock();
+  asm("" ::"s"(stop));
+  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+
+  volatile auto output = result;
+
+  const uint64_t measured = stop - start;
+  return measured > baseline ? (measured - baseline) : 0;
+}
+
+// Provides the *baseline* for throughput with 2 arguments: measures loop and
+// measurement costs without calling the f function
+template <typename T, size_t N>
+static LIBC_INLINE uint64_t throughput_baseline(
+    const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {
+  asm("" ::"v"(&inputs1), "v"(&inputs2));
+
+  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+  uint64_t start = gpu::processor_clock();
+  asm("" ::"s"(start));
 
+  T result{};
+
+#pragma clang loop unroll(disable)
+  for (size_t i = 0; i < N; i++) {
+    T x = inputs1[i];
+    T y = inputs2[i];
+    asm("" ::"v"(x), "v"(y));
+    result = x;
     asm("" ::"v"(result));
   }
 
@@ -126,24 +192,33 @@ throughput(F f, const cpp::array<T, N> &inputs) {
   asm("" ::"s"(stop));
   cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
 
-  // Return the time elapsed.
+  volatile auto output = result;
+
   return stop - start;
 }
 
 // Provides throughput benchmarking for 2 arguments (e.g. atan2())
 template <typename F, typename T, size_t N>
-[[gnu::noinline]] static LIBC_INLINE uint64_t throughput(
-    F f, const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {
+static LIBC_INLINE uint64_t throughput(F f, const cpp::array<T, N> &inputs1,
+                                       const cpp::array<T, N> &inputs2) {
+  uint64_t baseline = UINT64_MAX;
+  for (int i = 0; i < 5; ++i)
+    baseline = cpp::min(baseline, throughput_baseline<T, N>(inputs1, inputs2));
+
   asm("" ::"v"(&inputs1), "v"(&inputs2));
 
   cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
   uint64_t start = gpu::processor_clock();
-
   asm("" ::"s"(start));
 
-  for (size_t i = 0; i < inputs1.size(); i++) {
-    auto result = f(inputs1[i], inputs2[i]);
+  T result{};
 
+#pragma clang loop unroll(disable)
+  for (size_t i = 0; i < N; i++) {
+    T x = inputs1[i];
+    T y = inputs2[i];
+    asm("" ::"v"(x), "v"(y));
+    result = f(x, y);
     asm("" ::"v"(result));
   }
 
@@ -151,8 +226,10 @@ template <typename F, typename T, size_t N>
   asm("" ::"s"(stop));
   cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
 
-  // Return the time elapsed.
-  return stop - start;
+  volatile auto output = result;
+
+  const uint64_t measured = stop - start;
+  return measured > baseline ? (measured - baseline) : 0;
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/benchmarks/gpu/timing/nvptx/CMakeLists.txt b/libc/benchmarks/gpu/timing/nvptx/CMakeLists.txt
index a19c16ee4e44d..4615f53e3d247 100644
--- a/libc/benchmarks/gpu/timing/nvptx/CMakeLists.txt
+++ b/libc/benchmarks/gpu/timing/nvptx/CMakeLists.txt
@@ -4,9 +4,11 @@ add_header_library(
     timing.h
   DEPENDS
     libc.hdr.stdint_proxy
-    libc.src.__support.common
     libc.src.__support.macros.config
     libc.src.__support.macros.attributes
-    libc.src.__support.CPP.type_traits
+    libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.array
+    libc.src.__support.CPP.atomic
+    libc.src.__support.CPP.type_traits
+    libc.src.__support.GPU.utils
 )
diff --git a/libc/benchmarks/gpu/timing/nvptx/timing.h b/libc/benchmarks/gpu/timing/nvptx/timing.h
index 3c729636367aa..944d3732eae65 100644
--- a/libc/benchmarks/gpu/timing/nvptx/timing.h
+++ b/libc/benchmarks/gpu/timing/nvptx/timing.h
@@ -10,11 +10,10 @@
 #define LLVM_LIBC_UTILS_GPU_TIMING_NVPTX
 
 #include "hdr/stdint_proxy.h"
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/CPP/array.h"
 #include "src/__support/CPP/atomic.h"
-#include "src/__support/CPP/type_traits.h"
 #include "src/__support/GPU/utils.h"
-#include "src/__support/common.h"
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/config.h"
 
@@ -65,7 +64,7 @@ template <typename F, typename T>
   uint64_t stop = gpu::processor_clock();
   cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
   asm("" ::"r"(stop));
-  volatile T output = result;
+  volatile auto output = result;
 
   // Return the time elapsed.
   return stop - start;
@@ -95,18 +94,51 @@ static LIBC_INLINE uint64_t latency(F f, T1 t1, T2 t2) {
   return stop - start;
 }
 
-// Provides throughput benchmarking.
-template <typename F, typename T, size_t N>
-[[gnu::noinline]] static LIBC_INLINE uint64_t
-throughput(F f, const cpp::array<T, N> &inputs) {
+// Provides the *baseline* for throughput: measures loop and measurement costs
+// without calling the f function
+template <typename T, size_t N>
+static LIBC_INLINE uint64_t
+throughput_baseline(const cpp::array<T, N> &inputs) {
   asm("" ::"r"(&inputs));
 
   cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
   uint64_t start = gpu::processor_clock();
+  asm("" ::"llr"(start));
+
+  T result{};
+
+#pragma clang loop unroll(disable)
+  for (auto input : inputs) {
+    asm("" ::"r"(input));
+    result = input;
+    asm("" ::"r"(result));
+  }
 
+  uint64_t stop = gpu::processor_clock();
+  asm("" ::"r"(stop));
+  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+
+  volatile auto output = result;
+
+  return stop - start;
+}
+
+// Provides throughput benchmarking
+template <typename F, typename T, size_t N>
+static LIBC_INLINE uint64_t throughput(F f, const cpp::array<T, N> &inputs) {
+  uint64_t baseline = UINT64_MAX;
+  for (int i = 0; i < 5; ++i)
+    baseline = cpp::min(baseline, throughput_baseline<T, N>(inputs));
+
+  asm("" ::"r"(&inputs));
+
+  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+  uint64_t start = gpu::processor_clock();
   asm("" ::"llr"(start));
 
-  uint64_t result;
+  T result{};
+
+#pragma clang loop unroll(disable)
   for (auto input : inputs) {
     asm("" ::"r"(input));
     result = f(input);
@@ -114,39 +146,81 @@ throughput(F f, const cpp::array<T, N> &inputs) {
   }
 
   uint64_t stop = gpu::processor_clock();
+  asm("" ::"r"(stop));
+  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+
+  volatile auto output = result;
+
+  const uint64_t measured = stop - start;
+  return measured > baseline ? (measured - baseline) : 0;
+}
+
+// Provides the *baseline* for throughput with 2 arguments: measures loop and
+// measurement costs without calling the f function
+template <typename T, size_t N>
+static LIBC_INLINE uint64_t throughput_baseline(
+    const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {
+  asm("" ::"r"(&inputs1), "r"(&inputs2));
+
   cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+  uint64_t start = gpu::processor_clock();
+  asm("" ::"llr"(start));
+
+  T result{};
+
+#pragma clang loop unroll(disable)
+  for (size_t i = 0; i < N; i++) {
+    T x = inputs1[i];
+    T y = inputs2[i];
+    asm("" ::"r"(x), "r"(y));
+    result = x;
+    asm("" ::"r"(result));
+  }
+
+  uint64_t stop = gpu::processor_clock();
   asm("" ::"r"(stop));
+  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+
   volatile auto output = result;
 
-  // Return the time elapsed.
   return stop - start;
 }
 
 // Provides throughput benchmarking for 2 arguments (e.g. atan2())
 template <typename F, typename T, size_t N>
-[[gnu::noinline]] static LIBC_INLINE uint64_t throughput(
-    F f, const cpp::array<T, N> &inputs1, const cpp::array<T, N> &inputs2) {
+static LIBC_INLINE uint64_t throughput(F f, const cpp::array<T, N> &inputs1,
+                                       const cpp::array<T, N> &inputs2) {
+  uint64_t baseline = UINT64_MAX;
+  for (int i = 0; i < 5; ++i)
+    baseline = cpp::min(baseline, throughput_baseline<T, N>(inputs1, inputs2));
+
   asm("" ::"r"(&inputs1), "r"(&inputs2));
 
   cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
   uint64_t start = gpu::processor_clock();
-
   asm("" ::"llr"(start));
 
-  uint64_t result;
-  for (size_t i = 0; i < inputs1.size(); i++) {
-    result = f(inputs1[i], inputs2[i]);
+  T result{};
+
+#pragma clang loop unroll(disable)
+  for (size_t i = 0; i < N; i++) {
+    T x = inputs1[i];
+    T y = inputs2[i];
+    asm("" ::"r"(x), "r"(y));
+    result = f(x, y);
     asm("" ::"r"(result));
   }
 
   uint64_t stop = gpu::processor_clock();
-  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
   asm("" ::"r"(stop));
+  cpp::atomic_thread_fence(cpp::MemoryOrder::ACQ_REL);
+
   volatile auto output = result;
 
-  // Return the time elapsed.
-  return stop - start;
+  const uint64_t measured = stop - start;
+  return measured > baseline ? (measured - baseline) : 0;
 }
+
 } // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_UTILS_GPU_TIMING_NVPTX
diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index d85c3930474ab..2eb0f066e19e0 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -115,11 +115,31 @@ function(_get_compile_options_from_config output_var)
   set(${output_var} ${config_options} PARENT_SCOPE)
 endfunction(_get_compile_options_from_config)
 
+function(_get_compile_options_from_arch output_var)
+  # Set options that are not found in src/__support/macros/properties/architectures.h
+  # and src/__support/macros/properties/os.h
+  # TODO: we probably want to unify these at some point for consistency
+  set(config_options "")
+
+  if (LIBC_TARGET_OS_IS_BAREMETAL)
+    list(APPEND config_options "-DLIBC_TARGET_OS_IS_BAREMETAL")  
+  endif()
+  if (LIBC_TARGET_OS_IS_GPU)
+    list(APPEND config_options "-DLIBC_TARGET_OS_IS_GPU")  
+  endif()
+  if (LIBC_TARGET_OS_IS_UEFI)
+    list(APPEND config_options "-DLIBC_TARGET_OS_IS_UEFI")  
+  endif()
+
+  set(${output_var} ${config_options} PARENT_SCOPE)
+endfunction(_get_compile_options_from_arch)
+
 function(_get_common_compile_options output_var flags)
   _get_compile_options_from_flags(compile_flags ${flags})
   _get_compile_options_from_config(config_flags)
+  _get_compile_options_from_arch(arch_flags)
 
-  set(compile_options ${LIBC_COMPILE_OPTIONS_DEFAULT} ${compile_flags} ${config_flags})
+  set(compile_options ${LIBC_COMPILE_OPTIONS_DEFAULT} ${compile_flags} ${config_flags} ${arch_flags})
 
   if(LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE)
     list(APPEND compile_options "-fpie")
diff --git a/libc/cmake/modules/LLVMLibCObjectRules.cmake b/libc/cmake/modules/LLVMLibCObjectRules.cmake
index 805da91284ce8..030157bf64585 100644
--- a/libc/cmake/modules/LLVMLibCObjectRules.cmake
+++ b/libc/cmake/modules/LLVMLibCObjectRules.cmake
@@ -1,4 +1,33 @@
 set(OBJECT_LIBRARY_TARGET_TYPE "OBJECT_LIBRARY")
+set(ENTRYPOINT_OBJ_TARGET_TYPE "ENTRYPOINT_OBJ")
+set(ENTRYPOINT_EXT_TARGET_TYPE "ENTRYPOINT_EXT")
+
+# Rule to check if a list of dependencies contains any entrypoint objects. Returns a list in entrypoint_deps.
+function(check_entrypoint_deps entrypoint_deps)
+  set(PUBLIC_DEPS "")
+  set(fq_deps_list "")
+  list(APPEND fq_deps_list ${ARGN})
+
+  #don't warn for deps that are allowed, such as errno
+  set(ALLOWED_DEPS
+    "libc.src.errno.errno"
+    "libc.src.setjmp.longjmp"
+  )
+  list(REMOVE_ITEM fq_deps_list ${ALLOWED_DEPS})
+
+  foreach(dep IN LISTS fq_deps_list)
+    if(NOT TARGET ${dep})
+      continue()
+    endif()
+
+    get_target_property(target_type ${dep} "TARGET_TYPE")
+    if(${target_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE})
+      list(APPEND PUBLIC_DEPS ${dep})
+    endif()
+  endforeach()
+  set(${entrypoint_deps} ${PUBLIC_DEPS} PARENT_SCOPE)
+endfunction()
+
 
 # Rule which is essentially a wrapper over add_library to compile a set of
 # sources to object files.
@@ -65,6 +94,18 @@ function(create_object_library fq_target_name)
   target_include_directories(${fq_target_name} PRIVATE ${LIBC_SOURCE_DIR})
   target_compile_options(${fq_target_name} PRIVATE ${compile_options})
 
+  #loop through the deps, check if any have the TARGET_TYPE of ENTRYPOINT_OBJ_TARGET_TYPE, and print a warning if they do.
+  if(LIBC_CMAKE_VERBOSE_LOGGING)
+    set(entrypoint_deps "")
+    if(NOT "${fq_deps_list}" STREQUAL "")
+      check_entrypoint_deps(entrypoint_deps ${fq_deps_list})
+    endif()
+    if(NOT "${entrypoint_deps}" STREQUAL "")
+      message(WARNING "Object ${fq_target_name} depends on public entrypoint(s) ${entrypoint_deps}.
+      Depending on public entrypoints is not allowed in internal code.")
+    endif()
+  endif()
+
   if(SHOW_INTERMEDIATE_OBJECTS)
     message(STATUS "Adding object library ${fq_target_name}")
     if(${SHOW_INTERMEDIATE_OBJECTS} STREQUAL "DEPS")
@@ -110,7 +151,6 @@ function(add_object_library target_name)
     ${ARGN})
 endfunction(add_object_library)
 
-set(ENTRYPOINT_OBJ_TARGET_TYPE "ENTRYPOINT_OBJ")
 
 # A rule for entrypoint object targets.
 # Usage:
@@ -179,7 +219,6 @@ function(create_entrypoint_object fq_target_name)
 
     get_target_property(obj_type ${fq_dep_name} "TARGET_TYPE")
     if((NOT obj_type) OR (NOT ${obj_type} STREQUAL ${ENTRYPOINT_OBJ_TARGET_TYPE}))
-                              
       message(FATAL_ERROR "The aliasee of an entrypoint alias should be an entrypoint.")
     endif()
 
@@ -230,6 +269,19 @@ function(create_entrypoint_object fq_target_name)
   _get_common_compile_options(common_compile_options "${ADD_ENTRYPOINT_OBJ_FLAGS}")
   list(APPEND common_compile_options ${ADD_ENTRYPOINT_OBJ_COMPILE_OPTIONS})
   get_fq_deps_list(fq_deps_list ${ADD_ENTRYPOINT_OBJ_DEPENDS})
+
+  #loop through the deps, check if any have the TARGET_TYPE of entrypoint_target_type, and print a warning if they do.
+  if(LIBC_CMAKE_VERBOSE_LOGGING)
+    set(entrypoint_deps "")
+    if(NOT "${fq_deps_list}" STREQUAL "")
+      check_entrypoint_deps(entrypoint_deps ${fq_deps_list})
+    endif()
+    if(NOT "${entrypoint_deps}" STREQUAL "")
+      message(WARNING "Entrypoint ${fq_target_name} depends on public entrypoint(s) ${entrypoint_deps}.
+      Depending on public entrypoints is not allowed in internal code.")
+    endif()
+  endif()
+
   set(full_deps_list ${fq_deps_list} libc.src.__support.common)
 
   if(SHOW_INTERMEDIATE_OBJECTS)
@@ -390,8 +442,6 @@ function(add_entrypoint_object target_name)
   )
 endfunction(add_entrypoint_object)
 
-set(ENTRYPOINT_EXT_TARGET_TYPE "ENTRYPOINT_EXT")
-
 # A rule for external entrypoint targets.
 # Usage:
 #     add_entrypoint_external(
diff --git a/libc/cmake/modules/LLVMLibCTestRules.cmake b/libc/cmake/modules/LLVMLibCTestRules.cmake
index 267c32e956945..d92fdb2938f3c 100644
--- a/libc/cmake/modules/LLVMLibCTestRules.cmake
+++ b/libc/cmake/modules/LLVMLibCTestRules.cmake
@@ -1,6 +1,7 @@
 function(_get_common_test_compile_options output_var c_test flags)
   _get_compile_options_from_flags(compile_flags ${flags})
   _get_compile_options_from_config(config_flags)
+  _get_compile_options_from_arch(arch_flags)
 
   # Remove -fno-math-errno if it was added.
   if(LIBC_ADD_FNO_MATH_ERRNO)
@@ -16,7 +17,8 @@ function(_get_common_test_compile_options output_var c_test flags)
       ${LIBC_COMPILE_OPTIONS_DEFAULT}
       ${LIBC_TEST_COMPILE_OPTIONS_DEFAULT}
       ${compile_flags}
-      ${config_flags})
+      ${config_flags}
+      ${arch_flags})
 
   if(LLVM_LIBC_COMPILER_IS_GCC_COMPATIBLE)
     list(APPEND compile_options "-fpie")
@@ -836,7 +838,7 @@ function(add_libc_hermetic test_name)
                    ${fq_deps_list})
   # TODO: currently the dependency chain is broken such that getauxval cannot properly
   # propagate to hermetic tests. This is a temporary workaround.
-  if (LIBC_TARGET_ARCHITECTURE_IS_AARCH64)
+  if (LIBC_TARGET_ARCHITECTURE_IS_AARCH64 AND NOT(LIBC_TARGET_OS_IS_BAREMETAL))
     target_link_libraries(
       ${fq_build_target_name}
       PRIVATE
diff --git a/libc/config/baremetal/aarch64/entrypoints.txt b/libc/config/baremetal/aarch64/entrypoints.txt
index e24e2b9e2a7bb..5649c2b31f101 100644
--- a/libc/config/baremetal/aarch64/entrypoints.txt
+++ b/libc/config/baremetal/aarch64/entrypoints.txt
@@ -757,9 +757,73 @@ endif()
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
+if(LIBC_TYPES_HAS_FLOAT128)
+  list(APPEND TARGET_LIBM_ENTRYPOINTS
+    # math.h C++23 mixed bfloat16 and _Float128 entrypoints
+    libc.src.math.bf16addf128
+    libc.src.math.bf16divf128
+    libc.src.math.bf16fmaf128
+    libc.src.math.bf16mulf128
+    libc.src.math.bf16subf128
+  )
+endif()
+
 if(LIBC_COMPILER_HAS_FIXED_POINT)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # stdfix.h _Fract and _Accum entrypoints
diff --git a/libc/config/baremetal/arm/entrypoints.txt b/libc/config/baremetal/arm/entrypoints.txt
index 44e9c3edac353..ad3ceab3c6b01 100644
--- a/libc/config/baremetal/arm/entrypoints.txt
+++ b/libc/config/baremetal/arm/entrypoints.txt
@@ -760,9 +760,73 @@ endif()
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
+if(LIBC_TYPES_HAS_FLOAT128)
+  list(APPEND TARGET_LIBM_ENTRYPOINTS
+    # math.h C++23 mixed bfloat16 and _Float128 entrypoints
+    libc.src.math.bf16addf128
+    libc.src.math.bf16divf128
+    libc.src.math.bf16fmaf128
+    libc.src.math.bf16mulf128
+    libc.src.math.bf16subf128
+  )
+endif()
+
 if(LIBC_COMPILER_HAS_FIXED_POINT)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # stdfix.h _Fract and _Accum entrypoints
diff --git a/libc/config/baremetal/riscv/entrypoints.txt b/libc/config/baremetal/riscv/entrypoints.txt
index 29cf322a2e33f..8c50fd1e41df5 100644
--- a/libc/config/baremetal/riscv/entrypoints.txt
+++ b/libc/config/baremetal/riscv/entrypoints.txt
@@ -760,9 +760,73 @@ endif()
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
+if(LIBC_TYPES_HAS_FLOAT128)
+  list(APPEND TARGET_LIBM_ENTRYPOINTS
+    # math.h C++23 mixed bfloat16 and _Float128 entrypoints
+    libc.src.math.bf16addf128
+    libc.src.math.bf16divf128
+    libc.src.math.bf16fmaf128
+    libc.src.math.bf16mulf128
+    libc.src.math.bf16subf128
+  )
+endif()
+
 if(LIBC_COMPILER_HAS_FIXED_POINT)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # stdfix.h _Fract and _Accum entrypoints
diff --git a/libc/config/config.json b/libc/config/config.json
index 1b0546980e6ba..cfbe9a43948ea 100644
--- a/libc/config/config.json
+++ b/libc/config/config.json
@@ -2,7 +2,7 @@
   "errno": {
     "LIBC_CONF_ERRNO_MODE": {
       "value": "LIBC_ERRNO_MODE_DEFAULT",
-      "doc": "The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, LIBC_ERRNO_MODE_SYSTEM, and LIBC_ERRNO_MODE_SYSTEM_INLINE."
+      "doc": "The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, and LIBC_ERRNO_MODE_SYSTEM_INLINE."
     }
   },
   "threads": {
diff --git a/libc/config/darwin/aarch64/entrypoints.txt b/libc/config/darwin/aarch64/entrypoints.txt
index 03e00a35a4cbe..08bc10b403e56 100644
--- a/libc/config/darwin/aarch64/entrypoints.txt
+++ b/libc/config/darwin/aarch64/entrypoints.txt
@@ -590,9 +590,71 @@ endif()
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
+if(LIBC_TYPES_HAS_FLOAT128)
+  list(APPEND TARGET_LIBM_ENTRYPOINTS
+    # math.h C++23 mixed bfloat16 and _Float128 entrypoints
+    libc.src.math.bf16addf128
+    libc.src.math.bf16mulf128
+    libc.src.math.bf16subf128
+  )
+endif()
+
 if(LIBC_COMPILER_HAS_FIXED_POINT)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # stdfix.h _Fract and _Accum entrypoints
diff --git a/libc/config/darwin/x86_64/entrypoints.txt b/libc/config/darwin/x86_64/entrypoints.txt
index 00cedaba810c7..49f440404b9dd 100644
--- a/libc/config/darwin/x86_64/entrypoints.txt
+++ b/libc/config/darwin/x86_64/entrypoints.txt
@@ -233,7 +233,60 @@ set(TARGET_LIBM_ENTRYPOINTS
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
 set(TARGET_LLVMLIBC_ENTRYPOINTS
diff --git a/libc/config/gpu/amdgpu/entrypoints.txt b/libc/config/gpu/amdgpu/entrypoints.txt
index cf2b8c6a55878..b557046598392 100644
--- a/libc/config/gpu/amdgpu/entrypoints.txt
+++ b/libc/config/gpu/amdgpu/entrypoints.txt
@@ -616,7 +616,60 @@ endif()
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
 set(TARGET_LLVMLIBC_ENTRYPOINTS
diff --git a/libc/config/gpu/nvptx/entrypoints.txt b/libc/config/gpu/nvptx/entrypoints.txt
index 55b27e69d5571..f1eac1f3fe6d9 100644
--- a/libc/config/gpu/nvptx/entrypoints.txt
+++ b/libc/config/gpu/nvptx/entrypoints.txt
@@ -280,6 +280,7 @@ set(TARGET_LIBC_ENTRYPOINTS
 
 set(TARGET_LIBM_ENTRYPOINTS
     # math.h entrypoints
+    libc.src.math.acos
     libc.src.math.acosf
     libc.src.math.acoshf
     libc.src.math.asin
@@ -617,7 +618,60 @@ endif()
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
 set(TARGET_LLVMLIBC_ENTRYPOINTS
diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index d76cdc2f2db4b..63471f9905988 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -660,6 +660,7 @@ if(LIBC_TYPES_HAS_FLOAT16)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # math.h C23 _Float16 entrypoints
     # libc.src.math.acoshf16
+    libc.src.math.asinpif16
     libc.src.math.canonicalizef16
     libc.src.math.ceilf16
     libc.src.math.copysignf16
@@ -844,9 +845,73 @@ endif()
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
+if(LIBC_TYPES_HAS_FLOAT128)
+  list(APPEND TARGET_LIBM_ENTRYPOINTS
+    # math.h C++23 mixed bfloat16 and _Float128 entrypoints
+    libc.src.math.bf16addf128
+    libc.src.math.bf16divf128
+    libc.src.math.bf16fmaf128
+    libc.src.math.bf16mulf128
+    libc.src.math.bf16subf128
+  )
+endif()
+
 if(LLVM_LIBC_FULL_BUILD)
   list(APPEND TARGET_LIBC_ENTRYPOINTS
     # assert.h entrypoints
diff --git a/libc/config/linux/arm/config.json b/libc/config/linux/arm/config.json
new file mode 100644
index 0000000000000..e7ad4544b104d
--- /dev/null
+++ b/libc/config/linux/arm/config.json
@@ -0,0 +1,7 @@
+{
+  "string": {
+    "LIBC_CONF_STRING_UNSAFE_WIDE_READ": {
+      "value": false
+    }
+  }
+}
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index 813c34d498f46..5f95c80166941 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -460,7 +460,60 @@ set(TARGET_LIBM_ENTRYPOINTS
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
 set(TARGET_LLVMLIBC_ENTRYPOINTS
diff --git a/libc/config/linux/config.json b/libc/config/linux/config.json
new file mode 100644
index 0000000000000..30e8b2cdadabe
--- /dev/null
+++ b/libc/config/linux/config.json
@@ -0,0 +1,7 @@
+{
+  "string": {
+    "LIBC_CONF_STRING_UNSAFE_WIDE_READ": {
+      "value": true
+    }
+  }
+}
diff --git a/libc/config/linux/riscv/config.json b/libc/config/linux/riscv/config.json
new file mode 100644
index 0000000000000..e7ad4544b104d
--- /dev/null
+++ b/libc/config/linux/riscv/config.json
@@ -0,0 +1,7 @@
+{
+  "string": {
+    "LIBC_CONF_STRING_UNSAFE_WIDE_READ": {
+      "value": false
+    }
+  }
+}
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 190aef7dd7a37..7c4a287c4a330 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -863,9 +863,73 @@ endif()
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
+if(LIBC_TYPES_HAS_FLOAT128)
+  list(APPEND TARGET_LIBM_ENTRYPOINTS
+    # math.h C++23 mixed bfloat16 and _Float128 entrypoints
+    libc.src.math.bf16addf128
+    libc.src.math.bf16divf128
+    libc.src.math.bf16fmaf128
+    libc.src.math.bf16mulf128
+    libc.src.math.bf16subf128
+  )
+endif()
+
 if(LIBC_COMPILER_HAS_FIXED_POINT)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # stdfix.h _Fract and _Accum entrypoints
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index ec41069e98c91..5e8bfc6341b47 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -704,6 +704,7 @@ if(LIBC_TYPES_HAS_FLOAT16)
     libc.src.math.acospif16
     libc.src.math.asinf16
     libc.src.math.asinhf16
+    libc.src.math.asinpif16
     libc.src.math.atanf16
     libc.src.math.atanhf16
     libc.src.math.canonicalizef16
@@ -893,12 +894,75 @@ if(LIBC_TYPES_HAS_FLOAT128)
   )
 endif()
 
-
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
+if(LIBC_TYPES_HAS_FLOAT128)
+  list(APPEND TARGET_LIBM_ENTRYPOINTS
+    # math.h C++23 mixed bfloat16 and _Float128 entrypoints
+    libc.src.math.bf16addf128
+    libc.src.math.bf16divf128
+    libc.src.math.bf16fmaf128
+    libc.src.math.bf16mulf128
+    libc.src.math.bf16subf128
+  )
+endif()
+
 if(LIBC_COMPILER_HAS_FIXED_POINT)
   list(APPEND TARGET_LIBM_ENTRYPOINTS
     # stdfix.h _Fract and _Accum entrypoints
diff --git a/libc/config/windows/entrypoints.txt b/libc/config/windows/entrypoints.txt
index 3160d5790f33b..8a61845b750ce 100644
--- a/libc/config/windows/entrypoints.txt
+++ b/libc/config/windows/entrypoints.txt
@@ -306,7 +306,60 @@ set(TARGET_LIBM_ENTRYPOINTS
 
 list(APPEND TARGET_LIBM_ENTRYPOINTS
   # bfloat16 entrypoints
+  libc.src.math.bf16add
+  libc.src.math.bf16addf
+  libc.src.math.bf16addl
+  libc.src.math.bf16div
+  libc.src.math.bf16divf
+  libc.src.math.bf16divl
+  libc.src.math.bf16fma
+  libc.src.math.bf16fmaf
+  libc.src.math.bf16fmal
+  libc.src.math.bf16mul
+  libc.src.math.bf16mulf
+  libc.src.math.bf16mull
+  libc.src.math.bf16sub
+  libc.src.math.bf16subf
+  libc.src.math.bf16subl
+  libc.src.math.ceilbf16
   libc.src.math.fabsbf16
+  libc.src.math.floorbf16
+  libc.src.math.fmaxbf16
+  libc.src.math.fmaximumbf16
+  libc.src.math.fmaximum_magbf16
+  libc.src.math.fmaximum_mag_numbf16
+  libc.src.math.fmaximum_numbf16
+  libc.src.math.fminbf16
+  libc.src.math.fminimumbf16
+  libc.src.math.fminimum_magbf16
+  libc.src.math.fminimum_mag_numbf16
+  libc.src.math.fminimum_numbf16
+  libc.src.math.frexpbf16
+  libc.src.math.llrintbf16
+  libc.src.math.llroundbf16
+  libc.src.math.lrintbf16
+  libc.src.math.lroundbf16
+  libc.src.math.fromfpbf16
+  libc.src.math.fromfpxbf16
+  libc.src.math.getpayloadbf16
+  libc.src.math.ilogbbf16
+  libc.src.math.ldexpbf16
+  libc.src.math.llogbbf16
+  libc.src.math.logbbf16
+  libc.src.math.nanbf16
+  libc.src.math.nearbyintbf16
+  libc.src.math.nextafterbf16
+  libc.src.math.nextdownbf16
+  libc.src.math.nexttowardbf16
+  libc.src.math.nextupbf16
+  libc.src.math.roundbf16
+  libc.src.math.roundevenbf16
+  libc.src.math.rintbf16
+  libc.src.math.setpayloadbf16
+  libc.src.math.setpayloadsigbf16
+  libc.src.math.truncbf16
+  libc.src.math.ufromfpbf16
+  libc.src.math.ufromfpxbf16
 )
 
 set(TARGET_LLVMLIBC_ENTRYPOINTS
diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst
index 95c51b8517e64..e23fc824ce7c8 100644
--- a/libc/docs/configure.rst
+++ b/libc/docs/configure.rst
@@ -29,7 +29,7 @@ to learn about the defaults for your platform and target.
     - ``LIBC_CONF_ENABLE_STRONG_STACK_PROTECTOR``: Enable -fstack-protector-strong to defend against stack smashing attack.
     - ``LIBC_CONF_KEEP_FRAME_POINTER``: Keep frame pointer in functions for better debugging experience.
 * **"errno" options**
-    - ``LIBC_CONF_ERRNO_MODE``: The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, LIBC_ERRNO_MODE_SYSTEM, and LIBC_ERRNO_MODE_SYSTEM_INLINE.
+    - ``LIBC_CONF_ERRNO_MODE``: The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, and LIBC_ERRNO_MODE_SYSTEM_INLINE.
 * **"general" options**
     - ``LIBC_ADD_NULL_CHECKS``: Add nullptr checks in the library's implementations to some functions for which passing nullptr is undefined behavior.
 * **"math" options**
diff --git a/libc/docs/headers/math/index.rst b/libc/docs/headers/math/index.rst
index 9679c4a6c807f..6a2699e88aa9f 100644
--- a/libc/docs/headers/math/index.rst
+++ b/libc/docs/headers/math/index.rst
@@ -104,144 +104,153 @@ Implementation Status
 Basic Operations
 ================
 
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| <Func>           | <Func_f> (float) | <Func> (double) | <Func_l> (long double) | <Func_f16> (float16) | <Func_f128> (float128) | C23 Definition Section | C23 Error Handling Section |
-+==================+==================+=================+========================+======================+========================+========================+============================+
-| ceil             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.1               | F.10.6.1                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| canonicalize     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.7              | F.10.8.7                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| copysign         | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.1              | F.10.8.1                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| dadd             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | 7.12.14.1              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| ddiv             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | 7.12.14.4              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| dfma             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | 7.12.14.5              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| dmul             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | 7.12.14.3              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| dsub             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | 7.12.14.2              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| f16add           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | 7.12.14.1              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| f16div           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | 7.12.14.4              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| f16fma           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | 7.12.14.5              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| f16mul           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | 7.12.14.5              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| f16sub           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | 7.12.14.2              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fabs             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.7.3               | F.10.4.3                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fadd             | N/A              | |check|         | |check|                | N/A                  | |check|                | 7.12.14.1              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fdim             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.1              | F.10.9.1                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fdiv             | N/A              | |check|         | |check|                | N/A                  | |check|\*              | 7.12.14.4              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| ffma             | N/A              | |check|         | |check|                | N/A                  | |check|\*              | 7.12.14.5              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| floor            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.2               | F.10.6.2                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmax             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.2              | F.10.9.2                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmaximum         | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.4              | F.10.9.4                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmaximum_mag     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.6              | F.10.9.4                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmaximum_mag_num | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.10             | F.10.9.5                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmaximum_num     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.8              | F.10.9.5                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmin             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.3              | F.10.9.3                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fminimum         | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.5              | F.10.9.4                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fminimum_mag     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.7              | F.10.9.4                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fminimum_mag_num | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.11             | F.10.9.5                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fminimum_num     | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.12.9              | F.10.9.5                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmod             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.10.1              | F.10.7.1                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fmul             | N/A              | |check|         | |check|                | N/A                  | |check|\*              | 7.12.14.3              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| frexp            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.7               | F.10.3.7                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fromfp           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.10              | F.10.6.10                  |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fromfpx          | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.11              | F.10.6.11                  |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| fsub             | N/A              | |check|         | |check|                | N/A                  | |check|\*              | 7.12.14.2              | F.10.11                    |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| getpayload       | |check|          | |check|         | |check|                | |check|              | |check|                | F.10.13.1              | N/A                        |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| ilogb            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.8               | F.10.3.8                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| iscanonical      | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.3.2               | N/A                        |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| issignaling      | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.3.8               | N/A                        |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| ldexp            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.9               | F.10.3.9                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| llogb            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.10              | F.10.3.10                  |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| llrint           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.5               | F.10.6.5                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| llround          | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.7               | F.10.6.7                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| logb             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.17              | F.10.3.17                  |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| lrint            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.5               | F.10.6.5                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| lround           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.7               | F.10.6.7                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| modf             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.18              | F.10.3.18                  |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nan              | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.2              | F.10.8.2                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nearbyint        | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.3               | F.10.6.3                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nextafter        | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.3              | F.10.8.3                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nextdown         | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.6              | F.10.8.6                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nexttoward       | |check|          | |check|         | |check|                | |check|              | N/A                    | 7.12.11.4              | F.10.8.4                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| nextup           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.11.5              | F.10.8.5                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| remainder        | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.10.2              | F.10.7.2                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| remquo           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.10.3              | F.10.7.3                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| rint             | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.4               | F.10.6.4                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| round            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.6               | F.10.6.6                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| roundeven        | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.8               | F.10.6.8                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| scalbln          | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.19              | F.10.3.19                  |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| scalbn           | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.6.19              | F.10.3.19                  |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| setpayload       | |check|          | |check|         | |check|                | |check|              | |check|                | F.10.13.2              | N/A                        |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| setpayloadsig    | |check|          | |check|         | |check|                | |check|              | |check|                | F.10.13.3              | N/A                        |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| totalorder       | |check|          | |check|         | |check|                | |check|              | |check|                | F.10.12.1              | N/A                        |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| totalordermag    | |check|          | |check|         | |check|                | |check|              | |check|                | F.10.12.2              | N/A                        |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| trunc            | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.9               | F.10.6.9                   |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| ufromfp          | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.10              | F.10.6.10                  |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| ufromfpx         | |check|          | |check|         | |check|                | |check|              | |check|                | 7.12.9.11              | F.10.6.11                  |
-+------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| <Func>           | <Func_f> (float) | <Func> (double) | <Func_l> (long double) | <Func_f16> (float16) | <Func_f128> (float128) | <Func_bf16> (bfloat16) | C23 Definition Section | C23 Error Handling Section |
++==================+==================+=================+========================+======================+========================+========================+========================+============================+
+| ceil             | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.1               | F.10.6.1                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| canonicalize     | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.11.7              | F.10.8.7                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| copysign         | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.11.1              | F.10.8.1                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| dadd             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | N/A                    | 7.12.14.1              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| ddiv             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | N/A                    | 7.12.14.4              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| dfma             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | N/A                    | 7.12.14.5              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| dmul             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | N/A                    | 7.12.14.3              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| dsub             | N/A              | N/A             | |check|                | N/A                  | |check|\*              | N/A                    | 7.12.14.2              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| f16add           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.1              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| f16div           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.4              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| f16fma           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.5              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| f16mul           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.3              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| f16sub           | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.2              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| bf16add          | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.1              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| bf16div          | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.4              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| bf16fma          | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.5              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| bf16mul          | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.3              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| bf16sub          | |check|\*        | |check|\*       | |check|\*              | N/A                  | |check|                | N/A                    | 7.12.14.2              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fabs             | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.7.3               | F.10.4.3                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fadd             | N/A              | |check|         | |check|                | N/A                  | |check|                | N/A                    | 7.12.14.1              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fdim             | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.12.1              | F.10.9.1                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fdiv             | N/A              | |check|         | |check|                | N/A                  | |check|\*              | N/A                    | 7.12.14.4              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| ffma             | N/A              | |check|         | |check|                | N/A                  | |check|\*              | N/A                    | 7.12.14.5              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| floor            | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.2               | F.10.6.2                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fmax             | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.2              | F.10.9.2                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fmaximum         | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.4              | F.10.9.4                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fmaximum_mag     | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.6              | F.10.9.4                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fmaximum_mag_num | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.10             | F.10.9.5                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fmaximum_num     | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.8              | F.10.9.5                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fmin             | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.3              | F.10.9.3                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fminimum         | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.5              | F.10.9.4                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fminimum_mag     | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.7              | F.10.9.4                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fminimum_mag_num | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.11             | F.10.9.5                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fminimum_num     | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.12.9              | F.10.9.5                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fmod             | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.10.1              | F.10.7.1                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fmul             | N/A              | |check|         | |check|                | N/A                  | |check|\*              | N/A                    | 7.12.14.3              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| frexp            | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.6.7               | F.10.3.7                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fromfp           | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.10              | F.10.6.10                  |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fromfpx          | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.11              | F.10.6.11                  |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| fsub             | N/A              | |check|         | |check|                | N/A                  | |check|\*              | N/A                    | 7.12.14.2              | F.10.11                    |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| getpayload       | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | F.10.13.1              | N/A                        |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| ilogb            | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.6.8               | F.10.3.8                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| iscanonical      | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.3.2               | N/A                        |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| issignaling      | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.3.8               | N/A                        |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| ldexp            | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.6.9               | F.10.3.9                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| llogb            | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.6.10              | F.10.3.10                  |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| llrint           | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.5               | F.10.6.5                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| llround          | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.7               | F.10.6.7                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| logb             | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.6.17              | F.10.3.17                  |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| lrint            | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.5               | F.10.6.5                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| lround           | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.7               | F.10.6.7                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| modf             | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.6.18              | F.10.3.18                  |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| nan              | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.11.2              | F.10.8.2                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| nearbyint        | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.3               | F.10.6.3                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| nextafter        | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.11.3              | F.10.8.3                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| nextdown         | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.11.6              | F.10.8.6                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| nexttoward       | |check|          | |check|         | |check|                | |check|              | N/A                    | |check|                | 7.12.11.4              | F.10.8.4                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| nextup           | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.11.5              | F.10.8.5                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| remainder        | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.10.2              | F.10.7.2                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| remquo           | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.10.3              | F.10.7.3                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| rint             | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.4               | F.10.6.4                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| round            | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.6               | F.10.6.6                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| roundeven        | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.8               | F.10.6.8                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| scalbln          | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.6.19              | F.10.3.19                  |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| scalbn           | |check|          | |check|         | |check|                | |check|              | |check|                |                        | 7.12.6.19              | F.10.3.19                  |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| setpayload       | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | F.10.13.2              | N/A                        |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| setpayloadsig    | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | F.10.13.3              | N/A                        |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| totalorder       | |check|          | |check|         | |check|                | |check|              | |check|                |                        | F.10.12.1              | N/A                        |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| totalordermag    | |check|          | |check|         | |check|                | |check|              | |check|                |                        | F.10.12.2              | N/A                        |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| trunc            | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.9               | F.10.6.9                   |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| ufromfp          | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.10              | F.10.6.10                  |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
+| ufromfpx         | |check|          | |check|         | |check|                | |check|              | |check|                | |check|                | 7.12.9.11              | F.10.6.11                  |
++------------------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+------------------------+----------------------------+
 
 Higher Math Functions
 =====================
@@ -259,7 +268,7 @@ Higher Math Functions
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | asinh     | |check|          |                 |                        | |check|              |                        | 7.12.5.2               | F.10.2.2                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
-| asinpi    |                  |                 |                        |                      |                        | 7.12.4.9               | F.10.1.9                   |
+| asinpi    |                  |                 |                        | |check|              |                        | 7.12.4.9               | F.10.1.9                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
 | atan      | |check|          | 1 ULP           |                        | |check|              |                        | 7.12.4.3               | F.10.1.3                   |
 +-----------+------------------+-----------------+------------------------+----------------------+------------------------+------------------------+----------------------------+
@@ -456,3 +465,5 @@ References
 * `The CORE-MATH Project <https://core-math.gitlabpages.inria.fr/>`_.
 * `The GNU C Library (glibc) <https://www.gnu.org/software/libc/>`_.
 * `The GNU MPFR Library <https://www.mpfr.org/>`_.
+* `C++23 Standard <https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/n4950.pdf>`_.
+* `BFloat16 <https://en.wikipedia.org/wiki/Bfloat16_floating-point_format>`_.
\ No newline at end of file
diff --git a/libc/include/__llvm-libc-common.h b/libc/include/__llvm-libc-common.h
index c6fd33a55532c..1fe3f4d2aea9c 100644
--- a/libc/include/__llvm-libc-common.h
+++ b/libc/include/__llvm-libc-common.h
@@ -47,6 +47,13 @@
 #define __NOEXCEPT throw()
 #endif
 
+#undef _Returns_twice
+#if __cplusplus >= 201103L
+#define _Returns_twice [[gnu::returns_twice]]
+#else
+#define _Returns_twice __attribute__((returns_twice))
+#endif
+
 // This macro serves as a generic cast implementation for use in both C and C++,
 // similar to `__BIONIC_CAST` in Android.
 #undef __LLVM_LIBC_CAST
diff --git a/libc/include/dlfcn.yaml b/libc/include/dlfcn.yaml
index db2893aaff5d9..bf17a11111050 100644
--- a/libc/include/dlfcn.yaml
+++ b/libc/include/dlfcn.yaml
@@ -46,7 +46,7 @@ enums:
     standards:
       - gnu
     value: 2
-  - name: RTLD_DI_CONFIGADDR,
+  - name: RTLD_DI_CONFIGADDR
     standards:
       - gnu
     value: 3
@@ -127,5 +127,5 @@ functions:
       - POSIX
     return_type: int
     arguments:
-      - type: const void *
-      - type: Dl_info *
+      - type: const void *__restrict
+      - type: Dl_info *__restrict
diff --git a/libc/include/math.yaml b/libc/include/math.yaml
index e8ac7ee5033db..6f48e0e9fd35f 100644
--- a/libc/include/math.yaml
+++ b/libc/include/math.yaml
@@ -79,6 +79,13 @@ functions:
     arguments:
       - type: _Float16
     guard: LIBC_TYPES_HAS_FLOAT16
+  - name: asinpif16
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: _Float16
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: atan
     standards:
       - stdc
@@ -363,6 +370,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: exp10f16
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: _Float16
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: exp10m1f
     standards:
       - stdc
@@ -388,6 +402,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: exp2f16
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: _Float16
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: exp2m1f
     standards:
       - stdc
@@ -407,6 +428,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: expf16
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: _Float16
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: expm1
     standards:
       - stdc
@@ -419,6 +447,13 @@ functions:
     return_type: float
     arguments:
       - type: float
+  - name: expm1f16
+    standards:
+      - stdc
+    return_type: _Float16
+    arguments:
+      - type: _Float16
+    guard: LIBC_TYPES_HAS_FLOAT16
   - name: f16add
     standards:
       - llvm_libc_ext
@@ -2425,14 +2460,6 @@ functions:
     return_type: double
     arguments:
       - type: double
-  - name: sincosf
-    standards:
-      - gnu
-    return_type: void
-    arguments:
-      - type: float
-      - type: float *
-      - type: float *
   - name: sinf
     standards:
       - stdc
@@ -2446,6 +2473,22 @@ functions:
     arguments:
       - type: _Float16
     guard: LIBC_TYPES_HAS_FLOAT16
+  - name: sincos
+    standards:
+      - gnu
+    return_type: void
+    arguments:
+      - type: double
+      - type: double *
+      - type: double *
+  - name: sincosf
+    standards:
+      - gnu
+    return_type: void
+    arguments:
+      - type: float
+      - type: float *
+      - type: float *
   - name: sinhf
     standards:
       - stdc
diff --git a/libc/include/sched.yaml b/libc/include/sched.yaml
index f14799ddf33fa..8014aa7ed61fc 100644
--- a/libc/include/sched.yaml
+++ b/libc/include/sched.yaml
@@ -20,7 +20,7 @@ functions:
       - type: const cpu_set_t *
   - name: getcpu
     standards:
-      - POSIX
+      - Linux
     return_type: int
     arguments:
       - type: unsigned int *
diff --git a/libc/include/sys/ioctl.yaml b/libc/include/sys/ioctl.yaml
index 5f7b7f333191e..7eb66b657664a 100644
--- a/libc/include/sys/ioctl.yaml
+++ b/libc/include/sys/ioctl.yaml
@@ -5,4 +5,12 @@ macros: []
 types: []
 enums: []
 objects: []
-functions: []
+functions:
+  - name: ioctl
+    standards:
+      - Linux
+    return_type: int
+    arguments:
+      - type: int
+      - type: unsigned long
+      - type: '...'
diff --git a/libc/shared/math.h b/libc/shared/math.h
index 3714f380a27dc..69d785b3e0291 100644
--- a/libc/shared/math.h
+++ b/libc/shared/math.h
@@ -31,6 +31,14 @@
 #include "math/atanhf.h"
 #include "math/atanhf16.h"
 #include "math/cbrt.h"
+#include "math/cbrtf.h"
+#include "math/cos.h"
+#include "math/cosf.h"
+#include "math/cosf16.h"
+#include "math/coshf.h"
+#include "math/coshf16.h"
+#include "math/cospif.h"
+#include "math/cospif16.h"
 #include "math/erff.h"
 #include "math/exp.h"
 #include "math/exp10.h"
diff --git a/libc/shared/math/cbrtf.h b/libc/shared/math/cbrtf.h
new file mode 100644
index 0000000000000..09b86bed3fb7e
--- /dev/null
+++ b/libc/shared/math/cbrtf.h
@@ -0,0 +1,23 @@
+//===-- Shared cbrtf function -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_SHARED_MATH_CBRTF_H
+#define LIBC_SHARED_MATH_CBRTF_H
+
+#include "shared/libc_common.h"
+#include "src/__support/math/cbrtf.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::cbrtf;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_SHARED_MATH_CBRTF_H
diff --git a/libc/shared/math/cos.h b/libc/shared/math/cos.h
new file mode 100644
index 0000000000000..c498550f098b4
--- /dev/null
+++ b/libc/shared/math/cos.h
@@ -0,0 +1,23 @@
+//===-- Shared cos function -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_COS_H
+#define LLVM_LIBC_SHARED_MATH_COS_H
+
+#include "shared/libc_common.h"
+#include "src/__support/math/cos.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::cos;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_MATH_COS_H
diff --git a/libc/shared/math/cosf.h b/libc/shared/math/cosf.h
new file mode 100644
index 0000000000000..06182207a82f2
--- /dev/null
+++ b/libc/shared/math/cosf.h
@@ -0,0 +1,23 @@
+//===-- Shared cosf function ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_COSF_H
+#define LLVM_LIBC_SHARED_MATH_COSF_H
+
+#include "shared/libc_common.h"
+#include "src/__support/math/cosf.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::cosf;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_MATH_COSF_H
diff --git a/libc/shared/math/cosf16.h b/libc/shared/math/cosf16.h
new file mode 100644
index 0000000000000..8a19285c5755b
--- /dev/null
+++ b/libc/shared/math/cosf16.h
@@ -0,0 +1,28 @@
+//===-- Shared cosf16 function ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_COSF16_H
+#define LLVM_LIBC_SHARED_MATH_COSF16_H
+
+#include "shared/libc_common.h"
+
+#ifdef LIBC_TYPES_HAS_FLOAT16
+
+#include "src/__support/math/cosf16.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::cosf16;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_TYPES_HAS_FLOAT16
+
+#endif // LLVM_LIBC_SHARED_MATH_COSF16_H
diff --git a/libc/shared/math/coshf.h b/libc/shared/math/coshf.h
new file mode 100644
index 0000000000000..33c2580630d59
--- /dev/null
+++ b/libc/shared/math/coshf.h
@@ -0,0 +1,23 @@
+//===-- Shared coshf function -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_COSHF_H
+#define LLVM_LIBC_SHARED_MATH_COSHF_H
+
+#include "shared/libc_common.h"
+#include "src/__support/math/coshf.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::coshf;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_MATH_COSHF_H
diff --git a/libc/shared/math/coshf16.h b/libc/shared/math/coshf16.h
new file mode 100644
index 0000000000000..66e8d1431df65
--- /dev/null
+++ b/libc/shared/math/coshf16.h
@@ -0,0 +1,28 @@
+//===-- Shared coshf16 function ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_COSHF16_H
+#define LLVM_LIBC_SHARED_MATH_COSHF16_H
+
+#include "shared/libc_common.h"
+
+#ifdef LIBC_TYPES_HAS_FLOAT16
+
+#include "src/__support/math/coshf16.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::coshf16;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_TYPES_HAS_FLOAT16
+
+#endif // LLVM_LIBC_SHARED_MATH_COSHF16_H
diff --git a/libc/shared/math/cospif.h b/libc/shared/math/cospif.h
new file mode 100644
index 0000000000000..ce664e9c54859
--- /dev/null
+++ b/libc/shared/math/cospif.h
@@ -0,0 +1,23 @@
+//===-- Shared cospif function ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_COSPIF_H
+#define LLVM_LIBC_SHARED_MATH_COSPIF_H
+
+#include "shared/libc_common.h"
+#include "src/__support/math/cospif.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::cospif;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SHARED_MATH_COSPIF_H
diff --git a/libc/shared/math/cospif16.h b/libc/shared/math/cospif16.h
new file mode 100644
index 0000000000000..a1680b00d0b5b
--- /dev/null
+++ b/libc/shared/math/cospif16.h
@@ -0,0 +1,28 @@
+//===-- Shared cospif16 function --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SHARED_MATH_COSPIF16_H
+#define LLVM_LIBC_SHARED_MATH_COSPIF16_H
+
+#include "shared/libc_common.h"
+
+#ifdef LIBC_TYPES_HAS_FLOAT16
+
+#include "src/__support/math/cospif16.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace shared {
+
+using math::cospif16;
+
+} // namespace shared
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_TYPES_HAS_FLOAT16
+
+#endif // LLVM_LIBC_SHARED_MATH_COSPIF_H
diff --git a/libc/shared/rpc.h b/libc/shared/rpc.h
index 7295efd4eee33..dac2a7949a906 100644
--- a/libc/shared/rpc.h
+++ b/libc/shared/rpc.h
@@ -551,6 +551,9 @@ template <uint32_t opcode> RPC_ATTRS Client::Port Client::open() {
 /// port if it has a pending receive operation
 RPC_ATTRS rpc::optional<typename Server::Port>
 Server::try_open(uint32_t lane_size, uint32_t start) {
+  if (rpc::get_lane_id() >= lane_size)
+    return rpc::nullopt;
+
   // Perform a naive linear scan for a port that has a pending request.
   for (uint32_t index = start; index < process.port_count; ++index) {
     uint64_t lane_mask = rpc::get_lane_mask();
diff --git a/libc/src/__support/FPUtil/CMakeLists.txt b/libc/src/__support/FPUtil/CMakeLists.txt
index 6e447fcd47368..e8fc539fd32e8 100644
--- a/libc/src/__support/FPUtil/CMakeLists.txt
+++ b/libc/src/__support/FPUtil/CMakeLists.txt
@@ -231,6 +231,7 @@ add_header_library(
     Hypot.h
   DEPENDS
     .basic_operations
+    .cast
     .fenv_impl
     .fp_bits
     .rounding_mode
@@ -285,6 +286,9 @@ add_header_library(
     libc.hdr.stdint_proxy
     libc.src.__support.CPP.bit
     libc.src.__support.CPP.type_traits
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.FPUtil.generic.div
+    libc.src.__support.FPUtil.generic.mul
     libc.src.__support.macros.config
     libc.src.__support.macros.properties.types
 )
diff --git a/libc/src/__support/FPUtil/Hypot.h b/libc/src/__support/FPUtil/Hypot.h
index 94da259cd42f0..e23f8b52d8220 100644
--- a/libc/src/__support/FPUtil/Hypot.h
+++ b/libc/src/__support/FPUtil/Hypot.h
@@ -12,6 +12,7 @@
 #include "BasicOperations.h"
 #include "FEnvImpl.h"
 #include "FPBits.h"
+#include "cast.h"
 #include "rounding_mode.h"
 #include "src/__support/CPP/bit.h"
 #include "src/__support/CPP/type_traits.h"
@@ -133,8 +134,18 @@ LIBC_INLINE T hypot(T x, T y) {
   uint16_t a_exp = a_bits.get_biased_exponent();
   uint16_t b_exp = b_bits.get_biased_exponent();
 
-  if ((a_exp - b_exp >= FPBits_t::FRACTION_LEN + 2) || (x == 0) || (y == 0))
-    return x_abs.get_val() + y_abs.get_val();
+  if ((a_exp - b_exp >= FPBits_t::FRACTION_LEN + 2) || (x == 0) || (y == 0)) {
+#ifdef LIBC_TYPES_HAS_FLOAT16
+    if constexpr (cpp::is_same_v<T, float16>) {
+      // Compiler runtime for basic operations of float16 might not be correctly
+      // rounded for all rounding modes.
+      float af = fputil::cast<float>(x_abs.get_val());
+      float bf = fputil::cast<float>(y_abs.get_val());
+      return fputil::cast<float16>(af + bf);
+    } else
+#endif // LIBC_TYPES_HAS_FLOAT16
+      return x_abs.get_val() + y_abs.get_val();
+  }
 
   uint64_t out_exp = a_exp;
   StorageType a_mant = a_bits.get_mantissa();
diff --git a/libc/src/__support/FPUtil/bfloat16.h b/libc/src/__support/FPUtil/bfloat16.h
index fa45d73fba6c1..b607fd68e838b 100644
--- a/libc/src/__support/FPUtil/bfloat16.h
+++ b/libc/src/__support/FPUtil/bfloat16.h
@@ -15,6 +15,9 @@
 #include "src/__support/FPUtil/cast.h"
 #include "src/__support/FPUtil/comparison_operations.h"
 #include "src/__support/FPUtil/dyadic_float.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/FPUtil/generic/div.h"
+#include "src/__support/FPUtil/generic/mul.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/types.h"
 
@@ -26,8 +29,6 @@ struct BFloat16 {
 
   LIBC_INLINE BFloat16() = default;
 
-  LIBC_INLINE constexpr explicit BFloat16(uint16_t bits) : bits(bits) {}
-
   template <typename T> LIBC_INLINE constexpr explicit BFloat16(T value) {
     if constexpr (cpp::is_floating_point_v<T>) {
       bits = fputil::cast<bfloat16>(value).bits;
@@ -45,6 +46,8 @@ struct BFloat16 {
           xd(sign, 0, value);
       bits = xd.template as<bfloat16, /*ShouldSignalExceptions=*/true>().bits;
 
+    } else if constexpr (cpp::is_convertible_v<T, BFloat16>) {
+      bits = value.operator BFloat16().bits;
     } else {
       bits = fputil::cast<bfloat16>(static_cast<float>(value)).bits;
     }
@@ -58,6 +61,11 @@ struct BFloat16 {
     return cpp::bit_cast<float>(x_bits);
   }
 
+  template <typename T, cpp::enable_if_t<cpp::is_integral_v<T>, int> = 0>
+  LIBC_INLINE constexpr explicit operator T() const {
+    return static_cast<T>(static_cast<float>(*this));
+  }
+
   LIBC_INLINE bool operator==(BFloat16 other) const {
     return fputil::equals(*this, other);
   }
@@ -81,6 +89,28 @@ struct BFloat16 {
   LIBC_INLINE bool operator>=(BFloat16 other) const {
     return fputil::greater_than_or_equals(*this, other);
   }
+
+  LIBC_INLINE constexpr BFloat16 operator-() const {
+    fputil::FPBits<bfloat16> result(*this);
+    result.set_sign(result.is_pos() ? Sign::NEG : Sign::POS);
+    return result.get_val();
+  }
+
+  LIBC_INLINE BFloat16 operator+(BFloat16 other) const {
+    return fputil::generic::add<BFloat16>(*this, other);
+  }
+
+  LIBC_INLINE BFloat16 operator-(BFloat16 other) const {
+    return fputil::generic::sub<BFloat16>(*this, other);
+  }
+
+  LIBC_INLINE BFloat16 operator*(BFloat16 other) const {
+    return fputil::generic::mul<bfloat16>(*this, other);
+  }
+
+  LIBC_INLINE BFloat16 operator/(BFloat16 other) const {
+    return fputil::generic::div<bfloat16>(*this, other);
+  }
 }; // struct BFloat16
 
 } // namespace fputil
diff --git a/libc/src/__support/FPUtil/cast.h b/libc/src/__support/FPUtil/cast.h
index e999ece37871a..54c80e862523a 100644
--- a/libc/src/__support/FPUtil/cast.h
+++ b/libc/src/__support/FPUtil/cast.h
@@ -27,47 +27,47 @@ LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_floating_point_v<OutType> &&
                                        OutType>
 cast(InType x) {
   // Casting to the same type is a no-op.
-  if constexpr (cpp::is_same_v<InType, OutType>)
+  if constexpr (cpp::is_same_v<InType, OutType>) {
     return x;
-
-  // bfloat16 is always defined (for now)
-  if constexpr (cpp::is_same_v<OutType, bfloat16> ||
-                cpp::is_same_v<InType, bfloat16>
+  } else {
+    if constexpr (cpp::is_same_v<OutType, bfloat16> ||
+                  cpp::is_same_v<InType, bfloat16>
 #if defined(LIBC_TYPES_HAS_FLOAT16) && !defined(__LIBC_USE_FLOAT16_CONVERSION)
-                || cpp::is_same_v<OutType, float16> ||
-                cpp::is_same_v<InType, float16>
+                  || cpp::is_same_v<OutType, float16> ||
+                  cpp::is_same_v<InType, float16>
 #endif
-  ) {
-    using InFPBits = FPBits<InType>;
-    using InStorageType = typename InFPBits::StorageType;
-    using OutFPBits = FPBits<OutType>;
-    using OutStorageType = typename OutFPBits::StorageType;
+    ) {
+      using InFPBits = FPBits<InType>;
+      using InStorageType = typename InFPBits::StorageType;
+      using OutFPBits = FPBits<OutType>;
+      using OutStorageType = typename OutFPBits::StorageType;
 
-    InFPBits x_bits(x);
+      InFPBits x_bits(x);
 
-    if (x_bits.is_nan()) {
-      if (x_bits.is_signaling_nan()) {
-        raise_except_if_required(FE_INVALID);
-        return OutFPBits::quiet_nan().get_val();
-      }
+      if (x_bits.is_nan()) {
+        if (x_bits.is_signaling_nan()) {
+          raise_except_if_required(FE_INVALID);
+          return OutFPBits::quiet_nan().get_val();
+        }
 
-      InStorageType x_mant = x_bits.get_mantissa();
-      if (InFPBits::FRACTION_LEN > OutFPBits::FRACTION_LEN)
-        x_mant >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
-      return OutFPBits::quiet_nan(x_bits.sign(),
-                                  static_cast<OutStorageType>(x_mant))
-          .get_val();
-    }
+        InStorageType x_mant = x_bits.get_mantissa();
+        if (InFPBits::FRACTION_LEN > OutFPBits::FRACTION_LEN)
+          x_mant >>= InFPBits::FRACTION_LEN - OutFPBits::FRACTION_LEN;
+        return OutFPBits::quiet_nan(x_bits.sign(),
+                                    static_cast<OutStorageType>(x_mant))
+            .get_val();
+      }
 
-    if (x_bits.is_inf())
-      return OutFPBits::inf(x_bits.sign()).get_val();
+      if (x_bits.is_inf())
+        return OutFPBits::inf(x_bits.sign()).get_val();
 
-    constexpr size_t MAX_FRACTION_LEN =
-        cpp::max(OutFPBits::FRACTION_LEN, InFPBits::FRACTION_LEN);
-    DyadicFloat<cpp::bit_ceil(MAX_FRACTION_LEN)> xd(x);
-    return xd.template as<OutType, /*ShouldSignalExceptions=*/true>();
-  } else {
-    return static_cast<OutType>(x);
+      constexpr size_t MAX_FRACTION_LEN =
+          cpp::max(OutFPBits::FRACTION_LEN, InFPBits::FRACTION_LEN);
+      DyadicFloat<cpp::bit_ceil(MAX_FRACTION_LEN)> xd(x);
+      return xd.template as<OutType, /*ShouldSignalExceptions=*/true>();
+    } else {
+      return static_cast<OutType>(x);
+    }
   }
 }
 
diff --git a/libc/src/__support/FPUtil/dyadic_float.h b/libc/src/__support/FPUtil/dyadic_float.h
index 3464e4aa9423f..cc0710fbf7b02 100644
--- a/libc/src/__support/FPUtil/dyadic_float.h
+++ b/libc/src/__support/FPUtil/dyadic_float.h
@@ -576,7 +576,7 @@ LIBC_INLINE constexpr DyadicFloat<Bits> quick_mul(const DyadicFloat<Bits> &a,
     // Check the leading bit directly, should be faster than using clz in
     // normalize().
     if (result.mantissa.val[DyadicFloat<Bits>::MantissaType::WORD_COUNT - 1] >>
-            63 ==
+            (DyadicFloat<Bits>::MantissaType::WORD_SIZE - 1) ==
         0)
       result.shift_left(1);
   } else {
diff --git a/libc/src/__support/FPUtil/generic/CMakeLists.txt b/libc/src/__support/FPUtil/generic/CMakeLists.txt
index 117213fc2c59f..b75efc8eb2fe4 100644
--- a/libc/src/__support/FPUtil/generic/CMakeLists.txt
+++ b/libc/src/__support/FPUtil/generic/CMakeLists.txt
@@ -68,6 +68,7 @@ add_header_library(
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.attributes
     libc.src.__support.macros.optimization
+    libc.src.__support.macros.properties.types
 )
 
 add_header_library(
@@ -77,6 +78,7 @@ add_header_library(
   DEPENDS
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
+    libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.bit
     libc.src.__support.CPP.type_traits
     libc.src.__support.FPUtil.basic_operations
diff --git a/libc/src/__support/FPUtil/generic/add_sub.h b/libc/src/__support/FPUtil/generic/add_sub.h
index d4a4129496640..b2e9d81f4774c 100644
--- a/libc/src/__support/FPUtil/generic/add_sub.h
+++ b/libc/src/__support/FPUtil/generic/add_sub.h
@@ -104,13 +104,22 @@ add_or_sub(InType x, InType y) {
         }
       }
 
-      // volatile prevents Clang from converting tmp to OutType and then
-      // immediately back to InType before negating it, resulting in double
-      // rounding.
-      volatile InType tmp = y;
-      if constexpr (IsSub)
-        tmp = -tmp;
-      return cast<OutType>(tmp);
+      if constexpr (cpp::is_same_v<InType, bfloat16> &&
+                    cpp::is_same_v<OutType, bfloat16>) {
+        OutFPBits y_bits(y);
+        if constexpr (IsSub)
+          y_bits.set_sign(y_bits.sign().negate());
+        return y_bits.get_val();
+      } else {
+
+        // volatile prevents Clang from converting tmp to OutType and then
+        // immediately back to InType before negating it, resulting in double
+        // rounding.
+        volatile InType tmp = y;
+        if constexpr (IsSub)
+          tmp = -tmp;
+        return cast<OutType>(tmp);
+      }
     }
 
     if (y_bits.is_zero())
@@ -165,8 +174,8 @@ add_or_sub(InType x, InType y) {
     int alignment = (max_bits.get_biased_exponent() - max_bits.is_normal()) -
                     (min_bits.get_biased_exponent() - min_bits.is_normal());
 
-    InStorageType aligned_min_mant =
-        min_mant >> cpp::min(alignment, RESULT_MANTISSA_LEN);
+    InStorageType aligned_min_mant = static_cast<InStorageType>(
+        min_mant >> cpp::min(alignment, RESULT_MANTISSA_LEN));
     bool aligned_min_mant_sticky;
 
     if (alignment <= GUARD_BITS_LEN)
diff --git a/libc/src/__support/FPUtil/generic/div.h b/libc/src/__support/FPUtil/generic/div.h
index 0891ae010ce20..bf7d0b7112ca9 100644
--- a/libc/src/__support/FPUtil/generic/div.h
+++ b/libc/src/__support/FPUtil/generic/div.h
@@ -11,6 +11,7 @@
 
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
+#include "src/__support/CPP/algorithm.h"
 #include "src/__support/CPP/bit.h"
 #include "src/__support/CPP/type_traits.h"
 #include "src/__support/FPUtil/BasicOperations.h"
@@ -34,8 +35,9 @@ div(InType x, InType y) {
   using OutStorageType = typename OutFPBits::StorageType;
   using InFPBits = FPBits<InType>;
   using InStorageType = typename InFPBits::StorageType;
-  using DyadicFloat =
-      DyadicFloat<cpp::bit_ceil(static_cast<size_t>(InFPBits::SIG_LEN + 1))>;
+  using DyadicFloat = DyadicFloat<cpp::max(
+      static_cast<size_t>(16),
+      cpp::bit_ceil(static_cast<size_t>(InFPBits::SIG_LEN + 1)))>;
 
   InFPBits x_bits(x);
   InFPBits y_bits(y);
diff --git a/libc/src/__support/FPUtil/rounding_mode.h b/libc/src/__support/FPUtil/rounding_mode.h
index 4ee0a0b0490fc..fdc84986a4781 100644
--- a/libc/src/__support/FPUtil/rounding_mode.h
+++ b/libc/src/__support/FPUtil/rounding_mode.h
@@ -17,30 +17,24 @@
 namespace LIBC_NAMESPACE_DECL {
 namespace fputil {
 
+namespace generic {
+
 // Quick free-standing test whether fegetround() == FE_UPWARD.
 // Using the following observation:
 //   1.0f + 2^-25 = 1.0f        for FE_TONEAREST, FE_DOWNWARD, FE_TOWARDZERO
 //                = 0x1.000002f for FE_UPWARD.
-LIBC_INLINE static constexpr bool fenv_is_round_up() {
-  if (cpp::is_constant_evaluated()) {
-    return false;
-  } else {
-    volatile float x = 0x1.0p-25f;
-    return (1.0f + x != 1.0f);
-  }
+LIBC_INLINE bool fenv_is_round_up() {
+  static volatile float x = 0x1.0p-25f;
+  return (1.0f + x != 1.0f);
 }
 
 // Quick free-standing test whether fegetround() == FE_DOWNWARD.
 // Using the following observation:
 //   -1.0f - 2^-25 = -1.0f        for FE_TONEAREST, FE_UPWARD, FE_TOWARDZERO
 //                 = -0x1.000002f for FE_DOWNWARD.
-LIBC_INLINE static constexpr bool fenv_is_round_down() {
-  if (cpp::is_constant_evaluated()) {
-    return false;
-  } else {
-    volatile float x = 0x1.0p-25f;
-    return (-1.0f - x != -1.0f);
-  }
+LIBC_INLINE bool fenv_is_round_down() {
+  static volatile float x = 0x1.0p-25f;
+  return (-1.0f - x != -1.0f);
 }
 
 // Quick free-standing test whether fegetround() == FE_TONEAREST.
@@ -49,14 +43,10 @@ LIBC_INLINE static constexpr bool fenv_is_round_down() {
 //                = 0x1.100002p0f  for FE_UPWARD,
 //   1.5f - 2^-24 = 1.5f           for FE_TONEAREST, FE_UPWARD
 //                = 0x1.0ffffep-1f for FE_DOWNWARD, FE_TOWARDZERO
-LIBC_INLINE static constexpr bool fenv_is_round_to_nearest() {
-  if (cpp::is_constant_evaluated()) {
-    return true;
-  } else {
-    volatile float x = 0x1.0p-24f;
-    float y = 1.5f + x;
-    return (y == 1.5f - x);
-  }
+LIBC_INLINE bool fenv_is_round_to_nearest() {
+  static volatile float x = 0x1.0p-24f;
+  float y = 1.5f + x;
+  return (y == 1.5f - x);
 }
 
 // Quick free-standing test whether fegetround() == FE_TOWARDZERO.
@@ -69,13 +59,56 @@ LIBC_INLINE static constexpr bool fenv_is_round_to_nearest() {
 // (0x1.000002p0f + 2^-24) + (-1.0f - 2^-24) = 2^-23 for FE_TOWARDZERO
 //                                           = 2^-22 for FE_TONEAREST, FE_UPWARD
 //                                           = 0 for FE_DOWNWARD
+LIBC_INLINE bool fenv_is_round_to_zero() {
+  static volatile float x = 0x1.0p-24f;
+  float y = x;
+  return ((0x1.000002p0f + y) + (-1.0f - y) == 0x1.0p-23f);
+}
+
+// Quick free standing get rounding mode based on the above observations.
+LIBC_INLINE int quick_get_round() {
+  static volatile float x = 0x1.0p-24f;
+  float y = x;
+  float z = (0x1.000002p0f + y) + (-1.0f - y);
+
+  if (z == 0.0f)
+    return FE_DOWNWARD;
+  if (z == 0x1.0p-23f)
+    return FE_TOWARDZERO;
+  return (2.0f + y == 2.0f) ? FE_TONEAREST : FE_UPWARD;
+}
+
+} // namespace generic
+
+LIBC_INLINE static constexpr bool fenv_is_round_up() {
+  if (cpp::is_constant_evaluated()) {
+    return false;
+  } else {
+    return generic::fenv_is_round_up();
+  }
+}
+
+LIBC_INLINE static constexpr bool fenv_is_round_down() {
+  if (cpp::is_constant_evaluated()) {
+    return false;
+  } else {
+    return generic::fenv_is_round_down();
+  }
+}
+
+LIBC_INLINE static constexpr bool fenv_is_round_to_nearest() {
+  if (cpp::is_constant_evaluated()) {
+    return true;
+  } else {
+    return generic::fenv_is_round_to_nearest();
+  }
+}
+
 LIBC_INLINE static constexpr bool fenv_is_round_to_zero() {
   if (cpp::is_constant_evaluated()) {
     return false;
   } else {
-    volatile float x = 0x1.0p-24f;
-    volatile float y = 0x1.000002p0f + x;
-    return (y + (-1.0f - x) == 0x1.0p-23f);
+    return generic::fenv_is_round_to_zero();
   }
 }
 
@@ -84,15 +117,7 @@ LIBC_INLINE static constexpr int quick_get_round() {
   if (cpp::is_constant_evaluated()) {
     return FE_TONEAREST;
   } else {
-    volatile float x = 0x1.0p-24f;
-    volatile float y = 0x1.000002p0f + x;
-    float z = y + (-1.0f - x);
-
-    if (z == 0.0f)
-      return FE_DOWNWARD;
-    if (z == 0x1.0p-23f)
-      return FE_TOWARDZERO;
-    return (2.0f + x == 2.0f) ? FE_TONEAREST : FE_UPWARD;
+    return generic::quick_get_round();
   }
 }
 
diff --git a/libc/src/__support/File/file.cpp b/libc/src/__support/File/file.cpp
index 303852dbbb717..4217e73828388 100644
--- a/libc/src/__support/File/file.cpp
+++ b/libc/src/__support/File/file.cpp
@@ -123,7 +123,7 @@ FileIOResult File::write_unlocked_fbf(const uint8_t *data, size_t len) {
 
     FileIOResult result =
         platform_write(this, remainder.data(), remainder.size());
-    size_t bytes_written = buf_result.value;
+    size_t bytes_written = result.value;
 
     // If less bytes were written than expected, then an error occurred. Return
     // the number of bytes that have been written from |data|.
diff --git a/libc/src/__support/GPU/CMakeLists.txt b/libc/src/__support/GPU/CMakeLists.txt
index f8fdfeb9da9df..72a7879f72224 100644
--- a/libc/src/__support/GPU/CMakeLists.txt
+++ b/libc/src/__support/GPU/CMakeLists.txt
@@ -9,6 +9,12 @@ add_header_library(
     utils.h
 )
 
+add_header_library(
+  fixedstack
+  HDRS
+    fixedstack.h
+)
+
 add_object_library(
   allocator
   SRCS
@@ -23,4 +29,5 @@ add_object_library(
     libc.src.__support.CPP.bit
     libc.src.__support.CPP.new
     .utils
+    .fixedstack
 )
diff --git a/libc/src/__support/GPU/allocator.cpp b/libc/src/__support/GPU/allocator.cpp
index 250bebdbb7d81..3da339c63ccd0 100644
--- a/libc/src/__support/GPU/allocator.cpp
+++ b/libc/src/__support/GPU/allocator.cpp
@@ -20,6 +20,7 @@
 #include "src/__support/CPP/atomic.h"
 #include "src/__support/CPP/bit.h"
 #include "src/__support/CPP/new.h"
+#include "src/__support/GPU/fixedstack.h"
 #include "src/__support/GPU/utils.h"
 #include "src/__support/RPC/rpc_client.h"
 #include "src/__support/threads/sleep.h"
@@ -39,6 +40,9 @@ constexpr static uint32_t MIN_ALIGNMENT = MIN_SIZE - 1;
 // The number of times to attempt claiming an in-progress slab allocation.
 constexpr static uint32_t MAX_TRIES = 1024;
 
+// The number of previously allocated slabs we will keep in memory.
+constexpr static uint32_t CACHED_SLABS = 8;
+
 static_assert(!(ARRAY_SIZE & (ARRAY_SIZE - 1)), "Must be a power of two");
 
 namespace impl {
@@ -162,7 +166,11 @@ static inline uint32_t get_leader_id(uint64_t ballot, uint32_t id) {
 
 // We use a sentinal value to indicate a failed or in-progress allocation.
 template <typename T> bool is_sentinel(const T &x) {
-  return x == cpp::numeric_limits<T>::max();
+  if constexpr (cpp::is_pointer_v<T>)
+    return reinterpret_cast<uintptr_t>(x) ==
+           cpp::numeric_limits<uintptr_t>::max();
+  else
+    return x == cpp::numeric_limits<T>::max();
 }
 
 } // namespace impl
@@ -185,20 +193,35 @@ struct Slab {
   struct alignas(MIN_SIZE) Header {
     uint32_t chunk_size;
     uint32_t global_index;
+    uint32_t cached_chunk_size;
   };
 
   // Initialize the slab with its chunk size and index in the global table for
   // use when freeing.
   Slab(uint32_t chunk_size, uint32_t global_index) {
     Header *header = reinterpret_cast<Header *>(memory);
+    header->cached_chunk_size = cpp::numeric_limits<uint32_t>::max();
+    header->chunk_size = chunk_size;
+    header->global_index = global_index;
+  }
+
+  // Reset the memory with a new index and chunk size, not thread safe.
+  Slab *reset(uint32_t chunk_size, uint32_t global_index) {
+    Header *header = reinterpret_cast<Header *>(memory);
+    header->cached_chunk_size = header->chunk_size;
     header->chunk_size = chunk_size;
     header->global_index = global_index;
+    return this;
   }
 
   // Set the necessary bitfield bytes to zero in parallel using many lanes. This
   // must be called before the bitfield can be accessed safely, memory is not
   // guaranteed to be zero initialized in the current implementation.
   void initialize(uint64_t uniform) {
+    // If this is a re-used slab the memory is already set to zero.
+    if (get_cached_chunk_size() <= get_chunk_size())
+      return;
+
     uint32_t size = (bitfield_bytes(get_chunk_size()) + sizeof(uint32_t) - 1) /
                     sizeof(uint32_t);
     impl::uniform_memset(get_bitfield(), 0, size, uniform);
@@ -236,6 +259,11 @@ struct Slab {
     return reinterpret_cast<const Header *>(memory)->chunk_size;
   }
 
+  // Get the chunk size that was previously used.
+  uint32_t get_cached_chunk_size() const {
+    return reinterpret_cast<const Header *>(memory)->cached_chunk_size;
+  }
+
   // Get the location in the memory where we will store the global index.
   uint32_t get_global_index() const {
     return reinterpret_cast<const Header *>(memory)->global_index;
@@ -337,6 +365,9 @@ struct Slab {
   uint8_t memory[SLAB_SIZE];
 };
 
+// A global cache of previously allocated slabs for efficient reuse.
+static FixedStack<Slab *, CACHED_SLABS> slab_cache;
+
 /// A wait-free guard around a pointer resource to be created dynamically if
 /// space is available and freed once there are no more users.
 struct GuardPtr {
@@ -408,13 +439,24 @@ struct GuardPtr {
             reinterpret_cast<Slab *>(cpp::numeric_limits<uintptr_t>::max()),
             cpp::MemoryOrder::RELAXED, cpp::MemoryOrder::RELAXED)) {
       count = cpp::numeric_limits<uint32_t>::max();
+
+      Slab *cached = nullptr;
+      if (slab_cache.pop(cached))
+        return cached->reset(cpp::forward<Args>(args)...);
+
       void *raw = impl::rpc_allocate(sizeof(Slab));
       if (!raw)
         return nullptr;
       return new (raw) Slab(cpp::forward<Args>(args)...);
     }
 
-    if (!expected || impl::is_sentinel(reinterpret_cast<uintptr_t>(expected)))
+    // If there is a slab allocation in progress we retry a few times.
+    for (uint32_t t = 0; impl::is_sentinel(expected) && t < MAX_TRIES; ++t) {
+      sleep_briefly();
+      expected = ptr.load(cpp::MemoryOrder::RELAXED);
+    }
+
+    if (!expected || impl::is_sentinel(expected))
       return nullptr;
 
     if (!ref.acquire(n, count))
@@ -475,8 +517,10 @@ struct GuardPtr {
     if (gpu::get_lane_id() == uint32_t(cpp::countr_zero(mask)) &&
         ref.release(cpp::popcount(mask))) {
       Slab *p = ptr.load(cpp::MemoryOrder::RELAXED);
-      p->~Slab();
-      impl::rpc_free(p);
+      if (!slab_cache.push(p)) {
+        p->~Slab();
+        impl::rpc_free(p);
+      }
       cpp::atomic_thread_fence(cpp::MemoryOrder::RELEASE);
       ptr.store(nullptr, cpp::MemoryOrder::RELAXED);
     }
@@ -523,16 +567,6 @@ static Slab *find_slab(uint32_t chunk_size, uint64_t &uniform,
       Slab *slab = slots[index].try_lock(lane_mask, uniform & lane_mask,
                                          reserved, chunk_size, index);
 
-      // If there is a slab allocation in progress we retry a few times.
-      for (uint32_t retries = 0;
-           !slab && !impl::is_sentinel(reserved) && retries < MAX_TRIES;
-           retries++) {
-        uint64_t lane_mask = gpu::get_lane_mask();
-        slab = slots[index].try_lock(lane_mask, uniform & lane_mask, reserved,
-                                     chunk_size, index);
-        sleep_briefly();
-      }
-
       // If we find a slab with a matching chunk size then we store the result.
       // Otherwise, we need to free the claimed lock and continue. In the case
       // of out-of-memory we receive a sentinel value and return a failure.
diff --git a/libc/src/__support/GPU/fixedstack.h b/libc/src/__support/GPU/fixedstack.h
new file mode 100644
index 0000000000000..6ceaa2fc73355
--- /dev/null
+++ b/libc/src/__support/GPU/fixedstack.h
@@ -0,0 +1,111 @@
+//===-- A lock-free data structure for a fixed capacity stack ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_GPU_FIXEDSTACK_H
+#define LLVM_LIBC_SRC___SUPPORT_GPU_FIXEDSTACK_H
+
+#include "src/__support/CPP/atomic.h"
+#include "src/__support/threads/sleep.h"
+
+#include <stdint.h>
+
+namespace LIBC_NAMESPACE_DECL {
+
+// A lock-free fixed size stack backed by an underlying array of data. It
+// supports push and pop operations in a completely lock-free manner.
+template <typename T, uint32_t CAPACITY> struct alignas(16) FixedStack {
+  // The index is stored as a 20-bit value and cannot index into any more.
+  static_assert(CAPACITY < 1024 * 1024, "Invalid buffer size");
+
+  // The head of the free and used stacks. Represents as a 20-bit index combined
+  // with a 44-bit ABA tag that is updated in a single atomic operation.
+  uint64_t free;
+  uint64_t used;
+
+  // The stack is a linked list of indices into the underlying data
+  uint32_t next[CAPACITY];
+  T data[CAPACITY];
+
+  // Get the 20-bit index into the underlying array from the head.
+  LIBC_INLINE static constexpr uint32_t get_node(uint64_t head) {
+    return static_cast<uint32_t>(head & 0xfffff);
+  }
+
+  // Increment the old ABA tag and merge it into the new index.
+  LIBC_INLINE static constexpr uint64_t make_head(uint64_t orig,
+                                                  uint32_t node) {
+    return static_cast<uint64_t>(node) | (((orig >> 20ul) + 1ul) << 20ul);
+  }
+
+  // Attempts to pop data from the given stack by making it point to the next
+  // node. We repeatedly attempt to write to the head using compare-and-swap,
+  // expecting that it has not been changed by any other thread.
+  LIBC_INLINE uint32_t pop_impl(cpp::AtomicRef<uint64_t> head) {
+    uint64_t orig = head.load(cpp::MemoryOrder::RELAXED);
+
+    for (;;) {
+      if (get_node(orig) == CAPACITY)
+        return CAPACITY;
+
+      uint32_t node =
+          cpp::AtomicRef(next[get_node(orig)]).load(cpp::MemoryOrder::RELAXED);
+      if (head.compare_exchange_strong(orig, make_head(orig, node),
+                                       cpp::MemoryOrder::ACQUIRE,
+                                       cpp::MemoryOrder::RELAXED))
+        break;
+    }
+    return get_node(orig);
+  }
+
+  // Attempts to push data to the given stack by making it point to the new
+  // node. We repeatedly attempt to write to the head using compare-and-swap,
+  // expecting that it has not been changed by any other thread.
+  LIBC_INLINE uint32_t push_impl(cpp::AtomicRef<uint64_t> head, uint32_t node) {
+    uint64_t orig = head.load(cpp::MemoryOrder::RELAXED);
+    for (;;) {
+      next[node] = get_node(orig);
+      if (head.compare_exchange_strong(orig, make_head(orig, node),
+                                       cpp::MemoryOrder::RELEASE,
+                                       cpp::MemoryOrder::RELAXED))
+        break;
+    }
+    return get_node(head.load(cpp::MemoryOrder::RELAXED));
+  }
+
+public:
+  // Initialize the free stack to be full and the used stack to be empty. We use
+  // the capacity of the stack as a sentinel value.
+  LIBC_INLINE constexpr FixedStack() : free(0), used(CAPACITY), data{} {
+    for (uint32_t i = 0; i < CAPACITY; ++i)
+      next[i] = i + 1;
+  }
+
+  LIBC_INLINE bool push(const T &val) {
+    uint32_t node = pop_impl(cpp::AtomicRef(free));
+    if (node == CAPACITY)
+      return false;
+
+    data[node] = val;
+    push_impl(cpp::AtomicRef(used), node);
+    return true;
+  }
+
+  LIBC_INLINE bool pop(T &val) {
+    uint32_t node = pop_impl(cpp::AtomicRef(used));
+    if (node == CAPACITY)
+      return false;
+
+    val = data[node];
+    push_impl(cpp::AtomicRef(free), node);
+    return true;
+  }
+};
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_GPU_FIXEDSTACK_H
diff --git a/libc/src/__support/OSUtil/linux/aarch64/vdso.h b/libc/src/__support/OSUtil/linux/aarch64/vdso.h
index 3c4c6205071da..ee5777ad67f6d 100644
--- a/libc/src/__support/OSUtil/linux/aarch64/vdso.h
+++ b/libc/src/__support/OSUtil/linux/aarch64/vdso.h
@@ -23,6 +23,8 @@ LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) {
     return "__kernel_clock_gettime";
   case VDSOSym::ClockGetRes:
     return "__kernel_clock_getres";
+  case VDSOSym::GetRandom:
+    return "__kernel_getrandom";
   default:
     return "";
   }
diff --git a/libc/src/__support/OSUtil/linux/vdso_sym.h b/libc/src/__support/OSUtil/linux/vdso_sym.h
index 968e1536c4d27..01f0b72a4ed9c 100644
--- a/libc/src/__support/OSUtil/linux/vdso_sym.h
+++ b/libc/src/__support/OSUtil/linux/vdso_sym.h
@@ -35,7 +35,8 @@ enum class VDSOSym {
   RTSigReturn,
   FlushICache,
   RiscvHwProbe,
-  VDSOSymCount
+  GetRandom,
+  VDSOSymCount,
 };
 
 template <VDSOSym sym> LIBC_INLINE constexpr auto dispatcher() {
@@ -60,6 +61,9 @@ template <VDSOSym sym> LIBC_INLINE constexpr auto dispatcher() {
   else if constexpr (sym == VDSOSym::RiscvHwProbe)
     return static_cast<int (*)(riscv_hwprobe *, size_t, size_t, cpu_set_t *,
                                unsigned)>(nullptr);
+  else if constexpr (sym == VDSOSym::GetRandom)
+    return static_cast<int (*)(void *, size_t, unsigned int, void *, size_t)>(
+        nullptr);
   else
     return static_cast<void *>(nullptr);
 }
diff --git a/libc/src/__support/OSUtil/linux/x86_64/vdso.h b/libc/src/__support/OSUtil/linux/x86_64/vdso.h
index abe7c33e07cfa..f46fcb038f2e6 100644
--- a/libc/src/__support/OSUtil/linux/x86_64/vdso.h
+++ b/libc/src/__support/OSUtil/linux/x86_64/vdso.h
@@ -29,6 +29,8 @@ LIBC_INLINE constexpr cpp::string_view symbol_name(VDSOSym sym) {
     return "__vdso_time";
   case VDSOSym::ClockGetRes:
     return "__vdso_clock_getres";
+  case VDSOSym::GetRandom:
+    return "__vdso_getrandom";
   default:
     return "";
   }
diff --git a/libc/src/__support/libc_errno.h b/libc/src/__support/libc_errno.h
index ab5f6a9c4b9d9..3720cdebd5d2a 100644
--- a/libc/src/__support/libc_errno.h
+++ b/libc/src/__support/libc_errno.h
@@ -37,18 +37,11 @@
 // libc doesn't maintain any internal state, instead the embedder must define
 // `int *__llvm_libc_errno(void);` C function.
 #define LIBC_ERRNO_MODE_EXTERNAL 4
-// libc uses system `<errno.h>` `errno` macro directly in the overlay mode; in
-// fullbuild mode, effectively the same as `LIBC_ERRNO_MODE_EXTERNAL`.
-// In this mode, the public C++ symbol `LIBC_NAMESPACE::libc_errno ` is still
-// exported and get redirected to the system `errno` inside its implementation.
-
-// TODO: Investigate deprecating LIBC_ERRNO_MODE_SYSTEM in favor of
-//       LIBC_ERRNO_MODE_SYSTEM_INLINE.
-//       https://github.com/llvm/llvm-project/issues/143454
-#define LIBC_ERRNO_MODE_SYSTEM 5
+// DEPRECATED: #define LIBC_ERRNO_MODE_SYSTEM 5
 // In this mode, the libc_errno is simply a macro resolved to `errno` from the
 // system header <errno.h>.  There is no need to link against the
-// `libc.src.errno.errno` object.
+// `libc.src.errno.errno` object, and public C++ symbol
+// `LIBC_NAMESPACE::libc_errno` doesn't exist.
 #define LIBC_ERRNO_MODE_SYSTEM_INLINE 6
 
 #if !defined(LIBC_ERRNO_MODE) || LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_DEFAULT
@@ -56,7 +49,7 @@
 #if defined(LIBC_FULL_BUILD) || !defined(LIBC_COPT_PUBLIC_PACKAGING)
 #define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_THREAD_LOCAL
 #else
-#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_SYSTEM
+#define LIBC_ERRNO_MODE LIBC_ERRNO_MODE_SYSTEM_INLINE
 #endif
 #endif // LIBC_ERRNO_MODE
 
@@ -65,7 +58,6 @@
     LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_THREAD_LOCAL &&                         \
     LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SHARED &&                               \
     LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_EXTERNAL &&                             \
-    LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM &&                               \
     LIBC_ERRNO_MODE != LIBC_ERRNO_MODE_SYSTEM_INLINE
 #error LIBC_ERRNO_MODE must be one of the following values: \
 LIBC_ERRNO_MODE_DEFAULT, \
@@ -73,7 +65,6 @@ LIBC_ERRNO_MODE_UNDEFINED, \
 LIBC_ERRNO_MODE_THREAD_LOCAL, \
 LIBC_ERRNO_MODE_SHARED, \
 LIBC_ERRNO_MODE_EXTERNAL, \
-LIBC_ERRNO_MODE_SYSTEM, \
 LIBC_ERRNO_MODE_SYSTEM_INLINE.
 #endif
 
diff --git a/libc/src/__support/macros/attributes.h b/libc/src/__support/macros/attributes.h
index c6474673de85a..4ff374b0e4fbd 100644
--- a/libc/src/__support/macros/attributes.h
+++ b/libc/src/__support/macros/attributes.h
@@ -28,7 +28,32 @@
 #define LIBC_INLINE_ASM __asm__ __volatile__
 #define LIBC_UNUSED __attribute__((unused))
 
-#ifdef LIBC_TARGET_ARCH_IS_GPU
+// Uses the platform specific specialization
+#define LIBC_THREAD_MODE_PLATFORM 0
+
+// Mutex guards nothing, used in single-threaded implementations
+#define LIBC_THREAD_MODE_SINGLE 1
+
+// Vendor provides implementation
+#define LIBC_THREAD_MODE_EXTERNAL 2
+
+// libcxx doesn't define LIBC_THREAD_MODE, unless that is passed in the command
+// line in the CMake invocation. This defaults to the original implementation
+// (before changes in https://github.com/llvm/llvm-project/pull/145358)
+#ifndef LIBC_THREAD_MODE
+#define LIBC_THREAD_MODE LIBC_THREAD_MODE_PLATFORM
+#endif // LIBC_THREAD_MODE
+
+#if LIBC_THREAD_MODE != LIBC_THREAD_MODE_PLATFORM &&                           \
+    LIBC_THREAD_MODE != LIBC_THREAD_MODE_SINGLE &&                             \
+    LIBC_THREAD_MODE != LIBC_THREAD_MODE_EXTERNAL
+#error LIBC_THREAD_MODE must be one of the following values: \
+LIBC_THREAD_MODE_PLATFORM, \
+LIBC_THREAD_MODE_SINGLE, \
+LIBC_THREAD_MODE_EXTERNAL.
+#endif
+
+#if LIBC_THREAD_MODE == LIBC_THREAD_MODE_SINGLE
 #define LIBC_THREAD_LOCAL
 #else
 #define LIBC_THREAD_LOCAL thread_local
diff --git a/libc/src/__support/math/CMakeLists.txt b/libc/src/__support/math/CMakeLists.txt
index e1076edf1e61c..39dc0e57f4472 100644
--- a/libc/src/__support/math/CMakeLists.txt
+++ b/libc/src/__support/math/CMakeLists.txt
@@ -346,6 +346,117 @@ add_header_library(
     libc.src.__support.integer_literals
 )
 
+add_header_library(
+  cbrtf
+  HDRS
+    cbrtf.h
+  DEPENDS
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.macros.optimization
+)
+
+add_header_library(
+  cos
+  HDRS
+    cos.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.src.errno.errno
+    libc.src.__support.FPUtil.double_double
+    libc.src.__support.FPUtil.dyadic_float
+    libc.src.__support.FPUtil.except_value_utils
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.math.range_reduction_double
+    libc.src.__support.math.sincos_eval
+    libc.src.__support.macros.optimization
+)
+
+add_header_library(
+  cosf
+  HDRS
+    cosf.h
+  DEPENDS
+    .sincosf_utils
+    libc.src.errno.errno
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.except_value_utils
+    libc.src.__support.FPUtil.fma
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.macros.optimization
+)
+
+add_header_library(
+  cosf16
+  HDRS
+    cosf16.h
+  DEPENDS
+    .sincosf16_utils
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.except_value_utils
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.macros.optimization
+    libc.src.__support.macros.properties.types
+)
+
+add_header_library(
+  coshf
+  HDRS
+    coshf.h
+  DEPENDS
+    .sinhfcoshf_utils
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.rounding_mode
+    libc.src.__support.macros.optimization
+)
+
+add_header_library(
+  coshf16
+  HDRS
+    coshf16.h
+  DEPENDS
+    .expxf16_utils
+    libc.src.__support.FPUtil.except_value_utils
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.rounding_mode
+    libc.src.__support.macros.optimization
+)
+
+add_header_library(
+  cospif
+  HDRS
+    cospif.h
+  DEPENDS
+    .sincosf_utils
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.macros.optimization
+)
+
+add_header_library(
+  cospif16
+  HDRS
+    cospif16.h
+  DEPENDS
+    .sincosf16_utils
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.macros.optimization
+)
+
 add_header_library(
   erff
   HDRS
@@ -416,6 +527,21 @@ add_header_library(
     libc.include.llvm-libc-macros.float16_macros
 )
 
+add_header_library(
+  expxf16_utils
+  HDRS
+    expxf16_utils.h
+  DEPENDS
+    libc.hdr.stdint_proxy
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.nearest_integer
+    libc.src.__support.macros.attributes
+    libc.src.__support.math.expf16_utils
+    libc.src.__support.math.exp10_float16_constants
+)
+
 add_header_library(
   frexpf128
   HDRS
@@ -602,3 +728,75 @@ add_header_library(
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
 )
+
+add_header_library(
+  range_reduction_double
+  HDRS
+    range_reduction_double_common.h
+    range_reduction_double_fma.h
+    range_reduction_double_nofma.h
+  DEPENDS
+    libc.src.__support.FPUtil.double_double
+    libc.src.__support.FPUtil.dyadic_float
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.fma
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.nearest_integer
+    libc.src.__support.common
+    libc.src.__support.integer_literals
+)
+
+add_header_library(
+  range_reduction
+  HDRS
+    range_reduction.h
+    range_reduction_fma.h
+  DEPENDS
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.fma
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.nearest_integer
+    libc.src.__support.common
+)
+
+add_header_library(
+  sincos_eval
+  HDRS
+    sincos_eval.h
+  DEPENDS
+    libc.src.__support.FPUtil.double_double
+    libc.src.__support.FPUtil.dyadic_float
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.integer_literals
+)
+
+add_header_library(
+  sincosf_utils
+  HDRS
+    sincosf_utils.h
+  DEPENDS
+    .range_reduction
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.common
+)
+
+add_header_library(
+  sincosf16_utils
+  HDRS
+    sincosf16_utils.h
+  DEPENDS
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.FPUtil.nearest_integer
+    libc.src.__support.common
+)
+
+add_header_library(
+  sinhfcoshf_utils
+  HDRS
+    sinhfcoshf_utils.h
+  DEPENDS
+    .exp10f_utils
+    libc.src.__support.FPUtil.multiply_add
+)
diff --git a/libc/src/__support/math/cbrtf.h b/libc/src/__support/math/cbrtf.h
new file mode 100644
index 0000000000000..f82892bbbe61b
--- /dev/null
+++ b/libc/src/__support/math/cbrtf.h
@@ -0,0 +1,161 @@
+//===-- Implementation header for cbrtf -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_SRC___SUPPORT_MATH_CBRTF_H
+#define LIBC_SRC___SUPPORT_MATH_CBRTF_H
+
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+LIBC_INLINE static constexpr float cbrtf(float x) {
+  // Look up table for 2^(i/3) for i = 0, 1, 2.
+  constexpr double CBRT2[3] = {1.0, 0x1.428a2f98d728bp0, 0x1.965fea53d6e3dp0};
+
+  // Degree-7 polynomials approximation of ((1 + x)^(1/3) - 1)/x for 0 <= x <= 1
+  // generated by Sollya with:
+  // > for i from 0 to 15 do {
+  //     P = fpminimax(((1 + x)^(1/3) - 1)/x, 6, [|D...|], [i/16, (i + 1)/16]);
+  //     print("{", coeff(P, 0), ",", coeff(P, 1), ",", coeff(P, 2), ",",
+  //           coeff(P, 3), ",", coeff(P, 4), ",", coeff(P, 5), ",",
+  //           coeff(P, 6), "},");
+  // };
+  // Then (1 + x)^(1/3) ~ 1 + x * P(x).
+  constexpr double COEFFS[16][7] = {
+      {0x1.55555555554ebp-2, -0x1.c71c71c678c0cp-4, 0x1.f9add2776de81p-5,
+       -0x1.511e10aa964a7p-5, 0x1.ee44165937fa2p-6, -0x1.7c5c9e059345dp-6,
+       0x1.047f75e0aff14p-6},
+      {0x1.5555554d1149ap-2, -0x1.c71c676fcb5bp-4, 0x1.f9ab127dc57ebp-5,
+       -0x1.50ea8fd1d4c15p-5, 0x1.e9d68f28ced43p-6, -0x1.60e0e1e661311p-6,
+       0x1.716eca1d6e3bcp-7},
+      {0x1.5555546377d45p-2, -0x1.c71bc1c6d49d2p-4, 0x1.f9924cc0ed24dp-5,
+       -0x1.4fea3beb53b3bp-5, 0x1.de028a9a07b1bp-6, -0x1.3b090d2233524p-6,
+       0x1.0aeca34893785p-7},
+      {0x1.55554dce9f649p-2, -0x1.c7188b34b98f8p-4, 0x1.f93e1af34af49p-5,
+       -0x1.4d9a06be75c63p-5, 0x1.cb943f4f68992p-6, -0x1.139a685a5e3c4p-6,
+       0x1.88410674c6a5dp-8},
+      {0x1.5555347d211c3p-2, -0x1.c70f2a4b1a5fap-4, 0x1.f88420e8602c3p-5,
+       -0x1.49becfa4ed3ep-5, 0x1.b475cd9013162p-6, -0x1.dcfee1dd2f8efp-7,
+       0x1.249bb51a1c498p-8},
+      {0x1.5554f01b33dbap-2, -0x1.c6facb929dbf1p-4, 0x1.f73fb7861252ep-5,
+       -0x1.4459a4a0071fap-5, 0x1.9a8df2b504fc2p-6, -0x1.9a7ce3006d06ep-7,
+       0x1.ba9230918fa2ep-9},
+      {0x1.55545c695db5fp-2, -0x1.c6d6089f20275p-4, 0x1.f556e0ea80efp-5,
+       -0x1.3d91372d083f4p-5, 0x1.7f66cff331f4p-6, -0x1.606a562491737p-7,
+       0x1.52e3e17c71069p-9},
+      {0x1.55534a879232ap-2, -0x1.c69b836998b84p-4, 0x1.f2bb26dac0e4cp-5,
+       -0x1.359eed43716d7p-5, 0x1.64218cd824fbcp-6, -0x1.2e703e2e091e8p-7,
+       0x1.0677d9af6aad4p-9},
+      {0x1.5551836bb5494p-2, -0x1.c64658c15353bp-4, 0x1.ef68517451a6ep-5,
+       -0x1.2cc20a980dceep-5, 0x1.49843e0fad93ap-6, -0x1.03c59ccb68e54p-7,
+       0x1.9ad325dc7adcbp-10},
+      {0x1.554ecacb0d035p-2, -0x1.c5d2664026ffcp-4, 0x1.eb624796ba809p-5,
+       -0x1.233803d19a535p-5, 0x1.300decb1c3c28p-6, -0x1.befe18031ec3dp-8,
+       0x1.449f5ee175c69p-10},
+      {0x1.554ae1f5ae815p-2, -0x1.c53c6b14ff6b2p-4, 0x1.e6b2d5127bb5bp-5,
+       -0x1.19387336788a3p-5, 0x1.180955a6ab255p-6, -0x1.81696703ba369p-8,
+       0x1.02cb36389bd79p-10},
+      {0x1.55458a59f356ep-2, -0x1.c4820dd631ae9p-4, 0x1.e167af818bd15p-5,
+       -0x1.0ef35f6f72e52p-5, 0x1.019c33b65e4ebp-6, -0x1.4d25bdd52d3a5p-8,
+       0x1.a008ae91f5936p-11},
+      {0x1.553e878eafee1p-2, -0x1.c3a1d0b2a3db2p-4, 0x1.db90d8ed9f89bp-5,
+       -0x1.0490e20f1ae91p-5, 0x1.d9a5d1fc42fe3p-7, -0x1.20bf8227c2abfp-8,
+       0x1.50f8174cdb6e9p-11},
+      {0x1.5535a0dedf1b1p-2, -0x1.c29afb8bd01a1p-4, 0x1.d53f6371c1e27p-5,
+       -0x1.f463209b433e2p-6, 0x1.b35222a17e44p-7, -0x1.f5efbf505e133p-9,
+       0x1.12e0e94e8586dp-11},
+      {0x1.552aa25e57bfdp-2, -0x1.c16d811e4acadp-4, 0x1.ce8489b47aa51p-5,
+       -0x1.dfde7ff758ea8p-6, 0x1.901f43aac38c8p-7, -0x1.b581d07df5ad5p-9,
+       0x1.c3726535f1fc6p-12},
+      {0x1.551d5d9b204d3p-2, -0x1.c019e328f8db1p-4, 0x1.c7710f44fc3cep-5,
+       -0x1.cbbbe25ea8ba4p-6, 0x1.6fe270088623dp-7, -0x1.7e6fc79733761p-9,
+       0x1.75077abf18d84p-12},
+  };
+
+  using FloatBits = typename fputil::FPBits<float>;
+  using DoubleBits = typename fputil::FPBits<double>;
+
+  FloatBits x_bits(x);
+
+  uint32_t x_abs = x_bits.uintval() & 0x7fff'ffff;
+  uint32_t sign_bit = (x_bits.uintval() >> 31) << DoubleBits::EXP_LEN;
+
+  if (LIBC_UNLIKELY(x == 0.0f || x_abs >= 0x7f80'0000)) {
+    // x is 0, Inf, or NaN.
+    // Make sure it works for FTZ/DAZ modes.
+    return x + x;
+  }
+
+  double xd = static_cast<double>(x);
+  DoubleBits xd_bits(xd);
+
+  // When using biased exponent of x in double precision,
+  //   x_e = real_exponent_of_x + 1023
+  // Then:
+  //   x_e / 3 = real_exponent_of_x / 3 + 1023/3
+  //           = real_exponent_of_x / 3 + 341
+  // So to make it the correct biased exponent of x^(1/3), we add
+  //   1023 - 341 = 682
+  // to the quotient x_e / 3.
+  unsigned x_e = static_cast<unsigned>(xd_bits.get_biased_exponent());
+  unsigned out_e = (x_e / 3 + 682) | sign_bit;
+  unsigned shift_e = x_e % 3;
+
+  // Set x_m = 2^(x_e % 3) * (1.mantissa)
+  uint64_t x_m = xd_bits.get_mantissa();
+  // Use the leading 4 bits for look up table
+  unsigned idx = static_cast<unsigned>(x_m >> (DoubleBits::FRACTION_LEN - 4));
+
+  x_m |= static_cast<uint64_t>(DoubleBits::EXP_BIAS)
+         << DoubleBits::FRACTION_LEN;
+
+  double x_reduced = DoubleBits(x_m).get_val();
+  double dx = x_reduced - 1.0;
+
+  double dx_sq = dx * dx;
+  double c0 = fputil::multiply_add(dx, COEFFS[idx][0], 1.0);
+  double c1 = fputil::multiply_add(dx, COEFFS[idx][2], COEFFS[idx][1]);
+  double c2 = fputil::multiply_add(dx, COEFFS[idx][4], COEFFS[idx][3]);
+  double c3 = fputil::multiply_add(dx, COEFFS[idx][6], COEFFS[idx][5]);
+
+  double dx_4 = dx_sq * dx_sq;
+  double p0 = fputil::multiply_add(dx_sq, c1, c0);
+  double p1 = fputil::multiply_add(dx_sq, c3, c2);
+
+  double r = fputil::multiply_add(dx_4, p1, p0) * CBRT2[shift_e];
+
+  uint64_t r_m = DoubleBits(r).get_mantissa();
+  // Check if the output is exact.  To be exact, the smallest 1-bit of the
+  // output has to be at least 2^-7 or higher.  So we check the lowest 44 bits
+  // to see if they are within 2^(-52 + 3) errors from all zeros, then the
+  // result cube root is exact.
+  if (LIBC_UNLIKELY(((r_m + 8) & 0xfffffffffff) <= 16)) {
+    if ((r_m & 0xfffffffffff) <= 8)
+      r_m &= 0xffff'ffff'ffff'ffe0;
+    else
+      r_m = (r_m & 0xffff'ffff'ffff'ffe0) + 0x20;
+    fputil::clear_except_if_required(FE_INEXACT);
+  }
+  // Adjust exponent and sign.
+  uint64_t r_bits =
+      r_m | (static_cast<uint64_t>(out_e) << DoubleBits::FRACTION_LEN);
+
+  return static_cast<float>(DoubleBits(r_bits).get_val());
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_SRC___SUPPORT_MATH_CBRTF_H
diff --git a/libc/src/__support/math/cos.h b/libc/src/__support/math/cos.h
new file mode 100644
index 0000000000000..0802f9e4f6e49
--- /dev/null
+++ b/libc/src/__support/math/cos.h
@@ -0,0 +1,173 @@
+//===-- Implementation header for cos ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_SRC___SUPPORT_MATH_COS_H
+#define LIBC_SRC___SUPPORT_MATH_COS_H
+
+#include "range_reduction_double_common.h"
+#include "sincos_eval.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/double_double.h"
+#include "src/__support/FPUtil/dyadic_float.h"
+#include "src/__support/FPUtil/except_value_utils.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
+#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+
+#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+#include "range_reduction_double_fma.h"
+#else
+#include "range_reduction_double_nofma.h"
+#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+LIBC_INLINE static constexpr double cos(double x) {
+  using namespace range_reduction_double_internal;
+  using DoubleDouble = fputil::DoubleDouble;
+  using FPBits = typename fputil::FPBits<double>;
+  FPBits xbits(x);
+
+  uint16_t x_e = xbits.get_biased_exponent();
+
+  DoubleDouble y;
+  unsigned k = 0;
+  LargeRangeReduction range_reduction_large;
+
+  // |x| < 2^16.
+  if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT)) {
+    // |x| < 2^-7
+    if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 7)) {
+      // |x| < 2^-27
+      if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 27)) {
+        // Signed zeros.
+        if (LIBC_UNLIKELY(x == 0.0))
+          return 1.0;
+
+        // For |x| < 2^-27, |cos(x) - 1| < |x|^2/2 < 2^-54 = ulp(1 - 2^-53)/2.
+        return fputil::round_result_slightly_down(1.0);
+      }
+      // No range reduction needed.
+      k = 0;
+      y.lo = 0.0;
+      y.hi = x;
+    } else {
+      // Small range reduction.
+      k = range_reduction_small(x, y);
+    }
+  } else {
+    // Inf or NaN
+    if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) {
+      if (xbits.is_signaling_nan()) {
+        fputil::raise_except_if_required(FE_INVALID);
+        return FPBits::quiet_nan().get_val();
+      }
+      // cos(+-Inf) = NaN
+      if (xbits.get_mantissa() == 0) {
+        fputil::set_errno_if_required(EDOM);
+        fputil::raise_except_if_required(FE_INVALID);
+      }
+      return x + FPBits::quiet_nan().get_val();
+    }
+
+    // Large range reduction.
+    k = range_reduction_large.fast(x, y);
+  }
+
+  DoubleDouble sin_y, cos_y;
+
+  [[maybe_unused]] double err =
+      math::sincos_eval_internal::sincos_eval(y, sin_y, cos_y);
+
+  // Look up sin(k * pi/128) and cos(k * pi/128)
+#ifdef LIBC_MATH_HAS_SMALL_TABLES
+  // Memory saving versions.  Use 65-entry table.
+  auto get_idx_dd = [](unsigned kk) -> DoubleDouble {
+    unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
+    DoubleDouble ans = SIN_K_PI_OVER_128[idx];
+    if (kk & 128) {
+      ans.hi = -ans.hi;
+      ans.lo = -ans.lo;
+    }
+    return ans;
+  };
+  DoubleDouble msin_k = get_idx_dd(k + 128);
+  DoubleDouble cos_k = get_idx_dd(k + 64);
+#else
+  // Fast look up version, but needs 256-entry table.
+  // -sin(k * pi/128) = sin((k + 128) * pi/128)
+  // cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128).
+  DoubleDouble msin_k = SIN_K_PI_OVER_128[(k + 128) & 255];
+  DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 255];
+#endif // LIBC_MATH_HAS_SMALL_TABLES
+
+  // After range reduction, k = round(x * 128 / pi) and y = x - k * (pi / 128).
+  // So k is an integer and -pi / 256 <= y <= pi / 256.
+  // Then cos(x) = cos((k * pi/128 + y)
+  //             = cos(y) * cos(k*pi/128) - sin(y) * sin(k*pi/128)
+  DoubleDouble cos_k_cos_y = fputil::quick_mult(cos_y, cos_k);
+  DoubleDouble msin_k_sin_y = fputil::quick_mult(sin_y, msin_k);
+
+  DoubleDouble rr = fputil::exact_add<false>(cos_k_cos_y.hi, msin_k_sin_y.hi);
+  rr.lo += msin_k_sin_y.lo + cos_k_cos_y.lo;
+
+#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  return rr.hi + rr.lo;
+#else
+  using Float128 = typename fputil::DyadicFloat<128>;
+  double rlp = rr.lo + err;
+  double rlm = rr.lo - err;
+
+  double r_upper = rr.hi + rlp; // (rr.lo + ERR);
+  double r_lower = rr.hi + rlm; // (rr.lo - ERR);
+
+  // Ziv's rounding test.
+  if (LIBC_LIKELY(r_upper == r_lower))
+    return r_upper;
+
+  Float128 u_f128, sin_u, cos_u;
+  if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT))
+    u_f128 = range_reduction_small_f128(x);
+  else
+    u_f128 = range_reduction_large.accurate();
+
+  math::sincos_eval_internal::sincos_eval(u_f128, sin_u, cos_u);
+
+  auto get_sin_k = [](unsigned kk) -> Float128 {
+    unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
+    Float128 ans = SIN_K_PI_OVER_128_F128[idx];
+    if (kk & 128)
+      ans.sign = Sign::NEG;
+    return ans;
+  };
+
+  // -sin(k * pi/128) = sin((k + 128) * pi/128)
+  // cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128).
+  Float128 msin_k_f128 = get_sin_k(k + 128);
+  Float128 cos_k_f128 = get_sin_k(k + 64);
+
+  // cos(x) = cos((k * pi/128 + u)
+  //        = cos(u) * cos(k*pi/128) - sin(u) * sin(k*pi/128)
+  Float128 r = fputil::quick_add(fputil::quick_mul(cos_k_f128, cos_u),
+                                 fputil::quick_mul(msin_k_f128, sin_u));
+
+  // TODO: Add assertion if Ziv's accuracy tests fail in debug mode.
+  // https://github.com/llvm/llvm-project/issues/96452.
+
+  return static_cast<double>(r);
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_SRC___SUPPORT_MATH_COS_H
diff --git a/libc/src/__support/math/cosf.h b/libc/src/__support/math/cosf.h
new file mode 100644
index 0000000000000..074be0b314637
--- /dev/null
+++ b/libc/src/__support/math/cosf.h
@@ -0,0 +1,152 @@
+//===-- Implementation header for cosf --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIBC_SRC___SUPPORT_MATH_COSF_H
+#define LIBC_SRC___SUPPORT_MATH_COSF_H
+
+#include "sincosf_utils.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/except_value_utils.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
+#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+LIBC_INLINE static constexpr float cosf(float x) {
+
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  // Exceptional cases for cosf.
+  constexpr size_t N_EXCEPTS = 6;
+
+  constexpr fputil::ExceptValues<float, N_EXCEPTS> COSF_EXCEPTS{{
+      // (inputs, RZ output, RU offset, RD offset, RN offset)
+      // x = 0x1.64a032p43, cos(x) = 0x1.9d4ba4p-1 (RZ)
+      {0x55325019, 0x3f4ea5d2, 1, 0, 0},
+      // x = 0x1.4555p51, cos(x) = 0x1.115d7cp-1 (RZ)
+      {0x5922aa80, 0x3f08aebe, 1, 0, 1},
+      // x = 0x1.48a858p54, cos(x) = 0x1.f48148p-2 (RZ)
+      {0x5aa4542c, 0x3efa40a4, 1, 0, 0},
+      // x = 0x1.3170fp63, cos(x) = 0x1.fe2976p-1 (RZ)
+      {0x5f18b878, 0x3f7f14bb, 1, 0, 0},
+      // x = 0x1.2b9622p67, cos(x) = 0x1.f0285cp-1 (RZ)
+      {0x6115cb11, 0x3f78142e, 1, 0, 1},
+      // x = 0x1.ddebdep120, cos(x) = 0x1.114438p-1 (RZ)
+      {0x7beef5ef, 0x3f08a21c, 1, 0, 0},
+  }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+
+  using FPBits = typename fputil::FPBits<float>;
+
+  FPBits xbits(x);
+  xbits.set_sign(Sign::POS);
+
+  uint32_t x_abs = xbits.uintval();
+  double xd = static_cast<double>(xbits.get_val());
+
+  // Range reduction:
+  // For |x| > pi/16, we perform range reduction as follows:
+  // Find k and y such that:
+  //   x = (k + y) * pi/32
+  //   k is an integer
+  //   |y| < 0.5
+  // For small range (|x| < 2^45 when FMA instructions are available, 2^22
+  // otherwise), this is done by performing:
+  //   k = round(x * 32/pi)
+  //   y = x * 32/pi - k
+  // For large range, we will omit all the higher parts of 16/pi such that the
+  // least significant bits of their full products with x are larger than 63,
+  // since cos((k + y + 64*i) * pi/32) = cos(x + i * 2pi) = cos(x).
+  //
+  // When FMA instructions are not available, we store the digits of 32/pi in
+  // chunks of 28-bit precision.  This will make sure that the products:
+  //   x * THIRTYTWO_OVER_PI_28[i] are all exact.
+  // When FMA instructions are available, we simply store the digits of 32/pi in
+  // chunks of doubles (53-bit of precision).
+  // So when multiplying by the largest values of single precision, the
+  // resulting output should be correct up to 2^(-208 + 128) ~ 2^-80.  By the
+  // worst-case analysis of range reduction, |y| >= 2^-38, so this should give
+  // us more than 40 bits of accuracy. For the worst-case estimation of range
+  // reduction, see for instances:
+  //   Elementary Functions by J-M. Muller, Chapter 11,
+  //   Handbook of Floating-Point Arithmetic by J-M. Muller et. al.,
+  //   Chapter 10.2.
+  //
+  // Once k and y are computed, we then deduce the answer by the cosine of sum
+  // formula:
+  //   cos(x) = cos((k + y)*pi/32)
+  //          = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
+  // The values of sin(k*pi/32) and cos(k*pi/32) for k = 0..63 are precomputed
+  // and stored using a vector of 32 doubles. Sin(y*pi/32) and cos(y*pi/32) are
+  // computed using degree-7 and degree-6 minimax polynomials generated by
+  // Sollya respectively.
+
+  // |x| < 0x1.0p-12f
+  if (LIBC_UNLIKELY(x_abs < 0x3980'0000U)) {
+    // When |x| < 2^-12, the relative error of the approximation cos(x) ~ 1
+    // is:
+    //   |cos(x) - 1| < |x^2 / 2| = 2^-25 < epsilon(1)/2.
+    // So the correctly rounded values of cos(x) are:
+    //   = 1 - eps(x) if rounding mode = FE_TOWARDZERO or FE_DOWWARD,
+    //   = 1 otherwise.
+    // To simplify the rounding decision and make it more efficient and to
+    // prevent compiler to perform constant folding, we use
+    //   fma(x, -2^-25, 1) instead.
+    // Note: to use the formula 1 - 2^-25*x to decide the correct rounding, we
+    // do need fma(x, -2^-25, 1) to prevent underflow caused by -2^-25*x when
+    // |x| < 2^-125. For targets without FMA instructions, we simply use
+    // double for intermediate results as it is more efficient than using an
+    // emulated version of FMA.
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
+    return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f);
+#else
+    return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, 1.0));
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
+  }
+
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  if (auto r = COSF_EXCEPTS.lookup(x_abs); LIBC_UNLIKELY(r.has_value()))
+    return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+
+  // x is inf or nan.
+  if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) {
+    if (xbits.is_signaling_nan()) {
+      fputil::raise_except_if_required(FE_INVALID);
+      return FPBits::quiet_nan().get_val();
+    }
+
+    if (x_abs == 0x7f80'0000U) {
+      fputil::set_errno_if_required(EDOM);
+      fputil::raise_except_if_required(FE_INVALID);
+    }
+    return x + FPBits::quiet_nan().get_val();
+  }
+
+  // Combine the results with the sine of sum formula:
+  //   cos(x) = cos((k + y)*pi/32)
+  //          = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
+  //          = cosm1_y * cos_k + sin_y * sin_k
+  //          = (cosm1_y * cos_k + cos_k) + sin_y * sin_k
+  double sin_k = 0, cos_k = 0, sin_y = 0, cosm1_y = 0;
+
+  sincosf_eval(xd, x_abs, sin_k, cos_k, sin_y, cosm1_y);
+
+  return static_cast<float>(fputil::multiply_add(
+      sin_y, -sin_k, fputil::multiply_add(cosm1_y, cos_k, cos_k)));
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_SRC___SUPPORT_MATH_COSF_H
diff --git a/libc/src/__support/math/cosf16.h b/libc/src/__support/math/cosf16.h
new file mode 100644
index 0000000000000..50c9a8f765c2a
--- /dev/null
+++ b/libc/src/__support/math/cosf16.h
@@ -0,0 +1,106 @@
+//===-- Implementation header for cosf16 ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_COSF16_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_COSF16_H
+
+#include "include/llvm-libc-macros/float16-macros.h"
+
+#ifdef LIBC_TYPES_HAS_FLOAT16
+
+#include "sincosf16_utils.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/cast.h"
+#include "src/__support/FPUtil/except_value_utils.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/macros/optimization.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+LIBC_INLINE static constexpr float16 cosf16(float16 x) {
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  constexpr size_t N_EXCEPTS = 4;
+
+  constexpr fputil::ExceptValues<float16, N_EXCEPTS> COSF16_EXCEPTS{{
+      // (input, RZ output, RU offset, RD offset, RN offset)
+      {0x2b7c, 0x3bfc, 1, 0, 1},
+      {0x4ac1, 0x38b5, 1, 0, 0},
+      {0x5c49, 0xb8c6, 0, 1, 0},
+      {0x7acc, 0xa474, 0, 1, 0},
+  }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+
+  using namespace sincosf16_internal;
+  using FPBits = fputil::FPBits<float16>;
+  FPBits xbits(x);
+
+  uint16_t x_u = xbits.uintval();
+  uint16_t x_abs = x_u & 0x7fff;
+  float xf = x;
+
+  // Range reduction:
+  // For |x| > pi/32, we perform range reduction as follows:
+  // Find k and y such that:
+  //   x = (k + y) * pi/32
+  //   k is an integer, |y| < 0.5
+  //
+  // This is done by performing:
+  //   k = round(x * 32/pi)
+  //   y = x * 32/pi - k
+  //
+  // Once k and y are computed, we then deduce the answer by the cosine of sum
+  // formula:
+  //   cos(x) = cos((k + y) * pi/32)
+  //          = cos(k * pi/32) * cos(y * pi/32) -
+  //            sin(k * pi/32) * sin(y * pi/32)
+
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  // Handle exceptional values
+  if (auto r = COSF16_EXCEPTS.lookup(x_abs); LIBC_UNLIKELY(r.has_value()))
+    return r.value();
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+
+  // cos(+/-0) = 1
+  if (LIBC_UNLIKELY(x_abs == 0U))
+    return fputil::cast<float16>(1.0f);
+
+  // cos(+/-inf) = NaN, and cos(NaN) = NaN
+  if (xbits.is_inf_or_nan()) {
+    if (xbits.is_signaling_nan()) {
+      fputil::raise_except_if_required(FE_INVALID);
+      return FPBits::quiet_nan().get_val();
+    }
+
+    if (xbits.is_inf()) {
+      fputil::set_errno_if_required(EDOM);
+      fputil::raise_except_if_required(FE_INVALID);
+    }
+
+    return x + FPBits::quiet_nan().get_val();
+  }
+
+  float sin_k = 0.0f, cos_k = 0.0f, sin_y = 0.0f, cosm1_y = 0.0f;
+  sincosf16_eval(xf, sin_k, cos_k, sin_y, cosm1_y);
+  // Since, cosm1_y = cos_y - 1, therefore:
+  //   cos(x) = cos_k * cos_y - sin_k * sin_y
+  //          = cos_k * (cos_y - 1 + 1) - sin_k * sin_y
+  //          = cos_k * cosm1_y - sin_k * sin_y + cos_k
+  return fputil::cast<float16>(fputil::multiply_add(
+      cos_k, cosm1_y, fputil::multiply_add(-sin_k, sin_y, cos_k)));
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_TYPES_HAS_FLOAT16
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_COSF16_H
diff --git a/libc/src/__support/math/coshf.h b/libc/src/__support/math/coshf.h
new file mode 100644
index 0000000000000..0f233b87c5e2c
--- /dev/null
+++ b/libc/src/__support/math/coshf.h
@@ -0,0 +1,65 @@
+//===-- Implementation header for coshf -------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_COSHF_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_COSHF_H
+
+#include "sinhfcoshf_utils.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+LIBC_INLINE static constexpr float coshf(float x) {
+  using namespace sinhfcoshf_internal;
+  using FPBits = typename fputil::FPBits<float>;
+
+  FPBits xbits(x);
+  xbits.set_sign(Sign::POS);
+  x = xbits.get_val();
+
+  uint32_t x_u = xbits.uintval();
+
+  // When |x| >= 90, or x is inf or nan
+  if (LIBC_UNLIKELY(x_u >= 0x42b4'0000U || x_u <= 0x3280'0000U)) {
+    // |x| <= 2^-26
+    if (x_u <= 0x3280'0000U) {
+      return 1.0f + x;
+    }
+
+    if (xbits.is_inf_or_nan())
+      return x + FPBits::inf().get_val();
+
+    int rounding = fputil::quick_get_round();
+    if (LIBC_UNLIKELY(rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO))
+      return FPBits::max_normal().get_val();
+
+    fputil::set_errno_if_required(ERANGE);
+    fputil::raise_except_if_required(FE_OVERFLOW);
+
+    return x + FPBits::inf().get_val();
+  }
+
+  // TODO: We should be able to reduce the latency and reciprocal throughput
+  // further by using a low degree (maybe 3-7 ?) minimax polynomial for small
+  // but not too small inputs, such as |x| < 2^-2, or |x| < 2^-3.
+
+  // cosh(x) = (e^x + e^(-x)) / 2.
+  return static_cast<float>(exp_pm_eval</*is_sinh*/ false>(x));
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_COSHF_H
diff --git a/libc/src/__support/math/coshf16.h b/libc/src/__support/math/coshf16.h
new file mode 100644
index 0000000000000..4c96a78fa5254
--- /dev/null
+++ b/libc/src/__support/math/coshf16.h
@@ -0,0 +1,124 @@
+//===-- Implementation header for coshf16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_COSHF16_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_COSHF16_H
+
+#include "include/llvm-libc-macros/float16-macros.h"
+
+#ifdef LIBC_TYPES_HAS_FLOAT16
+
+#include "expxf16_utils.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/except_value_utils.h"
+#include "src/__support/FPUtil/rounding_mode.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+LIBC_INLINE static constexpr float16 coshf16(float16 x) {
+
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  constexpr fputil::ExceptValues<float16, 9> COSHF16_EXCEPTS_POS = {{
+      // x = 0x1.6ap-5, coshf16(x) = 0x1p+0 (RZ)
+      {0x29a8U, 0x3c00U, 1U, 0U, 1U},
+      // x = 0x1.8c4p+0, coshf16(x) = 0x1.3a8p+1 (RZ)
+      {0x3e31U, 0x40eaU, 1U, 0U, 0U},
+      // x = 0x1.994p+0, coshf16(x) = 0x1.498p+1 (RZ)
+      {0x3e65U, 0x4126U, 1U, 0U, 0U},
+      // x = 0x1.b6p+0, coshf16(x) = 0x1.6d8p+1 (RZ)
+      {0x3ed8U, 0x41b6U, 1U, 0U, 1U},
+      // x = 0x1.aap+1, coshf16(x) = 0x1.be8p+3 (RZ)
+      {0x42a8U, 0x4afaU, 1U, 0U, 1U},
+      // x = 0x1.cc4p+1, coshf16(x) = 0x1.23cp+4 (RZ)
+      {0x4331U, 0x4c8fU, 1U, 0U, 0U},
+      // x = 0x1.288p+2, coshf16(x) = 0x1.9b4p+5 (RZ)
+      {0x44a2U, 0x526dU, 1U, 0U, 0U},
+      // x = 0x1.958p+2, coshf16(x) = 0x1.1a4p+8 (RZ)
+      {0x4656U, 0x5c69U, 1U, 0U, 0U},
+      // x = 0x1.5fp+3, coshf16(x) = 0x1.c54p+14 (RZ)
+      {0x497cU, 0x7715U, 1U, 0U, 1U},
+  }};
+
+  constexpr fputil::ExceptValues<float16, 6> COSHF16_EXCEPTS_NEG = {{
+      // x = -0x1.6ap-5, coshf16(x) = 0x1p+0 (RZ)
+      {0xa9a8U, 0x3c00U, 1U, 0U, 1U},
+      // x = -0x1.b6p+0, coshf16(x) = 0x1.6d8p+1 (RZ)
+      {0xbed8U, 0x41b6U, 1U, 0U, 1U},
+      // x = -0x1.288p+2, coshf16(x) = 0x1.9b4p+5 (RZ)
+      {0xc4a2U, 0x526dU, 1U, 0U, 0U},
+      // x = -0x1.5fp+3, coshf16(x) = 0x1.c54p+14 (RZ)
+      {0xc97cU, 0x7715U, 1U, 0U, 1U},
+      // x = -0x1.8c4p+0, coshf16(x) = 0x1.3a8p+1 (RZ)
+      {0xbe31U, 0x40eaU, 1U, 0U, 0U},
+      // x = -0x1.994p+0, coshf16(x) = 0x1.498p+1 (RZ)
+      {0xbe65U, 0x4126U, 1U, 0U, 0U},
+  }};
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+
+  using namespace expxf16_internal;
+  using FPBits = fputil::FPBits<float16>;
+  FPBits x_bits(x);
+
+  uint16_t x_u = x_bits.uintval();
+  uint16_t x_abs = x_u & 0x7fffU;
+
+  // When |x| >= acosh(2^16), or x is NaN.
+  if (LIBC_UNLIKELY(x_abs >= 0x49e5U)) {
+    // cosh(NaN) = NaN
+    if (x_bits.is_nan()) {
+      if (x_bits.is_signaling_nan()) {
+        fputil::raise_except_if_required(FE_INVALID);
+        return FPBits::quiet_nan().get_val();
+      }
+
+      return x;
+    }
+
+    // When |x| >= acosh(2^16).
+    if (x_abs >= 0x49e5U) {
+      // cosh(+/-inf) = +inf
+      if (x_bits.is_inf())
+        return FPBits::inf().get_val();
+
+      switch (fputil::quick_get_round()) {
+      case FE_TONEAREST:
+      case FE_UPWARD:
+        fputil::set_errno_if_required(ERANGE);
+        fputil::raise_except_if_required(FE_OVERFLOW | FE_INEXACT);
+        return FPBits::inf().get_val();
+      default:
+        return FPBits::max_normal().get_val();
+      }
+    }
+  }
+
+#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+  if (x_bits.is_pos()) {
+    if (auto r = COSHF16_EXCEPTS_POS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
+      return r.value();
+  } else {
+    if (auto r = COSHF16_EXCEPTS_NEG.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
+      return r.value();
+  }
+#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
+
+  return eval_sinh_or_cosh</*IsSinh=*/false>(x);
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_TYPES_HAS_FLOAT16
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_COSHF16_H
diff --git a/libc/src/__support/math/cospif.h b/libc/src/__support/math/cospif.h
new file mode 100644
index 0000000000000..e921090459769
--- /dev/null
+++ b/libc/src/__support/math/cospif.h
@@ -0,0 +1,109 @@
+//===-- Implementation header for cospif ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_COSPIF_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_COSPIF_H
+
+#include "sincosf_utils.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
+#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+LIBC_INLINE static constexpr float cospif(float x) {
+  using FPBits = typename fputil::FPBits<float>;
+
+  FPBits xbits(x);
+  xbits.set_sign(Sign::POS);
+
+  uint32_t x_abs = xbits.uintval();
+  double xd = static_cast<double>(xbits.get_val());
+
+  // Range reduction:
+  // For |x| > 1/32, we perform range reduction as follows:
+  // Find k and y such that:
+  //   x = (k + y) * 1/32
+  //   k is an integer
+  //   |y| < 0.5
+  //
+  // This is done by performing:
+  //   k = round(x * 32)
+  //   y = x * 32 - k
+  //
+  // Once k and y are computed, we then deduce the answer by the cosine of sum
+  // formula:
+  //   cospi(x) = cos((k + y)*pi/32)
+  //          = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
+  // The values of sin(k*pi/32) and cos(k*pi/32) for k = 0..63 are precomputed
+  // and stored using a vector of 32 doubles. Sin(y*pi/32) and cos(y*pi/32) are
+  // computed using degree-7 and degree-6 minimax polynomials generated by
+  // Sollya respectively.
+
+  // The exhautive test passes for smaller values
+  if (LIBC_UNLIKELY(x_abs < 0x38A2'F984U)) {
+
+#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
+    return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f);
+#else
+    return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, 1.0));
+#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
+  }
+
+  // Numbers greater or equal to 2^23 are always integers or NaN
+  if (LIBC_UNLIKELY(x_abs >= 0x4B00'0000)) {
+
+    if (LIBC_UNLIKELY(x_abs < 0x4B80'0000)) {
+      return (x_abs & 0x1) ? -1.0f : 1.0f;
+    }
+
+    // x is inf or nan.
+    if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) {
+      if (xbits.is_signaling_nan()) {
+        fputil::raise_except_if_required(FE_INVALID);
+        return FPBits::quiet_nan().get_val();
+      }
+
+      if (x_abs == 0x7f80'0000U) {
+        fputil::set_errno_if_required(EDOM);
+        fputil::raise_except_if_required(FE_INVALID);
+      }
+      return x + FPBits::quiet_nan().get_val();
+    }
+
+    return 1.0f;
+  }
+
+  // Combine the results with the sine of sum formula:
+  //   cos(pi * x) = cos((k + y)*pi/32)
+  //          = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
+  //          = (cosm1_y + 1) * cos_k - sin_y * sin_k
+  //          = (cosm1_y * cos_k + cos_k) - sin_y * sin_k
+  double sin_k = 0, cos_k = 0, sin_y = 0, cosm1_y = 0;
+
+  sincospif_eval(xd, sin_k, cos_k, sin_y, cosm1_y);
+
+  if (LIBC_UNLIKELY(sin_y == 0 && cos_k == 0)) {
+    return 0.0f;
+  }
+
+  return static_cast<float>(fputil::multiply_add(
+      sin_y, -sin_k, fputil::multiply_add(cosm1_y, cos_k, cos_k)));
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_COSHF_H
diff --git a/libc/src/__support/math/cospif16.h b/libc/src/__support/math/cospif16.h
new file mode 100644
index 0000000000000..d07236cccac75
--- /dev/null
+++ b/libc/src/__support/math/cospif16.h
@@ -0,0 +1,99 @@
+//===-- Implementation header for cospif16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_COSPIF16_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_COSPIF16_H
+
+#include "include/llvm-libc-macros/float16-macros.h"
+
+#ifdef LIBC_TYPES_HAS_FLOAT16
+
+#include "sincosf16_utils.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/cast.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/macros/optimization.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace math {
+
+LIBC_INLINE static constexpr float16 cospif16(float16 x) {
+
+  using namespace sincosf16_internal;
+  using FPBits = typename fputil::FPBits<float16>;
+  FPBits xbits(x);
+
+  uint16_t x_u = xbits.uintval();
+  uint16_t x_abs = x_u & 0x7fff;
+  float xf = x;
+
+  // Range reduction:
+  // For |x| > 1/32, we perform range reduction as follows:
+  // Find k and y such that:
+  //   x = (k + y) * 1/32
+  //   k is an integer
+  //   |y| < 0.5
+  //
+  // This is done by performing:
+  //   k = round(x * 32)
+  //   y = x * 32 - k
+  //
+  // Once k and y are computed, we then deduce the answer by the cosine of sum
+  // formula:
+  //   cos(x * pi) = cos((k + y) * pi/32)
+  //               = cos(k * pi/32) * cos(y * pi/32) +
+  //                 sin(y * pi/32) * sin(k * pi/32)
+
+  // For signed zeros
+  if (LIBC_UNLIKELY(x_abs == 0U))
+    return fputil::cast<float16>(1.0f);
+
+  // Numbers greater or equal to 2^10 are integers, or infinity, or NaN
+  if (LIBC_UNLIKELY(x_abs >= 0x6400)) {
+    if (LIBC_UNLIKELY(x_abs <= 0x67FF))
+      return fputil::cast<float16>((x_abs & 0x1) ? -1.0f : 1.0f);
+
+    // Check for NaN or infintiy values
+    if (LIBC_UNLIKELY(x_abs >= 0x7c00)) {
+      if (xbits.is_signaling_nan()) {
+        fputil::raise_except_if_required(FE_INVALID);
+        return FPBits::quiet_nan().get_val();
+      }
+      // If value is equal to infinity
+      if (x_abs == 0x7c00) {
+        fputil::set_errno_if_required(EDOM);
+        fputil::raise_except_if_required(FE_INVALID);
+      }
+
+      return x + FPBits::quiet_nan().get_val();
+    }
+
+    return fputil::cast<float16>(1.0f);
+  }
+
+  float sin_k = 0, cos_k = 0, sin_y = 0, cosm1_y = 0;
+  sincospif16_eval(xf, sin_k, cos_k, sin_y, cosm1_y);
+
+  if (LIBC_UNLIKELY(sin_y == 0 && cos_k == 0))
+    return fputil::cast<float16>(0.0f);
+
+  // Since, cosm1_y = cos_y - 1, therefore:
+  //   cos(x * pi) = cos_k(cosm1_y) + cos_k - sin_k * sin_y
+  return fputil::cast<float16>(fputil::multiply_add(
+      cos_k, cosm1_y, fputil::multiply_add(-sin_k, sin_y, cos_k)));
+}
+
+} // namespace math
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LIBC_TYPES_HAS_FLOAT16
+
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_COSHF16_H
diff --git a/libc/src/math/generic/expxf16.h b/libc/src/__support/math/expxf16_utils.h
similarity index 95%
rename from libc/src/math/generic/expxf16.h
rename to libc/src/__support/math/expxf16_utils.h
index 562a427fc6cdc..5d3bd3845a74f 100644
--- a/libc/src/math/generic/expxf16.h
+++ b/libc/src/__support/math/expxf16_utils.h
@@ -21,7 +21,11 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-LIBC_INLINE ExpRangeReduction exp2_range_reduction(float16 x) {
+namespace math {
+
+namespace expxf16_internal {
+
+LIBC_INLINE static ExpRangeReduction exp2_range_reduction(float16 x) {
   // For -25 < x < 16, to compute 2^x, we perform the following range reduction:
   // find hi, mid, lo, such that:
   //   x = hi + mid + lo, in which
@@ -117,7 +121,8 @@ static constexpr cpp::array<uint32_t, 32> EXP2_MID_5_BITS = {
 // The main point of these formulas is that the expensive part of calculating
 // the polynomials approximating lower parts of e^x and e^(-x) is shared and
 // only done once.
-template <bool IsSinh> LIBC_INLINE float16 eval_sinh_or_cosh(float16 x) {
+template <bool IsSinh>
+LIBC_INLINE static constexpr float16 eval_sinh_or_cosh(float16 x) {
   float xf = x;
   float kf = fputil::nearest_integer(xf * (LOG2F_E * 0x1.0p+5f));
   int x_hi_mid_p = static_cast<int>(kf);
@@ -174,7 +179,7 @@ template <bool IsSinh> LIBC_INLINE float16 eval_sinh_or_cosh(float16 x) {
 // Generated by Sollya with the following commands:
 //   > display = hexadecimal;
 //   > for i from 0 to 31 do print(round(log(1 + i * 2^-5), SG, RN));
-constexpr cpp::array<float, 32> LOGF_F = {
+static constexpr cpp::array<float, 32> LOGF_F = {
     0x0p+0f,        0x1.f829bp-6f,  0x1.f0a30cp-5f, 0x1.6f0d28p-4f,
     0x1.e27076p-4f, 0x1.29553p-3f,  0x1.5ff308p-3f, 0x1.9525aap-3f,
     0x1.c8ff7cp-3f, 0x1.fb9186p-3f, 0x1.1675cap-2f, 0x1.2e8e2cp-2f,
@@ -188,7 +193,7 @@ constexpr cpp::array<float, 32> LOGF_F = {
 // Generated by Sollya with the following commands:
 //   > display = hexadecimal;
 //   > for i from 0 to 31 do print(round(log2(1 + i * 2^-5), SG, RN));
-constexpr cpp::array<float, 32> LOG2F_F = {
+static constexpr cpp::array<float, 32> LOG2F_F = {
     0x0p+0f,        0x1.6bad38p-5f, 0x1.663f7p-4f,  0x1.08c588p-3f,
     0x1.5c01a4p-3f, 0x1.acf5e2p-3f, 0x1.fbc16cp-3f, 0x1.24407ap-2f,
     0x1.49a784p-2f, 0x1.6e221cp-2f, 0x1.91bba8p-2f, 0x1.b47ecp-2f,
@@ -202,7 +207,7 @@ constexpr cpp::array<float, 32> LOG2F_F = {
 // Generated by Sollya with the following commands:
 //   > display = hexadecimal;
 //   > for i from 0 to 31 do print(round(log10(1 + i * 2^-5), SG, RN));
-constexpr cpp::array<float, 32> LOG10F_F = {
+static constexpr cpp::array<float, 32> LOG10F_F = {
     0x0p+0f,        0x1.b5e908p-7f, 0x1.af5f92p-6f, 0x1.3ed11ap-5f,
     0x1.a30a9ep-5f, 0x1.02428cp-4f, 0x1.31b306p-4f, 0x1.5fe804p-4f,
     0x1.8cf184p-4f, 0x1.b8de4ep-4f, 0x1.e3bc1ap-4f, 0x1.06cbd6p-3f,
@@ -216,7 +221,7 @@ constexpr cpp::array<float, 32> LOG10F_F = {
 // Generated by Sollya with the following commands:
 //   > display = hexadecimal;
 //   > for i from 0 to 31 do print(round(1 / (1 + i * 2^-5), SG, RN));
-constexpr cpp::array<float, 32> ONE_OVER_F_F = {
+static constexpr cpp::array<float, 32> ONE_OVER_F_F = {
     0x1p+0f,        0x1.f07c2p-1f,  0x1.e1e1e2p-1f, 0x1.d41d42p-1f,
     0x1.c71c72p-1f, 0x1.bacf92p-1f, 0x1.af286cp-1f, 0x1.a41a42p-1f,
     0x1.99999ap-1f, 0x1.8f9c18p-1f, 0x1.861862p-1f, 0x1.7d05f4p-1f,
@@ -227,6 +232,10 @@ constexpr cpp::array<float, 32> ONE_OVER_F_F = {
     0x1.111112p-1f, 0x1.0c9714p-1f, 0x1.08421p-1f,  0x1.041042p-1f,
 };
 
+} // namespace expxf16_internal
+
+} // namespace math
+
 } // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_SRC_MATH_GENERIC_EXPXF16_H
diff --git a/libc/src/math/generic/range_reduction.h b/libc/src/__support/math/range_reduction.h
similarity index 95%
rename from libc/src/math/generic/range_reduction.h
rename to libc/src/__support/math/range_reduction.h
index 9ea446d89bed5..e3f25e4fda3c1 100644
--- a/libc/src/math/generic/range_reduction.h
+++ b/libc/src/__support/math/range_reduction.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_H
-#define LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_H
+#ifndef LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_H
+#define LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_H
 
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/multiply_add.h"
@@ -87,4 +87,4 @@ LIBC_INLINE int64_t large_range_reduction(double x, int x_exp, double &y) {
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_H
+#endif // LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_H
diff --git a/libc/src/math/generic/range_reduction_double_common.h b/libc/src/__support/math/range_reduction_double_common.h
similarity index 97%
rename from libc/src/math/generic/range_reduction_double_common.h
rename to libc/src/__support/math/range_reduction_double_common.h
index a93ee25201813..a12c25da4fdd0 100644
--- a/libc/src/math/generic/range_reduction_double_common.h
+++ b/libc/src/__support/math/range_reduction_double_common.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_COMMON_H
-#define LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_COMMON_H
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_DOUBLE_COMMON_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_DOUBLE_COMMON_H
 
 #include "src/__support/FPUtil/double_double.h"
 #include "src/__support/FPUtil/dyadic_float.h"
@@ -20,6 +20,10 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+namespace math {
+
+namespace range_reduction_double_internal {
+
 #ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 static constexpr unsigned SPLIT = fputil::DefaultSplit<double>::VALUE;
 #else
@@ -40,7 +44,7 @@ using Float128 = LIBC_NAMESPACE::fputil::DyadicFloat<128>;
 // Error bound:
 //   |(x - k * pi/128) - (u_hi + u_lo)| <= max(ulp(ulp(u_hi)), 2^-119)
 //                                      <= 2^-111.
-LIBC_INLINE unsigned range_reduction_small(double x, DoubleDouble &u) {
+LIBC_INLINE static unsigned range_reduction_small(double x, DoubleDouble &u) {
   // Values of -pi/128 used for inputs with absolute value <= 2^16.
   // The first 3 parts are generated with (53 - 21 = 32)-bit precision, so that
   // the product k * MPI_OVER_128[i] is exact.
@@ -267,13 +271,15 @@ struct LargeRangeReduction {
   }
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
+  LIBC_INLINE LargeRangeReduction() = default;
+
 private:
   // Index of x in the look-up table ONE_TWENTY_EIGHT_OVER_PI.
-  unsigned idx;
+  unsigned idx = 0;
   // x scaled down by 2^(-16 *(idx - 3))).
-  double x_reduced;
+  double x_reduced = 0;
   // Parts of (x * 128/pi) mod 1.
-  double y_hi, y_lo;
+  double y_hi = 0, y_lo = 0;
   DoubleDouble y_mid;
 };
 
@@ -369,6 +375,10 @@ static constexpr Float128 SIN_K_PI_OVER_128_F128[65] = {
 };
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
+} // namespace range_reduction_double_internal
+
+} // namespace math
+
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_COMMON_H
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_DOUBLE_COMMON_H
diff --git a/libc/src/math/generic/range_reduction_double_fma.h b/libc/src/__support/math/range_reduction_double_fma.h
similarity index 97%
rename from libc/src/math/generic/range_reduction_double_fma.h
rename to libc/src/__support/math/range_reduction_double_fma.h
index 160fb2461fe21..7fa3e40522cf6 100644
--- a/libc/src/math/generic/range_reduction_double_fma.h
+++ b/libc/src/__support/math/range_reduction_double_fma.h
@@ -6,20 +6,22 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_FMA_H
-#define LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_FMA_H
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_DOUBLE_FMA_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_DOUBLE_FMA_H
 
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/double_double.h"
 #include "src/__support/FPUtil/multiply_add.h"
 #include "src/__support/FPUtil/nearest_integer.h"
-#include "src/__support/common.h"
 #include "src/__support/macros/config.h"
-#include "src/__support/macros/optimization.h"
-#include "src/math/generic/range_reduction_double_common.h"
+#include "src/__support/math/range_reduction_double_common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
+namespace math {
+
+namespace range_reduction_double_internal {
+
 using LIBC_NAMESPACE::fputil::DoubleDouble;
 
 LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) {
@@ -341,6 +343,10 @@ LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[] = {
 #endif // !LIBC_MATH_HAS_SMALL_TABLES
 };
 
+} // namespace range_reduction_double_internal
+
+} // namespace math
+
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_FMA_H
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_DOUBLE_FMA_H
diff --git a/libc/src/math/generic/range_reduction_double_nofma.h b/libc/src/__support/math/range_reduction_double_nofma.h
similarity index 97%
rename from libc/src/math/generic/range_reduction_double_nofma.h
rename to libc/src/__support/math/range_reduction_double_nofma.h
index 9d13d246ce91f..3990b9bcbd3bb 100644
--- a/libc/src/math/generic/range_reduction_double_nofma.h
+++ b/libc/src/__support/math/range_reduction_double_nofma.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_NOFMA_H
-#define LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_NOFMA_H
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_DOUBLE_NOFMA_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_DOUBLE_NOFMA_H
 
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/double_double.h"
@@ -16,10 +16,14 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
-#include "src/math/generic/range_reduction_double_common.h"
+#include "src/__support/math/range_reduction_double_common.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
+namespace math {
+
+namespace range_reduction_double_internal {
+
 using fputil::DoubleDouble;
 
 LIBC_INLINE unsigned LargeRangeReduction::fast(double x, DoubleDouble &u) {
@@ -342,6 +346,10 @@ LIBC_INLINE constexpr DoubleDouble SIN_K_PI_OVER_128[] = {
 #endif // !LIBC_MATH_HAS_SMALL_TABLES
 };
 
+} // namespace range_reduction_double_internal
+
+} // namespace math
+
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_DOUBLE_NOFMA_H
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_DOUBLE_NOFMA_H
diff --git a/libc/src/math/generic/range_reduction_fma.h b/libc/src/__support/math/range_reduction_fma.h
similarity index 95%
rename from libc/src/math/generic/range_reduction_fma.h
rename to libc/src/__support/math/range_reduction_fma.h
index 537d5729dc42c..c06a1d8e5760f 100644
--- a/libc/src/math/generic/range_reduction_fma.h
+++ b/libc/src/__support/math/range_reduction_fma.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_FMA_H
-#define LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_FMA_H
+#ifndef LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_FMA_H
+#define LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_FMA_H
 
 #include "src/__support/FPUtil/FMA.h"
 #include "src/__support/FPUtil/FPBits.h"
@@ -89,4 +89,4 @@ LIBC_INLINE int64_t large_range_reduction(double x, int x_exp, double &y) {
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_MATH_GENERIC_RANGE_REDUCTION_FMA_H
+#endif // LIBC_SRC___SUPPORT_MATH_RANGE_REDUCTION_FMA_H
diff --git a/libc/src/math/generic/sincos_eval.h b/libc/src/__support/math/sincos_eval.h
similarity index 98%
rename from libc/src/math/generic/sincos_eval.h
rename to libc/src/__support/math/sincos_eval.h
index 41a4c75849ff4..fc741af19551f 100644
--- a/libc/src/math/generic/sincos_eval.h
+++ b/libc/src/__support/math/sincos_eval.h
@@ -18,7 +18,9 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-namespace generic {
+namespace math {
+
+namespace sincos_eval_internal {
 
 using fputil::DoubleDouble;
 using Float128 = fputil::DyadicFloat<128>;
@@ -131,7 +133,9 @@ LIBC_INLINE void sincos_eval(const Float128 &u, Float128 &sin_u,
                            COS_COEFFS[6]);
 }
 
-} // namespace generic
+} // namespace sincos_eval_internal
+
+} // namespace math
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/math/generic/sincosf16_utils.h b/libc/src/__support/math/sincosf16_utils.h
similarity index 97%
rename from libc/src/math/generic/sincosf16_utils.h
rename to libc/src/__support/math/sincosf16_utils.h
index 05cab09d2089b..74f21fd0a9dc4 100644
--- a/libc/src/math/generic/sincosf16_utils.h
+++ b/libc/src/__support/math/sincosf16_utils.h
@@ -16,6 +16,8 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
+namespace sincosf16_internal {
+
 // Lookup table for sin(k * pi / 32) with k = 0, ..., 63.
 // Table is generated with Sollya as follows:
 // > display = hexadecimmal;
@@ -66,7 +68,7 @@ LIBC_INLINE int32_t range_reduction_sincosf16(float x, float &y) {
   return static_cast<int32_t>(kd);
 }
 
-static LIBC_INLINE void sincosf16_poly_eval(int32_t k, float y, float &sin_k,
+LIBC_INLINE static void sincosf16_poly_eval(int32_t k, float y, float &sin_k,
                                             float &cos_k, float &sin_y,
                                             float &cosm1_y) {
 
@@ -107,6 +109,8 @@ LIBC_INLINE void sincospif16_eval(float xf, float &sin_k, float &cos_k,
   sincosf16_poly_eval(k, y, sin_k, cos_k, sin_y, cosm1_y);
 }
 
+} // namespace sincosf16_internal
+
 } // namespace LIBC_NAMESPACE_DECL
 
 #endif // LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF16_UTILS_H
diff --git a/libc/src/math/generic/sincosf_utils.h b/libc/src/__support/math/sincosf_utils.h
similarity index 97%
rename from libc/src/math/generic/sincosf_utils.h
rename to libc/src/__support/math/sincosf_utils.h
index 6eaf820e5c1b0..ed9d9f6bda0eb 100644
--- a/libc/src/math/generic/sincosf_utils.h
+++ b/libc/src/__support/math/sincosf_utils.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF_UTILS_H
-#define LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF_UTILS_H
+#ifndef LIBC_SRC___SUPPORT_MATH_SINCOSF_UTILS_H
+#define LIBC_SRC___SUPPORT_MATH_SINCOSF_UTILS_H
 
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
@@ -122,4 +122,4 @@ LIBC_INLINE void sincospif_eval(double xd, double &sin_k, double &cos_k,
 
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_MATH_GENERIC_SINCOSF_UTILS_H
+#endif // LIBC_SRC___SUPPORT_MATH_SINCOSF_UTILS_H
diff --git a/libc/src/math/generic/explogxf.h b/libc/src/__support/math/sinhfcoshf_utils.h
similarity index 89%
rename from libc/src/math/generic/explogxf.h
rename to libc/src/__support/math/sinhfcoshf_utils.h
index 72f8da8c72c5e..5f19b81246980 100644
--- a/libc/src/math/generic/explogxf.h
+++ b/libc/src/__support/math/sinhfcoshf_utils.h
@@ -1,4 +1,4 @@
-//===-- Single-precision general exp/log functions ------------------------===//
+//===-- Single-precision general sinhf/coshf functions --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,21 +6,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIBC_SRC_MATH_GENERIC_EXPLOGXF_H
-#define LLVM_LIBC_SRC_MATH_GENERIC_EXPLOGXF_H
+#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_SINHFCOSHF_UTILS_H
+#define LLVM_LIBC_SRC___SUPPORT_MATH_SINHFCOSHF_UTILS_H
 
-#include "common_constants.h"
-
-#include "src/__support/common.h"
-#include "src/__support/macros/properties/cpu_features.h"
-#include "src/__support/math/acoshf_utils.h"
-#include "src/__support/math/exp10f_utils.h"
-#include "src/__support/math/exp_utils.h"
+#include "exp10f_utils.h"
+#include "src/__support/FPUtil/multiply_add.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-constexpr int LOG_P1_BITS = 6;
-constexpr int LOG_P1_SIZE = 1 << LOG_P1_BITS;
+namespace math {
+
+namespace sinhfcoshf_internal {
 
 // The function correctly calculates sinh(x) and cosh(x) by calculating exp(x)
 // and exp(-x) simultaneously.
@@ -121,6 +117,10 @@ template <bool is_sinh> LIBC_INLINE double exp_pm_eval(float x) {
   return r;
 }
 
+} // namespace sinhfcoshf_internal
+
+} // namespace math
+
 } // namespace LIBC_NAMESPACE_DECL
 
-#endif // LLVM_LIBC_SRC_MATH_GENERIC_EXPLOGXF_H
+#endif // LLVM_LIBC_SRC___SUPPORT_MATH_SINHFCOSHF_UTILS_H
diff --git a/libc/src/__support/math_extras.h b/libc/src/__support/math_extras.h
index 47df2a43250c8..954bcb1b6ef89 100644
--- a/libc/src/__support/math_extras.h
+++ b/libc/src/__support/math_extras.h
@@ -66,7 +66,7 @@ template <typename T>
 
 #define RETURN_IF(TYPE, BUILTIN)                                               \
   if constexpr (cpp::is_same_v<T, TYPE>)                                       \
-    return BUILTIN(a, b, carry_in, carry_out);
+    return BUILTIN(a, b, carry_in, &carry_out);
 
 // Returns the result of 'a + b' taking into account 'carry_in'.
 // The carry out is stored in 'carry_out' it not 'nullptr', dropped otherwise.
@@ -74,7 +74,7 @@ template <typename T>
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_unsigned_v<T>, T>
 add_with_carry(T a, T b, T carry_in, T &carry_out) {
-  if constexpr (!cpp::is_constant_evaluated()) {
+  if (!cpp::is_constant_evaluated()) {
 #if __has_builtin(__builtin_addcb)
     RETURN_IF(unsigned char, __builtin_addcb)
 #elif __has_builtin(__builtin_addcs)
@@ -100,7 +100,7 @@ add_with_carry(T a, T b, T carry_in, T &carry_out) {
 template <typename T>
 [[nodiscard]] LIBC_INLINE constexpr cpp::enable_if_t<cpp::is_unsigned_v<T>, T>
 sub_with_borrow(T a, T b, T carry_in, T &carry_out) {
-  if constexpr (!cpp::is_constant_evaluated()) {
+  if (!cpp::is_constant_evaluated()) {
 #if __has_builtin(__builtin_subcb)
     RETURN_IF(unsigned char, __builtin_subcb)
 #elif __has_builtin(__builtin_subcs)
diff --git a/libc/src/__support/threads/linux/barrier.h b/libc/src/__support/threads/linux/barrier.h
index f0655bfc52a10..a632aa45b2aa2 100644
--- a/libc/src/__support/threads/linux/barrier.h
+++ b/libc/src/__support/threads/linux/barrier.h
@@ -36,14 +36,19 @@ class Barrier {
   int wait();
 };
 
-static_assert(
-    sizeof(Barrier) == sizeof(pthread_barrier_t),
-    "The public pthread_barrier_t type cannot accommodate the internal "
-    "barrier type.");
-
-static_assert(alignof(Barrier) == alignof(pthread_barrier_t),
-              "The public pthread_barrier_t type has a different alignment "
-              "than the internal barrier type.");
+static_assert(sizeof(Barrier) <= sizeof(pthread_barrier_t),
+              "The public pthread_barrier_t type cannot accommodate the "
+              "internal barrier type.");
+
+static_assert(alignof(Barrier) <= alignof(pthread_barrier_t),
+              "The public pthread_barrier_t type has insufficient alignment "
+              "for the internal barrier type.");
+
+static_assert(sizeof(CndVar) <= 24,
+              "CndVar size exceeds the size in __barrier_type.h");
+
+static_assert(sizeof(Mutex) <= 24,
+              "Mutex size exceeds the size in __barrier_type.h");
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/__support/threads/mutex.h b/libc/src/__support/threads/mutex.h
index cbef0d00009b2..f64f7e7b40082 100644
--- a/libc/src/__support/threads/mutex.h
+++ b/libc/src/__support/threads/mutex.h
@@ -12,28 +12,6 @@
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/config.h"
 
-// Uses the platform specific specialization
-#define LIBC_THREAD_MODE_PLATFORM 0
-
-// Mutex guards nothing, used in single-threaded implementations
-#define LIBC_THREAD_MODE_SINGLE 1
-
-// Vendor provides implementation
-#define LIBC_THREAD_MODE_EXTERNAL 2
-
-#if !defined(LIBC_THREAD_MODE)
-#error LIBC_THREAD_MODE is undefined
-#endif // LIBC_THREAD_MODE
-
-#if LIBC_THREAD_MODE != LIBC_THREAD_MODE_PLATFORM &&                           \
-    LIBC_THREAD_MODE != LIBC_THREAD_MODE_SINGLE &&                             \
-    LIBC_THREAD_MODE != LIBC_THREAD_MODE_EXTERNAL
-#error LIBC_THREAD_MODE must be one of the following values: \
-LIBC_THREAD_MODE_PLATFORM, \
-LIBC_THREAD_MODE_SINGLE, \
-LIBC_THREAD_MODE_EXTERNAL.
-#endif
-
 #if LIBC_THREAD_MODE == LIBC_THREAD_MODE_PLATFORM
 
 // Platform independent code will include this header file which pulls
diff --git a/libc/src/__support/wchar/character_converter.cpp b/libc/src/__support/wchar/character_converter.cpp
index 15d0f478a18a9..26672884d7b16 100644
--- a/libc/src/__support/wchar/character_converter.cpp
+++ b/libc/src/__support/wchar/character_converter.cpp
@@ -132,12 +132,6 @@ ErrorOr<char32_t> CharacterConverter::pop_utf32() {
   return utf32;
 }
 
-size_t CharacterConverter::sizeAsUTF32() {
-  return 1; // a single utf-32 value can fit an entire character
-}
-
-size_t CharacterConverter::sizeAsUTF8() { return state->total_bytes; }
-
 ErrorOr<char8_t> CharacterConverter::pop_utf8() {
   if (isEmpty())
     return Error(-1);
@@ -170,5 +164,13 @@ ErrorOr<char8_t> CharacterConverter::pop_utf8() {
   return static_cast<char8_t>(output);
 }
 
+template <> ErrorOr<char8_t> CharacterConverter::pop() { return pop_utf8(); }
+template <> ErrorOr<char32_t> CharacterConverter::pop() { return pop_utf32(); }
+
+template <> size_t CharacterConverter::sizeAs<char8_t>() {
+  return state->total_bytes;
+}
+template <> size_t CharacterConverter::sizeAs<char32_t>() { return 1; }
+
 } // namespace internal
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index b6d918f2d2edc..2cc28abf2772a 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -12,6 +12,7 @@
 #include "hdr/types/char32_t.h"
 #include "hdr/types/char8_t.h"
 #include "hdr/types/size_t.h"
+#include "src/__support/CPP/type_traits.h"
 #include "src/__support/common.h"
 #include "src/__support/error_or.h"
 #include "src/__support/wchar/mbstate.h"
@@ -31,14 +32,14 @@ class CharacterConverter {
   bool isEmpty();
   bool isValidState();
 
-  size_t sizeAsUTF32();
-  size_t sizeAsUTF8();
+  template <typename CharType> size_t sizeAs();
 
   int push(char8_t utf8_byte);
   int push(char32_t utf32);
 
   ErrorOr<char8_t> pop_utf8();
   ErrorOr<char32_t> pop_utf32();
+  template <typename CharType> ErrorOr<CharType> pop();
 };
 
 } // namespace internal
diff --git a/libc/src/__support/wchar/mbsnrtowcs.h b/libc/src/__support/wchar/mbsnrtowcs.h
index 54e315210d95c..6abb836635772 100644
--- a/libc/src/__support/wchar/mbsnrtowcs.h
+++ b/libc/src/__support/wchar/mbsnrtowcs.h
@@ -36,7 +36,7 @@ LIBC_INLINE static ErrorOr<size_t> mbsnrtowcs(wchar_t *__restrict dst,
   StringConverter<char8_t> str_conv(reinterpret_cast<const char8_t *>(*src), ps,
                                     len, nmc);
   size_t dst_idx = 0;
-  ErrorOr<char32_t> converted = str_conv.popUTF32();
+  ErrorOr<char32_t> converted = str_conv.pop<char32_t>();
   while (converted.has_value()) {
     if (dst != nullptr)
       dst[dst_idx] = converted.value();
@@ -47,7 +47,7 @@ LIBC_INLINE static ErrorOr<size_t> mbsnrtowcs(wchar_t *__restrict dst,
       return dst_idx;
     }
     dst_idx++;
-    converted = str_conv.popUTF32();
+    converted = str_conv.pop<char32_t>();
   }
 
   if (converted.error() == -1) { // if we hit conversion limit
diff --git a/libc/src/__support/wchar/string_converter.h b/libc/src/__support/wchar/string_converter.h
index 869ebdfc8b390..ba628bd34cdc0 100644
--- a/libc/src/__support/wchar/string_converter.h
+++ b/libc/src/__support/wchar/string_converter.h
@@ -12,6 +12,7 @@
 #include "hdr/types/char32_t.h"
 #include "hdr/types/char8_t.h"
 #include "hdr/types/size_t.h"
+#include "src/__support/CPP/type_traits.h"
 #include "src/__support/common.h"
 #include "src/__support/error_or.h"
 #include "src/__support/wchar/character_converter.h"
@@ -53,9 +54,7 @@ template <typename T> class StringConverter {
                   size_t srclen = SIZE_MAX)
       : cr(ps), src(s), src_len(srclen), src_idx(0), num_to_write(dstlen) {}
 
-  // TODO: following functions are almost identical
-  // look into templating CharacterConverter pop functions
-  ErrorOr<char32_t> popUTF32() {
+  template <typename CharType> ErrorOr<CharType> pop() {
     if (num_to_write == 0)
       return Error(-1);
 
@@ -64,7 +63,7 @@ template <typename T> class StringConverter {
       if (!src_elements_read.has_value())
         return Error(src_elements_read.error());
 
-      if (cr.sizeAsUTF32() > num_to_write) {
+      if (cr.sizeAs<CharType>() > num_to_write) {
         cr.clear();
         return Error(-1);
       }
@@ -72,34 +71,9 @@ template <typename T> class StringConverter {
       src_idx += src_elements_read.value();
     }
 
-    auto out = cr.pop_utf32();
-    if (out.has_value() && out.value() == L'\0')
-      src_len = src_idx;
-
-    num_to_write--;
-
-    return out;
-  }
-
-  ErrorOr<char8_t> popUTF8() {
-    if (num_to_write == 0)
-      return Error(-1);
-
-    if (cr.isEmpty() || src_idx == 0) {
-      auto src_elements_read = pushFullCharacter();
-      if (!src_elements_read.has_value())
-        return Error(src_elements_read.error());
-
-      if (cr.sizeAsUTF8() > num_to_write) {
-        cr.clear();
-        return Error(-1);
-      }
-
-      src_idx += src_elements_read.value();
-    }
-
-    auto out = cr.pop_utf8();
-    if (out.has_value() && out.value() == '\0')
+    ErrorOr<CharType> out = cr.pop<CharType>();
+    // if out isn't null terminator or an error
+    if (out.has_value() && out.value() == 0)
       src_len = src_idx;
 
     num_to_write--;
diff --git a/libc/src/__support/wchar/wcsnrtombs.h b/libc/src/__support/wchar/wcsnrtombs.h
index 433097c937a42..f593a0e0dba87 100644
--- a/libc/src/__support/wchar/wcsnrtombs.h
+++ b/libc/src/__support/wchar/wcsnrtombs.h
@@ -39,7 +39,7 @@ wcsnrtombs(char *__restrict dest, const wchar_t **__restrict ptr_to_src,
       reinterpret_cast<const char32_t *>(*ptr_to_src), ps, dest_len,
       num_src_widechars);
   size_t dst_idx = 0;
-  ErrorOr<char8_t> converted = str_conv.popUTF8();
+  ErrorOr<char8_t> converted = str_conv.pop<char8_t>();
   while (converted.has_value()) {
     if (dest != nullptr)
       dest[dst_idx] = converted.value();
@@ -51,7 +51,7 @@ wcsnrtombs(char *__restrict dest, const wchar_t **__restrict ptr_to_src,
     }
 
     dst_idx++;
-    converted = str_conv.popUTF8();
+    converted = str_conv.pop<char8_t>();
   }
 
   if (dest != nullptr)
diff --git a/libc/src/dlfcn/CMakeLists.txt b/libc/src/dlfcn/CMakeLists.txt
index 8ef0540c01a2e..2ee3ac074267f 100644
--- a/libc/src/dlfcn/CMakeLists.txt
+++ b/libc/src/dlfcn/CMakeLists.txt
@@ -14,7 +14,6 @@ add_entrypoint_object(
     dlerror.h
   DEPENDS
     libc.include.dlfcn
-    libc.src.errno.errno
 )
 
 add_entrypoint_object(
@@ -25,7 +24,6 @@ add_entrypoint_object(
     dlopen.h
   DEPENDS
     libc.include.dlfcn
-    libc.src.errno.errno
 )
 
 add_entrypoint_object(
@@ -36,7 +34,6 @@ add_entrypoint_object(
     dlsym.h
   DEPENDS
     libc.include.dlfcn
-    libc.src.errno.errno
 )
 
 add_entrypoint_object(
@@ -47,7 +44,6 @@ add_entrypoint_object(
     dlinfo.h
   DEPENDS
     libc.include.dlfcn
-    libc.src.errno.errno
 )
 
 add_entrypoint_object(
@@ -58,5 +54,4 @@ add_entrypoint_object(
     dladdr.h
   DEPENDS
     libc.include.dlfcn
-    libc.src.errno.errno
 )
diff --git a/libc/src/dlfcn/dladdr.cpp b/libc/src/dlfcn/dladdr.cpp
index 61490fd9a64b5..3db68b4a4c729 100644
--- a/libc/src/dlfcn/dladdr.cpp
+++ b/libc/src/dlfcn/dladdr.cpp
@@ -14,7 +14,8 @@
 namespace LIBC_NAMESPACE_DECL {
 
 // TODO: https:// github.com/llvm/llvm-project/issues/97929
-LLVM_LIBC_FUNCTION(int, dladdr, (const void *addr, Dl_info *info)) {
+LLVM_LIBC_FUNCTION(int, dladdr,
+                   (const void *__restrict addr, Dl_info *__restrict info)) {
   return -1;
 }
 
diff --git a/libc/src/dlfcn/dladdr.h b/libc/src/dlfcn/dladdr.h
index 346fc8dc27aed..abbc9a9008d92 100644
--- a/libc/src/dlfcn/dladdr.h
+++ b/libc/src/dlfcn/dladdr.h
@@ -13,7 +13,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-int dladdr(const void *, Dl_info *);
+int dladdr(const void *__restrict, Dl_info *__restrict);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/dlfcn/dlinfo.cpp b/libc/src/dlfcn/dlinfo.cpp
index d78cade5ea593..e1938d19c9657 100644
--- a/libc/src/dlfcn/dlinfo.cpp
+++ b/libc/src/dlfcn/dlinfo.cpp
@@ -16,7 +16,8 @@ namespace LIBC_NAMESPACE_DECL {
 
 // TODO: https://github.com/llvm/llvm-project/issues/149911
 LLVM_LIBC_FUNCTION(int, dlinfo,
-                   (void *restrict handle, int request, void *restrict info)) {
+                   (void *__restrict handle, int request,
+                    void *__restrict info)) {
   return -1;
 }
 
diff --git a/libc/src/dlfcn/dlinfo.h b/libc/src/dlfcn/dlinfo.h
index c2c34f02bd6f1..bc13152e40f44 100644
--- a/libc/src/dlfcn/dlinfo.h
+++ b/libc/src/dlfcn/dlinfo.h
@@ -13,7 +13,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-int dlinfo(void *restrict, int, void *restrict);
+int dlinfo(void *__restrict, int, void *__restrict);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/dlfcn/dlsym.cpp b/libc/src/dlfcn/dlsym.cpp
index c075c203bd583..dc0da7d6d120f 100644
--- a/libc/src/dlfcn/dlsym.cpp
+++ b/libc/src/dlfcn/dlsym.cpp
@@ -14,6 +14,8 @@
 namespace LIBC_NAMESPACE_DECL {
 
 // TODO(@izaakschroeder): https://github.com/llvm/llvm-project/issues/97920
-LLVM_LIBC_FUNCTION(void *, dlsym, (void *, const char *)) { return nullptr; }
+LLVM_LIBC_FUNCTION(void *, dlsym, (void *__restrict, const char *__restrict)) {
+  return nullptr;
+}
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/dlfcn/dlsym.h b/libc/src/dlfcn/dlsym.h
index 70c6ab3fbb143..f879792d3c5f6 100644
--- a/libc/src/dlfcn/dlsym.h
+++ b/libc/src/dlfcn/dlsym.h
@@ -13,7 +13,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-void *dlsym(void *, const char *);
+void *dlsym(void *__restrict, const char *__restrict);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/src/errno/libc_errno.cpp b/libc/src/errno/libc_errno.cpp
index 8ff1eec1b1035..e8960fcd89500 100644
--- a/libc/src/errno/libc_errno.cpp
+++ b/libc/src/errno/libc_errno.cpp
@@ -46,11 +46,6 @@ Errno::operator int() { return shared_errno; }
 void Errno::operator=(int a) { *__llvm_libc_errno() = a; }
 Errno::operator int() { return *__llvm_libc_errno(); }
 
-#elif LIBC_ERRNO_MODE == LIBC_ERRNO_MODE_SYSTEM
-
-void Errno::operator=(int a) { errno = a; }
-Errno::operator int() { return errno; }
-
 #endif
 
 // Define the global `libc_errno` instance.
diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt
index 0522e0e15bad2..cbafa63b9c0d6 100644
--- a/libc/src/math/CMakeLists.txt
+++ b/libc/src/math/CMakeLists.txt
@@ -58,6 +58,8 @@ add_math_entrypoint_object(asinh)
 add_math_entrypoint_object(asinhf)
 add_math_entrypoint_object(asinhf16)
 
+add_math_entrypoint_object(asinpif16)
+
 add_math_entrypoint_object(atan)
 add_math_entrypoint_object(atanf)
 add_math_entrypoint_object(atanf16)
@@ -91,6 +93,7 @@ add_math_entrypoint_object(ceilf)
 add_math_entrypoint_object(ceill)
 add_math_entrypoint_object(ceilf16)
 add_math_entrypoint_object(ceilf128)
+add_math_entrypoint_object(ceilbf16)
 
 add_math_entrypoint_object(copysign)
 add_math_entrypoint_object(copysignf)
@@ -214,6 +217,7 @@ add_math_entrypoint_object(floorf)
 add_math_entrypoint_object(floorl)
 add_math_entrypoint_object(floorf16)
 add_math_entrypoint_object(floorf128)
+add_math_entrypoint_object(floorbf16)
 
 add_math_entrypoint_object(fma)
 add_math_entrypoint_object(fmaf)
@@ -224,60 +228,70 @@ add_math_entrypoint_object(fmaxf)
 add_math_entrypoint_object(fmaxl)
 add_math_entrypoint_object(fmaxf128)
 add_math_entrypoint_object(fmaxf16)
+add_math_entrypoint_object(fmaxbf16)
 
 add_math_entrypoint_object(fmin)
 add_math_entrypoint_object(fminf)
 add_math_entrypoint_object(fminl)
 add_math_entrypoint_object(fminf128)
 add_math_entrypoint_object(fminf16)
+add_math_entrypoint_object(fminbf16)
 
 add_math_entrypoint_object(fmaximum)
 add_math_entrypoint_object(fmaximumf)
 add_math_entrypoint_object(fmaximuml)
 add_math_entrypoint_object(fmaximumf16)
 add_math_entrypoint_object(fmaximumf128)
+add_math_entrypoint_object(fmaximumbf16)
 
 add_math_entrypoint_object(fmaximum_num)
 add_math_entrypoint_object(fmaximum_numf)
 add_math_entrypoint_object(fmaximum_numl)
 add_math_entrypoint_object(fmaximum_numf16)
 add_math_entrypoint_object(fmaximum_numf128)
+add_math_entrypoint_object(fmaximum_numbf16)
 
 add_math_entrypoint_object(fmaximum_mag)
 add_math_entrypoint_object(fmaximum_magf)
 add_math_entrypoint_object(fmaximum_magl)
 add_math_entrypoint_object(fmaximum_magf16)
 add_math_entrypoint_object(fmaximum_magf128)
+add_math_entrypoint_object(fmaximum_magbf16)
 
 add_math_entrypoint_object(fmaximum_mag_num)
 add_math_entrypoint_object(fmaximum_mag_numf)
 add_math_entrypoint_object(fmaximum_mag_numl)
 add_math_entrypoint_object(fmaximum_mag_numf16)
 add_math_entrypoint_object(fmaximum_mag_numf128)
+add_math_entrypoint_object(fmaximum_mag_numbf16)
 
 add_math_entrypoint_object(fminimum)
 add_math_entrypoint_object(fminimumf)
 add_math_entrypoint_object(fminimuml)
 add_math_entrypoint_object(fminimumf16)
 add_math_entrypoint_object(fminimumf128)
+add_math_entrypoint_object(fminimumbf16)
 
 add_math_entrypoint_object(fminimum_num)
 add_math_entrypoint_object(fminimum_numf)
 add_math_entrypoint_object(fminimum_numl)
 add_math_entrypoint_object(fminimum_numf16)
 add_math_entrypoint_object(fminimum_numf128)
+add_math_entrypoint_object(fminimum_numbf16)
 
 add_math_entrypoint_object(fminimum_mag)
 add_math_entrypoint_object(fminimum_magf)
 add_math_entrypoint_object(fminimum_magl)
 add_math_entrypoint_object(fminimum_magf16)
 add_math_entrypoint_object(fminimum_magf128)
+add_math_entrypoint_object(fminimum_magbf16)
 
 add_math_entrypoint_object(fminimum_mag_num)
 add_math_entrypoint_object(fminimum_mag_numf)
 add_math_entrypoint_object(fminimum_mag_numl)
 add_math_entrypoint_object(fminimum_mag_numf16)
 add_math_entrypoint_object(fminimum_mag_numf128)
+add_math_entrypoint_object(fminimum_mag_numbf16)
 
 add_math_entrypoint_object(fmul)
 add_math_entrypoint_object(fmull)
@@ -294,18 +308,21 @@ add_math_entrypoint_object(frexpf)
 add_math_entrypoint_object(frexpl)
 add_math_entrypoint_object(frexpf16)
 add_math_entrypoint_object(frexpf128)
+add_math_entrypoint_object(frexpbf16)
 
 add_math_entrypoint_object(fromfp)
 add_math_entrypoint_object(fromfpf)
 add_math_entrypoint_object(fromfpl)
 add_math_entrypoint_object(fromfpf16)
 add_math_entrypoint_object(fromfpf128)
+add_math_entrypoint_object(fromfpbf16)
 
 add_math_entrypoint_object(fromfpx)
 add_math_entrypoint_object(fromfpxf)
 add_math_entrypoint_object(fromfpxl)
 add_math_entrypoint_object(fromfpxf16)
 add_math_entrypoint_object(fromfpxf128)
+add_math_entrypoint_object(fromfpxbf16)
 
 add_math_entrypoint_object(fsub)
 add_math_entrypoint_object(fsubl)
@@ -316,6 +333,7 @@ add_math_entrypoint_object(getpayloadf)
 add_math_entrypoint_object(getpayloadl)
 add_math_entrypoint_object(getpayloadf16)
 add_math_entrypoint_object(getpayloadf128)
+add_math_entrypoint_object(getpayloadbf16)
 
 add_math_entrypoint_object(hypot)
 add_math_entrypoint_object(hypotf)
@@ -326,6 +344,7 @@ add_math_entrypoint_object(ilogbf)
 add_math_entrypoint_object(ilogbl)
 add_math_entrypoint_object(ilogbf16)
 add_math_entrypoint_object(ilogbf128)
+add_math_entrypoint_object(ilogbbf16)
 
 add_math_entrypoint_object(isnan)
 add_math_entrypoint_object(isnanf)
@@ -342,12 +361,14 @@ add_math_entrypoint_object(llogbf)
 add_math_entrypoint_object(llogbl)
 add_math_entrypoint_object(llogbf16)
 add_math_entrypoint_object(llogbf128)
+add_math_entrypoint_object(llogbbf16)
 
 add_math_entrypoint_object(ldexp)
 add_math_entrypoint_object(ldexpf)
 add_math_entrypoint_object(ldexpl)
 add_math_entrypoint_object(ldexpf16)
 add_math_entrypoint_object(ldexpf128)
+add_math_entrypoint_object(ldexpbf16)
 
 add_math_entrypoint_object(log10)
 add_math_entrypoint_object(log10f)
@@ -369,30 +390,35 @@ add_math_entrypoint_object(logbf)
 add_math_entrypoint_object(logbl)
 add_math_entrypoint_object(logbf16)
 add_math_entrypoint_object(logbf128)
+add_math_entrypoint_object(logbbf16)
 
 add_math_entrypoint_object(llrint)
 add_math_entrypoint_object(llrintf)
 add_math_entrypoint_object(llrintl)
 add_math_entrypoint_object(llrintf16)
 add_math_entrypoint_object(llrintf128)
+add_math_entrypoint_object(llrintbf16)
 
 add_math_entrypoint_object(llround)
 add_math_entrypoint_object(llroundf)
 add_math_entrypoint_object(llroundl)
 add_math_entrypoint_object(llroundf16)
 add_math_entrypoint_object(llroundf128)
+add_math_entrypoint_object(llroundbf16)
 
 add_math_entrypoint_object(lrint)
 add_math_entrypoint_object(lrintf)
 add_math_entrypoint_object(lrintl)
 add_math_entrypoint_object(lrintf16)
 add_math_entrypoint_object(lrintf128)
+add_math_entrypoint_object(lrintbf16)
 
 add_math_entrypoint_object(lround)
 add_math_entrypoint_object(lroundf)
 add_math_entrypoint_object(lroundl)
 add_math_entrypoint_object(lroundf16)
 add_math_entrypoint_object(lroundf128)
+add_math_entrypoint_object(lroundbf16)
 
 add_math_entrypoint_object(modf)
 add_math_entrypoint_object(modff)
@@ -405,35 +431,41 @@ add_math_entrypoint_object(nanf)
 add_math_entrypoint_object(nanl)
 add_math_entrypoint_object(nanf16)
 add_math_entrypoint_object(nanf128)
+add_math_entrypoint_object(nanbf16)
 
 add_math_entrypoint_object(nearbyint)
 add_math_entrypoint_object(nearbyintf)
 add_math_entrypoint_object(nearbyintl)
 add_math_entrypoint_object(nearbyintf16)
 add_math_entrypoint_object(nearbyintf128)
+add_math_entrypoint_object(nearbyintbf16)
 
 add_math_entrypoint_object(nextafter)
 add_math_entrypoint_object(nextafterf)
 add_math_entrypoint_object(nextafterl)
 add_math_entrypoint_object(nextafterf16)
 add_math_entrypoint_object(nextafterf128)
+add_math_entrypoint_object(nextafterbf16)
 
 add_math_entrypoint_object(nexttoward)
 add_math_entrypoint_object(nexttowardf)
 add_math_entrypoint_object(nexttowardl)
 add_math_entrypoint_object(nexttowardf16)
+add_math_entrypoint_object(nexttowardbf16)
 
 add_math_entrypoint_object(nextdown)
 add_math_entrypoint_object(nextdownf)
 add_math_entrypoint_object(nextdownl)
 add_math_entrypoint_object(nextdownf16)
 add_math_entrypoint_object(nextdownf128)
+add_math_entrypoint_object(nextdownbf16)
 
 add_math_entrypoint_object(nextup)
 add_math_entrypoint_object(nextupf)
 add_math_entrypoint_object(nextupl)
 add_math_entrypoint_object(nextupf16)
 add_math_entrypoint_object(nextupf128)
+add_math_entrypoint_object(nextupbf16)
 
 add_math_entrypoint_object(pow)
 add_math_entrypoint_object(powf)
@@ -457,18 +489,21 @@ add_math_entrypoint_object(rintf)
 add_math_entrypoint_object(rintl)
 add_math_entrypoint_object(rintf16)
 add_math_entrypoint_object(rintf128)
+add_math_entrypoint_object(rintbf16)
 
 add_math_entrypoint_object(round)
 add_math_entrypoint_object(roundf)
 add_math_entrypoint_object(roundl)
 add_math_entrypoint_object(roundf16)
 add_math_entrypoint_object(roundf128)
+add_math_entrypoint_object(roundbf16)
 
 add_math_entrypoint_object(roundeven)
 add_math_entrypoint_object(roundevenf)
 add_math_entrypoint_object(roundevenl)
 add_math_entrypoint_object(roundevenf16)
 add_math_entrypoint_object(roundevenf128)
+add_math_entrypoint_object(roundevenbf16)
 
 add_math_entrypoint_object(scalbln)
 add_math_entrypoint_object(scalblnf)
@@ -487,12 +522,14 @@ add_math_entrypoint_object(setpayloadf)
 add_math_entrypoint_object(setpayloadl)
 add_math_entrypoint_object(setpayloadf16)
 add_math_entrypoint_object(setpayloadf128)
+add_math_entrypoint_object(setpayloadbf16)
 
 add_math_entrypoint_object(setpayloadsig)
 add_math_entrypoint_object(setpayloadsigf)
 add_math_entrypoint_object(setpayloadsigl)
 add_math_entrypoint_object(setpayloadsigf16)
 add_math_entrypoint_object(setpayloadsigf128)
+add_math_entrypoint_object(setpayloadsigbf16)
 
 add_math_entrypoint_object(sincos)
 add_math_entrypoint_object(sincosf)
@@ -546,15 +583,43 @@ add_math_entrypoint_object(truncf)
 add_math_entrypoint_object(truncl)
 add_math_entrypoint_object(truncf16)
 add_math_entrypoint_object(truncf128)
+add_math_entrypoint_object(truncbf16)
 
 add_math_entrypoint_object(ufromfp)
 add_math_entrypoint_object(ufromfpf)
 add_math_entrypoint_object(ufromfpl)
 add_math_entrypoint_object(ufromfpf16)
 add_math_entrypoint_object(ufromfpf128)
+add_math_entrypoint_object(ufromfpbf16)
 
 add_math_entrypoint_object(ufromfpx)
 add_math_entrypoint_object(ufromfpxf)
 add_math_entrypoint_object(ufromfpxl)
 add_math_entrypoint_object(ufromfpxf16)
 add_math_entrypoint_object(ufromfpxf128)
+add_math_entrypoint_object(ufromfpxbf16)
+
+add_math_entrypoint_object(bf16add)
+add_math_entrypoint_object(bf16addf)
+add_math_entrypoint_object(bf16addl)
+add_math_entrypoint_object(bf16addf128)
+
+add_math_entrypoint_object(bf16div)
+add_math_entrypoint_object(bf16divf)
+add_math_entrypoint_object(bf16divl)
+add_math_entrypoint_object(bf16divf128)
+
+add_math_entrypoint_object(bf16fma)
+add_math_entrypoint_object(bf16fmaf)
+add_math_entrypoint_object(bf16fmal)
+add_math_entrypoint_object(bf16fmaf128)
+
+add_math_entrypoint_object(bf16mul)
+add_math_entrypoint_object(bf16mulf)
+add_math_entrypoint_object(bf16mull)
+add_math_entrypoint_object(bf16mulf128)
+
+add_math_entrypoint_object(bf16sub)
+add_math_entrypoint_object(bf16subf)
+add_math_entrypoint_object(bf16subl)
+add_math_entrypoint_object(bf16subf128)
diff --git a/libc/src/math/asinpif16.h b/libc/src/math/asinpif16.h
new file mode 100644
index 0000000000000..b97166af63f5d
--- /dev/null
+++ b/libc/src/math/asinpif16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for asinpif16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ASINPIF16_H
+#define LLVM_LIBC_SRC_MATH_ASINPIF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+float16 asinpif16(float16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ASINPIF16_H
diff --git a/libc/src/math/bf16add.h b/libc/src/math/bf16add.h
new file mode 100644
index 0000000000000..a29970eb334fa
--- /dev/null
+++ b/libc/src/math/bf16add.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16add -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16ADD_H
+#define LLVM_LIBC_SRC_MATH_BF16ADD_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16add(double x, double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16ADD_H
diff --git a/libc/src/math/bf16addf.h b/libc/src/math/bf16addf.h
new file mode 100644
index 0000000000000..80a5e2a7640df
--- /dev/null
+++ b/libc/src/math/bf16addf.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16addf ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16ADDF_H
+#define LLVM_LIBC_SRC_MATH_BF16ADDF_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16addf(float x, float y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16ADDF_H
diff --git a/libc/src/math/bf16addf128.h b/libc/src/math/bf16addf128.h
new file mode 100644
index 0000000000000..3c2f3a15eb39c
--- /dev/null
+++ b/libc/src/math/bf16addf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16addf128 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16ADDF128_H
+#define LLVM_LIBC_SRC_MATH_BF16ADDF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16addf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16ADDF128_H
diff --git a/libc/src/math/bf16addl.h b/libc/src/math/bf16addl.h
new file mode 100644
index 0000000000000..a9e7d68660728
--- /dev/null
+++ b/libc/src/math/bf16addl.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16addl ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16ADDL_H
+#define LLVM_LIBC_SRC_MATH_BF16ADDL_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16addl(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16ADDL_H
diff --git a/libc/src/math/bf16div.h b/libc/src/math/bf16div.h
new file mode 100644
index 0000000000000..ade9c067d3c3a
--- /dev/null
+++ b/libc/src/math/bf16div.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16div -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16DIV_H
+#define LLVM_LIBC_SRC_MATH_BF16DIV_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16div(double x, double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16DIV_H
diff --git a/libc/src/math/bf16divf.h b/libc/src/math/bf16divf.h
new file mode 100644
index 0000000000000..481b176e01ea3
--- /dev/null
+++ b/libc/src/math/bf16divf.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16divf ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16DIVF_H
+#define LLVM_LIBC_SRC_MATH_BF16DIVF_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16divf(float x, float y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16DIVF_H
diff --git a/libc/src/math/bf16divf128.h b/libc/src/math/bf16divf128.h
new file mode 100644
index 0000000000000..d99006695d1d7
--- /dev/null
+++ b/libc/src/math/bf16divf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16divf128 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16DIVF128_H
+#define LLVM_LIBC_SRC_MATH_BF16DIVF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16divf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16DIVF128_H
diff --git a/libc/src/math/bf16divl.h b/libc/src/math/bf16divl.h
new file mode 100644
index 0000000000000..b19ac873af3f0
--- /dev/null
+++ b/libc/src/math/bf16divl.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16divl ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16DIVL_H
+#define LLVM_LIBC_SRC_MATH_BF16DIVL_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16divl(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16DIVL_H
diff --git a/libc/src/math/bf16fma.h b/libc/src/math/bf16fma.h
new file mode 100644
index 0000000000000..aa54956ef4783
--- /dev/null
+++ b/libc/src/math/bf16fma.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16fma -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16FMA_H
+#define LLVM_LIBC_SRC_MATH_BF16FMA_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16fma(double x, double y, double z);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16FMA_H
diff --git a/libc/src/math/bf16fmaf.h b/libc/src/math/bf16fmaf.h
new file mode 100644
index 0000000000000..e8582bd6dff5f
--- /dev/null
+++ b/libc/src/math/bf16fmaf.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16fmaf ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16FMAF_H
+#define LLVM_LIBC_SRC_MATH_BF16FMAF_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16fmaf(float x, float y, float z);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16FMAF_H
diff --git a/libc/src/math/bf16fmaf128.h b/libc/src/math/bf16fmaf128.h
new file mode 100644
index 0000000000000..4215e54c82ff5
--- /dev/null
+++ b/libc/src/math/bf16fmaf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16fmaf128 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16FMAF128_H
+#define LLVM_LIBC_SRC_MATH_BF16FMAF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16fmaf128(float128 x, float128 y, float128 z);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16FMAF128_H
diff --git a/libc/src/math/bf16fmal.h b/libc/src/math/bf16fmal.h
new file mode 100644
index 0000000000000..b92f17b7ee8d6
--- /dev/null
+++ b/libc/src/math/bf16fmal.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16fmal ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16FMAL_H
+#define LLVM_LIBC_SRC_MATH_BF16FMAL_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16fmal(long double x, long double y, long double z);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16FMAL_H
diff --git a/libc/src/math/bf16mul.h b/libc/src/math/bf16mul.h
new file mode 100644
index 0000000000000..14e8a304c2b6c
--- /dev/null
+++ b/libc/src/math/bf16mul.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16mul -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16MUL_H
+#define LLVM_LIBC_SRC_MATH_BF16MUL_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16mul(double x, double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16MUL_H
diff --git a/libc/src/math/bf16mulf.h b/libc/src/math/bf16mulf.h
new file mode 100644
index 0000000000000..1d02c8e796b17
--- /dev/null
+++ b/libc/src/math/bf16mulf.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16mulf ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16MULF_H
+#define LLVM_LIBC_SRC_MATH_BF16MULF_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16mulf(float x, float y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16MULF_H
diff --git a/libc/src/math/bf16mulf128.h b/libc/src/math/bf16mulf128.h
new file mode 100644
index 0000000000000..6ba7cefb2ee8e
--- /dev/null
+++ b/libc/src/math/bf16mulf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16mulf128 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16MULF128_H
+#define LLVM_LIBC_SRC_MATH_BF16MULF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16mulf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16MULF128_H
diff --git a/libc/src/math/bf16mull.h b/libc/src/math/bf16mull.h
new file mode 100644
index 0000000000000..dad65236ce05d
--- /dev/null
+++ b/libc/src/math/bf16mull.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16mull ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16MULL_H
+#define LLVM_LIBC_SRC_MATH_BF16MULL_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16mull(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16MULL_H
diff --git a/libc/src/math/bf16sub.h b/libc/src/math/bf16sub.h
new file mode 100644
index 0000000000000..8108e9146859f
--- /dev/null
+++ b/libc/src/math/bf16sub.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16sub -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16SUB_H
+#define LLVM_LIBC_SRC_MATH_BF16SUB_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16sub(double x, double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16SUB_H
diff --git a/libc/src/math/bf16subf.h b/libc/src/math/bf16subf.h
new file mode 100644
index 0000000000000..1bd79bfe119ce
--- /dev/null
+++ b/libc/src/math/bf16subf.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16subf ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16SUBF_H
+#define LLVM_LIBC_SRC_MATH_BF16SUBF_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16subf(float x, float y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16SUBF_H
diff --git a/libc/src/math/bf16subf128.h b/libc/src/math/bf16subf128.h
new file mode 100644
index 0000000000000..19590e8c67508
--- /dev/null
+++ b/libc/src/math/bf16subf128.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16subf128 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16SUBF128_H
+#define LLVM_LIBC_SRC_MATH_BF16SUBF128_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16subf128(float128 x, float128 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16SUBF128_H
diff --git a/libc/src/math/bf16subl.h b/libc/src/math/bf16subl.h
new file mode 100644
index 0000000000000..13b2093a92ffa
--- /dev/null
+++ b/libc/src/math/bf16subl.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for bf16subl ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_BF16SUBL_H
+#define LLVM_LIBC_SRC_MATH_BF16SUBL_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 bf16subl(long double x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_BF16SUBL_H
diff --git a/libc/src/math/ceilbf16.h b/libc/src/math/ceilbf16.h
new file mode 100644
index 0000000000000..bf70f25ab2b91
--- /dev/null
+++ b/libc/src/math/ceilbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for ceilbf16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_CEILBF16_H
+#define LLVM_LIBC_SRC_MATH_CEILBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 ceilbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_CEILBF16_H
diff --git a/libc/src/math/floorbf16.h b/libc/src/math/floorbf16.h
new file mode 100644
index 0000000000000..9b5a30adadd86
--- /dev/null
+++ b/libc/src/math/floorbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for floorbf16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FLOORBF16_H
+#define LLVM_LIBC_SRC_MATH_FLOORBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 floorbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FLOORBF16_H
diff --git a/libc/src/math/fmaxbf16.h b/libc/src/math/fmaxbf16.h
new file mode 100644
index 0000000000000..bdbd14cd578d5
--- /dev/null
+++ b/libc/src/math/fmaxbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmaxbf16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXBF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fmaxbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXBF16_H
diff --git a/libc/src/math/fmaximum_mag_numbf16.h b/libc/src/math/fmaximum_mag_numbf16.h
new file mode 100644
index 0000000000000..766352595e4f2
--- /dev/null
+++ b/libc/src/math/fmaximum_mag_numbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmaximum_mag_numbf16 ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMBF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fmaximum_mag_numbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAG_NUMBF16_H
diff --git a/libc/src/math/fmaximum_magbf16.h b/libc/src/math/fmaximum_magbf16.h
new file mode 100644
index 0000000000000..ff0ff1a33ff49
--- /dev/null
+++ b/libc/src/math/fmaximum_magbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmaximum_magbf16 --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGBF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fmaximum_magbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_MAGBF16_H
diff --git a/libc/src/math/fmaximum_numbf16.h b/libc/src/math/fmaximum_numbf16.h
new file mode 100644
index 0000000000000..f23bc525298a1
--- /dev/null
+++ b/libc/src/math/fmaximum_numbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmaximum_numbf16 --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMBF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fmaximum_numbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUM_NUMBF16_H
diff --git a/libc/src/math/fmaximumbf16.h b/libc/src/math/fmaximumbf16.h
new file mode 100644
index 0000000000000..9842e99d43a20
--- /dev/null
+++ b/libc/src/math/fmaximumbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fmaximumbf16 ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMAXIMUMBF16_H
+#define LLVM_LIBC_SRC_MATH_FMAXIMUMBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fmaximumbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMAXIMUMBF16_H
diff --git a/libc/src/math/fminbf16.h b/libc/src/math/fminbf16.h
new file mode 100644
index 0000000000000..4c1ada95f33dc
--- /dev/null
+++ b/libc/src/math/fminbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fminbf16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINBF16_H
+#define LLVM_LIBC_SRC_MATH_FMINBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fminbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMINBF16_H
diff --git a/libc/src/math/fminimum_mag_numbf16.h b/libc/src/math/fminimum_mag_numbf16.h
new file mode 100644
index 0000000000000..2773381efbcef
--- /dev/null
+++ b/libc/src/math/fminimum_mag_numbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fminimum_mag_numbf16 ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMBF16_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fminimum_mag_numbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAG_NUMBF16_H
diff --git a/libc/src/math/fminimum_magbf16.h b/libc/src/math/fminimum_magbf16.h
new file mode 100644
index 0000000000000..fee5c4c8b1a1c
--- /dev/null
+++ b/libc/src/math/fminimum_magbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fminimum_magbf16 --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_MAGBF16_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_MAGBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fminimum_magbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_MAGBF16_H
diff --git a/libc/src/math/fminimum_numbf16.h b/libc/src/math/fminimum_numbf16.h
new file mode 100644
index 0000000000000..a3fd47403c415
--- /dev/null
+++ b/libc/src/math/fminimum_numbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fminimum_numbf16 --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUM_NUMBF16_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUM_NUMBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fminimum_numbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUM_NUMBF16_H
diff --git a/libc/src/math/fminimumbf16.h b/libc/src/math/fminimumbf16.h
new file mode 100644
index 0000000000000..07f1adaa4f075
--- /dev/null
+++ b/libc/src/math/fminimumbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fminimumbf16 ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FMINIMUMBF16_H
+#define LLVM_LIBC_SRC_MATH_FMINIMUMBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fminimumbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FMINIMUMBF16_H
diff --git a/libc/src/math/frexpbf16.h b/libc/src/math/frexpbf16.h
new file mode 100644
index 0000000000000..1e9bba16ea6c4
--- /dev/null
+++ b/libc/src/math/frexpbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for frexpbf16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FREXPFB16_H
+#define LLVM_LIBC_SRC_MATH_FREXPFB16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 frexpbf16(bfloat16 x, int *exp);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FREXPFB16_H
diff --git a/libc/src/math/fromfpbf16.h b/libc/src/math/fromfpbf16.h
new file mode 100644
index 0000000000000..bff991c6c1ed5
--- /dev/null
+++ b/libc/src/math/fromfpbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fromfpbf16 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFPBF16_H
+#define LLVM_LIBC_SRC_MATH_FROMFPBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fromfpbf16(bfloat16 x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFPBF16_H
diff --git a/libc/src/math/fromfpxbf16.h b/libc/src/math/fromfpxbf16.h
new file mode 100644
index 0000000000000..e40d975617ef2
--- /dev/null
+++ b/libc/src/math/fromfpxbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for fromfpxbf16 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_FROMFPXBF16_H
+#define LLVM_LIBC_SRC_MATH_FROMFPXBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 fromfpxbf16(bfloat16 x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_FROMFPXBF16_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index a86619576cfc6..7f4f250025dc3 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -156,6 +156,22 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+    ceilbf16
+  SRCS
+    ceilbf16.cpp
+  HDRS
+    ../ceilbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    ROUND_OPT
+)
+
 add_entrypoint_object(
   daddl
   SRCS
@@ -262,69 +278,6 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.generic.add_sub
 )
 
-add_header_library(
-  range_reduction
-  HDRS
-    range_reduction.h
-    range_reduction_fma.h
-  DEPENDS
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.fma
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.nearest_integer
-    libc.src.__support.common
-)
-
-add_header_library(
-  range_reduction_double
-  HDRS
-    range_reduction_double_common.h
-    range_reduction_double_fma.h
-    range_reduction_double_nofma.h
-  DEPENDS
-    libc.src.__support.FPUtil.double_double
-    libc.src.__support.FPUtil.dyadic_float
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.fma
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.nearest_integer
-    libc.src.__support.common
-    libc.src.__support.integer_literals
-)
-
-add_header_library(
-  sincosf_utils
-  HDRS
-    sincosf_utils.h
-  DEPENDS
-    .range_reduction
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.polyeval
-    libc.src.__support.common
-)
-
-add_header_library(
-  sincosf16_utils
-  HDRS
-    sincosf16_utils.h
-  DEPENDS
-    libc.src.__support.FPUtil.polyeval
-    libc.src.__support.FPUtil.nearest_integer
-    libc.src.__support.common
-)
-
-add_header_library(
-  sincos_eval
-  HDRS
-    sincos_eval.h
-  DEPENDS
-    libc.src.__support.FPUtil.double_double
-    libc.src.__support.FPUtil.dyadic_float
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.polyeval
-    libc.src.__support.integer_literals
-)
-
 add_entrypoint_object(
   cos
   SRCS
@@ -332,16 +285,7 @@ add_entrypoint_object(
   HDRS
     ../cos.h
   DEPENDS
-    .range_reduction_double
-    .sincos_eval
-    libc.hdr.errno_macros
-    libc.src.errno.errno
-    libc.src.__support.FPUtil.double_double
-    libc.src.__support.FPUtil.dyadic_float
-    libc.src.__support.FPUtil.except_value_utils
-    libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.macros.optimization
+    libc.src.__support.math.cos
 )
 
 add_entrypoint_object(
@@ -351,15 +295,7 @@ add_entrypoint_object(
   HDRS
     ../cosf.h
   DEPENDS
-    .sincosf_utils
-    libc.src.errno.errno
-    libc.src.__support.FPUtil.basic_operations
-    libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.except_value_utils
-    libc.src.__support.FPUtil.fma
-    libc.src.__support.FPUtil.polyeval
-    libc.src.__support.macros.optimization
+    libc.src.__support.math.cosf
 )
 
 add_entrypoint_object(
@@ -369,16 +305,7 @@ add_entrypoint_object(
   HDRS
     ../cosf16.h
   DEPENDS
-    .sincosf16_utils
-    libc.hdr.errno_macros
-    libc.hdr.fenv_macros
-    libc.src.__support.FPUtil.cast
-    libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.except_value_utils
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.macros.optimization
-    libc.src.__support.macros.properties.types
+    libc.src.__support.math.cosf16
 )
 
 add_entrypoint_object(
@@ -388,12 +315,7 @@ add_entrypoint_object(
   HDRS
     ../cospif.h
   DEPENDS
-    .sincosf_utils
-    libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.fma
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.macros.optimization
+    libc.src.__support.math.cospif
 )
 
 add_entrypoint_object(
@@ -403,14 +325,7 @@ add_entrypoint_object(
   HDRS
     ../cospif16.h
   DEPENDS
-    .sincosf16_utils
-    libc.hdr.errno_macros
-    libc.hdr.fenv_macros
-    libc.src.__support.FPUtil.cast
-    libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.macros.optimization
+    libc.src.__support.math.cospif16
 )
 
 add_entrypoint_object(
@@ -420,8 +335,8 @@ add_entrypoint_object(
   HDRS
     ../sin.h
   DEPENDS
-    .range_reduction_double
-    .sincos_eval
+    libc.src.__support.math.range_reduction_double
+    libc.src.__support.math.sincos_eval
     libc.hdr.errno_macros
     libc.src.errno.errno
     libc.src.__support.FPUtil.double_double
@@ -440,8 +355,8 @@ add_entrypoint_object(
   HDRS
     ../sinf.h
   DEPENDS
-    .range_reduction
-    .sincosf_utils
+    libc.src.__support.math.range_reduction
+    libc.src.__support.math.sincosf_utils
     libc.src.errno.errno
     libc.src.__support.FPUtil.basic_operations
     libc.src.__support.FPUtil.fenv_impl
@@ -459,7 +374,6 @@ add_entrypoint_object(
   HDRS
     ../sinf16.h
   DEPENDS
-    .sincosf16_utils
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
@@ -469,6 +383,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.types
+    libc.src.__support.math.sincosf16_utils
   COMPILE_OPTIONS
     ${libc_opt_high_flag}
 )
@@ -480,8 +395,8 @@ add_entrypoint_object(
   HDRS
     ../sincos.h
   DEPENDS
-    .range_reduction_double
-    .sincos_eval
+    libc.src.__support.math.range_reduction_double
+    libc.src.__support.math.sincos_eval
     libc.hdr.errno_macros
     libc.src.errno.errno
     libc.src.__support.FPUtil.double_double
@@ -500,7 +415,7 @@ add_entrypoint_object(
   HDRS
     ../sinpif.h
   DEPENDS
-    .sincosf_utils
+    libc.src.__support.math.sincosf_utils
     libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fma
@@ -517,8 +432,8 @@ add_entrypoint_object(
   HDRS
     ../sincosf.h
   DEPENDS
-    .range_reduction
-    .sincosf_utils
+    libc.src.__support.math.range_reduction
+    libc.src.__support.math.sincosf_utils
     libc.src.errno.errno
     libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
@@ -536,7 +451,6 @@ add_entrypoint_object(
   HDRS
     ../sinpif16.h
   DEPENDS
-    .sincosf16_utils
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
@@ -544,6 +458,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
+    libc.src.__support.math.sincosf16_utils
 )
 
 add_entrypoint_object(
@@ -553,7 +468,7 @@ add_entrypoint_object(
   HDRS
     ../tan.h
   DEPENDS
-    .range_reduction_double
+    libc.src.__support.math.range_reduction_double
     libc.hdr.errno_macros
     libc.src.errno.errno
     libc.src.__support.FPUtil.double_double
@@ -572,8 +487,8 @@ add_entrypoint_object(
   HDRS
     ../tanf.h
   DEPENDS
-    .range_reduction
-    .sincosf_utils
+    libc.src.__support.math.range_reduction
+    libc.src.__support.math.sincosf_utils
     libc.src.errno.errno
     libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fenv_impl
@@ -592,7 +507,6 @@ add_entrypoint_object(
   HDRS
     ../tanf16.h
   DEPENDS
-    .sincosf16_utils
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
@@ -602,6 +516,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.types
+    libc.src.__support.math.sincosf16_utils
 )
 
 add_entrypoint_object(
@@ -611,7 +526,7 @@ add_entrypoint_object(
   HDRS
     ../tanpif.h
   DEPENDS
-    .sincosf_utils
+    libc.src.__support.math.sincosf_utils
     libc.src.__support.FPUtil.except_value_utils
     libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
@@ -626,7 +541,6 @@ add_entrypoint_object(
   HDRS
     ../tanpif16.h
   DEPENDS
-    .sincosf16_utils
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
@@ -635,6 +549,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.except_value_utils
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.macros.optimization
+    libc.src.__support.math.sincosf16_utils
 )
 
 add_entrypoint_object(
@@ -801,6 +716,22 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+    truncbf16
+  SRCS
+    truncbf16.cpp
+  HDRS
+    ../truncbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    ROUND_OPT
+)
+
 add_entrypoint_object(
   floor
   SRCS
@@ -861,6 +792,22 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+    floorbf16
+  SRCS
+    floorbf16.cpp
+  HDRS
+    ../floorbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    ROUND_OPT
+)
+
 add_entrypoint_object(
   round
   SRCS
@@ -921,6 +868,22 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+    roundbf16
+  SRCS
+    roundbf16.cpp
+  HDRS
+    ../roundbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    ROUND_OPT
+)
+
 add_entrypoint_object(
   roundeven
   SRCS
@@ -981,6 +944,22 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+    roundevenbf16
+  SRCS
+    roundevenbf16.cpp
+  HDRS
+    ../roundevenbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    ROUND_OPT
+)
+
 add_entrypoint_object(
   lround
   SRCS
@@ -1033,6 +1012,19 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  lroundbf16
+  SRCS
+    lroundbf16.cpp
+  HDRS
+    ../lroundf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+)
+
 add_entrypoint_object(
   llround
   SRCS
@@ -1085,6 +1077,19 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  llroundbf16
+  SRCS
+    llroundbf16.cpp
+  HDRS
+    ../llroundf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+)
+
 add_entrypoint_object(
   rint
   SRCS
@@ -1145,6 +1150,21 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  rintbf16
+  SRCS
+    rintbf16.cpp
+  HDRS
+    ../rintbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+  FLAGS
+    ROUND_OPT
+)
+
 add_entrypoint_object(
   lrint
   SRCS
@@ -1197,6 +1217,19 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  lrintbf16
+  SRCS
+    lrintbf16.cpp
+  HDRS
+    ../lrintbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+)
+
 add_entrypoint_object(
   llrint
   SRCS
@@ -1249,6 +1282,17 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  llrintbf16
+  SRCS
+    llrintbf16.cpp
+  HDRS
+    ../llrintbf16.h
+  DEPENDS
+    libc.src.__support.macros.properties.types
+    libc.src.__support.FPUtil.nearest_integer_operations
+)
+
 add_entrypoint_object(
   nearbyint
   SRCS
@@ -1301,6 +1345,19 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  nearbyintbf16
+  SRCS
+    nearbyintbf16.cpp
+  HDRS
+    ../nearbyintbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+)
+
 add_entrypoint_object(
   erff
   SRCS
@@ -1353,7 +1410,6 @@ add_entrypoint_object(
     ../exp2.h
   DEPENDS
     .common_constants
-    .explogxf
     libc.src.__support.CPP.bit
     libc.src.__support.CPP.optional
     libc.src.__support.FPUtil.dyadic_float
@@ -1366,6 +1422,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.triple_double
     libc.src.__support.integer_literals
     libc.src.__support.macros.optimization
+    libc.src.__support.math.exp_utils
     libc.src.errno.errno
 )
 
@@ -1374,7 +1431,6 @@ add_header_library(
   HDRS
     exp2f_impl.h
   DEPENDS
-    .explogxf
     libc.src.__support.FPUtil.except_value_utils
     libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
@@ -1383,6 +1439,7 @@ add_header_library(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
+    libc.src.__support.math.exp10f_utils
     libc.src.__support.common
     libc.src.errno.errno
 )
@@ -1404,7 +1461,6 @@ add_entrypoint_object(
   HDRS
     ../exp2f16.h
   DEPENDS
-    .expxf16
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
@@ -1413,6 +1469,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
+    libc.src.__support.math.expxf16_utils
 )
 
 add_entrypoint_object(
@@ -1422,7 +1479,6 @@ add_entrypoint_object(
   HDRS
     ../exp2m1f.h
   DEPENDS
-    .explogxf
     libc.src.errno.errno
     libc.src.__support.common
     libc.src.__support.FPUtil.except_value_utils
@@ -1433,6 +1489,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
+    libc.src.__support.math.exp10f_utils
 )
 
 add_entrypoint_object(
@@ -1442,7 +1499,6 @@ add_entrypoint_object(
   HDRS
     ../exp2m1f16.h
   DEPENDS
-    .expxf16
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.common
@@ -1455,6 +1511,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
+    libc.src.__support.math.expxf16_utils
 )
 
 add_entrypoint_object(
@@ -1497,7 +1554,6 @@ add_entrypoint_object(
   HDRS
     ../exp10m1f.h
   DEPENDS
-    .explogxf
     libc.src.errno.errno
     libc.src.__support.common
     libc.src.__support.FPUtil.except_value_utils
@@ -1507,6 +1563,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
+    libc.src.__support.math.exp10f_utils
 )
 
 add_entrypoint_object(
@@ -1538,14 +1595,11 @@ add_entrypoint_object(
     ../expm1.h
   DEPENDS
     .common_constants
-    .explogxf
     libc.src.__support.CPP.bit
-    libc.src.__support.CPP.optional
     libc.src.__support.FPUtil.dyadic_float
     libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.nearest_integer
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.FPUtil.triple_double
@@ -1580,7 +1634,6 @@ add_entrypoint_object(
   HDRS
     ../expm1f16.h
   DEPENDS
-    .expxf16
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
@@ -1591,6 +1644,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
+    libc.src.__support.math.expxf16_utils
 )
 
 add_entrypoint_object(
@@ -1602,7 +1656,6 @@ add_entrypoint_object(
   DEPENDS
     .common_constants
     .exp2f_impl
-    .explogxf
     libc.src.__support.math.exp10f
     libc.src.__support.CPP.bit
     libc.src.__support.FPUtil.fenv_impl
@@ -1745,6 +1798,20 @@ add_entrypoint_object(
     libc.src.__support.math.frexpf128
 )
 
+add_entrypoint_object(
+  frexpbf16
+  SRCS
+    frexpbf16.cpp
+  HDRS
+    ../frexpbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   ilogb
   SRCS
@@ -1797,6 +1864,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  ilogbbf16
+  SRCS
+    ilogbbf16.cpp
+  HDRS
+    ../ilogbbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   llogb
   SRCS
@@ -1849,6 +1930,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  llogbbf16
+  SRCS
+    llogbbf16.cpp
+  HDRS
+    ../llogbbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   ldexp
   SRCS
@@ -1899,12 +1994,26 @@ add_entrypoint_object(
     libc.src.__support.math.ldexpf128
 )
 
-add_object_library(
-  common_constants
-  HDRS
-    common_constants.h
+add_entrypoint_object(
+  ldexpbf16
   SRCS
-    common_constants.cpp
+    ldexpbf16.cpp
+  HDRS
+    ../ldexpbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_object_library(
+  common_constants
+  HDRS
+    common_constants.h
+  SRCS
+    common_constants.cpp
   DEPENDS
     libc.src.__support.math.exp_constants
     libc.src.__support.math.acosh_float_constants
@@ -1962,7 +2071,6 @@ add_entrypoint_object(
   HDRS
     ../log10f16.h
   DEPENDS
-    .expxf16
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
@@ -1973,6 +2081,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
+    libc.src.__support.math.expxf16_utils
 )
 
 add_entrypoint_object(
@@ -2051,7 +2160,6 @@ add_entrypoint_object(
   HDRS
     ../log2f16.h
   DEPENDS
-    .expxf16
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
@@ -2062,6 +2170,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
+    libc.src.__support.math.expxf16_utils
 )
 
 add_entrypoint_object(
@@ -2106,7 +2215,6 @@ add_entrypoint_object(
   HDRS
     ../logf16.h
   DEPENDS
-    .expxf16
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.cast
@@ -2117,6 +2225,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
     libc.src.__support.macros.properties.cpu_features
+    libc.src.__support.math.expxf16_utils
 )
 
 add_entrypoint_object(
@@ -2171,6 +2280,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  logbbf16
+  SRCS
+    logbbf16.cpp
+  HDRS
+    ../logbbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   modf
   SRCS
@@ -2281,6 +2404,21 @@ add_entrypoint_object(
     MISC_MATH_BASIC_OPS_OPT
 )
 
+add_entrypoint_object(
+  fminbf16
+  SRCS
+    fminbf16.cpp
+  HDRS
+    ../fminbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
 
 add_entrypoint_object(
   fmax
@@ -2340,6 +2478,22 @@ add_entrypoint_object(
     MISC_MATH_BASIC_OPS_OPT
 )
 
+add_entrypoint_object(
+  fmaxbf16
+  SRCS
+    fmaxbf16.cpp
+  HDRS
+    ../fmaxbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
+
 add_entrypoint_object(
   fmaximum
   SRCS
@@ -2398,6 +2552,21 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  fmaximumbf16
+  SRCS
+    fmaximumbf16.cpp
+  HDRS
+    ../fmaximumbf16.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
+
 add_entrypoint_object(
   fmaximum_num
   SRCS
@@ -2456,6 +2625,21 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  fmaximum_numbf16
+  SRCS
+    fmaximum_numbf16.cpp
+  HDRS
+    ../fmaximum_numbf16.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
+
 add_entrypoint_object(
   fmaximum_mag
   SRCS
@@ -2514,6 +2698,21 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  fmaximum_magbf16
+  SRCS
+    fmaximum_magbf16.cpp
+  HDRS
+    ../fmaximum_magbf16.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
+
 add_entrypoint_object(
   fmaximum_mag_num
   SRCS
@@ -2572,6 +2771,21 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  fmaximum_mag_numbf16
+  SRCS
+    fmaximum_mag_numbf16.cpp
+  HDRS
+    ../fmaximum_mag_numbf16.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
+
 add_entrypoint_object(
   fminimum
   SRCS
@@ -2630,6 +2844,21 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  fminimumbf16
+  SRCS
+    fminimumbf16.cpp
+  HDRS
+    ../fminimumbf16.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
+
 add_entrypoint_object(
   fminimum_num
   SRCS
@@ -2688,6 +2917,21 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  fminimum_numbf16
+  SRCS
+    fminimum_numbf16.cpp
+  HDRS
+    ../fminimum_numbf16.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
+
 add_entrypoint_object(
   fminimum_mag
   SRCS
@@ -2746,6 +2990,21 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  fminimum_magbf16
+  SRCS
+    fminimum_magbf16.cpp
+  HDRS
+    ../fminimum_magbf16.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
+
 add_entrypoint_object(
   fminimum_mag_num
   SRCS
@@ -2804,6 +3063,21 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  fminimum_mag_numbf16
+  SRCS
+    fminimum_mag_numbf16.cpp
+  HDRS
+    ../fminimum_mag_numbf16.h
+  DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+  FLAGS
+    MISC_MATH_BASIC_OPS_OPT
+)
+
 add_entrypoint_object(
   fmul
   SRCS
@@ -3306,6 +3580,22 @@ add_entrypoint_object(
     libc.src.errno.errno
 )
 
+add_entrypoint_object(
+  nanbf16
+  SRCS
+    nanbf16.cpp
+  HDRS
+    ../nanbf16.h
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.libc_errno
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+    libc.src.__support.str_to_float
+)
+
 add_entrypoint_object(
   nextafter
   SRCS
@@ -3358,6 +3648,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  nextafterbf16
+  SRCS
+    nextafterbf16.cpp
+  HDRS
+    ../nextafterbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   nexttoward
   SRCS
@@ -3399,6 +3703,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  nexttowardbf16
+  SRCS
+    nexttowardbf16.cpp
+  HDRS
+    ../nexttowardbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   nextdown
   SRCS
@@ -3451,6 +3769,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  nextdownbf16
+  SRCS
+    nextdownbf16.cpp
+  HDRS
+    ../nextdownbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   nextup
   SRCS
@@ -3503,6 +3835,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_entrypoint_object(
+  nextupbf16
+  SRCS
+    nextupbf16.cpp
+  HDRS
+    ../nextupbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   fmod
   SRCS
@@ -3607,6 +3953,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  fromfpbf16
+  SRCS
+    fromfpbf16.cpp
+  HDRS
+    ../fromfpbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   fromfpx
   SRCS
@@ -3659,6 +4019,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  fromfpxbf16
+  SRCS
+    fromfpxbf16.cpp
+  HDRS
+    ../fromfpxbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   ufromfp
   SRCS
@@ -3711,6 +4085,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
+add_entrypoint_object(
+  ufromfpbf16
+  SRCS
+    ufromfpbf16.cpp
+  HDRS
+    ../ufromfpbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   ufromfpx
   SRCS
@@ -3763,17 +4151,18 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.nearest_integer_operations
 )
 
-#TODO: Add errno include to the hyperbolic functions.
-add_header_library(
-  explogxf
+add_entrypoint_object(
+  ufromfpxbf16
+  SRCS
+    ufromfpxbf16.cpp
   HDRS
-    explogxf.h
+    ../ufromfpxbf16.h
   DEPENDS
-    .common_constants
-    libc.src.__support.math.exp_utils
-    libc.src.__support.math.acoshf_utils
-    libc.src.__support.macros.properties.cpu_features
-    libc.src.errno.errno
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.nearest_integer_operations
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
 )
 
 add_entrypoint_object(
@@ -3783,11 +4172,7 @@ add_entrypoint_object(
   HDRS
     ../coshf.h
   DEPENDS
-    .explogxf
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.rounding_mode
-    libc.src.__support.macros.optimization
+    libc.src.__support.math.coshf
 )
 
 add_entrypoint_object(
@@ -3797,14 +4182,7 @@ add_entrypoint_object(
   HDRS
     ../coshf16.h
   DEPENDS
-    .expxf16
-    libc.hdr.errno_macros
-    libc.hdr.fenv_macros
-    libc.src.__support.FPUtil.except_value_utils
-    libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.rounding_mode
-    libc.src.__support.macros.optimization
+    libc.src.__support.math.coshf16
 )
 
 add_entrypoint_object(
@@ -3814,10 +4192,10 @@ add_entrypoint_object(
   HDRS
     ../sinhf.h
   DEPENDS
-    .explogxf
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
+    libc.src.__support.math.sinhfcoshf_utils
 )
 
 add_entrypoint_object(
@@ -3827,14 +4205,14 @@ add_entrypoint_object(
   HDRS
     ../sinhf16.h
   DEPENDS
-    .expxf16
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
     libc.src.__support.FPUtil.except_value_utils
-    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fenv_impl 
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
+    libc.src.__support.math.expxf16_utils
 )
 
 add_entrypoint_object(
@@ -3844,12 +4222,12 @@ add_entrypoint_object(
   HDRS
     ../tanhf.h
   DEPENDS
-    .explogxf
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.FPUtil.multiply_add
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
+    libc.src.__support.math.exp10f_utils
 )
 
 add_entrypoint_object(
@@ -3859,7 +4237,6 @@ add_entrypoint_object(
   HDRS
     ../tanhf16.h
   DEPENDS
-    .expxf16
     libc.hdr.fenv_macros
     libc.src.__support.CPP.array
     libc.src.__support.FPUtil.cast
@@ -3871,6 +4248,7 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.polyeval
     libc.src.__support.FPUtil.rounding_mode
     libc.src.__support.macros.optimization
+    libc.src.__support.math.expxf16_utils
 )
 
 add_entrypoint_object(
@@ -3880,7 +4258,6 @@ add_entrypoint_object(
   HDRS
     ../acoshf.h
   DEPENDS
-    .explogxf
     libc.src.__support.math.acoshf
 )
 
@@ -3915,6 +4292,25 @@ add_entrypoint_object(
     libc.src.__support.math.asinhf16
 )
 
+add_entrypoint_object(
+  asinpif16
+  SRCS
+    asinpif16.cpp
+  HDRS
+    ../asinpif16.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.cast
+    libc.src.__support.FPUtil.except_value_utils
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.polyeval
+    libc.src.__support.FPUtil.sqrt
+    libc.src.__support.macros.optimization
+)
+
 add_entrypoint_object(
   atanhf
   SRCS
@@ -4377,6 +4773,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  getpayloadbf16
+  SRCS
+    getpayloadbf16.cpp
+  HDRS
+    ../getpayloadbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   setpayload
   SRCS
@@ -4429,6 +4839,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  setpayloadbf16
+  SRCS
+    setpayloadbf16.cpp
+  HDRS
+    ../setpayloadbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   setpayloadsig
   SRCS
@@ -4481,6 +4905,20 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.basic_operations
 )
 
+add_entrypoint_object(
+  setpayloadsigbf16
+  SRCS
+    setpayloadsigbf16.cpp
+  HDRS
+    ../setpayloadsigbf16.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
 add_entrypoint_object(
   f16add
   SRCS
@@ -4739,11 +5177,7 @@ add_entrypoint_object(
   HDRS
     ../cbrtf.h
   DEPENDS
-    libc.hdr.fenv_macros
-    libc.src.__support.FPUtil.fenv_impl
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.macros.optimization
+    libc.src.__support.math.cbrtf
 )
 
 add_entrypoint_object(
@@ -4821,17 +5255,282 @@ add_entrypoint_object(
     libc.src.__support.FPUtil.generic.mul
 )
 
-add_header_library(
-  expxf16
+add_entrypoint_object(
+  bf16add
+  SRCS
+    bf16add.cpp
   HDRS
-    expxf16.h
+    ../bf16add.h
   DEPENDS
-    libc.hdr.stdint_proxy
-    libc.src.__support.FPUtil.fp_bits
-    libc.src.__support.FPUtil.cast
-    libc.src.__support.FPUtil.multiply_add
-    libc.src.__support.FPUtil.nearest_integer
-    libc.src.__support.macros.attributes
-    libc.src.__support.math.expf16_utils
-    libc.src.__support.math.exp10_float16_constants
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16addf
+  SRCS
+    bf16addf.cpp
+  HDRS
+    ../bf16addf.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16addl
+  SRCS
+    bf16addl.cpp
+  HDRS
+    ../bf16addl.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16addf128
+  SRCS
+    bf16addf128.cpp
+  HDRS
+    ../bf16addf128.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16div
+  SRCS
+    bf16div.cpp
+  HDRS
+    ../bf16div.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.div
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16divf
+  SRCS
+    bf16divf.cpp
+  HDRS
+    ../bf16divf.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.div
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16divl
+  SRCS
+    bf16divl.cpp
+  HDRS
+    ../bf16divl.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.div
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16divf128
+  SRCS
+    bf16divf128.cpp
+  HDRS
+    ../bf16divf128.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.div
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16fma
+  SRCS
+    bf16fma.cpp
+  HDRS
+    ../bf16fma.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fma
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16fmaf
+  SRCS
+    bf16fmaf.cpp
+  HDRS
+    ../bf16fmaf.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fma
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16fmal
+  SRCS
+    bf16fmal.cpp
+  HDRS
+    ../bf16fmal.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fma
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16fmaf128
+  SRCS
+    bf16fmaf128.cpp
+  HDRS
+    ../bf16fmaf128.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fma
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16mul
+  SRCS
+    bf16mul.cpp
+  HDRS
+    ../bf16mul.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.mul
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16mulf
+  SRCS
+    bf16mulf.cpp
+  HDRS
+    ../bf16mulf.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.mul
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16mull
+  SRCS
+    bf16mull.cpp
+  HDRS
+    ../bf16mull.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.mul
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16mulf128
+  SRCS
+    bf16mulf128.cpp
+  HDRS
+    ../bf16mulf128.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.mul
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16sub
+  SRCS
+    bf16sub.cpp
+  HDRS
+    ../bf16sub.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16subf
+  SRCS
+    bf16subf.cpp
+  HDRS
+    ../bf16subf.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16subl
+  SRCS
+    bf16subl.cpp
+  HDRS
+    ../bf16subl.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
+)
+
+add_entrypoint_object(
+  bf16subf128
+  SRCS
+    bf16subf128.cpp
+  HDRS
+    ../bf16subf128.h
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.config
+    libc.src.__support.macros.properties.types
 )
diff --git a/libc/src/math/generic/acoshf.cpp b/libc/src/math/generic/acoshf.cpp
index 5c04583650e62..c9646329f73d0 100644
--- a/libc/src/math/generic/acoshf.cpp
+++ b/libc/src/math/generic/acoshf.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/acoshf.h"
-
 #include "src/__support/math/acoshf.h"
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/math/generic/asinpif16.cpp b/libc/src/math/generic/asinpif16.cpp
new file mode 100644
index 0000000000000..aabc0863ba527
--- /dev/null
+++ b/libc/src/math/generic/asinpif16.cpp
@@ -0,0 +1,127 @@
+//===-- Half-precision asinpif16(x) function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception.
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/asinpif16.h"
+#include "hdr/errno_macros.h"
+#include "hdr/fenv_macros.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/PolyEval.h"
+#include "src/__support/FPUtil/cast.h"
+#include "src/__support/FPUtil/except_value_utils.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/FPUtil/sqrt.h"
+#include "src/__support/macros/optimization.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(float16, asinpif16, (float16 x)) {
+  using FPBits = fputil::FPBits<float16>;
+
+  FPBits xbits(x);
+  bool is_neg = xbits.is_neg();
+  double x_abs = fputil::cast<double>(xbits.abs().get_val());
+
+  auto signed_result = [is_neg](auto r) -> auto { return is_neg ? -r : r; };
+
+  if (LIBC_UNLIKELY(x_abs > 1.0)) {
+    // aspinf16(NaN) = NaN
+    if (xbits.is_nan()) {
+      if (xbits.is_signaling_nan()) {
+        fputil::raise_except_if_required(FE_INVALID);
+        return FPBits::quiet_nan().get_val();
+      }
+      return x;
+    }
+
+    // 1 < |x| <= +/-inf
+    fputil::raise_except_if_required(FE_INVALID);
+    fputil::set_errno_if_required(EDOM);
+
+    return FPBits::quiet_nan().get_val();
+  }
+
+  // the coefficients for the polynomial approximation of asin(x)/pi in the
+  // range [0, 0.5] extracted using python-sympy
+  //
+  // Python code to generate the coefficients:
+  //  > from sympy import *
+  //  > import math
+  //  > x = symbols('x')
+  //  > print(series(asin(x)/math.pi, x, 0, 21))
+  //
+  // OUTPUT:
+  //
+  // 0.318309886183791*x + 0.0530516476972984*x**3 + 0.0238732414637843*x**5 +
+  // 0.0142102627760621*x**7 + 0.00967087327815336*x**9 +
+  // 0.00712127941391293*x**11 + 0.00552355646848375*x**13 +
+  // 0.00444514782463692*x**15 + 0.00367705242846804*x**17 +
+  // 0.00310721681820837*x**19 + O(x**21)
+  //
+  // it's very accurate in the range [0, 0.5] and has a maximum error of
+  // 0.0000000000000001 in the range [0, 0.5].
+  constexpr double POLY_COEFFS[] = {
+      0x1.45f306dc9c889p-2, // x^1
+      0x1.b2995e7b7b5fdp-5, // x^3
+      0x1.8723a1d588a36p-6, // x^5
+      0x1.d1a452f20430dp-7, // x^7
+      0x1.3ce52a3a09f61p-7, // x^9
+      0x1.d2b33e303d375p-8, // x^11
+      0x1.69fde663c674fp-8, // x^13
+      0x1.235134885f19bp-8, // x^15
+  };
+  // polynomial evaluation using horner's method
+  // work only for |x| in [0, 0.5]
+  auto asinpi_polyeval = [](double x) -> double {
+    return x * fputil::polyeval(x * x, POLY_COEFFS[0], POLY_COEFFS[1],
+                                POLY_COEFFS[2], POLY_COEFFS[3], POLY_COEFFS[4],
+                                POLY_COEFFS[5], POLY_COEFFS[6], POLY_COEFFS[7]);
+  };
+
+  // if |x| <= 0.5:
+  if (LIBC_UNLIKELY(x_abs <= 0.5)) {
+    // Use polynomial approximation of asin(x)/pi in the range [0, 0.5]
+    double result = asinpi_polyeval(fputil::cast<double>(x));
+    return fputil::cast<float16>(result);
+  }
+
+  // If |x| > 0.5, we need to use the range reduction method:
+  //    y = asin(x) => x = sin(y)
+  //      because: sin(a) = cos(pi/2 - a)
+  //      therefore:
+  //    x = cos(pi/2 - y)
+  //      let z = pi/2 - y,
+  //    x = cos(z)
+  //      because: cos(2a) = 1 - 2 * sin^2(a), z = 2a, a = z/2
+  //      therefore:
+  //    cos(z) = 1 - 2 * sin^2(z/2)
+  //    sin(z/2) = sqrt((1 - cos(z))/2)
+  //    sin(z/2) = sqrt((1 - x)/2)
+  //      let u = (1 - x)/2
+  //      then:
+  //    sin(z/2) = sqrt(u)
+  //    z/2 = asin(sqrt(u))
+  //    z = 2 * asin(sqrt(u))
+  //    pi/2 - y = 2 * asin(sqrt(u))
+  //    y = pi/2 - 2 * asin(sqrt(u))
+  //    y/pi = 1/2 - 2 * asin(sqrt(u))/pi
+  //
+  // Finally, we can write:
+  //   asinpi(x) = 1/2 - 2 * asinpi(sqrt(u))
+  //     where u = (1 - x) /2
+  //             = 0.5 - 0.5 * x
+  //             = multiply_add(-0.5, x, 0.5)
+
+  double u = fputil::multiply_add(-0.5, x_abs, 0.5);
+  double asinpi_sqrt_u = asinpi_polyeval(fputil::sqrt<double>(u));
+  double result = fputil::multiply_add(-2.0, asinpi_sqrt_u, 0.5);
+
+  return fputil::cast<float16>(signed_result(result));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16add.cpp b/libc/src/math/generic/bf16add.cpp
new file mode 100644
index 0000000000000..257596afe66cf
--- /dev/null
+++ b/libc/src/math/generic/bf16add.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16add function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16add.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16add, (double x, double y)) {
+  return fputil::generic::add<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16addf.cpp b/libc/src/math/generic/bf16addf.cpp
new file mode 100644
index 0000000000000..65e6cbf6b1507
--- /dev/null
+++ b/libc/src/math/generic/bf16addf.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16addf function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16addf.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16addf, (float x, float y)) {
+  return fputil::generic::add<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16addf128.cpp b/libc/src/math/generic/bf16addf128.cpp
new file mode 100644
index 0000000000000..03f70af82e892
--- /dev/null
+++ b/libc/src/math/generic/bf16addf128.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16addf128 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16addf128.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16addf128, (float128 x, float128 y)) {
+  return fputil::generic::add<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16addl.cpp b/libc/src/math/generic/bf16addl.cpp
new file mode 100644
index 0000000000000..c212195c2b7c8
--- /dev/null
+++ b/libc/src/math/generic/bf16addl.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16addl function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16addl.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16addl, (long double x, long double y)) {
+  return fputil::generic::add<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16div.cpp b/libc/src/math/generic/bf16div.cpp
new file mode 100644
index 0000000000000..5e9b1b42e27f4
--- /dev/null
+++ b/libc/src/math/generic/bf16div.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16div function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16div.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/div.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16div, (double x, double y)) {
+  return fputil::generic::div<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16divf.cpp b/libc/src/math/generic/bf16divf.cpp
new file mode 100644
index 0000000000000..2054a6417b078
--- /dev/null
+++ b/libc/src/math/generic/bf16divf.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16divf function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16divf.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/div.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16divf, (float x, float y)) {
+  return fputil::generic::div<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16divf128.cpp b/libc/src/math/generic/bf16divf128.cpp
new file mode 100644
index 0000000000000..fbe9775ce4046
--- /dev/null
+++ b/libc/src/math/generic/bf16divf128.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16divf128 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16divf128.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/div.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16divf128, (float128 x, float128 y)) {
+  return fputil::generic::div<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16divl.cpp b/libc/src/math/generic/bf16divl.cpp
new file mode 100644
index 0000000000000..21dd6b150e07a
--- /dev/null
+++ b/libc/src/math/generic/bf16divl.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16divl function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16divl.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/div.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16divl, (long double x, long double y)) {
+  return fputil::generic::div<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16fma.cpp b/libc/src/math/generic/bf16fma.cpp
new file mode 100644
index 0000000000000..0f0fe8658dde0
--- /dev/null
+++ b/libc/src/math/generic/bf16fma.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16fma function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16fma.h"
+#include "src/__support/FPUtil/FMA.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16fma, (double x, double y, double z)) {
+  return fputil::fma<bfloat16>(x, y, z);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16fmaf.cpp b/libc/src/math/generic/bf16fmaf.cpp
new file mode 100644
index 0000000000000..739691cc50117
--- /dev/null
+++ b/libc/src/math/generic/bf16fmaf.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16fmaf function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16fmaf.h"
+#include "src/__support/FPUtil/FMA.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16fmaf, (float x, float y, float z)) {
+  return fputil::fma<bfloat16>(x, y, z);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16fmaf128.cpp b/libc/src/math/generic/bf16fmaf128.cpp
new file mode 100644
index 0000000000000..a29a0b0b27276
--- /dev/null
+++ b/libc/src/math/generic/bf16fmaf128.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of bf16fmaf128 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16fmaf128.h"
+#include "src/__support/FPUtil/FMA.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16fmaf128,
+                   (float128 x, float128 y, float128 z)) {
+  return fputil::fma<bfloat16>(x, y, z);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16fmal.cpp b/libc/src/math/generic/bf16fmal.cpp
new file mode 100644
index 0000000000000..f31ec6904760b
--- /dev/null
+++ b/libc/src/math/generic/bf16fmal.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of bf16fmal function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16fmal.h"
+
+#include "src/__support/FPUtil/FMA.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16fmal,
+                   (long double x, long double y, long double z)) {
+  return fputil::fma<bfloat16>(x, y, z);
+}
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16mul.cpp b/libc/src/math/generic/bf16mul.cpp
new file mode 100644
index 0000000000000..c50eec2b52e5c
--- /dev/null
+++ b/libc/src/math/generic/bf16mul.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16mul function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16mul.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16mul, (double x, double y)) {
+  return fputil::generic::mul<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16mulf.cpp b/libc/src/math/generic/bf16mulf.cpp
new file mode 100644
index 0000000000000..117fcd1d661ab
--- /dev/null
+++ b/libc/src/math/generic/bf16mulf.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16mulf function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16mulf.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16mulf, (float x, float y)) {
+  return fputil::generic::mul<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16mulf128.cpp b/libc/src/math/generic/bf16mulf128.cpp
new file mode 100644
index 0000000000000..ff2a081d82e6b
--- /dev/null
+++ b/libc/src/math/generic/bf16mulf128.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16mulf128 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16mulf128.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16mulf128, (float128 x, float128 y)) {
+  return fputil::generic::mul<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16mull.cpp b/libc/src/math/generic/bf16mull.cpp
new file mode 100644
index 0000000000000..e7c4fc085a3cd
--- /dev/null
+++ b/libc/src/math/generic/bf16mull.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16mull function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16mull.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/mul.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16mull, (long double x, long double y)) {
+  return fputil::generic::mul<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16sub.cpp b/libc/src/math/generic/bf16sub.cpp
new file mode 100644
index 0000000000000..65eb2095dd4e9
--- /dev/null
+++ b/libc/src/math/generic/bf16sub.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16sub function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16sub.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16sub, (double x, double y)) {
+  return fputil::generic::sub<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16subf.cpp b/libc/src/math/generic/bf16subf.cpp
new file mode 100644
index 0000000000000..6bba4be441be6
--- /dev/null
+++ b/libc/src/math/generic/bf16subf.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16subf function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16subf.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16subf, (float x, float y)) {
+  return fputil::generic::sub<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16subf128.cpp b/libc/src/math/generic/bf16subf128.cpp
new file mode 100644
index 0000000000000..e5fe1077f89a7
--- /dev/null
+++ b/libc/src/math/generic/bf16subf128.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16subf128 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16subf128.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16subf128, (float128 x, float128 y)) {
+  return fputil::generic::sub<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/bf16subl.cpp b/libc/src/math/generic/bf16subl.cpp
new file mode 100644
index 0000000000000..d3a970cade922
--- /dev/null
+++ b/libc/src/math/generic/bf16subl.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of bf16subl function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/bf16subl.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/FPUtil/generic/add_sub.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, bf16subl, (long double x, long double y)) {
+  return fputil::generic::sub<bfloat16>(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/cbrtf.cpp b/libc/src/math/generic/cbrtf.cpp
index 71b23c4a8c742..0bd8f71365300 100644
--- a/libc/src/math/generic/cbrtf.cpp
+++ b/libc/src/math/generic/cbrtf.cpp
@@ -7,153 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/cbrtf.h"
-#include "hdr/fenv_macros.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+#include "src/__support/math/cbrtf.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-namespace {
-
-// Look up table for 2^(i/3) for i = 0, 1, 2.
-constexpr double CBRT2[3] = {1.0, 0x1.428a2f98d728bp0, 0x1.965fea53d6e3dp0};
-
-// Degree-7 polynomials approximation of ((1 + x)^(1/3) - 1)/x for 0 <= x <= 1
-// generated by Sollya with:
-// > for i from 0 to 15 do {
-//     P = fpminimax(((1 + x)^(1/3) - 1)/x, 6, [|D...|], [i/16, (i + 1)/16]);
-//     print("{", coeff(P, 0), ",", coeff(P, 1), ",", coeff(P, 2), ",",
-//           coeff(P, 3), ",", coeff(P, 4), ",", coeff(P, 5), ",",
-//           coeff(P, 6), "},");
-// };
-// Then (1 + x)^(1/3) ~ 1 + x * P(x).
-constexpr double COEFFS[16][7] = {
-    {0x1.55555555554ebp-2, -0x1.c71c71c678c0cp-4, 0x1.f9add2776de81p-5,
-     -0x1.511e10aa964a7p-5, 0x1.ee44165937fa2p-6, -0x1.7c5c9e059345dp-6,
-     0x1.047f75e0aff14p-6},
-    {0x1.5555554d1149ap-2, -0x1.c71c676fcb5bp-4, 0x1.f9ab127dc57ebp-5,
-     -0x1.50ea8fd1d4c15p-5, 0x1.e9d68f28ced43p-6, -0x1.60e0e1e661311p-6,
-     0x1.716eca1d6e3bcp-7},
-    {0x1.5555546377d45p-2, -0x1.c71bc1c6d49d2p-4, 0x1.f9924cc0ed24dp-5,
-     -0x1.4fea3beb53b3bp-5, 0x1.de028a9a07b1bp-6, -0x1.3b090d2233524p-6,
-     0x1.0aeca34893785p-7},
-    {0x1.55554dce9f649p-2, -0x1.c7188b34b98f8p-4, 0x1.f93e1af34af49p-5,
-     -0x1.4d9a06be75c63p-5, 0x1.cb943f4f68992p-6, -0x1.139a685a5e3c4p-6,
-     0x1.88410674c6a5dp-8},
-    {0x1.5555347d211c3p-2, -0x1.c70f2a4b1a5fap-4, 0x1.f88420e8602c3p-5,
-     -0x1.49becfa4ed3ep-5, 0x1.b475cd9013162p-6, -0x1.dcfee1dd2f8efp-7,
-     0x1.249bb51a1c498p-8},
-    {0x1.5554f01b33dbap-2, -0x1.c6facb929dbf1p-4, 0x1.f73fb7861252ep-5,
-     -0x1.4459a4a0071fap-5, 0x1.9a8df2b504fc2p-6, -0x1.9a7ce3006d06ep-7,
-     0x1.ba9230918fa2ep-9},
-    {0x1.55545c695db5fp-2, -0x1.c6d6089f20275p-4, 0x1.f556e0ea80efp-5,
-     -0x1.3d91372d083f4p-5, 0x1.7f66cff331f4p-6, -0x1.606a562491737p-7,
-     0x1.52e3e17c71069p-9},
-    {0x1.55534a879232ap-2, -0x1.c69b836998b84p-4, 0x1.f2bb26dac0e4cp-5,
-     -0x1.359eed43716d7p-5, 0x1.64218cd824fbcp-6, -0x1.2e703e2e091e8p-7,
-     0x1.0677d9af6aad4p-9},
-    {0x1.5551836bb5494p-2, -0x1.c64658c15353bp-4, 0x1.ef68517451a6ep-5,
-     -0x1.2cc20a980dceep-5, 0x1.49843e0fad93ap-6, -0x1.03c59ccb68e54p-7,
-     0x1.9ad325dc7adcbp-10},
-    {0x1.554ecacb0d035p-2, -0x1.c5d2664026ffcp-4, 0x1.eb624796ba809p-5,
-     -0x1.233803d19a535p-5, 0x1.300decb1c3c28p-6, -0x1.befe18031ec3dp-8,
-     0x1.449f5ee175c69p-10},
-    {0x1.554ae1f5ae815p-2, -0x1.c53c6b14ff6b2p-4, 0x1.e6b2d5127bb5bp-5,
-     -0x1.19387336788a3p-5, 0x1.180955a6ab255p-6, -0x1.81696703ba369p-8,
-     0x1.02cb36389bd79p-10},
-    {0x1.55458a59f356ep-2, -0x1.c4820dd631ae9p-4, 0x1.e167af818bd15p-5,
-     -0x1.0ef35f6f72e52p-5, 0x1.019c33b65e4ebp-6, -0x1.4d25bdd52d3a5p-8,
-     0x1.a008ae91f5936p-11},
-    {0x1.553e878eafee1p-2, -0x1.c3a1d0b2a3db2p-4, 0x1.db90d8ed9f89bp-5,
-     -0x1.0490e20f1ae91p-5, 0x1.d9a5d1fc42fe3p-7, -0x1.20bf8227c2abfp-8,
-     0x1.50f8174cdb6e9p-11},
-    {0x1.5535a0dedf1b1p-2, -0x1.c29afb8bd01a1p-4, 0x1.d53f6371c1e27p-5,
-     -0x1.f463209b433e2p-6, 0x1.b35222a17e44p-7, -0x1.f5efbf505e133p-9,
-     0x1.12e0e94e8586dp-11},
-    {0x1.552aa25e57bfdp-2, -0x1.c16d811e4acadp-4, 0x1.ce8489b47aa51p-5,
-     -0x1.dfde7ff758ea8p-6, 0x1.901f43aac38c8p-7, -0x1.b581d07df5ad5p-9,
-     0x1.c3726535f1fc6p-12},
-    {0x1.551d5d9b204d3p-2, -0x1.c019e328f8db1p-4, 0x1.c7710f44fc3cep-5,
-     -0x1.cbbbe25ea8ba4p-6, 0x1.6fe270088623dp-7, -0x1.7e6fc79733761p-9,
-     0x1.75077abf18d84p-12},
-};
-
-} // anonymous namespace
-
-LLVM_LIBC_FUNCTION(float, cbrtf, (float x)) {
-  using FloatBits = typename fputil::FPBits<float>;
-  using DoubleBits = typename fputil::FPBits<double>;
-
-  FloatBits x_bits(x);
-
-  uint32_t x_abs = x_bits.uintval() & 0x7fff'ffff;
-  uint32_t sign_bit = (x_bits.uintval() >> 31) << DoubleBits::EXP_LEN;
-
-  if (LIBC_UNLIKELY(x == 0.0f || x_abs >= 0x7f80'0000)) {
-    // x is 0, Inf, or NaN.
-    // Make sure it works for FTZ/DAZ modes.
-    return x + x;
-  }
-
-  double xd = static_cast<double>(x);
-  DoubleBits xd_bits(xd);
-
-  // When using biased exponent of x in double precision,
-  //   x_e = real_exponent_of_x + 1023
-  // Then:
-  //   x_e / 3 = real_exponent_of_x / 3 + 1023/3
-  //           = real_exponent_of_x / 3 + 341
-  // So to make it the correct biased exponent of x^(1/3), we add
-  //   1023 - 341 = 682
-  // to the quotient x_e / 3.
-  unsigned x_e = static_cast<unsigned>(xd_bits.get_biased_exponent());
-  unsigned out_e = (x_e / 3 + 682) | sign_bit;
-  unsigned shift_e = x_e % 3;
-
-  // Set x_m = 2^(x_e % 3) * (1.mantissa)
-  uint64_t x_m = xd_bits.get_mantissa();
-  // Use the leading 4 bits for look up table
-  unsigned idx = static_cast<unsigned>(x_m >> (DoubleBits::FRACTION_LEN - 4));
-
-  x_m |= static_cast<uint64_t>(DoubleBits::EXP_BIAS)
-         << DoubleBits::FRACTION_LEN;
-
-  double x_reduced = DoubleBits(x_m).get_val();
-  double dx = x_reduced - 1.0;
-
-  double dx_sq = dx * dx;
-  double c0 = fputil::multiply_add(dx, COEFFS[idx][0], 1.0);
-  double c1 = fputil::multiply_add(dx, COEFFS[idx][2], COEFFS[idx][1]);
-  double c2 = fputil::multiply_add(dx, COEFFS[idx][4], COEFFS[idx][3]);
-  double c3 = fputil::multiply_add(dx, COEFFS[idx][6], COEFFS[idx][5]);
-
-  double dx_4 = dx_sq * dx_sq;
-  double p0 = fputil::multiply_add(dx_sq, c1, c0);
-  double p1 = fputil::multiply_add(dx_sq, c3, c2);
-
-  double r = fputil::multiply_add(dx_4, p1, p0) * CBRT2[shift_e];
-
-  uint64_t r_m = DoubleBits(r).get_mantissa();
-  // Check if the output is exact.  To be exact, the smallest 1-bit of the
-  // output has to be at least 2^-7 or higher.  So we check the lowest 44 bits
-  // to see if they are within 2^(-52 + 3) errors from all zeros, then the
-  // result cube root is exact.
-  if (LIBC_UNLIKELY(((r_m + 8) & 0xfffffffffff) <= 16)) {
-    if ((r_m & 0xfffffffffff) <= 8)
-      r_m &= 0xffff'ffff'ffff'ffe0;
-    else
-      r_m = (r_m & 0xffff'ffff'ffff'ffe0) + 0x20;
-    fputil::clear_except_if_required(FE_INEXACT);
-  }
-  // Adjust exponent and sign.
-  uint64_t r_bits =
-      r_m | (static_cast<uint64_t>(out_e) << DoubleBits::FRACTION_LEN);
-
-  return static_cast<float>(DoubleBits(r_bits).get_val());
-}
+LLVM_LIBC_FUNCTION(float, cbrtf, (float x)) { return math::cbrtf(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/ceilbf16.cpp b/libc/src/math/generic/ceilbf16.cpp
new file mode 100644
index 0000000000000..441dcf0e93d6f
--- /dev/null
+++ b/libc/src/math/generic/ceilbf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of ceilbf16 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ceilbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, ceilbf16, (bfloat16 x)) { return fputil::ceil(x); }
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/cos.cpp b/libc/src/math/generic/cos.cpp
index 5da0f86812a89..aabf3bc7edcb0 100644
--- a/libc/src/math/generic/cos.cpp
+++ b/libc/src/math/generic/cos.cpp
@@ -7,161 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/cos.h"
-#include "hdr/errno_macros.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/double_double.h"
-#include "src/__support/FPUtil/dyadic_float.h"
-#include "src/__support/FPUtil/except_value_utils.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-#include "src/math/generic/range_reduction_double_common.h"
-#include "src/math/generic/sincos_eval.h"
-
-#ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
-#include "range_reduction_double_fma.h"
-#else
-#include "range_reduction_double_nofma.h"
-#endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
+#include "src/__support/math/cos.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-using DoubleDouble = fputil::DoubleDouble;
-using Float128 = typename fputil::DyadicFloat<128>;
-
-LLVM_LIBC_FUNCTION(double, cos, (double x)) {
-  using FPBits = typename fputil::FPBits<double>;
-  FPBits xbits(x);
-
-  uint16_t x_e = xbits.get_biased_exponent();
-
-  DoubleDouble y;
-  unsigned k;
-  LargeRangeReduction range_reduction_large{};
-
-  // |x| < 2^16.
-  if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT)) {
-    // |x| < 2^-7
-    if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 7)) {
-      // |x| < 2^-27
-      if (LIBC_UNLIKELY(x_e < FPBits::EXP_BIAS - 27)) {
-        // Signed zeros.
-        if (LIBC_UNLIKELY(x == 0.0))
-          return 1.0;
-
-        // For |x| < 2^-27, |cos(x) - 1| < |x|^2/2 < 2^-54 = ulp(1 - 2^-53)/2.
-        return fputil::round_result_slightly_down(1.0);
-      }
-      // No range reduction needed.
-      k = 0;
-      y.lo = 0.0;
-      y.hi = x;
-    } else {
-      // Small range reduction.
-      k = range_reduction_small(x, y);
-    }
-  } else {
-    // Inf or NaN
-    if (LIBC_UNLIKELY(x_e > 2 * FPBits::EXP_BIAS)) {
-      if (xbits.is_signaling_nan()) {
-        fputil::raise_except_if_required(FE_INVALID);
-        return FPBits::quiet_nan().get_val();
-      }
-      // cos(+-Inf) = NaN
-      if (xbits.get_mantissa() == 0) {
-        fputil::set_errno_if_required(EDOM);
-        fputil::raise_except_if_required(FE_INVALID);
-      }
-      return x + FPBits::quiet_nan().get_val();
-    }
-
-    // Large range reduction.
-    k = range_reduction_large.fast(x, y);
-  }
-
-  DoubleDouble sin_y, cos_y;
-
-  [[maybe_unused]] double err = generic::sincos_eval(y, sin_y, cos_y);
-
-  // Look up sin(k * pi/128) and cos(k * pi/128)
-#ifdef LIBC_MATH_HAS_SMALL_TABLES
-  // Memory saving versions.  Use 65-entry table.
-  auto get_idx_dd = [](unsigned kk) -> DoubleDouble {
-    unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
-    DoubleDouble ans = SIN_K_PI_OVER_128[idx];
-    if (kk & 128) {
-      ans.hi = -ans.hi;
-      ans.lo = -ans.lo;
-    }
-    return ans;
-  };
-  DoubleDouble msin_k = get_idx_dd(k + 128);
-  DoubleDouble cos_k = get_idx_dd(k + 64);
-#else
-  // Fast look up version, but needs 256-entry table.
-  // -sin(k * pi/128) = sin((k + 128) * pi/128)
-  // cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128).
-  DoubleDouble msin_k = SIN_K_PI_OVER_128[(k + 128) & 255];
-  DoubleDouble cos_k = SIN_K_PI_OVER_128[(k + 64) & 255];
-#endif // LIBC_MATH_HAS_SMALL_TABLES
-
-  // After range reduction, k = round(x * 128 / pi) and y = x - k * (pi / 128).
-  // So k is an integer and -pi / 256 <= y <= pi / 256.
-  // Then cos(x) = cos((k * pi/128 + y)
-  //             = cos(y) * cos(k*pi/128) - sin(y) * sin(k*pi/128)
-  DoubleDouble cos_k_cos_y = fputil::quick_mult(cos_y, cos_k);
-  DoubleDouble msin_k_sin_y = fputil::quick_mult(sin_y, msin_k);
-
-  DoubleDouble rr = fputil::exact_add<false>(cos_k_cos_y.hi, msin_k_sin_y.hi);
-  rr.lo += msin_k_sin_y.lo + cos_k_cos_y.lo;
-
-#ifdef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-  return rr.hi + rr.lo;
-#else
-
-  double rlp = rr.lo + err;
-  double rlm = rr.lo - err;
-
-  double r_upper = rr.hi + rlp; // (rr.lo + ERR);
-  double r_lower = rr.hi + rlm; // (rr.lo - ERR);
-
-  // Ziv's rounding test.
-  if (LIBC_LIKELY(r_upper == r_lower))
-    return r_upper;
-
-  Float128 u_f128, sin_u, cos_u;
-  if (LIBC_LIKELY(x_e < FPBits::EXP_BIAS + FAST_PASS_EXPONENT))
-    u_f128 = range_reduction_small_f128(x);
-  else
-    u_f128 = range_reduction_large.accurate();
-
-  generic::sincos_eval(u_f128, sin_u, cos_u);
-
-  auto get_sin_k = [](unsigned kk) -> Float128 {
-    unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
-    Float128 ans = SIN_K_PI_OVER_128_F128[idx];
-    if (kk & 128)
-      ans.sign = Sign::NEG;
-    return ans;
-  };
-
-  // -sin(k * pi/128) = sin((k + 128) * pi/128)
-  // cos(k * pi/128) = sin(k * pi/128 + pi/2) = sin((k + 64) * pi/128).
-  Float128 msin_k_f128 = get_sin_k(k + 128);
-  Float128 cos_k_f128 = get_sin_k(k + 64);
-
-  // cos(x) = cos((k * pi/128 + u)
-  //        = cos(u) * cos(k*pi/128) - sin(u) * sin(k*pi/128)
-  Float128 r = fputil::quick_add(fputil::quick_mul(cos_k_f128, cos_u),
-                                 fputil::quick_mul(msin_k_f128, sin_u));
-
-  // TODO: Add assertion if Ziv's accuracy tests fail in debug mode.
-  // https://github.com/llvm/llvm-project/issues/96452.
-
-  return static_cast<double>(r);
-#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-}
+LLVM_LIBC_FUNCTION(double, cos, (double x)) { return math::cos(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/cosf.cpp b/libc/src/math/generic/cosf.cpp
index 7cdae09869588..5c23d99d52441 100644
--- a/libc/src/math/generic/cosf.cpp
+++ b/libc/src/math/generic/cosf.cpp
@@ -7,139 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/cosf.h"
-#include "sincosf_utils.h"
-#include "src/__support/FPUtil/BasicOperations.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/except_value_utils.h"
-#include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+#include "src/__support/math/cosf.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-// Exceptional cases for cosf.
-static constexpr size_t N_EXCEPTS = 6;
-
-static constexpr fputil::ExceptValues<float, N_EXCEPTS> COSF_EXCEPTS{{
-    // (inputs, RZ output, RU offset, RD offset, RN offset)
-    // x = 0x1.64a032p43, cos(x) = 0x1.9d4ba4p-1 (RZ)
-    {0x55325019, 0x3f4ea5d2, 1, 0, 0},
-    // x = 0x1.4555p51, cos(x) = 0x1.115d7cp-1 (RZ)
-    {0x5922aa80, 0x3f08aebe, 1, 0, 1},
-    // x = 0x1.48a858p54, cos(x) = 0x1.f48148p-2 (RZ)
-    {0x5aa4542c, 0x3efa40a4, 1, 0, 0},
-    // x = 0x1.3170fp63, cos(x) = 0x1.fe2976p-1 (RZ)
-    {0x5f18b878, 0x3f7f14bb, 1, 0, 0},
-    // x = 0x1.2b9622p67, cos(x) = 0x1.f0285cp-1 (RZ)
-    {0x6115cb11, 0x3f78142e, 1, 0, 1},
-    // x = 0x1.ddebdep120, cos(x) = 0x1.114438p-1 (RZ)
-    {0x7beef5ef, 0x3f08a21c, 1, 0, 0},
-}};
-#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-
-LLVM_LIBC_FUNCTION(float, cosf, (float x)) {
-  using FPBits = typename fputil::FPBits<float>;
-
-  FPBits xbits(x);
-  xbits.set_sign(Sign::POS);
-
-  uint32_t x_abs = xbits.uintval();
-  double xd = static_cast<double>(xbits.get_val());
-
-  // Range reduction:
-  // For |x| > pi/16, we perform range reduction as follows:
-  // Find k and y such that:
-  //   x = (k + y) * pi/32
-  //   k is an integer
-  //   |y| < 0.5
-  // For small range (|x| < 2^45 when FMA instructions are available, 2^22
-  // otherwise), this is done by performing:
-  //   k = round(x * 32/pi)
-  //   y = x * 32/pi - k
-  // For large range, we will omit all the higher parts of 16/pi such that the
-  // least significant bits of their full products with x are larger than 63,
-  // since cos((k + y + 64*i) * pi/32) = cos(x + i * 2pi) = cos(x).
-  //
-  // When FMA instructions are not available, we store the digits of 32/pi in
-  // chunks of 28-bit precision.  This will make sure that the products:
-  //   x * THIRTYTWO_OVER_PI_28[i] are all exact.
-  // When FMA instructions are available, we simply store the digits of 32/pi in
-  // chunks of doubles (53-bit of precision).
-  // So when multiplying by the largest values of single precision, the
-  // resulting output should be correct up to 2^(-208 + 128) ~ 2^-80.  By the
-  // worst-case analysis of range reduction, |y| >= 2^-38, so this should give
-  // us more than 40 bits of accuracy. For the worst-case estimation of range
-  // reduction, see for instances:
-  //   Elementary Functions by J-M. Muller, Chapter 11,
-  //   Handbook of Floating-Point Arithmetic by J-M. Muller et. al.,
-  //   Chapter 10.2.
-  //
-  // Once k and y are computed, we then deduce the answer by the cosine of sum
-  // formula:
-  //   cos(x) = cos((k + y)*pi/32)
-  //          = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
-  // The values of sin(k*pi/32) and cos(k*pi/32) for k = 0..63 are precomputed
-  // and stored using a vector of 32 doubles. Sin(y*pi/32) and cos(y*pi/32) are
-  // computed using degree-7 and degree-6 minimax polynomials generated by
-  // Sollya respectively.
-
-  // |x| < 0x1.0p-12f
-  if (LIBC_UNLIKELY(x_abs < 0x3980'0000U)) {
-    // When |x| < 2^-12, the relative error of the approximation cos(x) ~ 1
-    // is:
-    //   |cos(x) - 1| < |x^2 / 2| = 2^-25 < epsilon(1)/2.
-    // So the correctly rounded values of cos(x) are:
-    //   = 1 - eps(x) if rounding mode = FE_TOWARDZERO or FE_DOWWARD,
-    //   = 1 otherwise.
-    // To simplify the rounding decision and make it more efficient and to
-    // prevent compiler to perform constant folding, we use
-    //   fma(x, -2^-25, 1) instead.
-    // Note: to use the formula 1 - 2^-25*x to decide the correct rounding, we
-    // do need fma(x, -2^-25, 1) to prevent underflow caused by -2^-25*x when
-    // |x| < 2^-125. For targets without FMA instructions, we simply use
-    // double for intermediate results as it is more efficient than using an
-    // emulated version of FMA.
-#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
-    return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f);
-#else
-    return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, 1.0));
-#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
-  }
-
-#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-  if (auto r = COSF_EXCEPTS.lookup(x_abs); LIBC_UNLIKELY(r.has_value()))
-    return r.value();
-#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-
-  // x is inf or nan.
-  if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) {
-    if (xbits.is_signaling_nan()) {
-      fputil::raise_except_if_required(FE_INVALID);
-      return FPBits::quiet_nan().get_val();
-    }
-
-    if (x_abs == 0x7f80'0000U) {
-      fputil::set_errno_if_required(EDOM);
-      fputil::raise_except_if_required(FE_INVALID);
-    }
-    return x + FPBits::quiet_nan().get_val();
-  }
-
-  // Combine the results with the sine of sum formula:
-  //   cos(x) = cos((k + y)*pi/32)
-  //          = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
-  //          = cosm1_y * cos_k + sin_y * sin_k
-  //          = (cosm1_y * cos_k + cos_k) + sin_y * sin_k
-  double sin_k, cos_k, sin_y, cosm1_y;
-
-  sincosf_eval(xd, x_abs, sin_k, cos_k, sin_y, cosm1_y);
-
-  return static_cast<float>(fputil::multiply_add(
-      sin_y, -sin_k, fputil::multiply_add(cosm1_y, cos_k, cos_k)));
-}
+LLVM_LIBC_FUNCTION(float, cosf, (float x)) { return math::cosf(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/cosf16.cpp b/libc/src/math/generic/cosf16.cpp
index 99bb03eb71426..031c3e131d972 100644
--- a/libc/src/math/generic/cosf16.cpp
+++ b/libc/src/math/generic/cosf16.cpp
@@ -7,87 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/cosf16.h"
-#include "hdr/errno_macros.h"
-#include "hdr/fenv_macros.h"
-#include "sincosf16_utils.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/cast.h"
-#include "src/__support/FPUtil/except_value_utils.h"
-#include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/macros/optimization.h"
+#include "src/__support/math/cosf16.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-constexpr size_t N_EXCEPTS = 4;
-
-constexpr fputil::ExceptValues<float16, N_EXCEPTS> COSF16_EXCEPTS{{
-    // (input, RZ output, RU offset, RD offset, RN offset)
-    {0x2b7c, 0x3bfc, 1, 0, 1},
-    {0x4ac1, 0x38b5, 1, 0, 0},
-    {0x5c49, 0xb8c6, 0, 1, 0},
-    {0x7acc, 0xa474, 0, 1, 0},
-}};
-#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-
-LLVM_LIBC_FUNCTION(float16, cosf16, (float16 x)) {
-  using FPBits = fputil::FPBits<float16>;
-  FPBits xbits(x);
-
-  uint16_t x_u = xbits.uintval();
-  uint16_t x_abs = x_u & 0x7fff;
-  float xf = x;
-
-  // Range reduction:
-  // For |x| > pi/32, we perform range reduction as follows:
-  // Find k and y such that:
-  //   x = (k + y) * pi/32
-  //   k is an integer, |y| < 0.5
-  //
-  // This is done by performing:
-  //   k = round(x * 32/pi)
-  //   y = x * 32/pi - k
-  //
-  // Once k and y are computed, we then deduce the answer by the cosine of sum
-  // formula:
-  //   cos(x) = cos((k + y) * pi/32)
-  //          = cos(k * pi/32) * cos(y * pi/32) -
-  //            sin(k * pi/32) * sin(y * pi/32)
-
-#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-  // Handle exceptional values
-  if (auto r = COSF16_EXCEPTS.lookup(x_abs); LIBC_UNLIKELY(r.has_value()))
-    return r.value();
-#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-
-  // cos(+/-0) = 1
-  if (LIBC_UNLIKELY(x_abs == 0U))
-    return fputil::cast<float16>(1.0f);
-
-  // cos(+/-inf) = NaN, and cos(NaN) = NaN
-  if (xbits.is_inf_or_nan()) {
-    if (xbits.is_signaling_nan()) {
-      fputil::raise_except_if_required(FE_INVALID);
-      return FPBits::quiet_nan().get_val();
-    }
-
-    if (xbits.is_inf()) {
-      fputil::set_errno_if_required(EDOM);
-      fputil::raise_except_if_required(FE_INVALID);
-    }
-
-    return x + FPBits::quiet_nan().get_val();
-  }
-
-  float sin_k, cos_k, sin_y, cosm1_y;
-  sincosf16_eval(xf, sin_k, cos_k, sin_y, cosm1_y);
-  // Since, cosm1_y = cos_y - 1, therefore:
-  //   cos(x) = cos_k * cos_y - sin_k * sin_y
-  //          = cos_k * (cos_y - 1 + 1) - sin_k * sin_y
-  //          = cos_k * cosm1_y - sin_k * sin_y + cos_k
-  return fputil::cast<float16>(fputil::multiply_add(
-      cos_k, cosm1_y, fputil::multiply_add(-sin_k, sin_y, cos_k)));
-}
+LLVM_LIBC_FUNCTION(float16, cosf16, (float16 x)) { return math::cosf16(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/coshf.cpp b/libc/src/math/generic/coshf.cpp
index 9f87564d524a6..368c0fda32e21 100644
--- a/libc/src/math/generic/coshf.cpp
+++ b/libc/src/math/generic/coshf.cpp
@@ -7,50 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/coshf.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/rounding_mode.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
-#include "src/math/generic/explogxf.h"
+#include "src/__support/math/coshf.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, coshf, (float x)) {
-  using FPBits = typename fputil::FPBits<float>;
-
-  FPBits xbits(x);
-  xbits.set_sign(Sign::POS);
-  x = xbits.get_val();
-
-  uint32_t x_u = xbits.uintval();
-
-  // When |x| >= 90, or x is inf or nan
-  if (LIBC_UNLIKELY(x_u >= 0x42b4'0000U || x_u <= 0x3280'0000U)) {
-    // |x| <= 2^-26
-    if (x_u <= 0x3280'0000U) {
-      return 1.0f + x;
-    }
-
-    if (xbits.is_inf_or_nan())
-      return x + FPBits::inf().get_val();
-
-    int rounding = fputil::quick_get_round();
-    if (LIBC_UNLIKELY(rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO))
-      return FPBits::max_normal().get_val();
-
-    fputil::set_errno_if_required(ERANGE);
-    fputil::raise_except_if_required(FE_OVERFLOW);
-
-    return x + FPBits::inf().get_val();
-  }
-
-  // TODO: We should be able to reduce the latency and reciprocal throughput
-  // further by using a low degree (maybe 3-7 ?) minimax polynomial for small
-  // but not too small inputs, such as |x| < 2^-2, or |x| < 2^-3.
-
-  // cosh(x) = (e^x + e^(-x)) / 2.
-  return static_cast<float>(exp_pm_eval</*is_sinh*/ false>(x));
-}
+LLVM_LIBC_FUNCTION(float, coshf, (float x)) { return math::coshf(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/coshf16.cpp b/libc/src/math/generic/coshf16.cpp
index 689d16a55260f..d86edd98f29e9 100644
--- a/libc/src/math/generic/coshf16.cpp
+++ b/libc/src/math/generic/coshf16.cpp
@@ -7,105 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/coshf16.h"
-#include "expxf16.h"
-#include "hdr/errno_macros.h"
-#include "hdr/fenv_macros.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/except_value_utils.h"
-#include "src/__support/FPUtil/rounding_mode.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/optimization.h"
+#include "src/__support/math/coshf16.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-static constexpr fputil::ExceptValues<float16, 9> COSHF16_EXCEPTS_POS = {{
-    // x = 0x1.6ap-5, coshf16(x) = 0x1p+0 (RZ)
-    {0x29a8U, 0x3c00U, 1U, 0U, 1U},
-    // x = 0x1.8c4p+0, coshf16(x) = 0x1.3a8p+1 (RZ)
-    {0x3e31U, 0x40eaU, 1U, 0U, 0U},
-    // x = 0x1.994p+0, coshf16(x) = 0x1.498p+1 (RZ)
-    {0x3e65U, 0x4126U, 1U, 0U, 0U},
-    // x = 0x1.b6p+0, coshf16(x) = 0x1.6d8p+1 (RZ)
-    {0x3ed8U, 0x41b6U, 1U, 0U, 1U},
-    // x = 0x1.aap+1, coshf16(x) = 0x1.be8p+3 (RZ)
-    {0x42a8U, 0x4afaU, 1U, 0U, 1U},
-    // x = 0x1.cc4p+1, coshf16(x) = 0x1.23cp+4 (RZ)
-    {0x4331U, 0x4c8fU, 1U, 0U, 0U},
-    // x = 0x1.288p+2, coshf16(x) = 0x1.9b4p+5 (RZ)
-    {0x44a2U, 0x526dU, 1U, 0U, 0U},
-    // x = 0x1.958p+2, coshf16(x) = 0x1.1a4p+8 (RZ)
-    {0x4656U, 0x5c69U, 1U, 0U, 0U},
-    // x = 0x1.5fp+3, coshf16(x) = 0x1.c54p+14 (RZ)
-    {0x497cU, 0x7715U, 1U, 0U, 1U},
-}};
-
-static constexpr fputil::ExceptValues<float16, 6> COSHF16_EXCEPTS_NEG = {{
-    // x = -0x1.6ap-5, coshf16(x) = 0x1p+0 (RZ)
-    {0xa9a8U, 0x3c00U, 1U, 0U, 1U},
-    // x = -0x1.b6p+0, coshf16(x) = 0x1.6d8p+1 (RZ)
-    {0xbed8U, 0x41b6U, 1U, 0U, 1U},
-    // x = -0x1.288p+2, coshf16(x) = 0x1.9b4p+5 (RZ)
-    {0xc4a2U, 0x526dU, 1U, 0U, 0U},
-    // x = -0x1.5fp+3, coshf16(x) = 0x1.c54p+14 (RZ)
-    {0xc97cU, 0x7715U, 1U, 0U, 1U},
-    // x = -0x1.8c4p+0, coshf16(x) = 0x1.3a8p+1 (RZ)
-    {0xbe31U, 0x40eaU, 1U, 0U, 0U},
-    // x = -0x1.994p+0, coshf16(x) = 0x1.498p+1 (RZ)
-    {0xbe65U, 0x4126U, 1U, 0U, 0U},
-}};
-#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-
-LLVM_LIBC_FUNCTION(float16, coshf16, (float16 x)) {
-  using FPBits = fputil::FPBits<float16>;
-  FPBits x_bits(x);
-
-  uint16_t x_u = x_bits.uintval();
-  uint16_t x_abs = x_u & 0x7fffU;
-
-  // When |x| >= acosh(2^16), or x is NaN.
-  if (LIBC_UNLIKELY(x_abs >= 0x49e5U)) {
-    // cosh(NaN) = NaN
-    if (x_bits.is_nan()) {
-      if (x_bits.is_signaling_nan()) {
-        fputil::raise_except_if_required(FE_INVALID);
-        return FPBits::quiet_nan().get_val();
-      }
-
-      return x;
-    }
-
-    // When |x| >= acosh(2^16).
-    if (x_abs >= 0x49e5U) {
-      // cosh(+/-inf) = +inf
-      if (x_bits.is_inf())
-        return FPBits::inf().get_val();
-
-      switch (fputil::quick_get_round()) {
-      case FE_TONEAREST:
-      case FE_UPWARD:
-        fputil::set_errno_if_required(ERANGE);
-        fputil::raise_except_if_required(FE_OVERFLOW | FE_INEXACT);
-        return FPBits::inf().get_val();
-      default:
-        return FPBits::max_normal().get_val();
-      }
-    }
-  }
-
-#ifndef LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-  if (x_bits.is_pos()) {
-    if (auto r = COSHF16_EXCEPTS_POS.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
-      return r.value();
-  } else {
-    if (auto r = COSHF16_EXCEPTS_NEG.lookup(x_u); LIBC_UNLIKELY(r.has_value()))
-      return r.value();
-  }
-#endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
-
-  return eval_sinh_or_cosh</*IsSinh=*/false>(x);
-}
+LLVM_LIBC_FUNCTION(float16, coshf16, (float16 x)) { return math::coshf16(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/cospif.cpp b/libc/src/math/generic/cospif.cpp
index 5b6880f853b26..b9a4637da2ef4 100644
--- a/libc/src/math/generic/cospif.cpp
+++ b/libc/src/math/generic/cospif.cpp
@@ -7,95 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/cospif.h"
-#include "sincosf_utils.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/common.h"
-#include "src/__support/macros/config.h"
-#include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+#include "src/__support/math/cospif.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float, cospif, (float x)) {
-  using FPBits = typename fputil::FPBits<float>;
-
-  FPBits xbits(x);
-  xbits.set_sign(Sign::POS);
-
-  uint32_t x_abs = xbits.uintval();
-  double xd = static_cast<double>(xbits.get_val());
-
-  // Range reduction:
-  // For |x| > 1/32, we perform range reduction as follows:
-  // Find k and y such that:
-  //   x = (k + y) * 1/32
-  //   k is an integer
-  //   |y| < 0.5
-  //
-  // This is done by performing:
-  //   k = round(x * 32)
-  //   y = x * 32 - k
-  //
-  // Once k and y are computed, we then deduce the answer by the cosine of sum
-  // formula:
-  //   cospi(x) = cos((k + y)*pi/32)
-  //          = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
-  // The values of sin(k*pi/32) and cos(k*pi/32) for k = 0..63 are precomputed
-  // and stored using a vector of 32 doubles. Sin(y*pi/32) and cos(y*pi/32) are
-  // computed using degree-7 and degree-6 minimax polynomials generated by
-  // Sollya respectively.
-
-  // The exhautive test passes for smaller values
-  if (LIBC_UNLIKELY(x_abs < 0x38A2'F984U)) {
-
-#if defined(LIBC_TARGET_CPU_HAS_FMA_FLOAT)
-    return fputil::multiply_add(xbits.get_val(), -0x1.0p-25f, 1.0f);
-#else
-    return static_cast<float>(fputil::multiply_add(xd, -0x1.0p-25, 1.0));
-#endif // LIBC_TARGET_CPU_HAS_FMA_FLOAT
-  }
-
-  // Numbers greater or equal to 2^23 are always integers or NaN
-  if (LIBC_UNLIKELY(x_abs >= 0x4B00'0000)) {
-
-    if (LIBC_UNLIKELY(x_abs < 0x4B80'0000)) {
-      return (x_abs & 0x1) ? -1.0f : 1.0f;
-    }
-
-    // x is inf or nan.
-    if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) {
-      if (xbits.is_signaling_nan()) {
-        fputil::raise_except_if_required(FE_INVALID);
-        return FPBits::quiet_nan().get_val();
-      }
-
-      if (x_abs == 0x7f80'0000U) {
-        fputil::set_errno_if_required(EDOM);
-        fputil::raise_except_if_required(FE_INVALID);
-      }
-      return x + FPBits::quiet_nan().get_val();
-    }
-
-    return 1.0f;
-  }
-
-  // Combine the results with the sine of sum formula:
-  //   cos(pi * x) = cos((k + y)*pi/32)
-  //          = cos(y*pi/32) * cos(k*pi/32) - sin(y*pi/32) * sin(k*pi/32)
-  //          = (cosm1_y + 1) * cos_k - sin_y * sin_k
-  //          = (cosm1_y * cos_k + cos_k) - sin_y * sin_k
-  double sin_k, cos_k, sin_y, cosm1_y;
-
-  sincospif_eval(xd, sin_k, cos_k, sin_y, cosm1_y);
-
-  if (LIBC_UNLIKELY(sin_y == 0 && cos_k == 0)) {
-    return 0.0f;
-  }
-
-  return static_cast<float>(fputil::multiply_add(
-      sin_y, -sin_k, fputil::multiply_add(cosm1_y, cos_k, cos_k)));
-}
+LLVM_LIBC_FUNCTION(float, cospif, (float x)) { return math::cospif(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/cospif16.cpp b/libc/src/math/generic/cospif16.cpp
index 9dc25920d5cfe..a4bae8e13eac8 100644
--- a/libc/src/math/generic/cospif16.cpp
+++ b/libc/src/math/generic/cospif16.cpp
@@ -7,79 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/cospif16.h"
-#include "hdr/errno_macros.h"
-#include "hdr/fenv_macros.h"
-#include "sincosf16_utils.h"
-#include "src/__support/FPUtil/FEnvImpl.h"
-#include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/cast.h"
-#include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/macros/optimization.h"
+#include "src/__support/math/cospif16.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(float16, cospif16, (float16 x)) {
-  using FPBits = typename fputil::FPBits<float16>;
-  FPBits xbits(x);
-
-  uint16_t x_u = xbits.uintval();
-  uint16_t x_abs = x_u & 0x7fff;
-  float xf = x;
-
-  // Range reduction:
-  // For |x| > 1/32, we perform range reduction as follows:
-  // Find k and y such that:
-  //   x = (k + y) * 1/32
-  //   k is an integer
-  //   |y| < 0.5
-  //
-  // This is done by performing:
-  //   k = round(x * 32)
-  //   y = x * 32 - k
-  //
-  // Once k and y are computed, we then deduce the answer by the cosine of sum
-  // formula:
-  //   cos(x * pi) = cos((k + y) * pi/32)
-  //               = cos(k * pi/32) * cos(y * pi/32) +
-  //                 sin(y * pi/32) * sin(k * pi/32)
-
-  // For signed zeros
-  if (LIBC_UNLIKELY(x_abs == 0U))
-    return fputil::cast<float16>(1.0f);
-
-  // Numbers greater or equal to 2^10 are integers, or infinity, or NaN
-  if (LIBC_UNLIKELY(x_abs >= 0x6400)) {
-    if (LIBC_UNLIKELY(x_abs <= 0x67FF))
-      return fputil::cast<float16>((x_abs & 0x1) ? -1.0f : 1.0f);
-
-    // Check for NaN or infintiy values
-    if (LIBC_UNLIKELY(x_abs >= 0x7c00)) {
-      if (xbits.is_signaling_nan()) {
-        fputil::raise_except_if_required(FE_INVALID);
-        return FPBits::quiet_nan().get_val();
-      }
-      // If value is equal to infinity
-      if (x_abs == 0x7c00) {
-        fputil::set_errno_if_required(EDOM);
-        fputil::raise_except_if_required(FE_INVALID);
-      }
-
-      return x + FPBits::quiet_nan().get_val();
-    }
-
-    return fputil::cast<float16>(1.0f);
-  }
-
-  float sin_k, cos_k, sin_y, cosm1_y;
-  sincospif16_eval(xf, sin_k, cos_k, sin_y, cosm1_y);
-
-  if (LIBC_UNLIKELY(sin_y == 0 && cos_k == 0))
-    return fputil::cast<float16>(0.0f);
-
-  // Since, cosm1_y = cos_y - 1, therefore:
-  //   cos(x * pi) = cos_k(cosm1_y) + cos_k - sin_k * sin_y
-  return fputil::cast<float16>(fputil::multiply_add(
-      cos_k, cosm1_y, fputil::multiply_add(-sin_k, sin_y, cos_k)));
-}
+LLVM_LIBC_FUNCTION(float16, cospif16, (float16 x)) { return math::cospif16(x); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/exp10m1f.cpp b/libc/src/math/generic/exp10m1f.cpp
index 27729104e038d..8589e3fb6639d 100644
--- a/libc/src/math/generic/exp10m1f.cpp
+++ b/libc/src/math/generic/exp10m1f.cpp
@@ -17,8 +17,7 @@
 #include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
-
-#include "explogxf.h"
+#include "src/__support/math/exp10f_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/math/generic/exp2.cpp b/libc/src/math/generic/exp2.cpp
index 726f88b6457fc..154154f2b90c3 100644
--- a/libc/src/math/generic/exp2.cpp
+++ b/libc/src/math/generic/exp2.cpp
@@ -8,7 +8,6 @@
 
 #include "src/math/exp2.h"
 #include "common_constants.h" // Lookup tables EXP2_MID1 and EXP_M2.
-#include "explogxf.h"         // ziv_test_denorm.
 #include "src/__support/CPP/bit.h"
 #include "src/__support/CPP/optional.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -24,6 +23,7 @@
 #include "src/__support/integer_literals.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+#include "src/__support/math/exp_utils.h"      // ziv_test_denorm.
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/math/generic/exp2f16.cpp b/libc/src/math/generic/exp2f16.cpp
index 5c039c59df1af..5db0c3a9de08d 100644
--- a/libc/src/math/generic/exp2f16.cpp
+++ b/libc/src/math/generic/exp2f16.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/exp2f16.h"
-#include "expxf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -18,6 +17,7 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
+#include "src/__support/math/expxf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -34,6 +34,7 @@ static constexpr fputil::ExceptValues<float16, 3> EXP2F16_EXCEPTS = {{
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, exp2f16, (float16 x)) {
+  using namespace math::expxf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits x_bits(x);
 
diff --git a/libc/src/math/generic/exp2f_impl.h b/libc/src/math/generic/exp2f_impl.h
index 5c6c2bd415188..b85bb1548019f 100644
--- a/libc/src/math/generic/exp2f_impl.h
+++ b/libc/src/math/generic/exp2f_impl.h
@@ -20,8 +20,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
 #include "src/__support/macros/properties/cpu_features.h"
-
-#include "explogxf.h"
+#include "src/__support/math/exp10f_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 namespace generic {
diff --git a/libc/src/math/generic/exp2m1f.cpp b/libc/src/math/generic/exp2m1f.cpp
index 127c6eaa494d4..16244edb4c583 100644
--- a/libc/src/math/generic/exp2m1f.cpp
+++ b/libc/src/math/generic/exp2m1f.cpp
@@ -18,8 +18,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
 #include "src/__support/macros/properties/cpu_features.h"
-
-#include "explogxf.h"
+#include "src/__support/math/exp10f_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/math/generic/exp2m1f16.cpp b/libc/src/math/generic/exp2m1f16.cpp
index 61633cd2cfcfb..ce0cc60748f19 100644
--- a/libc/src/math/generic/exp2m1f16.cpp
+++ b/libc/src/math/generic/exp2m1f16.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/exp2m1f16.h"
-#include "expxf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -21,6 +20,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
 #include "src/__support/macros/properties/cpu_features.h"
+#include "src/__support/math/expxf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -76,6 +76,7 @@ static constexpr fputil::ExceptValues<float16, N_EXP2M1F16_EXCEPTS_HI>
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, exp2m1f16, (float16 x)) {
+  using namespace math::expxf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits x_bits(x);
 
diff --git a/libc/src/math/generic/expm1.cpp b/libc/src/math/generic/expm1.cpp
index a4dbf38ab27d0..c360554a8af8f 100644
--- a/libc/src/math/generic/expm1.cpp
+++ b/libc/src/math/generic/expm1.cpp
@@ -8,9 +8,7 @@
 
 #include "src/math/expm1.h"
 #include "common_constants.h" // Lookup tables EXP_M1 and EXP_M2.
-#include "explogxf.h"         // ziv_test_denorm.
 #include "src/__support/CPP/bit.h"
-#include "src/__support/CPP/optional.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
@@ -18,7 +16,6 @@
 #include "src/__support/FPUtil/dyadic_float.h"
 #include "src/__support/FPUtil/except_value_utils.h"
 #include "src/__support/FPUtil/multiply_add.h"
-#include "src/__support/FPUtil/nearest_integer.h"
 #include "src/__support/FPUtil/rounding_mode.h"
 #include "src/__support/FPUtil/triple_double.h"
 #include "src/__support/common.h"
diff --git a/libc/src/math/generic/expm1f16.cpp b/libc/src/math/generic/expm1f16.cpp
index 2188dfbda8ef9..c2231f0aca715 100644
--- a/libc/src/math/generic/expm1f16.cpp
+++ b/libc/src/math/generic/expm1f16.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/expm1f16.h"
-#include "expxf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -20,6 +19,7 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
+#include "src/__support/math/expxf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -51,6 +51,7 @@ static constexpr fputil::ExceptValues<float16, N_EXPM1F16_EXCEPTS_HI>
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, expm1f16, (float16 x)) {
+  using namespace math::expxf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits x_bits(x);
 
diff --git a/libc/src/math/generic/floorbf16.cpp b/libc/src/math/generic/floorbf16.cpp
new file mode 100644
index 0000000000000..d157096c3e62f
--- /dev/null
+++ b/libc/src/math/generic/floorbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of floorbf16 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/floorbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, floorbf16, (bfloat16 x)) {
+  return fputil::floor(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fmaxbf16.cpp b/libc/src/math/generic/fmaxbf16.cpp
new file mode 100644
index 0000000000000..01d395bc04fa6
--- /dev/null
+++ b/libc/src/math/generic/fmaxbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fmaxbf16 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaxbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fmaxbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fmax(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fmaximum_mag_numbf16.cpp b/libc/src/math/generic/fmaximum_mag_numbf16.cpp
new file mode 100644
index 0000000000000..485e32959b53a
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_mag_numbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fmaximum_mag_numbf16 function -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_mag_numbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fmaximum_mag_numbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fmaximum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fmaximum_magbf16.cpp b/libc/src/math/generic/fmaximum_magbf16.cpp
new file mode 100644
index 0000000000000..0654ed904e549
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_magbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fmaximum_magbf16 function -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_magbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fmaximum_magbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fmaximum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fmaximum_numbf16.cpp b/libc/src/math/generic/fmaximum_numbf16.cpp
new file mode 100644
index 0000000000000..b058d50a7fd6b
--- /dev/null
+++ b/libc/src/math/generic/fmaximum_numbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fmaximum_numbf16 function -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximum_numbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fmaximum_numbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fmaximum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fmaximumbf16.cpp b/libc/src/math/generic/fmaximumbf16.cpp
new file mode 100644
index 0000000000000..e10830b4693d5
--- /dev/null
+++ b/libc/src/math/generic/fmaximumbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fmaximumbf16 function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fmaximumbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fmaximumbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fmaximum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fminbf16.cpp b/libc/src/math/generic/fminbf16.cpp
new file mode 100644
index 0000000000000..c3e29ee10c8b6
--- /dev/null
+++ b/libc/src/math/generic/fminbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fminbf16 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fminbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fmin(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fminimum_mag_numbf16.cpp b/libc/src/math/generic/fminimum_mag_numbf16.cpp
new file mode 100644
index 0000000000000..5056fc7e0cad2
--- /dev/null
+++ b/libc/src/math/generic/fminimum_mag_numbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fminimum_mag_numbf16 function -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_mag_numbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fminimum_mag_numbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fminimum_mag_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fminimum_magbf16.cpp b/libc/src/math/generic/fminimum_magbf16.cpp
new file mode 100644
index 0000000000000..f61d2d27b4a60
--- /dev/null
+++ b/libc/src/math/generic/fminimum_magbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fminimum_magbf16 function -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_magbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fminimum_magbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fminimum_mag(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fminimum_numbf16.cpp b/libc/src/math/generic/fminimum_numbf16.cpp
new file mode 100644
index 0000000000000..079a83011ec37
--- /dev/null
+++ b/libc/src/math/generic/fminimum_numbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fminimum_numbf16 function -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimum_numbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fminimum_numbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fminimum_num(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fminimumbf16.cpp b/libc/src/math/generic/fminimumbf16.cpp
new file mode 100644
index 0000000000000..da976b96b987c
--- /dev/null
+++ b/libc/src/math/generic/fminimumbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of fminimumbf16 function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fminimumbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fminimumbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::fminimum(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/frexpbf16.cpp b/libc/src/math/generic/frexpbf16.cpp
new file mode 100644
index 0000000000000..004f64f282f7d
--- /dev/null
+++ b/libc/src/math/generic/frexpbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of frexpbf16 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/frexpbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, frexpbf16, (bfloat16 x, int *exp)) {
+  return fputil::frexp(x, *exp);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fromfpbf16.cpp b/libc/src/math/generic/fromfpbf16.cpp
new file mode 100644
index 0000000000000..db1b8f18f6810
--- /dev/null
+++ b/libc/src/math/generic/fromfpbf16.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of fromfpbf16 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfpbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fromfpbf16,
+                   (bfloat16 x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/fromfpxbf16.cpp b/libc/src/math/generic/fromfpxbf16.cpp
new file mode 100644
index 0000000000000..8c16c411265ab
--- /dev/null
+++ b/libc/src/math/generic/fromfpxbf16.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of fromfpxbf16 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/fromfpxbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, fromfpxbf16,
+                   (bfloat16 x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/true>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/getpayloadbf16.cpp b/libc/src/math/generic/getpayloadbf16.cpp
new file mode 100644
index 0000000000000..544ed0a2f5c9d
--- /dev/null
+++ b/libc/src/math/generic/getpayloadbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of getpayloadbf16 function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/getpayloadbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, getpayloadbf16, (const bfloat16 *x)) {
+  return fputil::getpayload(*x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/hypotf16.cpp b/libc/src/math/generic/hypotf16.cpp
index d782c2687cdb6..fa90069f9ff0d 100644
--- a/libc/src/math/generic/hypotf16.cpp
+++ b/libc/src/math/generic/hypotf16.cpp
@@ -48,16 +48,15 @@ LLVM_LIBC_FUNCTION(float16, hypotf16, (float16 x, float16 y)) {
     return a_bits.get_val();
   }
 
-  // TODO: Investigate why replacing the return line below with:
-  //   return x_bits.get_val() + y_bits.get_val();
-  // fails the hypotf16 smoke tests.
+  float af = fputil::cast<float>(a_bits.get_val());
+  float bf = fputil::cast<float>(b_bits.get_val());
+
+  // Compiler runtime basic operations for float16 might not be correctly
+  // rounded for all rounding modes.
   if (LIBC_UNLIKELY(a_u - b_u >=
                     static_cast<uint16_t>((FPBits::FRACTION_LEN + 2)
                                           << FPBits::FRACTION_LEN)))
-    return a_bits.get_val() + b_bits.get_val();
-
-  float af = fputil::cast<float>(a_bits.get_val());
-  float bf = fputil::cast<float>(b_bits.get_val());
+    return fputil::cast<float16>(af + bf);
 
   // These squares are exact.
   float a_sq = af * af;
diff --git a/libc/src/math/generic/ilogbbf16.cpp b/libc/src/math/generic/ilogbbf16.cpp
new file mode 100644
index 0000000000000..6811139c6dcfd
--- /dev/null
+++ b/libc/src/math/generic/ilogbbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of ilogbbf16 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ilogbbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, ilogbbf16, (bfloat16 x)) {
+  return fputil::intlogb<int>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/ldexpbf16.cpp b/libc/src/math/generic/ldexpbf16.cpp
new file mode 100644
index 0000000000000..42a5039e81bc0
--- /dev/null
+++ b/libc/src/math/generic/ldexpbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of ldexpbf16 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ldexpbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, ldexpbf16, (bfloat16 x, int exp)) {
+  return fputil::ldexp(x, exp);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/llogbbf16.cpp b/libc/src/math/generic/llogbbf16.cpp
new file mode 100644
index 0000000000000..74c27620053da
--- /dev/null
+++ b/libc/src/math/generic/llogbbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of llogbbf16 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/llogbbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(long, llogbbf16, (bfloat16 x)) {
+  return fputil::intlogb<long>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/llrintbf16.cpp b/libc/src/math/generic/llrintbf16.cpp
new file mode 100644
index 0000000000000..ec85454d788ff
--- /dev/null
+++ b/libc/src/math/generic/llrintbf16.cpp
@@ -0,0 +1,23 @@
+//===-- Implementation of llrintbf16 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/llrintbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(long long, llrintbf16, (bfloat16 x)) {
+  return fputil::round_to_signed_integer_using_current_rounding_mode<bfloat16,
+                                                                     long long>(
+      x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/llroundbf16.cpp b/libc/src/math/generic/llroundbf16.cpp
new file mode 100644
index 0000000000000..2497b6b4a7df3
--- /dev/null
+++ b/libc/src/math/generic/llroundbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of llroundbf16 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/llroundbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(long long, llroundbf16, (bfloat16 x)) {
+  return fputil::round_to_signed_integer<bfloat16, long long>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/log10f16.cpp b/libc/src/math/generic/log10f16.cpp
index 2626af4ddda15..4bb684a2206cc 100644
--- a/libc/src/math/generic/log10f16.cpp
+++ b/libc/src/math/generic/log10f16.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/log10f16.h"
-#include "expxf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -20,6 +19,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
 #include "src/__support/macros/properties/cpu_features.h"
+#include "src/__support/math/expxf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -75,6 +75,7 @@ static constexpr fputil::ExceptValues<float16, N_LOG10F16_EXCEPTS>
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, log10f16, (float16 x)) {
+  using namespace math::expxf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits x_bits(x);
 
diff --git a/libc/src/math/generic/log2f16.cpp b/libc/src/math/generic/log2f16.cpp
index 34be780388f2c..5b60323aabf35 100644
--- a/libc/src/math/generic/log2f16.cpp
+++ b/libc/src/math/generic/log2f16.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/log2f16.h"
-#include "expxf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -20,6 +19,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
 #include "src/__support/macros/properties/cpu_features.h"
+#include "src/__support/math/expxf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -61,6 +61,7 @@ static constexpr fputil::ExceptValues<float16, N_LOG2F16_EXCEPTS>
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, log2f16, (float16 x)) {
+  using namespace math::expxf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits x_bits(x);
 
diff --git a/libc/src/math/generic/logbbf16.cpp b/libc/src/math/generic/logbbf16.cpp
new file mode 100644
index 0000000000000..5a43ddfafde75
--- /dev/null
+++ b/libc/src/math/generic/logbbf16.cpp
@@ -0,0 +1,19 @@
+//===-- Implementation of logbbf16 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/logbbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, logbbf16, (bfloat16 x)) { return fputil::logb(x); }
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/logf16.cpp b/libc/src/math/generic/logf16.cpp
index 8e0d7d8f226cb..22e0dc8be34bf 100644
--- a/libc/src/math/generic/logf16.cpp
+++ b/libc/src/math/generic/logf16.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/logf16.h"
-#include "expxf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -20,6 +19,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
 #include "src/__support/macros/properties/cpu_features.h"
+#include "src/__support/math/expxf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -68,6 +68,7 @@ static constexpr fputil::ExceptValues<float16, N_LOGF16_EXCEPTS>
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, logf16, (float16 x)) {
+  using namespace math::expxf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits x_bits(x);
 
diff --git a/libc/src/math/generic/lrintbf16.cpp b/libc/src/math/generic/lrintbf16.cpp
new file mode 100644
index 0000000000000..4b378909413d3
--- /dev/null
+++ b/libc/src/math/generic/lrintbf16.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of lrintbf16 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/lrintbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(long, lrintbf16, (bfloat16 x)) {
+  return fputil::round_to_signed_integer_using_current_rounding_mode<bfloat16,
+                                                                     long>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/lroundbf16.cpp b/libc/src/math/generic/lroundbf16.cpp
new file mode 100644
index 0000000000000..89095d1786f63
--- /dev/null
+++ b/libc/src/math/generic/lroundbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of lroundbf16 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/lroundbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(long, lroundbf16, (bfloat16 x)) {
+  return fputil::round_to_signed_integer<bfloat16, long>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/nanbf16.cpp b/libc/src/math/generic/nanbf16.cpp
new file mode 100644
index 0000000000000..678dd6aed9ee3
--- /dev/null
+++ b/libc/src/math/generic/nanbf16.cpp
@@ -0,0 +1,25 @@
+//===-- Implementation of nanbf16 function --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nanbf16.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/str_to_float.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, nanbf16, (const char *arg)) {
+  auto result = internal::strtonan<bfloat16>(arg);
+  if (result.has_error())
+    libc_errno = result.error;
+  return result.value;
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/nearbyintbf16.cpp b/libc/src/math/generic/nearbyintbf16.cpp
new file mode 100644
index 0000000000000..a6391995963ff
--- /dev/null
+++ b/libc/src/math/generic/nearbyintbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of nearbyintbf16 function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nearbyintbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, nearbyintbf16, (bfloat16 x)) {
+  return fputil::round_using_current_rounding_mode(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/nextafterbf16.cpp b/libc/src/math/generic/nextafterbf16.cpp
new file mode 100644
index 0000000000000..e21a2dcb3d664
--- /dev/null
+++ b/libc/src/math/generic/nextafterbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of nextafterbf16 function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nextafterbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, nextafterbf16, (bfloat16 x, bfloat16 y)) {
+  return fputil::nextafter(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/nextdownbf16.cpp b/libc/src/math/generic/nextdownbf16.cpp
new file mode 100644
index 0000000000000..2115df95ff072
--- /dev/null
+++ b/libc/src/math/generic/nextdownbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of nextdownbf16 function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nextdownbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, nextdownbf16, (bfloat16 x)) {
+  return fputil::nextupdown</*IsDown=*/true>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/nexttowardbf16.cpp b/libc/src/math/generic/nexttowardbf16.cpp
new file mode 100644
index 0000000000000..3deab87575939
--- /dev/null
+++ b/libc/src/math/generic/nexttowardbf16.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of nexttowardbf16 function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nexttowardbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, nexttowardbf16, (bfloat16 x, long double y)) {
+  // nextafter<T, U> where T != U is nexttoward
+  return fputil::nextafter(x, y);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/nextupbf16.cpp b/libc/src/math/generic/nextupbf16.cpp
new file mode 100644
index 0000000000000..147ce37477d9f
--- /dev/null
+++ b/libc/src/math/generic/nextupbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of nextupbf16 function -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/nextupbf16.h"
+#include "src/__support/FPUtil/ManipulationFunctions.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, nextupbf16, (bfloat16 x)) {
+  return fputil::nextupdown</*IsDown=*/false>(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/rintbf16.cpp b/libc/src/math/generic/rintbf16.cpp
new file mode 100644
index 0000000000000..2ffe16a716205
--- /dev/null
+++ b/libc/src/math/generic/rintbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of rintbf16 function -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/rintbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, rintbf16, (bfloat16 x)) {
+  return fputil::round_using_current_rounding_mode(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/roundbf16.cpp b/libc/src/math/generic/roundbf16.cpp
new file mode 100644
index 0000000000000..cc7e5e2375d58
--- /dev/null
+++ b/libc/src/math/generic/roundbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of roundbf16 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/roundbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, roundbf16, (bfloat16 x)) {
+  return fputil::round(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/roundevenbf16.cpp b/libc/src/math/generic/roundevenbf16.cpp
new file mode 100644
index 0000000000000..39419e483fb9e
--- /dev/null
+++ b/libc/src/math/generic/roundevenbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of roundevenbf16 function --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/roundevenbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, roundevenbf16, (bfloat16 x)) {
+  return fputil::round_using_specific_rounding_mode(x, FP_INT_TONEAREST);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/setpayloadbf16.cpp b/libc/src/math/generic/setpayloadbf16.cpp
new file mode 100644
index 0000000000000..49f9b9cabd6a2
--- /dev/null
+++ b/libc/src/math/generic/setpayloadbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of setpayloadbf16 function -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/setpayloadbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, setpayloadbf16, (bfloat16 * res, bfloat16 pl)) {
+  return static_cast<int>(fputil::setpayload</*IsSignaling=*/false>(*res, pl));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/setpayloadsigbf16.cpp b/libc/src/math/generic/setpayloadsigbf16.cpp
new file mode 100644
index 0000000000000..7a2b7c7dc66dd
--- /dev/null
+++ b/libc/src/math/generic/setpayloadsigbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of setpayloadsigbf16 function ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/setpayloadsigbf16.h"
+#include "src/__support/FPUtil/BasicOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(int, setpayloadsigbf16, (bfloat16 * res, bfloat16 pl)) {
+  return static_cast<int>(fputil::setpayload</*IsSignaling=*/true>(*res, pl));
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/sin.cpp b/libc/src/math/generic/sin.cpp
index a614427bd7ee3..1b6310f99366c 100644
--- a/libc/src/math/generic/sin.cpp
+++ b/libc/src/math/generic/sin.cpp
@@ -18,13 +18,13 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-#include "src/math/generic/range_reduction_double_common.h"
-#include "src/math/generic/sincos_eval.h"
+#include "src/__support/math/range_reduction_double_common.h"
+#include "src/__support/math/sincos_eval.h"
 
 #ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
-#include "range_reduction_double_fma.h"
+#include "src/__support/math/range_reduction_double_fma.h"
 #else
-#include "range_reduction_double_nofma.h"
+#include "src/__support/math/range_reduction_double_nofma.h"
 #endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 namespace LIBC_NAMESPACE_DECL {
@@ -33,6 +33,7 @@ using DoubleDouble = fputil::DoubleDouble;
 using Float128 = typename fputil::DyadicFloat<128>;
 
 LLVM_LIBC_FUNCTION(double, sin, (double x)) {
+  using namespace math::range_reduction_double_internal;
   using FPBits = typename fputil::FPBits<double>;
   FPBits xbits(x);
 
@@ -95,7 +96,8 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) {
 
   DoubleDouble sin_y, cos_y;
 
-  [[maybe_unused]] double err = generic::sincos_eval(y, sin_y, cos_y);
+  [[maybe_unused]] double err =
+      math::sincos_eval_internal::sincos_eval(y, sin_y, cos_y);
 
   // Look up sin(k * pi/128) and cos(k * pi/128)
 #ifdef LIBC_MATH_HAS_SMALL_TABLES
@@ -149,7 +151,7 @@ LLVM_LIBC_FUNCTION(double, sin, (double x)) {
   else
     u_f128 = range_reduction_large.accurate();
 
-  generic::sincos_eval(u_f128, sin_u, cos_u);
+  math::sincos_eval_internal::sincos_eval(u_f128, sin_u, cos_u);
 
   auto get_sin_k = [](unsigned kk) -> Float128 {
     unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
diff --git a/libc/src/math/generic/sincos.cpp b/libc/src/math/generic/sincos.cpp
index 08c8a8298f029..38661deaa886a 100644
--- a/libc/src/math/generic/sincos.cpp
+++ b/libc/src/math/generic/sincos.cpp
@@ -19,13 +19,13 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-#include "src/math/generic/range_reduction_double_common.h"
-#include "src/math/generic/sincos_eval.h"
+#include "src/__support/math/range_reduction_double_common.h"
+#include "src/__support/math/sincos_eval.h"
 
 #ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
-#include "range_reduction_double_fma.h"
+#include "src/__support/math/range_reduction_double_fma.h"
 #else
-#include "range_reduction_double_nofma.h"
+#include "src/__support/math/range_reduction_double_nofma.h"
 #endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 namespace LIBC_NAMESPACE_DECL {
@@ -34,6 +34,7 @@ using DoubleDouble = fputil::DoubleDouble;
 using Float128 = typename fputil::DyadicFloat<128>;
 
 LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {
+  using namespace math::range_reduction_double_internal;
   using FPBits = typename fputil::FPBits<double>;
   FPBits xbits(x);
 
@@ -106,7 +107,8 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {
 
   DoubleDouble sin_y, cos_y;
 
-  [[maybe_unused]] double err = generic::sincos_eval(y, sin_y, cos_y);
+  [[maybe_unused]] double err =
+      math::sincos_eval_internal::sincos_eval(y, sin_y, cos_y);
 
   // Look up sin(k * pi/128) and cos(k * pi/128)
 #ifdef LIBC_MATH_HAS_SMALL_TABLES
@@ -179,7 +181,7 @@ LLVM_LIBC_FUNCTION(void, sincos, (double x, double *sin_x, double *cos_x)) {
   else
     u_f128 = range_reduction_large.accurate();
 
-  generic::sincos_eval(u_f128, sin_u, cos_u);
+  math::sincos_eval_internal::sincos_eval(u_f128, sin_u, cos_u);
 
   auto get_sin_k = [](unsigned kk) -> Float128 {
     unsigned idx = (kk & 64) ? 64 - (kk & 63) : (kk & 63);
diff --git a/libc/src/math/generic/sincosf.cpp b/libc/src/math/generic/sincosf.cpp
index 9c7bf181e485e..5179c985a2240 100644
--- a/libc/src/math/generic/sincosf.cpp
+++ b/libc/src/math/generic/sincosf.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/sincosf.h"
-#include "sincosf_utils.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/multiply_add.h"
@@ -16,6 +15,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+#include "src/__support/math/sincosf_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/math/generic/sinf.cpp b/libc/src/math/generic/sinf.cpp
index 38ea56f5f28c6..a8e634ce2f5ac 100644
--- a/libc/src/math/generic/sinf.cpp
+++ b/libc/src/math/generic/sinf.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/sinf.h"
-#include "sincosf_utils.h"
 #include "src/__support/FPUtil/BasicOperations.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
@@ -18,11 +17,12 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+#include "src/__support/math/sincosf_utils.h"
 
 #if defined(LIBC_TARGET_CPU_HAS_FMA_DOUBLE)
-#include "range_reduction_fma.h"
+#include "src/__support/math/range_reduction_fma.h"
 #else
-#include "range_reduction.h"
+#include "src/__support/math/range_reduction.h"
 #endif
 
 namespace LIBC_NAMESPACE_DECL {
diff --git a/libc/src/math/generic/sinf16.cpp b/libc/src/math/generic/sinf16.cpp
index 28debbd52a9a5..2b579202ea111 100644
--- a/libc/src/math/generic/sinf16.cpp
+++ b/libc/src/math/generic/sinf16.cpp
@@ -9,13 +9,13 @@
 #include "src/math/sinf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
-#include "sincosf16_utils.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/cast.h"
 #include "src/__support/FPUtil/except_value_utils.h"
 #include "src/__support/FPUtil/multiply_add.h"
 #include "src/__support/macros/optimization.h"
+#include "src/__support/math/sincosf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -32,6 +32,7 @@ constexpr fputil::ExceptValues<float16, N_EXCEPTS> SINF16_EXCEPTS{{
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, sinf16, (float16 x)) {
+  using namespace sincosf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits xbits(x);
 
diff --git a/libc/src/math/generic/sinhf.cpp b/libc/src/math/generic/sinhf.cpp
index 63111f84de141..5f2d0b5d9c71c 100644
--- a/libc/src/math/generic/sinhf.cpp
+++ b/libc/src/math/generic/sinhf.cpp
@@ -12,7 +12,7 @@
 #include "src/__support/FPUtil/rounding_mode.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
-#include "src/math/generic/explogxf.h"
+#include "src/__support/math/sinhfcoshf_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -73,7 +73,8 @@ LLVM_LIBC_FUNCTION(float, sinhf, (float x)) {
   }
 
   // sinh(x) = (e^x - e^(-x)) / 2.
-  return static_cast<float>(exp_pm_eval</*is_sinh*/ true>(x));
+  return static_cast<float>(
+      math::sinhfcoshf_internal::exp_pm_eval</*is_sinh*/ true>(x));
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/sinhf16.cpp b/libc/src/math/generic/sinhf16.cpp
index b426ea78b98c0..f6b5c9bea16b0 100644
--- a/libc/src/math/generic/sinhf16.cpp
+++ b/libc/src/math/generic/sinhf16.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/sinhf16.h"
-#include "expxf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -17,6 +16,7 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
+#include "src/__support/math/expxf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -89,6 +89,7 @@ static constexpr fputil::ExceptValues<float16, 13> SINHF16_EXCEPTS_NEG = {{
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, sinhf16, (float16 x)) {
+  using namespace math::expxf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits x_bits(x);
 
diff --git a/libc/src/math/generic/sinpif.cpp b/libc/src/math/generic/sinpif.cpp
index 492689d594d90..f3383f19db60f 100644
--- a/libc/src/math/generic/sinpif.cpp
+++ b/libc/src/math/generic/sinpif.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/sinpif.h"
-#include "sincosf_utils.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
@@ -15,6 +14,7 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+#include "src/__support/math/sincosf_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/math/generic/sinpif16.cpp b/libc/src/math/generic/sinpif16.cpp
index 68af484a6c5d3..311e6f989ebf1 100644
--- a/libc/src/math/generic/sinpif16.cpp
+++ b/libc/src/math/generic/sinpif16.cpp
@@ -9,15 +9,16 @@
 #include "src/math/sinpif16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
-#include "sincosf16_utils.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/cast.h"
 #include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/math/sincosf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(float16, sinpif16, (float16 x)) {
+  using namespace sincosf16_internal;
   using FPBits = typename fputil::FPBits<float16>;
   FPBits xbits(x);
 
diff --git a/libc/src/math/generic/tan.cpp b/libc/src/math/generic/tan.cpp
index 89b812cfc23a0..7ea40c9af75d2 100644
--- a/libc/src/math/generic/tan.cpp
+++ b/libc/src/math/generic/tan.cpp
@@ -20,12 +20,12 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-#include "src/math/generic/range_reduction_double_common.h"
+#include "src/__support/math/range_reduction_double_common.h"
 
 #ifdef LIBC_TARGET_CPU_HAS_FMA_DOUBLE
-#include "range_reduction_double_fma.h"
+#include "src/__support/math/range_reduction_double_fma.h"
 #else
-#include "range_reduction_double_nofma.h"
+#include "src/__support/math/range_reduction_double_nofma.h"
 #endif // LIBC_TARGET_CPU_HAS_FMA_DOUBLE
 
 namespace LIBC_NAMESPACE_DECL {
@@ -121,6 +121,7 @@ LIBC_INLINE double tan_eval(const DoubleDouble &u, DoubleDouble &result) {
 } // anonymous namespace
 
 LLVM_LIBC_FUNCTION(double, tan, (double x)) {
+  using namespace math::range_reduction_double_internal;
   using FPBits = typename fputil::FPBits<double>;
   FPBits xbits(x);
 
diff --git a/libc/src/math/generic/tanf.cpp b/libc/src/math/generic/tanf.cpp
index ca5e35dca4c91..a8c557b14808c 100644
--- a/libc/src/math/generic/tanf.cpp
+++ b/libc/src/math/generic/tanf.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/tanf.h"
-#include "sincosf_utils.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
@@ -18,6 +17,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
 #include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+#include "src/__support/math/sincosf_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/math/generic/tanf16.cpp b/libc/src/math/generic/tanf16.cpp
index 229f4a363670b..20323a88f3527 100644
--- a/libc/src/math/generic/tanf16.cpp
+++ b/libc/src/math/generic/tanf16.cpp
@@ -9,13 +9,13 @@
 #include "src/math/tanf16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
-#include "sincosf16_utils.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/cast.h"
 #include "src/__support/FPUtil/except_value_utils.h"
 #include "src/__support/FPUtil/multiply_add.h"
 #include "src/__support/macros/optimization.h"
+#include "src/__support/math/sincosf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -37,6 +37,7 @@ constexpr fputil::ExceptValues<float16, N_EXCEPTS> TANF16_EXCEPTS{{
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, tanf16, (float16 x)) {
+  using namespace sincosf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits xbits(x);
 
diff --git a/libc/src/math/generic/tanhf.cpp b/libc/src/math/generic/tanhf.cpp
index 32153c3e1c943..0c55047da4826 100644
--- a/libc/src/math/generic/tanhf.cpp
+++ b/libc/src/math/generic/tanhf.cpp
@@ -14,7 +14,7 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
 #include "src/__support/macros/properties/cpu_features.h"
-#include "src/math/generic/explogxf.h"
+#include "src/__support/math/exp10f_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/math/generic/tanhf16.cpp b/libc/src/math/generic/tanhf16.cpp
index 4c43cfd29153f..fc0e28b95a834 100644
--- a/libc/src/math/generic/tanhf16.cpp
+++ b/libc/src/math/generic/tanhf16.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/tanhf16.h"
-#include "expxf16.h"
 #include "hdr/fenv_macros.h"
 #include "src/__support/CPP/array.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -21,6 +20,7 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h"
+#include "src/__support/math/expxf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -34,6 +34,7 @@ static constexpr fputil::ExceptValues<float16, 2> TANHF16_EXCEPTS = {{
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, tanhf16, (float16 x)) {
+  using namespace math::expxf16_internal;
   using FPBits = fputil::FPBits<float16>;
   FPBits x_bits(x);
 
diff --git a/libc/src/math/generic/tanpif.cpp b/libc/src/math/generic/tanpif.cpp
index 58d46c9481aa5..b49f3cea4911b 100644
--- a/libc/src/math/generic/tanpif.cpp
+++ b/libc/src/math/generic/tanpif.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/math/tanpif.h"
-#include "sincosf_utils.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/cast.h"
@@ -16,6 +15,7 @@
 #include "src/__support/common.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+#include "src/__support/math/sincosf_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
diff --git a/libc/src/math/generic/tanpif16.cpp b/libc/src/math/generic/tanpif16.cpp
index 792d405b1bb9e..b137b09860f7c 100644
--- a/libc/src/math/generic/tanpif16.cpp
+++ b/libc/src/math/generic/tanpif16.cpp
@@ -9,13 +9,13 @@
 #include "src/math/tanpif16.h"
 #include "hdr/errno_macros.h"
 #include "hdr/fenv_macros.h"
-#include "sincosf16_utils.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/cast.h"
 #include "src/__support/FPUtil/except_value_utils.h"
 #include "src/__support/FPUtil/multiply_add.h"
 #include "src/__support/macros/optimization.h"
+#include "src/__support/math/sincosf16_utils.h"
 
 namespace LIBC_NAMESPACE_DECL {
 
@@ -39,6 +39,7 @@ constexpr fputil::ExceptValues<float16, N_EXCEPTS> TANPIF16_EXCEPTS{{
 #endif // !LIBC_MATH_HAS_SKIP_ACCURATE_PASS
 
 LLVM_LIBC_FUNCTION(float16, tanpif16, (float16 x)) {
+  using namespace sincosf16_internal;
   using FPBits = typename fputil::FPBits<float16>;
   FPBits xbits(x);
 
diff --git a/libc/src/math/generic/truncbf16.cpp b/libc/src/math/generic/truncbf16.cpp
new file mode 100644
index 0000000000000..dfbe83d003377
--- /dev/null
+++ b/libc/src/math/generic/truncbf16.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of truncbf16 function ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/truncbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, truncbf16, (bfloat16 x)) {
+  return fputil::trunc(x);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/ufromfpbf16.cpp b/libc/src/math/generic/ufromfpbf16.cpp
new file mode 100644
index 0000000000000..336771b695d97
--- /dev/null
+++ b/libc/src/math/generic/ufromfpbf16.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of ufromfpbf16 function ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfpbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, ufromfpbf16,
+                   (bfloat16 x, int rnd, unsigned int width)) {
+  return fputil::fromfp</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/generic/ufromfpxbf16.cpp b/libc/src/math/generic/ufromfpxbf16.cpp
new file mode 100644
index 0000000000000..ac9cf44470a8e
--- /dev/null
+++ b/libc/src/math/generic/ufromfpxbf16.cpp
@@ -0,0 +1,22 @@
+//===-- Implementation of ufromfpxbf16 function ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/ufromfpxbf16.h"
+#include "src/__support/FPUtil/NearestIntegerOperations.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/__support/common.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(bfloat16, ufromfpxbf16,
+                   (bfloat16 x, int rnd, unsigned int width)) {
+  return fputil::fromfpx</*IsSigned=*/false>(x, rnd, width);
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/math/getpayloadbf16.h b/libc/src/math/getpayloadbf16.h
new file mode 100644
index 0000000000000..e4767f0728139
--- /dev/null
+++ b/libc/src/math/getpayloadbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for getpayloadbf16 ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_GETPAYLOADBF16_H
+#define LLVM_LIBC_SRC_MATH_GETPAYLOADBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 getpayloadbf16(const bfloat16 *x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_GETPAYLOADBF16_H
diff --git a/libc/src/math/ilogbbf16.h b/libc/src/math/ilogbbf16.h
new file mode 100644
index 0000000000000..da2384b6d62f6
--- /dev/null
+++ b/libc/src/math/ilogbbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for ilogbbf16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ILOGBBF16_H
+#define LLVM_LIBC_SRC_MATH_ILOGBBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int ilogbbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ILOGBBF16_H
diff --git a/libc/src/math/ldexpbf16.h b/libc/src/math/ldexpbf16.h
new file mode 100644
index 0000000000000..7436d8dc5c149
--- /dev/null
+++ b/libc/src/math/ldexpbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for ldexpbf16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LDEXPBF16_H
+#define LLVM_LIBC_SRC_MATH_LDEXPBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 ldexpbf16(bfloat16 x, int exp);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_LDEXPBF16_H
diff --git a/libc/src/math/llogbbf16.h b/libc/src/math/llogbbf16.h
new file mode 100644
index 0000000000000..13f0570585d82
--- /dev/null
+++ b/libc/src/math/llogbbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for llogbbf16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLOGBBF16_H
+#define LLVM_LIBC_SRC_MATH_LLOGBBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+long llogbbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_LLOGBBF16_H
diff --git a/libc/src/math/llrintbf16.h b/libc/src/math/llrintbf16.h
new file mode 100644
index 0000000000000..23402a57806a9
--- /dev/null
+++ b/libc/src/math/llrintbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for llrintbf16 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLRINTBF16_H
+#define LLVM_LIBC_SRC_MATH_LLRINTBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+long long llrintbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_LLRINTBF16_H
diff --git a/libc/src/math/llroundbf16.h b/libc/src/math/llroundbf16.h
new file mode 100644
index 0000000000000..69878e43c4460
--- /dev/null
+++ b/libc/src/math/llroundbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for llroundbf16 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LLROUNDBF16_H
+#define LLVM_LIBC_SRC_MATH_LLROUNDBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+long long llroundbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_LLROUNDBF16_H
diff --git a/libc/src/math/logbbf16.h b/libc/src/math/logbbf16.h
new file mode 100644
index 0000000000000..2c0d77e9df8af
--- /dev/null
+++ b/libc/src/math/logbbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for logbbf16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LOGBBF16_H
+#define LLVM_LIBC_SRC_MATH_LOGBBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 logbbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_LOGBBF16_H
diff --git a/libc/src/math/lrintbf16.h b/libc/src/math/lrintbf16.h
new file mode 100644
index 0000000000000..ec244728053a3
--- /dev/null
+++ b/libc/src/math/lrintbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for lrintbf16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LRINTBF16_H
+#define LLVM_LIBC_SRC_MATH_LRINTBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+long lrintbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_LRINTBF16_H
diff --git a/libc/src/math/lroundbf16.h b/libc/src/math/lroundbf16.h
new file mode 100644
index 0000000000000..c08db7a645b3e
--- /dev/null
+++ b/libc/src/math/lroundbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for lroundbf16 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_LROUNDBF16_H
+#define LLVM_LIBC_SRC_MATH_LROUNDBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+long lroundbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_LROUNDBF16_H
diff --git a/libc/src/math/nanbf16.h b/libc/src/math/nanbf16.h
new file mode 100644
index 0000000000000..1551044677768
--- /dev/null
+++ b/libc/src/math/nanbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for nanbf16 -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NANBF16_H
+#define LLVM_LIBC_SRC_MATH_NANBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 nanbf16(const char *arg);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_NANBF16_H
diff --git a/libc/src/math/nearbyintbf16.h b/libc/src/math/nearbyintbf16.h
new file mode 100644
index 0000000000000..bc7eb3b6a315d
--- /dev/null
+++ b/libc/src/math/nearbyintbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for nearbyintbf16 -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NEARBYINTBF16_H
+#define LLVM_LIBC_SRC_MATH_NEARBYINTBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 nearbyintbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_NEARBYINTBF16_H
diff --git a/libc/src/math/nextafterbf16.h b/libc/src/math/nextafterbf16.h
new file mode 100644
index 0000000000000..f962c7c51e5cd
--- /dev/null
+++ b/libc/src/math/nextafterbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for nextafterbf16 -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NEXTAFTERBF16_H
+#define LLVM_LIBC_SRC_MATH_NEXTAFTERBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 nextafterbf16(bfloat16 x, bfloat16 y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_NEXTAFTERBF16_H
diff --git a/libc/src/math/nextdownbf16.h b/libc/src/math/nextdownbf16.h
new file mode 100644
index 0000000000000..36b0cd7cb9626
--- /dev/null
+++ b/libc/src/math/nextdownbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for nextdownbf16 ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NEXTDOWNBF16_H
+#define LLVM_LIBC_SRC_MATH_NEXTDOWNBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 nextdownbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_NEXTDOWNBF16_H
diff --git a/libc/src/math/nexttowardbf16.h b/libc/src/math/nexttowardbf16.h
new file mode 100644
index 0000000000000..930abf858ef51
--- /dev/null
+++ b/libc/src/math/nexttowardbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for nexttowardbf16 ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NEXTTOWARDBF16_H
+#define LLVM_LIBC_SRC_MATH_NEXTTOWARDBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 nexttowardbf16(bfloat16 x, long double y);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_NEXTTOWARDBF16_H
diff --git a/libc/src/math/nextupbf16.h b/libc/src/math/nextupbf16.h
new file mode 100644
index 0000000000000..872de84338d95
--- /dev/null
+++ b/libc/src/math/nextupbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for nextupbf16 --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_NEXTUPBF16_H
+#define LLVM_LIBC_SRC_MATH_NEXTUPBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 nextupbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_NEXTUPBF16_H
diff --git a/libc/src/math/rintbf16.h b/libc/src/math/rintbf16.h
new file mode 100644
index 0000000000000..aae1541b583fa
--- /dev/null
+++ b/libc/src/math/rintbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for rintbf16 ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_RINTBF16_H
+#define LLVM_LIBC_SRC_MATH_RINTBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 rintbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_RINTBF16_H
diff --git a/libc/src/math/roundbf16.h b/libc/src/math/roundbf16.h
new file mode 100644
index 0000000000000..0f74e43ba27be
--- /dev/null
+++ b/libc/src/math/roundbf16.h
@@ -0,0 +1,22 @@
+//===-- Implementation header for roundbf16 ---------------------*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ROUNDBF16_H
+#define LLVM_LIBC_SRC_MATH_ROUNDBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 roundbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ROUNDBF16_H
diff --git a/libc/src/math/roundevenbf16.h b/libc/src/math/roundevenbf16.h
new file mode 100644
index 0000000000000..f4374d2c24e5f
--- /dev/null
+++ b/libc/src/math/roundevenbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for roundevenbf16 -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_ROUNDEVENBF16_H
+#define LLVM_LIBC_SRC_MATH_ROUNDEVENBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 roundevenbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_ROUNDEVENBF16_H
diff --git a/libc/src/math/setpayloadbf16.h b/libc/src/math/setpayloadbf16.h
new file mode 100644
index 0000000000000..e3a60b2f2e021
--- /dev/null
+++ b/libc/src/math/setpayloadbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for setpayloadbf16 ----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_SETPAYLOADBF16_H
+#define LLVM_LIBC_SRC_MATH_SETPAYLOADBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int setpayloadbf16(bfloat16 *res, bfloat16 pl);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_SETPAYLOADBF16_H
diff --git a/libc/src/math/setpayloadsigbf16.h b/libc/src/math/setpayloadsigbf16.h
new file mode 100644
index 0000000000000..5baba95b45758
--- /dev/null
+++ b/libc/src/math/setpayloadsigbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for setpayloadsigbf16 -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_SETPAYLOADSIGBF16_H
+#define LLVM_LIBC_SRC_MATH_SETPAYLOADSIGBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+int setpayloadsigbf16(bfloat16 *res, bfloat16 pl);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_SETPAYLOADSIGBF16_H
diff --git a/libc/src/math/truncbf16.h b/libc/src/math/truncbf16.h
new file mode 100644
index 0000000000000..c87d4cc2b2c4d
--- /dev/null
+++ b/libc/src/math/truncbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for truncbf16 ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_TRUNCBF16_H
+#define LLVM_LIBC_SRC_MATH_TRUNCBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 truncbf16(bfloat16 x);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_TRUNCBF16_H
diff --git a/libc/src/math/ufromfpbf16.h b/libc/src/math/ufromfpbf16.h
new file mode 100644
index 0000000000000..1fd876a15e600
--- /dev/null
+++ b/libc/src/math/ufromfpbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for ufromfpbf16 -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFPBF16_H
+#define LLVM_LIBC_SRC_MATH_UFROMFPBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 ufromfpbf16(bfloat16 x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFPBF16_H
diff --git a/libc/src/math/ufromfpxbf16.h b/libc/src/math/ufromfpxbf16.h
new file mode 100644
index 0000000000000..ec63744c8045b
--- /dev/null
+++ b/libc/src/math/ufromfpxbf16.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for ufromfpxbf16 ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_MATH_UFROMFPXBF16_H
+#define LLVM_LIBC_SRC_MATH_UFROMFPXBF16_H
+
+#include "src/__support/macros/config.h"
+#include "src/__support/macros/properties/types.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+bfloat16 ufromfpxbf16(bfloat16 x, int rnd, unsigned int width);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_MATH_UFROMFPXBF16_H
diff --git a/libc/src/string/memory_utils/aarch64/inline_strlen.h b/libc/src/string/memory_utils/aarch64/inline_strlen.h
new file mode 100644
index 0000000000000..36fd1aa636b54
--- /dev/null
+++ b/libc/src/string/memory_utils/aarch64/inline_strlen.h
@@ -0,0 +1,53 @@
+//===-- Strlen implementation for aarch64 ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H
+
+#if defined(__ARM_NEON)
+#include "src/__support/CPP/bit.h" // countr_zero
+
+#include <arm_neon.h>
+#include <stddef.h> // size_t
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace neon {
+[[gnu::no_sanitize_address]] [[maybe_unused]] LIBC_INLINE static size_t
+string_length(const char *src) {
+  using Vector __attribute__((may_alias)) = uint8x8_t;
+
+  uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
+  const Vector *block_ptr =
+      reinterpret_cast<const Vector *>(src - misalign_bytes);
+  Vector v = *block_ptr;
+  Vector vcmp = vceqz_u8(v);
+  uint64x1_t cmp_mask = vreinterpret_u64_u8(vcmp);
+  uint64_t cmp = vget_lane_u64(cmp_mask, 0);
+  cmp = cmp >> (misalign_bytes << 3);
+  if (cmp)
+    return cpp::countr_zero(cmp) >> 3;
+
+  while (true) {
+    ++block_ptr;
+    v = *block_ptr;
+    vcmp = vceqz_u8(v);
+    cmp_mask = vreinterpret_u64_u8(vcmp);
+    cmp = vget_lane_u64(cmp_mask, 0);
+    if (cmp)
+      return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
+                                 reinterpret_cast<uintptr_t>(src) +
+                                 (cpp::countr_zero(cmp) >> 3));
+  }
+}
+} // namespace neon
+
+namespace string_length_impl = neon;
+
+} // namespace LIBC_NAMESPACE_DECL
+#endif // __ARM_NEON
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_AARCH64_INLINE_STRLEN_H
diff --git a/libc/src/string/memory_utils/x86_64/inline_strlen.h b/libc/src/string/memory_utils/x86_64/inline_strlen.h
new file mode 100644
index 0000000000000..739f8c1aaddbc
--- /dev/null
+++ b/libc/src/string/memory_utils/x86_64/inline_strlen.h
@@ -0,0 +1,107 @@
+//===-- Strlen implementation for x86_64 ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H
+#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H
+
+#include "src/__support/CPP/bit.h" // countr_zero
+
+#include <immintrin.h>
+#include <stddef.h> // size_t
+
+namespace LIBC_NAMESPACE_DECL {
+
+namespace string_length_internal {
+// Return a bit-mask with the nth bit set if the nth-byte in block_ptr is zero.
+template <typename Vector, typename Mask>
+[[gnu::no_sanitize_address]] LIBC_INLINE static Mask
+compare_and_mask(const Vector *block_ptr);
+
+template <typename Vector, typename Mask,
+          decltype(compare_and_mask<Vector, Mask>)>
+[[gnu::no_sanitize_address]] LIBC_INLINE static size_t
+string_length_vector(const char *src) {
+  uintptr_t misalign_bytes = reinterpret_cast<uintptr_t>(src) % sizeof(Vector);
+
+  const Vector *block_ptr =
+      reinterpret_cast<const Vector *>(src - misalign_bytes);
+  auto cmp = compare_and_mask<Vector, Mask>(block_ptr) >> misalign_bytes;
+  if (cmp)
+    return cpp::countr_zero(cmp);
+
+  while (true) {
+    block_ptr++;
+    cmp = compare_and_mask<Vector, Mask>(block_ptr);
+    if (cmp)
+      return static_cast<size_t>(reinterpret_cast<uintptr_t>(block_ptr) -
+                                 reinterpret_cast<uintptr_t>(src) +
+                                 cpp::countr_zero(cmp));
+  }
+}
+
+template <>
+LIBC_INLINE uint32_t
+compare_and_mask<__m128i, uint32_t>(const __m128i *block_ptr) {
+  __m128i v = _mm_load_si128(block_ptr);
+  __m128i z = _mm_setzero_si128();
+  __m128i c = _mm_cmpeq_epi8(z, v);
+  return _mm_movemask_epi8(c);
+}
+
+namespace sse2 {
+[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {
+  return string_length_vector<__m128i, uint32_t,
+                              compare_and_mask<__m128i, uint32_t>>(src);
+}
+} // namespace sse2
+
+#if defined(__AVX2__)
+template <>
+LIBC_INLINE uint32_t
+compare_and_mask<__m256i, uint32_t>(const __m256i *block_ptr) {
+  __m256i v = _mm256_load_si256(block_ptr);
+  __m256i z = _mm256_setzero_si256();
+  __m256i c = _mm256_cmpeq_epi8(z, v);
+  return _mm256_movemask_epi8(c);
+}
+
+namespace avx2 {
+[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {
+  return string_length_vector<__m256i, uint32_t,
+                              compare_and_mask<__m256i, uint32_t>>(src);
+}
+} // namespace avx2
+#endif
+
+#if defined(__AVX512F__)
+template <>
+LIBC_INLINE __mmask64
+compare_and_mask<__m512i, __mmask64>(const __m512i *block_ptr) {
+  __m512i v = _mm512_load_si512(block_ptr);
+  __m512i z = _mm512_setzero_si512();
+  return _mm512_cmp_epu8_mask(z, v, _MM_CMPINT_EQ);
+}
+namespace avx512 {
+[[maybe_unused]] LIBC_INLINE size_t string_length(const char *src) {
+  return string_length_vector<__m512i, __mmask64,
+                              compare_and_mask<__m512i, __mmask64>>(src);
+}
+} // namespace avx512
+#endif
+} // namespace string_length_internal
+
+#if defined(__AVX512F__)
+namespace string_length_impl = string_length_internal::avx512;
+#elif defined(__AVX2__)
+namespace string_length_impl = string_length_internal::avx2;
+#else
+namespace string_length_impl = string_length_internal::sse2;
+#endif
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_X86_64_INLINE_STRLEN_H
diff --git a/libc/src/string/string_utils.h b/libc/src/string/string_utils.h
index 80e5783c7890b..ce461581b9d95 100644
--- a/libc/src/string/string_utils.h
+++ b/libc/src/string/string_utils.h
@@ -22,6 +22,16 @@
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
 
+#if defined(LIBC_COPT_STRING_UNSAFE_WIDE_READ)
+#if defined(LIBC_TARGET_ARCH_IS_X86)
+#include "src/string/memory_utils/x86_64/inline_strlen.h"
+#elif defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_NEON)
+#include "src/string/memory_utils/aarch64/inline_strlen.h"
+#else
+namespace string_length_impl = LIBC_NAMESPACE::wide_read;
+#endif
+#endif
+
 namespace LIBC_NAMESPACE_DECL {
 namespace internal {
 
@@ -53,7 +63,7 @@ template <typename Word> LIBC_INLINE constexpr Word repeat_byte(Word byte) {
 // high bit set will no longer have it set, narrowing the list of bytes which
 // result in non-zero values to just the zero byte.
 template <typename Word> LIBC_INLINE constexpr bool has_zeroes(Word block) {
-  constexpr Word LOW_BITS = repeat_byte<Word>(0x01);
+  constexpr unsigned int LOW_BITS = repeat_byte<Word>(0x01);
   constexpr Word HIGH_BITS = repeat_byte<Word>(0x80);
   Word subtracted = block - LOW_BITS;
   Word inverted = ~block;
@@ -81,16 +91,23 @@ LIBC_INLINE size_t string_length_wide_read(const char *src) {
   return static_cast<size_t>(char_ptr - src);
 }
 
-// Returns the length of a string, denoted by the first occurrence
-// of a null terminator.
-template <typename T> LIBC_INLINE size_t string_length(const T *src) {
-#ifdef LIBC_COPT_STRING_UNSAFE_WIDE_READ
+namespace wide_read {
+LIBC_INLINE size_t string_length(const char *src) {
   // Unsigned int is the default size for most processors, and on x86-64 it
   // performs better than larger sizes when the src pointer can't be assumed to
   // be aligned to a word boundary, so it's the size we use for reading the
   // string a block at a time.
+  return string_length_wide_read<unsigned int>(src);
+}
+
+} // namespace wide_read
+
+// Returns the length of a string, denoted by the first occurrence
+// of a null terminator.
+template <typename T> LIBC_INLINE size_t string_length(const T *src) {
+#ifdef LIBC_COPT_STRING_UNSAFE_WIDE_READ
   if constexpr (cpp::is_same_v<T, char>)
-    return string_length_wide_read<unsigned int>(src);
+    return string_length_impl::string_length(src);
 #endif
   size_t length;
   for (length = 0; *src; ++src, ++length)
@@ -99,8 +116,9 @@ template <typename T> LIBC_INLINE size_t string_length(const T *src) {
 }
 
 template <typename Word>
-LIBC_INLINE void *find_first_character_wide_read(const unsigned char *src,
-                                                 unsigned char ch, size_t n) {
+[[gnu::no_sanitize_address]] LIBC_INLINE void *
+find_first_character_wide_read(const unsigned char *src, unsigned char ch,
+                               size_t n) {
   const unsigned char *char_ptr = src;
   size_t cur = 0;
 
@@ -184,33 +202,36 @@ LIBC_INLINE size_t complementary_span(const char *src, const char *segment) {
 template <bool SkipDelim = true>
 LIBC_INLINE char *string_token(char *__restrict src,
                                const char *__restrict delimiter_string,
-                               char **__restrict saveptr) {
-  // Return nullptr immediately if both src AND saveptr are nullptr
-  if (LIBC_UNLIKELY(src == nullptr && ((src = *saveptr) == nullptr)))
+                               char **__restrict context) {
+  // Return nullptr immediately if both src AND context are nullptr
+  if (LIBC_UNLIKELY(src == nullptr && ((src = *context) == nullptr)))
     return nullptr;
 
   static_assert(CHAR_BIT == 8, "bitset of 256 assumes char is 8 bits");
-  cpp::bitset<256> delimiter_set;
+  cpp::bitset<256> delims;
   for (; *delimiter_string != '\0'; ++delimiter_string)
-    delimiter_set.set(static_cast<size_t>(*delimiter_string));
+    delims.set(static_cast<size_t>(*delimiter_string));
 
+  char *tok_start = src;
   if constexpr (SkipDelim)
-    for (; *src != '\0' && delimiter_set.test(static_cast<size_t>(*src)); ++src)
-      ;
-  if (*src == '\0') {
-    *saveptr = src;
+    while (*tok_start != '\0' && delims.test(static_cast<size_t>(*tok_start)))
+      ++tok_start;
+  if (*tok_start == '\0' && SkipDelim) {
+    *context = nullptr;
     return nullptr;
   }
-  char *token = src;
-  for (; *src != '\0'; ++src) {
-    if (delimiter_set.test(static_cast<size_t>(*src))) {
-      *src = '\0';
-      ++src;
-      break;
-    }
+
+  char *tok_end = tok_start;
+  while (*tok_end != '\0' && !delims.test(static_cast<size_t>(*tok_end)))
+    ++tok_end;
+
+  if (*tok_end == '\0') {
+    *context = nullptr;
+  } else {
+    *tok_end = '\0';
+    *context = tok_end + 1;
   }
-  *saveptr = src;
-  return token;
+  return tok_start;
 }
 
 LIBC_INLINE size_t strlcpy(char *__restrict dst, const char *__restrict src,
diff --git a/libc/src/wchar/wcsncmp.cpp b/libc/src/wchar/wcsncmp.cpp
index f2e052b3c9fe3..baca12bffb188 100644
--- a/libc/src/wchar/wcsncmp.cpp
+++ b/libc/src/wchar/wcsncmp.cpp
@@ -30,8 +30,7 @@ LLVM_LIBC_FUNCTION(int, wcsncmp,
     if (!comp(lc, '\0') || comp(lc, *right))
       break;
   }
-  return comp(*reinterpret_cast<const wchar_t *>(left),
-              *reinterpret_cast<const wchar_t *>(right));
+  return comp(*left, *right);
 }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wctype/iswalpha.cpp b/libc/src/wctype/iswalpha.cpp
index e18f29370fbd0..09f55d391dbff 100644
--- a/libc/src/wctype/iswalpha.cpp
+++ b/libc/src/wctype/iswalpha.cpp
@@ -14,6 +14,6 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-LLVM_LIBC_FUNCTION(bool, iswalpha, (wint_t c)) { return internal::iswalpha(c); }
+LLVM_LIBC_FUNCTION(int, iswalpha, (wint_t c)) { return internal::iswalpha(c); }
 
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wctype/iswalpha.h b/libc/src/wctype/iswalpha.h
index 681fc6ba79a54..0353388607b60 100644
--- a/libc/src/wctype/iswalpha.h
+++ b/libc/src/wctype/iswalpha.h
@@ -14,7 +14,7 @@
 
 namespace LIBC_NAMESPACE_DECL {
 
-bool iswalpha(wint_t c);
+int iswalpha(wint_t c);
 
 } // namespace LIBC_NAMESPACE_DECL
 
diff --git a/libc/startup/baremetal/aarch64/CMakeLists.txt b/libc/startup/baremetal/aarch64/CMakeLists.txt
new file mode 100644
index 0000000000000..f75bd893c68bc
--- /dev/null
+++ b/libc/startup/baremetal/aarch64/CMakeLists.txt
@@ -0,0 +1,16 @@
+add_startup_object(
+  crt1
+  SRC
+    start.cpp
+  DEPENDS
+    libc.src.stdlib.atexit
+    libc.src.stdlib.exit
+    libc.src.string.memcpy
+    libc.src.string.memset
+    libc.startup.baremetal.init
+    libc.startup.baremetal.fini
+  COMPILE_OPTIONS
+    -ffreestanding # To avoid compiler warnings about calling the main function.
+    -fno-builtin
+    -Wno-global-constructors # To allow vector table initialization
+)
diff --git a/libc/startup/baremetal/aarch64/start.cpp b/libc/startup/baremetal/aarch64/start.cpp
new file mode 100644
index 0000000000000..ff3ea933c5240
--- /dev/null
+++ b/libc/startup/baremetal/aarch64/start.cpp
@@ -0,0 +1,112 @@
+//===-- Implementation of crt for aarch64 ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/stdint_proxy.h"
+#include "src/__support/macros/config.h"
+#include "src/stdlib/atexit.h"
+#include "src/stdlib/exit.h"
+#include "src/string/memcpy.h"
+#include "src/string/memset.h"
+#include "startup/baremetal/fini.h"
+#include "startup/baremetal/init.h"
+
+#include <arm_acle.h>
+
+extern "C" {
+int main(int argc, char **argv);
+void _start();
+
+// Semihosting library initialisation if applicable. Required for printf, etc.
+[[gnu::weak]] void _platform_init() {}
+
+// These symbols are provided by the linker. The exact names are not defined by
+// a standard.
+extern uintptr_t __stack;
+extern uintptr_t __data_source[];
+extern uintptr_t __data_start[];
+extern uintptr_t __data_size[];
+extern uintptr_t __bss_start[];
+extern uintptr_t __bss_size[];
+} // extern "C"
+
+namespace {
+// The Arm ARM for the A-profile architecture (D14.1.5) defines the exceptions.
+// However, for simplicity, we don't bother logging, and just exit.
+void GenericException_Handler() { LIBC_NAMESPACE::exit(1); }
+
+// The AArch64 exception vector table has 16 entries, each of which is 128
+// bytes long, and contains code. The whole table must be 2048-byte aligned.
+// For our purposes, each entry just contains one branch instruction to the
+// exception reporting function, since we never want to resume after an
+// exception.
+[[gnu::section(".vectors"), gnu::aligned(2048), gnu::used, gnu::naked]]
+void vector_table() {
+#define VECTOR_TABLE_ENTRY                                                     \
+  asm(".balign 128");                                                          \
+  asm("B %0" : : "X"(GenericException_Handler));
+
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+  VECTOR_TABLE_ENTRY;
+}
+} // namespace
+
+namespace LIBC_NAMESPACE_DECL {
+
+[[noreturn]] void do_start() {
+  // TODO: This startup code is not extensive, but rather the MVP for QEMU
+  // testing.
+  // TODO: Setup memory (MMU, page table, caches)
+  // TODO: Consider v8-R variants
+
+  // Set up exception handling
+  __arm_wsr64("VBAR_EL1", reinterpret_cast<uint64_t>(&vector_table));
+
+#ifdef __ARM_FP
+  // Do not trap FP/SME/SVE instructions
+  static constexpr uint64_t CPACR_SHIFT_FPEN = 20;
+  static constexpr uint64_t CPACR_SHIFT_SMEN = 24;
+  uint64_t cpacr = __arm_rsr64("CPACR_EL1");
+  cpacr |= (0x3 << CPACR_SHIFT_FPEN);
+  cpacr |= (0x3 << CPACR_SHIFT_SMEN);
+  __arm_wsr64("CPACR_EL1", cpacr);
+#endif
+
+  // Perform the equivalent of scatterloading
+  LIBC_NAMESPACE::memcpy(__data_start, __data_source,
+                         reinterpret_cast<uintptr_t>(__data_size));
+  LIBC_NAMESPACE::memset(__bss_start, '\0',
+                         reinterpret_cast<uintptr_t>(__bss_size));
+  __libc_init_array();
+
+  _platform_init();
+  LIBC_NAMESPACE::atexit(&__libc_fini_array);
+  LIBC_NAMESPACE::exit(main(0, 0));
+}
+} // namespace LIBC_NAMESPACE_DECL
+
+extern "C" {
+[[gnu::section(".text.init.enter"), gnu::naked]]
+void _start() {
+  asm volatile("mov sp, %0" : : "r"(&__stack));
+  asm volatile("bl %0" : : "X"(LIBC_NAMESPACE::do_start));
+}
+} // extern "C"
diff --git a/libc/startup/baremetal/arm/start.cpp b/libc/startup/baremetal/arm/start.cpp
index 123efc4750941..c089a14f5f782 100644
--- a/libc/startup/baremetal/arm/start.cpp
+++ b/libc/startup/baremetal/arm/start.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/stdint_proxy.h"
 #include "src/__support/macros/config.h"
 #include "src/stdlib/atexit.h"
 #include "src/stdlib/exit.h"
@@ -14,7 +15,7 @@
 #include "startup/baremetal/fini.h"
 #include "startup/baremetal/init.h"
 
-#include <stdint.h>
+#include <arm_acle.h> // For __arm_wsr
 
 extern "C" {
 int main(int argc, char **argv);
@@ -31,7 +32,10 @@ extern uintptr_t __data_start[];
 extern uintptr_t __data_size[];
 extern uintptr_t __bss_start[];
 extern uintptr_t __bss_size[];
+} // extern "C"
 
+namespace {
+#if __ARM_ARCH_PROFILE == 'M'
 // Based on
 // https://developer.arm.com/documentation/107565/0101/Use-case-examples/Generic-Information/What-is-inside-a-program-image-/Vector-table
 void NMI_Handler() {}
@@ -49,35 +53,89 @@ void SysTick_Handler() {}
 // to be zero and Cortex-M23 can require up to 10, so 1024-byte align the vector
 // table.
 using HandlerType = void (*)(void);
-const HandlerType vector_table[]
-    __attribute__((section(".vectors"), aligned(1024), used)) = {
-        (HandlerType)&__stack, // SP
-        _start,                // Reset
-        NMI_Handler,           // NMI Handler
-        HardFault_Handler,     // Hard Fault Handlerß
-        MemManage_Handler,     // MPU Fault Han`dler
-        BusFault_Handler,      // Bus Fault Handler
-        UsageFault_Handler,    // Usage Fault Handler
-        0,                     // Reserved
-        0,                     // Reserved
-        0,                     // Reserved
-        0,                     // Reserved
-        SVC_Handler,           // SVC Handler
-        DebugMon_Handler,      // Debug Monitor Handler
-        0,                     // Reserved
-        PendSV_Handler,        // PendSV Handler
-        SysTick_Handler,       // SysTick Handler
-                               // Unused
+[[gnu::section(".vectors"), gnu::aligned(1024), gnu::used]]
+const HandlerType vector_table[] = {
+    reinterpret_cast<HandlerType>(&__stack), // SP
+    _start,                                  // Reset
+    NMI_Handler,                             // NMI Handler
+    HardFault_Handler,                       // Hard Fault Handler
+    MemManage_Handler,                       // MPU Fault Handler
+    BusFault_Handler,                        // Bus Fault Handler
+    UsageFault_Handler,                      // Usage Fault Handler
+    0,                                       // Reserved
+    0,                                       // Reserved
+    0,                                       // Reserved
+    0,                                       // Reserved
+    SVC_Handler,                             // SVC Handler
+    DebugMon_Handler,                        // Debug Monitor Handler
+    0,                                       // Reserved
+    PendSV_Handler,                          // PendSV Handler
+    SysTick_Handler,                         // SysTick Handler
+                                             // Unused
 };
-} // extern "C"
+#else
+// Based on
+// https://developer.arm.com/documentation/den0013/0400/Boot-Code/Booting-a-bare-metal-system
+void Reset_Handler() { LIBC_NAMESPACE::exit(1); }
+void Undefined_Handler() { LIBC_NAMESPACE::exit(1); }
+void SWI_Handler() { LIBC_NAMESPACE::exit(1); }
+void PrefetchAbort_Handler() { LIBC_NAMESPACE::exit(1); }
+void DataAbort_Handler() { LIBC_NAMESPACE::exit(1); }
+void IRQ_Handler() { LIBC_NAMESPACE::exit(1); }
+void FIQ_Handler() { LIBC_NAMESPACE::exit(1); }
+
+// The AArch32 exception vector table has 8 entries, each of which is 4
+// bytes long, and contains code. The whole table must be 32-byte aligned.
+// The table may also be relocated, so we make it position-independent by
+// having a table of handler addresses and loading the address to pc.
+[[gnu::section(".vectors"), gnu::aligned(32), gnu::used, gnu::naked,
+  gnu::target("arm")]]
+void vector_table() {
+  asm("LDR pc, [pc, #24]");
+  asm("LDR pc, [pc, #24]");
+  asm("LDR pc, [pc, #24]");
+  asm("LDR pc, [pc, #24]");
+  asm("LDR pc, [pc, #24]");
+  asm("LDR pc, [pc, #24]");
+  asm("LDR pc, [pc, #24]");
+  asm("LDR pc, [pc, #24]");
+  asm(".word %c0" : : "X"(Reset_Handler));
+  asm(".word %c0" : : "X"(Undefined_Handler));
+  asm(".word %c0" : : "X"(SWI_Handler));
+  asm(".word %c0" : : "X"(PrefetchAbort_Handler));
+  asm(".word %c0" : : "X"(DataAbort_Handler));
+  asm(".word %c0" : : "X"(0));
+  asm(".word %c0" : : "X"(IRQ_Handler));
+  asm(".word %c0" : : "X"(FIQ_Handler));
+}
+#endif
+} // namespace
 
 namespace LIBC_NAMESPACE_DECL {
 [[noreturn]] void do_start() {
   // FIXME: set up the QEMU test environment
 
+#if __ARM_ARCH_PROFILE == 'A' || __ARM_ARCH_PROFILE == 'R'
+  // Set up registers to be used in exception handling
+  // Copy the current sp value to each of the banked copies of sp.
+  __arm_wsr("CPSR_c", 0x11); // FIQ
+  asm volatile("mov sp, %0" : : "r"(__builtin_frame_address(0)));
+  __arm_wsr("CPSR_c", 0x12); // IRQ
+  asm volatile("mov sp, %0" : : "r"(__builtin_frame_address(0)));
+  __arm_wsr("CPSR_c", 0x17); // ABT
+  asm volatile("mov sp, %0" : : "r"(__builtin_frame_address(0)));
+  __arm_wsr("CPSR_c", 0x1B); // UND
+  asm volatile("mov sp, %0" : : "r"(__builtin_frame_address(0)));
+  __arm_wsr("CPSR_c", 0x1F); // SYS
+  asm volatile("mov sp, %0" : : "r"(__builtin_frame_address(0)));
+  __arm_wsr("CPSR_c", 0x13); // SVC
+#endif
+
   // Perform the equivalent of scatterloading
-  LIBC_NAMESPACE::memcpy(__data_start, __data_source, (uintptr_t)__data_size);
-  LIBC_NAMESPACE::memset(__bss_start, '\0', (uintptr_t)__bss_size);
+  LIBC_NAMESPACE::memcpy(__data_start, __data_source,
+                         reinterpret_cast<uintptr_t>(__data_size));
+  LIBC_NAMESPACE::memset(__bss_start, '\0',
+                         reinterpret_cast<uintptr_t>(__bss_size));
   __libc_init_array();
 
   _platform_init();
@@ -86,7 +144,13 @@ namespace LIBC_NAMESPACE_DECL {
 }
 } // namespace LIBC_NAMESPACE_DECL
 
-extern "C" void _start() {
+extern "C" {
+#ifdef __ARM_ARCH_ISA_ARM
+// If ARM state is supported, it must be used (instead of Thumb)
+[[gnu::naked, gnu::target("arm")]]
+#endif
+void _start() {
   asm volatile("mov sp, %0" : : "r"(&__stack));
   asm volatile("bl %0" : : "X"(LIBC_NAMESPACE::do_start));
 }
+} // extern "C"
diff --git a/libc/test/IntegrationTest/test.cpp b/libc/test/IntegrationTest/test.cpp
index 0e4825202a9e2..19eb255e57b04 100644
--- a/libc/test/IntegrationTest/test.cpp
+++ b/libc/test/IntegrationTest/test.cpp
@@ -63,8 +63,8 @@ int atexit(void (*func)(void)) { return LIBC_NAMESPACE::atexit(func); }
 // which just hands out continuous blocks from a statically allocated chunk of
 // memory.
 
-static constexpr uint64_t ALIGNMENT = alignof(long double);
-static constexpr uint64_t MEMORY_SIZE = 65336;
+static constexpr uint64_t ALIGNMENT = alignof(double);
+static constexpr uint64_t MEMORY_SIZE = 256 * 1024 /* 256 KiB */;
 alignas(ALIGNMENT) static uint8_t memory[MEMORY_SIZE];
 static size_t ptr = 0;
 
@@ -75,7 +75,7 @@ void *malloc(size_t size) {
   size = (size + ALIGNMENT - 1) & ~(ALIGNMENT - 1);
   size_t old_ptr =
       ref.fetch_add(size, LIBC_NAMESPACE::cpp::MemoryOrder::RELAXED);
-  if (static_cast<size_t>(old_ptr + size) > MEMORY_SIZE)
+  if (static_cast<size_t>(old_ptr + size) >= MEMORY_SIZE)
     return nullptr;
   return &memory[old_ptr];
 }
diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt
index d92ab6feccb27..f1a83fc601e5e 100644
--- a/libc/test/UnitTest/CMakeLists.txt
+++ b/libc/test/UnitTest/CMakeLists.txt
@@ -125,6 +125,7 @@ add_unittest_framework_library(
     RoundingModeUtils.h
   DEPENDS
     LibcTest
+    libc.test.UnitTest.ErrnoCheckingTest
     libc.test.UnitTest.string_utils
     libc.src.__support.CPP.array
     libc.src.__support.FPUtil.fp_bits
diff --git a/libc/test/UnitTest/FEnvSafeTest.cpp b/libc/test/UnitTest/FEnvSafeTest.cpp
index 168b1d4159d8d..f644569695eec 100644
--- a/libc/test/UnitTest/FEnvSafeTest.cpp
+++ b/libc/test/UnitTest/FEnvSafeTest.cpp
@@ -9,8 +9,10 @@
 #include "FEnvSafeTest.h"
 
 #include "src/__support/FPUtil/FEnvImpl.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/architectures.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 
 namespace LIBC_NAMESPACE_DECL {
 namespace testing {
@@ -25,6 +27,10 @@ void FEnvSafeTest::TearDown() {
   if (!should_be_unchanged) {
     restore_fenv();
   }
+  // TODO (PR 135320): Remove this override once all FEnvSafeTest instances are
+  // updated to validate or ignore errno.
+  libc_errno = 0;
+  ErrnoCheckingTest::TearDown();
 }
 
 void FEnvSafeTest::get_fenv(fenv_t &fenv) {
diff --git a/libc/test/UnitTest/FEnvSafeTest.h b/libc/test/UnitTest/FEnvSafeTest.h
index a3c5e62120bc0..1e10629ba3be8 100644
--- a/libc/test/UnitTest/FEnvSafeTest.h
+++ b/libc/test/UnitTest/FEnvSafeTest.h
@@ -12,6 +12,7 @@
 #include "hdr/types/fenv_t.h"
 #include "src/__support/CPP/utility.h"
 #include "src/__support/macros/config.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/Test.h"
 
 namespace LIBC_NAMESPACE_DECL {
@@ -20,7 +21,7 @@ namespace testing {
 // This provides a test fixture (or base class for other test fixtures) that
 // asserts that each test does not leave the FPU state represented by `fenv_t`
 // (aka `FPState`) perturbed from its initial state.
-class FEnvSafeTest : public Test {
+class FEnvSafeTest : public ErrnoCheckingTest {
 public:
   void TearDown() override;
 
diff --git a/libc/test/UnitTest/FPMatcher.h b/libc/test/UnitTest/FPMatcher.h
index da15cf2907f7c..592cd1b3f37ab 100644
--- a/libc/test/UnitTest/FPMatcher.h
+++ b/libc/test/UnitTest/FPMatcher.h
@@ -14,8 +14,10 @@
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/fpbits_str.h"
+#include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/architectures.h"
+#include "test/UnitTest/ErrnoCheckingTest.h"
 #include "test/UnitTest/RoundingModeUtils.h"
 #include "test/UnitTest/StringUtils.h"
 #include "test/UnitTest/Test.h"
@@ -166,7 +168,7 @@ CFPMatcher<T, C> getMatcherComplex(T expectedValue) {
   return CFPMatcher<T, C>(expectedValue);
 }
 
-template <typename T> struct FPTest : public Test {
+template <typename T> struct FPTest : public ErrnoCheckingTest {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
   using StorageType = typename FPBits::StorageType;
   static constexpr StorageType STORAGE_MAX =
@@ -191,6 +193,13 @@ template <typename T> struct FPTest : public Test {
       fputil::testing::RoundingMode::Downward,
       fputil::testing::RoundingMode::TowardZero,
   };
+
+  void TearDown() override {
+    // TODO (PR 135320): Remove this override once all FPTest instances are
+    // updated to validate or ignore errno.
+    libc_errno = 0;
+    ErrnoCheckingTest::TearDown();
+  }
 };
 
 // Add facility to test Flush-Denormal-To-Zero (FTZ) and Denormal-As-Zero (DAZ)
diff --git a/libc/test/integration/src/__support/GPU/CMakeLists.txt b/libc/test/integration/src/__support/GPU/CMakeLists.txt
index e066830f6cc0d..1fb175b92ef2a 100644
--- a/libc/test/integration/src/__support/GPU/CMakeLists.txt
+++ b/libc/test/integration/src/__support/GPU/CMakeLists.txt
@@ -27,3 +27,16 @@ add_integration_test(
   LOADER_ARGS
     --threads 64
 )
+
+add_libc_test(
+  fixedstack_test
+  SUITE
+    libc-support-gpu-tests
+  SRCS
+    fixedstack_test.cpp
+  DEPENDS
+    libc.src.__support.GPU.fixedstack
+  LOADER_ARGS
+    --threads 32
+    --blocks 16
+)
diff --git a/libc/test/integration/src/__support/GPU/fixedstack_test.cpp b/libc/test/integration/src/__support/GPU/fixedstack_test.cpp
new file mode 100644
index 0000000000000..fde51df325ab8
--- /dev/null
+++ b/libc/test/integration/src/__support/GPU/fixedstack_test.cpp
@@ -0,0 +1,44 @@
+//===-- Integration test for the lock-free stack --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/GPU/fixedstack.h"
+#include "src/__support/GPU/utils.h"
+#include "test/IntegrationTest/test.h"
+
+using namespace LIBC_NAMESPACE;
+
+static FixedStack<uint32_t, 2048> global_stack;
+
+void run() {
+  // We need enough space in the stack as threads in flight can temporarily
+  // consume memory before they finish comitting it back to the stack.
+  ASSERT_EQ(gpu::get_num_blocks() * gpu::get_num_threads(), 512);
+
+  uint32_t val;
+  uint32_t num_threads = static_cast<uint32_t>(gpu::get_num_threads());
+  for (int i = 0; i < 256; ++i) {
+    EXPECT_TRUE(global_stack.push(UINT32_MAX))
+    EXPECT_TRUE(global_stack.pop(val))
+    ASSERT_TRUE(val < num_threads || val == UINT32_MAX);
+  }
+
+  EXPECT_TRUE(global_stack.push(static_cast<uint32_t>(gpu::get_thread_id())));
+  EXPECT_TRUE(global_stack.push(static_cast<uint32_t>(gpu::get_thread_id())));
+  EXPECT_TRUE(global_stack.pop(val));
+  ASSERT_TRUE(val < num_threads || val == UINT32_MAX);
+
+  // Fill the rest of the stack with the default value.
+  while (!global_stack.push(UINT32_MAX))
+    ;
+}
+
+TEST_MAIN(int argc, char **argv, char **envp) {
+  run();
+
+  return 0;
+}
diff --git a/libc/test/integration/src/__support/GPU/match.cpp b/libc/test/integration/src/__support/GPU/match.cpp
index 2d314c2805158..70c22b7f3a91b 100644
--- a/libc/test/integration/src/__support/GPU/match.cpp
+++ b/libc/test/integration/src/__support/GPU/match.cpp
@@ -21,7 +21,9 @@ static void test_match() {
             gpu::match_any(mask, gpu::get_lane_id()));
   EXPECT_EQ(mask, gpu::match_any(mask, 1));
 
-  uint64_t expected = gpu::get_lane_id() < 16 ? 0xffff : 0xffff0000;
+  uint64_t full_mask =
+      gpu::get_lane_size() > 32 ? 0xffffffffffffffff : 0xffffffff;
+  uint64_t expected = gpu::get_lane_id() < 16 ? 0xffff : full_mask & ~0xffff;
   EXPECT_EQ(expected, gpu::match_any(mask, gpu::get_lane_id() < 16));
   EXPECT_EQ(mask, gpu::match_all(mask, 1));
   EXPECT_EQ(0ull, gpu::match_all(mask, gpu::get_lane_id()));
diff --git a/libc/test/shared/CMakeLists.txt b/libc/test/shared/CMakeLists.txt
index f5ea510180366..48241d3f55287 100644
--- a/libc/test/shared/CMakeLists.txt
+++ b/libc/test/shared/CMakeLists.txt
@@ -27,6 +27,14 @@ add_fp_unittest(
     libc.src.__support.math.atanhf
     libc.src.__support.math.atanhf16
     libc.src.__support.math.cbrt
+    libc.src.__support.math.cbrtf
+    libc.src.__support.math.cos
+    libc.src.__support.math.cosf
+    libc.src.__support.math.cosf16
+    libc.src.__support.math.coshf
+    libc.src.__support.math.coshf16
+    libc.src.__support.math.cospif
+    libc.src.__support.math.cospif16
     libc.src.__support.math.erff
     libc.src.__support.math.exp
     libc.src.__support.math.exp10
diff --git a/libc/test/shared/shared_math_test.cpp b/libc/test/shared/shared_math_test.cpp
index 3d64e5e68882b..2e5a2d51146d4 100644
--- a/libc/test/shared/shared_math_test.cpp
+++ b/libc/test/shared/shared_math_test.cpp
@@ -21,7 +21,9 @@ TEST(LlvmLibcSharedMathTest, AllFloat16) {
   EXPECT_FP_EQ(0x0p+0f16, LIBC_NAMESPACE::shared::asinhf16(0.0f16));
   EXPECT_FP_EQ(0x0p+0f16, LIBC_NAMESPACE::shared::atanf16(0.0f16));
   EXPECT_FP_EQ(0x0p+0f16, LIBC_NAMESPACE::shared::atanhf16(0.0f16));
-
+  EXPECT_FP_EQ(0x1p+0f16, LIBC_NAMESPACE::shared::cosf16(0.0f16));
+  EXPECT_FP_EQ(0x1p+0f16, LIBC_NAMESPACE::shared::coshf16(0.0f16));
+  EXPECT_FP_EQ(0x1p+0f16, LIBC_NAMESPACE::shared::cospif16(0.0f16));
   EXPECT_FP_EQ(0x1p+0f16, LIBC_NAMESPACE::shared::exp10f16(0.0f16));
 
   EXPECT_FP_EQ(0x1p+0f16, LIBC_NAMESPACE::shared::expf16(0.0f16));
@@ -49,6 +51,10 @@ TEST(LlvmLibcSharedMathTest, AllFloat) {
   EXPECT_FP_EQ(0x0p+0f, LIBC_NAMESPACE::shared::atan2f(0.0f, 0.0f));
   EXPECT_FP_EQ(0x0p+0f, LIBC_NAMESPACE::shared::atanf(0.0f));
   EXPECT_FP_EQ(0x0p+0f, LIBC_NAMESPACE::shared::atanhf(0.0f));
+  EXPECT_FP_EQ(0x0p+0f, LIBC_NAMESPACE::shared::cbrtf(0.0f));
+  EXPECT_FP_EQ(0x1p+0f, LIBC_NAMESPACE::shared::cosf(0.0f));
+  EXPECT_FP_EQ(0x1p+0f, LIBC_NAMESPACE::shared::coshf(0.0f));
+  EXPECT_FP_EQ(0x1p+0f, LIBC_NAMESPACE::shared::cospif(0.0f));
   EXPECT_FP_EQ(0x0p+0f, LIBC_NAMESPACE::shared::erff(0.0f));
   EXPECT_FP_EQ(0x1p+0f, LIBC_NAMESPACE::shared::exp10f(0.0f));
   EXPECT_FP_EQ(0x1p+0f, LIBC_NAMESPACE::shared::expf(0.0f));
@@ -65,8 +71,9 @@ TEST(LlvmLibcSharedMathTest, AllDouble) {
   EXPECT_FP_EQ(0x1.921fb54442d18p+0, LIBC_NAMESPACE::shared::acos(0.0));
   EXPECT_FP_EQ(0x0p+0, LIBC_NAMESPACE::shared::asin(0.0));
   EXPECT_FP_EQ(0x0p+0, LIBC_NAMESPACE::shared::atan(0.0));
-  EXPECT_FP_EQ(0x0p+0, LIBC_NAMESPACE::shared::cbrt(0.0));
   EXPECT_FP_EQ(0x0p+0, LIBC_NAMESPACE::shared::atan2(0.0, 0.0));
+  EXPECT_FP_EQ(0x0p+0, LIBC_NAMESPACE::shared::cbrt(0.0));
+  EXPECT_FP_EQ(0x1p+0, LIBC_NAMESPACE::shared::cos(0.0));
   EXPECT_FP_EQ(0x1p+0, LIBC_NAMESPACE::shared::exp(0.0));
   EXPECT_FP_EQ(0x1p+0, LIBC_NAMESPACE::shared::exp10(0.0));
 }
diff --git a/libc/test/src/__support/File/file_test.cpp b/libc/test/src/__support/File/file_test.cpp
index b3c9f2ba49bce..17dad4d5ac917 100644
--- a/libc/test/src/__support/File/file_test.cpp
+++ b/libc/test/src/__support/File/file_test.cpp
@@ -493,3 +493,22 @@ TEST(LlvmLibcFileTest, WriteNothing) {
   ASSERT_EQ(f_lbf->close(), 0);
   ASSERT_EQ(f_nbf->close(), 0);
 }
+
+TEST(LlvmLibcFileTest, WriteSplit) {
+  constexpr size_t FILE_BUFFER_SIZE = 8;
+  char file_buffer[FILE_BUFFER_SIZE];
+  StringFile *f =
+      new_string_file(file_buffer, FILE_BUFFER_SIZE, _IOFBF, false, "w");
+
+  static constexpr size_t AVAIL = 12;
+  f->seek(-AVAIL, SEEK_END);
+
+  const char data[] = "hello";
+  ASSERT_EQ(sizeof(data) - 1, f->write(data, sizeof(data) - 1).value);
+
+  const char data2[] = " extra data";
+  static constexpr size_t WR_EXPECTED = AVAIL - (sizeof(data) - 1);
+  ASSERT_EQ(WR_EXPECTED, f->write(data2, sizeof(data2) - 1).value);
+  EXPECT_TRUE(f->error());
+  ASSERT_EQ(f->close(), 0);
+}
diff --git a/libc/test/src/__support/OSUtil/linux/vdso_test.cpp b/libc/test/src/__support/OSUtil/linux/vdso_test.cpp
index 2f68470e8f678..71892a055daba 100644
--- a/libc/test/src/__support/OSUtil/linux/vdso_test.cpp
+++ b/libc/test/src/__support/OSUtil/linux/vdso_test.cpp
@@ -110,8 +110,8 @@ TEST(LlvmLibcOSUtilVDSOTest, RtSigReturn) {
   using namespace testing::ErrnoSetterMatcher;
   // must use struct since there is a function of the same name in the same
   // scope.
-  struct sigaction sa {};
-  struct sigaction old_sa {};
+  struct sigaction sa{};
+  struct sigaction old_sa{};
   sa.sa_handler = sigprof_handler;
   sa.sa_flags = SA_RESTORER;
   vdso::TypedSymbol<vdso::VDSOSym::RTSigReturn> symbol;
@@ -158,4 +158,30 @@ TEST(LlvmLibcOSUtilVDSOTest, RiscvHwProbe) {
   }
 }
 
+TEST(LlvmLibcOSUtilVDSOTest, GetRandom) {
+  using namespace testing::ErrnoSetterMatcher;
+  vdso::TypedSymbol<vdso::VDSOSym::GetRandom> symbol;
+  if (!symbol)
+    return;
+  // This structure exists in kernel UAPI header; but we define it on our own to
+  // make sure we can test it even on platform without support.
+  struct VGetrandomOpaqueParams {
+    uint32_t size_of_opaque_states;
+    uint32_t mmap_prot;
+    uint32_t mmap_flags;
+    uint32_t reserved[13];
+  };
+  VGetrandomOpaqueParams param{0, 0, 0, {}};
+  // When getrandom vDSO symbol is called with special parameters (~0 for state
+  // size), it populates the desired configuration into VGetrandomOpaqueParams.
+  int res = symbol(
+      /*buf=*/nullptr, /*count=*/0, /*flags=*/0,
+      /*opaque_states=*/&param,
+      /*size_of_opaque_states=*/~0);
+  // Test that the size of the states are correctly populated after a successful
+  // call.
+  EXPECT_EQ(res, 0);
+  EXPECT_GT(param.size_of_opaque_states, 0u);
+}
+
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/__support/wchar/string_converter_test.cpp b/libc/test/src/__support/wchar/string_converter_test.cpp
index d514df9317852..e45358ddc68c4 100644
--- a/libc/test/src/__support/wchar/string_converter_test.cpp
+++ b/libc/test/src/__support/wchar/string_converter_test.cpp
@@ -34,32 +34,32 @@ TEST(LlvmLibcStringConverterTest, UTF8To32) {
   LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
       reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX);
 
-  auto res = sc.popUTF32();
+  auto res = sc.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
 
-  res = sc.popUTF32();
+  res = sc.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x2211);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 7);
 
-  res = sc.popUTF32();
+  res = sc.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xff);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 9);
 
-  res = sc.popUTF32();
+  res = sc.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x41);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 10);
 
-  res = sc.popUTF32();
+  res = sc.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 11);
 
-  res = sc.popUTF32();
+  res = sc.pop<char32_t>();
   ASSERT_FALSE(res.has_value());
   ASSERT_EQ(res.error(), -1);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 11);
@@ -75,66 +75,66 @@ TEST(LlvmLibcStringConverterTest, UTF32To8) {
   LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(
       reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX);
 
-  auto res = sc.popUTF8();
+  auto res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
   // end of clown emoji, sigma symbol begins
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xE2);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x88);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x91);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 2);
 
   // end of sigma symbol, y with diaeresis begins
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xC3);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 3);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xBF);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 3);
 
   // end of y with diaeresis, letter A begins
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x41);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
 
   // null byte
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_FALSE(res.has_value());
   ASSERT_EQ(res.error(), -1);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);
@@ -148,28 +148,28 @@ TEST(LlvmLibcStringConverterTest, UTF32To8PartialRead) {
   LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(
       reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX, 1);
 
-  auto res = sc.popUTF8();
+  auto res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
   // can only read 1 character from source string, so error on next pop
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_FALSE(res.has_value());
   ASSERT_EQ(res.error(), -1);
 }
@@ -181,12 +181,12 @@ TEST(LlvmLibcStringConverterTest, UTF8To32PartialRead) {
   LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
       reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX, 5);
 
-  auto res = sc.popUTF32();
+  auto res = sc.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
 
-  res = sc.popUTF32();
+  res = sc.pop<char32_t>();
   ASSERT_FALSE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.error()), -1);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 5);
@@ -200,27 +200,27 @@ TEST(LlvmLibcStringConverterTest, UTF32To8ErrorHandling) {
   LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(
       reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX);
 
-  auto res = sc.popUTF8();
+  auto res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_FALSE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.error()), EILSEQ);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
@@ -234,12 +234,12 @@ TEST(LlvmLibcStringConverterTest, UTF8To32ErrorHandling) {
   LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
       reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX);
 
-  auto res = sc.popUTF32();
+  auto res = sc.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
 
-  res = sc.popUTF32();
+  res = sc.pop<char32_t>();
   ASSERT_FALSE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.error()), EILSEQ);
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
@@ -257,12 +257,12 @@ TEST(LlvmLibcStringConverterTest, InvalidCharacterOutsideBounds) {
   LIBC_NAMESPACE::internal::StringConverter<char8_t> sc1(
       reinterpret_cast<const char8_t *>(src1), &ps1, 1);
 
-  auto res1 = sc1.popUTF32();
+  auto res1 = sc1.pop<char32_t>();
   ASSERT_TRUE(res1.has_value());
   ASSERT_EQ(static_cast<int>(res1.value()), 0x1f921);
   ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 4);
 
-  res1 = sc1.popUTF32();
+  res1 = sc1.pop<char32_t>();
   ASSERT_FALSE(res1.has_value());
   // no space to write error NOT invalid character error (EILSEQ)
   ASSERT_EQ(static_cast<int>(res1.error()), -1);
@@ -275,27 +275,27 @@ TEST(LlvmLibcStringConverterTest, InvalidCharacterOutsideBounds) {
   LIBC_NAMESPACE::internal::StringConverter<char32_t> sc2(
       reinterpret_cast<const char32_t *>(src2), &ps2, 4);
 
-  auto res2 = sc2.popUTF8();
+  auto res2 = sc2.pop<char8_t>();
   ASSERT_TRUE(res2.has_value());
   ASSERT_EQ(static_cast<int>(res2.value()), 0xF0);
   ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
 
-  res2 = sc2.popUTF8();
+  res2 = sc2.pop<char8_t>();
   ASSERT_TRUE(res2.has_value());
   ASSERT_EQ(static_cast<int>(res2.value()), 0x9F);
   ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
 
-  res2 = sc2.popUTF8();
+  res2 = sc2.pop<char8_t>();
   ASSERT_TRUE(res2.has_value());
   ASSERT_EQ(static_cast<int>(res2.value()), 0xA4);
   ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
 
-  res2 = sc2.popUTF8();
+  res2 = sc2.pop<char8_t>();
   ASSERT_TRUE(res2.has_value());
   ASSERT_EQ(static_cast<int>(res2.value()), 0xA1);
   ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
 
-  res2 = sc2.popUTF8();
+  res2 = sc2.pop<char8_t>();
   ASSERT_FALSE(res2.has_value());
   // no space to write error NOT invalid character error (EILSEQ)
   ASSERT_EQ(static_cast<int>(res2.error()), -1);
@@ -315,22 +315,22 @@ TEST(LlvmLibcStringConverterTest, MultipleStringConverters32To8) {
   LIBC_NAMESPACE::internal::StringConverter<char32_t> sc1(
       reinterpret_cast<const char32_t *>(src), &state, SIZE_MAX, 1);
 
-  auto res = sc1.popUTF8();
+  auto res = sc1.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xF0);
   ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);
 
-  res = sc1.popUTF8();
+  res = sc1.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x9F);
   ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);
 
-  res = sc1.popUTF8();
+  res = sc1.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xA4);
   ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);
 
-  res = sc1.popUTF8();
+  res = sc1.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xA1);
   ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 1);
@@ -340,12 +340,12 @@ TEST(LlvmLibcStringConverterTest, MultipleStringConverters32To8) {
       reinterpret_cast<const char32_t *>(src) + sc1.getSourceIndex(), &state,
       SIZE_MAX, 1);
 
-  res = sc2.popUTF8();
+  res = sc2.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xC3);
   ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
 
-  res = sc2.popUTF8();
+  res = sc2.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0xBF);
   ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 1);
@@ -357,7 +357,7 @@ TEST(LlvmLibcStringConverterTest, MultipleStringConverters8To32) {
   LIBC_NAMESPACE::internal::StringConverter<char8_t> sc1(
       reinterpret_cast<const char8_t *>(src), &state, SIZE_MAX, 2);
 
-  auto res = sc1.popUTF32();
+  auto res = sc1.pop<char32_t>();
   ASSERT_FALSE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.error()), -1);
   ASSERT_EQ(static_cast<int>(sc1.getSourceIndex()), 2);
@@ -367,12 +367,12 @@ TEST(LlvmLibcStringConverterTest, MultipleStringConverters8To32) {
       reinterpret_cast<const char8_t *>(src) + sc1.getSourceIndex(), &state,
       SIZE_MAX, 3);
 
-  res = sc2.popUTF32();
+  res = sc2.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0x1f921);
   ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 2);
 
-  res = sc2.popUTF32();
+  res = sc2.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(res.value()), 0);
   ASSERT_EQ(static_cast<int>(sc2.getSourceIndex()), 3);
@@ -384,11 +384,11 @@ TEST(LlvmLibcStringConverterTest, DestLimitUTF8To32) {
   LIBC_NAMESPACE::internal::StringConverter<char8_t> sc(
       reinterpret_cast<const char8_t *>(src), &state, 1);
 
-  auto res = sc.popUTF32();
+  auto res = sc.pop<char32_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 4);
 
-  res = sc.popUTF32(); // no space to pop this into
+  res = sc.pop<char32_t>(); // no space to pop this into
   ASSERT_FALSE(res.has_value());
 }
 
@@ -399,23 +399,23 @@ TEST(LlvmLibcStringConverterTest, DestLimitUTF32To8) {
   LIBC_NAMESPACE::internal::StringConverter<char32_t> sc(
       reinterpret_cast<const char32_t *>(src), &state, 5);
 
-  auto res = sc.popUTF8();
+  auto res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_TRUE(res.has_value());
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 
-  res = sc.popUTF8();
+  res = sc.pop<char8_t>();
   ASSERT_FALSE(res.has_value());
   ASSERT_EQ(static_cast<int>(sc.getSourceIndex()), 1);
 }
diff --git a/libc/test/src/math/CMakeLists.txt b/libc/test/src/math/CMakeLists.txt
index 43cde0d68873e..8e24c5d5c2afd 100644
--- a/libc/test/src/math/CMakeLists.txt
+++ b/libc/test/src/math/CMakeLists.txt
@@ -668,6 +668,25 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  lroundbf16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    lroundbf16_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.errno.errno
+    libc.src.math.lroundbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   llround_test
   NEED_MPFR
@@ -740,6 +759,25 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  llroundbf16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    llroundbf16_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.errno.errno
+    libc.src.math.llroundbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   nearbyint_test
   NEED_MPFR
@@ -800,6 +838,22 @@ add_fp_unittest(
     libc.src.__support.CPP.array
 )
 
+add_fp_unittest(
+  nearbyintbf16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    nearbyintbf16_test.cpp
+  HDRS
+    NearbyIntTest.h
+  DEPENDS
+    libc.src.math.nearbyintbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.CPP.array
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   rint_test
   NEED_MPFR
@@ -868,6 +922,24 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  rintbf16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    rintbf16_test.cpp
+  HDRS
+    RIntTest.h
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.math.rintbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   lrint_test
   NEED_MPFR
@@ -932,6 +1004,23 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  lrintbf16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    lrintbf16_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.src.math.lrintbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   llrint_test
   NEED_MPFR
@@ -996,6 +1085,23 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  llrintbf16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    llrintbf16_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.src.math.llrintbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   exp_test
   NEED_MPFR
@@ -2282,6 +2388,17 @@ add_fp_unittest(
     libc.src.math.asinf16  
 )
 
+add_fp_unittest(
+  asinpif16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    asinpif16_test.cpp
+  DEPENDS
+    libc.src.math.asinpif16
+)
+
 add_fp_unittest(
   acosf_test
   NEED_MPFR
@@ -2972,6 +3089,302 @@ add_fp_unittest(
     libc.src.__support.macros.properties.types
 )
 
+add_fp_unittest(
+  bf16add_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16add_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.src.math.bf16add
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16addf_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16addf_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.src.math.bf16addf
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16addl_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16addl_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.src.math.bf16addl
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16addf128_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16addf128_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.src.math.bf16addf128
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16div_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16div_test.cpp
+  HDRS
+    DivTest.h
+  DEPENDS
+    libc.src.math.bf16div
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16divf_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16divf_test.cpp
+  HDRS
+    DivTest.h
+  DEPENDS
+    libc.src.math.bf16divf
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16divl_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16divl_test.cpp
+  HDRS
+    DivTest.h
+  DEPENDS
+    libc.src.math.bf16divl
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16divf128_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16divf128_test.cpp
+  HDRS
+    DivTest.h
+  DEPENDS
+    libc.src.math.bf16divf128
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16fma_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16fma_test.cpp
+  HDRS
+    FmaTest.h
+  DEPENDS
+    libc.src.math.bf16fma
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16fmaf_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16fmaf_test.cpp
+  HDRS
+    FmaTest.h
+  DEPENDS
+    libc.src.math.bf16fmaf
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16fmal_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16fmal_test.cpp
+  HDRS
+    FmaTest.h
+  DEPENDS
+    libc.src.math.bf16fmal
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16fmaf128_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16fmaf128_test.cpp
+  HDRS
+    FmaTest.h
+  DEPENDS
+    libc.src.math.bf16fmaf128
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16mul_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16mul_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.src.math.bf16mul
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16mulf_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16mulf_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.src.math.bf16mulf
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16mull_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16mull_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.src.math.bf16mull
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16mulf128_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16mulf128_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.src.math.bf16mulf128
+    libc.src.stdlib.rand
+    libc.src.stdlib.srand
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16sub_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16sub_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.src.math.bf16sub
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16subf_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16subf_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.src.math.bf16subf
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16subl_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16subl_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.src.math.bf16subl
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16subf128_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    bf16subf128_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.src.math.bf16subf128
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_subdirectory(generic)
 add_subdirectory(smoke)
 
diff --git a/libc/test/src/math/RoundToIntegerTest.h b/libc/test/src/math/RoundToIntegerTest.h
index 6af9cfea0e0a5..e5e93869a3235 100644
--- a/libc/test/src/math/RoundToIntegerTest.h
+++ b/libc/test/src/math/RoundToIntegerTest.h
@@ -127,8 +127,8 @@ class RoundToIntegerTestTemplate
     test_one_input(func, FloatType(-1.0), IntType(-1), false);
     test_one_input(func, FloatType(10.0), IntType(10), false);
     test_one_input(func, FloatType(-10.0), IntType(-10), false);
-    test_one_input(func, FloatType(1234.0), IntType(1234), false);
-    test_one_input(func, FloatType(-1234.0), IntType(-1234), false);
+    test_one_input(func, FloatType(1232.0), IntType(1232), false);
+    test_one_input(func, FloatType(-1232.0), IntType(-1232), false);
 
     // The rest of this function compares with an equivalent MPFR function
     // which rounds floating point numbers to long values. There is no MPFR
diff --git a/libc/test/src/math/asinpif16_test.cpp b/libc/test/src/math/asinpif16_test.cpp
new file mode 100644
index 0000000000000..3718f39fd06a6
--- /dev/null
+++ b/libc/test/src/math/asinpif16_test.cpp
@@ -0,0 +1,40 @@
+//===-- Exhaustive test for asinpif16 -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/math/asinpif16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+using LlvmLibcAsinpif16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+
+// Range: [0, Inf]
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7c00U;
+
+// Range: [-Inf, 0]
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xfc00U;
+
+TEST_F(LlvmLibcAsinpif16Test, PositiveRange) {
+  for (uint16_t v = POS_START; v <= POS_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Asinpi, x,
+                                   LIBC_NAMESPACE::asinpif16(x), 0.5);
+  }
+}
+
+TEST_F(LlvmLibcAsinpif16Test, NegativeRange) {
+  for (uint16_t v = NEG_START; v <= NEG_STOP; ++v) {
+    float16 x = FPBits(v).get_val();
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Asinpi, x,
+                                   LIBC_NAMESPACE::asinpif16(x), 0.5);
+  }
+}
diff --git a/libc/test/src/math/bf16add_test.cpp b/libc/test/src/math/bf16add_test.cpp
new file mode 100644
index 0000000000000..9e9c594d7d7eb
--- /dev/null
+++ b/libc/test/src/math/bf16add_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16add ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16add.h"
+
+LIST_ADD_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16add)
diff --git a/libc/test/src/math/bf16addf128_test.cpp b/libc/test/src/math/bf16addf128_test.cpp
new file mode 100644
index 0000000000000..46f7ad329f237
--- /dev/null
+++ b/libc/test/src/math/bf16addf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16addf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16addf128.h"
+
+LIST_ADD_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16addf128)
diff --git a/libc/test/src/math/bf16addf_test.cpp b/libc/test/src/math/bf16addf_test.cpp
new file mode 100644
index 0000000000000..06d56cf21ebd2
--- /dev/null
+++ b/libc/test/src/math/bf16addf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16addf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16addf.h"
+
+LIST_ADD_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16addf)
diff --git a/libc/test/src/math/bf16addl_test.cpp b/libc/test/src/math/bf16addl_test.cpp
new file mode 100644
index 0000000000000..bf548272d31c1
--- /dev/null
+++ b/libc/test/src/math/bf16addl_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16addl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16addl.h"
+
+LIST_ADD_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16addl)
diff --git a/libc/test/src/math/bf16div_test.cpp b/libc/test/src/math/bf16div_test.cpp
new file mode 100644
index 0000000000000..4516351fb93a3
--- /dev/null
+++ b/libc/test/src/math/bf16div_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16div ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DivTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16div.h"
+
+LIST_DIV_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16div)
diff --git a/libc/test/src/math/bf16divf128_test.cpp b/libc/test/src/math/bf16divf128_test.cpp
new file mode 100644
index 0000000000000..c42c5bb067050
--- /dev/null
+++ b/libc/test/src/math/bf16divf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16divf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DivTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16divf128.h"
+
+LIST_DIV_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16divf128)
diff --git a/libc/test/src/math/bf16divf_test.cpp b/libc/test/src/math/bf16divf_test.cpp
new file mode 100644
index 0000000000000..873920b225a77
--- /dev/null
+++ b/libc/test/src/math/bf16divf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16divf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DivTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16divf.h"
+
+LIST_DIV_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16divf)
diff --git a/libc/test/src/math/bf16divl_test.cpp b/libc/test/src/math/bf16divl_test.cpp
new file mode 100644
index 0000000000000..32ecd6228de0b
--- /dev/null
+++ b/libc/test/src/math/bf16divl_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16divl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DivTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16divl.h"
+
+LIST_DIV_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16divl)
diff --git a/libc/test/src/math/bf16fma_test.cpp b/libc/test/src/math/bf16fma_test.cpp
new file mode 100644
index 0000000000000..81c73a0cb6947
--- /dev/null
+++ b/libc/test/src/math/bf16fma_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16fma ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FmaTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16fma.h"
+
+LIST_NARROWING_FMA_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16fma)
diff --git a/libc/test/src/math/bf16fmaf128_test.cpp b/libc/test/src/math/bf16fmaf128_test.cpp
new file mode 100644
index 0000000000000..dd8f4735c3497
--- /dev/null
+++ b/libc/test/src/math/bf16fmaf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16fmaf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FmaTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16fmaf128.h"
+
+LIST_NARROWING_FMA_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16fmaf128)
diff --git a/libc/test/src/math/bf16fmaf_test.cpp b/libc/test/src/math/bf16fmaf_test.cpp
new file mode 100644
index 0000000000000..04c674892833a
--- /dev/null
+++ b/libc/test/src/math/bf16fmaf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16fmaf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FmaTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16fmaf.h"
+
+LIST_NARROWING_FMA_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16fmaf)
diff --git a/libc/test/src/math/bf16fmal_test.cpp b/libc/test/src/math/bf16fmal_test.cpp
new file mode 100644
index 0000000000000..4c45e2c64312d
--- /dev/null
+++ b/libc/test/src/math/bf16fmal_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16fmal --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FmaTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16fmal.h"
+
+LIST_NARROWING_FMA_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16fmal)
diff --git a/libc/test/src/math/bf16mul_test.cpp b/libc/test/src/math/bf16mul_test.cpp
new file mode 100644
index 0000000000000..3682705556b0a
--- /dev/null
+++ b/libc/test/src/math/bf16mul_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for bf16mul ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/bf16mul.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+
+LIST_MUL_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16mul)
diff --git a/libc/test/src/math/bf16mulf128_test.cpp b/libc/test/src/math/bf16mulf128_test.cpp
new file mode 100644
index 0000000000000..6aee2687ae67a
--- /dev/null
+++ b/libc/test/src/math/bf16mulf128_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for bf16mulf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/bf16mulf128.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+
+LIST_MUL_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16mulf128)
diff --git a/libc/test/src/math/bf16mulf_test.cpp b/libc/test/src/math/bf16mulf_test.cpp
new file mode 100644
index 0000000000000..048b60f8e85cf
--- /dev/null
+++ b/libc/test/src/math/bf16mulf_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for bf16mulf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/bf16mulf.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+
+LIST_MUL_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16mulf)
diff --git a/libc/test/src/math/bf16mull_test.cpp b/libc/test/src/math/bf16mull_test.cpp
new file mode 100644
index 0000000000000..b8439b245c261
--- /dev/null
+++ b/libc/test/src/math/bf16mull_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for bf16mull --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/math/bf16mull.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+
+LIST_MUL_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16mull)
diff --git a/libc/test/src/math/bf16sub_test.cpp b/libc/test/src/math/bf16sub_test.cpp
new file mode 100644
index 0000000000000..4a793dc493891
--- /dev/null
+++ b/libc/test/src/math/bf16sub_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16sub ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16sub.h"
+
+LIST_SUB_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16sub)
diff --git a/libc/test/src/math/bf16subf128_test.cpp b/libc/test/src/math/bf16subf128_test.cpp
new file mode 100644
index 0000000000000..25d6711f71346
--- /dev/null
+++ b/libc/test/src/math/bf16subf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16subf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16subf128.h"
+
+LIST_SUB_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16subf128)
diff --git a/libc/test/src/math/bf16subf_test.cpp b/libc/test/src/math/bf16subf_test.cpp
new file mode 100644
index 0000000000000..e8c7440230f1d
--- /dev/null
+++ b/libc/test/src/math/bf16subf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16subf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16subf.h"
+
+LIST_SUB_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16subf)
diff --git a/libc/test/src/math/bf16subl_test.cpp b/libc/test/src/math/bf16subl_test.cpp
new file mode 100644
index 0000000000000..299736949d713
--- /dev/null
+++ b/libc/test/src/math/bf16subl_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16subl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16subl.h"
+
+LIST_SUB_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16subl)
diff --git a/libc/test/src/math/exhaustive/CMakeLists.txt b/libc/test/src/math/exhaustive/CMakeLists.txt
index 5246337dab64b..e40972ea9a879 100644
--- a/libc/test/src/math/exhaustive/CMakeLists.txt
+++ b/libc/test/src/math/exhaustive/CMakeLists.txt
@@ -567,3 +567,75 @@ add_fp_unittest(
   LINK_LIBRARIES
     -lpthread
 )
+
+add_fp_unittest(
+  bfloat16_add_test
+  NO_RUN_POSTBUILD
+  NEED_MPFR
+  SUITE
+    libc_math_exhaustive_tests
+  SRCS
+    bfloat16_add_test.cpp
+  COMPILE_OPTIONS
+    ${libc_opt_high_flag}
+  DEPENDS
+    .exhaustive_test
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+  LINK_LIBRARIES
+    -lpthread
+)
+
+add_fp_unittest(
+  bfloat16_div_test
+  NO_RUN_POSTBUILD
+  NEED_MPFR
+  SUITE
+    libc_math_exhaustive_tests
+  SRCS
+    bfloat16_div_test.cpp
+  COMPILE_OPTIONS
+    ${libc_opt_high_flag}
+  DEPENDS
+    .exhaustive_test
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+  LINK_LIBRARIES
+    -lpthread
+)
+
+add_fp_unittest(
+  bfloat16_mul_test
+  NO_RUN_POSTBUILD
+  NEED_MPFR
+  SUITE
+    libc_math_exhaustive_tests
+  SRCS
+    bfloat16_mul_test.cpp
+  COMPILE_OPTIONS
+    ${libc_opt_high_flag}
+  DEPENDS
+    .exhaustive_test
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+  LINK_LIBRARIES
+    -lpthread
+)
+
+add_fp_unittest(
+  bfloat16_sub_test
+  NO_RUN_POSTBUILD
+  NEED_MPFR
+  SUITE
+    libc_math_exhaustive_tests
+  SRCS
+    bfloat16_sub_test.cpp
+  COMPILE_OPTIONS
+    ${libc_opt_high_flag}
+  DEPENDS
+    .exhaustive_test
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+  LINK_LIBRARIES
+    -lpthread
+)
diff --git a/libc/test/src/math/exhaustive/bfloat16_add_test.cpp b/libc/test/src/math/exhaustive/bfloat16_add_test.cpp
new file mode 100644
index 0000000000000..3f4c77978a93d
--- /dev/null
+++ b/libc/test/src/math/exhaustive/bfloat16_add_test.cpp
@@ -0,0 +1,65 @@
+//===-- Exhaustive tests for bfloat16 addition ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "exhaustive_test.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "utils/MPFRWrapper/MPCommon.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LIBC_NAMESPACE::fputil::BFloat16;
+
+static BFloat16 add_func(BFloat16 x, BFloat16 y) { return x + y; }
+
+struct Bfloat16AddChecker : public virtual LIBC_NAMESPACE::testing::Test {
+  using FloatType = BFloat16;
+  using FPBits = LIBC_NAMESPACE::fputil::FPBits<bfloat16>;
+  using StorageType = typename FPBits::StorageType;
+
+  uint64_t check(uint16_t x_start, uint16_t x_stop, uint16_t y_start,
+                 uint16_t y_stop, mpfr::RoundingMode rounding) {
+    mpfr::ForceRoundingMode r(rounding);
+    if (!r.success)
+      return true;
+    uint16_t xbits = x_start;
+    uint64_t failed = 0;
+    do {
+      BFloat16 x = FPBits(xbits).get_val();
+      uint16_t ybits = xbits;
+      do {
+        BFloat16 y = FPBits(ybits).get_val();
+        mpfr::BinaryInput<BFloat16> input{x, y};
+        bool correct = TEST_MPFR_MATCH_ROUNDING_SILENTLY(
+            mpfr::Operation::Add, input, add_func(x, y), 0.5, rounding);
+        failed += (!correct);
+      } while (ybits++ < y_stop);
+    } while (xbits++ < x_stop);
+    return failed;
+  }
+};
+
+using LlvmLibcBfloat16ExhaustiveAddTest =
+    LlvmLibcExhaustiveMathTest<Bfloat16AddChecker, 1 << 2>;
+
+// range: [0, inf]
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7f80U;
+
+// range: [-0, -inf]
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xff80U;
+
+TEST_F(LlvmLibcBfloat16ExhaustiveAddTest, PositiveRange) {
+  test_full_range_all_roundings(POS_START, POS_STOP, POS_START, POS_STOP);
+}
+
+TEST_F(LlvmLibcBfloat16ExhaustiveAddTest, NegativeRange) {
+  test_full_range_all_roundings(NEG_START, NEG_STOP, NEG_START, NEG_STOP);
+}
diff --git a/libc/test/src/math/exhaustive/bfloat16_div_test.cpp b/libc/test/src/math/exhaustive/bfloat16_div_test.cpp
new file mode 100644
index 0000000000000..2648d5f775af5
--- /dev/null
+++ b/libc/test/src/math/exhaustive/bfloat16_div_test.cpp
@@ -0,0 +1,65 @@
+//===-- Exhaustive tests for bfloat16 division ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "exhaustive_test.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "utils/MPFRWrapper/MPCommon.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LIBC_NAMESPACE::fputil::BFloat16;
+
+static BFloat16 div_func(BFloat16 x, BFloat16 y) { return x / y; }
+
+struct Bfloat16DivChecker : public virtual LIBC_NAMESPACE::testing::Test {
+  using FloatType = BFloat16;
+  using FPBits = LIBC_NAMESPACE::fputil::FPBits<bfloat16>;
+  using StorageType = typename FPBits::StorageType;
+
+  uint64_t check(uint16_t x_start, uint16_t x_stop, uint16_t y_start,
+                 uint16_t y_stop, mpfr::RoundingMode rounding) {
+    mpfr::ForceRoundingMode r(rounding);
+    if (!r.success)
+      return true;
+    uint16_t xbits = x_start;
+    uint64_t failed = 0;
+    do {
+      BFloat16 x = FPBits(xbits).get_val();
+      uint16_t ybits = xbits;
+      do {
+        BFloat16 y = FPBits(ybits).get_val();
+        mpfr::BinaryInput<BFloat16> input{x, y};
+        bool correct = TEST_MPFR_MATCH_ROUNDING_SILENTLY(
+            mpfr::Operation::Div, input, div_func(x, y), 0.5, rounding);
+        failed += (!correct);
+      } while (ybits++ < y_stop);
+    } while (xbits++ < x_stop);
+    return failed;
+  }
+};
+
+using LlvmLibcBfloat16ExhaustiveDivTest =
+    LlvmLibcExhaustiveMathTest<Bfloat16DivChecker, 1 << 2>;
+
+// range: [0, inf]
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7f80U;
+
+// range: [-0, -inf]
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xff80U;
+
+TEST_F(LlvmLibcBfloat16ExhaustiveDivTest, PositiveRange) {
+  test_full_range_all_roundings(POS_START, POS_STOP, POS_START, POS_STOP);
+}
+
+TEST_F(LlvmLibcBfloat16ExhaustiveDivTest, NegativeRange) {
+  test_full_range_all_roundings(NEG_START, NEG_STOP, NEG_START, NEG_STOP);
+}
diff --git a/libc/test/src/math/exhaustive/bfloat16_mul_test.cpp b/libc/test/src/math/exhaustive/bfloat16_mul_test.cpp
new file mode 100644
index 0000000000000..3cbbcb500a4fb
--- /dev/null
+++ b/libc/test/src/math/exhaustive/bfloat16_mul_test.cpp
@@ -0,0 +1,65 @@
+//===-- Exhaustive tests for bfloat16 multiplication ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "exhaustive_test.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "utils/MPFRWrapper/MPCommon.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LIBC_NAMESPACE::fputil::BFloat16;
+
+static BFloat16 mul_func(BFloat16 x, BFloat16 y) { return x * y; }
+
+struct Bfloat16MulChecker : public virtual LIBC_NAMESPACE::testing::Test {
+  using FloatType = BFloat16;
+  using FPBits = LIBC_NAMESPACE::fputil::FPBits<bfloat16>;
+  using StorageType = typename FPBits::StorageType;
+
+  uint64_t check(uint16_t x_start, uint16_t x_stop, uint16_t y_start,
+                 uint16_t y_stop, mpfr::RoundingMode rounding) {
+    mpfr::ForceRoundingMode r(rounding);
+    if (!r.success)
+      return true;
+    uint16_t xbits = x_start;
+    uint64_t failed = 0;
+    do {
+      BFloat16 x = FPBits(xbits).get_val();
+      uint16_t ybits = xbits;
+      do {
+        BFloat16 y = FPBits(ybits).get_val();
+        mpfr::BinaryInput<BFloat16> input{x, y};
+        bool correct = TEST_MPFR_MATCH_ROUNDING_SILENTLY(
+            mpfr::Operation::Mul, input, mul_func(x, y), 0.5, rounding);
+        failed += (!correct);
+      } while (ybits++ < y_stop);
+    } while (xbits++ < x_stop);
+    return failed;
+  }
+};
+
+using LlvmLibcBfloat16ExhaustiveMulTest =
+    LlvmLibcExhaustiveMathTest<Bfloat16MulChecker, 1 << 2>;
+
+// range: [0, inf]
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7f80U;
+
+// range: [-0, -inf]
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xff80U;
+
+TEST_F(LlvmLibcBfloat16ExhaustiveMulTest, PositiveRange) {
+  test_full_range_all_roundings(POS_START, POS_STOP, POS_START, POS_STOP);
+}
+
+TEST_F(LlvmLibcBfloat16ExhaustiveMulTest, NegativeRange) {
+  test_full_range_all_roundings(NEG_START, NEG_STOP, NEG_START, NEG_STOP);
+}
diff --git a/libc/test/src/math/exhaustive/bfloat16_sub_test.cpp b/libc/test/src/math/exhaustive/bfloat16_sub_test.cpp
new file mode 100644
index 0000000000000..11bc6f59dd294
--- /dev/null
+++ b/libc/test/src/math/exhaustive/bfloat16_sub_test.cpp
@@ -0,0 +1,65 @@
+//===-- Exhaustive tests for bfloat16 subtraction -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "exhaustive_test.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "utils/MPFRWrapper/MPCommon.h"
+#include "utils/MPFRWrapper/MPFRUtils.h"
+
+namespace mpfr = LIBC_NAMESPACE::testing::mpfr;
+using LIBC_NAMESPACE::fputil::BFloat16;
+
+static BFloat16 sub_func(BFloat16 x, BFloat16 y) { return x - y; }
+
+struct Bfloat16SubChecker : public virtual LIBC_NAMESPACE::testing::Test {
+  using FloatType = BFloat16;
+  using FPBits = LIBC_NAMESPACE::fputil::FPBits<bfloat16>;
+  using StorageType = typename FPBits::StorageType;
+
+  uint64_t check(uint16_t x_start, uint16_t x_stop, uint16_t y_start,
+                 uint16_t y_stop, mpfr::RoundingMode rounding) {
+    mpfr::ForceRoundingMode r(rounding);
+    if (!r.success)
+      return true;
+    uint16_t xbits = x_start;
+    uint64_t failed = 0;
+    do {
+      BFloat16 x = FPBits(xbits).get_val();
+      uint16_t ybits = xbits;
+      do {
+        BFloat16 y = FPBits(ybits).get_val();
+        mpfr::BinaryInput<BFloat16> input{x, y};
+        bool correct = TEST_MPFR_MATCH_ROUNDING_SILENTLY(
+            mpfr::Operation::Sub, input, sub_func(x, y), 0.5, rounding);
+        failed += (!correct);
+      } while (ybits++ < y_stop);
+    } while (xbits++ < x_stop);
+    return failed;
+  }
+};
+
+using LlvmLibcBfloat16ExhaustiveSubTest =
+    LlvmLibcExhaustiveMathTest<Bfloat16SubChecker, 1 << 2>;
+
+// range: [0, inf]
+static constexpr uint16_t POS_START = 0x0000U;
+static constexpr uint16_t POS_STOP = 0x7f80U;
+
+// range: [-0, -inf]
+static constexpr uint16_t NEG_START = 0x8000U;
+static constexpr uint16_t NEG_STOP = 0xff80U;
+
+TEST_F(LlvmLibcBfloat16ExhaustiveSubTest, PositiveRange) {
+  test_full_range_all_roundings(POS_START, POS_STOP, POS_START, POS_STOP);
+}
+
+TEST_F(LlvmLibcBfloat16ExhaustiveSubTest, NegativeRange) {
+  test_full_range_all_roundings(NEG_START, NEG_STOP, NEG_START, NEG_STOP);
+}
diff --git a/libc/test/src/math/exhaustive/exhaustive_test.h b/libc/test/src/math/exhaustive/exhaustive_test.h
index cdf459c23bb9d..8be65ba69f725 100644
--- a/libc/test/src/math/exhaustive/exhaustive_test.h
+++ b/libc/test/src/math/exhaustive/exhaustive_test.h
@@ -164,7 +164,7 @@ struct LlvmLibcExhaustiveMathTest
 
             range_begin = current_value;
             if (stop >= Increment && stop - Increment >= current_value) {
-              range_end = current_value + Increment;
+              range_end = static_cast<StorageType>(current_value + Increment);
             } else {
               range_end = stop;
             }
diff --git a/libc/test/src/math/explogxf_test.cpp b/libc/test/src/math/explogxf_test.cpp
index 49cc96291a392..4d35309238619 100644
--- a/libc/test/src/math/explogxf_test.cpp
+++ b/libc/test/src/math/explogxf_test.cpp
@@ -9,11 +9,11 @@
 #include "hdr/math_macros.h"
 #include "in_float_range_test_helper.h"
 #include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/math/acoshf_utils.h"
+#include "src/__support/math/exp10f_utils.h"
 #include "src/math/fabs.h"
 #include "src/math/fabsf.h"
-#include "src/math/generic/explogxf.h"
 #include "test/UnitTest/FPMatcher.h"
-#include "test/UnitTest/Test.h"
 #include "utils/MPFRWrapper/MPFRUtils.h"
 
 using LlvmLibcExplogfTest = LIBC_NAMESPACE::testing::FPTest<float>;
diff --git a/libc/test/src/math/llrintbf16_test.cpp b/libc/test/src/math/llrintbf16_test.cpp
new file mode 100644
index 0000000000000..e841e627e2634
--- /dev/null
+++ b/libc/test/src/math/llrintbf16_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for llrintbf16 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/llrintbf16.h"
+
+LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(bfloat16, long long,
+                                       LIBC_NAMESPACE::llrintbf16)
diff --git a/libc/test/src/math/llroundbf16_test.cpp b/libc/test/src/math/llroundbf16_test.cpp
new file mode 100644
index 0000000000000..c3b7ea43c3e8a
--- /dev/null
+++ b/libc/test/src/math/llroundbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for llroundbf16 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/llroundbf16.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(bfloat16, long long, LIBC_NAMESPACE::llroundbf16)
diff --git a/libc/test/src/math/lrintbf16_test.cpp b/libc/test/src/math/lrintbf16_test.cpp
new file mode 100644
index 0000000000000..65a5633b81723
--- /dev/null
+++ b/libc/test/src/math/lrintbf16_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for lrintbf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/lrintbf16.h"
+
+LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(bfloat16, long,
+                                       LIBC_NAMESPACE::lrintbf16)
diff --git a/libc/test/src/math/lroundbf16_test.cpp b/libc/test/src/math/lroundbf16_test.cpp
new file mode 100644
index 0000000000000..2f2b7b1a71b36
--- /dev/null
+++ b/libc/test/src/math/lroundbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for lroundbf16 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/lroundbf16.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(bfloat16, long, LIBC_NAMESPACE::lroundbf16)
diff --git a/libc/test/src/math/nearbyintbf16_test.cpp b/libc/test/src/math/nearbyintbf16_test.cpp
new file mode 100644
index 0000000000000..2d64fc5a154ee
--- /dev/null
+++ b/libc/test/src/math/nearbyintbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for nearbyintbf16 ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NearbyIntTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/nearbyintbf16.h"
+
+LIST_NEARBYINT_TESTS(bfloat16, LIBC_NAMESPACE::nearbyintbf16)
diff --git a/libc/test/src/math/rintbf16_test.cpp b/libc/test/src/math/rintbf16_test.cpp
new file mode 100644
index 0000000000000..c78dcf6a31560
--- /dev/null
+++ b/libc/test/src/math/rintbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for rintbf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RIntTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/rintbf16.h"
+
+LIST_RINT_TESTS(bfloat16, LIBC_NAMESPACE::rintbf16)
diff --git a/libc/test/src/math/smoke/CMakeLists.txt b/libc/test/src/math/smoke/CMakeLists.txt
index 40b7a342e0c1a..f9da396202337 100644
--- a/libc/test/src/math/smoke/CMakeLists.txt
+++ b/libc/test/src/math/smoke/CMakeLists.txt
@@ -344,6 +344,19 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  truncbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    truncbf16_test.cpp
+  HDRS
+    TruncTest.h
+  DEPENDS
+    libc.src.math.truncbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   canonicalize_test
   SUITE
@@ -543,6 +556,19 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  ceilbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ceilbf16_test.cpp
+  HDRS
+    CeilTest.h
+  DEPENDS
+    libc.src.math.ceilbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   dfmal_test
   SUITE
@@ -663,6 +689,19 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  floorbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    floorbf16_test.cpp
+  HDRS
+    FloorTest.h
+  DEPENDS
+    libc.src.math.floorbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   round_test
   SUITE
@@ -727,6 +766,19 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  roundbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    roundbf16_test.cpp
+  HDRS
+    RoundTest.h
+  DEPENDS
+    libc.src.math.roundbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   roundeven_test
   SUITE
@@ -791,6 +843,19 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  roundevenbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    roundevenbf16_test.cpp
+  HDRS
+    RoundEvenTest.h
+  DEPENDS
+    libc.src.math.roundevenbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   lround_test
   SUITE
@@ -871,6 +936,23 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  lroundbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    lroundbf16_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.math.lroundbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   llround_test
   SUITE
@@ -951,6 +1033,23 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  llroundbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    llroundbf16_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.math.llroundbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   rint_test
   SUITE
@@ -1021,6 +1120,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  rintbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    rintbf16_test.cpp
+  HDRS
+    RIntTest.h
+  DEPENDS
+    libc.src.math.rintbf16
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   lrint_test
   SUITE
@@ -1101,6 +1215,23 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  lrintbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    lrintbf16_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.math.lrintbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   llrint_test
   SUITE
@@ -1181,6 +1312,23 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  llrintbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    llrintbf16_test.cpp
+  HDRS
+    RoundToIntegerTest.h
+  DEPENDS
+    libc.src.errno.errno
+    libc.src.math.llrintbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   exp_test
   SUITE
@@ -1471,6 +1619,19 @@ add_fp_unittest(
     libc.src.math.frexpf128
 )
 
+add_fp_unittest(
+  frexpbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    frexpbf16_test.cpp
+  HDRS
+    FrexpTest.h
+  DEPENDS
+    libc.src.math.frexpbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   fromfp_test
   SUITE
@@ -1531,6 +1692,19 @@ add_fp_unittest(
     libc.src.math.fromfpf128
 )
 
+add_fp_unittest(
+  fromfpbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfpbf16_test.cpp
+  HDRS
+    FromfpTest.h
+  DEPENDS
+    libc.src.math.fromfpbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   fromfpx_test
   SUITE
@@ -1591,6 +1765,20 @@ add_fp_unittest(
     libc.src.math.fromfpxf128
 )
 
+add_fp_unittest(
+  fromfpxbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fromfpxbf16_test.cpp
+  HDRS
+    FromfpxTest.h
+  DEPENDS
+    libc.src.math.fromfpxbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
+
 add_fp_unittest(
   ufromfp_test
   SUITE
@@ -1651,6 +1839,19 @@ add_fp_unittest(
     libc.src.math.ufromfpf128
 )
 
+add_fp_unittest(
+  ufromfpbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfpbf16_test.cpp
+  HDRS
+    UfromfpTest.h
+  DEPENDS
+    libc.src.math.ufromfpbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   ufromfpx_test
   SUITE
@@ -1711,6 +1912,19 @@ add_fp_unittest(
     libc.src.math.ufromfpxf128
 )
 
+add_fp_unittest(
+  ufromfpxbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ufromfpxbf16_test.cpp
+  HDRS
+    UfromfpxTest.h
+  DEPENDS
+    libc.src.math.ufromfpxbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   ilogb_test
   SUITE
@@ -1786,6 +2000,22 @@ add_fp_unittest(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_fp_unittest(
+  ilogbbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ilogbbf16_test.cpp
+  HDRS
+    ILogbTest.h
+  DEPENDS
+    libc.src.math.ilogbbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
 add_fp_unittest(
   issignaling_test
   SUITE
@@ -1921,6 +2151,22 @@ add_fp_unittest(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_fp_unittest(
+  llogbbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    llogbbf16_test.cpp
+  HDRS
+    ILogbTest.h
+  DEPENDS
+    libc.src.math.llogbbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
 add_fp_unittest(
   ldexp_test
   SUITE
@@ -1931,6 +2177,7 @@ add_fp_unittest(
     LdExpTest.h
   DEPENDS
     libc.src.math.ldexp
+    libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.limits
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.normal_float
@@ -1946,6 +2193,7 @@ add_fp_unittest(
     LdExpTest.h
   DEPENDS
     libc.src.math.ldexpf
+    libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.limits
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.normal_float
@@ -1961,6 +2209,7 @@ add_fp_unittest(
     LdExpTest.h
   DEPENDS
     libc.src.math.ldexpl
+    libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.limits
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.normal_float
@@ -1976,6 +2225,7 @@ add_fp_unittest(
     LdExpTest.h
   DEPENDS
     libc.src.math.ldexpf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.limits
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.normal_float
@@ -1991,7 +2241,25 @@ add_fp_unittest(
     LdExpTest.h
   DEPENDS
     libc.src.math.ldexpf128
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.CPP.limits
+    libc.src.__support.FPUtil.fp_bits
+    libc.src.__support.FPUtil.normal_float
+)
+
+add_fp_unittest(
+  ldexpbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    ldexpbf16_test.cpp
+  HDRS
+    LdExpTest.h
+  DEPENDS
+    libc.src.math.ldexpbf16
+    libc.src.__support.CPP.algorithm
     libc.src.__support.CPP.limits
+    libc.src.__support.FPUtil.bfloat16
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.normal_float
 )
@@ -2066,6 +2334,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.manipulation_functions
 )
 
+add_fp_unittest(
+  logbbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    logbbf16_test.cpp
+  HDRS
+    LogbTest.h
+  DEPENDS
+    libc.src.math.logbbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.manipulation_functions
+)
+
 add_fp_unittest(
   modf_test
   SUITE
@@ -2281,6 +2564,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fminbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminbf16_test.cpp
+  HDRS
+    FMinTest.h
+  DEPENDS
+    libc.src.math.fminbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   fmaxf_test
   SUITE
@@ -2351,6 +2649,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fmaxbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaxbf16_test.cpp
+  HDRS
+    FMaxTest.h
+  DEPENDS
+    libc.src.math.fmaxbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   fmaximuml_test
   SUITE
@@ -2436,15 +2749,30 @@ add_fp_unittest(
 )
 
 add_fp_unittest(
-  fmaximum_num_test
+  fmaximumbf16_test
   SUITE
     libc-math-smoke-tests
   SRCS
-    fmaximum_num_test.cpp
+    fmaximumbf16_test.cpp
   HDRS
-    FMaximumNumTest.h
+    FMaximumTest.h
   DEPENDS
-    libc.src.math.fmaximum_num
+    libc.src.math.fmaximumbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
+add_fp_unittest(
+  fmaximum_num_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_num_test.cpp
+  HDRS
+    FMaximumNumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_num
     libc.src.__support.CPP.algorithm
     libc.src.__support.FPUtil.fp_bits
 )
@@ -2491,6 +2819,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fmaximum_numbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_numbf16_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_numbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   fmaximum_magf_test
   SUITE
@@ -2566,6 +2909,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fmaximum_magbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_magbf16_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_magbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   fmaximum_mag_numf_test
   SUITE
@@ -2636,6 +2994,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fmaximum_mag_numbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fmaximum_mag_numbf16_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fmaximum_mag_numbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   fminimuml_test
   SUITE
@@ -2706,6 +3079,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fminimumbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimumbf16_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fminimumbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   fminimum_numf_test
   SUITE
@@ -2776,6 +3164,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fminimum_numbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_numf16_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fminimum_numf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   fminimum_magf_test
   SUITE
@@ -2846,6 +3249,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fminimum_magbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_magbf16_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fminimum_magbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   fminimum_mag_numf_test
   SUITE
@@ -2916,6 +3334,21 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  fminimum_mag_numbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    fminimum_mag_numbf16_test.cpp
+  HDRS
+    FMaximumTest.h
+  DEPENDS
+    libc.src.math.fminimum_mag_numbf16
+    libc.src.__support.CPP.algorithm
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   fmul_test
   SUITE
@@ -3247,6 +3680,22 @@ add_fp_unittest(
   UNIT_TEST_ONLY
 )
 
+add_fp_unittest(
+  nanbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nanbf16_test.cpp
+  DEPENDS
+    libc.hdr.signal_macros
+    libc.src.math.nanbf16
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fp_bits
+  # FIXME: The nan tests currently have death tests, which aren't supported for
+  # hermetic tests.
+  UNIT_TEST_ONLY
+)
+
 add_fp_unittest(
   nearbyint_test
   SUITE
@@ -3312,6 +3761,20 @@ add_fp_unittest(
     libc.src.math.nearbyintf128
 )
 
+add_fp_unittest(
+  nearbyintbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nearbyintbf16_test.cpp
+  HDRS
+    NearbyIntTest.h
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.math.nearbyintbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   nextafter_test
   SUITE
@@ -3392,6 +3855,23 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  nextafterbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nextafterbf16_test.cpp
+  HDRS
+    NextAfterTest.h
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.math.nextafterbf16
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 # FIXME: These tests are currently spurious for the GPU.
 if(NOT LIBC_TARGET_OS_IS_GPU)
   add_fp_unittest(
@@ -3459,6 +3939,23 @@ add_fp_unittest(
     libc.src.__support.FPUtil.fp_bits
 )
 
+add_fp_unittest(
+  nexttowardbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nexttowardbf16_test.cpp
+  HDRS
+    NextTowardTest.h
+  DEPENDS
+    libc.hdr.fenv_macros
+    libc.src.math.nexttowardbf16
+    libc.src.__support.CPP.bit
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.FPUtil.fenv_impl
+    libc.src.__support.FPUtil.fp_bits
+)
+
 add_fp_unittest(
   nextdown_test
   SUITE
@@ -3519,6 +4016,19 @@ add_fp_unittest(
     libc.src.math.nextdownf128
 )
 
+add_fp_unittest(
+  nextdownbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nextdownbf16_test.cpp
+  HDRS
+    NextDownTest.h
+  DEPENDS
+    libc.src.math.nextdownbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   nextup_test
   SUITE
@@ -3579,6 +4089,19 @@ add_fp_unittest(
     libc.src.math.nextupf128
 )
 
+add_fp_unittest(
+  nextupbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    nextupbf16_test.cpp
+  HDRS
+    NextUpTest.h
+  DEPENDS
+    libc.src.math.nextupbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 # TODO(lntue): The current implementation of fputil::general::fma<float> is only
 # correctly rounded for the default rounding mode round-to-nearest tie-to-even.
 add_fp_unittest(
@@ -4001,6 +4524,18 @@ add_fp_unittest(
     libc.src.math.asinhf16
 )
 
+add_fp_unittest(
+  asinpif16_test
+  NEED_MPFR
+  SUITE
+    libc-math-unittests
+  SRCS
+    asinpif16_test.cpp
+  DEPENDS
+    libc.src.math.asinpif16
+    libc.src.errno.errno
+)
+
 add_fp_unittest(
   acoshf_test
   SUITE
@@ -4533,6 +5068,19 @@ add_fp_unittest(
     libc.src.math.getpayloadf128
 )
 
+add_fp_unittest(
+  getpayloadbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    getpayloadbf16_test.cpp
+  HDRS
+    GetPayloadTest.h
+  DEPENDS
+    libc.src.math.getpayloadbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   setpayload_test
   SUITE
@@ -4593,6 +5141,19 @@ add_fp_unittest(
     libc.src.math.setpayloadf128
 )
 
+add_fp_unittest(
+  setpayloadbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    setpayloadbf16_test.cpp
+  HDRS
+    SetPayloadTest.h
+  DEPENDS
+    libc.src.math.setpayloadbf16
+    libc.src.__support.FPUtil.bfloat16
+)
+
 add_fp_unittest(
   setpayloadsig_test
   SUITE
@@ -4653,6 +5214,18 @@ add_fp_unittest(
     libc.src.math.setpayloadsigf128
 )
 
+add_fp_unittest(
+  setpayloadsigbf16_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    setpayloadsigbf16_test.cpp
+  HDRS
+    SetPayloadTest.h
+  DEPENDS
+    libc.src.math.setpayloadsigbf16
+    libc.src.__support.FPUtil.bfloat16
+)
 
 add_fp_unittest(
   f16add_test
@@ -5307,33 +5880,405 @@ add_fp_unittest(
 )
 
 add_fp_unittest(
-  add_same_type_test
+  bfloat16_add_test
   SUITE
     libc-math-smoke-tests
   SRCS
-    add_same_type_test.cpp
+    bfloat16_add_test.cpp
   HDRS
     AddTest.h
   DEPENDS
-    libc.hdr.errno_macros
-    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.bfloat16
     libc.src.__support.FPUtil.generic.add_sub
     libc.src.__support.macros.properties.os
     libc.src.__support.macros.properties.types
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
 )
 
 add_fp_unittest(
-  sub_same_type_test
+  bfloat16_div_test
   SUITE
     libc-math-smoke-tests
   SRCS
-    sub_same_type_test.cpp
+    bfloat16_div_test.cpp
   HDRS
-    SubTest.h
+    DivTest.h
+  DEPENDS
+    libc.src.__support.FPUtil.bfloat16
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+)
+
+add_fp_unittest(
+  bfloat16_mul_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bfloat16_mul_test.cpp
+  HDRS
+    MulTest.h
   DEPENDS
+    libc.src.__support.FPUtil.basic_operations
+    libc.src.__support.FPUtil.bfloat16
     libc.hdr.errno_macros
     libc.hdr.fenv_macros
+)
+
+add_fp_unittest(
+  bfloat16_sub_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bfloat16_sub_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.src.__support.FPUtil.bfloat16
     libc.src.__support.FPUtil.generic.add_sub
     libc.src.__support.macros.properties.os
     libc.src.__support.macros.properties.types
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+)
+
+add_fp_unittest(
+  add_same_type_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    add_same_type_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.properties.os
+    libc.src.__support.macros.properties.types
+)
+
+add_fp_unittest(
+  sub_same_type_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    sub_same_type_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.__support.FPUtil.generic.add_sub
+    libc.src.__support.macros.properties.os
+    libc.src.__support.macros.properties.types
+)
+
+add_fp_unittest(
+  bf16add_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16add_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16add
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.properties.os
+)
+
+add_fp_unittest(
+  bf16addf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16addf_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16addf
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.properties.os
+)
+
+add_fp_unittest(
+  bf16addl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16addl_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16addl
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.properties.os
+)
+
+add_fp_unittest(
+  bf16addf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16addf128_test.cpp
+  HDRS
+    AddTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16addf128
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.properties.os
+)
+
+add_fp_unittest(
+  bf16fma_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16fma_test.cpp
+  HDRS
+    FmaTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16fma
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16fmaf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16fmaf_test.cpp
+  HDRS
+    FmaTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16fmaf
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16fmal_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16fmal_test.cpp
+  HDRS
+    FmaTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16fmal
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16fmaf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16fmaf128_test.cpp
+  HDRS
+    FmaTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16fmaf128
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16div_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16div_test.cpp
+  HDRS
+    DivTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16div
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16divf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16divf_test.cpp
+  HDRS
+    DivTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16divf
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16divl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16divl_test.cpp
+  HDRS
+    DivTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16divl
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16divf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16divf128_test.cpp
+  HDRS
+    DivTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16divf128
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16mul_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16mul_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16mul
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16mulf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16mulf_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16mulf
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16mull_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16mull_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16mull
+    libc.src.__support.FPUtil.bfloat16
+)
+
+add_fp_unittest(
+  bf16mulf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16mulf128_test.cpp
+  HDRS
+    MulTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16mulf128
+    libc.src.__support.FPUtil.bfloat16
+)
+
+
+add_fp_unittest(
+  bf16sub_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16sub_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16sub
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.properties.os
+)
+
+add_fp_unittest(
+  bf16subf_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16subf_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16subf
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.properties.os
+)
+
+add_fp_unittest(
+  bf16subl_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16subl_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16subl
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.properties.os
+)
+
+add_fp_unittest(
+  bf16subf128_test
+  SUITE
+    libc-math-smoke-tests
+  SRCS
+    bf16subf128_test.cpp
+  HDRS
+    SubTest.h
+  DEPENDS
+    libc.hdr.errno_macros
+    libc.hdr.fenv_macros
+    libc.src.math.bf16subf128
+    libc.src.__support.FPUtil.bfloat16
+    libc.src.__support.macros.properties.os
 )
diff --git a/libc/test/src/math/smoke/CeilTest.h b/libc/test/src/math/smoke/CeilTest.h
index 7998eab62ec6b..1839db97887c4 100644
--- a/libc/test/src/math/smoke/CeilTest.h
+++ b/libc/test/src/math/smoke/CeilTest.h
@@ -59,10 +59,12 @@ class CeilTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(-10.0), func(T(-10.32)));
     EXPECT_FP_EQ(T(11.0), func(T(10.65)));
     EXPECT_FP_EQ(T(-10.0), func(T(-10.65)));
+    EXPECT_FP_EQ(T(50.0), func(T(49.62)));
+    EXPECT_FP_EQ(T(-50.0), func(T(-50.31)));
     EXPECT_FP_EQ(T(124.0), func(T(123.38)));
     EXPECT_FP_EQ(T(-123.0), func(T(-123.38)));
     EXPECT_FP_EQ(T(124.0), func(T(123.96)));
-    EXPECT_FP_EQ(T(-123.0), func(T(-123.96)));
+    EXPECT_FP_EQ(T(-123.0), func(T(-123.5)));
   }
 };
 
diff --git a/libc/test/src/math/smoke/FloorTest.h b/libc/test/src/math/smoke/FloorTest.h
index bc19e4f285915..cbcf276cddfff 100644
--- a/libc/test/src/math/smoke/FloorTest.h
+++ b/libc/test/src/math/smoke/FloorTest.h
@@ -59,10 +59,11 @@ class FloorTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(-11.0), func(T(-10.32)));
     EXPECT_FP_EQ(T(10.0), func(T(10.65)));
     EXPECT_FP_EQ(T(-11.0), func(T(-10.65)));
+    EXPECT_FP_EQ(T(50.0), func(T(50.31)));
+    EXPECT_FP_EQ(T(-50.0), func(T(-49.63)));
     EXPECT_FP_EQ(T(123.0), func(T(123.38)));
     EXPECT_FP_EQ(T(-124.0), func(T(-123.38)));
-    EXPECT_FP_EQ(T(123.0), func(T(123.96)));
-    EXPECT_FP_EQ(T(-124.0), func(T(-123.96)));
+    EXPECT_FP_EQ(T(-124.0), func(T(-123.5)));
   }
 };
 
diff --git a/libc/test/src/math/smoke/FromfpTest.h b/libc/test/src/math/smoke/FromfpTest.h
index 56205187d731c..708ec25fff438 100644
--- a/libc/test/src/math/smoke/FromfpTest.h
+++ b/libc/test/src/math/smoke/FromfpTest.h
@@ -85,10 +85,10 @@ class FromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(-10.0), func(T(-10.32), FP_INT_UPWARD, 5U));
     EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_UPWARD, 5U));
     EXPECT_FP_EQ(T(-10.0), func(T(-10.65), FP_INT_UPWARD, 5U));
-    EXPECT_FP_EQ(T(124.0), func(T(123.38), FP_INT_UPWARD, 8U));
-    EXPECT_FP_EQ(T(-123.0), func(T(-123.38), FP_INT_UPWARD, 8U));
-    EXPECT_FP_EQ(T(124.0), func(T(123.96), FP_INT_UPWARD, 8U));
-    EXPECT_FP_EQ(T(-123.0), func(T(-123.96), FP_INT_UPWARD, 8U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.25), FP_INT_UPWARD, 8U));
+    EXPECT_FP_EQ(T(-63.0), func(T(-63.25), FP_INT_UPWARD, 8U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.75), FP_INT_UPWARD, 8U));
+    EXPECT_FP_EQ(T(-63.0), func(T(-63.75), FP_INT_UPWARD, 8U));
   }
 
   void testFractionsUpwardOutsideRange(FromfpFunc func) {
@@ -139,10 +139,10 @@ class FromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(-11.0), func(T(-10.32), FP_INT_DOWNWARD, 5U));
     EXPECT_FP_EQ(T(10.0), func(T(10.65), FP_INT_DOWNWARD, 5U));
     EXPECT_FP_EQ(T(-11.0), func(T(-10.65), FP_INT_DOWNWARD, 5U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.38), FP_INT_DOWNWARD, 8U));
-    EXPECT_FP_EQ(T(-124.0), func(T(-123.38), FP_INT_DOWNWARD, 8U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.96), FP_INT_DOWNWARD, 8U));
-    EXPECT_FP_EQ(T(-124.0), func(T(-123.96), FP_INT_DOWNWARD, 8U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), FP_INT_DOWNWARD, 8U));
+    EXPECT_FP_EQ(T(-64.0), func(T(-63.25), FP_INT_DOWNWARD, 8U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.75), FP_INT_DOWNWARD, 8U));
+    EXPECT_FP_EQ(T(-64.0), func(T(-63.75), FP_INT_DOWNWARD, 8U));
   }
 
   void testFractionsDownwardOutsideRange(FromfpFunc func) {
@@ -193,10 +193,10 @@ class FromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(-10.0), func(T(-10.32), FP_INT_TOWARDZERO, 5U));
     EXPECT_FP_EQ(T(10.0), func(T(10.65), FP_INT_TOWARDZERO, 5U));
     EXPECT_FP_EQ(T(-10.0), func(T(-10.65), FP_INT_TOWARDZERO, 5U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.38), FP_INT_TOWARDZERO, 8U));
-    EXPECT_FP_EQ(T(-123.0), func(T(-123.38), FP_INT_TOWARDZERO, 8U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.96), FP_INT_TOWARDZERO, 8U));
-    EXPECT_FP_EQ(T(-123.0), func(T(-123.96), FP_INT_TOWARDZERO, 8U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), FP_INT_TOWARDZERO, 8U));
+    EXPECT_FP_EQ(T(-63.0), func(T(-63.25), FP_INT_TOWARDZERO, 8U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.75), FP_INT_TOWARDZERO, 8U));
+    EXPECT_FP_EQ(T(-63.0), func(T(-63.75), FP_INT_TOWARDZERO, 8U));
   }
 
   void testFractionsTowardZeroOutsideRange(FromfpFunc func) {
@@ -241,10 +241,10 @@ class FromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(-10.0), func(T(-10.32), FP_INT_TONEARESTFROMZERO, 5U));
     EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_TONEARESTFROMZERO, 5U));
     EXPECT_FP_EQ(T(-11.0), func(T(-10.65), FP_INT_TONEARESTFROMZERO, 5U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.38), FP_INT_TONEARESTFROMZERO, 8U));
-    EXPECT_FP_EQ(T(-123.0), func(T(-123.38), FP_INT_TONEARESTFROMZERO, 8U));
-    EXPECT_FP_EQ(T(124.0), func(T(123.96), FP_INT_TONEARESTFROMZERO, 8U));
-    EXPECT_FP_EQ(T(-124.0), func(T(-123.96), FP_INT_TONEARESTFROMZERO, 8U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), FP_INT_TONEARESTFROMZERO, 8U));
+    EXPECT_FP_EQ(T(-63.0), func(T(-63.25), FP_INT_TONEARESTFROMZERO, 8U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.75), FP_INT_TONEARESTFROMZERO, 8U));
+    EXPECT_FP_EQ(T(-64.0), func(T(-63.75), FP_INT_TONEARESTFROMZERO, 8U));
   }
 
   void testFractionsToNearestFromZeroOutsideRange(FromfpFunc func) {
@@ -297,10 +297,10 @@ class FromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(-10.0), func(T(-10.32), FP_INT_TONEAREST, 5U));
     EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_TONEAREST, 5U));
     EXPECT_FP_EQ(T(-11.0), func(T(-10.65), FP_INT_TONEAREST, 5U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.38), FP_INT_TONEAREST, 8U));
-    EXPECT_FP_EQ(T(-123.0), func(T(-123.38), FP_INT_TONEAREST, 8U));
-    EXPECT_FP_EQ(T(124.0), func(T(123.96), FP_INT_TONEAREST, 8U));
-    EXPECT_FP_EQ(T(-124.0), func(T(-123.96), FP_INT_TONEAREST, 8U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), FP_INT_TONEAREST, 8U));
+    EXPECT_FP_EQ(T(-63.0), func(T(-63.25), FP_INT_TONEAREST, 8U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.75), FP_INT_TONEAREST, 8U));
+    EXPECT_FP_EQ(T(-64.0), func(T(-63.75), FP_INT_TONEAREST, 8U));
 
     EXPECT_FP_EQ(T(2.0), func(T(2.3), FP_INT_TONEAREST, 3U));
     EXPECT_FP_EQ(T(-2.0), func(T(-2.3), FP_INT_TONEAREST, 2U));
@@ -391,14 +391,12 @@ class FromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(11.0), func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U));
     EXPECT_FP_EQ(T(-11.0),
                  func(T(-10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U));
-    EXPECT_FP_EQ(T(123.0),
-                 func(T(123.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U));
-    EXPECT_FP_EQ(T(-123.0),
-                 func(T(-123.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U));
-    EXPECT_FP_EQ(T(124.0),
-                 func(T(123.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U));
-    EXPECT_FP_EQ(T(-124.0),
-                 func(T(-123.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U));
+    EXPECT_FP_EQ(T(-63.0),
+                 func(T(-63.25), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U));
+    EXPECT_FP_EQ(T(-64.0),
+                 func(T(-63.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U));
 
     EXPECT_FP_EQ(T(2.0), func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 3U));
     EXPECT_FP_EQ(T(-2.0), func(T(-2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
diff --git a/libc/test/src/math/smoke/FromfpxTest.h b/libc/test/src/math/smoke/FromfpxTest.h
index a175a66e21b64..1930b22c80b11 100644
--- a/libc/test/src/math/smoke/FromfpxTest.h
+++ b/libc/test/src/math/smoke/FromfpxTest.h
@@ -101,13 +101,13 @@ class FromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
                                 FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(T(-10.0), func(T(-10.65), FP_INT_UPWARD, 5U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(124.0), func(T(123.38), FP_INT_UPWARD, 8U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(64.0), func(T(63.25), FP_INT_UPWARD, 8U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(-123.0), func(T(-123.38), FP_INT_UPWARD, 8U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-63.0), func(T(-63.25), FP_INT_UPWARD, 8U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(124.0), func(T(123.96), FP_INT_UPWARD, 8U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(64.0), func(T(63.75), FP_INT_UPWARD, 8U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(-123.0), func(T(-123.96), FP_INT_UPWARD, 8U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-63.0), func(T(-63.75), FP_INT_UPWARD, 8U),
                                 FE_INEXACT);
   }
 
@@ -175,14 +175,14 @@ class FromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
                                 FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(T(-11.0), func(T(-10.65), FP_INT_DOWNWARD, 5U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(123.0), func(T(123.38), FP_INT_DOWNWARD, 8U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.25), FP_INT_DOWNWARD, 8U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-124.0), func(T(-123.38), FP_INT_DOWNWARD, 8U), FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(123.0), func(T(123.96), FP_INT_DOWNWARD, 8U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-64.0), func(T(-63.25), FP_INT_DOWNWARD, 8U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.75), FP_INT_DOWNWARD, 8U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-64.0), func(T(-63.75), FP_INT_DOWNWARD, 8U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-124.0), func(T(-123.96), FP_INT_DOWNWARD, 8U), FE_INEXACT);
   }
 
   void testFractionsDownwardOutsideRange(FromfpxFunc func) {
@@ -249,14 +249,14 @@ class FromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
                                 FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
         T(-10.0), func(T(-10.65), FP_INT_TOWARDZERO, 5U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.25), FP_INT_TOWARDZERO, 8U),
+                                FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(123.0), func(T(123.38), FP_INT_TOWARDZERO, 8U), FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-123.0), func(T(-123.38), FP_INT_TOWARDZERO, 8U), FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(123.0), func(T(123.96), FP_INT_TOWARDZERO, 8U), FE_INEXACT);
+        T(-63.0), func(T(-63.25), FP_INT_TOWARDZERO, 8U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.75), FP_INT_TOWARDZERO, 8U),
+                                FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-123.0), func(T(-123.96), FP_INT_TOWARDZERO, 8U), FE_INEXACT);
+        T(-63.0), func(T(-63.75), FP_INT_TOWARDZERO, 8U), FE_INEXACT);
   }
 
   void testFractionsTowardZeroOutsideRange(FromfpxFunc func) {
@@ -318,13 +318,13 @@ class FromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ_WITH_EXCEPTION(
         T(-11.0), func(T(-10.65), FP_INT_TONEARESTFROMZERO, 5U), FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(123.0), func(T(123.38), FP_INT_TONEARESTFROMZERO, 8U), FE_INEXACT);
+        T(63.0), func(T(63.25), FP_INT_TONEARESTFROMZERO, 8U), FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-123.0), func(T(-123.38), FP_INT_TONEARESTFROMZERO, 8U), FE_INEXACT);
+        T(-63.0), func(T(-63.25), FP_INT_TONEARESTFROMZERO, 8U), FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(124.0), func(T(123.96), FP_INT_TONEARESTFROMZERO, 8U), FE_INEXACT);
+        T(64.0), func(T(63.75), FP_INT_TONEARESTFROMZERO, 8U), FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-124.0), func(T(-123.96), FP_INT_TONEARESTFROMZERO, 8U), FE_INEXACT);
+        T(-64.0), func(T(-63.75), FP_INT_TONEARESTFROMZERO, 8U), FE_INEXACT);
   }
 
   void testFractionsToNearestFromZeroOutsideRange(FromfpxFunc func) {
@@ -393,14 +393,14 @@ class FromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
                                 FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(T(-11.0), func(T(-10.65), FP_INT_TONEAREST, 5U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(123.0), func(T(123.38), FP_INT_TONEAREST, 8U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.25), FP_INT_TONEAREST, 8U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-123.0), func(T(-123.38), FP_INT_TONEAREST, 8U), FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(124.0), func(T(123.96), FP_INT_TONEAREST, 8U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-63.0), func(T(-63.25), FP_INT_TONEAREST, 8U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(64.0), func(T(63.75), FP_INT_TONEAREST, 8U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(-64.0), func(T(-63.75), FP_INT_TONEAREST, 8U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-124.0), func(T(-123.96), FP_INT_TONEAREST, 8U), FE_INEXACT);
 
     EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(2.3), FP_INT_TONEAREST, 3U),
                                 FE_INEXACT);
@@ -530,16 +530,16 @@ class FromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
         T(-11.0), func(T(-10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 5U),
         FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(123.0), func(T(123.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U),
+        T(63.0), func(T(63.25), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U),
         FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-123.0), func(T(-123.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U),
+        T(-63.0), func(T(-63.25), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U),
         FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(124.0), func(T(123.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U),
+        T(64.0), func(T(63.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U),
         FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(-124.0), func(T(-123.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U),
+        T(-64.0), func(T(-63.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 8U),
         FE_INEXACT);
 
     EXPECT_FP_EQ_WITH_EXCEPTION(
diff --git a/libc/test/src/math/smoke/GetPayloadTest.h b/libc/test/src/math/smoke/GetPayloadTest.h
index 1b1bf4f5d56ff..486ea42bf50c1 100644
--- a/libc/test/src/math/smoke/GetPayloadTest.h
+++ b/libc/test/src/math/smoke/GetPayloadTest.h
@@ -51,23 +51,50 @@ class GetPayloadTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(default_snan_payload, funcWrapper(func, sNaN));
     EXPECT_FP_EQ(default_snan_payload, funcWrapper(func, neg_sNaN));
 
-    T qnan_42 = FPBits::quiet_nan(Sign::POS, 0x42).get_val();
-    T neg_qnan_42 = FPBits::quiet_nan(Sign::NEG, 0x42).get_val();
-    T snan_42 = FPBits::signaling_nan(Sign::POS, 0x42).get_val();
-    T neg_snan_42 = FPBits::signaling_nan(Sign::NEG, 0x42).get_val();
-    EXPECT_FP_EQ(T(0x42.0p+0), funcWrapper(func, qnan_42));
-    EXPECT_FP_EQ(T(0x42.0p+0), funcWrapper(func, neg_qnan_42));
-    EXPECT_FP_EQ(T(0x42.0p+0), funcWrapper(func, snan_42));
-    EXPECT_FP_EQ(T(0x42.0p+0), funcWrapper(func, neg_snan_42));
-
-    T qnan_123 = FPBits::quiet_nan(Sign::POS, 0x123).get_val();
-    T neg_qnan_123 = FPBits::quiet_nan(Sign::NEG, 0x123).get_val();
-    T snan_123 = FPBits::signaling_nan(Sign::POS, 0x123).get_val();
-    T neg_snan_123 = FPBits::signaling_nan(Sign::NEG, 0x123).get_val();
-    EXPECT_FP_EQ(T(0x123.0p+0), funcWrapper(func, qnan_123));
-    EXPECT_FP_EQ(T(0x123.0p+0), funcWrapper(func, neg_qnan_123));
-    EXPECT_FP_EQ(T(0x123.0p+0), funcWrapper(func, snan_123));
-    EXPECT_FP_EQ(T(0x123.0p+0), funcWrapper(func, neg_snan_123));
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 6) {
+      // [S] [E..E] [QM..M] -> number of M bits should be at least 6
+      // 0x31 = 0b110001 = 6 bits
+      T qnan_31 = FPBits::quiet_nan(Sign::POS, 0x31).get_val();
+      T neg_qnan_31 = FPBits::quiet_nan(Sign::NEG, 0x31).get_val();
+      T snan_31 = FPBits::signaling_nan(Sign::POS, 0x31).get_val();
+      T neg_snan_31 = FPBits::signaling_nan(Sign::NEG, 0x31).get_val();
+      EXPECT_FP_EQ(T(0x31.0p+0), funcWrapper(func, qnan_31));
+      EXPECT_FP_EQ(T(0x31.0p+0), funcWrapper(func, neg_qnan_31));
+      EXPECT_FP_EQ(T(0x31.0p+0), funcWrapper(func, snan_31));
+      EXPECT_FP_EQ(T(0x31.0p+0), funcWrapper(func, neg_snan_31));
+
+      // 0x15 = 0b10101 = 5 bits
+      T qnan_15 = FPBits::quiet_nan(Sign::POS, 0x15).get_val();
+      T neg_qnan_15 = FPBits::quiet_nan(Sign::NEG, 0x15).get_val();
+      T snan_15 = FPBits::signaling_nan(Sign::POS, 0x15).get_val();
+      T neg_snan_15 = FPBits::signaling_nan(Sign::NEG, 0x15).get_val();
+      EXPECT_FP_EQ(T(0x15.0p+0), funcWrapper(func, qnan_15));
+      EXPECT_FP_EQ(T(0x15.0p+0), funcWrapper(func, neg_qnan_15));
+      EXPECT_FP_EQ(T(0x15.0p+0), funcWrapper(func, snan_15));
+      EXPECT_FP_EQ(T(0x15.0p+0), funcWrapper(func, neg_snan_15));
+    }
+
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 7) {
+      T qnan_42 = FPBits::quiet_nan(Sign::POS, 0x42).get_val();
+      T neg_qnan_42 = FPBits::quiet_nan(Sign::NEG, 0x42).get_val();
+      T snan_42 = FPBits::signaling_nan(Sign::POS, 0x42).get_val();
+      T neg_snan_42 = FPBits::signaling_nan(Sign::NEG, 0x42).get_val();
+      EXPECT_FP_EQ(T(0x42.0p+0), funcWrapper(func, qnan_42));
+      EXPECT_FP_EQ(T(0x42.0p+0), funcWrapper(func, neg_qnan_42));
+      EXPECT_FP_EQ(T(0x42.0p+0), funcWrapper(func, snan_42));
+      EXPECT_FP_EQ(T(0x42.0p+0), funcWrapper(func, neg_snan_42));
+    }
+
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 9) {
+      T qnan_123 = FPBits::quiet_nan(Sign::POS, 0x123).get_val();
+      T neg_qnan_123 = FPBits::quiet_nan(Sign::NEG, 0x123).get_val();
+      T snan_123 = FPBits::signaling_nan(Sign::POS, 0x123).get_val();
+      T neg_snan_123 = FPBits::signaling_nan(Sign::NEG, 0x123).get_val();
+      EXPECT_FP_EQ(T(0x123.0p+0), funcWrapper(func, qnan_123));
+      EXPECT_FP_EQ(T(0x123.0p+0), funcWrapper(func, neg_qnan_123));
+      EXPECT_FP_EQ(T(0x123.0p+0), funcWrapper(func, snan_123));
+      EXPECT_FP_EQ(T(0x123.0p+0), funcWrapper(func, neg_snan_123));
+    }
   }
 };
 
diff --git a/libc/test/src/math/smoke/LdExpTest.h b/libc/test/src/math/smoke/LdExpTest.h
index 8de70ad161ad6..d005f053c0c91 100644
--- a/libc/test/src/math/smoke/LdExpTest.h
+++ b/libc/test/src/math/smoke/LdExpTest.h
@@ -10,7 +10,8 @@
 #define LLVM_LIBC_TEST_SRC_MATH_LDEXPTEST_H
 
 #include "hdr/stdint_proxy.h"
-#include "src/__support/CPP/limits.h" // INT_MAX
+#include "src/__support/CPP/algorithm.h" // cpp::min
+#include "src/__support/CPP/limits.h"    // INT_MAX
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/NormalFloat.h"
 #include "test/UnitTest/FEnvSafeTest.h"
@@ -135,8 +136,8 @@ class LdExpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     // Normal which trigger mantissa overflow.
     T x = NormalFloat(Sign::POS, -FPBits::EXP_BIAS + 1,
                       StorageType(2) * NormalFloat::ONE - StorageType(1));
-    ASSERT_FP_EQ(func(x, -1), x / 2);
-    ASSERT_FP_EQ(func(-x, -1), -x / 2);
+    ASSERT_FP_EQ(func(x, -1), T(x / 2));
+    ASSERT_FP_EQ(func(-x, -1), -T(x / 2));
 
     // Start with a normal number high exponent but pass a very low number for
     // exp. The result should be a subnormal number.
@@ -154,7 +155,9 @@ class LdExpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
     // Start with a subnormal number but pass a very high number for exponent.
     // The result should not be infinity.
-    x = NormalFloat(Sign::POS, -FPBits::EXP_BIAS + 1, NormalFloat::ONE >> 10);
+    x = NormalFloat(Sign::POS, -FPBits::EXP_BIAS + 1,
+                    NormalFloat::ONE >>
+                        LIBC_NAMESPACE::cpp::min(FPBits::FRACTION_LEN, 10));
     exp = FPBits::MAX_BIASED_EXPONENT + 5;
     ASSERT_FALSE(FPBits(func(x, exp)).is_inf());
     // But if the exp is large enough to oversome than the normalization shift,
diff --git a/libc/test/src/math/smoke/MulTest.h b/libc/test/src/math/smoke/MulTest.h
index cf7f41abf5500..5020067d433a7 100644
--- a/libc/test/src/math/smoke/MulTest.h
+++ b/libc/test/src/math/smoke/MulTest.h
@@ -53,10 +53,12 @@ class MulTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ_ALL_ROUNDING(neg_zero, func(in.zero, in.neg_zero));
     EXPECT_FP_EQ_ALL_ROUNDING(neg_zero, func(in.neg_zero, in.zero));
 
-    EXPECT_FP_EQ_ALL_ROUNDING(OutType(1.0), func(1.0, 1.0));
-    EXPECT_FP_EQ_ALL_ROUNDING(OutType(15.0), func(3.0, 5.0));
-    EXPECT_FP_EQ_ALL_ROUNDING(OutType(0x1.0p-13), func(0x1.0p+1, 0x1.0p-14));
-    EXPECT_FP_EQ_ALL_ROUNDING(OutType(0x1.0p-10), func(0x1.0p+2, 0x1.0p-12));
+    EXPECT_FP_EQ_ALL_ROUNDING(OutType(1.0), func(InType(1.0), InType(1.0)));
+    EXPECT_FP_EQ_ALL_ROUNDING(OutType(15.0), func(InType(3.0), InType(5.0)));
+    EXPECT_FP_EQ_ALL_ROUNDING(OutType(0x1.0p-13),
+                              func(InType(0x1.0p+1), InType(0x1.0p-14)));
+    EXPECT_FP_EQ_ALL_ROUNDING(OutType(0x1.0p-10),
+                              func(InType(0x1.0p+2), InType(0x1.0p-12)));
   }
 
   void test_invalid_operations(MulFunc func) {
diff --git a/libc/test/src/math/smoke/NextAfterTest.h b/libc/test/src/math/smoke/NextAfterTest.h
index be27c9f188a72..b7e59f7cd8cfd 100644
--- a/libc/test/src/math/smoke/NextAfterTest.h
+++ b/libc/test/src/math/smoke/NextAfterTest.h
@@ -12,6 +12,7 @@
 #include "src/__support/CPP/bit.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/sign.h"
 #include "test/UnitTest/FEnvSafeTest.h"
 #include "test/UnitTest/FPMatcher.h"
 #include "test/UnitTest/Test.h"
@@ -42,6 +43,8 @@ class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
   const T neg_inf = FPBits::inf(Sign::NEG).get_val();
   const T zero = FPBits::zero(Sign::POS).get_val();
   const T neg_zero = FPBits::zero(Sign::NEG).get_val();
+  const T one = FPBits::one(Sign::POS).get_val();
+  const T neg_one = FPBits::one(Sign::NEG).get_val();
   const T nan = FPBits::quiet_nan().get_val();
 
   static constexpr StorageType min_subnormal =
@@ -55,8 +58,8 @@ class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
   typedef T (*NextAfterFunc)(T, T);
 
   void testNaN(NextAfterFunc func) {
-    ASSERT_FP_EQ(func(nan, 0), nan);
-    ASSERT_FP_EQ(func(0, nan), nan);
+    ASSERT_FP_EQ(func(nan, zero), nan);
+    ASSERT_FP_EQ(func(zero, nan), nan);
   }
 
   void testBoundaries(NextAfterFunc func) {
@@ -65,68 +68,68 @@ class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
     // 'from' is zero|neg_zero.
     T x = zero;
-    T result = func(x, T(1));
+    T result = func(x, one);
     StorageType expected_bits = 1;
     T expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
 
-    result = func(x, T(-1));
+    result = func(x, neg_one);
     expected_bits = FPBits::SIGN_MASK + 1;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
 
     x = neg_zero;
-    result = func(x, 1);
+    result = func(x, one);
     expected_bits = 1;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
 
-    result = func(x, -1);
+    result = func(x, neg_one);
     expected_bits = FPBits::SIGN_MASK + 1;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
 
     // 'from' is max subnormal value.
     x = LIBC_NAMESPACE::cpp::bit_cast<T>(max_subnormal);
-    result = func(x, 1);
+    result = func(x, one);
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(min_normal);
     ASSERT_FP_EQ(result, expected);
 
-    result = func(x, 0);
+    result = func(x, zero);
     expected_bits = max_subnormal - 1;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
 
     x = -x;
 
-    result = func(x, -1);
+    result = func(x, neg_one);
     expected_bits = FPBits::SIGN_MASK + min_normal;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ(result, expected);
 
-    result = func(x, 0);
+    result = func(x, zero);
     expected_bits = FPBits::SIGN_MASK + max_subnormal - 1;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
 
     // 'from' is min subnormal value.
     x = LIBC_NAMESPACE::cpp::bit_cast<T>(min_subnormal);
-    result = func(x, 1);
+    result = func(x, one);
     expected_bits = min_subnormal + 1;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
-    ASSERT_FP_EQ_WITH_UNDERFLOW(func(x, 0), zero);
+    ASSERT_FP_EQ_WITH_UNDERFLOW(func(x, zero), zero);
 
     x = -x;
-    result = func(x, -1);
+    result = func(x, neg_one);
     expected_bits = FPBits::SIGN_MASK + min_subnormal + 1;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
-    ASSERT_FP_EQ_WITH_UNDERFLOW(func(x, 0), T(-0.0));
+    ASSERT_FP_EQ_WITH_UNDERFLOW(func(x, zero), neg_zero);
 
     // 'from' is min normal.
     x = LIBC_NAMESPACE::cpp::bit_cast<T>(min_normal);
-    result = func(x, 0);
+    result = func(x, zero);
     expected_bits = max_subnormal;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
@@ -137,7 +140,7 @@ class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     ASSERT_FP_EQ(result, expected);
 
     x = -x;
-    result = func(x, 0);
+    result = func(x, zero);
     expected_bits = FPBits::SIGN_MASK + max_subnormal;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ_WITH_UNDERFLOW(result, expected);
@@ -157,14 +160,14 @@ class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
     // 'from' is infinity.
     x = inf;
-    result = func(x, 0);
+    result = func(x, zero);
     expected_bits = max_normal;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ(result, expected);
     ASSERT_FP_EQ(func(x, inf), inf);
 
     x = neg_inf;
-    result = func(x, 0);
+    result = func(x, zero);
     expected_bits = FPBits::SIGN_MASK + max_normal;
     expected = LIBC_NAMESPACE::cpp::bit_cast<T>(expected_bits);
     ASSERT_FP_EQ(result, expected);
@@ -172,7 +175,7 @@ class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
     // 'from' is a power of 2.
     x = T(32.0);
-    result = func(x, 0);
+    result = func(x, zero);
     FPBits x_bits = FPBits(x);
     FPBits result_bits = FPBits(result);
     ASSERT_EQ(result_bits.get_biased_exponent(),
@@ -187,7 +190,7 @@ class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
 
     x = -x;
 
-    result = func(x, 0);
+    result = func(x, zero);
     result_bits = FPBits(result);
     ASSERT_EQ(result_bits.get_biased_exponent(),
               uint16_t(x_bits.get_biased_exponent() - 1));
diff --git a/libc/test/src/math/smoke/NextTowardTest.h b/libc/test/src/math/smoke/NextTowardTest.h
index d2f352cdcbdf7..43e71c6a2d8f6 100644
--- a/libc/test/src/math/smoke/NextTowardTest.h
+++ b/libc/test/src/math/smoke/NextTowardTest.h
@@ -62,8 +62,8 @@ class NextTowardTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
   typedef T (*NextTowardFunc)(T, long double);
 
   void testNaN(NextTowardFunc func) {
-    ASSERT_FP_EQ(func(nan, 0), nan);
-    ASSERT_FP_EQ(func(0, to_nan), nan);
+    ASSERT_FP_EQ(func(nan, to_zero), nan);
+    ASSERT_FP_EQ(func(zero, to_nan), nan);
   }
 
   void testBoundaries(NextTowardFunc func) {
diff --git a/libc/test/src/math/smoke/RoundTest.h b/libc/test/src/math/smoke/RoundTest.h
index beb70008c330b..72889dada1378 100644
--- a/libc/test/src/math/smoke/RoundTest.h
+++ b/libc/test/src/math/smoke/RoundTest.h
@@ -59,8 +59,10 @@ class RoundTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(-10.0), func(T(-10.32)));
     EXPECT_FP_EQ(T(11.0), func(T(10.65)));
     EXPECT_FP_EQ(T(-11.0), func(T(-10.65)));
-    EXPECT_FP_EQ(T(123.0), func(T(123.38)));
-    EXPECT_FP_EQ(T(-123.0), func(T(-123.38)));
+    EXPECT_FP_EQ(T(50.0), func(T(49.63)));
+    EXPECT_FP_EQ(T(-50.0), func(T(-50.31)));
+    EXPECT_FP_EQ(T(124.0), func(T(123.5)));
+    EXPECT_FP_EQ(T(-124.0), func(T(-123.5)));
     EXPECT_FP_EQ(T(124.0), func(T(123.96)));
     EXPECT_FP_EQ(T(-124.0), func(T(-123.96)));
   }
diff --git a/libc/test/src/math/smoke/RoundToIntegerTest.h b/libc/test/src/math/smoke/RoundToIntegerTest.h
index 2b460aef6ef32..6866e9109ca0a 100644
--- a/libc/test/src/math/smoke/RoundToIntegerTest.h
+++ b/libc/test/src/math/smoke/RoundToIntegerTest.h
@@ -97,8 +97,8 @@ class RoundToIntegerTestTemplate
     test_one_input(func, F(-1.0), I(-1), false);
     test_one_input(func, F(10.0), I(10), false);
     test_one_input(func, F(-10.0), I(-10), false);
-    test_one_input(func, F(1234.0), I(1234), false);
-    test_one_input(func, F(-1234.0), I(-1234), false);
+    test_one_input(func, F(1232.0), I(1232), false);
+    test_one_input(func, F(-1232.0), I(-1232), false);
   }
 
   void testRoundNumbers(RoundToIntegerFunc func) {
@@ -124,7 +124,7 @@ class RoundToIntegerTestTemplate
         continue;
       // All subnormal numbers should round to zero.
       if (TestModes) {
-        if (x > 0) {
+        if (x > zero) {
           LIBC_NAMESPACE::fputil::set_round(FE_UPWARD);
           test_one_input(func, x, I(1), false);
           LIBC_NAMESPACE::fputil::set_round(FE_DOWNWARD);
diff --git a/libc/test/src/math/smoke/SetPayloadSigTest.h b/libc/test/src/math/smoke/SetPayloadSigTest.h
index f480479618a08..7b26c98b7ee35 100644
--- a/libc/test/src/math/smoke/SetPayloadSigTest.h
+++ b/libc/test/src/math/smoke/SetPayloadSigTest.h
@@ -54,15 +54,31 @@ class SetPayloadSigTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_EQ(FPBits::signaling_nan(Sign::POS, 1).uintval(),
               FPBits(res).uintval());
 
-    EXPECT_EQ(0, func(&res, T(0x42.0p+0)));
-    EXPECT_TRUE(FPBits(res).is_signaling_nan());
-    EXPECT_EQ(FPBits::signaling_nan(Sign::POS, 0x42).uintval(),
-              FPBits(res).uintval());
-
-    EXPECT_EQ(0, func(&res, T(0x123.0p+0)));
-    EXPECT_TRUE(FPBits(res).is_signaling_nan());
-    EXPECT_EQ(FPBits::signaling_nan(Sign::POS, 0x123).uintval(),
-              FPBits(res).uintval());
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 6) {
+      EXPECT_EQ(0, func(&res, T(0x31.0p+0)));
+      EXPECT_TRUE(FPBits(res).is_signaling_nan());
+      EXPECT_EQ(FPBits::signaling_nan(Sign::POS, 0x31).uintval(),
+                FPBits(res).uintval());
+
+      EXPECT_EQ(0, func(&res, T(0x15.0p+0)));
+      EXPECT_TRUE(FPBits(res).is_signaling_nan());
+      EXPECT_EQ(FPBits::signaling_nan(Sign::POS, 0x15).uintval(),
+                FPBits(res).uintval());
+    }
+
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 7) {
+      EXPECT_EQ(0, func(&res, T(0x42.0p+0)));
+      EXPECT_TRUE(FPBits(res).is_signaling_nan());
+      EXPECT_EQ(FPBits::signaling_nan(Sign::POS, 0x42).uintval(),
+                FPBits(res).uintval());
+    }
+
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 9) {
+      EXPECT_EQ(0, func(&res, T(0x123.0p+0)));
+      EXPECT_TRUE(FPBits(res).is_signaling_nan());
+      EXPECT_EQ(FPBits::signaling_nan(Sign::POS, 0x123).uintval(),
+                FPBits(res).uintval());
+    }
 
     FPBits nan_payload_bits = FPBits::one();
     nan_payload_bits.set_biased_exponent(FPBits::FRACTION_LEN - 2 +
diff --git a/libc/test/src/math/smoke/SetPayloadTest.h b/libc/test/src/math/smoke/SetPayloadTest.h
index 9ede5678fef1b..b86b3250f03a8 100644
--- a/libc/test/src/math/smoke/SetPayloadTest.h
+++ b/libc/test/src/math/smoke/SetPayloadTest.h
@@ -54,15 +54,33 @@ class SetPayloadTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_TRUE(FPBits(res).is_quiet_nan());
     EXPECT_EQ(FPBits::quiet_nan(Sign::POS, 1).uintval(), FPBits(res).uintval());
 
-    EXPECT_EQ(0, func(&res, T(0x42.0p+0)));
-    EXPECT_TRUE(FPBits(res).is_quiet_nan());
-    EXPECT_EQ(FPBits::quiet_nan(Sign::POS, 0x42).uintval(),
-              FPBits(res).uintval());
-
-    EXPECT_EQ(0, func(&res, T(0x123.0p+0)));
-    EXPECT_TRUE(FPBits(res).is_quiet_nan());
-    EXPECT_EQ(FPBits::quiet_nan(Sign::POS, 0x123).uintval(),
-              FPBits(res).uintval());
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 5) {
+      EXPECT_EQ(0, func(&res, T(0x15.0p+0)));
+      EXPECT_TRUE(FPBits(res).is_quiet_nan());
+      EXPECT_EQ(FPBits::quiet_nan(Sign::POS, 0x15).uintval(),
+                FPBits(res).uintval());
+    }
+
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 6) {
+      EXPECT_EQ(0, func(&res, T(0x31.0p+0)));
+      EXPECT_TRUE(FPBits(res).is_quiet_nan());
+      EXPECT_EQ(FPBits::quiet_nan(Sign::POS, 0x31).uintval(),
+                FPBits(res).uintval());
+    }
+
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 7) {
+      EXPECT_EQ(0, func(&res, T(0x42.0p+0)));
+      EXPECT_TRUE(FPBits(res).is_quiet_nan());
+      EXPECT_EQ(FPBits::quiet_nan(Sign::POS, 0x42).uintval(),
+                FPBits(res).uintval());
+    }
+
+    if constexpr (FPBits::FRACTION_LEN - 1 >= 9) {
+      EXPECT_EQ(0, func(&res, T(0x123.0p+0)));
+      EXPECT_TRUE(FPBits(res).is_quiet_nan());
+      EXPECT_EQ(FPBits::quiet_nan(Sign::POS, 0x123).uintval(),
+                FPBits(res).uintval());
+    }
 
     // The following code is creating a NaN payload manually to prevent a
     // conversion from BigInt to float128.
diff --git a/libc/test/src/math/smoke/TruncTest.h b/libc/test/src/math/smoke/TruncTest.h
index 49688e81707a1..e088f29db2cea 100644
--- a/libc/test/src/math/smoke/TruncTest.h
+++ b/libc/test/src/math/smoke/TruncTest.h
@@ -61,8 +61,8 @@ class TruncTest : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(-10.0), func(T(-10.65)));
     EXPECT_FP_EQ(T(123.0), func(T(123.38)));
     EXPECT_FP_EQ(T(-123.0), func(T(-123.38)));
-    EXPECT_FP_EQ(T(123.0), func(T(123.96)));
-    EXPECT_FP_EQ(T(-123.0), func(T(-123.96)));
+    EXPECT_FP_EQ(T(123.0), func(T(123.5)));
+    EXPECT_FP_EQ(T(-123.0), func(T(-123.5)));
   }
 };
 
diff --git a/libc/test/src/math/smoke/UfromfpTest.h b/libc/test/src/math/smoke/UfromfpTest.h
index 84c9f805193ed..16785dedbe17e 100644
--- a/libc/test/src/math/smoke/UfromfpTest.h
+++ b/libc/test/src/math/smoke/UfromfpTest.h
@@ -76,8 +76,8 @@ class UfromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(2.0), func(T(1.75), FP_INT_UPWARD, 2U));
     EXPECT_FP_EQ(T(11.0), func(T(10.32), FP_INT_UPWARD, 4U));
     EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_UPWARD, 4U));
-    EXPECT_FP_EQ(T(124.0), func(T(123.38), FP_INT_UPWARD, 7U));
-    EXPECT_FP_EQ(T(124.0), func(T(123.96), FP_INT_UPWARD, 7U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.25), FP_INT_UPWARD, 7U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.75), FP_INT_UPWARD, 7U));
   }
 
   void testFractionsUpwardOutsideRange(UfromfpFunc func) {
@@ -120,8 +120,8 @@ class UfromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(1.0), func(T(1.75), FP_INT_DOWNWARD, 1U));
     EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_DOWNWARD, 4U));
     EXPECT_FP_EQ(T(10.0), func(T(10.65), FP_INT_DOWNWARD, 4U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.38), FP_INT_DOWNWARD, 7U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.96), FP_INT_DOWNWARD, 7U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), FP_INT_DOWNWARD, 7U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.75), FP_INT_DOWNWARD, 7U));
   }
 
   void testFractionsDownwardOutsideRange(UfromfpFunc func) {
@@ -167,8 +167,8 @@ class UfromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(1.0), func(T(1.75), FP_INT_TOWARDZERO, 1U));
     EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_TOWARDZERO, 4U));
     EXPECT_FP_EQ(T(10.0), func(T(10.65), FP_INT_TOWARDZERO, 4U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.38), FP_INT_TOWARDZERO, 7U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.96), FP_INT_TOWARDZERO, 7U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), FP_INT_TOWARDZERO, 7U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.75), FP_INT_TOWARDZERO, 7U));
   }
 
   void testFractionsTowardZeroOutsideRange(UfromfpFunc func) {
@@ -206,8 +206,8 @@ class UfromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(2.0), func(T(1.75), FP_INT_TONEARESTFROMZERO, 2U));
     EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_TONEARESTFROMZERO, 4U));
     EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_TONEARESTFROMZERO, 4U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.38), FP_INT_TONEARESTFROMZERO, 7U));
-    EXPECT_FP_EQ(T(124.0), func(T(123.96), FP_INT_TONEARESTFROMZERO, 7U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), FP_INT_TONEARESTFROMZERO, 7U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.75), FP_INT_TONEARESTFROMZERO, 7U));
   }
 
   void testFractionsToNearestFromZeroOutsideRange(UfromfpFunc func) {
@@ -254,8 +254,8 @@ class UfromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(2.0), func(T(1.75), FP_INT_TONEAREST, 2U));
     EXPECT_FP_EQ(T(10.0), func(T(10.32), FP_INT_TONEAREST, 4U));
     EXPECT_FP_EQ(T(11.0), func(T(10.65), FP_INT_TONEAREST, 4U));
-    EXPECT_FP_EQ(T(123.0), func(T(123.38), FP_INT_TONEAREST, 7U));
-    EXPECT_FP_EQ(T(124.0), func(T(123.96), FP_INT_TONEAREST, 7U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), FP_INT_TONEAREST, 7U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.75), FP_INT_TONEAREST, 7U));
 
     EXPECT_FP_EQ(T(2.0), func(T(2.3), FP_INT_TONEAREST, 2U));
     EXPECT_FP_EQ(T(2.0), func(T(2.5), FP_INT_TONEAREST, 2U));
@@ -332,10 +332,8 @@ class UfromfpTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ(T(2.0), func(T(1.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
     EXPECT_FP_EQ(T(10.0), func(T(10.32), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
     EXPECT_FP_EQ(T(11.0), func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U));
-    EXPECT_FP_EQ(T(123.0),
-                 func(T(123.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 7U));
-    EXPECT_FP_EQ(T(124.0),
-                 func(T(123.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 7U));
+    EXPECT_FP_EQ(T(63.0), func(T(63.25), UNKNOWN_MATH_ROUNDING_DIRECTION, 7U));
+    EXPECT_FP_EQ(T(64.0), func(T(63.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 7U));
 
     EXPECT_FP_EQ(T(2.0), func(T(2.3), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
     EXPECT_FP_EQ(T(2.0), func(T(2.5), UNKNOWN_MATH_ROUNDING_DIRECTION, 2U));
diff --git a/libc/test/src/math/smoke/UfromfpxTest.h b/libc/test/src/math/smoke/UfromfpxTest.h
index 5916492d79187..1d4d3a3a12dbc 100644
--- a/libc/test/src/math/smoke/UfromfpxTest.h
+++ b/libc/test/src/math/smoke/UfromfpxTest.h
@@ -87,9 +87,9 @@ class UfromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
                                 FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(T(11.0), func(T(10.65), FP_INT_UPWARD, 4U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(124.0), func(T(123.38), FP_INT_UPWARD, 7U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(64.0), func(T(63.25), FP_INT_UPWARD, 7U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(124.0), func(T(123.96), FP_INT_UPWARD, 7U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(64.0), func(T(63.75), FP_INT_UPWARD, 7U),
                                 FE_INEXACT);
   }
 
@@ -141,9 +141,9 @@ class UfromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
                                 FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.65), FP_INT_DOWNWARD, 4U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(123.0), func(T(123.38), FP_INT_DOWNWARD, 7U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.25), FP_INT_DOWNWARD, 7U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(123.0), func(T(123.96), FP_INT_DOWNWARD, 7U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.75), FP_INT_DOWNWARD, 7U),
                                 FE_INEXACT);
   }
 
@@ -201,10 +201,10 @@ class UfromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
                                 FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(T(10.0), func(T(10.65), FP_INT_TOWARDZERO, 4U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(123.0), func(T(123.38), FP_INT_TOWARDZERO, 7U), FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(123.0), func(T(123.96), FP_INT_TOWARDZERO, 7U), FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.25), FP_INT_TOWARDZERO, 7U),
+                                FE_INEXACT);
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.75), FP_INT_TOWARDZERO, 7U),
+                                FE_INEXACT);
   }
 
   void testFractionsTowardZeroOutsideRange(UfromfpxFunc func) {
@@ -252,9 +252,9 @@ class UfromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
     EXPECT_FP_EQ_WITH_EXCEPTION(
         T(11.0), func(T(10.65), FP_INT_TONEARESTFROMZERO, 4U), FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(123.0), func(T(123.38), FP_INT_TONEARESTFROMZERO, 7U), FE_INEXACT);
+        T(63.0), func(T(63.25), FP_INT_TONEARESTFROMZERO, 7U), FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(124.0), func(T(123.96), FP_INT_TONEARESTFROMZERO, 7U), FE_INEXACT);
+        T(64.0), func(T(63.75), FP_INT_TONEARESTFROMZERO, 7U), FE_INEXACT);
   }
 
   void testFractionsToNearestFromZeroOutsideRange(UfromfpxFunc func) {
@@ -311,9 +311,9 @@ class UfromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
                                 FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(T(11.0), func(T(10.65), FP_INT_TONEAREST, 4U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(123.0), func(T(123.38), FP_INT_TONEAREST, 7U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(63.0), func(T(63.25), FP_INT_TONEAREST, 7U),
                                 FE_INEXACT);
-    EXPECT_FP_EQ_WITH_EXCEPTION(T(124.0), func(T(123.96), FP_INT_TONEAREST, 7U),
+    EXPECT_FP_EQ_WITH_EXCEPTION(T(64.0), func(T(63.75), FP_INT_TONEAREST, 7U),
                                 FE_INEXACT);
 
     EXPECT_FP_EQ_WITH_EXCEPTION(T(2.0), func(T(2.3), FP_INT_TONEAREST, 2U),
@@ -414,10 +414,10 @@ class UfromfpxTestTemplate : public LIBC_NAMESPACE::testing::FEnvSafeTest {
         T(11.0), func(T(10.65), UNKNOWN_MATH_ROUNDING_DIRECTION, 4U),
         FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(123.0), func(T(123.38), UNKNOWN_MATH_ROUNDING_DIRECTION, 7U),
+        T(63.0), func(T(63.25), UNKNOWN_MATH_ROUNDING_DIRECTION, 7U),
         FE_INEXACT);
     EXPECT_FP_EQ_WITH_EXCEPTION(
-        T(124.0), func(T(123.96), UNKNOWN_MATH_ROUNDING_DIRECTION, 7U),
+        T(64.0), func(T(63.75), UNKNOWN_MATH_ROUNDING_DIRECTION, 7U),
         FE_INEXACT);
 
     EXPECT_FP_EQ_WITH_EXCEPTION(
diff --git a/libc/test/src/math/smoke/asinpif16_test.cpp b/libc/test/src/math/smoke/asinpif16_test.cpp
new file mode 100644
index 0000000000000..5303eed8f5dae
--- /dev/null
+++ b/libc/test/src/math/smoke/asinpif16_test.cpp
@@ -0,0 +1,86 @@
+//===-- Unittests for asinpif16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/libc_errno.h"
+#include "src/math/asinpif16.h"
+#include "test/UnitTest/FPMatcher.h"
+
+using LlvmLibcAsinpif16Test = LIBC_NAMESPACE::testing::FPTest<float16>;
+
+TEST_F(LlvmLibcAsinpif16Test, SpecialNumbers) {
+  // zero
+  EXPECT_FP_EQ(zero, LIBC_NAMESPACE::asinpif16(zero));
+
+  // +/-1
+  EXPECT_FP_EQ(0.5f16, LIBC_NAMESPACE::asinpif16(1.0));
+  EXPECT_FP_EQ(-0.5f16, LIBC_NAMESPACE::asinpif16(-1.0));
+
+  // NaN inputs
+  EXPECT_FP_EQ(FPBits::quiet_nan().get_val(),
+               LIBC_NAMESPACE::asinpif16(FPBits::quiet_nan().get_val()));
+
+  EXPECT_FP_EQ(FPBits::quiet_nan().get_val(),
+               LIBC_NAMESPACE::asinpif16(FPBits::signaling_nan().get_val()));
+
+  // infinity inputs -> should return NaN
+  libc_errno = 0;
+  EXPECT_FP_EQ(FPBits::quiet_nan().get_val(), LIBC_NAMESPACE::asinpif16(inf));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  libc_errno = 0;
+  EXPECT_FP_EQ(FPBits::quiet_nan().get_val(),
+               LIBC_NAMESPACE::asinpif16(neg_inf));
+  EXPECT_MATH_ERRNO(EDOM);
+}
+
+TEST_F(LlvmLibcAsinpif16Test, OutOfRange) {
+  // Test values > 1
+  libc_errno = 0;
+  EXPECT_FP_EQ(FPBits::quiet_nan().get_val(),
+               LIBC_NAMESPACE::asinpif16(1.5f16));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  libc_errno = 0;
+  EXPECT_FP_EQ(FPBits::quiet_nan().get_val(),
+               LIBC_NAMESPACE::asinpif16(2.0f16));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  // Test values < -1
+  libc_errno = 0;
+  EXPECT_FP_EQ(FPBits::quiet_nan().get_val(),
+               LIBC_NAMESPACE::asinpif16(-1.5f16));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  libc_errno = 0;
+  EXPECT_FP_EQ(FPBits::quiet_nan().get_val(),
+               LIBC_NAMESPACE::asinpif16(-2.0f16));
+  EXPECT_MATH_ERRNO(EDOM);
+
+  // Test maximum normal value (should be > 1 for float16)
+  libc_errno = 0;
+  EXPECT_FP_EQ(FPBits::quiet_nan().get_val(),
+               LIBC_NAMESPACE::asinpif16(FPBits::max_normal().get_val()));
+  EXPECT_MATH_ERRNO(EDOM);
+}
+
+TEST_F(LlvmLibcAsinpif16Test, SymmetryProperty) {
+  // Test that asinpi(-x) = -asinpi(x)
+  constexpr float16 TEST_VALS[] = {0.1f16, 0.25f16, 0.5f16, 0.75f16,
+                                   0.9f16, 0.99f16, 1.0f16};
+
+  for (float16 x : TEST_VALS) {
+    FPBits neg_x_bits(x);
+    neg_x_bits.set_sign(Sign::NEG);
+    float16 neg_x = neg_x_bits.get_val();
+
+    float16 pos_result = LIBC_NAMESPACE::asinpif16(x);
+    float16 neg_result = LIBC_NAMESPACE::asinpif16(neg_x);
+
+    EXPECT_FP_EQ(pos_result, FPBits(neg_result).abs().get_val());
+  }
+}
diff --git a/libc/test/src/math/smoke/bf16add_test.cpp b/libc/test/src/math/smoke/bf16add_test.cpp
new file mode 100644
index 0000000000000..9e9c594d7d7eb
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16add_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16add ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16add.h"
+
+LIST_ADD_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16add)
diff --git a/libc/test/src/math/smoke/bf16addf128_test.cpp b/libc/test/src/math/smoke/bf16addf128_test.cpp
new file mode 100644
index 0000000000000..46f7ad329f237
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16addf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16addf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16addf128.h"
+
+LIST_ADD_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16addf128)
diff --git a/libc/test/src/math/smoke/bf16addf_test.cpp b/libc/test/src/math/smoke/bf16addf_test.cpp
new file mode 100644
index 0000000000000..06d56cf21ebd2
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16addf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16addf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16addf.h"
+
+LIST_ADD_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16addf)
diff --git a/libc/test/src/math/smoke/bf16addl_test.cpp b/libc/test/src/math/smoke/bf16addl_test.cpp
new file mode 100644
index 0000000000000..bf548272d31c1
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16addl_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16addl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16addl.h"
+
+LIST_ADD_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16addl)
diff --git a/libc/test/src/math/smoke/bf16div_test.cpp b/libc/test/src/math/smoke/bf16div_test.cpp
new file mode 100644
index 0000000000000..4516351fb93a3
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16div_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16div ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DivTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16div.h"
+
+LIST_DIV_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16div)
diff --git a/libc/test/src/math/smoke/bf16divf128_test.cpp b/libc/test/src/math/smoke/bf16divf128_test.cpp
new file mode 100644
index 0000000000000..c42c5bb067050
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16divf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16divf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DivTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16divf128.h"
+
+LIST_DIV_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16divf128)
diff --git a/libc/test/src/math/smoke/bf16divf_test.cpp b/libc/test/src/math/smoke/bf16divf_test.cpp
new file mode 100644
index 0000000000000..873920b225a77
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16divf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16divf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DivTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16divf.h"
+
+LIST_DIV_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16divf)
diff --git a/libc/test/src/math/smoke/bf16divl_test.cpp b/libc/test/src/math/smoke/bf16divl_test.cpp
new file mode 100644
index 0000000000000..32ecd6228de0b
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16divl_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16divl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DivTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16divl.h"
+
+LIST_DIV_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16divl)
diff --git a/libc/test/src/math/smoke/bf16fma_test.cpp b/libc/test/src/math/smoke/bf16fma_test.cpp
new file mode 100644
index 0000000000000..81c73a0cb6947
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16fma_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16fma ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FmaTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16fma.h"
+
+LIST_NARROWING_FMA_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16fma)
diff --git a/libc/test/src/math/smoke/bf16fmaf128_test.cpp b/libc/test/src/math/smoke/bf16fmaf128_test.cpp
new file mode 100644
index 0000000000000..dd8f4735c3497
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16fmaf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16fmaf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FmaTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16fmaf128.h"
+
+LIST_NARROWING_FMA_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16fmaf128)
diff --git a/libc/test/src/math/smoke/bf16fmaf_test.cpp b/libc/test/src/math/smoke/bf16fmaf_test.cpp
new file mode 100644
index 0000000000000..04c674892833a
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16fmaf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16fmaf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FmaTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16fmaf.h"
+
+LIST_NARROWING_FMA_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16fmaf)
diff --git a/libc/test/src/math/smoke/bf16fmal_test.cpp b/libc/test/src/math/smoke/bf16fmal_test.cpp
new file mode 100644
index 0000000000000..4c45e2c64312d
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16fmal_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16fmal --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FmaTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16fmal.h"
+
+LIST_NARROWING_FMA_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16fmal)
diff --git a/libc/test/src/math/smoke/bf16mul_test.cpp b/libc/test/src/math/smoke/bf16mul_test.cpp
new file mode 100644
index 0000000000000..db6f0cb7844db
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16mul_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16mul ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16mul.h"
+
+LIST_MUL_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16mul)
diff --git a/libc/test/src/math/smoke/bf16mulf128_test.cpp b/libc/test/src/math/smoke/bf16mulf128_test.cpp
new file mode 100644
index 0000000000000..03cb9f24228ba
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16mulf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16mulf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16mulf128.h"
+
+LIST_MUL_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16mulf128)
diff --git a/libc/test/src/math/smoke/bf16mulf_test.cpp b/libc/test/src/math/smoke/bf16mulf_test.cpp
new file mode 100644
index 0000000000000..ec40e98b20475
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16mulf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16mulf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16mulf.h"
+
+LIST_MUL_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16mulf)
diff --git a/libc/test/src/math/smoke/bf16mull_test.cpp b/libc/test/src/math/smoke/bf16mull_test.cpp
new file mode 100644
index 0000000000000..4cb9c9c730aae
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16mull_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16mull --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16mull.h"
+
+LIST_MUL_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16mull)
diff --git a/libc/test/src/math/smoke/bf16sub_test.cpp b/libc/test/src/math/smoke/bf16sub_test.cpp
new file mode 100644
index 0000000000000..4a793dc493891
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16sub_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16sub ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16sub.h"
+
+LIST_SUB_TESTS(bfloat16, double, LIBC_NAMESPACE::bf16sub)
diff --git a/libc/test/src/math/smoke/bf16subf128_test.cpp b/libc/test/src/math/smoke/bf16subf128_test.cpp
new file mode 100644
index 0000000000000..25d6711f71346
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16subf128_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16subf128 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16subf128.h"
+
+LIST_SUB_TESTS(bfloat16, float128, LIBC_NAMESPACE::bf16subf128)
diff --git a/libc/test/src/math/smoke/bf16subf_test.cpp b/libc/test/src/math/smoke/bf16subf_test.cpp
new file mode 100644
index 0000000000000..e8c7440230f1d
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16subf_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16subf --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16subf.h"
+
+LIST_SUB_TESTS(bfloat16, float, LIBC_NAMESPACE::bf16subf)
diff --git a/libc/test/src/math/smoke/bf16subl_test.cpp b/libc/test/src/math/smoke/bf16subl_test.cpp
new file mode 100644
index 0000000000000..299736949d713
--- /dev/null
+++ b/libc/test/src/math/smoke/bf16subl_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for bf16subl --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/bf16subl.h"
+
+LIST_SUB_TESTS(bfloat16, long double, LIBC_NAMESPACE::bf16subl)
diff --git a/libc/test/src/math/smoke/bfloat16_add_test.cpp b/libc/test/src/math/smoke/bfloat16_add_test.cpp
new file mode 100644
index 0000000000000..1db65a9be2b84
--- /dev/null
+++ b/libc/test/src/math/smoke/bfloat16_add_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for bfloat16 addition -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AddTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+
+static bfloat16 add_func(bfloat16 x, bfloat16 y) { return x + y; }
+
+LIST_ADD_TESTS(bfloat16, bfloat16, add_func)
diff --git a/libc/test/src/math/smoke/bfloat16_div_test.cpp b/libc/test/src/math/smoke/bfloat16_div_test.cpp
new file mode 100644
index 0000000000000..c1ab598df0608
--- /dev/null
+++ b/libc/test/src/math/smoke/bfloat16_div_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for bfloat16 division -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DivTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+
+static bfloat16 div_func(bfloat16 x, bfloat16 y) { return x / y; }
+
+LIST_DIV_TESTS(bfloat16, bfloat16, div_func)
diff --git a/libc/test/src/math/smoke/bfloat16_mul_test.cpp b/libc/test/src/math/smoke/bfloat16_mul_test.cpp
new file mode 100644
index 0000000000000..03e38d8b8e97a
--- /dev/null
+++ b/libc/test/src/math/smoke/bfloat16_mul_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for bfloat16 multiplication -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MulTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+
+static bfloat16 mul_func(bfloat16 x, bfloat16 y) { return x * y; }
+
+LIST_MUL_TESTS(bfloat16, bfloat16, mul_func)
diff --git a/libc/test/src/math/smoke/bfloat16_sub_test.cpp b/libc/test/src/math/smoke/bfloat16_sub_test.cpp
new file mode 100644
index 0000000000000..5eb4a9b7d062a
--- /dev/null
+++ b/libc/test/src/math/smoke/bfloat16_sub_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for bfloat16 subtraction --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SubTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+
+static bfloat16 sub_func(bfloat16 x, bfloat16 y) { return x - y; }
+
+LIST_SUB_TESTS(bfloat16, bfloat16, sub_func)
diff --git a/libc/test/src/math/smoke/ceilbf16_test.cpp b/libc/test/src/math/smoke/ceilbf16_test.cpp
new file mode 100644
index 0000000000000..dcaf058344ff0
--- /dev/null
+++ b/libc/test/src/math/smoke/ceilbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for ceilbf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CeilTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/ceilbf16.h"
+
+LIST_CEIL_TESTS(bfloat16, LIBC_NAMESPACE::ceilbf16)
diff --git a/libc/test/src/math/smoke/floorbf16_test.cpp b/libc/test/src/math/smoke/floorbf16_test.cpp
new file mode 100644
index 0000000000000..9cc77cd15aa1c
--- /dev/null
+++ b/libc/test/src/math/smoke/floorbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for floorbf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FloorTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/floorbf16.h"
+
+LIST_FLOOR_TESTS(bfloat16, LIBC_NAMESPACE::floorbf16)
diff --git a/libc/test/src/math/smoke/fmaxbf16_test.cpp b/libc/test/src/math/smoke/fmaxbf16_test.cpp
new file mode 100644
index 0000000000000..8ff9313a1994b
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaxbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fmaxbf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaxTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fmaxbf16.h"
+
+LIST_FMAX_TESTS(bfloat16, LIBC_NAMESPACE::fmaxbf16)
diff --git a/libc/test/src/math/smoke/fmaximum_mag_numbf16_test.cpp b/libc/test/src/math/smoke/fmaximum_mag_numbf16_test.cpp
new file mode 100644
index 0000000000000..9e435cc365080
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_mag_numbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fmaximum_mag_numbf16 --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagNumTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fmaximum_mag_numbf16.h"
+
+LIST_FMAXIMUM_MAG_NUM_TESTS(bfloat16, LIBC_NAMESPACE::fmaximum_mag_numbf16)
diff --git a/libc/test/src/math/smoke/fmaximum_magbf16_test.cpp b/libc/test/src/math/smoke/fmaximum_magbf16_test.cpp
new file mode 100644
index 0000000000000..22a0bc3380983
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_magbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fmaximum_magbf16 ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumMagTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fmaximum_magbf16.h"
+
+LIST_FMAXIMUM_MAG_TESTS(bfloat16, LIBC_NAMESPACE::fmaximum_magbf16)
diff --git a/libc/test/src/math/smoke/fmaximum_numbf16_test.cpp b/libc/test/src/math/smoke/fmaximum_numbf16_test.cpp
new file mode 100644
index 0000000000000..25dbe9f54603e
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximum_numbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fmaximum_numbf16 ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumNumTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fmaximum_numbf16.h"
+
+LIST_FMAXIMUM_NUM_TESTS(bfloat16, LIBC_NAMESPACE::fmaximum_numbf16)
diff --git a/libc/test/src/math/smoke/fmaximumbf16_test.cpp b/libc/test/src/math/smoke/fmaximumbf16_test.cpp
new file mode 100644
index 0000000000000..e2cf56bcea64a
--- /dev/null
+++ b/libc/test/src/math/smoke/fmaximumbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fmaximumbf16 ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMaximumTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fmaximumbf16.h"
+
+LIST_FMAXIMUM_TESTS(bfloat16, LIBC_NAMESPACE::fmaximumbf16)
diff --git a/libc/test/src/math/smoke/fminbf16_test.cpp b/libc/test/src/math/smoke/fminbf16_test.cpp
new file mode 100644
index 0000000000000..195a0ac07c8ff
--- /dev/null
+++ b/libc/test/src/math/smoke/fminbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fminbf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fminbf16.h"
+
+LIST_FMIN_TESTS(bfloat16, LIBC_NAMESPACE::fminbf16)
diff --git a/libc/test/src/math/smoke/fminimum_mag_numbf16_test.cpp b/libc/test/src/math/smoke/fminimum_mag_numbf16_test.cpp
new file mode 100644
index 0000000000000..ed3d4474a743d
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_mag_numbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fminimum_mag_numbf16 --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagNumTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fminimum_mag_numbf16.h"
+
+LIST_FMINIMUM_MAG_NUM_TESTS(bfloat16, LIBC_NAMESPACE::fminimum_mag_numbf16)
diff --git a/libc/test/src/math/smoke/fminimum_magbf16_test.cpp b/libc/test/src/math/smoke/fminimum_magbf16_test.cpp
new file mode 100644
index 0000000000000..94251e0f94501
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_magbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fminimum_magbf16 ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumMagTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fminimum_magbf16.h"
+
+LIST_FMINIMUM_MAG_TESTS(bfloat16, LIBC_NAMESPACE::fminimum_magbf16)
diff --git a/libc/test/src/math/smoke/fminimum_numbf16_test.cpp b/libc/test/src/math/smoke/fminimum_numbf16_test.cpp
new file mode 100644
index 0000000000000..fe8bb51ffc67f
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimum_numbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fminimum_numbf16 ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumNumTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fminimum_numbf16.h"
+
+LIST_FMINIMUM_NUM_TESTS(bfloat16, LIBC_NAMESPACE::fminimum_numbf16)
diff --git a/libc/test/src/math/smoke/fminimumbf16_test.cpp b/libc/test/src/math/smoke/fminimumbf16_test.cpp
new file mode 100644
index 0000000000000..3667e90745cbb
--- /dev/null
+++ b/libc/test/src/math/smoke/fminimumbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fminimumbf16 ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FMinimumTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fminimumbf16.h"
+
+LIST_FMINIMUM_TESTS(bfloat16, LIBC_NAMESPACE::fminimumbf16)
diff --git a/libc/test/src/math/smoke/frexpbf16_test.cpp b/libc/test/src/math/smoke/frexpbf16_test.cpp
new file mode 100644
index 0000000000000..3c80edbb433ba
--- /dev/null
+++ b/libc/test/src/math/smoke/frexpbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for frexpbf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FrexpTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/frexpbf16.h"
+
+LIST_FREXP_TESTS(bfloat16, LIBC_NAMESPACE::frexpbf16);
diff --git a/libc/test/src/math/smoke/fromfpbf16_test.cpp b/libc/test/src/math/smoke/fromfpbf16_test.cpp
new file mode 100644
index 0000000000000..45d70b2a05c4f
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfpbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fromfpbf16 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fromfpbf16.h"
+
+LIST_FROMFP_TESTS(bfloat16, LIBC_NAMESPACE::fromfpbf16)
diff --git a/libc/test/src/math/smoke/fromfpxbf16_test.cpp b/libc/test/src/math/smoke/fromfpxbf16_test.cpp
new file mode 100644
index 0000000000000..d930ba3f48734
--- /dev/null
+++ b/libc/test/src/math/smoke/fromfpxbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for fromfpxbf16 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "FromfpxTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/fromfpxbf16.h"
+
+LIST_FROMFPX_TESTS(bfloat16, LIBC_NAMESPACE::fromfpxbf16)
diff --git a/libc/test/src/math/smoke/getpayloadbf16_test.cpp b/libc/test/src/math/smoke/getpayloadbf16_test.cpp
new file mode 100644
index 0000000000000..51a5fad88cd97
--- /dev/null
+++ b/libc/test/src/math/smoke/getpayloadbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for getpayloadbf16 --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "GetPayloadTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/getpayloadbf16.h"
+
+LIST_GETPAYLOAD_TESTS(bfloat16, LIBC_NAMESPACE::getpayloadbf16)
diff --git a/libc/test/src/math/smoke/ilogbbf16_test.cpp b/libc/test/src/math/smoke/ilogbbf16_test.cpp
new file mode 100644
index 0000000000000..d9fa03a2e50cb
--- /dev/null
+++ b/libc/test/src/math/smoke/ilogbbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for ilogbbf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ILogbTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/ilogbbf16.h"
+
+LIST_INTLOGB_TESTS(int, bfloat16, LIBC_NAMESPACE::ilogbbf16);
diff --git a/libc/test/src/math/smoke/ldexpbf16_test.cpp b/libc/test/src/math/smoke/ldexpbf16_test.cpp
new file mode 100644
index 0000000000000..2e57eb8460c9e
--- /dev/null
+++ b/libc/test/src/math/smoke/ldexpbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for ldexpbf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LdExpTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/ldexpbf16.h"
+
+LIST_LDEXP_TESTS(bfloat16, LIBC_NAMESPACE::ldexpbf16);
diff --git a/libc/test/src/math/smoke/llogbbf16_test.cpp b/libc/test/src/math/smoke/llogbbf16_test.cpp
new file mode 100644
index 0000000000000..4454fbfb0644b
--- /dev/null
+++ b/libc/test/src/math/smoke/llogbbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for llogbbf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ILogbTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/llogbbf16.h"
+
+LIST_INTLOGB_TESTS(long, bfloat16, LIBC_NAMESPACE::llogbbf16);
diff --git a/libc/test/src/math/smoke/llrintbf16_test.cpp b/libc/test/src/math/smoke/llrintbf16_test.cpp
new file mode 100644
index 0000000000000..e841e627e2634
--- /dev/null
+++ b/libc/test/src/math/smoke/llrintbf16_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for llrintbf16 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/llrintbf16.h"
+
+LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(bfloat16, long long,
+                                       LIBC_NAMESPACE::llrintbf16)
diff --git a/libc/test/src/math/smoke/llroundbf16_test.cpp b/libc/test/src/math/smoke/llroundbf16_test.cpp
new file mode 100644
index 0000000000000..c3b7ea43c3e8a
--- /dev/null
+++ b/libc/test/src/math/smoke/llroundbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for llroundbf16 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/llroundbf16.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(bfloat16, long long, LIBC_NAMESPACE::llroundbf16)
diff --git a/libc/test/src/math/smoke/logbbf16_test.cpp b/libc/test/src/math/smoke/logbbf16_test.cpp
new file mode 100644
index 0000000000000..4ceec3d6984dd
--- /dev/null
+++ b/libc/test/src/math/smoke/logbbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for logbbf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LogbTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/logbbf16.h"
+
+LIST_LOGB_TESTS(bfloat16, LIBC_NAMESPACE::logbbf16)
diff --git a/libc/test/src/math/smoke/lrintbf16_test.cpp b/libc/test/src/math/smoke/lrintbf16_test.cpp
new file mode 100644
index 0000000000000..65a5633b81723
--- /dev/null
+++ b/libc/test/src/math/smoke/lrintbf16_test.cpp
@@ -0,0 +1,15 @@
+//===-- Unittests for lrintbf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/lrintbf16.h"
+
+LIST_ROUND_TO_INTEGER_TESTS_WITH_MODES(bfloat16, long,
+                                       LIBC_NAMESPACE::lrintbf16)
diff --git a/libc/test/src/math/smoke/lroundbf16_test.cpp b/libc/test/src/math/smoke/lroundbf16_test.cpp
new file mode 100644
index 0000000000000..2f2b7b1a71b36
--- /dev/null
+++ b/libc/test/src/math/smoke/lroundbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for lroundbf16 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundToIntegerTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/lroundbf16.h"
+
+LIST_ROUND_TO_INTEGER_TESTS(bfloat16, long, LIBC_NAMESPACE::lroundbf16)
diff --git a/libc/test/src/math/smoke/nanbf16_test.cpp b/libc/test/src/math/smoke/nanbf16_test.cpp
new file mode 100644
index 0000000000000..4154f6ed637c8
--- /dev/null
+++ b/libc/test/src/math/smoke/nanbf16_test.cpp
@@ -0,0 +1,55 @@
+//===-- Unittests for nanbf16 ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/signal_macros.h"
+#include "src/__support/FPUtil/FPBits.h"
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/nanbf16.h"
+#include "test/UnitTest/FEnvSafeTest.h"
+#include "test/UnitTest/FPMatcher.h"
+#include "test/UnitTest/Test.h"
+
+class LlvmLibcNanf16Test : public LIBC_NAMESPACE::testing::FEnvSafeTest {
+public:
+  using StorageType = LIBC_NAMESPACE::fputil::FPBits<bfloat16>::StorageType;
+
+  void run_test(const char *input_str, StorageType bits) {
+    bfloat16 result = LIBC_NAMESPACE::nanbf16(input_str);
+    auto actual_fp = LIBC_NAMESPACE::fputil::FPBits<bfloat16>(result);
+    auto expected_fp = LIBC_NAMESPACE::fputil::FPBits<bfloat16>(bits);
+    EXPECT_EQ(actual_fp.uintval(), expected_fp.uintval());
+  }
+};
+
+TEST_F(LlvmLibcNanf16Test, NCharSeq) {
+  run_test("", 0x7fc0);
+
+  // 0x7fc0 + 0x1f (31) = 0x7cdf
+  run_test("31", 0x7fdf);
+
+  // 0x7fc0 + 0x15 = 0x7fd5
+  run_test("0x15", 0x7fd5);
+
+  run_test("1a", 0x7fc0);
+  run_test("1234567890qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM_",
+           0x7fc0);
+  run_test("10000000000000000000000000000", 0x7fc0);
+}
+
+TEST_F(LlvmLibcNanf16Test, RandomString) {
+  run_test(" 1234", 0x7fc0);
+  run_test("-1234", 0x7fc0);
+  run_test("asd&f", 0x7fc0);
+  run_test("123 ", 0x7fc0);
+}
+
+#if defined(LIBC_ADD_NULL_CHECKS)
+TEST_F(LlvmLibcNanf16Test, InvalidInput) {
+  EXPECT_DEATH([] { LIBC_NAMESPACE::nanbf16(nullptr); }, WITH_SIGNAL(-1));
+}
+#endif // LIBC_ADD_NULL_CHECKS
diff --git a/libc/test/src/math/smoke/nearbyintbf16_test.cpp b/libc/test/src/math/smoke/nearbyintbf16_test.cpp
new file mode 100644
index 0000000000000..2d64fc5a154ee
--- /dev/null
+++ b/libc/test/src/math/smoke/nearbyintbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for nearbyintbf16 ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NearbyIntTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/nearbyintbf16.h"
+
+LIST_NEARBYINT_TESTS(bfloat16, LIBC_NAMESPACE::nearbyintbf16)
diff --git a/libc/test/src/math/smoke/nextafterbf16_test.cpp b/libc/test/src/math/smoke/nextafterbf16_test.cpp
new file mode 100644
index 0000000000000..b73a3cbef0b74
--- /dev/null
+++ b/libc/test/src/math/smoke/nextafterbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for nextafterbf16 ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NextAfterTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/nextafterbf16.h"
+
+LIST_NEXTAFTER_TESTS(bfloat16, LIBC_NAMESPACE::nextafterbf16)
diff --git a/libc/test/src/math/smoke/nextdownbf16_test.cpp b/libc/test/src/math/smoke/nextdownbf16_test.cpp
new file mode 100644
index 0000000000000..300206461e2de
--- /dev/null
+++ b/libc/test/src/math/smoke/nextdownbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for nextdownbf16 ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NextDownTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/nextdownbf16.h"
+
+LIST_NEXTDOWN_TESTS(bfloat16, LIBC_NAMESPACE::nextdownbf16)
diff --git a/libc/test/src/math/smoke/nexttowardbf16_test.cpp b/libc/test/src/math/smoke/nexttowardbf16_test.cpp
new file mode 100644
index 0000000000000..c1f35fcb96ef6
--- /dev/null
+++ b/libc/test/src/math/smoke/nexttowardbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for nexttowardbf16 --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NextTowardTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/nexttowardbf16.h"
+
+LIST_NEXTTOWARD_TESTS(bfloat16, LIBC_NAMESPACE::nexttowardbf16)
diff --git a/libc/test/src/math/smoke/nextupbf16_test.cpp b/libc/test/src/math/smoke/nextupbf16_test.cpp
new file mode 100644
index 0000000000000..49ad2f70b12a8
--- /dev/null
+++ b/libc/test/src/math/smoke/nextupbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for nextupbf16 ------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NextUpTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/nextupbf16.h"
+
+LIST_NEXTUP_TESTS(bfloat16, LIBC_NAMESPACE::nextupbf16)
diff --git a/libc/test/src/math/smoke/rintbf16_test.cpp b/libc/test/src/math/smoke/rintbf16_test.cpp
new file mode 100644
index 0000000000000..c78dcf6a31560
--- /dev/null
+++ b/libc/test/src/math/smoke/rintbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for rintbf16 --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RIntTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/rintbf16.h"
+
+LIST_RINT_TESTS(bfloat16, LIBC_NAMESPACE::rintbf16)
diff --git a/libc/test/src/math/smoke/roundbf16_test.cpp b/libc/test/src/math/smoke/roundbf16_test.cpp
new file mode 100644
index 0000000000000..51638689e8092
--- /dev/null
+++ b/libc/test/src/math/smoke/roundbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for roundbf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/roundbf16.h"
+
+LIST_ROUND_TESTS(bfloat16, LIBC_NAMESPACE::roundbf16)
diff --git a/libc/test/src/math/smoke/roundevenbf16_test.cpp b/libc/test/src/math/smoke/roundevenbf16_test.cpp
new file mode 100644
index 0000000000000..711c37a4e5115
--- /dev/null
+++ b/libc/test/src/math/smoke/roundevenbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for roundevenbf16 ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "RoundEvenTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/roundevenbf16.h"
+
+LIST_ROUNDEVEN_TESTS(bfloat16, LIBC_NAMESPACE::roundevenbf16)
diff --git a/libc/test/src/math/smoke/setpayloadbf16_test.cpp b/libc/test/src/math/smoke/setpayloadbf16_test.cpp
new file mode 100644
index 0000000000000..198b454ed78b0
--- /dev/null
+++ b/libc/test/src/math/smoke/setpayloadbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for setpayloadbf16 --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SetPayloadTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/setpayloadbf16.h"
+
+LIST_SETPAYLOAD_TESTS(bfloat16, LIBC_NAMESPACE::setpayloadbf16)
diff --git a/libc/test/src/math/smoke/setpayloadsigbf16_test.cpp b/libc/test/src/math/smoke/setpayloadsigbf16_test.cpp
new file mode 100644
index 0000000000000..aa6a3e9dcbf2f
--- /dev/null
+++ b/libc/test/src/math/smoke/setpayloadsigbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for setpayloadsigbf16 -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SetPayloadSigTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/setpayloadsigbf16.h"
+
+LIST_SETPAYLOADSIG_TESTS(bfloat16, LIBC_NAMESPACE::setpayloadsigbf16)
diff --git a/libc/test/src/math/smoke/truncbf16_test.cpp b/libc/test/src/math/smoke/truncbf16_test.cpp
new file mode 100644
index 0000000000000..970fa69b3c7a1
--- /dev/null
+++ b/libc/test/src/math/smoke/truncbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for truncbf16 -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TruncTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/truncbf16.h"
+
+LIST_TRUNC_TESTS(bfloat16, LIBC_NAMESPACE::truncbf16)
diff --git a/libc/test/src/math/smoke/ufromfpbf16_test.cpp b/libc/test/src/math/smoke/ufromfpbf16_test.cpp
new file mode 100644
index 0000000000000..2c516b0304ef7
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfpbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for ufromfpbf16 -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/ufromfpbf16.h"
+
+LIST_UFROMFP_TESTS(bfloat16, LIBC_NAMESPACE::ufromfpbf16)
diff --git a/libc/test/src/math/smoke/ufromfpxbf16_test.cpp b/libc/test/src/math/smoke/ufromfpxbf16_test.cpp
new file mode 100644
index 0000000000000..56e27276b4741
--- /dev/null
+++ b/libc/test/src/math/smoke/ufromfpxbf16_test.cpp
@@ -0,0 +1,14 @@
+//===-- Unittests for ufromfpxbf16 ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UfromfpxTest.h"
+
+#include "src/__support/FPUtil/bfloat16.h"
+#include "src/math/ufromfpxbf16.h"
+
+LIST_UFROMFPX_TESTS(bfloat16, LIBC_NAMESPACE::ufromfpxbf16)
diff --git a/libc/test/src/stdfix/BitsFxTest.h b/libc/test/src/stdfix/BitsFxTest.h
index eca6ab1bb28ab..cdb363f07053b 100644
--- a/libc/test/src/stdfix/BitsFxTest.h
+++ b/libc/test/src/stdfix/BitsFxTest.h
@@ -54,7 +54,7 @@ class BitsFxTest : public LIBC_NAMESPACE::testing::Test {
 
       if (max >= 11 && FXRep::FRACTION_LEN >= kMinFbits) {
         // (10.71875)_10 = (1010.1011100)_2
-        constexpr long long kExpected = 1372;
+        constexpr int64_t kExpected = 1372;
         EXPECT_EQ(
             static_cast<XType>(kExpected << (FXRep::FRACTION_LEN - kMinFbits)),
             func(special_num_t));
@@ -63,9 +63,11 @@ class BitsFxTest : public LIBC_NAMESPACE::testing::Test {
       if constexpr (FXRep::SIGN_LEN > 0) {
         if (min <= -11 && FXRep::FRACTION_LEN >= kMinFbits) {
           // (-10.71875)_10 = (-1010.1011100)_2
-          constexpr long long kExpected = -1372;
-          EXPECT_EQ(static_cast<XType>(kExpected
-                                       << (FXRep::FRACTION_LEN - kMinFbits)),
+          constexpr int64_t kExpected =
+              static_cast<int64_t>(static_cast<uint64_t>(-1372)
+                                   << (FXRep::FRACTION_LEN - kMinFbits));
+
+          EXPECT_EQ(static_cast<XType>(kExpected),
                     func(negative_special_num_t));
         }
       }
diff --git a/libc/test/src/string/strsep_test.cpp b/libc/test/src/string/strsep_test.cpp
index 06318dea4cb68..e2a5d52bbeddb 100644
--- a/libc/test/src/string/strsep_test.cpp
+++ b/libc/test/src/string/strsep_test.cpp
@@ -18,21 +18,21 @@ TEST(LlvmLibcStrsepTest, NullSrc) {
 TEST(LlvmLibcStrsepTest, NoTokenFound) {
   {
     char s[] = "";
-    char *string = s, *orig = s;
+    char *string = s;
     EXPECT_STREQ(LIBC_NAMESPACE::strsep(&string, ""), nullptr);
-    EXPECT_EQ(orig, string);
+    EXPECT_TRUE(string == nullptr);
   }
   {
     char s[] = "abcde";
     char *string = s, *orig = s;
     EXPECT_STREQ(LIBC_NAMESPACE::strsep(&string, ""), orig);
-    EXPECT_EQ(string, orig + 5);
+    EXPECT_TRUE(string == nullptr);
   }
   {
     char s[] = "abcde";
     char *string = s, *orig = s;
     EXPECT_STREQ(LIBC_NAMESPACE::strsep(&string, "fghijk"), orig);
-    EXPECT_EQ(string, orig + 5);
+    EXPECT_TRUE(string == nullptr);
   }
 }
 
@@ -53,6 +53,14 @@ TEST(LlvmLibcStrsepTest, DelimitersShouldNotBeIncludedInToken) {
   }
 }
 
+TEST(LlvmLibcStrsepTest, SubsequentSearchesReturnNull) {
+  char s[] = "a";
+  char *string = s;
+  ASSERT_STREQ(LIBC_NAMESPACE::strsep(&string, ":"), "a");
+  ASSERT_EQ(LIBC_NAMESPACE::strsep(&string, ":"), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::strsep(&string, ":"), nullptr);
+}
+
 #if defined(LIBC_ADD_NULL_CHECKS)
 
 TEST(LlvmLibcStrsepTest, CrashOnNullPtr) {
diff --git a/libc/test/src/string/strtok_r_test.cpp b/libc/test/src/string/strtok_r_test.cpp
index fdc27bae23c97..a19390d0b0c2d 100644
--- a/libc/test/src/string/strtok_r_test.cpp
+++ b/libc/test/src/string/strtok_r_test.cpp
@@ -122,3 +122,12 @@ TEST(LlvmLibcStrTokReentrantTest, DelimitersShouldNotBeIncludedInToken) {
   token = LIBC_NAMESPACE::strtok_r(nullptr, "_:,_", &reserve);
   ASSERT_STREQ(token, nullptr);
 }
+
+TEST(LlvmLibcStrTokReentrantTest, SubsequentSearchesReturnNull) {
+  char src[] = "a";
+  char *reserve = nullptr;
+  char *token = LIBC_NAMESPACE::strtok_r(src, ":", &reserve);
+  ASSERT_STREQ(token, "a");
+  ASSERT_EQ(LIBC_NAMESPACE::strtok_r(nullptr, ":", &reserve), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::strtok_r(nullptr, ":", &reserve), nullptr);
+}
diff --git a/libc/test/src/string/strtok_test.cpp b/libc/test/src/string/strtok_test.cpp
index b82065309e00c..76efeddda6f4a 100644
--- a/libc/test/src/string/strtok_test.cpp
+++ b/libc/test/src/string/strtok_test.cpp
@@ -76,3 +76,10 @@ TEST(LlvmLibcStrTokTest, DelimitersShouldNotBeIncludedInToken) {
   token = LIBC_NAMESPACE::strtok(nullptr, "_:,_");
   ASSERT_STREQ(token, nullptr);
 }
+
+TEST(LlvmLibcStrTokTest, SubsequentSearchesReturnNull) {
+  char src[] = "a";
+  ASSERT_STREQ("a", LIBC_NAMESPACE::strtok(src, ":"));
+  ASSERT_EQ(LIBC_NAMESPACE::strtok(nullptr, ":"), nullptr);
+  ASSERT_EQ(LIBC_NAMESPACE::strtok(nullptr, ":"), nullptr);
+}
diff --git a/libc/test/src/time/timespec_get_test.cpp b/libc/test/src/time/timespec_get_test.cpp
index 327bfefe934c1..97585016f1dd7 100644
--- a/libc/test/src/time/timespec_get_test.cpp
+++ b/libc/test/src/time/timespec_get_test.cpp
@@ -24,6 +24,8 @@ TEST(LlvmLibcTimespecGet, Utc) {
 #endif
 }
 
+// Baremetal implementation currently only supports TIME_UTC
+#ifndef LIBC_TARGET_OS_IS_BAREMETAL
 TEST(LlvmLibcTimespecGet, Monotonic) {
   timespec ts1, ts2;
   int result;
@@ -37,6 +39,7 @@ TEST(LlvmLibcTimespecGet, Monotonic) {
     ASSERT_GE(ts2.tv_nsec, ts1.tv_nsec);
   }
 }
+#endif
 
 TEST(LlvmLibcTimespecGet, Unknown) {
   timespec ts;
diff --git a/libc/test/src/wchar/mbrtowc_test.cpp b/libc/test/src/wchar/mbrtowc_test.cpp
index c406300b9ca34..ddf8fc73b055b 100644
--- a/libc/test/src/wchar/mbrtowc_test.cpp
+++ b/libc/test/src/wchar/mbrtowc_test.cpp
@@ -37,18 +37,18 @@ TEST_F(LlvmLibcMBRToWCTest, TwoByte) {
   const char ch[2] = {static_cast<char>(0xC2),
                       static_cast<char>(0x8E)}; // Ž car symbol
   wchar_t dest[2];
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
-  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, &mb);
   ASSERT_EQ(static_cast<int>(*dest), 142);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_ERRNO_SUCCESS();
 
   // Should fail since we have not read enough
-  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, &mb);
   ASSERT_EQ(static_cast<int>(n), -2);
   // Should pass after reading one more byte
-  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 1, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 1, &mb);
   ASSERT_EQ(static_cast<int>(n), 1);
   ASSERT_EQ(static_cast<int>(*dest), 142);
   ASSERT_ERRNO_SUCCESS();
@@ -58,19 +58,19 @@ TEST_F(LlvmLibcMBRToWCTest, ThreeByte) {
   const char ch[3] = {static_cast<char>(0xE2), static_cast<char>(0x88),
                       static_cast<char>(0x91)}; // ∑ sigma symbol
   wchar_t dest[2];
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
-  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 3, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 3, &mb);
   ASSERT_EQ(static_cast<int>(*dest), 8721);
   ASSERT_EQ(static_cast<int>(n), 3);
   ASSERT_ERRNO_SUCCESS();
 
   // Should fail since we have not read enough
-  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, &mb);
   ASSERT_EQ(static_cast<int>(n), -2);
   ASSERT_ERRNO_SUCCESS();
   // Should pass after reading two more bytes
-  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 2, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 2, &mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*dest), 8721);
   ASSERT_ERRNO_SUCCESS();
@@ -81,18 +81,18 @@ TEST_F(LlvmLibcMBRToWCTest, FourByte) {
                       static_cast<char>(0xA4),
                       static_cast<char>(0xA1)}; // 🤡 clown emoji
   wchar_t dest[2];
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
-  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, &mb);
   ASSERT_EQ(static_cast<int>(*dest), 129313);
   ASSERT_EQ(static_cast<int>(n), 4);
   ASSERT_ERRNO_SUCCESS();
   // Should fail since we have not read enough
-  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, &mb);
   ASSERT_EQ(static_cast<int>(n), -2);
   ASSERT_ERRNO_SUCCESS();
   // Should pass after reading two more bytes
-  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 2, 2, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 2, 2, &mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*dest), 129313);
   ASSERT_ERRNO_SUCCESS();
@@ -101,9 +101,9 @@ TEST_F(LlvmLibcMBRToWCTest, FourByte) {
 TEST_F(LlvmLibcMBRToWCTest, InvalidByte) {
   const char ch[1] = {static_cast<char>(0x80)};
   wchar_t dest[2];
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
-  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, &mb);
   ASSERT_EQ(static_cast<int>(n), -1);
   ASSERT_ERRNO_EQ(EILSEQ);
 }
@@ -113,18 +113,18 @@ TEST_F(LlvmLibcMBRToWCTest, InvalidMultiByte) {
                       static_cast<char>(0x80),
                       static_cast<char>(0x00)}; // invalid sequence of bytes
   wchar_t dest[2];
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
   // Trying to push all 4 should error
-  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, &mb);
   ASSERT_EQ(static_cast<int>(n), -1);
   ASSERT_ERRNO_EQ(EILSEQ);
   // Trying to push just the first one should error
-  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, &mb);
   ASSERT_EQ(static_cast<int>(n), -1);
   ASSERT_ERRNO_EQ(EILSEQ);
   // Trying to push the second and third should correspond to null wc
-  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 2, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 2, &mb);
   ASSERT_EQ(static_cast<int>(n), 0);
   ASSERT_TRUE(*dest == L'\0');
   ASSERT_ERRNO_SUCCESS();
@@ -136,10 +136,10 @@ TEST_F(LlvmLibcMBRToWCTest, InvalidLastByte) {
   const char ch[4] = {static_cast<char>(0xF1), static_cast<char>(0x80),
                       static_cast<char>(0x80), static_cast<char>(0xC0)};
   wchar_t dest[2];
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
   // Trying to push all 4 should error
-  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, &mb);
   ASSERT_EQ(static_cast<int>(n), -1);
   ASSERT_ERRNO_EQ(EILSEQ);
 }
@@ -148,10 +148,10 @@ TEST_F(LlvmLibcMBRToWCTest, ValidTwoByteWithExtraRead) {
   const char ch[3] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
                       static_cast<char>(0x80)};
   wchar_t dest[2];
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
   // Trying to push all 3 should return valid 2 byte
-  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 3, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 3, &mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*dest), 142);
   ASSERT_ERRNO_SUCCESS();
@@ -161,14 +161,14 @@ TEST_F(LlvmLibcMBRToWCTest, TwoValidTwoBytes) {
   const char ch[4] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
                       static_cast<char>(0xC7), static_cast<char>(0x8C)};
   wchar_t dest[2];
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
   // mbstate should reset after reading first one
-  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, &mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*dest), 142);
   ASSERT_ERRNO_SUCCESS();
-  n = LIBC_NAMESPACE::mbrtowc(dest + 1, ch + 2, 2, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest + 1, ch + 2, 2, &mb);
   ASSERT_EQ(static_cast<int>(n), 2);
   ASSERT_EQ(static_cast<int>(*(dest + 1)), 460);
   ASSERT_ERRNO_SUCCESS();
@@ -176,16 +176,16 @@ TEST_F(LlvmLibcMBRToWCTest, TwoValidTwoBytes) {
 
 TEST_F(LlvmLibcMBRToWCTest, NullString) {
   wchar_t dest[2] = {L'O', L'K'};
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
   // reading on nullptr should return 0
-  size_t n = LIBC_NAMESPACE::mbrtowc(dest, nullptr, 2, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, nullptr, 2, &mb);
   ASSERT_EQ(static_cast<int>(n), 0);
   ASSERT_TRUE(dest[0] == L'O');
   ASSERT_ERRNO_SUCCESS();
   // reading a null terminator should return 0
   const char *ch = "\0";
-  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, &mb);
   ASSERT_EQ(static_cast<int>(n), 0);
   ASSERT_ERRNO_SUCCESS();
 }
@@ -194,10 +194,10 @@ TEST_F(LlvmLibcMBRToWCTest, NullDest) {
   const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
                       static_cast<char>(0xA4),
                       static_cast<char>(0xA1)}; // 🤡 clown emoji
-  mbstate_t *mb;
+  mbstate_t mb;
   LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
   // reading nullptr should return correct size
-  size_t n = LIBC_NAMESPACE::mbrtowc(nullptr, ch, 10, mb);
+  size_t n = LIBC_NAMESPACE::mbrtowc(nullptr, ch, 10, &mb);
   ASSERT_EQ(static_cast<int>(n), 4);
   ASSERT_ERRNO_SUCCESS();
 }
diff --git a/libc/test/src/wctype/iswalpha_test.cpp b/libc/test/src/wctype/iswalpha_test.cpp
index f3f75f4dc7aa5..a82c0057c1807 100644
--- a/libc/test/src/wctype/iswalpha_test.cpp
+++ b/libc/test/src/wctype/iswalpha_test.cpp
@@ -9,46 +9,48 @@
 #include "src/__support/CPP/span.h"
 #include "src/wctype/iswalpha.h"
 
-#include "test/UnitTest/LibcTest.h"
 #include "test/UnitTest/Test.h"
 
-namespace {
-
-// TODO: Merge the wctype tests using this framework.
-constexpr char WALPHA_ARRAY[] = {
-    'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
-    'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
-    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
-    'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
-};
-
-bool in_span(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
-  for (size_t i = 0; i < arr.size(); ++i)
-    if (static_cast<int>(arr[i]) == ch)
-      return true;
-  return false;
-}
-
-} // namespace
-
 TEST(LlvmLibciswalpha, SimpleTest) {
-  EXPECT_TRUE(LIBC_NAMESPACE::iswalpha('a'));
-  EXPECT_TRUE(LIBC_NAMESPACE::iswalpha('B'));
-
-  EXPECT_FALSE(LIBC_NAMESPACE::iswalpha('3'));
-  EXPECT_FALSE(LIBC_NAMESPACE::iswalpha(' '));
-  EXPECT_FALSE(LIBC_NAMESPACE::iswalpha('?'));
-  EXPECT_FALSE(LIBC_NAMESPACE::iswalpha('\0'));
-  EXPECT_FALSE(LIBC_NAMESPACE::iswalpha(-1));
+  EXPECT_NE(LIBC_NAMESPACE::iswalpha('a'), 0);
+  EXPECT_NE(LIBC_NAMESPACE::iswalpha('B'), 0);
+
+  EXPECT_EQ(LIBC_NAMESPACE::iswalpha('3'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::iswalpha(' '), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::iswalpha('?'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::iswalpha('\0'), 0);
+  EXPECT_EQ(LIBC_NAMESPACE::iswalpha(-1), 0);
 }
 
-TEST(LlvmLibciswalpha, DefaultLocale) {
-  // Loops through all characters, verifying that letters return
-  // true and everything else returns false.
-  for (int ch = -255; ch < 255; ++ch) {
-    if (in_span(ch, WALPHA_ARRAY))
-      EXPECT_TRUE(LIBC_NAMESPACE::iswalpha(ch));
-    else
-      EXPECT_FALSE(LIBC_NAMESPACE::iswalpha(ch));
-  }
-}
+// TODO: once iswalpha supports more than just ascii-range characters add a
+// proper test.
+
+// namespace {
+
+// // TODO: Merge the wctype tests using this framework.
+// constexpr char WALPHA_ARRAY[] = {
+//     'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+//     'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+//     'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+//     'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+// };
+
+// bool in_span(int ch, LIBC_NAMESPACE::cpp::span<const char> arr) {
+//   for (size_t i = 0; i < arr.size(); ++i)
+//     if (static_cast<int>(arr[i]) == ch)
+//       return true;
+//   return false;
+// }
+
+// } // namespace
+
+// TEST(LlvmLibciswalpha, DefaultLocale) {
+//   // Loops through all characters, verifying that letters return
+//   // true and everything else returns false.
+//   for (int ch = -255; ch < 255; ++ch) {
+//     if (in_span(ch, WALPHA_ARRAY))
+//       EXPECT_TRUE(LIBC_NAMESPACE::iswalpha(ch));
+//     else
+//       EXPECT_FALSE(LIBC_NAMESPACE::iswalpha(ch));
+//   }
+// }
diff --git a/libc/utils/MPFRWrapper/CMakeLists.txt b/libc/utils/MPFRWrapper/CMakeLists.txt
index 2669b23ba43f9..73151c61a7fcc 100644
--- a/libc/utils/MPFRWrapper/CMakeLists.txt
+++ b/libc/utils/MPFRWrapper/CMakeLists.txt
@@ -43,6 +43,7 @@ if(LIBC_TESTS_CAN_USE_MPFR)
     libc.hdr.stdint_proxy
     libc.src.__support.CPP.array
     libc.src.__support.CPP.stringstream
+    libc.src.__support.FPUtil.bfloat16
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fpbits_str
     LibcTest.unit
diff --git a/libc/utils/MPFRWrapper/MPCommon.cpp b/libc/utils/MPFRWrapper/MPCommon.cpp
index 07339a06fff80..77039d4bf7df5 100644
--- a/libc/utils/MPFRWrapper/MPCommon.cpp
+++ b/libc/utils/MPFRWrapper/MPCommon.cpp
@@ -105,6 +105,21 @@ MPFRNumber MPFRNumber::asinh() const {
   return result;
 }
 
+MPFRNumber MPFRNumber::asinpi() const {
+  MPFRNumber result(*this);
+#if MPFR_VERSION >= MPFR_VERSION_NUM(4, 2, 0)
+  mpfr_asinpi(result.value, value, mpfr_rounding);
+  return result;
+#else
+  MPFRNumber value_asin(0.0, 1280);
+  mpfr_asin(value_asin.value, value, MPFR_RNDN);
+  MPFRNumber value_pi(0.0, 1280);
+  mpfr_const_pi(value_pi.value, MPFR_RNDN);
+  mpfr_div(result.value, value_asin.value, value_pi.value, mpfr_rounding);
+  return result;
+#endif
+}
+
 MPFRNumber MPFRNumber::atan() const {
   MPFRNumber result(*this);
   mpfr_atan(result.value, value, mpfr_rounding);
diff --git a/libc/utils/MPFRWrapper/MPCommon.h b/libc/utils/MPFRWrapper/MPCommon.h
index 8bcc69c247a34..47d6293c06af4 100644
--- a/libc/utils/MPFRWrapper/MPCommon.h
+++ b/libc/utils/MPFRWrapper/MPCommon.h
@@ -189,6 +189,7 @@ class MPFRNumber {
   MPFRNumber add(const MPFRNumber &b) const;
   MPFRNumber asin() const;
   MPFRNumber asinh() const;
+  MPFRNumber asinpi() const;
   MPFRNumber atan() const;
   MPFRNumber atan2(const MPFRNumber &b);
   MPFRNumber atanh() const;
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp
index 8853f96ef8f92..2155f324aa8a5 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.cpp
+++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp
@@ -11,6 +11,7 @@
 
 #include "src/__support/CPP/array.h"
 #include "src/__support/CPP/stringstream.h"
+#include "src/__support/FPUtil/bfloat16.h"
 #include "src/__support/FPUtil/fpbits_str.h"
 #include "src/__support/macros/config.h"
 #include "src/__support/macros/properties/types.h"
@@ -38,6 +39,8 @@ unary_operation(Operation op, InputType input, unsigned int precision,
     return mpfrInput.asin();
   case Operation::Asinh:
     return mpfrInput.asinh();
+  case Operation::Asinpi:
+    return mpfrInput.asinpi();
   case Operation::Atan:
     return mpfrInput.atan();
   case Operation::Atanh:
@@ -408,6 +411,23 @@ template void explain_binary_operation_one_output_error(
 template void explain_binary_operation_one_output_error(
     Operation, const BinaryInput<float128> &, float128, double, RoundingMode);
 #endif
+template void explain_binary_operation_one_output_error(
+    Operation, const BinaryInput<bfloat16> &, bfloat16, double, RoundingMode);
+template void
+explain_binary_operation_one_output_error(Operation, const BinaryInput<float> &,
+                                          bfloat16, double, RoundingMode);
+template void explain_binary_operation_one_output_error(
+    Operation, const BinaryInput<double> &, bfloat16, double, RoundingMode);
+template void
+explain_binary_operation_one_output_error(Operation,
+                                          const BinaryInput<long double> &,
+                                          bfloat16, double, RoundingMode);
+#if defined(LIBC_TYPES_HAS_FLOAT128) &&                                        \
+    defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
+template void explain_binary_operation_one_output_error(
+    Operation, const BinaryInput<float128> &, bfloat16, double, RoundingMode);
+#endif // defined(LIBC_TYPES_HAS_FLOAT128) &&
+       // defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
 
 template <typename InputType, typename OutputType>
 void explain_ternary_operation_one_output_error(
@@ -468,6 +488,21 @@ explain_ternary_operation_one_output_error(Operation,
                                            float16, double, RoundingMode);
 #endif
 
+template void explain_ternary_operation_one_output_error(
+    Operation, const TernaryInput<float> &, bfloat16, double, RoundingMode);
+template void explain_ternary_operation_one_output_error(
+    Operation, const TernaryInput<double> &, bfloat16, double, RoundingMode);
+template void
+explain_ternary_operation_one_output_error(Operation,
+                                           const TernaryInput<long double> &,
+                                           bfloat16, double, RoundingMode);
+#if defined(LIBC_TYPES_HAS_FLOAT128) &&                                        \
+    defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
+template void explain_ternary_operation_one_output_error(
+    Operation, const TernaryInput<float128> &, bfloat16, double, RoundingMode);
+#endif // defined(LIBC_TYPES_HAS_FLOAT128) &&
+       // defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
+
 template <typename InputType, typename OutputType>
 bool compare_unary_operation_single_output(Operation op, InputType input,
                                            OutputType libc_result,
@@ -641,7 +676,30 @@ template bool compare_binary_operation_one_output(Operation,
                                                   float128, double,
                                                   RoundingMode);
 #endif
+template bool compare_binary_operation_one_output(Operation,
+                                                  const BinaryInput<bfloat16> &,
+                                                  bfloat16, double,
+                                                  RoundingMode);
 
+template bool compare_binary_operation_one_output(Operation,
+                                                  const BinaryInput<float> &,
+                                                  bfloat16, double,
+                                                  RoundingMode);
+template bool compare_binary_operation_one_output(Operation,
+                                                  const BinaryInput<double> &,
+                                                  bfloat16, double,
+                                                  RoundingMode);
+template bool
+compare_binary_operation_one_output(Operation, const BinaryInput<long double> &,
+                                    bfloat16, double, RoundingMode);
+#if defined(LIBC_TYPES_HAS_FLOAT128) &&                                        \
+    defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
+template bool compare_binary_operation_one_output(Operation,
+                                                  const BinaryInput<float128> &,
+                                                  bfloat16, double,
+                                                  RoundingMode);
+#endif // defined(LIBC_TYPES_HAS_FLOAT128) &&
+       // defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
 template <typename InputType, typename OutputType>
 bool compare_ternary_operation_one_output(Operation op,
                                           const TernaryInput<InputType> &input,
@@ -693,6 +751,27 @@ compare_ternary_operation_one_output(Operation,
                                      double, RoundingMode);
 #endif
 
+template bool compare_ternary_operation_one_output(Operation,
+                                                   const TernaryInput<float> &,
+                                                   bfloat16, double,
+                                                   RoundingMode);
+template bool compare_ternary_operation_one_output(Operation,
+                                                   const TernaryInput<double> &,
+                                                   bfloat16, double,
+                                                   RoundingMode);
+template bool
+compare_ternary_operation_one_output(Operation,
+                                     const TernaryInput<long double> &,
+                                     bfloat16, double, RoundingMode);
+
+#if defined(LIBC_TYPES_HAS_FLOAT128) &&                                        \
+    defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
+template bool
+compare_ternary_operation_one_output(Operation, const TernaryInput<float128> &,
+                                     bfloat16, double, RoundingMode);
+#endif // defined(LIBC_TYPES_HAS_FLOAT128) &&
+       // defined(LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE)
+
 } // namespace internal
 
 template <typename T> bool round_to_long(T x, long &result) {
@@ -712,6 +791,8 @@ template bool round_to_long<float16>(float16, long &);
 template bool round_to_long<float128>(float128, long &);
 #endif // LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
 
+template bool round_to_long<bfloat16>(bfloat16, long &);
+
 template <typename T> bool round_to_long(T x, RoundingMode mode, long &result) {
   MPFRNumber mpfr(x);
   return mpfr.round_to_long(get_mpfr_rounding_mode(mode), result);
@@ -729,6 +810,8 @@ template bool round_to_long<float16>(float16, RoundingMode, long &);
 template bool round_to_long<float128>(float128, RoundingMode, long &);
 #endif // LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
 
+template bool round_to_long<bfloat16>(bfloat16, RoundingMode, long &);
+
 template <typename T> T round(T x, RoundingMode mode) {
   MPFRNumber mpfr(x);
   MPFRNumber result = mpfr.rint(get_mpfr_rounding_mode(mode));
@@ -747,6 +830,8 @@ template float16 round<float16>(float16, RoundingMode);
 template float128 round<float128>(float128, RoundingMode);
 #endif // LIBC_TYPES_FLOAT128_IS_NOT_LONG_DOUBLE
 
+template bfloat16 round<bfloat16>(bfloat16, RoundingMode);
+
 } // namespace mpfr
 } // namespace testing
 } // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.h b/libc/utils/MPFRWrapper/MPFRUtils.h
index 45468c6cb19ae..e805607328f63 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.h
+++ b/libc/utils/MPFRWrapper/MPFRUtils.h
@@ -30,6 +30,7 @@ enum class Operation : int {
   Acospi,
   Asin,
   Asinh,
+  Asinpi,
   Atan,
   Atanh,
   Cbrt,
diff --git a/libc/utils/hdrgen/hdrgen/header.py b/libc/utils/hdrgen/hdrgen/header.py
index b054ed4e03bd0..2118db6e5fb75 100644
--- a/libc/utils/hdrgen/hdrgen/header.py
+++ b/libc/utils/hdrgen/hdrgen/header.py
@@ -204,7 +204,7 @@ def relpath(file):
 
         current_guard = None
         for function in self.functions:
-            if function.guard == None:
+            if function.guard == None and current_guard == None:
                 content.append(str(function) + " __NOEXCEPT;")
                 content.append("")
             else:
@@ -221,7 +221,8 @@ def relpath(file):
                     content.append(f"#endif // {current_guard}")
                     content.append("")
                     current_guard = function.guard
-                    content.append(f"#ifdef {current_guard}")
+                    if current_guard is not None:
+                        content.append(f"#ifdef {current_guard}")
                     content.append(str(function) + " __NOEXCEPT;")
                     content.append("")
         if current_guard != None:
diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index adfabf48297af..756e097dcf123 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -31,6 +31,7 @@ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
   clc/lib/amdgcn/SOURCES;
   clc/lib/amdgpu/SOURCES;
   clc/lib/clspv/SOURCES;
+  clc/lib/ptx-nvidiacl/SOURCES;
   clc/lib/r600/SOURCES;
   clc/lib/spirv/SOURCES;
 )
diff --git a/libclc/clc/include/clc/atomic/atomic_decl.inc b/libclc/clc/include/clc/atomic/atomic_decl.inc
index b790a94c7d288..5e0f456e34009 100644
--- a/libclc/clc/include/clc/atomic/atomic_decl.inc
+++ b/libclc/clc/include/clc/atomic/atomic_decl.inc
@@ -14,23 +14,23 @@
 
 #ifdef __CLC_NO_VALUE_ARG
 #define __CLC_DECLARE_ATOMIC(ADDRSPACE)                                        \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(                        \
       volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder,                  \
       int MemoryScope);
 #elif defined(__CLC_RETURN_VOID)
 #define __CLC_DECLARE_ATOMIC(ADDRSPACE)                                        \
-  _CLC_OVERLOAD _CLC_DECL void FUNCTION(volatile ADDRSPACE __CLC_GENTYPE *Ptr, \
-                                        __CLC_GENTYPE Value, int MemoryOrder,  \
-                                        int MemoryScope);
+  _CLC_OVERLOAD _CLC_DECL void __CLC_FUNCTION(                                 \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value,              \
+      int MemoryOrder, int MemoryScope);
 #elif defined(__CLC_COMPARE_EXCHANGE)
 #define __CLC_DECLARE_ATOMIC(ADDRSPACE)                                        \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(                        \
       volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Comparator,         \
       __CLC_GENTYPE Value, int MemoryOrderEqual, int MemoryOrderUnequal,       \
       int MemoryScope);
 #else
 #define __CLC_DECLARE_ATOMIC(ADDRSPACE)                                        \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(                        \
       volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value,              \
       int MemoryOrder, int MemoryScope);
 #endif
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_compare_exchange.h b/libclc/clc/include/clc/atomic/clc_atomic_compare_exchange.h
index ae7918ac32e43..31b8167223502 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_compare_exchange.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_compare_exchange.h
@@ -11,7 +11,7 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_compare_exchange
+#define __CLC_FUNCTION __clc_atomic_compare_exchange
 #define __CLC_COMPARE_EXCHANGE
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
@@ -21,6 +21,6 @@
 #include <clc/math/gentype.inc>
 
 #undef __CLC_COMPARE_EXCHANGE
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_COMPARE_EXCHANGE_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_dec.h b/libclc/clc/include/clc/atomic/clc_atomic_dec.h
index ada36ba3ff9b3..66302b6b9ade2 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_dec.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_dec.h
@@ -11,13 +11,13 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_dec
+#define __CLC_FUNCTION __clc_atomic_dec
 #define __CLC_NO_VALUE_ARG
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
 
 #undef __CLC_NO_VALUE_ARG
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_DEC_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_exchange.h b/libclc/clc/include/clc/atomic/clc_atomic_exchange.h
index 7e626d4a8830b..321cfb4030851 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_exchange.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_exchange.h
@@ -11,7 +11,7 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_exchange
+#define __CLC_FUNCTION __clc_atomic_exchange
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_EXCHANGE_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_add.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_add.h
index ad0c2eb4607a7..4ef29fce50af8 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_fetch_add.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_add.h
@@ -11,7 +11,7 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_fetch_add
+#define __CLC_FUNCTION __clc_atomic_fetch_add
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_ADD_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_and.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_and.h
index 80810c38cbbb8..688c11287db50 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_fetch_and.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_and.h
@@ -11,11 +11,11 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_fetch_and
+#define __CLC_FUNCTION __clc_atomic_fetch_and
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_AND_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_max.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_max.h
index 56f511922e5c7..a4e44b88a697f 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_fetch_max.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_max.h
@@ -11,7 +11,7 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_fetch_max
+#define __CLC_FUNCTION __clc_atomic_fetch_max
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_MAX_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_min.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_min.h
index f17408d28a35d..b58b538649e19 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_fetch_min.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_min.h
@@ -11,7 +11,7 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_fetch_min
+#define __CLC_FUNCTION __clc_atomic_fetch_min
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_MIN_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_or.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_or.h
index b82069e6f960e..ab303cc673ba2 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_fetch_or.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_or.h
@@ -11,11 +11,11 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_fetch_or
+#define __CLC_FUNCTION __clc_atomic_fetch_or
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_OR_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_sub.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_sub.h
index 6cfd224629d60..d9deae085a61e 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_fetch_sub.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_sub.h
@@ -11,7 +11,7 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_fetch_sub
+#define __CLC_FUNCTION __clc_atomic_fetch_sub
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_SUB_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_fetch_xor.h b/libclc/clc/include/clc/atomic/clc_atomic_fetch_xor.h
index b007b47a9369d..fe13a641fc1cd 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_fetch_xor.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_fetch_xor.h
@@ -11,11 +11,11 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_fetch_xor
+#define __CLC_FUNCTION __clc_atomic_fetch_xor
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_FETCH_XOR_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_inc.h b/libclc/clc/include/clc/atomic/clc_atomic_inc.h
index 3ddef4a8bf355..c6fcdad30949b 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_inc.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_inc.h
@@ -11,13 +11,13 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_inc
+#define __CLC_FUNCTION __clc_atomic_inc
 #define __CLC_NO_VALUE_ARG
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
 
 #undef __CLC_NO_VALUE_ARG
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_INC_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_load.h b/libclc/clc/include/clc/atomic/clc_atomic_load.h
index a4899b34b88a1..3abfce86b1947 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_load.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_load.h
@@ -11,7 +11,7 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_load
+#define __CLC_FUNCTION __clc_atomic_load
 #define __CLC_NO_VALUE_ARG
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
@@ -21,6 +21,6 @@
 #include <clc/math/gentype.inc>
 
 #undef __CLC_NO_VALUE_ARG
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_LOAD_H__
diff --git a/libclc/clc/include/clc/atomic/clc_atomic_store.h b/libclc/clc/include/clc/atomic/clc_atomic_store.h
index 6baf0eb7ea32b..94d77621735d0 100644
--- a/libclc/clc/include/clc/atomic/clc_atomic_store.h
+++ b/libclc/clc/include/clc/atomic/clc_atomic_store.h
@@ -11,7 +11,7 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_atomic_store
+#define __CLC_FUNCTION __clc_atomic_store
 #define __CLC_RETURN_VOID
 
 #define __CLC_BODY <clc/atomic/atomic_decl.inc>
@@ -21,6 +21,6 @@
 #include <clc/math/gentype.inc>
 
 #undef __CLC_RETURN_VOID
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_ATOMIC_CLC_ATOMIC_STORE_H__
diff --git a/libclc/clc/include/clc/clcfunc.h b/libclc/clc/include/clc/clcfunc.h
index 10b9cdd099900..30feaf99a4d31 100644
--- a/libclc/clc/include/clc/clcfunc.h
+++ b/libclc/clc/include/clc/clcfunc.h
@@ -12,6 +12,7 @@
 #define _CLC_OVERLOAD __attribute__((overloadable))
 #define _CLC_DECL
 #define _CLC_INLINE __attribute__((always_inline)) inline
+#define _CLC_CONST __attribute__((const))
 
 // avoid inlines for SPIR-V related targets since we'll optimise later in the
 // chain
diff --git a/libclc/clc/include/clc/clcmacro.h b/libclc/clc/include/clc/clcmacro.h
index 5c67c937cb1d4..9fa11489b1457 100644
--- a/libclc/clc/include/clc/clcmacro.h
+++ b/libclc/clc/include/clc/clcmacro.h
@@ -12,58 +12,58 @@
 #include <clc/internal/clc.h>
 #include <clc/utils.h>
 
-#define _CLC_V_V_VP_VECTORIZE(DECLSPEC, RET_TYPE, FUNCTION, ARG1_TYPE,         \
+#define _CLC_V_V_VP_VECTORIZE(DECLSPEC, RET_TYPE, __CLC_FUNCTION, ARG1_TYPE,   \
                               ADDR_SPACE, ARG2_TYPE)                           \
   DECLSPEC __CLC_XCONCAT(RET_TYPE, 2)                                          \
-      FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 2) x,                                  \
-               ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 2) * y) {                   \
+      __CLC_FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 2) x,                            \
+                     ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 2) * y) {             \
     ADDR_SPACE ARG2_TYPE *ptr = (ADDR_SPACE ARG2_TYPE *)y;                     \
-    return (__CLC_XCONCAT(RET_TYPE, 2))(FUNCTION(x.s0, ptr),                   \
-                                        FUNCTION(x.s1, ptr + 1));              \
+    return (__CLC_XCONCAT(RET_TYPE, 2))(__CLC_FUNCTION(x.s0, ptr),             \
+                                        __CLC_FUNCTION(x.s1, ptr + 1));        \
   }                                                                            \
                                                                                \
   DECLSPEC __CLC_XCONCAT(RET_TYPE, 3)                                          \
-      FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 3) x,                                  \
-               ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 3) * y) {                   \
+      __CLC_FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 3) x,                            \
+                     ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 3) * y) {             \
     ADDR_SPACE ARG2_TYPE *ptr = (ADDR_SPACE ARG2_TYPE *)y;                     \
-    return (__CLC_XCONCAT(RET_TYPE, 3))(FUNCTION(x.s0, ptr),                   \
-                                        FUNCTION(x.s1, ptr + 1),               \
-                                        FUNCTION(x.s2, ptr + 2));              \
+    return (__CLC_XCONCAT(RET_TYPE, 3))(__CLC_FUNCTION(x.s0, ptr),             \
+                                        __CLC_FUNCTION(x.s1, ptr + 1),         \
+                                        __CLC_FUNCTION(x.s2, ptr + 2));        \
   }                                                                            \
                                                                                \
   DECLSPEC __CLC_XCONCAT(RET_TYPE, 4)                                          \
-      FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 4) x,                                  \
-               ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 4) * y) {                   \
+      __CLC_FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 4) x,                            \
+                     ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 4) * y) {             \
     ADDR_SPACE ARG2_TYPE *ptr = (ADDR_SPACE ARG2_TYPE *)y;                     \
     return (__CLC_XCONCAT(RET_TYPE, 4))(                                       \
-        FUNCTION(x.s0, ptr), FUNCTION(x.s1, ptr + 1), FUNCTION(x.s2, ptr + 2), \
-        FUNCTION(x.s3, ptr + 3));                                              \
+        __CLC_FUNCTION(x.s0, ptr), __CLC_FUNCTION(x.s1, ptr + 1),              \
+        __CLC_FUNCTION(x.s2, ptr + 2), __CLC_FUNCTION(x.s3, ptr + 3));         \
   }                                                                            \
                                                                                \
   DECLSPEC __CLC_XCONCAT(RET_TYPE, 8)                                          \
-      FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 8) x,                                  \
-               ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 8) * y) {                   \
+      __CLC_FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 8) x,                            \
+                     ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 8) * y) {             \
     ADDR_SPACE ARG2_TYPE *ptr = (ADDR_SPACE ARG2_TYPE *)y;                     \
     return (__CLC_XCONCAT(RET_TYPE, 8))(                                       \
-        FUNCTION(x.s0, ptr), FUNCTION(x.s1, ptr + 1), FUNCTION(x.s2, ptr + 2), \
-        FUNCTION(x.s3, ptr + 3), FUNCTION(x.s4, ptr + 4),                      \
-        FUNCTION(x.s5, ptr + 5), FUNCTION(x.s6, ptr + 6),                      \
-        FUNCTION(x.s7, ptr + 7));                                              \
+        __CLC_FUNCTION(x.s0, ptr), __CLC_FUNCTION(x.s1, ptr + 1),              \
+        __CLC_FUNCTION(x.s2, ptr + 2), __CLC_FUNCTION(x.s3, ptr + 3),          \
+        __CLC_FUNCTION(x.s4, ptr + 4), __CLC_FUNCTION(x.s5, ptr + 5),          \
+        __CLC_FUNCTION(x.s6, ptr + 6), __CLC_FUNCTION(x.s7, ptr + 7));         \
   }                                                                            \
                                                                                \
   DECLSPEC __CLC_XCONCAT(RET_TYPE, 16)                                         \
-      FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 16) x,                                 \
-               ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 16) * y) {                  \
+      __CLC_FUNCTION(__CLC_XCONCAT(ARG1_TYPE, 16) x,                           \
+                     ADDR_SPACE __CLC_XCONCAT(ARG2_TYPE, 16) * y) {            \
     ADDR_SPACE ARG2_TYPE *ptr = (ADDR_SPACE ARG2_TYPE *)y;                     \
     return (__CLC_XCONCAT(RET_TYPE, 16))(                                      \
-        FUNCTION(x.s0, ptr), FUNCTION(x.s1, ptr + 1), FUNCTION(x.s2, ptr + 2), \
-        FUNCTION(x.s3, ptr + 3), FUNCTION(x.s4, ptr + 4),                      \
-        FUNCTION(x.s5, ptr + 5), FUNCTION(x.s6, ptr + 6),                      \
-        FUNCTION(x.s7, ptr + 7), FUNCTION(x.s8, ptr + 8),                      \
-        FUNCTION(x.s9, ptr + 9), FUNCTION(x.sa, ptr + 10),                     \
-        FUNCTION(x.sb, ptr + 11), FUNCTION(x.sc, ptr + 12),                    \
-        FUNCTION(x.sd, ptr + 13), FUNCTION(x.se, ptr + 14),                    \
-        FUNCTION(x.sf, ptr + 15));                                             \
+        __CLC_FUNCTION(x.s0, ptr), __CLC_FUNCTION(x.s1, ptr + 1),              \
+        __CLC_FUNCTION(x.s2, ptr + 2), __CLC_FUNCTION(x.s3, ptr + 3),          \
+        __CLC_FUNCTION(x.s4, ptr + 4), __CLC_FUNCTION(x.s5, ptr + 5),          \
+        __CLC_FUNCTION(x.s6, ptr + 6), __CLC_FUNCTION(x.s7, ptr + 7),          \
+        __CLC_FUNCTION(x.s8, ptr + 8), __CLC_FUNCTION(x.s9, ptr + 9),          \
+        __CLC_FUNCTION(x.sa, ptr + 10), __CLC_FUNCTION(x.sb, ptr + 11),        \
+        __CLC_FUNCTION(x.sc, ptr + 12), __CLC_FUNCTION(x.sd, ptr + 13),        \
+        __CLC_FUNCTION(x.se, ptr + 14), __CLC_FUNCTION(x.sf, ptr + 15));       \
   }
 
 #endif // __CLC_CLCMACRO_H__
diff --git a/libclc/clc/include/clc/common/clc_degrees.h b/libclc/clc/include/clc/common/clc_degrees.h
index e633344c291c1..5f04fac1f2612 100644
--- a/libclc/clc/include/clc/common/clc_degrees.h
+++ b/libclc/clc/include/clc/common/clc_degrees.h
@@ -10,10 +10,10 @@
 #define __CLC_COMMON_CLC_DEGREES_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_degrees
+#define __CLC_FUNCTION __clc_degrees
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_COMMON_CLC_DEGREES_H__
diff --git a/libclc/clc/include/clc/common/clc_radians.h b/libclc/clc/include/clc/common/clc_radians.h
index 0b075efbcce2f..03d7c4b9e3014 100644
--- a/libclc/clc/include/clc/common/clc_radians.h
+++ b/libclc/clc/include/clc/common/clc_radians.h
@@ -10,10 +10,10 @@
 #define __CLC_COMMON_CLC_RADIANS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_radians
+#define __CLC_FUNCTION __clc_radians
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_COMMON_CLC_RADIANS_H__
diff --git a/libclc/clc/include/clc/common/clc_sign.h b/libclc/clc/include/clc/common/clc_sign.h
index 0a8c4cc94d80b..9737f1e54441b 100644
--- a/libclc/clc/include/clc/common/clc_sign.h
+++ b/libclc/clc/include/clc/common/clc_sign.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_COMMON_CLC_SIGN_H__
 #define __CLC_COMMON_CLC_SIGN_H__
 
-#define FUNCTION __clc_sign
+#define __CLC_FUNCTION __clc_sign
 #define __CLC_BODY <clc/math/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_COMMON_CLC_SIGN_H__
diff --git a/libclc/clc/include/clc/common/clc_smoothstep.inc b/libclc/clc/include/clc/common/clc_smoothstep.inc
index 158c578d73d95..e3752e893ac17 100644
--- a/libclc/clc/include/clc/common/clc_smoothstep.inc
+++ b/libclc/clc/include/clc/common/clc_smoothstep.inc
@@ -6,6 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_smoothstep(__CLC_GENTYPE edge0,
-                                                       __CLC_GENTYPE edge1,
-                                                       __CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__clc_smoothstep(__CLC_GENTYPE edge0, __CLC_GENTYPE edge1, __CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/common/clc_step.h b/libclc/clc/include/clc/common/clc_step.h
index 96b4759cabf77..6b093d06896c0 100644
--- a/libclc/clc/include/clc/common/clc_step.h
+++ b/libclc/clc/include/clc/common/clc_step.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_COMMON_CLC_STEP_H__
 #define __CLC_COMMON_CLC_STEP_H__
 
-#define FUNCTION __clc_step
+#define __CLC_FUNCTION __clc_step
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_COMMON_CLC_STEP_H__
diff --git a/libclc/clc/include/clc/geometric/binary_decl.inc b/libclc/clc/include/clc/geometric/binary_decl.inc
index 342d97a802659..ab64c9bece324 100644
--- a/libclc/clc/include/clc/geometric/binary_decl.inc
+++ b/libclc/clc/include/clc/geometric/binary_decl.inc
@@ -10,7 +10,7 @@
 #if (__CLC_VECSIZE_OR_1 == 1 || __CLC_VECSIZE_OR_1 == 2 ||                     \
      __CLC_VECSIZE_OR_1 == 3 || __CLC_VECSIZE_OR_1 == 4)
 
-_CLC_OVERLOAD _CLC_DECL __CLC_SCALAR_GENTYPE FUNCTION(__CLC_GENTYPE a,
-                                                      __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_SCALAR_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b);
 
 #endif
diff --git a/libclc/clc/include/clc/geometric/binary_def.inc b/libclc/clc/include/clc/geometric/binary_def.inc
index ad4fa4c01683b..bbd4f57980023 100644
--- a/libclc/clc/include/clc/geometric/binary_def.inc
+++ b/libclc/clc/include/clc/geometric/binary_def.inc
@@ -8,17 +8,17 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // Geometric functions are only defined for scalar, vec2, vec3 and vec4
 #if (__CLC_VECSIZE_OR_1 == 1 || __CLC_VECSIZE_OR_1 == 2 ||                     \
      __CLC_VECSIZE_OR_1 == 3 || __CLC_VECSIZE_OR_1 == 4)
 
-_CLC_OVERLOAD _CLC_DEF __CLC_SCALAR_GENTYPE FUNCTION(__CLC_GENTYPE a,
-                                                     __CLC_GENTYPE b) {
-  return __IMPL_FUNCTION(FUNCTION)(a, b);
+_CLC_OVERLOAD _CLC_DEF __CLC_SCALAR_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
+                                                           __CLC_GENTYPE b) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a, b);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/geometric/clc_cross.h b/libclc/clc/include/clc/geometric/clc_cross.h
index e5aa913abfa29..c0a9cacbc3383 100644
--- a/libclc/clc/include/clc/geometric/clc_cross.h
+++ b/libclc/clc/include/clc/geometric/clc_cross.h
@@ -11,22 +11,22 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL float3 __clc_cross(float3 p0, float3 p1);
-_CLC_OVERLOAD _CLC_DECL float4 __clc_cross(float4 p0, float4 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL float3 __clc_cross(float3 p0, float3 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL float4 __clc_cross(float4 p0, float4 p1);
 
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-_CLC_OVERLOAD _CLC_DECL double3 __clc_cross(double3 p0, double3 p1);
-_CLC_OVERLOAD _CLC_DECL double4 __clc_cross(double4 p0, double4 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL double3 __clc_cross(double3 p0, double3 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL double4 __clc_cross(double4 p0, double4 p1);
 
 #endif
 
 #ifdef cl_khr_fp16
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
-_CLC_OVERLOAD _CLC_DECL half3 __clc_cross(half3 p0, half3 p1);
-_CLC_OVERLOAD _CLC_DECL half4 __clc_cross(half4 p0, half4 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL half3 __clc_cross(half3 p0, half3 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL half4 __clc_cross(half4 p0, half4 p1);
 
 #endif
 
diff --git a/libclc/clc/include/clc/geometric/clc_distance.h b/libclc/clc/include/clc/geometric/clc_distance.h
index 666e2c881bc9a..1acea411a08b6 100644
--- a/libclc/clc/include/clc/geometric/clc_distance.h
+++ b/libclc/clc/include/clc/geometric/clc_distance.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_GEOMETRIC_CLC_DISTANCE_H__
 #define __CLC_GEOMETRIC_CLC_DISTANCE_H__
 
-#define FUNCTION __clc_distance
+#define __CLC_FUNCTION __clc_distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_DISTANCE_H__
diff --git a/libclc/clc/include/clc/geometric/clc_dot.h b/libclc/clc/include/clc/geometric/clc_dot.h
index 4bfed6e3abc51..a5a97d1bdd2bc 100644
--- a/libclc/clc/include/clc/geometric/clc_dot.h
+++ b/libclc/clc/include/clc/geometric/clc_dot.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_GEOMETRIC_CLC_DOT_H__
 #define __CLC_GEOMETRIC_CLC_DOT_H__
 
-#define FUNCTION __clc_dot
+#define __CLC_FUNCTION __clc_dot
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_DOT_H__
diff --git a/libclc/clc/include/clc/geometric/clc_fast_distance.h b/libclc/clc/include/clc/geometric/clc_fast_distance.h
index 2932d0147a6d8..47292130e9d68 100644
--- a/libclc/clc/include/clc/geometric/clc_fast_distance.h
+++ b/libclc/clc/include/clc/geometric/clc_fast_distance.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_GEOMETRIC_CLC_FAST_DISTANCE_H__
 #define __CLC_GEOMETRIC_CLC_FAST_DISTANCE_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_fast_distance
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_fast_distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_FAST_DISTANCE_H__
diff --git a/libclc/clc/include/clc/geometric/clc_fast_length.h b/libclc/clc/include/clc/geometric/clc_fast_length.h
index 903ecc8b91755..2244d8c77feaa 100644
--- a/libclc/clc/include/clc/geometric/clc_fast_length.h
+++ b/libclc/clc/include/clc/geometric/clc_fast_length.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_GEOMETRIC_CLC_FAST_LENGTH_H__
 #define __CLC_GEOMETRIC_CLC_FAST_LENGTH_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_fast_length
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_fast_length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_FAST_LENGTH_H__
diff --git a/libclc/clc/include/clc/geometric/clc_fast_normalize.h b/libclc/clc/include/clc/geometric/clc_fast_normalize.h
index a91862fe61cb7..ee9747c1af1e3 100644
--- a/libclc/clc/include/clc/geometric/clc_fast_normalize.h
+++ b/libclc/clc/include/clc/geometric/clc_fast_normalize.h
@@ -9,13 +9,13 @@
 #ifndef __CLC_GEOMETRIC_CLC_FAST_NORMALIZE_H__
 #define __CLC_GEOMETRIC_CLC_FAST_NORMALIZE_H__
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_GEOMETRIC_RET_GENTYPE
-#define FUNCTION __clc_fast_normalize
+#define __CLC_FUNCTION __clc_fast_normalize
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 #undef __CLC_GEOMETRIC_RET_GENTYPE
 
 #endif // __CLC_GEOMETRIC_CLC_FAST_NORMALIZE_H__
diff --git a/libclc/clc/include/clc/geometric/clc_length.h b/libclc/clc/include/clc/geometric/clc_length.h
index 35b206e6c3fbb..f8ce12db07871 100644
--- a/libclc/clc/include/clc/geometric/clc_length.h
+++ b/libclc/clc/include/clc/geometric/clc_length.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_GEOMETRIC_CLC_LENGTH_H__
 #define __CLC_GEOMETRIC_CLC_LENGTH_H__
 
-#define FUNCTION __clc_length
+#define __CLC_FUNCTION __clc_length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_GEOMETRIC_CLC_LENGTH_H__
diff --git a/libclc/clc/include/clc/geometric/clc_normalize.h b/libclc/clc/include/clc/geometric/clc_normalize.h
index 745ed7f4887d1..3058a72b2bbbe 100644
--- a/libclc/clc/include/clc/geometric/clc_normalize.h
+++ b/libclc/clc/include/clc/geometric/clc_normalize.h
@@ -10,11 +10,11 @@
 #define __CLC_GEOMETRIC_CLC_NORMALIZE_H__
 
 #define __CLC_GEOMETRIC_RET_GENTYPE
-#define FUNCTION __clc_normalize
+#define __CLC_FUNCTION __clc_normalize
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 #undef __CLC_GEOMETRIC_RET_GENTYPE
 
 #endif // __CLC_GEOMETRIC_CLC_NORMALIZE_H__
diff --git a/libclc/clc/include/clc/geometric/unary_decl.inc b/libclc/clc/include/clc/geometric/unary_decl.inc
index cff66000fcf77..6641d28d09678 100644
--- a/libclc/clc/include/clc/geometric/unary_decl.inc
+++ b/libclc/clc/include/clc/geometric/unary_decl.inc
@@ -10,12 +10,12 @@
 #if (__CLC_VECSIZE_OR_1 == 1 || __CLC_VECSIZE_OR_1 == 2 ||                     \
      __CLC_VECSIZE_OR_1 == 3 || __CLC_VECSIZE_OR_1 == 4)
 
-_CLC_OVERLOAD _CLC_DECL
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL
 #ifdef __CLC_GEOMETRIC_RET_GENTYPE
     __CLC_GENTYPE
 #else
     __CLC_SCALAR_GENTYPE
 #endif
-    FUNCTION(__CLC_GENTYPE a);
+    __CLC_FUNCTION(__CLC_GENTYPE a);
 
 #endif
diff --git a/libclc/clc/include/clc/geometric/unary_def.inc b/libclc/clc/include/clc/geometric/unary_def.inc
index ea90de0ec581d..78c144b35af2c 100644
--- a/libclc/clc/include/clc/geometric/unary_def.inc
+++ b/libclc/clc/include/clc/geometric/unary_def.inc
@@ -8,8 +8,8 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // Geometric functions are only defined for scalar, vec2, vec3 and vec4
@@ -22,8 +22,8 @@ _CLC_OVERLOAD _CLC_DEF
 #else
     __CLC_SCALAR_GENTYPE
 #endif
-    FUNCTION(__CLC_GENTYPE a) {
-  return __IMPL_FUNCTION(FUNCTION)(a);
+    __CLC_FUNCTION(__CLC_GENTYPE a) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/integer/clc_abs.inc b/libclc/clc/include/clc/integer/clc_abs.inc
index f1e2c5a5149cb..4c34b0de6f3ee 100644
--- a/libclc/clc/include/clc/integer/clc_abs.inc
+++ b/libclc/clc/include/clc/integer/clc_abs.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_U_GENTYPE __clc_abs(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_U_GENTYPE __clc_abs(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/integer/clc_abs_diff.inc b/libclc/clc/include/clc/integer/clc_abs_diff.inc
index 2514a37b78144..84ef15cdb8ae8 100644
--- a/libclc/clc/include/clc/integer/clc_abs_diff.inc
+++ b/libclc/clc/include/clc/integer/clc_abs_diff.inc
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_U_GENTYPE __clc_abs_diff(__CLC_GENTYPE x,
-                                                       __CLC_GENTYPE y);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_U_GENTYPE
+__clc_abs_diff(__CLC_GENTYPE x, __CLC_GENTYPE y);
diff --git a/libclc/clc/include/clc/integer/clc_add_sat.h b/libclc/clc/include/clc/integer/clc_add_sat.h
index be095d38225e6..8c3495f6c5193 100644
--- a/libclc/clc/include/clc/integer/clc_add_sat.h
+++ b/libclc/clc/include/clc/integer/clc_add_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_ADD_SAT_H__
 #define __CLC_INTEGER_CLC_ADD_SAT_H__
 
-#define FUNCTION __clc_add_sat
+#define __CLC_FUNCTION __clc_add_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_ADD_SAT_H__
diff --git a/libclc/clc/include/clc/integer/clc_bit_reverse.h b/libclc/clc/include/clc/integer/clc_bit_reverse.h
index c945e326c74fa..de0c354b3fa4e 100644
--- a/libclc/clc/include/clc/integer/clc_bit_reverse.h
+++ b/libclc/clc/include/clc/integer/clc_bit_reverse.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_BIT_REVERSE_H__
 #define __CLC_INTEGER_CLC_BIT_REVERSE_H__
 
-#define FUNCTION __clc_bit_reverse
+#define __CLC_FUNCTION __clc_bit_reverse
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_BIT_REVERSE_H__
diff --git a/libclc/clc/include/clc/integer/clc_bitfield_extract_decl.inc b/libclc/clc/include/clc/integer/clc_bitfield_extract_decl.inc
index c93eff08de0bc..2f23667011209 100644
--- a/libclc/clc/include/clc/integer/clc_bitfield_extract_decl.inc
+++ b/libclc/clc/include/clc/integer/clc_bitfield_extract_decl.inc
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __RETTYPE FUNCTION(__CLC_GENTYPE base, uint offset,
-                                           uint count);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_RETTYPE
+__CLC_FUNCTION(__CLC_GENTYPE base, uint offset, uint count);
diff --git a/libclc/clc/include/clc/integer/clc_bitfield_extract_signed.h b/libclc/clc/include/clc/integer/clc_bitfield_extract_signed.h
index 9c2e047b8be00..7998196512b30 100644
--- a/libclc/clc/include/clc/integer/clc_bitfield_extract_signed.h
+++ b/libclc/clc/include/clc/integer/clc_bitfield_extract_signed.h
@@ -11,13 +11,13 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_bitfield_extract_signed
-#define __RETTYPE __CLC_S_GENTYPE
+#define __CLC_FUNCTION __clc_bitfield_extract_signed
+#define __CLC_RETTYPE __CLC_S_GENTYPE
 
 #define __CLC_BODY <clc/integer/clc_bitfield_extract_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef __RETTYPE
-#undef FUNCTION
+#undef __CLC_RETTYPE
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_BITFIELD_EXTRACT_SIGNED_H__
diff --git a/libclc/clc/include/clc/integer/clc_bitfield_extract_unsigned.h b/libclc/clc/include/clc/integer/clc_bitfield_extract_unsigned.h
index 95305a3027e5d..84614056417f4 100644
--- a/libclc/clc/include/clc/integer/clc_bitfield_extract_unsigned.h
+++ b/libclc/clc/include/clc/integer/clc_bitfield_extract_unsigned.h
@@ -11,13 +11,13 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_bitfield_extract_unsigned
-#define __RETTYPE __CLC_U_GENTYPE
+#define __CLC_FUNCTION __clc_bitfield_extract_unsigned
+#define __CLC_RETTYPE __CLC_U_GENTYPE
 
 #define __CLC_BODY <clc/integer/clc_bitfield_extract_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef __RETTYPE
-#undef FUNCTION
+#undef __CLC_RETTYPE
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_BITFIELD_EXTRACT_SIGNED_H__
diff --git a/libclc/clc/include/clc/integer/clc_bitfield_insert.h b/libclc/clc/include/clc/integer/clc_bitfield_insert.h
index f4d36b2ad2d2e..b284c0bf932d5 100644
--- a/libclc/clc/include/clc/integer/clc_bitfield_insert.h
+++ b/libclc/clc/include/clc/integer/clc_bitfield_insert.h
@@ -11,7 +11,7 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_bitfield_insert
+#define __CLC_FUNCTION __clc_bitfield_insert
 #define __CLC_BODY <clc/integer/clc_bitfield_insert.inc>
 #include <clc/integer/gentype.inc>
 
diff --git a/libclc/clc/include/clc/integer/clc_bitfield_insert.inc b/libclc/clc/include/clc/integer/clc_bitfield_insert.inc
index 22f58bdc09830..00a363fa41129 100644
--- a/libclc/clc/include/clc/integer/clc_bitfield_insert.inc
+++ b/libclc/clc/include/clc/integer/clc_bitfield_insert.inc
@@ -6,6 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE base,
-                                               __CLC_GENTYPE insert,
-                                               uint offset, uint count);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(
+    __CLC_GENTYPE base, __CLC_GENTYPE insert, uint offset, uint count);
diff --git a/libclc/clc/include/clc/integer/clc_clz.h b/libclc/clc/include/clc/integer/clc_clz.h
index 3e0530bad6f11..dd62cc935f8cd 100644
--- a/libclc/clc/include/clc/integer/clc_clz.h
+++ b/libclc/clc/include/clc/integer/clc_clz.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_CLZ_H__
 #define __CLC_INTEGER_CLC_CLZ_H__
 
-#define FUNCTION __clc_clz
+#define __CLC_FUNCTION __clc_clz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_CLZ_H__
diff --git a/libclc/clc/include/clc/integer/clc_ctz.h b/libclc/clc/include/clc/integer/clc_ctz.h
index 1495dc5560f1d..3f2079e023f67 100644
--- a/libclc/clc/include/clc/integer/clc_ctz.h
+++ b/libclc/clc/include/clc/integer/clc_ctz.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_CTZ_H__
 #define __CLC_INTEGER_CLC_CTZ_H__
 
-#define FUNCTION __clc_ctz
+#define __CLC_FUNCTION __clc_ctz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_CTZ_H__
diff --git a/libclc/clc/include/clc/integer/clc_hadd.h b/libclc/clc/include/clc/integer/clc_hadd.h
index 7aaee9b601a26..679369d3c9d4e 100644
--- a/libclc/clc/include/clc/integer/clc_hadd.h
+++ b/libclc/clc/include/clc/integer/clc_hadd.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_HADD_H__
 #define __CLC_INTEGER_CLC_HADD_H__
 
-#define FUNCTION __clc_hadd
+#define __CLC_FUNCTION __clc_hadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_HADD_H__
diff --git a/libclc/clc/include/clc/integer/clc_mad24.h b/libclc/clc/include/clc/integer/clc_mad24.h
index 121840f81d63e..e19bd905d22eb 100644
--- a/libclc/clc/include/clc/integer/clc_mad24.h
+++ b/libclc/clc/include/clc/integer/clc_mad24.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MAD24_H__
 #define __CLC_INTEGER_CLC_MAD24_H__
 
-#define FUNCTION __clc_mad24
+#define __CLC_FUNCTION __clc_mad24
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MAD24_H__
diff --git a/libclc/clc/include/clc/integer/clc_mad_sat.h b/libclc/clc/include/clc/integer/clc_mad_sat.h
index 0c418068ca4ac..845378b3e7c46 100644
--- a/libclc/clc/include/clc/integer/clc_mad_sat.h
+++ b/libclc/clc/include/clc/integer/clc_mad_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MAD_SAT_H__
 #define __CLC_INTEGER_CLC_MAD_SAT_H__
 
-#define FUNCTION __clc_mad_sat
+#define __CLC_FUNCTION __clc_mad_sat
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MAD_SAT_H__
diff --git a/libclc/clc/include/clc/integer/clc_mul24.h b/libclc/clc/include/clc/integer/clc_mul24.h
index 9310458d59817..f813de55c71e5 100644
--- a/libclc/clc/include/clc/integer/clc_mul24.h
+++ b/libclc/clc/include/clc/integer/clc_mul24.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MUL24_H__
 #define __CLC_INTEGER_CLC_MUL24_H__
 
-#define FUNCTION __clc_mul24
+#define __CLC_FUNCTION __clc_mul24
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MUL24_H__
diff --git a/libclc/clc/include/clc/integer/clc_mul_hi.h b/libclc/clc/include/clc/integer/clc_mul_hi.h
index 6542f237d09bc..e4738e5570e81 100644
--- a/libclc/clc/include/clc/integer/clc_mul_hi.h
+++ b/libclc/clc/include/clc/integer/clc_mul_hi.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_MUL_HI_H__
 #define __CLC_INTEGER_CLC_MUL_HI_H__
 
-#define FUNCTION __clc_mul_hi
+#define __CLC_FUNCTION __clc_mul_hi
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_MUL_HI_H__
diff --git a/libclc/clc/include/clc/integer/clc_popcount.h b/libclc/clc/include/clc/integer/clc_popcount.h
index 8ece2e998862d..55ca6ccfcec28 100644
--- a/libclc/clc/include/clc/integer/clc_popcount.h
+++ b/libclc/clc/include/clc/integer/clc_popcount.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_INTEGER_CLC_POPCOUNT_H__
 #define __CLC_INTEGER_CLC_POPCOUNT_H__
 
-#define FUNCTION __clc_popcount
+#define __CLC_FUNCTION __clc_popcount
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
 #undef __CLC_INTRINSIC
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_POPCOUNT_H__
diff --git a/libclc/clc/include/clc/integer/clc_rhadd.h b/libclc/clc/include/clc/integer/clc_rhadd.h
index 882f950656dfb..b8b0b1a3a56ed 100644
--- a/libclc/clc/include/clc/integer/clc_rhadd.h
+++ b/libclc/clc/include/clc/integer/clc_rhadd.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_RHADD_H__
 #define __CLC_INTEGER_CLC_RHADD_H__
 
-#define FUNCTION __clc_rhadd
+#define __CLC_FUNCTION __clc_rhadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_RHADD_H__
diff --git a/libclc/clc/include/clc/integer/clc_rotate.h b/libclc/clc/include/clc/integer/clc_rotate.h
index 61cf08788ef3b..513bf98526233 100644
--- a/libclc/clc/include/clc/integer/clc_rotate.h
+++ b/libclc/clc/include/clc/integer/clc_rotate.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_ROTATE_H__
 #define __CLC_INTEGER_CLC_ROTATE_H__
 
-#define FUNCTION __clc_rotate
+#define __CLC_FUNCTION __clc_rotate
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_ROTATE_H__
diff --git a/libclc/clc/include/clc/integer/clc_sub_sat.h b/libclc/clc/include/clc/integer/clc_sub_sat.h
index 1b694a4a1780a..94a5484adefcf 100644
--- a/libclc/clc/include/clc/integer/clc_sub_sat.h
+++ b/libclc/clc/include/clc/integer/clc_sub_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTEGER_CLC_SUB_SAT_H__
 #define __CLC_INTEGER_CLC_SUB_SAT_H__
 
-#define FUNCTION __clc_sub_sat
+#define __CLC_FUNCTION __clc_sub_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTEGER_CLC_SUB_SAT_H__
diff --git a/libclc/clc/include/clc/internal/math/clc_sw_fma.h b/libclc/clc/include/clc/internal/math/clc_sw_fma.h
index 8497f4efd1137..5d6c76879ceb9 100644
--- a/libclc/clc/include/clc/internal/math/clc_sw_fma.h
+++ b/libclc/clc/include/clc/internal/math/clc_sw_fma.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
 #define __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
 
-#define FUNCTION __clc_sw_fma
+#define __CLC_FUNCTION __clc_sw_fma
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_INTERNAL_MATH_CLC_SW_FMA_H__
diff --git a/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc b/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc
index 1b2b9ff33521b..b9acf8c4af331 100644
--- a/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc
+++ b/libclc/clc/include/clc/math/binary_decl_with_scalar_second_arg.inc
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
-                                               __CLC_GENTYPE b);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
-                                               __CLC_SCALAR_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE a, __CLC_SCALAR_GENTYPE b);
diff --git a/libclc/clc/include/clc/math/binary_def_via_fp32.inc b/libclc/clc/include/clc/math/binary_def_via_fp32.inc
index b53a178bf0f10..ae22903dd3d0b 100644
--- a/libclc/clc/include/clc/math/binary_def_via_fp32.inc
+++ b/libclc/clc/include/clc/math/binary_def_via_fp32.inc
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                              __CLC_GENTYPE y) {
+_CLC_OVERLOAD _CLC_CONST _CLC_DEF __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y) {
   return __CLC_CONVERT_GENTYPE(
-      FUNCTION(__CLC_CONVERT_FLOATN(x), __CLC_CONVERT_FLOATN(y)));
+      __CLC_FUNCTION(__CLC_CONVERT_FLOATN(x), __CLC_CONVERT_FLOATN(y)));
 }
diff --git a/libclc/clc/include/clc/math/clc_acos.h b/libclc/clc/include/clc/math/clc_acos.h
index 130e3111ec1ca..1d484fa5b56c9 100644
--- a/libclc/clc/include/clc/math/clc_acos.h
+++ b/libclc/clc/include/clc/math/clc_acos.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ACOS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_acos
+#define __CLC_FUNCTION __clc_acos
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ACOS_H__
diff --git a/libclc/clc/include/clc/math/clc_acosh.h b/libclc/clc/include/clc/math/clc_acosh.h
index 9b8c7184b9742..9492724bb27ba 100644
--- a/libclc/clc/include/clc/math/clc_acosh.h
+++ b/libclc/clc/include/clc/math/clc_acosh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ACOSH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_acosh
+#define __CLC_FUNCTION __clc_acosh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ACOSH_H__
diff --git a/libclc/clc/include/clc/math/clc_acospi.h b/libclc/clc/include/clc/math/clc_acospi.h
index 6d4dc694900a0..90cfc06b7dffb 100644
--- a/libclc/clc/include/clc/math/clc_acospi.h
+++ b/libclc/clc/include/clc/math/clc_acospi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ACOSPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_acospi
+#define __CLC_FUNCTION __clc_acospi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ACOSPI_H__
diff --git a/libclc/clc/include/clc/math/clc_asin.h b/libclc/clc/include/clc/math/clc_asin.h
index f3d69a7bf9956..f6246c49f8022 100644
--- a/libclc/clc/include/clc/math/clc_asin.h
+++ b/libclc/clc/include/clc/math/clc_asin.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ASIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_asin
+#define __CLC_FUNCTION __clc_asin
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ASIN_H__
diff --git a/libclc/clc/include/clc/math/clc_asinh.h b/libclc/clc/include/clc/math/clc_asinh.h
index dc4530fa68b82..05e44fa438cee 100644
--- a/libclc/clc/include/clc/math/clc_asinh.h
+++ b/libclc/clc/include/clc/math/clc_asinh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ASINH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_asinh
+#define __CLC_FUNCTION __clc_asinh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ASINH_H__
diff --git a/libclc/clc/include/clc/math/clc_asinpi.h b/libclc/clc/include/clc/math/clc_asinpi.h
index 19d8ec3ad0c50..8f308dd06022e 100644
--- a/libclc/clc/include/clc/math/clc_asinpi.h
+++ b/libclc/clc/include/clc/math/clc_asinpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ASINPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_asinpi
+#define __CLC_FUNCTION __clc_asinpi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ASINPI_H__
diff --git a/libclc/clc/include/clc/math/clc_atan.h b/libclc/clc/include/clc/math/clc_atan.h
index 9f10efeae976c..203e46c2b8a4a 100644
--- a/libclc/clc/include/clc/math/clc_atan.h
+++ b/libclc/clc/include/clc/math/clc_atan.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_atan
+#define __CLC_FUNCTION __clc_atan
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ATAN_H__
diff --git a/libclc/clc/include/clc/math/clc_atan2.h b/libclc/clc/include/clc/math/clc_atan2.h
index 99bd0760f3fe8..6edc9dda48f4d 100644
--- a/libclc/clc/include/clc/math/clc_atan2.h
+++ b/libclc/clc/include/clc/math/clc_atan2.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATAN2_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_atan2
+#define __CLC_FUNCTION __clc_atan2
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ATAN2_H__
diff --git a/libclc/clc/include/clc/math/clc_atan2pi.h b/libclc/clc/include/clc/math/clc_atan2pi.h
index ae044b7bbfafb..2f26d1d9ae985 100644
--- a/libclc/clc/include/clc/math/clc_atan2pi.h
+++ b/libclc/clc/include/clc/math/clc_atan2pi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATAN2PI_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_atan2pi
+#define __CLC_FUNCTION __clc_atan2pi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ATAN2PI_H__
diff --git a/libclc/clc/include/clc/math/clc_atanh.h b/libclc/clc/include/clc/math/clc_atanh.h
index c1a95d4acba1b..85f9fc53be1a6 100644
--- a/libclc/clc/include/clc/math/clc_atanh.h
+++ b/libclc/clc/include/clc/math/clc_atanh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATANH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_atanh
+#define __CLC_FUNCTION __clc_atanh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ATANH_H__
diff --git a/libclc/clc/include/clc/math/clc_atanpi.h b/libclc/clc/include/clc/math/clc_atanpi.h
index 43f752ecfe98b..2b3c6b373ed42 100644
--- a/libclc/clc/include/clc/math/clc_atanpi.h
+++ b/libclc/clc/include/clc/math/clc_atanpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ATANPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_atanpi
+#define __CLC_FUNCTION __clc_atanpi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ATANPI_H__
diff --git a/libclc/clc/include/clc/math/clc_cbrt.inc b/libclc/clc/include/clc/math/clc_cbrt.inc
index 019b5174a9015..fd9b750c8fd89 100644
--- a/libclc/clc/include/clc/math/clc_cbrt.inc
+++ b/libclc/clc/include/clc/math/clc_cbrt.inc
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_CBRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_cbrt
+#define __CLC_FUNCTION __clc_cbrt
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_CBRT_H__
diff --git a/libclc/clc/include/clc/math/clc_ceil.h b/libclc/clc/include/clc/math/clc_ceil.h
index c60b794c83c6c..45668224c6176 100644
--- a/libclc/clc/include/clc/math/clc_ceil.h
+++ b/libclc/clc/include/clc/math/clc_ceil.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_CEIL_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_ceil
+#define __CLC_FUNCTION __clc_ceil
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_CEIL_H__
diff --git a/libclc/clc/include/clc/math/clc_copysign.h b/libclc/clc/include/clc/math/clc_copysign.h
index 5976e42de250f..563dd2827839a 100644
--- a/libclc/clc/include/clc/math/clc_copysign.h
+++ b/libclc/clc/include/clc/math/clc_copysign.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COPYSIGN_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_copysign
+#define __CLC_FUNCTION __clc_copysign
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_COPYSIGN_H__
diff --git a/libclc/clc/include/clc/math/clc_cos.h b/libclc/clc/include/clc/math/clc_cos.h
index 3a1ae9342e265..44681608efc37 100644
--- a/libclc/clc/include/clc/math/clc_cos.h
+++ b/libclc/clc/include/clc/math/clc_cos.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_cos
+#define __CLC_FUNCTION __clc_cos
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_COS_H__
diff --git a/libclc/clc/include/clc/math/clc_cosh.h b/libclc/clc/include/clc/math/clc_cosh.h
index 3033d76000dc0..7bcfa2ea26e06 100644
--- a/libclc/clc/include/clc/math/clc_cosh.h
+++ b/libclc/clc/include/clc/math/clc_cosh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COSH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_cosh
+#define __CLC_FUNCTION __clc_cosh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_COSH_H__
diff --git a/libclc/clc/include/clc/math/clc_cospi.h b/libclc/clc/include/clc/math/clc_cospi.h
index 3e4e73f096a06..4d82100649b01 100644
--- a/libclc/clc/include/clc/math/clc_cospi.h
+++ b/libclc/clc/include/clc/math/clc_cospi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_COSPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_cospi
+#define __CLC_FUNCTION __clc_cospi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_COSPI_H__
diff --git a/libclc/clc/include/clc/math/clc_erf.h b/libclc/clc/include/clc/math/clc_erf.h
index 04b990c65133f..01a21b36b352f 100644
--- a/libclc/clc/include/clc/math/clc_erf.h
+++ b/libclc/clc/include/clc/math/clc_erf.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ERF_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_erf
+#define __CLC_FUNCTION __clc_erf
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ERF_H__
diff --git a/libclc/clc/include/clc/math/clc_erfc.h b/libclc/clc/include/clc/math/clc_erfc.h
index be2578b4bfb04..efd581542879f 100644
--- a/libclc/clc/include/clc/math/clc_erfc.h
+++ b/libclc/clc/include/clc/math/clc_erfc.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ERFC_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_erfc
+#define __CLC_FUNCTION __clc_erfc
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ERFC_H__
diff --git a/libclc/clc/include/clc/math/clc_exp.h b/libclc/clc/include/clc/math/clc_exp.h
index 355148934427f..84369bb8fbdd6 100644
--- a/libclc/clc/include/clc/math/clc_exp.h
+++ b/libclc/clc/include/clc/math/clc_exp.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXP_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_exp
+#define __CLC_FUNCTION __clc_exp
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_EXP_H__
diff --git a/libclc/clc/include/clc/math/clc_exp10.h b/libclc/clc/include/clc/math/clc_exp10.h
index c281ee84dd3a1..500271cb07429 100644
--- a/libclc/clc/include/clc/math/clc_exp10.h
+++ b/libclc/clc/include/clc/math/clc_exp10.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXP10_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_exp10
+#define __CLC_FUNCTION __clc_exp10
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_EXP10_H__
diff --git a/libclc/clc/include/clc/math/clc_exp2.h b/libclc/clc/include/clc/math/clc_exp2.h
index bb10f12d34e2e..4ad0ddea48a7c 100644
--- a/libclc/clc/include/clc/math/clc_exp2.h
+++ b/libclc/clc/include/clc/math/clc_exp2.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXP2_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_exp2
+#define __CLC_FUNCTION __clc_exp2
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_EXP2_H__
diff --git a/libclc/clc/include/clc/math/clc_exp_helper.h b/libclc/clc/include/clc/math/clc_exp_helper.h
index 0c5028bd23858..a2b3f1bebaf49 100644
--- a/libclc/clc/include/clc/math/clc_exp_helper.h
+++ b/libclc/clc/include/clc/math/clc_exp_helper.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_MATH_CLC_EXP_HELPER
 #define __CLC_MATH_CLC_EXP_HELPER
 
-#define __DOUBLE_ONLY
+#define __CLC_DOUBLE_ONLY
 #define __CLC_BODY <clc/math/clc_exp_helper.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/include/clc/math/clc_exp_helper.inc b/libclc/clc/include/clc/math/clc_exp_helper.inc
index cdf650405c815..cb2a0d30b1974 100644
--- a/libclc/clc/include/clc/math/clc_exp_helper.inc
+++ b/libclc/clc/include/clc/math/clc_exp_helper.inc
@@ -6,8 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_DECL _CLC_OVERLOAD __CLC_GENTYPE __clc_exp_helper(__CLC_GENTYPE x,
-                                                       __CLC_GENTYPE x_min,
-                                                       __CLC_GENTYPE x_max,
-                                                       __CLC_GENTYPE r,
-                                                       __CLC_INTN n);
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST __CLC_GENTYPE
+__clc_exp_helper(__CLC_GENTYPE x, __CLC_GENTYPE x_min, __CLC_GENTYPE x_max,
+                 __CLC_GENTYPE r, __CLC_INTN n);
diff --git a/libclc/clc/include/clc/math/clc_expm1.h b/libclc/clc/include/clc/math/clc_expm1.h
index 7a199dca17cb3..5c60960ca5b03 100644
--- a/libclc/clc/include/clc/math/clc_expm1.h
+++ b/libclc/clc/include/clc/math/clc_expm1.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_EXPM1_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_expm1
+#define __CLC_FUNCTION __clc_expm1
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_EXPM1_H__
diff --git a/libclc/clc/include/clc/math/clc_fabs.h b/libclc/clc/include/clc/math/clc_fabs.h
index 8af3bb572142f..1ef38a8c75957 100644
--- a/libclc/clc/include/clc/math/clc_fabs.h
+++ b/libclc/clc/include/clc/math/clc_fabs.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_FABS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_fabs
+#define __CLC_FUNCTION __clc_fabs
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_FABS_H__
diff --git a/libclc/clc/include/clc/math/clc_fdim.h b/libclc/clc/include/clc/math/clc_fdim.h
index c678eb25f59ba..50ad045db62cf 100644
--- a/libclc/clc/include/clc/math/clc_fdim.h
+++ b/libclc/clc/include/clc/math/clc_fdim.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_FDIM_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_fdim
+#define __CLC_FUNCTION __clc_fdim
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_FDIM_H__
diff --git a/libclc/clc/include/clc/math/clc_floor.h b/libclc/clc/include/clc/math/clc_floor.h
index b3fe8ea95ce13..d243819e0a42c 100644
--- a/libclc/clc/include/clc/math/clc_floor.h
+++ b/libclc/clc/include/clc/math/clc_floor.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_FLOOR_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_floor
+#define __CLC_FUNCTION __clc_floor
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_FLOOR_H__
diff --git a/libclc/clc/include/clc/math/clc_fma.h b/libclc/clc/include/clc/math/clc_fma.h
index efe20bd150eb5..5e87f4c470850 100644
--- a/libclc/clc/include/clc/math/clc_fma.h
+++ b/libclc/clc/include/clc/math/clc_fma.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMA_H__
 #define __CLC_MATH_CLC_FMA_H__
 
-#define FUNCTION __clc_fma
+#define __CLC_FUNCTION __clc_fma
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_FMA_H__
diff --git a/libclc/clc/include/clc/math/clc_fmax.h b/libclc/clc/include/clc/math/clc_fmax.h
index fd5d38987ec11..f367a9b6ea1ca 100644
--- a/libclc/clc/include/clc/math/clc_fmax.h
+++ b/libclc/clc/include/clc/math/clc_fmax.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMAX_H__
 #define __CLC_MATH_CLC_FMAX_H__
 
-#define FUNCTION __clc_fmax
+#define __CLC_FUNCTION __clc_fmax
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_FMAX_H__
diff --git a/libclc/clc/include/clc/math/clc_fmin.h b/libclc/clc/include/clc/math/clc_fmin.h
index 9fcdb56475ff6..47ed3fad2d8c2 100644
--- a/libclc/clc/include/clc/math/clc_fmin.h
+++ b/libclc/clc/include/clc/math/clc_fmin.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMIN_H__
 #define __CLC_MATH_CLC_FMIN_H__
 
-#define FUNCTION __clc_fmin
+#define __CLC_FUNCTION __clc_fmin
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_FMIN_H__
diff --git a/libclc/clc/include/clc/math/clc_fmod.h b/libclc/clc/include/clc/math/clc_fmod.h
index 840bb850d73c9..44179babd7fed 100644
--- a/libclc/clc/include/clc/math/clc_fmod.h
+++ b/libclc/clc/include/clc/math/clc_fmod.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FMOD_H__
 #define __CLC_MATH_CLC_FMOD_H__
 
-#define FUNCTION __clc_fmod
+#define __CLC_FUNCTION __clc_fmod
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_FMOD_H__
diff --git a/libclc/clc/include/clc/math/clc_fract.h b/libclc/clc/include/clc/math/clc_fract.h
index 26c402c79641a..af762ecc59ea5 100644
--- a/libclc/clc/include/clc/math/clc_fract.h
+++ b/libclc/clc/include/clc/math/clc_fract.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_FRACT_H__
 #define __CLC_MATH_CLC_FRACT_H__
 
-#define FUNCTION __clc_fract
+#define __CLC_FUNCTION __clc_fract
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_FRACT_H__
diff --git a/libclc/clc/include/clc/math/clc_frexp.h b/libclc/clc/include/clc/math/clc_frexp.h
index e3c9f6b363dd1..d6ed2754d0d40 100644
--- a/libclc/clc/include/clc/math/clc_frexp.h
+++ b/libclc/clc/include/clc/math/clc_frexp.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_FREXP_H__
 #define __CLC_MATH_CLC_FREXP_H__
 
-#define FUNCTION __clc_frexp
+#define __CLC_FUNCTION __clc_frexp
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_FREXP_H__
diff --git a/libclc/clc/include/clc/math/clc_half_cos.h b/libclc/clc/include/clc/math/clc_half_cos.h
index 0f9f0b61443f9..d0122987899c3 100644
--- a/libclc/clc/include/clc/math/clc_half_cos.h
+++ b/libclc/clc/include/clc/math/clc_half_cos.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_COS_H__
 #define __CLC_MATH_CLC_HALF_COS_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_cos
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_cos
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_COS_H__
diff --git a/libclc/clc/include/clc/math/clc_half_divide.h b/libclc/clc/include/clc/math/clc_half_divide.h
index 44d90f39b4f61..e5d5cdc9df34e 100644
--- a/libclc/clc/include/clc/math/clc_half_divide.h
+++ b/libclc/clc/include/clc/math/clc_half_divide.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_DIVIDE_H__
 #define __CLC_MATH_CLC_HALF_DIVIDE_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_divide
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_divide
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_DIVIDE_H__
diff --git a/libclc/clc/include/clc/math/clc_half_exp.h b/libclc/clc/include/clc/math/clc_half_exp.h
index 158d28c16aabb..2284eaf8df360 100644
--- a/libclc/clc/include/clc/math/clc_half_exp.h
+++ b/libclc/clc/include/clc/math/clc_half_exp.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_EXP_H__
 #define __CLC_MATH_CLC_HALF_EXP_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_exp
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_exp
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_EXP_H__
diff --git a/libclc/clc/include/clc/math/clc_half_exp10.h b/libclc/clc/include/clc/math/clc_half_exp10.h
index 4982b618e505a..662df93e06855 100644
--- a/libclc/clc/include/clc/math/clc_half_exp10.h
+++ b/libclc/clc/include/clc/math/clc_half_exp10.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_EXP10_H__
 #define __CLC_MATH_CLC_HALF_EXP10_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_exp10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_exp10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_EXP10_H__
diff --git a/libclc/clc/include/clc/math/clc_half_exp2.h b/libclc/clc/include/clc/math/clc_half_exp2.h
index dc0f67659bf9f..d2af0f2e3fa9a 100644
--- a/libclc/clc/include/clc/math/clc_half_exp2.h
+++ b/libclc/clc/include/clc/math/clc_half_exp2.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_EXP2_H__
 #define __CLC_MATH_CLC_HALF_EXP2_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_exp2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_exp2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_EXP2_H__
diff --git a/libclc/clc/include/clc/math/clc_half_log.h b/libclc/clc/include/clc/math/clc_half_log.h
index df527e1a8fb57..8ae30e5abe5f8 100644
--- a/libclc/clc/include/clc/math/clc_half_log.h
+++ b/libclc/clc/include/clc/math/clc_half_log.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_LOG_H__
 #define __CLC_MATH_CLC_HALF_LOG_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_log
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_log
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_LOG_H__
diff --git a/libclc/clc/include/clc/math/clc_half_log10.h b/libclc/clc/include/clc/math/clc_half_log10.h
index 47db4cd325931..fe5248350abad 100644
--- a/libclc/clc/include/clc/math/clc_half_log10.h
+++ b/libclc/clc/include/clc/math/clc_half_log10.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_LOG10_H__
 #define __CLC_MATH_CLC_HALF_LOG10_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_log10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_log10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_LOG10_H__
diff --git a/libclc/clc/include/clc/math/clc_half_log2.h b/libclc/clc/include/clc/math/clc_half_log2.h
index 2b95b444b3f24..c57aa1bffb335 100644
--- a/libclc/clc/include/clc/math/clc_half_log2.h
+++ b/libclc/clc/include/clc/math/clc_half_log2.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_LOG2_H__
 #define __CLC_MATH_CLC_HALF_LOG2_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_log2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_log2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_LOG2_H__
diff --git a/libclc/clc/include/clc/math/clc_half_powr.h b/libclc/clc/include/clc/math/clc_half_powr.h
index 2fd62213b8241..188b32b8b49c6 100644
--- a/libclc/clc/include/clc/math/clc_half_powr.h
+++ b/libclc/clc/include/clc/math/clc_half_powr.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_POWR_H__
 #define __CLC_MATH_CLC_HALF_POWR_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_powr
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_powr
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_POWR_H__
diff --git a/libclc/clc/include/clc/math/clc_half_recip.h b/libclc/clc/include/clc/math/clc_half_recip.h
index d2ace8b7a17e4..f5f71fa1988c8 100644
--- a/libclc/clc/include/clc/math/clc_half_recip.h
+++ b/libclc/clc/include/clc/math/clc_half_recip.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_RECIP_H__
 #define __CLC_MATH_CLC_HALF_RECIP_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_recip
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_recip
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_RECIP_H__
diff --git a/libclc/clc/include/clc/math/clc_half_rsqrt.h b/libclc/clc/include/clc/math/clc_half_rsqrt.h
index a50dabccdaafc..d5a05ac8b84ac 100644
--- a/libclc/clc/include/clc/math/clc_half_rsqrt.h
+++ b/libclc/clc/include/clc/math/clc_half_rsqrt.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_RSQRT_H__
 #define __CLC_MATH_CLC_HALF_RSQRT_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_rsqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_rsqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_RSQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_half_sin.h b/libclc/clc/include/clc/math/clc_half_sin.h
index f79f6afc2b4ec..d2d3da9b11dbf 100644
--- a/libclc/clc/include/clc/math/clc_half_sin.h
+++ b/libclc/clc/include/clc/math/clc_half_sin.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_SIN_H__
 #define __CLC_MATH_CLC_HALF_SIN_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_sin
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_sin
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_SIN_H__
diff --git a/libclc/clc/include/clc/math/clc_half_sqrt.h b/libclc/clc/include/clc/math/clc_half_sqrt.h
index df5fc222a5c9f..a54b8122cb15e 100644
--- a/libclc/clc/include/clc/math/clc_half_sqrt.h
+++ b/libclc/clc/include/clc/math/clc_half_sqrt.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_SQRT_H__
 #define __CLC_MATH_CLC_HALF_SQRT_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_sqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_sqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_SQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_half_tan.h b/libclc/clc/include/clc/math/clc_half_tan.h
index 0f6ef87907225..503ebe29c3cec 100644
--- a/libclc/clc/include/clc/math/clc_half_tan.h
+++ b/libclc/clc/include/clc/math/clc_half_tan.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_HALF_TAN_H__
 #define __CLC_MATH_CLC_HALF_TAN_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_tan
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_tan
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HALF_TAN_H__
diff --git a/libclc/clc/include/clc/math/clc_hypot.h b/libclc/clc/include/clc/math/clc_hypot.h
index b2d69ebda96e2..40b354154fa53 100644
--- a/libclc/clc/include/clc/math/clc_hypot.h
+++ b/libclc/clc/include/clc/math/clc_hypot.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_HYPOT_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_hypot
+#define __CLC_FUNCTION __clc_hypot
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_HYPOT_H__
diff --git a/libclc/clc/include/clc/math/clc_ilogb.h b/libclc/clc/include/clc/math/clc_ilogb.h
index e02108ff1444b..6aeaf6be3ac6c 100644
--- a/libclc/clc/include/clc/math/clc_ilogb.h
+++ b/libclc/clc/include/clc/math/clc_ilogb.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_ILOGB_H__
 #define __CLC_MATH_CLC_ILOGB_H__
 
-#define FUNCTION __clc_ilogb
+#define __CLC_FUNCTION __clc_ilogb
 #define __CLC_BODY <clc/math/unary_decl_with_int_return.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ILOGB_H__
diff --git a/libclc/clc/include/clc/math/clc_ldexp.inc b/libclc/clc/include/clc/math/clc_ldexp.inc
index 9b88359d212d2..b0bea29bed8f2 100644
--- a/libclc/clc/include/clc/math/clc_ldexp.inc
+++ b/libclc/clc/include/clc/math/clc_ldexp.inc
@@ -6,4 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_DECL _CLC_OVERLOAD __CLC_GENTYPE __clc_ldexp(__CLC_GENTYPE, __CLC_INTN);
+_CLC_DECL _CLC_OVERLOAD _CLC_CONST __CLC_GENTYPE __clc_ldexp(__CLC_GENTYPE,
+                                                             __CLC_INTN);
diff --git a/libclc/clc/include/clc/math/clc_lgamma.h b/libclc/clc/include/clc/math/clc_lgamma.h
index 69b1ff9701ca5..f54c5e2b61b67 100644
--- a/libclc/clc/include/clc/math/clc_lgamma.h
+++ b/libclc/clc/include/clc/math/clc_lgamma.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_LGAMMA_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_lgamma
+#define __CLC_FUNCTION __clc_lgamma
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_LGAMMA_H__
diff --git a/libclc/clc/include/clc/math/clc_lgamma_r.h b/libclc/clc/include/clc/math/clc_lgamma_r.h
index a98f760d8032e..262e62372227a 100644
--- a/libclc/clc/include/clc/math/clc_lgamma_r.h
+++ b/libclc/clc/include/clc/math/clc_lgamma_r.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LGAMMA_R_H__
 #define __CLC_MATH_CLC_LGAMMA_R_H__
 
-#define FUNCTION __clc_lgamma_r
+#define __CLC_FUNCTION __clc_lgamma_r
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_LGAMMA_R_H__
diff --git a/libclc/clc/include/clc/math/clc_log.h b/libclc/clc/include/clc/math/clc_log.h
index 6bddb5ff746ec..8b0e44b425a89 100644
--- a/libclc/clc/include/clc/math/clc_log.h
+++ b/libclc/clc/include/clc/math/clc_log.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LOG_H__
 #define __CLC_MATH_CLC_LOG_H__
 
-#define FUNCTION __clc_log
+#define __CLC_FUNCTION __clc_log
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG_H__
diff --git a/libclc/clc/include/clc/math/clc_log10.h b/libclc/clc/include/clc/math/clc_log10.h
index e3a5b19eb4184..6e086690c6ada 100644
--- a/libclc/clc/include/clc/math/clc_log10.h
+++ b/libclc/clc/include/clc/math/clc_log10.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LOG10_H__
 #define __CLC_MATH_CLC_LOG10_H__
 
-#define FUNCTION __clc_log10
+#define __CLC_FUNCTION __clc_log10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG10_H__
diff --git a/libclc/clc/include/clc/math/clc_log1p.h b/libclc/clc/include/clc/math/clc_log1p.h
index f502802dafb22..1be8c8548d488 100644
--- a/libclc/clc/include/clc/math/clc_log1p.h
+++ b/libclc/clc/include/clc/math/clc_log1p.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_LOG1P_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_log1p
+#define __CLC_FUNCTION __clc_log1p
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG1P_H__
diff --git a/libclc/clc/include/clc/math/clc_log2.h b/libclc/clc/include/clc/math/clc_log2.h
index 7344cabaeffaa..c5f9e37f39845 100644
--- a/libclc/clc/include/clc/math/clc_log2.h
+++ b/libclc/clc/include/clc/math/clc_log2.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_LOG2_H__
 #define __CLC_MATH_CLC_LOG2_H__
 
-#define FUNCTION __clc_log2
+#define __CLC_FUNCTION __clc_log2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_LOG2_H__
diff --git a/libclc/clc/include/clc/math/clc_logb.h b/libclc/clc/include/clc/math/clc_logb.h
index 1339d5819171d..75a2ce7e578b1 100644
--- a/libclc/clc/include/clc/math/clc_logb.h
+++ b/libclc/clc/include/clc/math/clc_logb.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_LOGB_H__
 #define __CLC_MATH_CLC_LOGB_H__
 
-#define FUNCTION __clc_logb
+#define __CLC_FUNCTION __clc_logb
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_LOGB_H__
diff --git a/libclc/clc/include/clc/math/clc_mad.h b/libclc/clc/include/clc/math/clc_mad.h
index 72014b7c0ef7b..117e361380eda 100644
--- a/libclc/clc/include/clc/math/clc_mad.h
+++ b/libclc/clc/include/clc/math/clc_mad.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_MAD_H__
 
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
-#define FUNCTION __clc_mad
+#define __CLC_FUNCTION __clc_mad
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_MAD_H__
diff --git a/libclc/clc/include/clc/math/clc_maxmag.h b/libclc/clc/include/clc/math/clc_maxmag.h
index 058230e6ebfc9..857ad7867b05d 100644
--- a/libclc/clc/include/clc/math/clc_maxmag.h
+++ b/libclc/clc/include/clc/math/clc_maxmag.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_MAXMAG_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_maxmag
+#define __CLC_FUNCTION __clc_maxmag
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_MAXMAG_H__
diff --git a/libclc/clc/include/clc/math/clc_minmag.h b/libclc/clc/include/clc/math/clc_minmag.h
index 581b957828235..9615d029bfed2 100644
--- a/libclc/clc/include/clc/math/clc_minmag.h
+++ b/libclc/clc/include/clc/math/clc_minmag.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_MINMAG_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_minmag
+#define __CLC_FUNCTION __clc_minmag
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_MINMAG_H__
diff --git a/libclc/clc/include/clc/math/clc_modf.h b/libclc/clc/include/clc/math/clc_modf.h
index fbdd3f8ffd4b8..aa8f9e20892ab 100644
--- a/libclc/clc/include/clc/math/clc_modf.h
+++ b/libclc/clc/include/clc/math/clc_modf.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_MATH_CLC_MODF_H__
 #define __CLC_MATH_CLC_MODF_H__
 
-#define FUNCTION __clc_modf
+#define __CLC_FUNCTION __clc_modf
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_MODF_H__
diff --git a/libclc/clc/include/clc/math/clc_nan.h b/libclc/clc/include/clc/math/clc_nan.h
index 45d91184cf145..91901fd9369f2 100644
--- a/libclc/clc/include/clc/math/clc_nan.h
+++ b/libclc/clc/include/clc/math/clc_nan.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_NAN_H__
 #define __CLC_MATH_CLC_NAN_H__
 
-#define FUNCTION __clc_nan
+#define __CLC_FUNCTION __clc_nan
 #define __CLC_BODY <clc/math/clc_nan.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NAN_H__
diff --git a/libclc/clc/include/clc/math/clc_nan.inc b/libclc/clc/include/clc/math/clc_nan.inc
index 22604e0a5785e..9203499174857 100644
--- a/libclc/clc/include/clc/math/clc_nan.inc
+++ b/libclc/clc/include/clc/math/clc_nan.inc
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_U_GENTYPE code);
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_S_GENTYPE code);
+_CLC_OVERLOAD _CLC_CONST _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_U_GENTYPE code);
+_CLC_OVERLOAD _CLC_CONST _CLC_DEF __CLC_GENTYPE __clc_nan(__CLC_S_GENTYPE code);
diff --git a/libclc/clc/include/clc/math/clc_native_cos.h b/libclc/clc/include/clc/math/clc_native_cos.h
index 08d51baca1f61..eefd133911e48 100644
--- a/libclc/clc/include/clc/math/clc_native_cos.h
+++ b/libclc/clc/include/clc/math/clc_native_cos.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_COS_H__
 #define __CLC_MATH_CLC_NATIVE_COS_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_cos
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_cos
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_COS_H__
diff --git a/libclc/clc/include/clc/math/clc_native_divide.h b/libclc/clc/include/clc/math/clc_native_divide.h
index a10ed512f10d9..97d97c2670a6b 100644
--- a/libclc/clc/include/clc/math/clc_native_divide.h
+++ b/libclc/clc/include/clc/math/clc_native_divide.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_DIVIDE_H__
 #define __CLC_MATH_CLC_NATIVE_DIVIDE_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_divide
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_divide
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_DIVIDE_H__
diff --git a/libclc/clc/include/clc/math/clc_native_exp.h b/libclc/clc/include/clc/math/clc_native_exp.h
index 30801fe2f8ad3..30c0d1edf18fb 100644
--- a/libclc/clc/include/clc/math/clc_native_exp.h
+++ b/libclc/clc/include/clc/math/clc_native_exp.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_EXP_H__
 #define __CLC_MATH_CLC_NATIVE_EXP_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_exp
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_exp
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_EXP_H__
diff --git a/libclc/clc/include/clc/math/clc_native_exp10.h b/libclc/clc/include/clc/math/clc_native_exp10.h
index 91cf60ef6a2be..a88b5a3897377 100644
--- a/libclc/clc/include/clc/math/clc_native_exp10.h
+++ b/libclc/clc/include/clc/math/clc_native_exp10.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_EXP10_H__
 #define __CLC_MATH_CLC_NATIVE_EXP10_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_exp10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_exp10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_EXP10_H__
diff --git a/libclc/clc/include/clc/math/clc_native_exp2.h b/libclc/clc/include/clc/math/clc_native_exp2.h
index fe47280536198..09f4bfe6979dd 100644
--- a/libclc/clc/include/clc/math/clc_native_exp2.h
+++ b/libclc/clc/include/clc/math/clc_native_exp2.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_EXP2_H__
 #define __CLC_MATH_CLC_NATIVE_EXP2_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_exp2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_exp2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_EXP2_H__
diff --git a/libclc/clc/include/clc/math/clc_native_log.h b/libclc/clc/include/clc/math/clc_native_log.h
index 25a7054d2a599..c32fc812d4770 100644
--- a/libclc/clc/include/clc/math/clc_native_log.h
+++ b/libclc/clc/include/clc/math/clc_native_log.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_LOG_H__
 #define __CLC_MATH_CLC_NATIVE_LOG_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_log
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_log
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_LOG_H__
diff --git a/libclc/clc/include/clc/math/clc_native_log10.h b/libclc/clc/include/clc/math/clc_native_log10.h
index d46124d655ee7..329b8a7437556 100644
--- a/libclc/clc/include/clc/math/clc_native_log10.h
+++ b/libclc/clc/include/clc/math/clc_native_log10.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_LOG10_H__
 #define __CLC_MATH_CLC_NATIVE_LOG10_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_log10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_log10
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_LOG10_H__
diff --git a/libclc/clc/include/clc/math/clc_native_log2.h b/libclc/clc/include/clc/math/clc_native_log2.h
index 8998553b4984a..b8ad663b470d8 100644
--- a/libclc/clc/include/clc/math/clc_native_log2.h
+++ b/libclc/clc/include/clc/math/clc_native_log2.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_LOG2_H__
 #define __CLC_MATH_CLC_NATIVE_LOG2_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_log2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_log2
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_LOG2_H__
diff --git a/libclc/clc/include/clc/math/clc_native_powr.h b/libclc/clc/include/clc/math/clc_native_powr.h
index 932cfa66aeb9c..3a0c8cc8b9d58 100644
--- a/libclc/clc/include/clc/math/clc_native_powr.h
+++ b/libclc/clc/include/clc/math/clc_native_powr.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_POWR_H__
 #define __CLC_MATH_CLC_NATIVE_POWR_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_powr
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_powr
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_POWR_H__
diff --git a/libclc/clc/include/clc/math/clc_native_recip.h b/libclc/clc/include/clc/math/clc_native_recip.h
index 6b94cbc3d6c6e..a6540e846f2b6 100644
--- a/libclc/clc/include/clc/math/clc_native_recip.h
+++ b/libclc/clc/include/clc/math/clc_native_recip.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_RECIP_H__
 #define __CLC_MATH_CLC_NATIVE_RECIP_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_recip
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_recip
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_RECIP_H__
diff --git a/libclc/clc/include/clc/math/clc_native_rsqrt.h b/libclc/clc/include/clc/math/clc_native_rsqrt.h
index 5fa870b3f868a..0c8f3da039299 100644
--- a/libclc/clc/include/clc/math/clc_native_rsqrt.h
+++ b/libclc/clc/include/clc/math/clc_native_rsqrt.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_RSQRT_H__
 #define __CLC_MATH_CLC_NATIVE_RSQRT_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_rsqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_rsqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_RSQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_native_sin.h b/libclc/clc/include/clc/math/clc_native_sin.h
index d25968c74409e..22ab7868e20fb 100644
--- a/libclc/clc/include/clc/math/clc_native_sin.h
+++ b/libclc/clc/include/clc/math/clc_native_sin.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_SIN_H__
 #define __CLC_MATH_CLC_NATIVE_SIN_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_sin
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_sin
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_SIN_H__
diff --git a/libclc/clc/include/clc/math/clc_native_sqrt.h b/libclc/clc/include/clc/math/clc_native_sqrt.h
index e8a28b7f70463..fd071f1d463ab 100644
--- a/libclc/clc/include/clc/math/clc_native_sqrt.h
+++ b/libclc/clc/include/clc/math/clc_native_sqrt.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_SQRT_H__
 #define __CLC_MATH_CLC_NATIVE_SQRT_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_sqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_sqrt
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_SQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_native_tan.h b/libclc/clc/include/clc/math/clc_native_tan.h
index 36fdd6d8c011b..4b381312d8d51 100644
--- a/libclc/clc/include/clc/math/clc_native_tan.h
+++ b/libclc/clc/include/clc/math/clc_native_tan.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_NATIVE_TAN_H__
 #define __CLC_MATH_CLC_NATIVE_TAN_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_tan
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_tan
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NATIVE_TAN_H__
diff --git a/libclc/clc/include/clc/math/clc_nextafter.h b/libclc/clc/include/clc/math/clc_nextafter.h
index d080a40a06f03..f24742b9e9c37 100644
--- a/libclc/clc/include/clc/math/clc_nextafter.h
+++ b/libclc/clc/include/clc/math/clc_nextafter.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_NEXTAFTER_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_nextafter
+#define __CLC_FUNCTION __clc_nextafter
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_NEXTAFTER_H__
diff --git a/libclc/clc/include/clc/math/clc_pow.h b/libclc/clc/include/clc/math/clc_pow.h
index b33391e10a42a..5e37e5bf6da65 100644
--- a/libclc/clc/include/clc/math/clc_pow.h
+++ b/libclc/clc/include/clc/math/clc_pow.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_POW_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_pow
+#define __CLC_FUNCTION __clc_pow
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_POW_H__
diff --git a/libclc/clc/include/clc/math/clc_pown.h b/libclc/clc/include/clc/math/clc_pown.h
index 473bbe1603c52..30628efb19001 100644
--- a/libclc/clc/include/clc/math/clc_pown.h
+++ b/libclc/clc/include/clc/math/clc_pown.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_POWN_H__
 
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
-#define FUNCTION __clc_pown
+#define __CLC_FUNCTION __clc_pown
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_POWN_H__
diff --git a/libclc/clc/include/clc/math/clc_powr.h b/libclc/clc/include/clc/math/clc_powr.h
index a820516f18660..baa494cce6989 100644
--- a/libclc/clc/include/clc/math/clc_powr.h
+++ b/libclc/clc/include/clc/math/clc_powr.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_POWR_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION __clc_powr
+#define __CLC_FUNCTION __clc_powr
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_POWR_H__
diff --git a/libclc/clc/include/clc/math/clc_remainder.h b/libclc/clc/include/clc/math/clc_remainder.h
index bc53b043045cd..a7700c95103b4 100644
--- a/libclc/clc/include/clc/math/clc_remainder.h
+++ b/libclc/clc/include/clc/math/clc_remainder.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_REMAINDER_H__
 #define __CLC_MATH_CLC_REMAINDER_H__
 
-#define FUNCTION __clc_remainder
+#define __CLC_FUNCTION __clc_remainder
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_REMAINDER_H__
diff --git a/libclc/clc/include/clc/math/clc_remquo.h b/libclc/clc/include/clc/math/clc_remquo.h
index b46f94ba94477..48a8844a6e384 100644
--- a/libclc/clc/include/clc/math/clc_remquo.h
+++ b/libclc/clc/include/clc/math/clc_remquo.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_MATH_CLC_REMQUO_H__
 #define __CLC_MATH_CLC_REMQUO_H__
 
-#define FUNCTION __clc_remquo
+#define __CLC_FUNCTION __clc_remquo
 #define __CLC_BODY <clc/math/remquo_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_REMQUO_H__
diff --git a/libclc/clc/include/clc/math/clc_rint.h b/libclc/clc/include/clc/math/clc_rint.h
index c446c75764710..17b1549f439b8 100644
--- a/libclc/clc/include/clc/math/clc_rint.h
+++ b/libclc/clc/include/clc/math/clc_rint.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_RINT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_rint
+#define __CLC_FUNCTION __clc_rint
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_RINT_H__
diff --git a/libclc/clc/include/clc/math/clc_rootn.h b/libclc/clc/include/clc/math/clc_rootn.h
index fb7817ee250fd..90a25ad52d867 100644
--- a/libclc/clc/include/clc/math/clc_rootn.h
+++ b/libclc/clc/include/clc/math/clc_rootn.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ROOTN_H__
 
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
-#define FUNCTION __clc_rootn
+#define __CLC_FUNCTION __clc_rootn
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ROOTN_H__
diff --git a/libclc/clc/include/clc/math/clc_round.h b/libclc/clc/include/clc/math/clc_round.h
index fd1872a5cf224..78ebf4db99e9f 100644
--- a/libclc/clc/include/clc/math/clc_round.h
+++ b/libclc/clc/include/clc/math/clc_round.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_ROUND_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_round
+#define __CLC_FUNCTION __clc_round
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_ROUND_H__
diff --git a/libclc/clc/include/clc/math/clc_rsqrt.h b/libclc/clc/include/clc/math/clc_rsqrt.h
index 1690ede1b8926..59402fbead2b2 100644
--- a/libclc/clc/include/clc/math/clc_rsqrt.h
+++ b/libclc/clc/include/clc/math/clc_rsqrt.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_RSQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_rsqrt
+#define __CLC_FUNCTION __clc_rsqrt
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_RSQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_sin.h b/libclc/clc/include/clc/math/clc_sin.h
index 89ae444eb6e30..de4c722ca123f 100644
--- a/libclc/clc/include/clc/math/clc_sin.h
+++ b/libclc/clc/include/clc/math/clc_sin.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_sin
+#define __CLC_FUNCTION __clc_sin
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_SIN_H__
diff --git a/libclc/clc/include/clc/math/clc_sincos.h b/libclc/clc/include/clc/math/clc_sincos.h
index bf3e0806db381..e26dc7c024c9c 100644
--- a/libclc/clc/include/clc/math/clc_sincos.h
+++ b/libclc/clc/include/clc/math/clc_sincos.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SINCOS_H__
 
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
-#define FUNCTION __clc_sincos
+#define __CLC_FUNCTION __clc_sincos
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_SINCOS_H__
diff --git a/libclc/clc/include/clc/math/clc_sincos_helpers.h b/libclc/clc/include/clc/math/clc_sincos_helpers.h
index f9ceba3bf2cf7..909afe476ad90 100644
--- a/libclc/clc/include/clc/math/clc_sincos_helpers.h
+++ b/libclc/clc/include/clc/math/clc_sincos_helpers.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_MATH_CLC_SINCOS_HELPERS_H__
 #define __CLC_MATH_CLC_SINCOS_HELPERS_H__
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc/math/clc_sincos_helpers.inc>
 
 #include <clc/math/gentype.inc>
 
-#define __DOUBLE_ONLY
+#define __CLC_DOUBLE_ONLY
 #define __CLC_BODY <clc/math/clc_sincos_helpers_fp64.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/include/clc/math/clc_sinh.h b/libclc/clc/include/clc/math/clc_sinh.h
index c94d8e37acc64..fa9cabe79cb0e 100644
--- a/libclc/clc/include/clc/math/clc_sinh.h
+++ b/libclc/clc/include/clc/math/clc_sinh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SINH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_sinh
+#define __CLC_FUNCTION __clc_sinh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_SINH_H__
diff --git a/libclc/clc/include/clc/math/clc_sinpi.h b/libclc/clc/include/clc/math/clc_sinpi.h
index 725e559600edd..db563c2e7e14a 100644
--- a/libclc/clc/include/clc/math/clc_sinpi.h
+++ b/libclc/clc/include/clc/math/clc_sinpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SINPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_sinpi
+#define __CLC_FUNCTION __clc_sinpi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_SINPI_H__
diff --git a/libclc/clc/include/clc/math/clc_sqrt.h b/libclc/clc/include/clc/math/clc_sqrt.h
index 74aea8e858099..7084515533291 100644
--- a/libclc/clc/include/clc/math/clc_sqrt.h
+++ b/libclc/clc/include/clc/math/clc_sqrt.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_SQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_sqrt
+#define __CLC_FUNCTION __clc_sqrt
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_SQRT_H__
diff --git a/libclc/clc/include/clc/math/clc_tan.h b/libclc/clc/include/clc/math/clc_tan.h
index d16370420a030..028ff28ecd69d 100644
--- a/libclc/clc/include/clc/math/clc_tan.h
+++ b/libclc/clc/include/clc/math/clc_tan.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_tan
+#define __CLC_FUNCTION __clc_tan
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_TAN_H__
diff --git a/libclc/clc/include/clc/math/clc_tanh.h b/libclc/clc/include/clc/math/clc_tanh.h
index 2c860c165ca67..88fce2340d99b 100644
--- a/libclc/clc/include/clc/math/clc_tanh.h
+++ b/libclc/clc/include/clc/math/clc_tanh.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TANH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_tanh
+#define __CLC_FUNCTION __clc_tanh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_TANH_H__
diff --git a/libclc/clc/include/clc/math/clc_tanpi.h b/libclc/clc/include/clc/math/clc_tanpi.h
index 2691fa696c0a5..596bb72b4b8de 100644
--- a/libclc/clc/include/clc/math/clc_tanpi.h
+++ b/libclc/clc/include/clc/math/clc_tanpi.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TANPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_tanpi
+#define __CLC_FUNCTION __clc_tanpi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_TANPI_H__
diff --git a/libclc/clc/include/clc/math/clc_tgamma.h b/libclc/clc/include/clc/math/clc_tgamma.h
index d067703c2e9df..dd76e041875d5 100644
--- a/libclc/clc/include/clc/math/clc_tgamma.h
+++ b/libclc/clc/include/clc/math/clc_tgamma.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TGAMMA_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_tgamma
+#define __CLC_FUNCTION __clc_tgamma
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_TGAMMA_H__
diff --git a/libclc/clc/include/clc/math/clc_trunc.h b/libclc/clc/include/clc/math/clc_trunc.h
index 6a8353a4f6d45..33445f9fe15d6 100644
--- a/libclc/clc/include/clc/math/clc_trunc.h
+++ b/libclc/clc/include/clc/math/clc_trunc.h
@@ -10,10 +10,10 @@
 #define __CLC_MATH_CLC_TRUNC_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION __clc_trunc
+#define __CLC_FUNCTION __clc_trunc
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MATH_CLC_TRUNC_H__
diff --git a/libclc/clc/include/clc/math/gentype.inc b/libclc/clc/include/clc/math/gentype.inc
index d74a9932bd0a6..3373f5fa50024 100644
--- a/libclc/clc/include/clc/math/gentype.inc
+++ b/libclc/clc/include/clc/math/gentype.inc
@@ -70,7 +70,7 @@
 #define __CLC_CONVERT_S_GENTYPE __CLC_XCONCAT(__clc_convert_, __CLC_S_GENTYPE)
 #define __CLC_CONVERT_U_GENTYPE __CLC_XCONCAT(__clc_convert_, __CLC_U_GENTYPE)
 
-#if (!defined(__HALF_ONLY) && !defined(__DOUBLE_ONLY))
+#if (!defined(__CLC_HALF_ONLY) && !defined(__CLC_DOUBLE_ONLY))
 #define __CLC_SCALAR_GENTYPE float
 #define __CLC_FPSIZE 32
 #define __CLC_FP_LIT(x) (__CLC_GENTYPE) x##F
@@ -80,6 +80,7 @@
 #define __CLC_U_GENTYPE __CLC_XCONCAT(uint, __CLC_VECSIZE)
 
 #define __CLC_GENTYPE float
+#define __CLC_BIT_INT int
 #define __CLC_BIT_INTN int
 #define __CLC_SCALAR
 #define __CLC_VECSIZE
@@ -131,6 +132,7 @@
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INT
 #undef __CLC_BIT_INTN
 
 #undef __CLC_VECSIZE_OR_1
@@ -143,7 +145,7 @@
 
 #endif
 
-#if (!defined(__HALF_ONLY) && !defined(__FLOAT_ONLY))
+#if (!defined(__CLC_HALF_ONLY) && !defined(__CLC_FLOAT_ONLY))
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
@@ -159,6 +161,7 @@
 #define __CLC_VECSIZE
 #define __CLC_VECSIZE_OR_1 1
 #define __CLC_GENTYPE double
+#define __CLC_BIT_INT long
 #define __CLC_BIT_INTN long
 #include __CLC_BODY
 #undef __CLC_VECSIZE_OR_1
@@ -207,6 +210,7 @@
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INT
 #undef __CLC_BIT_INTN
 
 #undef __CLC_VECSIZE_OR_1
@@ -219,7 +223,7 @@
 #endif
 #endif
 
-#if (!defined(__FLOAT_ONLY) && !defined(__DOUBLE_ONLY))
+#if (!defined(__CLC_FLOAT_ONLY) && !defined(__CLC_DOUBLE_ONLY))
 #ifdef cl_khr_fp16
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
@@ -235,6 +239,7 @@
 #define __CLC_VECSIZE
 #define __CLC_VECSIZE_OR_1 1
 #define __CLC_GENTYPE half
+#define __CLC_BIT_INT short
 #define __CLC_BIT_INTN short
 #include __CLC_BODY
 #undef __CLC_GENTYPE
@@ -283,6 +288,7 @@
 #include __CLC_BODY
 #undef __CLC_VECSIZE
 #undef __CLC_GENTYPE
+#undef __CLC_BIT_INT
 #undef __CLC_BIT_INTN
 
 #undef __CLC_VECSIZE_OR_1
@@ -350,6 +356,6 @@
 #undef __CLC_AS_GENTYPE
 #undef __CLC_CONVERT_GENTYPE
 
-#undef __HALF_ONLY
-#undef __FLOAT_ONLY
-#undef __DOUBLE_ONLY
+#undef __CLC_HALF_ONLY
+#undef __CLC_FLOAT_ONLY
+#undef __CLC_DOUBLE_ONLY
diff --git a/libclc/clc/include/clc/math/math.h b/libclc/clc/include/clc/math/math.h
index 5df53f1b5e5db..c2647f66b4006 100644
--- a/libclc/clc/include/clc/math/math.h
+++ b/libclc/clc/include/clc/math/math.h
@@ -120,6 +120,4 @@ _CLC_OVERLOAD _CLC_INLINE float __clc_flush_denormal_if_not_supported(float x) {
 
 #endif // cl_khr_fp16
 
-#define ALIGNED(x) __attribute__((aligned(x)))
-
 #endif // __CLC_MATH_MATH_H__
diff --git a/libclc/clc/include/clc/math/remquo_decl.inc b/libclc/clc/include/clc/math/remquo_decl.inc
index 24d96b0488059..cba28a7244eb4 100644
--- a/libclc/clc/include/clc/math/remquo_decl.inc
+++ b/libclc/clc/include/clc/math/remquo_decl.inc
@@ -6,15 +6,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
-                                               private __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     __CLC_GENTYPE y,
+                                                     private __CLC_INTN *q);
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
-                                               global __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     __CLC_GENTYPE y,
+                                                     global __CLC_INTN *q);
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
-                                               local __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     __CLC_GENTYPE y,
+                                                     local __CLC_INTN *q);
 #if _CLC_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y,
-                                               generic __CLC_INTN *q);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     __CLC_GENTYPE y,
+                                                     generic __CLC_INTN *q);
 #endif
diff --git a/libclc/clc/include/clc/math/tables.h b/libclc/clc/include/clc/math/tables.h
index 0fec778b53679..ad4b0547f8c69 100644
--- a/libclc/clc/include/clc/math/tables.h
+++ b/libclc/clc/include/clc/math/tables.h
@@ -11,35 +11,36 @@
 
 #include <clc/clctypes.h>
 
-#define TABLE_SPACE __constant
+#define __CLC_TABLE_SPACE __constant
 
-#define TABLE_MANGLE(NAME) __clc_##NAME
+#define __CLC_TABLE_MANGLE(NAME) __clc_##NAME
 
-#define DECLARE_TABLE(TYPE, NAME, LENGTH) TABLE_SPACE TYPE NAME[LENGTH]
+#define __CLC_DECLARE_TABLE(TYPE, NAME, LENGTH)                                \
+  __CLC_TABLE_SPACE TYPE NAME[LENGTH]
 
-#define TABLE_FUNCTION(TYPE, TABLE, NAME)                                      \
-  TYPE TABLE_MANGLE(NAME)(size_t idx) { return TABLE[idx]; }
+#define __CLC_TABLE_FUNCTION(TYPE, TABLE, NAME)                                \
+  TYPE __CLC_TABLE_MANGLE(NAME)(size_t idx) { return TABLE[idx]; }
 
-#define CLC_TABLE_FUNCTION(TYPE, TABLE, NAME)                                  \
-  _CLC_DEF _CLC_OVERLOAD TYPE TABLE_MANGLE(NAME)(int idx) {                    \
+#define __CLC_TABLE_FUNCTION_VEC(TYPE, TABLE, NAME)                            \
+  _CLC_DEF _CLC_OVERLOAD TYPE __CLC_TABLE_MANGLE(NAME)(int idx) {              \
     return TABLE[idx];                                                         \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD TYPE##2 TABLE_MANGLE(NAME)(int##2 idx) {              \
+  _CLC_DEF _CLC_OVERLOAD TYPE##2 __CLC_TABLE_MANGLE(NAME)(int##2 idx) {        \
     return (TYPE##2){TABLE[idx.s0], TABLE[idx.s1]};                            \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD TYPE##3 TABLE_MANGLE(NAME)(int##3 idx) {              \
+  _CLC_DEF _CLC_OVERLOAD TYPE##3 __CLC_TABLE_MANGLE(NAME)(int##3 idx) {        \
     return (TYPE##3){TABLE[idx.s0], TABLE[idx.s1], TABLE[idx.s2]};             \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD TYPE##4 TABLE_MANGLE(NAME)(int##4 idx) {              \
+  _CLC_DEF _CLC_OVERLOAD TYPE##4 __CLC_TABLE_MANGLE(NAME)(int##4 idx) {        \
     return (TYPE##4){TABLE[idx.s0], TABLE[idx.s1], TABLE[idx.s2],              \
                      TABLE[idx.s3]};                                           \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD TYPE##8 TABLE_MANGLE(NAME)(int##8 idx) {              \
+  _CLC_DEF _CLC_OVERLOAD TYPE##8 __CLC_TABLE_MANGLE(NAME)(int##8 idx) {        \
     return (TYPE##8){TABLE[idx.s0], TABLE[idx.s1], TABLE[idx.s2],              \
                      TABLE[idx.s3], TABLE[idx.s4], TABLE[idx.s5],              \
                      TABLE[idx.s6], TABLE[idx.s7]};                            \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD TYPE##16 TABLE_MANGLE(NAME)(int##16 idx) {            \
+  _CLC_DEF _CLC_OVERLOAD TYPE##16 __CLC_TABLE_MANGLE(NAME)(int##16 idx) {      \
     return (TYPE##16){                                                         \
         TABLE[idx.s0], TABLE[idx.s1], TABLE[idx.s2], TABLE[idx.s3],            \
         TABLE[idx.s4], TABLE[idx.s5], TABLE[idx.s6], TABLE[idx.s7],            \
@@ -47,58 +48,59 @@
         TABLE[idx.sC], TABLE[idx.sD], TABLE[idx.sE], TABLE[idx.sF]};           \
   }
 
-#define TABLE_FUNCTION_DECL(TYPE, NAME) TYPE TABLE_MANGLE(NAME)(size_t idx);
-
-#define CLC_TABLE_FUNCTION_DECL(TYPE, NAME)                                    \
-  _CLC_DECL _CLC_OVERLOAD TYPE TABLE_MANGLE(NAME)(int idx);                    \
-  _CLC_DECL _CLC_OVERLOAD TYPE##2 TABLE_MANGLE(NAME)(int##2 idx);              \
-  _CLC_DECL _CLC_OVERLOAD TYPE##3 TABLE_MANGLE(NAME)(int##3 idx);              \
-  _CLC_DECL _CLC_OVERLOAD TYPE##4 TABLE_MANGLE(NAME)(int##4 idx);              \
-  _CLC_DECL _CLC_OVERLOAD TYPE##8 TABLE_MANGLE(NAME)(int##8 idx);              \
-  _CLC_DECL _CLC_OVERLOAD TYPE##16 TABLE_MANGLE(NAME)(int##16 idx);
-
-#define USE_TABLE(NAME, IDX) TABLE_MANGLE(NAME)(IDX)
-
-TABLE_FUNCTION_DECL(float2, log2_tbl);
-TABLE_FUNCTION_DECL(float2, log10_tbl);
-
-CLC_TABLE_FUNCTION_DECL(float, log_inv_tbl_ep_head);
-CLC_TABLE_FUNCTION_DECL(float, log_inv_tbl_ep_tail);
-CLC_TABLE_FUNCTION_DECL(float, loge_tbl_lo);
-CLC_TABLE_FUNCTION_DECL(float, loge_tbl_hi);
-CLC_TABLE_FUNCTION_DECL(float, log_inv_tbl);
-CLC_TABLE_FUNCTION_DECL(float, exp_tbl);
-CLC_TABLE_FUNCTION_DECL(float, exp_tbl_ep_head);
-CLC_TABLE_FUNCTION_DECL(float, exp_tbl_ep_tail);
-CLC_TABLE_FUNCTION_DECL(float, cbrt_tbl_head);
-CLC_TABLE_FUNCTION_DECL(float, cbrt_tbl_tail);
-CLC_TABLE_FUNCTION_DECL(float, sinhcosh_tbl_head);
-CLC_TABLE_FUNCTION_DECL(float, sinhcosh_tbl_tail);
-CLC_TABLE_FUNCTION_DECL(ulong, pibits_tbl);
+#define __CLC_TABLE_FUNCTION_DECL(TYPE, NAME)                                  \
+  TYPE __CLC_TABLE_MANGLE(NAME)(size_t idx);
+
+#define __CLC_TABLE_FUNCTION_DECL_VEC(TYPE, NAME)                              \
+  _CLC_DECL _CLC_OVERLOAD TYPE __CLC_TABLE_MANGLE(NAME)(int idx);              \
+  _CLC_DECL _CLC_OVERLOAD TYPE##2 __CLC_TABLE_MANGLE(NAME)(int##2 idx);        \
+  _CLC_DECL _CLC_OVERLOAD TYPE##3 __CLC_TABLE_MANGLE(NAME)(int##3 idx);        \
+  _CLC_DECL _CLC_OVERLOAD TYPE##4 __CLC_TABLE_MANGLE(NAME)(int##4 idx);        \
+  _CLC_DECL _CLC_OVERLOAD TYPE##8 __CLC_TABLE_MANGLE(NAME)(int##8 idx);        \
+  _CLC_DECL _CLC_OVERLOAD TYPE##16 __CLC_TABLE_MANGLE(NAME)(int##16 idx);
+
+#define __CLC_USE_TABLE(NAME, IDX) __CLC_TABLE_MANGLE(NAME)(IDX)
+
+__CLC_TABLE_FUNCTION_DECL(float2, log2_tbl);
+__CLC_TABLE_FUNCTION_DECL(float2, log10_tbl);
+
+__CLC_TABLE_FUNCTION_DECL_VEC(float, log_inv_tbl_ep_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, log_inv_tbl_ep_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, loge_tbl_lo);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, loge_tbl_hi);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, log_inv_tbl);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, exp_tbl);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, exp_tbl_ep_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, exp_tbl_ep_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, cbrt_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, cbrt_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, sinhcosh_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(float, sinhcosh_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(ulong, pibits_tbl);
 
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-CLC_TABLE_FUNCTION_DECL(double, ln_tbl_lo);
-CLC_TABLE_FUNCTION_DECL(double, ln_tbl_hi);
-CLC_TABLE_FUNCTION_DECL(double, atan_jby256_tbl_head);
-CLC_TABLE_FUNCTION_DECL(double, atan_jby256_tbl_tail);
-CLC_TABLE_FUNCTION_DECL(double, two_to_jby64_ep_tbl_head);
-CLC_TABLE_FUNCTION_DECL(double, two_to_jby64_ep_tbl_tail);
-CLC_TABLE_FUNCTION_DECL(double, sinh_tbl_head);
-CLC_TABLE_FUNCTION_DECL(double, sinh_tbl_tail);
-CLC_TABLE_FUNCTION_DECL(double, cosh_tbl_head);
-CLC_TABLE_FUNCTION_DECL(double, cosh_tbl_tail);
-CLC_TABLE_FUNCTION_DECL(double, cbrt_inv_tbl);
-CLC_TABLE_FUNCTION_DECL(double, cbrt_dbl_tbl_head);
-CLC_TABLE_FUNCTION_DECL(double, cbrt_dbl_tbl_tail);
-CLC_TABLE_FUNCTION_DECL(double, cbrt_rem_tbl_head);
-CLC_TABLE_FUNCTION_DECL(double, cbrt_rem_tbl_tail);
-CLC_TABLE_FUNCTION_DECL(double, powlog_tbl_head);
-CLC_TABLE_FUNCTION_DECL(double, powlog_tbl_tail);
-CLC_TABLE_FUNCTION_DECL(double, log_f_inv_tbl_head);
-CLC_TABLE_FUNCTION_DECL(double, log_f_inv_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, ln_tbl_lo);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, ln_tbl_hi);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, atan_jby256_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, atan_jby256_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, two_to_jby64_ep_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, two_to_jby64_ep_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, sinh_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, sinh_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, cosh_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, cosh_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, cbrt_inv_tbl);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, cbrt_dbl_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, cbrt_dbl_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, cbrt_rem_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, cbrt_rem_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, powlog_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, powlog_tbl_tail);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, log_f_inv_tbl_head);
+__CLC_TABLE_FUNCTION_DECL_VEC(double, log_f_inv_tbl_tail);
 
 #endif // cl_khr_fp64
 
diff --git a/libclc/clc/include/clc/math/unary_decl.inc b/libclc/clc/include/clc/math/unary_decl.inc
index ce1a71b8284aa..19f1c8a876e53 100644
--- a/libclc/clc/include/clc/math/unary_decl.inc
+++ b/libclc/clc/include/clc/math/unary_decl.inc
@@ -6,4 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc b/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc
index d62046a5292cd..8bfe17c3c6817 100644
--- a/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_decl_with_int_ptr.inc
@@ -6,13 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               global __CLC_INTN *iptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               local __CLC_INTN *iptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               private __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     global __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     local __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     private __CLC_INTN *iptr);
 #if _CLC_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               generic __CLC_INTN *iptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     generic __CLC_INTN *iptr);
 #endif
diff --git a/libclc/clc/include/clc/math/unary_decl_with_int_return.inc b/libclc/clc/include/clc/math/unary_decl_with_int_return.inc
index ed8d0a12506f2..2e86a310b018b 100644
--- a/libclc/clc/include/clc/math/unary_decl_with_int_return.inc
+++ b/libclc/clc/include/clc/math/unary_decl_with_int_return.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_INTN FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_INTN __CLC_FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/math/unary_decl_with_ptr.inc b/libclc/clc/include/clc/math/unary_decl_with_ptr.inc
index ca0077bec558f..63a20e3d21c5d 100644
--- a/libclc/clc/include/clc/math/unary_decl_with_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_decl_with_ptr.inc
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               global __CLC_GENTYPE *ptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               local __CLC_GENTYPE *ptr);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               private __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     global __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                     local __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE x, private __CLC_GENTYPE *ptr);
 
 #if _CLC_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               generic __CLC_GENTYPE *ptr);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE x, generic __CLC_GENTYPE *ptr);
 #endif
diff --git a/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc b/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc
index a8ed34ad04bbf..db6472ad0e645 100644
--- a/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_def_with_int_ptr.inc
@@ -8,28 +8,28 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                              private __CLC_INTN *iptr) {
-  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                    private __CLC_INTN *iptr) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, iptr);
 }
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                              global __CLC_INTN *iptr) {
-  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                    global __CLC_INTN *iptr) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, iptr);
 }
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                              local __CLC_INTN *iptr) {
-  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                    local __CLC_INTN *iptr) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, iptr);
 }
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                              generic __CLC_INTN *iptr) {
-  return __IMPL_FUNCTION(FUNCTION)(x, iptr);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                    generic __CLC_INTN *iptr) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, iptr);
 }
 #endif
diff --git a/libclc/clc/include/clc/math/unary_def_with_int_return.inc b/libclc/clc/include/clc/math/unary_def_with_int_return.inc
index c7bb84cf5f7c8..7a47aae8ce637 100644
--- a/libclc/clc/include/clc/math/unary_def_with_int_return.inc
+++ b/libclc/clc/include/clc/math/unary_def_with_int_return.inc
@@ -8,10 +8,10 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __CLC_INTN FUNCTION(__CLC_GENTYPE a) {
-  return __IMPL_FUNCTION(FUNCTION)(a);
+_CLC_OVERLOAD _CLC_DEF __CLC_INTN __CLC_FUNCTION(__CLC_GENTYPE a) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a);
 }
diff --git a/libclc/clc/include/clc/math/unary_def_with_ptr.inc b/libclc/clc/include/clc/math/unary_def_with_ptr.inc
index c3faa9aab87c9..8a490e1bba16d 100644
--- a/libclc/clc/include/clc/math/unary_def_with_ptr.inc
+++ b/libclc/clc/include/clc/math/unary_def_with_ptr.inc
@@ -8,28 +8,28 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                              private __CLC_GENTYPE *ptr) {
-  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE x, private __CLC_GENTYPE *ptr) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, ptr);
 }
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                              global __CLC_GENTYPE *ptr) {
-  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                    global __CLC_GENTYPE *ptr) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, ptr);
 }
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                              local __CLC_GENTYPE *ptr) {
-  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                    local __CLC_GENTYPE *ptr) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, ptr);
 }
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                              generic __CLC_GENTYPE *ptr) {
-  return __IMPL_FUNCTION(FUNCTION)(x, ptr);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE x, generic __CLC_GENTYPE *ptr) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, ptr);
 }
 #endif
diff --git a/libclc/clc/include/clc/misc/clc_shuffle.h b/libclc/clc/include/clc/misc/clc_shuffle.h
index 1e9ec6828f95a..ba3d7a2bddd62 100644
--- a/libclc/clc/include/clc/misc/clc_shuffle.h
+++ b/libclc/clc/include/clc/misc/clc_shuffle.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_MISC_CLC_SHUFFLE_H__
 #define __CLC_MISC_CLC_SHUFFLE_H__
 
-#define FUNCTION __clc_shuffle
+#define __CLC_FUNCTION __clc_shuffle
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MISC_CLC_SHUFFLE_H__
diff --git a/libclc/clc/include/clc/misc/clc_shuffle2.h b/libclc/clc/include/clc/misc/clc_shuffle2.h
index fb5361cfeb95a..84bafba12b211 100644
--- a/libclc/clc/include/clc/misc/clc_shuffle2.h
+++ b/libclc/clc/include/clc/misc/clc_shuffle2.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_MISC_CLC_SHUFFLE2_H__
 #define __CLC_MISC_CLC_SHUFFLE2_H__
 
-#define FUNCTION __clc_shuffle2
+#define __CLC_FUNCTION __clc_shuffle2
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_MISC_CLC_SHUFFLE2_H__
diff --git a/libclc/clc/include/clc/misc/shuffle2_decl.inc b/libclc/clc/include/clc/misc/shuffle2_decl.inc
index 47fd5e28a3142..37f83cba9939c 100644
--- a/libclc/clc/include/clc/misc/shuffle2_decl.inc
+++ b/libclc/clc/include/clc/misc/shuffle2_decl.inc
@@ -12,17 +12,17 @@
 // The return type is same base type as the input type, with the same vector
 // size as the mask. Elements in the mask must be the same size (number of bits)
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
-         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
-         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
-         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
-         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
+               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
+               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
+               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
+               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask);
 
 #endif
diff --git a/libclc/clc/include/clc/misc/shuffle2_def.inc b/libclc/clc/include/clc/misc/shuffle2_def.inc
index 0415b8c187291..f25d281af1169 100644
--- a/libclc/clc/include/clc/misc/shuffle2_def.inc
+++ b/libclc/clc/include/clc/misc/shuffle2_def.inc
@@ -11,32 +11,32 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // The return type is same base type as the input type, with the same vector
 // size as the mask. Elements in the mask must be the same size (number of bits)
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
-         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask) {
-  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x,
+               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) y, __CLC_U_GENTYPE mask) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, y, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
-         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask) {
-  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x,
+               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) y, __CLC_U_GENTYPE mask) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, y, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
-         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask) {
-  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x,
+               __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) y, __CLC_U_GENTYPE mask) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, y, mask);
 }
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
-         __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask) {
-  return __IMPL_FUNCTION(FUNCTION)(x, y, mask);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(
+    __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x,
+    __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) y, __CLC_U_GENTYPE mask) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, y, mask);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/misc/shuffle_decl.inc b/libclc/clc/include/clc/misc/shuffle_decl.inc
index c6c64d5b5fede..0056d12f0f428 100644
--- a/libclc/clc/include/clc/misc/shuffle_decl.inc
+++ b/libclc/clc/include/clc/misc/shuffle_decl.inc
@@ -12,13 +12,13 @@
 // The return type is same base type as the input type, with the same vector
 // size as the mask. Elements in the mask must be the same size (number of bits)
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask);
 
 #endif
diff --git a/libclc/clc/include/clc/misc/shuffle_def.inc b/libclc/clc/include/clc/misc/shuffle_def.inc
index d2a088498fc6c..49a47daf821fe 100644
--- a/libclc/clc/include/clc/misc/shuffle_def.inc
+++ b/libclc/clc/include/clc/misc/shuffle_def.inc
@@ -11,28 +11,28 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
 // The return type is same base type as the input type, with the same vector
 // size as the mask. Elements in the mask must be the same size (number of bits)
 // as the input value., e.g. char8 ret = shuffle(char2 x, uchar8 mask);
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask) {
-  return __IMPL_FUNCTION(FUNCTION)(x, mask);
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 2) x, __CLC_U_GENTYPE mask) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask) {
-  return __IMPL_FUNCTION(FUNCTION)(x, mask);
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 4) x, __CLC_U_GENTYPE mask) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, mask);
 }
 _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask) {
-  return __IMPL_FUNCTION(FUNCTION)(x, mask);
+__CLC_FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 8) x, __CLC_U_GENTYPE mask) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, mask);
 }
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE
-FUNCTION(__CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask) {
-  return __IMPL_FUNCTION(FUNCTION)(x, mask);
+_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(
+    __CLC_XCONCAT(__CLC_SCALAR_GENTYPE, 16) x, __CLC_U_GENTYPE mask) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, mask);
 }
 
 #endif
diff --git a/libclc/clc/include/clc/relational/binary_decl.inc b/libclc/clc/include/clc/relational/binary_decl.inc
index dc8ec9db7b7da..34749559c7e09 100644
--- a/libclc/clc/include/clc/relational/binary_decl.inc
+++ b/libclc/clc/include/clc/relational/binary_decl.inc
@@ -7,11 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #if __CLC_VECSIZE_OR_1 == 1
-#define __RETTYPE __CLC_INTN
+#define __CLC_RETTYPE __CLC_INTN
 #else
-#define __RETTYPE __CLC_BIT_INTN
+#define __CLC_RETTYPE __CLC_BIT_INTN
 #endif
 
-_CLC_OVERLOAD _CLC_DECL __RETTYPE FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_RETTYPE
+__CLC_FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b);
 
-#undef __RETTYPE
+#undef __CLC_RETTYPE
diff --git a/libclc/clc/include/clc/relational/clc_all.h b/libclc/clc/include/clc/relational/clc_all.h
index 4d01fb7795b72..272d879e08cd5 100644
--- a/libclc/clc/include/clc/relational/clc_all.h
+++ b/libclc/clc/include/clc/relational/clc_all.h
@@ -12,7 +12,8 @@
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
-#define _CLC_ALL_DECL(TYPE) _CLC_OVERLOAD _CLC_DECL int __clc_all(TYPE v);
+#define _CLC_ALL_DECL(TYPE)                                                    \
+  _CLC_OVERLOAD _CLC_CONST _CLC_DECL int __clc_all(TYPE v);
 
 #define _CLC_VECTOR_ALL_DECL(TYPE)                                             \
   _CLC_ALL_DECL(TYPE)                                                          \
diff --git a/libclc/clc/include/clc/relational/clc_any.h b/libclc/clc/include/clc/relational/clc_any.h
index 1e287af58d880..82d08c0129e91 100644
--- a/libclc/clc/include/clc/relational/clc_any.h
+++ b/libclc/clc/include/clc/relational/clc_any.h
@@ -12,7 +12,8 @@
 #include <clc/clcfunc.h>
 #include <clc/clctypes.h>
 
-#define _CLC_ANY_DECL(TYPE) _CLC_OVERLOAD _CLC_DECL int __clc_any(TYPE v);
+#define _CLC_ANY_DECL(TYPE)                                                    \
+  _CLC_OVERLOAD _CLC_CONST _CLC_DECL int __clc_any(TYPE v);
 
 #define _CLC_VECTOR_ANY_DECL(TYPE)                                             \
   _CLC_ANY_DECL(TYPE)                                                          \
diff --git a/libclc/clc/include/clc/relational/clc_bitselect.inc b/libclc/clc/include/clc/relational/clc_bitselect.inc
index b81cfc2ef0468..564f3b00b5f1c 100644
--- a/libclc/clc/include/clc/relational/clc_bitselect.inc
+++ b/libclc/clc/include/clc/relational/clc_bitselect.inc
@@ -6,6 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_bitselect(__CLC_GENTYPE x,
-                                                      __CLC_GENTYPE y,
-                                                      __CLC_GENTYPE z);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__clc_bitselect(__CLC_GENTYPE x, __CLC_GENTYPE y, __CLC_GENTYPE z);
diff --git a/libclc/clc/include/clc/relational/clc_isequal.h b/libclc/clc/include/clc/relational/clc_isequal.h
index f3789f595637e..2585dbdaf3cfe 100644
--- a/libclc/clc/include/clc/relational/clc_isequal.h
+++ b/libclc/clc/include/clc/relational/clc_isequal.h
@@ -13,7 +13,7 @@
 #include <clc/clctypes.h>
 
 #define _CLC_ISEQUAL_DECL(TYPE, RETTYPE)                                       \
-  _CLC_OVERLOAD _CLC_DECL RETTYPE __clc_isequal(TYPE x, TYPE y);
+  _CLC_OVERLOAD _CLC_CONST _CLC_DECL RETTYPE __clc_isequal(TYPE x, TYPE y);
 
 #define _CLC_VECTOR_ISEQUAL_DECL(TYPE, RETTYPE)                                \
   _CLC_ISEQUAL_DECL(TYPE##2, RETTYPE##2)                                       \
diff --git a/libclc/clc/include/clc/relational/clc_isfinite.h b/libclc/clc/include/clc/relational/clc_isfinite.h
index 444d733039819..596b81f51b853 100644
--- a/libclc/clc/include/clc/relational/clc_isfinite.h
+++ b/libclc/clc/include/clc/relational/clc_isfinite.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISFINITE_H__
 #define __CLC_RELATIONAL_CLC_ISFINITE_H__
 
-#define FUNCTION __clc_isfinite
+#define __CLC_FUNCTION __clc_isfinite
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISFINITE_H__
diff --git a/libclc/clc/include/clc/relational/clc_isgreater.h b/libclc/clc/include/clc/relational/clc_isgreater.h
index 88de46854961d..fc5b3cdad084a 100644
--- a/libclc/clc/include/clc/relational/clc_isgreater.h
+++ b/libclc/clc/include/clc/relational/clc_isgreater.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISGREATER_H__
 #define __CLC_RELATIONAL_CLC_ISGREATER_H__
 
-#define FUNCTION __clc_isgreater
+#define __CLC_FUNCTION __clc_isgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISGREATER_H__
diff --git a/libclc/clc/include/clc/relational/clc_isgreaterequal.h b/libclc/clc/include/clc/relational/clc_isgreaterequal.h
index 42308036f102f..6209a066cf46d 100644
--- a/libclc/clc/include/clc/relational/clc_isgreaterequal.h
+++ b/libclc/clc/include/clc/relational/clc_isgreaterequal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
 
-#define FUNCTION __clc_isgreaterequal
+#define __CLC_FUNCTION __clc_isgreaterequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISGREATEREQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isinf.h b/libclc/clc/include/clc/relational/clc_isinf.h
index 39a9de6c434e1..fcd87e94c6ee2 100644
--- a/libclc/clc/include/clc/relational/clc_isinf.h
+++ b/libclc/clc/include/clc/relational/clc_isinf.h
@@ -13,7 +13,7 @@
 #include <clc/clctypes.h>
 
 #define _CLC_ISINF_DECL(RET_TYPE, ARG_TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL RET_TYPE __clc_isinf(ARG_TYPE);
+  _CLC_OVERLOAD _CLC_CONST _CLC_DECL RET_TYPE __clc_isinf(ARG_TYPE);
 
 #define _CLC_VECTOR_ISINF_DECL(RET_TYPE, ARG_TYPE)                             \
   _CLC_ISINF_DECL(RET_TYPE##2, ARG_TYPE##2)                                    \
diff --git a/libclc/clc/include/clc/relational/clc_isless.h b/libclc/clc/include/clc/relational/clc_isless.h
index 6fdc6c54947c0..4e340db121773 100644
--- a/libclc/clc/include/clc/relational/clc_isless.h
+++ b/libclc/clc/include/clc/relational/clc_isless.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESS_H__
 #define __CLC_RELATIONAL_CLC_ISLESS_H__
 
-#define FUNCTION __clc_isless
+#define __CLC_FUNCTION __clc_isless
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISLESS_H__
diff --git a/libclc/clc/include/clc/relational/clc_islessequal.h b/libclc/clc/include/clc/relational/clc_islessequal.h
index e592287b23099..14c5a10301836 100644
--- a/libclc/clc/include/clc/relational/clc_islessequal.h
+++ b/libclc/clc/include/clc/relational/clc_islessequal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
 
-#define FUNCTION __clc_islessequal
+#define __CLC_FUNCTION __clc_islessequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISLESSEQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_islessgreater.h b/libclc/clc/include/clc/relational/clc_islessgreater.h
index a2f10707a677d..530564bbf0595 100644
--- a/libclc/clc/include/clc/relational/clc_islessgreater.h
+++ b/libclc/clc/include/clc/relational/clc_islessgreater.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
 #define __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
 
-#define FUNCTION __clc_islessgreater
+#define __CLC_FUNCTION __clc_islessgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISLESSGREATER_H__
diff --git a/libclc/clc/include/clc/relational/clc_isnan.h b/libclc/clc/include/clc/relational/clc_isnan.h
index d0821a3c52639..779e6cc94b91b 100644
--- a/libclc/clc/include/clc/relational/clc_isnan.h
+++ b/libclc/clc/include/clc/relational/clc_isnan.h
@@ -13,7 +13,7 @@
 #include <clc/clctypes.h>
 
 #define _CLC_ISNAN_DECL(RET_TYPE, ARG_TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL RET_TYPE __clc_isnan(ARG_TYPE);
+  _CLC_OVERLOAD _CLC_CONST _CLC_DECL RET_TYPE __clc_isnan(ARG_TYPE);
 
 #define _CLC_VECTOR_ISNAN_DECL(RET_TYPE, ARG_TYPE)                             \
   _CLC_ISNAN_DECL(RET_TYPE##2, ARG_TYPE##2)                                    \
diff --git a/libclc/clc/include/clc/relational/clc_isnormal.h b/libclc/clc/include/clc/relational/clc_isnormal.h
index 2281bc4245d03..9572b02004686 100644
--- a/libclc/clc/include/clc/relational/clc_isnormal.h
+++ b/libclc/clc/include/clc/relational/clc_isnormal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISNORMAL_H__
 #define __CLC_RELATIONAL_CLC_ISNORMAL_H__
 
-#define FUNCTION __clc_isnormal
+#define __CLC_FUNCTION __clc_isnormal
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISNORMAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isnotequal.h b/libclc/clc/include/clc/relational/clc_isnotequal.h
index c2640fc0899a6..f90622db9fe1d 100644
--- a/libclc/clc/include/clc/relational/clc_isnotequal.h
+++ b/libclc/clc/include/clc/relational/clc_isnotequal.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
 #define __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
 
-#define FUNCTION __clc_isnotequal
+#define __CLC_FUNCTION __clc_isnotequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISNOTEQUAL_H__
diff --git a/libclc/clc/include/clc/relational/clc_isordered.h b/libclc/clc/include/clc/relational/clc_isordered.h
index cb9be31311575..3f5cc097f4a72 100644
--- a/libclc/clc/include/clc/relational/clc_isordered.h
+++ b/libclc/clc/include/clc/relational/clc_isordered.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISORDERED_H__
 #define __CLC_RELATIONAL_CLC_ISORDERED_H__
 
-#define FUNCTION __clc_isordered
+#define __CLC_FUNCTION __clc_isordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISORDERED_H__
diff --git a/libclc/clc/include/clc/relational/clc_isunordered.h b/libclc/clc/include/clc/relational/clc_isunordered.h
index 36d314ff0e1be..6134ba849d229 100644
--- a/libclc/clc/include/clc/relational/clc_isunordered.h
+++ b/libclc/clc/include/clc/relational/clc_isunordered.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_ISUNORDERED_H__
 #define __CLC_RELATIONAL_CLC_ISUNORDERED_H__
 
-#define FUNCTION __clc_isunordered
+#define __CLC_FUNCTION __clc_isunordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_ISUNORDERED_H__
diff --git a/libclc/clc/include/clc/relational/clc_select_decl.inc b/libclc/clc/include/clc/relational/clc_select_decl.inc
index 1cf026dc8db36..a2350f7b78440 100644
--- a/libclc/clc/include/clc/relational/clc_select_decl.inc
+++ b/libclc/clc/include/clc/relational/clc_select_decl.inc
@@ -6,9 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_SELECT_FN(__CLC_GENTYPE x,
-                                                      __CLC_GENTYPE y,
-                                                      __CLC_S_GENTYPE z);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_SELECT_FN(__CLC_GENTYPE x,
-                                                      __CLC_GENTYPE y,
-                                                      __CLC_U_GENTYPE z);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_SELECT_FN(__CLC_GENTYPE x, __CLC_GENTYPE y, __CLC_S_GENTYPE z);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_SELECT_FN(__CLC_GENTYPE x, __CLC_GENTYPE y, __CLC_U_GENTYPE z);
diff --git a/libclc/clc/include/clc/relational/clc_signbit.h b/libclc/clc/include/clc/relational/clc_signbit.h
index 9e423ab448953..a8e04ce494f2c 100644
--- a/libclc/clc/include/clc/relational/clc_signbit.h
+++ b/libclc/clc/include/clc/relational/clc_signbit.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_RELATIONAL_CLC_SIGNBIT_H__
 #define __CLC_RELATIONAL_CLC_SIGNBIT_H__
 
-#define FUNCTION __clc_signbit
+#define __CLC_FUNCTION __clc_signbit
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_RELATIONAL_CLC_SIGNBIT_H__
diff --git a/libclc/clc/include/clc/relational/relational.h b/libclc/clc/include/clc/relational/relational.h
index e2b577019a50f..fa22232349944 100644
--- a/libclc/clc/include/clc/relational/relational.h
+++ b/libclc/clc/include/clc/relational/relational.h
@@ -14,29 +14,29 @@
  * when the result is true.
  */
 
-#define _CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(RET_TYPE, RET_TYPE_VEC, FUNCTION, \
-                                             ARG1_TYPE, ARG2_TYPE)             \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {         \
+#define _CLC_DEFINE_SIMPLE_RELATIONAL_BINARY(                                  \
+    RET_TYPE, RET_TYPE_VEC, __CLC_FUNCTION, ARG1_TYPE, ARG2_TYPE)              \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE __CLC_FUNCTION(ARG1_TYPE x, ARG2_TYPE y) {   \
     return _CLC_RELATIONAL_OP(x, y);                                           \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##2 FUNCTION(ARG1_TYPE##2 x,              \
-                                                  ARG2_TYPE##2 y) {            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##2 __CLC_FUNCTION(ARG1_TYPE##2 x,        \
+                                                        ARG2_TYPE##2 y) {      \
     return _CLC_RELATIONAL_OP(x, y);                                           \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##3 FUNCTION(ARG1_TYPE##3 x,              \
-                                                  ARG2_TYPE##3 y) {            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##3 __CLC_FUNCTION(ARG1_TYPE##3 x,        \
+                                                        ARG2_TYPE##3 y) {      \
     return _CLC_RELATIONAL_OP(x, y);                                           \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##4 FUNCTION(ARG1_TYPE##4 x,              \
-                                                  ARG2_TYPE##4 y) {            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##4 __CLC_FUNCTION(ARG1_TYPE##4 x,        \
+                                                        ARG2_TYPE##4 y) {      \
     return _CLC_RELATIONAL_OP(x, y);                                           \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##8 FUNCTION(ARG1_TYPE##8 x,              \
-                                                  ARG2_TYPE##8 y) {            \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##8 __CLC_FUNCTION(ARG1_TYPE##8 x,        \
+                                                        ARG2_TYPE##8 y) {      \
     return _CLC_RELATIONAL_OP(x, y);                                           \
   }                                                                            \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##16 FUNCTION(ARG1_TYPE##16 x,            \
-                                                   ARG2_TYPE##16 y) {          \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE_VEC##16 __CLC_FUNCTION(ARG1_TYPE##16 x,      \
+                                                         ARG2_TYPE##16 y) {    \
     return _CLC_RELATIONAL_OP(x, y);                                           \
   }
 
@@ -49,20 +49,25 @@
   (__FPCLASS_NEGNORMAL | __FPCLASS_NEGSUBNORMAL | __FPCLASS_NEGZERO)
 #define fcFinite (fcPosFinite | fcNegFinite)
 
-#define _CLC_DEFINE_ISFPCLASS_VEC(RET_TYPE, FUNCTION, MASK, ARG_TYPE)          \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
+#define _CLC_DEFINE_ISFPCLASS_VEC(RET_TYPE, __CLC_FUNCTION, MASK, ARG_TYPE)    \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE __CLC_FUNCTION(ARG_TYPE x) {                 \
     return (RET_TYPE)(__builtin_isfpclass(x, (MASK)) != (RET_TYPE)0);          \
   }
 
-#define _CLC_DEFINE_ISFPCLASS(RET_TYPE, VEC_RET_TYPE, FUNCTION, MASK,          \
+#define _CLC_DEFINE_ISFPCLASS(RET_TYPE, VEC_RET_TYPE, __CLC_FUNCTION, MASK,    \
                               ARG_TYPE)                                        \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE __CLC_FUNCTION(ARG_TYPE x) {                 \
     return __builtin_isfpclass(x, (MASK));                                     \
   }                                                                            \
-  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##2, FUNCTION, MASK, ARG_TYPE##2)      \
-  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##3, FUNCTION, MASK, ARG_TYPE##3)      \
-  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##4, FUNCTION, MASK, ARG_TYPE##4)      \
-  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##8, FUNCTION, MASK, ARG_TYPE##8)      \
-  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##16, FUNCTION, MASK, ARG_TYPE##16)
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##2, __CLC_FUNCTION, MASK,             \
+                            ARG_TYPE##2)                                       \
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##3, __CLC_FUNCTION, MASK,             \
+                            ARG_TYPE##3)                                       \
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##4, __CLC_FUNCTION, MASK,             \
+                            ARG_TYPE##4)                                       \
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##8, __CLC_FUNCTION, MASK,             \
+                            ARG_TYPE##8)                                       \
+  _CLC_DEFINE_ISFPCLASS_VEC(VEC_RET_TYPE##16, __CLC_FUNCTION, MASK,            \
+                            ARG_TYPE##16)
 
 #endif // __CLC_RELATIONAL_RELATIONAL_H__
diff --git a/libclc/clc/include/clc/relational/unary_decl.inc b/libclc/clc/include/clc/relational/unary_decl.inc
index cc3f2d065529b..df74194040253 100644
--- a/libclc/clc/include/clc/relational/unary_decl.inc
+++ b/libclc/clc/include/clc/relational/unary_decl.inc
@@ -7,11 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #if __CLC_VECSIZE_OR_1 == 1
-#define __RETTYPE __CLC_INTN
+#define __CLC_RETTYPE __CLC_INTN
 #else
-#define __RETTYPE __CLC_BIT_INTN
+#define __CLC_RETTYPE __CLC_BIT_INTN
 #endif
 
-_CLC_OVERLOAD _CLC_DECL __RETTYPE FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_RETTYPE
+__CLC_FUNCTION(__CLC_GENTYPE x);
 
-#undef __RETTYPE
+#undef __CLC_RETTYPE
diff --git a/libclc/clc/include/clc/shared/binary_decl.inc b/libclc/clc/include/clc/shared/binary_decl.inc
index ff4739dcdb8d1..f7f49519f6fca 100644
--- a/libclc/clc/include/clc/shared/binary_decl.inc
+++ b/libclc/clc/include/clc/shared/binary_decl.inc
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               __CLC_GENTYPE y);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE x, __CLC_GENTYPE y);
diff --git a/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc b/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc
index a86d89b4fc85f..25cf52c84811e 100644
--- a/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_decl_with_int_second_arg.inc
@@ -8,4 +8,5 @@
 
 #include <clc/utils.h>
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_INTN y);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                                __CLC_INTN y);
diff --git a/libclc/clc/include/clc/shared/binary_def.inc b/libclc/clc/include/clc/shared/binary_def.inc
index 9ab5a7c5a79e5..5487ad926768d 100644
--- a/libclc/clc/include/clc/shared/binary_def.inc
+++ b/libclc/clc/include/clc/shared/binary_def.inc
@@ -8,11 +8,11 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
-                                              __CLC_GENTYPE b) {
-  return __IMPL_FUNCTION(FUNCTION)(a, b);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
+                                                    __CLC_GENTYPE b) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a, b);
 }
diff --git a/libclc/clc/include/clc/shared/binary_def_scalarize.inc b/libclc/clc/include/clc/shared/binary_def_scalarize.inc
index 53d4d3fa23b13..2505f2ee0da2d 100644
--- a/libclc/clc/include/clc/shared/binary_def_scalarize.inc
+++ b/libclc/clc/include/clc/shared/binary_def_scalarize.inc
@@ -14,8 +14,8 @@
 #define __CLC_MIN_VECSIZE 2
 #endif
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION FUNCTION
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION __CLC_FUNCTION
 #endif
 
 #ifndef __CLC_DEF_SPEC
@@ -35,9 +35,9 @@
 #endif
 
 #if __CLC_MIN_VECSIZE == 1
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x,
-                                                     __CLC_ARG2_TYPE y) {
-  return __IMPL_FUNCTION(x, y);
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE __CLC_FUNCTION(__CLC_ARG1_TYPE x,
+                                                           __CLC_ARG2_TYPE y) {
+  return __CLC_IMPL_FUNCTION(x, y);
 }
 #endif // __CLC_MIN_VECSIZE == 1
 
@@ -45,10 +45,10 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x,
 #define __CLC_RET_TYPE2 __CLC_XCONCAT(__CLC_RET_TYPE, 2)
 #define __CLC_ARG1_TYPE2 __CLC_XCONCAT(__CLC_ARG1_TYPE, 2)
 #define __CLC_ARG2_TYPE2 __CLC_XCONCAT(__CLC_ARG2_TYPE, 2)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x,
-                                                      __CLC_ARG2_TYPE2 y) {
-  return (__CLC_RET_TYPE2)(__IMPL_FUNCTION(x.s0, y.s0),
-                           __IMPL_FUNCTION(x.s1, y.s1));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2
+__CLC_FUNCTION(__CLC_ARG1_TYPE2 x, __CLC_ARG2_TYPE2 y) {
+  return (__CLC_RET_TYPE2)(__CLC_IMPL_FUNCTION(x.s0, y.s0),
+                           __CLC_IMPL_FUNCTION(x.s1, y.s1));
 }
 #undef __CLC_RET_TYPE2
 #undef __CLC_ARG1_TYPE2
@@ -62,11 +62,11 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x,
 #define __CLC_RET_TYPE3 __CLC_XCONCAT(__CLC_RET_TYPE, 3)
 #define __CLC_ARG1_TYPE3 __CLC_XCONCAT(__CLC_ARG1_TYPE, 3)
 #define __CLC_ARG2_TYPE3 __CLC_XCONCAT(__CLC_ARG2_TYPE, 3)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x,
-                                                      __CLC_ARG2_TYPE3 y) {
-  return (__CLC_RET_TYPE3)(__IMPL_FUNCTION(x.s0, y.s0),
-                           __IMPL_FUNCTION(x.s1, y.s1),
-                           __IMPL_FUNCTION(x.s2, y.s2));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3
+__CLC_FUNCTION(__CLC_ARG1_TYPE3 x, __CLC_ARG2_TYPE3 y) {
+  return (__CLC_RET_TYPE3)(__CLC_IMPL_FUNCTION(x.s0, y.s0),
+                           __CLC_IMPL_FUNCTION(x.s1, y.s1),
+                           __CLC_IMPL_FUNCTION(x.s2, y.s2));
 }
 #undef __CLC_RET_TYPE3
 #undef __CLC_ARG1_TYPE3
@@ -75,12 +75,12 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x,
 #define __CLC_RET_TYPE4 __CLC_XCONCAT(__CLC_RET_TYPE, 4)
 #define __CLC_ARG1_TYPE4 __CLC_XCONCAT(__CLC_ARG1_TYPE, 4)
 #define __CLC_ARG2_TYPE4 __CLC_XCONCAT(__CLC_ARG2_TYPE, 4)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x,
-                                                      __CLC_ARG2_TYPE4 y) {
-  return (__CLC_RET_TYPE4)(__IMPL_FUNCTION(x.s0, y.s0),
-                           __IMPL_FUNCTION(x.s1, y.s1),
-                           __IMPL_FUNCTION(x.s2, y.s2),
-                           __IMPL_FUNCTION(x.s3, y.s3));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4
+__CLC_FUNCTION(__CLC_ARG1_TYPE4 x, __CLC_ARG2_TYPE4 y) {
+  return (__CLC_RET_TYPE4)(__CLC_IMPL_FUNCTION(x.s0, y.s0),
+                           __CLC_IMPL_FUNCTION(x.s1, y.s1),
+                           __CLC_IMPL_FUNCTION(x.s2, y.s2),
+                           __CLC_IMPL_FUNCTION(x.s3, y.s3));
 }
 #undef __CLC_RET_TYPE4
 #undef __CLC_ARG1_TYPE4
@@ -89,14 +89,16 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x,
 #define __CLC_RET_TYPE8 __CLC_XCONCAT(__CLC_RET_TYPE, 8)
 #define __CLC_ARG1_TYPE8 __CLC_XCONCAT(__CLC_ARG1_TYPE, 8)
 #define __CLC_ARG2_TYPE8 __CLC_XCONCAT(__CLC_ARG2_TYPE, 8)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x,
-                                                      __CLC_ARG2_TYPE8 y) {
-  return (
-      __CLC_RET_TYPE8)(__IMPL_FUNCTION(x.s0, y.s0), __IMPL_FUNCTION(x.s1, y.s1),
-                       __IMPL_FUNCTION(x.s2, y.s2), __IMPL_FUNCTION(x.s3, y.s3),
-                       __IMPL_FUNCTION(x.s4, y.s4), __IMPL_FUNCTION(x.s5, y.s5),
-                       __IMPL_FUNCTION(x.s6, y.s6),
-                       __IMPL_FUNCTION(x.s7, y.s7));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8
+__CLC_FUNCTION(__CLC_ARG1_TYPE8 x, __CLC_ARG2_TYPE8 y) {
+  return (__CLC_RET_TYPE8)(__CLC_IMPL_FUNCTION(x.s0, y.s0),
+                           __CLC_IMPL_FUNCTION(x.s1, y.s1),
+                           __CLC_IMPL_FUNCTION(x.s2, y.s2),
+                           __CLC_IMPL_FUNCTION(x.s3, y.s3),
+                           __CLC_IMPL_FUNCTION(x.s4, y.s4),
+                           __CLC_IMPL_FUNCTION(x.s5, y.s5),
+                           __CLC_IMPL_FUNCTION(x.s6, y.s6),
+                           __CLC_IMPL_FUNCTION(x.s7, y.s7));
 }
 #undef __CLC_RET_TYPE8
 #undef __CLC_ARG1_TYPE8
@@ -105,24 +107,24 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x,
 #define __CLC_RET_TYPE16 __CLC_XCONCAT(__CLC_RET_TYPE, 16)
 #define __CLC_ARG1_TYPE16 __CLC_XCONCAT(__CLC_ARG1_TYPE, 16)
 #define __CLC_ARG2_TYPE16 __CLC_XCONCAT(__CLC_ARG2_TYPE, 16)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE16 FUNCTION(__CLC_ARG1_TYPE16 x,
-                                                       __CLC_ARG2_TYPE16 y) {
-  return (__CLC_RET_TYPE16)(__IMPL_FUNCTION(x.s0, y.s0),
-                            __IMPL_FUNCTION(x.s1, y.s1),
-                            __IMPL_FUNCTION(x.s2, y.s2),
-                            __IMPL_FUNCTION(x.s3, y.s3),
-                            __IMPL_FUNCTION(x.s4, y.s4),
-                            __IMPL_FUNCTION(x.s5, y.s5),
-                            __IMPL_FUNCTION(x.s6, y.s6),
-                            __IMPL_FUNCTION(x.s7, y.s7),
-                            __IMPL_FUNCTION(x.s8, y.s8),
-                            __IMPL_FUNCTION(x.s9, y.s9),
-                            __IMPL_FUNCTION(x.sa, y.sa),
-                            __IMPL_FUNCTION(x.sb, y.sb),
-                            __IMPL_FUNCTION(x.sc, y.sc),
-                            __IMPL_FUNCTION(x.sd, y.sd),
-                            __IMPL_FUNCTION(x.se, y.se),
-                            __IMPL_FUNCTION(x.sf, y.sf));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE16
+__CLC_FUNCTION(__CLC_ARG1_TYPE16 x, __CLC_ARG2_TYPE16 y) {
+  return (__CLC_RET_TYPE16)(__CLC_IMPL_FUNCTION(x.s0, y.s0),
+                            __CLC_IMPL_FUNCTION(x.s1, y.s1),
+                            __CLC_IMPL_FUNCTION(x.s2, y.s2),
+                            __CLC_IMPL_FUNCTION(x.s3, y.s3),
+                            __CLC_IMPL_FUNCTION(x.s4, y.s4),
+                            __CLC_IMPL_FUNCTION(x.s5, y.s5),
+                            __CLC_IMPL_FUNCTION(x.s6, y.s6),
+                            __CLC_IMPL_FUNCTION(x.s7, y.s7),
+                            __CLC_IMPL_FUNCTION(x.s8, y.s8),
+                            __CLC_IMPL_FUNCTION(x.s9, y.s9),
+                            __CLC_IMPL_FUNCTION(x.sa, y.sa),
+                            __CLC_IMPL_FUNCTION(x.sb, y.sb),
+                            __CLC_IMPL_FUNCTION(x.sc, y.sc),
+                            __CLC_IMPL_FUNCTION(x.sd, y.sd),
+                            __CLC_IMPL_FUNCTION(x.se, y.se),
+                            __CLC_IMPL_FUNCTION(x.sf, y.sf));
 }
 #undef __CLC_RET_TYPE16
 #undef __CLC_ARG1_TYPE16
diff --git a/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc b/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc
index 262b6b21cc648..2c32754044db9 100644
--- a/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_def_with_int_second_arg.inc
@@ -8,10 +8,11 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x, __CLC_INTN y) {
-  return __IMPL_FUNCTION(FUNCTION)(x, y);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE x,
+                                                    __CLC_INTN y) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(x, y);
 }
diff --git a/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc b/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc
index 103f775ab87e5..7a6283d7190da 100644
--- a/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc
+++ b/libclc/clc/include/clc/shared/binary_def_with_scalar_second_arg.inc
@@ -8,18 +8,18 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
-                                              __CLC_GENTYPE b) {
-  return __IMPL_FUNCTION(FUNCTION)(a, b);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
+                                                    __CLC_GENTYPE b) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a, b);
 }
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a,
-                                              __CLC_SCALAR_GENTYPE b) {
-  return __IMPL_FUNCTION(FUNCTION)(a, (__CLC_GENTYPE)b);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
+                                                    __CLC_SCALAR_GENTYPE b) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a, (__CLC_GENTYPE)b);
 }
 #endif
diff --git a/libclc/clc/include/clc/shared/clc_clamp.inc b/libclc/clc/include/clc/shared/clc_clamp.inc
index c1280a6f699ac..343fa9d24e749 100644
--- a/libclc/clc/include/clc/shared/clc_clamp.inc
+++ b/libclc/clc/include/clc/shared/clc_clamp.inc
@@ -6,12 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_clamp(__CLC_GENTYPE x,
-                                                  __CLC_GENTYPE y,
-                                                  __CLC_GENTYPE z);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE __clc_clamp(__CLC_GENTYPE x,
+                                                             __CLC_GENTYPE y,
+                                                             __CLC_GENTYPE z);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_clamp(__CLC_GENTYPE x,
-                                                  __CLC_SCALAR_GENTYPE y,
-                                                  __CLC_SCALAR_GENTYPE z);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__clc_clamp(__CLC_GENTYPE x, __CLC_SCALAR_GENTYPE y, __CLC_SCALAR_GENTYPE z);
 #endif
diff --git a/libclc/clc/include/clc/shared/clc_max.inc b/libclc/clc/include/clc/shared/clc_max.inc
index ca99c1264ad62..e2a6de69701f7 100644
--- a/libclc/clc/include/clc/shared/clc_max.inc
+++ b/libclc/clc/include/clc/shared/clc_max.inc
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_max(__CLC_GENTYPE a,
-                                                __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE __clc_max(__CLC_GENTYPE a,
+                                                           __CLC_GENTYPE b);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_max(__CLC_GENTYPE a,
-                                                __CLC_SCALAR_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__clc_max(__CLC_GENTYPE a, __CLC_SCALAR_GENTYPE b);
 #endif
diff --git a/libclc/clc/include/clc/shared/clc_min.inc b/libclc/clc/include/clc/shared/clc_min.inc
index c7cfe23ec3cd6..86b5edee585a1 100644
--- a/libclc/clc/include/clc/shared/clc_min.inc
+++ b/libclc/clc/include/clc/shared/clc_min.inc
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_min(__CLC_GENTYPE a,
-                                                __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE __clc_min(__CLC_GENTYPE a,
+                                                           __CLC_GENTYPE b);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __clc_min(__CLC_GENTYPE a,
-                                                __CLC_SCALAR_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__clc_min(__CLC_GENTYPE a, __CLC_SCALAR_GENTYPE b);
 #endif
diff --git a/libclc/clc/include/clc/shared/clc_vload.inc b/libclc/clc/include/clc/shared/clc_vload.inc
index 8f3b00ec04454..70bca250e4135 100644
--- a/libclc/clc/include/clc/shared/clc_vload.inc
+++ b/libclc/clc/include/clc/shared/clc_vload.inc
@@ -6,29 +6,29 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define CLC_VLOAD_NAME __CLC_XCONCAT(__clc_vload, __CLC_VECSIZE)
-#define CLC_VLOAD_HALF_NAME __CLC_XCONCAT(__clc_vload_half, __CLC_VECSIZE)
-#define CLC_VLOADA_HALF_NAME __CLC_XCONCAT(__clc_vloada_half, __CLC_VECSIZE)
+#define __CLC_VLOAD_NAME __CLC_XCONCAT(__clc_vload, __CLC_VECSIZE)
+#define __CLC_VLOAD_HALF_NAME __CLC_XCONCAT(__clc_vload_half, __CLC_VECSIZE)
+#define __CLC_VLOADA_HALF_NAME __CLC_XCONCAT(__clc_vloada_half, __CLC_VECSIZE)
 
 #ifndef __CLC_SCALAR
 
-#define CLC_VLOAD_TY __CLC_XCONCAT(less_aligned_, __CLC_GENTYPE)
+#define __CLC_VLOAD_TY __CLC_XCONCAT(less_aligned_, __CLC_GENTYPE)
 
-#define CLC_VLOAD_DECL(ADDRSPACE)                                              \
-  _CLC_OVERLOAD _CLC_DECL CLC_VLOAD_TY CLC_VLOAD_NAME(                         \
+#define __CLC_VLOAD_DECL(ADDRSPACE)                                            \
+  _CLC_OVERLOAD _CLC_DECL __CLC_VLOAD_TY __CLC_VLOAD_NAME(                     \
       size_t offset, const ADDRSPACE __CLC_SCALAR_GENTYPE *x);
 
-CLC_VLOAD_DECL(__private)
-CLC_VLOAD_DECL(__local)
-CLC_VLOAD_DECL(__constant)
-CLC_VLOAD_DECL(__global)
+__CLC_VLOAD_DECL(__private)
+__CLC_VLOAD_DECL(__local)
+__CLC_VLOAD_DECL(__constant)
+__CLC_VLOAD_DECL(__global)
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-CLC_VLOAD_DECL(__generic)
+__CLC_VLOAD_DECL(__generic)
 #endif
 
-#undef CLC_VLOAD_DECL
-#undef CLC_VLOAD_TY
+#undef __CLC_VLOAD_DECL
+#undef __CLC_VLOAD_TY
 
 #endif // __CLC_SCALAR
 
@@ -38,27 +38,27 @@ CLC_VLOAD_DECL(__generic)
 #ifdef __CLC_FPSIZE
 #if __CLC_FPSIZE == 32
 
-#define CLC_VLOAD_HALF_DECL(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE CLC_VLOAD_HALF_NAME(                   \
+#define __CLC_VLOAD_HALF_DECL(ADDRSPACE)                                       \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_VLOAD_HALF_NAME(                 \
       size_t offset, const ADDRSPACE half *mem);                               \
                                                                                \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE CLC_VLOADA_HALF_NAME(                  \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_VLOADA_HALF_NAME(                \
       size_t offset, const ADDRSPACE half *mem);
 
-CLC_VLOAD_HALF_DECL(__private)
-CLC_VLOAD_HALF_DECL(__local)
-CLC_VLOAD_HALF_DECL(__constant)
-CLC_VLOAD_HALF_DECL(__global)
+__CLC_VLOAD_HALF_DECL(__private)
+__CLC_VLOAD_HALF_DECL(__local)
+__CLC_VLOAD_HALF_DECL(__constant)
+__CLC_VLOAD_HALF_DECL(__global)
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-CLC_VLOAD_HALF_DECL(__generic)
+__CLC_VLOAD_HALF_DECL(__generic)
 #endif
 
-#undef CLC_VLOAD_HALF_DECL
+#undef __CLC_VLOAD_HALF_DECL
 
 #endif
 #endif
 
-#undef CLC_VLOAD_NAME
-#undef CLC_VLOAD_HALF_NAME
-#undef CLC_VLOADA_HALF_NAME
+#undef __CLC_VLOAD_NAME
+#undef __CLC_VLOAD_HALF_NAME
+#undef __CLC_VLOADA_HALF_NAME
diff --git a/libclc/clc/include/clc/shared/clc_vstore.inc b/libclc/clc/include/clc/shared/clc_vstore.inc
index 38d54b2f1b67f..cd3377e077af6 100644
--- a/libclc/clc/include/clc/shared/clc_vstore.inc
+++ b/libclc/clc/include/clc/shared/clc_vstore.inc
@@ -6,28 +6,28 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define CLC_VSTORE_TY __CLC_XCONCAT(less_aligned_, __CLC_GENTYPE)
-#define CLC_VSTORE_NAME __CLC_XCONCAT(__clc_vstore, __CLC_VECSIZE)
-#define CLC_VSTORE_HALF_NAME(x)                                                \
+#define __CLC_VSTORE_TY __CLC_XCONCAT(less_aligned_, __CLC_GENTYPE)
+#define __CLC_VSTORE_NAME __CLC_XCONCAT(__clc_vstore, __CLC_VECSIZE)
+#define __CLC_VSTORE_HALF_NAME(x)                                              \
   __CLC_XCONCAT(__CLC_XCONCAT(__clc_vstore_half, __CLC_VECSIZE), x)
-#define CLC_VSTOREA_HALF_NAME(x)                                               \
+#define __CLC_VSTOREA_HALF_NAME(x)                                             \
   __CLC_XCONCAT(__CLC_XCONCAT(__clc_vstorea_half, __CLC_VECSIZE), x)
 
 #ifndef __CLC_SCALAR
 
-#define CLC_VSTORE_DECL(ADDRSPACE)                                             \
-  _CLC_OVERLOAD _CLC_DECL void CLC_VSTORE_NAME(                                \
-      CLC_VSTORE_TY data, size_t offset, ADDRSPACE __CLC_SCALAR_GENTYPE *p);
+#define __CLC_VSTORE_DECL(ADDRSPACE)                                           \
+  _CLC_OVERLOAD _CLC_DECL void __CLC_VSTORE_NAME(                              \
+      __CLC_VSTORE_TY data, size_t offset, ADDRSPACE __CLC_SCALAR_GENTYPE *p);
 
-CLC_VSTORE_DECL(__private)
-CLC_VSTORE_DECL(__local)
-CLC_VSTORE_DECL(__global)
+__CLC_VSTORE_DECL(__private)
+__CLC_VSTORE_DECL(__local)
+__CLC_VSTORE_DECL(__global)
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-CLC_VSTORE_DECL(__generic)
+__CLC_VSTORE_DECL(__generic)
 #endif
 
-#undef CLC_VSTORE_DECL
+#undef __CLC_VSTORE_DECL
 
 #endif // __CLC_SCALAR
 
@@ -36,35 +36,35 @@ CLC_VSTORE_DECL(__generic)
 #ifdef __CLC_FPSIZE
 #if __CLC_FPSIZE == 32 || __CLC_FPSIZE == 64
 
-#define CLC_VSTORE_HALF_DECL(ADDRSPACE, SUFFIX)                                \
-  _CLC_OVERLOAD _CLC_DECL void CLC_VSTORE_HALF_NAME(SUFFIX)(                   \
-      CLC_VSTORE_TY data, size_t offset, ADDRSPACE half *p);                   \
+#define __CLC_VSTORE_HALF_DECL(ADDRSPACE, SUFFIX)                              \
+  _CLC_OVERLOAD _CLC_DECL void __CLC_VSTORE_HALF_NAME(SUFFIX)(                 \
+      __CLC_VSTORE_TY data, size_t offset, ADDRSPACE half *p);                 \
                                                                                \
-  _CLC_OVERLOAD _CLC_DECL void CLC_VSTOREA_HALF_NAME(SUFFIX)(                  \
-      CLC_VSTORE_TY data, size_t offset, ADDRSPACE half *p);
+  _CLC_OVERLOAD _CLC_DECL void __CLC_VSTOREA_HALF_NAME(SUFFIX)(                \
+      __CLC_VSTORE_TY data, size_t offset, ADDRSPACE half *p);
 
-#define CLC_VSTORE_HALF_DECL_ALL_MODES(ADDRSPACE)                              \
-  CLC_VSTORE_HALF_DECL(ADDRSPACE, )                                            \
-  CLC_VSTORE_HALF_DECL(ADDRSPACE, _rtz)                                        \
-  CLC_VSTORE_HALF_DECL(ADDRSPACE, _rtn)                                        \
-  CLC_VSTORE_HALF_DECL(ADDRSPACE, _rtp)                                        \
-  CLC_VSTORE_HALF_DECL(ADDRSPACE, _rte)
+#define __CLC_VSTORE_HALF_DECL_ALL_MODES(ADDRSPACE)                            \
+  __CLC_VSTORE_HALF_DECL(ADDRSPACE, )                                          \
+  __CLC_VSTORE_HALF_DECL(ADDRSPACE, _rtz)                                      \
+  __CLC_VSTORE_HALF_DECL(ADDRSPACE, _rtn)                                      \
+  __CLC_VSTORE_HALF_DECL(ADDRSPACE, _rtp)                                      \
+  __CLC_VSTORE_HALF_DECL(ADDRSPACE, _rte)
 
-CLC_VSTORE_HALF_DECL_ALL_MODES(__private)
-CLC_VSTORE_HALF_DECL_ALL_MODES(__local)
-CLC_VSTORE_HALF_DECL_ALL_MODES(__global)
+__CLC_VSTORE_HALF_DECL_ALL_MODES(__private)
+__CLC_VSTORE_HALF_DECL_ALL_MODES(__local)
+__CLC_VSTORE_HALF_DECL_ALL_MODES(__global)
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-CLC_VSTORE_HALF_DECL_ALL_MODES(__generic)
+__CLC_VSTORE_HALF_DECL_ALL_MODES(__generic)
 #endif
 
-#undef CLC_VSTORE_HALF_DECL
-#undef CLC_VSTORE_HALF_DECL_ALL_MODES
+#undef __CLC_VSTORE_HALF_DECL
+#undef __CLC_VSTORE_HALF_DECL_ALL_MODES
 
 #endif
 #endif
 
-#undef CLC_VSTORE_TY
-#undef CLC_VSTORE_NAME
-#undef CLC_VSTORE_HALF_NAME
-#undef CLC_VSTOREA_HALF_NAME
+#undef __CLC_VSTORE_TY
+#undef __CLC_VSTORE_NAME
+#undef __CLC_VSTORE_HALF_NAME
+#undef __CLC_VSTOREA_HALF_NAME
diff --git a/libclc/clc/include/clc/shared/ternary_decl.inc b/libclc/clc/include/clc/shared/ternary_decl.inc
index a76db1eefb6a4..21acb0042518a 100644
--- a/libclc/clc/include/clc/shared/ternary_decl.inc
+++ b/libclc/clc/include/clc/shared/ternary_decl.inc
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b,
-                                               __CLC_GENTYPE c);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b, __CLC_GENTYPE c);
diff --git a/libclc/clc/include/clc/shared/ternary_def.inc b/libclc/clc/include/clc/shared/ternary_def.inc
index 92134e8cb6f27..566cfb4ca58c9 100644
--- a/libclc/clc/include/clc/shared/ternary_def.inc
+++ b/libclc/clc/include/clc/shared/ternary_def.inc
@@ -8,11 +8,12 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b,
-                                              __CLC_GENTYPE c) {
-  return __IMPL_FUNCTION(FUNCTION)(a, b, c);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
+                                                    __CLC_GENTYPE b,
+                                                    __CLC_GENTYPE c) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a, b, c);
 }
diff --git a/libclc/clc/include/clc/shared/ternary_def_scalarize.inc b/libclc/clc/include/clc/shared/ternary_def_scalarize.inc
index c216fa4db4cac..7bdaee85a612b 100644
--- a/libclc/clc/include/clc/shared/ternary_def_scalarize.inc
+++ b/libclc/clc/include/clc/shared/ternary_def_scalarize.inc
@@ -14,8 +14,8 @@
 #define __CLC_MIN_VECSIZE 2
 #endif
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION FUNCTION
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION __CLC_FUNCTION
 #endif
 
 #ifndef __CLC_DEF_SPEC
@@ -39,10 +39,10 @@
 #endif
 
 #if __CLC_MIN_VECSIZE == 1
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x,
-                                                     __CLC_ARG2_TYPE y,
-                                                     __CLC_ARG3_TYPE z) {
-  return __IMPL_FUNCTION(x, y, z);
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE __CLC_FUNCTION(__CLC_ARG1_TYPE x,
+                                                           __CLC_ARG2_TYPE y,
+                                                           __CLC_ARG3_TYPE z) {
+  return __CLC_IMPL_FUNCTION(x, y, z);
 }
 #endif // __CLC_MIN_VECSIZE == 1
 
@@ -51,11 +51,10 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x,
 #define __CLC_ARG1_TYPE2 __CLC_XCONCAT(__CLC_ARG1_TYPE, 2)
 #define __CLC_ARG2_TYPE2 __CLC_XCONCAT(__CLC_ARG2_TYPE, 2)
 #define __CLC_ARG3_TYPE2 __CLC_XCONCAT(__CLC_ARG3_TYPE, 2)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x,
-                                                      __CLC_ARG2_TYPE2 y,
-                                                      __CLC_ARG3_TYPE2 z) {
-  return (__CLC_RET_TYPE2)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
-                           __IMPL_FUNCTION(x.s1, y.s1, z.s1));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2
+__CLC_FUNCTION(__CLC_ARG1_TYPE2 x, __CLC_ARG2_TYPE2 y, __CLC_ARG3_TYPE2 z) {
+  return (__CLC_RET_TYPE2)(__CLC_IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __CLC_IMPL_FUNCTION(x.s1, y.s1, z.s1));
 }
 #undef __CLC_RET_TYPE2
 #undef __CLC_ARG1_TYPE2
@@ -71,12 +70,11 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x,
 #define __CLC_ARG1_TYPE3 __CLC_XCONCAT(__CLC_ARG1_TYPE, 3)
 #define __CLC_ARG2_TYPE3 __CLC_XCONCAT(__CLC_ARG2_TYPE, 3)
 #define __CLC_ARG3_TYPE3 __CLC_XCONCAT(__CLC_ARG3_TYPE, 3)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x,
-                                                      __CLC_ARG2_TYPE3 y,
-                                                      __CLC_ARG3_TYPE3 z) {
-  return (__CLC_RET_TYPE3)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
-                           __IMPL_FUNCTION(x.s1, y.s1, z.s1),
-                           __IMPL_FUNCTION(x.s2, y.s2, z.s2));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3
+__CLC_FUNCTION(__CLC_ARG1_TYPE3 x, __CLC_ARG2_TYPE3 y, __CLC_ARG3_TYPE3 z) {
+  return (__CLC_RET_TYPE3)(__CLC_IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __CLC_IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                           __CLC_IMPL_FUNCTION(x.s2, y.s2, z.s2));
 }
 #undef __CLC_RET_TYPE3
 #undef __CLC_ARG1_TYPE3
@@ -87,13 +85,12 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x,
 #define __CLC_ARG1_TYPE4 __CLC_XCONCAT(__CLC_ARG1_TYPE, 4)
 #define __CLC_ARG2_TYPE4 __CLC_XCONCAT(__CLC_ARG2_TYPE, 4)
 #define __CLC_ARG3_TYPE4 __CLC_XCONCAT(__CLC_ARG3_TYPE, 4)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x,
-                                                      __CLC_ARG2_TYPE4 y,
-                                                      __CLC_ARG3_TYPE4 z) {
-  return (__CLC_RET_TYPE4)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
-                           __IMPL_FUNCTION(x.s1, y.s1, z.s1),
-                           __IMPL_FUNCTION(x.s2, y.s2, z.s2),
-                           __IMPL_FUNCTION(x.s3, y.s3, z.s3));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4
+__CLC_FUNCTION(__CLC_ARG1_TYPE4 x, __CLC_ARG2_TYPE4 y, __CLC_ARG3_TYPE4 z) {
+  return (__CLC_RET_TYPE4)(__CLC_IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __CLC_IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                           __CLC_IMPL_FUNCTION(x.s2, y.s2, z.s2),
+                           __CLC_IMPL_FUNCTION(x.s3, y.s3, z.s3));
 }
 #undef __CLC_RET_TYPE4
 #undef __CLC_ARG1_TYPE4
@@ -104,17 +101,16 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x,
 #define __CLC_ARG1_TYPE8 __CLC_XCONCAT(__CLC_ARG1_TYPE, 8)
 #define __CLC_ARG2_TYPE8 __CLC_XCONCAT(__CLC_ARG2_TYPE, 8)
 #define __CLC_ARG3_TYPE8 __CLC_XCONCAT(__CLC_ARG3_TYPE, 8)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x,
-                                                      __CLC_ARG2_TYPE8 y,
-                                                      __CLC_ARG3_TYPE8 z) {
-  return (__CLC_RET_TYPE8)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
-                           __IMPL_FUNCTION(x.s1, y.s1, z.s1),
-                           __IMPL_FUNCTION(x.s2, y.s2, z.s2),
-                           __IMPL_FUNCTION(x.s3, y.s3, z.s3),
-                           __IMPL_FUNCTION(x.s4, y.s4, z.s4),
-                           __IMPL_FUNCTION(x.s5, y.s5, z.s5),
-                           __IMPL_FUNCTION(x.s6, y.s6, z.s6),
-                           __IMPL_FUNCTION(x.s7, y.s7, z.s7));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8
+__CLC_FUNCTION(__CLC_ARG1_TYPE8 x, __CLC_ARG2_TYPE8 y, __CLC_ARG3_TYPE8 z) {
+  return (__CLC_RET_TYPE8)(__CLC_IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                           __CLC_IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                           __CLC_IMPL_FUNCTION(x.s2, y.s2, z.s2),
+                           __CLC_IMPL_FUNCTION(x.s3, y.s3, z.s3),
+                           __CLC_IMPL_FUNCTION(x.s4, y.s4, z.s4),
+                           __CLC_IMPL_FUNCTION(x.s5, y.s5, z.s5),
+                           __CLC_IMPL_FUNCTION(x.s6, y.s6, z.s6),
+                           __CLC_IMPL_FUNCTION(x.s7, y.s7, z.s7));
 }
 #undef __CLC_RET_TYPE8
 #undef __CLC_ARG1_TYPE8
@@ -125,25 +121,24 @@ _CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x,
 #define __CLC_ARG1_TYPE16 __CLC_XCONCAT(__CLC_ARG1_TYPE, 16)
 #define __CLC_ARG2_TYPE16 __CLC_XCONCAT(__CLC_ARG2_TYPE, 16)
 #define __CLC_ARG3_TYPE16 __CLC_XCONCAT(__CLC_ARG3_TYPE, 16)
-_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE16 FUNCTION(__CLC_ARG1_TYPE16 x,
-                                                       __CLC_ARG2_TYPE16 y,
-                                                       __CLC_ARG3_TYPE16 z) {
-  return (__CLC_RET_TYPE16)(__IMPL_FUNCTION(x.s0, y.s0, z.s0),
-                            __IMPL_FUNCTION(x.s1, y.s1, z.s1),
-                            __IMPL_FUNCTION(x.s2, y.s2, z.s2),
-                            __IMPL_FUNCTION(x.s3, y.s3, z.s3),
-                            __IMPL_FUNCTION(x.s4, y.s4, z.s4),
-                            __IMPL_FUNCTION(x.s5, y.s5, z.s5),
-                            __IMPL_FUNCTION(x.s6, y.s6, z.s6),
-                            __IMPL_FUNCTION(x.s7, y.s7, z.s7),
-                            __IMPL_FUNCTION(x.s8, y.s8, z.s8),
-                            __IMPL_FUNCTION(x.s9, y.s9, z.s9),
-                            __IMPL_FUNCTION(x.sa, y.sa, z.sa),
-                            __IMPL_FUNCTION(x.sb, y.sb, z.sb),
-                            __IMPL_FUNCTION(x.sc, y.sc, z.sc),
-                            __IMPL_FUNCTION(x.sd, y.sd, z.sd),
-                            __IMPL_FUNCTION(x.se, y.se, z.se),
-                            __IMPL_FUNCTION(x.sf, y.sf, z.sf));
+_CLC_OVERLOAD __CLC_DEF_SPEC __CLC_RET_TYPE16
+__CLC_FUNCTION(__CLC_ARG1_TYPE16 x, __CLC_ARG2_TYPE16 y, __CLC_ARG3_TYPE16 z) {
+  return (__CLC_RET_TYPE16)(__CLC_IMPL_FUNCTION(x.s0, y.s0, z.s0),
+                            __CLC_IMPL_FUNCTION(x.s1, y.s1, z.s1),
+                            __CLC_IMPL_FUNCTION(x.s2, y.s2, z.s2),
+                            __CLC_IMPL_FUNCTION(x.s3, y.s3, z.s3),
+                            __CLC_IMPL_FUNCTION(x.s4, y.s4, z.s4),
+                            __CLC_IMPL_FUNCTION(x.s5, y.s5, z.s5),
+                            __CLC_IMPL_FUNCTION(x.s6, y.s6, z.s6),
+                            __CLC_IMPL_FUNCTION(x.s7, y.s7, z.s7),
+                            __CLC_IMPL_FUNCTION(x.s8, y.s8, z.s8),
+                            __CLC_IMPL_FUNCTION(x.s9, y.s9, z.s9),
+                            __CLC_IMPL_FUNCTION(x.sa, y.sa, z.sa),
+                            __CLC_IMPL_FUNCTION(x.sb, y.sb, z.sb),
+                            __CLC_IMPL_FUNCTION(x.sc, y.sc, z.sc),
+                            __CLC_IMPL_FUNCTION(x.sd, y.sd, z.sd),
+                            __CLC_IMPL_FUNCTION(x.se, y.se, z.se),
+                            __CLC_IMPL_FUNCTION(x.sf, y.sf, z.sf));
 }
 #undef __CLC_RET_TYPE16
 #undef __CLC_ARG1_TYPE16
diff --git a/libclc/clc/include/clc/shared/unary_decl.inc b/libclc/clc/include/clc/shared/unary_decl.inc
index ce1a71b8284aa..19f1c8a876e53 100644
--- a/libclc/clc/include/clc/shared/unary_decl.inc
+++ b/libclc/clc/include/clc/shared/unary_decl.inc
@@ -6,4 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE
+__CLC_FUNCTION(__CLC_GENTYPE x);
diff --git a/libclc/clc/include/clc/shared/unary_def.inc b/libclc/clc/include/clc/shared/unary_def.inc
index e746a6bc89f32..b6826b023c70b 100644
--- a/libclc/clc/include/clc/shared/unary_def.inc
+++ b/libclc/clc/include/clc/shared/unary_def.inc
@@ -8,10 +8,10 @@
 
 #include <clc/utils.h>
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(__CLC_GENTYPE a) {
-  return __IMPL_FUNCTION(FUNCTION)(a);
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(__CLC_GENTYPE a) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a);
 }
diff --git a/libclc/clc/include/clc/shared/unary_def_scalarize.inc b/libclc/clc/include/clc/shared/unary_def_scalarize.inc
index 6c82babad2b52..e941918312da4 100644
--- a/libclc/clc/include/clc/shared/unary_def_scalarize.inc
+++ b/libclc/clc/include/clc/shared/unary_def_scalarize.inc
@@ -14,8 +14,8 @@
 #define __CLC_MIN_VECSIZE 2
 #endif
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION FUNCTION
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION __CLC_FUNCTION
 #endif
 
 #ifndef __CLC_RET_TYPE
@@ -31,16 +31,17 @@
 #endif
 
 #if __CLC_MIN_VECSIZE == 1
-_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE FUNCTION(__CLC_ARG1_TYPE x) {
-  return __IMPL_FUNCTION(x);
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE __CLC_FUNCTION(__CLC_ARG1_TYPE x) {
+  return __CLC_IMPL_FUNCTION(x);
 }
 #endif // __CLC_MIN_VECSIZE == 1
 
 #if __CLC_MIN_VECSIZE <= 2
 #define __CLC_RET_TYPE2 __CLC_XCONCAT(__CLC_RET_TYPE, 2)
 #define __CLC_ARG1_TYPE2 __CLC_XCONCAT(__CLC_ARG1_TYPE, 2)
-_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x) {
-  return (__CLC_RET_TYPE2)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1));
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE2 __CLC_FUNCTION(__CLC_ARG1_TYPE2 x) {
+  return (__CLC_RET_TYPE2)(__CLC_IMPL_FUNCTION(x.s0),
+                           __CLC_IMPL_FUNCTION(x.s1));
 }
 #undef __CLC_RET_TYPE2
 #undef __CLC_ARG1_TYPE2
@@ -52,44 +53,47 @@ _CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE2 FUNCTION(__CLC_ARG1_TYPE2 x) {
 
 #define __CLC_RET_TYPE3 __CLC_XCONCAT(__CLC_RET_TYPE, 3)
 #define __CLC_ARG1_TYPE3 __CLC_XCONCAT(__CLC_ARG1_TYPE, 3)
-_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE3 FUNCTION(__CLC_ARG1_TYPE3 x) {
-  return (__CLC_RET_TYPE3)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
-                           __IMPL_FUNCTION(x.s2));
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE3 __CLC_FUNCTION(__CLC_ARG1_TYPE3 x) {
+  return (__CLC_RET_TYPE3)(__CLC_IMPL_FUNCTION(x.s0), __CLC_IMPL_FUNCTION(x.s1),
+                           __CLC_IMPL_FUNCTION(x.s2));
 }
 #undef __CLC_RET_TYPE3
 #undef __CLC_ARG1_TYPE3
 
 #define __CLC_RET_TYPE4 __CLC_XCONCAT(__CLC_RET_TYPE, 4)
 #define __CLC_ARG1_TYPE4 __CLC_XCONCAT(__CLC_ARG1_TYPE, 4)
-_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE4 FUNCTION(__CLC_ARG1_TYPE4 x) {
-  return (__CLC_RET_TYPE4)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
-                           __IMPL_FUNCTION(x.s2), __IMPL_FUNCTION(x.s3));
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE4 __CLC_FUNCTION(__CLC_ARG1_TYPE4 x) {
+  return (__CLC_RET_TYPE4)(__CLC_IMPL_FUNCTION(x.s0), __CLC_IMPL_FUNCTION(x.s1),
+                           __CLC_IMPL_FUNCTION(x.s2),
+                           __CLC_IMPL_FUNCTION(x.s3));
 }
 #undef __CLC_RET_TYPE4
 #undef __CLC_ARG1_TYPE4
 
 #define __CLC_RET_TYPE8 __CLC_XCONCAT(__CLC_RET_TYPE, 8)
 #define __CLC_ARG1_TYPE8 __CLC_XCONCAT(__CLC_ARG1_TYPE, 8)
-_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE8 FUNCTION(__CLC_ARG1_TYPE8 x) {
-  return (__CLC_RET_TYPE8)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
-                           __IMPL_FUNCTION(x.s2), __IMPL_FUNCTION(x.s3),
-                           __IMPL_FUNCTION(x.s4), __IMPL_FUNCTION(x.s5),
-                           __IMPL_FUNCTION(x.s6), __IMPL_FUNCTION(x.s7));
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE8 __CLC_FUNCTION(__CLC_ARG1_TYPE8 x) {
+  return (__CLC_RET_TYPE8)(__CLC_IMPL_FUNCTION(x.s0), __CLC_IMPL_FUNCTION(x.s1),
+                           __CLC_IMPL_FUNCTION(x.s2), __CLC_IMPL_FUNCTION(x.s3),
+                           __CLC_IMPL_FUNCTION(x.s4), __CLC_IMPL_FUNCTION(x.s5),
+                           __CLC_IMPL_FUNCTION(x.s6),
+                           __CLC_IMPL_FUNCTION(x.s7));
 }
 #undef __CLC_RET_TYPE8
 #undef __CLC_ARG1_TYPE8
 
 #define __CLC_RET_TYPE16 __CLC_XCONCAT(__CLC_RET_TYPE, 16)
 #define __CLC_ARG1_TYPE16 __CLC_XCONCAT(__CLC_ARG1_TYPE, 16)
-_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE16 FUNCTION(__CLC_ARG1_TYPE16 x) {
-  return (__CLC_RET_TYPE16)(__IMPL_FUNCTION(x.s0), __IMPL_FUNCTION(x.s1),
-                            __IMPL_FUNCTION(x.s2), __IMPL_FUNCTION(x.s3),
-                            __IMPL_FUNCTION(x.s4), __IMPL_FUNCTION(x.s5),
-                            __IMPL_FUNCTION(x.s6), __IMPL_FUNCTION(x.s7),
-                            __IMPL_FUNCTION(x.s8), __IMPL_FUNCTION(x.s9),
-                            __IMPL_FUNCTION(x.sa), __IMPL_FUNCTION(x.sb),
-                            __IMPL_FUNCTION(x.sc), __IMPL_FUNCTION(x.sd),
-                            __IMPL_FUNCTION(x.se), __IMPL_FUNCTION(x.sf));
+_CLC_OVERLOAD _CLC_DEF __CLC_RET_TYPE16 __CLC_FUNCTION(__CLC_ARG1_TYPE16 x) {
+  return (
+      __CLC_RET_TYPE16)(__CLC_IMPL_FUNCTION(x.s0), __CLC_IMPL_FUNCTION(x.s1),
+                        __CLC_IMPL_FUNCTION(x.s2), __CLC_IMPL_FUNCTION(x.s3),
+                        __CLC_IMPL_FUNCTION(x.s4), __CLC_IMPL_FUNCTION(x.s5),
+                        __CLC_IMPL_FUNCTION(x.s6), __CLC_IMPL_FUNCTION(x.s7),
+                        __CLC_IMPL_FUNCTION(x.s8), __CLC_IMPL_FUNCTION(x.s9),
+                        __CLC_IMPL_FUNCTION(x.sa), __CLC_IMPL_FUNCTION(x.sb),
+                        __CLC_IMPL_FUNCTION(x.sc), __CLC_IMPL_FUNCTION(x.sd),
+                        __CLC_IMPL_FUNCTION(x.se), __CLC_IMPL_FUNCTION(x.sf));
 }
 #undef __CLC_RET_TYPE16
 #undef __CLC_ARG1_TYPE16
diff --git a/libclc/clc/include/clc/workitem/clc_get_global_id.h b/libclc/clc/include/clc/workitem/clc_get_global_id.h
index da15ed8e53ca8..8a78c78bbc164 100644
--- a/libclc/clc/include/clc/workitem/clc_get_global_id.h
+++ b/libclc/clc/include/clc/workitem/clc_get_global_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL size_t __clc_get_global_id(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t __clc_get_global_id(uint dim);
 
 #endif // __CLC_WORKITEM_CLC_GET_GLOBAL_ID_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_global_offset.h b/libclc/clc/include/clc/workitem/clc_get_global_offset.h
index 639be48d0b958..bf6d4e5c7642d 100644
--- a/libclc/clc/include/clc/workitem/clc_get_global_offset.h
+++ b/libclc/clc/include/clc/workitem/clc_get_global_offset.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL size_t __clc_get_global_offset(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t __clc_get_global_offset(uint dim);
 
 #endif // __CLC_WORKITEM_CLC_GET_GLOBAL_OFFSET_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_global_size.h b/libclc/clc/include/clc/workitem/clc_get_global_size.h
index fce11f417bf9d..87b88d7d1b4b0 100644
--- a/libclc/clc/include/clc/workitem/clc_get_global_size.h
+++ b/libclc/clc/include/clc/workitem/clc_get_global_size.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL size_t __clc_get_global_size(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t __clc_get_global_size(uint dim);
 
 #endif // __CLC_WORKITEM_CLC_GET_GLOBAL_SIZE_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_group_id.h b/libclc/clc/include/clc/workitem/clc_get_group_id.h
index 3fd43cb77f893..cc1b0f13ad792 100644
--- a/libclc/clc/include/clc/workitem/clc_get_group_id.h
+++ b/libclc/clc/include/clc/workitem/clc_get_group_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL size_t __clc_get_group_id(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t __clc_get_group_id(uint dim);
 
 #endif // __CLC_WORKITEM_CLC_GET_GROUP_ID_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_local_id.h b/libclc/clc/include/clc/workitem/clc_get_local_id.h
index c446127a5f90c..a25e5ebaa5e45 100644
--- a/libclc/clc/include/clc/workitem/clc_get_local_id.h
+++ b/libclc/clc/include/clc/workitem/clc_get_local_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL size_t __clc_get_local_id(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t __clc_get_local_id(uint dim);
 
 #endif // __CLC_WORKITEM_CLC_GET_LOCAL_ID_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_local_linear_id.h b/libclc/clc/include/clc/workitem/clc_get_local_linear_id.h
index 34c47f2afcb30..9548644f05ca9 100644
--- a/libclc/clc/include/clc/workitem/clc_get_local_linear_id.h
+++ b/libclc/clc/include/clc/workitem/clc_get_local_linear_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL size_t __clc_get_local_linear_id();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t __clc_get_local_linear_id();
 
 #endif // __CLC_WORKITEM_CLC_GET_LOCAL_LINEAR_ID_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_local_size.h b/libclc/clc/include/clc/workitem/clc_get_local_size.h
index 7fe3c193e0ec1..1272227b65e1b 100644
--- a/libclc/clc/include/clc/workitem/clc_get_local_size.h
+++ b/libclc/clc/include/clc/workitem/clc_get_local_size.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL size_t __clc_get_local_size(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t __clc_get_local_size(uint dim);
 
 #endif // __CLC_WORKITEM_CLC_GET_LOCAL_SIZE_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_max_sub_group_size.h b/libclc/clc/include/clc/workitem/clc_get_max_sub_group_size.h
index 8b521fe32862d..c8ea2a1fead12 100644
--- a/libclc/clc/include/clc/workitem/clc_get_max_sub_group_size.h
+++ b/libclc/clc/include/clc/workitem/clc_get_max_sub_group_size.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_DEF _CLC_OVERLOAD uint __clc_get_max_sub_group_size();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint __clc_get_max_sub_group_size();
 
 #endif // __CLC_WORKITEM_CLC_GET_MAX_SUB_GROUP_SIZE_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_num_groups.h b/libclc/clc/include/clc/workitem/clc_get_num_groups.h
index f860944ad517f..6071fc3f141fb 100644
--- a/libclc/clc/include/clc/workitem/clc_get_num_groups.h
+++ b/libclc/clc/include/clc/workitem/clc_get_num_groups.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL size_t __clc_get_num_groups(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t __clc_get_num_groups(uint dim);
 
 #endif // __CLC_WORKITEM_CLC_GET_NUM_GROUPS_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_num_sub_groups.h b/libclc/clc/include/clc/workitem/clc_get_num_sub_groups.h
index 6965aef1bce30..f94e898268921 100644
--- a/libclc/clc/include/clc/workitem/clc_get_num_sub_groups.h
+++ b/libclc/clc/include/clc/workitem/clc_get_num_sub_groups.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_DEF _CLC_OVERLOAD uint __clc_get_num_sub_groups();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint __clc_get_num_sub_groups();
 
 #endif // __CLC_WORKITEM_CLC_GET_NUM_SUB_GROUPS_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_sub_group_id.h b/libclc/clc/include/clc/workitem/clc_get_sub_group_id.h
index ac3d7bd30e454..61b54ecde4766 100644
--- a/libclc/clc/include/clc/workitem/clc_get_sub_group_id.h
+++ b/libclc/clc/include/clc/workitem/clc_get_sub_group_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_DEF _CLC_OVERLOAD uint __clc_get_sub_group_id();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint __clc_get_sub_group_id();
 
 #endif // __CLC_WORKITEM_CLC_GET_SUB_GROUP_ID_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_sub_group_local_id.h b/libclc/clc/include/clc/workitem/clc_get_sub_group_local_id.h
index 06bb6f8b77a5c..a158153bde1e9 100644
--- a/libclc/clc/include/clc/workitem/clc_get_sub_group_local_id.h
+++ b/libclc/clc/include/clc/workitem/clc_get_sub_group_local_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_DEF _CLC_OVERLOAD uint __clc_get_sub_group_local_id();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint __clc_get_sub_group_local_id();
 
 #endif // __CLC_WORKITEM_CLC_GET_SUB_GROUP_LOCAL_ID_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_sub_group_size.h b/libclc/clc/include/clc/workitem/clc_get_sub_group_size.h
index a6e8e49470e4f..c4b1d928b20d2 100644
--- a/libclc/clc/include/clc/workitem/clc_get_sub_group_size.h
+++ b/libclc/clc/include/clc/workitem/clc_get_sub_group_size.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_DEF _CLC_OVERLOAD uint __clc_get_sub_group_size();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint __clc_get_sub_group_size();
 
 #endif // __CLC_WORKITEM_CLC_GET_SUB_GROUP_SIZE_H__
diff --git a/libclc/clc/include/clc/workitem/clc_get_work_dim.h b/libclc/clc/include/clc/workitem/clc_get_work_dim.h
index 7e2da4570c4f5..e93d3773b8ca9 100644
--- a/libclc/clc/include/clc/workitem/clc_get_work_dim.h
+++ b/libclc/clc/include/clc/workitem/clc_get_work_dim.h
@@ -11,6 +11,6 @@
 
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DECL uint __clc_get_work_dim();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint __clc_get_work_dim();
 
 #endif // __CLC_WORKITEM_CLC_GET_WORK_DIM_H__
diff --git a/libclc/clc/lib/amdgcn/SOURCES b/libclc/clc/lib/amdgcn/SOURCES
index 76c3266e3af7b..53bbe388f7dfc 100644
--- a/libclc/clc/lib/amdgcn/SOURCES
+++ b/libclc/clc/lib/amdgcn/SOURCES
@@ -5,4 +5,6 @@ workitem/clc_get_global_offset.cl
 workitem/clc_get_global_size.cl
 workitem/clc_get_group_id.cl
 workitem/clc_get_local_id.cl
+workitem/clc_get_local_size.cl
+workitem/clc_get_max_sub_group_size.cl
 workitem/clc_get_work_dim.cl
diff --git a/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl b/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl
index b2990647a00a7..90bd50ac15512 100644
--- a/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl
+++ b/libclc/clc/lib/amdgcn/math/clc_ldexp_override.cl
@@ -10,25 +10,25 @@
 #include <clc/internal/clc.h>
 #include <clc/math/clc_ldexp.h>
 
-#define FUNCTION __clc_ldexp
+#define __CLC_FUNCTION __clc_ldexp
 #define __CLC_ARG2_TYPE int
 #define __CLC_MIN_VECSIZE 1
 
 #ifdef __HAS_LDEXPF__
 // This defines all the ldexp(floatN, intN) variants.
-#define __FLOAT_ONLY
-#define __IMPL_FUNCTION __builtin_amdgcn_ldexpf
+#define __CLC_FLOAT_ONLY
+#define __CLC_IMPL_FUNCTION __builtin_amdgcn_ldexpf
 #define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
-#undef __IMPL_FUNCTION
+#undef __CLC_IMPL_FUNCTION
 #endif
 
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 // This defines all the ldexp(doubleN, intN) variants.
-#define __DOUBLE_ONLY
-#define __IMPL_FUNCTION __builtin_amdgcn_ldexp
+#define __CLC_DOUBLE_ONLY
+#define __CLC_IMPL_FUNCTION __builtin_amdgcn_ldexp
 #define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
-#undef __IMPL_FUNCTION
+#undef __CLC_IMPL_FUNCTION
 #endif
diff --git a/libclc/clc/lib/amdgcn/workitem/clc_get_group_id.cl b/libclc/clc/lib/amdgcn/workitem/clc_get_group_id.cl
index aea927c3460b4..4dab7905ba301 100644
--- a/libclc/clc/lib/amdgcn/workitem/clc_get_group_id.cl
+++ b/libclc/clc/lib/amdgcn/workitem/clc_get_group_id.cl
@@ -17,6 +17,6 @@ _CLC_DEF _CLC_OVERLOAD size_t __clc_get_group_id(uint dim) {
   case 2:
     return __builtin_amdgcn_workgroup_id_z();
   default:
-    return 1;
+    return 0;
   }
 }
diff --git a/libclc/clc/lib/amdgcn/workitem/clc_get_local_id.cl b/libclc/clc/lib/amdgcn/workitem/clc_get_local_id.cl
index b7b7a43e735d3..8d1d16d4762fd 100644
--- a/libclc/clc/lib/amdgcn/workitem/clc_get_local_id.cl
+++ b/libclc/clc/lib/amdgcn/workitem/clc_get_local_id.cl
@@ -17,6 +17,6 @@ _CLC_DEF _CLC_OVERLOAD size_t __clc_get_local_id(uint dim) {
   case 2:
     return __builtin_amdgcn_workitem_id_z();
   default:
-    return 1;
+    return 0;
   }
 }
diff --git a/libclc/clc/lib/amdgcn/workitem/clc_get_local_size.cl b/libclc/clc/lib/amdgcn/workitem/clc_get_local_size.cl
new file mode 100644
index 0000000000000..1e749404168d8
--- /dev/null
+++ b/libclc/clc/lib/amdgcn/workitem/clc_get_local_size.cl
@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/workitem/clc_get_local_size.h>
+
+_CLC_OVERLOAD _CLC_DEF size_t __clc_get_local_size(uint dim) {
+  switch (dim) {
+  case 0:
+    return __builtin_amdgcn_workgroup_size_x();
+  case 1:
+    return __builtin_amdgcn_workgroup_size_y();
+  case 2:
+    return __builtin_amdgcn_workgroup_size_z();
+  default:
+    return 1;
+  }
+}
diff --git a/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl b/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl
new file mode 100644
index 0000000000000..cc56f8d9c325d
--- /dev/null
+++ b/libclc/clc/lib/amdgcn/workitem/clc_get_max_sub_group_size.cl
@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/workitem/clc_get_max_sub_group_size.h>
+
+_CLC_OVERLOAD _CLC_DEF uint __clc_get_max_sub_group_size() {
+  return __builtin_amdgcn_wavefrontsize();
+}
diff --git a/libclc/clc/lib/amdgpu/math/clc_half_exp.cl b/libclc/clc/lib/amdgpu/math/clc_half_exp.cl
index 5fa9d0b9a9dea..c332708615e19 100644
--- a/libclc/clc/lib/amdgpu/math/clc_half_exp.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_half_exp.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_native_exp.h>
 
 #define __CLC_FUNC exp
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_native_unary.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_half_exp10.cl b/libclc/clc/lib/amdgpu/math/clc_half_exp10.cl
index 5c119eb5a535c..5560f7ac97965 100644
--- a/libclc/clc/lib/amdgpu/math/clc_half_exp10.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_half_exp10.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_native_exp10.h>
 
 #define __CLC_FUNC exp10
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_native_unary.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_half_exp2.cl b/libclc/clc/lib/amdgpu/math/clc_half_exp2.cl
index 9750d5008986b..3e38cbee3285c 100644
--- a/libclc/clc/lib/amdgpu/math/clc_half_exp2.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_half_exp2.cl
@@ -10,6 +10,6 @@
 #include <clc/math/clc_native_exp2.h>
 
 #define __CLC_FUNC exp2
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_native_unary.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_half_log.cl b/libclc/clc/lib/amdgpu/math/clc_half_log.cl
index ea19bd5a4c7a7..0b731c202e20b 100644
--- a/libclc/clc/lib/amdgpu/math/clc_half_log.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_half_log.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_native_log.h>
 
 #define __CLC_FUNC log
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_native_unary.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_half_log10.cl b/libclc/clc/lib/amdgpu/math/clc_half_log10.cl
index af3a2695396db..1fd8ba520411e 100644
--- a/libclc/clc/lib/amdgpu/math/clc_half_log10.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_half_log10.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_native_log10.h>
 
 #define __CLC_FUNC log10
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_native_unary.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_half_log2.cl b/libclc/clc/lib/amdgpu/math/clc_half_log2.cl
index 81795f012c77d..e4d27421f8fe0 100644
--- a/libclc/clc/lib/amdgpu/math/clc_half_log2.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_half_log2.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_native_log2.h>
 
 #define __CLC_FUNC log2
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_native_unary.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_half_recip.cl b/libclc/clc/lib/amdgpu/math/clc_half_recip.cl
index edbec07e44841..0a1c170375314 100644
--- a/libclc/clc/lib/amdgpu/math/clc_half_recip.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_half_recip.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_native_recip.h>
 
 #define __CLC_FUNC recip
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_native_unary.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_half_rsqrt.cl b/libclc/clc/lib/amdgpu/math/clc_half_rsqrt.cl
index c0a5489ec8fc0..a17ac508b5ad3 100644
--- a/libclc/clc/lib/amdgpu/math/clc_half_rsqrt.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_half_rsqrt.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_native_rsqrt.h>
 
 #define __CLC_FUNC rsqrt
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_native_unary.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_half_sqrt.cl b/libclc/clc/lib/amdgpu/math/clc_half_sqrt.cl
index 4dc6fa31f21a7..142a878818535 100644
--- a/libclc/clc/lib/amdgpu/math/clc_half_sqrt.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_half_sqrt.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_native_sqrt.h>
 
 #define __CLC_FUNC sqrt
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_native_unary.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_native_exp.cl b/libclc/clc/lib/amdgpu/math/clc_native_exp.cl
index 591ecb0ac00b5..946b9beed5be4 100644
--- a/libclc/clc/lib/amdgpu/math/clc_native_exp.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_native_exp.cl
@@ -11,5 +11,5 @@
 #include <clc/math/clc_native_exp2.h>
 
 #define __CLC_BODY <clc_native_exp.inc>
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl b/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl
index 63f28921dc177..5959ea2786a51 100644
--- a/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_native_exp2.cl
@@ -9,9 +9,9 @@
 #include <clc/clcmacro.h>
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_MIN_VECSIZE 1
-#define FUNCTION __clc_native_exp2
-#define __IMPL_FUNCTION __builtin_amdgcn_exp2f
+#define __CLC_FUNCTION __clc_native_exp2
+#define __CLC_IMPL_FUNCTION __builtin_amdgcn_exp2f
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_native_log10.cl b/libclc/clc/lib/amdgpu/math/clc_native_log10.cl
index 0668a635d24d7..c356b22b6b6e0 100644
--- a/libclc/clc/lib/amdgpu/math/clc_native_log10.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_native_log10.cl
@@ -11,5 +11,5 @@
 #include <clc/math/clc_native_log2.h>
 
 #define __CLC_BODY <clc_native_log10.inc>
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl b/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl
index b7cb635a2ae8e..d2790f3a82605 100644
--- a/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl
+++ b/libclc/clc/lib/amdgpu/math/clc_sqrt_fp64.cl
@@ -43,8 +43,8 @@ _CLC_OVERLOAD _CLC_DEF double __clc_sqrt(double x) {
   return (x == __builtin_inf() || (x == 0.0)) ? v01 : v23;
 }
 
-#define __DOUBLE_ONLY
-#define FUNCTION __clc_sqrt
+#define __CLC_DOUBLE_ONLY
+#define __CLC_FUNCTION __clc_sqrt
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/clspv/math/clc_sw_fma.cl b/libclc/clc/lib/clspv/math/clc_sw_fma.cl
index 266269644721a..1dc9a0e6407ba 100644
--- a/libclc/clc/lib/clspv/math/clc_sw_fma.cl
+++ b/libclc/clc/lib/clspv/math/clc_sw_fma.cl
@@ -269,7 +269,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
                         ((uint)st_fma.mantissa.lo & 0x7fffff));
 }
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_sw_fma
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_sw_fma
 #define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_dec.cl b/libclc/clc/lib/generic/atomic/clc_atomic_dec.cl
index f35a9624fd013..7984dba5731ee 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_dec.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_dec.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_dec.h>
 
-#define FUNCTION __clc_atomic_dec
-#define __IMPL_FUNCTION __scoped_atomic_fetch_add
+#define __CLC_FUNCTION __clc_atomic_dec
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_add
 #define __CLC_INC_DEC
 
 #define __CLC_BODY <clc_atomic_def.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_def.inc b/libclc/clc/lib/generic/atomic/clc_atomic_def.inc
index 2c45f49f60848..c1a0731eb8439 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_def.inc
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_def.inc
@@ -31,37 +31,37 @@
 
 #ifdef __CLC_NO_VALUE_ARG
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(                        \
       volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder,                  \
       int MemoryScope) {                                                       \
-    return __CLC_AS_RETTYPE(__IMPL_FUNCTION(                                   \
+    return __CLC_AS_RETTYPE(__CLC_IMPL_FUNCTION(                               \
         (ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, MemoryOrder, MemoryScope));       \
   }
 #elif defined(__CLC_INC_DEC)
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(                        \
       volatile ADDRSPACE __CLC_GENTYPE *Ptr, int MemoryOrder,                  \
       int MemoryScope) {                                                       \
     return __CLC_AS_RETTYPE(                                                   \
-        __IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, (__CLC_GENTYPE)1, \
-                        MemoryOrder, MemoryScope));                            \
+        __CLC_IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr,               \
+                            (__CLC_GENTYPE)1, MemoryOrder, MemoryScope));      \
   }
 #elif defined(__CLC_RETURN_VOID)
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DECL void FUNCTION(volatile ADDRSPACE __CLC_GENTYPE *Ptr, \
-                                        __CLC_GENTYPE Value, int MemoryOrder,  \
-                                        int MemoryScope) {                     \
-    __IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, Value, MemoryOrder,   \
-                    MemoryScope);                                              \
+  _CLC_OVERLOAD _CLC_DECL void __CLC_FUNCTION(                                 \
+      volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value,              \
+      int MemoryOrder, int MemoryScope) {                                      \
+    __CLC_IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, Value,            \
+                        MemoryOrder, MemoryScope);                             \
   }
 #else
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(                        \
       volatile ADDRSPACE __CLC_GENTYPE *Ptr, __CLC_GENTYPE Value,              \
       int MemoryOrder, int MemoryScope) {                                      \
     return __CLC_AS_RETTYPE(                                                   \
-        __IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, Value,            \
-                        MemoryOrder, MemoryScope));                            \
+        __CLC_IMPL_FUNCTION((ADDRSPACE __CLC_PTR_CASTTYPE *)Ptr, Value,        \
+                            MemoryOrder, MemoryScope));                        \
   }
 #endif
 
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_exchange.cl b/libclc/clc/lib/generic/atomic/clc_atomic_exchange.cl
index 52fd11afed6a2..ee80256d3dbb6 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_exchange.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_exchange.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_exchange.h>
 
-#define FUNCTION __clc_atomic_exchange
-#define __IMPL_FUNCTION __scoped_atomic_exchange_n
+#define __CLC_FUNCTION __clc_atomic_exchange
+#define __CLC_IMPL_FUNCTION __scoped_atomic_exchange_n
 
 #define __CLC_BODY <clc_atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_add.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_add.cl
index 0dc44919627b3..06fa21344ec84 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_add.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_add.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_fetch_add.h>
 
-#define FUNCTION __clc_atomic_fetch_add
-#define __IMPL_FUNCTION __scoped_atomic_fetch_add
+#define __CLC_FUNCTION __clc_atomic_fetch_add
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_add
 
 #define __CLC_BODY <clc_atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_and.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_and.cl
index ec89738bc0f62..7b9750d8d4db7 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_and.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_and.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_fetch_and.h>
 
-#define FUNCTION __clc_atomic_fetch_and
-#define __IMPL_FUNCTION __scoped_atomic_fetch_and
+#define __CLC_FUNCTION __clc_atomic_fetch_and
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_and
 
 #define __CLC_BODY <clc_atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_max.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_max.cl
index 0acac711aa96d..6401d3e9439f0 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_max.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_max.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_fetch_max.h>
 
-#define FUNCTION __clc_atomic_fetch_max
-#define __IMPL_FUNCTION __scoped_atomic_fetch_max
+#define __CLC_FUNCTION __clc_atomic_fetch_max
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_max
 
 #define __CLC_BODY <clc_atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_min.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_min.cl
index 7a098588ec005..51e3904a5c6e3 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_min.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_min.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_fetch_min.h>
 
-#define FUNCTION __clc_atomic_fetch_min
-#define __IMPL_FUNCTION __scoped_atomic_fetch_min
+#define __CLC_FUNCTION __clc_atomic_fetch_min
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_min
 
 #define __CLC_BODY <clc_atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_or.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_or.cl
index e0f48fa408350..e92e351db3f42 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_or.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_or.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_fetch_or.h>
 
-#define FUNCTION __clc_atomic_fetch_or
-#define __IMPL_FUNCTION __scoped_atomic_fetch_or
+#define __CLC_FUNCTION __clc_atomic_fetch_or
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_or
 
 #define __CLC_BODY <clc_atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_sub.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_sub.cl
index a4c2c1da1555c..fbbf7a4def566 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_sub.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_sub.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_fetch_sub.h>
 
-#define FUNCTION __clc_atomic_fetch_sub
-#define __IMPL_FUNCTION __scoped_atomic_fetch_sub
+#define __CLC_FUNCTION __clc_atomic_fetch_sub
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_sub
 
 #define __CLC_BODY <clc_atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_xor.cl b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_xor.cl
index 4424a298178fd..d041fd164d386 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_fetch_xor.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_fetch_xor.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_fetch_xor.h>
 
-#define FUNCTION __clc_atomic_fetch_xor
-#define __IMPL_FUNCTION __scoped_atomic_fetch_xor
+#define __CLC_FUNCTION __clc_atomic_fetch_xor
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_xor
 
 #define __CLC_BODY <clc_atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_inc.cl b/libclc/clc/lib/generic/atomic/clc_atomic_inc.cl
index 019aa8d9d6dd8..7171f84c30ce0 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_inc.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_inc.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_inc.h>
 
-#define FUNCTION __clc_atomic_inc
-#define __IMPL_FUNCTION __scoped_atomic_fetch_sub
+#define __CLC_FUNCTION __clc_atomic_inc
+#define __CLC_IMPL_FUNCTION __scoped_atomic_fetch_sub
 #define __CLC_INC_DEC
 
 #define __CLC_BODY <clc_atomic_def.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_load.cl b/libclc/clc/lib/generic/atomic/clc_atomic_load.cl
index 1f083073e43ff..f7fe2510569e4 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_load.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_load.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_load.h>
 
-#define FUNCTION __clc_atomic_load
-#define __IMPL_FUNCTION __scoped_atomic_load_n
+#define __CLC_FUNCTION __clc_atomic_load
+#define __CLC_IMPL_FUNCTION __scoped_atomic_load_n
 #define __CLC_NO_VALUE_ARG
 
 #define __CLC_BODY <clc_atomic_def.inc>
diff --git a/libclc/clc/lib/generic/atomic/clc_atomic_store.cl b/libclc/clc/lib/generic/atomic/clc_atomic_store.cl
index 8fd165b9a83b8..a93d21e8430ce 100644
--- a/libclc/clc/lib/generic/atomic/clc_atomic_store.cl
+++ b/libclc/clc/lib/generic/atomic/clc_atomic_store.cl
@@ -8,8 +8,8 @@
 
 #include <clc/atomic/clc_atomic_store.h>
 
-#define FUNCTION __clc_atomic_store
-#define __IMPL_FUNCTION __scoped_atomic_store_n
+#define __CLC_FUNCTION __clc_atomic_store
+#define __CLC_IMPL_FUNCTION __scoped_atomic_store_n
 #define __CLC_RETURN_VOID
 
 #define __CLC_BODY <clc_atomic_def.inc>
diff --git a/libclc/clc/lib/generic/common/clc_radians.cl b/libclc/clc/lib/generic/common/clc_radians.cl
index f8a1635ecb5cf..497263a15f516 100644
--- a/libclc/clc/lib/generic/common/clc_radians.cl
+++ b/libclc/clc/lib/generic/common/clc_radians.cl
@@ -9,27 +9,27 @@
 #include <clc/clcmacro.h>
 #include <clc/internal/clc.h>
 
-#define RADIANS_SINGLE_DEF(TYPE, LITERAL)                                      \
+#define __CLC_RADIANS_SINGLE_DEF(TYPE, LITERAL)                                \
   _CLC_OVERLOAD _CLC_DEF TYPE __clc_radians(TYPE radians) {                    \
     return (TYPE)LITERAL * radians;                                            \
   }
 
-#define RADIANS_DEF(TYPE, LITERAL)                                             \
-  RADIANS_SINGLE_DEF(TYPE, LITERAL)                                            \
-  RADIANS_SINGLE_DEF(TYPE##2, LITERAL)                                         \
-  RADIANS_SINGLE_DEF(TYPE##3, LITERAL)                                         \
-  RADIANS_SINGLE_DEF(TYPE##4, LITERAL)                                         \
-  RADIANS_SINGLE_DEF(TYPE##8, LITERAL)                                         \
-  RADIANS_SINGLE_DEF(TYPE##16, LITERAL)
+#define __CLC_RADIANS_DEF(TYPE, LITERAL)                                       \
+  __CLC_RADIANS_SINGLE_DEF(TYPE, LITERAL)                                      \
+  __CLC_RADIANS_SINGLE_DEF(TYPE##2, LITERAL)                                   \
+  __CLC_RADIANS_SINGLE_DEF(TYPE##3, LITERAL)                                   \
+  __CLC_RADIANS_SINGLE_DEF(TYPE##4, LITERAL)                                   \
+  __CLC_RADIANS_SINGLE_DEF(TYPE##8, LITERAL)                                   \
+  __CLC_RADIANS_SINGLE_DEF(TYPE##16, LITERAL)
 
 // pi/180 = ~0.01745329251994329577 or 0x1.1df46a2529d39p-6 or 0x1.1df46ap-6F
-RADIANS_DEF(float, 0x1.1df46ap-6F)
+__CLC_RADIANS_DEF(float, 0x1.1df46ap-6F)
 
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
 // pi/180 = ~0.01745329251994329577 or 0x1.1df46a2529d39p-6 or 0x1.1df46ap-6F
-RADIANS_DEF(double, 0x1.1df46a2529d39p-6)
+__CLC_RADIANS_DEF(double, 0x1.1df46a2529d39p-6)
 
 #endif
 
@@ -37,6 +37,6 @@ RADIANS_DEF(double, 0x1.1df46a2529d39p-6)
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // pi/180 = ~0.01745329251994329577 or 0x1.1df46a2529d39p-6 or 0x1.1df46ap-6F
-RADIANS_DEF(half, (half)0x1.1df46a2529d39p-6)
+__CLC_RADIANS_DEF(half, (half)0x1.1df46a2529d39p-6)
 
 #endif
diff --git a/libclc/clc/lib/generic/geometric/clc_fast_distance.cl b/libclc/clc/lib/generic/geometric/clc_fast_distance.cl
index 996039898480c..6b645df0b3ddb 100644
--- a/libclc/clc/lib/generic/geometric/clc_fast_distance.cl
+++ b/libclc/clc/lib/generic/geometric/clc_fast_distance.cl
@@ -9,6 +9,6 @@
 #include <clc/geometric/clc_fast_length.h>
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_fast_distance.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/geometric/clc_fast_normalize.cl b/libclc/clc/lib/generic/geometric/clc_fast_normalize.cl
index 85684d0f49bc1..a5883efe7bd90 100644
--- a/libclc/clc/lib/generic/geometric/clc_fast_normalize.cl
+++ b/libclc/clc/lib/generic/geometric/clc_fast_normalize.cl
@@ -10,6 +10,6 @@
 #include <clc/geometric/clc_normalize.h>
 #include <clc/math/clc_half_rsqrt.h>
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_fast_normalize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_add_sat.cl b/libclc/clc/lib/generic/integer/clc_add_sat.cl
index 8241ee5d547ac..08337bf533fc0 100644
--- a/libclc/clc/lib/generic/integer/clc_add_sat.cl
+++ b/libclc/clc/lib/generic/integer/clc_add_sat.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_add_sat
-#define __IMPL_FUNCTION(x) __builtin_elementwise_add_sat
+#define __CLC_FUNCTION __clc_add_sat
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_add_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_bit_reverse.cl b/libclc/clc/lib/generic/integer/clc_bit_reverse.cl
index 439957383f583..9bf57e2c7d83f 100644
--- a/libclc/clc/lib/generic/integer/clc_bit_reverse.cl
+++ b/libclc/clc/lib/generic/integer/clc_bit_reverse.cl
@@ -8,8 +8,8 @@
 
 #include <clc/integer/clc_bit_reverse.h>
 
-#define FUNCTION __clc_bit_reverse
-#define __IMPL_FUNCTION(x) __builtin_elementwise_bitreverse
+#define __CLC_FUNCTION __clc_bit_reverse
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_bitreverse
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_clz.cl b/libclc/clc/lib/generic/integer/clc_clz.cl
index c6e1da680b7bd..37ba4dbfd8094 100644
--- a/libclc/clc/lib/generic/integer/clc_clz.cl
+++ b/libclc/clc/lib/generic/integer/clc_clz.cl
@@ -10,38 +10,5 @@
 #include <clc/integer/clc_clz.h>
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DEF char __clc_clz(char x) {
-  return __clc_clz((ushort)(uchar)x) - 8;
-}
-
-_CLC_OVERLOAD _CLC_DEF uchar __clc_clz(uchar x) {
-  return __clc_clz((ushort)x) - 8;
-}
-
-_CLC_OVERLOAD _CLC_DEF short __clc_clz(short x) {
-  return x ? __builtin_clzs(x) : 16;
-}
-
-_CLC_OVERLOAD _CLC_DEF ushort __clc_clz(ushort x) {
-  return x ? __builtin_clzs(x) : 16;
-}
-
-_CLC_OVERLOAD _CLC_DEF int __clc_clz(int x) {
-  return x ? __builtin_clz(x) : 32;
-}
-
-_CLC_OVERLOAD _CLC_DEF uint __clc_clz(uint x) {
-  return x ? __builtin_clz(x) : 32;
-}
-
-_CLC_OVERLOAD _CLC_DEF long __clc_clz(long x) {
-  return x ? __builtin_clzl(x) : 64;
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong __clc_clz(ulong x) {
-  return x ? __builtin_clzl(x) : 64;
-}
-
-#define FUNCTION __clc_clz
-#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#define __CLC_BODY <clc_clz.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_clz.inc b/libclc/clc/lib/generic/integer/clc_clz.inc
new file mode 100644
index 0000000000000..ba7eb77d50036
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_clz.inc
@@ -0,0 +1,11 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_clz(__CLC_GENTYPE x) {
+  return __builtin_elementwise_ctlz(x, (__CLC_GENTYPE)__CLC_GENSIZE);
+}
diff --git a/libclc/clc/lib/generic/integer/clc_ctz.cl b/libclc/clc/lib/generic/integer/clc_ctz.cl
index d82d99d539df8..5cf4c19811db2 100644
--- a/libclc/clc/lib/generic/integer/clc_ctz.cl
+++ b/libclc/clc/lib/generic/integer/clc_ctz.cl
@@ -10,34 +10,5 @@
 #include <clc/integer/clc_ctz.h>
 #include <clc/internal/clc.h>
 
-_CLC_OVERLOAD _CLC_DEF char __clc_ctz(char x) {
-  return __clc_ctz(__clc_as_uchar(x));
-}
-
-_CLC_OVERLOAD _CLC_DEF uchar __clc_ctz(uchar x) { return __builtin_ctzg(x, 8); }
-
-_CLC_OVERLOAD _CLC_DEF short __clc_ctz(short x) {
-  return __clc_ctz(__clc_as_ushort(x));
-}
-
-_CLC_OVERLOAD _CLC_DEF ushort __clc_ctz(ushort x) {
-  return __builtin_ctzg(x, 16);
-}
-
-_CLC_OVERLOAD _CLC_DEF int __clc_ctz(int x) {
-  return __clc_ctz(__clc_as_uint(x));
-}
-
-_CLC_OVERLOAD _CLC_DEF uint __clc_ctz(uint x) { return __builtin_ctzg(x, 32); }
-
-_CLC_OVERLOAD _CLC_DEF long __clc_ctz(long x) {
-  return __clc_ctz(__clc_as_ulong(x));
-}
-
-_CLC_OVERLOAD _CLC_DEF ulong __clc_ctz(ulong x) {
-  return __builtin_ctzg(x, 64);
-}
-
-#define FUNCTION __clc_ctz
-#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#define __CLC_BODY <clc_ctz.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_ctz.inc b/libclc/clc/lib/generic/integer/clc_ctz.inc
new file mode 100644
index 0000000000000..cebd392542be9
--- /dev/null
+++ b/libclc/clc/lib/generic/integer/clc_ctz.inc
@@ -0,0 +1,11 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+_CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_ctz(__CLC_GENTYPE x) {
+  return __builtin_elementwise_cttz(x, (__CLC_GENTYPE)__CLC_GENSIZE);
+}
diff --git a/libclc/clc/lib/generic/integer/clc_popcount.cl b/libclc/clc/lib/generic/integer/clc_popcount.cl
index 078ceecc97691..2781ead638c97 100644
--- a/libclc/clc/lib/generic/integer/clc_popcount.cl
+++ b/libclc/clc/lib/generic/integer/clc_popcount.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_popcount
-#define __IMPL_FUNCTION(x) __builtin_elementwise_popcount
+#define __CLC_FUNCTION __clc_popcount
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_popcount
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/integer/clc_sub_sat.cl b/libclc/clc/lib/generic/integer/clc_sub_sat.cl
index e664f04b17481..8a7ffd06f8b0b 100644
--- a/libclc/clc/lib/generic/integer/clc_sub_sat.cl
+++ b/libclc/clc/lib/generic/integer/clc_sub_sat.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_sub_sat
-#define __IMPL_FUNCTION(x) __builtin_elementwise_sub_sat
+#define __CLC_FUNCTION __clc_sub_sat
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_sub_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_atan2.inc b/libclc/clc/lib/generic/math/clc_atan2.inc
index 61ffeebbc5d11..19eaaeee0092d 100644
--- a/libclc/clc/lib/generic/math/clc_atan2.inc
+++ b/libclc/clc/lib/generic/math/clc_atan2.inc
@@ -131,8 +131,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_atan2(__CLC_GENTYPE y,
   {
     __CLC_GENTYPE val = vbyu > 0.0625 ? vbyu : 0.063;
     __CLC_INTN index = __CLC_CONVERT_INTN(__clc_fma(256.0, val, 0.5));
-    q1 = USE_TABLE(atan_jby256_tbl_head, index - 16);
-    q2 = USE_TABLE(atan_jby256_tbl_tail, index - 16);
+    q1 = __CLC_USE_TABLE(atan_jby256_tbl_head, index - 16);
+    q2 = __CLC_USE_TABLE(atan_jby256_tbl_tail, index - 16);
     __CLC_GENTYPE c = __CLC_CONVERT_GENTYPE(index) * 0x1.0p-8;
 
     // We're going to scale u and v by 2^(-u_exponent) to bring them close to 1
diff --git a/libclc/clc/lib/generic/math/clc_atan2pi.inc b/libclc/clc/lib/generic/math/clc_atan2pi.inc
index 79b2551e077cc..9f901947887d8 100644
--- a/libclc/clc/lib/generic/math/clc_atan2pi.inc
+++ b/libclc/clc/lib/generic/math/clc_atan2pi.inc
@@ -110,8 +110,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_atan2pi(__CLC_GENTYPE y,
   {
     __CLC_GENTYPE val = vbyu > 0.0625 ? vbyu : 0.063;
     __CLC_INTN index = __CLC_CONVERT_INTN(__clc_fma(256.0, val, 0.5));
-    q1 = USE_TABLE(atan_jby256_tbl_head, (index - 16));
-    q2 = USE_TABLE(atan_jby256_tbl_tail, (index - 16));
+    q1 = __CLC_USE_TABLE(atan_jby256_tbl_head, (index - 16));
+    q2 = __CLC_USE_TABLE(atan_jby256_tbl_tail, (index - 16));
     __CLC_GENTYPE c = __CLC_CONVERT_GENTYPE(index) * 0x1.0p-8;
 
     // We're going to scale u and v by 2^(-u_exponent) to bring them close to 1
diff --git a/libclc/clc/lib/generic/math/clc_cbrt.inc b/libclc/clc/lib/generic/math/clc_cbrt.inc
index 2aba2484e33fc..ce0e0fe453169 100644
--- a/libclc/clc/lib/generic/math/clc_cbrt.inc
+++ b/libclc/clc/lib/generic/math/clc_cbrt.inc
@@ -32,7 +32,7 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_cbrt(__CLC_GENTYPE x) {
                     __CLC_AS_GENTYPE(indx | 0x3f000000);
 
   indx >>= 16;
-  __CLC_GENTYPE r = f * USE_TABLE(log_inv_tbl, __CLC_AS_INTN(indx));
+  __CLC_GENTYPE r = f * __CLC_USE_TABLE(log_inv_tbl, __CLC_AS_INTN(indx));
   __CLC_GENTYPE poly = __clc_mad(__clc_mad(r, 0x1.f9add4p-5f, -0x1.c71c72p-4f),
                                  r * r, r * 0x1.555556p-2f);
 
@@ -52,8 +52,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_cbrt(__CLC_GENTYPE x) {
   remH = rem == 2 ? 0x1.964000p+0f : remH;
   remT = rem == 2 ? 0x1.fea53ep-12f : remT;
 
-  __CLC_GENTYPE cbrtH = USE_TABLE(cbrt_tbl_head, __CLC_AS_INTN(indx));
-  __CLC_GENTYPE cbrtT = USE_TABLE(cbrt_tbl_tail, __CLC_AS_INTN(indx));
+  __CLC_GENTYPE cbrtH = __CLC_USE_TABLE(cbrt_tbl_head, __CLC_AS_INTN(indx));
+  __CLC_GENTYPE cbrtT = __CLC_USE_TABLE(cbrt_tbl_tail, __CLC_AS_INTN(indx));
 
   __CLC_GENTYPE bH = cbrtH * remH;
   __CLC_GENTYPE bT =
@@ -97,7 +97,7 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_cbrt(__CLC_GENTYPE x) {
   __CLC_GENTYPE F = __CLC_CONVERT_GENTYPE(index) * 0x1.0p-9;
 
   __CLC_GENTYPE f = Y - F;
-  __CLC_GENTYPE r = f * USE_TABLE(cbrt_inv_tbl, index - 256);
+  __CLC_GENTYPE r = f * __CLC_USE_TABLE(cbrt_inv_tbl, index - 256);
 
   __CLC_GENTYPE z =
       r * __clc_fma(
@@ -112,11 +112,11 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_cbrt(__CLC_GENTYPE x) {
                         -0x1.c71c71c71c71cp-4),
               0x1.5555555555555p-2);
 
-  __CLC_GENTYPE Rem_h = USE_TABLE(cbrt_rem_tbl_head, rem + 2);
-  __CLC_GENTYPE Rem_t = USE_TABLE(cbrt_rem_tbl_tail, rem + 2);
+  __CLC_GENTYPE Rem_h = __CLC_USE_TABLE(cbrt_rem_tbl_head, rem + 2);
+  __CLC_GENTYPE Rem_t = __CLC_USE_TABLE(cbrt_rem_tbl_tail, rem + 2);
 
-  __CLC_GENTYPE F_h = USE_TABLE(cbrt_dbl_tbl_head, index - 256);
-  __CLC_GENTYPE F_t = USE_TABLE(cbrt_dbl_tbl_tail, index - 256);
+  __CLC_GENTYPE F_h = __CLC_USE_TABLE(cbrt_dbl_tbl_head, index - 256);
+  __CLC_GENTYPE F_t = __CLC_USE_TABLE(cbrt_dbl_tbl_tail, index - 256);
 
   __CLC_GENTYPE b_h = F_h * Rem_h;
   __CLC_GENTYPE b_t = __clc_fma(Rem_t, F_h, __clc_fma(F_t, Rem_h, F_t * Rem_t));
diff --git a/libclc/clc/lib/generic/math/clc_ceil.cl b/libclc/clc/lib/generic/math/clc_ceil.cl
index b06cb364b58d9..e48ad1a482ef7 100644
--- a/libclc/clc/lib/generic/math/clc_ceil.cl
+++ b/libclc/clc/lib/generic/math/clc_ceil.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_ceil
-#define __IMPL_FUNCTION(x) __builtin_elementwise_ceil
+#define __CLC_FUNCTION __clc_ceil
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_ceil
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_copysign.cl b/libclc/clc/lib/generic/math/clc_copysign.cl
index 06f126b16dfde..89829e2375ea3 100644
--- a/libclc/clc/lib/generic/math/clc_copysign.cl
+++ b/libclc/clc/lib/generic/math/clc_copysign.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_copysign
-#define __IMPL_FUNCTION(x) __builtin_elementwise_copysign
+#define __CLC_FUNCTION __clc_copysign
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_copysign
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_cosh.inc b/libclc/clc/lib/generic/math/clc_cosh.inc
index a9fa1bb14140c..c80b1a38c7768 100644
--- a/libclc/clc/lib/generic/math/clc_cosh.inc
+++ b/libclc/clc/lib/generic/math/clc_cosh.inc
@@ -71,8 +71,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_cosh(__CLC_GENTYPE x) {
       0.500000000000000005911074e0f);
   cdy = __clc_mad(cdy, dy2, 1.0f);
 
-  __CLC_GENTYPE sinhcoshh = USE_TABLE(sinhcosh_tbl_head, ind);
-  __CLC_GENTYPE sinhcosht = USE_TABLE(sinhcosh_tbl_tail, ind);
+  __CLC_GENTYPE sinhcoshh = __CLC_USE_TABLE(sinhcosh_tbl_head, ind);
+  __CLC_GENTYPE sinhcosht = __CLC_USE_TABLE(sinhcosh_tbl_tail, ind);
   __CLC_GENTYPE z = __clc_mad(sinhcoshh, sdy, sinhcosht * cdy);
 
   // When exp(-x) is insignificant compared to exp(x), return exp(x)/2
@@ -162,10 +162,10 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_cosh(__CLC_GENTYPE x) {
 
   // At this point sinh(dy) is approximated by dy + sdy,
   // and cosh(dy) is approximated by 1 + cdy.
-  __CLC_GENTYPE cl = USE_TABLE(cosh_tbl_head, ind);
-  __CLC_GENTYPE ct = USE_TABLE(cosh_tbl_tail, ind);
-  __CLC_GENTYPE sl = USE_TABLE(sinh_tbl_head, ind);
-  __CLC_GENTYPE st = USE_TABLE(sinh_tbl_tail, ind);
+  __CLC_GENTYPE cl = __CLC_USE_TABLE(cosh_tbl_head, ind);
+  __CLC_GENTYPE ct = __CLC_USE_TABLE(cosh_tbl_tail, ind);
+  __CLC_GENTYPE sl = __CLC_USE_TABLE(sinh_tbl_head, ind);
+  __CLC_GENTYPE st = __CLC_USE_TABLE(sinh_tbl_tail, ind);
 
   __CLC_GENTYPE z =
       __clc_fma(
diff --git a/libclc/clc/lib/generic/math/clc_ep_log.inc b/libclc/clc/lib/generic/math/clc_ep_log.inc
index f51e487bedcaa..d8cc886ffc745 100644
--- a/libclc/clc/lib/generic/math/clc_ep_log.inc
+++ b/libclc/clc/lib/generic/math/clc_ep_log.inc
@@ -58,8 +58,8 @@ _CLC_DEF _CLC_OVERLOAD void __clc_ep_log(__CLC_GENTYPE x,
   __CLC_GENTYPE f2 = f - f1;
   __CLC_GENTYPE u2 = MATH_DIVIDE(f2, __clc_fma(0.5, f2, f1));
 
-  __CLC_GENTYPE z1 = USE_TABLE(ln_tbl_lo, (index - 64));
-  __CLC_GENTYPE q = USE_TABLE(ln_tbl_hi, (index - 64));
+  __CLC_GENTYPE z1 = __CLC_USE_TABLE(ln_tbl_lo, (index - 64));
+  __CLC_GENTYPE q = __CLC_USE_TABLE(ln_tbl_hi, (index - 64));
 
   z1 = near_one ? r : z1;
   q = near_one ? 0.0 : q;
diff --git a/libclc/clc/lib/generic/math/clc_erf.cl b/libclc/clc/lib/generic/math/clc_erf.cl
index bea392445c93c..34c7d586131e2 100644
--- a/libclc/clc/lib/generic/math/clc_erf.cl
+++ b/libclc/clc/lib/generic/math/clc_erf.cl
@@ -506,6 +506,6 @@ _CLC_OVERLOAD _CLC_DEF half __clc_erf(half x) {
 
 #endif
 
-#define FUNCTION __clc_erf
+#define __CLC_FUNCTION __clc_erf
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_erfc.cl b/libclc/clc/lib/generic/math/clc_erfc.cl
index fab6960046028..7922807818ea2 100644
--- a/libclc/clc/lib/generic/math/clc_erfc.cl
+++ b/libclc/clc/lib/generic/math/clc_erfc.cl
@@ -517,6 +517,6 @@ _CLC_OVERLOAD _CLC_DEF half __clc_erfc(half x) {
 
 #endif
 
-#define FUNCTION __clc_erfc
+#define __CLC_FUNCTION __clc_erfc
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_exp10.inc b/libclc/clc/lib/generic/math/clc_exp10.inc
index 96bc5331fef17..8bf4ec4c0b37d 100644
--- a/libclc/clc/lib/generic/math/clc_exp10.inc
+++ b/libclc/clc/lib/generic/math/clc_exp10.inc
@@ -69,7 +69,7 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_exp10(__CLC_GENTYPE x) {
                           0x1.000000p-1f),
                 r * r, r);
 
-  __CLC_GENTYPE two_to_jby64 = USE_TABLE(exp_tbl, j);
+  __CLC_GENTYPE two_to_jby64 = __CLC_USE_TABLE(exp_tbl, j);
   z2 = __clc_mad(two_to_jby64, z2, two_to_jby64);
 
   __CLC_GENTYPE z2s = z2 * __CLC_AS_GENTYPE((__CLC_UINTN)0x1 << (m + 149));
@@ -122,8 +122,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_exp10(__CLC_GENTYPE x) {
                         0x1.0000000000000p-1),
               1.0);
 
-  __CLC_GENTYPE tv0 = USE_TABLE(two_to_jby64_ep_tbl_head, j);
-  __CLC_GENTYPE tv1 = USE_TABLE(two_to_jby64_ep_tbl_tail, j);
+  __CLC_GENTYPE tv0 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_head, j);
+  __CLC_GENTYPE tv1 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_tail, j);
   z2 = __clc_fma(tv0 + tv1, z2, tv1) + tv0;
 
   __CLC_INTN small_value =
diff --git a/libclc/clc/lib/generic/math/clc_exp_helper.cl b/libclc/clc/lib/generic/math/clc_exp_helper.cl
index 92ff8f7fe4e6f..1670b73b6c980 100644
--- a/libclc/clc/lib/generic/math/clc_exp_helper.cl
+++ b/libclc/clc/lib/generic/math/clc_exp_helper.cl
@@ -14,7 +14,7 @@
 #include <clc/math/tables.h>
 #include <clc/relational/clc_isnan.h>
 
-#define __DOUBLE_ONLY
+#define __CLC_DOUBLE_ONLY
 #define __CLC_BODY <clc_exp_helper.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_exp_helper.inc b/libclc/clc/lib/generic/math/clc_exp_helper.inc
index 70ced7e9ea485..4ccf2d12b8cf5 100644
--- a/libclc/clc/lib/generic/math/clc_exp_helper.inc
+++ b/libclc/clc/lib/generic/math/clc_exp_helper.inc
@@ -29,8 +29,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_exp_helper(__CLC_GENTYPE x,
                         0x1.0000000000000p-1),
               1.0);
 
-  __CLC_GENTYPE tv0 = USE_TABLE(two_to_jby64_ep_tbl_head, j);
-  __CLC_GENTYPE tv1 = USE_TABLE(two_to_jby64_ep_tbl_tail, j);
+  __CLC_GENTYPE tv0 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_head, j);
+  __CLC_GENTYPE tv1 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_tail, j);
   z2 = __clc_fma(tv0 + tv1, z2, tv1) + tv0;
 
   __CLC_INTN small_value =
diff --git a/libclc/clc/lib/generic/math/clc_expm1.inc b/libclc/clc/lib/generic/math/clc_expm1.inc
index 6abee9b3f0cc9..a7ff90ea86ca0 100644
--- a/libclc/clc/lib/generic/math/clc_expm1.inc
+++ b/libclc/clc/lib/generic/math/clc_expm1.inc
@@ -37,8 +37,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_expm1(__CLC_GENTYPE x) {
       r);
 
   __CLC_GENTYPE m2 = __CLC_AS_GENTYPE((m + EXPBIAS_SP32) << EXPSHIFTBITS_SP32);
-  __CLC_GENTYPE exp_head = USE_TABLE(exp_tbl_ep_head, j);
-  __CLC_GENTYPE exp_tail = USE_TABLE(exp_tbl_ep_tail, j);
+  __CLC_GENTYPE exp_head = __CLC_USE_TABLE(exp_tbl_ep_head, j);
+  __CLC_GENTYPE exp_tail = __CLC_USE_TABLE(exp_tbl_ep_tail, j);
 
   __CLC_GENTYPE two_to_jby64_h = exp_head * m2;
   __CLC_GENTYPE two_to_jby64_t = exp_tail * m2;
@@ -108,8 +108,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_expm1(__CLC_GENTYPE x) {
   __CLC_INTN j = n & 0x3f;
   __CLC_INTN m = n >> 6;
 
-  __CLC_GENTYPE f1 = USE_TABLE(two_to_jby64_ep_tbl_head, j);
-  __CLC_GENTYPE f2 = USE_TABLE(two_to_jby64_ep_tbl_tail, j);
+  __CLC_GENTYPE f1 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_head, j);
+  __CLC_GENTYPE f2 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_tail, j);
   __CLC_GENTYPE f = f1 + f2;
 
   __CLC_GENTYPE dn = __CLC_CONVERT_GENTYPE(-n);
diff --git a/libclc/clc/lib/generic/math/clc_fabs.cl b/libclc/clc/lib/generic/math/clc_fabs.cl
index a0f0f3af07e25..e0c7685bfdb64 100644
--- a/libclc/clc/lib/generic/math/clc_fabs.cl
+++ b/libclc/clc/lib/generic/math/clc_fabs.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_fabs
-#define __IMPL_FUNCTION(x) __builtin_elementwise_abs
+#define __CLC_FUNCTION __clc_fabs
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_abs
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_floor.cl b/libclc/clc/lib/generic/math/clc_floor.cl
index a14adb98297e7..f31bed2e9ee64 100644
--- a/libclc/clc/lib/generic/math/clc_floor.cl
+++ b/libclc/clc/lib/generic/math/clc_floor.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_floor
-#define __IMPL_FUNCTION(x) __builtin_elementwise_floor
+#define __CLC_FUNCTION __clc_floor
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_floor
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_fmax.cl b/libclc/clc/lib/generic/math/clc_fmax.cl
index b334207365b98..0fdbaa5eef5f6 100644
--- a/libclc/clc/lib/generic/math/clc_fmax.cl
+++ b/libclc/clc/lib/generic/math/clc_fmax.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_fmax
-#define __IMPL_FUNCTION(x) __builtin_elementwise_maximumnum
+#define __CLC_FUNCTION __clc_fmax
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_maximumnum
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_fmin.cl b/libclc/clc/lib/generic/math/clc_fmin.cl
index d21bb8d076790..572d54b128a6f 100644
--- a/libclc/clc/lib/generic/math/clc_fmin.cl
+++ b/libclc/clc/lib/generic/math/clc_fmin.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_fmin
-#define __IMPL_FUNCTION(x) __builtin_elementwise_minimumnum
+#define __CLC_FUNCTION __clc_fmin
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_minimumnum
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_fmod.cl b/libclc/clc/lib/generic/math/clc_fmod.cl
index 3162ef60d46ef..3cb01e67292ed 100644
--- a/libclc/clc/lib/generic/math/clc_fmod.cl
+++ b/libclc/clc/lib/generic/math/clc_fmod.cl
@@ -64,11 +64,11 @@ _CLC_DEF _CLC_OVERLOAD float __clc_fmod(float x, float y) {
   return xr;
 }
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_fmod
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_fmod
 #define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -176,11 +176,11 @@ _CLC_DEF _CLC_OVERLOAD double __clc_fmod(double x, double y) {
   return ret;
 }
 
-#define __DOUBLE_ONLY
-#define FUNCTION __clc_fmod
+#define __CLC_DOUBLE_ONLY
+#define __CLC_FUNCTION __clc_fmod
 #define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif
 
@@ -189,8 +189,8 @@ _CLC_DEF _CLC_OVERLOAD double __clc_fmod(double x, double y) {
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // Forward the half version of this builtin onto the float one
-#define __HALF_ONLY
-#define FUNCTION __clc_fmod
+#define __CLC_HALF_ONLY
+#define __CLC_FUNCTION __clc_fmod
 #define __CLC_BODY <clc/math/binary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_fract.inc b/libclc/clc/lib/generic/math/clc_fract.inc
index f0466e339b341..5b12c74f293f2 100644
--- a/libclc/clc/lib/generic/math/clc_fract.inc
+++ b/libclc/clc/lib/generic/math/clc_fract.inc
@@ -23,7 +23,7 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_fract(__CLC_GENTYPE x,
   return r;
 }
 
-#define FRACT_DEF(addrspace)                                                   \
+#define __CLC_FRACT_DEF(addrspace)                                             \
   _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_fract(                            \
       __CLC_GENTYPE x, addrspace __CLC_GENTYPE *iptr) {                        \
     __CLC_GENTYPE private_iptr;                                                \
@@ -32,10 +32,10 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_fract(__CLC_GENTYPE x,
     return ret;                                                                \
   }
 
-FRACT_DEF(local);
-FRACT_DEF(global);
+__CLC_FRACT_DEF(local);
+__CLC_FRACT_DEF(global);
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-FRACT_DEF(generic);
+__CLC_FRACT_DEF(generic);
 #endif
 
 #undef MIN_CONSTANT
diff --git a/libclc/clc/lib/generic/math/clc_half_cos.cl b/libclc/clc/lib/generic/math/clc_half_cos.cl
index 79a7ee7b483f4..4420ad31afae5 100644
--- a/libclc/clc/lib/generic/math/clc_half_cos.cl
+++ b/libclc/clc/lib/generic/math/clc_half_cos.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_cos.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_cos
-#define __IMPL_FUNCTION(x) __clc_cos
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_cos
+#define __CLC_IMPL_FUNCTION(x) __clc_cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_divide.cl b/libclc/clc/lib/generic/math/clc_half_divide.cl
index 88676a4ac7856..16af197d29a75 100644
--- a/libclc/clc/lib/generic/math/clc_half_divide.cl
+++ b/libclc/clc/lib/generic/math/clc_half_divide.cl
@@ -8,7 +8,7 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_divide.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_exp.cl b/libclc/clc/lib/generic/math/clc_half_exp.cl
index b53454580e3ce..4b4717672d368 100644
--- a/libclc/clc/lib/generic/math/clc_half_exp.cl
+++ b/libclc/clc/lib/generic/math/clc_half_exp.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_exp.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_exp
-#define __IMPL_FUNCTION(x) __clc_exp
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_exp
+#define __CLC_IMPL_FUNCTION(x) __clc_exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_exp10.cl b/libclc/clc/lib/generic/math/clc_half_exp10.cl
index 7bd107bf59e85..5bacca1ee0b7d 100644
--- a/libclc/clc/lib/generic/math/clc_half_exp10.cl
+++ b/libclc/clc/lib/generic/math/clc_half_exp10.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_exp10.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_exp10
-#define __IMPL_FUNCTION(x) __clc_exp10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_exp10
+#define __CLC_IMPL_FUNCTION(x) __clc_exp10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_exp2.cl b/libclc/clc/lib/generic/math/clc_half_exp2.cl
index a20d79bfff6e3..9a8f2f980c5a7 100644
--- a/libclc/clc/lib/generic/math/clc_half_exp2.cl
+++ b/libclc/clc/lib/generic/math/clc_half_exp2.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_exp2.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_exp2
-#define __IMPL_FUNCTION(x) __clc_exp2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_exp2
+#define __CLC_IMPL_FUNCTION(x) __clc_exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_log.cl b/libclc/clc/lib/generic/math/clc_half_log.cl
index 26b2c756c9889..160b193e8040c 100644
--- a/libclc/clc/lib/generic/math/clc_half_log.cl
+++ b/libclc/clc/lib/generic/math/clc_half_log.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_log.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_log
-#define __IMPL_FUNCTION(x) __clc_log
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_log
+#define __CLC_IMPL_FUNCTION(x) __clc_log
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_log10.cl b/libclc/clc/lib/generic/math/clc_half_log10.cl
index 36f5bf55bb531..83d57db1bdf2f 100644
--- a/libclc/clc/lib/generic/math/clc_half_log10.cl
+++ b/libclc/clc/lib/generic/math/clc_half_log10.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_log10.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_log10
-#define __IMPL_FUNCTION(x) __clc_log10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_log10
+#define __CLC_IMPL_FUNCTION(x) __clc_log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_log2.cl b/libclc/clc/lib/generic/math/clc_half_log2.cl
index cbf1d350dd8e2..7de0053fd7507 100644
--- a/libclc/clc/lib/generic/math/clc_half_log2.cl
+++ b/libclc/clc/lib/generic/math/clc_half_log2.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_log2.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_log2
-#define __IMPL_FUNCTION(x) __clc_log2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_log2
+#define __CLC_IMPL_FUNCTION(x) __clc_log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_powr.cl b/libclc/clc/lib/generic/math/clc_half_powr.cl
index f7ef2074b840e..7065377ede66c 100644
--- a/libclc/clc/lib/generic/math/clc_half_powr.cl
+++ b/libclc/clc/lib/generic/math/clc_half_powr.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_powr.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_powr
-#define __IMPL_FUNCTION(x) __clc_powr
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_powr
+#define __CLC_IMPL_FUNCTION(x) __clc_powr
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_recip.cl b/libclc/clc/lib/generic/math/clc_half_recip.cl
index 0ae1e922d489f..a988ad5e05a8f 100644
--- a/libclc/clc/lib/generic/math/clc_half_recip.cl
+++ b/libclc/clc/lib/generic/math/clc_half_recip.cl
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_half_recip.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_rsqrt.cl b/libclc/clc/lib/generic/math/clc_half_rsqrt.cl
index 7e5d9e0526851..4e0cdd252f656 100644
--- a/libclc/clc/lib/generic/math/clc_half_rsqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_half_rsqrt.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_rsqrt.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_rsqrt
-#define __IMPL_FUNCTION(x) __clc_rsqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_rsqrt
+#define __CLC_IMPL_FUNCTION(x) __clc_rsqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_sin.cl b/libclc/clc/lib/generic/math/clc_half_sin.cl
index ef333dae3797c..acc5d43156ab8 100644
--- a/libclc/clc/lib/generic/math/clc_half_sin.cl
+++ b/libclc/clc/lib/generic/math/clc_half_sin.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_sin.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_sin
-#define __IMPL_FUNCTION(x) __clc_sin
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_sin
+#define __CLC_IMPL_FUNCTION(x) __clc_sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_sqrt.cl b/libclc/clc/lib/generic/math/clc_half_sqrt.cl
index 293eb81196b63..01e779960df6c 100644
--- a/libclc/clc/lib/generic/math/clc_half_sqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_half_sqrt.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_sqrt.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_sqrt
-#define __IMPL_FUNCTION(x) __clc_sqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_sqrt
+#define __CLC_IMPL_FUNCTION(x) __clc_sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_half_tan.cl b/libclc/clc/lib/generic/math/clc_half_tan.cl
index ecc9f8f328311..1427fe02555f1 100644
--- a/libclc/clc/lib/generic/math/clc_half_tan.cl
+++ b/libclc/clc/lib/generic/math/clc_half_tan.cl
@@ -8,9 +8,9 @@
 
 #include <clc/math/clc_tan.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_half_tan
-#define __IMPL_FUNCTION(x) __clc_tan
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_half_tan
+#define __CLC_IMPL_FUNCTION(x) __clc_tan
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_ldexp.cl b/libclc/clc/lib/generic/math/clc_ldexp.cl
index cb4185d89c729..a5327c58e424a 100644
--- a/libclc/clc/lib/generic/math/clc_ldexp.cl
+++ b/libclc/clc/lib/generic/math/clc_ldexp.cl
@@ -131,7 +131,7 @@ _CLC_OVERLOAD _CLC_DEF_ldexp half __clc_ldexp(half x, int n) {
 
 #endif
 
-#define FUNCTION __clc_ldexp
+#define __CLC_FUNCTION __clc_ldexp
 #define __CLC_DEF_SPEC _CLC_DEF_ldexp
 #define __CLC_ARG2_TYPE int
 #define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
diff --git a/libclc/clc/lib/generic/math/clc_log.cl b/libclc/clc/lib/generic/math/clc_log.cl
index cf5628f206cad..7eb0180de971e 100644
--- a/libclc/clc/lib/generic/math/clc_log.cl
+++ b/libclc/clc/lib/generic/math/clc_log.cl
@@ -39,6 +39,6 @@ _CLC_OVERLOAD _CLC_DEF half __clc_log(half x) {
 
 #endif // cl_khr_fp16
 
-#define FUNCTION __clc_log
+#define __CLC_FUNCTION __clc_log
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_log10.cl b/libclc/clc/lib/generic/math/clc_log10.cl
index f5f0e8cc70837..35489f4675629 100644
--- a/libclc/clc/lib/generic/math/clc_log10.cl
+++ b/libclc/clc/lib/generic/math/clc_log10.cl
@@ -22,6 +22,6 @@
 #include "clc_log_base.h"
 #undef COMPILING_LOG10
 
-#define FUNCTION __clc_log10
+#define __CLC_FUNCTION __clc_log10
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_log1p.inc b/libclc/clc/lib/generic/math/clc_log1p.inc
index 8c7dcfc48c18e..2dd616818c715 100644
--- a/libclc/clc/lib/generic/math/clc_log1p.inc
+++ b/libclc/clc/lib/generic/math/clc_log1p.inc
@@ -56,7 +56,7 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_log1p(__CLC_GENTYPE x) {
   __CLC_GENTYPE f = mf > 24.0f ? fg24 : fl24;
 
   indx = indx >> 16;
-  __CLC_GENTYPE r = f * USE_TABLE(log_inv_tbl, __CLC_CONVERT_INTN(indx));
+  __CLC_GENTYPE r = f * __CLC_USE_TABLE(log_inv_tbl, __CLC_CONVERT_INTN(indx));
 
   // 1/3, 1/2
   __CLC_GENTYPE poly =
@@ -65,8 +65,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_log1p(__CLC_GENTYPE x) {
   const __CLC_GENTYPE LOG2_HEAD = 0x1.62e000p-1f;  // 0.693115234
   const __CLC_GENTYPE LOG2_TAIL = 0x1.0bfbe8p-15f; // 0.0000319461833
 
-  __CLC_GENTYPE tv0 = USE_TABLE(loge_tbl_lo, __CLC_AS_INTN(indx));
-  __CLC_GENTYPE tv1 = USE_TABLE(loge_tbl_hi, __CLC_AS_INTN(indx));
+  __CLC_GENTYPE tv0 = __CLC_USE_TABLE(loge_tbl_lo, __CLC_AS_INTN(indx));
+  __CLC_GENTYPE tv1 = __CLC_USE_TABLE(loge_tbl_hi, __CLC_AS_INTN(indx));
   __CLC_GENTYPE z1 = __clc_mad(mf, LOG2_HEAD, tv0);
   __CLC_GENTYPE z2 = __clc_mad(mf, LOG2_TAIL, -poly) + tv1;
   __CLC_GENTYPE z = z1 + z2;
@@ -108,8 +108,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_log1p(__CLC_GENTYPE x) {
   f2 = __CLC_CONVERT_LONGN(xexp <= -2 || (xexp >= MANTLENGTH_DP64 + 8)) ? f2temp
                                                                         : f2;
 
-  __CLC_GENTYPE z1 = USE_TABLE(ln_tbl_lo, j);
-  __CLC_GENTYPE q = USE_TABLE(ln_tbl_hi, j);
+  __CLC_GENTYPE z1 = __CLC_USE_TABLE(ln_tbl_lo, j);
+  __CLC_GENTYPE q = __CLC_USE_TABLE(ln_tbl_hi, j);
 
   __CLC_GENTYPE u = MATH_DIVIDE(f2, __clc_fma(0.5, f2, f1));
   __CLC_GENTYPE v = u * u;
diff --git a/libclc/clc/lib/generic/math/clc_log2.cl b/libclc/clc/lib/generic/math/clc_log2.cl
index 335488af2f3de..d9d8cef542557 100644
--- a/libclc/clc/lib/generic/math/clc_log2.cl
+++ b/libclc/clc/lib/generic/math/clc_log2.cl
@@ -22,6 +22,6 @@
 #include "clc_log_base.h"
 #undef COMPILING_LOG2
 
-#define FUNCTION __clc_log2
+#define __CLC_FUNCTION __clc_log2
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_log_base.h b/libclc/clc/lib/generic/math/clc_log_base.h
index 9418535db827d..ba92679dbaaf4 100644
--- a/libclc/clc/lib/generic/math/clc_log_base.h
+++ b/libclc/clc/lib/generic/math/clc_log_base.h
@@ -142,21 +142,21 @@ __clc_log(float x)
             __clc_as_float(0x3f000000 | (xin & MANTBITS_SP32));
 
   indx = indx >> 16;
-  r = f * USE_TABLE(log_inv_tbl, indx);
+  r = f * __CLC_USE_TABLE(log_inv_tbl, indx);
 
   // 1/3,  1/2
   float poly = __clc_mad(__clc_mad(r, 0x1.555556p-2f, 0.5f), r * r, r);
 
 #if defined(COMPILING_LOG2)
-  float2 tv = USE_TABLE(log2_tbl, indx);
+  float2 tv = __CLC_USE_TABLE(log2_tbl, indx);
   z1 = tv.s0 + mf;
   z2 = __clc_mad(poly, -LOG2E, tv.s1);
 #elif defined(COMPILING_LOG10)
-  float2 tv = USE_TABLE(log10_tbl, indx);
+  float2 tv = __CLC_USE_TABLE(log10_tbl, indx);
   z1 = __clc_mad(mf, LOG10_2_HEAD, tv.s0);
   z2 = __clc_mad(poly, -LOG10E, mf * LOG10_2_TAIL) + tv.s1;
 #else
-  float2 tv = USE_TABLE(log_tbl, indx);
+  float2 tv = __CLC_USE_TABLE(log_tbl, indx);
   z1 = __clc_mad(mf, LOG2_HEAD, tv.s0);
   z2 = __clc_mad(mf, LOG2_TAIL, -poly) + tv.s1;
 #endif
@@ -261,8 +261,8 @@ __clc_log(double x)
   int index = __clc_as_int2(ux).hi >> 13;
   index = ((0x80 | (index & 0x7e)) >> 1) + (index & 0x1);
 
-  double z1 = USE_TABLE(ln_tbl_lo, index - 64);
-  double q = USE_TABLE(ln_tbl_hi, index - 64);
+  double z1 = __CLC_USE_TABLE(ln_tbl_lo, index - 64);
+  double q = __CLC_USE_TABLE(ln_tbl_hi, index - 64);
 
   double f1 = index * 0x1.0p-7;
   double f2 = f - f1;
diff --git a/libclc/clc/lib/generic/math/clc_modf.inc b/libclc/clc/lib/generic/math/clc_modf.inc
index 962d859ce9f68..4344c65a12bed 100644
--- a/libclc/clc/lib/generic/math/clc_modf.inc
+++ b/libclc/clc/lib/generic/math/clc_modf.inc
@@ -11,7 +11,7 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_modf(__CLC_GENTYPE x,
   return __clc_copysign(__clc_isinf(x) ? __CLC_FP_LIT(0.0) : x - *iptr, x);
 }
 
-#define CLC_MODF_DEF(addrspace)                                                \
+#define __CLC_MODF_DEF(addrspace)                                              \
   _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_modf(                             \
       __CLC_GENTYPE x, addrspace __CLC_GENTYPE *iptr) {                        \
     __CLC_GENTYPE private_iptr;                                                \
@@ -20,11 +20,11 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_modf(__CLC_GENTYPE x,
     return ret;                                                                \
   }
 
-CLC_MODF_DEF(local);
-CLC_MODF_DEF(global);
+__CLC_MODF_DEF(local);
+__CLC_MODF_DEF(global);
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-CLC_MODF_DEF(generic);
+__CLC_MODF_DEF(generic);
 #endif
 
-#undef CLC_MODF_DEF
+#undef __CLC_MODF_DEF
diff --git a/libclc/clc/lib/generic/math/clc_native_cos.cl b/libclc/clc/lib/generic/math/clc_native_cos.cl
index 2e63e9c9493f1..4e9a6c91b4260 100644
--- a/libclc/clc/lib/generic/math/clc_native_cos.cl
+++ b/libclc/clc/lib/generic/math/clc_native_cos.cl
@@ -8,9 +8,9 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_cos
-#define __IMPL_FUNCTION(x) __builtin_elementwise_cos
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_cos
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_divide.cl b/libclc/clc/lib/generic/math/clc_native_divide.cl
index 005089b1ba15d..abec9add4fd62 100644
--- a/libclc/clc/lib/generic/math/clc_native_divide.cl
+++ b/libclc/clc/lib/generic/math/clc_native_divide.cl
@@ -8,7 +8,7 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_native_divide.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_exp.cl b/libclc/clc/lib/generic/math/clc_native_exp.cl
index 2f50a0aefac92..63b4e56913972 100644
--- a/libclc/clc/lib/generic/math/clc_native_exp.cl
+++ b/libclc/clc/lib/generic/math/clc_native_exp.cl
@@ -8,9 +8,9 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_exp
-#define __IMPL_FUNCTION(x) __builtin_elementwise_exp
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_exp
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_exp10.cl b/libclc/clc/lib/generic/math/clc_native_exp10.cl
index 7ca5d1992c61c..2819cd59c4be9 100644
--- a/libclc/clc/lib/generic/math/clc_native_exp10.cl
+++ b/libclc/clc/lib/generic/math/clc_native_exp10.cl
@@ -10,7 +10,7 @@
 #include <clc/internal/clc.h>
 #include <clc/math/clc_native_exp2.h>
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_native_exp10.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_exp2.cl b/libclc/clc/lib/generic/math/clc_native_exp2.cl
index 25f557306c35b..a995b0347cac4 100644
--- a/libclc/clc/lib/generic/math/clc_native_exp2.cl
+++ b/libclc/clc/lib/generic/math/clc_native_exp2.cl
@@ -8,9 +8,9 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_exp2
-#define __IMPL_FUNCTION(x) __builtin_elementwise_exp2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_exp2
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_log.cl b/libclc/clc/lib/generic/math/clc_native_log.cl
index b9b9d274f8b9f..617908277fb6a 100644
--- a/libclc/clc/lib/generic/math/clc_native_log.cl
+++ b/libclc/clc/lib/generic/math/clc_native_log.cl
@@ -8,9 +8,9 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_log
-#define __IMPL_FUNCTION(x) __builtin_elementwise_log
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_log
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_log
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_log10.cl b/libclc/clc/lib/generic/math/clc_native_log10.cl
index 221aa406466c4..0558f8cb7049b 100644
--- a/libclc/clc/lib/generic/math/clc_native_log10.cl
+++ b/libclc/clc/lib/generic/math/clc_native_log10.cl
@@ -8,9 +8,9 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_log10
-#define __IMPL_FUNCTION(x) __builtin_elementwise_log10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_log10
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_log2.cl b/libclc/clc/lib/generic/math/clc_native_log2.cl
index c3008ce242c74..c897ac07acb45 100644
--- a/libclc/clc/lib/generic/math/clc_native_log2.cl
+++ b/libclc/clc/lib/generic/math/clc_native_log2.cl
@@ -8,9 +8,9 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_log2
-#define __IMPL_FUNCTION(x) __builtin_elementwise_log2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_log2
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_powr.cl b/libclc/clc/lib/generic/math/clc_native_powr.cl
index 7c7dcaf8c508e..dac9b7edf78b9 100644
--- a/libclc/clc/lib/generic/math/clc_native_powr.cl
+++ b/libclc/clc/lib/generic/math/clc_native_powr.cl
@@ -10,7 +10,7 @@
 #include <clc/math/clc_native_exp2.h>
 #include <clc/math/clc_native_log2.h>
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_native_powr.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_recip.cl b/libclc/clc/lib/generic/math/clc_native_recip.cl
index 4377f10b1543f..75d832c8467ee 100644
--- a/libclc/clc/lib/generic/math/clc_native_recip.cl
+++ b/libclc/clc/lib/generic/math/clc_native_recip.cl
@@ -8,7 +8,7 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_native_recip.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_rsqrt.cl b/libclc/clc/lib/generic/math/clc_native_rsqrt.cl
index d5e6fcdae491f..9b88e72126f88 100644
--- a/libclc/clc/lib/generic/math/clc_native_rsqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_native_rsqrt.cl
@@ -10,5 +10,5 @@
 #include <clc/math/clc_native_sqrt.h>
 
 #define __CLC_BODY <clc_native_rsqrt.inc>
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_sin.cl b/libclc/clc/lib/generic/math/clc_native_sin.cl
index 533f8d726d41a..941342f387a1a 100644
--- a/libclc/clc/lib/generic/math/clc_native_sin.cl
+++ b/libclc/clc/lib/generic/math/clc_native_sin.cl
@@ -8,9 +8,9 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_sin
-#define __IMPL_FUNCTION(x) __builtin_elementwise_sin
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_sin
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_sqrt.cl b/libclc/clc/lib/generic/math/clc_native_sqrt.cl
index e0f028e988b42..cc18a4b7ba1d3 100644
--- a/libclc/clc/lib/generic/math/clc_native_sqrt.cl
+++ b/libclc/clc/lib/generic/math/clc_native_sqrt.cl
@@ -8,9 +8,9 @@
 
 #include <clc/internal/clc.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_sqrt
-#define __IMPL_FUNCTION(x) __builtin_elementwise_sqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_sqrt
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_native_tan.cl b/libclc/clc/lib/generic/math/clc_native_tan.cl
index a9ac46688c8f3..6b69067be388e 100644
--- a/libclc/clc/lib/generic/math/clc_native_tan.cl
+++ b/libclc/clc/lib/generic/math/clc_native_tan.cl
@@ -10,7 +10,7 @@
 #include <clc/math/clc_native_cos.h>
 #include <clc/math/clc_native_sin.h>
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_native_tan.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_nextafter.cl b/libclc/clc/lib/generic/math/clc_nextafter.cl
index 6fc699ce711a7..40e1b5a2c57f9 100644
--- a/libclc/clc/lib/generic/math/clc_nextafter.cl
+++ b/libclc/clc/lib/generic/math/clc_nextafter.cl
@@ -14,67 +14,69 @@
 // This file provides OpenCL C implementations of __clc_nextafter for
 // targets that don't support the clang builtin.
 
-#define CLC_AS_TYPE(x) __clc_as_##x
+#define __CLC_CLC_AS_TYPE(x) __clc_as_##x
 
-#define NEXTAFTER(FLOAT_TYPE, UINT_TYPE, INT_TYPE, INT_TYPE_SCALAR)            \
+#define __CLC_NEXTAFTER(FLOAT_TYPE, UINT_TYPE, INT_TYPE, INT_TYPE_SCALAR)      \
   _CLC_OVERLOAD _CLC_DEF FLOAT_TYPE __clc_nextafter(FLOAT_TYPE x,              \
                                                     FLOAT_TYPE y) {            \
     const UINT_TYPE sign_bit = (UINT_TYPE)1                                    \
                                << (sizeof(INT_TYPE_SCALAR) * 8 - 1);           \
-    UINT_TYPE ix = CLC_AS_TYPE(UINT_TYPE)(x);                                  \
+    UINT_TYPE ix = __CLC_CLC_AS_TYPE(UINT_TYPE)(x);                            \
     FLOAT_TYPE absx = __clc_fabs(x);                                           \
     UINT_TYPE mxu = sign_bit - ix;                                             \
-    INT_TYPE mx = CLC_AS_TYPE(INT_TYPE)(mxu);                                  \
-    mx = CLC_AS_TYPE(INT_TYPE)(ix) < (INT_TYPE)0 ? mx                          \
-                                                 : CLC_AS_TYPE(INT_TYPE)(ix);  \
-    UINT_TYPE iy = CLC_AS_TYPE(UINT_TYPE)(y);                                  \
+    INT_TYPE mx = __CLC_CLC_AS_TYPE(INT_TYPE)(mxu);                            \
+    mx = __CLC_CLC_AS_TYPE(INT_TYPE)(ix) < (INT_TYPE)0                         \
+             ? mx                                                              \
+             : __CLC_CLC_AS_TYPE(INT_TYPE)(ix);                                \
+    UINT_TYPE iy = __CLC_CLC_AS_TYPE(UINT_TYPE)(y);                            \
     FLOAT_TYPE absy = __clc_fabs(y);                                           \
     UINT_TYPE myu = sign_bit - iy;                                             \
-    INT_TYPE my = CLC_AS_TYPE(INT_TYPE)(myu);                                  \
-    my = CLC_AS_TYPE(INT_TYPE)(iy) < (INT_TYPE)0 ? my                          \
-                                                 : CLC_AS_TYPE(INT_TYPE)(iy);  \
-    INT_TYPE t = mx + (mx < my ? (INT_TYPE)1 : (INT_TYPE)-1);                  \
-    UINT_TYPE r = sign_bit - CLC_AS_TYPE(UINT_TYPE)(t);                        \
+    INT_TYPE my = __CLC_CLC_AS_TYPE(INT_TYPE)(myu);                            \
+    my = __CLC_CLC_AS_TYPE(INT_TYPE)(iy) < (INT_TYPE)0                         \
+             ? my                                                              \
+             : __CLC_CLC_AS_TYPE(INT_TYPE)(iy);                                \
+    INT_TYPE t = mx + (mx < my ? (INT_TYPE)1 : (INT_TYPE) - 1);                \
+    UINT_TYPE r = sign_bit - __CLC_CLC_AS_TYPE(UINT_TYPE)(t);                  \
     r = (t < (INT_TYPE)0 || (t == (INT_TYPE)0 && mx < my))                     \
             ? r                                                                \
-            : CLC_AS_TYPE(UINT_TYPE)(t);                                       \
+            : __CLC_CLC_AS_TYPE(UINT_TYPE)(t);                                 \
     r = __clc_isnan(x) ? ix : r;                                               \
     r = __clc_isnan(y) ? iy : r;                                               \
-    r = ((CLC_AS_TYPE(UINT_TYPE)(absx) | CLC_AS_TYPE(UINT_TYPE)(absy)) ==      \
-             (UINT_TYPE)0 ||                                                   \
+    r = ((__CLC_CLC_AS_TYPE(UINT_TYPE)(absx) |                                 \
+          __CLC_CLC_AS_TYPE(UINT_TYPE)(absy)) == (UINT_TYPE)0 ||               \
          ix == iy)                                                             \
             ? iy                                                               \
             : r;                                                               \
-    return CLC_AS_TYPE(FLOAT_TYPE)(r);                                         \
+    return __CLC_CLC_AS_TYPE(FLOAT_TYPE)(r);                                   \
   }
 
-NEXTAFTER(float, uint, int, int)
-NEXTAFTER(float2, uint2, int2, int)
-NEXTAFTER(float3, uint3, int3, int)
-NEXTAFTER(float4, uint4, int4, int)
-NEXTAFTER(float8, uint8, int8, int)
-NEXTAFTER(float16, uint16, int16, int)
+__CLC_NEXTAFTER(float, uint, int, int)
+__CLC_NEXTAFTER(float2, uint2, int2, int)
+__CLC_NEXTAFTER(float3, uint3, int3, int)
+__CLC_NEXTAFTER(float4, uint4, int4, int)
+__CLC_NEXTAFTER(float8, uint8, int8, int)
+__CLC_NEXTAFTER(float16, uint16, int16, int)
 
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-NEXTAFTER(double, ulong, long, long)
-NEXTAFTER(double2, ulong2, long2, long)
-NEXTAFTER(double3, ulong3, long3, long)
-NEXTAFTER(double4, ulong4, long4, long)
-NEXTAFTER(double8, ulong8, long8, long)
-NEXTAFTER(double16, ulong16, long16, long)
+__CLC_NEXTAFTER(double, ulong, long, long)
+__CLC_NEXTAFTER(double2, ulong2, long2, long)
+__CLC_NEXTAFTER(double3, ulong3, long3, long)
+__CLC_NEXTAFTER(double4, ulong4, long4, long)
+__CLC_NEXTAFTER(double8, ulong8, long8, long)
+__CLC_NEXTAFTER(double16, ulong16, long16, long)
 
 #endif
 
 #ifdef cl_khr_fp16
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
-NEXTAFTER(half, ushort, short, short)
-NEXTAFTER(half2, ushort2, short2, short)
-NEXTAFTER(half3, ushort3, short3, short)
-NEXTAFTER(half4, ushort4, short4, short)
-NEXTAFTER(half8, ushort8, short8, short)
-NEXTAFTER(half16, ushort16, short16, short)
+__CLC_NEXTAFTER(half, ushort, short, short)
+__CLC_NEXTAFTER(half2, ushort2, short2, short)
+__CLC_NEXTAFTER(half3, ushort3, short3, short)
+__CLC_NEXTAFTER(half4, ushort4, short4, short)
+__CLC_NEXTAFTER(half8, ushort8, short8, short)
+__CLC_NEXTAFTER(half16, ushort16, short16, short)
 
 #endif
diff --git a/libclc/clc/lib/generic/math/clc_pow.inc b/libclc/clc/lib/generic/math/clc_pow.inc
index 8b1f820268ba0..35cbcdae8ffff 100644
--- a/libclc/clc/lib/generic/math/clc_pow.inc
+++ b/libclc/clc/lib/generic/math/clc_pow.inc
@@ -103,8 +103,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pow(__CLC_GENTYPE x,
                     __CLC_AS_GENTYPE(0x3f000000 | (ixn & MANTBITS_SP32));
 
   indx = indx >> 16;
-  __CLC_GENTYPE rh = f * USE_TABLE(log_inv_tbl_ep_head, indx);
-  __CLC_GENTYPE rt = f * USE_TABLE(log_inv_tbl_ep_tail, indx);
+  __CLC_GENTYPE rh = f * __CLC_USE_TABLE(log_inv_tbl_ep_head, indx);
+  __CLC_GENTYPE rt = f * __CLC_USE_TABLE(log_inv_tbl_ep_tail, indx);
   r = rh + rt;
 
   poly = __clc_mad(r, __clc_mad(r, 0x1.0p-2f, 0x1.555556p-2f), 0x1.0p-1f) *
@@ -113,8 +113,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pow(__CLC_GENTYPE x,
 
   const __CLC_GENTYPE LOG2_HEAD = 0x1.62e000p-1f;  /* 0.693115234 */
   const __CLC_GENTYPE LOG2_TAIL = 0x1.0bfbe8p-15f; /* 0.0000319461833 */
-  __CLC_GENTYPE logel = USE_TABLE(loge_tbl_lo, indx);
-  __CLC_GENTYPE logeh = USE_TABLE(loge_tbl_hi, indx);
+  __CLC_GENTYPE logel = __CLC_USE_TABLE(loge_tbl_lo, indx);
+  __CLC_GENTYPE logeh = __CLC_USE_TABLE(loge_tbl_hi, indx);
   __CLC_GENTYPE lth = -r;
   __CLC_GENTYPE ltt = __clc_mad(mfn, LOG2_TAIL, -poly) + logeh;
   __CLC_GENTYPE lt = lth + ltt;
@@ -161,8 +161,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pow(__CLC_GENTYPE x,
                              0x1.000000p-1f),
                    r * r, r);
 
-  __CLC_GENTYPE exp_head = USE_TABLE(exp_tbl_ep_head, j);
-  __CLC_GENTYPE exp_tail = USE_TABLE(exp_tbl_ep_tail, j);
+  __CLC_GENTYPE exp_head = __CLC_USE_TABLE(exp_tbl_ep_head, j);
+  __CLC_GENTYPE exp_tail = __CLC_USE_TABLE(exp_tbl_ep_tail, j);
 
   __CLC_GENTYPE expylogx =
       __clc_mad(exp_head, poly, __clc_mad(exp_tail, poly, exp_tail)) + exp_head;
@@ -275,8 +275,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pow(__CLC_GENTYPE x,
     __CLC_GENTYPE F = __CLC_AS_GENTYPE(rax | 0x3FE0000000000000L);
     __CLC_GENTYPE Y = __CLC_AS_GENTYPE(mantissa | 0x3FE0000000000000L);
     __CLC_GENTYPE f = F - Y;
-    __CLC_GENTYPE log_h = USE_TABLE(log_f_inv_tbl_head, index);
-    __CLC_GENTYPE log_t = USE_TABLE(log_f_inv_tbl_tail, index);
+    __CLC_GENTYPE log_h = __CLC_USE_TABLE(log_f_inv_tbl_head, index);
+    __CLC_GENTYPE log_t = __CLC_USE_TABLE(log_f_inv_tbl_tail, index);
     __CLC_GENTYPE f_inv = (log_h + log_t) * f;
     __CLC_GENTYPE r1 =
         __CLC_AS_GENTYPE(__CLC_AS_ULONGN(f_inv) & 0xfffffffff8000000L);
@@ -296,8 +296,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pow(__CLC_GENTYPE x,
     __CLC_GENTYPE poly0t = r1 - poly0h + hr1r1;
     poly = __clc_fma(r1, r2, __clc_fma(0.5 * r2, r2, poly)) + r2 + poly0t;
 
-    log_h = USE_TABLE(powlog_tbl_head, index);
-    log_t = USE_TABLE(powlog_tbl_tail, index);
+    log_h = __CLC_USE_TABLE(powlog_tbl_head, index);
+    log_t = __CLC_USE_TABLE(powlog_tbl_tail, index);
 
     __CLC_GENTYPE resT_t = __clc_fma(xexp, real_log2_tail, +log_t) - poly;
     __CLC_GENTYPE resT = resT_t - poly0h;
@@ -345,8 +345,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pow(__CLC_GENTYPE x,
     __CLC_INTN j = n & 0x0000003f;
     __CLC_INTN m = n >> 6;
 
-    __CLC_GENTYPE f1 = USE_TABLE(two_to_jby64_ep_tbl_head, j);
-    __CLC_GENTYPE f2 = USE_TABLE(two_to_jby64_ep_tbl_tail, j);
+    __CLC_GENTYPE f1 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_head, j);
+    __CLC_GENTYPE f2 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_tail, j);
     __CLC_GENTYPE f = f1 + f2;
 
     __CLC_GENTYPE r1 = __clc_fma(dn, -lnof2_by_64_head, v);
diff --git a/libclc/clc/lib/generic/math/clc_pown.inc b/libclc/clc/lib/generic/math/clc_pown.inc
index 483fd2faf2717..1a681b5e4b397 100644
--- a/libclc/clc/lib/generic/math/clc_pown.inc
+++ b/libclc/clc/lib/generic/math/clc_pown.inc
@@ -104,8 +104,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pown(__CLC_GENTYPE x,
                     __CLC_AS_GENTYPE(0x3f000000 | (ixn & MANTBITS_SP32));
 
   indx = indx >> 16;
-  __CLC_GENTYPE rh = f * USE_TABLE(log_inv_tbl_ep_head, indx);
-  __CLC_GENTYPE rt = f * USE_TABLE(log_inv_tbl_ep_tail, indx);
+  __CLC_GENTYPE rh = f * __CLC_USE_TABLE(log_inv_tbl_ep_head, indx);
+  __CLC_GENTYPE rt = f * __CLC_USE_TABLE(log_inv_tbl_ep_tail, indx);
   r = rh + rt;
 
   poly = __clc_mad(r, __clc_mad(r, 0x1.0p-2f, 0x1.555556p-2f), 0x1.0p-1f) *
@@ -114,8 +114,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pown(__CLC_GENTYPE x,
 
   const __CLC_GENTYPE LOG2_HEAD = 0x1.62e000p-1f;  // 0.693115234
   const __CLC_GENTYPE LOG2_TAIL = 0x1.0bfbe8p-15f; // 0.0000319461833
-  __CLC_GENTYPE logel = USE_TABLE(loge_tbl_lo, indx);
-  __CLC_GENTYPE logeh = USE_TABLE(loge_tbl_hi, indx);
+  __CLC_GENTYPE logel = __CLC_USE_TABLE(loge_tbl_lo, indx);
+  __CLC_GENTYPE logeh = __CLC_USE_TABLE(loge_tbl_hi, indx);
   __CLC_GENTYPE lth = -r;
   __CLC_GENTYPE ltt = __clc_mad(mfn, LOG2_TAIL, -poly) + logeh;
   __CLC_GENTYPE lt = lth + ltt;
@@ -162,8 +162,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pown(__CLC_GENTYPE x,
                              0x1.000000p-1f),
                    r * r, r);
 
-  __CLC_GENTYPE exp_head = USE_TABLE(exp_tbl_ep_head, j);
-  __CLC_GENTYPE exp_tail = USE_TABLE(exp_tbl_ep_tail, j);
+  __CLC_GENTYPE exp_head = __CLC_USE_TABLE(exp_tbl_ep_head, j);
+  __CLC_GENTYPE exp_tail = __CLC_USE_TABLE(exp_tbl_ep_tail, j);
 
   __CLC_GENTYPE expylogx =
       __clc_mad(exp_head, poly, __clc_mad(exp_tail, poly, exp_tail)) + exp_head;
@@ -256,8 +256,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pown(__CLC_GENTYPE x,
     __CLC_GENTYPE F = __CLC_AS_GENTYPE(rax | 0x3FE0000000000000L);
     __CLC_GENTYPE Y = __CLC_AS_GENTYPE(mantissa | 0x3FE0000000000000L);
     __CLC_GENTYPE f = F - Y;
-    __CLC_GENTYPE log_h = USE_TABLE(log_f_inv_tbl_head, index);
-    __CLC_GENTYPE log_t = USE_TABLE(log_f_inv_tbl_tail, index);
+    __CLC_GENTYPE log_h = __CLC_USE_TABLE(log_f_inv_tbl_head, index);
+    __CLC_GENTYPE log_t = __CLC_USE_TABLE(log_f_inv_tbl_tail, index);
     __CLC_GENTYPE f_inv = (log_h + log_t) * f;
     __CLC_GENTYPE r1 =
         __CLC_AS_GENTYPE(__CLC_AS_ULONGN(f_inv) & 0xfffffffff8000000L);
@@ -277,8 +277,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pown(__CLC_GENTYPE x,
     __CLC_GENTYPE poly0t = r1 - poly0h + hr1r1;
     poly = __clc_fma(r1, r2, __clc_fma(0.5 * r2, r2, poly)) + r2 + poly0t;
 
-    log_h = USE_TABLE(powlog_tbl_head, index);
-    log_t = USE_TABLE(powlog_tbl_tail, index);
+    log_h = __CLC_USE_TABLE(powlog_tbl_head, index);
+    log_t = __CLC_USE_TABLE(powlog_tbl_tail, index);
 
     __CLC_GENTYPE resT_t = __clc_fma(xexp, real_log2_tail, +log_t) - poly;
     __CLC_GENTYPE resT = resT_t - poly0h;
@@ -332,8 +332,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_pown(__CLC_GENTYPE x,
     __CLC_INTN j = n & 0x0000003f;
     __CLC_INTN m = n >> 6;
 
-    __CLC_GENTYPE f1 = USE_TABLE(two_to_jby64_ep_tbl_head, j);
-    __CLC_GENTYPE f2 = USE_TABLE(two_to_jby64_ep_tbl_tail, j);
+    __CLC_GENTYPE f1 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_head, j);
+    __CLC_GENTYPE f2 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_tail, j);
     __CLC_GENTYPE f = f1 + f2;
 
     __CLC_GENTYPE r1 = __clc_fma(dn, -lnof2_by_64_head, v);
diff --git a/libclc/clc/lib/generic/math/clc_powr.inc b/libclc/clc/lib/generic/math/clc_powr.inc
index 1244f7f6ac5d6..b94dbfdcbdeb7 100644
--- a/libclc/clc/lib/generic/math/clc_powr.inc
+++ b/libclc/clc/lib/generic/math/clc_powr.inc
@@ -102,8 +102,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_powr(__CLC_GENTYPE x,
                     __CLC_AS_GENTYPE(0x3f000000 | (ixn & MANTBITS_SP32));
 
   indx = indx >> 16;
-  __CLC_GENTYPE rh = f * USE_TABLE(log_inv_tbl_ep_head, indx);
-  __CLC_GENTYPE rt = f * USE_TABLE(log_inv_tbl_ep_tail, indx);
+  __CLC_GENTYPE rh = f * __CLC_USE_TABLE(log_inv_tbl_ep_head, indx);
+  __CLC_GENTYPE rt = f * __CLC_USE_TABLE(log_inv_tbl_ep_tail, indx);
   r = rh + rt;
 
   poly = __clc_mad(r, __clc_mad(r, 0x1.0p-2f, 0x1.555556p-2f), 0x1.0p-1f) *
@@ -112,8 +112,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_powr(__CLC_GENTYPE x,
 
   const __CLC_GENTYPE LOG2_HEAD = 0x1.62e000p-1f;  // 0.693115234
   const __CLC_GENTYPE LOG2_TAIL = 0x1.0bfbe8p-15f; // 0.0000319461833
-  __CLC_GENTYPE logel = USE_TABLE(loge_tbl_lo, indx);
-  __CLC_GENTYPE logeh = USE_TABLE(loge_tbl_hi, indx);
+  __CLC_GENTYPE logel = __CLC_USE_TABLE(loge_tbl_lo, indx);
+  __CLC_GENTYPE logeh = __CLC_USE_TABLE(loge_tbl_hi, indx);
   __CLC_GENTYPE lth = -r;
   __CLC_GENTYPE ltt = __clc_mad(mfn, LOG2_TAIL, -poly) + logeh;
   __CLC_GENTYPE lt = lth + ltt;
@@ -159,8 +159,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_powr(__CLC_GENTYPE x,
                              0x1.000000p-1f),
                    r * r, r);
 
-  __CLC_GENTYPE exp_head = USE_TABLE(exp_tbl_ep_head, j);
-  __CLC_GENTYPE exp_tail = USE_TABLE(exp_tbl_ep_tail, j);
+  __CLC_GENTYPE exp_head = __CLC_USE_TABLE(exp_tbl_ep_head, j);
+  __CLC_GENTYPE exp_tail = __CLC_USE_TABLE(exp_tbl_ep_tail, j);
 
   __CLC_GENTYPE expylogx =
       __clc_mad(exp_head, poly, __clc_mad(exp_tail, poly, exp_tail)) + exp_head;
@@ -261,8 +261,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_powr(__CLC_GENTYPE x,
     __CLC_GENTYPE F = __CLC_AS_GENTYPE(rax | 0x3FE0000000000000L);
     __CLC_GENTYPE Y = __CLC_AS_GENTYPE(mantissa | 0x3FE0000000000000L);
     __CLC_GENTYPE f = F - Y;
-    __CLC_GENTYPE log_h = USE_TABLE(log_f_inv_tbl_head, index);
-    __CLC_GENTYPE log_t = USE_TABLE(log_f_inv_tbl_tail, index);
+    __CLC_GENTYPE log_h = __CLC_USE_TABLE(log_f_inv_tbl_head, index);
+    __CLC_GENTYPE log_t = __CLC_USE_TABLE(log_f_inv_tbl_tail, index);
     __CLC_GENTYPE f_inv = (log_h + log_t) * f;
     __CLC_GENTYPE r1 =
         __CLC_AS_GENTYPE(__CLC_AS_ULONGN(f_inv) & 0xfffffffff8000000L);
@@ -282,8 +282,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_powr(__CLC_GENTYPE x,
     __CLC_GENTYPE poly0t = r1 - poly0h + hr1r1;
     poly = __clc_fma(r1, r2, __clc_fma(0.5 * r2, r2, poly)) + r2 + poly0t;
 
-    log_h = USE_TABLE(powlog_tbl_head, index);
-    log_t = USE_TABLE(powlog_tbl_tail, index);
+    log_h = __CLC_USE_TABLE(powlog_tbl_head, index);
+    log_t = __CLC_USE_TABLE(powlog_tbl_tail, index);
 
     __CLC_GENTYPE resT_t = __clc_fma(xexp, real_log2_tail, +log_t) - poly;
     __CLC_GENTYPE resT = resT_t - poly0h;
@@ -331,8 +331,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_powr(__CLC_GENTYPE x,
     __CLC_INTN j = n & 0x0000003f;
     __CLC_INTN m = n >> 6;
 
-    __CLC_GENTYPE f1 = USE_TABLE(two_to_jby64_ep_tbl_head, j);
-    __CLC_GENTYPE f2 = USE_TABLE(two_to_jby64_ep_tbl_tail, j);
+    __CLC_GENTYPE f1 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_head, j);
+    __CLC_GENTYPE f2 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_tail, j);
     __CLC_GENTYPE f = f1 + f2;
 
     __CLC_GENTYPE r1 = __clc_fma(dn, -lnof2_by_64_head, v);
diff --git a/libclc/clc/lib/generic/math/clc_remainder.cl b/libclc/clc/lib/generic/math/clc_remainder.cl
index 2b3d185a8bee6..488dde73f67a0 100644
--- a/libclc/clc/lib/generic/math/clc_remainder.cl
+++ b/libclc/clc/lib/generic/math/clc_remainder.cl
@@ -74,11 +74,11 @@ _CLC_DEF _CLC_OVERLOAD float __clc_remainder(float x, float y) {
   return xr;
 }
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_remainder
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_remainder
 #define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -212,11 +212,11 @@ _CLC_DEF _CLC_OVERLOAD double __clc_remainder(double x, double y) {
   return ret;
 }
 
-#define __DOUBLE_ONLY
-#define FUNCTION __clc_remainder
+#define __CLC_DOUBLE_ONLY
+#define __CLC_FUNCTION __clc_remainder
 #define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif
 
@@ -225,8 +225,8 @@ _CLC_DEF _CLC_OVERLOAD double __clc_remainder(double x, double y) {
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
 // Forward the half version of this builtin onto the float one
-#define __HALF_ONLY
-#define FUNCTION __clc_remainder
+#define __CLC_HALF_ONLY
+#define __CLC_FUNCTION __clc_remainder
 #define __CLC_BODY <clc/math/binary_def_via_fp32.inc>
 #include <clc/math/gentype.inc>
 
diff --git a/libclc/clc/lib/generic/math/clc_remquo.inc b/libclc/clc/lib/generic/math/clc_remquo.inc
index 28f51c4e2fc23..3a76ffed7f039 100644
--- a/libclc/clc/lib/generic/math/clc_remquo.inc
+++ b/libclc/clc/lib/generic/math/clc_remquo.inc
@@ -74,7 +74,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_remquo(float x, float y,
 }
 
 // remquo signature is special, we don't have macro for this
-#define __VEC_REMQUO(TYPE, VEC_SIZE, HALF_VEC_SIZE)                            \
+#define __CLC_VEC_REMQUO(TYPE, VEC_SIZE, HALF_VEC_SIZE)                        \
   _CLC_DEF _CLC_OVERLOAD TYPE##VEC_SIZE __clc_remquo(                          \
       TYPE##VEC_SIZE x, TYPE##VEC_SIZE y,                                      \
       __CLC_ADDRESS_SPACE int##VEC_SIZE *quo) {                                \
@@ -87,7 +87,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_remquo(float x, float y,
     return ret;                                                                \
   }
 
-#define __VEC3_REMQUO(TYPE)                                                    \
+#define __CLC_VEC3_REMQUO(TYPE)                                                \
   _CLC_DEF _CLC_OVERLOAD TYPE##3 __clc_remquo(                                 \
       TYPE##3 x, TYPE##3 y, __CLC_ADDRESS_SPACE int##3 * quo) {                \
     int2 lo;                                                                   \
@@ -99,11 +99,11 @@ _CLC_DEF _CLC_OVERLOAD float __clc_remquo(float x, float y,
     (*quo).s2 = hi;                                                            \
     return ret;                                                                \
   }
-__VEC_REMQUO(float, 2, )
-__VEC3_REMQUO(float)
-__VEC_REMQUO(float, 4, 2)
-__VEC_REMQUO(float, 8, 4)
-__VEC_REMQUO(float, 16, 8)
+__CLC_VEC_REMQUO(float, 2, )
+__CLC_VEC3_REMQUO(float)
+__CLC_VEC_REMQUO(float, 4, 2)
+__CLC_VEC_REMQUO(float, 8, 4)
+__CLC_VEC_REMQUO(float, 16, 8)
 
 #ifdef cl_khr_fp64
 
@@ -246,11 +246,11 @@ _CLC_DEF _CLC_OVERLOAD double __clc_remquo(double x, double y,
   *pquo = quo;
   return ret;
 }
-__VEC_REMQUO(double, 2, )
-__VEC3_REMQUO(double)
-__VEC_REMQUO(double, 4, 2)
-__VEC_REMQUO(double, 8, 4)
-__VEC_REMQUO(double, 16, 8)
+__CLC_VEC_REMQUO(double, 2, )
+__CLC_VEC3_REMQUO(double)
+__CLC_VEC_REMQUO(double, 4, 2)
+__CLC_VEC_REMQUO(double, 8, 4)
+__CLC_VEC_REMQUO(double, 16, 8)
 
 #endif
 
@@ -262,10 +262,10 @@ _CLC_OVERLOAD _CLC_DEF half __clc_remquo(half x, half y,
                                          __CLC_ADDRESS_SPACE int *pquo) {
   return (half)__clc_remquo((float)x, (float)y, pquo);
 }
-__VEC_REMQUO(half, 2, )
-__VEC3_REMQUO(half)
-__VEC_REMQUO(half, 4, 2)
-__VEC_REMQUO(half, 8, 4)
-__VEC_REMQUO(half, 16, 8)
+__CLC_VEC_REMQUO(half, 2, )
+__CLC_VEC3_REMQUO(half)
+__CLC_VEC_REMQUO(half, 4, 2)
+__CLC_VEC_REMQUO(half, 8, 4)
+__CLC_VEC_REMQUO(half, 16, 8)
 
 #endif
diff --git a/libclc/clc/lib/generic/math/clc_rint.cl b/libclc/clc/lib/generic/math/clc_rint.cl
index b3f94490e3b5a..e3aaae0ffec6f 100644
--- a/libclc/clc/lib/generic/math/clc_rint.cl
+++ b/libclc/clc/lib/generic/math/clc_rint.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_rint
-#define __IMPL_FUNCTION(x) __builtin_elementwise_rint
+#define __CLC_FUNCTION __clc_rint
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_rint
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_rootn.inc b/libclc/clc/lib/generic/math/clc_rootn.inc
index 996f88f145357..fd3d0becb0dff 100644
--- a/libclc/clc/lib/generic/math/clc_rootn.inc
+++ b/libclc/clc/lib/generic/math/clc_rootn.inc
@@ -103,8 +103,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,
                     __CLC_AS_GENTYPE(0x3f000000 | (ixn & MANTBITS_SP32));
 
   indx = indx >> 16;
-  __CLC_GENTYPE rh = f * USE_TABLE(log_inv_tbl_ep_head, indx);
-  __CLC_GENTYPE rt = f * USE_TABLE(log_inv_tbl_ep_tail, indx);
+  __CLC_GENTYPE rh = f * __CLC_USE_TABLE(log_inv_tbl_ep_head, indx);
+  __CLC_GENTYPE rt = f * __CLC_USE_TABLE(log_inv_tbl_ep_tail, indx);
   ;
   r = rh + rt;
 
@@ -116,9 +116,10 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,
   const __CLC_GENTYPE LOG2_TAIL = 0x1.0bfbe8p-15f; // 0.0000319461833
   __CLC_GENTYPE lth = -r;
   __CLC_GENTYPE ltt =
-      __clc_mad(mfn, LOG2_TAIL, -poly) + USE_TABLE(loge_tbl_hi, indx);
+      __clc_mad(mfn, LOG2_TAIL, -poly) + __CLC_USE_TABLE(loge_tbl_hi, indx);
   __CLC_GENTYPE lt = lth + ltt;
-  __CLC_GENTYPE lh = __clc_mad(mfn, LOG2_HEAD, USE_TABLE(loge_tbl_lo, indx));
+  __CLC_GENTYPE lh =
+      __clc_mad(mfn, LOG2_HEAD, __CLC_USE_TABLE(loge_tbl_lo, indx));
   __CLC_GENTYPE l = lh + lt;
 
   // Select near 1 or not
@@ -165,8 +166,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,
                              0x1.000000p-1f),
                    r * r, r);
 
-  __CLC_GENTYPE exph = USE_TABLE(exp_tbl_ep_head, j);
-  __CLC_GENTYPE expt = USE_TABLE(exp_tbl_ep_tail, j);
+  __CLC_GENTYPE exph = __CLC_USE_TABLE(exp_tbl_ep_head, j);
+  __CLC_GENTYPE expt = __CLC_USE_TABLE(exp_tbl_ep_tail, j);
 
   __CLC_GENTYPE expylogx =
       __clc_mad(exph, poly, __clc_mad(expt, poly, expt)) + exph;
@@ -263,8 +264,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,
     __CLC_GENTYPE F = __CLC_AS_GENTYPE(rax | 0x3FE0000000000000L);
     __CLC_GENTYPE Y = __CLC_AS_GENTYPE(mantissa | 0x3FE0000000000000L);
     __CLC_GENTYPE f = F - Y;
-    __CLC_GENTYPE log_h = USE_TABLE(log_f_inv_tbl_head, index);
-    __CLC_GENTYPE log_t = USE_TABLE(log_f_inv_tbl_tail, index);
+    __CLC_GENTYPE log_h = __CLC_USE_TABLE(log_f_inv_tbl_head, index);
+    __CLC_GENTYPE log_t = __CLC_USE_TABLE(log_f_inv_tbl_tail, index);
     __CLC_GENTYPE f_inv = (log_h + log_t) * f;
     __CLC_GENTYPE r1 =
         __CLC_AS_GENTYPE(__CLC_AS_ULONGN(f_inv) & 0xfffffffff8000000L);
@@ -284,8 +285,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,
     __CLC_GENTYPE poly0t = r1 - poly0h + hr1r1;
     poly = __clc_fma(r1, r2, __clc_fma(0.5 * r2, r2, poly)) + r2 + poly0t;
 
-    log_h = USE_TABLE(powlog_tbl_head, index);
-    log_t = USE_TABLE(powlog_tbl_tail, index);
+    log_h = __CLC_USE_TABLE(powlog_tbl_head, index);
+    log_t = __CLC_USE_TABLE(powlog_tbl_tail, index);
 
     __CLC_GENTYPE resT_t = __clc_fma(xexp, real_log2_tail, +log_t) - poly;
     __CLC_GENTYPE resT = resT_t - poly0h;
@@ -338,8 +339,8 @@ _CLC_DEF _CLC_OVERLOAD __CLC_GENTYPE __clc_rootn(__CLC_GENTYPE x,
     __CLC_INTN j = n & 0x0000003f;
     __CLC_INTN m = n >> 6;
 
-    __CLC_GENTYPE f1 = USE_TABLE(two_to_jby64_ep_tbl_head, j);
-    __CLC_GENTYPE f2 = USE_TABLE(two_to_jby64_ep_tbl_tail, j);
+    __CLC_GENTYPE f1 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_head, j);
+    __CLC_GENTYPE f2 = __CLC_USE_TABLE(two_to_jby64_ep_tbl_tail, j);
     __CLC_GENTYPE f = f1 + f2;
 
     __CLC_GENTYPE r1 = __clc_fma(dn, -lnof2_by_64_head, v);
diff --git a/libclc/clc/lib/generic/math/clc_round.cl b/libclc/clc/lib/generic/math/clc_round.cl
index 8c298d8c34835..5ed8d08ba4f45 100644
--- a/libclc/clc/lib/generic/math/clc_round.cl
+++ b/libclc/clc/lib/generic/math/clc_round.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_round
-#define __IMPL_FUNCTION(x) __builtin_elementwise_round
+#define __CLC_FUNCTION __clc_round
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_round
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_sincos_helpers.cl b/libclc/clc/lib/generic/math/clc_sincos_helpers.cl
index 0ea1195fffa70..19705c42f6f07 100644
--- a/libclc/clc/lib/generic/math/clc_sincos_helpers.cl
+++ b/libclc/clc/lib/generic/math/clc_sincos_helpers.cl
@@ -18,16 +18,16 @@
 
 #define bitalign(hi, lo, shift) ((hi) << (32 - (shift))) | ((lo) >> (shift));
 
-#define FULL_MUL(A, B, HI, LO)                                                 \
+#define __CLC_FULL_MUL(A, B, HI, LO)                                           \
   LO = A * B;                                                                  \
   HI = __clc_mul_hi(A, B)
 
-#define FULL_MAD(A, B, C, HI, LO)                                              \
+#define __CLC_FULL_MAD(A, B, C, HI, LO)                                        \
   LO = ((A) * (B) + (C));                                                      \
   HI = __clc_mul_hi(A, B);                                                     \
   HI += LO < C ? 1U : 0U;
 
-#define __FLOAT_ONLY
+#define __CLC_FLOAT_ONLY
 #define __CLC_BODY <clc_sincos_helpers.inc>
 
 #include <clc/math/gentype.inc>
@@ -45,7 +45,7 @@
       ((__CLC_CONVERT_LONGN((src0)) << 32) | __CLC_CONVERT_LONGN((src1))) >>   \
       (((src2) & 3) * 8)))
 
-#define __DOUBLE_ONLY
+#define __CLC_DOUBLE_ONLY
 #define __CLC_BODY <clc_sincos_helpers_fp64.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_sincos_helpers.inc b/libclc/clc/lib/generic/math/clc_sincos_helpers.inc
index 29c742136b7e9..bddc0998cf950 100644
--- a/libclc/clc/lib/generic/math/clc_sincos_helpers.inc
+++ b/libclc/clc/lib/generic/math/clc_sincos_helpers.inc
@@ -175,13 +175,13 @@ _CLC_DEF _CLC_OVERLOAD __CLC_INTN __clc_argReductionLargeS(
 
   __CLC_UINTN p0, p1, p2, p3, p4, p5, p6, p7, c0, c1;
 
-  FULL_MUL(xm, b0, c0, p0);
-  FULL_MAD(xm, b1, c0, c1, p1);
-  FULL_MAD(xm, b2, c1, c0, p2);
-  FULL_MAD(xm, b3, c0, c1, p3);
-  FULL_MAD(xm, b4, c1, c0, p4);
-  FULL_MAD(xm, b5, c0, c1, p5);
-  FULL_MAD(xm, b6, c1, p7, p6);
+  __CLC_FULL_MUL(xm, b0, c0, p0);
+  __CLC_FULL_MAD(xm, b1, c0, c1, p1);
+  __CLC_FULL_MAD(xm, b2, c1, c0, p2);
+  __CLC_FULL_MAD(xm, b3, c0, c1, p3);
+  __CLC_FULL_MAD(xm, b4, c1, c0, p4);
+  __CLC_FULL_MAD(xm, b5, c0, c1, p5);
+  __CLC_FULL_MAD(xm, b6, c1, p7, p6);
 
   __CLC_UINTN fbits = (__CLC_UINTN)224 + (__CLC_UINTN)23 - __CLC_AS_UINTN(xe);
 
diff --git a/libclc/clc/lib/generic/math/clc_sincos_helpers_fp64.inc b/libclc/clc/lib/generic/math/clc_sincos_helpers_fp64.inc
index 9b5776d9b05e9..8fae90c9cc5a5 100644
--- a/libclc/clc/lib/generic/math/clc_sincos_helpers_fp64.inc
+++ b/libclc/clc/lib/generic/math/clc_sincos_helpers_fp64.inc
@@ -67,11 +67,11 @@ __clc_remainder_piby2_large(__CLC_DOUBLEN x, private __CLC_DOUBLEN *r,
 
   // The following extracts 192 consecutive bits of 2/pi aligned on an arbitrary
   // byte boundary
-  __CLC_ULONGN q0 = USE_TABLE(pibits_tbl, j16);
-  __CLC_ULONGN q1 = USE_TABLE(pibits_tbl, (j16 + 8));
-  __CLC_ULONGN q2 = USE_TABLE(pibits_tbl, (j16 + 16));
-  __CLC_ULONGN q3 = USE_TABLE(pibits_tbl, (j16 + 24));
-  __CLC_ULONGN q4 = USE_TABLE(pibits_tbl, (j16 + 32));
+  __CLC_ULONGN q0 = __CLC_USE_TABLE(pibits_tbl, j16);
+  __CLC_ULONGN q1 = __CLC_USE_TABLE(pibits_tbl, (j16 + 8));
+  __CLC_ULONGN q2 = __CLC_USE_TABLE(pibits_tbl, (j16 + 16));
+  __CLC_ULONGN q3 = __CLC_USE_TABLE(pibits_tbl, (j16 + 24));
+  __CLC_ULONGN q4 = __CLC_USE_TABLE(pibits_tbl, (j16 + 32));
 
   __CLC_UINTN q0s0 = __CLC_CONVERT_UINTN(q0);
   __CLC_UINTN q0s1 = __CLC_CONVERT_UINTN(q0 >> 32);
diff --git a/libclc/clc/lib/generic/math/clc_sinh.inc b/libclc/clc/lib/generic/math/clc_sinh.inc
index 799cc32105084..d39059166ffb3 100644
--- a/libclc/clc/lib/generic/math/clc_sinh.inc
+++ b/libclc/clc/lib/generic/math/clc_sinh.inc
@@ -69,8 +69,8 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_sinh(__CLC_GENTYPE x) {
       0.500000000000000005911074e0f);
   cdy = __clc_mad(cdy, dy2, 1.0f);
 
-  __CLC_GENTYPE sinhcoshh = USE_TABLE(sinhcosh_tbl_head, ind);
-  __CLC_GENTYPE sinhcosht = USE_TABLE(sinhcosh_tbl_tail, ind);
+  __CLC_GENTYPE sinhcoshh = __CLC_USE_TABLE(sinhcosh_tbl_head, ind);
+  __CLC_GENTYPE sinhcosht = __CLC_USE_TABLE(sinhcosh_tbl_tail, ind);
   __CLC_GENTYPE z = __clc_mad(sinhcosht, sdy, sinhcoshh * cdy);
   z = __CLC_AS_GENTYPE(xs | __CLC_AS_UINTN(z));
 
@@ -167,10 +167,10 @@ _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __clc_sinh(__CLC_GENTYPE x) {
       __CLC_AS_GENTYPE(__CLC_AS_ULONGN(dy) & 0xfffffffff8000000UL);
   __CLC_GENTYPE sdy2 = sdy + (dy - sdy1);
 
-  __CLC_GENTYPE cl = USE_TABLE(cosh_tbl_head, ind);
-  __CLC_GENTYPE ct = USE_TABLE(cosh_tbl_tail, ind);
-  __CLC_GENTYPE sl = USE_TABLE(sinh_tbl_head, ind);
-  __CLC_GENTYPE st = USE_TABLE(sinh_tbl_tail, ind);
+  __CLC_GENTYPE cl = __CLC_USE_TABLE(cosh_tbl_head, ind);
+  __CLC_GENTYPE ct = __CLC_USE_TABLE(cosh_tbl_tail, ind);
+  __CLC_GENTYPE sl = __CLC_USE_TABLE(sinh_tbl_head, ind);
+  __CLC_GENTYPE st = __CLC_USE_TABLE(sinh_tbl_tail, ind);
 
   __CLC_GENTYPE z =
       __clc_fma(cl, sdy1,
diff --git a/libclc/clc/lib/generic/math/clc_sw_fma.cl b/libclc/clc/lib/generic/math/clc_sw_fma.cl
index ee4734078d698..550ca5e18f3f8 100644
--- a/libclc/clc/lib/generic/math/clc_sw_fma.cl
+++ b/libclc/clc/lib/generic/math/clc_sw_fma.cl
@@ -160,7 +160,7 @@ _CLC_DEF _CLC_OVERLOAD float __clc_sw_fma(float a, float b, float c) {
                         ((uint)st_fma.mantissa & 0x7fffff));
 }
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_sw_fma
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_sw_fma
 #define __CLC_BODY <clc/shared/ternary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_tables.cl b/libclc/clc/lib/generic/math/clc_tables.cl
index 7db00532c8be5..67a17525014db 100644
--- a/libclc/clc/lib/generic/math/clc_tables.cl
+++ b/libclc/clc/lib/generic/math/clc_tables.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 #include <clc/math/tables.h>
 
-DECLARE_TABLE(float, LOG_INV_TBL_EP_HEAD, 129) = {
+__CLC_DECLARE_TABLE(float, LOG_INV_TBL_EP_HEAD, 129) = {
     0x1.000000p+1f, 0x1.fc0000p+0f, 0x1.f80000p+0f, 0x1.f40000p+0f,
     0x1.f00000p+0f, 0x1.ec0000p+0f, 0x1.e80000p+0f, 0x1.e40000p+0f,
     0x1.e00000p+0f, 0x1.de0000p+0f, 0x1.da0000p+0f, 0x1.d60000p+0f,
@@ -45,9 +45,9 @@ DECLARE_TABLE(float, LOG_INV_TBL_EP_HEAD, 129) = {
     0x1.000000p+0f,
 };
 
-CLC_TABLE_FUNCTION(float, LOG_INV_TBL_EP_HEAD, log_inv_tbl_ep_head);
+__CLC_TABLE_FUNCTION_VEC(float, LOG_INV_TBL_EP_HEAD, log_inv_tbl_ep_head);
 
-DECLARE_TABLE(float, LOG_INV_TBL_EP_TAIL, 129) = {
+__CLC_DECLARE_TABLE(float, LOG_INV_TBL_EP_TAIL, 129) = {
     0x0.000000p+0f,  0x1.fc07f0p-14f, 0x1.f81f82p-12f, 0x1.196792p-10f,
     0x1.f07c20p-10f, 0x1.80f660p-9f,  0x1.131ac0p-8f,  0x1.73ac90p-8f,
     0x1.e1e1e2p-8f,  0x1.75b8fep-10f, 0x1.cc0ed8p-9f,  0x1.7b654cp-8f,
@@ -82,9 +82,9 @@ DECLARE_TABLE(float, LOG_INV_TBL_EP_TAIL, 129) = {
     0x1.041042p-12f, 0x1.091b52p-8f,  0x1.020408p-14f, 0x1.010102p-8f,
     0x0.000000p+0f,
 };
-CLC_TABLE_FUNCTION(float, LOG_INV_TBL_EP_TAIL, log_inv_tbl_ep_tail);
+__CLC_TABLE_FUNCTION_VEC(float, LOG_INV_TBL_EP_TAIL, log_inv_tbl_ep_tail);
 
-DECLARE_TABLE(float, LOGE_TBL_LO, 129) = {
+__CLC_DECLARE_TABLE(float, LOGE_TBL_LO, 129) = {
     0x0.000000p+0f, 0x1.fe0000p-8f, 0x1.fc0000p-7f, 0x1.7b8000p-6f,
     0x1.f82000p-6f, 0x1.39e000p-5f, 0x1.774000p-5f, 0x1.b42000p-5f,
     0x1.f0a000p-5f, 0x1.164000p-4f, 0x1.340000p-4f, 0x1.51a000p-4f,
@@ -120,7 +120,7 @@ DECLARE_TABLE(float, LOGE_TBL_LO, 129) = {
     0x1.62e000p-1f,
 };
 
-DECLARE_TABLE(float, LOGE_TBL_HI, 129) = {
+__CLC_DECLARE_TABLE(float, LOGE_TBL_HI, 129) = {
     0x0.000000p+0f,  0x1.535882p-23f, 0x1.5161f8p-20f, 0x1.1b07d4p-18f,
     0x1.361cf0p-19f, 0x1.0f73fcp-18f, 0x1.63d8cap-19f, 0x1.bae232p-18f,
     0x1.86008ap-20f, 0x1.36eea2p-16f, 0x1.d7961ap-16f, 0x1.073f06p-16f,
@@ -156,10 +156,10 @@ DECLARE_TABLE(float, LOGE_TBL_HI, 129) = {
     0x1.0bfbe8p-15f,
 };
 
-CLC_TABLE_FUNCTION(float, LOGE_TBL_LO, loge_tbl_lo);
-CLC_TABLE_FUNCTION(float, LOGE_TBL_HI, loge_tbl_hi);
+__CLC_TABLE_FUNCTION_VEC(float, LOGE_TBL_LO, loge_tbl_lo);
+__CLC_TABLE_FUNCTION_VEC(float, LOGE_TBL_HI, loge_tbl_hi);
 
-DECLARE_TABLE(float, LOG_INV_TBL, 129) = {
+__CLC_DECLARE_TABLE(float, LOG_INV_TBL, 129) = {
     0x1.000000p+1f, 0x1.fc07f0p+0f, 0x1.f81f82p+0f, 0x1.f4465ap+0f,
     0x1.f07c20p+0f, 0x1.ecc07cp+0f, 0x1.e9131ap+0f, 0x1.e573acp+0f,
     0x1.e1e1e2p+0f, 0x1.de5d6ep+0f, 0x1.dae608p+0f, 0x1.d77b66p+0f,
@@ -195,143 +195,142 @@ DECLARE_TABLE(float, LOG_INV_TBL, 129) = {
     0x1.000000p+0f,
 };
 
-CLC_TABLE_FUNCTION(float, LOG_INV_TBL, log_inv_tbl);
-
-DECLARE_TABLE(float2, LOG2_TBL, 129) = {
-    (float2)(0x0.000000p+0f, 0x0.000000p+0f),
-    (float2)(0x1.6f8000p-7f, 0x1.942dbap-17f),
-    (float2)(0x1.6e0000p-6f, 0x1.e5a170p-16f),
-    (float2)(0x1.118000p-5f, 0x1.347544p-15f),
-    (float2)(0x1.6b8000p-5f, 0x1.69bac6p-16f),
-    (float2)(0x1.c48000p-5f, 0x1.7eae42p-15f),
-    (float2)(0x1.0e8000p-4f, 0x1.9c4fd0p-15f),
-    (float2)(0x1.3a8000p-4f, 0x1.17ee92p-15f),
-    (float2)(0x1.660000p-4f, 0x1.fb7d64p-15f),
-    (float2)(0x1.918000p-4f, 0x1.42dc8cp-17f),
-    (float2)(0x1.bc8000p-4f, 0x1.0902b6p-18f),
-    (float2)(0x1.e70000p-4f, 0x1.7608bep-15f),
-    (float2)(0x1.088000p-3f, 0x1.162336p-13f),
-    (float2)(0x1.1d8000p-3f, 0x1.3465d4p-13f),
-    (float2)(0x1.328000p-3f, 0x1.74f13cp-14f),
-    (float2)(0x1.470000p-3f, 0x1.aa7e60p-13f),
-    (float2)(0x1.5c0000p-3f, 0x1.a39fbcp-19f),
-    (float2)(0x1.700000p-3f, 0x1.d0b53ap-13f),
-    (float2)(0x1.848000p-3f, 0x1.0af40ap-13f),
-    (float2)(0x1.988000p-3f, 0x1.b741dep-13f),
-    (float2)(0x1.ac8000p-3f, 0x1.d78b6cp-13f),
-    (float2)(0x1.c08000p-3f, 0x1.6db376p-13f),
-    (float2)(0x1.d48000p-3f, 0x1.ee4c32p-15f),
-    (float2)(0x1.e80000p-3f, 0x1.02f9d2p-13f),
-    (float2)(0x1.fb8000p-3f, 0x1.05ae40p-13f),
-    (float2)(0x1.078000p-2f, 0x1.0adbb0p-14f),
-    (float2)(0x1.110000p-2f, 0x1.83ed68p-13f),
-    (float2)(0x1.1a8000p-2f, 0x1.016ca4p-12f),
-    (float2)(0x1.240000p-2f, 0x1.01eac2p-12f),
-    (float2)(0x1.2d8000p-2f, 0x1.887e26p-13f),
-    (float2)(0x1.370000p-2f, 0x1.24cea4p-14f),
-    (float2)(0x1.400000p-2f, 0x1.918ec6p-12f),
-    (float2)(0x1.498000p-2f, 0x1.3c25e6p-13f),
-    (float2)(0x1.528000p-2f, 0x1.6f7f12p-12f),
-    (float2)(0x1.5c0000p-2f, 0x1.a39fbcp-18f),
-    (float2)(0x1.650000p-2f, 0x1.8fe466p-14f),
-    (float2)(0x1.6e0000p-2f, 0x1.10e6cep-13f),
-    (float2)(0x1.770000p-2f, 0x1.d2ba7ep-14f),
-    (float2)(0x1.800000p-2f, 0x1.4ac62cp-15f),
-    (float2)(0x1.888000p-2f, 0x1.a71cb8p-12f),
-    (float2)(0x1.918000p-2f, 0x1.dd448ep-13f),
-    (float2)(0x1.9a8000p-2f, 0x1.1c8f10p-21f),
-    (float2)(0x1.a30000p-2f, 0x1.bb053ep-13f),
-    (float2)(0x1.ab8000p-2f, 0x1.861e5ep-12f),
-    (float2)(0x1.b40000p-2f, 0x1.fafdcep-12f),
-    (float2)(0x1.bd0000p-2f, 0x1.e5d3cep-15f),
-    (float2)(0x1.c58000p-2f, 0x1.2fad28p-14f),
-    (float2)(0x1.ce0000p-2f, 0x1.492474p-15f),
-    (float2)(0x1.d60000p-2f, 0x1.d4f80cp-12f),
-    (float2)(0x1.de8000p-2f, 0x1.4ff510p-12f),
-    (float2)(0x1.e70000p-2f, 0x1.3550f2p-13f),
-    (float2)(0x1.ef0000p-2f, 0x1.b59ccap-12f),
-    (float2)(0x1.f78000p-2f, 0x1.42b464p-13f),
-    (float2)(0x1.ff8000p-2f, 0x1.5e66a0p-12f),
-    (float2)(0x1.038000p-1f, 0x1.f6a2e4p-11f),
-    (float2)(0x1.080000p-1f, 0x1.39e4fep-14f),
-    (float2)(0x1.0c0000p-1f, 0x1.0500d6p-13f),
-    (float2)(0x1.100000p-1f, 0x1.13b152p-13f),
-    (float2)(0x1.140000p-1f, 0x1.93f542p-14f),
-    (float2)(0x1.180000p-1f, 0x1.467b94p-16f),
-    (float2)(0x1.1b8000p-1f, 0x1.cc47a4p-11f),
-    (float2)(0x1.1f8000p-1f, 0x1.78f4c2p-11f),
-    (float2)(0x1.238000p-1f, 0x1.107508p-11f),
-    (float2)(0x1.278000p-1f, 0x1.2602c2p-12f),
-    (float2)(0x1.2b8000p-1f, 0x1.a39fbcp-20f),
-    (float2)(0x1.2f0000p-1f, 0x1.5a1d7ap-11f),
-    (float2)(0x1.330000p-1f, 0x1.3e355ap-12f),
-    (float2)(0x1.368000p-1f, 0x1.cffedap-11f),
-    (float2)(0x1.3a8000p-1f, 0x1.d9fd50p-12f),
-    (float2)(0x1.3e0000p-1f, 0x1.f64de6p-11f),
-    (float2)(0x1.420000p-1f, 0x1.d83f4cp-12f),
-    (float2)(0x1.458000p-1f, 0x1.cea628p-11f),
-    (float2)(0x1.498000p-1f, 0x1.3c25e6p-12f),
-    (float2)(0x1.4d0000p-1f, 0x1.5a96ccp-11f),
-    (float2)(0x1.510000p-1f, 0x1.18708ap-17f),
-    (float2)(0x1.548000p-1f, 0x1.374652p-12f),
-    (float2)(0x1.580000p-1f, 0x1.2089a6p-11f),
-    (float2)(0x1.5b8000p-1f, 0x1.93432cp-11f),
-    (float2)(0x1.5f0000p-1f, 0x1.f3fd06p-11f),
-    (float2)(0x1.630000p-1f, 0x1.0b8f54p-13f),
-    (float2)(0x1.668000p-1f, 0x1.004722p-12f),
-    (float2)(0x1.6a0000p-1f, 0x1.57cf2cp-12f),
-    (float2)(0x1.6d8000p-1f, 0x1.8cb53ap-12f),
-    (float2)(0x1.710000p-1f, 0x1.9f4d8ap-12f),
-    (float2)(0x1.748000p-1f, 0x1.8feb26p-12f),
-    (float2)(0x1.780000p-1f, 0x1.5edfeep-12f),
-    (float2)(0x1.7b8000p-1f, 0x1.0c7c9ap-12f),
-    (float2)(0x1.7f0000p-1f, 0x1.322182p-13f),
-    (float2)(0x1.828000p-1f, 0x1.3ab7cep-18f),
-    (float2)(0x1.858000p-1f, 0x1.a82c2cp-11f),
-    (float2)(0x1.890000p-1f, 0x1.3dd2c0p-11f),
-    (float2)(0x1.8c8000p-1f, 0x1.871da4p-12f),
-    (float2)(0x1.900000p-1f, 0x1.cc2c00p-14f),
-    (float2)(0x1.930000p-1f, 0x1.9fdb68p-11f),
-    (float2)(0x1.968000p-1f, 0x1.ed6956p-12f),
-    (float2)(0x1.9a0000p-1f, 0x1.f1a760p-14f),
-    (float2)(0x1.9d0000p-1f, 0x1.767f54p-11f),
-    (float2)(0x1.a08000p-1f, 0x1.3f6d26p-12f),
-    (float2)(0x1.a38000p-1f, 0x1.b9fce2p-11f),
-    (float2)(0x1.a70000p-1f, 0x1.8ae816p-12f),
-    (float2)(0x1.aa0000p-1f, 0x1.c23d60p-11f),
-    (float2)(0x1.ad8000p-1f, 0x1.60f388p-12f),
-    (float2)(0x1.b08000p-1f, 0x1.9049aep-11f),
-    (float2)(0x1.b40000p-1f, 0x1.8734a8p-13f),
-    (float2)(0x1.b70000p-1f, 0x1.2523d4p-11f),
-    (float2)(0x1.ba0000p-1f, 0x1.da6ce6p-11f),
-    (float2)(0x1.bd8000p-1f, 0x1.038e62p-12f),
-    (float2)(0x1.c08000p-1f, 0x1.1b511ep-11f),
-    (float2)(0x1.c38000p-1f, 0x1.a728b8p-11f),
-    (float2)(0x1.c70000p-1f, 0x1.2b5d22p-14f),
-    (float2)(0x1.ca0000p-1f, 0x1.2c6e54p-12f),
-    (float2)(0x1.cd0000p-1f, 0x1.f35064p-12f),
-    (float2)(0x1.d00000p-1f, 0x1.4fdb48p-11f),
-    (float2)(0x1.d30000p-1f, 0x1.98ec9ep-11f),
-    (float2)(0x1.d60000p-1f, 0x1.d4f80cp-11f),
-    (float2)(0x1.d98000p-1f, 0x1.0643d6p-17f),
-    (float2)(0x1.dc8000p-1f, 0x1.33567ep-14f),
-    (float2)(0x1.df8000p-1f, 0x1.e0410cp-14f),
-    (float2)(0x1.e28000p-1f, 0x1.142e0ep-13f),
-    (float2)(0x1.e58000p-1f, 0x1.063c88p-13f),
-    (float2)(0x1.e88000p-1f, 0x1.8d66c4p-14f),
-    (float2)(0x1.eb8000p-1f, 0x1.57e32ap-15f),
-    (float2)(0x1.ee0000p-1f, 0x1.ed1c6cp-11f),
-    (float2)(0x1.f10000p-1f, 0x1.b8a076p-11f),
-    (float2)(0x1.f40000p-1f, 0x1.7822f2p-11f),
-    (float2)(0x1.f70000p-1f, 0x1.2bbc3ap-11f),
-    (float2)(0x1.fa0000p-1f, 0x1.a708bap-12f),
-    (float2)(0x1.fd0000p-1f, 0x1.be4c7ep-13f),
-    (float2)(0x1.000000p+0f, 0x0.000000p+0f)
-};
-
-TABLE_FUNCTION(float2, LOG2_TBL, log2_tbl);
-
-DECLARE_TABLE(float2, LOG10_TBL, 129) = {
+__CLC_TABLE_FUNCTION_VEC(float, LOG_INV_TBL, log_inv_tbl);
+
+__CLC_DECLARE_TABLE(float2, LOG2_TBL,
+                    129) = {(float2)(0x0.000000p+0f, 0x0.000000p+0f),
+                            (float2)(0x1.6f8000p-7f, 0x1.942dbap-17f),
+                            (float2)(0x1.6e0000p-6f, 0x1.e5a170p-16f),
+                            (float2)(0x1.118000p-5f, 0x1.347544p-15f),
+                            (float2)(0x1.6b8000p-5f, 0x1.69bac6p-16f),
+                            (float2)(0x1.c48000p-5f, 0x1.7eae42p-15f),
+                            (float2)(0x1.0e8000p-4f, 0x1.9c4fd0p-15f),
+                            (float2)(0x1.3a8000p-4f, 0x1.17ee92p-15f),
+                            (float2)(0x1.660000p-4f, 0x1.fb7d64p-15f),
+                            (float2)(0x1.918000p-4f, 0x1.42dc8cp-17f),
+                            (float2)(0x1.bc8000p-4f, 0x1.0902b6p-18f),
+                            (float2)(0x1.e70000p-4f, 0x1.7608bep-15f),
+                            (float2)(0x1.088000p-3f, 0x1.162336p-13f),
+                            (float2)(0x1.1d8000p-3f, 0x1.3465d4p-13f),
+                            (float2)(0x1.328000p-3f, 0x1.74f13cp-14f),
+                            (float2)(0x1.470000p-3f, 0x1.aa7e60p-13f),
+                            (float2)(0x1.5c0000p-3f, 0x1.a39fbcp-19f),
+                            (float2)(0x1.700000p-3f, 0x1.d0b53ap-13f),
+                            (float2)(0x1.848000p-3f, 0x1.0af40ap-13f),
+                            (float2)(0x1.988000p-3f, 0x1.b741dep-13f),
+                            (float2)(0x1.ac8000p-3f, 0x1.d78b6cp-13f),
+                            (float2)(0x1.c08000p-3f, 0x1.6db376p-13f),
+                            (float2)(0x1.d48000p-3f, 0x1.ee4c32p-15f),
+                            (float2)(0x1.e80000p-3f, 0x1.02f9d2p-13f),
+                            (float2)(0x1.fb8000p-3f, 0x1.05ae40p-13f),
+                            (float2)(0x1.078000p-2f, 0x1.0adbb0p-14f),
+                            (float2)(0x1.110000p-2f, 0x1.83ed68p-13f),
+                            (float2)(0x1.1a8000p-2f, 0x1.016ca4p-12f),
+                            (float2)(0x1.240000p-2f, 0x1.01eac2p-12f),
+                            (float2)(0x1.2d8000p-2f, 0x1.887e26p-13f),
+                            (float2)(0x1.370000p-2f, 0x1.24cea4p-14f),
+                            (float2)(0x1.400000p-2f, 0x1.918ec6p-12f),
+                            (float2)(0x1.498000p-2f, 0x1.3c25e6p-13f),
+                            (float2)(0x1.528000p-2f, 0x1.6f7f12p-12f),
+                            (float2)(0x1.5c0000p-2f, 0x1.a39fbcp-18f),
+                            (float2)(0x1.650000p-2f, 0x1.8fe466p-14f),
+                            (float2)(0x1.6e0000p-2f, 0x1.10e6cep-13f),
+                            (float2)(0x1.770000p-2f, 0x1.d2ba7ep-14f),
+                            (float2)(0x1.800000p-2f, 0x1.4ac62cp-15f),
+                            (float2)(0x1.888000p-2f, 0x1.a71cb8p-12f),
+                            (float2)(0x1.918000p-2f, 0x1.dd448ep-13f),
+                            (float2)(0x1.9a8000p-2f, 0x1.1c8f10p-21f),
+                            (float2)(0x1.a30000p-2f, 0x1.bb053ep-13f),
+                            (float2)(0x1.ab8000p-2f, 0x1.861e5ep-12f),
+                            (float2)(0x1.b40000p-2f, 0x1.fafdcep-12f),
+                            (float2)(0x1.bd0000p-2f, 0x1.e5d3cep-15f),
+                            (float2)(0x1.c58000p-2f, 0x1.2fad28p-14f),
+                            (float2)(0x1.ce0000p-2f, 0x1.492474p-15f),
+                            (float2)(0x1.d60000p-2f, 0x1.d4f80cp-12f),
+                            (float2)(0x1.de8000p-2f, 0x1.4ff510p-12f),
+                            (float2)(0x1.e70000p-2f, 0x1.3550f2p-13f),
+                            (float2)(0x1.ef0000p-2f, 0x1.b59ccap-12f),
+                            (float2)(0x1.f78000p-2f, 0x1.42b464p-13f),
+                            (float2)(0x1.ff8000p-2f, 0x1.5e66a0p-12f),
+                            (float2)(0x1.038000p-1f, 0x1.f6a2e4p-11f),
+                            (float2)(0x1.080000p-1f, 0x1.39e4fep-14f),
+                            (float2)(0x1.0c0000p-1f, 0x1.0500d6p-13f),
+                            (float2)(0x1.100000p-1f, 0x1.13b152p-13f),
+                            (float2)(0x1.140000p-1f, 0x1.93f542p-14f),
+                            (float2)(0x1.180000p-1f, 0x1.467b94p-16f),
+                            (float2)(0x1.1b8000p-1f, 0x1.cc47a4p-11f),
+                            (float2)(0x1.1f8000p-1f, 0x1.78f4c2p-11f),
+                            (float2)(0x1.238000p-1f, 0x1.107508p-11f),
+                            (float2)(0x1.278000p-1f, 0x1.2602c2p-12f),
+                            (float2)(0x1.2b8000p-1f, 0x1.a39fbcp-20f),
+                            (float2)(0x1.2f0000p-1f, 0x1.5a1d7ap-11f),
+                            (float2)(0x1.330000p-1f, 0x1.3e355ap-12f),
+                            (float2)(0x1.368000p-1f, 0x1.cffedap-11f),
+                            (float2)(0x1.3a8000p-1f, 0x1.d9fd50p-12f),
+                            (float2)(0x1.3e0000p-1f, 0x1.f64de6p-11f),
+                            (float2)(0x1.420000p-1f, 0x1.d83f4cp-12f),
+                            (float2)(0x1.458000p-1f, 0x1.cea628p-11f),
+                            (float2)(0x1.498000p-1f, 0x1.3c25e6p-12f),
+                            (float2)(0x1.4d0000p-1f, 0x1.5a96ccp-11f),
+                            (float2)(0x1.510000p-1f, 0x1.18708ap-17f),
+                            (float2)(0x1.548000p-1f, 0x1.374652p-12f),
+                            (float2)(0x1.580000p-1f, 0x1.2089a6p-11f),
+                            (float2)(0x1.5b8000p-1f, 0x1.93432cp-11f),
+                            (float2)(0x1.5f0000p-1f, 0x1.f3fd06p-11f),
+                            (float2)(0x1.630000p-1f, 0x1.0b8f54p-13f),
+                            (float2)(0x1.668000p-1f, 0x1.004722p-12f),
+                            (float2)(0x1.6a0000p-1f, 0x1.57cf2cp-12f),
+                            (float2)(0x1.6d8000p-1f, 0x1.8cb53ap-12f),
+                            (float2)(0x1.710000p-1f, 0x1.9f4d8ap-12f),
+                            (float2)(0x1.748000p-1f, 0x1.8feb26p-12f),
+                            (float2)(0x1.780000p-1f, 0x1.5edfeep-12f),
+                            (float2)(0x1.7b8000p-1f, 0x1.0c7c9ap-12f),
+                            (float2)(0x1.7f0000p-1f, 0x1.322182p-13f),
+                            (float2)(0x1.828000p-1f, 0x1.3ab7cep-18f),
+                            (float2)(0x1.858000p-1f, 0x1.a82c2cp-11f),
+                            (float2)(0x1.890000p-1f, 0x1.3dd2c0p-11f),
+                            (float2)(0x1.8c8000p-1f, 0x1.871da4p-12f),
+                            (float2)(0x1.900000p-1f, 0x1.cc2c00p-14f),
+                            (float2)(0x1.930000p-1f, 0x1.9fdb68p-11f),
+                            (float2)(0x1.968000p-1f, 0x1.ed6956p-12f),
+                            (float2)(0x1.9a0000p-1f, 0x1.f1a760p-14f),
+                            (float2)(0x1.9d0000p-1f, 0x1.767f54p-11f),
+                            (float2)(0x1.a08000p-1f, 0x1.3f6d26p-12f),
+                            (float2)(0x1.a38000p-1f, 0x1.b9fce2p-11f),
+                            (float2)(0x1.a70000p-1f, 0x1.8ae816p-12f),
+                            (float2)(0x1.aa0000p-1f, 0x1.c23d60p-11f),
+                            (float2)(0x1.ad8000p-1f, 0x1.60f388p-12f),
+                            (float2)(0x1.b08000p-1f, 0x1.9049aep-11f),
+                            (float2)(0x1.b40000p-1f, 0x1.8734a8p-13f),
+                            (float2)(0x1.b70000p-1f, 0x1.2523d4p-11f),
+                            (float2)(0x1.ba0000p-1f, 0x1.da6ce6p-11f),
+                            (float2)(0x1.bd8000p-1f, 0x1.038e62p-12f),
+                            (float2)(0x1.c08000p-1f, 0x1.1b511ep-11f),
+                            (float2)(0x1.c38000p-1f, 0x1.a728b8p-11f),
+                            (float2)(0x1.c70000p-1f, 0x1.2b5d22p-14f),
+                            (float2)(0x1.ca0000p-1f, 0x1.2c6e54p-12f),
+                            (float2)(0x1.cd0000p-1f, 0x1.f35064p-12f),
+                            (float2)(0x1.d00000p-1f, 0x1.4fdb48p-11f),
+                            (float2)(0x1.d30000p-1f, 0x1.98ec9ep-11f),
+                            (float2)(0x1.d60000p-1f, 0x1.d4f80cp-11f),
+                            (float2)(0x1.d98000p-1f, 0x1.0643d6p-17f),
+                            (float2)(0x1.dc8000p-1f, 0x1.33567ep-14f),
+                            (float2)(0x1.df8000p-1f, 0x1.e0410cp-14f),
+                            (float2)(0x1.e28000p-1f, 0x1.142e0ep-13f),
+                            (float2)(0x1.e58000p-1f, 0x1.063c88p-13f),
+                            (float2)(0x1.e88000p-1f, 0x1.8d66c4p-14f),
+                            (float2)(0x1.eb8000p-1f, 0x1.57e32ap-15f),
+                            (float2)(0x1.ee0000p-1f, 0x1.ed1c6cp-11f),
+                            (float2)(0x1.f10000p-1f, 0x1.b8a076p-11f),
+                            (float2)(0x1.f40000p-1f, 0x1.7822f2p-11f),
+                            (float2)(0x1.f70000p-1f, 0x1.2bbc3ap-11f),
+                            (float2)(0x1.fa0000p-1f, 0x1.a708bap-12f),
+                            (float2)(0x1.fd0000p-1f, 0x1.be4c7ep-13f),
+                            (float2)(0x1.000000p+0f, 0x0.000000p+0f)};
+
+__CLC_TABLE_FUNCTION(float2, LOG2_TBL, log2_tbl);
+
+__CLC_DECLARE_TABLE(float2, LOG10_TBL, 129) = {
     (float2)(0x0.000000p+0f, 0x0.000000p+0f),
     (float2)(0x1.ba8000p-9f, 0x1.f51c88p-19f),
     (float2)(0x1.b90000p-8f, 0x1.1da93ep-18f),
@@ -463,10 +462,9 @@ DECLARE_TABLE(float2, LOG10_TBL, 129) = {
     (float2)(0x1.340000p-2f, 0x1.04d426p-12f),
 };
 
-TABLE_FUNCTION(float2, LOG10_TBL, log10_tbl);
-
+__CLC_TABLE_FUNCTION(float2, LOG10_TBL, log10_tbl);
 
-DECLARE_TABLE(float, EXP_TBL, 65) = {
+__CLC_DECLARE_TABLE(float, EXP_TBL, 65) = {
     0x1.000000p+0f, 0x1.02c9a4p+0f, 0x1.059b0ep+0f, 0x1.087452p+0f,
     0x1.0b5586p+0f, 0x1.0e3ec4p+0f, 0x1.11301ep+0f, 0x1.1429aap+0f,
     0x1.172b84p+0f, 0x1.1a35bep+0f, 0x1.1d4874p+0f, 0x1.2063b8p+0f,
@@ -486,9 +484,9 @@ DECLARE_TABLE(float, EXP_TBL, 65) = {
     0x1.000000p+1f,
 };
 
-CLC_TABLE_FUNCTION(float, EXP_TBL, exp_tbl);
+__CLC_TABLE_FUNCTION_VEC(float, EXP_TBL, exp_tbl);
 
-DECLARE_TABLE(float, EXP_TBL_EP_HEAD, 65) = {
+__CLC_DECLARE_TABLE(float, EXP_TBL_EP_HEAD, 65) = {
     0x1.000000p+0f, 0x1.02c000p+0f, 0x1.058000p+0f, 0x1.084000p+0f,
     0x1.0b4000p+0f, 0x1.0e0000p+0f, 0x1.110000p+0f, 0x1.140000p+0f,
     0x1.170000p+0f, 0x1.1a0000p+0f, 0x1.1d4000p+0f, 0x1.204000p+0f,
@@ -508,9 +506,9 @@ DECLARE_TABLE(float, EXP_TBL_EP_HEAD, 65) = {
     0x1.000000p+1f,
 };
 
-CLC_TABLE_FUNCTION(float, EXP_TBL_EP_HEAD, exp_tbl_ep_head);
+__CLC_TABLE_FUNCTION_VEC(float, EXP_TBL_EP_HEAD, exp_tbl_ep_head);
 
-DECLARE_TABLE(float, EXP_TBL_EP_TAIL, 65) = {
+__CLC_DECLARE_TABLE(float, EXP_TBL_EP_TAIL, 65) = {
     0x0.000000p+0f,  0x1.347ceep-13f, 0x1.b0d314p-12f, 0x1.a28c3ap-11f,
     0x1.586cf8p-12f, 0x1.f61968p-11f, 0x1.80e808p-11f, 0x1.4d5754p-11f,
     0x1.5c1e3ep-11f, 0x1.adf5b6p-11f, 0x1.0e62d0p-13f, 0x1.1dc430p-11f,
@@ -530,9 +528,9 @@ DECLARE_TABLE(float, EXP_TBL_EP_TAIL, 65) = {
     0x0.000000p+0f,
 };
 
-CLC_TABLE_FUNCTION(float, EXP_TBL_EP_TAIL, exp_tbl_ep_tail);
+__CLC_TABLE_FUNCTION_VEC(float, EXP_TBL_EP_TAIL, exp_tbl_ep_tail);
 
-DECLARE_TABLE(float, CBRT_TBL_HEAD, 129) = {
+__CLC_DECLARE_TABLE(float, CBRT_TBL_HEAD, 129) = {
     0x1.000000p+0f, 0x1.008000p+0f, 0x1.014000p+0f, 0x1.01c000p+0f,
     0x1.028000p+0f, 0x1.034000p+0f, 0x1.03c000p+0f, 0x1.048000p+0f,
     0x1.050000p+0f, 0x1.05c000p+0f, 0x1.068000p+0f, 0x1.070000p+0f,
@@ -568,9 +566,9 @@ DECLARE_TABLE(float, CBRT_TBL_HEAD, 129) = {
     0x1.428000p+0f,
 };
 
-CLC_TABLE_FUNCTION(float, CBRT_TBL_HEAD, cbrt_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(float, CBRT_TBL_HEAD, cbrt_tbl_head);
 
-DECLARE_TABLE(float, CBRT_TBL_TAIL, 129) = {
+__CLC_DECLARE_TABLE(float, CBRT_TBL_TAIL, 129) = {
     0x0.000000p+0f,  0x1.51cb0ap-11f, 0x1.39221ep-12f, 0x1.e06908p-11f,
     0x1.1d6978p-11f, 0x1.4ea1bep-13f, 0x1.833b8ep-11f, 0x1.587002p-12f,
     0x1.ceb290p-11f, 0x1.d57f34p-12f, 0x1.cc53acp-21f, 0x1.0fe098p-11f,
@@ -606,10 +604,10 @@ DECLARE_TABLE(float, CBRT_TBL_TAIL, 129) = {
     0x1.45f31ap-13f,
 };
 
-CLC_TABLE_FUNCTION(float, CBRT_TBL_TAIL, cbrt_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(float, CBRT_TBL_TAIL, cbrt_tbl_tail);
 
 // Tabulated values of sinh(i) and cosh(i) for i = 0,...,36.
-DECLARE_TABLE(float, SINHCOSH_TBL_HEAD, 37) = {
+__CLC_DECLARE_TABLE(float, SINHCOSH_TBL_HEAD, 37) = {
     0x0.000000p+0f,  0x1.2cd9fcp+0f,  0x1.d03cf6p+1f,  0x1.40926ep+3f,
     0x1.b4a380p+4f,  0x1.28d016p+6f,  0x1.936d22p+7f,  0x1.122876p+9f,
     0x1.749ea6p+10f, 0x1.fa7158p+11f, 0x1.5829dcp+13f, 0x1.d3c448p+14f,
@@ -622,9 +620,9 @@ DECLARE_TABLE(float, SINHCOSH_TBL_HEAD, 37) = {
     0x1.ea215ap+50f,
 };
 
-CLC_TABLE_FUNCTION(float, SINHCOSH_TBL_HEAD, sinhcosh_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(float, SINHCOSH_TBL_HEAD, sinhcosh_tbl_head);
 
-DECLARE_TABLE(float, SINHCOSH_TBL_TAIL, 37) = {
+__CLC_DECLARE_TABLE(float, SINHCOSH_TBL_TAIL, 37) = {
     0x1.000000p+0f,  0x1.8b0756p+0f,  0x1.e18fa0p+1f,  0x1.422a4ap+3f,
     0x1.b4ee86p+4f,  0x1.28d6fcp+6f,  0x1.936e68p+7f,  0x1.122894p+9f,
     0x1.749eaap+10f, 0x1.fa7158p+11f, 0x1.5829dep+13f, 0x1.d3c448p+14f,
@@ -637,13 +635,13 @@ DECLARE_TABLE(float, SINHCOSH_TBL_TAIL, 37) = {
     0x1.ea215ap+50f,
 };
 
-CLC_TABLE_FUNCTION(float, SINHCOSH_TBL_TAIL, sinhcosh_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(float, SINHCOSH_TBL_TAIL, sinhcosh_tbl_tail);
 
 #ifdef cl_khr_fp64
 
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-DECLARE_TABLE(double, LN_TBL_LO, 65) = {
+__CLC_DECLARE_TABLE(double, LN_TBL_LO, 65) = {
     0x0.0000000000000p+0, 0x1.fc0a800000000p-7, 0x1.f829800000000p-6,
     0x1.7745800000000p-5, 0x1.f0a3000000000p-5, 0x1.341d700000000p-4,
     0x1.6f0d200000000p-4, 0x1.a926d00000000p-4, 0x1.e270700000000p-4,
@@ -668,9 +666,9 @@ DECLARE_TABLE(double, LN_TBL_LO, 65) = {
     0x1.5ee02a0000000p-1, 0x1.62e42e0000000p-1,
 };
 
-CLC_TABLE_FUNCTION(double, LN_TBL_LO, ln_tbl_lo);
+__CLC_TABLE_FUNCTION_VEC(double, LN_TBL_LO, ln_tbl_lo);
 
-DECLARE_TABLE(double, LN_TBL_HI, 65) = {
+__CLC_DECLARE_TABLE(double, LN_TBL_HI, 65) = {
     0x0.0000000000000p+0,  0x1.61f807c79f3dbp-28, 0x1.873c1980267c8p-25,
     0x1.ec65b9f88c69ep-26, 0x1.8022c54cc2f99p-26, 0x1.2c37a3a125330p-25,
     0x1.15cad69737c93p-25, 0x1.d256ab1b285e9p-27, 0x1.b8abcb97a7aa2p-26,
@@ -695,14 +693,14 @@ DECLARE_TABLE(double, LN_TBL_HI, 65) = {
     0x1.2482ceae1ac12p-26, 0x1.efa39ef35793cp-25,
 };
 
-CLC_TABLE_FUNCTION(double, LN_TBL_HI, ln_tbl_hi);
+__CLC_TABLE_FUNCTION_VEC(double, LN_TBL_HI, ln_tbl_hi);
 
 // Arrays atan_jby256_head and atan_jby256_tail contain leading and trailing
 // parts respectively of precomputed values of atan(j/256), for j = 16, 17, ...,
 // 256. atan_jby256_head contains the first 21 bits of precision, and
 // atan_jby256_tail contains a further 53 bits precision.
 
-DECLARE_TABLE(double, ATAN_JBY256_TBL_HEAD, 241) = {
+__CLC_DECLARE_TABLE(double, ATAN_JBY256_TBL_HEAD, 241) = {
     0x1.ff55b00000000p-5, 0x1.0f99e00000000p-4, 0x1.1f86d00000000p-4,
     0x1.2f71900000000p-4, 0x1.3f59f00000000p-4, 0x1.4f3fd00000000p-4,
     0x1.5f23200000000p-4, 0x1.6f03b00000000p-4, 0x1.7ee1800000000p-4,
@@ -786,9 +784,9 @@ DECLARE_TABLE(double, ATAN_JBY256_TBL_HEAD, 241) = {
     0x1.921fb00000000p-1,
 };
 
-CLC_TABLE_FUNCTION(double, ATAN_JBY256_TBL_HEAD, atan_jby256_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(double, ATAN_JBY256_TBL_HEAD, atan_jby256_tbl_head);
 
-DECLARE_TABLE(double, ATAN_JBY256_TBL_TAIL, 241) = {
+__CLC_DECLARE_TABLE(double, ATAN_JBY256_TBL_TAIL, 241) = {
     0x1.6e59fbd38db2cp-26, 0x1.4e3aa54dedf96p-25, 0x1.7e105ab1bda88p-25,
     0x1.8c5254d013fd0p-27, 0x1.cf8ab3ad62670p-29, 0x1.9dca4bec80468p-26,
     0x1.3f4b5ec98a8dap-26, 0x1.b9d49619d81fep-25, 0x1.3017887460934p-27,
@@ -872,9 +870,9 @@ DECLARE_TABLE(double, ATAN_JBY256_TBL_TAIL, 241) = {
     0x1.5110b4611a626p-23,
 };
 
-CLC_TABLE_FUNCTION(double, ATAN_JBY256_TBL_TAIL, atan_jby256_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(double, ATAN_JBY256_TBL_TAIL, atan_jby256_tbl_tail);
 
-DECLARE_TABLE(double, LOG_F_INV_TBL_HEAD, 258) = {
+__CLC_DECLARE_TABLE(double, LOG_F_INV_TBL_HEAD, 258) = {
     0x1.0000000000000p+1, 0x1.fe00000000000p+0, 0x1.fc00000000000p+0,
     0x1.fa00000000000p+0, 0x1.f800000000000p+0, 0x1.f600000000000p+0,
     0x1.f400000000000p+0, 0x1.f200000000000p+0, 0x1.f000000000000p+0,
@@ -963,9 +961,9 @@ DECLARE_TABLE(double, LOG_F_INV_TBL_HEAD, 258) = {
     0x1.0000000000000p+0, 0x1.0000000000000p+0,
 };
 
-CLC_TABLE_FUNCTION(double, LOG_F_INV_TBL_HEAD, log_f_inv_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(double, LOG_F_INV_TBL_HEAD, log_f_inv_tbl_head);
 
-DECLARE_TABLE(double, LOG_F_INV_TBL_TAIL, 258) = {
+__CLC_DECLARE_TABLE(double, LOG_F_INV_TBL_TAIL, 258) = {
     0x0.0000000000000p+0,  0x1.fe01fe01fe020p-16, 0x1.fc07f01fc07f0p-14,
     0x1.1caa01fa11caap-12, 0x1.f81f81f81f820p-12, 0x1.8856506ddaba6p-11,
     0x1.196792909c560p-10, 0x1.7d9108c2ad433p-10, 0x1.f07c1f07c1f08p-10,
@@ -1054,9 +1052,9 @@ DECLARE_TABLE(double, LOG_F_INV_TBL_TAIL, 258) = {
     0x1.0080402010080p-9,  0x0.0000000000000p+0,
 };
 
-CLC_TABLE_FUNCTION(double, LOG_F_INV_TBL_TAIL, log_f_inv_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(double, LOG_F_INV_TBL_TAIL, log_f_inv_tbl_tail);
 
-DECLARE_TABLE(double, POWLOG_TBL_HEAD, 258) = {
+__CLC_DECLARE_TABLE(double, POWLOG_TBL_HEAD, 258) = {
     0x0.0000000000000p+0, 0x1.ff00aa0000000p-9, 0x1.fe02a60000000p-8,
     0x1.7dc4750000000p-7, 0x1.fc0a8b0000000p-7, 0x1.3cea440000000p-6,
     0x1.7b91b00000000p-6, 0x1.b9fc020000000p-6, 0x1.f829b00000000p-6,
@@ -1145,9 +1143,9 @@ DECLARE_TABLE(double, POWLOG_TBL_HEAD, 258) = {
     0x1.61e3ef0000000p-1, 0x1.62e42e0000000p-1, 0x0.0000000000000p+0,
 };
 
-CLC_TABLE_FUNCTION(double, POWLOG_TBL_HEAD, powlog_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(double, POWLOG_TBL_HEAD, powlog_tbl_head);
 
-DECLARE_TABLE(double, POWLOG_TBL_TAIL, 258) = {
+__CLC_DECLARE_TABLE(double, POWLOG_TBL_TAIL, 258) = {
     0x0.0000000000000p+0,  0x1.5885e0250435ap-36, 0x1.620cf11f86ed2p-33,
     0x1.f0214edba4a25p-32, 0x1.f807c79f3db4ep-36, 0x1.a352ba779a52bp-33,
     0x1.f56c46aa49fd5p-32, 0x1.ebe465fef5196p-32, 0x1.cf0660099f1f8p-31,
@@ -1236,9 +1234,9 @@ DECLARE_TABLE(double, POWLOG_TBL_TAIL, 258) = {
     0x1.b48c8cd2f246cp-26, 0x1.efa39ef35793cp-25, 0x0.0000000000000p+0,
 };
 
-CLC_TABLE_FUNCTION(double, POWLOG_TBL_TAIL, powlog_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(double, POWLOG_TBL_TAIL, powlog_tbl_tail);
 
-DECLARE_TABLE(double, TWO_TO_JBY64_EP_HEAD, 64) = {
+__CLC_DECLARE_TABLE(double, TWO_TO_JBY64_EP_HEAD, 64) = {
     0x1.0000000000000p+0, 0x1.02c9a30000000p+0, 0x1.059b0d0000000p+0,
     0x1.0874510000000p+0, 0x1.0b55860000000p+0, 0x1.0e3ec30000000p+0,
     0x1.11301d0000000p+0, 0x1.1429aa0000000p+0, 0x1.172b830000000p+0,
@@ -1263,9 +1261,10 @@ DECLARE_TABLE(double, TWO_TO_JBY64_EP_HEAD, 64) = {
     0x1.fa7c180000000p+0,
 };
 
-CLC_TABLE_FUNCTION(double, TWO_TO_JBY64_EP_HEAD, two_to_jby64_ep_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(double, TWO_TO_JBY64_EP_HEAD,
+                         two_to_jby64_ep_tbl_head);
 
-DECLARE_TABLE(double, TWO_TO_JBY64_EP_TAIL, 64) = {
+__CLC_DECLARE_TABLE(double, TWO_TO_JBY64_EP_TAIL, 64) = {
     0x0.0000000000000p+0,  0x1.cef00c1dcdef9p-25, 0x1.8ac2ba1d73e2ap-27,
     0x1.0eb37901186bep-25, 0x1.9f3121ec53172p-25, 0x1.69e8d10103a17p-27,
     0x1.25b50a4ebbf1ap-32, 0x1.d525bbf668203p-25, 0x1.8faa2f5b9bef9p-25,
@@ -1290,9 +1289,10 @@ DECLARE_TABLE(double, TWO_TO_JBY64_EP_TAIL, 64) = {
     0x1.9e90d82e90a7ep-28,
 };
 
-CLC_TABLE_FUNCTION(double, TWO_TO_JBY64_EP_TAIL, two_to_jby64_ep_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(double, TWO_TO_JBY64_EP_TAIL,
+                         two_to_jby64_ep_tbl_tail);
 
-DECLARE_TABLE(double, CBRT_INV_TBL, 257) = {
+__CLC_DECLARE_TABLE(double, CBRT_INV_TBL, 257) = {
     0x1.0000000000000p+1, 0x1.fe01fe01fe020p+0, 0x1.fc07f01fc07f0p+0,
     0x1.fa11caa01fa12p+0, 0x1.f81f81f81f820p+0, 0x1.f6310aca0dbb5p+0,
     0x1.f44659e4a4271p+0, 0x1.f25f644230ab5p+0, 0x1.f07c1f07c1f08p+0,
@@ -1381,9 +1381,9 @@ DECLARE_TABLE(double, CBRT_INV_TBL, 257) = {
     0x1.0080402010080p+0, 0x1.0000000000000p+0,
 };
 
-CLC_TABLE_FUNCTION(double, CBRT_INV_TBL, cbrt_inv_tbl);
+__CLC_TABLE_FUNCTION_VEC(double, CBRT_INV_TBL, cbrt_inv_tbl);
 
-DECLARE_TABLE(double, CBRT_DBL_TBL_HEAD, 257) = {
+__CLC_DECLARE_TABLE(double, CBRT_DBL_TBL_HEAD, 257) = {
     0x1.0000000000000p+0, 0x1.0055380000000p+0, 0x1.00aa390000000p+0,
     0x1.00ff010000000p+0, 0x1.0153920000000p+0, 0x1.01a7eb0000000p+0,
     0x1.01fc0d0000000p+0, 0x1.024ff80000000p+0, 0x1.02a3ad0000000p+0,
@@ -1472,9 +1472,9 @@ DECLARE_TABLE(double, CBRT_DBL_TBL_HEAD, 257) = {
     0x1.4254640000000p+0, 0x1.428a2f0000000p+0,
 };
 
-CLC_TABLE_FUNCTION(double, CBRT_DBL_TBL_HEAD, cbrt_dbl_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(double, CBRT_DBL_TBL_HEAD, cbrt_dbl_tbl_head);
 
-DECLARE_TABLE(double, CBRT_DBL_TBL_TAIL, 257) = {
+__CLC_DECLARE_TABLE(double, CBRT_DBL_TBL_TAIL, 257) = {
     0x0.0000000000000p+0,  0x1.e6a24c81e4294p-25, 0x1.8548511e3a785p-26,
     0x1.4eb9336ec07f6p-25, 0x1.0ea64b8b750e1p-27, 0x1.61637cff8a53cp-27,
     0x1.0733bf7bd1943p-27, 0x1.666911345ccedp-26, 0x1.77b7a3f592f14p-27,
@@ -1563,23 +1563,23 @@ DECLARE_TABLE(double, CBRT_DBL_TBL_TAIL, 257) = {
     0x1.dc43f1ed210b4p-25, 0x1.31ae515c447bbp-25,
 };
 
-CLC_TABLE_FUNCTION(double, CBRT_DBL_TBL_TAIL, cbrt_dbl_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(double, CBRT_DBL_TBL_TAIL, cbrt_dbl_tbl_tail);
 
-DECLARE_TABLE(double, CBRT_REM_TBL_HEAD, 5) = {
+__CLC_DECLARE_TABLE(double, CBRT_REM_TBL_HEAD, 5) = {
     0x1.428a2f0000000p-1, 0x1.965fea0000000p-1, 0x1.0000000000000p+0,
     0x1.428a2f0000000p+0, 0x1.965fea0000000p+0,
 };
 
-CLC_TABLE_FUNCTION(double, CBRT_REM_TBL_HEAD, cbrt_rem_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(double, CBRT_REM_TBL_HEAD, cbrt_rem_tbl_head);
 
-DECLARE_TABLE(double, CBRT_REM_TBL_TAIL, 5) = {
+__CLC_DECLARE_TABLE(double, CBRT_REM_TBL_TAIL, 5) = {
     0x1.31ae515c447bbp-26, 0x1.4f5b8f20ac166p-27, 0x0.0000000000000p+0,
     0x1.31ae515c447bbp-25, 0x1.4f5b8f20ac166p-26,
 };
 
-CLC_TABLE_FUNCTION(double, CBRT_REM_TBL_TAIL, cbrt_rem_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(double, CBRT_REM_TBL_TAIL, cbrt_rem_tbl_tail);
 
-DECLARE_TABLE(double, SINH_TBL_HEAD, 37) = {
+__CLC_DECLARE_TABLE(double, SINH_TBL_HEAD, 37) = {
     0x0.0000000000000p+0,  0x1.2cd9fc0000000p+0,  0x1.d03cf60000000p+1,
     0x1.40926e0000000p+3,  0x1.b4a3800000000p+4,  0x1.28d0160000000p+6,
     0x1.936d228000000p+7,  0x1.1228768000000p+9,  0x1.749ea50000000p+10,
@@ -1595,7 +1595,7 @@ DECLARE_TABLE(double, SINH_TBL_HEAD, 37) = {
     0x1.ea215a0000000p+50,
 };
 
-DECLARE_TABLE(double, SINH_TBL_TAIL, 37) = {
+__CLC_DECLARE_TABLE(double, SINH_TBL_TAIL, 37) = {
     0x0.0000000000000p+0,  0x1.13ae6096a0092p-26, 0x1.db70cfb79a640p-26,
     0x1.c2526b66dc067p-23, 0x1.b81b18647f380p-23, 0x1.bc1cdd1e1eb08p-20,
     0x1.d9f201534fb09p-19, 0x1.d1c064a4e9954p-18, 0x1.4eca65d06ea74p-18,
@@ -1611,7 +1611,7 @@ DECLARE_TABLE(double, SINH_TBL_TAIL, 37) = {
     0x1.d20d76744835cp+22,
 };
 
-DECLARE_TABLE(double, COSH_TBL_HEAD, 37) = {
+__CLC_DECLARE_TABLE(double, COSH_TBL_HEAD, 37) = {
     0x1.0000000000000p+0,  0x1.8b07550000000p+0,  0x1.e18fa08000000p+1,
     0x1.422a490000000p+3,  0x1.b4ee858000000p+4,  0x1.28d6fc8000000p+6,
     0x1.936e678000000p+7,  0x1.1228948000000p+9,  0x1.749eaa8000000p+10,
@@ -1627,7 +1627,7 @@ DECLARE_TABLE(double, COSH_TBL_HEAD, 37) = {
     0x1.ea215a0000000p+50,
 };
 
-DECLARE_TABLE(double, COSH_TBL_TAIL, 37) = {
+__CLC_DECLARE_TABLE(double, COSH_TBL_TAIL, 37) = {
     0x0.0000000000000p+0,  0x1.d9f5504c2bd28p-28, 0x1.7cb66f0a4c9fdp-25,
     0x1.f58617928e588p-23, 0x1.bc7d000c38d48p-25, 0x1.f7f9d4e329998p-21,
     0x1.6e6e464885269p-19, 0x1.ba3a8b946c154p-19, 0x1.3f4e76110d5a4p-18,
@@ -1643,12 +1643,12 @@ DECLARE_TABLE(double, COSH_TBL_TAIL, 37) = {
     0x1.d20d76744835cp+22,
 };
 
-CLC_TABLE_FUNCTION(double, SINH_TBL_HEAD, sinh_tbl_head);
-CLC_TABLE_FUNCTION(double, SINH_TBL_TAIL, sinh_tbl_tail);
-CLC_TABLE_FUNCTION(double, COSH_TBL_HEAD, cosh_tbl_head);
-CLC_TABLE_FUNCTION(double, COSH_TBL_TAIL, cosh_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(double, SINH_TBL_HEAD, sinh_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(double, SINH_TBL_TAIL, sinh_tbl_tail);
+__CLC_TABLE_FUNCTION_VEC(double, COSH_TBL_HEAD, cosh_tbl_head);
+__CLC_TABLE_FUNCTION_VEC(double, COSH_TBL_TAIL, cosh_tbl_tail);
 
-DECLARE_TABLE(uchar, PIBITS_TBL, ) = {
+__CLC_DECLARE_TABLE(uchar, PIBITS_TBL, ) = {
     224, 241, 27,  193, 12,  88,  33,  116, 53,  126, 196, 126, 237, 175, 169,
     75,  74,  41,  222, 231, 28,  244, 236, 197, 151, 175, 31,  235, 158, 212,
     181, 168, 127, 121, 154, 253, 24,  61,  221, 38,  44,  159, 60,  251, 217,
@@ -1661,25 +1661,25 @@ DECLARE_TABLE(uchar, PIBITS_TBL, ) = {
     119, 211, 212, 71,  95,  157, 240, 167, 84,  16,  57,  185, 13,  230, 139,
     2,   0,   0,   0,   0,   0,   0,   0};
 
-_CLC_DEF _CLC_OVERLOAD ulong TABLE_MANGLE(pibits_tbl)(int idx) {
+_CLC_DEF _CLC_OVERLOAD ulong __CLC_TABLE_MANGLE(pibits_tbl)(int idx) {
   return *(__constant ulong *)(PIBITS_TBL + idx);
 }
-_CLC_DEF _CLC_OVERLOAD ulong2 TABLE_MANGLE(pibits_tbl)(int2 idx) {
+_CLC_DEF _CLC_OVERLOAD ulong2 __CLC_TABLE_MANGLE(pibits_tbl)(int2 idx) {
   return (ulong2){*(__constant ulong *)(PIBITS_TBL + idx.s0),
                   *(__constant ulong *)(PIBITS_TBL + idx.s1)};
 }
-_CLC_DEF _CLC_OVERLOAD ulong3 TABLE_MANGLE(pibits_tbl)(int3 idx) {
+_CLC_DEF _CLC_OVERLOAD ulong3 __CLC_TABLE_MANGLE(pibits_tbl)(int3 idx) {
   return (ulong3){*(__constant ulong *)(PIBITS_TBL + idx.s0),
                   *(__constant ulong *)(PIBITS_TBL + idx.s1),
                   *(__constant ulong *)(PIBITS_TBL + idx.s2)};
 }
-_CLC_DEF _CLC_OVERLOAD ulong4 TABLE_MANGLE(pibits_tbl)(int4 idx) {
+_CLC_DEF _CLC_OVERLOAD ulong4 __CLC_TABLE_MANGLE(pibits_tbl)(int4 idx) {
   return (ulong4){*(__constant ulong *)(PIBITS_TBL + idx.s0),
                   *(__constant ulong *)(PIBITS_TBL + idx.s1),
                   *(__constant ulong *)(PIBITS_TBL + idx.s2),
                   *(__constant ulong *)(PIBITS_TBL + idx.s3)};
 }
-_CLC_DEF _CLC_OVERLOAD ulong8 TABLE_MANGLE(pibits_tbl)(int8 idx) {
+_CLC_DEF _CLC_OVERLOAD ulong8 __CLC_TABLE_MANGLE(pibits_tbl)(int8 idx) {
   return (ulong8){*(__constant ulong *)(PIBITS_TBL + idx.s0),
                   *(__constant ulong *)(PIBITS_TBL + idx.s1),
                   *(__constant ulong *)(PIBITS_TBL + idx.s2),
@@ -1689,7 +1689,7 @@ _CLC_DEF _CLC_OVERLOAD ulong8 TABLE_MANGLE(pibits_tbl)(int8 idx) {
                   *(__constant ulong *)(PIBITS_TBL + idx.s6),
                   *(__constant ulong *)(PIBITS_TBL + idx.s7)};
 }
-_CLC_DEF _CLC_OVERLOAD ulong16 TABLE_MANGLE(pibits_tbl)(int16 idx) {
+_CLC_DEF _CLC_OVERLOAD ulong16 __CLC_TABLE_MANGLE(pibits_tbl)(int16 idx) {
   return (ulong16){*(__constant ulong *)(PIBITS_TBL + idx.s0),
                    *(__constant ulong *)(PIBITS_TBL + idx.s1),
                    *(__constant ulong *)(PIBITS_TBL + idx.s2),
diff --git a/libclc/clc/lib/generic/math/clc_tgamma.cl b/libclc/clc/lib/generic/math/clc_tgamma.cl
index 83b09cc33ecc2..c379cd16b5a62 100644
--- a/libclc/clc/lib/generic/math/clc_tgamma.cl
+++ b/libclc/clc/lib/generic/math/clc_tgamma.cl
@@ -65,6 +65,6 @@ _CLC_OVERLOAD _CLC_DEF half __clc_tgamma(half x) {
 
 #endif
 
-#define FUNCTION __clc_tgamma
+#define __CLC_FUNCTION __clc_tgamma
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/math/clc_trunc.cl b/libclc/clc/lib/generic/math/clc_trunc.cl
index 6261d04cd26de..d9c54a1888d94 100644
--- a/libclc/clc/lib/generic/math/clc_trunc.cl
+++ b/libclc/clc/lib/generic/math/clc_trunc.cl
@@ -8,8 +8,8 @@
 
 #include <clc/internal/clc.h>
 
-#define FUNCTION __clc_trunc
-#define __IMPL_FUNCTION(x) __builtin_elementwise_trunc
+#define __CLC_FUNCTION __clc_trunc
+#define __CLC_IMPL_FUNCTION(x) __builtin_elementwise_trunc
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/generic/relational/clc_bitselect.cl b/libclc/clc/lib/generic/relational/clc_bitselect.cl
index 2976b8947e544..9c6e82e6be4be 100644
--- a/libclc/clc/lib/generic/relational/clc_bitselect.cl
+++ b/libclc/clc/lib/generic/relational/clc_bitselect.cl
@@ -12,7 +12,7 @@
 #define __CLC_BODY <clc_bitselect.inc>
 #include <clc/integer/gentype.inc>
 
-#define FLOAT_BITSELECT(f_type, i_type, width)                                 \
+#define __CLC_FLOAT_BITSELECT(f_type, i_type, width)                           \
   _CLC_OVERLOAD _CLC_DEF f_type##width __clc_bitselect(                        \
       f_type##width x, f_type##width y, f_type##width z) {                     \
     return __clc_as_##f_type##width(__clc_bitselect(                           \
@@ -20,33 +20,33 @@
         __clc_as_##i_type##width(z)));                                         \
   }
 
-FLOAT_BITSELECT(float, uint, )
-FLOAT_BITSELECT(float, uint, 2)
-FLOAT_BITSELECT(float, uint, 3)
-FLOAT_BITSELECT(float, uint, 4)
-FLOAT_BITSELECT(float, uint, 8)
-FLOAT_BITSELECT(float, uint, 16)
+__CLC_FLOAT_BITSELECT(float, uint, )
+__CLC_FLOAT_BITSELECT(float, uint, 2)
+__CLC_FLOAT_BITSELECT(float, uint, 3)
+__CLC_FLOAT_BITSELECT(float, uint, 4)
+__CLC_FLOAT_BITSELECT(float, uint, 8)
+__CLC_FLOAT_BITSELECT(float, uint, 16)
 
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
 
-FLOAT_BITSELECT(double, ulong, )
-FLOAT_BITSELECT(double, ulong, 2)
-FLOAT_BITSELECT(double, ulong, 3)
-FLOAT_BITSELECT(double, ulong, 4)
-FLOAT_BITSELECT(double, ulong, 8)
-FLOAT_BITSELECT(double, ulong, 16)
+__CLC_FLOAT_BITSELECT(double, ulong, )
+__CLC_FLOAT_BITSELECT(double, ulong, 2)
+__CLC_FLOAT_BITSELECT(double, ulong, 3)
+__CLC_FLOAT_BITSELECT(double, ulong, 4)
+__CLC_FLOAT_BITSELECT(double, ulong, 8)
+__CLC_FLOAT_BITSELECT(double, ulong, 16)
 
 #endif
 
 #ifdef cl_khr_fp16
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
 
-FLOAT_BITSELECT(half, ushort, )
-FLOAT_BITSELECT(half, ushort, 2)
-FLOAT_BITSELECT(half, ushort, 3)
-FLOAT_BITSELECT(half, ushort, 4)
-FLOAT_BITSELECT(half, ushort, 8)
-FLOAT_BITSELECT(half, ushort, 16)
+__CLC_FLOAT_BITSELECT(half, ushort, )
+__CLC_FLOAT_BITSELECT(half, ushort, 2)
+__CLC_FLOAT_BITSELECT(half, ushort, 3)
+__CLC_FLOAT_BITSELECT(half, ushort, 4)
+__CLC_FLOAT_BITSELECT(half, ushort, 8)
+__CLC_FLOAT_BITSELECT(half, ushort, 16)
 
 #endif
diff --git a/libclc/clc/lib/generic/relational/clc_signbit.cl b/libclc/clc/lib/generic/relational/clc_signbit.cl
index d8736a72b394f..cb7000ddb0757 100644
--- a/libclc/clc/lib/generic/relational/clc_signbit.cl
+++ b/libclc/clc/lib/generic/relational/clc_signbit.cl
@@ -9,50 +9,54 @@
 #include <clc/internal/clc.h>
 #include <clc/relational/relational.h>
 
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE, FUNCTION, ARG_TYPE)        \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.lo), FUNCTION(x.hi)} !=            \
-                      (RET_TYPE)0);                                            \
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE, __CLC_FUNCTION, ARG_TYPE)  \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE __CLC_FUNCTION(ARG_TYPE x) {                 \
+    return (RET_TYPE)((RET_TYPE){__CLC_FUNCTION(x.lo),                         \
+                                 __CLC_FUNCTION(x.hi)} != (RET_TYPE)0);        \
   }
 
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE, FUNCTION, ARG_TYPE)        \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1),               \
-                                 FUNCTION(x.s2)} != (RET_TYPE)0);              \
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE, __CLC_FUNCTION, ARG_TYPE)  \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE __CLC_FUNCTION(ARG_TYPE x) {                 \
+    return (RET_TYPE)((RET_TYPE){__CLC_FUNCTION(x.s0), __CLC_FUNCTION(x.s1),   \
+                                 __CLC_FUNCTION(x.s2)} != (RET_TYPE)0);        \
   }
 
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE, FUNCTION, ARG_TYPE)        \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1),               \
-                                 FUNCTION(x.s2),                               \
-                                 FUNCTION(x.s3)} != (RET_TYPE)0);              \
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE, __CLC_FUNCTION, ARG_TYPE)  \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE __CLC_FUNCTION(ARG_TYPE x) {                 \
+    return (RET_TYPE)((RET_TYPE){__CLC_FUNCTION(x.s0), __CLC_FUNCTION(x.s1),   \
+                                 __CLC_FUNCTION(x.s2),                         \
+                                 __CLC_FUNCTION(x.s3)} != (RET_TYPE)0);        \
   }
 
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE, FUNCTION, ARG_TYPE)        \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (                                                                   \
-        RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),   \
-                             FUNCTION(x.s3), FUNCTION(x.s4), FUNCTION(x.s5),   \
-                             FUNCTION(x.s6), FUNCTION(x.s7)} != (RET_TYPE)0);  \
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE, __CLC_FUNCTION, ARG_TYPE)  \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE __CLC_FUNCTION(ARG_TYPE x) {                 \
+    return (RET_TYPE)((RET_TYPE){__CLC_FUNCTION(x.s0), __CLC_FUNCTION(x.s1),   \
+                                 __CLC_FUNCTION(x.s2), __CLC_FUNCTION(x.s3),   \
+                                 __CLC_FUNCTION(x.s4), __CLC_FUNCTION(x.s5),   \
+                                 __CLC_FUNCTION(x.s6),                         \
+                                 __CLC_FUNCTION(x.s7)} != (RET_TYPE)0);        \
   }
 
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE, FUNCTION, ARG_TYPE)       \
-  _CLC_DEF _CLC_OVERLOAD RET_TYPE FUNCTION(ARG_TYPE x) {                       \
-    return (                                                                   \
-        RET_TYPE)((RET_TYPE){FUNCTION(x.s0), FUNCTION(x.s1), FUNCTION(x.s2),   \
-                             FUNCTION(x.s3), FUNCTION(x.s4), FUNCTION(x.s5),   \
-                             FUNCTION(x.s6), FUNCTION(x.s7), FUNCTION(x.s8),   \
-                             FUNCTION(x.s9), FUNCTION(x.sa), FUNCTION(x.sb),   \
-                             FUNCTION(x.sc), FUNCTION(x.sd), FUNCTION(x.se),   \
-                             FUNCTION(x.sf)} != (RET_TYPE)0);                  \
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE, __CLC_FUNCTION, ARG_TYPE) \
+  _CLC_DEF _CLC_OVERLOAD RET_TYPE __CLC_FUNCTION(ARG_TYPE x) {                 \
+    return (RET_TYPE)((RET_TYPE){__CLC_FUNCTION(x.s0), __CLC_FUNCTION(x.s1),   \
+                                 __CLC_FUNCTION(x.s2), __CLC_FUNCTION(x.s3),   \
+                                 __CLC_FUNCTION(x.s4), __CLC_FUNCTION(x.s5),   \
+                                 __CLC_FUNCTION(x.s6), __CLC_FUNCTION(x.s7),   \
+                                 __CLC_FUNCTION(x.s8), __CLC_FUNCTION(x.s9),   \
+                                 __CLC_FUNCTION(x.sa), __CLC_FUNCTION(x.sb),   \
+                                 __CLC_FUNCTION(x.sc), __CLC_FUNCTION(x.sd),   \
+                                 __CLC_FUNCTION(x.se),                         \
+                                 __CLC_FUNCTION(x.sf)} != (RET_TYPE)0);        \
   }
 
-#define _CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(RET_TYPE, FUNCTION, ARG_TYPE)     \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE##2, FUNCTION, ARG_TYPE##2)        \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE##3, FUNCTION, ARG_TYPE##3)        \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE##4, FUNCTION, ARG_TYPE##4)        \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE##8, FUNCTION, ARG_TYPE##8)        \
-  _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE##16, FUNCTION, ARG_TYPE##16)
+#define _CLC_DEFINE_RELATIONAL_UNARY_VEC_ALL(RET_TYPE, __CLC_FUNCTION,         \
+                                             ARG_TYPE)                         \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC2(RET_TYPE##2, __CLC_FUNCTION, ARG_TYPE##2)  \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC3(RET_TYPE##3, __CLC_FUNCTION, ARG_TYPE##3)  \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC4(RET_TYPE##4, __CLC_FUNCTION, ARG_TYPE##4)  \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC8(RET_TYPE##8, __CLC_FUNCTION, ARG_TYPE##8)  \
+  _CLC_DEFINE_RELATIONAL_UNARY_VEC16(RET_TYPE##16, __CLC_FUNCTION, ARG_TYPE##16)
 
 _CLC_DEF _CLC_OVERLOAD int __clc_signbit(float x) {
   return __builtin_signbitf(x);
diff --git a/libclc/clc/lib/generic/shared/clc_vload.cl b/libclc/clc/lib/generic/shared/clc_vload.cl
index e4003e4a96736..5942f10d2de5b 100644
--- a/libclc/clc/lib/generic/shared/clc_vload.cl
+++ b/libclc/clc/lib/generic/shared/clc_vload.cl
@@ -9,7 +9,7 @@
 #include <clc/internal/clc.h>
 #include <clc/shared/clc_vload.h>
 
-#define VLOAD_VECTORIZE(PRIM_TYPE, ADDR_SPACE)                                 \
+#define __CLC_VLOAD_VECTORIZE(PRIM_TYPE, ADDR_SPACE)                           \
   _CLC_OVERLOAD _CLC_DEF PRIM_TYPE##2 __clc_vload2(                            \
       size_t offset, const ADDR_SPACE PRIM_TYPE *x) {                          \
     return *(                                                                  \
@@ -42,89 +42,89 @@
   }
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-#define VLOAD_VECTORIZE_GENERIC VLOAD_VECTORIZE
+#define __CLC_VLOAD_VECTORIZE_GENERIC __CLC_VLOAD_VECTORIZE
 #else
 // The generic address space isn't available, so make the macro do nothing
-#define VLOAD_VECTORIZE_GENERIC(X, Y)
+#define __CLC_VLOAD_VECTORIZE_GENERIC(X, Y)
 #endif
 
-#define VLOAD_ADDR_SPACES(__CLC_SCALAR_GENTYPE)                                \
-  VLOAD_VECTORIZE(__CLC_SCALAR_GENTYPE, __private)                             \
-  VLOAD_VECTORIZE(__CLC_SCALAR_GENTYPE, __local)                               \
-  VLOAD_VECTORIZE(__CLC_SCALAR_GENTYPE, __constant)                            \
-  VLOAD_VECTORIZE(__CLC_SCALAR_GENTYPE, __global)                              \
-  VLOAD_VECTORIZE_GENERIC(__CLC_SCALAR_GENTYPE, __generic)
+#define __CLC_VLOAD_ADDR_SPACES(__CLC_SCALAR_GENTYPE)                          \
+  __CLC_VLOAD_VECTORIZE(__CLC_SCALAR_GENTYPE, __private)                       \
+  __CLC_VLOAD_VECTORIZE(__CLC_SCALAR_GENTYPE, __local)                         \
+  __CLC_VLOAD_VECTORIZE(__CLC_SCALAR_GENTYPE, __constant)                      \
+  __CLC_VLOAD_VECTORIZE(__CLC_SCALAR_GENTYPE, __global)                        \
+  __CLC_VLOAD_VECTORIZE_GENERIC(__CLC_SCALAR_GENTYPE, __generic)
 
-#define VLOAD_TYPES()                                                          \
-  VLOAD_ADDR_SPACES(char)                                                      \
-  VLOAD_ADDR_SPACES(uchar)                                                     \
-  VLOAD_ADDR_SPACES(short)                                                     \
-  VLOAD_ADDR_SPACES(ushort)                                                    \
-  VLOAD_ADDR_SPACES(int)                                                       \
-  VLOAD_ADDR_SPACES(uint)                                                      \
-  VLOAD_ADDR_SPACES(long)                                                      \
-  VLOAD_ADDR_SPACES(ulong)                                                     \
-  VLOAD_ADDR_SPACES(float)
+#define __CLC_VLOAD_TYPES()                                                    \
+  __CLC_VLOAD_ADDR_SPACES(char)                                                \
+  __CLC_VLOAD_ADDR_SPACES(uchar)                                               \
+  __CLC_VLOAD_ADDR_SPACES(short)                                               \
+  __CLC_VLOAD_ADDR_SPACES(ushort)                                              \
+  __CLC_VLOAD_ADDR_SPACES(int)                                                 \
+  __CLC_VLOAD_ADDR_SPACES(uint)                                                \
+  __CLC_VLOAD_ADDR_SPACES(long)                                                \
+  __CLC_VLOAD_ADDR_SPACES(ulong)                                               \
+  __CLC_VLOAD_ADDR_SPACES(float)
 
-VLOAD_TYPES()
+__CLC_VLOAD_TYPES()
 
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
-VLOAD_ADDR_SPACES(double)
+__CLC_VLOAD_ADDR_SPACES(double)
 #endif
 #ifdef cl_khr_fp16
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
-VLOAD_ADDR_SPACES(half)
+__CLC_VLOAD_ADDR_SPACES(half)
 #endif
 
 /* vload_half are legal even without cl_khr_fp16 */
 /* no vload_half for double */
-#define VEC_LOAD1(val, AS) val = __builtin_load_halff(&mem[offset++]);
-#define VEC_LOAD2(val, AS)                                                     \
-  VEC_LOAD1(val.lo, AS)                                                        \
-  VEC_LOAD1(val.hi, AS)
-#define VEC_LOAD3(val, AS)                                                     \
-  VEC_LOAD1(val.s0, AS)                                                        \
-  VEC_LOAD1(val.s1, AS)                                                        \
-  VEC_LOAD1(val.s2, AS)
-#define VEC_LOAD4(val, AS)                                                     \
-  VEC_LOAD2(val.lo, AS)                                                        \
-  VEC_LOAD2(val.hi, AS)
-#define VEC_LOAD8(val, AS)                                                     \
-  VEC_LOAD4(val.lo, AS)                                                        \
-  VEC_LOAD4(val.hi, AS)
-#define VEC_LOAD16(val, AS)                                                    \
-  VEC_LOAD8(val.lo, AS)                                                        \
-  VEC_LOAD8(val.hi, AS)
+#define __CLC_VEC_LOAD1(val, AS) val = __builtin_load_halff(&mem[offset++]);
+#define __CLC_VEC_LOAD2(val, AS)                                               \
+  __CLC_VEC_LOAD1(val.lo, AS)                                                  \
+  __CLC_VEC_LOAD1(val.hi, AS)
+#define __CLC_VEC_LOAD3(val, AS)                                               \
+  __CLC_VEC_LOAD1(val.s0, AS)                                                  \
+  __CLC_VEC_LOAD1(val.s1, AS)                                                  \
+  __CLC_VEC_LOAD1(val.s2, AS)
+#define __CLC_VEC_LOAD4(val, AS)                                               \
+  __CLC_VEC_LOAD2(val.lo, AS)                                                  \
+  __CLC_VEC_LOAD2(val.hi, AS)
+#define __CLC_VEC_LOAD8(val, AS)                                               \
+  __CLC_VEC_LOAD4(val.lo, AS)                                                  \
+  __CLC_VEC_LOAD4(val.hi, AS)
+#define __CLC_VEC_LOAD16(val, AS)                                              \
+  __CLC_VEC_LOAD8(val.lo, AS)                                                  \
+  __CLC_VEC_LOAD8(val.hi, AS)
 
-#define __FUNC(SUFFIX, VEC_SIZE, OFFSET_SIZE, TYPE, AS)                        \
+#define __CLC_FUNC_IMPL(SUFFIX, VEC_SIZE, OFFSET_SIZE, TYPE, AS)               \
   _CLC_OVERLOAD _CLC_DEF TYPE __clc_vload_half##SUFFIX(size_t offset,          \
                                                        const AS half *mem) {   \
     offset *= VEC_SIZE;                                                        \
     TYPE __tmp;                                                                \
-    VEC_LOAD##VEC_SIZE(__tmp, AS) return __tmp;                                \
+    __CLC_VEC_LOAD##VEC_SIZE(__tmp, AS) return __tmp;                          \
   }                                                                            \
   _CLC_OVERLOAD _CLC_DEF TYPE __clc_vloada_half##SUFFIX(size_t offset,         \
                                                         const AS half *mem) {  \
     offset *= OFFSET_SIZE;                                                     \
     TYPE __tmp;                                                                \
-    VEC_LOAD##VEC_SIZE(__tmp, AS) return __tmp;                                \
+    __CLC_VEC_LOAD##VEC_SIZE(__tmp, AS) return __tmp;                          \
   }
 
-#define FUNC(SUFFIX, VEC_SIZE, OFFSET_SIZE, TYPE, AS)                          \
-  __FUNC(SUFFIX, VEC_SIZE, OFFSET_SIZE, TYPE, AS)
+#define __CLC_FUNC(SUFFIX, VEC_SIZE, OFFSET_SIZE, TYPE, AS)                    \
+  __CLC_FUNC_IMPL(SUFFIX, VEC_SIZE, OFFSET_SIZE, TYPE, AS)
 
 #define __CLC_BODY "clc_vload_half.inc"
 #include <clc/math/gentype.inc>
-#undef FUNC
-#undef __FUNC
-#undef VEC_LOAD16
-#undef VEC_LOAD8
-#undef VEC_LOAD4
-#undef VEC_LOAD3
-#undef VEC_LOAD2
-#undef VEC_LOAD1
-#undef VLOAD_TYPES
-#undef VLOAD_ADDR_SPACES
-#undef VLOAD_VECTORIZE
-#undef VLOAD_VECTORIZE_GENERIC
+#undef __CLC_FUNC
+#undef __CLC_FUNC_IMPL
+#undef __CLC_VEC_LOAD16
+#undef __CLC_VEC_LOAD8
+#undef __CLC_VEC_LOAD4
+#undef __CLC_VEC_LOAD3
+#undef __CLC_VEC_LOAD2
+#undef __CLC_VEC_LOAD1
+#undef __CLC_VLOAD_TYPES
+#undef __CLC_VLOAD_ADDR_SPACES
+#undef __CLC_VLOAD_VECTORIZE
+#undef __CLC_VLOAD_VECTORIZE_GENERIC
diff --git a/libclc/clc/lib/generic/shared/clc_vload_half.inc b/libclc/clc/lib/generic/shared/clc_vload_half.inc
index aaf067d75daac..2d5fdb357b0af 100644
--- a/libclc/clc/lib/generic/shared/clc_vload_half.inc
+++ b/libclc/clc/lib/generic/shared/clc_vload_half.inc
@@ -16,22 +16,25 @@
 #define __CLC_OFFSET __CLC_VECSIZE
 #endif
 
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __private);
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __local);
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __global);
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __constant);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE,
+           __private);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __local);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __global);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE,
+           __constant);
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __generic);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE,
+           __generic);
 #endif
 
 #undef __CLC_OFFSET
 #else
-FUNC(, 1, 1, __CLC_GENTYPE, __private);
-FUNC(, 1, 1, __CLC_GENTYPE, __local);
-FUNC(, 1, 1, __CLC_GENTYPE, __global);
-FUNC(, 1, 1, __CLC_GENTYPE, __constant);
+__CLC_FUNC(, 1, 1, __CLC_GENTYPE, __private);
+__CLC_FUNC(, 1, 1, __CLC_GENTYPE, __local);
+__CLC_FUNC(, 1, 1, __CLC_GENTYPE, __global);
+__CLC_FUNC(, 1, 1, __CLC_GENTYPE, __constant);
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-FUNC(, 1, 1, __CLC_GENTYPE, __generic);
+__CLC_FUNC(, 1, 1, __CLC_GENTYPE, __generic);
 #endif
 #endif
 #endif
diff --git a/libclc/clc/lib/generic/shared/clc_vstore.cl b/libclc/clc/lib/generic/shared/clc_vstore.cl
index adde58aec9153..ac96927395120 100644
--- a/libclc/clc/lib/generic/shared/clc_vstore.cl
+++ b/libclc/clc/lib/generic/shared/clc_vstore.cl
@@ -17,7 +17,7 @@
 
 #pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
 
-#define VSTORE_VECTORIZE(PRIM_TYPE, ADDR_SPACE)                                \
+#define __CLC_VSTORE_VECTORIZE(PRIM_TYPE, ADDR_SPACE)                          \
   typedef PRIM_TYPE##2 less_aligned_##ADDR_SPACE##PRIM_TYPE##2                 \
       __attribute__((aligned(sizeof(PRIM_TYPE))));                             \
   _CLC_OVERLOAD _CLC_DEF void __clc_vstore2(PRIM_TYPE##2 vec, size_t offset,   \
@@ -58,67 +58,68 @@
   }
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-#define VSTORE_VECTORIZE_GENERIC VSTORE_VECTORIZE
+#define __CLC_VSTORE_VECTORIZE_GENERIC __CLC_VSTORE_VECTORIZE
 #else
 // The generic address space isn't available, so make the macro do nothing
-#define VSTORE_VECTORIZE_GENERIC(X, Y)
+#define __CLC_VSTORE_VECTORIZE_GENERIC(X, Y)
 #endif
 
-#define VSTORE_ADDR_SPACES(__CLC_SCALAR___CLC_GENTYPE)                         \
-  VSTORE_VECTORIZE(__CLC_SCALAR___CLC_GENTYPE, __private)                      \
-  VSTORE_VECTORIZE(__CLC_SCALAR___CLC_GENTYPE, __local)                        \
-  VSTORE_VECTORIZE(__CLC_SCALAR___CLC_GENTYPE, __global)                       \
-  VSTORE_VECTORIZE_GENERIC(__CLC_SCALAR___CLC_GENTYPE, __generic)
+#define __CLC_VSTORE_ADDR_SPACES(__CLC_SCALAR___CLC_GENTYPE)                   \
+  __CLC_VSTORE_VECTORIZE(__CLC_SCALAR___CLC_GENTYPE, __private)                \
+  __CLC_VSTORE_VECTORIZE(__CLC_SCALAR___CLC_GENTYPE, __local)                  \
+  __CLC_VSTORE_VECTORIZE(__CLC_SCALAR___CLC_GENTYPE, __global)                 \
+  __CLC_VSTORE_VECTORIZE_GENERIC(__CLC_SCALAR___CLC_GENTYPE, __generic)
 
-VSTORE_ADDR_SPACES(char)
-VSTORE_ADDR_SPACES(uchar)
-VSTORE_ADDR_SPACES(short)
-VSTORE_ADDR_SPACES(ushort)
-VSTORE_ADDR_SPACES(int)
-VSTORE_ADDR_SPACES(uint)
-VSTORE_ADDR_SPACES(long)
-VSTORE_ADDR_SPACES(ulong)
-VSTORE_ADDR_SPACES(float)
+__CLC_VSTORE_ADDR_SPACES(char)
+__CLC_VSTORE_ADDR_SPACES(uchar)
+__CLC_VSTORE_ADDR_SPACES(short)
+__CLC_VSTORE_ADDR_SPACES(ushort)
+__CLC_VSTORE_ADDR_SPACES(int)
+__CLC_VSTORE_ADDR_SPACES(uint)
+__CLC_VSTORE_ADDR_SPACES(long)
+__CLC_VSTORE_ADDR_SPACES(ulong)
+__CLC_VSTORE_ADDR_SPACES(float)
 
 #ifdef cl_khr_fp64
 #pragma OPENCL EXTENSION cl_khr_fp64 : enable
-VSTORE_ADDR_SPACES(double)
+__CLC_VSTORE_ADDR_SPACES(double)
 #endif
 
 #ifdef cl_khr_fp16
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
-VSTORE_ADDR_SPACES(half)
+__CLC_VSTORE_ADDR_SPACES(half)
 #endif
 
-#define VEC_STORE1(val, ROUNDF, BUILTIN) BUILTIN(ROUNDF(val), &mem[offset++]);
+#define __CLC_VEC_STORE1(val, ROUNDF, BUILTIN)                                 \
+  BUILTIN(ROUNDF(val), &mem[offset++]);
 
-#define VEC_STORE2(val, ROUNDF, BUILTIN)                                       \
-  VEC_STORE1(val.lo, ROUNDF, BUILTIN)                                          \
-  VEC_STORE1(val.hi, ROUNDF, BUILTIN)
-#define VEC_STORE3(val, ROUNDF, BUILTIN)                                       \
-  VEC_STORE1(val.s0, ROUNDF, BUILTIN)                                          \
-  VEC_STORE1(val.s1, ROUNDF, BUILTIN)                                          \
-  VEC_STORE1(val.s2, ROUNDF, BUILTIN)
-#define VEC_STORE4(val, ROUNDF, BUILTIN)                                       \
-  VEC_STORE2(val.lo, ROUNDF, BUILTIN)                                          \
-  VEC_STORE2(val.hi, ROUNDF, BUILTIN)
-#define VEC_STORE8(val, ROUNDF, BUILTIN)                                       \
-  VEC_STORE4(val.lo, ROUNDF, BUILTIN)                                          \
-  VEC_STORE4(val.hi, ROUNDF, BUILTIN)
-#define VEC_STORE16(val, ROUNDF, BUILTIN)                                      \
-  VEC_STORE8(val.lo, ROUNDF, BUILTIN)                                          \
-  VEC_STORE8(val.hi, ROUNDF, BUILTIN)
+#define __CLC_VEC_STORE2(val, ROUNDF, BUILTIN)                                 \
+  __CLC_VEC_STORE1(val.lo, ROUNDF, BUILTIN)                                    \
+  __CLC_VEC_STORE1(val.hi, ROUNDF, BUILTIN)
+#define __CLC_VEC_STORE3(val, ROUNDF, BUILTIN)                                 \
+  __CLC_VEC_STORE1(val.s0, ROUNDF, BUILTIN)                                    \
+  __CLC_VEC_STORE1(val.s1, ROUNDF, BUILTIN)                                    \
+  __CLC_VEC_STORE1(val.s2, ROUNDF, BUILTIN)
+#define __CLC_VEC_STORE4(val, ROUNDF, BUILTIN)                                 \
+  __CLC_VEC_STORE2(val.lo, ROUNDF, BUILTIN)                                    \
+  __CLC_VEC_STORE2(val.hi, ROUNDF, BUILTIN)
+#define __CLC_VEC_STORE8(val, ROUNDF, BUILTIN)                                 \
+  __CLC_VEC_STORE4(val.lo, ROUNDF, BUILTIN)                                    \
+  __CLC_VEC_STORE4(val.hi, ROUNDF, BUILTIN)
+#define __CLC_VEC_STORE16(val, ROUNDF, BUILTIN)                                \
+  __CLC_VEC_STORE8(val.lo, ROUNDF, BUILTIN)                                    \
+  __CLC_VEC_STORE8(val.hi, ROUNDF, BUILTIN)
 
-#define __FUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, ROUNDF, BUILTIN)            \
+#define __CLC_XFUNC_IMPL(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, ROUNDF, BUILTIN)  \
   _CLC_OVERLOAD _CLC_DEF void __clc_vstore_half##SUFFIX(                       \
       TYPE vec, size_t offset, AS half *mem) {                                 \
     offset *= VEC_SIZE;                                                        \
-    VEC_STORE##VEC_SIZE(vec, ROUNDF, BUILTIN)                                  \
+    __CLC_VEC_STORE##VEC_SIZE(vec, ROUNDF, BUILTIN)                            \
   }                                                                            \
   _CLC_OVERLOAD _CLC_DEF void __clc_vstorea_half##SUFFIX(                      \
       TYPE vec, size_t offset, AS half *mem) {                                 \
     offset *= OFFSET;                                                          \
-    VEC_STORE##VEC_SIZE(vec, ROUNDF, BUILTIN)                                  \
+    __CLC_VEC_STORE##VEC_SIZE(vec, ROUNDF, BUILTIN)                            \
   }
 
 _CLC_DEF _CLC_OVERLOAD float __clc_noop(float x) { return x; }
@@ -241,28 +242,30 @@ _CLC_DEF _CLC_OVERLOAD double __clc_rte(double x) {
 }
 #endif
 
-#define __XFUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, BUILTIN)                   \
-  __FUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, __clc_noop, BUILTIN)              \
-  __FUNC(SUFFIX##_rtz, VEC_SIZE, OFFSET, TYPE, AS, __clc_rtz, BUILTIN)         \
-  __FUNC(SUFFIX##_rtn, VEC_SIZE, OFFSET, TYPE, AS, __clc_rtn, BUILTIN)         \
-  __FUNC(SUFFIX##_rtp, VEC_SIZE, OFFSET, TYPE, AS, __clc_rtp, BUILTIN)         \
-  __FUNC(SUFFIX##_rte, VEC_SIZE, OFFSET, TYPE, AS, __clc_rte, BUILTIN)
+#define __CLC_XFUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, BUILTIN)               \
+  __CLC_XFUNC_IMPL(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, __clc_noop, BUILTIN)    \
+  __CLC_XFUNC_IMPL(SUFFIX##_rtz, VEC_SIZE, OFFSET, TYPE, AS, __clc_rtz,        \
+                   BUILTIN)                                                    \
+  __CLC_XFUNC_IMPL(SUFFIX##_rtn, VEC_SIZE, OFFSET, TYPE, AS, __clc_rtn,        \
+                   BUILTIN)                                                    \
+  __CLC_XFUNC_IMPL(SUFFIX##_rtp, VEC_SIZE, OFFSET, TYPE, AS, __clc_rtp,        \
+                   BUILTIN)                                                    \
+  __CLC_XFUNC_IMPL(SUFFIX##_rte, VEC_SIZE, OFFSET, TYPE, AS, __clc_rte, BUILTIN)
 
-#define FUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, BUILTIN)                      \
-  __XFUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, BUILTIN)
+#define __CLC_FUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, BUILTIN)                \
+  __CLC_XFUNC(SUFFIX, VEC_SIZE, OFFSET, TYPE, AS, BUILTIN)
 
 #define __CLC_BODY "clc_vstore_half.inc"
 #include <clc/math/gentype.inc>
-#undef FUNC
-#undef __XFUNC
-#undef __FUNC
-#undef VEC_LOAD16
-#undef VEC_LOAD8
-#undef VEC_LOAD4
-#undef VEC_LOAD3
-#undef VEC_LOAD2
-#undef VEC_LOAD1
-#undef DECLARE_HELPER
-#undef VSTORE_ADDR_SPACES
-#undef VSTORE_VECTORIZE
-#undef VSTORE_VECTORIZE_GENERIC
+#undef __CLC_FUNC
+#undef __CLC_XFUNC
+#undef __CLC_XFUNC_IMPL
+#undef __CLC_VEC_STORE16
+#undef __CLC_VEC_STORE8
+#undef __CLC_VEC_STORE4
+#undef __CLC_VEC_STORE3
+#undef __CLC_VEC_STORE2
+#undef __CLC_VEC_STORE1
+#undef __CLC_VSTORE_ADDR_SPACES
+#undef __CLC_VSTORE_VECTORIZE
+#undef __CLC_VSTORE_VECTORIZE_GENERIC
diff --git a/libclc/clc/lib/generic/shared/clc_vstore_half.inc b/libclc/clc/lib/generic/shared/clc_vstore_half.inc
index 02d645357f853..9188f96e29691 100644
--- a/libclc/clc/lib/generic/shared/clc_vstore_half.inc
+++ b/libclc/clc/lib/generic/shared/clc_vstore_half.inc
@@ -25,24 +25,24 @@
 #define __CLC_OFFSET __CLC_VECSIZE
 #endif
 
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __private,
-     STORE_HALF_BUILTIN);
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __local,
-     STORE_HALF_BUILTIN);
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __global,
-     STORE_HALF_BUILTIN);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __private,
+           STORE_HALF_BUILTIN);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __local,
+           STORE_HALF_BUILTIN);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __global,
+           STORE_HALF_BUILTIN);
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __generic,
-     STORE_HALF_BUILTIN);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_OFFSET, __CLC_GENTYPE, __generic,
+           STORE_HALF_BUILTIN);
 #endif
 
 #undef __CLC_OFFSET
 #else
-FUNC(, 1, 1, __CLC_GENTYPE, __private, STORE_HALF_BUILTIN);
-FUNC(, 1, 1, __CLC_GENTYPE, __local, STORE_HALF_BUILTIN);
-FUNC(, 1, 1, __CLC_GENTYPE, __global, STORE_HALF_BUILTIN);
+__CLC_FUNC(, 1, 1, __CLC_GENTYPE, __private, STORE_HALF_BUILTIN);
+__CLC_FUNC(, 1, 1, __CLC_GENTYPE, __local, STORE_HALF_BUILTIN);
+__CLC_FUNC(, 1, 1, __CLC_GENTYPE, __global, STORE_HALF_BUILTIN);
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-FUNC(, 1, 1, __CLC_GENTYPE, __generic, STORE_HALF_BUILTIN);
+__CLC_FUNC(, 1, 1, __CLC_GENTYPE, __generic, STORE_HALF_BUILTIN);
 #endif
 #endif
 
diff --git a/libclc/clc/lib/ptx-nvidiacl/SOURCES b/libclc/clc/lib/ptx-nvidiacl/SOURCES
index b6f50654f89c5..cafd90943f22e 100644
--- a/libclc/clc/lib/ptx-nvidiacl/SOURCES
+++ b/libclc/clc/lib/ptx-nvidiacl/SOURCES
@@ -1,3 +1,7 @@
+math/clc_log.cl
+math/clc_rsqrt.cl
+math/clc_sinpi.cl
+math/clc_sqrt.cl
 mem_fence/clc_mem_fence.cl
 synchronization/clc_work_group_barrier.cl
 workitem/clc_get_global_id.cl
@@ -7,3 +11,4 @@ workitem/clc_get_local_size.cl
 workitem/clc_get_max_sub_group_size.cl
 workitem/clc_get_num_groups.cl
 workitem/clc_get_sub_group_local_id.cl
+relational/clc_isinf.cl
diff --git a/libclc/clc/lib/ptx-nvidiacl/math/clc_log.cl b/libclc/clc/lib/ptx-nvidiacl/math/clc_log.cl
new file mode 100644
index 0000000000000..4ccdfaf4e3fc9
--- /dev/null
+++ b/libclc/clc/lib/ptx-nvidiacl/math/clc_log.cl
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/math/clc_log.h>
+
+float __nv_logf(float);
+double __nv_log(double);
+
+_CLC_OVERLOAD _CLC_DEF float __clc_log(float x) { return __nv_logf(x); }
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+
+_CLC_OVERLOAD _CLC_DEF double __clc_log(double x) { return __nv_log(x); }
+
+#endif
+
+#ifdef cl_khr_fp16
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+_CLC_OVERLOAD _CLC_DEF half __clc_log(half x) {
+  return (half)__clc_log((float)x);
+}
+
+#endif
+
+#define __CLC_FUNCTION __clc_log
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/ptx-nvidiacl/math/clc_rsqrt.cl b/libclc/clc/lib/ptx-nvidiacl/math/clc_rsqrt.cl
new file mode 100644
index 0000000000000..786d57046728d
--- /dev/null
+++ b/libclc/clc/lib/ptx-nvidiacl/math/clc_rsqrt.cl
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/math/clc_rsqrt.h>
+
+float __nv_rsqrtf(float);
+double __nv_rsqrt(double);
+
+_CLC_OVERLOAD _CLC_DEF float __clc_rsqrt(float x) { return __nv_rsqrtf(x); }
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+
+_CLC_OVERLOAD _CLC_DEF double __clc_rsqrt(double x) { return __nv_rsqrt(x); }
+
+#endif
+
+#ifdef cl_khr_fp16
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+_CLC_OVERLOAD _CLC_DEF half __clc_rsqrt(half x) {
+  return (half)__clc_rsqrt((float)x);
+}
+
+#endif
+
+#define __CLC_FUNCTION __clc_rsqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/ptx-nvidiacl/math/clc_sinpi.cl b/libclc/clc/lib/ptx-nvidiacl/math/clc_sinpi.cl
new file mode 100644
index 0000000000000..78f7038b26443
--- /dev/null
+++ b/libclc/clc/lib/ptx-nvidiacl/math/clc_sinpi.cl
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/math/clc_sinpi.h>
+
+float __nv_sinpif(float);
+double __nv_sinpi(double);
+
+_CLC_OVERLOAD _CLC_DEF float __clc_sinpi(float x) { return __nv_sinpif(x); }
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+
+_CLC_OVERLOAD _CLC_DEF double __clc_sinpi(double x) { return __nv_sinpi(x); }
+
+#endif
+
+#ifdef cl_khr_fp16
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+_CLC_OVERLOAD _CLC_DEF half __clc_sinpi(half x) {
+  return (half)__clc_sinpi((float)x);
+}
+
+#endif
+
+#define __CLC_FUNCTION __clc_sinpi
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/ptx-nvidiacl/math/clc_sqrt.cl b/libclc/clc/lib/ptx-nvidiacl/math/clc_sqrt.cl
new file mode 100644
index 0000000000000..ac86a57fca3d6
--- /dev/null
+++ b/libclc/clc/lib/ptx-nvidiacl/math/clc_sqrt.cl
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/math/clc_sqrt.h>
+
+float __nv_sqrtf(float);
+double __nv_sqrt(double);
+
+_CLC_OVERLOAD _CLC_DEF float __clc_sqrt(float x) { return __nv_sqrtf(x); }
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+
+_CLC_OVERLOAD _CLC_DEF double __clc_sqrt(double x) { return __nv_sqrt(x); }
+
+#endif
+
+#ifdef cl_khr_fp16
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+_CLC_OVERLOAD _CLC_DEF half __clc_sqrt(half x) {
+  return (half)__clc_sqrt((float)x);
+}
+
+#endif
+
+#define __CLC_FUNCTION __clc_sqrt
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/ptx-nvidiacl/relational/clc_isinf.cl b/libclc/clc/lib/ptx-nvidiacl/relational/clc_isinf.cl
new file mode 100644
index 0000000000000..93f14d79fe6b2
--- /dev/null
+++ b/libclc/clc/lib/ptx-nvidiacl/relational/clc_isinf.cl
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <clc/relational/clc_isinf.h>
+
+int __nv_isinff(float);
+int __nv_isinfd(double);
+
+_CLC_OVERLOAD _CLC_DEF int __clc_isinf(float x) { return __nv_isinff(x); }
+
+#ifdef cl_khr_fp64
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+
+_CLC_OVERLOAD _CLC_DEF int __clc_isinf(double x) { return __nv_isinfd(x); }
+
+#endif
+
+#ifdef cl_khr_fp16
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+_CLC_OVERLOAD _CLC_DEF int __clc_isinf(half x) { return __clc_isinf((float)x); }
+
+#endif
+
+#define __CLC_FUNCTION __clc_isinf
+#define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
+#define __CLC_RET_TYPE __CLC_BIT_INT
+#include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/ptx-nvidiacl/workitem/clc_get_local_size.cl b/libclc/clc/lib/ptx-nvidiacl/workitem/clc_get_local_size.cl
index 4525c85f1e382..2d547d3af249e 100644
--- a/libclc/clc/lib/ptx-nvidiacl/workitem/clc_get_local_size.cl
+++ b/libclc/clc/lib/ptx-nvidiacl/workitem/clc_get_local_size.cl
@@ -17,6 +17,6 @@ _CLC_OVERLOAD _CLC_DEF size_t __clc_get_local_size(uint dim) {
   case 2:
     return __nvvm_read_ptx_sreg_ntid_z();
   default:
-    return 0;
+    return 1;
   }
 }
diff --git a/libclc/clc/lib/ptx-nvidiacl/workitem/clc_get_num_groups.cl b/libclc/clc/lib/ptx-nvidiacl/workitem/clc_get_num_groups.cl
index 495864751ef68..87151ea4be625 100644
--- a/libclc/clc/lib/ptx-nvidiacl/workitem/clc_get_num_groups.cl
+++ b/libclc/clc/lib/ptx-nvidiacl/workitem/clc_get_num_groups.cl
@@ -17,6 +17,6 @@ _CLC_OVERLOAD _CLC_DEF size_t __clc_get_num_groups(uint dim) {
   case 2:
     return __nvvm_read_ptx_sreg_nctaid_z();
   default:
-    return 0;
+    return 1;
   }
 }
diff --git a/libclc/clc/lib/r600/math/clc_native_rsqrt.cl b/libclc/clc/lib/r600/math/clc_native_rsqrt.cl
index b5966570804c9..da001574afc0c 100644
--- a/libclc/clc/lib/r600/math/clc_native_rsqrt.cl
+++ b/libclc/clc/lib/r600/math/clc_native_rsqrt.cl
@@ -13,7 +13,7 @@ _CLC_OVERLOAD _CLC_DEF float __clc_native_rsqrt(float x) {
   return __builtin_r600_recipsqrt_ieeef(x);
 }
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_native_rsqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_native_rsqrt
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/r600/math/clc_rsqrt_override.cl b/libclc/clc/lib/r600/math/clc_rsqrt_override.cl
index 75355df56d32e..f20046b341177 100644
--- a/libclc/clc/lib/r600/math/clc_rsqrt_override.cl
+++ b/libclc/clc/lib/r600/math/clc_rsqrt_override.cl
@@ -13,11 +13,11 @@ _CLC_OVERLOAD _CLC_DEF float __clc_rsqrt(float x) {
   return __builtin_r600_recipsqrt_ieeef(x);
 }
 
-#define __FLOAT_ONLY
-#define FUNCTION __clc_rsqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION __clc_rsqrt
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #ifdef cl_khr_fp64
 
@@ -27,10 +27,10 @@ _CLC_OVERLOAD _CLC_DEF double __clc_rsqrt(double x) {
   return __builtin_r600_recipsqrt_ieee(x);
 }
 
-#define __DOUBLE_ONLY
-#define FUNCTION __clc_rsqrt
+#define __CLC_DOUBLE_ONLY
+#define __CLC_FUNCTION __clc_rsqrt
 #define __CLC_BODY <clc/shared/unary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif
diff --git a/libclc/clc/lib/spirv/math/clc_fmax.cl b/libclc/clc/lib/spirv/math/clc_fmax.cl
index be660fe2b29ba..f63a9d00c642d 100644
--- a/libclc/clc/lib/spirv/math/clc_fmax.cl
+++ b/libclc/clc/lib/spirv/math/clc_fmax.cl
@@ -27,6 +27,6 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmax(half x, half y) {
 }
 #endif
 
-#define FUNCTION __clc_fmax
+#define __CLC_FUNCTION __clc_fmax
 #define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/clc/lib/spirv/math/clc_fmin.cl b/libclc/clc/lib/spirv/math/clc_fmin.cl
index 9f3fa66c0ab7a..ddb0126035f8f 100644
--- a/libclc/clc/lib/spirv/math/clc_fmin.cl
+++ b/libclc/clc/lib/spirv/math/clc_fmin.cl
@@ -27,6 +27,6 @@ _CLC_DEF _CLC_OVERLOAD half __clc_fmin(half x, half y) {
 }
 #endif
 
-#define FUNCTION __clc_fmin
+#define __CLC_FUNCTION __clc_fmin
 #define __CLC_BODY <clc/shared/binary_def_scalarize.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/cmake/modules/AddLibclc.cmake b/libclc/cmake/modules/AddLibclc.cmake
index 47185586d4094..5cc202ddbaa8c 100644
--- a/libclc/cmake/modules/AddLibclc.cmake
+++ b/libclc/cmake/modules/AddLibclc.cmake
@@ -1,8 +1,6 @@
 # Compiles an OpenCL C - or assembles an LL file - to bytecode
 #
 # Arguments:
-# * TARGET <string>
-#     Custom target to create
 # * TRIPLE <string>
 #     Target triple for which to compile the bytecode file.
 # * INPUT <string>
@@ -19,7 +17,7 @@
 function(compile_to_bc)
   cmake_parse_arguments(ARG
     ""
-    "TARGET;TRIPLE;INPUT;OUTPUT"
+    "TRIPLE;INPUT;OUTPUT"
     "EXTRA_OPTS;DEPENDENCIES"
     ${ARGN}
   )
@@ -65,12 +63,6 @@ function(compile_to_bc)
       ${ARG_DEPENDENCIES}
     DEPFILE ${ARG_OUTPUT}.d
   )
-  # FIXME: The target is added to ensure the parallel build of source files.
-  # However, this may result in a large number of targets.
-  # Starting with CMake 3.27, DEPENDS_EXPLICIT_ONLY can be used with
-  # add_custom_command to enable parallel build.
-  # Refer to https://gitlab.kitware.com/cmake/cmake/-/issues/17097 for details.
-  add_custom_target( ${ARG_TARGET} DEPENDS ${ARG_OUTPUT}${TMP_SUFFIX} )
 
   if( ${FILE_EXT} STREQUAL ".ll" )
     add_custom_command(
@@ -132,6 +124,37 @@ function(link_bc)
   )
 endfunction()
 
+# Create a custom target for each bitcode file, which is the output of a custom
+# command. This is required for parallel compilation of the custom commands that
+# generate the bitcode files when using the CMake MSVC generator on Windows.
+#
+# Arguments:
+#  * compile_tgts
+#      Output list of compile targets
+#  * ARCH_SUFFIX <string>
+#      libclc architecture/triple suffix
+#  * FILES <string> ...
+#     List of bitcode files
+function(create_compile_targets compile_tgts)
+  cmake_parse_arguments( ARG "" "ARCH_SUFFIX" "FILES" ${ARGN} )
+
+  if( NOT ARG_ARCH_SUFFIX OR NOT ARG_FILES )
+    message( FATAL_ERROR "Must provide ARCH_SUFFIX, and FILES" )
+  endif()
+
+  set( tgts )
+  foreach( file IN LISTS ARG_FILES )
+    cmake_path( GET file STEM stem )
+    cmake_path( GET file PARENT_PATH parent_path )
+    cmake_path( GET parent_path STEM parent_path_stem )
+    set( tgt compile-${ARG_ARCH_SUFFIX}-${parent_path_stem}-${stem} )
+    add_custom_target( ${tgt} DEPENDS ${file} )
+    list( APPEND tgts ${tgt} )
+  endforeach()
+
+  set( compile_tgts ${tgts} PARENT_SCOPE )
+endfunction()
+
 # Decomposes and returns variables based on a libclc triple and architecture
 # combination. Returns data via one or more optional output variables.
 #
@@ -199,6 +222,33 @@ function(get_libclc_device_info)
   endif()
 endfunction()
 
+# Install libclc artifacts.
+#
+# Arguments:
+#  * FILES <string> ...
+#      List of libclc artifact files to be installed.
+function(libclc_install)
+  cmake_parse_arguments(ARG "" "" "FILES" ${ARGN})
+
+  if( NOT ARG_FILES )
+    message( FATAL_ERROR "Must provide FILES" )
+  endif()
+
+  if( NOT CMAKE_CFG_INTDIR STREQUAL "." )
+    # Replace CMAKE_CFG_INTDIR with CMAKE_INSTALL_CONFIG_NAME for multiple-
+    # configuration generators.
+    string( REPLACE ${CMAKE_CFG_INTDIR} "\$\{CMAKE_INSTALL_CONFIG_NAME\}"
+            files ${ARG_FILES} )
+  else()
+    set( files ${ARG_FILES} )
+  endif()
+
+  install(
+    FILES ${files}
+    DESTINATION "${CMAKE_INSTALL_DATADIR}/clc"
+  )
+endfunction()
+
 # Compiles a list of library source files (provided by LIB_FILES/GEN_FILES) and
 # compiles them to LLVM bytecode (or SPIR-V), links them together and optimizes
 # them.
@@ -248,7 +298,6 @@ function(add_libclc_builtin_set)
 
   set( bytecode_files )
   set( bytecode_ir_files )
-  set( compile_tgts )
   foreach( file IN LISTS ARG_GEN_FILES ARG_LIB_FILES )
     # We need to take each file and produce an absolute input file, as well
     # as a unique architecture-specific output file. We deal with a mix of
@@ -278,9 +327,6 @@ function(add_libclc_builtin_set)
 
     get_filename_component( file_dir ${file} DIRECTORY )
 
-    string( REPLACE "/" "-" replaced ${file} )
-    set( tgt compile_tgt-${ARG_ARCH_SUFFIX}${replaced})
-
     set( file_specific_compile_options )
     get_source_file_property( compile_opts ${file} COMPILE_OPTIONS)
     if( compile_opts )
@@ -288,7 +334,6 @@ function(add_libclc_builtin_set)
     endif()
 
     compile_to_bc(
-      TARGET ${tgt}
       TRIPLE ${ARG_TRIPLE}
       INPUT ${input_file}
       OUTPUT ${output_file}
@@ -297,7 +342,6 @@ function(add_libclc_builtin_set)
         -I${CMAKE_CURRENT_SOURCE_DIR}/${file_dir}
       DEPENDENCIES ${input_file_dep}
     )
-    list( APPEND compile_tgts ${tgt} )
 
     # Collect all files originating in LLVM IR separately
     get_filename_component( file_ext ${file} EXT )
@@ -308,6 +352,22 @@ function(add_libclc_builtin_set)
     endif()
   endforeach()
 
+  set( builtins_comp_lib_tgt builtins.comp.${ARG_ARCH_SUFFIX} )
+  if ( CMAKE_GENERATOR MATCHES "Visual Studio" )
+    # Don't put commands in one custom target to avoid serialized compilation.
+    create_compile_targets( compile_tgts
+      ARCH_SUFFIX ${ARG_ARCH_SUFFIX}
+      FILES ${bytecode_files}
+    )
+    add_custom_target( ${builtins_comp_lib_tgt} DEPENDS ${bytecode_ir_files} )
+    add_dependencies( ${builtins_comp_lib_tgt} ${compile_tgts} )
+  else()
+    add_custom_target( ${builtins_comp_lib_tgt}
+      DEPENDS ${bytecode_files} ${bytecode_ir_files}
+    )
+  endif()
+  set_target_properties( ${builtins_comp_lib_tgt} PROPERTIES FOLDER "libclc/Device IR/Comp" )
+
   # Prepend all LLVM IR files to the list so they are linked into the final
   # bytecode modules first. This helps to suppress unnecessary warnings
   # regarding different data layouts while linking. Any LLVM IR files without a
@@ -315,12 +375,6 @@ function(add_libclc_builtin_set)
   # process comes across.
   list( PREPEND bytecode_files ${bytecode_ir_files} )
 
-  set( builtins_comp_lib_tgt builtins.comp.${ARG_ARCH_SUFFIX} )
-  add_custom_target( ${builtins_comp_lib_tgt}
-    DEPENDS ${bytecode_files} ${compile_tgts}
-  )
-  set_target_properties( ${builtins_comp_lib_tgt} PROPERTIES FOLDER "libclc/Device IR/Comp" )
-
   if( NOT bytecode_files )
     message(FATAL_ERROR "Cannot create an empty builtins library")
   endif()
@@ -409,7 +463,10 @@ function(add_libclc_builtin_set)
 
   # Add a 'prepare' target
   add_custom_target( prepare-${obj_suffix} ALL DEPENDS ${libclc_builtins_lib} )
-  set_target_properties( "prepare-${obj_suffix}" PROPERTIES FOLDER "libclc/Device IR/Prepare" )
+  set_target_properties( "prepare-${obj_suffix}" PROPERTIES
+    TARGET_FILE ${libclc_builtins_lib}
+    FOLDER "libclc/Device IR/Prepare"
+  )
 
   # Also add a 'prepare' target for the triple. Since a triple may have
   # multiple devices, ensure we only try to create the triple target once. The
@@ -422,10 +479,7 @@ function(add_libclc_builtin_set)
   # targets dependent on libclc.
   add_dependencies( ${ARG_PARENT_TARGET} prepare-${ARG_TRIPLE} )
 
-  install(
-    FILES ${libclc_builtins_lib}
-    DESTINATION "${CMAKE_INSTALL_DATADIR}/clc"
-  )
+  libclc_install(FILES ${libclc_builtins_lib})
 
   # SPIR-V targets can exit early here
   if( ARG_ARCH STREQUAL spirv OR ARG_ARCH STREQUAL spirv64 )
@@ -467,10 +521,7 @@ function(add_libclc_builtin_set)
     set_target_properties( alias-${alias_suffix}
       PROPERTIES FOLDER "libclc/Device IR/Aliases"
     )
-    install(
-      FILES ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix}
-      DESTINATION "${CMAKE_INSTALL_DATADIR}/clc"
-    )
+    libclc_install(FILES ${LIBCLC_OUTPUT_LIBRARY_DIR}/${alias_suffix})
   endforeach( a )
 endfunction(add_libclc_builtin_set)
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_add.h b/libclc/opencl/include/clc/opencl/atomic/atom_add.h
index 7f68fca0fc3a5..b26a2a5361b6f 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_add.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_add.h
@@ -12,19 +12,19 @@
 #include <clc/opencl/opencl-base.h>
 
 #ifdef cl_khr_global_int32_base_atomics
-#define FUNCTION atom_add
+#define __CLC_FUNCTION atom_add
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_base_atomics
 
 #ifdef cl_khr_local_int32_base_atomics
-#define FUNCTION atom_add
+#define __CLC_FUNCTION atom_add
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_base_atomics
 
 #ifdef cl_khr_int64_base_atomics
-#define FUNCTION atom_add
+#define __CLC_FUNCTION atom_add
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_base_atomics
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_and.h b/libclc/opencl/include/clc/opencl/atomic/atom_and.h
index 5c40aa4db70ad..eacbd7a33b6ee 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_and.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_and.h
@@ -12,19 +12,19 @@
 #include <clc/opencl/opencl-base.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define FUNCTION atom_and
+#define __CLC_FUNCTION atom_and
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define FUNCTION atom_and
+#define __CLC_FUNCTION atom_and
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define FUNCTION atom_and
+#define __CLC_FUNCTION atom_and
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc
index b8631ceac3a42..866d8903db816 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int32.inc
@@ -10,12 +10,12 @@
 #include <clc/clctypes.h>
 
 #define __CLC_DECLARE_ATOM(ADDRSPACE, TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+  _CLC_OVERLOAD _CLC_DECL TYPE __CLC_FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
 
 __CLC_DECLARE_ATOM(__CLC_ADDRESS_SPACE, int)
 __CLC_DECLARE_ATOM(__CLC_ADDRESS_SPACE, uint)
 
 #undef __CLC_DECLARE_ATOM
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 #undef __CLC_ADDRESS_SPACE
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc
index 3918a7cae006b..146de3412fc2e 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_decl_int64.inc
@@ -10,7 +10,7 @@
 #include <clc/clctypes.h>
 
 #define __CLC_DECLARE_ATOM(ADDRSPACE, TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+  _CLC_OVERLOAD _CLC_DECL TYPE __CLC_FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
 
 __CLC_DECLARE_ATOM(local, long)
 __CLC_DECLARE_ATOM(local, ulong)
@@ -19,4 +19,4 @@ __CLC_DECLARE_ATOM(global, ulong)
 
 #undef __CLC_DECLARE_ATOM
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_max.h b/libclc/opencl/include/clc/opencl/atomic/atom_max.h
index 87c70f20058d9..35a1e2aa06d48 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_max.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_max.h
@@ -12,19 +12,19 @@
 #include <clc/opencl/opencl-base.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define FUNCTION atom_max
+#define __CLC_FUNCTION atom_max
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define FUNCTION atom_max
+#define __CLC_FUNCTION atom_max
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define FUNCTION atom_max
+#define __CLC_FUNCTION atom_max
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_min.h b/libclc/opencl/include/clc/opencl/atomic/atom_min.h
index 487364c617ab5..ded96a5b29a48 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_min.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_min.h
@@ -12,19 +12,19 @@
 #include <clc/opencl/opencl-base.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define FUNCTION atom_min
+#define __CLC_FUNCTION atom_min
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define FUNCTION atom_min
+#define __CLC_FUNCTION atom_min
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define FUNCTION atom_min
+#define __CLC_FUNCTION atom_min
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_or.h b/libclc/opencl/include/clc/opencl/atomic/atom_or.h
index 1c3ae6984eead..42a69bd3f7b46 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_or.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_or.h
@@ -12,19 +12,19 @@
 #include <clc/opencl/opencl-base.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define FUNCTION atom_or
+#define __CLC_FUNCTION atom_or
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define FUNCTION atom_or
+#define __CLC_FUNCTION atom_or
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define FUNCTION atom_or
+#define __CLC_FUNCTION atom_or
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_sub.h b/libclc/opencl/include/clc/opencl/atomic/atom_sub.h
index eb17aa8ce73fe..9763ee6033410 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_sub.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_sub.h
@@ -12,19 +12,19 @@
 #include <clc/opencl/opencl-base.h>
 
 #ifdef cl_khr_global_int32_base_atomics
-#define FUNCTION atom_sub
+#define __CLC_FUNCTION atom_sub
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_base_atomics
 
 #ifdef cl_khr_local_int32_base_atomics
-#define FUNCTION atom_sub
+#define __CLC_FUNCTION atom_sub
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_base_atomics
 
 #ifdef cl_khr_int64_base_atomics
-#define FUNCTION atom_sub
+#define __CLC_FUNCTION atom_sub
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_base_atomics
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h b/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h
index df17d31d6e67a..ab2c5b5156d68 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_xchg.h
@@ -12,19 +12,19 @@
 #include <clc/opencl/opencl-base.h>
 
 #ifdef cl_khr_global_int32_base_atomics
-#define FUNCTION atom_xchg
+#define __CLC_FUNCTION atom_xchg
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_base_atomics
 
 #ifdef cl_khr_local_int32_base_atomics
-#define FUNCTION atom_xchg
+#define __CLC_FUNCTION atom_xchg
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_base_atomics
 
 #ifdef cl_khr_int64_base_atomics
-#define FUNCTION atom_xchg
+#define __CLC_FUNCTION atom_xchg
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_base_atomics
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atom_xor.h b/libclc/opencl/include/clc/opencl/atomic/atom_xor.h
index 8a97818821582..b33168db72cf0 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atom_xor.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atom_xor.h
@@ -12,19 +12,19 @@
 #include <clc/opencl/opencl-base.h>
 
 #ifdef cl_khr_global_int32_extended_atomics
-#define FUNCTION atom_xor
+#define __CLC_FUNCTION atom_xor
 #define __CLC_ADDRESS_SPACE global
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_global_int32_extended_atomics
 
 #ifdef cl_khr_local_int32_extended_atomics
-#define FUNCTION atom_xor
+#define __CLC_FUNCTION atom_xor
 #define __CLC_ADDRESS_SPACE local
 #include <clc/opencl/atomic/atom_decl_int32.inc>
 #endif // cl_khr_local_int32_extended_atomics
 
 #ifdef cl_khr_int64_extended_atomics
-#define FUNCTION atom_xor
+#define __CLC_FUNCTION atom_xor
 #include <clc/opencl/atomic/atom_decl_int64.inc>
 #endif // cl_khr_int64_extended_atomics
 
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_add.h b/libclc/opencl/include/clc/opencl/atomic/atomic_add.h
index 50fb99d1362fc..edb13bd6bde49 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_add.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_add.h
@@ -11,7 +11,7 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION atomic_add
+#define __CLC_FUNCTION atomic_add
 #include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_ADD_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_and.h b/libclc/opencl/include/clc/opencl/atomic/atomic_and.h
index 8ce328c9739aa..4875f5374c692 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_and.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_and.h
@@ -11,7 +11,7 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION atomic_and
+#define __CLC_FUNCTION atomic_and
 #include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_AND_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h
index 76eeda7ba3469..59bfa0e87dd8f 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_strong.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__
 
-#define FUNCTION atomic_compare_exchange_strong
+#define __CLC_FUNCTION atomic_compare_exchange_strong
 #define __CLC_COMPARE_EXCHANGE
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
@@ -19,6 +19,6 @@
 #include <clc/math/gentype.inc>
 
 #undef __CLC_COMPARE_EXCHANGE
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_STRONG_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h
index 12788ad03a2d1..7106c3e061d65 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_compare_exchange_weak.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__
 
-#define FUNCTION atomic_compare_exchange_weak
+#define __CLC_FUNCTION atomic_compare_exchange_weak
 #define __CLC_COMPARE_EXCHANGE
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
@@ -19,6 +19,6 @@
 #include <clc/math/gentype.inc>
 
 #undef __CLC_COMPARE_EXCHANGE
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_COMPARE_EXCHANGE_WEAK_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc
index 1b2bf17bd6dfd..38d250f0693f7 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_decl.inc
@@ -10,36 +10,37 @@
 
 #if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) &&         \
                                  defined(cl_khr_int64_extended_atomics))
-#define HAVE_64_ATOMIC
+#define __CLC_HAVE_64_ATOMIC
 #endif
-#if defined(__CLC_FPSIZE) && (__CLC_FPSIZE < 64 || defined(HAVE_64_ATOMIC))
-#define HAVE_FP_ATOMIC
+#if defined(__CLC_FPSIZE) &&                                                   \
+    (__CLC_FPSIZE < 64 || defined(__CLC_HAVE_64_ATOMIC))
+#define __CLC_HAVE_FP_ATOMIC
 #endif
 #if defined(__CLC_GENSIZE) &&                                                  \
     ((__CLC_GENSIZE == 32) ||                                                  \
-     (__CLC_GENSIZE == 64 && defined(HAVE_64_ATOMIC)))
-#define HAVE_INT_ATOMIC
+     (__CLC_GENSIZE == 64 && defined(__CLC_HAVE_64_ATOMIC)))
+#define __CLC_HAVE_INT_ATOMIC
 #endif
-#if defined(HAVE_FP_ATOMIC) || defined(HAVE_INT_ATOMIC)
+#if defined(__CLC_HAVE_FP_ATOMIC) || defined(__CLC_HAVE_INT_ATOMIC)
 
 #define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE)
 
 #ifdef __CLC_NO_VALUE_ARG
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(                        \
       volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr);
 #elif defined(__CLC_RETURN_VOID)
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DECL void FUNCTION(                                       \
+  _CLC_OVERLOAD _CLC_DECL void __CLC_FUNCTION(                                 \
       volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value);
 #elif defined(__CLC_COMPARE_EXCHANGE)
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(                        \
       volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr,                            \
       ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired);
 #else
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(                              \
+  _CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE __CLC_FUNCTION(                        \
       volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value);
 #endif
 
@@ -51,10 +52,10 @@ __CLC_DEFINE_ATOMIC()
 
 #undef __CLC_DEFINE_ATOMIC
 
-#endif // HAVE_FP_ATOMIC || HAVE_INT_ATOMIC
+#endif // __CLC_HAVE_FP_ATOMIC || __CLC_HAVE_INT_ATOMIC
 
-#undef HAVE_INT_ATOMIC
-#undef HAVE_FP_ATOMIC
-#undef HAVE_64_ATOMIC
+#undef __CLC_HAVE_INT_ATOMIC
+#undef __CLC_HAVE_FP_ATOMIC
+#undef __CLC_HAVE_64_ATOMIC
 
 #endif // __CLC_SCALAR
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_decl_legacy.inc b/libclc/opencl/include/clc/opencl/atomic/atomic_decl_legacy.inc
index e060e3aaea161..0cfd4c3eab5f6 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_decl_legacy.inc
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_decl_legacy.inc
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #define __CLC_DECLARE_ATOMIC(ADDRSPACE, TYPE)                                  \
-  _CLC_OVERLOAD _CLC_DECL TYPE FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
+  _CLC_OVERLOAD _CLC_DECL TYPE __CLC_FUNCTION(volatile ADDRSPACE TYPE *, TYPE);
 
 #define __CLC_DECLARE_ATOMIC_ADDRSPACE(TYPE)                                   \
   __CLC_DECLARE_ATOMIC(global, TYPE)                                           \
@@ -19,4 +19,4 @@ __CLC_DECLARE_ATOMIC_ADDRSPACE(uint)
 #undef __CLC_DECLARE_ATOMIC_ADDRSPACE
 #undef __CLC_DECLARE_ATOMIC
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h b/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h
index 3949bc13401f2..9d949825b58c3 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_exchange.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__
 
-#define FUNCTION atomic_exchange
+#define __CLC_FUNCTION atomic_exchange
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -17,6 +17,6 @@
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_EXCHANGE_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h
index 972c1fa69fe7b..bae5a7a7e19bb 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_add.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__
 
-#define FUNCTION atomic_fetch_add
+#define __CLC_FUNCTION atomic_fetch_add
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -17,6 +17,6 @@
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_ADD_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h
index fdac049a74d3f..9f9d2225f910e 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_and.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__
 
-#define FUNCTION atomic_fetch_and
+#define __CLC_FUNCTION atomic_fetch_and
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_AND_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h
index 513b60fec2727..bef102dc82f48 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_max.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__
 
-#define FUNCTION atomic_fetch_max
+#define __CLC_FUNCTION atomic_fetch_max
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -17,6 +17,6 @@
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MAX_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h
index c961c4a641656..d7e346dc44368 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_min.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__
 
-#define FUNCTION atomic_fetch_min
+#define __CLC_FUNCTION atomic_fetch_min
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -17,6 +17,6 @@
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_MIN_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h
index 25923e3647e36..aa00982e15a56 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_or.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__
 
-#define FUNCTION atomic_fetch_or
+#define __CLC_FUNCTION atomic_fetch_or
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_OR_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h
index b307c30a298b3..3d04ed7ba34f8 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_sub.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__
 
-#define FUNCTION atomic_fetch_sub
+#define __CLC_FUNCTION atomic_fetch_sub
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
@@ -17,6 +17,6 @@
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_SUB_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h
index 52510d018574d..2cdff08069025 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_fetch_xor.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__
 
-#define FUNCTION atomic_fetch_xor
+#define __CLC_FUNCTION atomic_fetch_xor
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_FETCH_XOR_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_load.h b/libclc/opencl/include/clc/opencl/atomic/atomic_load.h
index 3998a4de9452b..7db259b136ec8 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_load.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_load.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__
 
-#define FUNCTION atomic_load
+#define __CLC_FUNCTION atomic_load
 #define __CLC_NO_VALUE_ARG
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
@@ -19,6 +19,6 @@
 #include <clc/math/gentype.inc>
 
 #undef __CLC_NO_VALUE_ARG
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_LOAD_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_max.h b/libclc/opencl/include/clc/opencl/atomic/atomic_max.h
index 6b95ad7e68d94..e16f4ba7122dc 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_max.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_max.h
@@ -11,7 +11,7 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION atomic_max
+#define __CLC_FUNCTION atomic_max
 #include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_MAX_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_min.h b/libclc/opencl/include/clc/opencl/atomic/atomic_min.h
index c1dfacb40b746..d422388614c2a 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_min.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_min.h
@@ -11,7 +11,7 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION atomic_min
+#define __CLC_FUNCTION atomic_min
 #include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_MIN_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_or.h b/libclc/opencl/include/clc/opencl/atomic/atomic_or.h
index 30c32fe4889d5..53c9b1e3e89b3 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_or.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_or.h
@@ -11,7 +11,7 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION atomic_or
+#define __CLC_FUNCTION atomic_or
 #include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_OR_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_store.h b/libclc/opencl/include/clc/opencl/atomic/atomic_store.h
index 4893a5b88df03..b3cdfc6ffaeae 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_store.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_store.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__
 #define __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__
 
-#define FUNCTION atomic_store
+#define __CLC_FUNCTION atomic_store
 #define __CLC_RETURN_VOID
 
 #define __CLC_BODY <clc/opencl/atomic/atomic_decl.inc>
@@ -19,6 +19,6 @@
 #include <clc/math/gentype.inc>
 
 #undef __CLC_RETURN_VOID
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_STORE_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h b/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h
index 1e7ac5505b071..1b77bd7ced462 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_sub.h
@@ -11,7 +11,7 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION atomic_sub
+#define __CLC_FUNCTION atomic_sub
 #include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_SUB_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h b/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h
index 043d7825483e4..47d4a4638db34 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_xchg.h
@@ -11,10 +11,10 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION atomic_xchg
+#define __CLC_FUNCTION atomic_xchg
 
-_CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile local float *, float);
-_CLC_OVERLOAD _CLC_DECL float FUNCTION(volatile global float *, float);
+_CLC_OVERLOAD _CLC_DECL float __CLC_FUNCTION(volatile local float *, float);
+_CLC_OVERLOAD _CLC_DECL float __CLC_FUNCTION(volatile global float *, float);
 #include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_XCHG_H__
diff --git a/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h b/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h
index a9bee007b9344..9c808d802594e 100644
--- a/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h
+++ b/libclc/opencl/include/clc/opencl/atomic/atomic_xor.h
@@ -11,7 +11,7 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION atomic_xor
+#define __CLC_FUNCTION atomic_xor
 #include <clc/opencl/atomic/atomic_decl_legacy.inc>
 
 #endif // __CLC_OPENCL_ATOMIC_ATOMIC_XOR_H__
diff --git a/libclc/opencl/include/clc/opencl/common/degrees.h b/libclc/opencl/include/clc/opencl/common/degrees.h
index 5bd6f4e8f7fd5..8a3e77e559cfa 100644
--- a/libclc/opencl/include/clc/opencl/common/degrees.h
+++ b/libclc/opencl/include/clc/opencl/common/degrees.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_COMMON_DEGREES_H__
 #define __CLC_OPENCL_COMMON_DEGREES_H__
 
-#define FUNCTION degrees
+#define __CLC_FUNCTION degrees
 #define __CLC_BODY <clc/math/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_COMMON_DEGREES_H__
diff --git a/libclc/opencl/include/clc/opencl/common/mix.inc b/libclc/opencl/include/clc/opencl/common/mix.inc
index 2b8350f5baae1..cd32085467bbf 100644
--- a/libclc/opencl/include/clc/opencl/common/mix.inc
+++ b/libclc/opencl/include/clc/opencl/common/mix.inc
@@ -6,10 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE mix(__CLC_GENTYPE a, __CLC_GENTYPE b,
-                                          __CLC_GENTYPE c);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE mix(__CLC_GENTYPE a,
+                                                     __CLC_GENTYPE b,
+                                                     __CLC_GENTYPE c);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE mix(__CLC_GENTYPE a, __CLC_GENTYPE b,
-                                          __CLC_SCALAR_GENTYPE c);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE mix(__CLC_GENTYPE a,
+                                                     __CLC_GENTYPE b,
+                                                     __CLC_SCALAR_GENTYPE c);
 #endif
diff --git a/libclc/opencl/include/clc/opencl/common/radians.h b/libclc/opencl/include/clc/opencl/common/radians.h
index 3761a7e875be3..accef6eb62609 100644
--- a/libclc/opencl/include/clc/opencl/common/radians.h
+++ b/libclc/opencl/include/clc/opencl/common/radians.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_COMMON_RADIANS_H__
 #define __CLC_OPENCL_COMMON_RADIANS_H__
 
-#define FUNCTION radians
+#define __CLC_FUNCTION radians
 #define __CLC_BODY <clc/math/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_COMMON_RADIANS_H__
diff --git a/libclc/opencl/include/clc/opencl/common/sign.h b/libclc/opencl/include/clc/opencl/common/sign.h
index 619e68ff1e14d..58669b4e1fc9a 100644
--- a/libclc/opencl/include/clc/opencl/common/sign.h
+++ b/libclc/opencl/include/clc/opencl/common/sign.h
@@ -9,9 +9,9 @@
 #ifndef __CLC_OPENCL_COMMON_SIGN_H__
 #define __CLC_OPENCL_COMMON_SIGN_H__
 
-#define FUNCTION sign
+#define __CLC_FUNCTION sign
 #define __CLC_BODY <clc/math/unary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_COMMON_SIGN_H__
diff --git a/libclc/opencl/include/clc/opencl/common/smoothstep.inc b/libclc/opencl/include/clc/opencl/common/smoothstep.inc
index 4547483364ba2..f1402e3c80017 100644
--- a/libclc/opencl/include/clc/opencl/common/smoothstep.inc
+++ b/libclc/opencl/include/clc/opencl/common/smoothstep.inc
@@ -6,13 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE smoothstep(__CLC_GENTYPE edge0,
-                                                 __CLC_GENTYPE edge1,
-                                                 __CLC_GENTYPE x);
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE smoothstep(float edge0, float edge1,
-                                                 __CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE smoothstep(__CLC_GENTYPE edge0,
+                                                            __CLC_GENTYPE edge1,
+                                                            __CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE smoothstep(float edge0,
+                                                            float edge1,
+                                                            __CLC_GENTYPE x);
 
 #ifdef cl_khr_fp64
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE smoothstep(double edge0, double edge1,
-                                                 __CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE smoothstep(double edge0,
+                                                            double edge1,
+                                                            __CLC_GENTYPE x);
 #endif
diff --git a/libclc/opencl/include/clc/opencl/common/step.inc b/libclc/opencl/include/clc/opencl/common/step.inc
index 86d4ed03dbcab..ab4b61e606feb 100644
--- a/libclc/opencl/include/clc/opencl/common/step.inc
+++ b/libclc/opencl/include/clc/opencl/common/step.inc
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE step(__CLC_GENTYPE edge, __CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE step(__CLC_GENTYPE edge,
+                                                      __CLC_GENTYPE x);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE step(__CLC_SCALAR_GENTYPE edge,
-                                           __CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE step(__CLC_SCALAR_GENTYPE edge,
+                                                      __CLC_GENTYPE x);
 #endif
diff --git a/libclc/opencl/include/clc/opencl/geometric/cross.h b/libclc/opencl/include/clc/opencl/geometric/cross.h
index be137965aea4b..10e8f5641e730 100644
--- a/libclc/opencl/include/clc/opencl/geometric/cross.h
+++ b/libclc/opencl/include/clc/opencl/geometric/cross.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_OPENCL_GEOMETRIC_CROSS_H__
 #define __CLC_OPENCL_GEOMETRIC_CROSS_H__
 
-_CLC_OVERLOAD _CLC_DECL float3 cross(float3 p0, float3 p1);
-_CLC_OVERLOAD _CLC_DECL float4 cross(float4 p0, float4 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL float3 cross(float3 p0, float3 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL float4 cross(float4 p0, float4 p1);
 
 #ifdef cl_khr_fp64
-_CLC_OVERLOAD _CLC_DECL double3 cross(double3 p0, double3 p1);
-_CLC_OVERLOAD _CLC_DECL double4 cross(double4 p0, double4 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL double3 cross(double3 p0, double3 p1);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL double4 cross(double4 p0, double4 p1);
 #endif
 
 #endif // __CLC_OPENCL_GEOMETRIC_CROSS_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/distance.h b/libclc/opencl/include/clc/opencl/geometric/distance.h
index f9c219c44a49b..dc5abf11c10c0 100644
--- a/libclc/opencl/include/clc/opencl/geometric/distance.h
+++ b/libclc/opencl/include/clc/opencl/geometric/distance.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_GEOMETRIC_DISTANCE_H__
 #define __CLC_OPENCL_GEOMETRIC_DISTANCE_H__
 
-#define FUNCTION distance
+#define __CLC_FUNCTION distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_GEOMETRIC_DISTANCE_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/dot.h b/libclc/opencl/include/clc/opencl/geometric/dot.h
index 0b391a10aa075..8cd702961fa41 100644
--- a/libclc/opencl/include/clc/opencl/geometric/dot.h
+++ b/libclc/opencl/include/clc/opencl/geometric/dot.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_GEOMETRIC_DOT_H__
 #define __CLC_OPENCL_GEOMETRIC_DOT_H__
 
-#define FUNCTION dot
+#define __CLC_FUNCTION dot
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_GEOMETRIC_DOT_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/fast_distance.h b/libclc/opencl/include/clc/opencl/geometric/fast_distance.h
index 95b4966dfac1a..3eb7d70cdddd2 100644
--- a/libclc/opencl/include/clc/opencl/geometric/fast_distance.h
+++ b/libclc/opencl/include/clc/opencl/geometric/fast_distance.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_OPENCL_GEOMETRIC_FAST_DISTANCE_H__
 #define __CLC_OPENCL_GEOMETRIC_FAST_DISTANCE_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION fast_distance
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION fast_distance
 #define __CLC_BODY <clc/geometric/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_GEOMETRIC_FAST_DISTANCE_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/fast_length.h b/libclc/opencl/include/clc/opencl/geometric/fast_length.h
index 78ad1fea981c3..ff383de9ec72d 100644
--- a/libclc/opencl/include/clc/opencl/geometric/fast_length.h
+++ b/libclc/opencl/include/clc/opencl/geometric/fast_length.h
@@ -9,12 +9,12 @@
 #ifndef __CLC_OPENCL_GEOMETRIC_FAST_LENGTH_H__
 #define __CLC_OPENCL_GEOMETRIC_FAST_LENGTH_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION fast_length
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION fast_length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_GEOMETRIC_FAST_LENGTH_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h b/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h
index 7a70dd5df5196..fb677d7d3f791 100644
--- a/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h
+++ b/libclc/opencl/include/clc/opencl/geometric/fast_normalize.h
@@ -9,14 +9,14 @@
 #ifndef __CLC_OPENCL_GEOMETRIC_FAST_NORMALIZE_H__
 #define __CLC_OPENCL_GEOMETRIC_FAST_NORMALIZE_H__
 
-#define __FLOAT_ONLY
-#define FUNCTION fast_normalize
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION fast_normalize
 #define __CLC_GEOMETRIC_RET_GENTYPE
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 #undef __CLC_GEOMETRIC_RET_GENTYPE
 
 #endif // __CLC_OPENCL_GEOMETRIC_FAST_NORMALIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/length.h b/libclc/opencl/include/clc/opencl/geometric/length.h
index 5dd6ff5a9cad4..686d554d60420 100644
--- a/libclc/opencl/include/clc/opencl/geometric/length.h
+++ b/libclc/opencl/include/clc/opencl/geometric/length.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_GEOMETRIC_LENGTH_H__
 #define __CLC_OPENCL_GEOMETRIC_LENGTH_H__
 
-#define FUNCTION length
+#define __CLC_FUNCTION length
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_GEOMETRIC_LENGTH_H__
diff --git a/libclc/opencl/include/clc/opencl/geometric/normalize.h b/libclc/opencl/include/clc/opencl/geometric/normalize.h
index 7d3e8fa98f8ed..3ed6dae53269e 100644
--- a/libclc/opencl/include/clc/opencl/geometric/normalize.h
+++ b/libclc/opencl/include/clc/opencl/geometric/normalize.h
@@ -9,13 +9,13 @@
 #ifndef __CLC_OPENCL_GEOMETRIC_NORMALIZE_H__
 #define __CLC_OPENCL_GEOMETRIC_NORMALIZE_H__
 
-#define FUNCTION normalize
+#define __CLC_FUNCTION normalize
 #define __CLC_GEOMETRIC_RET_GENTYPE
 #define __CLC_BODY <clc/geometric/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
 #undef __CLC_GEOMETRIC_RET_GENTYPE
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_GEOMETRIC_NORMALIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/abs.inc b/libclc/opencl/include/clc/opencl/integer/abs.inc
index 6620ce36f8c6c..352babb393212 100644
--- a/libclc/opencl/include/clc/opencl/integer/abs.inc
+++ b/libclc/opencl/include/clc/opencl/integer/abs.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_U_GENTYPE abs(__CLC_GENTYPE x);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_U_GENTYPE abs(__CLC_GENTYPE x);
diff --git a/libclc/opencl/include/clc/opencl/integer/abs_diff.inc b/libclc/opencl/include/clc/opencl/integer/abs_diff.inc
index 41873854021eb..1f8fd635fad9e 100644
--- a/libclc/opencl/include/clc/opencl/integer/abs_diff.inc
+++ b/libclc/opencl/include/clc/opencl/integer/abs_diff.inc
@@ -6,5 +6,5 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_U_GENTYPE abs_diff(__CLC_GENTYPE x,
-                                                 __CLC_GENTYPE y);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_U_GENTYPE abs_diff(__CLC_GENTYPE x,
+                                                            __CLC_GENTYPE y);
diff --git a/libclc/opencl/include/clc/opencl/integer/add_sat.h b/libclc/opencl/include/clc/opencl/integer/add_sat.h
index cd2c17a096299..e77eee0c0b05a 100644
--- a/libclc/opencl/include/clc/opencl/integer/add_sat.h
+++ b/libclc/opencl/include/clc/opencl/integer/add_sat.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION add_sat
+#define __CLC_FUNCTION add_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_ADD_SAT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/bit_reverse.h b/libclc/opencl/include/clc/opencl/integer/bit_reverse.h
index 46b589557631d..9c3e45d08519a 100644
--- a/libclc/opencl/include/clc/opencl/integer/bit_reverse.h
+++ b/libclc/opencl/include/clc/opencl/integer/bit_reverse.h
@@ -13,12 +13,12 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION bit_reverse
+#define __CLC_FUNCTION bit_reverse
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // cl_khr_extended_bit_ops
 
diff --git a/libclc/opencl/include/clc/opencl/integer/bitfield_extract_signed.h b/libclc/opencl/include/clc/opencl/integer/bitfield_extract_signed.h
index 0a902b2a21d6d..c6e48d9820dba 100644
--- a/libclc/opencl/include/clc/opencl/integer/bitfield_extract_signed.h
+++ b/libclc/opencl/include/clc/opencl/integer/bitfield_extract_signed.h
@@ -13,14 +13,14 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION bitfield_extract_signed
-#define __RETTYPE __CLC_S_GENTYPE
+#define __CLC_FUNCTION bitfield_extract_signed
+#define __CLC_RETTYPE __CLC_S_GENTYPE
 
 #define __CLC_BODY <clc/integer/clc_bitfield_extract_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef __RETTYPE
-#undef FUNCTION
+#undef __CLC_RETTYPE
+#undef __CLC_FUNCTION
 
 #endif // cl_khr_extended_bit_ops
 
diff --git a/libclc/opencl/include/clc/opencl/integer/bitfield_extract_unsigned.h b/libclc/opencl/include/clc/opencl/integer/bitfield_extract_unsigned.h
index 28064c08b113e..63dd4368f97a6 100644
--- a/libclc/opencl/include/clc/opencl/integer/bitfield_extract_unsigned.h
+++ b/libclc/opencl/include/clc/opencl/integer/bitfield_extract_unsigned.h
@@ -13,14 +13,14 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION bitfield_extract_unsigned
-#define __RETTYPE __CLC_U_GENTYPE
+#define __CLC_FUNCTION bitfield_extract_unsigned
+#define __CLC_RETTYPE __CLC_U_GENTYPE
 
 #define __CLC_BODY <clc/integer/clc_bitfield_extract_decl.inc>
 #include <clc/integer/gentype.inc>
 
-#undef __RETTYPE
-#undef FUNCTION
+#undef __CLC_RETTYPE
+#undef __CLC_FUNCTION
 
 #endif // cl_khr_extended_bit_ops
 
diff --git a/libclc/opencl/include/clc/opencl/integer/bitfield_insert.h b/libclc/opencl/include/clc/opencl/integer/bitfield_insert.h
index e77d7a4f0b957..40c7d59825852 100644
--- a/libclc/opencl/include/clc/opencl/integer/bitfield_insert.h
+++ b/libclc/opencl/include/clc/opencl/integer/bitfield_insert.h
@@ -16,7 +16,7 @@
 #define __CLC_BODY <clc/integer/clc_bitfield_insert.inc>
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // cl_khr_extended_bit_ops
 
diff --git a/libclc/opencl/include/clc/opencl/integer/clz.h b/libclc/opencl/include/clc/opencl/integer/clz.h
index 77032ceeaf0f4..1b47e975b52a1 100644
--- a/libclc/opencl/include/clc/opencl/integer/clz.h
+++ b/libclc/opencl/include/clc/opencl/integer/clz.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION clz
+#define __CLC_FUNCTION clz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_CLZ_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/ctz.h b/libclc/opencl/include/clc/opencl/integer/ctz.h
index b861225846ee4..ba01161407303 100644
--- a/libclc/opencl/include/clc/opencl/integer/ctz.h
+++ b/libclc/opencl/include/clc/opencl/integer/ctz.h
@@ -13,12 +13,12 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION ctz
+#define __CLC_FUNCTION ctz
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __OPENCL_C_VERSION__ >= CL_VERSION_2_0
 
diff --git a/libclc/opencl/include/clc/opencl/integer/hadd.h b/libclc/opencl/include/clc/opencl/integer/hadd.h
index c4c8f7daf4014..f200c7019435b 100644
--- a/libclc/opencl/include/clc/opencl/integer/hadd.h
+++ b/libclc/opencl/include/clc/opencl/integer/hadd.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION hadd
+#define __CLC_FUNCTION hadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_HADD_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mad24.h b/libclc/opencl/include/clc/opencl/integer/mad24.h
index daee9a0383477..5bf3d385a432b 100644
--- a/libclc/opencl/include/clc/opencl/integer/mad24.h
+++ b/libclc/opencl/include/clc/opencl/integer/mad24.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION mad24
+#define __CLC_FUNCTION mad24
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_MAD24_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mad_hi.h b/libclc/opencl/include/clc/opencl/integer/mad_hi.h
index bbcdcd9c82ebd..98b525ba82689 100644
--- a/libclc/opencl/include/clc/opencl/integer/mad_hi.h
+++ b/libclc/opencl/include/clc/opencl/integer/mad_hi.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION mad_hi
+#define __CLC_FUNCTION mad_hi
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_MAD_HI_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mad_sat.h b/libclc/opencl/include/clc/opencl/integer/mad_sat.h
index 28045477975ed..9e2afe4f1d9f1 100644
--- a/libclc/opencl/include/clc/opencl/integer/mad_sat.h
+++ b/libclc/opencl/include/clc/opencl/integer/mad_sat.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION mad_sat
+#define __CLC_FUNCTION mad_sat
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_MAD_SAT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mul24.h b/libclc/opencl/include/clc/opencl/integer/mul24.h
index 6aa799a28b7ec..434da55a71731 100644
--- a/libclc/opencl/include/clc/opencl/integer/mul24.h
+++ b/libclc/opencl/include/clc/opencl/integer/mul24.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION mul24
+#define __CLC_FUNCTION mul24
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype24.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_MUL24_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/mul_hi.h b/libclc/opencl/include/clc/opencl/integer/mul_hi.h
index ed7ac7319cab8..7a9317f94cd24 100644
--- a/libclc/opencl/include/clc/opencl/integer/mul_hi.h
+++ b/libclc/opencl/include/clc/opencl/integer/mul_hi.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION mul_hi
+#define __CLC_FUNCTION mul_hi
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_MUL_HI_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/popcount.h b/libclc/opencl/include/clc/opencl/integer/popcount.h
index 43d5bc0f0b335..112640c7ce635 100644
--- a/libclc/opencl/include/clc/opencl/integer/popcount.h
+++ b/libclc/opencl/include/clc/opencl/integer/popcount.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION popcount
+#define __CLC_FUNCTION popcount
 #define __CLC_BODY <clc/shared/unary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_POPCOUNT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/rhadd.h b/libclc/opencl/include/clc/opencl/integer/rhadd.h
index 2c93c100cf02f..5ed8b27df8fce 100644
--- a/libclc/opencl/include/clc/opencl/integer/rhadd.h
+++ b/libclc/opencl/include/clc/opencl/integer/rhadd.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_INTEGER_RHADD_H__
 #define __CLC_OPENCL_INTEGER_RHADD_H__
 
-#define FUNCTION rhadd
+#define __CLC_FUNCTION rhadd
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_RHADD_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/rotate.h b/libclc/opencl/include/clc/opencl/integer/rotate.h
index 156155ae0638a..cc92a1254fed3 100644
--- a/libclc/opencl/include/clc/opencl/integer/rotate.h
+++ b/libclc/opencl/include/clc/opencl/integer/rotate.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION rotate
+#define __CLC_FUNCTION rotate
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_ROTATE_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/sub_sat.h b/libclc/opencl/include/clc/opencl/integer/sub_sat.h
index a7861ee1ac4a1..1ff26d94778cb 100644
--- a/libclc/opencl/include/clc/opencl/integer/sub_sat.h
+++ b/libclc/opencl/include/clc/opencl/integer/sub_sat.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_INTEGER_SUB_SAT_H__
 #define __CLC_OPENCL_INTEGER_SUB_SAT_H__
 
-#define FUNCTION sub_sat
+#define __CLC_FUNCTION sub_sat
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/integer/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_INTEGER_SUB_SAT_H__
diff --git a/libclc/opencl/include/clc/opencl/integer/upsample.h b/libclc/opencl/include/clc/opencl/integer/upsample.h
index a713f2b392f27..4432d49ccb062 100644
--- a/libclc/opencl/include/clc/opencl/integer/upsample.h
+++ b/libclc/opencl/include/clc/opencl/integer/upsample.h
@@ -12,7 +12,7 @@
 #include <clc/opencl/opencl-base.h>
 
 #define __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE)                       \
-  _CLC_OVERLOAD _CLC_DECL BGENTYPE upsample(GENTYPE hi, UGENTYPE lo);
+  _CLC_OVERLOAD _CLC_CONST _CLC_DECL BGENTYPE upsample(GENTYPE hi, UGENTYPE lo);
 
 #define __CLC_UPSAMPLE_VEC(BGENTYPE, GENTYPE, UGENTYPE)                        \
   __CLC_UPSAMPLE_DECL(BGENTYPE, GENTYPE, UGENTYPE)                             \
diff --git a/libclc/opencl/include/clc/opencl/math/acos.h b/libclc/opencl/include/clc/opencl/math/acos.h
index f9e9b44bb9041..aa0d7af13ee9a 100644
--- a/libclc/opencl/include/clc/opencl/math/acos.h
+++ b/libclc/opencl/include/clc/opencl/math/acos.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ACOS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION acos
+#define __CLC_FUNCTION acos
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ACOS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/acosh.h b/libclc/opencl/include/clc/opencl/math/acosh.h
index 241b305e6bc87..333ab6986d8c3 100644
--- a/libclc/opencl/include/clc/opencl/math/acosh.h
+++ b/libclc/opencl/include/clc/opencl/math/acosh.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ACOSH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION acosh
+#define __CLC_FUNCTION acosh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ACOSH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/acospi.h b/libclc/opencl/include/clc/opencl/math/acospi.h
index 7014889dfb7ef..fd838acd5e522 100644
--- a/libclc/opencl/include/clc/opencl/math/acospi.h
+++ b/libclc/opencl/include/clc/opencl/math/acospi.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ACOSPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION acospi
+#define __CLC_FUNCTION acospi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ACOSPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/asin.h b/libclc/opencl/include/clc/opencl/math/asin.h
index 1ba1f459a9024..de3b8ad219d1b 100644
--- a/libclc/opencl/include/clc/opencl/math/asin.h
+++ b/libclc/opencl/include/clc/opencl/math/asin.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ASIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION asin
+#define __CLC_FUNCTION asin
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ASIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/asinh.h b/libclc/opencl/include/clc/opencl/math/asinh.h
index 9a530a51fa1ee..a550ec8dee4ca 100644
--- a/libclc/opencl/include/clc/opencl/math/asinh.h
+++ b/libclc/opencl/include/clc/opencl/math/asinh.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ASINH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION asinh
+#define __CLC_FUNCTION asinh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ASINH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/asinpi.h b/libclc/opencl/include/clc/opencl/math/asinpi.h
index 844848fff0433..cb6133ddc3f51 100644
--- a/libclc/opencl/include/clc/opencl/math/asinpi.h
+++ b/libclc/opencl/include/clc/opencl/math/asinpi.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ASINPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION asinpi
+#define __CLC_FUNCTION asinpi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ASINPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atan.h b/libclc/opencl/include/clc/opencl/math/atan.h
index f2312b8fb0737..165bbc3dfb18e 100644
--- a/libclc/opencl/include/clc/opencl/math/atan.h
+++ b/libclc/opencl/include/clc/opencl/math/atan.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ATAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION atan
+#define __CLC_FUNCTION atan
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ATAN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atan2.h b/libclc/opencl/include/clc/opencl/math/atan2.h
index 72412d44aafb1..2874876707098 100644
--- a/libclc/opencl/include/clc/opencl/math/atan2.h
+++ b/libclc/opencl/include/clc/opencl/math/atan2.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_MATH_ATAN2_H__
 #define __CLC_OPENCL_MATH_ATAN2_H__
 
-#define FUNCTION atan2
+#define __CLC_FUNCTION atan2
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ATAN2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atan2pi.h b/libclc/opencl/include/clc/opencl/math/atan2pi.h
index f12a19776ef23..7c09a94fffa82 100644
--- a/libclc/opencl/include/clc/opencl/math/atan2pi.h
+++ b/libclc/opencl/include/clc/opencl/math/atan2pi.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_MATH_ATAN2PI_H__
 #define __CLC_OPENCL_MATH_ATAN2PI_H__
 
-#define FUNCTION atan2pi
+#define __CLC_FUNCTION atan2pi
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ATAN2PI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atanh.h b/libclc/opencl/include/clc/opencl/math/atanh.h
index 147a76b543e61..ccfe41fdb2cf1 100644
--- a/libclc/opencl/include/clc/opencl/math/atanh.h
+++ b/libclc/opencl/include/clc/opencl/math/atanh.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ATANH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION atanh
+#define __CLC_FUNCTION atanh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ATANH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/atanpi.h b/libclc/opencl/include/clc/opencl/math/atanpi.h
index 12e9ea2501705..8041cc3e813e7 100644
--- a/libclc/opencl/include/clc/opencl/math/atanpi.h
+++ b/libclc/opencl/include/clc/opencl/math/atanpi.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ATANPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION atanpi
+#define __CLC_FUNCTION atanpi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ATANPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/cbrt.h b/libclc/opencl/include/clc/opencl/math/cbrt.h
index fdb408ac96858..3beb5b4b596af 100644
--- a/libclc/opencl/include/clc/opencl/math/cbrt.h
+++ b/libclc/opencl/include/clc/opencl/math/cbrt.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_CBRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION cbrt
+#define __CLC_FUNCTION cbrt
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_CBRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/ceil.h b/libclc/opencl/include/clc/opencl/math/ceil.h
index dad2ea4f69807..ab32419225ce9 100644
--- a/libclc/opencl/include/clc/opencl/math/ceil.h
+++ b/libclc/opencl/include/clc/opencl/math/ceil.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_CEIL_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION ceil
+#define __CLC_FUNCTION ceil
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_CEIL_H__
diff --git a/libclc/opencl/include/clc/opencl/math/copysign.h b/libclc/opencl/include/clc/opencl/math/copysign.h
index 10d6c90edf81f..626ce2e01cc34 100644
--- a/libclc/opencl/include/clc/opencl/math/copysign.h
+++ b/libclc/opencl/include/clc/opencl/math/copysign.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_MATH_COPYSIGN_H__
 #define __CLC_OPENCL_MATH_COPYSIGN_H__
 
-#define FUNCTION copysign
+#define __CLC_FUNCTION copysign
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_COPYSIGN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/cos.h b/libclc/opencl/include/clc/opencl/math/cos.h
index 4e3aec6c69628..8d26f0f78d2d8 100644
--- a/libclc/opencl/include/clc/opencl/math/cos.h
+++ b/libclc/opencl/include/clc/opencl/math/cos.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_COS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION cos
+#define __CLC_FUNCTION cos
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_COS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/cosh.h b/libclc/opencl/include/clc/opencl/math/cosh.h
index f316433a23514..2afe77fae6c70 100644
--- a/libclc/opencl/include/clc/opencl/math/cosh.h
+++ b/libclc/opencl/include/clc/opencl/math/cosh.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_COSH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION cosh
+#define __CLC_FUNCTION cosh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_COSH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/cospi.h b/libclc/opencl/include/clc/opencl/math/cospi.h
index b7dfb0dbafb58..8e848ac0f883b 100644
--- a/libclc/opencl/include/clc/opencl/math/cospi.h
+++ b/libclc/opencl/include/clc/opencl/math/cospi.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_COSPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION cospi
+#define __CLC_FUNCTION cospi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_COSPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/erf.h b/libclc/opencl/include/clc/opencl/math/erf.h
index 8ce262f276e39..71c211e693bde 100644
--- a/libclc/opencl/include/clc/opencl/math/erf.h
+++ b/libclc/opencl/include/clc/opencl/math/erf.h
@@ -12,10 +12,10 @@
 #undef erfc
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION erf
+#define __CLC_FUNCTION erf
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ERF_H__
diff --git a/libclc/opencl/include/clc/opencl/math/erfc.h b/libclc/opencl/include/clc/opencl/math/erfc.h
index 26afe1315df60..3939108820451 100644
--- a/libclc/opencl/include/clc/opencl/math/erfc.h
+++ b/libclc/opencl/include/clc/opencl/math/erfc.h
@@ -12,10 +12,10 @@
 #undef erfc
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION erfc
+#define __CLC_FUNCTION erfc
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ERFC_H__
diff --git a/libclc/opencl/include/clc/opencl/math/exp.h b/libclc/opencl/include/clc/opencl/math/exp.h
index fb8f674a8c7e5..acf6d4885dbd3 100644
--- a/libclc/opencl/include/clc/opencl/math/exp.h
+++ b/libclc/opencl/include/clc/opencl/math/exp.h
@@ -12,10 +12,10 @@
 #undef exp
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION exp
+#define __CLC_FUNCTION exp
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_EXP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/exp10.h b/libclc/opencl/include/clc/opencl/math/exp10.h
index cfd5c6df9d15c..11dcae74e0613 100644
--- a/libclc/opencl/include/clc/opencl/math/exp10.h
+++ b/libclc/opencl/include/clc/opencl/math/exp10.h
@@ -12,10 +12,10 @@
 #undef exp10
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION exp10
+#define __CLC_FUNCTION exp10
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_EXP10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/exp2.h b/libclc/opencl/include/clc/opencl/math/exp2.h
index db0405eb13341..e220654ac72e7 100644
--- a/libclc/opencl/include/clc/opencl/math/exp2.h
+++ b/libclc/opencl/include/clc/opencl/math/exp2.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_EXP2_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION exp2
+#define __CLC_FUNCTION exp2
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_EXP2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/expm1.h b/libclc/opencl/include/clc/opencl/math/expm1.h
index 0b3628d48468e..556dbea8fae02 100644
--- a/libclc/opencl/include/clc/opencl/math/expm1.h
+++ b/libclc/opencl/include/clc/opencl/math/expm1.h
@@ -12,10 +12,10 @@
 #undef exp
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION expm1
+#define __CLC_FUNCTION expm1
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_EXPM1_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fabs.h b/libclc/opencl/include/clc/opencl/math/fabs.h
index f20016f4eb8ba..ad8c05198a705 100644
--- a/libclc/opencl/include/clc/opencl/math/fabs.h
+++ b/libclc/opencl/include/clc/opencl/math/fabs.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_FABS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION fabs
+#define __CLC_FUNCTION fabs
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_FABS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fdim.h b/libclc/opencl/include/clc/opencl/math/fdim.h
index 82f70fd832361..ed30904e3c247 100644
--- a/libclc/opencl/include/clc/opencl/math/fdim.h
+++ b/libclc/opencl/include/clc/opencl/math/fdim.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_MATH_FDIM_H__
 #define __CLC_OPENCL_MATH_FDIM_H__
 
-#define FUNCTION fdim
+#define __CLC_FUNCTION fdim
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_FDIM_H__
diff --git a/libclc/opencl/include/clc/opencl/math/floor.h b/libclc/opencl/include/clc/opencl/math/floor.h
index 1930aeca74ff0..8d7a0358c4652 100644
--- a/libclc/opencl/include/clc/opencl/math/floor.h
+++ b/libclc/opencl/include/clc/opencl/math/floor.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_FLOOR_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION floor
+#define __CLC_FUNCTION floor
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_FLOOR_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fma.h b/libclc/opencl/include/clc/opencl/math/fma.h
index 6fed3fc9b2560..7a002318384a9 100644
--- a/libclc/opencl/include/clc/opencl/math/fma.h
+++ b/libclc/opencl/include/clc/opencl/math/fma.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_FMA_H__
 
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
-#define FUNCTION fma
+#define __CLC_FUNCTION fma
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_FMA_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fmax.h b/libclc/opencl/include/clc/opencl/math/fmax.h
index 84c754e2c0c71..8ada84403e0ab 100644
--- a/libclc/opencl/include/clc/opencl/math/fmax.h
+++ b/libclc/opencl/include/clc/opencl/math/fmax.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_FMAX_H__
 
 #define __CLC_BODY <clc/math/binary_decl_with_scalar_second_arg.inc>
-#define FUNCTION fmax
+#define __CLC_FUNCTION fmax
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_FMAX_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fmin.h b/libclc/opencl/include/clc/opencl/math/fmin.h
index b35573e4f6337..3a464a63db735 100644
--- a/libclc/opencl/include/clc/opencl/math/fmin.h
+++ b/libclc/opencl/include/clc/opencl/math/fmin.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_FMIN_H__
 
 #define __CLC_BODY <clc/math/binary_decl_with_scalar_second_arg.inc>
-#define FUNCTION fmin
+#define __CLC_FUNCTION fmin
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_FMIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fmod.h b/libclc/opencl/include/clc/opencl/math/fmod.h
index 13515006fd2be..f826c8a6f6c3e 100644
--- a/libclc/opencl/include/clc/opencl/math/fmod.h
+++ b/libclc/opencl/include/clc/opencl/math/fmod.h
@@ -9,9 +9,9 @@
 #ifndef __CLC_OPENCL_MATH_FMOD_H__
 #define __CLC_OPENCL_MATH_FMOD_H__
 
-#define FUNCTION fmod
+#define __CLC_FUNCTION fmod
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_FMOD_H__
diff --git a/libclc/opencl/include/clc/opencl/math/fract.h b/libclc/opencl/include/clc/opencl/math/fract.h
index de953f78c5940..342280f70ed6c 100644
--- a/libclc/opencl/include/clc/opencl/math/fract.h
+++ b/libclc/opencl/include/clc/opencl/math/fract.h
@@ -9,9 +9,9 @@
 #ifndef __CLC_OPENCL_MATH_FRACT_H__
 #define __CLC_OPENCL_MATH_FRACT_H__
 
-#define FUNCTION fract
+#define __CLC_FUNCTION fract
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_FRACT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/frexp.h b/libclc/opencl/include/clc/opencl/math/frexp.h
index c7ed7cbd1cbc4..9f0701aff10a2 100644
--- a/libclc/opencl/include/clc/opencl/math/frexp.h
+++ b/libclc/opencl/include/clc/opencl/math/frexp.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_OPENCL_MATH_FREXP_H__
 #define __CLC_OPENCL_MATH_FREXP_H__
 
-#define FUNCTION frexp
+#define __CLC_FUNCTION frexp
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_FREXP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_cos.h b/libclc/opencl/include/clc/opencl/math/half_cos.h
index 5ab0b69ca4224..9f95f5d7e2775 100644
--- a/libclc/opencl/include/clc/opencl/math/half_cos.h
+++ b/libclc/opencl/include/clc/opencl/math/half_cos.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_COS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_cos
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_cos
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_COS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_divide.h b/libclc/opencl/include/clc/opencl/math/half_divide.h
index ff506d0c9fd6d..c61fb08e2119d 100644
--- a/libclc/opencl/include/clc/opencl/math/half_divide.h
+++ b/libclc/opencl/include/clc/opencl/math/half_divide.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_HALF_DIVIDE_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION half_divide
+#define __CLC_FUNCTION half_divide
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_DIVIDE_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_exp.h b/libclc/opencl/include/clc/opencl/math/half_exp.h
index 22780c144923c..53b75a62f4dfe 100644
--- a/libclc/opencl/include/clc/opencl/math/half_exp.h
+++ b/libclc/opencl/include/clc/opencl/math/half_exp.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_EXP_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_exp
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_exp
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_EXP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_exp10.h b/libclc/opencl/include/clc/opencl/math/half_exp10.h
index 1a8195e67eee0..88b05a4df0811 100644
--- a/libclc/opencl/include/clc/opencl/math/half_exp10.h
+++ b/libclc/opencl/include/clc/opencl/math/half_exp10.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_EXP10_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_exp10
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_exp10
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_EXP10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_exp2.h b/libclc/opencl/include/clc/opencl/math/half_exp2.h
index 2ca69aa1ea076..e440b996e58ab 100644
--- a/libclc/opencl/include/clc/opencl/math/half_exp2.h
+++ b/libclc/opencl/include/clc/opencl/math/half_exp2.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_EXP2_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_exp2
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_exp2
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_EXP2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_log.h b/libclc/opencl/include/clc/opencl/math/half_log.h
index d2eef868790f5..d4432312c6aff 100644
--- a/libclc/opencl/include/clc/opencl/math/half_log.h
+++ b/libclc/opencl/include/clc/opencl/math/half_log.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_LOG_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_log
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_log
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_LOG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_log10.h b/libclc/opencl/include/clc/opencl/math/half_log10.h
index e0f85fff3f3ea..f219f950f9492 100644
--- a/libclc/opencl/include/clc/opencl/math/half_log10.h
+++ b/libclc/opencl/include/clc/opencl/math/half_log10.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_LOG10_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_log10
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_log10
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_LOG10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_log2.h b/libclc/opencl/include/clc/opencl/math/half_log2.h
index ba0105d883568..932f546e87612 100644
--- a/libclc/opencl/include/clc/opencl/math/half_log2.h
+++ b/libclc/opencl/include/clc/opencl/math/half_log2.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_LOG2_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_log2
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_log2
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_LOG2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_powr.h b/libclc/opencl/include/clc/opencl/math/half_powr.h
index 4b792c0c6cba4..e9af720aa5ac5 100644
--- a/libclc/opencl/include/clc/opencl/math/half_powr.h
+++ b/libclc/opencl/include/clc/opencl/math/half_powr.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_HALF_POWR_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION half_powr
+#define __CLC_FUNCTION half_powr
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_POWR_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_recip.h b/libclc/opencl/include/clc/opencl/math/half_recip.h
index 3425df4e476d8..6fb7f02138a6e 100644
--- a/libclc/opencl/include/clc/opencl/math/half_recip.h
+++ b/libclc/opencl/include/clc/opencl/math/half_recip.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_RECIP_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_recip
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_recip
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_RECIP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_rsqrt.h b/libclc/opencl/include/clc/opencl/math/half_rsqrt.h
index e22a7375cacda..a130d3bb080a1 100644
--- a/libclc/opencl/include/clc/opencl/math/half_rsqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/half_rsqrt.h
@@ -10,9 +10,9 @@
 #define __CLC_OPENCL_MATH_HALF_RSQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_rsqrt
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_rsqrt
+#define __CLC_FLOAT_ONLY
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_RSQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_sin.h b/libclc/opencl/include/clc/opencl/math/half_sin.h
index cb0462c45436f..cde9549bd3b35 100644
--- a/libclc/opencl/include/clc/opencl/math/half_sin.h
+++ b/libclc/opencl/include/clc/opencl/math/half_sin.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_SIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_sin
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_sin
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_SIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_sqrt.h b/libclc/opencl/include/clc/opencl/math/half_sqrt.h
index f755fee7b4be2..017bed1c066e4 100644
--- a/libclc/opencl/include/clc/opencl/math/half_sqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/half_sqrt.h
@@ -10,9 +10,9 @@
 #define __CLC_OPENCL_MATH_HALF_SQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_sqrt
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_sqrt
+#define __CLC_FLOAT_ONLY
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_SQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/half_tan.h b/libclc/opencl/include/clc/opencl/math/half_tan.h
index 91b5b11ae1789..b8e71fcba43db 100644
--- a/libclc/opencl/include/clc/opencl/math/half_tan.h
+++ b/libclc/opencl/include/clc/opencl/math/half_tan.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_HALF_TAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION half_tan
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION half_tan
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HALF_TAN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/hypot.h b/libclc/opencl/include/clc/opencl/math/hypot.h
index 76ff6ef0655a9..53bef98772e41 100644
--- a/libclc/opencl/include/clc/opencl/math/hypot.h
+++ b/libclc/opencl/include/clc/opencl/math/hypot.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_MATH_HYPOT_H__
 #define __CLC_OPENCL_MATH_HYPOT_H__
 
-#define FUNCTION hypot
+#define __CLC_FUNCTION hypot
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_HYPOT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/ilogb.h b/libclc/opencl/include/clc/opencl/math/ilogb.h
index d9752a644bee4..29d92632f7f50 100644
--- a/libclc/opencl/include/clc/opencl/math/ilogb.h
+++ b/libclc/opencl/include/clc/opencl/math/ilogb.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_MATH_ILOGB_H__
 #define __CLC_OPENCL_MATH_ILOGB_H__
 
-#define FUNCTION ilogb
+#define __CLC_FUNCTION ilogb
 #define __CLC_BODY <clc/math/unary_decl_with_int_return.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ILOGB_H__
diff --git a/libclc/opencl/include/clc/opencl/math/ldexp.h b/libclc/opencl/include/clc/opencl/math/ldexp.h
index b0194d9c9adad..cbbd030d73400 100644
--- a/libclc/opencl/include/clc/opencl/math/ldexp.h
+++ b/libclc/opencl/include/clc/opencl/math/ldexp.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_OPENCL_MATH_LDEXP_H__
 #define __CLC_OPENCL_MATH_LDEXP_H__
 
-#define FUNCTION ldexp
+#define __CLC_FUNCTION ldexp
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #define __CLC_BODY <clc/opencl/math/ldexp.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/include/clc/opencl/math/ldexp.inc b/libclc/opencl/include/clc/opencl/math/ldexp.inc
index b5a5cfcafdd53..9f3ffeeef27e3 100644
--- a/libclc/opencl/include/clc/opencl/math/ldexp.inc
+++ b/libclc/opencl/include/clc/opencl/math/ldexp.inc
@@ -8,6 +8,6 @@
 
 #ifndef __CLC_SCALAR
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE ldexp(__CLC_GENTYPE x, int n);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE ldexp(__CLC_GENTYPE x, int n);
 
 #endif
diff --git a/libclc/opencl/include/clc/opencl/math/lgamma.h b/libclc/opencl/include/clc/opencl/math/lgamma.h
index 95ed07c3531ee..42c260f65942b 100644
--- a/libclc/opencl/include/clc/opencl/math/lgamma.h
+++ b/libclc/opencl/include/clc/opencl/math/lgamma.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_LGAMMA_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION lgamma
+#define __CLC_FUNCTION lgamma
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_LGAMMA_H__
diff --git a/libclc/opencl/include/clc/opencl/math/lgamma_r.h b/libclc/opencl/include/clc/opencl/math/lgamma_r.h
index 4853d2dd5c283..03f47b30fb717 100644
--- a/libclc/opencl/include/clc/opencl/math/lgamma_r.h
+++ b/libclc/opencl/include/clc/opencl/math/lgamma_r.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_MATH_LGAMMA_R_H__
 #define __CLC_OPENCL_MATH_LGAMMA_R_H__
 
-#define FUNCTION lgamma_r
+#define __CLC_FUNCTION lgamma_r
 #define __CLC_BODY <clc/math/unary_decl_with_int_ptr.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_LGAMMA_R_H__
diff --git a/libclc/opencl/include/clc/opencl/math/log.h b/libclc/opencl/include/clc/opencl/math/log.h
index 2d536d6b09266..f863dee3e8f2d 100644
--- a/libclc/opencl/include/clc/opencl/math/log.h
+++ b/libclc/opencl/include/clc/opencl/math/log.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_LOG_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION log
+#define __CLC_FUNCTION log
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_LOG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/log10.h b/libclc/opencl/include/clc/opencl/math/log10.h
index 0f83b8c2a03e9..d328057ecf5e0 100644
--- a/libclc/opencl/include/clc/opencl/math/log10.h
+++ b/libclc/opencl/include/clc/opencl/math/log10.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_LOG10_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION log10
+#define __CLC_FUNCTION log10
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_LOG10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/log1p.h b/libclc/opencl/include/clc/opencl/math/log1p.h
index bd8adeda88d4e..0dba062aa9fde 100644
--- a/libclc/opencl/include/clc/opencl/math/log1p.h
+++ b/libclc/opencl/include/clc/opencl/math/log1p.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_LOG1P_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION log1p
+#define __CLC_FUNCTION log1p
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_LOG1P_H__
diff --git a/libclc/opencl/include/clc/opencl/math/log2.h b/libclc/opencl/include/clc/opencl/math/log2.h
index e4f6cb619acf2..3903b2730e292 100644
--- a/libclc/opencl/include/clc/opencl/math/log2.h
+++ b/libclc/opencl/include/clc/opencl/math/log2.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_LOG2_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION log2
+#define __CLC_FUNCTION log2
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_LOG2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/logb.h b/libclc/opencl/include/clc/opencl/math/logb.h
index 3bbc8476d2ffa..ef4f7b763c59f 100644
--- a/libclc/opencl/include/clc/opencl/math/logb.h
+++ b/libclc/opencl/include/clc/opencl/math/logb.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_LOGB_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION logb
+#define __CLC_FUNCTION logb
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_LOGB_H__
diff --git a/libclc/opencl/include/clc/opencl/math/mad.h b/libclc/opencl/include/clc/opencl/math/mad.h
index 18732eb7be668..440892cfbb17b 100644
--- a/libclc/opencl/include/clc/opencl/math/mad.h
+++ b/libclc/opencl/include/clc/opencl/math/mad.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_MAD_H__
 
 #define __CLC_BODY <clc/shared/ternary_decl.inc>
-#define FUNCTION mad
+#define __CLC_FUNCTION mad
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_MAD_H__
diff --git a/libclc/opencl/include/clc/opencl/math/maxmag.h b/libclc/opencl/include/clc/opencl/math/maxmag.h
index c8f3d5700fa28..1f3df1034d8e4 100644
--- a/libclc/opencl/include/clc/opencl/math/maxmag.h
+++ b/libclc/opencl/include/clc/opencl/math/maxmag.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_MAXMAG_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION maxmag
+#define __CLC_FUNCTION maxmag
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_MAXMAG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/minmag.h b/libclc/opencl/include/clc/opencl/math/minmag.h
index c90a9d9bdbeb1..c87fc1a20e401 100644
--- a/libclc/opencl/include/clc/opencl/math/minmag.h
+++ b/libclc/opencl/include/clc/opencl/math/minmag.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_MINMAG_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION minmag
+#define __CLC_FUNCTION minmag
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_MINMAG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/modf.h b/libclc/opencl/include/clc/opencl/math/modf.h
index a336c9e242c1e..df07f7f5948dd 100644
--- a/libclc/opencl/include/clc/opencl/math/modf.h
+++ b/libclc/opencl/include/clc/opencl/math/modf.h
@@ -9,10 +9,10 @@
 #ifndef __CLC_OPENCL_MATH_MODF_H__
 #define __CLC_OPENCL_MATH_MODF_H__
 
-#define FUNCTION modf
+#define __CLC_FUNCTION modf
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_MODF_H__
diff --git a/libclc/opencl/include/clc/opencl/math/nan.inc b/libclc/opencl/include/clc/opencl/math/nan.inc
index 45b96f81d7285..3134e56eaf83e 100644
--- a/libclc/opencl/include/clc/opencl/math/nan.inc
+++ b/libclc/opencl/include/clc/opencl/math/nan.inc
@@ -6,4 +6,4 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE nan(__CLC_U_GENTYPE code);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE nan(__CLC_U_GENTYPE code);
diff --git a/libclc/opencl/include/clc/opencl/math/native_cos.h b/libclc/opencl/include/clc/opencl/math/native_cos.h
index 566c32e95b32c..9c6728103adc3 100644
--- a/libclc/opencl/include/clc/opencl/math/native_cos.h
+++ b/libclc/opencl/include/clc/opencl/math/native_cos.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_COS_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_cos
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_cos
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_COS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_divide.h b/libclc/opencl/include/clc/opencl/math/native_divide.h
index 23360b4bdeb9b..66c42f20d222c 100644
--- a/libclc/opencl/include/clc/opencl/math/native_divide.h
+++ b/libclc/opencl/include/clc/opencl/math/native_divide.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_NATIVE_DIVIDE_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION native_divide
+#define __CLC_FUNCTION native_divide
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_DIVIDE_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_exp.h b/libclc/opencl/include/clc/opencl/math/native_exp.h
index 5133469509ffb..d8c6f70d3f2c2 100644
--- a/libclc/opencl/include/clc/opencl/math/native_exp.h
+++ b/libclc/opencl/include/clc/opencl/math/native_exp.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_EXP_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_exp
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_exp
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_EXP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_exp10.h b/libclc/opencl/include/clc/opencl/math/native_exp10.h
index 8a2788cd3bcad..484eeb5de0a51 100644
--- a/libclc/opencl/include/clc/opencl/math/native_exp10.h
+++ b/libclc/opencl/include/clc/opencl/math/native_exp10.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_EXP10_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_exp10
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_exp10
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_EXP10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_exp2.h b/libclc/opencl/include/clc/opencl/math/native_exp2.h
index 5a6aed09a838e..bc1e6babc62b0 100644
--- a/libclc/opencl/include/clc/opencl/math/native_exp2.h
+++ b/libclc/opencl/include/clc/opencl/math/native_exp2.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_EXP2_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_exp2
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_exp2
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_EXP2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_log.h b/libclc/opencl/include/clc/opencl/math/native_log.h
index 58164960644ff..c4bf498738f4a 100644
--- a/libclc/opencl/include/clc/opencl/math/native_log.h
+++ b/libclc/opencl/include/clc/opencl/math/native_log.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_LOG_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_log
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_log
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_LOG_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_log10.h b/libclc/opencl/include/clc/opencl/math/native_log10.h
index c215c733c0422..ab61ee3df5c03 100644
--- a/libclc/opencl/include/clc/opencl/math/native_log10.h
+++ b/libclc/opencl/include/clc/opencl/math/native_log10.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_LOG10_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_log10
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_log10
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_LOG10_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_log2.h b/libclc/opencl/include/clc/opencl/math/native_log2.h
index 783abe6cd8e46..6ba0d3bf490f0 100644
--- a/libclc/opencl/include/clc/opencl/math/native_log2.h
+++ b/libclc/opencl/include/clc/opencl/math/native_log2.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_LOG2_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_log2
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_log2
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_LOG2_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_powr.h b/libclc/opencl/include/clc/opencl/math/native_powr.h
index 9c8e171afc21a..6b63138aa7822 100644
--- a/libclc/opencl/include/clc/opencl/math/native_powr.h
+++ b/libclc/opencl/include/clc/opencl/math/native_powr.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_NATIVE_POWR_H__
 
 #define __CLC_BODY <clc/shared/binary_decl.inc>
-#define FUNCTION native_powr
+#define __CLC_FUNCTION native_powr
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_POWR_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_recip.h b/libclc/opencl/include/clc/opencl/math/native_recip.h
index 1c88c076e03c7..967eea38d80d9 100644
--- a/libclc/opencl/include/clc/opencl/math/native_recip.h
+++ b/libclc/opencl/include/clc/opencl/math/native_recip.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_RECIP_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_recip
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_recip
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_RECIP_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_rsqrt.h b/libclc/opencl/include/clc/opencl/math/native_rsqrt.h
index 2806f9128d94f..9146df8913720 100644
--- a/libclc/opencl/include/clc/opencl/math/native_rsqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/native_rsqrt.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_RSQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_rsqrt
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_rsqrt
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_RSQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_sin.h b/libclc/opencl/include/clc/opencl/math/native_sin.h
index 81f664153cee1..766b76781a9b7 100644
--- a/libclc/opencl/include/clc/opencl/math/native_sin.h
+++ b/libclc/opencl/include/clc/opencl/math/native_sin.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_SIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_sin
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_sin
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_SIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_sqrt.h b/libclc/opencl/include/clc/opencl/math/native_sqrt.h
index dca0e542e0ac7..5d44f28db9286 100644
--- a/libclc/opencl/include/clc/opencl/math/native_sqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/native_sqrt.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_SQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_sqrt
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_sqrt
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_SQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/native_tan.h b/libclc/opencl/include/clc/opencl/math/native_tan.h
index cf942110e3124..075aa59757b4b 100644
--- a/libclc/opencl/include/clc/opencl/math/native_tan.h
+++ b/libclc/opencl/include/clc/opencl/math/native_tan.h
@@ -10,11 +10,11 @@
 #define __CLC_OPENCL_MATH_NATIVE_TAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION native_tan
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION native_tan
+#define __CLC_FLOAT_ONLY
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NATIVE_TAN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/nextafter.h b/libclc/opencl/include/clc/opencl/math/nextafter.h
index e338f5bf6c2a7..6843f29770251 100644
--- a/libclc/opencl/include/clc/opencl/math/nextafter.h
+++ b/libclc/opencl/include/clc/opencl/math/nextafter.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_MATH_NEXTAFTER_H__
 #define __CLC_OPENCL_MATH_NEXTAFTER_H__
 
-#define FUNCTION nextafter
+#define __CLC_FUNCTION nextafter
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_NEXTAFTER_H__
diff --git a/libclc/opencl/include/clc/opencl/math/pow.h b/libclc/opencl/include/clc/opencl/math/pow.h
index 3f4c1746faa91..ce17b66b541d0 100644
--- a/libclc/opencl/include/clc/opencl/math/pow.h
+++ b/libclc/opencl/include/clc/opencl/math/pow.h
@@ -9,9 +9,9 @@
 #ifndef __CLC_OPENCL_MATH_POW_H__
 #define __CLC_OPENCL_MATH_POW_H__
 
-#define FUNCTION pow
+#define __CLC_FUNCTION pow
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_POW_H__
diff --git a/libclc/opencl/include/clc/opencl/math/pown.h b/libclc/opencl/include/clc/opencl/math/pown.h
index fd97a705c8c87..24fc298da4f1e 100644
--- a/libclc/opencl/include/clc/opencl/math/pown.h
+++ b/libclc/opencl/include/clc/opencl/math/pown.h
@@ -9,11 +9,11 @@
 #ifndef __CLC_OPENCL_MATH_POWN_H__
 #define __CLC_OPENCL_MATH_POWN_H__
 
-#define FUNCTION pown
+#define __CLC_FUNCTION pown
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_POWN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/powr.h b/libclc/opencl/include/clc/opencl/math/powr.h
index 47f653a73afa3..6aebc346080f9 100644
--- a/libclc/opencl/include/clc/opencl/math/powr.h
+++ b/libclc/opencl/include/clc/opencl/math/powr.h
@@ -9,9 +9,9 @@
 #ifndef __CLC_OPENCL_MATH_POWR_H__
 #define __CLC_OPENCL_MATH_POWR_H__
 
-#define FUNCTION powr
+#define __CLC_FUNCTION powr
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_POWR_H__
diff --git a/libclc/opencl/include/clc/opencl/math/remainder.h b/libclc/opencl/include/clc/opencl/math/remainder.h
index 416a0422c9a62..d30a315a4ce38 100644
--- a/libclc/opencl/include/clc/opencl/math/remainder.h
+++ b/libclc/opencl/include/clc/opencl/math/remainder.h
@@ -9,9 +9,9 @@
 #ifndef __CLC_OPENCL_MATH_REMAINDER_H__
 #define __CLC_OPENCL_MATH_REMAINDER_H__
 
-#define FUNCTION remainder
+#define __CLC_FUNCTION remainder
 #define __CLC_BODY <clc/shared/binary_decl.inc>
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_REMAINDER_H__
diff --git a/libclc/opencl/include/clc/opencl/math/remquo.h b/libclc/opencl/include/clc/opencl/math/remquo.h
index 3525c6bd0f685..28fc74c00fd90 100644
--- a/libclc/opencl/include/clc/opencl/math/remquo.h
+++ b/libclc/opencl/include/clc/opencl/math/remquo.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_MATH_REMQUO_H__
 #define __CLC_OPENCL_MATH_REMQUO_H__
 
-#define FUNCTION remquo
+#define __CLC_FUNCTION remquo
 
 #define __CLC_BODY <clc/math/remquo_decl.inc>
 #include <clc/math/gentype.inc>
@@ -21,6 +21,6 @@
 #undef __CLC_ADDRESS_SPACE
 #endif
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_REMQUO_H__
diff --git a/libclc/opencl/include/clc/opencl/math/rint.h b/libclc/opencl/include/clc/opencl/math/rint.h
index 8f08fd20c3c92..ceca56535a864 100644
--- a/libclc/opencl/include/clc/opencl/math/rint.h
+++ b/libclc/opencl/include/clc/opencl/math/rint.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_RINT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION rint
+#define __CLC_FUNCTION rint
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_RINT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/rootn.h b/libclc/opencl/include/clc/opencl/math/rootn.h
index 20b52e7c01dc5..9dce895a3f2d7 100644
--- a/libclc/opencl/include/clc/opencl/math/rootn.h
+++ b/libclc/opencl/include/clc/opencl/math/rootn.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ROOTN_H__
 
 #define __CLC_BODY <clc/shared/binary_decl_with_int_second_arg.inc>
-#define FUNCTION rootn
+#define __CLC_FUNCTION rootn
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ROOTN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/round.h b/libclc/opencl/include/clc/opencl/math/round.h
index 65bfc265ced0e..740ae9241f692 100644
--- a/libclc/opencl/include/clc/opencl/math/round.h
+++ b/libclc/opencl/include/clc/opencl/math/round.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_ROUND_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION round
+#define __CLC_FUNCTION round
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_ROUND_H__
diff --git a/libclc/opencl/include/clc/opencl/math/rsqrt.h b/libclc/opencl/include/clc/opencl/math/rsqrt.h
index 41fb54ea9bb5f..ea141d4bc45bd 100644
--- a/libclc/opencl/include/clc/opencl/math/rsqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/rsqrt.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_RSQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION rsqrt
+#define __CLC_FUNCTION rsqrt
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_RSQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sin.h b/libclc/opencl/include/clc/opencl/math/sin.h
index fc7741e164fda..60de98de09584 100644
--- a/libclc/opencl/include/clc/opencl/math/sin.h
+++ b/libclc/opencl/include/clc/opencl/math/sin.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_SIN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION sin
+#define __CLC_FUNCTION sin
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_SIN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sincos.h b/libclc/opencl/include/clc/opencl/math/sincos.h
index 0231560a6b9de..4989bcf21c038 100644
--- a/libclc/opencl/include/clc/opencl/math/sincos.h
+++ b/libclc/opencl/include/clc/opencl/math/sincos.h
@@ -10,8 +10,8 @@
 #define __CLC_OPENCL_MATH_SINCOS_H__
 
 #define __CLC_BODY <clc/math/unary_decl_with_ptr.inc>
-#define FUNCTION sincos
+#define __CLC_FUNCTION sincos
 #include <clc/math/gentype.inc>
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_SINCOS_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sinh.h b/libclc/opencl/include/clc/opencl/math/sinh.h
index cbe30551ec641..0c9f0985be52b 100644
--- a/libclc/opencl/include/clc/opencl/math/sinh.h
+++ b/libclc/opencl/include/clc/opencl/math/sinh.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_SINH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION sinh
+#define __CLC_FUNCTION sinh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_SINH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sinpi.h b/libclc/opencl/include/clc/opencl/math/sinpi.h
index 3ef55a1cd42dc..a24339f8cf03f 100644
--- a/libclc/opencl/include/clc/opencl/math/sinpi.h
+++ b/libclc/opencl/include/clc/opencl/math/sinpi.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_SINPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION sinpi
+#define __CLC_FUNCTION sinpi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_SINPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/sqrt.h b/libclc/opencl/include/clc/opencl/math/sqrt.h
index 8da37e94e756c..41f604f9652dd 100644
--- a/libclc/opencl/include/clc/opencl/math/sqrt.h
+++ b/libclc/opencl/include/clc/opencl/math/sqrt.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_SQRT_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION sqrt
+#define __CLC_FUNCTION sqrt
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_SQRT_H__
diff --git a/libclc/opencl/include/clc/opencl/math/tan.h b/libclc/opencl/include/clc/opencl/math/tan.h
index b7bab825c935c..5c3f5f6a3a417 100644
--- a/libclc/opencl/include/clc/opencl/math/tan.h
+++ b/libclc/opencl/include/clc/opencl/math/tan.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_TAN_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION tan
+#define __CLC_FUNCTION tan
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_TAN_H__
diff --git a/libclc/opencl/include/clc/opencl/math/tanh.h b/libclc/opencl/include/clc/opencl/math/tanh.h
index 168c06d50cb29..dce89867f25e0 100644
--- a/libclc/opencl/include/clc/opencl/math/tanh.h
+++ b/libclc/opencl/include/clc/opencl/math/tanh.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_TANH_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION tanh
+#define __CLC_FUNCTION tanh
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_TANH_H__
diff --git a/libclc/opencl/include/clc/opencl/math/tanpi.h b/libclc/opencl/include/clc/opencl/math/tanpi.h
index 0475027086f63..2a903b9a1fe75 100644
--- a/libclc/opencl/include/clc/opencl/math/tanpi.h
+++ b/libclc/opencl/include/clc/opencl/math/tanpi.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_TANPI_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION tanpi
+#define __CLC_FUNCTION tanpi
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_TANPI_H__
diff --git a/libclc/opencl/include/clc/opencl/math/tgamma.h b/libclc/opencl/include/clc/opencl/math/tgamma.h
index 6629c02002455..5cb0a7fd48f93 100644
--- a/libclc/opencl/include/clc/opencl/math/tgamma.h
+++ b/libclc/opencl/include/clc/opencl/math/tgamma.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_TGAMMA_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION tgamma
+#define __CLC_FUNCTION tgamma
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_TGAMMA_H__
diff --git a/libclc/opencl/include/clc/opencl/math/trunc.h b/libclc/opencl/include/clc/opencl/math/trunc.h
index 0e61f6e1679f7..ccb60a6a54d6a 100644
--- a/libclc/opencl/include/clc/opencl/math/trunc.h
+++ b/libclc/opencl/include/clc/opencl/math/trunc.h
@@ -10,10 +10,10 @@
 #define __CLC_OPENCL_MATH_TRUNC_H__
 
 #define __CLC_BODY <clc/math/unary_decl.inc>
-#define FUNCTION trunc
+#define __CLC_FUNCTION trunc
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MATH_TRUNC_H__
diff --git a/libclc/opencl/include/clc/opencl/misc/shuffle.h b/libclc/opencl/include/clc/opencl/misc/shuffle.h
index 03c2718b0c4ff..a1d4b6aa38248 100644
--- a/libclc/opencl/include/clc/opencl/misc/shuffle.h
+++ b/libclc/opencl/include/clc/opencl/misc/shuffle.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_MISC_SHUFFLE_H__
 #define __CLC_OPENCL_MISC_SHUFFLE_H__
 
-#define FUNCTION shuffle
+#define __CLC_FUNCTION shuffle
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MISC_SHUFFLE_H__
diff --git a/libclc/opencl/include/clc/opencl/misc/shuffle2.h b/libclc/opencl/include/clc/opencl/misc/shuffle2.h
index 5b6ba99686516..0f93a802b3f36 100644
--- a/libclc/opencl/include/clc/opencl/misc/shuffle2.h
+++ b/libclc/opencl/include/clc/opencl/misc/shuffle2.h
@@ -9,7 +9,7 @@
 #ifndef __CLC_OPENCL_MISC_SHUFFLE2_H__
 #define __CLC_OPENCL_MISC_SHUFFLE2_H__
 
-#define FUNCTION shuffle2
+#define __CLC_FUNCTION shuffle2
 
 // Integer-type decls
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
@@ -19,6 +19,6 @@
 #define __CLC_BODY <clc/misc/shuffle2_decl.inc>
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_MISC_SHUFFLE2_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/all.h b/libclc/opencl/include/clc/opencl/relational/all.h
index 6c19d83e25bbf..f7f14006309fb 100644
--- a/libclc/opencl/include/clc/opencl/relational/all.h
+++ b/libclc/opencl/include/clc/opencl/relational/all.h
@@ -11,7 +11,7 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define _CLC_ALL_DECL(TYPE) _CLC_OVERLOAD _CLC_DECL int all(TYPE v);
+#define _CLC_ALL_DECL(TYPE) _CLC_OVERLOAD _CLC_CONST _CLC_DECL int all(TYPE v);
 
 #define _CLC_VECTOR_ALL_DECL(TYPE)                                             \
   _CLC_ALL_DECL(TYPE)                                                          \
diff --git a/libclc/opencl/include/clc/opencl/relational/any.h b/libclc/opencl/include/clc/opencl/relational/any.h
index 4f6b2e22460c1..7b1ad333cef34 100644
--- a/libclc/opencl/include/clc/opencl/relational/any.h
+++ b/libclc/opencl/include/clc/opencl/relational/any.h
@@ -11,7 +11,7 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define _CLC_ANY_DECL(TYPE) _CLC_OVERLOAD _CLC_DECL int any(TYPE v);
+#define _CLC_ANY_DECL(TYPE) _CLC_OVERLOAD _CLC_CONST _CLC_DECL int any(TYPE v);
 
 #define _CLC_VECTOR_ANY_DECL(TYPE)                                             \
   _CLC_ANY_DECL(TYPE)                                                          \
diff --git a/libclc/opencl/include/clc/opencl/relational/bitselect.inc b/libclc/opencl/include/clc/opencl/relational/bitselect.inc
index d821bb86a1409..429ef372d1214 100644
--- a/libclc/opencl/include/clc/opencl/relational/bitselect.inc
+++ b/libclc/opencl/include/clc/opencl/relational/bitselect.inc
@@ -6,6 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE bitselect(__CLC_GENTYPE x,
-                                                __CLC_GENTYPE y,
-                                                __CLC_GENTYPE z);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE bitselect(__CLC_GENTYPE x,
+                                                           __CLC_GENTYPE y,
+                                                           __CLC_GENTYPE z);
diff --git a/libclc/opencl/include/clc/opencl/relational/isequal.h b/libclc/opencl/include/clc/opencl/relational/isequal.h
index c679b2cfc3feb..b3dff2e924d2f 100644
--- a/libclc/opencl/include/clc/opencl/relational/isequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isequal.h
@@ -12,7 +12,7 @@
 #include <clc/opencl/opencl-base.h>
 
 #define _CLC_ISEQUAL_DECL(TYPE, RETTYPE)                                       \
-  _CLC_OVERLOAD _CLC_DECL RETTYPE isequal(TYPE x, TYPE y);
+  _CLC_OVERLOAD _CLC_CONST _CLC_DECL RETTYPE isequal(TYPE x, TYPE y);
 
 #define _CLC_VECTOR_ISEQUAL_DECL(TYPE, RETTYPE)                                \
   _CLC_ISEQUAL_DECL(TYPE##2, RETTYPE##2)                                       \
diff --git a/libclc/opencl/include/clc/opencl/relational/isfinite.h b/libclc/opencl/include/clc/opencl/relational/isfinite.h
index ac3db6764073a..ff996befa937d 100644
--- a/libclc/opencl/include/clc/opencl/relational/isfinite.h
+++ b/libclc/opencl/include/clc/opencl/relational/isfinite.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION isfinite
+#define __CLC_FUNCTION isfinite
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISFINITE_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isgreater.h b/libclc/opencl/include/clc/opencl/relational/isgreater.h
index 2230055115bcd..1885faac0d6d2 100644
--- a/libclc/opencl/include/clc/opencl/relational/isgreater.h
+++ b/libclc/opencl/include/clc/opencl/relational/isgreater.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION isgreater
+#define __CLC_FUNCTION isgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISGREATER_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h b/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h
index f99a620dabd78..36def480081bc 100644
--- a/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isgreaterequal.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION isgreaterequal
+#define __CLC_FUNCTION isgreaterequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISGREATEREQUAL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isinf.h b/libclc/opencl/include/clc/opencl/relational/isinf.h
index 561cb1cda47b7..0dd7246a88fea 100644
--- a/libclc/opencl/include/clc/opencl/relational/isinf.h
+++ b/libclc/opencl/include/clc/opencl/relational/isinf.h
@@ -12,7 +12,7 @@
 #include <clc/opencl/opencl-base.h>
 
 #define _CLC_ISINF_DECL(RET_TYPE, ARG_TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL RET_TYPE isinf(ARG_TYPE);
+  _CLC_OVERLOAD _CLC_CONST _CLC_DECL RET_TYPE isinf(ARG_TYPE);
 
 #define _CLC_VECTOR_ISINF_DECL(RET_TYPE, ARG_TYPE)                             \
   _CLC_ISINF_DECL(RET_TYPE##2, ARG_TYPE##2)                                    \
diff --git a/libclc/opencl/include/clc/opencl/relational/isless.h b/libclc/opencl/include/clc/opencl/relational/isless.h
index 74280e543e0b5..ce516890f19e5 100644
--- a/libclc/opencl/include/clc/opencl/relational/isless.h
+++ b/libclc/opencl/include/clc/opencl/relational/isless.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION isless
+#define __CLC_FUNCTION isless
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISLESS_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/islessequal.h b/libclc/opencl/include/clc/opencl/relational/islessequal.h
index dcc26c37b73c1..571ed13c0ab86 100644
--- a/libclc/opencl/include/clc/opencl/relational/islessequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/islessequal.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION islessequal
+#define __CLC_FUNCTION islessequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISLESSEQUAL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/islessgreater.h b/libclc/opencl/include/clc/opencl/relational/islessgreater.h
index 15a1eb5577531..eefd2c41b3469 100644
--- a/libclc/opencl/include/clc/opencl/relational/islessgreater.h
+++ b/libclc/opencl/include/clc/opencl/relational/islessgreater.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION islessgreater
+#define __CLC_FUNCTION islessgreater
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISLESSGREATER_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isnan.h b/libclc/opencl/include/clc/opencl/relational/isnan.h
index fe1e5721c1d24..f02ec5d04c64f 100644
--- a/libclc/opencl/include/clc/opencl/relational/isnan.h
+++ b/libclc/opencl/include/clc/opencl/relational/isnan.h
@@ -12,7 +12,7 @@
 #include <clc/opencl/opencl-base.h>
 
 #define _CLC_ISNAN_DECL(RET_TYPE, ARG_TYPE)                                    \
-  _CLC_OVERLOAD _CLC_DECL RET_TYPE isnan(ARG_TYPE);
+  _CLC_OVERLOAD _CLC_CONST _CLC_DECL RET_TYPE isnan(ARG_TYPE);
 
 #define _CLC_VECTOR_ISNAN_DECL(RET_TYPE, ARG_TYPE)                             \
   _CLC_ISNAN_DECL(RET_TYPE##2, ARG_TYPE##2)                                    \
diff --git a/libclc/opencl/include/clc/opencl/relational/isnormal.h b/libclc/opencl/include/clc/opencl/relational/isnormal.h
index bbb06aad0df2a..29ef762174f09 100644
--- a/libclc/opencl/include/clc/opencl/relational/isnormal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isnormal.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION isnormal
+#define __CLC_FUNCTION isnormal
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISNORMAL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isnotequal.h b/libclc/opencl/include/clc/opencl/relational/isnotequal.h
index c13aca8ef4be8..6670b9946d186 100644
--- a/libclc/opencl/include/clc/opencl/relational/isnotequal.h
+++ b/libclc/opencl/include/clc/opencl/relational/isnotequal.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION isnotequal
+#define __CLC_FUNCTION isnotequal
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISNOTEQUAL_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isordered.h b/libclc/opencl/include/clc/opencl/relational/isordered.h
index ea4ba3fa6fe8d..09531ed86721c 100644
--- a/libclc/opencl/include/clc/opencl/relational/isordered.h
+++ b/libclc/opencl/include/clc/opencl/relational/isordered.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION isordered
+#define __CLC_FUNCTION isordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISORDERED_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/isunordered.h b/libclc/opencl/include/clc/opencl/relational/isunordered.h
index 76bf85604d1c7..d1f6dabbe20c3 100644
--- a/libclc/opencl/include/clc/opencl/relational/isunordered.h
+++ b/libclc/opencl/include/clc/opencl/relational/isunordered.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION isunordered
+#define __CLC_FUNCTION isunordered
 #define __CLC_BODY <clc/relational/binary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_ISUNORDERED_H__
diff --git a/libclc/opencl/include/clc/opencl/relational/signbit.h b/libclc/opencl/include/clc/opencl/relational/signbit.h
index 6ad6595c7e294..fff6f33508fad 100644
--- a/libclc/opencl/include/clc/opencl/relational/signbit.h
+++ b/libclc/opencl/include/clc/opencl/relational/signbit.h
@@ -11,11 +11,11 @@
 
 #include <clc/opencl/opencl-base.h>
 
-#define FUNCTION signbit
+#define __CLC_FUNCTION signbit
 #define __CLC_BODY <clc/relational/unary_decl.inc>
 
 #include <clc/math/gentype.inc>
 
-#undef FUNCTION
+#undef __CLC_FUNCTION
 
 #endif // __CLC_OPENCL_RELATIONAL_SIGNBIT_H__
diff --git a/libclc/opencl/include/clc/opencl/shared/clamp.inc b/libclc/opencl/include/clc/opencl/shared/clamp.inc
index 75dec9c258e8a..74d2f704fadd9 100644
--- a/libclc/opencl/include/clc/opencl/shared/clamp.inc
+++ b/libclc/opencl/include/clc/opencl/shared/clamp.inc
@@ -6,11 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE clamp(__CLC_GENTYPE x, __CLC_GENTYPE y,
-                                            __CLC_GENTYPE z);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE clamp(__CLC_GENTYPE x,
+                                                       __CLC_GENTYPE y,
+                                                       __CLC_GENTYPE z);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE clamp(__CLC_GENTYPE x,
-                                            __CLC_SCALAR_GENTYPE y,
-                                            __CLC_SCALAR_GENTYPE z);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE clamp(__CLC_GENTYPE x,
+                                                       __CLC_SCALAR_GENTYPE y,
+                                                       __CLC_SCALAR_GENTYPE z);
 #endif
diff --git a/libclc/opencl/include/clc/opencl/shared/max.inc b/libclc/opencl/include/clc/opencl/shared/max.inc
index 98610dfdb1018..99c28a7e8cd74 100644
--- a/libclc/opencl/include/clc/opencl/shared/max.inc
+++ b/libclc/opencl/include/clc/opencl/shared/max.inc
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE max(__CLC_GENTYPE a, __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE max(__CLC_GENTYPE a,
+                                                     __CLC_GENTYPE b);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE max(__CLC_GENTYPE a,
-                                          __CLC_SCALAR_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE max(__CLC_GENTYPE a,
+                                                     __CLC_SCALAR_GENTYPE b);
 #endif
diff --git a/libclc/opencl/include/clc/opencl/shared/min.inc b/libclc/opencl/include/clc/opencl/shared/min.inc
index 3140b637f3953..ce5d906fe26c7 100644
--- a/libclc/opencl/include/clc/opencl/shared/min.inc
+++ b/libclc/opencl/include/clc/opencl/shared/min.inc
@@ -6,9 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE min(__CLC_GENTYPE a, __CLC_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE min(__CLC_GENTYPE a,
+                                                     __CLC_GENTYPE b);
 
 #ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE min(__CLC_GENTYPE a,
-                                          __CLC_SCALAR_GENTYPE b);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL __CLC_GENTYPE min(__CLC_GENTYPE a,
+                                                     __CLC_SCALAR_GENTYPE b);
 #endif
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_global_id.h b/libclc/opencl/include/clc/opencl/workitem/get_global_id.h
index 98217c0ba6b4d..d64da9d2cc643 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_global_id.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_global_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_DECL _CLC_OVERLOAD size_t get_global_id(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t get_global_id(uint dim);
 
 #endif // __CLC_OPENCL_WORKITEM_GET_GLOBAL_ID_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_global_offset.h b/libclc/opencl/include/clc/opencl/workitem/get_global_offset.h
index 8c820331e6ab2..ac9704e274f2d 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_global_offset.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_global_offset.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_DECL _CLC_OVERLOAD size_t get_global_offset(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t get_global_offset(uint dim);
 
 #endif // __CLC_OPENCL_WORKITEM_GET_GLOBAL_OFFSET_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_global_size.h b/libclc/opencl/include/clc/opencl/workitem/get_global_size.h
index 5c7426047d7d0..4809cc4b37712 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_global_size.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_global_size.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_DECL _CLC_OVERLOAD size_t get_global_size(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t get_global_size(uint dim);
 
 #endif // __CLC_OPENCL_WORKITEM_GET_GLOBAL_SIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_group_id.h b/libclc/opencl/include/clc/opencl/workitem/get_group_id.h
index 3d0f51aeb8dc2..e3d8b07e4ad9a 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_group_id.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_group_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_DECL _CLC_OVERLOAD size_t get_group_id(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t get_group_id(uint dim);
 
 #endif // __CLC_OPENCL_WORKITEM_GET_GROUP_ID_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_local_id.h b/libclc/opencl/include/clc/opencl/workitem/get_local_id.h
index 31fe9d2943f48..88fbad98dfb19 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_local_id.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_local_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_DECL _CLC_OVERLOAD size_t get_local_id(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t get_local_id(uint dim);
 
 #endif // __CLC_OPENCL_WORKITEM_GET_LOCAL_ID_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_local_linear_id.h b/libclc/opencl/include/clc/opencl/workitem/get_local_linear_id.h
index 9a32aa46b8191..072318d1786f1 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_local_linear_id.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_local_linear_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_OVERLOAD _CLC_DECL size_t get_local_linear_id();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t get_local_linear_id();
 
 #endif // __CLC_OPENCL_WORKITEM_GET_LOCAL_LINEAR_ID_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_local_size.h b/libclc/opencl/include/clc/opencl/workitem/get_local_size.h
index c93a7a27bd7cf..49077afde34fd 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_local_size.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_local_size.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_DECL _CLC_OVERLOAD size_t get_local_size(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t get_local_size(uint dim);
 
 #endif // __CLC_OPENCL_WORKITEM_GET_LOCAL_SIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_max_sub_group_size.h b/libclc/opencl/include/clc/opencl/workitem/get_max_sub_group_size.h
index 140f22426ef6b..b12737618c64b 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_max_sub_group_size.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_max_sub_group_size.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_OVERLOAD _CLC_DECL uint get_max_sub_group_size();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint get_max_sub_group_size();
 
 #endif // __CLC_OPENCL_WORKITEM_GET_MAX_SUB_GROUP_SIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_num_groups.h b/libclc/opencl/include/clc/opencl/workitem/get_num_groups.h
index 0c47a5ef462b3..29783169415ff 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_num_groups.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_num_groups.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_DECL _CLC_OVERLOAD size_t get_num_groups(uint dim);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL size_t get_num_groups(uint dim);
 
 #endif // __CLC_OPENCL_WORKITEM_GET_NUM_GROUPS_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_num_sub_groups.h b/libclc/opencl/include/clc/opencl/workitem/get_num_sub_groups.h
index 5c72db0b40c72..f9fd2cddbf196 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_num_sub_groups.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_num_sub_groups.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_OVERLOAD _CLC_DECL uint get_num_sub_groups();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint get_num_sub_groups();
 
 #endif // __CLC_OPENCL_WORKITEM_GET_NUM_SUB_GROUPS_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_sub_group_id.h b/libclc/opencl/include/clc/opencl/workitem/get_sub_group_id.h
index 3dbf39b8cfbad..0a67c022a5131 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_sub_group_id.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_sub_group_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_OVERLOAD _CLC_DECL uint get_sub_group_id();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint get_sub_group_id();
 
 #endif // __CLC_OPENCL_WORKITEM_GET_SUB_GROUP_ID_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_sub_group_local_id.h b/libclc/opencl/include/clc/opencl/workitem/get_sub_group_local_id.h
index 7e139a857a7d7..37891d8f28304 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_sub_group_local_id.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_sub_group_local_id.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_OVERLOAD _CLC_DECL uint get_sub_group_local_id();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint get_sub_group_local_id();
 
 #endif // __CLC_OPENCL_WORKITEM_GET_SUB_GROUP_LOCAL_ID_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_sub_group_size.h b/libclc/opencl/include/clc/opencl/workitem/get_sub_group_size.h
index dd4f89da18a14..74e9b033fd07f 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_sub_group_size.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_sub_group_size.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_OVERLOAD _CLC_DECL uint get_sub_group_size();
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint get_sub_group_size();
 
 #endif // __CLC_OPENCL_WORKITEM_GET_SUB_GROUP_SIZE_H__
diff --git a/libclc/opencl/include/clc/opencl/workitem/get_work_dim.h b/libclc/opencl/include/clc/opencl/workitem/get_work_dim.h
index c4c3c1b224a9a..344ffaa43e108 100644
--- a/libclc/opencl/include/clc/opencl/workitem/get_work_dim.h
+++ b/libclc/opencl/include/clc/opencl/workitem/get_work_dim.h
@@ -11,6 +11,6 @@
 
 #include <clc/opencl/opencl-base.h>
 
-_CLC_DECL _CLC_OVERLOAD uint get_work_dim(void);
+_CLC_OVERLOAD _CLC_CONST _CLC_DECL uint get_work_dim(void);
 
 #endif // __CLC_OPENCL_WORKITEM_GET_WORK_DIM_H__
diff --git a/libclc/opencl/lib/clspv/math/fma.cl b/libclc/opencl/lib/clspv/math/fma.cl
index 83504f63772ca..0e328903ba263 100644
--- a/libclc/opencl/lib/clspv/math/fma.cl
+++ b/libclc/opencl/lib/clspv/math/fma.cl
@@ -9,9 +9,9 @@
 #include <clc/internal/math/clc_sw_fma.h>
 #include <clc/opencl/math/fma.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION fma
-#define __IMPL_FUNCTION(x) __clc_sw_fma
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION fma
+#define __CLC_IMPL_FUNCTION(x) __clc_sw_fma
 #define __CLC_BODY <clc/shared/ternary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/clspv/shared/vstore_half.cl b/libclc/opencl/lib/clspv/shared/vstore_half.cl
index df5f30a711df5..66973eb9ac4dc 100644
--- a/libclc/opencl/lib/clspv/shared/vstore_half.cl
+++ b/libclc/opencl/lib/clspv/shared/vstore_half.cl
@@ -18,37 +18,37 @@
 
 #pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable
 
-#define ROUND_VEC1(out, in, ROUNDF) out = ROUNDF(in);
-#define ROUND_VEC2(out, in, ROUNDF)                                            \
-  ROUND_VEC1(out.lo, in.lo, ROUNDF);                                           \
-  ROUND_VEC1(out.hi, in.hi, ROUNDF);
-#define ROUND_VEC3(out, in, ROUNDF)                                            \
-  ROUND_VEC1(out.s0, in.s0, ROUNDF);                                           \
-  ROUND_VEC1(out.s1, in.s1, ROUNDF);                                           \
-  ROUND_VEC1(out.s2, in.s2, ROUNDF);
-#define ROUND_VEC4(out, in, ROUNDF)                                            \
-  ROUND_VEC2(out.lo, in.lo, ROUNDF);                                           \
-  ROUND_VEC2(out.hi, in.hi, ROUNDF);
-#define ROUND_VEC8(out, in, ROUNDF)                                            \
-  ROUND_VEC4(out.lo, in.lo, ROUNDF);                                           \
-  ROUND_VEC4(out.hi, in.hi, ROUNDF);
-#define ROUND_VEC16(out, in, ROUNDF)                                           \
-  ROUND_VEC8(out.lo, in.lo, ROUNDF);                                           \
-  ROUND_VEC8(out.hi, in.hi, ROUNDF);
-
-#define __FUNC(SUFFIX, VEC_SIZE, TYPE, AS, ROUNDF)                             \
+#define __CLC_ROUND_VEC1(out, in, ROUNDF) out = ROUNDF(in);
+#define __CLC_ROUND_VEC2(out, in, ROUNDF)                                      \
+  __CLC_ROUND_VEC1(out.lo, in.lo, ROUNDF);                                     \
+  __CLC_ROUND_VEC1(out.hi, in.hi, ROUNDF);
+#define __CLC_ROUND_VEC3(out, in, ROUNDF)                                      \
+  __CLC_ROUND_VEC1(out.s0, in.s0, ROUNDF);                                     \
+  __CLC_ROUND_VEC1(out.s1, in.s1, ROUNDF);                                     \
+  __CLC_ROUND_VEC1(out.s2, in.s2, ROUNDF);
+#define __CLC_ROUND_VEC4(out, in, ROUNDF)                                      \
+  __CLC_ROUND_VEC2(out.lo, in.lo, ROUNDF);                                     \
+  __CLC_ROUND_VEC2(out.hi, in.hi, ROUNDF);
+#define __CLC_ROUND_VEC8(out, in, ROUNDF)                                      \
+  __CLC_ROUND_VEC4(out.lo, in.lo, ROUNDF);                                     \
+  __CLC_ROUND_VEC4(out.hi, in.hi, ROUNDF);
+#define __CLC_ROUND_VEC16(out, in, ROUNDF)                                     \
+  __CLC_ROUND_VEC8(out.lo, in.lo, ROUNDF);                                     \
+  __CLC_ROUND_VEC8(out.hi, in.hi, ROUNDF);
+
+#define __CLC_XFUNC_IMPL(SUFFIX, VEC_SIZE, TYPE, AS, ROUNDF)                   \
   void _CLC_OVERLOAD vstore_half_##VEC_SIZE(TYPE, size_t, AS half *);          \
   _CLC_OVERLOAD _CLC_DEF void vstore_half##SUFFIX(TYPE vec, size_t offset,     \
                                                   AS half *mem) {              \
     TYPE rounded_vec;                                                          \
-    ROUND_VEC##VEC_SIZE(rounded_vec, vec, ROUNDF);                             \
+    __CLC_ROUND_VEC##VEC_SIZE(rounded_vec, vec, ROUNDF);                       \
     vstore_half_##VEC_SIZE(rounded_vec, offset, mem);                          \
   }                                                                            \
   void _CLC_OVERLOAD vstorea_half_##VEC_SIZE(TYPE, size_t, AS half *);         \
   _CLC_OVERLOAD _CLC_DEF void vstorea_half##SUFFIX(TYPE vec, size_t offset,    \
                                                    AS half *mem) {             \
     TYPE rounded_vec;                                                          \
-    ROUND_VEC##VEC_SIZE(rounded_vec, vec, ROUNDF);                             \
+    __CLC_ROUND_VEC##VEC_SIZE(rounded_vec, vec, ROUNDF);                       \
     vstorea_half_##VEC_SIZE(rounded_vec, offset, mem);                         \
   }
 
@@ -134,17 +134,18 @@ _CLC_DEF _CLC_OVERLOAD float __clc_rte(float x) {
   return roundup ? __clc_rti(x) : __clc_rtz(x);
 }
 
-#define __XFUNC(SUFFIX, VEC_SIZE, TYPE, AS)                                    \
-  __FUNC(SUFFIX, VEC_SIZE, TYPE, AS, __clc_rte)                                \
-  __FUNC(SUFFIX##_rtz, VEC_SIZE, TYPE, AS, __clc_rtz)                          \
-  __FUNC(SUFFIX##_rtn, VEC_SIZE, TYPE, AS, __clc_rtn)                          \
-  __FUNC(SUFFIX##_rtp, VEC_SIZE, TYPE, AS, __clc_rtp)                          \
-  __FUNC(SUFFIX##_rte, VEC_SIZE, TYPE, AS, __clc_rte)
+#define __CLC_XFUNC(SUFFIX, VEC_SIZE, TYPE, AS)                                \
+  __CLC_XFUNC_IMPL(SUFFIX, VEC_SIZE, TYPE, AS, __clc_rte)                      \
+  __CLC_XFUNC_IMPL(SUFFIX##_rtz, VEC_SIZE, TYPE, AS, __clc_rtz)                \
+  __CLC_XFUNC_IMPL(SUFFIX##_rtn, VEC_SIZE, TYPE, AS, __clc_rtn)                \
+  __CLC_XFUNC_IMPL(SUFFIX##_rtp, VEC_SIZE, TYPE, AS, __clc_rtp)                \
+  __CLC_XFUNC_IMPL(SUFFIX##_rte, VEC_SIZE, TYPE, AS, __clc_rte)
 
-#define FUNC(SUFFIX, VEC_SIZE, TYPE, AS) __XFUNC(SUFFIX, VEC_SIZE, TYPE, AS)
+#define __CLC_FUNC(SUFFIX, VEC_SIZE, TYPE, AS)                                 \
+  __CLC_XFUNC(SUFFIX, VEC_SIZE, TYPE, AS)
 
 #define __CLC_BODY "vstore_half.inc"
 #include <clc/math/gentype.inc>
-#undef FUNC
-#undef __XFUNC
-#undef __FUNC
+#undef __CLC_FUNC
+#undef __CLC_XFUNC
+#undef __CLC_XFUNC_IMPL
diff --git a/libclc/opencl/lib/clspv/shared/vstore_half.inc b/libclc/opencl/lib/clspv/shared/vstore_half.inc
index f2c02748246a8..d78fbb80976b2 100644
--- a/libclc/opencl/lib/clspv/shared/vstore_half.inc
+++ b/libclc/opencl/lib/clspv/shared/vstore_half.inc
@@ -10,20 +10,20 @@
 #if __CLC_FPSIZE == 32
 #ifndef __CLC_SCALAR
 
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_GENTYPE, __private);
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_GENTYPE, __local);
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_GENTYPE, __global);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_GENTYPE, __private);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_GENTYPE, __local);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_GENTYPE, __global);
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_GENTYPE, __generic);
+__CLC_FUNC(__CLC_VECSIZE, __CLC_VECSIZE, __CLC_GENTYPE, __generic);
 #endif
 
 #undef __CLC_OFFSET
 #else
-FUNC(, 1, __CLC_GENTYPE, __private);
-FUNC(, 1, __CLC_GENTYPE, __local);
-FUNC(, 1, __CLC_GENTYPE, __global);
+__CLC_FUNC(, 1, __CLC_GENTYPE, __private);
+__CLC_FUNC(, 1, __CLC_GENTYPE, __local);
+__CLC_FUNC(, 1, __CLC_GENTYPE, __global);
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-FUNC(, 1, __CLC_GENTYPE, __generic);
+__CLC_FUNC(, 1, __CLC_GENTYPE, __generic);
 #endif
 #endif
 #endif
diff --git a/libclc/opencl/lib/generic/async/async_work_group_strided_copy.inc b/libclc/opencl/lib/generic/async/async_work_group_strided_copy.inc
index c9c32a7d94315..f421298e2ca0f 100644
--- a/libclc/opencl/lib/generic/async/async_work_group_strided_copy.inc
+++ b/libclc/opencl/lib/generic/async/async_work_group_strided_copy.inc
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define STRIDED_COPY(dst, src, num_gentypes, dst_stride, src_stride)           \
+#define __CLC_STRIDED_COPY(dst, src, num_gentypes, dst_stride, src_stride)     \
   size_t size = get_local_size(0) * get_local_size(1) * get_local_size(2);     \
   size_t id = (get_local_size(0) * get_local_size(1) * get_local_id(2)) +      \
               (get_local_size(0) * get_local_id(1)) + get_local_id(0);         \
@@ -20,7 +20,7 @@ _CLC_OVERLOAD _CLC_DEF event_t async_work_group_strided_copy(
     local __CLC_GENTYPE *dst, const global __CLC_GENTYPE *src,
     size_t num_gentypes, size_t src_stride, event_t event) {
 
-  STRIDED_COPY(dst, src, num_gentypes, 1, src_stride);
+  __CLC_STRIDED_COPY(dst, src, num_gentypes, 1, src_stride);
   return event;
 }
 
@@ -28,6 +28,6 @@ _CLC_OVERLOAD _CLC_DEF event_t async_work_group_strided_copy(
     global __CLC_GENTYPE *dst, const local __CLC_GENTYPE *src,
     size_t num_gentypes, size_t dst_stride, event_t event) {
 
-  STRIDED_COPY(dst, src, num_gentypes, dst_stride, 1);
+  __CLC_STRIDED_COPY(dst, src, num_gentypes, dst_stride, 1);
   return event;
 }
diff --git a/libclc/opencl/lib/generic/atomic/atom_add.cl b/libclc/opencl/lib/generic/atomic/atom_add.cl
index 65d7ad595ace6..08fb3fecd5bc9 100644
--- a/libclc/opencl/lib/generic/atomic/atom_add.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_add.cl
@@ -23,15 +23,15 @@
 
 #ifdef cl_khr_int64_base_atomics
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_add(volatile AS TYPE *p, TYPE val) {        \
     return __sync_fetch_and_add_8(p, val);                                     \
   }
 
-IMPL(global, long)
-IMPL(global, unsigned long)
-IMPL(local, long)
-IMPL(local, unsigned long)
-#undef IMPL
+__CLC_IMPL(global, long)
+__CLC_IMPL(global, unsigned long)
+__CLC_IMPL(local, long)
+__CLC_IMPL(local, unsigned long)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_base_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_and.cl b/libclc/opencl/lib/generic/atomic/atom_and.cl
index 371e5f0aa4d0d..1dddd8e72f305 100644
--- a/libclc/opencl/lib/generic/atomic/atom_and.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_and.cl
@@ -23,15 +23,15 @@
 
 #ifdef cl_khr_int64_extended_atomics
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_and(volatile AS TYPE *p, TYPE val) {        \
     return __sync_fetch_and_and_8(p, val);                                     \
   }
 
-IMPL(global, long)
-IMPL(global, unsigned long)
-IMPL(local, long)
-IMPL(local, unsigned long)
-#undef IMPL
+__CLC_IMPL(global, long)
+__CLC_IMPL(global, unsigned long)
+__CLC_IMPL(local, long)
+__CLC_IMPL(local, unsigned long)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_cmpxchg.cl b/libclc/opencl/lib/generic/atomic/atom_cmpxchg.cl
index f129be9809e51..5ae6aa30a8358 100644
--- a/libclc/opencl/lib/generic/atomic/atom_cmpxchg.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_cmpxchg.cl
@@ -9,35 +9,35 @@
 #include <clc/opencl/atomic/atom_cmpxchg.h>
 #include <clc/opencl/atomic/atomic_cmpxchg.h>
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_cmpxchg(volatile AS TYPE *p, TYPE cmp,      \
                                            TYPE val) {                         \
     return atomic_cmpxchg(p, cmp, val);                                        \
   }
 
 #ifdef cl_khr_global_int32_base_atomics
-IMPL(global, int)
-IMPL(global, unsigned int)
+__CLC_IMPL(global, int)
+__CLC_IMPL(global, unsigned int)
 #endif // cl_khr_global_int32_base_atomics
 #ifdef cl_khr_local_int32_base_atomics
-IMPL(local, int)
-IMPL(local, unsigned int)
+__CLC_IMPL(local, int)
+__CLC_IMPL(local, unsigned int)
 #endif // cl_khr_local_int32_base_atomics
 
-#undef IMPL
+#undef __CLC_IMPL
 
 #ifdef cl_khr_int64_base_atomics
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_cmpxchg(volatile AS TYPE *p, TYPE cmp,      \
                                            TYPE val) {                         \
     return __sync_val_compare_and_swap_8(p, cmp, val);                         \
   }
 
-IMPL(global, long)
-IMPL(global, unsigned long)
-IMPL(local, long)
-IMPL(local, unsigned long)
-#undef IMPL
+__CLC_IMPL(global, long)
+__CLC_IMPL(global, unsigned long)
+__CLC_IMPL(local, long)
+__CLC_IMPL(local, unsigned long)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_base_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_dec.cl b/libclc/opencl/lib/generic/atomic/atom_dec.cl
index 2df721cae365a..af811042d307a 100644
--- a/libclc/opencl/lib/generic/atomic/atom_dec.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_dec.cl
@@ -10,33 +10,33 @@
 #include <clc/opencl/atomic/atom_sub.h>
 #include <clc/opencl/atomic/atomic_dec.h>
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_dec(volatile AS TYPE *p) {                  \
     return atomic_dec(p);                                                      \
   }
 
 #ifdef cl_khr_global_int32_base_atomics
-IMPL(global, int)
-IMPL(global, unsigned int)
+__CLC_IMPL(global, int)
+__CLC_IMPL(global, unsigned int)
 #endif // cl_khr_global_int32_base_atomics
 #ifdef cl_khr_local_int32_base_atomics
-IMPL(local, int)
-IMPL(local, unsigned int)
+__CLC_IMPL(local, int)
+__CLC_IMPL(local, unsigned int)
 #endif // cl_khr_local_int32_base_atomics
 
-#undef IMPL
+#undef __CLC_IMPL
 
 #ifdef cl_khr_int64_base_atomics
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_dec(volatile AS TYPE *p) {                  \
     return atom_sub(p, (TYPE)1);                                               \
   }
 
-IMPL(global, long)
-IMPL(global, unsigned long)
-IMPL(local, long)
-IMPL(local, unsigned long)
-#undef IMPL
+__CLC_IMPL(global, long)
+__CLC_IMPL(global, unsigned long)
+__CLC_IMPL(local, long)
+__CLC_IMPL(local, unsigned long)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_base_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_inc.cl b/libclc/opencl/lib/generic/atomic/atom_inc.cl
index dc7699a478020..f881b3a3caa6d 100644
--- a/libclc/opencl/lib/generic/atomic/atom_inc.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_inc.cl
@@ -10,33 +10,33 @@
 #include <clc/opencl/atomic/atom_inc.h>
 #include <clc/opencl/atomic/atomic_inc.h>
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_inc(volatile AS TYPE *p) {                  \
     return atomic_inc(p);                                                      \
   }
 
 #ifdef cl_khr_global_int32_base_atomics
-IMPL(global, int)
-IMPL(global, unsigned int)
+__CLC_IMPL(global, int)
+__CLC_IMPL(global, unsigned int)
 #endif // cl_khr_global_int32_base_atomics
 #ifdef cl_khr_local_int32_base_atomics
-IMPL(local, int)
-IMPL(local, unsigned int)
+__CLC_IMPL(local, int)
+__CLC_IMPL(local, unsigned int)
 #endif // cl_khr_local_int32_base_atomics
 
-#undef IMPL
+#undef __CLC_IMPL
 
 #ifdef cl_khr_int64_base_atomics
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_inc(volatile AS TYPE *p) {                  \
     return atom_add(p, (TYPE)1);                                               \
   }
 
-IMPL(global, long)
-IMPL(global, unsigned long)
-IMPL(local, long)
-IMPL(local, unsigned long)
-#undef IMPL
+__CLC_IMPL(global, long)
+__CLC_IMPL(global, unsigned long)
+__CLC_IMPL(local, long)
+__CLC_IMPL(local, unsigned long)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_base_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_max.cl b/libclc/opencl/lib/generic/atomic/atom_max.cl
index 2542191d04f53..83b532ac19a1e 100644
--- a/libclc/opencl/lib/generic/atomic/atom_max.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_max.cl
@@ -31,15 +31,15 @@ unsigned long
 __clc__sync_fetch_and_umax_global_8(volatile global unsigned long *,
                                     unsigned long);
 
-#define IMPL(AS, TYPE, OP)                                                     \
+#define __CLC_IMPL(AS, TYPE, OP)                                               \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_max(volatile AS TYPE *p, TYPE val) {        \
     return __clc__sync_fetch_and_##OP##_##AS##_8(p, val);                      \
   }
 
-IMPL(global, long, max)
-IMPL(global, unsigned long, umax)
-IMPL(local, long, max)
-IMPL(local, unsigned long, umax)
-#undef IMPL
+__CLC_IMPL(global, long, max)
+__CLC_IMPL(global, unsigned long, umax)
+__CLC_IMPL(local, long, max)
+__CLC_IMPL(local, unsigned long, umax)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_min.cl b/libclc/opencl/lib/generic/atomic/atom_min.cl
index 4e62804824d82..b52e34769cdd8 100644
--- a/libclc/opencl/lib/generic/atomic/atom_min.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_min.cl
@@ -31,15 +31,15 @@ unsigned long
 __clc__sync_fetch_and_umin_global_8(volatile global unsigned long *,
                                     unsigned long);
 
-#define IMPL(AS, TYPE, OP)                                                     \
+#define __CLC_IMPL(AS, TYPE, OP)                                               \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_min(volatile AS TYPE *p, TYPE val) {        \
     return __clc__sync_fetch_and_##OP##_##AS##_8(p, val);                      \
   }
 
-IMPL(global, long, min)
-IMPL(global, unsigned long, umin)
-IMPL(local, long, min)
-IMPL(local, unsigned long, umin)
-#undef IMPL
+__CLC_IMPL(global, long, min)
+__CLC_IMPL(global, unsigned long, umin)
+__CLC_IMPL(local, long, min)
+__CLC_IMPL(local, unsigned long, umin)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_or.cl b/libclc/opencl/lib/generic/atomic/atom_or.cl
index 30ad8e52ca232..fa9737f5f28ee 100644
--- a/libclc/opencl/lib/generic/atomic/atom_or.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_or.cl
@@ -23,15 +23,15 @@
 
 #ifdef cl_khr_int64_extended_atomics
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_or(volatile AS TYPE *p, TYPE val) {         \
     return __sync_fetch_and_or_8(p, val);                                      \
   }
 
-IMPL(global, long)
-IMPL(global, unsigned long)
-IMPL(local, long)
-IMPL(local, unsigned long)
-#undef IMPL
+__CLC_IMPL(global, long)
+__CLC_IMPL(global, unsigned long)
+__CLC_IMPL(local, long)
+__CLC_IMPL(local, unsigned long)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_sub.cl b/libclc/opencl/lib/generic/atomic/atom_sub.cl
index f6ee94db7469f..9a8acfa9116b8 100644
--- a/libclc/opencl/lib/generic/atomic/atom_sub.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_sub.cl
@@ -23,15 +23,15 @@
 
 #ifdef cl_khr_int64_base_atomics
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_sub(volatile AS TYPE *p, TYPE val) {        \
     return __sync_fetch_and_sub_8(p, val);                                     \
   }
 
-IMPL(global, long)
-IMPL(global, unsigned long)
-IMPL(local, long)
-IMPL(local, unsigned long)
-#undef IMPL
+__CLC_IMPL(global, long)
+__CLC_IMPL(global, unsigned long)
+__CLC_IMPL(local, long)
+__CLC_IMPL(local, unsigned long)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_base_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_xchg.cl b/libclc/opencl/lib/generic/atomic/atom_xchg.cl
index 3fa17c950a82e..03f8a9c466c5f 100644
--- a/libclc/opencl/lib/generic/atomic/atom_xchg.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_xchg.cl
@@ -23,15 +23,15 @@
 
 #ifdef cl_khr_int64_base_atomics
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_xchg(volatile AS TYPE *p, TYPE val) {       \
     return __sync_swap_8(p, val);                                              \
   }
 
-IMPL(global, long)
-IMPL(global, unsigned long)
-IMPL(local, long)
-IMPL(local, unsigned long)
-#undef IMPL
+__CLC_IMPL(global, long)
+__CLC_IMPL(global, unsigned long)
+__CLC_IMPL(local, long)
+__CLC_IMPL(local, unsigned long)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_base_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atom_xor.cl b/libclc/opencl/lib/generic/atomic/atom_xor.cl
index ac08ef63b1a4b..392a4b794c698 100644
--- a/libclc/opencl/lib/generic/atomic/atom_xor.cl
+++ b/libclc/opencl/lib/generic/atomic/atom_xor.cl
@@ -23,15 +23,15 @@
 
 #ifdef cl_khr_int64_extended_atomics
 
-#define IMPL(AS, TYPE)                                                         \
+#define __CLC_IMPL(AS, TYPE)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atom_xor(volatile AS TYPE *p, TYPE val) {        \
     return __sync_fetch_and_xor_8(p, val);                                     \
   }
 
-IMPL(global, long)
-IMPL(global, unsigned long)
-IMPL(local, long)
-IMPL(local, unsigned long)
-#undef IMPL
+__CLC_IMPL(global, long)
+__CLC_IMPL(global, unsigned long)
+__CLC_IMPL(local, long)
+__CLC_IMPL(local, unsigned long)
+#undef __CLC_IMPL
 
 #endif // cl_khr_int64_extended_atomics
diff --git a/libclc/opencl/lib/generic/atomic/atomic_add.cl b/libclc/opencl/lib/generic/atomic/atomic_add.cl
index a0effced7dc62..d005c1dd6ac51 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_add.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_add.cl
@@ -8,13 +8,13 @@
 
 #include <clc/opencl/atomic/atomic_add.h>
 
-#define IMPL(TYPE, AS)                                                         \
+#define __CLC_IMPL(TYPE, AS)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_add(volatile AS TYPE *p, TYPE val) {      \
     return __sync_fetch_and_add(p, val);                                       \
   }
 
-IMPL(int, global)
-IMPL(unsigned int, global)
-IMPL(int, local)
-IMPL(unsigned int, local)
-#undef IMPL
+__CLC_IMPL(int, global)
+__CLC_IMPL(unsigned int, global)
+__CLC_IMPL(int, local)
+__CLC_IMPL(unsigned int, local)
+#undef __CLC_IMPL
diff --git a/libclc/opencl/lib/generic/atomic/atomic_and.cl b/libclc/opencl/lib/generic/atomic/atomic_and.cl
index 629e6638d3bc2..12558568b0e4e 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_and.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_and.cl
@@ -8,13 +8,13 @@
 
 #include <clc/opencl/atomic/atomic_and.h>
 
-#define IMPL(TYPE, AS)                                                         \
+#define __CLC_IMPL(TYPE, AS)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_and(volatile AS TYPE *p, TYPE val) {      \
     return __sync_fetch_and_and(p, val);                                       \
   }
 
-IMPL(int, global)
-IMPL(unsigned int, global)
-IMPL(int, local)
-IMPL(unsigned int, local)
-#undef IMPL
+__CLC_IMPL(int, global)
+__CLC_IMPL(unsigned int, global)
+__CLC_IMPL(int, local)
+__CLC_IMPL(unsigned int, local)
+#undef __CLC_IMPL
diff --git a/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl b/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl
index db0495b004c1a..1045020a553fc 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_cmpxchg.cl
@@ -8,14 +8,14 @@
 
 #include <clc/opencl/atomic/atomic_cmpxchg.h>
 
-#define IMPL(TYPE, AS)                                                         \
+#define __CLC_IMPL(TYPE, AS)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_cmpxchg(volatile AS TYPE *p, TYPE cmp,    \
                                              TYPE val) {                       \
     return __sync_val_compare_and_swap(p, cmp, val);                           \
   }
 
-IMPL(int, global)
-IMPL(unsigned int, global)
-IMPL(int, local)
-IMPL(unsigned int, local)
-#undef IMPL
+__CLC_IMPL(int, global)
+__CLC_IMPL(unsigned int, global)
+__CLC_IMPL(int, local)
+__CLC_IMPL(unsigned int, local)
+#undef __CLC_IMPL
diff --git a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl
index 422c03f292071..2c1f07d8ca485 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_strong.cl
@@ -12,7 +12,7 @@
 #include <clc/atomic/clc_atomic_compare_exchange.h>
 #include <clc/opencl/atomic/atomic_compare_exchange_strong.h>
 
-#define FUNCTION atomic_compare_exchange_strong
+#define __CLC_FUNCTION atomic_compare_exchange_strong
 #define __CLC_COMPARE_EXCHANGE
 
 #define __CLC_BODY <atomic_def.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl
index 8a6b3c4f0110e..69bdf37250f70 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_compare_exchange_weak.cl
@@ -12,7 +12,7 @@
 #include <clc/atomic/clc_atomic_compare_exchange.h>
 #include <clc/opencl/atomic/atomic_compare_exchange_weak.h>
 
-#define FUNCTION atomic_compare_exchange_weak
+#define __CLC_FUNCTION atomic_compare_exchange_weak
 #define __CLC_COMPARE_EXCHANGE
 
 #define __CLC_BODY <atomic_def.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_dec.cl b/libclc/opencl/lib/generic/atomic/atomic_dec.cl
index 6de55bc0b9845..ee38267723369 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_dec.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_dec.cl
@@ -9,8 +9,8 @@
 #include <clc/atomic/clc_atomic_dec.h>
 #include <clc/opencl/atomic/atomic_dec.h>
 
-#define FUNCTION atomic_dec
-#define __IMPL_FUNCTION __clc_atomic_dec
+#define __CLC_FUNCTION atomic_dec
+#define __CLC_IMPL_FUNCTION __clc_atomic_dec
 
 #define __CLC_BODY <atomic_inc_dec.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_def.inc b/libclc/opencl/lib/generic/atomic/atomic_def.inc
index ce192bf844938..a4ccab5990888 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_def.inc
+++ b/libclc/opencl/lib/generic/atomic/atomic_def.inc
@@ -10,37 +10,37 @@
 
 #if defined(__opencl_c_fp64) && (defined(cl_khr_int64_base_atomics) &&         \
                                  defined(cl_khr_int64_extended_atomics))
-#define HAVE_64_ATOMIC
+#define __CLC_HAVE_64_ATOMIC
 #endif
-#if defined(__CLC_FPSIZE) && (__CLC_FPSIZE < 64 || defined(HAVE_64_ATOMIC)
-#define HAVE_FP_ATOMIC
+#if defined(__CLC_FPSIZE) && (__CLC_FPSIZE < 64 || defined(__CLC_HAVE_64_ATOMIC)
+#define __CLC_HAVE_FP_ATOMIC
 #endif
 #if defined(__CLC_GENSIZE) &&                                                  \
     ((__CLC_GENSIZE == 32) ||                                                  \
-     (__CLC_GENSIZE == 64 && defined(HAVE_64_ATOMIC)))
-#define HAVE_INT_ATOMIC
+     (__CLC_GENSIZE == 64 && defined(__CLC_HAVE_64_ATOMIC)))
+#define __CLC_HAVE_INT_ATOMIC
 #endif
-#if defined(HAVE_FP_ATOMIC) || defined(HAVE_INT_ATOMIC)
+#if defined(__CLC_HAVE_FP_ATOMIC) || defined(__CLC_HAVE_INT_ATOMIC)
 
 #define __CLC_ATOMIC_GENTYPE __CLC_XCONCAT(atomic_, __CLC_GENTYPE)
 
 #ifdef __CLC_NO_VALUE_ARG
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(                               \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(                         \
       volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr) {                          \
-    return __IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr,            \
-                           __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);           \
+    return __CLC_IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr,        \
+                               __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);       \
   }
 #elif defined(__CLC_RETURN_VOID)
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DEF void FUNCTION(                                        \
+  _CLC_OVERLOAD _CLC_DEF void __CLC_FUNCTION(                                  \
       volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) {     \
-    __IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value,            \
-                    __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);                  \
+    __CLC_IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value,        \
+                        __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);              \
   }
 #elif defined(__CLC_COMPARE_EXCHANGE)
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(                               \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(                         \
       volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr,                            \
       ADDRSPACE __CLC_GENTYPE *Expected, __CLC_GENTYPE Desired) {              \
     __CLC_GENTYPE Comparator = *Expected;                                      \
@@ -55,10 +55,10 @@
   }
 #else
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(                               \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(                         \
       volatile ADDRSPACE __CLC_ATOMIC_GENTYPE *Ptr, __CLC_GENTYPE Value) {     \
-    return __IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value,     \
-                           __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);           \
+    return __CLC_IMPL_FUNCTION((volatile ADDRSPACE __CLC_GENTYPE *)Ptr, Value, \
+                               __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);       \
   }
 #endif
 
@@ -70,10 +70,10 @@ __CLC_DEFINE_ATOMIC()
 
 #undef __CLC_DEFINE_ATOMIC
 
-#endif // HAVE_FP_ATOMIC || HAVE_INT_ATOMIC
+#endif // __CLC_HAVE_FP_ATOMIC || __CLC_HAVE_INT_ATOMIC
 
-#undef HAVE_INT_ATOMIC
-#undef HAVE_FP_ATOMIC
-#undef HAVE_64_ATOMIC
+#undef __CLC_HAVE_INT_ATOMIC
+#undef __CLC_HAVE_FP_ATOMIC
+#undef __CLC_HAVE_64_ATOMIC
 
 #endif // __CLC_SCALAR
diff --git a/libclc/opencl/lib/generic/atomic/atomic_exchange.cl b/libclc/opencl/lib/generic/atomic/atomic_exchange.cl
index 6dae6c0a77599..5f7e2fa593e3f 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_exchange.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_exchange.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_exchange.h>
 #include <clc/opencl/atomic/atomic_exchange.h>
 
-#define FUNCTION atomic_exchange
-#define __IMPL_FUNCTION __clc_atomic_exchange
+#define __CLC_FUNCTION atomic_exchange
+#define __CLC_IMPL_FUNCTION __clc_atomic_exchange
 
 #define __CLC_BODY <atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl
index bbaa1c2b0dacf..0362ff89d1d78 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_add.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_fetch_add.h>
 #include <clc/opencl/atomic/atomic_fetch_add.h>
 
-#define FUNCTION atomic_fetch_add
-#define __IMPL_FUNCTION __clc_atomic_fetch_add
+#define __CLC_FUNCTION atomic_fetch_add
+#define __CLC_IMPL_FUNCTION __clc_atomic_fetch_add
 
 #define __CLC_BODY <atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl
index 73925844c9357..a1796f20c6e44 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_and.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_fetch_and.h>
 #include <clc/opencl/atomic/atomic_fetch_and.h>
 
-#define FUNCTION atomic_fetch_and
-#define __IMPL_FUNCTION __clc_atomic_fetch_and
+#define __CLC_FUNCTION atomic_fetch_and
+#define __CLC_IMPL_FUNCTION __clc_atomic_fetch_and
 
 #define __CLC_BODY <atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl
index 8c8ce11cc575f..03b5d1d8ae7bd 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_max.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_fetch_max.h>
 #include <clc/opencl/atomic/atomic_fetch_max.h>
 
-#define FUNCTION atomic_fetch_max
-#define __IMPL_FUNCTION __clc_atomic_fetch_max
+#define __CLC_FUNCTION atomic_fetch_max
+#define __CLC_IMPL_FUNCTION __clc_atomic_fetch_max
 
 #define __CLC_BODY <atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl
index 550459cee32d6..60ffeff04cc6a 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_min.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_fetch_min.h>
 #include <clc/opencl/atomic/atomic_fetch_min.h>
 
-#define FUNCTION atomic_fetch_min
-#define __IMPL_FUNCTION __clc_atomic_fetch_min
+#define __CLC_FUNCTION atomic_fetch_min
+#define __CLC_IMPL_FUNCTION __clc_atomic_fetch_min
 
 #define __CLC_BODY <atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl
index 2606ff3c99673..8f4100bb150e3 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_or.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_fetch_or.h>
 #include <clc/opencl/atomic/atomic_fetch_or.h>
 
-#define FUNCTION atomic_fetch_or
-#define __IMPL_FUNCTION __clc_atomic_fetch_or
+#define __CLC_FUNCTION atomic_fetch_or
+#define __CLC_IMPL_FUNCTION __clc_atomic_fetch_or
 
 #define __CLC_BODY <atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl
index 33772233bebed..ecb5b4315ee86 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_sub.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_fetch_sub.h>
 #include <clc/opencl/atomic/atomic_fetch_sub.h>
 
-#define FUNCTION atomic_fetch_sub
-#define __IMPL_FUNCTION __clc_atomic_fetch_sub
+#define __CLC_FUNCTION atomic_fetch_sub
+#define __CLC_IMPL_FUNCTION __clc_atomic_fetch_sub
 
 #define __CLC_BODY <atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl b/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl
index 6f6503e588b6f..c49a55820c8d4 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_fetch_xor.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_fetch_xor.h>
 #include <clc/opencl/atomic/atomic_fetch_xor.h>
 
-#define FUNCTION atomic_fetch_xor
-#define __IMPL_FUNCTION __clc_atomic_fetch_xor
+#define __CLC_FUNCTION atomic_fetch_xor
+#define __CLC_IMPL_FUNCTION __clc_atomic_fetch_xor
 
 #define __CLC_BODY <atomic_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_inc.cl b/libclc/opencl/lib/generic/atomic/atomic_inc.cl
index a160b2e2370fc..7cf82b25a0d43 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_inc.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_inc.cl
@@ -9,8 +9,8 @@
 #include <clc/atomic/clc_atomic_inc.h>
 #include <clc/opencl/atomic/atomic_inc.h>
 
-#define FUNCTION atomic_inc
-#define __IMPL_FUNCTION __clc_atomic_inc
+#define __CLC_FUNCTION atomic_inc
+#define __CLC_IMPL_FUNCTION __clc_atomic_inc
 
 #define __CLC_BODY <atomic_inc_dec.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc b/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc
index 0bcf300dd284a..03eb5d1b33057 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc
+++ b/libclc/opencl/lib/generic/atomic/atomic_inc_dec.inc
@@ -11,9 +11,9 @@
 #if __CLC_GENSIZE == 32
 
 #define __CLC_DEFINE_ATOMIC(ADDRSPACE)                                         \
-  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE FUNCTION(                               \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_FUNCTION(                         \
       volatile ADDRSPACE __CLC_GENTYPE *Ptr) {                                 \
-    return __IMPL_FUNCTION(Ptr, __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);      \
+    return __CLC_IMPL_FUNCTION(Ptr, __ATOMIC_SEQ_CST, __MEMORY_SCOPE_DEVICE);  \
   }
 
 __CLC_DEFINE_ATOMIC(global)
diff --git a/libclc/opencl/lib/generic/atomic/atomic_load.cl b/libclc/opencl/lib/generic/atomic/atomic_load.cl
index 459265473a8c8..e904330be0064 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_load.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_load.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_load.h>
 #include <clc/opencl/atomic/atomic_load.h>
 
-#define FUNCTION atomic_load
-#define __IMPL_FUNCTION __clc_atomic_load
+#define __CLC_FUNCTION atomic_load
+#define __CLC_IMPL_FUNCTION __clc_atomic_load
 #define __CLC_NO_VALUE_ARG
 
 #define __CLC_BODY <atomic_def.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_max.cl b/libclc/opencl/lib/generic/atomic/atomic_max.cl
index ae929f0523dc2..aa482a8f46397 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_max.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_max.cl
@@ -8,13 +8,13 @@
 
 #include <clc/opencl/atomic/atomic_max.h>
 
-#define IMPL(TYPE, AS, OP)                                                     \
+#define __CLC_IMPL(TYPE, AS, OP)                                               \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_max(volatile AS TYPE *p, TYPE val) {      \
     return __sync_fetch_and_##OP(p, val);                                      \
   }
 
-IMPL(int, global, max)
-IMPL(unsigned int, global, umax)
-IMPL(int, local, max)
-IMPL(unsigned int, local, umax)
-#undef IMPL
+__CLC_IMPL(int, global, max)
+__CLC_IMPL(unsigned int, global, umax)
+__CLC_IMPL(int, local, max)
+__CLC_IMPL(unsigned int, local, umax)
+#undef __CLC_IMPL
diff --git a/libclc/opencl/lib/generic/atomic/atomic_min.cl b/libclc/opencl/lib/generic/atomic/atomic_min.cl
index c7ebe71ee8295..7f39e94316846 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_min.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_min.cl
@@ -8,13 +8,13 @@
 
 #include <clc/opencl/atomic/atomic_min.h>
 
-#define IMPL(TYPE, AS, OP)                                                     \
+#define __CLC_IMPL(TYPE, AS, OP)                                               \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_min(volatile AS TYPE *p, TYPE val) {      \
     return __sync_fetch_and_##OP(p, val);                                      \
   }
 
-IMPL(int, global, min)
-IMPL(unsigned int, global, umin)
-IMPL(int, local, min)
-IMPL(unsigned int, local, umin)
-#undef IMPL
+__CLC_IMPL(int, global, min)
+__CLC_IMPL(unsigned int, global, umin)
+__CLC_IMPL(int, local, min)
+__CLC_IMPL(unsigned int, local, umin)
+#undef __CLC_IMPL
diff --git a/libclc/opencl/lib/generic/atomic/atomic_or.cl b/libclc/opencl/lib/generic/atomic/atomic_or.cl
index 45fb865689655..ad14cd2178555 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_or.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_or.cl
@@ -8,13 +8,13 @@
 
 #include <clc/opencl/atomic/atomic_or.h>
 
-#define IMPL(TYPE, AS)                                                         \
+#define __CLC_IMPL(TYPE, AS)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_or(volatile AS TYPE *p, TYPE val) {       \
     return __sync_fetch_and_or(p, val);                                        \
   }
 
-IMPL(int, global)
-IMPL(unsigned int, global)
-IMPL(int, local)
-IMPL(unsigned int, local)
-#undef IMPL
+__CLC_IMPL(int, global)
+__CLC_IMPL(unsigned int, global)
+__CLC_IMPL(int, local)
+__CLC_IMPL(unsigned int, local)
+#undef __CLC_IMPL
diff --git a/libclc/opencl/lib/generic/atomic/atomic_store.cl b/libclc/opencl/lib/generic/atomic/atomic_store.cl
index 67f2c8457fc10..584e29ef99a5f 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_store.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_store.cl
@@ -12,8 +12,8 @@
 #include <clc/atomic/clc_atomic_store.h>
 #include <clc/opencl/atomic/atomic_store.h>
 
-#define FUNCTION atomic_store
-#define __IMPL_FUNCTION __clc_atomic_store
+#define __CLC_FUNCTION atomic_store
+#define __CLC_IMPL_FUNCTION __clc_atomic_store
 #define __CLC_RETURN_VOID
 
 #define __CLC_BODY <atomic_def.inc>
diff --git a/libclc/opencl/lib/generic/atomic/atomic_sub.cl b/libclc/opencl/lib/generic/atomic/atomic_sub.cl
index 74977f15155fb..2e51c4c2ce02f 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_sub.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_sub.cl
@@ -8,13 +8,13 @@
 
 #include <clc/opencl/atomic/atomic_sub.h>
 
-#define IMPL(TYPE, AS)                                                         \
+#define __CLC_IMPL(TYPE, AS)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_sub(volatile AS TYPE *p, TYPE val) {      \
     return __sync_fetch_and_sub(p, val);                                       \
   }
 
-IMPL(int, global)
-IMPL(unsigned int, global)
-IMPL(int, local)
-IMPL(unsigned int, local)
-#undef IMPL
+__CLC_IMPL(int, global)
+__CLC_IMPL(unsigned int, global)
+__CLC_IMPL(int, local)
+__CLC_IMPL(unsigned int, local)
+#undef __CLC_IMPL
diff --git a/libclc/opencl/lib/generic/atomic/atomic_xchg.cl b/libclc/opencl/lib/generic/atomic/atomic_xchg.cl
index 883132caf9fbb..2585a5427392e 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_xchg.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_xchg.cl
@@ -17,13 +17,13 @@ _CLC_OVERLOAD _CLC_DEF float atomic_xchg(volatile local float *p, float val) {
   return as_float(atomic_xchg((volatile local uint *)p, as_uint(val)));
 }
 
-#define IMPL(TYPE, AS)                                                         \
+#define __CLC_IMPL(TYPE, AS)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_xchg(volatile AS TYPE *p, TYPE val) {     \
     return __sync_swap_4(p, val);                                              \
   }
 
-IMPL(int, global)
-IMPL(unsigned int, global)
-IMPL(int, local)
-IMPL(unsigned int, local)
-#undef IMPL
+__CLC_IMPL(int, global)
+__CLC_IMPL(unsigned int, global)
+__CLC_IMPL(int, local)
+__CLC_IMPL(unsigned int, local)
+#undef __CLC_IMPL
diff --git a/libclc/opencl/lib/generic/atomic/atomic_xor.cl b/libclc/opencl/lib/generic/atomic/atomic_xor.cl
index 246752c258374..0228134397464 100644
--- a/libclc/opencl/lib/generic/atomic/atomic_xor.cl
+++ b/libclc/opencl/lib/generic/atomic/atomic_xor.cl
@@ -8,13 +8,13 @@
 
 #include <clc/opencl/atomic/atomic_xor.h>
 
-#define IMPL(TYPE, AS)                                                         \
+#define __CLC_IMPL(TYPE, AS)                                                   \
   _CLC_OVERLOAD _CLC_DEF TYPE atomic_xor(volatile AS TYPE *p, TYPE val) {      \
     return __sync_fetch_and_xor(p, val);                                       \
   }
 
-IMPL(int, global)
-IMPL(unsigned int, global)
-IMPL(int, local)
-IMPL(unsigned int, local)
-#undef IMPL
+__CLC_IMPL(int, global)
+__CLC_IMPL(unsigned int, global)
+__CLC_IMPL(int, local)
+__CLC_IMPL(unsigned int, local)
+#undef __CLC_IMPL
diff --git a/libclc/opencl/lib/generic/common/degrees.cl b/libclc/opencl/lib/generic/common/degrees.cl
index d989fa6382edb..15b85af997b4d 100644
--- a/libclc/opencl/lib/generic/common/degrees.cl
+++ b/libclc/opencl/lib/generic/common/degrees.cl
@@ -9,7 +9,7 @@
 #include <clc/common/clc_degrees.h>
 #include <clc/opencl/common/degrees.h>
 
-#define FUNCTION degrees
+#define __CLC_FUNCTION degrees
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/common/radians.cl b/libclc/opencl/lib/generic/common/radians.cl
index 1da38920d43fb..6550cf458a7a8 100644
--- a/libclc/opencl/lib/generic/common/radians.cl
+++ b/libclc/opencl/lib/generic/common/radians.cl
@@ -9,7 +9,7 @@
 #include <clc/common/clc_radians.h>
 #include <clc/opencl/common/radians.h>
 
-#define FUNCTION radians
+#define __CLC_FUNCTION radians
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/common/sign.cl b/libclc/opencl/lib/generic/common/sign.cl
index 9afcb7375e96a..7add739be6a9c 100644
--- a/libclc/opencl/lib/generic/common/sign.cl
+++ b/libclc/opencl/lib/generic/common/sign.cl
@@ -10,7 +10,7 @@
 #include <clc/common/clc_sign.h>
 #include <clc/opencl/common/sign.h>
 
-#define FUNCTION sign
+#define __CLC_FUNCTION sign
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/geometric/distance.cl b/libclc/opencl/lib/generic/geometric/distance.cl
index 54c78d3609ba7..4d9f5403f8c2d 100644
--- a/libclc/opencl/lib/generic/geometric/distance.cl
+++ b/libclc/opencl/lib/generic/geometric/distance.cl
@@ -9,6 +9,6 @@
 #include <clc/geometric/clc_distance.h>
 #include <clc/opencl/geometric/distance.h>
 
-#define FUNCTION distance
+#define __CLC_FUNCTION distance
 #define __CLC_BODY <clc/geometric/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/geometric/dot.cl b/libclc/opencl/lib/generic/geometric/dot.cl
index 05425aafb0404..fc207c8fc155a 100644
--- a/libclc/opencl/lib/generic/geometric/dot.cl
+++ b/libclc/opencl/lib/generic/geometric/dot.cl
@@ -9,6 +9,6 @@
 #include <clc/geometric/clc_dot.h>
 #include <clc/opencl/geometric/dot.h>
 
-#define FUNCTION dot
+#define __CLC_FUNCTION dot
 #define __CLC_BODY <clc/geometric/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/geometric/fast_distance.cl b/libclc/opencl/lib/generic/geometric/fast_distance.cl
index bc30c298a2b9a..a9c2c66dc9a19 100644
--- a/libclc/opencl/lib/generic/geometric/fast_distance.cl
+++ b/libclc/opencl/lib/generic/geometric/fast_distance.cl
@@ -9,7 +9,7 @@
 #include <clc/geometric/clc_fast_distance.h>
 #include <clc/opencl/geometric/fast_distance.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION fast_distance
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION fast_distance
 #define __CLC_BODY <clc/geometric/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/geometric/fast_length.cl b/libclc/opencl/lib/generic/geometric/fast_length.cl
index 407080900cceb..8e6ce1aa5c564 100644
--- a/libclc/opencl/lib/generic/geometric/fast_length.cl
+++ b/libclc/opencl/lib/generic/geometric/fast_length.cl
@@ -9,7 +9,7 @@
 #include <clc/geometric/clc_fast_length.h>
 #include <clc/opencl/geometric/fast_length.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION fast_length
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION fast_length
 #define __CLC_BODY <clc/geometric/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/geometric/fast_normalize.cl b/libclc/opencl/lib/generic/geometric/fast_normalize.cl
index 195081371a5b2..d2f7111516f44 100644
--- a/libclc/opencl/lib/generic/geometric/fast_normalize.cl
+++ b/libclc/opencl/lib/generic/geometric/fast_normalize.cl
@@ -9,8 +9,8 @@
 #include <clc/geometric/clc_fast_normalize.h>
 #include <clc/opencl/geometric/fast_normalize.h>
 
-#define FUNCTION fast_normalize
-#define __FLOAT_ONLY
+#define __CLC_FUNCTION fast_normalize
+#define __CLC_FLOAT_ONLY
 #define __CLC_GEOMETRIC_RET_GENTYPE
 #define __CLC_BODY <clc/geometric/unary_def.inc>
 
diff --git a/libclc/opencl/lib/generic/geometric/length.cl b/libclc/opencl/lib/generic/geometric/length.cl
index 0b0ebaa74d951..59e2055020d88 100644
--- a/libclc/opencl/lib/generic/geometric/length.cl
+++ b/libclc/opencl/lib/generic/geometric/length.cl
@@ -9,6 +9,6 @@
 #include <clc/geometric/clc_length.h>
 #include <clc/opencl/geometric/length.h>
 
-#define FUNCTION length
+#define __CLC_FUNCTION length
 #define __CLC_BODY <clc/geometric/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/geometric/normalize.cl b/libclc/opencl/lib/generic/geometric/normalize.cl
index eb84ffd382129..968d4001ae999 100644
--- a/libclc/opencl/lib/generic/geometric/normalize.cl
+++ b/libclc/opencl/lib/generic/geometric/normalize.cl
@@ -9,7 +9,7 @@
 #include <clc/geometric/clc_normalize.h>
 #include <clc/opencl/geometric/normalize.h>
 
-#define FUNCTION normalize
+#define __CLC_FUNCTION normalize
 #define __CLC_GEOMETRIC_RET_GENTYPE
 #define __CLC_BODY <clc/geometric/unary_def.inc>
 
diff --git a/libclc/opencl/lib/generic/integer/add_sat.cl b/libclc/opencl/lib/generic/integer/add_sat.cl
index 50d989a47211b..6cde612e5fb0f 100644
--- a/libclc/opencl/lib/generic/integer/add_sat.cl
+++ b/libclc/opencl/lib/generic/integer/add_sat.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_add_sat.h>
 #include <clc/opencl/integer/add_sat.h>
 
-#define FUNCTION add_sat
+#define __CLC_FUNCTION add_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/bit_reverse.cl b/libclc/opencl/lib/generic/integer/bit_reverse.cl
index 23181b6b3eba5..b35a5c473a1e6 100644
--- a/libclc/opencl/lib/generic/integer/bit_reverse.cl
+++ b/libclc/opencl/lib/generic/integer/bit_reverse.cl
@@ -11,7 +11,7 @@
 #include <clc/integer/clc_bit_reverse.h>
 #include <clc/opencl/integer/bit_reverse.h>
 
-#define FUNCTION bit_reverse
+#define __CLC_FUNCTION bit_reverse
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/bitfield_extract_def.inc b/libclc/opencl/lib/generic/integer/bitfield_extract_def.inc
index 0262f67732afc..d3b8badcc8c99 100644
--- a/libclc/opencl/lib/generic/integer/bitfield_extract_def.inc
+++ b/libclc/opencl/lib/generic/integer/bitfield_extract_def.inc
@@ -6,11 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef __IMPL_FUNCTION
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#ifndef __CLC_IMPL_FUNCTION
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __RETTYPE FUNCTION(__CLC_GENTYPE base, uint offset,
-                                          uint count) {
-  return __IMPL_FUNCTION(FUNCTION)(base, offset, count);
+_CLC_OVERLOAD _CLC_DEF __CLC_RETTYPE __CLC_FUNCTION(__CLC_GENTYPE base,
+                                                    uint offset, uint count) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(base, offset, count);
 }
diff --git a/libclc/opencl/lib/generic/integer/bitfield_extract_signed.cl b/libclc/opencl/lib/generic/integer/bitfield_extract_signed.cl
index eaa4ac779cfd1..ad49cd36b766e 100644
--- a/libclc/opencl/lib/generic/integer/bitfield_extract_signed.cl
+++ b/libclc/opencl/lib/generic/integer/bitfield_extract_signed.cl
@@ -11,8 +11,8 @@
 #include <clc/integer/clc_bitfield_extract_signed.h>
 #include <clc/opencl/integer/bitfield_extract_signed.h>
 
-#define FUNCTION bitfield_extract_signed
-#define __RETTYPE __CLC_S_GENTYPE
+#define __CLC_FUNCTION bitfield_extract_signed
+#define __CLC_RETTYPE __CLC_S_GENTYPE
 
 #define __CLC_BODY <bitfield_extract_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/bitfield_extract_unsigned.cl b/libclc/opencl/lib/generic/integer/bitfield_extract_unsigned.cl
index fd63d5d6dee30..e2fe92cfcce82 100644
--- a/libclc/opencl/lib/generic/integer/bitfield_extract_unsigned.cl
+++ b/libclc/opencl/lib/generic/integer/bitfield_extract_unsigned.cl
@@ -11,8 +11,8 @@
 #include <clc/integer/clc_bitfield_extract_unsigned.h>
 #include <clc/opencl/integer/bitfield_extract_unsigned.h>
 
-#define FUNCTION bitfield_extract_unsigned
-#define __RETTYPE __CLC_U_GENTYPE
+#define __CLC_FUNCTION bitfield_extract_unsigned
+#define __CLC_RETTYPE __CLC_U_GENTYPE
 
 #define __CLC_BODY <bitfield_extract_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/bitfield_insert.cl b/libclc/opencl/lib/generic/integer/bitfield_insert.cl
index 6b441155f393b..c165bd756ffef 100644
--- a/libclc/opencl/lib/generic/integer/bitfield_insert.cl
+++ b/libclc/opencl/lib/generic/integer/bitfield_insert.cl
@@ -11,7 +11,7 @@
 #include <clc/integer/clc_bitfield_insert.h>
 #include <clc/opencl/integer/bitfield_insert.h>
 
-#define FUNCTION bitfield_insert
+#define __CLC_FUNCTION bitfield_insert
 #define __CLC_BODY <clc/integer/clc_bitfield_insert.inc>
 #include <clc/integer/gentype.inc>
 
diff --git a/libclc/opencl/lib/generic/integer/clz.cl b/libclc/opencl/lib/generic/integer/clz.cl
index 58110df1eec24..d0509cba5913a 100644
--- a/libclc/opencl/lib/generic/integer/clz.cl
+++ b/libclc/opencl/lib/generic/integer/clz.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_clz.h>
 #include <clc/opencl/integer/clz.h>
 
-#define FUNCTION clz
+#define __CLC_FUNCTION clz
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/ctz.cl b/libclc/opencl/lib/generic/integer/ctz.cl
index cbb167275a4a6..640ccfa5f3117 100644
--- a/libclc/opencl/lib/generic/integer/ctz.cl
+++ b/libclc/opencl/lib/generic/integer/ctz.cl
@@ -11,7 +11,7 @@
 #include <clc/integer/clc_ctz.h>
 #include <clc/opencl/integer/ctz.h>
 
-#define FUNCTION ctz
+#define __CLC_FUNCTION ctz
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/hadd.cl b/libclc/opencl/lib/generic/integer/hadd.cl
index e9edc88bce02c..0082524b55af2 100644
--- a/libclc/opencl/lib/generic/integer/hadd.cl
+++ b/libclc/opencl/lib/generic/integer/hadd.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_hadd.h>
 #include <clc/opencl/integer/hadd.h>
 
-#define FUNCTION hadd
+#define __CLC_FUNCTION hadd
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/mad24.cl b/libclc/opencl/lib/generic/integer/mad24.cl
index 5e9dfb3b3031a..fc8e0e3171334 100644
--- a/libclc/opencl/lib/generic/integer/mad24.cl
+++ b/libclc/opencl/lib/generic/integer/mad24.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_mad24.h>
 #include <clc/opencl/integer/mad24.h>
 
-#define FUNCTION mad24
+#define __CLC_FUNCTION mad24
 #define __CLC_BODY <clc/shared/ternary_def.inc>
 
 #include <clc/integer/gentype24.inc>
diff --git a/libclc/opencl/lib/generic/integer/mad_hi.cl b/libclc/opencl/lib/generic/integer/mad_hi.cl
index 786614e42bc8b..20d096c59650d 100644
--- a/libclc/opencl/lib/generic/integer/mad_hi.cl
+++ b/libclc/opencl/lib/generic/integer/mad_hi.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_mad_hi.h>
 #include <clc/opencl/integer/mad_hi.h>
 
-#define FUNCTION mad_hi
+#define __CLC_FUNCTION mad_hi
 #define __CLC_BODY <clc/shared/ternary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/mad_sat.cl b/libclc/opencl/lib/generic/integer/mad_sat.cl
index 54ddc0eed2d17..310b31a0ca239 100644
--- a/libclc/opencl/lib/generic/integer/mad_sat.cl
+++ b/libclc/opencl/lib/generic/integer/mad_sat.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_mad_sat.h>
 #include <clc/opencl/integer/mad_sat.h>
 
-#define FUNCTION mad_sat
+#define __CLC_FUNCTION mad_sat
 #define __CLC_BODY <clc/shared/ternary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/mul24.cl b/libclc/opencl/lib/generic/integer/mul24.cl
index e6db6b89c5d98..243bff77da91a 100644
--- a/libclc/opencl/lib/generic/integer/mul24.cl
+++ b/libclc/opencl/lib/generic/integer/mul24.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_mul24.h>
 #include <clc/opencl/integer/mul24.h>
 
-#define FUNCTION mul24
+#define __CLC_FUNCTION mul24
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype24.inc>
diff --git a/libclc/opencl/lib/generic/integer/mul_hi.cl b/libclc/opencl/lib/generic/integer/mul_hi.cl
index 4c7b8c2caa994..99d3051a0df9e 100644
--- a/libclc/opencl/lib/generic/integer/mul_hi.cl
+++ b/libclc/opencl/lib/generic/integer/mul_hi.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_mul_hi.h>
 #include <clc/opencl/integer/mul_hi.h>
 
-#define FUNCTION mul_hi
+#define __CLC_FUNCTION mul_hi
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/popcount.cl b/libclc/opencl/lib/generic/integer/popcount.cl
index 3c33c82302c39..e4219b4ff9c37 100644
--- a/libclc/opencl/lib/generic/integer/popcount.cl
+++ b/libclc/opencl/lib/generic/integer/popcount.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_popcount.h>
 #include <clc/opencl/integer/popcount.h>
 
-#define FUNCTION popcount
+#define __CLC_FUNCTION popcount
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/rhadd.cl b/libclc/opencl/lib/generic/integer/rhadd.cl
index f835511899429..df9f1eff3e2d5 100644
--- a/libclc/opencl/lib/generic/integer/rhadd.cl
+++ b/libclc/opencl/lib/generic/integer/rhadd.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_rhadd.h>
 #include <clc/opencl/integer/rhadd.h>
 
-#define FUNCTION rhadd
+#define __CLC_FUNCTION rhadd
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/rotate.cl b/libclc/opencl/lib/generic/integer/rotate.cl
index 282270f3b3ff2..f5b6195275deb 100644
--- a/libclc/opencl/lib/generic/integer/rotate.cl
+++ b/libclc/opencl/lib/generic/integer/rotate.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_rotate.h>
 #include <clc/opencl/integer/rotate.h>
 
-#define FUNCTION rotate
+#define __CLC_FUNCTION rotate
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/integer/sub_sat.cl b/libclc/opencl/lib/generic/integer/sub_sat.cl
index 0b45df1118f14..81517022160ae 100644
--- a/libclc/opencl/lib/generic/integer/sub_sat.cl
+++ b/libclc/opencl/lib/generic/integer/sub_sat.cl
@@ -9,7 +9,7 @@
 #include <clc/integer/clc_sub_sat.h>
 #include <clc/opencl/integer/sub_sat.h>
 
-#define FUNCTION sub_sat
+#define __CLC_FUNCTION sub_sat
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/acos.cl b/libclc/opencl/lib/generic/math/acos.cl
index 040b2b82f6f98..bdaeedf02e908 100644
--- a/libclc/opencl/lib/generic/math/acos.cl
+++ b/libclc/opencl/lib/generic/math/acos.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_acos.h>
 #include <clc/opencl/math/acos.h>
 
-#define FUNCTION acos
+#define __CLC_FUNCTION acos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/acosh.cl b/libclc/opencl/lib/generic/math/acosh.cl
index 55b60bc99614a..c46532dbcbd95 100644
--- a/libclc/opencl/lib/generic/math/acosh.cl
+++ b/libclc/opencl/lib/generic/math/acosh.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_acosh.h>
 #include <clc/opencl/math/acosh.h>
 
-#define FUNCTION acosh
+#define __CLC_FUNCTION acosh
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/acospi.cl b/libclc/opencl/lib/generic/math/acospi.cl
index 2537c992e2ada..b6b0fb91250d0 100644
--- a/libclc/opencl/lib/generic/math/acospi.cl
+++ b/libclc/opencl/lib/generic/math/acospi.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_acospi.h>
 #include <clc/opencl/math/acospi.h>
 
-#define FUNCTION acospi
+#define __CLC_FUNCTION acospi
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/asin.cl b/libclc/opencl/lib/generic/math/asin.cl
index 08dde1201392e..31fc36d746019 100644
--- a/libclc/opencl/lib/generic/math/asin.cl
+++ b/libclc/opencl/lib/generic/math/asin.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_asin.h>
 #include <clc/opencl/math/asin.h>
 
-#define FUNCTION asin
+#define __CLC_FUNCTION asin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/asinh.cl b/libclc/opencl/lib/generic/math/asinh.cl
index 189392f50b5ee..3bdd8099f079b 100644
--- a/libclc/opencl/lib/generic/math/asinh.cl
+++ b/libclc/opencl/lib/generic/math/asinh.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_asinh.h>
 #include <clc/opencl/math/asinh.h>
 
-#define FUNCTION asinh
+#define __CLC_FUNCTION asinh
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/asinpi.cl b/libclc/opencl/lib/generic/math/asinpi.cl
index b9327faaccba5..6df9047ced3c5 100644
--- a/libclc/opencl/lib/generic/math/asinpi.cl
+++ b/libclc/opencl/lib/generic/math/asinpi.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_asinpi.h>
 #include <clc/opencl/math/asinpi.h>
 
-#define FUNCTION asinpi
+#define __CLC_FUNCTION asinpi
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/atan.cl b/libclc/opencl/lib/generic/math/atan.cl
index e2b1530f9e4cf..157e50dd0e07a 100644
--- a/libclc/opencl/lib/generic/math/atan.cl
+++ b/libclc/opencl/lib/generic/math/atan.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_atan.h>
 #include <clc/opencl/math/atan.h>
 
-#define FUNCTION atan
+#define __CLC_FUNCTION atan
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/atan2.cl b/libclc/opencl/lib/generic/math/atan2.cl
index 9f3d4a965e58a..7db630608867c 100644
--- a/libclc/opencl/lib/generic/math/atan2.cl
+++ b/libclc/opencl/lib/generic/math/atan2.cl
@@ -10,7 +10,7 @@
 #include <clc/math/clc_atan2.h>
 #include <clc/opencl/math/atan2.h>
 
-#define FUNCTION atan2
+#define __CLC_FUNCTION atan2
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/atan2pi.cl b/libclc/opencl/lib/generic/math/atan2pi.cl
index 9b3fea163f685..ed57c920f1b2d 100644
--- a/libclc/opencl/lib/generic/math/atan2pi.cl
+++ b/libclc/opencl/lib/generic/math/atan2pi.cl
@@ -10,7 +10,7 @@
 #include <clc/math/clc_atan2pi.h>
 #include <clc/opencl/math/atan2pi.h>
 
-#define FUNCTION atan2pi
+#define __CLC_FUNCTION atan2pi
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/atanh.cl b/libclc/opencl/lib/generic/math/atanh.cl
index 5e4564d347026..d0337ff1ffac3 100644
--- a/libclc/opencl/lib/generic/math/atanh.cl
+++ b/libclc/opencl/lib/generic/math/atanh.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_atanh.h>
 #include <clc/opencl/math/atanh.h>
 
-#define FUNCTION atanh
+#define __CLC_FUNCTION atanh
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/atanpi.cl b/libclc/opencl/lib/generic/math/atanpi.cl
index ddeb38e64e993..11630db430c71 100644
--- a/libclc/opencl/lib/generic/math/atanpi.cl
+++ b/libclc/opencl/lib/generic/math/atanpi.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_atanpi.h>
 #include <clc/opencl/math/atanpi.h>
 
-#define FUNCTION atanpi
+#define __CLC_FUNCTION atanpi
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/cbrt.cl b/libclc/opencl/lib/generic/math/cbrt.cl
index dccb49a47dc03..0d670150ed4c9 100644
--- a/libclc/opencl/lib/generic/math/cbrt.cl
+++ b/libclc/opencl/lib/generic/math/cbrt.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_cbrt.inc>
 #include <clc/opencl/math/cbrt.h>
 
-#define FUNCTION cbrt
+#define __CLC_FUNCTION cbrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/ceil.cl b/libclc/opencl/lib/generic/math/ceil.cl
index e312281b98a23..e1bffbcb68600 100644
--- a/libclc/opencl/lib/generic/math/ceil.cl
+++ b/libclc/opencl/lib/generic/math/ceil.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_ceil.h>
 #include <clc/opencl/math/ceil.h>
 
-#define FUNCTION ceil
+#define __CLC_FUNCTION ceil
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/copysign.cl b/libclc/opencl/lib/generic/math/copysign.cl
index 7aa4cb2da0eec..5234b534631c8 100644
--- a/libclc/opencl/lib/generic/math/copysign.cl
+++ b/libclc/opencl/lib/generic/math/copysign.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_copysign.h>
 #include <clc/opencl/math/copysign.h>
 
-#define FUNCTION copysign
+#define __CLC_FUNCTION copysign
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/cos.cl b/libclc/opencl/lib/generic/math/cos.cl
index fb40b6c00e3c8..69c9c37cf3e84 100644
--- a/libclc/opencl/lib/generic/math/cos.cl
+++ b/libclc/opencl/lib/generic/math/cos.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_cos.h>
 #include <clc/opencl/math/cos.h>
 
-#define FUNCTION cos
+#define __CLC_FUNCTION cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/cosh.cl b/libclc/opencl/lib/generic/math/cosh.cl
index 7106fc9f18fa5..de9e750c1a7f0 100644
--- a/libclc/opencl/lib/generic/math/cosh.cl
+++ b/libclc/opencl/lib/generic/math/cosh.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_cosh.h>
 #include <clc/opencl/math/cosh.h>
 
-#define FUNCTION cosh
+#define __CLC_FUNCTION cosh
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/cospi.cl b/libclc/opencl/lib/generic/math/cospi.cl
index 0e242ad0d4779..d9afd05bbabf3 100644
--- a/libclc/opencl/lib/generic/math/cospi.cl
+++ b/libclc/opencl/lib/generic/math/cospi.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_cospi.h>
 #include <clc/opencl/math/cospi.h>
 
-#define FUNCTION cospi
+#define __CLC_FUNCTION cospi
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/erf.cl b/libclc/opencl/lib/generic/math/erf.cl
index 83c38867e0460..8d5a80ba10e47 100644
--- a/libclc/opencl/lib/generic/math/erf.cl
+++ b/libclc/opencl/lib/generic/math/erf.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_erf.h>
 #include <clc/opencl/math/erf.h>
 
-#define FUNCTION erf
+#define __CLC_FUNCTION erf
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/erfc.cl b/libclc/opencl/lib/generic/math/erfc.cl
index 1c473999bba93..a3fda46237d99 100644
--- a/libclc/opencl/lib/generic/math/erfc.cl
+++ b/libclc/opencl/lib/generic/math/erfc.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_erfc.h>
 #include <clc/opencl/math/erfc.h>
 
-#define FUNCTION erfc
+#define __CLC_FUNCTION erfc
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/exp.cl b/libclc/opencl/lib/generic/math/exp.cl
index 8125ce314e8e4..a9fd07a26d0e1 100644
--- a/libclc/opencl/lib/generic/math/exp.cl
+++ b/libclc/opencl/lib/generic/math/exp.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_exp.h>
 #include <clc/opencl/math/exp.h>
 
-#define FUNCTION exp
+#define __CLC_FUNCTION exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/exp10.cl b/libclc/opencl/lib/generic/math/exp10.cl
index f8c0f94a990bc..7db7bafd226f4 100644
--- a/libclc/opencl/lib/generic/math/exp10.cl
+++ b/libclc/opencl/lib/generic/math/exp10.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_exp10.h>
 #include <clc/opencl/math/exp10.h>
 
-#define FUNCTION exp10
+#define __CLC_FUNCTION exp10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/exp2.cl b/libclc/opencl/lib/generic/math/exp2.cl
index dc75c2201a460..1ffc34ae13ce0 100644
--- a/libclc/opencl/lib/generic/math/exp2.cl
+++ b/libclc/opencl/lib/generic/math/exp2.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_exp2.h>
 #include <clc/opencl/math/exp2.h>
 
-#define FUNCTION exp2
+#define __CLC_FUNCTION exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/expm1.cl b/libclc/opencl/lib/generic/math/expm1.cl
index 7507786c2ad09..69c852fc7e79c 100644
--- a/libclc/opencl/lib/generic/math/expm1.cl
+++ b/libclc/opencl/lib/generic/math/expm1.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_expm1.h>
 #include <clc/opencl/math/expm1.h>
 
-#define FUNCTION expm1
+#define __CLC_FUNCTION expm1
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/fabs.cl b/libclc/opencl/lib/generic/math/fabs.cl
index 0d3128f3b2a1f..d88c6c84be6d6 100644
--- a/libclc/opencl/lib/generic/math/fabs.cl
+++ b/libclc/opencl/lib/generic/math/fabs.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_fabs.h>
 #include <clc/opencl/math/fabs.h>
 
-#define FUNCTION fabs
+#define __CLC_FUNCTION fabs
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/fdim.cl b/libclc/opencl/lib/generic/math/fdim.cl
index ecdcd3aef3a7d..edaa8a3daef45 100644
--- a/libclc/opencl/lib/generic/math/fdim.cl
+++ b/libclc/opencl/lib/generic/math/fdim.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_fdim.h>
 #include <clc/opencl/math/fdim.h>
 
-#define FUNCTION fdim
+#define __CLC_FUNCTION fdim
 #define __CLC_BODY <clc/shared/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/floor.cl b/libclc/opencl/lib/generic/math/floor.cl
index cda8363f91360..2edc1df21bc59 100644
--- a/libclc/opencl/lib/generic/math/floor.cl
+++ b/libclc/opencl/lib/generic/math/floor.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_floor.h>
 #include <clc/opencl/math/floor.h>
 
-#define FUNCTION floor
+#define __CLC_FUNCTION floor
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/fma.cl b/libclc/opencl/lib/generic/math/fma.cl
index 867b975069013..199d2683a8988 100644
--- a/libclc/opencl/lib/generic/math/fma.cl
+++ b/libclc/opencl/lib/generic/math/fma.cl
@@ -10,7 +10,7 @@
 #include <clc/math/math.h>
 #include <clc/opencl/math/fma.h>
 
-#define FUNCTION fma
+#define __CLC_FUNCTION fma
 #define __CLC_BODY <clc/shared/ternary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/fmax.cl b/libclc/opencl/lib/generic/math/fmax.cl
index 7dbd6fe0bd7f4..ae7a70a69fe38 100644
--- a/libclc/opencl/lib/generic/math/fmax.cl
+++ b/libclc/opencl/lib/generic/math/fmax.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_fmax.h>
 #include <clc/opencl/math/fmax.h>
 
-#define FUNCTION fmax
+#define __CLC_FUNCTION fmax
 #define __CLC_BODY <clc/shared/binary_def_with_scalar_second_arg.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/fmin.cl b/libclc/opencl/lib/generic/math/fmin.cl
index ee0dca480073e..4b55a93bb70fa 100644
--- a/libclc/opencl/lib/generic/math/fmin.cl
+++ b/libclc/opencl/lib/generic/math/fmin.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_fmin.h>
 #include <clc/opencl/math/fmin.h>
 
-#define FUNCTION fmin
+#define __CLC_FUNCTION fmin
 #define __CLC_BODY <clc/shared/binary_def_with_scalar_second_arg.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/fmod.cl b/libclc/opencl/lib/generic/math/fmod.cl
index ec1904d213d87..338b803347217 100644
--- a/libclc/opencl/lib/generic/math/fmod.cl
+++ b/libclc/opencl/lib/generic/math/fmod.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_fmod.h>
 #include <clc/opencl/math/fmod.h>
 
-#define FUNCTION fmod
+#define __CLC_FUNCTION fmod
 #define __CLC_BODY <clc/shared/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/fract.cl b/libclc/opencl/lib/generic/math/fract.cl
index 9567072bf4c39..00e8eb456cf64 100644
--- a/libclc/opencl/lib/generic/math/fract.cl
+++ b/libclc/opencl/lib/generic/math/fract.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_fract.h>
 #include <clc/opencl/math/fract.h>
 
-#define FUNCTION fract
+#define __CLC_FUNCTION fract
 #define __CLC_BODY <clc/math/unary_def_with_ptr.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/frexp.cl b/libclc/opencl/lib/generic/math/frexp.cl
index 3ac9be792ae65..207f9de1c99ec 100644
--- a/libclc/opencl/lib/generic/math/frexp.cl
+++ b/libclc/opencl/lib/generic/math/frexp.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_frexp.h>
 #include <clc/opencl/math/frexp.h>
 
-#define FUNCTION frexp
+#define __CLC_FUNCTION frexp
 #define __CLC_BODY <clc/math/unary_def_with_int_ptr.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_cos.cl b/libclc/opencl/lib/generic/math/half_cos.cl
index 377c4d1642599..d1a5352b52670 100644
--- a/libclc/opencl/lib/generic/math/half_cos.cl
+++ b/libclc/opencl/lib/generic/math/half_cos.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_cos.h>
 #include <clc/opencl/math/half_cos.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_cos
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_divide.cl b/libclc/opencl/lib/generic/math/half_divide.cl
index 730617ba67fb3..10c418b9312db 100644
--- a/libclc/opencl/lib/generic/math/half_divide.cl
+++ b/libclc/opencl/lib/generic/math/half_divide.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_divide.h>
 #include <clc/opencl/math/half_divide.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_divide
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_divide
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_exp.cl b/libclc/opencl/lib/generic/math/half_exp.cl
index 05b8753b2faf4..fbbf564cb697a 100644
--- a/libclc/opencl/lib/generic/math/half_exp.cl
+++ b/libclc/opencl/lib/generic/math/half_exp.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_exp.h>
 #include <clc/opencl/math/half_exp.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_exp
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_exp10.cl b/libclc/opencl/lib/generic/math/half_exp10.cl
index b4e46aaa771cd..e655f5f89ff69 100644
--- a/libclc/opencl/lib/generic/math/half_exp10.cl
+++ b/libclc/opencl/lib/generic/math/half_exp10.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_exp10.h>
 #include <clc/opencl/math/half_exp10.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_exp10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_exp10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_exp2.cl b/libclc/opencl/lib/generic/math/half_exp2.cl
index 51fe9f72477a9..417d8b978e7de 100644
--- a/libclc/opencl/lib/generic/math/half_exp2.cl
+++ b/libclc/opencl/lib/generic/math/half_exp2.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_exp2.h>
 #include <clc/opencl/math/half_exp2.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_exp2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_log.cl b/libclc/opencl/lib/generic/math/half_log.cl
index b21835204a173..89482c263f83f 100644
--- a/libclc/opencl/lib/generic/math/half_log.cl
+++ b/libclc/opencl/lib/generic/math/half_log.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_log.h>
 #include <clc/opencl/math/half_log.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_log
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_log
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_log10.cl b/libclc/opencl/lib/generic/math/half_log10.cl
index acb50f99af192..b5ffe7edd363a 100644
--- a/libclc/opencl/lib/generic/math/half_log10.cl
+++ b/libclc/opencl/lib/generic/math/half_log10.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_log10.h>
 #include <clc/opencl/math/half_log10.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_log10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_log2.cl b/libclc/opencl/lib/generic/math/half_log2.cl
index c97e9e093878d..cf13cf927c9c9 100644
--- a/libclc/opencl/lib/generic/math/half_log2.cl
+++ b/libclc/opencl/lib/generic/math/half_log2.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_log2.h>
 #include <clc/opencl/math/half_log2.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_log2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_powr.cl b/libclc/opencl/lib/generic/math/half_powr.cl
index d61a18ab91321..d676f353f8647 100644
--- a/libclc/opencl/lib/generic/math/half_powr.cl
+++ b/libclc/opencl/lib/generic/math/half_powr.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_powr.h>
 #include <clc/opencl/math/half_powr.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_powr
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_powr
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_recip.cl b/libclc/opencl/lib/generic/math/half_recip.cl
index 1917448c4e695..40a6c587dbaf9 100644
--- a/libclc/opencl/lib/generic/math/half_recip.cl
+++ b/libclc/opencl/lib/generic/math/half_recip.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_recip.h>
 #include <clc/opencl/math/half_recip.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_recip
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_recip
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_rsqrt.cl b/libclc/opencl/lib/generic/math/half_rsqrt.cl
index d6ffa651fbe6a..944976855dacf 100644
--- a/libclc/opencl/lib/generic/math/half_rsqrt.cl
+++ b/libclc/opencl/lib/generic/math/half_rsqrt.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_rsqrt.h>
 #include <clc/opencl/math/half_rsqrt.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_rsqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_rsqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_sin.cl b/libclc/opencl/lib/generic/math/half_sin.cl
index baba0cca49849..3376f8344f394 100644
--- a/libclc/opencl/lib/generic/math/half_sin.cl
+++ b/libclc/opencl/lib/generic/math/half_sin.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_sin.h>
 #include <clc/opencl/math/half_sin.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_sin
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_sqrt.cl b/libclc/opencl/lib/generic/math/half_sqrt.cl
index 7a59744ab11d0..4939d6bd199dc 100644
--- a/libclc/opencl/lib/generic/math/half_sqrt.cl
+++ b/libclc/opencl/lib/generic/math/half_sqrt.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_sqrt.h>
 #include <clc/opencl/math/half_sqrt.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_sqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/half_tan.cl b/libclc/opencl/lib/generic/math/half_tan.cl
index acd4b011173db..2b77b3ae0c623 100644
--- a/libclc/opencl/lib/generic/math/half_tan.cl
+++ b/libclc/opencl/lib/generic/math/half_tan.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_half_tan.h>
 #include <clc/opencl/math/half_tan.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION half_tan
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION half_tan
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/hypot.cl b/libclc/opencl/lib/generic/math/hypot.cl
index 4c59e4696e5ae..41aff69069879 100644
--- a/libclc/opencl/lib/generic/math/hypot.cl
+++ b/libclc/opencl/lib/generic/math/hypot.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_hypot.h>
 #include <clc/opencl/math/hypot.h>
 
-#define FUNCTION hypot
+#define __CLC_FUNCTION hypot
 #define __CLC_BODY <clc/shared/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/ilogb.cl b/libclc/opencl/lib/generic/math/ilogb.cl
index 3829ce68fb577..027d587a4d46d 100644
--- a/libclc/opencl/lib/generic/math/ilogb.cl
+++ b/libclc/opencl/lib/generic/math/ilogb.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_ilogb.h>
 #include <clc/opencl/math/ilogb.h>
 
-#define FUNCTION ilogb
+#define __CLC_FUNCTION ilogb
 #define __CLC_BODY <clc/math/unary_def_with_int_return.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/ldexp.cl b/libclc/opencl/lib/generic/math/ldexp.cl
index 88c60716d7829..3cdb9e1a760a1 100644
--- a/libclc/opencl/lib/generic/math/ldexp.cl
+++ b/libclc/opencl/lib/generic/math/ldexp.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_ldexp.h>
 #include <clc/opencl/math/ldexp.h>
 
-#define FUNCTION ldexp
-#define __IMPL_FUNCTION(x) __clc_ldexp
+#define __CLC_FUNCTION ldexp
+#define __CLC_IMPL_FUNCTION(x) __clc_ldexp
 #define __CLC_BODY <clc/shared/binary_def_with_int_second_arg.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/lgamma.cl b/libclc/opencl/lib/generic/math/lgamma.cl
index 253ce31c15771..da8ceac986e43 100644
--- a/libclc/opencl/lib/generic/math/lgamma.cl
+++ b/libclc/opencl/lib/generic/math/lgamma.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_lgamma.h>
 #include <clc/opencl/math/lgamma.h>
 
-#define FUNCTION lgamma
+#define __CLC_FUNCTION lgamma
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/lgamma_r.cl b/libclc/opencl/lib/generic/math/lgamma_r.cl
index 8753f6c75d8b1..bb2d9bd283df6 100644
--- a/libclc/opencl/lib/generic/math/lgamma_r.cl
+++ b/libclc/opencl/lib/generic/math/lgamma_r.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_lgamma_r.h>
 #include <clc/opencl/math/lgamma_r.h>
 
-#define FUNCTION lgamma_r
+#define __CLC_FUNCTION lgamma_r
 #define __CLC_BODY <clc/math/unary_def_with_int_ptr.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/log.cl b/libclc/opencl/lib/generic/math/log.cl
index dbe3e35727033..06209999bf2f1 100644
--- a/libclc/opencl/lib/generic/math/log.cl
+++ b/libclc/opencl/lib/generic/math/log.cl
@@ -10,7 +10,7 @@
 #include <clc/math/clc_log.h>
 #include <clc/opencl/math/log.h>
 
-#define FUNCTION log
+#define __CLC_FUNCTION log
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/log10.cl b/libclc/opencl/lib/generic/math/log10.cl
index d5137a7cab7ab..466b602e18963 100644
--- a/libclc/opencl/lib/generic/math/log10.cl
+++ b/libclc/opencl/lib/generic/math/log10.cl
@@ -10,7 +10,7 @@
 #include <clc/math/clc_log10.h>
 #include <clc/opencl/math/log10.h>
 
-#define FUNCTION log10
+#define __CLC_FUNCTION log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/log1p.cl b/libclc/opencl/lib/generic/math/log1p.cl
index d389ab72908c8..c1a8711e11858 100644
--- a/libclc/opencl/lib/generic/math/log1p.cl
+++ b/libclc/opencl/lib/generic/math/log1p.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_log1p.h>
 #include <clc/opencl/math/log1p.h>
 
-#define FUNCTION log1p
+#define __CLC_FUNCTION log1p
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/log2.cl b/libclc/opencl/lib/generic/math/log2.cl
index d1433bc825813..a52a52e28f63f 100644
--- a/libclc/opencl/lib/generic/math/log2.cl
+++ b/libclc/opencl/lib/generic/math/log2.cl
@@ -10,7 +10,7 @@
 #include <clc/math/clc_log2.h>
 #include <clc/opencl/math/log2.h>
 
-#define FUNCTION log2
+#define __CLC_FUNCTION log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/logb.cl b/libclc/opencl/lib/generic/math/logb.cl
index 037380a26a727..09d43fac3ad2a 100644
--- a/libclc/opencl/lib/generic/math/logb.cl
+++ b/libclc/opencl/lib/generic/math/logb.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_logb.h>
 #include <clc/opencl/math/logb.h>
 
-#define FUNCTION logb
+#define __CLC_FUNCTION logb
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/mad.cl b/libclc/opencl/lib/generic/math/mad.cl
index 3fe2dd9bed5fb..05bd228251c18 100644
--- a/libclc/opencl/lib/generic/math/mad.cl
+++ b/libclc/opencl/lib/generic/math/mad.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_mad.h>
 #include <clc/opencl/math/mad.h>
 
-#define FUNCTION mad
+#define __CLC_FUNCTION mad
 #define __CLC_BODY <clc/shared/ternary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/maxmag.cl b/libclc/opencl/lib/generic/math/maxmag.cl
index 4f9dbeb8d27e7..30e7bb21302d7 100644
--- a/libclc/opencl/lib/generic/math/maxmag.cl
+++ b/libclc/opencl/lib/generic/math/maxmag.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_maxmag.h>
 #include <clc/opencl/math/maxmag.h>
 
-#define FUNCTION maxmag
+#define __CLC_FUNCTION maxmag
 #define __CLC_BODY <clc/shared/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/minmag.cl b/libclc/opencl/lib/generic/math/minmag.cl
index efe72b6f545c2..3732dbfe7af45 100644
--- a/libclc/opencl/lib/generic/math/minmag.cl
+++ b/libclc/opencl/lib/generic/math/minmag.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_minmag.h>
 #include <clc/opencl/math/minmag.h>
 
-#define FUNCTION minmag
+#define __CLC_FUNCTION minmag
 #define __CLC_BODY <clc/shared/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/modf.cl b/libclc/opencl/lib/generic/math/modf.cl
index de6524e02f73f..306070bf89238 100644
--- a/libclc/opencl/lib/generic/math/modf.cl
+++ b/libclc/opencl/lib/generic/math/modf.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_modf.h>
 #include <clc/opencl/math/modf.h>
 
-#define FUNCTION modf
+#define __CLC_FUNCTION modf
 #define __CLC_BODY <clc/math/unary_def_with_ptr.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_cos.cl b/libclc/opencl/lib/generic/math/native_cos.cl
index 85944a03c5468..8508df37a9b62 100644
--- a/libclc/opencl/lib/generic/math/native_cos.cl
+++ b/libclc/opencl/lib/generic/math/native_cos.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_cos.h>
 #include <clc/opencl/math/native_cos.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_cos
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_cos
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_divide.cl b/libclc/opencl/lib/generic/math/native_divide.cl
index 8efd8cc21b5d8..ea28bf6e195cc 100644
--- a/libclc/opencl/lib/generic/math/native_divide.cl
+++ b/libclc/opencl/lib/generic/math/native_divide.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_divide.h>
 #include <clc/opencl/math/native_divide.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_divide
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_divide
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_exp.cl b/libclc/opencl/lib/generic/math/native_exp.cl
index 42208371e195c..aedb0953c5349 100644
--- a/libclc/opencl/lib/generic/math/native_exp.cl
+++ b/libclc/opencl/lib/generic/math/native_exp.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_exp.h>
 #include <clc/opencl/math/native_exp.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_exp
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_exp
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_exp10.cl b/libclc/opencl/lib/generic/math/native_exp10.cl
index aa2585ff4311d..eb184647639f9 100644
--- a/libclc/opencl/lib/generic/math/native_exp10.cl
+++ b/libclc/opencl/lib/generic/math/native_exp10.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_exp10.h>
 #include <clc/opencl/math/native_exp10.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_exp10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_exp10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_exp2.cl b/libclc/opencl/lib/generic/math/native_exp2.cl
index 8955b28b367cb..52e814f5ee53c 100644
--- a/libclc/opencl/lib/generic/math/native_exp2.cl
+++ b/libclc/opencl/lib/generic/math/native_exp2.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_exp2.h>
 #include <clc/opencl/math/native_exp2.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_exp2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_exp2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_log.cl b/libclc/opencl/lib/generic/math/native_log.cl
index 334f7c04e3899..d1d4ae2f15f18 100644
--- a/libclc/opencl/lib/generic/math/native_log.cl
+++ b/libclc/opencl/lib/generic/math/native_log.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_log.h>
 #include <clc/opencl/math/native_log.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_log
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_log
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_log10.cl b/libclc/opencl/lib/generic/math/native_log10.cl
index a65938ee4c6cb..680b9cb341bf2 100644
--- a/libclc/opencl/lib/generic/math/native_log10.cl
+++ b/libclc/opencl/lib/generic/math/native_log10.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_log10.h>
 #include <clc/opencl/math/native_log10.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_log10
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_log10
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_log2.cl b/libclc/opencl/lib/generic/math/native_log2.cl
index f10b533f91f40..3a104357afa8c 100644
--- a/libclc/opencl/lib/generic/math/native_log2.cl
+++ b/libclc/opencl/lib/generic/math/native_log2.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_log2.h>
 #include <clc/opencl/math/native_log2.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_log2
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_log2
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_powr.cl b/libclc/opencl/lib/generic/math/native_powr.cl
index 8301443cfbcfd..f325d53321b1d 100644
--- a/libclc/opencl/lib/generic/math/native_powr.cl
+++ b/libclc/opencl/lib/generic/math/native_powr.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_powr.h>
 #include <clc/opencl/math/native_powr.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_powr
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_powr
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_recip.cl b/libclc/opencl/lib/generic/math/native_recip.cl
index 145845b455147..20d81ed9f1b79 100644
--- a/libclc/opencl/lib/generic/math/native_recip.cl
+++ b/libclc/opencl/lib/generic/math/native_recip.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_recip.h>
 #include <clc/opencl/math/native_recip.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_recip
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_recip
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_rsqrt.cl b/libclc/opencl/lib/generic/math/native_rsqrt.cl
index cbb5a754fc967..8f308e74f334c 100644
--- a/libclc/opencl/lib/generic/math/native_rsqrt.cl
+++ b/libclc/opencl/lib/generic/math/native_rsqrt.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_rsqrt.h>
 #include <clc/opencl/math/native_rsqrt.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_rsqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_rsqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_sin.cl b/libclc/opencl/lib/generic/math/native_sin.cl
index dfcc882614a63..ea7167822a818 100644
--- a/libclc/opencl/lib/generic/math/native_sin.cl
+++ b/libclc/opencl/lib/generic/math/native_sin.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_sin.h>
 #include <clc/opencl/math/native_sin.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_sin
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_sqrt.cl b/libclc/opencl/lib/generic/math/native_sqrt.cl
index a0be41d33fa43..fa38bdb3301d4 100644
--- a/libclc/opencl/lib/generic/math/native_sqrt.cl
+++ b/libclc/opencl/lib/generic/math/native_sqrt.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_sqrt.h>
 #include <clc/opencl/math/native_sqrt.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_sqrt
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/native_tan.cl b/libclc/opencl/lib/generic/math/native_tan.cl
index 86eee4f5913f0..8cebf93bae91c 100644
--- a/libclc/opencl/lib/generic/math/native_tan.cl
+++ b/libclc/opencl/lib/generic/math/native_tan.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_native_tan.h>
 #include <clc/opencl/math/native_tan.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION native_tan
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION native_tan
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/nextafter.cl b/libclc/opencl/lib/generic/math/nextafter.cl
index 9c30e5668a56b..256f06bf58550 100644
--- a/libclc/opencl/lib/generic/math/nextafter.cl
+++ b/libclc/opencl/lib/generic/math/nextafter.cl
@@ -9,8 +9,8 @@
 #include <clc/math/clc_nextafter.h>
 #include <clc/opencl/math/nextafter.h>
 
-#define FUNCTION nextafter
-#define __IMPL_FUNCTION(x) __clc_nextafter
+#define __CLC_FUNCTION nextafter
+#define __CLC_IMPL_FUNCTION(x) __clc_nextafter
 #define __CLC_BODY <clc/shared/binary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/pow.cl b/libclc/opencl/lib/generic/math/pow.cl
index 2fddb7e889649..bf43519265731 100644
--- a/libclc/opencl/lib/generic/math/pow.cl
+++ b/libclc/opencl/lib/generic/math/pow.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_pow.h>
 #include <clc/opencl/math/pow.h>
 
-#define FUNCTION pow
+#define __CLC_FUNCTION pow
 #define __CLC_BODY <clc/shared/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/pown.cl b/libclc/opencl/lib/generic/math/pown.cl
index 0a5ee893a45a1..8c03be6bee168 100644
--- a/libclc/opencl/lib/generic/math/pown.cl
+++ b/libclc/opencl/lib/generic/math/pown.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_pown.h>
 #include <clc/opencl/math/pown.h>
 
-#define FUNCTION pown
+#define __CLC_FUNCTION pown
 #define __CLC_BODY <clc/shared/binary_def_with_int_second_arg.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/powr.cl b/libclc/opencl/lib/generic/math/powr.cl
index 13af03ff13274..680f9be894abb 100644
--- a/libclc/opencl/lib/generic/math/powr.cl
+++ b/libclc/opencl/lib/generic/math/powr.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_powr.h>
 #include <clc/opencl/math/powr.h>
 
-#define FUNCTION powr
+#define __CLC_FUNCTION powr
 #define __CLC_BODY <clc/shared/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/remainder.cl b/libclc/opencl/lib/generic/math/remainder.cl
index 224a66bc9b279..42b37f847f6d1 100644
--- a/libclc/opencl/lib/generic/math/remainder.cl
+++ b/libclc/opencl/lib/generic/math/remainder.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_remainder.h>
 #include <clc/opencl/math/remainder.h>
 
-#define FUNCTION remainder
+#define __CLC_FUNCTION remainder
 #define __CLC_BODY <clc/shared/binary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/rint.cl b/libclc/opencl/lib/generic/math/rint.cl
index 489883c97c141..8beaa7ae064c5 100644
--- a/libclc/opencl/lib/generic/math/rint.cl
+++ b/libclc/opencl/lib/generic/math/rint.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_rint.h>
 #include <clc/opencl/math/rint.h>
 
-#define FUNCTION rint
+#define __CLC_FUNCTION rint
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/rootn.cl b/libclc/opencl/lib/generic/math/rootn.cl
index e8dba17b9395d..8f25ee1d31b47 100644
--- a/libclc/opencl/lib/generic/math/rootn.cl
+++ b/libclc/opencl/lib/generic/math/rootn.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_rootn.h>
 #include <clc/opencl/math/rootn.h>
 
-#define FUNCTION rootn
+#define __CLC_FUNCTION rootn
 #define __CLC_BODY <clc/shared/binary_def_with_int_second_arg.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/round.cl b/libclc/opencl/lib/generic/math/round.cl
index c45e681b1c222..79b04752e9dec 100644
--- a/libclc/opencl/lib/generic/math/round.cl
+++ b/libclc/opencl/lib/generic/math/round.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_round.h>
 #include <clc/opencl/math/round.h>
 
-#define FUNCTION round
+#define __CLC_FUNCTION round
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/rsqrt.cl b/libclc/opencl/lib/generic/math/rsqrt.cl
index 47e4f450ed1a6..f7137f5a007d3 100644
--- a/libclc/opencl/lib/generic/math/rsqrt.cl
+++ b/libclc/opencl/lib/generic/math/rsqrt.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_rsqrt.h>
 #include <clc/opencl/math/rsqrt.h>
 
-#define FUNCTION rsqrt
+#define __CLC_FUNCTION rsqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/sin.cl b/libclc/opencl/lib/generic/math/sin.cl
index ff7db2a5b3cb8..e198198bcbe86 100644
--- a/libclc/opencl/lib/generic/math/sin.cl
+++ b/libclc/opencl/lib/generic/math/sin.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_sin.h>
 #include <clc/opencl/math/sin.h>
 
-#define FUNCTION sin
+#define __CLC_FUNCTION sin
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/sincos.cl b/libclc/opencl/lib/generic/math/sincos.cl
index c8871fd2b0020..845e966966c51 100644
--- a/libclc/opencl/lib/generic/math/sincos.cl
+++ b/libclc/opencl/lib/generic/math/sincos.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_sincos.h>
 #include <clc/opencl/math/sincos.h>
 
-#define FUNCTION sincos
+#define __CLC_FUNCTION sincos
 #define __CLC_BODY <clc/math/unary_def_with_ptr.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/sinh.cl b/libclc/opencl/lib/generic/math/sinh.cl
index bd6ea20e7e898..6d5f1b1c269c1 100644
--- a/libclc/opencl/lib/generic/math/sinh.cl
+++ b/libclc/opencl/lib/generic/math/sinh.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_sinh.h>
 #include <clc/opencl/math/sinh.h>
 
-#define FUNCTION sinh
+#define __CLC_FUNCTION sinh
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/sinpi.cl b/libclc/opencl/lib/generic/math/sinpi.cl
index badecd0909856..33b04532e7763 100644
--- a/libclc/opencl/lib/generic/math/sinpi.cl
+++ b/libclc/opencl/lib/generic/math/sinpi.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_sinpi.h>
 #include <clc/opencl/math/sinpi.h>
 
-#define FUNCTION sinpi
+#define __CLC_FUNCTION sinpi
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/sqrt.cl b/libclc/opencl/lib/generic/math/sqrt.cl
index 3b05b25864485..9984ddb9d5a41 100644
--- a/libclc/opencl/lib/generic/math/sqrt.cl
+++ b/libclc/opencl/lib/generic/math/sqrt.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_sqrt.h>
 #include <clc/opencl/math/sqrt.h>
 
-#define FUNCTION sqrt
+#define __CLC_FUNCTION sqrt
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/tan.cl b/libclc/opencl/lib/generic/math/tan.cl
index 95409fc5b092b..89a5251111675 100644
--- a/libclc/opencl/lib/generic/math/tan.cl
+++ b/libclc/opencl/lib/generic/math/tan.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_tan.h>
 #include <clc/opencl/math/tan.h>
 
-#define FUNCTION tan
+#define __CLC_FUNCTION tan
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/tanh.cl b/libclc/opencl/lib/generic/math/tanh.cl
index a77878bb79667..92307eafdd05f 100644
--- a/libclc/opencl/lib/generic/math/tanh.cl
+++ b/libclc/opencl/lib/generic/math/tanh.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_tanh.h>
 #include <clc/opencl/math/tanh.h>
 
-#define FUNCTION tanh
+#define __CLC_FUNCTION tanh
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/tanpi.cl b/libclc/opencl/lib/generic/math/tanpi.cl
index 6b5805ec3cd04..47e943ba77416 100644
--- a/libclc/opencl/lib/generic/math/tanpi.cl
+++ b/libclc/opencl/lib/generic/math/tanpi.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_tanpi.h>
 #include <clc/opencl/math/tanpi.h>
 
-#define FUNCTION tanpi
+#define __CLC_FUNCTION tanpi
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/tgamma.cl b/libclc/opencl/lib/generic/math/tgamma.cl
index 8f35d22426a70..2749cf3dfc0f4 100644
--- a/libclc/opencl/lib/generic/math/tgamma.cl
+++ b/libclc/opencl/lib/generic/math/tgamma.cl
@@ -9,6 +9,6 @@
 #include <clc/math/clc_tgamma.h>
 #include <clc/opencl/math/tgamma.h>
 
-#define FUNCTION tgamma
+#define __CLC_FUNCTION tgamma
 #define __CLC_BODY <clc/shared/unary_def.inc>
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/math/trunc.cl b/libclc/opencl/lib/generic/math/trunc.cl
index ea56329bc8dea..b276996995344 100644
--- a/libclc/opencl/lib/generic/math/trunc.cl
+++ b/libclc/opencl/lib/generic/math/trunc.cl
@@ -9,7 +9,7 @@
 #include <clc/math/clc_trunc.h>
 #include <clc/opencl/math/trunc.h>
 
-#define FUNCTION trunc
+#define __CLC_FUNCTION trunc
 #define __CLC_BODY <clc/shared/unary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/misc/shuffle.cl b/libclc/opencl/lib/generic/misc/shuffle.cl
index 15295bf7d907a..f9187be453e65 100644
--- a/libclc/opencl/lib/generic/misc/shuffle.cl
+++ b/libclc/opencl/lib/generic/misc/shuffle.cl
@@ -9,7 +9,7 @@
 #include <clc/misc/clc_shuffle.h>
 #include <clc/opencl/misc/shuffle.h>
 
-#define FUNCTION shuffle
+#define __CLC_FUNCTION shuffle
 
 #define __CLC_BODY <clc/misc/shuffle_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/misc/shuffle2.cl b/libclc/opencl/lib/generic/misc/shuffle2.cl
index 7e4c5f0922682..0008340b622df 100644
--- a/libclc/opencl/lib/generic/misc/shuffle2.cl
+++ b/libclc/opencl/lib/generic/misc/shuffle2.cl
@@ -9,7 +9,7 @@
 #include <clc/misc/clc_shuffle2.h>
 #include <clc/opencl/misc/shuffle2.h>
 
-#define FUNCTION shuffle2
+#define __CLC_FUNCTION shuffle2
 
 #define __CLC_BODY <clc/misc/shuffle2_def.inc>
 #include <clc/integer/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/all.cl b/libclc/opencl/lib/generic/relational/all.cl
index 5ac4a465365a4..f72da2cb622c9 100644
--- a/libclc/opencl/lib/generic/relational/all.cl
+++ b/libclc/opencl/lib/generic/relational/all.cl
@@ -9,17 +9,17 @@
 #include <clc/opencl/relational/all.h>
 #include <clc/relational/clc_all.h>
 
-#define ALL_ID(TYPE) _CLC_OVERLOAD _CLC_DEF int all(TYPE v)
+#define __CLC_ALL_ID(TYPE) _CLC_OVERLOAD _CLC_DEF int all(TYPE v)
 
-#define ALL_VECTORIZE(TYPE)                                                    \
-  ALL_ID(TYPE) { return __clc_all(v); }                                        \
-  ALL_ID(TYPE##2) { return __clc_all(v); }                                     \
-  ALL_ID(TYPE##3) { return __clc_all(v); }                                     \
-  ALL_ID(TYPE##4) { return __clc_all(v); }                                     \
-  ALL_ID(TYPE##8) { return __clc_all(v); }                                     \
-  ALL_ID(TYPE##16) { return __clc_all(v); }
+#define __CLC_ALL_VECTORIZE(TYPE)                                              \
+  __CLC_ALL_ID(TYPE) { return __clc_all(v); }                                  \
+  __CLC_ALL_ID(TYPE##2) { return __clc_all(v); }                               \
+  __CLC_ALL_ID(TYPE##3) { return __clc_all(v); }                               \
+  __CLC_ALL_ID(TYPE##4) { return __clc_all(v); }                               \
+  __CLC_ALL_ID(TYPE##8) { return __clc_all(v); }                               \
+  __CLC_ALL_ID(TYPE##16) { return __clc_all(v); }
 
-ALL_VECTORIZE(char)
-ALL_VECTORIZE(short)
-ALL_VECTORIZE(int)
-ALL_VECTORIZE(long)
+__CLC_ALL_VECTORIZE(char)
+__CLC_ALL_VECTORIZE(short)
+__CLC_ALL_VECTORIZE(int)
+__CLC_ALL_VECTORIZE(long)
diff --git a/libclc/opencl/lib/generic/relational/any.cl b/libclc/opencl/lib/generic/relational/any.cl
index 507f2b170693e..b2b48dbf1ab04 100644
--- a/libclc/opencl/lib/generic/relational/any.cl
+++ b/libclc/opencl/lib/generic/relational/any.cl
@@ -9,17 +9,17 @@
 #include <clc/opencl/relational/any.h>
 #include <clc/relational/clc_any.h>
 
-#define ANY_ID(TYPE) _CLC_OVERLOAD _CLC_DEF int any(TYPE v)
+#define __CLC_ANY_ID(TYPE) _CLC_OVERLOAD _CLC_DEF int any(TYPE v)
 
-#define ANY_VECTORIZE(TYPE)                                                    \
-  ANY_ID(TYPE) { return __clc_any(v); }                                        \
-  ANY_ID(TYPE##2) { return __clc_any(v); }                                     \
-  ANY_ID(TYPE##3) { return __clc_any(v); }                                     \
-  ANY_ID(TYPE##4) { return __clc_any(v); }                                     \
-  ANY_ID(TYPE##8) { return __clc_any(v); }                                     \
-  ANY_ID(TYPE##16) { return __clc_any(v); }
+#define __CLC_ANY_VECTORIZE(TYPE)                                              \
+  __CLC_ANY_ID(TYPE) { return __clc_any(v); }                                  \
+  __CLC_ANY_ID(TYPE##2) { return __clc_any(v); }                               \
+  __CLC_ANY_ID(TYPE##3) { return __clc_any(v); }                               \
+  __CLC_ANY_ID(TYPE##4) { return __clc_any(v); }                               \
+  __CLC_ANY_ID(TYPE##8) { return __clc_any(v); }                               \
+  __CLC_ANY_ID(TYPE##16) { return __clc_any(v); }
 
-ANY_VECTORIZE(char)
-ANY_VECTORIZE(short)
-ANY_VECTORIZE(int)
-ANY_VECTORIZE(long)
+__CLC_ANY_VECTORIZE(char)
+__CLC_ANY_VECTORIZE(short)
+__CLC_ANY_VECTORIZE(int)
+__CLC_ANY_VECTORIZE(long)
diff --git a/libclc/opencl/lib/generic/relational/binary_def.inc b/libclc/opencl/lib/generic/relational/binary_def.inc
index 8416da0475a2c..b60d9bbe45e11 100644
--- a/libclc/opencl/lib/generic/relational/binary_def.inc
+++ b/libclc/opencl/lib/generic/relational/binary_def.inc
@@ -8,16 +8,17 @@
 
 #include <clc/utils.h>
 
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 
 #if __CLC_VECSIZE_OR_1 == 1
-#define __RETTYPE __CLC_INTN
+#define __CLC_RETTYPE __CLC_INTN
 #else
-#define __RETTYPE __CLC_BIT_INTN
+#define __CLC_RETTYPE __CLC_BIT_INTN
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __RETTYPE FUNCTION(__CLC_GENTYPE a, __CLC_GENTYPE b) {
-  return __IMPL_FUNCTION(FUNCTION)(a, b);
+_CLC_OVERLOAD _CLC_DEF __CLC_RETTYPE __CLC_FUNCTION(__CLC_GENTYPE a,
+                                                    __CLC_GENTYPE b) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a, b);
 }
 
-#undef __RETTYPE
+#undef __CLC_RETTYPE
diff --git a/libclc/opencl/lib/generic/relational/isequal.cl b/libclc/opencl/lib/generic/relational/isequal.cl
index 83002c28ceab3..40718dbca9d05 100644
--- a/libclc/opencl/lib/generic/relational/isequal.cl
+++ b/libclc/opencl/lib/generic/relational/isequal.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isequal.h>
 #include <clc/relational/clc_isequal.h>
 
-#define FUNCTION isequal
+#define __CLC_FUNCTION isequal
 #define __CLC_BODY "binary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isfinite.cl b/libclc/opencl/lib/generic/relational/isfinite.cl
index a2017133cead8..f055e295416c9 100644
--- a/libclc/opencl/lib/generic/relational/isfinite.cl
+++ b/libclc/opencl/lib/generic/relational/isfinite.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isfinite.h>
 #include <clc/relational/clc_isfinite.h>
 
-#define FUNCTION isfinite
+#define __CLC_FUNCTION isfinite
 #define __CLC_BODY "unary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isgreater.cl b/libclc/opencl/lib/generic/relational/isgreater.cl
index 6eeb2b21c0493..ae73769267dc8 100644
--- a/libclc/opencl/lib/generic/relational/isgreater.cl
+++ b/libclc/opencl/lib/generic/relational/isgreater.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isgreater.h>
 #include <clc/relational/clc_isgreater.h>
 
-#define FUNCTION isgreater
+#define __CLC_FUNCTION isgreater
 #define __CLC_BODY "binary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isgreaterequal.cl b/libclc/opencl/lib/generic/relational/isgreaterequal.cl
index e4e4535fd30d3..725c3289bb05f 100644
--- a/libclc/opencl/lib/generic/relational/isgreaterequal.cl
+++ b/libclc/opencl/lib/generic/relational/isgreaterequal.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isgreaterequal.h>
 #include <clc/relational/clc_isgreaterequal.h>
 
-#define FUNCTION isgreaterequal
+#define __CLC_FUNCTION isgreaterequal
 #define __CLC_BODY "binary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isinf.cl b/libclc/opencl/lib/generic/relational/isinf.cl
index 2ab8c182e02a6..9558b0f9eb5ee 100644
--- a/libclc/opencl/lib/generic/relational/isinf.cl
+++ b/libclc/opencl/lib/generic/relational/isinf.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isinf.h>
 #include <clc/relational/clc_isinf.h>
 
-#define FUNCTION isinf
+#define __CLC_FUNCTION isinf
 #define __CLC_BODY "unary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isless.cl b/libclc/opencl/lib/generic/relational/isless.cl
index 4212970e7671a..6ed0857d354a3 100644
--- a/libclc/opencl/lib/generic/relational/isless.cl
+++ b/libclc/opencl/lib/generic/relational/isless.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isless.h>
 #include <clc/relational/clc_isless.h>
 
-#define FUNCTION isless
+#define __CLC_FUNCTION isless
 #define __CLC_BODY "binary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/islessequal.cl b/libclc/opencl/lib/generic/relational/islessequal.cl
index e7aec262fc762..c4697ca56f982 100644
--- a/libclc/opencl/lib/generic/relational/islessequal.cl
+++ b/libclc/opencl/lib/generic/relational/islessequal.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/islessequal.h>
 #include <clc/relational/clc_islessequal.h>
 
-#define FUNCTION islessequal
+#define __CLC_FUNCTION islessequal
 #define __CLC_BODY "binary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/islessgreater.cl b/libclc/opencl/lib/generic/relational/islessgreater.cl
index b775d2484550c..eb4cdc24c0679 100644
--- a/libclc/opencl/lib/generic/relational/islessgreater.cl
+++ b/libclc/opencl/lib/generic/relational/islessgreater.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/islessgreater.h>
 #include <clc/relational/clc_islessgreater.h>
 
-#define FUNCTION islessgreater
+#define __CLC_FUNCTION islessgreater
 #define __CLC_BODY "binary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isnan.cl b/libclc/opencl/lib/generic/relational/isnan.cl
index 4b7eeb5b919b6..4bae2ae077c13 100644
--- a/libclc/opencl/lib/generic/relational/isnan.cl
+++ b/libclc/opencl/lib/generic/relational/isnan.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isnan.h>
 #include <clc/relational/clc_isnan.h>
 
-#define FUNCTION isnan
+#define __CLC_FUNCTION isnan
 #define __CLC_BODY "unary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isnormal.cl b/libclc/opencl/lib/generic/relational/isnormal.cl
index 60ce9dccaeaf3..d4ea20a27cdb6 100644
--- a/libclc/opencl/lib/generic/relational/isnormal.cl
+++ b/libclc/opencl/lib/generic/relational/isnormal.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isnormal.h>
 #include <clc/relational/clc_isnormal.h>
 
-#define FUNCTION isnormal
+#define __CLC_FUNCTION isnormal
 #define __CLC_BODY "unary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isnotequal.cl b/libclc/opencl/lib/generic/relational/isnotequal.cl
index abb4d3a859663..e24d57564cb8c 100644
--- a/libclc/opencl/lib/generic/relational/isnotequal.cl
+++ b/libclc/opencl/lib/generic/relational/isnotequal.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isnotequal.h>
 #include <clc/relational/clc_isnotequal.h>
 
-#define FUNCTION isnotequal
+#define __CLC_FUNCTION isnotequal
 #define __CLC_BODY "binary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isordered.cl b/libclc/opencl/lib/generic/relational/isordered.cl
index 684ee425e1203..773cf1947e68e 100644
--- a/libclc/opencl/lib/generic/relational/isordered.cl
+++ b/libclc/opencl/lib/generic/relational/isordered.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isordered.h>
 #include <clc/relational/clc_isordered.h>
 
-#define FUNCTION isordered
+#define __CLC_FUNCTION isordered
 #define __CLC_BODY "binary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/isunordered.cl b/libclc/opencl/lib/generic/relational/isunordered.cl
index 84aa8cafb111a..44427ea5e6046 100644
--- a/libclc/opencl/lib/generic/relational/isunordered.cl
+++ b/libclc/opencl/lib/generic/relational/isunordered.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/isunordered.h>
 #include <clc/relational/clc_isunordered.h>
 
-#define FUNCTION isunordered
+#define __CLC_FUNCTION isunordered
 #define __CLC_BODY "binary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/signbit.cl b/libclc/opencl/lib/generic/relational/signbit.cl
index d30fea7b9f6f5..b1f3ac89933f7 100644
--- a/libclc/opencl/lib/generic/relational/signbit.cl
+++ b/libclc/opencl/lib/generic/relational/signbit.cl
@@ -9,7 +9,7 @@
 #include <clc/opencl/relational/signbit.h>
 #include <clc/relational/clc_signbit.h>
 
-#define FUNCTION signbit
+#define __CLC_FUNCTION signbit
 #define __CLC_BODY "unary_def.inc"
 
 #include <clc/math/gentype.inc>
diff --git a/libclc/opencl/lib/generic/relational/unary_def.inc b/libclc/opencl/lib/generic/relational/unary_def.inc
index f184e3cf0be56..1655ace114aa6 100644
--- a/libclc/opencl/lib/generic/relational/unary_def.inc
+++ b/libclc/opencl/lib/generic/relational/unary_def.inc
@@ -8,16 +8,16 @@
 
 #include <clc/utils.h>
 
-#define __IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
+#define __CLC_IMPL_FUNCTION(x) __CLC_CONCAT(__clc_, x)
 
 #if __CLC_VECSIZE_OR_1 == 1
-#define __RETTYPE __CLC_INTN
+#define __CLC_RETTYPE __CLC_INTN
 #else
-#define __RETTYPE __CLC_BIT_INTN
+#define __CLC_RETTYPE __CLC_BIT_INTN
 #endif
 
-_CLC_OVERLOAD _CLC_DEF __RETTYPE FUNCTION(__CLC_GENTYPE a) {
-  return __IMPL_FUNCTION(FUNCTION)(a);
+_CLC_OVERLOAD _CLC_DEF __CLC_RETTYPE __CLC_FUNCTION(__CLC_GENTYPE a) {
+  return __CLC_IMPL_FUNCTION(__CLC_FUNCTION)(a);
 }
 
-#undef __RETTYPE
+#undef __CLC_RETTYPE
diff --git a/libclc/opencl/lib/generic/shared/vload.inc b/libclc/opencl/lib/generic/shared/vload.inc
index 62cb040aad180..955b77b48ec35 100644
--- a/libclc/opencl/lib/generic/shared/vload.inc
+++ b/libclc/opencl/lib/generic/shared/vload.inc
@@ -6,33 +6,34 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define CLC_VLOAD_NAME(x) __CLC_XCONCAT(__CLC_XCONCAT(x, vload), __CLC_VECSIZE)
-#define CLC_VLOAD_HALF_NAME(x)                                                 \
+#define __CLC_VLOAD_NAME(x)                                                    \
+  __CLC_XCONCAT(__CLC_XCONCAT(x, vload), __CLC_VECSIZE)
+#define __CLC_VLOAD_HALF_NAME(x)                                               \
   __CLC_XCONCAT(__CLC_XCONCAT(x, vload_half), __CLC_VECSIZE)
-#define CLC_VLOADA_HALF_NAME(x)                                                \
+#define __CLC_VLOADA_HALF_NAME(x)                                              \
   __CLC_XCONCAT(__CLC_XCONCAT(x, vloada_half), __CLC_VECSIZE)
 
 #ifndef __CLC_SCALAR
 
-#define CLC_VLOAD_TY __CLC_XCONCAT(less_aligned_, __CLC_GENTYPE)
+#define __CLC_VLOAD_TY __CLC_XCONCAT(less_aligned_, __CLC_GENTYPE)
 
-#define VLOAD_DEF(ADDRSPACE)                                                   \
-  _CLC_OVERLOAD _CLC_DEF CLC_VLOAD_TY CLC_VLOAD_NAME()(                        \
+#define __CLC_VLOAD_DEF(ADDRSPACE)                                             \
+  _CLC_OVERLOAD _CLC_DEF __CLC_VLOAD_TY __CLC_VLOAD_NAME()(                    \
       size_t offset, const ADDRSPACE __CLC_SCALAR_GENTYPE *x) {                \
-    return CLC_VLOAD_NAME(__clc_)(offset, x);                                  \
+    return __CLC_VLOAD_NAME(__clc_)(offset, x);                                \
   }
 
-VLOAD_DEF(__private)
-VLOAD_DEF(__local)
-VLOAD_DEF(__constant)
-VLOAD_DEF(__global)
+__CLC_VLOAD_DEF(__private)
+__CLC_VLOAD_DEF(__local)
+__CLC_VLOAD_DEF(__constant)
+__CLC_VLOAD_DEF(__global)
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-VLOAD_DEF(__generic)
+__CLC_VLOAD_DEF(__generic)
 #endif
 
-#undef VLOAD_DEF
-#undef CLC_VLOAD_TY
+#undef __CLC_VLOAD_DEF
+#undef __CLC_VLOAD_TY
 
 #endif
 
@@ -42,30 +43,30 @@ VLOAD_DEF(__generic)
 #ifdef __CLC_FPSIZE
 #if __CLC_FPSIZE == 32
 
-#define VLOAD_HALF_DEF(ADDRSPACE)                                              \
-  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE CLC_VLOAD_HALF_NAME()(                  \
+#define __CLC_VLOAD_HALF_DEF(ADDRSPACE)                                        \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_VLOAD_HALF_NAME()(                \
       size_t offset, const ADDRSPACE half *mem) {                              \
-    return CLC_VLOAD_HALF_NAME(__clc_)(offset, mem);                           \
+    return __CLC_VLOAD_HALF_NAME(__clc_)(offset, mem);                         \
   }                                                                            \
                                                                                \
-  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE CLC_VLOADA_HALF_NAME()(                 \
+  _CLC_OVERLOAD _CLC_DEF __CLC_GENTYPE __CLC_VLOADA_HALF_NAME()(               \
       size_t offset, const ADDRSPACE half *mem) {                              \
-    return CLC_VLOADA_HALF_NAME(__clc_)(offset, mem);                          \
+    return __CLC_VLOADA_HALF_NAME(__clc_)(offset, mem);                        \
   }
 
-VLOAD_HALF_DEF(__private)
-VLOAD_HALF_DEF(__local)
-VLOAD_HALF_DEF(__constant)
-VLOAD_HALF_DEF(__global)
+__CLC_VLOAD_HALF_DEF(__private)
+__CLC_VLOAD_HALF_DEF(__local)
+__CLC_VLOAD_HALF_DEF(__constant)
+__CLC_VLOAD_HALF_DEF(__global)
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-VLOAD_HALF_DEF(__generic)
+__CLC_VLOAD_HALF_DEF(__generic)
 #endif
 
-#undef VLOAD_HALF_DEF
+#undef __CLC_VLOAD_HALF_DEF
 #endif
 #endif
 
-#undef CLC_VLOAD_NAME
-#undef CLC_VLOAD_HALF_NAME
-#undef CLC_VLOADA_HALF_NAME
+#undef __CLC_VLOAD_NAME
+#undef __CLC_VLOAD_HALF_NAME
+#undef __CLC_VLOADA_HALF_NAME
diff --git a/libclc/opencl/lib/generic/shared/vstore.inc b/libclc/opencl/lib/generic/shared/vstore.inc
index 4bdce0719912d..79ae9f9c9df20 100644
--- a/libclc/opencl/lib/generic/shared/vstore.inc
+++ b/libclc/opencl/lib/generic/shared/vstore.inc
@@ -6,31 +6,32 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define CLC_VSTORE_TY __CLC_XCONCAT(less_aligned_, __CLC_GENTYPE)
-#define CLC_VSTORE_NAME(x)                                                     \
+#define __CLC_VSTORE_TY __CLC_XCONCAT(less_aligned_, __CLC_GENTYPE)
+#define __CLC_VSTORE_NAME(x)                                                   \
   __CLC_XCONCAT(__CLC_XCONCAT(x, vstore), __CLC_VECSIZE)
-#define CLC_VSTORE_HALF_NAME(x, y)                                             \
+#define __CLC_VSTORE_HALF_NAME(x, y)                                           \
   __CLC_XCONCAT(__CLC_XCONCAT(__CLC_XCONCAT(x, vstore_half), __CLC_VECSIZE), y)
-#define CLC_VSTOREA_HALF_NAME(x, y)                                            \
+#define __CLC_VSTOREA_HALF_NAME(x, y)                                          \
   __CLC_XCONCAT(__CLC_XCONCAT(__CLC_XCONCAT(x, vstorea_half), __CLC_VECSIZE), y)
 
 #ifndef __CLC_SCALAR
 
-#define CLC_VSTORE_DEF(ADDRSPACE)                                              \
-  _CLC_OVERLOAD _CLC_DEF void CLC_VSTORE_NAME()(                               \
-      CLC_VSTORE_TY data, size_t offset, ADDRSPACE __CLC_SCALAR_GENTYPE *p) {  \
-    return CLC_VSTORE_NAME(__clc_)(data, offset, p);                           \
+#define __CLC_VSTORE_DEF(ADDRSPACE)                                            \
+  _CLC_OVERLOAD _CLC_DEF void __CLC_VSTORE_NAME()(                             \
+      __CLC_VSTORE_TY data, size_t offset,                                     \
+      ADDRSPACE __CLC_SCALAR_GENTYPE *p) {                                     \
+    return __CLC_VSTORE_NAME(__clc_)(data, offset, p);                         \
   }
 
-CLC_VSTORE_DEF(__private)
-CLC_VSTORE_DEF(__local)
-CLC_VSTORE_DEF(__global)
+__CLC_VSTORE_DEF(__private)
+__CLC_VSTORE_DEF(__local)
+__CLC_VSTORE_DEF(__global)
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-CLC_VSTORE_DEF(__generic)
+__CLC_VSTORE_DEF(__generic)
 #endif
 
-#undef CLC_VSTORE_DEF
+#undef __CLC_VSTORE_DEF
 
 #endif // __CLC_SCALAR
 
@@ -39,39 +40,39 @@ CLC_VSTORE_DEF(__generic)
 #ifdef __CLC_FPSIZE
 #if __CLC_FPSIZE == 32 || __CLC_FPSIZE == 64
 
-#define CLC_VSTORE_HALF_DEF(ADDRSPACE, SUFFIX)                                 \
-  _CLC_OVERLOAD _CLC_DEF void CLC_VSTORE_HALF_NAME(, SUFFIX)(                  \
-      CLC_VSTORE_TY data, size_t offset, ADDRSPACE half *p) {                  \
-    CLC_VSTORE_HALF_NAME(__clc_, SUFFIX)(data, offset, p);                     \
+#define __CLC_VSTORE_HALF_DEF(ADDRSPACE, SUFFIX)                               \
+  _CLC_OVERLOAD _CLC_DEF void __CLC_VSTORE_HALF_NAME(, SUFFIX)(                \
+      __CLC_VSTORE_TY data, size_t offset, ADDRSPACE half *p) {                \
+    __CLC_VSTORE_HALF_NAME(__clc_, SUFFIX)(data, offset, p);                   \
   }                                                                            \
                                                                                \
-  _CLC_OVERLOAD _CLC_DEF void CLC_VSTOREA_HALF_NAME(, SUFFIX)(                 \
-      CLC_VSTORE_TY data, size_t offset, ADDRSPACE half *p) {                  \
-    CLC_VSTOREA_HALF_NAME(__clc_, SUFFIX)(data, offset, p);                    \
+  _CLC_OVERLOAD _CLC_DEF void __CLC_VSTOREA_HALF_NAME(, SUFFIX)(               \
+      __CLC_VSTORE_TY data, size_t offset, ADDRSPACE half *p) {                \
+    __CLC_VSTOREA_HALF_NAME(__clc_, SUFFIX)(data, offset, p);                  \
   }
 
-#define CLC_VSTORE_HALF_DEF_ALL_MODES(ADDRSPACE)                               \
-  CLC_VSTORE_HALF_DEF(ADDRSPACE, )                                             \
-  CLC_VSTORE_HALF_DEF(ADDRSPACE, _rtz)                                         \
-  CLC_VSTORE_HALF_DEF(ADDRSPACE, _rtn)                                         \
-  CLC_VSTORE_HALF_DEF(ADDRSPACE, _rtp)                                         \
-  CLC_VSTORE_HALF_DEF(ADDRSPACE, _rte)
+#define __CLC_VSTORE_HALF_DEF_ALL_MODES(ADDRSPACE)                             \
+  __CLC_VSTORE_HALF_DEF(ADDRSPACE, )                                           \
+  __CLC_VSTORE_HALF_DEF(ADDRSPACE, _rtz)                                       \
+  __CLC_VSTORE_HALF_DEF(ADDRSPACE, _rtn)                                       \
+  __CLC_VSTORE_HALF_DEF(ADDRSPACE, _rtp)                                       \
+  __CLC_VSTORE_HALF_DEF(ADDRSPACE, _rte)
 
-CLC_VSTORE_HALF_DEF_ALL_MODES(__private)
-CLC_VSTORE_HALF_DEF_ALL_MODES(__local)
-CLC_VSTORE_HALF_DEF_ALL_MODES(__global)
+__CLC_VSTORE_HALF_DEF_ALL_MODES(__private)
+__CLC_VSTORE_HALF_DEF_ALL_MODES(__local)
+__CLC_VSTORE_HALF_DEF_ALL_MODES(__global)
 
 #if _CLC_DISTINCT_GENERIC_AS_SUPPORTED
-CLC_VSTORE_HALF_DEF_ALL_MODES(__generic)
+__CLC_VSTORE_HALF_DEF_ALL_MODES(__generic)
 #endif
 
-#undef CLC_VSTORE_HALF_DEF
-#undef CLC_VSTORE_HALF_DEF_ALL_MODES
+#undef __CLC_VSTORE_HALF_DEF
+#undef __CLC_VSTORE_HALF_DEF_ALL_MODES
 
 #endif
 #endif
 
-#undef CLC_VSTORE_TY
-#undef CLC_VSTORE_NAME
-#undef CLC_VSTORE_HALF_NAME
-#undef CLC_VSTOREA_HALF_NAME
+#undef __CLC_VSTORE_TY
+#undef __CLC_VSTORE_NAME
+#undef __CLC_VSTORE_HALF_NAME
+#undef __CLC_VSTOREA_HALF_NAME
diff --git a/libclc/opencl/lib/spirv/math/fma.cl b/libclc/opencl/lib/spirv/math/fma.cl
index 83504f63772ca..0e328903ba263 100644
--- a/libclc/opencl/lib/spirv/math/fma.cl
+++ b/libclc/opencl/lib/spirv/math/fma.cl
@@ -9,9 +9,9 @@
 #include <clc/internal/math/clc_sw_fma.h>
 #include <clc/opencl/math/fma.h>
 
-#define __FLOAT_ONLY
-#define FUNCTION fma
-#define __IMPL_FUNCTION(x) __clc_sw_fma
+#define __CLC_FLOAT_ONLY
+#define __CLC_FUNCTION fma
+#define __CLC_IMPL_FUNCTION(x) __clc_sw_fma
 #define __CLC_BODY <clc/shared/ternary_def.inc>
 
 #include <clc/math/gentype.inc>
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 85514cc7547a9..5162963bafd63 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -113,7 +113,7 @@ option(LIBCXX_ENABLE_MONOTONIC_CLOCK
 #
 # TODO TZDB make the default always ON when most platforms ship with the IANA
 # database.
-if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "Linux")
   set(ENABLE_TIME_ZONE_DATABASE_DEFAULT ON)
 else()
   set(ENABLE_TIME_ZONE_DATABASE_DEFAULT OFF)
@@ -159,7 +159,7 @@ set(LIBCXX_TEST_PARAMS "" CACHE STRING
     "A list of parameters to run the Lit test suite with.")
 
 # TODO: Figure out how to build GoogleBenchmark on those platforms, and how to build when exceptions or RTTI is disabled
-if (WIN32 OR MINGW OR ANDROID OR ${CMAKE_SYSTEM_NAME} MATCHES "AIX"
+if (WIN32 OR MINGW OR ANDROID OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX"
     OR NOT LIBCXX_ENABLE_LOCALIZATION
     OR NOT LIBCXX_ENABLE_THREADS
     OR NOT LIBCXX_ENABLE_FILESYSTEM
@@ -466,7 +466,7 @@ include(HandleLibcxxFlags)
 # 'config-ix' use them during feature checks. It also adds them to both
 # 'LIBCXX_COMPILE_FLAGS' and 'LIBCXX_LINK_FLAGS'
 
-if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   add_flags_if_supported("-mdefault-visibility-export-mapping=explicit")
   set(CMAKE_AIX_EXPORT_ALL_SYMBOLS OFF)
 endif()
diff --git a/libcxx/cmake/caches/Generic-hardening-mode-fast-with-abi-breaks.cmake b/libcxx/cmake/caches/Generic-hardening-mode-fast-with-abi-breaks.cmake
index 699d3f8866861..d4ce32ce5b17f 100644
--- a/libcxx/cmake/caches/Generic-hardening-mode-fast-with-abi-breaks.cmake
+++ b/libcxx/cmake/caches/Generic-hardening-mode-fast-with-abi-breaks.cmake
@@ -5,5 +5,6 @@ set(_defines
     _LIBCPP_ABI_BOUNDED_ITERATORS_IN_VECTOR
     _LIBCPP_ABI_BOUNDED_UNIQUE_PTR
     _LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY
+    _LIBCPP_ABI_BOUNDED_ITERATORS_IN_OPTIONAL
 )
 set(LIBCXX_ABI_DEFINES "${_defines}" CACHE STRING "")
diff --git a/libcxx/docs/Contributing.rst b/libcxx/docs/Contributing.rst
index 6aaa70764c2fa..ac856195ad68e 100644
--- a/libcxx/docs/Contributing.rst
+++ b/libcxx/docs/Contributing.rst
@@ -174,10 +174,11 @@ Pre-commit CI
 Introduction
 ------------
 
-Unlike most parts of the LLVM project, libc++ uses a pre-commit CI [#]_. This
-CI is hosted on `Buildkite <https://buildkite.com/llvm-project/libcxx-ci>`__ and
-the build results are visible in the review on GitHub. Please make sure
-the CI is green before committing a patch.
+Unlike most parts of the LLVM project, libc++ uses a pre-commit CI [#]_. Some of
+this CI is hosted on `Buildkite <https://buildkite.com/llvm-project/libcxx-ci>`__,
+but some has migrated to the LLVM CI infrastructure. The build results are
+visible in the review on GitHub. Please make sure the CI is green before
+committing a patch.
 
 The CI tests libc++ for all :ref:`supported platforms <SupportedPlatforms>`.
 The build is started for every commit added to a Pull Request. A complete CI
@@ -246,21 +247,89 @@ Below is a short description of the most interesting CI builds [#]_:
 Infrastructure
 --------------
 
-All files of the CI infrastructure are in the directory ``libcxx/utils/ci``.
-Note that quite a bit of this infrastructure is heavily Linux focused. This is
-the platform used by most of libc++'s Buildkite runners and developers.
+The files for the CI infrastructure are split between the llvm-project
+and the llvm-zorg repositories. All files of the CI infrastructure in
+the llvm-project are in the directory ``libcxx/utils/ci``. Note that
+quite a bit of this infrastructure is heavily Linux focused. This is
+the platform used by most of libc++'s Buildkite runners and
+developers.
 
-Dockerfile
-~~~~~~~~~~
+Dockerfile/Container Images
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Contains the Docker image for the Ubuntu CI. Because the same Docker image is
 used for the ``main`` and ``release`` branch, it should contain no hard-coded
-versions.  It contains the used versions of Clang, various clang-tools,
+versions. It contains the used versions of Clang, various clang-tools,
 GCC, and CMake.
 
 .. note:: This image is pulled from Docker hub and not rebuild when changing
    the Dockerfile.
 
+Updating the CI testing container images
+----------------------------------------
+
+The libcxx linux premerge testing can run on one of three sets of runner
+groups. The three runner group names are "llvm-premerge-libcxx-runners",
+"llvm-premerge-libcxx-release-runners" and "llvm-premerge-libcxx-next-runners".
+Which runner set to use is controlled by the contents of
+https://github.com/llvm/llvm-project/blob/main/.github/workflows/libcxx-build-and-test.yaml.
+By default, it uses "llvm-premerge-libcxx-runners". To switch to one of the
+other runner sets, just replace all uses of "llvm-premerge-libcxx-runners" in
+the yaml file with the desired runner set.
+
+Which container image is used by these three runner sets is controlled
+and set by the variable values in
+https://github.com/llvm/llvm-zorg/blob/main/premerge/premerge_resources/variables.tf.
+The table below shows the variable names and
+the runner sets to which they correspond. To see their values, follow the
+link above (to variables.tf in llvm-zorg).
+
++------------------------------------+---------------------------+
+|Runner Set                          |Variable                   |
++====================================+===========================+
+|llvm-premerge-libcxx-runners        |libcxx_runner_image        |
++------------------------------------+---------------------------+
+|llvm-premerge-libcxx-release-runners|libcxx_release_runner_image|
++------------------------------------+---------------------------+
+|llvm-premerge-libcxx-next-runners   |libcxx_next_runner_image   |
++------------------------------------+---------------------------+
+
+
+When updating the container image you can either update just the
+runner binary (the part the connects to Github), or you can update
+everything (tools, etc.). Whether to update just the runner or to update
+everything is controlled by the value of ``ACTIONS_BASE_IMAGE``, under
+``actions-builder`` in ``libcxx/utils/ci/docker-compose.yml``.
+
+To update just the runner binary, change the value of ``ACTIONS_BASE_IMAGE``
+to be a modified version of one of the libcxx runner variable images from
+https://github.com/llvm/llvm-zorg/blob/main/premerge/premerge_resources/variables.tf,
+as follows: Find the libcxx runner image name you want to use from the
+variables.tf file. The name will be something like
+``ghcr.io/llvm/libcxx-linux-builder:<some-commit-SHA>``. Replace
+``libcxx-linux-builder`` with ``libcxx-linux-builder-base``. Use this new image
+name as the value you assign to ``ACTIONS_BASE_IMAGE``.
+
+To update the entire container image, set the value of ``ACTIONS_BASE_IMAGE``
+to ``builder-base``. If the value is already ``builder-base`` (there
+have been no just-the-runner updates since the last complete update), then you
+need to find the line containing ``RUN echo "Last forced update executed on``
+in ``libcxx/utils/ci/Dockerfile`` and update the date to be the current date.
+
+Once you have created and merged a PR with those changes, a new image
+will be created, and a link to it can be found at
+https://github.com/llvm/llvm-project/pkgs/container/libcxx-linux-builder,
+where the actual image name should be
+``ghcr.io/llvm/libcxx-linux-builder:<SHA-of-committed-change-from-PR>``.
+
+Lastly you need to create a PR in the llvm-zorg repository,
+updating the the value of the appropriate libcxx runner variable in
+the variables.tf file mentioned above to the name of your newly created
+image (see above paragraph about finding the image name). Once that change
+has been merged, an LLVM premerge maintainer (a Google employee) must use
+terraform to apply the change to the running GKE cluster.
+
+
 run-buildbot-container
 ~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index a36848ebd24b4..358889d8dbc37 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -480,7 +480,7 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_not_fn``                                       ``202306L``
     ---------------------------------------------------------- -----------------
-    ``__cpp_lib_optional_range_support``                       *unimplemented*
+    ``__cpp_lib_optional_range_support``                       ``202406L``
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_out_ptr``                                      ``202311L``
     ---------------------------------------------------------- -----------------
diff --git a/libcxx/docs/ReleaseNotes/21.rst b/libcxx/docs/ReleaseNotes/21.rst
index 91123ffa3e34b..642bf7dc1bc84 100644
--- a/libcxx/docs/ReleaseNotes/21.rst
+++ b/libcxx/docs/ReleaseNotes/21.rst
@@ -148,6 +148,21 @@ ABI Affecting Changes
   comparison between shared libraries, since all RTTI has the correct visibility now. There is no behaviour change on
   Clang.
 
+- LLVM 20 contained an ABI break that can result in the size of ``std::unordered_{map,set,multimap,multiset}`` and ``std::deque`` changing when used with an allocator type that is empty and contains a base class that is the same across rebound allocator instantiations (e.g. ``Allocator<int>`` and ``Allocator<char>`` are both empty and contain the same base class).
+  In addition, the layout of a user-defined type that:
+
+    - contains one of the following containers: ``std::unordered_{map,set,multimap,multiset}``, ``std::deque``, ``std::map``, ``std::set``, ``std::multimap``, ``std::multiset``, ``std::list`` or ``std::vector``, and
+    - passes an empty allocator, comparator or hasher type to that container, and
+    - has a member of that same empty allocator, comparator or hasher type inside the enclosing struct, and
+    - that member is either marked with ``[[no_unique_address]]`` or optimized out via the EBO (empty base optimization) technique
+
+  saw its size increase from LLVM 19 to LLVM 20. This was caused by the usage of ``[[no_unique_address]]`` within some of libc++'s containers in a way that allowed subtle interactions with enclosing objects. This is fixed in LLVM 21 when using the Clang compiler (returning to the LLVM 19 ABI), however that implies an ABI break from LLVM 20 to LLVM 21.
+
+  Furthermore, fixing this causes a slight regression to constant evaluation support in ``std::unique_ptr``. Specifically, constant evaluation will now fail when the deleter relies on being value-initialized for constant-evaluation admissibility. If a default-initialized deleter can be used during constant evaluation, or if the default constructor is non-trivial, the ``unique_ptr`` is not affected by this regression. In particular, this regression does not impact any ``unique_ptr`` using the default deleter.
+
+  Note that there is currently no way to realistically fix this ABI break on GCC, therefore GCC will remain on the ABI introduced in LLVM 19. That also means that Clang and GCC will have a slightly different ABI for the small subset of types listed above until we are able to apply the same fix we did with Clang on GCC.
+
+  For more details see https://llvm.org/PR154146.
 
 Build System Changes
 --------------------
diff --git a/libcxx/docs/ReleaseNotes/22.rst b/libcxx/docs/ReleaseNotes/22.rst
index 8b8dce5083149..a26c5476d421b 100644
--- a/libcxx/docs/ReleaseNotes/22.rst
+++ b/libcxx/docs/ReleaseNotes/22.rst
@@ -39,12 +39,21 @@ Implemented Papers
 ------------------
 
 - P2321R2: ``zip`` (`Github <https://github.com/llvm/llvm-project/issues/105169>`__) (The paper is partially implemented. ``zip_transform_view`` is implemented in this release)
+- P3168R2: Give ``std::optional`` Range Support (`Github <https://github.com/llvm/llvm-project/issues/105430>`__)
 
 Improvements and New Features
 -----------------------------
 
 - The performance of ``map::map(const map&)`` has been improved up to 2.3x
 - The performance of ``map::operator=(const map&)`` has been improved by up to 11x
+- The performance of ``unordered_set::unordered_set(const unordered_set&)`` has been improved by up to 3.3x.
+- The performance of ``unordered_set::operator=(const unordered_set&)`` has been improved by up to 5x.
+- The performance of ``map::erase`` and ``set::erase`` has been improved by up to 2x
+- The performance of ``find(key)`` in ``map``, ``set``, ``multimap`` and ``multiset`` has been improved by up to 2.3x
+- Some reallocations are now avoided in `std::filesystem::path::lexically_relative`, resulting in a performance improvement of up to 1.7x.
+- The performance of the ``(iterator, iterator)`` constructors of ``multimap`` and ``multiset``
+  has been improved by up to 3x
+- The performance of ``insert(iterator, iterator)`` of ``multimap`` and ``multiset`` has been improved by up to 2.5x
 
 Deprecations and Removals
 -------------------------
diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index 189f8452e0678..d1546f4a452b5 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -209,7 +209,7 @@
 "`LWG3746 <https://wg21.link/LWG3746>`__","``optional``'s spaceship with ``U`` with a type derived from optional causes infinite constraint meta-recursion","2022-11 (Kona)","|Complete|","17",""
 "`LWG3747 <https://wg21.link/LWG3747>`__","``ranges::uninitialized_copy_n``, ``ranges::uninitialized_move_n``, and ``ranges::destroy_n`` should use ``std::move``","2022-11 (Kona)","","",""
 "`LWG3750 <https://wg21.link/LWG3750>`__","Too many papers bump ``__cpp_lib_format``","2022-11 (Kona)","|Partial|","","Only ``__cpp_lib_format_ranges`` is fully implemented"
-"`LWG3751 <https://wg21.link/LWG3751>`__","Missing feature macro for ``flat_set``","2022-11 (Kona)","","",""
+"`LWG3751 <https://wg21.link/LWG3751>`__","Missing feature macro for ``flat_set``","2022-11 (Kona)","|Complete|","21",""
 "`LWG3753 <https://wg21.link/LWG3753>`__","Clarify entity vs. freestanding entity","2022-11 (Kona)","","",""
 "`LWG3754 <https://wg21.link/LWG3754>`__","Class template expected synopsis contains declarations that do not match the detailed description","2022-11 (Kona)","|Nothing To Do|","",""
 "`LWG3755 <https://wg21.link/LWG3755>`__","``tuple-for-each`` can call ``user-defined`` ``operator,``","2022-11 (Kona)","|Complete|","17",""
@@ -223,14 +223,14 @@
 "`LWG3766 <https://wg21.link/LWG3766>`__","``view_interface::cbegin`` is underconstrained","2022-11 (Kona)","","",""
 "`LWG3770 <https://wg21.link/LWG3770>`__","``const_sentinel_t`` is missing","2022-11 (Kona)","","",""
 "`LWG3773 <https://wg21.link/LWG3773>`__","``views::zip_transform`` still requires ``F`` to be ``copy_constructible`` when empty pack","2022-11 (Kona)","|Complete|","22",""
-"`LWG3774 <https://wg21.link/LWG3774>`__","``<flat_set>`` should include ``<compare>``","2022-11 (Kona)","","",""
+"`LWG3774 <https://wg21.link/LWG3774>`__","``<flat_set>`` should include ``<compare>``","2022-11 (Kona)","|Complete|","21",""
 "`LWG3775 <https://wg21.link/LWG3775>`__","Broken dependencies in the ``Cpp17Allocator`` requirements","2022-11 (Kona)","","",""
 "`LWG3778 <https://wg21.link/LWG3778>`__","``vector<bool>`` missing exception specifications","2022-11 (Kona)","|Complete|","3.7",""
 "`LWG3781 <https://wg21.link/LWG3781>`__","The exposition-only alias templates ``cont-key-type`` and ``cont-mapped-type`` should be removed","2022-11 (Kona)","|Nothing To Do|","",""
 "`LWG3782 <https://wg21.link/LWG3782>`__","Should ``<math.h>`` declare ``::lerp``?","2022-11 (Kona)","|Complete|","17",""
 "`LWG3784 <https://wg21.link/LWG3784>`__","std.compat should not provide ``::byte`` and its friends","2022-11 (Kona)","|Complete|","19",""
 "`LWG3785 <https://wg21.link/LWG3785>`__","``ranges::to`` is over-constrained on the destination type being a range","2022-11 (Kona)","","",""
-"`LWG3788 <https://wg21.link/LWG3788>`__","``jthread::operator=(jthread&&)`` postconditions are unimplementable under self-assignment","2022-11 (Kona)","","",""
+"`LWG3788 <https://wg21.link/LWG3788>`__","``jthread::operator=(jthread&&)`` postconditions are unimplementable under self-assignment","2022-11 (Kona)","|Complete|","18",""
 "`LWG3792 <https://wg21.link/LWG3792>`__","``__cpp_lib_constexpr_algorithms`` should also be defined in ``<utility>``","2022-11 (Kona)","|Complete|","16",""
 "`LWG3795 <https://wg21.link/LWG3795>`__","Self-move-assignment of ``std::future`` and ``std::shared_future`` have unimplementable postconditions","2022-11 (Kona)","","",""
 "`LWG3796 <https://wg21.link/LWG3796>`__","``movable-box`` as member should use ``default-initialization`` instead of ``copy-initialization``","2022-11 (Kona)","","",""
diff --git a/libcxx/docs/Status/Cxx2cIssues.csv b/libcxx/docs/Status/Cxx2cIssues.csv
index c6225127a74b9..082a86a65c852 100644
--- a/libcxx/docs/Status/Cxx2cIssues.csv
+++ b/libcxx/docs/Status/Cxx2cIssues.csv
@@ -71,7 +71,7 @@
 "`LWG4079 <https://wg21.link/LWG4079>`__","Missing Preconditions in ``concat_view::iterator``\`s conversion constructor","2024-06 (St. Louis)","","",""
 "`LWG4082 <https://wg21.link/LWG4082>`__","``views::concat(r)`` is well-formed when ``r`` is an ``output_range``","2024-06 (St. Louis)","","",""
 "`LWG4083 <https://wg21.link/LWG4083>`__","``views::as_rvalue`` should reject non-input ranges","2024-06 (St. Louis)","","",""
-"`LWG4096 <https://wg21.link/LWG4096>`__","``views::iota(views::iota(0))`` should be rejected","2024-06 (St. Louis)","","",""
+"`LWG4096 <https://wg21.link/LWG4096>`__","``views::iota(views::iota(0))`` should be rejected","2024-06 (St. Louis)","|Complete|","22",""
 "`LWG4098 <https://wg21.link/LWG4098>`__","``views::adjacent<0>`` should reject non-forward ranges","2024-06 (St. Louis)","","",""
 "`LWG4105 <https://wg21.link/LWG4105>`__","``ranges::ends_with``\`s Returns misses difference casting","2024-06 (St. Louis)","","",""
 "`LWG4106 <https://wg21.link/LWG4106>`__","``basic_format_args`` should not be default-constructible","2024-06 (St. Louis)","|Complete|","19",""
@@ -132,7 +132,7 @@
 "`LWG4200 <https://wg21.link/LWG4200>`__","The ``operation_state`` concept can be simplified","2025-06 (Sofia)","","",""
 "`LWG4201 <https://wg21.link/LWG4201>`__","``with-await-transform::await_transform`` should not use a deduced return type","2025-06 (Sofia)","","",""
 "`LWG4217 <https://wg21.link/LWG4217>`__","Clarify ``mdspan`` layout mapping requirements for ``rank == 0``","2025-06 (Sofia)","","",""
-"`LWG4222 <https://wg21.link/LWG4222>`__","``expected`` constructor from a single value missing a constraint","2025-06 (Sofia)","","",""
+"`LWG4222 <https://wg21.link/LWG4222>`__","``expected`` constructor from a single value missing a constraint","2025-06 (Sofia)","|Complete|","22",""
 "`LWG4224 <https://wg21.link/LWG4224>`__","Philox engines should be freestanding","2025-06 (Sofia)","","",""
 "`LWG4227 <https://wg21.link/LWG4227>`__","Missing ``noexcept`` operator in [exec.when.all]","2025-06 (Sofia)","","",""
 "`LWG4231 <https://wg21.link/LWG4231>`__","``datapar::chunk<N>`` should use ``simd-size-type`` instead of ``size_t``","2025-06 (Sofia)","","",""
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index e8b0c9559f40b..3b8b2b7ad0b3f 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -66,7 +66,7 @@
 "`P2747R2 <https://wg21.link/P2747R2>`__","``constexpr`` placement new","2024-06 (St. Louis)","|Complete|","20",""
 "`P2997R1 <https://wg21.link/P2997R1>`__","Removing the common reference requirement from the indirectly invocable concepts","2024-06 (St. Louis)","|Complete|","19","Implemented as a DR against C++20. (MSVC STL and libstdc++ will do the same.)"
 "`P2389R2 <https://wg21.link/P2389R2>`__","``dextents`` Index Type Parameter","2024-06 (St. Louis)","|Complete|","19",""
-"`P3168R2 <https://wg21.link/P3168R2>`__","Give ``std::optional`` Range Support","2024-06 (St. Louis)","","",""
+"`P3168R2 <https://wg21.link/P3168R2>`__","Give ``std::optional`` Range Support","2024-06 (St. Louis)","|Complete|","22",""
 "`P3217R0 <https://wg21.link/P3217R0>`__","Adjoints to 'Enabling list-initialization for algorithms': find_last","2024-06 (St. Louis)","","",""
 "`P2985R0 <https://wg21.link/P2985R0>`__","A type trait for detecting virtual base classes","2024-06 (St. Louis)","|Complete|","20",""
 "`P0843R14 <https://wg21.link/P0843R14>`__","``inplace_vector``","2024-06 (St. Louis)","","",""
diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 51444ec668e2b..c6b87a34a43e9 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -447,14 +447,12 @@ set(files
   __fwd/get.h
   __fwd/ios.h
   __fwd/istream.h
-  __fwd/map.h
   __fwd/mdspan.h
   __fwd/memory.h
   __fwd/memory_resource.h
   __fwd/ostream.h
   __fwd/pair.h
   __fwd/queue.h
-  __fwd/set.h
   __fwd/span.h
   __fwd/sstream.h
   __fwd/stack.h
diff --git a/libcxx/include/__expected/expected.h b/libcxx/include/__expected/expected.h
index 0f446b870723b..38a34121040f6 100644
--- a/libcxx/include/__expected/expected.h
+++ b/libcxx/include/__expected/expected.h
@@ -557,7 +557,8 @@ class expected : private __expected_base<_Tp, _Err> {
 
   template <class _Up = _Tp>
     requires(!is_same_v<remove_cvref_t<_Up>, in_place_t> && !is_same_v<expected, remove_cvref_t<_Up>> &&
-             is_constructible_v<_Tp, _Up> && !__is_std_unexpected<remove_cvref_t<_Up>>::value &&
+             !is_same_v<remove_cvref_t<_Up>, unexpect_t> && is_constructible_v<_Tp, _Up> &&
+             !__is_std_unexpected<remove_cvref_t<_Up>>::value &&
              (!is_same_v<remove_cv_t<_Tp>, bool> || !__is_std_expected<remove_cvref_t<_Up>>::value))
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!is_convertible_v<_Up, _Tp>)
       expected(_Up&& __u) noexcept(is_nothrow_constructible_v<_Tp, _Up>) // strengthened
diff --git a/libcxx/include/__fwd/map.h b/libcxx/include/__fwd/map.h
deleted file mode 100644
index 940298dd3e563..0000000000000
--- a/libcxx/include/__fwd/map.h
+++ /dev/null
@@ -1,31 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___FWD_MAP_H
-#define _LIBCPP___FWD_MAP_H
-
-#include <__config>
-#include <__fwd/functional.h>
-#include <__fwd/memory.h>
-#include <__fwd/pair.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > >
-class map;
-
-template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > >
-class multimap;
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___FWD_MAP_H
diff --git a/libcxx/include/__fwd/set.h b/libcxx/include/__fwd/set.h
deleted file mode 100644
index d5ef8d56b0015..0000000000000
--- a/libcxx/include/__fwd/set.h
+++ /dev/null
@@ -1,30 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef _LIBCPP___FWD_SET_H
-#define _LIBCPP___FWD_SET_H
-
-#include <__config>
-#include <__fwd/functional.h>
-#include <__fwd/memory.h>
-
-#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
-#endif
-
-_LIBCPP_BEGIN_NAMESPACE_STD
-
-template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> >
-class set;
-
-template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> >
-class multiset;
-
-_LIBCPP_END_NAMESPACE_STD
-
-#endif // _LIBCPP___FWD_SET_H
diff --git a/libcxx/include/__hash_table b/libcxx/include/__hash_table
index dacc152030e14..996ec9fa31ac3 100644
--- a/libcxx/include/__hash_table
+++ b/libcxx/include/__hash_table
@@ -10,6 +10,7 @@
 #ifndef _LIBCPP___HASH_TABLE
 #define _LIBCPP___HASH_TABLE
 
+#include <__algorithm/fill_n.h>
 #include <__algorithm/max.h>
 #include <__algorithm/min.h>
 #include <__assert>
@@ -700,6 +701,38 @@ private:
 
   _LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __size_; }
 
+  _LIBCPP_HIDE_FROM_ABI void
+  __copy_construct(__next_pointer __other_iter, __next_pointer __own_iter, size_t __current_chash) {
+    auto __bucket_count = bucket_count();
+
+    for (; __other_iter; __other_iter = __other_iter->__next_) {
+      __node_holder __new_node = __construct_node_hash(__other_iter->__hash(), __other_iter->__upcast()->__get_value());
+
+      size_t __new_chash = std::__constrain_hash(__new_node->__hash(), __bucket_count);
+      if (__new_chash != __current_chash) {
+        __bucket_list_[__new_chash] = __own_iter;
+        __current_chash             = __new_chash;
+      }
+
+      __own_iter->__next_ = static_cast<__next_pointer>(__new_node.release());
+      __own_iter          = __own_iter->__next_;
+    }
+  }
+
+  _LIBCPP_HIDE_FROM_ABI void __copy_construct(__next_pointer __other_iter) {
+    __next_pointer __own_iter = __first_node_.__ptr();
+    {
+      __node_holder __new_node = __construct_node_hash(__other_iter->__hash(), __other_iter->__upcast()->__get_value());
+      __own_iter->__next_      = static_cast<__next_pointer>(__new_node.release());
+    }
+
+    size_t __current_chash          = std::__constrain_hash(__own_iter->__next_->__hash(), bucket_count());
+    __bucket_list_[__current_chash] = __own_iter;
+    __other_iter                    = __other_iter->__next_;
+    __own_iter                      = __own_iter->__next_;
+    __copy_construct(__other_iter, __own_iter, __current_chash);
+  }
+
 public:
   _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }
 
@@ -1048,16 +1081,29 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const allocator_type& __a
       __max_load_factor_(1.0f) {}
 
 template <class _Tp, class _Hash, class _Equal, class _Alloc>
-__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u)
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __other)
     : __bucket_list_(nullptr,
-                     __bucket_list_deleter(allocator_traits<__pointer_allocator>::select_on_container_copy_construction(
-                                               __u.__bucket_list_.get_deleter().__alloc()),
+                     __bucket_list_deleter(__pointer_alloc_traits::select_on_container_copy_construction(
+                                               __other.__bucket_list_.get_deleter().__alloc()),
                                            0)),
-      __node_alloc_(allocator_traits<__node_allocator>::select_on_container_copy_construction(__u.__node_alloc())),
+      __node_alloc_(__node_traits::select_on_container_copy_construction(__other.__node_alloc())),
       __size_(0),
-      __hasher_(__u.hash_function()),
-      __max_load_factor_(__u.__max_load_factor_),
-      __key_eq_(__u.__key_eq_) {}
+      __hasher_(__other.hash_function()),
+      __max_load_factor_(__other.__max_load_factor_),
+      __key_eq_(__other.__key_eq_) {
+  if (__other.size() == 0)
+    return;
+
+  auto& __bucket_list_del = __bucket_list_.get_deleter();
+  auto __bucket_count     = __other.bucket_count();
+  __bucket_list_.reset(__pointer_alloc_traits::allocate(__bucket_list_del.__alloc(), __bucket_count));
+  __bucket_list_del.size() = __bucket_count;
+
+  std::fill_n(__bucket_list_.get(), __bucket_count, nullptr);
+
+  __copy_construct(__other.__first_node_.__next_);
+  __size_ = __other.size();
+}
 
 template <class _Tp, class _Hash, class _Equal, class _Alloc>
 __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u, const allocator_type& __a)
@@ -1131,14 +1177,75 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__copy_assign_alloc(const __hash_
 }
 
 template <class _Tp, class _Hash, class _Equal, class _Alloc>
-__hash_table<_Tp, _Hash, _Equal, _Alloc>& __hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __u) {
-  if (this != std::addressof(__u)) {
-    __copy_assign_alloc(__u);
-    hash_function()   = __u.hash_function();
-    key_eq()          = __u.key_eq();
-    max_load_factor() = __u.max_load_factor();
-    __assign_multi(__u.begin(), __u.end());
+__hash_table<_Tp, _Hash, _Equal, _Alloc>&
+__hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __other) {
+  if (this == std::addressof(__other))
+    return *this;
+
+  __copy_assign_alloc(__other);
+  hash_function()   = __other.hash_function();
+  key_eq()          = __other.key_eq();
+  max_load_factor() = __other.max_load_factor();
+
+  if (__other.size() == 0) {
+    clear();
+    return *this;
+  }
+
+  auto __bucket_count = __other.bucket_count();
+  if (__bucket_count != bucket_count()) {
+    auto& __bucket_list_del = __bucket_list_.get_deleter();
+    __bucket_list_.reset(__pointer_alloc_traits::allocate(__bucket_list_del.__alloc(), __bucket_count));
+    __bucket_list_del.size() = __bucket_count;
+  }
+  std::fill_n(__bucket_list_.get(), __bucket_count, nullptr);
+
+  if (!__first_node_.__next_) {
+    __copy_construct(__other.__first_node_.__next_);
+    __size_ = __other.size();
+    return *this;
   }
+
+  __next_pointer __other_iter = __other.__first_node_.__next_;
+  __next_pointer __own_iter   = __first_node_.__ptr();
+  {
+    __node_pointer __next = __own_iter->__next_->__upcast();
+    __assign_value(__next->__get_value(), __other_iter->__upcast()->__get_value());
+    __next->__hash_ = __other_iter->__hash();
+  }
+  size_t __current_chash          = std::__constrain_hash(__own_iter->__next_->__hash(), __bucket_count);
+  __bucket_list_[__current_chash] = __own_iter;
+  __other_iter                    = __other_iter->__next_;
+  __own_iter                      = __own_iter->__next_;
+
+  // Go through the nodes of the incoming hash table and copy then into the destination hash table, reusing as many
+  // existing nodes as posssible in the destination.
+  while (__other_iter && __own_iter->__next_) {
+    __node_pointer __next = __own_iter->__next_->__upcast();
+    __assign_value(__next->__get_value(), __other_iter->__upcast()->__get_value());
+    __next->__hash_ = __other_iter->__hash();
+
+    size_t __new_chash = std::__constrain_hash(__next->__hash_, __bucket_count);
+    if (__new_chash != __current_chash) {
+      __bucket_list_[__new_chash] = __own_iter;
+      __current_chash             = __new_chash;
+    }
+
+    __other_iter = __other_iter->__next_;
+    __own_iter   = __own_iter->__next_;
+  }
+
+  // At this point we either have consumed the whole incoming hash table, or we don't have any more nodes to reuse in
+  // the destination. Either continue with constructing new nodes, or deallocate the left over nodes.
+  if (__own_iter->__next_) {
+    __deallocate_node(__own_iter->__next_);
+    __own_iter->__next_ = nullptr;
+  } else {
+    __copy_construct(__other_iter, __own_iter, __current_chash);
+  }
+
+  __size_ = __other.size();
+
   return *this;
 }
 
diff --git a/libcxx/include/__iterator/wrap_iter.h b/libcxx/include/__iterator/wrap_iter.h
index 2b5bc489dd44c..7610586ddecbb 100644
--- a/libcxx/include/__iterator/wrap_iter.h
+++ b/libcxx/include/__iterator/wrap_iter.h
@@ -117,6 +117,8 @@ class __wrap_iter {
   friend class span;
   template <class _Tp, size_t _Size>
   friend struct array;
+  template <class _Tp>
+  friend class optional;
 };
 
 template <class _Iter1>
diff --git a/libcxx/include/__memory/compressed_pair.h b/libcxx/include/__memory/compressed_pair.h
index fb7b7b7afcc8c..29e503931b0be 100644
--- a/libcxx/include/__memory/compressed_pair.h
+++ b/libcxx/include/__memory/compressed_pair.h
@@ -80,21 +80,45 @@ class __compressed_pair_padding {
 template <class _ToPad>
 class __compressed_pair_padding<_ToPad, true> {};
 
-#  define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2)                                                  \
-    _LIBCPP_NO_UNIQUE_ADDRESS __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>))) T1 Initializer1;    \
-    _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _);          \
-    _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2;                                                                         \
-    _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _)
-
-#  define _LIBCPP_COMPRESSED_TRIPLE(T1, Initializer1, T2, Initializer2, T3, Initializer3)                              \
-    _LIBCPP_NO_UNIQUE_ADDRESS                                                                                          \
-    __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>),                                                \
-                   __aligned__(::std::__compressed_pair_alignment<T3>))) T1 Initializer1;                              \
-    _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _);          \
-    _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2;                                                                         \
-    _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _);          \
-    _LIBCPP_NO_UNIQUE_ADDRESS T3 Initializer3;                                                                         \
-    _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T3> _LIBCPP_CONCAT3(__padding3_, __LINE__, _)
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef _LIBCPP_COMPILER_GCC
+#    define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2)                                                \
+      _LIBCPP_NO_UNIQUE_ADDRESS __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>))) T1 Initializer1;  \
+      _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _);        \
+      _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2;                                                                       \
+      _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _)
+
+#    define _LIBCPP_COMPRESSED_TRIPLE(T1, Initializer1, T2, Initializer2, T3, Initializer3)                            \
+      _LIBCPP_NO_UNIQUE_ADDRESS                                                                                        \
+      __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>),                                              \
+                     __aligned__(::std::__compressed_pair_alignment<T3>))) T1 Initializer1;                            \
+      _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _);        \
+      _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2;                                                                       \
+      _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _);        \
+      _LIBCPP_NO_UNIQUE_ADDRESS T3 Initializer3;                                                                       \
+      _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T3> _LIBCPP_CONCAT3(__padding3_, __LINE__, _)
+#  else
+#    define _LIBCPP_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2)                                                \
+      struct {                                                                                                         \
+        _LIBCPP_NO_UNIQUE_ADDRESS                                                                                      \
+        __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>))) T1 Initializer1;                          \
+        _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _);      \
+        _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2;                                                                     \
+        _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _);      \
+      }
+
+#    define _LIBCPP_COMPRESSED_TRIPLE(T1, Initializer1, T2, Initializer2, T3, Initializer3)                            \
+      struct {                                                                                                         \
+        _LIBCPP_NO_UNIQUE_ADDRESS                                                                                      \
+        __attribute__((__aligned__(::std::__compressed_pair_alignment<T2>),                                            \
+                       __aligned__(::std::__compressed_pair_alignment<T3>))) T1 Initializer1;                          \
+        _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T1> _LIBCPP_CONCAT3(__padding1_, __LINE__, _);      \
+        _LIBCPP_NO_UNIQUE_ADDRESS T2 Initializer2;                                                                     \
+        _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T2> _LIBCPP_CONCAT3(__padding2_, __LINE__, _);      \
+        _LIBCPP_NO_UNIQUE_ADDRESS T3 Initializer3;                                                                     \
+        _LIBCPP_NO_UNIQUE_ADDRESS ::std::__compressed_pair_padding<T3> _LIBCPP_CONCAT3(__padding3_, __LINE__, _);      \
+      }
+#  endif
 
 #else
 #  define _LIBCPP_COMPRESSED_PAIR(T1, Name1, T2, Name2)                                                                \
diff --git a/libcxx/include/__ranges/iota_view.h b/libcxx/include/__ranges/iota_view.h
index 4b84585258b91..32ff3408e54ea 100644
--- a/libcxx/include/__ranges/iota_view.h
+++ b/libcxx/include/__ranges/iota_view.h
@@ -30,6 +30,7 @@
 #include <__ranges/movable_box.h>
 #include <__ranges/view_interface.h>
 #include <__type_traits/conditional.h>
+#include <__type_traits/decay.h>
 #include <__type_traits/is_nothrow_constructible.h>
 #include <__type_traits/make_unsigned.h>
 #include <__type_traits/type_identity.h>
@@ -374,10 +375,10 @@ namespace views {
 namespace __iota {
 struct __fn {
   template <class _Start>
+    requires(requires(_Start __s) { ranges::iota_view<decay_t<_Start>>(std::forward<_Start>(__s)); })
   _LIBCPP_HIDE_FROM_ABI constexpr auto operator()(_Start&& __start) const
-      noexcept(noexcept(ranges::iota_view(std::forward<_Start>(__start))))
-          -> decltype(ranges::iota_view(std::forward<_Start>(__start))) {
-    return ranges::iota_view(std::forward<_Start>(__start));
+      noexcept(noexcept(ranges::iota_view<decay_t<_Start>>(std::forward<_Start>(__start)))) {
+    return ranges::iota_view<decay_t<_Start>>(std::forward<_Start>(__start));
   }
 
   template <class _Start, class _BoundSentinel>
diff --git a/libcxx/include/__tree b/libcxx/include/__tree
index 6ca1a623536f2..0f3640ef6a834 100644
--- a/libcxx/include/__tree
+++ b/libcxx/include/__tree
@@ -13,9 +13,7 @@
 #include <__algorithm/min.h>
 #include <__assert>
 #include <__config>
-#include <__fwd/map.h>
 #include <__fwd/pair.h>
-#include <__fwd/set.h>
 #include <__iterator/distance.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/next.h>
@@ -29,15 +27,13 @@
 #include <__type_traits/copy_cvref.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/invoke.h>
-#include <__type_traits/is_const.h>
 #include <__type_traits/is_constructible.h>
 #include <__type_traits/is_nothrow_assignable.h>
 #include <__type_traits/is_nothrow_constructible.h>
 #include <__type_traits/is_same.h>
+#include <__type_traits/is_specialization.h>
 #include <__type_traits/is_swappable.h>
 #include <__type_traits/remove_const.h>
-#include <__type_traits/remove_const_ref.h>
-#include <__type_traits/remove_cvref.h>
 #include <__utility/forward.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
@@ -51,14 +47,31 @@
 _LIBCPP_PUSH_MACROS
 #include <__undef_macros>
 
-_LIBCPP_BEGIN_NAMESPACE_STD
+_LIBCPP_DIAGNOSTIC_PUSH
+// GCC complains about the backslashes at the end, see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121528
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wcomment")
+// __tree is a red-black-tree implementation used for the associative containers (i.e. (multi)map/set). It stores
+// - (1) a pointer to the node with the smallest (i.e. leftmost) element, namely __begin_node_
+// - (2) the number of nodes in the tree, namely __size_
+// - (3) a pointer to the root of the tree, namely __end_node_
+//
+// Storing (1) and (2) is required to allow for constant time lookups. A tree looks like this in memory:
+//
+//      __end_node_
+//           |
+//          root
+//         /    \
+//       l1       r1
+//      /  \     /  \
+//    ...  ... ...  ...
+//
+// All nodes except __end_node_ have a __left_ and __right_ pointer as well as a __parent_ pointer.
+// __end_node_ only contains a __left_ pointer, which points to the root of the tree.
+// This layout allows for iteration through the tree without a need for special handling of the end node. See
+// __tree_next_iter and __tree_prev_iter for more details.
+_LIBCPP_DIAGNOSTIC_POP
 
-template <class _Tp, class _Compare, class _Allocator>
-class __tree;
-template <class _Tp, class _NodePtr, class _DiffType>
-class __tree_iterator;
-template <class _Tp, class _ConstNodePtr, class _DiffType>
-class __tree_const_iterator;
+_LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Pointer>
 class __tree_end_node;
@@ -70,13 +83,6 @@ class __tree_node;
 template <class _Key, class _Value>
 struct __value_type;
 
-template <class _Allocator>
-class __map_node_destructor;
-template <class _TreeIterator>
-class __map_iterator;
-template <class _TreeIterator>
-class __map_const_iterator;
-
 /*
 
 _NodePtr algorithms
@@ -185,6 +191,11 @@ _LIBCPP_HIDE_FROM_ABI _NodePtr __tree_next(_NodePtr __x) _NOEXCEPT {
   return __x->__parent_unsafe();
 }
 
+// __tree_next_iter and __tree_prev_iter implement iteration through the tree. The order is as follows:
+// left sub-tree -> node -> right sub-tree. When the right-most node of a sub-tree is reached, we walk up the tree until
+// we find a node where we were in the left sub-tree. We are _always_ in a left sub-tree, since the __end_node_ points
+// to the actual root of the tree through a __left_ pointer. Incrementing the end() pointer is UB, so we can assume that
+// never happens.
 template <class _EndNodePtr, class _NodePtr>
 inline _LIBCPP_HIDE_FROM_ABI _EndNodePtr __tree_next_iter(_NodePtr __x) _NOEXCEPT {
   _LIBCPP_ASSERT_INTERNAL(__x != nullptr, "node shouldn't be null");
@@ -494,16 +505,7 @@ _LIBCPP_HIDE_FROM_ABI void __tree_remove(_NodePtr __root, _NodePtr __z) _NOEXCEP
 // node traits
 
 template <class _Tp>
-struct __is_tree_value_type_imp : false_type {};
-
-template <class _Key, class _Value>
-struct __is_tree_value_type_imp<__value_type<_Key, _Value> > : true_type {};
-
-template <class... _Args>
-struct __is_tree_value_type : false_type {};
-
-template <class _One>
-struct __is_tree_value_type<_One> : __is_tree_value_type_imp<__remove_cvref_t<_One> > {};
+inline const bool __is_tree_value_type_v = __is_specialization_v<_Tp, __value_type>;
 
 template <class _Tp>
 struct __get_tree_key_type {
@@ -580,8 +582,10 @@ class _LIBCPP_STANDALONE_DEBUG __tree_node : public __tree_node_base<_VoidPtr> {
 public:
   using __node_value_type _LIBCPP_NODEBUG = __get_node_value_type_t<_Tp>;
 
+private:
   __node_value_type __value_;
 
+public:
   _LIBCPP_HIDE_FROM_ABI __node_value_type& __get_value() { return __value_; }
 
   ~__tree_node()                             = delete;
@@ -612,7 +616,7 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
     if (__value_constructed)
-      __alloc_traits::destroy(__na_, std::addressof(__p->__value_));
+      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value()));
     if (__p)
       __alloc_traits::deallocate(__na_, __p, 1);
   }
@@ -648,8 +652,10 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI __tree_iterator() _NOEXCEPT : __ptr_(nullptr) {}
 
-  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__value_; }
-  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__get_np()->__value_); }
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    return pointer_traits<pointer>::pointer_to(__get_np()->__get_value());
+  }
 
   _LIBCPP_HIDE_FROM_ABI __tree_iterator& operator++() {
     __ptr_ = std::__tree_next_iter<__end_node_pointer>(static_cast<__node_base_pointer>(__ptr_));
@@ -686,16 +692,6 @@ private:
   friend class __tree;
   template <class, class, class>
   friend class __tree_const_iterator;
-  template <class>
-  friend class __map_iterator;
-  template <class, class, class, class>
-  friend class map;
-  template <class, class, class, class>
-  friend class multimap;
-  template <class, class, class>
-  friend class set;
-  template <class, class, class>
-  friend class multiset;
 };
 
 template <class _Tp, class _NodePtr, class _DiffType>
@@ -709,22 +705,21 @@ class __tree_const_iterator {
   __end_node_pointer __ptr_;
 
 public:
-  using iterator_category = bidirectional_iterator_tag;
-  using value_type        = __get_node_value_type_t<_Tp>;
-  using difference_type   = _DiffType;
-  using reference         = const value_type&;
-  using pointer           = __rebind_pointer_t<_NodePtr, const value_type>;
+  using iterator_category                    = bidirectional_iterator_tag;
+  using value_type                           = __get_node_value_type_t<_Tp>;
+  using difference_type                      = _DiffType;
+  using reference                            = const value_type&;
+  using pointer                              = __rebind_pointer_t<_NodePtr, const value_type>;
+  using __non_const_iterator _LIBCPP_NODEBUG = __tree_iterator<_Tp, __node_pointer, difference_type>;
 
   _LIBCPP_HIDE_FROM_ABI __tree_const_iterator() _NOEXCEPT : __ptr_(nullptr) {}
 
-private:
-  typedef __tree_iterator<_Tp, __node_pointer, difference_type> __non_const_iterator;
-
-public:
   _LIBCPP_HIDE_FROM_ABI __tree_const_iterator(__non_const_iterator __p) _NOEXCEPT : __ptr_(__p.__ptr_) {}
 
-  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__value_; }
-  _LIBCPP_HIDE_FROM_ABI pointer operator->() const { return pointer_traits<pointer>::pointer_to(__get_np()->__value_); }
+  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __get_np()->__get_value(); }
+  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+    return pointer_traits<pointer>::pointer_to(__get_np()->__get_value());
+  }
 
   _LIBCPP_HIDE_FROM_ABI __tree_const_iterator& operator++() {
     __ptr_ = std::__tree_next_iter<__end_node_pointer>(static_cast<__node_base_pointer>(__ptr_));
@@ -762,16 +757,6 @@ private:
 
   template <class, class, class>
   friend class __tree;
-  template <class, class, class, class>
-  friend class map;
-  template <class, class, class, class>
-  friend class multimap;
-  template <class, class, class>
-  friend class set;
-  template <class, class, class>
-  friend class multiset;
-  template <class>
-  friend class __map_const_iterator;
 };
 
 template <class _Tp, class _Compare>
@@ -999,27 +984,56 @@ public:
     return __emplace_hint_unique_key_args(__p, __x.first, std::forward<_Pp>(__x)).first;
   }
 
-  template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type<_ValueT>::value, int> = 0>
+  template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI void
   __insert_unique_from_orphaned_node(const_iterator __p, __get_node_value_type_t<_Tp>&& __value) {
     __emplace_hint_unique(__p, const_cast<key_type&&>(__value.first), std::move(__value.second));
   }
 
-  template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type<_ValueT>::value, int> = 0>
+  template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI void __insert_unique_from_orphaned_node(const_iterator __p, _Tp&& __value) {
     __emplace_hint_unique(__p, std::move(__value));
   }
 
-  template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type<_ValueT>::value, int> = 0>
+  template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(const_iterator __p, value_type&& __value) {
     __emplace_hint_multi(__p, const_cast<key_type&&>(__value.first), std::move(__value.second));
   }
 
-  template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type<_ValueT>::value, int> = 0>
+  template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(const_iterator __p, _Tp&& __value) {
     __emplace_hint_multi(__p, std::move(__value));
   }
 
+  template <class _InIter, class _Sent>
+  _LIBCPP_HIDE_FROM_ABI void __insert_range_multi(_InIter __first, _Sent __last) {
+    if (__first == __last)
+      return;
+
+    if (__root() == nullptr) { // Make sure we always have a root node
+      __insert_node_at(
+          __end_node(), __end_node()->__left_, static_cast<__node_base_pointer>(__construct_node(*__first).release()));
+      ++__first;
+    }
+
+    auto __max_node = static_cast<__node_pointer>(std::__tree_max(static_cast<__node_base_pointer>(__root())));
+
+    for (; __first != __last; ++__first) {
+      __node_holder __nd = __construct_node(*__first);
+      // Always check the max node first. This optimizes for sorted ranges inserted at the end.
+      if (!value_comp()(__nd->__get_value(), __max_node->__get_value())) { // __node >= __max_val
+        __insert_node_at(static_cast<__end_node_pointer>(__max_node),
+                         __max_node->__right_,
+                         static_cast<__node_base_pointer>(__nd.get()));
+        __max_node = __nd.release();
+      } else {
+        __end_node_pointer __parent;
+        __node_base_pointer& __child = __find_leaf_high(__parent, __nd->__get_value());
+        __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd.release()));
+      }
+    }
+  }
+
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __node_assign_unique(const value_type& __v, __node_pointer __dest);
 
   _LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(__node_pointer __nd);
@@ -1059,9 +1073,22 @@ public:
   __insert_node_at(__end_node_pointer __parent, __node_base_pointer& __child, __node_base_pointer __new_node) _NOEXCEPT;
 
   template <class _Key>
-  _LIBCPP_HIDE_FROM_ABI iterator find(const _Key& __v);
+  _LIBCPP_HIDE_FROM_ABI iterator find(const _Key& __key) {
+    __end_node_pointer __parent;
+    __node_base_pointer __match = __find_equal(__parent, __key);
+    if (__match == nullptr)
+      return end();
+    return iterator(static_cast<__node_pointer>(__match));
+  }
+
   template <class _Key>
-  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Key& __v) const;
+  _LIBCPP_HIDE_FROM_ABI const_iterator find(const _Key& __key) const {
+    __end_node_pointer __parent;
+    __node_base_pointer __match = __find_equal(__parent, __key);
+    if (__match == nullptr)
+      return end();
+    return const_iterator(static_cast<__node_pointer>(__match));
+  }
 
   template <class _Key>
   _LIBCPP_HIDE_FROM_ABI size_type __count_unique(const _Key& __k) const;
@@ -1162,7 +1189,7 @@ private:
   }
   _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__tree&, false_type) _NOEXCEPT {}
 
-  template <class _From, class _ValueT = _Tp, __enable_if_t<__is_tree_value_type<_ValueT>::value, int> = 0>
+  template <class _From, class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI static void __assign_value(__get_node_value_type_t<value_type>& __lhs, _From&& __rhs) {
     using __key_type = __remove_const_t<typename value_type::first_type>;
 
@@ -1172,7 +1199,7 @@ private:
     __lhs.second                         = std::forward<_From>(__rhs).second;
   }
 
-  template <class _To, class _From, class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type<_ValueT>::value, int> = 0>
+  template <class _To, class _From, class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
   _LIBCPP_HIDE_FROM_ABI static void __assign_value(_To& __lhs, _From&& __rhs) {
     __lhs = std::forward<_From>(__rhs);
   }
@@ -1234,7 +1261,7 @@ private:
 
       auto __right = __ptr->__right_;
 
-      __node_traits::destroy(__alloc_, std::addressof(__ptr->__value_));
+      __node_traits::destroy(__alloc_, std::addressof(__ptr->__get_value()));
       __node_traits::deallocate(__alloc_, __ptr, 1);
 
       (*this)(static_cast<__node_pointer>(__right));
@@ -1253,7 +1280,7 @@ private:
     if (!__src)
       return nullptr;
 
-    __node_holder __new_node = __construct_node(__src->__value_);
+    __node_holder __new_node = __construct_node(__src->__get_value());
 
     unique_ptr<__node, __tree_deleter> __left(
         __copy_construct_tree(static_cast<__node_pointer>(__src->__left_)), __node_alloc_);
@@ -1284,7 +1311,7 @@ private:
       return nullptr;
     }
 
-    __assign_value(__dest->__value_, __src->__value_);
+    __assign_value(__dest->__get_value(), __src->__get_value());
     __dest->__is_black_ = __src->__is_black_;
 
     // If we already have a left node in the destination tree, reuse it and copy-assign recursively
@@ -1388,8 +1415,9 @@ __tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(
     if (__root())
       __root()->__parent_ = __end_node();
   }
-  __begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(static_cast<__node_base_pointer>(__end_node())));
-  __size_       = __t.size();
+  __begin_node_ =
+      __end_node()->__left_ ? static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_)) : __end_node();
+  __size_ = __t.size();
 
   return *this;
 }
@@ -1424,7 +1452,7 @@ void __tree<_Tp, _Compare, _Allocator>::__assign_multi(_InputIterator __first, _
   if (__size_ != 0) {
     _DetachedTreeCache __cache(this);
     for (; __cache.__get() && __first != __last; ++__first) {
-      __assign_value(__cache.__get()->__value_, *__first);
+      __assign_value(__cache.__get()->__get_value(), *__first);
       __node_insert_multi(__cache.__get());
       __cache.__advance();
     }
@@ -1516,13 +1544,13 @@ void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, false_type) {
     if (__size_ != 0) {
       _DetachedTreeCache __cache(this);
       while (__cache.__get() != nullptr && __t.__size_ != 0) {
-        __assign_value(__cache.__get()->__value_, std::move(__t.remove(__t.begin())->__value_));
+        __assign_value(__cache.__get()->__get_value(), std::move(__t.remove(__t.begin())->__get_value()));
         __node_insert_multi(__cache.__get());
         __cache.__advance();
       }
     }
     while (__t.__size_ != 0) {
-      __insert_multi_from_orphaned_node(__e, std::move(__t.remove(__t.begin())->__value_));
+      __insert_multi_from_orphaned_node(__e, std::move(__t.remove(__t.begin())->__get_value()));
     }
   }
 }
@@ -1590,7 +1618,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_leaf_low(__end_node_pointer& __parent,
   __node_pointer __nd = __root();
   if (__nd != nullptr) {
     while (true) {
-      if (value_comp()(__nd->__value_, __v)) {
+      if (value_comp()(__nd->__get_value(), __v)) {
         if (__nd->__right_ != nullptr)
           __nd = static_cast<__node_pointer>(__nd->__right_);
         else {
@@ -1620,7 +1648,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_leaf_high(__end_node_pointer& __parent
   __node_pointer __nd = __root();
   if (__nd != nullptr) {
     while (true) {
-      if (value_comp()(__v, __nd->__value_)) {
+      if (value_comp()(__v, __nd->__get_value())) {
         if (__nd->__left_ != nullptr)
           __nd = static_cast<__node_pointer>(__nd->__left_);
         else {
@@ -1683,7 +1711,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_equal(__end_node_pointer& __parent, co
   __node_base_pointer* __nd_ptr = __root_ptr();
   if (__nd != nullptr) {
     while (true) {
-      if (value_comp()(__v, __nd->__value_)) {
+      if (value_comp()(__v, __nd->__get_value())) {
         if (__nd->__left_ != nullptr) {
           __nd_ptr = std::addressof(__nd->__left_);
           __nd     = static_cast<__node_pointer>(__nd->__left_);
@@ -1691,7 +1719,7 @@ __tree<_Tp, _Compare, _Allocator>::__find_equal(__end_node_pointer& __parent, co
           __parent = static_cast<__end_node_pointer>(__nd);
           return __parent->__left_;
         }
-      } else if (value_comp()(__nd->__value_, __v)) {
+      } else if (value_comp()(__nd->__get_value(), __v)) {
         if (__nd->__right_ != nullptr) {
           __nd_ptr = std::addressof(__nd->__right_);
           __nd     = static_cast<__node_pointer>(__nd->__right_);
@@ -1815,7 +1843,7 @@ typename __tree<_Tp, _Compare, _Allocator>::__node_holder
 __tree<_Tp, _Compare, _Allocator>::__construct_node(_Args&&... __args) {
   __node_allocator& __na = __node_alloc();
   __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
-  __node_traits::construct(__na, std::addressof(__h->__value_), std::forward<_Args>(__args)...);
+  __node_traits::construct(__na, std::addressof(__h->__get_value()), std::forward<_Args>(__args)...);
   __h.get_deleter().__value_constructed = true;
   return __h;
 }
@@ -1826,7 +1854,7 @@ pair<typename __tree<_Tp, _Compare, _Allocator>::iterator, bool>
 __tree<_Tp, _Compare, _Allocator>::__emplace_unique_impl(_Args&&... __args) {
   __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
   __end_node_pointer __parent;
-  __node_base_pointer& __child = __find_equal(__parent, __h->__value_);
+  __node_base_pointer& __child = __find_equal(__parent, __h->__get_value());
   __node_pointer __r           = static_cast<__node_pointer>(__child);
   bool __inserted              = false;
   if (__child == nullptr) {
@@ -1844,7 +1872,7 @@ __tree<_Tp, _Compare, _Allocator>::__emplace_hint_unique_impl(const_iterator __p
   __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
   __end_node_pointer __parent;
   __node_base_pointer __dummy;
-  __node_base_pointer& __child = __find_equal(__p, __parent, __dummy, __h->__value_);
+  __node_base_pointer& __child = __find_equal(__p, __parent, __dummy, __h->__get_value());
   __node_pointer __r           = static_cast<__node_pointer>(__child);
   if (__child == nullptr) {
     __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
@@ -1859,7 +1887,7 @@ typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__emplace_multi(_Args&&... __args) {
   __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
   __end_node_pointer __parent;
-  __node_base_pointer& __child = __find_leaf_high(__parent, __h->__value_);
+  __node_base_pointer& __child = __find_leaf_high(__parent, __h->__get_value());
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
   return iterator(static_cast<__node_pointer>(__h.release()));
 }
@@ -1870,7 +1898,7 @@ typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__emplace_hint_multi(const_iterator __p, _Args&&... __args) {
   __node_holder __h = __construct_node(std::forward<_Args>(__args)...);
   __end_node_pointer __parent;
-  __node_base_pointer& __child = __find_leaf(__p, __parent, __h->__value_);
+  __node_base_pointer& __child = __find_leaf(__p, __parent, __h->__get_value());
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
   return iterator(static_cast<__node_pointer>(__h.release()));
 }
@@ -1883,7 +1911,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_assign_unique(const value_type& __v, _
   __node_pointer __r           = static_cast<__node_pointer>(__child);
   bool __inserted              = false;
   if (__child == nullptr) {
-    __assign_value(__nd->__value_, __v);
+    __assign_value(__nd->__get_value(), __v);
     __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
     __r        = __nd;
     __inserted = true;
@@ -1895,7 +1923,7 @@ template <class _Tp, class _Compare, class _Allocator>
 typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__node_insert_multi(__node_pointer __nd) {
   __end_node_pointer __parent;
-  __node_base_pointer& __child = __find_leaf_high(__parent, __nd->__value_);
+  __node_base_pointer& __child = __find_leaf_high(__parent, __nd->__get_value());
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
   return iterator(__nd);
 }
@@ -1904,7 +1932,7 @@ template <class _Tp, class _Compare, class _Allocator>
 typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__node_insert_multi(const_iterator __p, __node_pointer __nd) {
   __end_node_pointer __parent;
-  __node_base_pointer& __child = __find_leaf(__p, __parent, __nd->__value_);
+  __node_base_pointer& __child = __find_leaf(__p, __parent, __nd->__get_value());
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd));
   return iterator(__nd);
 }
@@ -1931,7 +1959,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_unique(_NodeHandle&& __n
 
   __node_pointer __ptr = __nh.__ptr_;
   __end_node_pointer __parent;
-  __node_base_pointer& __child = __find_equal(__parent, __ptr->__value_);
+  __node_base_pointer& __child = __find_equal(__parent, __ptr->__get_value());
   if (__child != nullptr)
     return _InsertReturnType{iterator(static_cast<__node_pointer>(__child)), false, std::move(__nh)};
 
@@ -1950,7 +1978,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_unique(const_iterator __
   __node_pointer __ptr = __nh.__ptr_;
   __end_node_pointer __parent;
   __node_base_pointer __dummy;
-  __node_base_pointer& __child = __find_equal(__hint, __parent, __dummy, __ptr->__value_);
+  __node_base_pointer& __child = __find_equal(__hint, __parent, __dummy, __ptr->__get_value());
   __node_pointer __r           = static_cast<__node_pointer>(__child);
   if (__child == nullptr) {
     __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
@@ -1985,7 +2013,7 @@ _LIBCPP_HIDE_FROM_ABI void __tree<_Tp, _Compare, _Allocator>::__node_handle_merg
   for (typename _Tree::iterator __i = __source.begin(); __i != __source.end();) {
     __node_pointer __src_ptr = __i.__get_np();
     __end_node_pointer __parent;
-    __node_base_pointer& __child = __find_equal(__parent, __src_ptr->__value_);
+    __node_base_pointer& __child = __find_equal(__parent, __src_ptr->__get_value());
     ++__i;
     if (__child != nullptr)
       continue;
@@ -2002,7 +2030,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_multi(_NodeHandle&& __nh
     return end();
   __node_pointer __ptr = __nh.__ptr_;
   __end_node_pointer __parent;
-  __node_base_pointer& __child = __find_leaf_high(__parent, __ptr->__value_);
+  __node_base_pointer& __child = __find_leaf_high(__parent, __ptr->__get_value());
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
   __nh.__release_ptr();
   return iterator(__ptr);
@@ -2017,7 +2045,7 @@ __tree<_Tp, _Compare, _Allocator>::__node_handle_insert_multi(const_iterator __h
 
   __node_pointer __ptr = __nh.__ptr_;
   __end_node_pointer __parent;
-  __node_base_pointer& __child = __find_leaf(__hint, __parent, __ptr->__value_);
+  __node_base_pointer& __child = __find_leaf(__hint, __parent, __ptr->__get_value());
   __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__ptr));
   __nh.__release_ptr();
   return iterator(__ptr);
@@ -2031,7 +2059,7 @@ _LIBCPP_HIDE_FROM_ABI void __tree<_Tp, _Compare, _Allocator>::__node_handle_merg
   for (typename _Tree::iterator __i = __source.begin(); __i != __source.end();) {
     __node_pointer __src_ptr = __i.__get_np();
     __end_node_pointer __parent;
-    __node_base_pointer& __child = __find_leaf_high(__parent, __src_ptr->__value_);
+    __node_base_pointer& __child = __find_leaf_high(__parent, __src_ptr->__get_value());
     ++__i;
     __source.__remove_node_pointer(__src_ptr);
     __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__src_ptr));
@@ -2080,34 +2108,15 @@ __tree<_Tp, _Compare, _Allocator>::__erase_multi(const _Key& __k) {
   return __r;
 }
 
-template <class _Tp, class _Compare, class _Allocator>
-template <class _Key>
-typename __tree<_Tp, _Compare, _Allocator>::iterator __tree<_Tp, _Compare, _Allocator>::find(const _Key& __v) {
-  iterator __p = __lower_bound(__v, __root(), __end_node());
-  if (__p != end() && !value_comp()(__v, *__p))
-    return __p;
-  return end();
-}
-
-template <class _Tp, class _Compare, class _Allocator>
-template <class _Key>
-typename __tree<_Tp, _Compare, _Allocator>::const_iterator
-__tree<_Tp, _Compare, _Allocator>::find(const _Key& __v) const {
-  const_iterator __p = __lower_bound(__v, __root(), __end_node());
-  if (__p != end() && !value_comp()(__v, *__p))
-    return __p;
-  return end();
-}
-
 template <class _Tp, class _Compare, class _Allocator>
 template <class _Key>
 typename __tree<_Tp, _Compare, _Allocator>::size_type
 __tree<_Tp, _Compare, _Allocator>::__count_unique(const _Key& __k) const {
   __node_pointer __rt = __root();
   while (__rt != nullptr) {
-    if (value_comp()(__k, __rt->__value_)) {
+    if (value_comp()(__k, __rt->__get_value())) {
       __rt = static_cast<__node_pointer>(__rt->__left_);
-    } else if (value_comp()(__rt->__value_, __k))
+    } else if (value_comp()(__rt->__get_value(), __k))
       __rt = static_cast<__node_pointer>(__rt->__right_);
     else
       return 1;
@@ -2122,10 +2131,10 @@ __tree<_Tp, _Compare, _Allocator>::__count_multi(const _Key& __k) const {
   __end_node_pointer __result = __end_node();
   __node_pointer __rt         = __root();
   while (__rt != nullptr) {
-    if (value_comp()(__k, __rt->__value_)) {
+    if (value_comp()(__k, __rt->__get_value())) {
       __result = static_cast<__end_node_pointer>(__rt);
       __rt     = static_cast<__node_pointer>(__rt->__left_);
-    } else if (value_comp()(__rt->__value_, __k))
+    } else if (value_comp()(__rt->__get_value(), __k))
       __rt = static_cast<__node_pointer>(__rt->__right_);
     else
       return std::distance(
@@ -2140,7 +2149,7 @@ template <class _Key>
 typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__lower_bound(const _Key& __v, __node_pointer __root, __end_node_pointer __result) {
   while (__root != nullptr) {
-    if (!value_comp()(__root->__value_, __v)) {
+    if (!value_comp()(__root->__get_value(), __v)) {
       __result = static_cast<__end_node_pointer>(__root);
       __root   = static_cast<__node_pointer>(__root->__left_);
     } else
@@ -2154,7 +2163,7 @@ template <class _Key>
 typename __tree<_Tp, _Compare, _Allocator>::const_iterator __tree<_Tp, _Compare, _Allocator>::__lower_bound(
     const _Key& __v, __node_pointer __root, __end_node_pointer __result) const {
   while (__root != nullptr) {
-    if (!value_comp()(__root->__value_, __v)) {
+    if (!value_comp()(__root->__get_value(), __v)) {
       __result = static_cast<__end_node_pointer>(__root);
       __root   = static_cast<__node_pointer>(__root->__left_);
     } else
@@ -2168,7 +2177,7 @@ template <class _Key>
 typename __tree<_Tp, _Compare, _Allocator>::iterator
 __tree<_Tp, _Compare, _Allocator>::__upper_bound(const _Key& __v, __node_pointer __root, __end_node_pointer __result) {
   while (__root != nullptr) {
-    if (value_comp()(__v, __root->__value_)) {
+    if (value_comp()(__v, __root->__get_value())) {
       __result = static_cast<__end_node_pointer>(__root);
       __root   = static_cast<__node_pointer>(__root->__left_);
     } else
@@ -2182,7 +2191,7 @@ template <class _Key>
 typename __tree<_Tp, _Compare, _Allocator>::const_iterator __tree<_Tp, _Compare, _Allocator>::__upper_bound(
     const _Key& __v, __node_pointer __root, __end_node_pointer __result) const {
   while (__root != nullptr) {
-    if (value_comp()(__v, __root->__value_)) {
+    if (value_comp()(__v, __root->__get_value())) {
       __result = static_cast<__end_node_pointer>(__root);
       __root   = static_cast<__node_pointer>(__root->__left_);
     } else
@@ -2199,10 +2208,10 @@ __tree<_Tp, _Compare, _Allocator>::__equal_range_unique(const _Key& __k) {
   __end_node_pointer __result = __end_node();
   __node_pointer __rt         = __root();
   while (__rt != nullptr) {
-    if (value_comp()(__k, __rt->__value_)) {
+    if (value_comp()(__k, __rt->__get_value())) {
       __result = static_cast<__end_node_pointer>(__rt);
       __rt     = static_cast<__node_pointer>(__rt->__left_);
-    } else if (value_comp()(__rt->__value_, __k))
+    } else if (value_comp()(__rt->__get_value(), __k))
       __rt = static_cast<__node_pointer>(__rt->__right_);
     else
       return _Pp(iterator(__rt),
@@ -2221,10 +2230,10 @@ __tree<_Tp, _Compare, _Allocator>::__equal_range_unique(const _Key& __k) const {
   __end_node_pointer __result = __end_node();
   __node_pointer __rt         = __root();
   while (__rt != nullptr) {
-    if (value_comp()(__k, __rt->__value_)) {
+    if (value_comp()(__k, __rt->__get_value())) {
       __result = static_cast<__end_node_pointer>(__rt);
       __rt     = static_cast<__node_pointer>(__rt->__left_);
-    } else if (value_comp()(__rt->__value_, __k))
+    } else if (value_comp()(__rt->__get_value(), __k))
       __rt = static_cast<__node_pointer>(__rt->__right_);
     else
       return _Pp(
@@ -2243,10 +2252,10 @@ __tree<_Tp, _Compare, _Allocator>::__equal_range_multi(const _Key& __k) {
   __end_node_pointer __result = __end_node();
   __node_pointer __rt     = __root();
   while (__rt != nullptr) {
-    if (value_comp()(__k, __rt->__value_)) {
+    if (value_comp()(__k, __rt->__get_value())) {
       __result = static_cast<__end_node_pointer>(__rt);
       __rt     = static_cast<__node_pointer>(__rt->__left_);
-    } else if (value_comp()(__rt->__value_, __k))
+    } else if (value_comp()(__rt->__get_value(), __k))
       __rt = static_cast<__node_pointer>(__rt->__right_);
     else
       return _Pp(__lower_bound(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__end_node_pointer>(__rt)),
@@ -2264,10 +2273,10 @@ __tree<_Tp, _Compare, _Allocator>::__equal_range_multi(const _Key& __k) const {
   __end_node_pointer __result = __end_node();
   __node_pointer __rt     = __root();
   while (__rt != nullptr) {
-    if (value_comp()(__k, __rt->__value_)) {
+    if (value_comp()(__k, __rt->__get_value())) {
       __result = static_cast<__end_node_pointer>(__rt);
       __rt     = static_cast<__node_pointer>(__rt->__left_);
-    } else if (value_comp()(__rt->__value_, __k))
+    } else if (value_comp()(__rt->__get_value(), __k))
       __rt = static_cast<__node_pointer>(__rt->__right_);
     else
       return _Pp(__lower_bound(__k, static_cast<__node_pointer>(__rt->__left_), static_cast<__end_node_pointer>(__rt)),
diff --git a/libcxx/include/__type_traits/invoke.h b/libcxx/include/__type_traits/invoke.h
index 5ff2efbe5faaf..24702b6d5b52b 100644
--- a/libcxx/include/__type_traits/invoke.h
+++ b/libcxx/include/__type_traits/invoke.h
@@ -67,20 +67,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__builtin_invoke)
 
-template <class... _Args>
-using __invoke_result_t _LIBCPP_NODEBUG = decltype(__builtin_invoke(std::declval<_Args>()...));
-
 template <class, class... _Args>
 struct __invoke_result_impl {};
 
 template <class... _Args>
-struct __invoke_result_impl<__void_t<__invoke_result_t<_Args...> >, _Args...> {
-  using type _LIBCPP_NODEBUG = __invoke_result_t<_Args...>;
+struct __invoke_result_impl<__void_t<decltype(__builtin_invoke(std::declval<_Args>()...))>, _Args...> {
+  using type _LIBCPP_NODEBUG = decltype(__builtin_invoke(std::declval<_Args>()...));
 };
 
 template <class... _Args>
 using __invoke_result _LIBCPP_NODEBUG = __invoke_result_impl<void, _Args...>;
 
+template <class... _Args>
+using __invoke_result_t _LIBCPP_NODEBUG = typename __invoke_result<_Args...>::type;
+
 template <class... _Args>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __invoke_result_t<_Args...> __invoke(_Args&&... __args)
     _NOEXCEPT_(noexcept(__builtin_invoke(std::forward<_Args>(__args)...))) {
@@ -112,8 +112,10 @@ inline const bool __is_invocable_r_v = __is_invocable_r_impl<_Ret, __is_invocabl
 template <bool __is_invocable, class... _Args>
 inline const bool __is_nothrow_invocable_impl = false;
 
+#  ifndef _LIBCPP_CXX03_LANG
 template <class... _Args>
 inline const bool __is_nothrow_invocable_impl<true, _Args...> = noexcept(__builtin_invoke(std::declval<_Args>()...));
+#  endif
 
 template <class... _Args>
 inline const bool __is_nothrow_invocable_v = __is_nothrow_invocable_impl<__is_invocable_v<_Args...>, _Args...>;
diff --git a/libcxx/include/__type_traits/is_specialization.h b/libcxx/include/__type_traits/is_specialization.h
index 9b75636b1a511..f14ab93c3c7dc 100644
--- a/libcxx/include/__type_traits/is_specialization.h
+++ b/libcxx/include/__type_traits/is_specialization.h
@@ -30,15 +30,11 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if _LIBCPP_STD_VER >= 17
-
 template <class _Tp, template <class...> class _Template>
-inline constexpr bool __is_specialization_v = false; // true if and only if _Tp is a specialization of _Template
+inline const bool __is_specialization_v = false; // true if and only if _Tp is a specialization of _Template
 
 template <template <class...> class _Template, class... _Args>
-inline constexpr bool __is_specialization_v<_Template<_Args...>, _Template> = true;
-
-#endif // _LIBCPP_STD_VER >= 17
+inline const bool __is_specialization_v<_Template<_Args...>, _Template> = true;
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/iterator b/libcxx/include/iterator
index d25fdfd2a8b33..fc8bdc5e6bcf6 100644
--- a/libcxx/include/iterator
+++ b/libcxx/include/iterator
@@ -737,6 +737,16 @@ template <class E> constexpr const E* data(initializer_list<E> il) noexcept;
 #  include <compare>
 #  include <concepts>
 
+// [range.access.general]
+#  if _LIBCPP_STD_VER >= 20
+#    include <__ranges/access.h>
+#    include <__ranges/data.h>
+#    include <__ranges/empty.h>
+#    include <__ranges/rbegin.h>
+#    include <__ranges/rend.h>
+#    include <__ranges/size.h>
+#  endif
+
 #  if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #    pragma GCC system_header
 #  endif
diff --git a/libcxx/include/map b/libcxx/include/map
index 0a43bd09a0b16..9bd2282e77a3c 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -582,7 +582,6 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
 #  include <__functional/binary_function.h>
 #  include <__functional/is_transparent.h>
 #  include <__functional/operations.h>
-#  include <__fwd/map.h>
 #  include <__iterator/erase_if_container.h>
 #  include <__iterator/iterator_traits.h>
 #  include <__iterator/ranges_iterator_traits.h>
@@ -594,6 +593,7 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
 #  include <__memory/unique_ptr.h>
 #  include <__memory_resource/polymorphic_allocator.h>
 #  include <__node_handle>
+#  include <__ranges/access.h>
 #  include <__ranges/concepts.h>
 #  include <__ranges/container_compatible_range.h>
 #  include <__ranges/from_range.h>
@@ -692,12 +692,12 @@ public:
 #  if _LIBCPP_STD_VER >= 14
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _K2& __x, const _CP& __y) const {
-    return __comp_(__x, __y.__get_value().first);
+    return __comp_(__x, __y.first);
   }
 
   template <typename _K2>
   _LIBCPP_HIDE_FROM_ABI bool operator()(const _CP& __x, const _K2& __y) const {
-    return __comp_(__x.__get_value().first, __y);
+    return __comp_(__x.first, __y);
   }
 #  endif
 };
@@ -742,9 +742,9 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI void operator()(pointer __p) _NOEXCEPT {
     if (__second_constructed)
-      __alloc_traits::destroy(__na_, std::addressof(__p->__value_.second));
+      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().second));
     if (__first_constructed)
-      __alloc_traits::destroy(__na_, std::addressof(__p->__value_.first));
+      __alloc_traits::destroy(__na_, std::addressof(__p->__get_value().first));
     if (__p)
       __alloc_traits::deallocate(__na_, __p, 1);
   }
@@ -861,7 +861,10 @@ public:
   friend class __tree_const_iterator;
 };
 
-template <class _Key, class _Tp, class _Compare, class _Allocator>
+template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > >
+class multimap;
+
+template <class _Key, class _Tp, class _Compare = less<_Key>, class _Allocator = allocator<pair<const _Key, _Tp> > >
 class map {
 public:
   // types:
@@ -976,11 +979,11 @@ public:
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-  _LIBCPP_HIDE_FROM_ABI map(map&& __m) noexcept(is_nothrow_move_constructible<__base>::value) = default;
+  _LIBCPP_HIDE_FROM_ABI map(map&& __m) = default;
 
   _LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a);
 
-  _LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) noexcept(is_nothrow_move_assignable<__base>::value) = default;
+  _LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) = default;
 
   _LIBCPP_HIDE_FROM_ABI map(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : __tree_(__vc(__comp)) {
@@ -1387,7 +1390,8 @@ map<_Key, _Tp, _Compare, _Allocator>::map(map&& __m, const allocator_type& __a)
   if (__a != __m.get_allocator()) {
     const_iterator __e = cend();
     while (!__m.empty()) {
-      __tree_.__insert_unique_from_orphaned_node(__e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__value_));
+      __tree_.__insert_unique_from_orphaned_node(
+          __e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__get_value()));
     }
   }
 }
@@ -1417,9 +1421,9 @@ typename map<_Key, _Tp, _Compare, _Allocator>::__node_holder
 map<_Key, _Tp, _Compare, _Allocator>::__construct_node_with_key(const key_type& __k) {
   __node_allocator& __na = __tree_.__node_alloc();
   __node_holder __h(__node_traits::allocate(__na, 1), _Dp(__na));
-  __node_traits::construct(__na, std::addressof(__h->__value_.first), __k);
+  __node_traits::construct(__na, std::addressof(__h->__get_value().first), __k);
   __h.get_deleter().__first_constructed = true;
-  __node_traits::construct(__na, std::addressof(__h->__value_.second));
+  __node_traits::construct(__na, std::addressof(__h->__get_value().second));
   __h.get_deleter().__second_constructed = true;
   return __h;
 }
@@ -1434,7 +1438,7 @@ _Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
     __tree_.__insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__h.get()));
     __r = __h.release();
   }
-  return __r->__value_.second;
+  return __r->__get_value().second;
 }
 
 #  endif // _LIBCPP_CXX03_LANG
@@ -1445,7 +1449,7 @@ _Tp& map<_Key, _Tp, _Compare, _Allocator>::at(const key_type& __k) {
   __node_base_pointer& __child = __tree_.__find_equal(__parent, __k);
   if (__child == nullptr)
     std::__throw_out_of_range("map::at:  key not found");
-  return static_cast<__node_pointer>(__child)->__value_.second;
+  return static_cast<__node_pointer>(__child)->__get_value().second;
 }
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
@@ -1454,7 +1458,7 @@ const _Tp& map<_Key, _Tp, _Compare, _Allocator>::at(const key_type& __k) const {
   __node_base_pointer __child = __tree_.__find_equal(__parent, __k);
   if (__child == nullptr)
     std::__throw_out_of_range("map::at:  key not found");
-  return static_cast<__node_pointer>(__child)->__value_.second;
+  return static_cast<__node_pointer>(__child)->__get_value().second;
 }
 
 template <class _Key, class _Tp, class _Compare, class _Allocator>
@@ -1643,12 +1647,11 @@ public:
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-  _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) noexcept(is_nothrow_move_constructible<__base>::value) = default;
+  _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) = default;
 
   _LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a);
 
-  _LIBCPP_HIDE_FROM_ABI multimap&
-  operator=(multimap&& __m) noexcept(is_nothrow_move_assignable<__base>::value) = default;
+  _LIBCPP_HIDE_FROM_ABI multimap& operator=(multimap&& __m) = default;
 
   _LIBCPP_HIDE_FROM_ABI multimap(initializer_list<value_type> __il, const key_compare& __comp = key_compare())
       : __tree_(__vc(__comp)) {
@@ -1747,17 +1750,13 @@ public:
 
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
-    for (const_iterator __e = cend(); __f != __l; ++__f)
-      __tree_.__emplace_hint_multi(__e.__i_, *__f);
+    __tree_.__insert_range_multi(__f, __l);
   }
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
-    const_iterator __end = cend();
-    for (auto&& __element : __range) {
-      __tree_.__emplace_hint_multi(__end.__i_, std::forward<decltype(__element)>(__element));
-    }
+    __tree_.__insert_range_multi(ranges::begin(__range), ranges::end(__range));
   }
 #  endif
 
@@ -1957,7 +1956,8 @@ multimap<_Key, _Tp, _Compare, _Allocator>::multimap(multimap&& __m, const alloca
   if (__a != __m.get_allocator()) {
     const_iterator __e = cend();
     while (!__m.empty())
-      __tree_.__insert_multi_from_orphaned_node(__e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__value_));
+      __tree_.__insert_multi_from_orphaned_node(
+          __e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__get_value()));
   }
 }
 #  endif
diff --git a/libcxx/include/module.modulemap.in b/libcxx/include/module.modulemap.in
index b07a153eacfae..c431c0cb407f3 100644
--- a/libcxx/include/module.modulemap.in
+++ b/libcxx/include/module.modulemap.in
@@ -1607,7 +1607,6 @@ module std [system] {
   }
 
   module map {
-    module fwd { header "__fwd/map.h" }
     header "map"
     export *
     export std.iterator.reverse_iterator
@@ -1974,7 +1973,6 @@ module std [system] {
   }
 
   module set {
-    module fwd { header "__fwd/set.h" }
     header "set"
     export *
     export std.iterator.reverse_iterator
diff --git a/libcxx/include/optional b/libcxx/include/optional
index e81bff50daad6..39fcaa2c2ec18 100644
--- a/libcxx/include/optional
+++ b/libcxx/include/optional
@@ -20,6 +20,11 @@ namespace std {
   template <class T>
     class optional;
 
+  template<class T>
+   constexpr bool ranges::enable_view<optional<T>> = true;
+  template<class T>
+   constexpr auto format_kind<optional<T>> = range_format::disabled;
+
   template<class T>
     concept is-derived-from-optional = requires(const T& t) {       // exposition only
       []<class U>(const optional<U>&){ }(t);
@@ -102,6 +107,8 @@ namespace std {
   class optional {
   public:
     using value_type = T;
+    using iterator               = implementation-defined; // see [optional.iterators]
+    using const_iterator         = implementation-defined; // see [optional.iterators]
 
     // [optional.ctor], constructors
     constexpr optional() noexcept;
@@ -135,6 +142,12 @@ namespace std {
     // [optional.swap], swap
     void swap(optional &) noexcept(see below ); // constexpr in C++20
 
+    // [optional.iterators], iterator support
+    constexpr iterator begin() noexcept;
+    constexpr const_iterator begin() const noexcept;
+    constexpr iterator end() noexcept;
+    constexpr const_iterator end() const noexcept;
+
     // [optional.observe], observers
     constexpr T const *operator->() const noexcept;
     constexpr T *operator->() noexcept;
@@ -186,13 +199,18 @@ namespace std {
 #  include <__compare/three_way_comparable.h>
 #  include <__concepts/invocable.h>
 #  include <__config>
+#  include <__cstddef/ptrdiff_t.h>
 #  include <__exception/exception.h>
+#  include <__format/range_format.h>
 #  include <__functional/hash.h>
 #  include <__functional/invoke.h>
 #  include <__functional/unary_function.h>
 #  include <__fwd/functional.h>
+#  include <__iterator/bounded_iter.h>
+#  include <__iterator/wrap_iter.h>
 #  include <__memory/addressof.h>
 #  include <__memory/construct_at.h>
+#  include <__ranges/enable_view.h>
 #  include <__tuple/sfinae_helpers.h>
 #  include <__type_traits/add_pointer.h>
 #  include <__type_traits/conditional.h>
@@ -207,6 +225,7 @@ namespace std {
 #  include <__type_traits/is_convertible.h>
 #  include <__type_traits/is_core_convertible.h>
 #  include <__type_traits/is_destructible.h>
+#  include <__type_traits/is_function.h>
 #  include <__type_traits/is_nothrow_assignable.h>
 #  include <__type_traits/is_nothrow_constructible.h>
 #  include <__type_traits/is_object.h>
@@ -219,6 +238,7 @@ namespace std {
 #  include <__type_traits/is_trivially_constructible.h>
 #  include <__type_traits/is_trivially_destructible.h>
 #  include <__type_traits/is_trivially_relocatable.h>
+#  include <__type_traits/is_unbounded_array.h>
 #  include <__type_traits/negation.h>
 #  include <__type_traits/remove_const.h>
 #  include <__type_traits/remove_cv.h>
@@ -567,6 +587,14 @@ using __optional_sfinae_assign_base_t _LIBCPP_NODEBUG =
 template <class _Tp>
 class optional;
 
+#    if _LIBCPP_STD_VER >= 26
+template <class _Tp>
+constexpr bool ranges::enable_view<optional<_Tp>> = true;
+
+template <class _Tp>
+constexpr range_format format_kind<optional<_Tp>> = range_format::disabled;
+#    endif
+
 #    if _LIBCPP_STD_VER >= 20
 
 template <class _Tp>
@@ -586,9 +614,21 @@ class _LIBCPP_DECLSPEC_EMPTY_BASES optional
       private __optional_sfinae_assign_base_t<_Tp> {
   using __base _LIBCPP_NODEBUG = __optional_move_assign_base<_Tp>;
 
+  using __pointer _LIBCPP_NODEBUG       = std::add_pointer_t<_Tp>;
+  using __const_pointer _LIBCPP_NODEBUG = std::add_pointer_t<const _Tp>;
+
 public:
   using value_type = _Tp;
 
+#    if _LIBCPP_STD_VER >= 26
+#      ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_OPTIONAL
+  using iterator       = __bounded_iter<__wrap_iter<__pointer>>;
+  using const_iterator = __bounded_iter<__wrap_iter<__const_pointer>>;
+#      else
+  using iterator       = __wrap_iter<__pointer>;
+  using const_iterator = __wrap_iter<__const_pointer>;
+#      endif
+#    endif
   using __trivially_relocatable _LIBCPP_NODEBUG =
       conditional_t<__libcpp_is_trivially_relocatable<_Tp>::value, optional, void>;
   using __replaceable _LIBCPP_NODEBUG = conditional_t<__is_replaceable_v<_Tp>, optional, void>;
@@ -792,6 +832,34 @@ public:
     }
   }
 
+#    if _LIBCPP_STD_VER >= 26
+  // [optional.iterators], iterator support
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator begin() noexcept {
+#      ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_OPTIONAL
+    return std::__make_bounded_iter(
+        std::__wrap_iter<__pointer>(std::addressof(this->__get())),
+        std::__wrap_iter<__pointer>(std::addressof(this->__get())),
+        std::__wrap_iter<__pointer>(std::addressof(this->__get()) + (this->has_value() ? 1 : 0)));
+#      else
+    return iterator(std::addressof(this->__get()));
+#      endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr const_iterator begin() const noexcept {
+#      ifdef _LIBCPP_ABI_BOUNDED_ITERATORS_IN_OPTIONAL
+    return std::__make_bounded_iter(
+        std::__wrap_iter<__const_pointer>(std::addressof(this->__get())),
+        std::__wrap_iter<__const_pointer>(std::addressof(this->__get())),
+        std::__wrap_iter<__const_pointer>(std::addressof(this->__get()) + (this->has_value() ? 1 : 0)));
+#      else
+    return const_iterator(std::addressof(this->__get()));
+#      endif
+  }
+
+  _LIBCPP_HIDE_FROM_ABI constexpr iterator end() noexcept { return begin() + (this->has_value() ? 1 : 0); }
+  _LIBCPP_HIDE_FROM_ABI constexpr const_iterator end() const noexcept { return begin() + (this->has_value() ? 1 : 0); }
+#    endif
+
   _LIBCPP_HIDE_FROM_ABI constexpr add_pointer_t<value_type const> operator->() const noexcept {
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(this->has_value(), "optional operator-> called on a disengaged value");
     return std::addressof(this->__get());
diff --git a/libcxx/include/set b/libcxx/include/set
index 342a5294c814f..5190fc1f406ba 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -522,7 +522,6 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred);  // C++20
 #  include <__config>
 #  include <__functional/is_transparent.h>
 #  include <__functional/operations.h>
-#  include <__fwd/set.h>
 #  include <__iterator/erase_if_container.h>
 #  include <__iterator/iterator_traits.h>
 #  include <__iterator/ranges_iterator_traits.h>
@@ -531,6 +530,7 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred);  // C++20
 #  include <__memory/allocator_traits.h>
 #  include <__memory_resource/polymorphic_allocator.h>
 #  include <__node_handle>
+#  include <__ranges/access.h>
 #  include <__ranges/concepts.h>
 #  include <__ranges/container_compatible_range.h>
 #  include <__ranges/from_range.h>
@@ -570,7 +570,10 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Key, class _Compare, class _Allocator>
+template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> >
+class multiset;
+
+template <class _Key, class _Compare = less<_Key>, class _Allocator = allocator<_Key> >
 class set {
 public:
   // types:
@@ -665,7 +668,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI set(set&& __s) noexcept(is_nothrow_move_constructible<__base>::value) = default;
+  _LIBCPP_HIDE_FROM_ABI set(set&& __s) = default;
 #  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI explicit set(const allocator_type& __a) : __tree_(__a) {}
@@ -697,10 +700,7 @@ public:
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI set& operator=(set&& __s) noexcept(is_nothrow_move_assignable<__base>::value) {
-    __tree_ = std::move(__s.__tree_);
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI set& operator=(set&& __s) = default;
 #  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI ~set() { static_assert(sizeof(std::__diagnose_non_const_comparator<_Key, _Compare>()), ""); }
@@ -951,7 +951,7 @@ set<_Key, _Compare, _Allocator>::set(set&& __s, const allocator_type& __a) : __t
   if (__a != __s.get_allocator()) {
     const_iterator __e = cend();
     while (!__s.empty())
-      insert(__e, std::move(__s.__tree_.remove(__s.begin())->__value_));
+      insert(__e, std::move(__s.__tree_.remove(__s.begin())->__get_value()));
   }
 }
 
@@ -1124,7 +1124,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) = default;
 
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) noexcept(is_nothrow_move_constructible<__base>::value) = default;
+  _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) = default;
 
   _LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a);
 #  endif // _LIBCPP_CXX03_LANG
@@ -1156,10 +1156,7 @@ public:
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI multiset& operator=(multiset&& __s) _NOEXCEPT_(is_nothrow_move_assignable<__base>::value) {
-    __tree_ = std::move(__s.__tree_);
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI multiset& operator=(multiset&& __s) = default;
 #  endif // _LIBCPP_CXX03_LANG
 
   _LIBCPP_HIDE_FROM_ABI ~multiset() {
@@ -1203,18 +1200,14 @@ public:
   }
 
   template <class _InputIterator>
-  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
-    for (const_iterator __e = cend(); __f != __l; ++__f)
-      __tree_.__emplace_hint_multi(__e, *__f);
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
+    __tree_.__insert_range_multi(__first, __last);
   }
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
-    const_iterator __end = cend();
-    for (auto&& __element : __range) {
-      __tree_.__emplace_hint_multi(__end, std::forward<decltype(__element)>(__element));
-    }
+    __tree_.__insert_range_multi(ranges::begin(__range), ranges::end(__range));
   }
 #  endif
 
@@ -1414,7 +1407,7 @@ multiset<_Key, _Compare, _Allocator>::multiset(multiset&& __s, const allocator_t
   if (__a != __s.get_allocator()) {
     const_iterator __e = cend();
     while (!__s.empty())
-      insert(__e, std::move(__s.__tree_.remove(__s.begin())->__value_));
+      insert(__e, std::move(__s.__tree_.remove(__s.begin())->__get_value()));
   }
 }
 
diff --git a/libcxx/include/streambuf b/libcxx/include/streambuf
index 585ae7af65aa8..7dc4e31cc2324 100644
--- a/libcxx/include/streambuf
+++ b/libcxx/include/streambuf
@@ -119,6 +119,7 @@ protected:
 #    include <__locale>
 #    include <__type_traits/is_same.h>
 #    include <__utility/is_valid_range.h>
+#    include <__utility/scope_guard.h>
 #    include <climits>
 #    include <ios>
 #    include <iosfwd>
@@ -178,18 +179,27 @@ public:
   // Get and put areas:
   // 27.6.2.2.3 Get area:
   inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 streamsize in_avail() {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     if (gptr() < egptr())
       return static_cast<streamsize>(egptr() - gptr());
     return showmanyc();
   }
 
   inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type snextc() {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     if (sbumpc() == traits_type::eof())
       return traits_type::eof();
     return sgetc();
   }
 
   inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sbumpc() {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     if (gptr() == egptr())
       return uflow();
     int_type __c = traits_type::to_int_type(*gptr());
@@ -198,6 +208,9 @@ public:
   }
 
   inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sgetc() {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     if (gptr() == egptr())
       return underflow();
     return traits_type::to_int_type(*gptr());
@@ -207,6 +220,9 @@ public:
 
   // 27.6.2.2.4 Putback:
   inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sputbackc(char_type __c) {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     if (eback() == gptr() || !traits_type::eq(__c, *(gptr() - 1)))
       return pbackfail(traits_type::to_int_type(__c));
     this->gbump(-1);
@@ -214,6 +230,9 @@ public:
   }
 
   inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sungetc() {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     if (eback() == gptr())
       return pbackfail();
     this->gbump(-1);
@@ -222,6 +241,9 @@ public:
 
   // 27.6.2.2.5 Put area:
   inline _LIBCPP_HIDE_FROM_ABI_AFTER_V1 int_type sputc(char_type __c) {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     if (pptr() == epptr())
       return overflow(traits_type::to_int_type(__c));
     *pptr() = __c;
@@ -317,6 +339,9 @@ protected:
   virtual streamsize showmanyc() { return 0; }
 
   virtual streamsize xsgetn(char_type* __s, streamsize __n) {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     int_type __c;
     streamsize __i = 0;
     while (__i < __n) {
@@ -338,6 +363,9 @@ protected:
 
   virtual int_type underflow() { return traits_type::eof(); }
   virtual int_type uflow() {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     if (underflow() == traits_type::eof())
       return traits_type::eof();
     int_type __c = traits_type::to_int_type(*gptr());
@@ -350,6 +378,9 @@ protected:
 
   // 27.6.2.4.5 Put area:
   virtual streamsize xsputn(const char_type* __s, streamsize __n) {
+    __check_invariants();
+    auto __guard = std::__make_scope_guard([this] { this->__check_invariants(); });
+
     streamsize __i = 0;
     while (__i < __n) {
       if (pptr() >= epptr()) {
@@ -370,6 +401,15 @@ protected:
 
   virtual int_type overflow(int_type = traits_type::eof()) { return traits_type::eof(); }
 
+  // This function checks some invariants of the class (it isn't exhaustive).
+  _LIBCPP_HIDE_FROM_ABI void __check_invariants() const {
+    _LIBCPP_ASSERT_INTERNAL(pbase() <= pptr(), "this is an invariant of the class");
+    _LIBCPP_ASSERT_INTERNAL(pptr() <= epptr(), "this is an invariant of the class");
+
+    _LIBCPP_ASSERT_INTERNAL(eback() <= gptr(), "this is an invariant of the class");
+    _LIBCPP_ASSERT_INTERNAL(gptr() <= egptr(), "this is an invariant of the class");
+  }
+
 private:
   locale __loc_;
   char_type* __binp_ = nullptr;
diff --git a/libcxx/include/string b/libcxx/include/string
index 98297d04d0c61..1d197654b9fee 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -974,7 +974,12 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string()
       _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
-      : __rep_() {
+#  if _LIBCPP_STD_VER >= 20 // TODO(LLVM 23): Remove this condition; this is a workaround for https://llvm.org/PR154567
+      : __rep_(__short())
+#  else
+      : __rep_()
+#  endif
+  {
     __annotate_new(0);
   }
 
@@ -984,7 +989,12 @@ public:
 #  else
       _NOEXCEPT
 #  endif
-      : __rep_(), __alloc_(__a) {
+#  if _LIBCPP_STD_VER >= 20 // TODO(LLVM 23): Remove this condition; this is a workaround for https://llvm.org/PR154567
+      : __rep_(__short()),
+#  else
+      : __rep_(),
+#  endif
+        __alloc_(__a) {
     __annotate_new(0);
   }
 
@@ -1417,20 +1427,21 @@ public:
     size_type __sz  = size();
     size_type __cap = capacity();
     size_type __n   = static_cast<size_type>(std::distance(__first, __last));
-    if (__n) {
-      if (__string_is_trivial_iterator_v<_ForwardIterator> && !__addr_in_range(*__first)) {
-        if (__cap - __sz < __n)
-          __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
-        __annotate_increase(__n);
-        auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__get_pointer() + __sz));
-        traits_type::assign(*__end, value_type());
-        __set_size(__sz + __n);
-      } else {
-        const basic_string __temp(__first, __last, __alloc_);
-        append(__temp.data(), __temp.size());
-      }
+    if (__n == 0)
+      return *this;
+
+    if (__string_is_trivial_iterator_v<_ForwardIterator> && !__addr_in_range(*__first)) {
+      if (__cap - __sz < __n)
+        __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
+      __annotate_increase(__n);
+      auto __end = __copy_non_overlapping_range(__first, __last, std::__to_address(__get_pointer() + __sz));
+      traits_type::assign(*__end, value_type());
+      __set_size(__sz + __n);
+      return *this;
+    } else {
+      const basic_string __temp(__first, __last, __alloc_);
+      return append(__temp.data(), __temp.size());
     }
-    return *this;
   }
 
 #  if _LIBCPP_STD_VER >= 23
@@ -2790,24 +2801,23 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE basic_string<_CharT, _Traits, _Al
 basic_string<_CharT, _Traits, _Allocator>::__assign_no_alias(const value_type* __s, size_type __n) {
   const auto __cap  = __is_short ? static_cast<size_type>(__min_cap) : __get_long_cap();
   const auto __size = __is_short ? __get_short_size() : __get_long_size();
-  if (__n < __cap) {
-    if (__n > __size)
-      __annotate_increase(__n - __size);
-    pointer __p;
-    if (__is_short) {
-      __p = __get_short_pointer();
-      __set_short_size(__n);
-    } else {
-      __p = __get_long_pointer();
-      __set_long_size(__n);
-    }
-    traits_type::copy(std::__to_address(__p), __s, __n);
-    traits_type::assign(__p[__n], value_type());
-    if (__size > __n)
-      __annotate_shrink(__size);
-  } else {
+  if (__n >= __cap) {
     __grow_by_and_replace(__cap - 1, __n - __cap + 1, __size, 0, __size, __n, __s);
+    return *this;
+  }
+
+  __annotate_delete();
+  auto __guard = std::__make_scope_guard(__annotate_new_size(*this));
+  pointer __p;
+  if (__is_short) {
+    __p = __get_short_pointer();
+    __set_short_size(__n);
+  } else {
+    __p = __get_long_pointer();
+    __set_long_size(__n);
   }
+  traits_type::copy(std::__to_address(__p), __s, __n);
+  traits_type::assign(__p[__n], value_type());
   return *this;
 }
 
@@ -3021,52 +3031,56 @@ basic_string<_CharT, _Traits, _Allocator>::append(const value_type* __s, size_ty
   _LIBCPP_ASSERT_NON_NULL(__n == 0 || __s != nullptr, "string::append received nullptr");
   size_type __cap = capacity();
   size_type __sz  = size();
-  if (__cap - __sz >= __n) {
-    if (__n) {
-      __annotate_increase(__n);
-      value_type* __p = std::__to_address(__get_pointer());
-      traits_type::copy(__p + __sz, __s, __n);
-      __sz += __n;
-      __set_size(__sz);
-      traits_type::assign(__p[__sz], value_type());
-    }
-  } else
+  if (__cap - __sz < __n) {
     __grow_by_and_replace(__cap, __sz + __n - __cap, __sz, __sz, 0, __n, __s);
+    return *this;
+  }
+
+  if (__n == 0)
+    return *this;
+
+  __annotate_increase(__n);
+  value_type* __p = std::__to_address(__get_pointer());
+  traits_type::copy(__p + __sz, __s, __n);
+  __sz += __n;
+  __set_size(__sz);
+  traits_type::assign(__p[__sz], value_type());
   return *this;
 }
 
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::append(size_type __n, value_type __c) {
-  if (__n) {
-    size_type __cap = capacity();
-    size_type __sz  = size();
-    if (__cap - __sz < __n)
-      __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
-    __annotate_increase(__n);
-    pointer __p = __get_pointer();
-    traits_type::assign(std::__to_address(__p) + __sz, __n, __c);
-    __sz += __n;
-    __set_size(__sz);
-    traits_type::assign(__p[__sz], value_type());
-  }
+  if (__n == 0)
+    return *this;
+
+  size_type __cap = capacity();
+  size_type __sz  = size();
+  if (__cap - __sz < __n)
+    __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
+  __annotate_increase(__n);
+  pointer __p = __get_pointer();
+  traits_type::assign(std::__to_address(__p) + __sz, __n, __c);
+  __sz += __n;
+  __set_size(__sz);
+  traits_type::assign(__p[__sz], value_type());
   return *this;
 }
 
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 inline void
 basic_string<_CharT, _Traits, _Allocator>::__append_default_init(size_type __n) {
-  if (__n) {
-    size_type __cap = capacity();
-    size_type __sz  = size();
-    if (__cap - __sz < __n)
-      __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
-    __annotate_increase(__n);
-    pointer __p = __get_pointer();
-    __sz += __n;
-    __set_size(__sz);
-    traits_type::assign(__p[__sz], value_type());
-  }
+  if (__n == 0)
+    return;
+  size_type __cap = capacity();
+  size_type __sz  = size();
+  if (__cap - __sz < __n)
+    __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
+  __annotate_increase(__n);
+  pointer __p = __get_pointer();
+  __sz += __n;
+  __set_size(__sz);
+  traits_type::assign(__p[__sz], value_type());
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3124,23 +3138,27 @@ basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, const value_t
   if (__pos > __sz)
     this->__throw_out_of_range();
   size_type __cap = capacity();
-  if (__cap - __sz >= __n) {
-    if (__n) {
-      __annotate_increase(__n);
-      value_type* __p    = std::__to_address(__get_pointer());
-      size_type __n_move = __sz - __pos;
-      if (__n_move != 0) {
-        if (std::__is_pointer_in_range(__p + __pos, __p + __sz, __s))
-          __s += __n;
-        traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
-      }
-      traits_type::move(__p + __pos, __s, __n);
-      __sz += __n;
-      __set_size(__sz);
-      traits_type::assign(__p[__sz], value_type());
-    }
-  } else
+
+  if (__cap - __sz < __n) {
     __grow_by_and_replace(__cap, __sz + __n - __cap, __sz, __pos, 0, __n, __s);
+    return *this;
+  }
+
+  if (__n == 0)
+    return *this;
+
+  __annotate_increase(__n);
+  value_type* __p    = std::__to_address(__get_pointer());
+  size_type __n_move = __sz - __pos;
+  if (__n_move != 0) {
+    if (std::__is_pointer_in_range(__p + __pos, __p + __sz, __s))
+      __s += __n;
+    traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
+  }
+  traits_type::move(__p + __pos, __s, __n);
+  __sz += __n;
+  __set_size(__sz);
+  traits_type::assign(__p[__sz], value_type());
   return *this;
 }
 
@@ -3150,24 +3168,26 @@ basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, size_type __n
   size_type __sz = size();
   if (__pos > __sz)
     this->__throw_out_of_range();
-  if (__n) {
-    size_type __cap = capacity();
-    value_type* __p;
-    if (__cap - __sz >= __n) {
-      __annotate_increase(__n);
-      __p                = std::__to_address(__get_pointer());
-      size_type __n_move = __sz - __pos;
-      if (__n_move != 0)
-        traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
-    } else {
-      __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __pos, 0, __n);
-      __p = std::__to_address(__get_long_pointer());
-    }
-    traits_type::assign(__p + __pos, __n, __c);
-    __sz += __n;
-    __set_size(__sz);
-    traits_type::assign(__p[__sz], value_type());
+
+  if (__n == 0)
+    return *this;
+
+  size_type __cap = capacity();
+  value_type* __p;
+  if (__cap - __sz >= __n) {
+    __annotate_increase(__n);
+    __p                = std::__to_address(__get_pointer());
+    size_type __n_move = __sz - __pos;
+    if (__n_move != 0)
+      traits_type::move(__p + __pos + __n, __p + __pos, __n_move);
+  } else {
+    __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __pos, 0, __n);
+    __p = std::__to_address(__get_long_pointer());
   }
+  traits_type::assign(__p + __pos, __n, __c);
+  __sz += __n;
+  __set_size(__sz);
+  traits_type::assign(__p[__sz], value_type());
   return *this;
 }
 
@@ -3241,38 +3261,38 @@ basic_string<_CharT, _Traits, _Allocator>::replace(
     this->__throw_out_of_range();
   __n1            = std::min(__n1, __sz - __pos);
   size_type __cap = capacity();
-  if (__cap - __sz + __n1 >= __n2) {
-    value_type* __p = std::__to_address(__get_pointer());
-    if (__n1 != __n2) {
-      if (__n2 > __n1)
-        __annotate_increase(__n2 - __n1);
-      size_type __n_move = __sz - __pos - __n1;
-      if (__n_move != 0) {
-        if (__n1 > __n2) {
-          traits_type::move(__p + __pos, __s, __n2);
-          traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
-          return __null_terminate_at(__p, __sz + (__n2 - __n1));
-        }
-        if (std::__is_pointer_in_range(__p + __pos + 1, __p + __sz, __s)) {
-          if (__p + __pos + __n1 <= __s)
-            __s += __n2 - __n1;
-          else // __p + __pos < __s < __p + __pos + __n1
-          {
-            traits_type::move(__p + __pos, __s, __n1);
-            __pos += __n1;
-            __s += __n2;
-            __n2 -= __n1;
-            __n1 = 0;
-          }
-        }
+  if (__cap - __sz + __n1 < __n2) {
+    __grow_by_and_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2, __s);
+    return *this;
+  }
+
+  value_type* __p = std::__to_address(__get_pointer());
+  if (__n1 != __n2) {
+    if (__n2 > __n1)
+      __annotate_increase(__n2 - __n1);
+    size_type __n_move = __sz - __pos - __n1;
+    if (__n_move != 0) {
+      if (__n1 > __n2) {
+        traits_type::move(__p + __pos, __s, __n2);
         traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
+        return __null_terminate_at(__p, __sz + (__n2 - __n1));
+      }
+      if (std::__is_pointer_in_range(__p + __pos + 1, __p + __sz, __s)) {
+        if (__p + __pos + __n1 <= __s) {
+          __s += __n2 - __n1;
+        } else { // __p + __pos < __s < __p + __pos + __n1
+          traits_type::move(__p + __pos, __s, __n1);
+          __pos += __n1;
+          __s += __n2;
+          __n2 -= __n1;
+          __n1 = 0;
+        }
       }
+      traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
     }
-    traits_type::move(__p + __pos, __s, __n2);
-    return __null_terminate_at(__p, __sz + (__n2 - __n1));
-  } else
-    __grow_by_and_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2, __s);
-  return *this;
+  }
+  traits_type::move(__p + __pos, __s, __n2);
+  return __null_terminate_at(__p, __sz + (__n2 - __n1));
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3325,15 +3345,16 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __
 template <class _CharT, class _Traits, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_NOINLINE void
 basic_string<_CharT, _Traits, _Allocator>::__erase_external_with_move(size_type __pos, size_type __n) {
-  if (__n) {
-    size_type __sz     = size();
-    value_type* __p    = std::__to_address(__get_pointer());
-    __n                = std::min(__n, __sz - __pos);
-    size_type __n_move = __sz - __pos - __n;
-    if (__n_move != 0)
-      traits_type::move(__p + __pos, __p + __pos + __n, __n_move);
-    __null_terminate_at(__p, __sz - __n);
-  }
+  if (__n == 0)
+    return;
+
+  size_type __sz     = size();
+  value_type* __p    = std::__to_address(__get_pointer());
+  __n                = std::min(__n, __sz - __pos);
+  size_type __n_move = __sz - __pos - __n;
+  if (__n_move != 0)
+    traits_type::move(__p + __pos, __p + __pos + __n, __n_move);
+  __null_terminate_at(__p, __sz - __n);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index f86b2722aca6c..9a20bb6f4a0bb 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -320,7 +320,7 @@ public:
   _LIBCPP_CONSTEXPR _LIBCPP_HIDE_FROM_ABI basic_string_view(const _CharT* __s, size_type __len) _NOEXCEPT
       _LIBCPP_DIAGNOSE_NULLPTR_IF(__len != 0 && __s == nullptr, " if len is not zero")
       : __data_(__s), __size_(__len) {
-#  if _LIBCPP_STD_VER >= 14
+#  if !defined(_LIBCPP_CXX03_LANG) && (!defined(_LIBCPP_COMPILER_GCC) || _LIBCPP_STD_VER >= 14)
     // Allocations must fit in `ptrdiff_t` for pointer arithmetic to work. If `__len` exceeds it, the input
     // range could not have been valid. Most likely the caller underflowed some arithmetic and inadvertently
     // passed in a negative length.
@@ -502,7 +502,6 @@ public:
   // find
   _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
   find(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT {
-    _LIBCPP_ASSERT_NON_NULL(__s.size() == 0 || __s.data() != nullptr, "string_view::find(): received nullptr");
     return std::__str_find<value_type, size_type, traits_type, npos>(data(), size(), __s.data(), __pos, __s.size());
   }
 
@@ -527,7 +526,6 @@ public:
   // rfind
   _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
   rfind(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT {
-    _LIBCPP_ASSERT_NON_NULL(__s.size() == 0 || __s.data() != nullptr, "string_view::find(): received nullptr");
     return std::__str_rfind<value_type, size_type, traits_type, npos>(data(), size(), __s.data(), __pos, __s.size());
   }
 
@@ -553,7 +551,6 @@ public:
   // find_first_of
   _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
   find_first_of(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT {
-    _LIBCPP_ASSERT_NON_NULL(__s.size() == 0 || __s.data() != nullptr, "string_view::find_first_of(): received nullptr");
     return std::__str_find_first_of<value_type, size_type, traits_type, npos>(
         data(), size(), __s.data(), __pos, __s.size());
   }
@@ -580,7 +577,6 @@ public:
   // find_last_of
   _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
   find_last_of(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT {
-    _LIBCPP_ASSERT_NON_NULL(__s.size() == 0 || __s.data() != nullptr, "string_view::find_last_of(): received nullptr");
     return std::__str_find_last_of<value_type, size_type, traits_type, npos>(
         data(), size(), __s.data(), __pos, __s.size());
   }
@@ -607,8 +603,6 @@ public:
   // find_first_not_of
   _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
   find_first_not_of(basic_string_view __s, size_type __pos = 0) const _NOEXCEPT {
-    _LIBCPP_ASSERT_NON_NULL(
-        __s.size() == 0 || __s.data() != nullptr, "string_view::find_first_not_of(): received nullptr");
     return std::__str_find_first_not_of<value_type, size_type, traits_type, npos>(
         data(), size(), __s.data(), __pos, __s.size());
   }
@@ -635,8 +629,6 @@ public:
   // find_last_not_of
   _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI size_type
   find_last_not_of(basic_string_view __s, size_type __pos = npos) const _NOEXCEPT {
-    _LIBCPP_ASSERT_NON_NULL(
-        __s.size() == 0 || __s.data() != nullptr, "string_view::find_last_not_of(): received nullptr");
     return std::__str_find_last_not_of<value_type, size_type, traits_type, npos>(
         data(), size(), __s.data(), __pos, __s.size());
   }
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index be30ab5b2173d..8cc061c70b891 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -516,6 +516,7 @@ struct __tuple_impl;
 
 struct __forward_args {};
 struct __value_init {};
+struct __from_tuple {};
 
 template <size_t... _Indx, class... _Tp>
 struct _LIBCPP_DECLSPEC_EMPTY_BASES
@@ -538,7 +539,7 @@ struct _LIBCPP_DECLSPEC_EMPTY_BASES
       : __tuple_leaf<_Indx, _Tp>(__uses_alloc_ctor<_Tp, _Alloc, _Args>(), __alloc, std::forward<_Args>(__args))... {}
 
   template <class _Tuple>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_impl(_Tuple&& __t) noexcept(
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_impl(__from_tuple, _Tuple&& __t) noexcept(
       (__all<is_nothrow_constructible<
            _Tp,
            typename tuple_element<_Indx, typename __make_tuple_types<_Tuple>::type>::type>::value...>::value))
@@ -547,7 +548,8 @@ struct _LIBCPP_DECLSPEC_EMPTY_BASES
                 std::get<_Indx>(__t)))... {}
 
   template <class _Alloc, class _Tuple>
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 __tuple_impl(allocator_arg_t, const _Alloc& __a, _Tuple&& __t)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
+  __tuple_impl(allocator_arg_t, const _Alloc& __a, __from_tuple, _Tuple&& __t)
       : __tuple_leaf<_Indx, _Tp>(
             __uses_alloc_ctor<_Tp,
                               _Alloc,
@@ -673,13 +675,13 @@ public:
             template <class...> class _And                                  = _And,
             __enable_if_t< _And<is_copy_constructible<_Tp>...>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc& __alloc, const tuple& __t)
-      : __base_(allocator_arg_t(), __alloc, __t) {}
+      : __base_(allocator_arg_t(), __alloc, __from_tuple(), __t) {}
 
   template <class _Alloc,
             template <class...> class _And                                  = _And,
             __enable_if_t< _And<is_move_constructible<_Tp>...>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 tuple(allocator_arg_t, const _Alloc& __alloc, tuple&& __t)
-      : __base_(allocator_arg_t(), __alloc, std::move(__t)) {}
+      : __base_(allocator_arg_t(), __alloc, __from_tuple(), std::move(__t)) {}
 
   // tuple(const tuple<U...>&) constructors (including allocator_arg_t variants)
 
@@ -712,7 +714,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> >::value)
       tuple(const tuple<_Up...>& __t) noexcept(_And<is_nothrow_constructible<_Tp, const _Up&>...>::value)
-      : __base_(__t) {}
+      : __base_(__from_tuple(), __t) {}
 
   template <class... _Up,
             class _Alloc,
@@ -720,33 +722,33 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<const _Up&, _Tp>...> >::value)
       tuple(allocator_arg_t, const _Alloc& __a, const tuple<_Up...>& __t)
-      : __base_(allocator_arg_t(), __a, __t) {}
+      : __base_(allocator_arg_t(), __a, __from_tuple(), __t) {}
 
 #    if _LIBCPP_STD_VER >= 23
   // tuple(tuple<U...>&) constructors (including allocator_arg_t variants)
 
   template <class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<tuple<_Up...>&>::value>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<_Up&, _Tp>...>::value) tuple(tuple<_Up...>& __t)
-      : __base_(__t) {}
+      : __base_(__from_tuple(), __t) {}
 
   template <class _Alloc, class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<tuple<_Up...>&>::value>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<_Up&, _Tp>...>::value)
       tuple(allocator_arg_t, const _Alloc& __alloc, tuple<_Up...>& __t)
-      : __base_(allocator_arg_t(), __alloc, __t) {}
+      : __base_(allocator_arg_t(), __alloc, __from_tuple(), __t) {}
 #    endif // _LIBCPP_STD_VER >= 23
 
   // tuple(tuple<U...>&&) constructors (including allocator_arg_t variants)
   template <class... _Up, __enable_if_t< _And< _EnableCtorFromUTypesTuple<tuple<_Up...>&&> >::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
       tuple(tuple<_Up...>&& __t) noexcept(_And<is_nothrow_constructible<_Tp, _Up>...>::value)
-      : __base_(std::move(__t)) {}
+      : __base_(__from_tuple(), std::move(__t)) {}
 
   template <class _Alloc,
             class... _Up,
             __enable_if_t< _And< _EnableCtorFromUTypesTuple<tuple<_Up...>&&> >::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_Lazy<_And, is_convertible<_Up, _Tp>...> >::value)
       tuple(allocator_arg_t, const _Alloc& __a, tuple<_Up...>&& __t)
-      : __base_(allocator_arg_t(), __a, std::move(__t)) {}
+      : __base_(allocator_arg_t(), __a, __from_tuple(), std::move(__t)) {}
 
 #    if _LIBCPP_STD_VER >= 23
   // tuple(const tuple<U...>&&) constructors (including allocator_arg_t variants)
@@ -754,14 +756,14 @@ public:
   template <class... _Up, enable_if_t< _EnableCtorFromUTypesTuple<const tuple<_Up...>&&>::value>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<const _Up&&, _Tp>...>::value)
       tuple(const tuple<_Up...>&& __t)
-      : __base_(std::move(__t)) {}
+      : __base_(__from_tuple(), std::move(__t)) {}
 
   template <class _Alloc,
             class... _Up,
             enable_if_t< _EnableCtorFromUTypesTuple<const tuple<_Up...>&&>::value>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_Lazy<_And, is_convertible<const _Up&&, _Tp>...>::value)
       tuple(allocator_arg_t, const _Alloc& __alloc, const tuple<_Up...>&& __t)
-      : __base_(allocator_arg_t(), __alloc, std::move(__t)) {}
+      : __base_(allocator_arg_t(), __alloc, __from_tuple(), std::move(__t)) {}
 #    endif // _LIBCPP_STD_VER >= 23
 
   // tuple(const pair<U1, U2>&) constructors (including allocator_arg_t variants)
@@ -796,7 +798,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> >::value)
       tuple(const pair<_Up1, _Up2>& __p) noexcept(_NothrowConstructibleFromPair<const pair<_Up1, _Up2>&>::value)
-      : __base_(__p) {}
+      : __base_(__from_tuple(), __p) {}
 
   template <class _Alloc,
             class _Up1,
@@ -806,7 +808,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_BothImplicitlyConvertible<const pair<_Up1, _Up2>&> >::value)
       tuple(allocator_arg_t, const _Alloc& __a, const pair<_Up1, _Up2>& __p)
-      : __base_(allocator_arg_t(), __a, __p) {}
+      : __base_(allocator_arg_t(), __a, __from_tuple(), __p) {}
 
 #    if _LIBCPP_STD_VER >= 23
   // tuple(pair<U1, U2>&) constructors (including allocator_arg_t variants)
@@ -814,7 +816,7 @@ public:
   template <class _U1, class _U2, enable_if_t< _EnableCtorFromPair<pair<_U1, _U2>&>::value>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<pair<_U1, _U2>&>::value)
       tuple(pair<_U1, _U2>& __p)
-      : __base_(__p) {}
+      : __base_(__from_tuple(), __p) {}
 
   template <class _Alloc,
             class _U1,
@@ -822,7 +824,7 @@ public:
             enable_if_t< _EnableCtorFromPair<std::pair<_U1, _U2>&>::value>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<pair<_U1, _U2>&>::value)
       tuple(allocator_arg_t, const _Alloc& __alloc, pair<_U1, _U2>& __p)
-      : __base_(allocator_arg_t(), __alloc, __p) {}
+      : __base_(allocator_arg_t(), __alloc, __from_tuple(), __p) {}
 #    endif
 
   // tuple(pair<U1, U2>&&) constructors (including allocator_arg_t variants)
@@ -834,7 +836,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   _LIBCPP_CONSTEXPR_SINCE_CXX14 explicit(_Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> >::value)
       tuple(pair<_Up1, _Up2>&& __p) noexcept(_NothrowConstructibleFromPair<pair<_Up1, _Up2>&&>::value)
-      : __base_(std::move(__p)) {}
+      : __base_(__from_tuple(), std::move(__p)) {}
 
   template <class _Alloc,
             class _Up1,
@@ -844,7 +846,7 @@ public:
   _LIBCPP_HIDE_FROM_ABI
   _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit(_Not<_BothImplicitlyConvertible<pair<_Up1, _Up2>&&> >::value)
       tuple(allocator_arg_t, const _Alloc& __a, pair<_Up1, _Up2>&& __p)
-      : __base_(allocator_arg_t(), __a, std::move(__p)) {}
+      : __base_(allocator_arg_t(), __a, __from_tuple(), std::move(__p)) {}
 
 #    if _LIBCPP_STD_VER >= 23
   // tuple(const pair<U1, U2>&&) constructors (including allocator_arg_t variants)
@@ -852,7 +854,7 @@ public:
   template <class _U1, class _U2, enable_if_t< _EnableCtorFromPair<const pair<_U1, _U2>&&>::value>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<const pair<_U1, _U2>&&>::value)
       tuple(const pair<_U1, _U2>&& __p)
-      : __base_(std::move(__p)) {}
+      : __base_(__from_tuple(), std::move(__p)) {}
 
   template <class _Alloc,
             class _U1,
@@ -860,7 +862,7 @@ public:
             enable_if_t< _EnableCtorFromPair<const pair<_U1, _U2>&&>::value>* = nullptr>
   _LIBCPP_HIDE_FROM_ABI constexpr explicit(!_BothImplicitlyConvertible<const pair<_U1, _U2>&&>::value)
       tuple(allocator_arg_t, const _Alloc& __alloc, const pair<_U1, _U2>&& __p)
-      : __base_(allocator_arg_t(), __alloc, std::move(__p)) {}
+      : __base_(allocator_arg_t(), __alloc, __from_tuple(), std::move(__p)) {}
 #    endif // _LIBCPP_STD_VER >= 23
 
   // [tuple.assign]
@@ -1432,6 +1434,7 @@ inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp __make_from_tuple_impl(_Tuple&& __t,
     enable_if_t<is_constructible_v<_Tp, decltype(std::get<_Idx>(std::forward<_Tuple>(__t)))...>> * = nullptr)
     _LIBCPP_NOEXCEPT_RETURN(_Tp(std::get<_Idx>(std::forward<_Tuple>(__t))...))
 #endif // _LIBCPP_STD_VER >= 20
+#undef _LIBCPP_NOEXCEPT_RETURN
 
 template <class _Tp, class _Tuple,
           class _Seq = make_index_sequence<tuple_size_v<remove_reference_t<_Tuple>>>, class = void>
@@ -1451,10 +1454,19 @@ template <class _Tp, class _Tuple>
 #else
 template <class _Tp, class _Tuple, class = enable_if_t<__can_make_from_tuple<_Tp, _Tuple>>> // strengthen
 #endif // _LIBCPP_STD_VER >= 20
+
 inline _LIBCPP_HIDE_FROM_ABI constexpr _Tp make_from_tuple(_Tuple&& __t)
-    _LIBCPP_NOEXCEPT_RETURN(std::__make_from_tuple_impl<_Tp>(
-        std::forward<_Tuple>(__t), make_index_sequence<tuple_size_v<remove_reference_t<_Tuple>>>()))
-#    undef _LIBCPP_NOEXCEPT_RETURN
+  noexcept(noexcept(std::__make_from_tuple_impl<_Tp>(std::forward<_Tuple>(__t),
+                    make_index_sequence<tuple_size_v<remove_reference_t<_Tuple>>>()))) {
+#if _LIBCPP_STD_VER >= 23 && __has_builtin(__reference_constructs_from_temporary)
+  if constexpr (tuple_size_v<remove_reference_t<_Tuple>> == 1) {
+    static_assert(!std::reference_constructs_from_temporary_v<_Tp, decltype(std::get<0>(std::declval<_Tuple>()))>,
+                  "Attempted construction of reference element binds to a temporary whose lifetime has ended");
+  }
+#endif // _LIBCPP_STD_VER >= 23
+  return std::__make_from_tuple_impl<_Tp>(
+        std::forward<_Tuple>(__t), make_index_sequence<tuple_size_v<remove_reference_t<_Tuple>>>());
+}
 
 #  endif // _LIBCPP_STD_VER >= 17
 
diff --git a/libcxx/include/unordered_map b/libcxx/include/unordered_map
index 97c2c52eba337..a934953674967 100644
--- a/libcxx/include/unordered_map
+++ b/libcxx/include/unordered_map
@@ -1046,10 +1046,10 @@ public:
 #  endif
 
   _LIBCPP_HIDE_FROM_ABI explicit unordered_map(const allocator_type& __a);
-  _LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u);
+  _LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u, const allocator_type& __a);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u, const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_map(initializer_list<value_type> __il);
   _LIBCPP_HIDE_FROM_ABI
@@ -1099,24 +1099,9 @@ public:
     static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
   }
 
-  _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) {
+  _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) = default;
 #  ifndef _LIBCPP_CXX03_LANG
-    __table_ = __u.__table_;
-#  else
-    if (this != std::addressof(__u)) {
-      __table_.clear();
-      __table_.hash_function()   = __u.__table_.hash_function();
-      __table_.key_eq()          = __u.__table_.key_eq();
-      __table_.max_load_factor() = __u.__table_.max_load_factor();
-      __table_.__copy_assign_alloc(__u.__table_);
-      insert(__u.begin(), __u.end());
-    }
-#  endif
-    return *this;
-  }
-#  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(unordered_map&& __u)
-      _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(unordered_map&& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_map& operator=(initializer_list<value_type> __il);
 #  endif // _LIBCPP_CXX03_LANG
 
@@ -1563,12 +1548,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
   insert(__first, __last);
 }
 
-template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
-unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u) : __table_(__u.__table_) {
-  __table_.__rehash_unique(__u.bucket_count());
-  insert(__u.begin(), __u.end());
-}
-
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u, const allocator_type& __a)
     : __table_(__u.__table_, typename __table::allocator_type(__a)) {
@@ -1578,11 +1557,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_ma
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
-inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u)
-    _NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
-    : __table_(std::move(__u.__table_)) {}
-
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u, const allocator_type& __a)
     : __table_(std::move(__u.__table_), typename __table::allocator_type(__a)) {
@@ -1618,14 +1592,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
   insert(__il.begin(), __il.end());
 }
 
-template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
-inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
-unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_map&& __u)
-    _NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
-  __table_ = std::move(__u.__table_);
-  return *this;
-}
-
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
 unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
@@ -1852,11 +1818,10 @@ public:
 #  endif
 
   _LIBCPP_HIDE_FROM_ABI explicit unordered_multimap(const allocator_type& __a);
-  _LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u);
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u, const allocator_type& __a);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u)
-      _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u, const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(initializer_list<value_type> __il);
   _LIBCPP_HIDE_FROM_ABI unordered_multimap(
@@ -1906,24 +1871,9 @@ public:
     static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
   }
 
-  _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(const unordered_multimap& __u) {
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(const unordered_multimap& __u) = default;
 #  ifndef _LIBCPP_CXX03_LANG
-    __table_ = __u.__table_;
-#  else
-    if (this != std::addressof(__u)) {
-      __table_.clear();
-      __table_.hash_function()   = __u.__table_.hash_function();
-      __table_.key_eq()          = __u.__table_.key_eq();
-      __table_.max_load_factor() = __u.__table_.max_load_factor();
-      __table_.__copy_assign_alloc(__u.__table_);
-      insert(__u.begin(), __u.end());
-    }
-#  endif
-    return *this;
-  }
-#  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(unordered_multimap&& __u)
-      _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(unordered_multimap&& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(initializer_list<value_type> __il);
 #  endif // _LIBCPP_CXX03_LANG
 
@@ -2315,13 +2265,6 @@ template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const allocator_type& __a)
     : __table_(typename __table::allocator_type(__a)) {}
 
-template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
-unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const unordered_multimap& __u)
-    : __table_(__u.__table_) {
-  __table_.__rehash_multi(__u.bucket_count());
-  insert(__u.begin(), __u.end());
-}
-
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
     const unordered_multimap& __u, const allocator_type& __a)
@@ -2332,11 +2275,6 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
-inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(unordered_multimap&& __u)
-    _NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
-    : __table_(std::move(__u.__table_)) {}
-
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
     unordered_multimap&& __u, const allocator_type& __a)
@@ -2373,14 +2311,6 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
   insert(__il.begin(), __il.end());
 }
 
-template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
-inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
-unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_multimap&& __u)
-    _NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
-  __table_ = std::move(__u.__table_);
-  return *this;
-}
-
 template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
 inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
 unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
diff --git a/libcxx/include/unordered_set b/libcxx/include/unordered_set
index 475715db62bdb..4bc9c1afa2639 100644
--- a/libcxx/include/unordered_set
+++ b/libcxx/include/unordered_set
@@ -703,10 +703,10 @@ public:
 #  endif
 
   _LIBCPP_HIDE_FROM_ABI explicit unordered_set(const allocator_type& __a);
-  _LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u);
+  _LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_set(const unordered_set& __u, const allocator_type& __a);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_set(unordered_set&& __u, const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_set(initializer_list<value_type> __il);
   _LIBCPP_HIDE_FROM_ABI
@@ -733,13 +733,9 @@ public:
     static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), "");
   }
 
-  _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(const unordered_set& __u) {
-    __table_ = __u.__table_;
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(const unordered_set& __u) = default;
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(unordered_set&& __u)
-      _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(unordered_set&& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_set& operator=(initializer_list<value_type> __il);
 #  endif // _LIBCPP_CXX03_LANG
 
@@ -1070,12 +1066,6 @@ unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 inline unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const allocator_type& __a) : __table_(__a) {}
 
-template <class _Value, class _Hash, class _Pred, class _Alloc>
-unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const unordered_set& __u) : __table_(__u.__table_) {
-  __table_.__rehash_unique(__u.bucket_count());
-  insert(__u.begin(), __u.end());
-}
-
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const unordered_set& __u, const allocator_type& __a)
     : __table_(__u.__table_, __a) {
@@ -1085,11 +1075,6 @@ unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(const unordered_set&
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-template <class _Value, class _Hash, class _Pred, class _Alloc>
-inline unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(unordered_set&& __u)
-    _NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
-    : __table_(std::move(__u.__table_)) {}
-
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(unordered_set&& __u, const allocator_type& __a)
     : __table_(std::move(__u.__table_), __a) {
@@ -1125,14 +1110,6 @@ unordered_set<_Value, _Hash, _Pred, _Alloc>::unordered_set(
   insert(__il.begin(), __il.end());
 }
 
-template <class _Value, class _Hash, class _Pred, class _Alloc>
-inline unordered_set<_Value, _Hash, _Pred, _Alloc>&
-unordered_set<_Value, _Hash, _Pred, _Alloc>::operator=(unordered_set&& __u)
-    _NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
-  __table_ = std::move(__u.__table_);
-  return *this;
-}
-
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 inline unordered_set<_Value, _Hash, _Pred, _Alloc>&
 unordered_set<_Value, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
@@ -1308,11 +1285,10 @@ public:
 #  endif
 
   _LIBCPP_HIDE_FROM_ABI explicit unordered_multiset(const allocator_type& __a);
-  _LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u);
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(const unordered_multiset& __u, const allocator_type& __a);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u)
-      _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(unordered_multiset&& __u, const allocator_type& __a);
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(initializer_list<value_type> __il);
   _LIBCPP_HIDE_FROM_ABI unordered_multiset(
@@ -1339,13 +1315,9 @@ public:
     static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Value, _Hash, _Pred>(0)), "");
   }
 
-  _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(const unordered_multiset& __u) {
-    __table_ = __u.__table_;
-    return *this;
-  }
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(const unordered_multiset& __u) = default;
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(unordered_multiset&& __u)
-      _NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
+  _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(unordered_multiset&& __u) = default;
   _LIBCPP_HIDE_FROM_ABI unordered_multiset& operator=(initializer_list<value_type> __il);
 #  endif // _LIBCPP_CXX03_LANG
 
@@ -1685,13 +1657,6 @@ template <class _Value, class _Hash, class _Pred, class _Alloc>
 inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(const allocator_type& __a)
     : __table_(__a) {}
 
-template <class _Value, class _Hash, class _Pred, class _Alloc>
-unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(const unordered_multiset& __u)
-    : __table_(__u.__table_) {
-  __table_.__rehash_multi(__u.bucket_count());
-  insert(__u.begin(), __u.end());
-}
-
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
     const unordered_multiset& __u, const allocator_type& __a)
@@ -1702,11 +1667,6 @@ unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
 
 #  ifndef _LIBCPP_CXX03_LANG
 
-template <class _Value, class _Hash, class _Pred, class _Alloc>
-inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(unordered_multiset&& __u)
-    _NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
-    : __table_(std::move(__u.__table_)) {}
-
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
     unordered_multiset&& __u, const allocator_type& __a)
@@ -1743,14 +1703,6 @@ unordered_multiset<_Value, _Hash, _Pred, _Alloc>::unordered_multiset(
   insert(__il.begin(), __il.end());
 }
 
-template <class _Value, class _Hash, class _Pred, class _Alloc>
-inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>&
-unordered_multiset<_Value, _Hash, _Pred, _Alloc>::operator=(unordered_multiset&& __u)
-    _NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
-  __table_ = std::move(__u.__table_);
-  return *this;
-}
-
 template <class _Value, class _Hash, class _Pred, class _Alloc>
 inline unordered_multiset<_Value, _Hash, _Pred, _Alloc>&
 unordered_multiset<_Value, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
diff --git a/libcxx/include/version b/libcxx/include/version
index aae9277a7dfc6..16917a3bd9ddd 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -585,7 +585,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # define __cpp_lib_mdspan                               202406L
 # undef  __cpp_lib_not_fn
 # define __cpp_lib_not_fn                               202306L
-// # define __cpp_lib_optional_range_support               202406L
+# define __cpp_lib_optional_range_support               202406L
 # undef  __cpp_lib_out_ptr
 # define __cpp_lib_out_ptr                              202311L
 // # define __cpp_lib_philox_engine                        202406L
diff --git a/libcxx/modules/std/optional.inc b/libcxx/modules/std/optional.inc
index 0f812bc0e24a4..9ee51117277ce 100644
--- a/libcxx/modules/std/optional.inc
+++ b/libcxx/modules/std/optional.inc
@@ -10,7 +10,12 @@
 export namespace std {
   // [optional.optional], class template optional
   using std::optional;
-
+#if _LIBCPP_STD_VER >= 26
+  // [optional.iterators], iterator support
+  namespace ranges {
+    using std::ranges::enable_view;
+  }
+#endif
   // [optional.nullopt], no-value state indicator
   using std::nullopt;
   using std::nullopt_t;
@@ -18,6 +23,10 @@ export namespace std {
   // [optional.bad.access], class bad_optional_access
   using std::bad_optional_access;
 
+#if _LIBCPP_STD_VER >= 26
+  using std::format_kind;
+#endif
+
   // [optional.relops], relational operators
   using std::operator==;
   using std::operator!=;
diff --git a/libcxx/src/filesystem/path.cpp b/libcxx/src/filesystem/path.cpp
index 9f7dc54fdf156..400b6e8988569 100644
--- a/libcxx/src/filesystem/path.cpp
+++ b/libcxx/src/filesystem/path.cpp
@@ -292,7 +292,9 @@ path path::lexically_relative(const path& base) const {
   // return a path constructed with 'n' dot-dot elements, followed by the
   // elements of '*this' after the mismatch.
   path Result;
-  // FIXME: Reserve enough room in Result that it won't have to re-allocate.
+  constexpr size_t ElemSize      = 2; // ".."
+  constexpr size_t SeparatorSize = 1; // separator is always a single char
+  Result.__reserve(ElemCount * (ElemSize + SeparatorSize) + SeparatorSize + PP.Path.size());
   while (ElemCount--)
     Result /= PATHSTR("..");
   for (; PP; ++PP)
diff --git a/libcxx/test/benchmarks/filesystem.bench.cpp b/libcxx/test/benchmarks/filesystem.bench.cpp
index dc6b0ac537f7e..c058a5d41a150 100644
--- a/libcxx/test/benchmarks/filesystem.bench.cpp
+++ b/libcxx/test/benchmarks/filesystem.bench.cpp
@@ -171,4 +171,24 @@ BENCHMARK_CAPTURE(BM_LexicallyNormal, large_path, getRandomPaths, /*PathLen*/ 32
     ->Range(2, 256)
     ->Complexity();
 
+template <class GenInput>
+void BM_LexicallyRelative(benchmark::State& st, GenInput gen, size_t PathLen) {
+  auto BasePath   = gen(st.range(0), PathLen);
+  auto TargetPath = gen(st.range(0), PathLen);
+  benchmark::DoNotOptimize(&BasePath);
+  benchmark::DoNotOptimize(&TargetPath);
+  for (auto _ : st) {
+    benchmark::DoNotOptimize(TargetPath.lexically_relative(BasePath));
+  }
+  st.SetComplexityN(st.range(0));
+}
+BENCHMARK_CAPTURE(BM_LexicallyRelative, small_path, getRandomPaths, /*PathLen*/ 5)
+    ->RangeMultiplier(2)
+    ->Range(2, 256)
+    ->Complexity();
+BENCHMARK_CAPTURE(BM_LexicallyRelative, large_path, getRandomPaths, /*PathLen*/ 32)
+    ->RangeMultiplier(2)
+    ->Range(2, 256)
+    ->Complexity();
+
 BENCHMARK_MAIN();
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp b/libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp
rename to libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_annotations_not_enabled.pass.cpp
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp b/libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_lock_guard.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_guard.pass.cpp
rename to libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_lock_guard.pass.cpp
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp b/libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp
rename to libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_lock_unlock.pass.cpp
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp b/libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
similarity index 100%
rename from libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
rename to libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp b/libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_requires_capability.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/thread/thread.mutex/thread_safety_requires_capability.pass.cpp
rename to libcxx/test/extensions/clang/thread/thread.mutex/thread_safety_requires_capability.pass.cpp
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/fstream.cons/wchar_pointer.pass.cpp b/libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/fstream.cons/wchar_pointer.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/input.output/file.streams/fstreams/fstream.cons/wchar_pointer.pass.cpp
rename to libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/fstream.cons/wchar_pointer.pass.cpp
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/fstream.members/open_wchar_pointer.pass.cpp b/libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/fstream.members/open_wchar_pointer.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/input.output/file.streams/fstreams/fstream.members/open_wchar_pointer.pass.cpp
rename to libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/fstream.members/open_wchar_pointer.pass.cpp
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/ifstream.cons/test.dat b/libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ifstream.cons/test.dat
similarity index 100%
rename from libcxx/test/libcxx/input.output/file.streams/fstreams/ifstream.cons/test.dat
rename to libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ifstream.cons/test.dat
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/ifstream.cons/wchar_pointer.pass.cpp b/libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ifstream.cons/wchar_pointer.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/input.output/file.streams/fstreams/ifstream.cons/wchar_pointer.pass.cpp
rename to libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ifstream.cons/wchar_pointer.pass.cpp
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/ifstream.members/open_wchar_pointer.pass.cpp b/libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ifstream.members/open_wchar_pointer.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/input.output/file.streams/fstreams/ifstream.members/open_wchar_pointer.pass.cpp
rename to libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ifstream.members/open_wchar_pointer.pass.cpp
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/ifstream.members/test.dat b/libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ifstream.members/test.dat
similarity index 100%
rename from libcxx/test/libcxx/input.output/file.streams/fstreams/ifstream.members/test.dat
rename to libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ifstream.members/test.dat
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/ofstream.cons/wchar_pointer.pass.cpp b/libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ofstream.cons/wchar_pointer.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/input.output/file.streams/fstreams/ofstream.cons/wchar_pointer.pass.cpp
rename to libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ofstream.cons/wchar_pointer.pass.cpp
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/ofstream.members/open_wchar_pointer.pass.cpp b/libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ofstream.members/open_wchar_pointer.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/input.output/file.streams/fstreams/ofstream.members/open_wchar_pointer.pass.cpp
rename to libcxx/test/extensions/libcxx/input.output/file.streams/fstreams/ofstream.members/open_wchar_pointer.pass.cpp
diff --git a/libcxx/test/libcxx/iterators/stream.iterators/ostreambuf.iterator/ostreambuf.iter.ops/failed.pass.cpp b/libcxx/test/extensions/libcxx/iterators/stream.iterators/ostreambuf.iterator/ostreambuf.iter.ops/failed.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/iterators/stream.iterators/ostreambuf.iterator/ostreambuf.iter.ops/failed.pass.cpp
rename to libcxx/test/extensions/libcxx/iterators/stream.iterators/ostreambuf.iterator/ostreambuf.iter.ops/failed.pass.cpp
diff --git a/libcxx/test/libcxx/odr_signature.exceptions.sh.cpp b/libcxx/test/extensions/libcxx/odr_signature.exceptions.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/odr_signature.exceptions.sh.cpp
rename to libcxx/test/extensions/libcxx/odr_signature.exceptions.sh.cpp
diff --git a/libcxx/test/libcxx/odr_signature.hardening.sh.cpp b/libcxx/test/extensions/libcxx/odr_signature.hardening.sh.cpp
similarity index 100%
rename from libcxx/test/libcxx/odr_signature.hardening.sh.cpp
rename to libcxx/test/extensions/libcxx/odr_signature.hardening.sh.cpp
diff --git a/libcxx/test/libcxx/strings/basic.string/string.capacity/PR53170.pass.cpp b/libcxx/test/extensions/libcxx/strings/basic.string/string.capacity/PR53170.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/strings/basic.string/string.capacity/PR53170.pass.cpp
rename to libcxx/test/extensions/libcxx/strings/basic.string/string.capacity/PR53170.pass.cpp
diff --git a/libcxx/test/libcxx/xopen_source.gen.py b/libcxx/test/extensions/posix/xopen_source.gen.py
similarity index 100%
rename from libcxx/test/libcxx/xopen_source.gen.py
rename to libcxx/test/extensions/posix/xopen_source.gen.py
diff --git a/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_add.verify.cpp b/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_add.verify.cpp
index 320ef57dcb6f9..222799f9fa2e7 100644
--- a/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_add.verify.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_add.verify.cpp
@@ -48,12 +48,12 @@ void pointer_to_incomplete_type() {
 void function_pointer() {
   {
     volatile std::atomic<void (*)(int)> fun;
-    // expected-error-re@*:* {{static assertion failed due to requirement '!is_function<void (int)>::value'{{.*}}Pointer to function isn't allowed}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Pointer to function isn't allowed}}
     std::atomic_fetch_add(&fun, 0);
   }
   {
     std::atomic<void (*)(int)> fun;
-    // expected-error-re@*:* {{static assertion failed due to requirement '!is_function<void (int)>::value'{{.*}}Pointer to function isn't allowed}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Pointer to function isn't allowed}}
     std::atomic_fetch_add(&fun, 0);
   }
 }
diff --git a/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_add_explicit.verify.cpp b/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_add_explicit.verify.cpp
index bdd8089feb281..3bde3ad1c3384 100644
--- a/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_add_explicit.verify.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_add_explicit.verify.cpp
@@ -51,12 +51,12 @@ void pointer_to_incomplete_type() {
 void function_pointer() {
   {
     volatile std::atomic<void (*)(int)> fun;
-    // expected-error-re@*:* {{static assertion failed due to requirement '!is_function<void (int)>::value'{{.*}}Pointer to function isn't allowed}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Pointer to function isn't allowed}}
     std::atomic_fetch_add_explicit(&fun, 0, std::memory_order_relaxed);
   }
   {
     std::atomic<void (*)(int)> fun;
-    // expected-error-re@*:* {{static assertion failed due to requirement '!is_function<void (int)>::value'{{.*}}Pointer to function isn't allowed}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Pointer to function isn't allowed}}
     std::atomic_fetch_add_explicit(&fun, 0, std::memory_order_relaxed);
   }
 }
diff --git a/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_sub.verify.cpp b/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_sub.verify.cpp
index 2c9f89891d5be..805ca346c8540 100644
--- a/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_sub.verify.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_sub.verify.cpp
@@ -48,12 +48,12 @@ void pointer_to_incomplete_type() {
 void function_pointer() {
   {
     volatile std::atomic<void (*)(int)> fun;
-    // expected-error-re@*:* {{static assertion failed due to requirement '!is_function<void (int)>::value'{{.*}}Pointer to function isn't allowed}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Pointer to function isn't allowed}}
     std::atomic_fetch_sub(&fun, 0);
   }
   {
     std::atomic<void (*)(int)> fun;
-    // expected-error-re@*:* {{static assertion failed due to requirement '!is_function<void (int)>::value'{{.*}}Pointer to function isn't allowed}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Pointer to function isn't allowed}}
     std::atomic_fetch_sub(&fun, 0);
   }
 }
diff --git a/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_sub_explicit.verify.cpp b/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_sub_explicit.verify.cpp
index 88c42750b608a..37a4237a9b214 100644
--- a/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_sub_explicit.verify.cpp
+++ b/libcxx/test/libcxx/atomics/atomics.types.operations/atomics.types.operations.req/atomic_fetch_sub_explicit.verify.cpp
@@ -51,12 +51,12 @@ void pointer_to_incomplete_type() {
 void function_pointer() {
   {
     volatile std::atomic<void (*)(int)> fun;
-    // expected-error-re@*:* {{static assertion failed due to requirement '!is_function<void (int)>::value'{{.*}}Pointer to function isn't allowed}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Pointer to function isn't allowed}}
     std::atomic_fetch_sub_explicit(&fun, 0, std::memory_order_relaxed);
   }
   {
     std::atomic<void (*)(int)> fun;
-    // expected-error-re@*:* {{static assertion failed due to requirement '!is_function<void (int)>::value'{{.*}}Pointer to function isn't allowed}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Pointer to function isn't allowed}}
     std::atomic_fetch_sub_explicit(&fun, 0, std::memory_order_relaxed);
   }
 }
diff --git a/libcxx/test/libcxx/containers/associative/map/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/associative/map/abi.compile.pass.cpp
new file mode 100644
index 0000000000000..e0598b4ff1746
--- /dev/null
+++ b/libcxx/test/libcxx/containers/associative/map/abi.compile.pass.cpp
@@ -0,0 +1,168 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: libcpp-abi-no-compressed-pair-padding
+
+#include <cstdint>
+#include <map>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+template <class T>
+class small_pointer {
+  std::uint16_t offset;
+};
+
+template <class T>
+class small_iter_allocator {
+public:
+  using value_type      = T;
+  using pointer         = small_pointer<T>;
+  using size_type       = std::uint16_t;
+  using difference_type = std::int16_t;
+
+  small_iter_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  small_iter_allocator(small_iter_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(small_iter_allocator, small_iter_allocator) { return true; }
+  friend bool operator!=(small_iter_allocator, small_iter_allocator) { return false; }
+};
+
+template <class T>
+class final_small_iter_allocator final {
+public:
+  using value_type      = T;
+  using pointer         = small_pointer<T>;
+  using size_type       = std::uint16_t;
+  using difference_type = std::int16_t;
+
+  final_small_iter_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  final_small_iter_allocator(final_small_iter_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(final_small_iter_allocator, final_small_iter_allocator) { return true; }
+  friend bool operator!=(final_small_iter_allocator, final_small_iter_allocator) { return false; }
+};
+
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
+template <class T, class Alloc>
+using map_alloc = std::map<T, T, std::less<T>, Alloc>;
+
+struct user_struct {
+  map_alloc<int, common_base_allocator<std::pair<const int, int> > > v;
+  [[no_unique_address]] common_base_allocator<int> a;
+};
+
+#if __SIZE_WIDTH__ == 64
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 32, "");
+#  else
+static_assert(sizeof(user_struct) == 24, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 8, "");
+
+static_assert(sizeof(map_alloc<int, std::allocator<std::pair<const int, int> > >) == 24, "");
+static_assert(sizeof(map_alloc<int, min_allocator<std::pair<const int, int> > >) == 24, "");
+static_assert(sizeof(map_alloc<int, test_allocator<std::pair<const int, int> > >) == 40, "");
+static_assert(sizeof(map_alloc<int, small_iter_allocator<std::pair<const int, int> > >) == 6, "");
+static_assert(sizeof(map_alloc<int, final_small_iter_allocator<std::pair<const int, int> > >) == 8, "");
+
+static_assert(sizeof(map_alloc<char, std::allocator<std::pair<const char, char> > >) == 24, "");
+static_assert(sizeof(map_alloc<char, min_allocator<std::pair<const char, char> > >) == 24, "");
+static_assert(sizeof(map_alloc<char, test_allocator<std::pair<const char, char> > >) == 40, "");
+static_assert(sizeof(map_alloc<char, small_iter_allocator<std::pair<const char, char> > >) == 6, "");
+static_assert(sizeof(map_alloc<char, final_small_iter_allocator<std::pair<const char, char> > >) == 8, "");
+
+static_assert(TEST_ALIGNOF(map_alloc<int, std::allocator<std::pair<const int, int> > >) == 8, "");
+static_assert(TEST_ALIGNOF(map_alloc<int, min_allocator<std::pair<const int, int> > >) == 8, "");
+static_assert(TEST_ALIGNOF(map_alloc<int, test_allocator<std::pair<const int, int> > >) == 8, "");
+static_assert(TEST_ALIGNOF(map_alloc<int, small_iter_allocator<std::pair<const int, int> > >) == 2, "");
+static_assert(TEST_ALIGNOF(map_alloc<int, final_small_iter_allocator<std::pair<const int, int> > >) == 2, "");
+
+static_assert(TEST_ALIGNOF(map_alloc<char, std::allocator<std::pair<const char, char> > >) == 8, "");
+static_assert(TEST_ALIGNOF(map_alloc<char, min_allocator<std::pair<const char, char> > >) == 8, "");
+static_assert(TEST_ALIGNOF(map_alloc<char, test_allocator<std::pair<const char, char> > >) == 8, "");
+static_assert(TEST_ALIGNOF(map_alloc<char, small_iter_allocator<std::pair<const char, char> > >) == 2, "");
+static_assert(TEST_ALIGNOF(map_alloc<char, final_small_iter_allocator<std::pair<const char, char> > >) == 2, "");
+
+struct TEST_ALIGNAS(32) AlignedLess {};
+
+static_assert(sizeof(std::map<int, int, AlignedLess>) == 64, "");
+static_assert(TEST_ALIGNOF(std::map<int, int, AlignedLess>) == 32, "");
+
+#elif __SIZE_WIDTH__ == 32
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 16, "");
+#  else
+static_assert(sizeof(user_struct) == 12, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 4, "");
+
+static_assert(sizeof(map_alloc<int, std::allocator<std::pair<const int, int> > >) == 12, "");
+static_assert(sizeof(map_alloc<int, min_allocator<std::pair<const int, int> > >) == 12, "");
+static_assert(sizeof(map_alloc<int, test_allocator<std::pair<const int, int> > >) == 24, "");
+static_assert(sizeof(map_alloc<int, small_iter_allocator<std::pair<const int, int> > >) == 6, "");
+static_assert(sizeof(map_alloc<int, final_small_iter_allocator<std::pair<const int, int> > >) == 8, "");
+
+static_assert(sizeof(map_alloc<char, std::allocator<std::pair<const char, char> > >) == 12, "");
+static_assert(sizeof(map_alloc<char, min_allocator<std::pair<const char, char> > >) == 12, "");
+static_assert(sizeof(map_alloc<char, test_allocator<std::pair<const char, char> > >) == 24, "");
+static_assert(sizeof(map_alloc<char, small_iter_allocator<std::pair<const char, char> > >) == 6, "");
+static_assert(sizeof(map_alloc<char, final_small_iter_allocator<std::pair<const char, char> > >) == 8, "");
+
+static_assert(TEST_ALIGNOF(map_alloc<int, std::allocator<std::pair<const int, int> > >) == 4, "");
+static_assert(TEST_ALIGNOF(map_alloc<int, min_allocator<std::pair<const int, int> > >) == 4, "");
+static_assert(TEST_ALIGNOF(map_alloc<int, test_allocator<std::pair<const int, int> > >) == 4, "");
+static_assert(TEST_ALIGNOF(map_alloc<int, small_iter_allocator<std::pair<const int, int> > >) == 2, "");
+static_assert(TEST_ALIGNOF(map_alloc<int, final_small_iter_allocator<std::pair<const int, int> > >) == 2, "");
+
+static_assert(TEST_ALIGNOF(map_alloc<char, std::allocator<std::pair<const char, char> > >) == 4, "");
+static_assert(TEST_ALIGNOF(map_alloc<char, min_allocator<std::pair<const char, char> > >) == 4, "");
+static_assert(TEST_ALIGNOF(map_alloc<char, test_allocator<std::pair<const char, char> > >) == 4, "");
+static_assert(TEST_ALIGNOF(map_alloc<char, small_iter_allocator<std::pair<const char, char> > >) == 2, "");
+static_assert(TEST_ALIGNOF(map_alloc<char, final_small_iter_allocator<std::pair<const char, char> > >) == 2, "");
+
+struct TEST_ALIGNAS(32) AlignedLess {};
+
+static_assert(sizeof(std::map<int, int, AlignedLess>) == 64);
+static_assert(TEST_ALIGNOF(std::map<int, int, AlignedLess>) == 32);
+
+#else
+#  error std::size_t has an unexpected size
+#endif
diff --git a/libcxx/test/libcxx/containers/associative/set/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/associative/set/abi.compile.pass.cpp
new file mode 100644
index 0000000000000..14f89b6c9deae
--- /dev/null
+++ b/libcxx/test/libcxx/containers/associative/set/abi.compile.pass.cpp
@@ -0,0 +1,168 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: libcpp-abi-no-compressed-pair-padding
+
+#include <cstdint>
+#include <set>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+template <class T>
+class small_pointer {
+  std::uint16_t offset;
+};
+
+template <class T>
+class small_iter_allocator {
+public:
+  using value_type      = T;
+  using pointer         = small_pointer<T>;
+  using size_type       = std::uint16_t;
+  using difference_type = std::int16_t;
+
+  small_iter_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  small_iter_allocator(small_iter_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(small_iter_allocator, small_iter_allocator) { return true; }
+  friend bool operator!=(small_iter_allocator, small_iter_allocator) { return false; }
+};
+
+template <class T>
+class final_small_iter_allocator final {
+public:
+  using value_type      = T;
+  using pointer         = small_pointer<T>;
+  using size_type       = std::uint16_t;
+  using difference_type = std::int16_t;
+
+  final_small_iter_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  final_small_iter_allocator(final_small_iter_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(final_small_iter_allocator, final_small_iter_allocator) { return true; }
+  friend bool operator!=(final_small_iter_allocator, final_small_iter_allocator) { return false; }
+};
+
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
+template <class T, class Alloc>
+using set_alloc = std::set<T, std::less<T>, Alloc>;
+
+struct user_struct {
+  set_alloc<int, common_base_allocator<int> > v;
+  [[no_unique_address]] common_base_allocator<int> a;
+};
+
+#if __SIZE_WIDTH__ == 64
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 32, "");
+#  else
+static_assert(sizeof(user_struct) == 24, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 8, "");
+
+static_assert(sizeof(set_alloc<int, std::allocator<int> >) == 24, "");
+static_assert(sizeof(set_alloc<int, min_allocator<int> >) == 24, "");
+static_assert(sizeof(set_alloc<int, test_allocator<int> >) == 40, "");
+static_assert(sizeof(set_alloc<int, small_iter_allocator<int> >) == 6, "");
+static_assert(sizeof(set_alloc<int, final_small_iter_allocator<int> >) == 8, "");
+
+static_assert(sizeof(set_alloc<char, std::allocator<char> >) == 24, "");
+static_assert(sizeof(set_alloc<char, min_allocator<char> >) == 24, "");
+static_assert(sizeof(set_alloc<char, test_allocator<char> >) == 40, "");
+static_assert(sizeof(set_alloc<char, small_iter_allocator<char> >) == 6, "");
+static_assert(sizeof(set_alloc<char, final_small_iter_allocator<char> >) == 8, "");
+
+static_assert(TEST_ALIGNOF(set_alloc<int, std::allocator<int> >) == 8, "");
+static_assert(TEST_ALIGNOF(set_alloc<int, min_allocator<int> >) == 8, "");
+static_assert(TEST_ALIGNOF(set_alloc<int, test_allocator<int> >) == 8, "");
+static_assert(TEST_ALIGNOF(set_alloc<int, small_iter_allocator<int> >) == 2, "");
+static_assert(TEST_ALIGNOF(set_alloc<int, final_small_iter_allocator<int> >) == 2, "");
+
+static_assert(TEST_ALIGNOF(set_alloc<char, std::allocator<char> >) == 8, "");
+static_assert(TEST_ALIGNOF(set_alloc<char, min_allocator<char> >) == 8, "");
+static_assert(TEST_ALIGNOF(set_alloc<char, test_allocator<char> >) == 8, "");
+static_assert(TEST_ALIGNOF(set_alloc<char, small_iter_allocator<char> >) == 2, "");
+static_assert(TEST_ALIGNOF(set_alloc<char, final_small_iter_allocator<char> >) == 2, "");
+
+struct TEST_ALIGNAS(32) AlignedLess {};
+
+static_assert(sizeof(std::set<int, AlignedLess>) == 64, "");
+static_assert(TEST_ALIGNOF(std::set<int, AlignedLess>) == 32, "");
+
+#elif __SIZE_WIDTH__ == 32
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 16, "");
+#  else
+static_assert(sizeof(user_struct) == 12, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 4, "");
+
+static_assert(sizeof(set_alloc<int, std::allocator<int> >) == 12, "");
+static_assert(sizeof(set_alloc<int, min_allocator<int> >) == 12, "");
+static_assert(sizeof(set_alloc<int, test_allocator<int> >) == 24, "");
+static_assert(sizeof(set_alloc<int, small_iter_allocator<int> >) == 6, "");
+static_assert(sizeof(set_alloc<int, final_small_iter_allocator<int> >) == 8, "");
+
+static_assert(sizeof(set_alloc<char, std::allocator<char> >) == 12, "");
+static_assert(sizeof(set_alloc<char, min_allocator<char> >) == 12, "");
+static_assert(sizeof(set_alloc<char, test_allocator<char> >) == 24, "");
+static_assert(sizeof(set_alloc<char, small_iter_allocator<char> >) == 6, "");
+static_assert(sizeof(set_alloc<char, final_small_iter_allocator<char> >) == 8, "");
+
+static_assert(TEST_ALIGNOF(set_alloc<int, std::allocator<int> >) == 4, "");
+static_assert(TEST_ALIGNOF(set_alloc<int, min_allocator<int> >) == 4, "");
+static_assert(TEST_ALIGNOF(set_alloc<int, test_allocator<int> >) == 4, "");
+static_assert(TEST_ALIGNOF(set_alloc<int, small_iter_allocator<int> >) == 2, "");
+static_assert(TEST_ALIGNOF(set_alloc<int, final_small_iter_allocator<int> >) == 2, "");
+
+static_assert(TEST_ALIGNOF(set_alloc<char, std::allocator<char> >) == 4, "");
+static_assert(TEST_ALIGNOF(set_alloc<char, min_allocator<char> >) == 4, "");
+static_assert(TEST_ALIGNOF(set_alloc<char, test_allocator<char> >) == 4, "");
+static_assert(TEST_ALIGNOF(set_alloc<char, small_iter_allocator<char> >) == 2, "");
+static_assert(TEST_ALIGNOF(set_alloc<char, final_small_iter_allocator<char> >) == 2, "");
+
+struct TEST_ALIGNAS(32) AlignedLess {};
+
+static_assert(sizeof(std::set<int, AlignedLess>) == 64);
+static_assert(TEST_ALIGNOF(std::set<int, AlignedLess>) == 32);
+
+#else
+#  error std::size_t has an unexpected size
+#endif
diff --git a/libcxx/test/libcxx/containers/associative/unord.map/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/associative/unord.map/abi.compile.pass.cpp
index 55d42a8d017e1..60c3e5bf31b00 100644
--- a/libcxx/test/libcxx/containers/associative/unord.map/abi.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/unord.map/abi.compile.pass.cpp
@@ -12,8 +12,6 @@
 // unordered containers changes when bounded unique_ptr is enabled.
 // UNSUPPORTED: libcpp-has-abi-bounded-unique_ptr
 
-// XFAIL: FROZEN-CXX03-HEADERS-FIXME
-
 #include <cstdint>
 #include <unordered_map>
 
@@ -66,10 +64,41 @@ class final_small_iter_allocator final {
   friend bool operator!=(final_small_iter_allocator, final_small_iter_allocator) { return false; }
 };
 
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
 template <class T, class Alloc>
 using unordered_map_alloc = std::unordered_map<T, T, std::hash<T>, std::equal_to<T>, Alloc>;
 
+struct user_struct {
+  unordered_map_alloc<int, common_base_allocator<std::pair<const int, int> > > v;
+  [[no_unique_address]] common_base_allocator<int> a;
+};
+
 #if __SIZE_WIDTH__ == 64
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 48, "");
+#  else
+static_assert(sizeof(user_struct) == 40, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 8, "");
 
 static_assert(sizeof(unordered_map_alloc<int, std::allocator<std::pair<const int, int> > >) == 40, "");
 static_assert(sizeof(unordered_map_alloc<int, min_allocator<std::pair<const int, int> > >) == 40, "");
@@ -98,12 +127,22 @@ static_assert(TEST_ALIGNOF(unordered_map_alloc<char, final_small_iter_allocator<
 
 struct TEST_ALIGNAS(32) AlignedHash {};
 struct UnalignedEqualTo {};
-
-// This part of the ABI has been broken between LLVM 19 and LLVM 20.
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
 static_assert(sizeof(std::unordered_map<int, int, AlignedHash, UnalignedEqualTo>) == 64, "");
+#  else
+static_assert(sizeof(std::unordered_map<int, int, AlignedHash, UnalignedEqualTo>) == 96, "");
+#  endif
 static_assert(TEST_ALIGNOF(std::unordered_map<int, int, AlignedHash, UnalignedEqualTo>) == 32, "");
 
 #elif __SIZE_WIDTH__ == 32
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 24, "");
+#  else
+static_assert(sizeof(user_struct) == 20, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 4, "");
 
 static_assert(sizeof(unordered_map_alloc<int, std::allocator<std::pair<const int, int> > >) == 20, "");
 static_assert(sizeof(unordered_map_alloc<int, min_allocator<std::pair<const int, int> > >) == 20, "");
@@ -133,7 +172,12 @@ static_assert(TEST_ALIGNOF(unordered_map_alloc<char, final_small_iter_allocator<
 struct TEST_ALIGNAS(32) AlignedHash {};
 struct UnalignedEqualTo {};
 
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
 static_assert(sizeof(std::unordered_map<int, int, AlignedHash, UnalignedEqualTo>) == 64);
+#  else
+static_assert(sizeof(std::unordered_map<int, int, AlignedHash, UnalignedEqualTo>) == 96);
+#  endif
 static_assert(TEST_ALIGNOF(std::unordered_map<int, int, AlignedHash, UnalignedEqualTo>) == 32);
 
 #else
diff --git a/libcxx/test/libcxx/containers/associative/unord.set/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/associative/unord.set/abi.compile.pass.cpp
index bee2012bbea29..0216cb1213edf 100644
--- a/libcxx/test/libcxx/containers/associative/unord.set/abi.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/associative/unord.set/abi.compile.pass.cpp
@@ -12,8 +12,6 @@
 // unordered containers changes when bounded unique_ptr is enabled.
 // UNSUPPORTED: libcpp-has-abi-bounded-unique_ptr
 
-// XFAIL: FROZEN-CXX03-HEADERS-FIXME
-
 #include <cstdint>
 #include <unordered_set>
 
@@ -66,10 +64,41 @@ class final_small_iter_allocator final {
   friend bool operator!=(final_small_iter_allocator, final_small_iter_allocator) { return false; }
 };
 
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
 template <class T, class Alloc>
 using unordered_set_alloc = std::unordered_set<T, std::hash<T>, std::equal_to<T>, Alloc>;
 
+struct user_struct {
+  unordered_set_alloc<int, common_base_allocator<int> > v;
+  [[no_unique_address]] common_base_allocator<int> a;
+};
+
 #if __SIZE_WIDTH__ == 64
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 48, "");
+#  else
+static_assert(sizeof(user_struct) == 40, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 8, "");
 
 static_assert(sizeof(unordered_set_alloc<int, std::allocator<int> >) == 40, "");
 static_assert(sizeof(unordered_set_alloc<int, min_allocator<int> >) == 40, "");
@@ -98,11 +127,22 @@ static_assert(TEST_ALIGNOF(unordered_set_alloc<char, final_small_iter_allocator<
 struct TEST_ALIGNAS(32) AlignedHash {};
 struct UnalignedEqualTo {};
 
-// This part of the ABI has been broken between LLVM 19 and LLVM 20.
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
 static_assert(sizeof(std::unordered_set<int, AlignedHash, UnalignedEqualTo>) == 64, "");
+#  else
+static_assert(sizeof(std::unordered_set<int, AlignedHash, UnalignedEqualTo>) == 96, "");
+#  endif
 static_assert(TEST_ALIGNOF(std::unordered_set<int, AlignedHash, UnalignedEqualTo>) == 32, "");
 
 #elif __SIZE_WIDTH__ == 32
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 24, "");
+#  else
+static_assert(sizeof(user_struct) == 20, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 4, "");
 
 static_assert(sizeof(unordered_set_alloc<int, std::allocator<int> >) == 20, "");
 static_assert(sizeof(unordered_set_alloc<int, min_allocator<int> >) == 20, "");
@@ -131,7 +171,12 @@ static_assert(TEST_ALIGNOF(unordered_set_alloc<char, final_small_iter_allocator<
 struct TEST_ALIGNAS(32) AlignedHash {};
 struct UnalignedEqualTo {};
 
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
 static_assert(sizeof(std::unordered_set<int, AlignedHash, UnalignedEqualTo>) == 64);
+#  else
+static_assert(sizeof(std::unordered_set<int, AlignedHash, UnalignedEqualTo>) == 96);
+#  endif
 static_assert(TEST_ALIGNOF(std::unordered_set<int, AlignedHash, UnalignedEqualTo>) == 32);
 
 #else
diff --git a/libcxx/test/libcxx/containers/sequences/deque/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/sequences/deque/abi.compile.pass.cpp
index 30586d8b2422c..7eaf64ea09d13 100644
--- a/libcxx/test/libcxx/containers/sequences/deque/abi.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/deque/abi.compile.pass.cpp
@@ -60,6 +60,25 @@ class final_small_iter_allocator final {
   friend bool operator!=(final_small_iter_allocator, final_small_iter_allocator) { return false; }
 };
 
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
 #if __SIZE_WIDTH__ == 64
 
 static_assert(sizeof(std::deque<int>) == 48, "");
@@ -67,24 +86,38 @@ static_assert(sizeof(std::deque<int, min_allocator<int> >) == 48, "");
 static_assert(sizeof(std::deque<int, test_allocator<int> >) == 80, "");
 static_assert(sizeof(std::deque<int, small_iter_allocator<int> >) == 12, "");
 static_assert(sizeof(std::deque<int, final_small_iter_allocator<int> >) == 16, "");
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(std::deque<int, common_base_allocator<int> >) == 56, "");
+#  else
+static_assert(sizeof(std::deque<int, common_base_allocator<int> >) == 48, "");
+#  endif
 
 static_assert(sizeof(std::deque<char>) == 48, "");
 static_assert(sizeof(std::deque<char, min_allocator<char> >) == 48, "");
 static_assert(sizeof(std::deque<char, test_allocator<char> >) == 80, "");
 static_assert(sizeof(std::deque<char, small_iter_allocator<char> >) == 12, "");
 static_assert(sizeof(std::deque<char, final_small_iter_allocator<char> >) == 16, "");
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(std::deque<char, common_base_allocator<char> >) == 56, "");
+#  else
+static_assert(sizeof(std::deque<char, common_base_allocator<char> >) == 48, "");
+#  endif
 
 static_assert(TEST_ALIGNOF(std::deque<int>) == 8, "");
 static_assert(TEST_ALIGNOF(std::deque<int, min_allocator<int> >) == 8, "");
 static_assert(TEST_ALIGNOF(std::deque<int, test_allocator<int> >) == 8, "");
 static_assert(TEST_ALIGNOF(std::deque<int, small_iter_allocator<int> >) == 2, "");
 static_assert(TEST_ALIGNOF(std::deque<int, final_small_iter_allocator<int> >) == 2, "");
+static_assert(TEST_ALIGNOF(std::deque<int, common_base_allocator<int> >) == 8, "");
 
 static_assert(TEST_ALIGNOF(std::deque<char>) == 8, "");
 static_assert(TEST_ALIGNOF(std::deque<char, min_allocator<char> >) == 8, "");
 static_assert(TEST_ALIGNOF(std::deque<char, test_allocator<char> >) == 8, "");
 static_assert(TEST_ALIGNOF(std::deque<char, small_iter_allocator<char> >) == 2, "");
 static_assert(TEST_ALIGNOF(std::deque<char, final_small_iter_allocator<char> >) == 2, "");
+static_assert(TEST_ALIGNOF(std::deque<char, common_base_allocator<char> >) == 8, "");
 
 #elif __SIZE_WIDTH__ == 32
 
@@ -93,24 +126,38 @@ static_assert(sizeof(std::deque<int, min_allocator<int> >) == 24, "");
 static_assert(sizeof(std::deque<int, test_allocator<int> >) == 48, "");
 static_assert(sizeof(std::deque<int, small_iter_allocator<int> >) == 12, "");
 static_assert(sizeof(std::deque<int, final_small_iter_allocator<int> >) == 16, "");
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(std::deque<int, common_base_allocator<int> >) == 28, "");
+#  else
+static_assert(sizeof(std::deque<int, common_base_allocator<int> >) == 24, "");
+#  endif
 
 static_assert(sizeof(std::deque<char>) == 24, "");
 static_assert(sizeof(std::deque<char, min_allocator<char> >) == 24, "");
 static_assert(sizeof(std::deque<char, test_allocator<char> >) == 48, "");
 static_assert(sizeof(std::deque<char, small_iter_allocator<char> >) == 12, "");
 static_assert(sizeof(std::deque<char, final_small_iter_allocator<char> >) == 16, "");
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(std::deque<char, common_base_allocator<char> >) == 28, "");
+#  else
+static_assert(sizeof(std::deque<char, common_base_allocator<char> >) == 24, "");
+#  endif
 
 static_assert(TEST_ALIGNOF(std::deque<int>) == 4, "");
 static_assert(TEST_ALIGNOF(std::deque<int, min_allocator<int> >) == 4, "");
 static_assert(TEST_ALIGNOF(std::deque<int, test_allocator<int> >) == 4, "");
 static_assert(TEST_ALIGNOF(std::deque<int, small_iter_allocator<int> >) == 2, "");
 static_assert(TEST_ALIGNOF(std::deque<int, final_small_iter_allocator<int> >) == 2, "");
+static_assert(TEST_ALIGNOF(std::deque<int, common_base_allocator<int> >) == 4, "");
 
 static_assert(TEST_ALIGNOF(std::deque<char>) == 4, "");
 static_assert(TEST_ALIGNOF(std::deque<char, min_allocator<char> >) == 4, "");
 static_assert(TEST_ALIGNOF(std::deque<char, test_allocator<char> >) == 4, "");
 static_assert(TEST_ALIGNOF(std::deque<char, small_iter_allocator<char> >) == 2, "");
 static_assert(TEST_ALIGNOF(std::deque<char, final_small_iter_allocator<char> >) == 2, "");
+static_assert(TEST_ALIGNOF(std::deque<char, common_base_allocator<char> >) == 4, "");
 
 #else
 #  error std::size_t has an unexpected size
diff --git a/libcxx/test/libcxx/containers/sequences/forwardlist/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/sequences/forwardlist/abi.compile.pass.cpp
new file mode 100644
index 0000000000000..cb500f9c4d61b
--- /dev/null
+++ b/libcxx/test/libcxx/containers/sequences/forwardlist/abi.compile.pass.cpp
@@ -0,0 +1,115 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstdint>
+#include <forward_list>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+template <class T>
+class small_pointer {
+  std::uint16_t offset;
+};
+
+template <class T>
+class small_iter_allocator {
+public:
+  using value_type      = T;
+  using pointer         = small_pointer<T>;
+  using size_type       = std::int16_t;
+  using difference_type = std::int16_t;
+
+  small_iter_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  small_iter_allocator(small_iter_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(small_iter_allocator, small_iter_allocator) { return true; }
+  friend bool operator!=(small_iter_allocator, small_iter_allocator) { return false; }
+};
+
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
+struct user_struct {
+  std::forward_list<int, common_base_allocator<int> > v;
+  [[no_unique_address]] common_base_allocator<int> a;
+};
+
+#if __SIZE_WIDTH__ == 64
+static_assert(sizeof(user_struct) == 16, "");
+static_assert(TEST_ALIGNOF(user_struct) == 8, "");
+
+static_assert(sizeof(std::forward_list<int>) == 8, "");
+static_assert(sizeof(std::forward_list<int, min_allocator<int> >) == 8, "");
+static_assert(sizeof(std::forward_list<int, test_allocator<int> >) == 24, "");
+static_assert(sizeof(std::forward_list<int, small_iter_allocator<int> >) == 2, "");
+
+static_assert(sizeof(std::forward_list<char>) == 8, "");
+static_assert(sizeof(std::forward_list<char, min_allocator<char> >) == 8, "");
+static_assert(sizeof(std::forward_list<char, test_allocator<char> >) == 24, "");
+static_assert(sizeof(std::forward_list<char, small_iter_allocator<char> >) == 2, "");
+
+static_assert(TEST_ALIGNOF(std::forward_list<int>) == 8, "");
+static_assert(TEST_ALIGNOF(std::forward_list<int, min_allocator<int> >) == 8, "");
+static_assert(TEST_ALIGNOF(std::forward_list<int, test_allocator<int> >) == 8, "");
+static_assert(TEST_ALIGNOF(std::forward_list<int, small_iter_allocator<int> >) == 2, "");
+
+static_assert(TEST_ALIGNOF(std::forward_list<char>) == 8, "");
+static_assert(TEST_ALIGNOF(std::forward_list<char, min_allocator<char> >) == 8, "");
+static_assert(TEST_ALIGNOF(std::forward_list<char, test_allocator<char> >) == 8, "");
+static_assert(TEST_ALIGNOF(std::forward_list<char, small_iter_allocator<char> >) == 2, "");
+
+#elif __SIZE_WIDTH__ == 32
+static_assert(sizeof(user_struct) == 8, "");
+static_assert(TEST_ALIGNOF(user_struct) == 4, "");
+
+static_assert(sizeof(std::forward_list<int>) == 4, "");
+static_assert(sizeof(std::forward_list<int, min_allocator<int> >) == 4, "");
+static_assert(sizeof(std::forward_list<int, test_allocator<int> >) == 16, "");
+static_assert(sizeof(std::forward_list<int, small_iter_allocator<int> >) == 2, "");
+
+static_assert(sizeof(std::forward_list<char>) == 4, "");
+static_assert(sizeof(std::forward_list<char, min_allocator<char> >) == 4, "");
+static_assert(sizeof(std::forward_list<char, test_allocator<char> >) == 16, "");
+static_assert(sizeof(std::forward_list<char, small_iter_allocator<char> >) == 2, "");
+
+static_assert(TEST_ALIGNOF(std::forward_list<int>) == 4, "");
+static_assert(TEST_ALIGNOF(std::forward_list<int, min_allocator<int> >) == 4, "");
+static_assert(TEST_ALIGNOF(std::forward_list<int, test_allocator<int> >) == 4, "");
+static_assert(TEST_ALIGNOF(std::forward_list<int, small_iter_allocator<int> >) == 2, "");
+
+static_assert(TEST_ALIGNOF(std::forward_list<char>) == 4, "");
+static_assert(TEST_ALIGNOF(std::forward_list<char, min_allocator<char> >) == 4, "");
+static_assert(TEST_ALIGNOF(std::forward_list<char, test_allocator<char> >) == 4, "");
+static_assert(TEST_ALIGNOF(std::forward_list<char, small_iter_allocator<char> >) == 2, "");
+
+#else
+#  error std::size_t has an unexpected size
+#endif
diff --git a/libcxx/test/libcxx/containers/sequences/list/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/sequences/list/abi.compile.pass.cpp
index a16ae1d527921..4b9c295c46f2c 100644
--- a/libcxx/test/libcxx/containers/sequences/list/abi.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/list/abi.compile.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: libcpp-abi-no-compressed-pair-padding
+
 #include <cstdint>
 #include <list>
 
@@ -38,7 +40,38 @@ class small_iter_allocator {
   friend bool operator!=(small_iter_allocator, small_iter_allocator) { return false; }
 };
 
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
+struct user_struct {
+  std::list<int, common_base_allocator<int> > v;
+  [[no_unique_address]] common_base_allocator<int> a;
+};
+
 #if __SIZE_WIDTH__ == 64
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 32, "");
+#  else
+static_assert(sizeof(user_struct) == 24, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 8, "");
 
 static_assert(sizeof(std::list<int>) == 24, "");
 static_assert(sizeof(std::list<int, min_allocator<int> >) == 24, "");
@@ -61,6 +94,13 @@ static_assert(TEST_ALIGNOF(std::list<char, test_allocator<char> >) == 8, "");
 static_assert(TEST_ALIGNOF(std::list<char, small_iter_allocator<char> >) == 2, "");
 
 #elif __SIZE_WIDTH__ == 32
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 16, "");
+#  else
+static_assert(sizeof(user_struct) == 12, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 4, "");
 
 static_assert(sizeof(std::list<int>) == 12, "");
 static_assert(sizeof(std::list<int, min_allocator<int> >) == 12, "");
diff --git a/libcxx/test/libcxx/containers/sequences/vector.bool/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector.bool/abi.compile.pass.cpp
index cc6b0d94e7daf..598d54ffb91a0 100644
--- a/libcxx/test/libcxx/containers/sequences/vector.bool/abi.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector.bool/abi.compile.pass.cpp
@@ -40,7 +40,37 @@ class small_iter_allocator {
   friend bool operator!=(small_iter_allocator, small_iter_allocator) { return false; }
 };
 
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
+struct user_struct {
+  std::vector<bool, common_base_allocator<bool> > v;
+  [[no_unique_address]] common_base_allocator<bool> a;
+};
+
 #if __SIZE_WIDTH__ == 64
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 32, "");
+#  else
+static_assert(sizeof(user_struct) == 24, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 8, "");
 
 static_assert(sizeof(std::vector<bool>) == 24, "");
 static_assert(sizeof(std::vector<bool, min_allocator<bool> >) == 24, "");
@@ -53,6 +83,13 @@ static_assert(TEST_ALIGNOF(std::vector<bool, test_allocator<bool> >) == 8, "");
 static_assert(TEST_ALIGNOF(std::vector<bool, small_iter_allocator<bool> >) == 2, "");
 
 #elif __SIZE_WIDTH__ == 32
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 16, "");
+#  else
+static_assert(sizeof(user_struct) == 12, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 4, "");
 
 static_assert(sizeof(std::vector<bool>) == 12, "");
 static_assert(sizeof(std::vector<bool, min_allocator<bool> >) == 12, "");
diff --git a/libcxx/test/libcxx/containers/sequences/vector/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/abi.compile.pass.cpp
index 57684951c8e8e..6dc0504991b39 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/abi.compile.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/abi.compile.pass.cpp
@@ -6,6 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+// UNSUPPORTED: libcpp-abi-no-compressed-pair-padding
+
 #include <cstdint>
 #include <vector>
 
@@ -46,7 +48,38 @@ class small_iter_allocator {
   friend bool operator!=(small_iter_allocator, small_iter_allocator) { return false; }
 };
 
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
+struct user_struct {
+  std::vector<int, common_base_allocator<int> > v;
+  [[no_unique_address]] common_base_allocator<int> a;
+};
+
 #if __SIZE_WIDTH__ == 64
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 32, "");
+#  else
+static_assert(sizeof(user_struct) == 24, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 8, "");
 
 static_assert(sizeof(std::vector<int>) == 24, "");
 static_assert(sizeof(std::vector<int, min_allocator<int> >) == 24, "");
@@ -69,6 +102,13 @@ static_assert(TEST_ALIGNOF(std::vector<char, test_allocator<char> >) == 8, "");
 static_assert(TEST_ALIGNOF(std::vector<char, small_iter_allocator<char> >) == 2, "");
 
 #elif __SIZE_WIDTH__ == 32
+// TODO: Fix the ABI for GCC as well once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121637 is fixed
+#  ifdef TEST_COMPILER_GCC
+static_assert(sizeof(user_struct) == 16, "");
+#  else
+static_assert(sizeof(user_struct) == 12, "");
+#  endif
+static_assert(TEST_ALIGNOF(user_struct) == 4, "");
 
 static_assert(sizeof(std::vector<int>) == 12, "");
 static_assert(sizeof(std::vector<int, min_allocator<int> >) == 12, "");
diff --git a/libcxx/test/libcxx/containers/strings/basic.string/abi.compile.pass.cpp b/libcxx/test/libcxx/containers/strings/basic.string/abi.compile.pass.cpp
new file mode 100644
index 0000000000000..cf802e214d07c
--- /dev/null
+++ b/libcxx/test/libcxx/containers/strings/basic.string/abi.compile.pass.cpp
@@ -0,0 +1,106 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstdint>
+#include <string>
+
+#include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
+
+template <class T>
+class small_pointer {
+public:
+  using value_type        = T;
+  using difference_type   = std::int16_t;
+  using pointer           = small_pointer;
+  using reference         = T&;
+  using iterator_category = std::random_access_iterator_tag;
+
+private:
+  std::uint16_t offset;
+};
+
+template <class T>
+class small_iter_allocator {
+public:
+  using value_type      = T;
+  using pointer         = small_pointer<T>;
+  using size_type       = std::int16_t;
+  using difference_type = std::int16_t;
+
+  small_iter_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  small_iter_allocator(small_iter_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(small_iter_allocator, small_iter_allocator) { return true; }
+  friend bool operator!=(small_iter_allocator, small_iter_allocator) { return false; }
+};
+
+struct allocator_base {};
+
+// Make sure that types with a common base type don't get broken. See https://llvm.org/PR154146
+template <class T>
+struct common_base_allocator : allocator_base {
+  using value_type = T;
+
+  common_base_allocator() TEST_NOEXCEPT {}
+
+  template <class U>
+  common_base_allocator(common_base_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n);
+  void deallocate(T* p, std::size_t);
+
+  friend bool operator==(common_base_allocator, common_base_allocator) { return true; }
+  friend bool operator!=(common_base_allocator, common_base_allocator) { return false; }
+};
+
+template <class Alloc>
+using string_alloc = std::basic_string<char, std::char_traits<char>, Alloc>;
+
+struct user_struct {
+  string_alloc<common_base_allocator<char> > v;
+  [[no_unique_address]] common_base_allocator<int> a;
+};
+
+#if __SIZE_WIDTH__ == 64
+static_assert(sizeof(user_struct) == 32, "");
+static_assert(TEST_ALIGNOF(user_struct) == 8, "");
+
+static_assert(sizeof(string_alloc<std::allocator<char> >) == 24, "");
+static_assert(sizeof(string_alloc<min_allocator<char> >) == 24, "");
+static_assert(sizeof(string_alloc<test_allocator<char> >) == 32, "");
+static_assert(sizeof(string_alloc<small_iter_allocator<char> >) == 6, "");
+
+static_assert(TEST_ALIGNOF(string_alloc<std::allocator<char> >) == 8, "");
+static_assert(TEST_ALIGNOF(string_alloc<min_allocator<char> >) == 8, "");
+static_assert(TEST_ALIGNOF(string_alloc<test_allocator<char> >) == 8, "");
+static_assert(TEST_ALIGNOF(string_alloc<small_iter_allocator<char> >) == 2, "");
+
+#elif __SIZE_WIDTH__ == 32
+static_assert(sizeof(user_struct) == 16, "");
+static_assert(TEST_ALIGNOF(user_struct) == 4, "");
+
+static_assert(sizeof(string_alloc<std::allocator<char> >) == 12, "");
+static_assert(sizeof(string_alloc<min_allocator<char> >) == 12, "");
+static_assert(sizeof(string_alloc<test_allocator<char> >) == 24, "");
+static_assert(sizeof(string_alloc<small_iter_allocator<char> >) == 6, "");
+
+static_assert(TEST_ALIGNOF(string_alloc<std::allocator<char> >) == 4, "");
+static_assert(TEST_ALIGNOF(string_alloc<min_allocator<char> >) == 4, "");
+static_assert(TEST_ALIGNOF(string_alloc<test_allocator<char> >) == 4, "");
+static_assert(TEST_ALIGNOF(string_alloc<small_iter_allocator<char> >) == 2, "");
+
+#else
+#  error std::size_t has an unexpected size
+#endif
diff --git a/libcxx/test/libcxx/language.support/support.c.headers/support.c.headers.other/math.lerp.verify.cpp b/libcxx/test/libcxx/language.support/support.c.headers/support.c.headers.other/math.lerp.verify.cpp
deleted file mode 100644
index 1b84a9bfe47da..0000000000000
--- a/libcxx/test/libcxx/language.support/support.c.headers/support.c.headers.other/math.lerp.verify.cpp
+++ /dev/null
@@ -1,39 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// When built with modules, this test gives diagnostics like
-//  declaration of 'lerp' must be imported from module 'std.compat.cmath'
-//  before it is required
-// therefore disable the test in this configuration.
-// UNSUPPORTED: clang-modules-build
-
-// <math.h>
-
-// [support.c.headers.other]/1
-//   ... except for the functions described in [sf.cmath], the
-//   std::lerp function overloads ([c.math.lerp]) ...
-
-#include <math.h>
-
-void f() {
-  {
-    float f;
-    ::lerp(f, f, f);    // expected-error {{no member named 'lerp' in the global namespace}}
-    std::lerp(f, f, f); // expected-error {{no member named 'lerp' in namespace 'std'}}
-  }
-  {
-    double d;
-    ::lerp(d, d, d);    // expected-error {{no member named 'lerp' in the global namespace}}
-    std::lerp(d, d, d); // expected-error {{no member named 'lerp' in namespace 'std'}}
-  }
-  {
-    long double l;
-    ::lerp(l, l, l);    // expected-error {{no member named 'lerp' in the global namespace}}
-    std::lerp(l, l, l); // expected-error {{no member named 'lerp' in namespace 'std'}}
-  }
-}
diff --git a/libcxx/test/libcxx/strings/string.view/assert.ctor.length.pass.cpp b/libcxx/test/libcxx/strings/string.view/assert.ctor.length.pass.cpp
index af8b393f9e0a7..e47b5f5963109 100644
--- a/libcxx/test/libcxx/strings/string.view/assert.ctor.length.pass.cpp
+++ b/libcxx/test/libcxx/strings/string.view/assert.ctor.length.pass.cpp
@@ -7,7 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 // REQUIRES: has-unix-headers
-// UNSUPPORTED: c++03, c++11
+// UNSUPPORTED: c++03
+// UNSUPPORTED: c++11 && gcc
 // REQUIRES: libcpp-hardening-mode={{extensive|debug}}
 // XFAIL: libcpp-hardening-mode=debug && availability-verbose_abort-missing
 
@@ -17,10 +18,17 @@
 #include <string_view>
 
 #include "check_assertion.h"
+#include "test_macros.h"
+
+// We're testing for assertions here, so let's not diagnose the misuses at compile time
+// FIXME: This should really be in ADDITIONAL_COMPILE_FLAGS, but it that doesn't work due to a Clang bug
+TEST_CLANG_DIAGNOSTIC_IGNORED("-Wnonnull")
 
 int main(int, char**) {
   char c = 0;
   TEST_LIBCPP_ASSERT_FAILURE(
       std::string_view(&c, -1), "string_view::string_view(_CharT *, size_t): length does not fit in difference_type");
+  TEST_LIBCPP_ASSERT_FAILURE(
+      std::string_view(nullptr, 1), "string_view::string_view(_CharT *, size_t): received nullptr");
   return 0;
 }
diff --git a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_is_same.sh.cpp b/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_is_same.sh.cpp
deleted file mode 100644
index cfd8adb52d854..0000000000000
--- a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_is_same.sh.cpp
+++ /dev/null
@@ -1,55 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This is a dummy feature that prevents this test from running by default.
-// REQUIRES: template-cost-testing
-
-// The table below compares the compile time and object size for each of the
-// variants listed in the RUN script.
-//
-//  Impl          Compile Time    Object Size
-// -------------------------------------------
-// std::_IsSame:    689.634 ms     356 K
-// std::is_same:  8,129.180 ms     560 K
-//
-// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/orig.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -std=c++17
-// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/new.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -std=c++17 -DTEST_NEW
-
-#include <type_traits>
-#include <cassert>
-
-#include "test_macros.h"
-#include "template_cost_testing.h"
-
-template <int N> struct Arg { enum { value = 1 }; };
-
-#ifdef TEST_NEW
-#define IS_SAME  std::_IsSame
-#else
-#define IS_SAME std::is_same
-#endif
-
-#define TEST_CASE_NOP() IS_SAME < Arg< __COUNTER__ >, Arg < __COUNTER__ > >::value,
-#define TEST_CASE_TYPE() IS_SAME < Arg< __COUNTER__ >, Arg < __COUNTER__ > >,
-
-int sink(...);
-
-int x = sink(
-  REPEAT_10000(TEST_CASE_NOP)
-  REPEAT_10000(TEST_CASE_NOP) 42
-);
-
-void Foo( REPEAT_1000(TEST_CASE_TYPE) int) { }
-
-static_assert(__COUNTER__ > 10000, "");
-
-void escape() {
-
-sink(&x);
-sink(&Foo);
-}
diff --git a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_metafunctions.sh.cpp b/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_metafunctions.sh.cpp
deleted file mode 100644
index 4815ce37b0883..0000000000000
--- a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_metafunctions.sh.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This is a dummy feature that prevents this test from running by default.
-// REQUIRES: template-cost-testing
-
-// The table below compares the compile time and object size for each of the
-// variants listed in the RUN script.
-//
-//  Impl          Compile Time    Object Size
-// -------------------------------------------
-// _And:         3,498.639 ms     158 M
-// __lazy_and:  10,138.982 ms     334 M
-// __and_:      14,181.851 ms     648 M
-//
-
-// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/new.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -std=c++17
-// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/lazy.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace  -std=c++17 -DTEST_LAZY_AND
-// RUN: %{cxx} %{flags} %{compile_flags} -c %s -o %S/std.o -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace   -std=c++17 -DTEST_STD_AND
-
-#include <type_traits>
-#include <cassert>
-
-#include "test_macros.h"
-#include "template_cost_testing.h"
-using std::true_type;
-using std::false_type;
-
-#define FALSE_T() std::false_type,
-#define TRUE_T() std::true_type,
-
-#ifdef TEST_LAZY_AND
-#define TEST_AND std::__lazy_and
-#define TEST_OR std::__lazy_or
-#elif defined(TEST_STD_AND)
-#define TEST_AND std::__and_
-#define TEST_OR std::__or_
-#else
-#define TEST_AND std::_And
-#define TEST_OR std::_Or
-#endif
-
-void sink(...);
-
-void Foo1(TEST_AND < REPEAT_1000(TRUE_T) true_type > t1) { sink(&t1); }
-void Foo2(TEST_AND < REPEAT_1000(TRUE_T) REPEAT_1000(TRUE_T) true_type > t2) { sink(&t2); }
-void Foo3(TEST_AND < REPEAT_1000(TRUE_T) true_type, false_type > t3) { sink(&t3); }
-void Foo4(TEST_AND < REPEAT_1000(TRUE_T) REPEAT_1000(TRUE_T) true_type, false_type > t4) { sink(&t4); }
-void Foo5(TEST_AND < false_type, REPEAT_1000(TRUE_T) true_type > t5) { sink(&t5); }
-void Foo6(TEST_AND < false_type, REPEAT_1000(TRUE_T) REPEAT_1000(TRUE_T) true_type > t6) { sink(&t6); }
-
-void escape() {
-
-sink(&Foo1);
-sink(&Foo2);
-sink(&Foo3);
-sink(&Foo4);
-sink(&Foo5);
-sink(&Foo6);
-}
diff --git a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_variant_overloads_impl.sh.cpp b/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_variant_overloads_impl.sh.cpp
deleted file mode 100644
index b0512325d4c2c..0000000000000
--- a/libcxx/test/libcxx/utilities/meta/stress_tests/stress_test_variant_overloads_impl.sh.cpp
+++ /dev/null
@@ -1,119 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This is a dummy feature that prevents this test from running by default.
-// REQUIRES: template-cost-testing
-
-// Test the cost of the mechanism used to create an overload set used by variant
-// to determine which alternative to construct.
-
-// The table below compares the compile time and object size for each of the
-// variants listed in the RUN script.
-//
-//  Impl           Compile Time  Object Size
-// -----------------------------------------------------
-// flat:              959 ms        792 KiB
-// recursive:      23,444 ms     23,000 KiB
-// -----------------------------------------------------
-// variant_old:    16,894 ms     17,000 KiB
-// variant_new:     1,105 ms        828 KiB
-
-
-// RUN: %{cxx} %{flags} %{compile_flags} -std=c++17 -c %s \
-// RUN:    -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -g \
-// RUN:    -DTEST_NS=flat_impl -o %S/flat.o
-// RUN: %{cxx} %{flags} %{compile_flags} -std=c++17 -c %s \
-// RUN:    -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -g \
-// RUN:    -DTEST_NS=rec_impl -o %S/rec.o
-// RUN: %{cxx} %{flags} %{compile_flags} -std=c++17 -c %s \
-// RUN:    -ggdb  -ggnu-pubnames -ftemplate-depth=5000 -ftime-trace -g \
-// RUN:    -DTEST_NS=variant_impl -o %S/variant.o
-
-#include <type_traits>
-#include <tuple>
-#include <cassert>
-#include <variant>
-
-#include "test_macros.h"
-#include "template_cost_testing.h"
-
-template <std::size_t Idx>
-struct TestType {};
-
-template <class T>
-struct ID {
-  using type = T;
-};
-
-namespace flat_impl {
-
-struct OverloadBase { void operator()() const; };
-
-template <class Tp, std::size_t Idx>
-struct Overload {
-  auto operator()(Tp, Tp) const -> ID<Tp>;
-};
-
-template <class ...Bases>
-struct AllOverloads : OverloadBase, Bases... {};
-
-template <class IdxSeq>
-struct MakeOverloads;
-
-template <std::size_t ..._Idx>
-struct MakeOverloads<std::__tuple_indices<_Idx...> > {
-  template <class ...Types>
-  using Apply = AllOverloads<Overload<Types, _Idx>...>;
-};
-
-template <class ...Types>
-using Overloads = typename MakeOverloads<
-    std::__make_indices_imp<sizeof...(Types), 0> >::template Apply<Types...>;
-
-} // namespace flat_impl
-
-
-namespace rec_impl {
-
-template <class... Types> struct Overload;
-
-template <>
-struct Overload<> { void operator()() const; };
-
-template <class Tp, class... Types>
-struct Overload<Tp, Types...> : Overload<Types...> {
-  using Overload<Types...>::operator();
-  auto operator()(Tp, Tp) const -> ID<Tp>;
-};
-
-template <class... Types>
-using Overloads = Overload<Types...>;
-
-} // namespace rec_impl
-
-namespace variant_impl {
-  template <class ...Types>
-  using Overloads = std::__variant_detail::_MakeOverloads<Types...>;
-  } // namespace variant_impl
-
-#ifndef TEST_NS
-#error TEST_NS must be defined
-#endif
-
-#define TEST_TYPE() TestType< __COUNTER__ >,
-using T1 = TEST_NS::Overloads<REPEAT_1000(TEST_TYPE) TestType<1>, TestType<1>, int>;
-static_assert(__COUNTER__ >= 1000, "");
-
-void fn1(T1 x) { DoNotOptimize(&x); }
-void fn2(typename std::invoke_result_t<T1, int, int>::type x) { DoNotOptimize(&x); }
-
-int main(int, char**) {
-  DoNotOptimize(&fn1);
-  DoNotOptimize(&fn2);
-  return 0;
-}
diff --git a/libcxx/test/libcxx/utilities/optional/optional.iterator/iterator.compile.pass.cpp b/libcxx/test/libcxx/utilities/optional/optional.iterator/iterator.compile.pass.cpp
new file mode 100644
index 0000000000000..3cdd7553e2e5d
--- /dev/null
+++ b/libcxx/test/libcxx/utilities/optional/optional.iterator/iterator.compile.pass.cpp
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++26
+
+// <optional>
+
+// template <class T> class optional::iterator;
+// template <class T> class optional::const_iterator;
+
+#include <optional>
+
+template <typename T>
+concept has_iterator_aliases = requires {
+  typename T::iterator;
+  typename T::const_iterator;
+};
+
+static_assert(has_iterator_aliases<std::optional<int>>);
+static_assert(has_iterator_aliases<std::optional<const int>>);
+
+// TODO: Uncomment these once P2988R12 is implemented, as they would be testing optional<T&>
+
+// static_assert(!has_iterator_aliases<std::optional<int (&)[]>>);
+// static_assert(!has_iterator_aliases<std::optional<void (&)(int, char)>>);
diff --git a/libcxx/test/std/algorithms/algorithms.results/out_value_result.pass.cpp b/libcxx/test/std/algorithms/algorithms.results/out_value_result.pass.cpp
index 7bdbb7f60e9a2..9f45976c29681 100644
--- a/libcxx/test/std/algorithms/algorithms.results/out_value_result.pass.cpp
+++ b/libcxx/test/std/algorithms/algorithms.results/out_value_result.pass.cpp
@@ -123,7 +123,7 @@ constexpr bool test() {
     static_assert(std::is_same_v<double, decltype(res.value)>);
   }
   {
-    // Check aggregate initiazliation
+    // Check aggregate initialization
     out_value_result<int, int> res = {1, 2};
     assert(res.out == 1);
     assert(res.value == 2);
diff --git a/libcxx/test/std/atomics/types.pass.cpp b/libcxx/test/std/atomics/types.pass.cpp
index 8737694e2a070..c979392290b35 100644
--- a/libcxx/test/std/atomics/types.pass.cpp
+++ b/libcxx/test/std/atomics/types.pass.cpp
@@ -68,7 +68,7 @@ struct test_atomic<T, /*Integral=*/true, false, false> {
 };
 
 template <class T>
-struct test_atomic<T, false, /*Foating=*/true, false> {
+struct test_atomic<T, false, /*Floating=*/true, false> {
   test_atomic() {
     using A = std::atomic<T>;
     A a;
diff --git a/libcxx/test/std/containers/associative/map/map.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/associative/map/map.cons/copy_assign.pass.cpp
index e01a10b679ac6..56c6cdd7e1e99 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/copy_assign.pass.cpp
@@ -70,7 +70,7 @@ template <class Alloc, class AllocatorInvariant = NoOp>
 void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
                 const Alloc& rhs_alloc                   = Alloc(),
                 AllocatorInvariant check_alloc_invariant = NoOp()) {
-  {   // Test empty/non-empy map combinations
+  {   // Test empty/non-empty map combinations
     { // assign from a non-empty container into an empty one
       using V   = std::pair<const int, int>;
       using Map = std::map<int, int, std::less<int>, Alloc>;
@@ -225,7 +225,7 @@ void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
     }
     check_alloc_invariant();
   }
-  { // Make a somewhat larget set to exercise the algorithm a bit
+  { // Make a somewhat larger set to exercise the algorithm a bit
     using V   = std::pair<const int, int>;
     using Map = std::map<int, int, std::less<int>, Alloc>;
 
diff --git a/libcxx/test/std/containers/associative/map/map.nonmember/compare.three_way.verify.cpp b/libcxx/test/std/containers/associative/map/map.nonmember/compare.three_way.verify.cpp
index 2427525664558..5352100226caf 100644
--- a/libcxx/test/std/containers/associative/map/map.nonmember/compare.three_way.verify.cpp
+++ b/libcxx/test/std/containers/associative/map/map.nonmember/compare.three_way.verify.cpp
@@ -25,9 +25,9 @@ int main(int, char**) {
   {
     std::map<int, int, std::less<int>, std::allocator<int>> s1;
     std::map<int, int, std::less<int>, test_allocator<int>> s2;
-    // expected-error-re@*:* {{static assertion failed due to requirement 'is_same<int, std::pair<const int, int>>::value'{{.*}}Allocator::value_type must be same type as value_type}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Allocator::value_type must be same type as value_type}}
     s1 <=> s2;
-    // expected-error-re@*:* {{static assertion failed due to requirement 'is_same<int, std::pair<const int, int>>::value'{{.*}}Allocator::value_type must be same type as value_type}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Allocator::value_type must be same type as value_type}}
     s2 <=> s1;
   }
   // Mismatching comparision functions
diff --git a/libcxx/test/std/containers/associative/map/map.ops/count0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/count0.pass.cpp
index c7ba765178969..62491e2bc4438 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/count0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/count0.pass.cpp
@@ -33,6 +33,10 @@ int main(int, char**) {
     typedef std::map<int, double, transparent_less_not_referenceable> M;
     assert(M().count(C2Int{5}) == 0);
   }
+  {
+    using M = std::map<int, double, transparent_less_nonempty>;
+    assert(M().count(C2Int{5}) == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/equal_range0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/equal_range0.pass.cpp
index 75724bdb387ce..57ce9339f6323 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/equal_range0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/equal_range0.pass.cpp
@@ -40,6 +40,13 @@ int main(int, char**) {
     P result = example.equal_range(C2Int{5});
     assert(result.first == result.second);
   }
+  {
+    using M = std::map<int, double, transparent_less_nonempty>;
+    using P = std::pair<typename M::iterator, typename M::iterator>;
+    M example;
+    P result = example.equal_range(C2Int{5});
+    assert(result.first == result.second);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/find0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/find0.pass.cpp
index 9825d6c5879b1..3f09d5608772d 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/find0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/find0.pass.cpp
@@ -36,6 +36,11 @@ int main(int, char**) {
     M example;
     assert(example.find(C2Int{5}) == example.end());
   }
+  {
+    using M = std::map<int, double, transparent_less_nonempty>;
+    M example;
+    assert(example.find(C2Int{5}) == example.end());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/lower_bound0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/lower_bound0.pass.cpp
index fe7fe381a86eb..308a2ed1e3af0 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/lower_bound0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/lower_bound0.pass.cpp
@@ -36,6 +36,11 @@ int main(int, char**) {
     M example;
     assert(example.lower_bound(C2Int{5}) == example.end());
   }
+  {
+    using M = std::map<int, double, transparent_less_nonempty>;
+    M example;
+    assert(example.lower_bound(C2Int{5}) == example.end());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/map/map.ops/upper_bound0.pass.cpp b/libcxx/test/std/containers/associative/map/map.ops/upper_bound0.pass.cpp
index 525aa673ea74b..332b71a84e9fb 100644
--- a/libcxx/test/std/containers/associative/map/map.ops/upper_bound0.pass.cpp
+++ b/libcxx/test/std/containers/associative/map/map.ops/upper_bound0.pass.cpp
@@ -36,6 +36,11 @@ int main(int, char**) {
     M example;
     assert(example.upper_bound(C2Int{5}) == example.end());
   }
+  {
+    using M = std::map<int, double, transparent_less_nonempty>;
+    M example;
+    assert(example.upper_bound(C2Int{5}) == example.end());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_assign.pass.cpp
index 5830283e9b0c5..e52da3a4a6313 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/copy_assign.pass.cpp
@@ -68,7 +68,7 @@ template <class Alloc, class AllocatorInvariant = NoOp>
 void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
                 const Alloc& rhs_alloc                   = Alloc(),
                 AllocatorInvariant check_alloc_invariant = NoOp()) {
-  {   // Test empty/non-empy multimap combinations
+  {   // Test empty/non-empty multimap combinations
     { // assign from a non-empty container into an empty one
       using V   = std::pair<const int, int>;
       using Map = std::multimap<int, int, std::less<int>, Alloc>;
@@ -223,7 +223,7 @@ void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
     }
     check_alloc_invariant();
   }
-  { // Make a somewhat larget set to exercise the algorithm a bit
+  { // Make a somewhat larger set to exercise the algorithm a bit
     using V   = std::pair<const int, int>;
     using Map = std::multimap<int, int, std::less<int>, Alloc>;
 
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_iter.pass.cpp
index 12df31e9dd6e7..3f145cc81ff46 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_iter.pass.cpp
@@ -13,88 +13,195 @@
 // template <class InputIterator>
 //   void insert(InputIterator first, InputIterator last);
 
-#include <map>
+#include <array>
 #include <cassert>
+#include <map>
 
-#include "test_macros.h"
-#include "test_iterators.h"
 #include "min_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef std::multimap<int, double> M;
-    typedef std::pair<int, double> P;
-    P ar[] = {
-        P(1, 1),
-        P(1, 1.5),
-        P(1, 2),
-        P(2, 1),
-        P(2, 1.5),
-        P(2, 2),
-        P(3, 1),
-        P(3, 1.5),
-        P(3, 2),
-    };
-    M m;
-    m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar) / sizeof(ar[0])));
-    assert(m.size() == 9);
-    assert(m.begin()->first == 1);
-    assert(m.begin()->second == 1);
-    assert(std::next(m.begin())->first == 1);
-    assert(std::next(m.begin())->second == 1.5);
-    assert(std::next(m.begin(), 2)->first == 1);
-    assert(std::next(m.begin(), 2)->second == 2);
-    assert(std::next(m.begin(), 3)->first == 2);
-    assert(std::next(m.begin(), 3)->second == 1);
-    assert(std::next(m.begin(), 4)->first == 2);
-    assert(std::next(m.begin(), 4)->second == 1.5);
-    assert(std::next(m.begin(), 5)->first == 2);
-    assert(std::next(m.begin(), 5)->second == 2);
-    assert(std::next(m.begin(), 6)->first == 3);
-    assert(std::next(m.begin(), 6)->second == 1);
-    assert(std::next(m.begin(), 7)->first == 3);
-    assert(std::next(m.begin(), 7)->second == 1.5);
-    assert(std::next(m.begin(), 8)->first == 3);
-    assert(std::next(m.begin(), 8)->second == 2);
+template <class Iter, class Alloc>
+void test_alloc() {
+  {   // Check that an empty range works correctly
+    { // Without elements in the container
+      using Map = std::multimap<int, int, std::less<int>, Alloc>;
+
+      std::array<std::pair<const int, int>, 0> arr;
+
+      Map map;
+      map.insert(Iter(arr.data()), Iter(arr.data() + arr.size()));
+      assert(map.size() == 0);
+      assert(map.begin() == map.end());
+    }
+    { // With 1 element in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      std::array<Pair, 0> arr;
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Iter(arr.data()), Iter(arr.data() + arr.size()));
+      assert(map.size() == 1);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(std::next(map.begin(), 1) == map.end());
+    }
+    { // With multiple elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      std::array<Pair, 0> arr;
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Pair(1, 1));
+      map.insert(Pair(2, 2));
+      map.insert(Iter(arr.data()), Iter(arr.data() + arr.size()));
+      assert(map.size() == 3);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(2, 2));
+      assert(std::next(map.begin(), 3) == map.end());
+    }
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-    typedef std::pair<int, double> P;
-    P ar[] = {
-        P(1, 1),
-        P(1, 1.5),
-        P(1, 2),
-        P(2, 1),
-        P(2, 1.5),
-        P(2, 2),
-        P(3, 1),
-        P(3, 1.5),
-        P(3, 2),
-    };
-    M m;
-    m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar) / sizeof(ar[0])));
-    assert(m.size() == 9);
-    assert(m.begin()->first == 1);
-    assert(m.begin()->second == 1);
-    assert(std::next(m.begin())->first == 1);
-    assert(std::next(m.begin())->second == 1.5);
-    assert(std::next(m.begin(), 2)->first == 1);
-    assert(std::next(m.begin(), 2)->second == 2);
-    assert(std::next(m.begin(), 3)->first == 2);
-    assert(std::next(m.begin(), 3)->second == 1);
-    assert(std::next(m.begin(), 4)->first == 2);
-    assert(std::next(m.begin(), 4)->second == 1.5);
-    assert(std::next(m.begin(), 5)->first == 2);
-    assert(std::next(m.begin(), 5)->second == 2);
-    assert(std::next(m.begin(), 6)->first == 3);
-    assert(std::next(m.begin(), 6)->second == 1);
-    assert(std::next(m.begin(), 7)->first == 3);
-    assert(std::next(m.begin(), 7)->second == 1.5);
-    assert(std::next(m.begin(), 8)->first == 3);
-    assert(std::next(m.begin(), 8)->second == 2);
+  {   // Check that 1 element is inserted correctly
+    { // Without elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1)};
+
+      Map map;
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 1);
+      assert(*std::next(map.begin(), 0) == Pair(1, 1));
+      assert(std::next(map.begin(), 1) == map.end());
+    }
+    { // With 1 element in the container - a different key
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1)};
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 2);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(std::next(map.begin(), 2) == map.end());
+    }
+    { // With 1 element in the container - the same key
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1)};
+
+      Map map;
+      map.insert(Pair(1, 1));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 2);
+      assert(*std::next(map.begin(), 0) == Pair(1, 1));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(std::next(map.begin(), 2) == map.end());
+    }
+    { // With multiple elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1)};
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Pair(1, 1));
+      map.insert(Pair(2, 2));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 4);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(1, 1));
+      assert(*std::next(map.begin(), 3) == Pair(2, 2));
+      assert(std::next(map.begin(), 4) == map.end());
+    }
   }
-#endif
+  {   // Check that multiple elements are inserted correctly
+    { // Without elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1), Pair(1, 1), Pair(3, 3)};
+
+      Map map;
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 3);
+      assert(*std::next(map.begin(), 0) == Pair(1, 1));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(3, 3));
+      assert(std::next(map.begin(), 3) == map.end());
+    }
+    { // With 1 element in the container - a different key
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1), Pair(1, 1), Pair(3, 3)};
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 4);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(1, 1));
+      assert(*std::next(map.begin(), 3) == Pair(3, 3));
+      assert(std::next(map.begin(), 4) == map.end());
+    }
+    { // With 1 element in the container - the same key
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1), Pair(2, 2), Pair(3, 3)};
+
+      Map map;
+      map.insert(Pair(1, 1));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 4);
+      assert(*std::next(map.begin(), 0) == Pair(1, 1));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(2, 2));
+      assert(*std::next(map.begin(), 3) == Pair(3, 3));
+      assert(std::next(map.begin(), 4) == map.end());
+    }
+    { // With multiple elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1), Pair(3, 3), Pair(4, 4)};
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Pair(1, 1));
+      map.insert(Pair(2, 2));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 6);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(1, 1));
+      assert(*std::next(map.begin(), 3) == Pair(2, 2));
+      assert(*std::next(map.begin(), 4) == Pair(3, 3));
+      assert(*std::next(map.begin(), 5) == Pair(4, 4));
+      assert(std::next(map.begin(), 6) == map.end());
+    }
+  }
+}
+
+void test() {
+  test_alloc<cpp17_input_iterator<std::pair<const int, int>*>, std::allocator<std::pair<const int, int> > >();
+  test_alloc<cpp17_input_iterator<std::pair<const int, int>*>, min_allocator<std::pair<const int, int> > >();
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.nonmember/compare.three_way.verify.cpp b/libcxx/test/std/containers/associative/multimap/multimap.nonmember/compare.three_way.verify.cpp
index 239ad0af9a637..e70e47e6d3b49 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.nonmember/compare.three_way.verify.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.nonmember/compare.three_way.verify.cpp
@@ -25,9 +25,9 @@ int main(int, char**) {
   {
     std::multimap<int, int, std::less<int>, std::allocator<int>> s1;
     std::multimap<int, int, std::less<int>, test_allocator<int>> s2;
-    // expected-error-re@*:* {{static assertion failed due to requirement 'is_same<int, std::pair<const int, int>>::value'{{.*}}Allocator::value_type must be same type as value_type}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Allocator::value_type must be same type as value_type}}
     s1 <=> s2;
-    // expected-error-re@*:* {{static assertion failed due to requirement 'is_same<int, std::pair<const int, int>>::value'{{.*}}Allocator::value_type must be same type as value_type}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}Allocator::value_type must be same type as value_type}}
     s2 <=> s1;
   }
   // Mismatching comparision functions
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/count0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/count0.pass.cpp
index 233d1a11e1d6c..36f0ac2647ba3 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/count0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/count0.pass.cpp
@@ -33,6 +33,10 @@ int main(int, char**) {
     typedef std::multimap<int, double, transparent_less_not_referenceable> M;
     assert(M().count(C2Int{5}) == 0);
   }
+  {
+    using M = std::multimap<int, double, transparent_less_nonempty>;
+    assert(M().count(C2Int{5}) == 0);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range0.pass.cpp
index 0bead6c7938db..a362c03e26385 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/equal_range0.pass.cpp
@@ -40,6 +40,13 @@ int main(int, char**) {
     P result = example.equal_range(C2Int{5});
     assert(result.first == result.second);
   }
+  {
+    using M = std::multimap<int, double, transparent_less_nonempty>;
+    using P = std::pair<typename M::iterator, typename M::iterator>;
+    M example;
+    P result = example.equal_range(C2Int{5});
+    assert(result.first == result.second);
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find.pass.cpp
index 6d5018ff5263e..15df6c15bfa78 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find.pass.cpp
@@ -21,6 +21,15 @@
 #include "private_constructor.h"
 #include "is_transparent.h"
 
+template <class Iter>
+bool iter_in_range(Iter first, Iter last, Iter to_find) {
+  for (; first != last; ++first) {
+    if (first == to_find)
+      return true;
+  }
+  return false;
+}
+
 int main(int, char**) {
   typedef std::pair<const int, double> V;
   {
@@ -30,15 +39,15 @@ int main(int, char**) {
       V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
       M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
       R r = m.find(5);
-      assert(r == m.begin());
+      assert(iter_in_range(std::next(m.begin(), 0), std::next(m.begin(), 3), r));
       r = m.find(6);
       assert(r == m.end());
       r = m.find(7);
-      assert(r == std::next(m.begin(), 3));
+      assert(iter_in_range(std::next(m.begin(), 3), std::next(m.begin(), 6), r));
       r = m.find(8);
       assert(r == m.end());
       r = m.find(9);
-      assert(r == std::next(m.begin(), 6));
+      assert(iter_in_range(std::next(m.begin(), 6), std::next(m.begin(), 9), r));
       r = m.find(10);
       assert(r == m.end());
     }
@@ -47,15 +56,15 @@ int main(int, char**) {
       V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
       const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
       R r = m.find(5);
-      assert(r == m.begin());
+      assert(iter_in_range(std::next(m.begin(), 0), std::next(m.begin(), 3), r));
       r = m.find(6);
       assert(r == m.end());
       r = m.find(7);
-      assert(r == std::next(m.begin(), 3));
+      assert(iter_in_range(std::next(m.begin(), 3), std::next(m.begin(), 6), r));
       r = m.find(8);
       assert(r == m.end());
       r = m.find(9);
-      assert(r == std::next(m.begin(), 6));
+      assert(iter_in_range(std::next(m.begin(), 6), std::next(m.begin(), 9), r));
       r = m.find(10);
       assert(r == m.end());
     }
@@ -68,15 +77,15 @@ int main(int, char**) {
       V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
       M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
       R r = m.find(5);
-      assert(r == m.begin());
+      assert(iter_in_range(std::next(m.begin(), 0), std::next(m.begin(), 3), r));
       r = m.find(6);
       assert(r == m.end());
       r = m.find(7);
-      assert(r == std::next(m.begin(), 3));
+      assert(iter_in_range(std::next(m.begin(), 3), std::next(m.begin(), 6), r));
       r = m.find(8);
       assert(r == m.end());
       r = m.find(9);
-      assert(r == std::next(m.begin(), 6));
+      assert(iter_in_range(std::next(m.begin(), 6), std::next(m.begin(), 9), r));
       r = m.find(10);
       assert(r == m.end());
     }
@@ -85,15 +94,15 @@ int main(int, char**) {
       V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
       const M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
       R r = m.find(5);
-      assert(r == m.begin());
+      assert(iter_in_range(std::next(m.begin(), 0), std::next(m.begin(), 3), r));
       r = m.find(6);
       assert(r == m.end());
       r = m.find(7);
-      assert(r == std::next(m.begin(), 3));
+      assert(iter_in_range(std::next(m.begin(), 3), std::next(m.begin(), 6), r));
       r = m.find(8);
       assert(r == m.end());
       r = m.find(9);
-      assert(r == std::next(m.begin(), 6));
+      assert(iter_in_range(std::next(m.begin(), 6), std::next(m.begin(), 9), r));
       r = m.find(10);
       assert(r == m.end());
     }
@@ -107,28 +116,28 @@ int main(int, char**) {
     V ar[] = {V(5, 1), V(5, 2), V(5, 3), V(7, 1), V(7, 2), V(7, 3), V(9, 1), V(9, 2), V(9, 3)};
     M m(ar, ar + sizeof(ar) / sizeof(ar[0]));
     R r = m.find(5);
-    assert(r == m.begin());
+    assert(iter_in_range(std::next(m.begin(), 0), std::next(m.begin(), 3), r));
     r = m.find(6);
     assert(r == m.end());
     r = m.find(7);
-    assert(r == std::next(m.begin(), 3));
+    assert(iter_in_range(std::next(m.begin(), 3), std::next(m.begin(), 6), r));
     r = m.find(8);
     assert(r == m.end());
     r = m.find(9);
-    assert(r == std::next(m.begin(), 6));
+    assert(iter_in_range(std::next(m.begin(), 6), std::next(m.begin(), 9), r));
     r = m.find(10);
     assert(r == m.end());
 
     r = m.find(C2Int(5));
-    assert(r == m.begin());
+    assert(iter_in_range(std::next(m.begin(), 0), std::next(m.begin(), 3), r));
     r = m.find(C2Int(6));
     assert(r == m.end());
     r = m.find(C2Int(7));
-    assert(r == std::next(m.begin(), 3));
+    assert(iter_in_range(std::next(m.begin(), 3), std::next(m.begin(), 6), r));
     r = m.find(C2Int(8));
     assert(r == m.end());
     r = m.find(C2Int(9));
-    assert(r == std::next(m.begin(), 6));
+    assert(iter_in_range(std::next(m.begin(), 6), std::next(m.begin(), 9), r));
     r = m.find(C2Int(10));
     assert(r == m.end());
   }
@@ -150,15 +159,15 @@ int main(int, char**) {
     m.insert(std::make_pair<PC, double>(PC::make(9), 3));
 
     R r = m.find(5);
-    assert(r == m.begin());
+    assert(iter_in_range(std::next(m.begin(), 0), std::next(m.begin(), 3), r));
     r = m.find(6);
     assert(r == m.end());
     r = m.find(7);
-    assert(r == std::next(m.begin(), 3));
+    assert(iter_in_range(std::next(m.begin(), 3), std::next(m.begin(), 6), r));
     r = m.find(8);
     assert(r == m.end());
     r = m.find(9);
-    assert(r == std::next(m.begin(), 6));
+    assert(iter_in_range(std::next(m.begin(), 6), std::next(m.begin(), 9), r));
     r = m.find(10);
     assert(r == m.end());
   }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/find0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/find0.pass.cpp
index 701d4e314e7a8..ccb0900e76835 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/find0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/find0.pass.cpp
@@ -36,6 +36,11 @@ int main(int, char**) {
     M example;
     assert(example.find(C2Int{5}) == example.end());
   }
+  {
+    using M = std::multimap<int, double, transparent_less_nonempty>;
+    M example;
+    assert(example.find(C2Int{5}) == example.end());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound0.pass.cpp
index 79f994847f131..4b4853062001f 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/lower_bound0.pass.cpp
@@ -36,6 +36,11 @@ int main(int, char**) {
     M example;
     assert(example.lower_bound(C2Int{5}) == example.end());
   }
+  {
+    using M = std::multimap<int, double, transparent_less_nonempty>;
+    M example;
+    assert(example.lower_bound(C2Int{5}) == example.end());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound0.pass.cpp b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound0.pass.cpp
index 62f52416b9156..f2ae94577b6c1 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound0.pass.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.ops/upper_bound0.pass.cpp
@@ -36,6 +36,11 @@ int main(int, char**) {
     M example;
     assert(example.upper_bound(C2Int{5}) == example.end());
   }
+  {
+    using M = std::multimap<int, double, transparent_less_nonempty>;
+    M example;
+    assert(example.upper_bound(C2Int{5}) == example.end());
+  }
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/insert_iter_iter.pass.cpp b/libcxx/test/std/containers/associative/multiset/insert_iter_iter.pass.cpp
index f3dd91794c9d5..56483714cd50c 100644
--- a/libcxx/test/std/containers/associative/multiset/insert_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/insert_iter_iter.pass.cpp
@@ -13,50 +13,184 @@
 // template <class InputIterator>
 //   void insert(InputIterator first, InputIterator last);
 
-#include <set>
+#include <array>
 #include <cassert>
+#include <set>
 
-#include "test_macros.h"
-#include "test_iterators.h"
 #include "min_allocator.h"
+#include "test_iterators.h"
 
-int main(int, char**) {
-  {
-    typedef std::multiset<int> M;
-    typedef int V;
-    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
-    M m;
-    m.insert(cpp17_input_iterator<const V*>(ar), cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])));
-    assert(m.size() == 9);
-    assert(*std::next(m.begin(), 0) == 1);
-    assert(*std::next(m.begin(), 1) == 1);
-    assert(*std::next(m.begin(), 2) == 1);
-    assert(*std::next(m.begin(), 3) == 2);
-    assert(*std::next(m.begin(), 4) == 2);
-    assert(*std::next(m.begin(), 5) == 2);
-    assert(*std::next(m.begin(), 6) == 3);
-    assert(*std::next(m.begin(), 7) == 3);
-    assert(*std::next(m.begin(), 8) == 3);
+template <class Iter, class Alloc>
+void test_alloc() {
+  {   // Check that an empty range works correctly
+    { // Without elements in the container
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      std::array<int, 0> arr;
+
+      Map map;
+      map.insert(Iter(arr.data()), Iter(arr.data() + arr.size()));
+      assert(map.size() == 0);
+      assert(map.begin() == map.end());
+    }
+    { // With 1 element in the container
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      std::array<int, 0> arr;
+
+      Map map;
+      map.insert(0);
+      map.insert(Iter(arr.data()), Iter(arr.data() + arr.size()));
+      assert(map.size() == 1);
+      assert(*std::next(map.begin(), 0) == 0);
+      assert(std::next(map.begin(), 1) == map.end());
+    }
+    { // With multiple elements in the container
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      std::array<int, 0> arr;
+
+      Map map;
+      map.insert(0);
+      map.insert(1);
+      map.insert(2);
+      map.insert(Iter(arr.data()), Iter(arr.data() + arr.size()));
+      assert(map.size() == 3);
+      assert(*std::next(map.begin(), 0) == 0);
+      assert(*std::next(map.begin(), 1) == 1);
+      assert(*std::next(map.begin(), 2) == 2);
+      assert(std::next(map.begin(), 3) == map.end());
+    }
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::multiset<int, std::less<int>, min_allocator<int>> M;
-    typedef int V;
-    V ar[] = {1, 1, 1, 2, 2, 2, 3, 3, 3};
-    M m;
-    m.insert(cpp17_input_iterator<const V*>(ar), cpp17_input_iterator<const V*>(ar + sizeof(ar) / sizeof(ar[0])));
-    assert(m.size() == 9);
-    assert(*std::next(m.begin(), 0) == 1);
-    assert(*std::next(m.begin(), 1) == 1);
-    assert(*std::next(m.begin(), 2) == 1);
-    assert(*std::next(m.begin(), 3) == 2);
-    assert(*std::next(m.begin(), 4) == 2);
-    assert(*std::next(m.begin(), 5) == 2);
-    assert(*std::next(m.begin(), 6) == 3);
-    assert(*std::next(m.begin(), 7) == 3);
-    assert(*std::next(m.begin(), 8) == 3);
+  {   // Check that 1 element is inserted correctly
+    { // Without elements in the container
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      int arr[] = {1};
+
+      Map map;
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 1);
+      assert(*std::next(map.begin(), 0) == 1);
+      assert(std::next(map.begin(), 1) == map.end());
+    }
+    { // With 1 element in the container - a different key
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      int arr[] = {1};
+
+      Map map;
+      map.insert(0);
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 2);
+      assert(*std::next(map.begin(), 0) == 0);
+      assert(*std::next(map.begin(), 1) == 1);
+      assert(std::next(map.begin(), 2) == map.end());
+    }
+    { // With 1 element in the container - the same key
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      int arr[] = {1};
+
+      Map map;
+      map.insert(1);
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 2);
+      assert(*std::next(map.begin(), 0) == 1);
+      assert(*std::next(map.begin(), 1) == 1);
+      assert(std::next(map.begin(), 2) == map.end());
+    }
+    { // With multiple elements in the container
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      int arr[] = {1};
+
+      Map map;
+      map.insert(0);
+      map.insert(1);
+      map.insert(2);
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 4);
+      assert(*std::next(map.begin(), 0) == 0);
+      assert(*std::next(map.begin(), 1) == 1);
+      assert(*std::next(map.begin(), 2) == 1);
+      assert(*std::next(map.begin(), 3) == 2);
+      assert(std::next(map.begin(), 4) == map.end());
+    }
+  }
+  {   // Check that multiple elements are inserted correctly
+    { // Without elements in the container
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      int arr[] = {1, 1, 3};
+
+      Map map;
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 3);
+      assert(*std::next(map.begin(), 0) == 1);
+      assert(*std::next(map.begin(), 1) == 1);
+      assert(*std::next(map.begin(), 2) == 3);
+      assert(std::next(map.begin(), 3) == map.end());
+    }
+    { // With 1 element in the container - a different key
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      int arr[] = {1, 1, 3};
+
+      Map map;
+      map.insert(0);
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 4);
+      assert(*std::next(map.begin(), 0) == 0);
+      assert(*std::next(map.begin(), 1) == 1);
+      assert(*std::next(map.begin(), 2) == 1);
+      assert(*std::next(map.begin(), 3) == 3);
+      assert(std::next(map.begin(), 4) == map.end());
+    }
+    { // With 1 element in the container - the same key
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      int arr[] = {1, 2, 3};
+
+      Map map;
+      map.insert(1);
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 4);
+      assert(*std::next(map.begin(), 0) == 1);
+      assert(*std::next(map.begin(), 1) == 1);
+      assert(*std::next(map.begin(), 2) == 2);
+      assert(*std::next(map.begin(), 3) == 3);
+      assert(std::next(map.begin(), 4) == map.end());
+    }
+    { // With multiple elements in the container
+      using Map = std::multiset<int, std::less<int>, Alloc>;
+
+      int arr[] = {1, 3, 4};
+
+      Map map;
+      map.insert(0);
+      map.insert(1);
+      map.insert(2);
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 6);
+      assert(*std::next(map.begin(), 0) == 0);
+      assert(*std::next(map.begin(), 1) == 1);
+      assert(*std::next(map.begin(), 2) == 1);
+      assert(*std::next(map.begin(), 3) == 2);
+      assert(*std::next(map.begin(), 4) == 3);
+      assert(*std::next(map.begin(), 5) == 4);
+      assert(std::next(map.begin(), 6) == map.end());
+    }
   }
-#endif
+}
+
+void test() {
+  test_alloc<cpp17_input_iterator<int*>, std::allocator<int> >();
+  test_alloc<cpp17_input_iterator<int*>, min_allocator<int> >();
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_assign.pass.cpp
index 8511f05ab57e4..0a5fe62eed69e 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/copy_assign.pass.cpp
@@ -68,7 +68,7 @@ template <class Alloc, class AllocatorInvariant = NoOp>
 void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
                 const Alloc& rhs_alloc                   = Alloc(),
                 AllocatorInvariant check_alloc_invariant = NoOp()) {
-  {   // Test empty/non-empy multiset combinations
+  {   // Test empty/non-empty multiset combinations
     { // assign from a non-empty container into an empty one
       using Set = std::multiset<int, std::less<int>, Alloc>;
 
@@ -216,7 +216,7 @@ void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
       assert(std::next(orig.begin(), 3) == orig.end());
     }
     check_alloc_invariant();
-    { // Make a somewhat larget multiset to exercise the algorithm a bit
+    { // Make a somewhat larger multiset to exercise the algorithm a bit
       using Set = std::multiset<int, std::less<int>, Alloc>;
 
       Set orig(rhs_alloc);
diff --git a/libcxx/test/std/containers/associative/set/set.cons/copy_assign.pass.cpp b/libcxx/test/std/containers/associative/set/set.cons/copy_assign.pass.cpp
index 85bc4e761f2c3..dd1dbc86190f8 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/copy_assign.pass.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/copy_assign.pass.cpp
@@ -70,7 +70,7 @@ template <class Alloc, class AllocatorInvariant = NoOp>
 void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
                 const Alloc& rhs_alloc                   = Alloc(),
                 AllocatorInvariant check_alloc_invariant = NoOp()) {
-  {   // Test empty/non-empy set combinations
+  {   // Test empty/non-empty set combinations
     { // assign from a non-empty container into an empty one
       using Set = std::set<int, std::less<int>, Alloc>;
 
@@ -218,7 +218,7 @@ void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
       assert(std::next(orig.begin(), 3) == orig.end());
     }
     check_alloc_invariant();
-    { // Make a somewhat larget set to exercise the algorithm a bit
+    { // Make a somewhat larger set to exercise the algorithm a bit
       using Set = std::set<int, std::less<int>, Alloc>;
 
       Set orig(rhs_alloc);
diff --git a/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp b/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp
index 0bca4299c8516..6d5427f3e3637 100644
--- a/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.cons/initialization.pass.cpp
@@ -20,7 +20,7 @@ struct NoDefault {
 struct test_initialization {
   template <typename T>
   TEST_CONSTEXPR_CXX14 void operator()() const {
-    // Check default initalization
+    // Check default initialization
     {
       std::array<T, 0> a0;
       (void)a0;
diff --git a/libcxx/test/std/containers/sequences/array/array.overview/nttp.verify.cpp b/libcxx/test/std/containers/sequences/array/array.overview/nttp.verify.cpp
index f50febf5f2485..e268dd91a26e1 100644
--- a/libcxx/test/std/containers/sequences/array/array.overview/nttp.verify.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.overview/nttp.verify.cpp
@@ -45,7 +45,7 @@ using E = test<NotALiteral, 1, std::array<NotALiteral, 1>{}>;
 // expected-error-re@*:* {{non-type template parameter has non-literal type 'std::array<NotALiteral, 1U{{L{0,2}.*}}>'}}
 
 using F = test<std::string, 2, std::array<std::string, 2>{}>;
-// expected-error-re@*:* {{type 'std::array<{{(std::)?}}string, 2U{{L{0,2}.*}}>' {{(\(aka 'array<basic_string<char>, 2UL{0,2}>'\) )?}}of non-type template parameter is not a structural type}}
+// expected-error-re@*:* {{type {{.+}} of non-type template parameter is not a structural type}}
 } // namespace test_full_type
 
 namespace test_ctad {
@@ -77,5 +77,5 @@ using E = test<std::array<NotALiteral, 1>{}>;
 // expected-error@-1 {{non-type template parameter has non-literal type 'std::array<NotALiteral, 1>'}}
 
 using F = test<std::array<std::string, 2>{}>;
-// expected-error@-1 {{type 'std::array<std::string, 2>' (aka 'array<basic_string<char>, 2>') of non-type template parameter is not a structural type}}
+// expected-error-re@-1 {{type {{.+}} of non-type template parameter is not a structural type}}
 } // namespace test_auto
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/max_size.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/max_size.pass.cpp
index a3f44f33028e4..af57e5ffc5cf9 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/max_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/max_size.pass.cpp
@@ -66,7 +66,7 @@ TEST_CONSTEXPR_CXX20 bool tests() {
 
 #if TEST_STD_VER >= 11
 
-  // Test with various allocators and diffrent size_type
+  // Test with various allocators and different size_type
   {
     test(std::vector<int>());
     test(std::vector<short, std::allocator<short> >());
diff --git a/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit_exceptions.pass.cpp b/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit_exceptions.pass.cpp
index 521a25fdeda0f..0bc3387e26b87 100644
--- a/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit_exceptions.pass.cpp
+++ b/libcxx/test/std/containers/sequences/vector/vector.capacity/shrink_to_fit_exceptions.pass.cpp
@@ -39,7 +39,7 @@ void test_allocation_exception_for_strong_guarantee(std::vector<T, Alloc>& v, co
   } catch (...) {
   }
 
-  // As shrink_to_fit may swallow any exceptions, we place the checks outisde the catch block.
+  // As shrink_to_fit may swallow any exceptions, we place the checks outside the catch block.
   assert(v.data() == old_data);
   assert(v.size() == old_size);
   assert(v.capacity() == old_cap);
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_copy.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_copy.pass.cpp
index 34dec07b03e08..3e4c5b1c03fcd 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/assign_copy.pass.cpp
@@ -14,13 +14,16 @@
 
 // unordered_map& operator=(const unordered_map& u);
 
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+
 #include <algorithm>
-#include <unordered_map>
-#include <string>
 #include <cassert>
 #include <cfloat>
 #include <cmath>
 #include <cstddef>
+#include <string>
+#include <unordered_map>
+#include <vector>
 
 #include "test_macros.h"
 #include "../../../test_compare.h"
@@ -28,116 +31,437 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
-  {
-    typedef test_allocator<std::pair<const int, std::string> > A;
-    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
-    c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A(4));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class T>
+class tracking_allocator {
+  std::vector<void*>* allocs_;
+
+  template <class U>
+  friend class tracking_allocator;
+
+public:
+  using value_type                             = T;
+  using propagate_on_container_copy_assignment = std::true_type;
+
+  tracking_allocator(std::vector<void*>& allocs) : allocs_(&allocs) {}
+
+  template <class U>
+  tracking_allocator(const tracking_allocator<U>& other) : allocs_(other.allocs_) {}
+
+  T* allocate(std::size_t n) {
+    T* allocation = std::allocator<T>().allocate(n);
+    allocs_->push_back(allocation);
+    return allocation;
   }
-  {
-    typedef std::unordered_map<int, std::string> C;
-    typedef std::pair<const int, std::string> P;
-    const P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c(a, a + sizeof(a) / sizeof(a[0]));
-    C* p = &c;
-    c    = *p;
-    assert(c.size() == 4);
-    assert(std::is_permutation(c.begin(), c.end(), a));
+
+  void deallocate(T* ptr, std::size_t n) TEST_NOEXCEPT {
+    auto res = std::remove(allocs_->begin(), allocs_->end(), ptr);
+    assert(res != allocs_->end() && "Trying to deallocate memory from different allocator?");
+    allocs_->erase(res);
+    std::allocator<T>().deallocate(ptr, n);
   }
-  {
-    typedef other_allocator<std::pair<const int, std::string> > A;
-    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
-    c = c0;
-    assert(c.bucket_count() >= 5);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A(10));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+  friend bool operator==(const tracking_allocator& lhs, const tracking_allocator& rhs) {
+    return lhs.allocs_ == rhs.allocs_;
+  }
+
+  friend bool operator!=(const tracking_allocator& lhs, const tracking_allocator& rhs) {
+    return lhs.allocs_ != rhs.allocs_;
+  }
+};
+
+struct NoOp {
+  void operator()() {}
+};
+
+template <class Alloc, class AllocatorInvariant = NoOp>
+void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
+                const Alloc& rhs_alloc                   = Alloc(),
+                AllocatorInvariant check_alloc_invariant = NoOp()) {
+  {   // Test empty/non-empty combinations
+    { // assign from a non-empty container into an empty one
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      Map copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.at(1) == 1);
+      assert(copy.at(2) == 3);
+      assert(copy.at(4) == 4);
+      assert(copy.at(5) == 2);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.at(1) == 1);
+      assert(orig.at(2) == 3);
+      assert(orig.at(4) == 4);
+      assert(orig.at(5) == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // assign from an empty container into an empty one
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      const Map orig(rhs_alloc);
+      Map copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+      assert(copy.begin() == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+    { // assign from an empty container into a non-empty one
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      const Map orig(rhs_alloc);
+      Map copy(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), lhs_alloc);
+      copy = orig;
+      // Depending on whether the allocator is propagated the bucked count can change
+      LIBCPP_ASSERT(copy.bucket_count() == 5 || copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+      assert(copy.begin() == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+  }
+  {   // Test empty/one-element copies. In our implementation that's a special case.
+    { // assign from a single-element container into an empty one
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1)};
+      const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      Map copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 2);
+      assert(copy.size() == 1);
+      assert(copy.at(1) == 1);
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 2);
+      assert(orig.size() == 1);
+      assert(orig.at(1) == 1);
+    }
+    { // assign from an empty container into a single-element one
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1)};
+      const Map orig(rhs_alloc);
+      Map copy(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), lhs_alloc);
+      copy = orig;
+      // Depending on whether the allocator is propagated the bucked count can change
+      LIBCPP_ASSERT(copy.bucket_count() == 2 || copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+    }
+  }
+  {   // Ensure that self-assignment works correctly
+    { // with a non-empty map
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      orig = static_cast<const Map&>(orig);
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.at(1) == 1);
+      assert(orig.at(2) == 3);
+      assert(orig.at(4) == 4);
+      assert(orig.at(5) == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // with an empty map
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      Map orig(rhs_alloc);
+      orig = static_cast<const Map&>(orig);
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+  }
+  {   // check assignment into a non-empty map
+    { // LHS already contains elements, but fewer than the RHS
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V lhs_arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      const Map orig(begin(lhs_arr), end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      V rhs_arr[] = {V(10, 4), V(13, 5)};
+      Map copy(begin(rhs_arr), end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.at(1) == 1);
+      assert(copy.at(2) == 3);
+      assert(copy.at(4) == 4);
+      assert(copy.at(5) == 2);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.at(1) == 1);
+      assert(orig.at(2) == 3);
+      assert(orig.at(4) == 4);
+      assert(orig.at(5) == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // LHS contains the same number of elements as the RHS
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V lhs_arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      const Map orig(begin(lhs_arr), end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      V rhs_arr[] = {V(10, 4), V(13, 5), V(12, 324), V(0, 54)};
+      Map copy(begin(rhs_arr), end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.at(1) == 1);
+      assert(copy.at(2) == 3);
+      assert(copy.at(4) == 4);
+      assert(copy.at(5) == 2);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.at(1) == 1);
+      assert(orig.at(2) == 3);
+      assert(orig.at(4) == 4);
+      assert(orig.at(5) == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // LHS already contains more elements than the RHS
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V lhs_arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      const Map orig(begin(lhs_arr), end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      V rhs_arr[] = {V(10, 4), V(13, 5), V(12, 324), V(0, 54), V(50, 5), V(2, 5)};
+      Map copy(begin(rhs_arr), end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.at(1) == 1);
+      assert(copy.at(2) == 3);
+      assert(copy.at(4) == 4);
+      assert(copy.at(5) == 2);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.at(1) == 1);
+      assert(orig.at(2) == 3);
+      assert(orig.at(4) == 4);
+      assert(orig.at(5) == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef min_allocator<std::pair<const int, std::string> > A;
-    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
+}
+
+void test() {
+  test_alloc<std::allocator<std::pair<const int, int> > >();
+  test_alloc<min_allocator<std::pair<const int, int> > >();
+
+  { // Make sure we're allocating/deallocating nodes with the correct allocator
+    // See https://llvm.org/PR29001 (report is for std::map, but the unordered containers have the same optimization)
+    class AssertEmpty {
+      std::vector<void*>* lhs_allocs_;
+      std::vector<void*>* rhs_allocs_;
+
+    public:
+      AssertEmpty(std::vector<void*>& lhs_allocs, std::vector<void*>& rhs_allocs)
+          : lhs_allocs_(&lhs_allocs), rhs_allocs_(&rhs_allocs) {}
+
+      void operator()() {
+        assert(lhs_allocs_->empty());
+        assert(rhs_allocs_->empty());
+      }
     };
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
-    c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A());
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+    std::vector<void*> lhs_allocs;
+    std::vector<void*> rhs_allocs;
+    test_alloc<tracking_allocator<std::pair<const int, int> > >(
+        lhs_allocs, rhs_allocs, AssertEmpty(lhs_allocs, rhs_allocs));
+  }
+
+  {   // Ensure that the hasher is copied
+    { // when the container is non-empty
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, test_hash<int> >;
+
+      V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+      const Map orig(begin(arr), end(arr), 0, test_hash<int>(5));
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.hash_function() == test_hash<int>(5));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(orig.hash_function() == test_hash<int>(5));
+    }
+    { // when the container is empty
+      using Map = std::unordered_map<int, int, test_hash<int> >;
+
+      const Map orig(0, test_hash<int>(5));
+      Map copy;
+      copy = orig;
+      assert(copy.empty());
+      assert(copy.hash_function() == test_hash<int>(5));
+
+      // Check that orig is still what is expected
+      assert(orig.empty());
+      assert(orig.hash_function() == test_hash<int>(5));
+    }
   }
-#endif
+
+  {   // Ensure that the equality comparator is copied
+    { // when the container is non-empty
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int, std::hash<int>, test_equal_to<int> >;
+
+      V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+      const Map orig(begin(arr), end(arr), 0, std::hash<int>(), test_equal_to<int>(23));
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+    }
+    { // when the container is empty
+      using Map = std::unordered_map<int, int, std::hash<int>, test_equal_to<int> >;
+
+      const Map orig(0, std::hash<int>(), test_equal_to<int>(23));
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 0);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 0);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+    }
+  }
+
+  {   // Ensure that the max load factor is copied
+    { // when the container is non-empty
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_map<int, int>;
+
+      V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+      Map orig(begin(arr), end(arr));
+      orig.max_load_factor(33.f);
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.max_load_factor() == 33.f);
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(orig.max_load_factor() == 33.f);
+    }
+    { // when the container is empty
+      using Map = std::unordered_map<int, int>;
+
+      Map orig;
+      orig.max_load_factor(17.f);
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 0);
+      assert(copy.max_load_factor() == 17.f);
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 0);
+      assert(orig.max_load_factor() == 17.f);
+    }
+  }
+
+  {     // Check that pocca is handled properly
+    {   // pocca == true_type
+      { // when the container is non-empty
+        using V     = std::pair<const int, int>;
+        using Alloc = other_allocator<V>;
+        using Map   = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+        const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), Alloc(3));
+        Map copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(3));
+      }
+      { // when the container is empty
+        using Alloc = other_allocator<std::pair<const int, int> >;
+        using Map   = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        const Map orig(Alloc(3));
+        Map copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(3));
+      }
+    }
+    {   // pocca == false_type
+      { // when the container is non-empty
+        using V     = std::pair<const int, int>;
+        using Alloc = test_allocator<V>;
+        using Map   = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+        const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), Alloc(3));
+        Map copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(1));
+      }
+      { // when the container is empty
+        using Alloc = test_allocator<std::pair<const int, int> >;
+        using Map   = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        const Map orig(Alloc(3));
+        Map copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(1));
+      }
+    }
+  }
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy.pass.cpp
index 793807babbdca..36083d54228fa 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy.pass.cpp
@@ -14,136 +14,134 @@
 
 // unordered_map(const unordered_map& u);
 
-#include <unordered_map>
-#include <string>
 #include <cassert>
 #include <cfloat>
 #include <cmath>
 #include <cstddef>
+#include <unordered_map>
 
-#include "test_macros.h"
 #include "../../../test_compare.h"
 #include "../../../test_hash.h"
-#include "test_allocator.h"
 #include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef std::unordered_map<int,
-                               std::string,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               test_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         test_allocator<std::pair<const int, std::string> >(10));
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(10)));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class Alloc>
+void test_alloc() {
+  { // Simple check
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    V arr[] = {V(1, 2), V(2, 4), V(3, 1)};
+    const Map orig(std::begin(arr), std::end(arr));
+    Map copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 5);
+    assert(copy.size() == 3);
+    assert(copy.at(1) == 2);
+    assert(copy.at(2) == 4);
+    assert(copy.at(3) == 1);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 5);
+    assert(orig.size() == 3);
+    assert(orig.at(1) == 2);
+    assert(orig.at(2) == 4);
+    assert(orig.at(3) == 1);
+  }
+  { // single element copy
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    V arr[] = {V(1, 2)};
+    const Map orig(std::begin(arr), std::end(arr));
+    Map copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 2);
+    assert(copy.size() == 1);
+    assert(copy.at(1) == 2);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 2);
+    assert(orig.size() == 1);
+    assert(orig.at(1) == 2);
+  }
+  { // Copy empty map
+    using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    const Map orig;
+    Map copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 0);
+    assert(copy.size() == 0);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 0);
+    assert(orig.size() == 0);
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::unordered_map<int,
-                               std::string,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               other_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         other_allocator<std::pair<const int, std::string> >(10));
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (other_allocator<std::pair<const int, std::string> >(-2)));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  { // Ensure that the hash function is copied
+    using Map = std::unordered_map<int, int, test_hash<int>, std::equal_to<int>, Alloc>;
+    const Map orig(0, test_hash<int>(23));
+    Map copy = orig;
+    assert(copy.hash_function() == test_hash<int>(23));
+
+    // Check that orig is still what is expected
+    assert(orig.hash_function() == test_hash<int>(23));
+  }
+  { // Ensure that the quality comparator is copied
+    using Map = std::unordered_map<int, int, std::hash<int>, test_equal_to<int>, Alloc>;
+    const Map orig(0, std::hash<int>(), test_equal_to<int>(56));
+    Map copy = orig;
+    assert(copy.key_eq() == test_equal_to<int>(56));
+
+    // Check that orig is still what is expected
+    assert(orig.key_eq() == test_equal_to<int>(56));
   }
-  {
-    typedef std::unordered_map<int,
-                               std::string,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               min_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         min_allocator<std::pair<const int, std::string> >());
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+}
+
+void test() {
+  test_alloc<std::allocator<std::pair<const int, int> > >();
+  test_alloc<min_allocator<std::pair<const int, int> > >();
+
+  { // Ensure that the allocator is copied
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, test_allocator<V> >;
+
+    V arr[] = {V(1, 1), V(2, 3), V(3, 6)};
+    const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), test_allocator<V>(10));
+    Map copy = orig;
+    assert(copy.size() == 3);
+    assert(copy.get_allocator() == test_allocator<V>(10));
+
+    // Check that orig is still what is expected
+    assert(orig.size() == 3);
+    assert(orig.get_allocator() == test_allocator<V>(10));
+    assert(orig.get_allocator().get_id() != test_alloc_base::moved_value);
   }
-#endif
+
+  { // Ensure that soccc is handled properly
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, other_allocator<V> >;
+
+    V arr[] = {V(1, 1), V(2, 3), V(3, 6)};
+    const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), other_allocator<V>(10));
+    Map copy = orig;
+    assert(copy.size() == 3);
+    assert(copy.get_allocator() == other_allocator<V>(-2));
+
+    // Check that orig is still what is expected
+    assert(orig.size() == 3);
+    assert(orig.get_allocator() == other_allocator<V>(10));
+  }
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy_alloc.pass.cpp
index 65c49f4fdcd00..652e8ce10ea45 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/copy_alloc.pass.cpp
@@ -14,127 +14,109 @@
 
 // unordered_map(const unordered_map& u, const allocator_type& a);
 
-#include <unordered_map>
-#include <string>
 #include <cassert>
 #include <cfloat>
 #include <cmath>
 #include <cstddef>
+#include <unordered_map>
 
-#include "test_macros.h"
 #include "../../../test_compare.h"
 #include "../../../test_hash.h"
-#include "test_allocator.h"
 #include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef std::unordered_map<int,
-                               std::string,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               test_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         test_allocator<std::pair<const int, std::string> >(10));
-    C c(c0, test_allocator<std::pair<const int, std::string> >(5));
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(5)));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class Alloc>
+void test_alloc(const Alloc& new_alloc) {
+  { // Simple check
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    V arr[] = {V(1, 2), V(2, 4), V(3, 1)};
+    const Map orig(std::begin(arr), std::end(arr));
+    Map copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 5);
+    assert(copy.size() == 3);
+    assert(copy.at(1) == 2);
+    assert(copy.at(2) == 4);
+    assert(copy.at(3) == 1);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 5);
+    assert(orig.size() == 3);
+    assert(orig.at(1) == 2);
+    assert(orig.at(2) == 4);
+    assert(orig.at(3) == 1);
+  }
+  { // single element check
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    V arr[] = {V(1, 2)};
+    const Map orig(std::begin(arr), std::end(arr));
+    Map copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 2);
+    assert(copy.size() == 1);
+    assert(copy.at(1) == 2);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 2);
+    assert(orig.size() == 1);
+    assert(orig.at(1) == 2);
+  }
+  { // Copy empty map
+    using Map = std::unordered_map<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    const Map orig;
+    Map copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 0);
+    assert(copy.size() == 0);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 0);
+    assert(orig.size() == 0);
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::unordered_map<int,
-                               std::string,
-                               test_hash<int>,
-                               test_equal_to<int>,
-                               min_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         min_allocator<std::pair<const int, std::string> >());
-    C c(c0, min_allocator<std::pair<const int, std::string> >());
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  { // Ensure that the hash function is copied
+    using Map = std::unordered_map<int, int, test_hash<int>, std::equal_to<int>, Alloc>;
+    const Map orig(0, test_hash<int>(23));
+    Map copy(orig, new_alloc);
+    assert(copy.hash_function() == test_hash<int>(23));
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    assert(orig.hash_function() == test_hash<int>(23));
   }
-  {
-    typedef explicit_allocator<std::pair<const int, std::string>> A;
-    typedef std::unordered_map<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A{});
-    C c(c0, A{});
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.at(1) == "one");
-    assert(c.at(2) == "two");
-    assert(c.at(3) == "three");
-    assert(c.at(4) == "four");
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A{});
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  { // Ensure that the quality comparator is copied
+    using Map = std::unordered_map<int, int, std::hash<int>, test_equal_to<int>, Alloc>;
+    const Map orig(0, std::hash<int>(), test_equal_to<int>(56));
+    Map copy(orig, new_alloc);
+    assert(copy.key_eq() == test_equal_to<int>(56));
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    assert(orig.key_eq() == test_equal_to<int>(56));
   }
-#endif
+}
+
+void test() {
+  test_alloc(std::allocator<std::pair<const int, int> >());
+  test_alloc(min_allocator<std::pair<const int, int> >());
+  test_alloc(test_allocator<std::pair<const int, int> >(25));
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_copy.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_copy.pass.cpp
index 8897b6abd1439..938b6beccd141 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/assign_copy.pass.cpp
@@ -14,153 +14,473 @@
 
 // unordered_multimap& operator=(const unordered_multimap& u);
 
-#include <unordered_map>
-#include <string>
-#include <set>
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+
+#include <algorithm>
 #include <cassert>
 #include <cfloat>
-#include <cmath>
-#include <algorithm>
 #include <cstddef>
+#include <unordered_map>
+#include <vector>
 
-#include "test_macros.h"
 #include "../../../check_consecutive.h"
 #include "../../../test_compare.h"
 #include "../../../test_hash.h"
-#include "test_allocator.h"
 #include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef test_allocator<std::pair<const int, std::string> > A;
-    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
-    c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-    s.insert("three");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A(4));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class T>
+class tracking_allocator {
+  std::vector<void*>* allocs_;
+
+  template <class U>
+  friend class tracking_allocator;
+
+public:
+  using value_type                             = T;
+  using propagate_on_container_copy_assignment = std::true_type;
+
+  tracking_allocator(std::vector<void*>& allocs) : allocs_(&allocs) {}
+
+  template <class U>
+  tracking_allocator(const tracking_allocator<U>& other) : allocs_(other.allocs_) {}
+
+  T* allocate(std::size_t n) {
+    T* allocation = std::allocator<T>().allocate(n);
+    allocs_->push_back(allocation);
+    return allocation;
   }
-  {
-    typedef std::unordered_multimap<int, std::string> C;
-    typedef std::pair<const int, std::string> P;
-    const P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c(a, a + sizeof(a) / sizeof(a[0]));
-    C* p = &c;
-    c    = *p;
-    assert(c.size() == 6);
-    assert(std::is_permutation(c.begin(), c.end(), a));
+
+  void deallocate(T* ptr, std::size_t n) TEST_NOEXCEPT {
+    auto res = std::remove(allocs_->begin(), allocs_->end(), ptr);
+    assert(res != allocs_->end() && "Trying to deallocate memory from different allocator?");
+    allocs_->erase(res);
+    std::allocator<T>().deallocate(ptr, n);
   }
-  {
-    typedef other_allocator<std::pair<const int, std::string> > A;
-    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
-    c = c0;
-    assert(c.bucket_count() >= 7);
-    assert(c.size() == 6);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-    s.insert("three");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A(10));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+  friend bool operator==(const tracking_allocator& lhs, const tracking_allocator& rhs) {
+    return lhs.allocs_ == rhs.allocs_;
+  }
+
+  friend bool operator!=(const tracking_allocator& lhs, const tracking_allocator& rhs) {
+    return lhs.allocs_ != rhs.allocs_;
+  }
+};
+
+struct NoOp {
+  void operator()() {}
+};
+
+template <class Alloc, class AllocatorInvariant = NoOp>
+void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
+                const Alloc& rhs_alloc                   = Alloc(),
+                AllocatorInvariant check_alloc_invariant = NoOp()) {
+  {   // Test empty/non-empty combinations
+    { // assign from a non-empty container into an empty one
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1), V(2, 3), V(4, 4), V(4, 2)};
+      const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      Map copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.find(1)->second == 1);
+      assert(copy.find(2)->second == 3);
+      {
+        auto range          = copy.equal_range(4);
+        auto first_element  = std::next(range.first, 0);
+        auto second_element = std::next(range.first, 1);
+        auto end            = std::next(range.first, 2);
+
+        assert(range.second == end);
+
+        assert(first_element->second == 2 || first_element->second == 4);
+        assert(second_element->second == 2 || second_element->second == 4);
+        assert(second_element->second != range.first->second);
+      }
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.find(1)->second == 1);
+      assert(orig.find(2)->second == 3);
+      {
+        auto range          = orig.equal_range(4);
+        auto first_element  = std::next(range.first, 0);
+        auto second_element = std::next(range.first, 1);
+        auto end            = std::next(range.first, 2);
+
+        assert(range.second == end);
+
+        assert(first_element->second == 2 || first_element->second == 4);
+        assert(second_element->second == 2 || second_element->second == 4);
+        assert(second_element->second != range.first->second);
+      }
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // assign from an empty container into an empty one
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      const Map orig(rhs_alloc);
+      Map copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+      assert(copy.begin() == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+    { // assign from an empty container into a non-empty one
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      const Map orig(rhs_alloc);
+      Map copy(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), lhs_alloc);
+      copy = orig;
+      // Depending on whether the allocator is propagated the bucked count can change
+      LIBCPP_ASSERT(copy.bucket_count() == 5 || copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+      assert(copy.begin() == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+  }
+  {   // Test empty/one-element copies. In our implementation that's a special case.
+    { // assign from a single-element container into an empty one
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1)};
+      const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      Map copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 2);
+      assert(copy.size() == 1);
+      assert(copy.find(1)->second == 1);
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 2);
+      assert(orig.size() == 1);
+      assert(orig.find(1)->second == 1);
+    }
+    { // assign from an empty container into a single-element one
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1)};
+      const Map orig(rhs_alloc);
+      Map copy(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), lhs_alloc);
+      copy = orig;
+      // Depending on whether the allocator is propagated the bucked count can change
+      LIBCPP_ASSERT(copy.bucket_count() == 2 || copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+    }
+  }
+  {   // Ensure that self-assignment works correctly
+    { // with a non-empty map
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      orig = static_cast<const Map&>(orig);
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.find(1)->second == 1);
+      assert(orig.find(2)->second == 3);
+      assert(orig.find(4)->second == 4);
+      assert(orig.find(5)->second == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // with an empty map
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      Map orig(rhs_alloc);
+      orig = static_cast<const Map&>(orig);
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef min_allocator<std::pair<const int, std::string> > A;
-    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
+  {   // check assignment into a non-empty map
+    { // LHS already contains elements, but fewer than the RHS
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V lhs_arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      const Map orig(begin(lhs_arr), end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      V rhs_arr[] = {V(10, 4), V(13, 5)};
+      Map copy(begin(rhs_arr), end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.find(1)->second == 1);
+      assert(copy.find(2)->second == 3);
+      assert(copy.find(4)->second == 4);
+      assert(copy.find(5)->second == 2);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.find(1)->second == 1);
+      assert(orig.find(2)->second == 3);
+      assert(orig.find(4)->second == 4);
+      assert(orig.find(5)->second == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // LHS contains the same number of elements as the RHS
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V lhs_arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      const Map orig(begin(lhs_arr), end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      V rhs_arr[] = {V(10, 4), V(13, 5), V(12, 324), V(0, 54)};
+      Map copy(begin(rhs_arr), end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.find(1)->second == 1);
+      assert(copy.find(2)->second == 3);
+      assert(copy.find(4)->second == 4);
+      assert(copy.find(5)->second == 2);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.find(1)->second == 1);
+      assert(orig.find(2)->second == 3);
+      assert(orig.find(4)->second == 4);
+      assert(orig.find(5)->second == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // LHS already contains more elements than the RHS
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      V lhs_arr[] = {V(1, 1), V(2, 3), V(4, 4), V(5, 2)};
+      const Map orig(begin(lhs_arr), end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      V rhs_arr[] = {V(10, 4), V(13, 5), V(12, 324), V(0, 54), V(50, 5), V(2, 5)};
+      Map copy(begin(rhs_arr), end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.find(1)->second == 1);
+      assert(copy.find(2)->second == 3);
+      assert(copy.find(4)->second == 4);
+      assert(copy.find(5)->second == 2);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.find(1)->second == 1);
+      assert(orig.find(2)->second == 3);
+      assert(orig.find(4)->second == 4);
+      assert(orig.find(5)->second == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+  }
+}
+
+void test() {
+  test_alloc<std::allocator<std::pair<const int, int> > >();
+  test_alloc<min_allocator<std::pair<const int, int> > >();
+
+  { // Make sure we're allocating/deallocating nodes with the correct allocator
+    // See https://llvm.org/PR29001 (report is for std::map, but the unordered containers have the same optimization)
+    class AssertEmpty {
+      std::vector<void*>* lhs_allocs_;
+      std::vector<void*>* rhs_allocs_;
+
+    public:
+      AssertEmpty(std::vector<void*>& lhs_allocs, std::vector<void*>& rhs_allocs)
+          : lhs_allocs_(&lhs_allocs), rhs_allocs_(&rhs_allocs) {}
+
+      void operator()() {
+        assert(lhs_allocs_->empty());
+        assert(rhs_allocs_->empty());
+      }
     };
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
-    c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-    s.insert("three");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A());
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+    std::vector<void*> lhs_allocs;
+    std::vector<void*> rhs_allocs;
+    test_alloc<tracking_allocator<std::pair<const int, int> > >(
+        lhs_allocs, rhs_allocs, AssertEmpty(lhs_allocs, rhs_allocs));
+  }
+
+  {   // Ensure that the hasher is copied
+    { // when the container is non-empty
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, test_hash<int> >;
+
+      V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+      const Map orig(begin(arr), end(arr), 0, test_hash<int>(5));
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.hash_function() == test_hash<int>(5));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(orig.hash_function() == test_hash<int>(5));
+    }
+    { // when the container is empty
+      using Map = std::unordered_multimap<int, int, test_hash<int> >;
+
+      const Map orig(0, test_hash<int>(5));
+      Map copy;
+      copy = orig;
+      assert(copy.empty());
+      assert(copy.hash_function() == test_hash<int>(5));
+
+      // Check that orig is still what is expected
+      assert(orig.empty());
+      assert(orig.hash_function() == test_hash<int>(5));
+    }
   }
-#endif
+
+  {   // Ensure that the equality comparator is copied
+    { // when the container is non-empty
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int, std::hash<int>, test_equal_to<int> >;
+
+      V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+      const Map orig(begin(arr), end(arr), 0, std::hash<int>(), test_equal_to<int>(23));
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+    }
+    { // when the container is empty
+      using Map = std::unordered_multimap<int, int, std::hash<int>, test_equal_to<int> >;
+
+      const Map orig(0, std::hash<int>(), test_equal_to<int>(23));
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 0);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 0);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+    }
+  }
+
+  {   // Ensure that the max load factor is copied
+    { // when the container is non-empty
+      using V   = std::pair<const int, int>;
+      using Map = std::unordered_multimap<int, int>;
+
+      V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+      Map orig(begin(arr), end(arr));
+      orig.max_load_factor(33.f);
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.max_load_factor() == 33.f);
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(orig.max_load_factor() == 33.f);
+    }
+    { // when the container is empty
+      using Map = std::unordered_multimap<int, int>;
+
+      Map orig;
+      orig.max_load_factor(17.f);
+      Map copy;
+      copy = orig;
+      assert(copy.size() == 0);
+      assert(copy.max_load_factor() == 17.f);
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 0);
+      assert(orig.max_load_factor() == 17.f);
+    }
+  }
+
+  {     // Check that pocca is handled properly
+    {   // pocca == true_type
+      { // when the container is non-empty
+        using V     = std::pair<const int, int>;
+        using Alloc = other_allocator<V>;
+        using Map   = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+        const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), Alloc(3));
+        Map copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(3));
+      }
+      { // when the container is empty
+        using Alloc = other_allocator<std::pair<const int, int> >;
+        using Map   = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        const Map orig(Alloc(3));
+        Map copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(3));
+      }
+    }
+    {   // pocca == false_type
+      { // when the container is non-empty
+        using V     = std::pair<const int, int>;
+        using Alloc = test_allocator<V>;
+        using Map   = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        V arr[] = {V(1, 1), V(2, 2), V(3, 3)};
+        const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), Alloc(3));
+        Map copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(1));
+      }
+      { // when the container is empty
+        using Alloc = test_allocator<std::pair<const int, int> >;
+        using Map   = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        const Map orig(Alloc(3));
+        Map copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(1));
+      }
+    }
+  }
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy.pass.cpp
index e369ce2ac268e..36e6a0487a073 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy.pass.cpp
@@ -29,144 +29,144 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
-  {
-    typedef std::unordered_multimap<int,
-                                    std::string,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    test_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         test_allocator<std::pair<const int, std::string> >(10));
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-    s.insert("three");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(10)));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class Alloc>
+void test_alloc() {
+  { // Simple check
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    V arr[] = {V(1, 2), V(2, 4), V(3, 1), V(3, 2)};
+    const Map orig(std::begin(arr), std::end(arr));
+    Map copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 5);
+    assert(copy.size() == 4);
+    assert(copy.find(1)->second == 2);
+    assert(copy.find(2)->second == 4);
+    {
+      auto range          = copy.equal_range(3);
+      auto first_element  = std::next(range.first, 0);
+      auto second_element = std::next(range.first, 1);
+      auto end            = std::next(range.first, 2);
+
+      assert(range.second == end);
+
+      assert(first_element->second == 1 || first_element->second == 2);
+      assert(second_element->second == 1 || second_element->second == 2);
+      assert(second_element->second != range.first->second);
+    }
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 5);
+    assert(orig.size() == 4);
+    assert(orig.find(1)->second == 2);
+    assert(orig.find(2)->second == 4);
+    {
+      auto range          = orig.equal_range(3);
+      auto first_element  = std::next(range.first, 0);
+      auto second_element = std::next(range.first, 1);
+      auto end            = std::next(range.first, 2);
+
+      assert(range.second == end);
+
+      assert(first_element->second == 1 || first_element->second == 2);
+      assert(second_element->second == 1 || second_element->second == 2);
+      assert(second_element->second != range.first->second);
+    }
+  }
+  { // single element copy
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    V arr[] = {V(1, 2)};
+    const Map orig(std::begin(arr), std::end(arr));
+    Map copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 2);
+    assert(copy.size() == 1);
+    assert(copy.find(1)->second == 2);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 2);
+    assert(orig.size() == 1);
+    assert(orig.find(1)->second == 2);
+  }
+  { // Copy empty map
+    using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    const Map orig;
+    Map copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 0);
+    assert(copy.size() == 0);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 0);
+    assert(orig.size() == 0);
+  }
+  { // Ensure that the hash function is copied
+    using Map = std::unordered_multimap<int, int, test_hash<int>, std::equal_to<int>, Alloc>;
+    const Map orig(0, test_hash<int>(23));
+    Map copy = orig;
+    assert(copy.hash_function() == test_hash<int>(23));
+
+    // Check that orig is still what is expected
+    assert(orig.hash_function() == test_hash<int>(23));
+  }
+  { // Ensure that the quality comparator is copied
+    using Map = std::unordered_multimap<int, int, std::hash<int>, test_equal_to<int>, Alloc>;
+    const Map orig(0, std::hash<int>(), test_equal_to<int>(56));
+    Map copy = orig;
+    assert(copy.key_eq() == test_equal_to<int>(56));
+
+    // Check that orig is still what is expected
+    assert(orig.key_eq() == test_equal_to<int>(56));
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::unordered_multimap<int,
-                                    std::string,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    other_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         other_allocator<std::pair<const int, std::string> >(10));
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-    s.insert("three");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (other_allocator<std::pair<const int, std::string> >(-2)));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+}
+
+void test() {
+  test_alloc<std::allocator<std::pair<const int, int> > >();
+  test_alloc<min_allocator<std::pair<const int, int> > >();
+
+  { // Ensure that the allocator is copied
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, test_allocator<V> >;
+
+    V arr[] = {V(1, 1), V(2, 3), V(3, 6)};
+    const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), test_allocator<V>(10));
+    Map copy = orig;
+    assert(copy.size() == 3);
+    assert(copy.get_allocator() == test_allocator<V>(10));
+
+    // Check that orig is still what is expected
+    assert(orig.size() == 3);
+    assert(orig.get_allocator() == test_allocator<V>(10));
+    assert(orig.get_allocator().get_id() != test_alloc_base::moved_value);
   }
-  {
-    typedef std::unordered_multimap<int,
-                                    std::string,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    min_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         min_allocator<std::pair<const int, std::string> >());
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-    s.insert("three");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+  { // Ensure that soccc is handled properly
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, other_allocator<V> >;
+
+    V arr[] = {V(1, 1), V(2, 3), V(3, 6)};
+    const Map orig(begin(arr), end(arr), 0, std::hash<int>(), std::equal_to<int>(), other_allocator<V>(10));
+    Map copy = orig;
+    assert(copy.size() == 3);
+    assert(copy.get_allocator() == other_allocator<V>(-2));
+
+    // Check that orig is still what is expected
+    assert(orig.size() == 3);
+    assert(orig.get_allocator() == other_allocator<V>(10));
   }
-#endif
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy_alloc.pass.cpp
index 25348786a3607..e9e3668b8b2be 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/copy_alloc.pass.cpp
@@ -29,135 +29,120 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
-  {
-    typedef std::unordered_multimap<int,
-                                    std::string,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    test_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         test_allocator<std::pair<const int, std::string> >(10));
-    C c(c0, test_allocator<std::pair<const int, std::string> >(5));
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-    s.insert("three");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (test_allocator<std::pair<const int, std::string> >(5)));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class Alloc>
+void test_alloc(const Alloc& new_alloc) {
+  { // Simple check
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    V arr[] = {V(1, 2), V(2, 4), V(3, 1), V(3, 2)};
+    const Map orig(std::begin(arr), std::end(arr));
+    Map copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 5);
+    assert(copy.size() == 4);
+    assert(copy.find(1)->second == 2);
+    assert(copy.find(2)->second == 4);
+    {
+      auto range          = copy.equal_range(3);
+      auto first_element  = std::next(range.first, 0);
+      auto second_element = std::next(range.first, 1);
+      auto end            = std::next(range.first, 2);
+
+      assert(range.second == end);
+
+      assert(first_element->second == 1 || first_element->second == 2);
+      assert(second_element->second == 1 || second_element->second == 2);
+      assert(second_element->second != range.first->second);
+    }
+
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 5);
+    assert(orig.size() == 4);
+    assert(orig.find(1)->second == 2);
+    assert(orig.find(2)->second == 4);
+    {
+      auto range          = orig.equal_range(3);
+      auto first_element  = std::next(range.first, 0);
+      auto second_element = std::next(range.first, 1);
+      auto end            = std::next(range.first, 2);
+
+      assert(range.second == end);
+
+      assert(first_element->second == 1 || first_element->second == 2);
+      assert(second_element->second == 1 || second_element->second == 2);
+      assert(second_element->second != range.first->second);
+    }
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::unordered_multimap<int,
-                                    std::string,
-                                    test_hash<int>,
-                                    test_equal_to<int>,
-                                    min_allocator<std::pair<const int, std::string> > >
-        C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a,
-         a + sizeof(a) / sizeof(a[0]),
-         7,
-         test_hash<int>(8),
-         test_equal_to<int>(9),
-         min_allocator<std::pair<const int, std::string> >());
-    C c(c0, min_allocator<std::pair<const int, std::string> >());
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-    s.insert("three");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == (min_allocator<std::pair<const int, std::string> >()));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  { // single element check
+    using V   = std::pair<const int, int>;
+    using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    V arr[] = {V(1, 2)};
+    const Map orig(std::begin(arr), std::end(arr));
+    Map copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 2);
+    assert(copy.size() == 1);
+    assert(copy.find(1)->second == 2);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 2);
+    assert(orig.size() == 1);
+    assert(orig.find(1)->second == 2);
   }
-  {
-    typedef explicit_allocator<std::pair<const int, std::string>> A;
-    typedef std::unordered_multimap<int, std::string, test_hash<int>, test_equal_to<int>, A > C;
-    typedef std::pair<int, std::string> P;
-    P a[] = {
-        P(1, "one"),
-        P(2, "two"),
-        P(3, "three"),
-        P(4, "four"),
-        P(1, "four"),
-        P(2, "four"),
-    };
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A{});
-    C c(c0, A{});
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    std::multiset<std::string> s;
-    s.insert("one");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(1), c.end(), 1, s);
-    s.insert("two");
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(2), c.end(), 2, s);
-    s.insert("three");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(3), c.end(), 3, s);
-    s.insert("four");
-    CheckConsecutiveKeys<C::const_iterator>(c.find(4), c.end(), 4, s);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A{});
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  { // Copy empty map
+    using Map = std::unordered_multimap<int, int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    const Map orig;
+    Map copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 0);
+    assert(copy.size() == 0);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 0);
+    assert(orig.size() == 0);
+  }
+  { // Ensure that the hash function is copied
+    using Map = std::unordered_multimap<int, int, test_hash<int>, std::equal_to<int>, Alloc>;
+    const Map orig(0, test_hash<int>(23));
+    Map copy(orig, new_alloc);
+    assert(copy.hash_function() == test_hash<int>(23));
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    assert(orig.hash_function() == test_hash<int>(23));
+  }
+  { // Ensure that the quality comparator is copied
+    using Map = std::unordered_multimap<int, int, std::hash<int>, test_equal_to<int>, Alloc>;
+    const Map orig(0, std::hash<int>(), test_equal_to<int>(56));
+    Map copy(orig, new_alloc);
+    assert(copy.key_eq() == test_equal_to<int>(56));
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    assert(orig.key_eq() == test_equal_to<int>(56));
   }
-#endif
+}
+
+void test() {
+  test_alloc(std::allocator<std::pair<const int, int> >());
+  test_alloc(min_allocator<std::pair<const int, int> >());
+  test_alloc(test_allocator<std::pair<const int, int> >(25));
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_copy.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_copy.pass.cpp
index 248d446b92eb7..e415253c5f60f 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/assign_copy.pass.cpp
@@ -14,103 +14,437 @@
 
 // unordered_multiset& operator=(const unordered_multiset& u);
 
-#include <unordered_set>
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+
 #include <algorithm>
 #include <cassert>
 #include <cfloat>
 #include <cmath>
 #include <cstddef>
+#include <unordered_set>
+#include <vector>
 
-#include "test_macros.h"
-#include "../../../check_consecutive.h"
 #include "../../../test_compare.h"
 #include "../../../test_hash.h"
-#include "test_allocator.h"
 #include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef test_allocator<int> A;
-    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
-    c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A(4));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class T>
+class tracking_allocator {
+  std::vector<void*>* allocs_;
+
+  template <class U>
+  friend class tracking_allocator;
+
+public:
+  using value_type                             = T;
+  using propagate_on_container_copy_assignment = std::true_type;
+
+  tracking_allocator(std::vector<void*>& allocs) : allocs_(&allocs) {}
+
+  template <class U>
+  tracking_allocator(const tracking_allocator<U>& other) : allocs_(other.allocs_) {}
+
+  T* allocate(std::size_t n) {
+    T* allocation = std::allocator<T>().allocate(n);
+    allocs_->push_back(allocation);
+    return allocation;
   }
-  {
-    typedef std::unordered_multiset<int> C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c(a, a + sizeof(a) / sizeof(a[0]));
-    C* p = &c;
-    c    = *p;
-
-    assert(c.size() == 6);
-    assert(std::is_permutation(c.begin(), c.end(), a));
+
+  void deallocate(T* ptr, std::size_t n) TEST_NOEXCEPT {
+    auto res = std::remove(allocs_->begin(), allocs_->end(), ptr);
+    assert(res != allocs_->end() && "Trying to deallocate memory from different allocator?");
+    allocs_->erase(res);
+    std::allocator<T>().deallocate(ptr, n);
   }
-  {
-    typedef other_allocator<int> A;
-    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
-    c = c0;
-    assert(c.bucket_count() >= 7);
-    assert(c.size() == 6);
-    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A(10));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+  friend bool operator==(const tracking_allocator& lhs, const tracking_allocator& rhs) {
+    return lhs.allocs_ == rhs.allocs_;
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef min_allocator<int> A;
-    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, A > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
-    c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A());
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+  friend bool operator!=(const tracking_allocator& lhs, const tracking_allocator& rhs) {
+    return lhs.allocs_ != rhs.allocs_;
   }
-#endif
+};
+
+struct NoOp {
+  void operator()() {}
+};
+
+template <class Alloc, class AllocatorInvariant = NoOp>
+void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
+                const Alloc& rhs_alloc                   = Alloc(),
+                AllocatorInvariant check_alloc_invariant = NoOp()) {
+  {   // Test empty/non-empty combinations
+    { // assign from a non-empty container into an empty one
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1, 2, 4, 4};
+      const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      Set copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.count(1) == 1);
+      assert(copy.count(2) == 1);
+      assert(copy.count(4) == 2);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 2);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // assign from an empty container into an empty one
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      const Set orig(rhs_alloc);
+      Set copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+      assert(copy.begin() == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+    { // assign from an empty container into a non-empty one
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1, 2, 4, 4};
+      const Set orig(rhs_alloc);
+      Set copy(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), lhs_alloc);
+      copy = orig;
+      // Depending on whether the allocator is propagated the bucked count can change
+      LIBCPP_ASSERT(copy.bucket_count() == 5 || copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+      assert(copy.begin() == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+  }
+  {   // Test empty/one-element copies. In our implementation that's a special case.
+    { // assign from a single-element container into an empty one
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1};
+      const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      Set copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 2);
+      assert(copy.size() == 1);
+      assert(copy.count(1) == 1);
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 2);
+      assert(orig.size() == 1);
+      assert(orig.count(1) == 1);
+    }
+    { // assign from an empty container into a single-element one
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1};
+      const Set orig(rhs_alloc);
+      Set copy(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), lhs_alloc);
+      copy = orig;
+      // Depending on whether the allocator is propagated the bucked count can change
+      LIBCPP_ASSERT(copy.bucket_count() == 2 || copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+    }
+  }
+  {   // Ensure that self-assignment works correctly
+    { // with a non-empty map
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1, 2, 4, 5};
+      Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      orig = static_cast<const Set&>(orig);
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 1);
+      assert(orig.count(5) == 1);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // with an empty map
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      Set orig(rhs_alloc);
+      orig = static_cast<const Set&>(orig);
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+  }
+  {   // check assignment into a non-empty map
+    { // LHS already contains elements, but fewer than the RHS
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int lhs_arr[] = {1, 2, 4, 5};
+      const Set orig(std::begin(lhs_arr), std::end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      int rhs_arr[] = {10, 13};
+      Set copy(std::begin(rhs_arr), std::end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.count(1) == 1);
+      assert(copy.count(2) == 1);
+      assert(copy.count(4) == 1);
+      assert(copy.count(5) == 1);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 1);
+      assert(orig.count(5) == 1);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // LHS contains the same number of elements as the RHS
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int lhs_arr[] = {1, 2, 4, 5};
+      const Set orig(std::begin(lhs_arr), std::end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      int rhs_arr[] = {10, 13, 12, 0};
+      Set copy(std::begin(rhs_arr), std::end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.count(1) == 1);
+      assert(copy.count(2) == 1);
+      assert(copy.count(4) == 1);
+      assert(copy.count(5) == 1);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 1);
+      assert(orig.count(5) == 1);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // LHS already contains more elements than the RHS
+      using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int lhs_arr[] = {1, 2, 4, 5};
+      const Set orig(std::begin(lhs_arr), std::end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      int rhs_arr[] = {10, 13, 12, 0, 50, 2};
+      Set copy(std::begin(rhs_arr), std::end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.count(1) == 1);
+      assert(copy.count(2) == 1);
+      assert(copy.count(4) == 1);
+      assert(copy.count(5) == 1);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 1);
+      assert(orig.count(5) == 1);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+  }
+}
+
+void test() {
+  test_alloc<std::allocator<int> >();
+  test_alloc<min_allocator<int> >();
+
+  { // Make sure we're allocating/deallocating nodes with the correct allocator
+    // See https://llvm.org/PR29001 (report is for std::map, but the unordered containers have the same optimization)
+    class AssertEmpty {
+      std::vector<void*>* lhs_allocs_;
+      std::vector<void*>* rhs_allocs_;
+
+    public:
+      AssertEmpty(std::vector<void*>& lhs_allocs, std::vector<void*>& rhs_allocs)
+          : lhs_allocs_(&lhs_allocs), rhs_allocs_(&rhs_allocs) {}
+
+      void operator()() {
+        assert(lhs_allocs_->empty());
+        assert(rhs_allocs_->empty());
+      }
+    };
+
+    std::vector<void*> lhs_allocs;
+    std::vector<void*> rhs_allocs;
+    test_alloc<tracking_allocator<int> >(lhs_allocs, rhs_allocs, AssertEmpty(lhs_allocs, rhs_allocs));
+  }
+
+  {   // Ensure that the hasher is copied
+    { // when the container is non-empty
+      using Set = std::unordered_multiset<int, test_hash<int> >;
+
+      int arr[] = {1, 2, 3};
+      const Set orig(std::begin(arr), std::end(arr), 0, test_hash<int>(5));
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.hash_function() == test_hash<int>(5));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(orig.hash_function() == test_hash<int>(5));
+    }
+    { // when the container is empty
+      using Set = std::unordered_multiset<int, test_hash<int> >;
+
+      const Set orig(0, test_hash<int>(5));
+      Set copy;
+      copy = orig;
+      assert(copy.empty());
+      assert(copy.hash_function() == test_hash<int>(5));
+
+      // Check that orig is still what is expected
+      assert(orig.empty());
+      assert(orig.hash_function() == test_hash<int>(5));
+    }
+  }
+
+  {   // Ensure that the equality comparator is copied
+    { // when the container is non-empty
+      using Set = std::unordered_multiset<int, std::hash<int>, test_equal_to<int> >;
+
+      int arr[] = {1, 2, 3};
+      const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), test_equal_to<int>(23));
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+    }
+    { // when the container is empty
+      using Set = std::unordered_multiset<int, std::hash<int>, test_equal_to<int> >;
+
+      const Set orig(0, std::hash<int>(), test_equal_to<int>(23));
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 0);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 0);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+    }
+  }
+
+  {   // Ensure that the max load factor is copied
+    { // when the container is non-empty
+      using Set = std::unordered_multiset<int>;
+
+      int arr[] = {1, 2, 3};
+      Set orig(std::begin(arr), std::end(arr));
+      orig.max_load_factor(33.f);
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.max_load_factor() == 33.f);
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(orig.max_load_factor() == 33.f);
+    }
+    { // when the container is empty
+      using Set = std::unordered_multiset<int>;
+
+      Set orig;
+      orig.max_load_factor(17.f);
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 0);
+      assert(copy.max_load_factor() == 17.f);
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 0);
+      assert(orig.max_load_factor() == 17.f);
+    }
+  }
+
+  {     // Check that pocca is handled properly
+    {   // pocca == true_type
+      { // when the container is non-empty
+        using Alloc = other_allocator<int>;
+        using Set   = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        int arr[] = {1, 2, 3};
+        const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), Alloc(3));
+        Set copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(3));
+      }
+      { // when the container is empty
+        using Alloc = other_allocator<int>;
+        using Set   = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        const Set orig(Alloc(3));
+        Set copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(3));
+      }
+    }
+    {   // pocca == false_type
+      { // when the container is non-empty
+        using Alloc = test_allocator<int>;
+        using Set   = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        int arr[] = {1, 2, 3};
+        const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), Alloc(3));
+        Set copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(1));
+      }
+      { // when the container is empty
+        using Alloc = test_allocator<int>;
+        using Set   = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        const Set orig(Alloc(3));
+        Set copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(1));
+      }
+    }
+  }
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy.pass.cpp
index fc577f323e43b..bc74d4a2bbc57 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy.pass.cpp
@@ -27,72 +27,118 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
-  {
-    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == test_allocator<int>(10));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class Alloc>
+void test_alloc() {
+  { // Simple check
+    using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    int arr[] = {1, 2, 3, 3};
+    const Set orig(std::begin(arr), std::end(arr));
+    Set copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 5);
+    assert(copy.size() == 4);
+    assert(copy.count(1) == 1);
+    assert(copy.count(2) == 1);
+    assert(copy.count(3) == 2);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 5);
+    assert(orig.size() == 4);
+    assert(orig.count(1) == 1);
+    assert(orig.count(2) == 1);
+    assert(orig.count(3) == 2);
+  }
+  { // single element copy
+    using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    int arr[] = {1};
+    const Set orig(std::begin(arr), std::end(arr));
+    Set copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 2);
+    assert(copy.size() == 1);
+    assert(copy.count(1) == 1);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 2);
+    assert(orig.size() == 1);
+    assert(orig.count(1) == 1);
+  }
+  { // Copy empty map
+    using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    const Set orig;
+    Set copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 0);
+    assert(copy.size() == 0);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 0);
+    assert(orig.size() == 0);
+  }
+  { // Ensure that the hash function is copied
+    using Set = std::unordered_multiset<int, test_hash<int>, std::equal_to<int>, Alloc>;
+    const Set orig(0, test_hash<int>(23));
+    Set copy = orig;
+    assert(copy.hash_function() == test_hash<int>(23));
+
+    // Check that orig is still what is expected
+    assert(orig.hash_function() == test_hash<int>(23));
+  }
+  { // Ensure that the quality comparator is copied
+    using Set = std::unordered_multiset<int, std::hash<int>, test_equal_to<int>, Alloc>;
+    const Set orig(0, std::hash<int>(), test_equal_to<int>(56));
+    Set copy = orig;
+    assert(copy.key_eq() == test_equal_to<int>(56));
+
+    // Check that orig is still what is expected
+    assert(orig.key_eq() == test_equal_to<int>(56));
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, other_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), other_allocator<int>(10));
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == other_allocator<int>(-2));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+}
+
+void test() {
+  test_alloc<std::allocator<int> >();
+  test_alloc<min_allocator<int> >();
+
+  { // Ensure that the allocator is copied
+    using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, test_allocator<int> >;
+
+    int arr[] = {1, 2, 3};
+    const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), test_allocator<int>(10));
+    Set copy = orig;
+    assert(copy.size() == 3);
+    assert(copy.get_allocator() == test_allocator<int>(10));
+
+    // Check that orig is still what is expected
+    assert(orig.size() == 3);
+    assert(orig.get_allocator() == test_allocator<int>(10));
+    assert(orig.get_allocator().get_id() != test_alloc_base::moved_value);
   }
-  {
-    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == min_allocator<int>());
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+  { // Ensure that soccc is handled properly
+    using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, other_allocator<int> >;
+
+    int arr[] = {1, 2, 3};
+    const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), other_allocator<int>(10));
+    Set copy = orig;
+    assert(copy.size() == 3);
+    assert(copy.get_allocator() == other_allocator<int>(-2));
+
+    // Check that orig is still what is expected
+    assert(orig.size() == 3);
+    assert(orig.get_allocator() == other_allocator<int>(10));
   }
-#endif
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy_alloc.pass.cpp
index 7f45a3f2e4bd4..944865183cdda 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/copy_alloc.pass.cpp
@@ -14,64 +14,107 @@
 
 // unordered_multiset(const unordered_multiset& u, const allocator_type& a);
 
-#include <unordered_set>
 #include <cassert>
 #include <cfloat>
 #include <cmath>
 #include <cstddef>
+#include <unordered_set>
 
-#include "test_macros.h"
-#include "../../../check_consecutive.h"
 #include "../../../test_compare.h"
 #include "../../../test_hash.h"
-#include "test_allocator.h"
 #include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
-    C c(c0, test_allocator<int>(5));
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == test_allocator<int>(5));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class Alloc>
+void test_alloc(const Alloc& new_alloc) {
+  { // Simple check
+    using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    int arr[] = {1, 2, 3, 3};
+    const Set orig(std::begin(arr), std::end(arr));
+    Set copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 5);
+    assert(copy.size() == 4);
+    assert(copy.count(1) == 1);
+    assert(copy.count(2) == 1);
+    assert(copy.count(3) == 2);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 5);
+    assert(orig.size() == 4);
+    assert(orig.count(1) == 1);
+    assert(orig.count(2) == 1);
+    assert(orig.count(3) == 2);
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::unordered_multiset<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
-    C c(c0, min_allocator<int>());
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 6);
-    CheckConsecutiveValues<C::const_iterator>(c.find(1), c.end(), 1, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(2), c.end(), 2, 2);
-    CheckConsecutiveValues<C::const_iterator>(c.find(3), c.end(), 3, 1);
-    CheckConsecutiveValues<C::const_iterator>(c.find(4), c.end(), 4, 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == min_allocator<int>());
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  { // single element check
+    using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    int arr[] = {1};
+    const Set orig(std::begin(arr), std::end(arr));
+    Set copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 2);
+    assert(copy.size() == 1);
+    assert(copy.count(1) == 1);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 2);
+    assert(orig.size() == 1);
+    assert(orig.count(1) == 1);
   }
-#endif
+  { // Copy empty map
+    using Set = std::unordered_multiset<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    const Set orig;
+    Set copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 0);
+    assert(copy.size() == 0);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 0);
+    assert(orig.size() == 0);
+  }
+  { // Ensure that the hash function is copied
+    using Set = std::unordered_multiset<int, test_hash<int>, std::equal_to<int>, Alloc>;
+    const Set orig(0, test_hash<int>(23));
+    Set copy(orig, new_alloc);
+    assert(copy.hash_function() == test_hash<int>(23));
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    assert(orig.hash_function() == test_hash<int>(23));
+  }
+  { // Ensure that the quality comparator is copied
+    using Set = std::unordered_multiset<int, std::hash<int>, test_equal_to<int>, Alloc>;
+    const Set orig(0, std::hash<int>(), test_equal_to<int>(56));
+    Set copy(orig, new_alloc);
+    assert(copy.key_eq() == test_equal_to<int>(56));
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    assert(orig.key_eq() == test_equal_to<int>(56));
+  }
+}
+
+void test() {
+  test_alloc(std::allocator<int>());
+  test_alloc(min_allocator<int>());
+  test_alloc(test_allocator<int>(25));
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp
index e606e538efcbe..9828b8b459c89 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/assign_copy.pass.cpp
@@ -14,115 +14,440 @@
 
 // unordered_set& operator=(const unordered_set& u);
 
-#include <unordered_set>
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+
 #include <algorithm>
 #include <cassert>
 #include <cfloat>
 #include <cmath>
 #include <cstddef>
+#include <unordered_set>
 #include <utility>
+#include <vector>
 
-#include "test_macros.h"
 #include "../../../test_compare.h"
 #include "../../../test_hash.h"
-#include "test_allocator.h"
 #include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef test_allocator<int> A;
-    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
-    c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A(4));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class T>
+class tracking_allocator {
+  std::vector<void*>* allocs_;
+
+  template <class U>
+  friend class tracking_allocator;
+
+public:
+  using value_type                             = T;
+  using propagate_on_container_copy_assignment = std::true_type;
+
+  tracking_allocator(std::vector<void*>& allocs) : allocs_(&allocs) {}
+
+  template <class U>
+  tracking_allocator(const tracking_allocator<U>& other) : allocs_(other.allocs_) {}
+
+  T* allocate(std::size_t n) {
+    T* allocation = std::allocator<T>().allocate(n);
+    allocs_->push_back(allocation);
+    return allocation;
   }
-  {
-    typedef std::unordered_set<int> C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c(a, a + sizeof(a) / sizeof(a[0]));
-    C* p = &c;
-    c    = *p;
-    assert(c.size() == 4);
-    assert(std::is_permutation(c.begin(), c.end(), a));
+
+  void deallocate(T* ptr, std::size_t n) TEST_NOEXCEPT {
+    auto res = std::remove(allocs_->begin(), allocs_->end(), ptr);
+    assert(res != allocs_->end() && "Trying to deallocate memory from different allocator?");
+    allocs_->erase(res);
+    std::allocator<T>().deallocate(ptr, n);
   }
-  {
-    typedef other_allocator<int> A;
-    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A(10));
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A(4));
-    c = c0;
-    assert(c.bucket_count() >= 5);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A(10));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+  friend bool operator==(const tracking_allocator& lhs, const tracking_allocator& rhs) {
+    return lhs.allocs_ == rhs.allocs_;
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef min_allocator<int> A;
-    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, A > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), A());
-    C c(a, a + 2, 7, test_hash<int>(2), test_equal_to<int>(3), A());
-    c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == A());
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+
+  friend bool operator!=(const tracking_allocator& lhs, const tracking_allocator& rhs) {
+    return lhs.allocs_ != rhs.allocs_;
+  }
+};
+
+struct NoOp {
+  void operator()() {}
+};
+
+template <class Alloc, class AllocatorInvariant = NoOp>
+void test_alloc(const Alloc& lhs_alloc                   = Alloc(),
+                const Alloc& rhs_alloc                   = Alloc(),
+                AllocatorInvariant check_alloc_invariant = NoOp()) {
+  {   // Test empty/non-empty combinations
+    { // assign from a non-empty container into an empty one
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1, 2, 4, 5};
+      const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      Set copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.count(1) == 1);
+      assert(copy.count(2) == 1);
+      assert(copy.count(4) == 1);
+      assert(copy.count(5) == 1);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 1);
+      assert(orig.count(5) == 1);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // assign from an empty container into an empty one
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      const Set orig(rhs_alloc);
+      Set copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+      assert(copy.begin() == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+    { // assign from an empty container into a non-empty one
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1, 2, 4, 5};
+      const Set orig(rhs_alloc);
+      Set copy(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), lhs_alloc);
+      copy = orig;
+      // Depending on whether the allocator is propagated the bucked count can change
+      LIBCPP_ASSERT(copy.bucket_count() == 5 || copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+      assert(copy.begin() == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+  }
+  {   // Test empty/one-element copies. In our implementation that's a special case.
+    { // assign from a single-element container into an empty one
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1};
+      const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      Set copy(lhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 2);
+      assert(copy.size() == 1);
+      assert(copy.count(1) == 1);
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 2);
+      assert(orig.size() == 1);
+      assert(orig.count(1) == 1);
+    }
+    { // assign from an empty container into a single-element one
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1};
+      const Set orig(rhs_alloc);
+      Set copy(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), lhs_alloc);
+      copy = orig;
+      // Depending on whether the allocator is propagated the bucked count can change
+      LIBCPP_ASSERT(copy.bucket_count() == 2 || copy.bucket_count() == 0);
+      assert(copy.size() == 0);
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+    }
+  }
+  {   // Ensure that self-assignment works correctly
+    { // with a non-empty map
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int arr[] = {1, 2, 4, 5};
+      Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      orig = static_cast<const Set&>(orig);
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 1);
+      assert(orig.count(5) == 1);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // with an empty map
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      Set orig(rhs_alloc);
+      orig = static_cast<const Set&>(orig);
+      LIBCPP_ASSERT(orig.bucket_count() == 0);
+      assert(orig.size() == 0);
+      assert(orig.begin() == orig.end());
+    }
+    check_alloc_invariant();
+  }
+  {   // check assignment into a non-empty map
+    { // LHS already contains elements, but fewer than the RHS
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int lhs_arr[] = {1, 2, 4, 5};
+      const Set orig(std::begin(lhs_arr), std::end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      int rhs_arr[] = {10, 13};
+      Set copy(std::begin(rhs_arr), std::end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.count(1) == 1);
+      assert(copy.count(2) == 1);
+      assert(copy.count(4) == 1);
+      assert(copy.count(5) == 1);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 1);
+      assert(orig.count(5) == 1);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // LHS contains the same number of elements as the RHS
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int lhs_arr[] = {1, 2, 4, 5};
+      const Set orig(std::begin(lhs_arr), std::end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      int rhs_arr[] = {10, 13, 12, 0};
+      Set copy(std::begin(rhs_arr), std::end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.count(1) == 1);
+      assert(copy.count(2) == 1);
+      assert(copy.count(4) == 1);
+      assert(copy.count(5) == 1);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 1);
+      assert(orig.count(5) == 1);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
+    { // LHS already contains more elements than the RHS
+      using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+      int lhs_arr[] = {1, 2, 4, 5};
+      const Set orig(std::begin(lhs_arr), std::end(lhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+
+      int rhs_arr[] = {10, 13, 12, 0, 50, 2};
+      Set copy(std::begin(rhs_arr), std::end(rhs_arr), 0, std::hash<int>(), std::equal_to<int>(), rhs_alloc);
+      copy = orig;
+      LIBCPP_ASSERT(copy.bucket_count() == 5);
+      assert(copy.size() == 4);
+      assert(copy.count(1) == 1);
+      assert(copy.count(2) == 1);
+      assert(copy.count(4) == 1);
+      assert(copy.count(5) == 1);
+      assert(std::next(copy.begin(), 4) == copy.end());
+
+      // Check that orig is still what is expected
+      LIBCPP_ASSERT(orig.bucket_count() == 5);
+      assert(orig.size() == 4);
+      assert(orig.count(1) == 1);
+      assert(orig.count(2) == 1);
+      assert(orig.count(4) == 1);
+      assert(orig.count(5) == 1);
+      assert(std::next(orig.begin(), 4) == orig.end());
+    }
+    check_alloc_invariant();
   }
-#endif
-  { // Test with std::pair, since we have some special handling for pairs inside __hash_table
-    struct pair_hash {
-      size_t operator()(std::pair<int, int> val) const TEST_NOEXCEPT { return val.first | val.second; }
+}
+
+void test() {
+  test_alloc<std::allocator<int> >();
+  test_alloc<min_allocator<int> >();
+
+  { // Make sure we're allocating/deallocating nodes with the correct allocator
+    // See https://llvm.org/PR29001 (report is for std::map, but the unordered containers have the same optimization)
+    class AssertEmpty {
+      std::vector<void*>* lhs_allocs_;
+      std::vector<void*>* rhs_allocs_;
+
+    public:
+      AssertEmpty(std::vector<void*>& lhs_allocs, std::vector<void*>& rhs_allocs)
+          : lhs_allocs_(&lhs_allocs), rhs_allocs_(&rhs_allocs) {}
+
+      void operator()() {
+        assert(lhs_allocs_->empty());
+        assert(rhs_allocs_->empty());
+      }
     };
 
-    std::pair<int, int> arr[] = {
-        std::make_pair(1, 2), std::make_pair(2, 3), std::make_pair(3, 4), std::make_pair(4, 5)};
-    std::unordered_set<std::pair<int, int>, pair_hash> a(arr, arr + 4);
-    std::unordered_set<std::pair<int, int>, pair_hash> b;
+    std::vector<void*> lhs_allocs;
+    std::vector<void*> rhs_allocs;
+    test_alloc<tracking_allocator<int> >(lhs_allocs, rhs_allocs, AssertEmpty(lhs_allocs, rhs_allocs));
+  }
+
+  {   // Ensure that the hasher is copied
+    { // when the container is non-empty
+      using Set = std::unordered_set<int, test_hash<int> >;
+
+      int arr[] = {1, 2, 3};
+      const Set orig(std::begin(arr), std::end(arr), 0, test_hash<int>(5));
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.hash_function() == test_hash<int>(5));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(orig.hash_function() == test_hash<int>(5));
+    }
+    { // when the container is empty
+      using Set = std::unordered_set<int, test_hash<int> >;
 
-    b = a;
-    assert(a == b);
+      const Set orig(0, test_hash<int>(5));
+      Set copy;
+      copy = orig;
+      assert(copy.empty());
+      assert(copy.hash_function() == test_hash<int>(5));
+
+      // Check that orig is still what is expected
+      assert(orig.empty());
+      assert(orig.hash_function() == test_hash<int>(5));
+    }
   }
 
+  {   // Ensure that the equality comparator is copied
+    { // when the container is non-empty
+      using Set = std::unordered_set<int, std::hash<int>, test_equal_to<int> >;
+
+      int arr[] = {1, 2, 3};
+      const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), test_equal_to<int>(23));
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+    }
+    { // when the container is empty
+      using Set = std::unordered_set<int, std::hash<int>, test_equal_to<int> >;
+
+      const Set orig(0, std::hash<int>(), test_equal_to<int>(23));
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 0);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 0);
+      assert(copy.key_eq() == test_equal_to<int>(23));
+    }
+  }
+
+  {   // Ensure that the max load factor is copied
+    { // when the container is non-empty
+      using Set = std::unordered_set<int>;
+
+      int arr[] = {1, 2, 3};
+      Set orig(std::begin(arr), std::end(arr));
+      orig.max_load_factor(33.f);
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 3);
+      assert(copy.max_load_factor() == 33.f);
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 3);
+      assert(orig.max_load_factor() == 33.f);
+    }
+    { // when the container is empty
+      using Set = std::unordered_set<int>;
+
+      Set orig;
+      orig.max_load_factor(17.f);
+      Set copy;
+      copy = orig;
+      assert(copy.size() == 0);
+      assert(copy.max_load_factor() == 17.f);
+
+      // Check that orig is still what is expected
+      assert(orig.size() == 0);
+      assert(orig.max_load_factor() == 17.f);
+    }
+  }
+
+  {     // Check that pocca is handled properly
+    {   // pocca == true_type
+      { // when the container is non-empty
+        using Alloc = other_allocator<int>;
+        using Set   = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        int arr[] = {1, 2, 3};
+        const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), Alloc(3));
+        Set copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(3));
+      }
+      { // when the container is empty
+        using Alloc = other_allocator<int>;
+        using Set   = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        const Set orig(Alloc(3));
+        Set copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(3));
+      }
+    }
+    {   // pocca == false_type
+      { // when the container is non-empty
+        using Alloc = test_allocator<int>;
+        using Set   = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        int arr[] = {1, 2, 3};
+        const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), Alloc(3));
+        Set copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(1));
+      }
+      { // when the container is empty
+        using Alloc = test_allocator<int>;
+        using Set   = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+        const Set orig(Alloc(3));
+        Set copy(Alloc(1));
+        copy = orig;
+        assert(copy.get_allocator() == Alloc(1));
+      }
+    }
+  }
+}
+
+int main(int, char**) {
+  test();
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy.pass.cpp
index e795d810e9fa1..ff8fa13cffe64 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy.pass.cpp
@@ -14,84 +14,130 @@
 
 // unordered_set(const unordered_set& u);
 
-#include <unordered_set>
 #include <cassert>
 #include <cfloat>
 #include <cmath>
 #include <cstddef>
+#include <unordered_set>
 
-#include "test_macros.h"
 #include "../../../test_compare.h"
 #include "../../../test_hash.h"
-#include "test_allocator.h"
 #include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == test_allocator<int>(10));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class Alloc>
+void test_alloc() {
+  { // Simple check
+    using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    int arr[] = {1, 2, 3};
+    const Set orig(std::begin(arr), std::end(arr));
+    Set copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 5);
+    assert(copy.size() == 3);
+    assert(copy.count(1) == 1);
+    assert(copy.count(2) == 1);
+    assert(copy.count(3) == 1);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 5);
+    assert(orig.size() == 3);
+    assert(orig.count(1) == 1);
+    assert(orig.count(2) == 1);
+    assert(orig.count(3) == 1);
+  }
+  { // single element copy
+    using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    int arr[] = {1};
+    const Set orig(std::begin(arr), std::end(arr));
+    Set copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 2);
+    assert(copy.size() == 1);
+    assert(copy.count(1) == 1);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 2);
+    assert(orig.size() == 1);
+    assert(orig.count(1) == 1);
+  }
+  { // Copy empty map
+    using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    const Set orig;
+    Set copy = orig;
+    LIBCPP_ASSERT(copy.bucket_count() == 0);
+    assert(copy.size() == 0);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(copy.max_load_factor() == 1.f);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 0);
+    assert(orig.size() == 0);
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, other_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), other_allocator<int>(10));
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == other_allocator<int>(-2));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  { // Ensure that the hash function is copied
+    using Set = std::unordered_set<int, test_hash<int>, std::equal_to<int>, Alloc>;
+    const Set orig(0, test_hash<int>(23));
+    Set copy = orig;
+    assert(copy.hash_function() == test_hash<int>(23));
+
+    // Check that orig is still what is expected
+    assert(orig.hash_function() == test_hash<int>(23));
+  }
+  { // Ensure that the quality comparator is copied
+    using Set = std::unordered_set<int, std::hash<int>, test_equal_to<int>, Alloc>;
+    const Set orig(0, std::hash<int>(), test_equal_to<int>(56));
+    Set copy = orig;
+    assert(copy.key_eq() == test_equal_to<int>(56));
+
+    // Check that orig is still what is expected
+    assert(orig.key_eq() == test_equal_to<int>(56));
   }
-  {
-    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
-    C c = c0;
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == min_allocator<int>());
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+}
+
+void test() {
+  test_alloc<std::allocator<int> >();
+  test_alloc<min_allocator<int> >();
+
+  { // Ensure that the allocator is copied
+    using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, test_allocator<int> >;
+
+    int arr[] = {1, 2, 3};
+    const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), test_allocator<int>(10));
+    Set copy = orig;
+    assert(copy.size() == 3);
+    assert(copy.get_allocator() == test_allocator<int>(10));
+
+    // Check that orig is still what is expected
+    assert(orig.size() == 3);
+    assert(orig.get_allocator() == test_allocator<int>(10));
+    assert(orig.get_allocator().get_id() != test_alloc_base::moved_value);
   }
-#endif
+
+  { // Ensure that soccc is handled properly
+    using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, other_allocator<int> >;
+
+    int arr[] = {1, 2, 3};
+    const Set orig(std::begin(arr), std::end(arr), 0, std::hash<int>(), std::equal_to<int>(), other_allocator<int>(10));
+    Set copy = orig;
+    assert(copy.size() == 3);
+    assert(copy.get_allocator() == other_allocator<int>(-2));
+
+    // Check that orig is still what is expected
+    assert(orig.size() == 3);
+    assert(orig.get_allocator() == other_allocator<int>(10));
+  }
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy_alloc.pass.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy_alloc.pass.cpp
index 49a5f9406e1e1..388c5c621db15 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/copy_alloc.pass.cpp
@@ -14,63 +14,107 @@
 
 // unordered_set(const unordered_set& u, const allocator_type& a);
 
-#include <unordered_set>
 #include <cassert>
 #include <cfloat>
 #include <cmath>
 #include <cstddef>
+#include <unordered_set>
 
-#include "test_macros.h"
 #include "../../../test_compare.h"
 #include "../../../test_hash.h"
-#include "test_allocator.h"
 #include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, test_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), test_allocator<int>(10));
-    C c(c0, test_allocator<int>(5));
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == test_allocator<int>(5));
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+template <class Alloc>
+void test_alloc(const Alloc& new_alloc) {
+  { // Simple check
+    using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    int arr[] = {1, 2, 3};
+    const Set orig(std::begin(arr), std::end(arr));
+    Set copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 5);
+    assert(copy.size() == 3);
+    assert(copy.count(1) == 1);
+    assert(copy.count(2) == 1);
+    assert(copy.count(3) == 1);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 5);
+    assert(orig.size() == 3);
+    assert(orig.count(1) == 1);
+    assert(orig.count(2) == 1);
+    assert(orig.count(3) == 1);
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::unordered_set<int, test_hash<int>, test_equal_to<int>, min_allocator<int> > C;
-    typedef int P;
-    P a[] = {P(1), P(2), P(3), P(4), P(1), P(2)};
-    C c0(a, a + sizeof(a) / sizeof(a[0]), 7, test_hash<int>(8), test_equal_to<int>(9), min_allocator<int>());
-    C c(c0, min_allocator<int>());
-    LIBCPP_ASSERT(c.bucket_count() == 7);
-    assert(c.size() == 4);
-    assert(c.count(1) == 1);
-    assert(c.count(2) == 1);
-    assert(c.count(3) == 1);
-    assert(c.count(4) == 1);
-    assert(c.hash_function() == test_hash<int>(8));
-    assert(c.key_eq() == test_equal_to<int>(9));
-    assert(c.get_allocator() == min_allocator<int>());
-    assert(!c.empty());
-    assert(static_cast<std::size_t>(std::distance(c.begin(), c.end())) == c.size());
-    assert(static_cast<std::size_t>(std::distance(c.cbegin(), c.cend())) == c.size());
-    assert(std::fabs(c.load_factor() - (float)c.size() / c.bucket_count()) < FLT_EPSILON);
-    assert(c.max_load_factor() == 1);
+  { // single element check
+    using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    int arr[] = {1};
+    const Set orig(std::begin(arr), std::end(arr));
+    Set copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 2);
+    assert(copy.size() == 1);
+    assert(copy.count(1) == 1);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(std::fabs(copy.load_factor() - static_cast<float>(copy.size()) / copy.bucket_count()) < FLT_EPSILON);
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 2);
+    assert(orig.size() == 1);
+    assert(orig.count(1) == 1);
   }
-#endif
+  { // Copy empty map
+    using Set = std::unordered_set<int, std::hash<int>, std::equal_to<int>, Alloc>;
+
+    const Set orig;
+    Set copy(orig, new_alloc);
+    LIBCPP_ASSERT(copy.bucket_count() == 0);
+    assert(copy.size() == 0);
+    assert(static_cast<std::size_t>(std::distance(copy.begin(), copy.end())) == copy.size());
+    assert(copy.max_load_factor() == 1.f);
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    LIBCPP_ASSERT(orig.bucket_count() == 0);
+    assert(orig.size() == 0);
+  }
+  { // Ensure that the hash function is copied
+    using Set = std::unordered_set<int, test_hash<int>, std::equal_to<int>, Alloc>;
+    const Set orig(0, test_hash<int>(23));
+    Set copy(orig, new_alloc);
+    assert(copy.hash_function() == test_hash<int>(23));
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    assert(orig.hash_function() == test_hash<int>(23));
+  }
+  { // Ensure that the quality comparator is copied
+    using Set = std::unordered_set<int, std::hash<int>, test_equal_to<int>, Alloc>;
+    const Set orig(0, std::hash<int>(), test_equal_to<int>(56));
+    Set copy(orig, new_alloc);
+    assert(copy.key_eq() == test_equal_to<int>(56));
+    assert(copy.get_allocator() == new_alloc);
+
+    // Check that orig is still what is expected
+    assert(orig.key_eq() == test_equal_to<int>(56));
+  }
+}
+
+void test() {
+  test_alloc(std::allocator<int>());
+  test_alloc(min_allocator<int>());
+  test_alloc(test_allocator<int>(25));
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/views/mdspan/aligned_accessor/access.pass.cpp b/libcxx/test/std/containers/views/mdspan/aligned_accessor/access.pass.cpp
index deca1226e47ca..91a910899abcb 100644
--- a/libcxx/test/std/containers/views/mdspan/aligned_accessor/access.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/aligned_accessor/access.pass.cpp
@@ -23,7 +23,7 @@
 #include "test_macros.h"
 
 // We are not using MinimalElementType.h because MinimalElementType is not
-// default consructible and uninitialized storage does not work in constexpr.
+// default constructible and uninitialized storage does not work in constexpr.
 
 // Same as MinimalElementType but with a defaulted default constructor
 struct MyMinimalElementType {
diff --git a/libcxx/test/std/containers/views/mdspan/aligned_accessor/offset.pass.cpp b/libcxx/test/std/containers/views/mdspan/aligned_accessor/offset.pass.cpp
index 66c30da787a8d..3ed8cd7520176 100644
--- a/libcxx/test/std/containers/views/mdspan/aligned_accessor/offset.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/aligned_accessor/offset.pass.cpp
@@ -23,7 +23,7 @@
 #include "test_macros.h"
 
 // We are not using MinimalElementType.h because MinimalElementType is not
-// default consructible and uninitialized storage does not work in constexpr.
+// default constructible and uninitialized storage does not work in constexpr.
 
 // Same as MinimalElementType but with a defaulted default constructor
 struct MyMinimalElementType {
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/initializer_list.pass.cpp
index bc76e23fea3c0..b413b71ec9458 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/initializer_list.pass.cpp
@@ -34,7 +34,7 @@ static_assert(!ConstElementType<std::span<int>>);
 static_assert(ConstElementType<std::span<const int, 94>>);
 static_assert(!ConstElementType<std::span<int, 94>>);
 
-// Constructor constraings
+// Constructor constraints
 
 template <typename I, typename T, std::size_t... N>
 concept HasInitializerListCtr = requires(I il) { std::span<T, N...>{il}; };
diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
index 3bebf38a2a7c3..56e35736d4f78 100644
--- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.compare.pass.cpp
@@ -132,7 +132,7 @@ void test_compare_basic() {
       AssertOrderReturn<std::strong_ordering, path>();
 #endif
 
-      // check comarison results
+      // check comparison results
       assert(testComparisons(p1, p2, /*isEqual*/ E == 0, /*isLess*/ E < 0));
 #if TEST_STD_VER > 17
       assert(testOrder(p1, p2, E <=> 0));
diff --git a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/cache_refresh_iter.pass.cpp b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/cache_refresh_iter.pass.cpp
index 3d45828968de9..ff7035096e702 100644
--- a/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/cache_refresh_iter.pass.cpp
+++ b/libcxx/test/std/input.output/filesystems/class.rec.dir.itr/cache_refresh_iter.pass.cpp
@@ -147,7 +147,7 @@ static void test_cached_values_in_iteration() {
     const std::set<fs::path> dir_contents(static_env.RecDirIterationList.begin(), static_env.RecDirIterationList.end());
     dir_entries = get_directory_entries_for(testDir, dir_contents);
   }
-  // Testing folder should be deleted after destoying static_test_env.
+  // Testing folder should be deleted after destroying static_test_env.
 
   for (const auto& dir_entry : dir_entries) {
     // During iteration Windows provides information only about symlink itself
diff --git a/libcxx/test/std/iterators/iterator.requirements/indirectcallable/projected/projected.compile.pass.cpp b/libcxx/test/std/iterators/iterator.requirements/indirectcallable/projected/projected.compile.pass.cpp
index 06d1171c370a5..7ab92660fb068 100644
--- a/libcxx/test/std/iterators/iterator.requirements/indirectcallable/projected/projected.compile.pass.cpp
+++ b/libcxx/test/std/iterators/iterator.requirements/indirectcallable/projected/projected.compile.pass.cpp
@@ -61,4 +61,4 @@ constexpr bool projectable = requires {
 
 static_assert(!projectable<int, void (*)(int)>); // int isn't indirectly_readable
 static_assert(!projectable<S, void (*)(int)>);   // S isn't weakly_incrementable
-static_assert(!projectable<int*, void(int)>);    // void(int) doesn't satisfy indirectly_regular_unary_invcable
+static_assert(!projectable<int*, void(int)>);    // void(int) doesn't satisfy indirectly_regular_unary_invocable
diff --git a/libcxx/test/std/iterators/stream.iterators/istream.iterator/istream.iterator.cons/copy.pass.cpp b/libcxx/test/std/iterators/stream.iterators/istream.iterator/istream.iterator.cons/copy.pass.cpp
index 050f4ac7cc6d5..7474a8d3dde96 100644
--- a/libcxx/test/std/iterators/stream.iterators/istream.iterator/istream.iterator.cons/copy.pass.cpp
+++ b/libcxx/test/std/iterators/stream.iterators/istream.iterator/istream.iterator.cons/copy.pass.cpp
@@ -27,9 +27,9 @@ TEST_CONSTEXPR_CXX14 bool test_constexpr() {
   return true;
 }
 
-struct thowing_copy_constructor {
-  thowing_copy_constructor() {}
-  thowing_copy_constructor(const thowing_copy_constructor&) TEST_NOEXCEPT_FALSE {}
+struct throwing_copy_constructor {
+  throwing_copy_constructor() {}
+  throwing_copy_constructor(const throwing_copy_constructor&) TEST_NOEXCEPT_FALSE {}
 };
 
 int main(int, char**)
@@ -43,11 +43,11 @@ int main(int, char**)
 #endif
     }
     {
-      std::istream_iterator<thowing_copy_constructor> io;
-      std::istream_iterator<thowing_copy_constructor> i = io;
-      assert(i == std::istream_iterator<thowing_copy_constructor>());
+      std::istream_iterator<throwing_copy_constructor> io;
+      std::istream_iterator<throwing_copy_constructor> i = io;
+      assert(i == std::istream_iterator<throwing_copy_constructor>());
 #if TEST_STD_VER >= 11
-      static_assert(!std::is_nothrow_copy_constructible<std::istream_iterator<thowing_copy_constructor>>::value, "");
+      static_assert(!std::is_nothrow_copy_constructible<std::istream_iterator<throwing_copy_constructor>>::value, "");
 #endif
     }
     {
diff --git a/libcxx/test/std/language.support/support.c.headers/support.c.headers.other/math.lerp.pass.cpp b/libcxx/test/std/language.support/support.c.headers/support.c.headers.other/math.lerp.pass.cpp
new file mode 100644
index 0000000000000..f1dd021d28b87
--- /dev/null
+++ b/libcxx/test/std/language.support/support.c.headers/support.c.headers.other/math.lerp.pass.cpp
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <math.h>
+
+// [support.c.headers.other]/1
+//   ... except for the functions described in [sf.cmath], the
+//   std::lerp function overloads ([c.math.lerp]) ...
+
+#include <cassert>
+#include <math.h>
+
+template <class = int>
+int lerp(float, float, float) {
+  return 32;
+}
+
+template <class = int>
+int lerp(double, double, double) {
+  return 32;
+}
+
+template <class = int>
+int lerp(long double, long double, long double) {
+  return 32;
+}
+
+int main(int, char**) {
+  assert(lerp(0.f, 0.f, 0.f) == 32);
+  assert(lerp(0., 0., 0.) == 32);
+  assert(lerp(0.l, 0.l, 0.l) == 32);
+
+  return 0;
+}
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp
index ccdb1a8c11a0b..aca6290f5a4bf 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp
@@ -146,17 +146,11 @@
 #    error "__cpp_lib_optional should have the value 202110L in c++26"
 #  endif
 
-#  if !defined(_LIBCPP_VERSION)
-#    ifndef __cpp_lib_optional_range_support
-#      error "__cpp_lib_optional_range_support should be defined in c++26"
-#    endif
-#    if __cpp_lib_optional_range_support != 202406L
-#      error "__cpp_lib_optional_range_support should have the value 202406L in c++26"
-#    endif
-#  else
-#    ifdef __cpp_lib_optional_range_support
-#      error "__cpp_lib_optional_range_support should not be defined because it is unimplemented in libc++!"
-#    endif
+#  ifndef __cpp_lib_optional_range_support
+#    error "__cpp_lib_optional_range_support should be defined in c++26"
+#  endif
+#  if __cpp_lib_optional_range_support != 202406L
+#    error "__cpp_lib_optional_range_support should have the value 202406L in c++26"
 #  endif
 
 #endif // TEST_STD_VER > 23
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index 7bd8e8979e6f3..cde2f258b7732 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -7437,17 +7437,11 @@
 #    error "__cpp_lib_optional should have the value 202110L in c++26"
 #  endif
 
-#  if !defined(_LIBCPP_VERSION)
-#    ifndef __cpp_lib_optional_range_support
-#      error "__cpp_lib_optional_range_support should be defined in c++26"
-#    endif
-#    if __cpp_lib_optional_range_support != 202406L
-#      error "__cpp_lib_optional_range_support should have the value 202406L in c++26"
-#    endif
-#  else
-#    ifdef __cpp_lib_optional_range_support
-#      error "__cpp_lib_optional_range_support should not be defined because it is unimplemented in libc++!"
-#    endif
+#  ifndef __cpp_lib_optional_range_support
+#    error "__cpp_lib_optional_range_support should be defined in c++26"
+#  endif
+#  if __cpp_lib_optional_range_support != 202406L
+#    error "__cpp_lib_optional_range_support should have the value 202406L in c++26"
 #  endif
 
 #  ifndef __cpp_lib_out_ptr
diff --git a/libcxx/test/std/library/description/conventions/customization.point.object/cpo.compile.pass.cpp b/libcxx/test/std/library/description/conventions/customization.point.object/cpo.compile.pass.cpp
index 678483b9b2f2f..49497875dcf95 100644
--- a/libcxx/test/std/library/description/conventions/customization.point.object/cpo.compile.pass.cpp
+++ b/libcxx/test/std/library/description/conventions/customization.point.object/cpo.compile.pass.cpp
@@ -81,6 +81,10 @@ static_assert(test(std::ranges::rend, a));
 static_assert(test(std::ranges::size, a));
 static_assert(test(std::ranges::ssize, a));
 
+#if TEST_STD_VER >= 26
+// static_assert(test(std::views::reserve_hint, a));
+#endif
+
 // [range.factories]
 // views::empty<T> is not a CPO
 static_assert(test(std::views::iota, 1));
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.hex.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.hex.pass.cpp
index 06a974ad1bee0..8db40b9e0dcbc 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.hex.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.nm.put/facet.num.put.members/put_long_double.hex.pass.cpp
@@ -1858,7 +1858,7 @@ void test2() {
   // 64 mantissa bits are grouped by nibble as they are stored in the long
   // double representation (nibble aligned at the end of the least significant
   // bits). This makes 1.0L to be formatted as "0x8p-3" (where the leading
-  // bit of the mantissa is the higest bit in the 0x8 nibble), and makes
+  // bit of the mantissa is the highest bit in the 0x8 nibble), and makes
   // __LDBL_MAX__ be formatted as "0xf.fffffffffffffffp+16380".
   //
   // FreeBSD normalizes/aligns the leading bit of the mantissa as a separate
diff --git a/libcxx/test/std/numerics/c.math/hermite.pass.cpp b/libcxx/test/std/numerics/c.math/hermite.pass.cpp
index f1a37c5ddc15d..2905950942581 100644
--- a/libcxx/test/std/numerics/c.math/hermite.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/hermite.pass.cpp
@@ -319,7 +319,7 @@ void test() {
       static_assert(sizeof(double) == 8);
       for (unsigned n = 0; n < get_maximal_order<Real>(); ++n) {
         // Q: Why n=111 and x=300?
-        // A: Both are chosen s.t. the first overlow occurs for some `n<get_maximal_order<Real>()`.
+        // A: Both are chosen s.t. the first overflow occurs for some `n<get_maximal_order<Real>()`.
         if (n < 111) {
           assert(std::isfinite(std::hermite(n, +300.0)));
           assert(std::isfinite(std::hermite(n, -300.0)));
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/and_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/and_valarray.pass.cpp
index 15a2b03bf84bf..594b99bc09583 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/and_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/and_valarray.pass.cpp
@@ -21,7 +21,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // Validate the function works for valarray, the proxies, and __val_expr.
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/divide_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/divide_valarray.pass.cpp
index e4fe675a9ec38..8c1fb2afd96fa 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/divide_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/divide_valarray.pass.cpp
@@ -21,7 +21,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // Validate the function works for valarray, the proxies, and __val_expr.
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/minus_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/minus_valarray.pass.cpp
index 54aa7fd4c269f..6e13d2618b6bb 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/minus_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/minus_valarray.pass.cpp
@@ -21,7 +21,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // Validate the function works for valarray, the proxies, and __val_expr.
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/modulo_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/modulo_valarray.pass.cpp
index e52097ae68e21..1246ab35547bf 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/modulo_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/modulo_valarray.pass.cpp
@@ -21,7 +21,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // Validate the function works for valarray, the proxies, and __val_expr.
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/or_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/or_valarray.pass.cpp
index 429953e272b61..9387fccbf1538 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/or_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/or_valarray.pass.cpp
@@ -21,7 +21,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // Validate the function works for valarray, the proxies, and __val_expr.
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/plus_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/plus_valarray.pass.cpp
index 6aacf4f68b98f..d1737a016e53c 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/plus_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/plus_valarray.pass.cpp
@@ -21,7 +21,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // Validate the function works for valarray, the proxies, and __val_expr.
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/shift_left_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/shift_left_valarray.pass.cpp
index 3ff264308f882..cb1af636cde5b 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/shift_left_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/shift_left_valarray.pass.cpp
@@ -19,7 +19,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // Validate the function works for valarray, the proxies, and __val_expr.
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/shift_right_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/shift_right_valarray.pass.cpp
index 140ec4dad5f42..11bbf01ea6876 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/shift_right_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/shift_right_valarray.pass.cpp
@@ -19,7 +19,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // valarray& operator>>=(const valarray& v);
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/times_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/times_valarray.pass.cpp
index 58361fa697b6f..10333ee910299 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/times_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/times_valarray.pass.cpp
@@ -19,7 +19,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // valarray& operator*=(const valarray& v);
diff --git a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/xor_valarray.pass.cpp b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/xor_valarray.pass.cpp
index 0b3ffe4f93244..484189e806beb 100644
--- a/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/xor_valarray.pass.cpp
+++ b/libcxx/test/std/numerics/numarray/template.valarray/valarray.cassign/xor_valarray.pass.cpp
@@ -19,7 +19,7 @@
 //
 // The const overloads of
 //   valarray::operator[](...) const
-// return propxy objects. These proxies are implicitly convertible to
+// return proxy objects. These proxies are implicitly convertible to
 // std::valarray.
 //
 // valarray& operator^=(const valarray& v);
diff --git a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/params.verify.cpp b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/params.verify.cpp
index bc9b5636e415d..0e0316162dc12 100644
--- a/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/params.verify.cpp
+++ b/libcxx/test/std/numerics/rand/rand.eng/rand.eng.lcong/params.verify.cpp
@@ -24,7 +24,7 @@ int main(int, char**)
     // expected-error-re@*:* {{static assertion failed due to requirement '1ULL == 0 || 1ULL < 1ULL'{{.*}}linear_congruential_engine invalid parameters}}
     std::linear_congruential_engine<T, 0, 1, 1> e3;
     std::linear_congruential_engine<T, 1, 0, 1> e4;
-    // expected-error-re@*:* {{static assertion failed due to requirement 'is_unsigned<int>::value'{{.*}}_UIntType must be unsigned type}}
+    // expected-error-re@*:* {{static assertion failed due to requirement {{.+}}_UIntType must be unsigned type}}
     std::linear_congruential_engine<int, 0, 0, 0> e5;
 
     return 0;
diff --git a/libcxx/test/std/ranges/range.access/include.iterator.pass.cpp b/libcxx/test/std/ranges/range.access/include.iterator.pass.cpp
new file mode 100644
index 0000000000000..bb2cda0e4d906
--- /dev/null
+++ b/libcxx/test/std/ranges/range.access/include.iterator.pass.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++20
+
+// [range.access.general]/1:
+// In addition to being available via inclusion of the <ranges> header, the customization point objects in
+// [range.access] are available when the header <iterator> is included.
+
+#include <iterator>
+#include <type_traits>
+
+#include "test_macros.h"
+
+template <class CPO, class... Args>
+constexpr void test(CPO& o, Args&&... args) {
+  static_assert(std::is_const_v<CPO>);
+  static_assert(std::is_class_v<CPO>);
+  static_assert(std::is_trivially_copyable_v<CPO>);
+  static_assert(std::is_trivially_default_constructible_v<CPO>);
+
+  auto p  = o;
+  using T = decltype(p);
+  (void)o(args...); // to make sure the CPO can actually be used
+
+  // The type of a customization point object, ignoring cv-qualifiers, shall model semiregular.
+  static_assert(std::semiregular<T>);
+
+  // The type T of a customization point object, ignoring cv-qualifiers, shall model...
+  static_assert(std::invocable<T&, Args...>);
+  static_assert(std::invocable<const T&, Args...>);
+  static_assert(std::invocable<T, Args...>);
+  static_assert(std::invocable<const T, Args...>);
+}
+
+int a[10];
+
+constexpr bool test() {
+  test(std::ranges::begin, a);
+  test(std::ranges::end, a);
+  test(std::ranges::cbegin, a);
+  test(std::ranges::cdata, a);
+  test(std::ranges::cend, a);
+  test(std::ranges::crbegin, a);
+  test(std::ranges::crend, a);
+  test(std::ranges::data, a);
+  test(std::ranges::empty, a);
+  test(std::ranges::rbegin, a);
+  test(std::ranges::rend, a);
+  test(std::ranges::size, a);
+  test(std::ranges::ssize, a);
+
+#if TEST_STD_VER >= 26
+  // test(std::views::reserve_hint, a);
+#endif
+
+  return true;
+}
+
+int main() {
+  test();
+  static_assert(test());
+}
diff --git a/libcxx/test/std/ranges/range.adaptors/range.zip.transform/general.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.zip.transform/general.pass.cpp
index 3c35de27deb69..4b057946abba2 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.zip.transform/general.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.zip.transform/general.pass.cpp
@@ -8,7 +8,7 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
-// Some basic examples of how zip_tranform_view might be used in the wild. This is a general
+// Some basic examples of how zip_transform_view might be used in the wild. This is a general
 // collection of sample algorithms and functions that try to mock general usage of
 // this view.
 
diff --git a/libcxx/test/std/ranges/range.factories/range.iota.view/views_iota.pass.cpp b/libcxx/test/std/ranges/range.factories/range.iota.view/views_iota.pass.cpp
index a6b268fd6a8d8..4f06b260d0d23 100644
--- a/libcxx/test/std/ranges/range.factories/range.iota.view/views_iota.pass.cpp
+++ b/libcxx/test/std/ranges/range.factories/range.iota.view/views_iota.pass.cpp
@@ -46,6 +46,9 @@ constexpr void testType(U u) {
 
 struct X {};
 
+template <typename IntT>
+concept CanDoubleWrap = requires(IntT i) { std::views::iota(std::views::iota(i)); };
+
 constexpr bool test() {
   testType<SomeInt>(SomeInt(10));
   testType<SomeInt>(IntComparableWith(SomeInt(10)));
@@ -68,6 +71,15 @@ constexpr bool test() {
   {
     static_assert(std::same_as<decltype(std::views::iota), decltype(std::ranges::views::iota)>);
   }
+  { // LWG4096: views::iota(views::iota(0)) should be rejected
+    static_assert(!CanDoubleWrap<int>);
+    static_assert(!CanDoubleWrap<SomeInt>);
+
+    static_assert(!std::is_invocable_v<decltype(std::views::iota), decltype(std::views::iota(0))>);
+    static_assert(!std::is_invocable_v<decltype(std::views::iota), decltype(std::views::iota(82))>);
+    static_assert(!std::is_invocable_v<decltype(std::views::iota), decltype(std::views::iota(SomeInt(0)))>);
+    static_assert(!std::is_invocable_v<decltype(std::views::iota), decltype(std::views::iota(SomeInt(94)))>);
+  }
 
   return true;
 }
diff --git a/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp b/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
index 4e087fd4226d1..70009589e985b 100644
--- a/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
+++ b/libcxx/test/std/thread/futures/futures.async/thread_create_failure.pass.cpp
@@ -8,7 +8,7 @@
 //
 // UNSUPPORTED: no-threads, no-exceptions
 
-// ASan seems to try to create threadsm which obviouly doesn't work in this test.
+// ASan seems to try to create threadsm which obviously doesn't work in this test.
 // UNSUPPORTED: asan, hwasan
 
 // UNSUPPORTED: c++03
diff --git a/libcxx/test/std/thread/futures/futures.task/futures.task.members/ctor1.verify.cpp b/libcxx/test/std/thread/futures/futures.task/futures.task.members/ctor1.verify.cpp
index 3c61c76ea9802..efa1316b5230f 100644
--- a/libcxx/test/std/thread/futures/futures.task/futures.task.members/ctor1.verify.cpp
+++ b/libcxx/test/std/thread/futures/futures.task/futures.task.members/ctor1.verify.cpp
@@ -31,7 +31,7 @@ int main(int, char**)
     auto const& c_init = init;
     PT p1{init}; // expected-error {{no matching constructor}}
     PT p2{c_init}; // expected-error {{no matching constructor}}
-    PT p3{std::move(init)}; // expected-error {{no matching constructor for initialization of 'PT' (aka 'packaged_task<A (int, char)>')}}
+    PT p3{std::move(init)}; // expected-error-re {{no matching constructor for initialization of 'PT' (aka {{.+}})}}
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp b/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp
index b714cc58cbd3b..8527af1344e27 100644
--- a/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp
+++ b/libcxx/test/std/thread/thread.jthread/assign.move.pass.cpp
@@ -112,5 +112,14 @@ int main(int, char**) {
     assert(j1.get_id() == j2Id);
   }
 
+  // LWG3788: self-assignment
+  {
+    std::jthread j = support::make_test_jthread([] {});
+    auto oldId     = j.get_id();
+    j              = std::move(j);
+
+    assert(j.get_id() == oldId);
+  }
+
   return 0;
 }
diff --git a/libcxx/test/libcxx/thread/thread.threads/thread.thread.this/sleep_for.pass.cpp b/libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_for.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/thread/thread.threads/thread.thread.this/sleep_for.pass.cpp
rename to libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_for.pass.cpp
diff --git a/libcxx/test/libcxx/thread/thread.threads/thread.thread.this/sleep_for.signals.pass.cpp b/libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_for.signals.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/thread/thread.threads/thread.thread.this/sleep_for.signals.pass.cpp
rename to libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_for.signals.pass.cpp
diff --git a/libcxx/test/std/time/time.hms/time.hms.nonmembers/ostream.pass.cpp b/libcxx/test/std/time/time.hms/time.hms.nonmembers/ostream.pass.cpp
index 6ee419fc07c0e..e281098de55d0 100644
--- a/libcxx/test/std/time/time.hms/time.hms.nonmembers/ostream.pass.cpp
+++ b/libcxx/test/std/time/time.hms/time.hms.nonmembers/ostream.pass.cpp
@@ -63,7 +63,7 @@ template <class CharT>
 static void test() {
   // Note std::atto can't be tested since the ratio conversion from std::atto
   // std::chrono::seconds to std::chrono::hours overflows when intmax_t is a
-  // 64-bit type. This is a limitiation in the constructor of
+  // 64-bit type. This is a limitation in the constructor of
   // std::chrono::hh_mm_ss.
 
   // C locale - integral power of 10 ratios
diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.local_time.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.local_time.pass.cpp
index 76dd8bc746fcd..1d0234468f419 100644
--- a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.local_time.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.local_time.pass.cpp
@@ -1017,8 +1017,8 @@ static void test_europe_dublin() {
 
 static void test_america_st_johns() {
   // A more typical entry,
-  // Uses letters both when DST is ative and not and has multiple
-  // letters. Uses negetive offsets.
+  // Uses letters both when DST is active and not and has multiple
+  // letters. Uses negative offsets.
   // Switches several times between their own and Canadian rules
   // Switches the stdoff from -3:30:52 to -3:30 while observing the same rule
 
diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
index 1a1705d5ae59a..1924af4bd6718 100644
--- a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
@@ -34,7 +34,7 @@
 // The data in the tests can be validated by using the zdump tool. For
 // example
 //   zdump -v Asia/Hong_Kong
-// show all transistions in the Hong Kong time zone. Or
+// show all transitions in the Hong Kong time zone. Or
 //   zdump -c1970,1980 -v Asia/Hong_Kong
 // shows all transitions in Hong Kong between 1970 and 1980.
 
@@ -177,7 +177,7 @@ static void test_antarctica_syowa() {
   // This change uses an ON field with a day number
   //
   // There don't seem to be rule-less zones that use last day or a
-  // contrained day
+  // constrained day
 
   // Z Antarctica/Syowa 0 - -00 1957 Ja 29
   // 3 - +03
@@ -605,8 +605,8 @@ static void test_europe_berlin() {
 
 static void test_america_st_johns() {
   // A more typical entry,
-  // Uses letters both when DST is ative and not and has multiple
-  // letters. Uses negetive offsets.
+  // Uses letters both when DST is active and not and has multiple
+  // letters. Uses negative offsets.
   // Switches several times between their own and Canadian rules
   // Switches the stdoff from -3:30:52 to -3:30 while observing the same rule
 
@@ -1036,7 +1036,7 @@ static void test_africa_freetown() {
   using namespace std::literals::chrono_literals;
   const std::chrono::time_zone* tz = std::chrono::locate_zone("Africa/Freetown");
 
-  // When a continuation has a named rule, the tranisition time determined by
+  // When a continuation has a named rule, the transition time determined by
   // the active rule can be wrong. The next continuation may set the clock to an
   // earlier time. This is tested for San Luis. This tests the rule is not used
   // when the rule is not a named rule.
diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys_choose.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys_choose.pass.cpp
index 1147c9fadf9ae..b2dbdb1bb66bb 100644
--- a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys_choose.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/to_sys_choose.pass.cpp
@@ -141,7 +141,7 @@ static void test_ambiguous() {
   assert(tz->to_sys(time + 0ms, std::chrono::choose::earliest) == earlier);
   assert(tz->to_sys(time + 0s, std::chrono::choose::latest) == later);
 
-  // Test times in the ambigious hour
+  // Test times in the ambiguous hour
   assert(tz->to_sys(time + 1s, std::chrono::choose::earliest) == earlier + 1s);
   assert(tz->to_sys(time + 1min, std::chrono::choose::latest) == later + 1min);
   assert(tz->to_sys(time + 30min, std::chrono::choose::earliest) == earlier + 30min);
diff --git a/libcxx/test/std/time/time.zone/time.zone.zonedtime/types.compile.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.zonedtime/types.compile.pass.cpp
index 497da859265c2..96f17b38f862b 100644
--- a/libcxx/test/std/time/time.zone/time.zone.zonedtime/types.compile.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.zonedtime/types.compile.pass.cpp
@@ -58,7 +58,7 @@ static_assert(std::is_copy_assignable_v<std::chrono::zoned_time<std::chrono::day
 static_assert(std::is_move_assignable_v<std::chrono::zoned_time<std::chrono::days, int>>);
 
 // Test with a move only type, since the copy constructor is defined, no move
-// constuctor is generated.
+// constructor is generated.
 static_assert(!std::is_copy_constructible_v<std::chrono::zoned_time< std::chrono::days, std::unique_ptr<int>>>);
 static_assert(!std::is_move_constructible_v<std::chrono::zoned_time< std::chrono::days, std::unique_ptr<int>>>);
 static_assert(!std::is_copy_assignable_v<std::chrono::zoned_time< std::chrono::days, std::unique_ptr<int>>>);
diff --git a/libcxx/test/std/utilities/charconv/charconv.from.chars/floating_point.pass.cpp b/libcxx/test/std/utilities/charconv/charconv.from.chars/floating_point.pass.cpp
index 6faf0499c4c9b..a02c7ac466f04 100644
--- a/libcxx/test/std/utilities/charconv/charconv.from.chars/floating_point.pass.cpp
+++ b/libcxx/test/std/utilities/charconv/charconv.from.chars/floating_point.pass.cpp
@@ -396,7 +396,7 @@ struct test_fixed {
       assert(x == F(-0.5));
     }
 
-    { // double deciamal point
+    { // double decimal point
       const char* s = "1.25.78";
 
       // This number is halfway between two float values.
@@ -405,7 +405,7 @@ struct test_fixed {
       assert(r.ptr == s + 4);
       assert(x == F(1.25));
     }
-    { // exponenent no sign
+    { // exponent no sign
       const char* s = "1.5e10";
       r             = std::from_chars(s, s + std::strlen(s), x, std::chars_format::fixed);
 
@@ -413,7 +413,7 @@ struct test_fixed {
       assert(r.ptr == s + 3);
       assert(x == F(1.5));
     }
-    { // exponenent capitalized no sign
+    { // exponent capitalized no sign
       const char* s = "1.5E10";
       r             = std::from_chars(s, s + std::strlen(s), x, std::chars_format::fixed);
 
@@ -421,7 +421,7 @@ struct test_fixed {
       assert(r.ptr == s + 3);
       assert(x == F(1.5));
     }
-    { // exponenent + sign
+    { // exponent + sign
       const char* s = "1.5e+10";
       r             = std::from_chars(s, s + std::strlen(s), x, std::chars_format::fixed);
 
@@ -429,7 +429,7 @@ struct test_fixed {
       assert(r.ptr == s + 3);
       assert(x == F(1.5));
     }
-    { // exponenent - sign
+    { // exponent - sign
       const char* s = "1.5e-10";
       r             = std::from_chars(s, s + std::strlen(s), x, std::chars_format::fixed);
 
@@ -739,7 +739,7 @@ struct test_scientific {
       assert(x == F(1.0));
     }
 
-    { // double deciamal point
+    { // double decimal point
       const char* s = "1.25e0.78";
 
       // This number is halfway between two float values.
@@ -749,7 +749,7 @@ struct test_scientific {
       assert(x == F(1.25));
     }
 
-    { // exponenent no sign
+    { // exponent no sign
       const char* s = "1.5e10";
 
       r = std::from_chars(s, s + std::strlen(s), x, std::chars_format::scientific);
@@ -757,7 +757,7 @@ struct test_scientific {
       assert(r.ptr == s + 6);
       assert(x == F(1.5e10));
     }
-    { // exponenent capitalized no sign
+    { // exponent capitalized no sign
       const char* s = "1.5E10";
 
       r = std::from_chars(s, s + std::strlen(s), x, std::chars_format::scientific);
@@ -765,7 +765,7 @@ struct test_scientific {
       assert(r.ptr == s + 6);
       assert(x == F(1.5e10));
     }
-    { // exponenent + sign
+    { // exponent + sign
       const char* s = "1.5e+10";
 
       r = std::from_chars(s, s + std::strlen(s), x, std::chars_format::scientific);
@@ -773,7 +773,7 @@ struct test_scientific {
       assert(r.ptr == s + 7);
       assert(x == F(1.5e10));
     }
-    { // exponenent - sign
+    { // exponent - sign
       const char* s = "1.5e-10";
 
       r = std::from_chars(s, s + std::strlen(s), x, std::chars_format::scientific);
@@ -958,7 +958,7 @@ struct test_general {
       assert(r.ptr == s + 3);
       assert(x == F(-0.5));
     }
-    { // double deciamal point
+    { // double decimal point
       const char* s = "1.25.78";
 
       // This number is halfway between two float values.
@@ -967,7 +967,7 @@ struct test_general {
       assert(r.ptr == s + 4);
       assert(x == F(1.25));
     }
-    { // exponenent no sign
+    { // exponent no sign
       const char* s = "1.5e10";
 
       r = std::from_chars(s, s + std::strlen(s), x);
@@ -975,7 +975,7 @@ struct test_general {
       assert(r.ptr == s + 6);
       assert(x == F(1.5e10));
     }
-    { // exponenent capitalized no sign
+    { // exponent capitalized no sign
       const char* s = "1.5E10";
 
       r = std::from_chars(s, s + std::strlen(s), x);
@@ -983,7 +983,7 @@ struct test_general {
       assert(r.ptr == s + 6);
       assert(x == F(1.5e10));
     }
-    { // exponenent + sign
+    { // exponent + sign
       const char* s = "1.5e+10";
 
       r = std::from_chars(s, s + std::strlen(s), x);
@@ -991,7 +991,7 @@ struct test_general {
       assert(r.ptr == s + 7);
       assert(x == F(1.5e10));
     }
-    { // exponenent - sign
+    { // exponent - sign
       const char* s = "1.5e-10";
 
       r = std::from_chars(s, s + std::strlen(s), x);
@@ -1278,7 +1278,7 @@ struct test_hex {
       assert(r.ptr == s + 3);
       assert(x == F(-0x0.5p0));
     }
-    { // double deciamal point
+    { // double decimal point
       const char* s = "1.25.78";
 
       // This number is halfway between two float values.
@@ -1287,7 +1287,7 @@ struct test_hex {
       assert(r.ptr == s + 4);
       assert(x == F(0x1.25p0));
     }
-    { // exponenent no sign
+    { // exponent no sign
       const char* s = "1.5p10";
 
       r = std::from_chars(s, s + std::strlen(s), x, std::chars_format::hex);
@@ -1295,7 +1295,7 @@ struct test_hex {
       assert(r.ptr == s + 6);
       assert(x == F(0x1.5p10));
     }
-    { // exponenent capitalized no sign
+    { // exponent capitalized no sign
       const char* s = "1.5P10";
 
       r = std::from_chars(s, s + std::strlen(s), x, std::chars_format::hex);
@@ -1303,7 +1303,7 @@ struct test_hex {
       assert(r.ptr == s + 6);
       assert(x == F(0x1.5p10));
     }
-    { // exponenent + sign
+    { // exponent + sign
       const char* s = "1.5p+10";
 
       r = std::from_chars(s, s + std::strlen(s), x, std::chars_format::hex);
@@ -1311,7 +1311,7 @@ struct test_hex {
       assert(r.ptr == s + 7);
       assert(x == F(0x1.5p10));
     }
-    { // exponenent - sign
+    { // exponent - sign
       const char* s = "1.5p-10";
 
       r = std::from_chars(s, s + std::strlen(s), x, std::chars_format::hex);
diff --git a/libcxx/test/std/utilities/expected/expected.expected/ctor/ctor.u.pass.cpp b/libcxx/test/std/utilities/expected/expected.expected/ctor/ctor.u.pass.cpp
index fa9f5a115ccf5..13c0da27bc533 100644
--- a/libcxx/test/std/utilities/expected/expected.expected/ctor/ctor.u.pass.cpp
+++ b/libcxx/test/std/utilities/expected/expected.expected/ctor/ctor.u.pass.cpp
@@ -14,6 +14,7 @@
 // Constraints:
 // - is_same_v<remove_cvref_t<U>, in_place_t> is false; and
 // - is_same_v<expected, remove_cvref_t<U>> is false; and
+// - is_same_v<remove_cvref_t<U>, unexpect_t> is false; and
 // - remove_cvref_t<U> is not a specialization of unexpected; and
 // - is_constructible_v<T, U> is true.
 //
@@ -46,6 +47,16 @@ static_assert(!std::is_constructible_v<std::expected<FromJustInplace, int>, std:
 // Note that result is true because it is covered by the constructors that take expected
 static_assert(std::is_constructible_v<std::expected<int, int>, std::expected<int, int>&>);
 
+struct T {
+  explicit T(auto) {}
+};
+struct E {
+  E(int) {}
+};
+
+// is_same_v<remove_cvref_t<U>, unexpect_t> is false;
+static_assert(!std::is_constructible_v<std::expected<T, E>, std::unexpect_t>);
+
 // remove_cvref_t<U> is a specialization of unexpected
 // Note that result is true because it is covered by the constructors that take unexpected
 static_assert(std::is_constructible_v<std::expected<int, int>, std::unexpected<int>&>);
diff --git a/libcxx/test/libcxx/vendor/ibm/bad_function_call.pass.cpp b/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.badcall/visibility_inlines_hidden.cpp
similarity index 97%
rename from libcxx/test/libcxx/vendor/ibm/bad_function_call.pass.cpp
rename to libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.badcall/visibility_inlines_hidden.cpp
index 3714e4037a2dc..fe5e49f715787 100644
--- a/libcxx/test/libcxx/vendor/ibm/bad_function_call.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/func.wrap/func.wrap.badcall/visibility_inlines_hidden.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-// REQUIRES: target={{.+}}-aix{{.*}}
 // ADDITIONAL_COMPILE_FLAGS: -fvisibility-inlines-hidden
 
 // When there is a weak hidden symbol in user code and a strong definition
diff --git a/libcxx/test/libcxx/utilities/function.objects/refwrap/binary.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/binary.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/utilities/function.objects/refwrap/binary.pass.cpp
rename to libcxx/test/std/utilities/function.objects/refwrap/binary.pass.cpp
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/compare.three_way.refwrap.const_ref.pass.cpp
similarity index 100%
rename from libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.const_ref.pass.cpp
rename to libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/compare.three_way.refwrap.const_ref.pass.cpp
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/compare.three_way.refwrap.refwrap.pass.cpp
similarity index 100%
rename from libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap.pass.cpp
rename to libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/compare.three_way.refwrap.refwrap.pass.cpp
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/compare.three_way.refwrap.refwrap_const.pass.cpp
similarity index 100%
rename from libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/compare.three_way.refwrap.refwrap_const.pass.cpp
rename to libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/compare.three_way.refwrap.refwrap_const.pass.cpp
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/equal.refwrap.const_ref.pass.cpp
similarity index 100%
rename from libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.const_ref.pass.cpp
rename to libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/equal.refwrap.const_ref.pass.cpp
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/equal.refwrap.refwrap.pass.cpp
similarity index 100%
rename from libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap.pass.cpp
rename to libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/equal.refwrap.refwrap.pass.cpp
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/equal.refwrap.refwrap_const.pass.cpp
similarity index 100%
rename from libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparissons/equal.refwrap.refwrap_const.pass.cpp
rename to libcxx/test/std/utilities/function.objects/refwrap/refwrap.comparisons/equal.refwrap.refwrap_const.pass.cpp
diff --git a/libcxx/test/libcxx/utilities/function.objects/refwrap/unary.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/unary.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/utilities/function.objects/refwrap/unary.pass.cpp
rename to libcxx/test/std/utilities/function.objects/refwrap/unary.pass.cpp
diff --git a/libcxx/test/libcxx/utilities/memory/util.smartptr/race_condition.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/race_condition.pass.cpp
similarity index 100%
rename from libcxx/test/libcxx/utilities/memory/util.smartptr/race_condition.pass.cpp
rename to libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/race_condition.pass.cpp
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/shared_ptr_array.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/shared_ptr_array.pass.cpp
index 9bc695cd7fb76..f71f688afc52e 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/shared_ptr_array.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/shared_ptr_array.pass.cpp
@@ -20,11 +20,11 @@
 #include <memory>
 
 int main(int, char**) {
-  std::allocate_shared<int[]>(std::allocator<int>{}, 10);
-  std::make_shared<int[]>(10);
+  (void)std::allocate_shared<int[]>(std::allocator<int>{}, 10);
+  (void)std::make_shared<int[]>(10);
 
-  std::allocate_shared<int[10]>(std::allocator<int>{});
-  std::make_shared<int[10]>();
+  (void)std::allocate_shared<int[10]>(std::allocator<int>{});
+  (void)std::make_shared<int[10]>();
 
   return 0;
 }
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
index b7fd0937eb216..a68f3f40e3647 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_implicit_lifetime.pass.cpp
@@ -35,10 +35,10 @@ enum class UnsignedEnumClass : unsigned int {};
 struct EmptyStruct {};
 struct IncompleteStruct;
 
-struct NoEligibleTrivialContructor {
-  NoEligibleTrivialContructor() {};
-  NoEligibleTrivialContructor(const NoEligibleTrivialContructor&) {}
-  NoEligibleTrivialContructor(NoEligibleTrivialContructor&&) {}
+struct NoEligibleTrivialConstructor {
+  NoEligibleTrivialConstructor() {};
+  NoEligibleTrivialConstructor(const NoEligibleTrivialConstructor&) {}
+  NoEligibleTrivialConstructor(NoEligibleTrivialConstructor&&) {}
 };
 
 struct OnlyDefaultConstructorIsTrivial {
@@ -47,22 +47,22 @@ struct OnlyDefaultConstructorIsTrivial {
   OnlyDefaultConstructorIsTrivial(OnlyDefaultConstructorIsTrivial&&) {}
 };
 
-struct AllContstructorsAreTrivial {
-  AllContstructorsAreTrivial()                                  = default;
-  AllContstructorsAreTrivial(const AllContstructorsAreTrivial&) = default;
-  AllContstructorsAreTrivial(AllContstructorsAreTrivial&&)      = default;
+struct AllConstructorsAreTrivial {
+  AllConstructorsAreTrivial()                                 = default;
+  AllConstructorsAreTrivial(const AllConstructorsAreTrivial&) = default;
+  AllConstructorsAreTrivial(AllConstructorsAreTrivial&&)      = default;
 };
 
-struct InheritedNoEligibleTrivialConstructor : NoEligibleTrivialContructor {
-  using NoEligibleTrivialContructor::NoEligibleTrivialContructor;
+struct InheritedNoEligibleTrivialConstructor : NoEligibleTrivialConstructor {
+  using NoEligibleTrivialConstructor::NoEligibleTrivialConstructor;
 };
 
 struct InheritedOnlyDefaultConstructorIsTrivial : OnlyDefaultConstructorIsTrivial {
   using OnlyDefaultConstructorIsTrivial::OnlyDefaultConstructorIsTrivial;
 };
 
-struct InheritedAllContstructorsAreTrivial : AllContstructorsAreTrivial {
-  using AllContstructorsAreTrivial::AllContstructorsAreTrivial;
+struct InheritedAllConstructorsAreTrivial : AllConstructorsAreTrivial {
+  using AllConstructorsAreTrivial::AllConstructorsAreTrivial;
 };
 
 struct UserDeclaredDestructor {
@@ -136,7 +136,7 @@ constexpr void test_is_implicit_lifetime() {
   test_is_implicit_lifetime<T[94], true>();
 }
 
-struct AritmeticTypesTest {
+struct ArithmeticTypesTest {
   template <class T>
   constexpr void operator()() {
     test_is_implicit_lifetime<T>();
@@ -152,7 +152,7 @@ constexpr bool test() {
   test_is_implicit_lifetime<const void, false>();
   test_is_implicit_lifetime<volatile void, false>();
 
-  types::for_each(types::arithmetic_types(), AritmeticTypesTest{});
+  types::for_each(types::arithmetic_types(), ArithmeticTypesTest{});
 
   test_is_implicit_lifetime<Enum>();
   test_is_implicit_lifetime<SignedEnum>();
@@ -184,17 +184,17 @@ constexpr bool test() {
 
   test_is_implicit_lifetime<UserProvidedDestructor, false>();
 
-  test_is_implicit_lifetime<NoEligibleTrivialContructor, false>();
+  test_is_implicit_lifetime<NoEligibleTrivialConstructor, false>();
 
   test_is_implicit_lifetime<OnlyDefaultConstructorIsTrivial, true>();
 
-  test_is_implicit_lifetime<AllContstructorsAreTrivial, true>();
+  test_is_implicit_lifetime<AllConstructorsAreTrivial, true>();
 
   test_is_implicit_lifetime<InheritedNoEligibleTrivialConstructor, false>();
 
   test_is_implicit_lifetime<InheritedOnlyDefaultConstructorIsTrivial, true>();
 
-  test_is_implicit_lifetime<InheritedAllContstructorsAreTrivial, true>();
+  test_is_implicit_lifetime<InheritedAllConstructorsAreTrivial, true>();
 
   test_is_implicit_lifetime<UserDeletedDestructorInAggregate, true>();
 
diff --git a/libcxx/test/std/utilities/optional/optional.iterator/begin.pass.cpp b/libcxx/test/std/utilities/optional/optional.iterator/begin.pass.cpp
new file mode 100644
index 0000000000000..df95a8df3793f
--- /dev/null
+++ b/libcxx/test/std/utilities/optional/optional.iterator/begin.pass.cpp
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++26
+
+// <optional>
+
+// constexpr iterator optional::begin() noexcept;
+// constexpr const_iterator optional::begin() const noexcept;
+
+#include <cassert>
+#include <iterator>
+#include <optional>
+#include <type_traits>
+#include <utility>
+
+template <typename T>
+constexpr bool test() {
+  std::optional<T> opt{T{}};
+
+  { // begin() is marked noexcept
+    static_assert(noexcept(opt.begin()));
+    static_assert(noexcept(std::as_const(opt).begin()));
+  }
+
+  { // Dereferencing an iterator at the beginning == indexing the 0th element, and that calling begin() again return the same iterator.
+    auto iter1 = opt.begin();
+    auto iter2 = std::as_const(opt).begin();
+    assert(*iter1 == iter1[0]);
+    assert(*iter2 == iter2[0]);
+    assert(iter1 == opt.begin());
+    assert(iter2 == std::as_const(opt).begin());
+  }
+
+  { // Calling begin() multiple times on a disengaged optional returns the same iterator.
+    std::optional<T> disengaged{std::nullopt};
+    auto iter1 = disengaged.begin();
+    auto iter2 = std::as_const(disengaged).begin();
+    assert(iter1 == disengaged.begin());
+    assert(iter2 == std::as_const(disengaged).begin());
+  }
+
+  return true;
+}
+
+constexpr bool tests() {
+  assert(test<int>());
+  assert(test<char>());
+  assert(test<const int>());
+  assert(test<const char>());
+  return true;
+}
+
+int main(int, char**) {
+  assert(tests());
+  static_assert(tests());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/optional/optional.iterator/end.pass.cpp b/libcxx/test/std/utilities/optional/optional.iterator/end.pass.cpp
new file mode 100644
index 0000000000000..966c3e7441880
--- /dev/null
+++ b/libcxx/test/std/utilities/optional/optional.iterator/end.pass.cpp
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++26
+
+// <optional>
+
+// constexpr iterator optional::end() noexcept;
+// constexpr const_iterator optional::end() const noexcept;
+
+#include <cassert>
+#include <iterator>
+#include <optional>
+#include <ranges>
+#include <utility>
+
+template <typename T>
+constexpr bool test() {
+  std::optional<T> disengaged{std::nullopt};
+
+  { // end() is marked noexcept
+    static_assert(noexcept(disengaged.end()));
+    static_assert(noexcept(std::as_const(disengaged).end()));
+  }
+
+  { // end() == begin() and end() == end() if the optional is disengaged
+    auto it  = disengaged.end();
+    auto it2 = std::as_const(disengaged).end();
+
+    assert(it == disengaged.begin());
+    assert(disengaged.begin() == it);
+    assert(it == disengaged.end());
+
+    assert(it2 == std::as_const(disengaged).begin());
+    assert(std::as_const(disengaged).begin() == it2);
+    assert(it2 == std::as_const(disengaged).end());
+  }
+
+  std::optional<T> engaged{T{}};
+
+  { // end() != begin() if the optional is engaged
+    auto it  = engaged.end();
+    auto it2 = std::as_const(engaged).end();
+
+    assert(it != engaged.begin());
+    assert(engaged.begin() != it);
+
+    assert(it2 != std::as_const(engaged).begin());
+    assert(std::as_const(engaged).begin() != it2);
+  }
+
+  return true;
+}
+
+constexpr bool tests() {
+  assert(test<int>());
+  assert(test<char>());
+  assert(test<const int>());
+  assert(test<const char>());
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(tests());
+  static_assert(tests());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/optional/optional.iterator/iterator.pass.cpp b/libcxx/test/std/utilities/optional/optional.iterator/iterator.pass.cpp
new file mode 100644
index 0000000000000..1203290a0290a
--- /dev/null
+++ b/libcxx/test/std/utilities/optional/optional.iterator/iterator.pass.cpp
@@ -0,0 +1,98 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++26
+
+// <optional>
+
+// template <class T> class optional::iterator;
+// template <class T> class optional::const_iterator;
+
+#include <cassert>
+#include <iterator>
+#include <optional>
+#include <ranges>
+#include <type_traits>
+#include <utility>
+
+template <typename T, T __val>
+constexpr bool test() {
+  std::optional<T> opt{__val};
+
+  { // Dereferencing an iterator of an engaged optional will return the same value that the optional holds.
+    auto it  = opt.begin();
+    auto it2 = std::as_const(opt).begin();
+    assert(*it == *opt);
+    assert(*it2 == *std::as_const(opt));
+  }
+
+  { // optional::iterator and optional::const_iterator satisfy the Cpp17RandomAccessIterator and contiguous iterator.
+    auto it  = opt.begin();
+    auto it2 = std::as_const(opt).begin();
+    assert(std::contiguous_iterator<decltype(it)>);
+    assert(std::contiguous_iterator<decltype(it2)>);
+
+    assert(std::random_access_iterator<decltype(it)>);
+    assert(std::random_access_iterator<decltype(it2)>);
+  }
+
+  { // const_iterator::value_type == std::remove_cv_t<T>, const_iterator::reference == const T&, iterator::value_type = std::remove_cv_t<T>, iterator::reference == T&
+    auto it  = opt.begin();
+    auto it2 = std::as_const(opt).begin();
+    assert((std::is_same_v<typename decltype(it)::value_type, std::remove_cv_t<T>>));
+    assert((std::is_same_v<typename decltype(it)::reference, T&>));
+    assert((std::is_same_v<typename decltype(it2)::value_type, std::remove_cv_t<T>>));
+    assert((std::is_same_v<typename decltype(it2)::reference, const T&>));
+  }
+
+  { // std::ranges::size for an engaged optional<T> == 1, disengaged optional<T> == 0
+    const std::optional<T> disengaged{std::nullopt};
+    std::optional<T> disengaged2{std::nullopt};
+    assert(std::ranges::size(opt) == 1);
+    assert(std::ranges::size(std::as_const(opt)) == 1);
+
+    assert(std::ranges::size(disengaged) == 0);
+    assert(std::ranges::size(disengaged2) == 0);
+  }
+
+  { // std::ranges::enable_view<optional<T>> == true, and std::format_kind<optional<T>> == true
+    static_assert(std::ranges::enable_view<std::optional<T>> == true);
+    static_assert(std::format_kind<std::optional<T>> == std::range_format::disabled);
+  }
+
+  // An optional with value that is reset will have a begin() == end(), then when it is reassigned a value,
+  // begin() != end(), and *begin() will contain the new value.
+  {
+    std::optional<T> val{__val};
+    assert(val.begin() != val.end());
+    val.reset();
+    assert(val.begin() == val.end());
+    val.emplace(__val);
+    assert(val.begin() != val.end());
+    assert(*(val.begin()) == __val);
+  }
+
+  return true;
+}
+
+constexpr bool tests() {
+  assert((test<int, 1>()));
+  assert((test<char, 'a'>()));
+  assert((test<bool, true>()));
+  assert((test<const int, 2>()));
+  assert((test<const char, 'b'>()));
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(tests());
+  static_assert(tests());
+
+  return 0;
+}
diff --git a/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.general.pass.cpp b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.general.pass.cpp
index d55a0c2abd464..864b5f444fd84 100644
--- a/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.general.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/adapt/inout_ptr/inout_ptr.general.pass.cpp
@@ -54,7 +54,7 @@ void test_replace_int_p() {
 
     replace_int_p(std::inout_ptr(uPtr, std::move(del)));
     assert(*uPtr == 84);
-    assert(uPtr.get_deleter().wasMoveInitilized == true);
+    assert(uPtr.get_deleter().wasMoveInitialized == true);
   }
 
   // pointer-like ConstructiblePtr
diff --git a/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.general.pass.cpp b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.general.pass.cpp
index a78e22f69abc0..3186b6c757582 100644
--- a/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.general.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/adapt/out_ptr/out_ptr.general.pass.cpp
@@ -49,7 +49,7 @@ void test_get_int_p() {
 
     get_int_p(std::out_ptr(uPtr, std::move(del)));
     assert(*uPtr == 84);
-    assert(uPtr.get_deleter().wasMoveInitilized == true);
+    assert(uPtr.get_deleter().wasMoveInitialized == true);
   }
 
   // std::shared_ptr
diff --git a/libcxx/test/std/utilities/smartptr/adapt/types.h b/libcxx/test/std/utilities/smartptr/adapt/types.h
index 0da6007ba394f..886bb88cbbeb1 100644
--- a/libcxx/test/std/utilities/smartptr/adapt/types.h
+++ b/libcxx/test/std/utilities/smartptr/adapt/types.h
@@ -21,12 +21,12 @@ struct MoveOnlyDeleter {
   MoveOnlyDeleter()                                  = default;
   MoveOnlyDeleter(const MoveOnlyDeleter&)            = delete;
   MoveOnlyDeleter& operator=(const MoveOnlyDeleter&) = delete;
-  MoveOnlyDeleter(MoveOnlyDeleter&&) : wasMoveInitilized{true} {}
+  MoveOnlyDeleter(MoveOnlyDeleter&&) : wasMoveInitialized{true} {}
   MoveOnlyDeleter& operator=(MoveOnlyDeleter&&) = default;
 
   void operator()(T* p) const { delete p; }
 
-  bool wasMoveInitilized = false;
+  bool wasMoveInitialized = false;
 };
 
 // Custom pointer types.
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/default.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/default.pass.cpp
index 7f4c90922d6c3..e2abfd17d27c3 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/default.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/default.pass.cpp
@@ -82,6 +82,10 @@ TEST_CONSTEXPR_CXX23 bool test_basic() {
     p.get_deleter().set_state(5);
     assert(p.get_deleter().state() == 5);
   }
+// TODO: Remove this check once https://llvm.org/PR154567 is fixed
+#if TEST_STD_VER >= 23 && defined(TEST_COMPILER_CLANG)
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
   {
     std::unique_ptr<ElemType, DefaultCtorDeleter<ElemType> > p;
     assert(p.get() == 0);
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/nullptr.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/nullptr.pass.cpp
index 45017a03b95dd..f5e0541684d13 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/nullptr.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/nullptr.pass.cpp
@@ -47,6 +47,10 @@ TEST_CONSTEXPR_CXX23 void test_basic() {
     assert(p.get() == 0);
     assert(p.get_deleter().state() == 0);
   }
+// TODO: Remove this check once https://llvm.org/PR154567 is fixed
+#if TEST_STD_VER >= 23 && defined(TEST_COMPILER_CLANG)
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
   {
     std::unique_ptr<VT, DefaultCtorDeleter<VT> > p(nullptr);
     assert(p.get() == 0);
diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer.pass.cpp
index cbce5c9c74c5a..e9912d4574af6 100644
--- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer.pass.cpp
+++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer.pass.cpp
@@ -75,6 +75,10 @@ TEST_CONSTEXPR_CXX23 void test_pointer() {
   }
   if (!TEST_IS_CONSTANT_EVALUATED)
     assert(A::count == 0);
+// TODO: Remove this check once https://llvm.org/PR154567 is fixed
+#if TEST_STD_VER >= 23 && defined(TEST_COMPILER_CLANG)
+  if (!TEST_IS_CONSTANT_EVALUATED)
+#endif
   {
     A* p = newValue<ValueT>(expect_alive);
     if (!TEST_IS_CONSTANT_EVALUATED)
diff --git a/libcxx/test/std/utilities/template.bitset/bitset.members/to_ullong.pass.cpp b/libcxx/test/std/utilities/template.bitset/bitset.members/to_ullong.pass.cpp
index 3c5a57d1c7fec..980dbe9ba15d0 100644
--- a/libcxx/test/std/utilities/template.bitset/bitset.members/to_ullong.pass.cpp
+++ b/libcxx/test/std/utilities/template.bitset/bitset.members/to_ullong.pass.cpp
@@ -65,7 +65,7 @@ TEST_CONSTEXPR_CXX23 bool test() {
     std::bitset<std::numeric_limits<unsigned long long>::digits + 1> q(0);
     q.flip();
     try {
-      q.to_ullong(); // throws
+      (void)q.to_ullong(); // throws
       assert(false);
     } catch (const std::overflow_error&) {
       // expected
diff --git a/libcxx/test/std/utilities/template.bitset/bitset.members/to_ulong.pass.cpp b/libcxx/test/std/utilities/template.bitset/bitset.members/to_ulong.pass.cpp
index d943030a16287..47ad127513ade 100644
--- a/libcxx/test/std/utilities/template.bitset/bitset.members/to_ulong.pass.cpp
+++ b/libcxx/test/std/utilities/template.bitset/bitset.members/to_ulong.pass.cpp
@@ -64,7 +64,7 @@ TEST_CONSTEXPR_CXX23 bool test() {
     std::bitset<std::numeric_limits<unsigned long>::digits + 1> q(0);
     q.flip();
     try {
-      q.to_ulong(); // throws
+      (void)q.to_ulong(); // throws
       assert(false);
     } catch (const std::overflow_error&) {
       // expected
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/move_ctor_sfinae.compile.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/move_ctor_sfinae.compile.pass.cpp
new file mode 100644
index 0000000000000..dc50385b4654c
--- /dev/null
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/move_ctor_sfinae.compile.pass.cpp
@@ -0,0 +1,27 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Ensure that tuple's move constructor properly SFINAES.
+// This is a regression test for https://github.com/llvm/llvm-project/pull/151654#issuecomment-3205410955
+
+// UNSUPPORTED: c++03, c++11, c++14
+
+#include <tuple>
+#include <variant>
+#include <type_traits>
+
+struct S {
+  S(const S&)            = delete;
+  S& operator=(const S&) = delete;
+  S(S&&)                 = default;
+  S& operator=(S&&)      = default;
+};
+
+using T = std::tuple<const std::variant<S>>;
+
+void func() { (void)std::is_trivially_move_constructible<T>::value; }
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp
index 2c14de8c796c1..8b9e94a1c1ad9 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.pass.cpp
@@ -286,6 +286,8 @@ LIBCPP_STATIC_ASSERT(can_make_from_tuple_impl<float, std::tuple<double>>);
 
 } // namespace LWG3528
 
+static_assert(LWG3528::can_make_from_tuple<int, std::tuple<>>);
+
 int main(int, char**)
 {
     test_constexpr_construction();
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.verify.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.verify.cpp
new file mode 100644
index 0000000000000..2dfbae9138864
--- /dev/null
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.apply/make_from_tuple.verify.cpp
@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: std-at-least-c++23
+
+// <tuple>
+
+// template <class T, class Tuple> constexpr T make_from_tuple(Tuple&&);
+// Mandates: If tuple_size_v<remove_reference_t<Tuple>> is 1, then reference_constructs_from_temporary_v<T, decltype(get<0>(declval<Tuple>()))> is false.
+
+#include <tuple>
+#include <utility>
+
+#include "test_macros.h"
+
+void test() {
+  // FreeBSD ci use clang 19.1.1, which hasn't implement __reference_constructs_from_temporary.
+  // The static_assert inner std::make_from_tuple will not triggered.
+#if __has_builtin(__reference_constructs_from_temporary)
+  // expected-error@*:* {{static assertion failed}}
+#endif
+
+  // Turns to an error since C++26 (Disallow Binding a Returned Glvalue to a Temporary https://wg21.link/P2748R5).
+#if TEST_STD_VER >= 26
+  // expected-error@tuple:* {{returning reference to local temporary object}}
+#else
+  // expected-warning@tuple:* {{returning reference to local temporary object}}
+#endif
+  std::ignore = std::make_from_tuple<const int&>(std::tuple<char>{});
+}
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.verify.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.verify.cpp
index 9a39c18f138fb..f440b95653ccd 100644
--- a/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.verify.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/nttp.verify.cpp
@@ -74,7 +74,7 @@ using H = test<std::pair<NotALiteral, NotALiteral>{}>;
 // expected-error@-1 {{non-type template parameter has non-literal type 'std::pair<NotALiteral, NotALiteral>'}}
 
 using I = test<std::pair<std::string, std::string>{}>;
-// expected-error-re@-1 {{type 'std::pair<{{(std::)?}}string, {{(std::)?}}string>'{{( \(aka 'std::pair<std::string, std::string>'\))?}} of non-type template parameter is not a structural type}}
+// expected-error-re@*:* {{type {{.+}} of non-type template parameter is not a structural type}}
 } // namespace test_ctad
 
 namespace test_auto {
@@ -95,5 +95,5 @@ using H = test<std::pair<NotALiteral, NotALiteral>{}>;
 // expected-error@-1 {{non-type template parameter has non-literal type 'std::pair<NotALiteral, NotALiteral>'}}
 
 using I = test<std::pair<std::string, std::string>{}>;
-// expected-error@-1 {{type 'std::pair<std::string, std::string>' (aka 'pair<basic_string<char>, basic_string<char>>') of non-type template parameter is not a structural type}}
+// expected-error-re@-1 {{type {{.+}} of non-type template parameter is not a structural type}}
 } // namespace test_auto
diff --git a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/pair.incomplete.compile.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/pair.incomplete.compile.pass.cpp
similarity index 91%
rename from libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/pair.incomplete.compile.pass.cpp
rename to libcxx/test/std/utilities/utility/pairs/pairs.pair/pair.incomplete.compile.pass.cpp
index 16ee000cd90fa..2c670d7432ac3 100644
--- a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/pair.incomplete.compile.pass.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/pair.incomplete.compile.pass.cpp
@@ -9,7 +9,7 @@
 // Check that instantiating pair doesn't look up type traits "too early", before
 // the contained types have been completed.
 //
-// This is a regression test, to prevent a reoccurrance of the issue introduced
+// This is a regression test, to prevent a reoccurrence of the issue introduced
 // in 5e1de27f680591a870d78e9952b23f76aed7f456.
 
 #include <utility>
diff --git a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp
index 142da1d820d9a..6111138726dbc 100644
--- a/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp
+++ b/libcxx/test/std/utilities/variant/variant.variant/variant.ctor/T.pass.cpp
@@ -173,6 +173,11 @@ void test_vector_bool() {
   assert(std::get<0>(v) == true);
 }
 
+struct ConvertibleFromAny {
+  template <class V>
+  ConvertibleFromAny(V) {}
+};
+
 int main(int, char**) {
   test_T_ctor_basic();
   test_T_ctor_noexcept();
@@ -180,5 +185,16 @@ int main(int, char**) {
   test_no_narrowing_check_for_class_types();
   test_construction_with_repeated_types();
   test_vector_bool();
+
+  { // Check that the constraints are evaluated lazily (see https://github.com/llvm/llvm-project/issues/151328)
+    struct Matcher {
+      Matcher() {}
+      Matcher(std::variant<ConvertibleFromAny>) {}
+    };
+
+    Matcher vec;
+    [[maybe_unused]] Matcher m = std::move(vec);
+  }
+
   return 0;
 }
diff --git a/libcxx/test/support/is_transparent.h b/libcxx/test/support/is_transparent.h
index 700c894a8b60f..4b2a458f574aa 100644
--- a/libcxx/test/support/is_transparent.h
+++ b/libcxx/test/support/is_transparent.h
@@ -36,6 +36,17 @@ struct transparent_less_not_referenceable
     using is_transparent = void () const &;  // it's a type; a weird one, but a type
 };
 
+// Prevent regression when empty base class optimization is not suitable.
+// See https://github.com/llvm/llvm-project/issues/152543.
+struct transparent_less_nonempty {
+  template <class T, class U>
+  constexpr bool operator()(T&& t, U&& u) const {
+    return std::forward<T>(t) < std::forward<U>(u);
+  }
+  struct is_transparent {
+  } pad_; // making this comparator non-empty
+};
+
 struct transparent_less_no_type
 {
     template <class T, class U>
diff --git a/libcxx/utils/ci/docker-compose.yml b/libcxx/utils/ci/docker-compose.yml
index 36b8dd77c47fb..d8ba8e5fba2bd 100644
--- a/libcxx/utils/ci/docker-compose.yml
+++ b/libcxx/utils/ci/docker-compose.yml
@@ -1,6 +1,10 @@
 x-versions: &compiler_versions
   GCC_HEAD_VERSION: 16
-  LLVM_HEAD_VERSION: 21
+  LLVM_HEAD_VERSION: 22
+
+x-image-versions: &image_versions
+  BASE_IMAGE: ubuntu:jammy
+  ACTIONS_BASE_IMAGE: ghcr.io/llvm/libcxx-linux-builder-base:77cb0980bcc2675b27d08141526939423fa0be76
 
 services:
   builder-base:
@@ -10,8 +14,7 @@ services:
       dockerfile: Dockerfile
       target: builder-base
       args:
-        BASE_IMAGE: ubuntu:jammy
-        <<: *compiler_versions
+        <<: [*image_versions, *compiler_versions]
 
   actions-builder:
     image: ghcr.io/llvm/libcxx-linux-builder:${TAG}
@@ -20,10 +23,8 @@ services:
       dockerfile: Dockerfile
       target: actions-builder
       args:
-        BASE_IMAGE: ubuntu:jammy
-        ACTIONS_BASE_IMAGE: builder-base
-        GITHUB_RUNNER_VERSION: "2.326.0"
-        <<: *compiler_versions
+        GITHUB_RUNNER_VERSION: "2.328.0"
+        <<: [*image_versions, *compiler_versions]
 
   android-buildkite-builder:
     image: ghcr.io/llvm/libcxx-android-builder:${TAG}
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index d9317e00e3f4a..8d57a07b8836b 100644
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -1012,7 +1012,6 @@ def add_version_header(tc):
             "name": "__cpp_lib_optional_range_support",
             "values": {"c++26": 202406},  # P3168R2 Give std::optional Range Support
             "headers": ["optional"],
-            "unimplemented": True,
         },
         {
             "name": "__cpp_lib_out_ptr",
diff --git a/libcxxabi/CMakeLists.txt b/libcxxabi/CMakeLists.txt
index cf478bcee01fa..3dabd87b9c587 100644
--- a/libcxxabi/CMakeLists.txt
+++ b/libcxxabi/CMakeLists.txt
@@ -244,7 +244,7 @@ include(HandleLibcxxabiFlags)
 #===============================================================================
 
 # Configure target flags
-if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   add_flags_if_supported("-mdefault-visibility-export-mapping=explicit")
   set(CMAKE_AIX_EXPORT_ALL_SYMBOLS OFF)
 endif()
@@ -458,7 +458,7 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${LIBCXXABI_C_FLAGS}")
 
 # On AIX, avoid picking up VMX extensions(i.e. vec_malloc) which would change
 # the default alignment of the allocators here.
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   add_definitions("-D_XOPEN_SOURCE=700")
 endif()
 
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index 0675577aed6a1..38a54b16278a7 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -37,7 +37,7 @@ else()
 endif()
 
 if (LIBCXXABI_ENABLE_THREADS AND (UNIX OR FUCHSIA) AND NOT (APPLE OR CYGWIN)
-    AND NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX"))
+    AND NOT ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX"))
   list(APPEND LIBCXXABI_SOURCES
     cxa_thread_atexit.cpp
   )
diff --git a/libunwind/include/__libunwind_config.h b/libunwind/include/__libunwind_config.h
index bb7fe4c83a3c1..544b3ec96216a 100644
--- a/libunwind/include/__libunwind_config.h
+++ b/libunwind/include/__libunwind_config.h
@@ -73,11 +73,11 @@
 #  define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_PPC
 # elif defined(__aarch64__)
 #  define _LIBUNWIND_TARGET_AARCH64 1
-#  define _LIBUNWIND_CONTEXT_SIZE 66
+#define _LIBUNWIND_CONTEXT_SIZE 67
 #  if defined(__SEH__)
 #    define _LIBUNWIND_CURSOR_SIZE 164
 #  else
-#    define _LIBUNWIND_CURSOR_SIZE 78
+#define _LIBUNWIND_CURSOR_SIZE 79
 #  endif
 #  define _LIBUNWIND_HIGHEST_DWARF_REGISTER _LIBUNWIND_HIGHEST_DWARF_REGISTER_ARM64
 # elif defined(__arm__)
diff --git a/libunwind/include/libunwind.h b/libunwind/include/libunwind.h
index b2dae8feed9a3..94928f436025a 100644
--- a/libunwind/include/libunwind.h
+++ b/libunwind/include/libunwind.h
@@ -532,6 +532,7 @@ enum {
   UNW_AARCH64_X31 = 31,
   UNW_AARCH64_SP = 31,
   UNW_AARCH64_PC = 32,
+  UNW_AARCH64_VG = 46,
 
   // reserved block
   UNW_AARCH64_RA_SIGN_STATE = 34,
diff --git a/libunwind/src/CMakeLists.txt b/libunwind/src/CMakeLists.txt
index 71663a22cdf41..514c2fcd5bf4e 100644
--- a/libunwind/src/CMakeLists.txt
+++ b/libunwind/src/CMakeLists.txt
@@ -6,7 +6,7 @@ set(LIBUNWIND_CXX_SOURCES
     Unwind-seh.cpp
     )
 
-if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   list(APPEND LIBUNWIND_CXX_SOURCES
     Unwind_AIXExtras.cpp
     )
diff --git a/libunwind/src/Registers.hpp b/libunwind/src/Registers.hpp
index 2c3bfb7e8428a..8b3055f5cd7cd 100644
--- a/libunwind/src/Registers.hpp
+++ b/libunwind/src/Registers.hpp
@@ -1851,6 +1851,8 @@ class _LIBUNWIND_HIDDEN Registers_arm64 {
   void      setFP(uint64_t value) { _registers.__fp = value; }
 
 private:
+  uint64_t lazyGetVG() const;
+
   struct GPRs {
     uint64_t __x[29]; // x0-x28
     uint64_t __fp;    // Frame pointer x29
@@ -1860,12 +1862,22 @@ class _LIBUNWIND_HIDDEN Registers_arm64 {
     uint64_t __ra_sign_state; // RA sign state register
   };
 
-  GPRs    _registers;
-  double  _vectorHalfRegisters[32];
+  struct Misc {
+    mutable uint64_t __vg = 0; // Vector Granule
+  };
+
+  GPRs _registers;
   // Currently only the lower double in 128-bit vectore registers
   // is perserved during unwinding.  We could define new register
   // numbers (> 96) which mean whole vector registers, then this
   // struct would need to change to contain whole vector registers.
+  double _vectorHalfRegisters[32];
+
+  // Miscellaneous/virtual registers. These are stored below the GPRs and FPRs
+  // as they do not correspond to physical registers, so do not need to be
+  // saved/restored in UnwindRegistersRestore.S and UnwindRegistersSave.S, and
+  // we don't want to modify the existing offsets for GPRs and FPRs.
+  Misc _misc_registers;
 };
 
 inline Registers_arm64::Registers_arm64(const void *registers) {
@@ -1895,11 +1907,27 @@ inline bool Registers_arm64::validRegister(int regNum) const {
     return false;
   if (regNum == UNW_AARCH64_RA_SIGN_STATE)
     return true;
+  if (regNum == UNW_AARCH64_VG)
+    return true;
   if ((regNum > 32) && (regNum < 64))
     return false;
   return true;
 }
 
+inline uint64_t Registers_arm64::lazyGetVG() const {
+  if (!_misc_registers.__vg) {
+#if defined(__aarch64__)
+    register uint64_t vg asm("x0");
+    asm(".inst 0x04e0e3e0" // CNTD x0
+        : "=r"(vg));
+    _misc_registers.__vg = vg;
+#else
+    _LIBUNWIND_ABORT("arm64 VG undefined");
+#endif
+  }
+  return _misc_registers.__vg;
+}
+
 inline uint64_t Registers_arm64::getRegister(int regNum) const {
   if (regNum == UNW_REG_IP || regNum == UNW_AARCH64_PC)
     return _registers.__pc;
@@ -1911,6 +1939,8 @@ inline uint64_t Registers_arm64::getRegister(int regNum) const {
     return _registers.__fp;
   if (regNum == UNW_AARCH64_LR)
     return _registers.__lr;
+  if (regNum == UNW_AARCH64_VG)
+    return lazyGetVG();
   if ((regNum >= 0) && (regNum < 29))
     return _registers.__x[regNum];
   _LIBUNWIND_ABORT("unsupported arm64 register");
@@ -1927,6 +1957,8 @@ inline void Registers_arm64::setRegister(int regNum, uint64_t value) {
     _registers.__fp = value;
   else if (regNum == UNW_AARCH64_LR)
     _registers.__lr = value;
+  else if (regNum == UNW_AARCH64_VG)
+    _misc_registers.__vg = value;
   else if ((regNum >= 0) && (regNum < 29))
     _registers.__x[regNum] = value;
   else
diff --git a/libunwind/test/aarch64_vg_unwind.pass.cpp b/libunwind/test/aarch64_vg_unwind.pass.cpp
new file mode 100644
index 0000000000000..b6bb9693c1e0f
--- /dev/null
+++ b/libunwind/test/aarch64_vg_unwind.pass.cpp
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// REQUIRES: linux && target={{aarch64-.+}}
+
+#include <libunwind.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Basic test of VG (Vector Granule) unwinding. This is meant to mimic SVE/SME
+// unwind info without requiring those features for this test.
+
+#define VG_REGNUM 46
+
+__attribute__((noinline)) void baz() {
+  // The previous value of VG is 2
+  asm(".cfi_escape 0x16, 0x2e, 0x01, 0x32");
+
+  unw_context_t context;
+  unw_cursor_t cursor;
+  unw_getcontext(&context);
+  unw_init_local(&cursor, &context);
+
+  // Note: At this point VG is not defined (until we unw_step).
+
+  uint16_t expected_vgs[]{/*qux*/ 2, /*bar*/ 2, /*foo*/ 8, /*main*/ 2};
+  for (uint16_t expected_vg : expected_vgs) {
+    unw_step(&cursor);
+    unw_word_t vg;
+    unw_get_reg(&cursor, VG_REGNUM, &vg);
+    if (vg != expected_vg)
+      exit(1);
+  }
+  exit(0);
+}
+
+__attribute__((noinline)) void qux() { baz(); }
+
+__attribute__((noinline)) void bar() {
+  // The previous value of VG is 8
+  asm(".cfi_escape 0x16, 0x2e, 0x01, 0x38");
+  // The previous value of W21 is VG (used to force an evaluation of VG).
+  asm(".cfi_escape 0x16, 0x15, 0x03, 0x92, 0x2e, 0x00");
+
+  // smstop sm
+  qux();
+  // smstart sm
+}
+__attribute__((noinline)) void foo() {
+  // The previous value of VG is 2
+  asm(".cfi_escape 0x16, 0x2e, 0x01, 0x32");
+  // The previous value of W21 is VG (used to force an evaluation of VG).
+  asm(".cfi_escape 0x16, 0x15, 0x03, 0x92, 0x2e, 0x00");
+
+  // smstart sm
+  bar();
+  // smstop sm
+}
+
+int main() { foo(); }
diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp
index e11d2c6dac83e..0a88807c00dd5 100644
--- a/lld/COFF/SymbolTable.cpp
+++ b/lld/COFF/SymbolTable.cpp
@@ -1399,6 +1399,7 @@ void SymbolTable::resolveAlternateNames() {
       auto toUndef = dyn_cast<Undefined>(toSym);
       if (toUndef && (!toUndef->weakAlias || toUndef->isAntiDep))
         continue;
+      toSym->isUsedInRegularObj = true;
       if (toSym->isLazy())
         forceLazy(toSym);
       u->setWeakAlias(toSym);
diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index 1812f2af419d2..27e77e943c197 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -154,6 +154,12 @@ RelExpr AArch64::getRelExpr(RelType type, const Symbol &s,
   case R_AARCH64_MOVW_UABS_G2_NC:
   case R_AARCH64_MOVW_UABS_G3:
     return R_ABS;
+  case R_AARCH64_PATCHINST:
+    if (!isAbsolute(s))
+      Err(ctx) << getErrorLoc(ctx, loc)
+               << "R_AARCH64_PATCHINST relocation against non-absolute symbol "
+               << &s;
+    return R_ABS;
   case R_AARCH64_AUTH_ABS64:
     return RE_AARCH64_AUTH;
   case R_AARCH64_TLSDESC_ADR_PAGE21:
@@ -506,6 +512,12 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
     checkIntUInt(ctx, loc, val, 32, rel);
     write32(ctx, loc, val);
     break;
+  case R_AARCH64_PATCHINST:
+    if (!rel.sym->isUndefined()) {
+      checkUInt(ctx, loc, val, 32, rel);
+      write32le(loc, val);
+    }
+    break;
   case R_AARCH64_PLT32:
   case R_AARCH64_GOTPCREL32:
     checkInt(ctx, loc, val, 32, rel);
diff --git a/lld/ELF/Arch/LoongArch.cpp b/lld/ELF/Arch/LoongArch.cpp
index 8802c8c2e7f01..db2c71c3b42b9 100644
--- a/lld/ELF/Arch/LoongArch.cpp
+++ b/lld/ELF/Arch/LoongArch.cpp
@@ -39,6 +39,7 @@ class LoongArch final : public TargetInfo {
   void relocate(uint8_t *loc, const Relocation &rel,
                 uint64_t val) const override;
   bool relaxOnce(int pass) const override;
+  bool synthesizeAlign(uint64_t &dot, InputSection *sec) override;
   RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;
   void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override;
   void finalizeRelax(int passes) const override;
@@ -48,6 +49,19 @@ class LoongArch final : public TargetInfo {
   void tlsdescToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
   bool tryGotToPCRel(uint8_t *loc, const Relocation &rHi20,
                      const Relocation &rLo12, uint64_t secAddr) const;
+  template <class ELFT, class RelTy>
+  bool synthesizeAlignForInput(uint64_t &dot, InputSection *sec,
+                               Relocs<RelTy> rels);
+  template <class ELFT, class RelTy>
+  void finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,
+                                Relocs<RelTy> rels);
+  template <class ELFT>
+  bool synthesizeAlignAux(uint64_t &dot, InputSection *sec);
+
+  // The following two variables are used by synthesized ALIGN relocations.
+  InputSection *baseSec = nullptr;
+  // r_offset and r_addend pairs.
+  SmallVector<std::pair<uint64_t, uint64_t>, 0> synthesizedAligns;
 };
 } // end anonymous namespace
 
@@ -766,6 +780,117 @@ void LoongArch::relocate(uint8_t *loc, const Relocation &rel,
   }
 }
 
+// If the section alignment is > 4, advance `dot` to insert NOPs and synthesize
+// an ALIGN relocation. Otherwise, return false to use default handling.
+template <class ELFT, class RelTy>
+bool LoongArch::synthesizeAlignForInput(uint64_t &dot, InputSection *sec,
+                                        Relocs<RelTy> rels) {
+  if (!baseSec) {
+    // Record the first input section with RELAX relocations. We will synthesize
+    // ALIGN relocations here.
+    for (auto rel : rels) {
+      if (rel.getType(false) == R_LARCH_RELAX) {
+        baseSec = sec;
+        break;
+      }
+    }
+  } else if (sec->addralign > 4) {
+    // If the alignment is > 4 and the section does not start with an ALIGN
+    // relocation, synthesize one.
+    bool hasAlignRel = llvm::any_of(rels, [](const RelTy &rel) {
+      return rel.r_offset == 0 && rel.getType(false) == R_LARCH_ALIGN;
+    });
+    if (!hasAlignRel) {
+      synthesizedAligns.emplace_back(dot - baseSec->getVA(),
+                                     sec->addralign - 4);
+      dot += sec->addralign - 4;
+      return true;
+    }
+  }
+  return false;
+}
+
+// Finalize the relocation section by appending synthesized ALIGN relocations
+// after processing all input sections.
+template <class ELFT, class RelTy>
+void LoongArch::finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,
+                                         Relocs<RelTy> rels) {
+  auto *f = cast<ObjFile<ELFT>>(baseSec->file);
+  auto shdr = f->template getELFShdrs<ELFT>()[baseSec->relSecIdx];
+  // Create a copy of InputSection.
+  sec = make<InputSection>(*f, shdr, baseSec->name);
+  auto *baseRelSec = cast<InputSection>(f->getSections()[baseSec->relSecIdx]);
+  *sec = *baseRelSec;
+  baseSec = nullptr;
+
+  // Allocate buffer for original and synthesized relocations in RELA format.
+  // If CREL is used, OutputSection::finalizeNonAllocCrel will convert RELA to
+  // CREL.
+  auto newSize = rels.size() + synthesizedAligns.size();
+  auto *relas = makeThreadLocalN<typename ELFT::Rela>(newSize);
+  sec->size = newSize * sizeof(typename ELFT::Rela);
+  sec->content_ = reinterpret_cast<uint8_t *>(relas);
+  sec->type = SHT_RELA;
+  // Copy original relocations to the new buffer, potentially converting CREL to
+  // RELA.
+  for (auto [i, r] : llvm::enumerate(rels)) {
+    relas[i].r_offset = r.r_offset;
+    relas[i].setSymbolAndType(r.getSymbol(0), r.getType(0), false);
+    if constexpr (RelTy::HasAddend)
+      relas[i].r_addend = r.r_addend;
+  }
+  // Append synthesized ALIGN relocations to the buffer.
+  for (auto [i, r] : llvm::enumerate(synthesizedAligns)) {
+    auto &rela = relas[rels.size() + i];
+    rela.r_offset = r.first;
+    rela.setSymbolAndType(0, R_LARCH_ALIGN, false);
+    rela.r_addend = r.second;
+  }
+  synthesizedAligns.clear();
+  // Replace the old relocation section with the new one in the output section.
+  // addOrphanSections ensures that the output relocation section is processed
+  // after osec.
+  for (SectionCommand *cmd : sec->getParent()->commands) {
+    auto *isd = dyn_cast<InputSectionDescription>(cmd);
+    if (!isd)
+      continue;
+    for (auto *&isec : isd->sections)
+      if (isec == baseRelSec)
+        isec = sec;
+  }
+}
+
+template <class ELFT>
+bool LoongArch::synthesizeAlignAux(uint64_t &dot, InputSection *sec) {
+  bool ret = false;
+  if (sec) {
+    invokeOnRelocs(*sec, ret = synthesizeAlignForInput<ELFT>, dot, sec);
+  } else if (baseSec) {
+    invokeOnRelocs(*baseSec, finalizeSynthesizeAligns<ELFT>, dot, sec);
+  }
+  return ret;
+}
+
+// Without linker relaxation enabled for a particular relocatable file or
+// section, the assembler will not generate R_LARCH_ALIGN relocations for
+// alignment directives. This becomes problematic in a two-stage linking
+// process: ld -r a.o b.o -o ab.o; ld ab.o -o ab. This function synthesizes an
+// R_LARCH_ALIGN relocation at section start when needed.
+//
+// When called with an input section (`sec` is not null): If the section
+// alignment is > 4, advance `dot` to insert NOPs and synthesize an ALIGN
+// relocation.
+//
+// When called after all input sections are processed (`sec` is null): The
+// output relocation section is updated with all the newly synthesized ALIGN
+// relocations.
+bool LoongArch::synthesizeAlign(uint64_t &dot, InputSection *sec) {
+  assert(ctx.arg.relocatable);
+  if (ctx.arg.is64)
+    return synthesizeAlignAux<ELF64LE>(dot, sec);
+  return synthesizeAlignAux<ELF32LE>(dot, sec);
+}
+
 static bool relaxable(ArrayRef<Relocation> relocs, size_t i) {
   return i + 1 < relocs.size() && relocs[i + 1].type == R_LARCH_RELAX;
 }
@@ -1396,9 +1521,6 @@ void LoongArch::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
 // change in section sizes can have cascading effect and require another
 // relaxation pass.
 bool LoongArch::relaxOnce(int pass) const {
-  if (ctx.arg.relocatable)
-    return false;
-
   if (pass == 0)
     initSymbolAnchors(ctx);
 
diff --git a/lld/ELF/Arch/RISCV.cpp b/lld/ELF/Arch/RISCV.cpp
index 72d83159ad8ac..85ddf1b6da283 100644
--- a/lld/ELF/Arch/RISCV.cpp
+++ b/lld/ELF/Arch/RISCV.cpp
@@ -45,7 +45,21 @@ class RISCV final : public TargetInfo {
                 uint64_t val) const override;
   void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override;
   bool relaxOnce(int pass) const override;
+  template <class ELFT, class RelTy>
+  bool synthesizeAlignForInput(uint64_t &dot, InputSection *sec,
+                               Relocs<RelTy> rels);
+  template <class ELFT, class RelTy>
+  void finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,
+                                Relocs<RelTy> rels);
+  template <class ELFT>
+  bool synthesizeAlignAux(uint64_t &dot, InputSection *sec);
+  bool synthesizeAlign(uint64_t &dot, InputSection *sec) override;
   void finalizeRelax(int passes) const override;
+
+  // The following two variables are used by synthesized ALIGN relocations.
+  InputSection *baseSec = nullptr;
+  // r_offset and r_addend pairs.
+  SmallVector<std::pair<uint64_t, uint64_t>, 0> synthesizedAligns;
 };
 
 } // end anonymous namespace
@@ -942,9 +956,6 @@ static bool relax(Ctx &ctx, int pass, InputSection &sec) {
 // relaxation pass.
 bool RISCV::relaxOnce(int pass) const {
   llvm::TimeTraceScope timeScope("RISC-V relaxOnce");
-  if (ctx.arg.relocatable)
-    return false;
-
   if (pass == 0)
     initSymbolAnchors(ctx);
 
@@ -959,6 +970,117 @@ bool RISCV::relaxOnce(int pass) const {
   return changed;
 }
 
+// If the section alignment is >= 4, advance `dot` to insert NOPs and synthesize
+// an ALIGN relocation. Otherwise, return false to use default handling.
+template <class ELFT, class RelTy>
+bool RISCV::synthesizeAlignForInput(uint64_t &dot, InputSection *sec,
+                                    Relocs<RelTy> rels) {
+  if (!baseSec) {
+    // Record the first input section with RELAX relocations. We will synthesize
+    // ALIGN relocations here.
+    for (auto rel : rels) {
+      if (rel.getType(false) == R_RISCV_RELAX) {
+        baseSec = sec;
+        break;
+      }
+    }
+  } else if (sec->addralign >= 4) {
+    // If the alignment is >= 4 and the section does not start with an ALIGN
+    // relocation, synthesize one.
+    bool hasAlignRel = llvm::any_of(rels, [](const RelTy &rel) {
+      return rel.r_offset == 0 && rel.getType(false) == R_RISCV_ALIGN;
+    });
+    if (!hasAlignRel) {
+      synthesizedAligns.emplace_back(dot - baseSec->getVA(),
+                                     sec->addralign - 2);
+      dot += sec->addralign - 2;
+      return true;
+    }
+  }
+  return false;
+}
+
+// Finalize the relocation section by appending synthesized ALIGN relocations
+// after processing all input sections.
+template <class ELFT, class RelTy>
+void RISCV::finalizeSynthesizeAligns(uint64_t &dot, InputSection *sec,
+                                     Relocs<RelTy> rels) {
+  auto *f = cast<ObjFile<ELFT>>(baseSec->file);
+  auto shdr = f->template getELFShdrs<ELFT>()[baseSec->relSecIdx];
+  // Create a copy of InputSection.
+  sec = make<InputSection>(*f, shdr, baseSec->name);
+  auto *baseRelSec = cast<InputSection>(f->getSections()[baseSec->relSecIdx]);
+  *sec = *baseRelSec;
+  baseSec = nullptr;
+
+  // Allocate buffer for original and synthesized relocations in RELA format.
+  // If CREL is used, OutputSection::finalizeNonAllocCrel will convert RELA to
+  // CREL.
+  auto newSize = rels.size() + synthesizedAligns.size();
+  auto *relas = makeThreadLocalN<typename ELFT::Rela>(newSize);
+  sec->size = newSize * sizeof(typename ELFT::Rela);
+  sec->content_ = reinterpret_cast<uint8_t *>(relas);
+  sec->type = SHT_RELA;
+  // Copy original relocations to the new buffer, potentially converting CREL to
+  // RELA.
+  for (auto [i, r] : llvm::enumerate(rels)) {
+    relas[i].r_offset = r.r_offset;
+    relas[i].setSymbolAndType(r.getSymbol(0), r.getType(0), false);
+    if constexpr (RelTy::HasAddend)
+      relas[i].r_addend = r.r_addend;
+  }
+  // Append synthesized ALIGN relocations to the buffer.
+  for (auto [i, r] : llvm::enumerate(synthesizedAligns)) {
+    auto &rela = relas[rels.size() + i];
+    rela.r_offset = r.first;
+    rela.setSymbolAndType(0, R_RISCV_ALIGN, false);
+    rela.r_addend = r.second;
+  }
+  synthesizedAligns.clear();
+  // Replace the old relocation section with the new one in the output section.
+  // addOrphanSections ensures that the output relocation section is processed
+  // after osec.
+  for (SectionCommand *cmd : sec->getParent()->commands) {
+    auto *isd = dyn_cast<InputSectionDescription>(cmd);
+    if (!isd)
+      continue;
+    for (auto *&isec : isd->sections)
+      if (isec == baseRelSec)
+        isec = sec;
+  }
+}
+
+template <class ELFT>
+bool RISCV::synthesizeAlignAux(uint64_t &dot, InputSection *sec) {
+  bool ret = false;
+  if (sec) {
+    invokeOnRelocs(*sec, ret = synthesizeAlignForInput<ELFT>, dot, sec);
+  } else if (baseSec) {
+    invokeOnRelocs(*baseSec, finalizeSynthesizeAligns<ELFT>, dot, sec);
+  }
+  return ret;
+}
+
+// Without linker relaxation enabled for a particular relocatable file or
+// section, the assembler will not generate R_RISCV_ALIGN relocations for
+// alignment directives. This becomes problematic in a two-stage linking
+// process: ld -r a.o b.o -o ab.o; ld ab.o -o ab. This function synthesizes an
+// R_RISCV_ALIGN relocation at section start when needed.
+//
+// When called with an input section (`sec` is not null): If the section
+// alignment is >= 4, advance `dot` to insert NOPs and synthesize an ALIGN
+// relocation.
+//
+// When called after all input sections are processed (`sec` is null): The
+// output relocation section is updated with all the newly synthesized ALIGN
+// relocations.
+bool RISCV::synthesizeAlign(uint64_t &dot, InputSection *sec) {
+  assert(ctx.arg.relocatable);
+  if (ctx.arg.is64)
+    return synthesizeAlignAux<ELF64LE>(dot, sec);
+  return synthesizeAlignAux<ELF32LE>(dot, sec);
+}
+
 void RISCV::finalizeRelax(int passes) const {
   llvm::TimeTraceScope timeScope("Finalize RISC-V relaxation");
   Log(ctx) << "relaxation passes: " << passes;
diff --git a/lld/ELF/BPSectionOrderer.cpp b/lld/ELF/BPSectionOrderer.cpp
index 06152046d13d4..38f15b93050c9 100644
--- a/lld/ELF/BPSectionOrderer.cpp
+++ b/lld/ELF/BPSectionOrderer.cpp
@@ -76,10 +76,12 @@ DenseMap<const InputSectionBase *, int> elf::runBalancedPartitioning(
     if (!d)
       return;
     auto *sec = dyn_cast_or_null<InputSection>(d->section);
-    // Skip empty, discarded, ICF folded sections, .bss. Skipping ICF folded
-    // sections reduces duplicate detection work in BPSectionOrderer.
-    if (!sec || sec->size == 0 || !sec->isLive() || sec->repl != sec ||
-        !sec->content().data() || !orderer.secToSym.try_emplace(sec, d).second)
+    // Skip section symbols. Skip empty, discarded, ICF folded sections, .bss.
+    // Skipping ICF folded sections reduces duplicate detection work in
+    // BPSectionOrderer.
+    if (sym.isSection() || !sec || sec->size == 0 || !sec->isLive() ||
+        sec->repl != sec || !sec->content().data() ||
+        !orderer.secToSym.try_emplace(sec, d).second)
       return;
     rootSymbolToSectionIdxs[CachedHashStringRef(
                                 lld::utils::getRootSymbol(sym.getName()))]
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index a5d08f4979dab..921128dae2bdb 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -1230,6 +1230,9 @@ bool LinkerScript::assignOffsets(OutputSection *sec) {
   if (sec->firstInOverlay)
     state->overlaySize = 0;
 
+  bool synthesizeAlign =
+      ctx.arg.relocatable && ctx.arg.relax && (sec->flags & SHF_EXECINSTR) &&
+      (ctx.arg.emachine == EM_LOONGARCH || ctx.arg.emachine == EM_RISCV);
   // We visited SectionsCommands from processSectionCommands to
   // layout sections. Now, we visit SectionsCommands again to fix
   // section offsets.
@@ -1260,7 +1263,10 @@ bool LinkerScript::assignOffsets(OutputSection *sec) {
       if (isa<PotentialSpillSection>(isec))
         continue;
       const uint64_t pos = dot;
-      dot = alignToPowerOf2(dot, isec->addralign);
+      // If synthesized ALIGN may be needed, call maybeSynthesizeAlign and
+      // disable the default handling if the return value is true.
+      if (!(synthesizeAlign && ctx.target->synthesizeAlign(dot, isec)))
+        dot = alignToPowerOf2(dot, isec->addralign);
       isec->outSecOff = dot - sec->addr;
       dot += isec->getSize();
 
@@ -1276,6 +1282,12 @@ bool LinkerScript::assignOffsets(OutputSection *sec) {
   if (ctx.in.relroPadding && sec == ctx.in.relroPadding->getParent())
     expandOutputSection(alignToPowerOf2(dot, ctx.arg.commonPageSize) - dot);
 
+  if (synthesizeAlign) {
+    const uint64_t pos = dot;
+    ctx.target->synthesizeAlign(dot, nullptr);
+    expandOutputSection(dot - pos);
+  }
+
   // Non-SHF_ALLOC sections do not affect the addresses of other OutputSections
   // as they are not part of the process image.
   if (!(sec->flags & SHF_ALLOC)) {
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 1020dd9f2569e..c9c63b3620b15 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -889,9 +889,19 @@ void OutputSection::sortInitFini() {
 std::array<uint8_t, 4> OutputSection::getFiller(Ctx &ctx) {
   if (filler)
     return *filler;
-  if (flags & SHF_EXECINSTR)
-    return ctx.target->trapInstr;
-  return {0, 0, 0, 0};
+  if (!(flags & SHF_EXECINSTR))
+    return {0, 0, 0, 0};
+  if (ctx.arg.relocatable && ctx.arg.emachine == EM_RISCV) {
+    // See RISCV::maybeSynthesizeAlign: Synthesized NOP bytes and ALIGN
+    // relocations might be needed between two input sections. Use a NOP for the
+    // filler.
+    if (ctx.arg.eflags & EF_RISCV_RVC)
+      return {1, 0, 1, 0};
+    return {0x13, 0, 0, 0};
+  }
+  if (ctx.arg.relocatable && ctx.arg.emachine == EM_LOONGARCH)
+    return {0, 0, 0x40, 0x03};
+  return ctx.target->trapInstr;
 }
 
 void OutputSection::checkDynRelAddends(Ctx &ctx) {
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 666863b700caa..6f55bac2ecf16 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -176,7 +176,7 @@ static RelType getMipsPairType(RelType type, bool isLocal) {
 
 // True if non-preemptable symbol always has the same value regardless of where
 // the DSO is loaded.
-static bool isAbsolute(const Symbol &sym) {
+bool elf::isAbsolute(const Symbol &sym) {
   if (sym.isUndefined())
     return true;
   if (const auto *dr = dyn_cast<Defined>(&sym))
@@ -273,12 +273,12 @@ template <class ELFT> static bool isReadOnly(SharedSymbol &ss) {
 // them are copied by a copy relocation, all of them need to be copied.
 // Otherwise, they would refer to different places at runtime.
 template <class ELFT>
-static SmallSet<SharedSymbol *, 4> getSymbolsAt(Ctx &ctx, SharedSymbol &ss) {
+static SmallPtrSet<SharedSymbol *, 4> getSymbolsAt(Ctx &ctx, SharedSymbol &ss) {
   using Elf_Sym = typename ELFT::Sym;
 
   const auto &file = cast<SharedFile>(*ss.file);
 
-  SmallSet<SharedSymbol *, 4> ret;
+  SmallPtrSet<SharedSymbol *, 4> ret;
   for (const Elf_Sym &s : file.template getGlobalELFSyms<ELFT>()) {
     if (s.st_shndx == SHN_UNDEF || s.st_shndx == SHN_ABS ||
         s.getType() == STT_TLS || s.st_value != ss.value)
diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h
index c1c4860ceaa92..7ea03c359bbf4 100644
--- a/lld/ELF/Relocations.h
+++ b/lld/ELF/Relocations.h
@@ -165,6 +165,8 @@ void addGotEntry(Ctx &ctx, Symbol &sym);
 void hexagonTLSSymbolUpdate(Ctx &ctx);
 bool hexagonNeedsTLSSymbol(ArrayRef<OutputSection *> outputSections);
 
+bool isAbsolute(const Symbol &sym);
+
 class ThunkSection;
 class Thunk;
 class InputSectionDescription;
diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h
index fdc0c20f9cd02..e121fa1183e9e 100644
--- a/lld/ELF/Target.h
+++ b/lld/ELF/Target.h
@@ -96,6 +96,9 @@ class TargetInfo {
 
   // Do a linker relaxation pass and return true if we changed something.
   virtual bool relaxOnce(int pass) const { return false; }
+  virtual bool synthesizeAlign(uint64_t &dot, InputSection *sec) {
+    return false;
+  }
   // Do finalize relaxation after collecting relaxation infos.
   virtual void finalizeRelax(int passes) const {}
 
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 2b0e097766d2c..4fa80397cbfa7 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -1541,8 +1541,10 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
   if (ctx.arg.randomizeSectionPadding)
     randomizeSectionPadding(ctx);
 
+  // Iterate until a fixed point is reached, skipping relocatable links since
+  // the final addresses are unavailable.
   uint32_t pass = 0, assignPasses = 0;
-  for (;;) {
+  while (!ctx.arg.relocatable) {
     bool changed = ctx.target->needsThunks
                        ? tc.createThunks(pass, ctx.outputSections)
                        : ctx.target->relaxOnce(pass);
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 9eb391c4ee1b9..f11c65a5583ad 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -1635,27 +1635,21 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
 
   config->osoPrefix = args.getLastArgValue(OPT_oso_prefix);
   if (!config->osoPrefix.empty()) {
-    // Expand special characters, such as ".", "..", or  "~", if present.
-    // Note: LD64 only expands "." and not other special characters.
-    // That seems silly to imitate so we will not try to follow it, but rather
-    // just use real_path() to do it.
-
     // The max path length is 4096, in theory. However that seems quite long
     // and seems unlikely that any one would want to strip everything from the
     // path. Hence we've picked a reasonably large number here.
     SmallString<1024> expanded;
-    if (!fs::real_path(config->osoPrefix, expanded,
-                       /*expand_tilde=*/true)) {
-      // Note: LD64 expands "." to be `<current_dir>/`
-      // (ie., it has a slash suffix) whereas real_path() doesn't.
-      // So we have to append '/' to be consistent.
-      StringRef sep = sys::path::get_separator();
-      // real_path removes trailing slashes as part of the normalization, but
-      // these are meaningful for our text based stripping
-      if (config->osoPrefix == "." || config->osoPrefix.ends_with(sep))
-        expanded += sep;
-      config->osoPrefix = saver().save(expanded.str());
+    // Expand "." into the current working directory.
+    if (config->osoPrefix == "." && !fs::current_path(expanded)) {
+      // Note: LD64 expands "." to be `<current_dir>/
+      // (ie., it has a slash suffix) whereas current_path() doesn't.
+      // So we have to append '/' to be consistent because this is
+      // meaningful for our text based stripping.
+      expanded += sys::path::get_separator();
+    } else {
+      expanded = config->osoPrefix;
     }
+    config->osoPrefix = saver().save(expanded.str());
   }
 
   bool pie = args.hasFlag(OPT_pie, OPT_no_pie, true);
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 6f60efd87c975..cc1628f48bb0e 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -1,3 +1,6 @@
+.. If you want to modify sections/contents permanently, you should modify both
+   ReleaseNotes.rst and ReleaseNotesTemplate.txt.
+
 ===========================
 lld |release| Release Notes
 ===========================
diff --git a/lld/docs/ReleaseNotesTemplate.txt b/lld/docs/ReleaseNotesTemplate.txt
new file mode 100644
index 0000000000000..cc1628f48bb0e
--- /dev/null
+++ b/lld/docs/ReleaseNotesTemplate.txt
@@ -0,0 +1,48 @@
+.. If you want to modify sections/contents permanently, you should modify both
+   ReleaseNotes.rst and ReleaseNotesTemplate.txt.
+
+===========================
+lld |release| Release Notes
+===========================
+
+.. contents::
+    :local:
+
+.. only:: PreRelease
+
+  .. warning::
+     These are in-progress notes for the upcoming LLVM |release| release.
+     Release notes for previous releases can be found on
+     `the Download Page <https://releases.llvm.org/download.html>`_.
+
+Introduction
+============
+
+This document contains the release notes for the lld linker, release |release|.
+Here we describe the status of lld, including major improvements
+from the previous release. All lld releases may be downloaded
+from the `LLVM releases web site <https://llvm.org/releases/>`_.
+
+Non-comprehensive list of changes in this release
+=================================================
+
+ELF Improvements
+----------------
+
+Breaking changes
+----------------
+
+COFF Improvements
+-----------------
+
+MinGW Improvements
+------------------
+
+MachO Improvements
+------------------
+
+WebAssembly Improvements
+------------------------
+
+Fixes
+#####
diff --git a/lld/test/COFF/alternatename-lto.ll b/lld/test/COFF/alternatename-lto.ll
new file mode 100644
index 0000000000000..c3666cd3501df
--- /dev/null
+++ b/lld/test/COFF/alternatename-lto.ll
@@ -0,0 +1,25 @@
+; REQUIRES: x86
+; RUN: mkdir -p %t.dir
+; RUN: llvm-as -o %t.obj %s
+; RUN: lld-link -out:%t.dll -dll -noentry %t.obj -export:test
+
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-windows-msvc19.33.0"
+
+$alt = comdat any
+
+@alt = weak_odr dso_local global i32 0, comdat, align 4
+@ext = external dso_local global i32, align 4
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @test() #0 {
+entry:
+  %0 = load i32, ptr @ext, align 4
+  ret i32 %0
+}
+
+attributes #0 = { noinline nounwind optnone uwtable }
+
+!llvm.linker.options = !{!0}
+
+!0 = !{!"/alternatename:ext=alt"}
diff --git a/lld/test/ELF/aarch64-patchinst.s b/lld/test/ELF/aarch64-patchinst.s
new file mode 100644
index 0000000000000..f91e4f9b209c3
--- /dev/null
+++ b/lld/test/ELF/aarch64-patchinst.s
@@ -0,0 +1,88 @@
+# REQUIRES: aarch64
+# RUN: rm -rf %t && split-file %s %t
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %t/use.s -o %t/use-le.o
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %t/def.s -o %t/def-le.o
+# RUN: llvm-mc -filetype=obj -triple=aarch64 %t/rel.s -o %t/rel-le.o
+
+## Deactivation symbol used without being defined: instruction emitted as usual.
+# RUN: ld.lld -o %t/undef-le %t/use-le.o --emit-relocs
+# RUN: llvm-objdump -r %t/undef-le | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/undef-le | FileCheck --check-prefix=UNDEF %s
+# RUN: ld.lld -pie -o %t/undef-le %t/use-le.o --emit-relocs
+# RUN: llvm-objdump -r %t/undef-le | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/undef-le | FileCheck --check-prefix=UNDEF %s
+
+## Deactivation symbol defined: instructions overwritten with NOPs.
+# RUN: ld.lld -o %t/def-le %t/use-le.o %t/def-le.o --emit-relocs
+# RUN: llvm-objdump -r %t/def-le | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/def-le | FileCheck --check-prefix=DEF %s
+# RUN: ld.lld -pie -o %t/def-le %t/use-le.o %t/def-le.o --emit-relocs
+# RUN: llvm-objdump -r %t/def-le | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/def-le | FileCheck --check-prefix=DEF %s
+
+## Relocation pointing to a non-SHN_UNDEF non-SHN_ABS symbol is an error.
+# RUN: not ld.lld -o %t/rel-le %t/use-le.o %t/rel-le.o 2>&1 | FileCheck --check-prefix=ERROR %s
+# RUN: not ld.lld -pie -o %t/rel-le %t/use-le.o %t/rel-le.o 2>&1 | FileCheck --check-prefix=ERROR %s
+
+## Behavior unchanged by endianness: relocation always written as little endian.
+# RUN: llvm-mc -filetype=obj -triple=aarch64_be %t/use.s -o %t/use-be.o
+# RUN: llvm-mc -filetype=obj -triple=aarch64_be %t/def.s -o %t/def-be.o
+# RUN: llvm-mc -filetype=obj -triple=aarch64_be %t/rel.s -o %t/rel-be.o
+# RUN: ld.lld -o %t/undef-be %t/use-be.o --emit-relocs
+# RUN: llvm-objdump -r %t/undef-be | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/undef-be | FileCheck --check-prefix=UNDEF %s
+# RUN: ld.lld -pie -o %t/undef-be %t/use-be.o --emit-relocs
+# RUN: llvm-objdump -r %t/undef-be | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/undef-be | FileCheck --check-prefix=UNDEF %s
+# RUN: ld.lld -o %t/def-be %t/use-be.o %t/def-be.o --emit-relocs
+# RUN: llvm-objdump -r %t/def-be | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/def-be | FileCheck --check-prefix=DEF %s
+# RUN: ld.lld -pie -o %t/def-be %t/use-be.o %t/def-be.o --emit-relocs
+# RUN: llvm-objdump -r %t/def-be | FileCheck --check-prefix=RELOC %s
+# RUN: llvm-objdump -d %t/def-be | FileCheck --check-prefix=DEF %s
+# RUN: not ld.lld -o %t/rel-be %t/use-be.o %t/rel-be.o 2>&1 | FileCheck --check-prefix=ERROR %s
+# RUN: not ld.lld -pie -o %t/rel-be %t/use-be.o %t/rel-be.o 2>&1 | FileCheck --check-prefix=ERROR %s
+
+# RELOC:      R_AARCH64_JUMP26
+# RELOC-NEXT: R_AARCH64_PATCHINST ds
+# RELOC-NEXT: R_AARCH64_PATCHINST ds
+# RELOC-NEXT: R_AARCH64_PATCHINST ds0+0xd503201f
+
+#--- use.s
+.weak ds
+.weak ds0
+# This instruction has a single relocation: the DS relocation.
+# UNDEF: add x0, x1, x2
+# DEF: nop
+# ERROR: R_AARCH64_PATCHINST relocation against non-absolute symbol ds
+.reloc ., R_AARCH64_PATCHINST, ds
+add x0, x1, x2
+# This instruction has two relocations: the DS relocation and the JUMP26 to f1.
+# Make sure that the DS relocation takes precedence.
+.reloc ., R_AARCH64_PATCHINST, ds
+# UNDEF: b {{.*}} <f1>
+# DEF: nop
+# ERROR: R_AARCH64_PATCHINST relocation against non-absolute symbol ds
+b f1
+# Alternative representation: instruction opcode stored in addend.
+# UNDEF: add x3, x4, x5
+# DEF: nop
+# ERROR: R_AARCH64_PATCHINST relocation against non-absolute symbol ds0
+.reloc ., R_AARCH64_PATCHINST, ds0 + 0xd503201f
+add x3, x4, x5
+
+.section .text.f1,"ax",@progbits
+f1:
+ret
+
+#--- def.s
+.globl ds
+ds = 0xd503201f
+.globl ds0
+ds0 = 0
+
+#--- rel.s
+.globl ds
+ds:
+.globl ds0
+ds0:
diff --git a/lld/test/ELF/bp-section-orderer.s b/lld/test/ELF/bp-section-orderer.s
index 438d7c2da0f76..9a83c70dba315 100644
--- a/lld/test/ELF/bp-section-orderer.s
+++ b/lld/test/ELF/bp-section-orderer.s
@@ -21,38 +21,38 @@
 # RUN: llvm-profdata merge a.proftext -o a.profdata
 # RUN: ld.lld a.o --irpgo-profile=a.profdata --bp-startup-sort=function --verbose-bp-section-orderer --icf=all --gc-sections 2>&1 | FileCheck %s --check-prefix=STARTUP-FUNC-ORDER
 
-# STARTUP-FUNC-ORDER: Ordered 3 sections ([[#]] bytes) using balanced partitioning
-# STARTUP-FUNC-ORDER: Total area under the page fault curve: 3.
+# STARTUP-FUNC-ORDER: Ordered 4 sections ([[#]] bytes) using balanced partitioning
+# STARTUP-FUNC-ORDER: Total area under the page fault curve: 4.
 
 # RUN: ld.lld -o out.s a.o --irpgo-profile=a.profdata --bp-startup-sort=function
 # RUN: llvm-nm -jn out.s | tr '\n' , | FileCheck %s --check-prefix=STARTUP
-# STARTUP: s5,s4,s3,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d3,d2,d1,g1,{{$}}
+# STARTUP: s5,s4,s3,s2,s1,A,B,C,L1,F,E,D,merged1,merged2,G,_start,d4,d3,d2,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.os a.o --irpgo-profile=a.profdata --bp-startup-sort=function --symbol-ordering-file a.txt
 # RUN: llvm-nm -jn out.os | tr '\n' , | FileCheck %s --check-prefix=ORDER-STARTUP
-# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,merged1,merged2,_start,d3,d2,d4,d1,g1,{{$}}
+# ORDER-STARTUP: s2,s1,s5,s4,s3,A,F,E,D,B,C,L1,merged1,merged2,G,_start,d3,d2,d4,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.cf a.o --verbose-bp-section-orderer --bp-compression-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-FUNC
 # RUN: ld.lld -o out.cf.icf a.o --verbose-bp-section-orderer --bp-compression-sort=function --icf=all --gc-sections 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-ICF-FUNC
 # RUN: llvm-nm -jn out.cf | tr '\n' , | FileCheck %s --check-prefix=CFUNC
-# CFUNC: s5,s4,s3,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d3,d2,d1,g1,{{$}}
+# CFUNC: s5,s4,s3,s2,s1,C,E,D,B,G,F,merged1,merged2,A,_start,L1,d4,d3,d2,d1,g1,{{$}}
 
 # RUN: ld.lld -o out.cd a.o --verbose-bp-section-orderer --bp-compression-sort=data 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-DATA
 # RUN: llvm-nm -jn out.cd | tr '\n' , | FileCheck %s --check-prefix=CDATA
-# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
+# CDATA: s5,s3,s4,s2,s1,F,C,E,D,B,A,merged1,merged2,L1,G,_start,d4,d1,d3,d2,g1,{{$}}
 
 # RUN: ld.lld -o out.cb a.o --verbose-bp-section-orderer --bp-compression-sort=both 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
 # RUN: llvm-nm -jn out.cb | tr '\n' , | FileCheck %s --check-prefix=CBOTH
-# CBOTH: s5,s3,s4,s2,s1,A,F,merged1,merged2,C,E,D,B,_start,d4,d1,d3,d2,g1,{{$}}
+# CBOTH: s5,s3,s4,s2,s1,C,E,D,B,G,F,merged1,merged2,A,_start,L1,d4,d1,d3,d2,g1,{{$}}
 
 # RUN: ld.lld -o out.cbs a.o --verbose-bp-section-orderer --bp-compression-sort=both --irpgo-profile=a.profdata --bp-startup-sort=function 2>&1 | FileCheck %s --check-prefix=BP-COMPRESSION-BOTH
 # RUN: llvm-nm -jn out.cbs | tr '\n' , | FileCheck %s --check-prefix=CBOTH-STARTUP
-# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,F,E,D,merged1,merged2,_start,d4,d1,d3,d2,g1,{{$}}
+# CBOTH-STARTUP: s5,s3,s4,s2,s1,A,B,C,L1,F,E,D,merged1,merged2,G,_start,d4,d1,d3,d2,g1,{{$}}
 
-# BP-COMPRESSION-FUNC: Ordered 9 sections ([[#]] bytes) using balanced partitioning
-# BP-COMPRESSION-ICF-FUNC: Ordered 8 sections ([[#]] bytes) using balanced partitioning
+# BP-COMPRESSION-FUNC: Ordered 11 sections ([[#]] bytes) using balanced partitioning
+# BP-COMPRESSION-ICF-FUNC: Ordered 9 sections ([[#]] bytes) using balanced partitioning
 # BP-COMPRESSION-DATA: Ordered 9 sections ([[#]] bytes) using balanced partitioning
-# BP-COMPRESSION-BOTH: Ordered 18 sections ([[#]] bytes) using balanced partitioning
+# BP-COMPRESSION-BOTH: Ordered 20 sections ([[#]] bytes) using balanced partitioning
 
 #--- a.proftext
 :ir
@@ -63,7 +63,7 @@
 1
 # Weight
 1
-A, B, C
+A, B, C, L1
 
 A
 # Func Hash:
@@ -97,6 +97,14 @@ D
 # Counter Values:
 1
 
+L1
+# Func Hash:
+5555
+# Num Counters:
+1
+# Counter Values:
+1
+
 #--- a.txt
 A
 F
@@ -137,6 +145,9 @@ void A() {}
 RETAIN int merged1(int a) { return F(a + 101); }
 int merged2(int a) { return F(a + 101); }
 
+RETAIN static int L1(int a) { return a + 103; }
+int G(int a) { return L1(a); }
+
 int _start() { return 0; }
 
 #--- gen
@@ -148,7 +159,7 @@ clang --target=aarch64-linux-gnu -O0 -ffunction-sections -fdata-sections -fno-as
 	.p2align	2
 	.type	F,@function
 F:                                      // @F
-// %bb.0:                               // %entry
+// %bb.0:
 	sub	sp, sp, #32
 	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
 	add	x29, sp, #16
@@ -167,7 +178,7 @@ F:                                      // @F
 	.p2align	2
 	.type	C,@function
 C:                                      // @C
-// %bb.0:                               // %entry
+// %bb.0:
 	sub	sp, sp, #32
 	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
 	add	x29, sp, #16
@@ -186,7 +197,7 @@ C:                                      // @C
 	.p2align	2
 	.type	E,@function
 E:                                      // @E
-// %bb.0:                               // %entry
+// %bb.0:
 	sub	sp, sp, #32
 	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
 	add	x29, sp, #16
@@ -205,7 +216,7 @@ E:                                      // @E
 	.p2align	2
 	.type	D,@function
 D:                                      // @D
-// %bb.0:                               // %entry
+// %bb.0:
 	sub	sp, sp, #32
 	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
 	add	x29, sp, #16
@@ -224,7 +235,7 @@ D:                                      // @D
 	.p2align	2
 	.type	B,@function
 B:                                      // @B
-// %bb.0:                               // %entry
+// %bb.0:
 	sub	sp, sp, #32
 	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
 	add	x29, sp, #16
@@ -243,7 +254,7 @@ B:                                      // @B
 	.p2align	2
 	.type	A,@function
 A:                                      // @A
-// %bb.0:                               // %entry
+// %bb.0:
 	ret
 .Lfunc_end5:
 	.size	A, .Lfunc_end5-A
@@ -253,7 +264,7 @@ A:                                      // @A
 	.p2align	2
 	.type	merged1,@function
 merged1:                                // @merged1
-// %bb.0:                               // %entry
+// %bb.0:
 	sub	sp, sp, #32
 	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
 	add	x29, sp, #16
@@ -272,7 +283,7 @@ merged1:                                // @merged1
 	.p2align	2
 	.type	merged2,@function
 merged2:                                // @merged2
-// %bb.0:                               // %entry
+// %bb.0:
 	sub	sp, sp, #32
 	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
 	add	x29, sp, #16
@@ -286,16 +297,48 @@ merged2:                                // @merged2
 .Lfunc_end7:
 	.size	merged2, .Lfunc_end7-merged2
                                         // -- End function
+	.section	.text.L1,"axR",@progbits
+	.p2align	2                               // -- Begin function L1
+	.type	L1,@function
+L1:                                     // @L1
+// %bb.0:
+	sub	sp, sp, #16
+	str	w0, [sp, #12]
+	ldr	w8, [sp, #12]
+	add	w0, w8, #103
+	add	sp, sp, #16
+	ret
+.Lfunc_end8:
+	.size	L1, .Lfunc_end8-L1
+                                        // -- End function
+	.section	.text.G,"ax",@progbits
+	.globl	G                               // -- Begin function G
+	.p2align	2
+	.type	G,@function
+G:                                      // @G
+// %bb.0:
+	sub	sp, sp, #32
+	stp	x29, x30, [sp, #16]             // 16-byte Folded Spill
+	add	x29, sp, #16
+	stur	w0, [x29, #-4]
+	ldur	w0, [x29, #-4]
+	bl	L1
+	ldp	x29, x30, [sp, #16]             // 16-byte Folded Reload
+	add	sp, sp, #32
+	ret
+.Lfunc_end9:
+	.size	G, .Lfunc_end9-G
+                                        // -- End function
 	.section	.text._start,"ax",@progbits
 	.globl	_start                          // -- Begin function _start
 	.p2align	2
 	.type	_start,@function
 _start:                                 // @_start
-// %bb.0:                               // %entry
+// %bb.0:
 	mov	w0, wzr
 	ret
-.Lfunc_end8:
-	.size	_start, .Lfunc_end8-_start
+.Lfunc_end10:
+	.size	_start, .Lfunc_end10-_start
                                         // -- End function
 	.type	s5,@object                      // @s5
 	.section	.rodata.s5,"a",@progbits
@@ -395,3 +438,4 @@ g1:
 	.addrsig_sym B
 	.addrsig_sym A
 	.addrsig_sym merged1
+	.addrsig_sym L1
diff --git a/lld/test/ELF/loongarch-relocatable-align.s b/lld/test/ELF/loongarch-relocatable-align.s
new file mode 100644
index 0000000000000..747a58ed61eaa
--- /dev/null
+++ b/lld/test/ELF/loongarch-relocatable-align.s
@@ -0,0 +1,147 @@
+# REQUIRES: loongarch
+
+## Test LA64.
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=loongarch64 -mattr=+relax a.s -o a.o
+# RUN: llvm-mc -filetype=obj -triple=loongarch64 -mattr=+relax --defsym ELF64=1 b.s -o b.o
+# RUN: llvm-mc -filetype=obj -triple=loongarch64 -mattr=+relax b1.s -o b1.o
+# RUN: llvm-mc -filetype=obj -triple=loongarch64 -mattr=+relax c.s -o c.o
+# RUN: llvm-mc -filetype=obj -triple=loongarch64 d.s -o d.o
+
+## No RELAX. Don't synthesize ALIGN.
+# RUN: ld.lld -r b.o d.o -o bd.ro
+# RUN: llvm-readelf -r bd.ro | FileCheck %s --check-prefix=NOREL
+
+# NOREL: no relocations
+
+# RUN: ld.lld -r b.o b.o a.o b.o b1.o c.o d.o -o out.ro
+# RUN: llvm-objdump -dr --no-show-raw-insn out.ro | FileCheck %s
+# RUN: llvm-readelf -r out.ro | FileCheck %s --check-prefix=CHECK-REL
+
+# CHECK:      <b0>:
+# CHECK-NEXT:   0:    addi.d $a0, $a1, 1
+# CHECK-NEXT:   4:    nop
+# CHECK-EMPTY:
+# CHECK-NEXT: <b0>:
+# CHECK-NEXT:   8:    addi.d $a0, $a1, 1
+# CHECK-EMPTY:
+# CHECK-NEXT: <_start>:
+# CHECK-NEXT:   c:    pcalau12i $a0, 0
+# CHECK-NEXT:           000000000000000c:  R_LARCH_PCALA_HI20   .Ltext1_start
+# CHECK-NEXT:           000000000000000c:  R_LARCH_RELAX        *ABS*
+# CHECK-NEXT:   10:   addi.d $a0, $a0, 0
+# CHECK-NEXT:           0000000000000010:  R_LARCH_PCALA_LO12   .Ltext1_start
+# CHECK-NEXT:           0000000000000010:  R_LARCH_RELAX        *ABS*
+# CHECK-NEXT:   14:   nop
+# CHECK-NEXT:           0000000000000014:  R_LARCH_ALIGN        *ABS*+0x4
+# CHECK-EMPTY:
+# CHECK-NEXT: <b0>:
+# CHECK-NEXT:   18:   addi.d $a0, $a1, 1
+# CHECK-NEXT:   1c:   nop
+# CHECK-NEXT:   20:   nop
+# CHECK-NEXT:           0000000000000020:  R_LARCH_ALIGN        *ABS*+0x4
+# CHECK-NEXT:   24:   nop
+# CHECK-EMPTY:
+# CHECK-NEXT: <b1>:
+# CHECK-NEXT:   28:   addi.d $a0, $a1, 3
+# CHECK-EMPTY:
+# CHECK-NEXT: <c0>:
+# CHECK-NEXT:   2c:   addi.d $a0, $a1, 4
+# CHECK-NEXT:   30:   nop
+# CHECK-NEXT:           0000000000000030:  R_LARCH_ALIGN        *ABS*+0x4
+# CHECK-EMPTY:
+# CHECK-NEXT: <d0>:
+# CHECK-NEXT:   34:   addi.d $a0, $a1, 5
+
+# CHECK-REL:  Relocation section '.rela.text' at offset {{.*}} contains 7 entries:
+# CHECK-REL:  Relocation section '.rela.text1' at offset {{.*}} contains 5 entries:
+
+## Test LA32.
+# RUN: llvm-mc -filetype=obj -triple=loongarch32 -mattr=+relax a.s -o a.32.o
+# RUN: llvm-mc -filetype=obj -triple=loongarch32 -mattr=+relax b.s -o b.32.o
+# RUN: ld.lld -r a.32.o b.32.o -o out.32.ro
+# RUN: ld.lld -Ttext=0x10000 out.32.ro -o out32
+# RUN: llvm-objdump -dr --no-show-raw-insn out32 | FileCheck %s --check-prefix=CHECK32
+
+# CHECK32:      <_start>:
+# CHECK32-NEXT:   10000:    pcaddi $a0, 4
+# CHECK32-NEXT:   10004:    nop
+# CHECK32-EMPTY:
+# CHECK32-NEXT: <b0>:
+# CHECK32-NEXT:   10008:    addi.w $a0, $a1, 1
+# CHECK32:      <.Ltext1_start>:
+# CHECK32-NEXT:   10010:    pcaddi $a1, 0
+# CHECK32-NEXT:   10014:    nop
+# CHECK32-NEXT:   10018:    addi.w $a0, $a1, 2
+
+## Test CREL.
+# RUN: llvm-mc -filetype=obj -triple=loongarch64 -mattr=+relax --crel a.s -o acrel.o
+# RUN: ld.lld -r acrel.o b.o -o out.crel.ro
+# RUN: llvm-objdump -dr --no-show-raw-insn out.crel.ro | FileCheck %s --check-prefix=CHECKC
+
+# CHECKC:      <_start>:
+# CHECKC-NEXT:   0:    pcalau12i $a0, 0
+# CHECKC-NEXT:           0000000000000000:  R_LARCH_PCALA_HI20   .Ltext1_start
+# CHECKC-NEXT:           0000000000000000:  R_LARCH_RELAX        *ABS*
+# CHECKC-NEXT:   4:    addi.d $a0, $a0, 0
+# CHECKC-NEXT:           0000000000000004:  R_LARCH_PCALA_LO12   .Ltext1_start
+# CHECKC-NEXT:           0000000000000004:  R_LARCH_RELAX        *ABS*
+# CHECKC-NEXT:   8:    nop
+# CHECKC-NEXT:           0000000000000008:  R_LARCH_ALIGN        *ABS*+0x4
+# CHECKC-EMPTY:
+# CHECKC-NEXT: <b0>:
+# CHECKC-NEXT:   c:    addi.d $a0, $a1, 1
+
+#--- a.s
+.globl _start
+_start:
+  la.pcrel $a0, .Ltext1_start
+
+.section .text1,"ax"
+.Ltext1_start:
+  la.pcrel $a1, .Ltext1_start
+
+#--- b.s
+.macro addi dst, src1, src2
+.ifdef ELF64
+  addi.d \dst, \src1, \src2
+.else
+  addi.w \dst, \src1, \src2
+.endif
+.endm
+
+## Needs synthesized ALIGN.
+.option push
+.option norelax
+.balign 8
+b0:
+  addi $a0, $a1, 1
+
+.section .text1,"ax"
+.balign 8
+  addi $a0, $a1, 2
+
+.option pop
+
+#--- b1.s
+# Starts with an ALIGN relocation, don't need synthesized ALIGN.
+.option push
+.option norelax
+  .reloc ., R_LARCH_ALIGN, 4
+  nop
+.balign 8
+b1:
+  addi.d $a0, $a1, 3
+.option pop
+
+#--- c.s
+## Alignment == 4, don't need synthesized ALIGN.
+.balign 4
+c0:
+  addi.d $a0, $a1, 4
+
+#--- d.s
+## Needs synthesized ALIGN.
+.balign 8
+d0:
+  addi.d $a0, $a1, 5
diff --git a/lld/test/ELF/riscv-relax-align.s b/lld/test/ELF/riscv-relax-align.s
index 3efc774f76dd4..b3db2c943d85d 100644
--- a/lld/test/ELF/riscv-relax-align.s
+++ b/lld/test/ELF/riscv-relax-align.s
@@ -10,7 +10,7 @@
 # RUN: ld.lld -Ttext=0x10000 --no-relax 32.o -o 32.norelax
 # RUN: llvm-objdump -td --no-show-raw-insn -M no-aliases 32.norelax | FileCheck %s
 
-# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+relax %s -o 64.o
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+relax %s -riscv-align-rvc=0 -o 64.o
 # RUN: ld.lld -Ttext=0x10000 64.o -o 64
 # RUN: llvm-objdump -td --no-show-raw-insn -M no-aliases 64 | FileCheck %s
 # RUN: ld.lld -Ttext=0x10000 --no-relax 64.o -o 64.norelax
@@ -29,7 +29,7 @@
 # CHECK-DAG: 00010000 g       .text  {{0*}}38 _start
 
 # CHECK:       <_start>:
-# CHECK-NEXT:            addi    a0, a0, 0x1
+# CHECK-NEXT:            lui     a0, 0x10
 # CHECK-EMPTY:
 # CHECK-NEXT:  <a>:
 # CHECK-NEXT:            addi    a0, a0, 0x2
@@ -82,7 +82,9 @@
 # GC-NOT:       <d>:
 
 # CHECKR:       <_start>:
-# CHECKR-NEXT:          addi    a0, a0, 0x1
+# CHECKR-NEXT:          lui     a0, 0x0
+# CHECKR-NEXT:          0000000000000000:  R_RISCV_HI20         _start
+# CHECKR-NEXT:          0000000000000000:  R_RISCV_RELAX        *ABS*
 # CHECKR-EMPTY:
 # CHECKR-NEXT:  <a>:
 # CHECKR-NEXT:          addi    a0, a0, 0x2
@@ -116,7 +118,7 @@
 
 .global _start
 _start:
-  addi a0, a0, 0x1
+  lui a0, %hi(_start)
 a:
   addi a0, a0, 0x2
 b:
diff --git a/lld/test/ELF/riscv-relax-emit-relocs.s b/lld/test/ELF/riscv-relax-emit-relocs.s
index 26ea0d10d8c50..358a288777e93 100644
--- a/lld/test/ELF/riscv-relax-emit-relocs.s
+++ b/lld/test/ELF/riscv-relax-emit-relocs.s
@@ -3,11 +3,11 @@
 
 # RUN: rm -rf %t && mkdir %t && cd %t
 
-# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+relax %s -o 32.o
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+relax %s -o 32.o -riscv-align-rvc=0
 # RUN: ld.lld -Ttext=0x10000 --emit-relocs 32.o -o 32
 # RUN: llvm-objdump -dr --no-show-raw-insn -M no-aliases 32 | FileCheck %s
 
-# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+relax %s -o 64.o
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+relax %s -o 64.o -riscv-align-rvc=0
 # RUN: ld.lld -Ttext=0x10000 --emit-relocs 64.o -o 64
 # RUN: llvm-objdump -dr --no-show-raw-insn -M no-aliases 64 | FileCheck %s
 
diff --git a/lld/test/ELF/riscv-relocatable-align.s b/lld/test/ELF/riscv-relocatable-align.s
new file mode 100644
index 0000000000000..9cd59e96a7196
--- /dev/null
+++ b/lld/test/ELF/riscv-relocatable-align.s
@@ -0,0 +1,140 @@
+# REQUIRES: riscv
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c,+relax a.s -o ac.o
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c,+relax b.s -o bc.o
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c,+relax b1.s -o b1c.o
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c,+relax c.s -o cc.o
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c d.s -o dc.o
+
+## No RELAX. Don't synthesize ALIGN.
+# RUN: ld.lld -r bc.o dc.o -o bd.ro
+
+# NOREL: no relocations
+
+# RUN: ld.lld -r bc.o bc.o ac.o bc.o b1c.o cc.o dc.o -o out.ro
+# RUN: llvm-objdump -dr -M no-aliases out.ro | FileCheck %s
+# RUN: llvm-readelf -r out.ro | FileCheck %s --check-prefix=CHECK-REL
+
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+relax a.s -o a.o
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+relax b.s -o b.o
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+relax d.s -o d.o
+# RUN: ld.lld -r a.o b.o d.o -o out0.ro
+# RUN: ld.lld -Ttext=0x10000 out0.ro -o out0
+# RUN: llvm-objdump -dr -M no-aliases out0 | FileCheck %s --check-prefix=CHECK1
+
+# CHECK:      <b0>:
+# CHECK-NEXT:   0: 00158513             addi    a0, a1, 0x1
+# CHECK-NEXT:   4: 0001                 c.nop
+# CHECK-NEXT:   6: 0001                 c.nop
+# CHECK-EMPTY:
+# CHECK-NEXT: <b0>:
+# CHECK-NEXT:   8: 00158513             addi    a0, a1, 0x1
+# CHECK-EMPTY:
+# CHECK-NEXT: <_start>:
+# CHECK-NEXT:   c: 00000097             auipc   ra, 0x0
+# CHECK-NEXT:           000000000000000c:  R_RISCV_CALL_PLT     foo
+# CHECK-NEXT:           000000000000000c:  R_RISCV_RELAX        *ABS*
+# CHECK-NEXT:  10: 000080e7             jalr    ra, 0x0(ra) <_start>
+# CHECK-NEXT:  14: 0001                 c.nop
+# CHECK-NEXT:           0000000000000014:  R_RISCV_ALIGN        *ABS*+0x6
+# CHECK-NEXT:  16: 0001                 c.nop
+# CHECK-NEXT:  18: 0001                 c.nop
+# CHECK-EMPTY:
+# CHECK-NEXT: <b0>:
+# CHECK-NEXT:  1a: 00158513             addi    a0, a1, 0x1
+# CHECK-NEXT:  1e: 0001                 c.nop
+# CHECK-NEXT:  20: 0001                 c.nop
+# CHECK-NEXT:           0000000000000020:  R_RISCV_ALIGN        *ABS*+0x6
+# CHECK-NEXT:  22: 0001                 c.nop
+# CHECK-NEXT:  24: 00000013             addi    zero, zero, 0x0
+# CHECK-EMPTY:
+# CHECK-NEXT: <b0>:
+# CHECK-NEXT:  28: 00158513             addi    a0, a1, 0x1
+# CHECK-EMPTY:
+# CHECK-NEXT: <c0>:
+# CHECK-NEXT:  2c: 00000097             auipc   ra, 0x0
+# CHECK-NEXT:           000000000000002c:  R_RISCV_CALL_PLT     foo
+# CHECK-NEXT:           000000000000002c:  R_RISCV_RELAX        *ABS*
+# CHECK-NEXT:  30: 000080e7             jalr    ra, 0x0(ra) <c0>
+# CHECK-NEXT:  34: 0001                 c.nop
+# CHECK-NEXT:           0000000000000034:  R_RISCV_ALIGN        *ABS*+0x2
+# CHECK-EMPTY:
+# CHECK-NEXT: <d0>:
+# CHECK-NEXT:  36: 00258513             addi    a0, a1, 0x2
+
+# CHECK-REL:  Relocation section '.rela.text' at offset {{.*}} contains 7 entries:
+# CHECK-REL:  Relocation section '.rela.text1' at offset {{.*}} contains 3 entries:
+
+# CHECK1:      <_start>:
+# CHECK1-NEXT:    010000ef      jal     ra, 0x10010 <foo>
+# CHECK1-NEXT:    00000013      addi zero, zero, 0x0
+# CHECK1-EMPTY:
+# CHECK1-NEXT: <b0>:
+# CHECK1-NEXT:    00158513      addi    a0, a1, 0x1
+# CHECK1-EMPTY:
+# CHECK1-NEXT: <d0>:
+# CHECK1-NEXT:    00258513      addi    a0, a1, 0x2
+
+## Test CREL.
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c,+relax --crel a.s -o acrel.o
+# RUN: ld.lld -r acrel.o bc.o -o out1.ro
+# RUN: llvm-objdump -dr -M no-aliases out1.ro | FileCheck %s --check-prefix=CHECK2
+
+# CHECK2:      <_start>:
+# CHECK2-NEXT:   0: 00000097             auipc   ra, 0x0
+# CHECK2-NEXT:           0000000000000000:  R_RISCV_CALL_PLT     foo
+# CHECK2-NEXT:           0000000000000000:  R_RISCV_RELAX        *ABS*
+# CHECK2-NEXT:   4: 000080e7             jalr    ra, 0x0(ra) <_start>
+# CHECK2-NEXT:   8: 0001                 c.nop
+# CHECK2-NEXT:           0000000000000008:  R_RISCV_ALIGN        *ABS*+0x6
+# CHECK2-NEXT:   a: 0001                 c.nop
+# CHECK2-NEXT:   c: 0001                 c.nop
+# CHECK2-EMPTY:
+# CHECK2-NEXT: <b0>:
+# CHECK2-NEXT:   e: 00158513             addi    a0, a1, 0x1
+
+#--- a.s
+.globl _start
+_start:
+  call foo
+
+.section .text1,"ax"
+.globl foo
+foo:
+  call foo
+
+#--- b.s
+## Needs synthesized ALIGN
+.option push
+.option norelax
+.balign 8
+b0:
+  addi a0, a1, 1
+
+.section .text1,"ax"
+.balign 8
+  addi a0, a1, 1
+
+.option pop
+
+#--- b1.s
+.option push
+.option norelax
+  .reloc ., R_RISCV_ALIGN, 6
+  addi x0, x0, 0
+  c.nop
+.balign 8
+b0:
+  addi a0, a1, 1
+.option pop
+
+#--- c.s
+.balign 2
+c0:
+  call foo
+
+#--- d.s
+## Needs synthesized ALIGN
+.balign 4
+d0:
+  addi a0, a1, 2
diff --git a/lld/test/MachO/stabs.s b/lld/test/MachO/stabs.s
index 968656dba5cba..e32b9fc5b50d6 100644
--- a/lld/test/MachO/stabs.s
+++ b/lld/test/MachO/stabs.s
@@ -63,9 +63,18 @@
 # RUN: dsymutil -s  %t/test-rel | grep 'N_OSO' | FileCheck %s  -D#TEST_TIME=0x10 -D#FOO_TIME=0x20 --check-prefix=REL-PATH-NO-SLASH
 # RUN: cd %t && ZERO_AR_DATE=0 %lld -lSystem test.o foo.o no-debug.o -oso_prefix "." -o %t/test-rel-dot
 # RUN: dsymutil -s  %t/test-rel-dot | grep 'N_OSO' | FileCheck %s  -D#TEST_TIME=0x10 -D#FOO_TIME=0x20 --check-prefix=REL-DOT
-## Set HOME to %t (for ~ to expand to)
-# RUN: cd %t && env HOME=%t ZERO_AR_DATE=0 %lld -lSystem test.o foo.o no-debug.o -oso_prefix "~" -o %t/test-rel-tilde
-# RUN: dsymutil -s  %t/test-rel-tilde | grep 'N_OSO' | FileCheck %s  -D#TEST_TIME=0x10 -D#FOO_TIME=0x20 --check-prefix=REL-PATH
+# RUN: cd %t && ZERO_AR_DATE=0 %lld -lSystem ./test.o ./foo.o ./no-debug.o -oso_prefix "." -o %t/test-rel-dot
+# RUN: dsymutil -s  %t/test-rel-dot | grep 'N_OSO' | FileCheck %s  -D#TEST_TIME=0x10 -D#FOO_TIME=0x20 --check-prefix=REL-DOT-EXPLICIT
+
+## Check that symlinks are not expanded when -oso_prefix . is used.
+# RUN: mkdir -p %t/private/var/folders/tmp && ln -s private/var %t/var
+# RUN: cp %t/test.o %t/foo.o %t/no-debug.o %t/private/var/folders/tmp
+# RUN: env TZ=GMT touch -t "197001010000.16" %t/private/var/folders/tmp/test.o
+# RUN: env TZ=GMT touch -t "197001010000.32" %t/private/var/folders/tmp/foo.o
+# RUN: cd %t/var/folders/tmp && ZERO_AR_DATE=0 %lld -lSystem test.o foo.o no-debug.o -oso_prefix "." -o test-rel-symlink
+# RUN: dsymutil -s  %t/private/var/folders/tmp/test-rel-symlink | grep 'N_OSO' | FileCheck %s  -D#TEST_TIME=0x10 -D#FOO_TIME=0x20 --check-prefix=REL-DOT
+# RUN: cd %t/var/folders/tmp && ZERO_AR_DATE=0 %lld -lSystem ./test.o ./foo.o ./no-debug.o -oso_prefix "." -o test-rel-symlink
+# RUN: dsymutil -s  %t/private/var/folders/tmp/test-rel-symlink | grep 'N_OSO' | FileCheck %s  -D#TEST_TIME=0x10 -D#FOO_TIME=0x20 --check-prefix=REL-DOT-EXPLICIT
 
 ## Check that we don't emit DWARF or stabs when -S is used
 # RUN: %lld -lSystem test.o foo.o no-debug.o -S -o %t/test-no-debug
@@ -91,6 +100,7 @@
 # REL-PATH:   (N_OSO        ) 03                         0001   [[#%.16x,TEST_TIME]] '/test.o'
 # REL-PATH-NO-SLASH:  (N_OSO        ) 03                 0001   [[#%.16x,TEST_TIME]] 'test.o'
 # REL-DOT:    (N_OSO        ) 03                         0001   [[#%.16x,TEST_TIME]] 'test.o'
+# REL-DOT-EXPLICIT:    (N_OSO        ) 03                0001   [[#%.16x,TEST_TIME]] './test.o'
 # CHECK-NEXT: (N_STSYM      ) [[#%.2d,MORE_DATA_ID + 1]] 0000   [[#%.16x,STATIC:]] '_static_var'
 # CHECK-NEXT: (N_FUN        ) [[#%.2d,TEXT_ID + 1]]      0000   [[#%.16x,MAIN:]]   '_main'
 # CHECK-NEXT: (N_FUN        ) 00                         0000   0000000000000006{{$}}
diff --git a/lldb/cmake/caches/Apple-lldb-Xcode.cmake b/lldb/cmake/caches/Apple-lldb-Xcode.cmake
index 98d68b1605e21..2372239a3db89 100644
--- a/lldb/cmake/caches/Apple-lldb-Xcode.cmake
+++ b/lldb/cmake/caches/Apple-lldb-Xcode.cmake
@@ -1,7 +1,7 @@
 include(${CMAKE_CURRENT_LIST_DIR}/Apple-lldb-base.cmake)
 
 set(CMAKE_GENERATOR Xcode CACHE STRING "")
-set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12 CACHE STRING "")
+set(CMAKE_OSX_DEPLOYMENT_TARGET 10.13 CACHE STRING "")
 set(CMAKE_XCODE_GENERATE_SCHEME ON CACHE BOOL "")
 
 set(LLDB_BUILD_FRAMEWORK ON CACHE BOOL "")
diff --git a/lldb/cmake/modules/LLDBConfig.cmake b/lldb/cmake/modules/LLDBConfig.cmake
index a9679d63e010d..c65c7e24badd9 100644
--- a/lldb/cmake/modules/LLDBConfig.cmake
+++ b/lldb/cmake/modules/LLDBConfig.cmake
@@ -68,7 +68,6 @@ add_optional_dependency(LLDB_ENABLE_FBSDVMCORE "Enable libfbsdvmcore support in
 option(LLDB_USE_ENTITLEMENTS "When codesigning, use entitlements if available" ON)
 option(LLDB_BUILD_FRAMEWORK "Build LLDB.framework (Darwin only)" OFF)
 option(LLDB_ENABLE_PROTOCOL_SERVERS "Enable protocol servers (e.g. MCP) in LLDB" ON)
-option(LLDB_ENABLE_PYTHON_LIMITED_API "Force LLDB to only use the Python Limited API (requires SWIG 4.2 or later)" OFF)
 option(LLDB_NO_INSTALL_DEFAULT_RPATH "Disable default RPATH settings in binaries" OFF)
 option(LLDB_USE_SYSTEM_DEBUGSERVER "Use the system's debugserver for testing (Darwin only)." OFF)
 option(LLDB_SKIP_STRIP "Whether to skip stripping of binaries when installing lldb." OFF)
@@ -174,11 +173,20 @@ if (LLDB_ENABLE_PYTHON)
     ${default_embed_python_home})
 
   include_directories(${Python3_INCLUDE_DIRS})
+
   if (LLDB_EMBED_PYTHON_HOME)
     get_filename_component(PYTHON_HOME "${Python3_EXECUTABLE}" DIRECTORY)
     set(LLDB_PYTHON_HOME "${PYTHON_HOME}" CACHE STRING
       "Path to use as PYTHONHOME in lldb. If a relative path is specified, it will be resolved at runtime relative to liblldb directory.")
   endif()
+
+  if (SWIG_VERSION VERSION_GREATER_EQUAL "4.2" AND NOT LLDB_EMBED_PYTHON_HOME)
+    set(default_enable_python_limited_api ON)
+  else()
+    set(default_enable_python_limited_api OFF)
+  endif()
+  option(LLDB_ENABLE_PYTHON_LIMITED_API "Force LLDB to only use the Python Limited API (requires SWIG 4.2 or later)"
+    ${default_enable_python_limited_api})
 endif()
 
 if (LLVM_EXTERNAL_CLANG_SOURCE_DIR)
diff --git a/lldb/docs/resources/lldbdap.md b/lldb/docs/resources/lldbdap.md
index 955713dd1b594..3838c82ab5dfb 100644
--- a/lldb/docs/resources/lldbdap.md
+++ b/lldb/docs/resources/lldbdap.md
@@ -170,3 +170,26 @@ This is also very simple, just run:
 ```bash
 npm run format
 ```
+
+## Working with the VS Code extension from another extension
+
+The VS Code extension exposes the following [VS Code
+commands](https://code.visualstudio.com/api/extension-guides/command),
+which can be invoked by other debugger extensions to leverage this extension's
+settings and logic. The commands help resolve configuration, create adapter
+descriptor, and get the lldb-dap process for state tracking, additional
+interaction, and telemetry.
+
+```
+// Resolve debug configuration
+const resolvedConfiguration = await vscode.commands.executeCommand("lldb-dap.resolveDebugConfiguration", folder, configuration, token);
+
+// Resolve debug configuration with substituted variables
+const resolvedConfigurationWithSubstitutedVariables = await vscode.commands.executeCommand("lldb-dap.resolveDebugConfigurationWithSubstitutedVariables", folder, configuration, token);
+
+// Create debug adapter descriptor
+const adapterDescriptor = await vscode.commands.executeCommand("lldb-dap.createDebugAdapterDescriptor", session, executable);
+
+// Get DAP server process
+const process = await vscode.commands.executeCommand("lldb-dap.getServerProcess");
+```
diff --git a/lldb/docs/use/map.rst b/lldb/docs/use/map.rst
index ea166bf8f1e9a..da566e7afe058 100644
--- a/lldb/docs/use/map.rst
+++ b/lldb/docs/use/map.rst
@@ -1126,6 +1126,20 @@ Save binary memory data starting at ``0x1000`` and ending at ``0x2000`` to a fil
   (lldb) memory read --outfile /tmp/mem.bin --binary 0x1000 0x2000
   (lldb) me r -o /tmp/mem.bin -b 0x1000 0x2000
 
+
+Print information about memory regions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: shell
+
+  (gdb) info proc mappings
+
+.. code-block:: shell
+
+  (lldb) memory region --all
+  (lldb) me reg --all
+
+
 Get information about a specific heap allocation (macOS only)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py
index bb20f3a25c1c1..b466be6a62428 100755
--- a/lldb/examples/python/crashlog.py
+++ b/lldb/examples/python/crashlog.py
@@ -1540,13 +1540,19 @@ def load_crashlog_in_scripted_process(debugger, crashlog_path, options, result):
             }
         )
     )
+
+    crashlog_sd = lldb.SBStructuredData()
+    crashlog_sd.SetGenericValue(
+        lldb.SBScriptObject(crashlog, lldb.eScriptLanguagePython)
+    )
+    structured_data.SetValueForKey("crashlog", crashlog_sd)
+
     launch_info = lldb.SBLaunchInfo(None)
     launch_info.SetProcessPluginName("ScriptedProcess")
     launch_info.SetScriptedProcessClassName(
         "crashlog_scripted_process.CrashLogScriptedProcess"
     )
     launch_info.SetScriptedProcessDictionary(structured_data)
-    launch_info.SetLaunchFlags(lldb.eLaunchFlagStopAtEntry)
 
     error = lldb.SBError()
     process = target.Launch(launch_info, error)
@@ -1554,9 +1560,6 @@ def load_crashlog_in_scripted_process(debugger, crashlog_path, options, result):
     if not process or error.Fail():
         raise InteractiveCrashLogException("couldn't launch Scripted Process", error)
 
-    process.GetScriptedImplementation().set_crashlog(crashlog)
-    process.Continue()
-
     if not options.skip_status:
 
         @contextlib.contextmanager
diff --git a/lldb/examples/python/crashlog_scripted_process.py b/lldb/examples/python/crashlog_scripted_process.py
index f54a8df0479e7..6c6eec8d12b96 100644
--- a/lldb/examples/python/crashlog_scripted_process.py
+++ b/lldb/examples/python/crashlog_scripted_process.py
@@ -10,8 +10,7 @@
 
 
 class CrashLogScriptedProcess(ScriptedProcess):
-    def set_crashlog(self, crashlog):
-        self.crashlog = crashlog
+    def parse_crashlog(self):
         if self.crashlog.process_id:
             if type(self.crashlog.process_id) is int:
                 self.pid = self.crashlog.process_id
@@ -29,8 +28,6 @@ def set_crashlog(self, crashlog):
         if hasattr(self.crashlog, "asb"):
             self.extended_thread_info = self.crashlog.asb
 
-        crashlog.load_images(self.options, self.loaded_images)
-
         for thread in self.crashlog.threads:
             if (
                 hasattr(thread, "app_specific_backtrace")
@@ -92,10 +89,21 @@ def __init__(self, exe_ctx: lldb.SBExecutionContext, args: lldb.SBStructuredData
                     no_parallel_image_loading.GetBooleanValue()
                 )
 
+        self.crashlog = None
+        crashlog = args.GetValueForKey("crashlog")
+        if crashlog and crashlog.IsValid():
+            if crashlog.GetType() == lldb.eStructuredDataTypeGeneric:
+                self.crashlog = crashlog.GetGenericValue()
+
+        if not self.crashlog:
+            # Return error
+            return
+
         self.pid = super().get_process_id()
         self.crashed_thread_idx = 0
         self.exception = None
         self.extended_thread_info = None
+        self.parse_crashlog()
 
     def read_memory_at_address(
         self, addr: int, size: int, error: lldb.SBError
@@ -104,8 +112,8 @@ def read_memory_at_address(
         return lldb.SBData()
 
     def get_loaded_images(self):
-        # TODO: Iterate over corefile_target modules and build a data structure
-        # from it.
+        if len(self.loaded_images) == 0:
+            self.crashlog.load_images(self.options, self.loaded_images)
         return self.loaded_images
 
     def should_stop(self) -> bool:
diff --git a/lldb/include/lldb/API/SBError.h b/lldb/include/lldb/API/SBError.h
index 58b45ebd793af..6a4a39c4db314 100644
--- a/lldb/include/lldb/API/SBError.h
+++ b/lldb/include/lldb/API/SBError.h
@@ -87,6 +87,7 @@ class LLDB_API SBError {
   friend class SBDebugger;
   friend class SBFile;
   friend class SBFormat;
+  friend class SBFrame;
   friend class SBHostOS;
   friend class SBPlatform;
   friend class SBProcess;
diff --git a/lldb/include/lldb/API/SBFrame.h b/lldb/include/lldb/API/SBFrame.h
index 3635ee5a537ad..08de0605b3240 100644
--- a/lldb/include/lldb/API/SBFrame.h
+++ b/lldb/include/lldb/API/SBFrame.h
@@ -233,6 +233,10 @@ class LLDB_API SBFrame {
 
   void SetFrameSP(const lldb::StackFrameSP &lldb_object_sp);
 
+  /// Return an SBValue containing an error message that warns the process is
+  /// not currently stopped.
+  static SBValue CreateProcessIsRunningExprEvalError();
+
   lldb::ExecutionContextRefSP m_opaque_sp;
 };
 
diff --git a/lldb/include/lldb/API/SBModule.h b/lldb/include/lldb/API/SBModule.h
index 85332066ee687..4009ca1461e51 100644
--- a/lldb/include/lldb/API/SBModule.h
+++ b/lldb/include/lldb/API/SBModule.h
@@ -296,6 +296,11 @@ class LLDB_API SBModule {
   /// Remove any global modules which are no longer needed.
   static void GarbageCollectAllocatedModules();
 
+  /// If this Module represents a specific object or part within a larger file,
+  /// returns the name of that object or part. Otherwise, returns
+  /// nullptr.
+  const char *GetObjectName() const;
+
 private:
   friend class SBAddress;
   friend class SBFrame;
diff --git a/lldb/include/lldb/API/SBStructuredData.h b/lldb/include/lldb/API/SBStructuredData.h
index f96e169f236ed..dfd8ec0e180ce 100644
--- a/lldb/include/lldb/API/SBStructuredData.h
+++ b/lldb/include/lldb/API/SBStructuredData.h
@@ -109,6 +109,35 @@ class SBStructuredData {
   /// Return the generic pointer if this data structure is a generic type.
   lldb::SBScriptObject GetGenericValue() const;
 
+  /// Set the value corresponding to a key. If this data structure
+  /// is not a dictionary type, reset the type to be dictionary and overwrite
+  /// the previous data.
+  void SetValueForKey(const char *key, SBStructuredData &value);
+
+  /// Change the type to unsigned interger and overwrite the previous data with
+  /// the new value.
+  void SetUnsignedIntegerValue(uint64_t value);
+
+  /// Change the type to signed interger and overwrite the previous data with
+  /// the new value.
+  void SetSignedIntegerValue(int64_t value);
+
+  /// Change the type to float and overwrite the previous data with the new
+  /// value.
+  void SetFloatValue(double value);
+
+  /// Change the type to boolean and overwrite the previous data with the new
+  /// value.
+  void SetBooleanValue(bool value);
+
+  /// Change the type to string and overwrite the previous data with the new
+  /// value.
+  void SetStringValue(const char *value);
+
+  /// Change the type to generic and overwrite the previous data with the new
+  /// value.
+  void SetGenericValue(SBScriptObject value);
+
 protected:
   friend class SBAttachInfo;
   friend class SBCommandReturnObject;
diff --git a/lldb/include/lldb/API/SBSymbol.h b/lldb/include/lldb/API/SBSymbol.h
index 94521881f82f9..a93bc7a7ae074 100644
--- a/lldb/include/lldb/API/SBSymbol.h
+++ b/lldb/include/lldb/API/SBSymbol.h
@@ -85,6 +85,12 @@ class LLDB_API SBSymbol {
 
   SymbolType GetType();
 
+  /// Get the ID of this symbol, usually the original symbol table index.
+  ///
+  /// \returns
+  ///   Returns the ID of this symbol.
+  uint32_t GetID();
+
   bool operator==(const lldb::SBSymbol &rhs) const;
 
   bool operator!=(const lldb::SBSymbol &rhs) const;
@@ -99,6 +105,15 @@ class LLDB_API SBSymbol {
   // other than the actual symbol table itself in the object file.
   bool IsSynthetic();
 
+  /// Returns true if the symbol is a debug symbol.
+  bool IsDebug();
+
+  /// Get the string representation of a symbol type.
+  static const char *GetTypeAsString(lldb::SymbolType symbol_type);
+
+  /// Get the symbol type from a string representation.
+  static lldb::SymbolType GetTypeFromString(const char *str);
+
 protected:
   lldb_private::Symbol *get();
 
diff --git a/lldb/include/lldb/API/SBTarget.h b/lldb/include/lldb/API/SBTarget.h
index 2776a8f9010fe..62cdd342a05e4 100644
--- a/lldb/include/lldb/API/SBTarget.h
+++ b/lldb/include/lldb/API/SBTarget.h
@@ -324,6 +324,16 @@ class LLDB_API SBTarget {
 
   lldb::SBModule FindModule(const lldb::SBFileSpec &file_spec);
 
+  /// Find a module with the given module specification.
+  ///
+  /// \param[in] module_spec
+  ///     A lldb::SBModuleSpec object that contains module specification.
+  ///
+  /// \return
+  ///     A lldb::SBModule object that represents the found module, or an
+  ///     invalid SBModule object if no module was found.
+  lldb::SBModule FindModule(const lldb::SBModuleSpec &module_spec);
+
   /// Find compile units related to *this target and passed source
   /// file.
   ///
@@ -658,6 +668,14 @@ class LLDB_API SBTarget {
       lldb::LanguageType symbol_language,
       const SBFileSpecList &module_list, const SBFileSpecList &comp_unit_list);
 
+  lldb::SBBreakpoint BreakpointCreateByName(
+      const char *symbol_name,
+      uint32_t
+          name_type_mask, // Logical OR one or more FunctionNameType enum bits
+      lldb::LanguageType symbol_language, lldb::addr_t offset,
+      bool offset_is_insn_count, const SBFileSpecList &module_list,
+      const SBFileSpecList &comp_unit_list);
+
 #ifdef SWIG
   lldb::SBBreakpoint BreakpointCreateByNames(
       const char **symbol_name, uint32_t num_names,
diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolver.h b/lldb/include/lldb/Breakpoint/BreakpointResolver.h
index 52cd70e934e6d..243aceeb6fd6a 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointResolver.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointResolver.h
@@ -45,9 +45,9 @@ class BreakpointResolver : public Searcher {
   ///   The breakpoint that owns this resolver.
   /// \param[in] resolverType
   ///   The concrete breakpoint resolver type for this breakpoint.
-  BreakpointResolver(const lldb::BreakpointSP &bkpt,
-                     unsigned char resolverType,
-                     lldb::addr_t offset = 0);
+  BreakpointResolver(const lldb::BreakpointSP &bkpt, unsigned char resolverType,
+                     lldb::addr_t offset = 0,
+                     bool offset_is_insn_count = false);
 
   /// The Destructor is virtual, all significant breakpoint resolvers derive
   /// from this class.
@@ -76,6 +76,7 @@ class BreakpointResolver : public Searcher {
   void SetOffset(lldb::addr_t offset);
 
   lldb::addr_t GetOffset() const { return m_offset; }
+  lldb::addr_t GetOffsetIsInsnCount() const { return m_offset_is_insn_count; }
 
   /// In response to this method the resolver scans all the modules in the
   /// breakpoint's target, and adds any new locations it finds.
@@ -220,6 +221,8 @@ class BreakpointResolver : public Searcher {
   lldb::BreakpointWP m_breakpoint; // This is the breakpoint we add locations to.
   lldb::addr_t m_offset;    // A random offset the user asked us to add to any
                             // breakpoints we set.
+  bool m_offset_is_insn_count; // Use the offset as an instruction count
+                               // instead of an address offset.
 
   // Subclass identifier (for llvm isa/dyn_cast)
   const unsigned char SubclassID;
diff --git a/lldb/include/lldb/Breakpoint/BreakpointResolverName.h b/lldb/include/lldb/Breakpoint/BreakpointResolverName.h
index c83814c174e88..48b3edabd808f 100644
--- a/lldb/include/lldb/Breakpoint/BreakpointResolverName.h
+++ b/lldb/include/lldb/Breakpoint/BreakpointResolverName.h
@@ -27,7 +27,7 @@ class BreakpointResolverName : public BreakpointResolver {
                          lldb::FunctionNameType name_type_mask,
                          lldb::LanguageType language,
                          Breakpoint::MatchType type, lldb::addr_t offset,
-                         bool skip_prologue);
+                         bool offset_is_insn_count, bool skip_prologue);
 
   // This one takes an array of names.  It is always MatchType = Exact.
   BreakpointResolverName(const lldb::BreakpointSP &bkpt, const char *names[],
diff --git a/lldb/include/lldb/Core/Disassembler.h b/lldb/include/lldb/Core/Disassembler.h
index 21bacb14f9b25..50a5d87835844 100644
--- a/lldb/include/lldb/Core/Disassembler.h
+++ b/lldb/include/lldb/Core/Disassembler.h
@@ -291,6 +291,8 @@ class InstructionList {
 
   size_t GetSize() const;
 
+  size_t GetTotalByteSize() const;
+
   uint32_t GetMaxOpcocdeByteSize() const;
 
   lldb::InstructionSP GetInstructionAtIndex(size_t idx) const;
diff --git a/lldb/include/lldb/Core/StructuredDataImpl.h b/lldb/include/lldb/Core/StructuredDataImpl.h
index fd0a7b94d3a6c..b88962bc774dc 100644
--- a/lldb/include/lldb/Core/StructuredDataImpl.h
+++ b/lldb/include/lldb/Core/StructuredDataImpl.h
@@ -81,6 +81,41 @@ class StructuredDataImpl {
 
   void SetObjectSP(const StructuredData::ObjectSP &obj) { m_data_sp = obj; }
 
+  void SetValueForKey(llvm::StringRef key,
+                      const StructuredData::ObjectSP &value) {
+    if (!m_data_sp ||
+        m_data_sp->GetType() != lldb::eStructuredDataTypeDictionary) {
+      m_data_sp = StructuredData::FromKeyValue(key, value);
+    } else if (StructuredData::Dictionary *dict =
+                   m_data_sp->GetAsDictionary()) {
+      dict->AddItem(key, value);
+    }
+  }
+
+  void SetUnsignedIntegerValue(uint64_t value) {
+    m_data_sp = StructuredData::FromInteger(value);
+  }
+
+  void SetSignedIntegerValue(int64_t value) {
+    m_data_sp = StructuredData::FromInteger(value);
+  }
+
+  void SetFloatValue(double value) {
+    m_data_sp = StructuredData::FromFloat(value);
+  }
+
+  void SetBooleanValue(bool value) {
+    m_data_sp = StructuredData::FromBoolean(value);
+  }
+
+  void SetStringValue(std::string value) {
+    m_data_sp = StructuredData::FromString(std::move(value));
+  }
+
+  void SetGenericValue(void *value) {
+    m_data_sp = StructuredData::FromGeneric(value);
+  }
+
   lldb::StructuredDataType GetType() const {
     return (m_data_sp ? m_data_sp->GetType() :
         lldb::eStructuredDataTypeInvalid);
diff --git a/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h b/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h
index cdb620e2148de..70166f33cfc45 100644
--- a/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h
+++ b/lldb/include/lldb/DataFormatters/DumpValueObjectOptions.h
@@ -76,7 +76,9 @@ class DumpValueObjectOptions {
 
   DumpValueObjectOptions &SetShowLocation(bool show = false);
 
-  DumpValueObjectOptions &SetUseObjectiveC(bool use = false);
+  DumpValueObjectOptions &DisableObjectDescription();
+
+  DumpValueObjectOptions &SetUseObjectDescription(bool use = false);
 
   DumpValueObjectOptions &SetShowSummary(bool show = true);
 
@@ -143,13 +145,14 @@ class DumpValueObjectOptions {
   ChildPrintingDecider m_child_printing_decider;
   PointerAsArraySettings m_pointer_as_array;
   unsigned m_expand_ptr_type_flags = 0;
+  // The following flags commonly default to false.
   bool m_use_synthetic : 1;
   bool m_scope_already_checked : 1;
   bool m_flat_output : 1;
   bool m_ignore_cap : 1;
   bool m_show_types : 1;
   bool m_show_location : 1;
-  bool m_use_objc : 1;
+  bool m_use_object_desc : 1;
   bool m_hide_root_type : 1;
   bool m_hide_root_name : 1;
   bool m_hide_name : 1;
diff --git a/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h b/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h
index f9deb6ebf8d6d..fdea3682f8844 100644
--- a/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h
+++ b/lldb/include/lldb/DataFormatters/ValueObjectPrinter.h
@@ -75,8 +75,6 @@ class ValueObjectPrinter {
 
   void SetupMostSpecializedValue();
 
-  llvm::Expected<std::string> GetDescriptionForDisplay();
-
   const char *GetRootNameForDisplay();
 
   bool ShouldPrintValueObject();
@@ -108,8 +106,7 @@ class ValueObjectPrinter {
 
   bool PrintValueAndSummaryIfNeeded(bool &value_printed, bool &summary_printed);
 
-  llvm::Error PrintObjectDescriptionIfNeeded(bool value_printed,
-                                             bool summary_printed);
+  void PrintObjectDescriptionIfNeeded(std::optional<std::string> object_desc);
 
   bool
   ShouldPrintChildren(DumpValueObjectOptions::PointerDepth &curr_ptr_depth);
@@ -141,6 +138,7 @@ class ValueObjectPrinter {
 
 private:
   bool ShouldShowName() const;
+  bool ShouldPrintObjectDescription();
 
   ValueObject &m_orig_valobj;
   /// Cache the current "most specialized" value.  Don't use this
diff --git a/lldb/include/lldb/Host/JSONTransport.h b/lldb/include/lldb/Host/JSONTransport.h
index 4087cdf2b42f7..0be60a8f3f96a 100644
--- a/lldb/include/lldb/Host/JSONTransport.h
+++ b/lldb/include/lldb/Host/JSONTransport.h
@@ -13,125 +13,284 @@
 #ifndef LLDB_HOST_JSONTRANSPORT_H
 #define LLDB_HOST_JSONTRANSPORT_H
 
+#include "lldb/Host/MainLoop.h"
+#include "lldb/Host/MainLoopBase.h"
+#include "lldb/Utility/IOObject.h"
+#include "lldb/Utility/Status.h"
 #include "lldb/lldb-forward.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/JSON.h"
-#include <chrono>
+#include "llvm/Support/raw_ostream.h"
+#include <string>
 #include <system_error>
+#include <variant>
+#include <vector>
 
 namespace lldb_private {
 
-class TransportEOFError : public llvm::ErrorInfo<TransportEOFError> {
+class TransportUnhandledContentsError
+    : public llvm::ErrorInfo<TransportUnhandledContentsError> {
 public:
   static char ID;
 
-  TransportEOFError() = default;
+  explicit TransportUnhandledContentsError(std::string unhandled_contents);
 
-  void log(llvm::raw_ostream &OS) const override {
-    OS << "transport end of file reached";
-  }
-  std::error_code convertToErrorCode() const override {
-    return llvm::inconvertibleErrorCode();
+  void log(llvm::raw_ostream &OS) const override;
+  std::error_code convertToErrorCode() const override;
+
+  const std::string &getUnhandledContents() const {
+    return m_unhandled_contents;
   }
+
+private:
+  std::string m_unhandled_contents;
 };
 
-class TransportTimeoutError : public llvm::ErrorInfo<TransportTimeoutError> {
+/// A transport is responsible for maintaining the connection to a client
+/// application, and reading/writing structured messages to it.
+///
+/// Transports have limited thread safety requirements:
+///  - Messages will not be sent concurrently.
+///  - Messages MAY be sent while Run() is reading, or its callback is active.
+template <typename Req, typename Resp, typename Evt> class Transport {
 public:
-  static char ID;
+  using Message = std::variant<Req, Resp, Evt>;
 
-  TransportTimeoutError() = default;
+  virtual ~Transport() = default;
 
-  void log(llvm::raw_ostream &OS) const override {
-    OS << "transport operation timed out";
-  }
-  std::error_code convertToErrorCode() const override {
-    return std::make_error_code(std::errc::timed_out);
-  }
-};
+  /// Sends an event, a message that does not require a response.
+  virtual llvm::Error Send(const Evt &) = 0;
+  /// Sends a request, a message that expects a response.
+  virtual llvm::Error Send(const Req &) = 0;
+  /// Sends a response to a specific request.
+  virtual llvm::Error Send(const Resp &) = 0;
 
-class TransportInvalidError : public llvm::ErrorInfo<TransportInvalidError> {
-public:
-  static char ID;
+  /// Implemented to handle incoming messages. (See Run() below).
+  class MessageHandler {
+  public:
+    virtual ~MessageHandler() = default;
+    /// Called when an event is received.
+    virtual void Received(const Evt &) = 0;
+    /// Called when a request is received.
+    virtual void Received(const Req &) = 0;
+    /// Called when a response is received.
+    virtual void Received(const Resp &) = 0;
 
-  TransportInvalidError() = default;
+    /// Called when an error occurs while reading from the transport.
+    ///
+    /// NOTE: This does *NOT* indicate that a specific request failed, but that
+    /// there was an error in the underlying transport.
+    virtual void OnError(llvm::Error) = 0;
 
-  void log(llvm::raw_ostream &OS) const override {
-    OS << "transport IO object invalid";
-  }
-  std::error_code convertToErrorCode() const override {
-    return std::make_error_code(std::errc::not_connected);
+    /// Called on EOF or client disconnect.
+    virtual void OnClosed() = 0;
+  };
+
+  using MessageHandlerSP = std::shared_ptr<MessageHandler>;
+
+  /// RegisterMessageHandler registers the Transport with the given MainLoop and
+  /// handles any incoming messages using the given MessageHandler.
+  ///
+  /// If an unexpected error occurs, the MainLoop will be terminated and a log
+  /// message will include additional information about the termination reason.
+  virtual llvm::Expected<MainLoop::ReadHandleUP>
+  RegisterMessageHandler(MainLoop &loop, MessageHandler &handler) = 0;
+
+protected:
+  template <typename... Ts> inline auto Logv(const char *Fmt, Ts &&...Vals) {
+    Log(llvm::formatv(Fmt, std::forward<Ts>(Vals)...).str());
   }
+  virtual void Log(llvm::StringRef message) = 0;
 };
 
-/// A transport class that uses JSON for communication.
-class JSONTransport {
+/// A JSONTransport will encode and decode messages using JSON.
+template <typename Req, typename Resp, typename Evt>
+class JSONTransport : public Transport<Req, Resp, Evt> {
 public:
-  JSONTransport(lldb::IOObjectSP input, lldb::IOObjectSP output);
-  virtual ~JSONTransport() = default;
-
-  /// Transport is not copyable.
-  /// @{
-  JSONTransport(const JSONTransport &rhs) = delete;
-  void operator=(const JSONTransport &rhs) = delete;
-  /// @}
-
-  /// Writes a message to the output stream.
-  template <typename T> llvm::Error Write(const T &t) {
-    const std::string message = llvm::formatv("{0}", toJSON(t)).str();
-    return WriteImpl(message);
-  }
+  using Transport<Req, Resp, Evt>::Transport;
+  using MessageHandler = typename Transport<Req, Resp, Evt>::MessageHandler;
 
-  /// Reads the next message from the input stream.
-  template <typename T>
-  llvm::Expected<T> Read(const std::chrono::microseconds &timeout) {
-    llvm::Expected<std::string> message = ReadImpl(timeout);
-    if (!message)
-      return message.takeError();
-    return llvm::json::parse<T>(/*JSON=*/*message);
+  JSONTransport(lldb::IOObjectSP in, lldb::IOObjectSP out)
+      : m_in(in), m_out(out) {}
+
+  llvm::Error Send(const Evt &evt) override { return Write(evt); }
+  llvm::Error Send(const Req &req) override { return Write(req); }
+  llvm::Error Send(const Resp &resp) override { return Write(resp); }
+
+  llvm::Expected<MainLoop::ReadHandleUP>
+  RegisterMessageHandler(MainLoop &loop, MessageHandler &handler) override {
+    Status status;
+    MainLoop::ReadHandleUP read_handle = loop.RegisterReadObject(
+        m_in,
+        std::bind(&JSONTransport::OnRead, this, std::placeholders::_1,
+                  std::ref(handler)),
+        status);
+    if (status.Fail()) {
+      return status.takeError();
+    }
+    return read_handle;
   }
 
+  /// Public for testing purposes, otherwise this should be an implementation
+  /// detail.
+  static constexpr size_t kReadBufferSize = 1024;
+
 protected:
-  virtual void Log(llvm::StringRef message);
+  virtual llvm::Expected<std::vector<std::string>> Parse() = 0;
+  virtual std::string Encode(const llvm::json::Value &message) = 0;
+  llvm::Error Write(const llvm::json::Value &message) {
+    this->Logv("<-- {0}", message);
+    std::string output = Encode(message);
+    size_t bytes_written = output.size();
+    return m_out->Write(output.data(), bytes_written).takeError();
+  }
+
+  llvm::SmallString<kReadBufferSize> m_buffer;
 
-  virtual llvm::Error WriteImpl(const std::string &message) = 0;
-  virtual llvm::Expected<std::string>
-  ReadImpl(const std::chrono::microseconds &timeout) = 0;
+private:
+  void OnRead(MainLoopBase &loop, MessageHandler &handler) {
+    char buf[kReadBufferSize];
+    size_t num_bytes = sizeof(buf);
+    if (Status status = m_in->Read(buf, num_bytes); status.Fail()) {
+      handler.OnError(status.takeError());
+      return;
+    }
 
-  lldb::IOObjectSP m_input;
-  lldb::IOObjectSP m_output;
+    if (num_bytes)
+      m_buffer.append(llvm::StringRef(buf, num_bytes));
+
+    // If the buffer has contents, try parsing any pending messages.
+    if (!m_buffer.empty()) {
+      llvm::Expected<std::vector<std::string>> raw_messages = Parse();
+      if (llvm::Error error = raw_messages.takeError()) {
+        handler.OnError(std::move(error));
+        return;
+      }
+
+      for (const std::string &raw_message : *raw_messages) {
+        llvm::Expected<typename Transport<Req, Resp, Evt>::Message> message =
+            llvm::json::parse<typename Transport<Req, Resp, Evt>::Message>(
+                raw_message);
+        if (!message) {
+          handler.OnError(message.takeError());
+          return;
+        }
+
+        std::visit([&handler](auto &&msg) { handler.Received(msg); }, *message);
+      }
+    }
+
+    // Check if we reached EOF.
+    if (num_bytes == 0) {
+      // EOF reached, but there may still be unhandled contents in the buffer.
+      if (!m_buffer.empty())
+        handler.OnError(llvm::make_error<TransportUnhandledContentsError>(
+            std::string(m_buffer.str())));
+      handler.OnClosed();
+    }
+  }
+
+  lldb::IOObjectSP m_in;
+  lldb::IOObjectSP m_out;
 };
 
 /// A transport class for JSON with a HTTP header.
-class HTTPDelimitedJSONTransport : public JSONTransport {
+template <typename Req, typename Resp, typename Evt>
+class HTTPDelimitedJSONTransport : public JSONTransport<Req, Resp, Evt> {
 public:
-  HTTPDelimitedJSONTransport(lldb::IOObjectSP input, lldb::IOObjectSP output)
-      : JSONTransport(input, output) {}
-  virtual ~HTTPDelimitedJSONTransport() = default;
+  using JSONTransport<Req, Resp, Evt>::JSONTransport;
 
 protected:
-  virtual llvm::Error WriteImpl(const std::string &message) override;
-  virtual llvm::Expected<std::string>
-  ReadImpl(const std::chrono::microseconds &timeout) override;
-
-  // FIXME: Support any header.
-  static constexpr llvm::StringLiteral kHeaderContentLength =
-      "Content-Length: ";
-  static constexpr llvm::StringLiteral kHeaderSeparator = "\r\n\r\n";
+  /// Encodes messages based on
+  /// https://microsoft.github.io/debug-adapter-protocol/overview#base-protocol
+  std::string Encode(const llvm::json::Value &message) override {
+    std::string output;
+    std::string raw_message = llvm::formatv("{0}", message).str();
+    llvm::raw_string_ostream OS(output);
+    OS << kHeaderContentLength << kHeaderFieldSeparator << ' '
+       << std::to_string(raw_message.size()) << kEndOfHeader << raw_message;
+    return output;
+  }
+
+  /// Parses messages based on
+  /// https://microsoft.github.io/debug-adapter-protocol/overview#base-protocol
+  llvm::Expected<std::vector<std::string>> Parse() override {
+    std::vector<std::string> messages;
+    llvm::StringRef buffer = this->m_buffer;
+    while (buffer.contains(kEndOfHeader)) {
+      auto [headers, rest] = buffer.split(kEndOfHeader);
+      size_t content_length = 0;
+      // HTTP Headers are formatted like `<field-name> ':' [<field-value>]`.
+      for (const llvm::StringRef &header :
+           llvm::split(headers, kHeaderSeparator)) {
+        auto [key, value] = header.split(kHeaderFieldSeparator);
+        // 'Content-Length' is the only meaningful key at the moment. Others are
+        // ignored.
+        if (!key.equals_insensitive(kHeaderContentLength))
+          continue;
+
+        value = value.trim();
+        if (!llvm::to_integer(value, content_length, 10)) {
+          // Clear the buffer to avoid re-parsing this malformed message.
+          this->m_buffer.clear();
+          return llvm::createStringError(std::errc::invalid_argument,
+                                         "invalid content length: %s",
+                                         value.str().c_str());
+        }
+      }
+
+      // Check if we have enough data.
+      if (content_length > rest.size())
+        break;
+
+      llvm::StringRef body = rest.take_front(content_length);
+      buffer = rest.drop_front(content_length);
+      messages.emplace_back(body.str());
+      this->Logv("--> {0}", body);
+    }
+
+    // Store the remainder of the buffer for the next read callback.
+    this->m_buffer = buffer.str();
+
+    return std::move(messages);
+  }
+
+  static constexpr llvm::StringLiteral kHeaderContentLength = "Content-Length";
+  static constexpr llvm::StringLiteral kHeaderFieldSeparator = ":";
+  static constexpr llvm::StringLiteral kHeaderSeparator = "\r\n";
+  static constexpr llvm::StringLiteral kEndOfHeader = "\r\n\r\n";
 };
 
 /// A transport class for JSON RPC.
-class JSONRPCTransport : public JSONTransport {
+template <typename Req, typename Resp, typename Evt>
+class JSONRPCTransport : public JSONTransport<Req, Resp, Evt> {
 public:
-  JSONRPCTransport(lldb::IOObjectSP input, lldb::IOObjectSP output)
-      : JSONTransport(input, output) {}
-  virtual ~JSONRPCTransport() = default;
+  using JSONTransport<Req, Resp, Evt>::JSONTransport;
 
 protected:
-  virtual llvm::Error WriteImpl(const std::string &message) override;
-  virtual llvm::Expected<std::string>
-  ReadImpl(const std::chrono::microseconds &timeout) override;
+  std::string Encode(const llvm::json::Value &message) override {
+    return llvm::formatv("{0}{1}", message, kMessageSeparator).str();
+  }
+
+  llvm::Expected<std::vector<std::string>> Parse() override {
+    std::vector<std::string> messages;
+    llvm::StringRef buf = this->m_buffer;
+    while (buf.contains(kMessageSeparator)) {
+      auto [raw_json, rest] = buf.split(kMessageSeparator);
+      buf = rest;
+      messages.emplace_back(raw_json.str());
+      this->Logv("--> {0}", raw_json);
+    }
+
+    // Store the remainder of the buffer for the next read callback.
+    this->m_buffer = buf.str();
+
+    return messages;
+  }
 
   static constexpr llvm::StringLiteral kMessageSeparator = "\n";
 };
diff --git a/lldb/include/lldb/Host/ProcessRunLock.h b/lldb/include/lldb/Host/ProcessRunLock.h
index 94683673024d5..27d10942ddb4c 100644
--- a/lldb/include/lldb/Host/ProcessRunLock.h
+++ b/lldb/include/lldb/Host/ProcessRunLock.h
@@ -41,9 +41,22 @@ class ProcessRunLock {
   class ProcessRunLocker {
   public:
     ProcessRunLocker() = default;
+    ProcessRunLocker(ProcessRunLocker &&other) : m_lock(other.m_lock) {
+      other.m_lock = nullptr;
+    }
+    ProcessRunLocker &operator=(ProcessRunLocker &&other) {
+      if (this != &other) {
+        Unlock();
+        m_lock = other.m_lock;
+        other.m_lock = nullptr;
+      }
+      return *this;
+    }
 
     ~ProcessRunLocker() { Unlock(); }
 
+    bool IsLocked() const { return m_lock; }
+
     // Try to lock the read lock, but only do so if there are no writers.
     bool TryLock(ProcessRunLock *lock) {
       if (m_lock) {
diff --git a/lldb/include/lldb/Interpreter/OptionGroupValueObjectDisplay.h b/lldb/include/lldb/Interpreter/OptionGroupValueObjectDisplay.h
index ebf26cea95cf3..4fd552a6700b8 100644
--- a/lldb/include/lldb/Interpreter/OptionGroupValueObjectDisplay.h
+++ b/lldb/include/lldb/Interpreter/OptionGroupValueObjectDisplay.h
@@ -31,7 +31,7 @@ class OptionGroupValueObjectDisplay : public OptionGroup {
 
   bool AnyOptionWasSet() const {
     return show_types || no_summary_depth != 0 || show_location ||
-           flat_output || use_objc || max_depth != UINT32_MAX ||
+           flat_output || use_object_desc || max_depth != UINT32_MAX ||
            ptr_depth != 0 || !use_synth || be_raw || ignore_cap ||
            run_validator;
   }
@@ -42,7 +42,7 @@ class OptionGroupValueObjectDisplay : public OptionGroup {
       lldb::Format format = lldb::eFormatDefault,
       lldb::TypeSummaryImplSP summary_sp = lldb::TypeSummaryImplSP());
 
-  bool show_types : 1, show_location : 1, flat_output : 1, use_objc : 1,
+  bool show_types : 1, show_location : 1, flat_output : 1, use_object_desc : 1,
       use_synth : 1, be_raw : 1, ignore_cap : 1, run_validator : 1,
       max_depth_is_default : 1;
 
diff --git a/lldb/include/lldb/Interpreter/OptionValue.h b/lldb/include/lldb/Interpreter/OptionValue.h
index f293a3a33bfa0..9c992821251cb 100644
--- a/lldb/include/lldb/Interpreter/OptionValue.h
+++ b/lldb/include/lldb/Interpreter/OptionValue.h
@@ -17,6 +17,7 @@
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/FileSpecList.h"
 #include "lldb/Utility/Status.h"
+#include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StringList.h"
 #include "lldb/Utility/UUID.h"
 #include "lldb/lldb-defines.h"
@@ -61,6 +62,7 @@ class OptionValue {
     eDumpOptionDescription = (1u << 3),
     eDumpOptionRaw = (1u << 4),
     eDumpOptionCommand = (1u << 5),
+    eDumpOptionDefaultValue = (1u << 6),
     eDumpGroupValue = (eDumpOptionName | eDumpOptionType | eDumpOptionValue),
     eDumpGroupHelp =
         (eDumpOptionName | eDumpOptionType | eDumpOptionDescription),
@@ -338,6 +340,20 @@ class OptionValue {
   // DeepCopy to it. Inherit from Cloneable to avoid doing this manually.
   virtual lldb::OptionValueSP Clone() const = 0;
 
+  class DefaultValueFormat {
+  public:
+    DefaultValueFormat(Stream &stream) : stream(stream) {
+      stream.PutCString(" (default: ");
+    }
+    ~DefaultValueFormat() { stream.PutChar(')'); }
+
+    DefaultValueFormat(const DefaultValueFormat &) = delete;
+    DefaultValueFormat &operator=(const DefaultValueFormat &) = delete;
+
+  private:
+    Stream &stream;
+  };
+
   lldb::OptionValueWP m_parent_wp;
   std::function<void()> m_callback;
   bool m_value_was_set = false; // This can be used to see if a value has been
diff --git a/lldb/include/lldb/Interpreter/OptionValueEnumeration.h b/lldb/include/lldb/Interpreter/OptionValueEnumeration.h
index a3a13ca7b15eb..91ab454b2065e 100644
--- a/lldb/include/lldb/Interpreter/OptionValueEnumeration.h
+++ b/lldb/include/lldb/Interpreter/OptionValueEnumeration.h
@@ -72,6 +72,7 @@ class OptionValueEnumeration
 
 protected:
   void SetEnumerations(const OptionEnumValues &enumerators);
+  void DumpEnum(Stream &strm, enum_type value);
 
   enum_type m_current_value;
   enum_type m_default_value;
diff --git a/lldb/include/lldb/Protocol/MCP/MCPError.h b/lldb/include/lldb/Protocol/MCP/MCPError.h
index 2bdbb9b7a6874..55dd40f124a15 100644
--- a/lldb/include/lldb/Protocol/MCP/MCPError.h
+++ b/lldb/include/lldb/Protocol/MCP/MCPError.h
@@ -26,7 +26,7 @@ class MCPError : public llvm::ErrorInfo<MCPError> {
 
   const std::string &getMessage() const { return m_message; }
 
-  lldb_protocol::mcp::Error toProtcolError() const;
+  lldb_protocol::mcp::Error toProtocolError() const;
 
   static constexpr int64_t kResourceNotFound = -32002;
   static constexpr int64_t kInternalError = -32603;
diff --git a/lldb/include/lldb/Protocol/MCP/Protocol.h b/lldb/include/lldb/Protocol/MCP/Protocol.h
index c43b06809bd3f..49f9490221755 100644
--- a/lldb/include/lldb/Protocol/MCP/Protocol.h
+++ b/lldb/include/lldb/Protocol/MCP/Protocol.h
@@ -21,52 +21,78 @@
 
 namespace lldb_protocol::mcp {
 
-static llvm::StringLiteral kVersion = "2024-11-05";
+static llvm::StringLiteral kProtocolVersion = "2024-11-05";
+
+/// A Request or Response 'id'.
+///
+/// NOTE: This differs from the JSON-RPC 2.0 spec. The MCP spec says this must
+/// be a string or number, excluding a json 'null' as a valid id.
+using Id = std::variant<int64_t, std::string>;
 
 /// A request that expects a response.
 struct Request {
-  uint64_t id = 0;
+  /// The request id.
+  Id id = 0;
+  /// The method to be invoked.
   std::string method;
+  /// The method's params.
   std::optional<llvm::json::Value> params;
 };
 
 llvm::json::Value toJSON(const Request &);
 bool fromJSON(const llvm::json::Value &, Request &, llvm::json::Path);
+bool operator==(const Request &, const Request &);
 
-struct ErrorInfo {
+struct Error {
+  /// The error type that occurred.
   int64_t code = 0;
+  /// A short description of the error. The message SHOULD be limited to a
+  /// concise single sentence.
   std::string message;
-  std::string data;
-};
-
-llvm::json::Value toJSON(const ErrorInfo &);
-bool fromJSON(const llvm::json::Value &, ErrorInfo &, llvm::json::Path);
-
-struct Error {
-  uint64_t id = 0;
-  ErrorInfo error;
+  /// Additional information about the error. The value of this member is
+  /// defined by the sender (e.g. detailed error information, nested errors
+  /// etc.).
+  std::optional<llvm::json::Value> data;
 };
 
 llvm::json::Value toJSON(const Error &);
 bool fromJSON(const llvm::json::Value &, Error &, llvm::json::Path);
+bool operator==(const Error &, const Error &);
 
+/// A response to a request, either an error or a result.
 struct Response {
-  uint64_t id = 0;
-  std::optional<llvm::json::Value> result;
-  std::optional<ErrorInfo> error;
+  /// The request id.
+  Id id = 0;
+  /// The result of the request, either an Error or the JSON value of the
+  /// response.
+  std::variant<Error, llvm::json::Value> result;
 };
 
 llvm::json::Value toJSON(const Response &);
 bool fromJSON(const llvm::json::Value &, Response &, llvm::json::Path);
+bool operator==(const Response &, const Response &);
 
 /// A notification which does not expect a response.
 struct Notification {
+  /// The method to be invoked.
   std::string method;
+  /// The notification's params.
   std::optional<llvm::json::Value> params;
 };
 
 llvm::json::Value toJSON(const Notification &);
 bool fromJSON(const llvm::json::Value &, Notification &, llvm::json::Path);
+bool operator==(const Notification &, const Notification &);
+
+/// A general message as defined by the JSON-RPC 2.0 spec.
+using Message = std::variant<Request, Response, Notification>;
+// With clang-cl and MSVC STL 202208, convertible can be false later if we do
+// not force it to be checked early here.
+static_assert(std::is_convertible_v<Message, Message>,
+              "Message is not convertible to itself");
+
+bool fromJSON(const llvm::json::Value &, Message &, llvm::json::Path);
+llvm::json::Value toJSON(const Message &);
 
 struct ToolCapability {
   /// Whether this server supports notifications for changes to the tool list.
@@ -176,11 +202,6 @@ struct ToolDefinition {
 llvm::json::Value toJSON(const ToolDefinition &);
 bool fromJSON(const llvm::json::Value &, ToolDefinition &, llvm::json::Path);
 
-using Message = std::variant<Request, Response, Notification, Error>;
-
-bool fromJSON(const llvm::json::Value &, Message &, llvm::json::Path);
-llvm::json::Value toJSON(const Message &);
-
 using ToolArguments = std::variant<std::monostate, llvm::json::Value>;
 
 } // namespace lldb_protocol::mcp
diff --git a/lldb/include/lldb/Protocol/MCP/Server.h b/lldb/include/lldb/Protocol/MCP/Server.h
new file mode 100644
index 0000000000000..2ac05880de86b
--- /dev/null
+++ b/lldb/include/lldb/Protocol/MCP/Server.h
@@ -0,0 +1,70 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_PROTOCOL_MCP_SERVER_H
+#define LLDB_PROTOCOL_MCP_SERVER_H
+
+#include "lldb/Protocol/MCP/Protocol.h"
+#include "lldb/Protocol/MCP/Resource.h"
+#include "lldb/Protocol/MCP/Tool.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/Error.h"
+#include <mutex>
+
+namespace lldb_protocol::mcp {
+
+class Server {
+public:
+  Server(std::string name, std::string version);
+  virtual ~Server() = default;
+
+  void AddTool(std::unique_ptr<Tool> tool);
+  void AddResourceProvider(std::unique_ptr<ResourceProvider> resource_provider);
+
+protected:
+  virtual Capabilities GetCapabilities() = 0;
+
+  using RequestHandler =
+      std::function<llvm::Expected<Response>(const Request &)>;
+  using NotificationHandler = std::function<void(const Notification &)>;
+
+  void AddRequestHandlers();
+
+  void AddRequestHandler(llvm::StringRef method, RequestHandler handler);
+  void AddNotificationHandler(llvm::StringRef method,
+                              NotificationHandler handler);
+
+  llvm::Expected<std::optional<Message>> HandleData(llvm::StringRef data);
+
+  llvm::Expected<Response> Handle(Request request);
+  void Handle(Notification notification);
+
+  llvm::Expected<Response> InitializeHandler(const Request &);
+
+  llvm::Expected<Response> ToolsListHandler(const Request &);
+  llvm::Expected<Response> ToolsCallHandler(const Request &);
+
+  llvm::Expected<Response> ResourcesListHandler(const Request &);
+  llvm::Expected<Response> ResourcesReadHandler(const Request &);
+
+  std::mutex m_mutex;
+
+private:
+  const std::string m_name;
+  const std::string m_version;
+
+  llvm::StringMap<std::unique_ptr<Tool>> m_tools;
+  std::vector<std::unique_ptr<ResourceProvider>> m_resource_providers;
+
+  llvm::StringMap<RequestHandler> m_request_handlers;
+  llvm::StringMap<NotificationHandler> m_notification_handlers;
+};
+
+} // namespace lldb_protocol::mcp
+
+#endif
diff --git a/lldb/include/lldb/Symbol/Symbol.h b/lldb/include/lldb/Symbol/Symbol.h
index 688c8a5931feb..b994c34e46493 100644
--- a/lldb/include/lldb/Symbol/Symbol.h
+++ b/lldb/include/lldb/Symbol/Symbol.h
@@ -15,6 +15,7 @@
 #include "lldb/Symbol/SymbolContextScope.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/UserID.h"
+#include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-private.h"
 #include "llvm/Support/JSON.h"
 
@@ -167,7 +168,7 @@ class Symbol : public SymbolContextScope {
 
   lldb::SymbolType GetType() const { return (lldb::SymbolType)m_type; }
 
-  void SetType(lldb::SymbolType type) { m_type = (lldb::SymbolType)type; }
+  void SetType(lldb::SymbolType type) { m_type = type; }
 
   const char *GetTypeAsString() const;
 
@@ -301,6 +302,10 @@ class Symbol : public SymbolContextScope {
 
   bool operator==(const Symbol &rhs) const;
 
+  static const char *GetTypeAsString(lldb::SymbolType symbol_type);
+
+  static lldb::SymbolType GetTypeFromString(const char *str);
+
 protected:
   // This is the internal guts of ResolveReExportedSymbol, it assumes
   // reexport_name is not null, and that module_spec is valid.  We track the
diff --git a/lldb/include/lldb/Target/ExecutionContext.h b/lldb/include/lldb/Target/ExecutionContext.h
index aebd0d5308e72..f105e38fa69aa 100644
--- a/lldb/include/lldb/Target/ExecutionContext.h
+++ b/lldb/include/lldb/Target/ExecutionContext.h
@@ -11,6 +11,7 @@
 
 #include <mutex>
 
+#include "lldb/Host/ProcessRunLock.h"
 #include "lldb/Target/StackID.h"
 #include "lldb/lldb-private.h"
 
@@ -315,14 +316,6 @@ class ExecutionContext {
   ExecutionContext(const ExecutionContextRef *exe_ctx_ref,
                    bool thread_and_frame_only_if_stopped = false);
 
-  // These two variants take in a locker, and grab the target, lock the API
-  // mutex into locker, then fill in the rest of the shared pointers.
-  ExecutionContext(const ExecutionContextRef &exe_ctx_ref,
-                   std::unique_lock<std::recursive_mutex> &locker)
-      : ExecutionContext(&exe_ctx_ref, locker) {}
-
-  ExecutionContext(const ExecutionContextRef *exe_ctx_ref,
-                   std::unique_lock<std::recursive_mutex> &locker);
   // Create execution contexts from execution context scopes
   ExecutionContext(ExecutionContextScope *exe_scope);
   ExecutionContext(ExecutionContextScope &exe_scope);
@@ -566,6 +559,53 @@ class ExecutionContext {
   lldb::StackFrameSP m_frame_sp; ///< The stack frame in thread.
 };
 
+/// A wrapper class representing an execution context with non-null Target
+/// and Process pointers, a locked API mutex and a locked ProcessRunLock.
+/// The locks are private by design: to unlock them, destroy the
+/// StoppedExecutionContext.
+struct StoppedExecutionContext : ExecutionContext {
+  StoppedExecutionContext(lldb::TargetSP &target_sp,
+                          lldb::ProcessSP &process_sp,
+                          lldb::ThreadSP &thread_sp,
+                          lldb::StackFrameSP &frame_sp,
+                          std::unique_lock<std::recursive_mutex> api_lock,
+                          ProcessRunLock::ProcessRunLocker stop_locker)
+      : m_api_lock(std::move(api_lock)), m_stop_locker(std::move(stop_locker)) {
+    assert(target_sp);
+    assert(process_sp);
+    assert(m_api_lock.owns_lock());
+    assert(m_stop_locker.IsLocked());
+    SetTargetSP(target_sp);
+    SetProcessSP(process_sp);
+    SetThreadSP(thread_sp);
+    SetFrameSP(frame_sp);
+  }
+
+  /// Transfers ownership of the locks from `other` to `this`, making `other`
+  /// unusable.
+  StoppedExecutionContext(StoppedExecutionContext &&other)
+      : StoppedExecutionContext(other.m_target_sp, other.m_process_sp,
+                                other.m_thread_sp, other.m_frame_sp,
+                                std::move(other.m_api_lock),
+                                std::move(other.m_stop_locker)) {
+    other.Clear();
+  }
+
+  /// Clears this context, unlocking the ProcessRunLock and returning the
+  /// locked API lock, allowing callers to resume the process. Similar to
+  /// a move operation, this object is no longer usable.
+  [[nodiscard]] std::unique_lock<std::recursive_mutex> AllowResume();
+
+private:
+  std::unique_lock<std::recursive_mutex> m_api_lock;
+  ProcessRunLock::ProcessRunLocker m_stop_locker;
+};
+
+llvm::Expected<StoppedExecutionContext>
+GetStoppedExecutionContext(const ExecutionContextRef *exe_ctx_ref_ptr);
+llvm::Expected<StoppedExecutionContext>
+GetStoppedExecutionContext(const lldb::ExecutionContextRefSP &exe_ctx_ref_ptr);
+
 } // namespace lldb_private
 
 #endif // LLDB_TARGET_EXECUTIONCONTEXT_H
diff --git a/lldb/include/lldb/Target/StackFrame.h b/lldb/include/lldb/Target/StackFrame.h
index 3f51c9a7f22f0..d4104bfe49d20 100644
--- a/lldb/include/lldb/Target/StackFrame.h
+++ b/lldb/include/lldb/Target/StackFrame.h
@@ -241,8 +241,9 @@ class StackFrame : public ExecutionContextScope,
     return m_reg_context_sp;
   }
 
-  /// Retrieve the list of variables that are in scope at this StackFrame's
-  /// pc.
+  /// Retrieve the list of variables whose scope either:
+  /// * contains this StackFrame's pc,
+  /// * is a child of this StackFrame's current scope.
   ///
   /// A frame that is not live may return an empty VariableList for a given
   /// pc value even though variables would be available at this point if it
@@ -274,6 +275,9 @@ class StackFrame : public ExecutionContextScope,
   ///     that are visible to the entire compilation unit (e.g. file
   ///     static in C, globals that are homed in this CU).
   ///
+  /// \param[in] must_have_valid_location
+  ///     Whether to filter variables whose location is not available at this
+  ///     StackFrame's pc.
   /// \return
   ///     A pointer to a list of variables.
   lldb::VariableListSP
diff --git a/lldb/include/lldb/Target/StackID.h b/lldb/include/lldb/Target/StackID.h
index 09392792cb501..fddbc8e48dfdc 100644
--- a/lldb/include/lldb/Target/StackID.h
+++ b/lldb/include/lldb/Target/StackID.h
@@ -14,14 +14,15 @@
 
 namespace lldb_private {
 
+class Process;
+
 class StackID {
 public:
   // Constructors and Destructors
   StackID() = default;
 
   explicit StackID(lldb::addr_t pc, lldb::addr_t cfa,
-                   SymbolContextScope *symbol_scope)
-      : m_pc(pc), m_cfa(cfa), m_symbol_scope(symbol_scope) {}
+                   SymbolContextScope *symbol_scope, Process *process);
 
   StackID(const StackID &rhs)
       : m_pc(rhs.m_pc), m_cfa(rhs.m_cfa), m_symbol_scope(rhs.m_symbol_scope) {}
@@ -63,9 +64,8 @@ class StackID {
 protected:
   friend class StackFrame;
 
-  void SetPC(lldb::addr_t pc) { m_pc = pc; }
-
-  void SetCFA(lldb::addr_t cfa) { m_cfa = cfa; }
+  void SetPC(lldb::addr_t pc, Process *process);
+  void SetCFA(lldb::addr_t cfa, Process *process);
 
   lldb::addr_t m_pc =
       LLDB_INVALID_ADDRESS; // The pc value for the function/symbol for this
diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h
index 7b23c8abe8d2f..14a09f29094d5 100644
--- a/lldb/include/lldb/Target/Target.h
+++ b/lldb/include/lldb/Target/Target.h
@@ -18,6 +18,7 @@
 #include "lldb/Breakpoint/BreakpointList.h"
 #include "lldb/Breakpoint/BreakpointName.h"
 #include "lldb/Breakpoint/WatchpointList.h"
+#include "lldb/Core/Address.h"
 #include "lldb/Core/Architecture.h"
 #include "lldb/Core/Disassembler.h"
 #include "lldb/Core/ModuleList.h"
@@ -723,7 +724,7 @@ class Target : public std::enable_shared_from_this<Target>,
   lldb::BreakpointSP CreateBreakpoint(lldb::addr_t load_addr, bool internal,
                                       bool request_hardware);
 
-  // Use this to create a breakpoint from a load address and a module file spec
+  // Use this to create a breakpoint from a file address and a module file spec
   lldb::BreakpointSP CreateAddressInModuleBreakpoint(lldb::addr_t file_addr,
                                                      bool internal,
                                                      const FileSpec &file_spec,
@@ -752,8 +753,8 @@ class Target : public std::enable_shared_from_this<Target>,
       const FileSpecList *containingModules,
       const FileSpecList *containingSourceFiles, const char *func_name,
       lldb::FunctionNameType func_name_type_mask, lldb::LanguageType language,
-      lldb::addr_t offset, LazyBool skip_prologue, bool internal,
-      bool request_hardware);
+      lldb::addr_t offset, bool offset_is_insn_count, LazyBool skip_prologue,
+      bool internal, bool request_hardware);
 
   lldb::BreakpointSP
   CreateExceptionBreakpoint(enum lldb::LanguageType language, bool catch_bp,
@@ -1334,6 +1335,10 @@ class Target : public std::enable_shared_from_this<Target>,
                                const lldb_private::RegisterFlags &flags,
                                uint32_t byte_size);
 
+  llvm::Expected<lldb::DisassemblerSP>
+  ReadInstructions(const Address &start_addr, uint32_t count,
+                   const char *flavor_string = nullptr);
+
   // Target Stop Hooks
   class StopHook : public UserID {
   public:
diff --git a/lldb/include/lldb/Utility/ArchSpec.h b/lldb/include/lldb/Utility/ArchSpec.h
index 7e9bc23a75acb..96bd5e3597b68 100644
--- a/lldb/include/lldb/Utility/ArchSpec.h
+++ b/lldb/include/lldb/Utility/ArchSpec.h
@@ -564,6 +564,7 @@ class ArchSpec {
 /// \return true if \a lhs is less than \a rhs
 bool operator<(const ArchSpec &lhs, const ArchSpec &rhs);
 bool operator==(const ArchSpec &lhs, const ArchSpec &rhs);
+bool operator!=(const ArchSpec &lhs, const ArchSpec &rhs);
 
 bool ParseMachCPUDashSubtypeTriple(llvm::StringRef triple_str, ArchSpec &arch);
 
diff --git a/lldb/include/lldb/Utility/StructuredData.h b/lldb/include/lldb/Utility/StructuredData.h
index 5e63ef92fac3e..59a2fd60b3989 100644
--- a/lldb/include/lldb/Utility/StructuredData.h
+++ b/lldb/include/lldb/Utility/StructuredData.h
@@ -432,7 +432,7 @@ class StructuredData {
       }
       return success;
     }
-      
+
     template <class IntType>
     bool GetValueForKeyAsInteger(llvm::StringRef key, IntType &result) const {
       ObjectSP value_sp = GetValueForKey(key);
@@ -574,6 +574,30 @@ class StructuredData {
     void *m_object;
   };
 
+  template <typename T> static ObjectSP FromInteger(T value) {
+    return std::make_shared<Integer<T>>(value);
+  }
+
+  static StructuredData::ObjectSP FromFloat(double value) {
+    return std::make_shared<StructuredData::Float>(value);
+  }
+  static StructuredData::ObjectSP FromBoolean(bool value) {
+    return std::make_shared<StructuredData::Boolean>(value);
+  }
+  static StructuredData::ObjectSP FromString(std::string value) {
+    return std::make_shared<StructuredData::String>(value);
+  }
+  static StructuredData::ObjectSP FromGeneric(void *value) {
+    return std::make_shared<StructuredData::Generic>(value);
+  }
+
+  static StructuredData::ObjectSP
+  FromKeyValue(llvm::StringRef key, const StructuredData::ObjectSP &value_sp) {
+    auto dict_sp = std::make_shared<StructuredData::Dictionary>();
+    dict_sp->AddItem(key, value_sp);
+    return dict_sp;
+  }
+
   static ObjectSP ParseJSON(llvm::StringRef json_text);
   static ObjectSP ParseJSONFromFile(const FileSpec &file, Status &error);
   static bool IsRecordType(const ObjectSP object_sp);
diff --git a/lldb/include/lldb/Utility/XcodeSDK.h b/lldb/include/lldb/Utility/XcodeSDK.h
index a1a0ec415b90e..5b345a4965cf9 100644
--- a/lldb/include/lldb/Utility/XcodeSDK.h
+++ b/lldb/include/lldb/Utility/XcodeSDK.h
@@ -86,8 +86,6 @@ class XcodeSDK {
   Type GetType() const;
   llvm::StringRef GetString() const;
   const FileSpec &GetSysroot() const;
-  /// Whether this Xcode SDK supports Swift.
-  bool SupportsSwift() const;
 
   /// Whether LLDB feels confident importing Clang modules from this SDK.
   static bool SDKSupportsModules(Type type, llvm::VersionTuple version);
diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h
index c63c1f03e58da..fec9fdef44df9 100644
--- a/lldb/include/lldb/lldb-enumerations.h
+++ b/lldb/include/lldb/lldb-enumerations.h
@@ -777,6 +777,7 @@ enum SectionType {
   eSectionTypeLLDBTypeSummaries,
   eSectionTypeLLDBFormatters,
   eSectionTypeSwiftModules,
+  eSectionTypeWasmName,
 };
 
 FLAGS_ENUM(EmulateInstructionOptions){
diff --git a/lldb/include/lldb/lldb-private-enumerations.h b/lldb/include/lldb/lldb-private-enumerations.h
index 98c1e956bf8f7..cc4657c87cb0f 100644
--- a/lldb/include/lldb/lldb-private-enumerations.h
+++ b/lldb/include/lldb/lldb-private-enumerations.h
@@ -248,6 +248,22 @@ enum class IterationAction {
   Stop,
 };
 
+/// Specifies the type of PCs when creating a `HistoryThread`.
+/// - `Returns` - Usually, when LLDB unwinds the stack or we retrieve a stack
+///   trace via `backtrace()` we are collecting return addresses (except for the
+///   topmost frame which is the actual PC).  LLDB then maps these return
+///   addresses back to call addresses to give accurate source line annotations.
+/// - `ReturnsNoZerothFrame` - Some trace providers (e.g., libsanitizers traces)
+///   collect return addresses but prune the topmost frames, so we should skip
+///   the special treatment of frame 0.
+/// - `Calls` - Other trace providers (e.g., ASan compiler-rt runtime) already
+///   perform this mapping, so we need to prevent LLDB from doing it again.
+enum class HistoryPCType {
+  Returns,              ///< PCs are return addresses, except for topmost frame.
+  ReturnsNoZerothFrame, ///< All PCs are return addresses.
+  Calls                 ///< PCs are call addresses.
+};
+
 inline std::string GetStatDescription(lldb_private::StatisticKind K) {
    switch (K) {
    case StatisticKind::ExpressionSuccessful:
diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py
index 8112705438c1f..b8a78b71f5ec1 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py
@@ -1464,8 +1464,8 @@ def format(self, value, buffer=None):
         return output.getvalue()
 
 
-class RecursiveDecentFormatter(BasicFormatter):
-    """The recursive decent formatter prints the value and the decendents.
+class RecursiveDescentFormatter(BasicFormatter):
+    """The recursive descent formatter prints the value and the descendents.
 
     The constructor takes two keyword args: indent_level, which defaults to 0,
     and indent_child, which defaults to 2.  The current indentation level is
@@ -1482,7 +1482,6 @@ def format(self, value, buffer=None):
             output = io.StringIO()
         else:
             output = buffer
-
         BasicFormatter.format(self, value, buffer=output, indent=self.lindent)
         new_indent = self.lindent + self.cindent
         for child in value:
@@ -1490,7 +1489,7 @@ def format(self, value, buffer=None):
                 BasicFormatter.format(self, child, buffer=output, indent=new_indent)
             else:
                 if child.GetNumChildren() > 0:
-                    rdf = RecursiveDecentFormatter(indent_level=new_indent)
+                    rdf = RecursiveDescentFormatter(indent_level=new_indent)
                     rdf.format(child, buffer=output)
                 else:
                     BasicFormatter.format(self, child, buffer=output, indent=new_indent)
diff --git a/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h b/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h
index 35649b10ce45e..491acb52be87d 100644
--- a/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h
+++ b/lldb/packages/Python/lldbsuite/test/make/libcxx-simulators-common/compressed_pair.h
@@ -4,6 +4,12 @@
 #include <type_traits>
 #include <utility> // for std::forward
 
+// COMPRESSED_PAIR_REV versions:
+// 0 -> Post-c88580c layout
+// 1 -> Post-27c83382d83dc layout
+// 2 -> Post-769c42f4a552a layout
+// 3 -> padding-less no_unique_address-based layout (introduced in 27c83382d83dc)
+
 namespace std {
 namespace __lldb {
 
@@ -13,7 +19,50 @@ namespace __lldb {
 #define _LLDB_NO_UNIQUE_ADDRESS [[__no_unique_address__]]
 #endif
 
-#if COMPRESSED_PAIR_REV == 0 // Post-c88580c layout
+// From libc++ datasizeof.h
+template <class _Tp> struct _FirstPaddingByte {
+  _LLDB_NO_UNIQUE_ADDRESS _Tp __v_;
+  char __first_padding_byte_;
+};
+
+template <class _Tp>
+inline const size_t __datasizeof_v =
+    __builtin_offsetof(_FirstPaddingByte<_Tp>, __first_padding_byte_);
+
+template <class _Tp>
+struct __lldb_is_final : public integral_constant<bool, __is_final(_Tp)> {};
+
+// The legacy layout has been patched, see
+// https://github.com/llvm/llvm-project/pull/142516.
+#if COMPRESSED_PAIR_REV == 1
+template <class _ToPad> class __compressed_pair_padding {
+  char __padding_[((is_empty<_ToPad>::value &&
+                    !__lldb_is_final<_ToPad>::value) ||
+                   is_reference<_ToPad>::value)
+                      ? 0
+                      : sizeof(_ToPad) - __datasizeof_v<_ToPad>];
+};
+#elif COMPRESSED_PAIR_REV > 1 && COMPRESSED_PAIR_REV < 3
+template <class _ToPad>
+inline const bool __is_reference_or_unpadded_object =
+    (std::is_empty<_ToPad>::value && !__lldb_is_final<_ToPad>::value) ||
+    sizeof(_ToPad) == __datasizeof_v<_ToPad>;
+
+template <class _Tp>
+inline const bool __is_reference_or_unpadded_object<_Tp &> = true;
+
+template <class _Tp>
+inline const bool __is_reference_or_unpadded_object<_Tp &&> = true;
+
+template <class _ToPad, bool _Empty = __is_reference_or_unpadded_object<_ToPad>>
+class __compressed_pair_padding {
+  char __padding_[sizeof(_ToPad) - __datasizeof_v<_ToPad>] = {};
+};
+
+template <class _ToPad> class __compressed_pair_padding<_ToPad, true> {};
+#endif // COMPRESSED_PAIR_REV == 1
+
+#if COMPRESSED_PAIR_REV == 0
 struct __value_init_tag {};
 struct __default_init_tag {};
 
@@ -59,49 +108,6 @@ class __compressed_pair : private __compressed_pair_elem<_T1, 0>,
   _T1 &first() { return static_cast<_Base1 &>(*this).__get(); }
 };
 #elif COMPRESSED_PAIR_REV == 1 || COMPRESSED_PAIR_REV == 2
-// From libc++ datasizeof.h
-template <class _Tp> struct _FirstPaddingByte {
-  _LLDB_NO_UNIQUE_ADDRESS _Tp __v_;
-  char __first_padding_byte_;
-};
-
-template <class _Tp>
-inline const size_t __datasizeof_v =
-    __builtin_offsetof(_FirstPaddingByte<_Tp>, __first_padding_byte_);
-
-template <class _Tp>
-struct __lldb_is_final : public integral_constant<bool, __is_final(_Tp)> {};
-
-// The legacy layout has been patched, see
-// https://github.com/llvm/llvm-project/pull/142516.
-#if COMPRESSED_PAIR_REV == 1
-template <class _ToPad> class __compressed_pair_padding {
-  char __padding_[((is_empty<_ToPad>::value &&
-                    !__lldb_is_final<_ToPad>::value) ||
-                   is_reference<_ToPad>::value)
-                      ? 0
-                      : sizeof(_ToPad) - __datasizeof_v<_ToPad>];
-};
-#else
-template <class _ToPad>
-inline const bool __is_reference_or_unpadded_object =
-    (std::is_empty<_ToPad>::value && !__lldb_is_final<_ToPad>::value) ||
-    sizeof(_ToPad) == __datasizeof_v<_ToPad>;
-
-template <class _Tp>
-inline const bool __is_reference_or_unpadded_object<_Tp &> = true;
-
-template <class _Tp>
-inline const bool __is_reference_or_unpadded_object<_Tp &&> = true;
-
-template <class _ToPad, bool _Empty = __is_reference_or_unpadded_object<_ToPad>>
-class __compressed_pair_padding {
-  char __padding_[sizeof(_ToPad) - __datasizeof_v<_ToPad>] = {};
-};
-
-template <class _ToPad> class __compressed_pair_padding<_ToPad, true> {};
-#endif
-
 #define _LLDB_COMPRESSED_PAIR(T1, Initializer1, T2, Initializer2)              \
   [[__gnu__::__aligned__(                                                      \
       alignof(T2))]] _LLDB_NO_UNIQUE_ADDRESS T1 Initializer1;                  \
@@ -127,7 +133,7 @@ template <class _ToPad> class __compressed_pair_padding<_ToPad, true> {};
   _LLDB_NO_UNIQUE_ADDRESS T1 Name1;                                            \
   _LLDB_NO_UNIQUE_ADDRESS T2 Name2;                                            \
   _LLDB_NO_UNIQUE_ADDRESS T3 Name3
-#endif
+#endif // COMPRESSED_PAIR_REV == 3
 } // namespace __lldb
 } // namespace std
 
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
index 0b09893c7ed5b..0608ac3fd83be 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py
@@ -12,15 +12,91 @@
 import sys
 import threading
 import time
-from typing import Any, Optional, Union, BinaryIO, TextIO
+from typing import (
+    Any,
+    Optional,
+    Dict,
+    cast,
+    List,
+    Callable,
+    IO,
+    Union,
+    BinaryIO,
+    TextIO,
+    TypedDict,
+    Literal,
+)
 
 ## DAP type references
-Event = dict[str, Any]
-Request = dict[str, Any]
-Response = dict[str, Any]
+
+
+class Event(TypedDict):
+    type: Literal["event"]
+    seq: int
+    event: str
+    body: Any
+
+
+class Request(TypedDict, total=False):
+    type: Literal["request"]
+    seq: int
+    command: str
+    arguments: Any
+
+
+class Response(TypedDict):
+    type: Literal["response"]
+    seq: int
+    request_seq: int
+    success: bool
+    command: str
+    message: Optional[str]
+    body: Any
+
+
 ProtocolMessage = Union[Event, Request, Response]
 
 
+class Source(TypedDict, total=False):
+    name: str
+    path: str
+    sourceReference: int
+
+    @staticmethod
+    def build(
+        *,
+        name: Optional[str] = None,
+        path: Optional[str] = None,
+        source_reference: Optional[int] = None,
+    ) -> "Source":
+        """Builds a source from the given name, path or source_reference."""
+        if not name and not path and not source_reference:
+            raise ValueError(
+                "Source.build requires either name, path, or source_reference"
+            )
+
+        s = Source()
+        if name:
+            s["name"] = name
+        if path:
+            if not name:
+                s["name"] = os.path.basename(path)
+            s["path"] = path
+        if source_reference is not None:
+            s["sourceReference"] = source_reference
+        return s
+
+
+class Breakpoint(TypedDict, total=False):
+    id: int
+    verified: bool
+    source: Source
+
+    @staticmethod
+    def is_verified(src: "Breakpoint") -> bool:
+        return src.get("verified", False)
+
+
 def dump_memory(base_addr, data, num_per_line, outfile):
     data_len = len(data)
     hex_string = binascii.hexlify(data)
@@ -58,7 +134,9 @@ def dump_memory(base_addr, data, num_per_line, outfile):
         outfile.write("\n")
 
 
-def read_packet(f, verbose=False, trace_file=None):
+def read_packet(
+    f: IO[bytes], trace_file: Optional[IO[str]] = None
+) -> Optional[ProtocolMessage]:
     """Decode a JSON packet that starts with the content length and is
     followed by the JSON bytes from a file 'f'. Returns None on EOF.
     """
@@ -70,19 +148,13 @@ def read_packet(f, verbose=False, trace_file=None):
     prefix = "Content-Length: "
     if line.startswith(prefix):
         # Decode length of JSON bytes
-        if verbose:
-            print('content: "%s"' % (line))
         length = int(line[len(prefix) :])
-        if verbose:
-            print('length: "%u"' % (length))
         # Skip empty line
-        line = f.readline()
-        if verbose:
-            print('empty: "%s"' % (line))
+        separator = f.readline().decode()
+        if separator != "":
+            Exception("malformed DAP content header, unexpected line: " + separator)
         # Read JSON bytes
-        json_str = f.read(length)
-        if verbose:
-            print('json: "%s"' % (json_str))
+        json_str = f.read(length).decode()
         if trace_file:
             trace_file.write("from adapter:\n%s\n" % (json_str))
         # Decode the JSON bytes into a python dictionary
@@ -95,7 +167,7 @@ def packet_type_is(packet, packet_type):
     return "type" in packet and packet["type"] == packet_type
 
 
-def dump_dap_log(log_file):
+def dump_dap_log(log_file: Optional[str]) -> None:
     print("========= DEBUG ADAPTER PROTOCOL LOGS =========", file=sys.stderr)
     if log_file is None:
         print("no log file available", file=sys.stderr)
@@ -105,36 +177,6 @@ def dump_dap_log(log_file):
     print("========= END =========", file=sys.stderr)
 
 
-class Source(object):
-    def __init__(
-        self, path: Optional[str] = None, source_reference: Optional[int] = None
-    ):
-        self._name = None
-        self._path = None
-        self._source_reference = None
-
-        if path is not None:
-            self._name = os.path.basename(path)
-            self._path = path
-        elif source_reference is not None:
-            self._source_reference = source_reference
-        else:
-            raise ValueError("Either path or source_reference must be provided")
-
-    def __str__(self):
-        return f"Source(name={self.name}, path={self.path}), source_reference={self.source_reference})"
-
-    def as_dict(self):
-        source_dict = {}
-        if self._name is not None:
-            source_dict["name"] = self._name
-        if self._path is not None:
-            source_dict["path"] = self._path
-        if self._source_reference is not None:
-            source_dict["sourceReference"] = self._source_reference
-        return source_dict
-
-
 class NotSupportedError(KeyError):
     """Raised if a feature is not supported due to its capabilities."""
 
@@ -152,25 +194,41 @@ def __init__(
         self.log_file = log_file
         self.send = send
         self.recv = recv
-        self.recv_packets: list[Optional[ProtocolMessage]] = []
-        self.recv_condition = threading.Condition()
-        self.recv_thread = threading.Thread(target=self._read_packet_thread)
-        self.process_event_body = None
-        self.exit_status: Optional[int] = None
-        self.capabilities: dict[str, Any] = {}
-        self.progress_events: list[Event] = []
-        self.reverse_requests = []
-        self.sequence = 1
-        self.threads = None
-        self.thread_stop_reasons = {}
-        self.recv_thread.start()
-        self.output_condition = threading.Condition()
-        self.output: dict[str, list[str]] = {}
-        self.configuration_done_sent = False
-        self.initialized = False
-        self.frame_scopes = {}
+
+        # Packets that have been received and processed but have not yet been
+        # requested by a test case.
+        self._pending_packets: List[Optional[ProtocolMessage]] = []
+        # Received packets that have not yet been processed.
+        self._recv_packets: List[Optional[ProtocolMessage]] = []
+        # Used as a mutex for _recv_packets and for notify when _recv_packets
+        # changes.
+        self._recv_condition = threading.Condition()
+        self._recv_thread = threading.Thread(target=self._read_packet_thread)
+
+        # session state
         self.init_commands = init_commands
-        self.resolved_breakpoints = {}
+        self.exit_status: Optional[int] = None
+        self.capabilities: Dict = {}
+        self.initialized: bool = False
+        self.configuration_done_sent: bool = False
+        self.process_event_body: Optional[Dict] = None
+        self.terminated: bool = False
+        self.events: List[Event] = []
+        self.progress_events: List[Event] = []
+        self.reverse_requests: List[Request] = []
+        self.module_events: List[Dict] = []
+        self.sequence: int = 1
+        self.output: Dict[str, str] = {}
+
+        # debuggee state
+        self.threads: Optional[dict] = None
+        self.thread_stop_reasons: Dict[str, Any] = {}
+        self.frame_scopes: Dict[str, Any] = {}
+        # keyed by breakpoint id
+        self.resolved_breakpoints: dict[str, Breakpoint] = {}
+
+        # trigger enqueue thread
+        self._recv_thread.start()
 
     @classmethod
     def encode_content(cls, s: str) -> bytes:
@@ -188,267 +246,324 @@ def validate_response(cls, command, response):
             )
 
     def _read_packet_thread(self):
-        done = False
         try:
-            while not done:
+            while True:
                 packet = read_packet(self.recv, trace_file=self.trace_file)
                 # `packet` will be `None` on EOF. We want to pass it down to
                 # handle_recv_packet anyway so the main thread can handle unexpected
                 # termination of lldb-dap and stop waiting for new packets.
-                done = not self._handle_recv_packet(packet)
+                if not self._handle_recv_packet(packet):
+                    break
         finally:
             dump_dap_log(self.log_file)
 
-    def get_modules(self, startModule: int = 0, moduleCount: int = 0):
-        module_list = self.request_modules(startModule, moduleCount)["body"]["modules"]
+    def get_modules(
+        self, start_module: Optional[int] = None, module_count: Optional[int] = None
+    ) -> Dict:
+        resp = self.request_modules(start_module, module_count)
+        if not resp["success"]:
+            raise ValueError(f"request_modules failed: {resp!r}")
         modules = {}
+        module_list = resp["body"]["modules"]
         for module in module_list:
             modules[module["name"]] = module
         return modules
 
-    def get_output(self, category, timeout=0.0, clear=True):
-        self.output_condition.acquire()
-        output = None
+    def get_output(self, category: str, clear=True) -> str:
+        output = ""
         if category in self.output:
-            output = self.output[category]
+            output = self.output.get(category, "")
             if clear:
                 del self.output[category]
-        elif timeout != 0.0:
-            self.output_condition.wait(timeout)
-            if category in self.output:
-                output = self.output[category]
-                if clear:
-                    del self.output[category]
-        self.output_condition.release()
         return output
 
-    def collect_output(self, category, timeout_secs, pattern, clear=True):
-        end_time = time.time() + timeout_secs
-        collected_output = ""
-        while end_time > time.time():
-            output = self.get_output(category, timeout=0.25, clear=clear)
-            if output:
-                collected_output += output
-                if pattern is not None and pattern in output:
-                    break
-        return collected_output if collected_output else None
+    def collect_output(
+        self,
+        category: str,
+        timeout: float,
+        pattern: Optional[str] = None,
+        clear=True,
+    ) -> str:
+        """Collect output from 'output' events.
+        Args:
+            category: The category to collect.
+            timeout: The max duration for collecting output.
+            pattern:
+                Optional, if set, return once this pattern is detected in the
+                collected output.
+        Returns:
+            The collected output.
+        """
+        deadline = time.monotonic() + timeout
+        output = self.get_output(category, clear)
+        while deadline >= time.monotonic() and (
+            pattern is None or pattern not in output
+        ):
+            event = self.wait_for_event(["output"], timeout=deadline - time.monotonic())
+            if not event:  # Timeout or EOF
+                break
+            output += self.get_output(category, clear=clear)
+        return output
 
     def _enqueue_recv_packet(self, packet: Optional[ProtocolMessage]):
-        self.recv_condition.acquire()
-        self.recv_packets.append(packet)
-        self.recv_condition.notify()
-        self.recv_condition.release()
+        with self.recv_condition:
+            self.recv_packets.append(packet)
+            self.recv_condition.notify()
 
     def _handle_recv_packet(self, packet: Optional[ProtocolMessage]) -> bool:
-        """Called by the read thread that is waiting for all incoming packets
-        to store the incoming packet in "self.recv_packets" in a thread safe
-        way. This function will then signal the "self.recv_condition" to
-        indicate a new packet is available. Returns True if the caller
-        should keep calling this function for more packets.
+        """Handles an incoming packet.
+
+        Called by the read thread that is waiting for all incoming packets
+        to store the incoming packet in "self._recv_packets" in a thread safe
+        way. This function will then signal the "self._recv_condition" to
+        indicate a new packet is available.
+
+        Args:
+            packet: A new packet to store.
+
+        Returns:
+            True if the caller should keep calling this function for more
+            packets.
         """
-        # If EOF, notify the read thread by enqueuing a None.
-        if not packet:
-            self._enqueue_recv_packet(None)
-            return False
-
-        # Check the packet to see if is an event packet
-        keepGoing = True
-        packet_type = packet["type"]
-        if packet_type == "event":
-            event = packet["event"]
-            body = None
-            if "body" in packet:
-                body = packet["body"]
-            # Handle the event packet and cache information from these packets
-            # as they come in
-            if event == "output":
-                # Store any output we receive so clients can retrieve it later.
-                category = body["category"]
-                output = body["output"]
-                self.output_condition.acquire()
-                if category in self.output:
-                    self.output[category] += output
-                else:
-                    self.output[category] = output
-                self.output_condition.notify()
-                self.output_condition.release()
-                # no need to add 'output' event packets to our packets list
-                return keepGoing
-            elif event == "initialized":
-                self.initialized = True
-            elif event == "process":
-                # When a new process is attached or launched, remember the
-                # details that are available in the body of the event
-                self.process_event_body = body
-            elif event == "exited":
-                # Process exited, mark the status to indicate the process is not
-                # alive.
-                self.exit_status = body["exitCode"]
-            elif event == "continued":
-                # When the process continues, clear the known threads and
-                # thread_stop_reasons.
-                all_threads_continued = body.get("allThreadsContinued", True)
-                tid = body["threadId"]
-                if tid in self.thread_stop_reasons:
-                    del self.thread_stop_reasons[tid]
-                self._process_continued(all_threads_continued)
-            elif event == "stopped":
-                # Each thread that stops with a reason will send a
-                # 'stopped' event. We need to remember the thread stop
-                # reasons since the 'threads' command doesn't return
-                # that information.
-                self._process_stopped()
-                tid = body["threadId"]
-                self.thread_stop_reasons[tid] = body
-            elif event.startswith("progress"):
-                # Progress events come in as 'progressStart', 'progressUpdate',
-                # and 'progressEnd' events. Keep these around in case test
-                # cases want to verify them.
-                self.progress_events.append(packet)
-            elif event == "breakpoint":
-                # Breakpoint events are sent when a breakpoint is resolved
-                self._update_verified_breakpoints([body["breakpoint"]])
-            elif event == "capabilities":
-                # Update the capabilities with new ones from the event.
-                self.capabilities.update(body["capabilities"])
-
-        elif packet_type == "response":
-            if packet["command"] == "disconnect":
-                keepGoing = False
-        self._enqueue_recv_packet(packet)
-        return keepGoing
+        with self._recv_condition:
+            self._recv_packets.append(packet)
+            self._recv_condition.notify()
+            # packet is None on EOF
+            return packet is not None and not (
+                packet["type"] == "response" and packet["command"] == "disconnect"
+            )
+
+    def _recv_packet(
+        self,
+        *,
+        predicate: Optional[Callable[[ProtocolMessage], bool]] = None,
+        timeout: Optional[float] = None,
+    ) -> Optional[ProtocolMessage]:
+        """Processes received packets from the adapter.
+        Updates the DebugCommunication stateful properties based on the received
+        packets in the order they are received.
+        NOTE: The only time the session state properties should be updated is
+        during this call to ensure consistency during tests.
+        Args:
+            predicate:
+                Optional, if specified, returns the first packet that matches
+                the given predicate.
+            timeout:
+                Optional, if specified, processes packets until either the
+                timeout occurs or the predicate matches a packet, whichever
+                occurs first.
+        Returns:
+            The first matching packet for the given predicate, if specified,
+            otherwise None.
+        """
+        assert (
+            threading.current_thread != self._recv_thread
+        ), "Must not be called from the _recv_thread"
+
+        def process_until_match():
+            self._process_recv_packets()
+            for i, packet in enumerate(self._pending_packets):
+                if packet is None:
+                    # We need to return a truthy value to break out of the
+                    # wait_for, use `EOFError` as an indicator of EOF.
+                    return EOFError()
+                if predicate and predicate(packet):
+                    self._pending_packets.pop(i)
+                    return packet
+
+        with self._recv_condition:
+            packet = self._recv_condition.wait_for(process_until_match, timeout)
+            return None if isinstance(packet, EOFError) else packet
+
+    def _process_recv_packets(self) -> None:
+        """Process received packets, updating the session state."""
+        with self._recv_condition:
+            for packet in self._recv_packets:
+                # Handle events that may modify any stateful properties of
+                # the DAP session.
+                if packet and packet["type"] == "event":
+                    self._handle_event(packet)
+                elif packet and packet["type"] == "request":
+                    # Handle reverse requests and keep processing.
+                    self._handle_reverse_request(packet)
+                # Move the packet to the pending queue.
+                self._pending_packets.append(packet)
+            self._recv_packets.clear()
+
+    def _handle_event(self, packet: Event) -> None:
+        """Handle any events that modify debug session state we track."""
+        event = packet["event"]
+        body: Optional[Dict] = packet.get("body", None)
+
+        if event == "output" and body:
+            # Store any output we receive so clients can retrieve it later.
+            category = body["category"]
+            output = body["output"]
+            if category in self.output:
+                self.output[category] += output
+            else:
+                self.output[category] = output
+        elif event == "initialized":
+            self.initialized = True
+        elif event == "process":
+            # When a new process is attached or launched, remember the
+            # details that are available in the body of the event
+            self.process_event_body = body
+        elif event == "exited" and body:
+            # Process exited, mark the status to indicate the process is not
+            # alive.
+            self.exit_status = body["exitCode"]
+        elif event == "continued" and body:
+            # When the process continues, clear the known threads and
+            # thread_stop_reasons.
+            all_threads_continued = body.get("allThreadsContinued", True)
+            tid = body["threadId"]
+            if tid in self.thread_stop_reasons:
+                del self.thread_stop_reasons[tid]
+            self._process_continued(all_threads_continued)
+        elif event == "stopped" and body:
+            # Each thread that stops with a reason will send a
+            # 'stopped' event. We need to remember the thread stop
+            # reasons since the 'threads' command doesn't return
+            # that information.
+            self._process_stopped()
+            tid = body["threadId"]
+            self.thread_stop_reasons[tid] = body
+        elif event.startswith("progress"):
+            # Progress events come in as 'progressStart', 'progressUpdate',
+            # and 'progressEnd' events. Keep these around in case test
+            # cases want to verify them.
+            self.progress_events.append(packet)
+        elif event == "breakpoint" and body:
+            # Breakpoint events are sent when a breakpoint is resolved
+            self._update_verified_breakpoints([body["breakpoint"]])
+        elif event == "capabilities" and body:
+            # Update the capabilities with new ones from the event.
+            self.capabilities.update(body["capabilities"])
+
+    def _handle_reverse_request(self, request: Request) -> None:
+        if request in self.reverse_requests:
+            return
+        self.reverse_requests.append(request)
+        arguments = request.get("arguments")
+        if request["command"] == "runInTerminal" and arguments is not None:
+            in_shell = arguments.get("argsCanBeInterpretedByShell", False)
+            print("spawning...", arguments["args"])
+            proc = subprocess.Popen(
+                arguments["args"],
+                env=arguments.get("env", {}),
+                cwd=arguments.get("cwd", None),
+                stdin=subprocess.DEVNULL,
+                stdout=sys.stderr,
+                stderr=sys.stderr,
+                shell=in_shell,
+            )
+            body = {}
+            if in_shell:
+                body["shellProcessId"] = proc.pid
+            else:
+                body["processId"] = proc.pid
+            self.send_packet(
+                {
+                    "type": "response",
+                    "seq": 0,
+                    "request_seq": request["seq"],
+                    "success": True,
+                    "command": "runInTerminal",
+                    "body": body,
+                }
+            )
+        elif request["command"] == "startDebugging":
+            self.send_packet(
+                {
+                    "type": "response",
+                    "seq": 0,
+                    "request_seq": request["seq"],
+                    "success": True,
+                    "message": None,
+                    "command": "startDebugging",
+                    "body": {},
+                }
+            )
+        else:
+            desc = 'unknown reverse request "%s"' % (request["command"])
+            raise ValueError(desc)
 
     def _process_continued(self, all_threads_continued: bool):
         self.frame_scopes = {}
         if all_threads_continued:
             self.thread_stop_reasons = {}
 
-    def _update_verified_breakpoints(self, breakpoints: list[Event]):
-        for breakpoint in breakpoints:
-            if "id" in breakpoint:
-                self.resolved_breakpoints[str(breakpoint["id"])] = breakpoint.get(
-                    "verified", False
-                )
+    def _update_verified_breakpoints(self, breakpoints: list[Breakpoint]):
+        for bp in breakpoints:
+            # If no id is set, we cannot correlate the given breakpoint across
+            # requests, ignore it.
+            if "id" not in bp:
+                continue
+
+            self.resolved_breakpoints[str(bp["id"])] = bp
+
+    def send_packet(self, packet: ProtocolMessage) -> int:
+        """Takes a dictionary representation of a DAP request and send the request to the debug adapter.
 
-    def send_packet(self, command_dict: Request, set_sequence=True):
-        """Take the "command_dict" python dictionary and encode it as a JSON
-        string and send the contents as a packet to the VSCode debug
-        adapter"""
-        # Set the sequence ID for this command automatically
-        if set_sequence:
-            command_dict["seq"] = self.sequence
+        Returns the seq number of the request.
+        """
+        # Set the seq for requests.
+        if packet["type"] == "request":
+            packet["seq"] = self.sequence
             self.sequence += 1
+        else:
+            packet["seq"] = 0
+
         # Encode our command dictionary as a JSON string
-        json_str = json.dumps(command_dict, separators=(",", ":"))
+        json_str = json.dumps(packet, separators=(",", ":"))
+
         if self.trace_file:
             self.trace_file.write("to adapter:\n%s\n" % (json_str))
+
         length = len(json_str)
         if length > 0:
             # Send the encoded JSON packet and flush the 'send' file
             self.send.write(self.encode_content(json_str))
             self.send.flush()
 
-    def recv_packet(
-        self,
-        filter_type: Optional[str] = None,
-        filter_event: Optional[Union[str, list[str]]] = None,
-        timeout: Optional[float] = None,
-    ) -> Optional[ProtocolMessage]:
-        """Get a JSON packet from the VSCode debug adapter. This function
-        assumes a thread that reads packets is running and will deliver
-        any received packets by calling handle_recv_packet(...). This
-        function will wait for the packet to arrive and return it when
-        it does."""
-        while True:
-            try:
-                self.recv_condition.acquire()
-                packet = None
-                while True:
-                    for i, curr_packet in enumerate(self.recv_packets):
-                        if not curr_packet:
-                            raise EOFError
-                        packet_type = curr_packet["type"]
-                        if filter_type is None or packet_type in filter_type:
-                            if filter_event is None or (
-                                packet_type == "event"
-                                and curr_packet["event"] in filter_event
-                            ):
-                                packet = self.recv_packets.pop(i)
-                                break
-                    if packet:
-                        break
-                    # Sleep until packet is received
-                    len_before = len(self.recv_packets)
-                    self.recv_condition.wait(timeout)
-                    len_after = len(self.recv_packets)
-                    if len_before == len_after:
-                        return None  # Timed out
-                return packet
-            except EOFError:
-                return None
-            finally:
-                self.recv_condition.release()
-
-    def send_recv(self, command):
+        return packet["seq"]
+
+    def _send_recv(self, request: Request) -> Optional[Response]:
         """Send a command python dictionary as JSON and receive the JSON
         response. Validates that the response is the correct sequence and
         command in the reply. Any events that are received are added to the
         events list in this object"""
-        self.send_packet(command)
-        done = False
-        while not done:
-            response_or_request = self.recv_packet(filter_type=["response", "request"])
-            if response_or_request is None:
-                desc = 'no response for "%s"' % (command["command"])
-                raise ValueError(desc)
-            if response_or_request["type"] == "response":
-                self.validate_response(command, response_or_request)
-                return response_or_request
-            else:
-                self.reverse_requests.append(response_or_request)
-                if response_or_request["command"] == "runInTerminal":
-                    subprocess.Popen(
-                        response_or_request["arguments"].get("args"),
-                        env=response_or_request["arguments"].get("env", {}),
-                    )
-                    self.send_packet(
-                        {
-                            "type": "response",
-                            "request_seq": response_or_request["seq"],
-                            "success": True,
-                            "command": "runInTerminal",
-                            "body": {},
-                        },
-                    )
-                elif response_or_request["command"] == "startDebugging":
-                    self.send_packet(
-                        {
-                            "type": "response",
-                            "request_seq": response_or_request["seq"],
-                            "success": True,
-                            "command": "startDebugging",
-                            "body": {},
-                        },
-                    )
-                else:
-                    desc = 'unknown reverse request "%s"' % (
-                        response_or_request["command"]
-                    )
-                    raise ValueError(desc)
+        seq = self.send_packet(request)
+        response = self.receive_response(seq)
+        if response is None:
+            raise ValueError(f"no response for {request!r}")
+        self.validate_response(request, response)
+        return response
 
-        return None
+    def receive_response(self, seq: int) -> Optional[Response]:
+        """Waits for a response with the associated request_sec."""
+
+        def predicate(p: ProtocolMessage):
+            return p["type"] == "response" and p["request_seq"] == seq
+
+        return cast(Optional[Response], self._recv_packet(predicate=predicate))
 
     def wait_for_event(
-        self, filter: Union[str, list[str]], timeout: Optional[float] = None
+        self, filter: List[str] = [], timeout: Optional[float] = None
     ) -> Optional[Event]:
         """Wait for the first event that matches the filter."""
-        return self.recv_packet(
-            filter_type="event", filter_event=filter, timeout=timeout
+
+        def predicate(p: ProtocolMessage):
+            return p["type"] == "event" and p["event"] in filter
+
+        return cast(
+            Optional[Event], self._recv_packet(predicate=predicate, timeout=timeout)
         )
 
     def wait_for_stopped(
         self, timeout: Optional[float] = None
-    ) -> Optional[list[Event]]:
+    ) -> Optional[List[Event]]:
         stopped_events = []
         stopped_event = self.wait_for_event(
             filter=["stopped", "exited"], timeout=timeout
@@ -469,7 +584,7 @@ def wait_for_stopped(
     def wait_for_breakpoint_events(self, timeout: Optional[float] = None):
         breakpoint_events: list[Event] = []
         while True:
-            event = self.wait_for_event("breakpoint", timeout=timeout)
+            event = self.wait_for_event(["breakpoint"], timeout=timeout)
             if not event:
                 break
             breakpoint_events.append(event)
@@ -480,20 +595,27 @@ def wait_for_breakpoints_to_be_verified(
     ):
         """Wait for all breakpoints to be verified. Return all unverified breakpoints."""
         while any(id not in self.resolved_breakpoints for id in breakpoint_ids):
-            breakpoint_event = self.wait_for_event("breakpoint", timeout=timeout)
+            breakpoint_event = self.wait_for_event(["breakpoint"], timeout=timeout)
             if breakpoint_event is None:
                 break
 
-        return [id for id in breakpoint_ids if id not in self.resolved_breakpoints]
+        return [
+            id
+            for id in breakpoint_ids
+            if (
+                id not in self.resolved_breakpoints
+                or not Breakpoint.is_verified(self.resolved_breakpoints[id])
+            )
+        ]
 
     def wait_for_exited(self, timeout: Optional[float] = None):
-        event_dict = self.wait_for_event("exited", timeout=timeout)
+        event_dict = self.wait_for_event(["exited"], timeout=timeout)
         if event_dict is None:
             raise ValueError("didn't get exited event")
         return event_dict
 
     def wait_for_terminated(self, timeout: Optional[float] = None):
-        event_dict = self.wait_for_event("terminated", timeout)
+        event_dict = self.wait_for_event(["terminated"], timeout)
         if event_dict is None:
             raise ValueError("didn't get terminated event")
         return event_dict
@@ -704,7 +826,7 @@ def request_attach(
         if gdbRemoteHostname is not None:
             args_dict["gdb-remote-hostname"] = gdbRemoteHostname
         command_dict = {"command": "attach", "type": "request", "arguments": args_dict}
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_breakpointLocations(
         self, file_path, line, end_line=None, column=None, end_column=None
@@ -726,7 +848,7 @@ def request_breakpointLocations(
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_configurationDone(self):
         command_dict = {
@@ -734,7 +856,7 @@ def request_configurationDone(self):
             "type": "request",
             "arguments": {},
         }
-        response = self.send_recv(command_dict)
+        response = self._send_recv(command_dict)
         if response:
             self.configuration_done_sent = True
             self.request_threads()
@@ -763,7 +885,7 @@ def request_continue(self, threadId=None, singleThread=False):
             "type": "request",
             "arguments": args_dict,
         }
-        response = self.send_recv(command_dict)
+        response = self._send_recv(command_dict)
         if response["success"]:
             self._process_continued(response["body"]["allThreadsContinued"])
         # Caller must still call wait_for_stopped.
@@ -780,7 +902,7 @@ def request_restart(self, restartArguments=None):
         if restartArguments:
             command_dict["arguments"] = restartArguments
 
-        response = self.send_recv(command_dict)
+        response = self._send_recv(command_dict)
         # Caller must still call wait_for_stopped.
         return response
 
@@ -796,7 +918,7 @@ def request_disconnect(self, terminateDebuggee=None):
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_disassemble(
         self,
@@ -816,7 +938,7 @@ def request_disassemble(
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)["body"]["instructions"]
+        return self._send_recv(command_dict)["body"]["instructions"]
 
     def request_readMemory(self, memoryReference, offset, count):
         args_dict = {
@@ -829,7 +951,7 @@ def request_readMemory(self, memoryReference, offset, count):
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_writeMemory(self, memoryReference, data, offset=0, allowPartial=False):
         args_dict = {
@@ -847,7 +969,7 @@ def request_writeMemory(self, memoryReference, data, offset=0, allowPartial=Fals
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_evaluate(self, expression, frameIndex=0, threadId=None, context=None):
         stackFrame = self.get_stackFrame(frameIndex=frameIndex, threadId=threadId)
@@ -863,7 +985,7 @@ def request_evaluate(self, expression, frameIndex=0, threadId=None, context=None
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_exceptionInfo(self, threadId=None):
         if threadId is None:
@@ -874,7 +996,7 @@ def request_exceptionInfo(self, threadId=None):
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_initialize(self, sourceInitFile=False):
         command_dict = {
@@ -895,10 +1017,10 @@ def request_initialize(self, sourceInitFile=False):
                 "$__lldb_sourceInitFile": sourceInitFile,
             },
         }
-        response = self.send_recv(command_dict)
+        response = self._send_recv(command_dict)
         if response:
             if "body" in response:
-                self.capabilities = response["body"]
+                self.capabilities.update(response.get("body", {}))
         return response
 
     def request_launch(
@@ -978,14 +1100,14 @@ def request_launch(
         if commandEscapePrefix is not None:
             args_dict["commandEscapePrefix"] = commandEscapePrefix
         command_dict = {"command": "launch", "type": "request", "arguments": args_dict}
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_next(self, threadId, granularity="statement"):
         if self.exit_status is not None:
             raise ValueError("request_continue called after process exited")
         args_dict = {"threadId": threadId, "granularity": granularity}
         command_dict = {"command": "next", "type": "request", "arguments": args_dict}
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_stepIn(self, threadId, targetId, granularity="statement"):
         if self.exit_status is not None:
@@ -998,7 +1120,7 @@ def request_stepIn(self, threadId, targetId, granularity="statement"):
             "granularity": granularity,
         }
         command_dict = {"command": "stepIn", "type": "request", "arguments": args_dict}
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_stepInTargets(self, frameId):
         if self.exit_status is not None:
@@ -1010,14 +1132,14 @@ def request_stepInTargets(self, frameId):
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_stepOut(self, threadId):
         if self.exit_status is not None:
             raise ValueError("request_stepOut called after process exited")
         args_dict = {"threadId": threadId}
         command_dict = {"command": "stepOut", "type": "request", "arguments": args_dict}
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_pause(self, threadId=None):
         if self.exit_status is not None:
@@ -1026,12 +1148,12 @@ def request_pause(self, threadId=None):
             threadId = self.get_thread_id()
         args_dict = {"threadId": threadId}
         command_dict = {"command": "pause", "type": "request", "arguments": args_dict}
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_scopes(self, frameId):
         args_dict = {"frameId": frameId}
         command_dict = {"command": "scopes", "type": "request", "arguments": args_dict}
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_setBreakpoints(self, source: Source, line_array, data=None):
         """data is array of parameters for breakpoints in line_array.
@@ -1039,7 +1161,7 @@ def request_setBreakpoints(self, source: Source, line_array, data=None):
         It contains optional location/hitCondition/logMessage parameters.
         """
         args_dict = {
-            "source": source.as_dict(),
+            "source": source,
             "sourceModified": False,
         }
         if line_array is not None:
@@ -1067,7 +1189,7 @@ def request_setBreakpoints(self, source: Source, line_array, data=None):
             "type": "request",
             "arguments": args_dict,
         }
-        response = self.send_recv(command_dict)
+        response = self._send_recv(command_dict)
         if response["success"]:
             self._update_verified_breakpoints(response["body"]["breakpoints"])
         return response
@@ -1083,7 +1205,7 @@ def request_setExceptionBreakpoints(
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_setFunctionBreakpoints(self, names, condition=None, hitCondition=None):
         breakpoints = []
@@ -1100,7 +1222,7 @@ def request_setFunctionBreakpoints(self, names, condition=None, hitCondition=Non
             "type": "request",
             "arguments": args_dict,
         }
-        response = self.send_recv(command_dict)
+        response = self._send_recv(command_dict)
         if response["success"]:
             self._update_verified_breakpoints(response["body"]["breakpoints"])
         return response
@@ -1121,7 +1243,7 @@ def request_dataBreakpointInfo(
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_setDataBreakpoint(self, dataBreakpoints):
         """dataBreakpoints is a list of dictionary with following fields:
@@ -1138,7 +1260,7 @@ def request_setDataBreakpoint(self, dataBreakpoints):
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_compileUnits(self, moduleId):
         args_dict = {"moduleId": moduleId}
@@ -1147,7 +1269,7 @@ def request_compileUnits(self, moduleId):
             "type": "request",
             "arguments": args_dict,
         }
-        response = self.send_recv(command_dict)
+        response = self._send_recv(command_dict)
         return response
 
     def request_completions(self, text, frameId=None):
@@ -1159,17 +1281,43 @@ def request_completions(self, text, frameId=None):
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
-
-    def request_modules(self, startModule: int, moduleCount: int):
-        return self.send_recv(
-            {
-                "command": "modules",
-                "type": "request",
-                "arguments": {"startModule": startModule, "moduleCount": moduleCount},
-            }
+        return self._send_recv(command_dict)
+
+    def request_modules(
+        self,
+        start_module: Optional[int] = None,
+        module_count: Optional[int] = None,
+    ):
+        args_dict = {}
+
+        if start_module is not None:
+            args_dict["startModule"] = start_module
+        if module_count is not None:
+            args_dict["moduleCount"] = module_count
+
+        return self._send_recv(
+            {"command": "modules", "type": "request", "arguments": args_dict}
         )
 
+    def request_moduleSymbols(
+        self,
+        moduleId: str = "",
+        moduleName: str = "",
+        startIndex: int = 0,
+        count: int = 0,
+    ):
+        command_dict = {
+            "command": "__lldb_moduleSymbols",
+            "type": "request",
+            "arguments": {
+                "moduleId": moduleId,
+                "moduleName": moduleName,
+                "startIndex": startIndex,
+                "count": count,
+            },
+        }
+        return self._send_recv(command_dict)
+
     def request_stackTrace(
         self, threadId=None, startFrame=None, levels=None, format=None, dump=False
     ):
@@ -1187,7 +1335,7 @@ def request_stackTrace(
             "type": "request",
             "arguments": args_dict,
         }
-        response = self.send_recv(command_dict)
+        response = self._send_recv(command_dict)
         if dump:
             for idx, frame in enumerate(response["body"]["stackFrames"]):
                 name = frame["name"]
@@ -1202,18 +1350,30 @@ def request_stackTrace(
                 print("[%3u] %s" % (idx, name))
         return response
 
-    def request_source(self, sourceReference):
+    def request_source(
+        self, *, source: Optional[Source] = None, sourceReference: Optional[int] = None
+    ):
         """Request a source from a 'Source' reference."""
+        if source is None and sourceReference is None:
+            raise ValueError("request_source requires either source or sourceReference")
+        elif source is not None:
+            sourceReference = source["sourceReference"]
+        elif sourceReference is not None:
+            source = {"sourceReference": sourceReference}
+        else:
+            raise ValueError(
+                "request_source requires either source or sourceReference not both"
+            )
         command_dict = {
             "command": "source",
             "type": "request",
             "arguments": {
-                "source": {"sourceReference": sourceReference},
+                "source": source,
                 # legacy version of the request
                 "sourceReference": sourceReference,
             },
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_threads(self):
         """Request a list of all threads and combine any information from any
@@ -1221,7 +1381,7 @@ def request_threads(self):
         thread actually stopped. Returns an array of thread dictionaries
         with information about all threads"""
         command_dict = {"command": "threads", "type": "request", "arguments": {}}
-        response = self.send_recv(command_dict)
+        response = self._send_recv(command_dict)
         if not response["success"]:
             self.threads = None
             return response
@@ -1261,7 +1421,7 @@ def request_variables(
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_setVariable(self, containingVarRef, name, value, id=None):
         args_dict = {
@@ -1276,7 +1436,7 @@ def request_setVariable(self, containingVarRef, name, value, id=None):
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_locations(self, locationReference):
         args_dict = {
@@ -1287,7 +1447,7 @@ def request_locations(self, locationReference):
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def request_testGetTargetBreakpoints(self):
         """A request packet used in the LLDB test suite to get all currently
@@ -1299,12 +1459,12 @@ def request_testGetTargetBreakpoints(self):
             "type": "request",
             "arguments": {},
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
     def terminate(self):
         self.send.close()
-        if self.recv_thread.is_alive():
-            self.recv_thread.join()
+        if self._recv_thread.is_alive():
+            self._recv_thread.join()
 
     def request_setInstructionBreakpoints(self, memory_reference=[]):
         breakpoints = []
@@ -1319,7 +1479,7 @@ def request_setInstructionBreakpoints(self, memory_reference=[]):
             "type": "request",
             "arguments": args_dict,
         }
-        return self.send_recv(command_dict)
+        return self._send_recv(command_dict)
 
 
 class DebugAdapterServer(DebugCommunication):
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
index 1567462839748..c23b2e73fb45e 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py
@@ -1,6 +1,6 @@
 import os
 import time
-from typing import Optional
+from typing import Optional, Callable, Any, List, Union
 import uuid
 
 import dap_server
@@ -59,24 +59,25 @@ def set_source_breakpoints(
         Each object in data is 1:1 mapping with the entry in lines.
         It contains optional location/hitCondition/logMessage parameters.
         """
-        response = self.dap_server.request_setBreakpoints(
-            Source(source_path), lines, data
+        return self.set_source_breakpoints_from_source(
+            Source(path=source_path), lines, data, wait_for_resolve
         )
-        if response is None or not response["success"]:
-            return []
-        breakpoints = response["body"]["breakpoints"]
-        breakpoint_ids = []
-        for breakpoint in breakpoints:
-            breakpoint_ids.append("%i" % (breakpoint["id"]))
-        if wait_for_resolve:
-            self.wait_for_breakpoints_to_resolve(breakpoint_ids)
-        return breakpoint_ids
 
     def set_source_breakpoints_assembly(
         self, source_reference, lines, data=None, wait_for_resolve=True
+    ):
+        return self.set_source_breakpoints_from_source(
+            Source.build(source_reference=source_reference),
+            lines,
+            data,
+            wait_for_resolve,
+        )
+
+    def set_source_breakpoints_from_source(
+        self, source: Source, lines, data=None, wait_for_resolve=True
     ):
         response = self.dap_server.request_setBreakpoints(
-            Source(source_reference=source_reference),
+            source,
             lines,
             data,
         )
@@ -122,11 +123,19 @@ def wait_for_breakpoints_to_resolve(
             f"Expected to resolve all breakpoints. Unresolved breakpoint ids: {unresolved_breakpoints}",
         )
 
-    def waitUntil(self, condition_callback):
-        for _ in range(20):
-            if condition_callback():
+    def wait_until(
+        self,
+        predicate: Callable[[], bool],
+        delay: float = 0.5,
+        timeout: float = DEFAULT_TIMEOUT,
+    ) -> bool:
+        """Repeatedly run the predicate until either the predicate returns True
+        or a timeout has occurred."""
+        deadline = time.monotonic() + timeout
+        while deadline > time.monotonic():
+            if predicate():
                 return True
-            time.sleep(0.5)
+            time.sleep(delay)
         return False
 
     def assertCapabilityIsSet(self, key: str, msg: Optional[str] = None) -> None:
@@ -139,13 +148,16 @@ def assertCapabilityIsNotSet(self, key: str, msg: Optional[str] = None) -> None:
         if key in self.dap_server.capabilities:
             self.assertEqual(self.dap_server.capabilities[key], False, msg)
 
-    def verify_breakpoint_hit(self, breakpoint_ids, timeout=DEFAULT_TIMEOUT):
+    def verify_breakpoint_hit(
+        self, breakpoint_ids: List[Union[int, str]], timeout: float = DEFAULT_TIMEOUT
+    ):
         """Wait for the process we are debugging to stop, and verify we hit
         any breakpoint location in the "breakpoint_ids" array.
         "breakpoint_ids" should be a list of breakpoint ID strings
         (["1", "2"]). The return value from self.set_source_breakpoints()
         or self.set_function_breakpoints() can be passed to this function"""
         stopped_events = self.dap_server.wait_for_stopped(timeout)
+        normalized_bp_ids = [str(b) for b in breakpoint_ids]
         for stopped_event in stopped_events:
             if "body" in stopped_event:
                 body = stopped_event["body"]
@@ -156,22 +168,16 @@ def verify_breakpoint_hit(self, breakpoint_ids, timeout=DEFAULT_TIMEOUT):
                     and body["reason"] != "instruction breakpoint"
                 ):
                     continue
-                if "description" not in body:
+                if "hitBreakpointIds" not in body:
                     continue
-                # Descriptions for breakpoints will be in the form
-                # "breakpoint 1.1", so look for any description that matches
-                # ("breakpoint 1.") in the description field as verification
-                # that one of the breakpoint locations was hit. DAP doesn't
-                # allow breakpoints to have multiple locations, but LLDB does.
-                # So when looking at the description we just want to make sure
-                # the right breakpoint matches and not worry about the actual
-                # location.
-                description = body["description"]
-                for breakpoint_id in breakpoint_ids:
-                    match_desc = f"breakpoint {breakpoint_id}."
-                    if match_desc in description:
+                hit_breakpoint_ids = body["hitBreakpointIds"]
+                for bp in hit_breakpoint_ids:
+                    if str(bp) in normalized_bp_ids:
                         return
-        self.assertTrue(False, f"breakpoint not hit, stopped_events={stopped_events}")
+        self.assertTrue(
+            False,
+            f"breakpoint not hit, wanted breakpoint_ids {breakpoint_ids} in stopped_events {stopped_events}",
+        )
 
     def verify_all_breakpoints_hit(self, breakpoint_ids, timeout=DEFAULT_TIMEOUT):
         """Wait for the process we are debugging to stop, and verify we hit
@@ -215,7 +221,7 @@ def verify_stop_exception_info(self, expected_description, timeout=DEFAULT_TIMEO
                     return True
         return False
 
-    def verify_commands(self, flavor, output, commands):
+    def verify_commands(self, flavor: str, output: str, commands: list[str]):
         self.assertTrue(output and len(output) > 0, "expect console output")
         lines = output.splitlines()
         prefix = "(lldb) "
@@ -228,10 +234,11 @@ def verify_commands(self, flavor, output, commands):
                     found = True
                     break
             self.assertTrue(
-                found, "verify '%s' found in console output for '%s'" % (cmd, flavor)
+                found,
+                f"Command '{flavor}' - '{cmd}' not found in output: {output}",
             )
 
-    def get_dict_value(self, d, key_path):
+    def get_dict_value(self, d: dict, key_path: list[str]) -> Any:
         """Verify each key in the key_path array is in contained in each
         dictionary within "d". Assert if any key isn't in the
         corresponding dictionary. This is handy for grabbing values from VS
@@ -300,28 +307,34 @@ def get_source_and_line(self, threadId=None, frameIndex=0):
                         return (source["path"], stackFrame["line"])
         return ("", 0)
 
-    def get_stdout(self, timeout=0.0):
-        return self.dap_server.get_output("stdout", timeout=timeout)
+    def get_stdout(self):
+        return self.dap_server.get_output("stdout")
 
-    def get_console(self, timeout=0.0):
-        return self.dap_server.get_output("console", timeout=timeout)
+    def get_console(self):
+        return self.dap_server.get_output("console")
 
-    def get_important(self, timeout=0.0):
-        return self.dap_server.get_output("important", timeout=timeout)
+    def get_important(self):
+        return self.dap_server.get_output("important")
 
-    def collect_stdout(self, timeout_secs, pattern=None):
+    def collect_stdout(
+        self, timeout: float = DEFAULT_TIMEOUT, pattern: Optional[str] = None
+    ) -> str:
         return self.dap_server.collect_output(
-            "stdout", timeout_secs=timeout_secs, pattern=pattern
+            "stdout", timeout=timeout, pattern=pattern
         )
 
-    def collect_console(self, timeout_secs, pattern=None):
+    def collect_console(
+        self, timeout: float = DEFAULT_TIMEOUT, pattern: Optional[str] = None
+    ) -> str:
         return self.dap_server.collect_output(
-            "console", timeout_secs=timeout_secs, pattern=pattern
+            "console", timeout=timeout, pattern=pattern
         )
 
-    def collect_important(self, timeout_secs, pattern=None):
+    def collect_important(
+        self, timeout: float = DEFAULT_TIMEOUT, pattern: Optional[str] = None
+    ) -> str:
         return self.dap_server.collect_output(
-            "important", timeout_secs=timeout_secs, pattern=pattern
+            "important", timeout=timeout, pattern=pattern
         )
 
     def get_local_as_int(self, name, threadId=None):
diff --git a/lldb/scripts/framework-header-fix.sh b/lldb/scripts/framework-header-fix.sh
deleted file mode 100755
index 345579c80cdf5..0000000000000
--- a/lldb/scripts/framework-header-fix.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-# Usage: framework-header-fix.sh <source header dir> <LLDB Version>
-
-set -e
-
-for file in `find $1 -name "*.h"`
-do
-  /usr/bin/sed -i.bak 's/\(#include\)[ ]*"lldb\/\(API\/\)\{0,1\}\(.*\)"/\1 <LLDB\/\3>/1' "$file"
-  /usr/bin/sed -i.bak 's|<LLDB/Utility|<LLDB|' "$file"
-  rm -f "$file.bak"
-done
diff --git a/lldb/source/API/SBFrame.cpp b/lldb/source/API/SBFrame.cpp
index 5b69cf1ee2641..b12cfceacd75d 100644
--- a/lldb/source/API/SBFrame.cpp
+++ b/lldb/source/API/SBFrame.cpp
@@ -97,226 +97,176 @@ bool SBFrame::IsValid() const {
 }
 SBFrame::operator bool() const {
   LLDB_INSTRUMENT_VA(this);
-
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock()))
-      return GetFrameSP().get() != nullptr;
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
   }
 
-  // Without a target & process we can't have a valid stack frame.
-  return false;
+  return GetFrameSP().get() != nullptr;
 }
 
 SBSymbolContext SBFrame::GetSymbolContext(uint32_t resolve_scope) const {
   LLDB_INSTRUMENT_VA(this, resolve_scope);
 
   SBSymbolContext sb_sym_ctx;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-  SymbolContextItem scope = static_cast<SymbolContextItem>(resolve_scope);
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      if (StackFrame *frame = exe_ctx.GetFramePtr())
-        sb_sym_ctx = frame->GetSymbolContext(scope);
-    }
+
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return sb_sym_ctx;
   }
 
+  SymbolContextItem scope = static_cast<SymbolContextItem>(resolve_scope);
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    sb_sym_ctx = frame->GetSymbolContext(scope);
+
   return sb_sym_ctx;
 }
 
 SBModule SBFrame::GetModule() const {
   LLDB_INSTRUMENT_VA(this);
 
-  SBModule sb_module;
-  ModuleSP module_sp;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        module_sp = frame->GetSymbolContext(eSymbolContextModule).module_sp;
-        sb_module.SetSP(module_sp);
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBModule();
   }
 
+  ModuleSP module_sp;
+  StackFrame *frame = exe_ctx->GetFramePtr();
+  if (!frame)
+    return SBModule();
+
+  SBModule sb_module;
+  module_sp = frame->GetSymbolContext(eSymbolContextModule).module_sp;
+  sb_module.SetSP(module_sp);
   return sb_module;
 }
 
 SBCompileUnit SBFrame::GetCompileUnit() const {
   LLDB_INSTRUMENT_VA(this);
 
-  SBCompileUnit sb_comp_unit;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        sb_comp_unit.reset(
-            frame->GetSymbolContext(eSymbolContextCompUnit).comp_unit);
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBCompileUnit();
   }
 
-  return sb_comp_unit;
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return SBCompileUnit(
+        frame->GetSymbolContext(eSymbolContextCompUnit).comp_unit);
+  return SBCompileUnit();
 }
 
 SBFunction SBFrame::GetFunction() const {
   LLDB_INSTRUMENT_VA(this);
 
-  SBFunction sb_function;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        sb_function.reset(
-            frame->GetSymbolContext(eSymbolContextFunction).function);
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBFunction();
   }
 
-  return sb_function;
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return SBFunction(frame->GetSymbolContext(eSymbolContextFunction).function);
+  return SBFunction();
 }
 
 SBSymbol SBFrame::GetSymbol() const {
   LLDB_INSTRUMENT_VA(this);
 
-  SBSymbol sb_symbol;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        sb_symbol.reset(frame->GetSymbolContext(eSymbolContextSymbol).symbol);
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBSymbol();
   }
 
-  return sb_symbol;
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return SBSymbol(frame->GetSymbolContext(eSymbolContextSymbol).symbol);
+  return SBSymbol();
 }
 
 SBBlock SBFrame::GetBlock() const {
   LLDB_INSTRUMENT_VA(this);
 
-  SBBlock sb_block;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame)
-        sb_block.SetPtr(frame->GetSymbolContext(eSymbolContextBlock).block);
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBBlock();
   }
-  return sb_block;
+
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return SBBlock(frame->GetSymbolContext(eSymbolContextBlock).block);
+  return SBBlock();
 }
 
 SBBlock SBFrame::GetFrameBlock() const {
   LLDB_INSTRUMENT_VA(this);
 
-  SBBlock sb_block;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame)
-        sb_block.SetPtr(frame->GetFrameBlock());
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBBlock();
   }
-  return sb_block;
+
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return SBBlock(frame->GetFrameBlock());
+  return SBBlock();
 }
 
 SBLineEntry SBFrame::GetLineEntry() const {
   LLDB_INSTRUMENT_VA(this);
 
-  SBLineEntry sb_line_entry;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        sb_line_entry.SetLineEntry(
-            frame->GetSymbolContext(eSymbolContextLineEntry).line_entry);
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBLineEntry();
   }
-  return sb_line_entry;
+
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return SBLineEntry(
+        &frame->GetSymbolContext(eSymbolContextLineEntry).line_entry);
+  return SBLineEntry();
 }
 
 uint32_t SBFrame::GetFrameID() const {
   LLDB_INSTRUMENT_VA(this);
 
-  uint32_t frame_idx = UINT32_MAX;
-
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  constexpr uint32_t error_frame_idx = UINT32_MAX;
 
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  if (frame)
-    frame_idx = frame->GetFrameIndex();
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return error_frame_idx;
+  }
 
-  return frame_idx;
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return frame->GetFrameIndex();
+  return error_frame_idx;
 }
 
 lldb::addr_t SBFrame::GetCFA() const {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return LLDB_INVALID_ADDRESS;
+  }
 
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  if (frame)
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
     return frame->GetStackID().GetCallFrameAddress();
   return LLDB_INVALID_ADDRESS;
 }
@@ -325,114 +275,86 @@ addr_t SBFrame::GetPC() const {
   LLDB_INSTRUMENT_VA(this);
 
   addr_t addr = LLDB_INVALID_ADDRESS;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        addr = frame->GetFrameCodeAddress().GetOpcodeLoadAddress(
-            target, AddressClass::eCode);
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return addr;
   }
 
+  Target *target = exe_ctx->GetTargetPtr();
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return frame->GetFrameCodeAddress().GetOpcodeLoadAddress(
+        target, AddressClass::eCode);
+
   return addr;
 }
 
 bool SBFrame::SetPC(addr_t new_pc) {
   LLDB_INSTRUMENT_VA(this, new_pc);
 
-  bool ret_val = false;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      if (StackFrame *frame = exe_ctx.GetFramePtr()) {
-        if (RegisterContextSP reg_ctx_sp = frame->GetRegisterContext()) {
-          ret_val = reg_ctx_sp->SetPC(new_pc);
-        }
-      }
-    }
+  constexpr bool error_ret_val = false;
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return error_ret_val;
   }
 
-  return ret_val;
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    if (RegisterContextSP reg_ctx_sp = frame->GetRegisterContext())
+      return reg_ctx_sp->SetPC(new_pc);
+
+  return error_ret_val;
 }
 
 addr_t SBFrame::GetSP() const {
   LLDB_INSTRUMENT_VA(this);
 
-  addr_t addr = LLDB_INVALID_ADDRESS;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      if (StackFrame *frame = exe_ctx.GetFramePtr()) {
-        if (RegisterContextSP reg_ctx_sp = frame->GetRegisterContext()) {
-          addr = reg_ctx_sp->GetSP();
-        }
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return LLDB_INVALID_ADDRESS;
   }
 
-  return addr;
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    if (RegisterContextSP reg_ctx_sp = frame->GetRegisterContext())
+      return reg_ctx_sp->GetSP();
+
+  return LLDB_INVALID_ADDRESS;
 }
 
 addr_t SBFrame::GetFP() const {
   LLDB_INSTRUMENT_VA(this);
 
-  addr_t addr = LLDB_INVALID_ADDRESS;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      if (StackFrame *frame = exe_ctx.GetFramePtr()) {
-        if (RegisterContextSP reg_ctx_sp = frame->GetRegisterContext()) {
-          addr = reg_ctx_sp->GetFP();
-        }
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return LLDB_INVALID_ADDRESS;
   }
 
-  return addr;
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    if (RegisterContextSP reg_ctx_sp = frame->GetRegisterContext())
+      return reg_ctx_sp->GetFP();
+
+  return LLDB_INVALID_ADDRESS;
 }
 
 SBAddress SBFrame::GetPCAddress() const {
   LLDB_INSTRUMENT_VA(this);
 
-  SBAddress sb_addr;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame)
-        sb_addr.SetAddress(frame->GetFrameCodeAddress());
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBAddress();
   }
-  return sb_addr;
+
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return SBAddress(frame->GetFrameCodeAddress());
+  return SBAddress();
 }
 
 void SBFrame::Clear() {
@@ -445,12 +367,14 @@ lldb::SBValue SBFrame::GetValueForVariablePath(const char *var_path) {
   LLDB_INSTRUMENT_VA(this, var_path);
 
   SBValue sb_value;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return sb_value;
+  }
 
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  Target *target = exe_ctx.GetTargetPtr();
-  if (frame && target) {
+  if (StackFrame *frame = exe_ctx->GetFramePtr()) {
     lldb::DynamicValueType use_dynamic =
         frame->CalculateTarget()->GetPreferDynamicValue();
     sb_value = GetValueForVariablePath(var_path, use_dynamic);
@@ -467,27 +391,22 @@ lldb::SBValue SBFrame::GetValueForVariablePath(const char *var_path,
     return sb_value;
   }
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        VariableSP var_sp;
-        Status error;
-        ValueObjectSP value_sp(frame->GetValueForVariableExpressionPath(
-            var_path, eNoDynamicValues,
-            StackFrame::eExpressionPathOptionCheckPtrVsMember |
-                StackFrame::eExpressionPathOptionsAllowDirectIVarAccess,
-            var_sp, error));
-        sb_value.SetSP(value_sp, use_dynamic);
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return sb_value;
+  }
+
+  if (StackFrame *frame = exe_ctx->GetFramePtr()) {
+    VariableSP var_sp;
+    Status error;
+    ValueObjectSP value_sp(frame->GetValueForVariableExpressionPath(
+        var_path, eNoDynamicValues,
+        StackFrame::eExpressionPathOptionCheckPtrVsMember |
+            StackFrame::eExpressionPathOptionsAllowDirectIVarAccess,
+        var_sp, error));
+    sb_value.SetSP(value_sp, use_dynamic);
   }
   return sb_value;
 }
@@ -495,18 +414,19 @@ lldb::SBValue SBFrame::GetValueForVariablePath(const char *var_path,
 SBValue SBFrame::FindVariable(const char *name) {
   LLDB_INSTRUMENT_VA(this, name);
 
-  SBValue value;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBValue();
+  }
 
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  Target *target = exe_ctx.GetTargetPtr();
-  if (frame && target) {
+  if (StackFrame *frame = exe_ctx->GetFramePtr()) {
     lldb::DynamicValueType use_dynamic =
         frame->CalculateTarget()->GetPreferDynamicValue();
-    value = FindVariable(name, use_dynamic);
+    return FindVariable(name, use_dynamic);
   }
-  return value;
+  return SBValue();
 }
 
 SBValue SBFrame::FindVariable(const char *name,
@@ -520,26 +440,17 @@ SBValue SBFrame::FindVariable(const char *name,
     return sb_value;
   }
 
-  ValueObjectSP value_sp;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        value_sp = frame->FindVariable(ConstString(name));
-
-        if (value_sp)
-          sb_value.SetSP(value_sp, use_dynamic);
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return sb_value;
   }
 
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    if (ValueObjectSP value_sp = frame->FindVariable(ConstString(name)))
+      sb_value.SetSP(value_sp, use_dynamic);
+
   return sb_value;
 }
 
@@ -547,12 +458,14 @@ SBValue SBFrame::FindValue(const char *name, ValueType value_type) {
   LLDB_INSTRUMENT_VA(this, name, value_type);
 
   SBValue value;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return value;
+  }
 
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  Target *target = exe_ctx.GetTargetPtr();
-  if (frame && target) {
+  if (StackFrame *frame = exe_ctx->GetFramePtr()) {
     lldb::DynamicValueType use_dynamic =
         frame->CalculateTarget()->GetPreferDynamicValue();
     value = FindValue(name, value_type, use_dynamic);
@@ -571,17 +484,17 @@ SBValue SBFrame::FindValue(const char *name, ValueType value_type,
   }
 
   ValueObjectSP value_sp;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return value_sp;
+  } else {
+    Target *target = exe_ctx->GetTargetPtr();
+    Process *process = exe_ctx->GetProcessPtr();
+    if (target && process) { // FIXME: this check is redundant.
+      if (StackFrame *frame = exe_ctx->GetFramePtr()) {
         VariableList variable_list;
 
         switch (value_type) {
@@ -698,10 +611,14 @@ bool SBFrame::operator!=(const SBFrame &rhs) const {
 SBThread SBFrame::GetThread() const {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBThread();
+  }
 
-  ThreadSP thread_sp(exe_ctx.GetThreadSP());
+  ThreadSP thread_sp(exe_ctx->GetThreadSP());
   SBThread sb_thread(thread_sp);
 
   return sb_thread;
@@ -710,19 +627,16 @@ SBThread SBFrame::GetThread() const {
 const char *SBFrame::Disassemble() const {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (!target || !process)
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
     return nullptr;
-
-  Process::StopLocker stop_locker;
-  if (stop_locker.TryLock(&process->GetRunLock())) {
-    if (auto *frame = exe_ctx.GetFramePtr())
-      return ConstString(frame->Disassemble()).GetCString();
   }
 
+  if (auto *frame = exe_ctx->GetFramePtr())
+    return ConstString(frame->Disassemble()).GetCString();
+
   return nullptr;
 }
 
@@ -731,12 +645,15 @@ SBValueList SBFrame::GetVariables(bool arguments, bool locals, bool statics,
   LLDB_INSTRUMENT_VA(this, arguments, locals, statics, in_scope_only);
 
   SBValueList value_list;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return value_list;
+  }
 
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  Target *target = exe_ctx.GetTargetPtr();
-  if (frame && target) {
+  if (StackFrame *frame = exe_ctx->GetFramePtr()) {
+    Target *target = exe_ctx->GetTargetPtr();
     lldb::DynamicValueType use_dynamic =
         frame->CalculateTarget()->GetPreferDynamicValue();
     const bool include_runtime_support_values =
@@ -761,12 +678,16 @@ lldb::SBValueList SBFrame::GetVariables(bool arguments, bool locals,
   LLDB_INSTRUMENT_VA(this, arguments, locals, statics, in_scope_only,
                      use_dynamic);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBValueList();
+  }
 
-  Target *target = exe_ctx.GetTargetPtr();
+  Target *target = exe_ctx->GetTargetPtr();
   const bool include_runtime_support_values =
-      target ? target->GetDisplayRuntimeSupportValues() : false;
+      target->GetDisplayRuntimeSupportValues();
   SBVariablesOptions options;
   options.SetIncludeArguments(arguments);
   options.SetIncludeLocals(locals);
@@ -781,30 +702,26 @@ SBValueList SBFrame::GetVariables(const lldb::SBVariablesOptions &options) {
   LLDB_INSTRUMENT_VA(this, options);
 
   SBValueList value_list;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-
-  const bool statics = options.GetIncludeStatics();
-  const bool arguments = options.GetIncludeArguments();
-  const bool recognized_arguments =
-        options.GetIncludeRecognizedArguments(SBTarget(exe_ctx.GetTargetSP()));
-  const bool locals = options.GetIncludeLocals();
-  const bool in_scope_only = options.GetInScopeOnly();
-  const bool include_runtime_support_values =
-      options.GetIncludeRuntimeSupportValues();
-  const lldb::DynamicValueType use_dynamic = options.GetUseDynamic();
-
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBValueList();
+  } else {
+    const bool statics = options.GetIncludeStatics();
+    const bool arguments = options.GetIncludeArguments();
+    const bool recognized_arguments =
+        options.GetIncludeRecognizedArguments(SBTarget(exe_ctx->GetTargetSP()));
+    const bool locals = options.GetIncludeLocals();
+    const bool in_scope_only = options.GetInScopeOnly();
+    const bool include_runtime_support_values =
+        options.GetIncludeRuntimeSupportValues();
+    const lldb::DynamicValueType use_dynamic = options.GetUseDynamic();
 
-  std::set<VariableSP> variable_set;
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
+    std::set<VariableSP> variable_set;
+    Process *process = exe_ctx->GetProcessPtr();
+    if (process) { // FIXME: this check is redundant.
+      if (StackFrame *frame = exe_ctx->GetFramePtr()) {
         Debugger &dbg = process->GetTarget().GetDebugger();
         VariableList *variable_list = nullptr;
         Status var_error;
@@ -892,17 +809,16 @@ SBValueList SBFrame::GetRegisters() {
   LLDB_INSTRUMENT_VA(this);
 
   SBValueList value_list;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBValueList();
+  } else {
+    Target *target = exe_ctx->GetTargetPtr();
+    Process *process = exe_ctx->GetProcessPtr();
+    if (target && process) { // FIXME: this check is redundant.
+      if (StackFrame *frame = exe_ctx->GetFramePtr()) {
         RegisterContextSP reg_ctx(frame->GetRegisterContext());
         if (reg_ctx) {
           const uint32_t num_sets = reg_ctx->GetRegisterSetCount();
@@ -923,17 +839,16 @@ SBValue SBFrame::FindRegister(const char *name) {
 
   SBValue result;
   ValueObjectSP value_sp;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBValue();
+  } else {
+    Target *target = exe_ctx->GetTargetPtr();
+    Process *process = exe_ctx->GetProcessPtr();
+    if (target && process) { // FIXME: this check is redundant.
+      if (StackFrame *frame = exe_ctx->GetFramePtr()) {
         RegisterContextSP reg_ctx(frame->GetRegisterContext());
         if (reg_ctx) {
           if (const RegisterInfo *reg_info =
@@ -953,12 +868,11 @@ SBError SBFrame::GetDescriptionWithFormat(const SBFormat &format,
                                           SBStream &output) {
   Stream &strm = output.ref();
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx)
+    return Status::FromError(exe_ctx.takeError());
 
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
   SBError error;
 
   if (!format) {
@@ -966,16 +880,9 @@ SBError SBFrame::GetDescriptionWithFormat(const SBFormat &format,
     return error;
   }
 
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame &&
-          frame->DumpUsingFormat(strm, format.GetFormatEntrySP().get())) {
-        return error;
-      }
-    }
-  }
+  if (StackFrame *frame = exe_ctx->GetFramePtr();
+      frame && frame->DumpUsingFormat(strm, format.GetFormatEntrySP().get()))
+    return error;
   error.SetErrorStringWithFormat(
       "It was not possible to generate a frame "
       "description with the given format string '%s'",
@@ -988,23 +895,16 @@ bool SBFrame::GetDescription(SBStream &description) {
 
   Stream &strm = description.ref();
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        frame->DumpUsingSettingsFormat(&strm);
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    strm.PutCString("Error: process is not stopped.");
+    return true;
+  }
 
-  } else
-    strm.PutCString("No value");
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    frame->DumpUsingSettingsFormat(&strm);
 
   return true;
 }
@@ -1012,22 +912,24 @@ bool SBFrame::GetDescription(SBStream &description) {
 SBValue SBFrame::EvaluateExpression(const char *expr) {
   LLDB_INSTRUMENT_VA(this, expr);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return CreateProcessIsRunningExprEvalError();
+  }
 
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  Target *target = exe_ctx.GetTargetPtr();
   SBExpressionOptions options;
-  if (frame && target) {
+  StackFrame *frame = exe_ctx->GetFramePtr();
+  if (frame) {
     lldb::DynamicValueType fetch_dynamic_value =
         frame->CalculateTarget()->GetPreferDynamicValue();
     options.SetFetchDynamicValue(fetch_dynamic_value);
   }
   options.SetUnwindOnError(true);
   options.SetIgnoreBreakpoints(true);
-  SourceLanguage language;
-  if (target)
-    language = target->GetLanguage();
+  Target *target = exe_ctx->GetTargetPtr();
+  SourceLanguage language = target->GetLanguage();
   if (!language && frame)
     language = frame->GetLanguage();
   options.SetLanguage((SBSourceLanguageName)language.name, language.version);
@@ -1043,14 +945,16 @@ SBFrame::EvaluateExpression(const char *expr,
   options.SetFetchDynamicValue(fetch_dynamic_value);
   options.SetUnwindOnError(true);
   options.SetIgnoreBreakpoints(true);
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  Target *target = exe_ctx.GetTargetPtr();
-  SourceLanguage language;
-  if (target)
-    language = target->GetLanguage();
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return CreateProcessIsRunningExprEvalError();
+  }
+
+  StackFrame *frame = exe_ctx->GetFramePtr();
+  Target *target = exe_ctx->GetTargetPtr();
+  SourceLanguage language = target->GetLanguage();
   if (!language && frame)
     language = frame->GetLanguage();
   options.SetLanguage((SBSourceLanguageName)language.name, language.version);
@@ -1063,23 +967,35 @@ SBValue SBFrame::EvaluateExpression(const char *expr,
   LLDB_INSTRUMENT_VA(this, expr, fetch_dynamic_value, unwind_on_error);
 
   SBExpressionOptions options;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return CreateProcessIsRunningExprEvalError();
+  }
 
   options.SetFetchDynamicValue(fetch_dynamic_value);
   options.SetUnwindOnError(unwind_on_error);
   options.SetIgnoreBreakpoints(true);
-  StackFrame *frame = exe_ctx.GetFramePtr();
-  Target *target = exe_ctx.GetTargetPtr();
-  SourceLanguage language;
-  if (target)
-    language = target->GetLanguage();
+  StackFrame *frame = exe_ctx->GetFramePtr();
+  Target *target = exe_ctx->GetTargetPtr();
+  SourceLanguage language = target->GetLanguage();
   if (!language && frame)
     language = frame->GetLanguage();
   options.SetLanguage((SBSourceLanguageName)language.name, language.version);
   return EvaluateExpression(expr, options);
 }
 
+lldb::SBValue SBFrame::CreateProcessIsRunningExprEvalError() {
+  auto error = Status::FromErrorString("can't evaluate expressions when the "
+                                       "process is running.");
+  ValueObjectSP expr_value_sp =
+      ValueObjectConstResult::Create(nullptr, std::move(error));
+  SBValue expr_result;
+  expr_result.SetSP(expr_value_sp, false);
+  return expr_result;
+}
+
 lldb::SBValue SBFrame::EvaluateExpression(const char *expr,
                                           const SBExpressionOptions &options) {
   LLDB_INSTRUMENT_VA(this, expr, options);
@@ -1094,18 +1010,16 @@ lldb::SBValue SBFrame::EvaluateExpression(const char *expr,
 
   ValueObjectSP expr_value_sp;
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    expr_result = CreateProcessIsRunningExprEvalError();
+  } else {
+    Target *target = exe_ctx->GetTargetPtr();
+    Process *process = exe_ctx->GetProcessPtr();
+    if (target && process) { // FIXME: this check is redundant.
+      if (StackFrame *frame = exe_ctx->GetFramePtr()) {
         std::unique_ptr<llvm::PrettyStackTraceFormat> stack_trace;
         if (target->GetDisplayExpressionsInCrashlogs()) {
           StreamString frame_description;
@@ -1121,17 +1035,11 @@ lldb::SBValue SBFrame::EvaluateExpression(const char *expr,
         expr_result.SetSP(expr_value_sp, options.GetFetchDynamicValue());
       }
     } else {
-      Status error;
-      error = Status::FromErrorString("can't evaluate expressions when the "
-                                      "process is running.");
-      expr_value_sp = ValueObjectConstResult::Create(nullptr, std::move(error));
-      expr_result.SetSP(expr_value_sp, false);
-    }
-  } else {
       Status error;
       error = Status::FromErrorString("sbframe object is not valid.");
       expr_value_sp = ValueObjectConstResult::Create(nullptr, std::move(error));
       expr_result.SetSP(expr_value_sp, false);
+    }
   }
 
   if (expr_result.GetError().Success())
@@ -1152,9 +1060,13 @@ SBStructuredData SBFrame::GetLanguageSpecificData() const {
   LLDB_INSTRUMENT_VA(this);
 
   SBStructuredData sb_data;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-  StackFrame *frame = exe_ctx.GetFramePtr();
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return sb_data;
+  }
+  StackFrame *frame = exe_ctx->GetFramePtr();
   if (!frame)
     return sb_data;
 
@@ -1172,20 +1084,15 @@ bool SBFrame::IsInlined() {
 bool SBFrame::IsInlined() const {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame)
-        return frame->IsInlined();
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
   }
+
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return frame->IsInlined();
   return false;
 }
 
@@ -1198,10 +1105,14 @@ bool SBFrame::IsArtificial() {
 bool SBFrame::IsArtificial() const {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
+  }
 
-  if (StackFrame *frame = exe_ctx.GetFramePtr())
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
     return frame->IsArtificial();
 
   return false;
@@ -1210,10 +1121,14 @@ bool SBFrame::IsArtificial() const {
 bool SBFrame::IsHidden() const {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
+  }
 
-  if (StackFrame *frame = exe_ctx.GetFramePtr())
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
     return frame->IsHidden();
 
   return false;
@@ -1228,63 +1143,44 @@ const char *SBFrame::GetFunctionName() {
 lldb::LanguageType SBFrame::GuessLanguage() const {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame) {
-        return frame->GuessLanguage().AsLanguageType();
-      }
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return eLanguageTypeUnknown;
   }
+
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return frame->GuessLanguage().AsLanguageType();
   return eLanguageTypeUnknown;
 }
 
 const char *SBFrame::GetFunctionName() const {
   LLDB_INSTRUMENT_VA(this);
 
-  const char *name = nullptr;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame)
-        return frame->GetFunctionName();
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return nullptr;
   }
-  return name;
+
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return frame->GetFunctionName();
+  return nullptr;
 }
 
 const char *SBFrame::GetDisplayFunctionName() {
   LLDB_INSTRUMENT_VA(this);
 
-  const char *name = nullptr;
-
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  StackFrame *frame = nullptr;
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock())) {
-      frame = exe_ctx.GetFramePtr();
-      if (frame)
-        return frame->GetDisplayFunctionName();
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return nullptr;
   }
-  return name;
+
+  if (StackFrame *frame = exe_ctx->GetFramePtr())
+    return frame->GetDisplayFunctionName();
+  return nullptr;
 }
diff --git a/lldb/source/API/SBModule.cpp b/lldb/source/API/SBModule.cpp
index 985107ec68efd..5a57f45f0d475 100644
--- a/lldb/source/API/SBModule.cpp
+++ b/lldb/source/API/SBModule.cpp
@@ -671,3 +671,11 @@ void SBModule::GarbageCollectAllocatedModules() {
   const bool mandatory = false;
   ModuleList::RemoveOrphanSharedModules(mandatory);
 }
+
+const char *SBModule::GetObjectName() const {
+  LLDB_INSTRUMENT_VA(this);
+
+  if (!m_opaque_sp)
+    return nullptr;
+  return m_opaque_sp->GetObjectName().AsCString();
+}
diff --git a/lldb/source/API/SBStructuredData.cpp b/lldb/source/API/SBStructuredData.cpp
index b891a34bd7c76..8e2c18ed42b47 100644
--- a/lldb/source/API/SBStructuredData.cpp
+++ b/lldb/source/API/SBStructuredData.cpp
@@ -232,3 +232,47 @@ lldb::SBScriptObject SBStructuredData::GetGenericValue() const {
 
   return {m_impl_up->GetGenericValue(), eScriptLanguageDefault};
 }
+
+void SBStructuredData::SetValueForKey(const char *key,
+                                      SBStructuredData &value) {
+  LLDB_INSTRUMENT_VA(this, key, value);
+
+  if (StructuredData::ObjectSP obj_sp = value.m_impl_up->GetObjectSP())
+    m_impl_up->SetValueForKey(key, obj_sp);
+}
+
+void SBStructuredData::SetUnsignedIntegerValue(uint64_t value) {
+  LLDB_INSTRUMENT_VA(this, value);
+
+  m_impl_up->SetUnsignedIntegerValue(value);
+}
+
+void SBStructuredData::SetSignedIntegerValue(int64_t value) {
+  LLDB_INSTRUMENT_VA(this, value);
+
+  m_impl_up->SetSignedIntegerValue(value);
+}
+
+void SBStructuredData::SetFloatValue(double value) {
+  LLDB_INSTRUMENT_VA(this, value);
+
+  m_impl_up->SetFloatValue(value);
+}
+
+void SBStructuredData::SetBooleanValue(bool value) {
+  LLDB_INSTRUMENT_VA(this, value);
+
+  m_impl_up->SetBooleanValue(value);
+}
+
+void SBStructuredData::SetStringValue(const char *value) {
+  LLDB_INSTRUMENT_VA(this, value);
+
+  m_impl_up->SetStringValue(value);
+}
+
+void SBStructuredData::SetGenericValue(SBScriptObject value) {
+  LLDB_INSTRUMENT_VA(this, value);
+
+  m_impl_up->SetGenericValue(value.GetPointer());
+}
diff --git a/lldb/source/API/SBSymbol.cpp b/lldb/source/API/SBSymbol.cpp
index 79477dd3a70fc..3b59119494f37 100644
--- a/lldb/source/API/SBSymbol.cpp
+++ b/lldb/source/API/SBSymbol.cpp
@@ -193,6 +193,14 @@ SymbolType SBSymbol::GetType() {
   return eSymbolTypeInvalid;
 }
 
+uint32_t SBSymbol::GetID() {
+  LLDB_INSTRUMENT_VA(this);
+
+  if (m_opaque_ptr)
+    return m_opaque_ptr->GetID();
+  return 0;
+}
+
 bool SBSymbol::IsExternal() {
   LLDB_INSTRUMENT_VA(this);
 
@@ -208,3 +216,23 @@ bool SBSymbol::IsSynthetic() {
     return m_opaque_ptr->IsSynthetic();
   return false;
 }
+
+bool SBSymbol::IsDebug() {
+  LLDB_INSTRUMENT_VA(this);
+
+  if (m_opaque_ptr)
+    return m_opaque_ptr->IsDebug();
+  return false;
+}
+
+const char *SBSymbol::GetTypeAsString(lldb::SymbolType symbol_type) {
+  LLDB_INSTRUMENT_VA(symbol_type);
+
+  return Symbol::GetTypeAsString(symbol_type);
+}
+
+lldb::SymbolType SBSymbol::GetTypeFromString(const char *str) {
+  LLDB_INSTRUMENT_VA(str);
+
+  return Symbol::GetTypeFromString(str);
+}
diff --git a/lldb/source/API/SBTarget.cpp b/lldb/source/API/SBTarget.cpp
index f26f7951edc6f..eb56337de3c44 100644
--- a/lldb/source/API/SBTarget.cpp
+++ b/lldb/source/API/SBTarget.cpp
@@ -766,16 +766,19 @@ SBBreakpoint SBTarget::BreakpointCreateByName(const char *symbol_name,
     const bool hardware = false;
     const LazyBool skip_prologue = eLazyBoolCalculate;
     const lldb::addr_t offset = 0;
+    const bool offset_is_insn_count = false;
     if (module_name && module_name[0]) {
       FileSpecList module_spec_list;
       module_spec_list.Append(FileSpec(module_name));
       sb_bp = target_sp->CreateBreakpoint(
           &module_spec_list, nullptr, symbol_name, eFunctionNameTypeAuto,
-          eLanguageTypeUnknown, offset, skip_prologue, internal, hardware);
+          eLanguageTypeUnknown, offset, offset_is_insn_count, skip_prologue,
+          internal, hardware);
     } else {
       sb_bp = target_sp->CreateBreakpoint(
           nullptr, nullptr, symbol_name, eFunctionNameTypeAuto,
-          eLanguageTypeUnknown, offset, skip_prologue, internal, hardware);
+          eLanguageTypeUnknown, offset, offset_is_insn_count, skip_prologue,
+          internal, hardware);
     }
   }
 
@@ -811,6 +814,17 @@ lldb::SBBreakpoint SBTarget::BreakpointCreateByName(
     const SBFileSpecList &comp_unit_list) {
   LLDB_INSTRUMENT_VA(this, symbol_name, name_type_mask, symbol_language,
                      module_list, comp_unit_list);
+  return BreakpointCreateByName(symbol_name, name_type_mask, symbol_language, 0,
+                                false, module_list, comp_unit_list);
+}
+
+lldb::SBBreakpoint SBTarget::BreakpointCreateByName(
+    const char *symbol_name, uint32_t name_type_mask,
+    LanguageType symbol_language, lldb::addr_t offset,
+    bool offset_is_insn_count, const SBFileSpecList &module_list,
+    const SBFileSpecList &comp_unit_list) {
+  LLDB_INSTRUMENT_VA(this, symbol_name, name_type_mask, symbol_language, offset,
+                     offset_is_insn_count, module_list, comp_unit_list);
 
   SBBreakpoint sb_bp;
   if (TargetSP target_sp = GetSP();
@@ -821,7 +835,8 @@ lldb::SBBreakpoint SBTarget::BreakpointCreateByName(
     std::lock_guard<std::recursive_mutex> guard(target_sp->GetAPIMutex());
     FunctionNameType mask = static_cast<FunctionNameType>(name_type_mask);
     sb_bp = target_sp->CreateBreakpoint(module_list.get(), comp_unit_list.get(),
-                                        symbol_name, mask, symbol_language, 0,
+                                        symbol_name, mask, symbol_language,
+                                        offset, offset_is_insn_count,
                                         skip_prologue, internal, hardware);
   }
 
@@ -1555,6 +1570,18 @@ SBModule SBTarget::FindModule(const SBFileSpec &sb_file_spec) {
   return sb_module;
 }
 
+SBModule SBTarget::FindModule(const SBModuleSpec &sb_module_spec) {
+  LLDB_INSTRUMENT_VA(this, sb_module_spec);
+
+  SBModule sb_module;
+  if (TargetSP target_sp = GetSP(); target_sp && sb_module_spec.IsValid()) {
+    // The module list is thread safe, no need to lock.
+    sb_module.SetSP(
+        target_sp->GetImages().FindFirstModule(*sb_module_spec.m_opaque_up));
+  }
+  return sb_module;
+}
+
 SBSymbolContextList SBTarget::FindCompileUnits(const SBFileSpec &sb_file_spec) {
   LLDB_INSTRUMENT_VA(this, sb_file_spec);
 
@@ -1955,29 +1982,10 @@ lldb::SBInstructionList SBTarget::ReadInstructions(lldb::SBAddress base_addr,
 
   if (TargetSP target_sp = GetSP()) {
     if (Address *addr_ptr = base_addr.get()) {
-      DataBufferHeap data(
-          target_sp->GetArchitecture().GetMaximumOpcodeByteSize() * count, 0);
-      bool force_live_memory = true;
-      lldb_private::Status error;
-      lldb::addr_t load_addr = LLDB_INVALID_ADDRESS;
-      const size_t bytes_read =
-          target_sp->ReadMemory(*addr_ptr, data.GetBytes(), data.GetByteSize(),
-                                error, force_live_memory, &load_addr);
-
-      const bool data_from_file = load_addr == LLDB_INVALID_ADDRESS;
-      if (!flavor_string || flavor_string[0] == '\0') {
-        // FIXME - we don't have the mechanism in place to do per-architecture
-        // settings.  But since we know that for now we only support flavors on
-        // x86 & x86_64,
-        const llvm::Triple::ArchType arch =
-            target_sp->GetArchitecture().GetTriple().getArch();
-        if (arch == llvm::Triple::x86 || arch == llvm::Triple::x86_64)
-          flavor_string = target_sp->GetDisassemblyFlavor();
+      if (llvm::Expected<DisassemblerSP> disassembler =
+              target_sp->ReadInstructions(*addr_ptr, count, flavor_string)) {
+        sb_instructions.SetDisassembler(*disassembler);
       }
-      sb_instructions.SetDisassembler(Disassembler::DisassembleBytes(
-          target_sp->GetArchitecture(), nullptr, flavor_string,
-          target_sp->GetDisassemblyCPU(), target_sp->GetDisassemblyFeatures(),
-          *addr_ptr, data.GetBytes(), bytes_read, count, data_from_file));
     }
   }
 
diff --git a/lldb/source/API/SBThread.cpp b/lldb/source/API/SBThread.cpp
index 74bc66c4f16f1..ec68b2a4b6f31 100644
--- a/lldb/source/API/SBThread.cpp
+++ b/lldb/source/API/SBThread.cpp
@@ -90,16 +90,17 @@ lldb::SBQueue SBThread::GetQueue() const {
 
   SBQueue sb_queue;
   QueueSP queue_sp;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      queue_sp = exe_ctx.GetThreadPtr()->GetQueue();
-      if (queue_sp) {
-        sb_queue.SetQueue(queue_sp);
-      }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBQueue();
+  }
+
+  if (exe_ctx->HasThreadScope()) {
+    queue_sp = exe_ctx->GetThreadPtr()->GetQueue();
+    if (queue_sp) {
+      sb_queue.SetQueue(queue_sp);
     }
   }
 
@@ -112,19 +113,17 @@ bool SBThread::IsValid() const {
 }
 SBThread::operator bool() const {
   LLDB_INSTRUMENT_VA(this);
+  if (!m_opaque_sp)
+    return false;
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  Target *target = exe_ctx.GetTargetPtr();
-  Process *process = exe_ctx.GetProcessPtr();
-  if (target && process) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&process->GetRunLock()))
-      return m_opaque_sp->GetThreadSP().get() != nullptr;
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
   }
-  // Without a valid target & process, this thread can't be valid.
-  return false;
+
+  return m_opaque_sp->GetThreadSP().get() != nullptr;
 }
 
 void SBThread::Clear() {
@@ -137,29 +136,27 @@ StopReason SBThread::GetStopReason() {
   LLDB_INSTRUMENT_VA(this);
 
   StopReason reason = eStopReasonInvalid;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      return exe_ctx.GetThreadPtr()->GetStopReason();
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return reason;
   }
 
+  if (exe_ctx->HasThreadScope())
+    return exe_ctx->GetThreadPtr()->GetStopReason();
+
   return reason;
 }
 
 size_t SBThread::GetStopReasonDataCount() {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      StopInfoSP stop_info_sp = exe_ctx.GetThreadPtr()->GetStopInfo();
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (exe_ctx) {
+    if (exe_ctx->HasThreadScope()) {
+      StopInfoSP stop_info_sp = exe_ctx->GetThreadPtr()->GetStopInfo();
       if (stop_info_sp) {
         StopReason reason = stop_info_sp->GetStopReason();
         switch (reason) {
@@ -179,7 +176,7 @@ size_t SBThread::GetStopReasonDataCount() {
         case eStopReasonBreakpoint: {
           break_id_t site_id = stop_info_sp->GetValue();
           lldb::BreakpointSiteSP bp_site_sp(
-              exe_ctx.GetProcessPtr()->GetBreakpointSiteList().FindByID(
+              exe_ctx->GetProcessPtr()->GetBreakpointSiteList().FindByID(
                   site_id));
           if (bp_site_sp)
             return bp_site_sp->GetNumberOfConstituents() * 2;
@@ -207,6 +204,9 @@ size_t SBThread::GetStopReasonDataCount() {
         }
       }
     }
+  } else {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return 0;
   }
   return 0;
 }
@@ -214,13 +214,11 @@ size_t SBThread::GetStopReasonDataCount() {
 uint64_t SBThread::GetStopReasonDataAtIndex(uint32_t idx) {
   LLDB_INSTRUMENT_VA(this, idx);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      Thread *thread = exe_ctx.GetThreadPtr();
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (exe_ctx) {
+    if (exe_ctx->HasThreadScope()) {
+      Thread *thread = exe_ctx->GetThreadPtr();
       StopInfoSP stop_info_sp = thread->GetStopInfo();
       if (stop_info_sp) {
         StopReason reason = stop_info_sp->GetStopReason();
@@ -241,7 +239,7 @@ uint64_t SBThread::GetStopReasonDataAtIndex(uint32_t idx) {
         case eStopReasonBreakpoint: {
           break_id_t site_id = stop_info_sp->GetValue();
           lldb::BreakpointSiteSP bp_site_sp(
-              exe_ctx.GetProcessPtr()->GetBreakpointSiteList().FindByID(
+              exe_ctx->GetProcessPtr()->GetBreakpointSiteList().FindByID(
                   site_id));
           if (bp_site_sp) {
             uint32_t bp_index = idx / 2;
@@ -280,6 +278,9 @@ uint64_t SBThread::GetStopReasonDataAtIndex(uint32_t idx) {
         }
       }
     }
+  } else {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return 0;
   }
   return 0;
 }
@@ -289,13 +290,17 @@ bool SBThread::GetStopReasonExtendedInfoAsJSON(lldb::SBStream &stream) {
 
   Stream &strm = stream.ref();
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
+  }
 
-  if (!exe_ctx.HasThreadScope())
+  if (!exe_ctx->HasThreadScope())
     return false;
 
-  StopInfoSP stop_info = exe_ctx.GetThreadPtr()->GetStopInfo();
+  StopInfoSP stop_info = exe_ctx->GetThreadPtr()->GetStopInfo();
   StructuredData::ObjectSP info = stop_info->GetExtendedInfo();
   if (!info)
     return false;
@@ -311,15 +316,19 @@ SBThread::GetStopReasonExtendedBacktraces(InstrumentationRuntimeType type) {
 
   SBThreadCollection threads;
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBThreadCollection();
+  }
 
-  if (!exe_ctx.HasThreadScope())
+  if (!exe_ctx->HasThreadScope())
     return SBThreadCollection();
 
-  ProcessSP process_sp = exe_ctx.GetProcessSP();
+  ProcessSP process_sp = exe_ctx->GetProcessSP();
 
-  StopInfoSP stop_info = exe_ctx.GetThreadPtr()->GetStopInfo();
+  StopInfoSP stop_info = exe_ctx->GetThreadPtr()->GetStopInfo();
   StructuredData::ObjectSP info = stop_info->GetExtendedInfo();
   if (!info)
     return threads;
@@ -332,20 +341,20 @@ SBThread::GetStopReasonExtendedBacktraces(InstrumentationRuntimeType type) {
 size_t SBThread::GetStopDescription(char *dst, size_t dst_len) {
   LLDB_INSTRUMENT_VA(this, dst, dst_len);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
   if (dst)
     *dst = 0;
 
-  if (!exe_ctx.HasThreadScope())
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
     return 0;
+  }
 
-  Process::StopLocker stop_locker;
-  if (!stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock()))
+  if (!exe_ctx->HasThreadScope())
     return 0;
 
-  std::string thread_stop_desc = exe_ctx.GetThreadPtr()->GetStopDescription();
+  std::string thread_stop_desc = exe_ctx->GetThreadPtr()->GetStopDescription();
   if (thread_stop_desc.empty())
     return 0;
 
@@ -361,16 +370,17 @@ SBValue SBThread::GetStopReturnValue() {
   LLDB_INSTRUMENT_VA(this);
 
   ValueObjectSP return_valobj_sp;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBValue();
+  }
 
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      StopInfoSP stop_info_sp = exe_ctx.GetThreadPtr()->GetStopInfo();
-      if (stop_info_sp) {
-        return_valobj_sp = StopInfo::GetReturnValueObject(stop_info_sp);
-      }
+  if (exe_ctx->HasThreadScope()) {
+    StopInfoSP stop_info_sp = exe_ctx->GetThreadPtr()->GetStopInfo();
+    if (stop_info_sp) {
+      return_valobj_sp = StopInfo::GetReturnValueObject(stop_info_sp);
     }
   }
 
@@ -402,47 +412,48 @@ uint32_t SBThread::GetIndexID() const {
 const char *SBThread::GetName() const {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (!exe_ctx.HasThreadScope())
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
     return nullptr;
+  }
 
-  Process::StopLocker stop_locker;
-  if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock()))
-    return ConstString(exe_ctx.GetThreadPtr()->GetName()).GetCString();
+  if (!exe_ctx->HasThreadScope())
+    return nullptr;
 
-  return nullptr;
+  return ConstString(exe_ctx->GetThreadPtr()->GetName()).GetCString();
 }
 
 const char *SBThread::GetQueueName() const {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (!exe_ctx.HasThreadScope())
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
     return nullptr;
+  }
 
-  Process::StopLocker stop_locker;
-  if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock()))
-    return ConstString(exe_ctx.GetThreadPtr()->GetQueueName()).GetCString();
+  if (!exe_ctx->HasThreadScope())
+    return nullptr;
 
-  return nullptr;
+  return ConstString(exe_ctx->GetThreadPtr()->GetQueueName()).GetCString();
 }
 
 lldb::queue_id_t SBThread::GetQueueID() const {
   LLDB_INSTRUMENT_VA(this);
 
   queue_id_t id = LLDB_INVALID_QUEUE_ID;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return id;
+  }
 
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      id = exe_ctx.GetThreadPtr()->GetQueueID();
-    }
+  if (exe_ctx->HasThreadScope()) {
+    id = exe_ctx->GetThreadPtr()->GetQueueID();
   }
 
   return id;
@@ -452,13 +463,11 @@ bool SBThread::GetInfoItemByPathAsString(const char *path, SBStream &strm) {
   LLDB_INSTRUMENT_VA(this, path, strm);
 
   bool success = false;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      Thread *thread = exe_ctx.GetThreadPtr();
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (exe_ctx) {
+    if (exe_ctx->HasThreadScope()) {
+      Thread *thread = exe_ctx->GetThreadPtr();
       StructuredData::ObjectSP info_root_sp = thread->GetExtendedInfo();
       if (info_root_sp) {
         StructuredData::ObjectSP node =
@@ -490,16 +499,16 @@ bool SBThread::GetInfoItemByPathAsString(const char *path, SBStream &strm) {
         }
       }
     }
+  } else {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return success;
   }
 
   return success;
 }
 
-static Status ResumeNewPlan(ExecutionContext &exe_ctx, ThreadPlan *new_plan) {
-  Process *process = exe_ctx.GetProcessPtr();
-  if (!process)
-    return Status::FromErrorString("No process in SBThread::ResumeNewPlan");
-
+static Status ResumeNewPlan(StoppedExecutionContext exe_ctx,
+                            ThreadPlan *new_plan) {
   Thread *thread = exe_ctx.GetThreadPtr();
   if (!thread)
     return Status::FromErrorString("No thread in SBThread::ResumeNewPlan");
@@ -512,8 +521,11 @@ static Status ResumeNewPlan(ExecutionContext &exe_ctx, ThreadPlan *new_plan) {
   }
 
   // Why do we need to set the current thread by ID here???
+  Process *process = exe_ctx.GetProcessPtr();
   process->GetThreadList().SetSelectedThreadByID(thread->GetID());
 
+  // Release the run lock but keep the API lock.
+  std::unique_lock<std::recursive_mutex> api_lock = exe_ctx.AllowResume();
   if (process->GetTarget().GetDebugger().GetAsyncExecution())
     return process->Resume();
   return process->ResumeSynchronous(nullptr);
@@ -529,15 +541,19 @@ void SBThread::StepOver(lldb::RunMode stop_other_threads) {
 void SBThread::StepOver(lldb::RunMode stop_other_threads, SBError &error) {
   LLDB_INSTRUMENT_VA(this, stop_other_threads, error);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    error = Status::FromError(exe_ctx.takeError());
+    return;
+  }
 
-  if (!exe_ctx.HasThreadScope()) {
+  if (!exe_ctx->HasThreadScope()) {
     error = Status::FromErrorString("this SBThread object is invalid");
     return;
   }
 
-  Thread *thread = exe_ctx.GetThreadPtr();
+  Thread *thread = exe_ctx->GetThreadPtr();
   bool abort_other_plans = false;
   StackFrameSP frame_sp(thread->GetStackFrameAtIndex(0));
 
@@ -555,7 +571,7 @@ void SBThread::StepOver(lldb::RunMode stop_other_threads, SBError &error) {
           true, abort_other_plans, stop_other_threads, new_plan_status);
     }
   }
-  error = ResumeNewPlan(exe_ctx, new_plan_sp.get());
+  error = ResumeNewPlan(std::move(*exe_ctx), new_plan_sp.get());
 }
 
 void SBThread::StepInto(lldb::RunMode stop_other_threads) {
@@ -576,17 +592,21 @@ void SBThread::StepInto(const char *target_name, uint32_t end_line,
                         SBError &error, lldb::RunMode stop_other_threads) {
   LLDB_INSTRUMENT_VA(this, target_name, end_line, error, stop_other_threads);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    error = Status::FromError(exe_ctx.takeError());
+    return;
+  }
 
-  if (!exe_ctx.HasThreadScope()) {
+  if (!exe_ctx->HasThreadScope()) {
     error = Status::FromErrorString("this SBThread object is invalid");
     return;
   }
 
   bool abort_other_plans = false;
 
-  Thread *thread = exe_ctx.GetThreadPtr();
+  Thread *thread = exe_ctx->GetThreadPtr();
   StackFrameSP frame_sp(thread->GetStackFrameAtIndex(0));
   ThreadPlanSP new_plan_sp;
   Status new_plan_status;
@@ -618,7 +638,7 @@ void SBThread::StepInto(const char *target_name, uint32_t end_line,
   }
 
   if (new_plan_status.Success())
-    error = ResumeNewPlan(exe_ctx, new_plan_sp.get());
+    error = ResumeNewPlan(std::move(*exe_ctx), new_plan_sp.get());
   else
     error = Status::FromErrorString(new_plan_status.AsCString());
 }
@@ -633,10 +653,14 @@ void SBThread::StepOut() {
 void SBThread::StepOut(SBError &error) {
   LLDB_INSTRUMENT_VA(this, error);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    error = Status::FromError(exe_ctx.takeError());
+    return;
+  }
 
-  if (!exe_ctx.HasThreadScope()) {
+  if (!exe_ctx->HasThreadScope()) {
     error = Status::FromErrorString("this SBThread object is invalid");
     return;
   }
@@ -644,7 +668,7 @@ void SBThread::StepOut(SBError &error) {
   bool abort_other_plans = false;
   bool stop_other_threads = false;
 
-  Thread *thread = exe_ctx.GetThreadPtr();
+  Thread *thread = exe_ctx->GetThreadPtr();
 
   const LazyBool avoid_no_debug = eLazyBoolCalculate;
   Status new_plan_status;
@@ -653,7 +677,7 @@ void SBThread::StepOut(SBError &error) {
       eVoteNoOpinion, 0, new_plan_status, avoid_no_debug));
 
   if (new_plan_status.Success())
-    error = ResumeNewPlan(exe_ctx, new_plan_sp.get());
+    error = ResumeNewPlan(std::move(*exe_ctx), new_plan_sp.get());
   else
     error = Status::FromErrorString(new_plan_status.AsCString());
 }
@@ -668,8 +692,12 @@ void SBThread::StepOutOfFrame(SBFrame &sb_frame) {
 void SBThread::StepOutOfFrame(SBFrame &sb_frame, SBError &error) {
   LLDB_INSTRUMENT_VA(this, sb_frame, error);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    error = Status::FromError(exe_ctx.takeError());
+    return;
+  }
 
   if (!sb_frame.IsValid()) {
     error = Status::FromErrorString("passed invalid SBFrame object");
@@ -678,14 +706,14 @@ void SBThread::StepOutOfFrame(SBFrame &sb_frame, SBError &error) {
 
   StackFrameSP frame_sp(sb_frame.GetFrameSP());
 
-  if (!exe_ctx.HasThreadScope()) {
+  if (!exe_ctx->HasThreadScope()) {
     error = Status::FromErrorString("this SBThread object is invalid");
     return;
   }
 
   bool abort_other_plans = false;
   bool stop_other_threads = false;
-  Thread *thread = exe_ctx.GetThreadPtr();
+  Thread *thread = exe_ctx->GetThreadPtr();
   if (sb_frame.GetThread().GetThreadID() != thread->GetID()) {
     error = Status::FromErrorString("passed a frame from another thread");
     return;
@@ -697,7 +725,7 @@ void SBThread::StepOutOfFrame(SBFrame &sb_frame, SBError &error) {
       eVoteNoOpinion, frame_sp->GetFrameIndex(), new_plan_status));
 
   if (new_plan_status.Success())
-    error = ResumeNewPlan(exe_ctx, new_plan_sp.get());
+    error = ResumeNewPlan(std::move(*exe_ctx), new_plan_sp.get());
   else
     error = Status::FromErrorString(new_plan_status.AsCString());
 }
@@ -712,21 +740,25 @@ void SBThread::StepInstruction(bool step_over) {
 void SBThread::StepInstruction(bool step_over, SBError &error) {
   LLDB_INSTRUMENT_VA(this, step_over, error);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    error = Status::FromError(exe_ctx.takeError());
+    return;
+  }
 
-  if (!exe_ctx.HasThreadScope()) {
+  if (!exe_ctx->HasThreadScope()) {
     error = Status::FromErrorString("this SBThread object is invalid");
     return;
   }
 
-  Thread *thread = exe_ctx.GetThreadPtr();
+  Thread *thread = exe_ctx->GetThreadPtr();
   Status new_plan_status;
   ThreadPlanSP new_plan_sp(thread->QueueThreadPlanForStepSingleInstruction(
       step_over, false, true, new_plan_status));
 
   if (new_plan_status.Success())
-    error = ResumeNewPlan(exe_ctx, new_plan_sp.get());
+    error = ResumeNewPlan(std::move(*exe_ctx), new_plan_sp.get());
   else
     error = Status::FromErrorString(new_plan_status.AsCString());
 }
@@ -741,10 +773,14 @@ void SBThread::RunToAddress(lldb::addr_t addr) {
 void SBThread::RunToAddress(lldb::addr_t addr, SBError &error) {
   LLDB_INSTRUMENT_VA(this, addr, error);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    error = Status::FromError(exe_ctx.takeError());
+    return;
+  }
 
-  if (!exe_ctx.HasThreadScope()) {
+  if (!exe_ctx->HasThreadScope()) {
     error = Status::FromErrorString("this SBThread object is invalid");
     return;
   }
@@ -754,14 +790,14 @@ void SBThread::RunToAddress(lldb::addr_t addr, SBError &error) {
 
   Address target_addr(addr);
 
-  Thread *thread = exe_ctx.GetThreadPtr();
+  Thread *thread = exe_ctx->GetThreadPtr();
 
   Status new_plan_status;
   ThreadPlanSP new_plan_sp(thread->QueueThreadPlanForRunToAddress(
       abort_other_plans, target_addr, stop_other_threads, new_plan_status));
 
   if (new_plan_status.Success())
-    error = ResumeNewPlan(exe_ctx, new_plan_sp.get());
+    error = ResumeNewPlan(std::move(*exe_ctx), new_plan_sp.get());
   else
     error = Status::FromErrorString(new_plan_status.AsCString());
 }
@@ -773,14 +809,16 @@ SBError SBThread::StepOverUntil(lldb::SBFrame &sb_frame,
   SBError sb_error;
   char path[PATH_MAX];
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx)
+    return Status::FromError(exe_ctx.takeError());
 
   StackFrameSP frame_sp(sb_frame.GetFrameSP());
 
-  if (exe_ctx.HasThreadScope()) {
-    Target *target = exe_ctx.GetTargetPtr();
-    Thread *thread = exe_ctx.GetThreadPtr();
+  if (exe_ctx->HasThreadScope()) {
+    Target *target = exe_ctx->GetTargetPtr();
+    Thread *thread = exe_ctx->GetThreadPtr();
 
     if (line == 0) {
       sb_error = Status::FromErrorString("invalid line argument");
@@ -875,7 +913,7 @@ SBError SBThread::StepOverUntil(lldb::SBFrame &sb_frame,
           frame_sp->GetFrameIndex(), new_plan_status));
 
       if (new_plan_status.Success())
-        sb_error = ResumeNewPlan(exe_ctx, new_plan_sp.get());
+        sb_error = ResumeNewPlan(std::move(*exe_ctx), new_plan_sp.get());
       else
         sb_error = Status::FromErrorString(new_plan_status.AsCString());
     }
@@ -907,15 +945,17 @@ SBError SBThread::StepUsingScriptedThreadPlan(const char *script_class_name,
 
   SBError error;
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx)
+    return Status::FromError(exe_ctx.takeError());
 
-  if (!exe_ctx.HasThreadScope()) {
+  if (!exe_ctx->HasThreadScope()) {
     error = Status::FromErrorString("this SBThread object is invalid");
     return error;
   }
 
-  Thread *thread = exe_ctx.GetThreadPtr();
+  Thread *thread = exe_ctx->GetThreadPtr();
   Status new_plan_status;
   StructuredData::ObjectSP obj_sp = args_data.m_impl_up->GetObjectSP();
 
@@ -931,7 +971,7 @@ SBError SBThread::StepUsingScriptedThreadPlan(const char *script_class_name,
     return error;
 
   if (new_plan_status.Success())
-    error = ResumeNewPlan(exe_ctx, new_plan_sp.get());
+    error = ResumeNewPlan(std::move(*exe_ctx), new_plan_sp.get());
   else
     error = Status::FromErrorString(new_plan_status.AsCString());
 
@@ -943,15 +983,17 @@ SBError SBThread::JumpToLine(lldb::SBFileSpec &file_spec, uint32_t line) {
 
   SBError sb_error;
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx)
+    return Status::FromError(exe_ctx.takeError());
 
-  if (!exe_ctx.HasThreadScope()) {
+  if (!exe_ctx->HasThreadScope()) {
     sb_error = Status::FromErrorString("this SBThread object is invalid");
     return sb_error;
   }
 
-  Thread *thread = exe_ctx.GetThreadPtr();
+  Thread *thread = exe_ctx->GetThreadPtr();
 
   Status err = thread->JumpToLine(file_spec.ref(), line, true);
   sb_error.SetError(std::move(err));
@@ -963,11 +1005,13 @@ SBError SBThread::ReturnFromFrame(SBFrame &frame, SBValue &return_value) {
 
   SBError sb_error;
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx)
+    return Status::FromError(exe_ctx.takeError());
 
-  if (exe_ctx.HasThreadScope()) {
-    Thread *thread = exe_ctx.GetThreadPtr();
+  if (exe_ctx->HasThreadScope()) {
+    Thread *thread = exe_ctx->GetThreadPtr();
     sb_error.SetError(
         thread->ReturnFromFrame(frame.GetFrameSP(), return_value.GetSP()));
   }
@@ -980,11 +1024,13 @@ SBError SBThread::UnwindInnermostExpression() {
 
   SBError sb_error;
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx)
+    return Status::FromError(exe_ctx.takeError());
 
-  if (exe_ctx.HasThreadScope()) {
-    Thread *thread = exe_ctx.GetThreadPtr();
+  if (exe_ctx->HasThreadScope()) {
+    Thread *thread = exe_ctx->GetThreadPtr();
     sb_error.SetError(thread->UnwindInnermostExpression());
     if (sb_error.Success())
       thread->SetSelectedFrameByIndex(0, false);
@@ -1003,18 +1049,17 @@ bool SBThread::Suspend() {
 bool SBThread::Suspend(SBError &error) {
   LLDB_INSTRUMENT_VA(this, error);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    error = Status::FromError(exe_ctx.takeError());
+    return false;
+  }
 
   bool result = false;
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      exe_ctx.GetThreadPtr()->SetResumeState(eStateSuspended);
-      result = true;
-    } else {
-      error = Status::FromErrorString("process is running");
-    }
+  if (exe_ctx->HasThreadScope()) {
+    exe_ctx->GetThreadPtr()->SetResumeState(eStateSuspended);
+    result = true;
   } else
     error = Status::FromErrorString("this SBThread object is invalid");
   return result;
@@ -1030,19 +1075,19 @@ bool SBThread::Resume() {
 bool SBThread::Resume(SBError &error) {
   LLDB_INSTRUMENT_VA(this, error);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    error = Status::FromErrorString("process is running");
+    return false;
+  }
 
   bool result = false;
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      const bool override_suspend = true;
-      exe_ctx.GetThreadPtr()->SetResumeState(eStateRunning, override_suspend);
-      result = true;
-    } else {
-      error = Status::FromErrorString("process is running");
-    }
+  if (exe_ctx->HasThreadScope()) {
+    const bool override_suspend = true;
+    exe_ctx->GetThreadPtr()->SetResumeState(eStateRunning, override_suspend);
+    result = true;
   } else
     error = Status::FromErrorString("this SBThread object is invalid");
   return result;
@@ -1051,22 +1096,30 @@ bool SBThread::Resume(SBError &error) {
 bool SBThread::IsSuspended() {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
+  }
 
-  if (exe_ctx.HasThreadScope())
-    return exe_ctx.GetThreadPtr()->GetResumeState() == eStateSuspended;
+  if (exe_ctx->HasThreadScope())
+    return exe_ctx->GetThreadPtr()->GetResumeState() == eStateSuspended;
   return false;
 }
 
 bool SBThread::IsStopped() {
   LLDB_INSTRUMENT_VA(this);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
+  }
 
-  if (exe_ctx.HasThreadScope())
-    return StateIsStoppedState(exe_ctx.GetThreadPtr()->GetState(), true);
+  if (exe_ctx->HasThreadScope())
+    return StateIsStoppedState(exe_ctx->GetThreadPtr()->GetState(), true);
   return false;
 }
 
@@ -1074,13 +1127,17 @@ SBProcess SBThread::GetProcess() {
   LLDB_INSTRUMENT_VA(this);
 
   SBProcess sb_process;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBProcess();
+  }
 
-  if (exe_ctx.HasThreadScope()) {
+  if (exe_ctx->HasThreadScope()) {
     // Have to go up to the target so we can get a shared pointer to our
     // process...
-    sb_process.SetSP(exe_ctx.GetProcessSP());
+    sb_process.SetSP(exe_ctx->GetProcessSP());
   }
 
   return sb_process;
@@ -1089,34 +1146,33 @@ SBProcess SBThread::GetProcess() {
 uint32_t SBThread::GetNumFrames() {
   LLDB_INSTRUMENT_VA(this);
 
-  uint32_t num_frames = 0;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      num_frames = exe_ctx.GetThreadPtr()->GetStackFrameCount();
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return 0;
   }
 
-  return num_frames;
+  if (exe_ctx->HasThreadScope())
+    return exe_ctx->GetThreadPtr()->GetStackFrameCount();
+
+  return 0;
 }
 
 SBFrame SBThread::GetFrameAtIndex(uint32_t idx) {
   LLDB_INSTRUMENT_VA(this, idx);
 
   SBFrame sb_frame;
-  StackFrameSP frame_sp;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBFrame();
+  }
 
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      frame_sp = exe_ctx.GetThreadPtr()->GetStackFrameAtIndex(idx);
-      sb_frame.SetFrameSP(frame_sp);
-    }
+  if (exe_ctx->HasThreadScope()) {
+    StackFrameSP frame_sp = exe_ctx->GetThreadPtr()->GetStackFrameAtIndex(idx);
+    sb_frame.SetFrameSP(frame_sp);
   }
 
   return sb_frame;
@@ -1126,17 +1182,17 @@ lldb::SBFrame SBThread::GetSelectedFrame() {
   LLDB_INSTRUMENT_VA(this);
 
   SBFrame sb_frame;
-  StackFrameSP frame_sp;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      frame_sp =
-          exe_ctx.GetThreadPtr()->GetSelectedFrame(SelectMostRelevantFrame);
-      sb_frame.SetFrameSP(frame_sp);
-    }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBFrame();
+  }
+
+  if (exe_ctx->HasThreadScope()) {
+    StackFrameSP frame_sp =
+        exe_ctx->GetThreadPtr()->GetSelectedFrame(SelectMostRelevantFrame);
+    sb_frame.SetFrameSP(frame_sp);
   }
 
   return sb_frame;
@@ -1147,18 +1203,19 @@ lldb::SBFrame SBThread::SetSelectedFrame(uint32_t idx) {
 
   SBFrame sb_frame;
   StackFrameSP frame_sp;
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
-
-  if (exe_ctx.HasThreadScope()) {
-    Process::StopLocker stop_locker;
-    if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-      Thread *thread = exe_ctx.GetThreadPtr();
-      frame_sp = thread->GetStackFrameAtIndex(idx);
-      if (frame_sp) {
-        thread->SetSelectedFrame(frame_sp.get());
-        sb_frame.SetFrameSP(frame_sp);
-      }
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return SBFrame();
+  }
+
+  if (exe_ctx->HasThreadScope()) {
+    Thread *thread = exe_ctx->GetThreadPtr();
+    frame_sp = thread->GetStackFrameAtIndex(idx);
+    if (frame_sp) {
+      thread->SetSelectedFrame(frame_sp.get());
+      sb_frame.SetFrameSP(frame_sp);
     }
   }
 
@@ -1202,12 +1259,16 @@ bool SBThread::GetStatus(SBStream &status) const {
 
   Stream &strm = status.ref();
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
+  }
 
-  if (exe_ctx.HasThreadScope()) {
-    exe_ctx.GetThreadPtr()->GetStatus(strm, 0, 1, 1, true,
-                                      /*show_hidden=*/true);
+  if (exe_ctx->HasThreadScope()) {
+    exe_ctx->GetThreadPtr()->GetStatus(strm, 0, 1, 1, true,
+                                       /*show_hidden=*/true);
   } else
     strm.PutCString("No status");
 
@@ -1225,11 +1286,15 @@ bool SBThread::GetDescription(SBStream &description, bool stop_format) const {
 
   Stream &strm = description.ref();
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return false;
+  }
 
-  if (exe_ctx.HasThreadScope()) {
-    exe_ctx.GetThreadPtr()->DumpUsingSettingsFormat(
+  if (exe_ctx->HasThreadScope()) {
+    exe_ctx->GetThreadPtr()->DumpUsingSettingsFormat(
         strm, LLDB_INVALID_THREAD_ID, stop_format);
   } else
     strm.PutCString("No value");
@@ -1247,11 +1312,15 @@ SBError SBThread::GetDescriptionWithFormat(const SBFormat &format,
     return error;
   }
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
+  if (!exe_ctx) {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
+    return error;
+  }
 
-  if (exe_ctx.HasThreadScope()) {
-    if (exe_ctx.GetThreadPtr()->DumpUsingFormat(
+  if (exe_ctx->HasThreadScope()) {
+    if (exe_ctx->GetThreadPtr()->DumpUsingFormat(
             strm, LLDB_INVALID_THREAD_ID, format.GetFormatEntrySP().get())) {
       return error;
     }
@@ -1267,17 +1336,15 @@ SBError SBThread::GetDescriptionWithFormat(const SBFormat &format,
 SBThread SBThread::GetExtendedBacktraceThread(const char *type) {
   LLDB_INSTRUMENT_VA(this, type);
 
-  std::unique_lock<std::recursive_mutex> lock;
-  ExecutionContext exe_ctx(m_opaque_sp.get(), lock);
+  llvm::Expected<StoppedExecutionContext> exe_ctx =
+      GetStoppedExecutionContext(m_opaque_sp);
   SBThread sb_origin_thread;
-
-  Process::StopLocker stop_locker;
-  if (stop_locker.TryLock(&exe_ctx.GetProcessPtr()->GetRunLock())) {
-    if (exe_ctx.HasThreadScope()) {
-      ThreadSP real_thread(exe_ctx.GetThreadSP());
+  if (exe_ctx) {
+    if (exe_ctx->HasThreadScope()) {
+      ThreadSP real_thread(exe_ctx->GetThreadSP());
       if (real_thread) {
         ConstString type_const(type);
-        Process *process = exe_ctx.GetProcessPtr();
+        Process *process = exe_ctx->GetProcessPtr();
         if (process) {
           SystemRuntime *runtime = process->GetSystemRuntime();
           if (runtime) {
@@ -1293,6 +1360,8 @@ SBThread SBThread::GetExtendedBacktraceThread(const char *type) {
         }
       }
     }
+  } else {
+    LLDB_LOG_ERROR(GetLog(LLDBLog::API), exe_ctx.takeError(), "{0}");
   }
 
   return sb_origin_thread;
diff --git a/lldb/source/Breakpoint/BreakpointResolver.cpp b/lldb/source/Breakpoint/BreakpointResolver.cpp
index 91fdff4a455da..4ac40501a5df5 100644
--- a/lldb/source/Breakpoint/BreakpointResolver.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolver.cpp
@@ -42,9 +42,9 @@ const char *BreakpointResolver::g_ty_to_name[] = {"FileAndLine", "Address",
 
 const char *BreakpointResolver::g_option_names[static_cast<uint32_t>(
     BreakpointResolver::OptionNames::LastOptionName)] = {
-    "AddressOffset", "Exact",     "FileName",     "Inlines",     "Language",
-    "LineNumber",    "Column",    "ModuleName",   "NameMask",    "Offset",
-    "PythonClass",   "Regex",     "ScriptArgs",   "SectionName", "SearchDepth",
+    "AddressOffset", "Exact",      "FileName",   "Inlines",     "Language",
+    "LineNumber",    "Column",     "ModuleName", "NameMask",    "Offset",
+    "PythonClass",   "Regex",      "ScriptArgs", "SectionName", "SearchDepth",
     "SkipPrologue",  "SymbolNames"};
 
 const char *BreakpointResolver::ResolverTyToName(enum ResolverTy type) {
@@ -65,8 +65,10 @@ BreakpointResolver::NameToResolverTy(llvm::StringRef name) {
 
 BreakpointResolver::BreakpointResolver(const BreakpointSP &bkpt,
                                        const unsigned char resolverTy,
-                                       lldb::addr_t offset)
-    : m_breakpoint(bkpt), m_offset(offset), SubclassID(resolverTy) {}
+                                       lldb::addr_t offset,
+                                       bool offset_is_insn_count)
+    : m_breakpoint(bkpt), m_offset(offset),
+      m_offset_is_insn_count(offset_is_insn_count), SubclassID(resolverTy) {}
 
 BreakpointResolver::~BreakpointResolver() = default;
 
@@ -364,7 +366,32 @@ void BreakpointResolver::AddLocation(SearchFilter &filter,
 
 BreakpointLocationSP BreakpointResolver::AddLocation(Address loc_addr,
                                                      bool *new_location) {
-  loc_addr.Slide(m_offset);
+  if (m_offset_is_insn_count) {
+    Target &target = GetBreakpoint()->GetTarget();
+    llvm::Expected<DisassemblerSP> expected_instructions =
+        target.ReadInstructions(loc_addr, m_offset);
+    if (!expected_instructions) {
+      LLDB_LOG_ERROR(GetLog(LLDBLog::Breakpoints),
+                     expected_instructions.takeError(),
+                     "error: Unable to read instructions at address 0x{0:x}",
+                     loc_addr.GetLoadAddress(&target));
+      return BreakpointLocationSP();
+    }
+
+    const DisassemblerSP instructions = *expected_instructions;
+    if (!instructions ||
+        instructions->GetInstructionList().GetSize() != m_offset) {
+      LLDB_LOG(GetLog(LLDBLog::Breakpoints),
+               "error: Unable to read {0} instructions at address 0x{1:x}",
+               m_offset, loc_addr.GetLoadAddress(&target));
+      return BreakpointLocationSP();
+    }
+
+    loc_addr.Slide(instructions->GetInstructionList().GetTotalByteSize());
+  } else {
+    loc_addr.Slide(m_offset);
+  }
+
   return GetBreakpoint()->AddLocation(loc_addr, new_location);
 }
 
diff --git a/lldb/source/Breakpoint/BreakpointResolverAddress.cpp b/lldb/source/Breakpoint/BreakpointResolverAddress.cpp
index 828647ceef637..70360d9d960ec 100644
--- a/lldb/source/Breakpoint/BreakpointResolverAddress.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolverAddress.cpp
@@ -133,6 +133,11 @@ Searcher::CallbackReturn BreakpointResolverAddress::SearchCallback(
           Address tmp_address;
           if (module_sp->ResolveFileAddress(m_addr.GetOffset(), tmp_address))
             m_addr = tmp_address;
+          else
+            return Searcher::eCallbackReturnStop;
+        } else {
+          // If we didn't find the module, then we can't resolve the address.
+          return Searcher::eCallbackReturnStop;
         }
       }
 
diff --git a/lldb/source/Breakpoint/BreakpointResolverName.cpp b/lldb/source/Breakpoint/BreakpointResolverName.cpp
index 21024a4198e1d..6372595a0f21f 100644
--- a/lldb/source/Breakpoint/BreakpointResolverName.cpp
+++ b/lldb/source/Breakpoint/BreakpointResolverName.cpp
@@ -24,11 +24,13 @@
 using namespace lldb;
 using namespace lldb_private;
 
-BreakpointResolverName::BreakpointResolverName(const BreakpointSP &bkpt,
-    const char *name_cstr, FunctionNameType name_type_mask,
-    LanguageType language, Breakpoint::MatchType type, lldb::addr_t offset,
+BreakpointResolverName::BreakpointResolverName(
+    const BreakpointSP &bkpt, const char *name_cstr,
+    FunctionNameType name_type_mask, LanguageType language,
+    Breakpoint::MatchType type, lldb::addr_t offset, bool offset_is_insn_count,
     bool skip_prologue)
-    : BreakpointResolver(bkpt, BreakpointResolver::NameResolver, offset),
+    : BreakpointResolver(bkpt, BreakpointResolver::NameResolver, offset,
+                         offset_is_insn_count),
       m_match_type(type), m_language(language), m_skip_prologue(skip_prologue) {
   if (m_match_type == Breakpoint::Regexp) {
     m_regex = RegularExpression(name_cstr);
@@ -81,7 +83,7 @@ BreakpointResolverName::BreakpointResolverName(const BreakpointSP &bkpt,
 BreakpointResolverName::BreakpointResolverName(
     const BreakpointResolverName &rhs)
     : BreakpointResolver(rhs.GetBreakpoint(), BreakpointResolver::NameResolver,
-                         rhs.GetOffset()),
+                         rhs.GetOffset(), rhs.GetOffsetIsInsnCount()),
       m_lookups(rhs.m_lookups), m_class_name(rhs.m_class_name),
       m_regex(rhs.m_regex), m_match_type(rhs.m_match_type),
       m_language(rhs.m_language), m_skip_prologue(rhs.m_skip_prologue) {}
@@ -177,7 +179,8 @@ BreakpointResolverSP BreakpointResolverName::CreateFromStructuredData(
     std::shared_ptr<BreakpointResolverName> resolver_sp =
         std::make_shared<BreakpointResolverName>(
             nullptr, names[0].c_str(), name_masks[0], language,
-            Breakpoint::MatchType::Exact, offset, skip_prologue);
+            Breakpoint::MatchType::Exact, offset,
+            /*offset_is_insn_count = */ false, skip_prologue);
     for (size_t i = 1; i < num_elem; i++) {
       resolver_sp->AddNameLookup(ConstString(names[i]), name_masks[i]);
     }
diff --git a/lldb/source/Commands/CommandObjectDWIMPrint.cpp b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
index d7589cc0396f6..0d9eb45732161 100644
--- a/lldb/source/Commands/CommandObjectDWIMPrint.cpp
+++ b/lldb/source/Commands/CommandObjectDWIMPrint.cpp
@@ -18,14 +18,11 @@
 #include "lldb/Interpreter/OptionGroupValueObjectDisplay.h"
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Utility/ConstString.h"
-#include "lldb/Utility/LLDBLog.h"
-#include "lldb/Utility/Log.h"
 #include "lldb/ValueObject/ValueObject.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Error.h"
 
 #include <regex>
 
@@ -93,7 +90,7 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
   dump_options.SetHideRootName(suppress_result)
       .SetExpandPointerTypeFlags(lldb::eTypeIsObjC);
 
-  bool is_po = m_varobj_options.use_objc;
+  bool is_po = m_varobj_options.use_object_desc;
 
   StackFrame *frame = m_exe_ctx.GetFramePtr();
 
@@ -135,22 +132,27 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
   };
 
   // Dump `valobj` according to whether `po` was requested or not.
-  auto dump_val_object = [&](ValueObject &valobj) -> Error {
+  auto dump_val_object = [&](ValueObject &valobj) {
     if (is_po) {
       StreamString temp_result_stream;
-      if (Error err = valobj.Dump(temp_result_stream, dump_options))
-        return err;
+      if (llvm::Error error = valobj.Dump(temp_result_stream, dump_options)) {
+        result.AppendError(toString(std::move(error)));
+        return;
+      }
       llvm::StringRef output = temp_result_stream.GetString();
       maybe_add_hint(output);
       result.GetOutputStream() << output;
     } else {
-      if (Error err = valobj.Dump(result.GetOutputStream(), dump_options))
-        return err;
+      llvm::Error error =
+        valobj.Dump(result.GetOutputStream(), dump_options);
+      if (error) {
+        result.AppendError(toString(std::move(error)));
+        return;
+      }
     }
     m_interpreter.PrintWarningsIfNecessary(result.GetOutputStream(),
                                            m_cmd_name);
     result.SetStatus(eReturnStatusSuccessFinishResult);
-    return Error::success();
   };
 
   // First, try `expr` as a _limited_ frame variable expression path: only the
@@ -184,13 +186,8 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
                                      expr);
       }
 
-      Error err = dump_val_object(*valobj_sp);
-      if (!err)
-        return;
-
-      // Dump failed, continue on to expression evaluation.
-      LLDB_LOG_ERROR(GetLog(LLDBLog::Expressions), std::move(err),
-                     "could not print frame variable '{1}': {0}", expr);
+      dump_val_object(*valobj_sp);
+      return;
     }
   }
 
@@ -199,14 +196,8 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
     if (auto *state = target.GetPersistentExpressionStateForLanguage(language))
       if (auto var_sp = state->GetVariable(expr))
         if (auto valobj_sp = var_sp->GetValueObject()) {
-          Error err = dump_val_object(*valobj_sp);
-          if (!err)
-            return;
-
-          // Dump failed, continue on to expression evaluation.
-          LLDB_LOG_ERROR(GetLog(LLDBLog::Expressions), std::move(err),
-                         "could not print persistent variable '{1}': {0}",
-                         expr);
+          dump_val_object(*valobj_sp);
+          return;
         }
 
   // Third, and lastly, try `expr` as a source expression to evaluate.
@@ -257,12 +248,10 @@ void CommandObjectDWIMPrint::DoExecute(StringRef command,
       result.AppendNoteWithFormatv("ran `expression {0}{1}`", flags, expr);
     }
 
-    if (valobj_sp->GetError().GetError() != UserExpression::kNoResult) {
-      if (Error err = dump_val_object(*valobj_sp))
-        result.SetError(std::move(err));
-    } else {
+    if (valobj_sp->GetError().GetError() != UserExpression::kNoResult)
+      dump_val_object(*valobj_sp);
+    else
       result.SetStatus(eReturnStatusSuccessFinishNoResult);
-    }
 
     if (suppress_result)
       if (auto result_var_sp =
diff --git a/lldb/source/Commands/CommandObjectExpression.cpp b/lldb/source/Commands/CommandObjectExpression.cpp
index c5b91678103d5..197bffe9c982f 100644
--- a/lldb/source/Commands/CommandObjectExpression.cpp
+++ b/lldb/source/Commands/CommandObjectExpression.cpp
@@ -202,7 +202,7 @@ EvaluateExpressionOptions
 CommandObjectExpression::CommandOptions::GetEvaluateExpressionOptions(
     const Target &target, const OptionGroupValueObjectDisplay &display_opts) {
   EvaluateExpressionOptions options;
-  options.SetCoerceToId(display_opts.use_objc);
+  options.SetCoerceToId(display_opts.use_object_desc);
   options.SetUnwindOnError(unwind_on_error);
   options.SetIgnoreBreakpoints(ignore_breakpoints);
   options.SetKeepInMemory(true);
@@ -241,11 +241,11 @@ CommandObjectExpression::CommandOptions::GetEvaluateExpressionOptions(
 bool CommandObjectExpression::CommandOptions::ShouldSuppressResult(
     const OptionGroupValueObjectDisplay &display_opts) const {
   // Explicitly disabling persistent results takes precedence over the
-  // m_verbosity/use_objc logic.
+  // m_verbosity/use_object_desc logic.
   if (suppress_persistent_result != eLazyBoolCalculate)
     return suppress_persistent_result == eLazyBoolYes;
 
-  return display_opts.use_objc &&
+  return display_opts.use_object_desc &&
          m_verbosity == eLanguageRuntimeDescriptionDisplayVerbosityCompact;
 }
 
@@ -332,7 +332,7 @@ Options *CommandObjectExpression::GetOptions() { return &m_option_group; }
 
 void CommandObjectExpression::HandleCompletion(CompletionRequest &request) {
   EvaluateExpressionOptions options;
-  options.SetCoerceToId(m_varobj_options.use_objc);
+  options.SetCoerceToId(m_varobj_options.use_object_desc);
   options.SetLanguage(m_command_options.language);
   options.SetExecutionPolicy(lldb_private::eExecutionPolicyNever);
   options.SetAutoApplyFixIts(false);
diff --git a/lldb/source/Commands/CommandObjectSettings.cpp b/lldb/source/Commands/CommandObjectSettings.cpp
index 7bbb0dd567ab1..126f57c738115 100644
--- a/lldb/source/Commands/CommandObjectSettings.cpp
+++ b/lldb/source/Commands/CommandObjectSettings.cpp
@@ -237,28 +237,62 @@ insert-before or insert-after.");
 };
 
 // CommandObjectSettingsShow -- Show current values
+#define LLDB_OPTIONS_settings_show
+#include "CommandOptions.inc"
 
 class CommandObjectSettingsShow : public CommandObjectParsed {
 public:
   CommandObjectSettingsShow(CommandInterpreter &interpreter)
       : CommandObjectParsed(interpreter, "settings show",
                             "Show matching debugger settings and their current "
-                            "values.  Defaults to showing all settings.",
-                            nullptr) {
+                            "values.  Defaults to showing all settings.") {
     AddSimpleArgumentList(eArgTypeSettingVariableName, eArgRepeatOptional);
   }
 
   ~CommandObjectSettingsShow() override = default;
 
+  Options *GetOptions() override { return &m_options; }
+
+  class CommandOptions : public Options {
+  public:
+    ~CommandOptions() override = default;
+
+    Status SetOptionValue(uint32_t option_idx, llvm::StringRef option_arg,
+                          ExecutionContext *execution_context) override {
+      const int short_option = m_getopt_table[option_idx].val;
+      switch (short_option) {
+      case 'd':
+        m_include_defaults = true;
+        break;
+      default:
+        llvm_unreachable("Unimplemented option");
+      }
+      return {};
+    }
+
+    void OptionParsingStarting(ExecutionContext *execution_context) override {
+      m_include_defaults = false;
+    }
+
+    llvm::ArrayRef<OptionDefinition> GetDefinitions() override {
+      return g_settings_show_options;
+    }
+
+    bool m_include_defaults = false;
+  };
+
 protected:
   void DoExecute(Args &args, CommandReturnObject &result) override {
     result.SetStatus(eReturnStatusSuccessFinishResult);
 
+    uint32_t dump_mask = OptionValue::eDumpGroupValue;
+    if (m_options.m_include_defaults)
+      dump_mask |= OptionValue::eDumpOptionDefaultValue;
+
     if (!args.empty()) {
       for (const auto &arg : args) {
         Status error(GetDebugger().DumpPropertyValue(
-            &m_exe_ctx, result.GetOutputStream(), arg.ref(),
-            OptionValue::eDumpGroupValue));
+            &m_exe_ctx, result.GetOutputStream(), arg.ref(), dump_mask));
         if (error.Success()) {
           result.GetOutputStream().EOL();
         } else {
@@ -267,9 +301,12 @@ class CommandObjectSettingsShow : public CommandObjectParsed {
       }
     } else {
       GetDebugger().DumpAllPropertyValues(&m_exe_ctx, result.GetOutputStream(),
-                                          OptionValue::eDumpGroupValue);
+                                          dump_mask);
     }
   }
+
+private:
+  CommandOptions m_options;
 };
 
 // CommandObjectSettingsWrite -- Write settings to file
diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td
index acb741081cac3..17d72cdcad129 100644
--- a/lldb/source/Commands/Options.td
+++ b/lldb/source/Commands/Options.td
@@ -56,6 +56,11 @@ let Command = "settings clear" in {
     Desc<"Clear all settings.">;
 }
 
+let Command = "settings show" in {
+  def setshow_defaults : Option<"defaults", "d">,
+    Desc<"Include default values if defined.">;
+}
+
 let Command = "breakpoint list" in {
   // FIXME: We need to add an "internal" command, and then add this sort of
   // thing to it. But I need to see it for now, and don't want to wait.
@@ -205,7 +210,7 @@ let Command = "breakpoint set" in {
     "identifiers). If not set the target.language setting is used.">;
   def breakpoint_set_skip_prologue : Option<"skip-prologue", "K">,
     Arg<"Boolean">, Groups<[1,3,4,5,6,7,8,12]>,
-    Desc<"sKip the prologue if the breakpoint is at the beginning of a "
+    Desc<"Skip the prologue if the breakpoint is at the beginning of a "
     "function. If not set the target.skip-prologue setting is used.">;
   def breakpoint_set_breakpoint_name : Option<"breakpoint-name", "N">,
     Arg<"BreakpointName">,
diff --git a/lldb/source/Core/DemangledNameInfo.cpp b/lldb/source/Core/DemangledNameInfo.cpp
index 00227f02bbff8..76f8987c5149c 100644
--- a/lldb/source/Core/DemangledNameInfo.cpp
+++ b/lldb/source/Core/DemangledNameInfo.cpp
@@ -111,6 +111,11 @@ void TrackingOutputBuffer::finalizeEnd() {
   if (NameInfo.ScopeRange.first > NameInfo.ScopeRange.second)
     NameInfo.ScopeRange.second = NameInfo.ScopeRange.first;
   NameInfo.BasenameRange.first = NameInfo.ScopeRange.second;
+
+  // We call anything past the FunctionEncoding the "suffix".
+  // In practice this would be nodes like `DotSuffix` that wrap
+  // a FunctionEncoding.
+  NameInfo.SuffixRange.first = getCurrentPosition();
 }
 
 ScopedOverride<unsigned> TrackingOutputBuffer::enterFunctionTypePrinting() {
@@ -138,6 +143,9 @@ void TrackingOutputBuffer::printLeft(const Node &N) {
   default:
     OutputBuffer::printLeft(N);
   }
+
+  // Keep updating suffix until we reach the end.
+  NameInfo.SuffixRange.second = getCurrentPosition();
 }
 
 void TrackingOutputBuffer::printRight(const Node &N) {
@@ -151,6 +159,9 @@ void TrackingOutputBuffer::printRight(const Node &N) {
   default:
     OutputBuffer::printRight(N);
   }
+
+  // Keep updating suffix until we reach the end.
+  NameInfo.SuffixRange.second = getCurrentPosition();
 }
 
 void TrackingOutputBuffer::printLeftImpl(const FunctionType &N) {
diff --git a/lldb/source/Core/Disassembler.cpp b/lldb/source/Core/Disassembler.cpp
index 925de2a5c836c..e0a7d69345706 100644
--- a/lldb/source/Core/Disassembler.cpp
+++ b/lldb/source/Core/Disassembler.cpp
@@ -1016,6 +1016,16 @@ uint32_t InstructionList::GetMaxOpcocdeByteSize() const {
   return max_inst_size;
 }
 
+size_t InstructionList::GetTotalByteSize() const {
+  size_t total_byte_size = 0;
+  collection::const_iterator pos, end;
+  for (pos = m_instructions.begin(), end = m_instructions.end(); pos != end;
+       ++pos) {
+    total_byte_size += (*pos)->GetOpcode().GetByteSize();
+  }
+  return total_byte_size;
+}
+
 InstructionSP InstructionList::GetInstructionAtIndex(size_t idx) const {
   InstructionSP inst_sp;
   if (idx < m_instructions.size())
diff --git a/lldb/source/Core/DynamicLoader.cpp b/lldb/source/Core/DynamicLoader.cpp
index 4be9f3eb9abc5..7580b15c02ce1 100644
--- a/lldb/source/Core/DynamicLoader.cpp
+++ b/lldb/source/Core/DynamicLoader.cpp
@@ -211,7 +211,7 @@ ModuleSP DynamicLoader::LoadBinaryWithUUIDAndAddress(
   if (uuid.IsValid())
     prog_str << uuid.GetAsString();
   if (value_is_offset == 0 && value != LLDB_INVALID_ADDRESS) {
-    prog_str << "at 0x";
+    prog_str << " at 0x";
     prog_str.PutHex64(value);
   }
 
diff --git a/lldb/source/Core/Mangled.cpp b/lldb/source/Core/Mangled.cpp
index 3663f430111c2..ce4db4e0daa8b 100644
--- a/lldb/source/Core/Mangled.cpp
+++ b/lldb/source/Core/Mangled.cpp
@@ -172,9 +172,6 @@ GetItaniumDemangledStr(const char *M) {
 
     TrackingOutputBuffer OB(demangled_cstr, demangled_size);
     demangled_cstr = ipd.finishDemangle(&OB);
-    // TODO: we should set the SuffixRange inside the TrackingOutputBuffer.
-    OB.NameInfo.SuffixRange.first = OB.NameInfo.QualifiersRange.second;
-    OB.NameInfo.SuffixRange.second = std::string_view(OB).size();
     info = std::move(OB.NameInfo);
 
     assert(demangled_cstr &&
diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp
index d5ddc2b249e56..060732f421f88 100644
--- a/lldb/source/Core/ModuleList.cpp
+++ b/lldb/source/Core/ModuleList.cpp
@@ -755,8 +755,234 @@ size_t ModuleList::GetIndexForModule(const Module *module) const {
 }
 
 namespace {
+/// A wrapper around ModuleList for shared modules. Provides fast lookups for
+/// file-based ModuleSpec queries.
+class SharedModuleList {
+public:
+  /// Finds all the modules matching the module_spec, and adds them to \p
+  /// matching_module_list.
+  void FindModules(const ModuleSpec &module_spec,
+                   ModuleList &matching_module_list) const {
+    std::lock_guard<std::recursive_mutex> guard(GetMutex());
+    // Try map first for performance - if found, skip expensive full list
+    // search.
+    FindModulesInMap(module_spec, matching_module_list);
+    if (!matching_module_list.IsEmpty())
+      return;
+    m_list.FindModules(module_spec, matching_module_list);
+    // Assert that modules were found in the list but not the map, it's
+    // because the module_spec has no filename or the found module has a
+    // different filename. For example, when searching by UUID and finding a
+    // module with an alias.
+    assert((matching_module_list.IsEmpty() ||
+            module_spec.GetFileSpec().GetFilename().IsEmpty() ||
+            module_spec.GetFileSpec().GetFilename() !=
+                matching_module_list.GetModuleAtIndex(0)
+                    ->GetFileSpec()
+                    .GetFilename()) &&
+           "Search by name not found in SharedModuleList's map");
+  }
+
+  ModuleSP FindModule(const Module &module) {
+
+    std::lock_guard<std::recursive_mutex> guard(GetMutex());
+    if (ModuleSP result = FindModuleInMap(module))
+      return result;
+    return m_list.FindModule(&module);
+  }
+
+  // UUID searches bypass map since UUIDs aren't indexed by filename.
+  ModuleSP FindModule(const UUID &uuid) const {
+    return m_list.FindModule(uuid);
+  }
+
+  void Append(const ModuleSP &module_sp, bool use_notifier) {
+    if (!module_sp)
+      return;
+    std::lock_guard<std::recursive_mutex> guard(GetMutex());
+    m_list.Append(module_sp, use_notifier);
+    AddToMap(module_sp);
+  }
+
+  size_t RemoveOrphans(bool mandatory) {
+    std::unique_lock<std::recursive_mutex> lock(GetMutex(), std::defer_lock);
+    if (mandatory) {
+      lock.lock();
+    } else {
+      if (!lock.try_lock())
+        return 0;
+    }
+    size_t total_count = 0;
+    size_t run_count;
+    do {
+      // Remove indexed orphans first, then remove non-indexed orphans. This
+      // order is important because the shared count will be different if a
+      // module is indexed or not.
+      run_count = RemoveOrphansFromMapAndList();
+      run_count += m_list.RemoveOrphans(mandatory);
+      total_count += run_count;
+      // Because removing orphans might make new orphans, remove from both
+      // containers until a fixed-point is reached.
+    } while (run_count != 0);
+
+    return total_count;
+  }
+
+  bool Remove(const ModuleSP &module_sp, bool use_notifier = true) {
+    if (!module_sp)
+      return false;
+    std::lock_guard<std::recursive_mutex> guard(GetMutex());
+    RemoveFromMap(*module_sp.get());
+    return m_list.Remove(module_sp, use_notifier);
+  }
+
+  void ReplaceEquivalent(const ModuleSP &module_sp,
+                         llvm::SmallVectorImpl<lldb::ModuleSP> *old_modules) {
+    std::lock_guard<std::recursive_mutex> guard(GetMutex());
+    m_list.ReplaceEquivalent(module_sp, old_modules);
+    ReplaceEquivalentInMap(module_sp);
+  }
+
+  bool RemoveIfOrphaned(const Module *module_ptr) {
+    if (!module_ptr)
+      return false;
+    std::lock_guard<std::recursive_mutex> guard(GetMutex());
+    RemoveFromMap(*module_ptr, /*if_orphaned=*/true);
+    return m_list.RemoveIfOrphaned(module_ptr);
+  }
+
+  std::recursive_mutex &GetMutex() const { return m_list.GetMutex(); }
+
+private:
+  ModuleSP FindModuleInMap(const Module &module) const {
+    if (!module.GetFileSpec().GetFilename())
+      return ModuleSP();
+    ConstString name = module.GetFileSpec().GetFilename();
+    auto it = m_name_to_modules.find(name);
+    if (it == m_name_to_modules.end())
+      return ModuleSP();
+    const llvm::SmallVectorImpl<ModuleSP> &vector = it->second;
+    for (const ModuleSP &module_sp : vector) {
+      if (module_sp.get() == &module)
+        return module_sp;
+    }
+    return ModuleSP();
+  }
+
+  void FindModulesInMap(const ModuleSpec &module_spec,
+                        ModuleList &matching_module_list) const {
+    auto it = m_name_to_modules.find(module_spec.GetFileSpec().GetFilename());
+    if (it == m_name_to_modules.end())
+      return;
+    const llvm::SmallVectorImpl<ModuleSP> &vector = it->second;
+    for (const ModuleSP &module_sp : vector) {
+      if (module_sp->MatchesModuleSpec(module_spec))
+        matching_module_list.Append(module_sp);
+    }
+  }
+
+  void AddToMap(const ModuleSP &module_sp) {
+    ConstString name = module_sp->GetFileSpec().GetFilename();
+    if (name.IsEmpty())
+      return;
+    m_name_to_modules[name].push_back(module_sp);
+  }
+
+  void RemoveFromMap(const Module &module, bool if_orphaned = false) {
+    ConstString name = module.GetFileSpec().GetFilename();
+    if (!m_name_to_modules.contains(name))
+      return;
+    llvm::SmallVectorImpl<ModuleSP> &vec = m_name_to_modules[name];
+    for (auto *it = vec.begin(); it != vec.end(); ++it) {
+      if (it->get() == &module) {
+        if (!if_orphaned || it->use_count() == kUseCountOrphaned) {
+          vec.erase(it);
+          break;
+        }
+      }
+    }
+  }
+
+  void ReplaceEquivalentInMap(const ModuleSP &module_sp) {
+    RemoveEquivalentModulesFromMap(module_sp);
+    AddToMap(module_sp);
+  }
+
+  void RemoveEquivalentModulesFromMap(const ModuleSP &module_sp) {
+    ConstString name = module_sp->GetFileSpec().GetFilename();
+    if (name.IsEmpty())
+      return;
+
+    auto it = m_name_to_modules.find(name);
+    if (it == m_name_to_modules.end())
+      return;
+
+    // First remove any equivalent modules. Equivalent modules are modules
+    // whose path, platform path and architecture match.
+    ModuleSpec equivalent_module_spec(module_sp->GetFileSpec(),
+                                      module_sp->GetArchitecture());
+    equivalent_module_spec.GetPlatformFileSpec() =
+        module_sp->GetPlatformFileSpec();
+
+    llvm::SmallVectorImpl<ModuleSP> &vec = it->second;
+    llvm::erase_if(vec, [&equivalent_module_spec](ModuleSP &element) {
+      return element->MatchesModuleSpec(equivalent_module_spec);
+    });
+  }
+
+  /// Remove orphans from the vector and return the removed modules.
+  ModuleList RemoveOrphansFromVector(llvm::SmallVectorImpl<ModuleSP> &vec) {
+    // remove_if moves the elements that match the condition to the end of the
+    // container, and returns an iterator to the first element that was moved.
+    auto *to_remove_start = llvm::remove_if(vec, [](const ModuleSP &module) {
+      return module.use_count() == kUseCountOrphaned;
+    });
+
+    ModuleList to_remove;
+    for (ModuleSP *it = to_remove_start; it != vec.end(); ++it)
+      to_remove.Append(*it);
+
+    vec.erase(to_remove_start, vec.end());
+    return to_remove;
+  }
+
+  /// Remove orphans that exist in both the map and list. This does not remove
+  /// any orphans that exist exclusively on the list.
+  ///
+  /// The mutex must be locked by the caller.
+  int RemoveOrphansFromMapAndList() {
+    // Modules might hold shared pointers to other modules, so removing one
+    // module might orphan other modules. Keep removing modules until
+    // there are no further modules that can be removed.
+    int remove_count = 0;
+    int previous_remove_count;
+    do {
+      previous_remove_count = remove_count;
+      for (auto &[name, vec] : m_name_to_modules) {
+        if (vec.empty())
+          continue;
+        ModuleList to_remove = RemoveOrphansFromVector(vec);
+        remove_count += to_remove.GetSize();
+        m_list.Remove(to_remove);
+      }
+      // Break when fixed-point is reached.
+    } while (previous_remove_count != remove_count);
+
+    return remove_count;
+  }
+
+  ModuleList m_list;
+
+  /// A hash map from a module's filename to all the modules that share that
+  /// filename, for fast module lookups by name.
+  llvm::DenseMap<ConstString, llvm::SmallVector<ModuleSP, 1>> m_name_to_modules;
+
+  /// The use count of a module held only by m_list and m_name_to_modules.
+  static constexpr long kUseCountOrphaned = 2;
+};
+
 struct SharedModuleListInfo {
-  ModuleList module_list;
+  SharedModuleList module_list;
   ModuleListProperties module_list_properties;
 };
 }
@@ -774,7 +1000,7 @@ static SharedModuleListInfo &GetSharedModuleListInfo()
   return *g_shared_module_list_info;
 }
 
-static ModuleList &GetSharedModuleList() {
+static SharedModuleList &GetSharedModuleList() {
   return GetSharedModuleListInfo().module_list;
 }
 
@@ -784,8 +1010,8 @@ ModuleListProperties &ModuleList::GetGlobalModuleListProperties() {
 
 bool ModuleList::ModuleIsInCache(const Module *module_ptr) {
   if (module_ptr) {
-    ModuleList &shared_module_list = GetSharedModuleList();
-    return shared_module_list.FindModule(module_ptr).get() != nullptr;
+    SharedModuleList &shared_module_list = GetSharedModuleList();
+    return shared_module_list.FindModule(*module_ptr).get() != nullptr;
   }
   return false;
 }
@@ -808,9 +1034,8 @@ ModuleList::GetSharedModule(const ModuleSpec &module_spec, ModuleSP &module_sp,
                             const FileSpecList *module_search_paths_ptr,
                             llvm::SmallVectorImpl<lldb::ModuleSP> *old_modules,
                             bool *did_create_ptr, bool always_create) {
-  ModuleList &shared_module_list = GetSharedModuleList();
-  std::lock_guard<std::recursive_mutex> guard(
-      shared_module_list.m_modules_mutex);
+  SharedModuleList &shared_module_list = GetSharedModuleList();
+  std::lock_guard<std::recursive_mutex> guard(shared_module_list.GetMutex());
   char path[PATH_MAX];
 
   Status error;
diff --git a/lldb/source/Core/Section.cpp b/lldb/source/Core/Section.cpp
index 27dcf987b0278..02d9d86fe5374 100644
--- a/lldb/source/Core/Section.cpp
+++ b/lldb/source/Core/Section.cpp
@@ -153,6 +153,8 @@ const char *Section::GetTypeAsCString() const {
     return "lldb-formatters";
   case eSectionTypeSwiftModules:
     return "swift-modules";
+  case eSectionTypeWasmName:
+    return "wasm-name";
   case eSectionTypeOther:
     return "regular";
   }
@@ -415,6 +417,7 @@ bool Section::ContainsOnlyDebugInfo() const {
   case eSectionTypeCompactUnwind:
   case eSectionTypeGoSymtab:
   case eSectionTypeAbsoluteAddress:
+  case eSectionTypeWasmName:
   case eSectionTypeOther:
   // Used for "__dof_cache" in mach-o or ".debug" for COFF which isn't debug
   // information that we parse at all. This was causing system files with no
diff --git a/lldb/source/Core/Value.cpp b/lldb/source/Core/Value.cpp
index 028f0587c5790..f9e65d397f05f 100644
--- a/lldb/source/Core/Value.cpp
+++ b/lldb/source/Core/Value.cpp
@@ -347,6 +347,9 @@ Status Value::GetValueAsData(ExecutionContext *exe_ctx, DataExtractor &data,
     else
       data.SetAddressByteSize(sizeof(void *));
 
+    if (!type_size)
+      return Status::FromErrorString("type does not have a size");
+
     uint32_t result_byte_size = *type_size;
     if (m_value.GetData(data, result_byte_size))
       return error; // Success;
@@ -488,9 +491,11 @@ Status Value::GetValueAsData(ExecutionContext *exe_ctx, DataExtractor &data,
     address = m_value.ULongLong(LLDB_INVALID_ADDRESS);
     address_type = eAddressTypeHost;
     if (exe_ctx) {
-      Target *target = exe_ctx->GetTargetPtr();
-      if (target) {
-        data.SetByteOrder(target->GetArchitecture().GetByteOrder());
+      if (Target *target = exe_ctx->GetTargetPtr()) {
+        // Registers are always stored in host endian.
+        data.SetByteOrder(m_context_type == ContextType::RegisterInfo
+                              ? endian::InlHostByteOrder()
+                              : target->GetArchitecture().GetByteOrder());
         data.SetAddressByteSize(target->GetArchitecture().GetAddressByteSize());
         break;
       }
diff --git a/lldb/source/DataFormatters/DumpValueObjectOptions.cpp b/lldb/source/DataFormatters/DumpValueObjectOptions.cpp
index c343e6083df19..e1df9522256fa 100644
--- a/lldb/source/DataFormatters/DumpValueObjectOptions.cpp
+++ b/lldb/source/DataFormatters/DumpValueObjectOptions.cpp
@@ -17,7 +17,7 @@ DumpValueObjectOptions::DumpValueObjectOptions()
     : m_summary_sp(), m_root_valobj_name(), m_decl_printing_helper(),
       m_child_printing_decider(), m_pointer_as_array(), m_use_synthetic(true),
       m_scope_already_checked(false), m_flat_output(false), m_ignore_cap(false),
-      m_show_types(false), m_show_location(false), m_use_objc(false),
+      m_show_types(false), m_show_location(false), m_use_object_desc(false),
       m_hide_root_type(false), m_hide_root_name(false), m_hide_name(false),
       m_hide_value(false), m_run_validator(false),
       m_use_type_display_name(true), m_allow_oneliner_mode(true),
@@ -65,8 +65,19 @@ DumpValueObjectOptions &DumpValueObjectOptions::SetShowLocation(bool show) {
   return *this;
 }
 
-DumpValueObjectOptions &DumpValueObjectOptions::SetUseObjectiveC(bool use) {
-  m_use_objc = use;
+DumpValueObjectOptions &DumpValueObjectOptions::DisableObjectDescription() {
+  // Reset these options to their default values.
+  SetUseObjectDescription(false);
+  SetHideRootType(false);
+  SetHideName(false);
+  SetHideValue(false);
+  SetShowSummary(true);
+  return *this;
+}
+
+DumpValueObjectOptions &
+DumpValueObjectOptions::SetUseObjectDescription(bool use) {
+  m_use_object_desc = use;
   return *this;
 }
 
diff --git a/lldb/source/DataFormatters/ValueObjectPrinter.cpp b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
index c2f8bb3ea05bc..e20051140806b 100644
--- a/lldb/source/DataFormatters/ValueObjectPrinter.cpp
+++ b/lldb/source/DataFormatters/ValueObjectPrinter.cpp
@@ -14,9 +14,11 @@
 #include "lldb/Target/Target.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/ValueObject/ValueObject.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/MathExtras.h"
 #include <cstdint>
 #include <memory>
+#include <optional>
 
 using namespace lldb;
 using namespace lldb_private;
@@ -69,6 +71,18 @@ void ValueObjectPrinter::Init(
   SetupMostSpecializedValue();
 }
 
+static const char *maybeNewline(const std::string &s) {
+  // If the string already ends with a \n don't add another one.
+  if (s.empty() || s.back() != '\n')
+    return "\n";
+  return "";
+}
+
+bool ValueObjectPrinter::ShouldPrintObjectDescription() {
+  return ShouldPrintValueObject() && m_options.m_use_object_desc && !IsNil() &&
+         !IsUninitialized() && !m_options.m_pointer_as_array;
+}
+
 llvm::Error ValueObjectPrinter::PrintValueObject() {
   // If the incoming ValueObject is in an error state, the best we're going to 
   // get out of it is its type.  But if we don't even have that, just print
@@ -77,6 +91,25 @@ llvm::Error ValueObjectPrinter::PrintValueObject() {
       !m_orig_valobj.GetCompilerType().IsValid())
     return m_orig_valobj.GetError().ToError();
 
+  std::optional<std::string> object_desc;
+  if (ShouldPrintObjectDescription()) {
+    // The object description is invoked now, but not printed until after
+    // value/summary. Calling GetObjectDescription at the outset of printing
+    // allows for early discovery of errors. In the case of an error, the value
+    // object is printed normally.
+    llvm::Expected<std::string> object_desc_or_err =
+        GetMostSpecializedValue().GetObjectDescription();
+    if (!object_desc_or_err) {
+      auto error_msg = toString(object_desc_or_err.takeError());
+      *m_stream << "error: " << error_msg << maybeNewline(error_msg);
+
+      // Print the value object directly.
+      m_options.DisableObjectDescription();
+    } else {
+      object_desc = *object_desc_or_err;
+    }
+  }
+
   if (ShouldPrintValueObject()) {
     PrintLocationIfNeeded();
     m_stream->Indent();
@@ -90,8 +123,10 @@ llvm::Error ValueObjectPrinter::PrintValueObject() {
   m_val_summary_ok =
       PrintValueAndSummaryIfNeeded(value_printed, summary_printed);
 
-  if (m_val_summary_ok)
+  if (m_val_summary_ok) {
+    PrintObjectDescriptionIfNeeded(object_desc);
     return PrintChildrenIfNeeded(value_printed, summary_printed);
+  }
   m_stream->EOL();
 
   return llvm::Error::success();
@@ -144,24 +179,6 @@ void ValueObjectPrinter::SetupMostSpecializedValue() {
          "SetupMostSpecialized value must compute a valid ValueObject");
 }
 
-llvm::Expected<std::string> ValueObjectPrinter::GetDescriptionForDisplay() {
-  ValueObject &valobj = GetMostSpecializedValue();
-  llvm::Expected<std::string> maybe_str = valobj.GetObjectDescription();
-  if (maybe_str)
-    return maybe_str;
-
-  const char *str = nullptr;
-  if (!str)
-    str = valobj.GetSummaryAsCString();
-  if (!str)
-    str = valobj.GetValueAsCString();
-
-  if (!str)
-    return maybe_str;
-  llvm::consumeError(maybe_str.takeError());
-  return str;
-}
-
 const char *ValueObjectPrinter::GetRootNameForDisplay() {
   const char *root_valobj_name =
       m_options.m_root_valobj_name.empty()
@@ -468,38 +485,14 @@ bool ValueObjectPrinter::PrintValueAndSummaryIfNeeded(bool &value_printed,
   return !error_printed;
 }
 
-llvm::Error
-ValueObjectPrinter::PrintObjectDescriptionIfNeeded(bool value_printed,
-                                                   bool summary_printed) {
-  if (ShouldPrintValueObject()) {
-    // let's avoid the overly verbose no description error for a nil thing
-    if (m_options.m_use_objc && !IsNil() && !IsUninitialized() &&
-        (!m_options.m_pointer_as_array)) {
-      if (!m_options.m_hide_value || ShouldShowName())
-        *m_stream << ' ';
-      llvm::Expected<std::string> object_desc =
-          (value_printed || summary_printed)
-              ? GetMostSpecializedValue().GetObjectDescription()
-              : GetDescriptionForDisplay();
-      if (!object_desc) {
-        // If no value or summary was printed, surface the error.
-        if (!value_printed && !summary_printed)
-          return object_desc.takeError();
-        // Otherwise gently nudge the user that they should have used
-        // `p` instead of `po`. Unfortunately we cannot be more direct
-        // about this, because we don't actually know what the user did.
-        *m_stream << "warning: no object description available\n";
-        llvm::consumeError(object_desc.takeError());
-      } else {
-        *m_stream << *object_desc;
-        // If the description already ends with a \n don't add another one.
-        if (object_desc->empty() || object_desc->back() != '\n')
-          *m_stream << '\n';
-      }
-      return llvm::Error::success();
-    }
-  }
-  return llvm::Error::success();
+void ValueObjectPrinter::PrintObjectDescriptionIfNeeded(
+    std::optional<std::string> object_desc) {
+  if (!object_desc)
+    return;
+
+  if (!m_options.m_hide_value || ShouldShowName())
+    *m_stream << ' ';
+  *m_stream << *object_desc << maybeNewline(*object_desc);
 }
 
 bool DumpValueObjectOptions::PointerDepth::CanAllowExpansion() const {
@@ -524,7 +517,7 @@ bool ValueObjectPrinter::ShouldPrintChildren(
   if (m_options.m_pointer_as_array)
     return true;
 
-  if (m_options.m_use_objc)
+  if (m_options.m_use_object_desc)
     return false;
 
   bool print_children = true;
@@ -819,9 +812,6 @@ bool ValueObjectPrinter::PrintChildrenOneLiner(bool hide_names) {
 
 llvm::Error ValueObjectPrinter::PrintChildrenIfNeeded(bool value_printed,
                                                       bool summary_printed) {
-  auto error = PrintObjectDescriptionIfNeeded(value_printed, summary_printed);
-  if (error)
-    return error;
 
   ValueObject &valobj = GetMostSpecializedValue();
 
diff --git a/lldb/source/Expression/IRExecutionUnit.cpp b/lldb/source/Expression/IRExecutionUnit.cpp
index e7a26d3c2dcf4..d557084acb745 100644
--- a/lldb/source/Expression/IRExecutionUnit.cpp
+++ b/lldb/source/Expression/IRExecutionUnit.cpp
@@ -799,7 +799,7 @@ ResolveFunctionCallLabel(const FunctionCallLabel &label,
   auto sc_or_err = symbol_file->ResolveFunctionCallLabel(label);
   if (!sc_or_err)
     return llvm::joinErrors(
-        llvm::createStringError("failed to resolve function by UID"),
+        llvm::createStringError("failed to resolve function by UID:"),
         sc_or_err.takeError());
 
   SymbolContextList sc_list;
diff --git a/lldb/source/Expression/Materializer.cpp b/lldb/source/Expression/Materializer.cpp
index 329768dd7915a..771a9ab84a20c 100644
--- a/lldb/source/Expression/Materializer.cpp
+++ b/lldb/source/Expression/Materializer.cpp
@@ -1377,29 +1377,26 @@ class EntityRegister : public Materializer::Entity {
       return;
     }
 
-    DataExtractor register_data;
-
-    if (!reg_value.GetData(register_data)) {
-      err = Status::FromErrorStringWithFormat(
-          "couldn't get the data for register %s", m_register_info.name);
-      return;
-    }
-
-    if (register_data.GetByteSize() != m_register_info.byte_size) {
+    if (reg_value.GetByteSize() != m_register_info.byte_size) {
       err = Status::FromErrorStringWithFormat(
           "data for register %s had size %llu but we expected %llu",
-          m_register_info.name, (unsigned long long)register_data.GetByteSize(),
+          m_register_info.name, (unsigned long long)reg_value.GetByteSize(),
           (unsigned long long)m_register_info.byte_size);
       return;
     }
 
-    m_register_contents = std::make_shared<DataBufferHeap>(
-        register_data.GetDataStart(), register_data.GetByteSize());
+    lldb_private::DataBufferHeap buf(reg_value.GetByteSize(), 0);
+    reg_value.GetAsMemoryData(m_register_info, buf.GetBytes(),
+                              buf.GetByteSize(), map.GetByteOrder(), err);
+    if (!err.Success())
+      return;
+
+    m_register_contents = std::make_shared<DataBufferHeap>(buf);
 
     Status write_error;
 
-    map.WriteMemory(load_addr, register_data.GetDataStart(),
-                    register_data.GetByteSize(), write_error);
+    map.WriteMemory(load_addr, buf.GetBytes(), reg_value.GetByteSize(),
+                    write_error);
 
     if (!write_error.Success()) {
       err = Status::FromErrorStringWithFormat(
diff --git a/lldb/source/Expression/REPL.cpp b/lldb/source/Expression/REPL.cpp
index e5377d3114af3..92017d271a2df 100644
--- a/lldb/source/Expression/REPL.cpp
+++ b/lldb/source/Expression/REPL.cpp
@@ -321,7 +321,7 @@ void REPL::IOHandlerInputComplete(IOHandler &io_handler, std::string &code) {
       const bool colorize_err = error_sp->GetFile().GetIsTerminalWithColors();
 
       EvaluateExpressionOptions expr_options = m_expr_options;
-      expr_options.SetCoerceToId(m_varobj_options.use_objc);
+      expr_options.SetCoerceToId(m_varobj_options.use_object_desc);
       expr_options.SetKeepInMemory(true);
       expr_options.SetUseDynamic(m_varobj_options.use_dynamic);
       expr_options.SetGenerateDebugInfo(true);
diff --git a/lldb/source/Host/CMakeLists.txt b/lldb/source/Host/CMakeLists.txt
index b15d72e61b6e5..c9e8afe48fcde 100644
--- a/lldb/source/Host/CMakeLists.txt
+++ b/lldb/source/Host/CMakeLists.txt
@@ -7,7 +7,7 @@ if (APPLE AND LLVM_ENABLE_LOCAL_SUBMODULE_VISIBILITY)
   endif()
 endif()
 
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   add_definitions("-D_ALL_SOURCE")
 endif()
 
diff --git a/lldb/source/Host/common/JSONTransport.cpp b/lldb/source/Host/common/JSONTransport.cpp
index 546c12c8f7114..c4b42eafc85d3 100644
--- a/lldb/source/Host/common/JSONTransport.cpp
+++ b/lldb/source/Host/common/JSONTransport.cpp
@@ -7,171 +7,26 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Host/JSONTransport.h"
-#include "lldb/Utility/IOObject.h"
-#include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
-#include "lldb/Utility/SelectHelper.h"
 #include "lldb/Utility/Status.h"
-#include "lldb/lldb-forward.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
-#include <optional>
 #include <string>
-#include <utility>
 
 using namespace llvm;
 using namespace lldb;
 using namespace lldb_private;
 
-/// ReadFull attempts to read the specified number of bytes. If EOF is
-/// encountered, an empty string is returned.
-static Expected<std::string>
-ReadFull(IOObject &descriptor, size_t length,
-         std::optional<std::chrono::microseconds> timeout = std::nullopt) {
-  if (!descriptor.IsValid())
-    return llvm::make_error<TransportInvalidError>();
+char TransportUnhandledContentsError::ID;
 
-  bool timeout_supported = true;
-  // FIXME: SelectHelper does not work with NativeFile on Win32.
-#if _WIN32
-  timeout_supported = descriptor.GetFdType() == IOObject::eFDTypeSocket;
-#endif
+TransportUnhandledContentsError::TransportUnhandledContentsError(
+    std::string unhandled_contents)
+    : m_unhandled_contents(unhandled_contents) {}
 
-  if (timeout && timeout_supported) {
-    SelectHelper sh;
-    sh.SetTimeout(*timeout);
-    sh.FDSetRead(
-        reinterpret_cast<lldb::socket_t>(descriptor.GetWaitableHandle()));
-    Status status = sh.Select();
-    if (status.Fail()) {
-      // Convert timeouts into a specific error.
-      if (status.GetType() == lldb::eErrorTypePOSIX &&
-          status.GetError() == ETIMEDOUT)
-        return make_error<TransportTimeoutError>();
-      return status.takeError();
-    }
-  }
-
-  std::string data;
-  data.resize(length);
-  Status status = descriptor.Read(data.data(), length);
-  if (status.Fail())
-    return status.takeError();
-
-  // Read returns '' on EOF.
-  if (length == 0)
-    return make_error<TransportEOFError>();
-
-  // Return the actual number of bytes read.
-  return data.substr(0, length);
-}
-
-static Expected<std::string>
-ReadUntil(IOObject &descriptor, StringRef delimiter,
-          std::optional<std::chrono::microseconds> timeout = std::nullopt) {
-  std::string buffer;
-  buffer.reserve(delimiter.size() + 1);
-  while (!llvm::StringRef(buffer).ends_with(delimiter)) {
-    Expected<std::string> next =
-        ReadFull(descriptor, buffer.empty() ? delimiter.size() : 1, timeout);
-    if (auto Err = next.takeError())
-      return std::move(Err);
-    buffer += *next;
-  }
-  return buffer.substr(0, buffer.size() - delimiter.size());
-}
-
-JSONTransport::JSONTransport(IOObjectSP input, IOObjectSP output)
-    : m_input(std::move(input)), m_output(std::move(output)) {}
-
-void JSONTransport::Log(llvm::StringRef message) {
-  LLDB_LOG(GetLog(LLDBLog::Host), "{0}", message);
+void TransportUnhandledContentsError::log(llvm::raw_ostream &OS) const {
+  OS << "transport EOF with unhandled contents: '" << m_unhandled_contents
+     << "'";
 }
-
-Expected<std::string>
-HTTPDelimitedJSONTransport::ReadImpl(const std::chrono::microseconds &timeout) {
-  if (!m_input || !m_input->IsValid())
-    return llvm::make_error<TransportInvalidError>();
-
-  IOObject *input = m_input.get();
-  Expected<std::string> message_header =
-      ReadFull(*input, kHeaderContentLength.size(), timeout);
-  if (!message_header)
-    return message_header.takeError();
-  if (*message_header != kHeaderContentLength)
-    return createStringError(formatv("expected '{0}' and got '{1}'",
-                                     kHeaderContentLength, *message_header)
-                                 .str());
-
-  Expected<std::string> raw_length = ReadUntil(*input, kHeaderSeparator);
-  if (!raw_length)
-    return handleErrors(raw_length.takeError(),
-                        [&](const TransportEOFError &E) -> llvm::Error {
-                          return createStringError(
-                              "unexpected EOF while reading header separator");
-                        });
-
-  size_t length;
-  if (!to_integer(*raw_length, length))
-    return createStringError(
-        formatv("invalid content length {0}", *raw_length).str());
-
-  Expected<std::string> raw_json = ReadFull(*input, length);
-  if (!raw_json)
-    return handleErrors(
-        raw_json.takeError(), [&](const TransportEOFError &E) -> llvm::Error {
-          return createStringError("unexpected EOF while reading JSON");
-        });
-
-  Log(llvm::formatv("--> {0}", *raw_json).str());
-
-  return raw_json;
+std::error_code TransportUnhandledContentsError::convertToErrorCode() const {
+  return std::make_error_code(std::errc::bad_message);
 }
-
-Error HTTPDelimitedJSONTransport::WriteImpl(const std::string &message) {
-  if (!m_output || !m_output->IsValid())
-    return llvm::make_error<TransportInvalidError>();
-
-  Log(llvm::formatv("<-- {0}", message).str());
-
-  std::string Output;
-  raw_string_ostream OS(Output);
-  OS << kHeaderContentLength << message.length() << kHeaderSeparator << message;
-  size_t num_bytes = Output.size();
-  return m_output->Write(Output.data(), num_bytes).takeError();
-}
-
-Expected<std::string>
-JSONRPCTransport::ReadImpl(const std::chrono::microseconds &timeout) {
-  if (!m_input || !m_input->IsValid())
-    return make_error<TransportInvalidError>();
-
-  IOObject *input = m_input.get();
-  Expected<std::string> raw_json =
-      ReadUntil(*input, kMessageSeparator, timeout);
-  if (!raw_json)
-    return raw_json.takeError();
-
-  Log(llvm::formatv("--> {0}", *raw_json).str());
-
-  return *raw_json;
-}
-
-Error JSONRPCTransport::WriteImpl(const std::string &message) {
-  if (!m_output || !m_output->IsValid())
-    return llvm::make_error<TransportInvalidError>();
-
-  Log(llvm::formatv("<-- {0}", message).str());
-
-  std::string Output;
-  llvm::raw_string_ostream OS(Output);
-  OS << message << kMessageSeparator;
-  size_t num_bytes = Output.size();
-  return m_output->Write(Output.data(), num_bytes).takeError();
-}
-
-char TransportEOFError::ID;
-char TransportTimeoutError::ID;
-char TransportInvalidError::ID;
diff --git a/lldb/source/Host/windows/Host.cpp b/lldb/source/Host/windows/Host.cpp
index 4e747f77afc0e..4277b8edb38e5 100644
--- a/lldb/source/Host/windows/Host.cpp
+++ b/lldb/source/Host/windows/Host.cpp
@@ -22,6 +22,7 @@
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StructuredData.h"
 
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/ManagedStatic.h"
@@ -32,6 +33,8 @@
 using namespace lldb;
 using namespace lldb_private;
 
+using llvm::sys::windows::UTF8ToUTF16;
+
 static bool GetTripleForProcess(const FileSpec &executable,
                                 llvm::Triple &triple) {
   // Open the PE File as a binary file, and parse just enough information to
@@ -325,31 +328,18 @@ class WindowsEventLog {
 
 static llvm::ManagedStatic<WindowsEventLog> event_log;
 
-static std::wstring AnsiToUtf16(const std::string &ansi) {
-  if (ansi.empty())
-    return {};
-
-  const int unicode_length =
-      MultiByteToWideChar(CP_ACP, 0, ansi.c_str(), -1, nullptr, 0);
-  if (unicode_length == 0)
-    return {};
-
-  std::wstring unicode(unicode_length, L'\0');
-  MultiByteToWideChar(CP_ACP, 0, ansi.c_str(), -1, &unicode[0], unicode_length);
-  return unicode;
-}
-
 void Host::SystemLog(Severity severity, llvm::StringRef message) {
+  if (message.empty())
+    return;
+
   HANDLE h = event_log->GetHandle();
   if (!h)
     return;
 
-  std::wstring wide_message = AnsiToUtf16(message.str());
-  if (wide_message.empty())
+  llvm::SmallVector<wchar_t, 1> argsUTF16;
+  if (UTF8ToUTF16(message.str(), argsUTF16))
     return;
 
-  LPCWSTR msg_ptr = wide_message.c_str();
-
   WORD event_type;
   switch (severity) {
   case lldb::eSeverityWarning:
@@ -363,5 +353,7 @@ void Host::SystemLog(Severity severity, llvm::StringRef message) {
     event_type = EVENTLOG_INFORMATION_TYPE;
   }
 
-  ReportEventW(h, event_type, 0, 0, nullptr, 1, 0, &msg_ptr, nullptr);
+  LPCWSTR messages[1] = {argsUTF16.data()};
+  ReportEventW(h, event_type, 0, 0, nullptr, std::size(messages), 0, messages,
+               nullptr);
 }
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index a0080cfff57c1..650b754fd8ace 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -335,7 +335,7 @@ void CommandInterpreter::Initialize() {
     AddAlias("ni", cmd_obj_sp);
   }
 
-  cmd_obj_sp = GetCommandSPExact("thread step-in");
+  cmd_obj_sp = GetCommandSPExact("_regexp-step");
   if (cmd_obj_sp) {
     AddAlias("s", cmd_obj_sp);
     AddAlias("step", cmd_obj_sp);
@@ -946,6 +946,27 @@ void CommandInterpreter::LoadCommandDictionary() {
           jump_regex_cmd_sp;
     }
   }
+
+  std::shared_ptr<CommandObjectRegexCommand> step_regex_cmd_sp(
+      new CommandObjectRegexCommand(
+          *this, "_regexp-step",
+          "Single step, optionally to a specific function.",
+          "\n"
+          "_regexp-step                 // Single step\n"
+          "_regexp-step <function-name> // Step into the named function\n",
+          0, false));
+  if (step_regex_cmd_sp) {
+    if (step_regex_cmd_sp->AddRegexCommand("^[[:space:]]*$",
+                                           "thread step-in") &&
+        step_regex_cmd_sp->AddRegexCommand("^[[:space:]]*(-.+)$",
+                                           "thread step-in %1") &&
+        step_regex_cmd_sp->AddRegexCommand(
+            "^[[:space:]]*(.+)[[:space:]]*$",
+            "thread step-in --end-linenumber block --step-in-target %1")) {
+      m_command_dict[std::string(step_regex_cmd_sp->GetCommandName())] =
+          step_regex_cmd_sp;
+    }
+  }
 }
 
 int CommandInterpreter::GetCommandNamesMatchingPartialString(
diff --git a/lldb/source/Interpreter/OptionGroupValueObjectDisplay.cpp b/lldb/source/Interpreter/OptionGroupValueObjectDisplay.cpp
index d633c469e603e..1939d79fec6a9 100644
--- a/lldb/source/Interpreter/OptionGroupValueObjectDisplay.cpp
+++ b/lldb/source/Interpreter/OptionGroupValueObjectDisplay.cpp
@@ -90,7 +90,7 @@ Status OptionGroupValueObjectDisplay::SetOptionValue(
     flat_output = true;
     break;
   case 'O':
-    use_objc = true;
+    use_object_desc = true;
     break;
   case 'R':
     be_raw = true;
@@ -163,7 +163,7 @@ void OptionGroupValueObjectDisplay::OptionParsingStarting(
   no_summary_depth = 0;
   show_location = false;
   flat_output = false;
-  use_objc = false;
+  use_object_desc = false;
   max_depth = UINT32_MAX;
   max_depth_is_default = true;
   ptr_depth = 0;
@@ -191,14 +191,14 @@ DumpValueObjectOptions OptionGroupValueObjectDisplay::GetAsDumpOptions(
     lldb::Format format, lldb::TypeSummaryImplSP summary_sp) {
   DumpValueObjectOptions options;
   options.SetMaximumPointerDepth(ptr_depth);
-  if (use_objc)
+  if (use_object_desc)
     options.SetShowSummary(false);
   else
     options.SetOmitSummaryDepth(no_summary_depth);
   options.SetMaximumDepth(max_depth, max_depth_is_default)
       .SetShowTypes(show_types)
       .SetShowLocation(show_location)
-      .SetUseObjectiveC(use_objc)
+      .SetUseObjectDescription(use_object_desc)
       .SetUseDynamicType(use_dynamic)
       .SetUseSyntheticValue(use_synth)
       .SetFlatOutput(flat_output)
@@ -208,8 +208,9 @@ DumpValueObjectOptions OptionGroupValueObjectDisplay::GetAsDumpOptions(
 
   if (lang_descr_verbosity ==
       eLanguageRuntimeDescriptionDisplayVerbosityCompact)
-    options.SetHideRootType(use_objc).SetHideName(use_objc).SetHideValue(
-        use_objc);
+    options.SetHideRootType(use_object_desc)
+        .SetHideName(use_object_desc)
+        .SetHideValue(use_object_desc);
 
   if (be_raw)
     options.SetRawDisplay();
diff --git a/lldb/source/Interpreter/OptionValueArch.cpp b/lldb/source/Interpreter/OptionValueArch.cpp
index ea15ccaaefe2b..a960e39d35a62 100644
--- a/lldb/source/Interpreter/OptionValueArch.cpp
+++ b/lldb/source/Interpreter/OptionValueArch.cpp
@@ -11,6 +11,7 @@
 #include "lldb/DataFormatters/FormatManager.h"
 #include "lldb/Interpreter/CommandCompletions.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/State.h"
 
@@ -30,6 +31,12 @@ void OptionValueArch::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
       if (arch_name)
         strm.PutCString(arch_name);
     }
+
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value && m_default_value.IsValid()) {
+      DefaultValueFormat label(strm);
+      strm.PutCString(m_default_value.GetArchitectureName());
+    }
   }
 }
 
diff --git a/lldb/source/Interpreter/OptionValueArray.cpp b/lldb/source/Interpreter/OptionValueArray.cpp
index f6c14dee525e9..5600333c111a7 100644
--- a/lldb/source/Interpreter/OptionValueArray.cpp
+++ b/lldb/source/Interpreter/OptionValueArray.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Interpreter/OptionValueArray.h"
 
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/Stream.h"
 
@@ -27,8 +28,15 @@ void OptionValueArray::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
   if (dump_mask & eDumpOptionValue) {
     const bool one_line = dump_mask & eDumpOptionCommand;
     const uint32_t size = m_values.size();
-    if (dump_mask & eDumpOptionType)
-      strm.Printf(" =%s", (m_values.size() > 0 && !one_line) ? "\n" : "");
+    if (dump_mask & (eDumpOptionType | eDumpOptionDefaultValue)) {
+      strm.PutCString(" =");
+      if (dump_mask & eDumpOptionDefaultValue && !m_values.empty()) {
+        DefaultValueFormat label(strm);
+        strm.PutCString("empty");
+      }
+      if (!m_values.empty() && !one_line)
+        strm.PutCString("\n");
+    }
     if (!one_line)
       strm.IndentMore();
     for (uint32_t i = 0; i < size; ++i) {
diff --git a/lldb/source/Interpreter/OptionValueBoolean.cpp b/lldb/source/Interpreter/OptionValueBoolean.cpp
index d4fda762fea6b..023c243b3efc1 100644
--- a/lldb/source/Interpreter/OptionValueBoolean.cpp
+++ b/lldb/source/Interpreter/OptionValueBoolean.cpp
@@ -10,6 +10,7 @@
 
 #include "lldb/Host/PosixApi.h"
 #include "lldb/Interpreter/OptionArgParser.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StringList.h"
 #include "llvm/ADT/STLExtras.h"
@@ -27,6 +28,11 @@ void OptionValueBoolean::DumpValue(const ExecutionContext *exe_ctx,
     if (dump_mask & eDumpOptionType)
       strm.PutCString(" = ");
     strm.PutCString(m_current_value ? "true" : "false");
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value) {
+      DefaultValueFormat label(strm);
+      strm.PutCString(m_default_value ? "true" : "false");
+    }
   }
 }
 
diff --git a/lldb/source/Interpreter/OptionValueChar.cpp b/lldb/source/Interpreter/OptionValueChar.cpp
index 2aadcff158388..595dcba49b61a 100644
--- a/lldb/source/Interpreter/OptionValueChar.cpp
+++ b/lldb/source/Interpreter/OptionValueChar.cpp
@@ -9,6 +9,7 @@
 #include "lldb/Interpreter/OptionValueChar.h"
 
 #include "lldb/Interpreter/OptionArgParser.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StringList.h"
 #include "llvm/ADT/STLExtras.h"
@@ -16,6 +17,13 @@
 using namespace lldb;
 using namespace lldb_private;
 
+static void DumpChar(Stream &strm, char value) {
+  if (value != '\0')
+    strm.PutChar(value);
+  else
+    strm.PutCString("(null)");
+}
+
 void OptionValueChar::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
                                 uint32_t dump_mask) {
   if (dump_mask & eDumpOptionType)
@@ -24,10 +32,12 @@ void OptionValueChar::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
   if (dump_mask & eDumpOptionValue) {
     if (dump_mask & eDumpOptionType)
       strm.PutCString(" = ");
-    if (m_current_value != '\0')
-      strm.PutChar(m_current_value);
-    else
-      strm.PutCString("(null)");
+    DumpChar(strm, m_current_value);
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value) {
+      DefaultValueFormat label(strm);
+      DumpChar(strm, m_default_value);
+    }
   }
 }
 
diff --git a/lldb/source/Interpreter/OptionValueDictionary.cpp b/lldb/source/Interpreter/OptionValueDictionary.cpp
index 19e21dd6f4c9a..0efc76bb7cd94 100644
--- a/lldb/source/Interpreter/OptionValueDictionary.cpp
+++ b/lldb/source/Interpreter/OptionValueDictionary.cpp
@@ -9,6 +9,7 @@
 #include "lldb/Interpreter/OptionValueDictionary.h"
 
 #include "lldb/DataFormatters/FormatManager.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Interpreter/OptionValueEnumeration.h"
 #include "lldb/Interpreter/OptionValueString.h"
 #include "lldb/Utility/Args.h"
@@ -30,8 +31,13 @@ void OptionValueDictionary::DumpValue(const ExecutionContext *exe_ctx,
   }
   if (dump_mask & eDumpOptionValue) {
     const bool one_line = dump_mask & eDumpOptionCommand;
-    if (dump_mask & eDumpOptionType)
+    if (dump_mask & (eDumpOptionType | eDumpOptionDefaultValue)) {
       strm.PutCString(" =");
+      if (dump_mask & eDumpOptionDefaultValue && !m_values.empty()) {
+        DefaultValueFormat label(strm);
+        strm.PutCString("empty");
+      }
+    }
 
     if (!one_line)
       strm.IndentMore();
diff --git a/lldb/source/Interpreter/OptionValueEnumeration.cpp b/lldb/source/Interpreter/OptionValueEnumeration.cpp
index cf646233c80da..eb31bde498b7c 100644
--- a/lldb/source/Interpreter/OptionValueEnumeration.cpp
+++ b/lldb/source/Interpreter/OptionValueEnumeration.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Interpreter/OptionValueEnumeration.h"
 
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/StringList.h"
 
 using namespace lldb;
@@ -19,6 +20,17 @@ OptionValueEnumeration::OptionValueEnumeration(
   SetEnumerations(enumerators);
 }
 
+void OptionValueEnumeration::DumpEnum(Stream &strm, enum_type value) {
+  const size_t count = m_enumerations.GetSize();
+  for (size_t i = 0; i < count; ++i)
+    if (m_enumerations.GetValueAtIndexUnchecked(i).value == value) {
+      strm.PutCString(m_enumerations.GetCStringAtIndex(i));
+      return;
+    }
+
+  strm.Printf("%" PRIu64, (uint64_t)value);
+}
+
 void OptionValueEnumeration::DumpValue(const ExecutionContext *exe_ctx,
                                        Stream &strm, uint32_t dump_mask) {
   if (dump_mask & eDumpOptionType)
@@ -26,14 +38,12 @@ void OptionValueEnumeration::DumpValue(const ExecutionContext *exe_ctx,
   if (dump_mask & eDumpOptionValue) {
     if (dump_mask & eDumpOptionType)
       strm.PutCString(" = ");
-    const size_t count = m_enumerations.GetSize();
-    for (size_t i = 0; i < count; ++i) {
-      if (m_enumerations.GetValueAtIndexUnchecked(i).value == m_current_value) {
-        strm.PutCString(m_enumerations.GetCStringAtIndex(i).GetStringRef());
-        return;
-      }
+    DumpEnum(strm, m_current_value);
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value) {
+      DefaultValueFormat label(strm);
+      DumpEnum(strm, m_default_value);
     }
-    strm.Printf("%" PRIu64, (uint64_t)m_current_value);
   }
 }
 
diff --git a/lldb/source/Interpreter/OptionValueFileSpec.cpp b/lldb/source/Interpreter/OptionValueFileSpec.cpp
index 6fa6af4ace02a..8d4966dcda0bc 100644
--- a/lldb/source/Interpreter/OptionValueFileSpec.cpp
+++ b/lldb/source/Interpreter/OptionValueFileSpec.cpp
@@ -12,6 +12,7 @@
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Interpreter/CommandCompletions.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/State.h"
 
@@ -41,7 +42,12 @@ void OptionValueFileSpec::DumpValue(const ExecutionContext *exe_ctx,
       strm.PutCString(" = ");
 
     if (m_current_value) {
-      strm << '"' << m_current_value.GetPath().c_str() << '"';
+      strm << '"' << m_current_value.GetPath() << '"';
+    }
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value && m_default_value) {
+      DefaultValueFormat label(strm);
+      strm << '"' << m_default_value.GetPath() << '"';
     }
   }
 }
diff --git a/lldb/source/Interpreter/OptionValueFileSpecList.cpp b/lldb/source/Interpreter/OptionValueFileSpecList.cpp
index f252dc4603cc1..f331c5d13f461 100644
--- a/lldb/source/Interpreter/OptionValueFileSpecList.cpp
+++ b/lldb/source/Interpreter/OptionValueFileSpecList.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Interpreter/OptionValueFileSpecList.h"
 
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/Stream.h"
 
@@ -22,9 +23,15 @@ void OptionValueFileSpecList::DumpValue(const ExecutionContext *exe_ctx,
   if (dump_mask & eDumpOptionValue) {
     const bool one_line = dump_mask & eDumpOptionCommand;
     const uint32_t size = m_current_value.GetSize();
-    if (dump_mask & eDumpOptionType)
-      strm.Printf(" =%s",
-                  (m_current_value.GetSize() > 0 && !one_line) ? "\n" : "");
+    if (dump_mask & (eDumpOptionType | eDumpOptionDefaultValue)) {
+      strm.Printf(" =");
+      if (dump_mask & eDumpOptionDefaultValue && !m_current_value.IsEmpty()) {
+        DefaultValueFormat label(strm);
+        strm.PutCString("empty");
+      }
+      if (!m_current_value.IsEmpty() && !one_line)
+        strm.PutCString("\n");
+    }
     if (!one_line)
       strm.IndentMore();
     for (uint32_t i = 0; i < size; ++i) {
diff --git a/lldb/source/Interpreter/OptionValueFormat.cpp b/lldb/source/Interpreter/OptionValueFormat.cpp
index bc4e77923d10e..05990fb50c356 100644
--- a/lldb/source/Interpreter/OptionValueFormat.cpp
+++ b/lldb/source/Interpreter/OptionValueFormat.cpp
@@ -10,6 +10,7 @@
 
 #include "lldb/DataFormatters/FormatManager.h"
 #include "lldb/Interpreter/OptionArgParser.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Stream.h"
 
 using namespace lldb;
@@ -23,6 +24,11 @@ void OptionValueFormat::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
     if (dump_mask & eDumpOptionType)
       strm.PutCString(" = ");
     strm.PutCString(FormatManager::GetFormatAsCString(m_current_value));
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value) {
+      DefaultValueFormat label(strm);
+      strm.PutCString(FormatManager::GetFormatAsCString(m_default_value));
+    }
   }
 }
 
diff --git a/lldb/source/Interpreter/OptionValueFormatEntity.cpp b/lldb/source/Interpreter/OptionValueFormatEntity.cpp
index d8b830115005c..b31dd4e475878 100644
--- a/lldb/source/Interpreter/OptionValueFormatEntity.cpp
+++ b/lldb/source/Interpreter/OptionValueFormatEntity.cpp
@@ -10,6 +10,7 @@
 
 #include "lldb/Core/Module.h"
 #include "lldb/Interpreter/CommandInterpreter.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StringList.h"
 using namespace lldb;
@@ -33,10 +34,9 @@ void OptionValueFormatEntity::Clear() {
   m_value_was_set = false;
 }
 
-static void EscapeBackticks(llvm::StringRef str, std::string &dst) {
-  dst.clear();
+static std::string EscapeBackticks(llvm::StringRef str) {
+  std::string dst;
   dst.reserve(str.size());
-
   for (size_t i = 0, e = str.size(); i != e; ++i) {
     char c = str[i];
     if (c == '`') {
@@ -45,6 +45,7 @@ static void EscapeBackticks(llvm::StringRef str, std::string &dst) {
     }
     dst += c;
   }
+  return dst;
 }
 
 void OptionValueFormatEntity::DumpValue(const ExecutionContext *exe_ctx,
@@ -54,17 +55,18 @@ void OptionValueFormatEntity::DumpValue(const ExecutionContext *exe_ctx,
   if (dump_mask & eDumpOptionValue) {
     if (dump_mask & eDumpOptionType)
       strm.PutCString(" = ");
-    std::string escaped;
-    EscapeBackticks(m_current_format, escaped);
-    strm << '"' << escaped << '"';
+    strm << '"' << EscapeBackticks(m_current_format) << '"';
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_format != m_default_format) {
+      DefaultValueFormat label(strm);
+      strm << '"' << EscapeBackticks(m_default_format) << '"';
+    }
   }
 }
 
 llvm::json::Value
 OptionValueFormatEntity::ToJSON(const ExecutionContext *exe_ctx) const {
-  std::string escaped;
-  EscapeBackticks(m_current_format, escaped);
-  return escaped;
+  return EscapeBackticks(m_current_format);
 }
 
 Status OptionValueFormatEntity::SetValueFromString(llvm::StringRef value_str,
diff --git a/lldb/source/Interpreter/OptionValueLanguage.cpp b/lldb/source/Interpreter/OptionValueLanguage.cpp
index 0fdaacb02594e..bb28dc8debe6d 100644
--- a/lldb/source/Interpreter/OptionValueLanguage.cpp
+++ b/lldb/source/Interpreter/OptionValueLanguage.cpp
@@ -9,8 +9,9 @@
 #include "lldb/Interpreter/OptionValueLanguage.h"
 
 #include "lldb/DataFormatters/FormatManager.h"
-#include "lldb/Target/Language.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Symbol/TypeSystem.h"
+#include "lldb/Target/Language.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/Stream.h"
 
@@ -26,6 +27,12 @@ void OptionValueLanguage::DumpValue(const ExecutionContext *exe_ctx,
       strm.PutCString(" = ");
     if (m_current_value != eLanguageTypeUnknown)
       strm.PutCString(Language::GetNameForLanguageType(m_current_value));
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value &&
+        m_default_value != eLanguageTypeUnknown) {
+      DefaultValueFormat label(strm);
+      strm.PutCString(Language::GetNameForLanguageType(m_default_value));
+    }
   }
 }
 
diff --git a/lldb/source/Interpreter/OptionValuePathMappings.cpp b/lldb/source/Interpreter/OptionValuePathMappings.cpp
index 95b8e64171455..abf4d429602e4 100644
--- a/lldb/source/Interpreter/OptionValuePathMappings.cpp
+++ b/lldb/source/Interpreter/OptionValuePathMappings.cpp
@@ -9,6 +9,7 @@
 #include "lldb/Interpreter/OptionValuePathMappings.h"
 
 #include "lldb/Host/FileSystem.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/FileSpec.h"
 #include "lldb/Utility/Stream.h"
@@ -28,8 +29,15 @@ void OptionValuePathMappings::DumpValue(const ExecutionContext *exe_ctx,
   if (dump_mask & eDumpOptionType)
     strm.Printf("(%s)", GetTypeAsCString());
   if (dump_mask & eDumpOptionValue) {
-    if (dump_mask & eDumpOptionType)
-      strm.Printf(" =%s", (m_path_mappings.GetSize() > 0) ? "\n" : "");
+    if (dump_mask & (eDumpOptionType | eDumpOptionDefaultValue)) {
+      strm.Printf(" =");
+      if (dump_mask & eDumpOptionDefaultValue && !m_path_mappings.IsEmpty()) {
+        DefaultValueFormat label(strm);
+        strm.PutCString("empty");
+      }
+      if (!m_path_mappings.IsEmpty())
+        strm.PutCString("\n");
+    }
     m_path_mappings.Dump(&strm);
   }
 }
diff --git a/lldb/source/Interpreter/OptionValueRegex.cpp b/lldb/source/Interpreter/OptionValueRegex.cpp
index 91ec41df6ee50..30e307a7240ef 100644
--- a/lldb/source/Interpreter/OptionValueRegex.cpp
+++ b/lldb/source/Interpreter/OptionValueRegex.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Interpreter/OptionValueRegex.h"
 
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Stream.h"
 
 using namespace lldb;
@@ -24,6 +25,12 @@ void OptionValueRegex::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
       llvm::StringRef regex_text = m_regex.GetText();
       strm.Printf("%s", regex_text.str().c_str());
     }
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_regex.GetText() != m_default_regex_str &&
+        !m_default_regex_str.empty()) {
+      DefaultValueFormat label(strm);
+      strm.PutCString(m_default_regex_str);
+    }
   }
 }
 
diff --git a/lldb/source/Interpreter/OptionValueSInt64.cpp b/lldb/source/Interpreter/OptionValueSInt64.cpp
index df7aee99e212c..00f1cc37dddac 100644
--- a/lldb/source/Interpreter/OptionValueSInt64.cpp
+++ b/lldb/source/Interpreter/OptionValueSInt64.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Interpreter/OptionValueSInt64.h"
 
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Stream.h"
 
 using namespace lldb;
@@ -26,6 +27,11 @@ void OptionValueSInt64::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
     if (dump_mask & eDumpOptionType)
       strm.PutCString(" = ");
     strm.Printf("%" PRIi64, m_current_value);
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value) {
+      DefaultValueFormat label(strm);
+      strm.Printf("%" PRIi64, m_default_value);
+    }
   }
 }
 
diff --git a/lldb/source/Interpreter/OptionValueString.cpp b/lldb/source/Interpreter/OptionValueString.cpp
index ae30661a56d05..022a27d069f48 100644
--- a/lldb/source/Interpreter/OptionValueString.cpp
+++ b/lldb/source/Interpreter/OptionValueString.cpp
@@ -9,12 +9,28 @@
 #include "lldb/Interpreter/OptionValueString.h"
 
 #include "lldb/Host/OptionParser.h"
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/Stream.h"
 
 using namespace lldb;
 using namespace lldb_private;
 
+static void DumpString(Stream &strm, const std::string &str, bool escape,
+                       bool raw) {
+  if (escape) {
+    std::string escaped_str;
+    Args::ExpandEscapedCharacters(str.c_str(), escaped_str);
+    DumpString(strm, escaped_str, false, raw);
+    return;
+  }
+
+  if (raw)
+    strm.PutCString(str);
+  else
+    strm.QuotedCString(str.c_str());
+}
+
 void OptionValueString::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
                                   uint32_t dump_mask) {
   if (dump_mask & eDumpOptionType)
@@ -22,21 +38,15 @@ void OptionValueString::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
   if (dump_mask & eDumpOptionValue) {
     if (dump_mask & eDumpOptionType)
       strm.PutCString(" = ");
-    if (!m_current_value.empty() || m_value_was_set) {
-      if (m_options.Test(eOptionEncodeCharacterEscapeSequences)) {
-        std::string expanded_escape_value;
-        Args::ExpandEscapedCharacters(m_current_value.c_str(),
-                                      expanded_escape_value);
-        if (dump_mask & eDumpOptionRaw)
-          strm.Printf("%s", expanded_escape_value.c_str());
-        else
-          strm.Printf("\"%s\"", expanded_escape_value.c_str());
-      } else {
-        if (dump_mask & eDumpOptionRaw)
-          strm.Printf("%s", m_current_value.c_str());
-        else
-          strm.Printf("\"%s\"", m_current_value.c_str());
-      }
+    const bool escape = m_options.Test(eOptionEncodeCharacterEscapeSequences);
+    const bool raw = dump_mask & eDumpOptionRaw;
+    if (!m_current_value.empty() || m_value_was_set)
+      DumpString(strm, m_current_value, escape, raw);
+
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value && !m_default_value.empty()) {
+      DefaultValueFormat label(strm);
+      DumpString(strm, m_default_value, escape, raw);
     }
   }
 }
diff --git a/lldb/source/Interpreter/OptionValueUInt64.cpp b/lldb/source/Interpreter/OptionValueUInt64.cpp
index aa5e9a21c8c28..63f83cbcd60a3 100644
--- a/lldb/source/Interpreter/OptionValueUInt64.cpp
+++ b/lldb/source/Interpreter/OptionValueUInt64.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Interpreter/OptionValueUInt64.h"
 
+#include "lldb/Interpreter/OptionValue.h"
 #include "lldb/Utility/Stream.h"
 
 using namespace lldb;
@@ -30,6 +31,11 @@ void OptionValueUInt64::DumpValue(const ExecutionContext *exe_ctx, Stream &strm,
     if (dump_mask & eDumpOptionType)
       strm.PutCString(" = ");
     strm.Printf("%" PRIu64, m_current_value);
+    if (dump_mask & eDumpOptionDefaultValue &&
+        m_current_value != m_default_value) {
+      DefaultValueFormat label(strm);
+      strm.Printf("%" PRIu64, m_default_value);
+    }
   }
 }
 
diff --git a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
index fe9f5d086da2c..1d210ea78df1a 100644
--- a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
+++ b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
@@ -1561,7 +1561,8 @@ void DynamicLoaderDarwinKernel::SetNotificationBreakpointIfNeeded() {
             .CreateBreakpoint(&module_spec_list, nullptr,
                               "OSKextLoadedKextSummariesUpdated",
                               eFunctionNameTypeFull, eLanguageTypeUnknown, 0,
-                              skip_prologue, internal_bp, hardware)
+                              /*offset_is_insn_count = */ false, skip_prologue,
+                              internal_bp, hardware)
             .get();
 
     bp->SetCallback(DynamicLoaderDarwinKernel::BreakpointHitCallback, this,
diff --git a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp
index 08bef4999eb9a..efb9ccc76b507 100644
--- a/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp
+++ b/lldb/source/Plugins/DynamicLoader/MacOSX-DYLD/DynamicLoaderMacOS.cpp
@@ -530,7 +530,7 @@ bool DynamicLoaderMacOS::SetNotificationBreakpoint() {
           m_process->GetTarget()
               .CreateBreakpoint(&dyld_filelist, source_files,
                                 "lldb_image_notifier", eFunctionNameTypeFull,
-                                eLanguageTypeUnknown, 0, skip_prologue,
+                                eLanguageTypeUnknown, 0, false, skip_prologue,
                                 internal, hardware)
               .get();
       breakpoint->SetCallback(DynamicLoaderMacOS::NotifyBreakpointHit, this,
@@ -546,8 +546,9 @@ bool DynamicLoaderMacOS::SetNotificationBreakpoint() {
             m_process->GetTarget()
                 .CreateBreakpoint(&dyld_filelist, source_files,
                                   "gdb_image_notifier", eFunctionNameTypeFull,
-                                  eLanguageTypeUnknown, 0, skip_prologue,
-                                  internal, hardware)
+                                  eLanguageTypeUnknown, 0,
+                                  /*offset_is_insn_count = */ false,
+                                  skip_prologue, internal, hardware)
                 .get();
         breakpoint->SetCallback(DynamicLoaderMacOS::NotifyBreakpointHit, this,
                                 true);
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
index 2529e78f78bca..92094c01e2a54 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.cpp
@@ -123,6 +123,12 @@ class DeclContextOverride {
 
     decl->setDeclContext(decl->getASTContext().getTranslationUnitDecl());
     decl->setLexicalDeclContext(decl->getASTContext().getTranslationUnitDecl());
+    // Changing the DeclContext might change the linkage. For example, if the
+    // entity was previously declared inside a function, it will not be
+    // external, but changing the declaration context to the TU will make it
+    // external. Make sure this will recompute the linkage if it was computed
+    // before.
+    decl->invalidateCachedLinkage();
   }
 
   bool ChainPassesThrough(
@@ -229,6 +235,35 @@ class CompleteTagDeclsScope : public ClangASTImporter::NewDeclListener {
   clang::ASTContext *m_src_ctx;
   ClangASTImporter &importer;
 
+  void CompleteDecl(
+      Decl *decl,
+      lldb_private::ClangASTImporter::ASTContextMetadata const &to_context_md) {
+    // The decl that should be completed has to be imported into the target
+    // context from some other context.
+    assert(to_context_md.hasOrigin(decl));
+    // We should only complete decls coming from the source context.
+    assert(to_context_md.getOrigin(decl).ctx == m_src_ctx);
+
+    Decl *original_decl = to_context_md.getOrigin(decl).decl;
+
+    // Complete the decl now.
+    TypeSystemClang::GetCompleteDecl(m_src_ctx, original_decl);
+    if (auto *tag_decl = dyn_cast<TagDecl>(decl)) {
+      if (auto *original_tag_decl = dyn_cast<TagDecl>(original_decl)) {
+        if (original_tag_decl->isCompleteDefinition()) {
+          m_delegate->ImportDefinitionTo(tag_decl, original_tag_decl);
+          tag_decl->setCompleteDefinition(true);
+        }
+      }
+
+      tag_decl->setHasExternalLexicalStorage(false);
+      tag_decl->setHasExternalVisibleStorage(false);
+    } else if (auto *container_decl = dyn_cast<ObjCContainerDecl>(decl)) {
+      container_decl->setHasExternalLexicalStorage(false);
+      container_decl->setHasExternalVisibleStorage(false);
+    }
+  }
+
 public:
   /// Constructs a CompleteTagDeclsScope.
   /// \param importer The ClangASTImporter that we should observe.
@@ -251,30 +286,7 @@ class CompleteTagDeclsScope : public ClangASTImporter::NewDeclListener {
       NamedDecl *decl = m_decls_to_complete.pop_back_val();
       m_decls_already_completed.insert(decl);
 
-      // The decl that should be completed has to be imported into the target
-      // context from some other context.
-      assert(to_context_md->hasOrigin(decl));
-      // We should only complete decls coming from the source context.
-      assert(to_context_md->getOrigin(decl).ctx == m_src_ctx);
-
-      Decl *original_decl = to_context_md->getOrigin(decl).decl;
-
-      // Complete the decl now.
-      TypeSystemClang::GetCompleteDecl(m_src_ctx, original_decl);
-      if (auto *tag_decl = dyn_cast<TagDecl>(decl)) {
-        if (auto *original_tag_decl = dyn_cast<TagDecl>(original_decl)) {
-          if (original_tag_decl->isCompleteDefinition()) {
-            m_delegate->ImportDefinitionTo(tag_decl, original_tag_decl);
-            tag_decl->setCompleteDefinition(true);
-          }
-        }
-
-        tag_decl->setHasExternalLexicalStorage(false);
-        tag_decl->setHasExternalVisibleStorage(false);
-      } else if (auto *container_decl = dyn_cast<ObjCContainerDecl>(decl)) {
-        container_decl->setHasExternalLexicalStorage(false);
-        container_decl->setHasExternalVisibleStorage(false);
-      }
+      CompleteDecl(decl, *to_context_md);
 
       to_context_md->removeOrigin(decl);
     }
@@ -320,7 +332,8 @@ CompilerType ClangASTImporter::DeportType(TypeSystemClang &dst,
   DeclContextOverride decl_context_override;
 
   if (auto *t = ClangUtil::GetQualType(src_type)->getAs<TagType>())
-    decl_context_override.OverrideAllDeclsFromContainingFunction(t->getDecl());
+    decl_context_override.OverrideAllDeclsFromContainingFunction(
+        t->getOriginalDecl());
 
   CompleteTagDeclsScope complete_scope(*this, &dst.getASTContext(),
                                        &src_ctxt->getASTContext());
@@ -358,6 +371,16 @@ clang::Decl *ClangASTImporter::DeportDecl(clang::ASTContext *dst_ctx,
   return result;
 }
 
+bool ClangASTImporter::CanImport(const Decl *d) {
+  if (!d)
+    return false;
+  if (isa<TagDecl>(d))
+    return GetDeclOrigin(d).Valid();
+  if (isa<ObjCInterfaceDecl>(d))
+    return GetDeclOrigin(d).Valid();
+  return false;
+}
+
 bool ClangASTImporter::CanImport(const CompilerType &type) {
   if (!ClangUtil::IsClangType(type))
     return false;
@@ -367,24 +390,10 @@ bool ClangASTImporter::CanImport(const CompilerType &type) {
 
   const clang::Type::TypeClass type_class = qual_type->getTypeClass();
   switch (type_class) {
-  case clang::Type::Record: {
-    const clang::CXXRecordDecl *cxx_record_decl =
-        qual_type->getAsCXXRecordDecl();
-    if (cxx_record_decl) {
-      if (GetDeclOrigin(cxx_record_decl).Valid())
-        return true;
-    }
-  } break;
-
-  case clang::Type::Enum: {
-    clang::EnumDecl *enum_decl =
-        llvm::cast<clang::EnumType>(qual_type)->getDecl();
-    if (enum_decl) {
-      if (GetDeclOrigin(enum_decl).Valid())
-        return true;
-    }
-  } break;
-
+  case clang::Type::Record:
+    return CanImport(qual_type->getAsCXXRecordDecl());
+  case clang::Type::Enum:
+    return CanImport(llvm::cast<clang::EnumType>(qual_type)->getOriginalDecl());
   case clang::Type::ObjCObject:
   case clang::Type::ObjCInterface: {
     const clang::ObjCObjectType *objc_class_type =
@@ -394,10 +403,7 @@ bool ClangASTImporter::CanImport(const CompilerType &type) {
           objc_class_type->getInterface();
       // We currently can't complete objective C types through the newly added
       // ASTContext because it only supports TagDecl objects right now...
-      if (class_interface_decl) {
-        if (GetDeclOrigin(class_interface_decl).Valid())
-          return true;
-      }
+      return CanImport(class_interface_decl);
     }
   } break;
 
@@ -414,12 +420,6 @@ bool ClangASTImporter::CanImport(const CompilerType &type) {
                                       ->getDeducedType()
                                       .getAsOpaquePtr()));
 
-  case clang::Type::Elaborated:
-    return CanImport(CompilerType(type.GetTypeSystem(),
-                                  llvm::cast<clang::ElaboratedType>(qual_type)
-                                      ->getNamedType()
-                                      .getAsOpaquePtr()));
-
   case clang::Type::Paren:
     return CanImport(CompilerType(
         type.GetTypeSystem(),
@@ -452,7 +452,7 @@ bool ClangASTImporter::Import(const CompilerType &type) {
 
   case clang::Type::Enum: {
     clang::EnumDecl *enum_decl =
-        llvm::cast<clang::EnumType>(qual_type)->getDecl();
+        llvm::cast<clang::EnumType>(qual_type)->getOriginalDecl();
     if (enum_decl) {
       if (GetDeclOrigin(enum_decl).Valid())
         return CompleteAndFetchChildren(qual_type);
@@ -488,12 +488,6 @@ bool ClangASTImporter::Import(const CompilerType &type) {
                                    ->getDeducedType()
                                    .getAsOpaquePtr()));
 
-  case clang::Type::Elaborated:
-    return Import(CompilerType(type.GetTypeSystem(),
-                               llvm::cast<clang::ElaboratedType>(qual_type)
-                                   ->getNamedType()
-                                   .getAsOpaquePtr()));
-
   case clang::Type::Paren:
     return Import(CompilerType(
         type.GetTypeSystem(),
@@ -597,7 +591,7 @@ bool ExtractBaseOffsets(const ASTRecordLayout &record_layout,
       return false;
 
     DeclFromUser<RecordDecl> origin_base_record(
-        origin_base_record_type->getDecl());
+        origin_base_record_type->getOriginalDecl());
 
     if (origin_base_record.IsInvalid())
       return false;
@@ -728,7 +722,8 @@ bool ClangASTImporter::importRecordLayoutFromOrigin(
 
         QualType base_type = bi->getType();
         const RecordType *base_record_type = base_type->getAs<RecordType>();
-        DeclFromParser<RecordDecl> base_record(base_record_type->getDecl());
+        DeclFromParser<RecordDecl> base_record(
+            base_record_type->getOriginalDecl());
         DeclFromParser<CXXRecordDecl> base_cxx_record =
             DynCast<CXXRecordDecl>(base_record);
 
@@ -860,7 +855,7 @@ bool ClangASTImporter::CompleteAndFetchChildren(clang::QualType type) {
   Log *log = GetLog(LLDBLog::Expressions);
 
   if (const TagType *tag_type = type->getAs<TagType>()) {
-    TagDecl *tag_decl = tag_type->getDecl();
+    TagDecl *tag_decl = tag_type->getOriginalDecl();
 
     DeclOrigin decl_origin = GetDeclOrigin(tag_decl);
 
@@ -928,9 +923,9 @@ bool ClangASTImporter::RequireCompleteType(clang::QualType type) {
     return false;
 
   if (const TagType *tag_type = type->getAs<TagType>()) {
-    TagDecl *tag_decl = tag_type->getDecl();
+    TagDecl *tag_decl = tag_type->getOriginalDecl();
 
-    if (tag_decl->getDefinition() || tag_decl->isBeingDefined())
+    if (tag_decl->getDefinition())
       return true;
 
     return CompleteTagDecl(tag_decl);
@@ -1053,6 +1048,16 @@ ClangASTImporter::MapCompleter::~MapCompleter() = default;
 
 llvm::Expected<Decl *>
 ClangASTImporter::ASTImporterDelegate::ImportImpl(Decl *From) {
+  // FIXME: The Minimal import mode of clang::ASTImporter does not correctly
+  // import Lambda definitions. Work around this for now by not importing
+  // lambdas at all. This is most likely encountered when importing decls from
+  // the `std` module (not from debug-info), where lambdas can be defined in
+  // inline function bodies. Those will be imported by LLDB.
+  if (const auto *CXX = llvm::dyn_cast<clang::CXXRecordDecl>(From))
+    if (CXX->isLambda())
+      return llvm::make_error<ASTImportError>(
+          ASTImportError::UnsupportedConstruct);
+
   if (m_std_handler) {
     std::optional<Decl *> D = m_std_handler->Import(From);
     if (D) {
@@ -1375,6 +1380,18 @@ void ClangASTImporter::ASTImporterDelegate::Imported(clang::Decl *from,
              from, m_source_ctx, &to->getASTContext());
   }
 
+  if (auto *to_namespace_decl = dyn_cast<NamespaceDecl>(to)) {
+    m_main.BuildNamespaceMap(to_namespace_decl);
+    to_namespace_decl->setHasExternalVisibleStorage();
+  }
+
+  MarkDeclImported(from, to);
+}
+
+void ClangASTImporter::ASTImporterDelegate::MarkDeclImported(Decl *from,
+                                                             Decl *to) {
+  Log *log = GetLog(LLDBLog::Expressions);
+
   if (auto *to_tag_decl = dyn_cast<TagDecl>(to)) {
     to_tag_decl->setHasExternalLexicalStorage();
     to_tag_decl->getPrimaryContext()->setMustBuildLookupTable();
@@ -1389,11 +1406,6 @@ void ClangASTImporter::ASTImporterDelegate::Imported(clang::Decl *from,
         (to_tag_decl->isCompleteDefinition() ? "complete" : "incomplete"));
   }
 
-  if (auto *to_namespace_decl = dyn_cast<NamespaceDecl>(to)) {
-    m_main.BuildNamespaceMap(to_namespace_decl);
-    to_namespace_decl->setHasExternalVisibleStorage();
-  }
-
   if (auto *to_container_decl = dyn_cast<ObjCContainerDecl>(to)) {
     to_container_decl->setHasExternalLexicalStorage();
     to_container_decl->setHasExternalVisibleStorage();
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.h
index 47b137a1314f8..03d2556ca6f23 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTImporter.h
@@ -157,6 +157,8 @@ class ClangASTImporter {
   /// \see ClangASTImporter::Import
   bool CanImport(const CompilerType &type);
 
+  bool CanImport(const clang::Decl *d);
+
   /// If the given type was copied from another TypeSystemClang then copy over
   /// all missing information (e.g., the definition of a 'class' type).
   ///
@@ -346,6 +348,8 @@ class ClangASTImporter {
     llvm::Expected<clang::Decl *> ImportImpl(clang::Decl *From) override;
 
   private:
+    void MarkDeclImported(clang::Decl *from, clang::Decl *to);
+
     /// Decls we should ignore when mapping decls back to their original
     /// ASTContext. Used by the CxxModuleHandler to mark declarations that
     /// were created from the 'std' C++ module to prevent that the Importer
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
index 4b52f6aafcb75..21a930745893a 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
@@ -223,7 +223,7 @@ TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) {
           continue;
 
         TagDecl *candidate_tag_decl =
-            const_cast<TagDecl *>(tag_type->getDecl());
+            tag_type->getOriginalDecl()->getDefinitionOrSelf();
 
         if (TypeSystemClang::GetCompleteDecl(
                 &candidate_tag_decl->getASTContext(), candidate_tag_decl))
@@ -250,7 +250,8 @@ TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) {
       if (!tag_type)
         continue;
 
-      TagDecl *candidate_tag_decl = const_cast<TagDecl *>(tag_type->getDecl());
+      TagDecl *candidate_tag_decl =
+          tag_type->getOriginalDecl()->getDefinitionOrSelf();
 
       if (TypeSystemClang::GetCompleteDecl(&candidate_tag_decl->getASTContext(),
                                            candidate_tag_decl))
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
index 214e260413aa4..8a68282bbbf3e 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
@@ -839,7 +839,7 @@ void ClangExpressionDeclMap::LookUpLldbClass(NameSearchContext &context) {
 
     clang::CXXRecordDecl *class_decl = method_decl->getParent();
 
-    QualType class_qual_type(class_decl->getTypeForDecl(), 0);
+    QualType class_qual_type = m_ast_context->getCanonicalTagType(class_decl);
 
     TypeFromUser class_user_type(
         class_qual_type.getAsOpaquePtr(),
@@ -1561,7 +1561,7 @@ ClangExpressionDeclMap::AddExpressionVariable(NameSearchContext &context,
 
   if (const clang::Type *parser_type = parser_opaque_type.getTypePtr()) {
     if (const TagType *tag_type = dyn_cast<TagType>(parser_type))
-      CompleteType(tag_type->getDecl());
+      CompleteType(tag_type->getOriginalDecl()->getDefinitionOrSelf());
     if (const ObjCObjectPointerType *objc_object_ptr_type =
             dyn_cast<ObjCObjectPointerType>(parser_type))
       CompleteType(objc_object_ptr_type->getInterfaceDecl());
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangPersistentVariables.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangPersistentVariables.cpp
index aa0e6e37d63e0..319ff3fe8b447 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangPersistentVariables.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangPersistentVariables.cpp
@@ -79,9 +79,11 @@ ClangPersistentVariables::GetCompilerTypeFromPersistentDecl(
   if (p.m_decl == nullptr)
     return std::nullopt;
 
+  auto ctx = std::static_pointer_cast<TypeSystemClang>(p.m_context.lock());
   if (clang::TypeDecl *tdecl = llvm::dyn_cast<clang::TypeDecl>(p.m_decl)) {
-    opaque_compiler_type_t t = static_cast<opaque_compiler_type_t>(
-        const_cast<clang::Type *>(tdecl->getTypeForDecl()));
+    opaque_compiler_type_t t =
+        static_cast<opaque_compiler_type_t>(const_cast<clang::Type *>(
+            ctx->getASTContext().getTypeDeclType(tdecl).getTypePtr()));
     return CompilerType(p.m_context, t);
   }
   return std::nullopt;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/NameSearchContext.cpp b/lldb/source/Plugins/ExpressionParser/Clang/NameSearchContext.cpp
index 45ad4f1ad98b9..6f57c18063672 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/NameSearchContext.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/NameSearchContext.cpp
@@ -153,7 +153,7 @@ NameSearchContext::AddTypeDecl(const CompilerType &clang_type) {
 
       return (NamedDecl *)typedef_name_decl;
     } else if (const TagType *tag_type = qual_type->getAs<TagType>()) {
-      TagDecl *tag_decl = tag_type->getDecl();
+      TagDecl *tag_decl = tag_type->getOriginalDecl()->getDefinitionOrSelf();
 
       m_decls.push_back(tag_decl);
 
diff --git a/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/InstrumentationRuntimeMainThreadChecker.cpp b/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/InstrumentationRuntimeMainThreadChecker.cpp
index e67e60b4a3957..b89a6aa176919 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/InstrumentationRuntimeMainThreadChecker.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/MainThreadChecker/InstrumentationRuntimeMainThreadChecker.cpp
@@ -266,9 +266,9 @@ InstrumentationRuntimeMainThreadChecker::GetBacktracesFromExtendedStopInfo(
 
   // We gather symbolication addresses above, so no need for HistoryThread to
   // try to infer the call addresses.
-  bool pcs_are_call_addresses = true;
-  ThreadSP new_thread_sp = std::make_shared<HistoryThread>(
-      *process_sp, tid, PCs, pcs_are_call_addresses);
+  auto pc_type = HistoryPCType::Calls;
+  ThreadSP new_thread_sp =
+      std::make_shared<HistoryThread>(*process_sp, tid, PCs, pc_type);
 
   // Save this in the Process' ExtendedThreadList so a strong pointer retains
   // the object
diff --git a/lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.cpp b/lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.cpp
index c2db3540a797b..565fd353a98e5 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/UBSan/InstrumentationRuntimeUBSan.cpp
@@ -324,9 +324,9 @@ InstrumentationRuntimeUBSan::GetBacktracesFromExtendedStopInfo(
 
   // We gather symbolication addresses above, so no need for HistoryThread to
   // try to infer the call addresses.
-  bool pcs_are_call_addresses = true;
-  ThreadSP new_thread_sp = std::make_shared<HistoryThread>(
-      *process_sp, tid, PCs, pcs_are_call_addresses);
+  auto pc_type = HistoryPCType::Calls;
+  ThreadSP new_thread_sp =
+      std::make_shared<HistoryThread>(*process_sp, tid, PCs, pc_type);
   std::string stop_reason_description = GetStopReasonDescription(info);
   new_thread_sp->SetName(stop_reason_description.c_str());
 
diff --git a/lldb/source/Plugins/InstrumentationRuntime/Utility/ReportRetriever.cpp b/lldb/source/Plugins/InstrumentationRuntime/Utility/ReportRetriever.cpp
index 96489248022eb..38c334bfb78ac 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/Utility/ReportRetriever.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/Utility/ReportRetriever.cpp
@@ -83,7 +83,7 @@ ReportRetriever::RetrieveReportData(const ProcessSP process_sp) {
   options.SetAutoApplyFixIts(false);
   options.SetLanguage(eLanguageTypeObjC_plus_plus);
 
-  if (auto m = GetPreferredAsanModule(process_sp->GetTarget())) {
+  if (auto [m, _] = GetPreferredAsanModule(process_sp->GetTarget()); m) {
     SymbolContextList sc_list;
     sc_list.Append(SymbolContext(std::move(m)));
     options.SetPreferredSymbolContexts(std::move(sc_list));
diff --git a/lldb/source/Plugins/InstrumentationRuntime/Utility/Utility.cpp b/lldb/source/Plugins/InstrumentationRuntime/Utility/Utility.cpp
index a5cee5d324325..09c6988b07050 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/Utility/Utility.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/Utility/Utility.cpp
@@ -13,10 +13,12 @@
 
 namespace lldb_private {
 
-lldb::ModuleSP GetPreferredAsanModule(const Target &target) {
-  // Currently only supported on Darwin.
+std::tuple<lldb::ModuleSP, HistoryPCType>
+GetPreferredAsanModule(const Target &target) {
+  // Currently only Darwin provides ASan runtime support as part of the OS
+  // (libsanitizers).
   if (!target.GetArchitecture().GetTriple().isOSDarwin())
-    return nullptr;
+    return {nullptr, HistoryPCType::Calls};
 
   lldb::ModuleSP module;
   llvm::Regex pattern(R"(libclang_rt\.asan_.*_dynamic\.dylib)");
@@ -29,7 +31,16 @@ lldb::ModuleSP GetPreferredAsanModule(const Target &target) {
     return IterationAction::Continue;
   });
 
-  return module;
+  // `Calls` - The ASan compiler-rt runtime already massages the return
+  //   addresses into call addresses, so we don't want LLDB's unwinder to try to
+  //   locate the previous instruction again as this might lead to us reporting
+  //   a different line.
+  // `ReturnsNoZerothFrame` - Darwin, but not ASan compiler-rt implies
+  //   libsanitizers which collects return addresses.  It also discards a few
+  //   non-user frames at the top of the stack.
+  auto pc_type =
+      (module ? HistoryPCType::Calls : HistoryPCType::ReturnsNoZerothFrame);
+  return {module, pc_type};
 }
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/InstrumentationRuntime/Utility/Utility.h b/lldb/source/Plugins/InstrumentationRuntime/Utility/Utility.h
index 425f0a2d35640..e26d2bc559e58 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/Utility/Utility.h
+++ b/lldb/source/Plugins/InstrumentationRuntime/Utility/Utility.h
@@ -9,18 +9,20 @@
 #ifndef LLDB_SOURCE_PLUGINS_INSTRUMENTATIONRUNTIME_UTILITY_UTILITY_H
 #define LLDB_SOURCE_PLUGINS_INSTRUMENTATIONRUNTIME_UTILITY_UTILITY_H
 
+#include <tuple>
+
 #include "lldb/lldb-forward.h"
+#include "lldb/lldb-private-enumerations.h"
 
 namespace lldb_private {
 
-class Target;
-
 /// On Darwin, if LLDB loaded libclang_rt, it's coming from a locally built
 /// compiler-rt, and we should prefer it in favour of the system sanitizers
 /// when running InstrumentationRuntime utility expressions that use symbols
 /// from the sanitizer libraries. This helper searches the target for such a
 /// dylib. Returns nullptr if no such dylib was found.
-lldb::ModuleSP GetPreferredAsanModule(const Target &target);
+std::tuple<lldb::ModuleSP, HistoryPCType>
+GetPreferredAsanModule(const Target &target);
 
 } // namespace lldb_private
 
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index 3bc870810dc81..b4207439f5285 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -105,7 +105,9 @@ CPlusPlusLanguage::GetFunctionNameInfo(ConstString name) const {
 
 bool CPlusPlusLanguage::SymbolNameFitsToLanguage(Mangled mangled) const {
   const char *mangled_name = mangled.GetMangledName().GetCString();
-  return mangled_name && Mangled::IsMangledName(mangled_name);
+  auto mangling_scheme = Mangled::GetManglingScheme(mangled_name);
+  return mangled_name && (mangling_scheme == Mangled::eManglingSchemeItanium ||
+                          mangling_scheme == Mangled::eManglingSchemeMSVC);
 }
 
 ConstString CPlusPlusLanguage::GetDemangledFunctionNameWithoutArguments(
@@ -392,13 +394,16 @@ GetDemangledScope(const SymbolContext &sc) {
   return CPlusPlusLanguage::GetDemangledScope(demangled_name, info);
 }
 
-/// Handles anything printed after the FunctionEncoding ItaniumDemangle
-/// node. Most notably the DotSuffix node.
-///
-/// FIXME: the suffix should also have an associated
-/// CPlusPlusLanguage::GetDemangledFunctionSuffix
-/// once we start setting the `DemangledNameInfo::SuffixRange`
-/// from inside the `TrackingOutputBuffer`.
+llvm::Expected<llvm::StringRef>
+CPlusPlusLanguage::GetDemangledFunctionSuffix(llvm::StringRef demangled,
+                                              const DemangledNameInfo &info) {
+  if (!info.hasSuffix())
+    return llvm::createStringError("Suffix range for '%s' is invalid.",
+                                   demangled.data());
+
+  return demangled.slice(info.SuffixRange.first, info.SuffixRange.second);
+}
+
 static llvm::Expected<llvm::StringRef>
 GetDemangledFunctionSuffix(const SymbolContext &sc) {
   auto info_or_err = GetAndValidateInfo(sc);
@@ -407,11 +412,7 @@ GetDemangledFunctionSuffix(const SymbolContext &sc) {
 
   auto [demangled_name, info] = *info_or_err;
 
-  if (!info.hasSuffix())
-    return llvm::createStringError("Suffix range for '%s' is invalid.",
-                                   demangled_name.data());
-
-  return demangled_name.slice(info.SuffixRange.first, info.SuffixRange.second);
+  return CPlusPlusLanguage::GetDemangledFunctionSuffix(demangled_name, info);
 }
 
 llvm::Expected<llvm::StringRef>
@@ -2424,7 +2425,7 @@ bool CPlusPlusLanguage::HandleFrameFormatVariable(
     return true;
   }
   case FormatEntity::Entry::Type::FunctionSuffix: {
-    auto suffix_or_err = GetDemangledFunctionSuffix(sc);
+    auto suffix_or_err = ::GetDemangledFunctionSuffix(sc);
     if (!suffix_or_err) {
       LLDB_LOG_ERROR(
           GetLog(LLDBLog::Language), suffix_or_err.takeError(),
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h
index 4f449f11257a6..4a30299dd2658 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.h
@@ -138,6 +138,10 @@ class CPlusPlusLanguage : public Language {
   GetDemangledFunctionArguments(llvm::StringRef demangled,
                                 const DemangledNameInfo &info);
 
+  static llvm::Expected<llvm::StringRef>
+  GetDemangledFunctionSuffix(llvm::StringRef demangled,
+                             const DemangledNameInfo &info);
+
   // Extract C++ context and identifier from a string using heuristic matching
   // (as opposed to
   // CPlusPlusLanguage::CxxMethodName which has to have a fully qualified C++
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
index 595e835b37df9..f4a695e036999 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibStdcpp.cpp
@@ -241,10 +241,11 @@ VectorIteratorSyntheticFrontEnd::GetIndexOfChildWithName(ConstString name) {
 bool lldb_private::formatters::LibStdcppStringSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   ValueObjectSP ptr = valobj.GetChildAtNamePath({"_M_dataplus", "_M_p"});
-  if (!ptr)
-    return false;
+  if (!ptr || !ptr->GetError().Success())
+    stream << "Summary Unavailable";
+  else
+    stream << ptr->GetSummaryAsCString();
 
-  stream << ptr->GetSummaryAsCString();
   return true;
 }
 
diff --git a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
index ef1c2c89fe125..24e84899e683c 100644
--- a/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSDictionary.cpp
@@ -73,7 +73,8 @@ static CompilerType GetLLDBNSPairType(TargetSP target_sp) {
 
   static constexpr llvm::StringLiteral g_lldb_autogen_nspair("__lldb_autogen_nspair");
 
-  compiler_type = scratch_ts_sp->GetTypeForIdentifier<clang::CXXRecordDecl>(g_lldb_autogen_nspair);
+  compiler_type = scratch_ts_sp->GetTypeForIdentifier<clang::CXXRecordDecl>(
+      scratch_ts_sp->getASTContext(), g_lldb_autogen_nspair);
 
   if (!compiler_type) {
     compiler_type = scratch_ts_sp->CreateRecordType(
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp
index 24a73717266a4..b1f2a661c4af6 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV1.cpp
@@ -102,7 +102,7 @@ AppleObjCRuntimeV1::CreateExceptionResolver(const BreakpointSP &bkpt,
     resolver_sp = std::make_shared<BreakpointResolverName>(
         bkpt, std::get<1>(GetExceptionThrowLocation()).AsCString(),
         eFunctionNameTypeBase, eLanguageTypeUnknown, Breakpoint::Exact, 0,
-        eLazyBoolNo);
+        /*offset_is_insn_count = */ false, eLazyBoolNo);
   // FIXME: don't do catch yet.
   return resolver_sp;
 }
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
index cca721e842f0d..9beb133f5595f 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCRuntimeV2.cpp
@@ -1163,7 +1163,7 @@ AppleObjCRuntimeV2::CreateExceptionResolver(const BreakpointSP &bkpt,
     resolver_sp = std::make_shared<BreakpointResolverName>(
         bkpt, std::get<1>(GetExceptionThrowLocation()).AsCString(),
         eFunctionNameTypeBase, eLanguageTypeUnknown, Breakpoint::Exact, 0,
-        eLazyBoolNo);
+        /*offset_is_insn_count = */ false, eLazyBoolNo);
   // FIXME: We don't do catch breakpoints for ObjC yet.
   // Should there be some way for the runtime to specify what it can do in this
   // regard?
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp
index a4b3e26474a55..8dc5f511c6291 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/GNUstepObjCRuntime/GNUstepObjCRuntime.cpp
@@ -169,7 +169,8 @@ GNUstepObjCRuntime::CreateExceptionResolver(const BreakpointSP &bkpt,
   if (throw_bp)
     resolver_sp = std::make_shared<BreakpointResolverName>(
         bkpt, "objc_exception_throw", eFunctionNameTypeBase,
-        eLanguageTypeUnknown, Breakpoint::Exact, 0, eLazyBoolNo);
+        eLanguageTypeUnknown, Breakpoint::Exact, 0,
+        /*offset_is_insn_count = */ false, eLazyBoolNo);
 
   return resolver_sp;
 }
diff --git a/lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.cpp b/lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.cpp
index afaaa57b09587..206a47165f762 100644
--- a/lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.cpp
+++ b/lldb/source/Plugins/MemoryHistory/asan/MemoryHistoryASan.cpp
@@ -91,11 +91,9 @@ const char *memory_history_asan_command_format =
     t;
 )";
 
-static void CreateHistoryThreadFromValueObject(ProcessSP process_sp,
-                                               ValueObjectSP return_value_sp,
-                                               const char *type,
-                                               const char *thread_name,
-                                               HistoryThreads &result) {
+static void CreateHistoryThreadFromValueObject(
+    ProcessSP process_sp, ValueObjectSP return_value_sp, HistoryPCType pc_type,
+    const char *type, const char *thread_name, HistoryThreads &result) {
   std::string count_path = "." + std::string(type) + "_count";
   std::string tid_path = "." + std::string(type) + "_tid";
   std::string trace_path = "." + std::string(type) + "_trace";
@@ -128,12 +126,8 @@ static void CreateHistoryThreadFromValueObject(ProcessSP process_sp,
     pcs.push_back(pc);
   }
 
-  // The ASAN runtime already massages the return addresses into call
-  // addresses, we don't want LLDB's unwinder to try to locate the previous
-  // instruction again as this might lead to us reporting a different line.
-  bool pcs_are_call_addresses = true;
   HistoryThread *history_thread =
-      new HistoryThread(*process_sp, tid, pcs, pcs_are_call_addresses);
+      new HistoryThread(*process_sp, tid, pcs, pc_type);
   ThreadSP new_thread_sp(history_thread);
   std::ostringstream thread_name_with_number;
   thread_name_with_number << thread_name << " Thread " << tid;
@@ -176,7 +170,8 @@ HistoryThreads MemoryHistoryASan::GetHistoryThreads(lldb::addr_t address) {
   options.SetAutoApplyFixIts(false);
   options.SetLanguage(eLanguageTypeObjC_plus_plus);
 
-  if (auto m = GetPreferredAsanModule(process_sp->GetTarget())) {
+  auto [m, pc_type] = GetPreferredAsanModule(process_sp->GetTarget());
+  if (m) {
     SymbolContextList sc_list;
     sc_list.Append(SymbolContext(std::move(m)));
     options.SetPreferredSymbolContexts(std::move(sc_list));
@@ -197,10 +192,10 @@ HistoryThreads MemoryHistoryASan::GetHistoryThreads(lldb::addr_t address) {
   if (!return_value_sp)
     return result;
 
-  CreateHistoryThreadFromValueObject(process_sp, return_value_sp, "free",
-                                     "Memory deallocated by", result);
-  CreateHistoryThreadFromValueObject(process_sp, return_value_sp, "alloc",
-                                     "Memory allocated by", result);
+  CreateHistoryThreadFromValueObject(process_sp, return_value_sp, pc_type,
+                                     "free", "Memory deallocated by", result);
+  CreateHistoryThreadFromValueObject(process_sp, return_value_sp, pc_type,
+                                     "alloc", "Memory allocated by", result);
 
   return result;
 }
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index 13df6e2f26b53..d7cb60e3f0c38 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -1156,6 +1156,7 @@ AddressClass ObjectFileMachO::GetAddressClass(lldb::addr_t file_addr) {
         case eSectionTypeDataObjCMessageRefs:
         case eSectionTypeDataObjCCFStrings:
         case eSectionTypeGoSymtab:
+        case eSectionTypeWasmName:
           return AddressClass::eData;
 
         case eSectionTypeDebug:
diff --git a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
index 25e98882c20c9..c361087730828 100644
--- a/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
+++ b/lldb/source/Plugins/ObjectFile/Minidump/MinidumpFileBuilder.cpp
@@ -308,40 +308,49 @@ Status MinidumpFileBuilder::AddModuleList() {
   // the llvm::minidump::Module's structures into helper data
   size_t size_before = GetCurrentDataEndOffset();
 
-  // This is the size of the main part of the ModuleList stream.
-  // It consists of a module number and corresponding number of
-  // structs describing individual modules
-  size_t module_stream_size =
-      sizeof(llvm::support::ulittle32_t) + modules_count * minidump_module_size;
-
-  // Adding directory describing this stream.
-  error = AddDirectory(StreamType::ModuleList, module_stream_size);
-  if (error.Fail())
-    return error;
-
-  m_data.AppendData(&modules_count, sizeof(llvm::support::ulittle32_t));
-
   // Temporary storage for the helper data (of variable length)
   // as these cannot be dumped to m_data before dumping entire
   // array of module structures.
   DataBufferHeap helper_data;
 
+  // Vector to store modules that pass validation.
+  std::vector<std::pair<ModuleSP, uint64_t>> valid_modules;
+
   for (size_t i = 0; i < modules_count; ++i) {
     ModuleSP mod = modules.GetModuleAtIndex(i);
     std::string module_name = mod->GetSpecificationDescription();
     auto maybe_mod_size = getModuleFileSize(target, mod);
     if (!maybe_mod_size) {
       llvm::Error mod_size_err = maybe_mod_size.takeError();
-      llvm::handleAllErrors(std::move(mod_size_err),
-                            [&](const llvm::ErrorInfoBase &E) {
-                              error = Status::FromErrorStringWithFormat(
-                                  "Unable to get the size of module %s: %s.",
-                                  module_name.c_str(), E.message().c_str());
-                            });
-      return error;
+      Log *log = GetLog(LLDBLog::Object);
+      llvm::handleAllErrors(
+          std::move(mod_size_err), [&](const llvm::ErrorInfoBase &E) {
+            if (log) {
+              LLDB_LOGF(log, "Unable to get the size of module %s: %s",
+                        module_name.c_str(), E.message().c_str());
+            }
+          });
+      continue;
     }
+    valid_modules.emplace_back(mod, *maybe_mod_size);
+  }
+
+  size_t module_stream_size = sizeof(llvm::support::ulittle32_t) +
+                              valid_modules.size() * minidump_module_size;
+
+  error = AddDirectory(StreamType::ModuleList, module_stream_size);
+  if (error.Fail())
+    return error;
 
-    uint64_t mod_size = std::move(*maybe_mod_size);
+  // Setting the header with the number of modules.
+  llvm::support::ulittle32_t count =
+      static_cast<llvm::support::ulittle32_t>(valid_modules.size());
+  m_data.AppendData(&count, sizeof(llvm::support::ulittle32_t));
+
+  for (const auto &valid_module : valid_modules) {
+    ModuleSP mod = valid_module.first;
+    uint64_t module_size = valid_module.second;
+    std::string module_name = mod->GetSpecificationDescription();
 
     llvm::support::ulittle32_t signature =
         static_cast<llvm::support::ulittle32_t>(
@@ -381,7 +390,7 @@ Status MinidumpFileBuilder::AddModuleList() {
     llvm::minidump::Module m{};
     m.BaseOfImage = static_cast<llvm::support::ulittle64_t>(
         mod->GetObjectFile()->GetBaseAddress().GetLoadAddress(&target));
-    m.SizeOfImage = static_cast<llvm::support::ulittle32_t>(mod_size);
+    m.SizeOfImage = static_cast<llvm::support::ulittle32_t>(module_size);
     m.Checksum = static_cast<llvm::support::ulittle32_t>(0);
     m.TimeDateStamp =
         static_cast<llvm::support::ulittle32_t>(std::time(nullptr));
diff --git a/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp b/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp
index b1efd25949379..777b20e9bb0f6 100644
--- a/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp
+++ b/lldb/source/Plugins/ObjectFile/wasm/ObjectFileWasm.cpp
@@ -22,6 +22,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/BinaryFormat/Wasm.h"
+#include "llvm/Support/CheckedArithmetic.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Format.h"
 #include <optional>
@@ -35,6 +36,41 @@ LLDB_PLUGIN_DEFINE(ObjectFileWasm)
 static const uint32_t kWasmHeaderSize =
     sizeof(llvm::wasm::WasmMagic) + sizeof(llvm::wasm::WasmVersion);
 
+/// Helper to read a 32-bit ULEB using LLDB's DataExtractor.
+static inline llvm::Expected<uint32_t> GetULEB32(DataExtractor &data,
+                                                 lldb::offset_t &offset) {
+  const uint64_t value = data.GetULEB128(&offset);
+  if (value > std::numeric_limits<uint32_t>::max())
+    return llvm::createStringError("ULEB exceeds 32 bits");
+  return value;
+}
+
+/// Helper to read a 32-bit ULEB using LLVM's DataExtractor.
+static inline llvm::Expected<uint32_t>
+GetULEB32(llvm::DataExtractor &data, llvm::DataExtractor::Cursor &c) {
+  const uint64_t value = data.getULEB128(c);
+  if (!c)
+    return c.takeError();
+  if (value > std::numeric_limits<uint32_t>::max())
+    return llvm::createStringError("ULEB exceeds 32 bits");
+  return value;
+}
+
+/// Helper to read a Wasm string, whcih is encoded as a vector of UTF-8 codes.
+static inline llvm::Expected<std::string>
+GetWasmString(llvm::DataExtractor &data, llvm::DataExtractor::Cursor &c) {
+  llvm::Expected<uint32_t> len = GetULEB32(data, c);
+  if (!len)
+    return len.takeError();
+
+  llvm::SmallVector<uint8_t, 32> str_storage;
+  data.getU8(c, str_storage, *len);
+  if (!c)
+    return c.takeError();
+
+  return std::string(toStringRef(llvm::ArrayRef(str_storage)));
+}
+
 /// Checks whether the data buffer starts with a valid Wasm module header.
 static bool ValidateModuleHeader(const DataBufferSP &data_sp) {
   if (!data_sp || data_sp->GetByteSize() < kWasmHeaderSize)
@@ -50,32 +86,6 @@ static bool ValidateModuleHeader(const DataBufferSP &data_sp) {
   return version == llvm::wasm::WasmVersion;
 }
 
-static std::optional<ConstString>
-GetWasmString(llvm::DataExtractor &data, llvm::DataExtractor::Cursor &c) {
-  // A Wasm string is encoded as a vector of UTF-8 codes.
-  // Vectors are encoded with their u32 length followed by the element
-  // sequence.
-  uint64_t len = data.getULEB128(c);
-  if (!c) {
-    consumeError(c.takeError());
-    return std::nullopt;
-  }
-
-  if (len >= (uint64_t(1) << 32)) {
-    return std::nullopt;
-  }
-
-  llvm::SmallVector<uint8_t, 32> str_storage;
-  data.getU8(c, str_storage, len);
-  if (!c) {
-    consumeError(c.takeError());
-    return std::nullopt;
-  }
-
-  llvm::StringRef str = toStringRef(llvm::ArrayRef(str_storage));
-  return ConstString(str);
-}
-
 char ObjectFileWasm::ID;
 
 void ObjectFileWasm::Initialize() {
@@ -174,7 +184,7 @@ bool ObjectFileWasm::DecodeNextSection(lldb::offset_t *offset_ptr) {
   if (!c)
     return !llvm::errorToBool(c.takeError());
 
-  if (payload_len >= (uint64_t(1) << 32))
+  if (payload_len > std::numeric_limits<uint32_t>::max())
     return false;
 
   if (section_id == llvm::wasm::WASM_SEC_CUSTOM) {
@@ -182,16 +192,19 @@ bool ObjectFileWasm::DecodeNextSection(lldb::offset_t *offset_ptr) {
     // identifying the custom section, followed by an uninterpreted sequence
     // of bytes.
     lldb::offset_t prev_offset = c.tell();
-    std::optional<ConstString> sect_name = GetWasmString(data, c);
-    if (!sect_name)
+    llvm::Expected<std::string> sect_name = GetWasmString(data, c);
+    if (!sect_name) {
+      LLDB_LOG_ERROR(GetLog(LLDBLog::Object), sect_name.takeError(),
+                     "failed to parse section name: {0}");
       return false;
+    }
 
     if (payload_len < c.tell() - prev_offset)
       return false;
 
     uint32_t section_length = payload_len - (c.tell() - prev_offset);
     m_sect_infos.push_back(section_info{*offset_ptr + c.tell(), section_length,
-                                        section_id, *sect_name});
+                                        section_id, ConstString(*sect_name)});
     *offset_ptr += (c.tell() + section_length);
   } else if (section_id <= llvm::wasm::WASM_SEC_LAST_KNOWN) {
     m_sect_infos.push_back(section_info{*offset_ptr + c.tell(),
@@ -248,12 +261,258 @@ bool ObjectFileWasm::ParseHeader() {
   return true;
 }
 
-void ObjectFileWasm::ParseSymtab(Symtab &symtab) {}
+static llvm::Expected<std::vector<AddressRange>>
+ParseFunctions(SectionSP code_section_sp) {
+  DataExtractor data;
+  code_section_sp->GetSectionData(data);
+  lldb::offset_t offset = 0;
+
+  llvm::Expected<uint32_t> function_count = GetULEB32(data, offset);
+  if (!function_count)
+    return function_count.takeError();
+
+  std::vector<AddressRange> functions;
+  functions.reserve(*function_count);
+
+  for (uint32_t i = 0; i < *function_count; ++i) {
+    llvm::Expected<uint32_t> function_size = GetULEB32(data, offset);
+    if (!function_size)
+      return function_size.takeError();
+    // llvm-objdump considers the ULEB with the function size to be part of the
+    // function. We can't do that here because that would break symbolic
+    // breakpoints, as that address is never executed.
+    functions.emplace_back(code_section_sp, offset, *function_size);
+
+    std::optional<lldb::offset_t> next_offset =
+        llvm::checkedAddUnsigned<lldb::offset_t>(offset, *function_size);
+    if (!next_offset)
+      return llvm::createStringError("function offset overflows 64 bits");
+    offset = *next_offset;
+  }
+
+  return functions;
+}
+
+struct WasmSegment {
+  WasmSegment(SectionSP section_sp, lldb::offset_t offset, uint32_t size)
+      : address_range(section_sp, offset, size) {};
+  std::string name;
+  AddressRange address_range;
+};
+
+static llvm::Expected<std::vector<WasmSegment>>
+ParseData(SectionSP data_section_sp) {
+  DataExtractor data;
+  data_section_sp->GetSectionData(data);
+
+  lldb::offset_t offset = 0;
+
+  llvm::Expected<uint32_t> segment_count = GetULEB32(data, offset);
+  if (!segment_count)
+    return segment_count.takeError();
+
+  std::vector<WasmSegment> segments;
+  segments.reserve(*segment_count);
+
+  for (uint32_t i = 0; i < *segment_count; ++i) {
+    llvm::Expected<uint32_t> flags = GetULEB32(data, offset);
+    if (!flags)
+      return flags.takeError();
+
+    // Data segments have a mode that identifies them as either passive or
+    // active. An active data segment copies its contents into a memory during
+    // instantiation, as specified by a memory index and a constant expression
+    // defining an offset into that memory.
+    if (*flags & llvm::wasm::WASM_DATA_SEGMENT_HAS_MEMINDEX) {
+      llvm::Expected<uint32_t> memidx = GetULEB32(data, offset);
+      if (!memidx)
+        return memidx.takeError();
+    }
+
+    if ((*flags & llvm::wasm::WASM_DATA_SEGMENT_IS_PASSIVE) == 0) {
+      // Skip over the constant expression.
+      for (uint8_t b = 0; b != llvm::wasm::WASM_OPCODE_END;)
+        b = data.GetU8(&offset);
+    }
+
+    llvm::Expected<uint32_t> segment_size = GetULEB32(data, offset);
+    if (!segment_size)
+      return segment_size.takeError();
+
+    segments.emplace_back(data_section_sp, offset, *segment_size);
+
+    std::optional<lldb::offset_t> next_offset =
+        llvm::checkedAddUnsigned<lldb::offset_t>(offset, *segment_size);
+    if (!next_offset)
+      return llvm::createStringError("segment offset overflows 64 bits");
+    offset = *next_offset;
+  }
+
+  return segments;
+}
+
+static llvm::Expected<std::vector<Symbol>>
+ParseNames(SectionSP name_section_sp,
+           const std::vector<AddressRange> &function_ranges,
+           std::vector<WasmSegment> &segments) {
+  DataExtractor name_section_data;
+  name_section_sp->GetSectionData(name_section_data);
+
+  llvm::DataExtractor data = name_section_data.GetAsLLVM();
+  llvm::DataExtractor::Cursor c(0);
+  std::vector<Symbol> symbols;
+  while (c && c.tell() < data.size()) {
+    const uint8_t type = data.getU8(c);
+    llvm::Expected<uint32_t> size = GetULEB32(data, c);
+    if (!size)
+      return size.takeError();
+
+    switch (type) {
+    case llvm::wasm::WASM_NAMES_FUNCTION: {
+      const uint64_t count = data.getULEB128(c);
+      if (count > std::numeric_limits<uint32_t>::max())
+        return llvm::createStringError("function count overflows uint32_t");
+
+      for (uint64_t i = 0; c && i < count; ++i) {
+        llvm::Expected<uint32_t> idx = GetULEB32(data, c);
+        if (!idx)
+          return idx.takeError();
+        llvm::Expected<std::string> name = GetWasmString(data, c);
+        if (!name)
+          return name.takeError();
+        if (*idx >= function_ranges.size())
+          continue;
+        symbols.emplace_back(
+            symbols.size(), Mangled(*name), lldb::eSymbolTypeCode,
+            /*external=*/false, /*is_debug=*/false, /*is_trampoline=*/false,
+            /*is_artificial=*/false, function_ranges[*idx],
+            /*size_is_valid=*/true, /*contains_linker_annotations=*/false,
+            /*flags=*/0);
+      }
+    } break;
+    case llvm::wasm::WASM_NAMES_DATA_SEGMENT: {
+      llvm::Expected<uint32_t> count = GetULEB32(data, c);
+      if (!count)
+        return count.takeError();
+      for (uint32_t i = 0; c && i < *count; ++i) {
+        llvm::Expected<uint32_t> idx = GetULEB32(data, c);
+        if (!idx)
+          return idx.takeError();
+        llvm::Expected<std::string> name = GetWasmString(data, c);
+        if (!name)
+          return name.takeError();
+        if (*idx >= segments.size())
+          continue;
+        // Update the segment name.
+        segments[i].name = *name;
+        symbols.emplace_back(
+            symbols.size(), Mangled(*name), lldb::eSymbolTypeData,
+            /*external=*/false, /*is_debug=*/false, /*is_trampoline=*/false,
+            /*is_artificial=*/false, segments[i].address_range,
+            /*size_is_valid=*/true, /*contains_linker_annotations=*/false,
+            /*flags=*/0);
+      }
+
+    } break;
+    case llvm::wasm::WASM_NAMES_GLOBAL:
+    case llvm::wasm::WASM_NAMES_LOCAL:
+    default:
+      std::optional<lldb::offset_t> offset =
+          llvm::checkedAddUnsigned<lldb::offset_t>(c.tell(), *size);
+      if (!offset)
+        return llvm::createStringError("offset overflows 64 bits");
+      c.seek(*offset);
+    }
+  }
+
+  if (!c)
+    return c.takeError();
+
+  return symbols;
+}
+
+void ObjectFileWasm::ParseSymtab(Symtab &symtab) {
+  assert(m_sections_up && "sections must be parsed");
+  Log *log = GetLog(LLDBLog::Object);
+
+  // The name section contains names and indexes. First parse the data from the
+  // relevant sections so we can access it by its index.
+  std::vector<AddressRange> functions;
+  std::vector<WasmSegment> segments;
+
+  // Parse the code section.
+  if (SectionSP code_section_sp =
+          m_sections_up->FindSectionByType(lldb::eSectionTypeCode, false)) {
+    llvm::Expected<std::vector<AddressRange>> maybe_functions =
+        ParseFunctions(code_section_sp);
+    if (!maybe_functions) {
+      LLDB_LOG_ERROR(log, maybe_functions.takeError(),
+                     "Failed to parse Wasm code section: {0}");
+      return;
+    }
+    functions = *maybe_functions;
+  }
+
+  // Parse the data section.
+  SectionSP data_section_sp =
+      m_sections_up->FindSectionByType(lldb::eSectionTypeData, false);
+  if (data_section_sp) {
+    llvm::Expected<std::vector<WasmSegment>> maybe_segments =
+        ParseData(data_section_sp);
+    if (!maybe_segments) {
+      LLDB_LOG_ERROR(log, maybe_segments.takeError(),
+                     "Failed to parse Wasm data section: {0}");
+      return;
+    }
+    segments = *maybe_segments;
+  }
+
+  // Parse the name section.
+  SectionSP name_section_sp =
+      m_sections_up->FindSectionByType(lldb::eSectionTypeWasmName, false);
+  if (!name_section_sp) {
+    LLDB_LOG(log, "Failed to parse Wasm symbol table: no names section");
+    return;
+  }
+
+  llvm::Expected<std::vector<Symbol>> symbols =
+      ParseNames(name_section_sp, functions, segments);
+  if (!symbols) {
+    LLDB_LOG_ERROR(log, symbols.takeError(), "Failed to parse Wasm names: {0}");
+    return;
+  }
+
+  for (const Symbol &symbol : *symbols)
+    symtab.AddSymbol(symbol);
+
+  lldb::user_id_t segment_id = 0;
+  for (const WasmSegment &segment : segments) {
+    const lldb::addr_t segment_addr =
+        segment.address_range.GetBaseAddress().GetFileAddress();
+    const size_t segment_size = segment.address_range.GetByteSize();
+    SectionSP segment_sp = std::make_shared<Section>(
+        /*parent_section_sp=*/data_section_sp, GetModule(),
+        /*obj_file=*/this,
+        ++segment_id << 8, // 1-based segment index, shifted by 8 bits to avoid
+                           // collision with section IDs.
+        ConstString(segment.name), eSectionTypeData,
+        /*file_vm_addr=*/segment_addr,
+        /*vm_size=*/segment_size,
+        /*file_offset=*/segment_addr,
+        /*file_size=*/segment_size,
+        /*log2align=*/0, /*flags=*/0);
+    m_sections_up->AddSection(segment_sp);
+    GetModule()->GetSectionList()->AddSection(segment_sp);
+  }
+
+  symtab.Finalize();
+}
 
 static SectionType GetSectionTypeFromName(llvm::StringRef Name) {
-  if (Name.consume_front(".debug_") || Name.consume_front(".zdebug_")) {
+  if (Name == "name")
+    return lldb::eSectionTypeWasmName;
+  if (Name.consume_front(".debug_") || Name.consume_front(".zdebug_"))
     return ObjectFile::GetDWARFSectionTypeFromName(Name);
-  }
   return eSectionTypeOther;
 }
 
@@ -283,6 +542,9 @@ void ObjectFileWasm::CreateSections(SectionList &unified_section_list) {
       // For this reason Section::GetFileAddress() must return zero for the
       // Code section.
       vm_addr = 0;
+    } else if (llvm::wasm::WASM_SEC_DATA == sect_info.id) {
+      section_type = eSectionTypeData;
+      section_name = ConstString("data");
     } else {
       section_type = GetSectionTypeFromName(sect_info.name.GetStringRef());
       if (section_type == eSectionTypeOther)
@@ -395,11 +657,15 @@ std::optional<FileSpec> ObjectFileWasm::GetExternalDebugInfoFileSpec() {
       const uint32_t kBufferSize = 1024;
       DataExtractor section_header_data =
           ReadImageData(sect_info.offset, kBufferSize);
+
       llvm::DataExtractor data = section_header_data.GetAsLLVM();
       llvm::DataExtractor::Cursor c(0);
-      std::optional<ConstString> symbols_url = GetWasmString(data, c);
-      if (symbols_url)
-        return FileSpec(symbols_url->GetStringRef());
+      llvm::Expected<std::string> symbols_url = GetWasmString(data, c);
+      if (!symbols_url) {
+        llvm::consumeError(symbols_url.takeError());
+        return std::nullopt;
+      }
+      return FileSpec(*symbols_url);
     }
   }
   return std::nullopt;
diff --git a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
index 1db7bc78013d7..cd72454fe0287 100644
--- a/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
+++ b/lldb/source/Plugins/Platform/MacOSX/PlatformDarwin.cpp
@@ -1039,7 +1039,12 @@ ResolveSDKPathFromDebugInfo(lldb_private::Target *target) {
 
   SymbolFile *sym_file = exe_module_sp->GetSymbolFile();
   if (!sym_file)
-    return llvm::createStringError("Failed to get symbol file from module");
+    return llvm::createStringError("Failed to get symbol file from executable");
+
+  if (sym_file->GetNumCompileUnits() == 0)
+    return llvm::createStringError(
+        "Failed to resolve SDK for target: executable's symbol file has no "
+        "compile units");
 
   XcodeSDK merged_sdk;
   for (unsigned i = 0; i < sym_file->GetNumCompileUnits(); ++i) {
@@ -1397,6 +1402,12 @@ PlatformDarwin::GetSDKPathFromDebugInfo(Module &module) {
         llvm::formatv("No symbol file available for module '{0}'",
                       module.GetFileSpec().GetFilename().AsCString("")));
 
+  if (sym_file->GetNumCompileUnits() == 0)
+    return llvm::createStringError(
+        llvm::formatv("Could not resolve SDK for module '{0}'. Symbol file has "
+                      "no compile units.",
+                      module.GetFileSpec()));
+
   bool found_public_sdk = false;
   bool found_internal_sdk = false;
   XcodeSDK merged_sdk;
diff --git a/lldb/source/Plugins/Process/FreeBSDKernel/CMakeLists.txt b/lldb/source/Plugins/Process/FreeBSDKernel/CMakeLists.txt
index b0d1370286a8c..c35b4def24e25 100644
--- a/lldb/source/Plugins/Process/FreeBSDKernel/CMakeLists.txt
+++ b/lldb/source/Plugins/Process/FreeBSDKernel/CMakeLists.txt
@@ -2,7 +2,7 @@ set(FBSDKERNEL_LIBS)
 if(FBSDVMCore_FOUND)
   list(APPEND FBSDKERNEL_LIBS fbsdvmcore)
 endif()
-if(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "FreeBSD")
   list(APPEND FBSDKERNEL_LIBS kvm)
 endif()
 
diff --git a/lldb/source/Plugins/Process/Utility/HistoryThread.cpp b/lldb/source/Plugins/Process/Utility/HistoryThread.cpp
index bc06757c806a9..93efa2a748880 100644
--- a/lldb/source/Plugins/Process/Utility/HistoryThread.cpp
+++ b/lldb/source/Plugins/Process/Utility/HistoryThread.cpp
@@ -27,13 +27,12 @@ using namespace lldb_private;
 
 HistoryThread::HistoryThread(lldb_private::Process &process, lldb::tid_t tid,
                              std::vector<lldb::addr_t> pcs,
-                             bool pcs_are_call_addresses)
+                             HistoryPCType pc_type)
     : Thread(process, tid, true), m_framelist_mutex(), m_framelist(),
       m_pcs(pcs), m_extended_unwind_token(LLDB_INVALID_ADDRESS), m_queue_name(),
       m_thread_name(), m_originating_unique_thread_id(tid),
       m_queue_id(LLDB_INVALID_QUEUE_ID) {
-  m_unwinder_up =
-      std::make_unique<HistoryUnwind>(*this, pcs, pcs_are_call_addresses);
+  m_unwinder_up = std::make_unique<HistoryUnwind>(*this, pcs, pc_type);
   Log *log = GetLog(LLDBLog::Object);
   LLDB_LOGF(log, "%p HistoryThread::HistoryThread", static_cast<void *>(this));
 }
diff --git a/lldb/source/Plugins/Process/Utility/HistoryThread.h b/lldb/source/Plugins/Process/Utility/HistoryThread.h
index a66e0f2d4207c..cdc3b090f4848 100644
--- a/lldb/source/Plugins/Process/Utility/HistoryThread.h
+++ b/lldb/source/Plugins/Process/Utility/HistoryThread.h
@@ -27,14 +27,14 @@ namespace lldb_private {
 /// process execution
 ///
 /// This subclass of Thread is used to provide a backtrace from earlier in
-/// process execution.  It is given a backtrace list of pc addresses and it
-/// will create stack frames for them.
+/// process execution.  It is given a backtrace list of pcs (return or call
+/// addresses) and it will create stack frames for them.
 
 class HistoryThread : public lldb_private::Thread {
 public:
   HistoryThread(lldb_private::Process &process, lldb::tid_t tid,
                 std::vector<lldb::addr_t> pcs,
-                bool pcs_are_call_addresses = false);
+                HistoryPCType pc_type = HistoryPCType::Returns);
 
   ~HistoryThread() override;
 
diff --git a/lldb/source/Plugins/Process/Utility/HistoryUnwind.cpp b/lldb/source/Plugins/Process/Utility/HistoryUnwind.cpp
index 7749dc6f5d514..3b0618fa10374 100644
--- a/lldb/source/Plugins/Process/Utility/HistoryUnwind.cpp
+++ b/lldb/source/Plugins/Process/Utility/HistoryUnwind.cpp
@@ -24,9 +24,8 @@ using namespace lldb_private;
 // Constructor
 
 HistoryUnwind::HistoryUnwind(Thread &thread, std::vector<lldb::addr_t> pcs,
-                             bool pcs_are_call_addresses)
-    : Unwind(thread), m_pcs(pcs),
-      m_pcs_are_call_addresses(pcs_are_call_addresses) {}
+                             HistoryPCType pc_type)
+    : Unwind(thread), m_pcs(pcs), m_pc_type(pc_type) {}
 
 // Destructor
 
@@ -52,6 +51,17 @@ HistoryUnwind::DoCreateRegisterContextForFrame(StackFrame *frame) {
   return rctx;
 }
 
+static bool BehavesLikeZerothFrame(HistoryPCType pc_type, uint32_t frame_idx) {
+  switch (pc_type) {
+  case HistoryPCType::Returns:
+    return (frame_idx == 0);
+  case HistoryPCType::ReturnsNoZerothFrame:
+    return false;
+  case HistoryPCType::Calls:
+    return true;
+  }
+}
+
 bool HistoryUnwind::DoGetFrameInfoAtIndex(uint32_t frame_idx, lldb::addr_t &cfa,
                                           lldb::addr_t &pc,
                                           bool &behaves_like_zeroth_frame) {
@@ -61,10 +71,7 @@ bool HistoryUnwind::DoGetFrameInfoAtIndex(uint32_t frame_idx, lldb::addr_t &cfa,
   if (frame_idx < m_pcs.size()) {
     cfa = frame_idx;
     pc = m_pcs[frame_idx];
-    if (m_pcs_are_call_addresses)
-      behaves_like_zeroth_frame = true;
-    else
-      behaves_like_zeroth_frame = (frame_idx == 0);
+    behaves_like_zeroth_frame = BehavesLikeZerothFrame(m_pc_type, frame_idx);
     return true;
   }
   return false;
diff --git a/lldb/source/Plugins/Process/Utility/HistoryUnwind.h b/lldb/source/Plugins/Process/Utility/HistoryUnwind.h
index cb72b5d0a1764..3cf70a4255254 100644
--- a/lldb/source/Plugins/Process/Utility/HistoryUnwind.h
+++ b/lldb/source/Plugins/Process/Utility/HistoryUnwind.h
@@ -19,7 +19,7 @@ namespace lldb_private {
 class HistoryUnwind : public lldb_private::Unwind {
 public:
   HistoryUnwind(Thread &thread, std::vector<lldb::addr_t> pcs,
-                bool pcs_are_call_addresses = false);
+                HistoryPCType pc_type = HistoryPCType::Returns);
 
   ~HistoryUnwind() override;
 
@@ -36,9 +36,7 @@ class HistoryUnwind : public lldb_private::Unwind {
 
 private:
   std::vector<lldb::addr_t> m_pcs;
-  /// This boolean indicates that the PCs in the non-0 frames are call
-  /// addresses and not return addresses.
-  bool m_pcs_are_call_addresses;
+  HistoryPCType m_pc_type;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
index 021ec49b3d2be..bca5bff24bff0 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
@@ -46,9 +46,9 @@ class RegisterInfoPOSIX_arm64
 
   // based on RegisterContextDarwin_arm64.h
   // Pack this so there are no extra bytes, but align its start address to at
-  // least 4 bytes to prevent alignment errors on Arm 32-bit.
+  // least 8 bytes to prevent alignment errors.
   LLVM_PACKED_START
-  struct alignas(4) GPR {
+  struct alignas(8) GPR {
     uint64_t x[29]; // x0-x28
     uint64_t fp;    // x29
     uint64_t lr;    // x30
diff --git a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp
index c9fe474d45c49..c359663239dcc 100644
--- a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp
+++ b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp
@@ -27,25 +27,12 @@ using namespace llvm;
 LLDB_PLUGIN_DEFINE(ProtocolServerMCP)
 
 static constexpr size_t kChunkSize = 1024;
+static constexpr llvm::StringLiteral kName = "lldb-mcp";
+static constexpr llvm::StringLiteral kVersion = "0.1.0";
 
-ProtocolServerMCP::ProtocolServerMCP() : ProtocolServer() {
-  AddRequestHandler("initialize",
-                    std::bind(&ProtocolServerMCP::InitializeHandler, this,
-                              std::placeholders::_1));
-
-  AddRequestHandler("tools/list",
-                    std::bind(&ProtocolServerMCP::ToolsListHandler, this,
-                              std::placeholders::_1));
-  AddRequestHandler("tools/call",
-                    std::bind(&ProtocolServerMCP::ToolsCallHandler, this,
-                              std::placeholders::_1));
-
-  AddRequestHandler("resources/list",
-                    std::bind(&ProtocolServerMCP::ResourcesListHandler, this,
-                              std::placeholders::_1));
-  AddRequestHandler("resources/read",
-                    std::bind(&ProtocolServerMCP::ResourcesReadHandler, this,
-                              std::placeholders::_1));
+ProtocolServerMCP::ProtocolServerMCP()
+    : ProtocolServer(),
+      lldb_protocol::mcp::Server(std::string(kName), std::string(kVersion)) {
   AddNotificationHandler("notifications/initialized",
                          [](const lldb_protocol::mcp::Notification &) {
                            LLDB_LOG(GetLog(LLDBLog::Host),
@@ -77,32 +64,6 @@ llvm::StringRef ProtocolServerMCP::GetPluginDescriptionStatic() {
   return "MCP Server.";
 }
 
-llvm::Expected<lldb_protocol::mcp::Response>
-ProtocolServerMCP::Handle(lldb_protocol::mcp::Request request) {
-  auto it = m_request_handlers.find(request.method);
-  if (it != m_request_handlers.end()) {
-    llvm::Expected<lldb_protocol::mcp::Response> response = it->second(request);
-    if (!response)
-      return response;
-    response->id = request.id;
-    return *response;
-  }
-
-  return make_error<MCPError>(
-      llvm::formatv("no handler for request: {0}", request.method).str());
-}
-
-void ProtocolServerMCP::Handle(lldb_protocol::mcp::Notification notification) {
-  auto it = m_notification_handlers.find(notification.method);
-  if (it != m_notification_handlers.end()) {
-    it->second(notification);
-    return;
-  }
-
-  LLDB_LOG(GetLog(LLDBLog::Host), "MPC notification: {0} ({1})",
-           notification.method, notification.params);
-}
-
 void ProtocolServerMCP::AcceptCallback(std::unique_ptr<Socket> socket) {
   LLDB_LOG(GetLog(LLDBLog::Host), "New MCP client ({0}) connected",
            m_clients.size() + 1);
@@ -157,7 +118,7 @@ llvm::Error ProtocolServerMCP::ReadCallback(Client &client) {
 }
 
 llvm::Error ProtocolServerMCP::Start(ProtocolServer::Connection connection) {
-  std::lock_guard<std::mutex> guard(m_server_mutex);
+  std::lock_guard<std::mutex> guard(m_mutex);
 
   if (m_running)
     return llvm::createStringError("the MCP server is already running");
@@ -189,7 +150,7 @@ llvm::Error ProtocolServerMCP::Start(ProtocolServer::Connection connection) {
 
 llvm::Error ProtocolServerMCP::Stop() {
   {
-    std::lock_guard<std::mutex> guard(m_server_mutex);
+    std::lock_guard<std::mutex> guard(m_mutex);
     if (!m_running)
       return createStringError("the MCP sever is not running");
     m_running = false;
@@ -204,7 +165,7 @@ llvm::Error ProtocolServerMCP::Stop() {
     m_loop_thread.join();
 
   {
-    std::lock_guard<std::mutex> guard(m_server_mutex);
+    std::lock_guard<std::mutex> guard(m_mutex);
     m_listener.reset();
     m_listen_handlers.clear();
     m_clients.clear();
@@ -213,48 +174,6 @@ llvm::Error ProtocolServerMCP::Stop() {
   return llvm::Error::success();
 }
 
-llvm::Expected<std::optional<lldb_protocol::mcp::Message>>
-ProtocolServerMCP::HandleData(llvm::StringRef data) {
-  auto message = llvm::json::parse<lldb_protocol::mcp::Message>(/*JSON=*/data);
-  if (!message)
-    return message.takeError();
-
-  if (const lldb_protocol::mcp::Request *request =
-          std::get_if<lldb_protocol::mcp::Request>(&(*message))) {
-    llvm::Expected<lldb_protocol::mcp::Response> response = Handle(*request);
-
-    // Handle failures by converting them into an Error message.
-    if (!response) {
-      lldb_protocol::mcp::Error protocol_error;
-      llvm::handleAllErrors(
-          response.takeError(),
-          [&](const MCPError &err) { protocol_error = err.toProtcolError(); },
-          [&](const llvm::ErrorInfoBase &err) {
-            protocol_error.error.code = MCPError::kInternalError;
-            protocol_error.error.message = err.message();
-          });
-      protocol_error.id = request->id;
-      return protocol_error;
-    }
-
-    return *response;
-  }
-
-  if (const lldb_protocol::mcp::Notification *notification =
-          std::get_if<lldb_protocol::mcp::Notification>(&(*message))) {
-    Handle(*notification);
-    return std::nullopt;
-  }
-
-  if (std::get_if<lldb_protocol::mcp::Error>(&(*message)))
-    return llvm::createStringError("unexpected MCP message: error");
-
-  if (std::get_if<lldb_protocol::mcp::Response>(&(*message)))
-    return llvm::createStringError("unexpected MCP message: response");
-
-  llvm_unreachable("all message types handled");
-}
-
 lldb_protocol::mcp::Capabilities ProtocolServerMCP::GetCapabilities() {
   lldb_protocol::mcp::Capabilities capabilities;
   capabilities.tools.listChanged = true;
@@ -263,158 +182,3 @@ lldb_protocol::mcp::Capabilities ProtocolServerMCP::GetCapabilities() {
   capabilities.resources.listChanged = false;
   return capabilities;
 }
-
-void ProtocolServerMCP::AddTool(std::unique_ptr<Tool> tool) {
-  std::lock_guard<std::mutex> guard(m_server_mutex);
-
-  if (!tool)
-    return;
-  m_tools[tool->GetName()] = std::move(tool);
-}
-
-void ProtocolServerMCP::AddResourceProvider(
-    std::unique_ptr<ResourceProvider> resource_provider) {
-  std::lock_guard<std::mutex> guard(m_server_mutex);
-
-  if (!resource_provider)
-    return;
-  m_resource_providers.push_back(std::move(resource_provider));
-}
-
-void ProtocolServerMCP::AddRequestHandler(llvm::StringRef method,
-                                          RequestHandler handler) {
-  std::lock_guard<std::mutex> guard(m_server_mutex);
-  m_request_handlers[method] = std::move(handler);
-}
-
-void ProtocolServerMCP::AddNotificationHandler(llvm::StringRef method,
-                                               NotificationHandler handler) {
-  std::lock_guard<std::mutex> guard(m_server_mutex);
-  m_notification_handlers[method] = std::move(handler);
-}
-
-llvm::Expected<lldb_protocol::mcp::Response>
-ProtocolServerMCP::InitializeHandler(
-    const lldb_protocol::mcp::Request &request) {
-  lldb_protocol::mcp::Response response;
-  response.result.emplace(llvm::json::Object{
-      {"protocolVersion", lldb_protocol::mcp::kVersion},
-      {"capabilities", GetCapabilities()},
-      {"serverInfo",
-       llvm::json::Object{{"name", kName}, {"version", kVersion}}}});
-  return response;
-}
-
-llvm::Expected<lldb_protocol::mcp::Response>
-ProtocolServerMCP::ToolsListHandler(
-    const lldb_protocol::mcp::Request &request) {
-  lldb_protocol::mcp::Response response;
-
-  llvm::json::Array tools;
-  for (const auto &tool : m_tools)
-    tools.emplace_back(toJSON(tool.second->GetDefinition()));
-
-  response.result.emplace(llvm::json::Object{{"tools", std::move(tools)}});
-
-  return response;
-}
-
-llvm::Expected<lldb_protocol::mcp::Response>
-ProtocolServerMCP::ToolsCallHandler(
-    const lldb_protocol::mcp::Request &request) {
-  lldb_protocol::mcp::Response response;
-
-  if (!request.params)
-    return llvm::createStringError("no tool parameters");
-
-  const json::Object *param_obj = request.params->getAsObject();
-  if (!param_obj)
-    return llvm::createStringError("no tool parameters");
-
-  const json::Value *name = param_obj->get("name");
-  if (!name)
-    return llvm::createStringError("no tool name");
-
-  llvm::StringRef tool_name = name->getAsString().value_or("");
-  if (tool_name.empty())
-    return llvm::createStringError("no tool name");
-
-  auto it = m_tools.find(tool_name);
-  if (it == m_tools.end())
-    return llvm::createStringError(llvm::formatv("no tool \"{0}\"", tool_name));
-
-  lldb_protocol::mcp::ToolArguments tool_args;
-  if (const json::Value *args = param_obj->get("arguments"))
-    tool_args = *args;
-
-  llvm::Expected<lldb_protocol::mcp::TextResult> text_result =
-      it->second->Call(tool_args);
-  if (!text_result)
-    return text_result.takeError();
-
-  response.result.emplace(toJSON(*text_result));
-
-  return response;
-}
-
-llvm::Expected<lldb_protocol::mcp::Response>
-ProtocolServerMCP::ResourcesListHandler(
-    const lldb_protocol::mcp::Request &request) {
-  lldb_protocol::mcp::Response response;
-
-  llvm::json::Array resources;
-
-  std::lock_guard<std::mutex> guard(m_server_mutex);
-  for (std::unique_ptr<ResourceProvider> &resource_provider_up :
-       m_resource_providers) {
-    for (const lldb_protocol::mcp::Resource &resource :
-         resource_provider_up->GetResources())
-      resources.push_back(resource);
-  }
-  response.result.emplace(
-      llvm::json::Object{{"resources", std::move(resources)}});
-
-  return response;
-}
-
-llvm::Expected<lldb_protocol::mcp::Response>
-ProtocolServerMCP::ResourcesReadHandler(
-    const lldb_protocol::mcp::Request &request) {
-  lldb_protocol::mcp::Response response;
-
-  if (!request.params)
-    return llvm::createStringError("no resource parameters");
-
-  const json::Object *param_obj = request.params->getAsObject();
-  if (!param_obj)
-    return llvm::createStringError("no resource parameters");
-
-  const json::Value *uri = param_obj->get("uri");
-  if (!uri)
-    return llvm::createStringError("no resource uri");
-
-  llvm::StringRef uri_str = uri->getAsString().value_or("");
-  if (uri_str.empty())
-    return llvm::createStringError("no resource uri");
-
-  std::lock_guard<std::mutex> guard(m_server_mutex);
-  for (std::unique_ptr<ResourceProvider> &resource_provider_up :
-       m_resource_providers) {
-    llvm::Expected<lldb_protocol::mcp::ResourceResult> result =
-        resource_provider_up->ReadResource(uri_str);
-    if (result.errorIsA<UnsupportedURI>()) {
-      llvm::consumeError(result.takeError());
-      continue;
-    }
-    if (!result)
-      return result.takeError();
-
-    lldb_protocol::mcp::Response response;
-    response.result.emplace(std::move(*result));
-    return response;
-  }
-
-  return make_error<MCPError>(
-      llvm::formatv("no resource handler for uri: {0}", uri_str).str(),
-      MCPError::kResourceNotFound);
-}
diff --git a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h
index 2ea9585a2334b..7fe909a728b85 100644
--- a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h
+++ b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h
@@ -13,14 +13,13 @@
 #include "lldb/Host/MainLoop.h"
 #include "lldb/Host/Socket.h"
 #include "lldb/Protocol/MCP/Protocol.h"
-#include "lldb/Protocol/MCP/Resource.h"
-#include "lldb/Protocol/MCP/Tool.h"
-#include "llvm/ADT/StringMap.h"
+#include "lldb/Protocol/MCP/Server.h"
 #include <thread>
 
 namespace lldb_private::mcp {
 
-class ProtocolServerMCP : public ProtocolServer {
+class ProtocolServerMCP : public ProtocolServer,
+                          public lldb_protocol::mcp::Server {
 public:
   ProtocolServerMCP();
   virtual ~ProtocolServerMCP() override;
@@ -40,48 +39,10 @@ class ProtocolServerMCP : public ProtocolServer {
 
   Socket *GetSocket() const override { return m_listener.get(); }
 
-protected:
-  using RequestHandler =
-      std::function<llvm::Expected<lldb_protocol::mcp::Response>(
-          const lldb_protocol::mcp::Request &)>;
-  using NotificationHandler =
-      std::function<void(const lldb_protocol::mcp::Notification &)>;
-
-  void AddTool(std::unique_ptr<lldb_protocol::mcp::Tool> tool);
-  void AddResourceProvider(
-      std::unique_ptr<lldb_protocol::mcp::ResourceProvider> resource_provider);
-
-  void AddRequestHandler(llvm::StringRef method, RequestHandler handler);
-  void AddNotificationHandler(llvm::StringRef method,
-                              NotificationHandler handler);
-
 private:
   void AcceptCallback(std::unique_ptr<Socket> socket);
 
-  llvm::Expected<std::optional<lldb_protocol::mcp::Message>>
-  HandleData(llvm::StringRef data);
-
-  llvm::Expected<lldb_protocol::mcp::Response>
-  Handle(lldb_protocol::mcp::Request request);
-  void Handle(lldb_protocol::mcp::Notification notification);
-
-  llvm::Expected<lldb_protocol::mcp::Response>
-  InitializeHandler(const lldb_protocol::mcp::Request &);
-
-  llvm::Expected<lldb_protocol::mcp::Response>
-  ToolsListHandler(const lldb_protocol::mcp::Request &);
-  llvm::Expected<lldb_protocol::mcp::Response>
-  ToolsCallHandler(const lldb_protocol::mcp::Request &);
-
-  llvm::Expected<lldb_protocol::mcp::Response>
-  ResourcesListHandler(const lldb_protocol::mcp::Request &);
-  llvm::Expected<lldb_protocol::mcp::Response>
-  ResourcesReadHandler(const lldb_protocol::mcp::Request &);
-
-  lldb_protocol::mcp::Capabilities GetCapabilities();
-
-  llvm::StringLiteral kName = "lldb-mcp";
-  llvm::StringLiteral kVersion = "0.1.0";
+  lldb_protocol::mcp::Capabilities GetCapabilities() override;
 
   bool m_running = false;
 
@@ -98,14 +59,6 @@ class ProtocolServerMCP : public ProtocolServer {
   };
   llvm::Error ReadCallback(Client &client);
   std::vector<std::unique_ptr<Client>> m_clients;
-
-  std::mutex m_server_mutex;
-  llvm::StringMap<std::unique_ptr<lldb_protocol::mcp::Tool>> m_tools;
-  std::vector<std::unique_ptr<lldb_protocol::mcp::ResourceProvider>>
-      m_resource_providers;
-
-  llvm::StringMap<RequestHandler> m_request_handlers;
-  llvm::StringMap<NotificationHandler> m_notification_handlers;
 };
 } // namespace lldb_private::mcp
 
diff --git a/lldb/source/Plugins/RegisterTypeBuilder/RegisterTypeBuilderClang.cpp b/lldb/source/Plugins/RegisterTypeBuilder/RegisterTypeBuilderClang.cpp
index f19dc8b1e6e58..eb9e013f4429a 100644
--- a/lldb/source/Plugins/RegisterTypeBuilder/RegisterTypeBuilderClang.cpp
+++ b/lldb/source/Plugins/RegisterTypeBuilder/RegisterTypeBuilderClang.cpp
@@ -47,7 +47,7 @@ CompilerType RegisterTypeBuilderClang::GetRegisterType(
   // See if we have made this type before and can reuse it.
   CompilerType fields_type =
       type_system->GetTypeForIdentifier<clang::CXXRecordDecl>(
-          register_type_name);
+          type_system->getASTContext(), register_type_name);
 
   if (!fields_type) {
     // In most ABI, a change of field type means a change in storage unit.
@@ -83,7 +83,7 @@ CompilerType RegisterTypeBuilderClang::GetRegisterType(
           // may have built this one already.
           CompilerType field_enum_type =
               type_system->GetTypeForIdentifier<clang::EnumDecl>(
-                  enum_type_name);
+                  type_system->getASTContext(), enum_type_name);
 
           if (field_enum_type)
             field_type = field_enum_type;
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
index 82aa022bbae0b..a2a287a6714db 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
@@ -71,24 +71,11 @@ Expected<std::string> python::As<std::string>(Expected<PythonObject> &&obj) {
   return std::string(utf8.get());
 }
 
-static bool python_is_finalizing() {
-#if PY_VERSION_HEX >= 0x030d0000
-  return Py_IsFinalizing();
-#else
-  return _Py_IsFinalizing();
-#endif
-}
-
 void PythonObject::Reset() {
   if (m_py_obj && Py_IsInitialized()) {
-    if (python_is_finalizing()) {
-      // Leak m_py_obj rather than crashing the process.
-      // https://docs.python.org/3/c-api/init.html#c.PyGILState_Ensure
-    } else {
-      PyGILState_STATE state = PyGILState_Ensure();
-      Py_DECREF(m_py_obj);
-      PyGILState_Release(state);
-    }
+    PyGILState_STATE state = PyGILState_Ensure();
+    Py_DECREF(m_py_obj);
+    PyGILState_Release(state);
   }
   m_py_obj = nullptr;
 }
@@ -418,15 +405,33 @@ Expected<llvm::StringRef> PythonString::AsUTF8() const {
   if (!IsValid())
     return nullDeref();
 
-  Py_ssize_t size;
-  const char *data;
+  // PyUnicode_AsUTF8AndSize caches the UTF-8 representation of the string in
+  // the Unicode object, which makes it more efficient and ties the lifetime of
+  // the data to the Python string. However, it was only added to the Stable API
+  // in Python 3.10. Older versions that want to use the Stable API must use
+  // PyUnicode_AsUTF8String in combination with ConstString.
+#if defined(Py_LIMITED_API) && (Py_LIMITED_API < 0x030a0000)
+  PyObject *py_bytes = PyUnicode_AsUTF8String(m_py_obj);
+  if (!py_bytes)
+    return exception();
+  auto release_py_str =
+      llvm::make_scope_exit([py_bytes] { Py_DECREF(py_bytes); });
+  Py_ssize_t size = PyBytes_Size(py_bytes);
+  const char *str = PyBytes_AsString(py_bytes);
+
+  if (!str)
+    return exception();
 
-  data = PyUnicode_AsUTF8AndSize(m_py_obj, &size);
+  return ConstString(str, size).GetStringRef();
+#else
+  Py_ssize_t size;
+  const char *str = PyUnicode_AsUTF8AndSize(m_py_obj, &size);
 
-  if (!data)
+  if (!str)
     return exception();
 
-  return llvm::StringRef(data, size);
+  return llvm::StringRef(str, size);
+#endif
 }
 
 size_t PythonString::GetSize() const {
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index 24d604f22a765..9330a634489a2 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -92,10 +92,23 @@ namespace {
 struct InitializePythonRAII {
 public:
   InitializePythonRAII() {
+    // The table of built-in modules can only be extended before Python is
+    // initialized.
+    if (!Py_IsInitialized()) {
+#ifdef LLDB_USE_LIBEDIT_READLINE_COMPAT_MODULE
+      // Python's readline is incompatible with libedit being linked into lldb.
+      // Provide a patched version local to the embedded interpreter.
+      PyImport_AppendInittab("readline", initlldb_readline);
+#endif
+
+      // Register _lldb as a built-in module.
+      PyImport_AppendInittab("_lldb", LLDBSwigPyInit);
+    }
+
+#if LLDB_EMBED_PYTHON_HOME
     PyConfig config;
     PyConfig_InitPythonConfig(&config);
 
-#if LLDB_EMBED_PYTHON_HOME
     static std::string g_python_home = []() -> std::string {
       if (llvm::sys::path::is_absolute(LLDB_PYTHON_HOME))
         return LLDB_PYTHON_HOME;
@@ -109,34 +122,13 @@ struct InitializePythonRAII {
     if (!g_python_home.empty()) {
       PyConfig_SetBytesString(&config, &config.home, g_python_home.c_str());
     }
-#endif
-
-    // The table of built-in modules can only be extended before Python is
-    // initialized.
-    if (!Py_IsInitialized()) {
-#ifdef LLDB_USE_LIBEDIT_READLINE_COMPAT_MODULE
-      // Python's readline is incompatible with libedit being linked into lldb.
-      // Provide a patched version local to the embedded interpreter.
-      bool ReadlinePatched = false;
-      for (auto *p = PyImport_Inittab; p->name != nullptr; p++) {
-        if (strcmp(p->name, "readline") == 0) {
-          p->initfunc = initlldb_readline;
-          break;
-        }
-      }
-      if (!ReadlinePatched) {
-        PyImport_AppendInittab("readline", initlldb_readline);
-        ReadlinePatched = true;
-      }
-#endif
-
-      // Register _lldb as a built-in module.
-      PyImport_AppendInittab("_lldb", LLDBSwigPyInit);
-    }
 
     config.install_signal_handlers = 0;
     Py_InitializeFromConfig(&config);
     PyConfig_Clear(&config);
+#else
+    Py_InitializeEx(/*install_sigs=*/0);
+#endif
 
     // The only case we should go further and acquire the GIL: it is unlocked.
     PyGILState_STATE gil_state = PyGILState_Ensure();
@@ -907,11 +899,11 @@ bool ScriptInterpreterPythonImpl::Interrupt() {
   Log *log = GetLog(LLDBLog::Script);
 
   if (IsExecutingPython()) {
-    PyThreadState *state = PyThreadState_GET();
+    PyThreadState *state = PyThreadState_Get();
     if (!state)
       state = GetThreadState();
     if (state) {
-      long tid = state->thread_id;
+      long tid = PyThread_get_thread_ident();
       PyThreadState_Swap(state);
       int num_threads = PyThreadState_SetAsyncExc(tid, PyExc_KeyboardInterrupt);
       LLDB_LOGF(log,
diff --git a/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp b/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp
index 867f6a6393bca..70093c9624f09 100644
--- a/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp
+++ b/lldb/source/Plugins/StructuredData/DarwinLog/StructuredDataDarwinLog.cpp
@@ -1601,6 +1601,7 @@ void StructuredDataDarwinLog::AddInitCompletionHook(Process &process) {
 
   const char *func_name = "_libtrace_init";
   const lldb::addr_t offset = 0;
+  const bool offset_is_insn_count = false;
   const LazyBool skip_prologue = eLazyBoolCalculate;
   // This is an internal breakpoint - the user shouldn't see it.
   const bool internal = true;
@@ -1608,7 +1609,8 @@ void StructuredDataDarwinLog::AddInitCompletionHook(Process &process) {
 
   auto breakpoint_sp = target.CreateBreakpoint(
       &module_spec_list, source_spec_list, func_name, eFunctionNameTypeFull,
-      eLanguageTypeC, offset, skip_prologue, internal, hardware);
+      eLanguageTypeC, offset, offset_is_insn_count, skip_prologue, internal,
+      hardware);
   if (!breakpoint_sp) {
     // Huh?  Bail here.
     LLDB_LOGF(log,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
index a9345c79bd2bb..1bfa17f08a3b9 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
@@ -44,7 +44,7 @@ class ElaboratingDIEIterator
   // Container sizes are optimized for the case of following DW_AT_specification
   // and DW_AT_abstract_origin just once.
   llvm::SmallVector<DWARFDIE, 2> m_worklist;
-  llvm::SmallSet<DWARFDebugInfoEntry *, 3> m_seen;
+  llvm::SmallPtrSet<DWARFDebugInfoEntry *, 3> m_seen;
 
   void Next() {
     assert(!m_worklist.empty() && "Incrementing end iterator?");
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
index f968eee091344..849f5738b62d7 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
@@ -284,9 +284,10 @@ bool DWARFDebugInfoEntry::GetDIENamesAndRanges(
 /// Adds all attributes of the DIE at the top of the \c worklist to the
 /// \c attributes list. Specifcations and abstract origins are added
 /// to the \c worklist if the referenced DIE has not been seen before.
-static bool GetAttributes(llvm::SmallVectorImpl<DWARFDIE> &worklist,
-                          llvm::SmallSet<DWARFDebugInfoEntry const *, 3> &seen,
-                          DWARFAttributes &attributes) {
+static bool
+GetAttributes(llvm::SmallVectorImpl<DWARFDIE> &worklist,
+              llvm::SmallPtrSet<DWARFDebugInfoEntry const *, 3> &seen,
+              DWARFAttributes &attributes) {
   assert(!worklist.empty() && "Need at least one DIE to visit.");
   assert(seen.size() >= 1 &&
          "Need to have seen at least the currently visited entry.");
@@ -366,7 +367,7 @@ DWARFAttributes DWARFDebugInfoEntry::GetAttributes(const DWARFUnit *cu,
   // Keep track if DIEs already seen to prevent infinite recursion.
   // Value of '3' was picked for the same reason that
   // DWARFDie::findRecursively does.
-  llvm::SmallSet<DWARFDebugInfoEntry const *, 3> seen;
+  llvm::SmallPtrSet<DWARFDebugInfoEntry const *, 3> seen;
   seen.insert(this);
 
   do {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 84b3da37367c4..b15e0c15fedb8 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -8,6 +8,7 @@
 
 #include "SymbolFileDWARF.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/DebugInfo/DWARF/DWARFAddressRange.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
 #include "llvm/Support/Casting.h"
@@ -998,12 +999,12 @@ XcodeSDK SymbolFileDWARF::ParseXcodeSDK(CompileUnit &comp_unit) {
   const char *sdk = cu_die.GetAttributeValueAsString(DW_AT_APPLE_sdk, nullptr);
   if (!sdk)
     return {};
-  std::string sysroot =
+  llvm::StringRef sysroot =
       cu_die.GetAttributeValueAsString(DW_AT_LLVM_sysroot, "");
 
   // RegisterXcodeSDK calls into xcrun which is not aware of CLT, which is
   // expensive.
-  if (sysroot.find("/Library/Developer/CommandLineTools/SDKs") != 0) {
+  if (!sysroot.starts_with("/Library/Developer/CommandLineTools/SDKs")) {
     // Register the sysroot path remapping with the module belonging to
     // the CU as well as the one belonging to the symbol file. The two
     // would be different if this is an OSO object and module is the
@@ -1017,7 +1018,7 @@ XcodeSDK SymbolFileDWARF::ParseXcodeSDK(CompileUnit &comp_unit) {
       local_module_sp->RegisterXcodeSDK(sdk, sysroot);
   }
 
-  return {sdk, FileSpec{std::move(sysroot)}};
+  return {sdk, FileSpec(sysroot)};
 }
 
 size_t SymbolFileDWARF::ParseFunctions(CompileUnit &comp_unit) {
@@ -2483,6 +2484,30 @@ bool SymbolFileDWARF::ResolveFunction(const DWARFDIE &orig_die,
   return false;
 }
 
+DWARFDIE
+SymbolFileDWARF::FindFunctionDefinition(const FunctionCallLabel &label) {
+  DWARFDIE definition;
+  Module::LookupInfo info(ConstString(label.lookup_name),
+                          lldb::eFunctionNameTypeFull,
+                          lldb::eLanguageTypeUnknown);
+
+  m_index->GetFunctions(info, *this, {}, [&](DWARFDIE entry) {
+    if (entry.GetAttributeValueAsUnsigned(llvm::dwarf::DW_AT_declaration, 0))
+      return IterationAction::Continue;
+
+    // We don't check whether the specification DIE for this function
+    // corresponds to the declaration DIE because the declaration might be in
+    // a type-unit but the definition in the compile-unit (and it's
+    // specifcation would point to the declaration in the compile-unit). We
+    // rely on the mangled name within the module to be enough to find us the
+    // unique definition.
+    definition = entry;
+    return IterationAction::Stop;
+  });
+
+  return definition;
+}
+
 llvm::Expected<SymbolContext>
 SymbolFileDWARF::ResolveFunctionCallLabel(const FunctionCallLabel &label) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
@@ -2495,37 +2520,19 @@ SymbolFileDWARF::ResolveFunctionCallLabel(const FunctionCallLabel &label) {
   // Label was created using a declaration DIE. Need to fetch the definition
   // to resolve the function call.
   if (die.GetAttributeValueAsUnsigned(llvm::dwarf::DW_AT_declaration, 0)) {
-    Module::LookupInfo info(ConstString(label.lookup_name),
-                            lldb::eFunctionNameTypeFull,
-                            lldb::eLanguageTypeUnknown);
-
-    m_index->GetFunctions(info, *this, {}, [&](DWARFDIE entry) {
-      if (entry.GetAttributeValueAsUnsigned(llvm::dwarf::DW_AT_declaration, 0))
-        return IterationAction::Continue;
-
-      // We don't check whether the specification DIE for this function
-      // corresponds to the declaration DIE because the declaration might be in
-      // a type-unit but the definition in the compile-unit (and it's
-      // specifcation would point to the declaration in the compile-unit). We
-      // rely on the mangled name within the module to be enough to find us the
-      // unique definition.
-      die = entry;
-      return IterationAction::Stop;
-    });
+    auto definition = FindFunctionDefinition(label);
+    if (!definition)
+      return llvm::createStringError("failed to find definition DIE");
 
-    if (die.GetAttributeValueAsUnsigned(llvm::dwarf::DW_AT_declaration, 0))
-      return llvm::createStringError(
-          llvm::formatv("failed to find definition DIE for {0}", label));
+    die = std::move(definition);
   }
 
   SymbolContextList sc_list;
   if (!ResolveFunction(die, /*include_inlines=*/false, sc_list))
-    return llvm::createStringError(
-        llvm::formatv("failed to resolve function for {0}", label));
+    return llvm::createStringError("failed to resolve function");
 
   if (sc_list.IsEmpty())
-    return llvm::createStringError(
-        llvm::formatv("failed to find function for {0}", label));
+    return llvm::createStringError("failed to find function");
 
   assert(sc_list.GetSize() == 1);
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index 5042d919d9d42..d7db8a3c0869f 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -373,6 +373,13 @@ class SymbolFileDWARF : public SymbolFileCommon {
   /// Returns the DWARFIndex for this symbol, if it exists.
   DWARFIndex *getIndex() { return m_index.get(); }
 
+private:
+  /// Find the definition DIE for the specified \c label in this
+  /// SymbolFile.
+  ///
+  /// \returns A valid definition DIE on success.
+  DWARFDIE FindFunctionDefinition(const FunctionCallLabel &label);
+
 protected:
   SymbolFileDWARF(const SymbolFileDWARF &) = delete;
   const SymbolFileDWARF &operator=(const SymbolFileDWARF &) = delete;
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
index f01fba3c48ce9..709281cb32709 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/PdbAstBuilder.cpp
@@ -449,7 +449,7 @@ bool PdbAstBuilder::CompleteTagDecl(clang::TagDecl &tag) {
                         ->GetIndex();
   lldbassert(IsTagRecord(type_id, index.tpi()));
 
-  clang::QualType tag_qt = m_clang.getASTContext().getTypeDeclType(&tag);
+  clang::QualType tag_qt = m_clang.getASTContext().getCanonicalTagType(&tag);
   TypeSystemClang::SetHasExternalStorage(tag_qt.getAsOpaquePtr(), false);
 
   TypeIndex tag_ti = type_id.index;
@@ -562,7 +562,8 @@ clang::QualType PdbAstBuilder::CreatePointerType(const PointerRecord &pointer) {
           m_clang.getASTContext(), spelling));
     }
     return m_clang.getASTContext().getMemberPointerType(
-        pointee_type, /*Qualifier=*/nullptr, class_type->getAsCXXRecordDecl());
+        pointee_type, /*Qualifier=*/std::nullopt,
+        class_type->getAsCXXRecordDecl());
   }
 
   clang::QualType pointer_type;
@@ -862,9 +863,9 @@ PdbAstBuilder::CreateFunctionDecl(PdbCompilandSymId func_id,
     SymbolFileNativePDB *pdb = static_cast<SymbolFileNativePDB *>(
         m_clang.GetSymbolFile()->GetBackingSymbolFile());
     PdbIndex &index = pdb->GetIndex();
-    clang::QualType parent_qt = llvm::cast<clang::TypeDecl>(parent)
-                                    ->getTypeForDecl()
-                                    ->getCanonicalTypeInternal();
+    clang::CanQualType parent_qt =
+        m_clang.getASTContext().getCanonicalTypeDeclType(
+            llvm::cast<clang::TypeDecl>(parent));
     lldb::opaque_compiler_type_t parent_opaque_ty =
         ToCompilerType(parent_qt).GetOpaqueQualType();
     // FIXME: Remove this workaround.
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
index dcea33dd9f854..112eb06e462fc 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
@@ -39,6 +39,7 @@
 #include "llvm/DebugInfo/PDB/Native/ModuleDebugStream.h"
 #include "llvm/DebugInfo/PDB/Native/NativeSession.h"
 #include "llvm/DebugInfo/PDB/Native/PDBFile.h"
+#include "llvm/DebugInfo/PDB/Native/PublicsStream.h"
 #include "llvm/DebugInfo/PDB/Native/SymbolStream.h"
 #include "llvm/DebugInfo/PDB/Native/TpiStream.h"
 #include "llvm/DebugInfo/PDB/PDB.h"
@@ -643,8 +644,14 @@ SymbolFileNativePDB::CreateClassStructUnion(PdbTypeSymId type_id,
 
   std::string uname = GetUnqualifiedTypeName(record);
 
-  // FIXME: Search IPI stream for LF_UDT_MOD_SRC_LINE.
+  llvm::Expected<Declaration> maybeDecl = ResolveUdtDeclaration(type_id);
   Declaration decl;
+  if (maybeDecl)
+    decl = std::move(*maybeDecl);
+  else
+    LLDB_LOG_ERROR(GetLog(LLDBLog::Symbols), maybeDecl.takeError(),
+                   "Failed to resolve declaration for '{1}': {0}", uname);
+
   return MakeType(toOpaqueUid(type_id), ConstString(uname), size, nullptr,
                   LLDB_INVALID_UID, Type::eEncodingIsUID, decl, ct,
                   Type::ResolveState::Forward);
@@ -667,7 +674,14 @@ lldb::TypeSP SymbolFileNativePDB::CreateTagType(PdbTypeSymId type_id,
                                                 CompilerType ct) {
   std::string uname = GetUnqualifiedTypeName(er);
 
+  llvm::Expected<Declaration> maybeDecl = ResolveUdtDeclaration(type_id);
   Declaration decl;
+  if (maybeDecl)
+    decl = std::move(*maybeDecl);
+  else
+    LLDB_LOG_ERROR(GetLog(LLDBLog::Symbols), maybeDecl.takeError(),
+                   "Failed to resolve declaration for '{1}': {0}", uname);
+
   TypeSP underlying_type = GetOrCreateType(er.UnderlyingType);
 
   return MakeType(
@@ -1641,6 +1655,94 @@ void SymbolFileNativePDB::DumpClangAST(Stream &s, llvm::StringRef filter) {
   clang->GetNativePDBParser()->Dump(s, filter);
 }
 
+void SymbolFileNativePDB::CacheFunctionNames() {
+  if (!m_func_full_names.IsEmpty())
+    return;
+
+  // (segment, code offset) -> gid
+  std::map<std::pair<uint16_t, uint32_t>, uint32_t> addr_ids;
+
+  // First, find all function references in the globals table.
+  for (const uint32_t gid : m_index->globals().getGlobalsTable()) {
+    CVSymbol ref_sym = m_index->symrecords().readRecord(gid);
+    auto kind = ref_sym.kind();
+    if (kind != S_PROCREF && kind != S_LPROCREF)
+      continue;
+
+    ProcRefSym ref =
+        llvm::cantFail(SymbolDeserializer::deserializeAs<ProcRefSym>(ref_sym));
+    if (ref.Name.empty())
+      continue;
+
+    // Find the function this is referencing.
+    CompilandIndexItem &cci =
+        m_index->compilands().GetOrCreateCompiland(ref.modi());
+    auto iter = cci.m_debug_stream.getSymbolArray().at(ref.SymOffset);
+    if (iter == cci.m_debug_stream.getSymbolArray().end())
+      continue;
+    kind = iter->kind();
+    if (kind != S_GPROC32 && kind != S_LPROC32)
+      continue;
+
+    ProcSym proc =
+        llvm::cantFail(SymbolDeserializer::deserializeAs<ProcSym>(*iter));
+    if ((proc.Flags & ProcSymFlags::IsUnreachable) != ProcSymFlags::None)
+      continue;
+    if (proc.Name.empty() || proc.FunctionType.isSimple())
+      continue;
+
+    // The function/procedure symbol only contains the demangled name.
+    // The mangled names are in the publics table. Save the address of this
+    // function to lookup the mangled name later.
+    addr_ids.emplace(std::make_pair(proc.Segment, proc.CodeOffset), gid);
+
+    llvm::StringRef basename = MSVCUndecoratedNameParser::DropScope(proc.Name);
+    if (basename.empty())
+      basename = proc.Name;
+
+    m_func_base_names.Append(ConstString(basename), gid);
+    m_func_full_names.Append(ConstString(proc.Name), gid);
+
+    // To see if this is a member function, check the type.
+    auto type = m_index->tpi().getType(proc.FunctionType);
+    if (type.kind() == LF_MFUNCTION) {
+      MemberFunctionRecord mfr;
+      llvm::cantFail(
+          TypeDeserializer::deserializeAs<MemberFunctionRecord>(type, mfr));
+      if (!mfr.getThisType().isNoneType())
+        m_func_method_names.Append(ConstString(basename), gid);
+    }
+  }
+
+  // The publics stream contains all mangled function names and their address.
+  for (auto pid : m_index->publics().getPublicsTable()) {
+    PdbGlobalSymId global{pid, true};
+    CVSymbol sym = m_index->ReadSymbolRecord(global);
+    auto kind = sym.kind();
+    if (kind != S_PUB32)
+      continue;
+    PublicSym32 pub =
+        llvm::cantFail(SymbolDeserializer::deserializeAs<PublicSym32>(sym));
+    // We only care about mangled names - if the name isn't mangled, it's
+    // already in the full name map.
+    if (!Mangled::IsMangledName(pub.Name))
+      continue;
+
+    // Check if this symbol is for one of our functions.
+    auto it = addr_ids.find({pub.Segment, pub.Offset});
+    if (it != addr_ids.end())
+      m_func_full_names.Append(ConstString(pub.Name), it->second);
+  }
+
+  // Sort them before value searching is working properly.
+  m_func_full_names.Sort();
+  m_func_full_names.SizeToFit();
+  m_func_method_names.Sort();
+  m_func_method_names.SizeToFit();
+  m_func_base_names.Sort();
+  m_func_base_names.SizeToFit();
+}
+
 void SymbolFileNativePDB::FindGlobalVariables(
     ConstString name, const CompilerDeclContext &parent_decl_ctx,
     uint32_t max_matches, VariableList &variables) {
@@ -1677,34 +1779,60 @@ void SymbolFileNativePDB::FindFunctions(
   if (name_type_mask & eFunctionNameTypeFull)
     name = lookup_info.GetName();
 
-  // For now we only support lookup by method name or full name.
   if (!(name_type_mask & eFunctionNameTypeFull ||
+        name_type_mask & eFunctionNameTypeBase ||
         name_type_mask & eFunctionNameTypeMethod))
     return;
+  CacheFunctionNames();
 
-  using SymbolAndOffset = std::pair<uint32_t, llvm::codeview::CVSymbol>;
+  std::set<uint32_t> resolved_ids; // avoid duplicate lookups
+  auto resolve_from = [&](UniqueCStringMap<uint32_t> &Names) {
+    std::vector<uint32_t> ids;
+    if (!Names.GetValues(name, ids))
+      return;
 
-  std::vector<SymbolAndOffset> matches = m_index->globals().findRecordsByName(
-      name.GetStringRef(), m_index->symrecords());
-  for (const SymbolAndOffset &match : matches) {
-    if (match.second.kind() != S_PROCREF && match.second.kind() != S_LPROCREF)
-      continue;
-    ProcRefSym proc(match.second.kind());
-    cantFail(SymbolDeserializer::deserializeAs<ProcRefSym>(match.second, proc));
+    for (uint32_t id : ids) {
+      if (!resolved_ids.insert(id).second)
+        continue;
 
-    if (!IsValidRecord(proc))
-      continue;
+      PdbGlobalSymId global{id, false};
+      if (parent_decl_ctx.IsValid() &&
+          GetDeclContextContainingUID(toOpaqueUid(global)) != parent_decl_ctx)
+        continue;
 
-    CompilandIndexItem &cci =
-        m_index->compilands().GetOrCreateCompiland(proc.modi());
-    SymbolContext sc;
+      CVSymbol sym = m_index->ReadSymbolRecord(global);
+      auto kind = sym.kind();
+      lldbassert(kind == S_PROCREF || kind == S_LPROCREF);
 
-    sc.comp_unit = GetOrCreateCompileUnit(cci).get();
-    PdbCompilandSymId func_id(proc.modi(), proc.SymOffset);
-    sc.function = GetOrCreateFunction(func_id, *sc.comp_unit).get();
+      ProcRefSym proc =
+          cantFail(SymbolDeserializer::deserializeAs<ProcRefSym>(sym));
 
-    sc_list.Append(sc);
-  }
+      if (!IsValidRecord(proc))
+        continue;
+
+      CompilandIndexItem &cci =
+          m_index->compilands().GetOrCreateCompiland(proc.modi());
+      SymbolContext sc;
+
+      sc.comp_unit = GetOrCreateCompileUnit(cci).get();
+      if (!sc.comp_unit)
+        continue;
+
+      PdbCompilandSymId func_id(proc.modi(), proc.SymOffset);
+      sc.function = GetOrCreateFunction(func_id, *sc.comp_unit).get();
+      if (!sc.function)
+        continue;
+
+      sc_list.Append(sc);
+    }
+  };
+
+  if (name_type_mask & eFunctionNameTypeFull)
+    resolve_from(m_func_full_names);
+  if (name_type_mask & eFunctionNameTypeBase)
+    resolve_from(m_func_base_names);
+  if (name_type_mask & eFunctionNameTypeMethod)
+    resolve_from(m_func_method_names);
 }
 
 void SymbolFileNativePDB::FindFunctions(const RegularExpression &regex,
@@ -2441,3 +2569,70 @@ SymbolFileNativePDB::GetContextForType(TypeIndex ti) {
   }
   return ctx;
 }
+
+void SymbolFileNativePDB::CacheUdtDeclarations() {
+  for (CVType cvt : m_index->ipi().typeArray()) {
+    switch (cvt.kind()) {
+    case LF_UDT_SRC_LINE: {
+      UdtSourceLineRecord udt_src;
+      llvm::cantFail(TypeDeserializer::deserializeAs(cvt, udt_src));
+      m_udt_declarations.try_emplace(
+          udt_src.UDT, UdtDeclaration{/*FileNameIndex=*/udt_src.SourceFile,
+                                      /*IsIpiIndex=*/true,
+                                      /*Line=*/udt_src.LineNumber});
+    } break;
+    case LF_UDT_MOD_SRC_LINE: {
+      UdtModSourceLineRecord udt_mod_src;
+      llvm::cantFail(TypeDeserializer::deserializeAs(cvt, udt_mod_src));
+      // Some types might be contributed by multiple modules. We assume that
+      // they all point to the same file and line because we can only provide
+      // one location.
+      m_udt_declarations.try_emplace(
+          udt_mod_src.UDT,
+          UdtDeclaration{/*FileNameIndex=*/udt_mod_src.SourceFile,
+                         /*IsIpiIndex=*/false,
+                         /*Line=*/udt_mod_src.LineNumber});
+    } break;
+    default:
+      break;
+    }
+  }
+}
+
+llvm::Expected<Declaration>
+SymbolFileNativePDB::ResolveUdtDeclaration(PdbTypeSymId type_id) {
+  std::call_once(m_cached_udt_declarations, [this] { CacheUdtDeclarations(); });
+
+  auto it = m_udt_declarations.find(type_id.index);
+  if (it == m_udt_declarations.end())
+    return llvm::createStringError("No UDT declaration found");
+
+  llvm::StringRef file_name;
+  if (it->second.IsIpiIndex) {
+    CVType cvt = m_index->ipi().getType(it->second.FileNameIndex);
+    if (cvt.kind() != LF_STRING_ID)
+      return llvm::createStringError("File name was not a LF_STRING_ID");
+
+    StringIdRecord sid;
+    llvm::cantFail(TypeDeserializer::deserializeAs(cvt, sid));
+    file_name = sid.String;
+  } else {
+    // The file name index is an index into the string table
+    auto string_table = m_index->pdb().getStringTable();
+    if (!string_table)
+      return string_table.takeError();
+
+    llvm::Expected<llvm::StringRef> string =
+        string_table->getStringTable().getString(
+            it->second.FileNameIndex.getIndex());
+    if (!string)
+      return string.takeError();
+    file_name = *string;
+  }
+
+  // rustc sets the filename to "<unknown>" for some files
+  if (file_name == "\\<unknown>")
+    return Declaration();
+
+  return Declaration(FileSpec(file_name), it->second.Line);
+}
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
index eda375d4cebe7..cfa00416d9673 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
@@ -260,6 +260,11 @@ class SymbolFileNativePDB : public SymbolFileCommon {
 
   std::vector<CompilerContext> GetContextForType(llvm::codeview::TypeIndex ti);
 
+  void CacheFunctionNames();
+
+  void CacheUdtDeclarations();
+  llvm::Expected<Declaration> ResolveUdtDeclaration(PdbTypeSymId type_id);
+
   llvm::BumpPtrAllocator m_allocator;
 
   lldb::addr_t m_obj_load_address = 0;
@@ -281,7 +286,26 @@ class SymbolFileNativePDB : public SymbolFileCommon {
   llvm::DenseMap<llvm::codeview::TypeIndex, llvm::codeview::TypeIndex>
       m_parent_types;
 
+  struct UdtDeclaration {
+    /// This could either be an index into the `/names` section (string table,
+    /// LF_UDT_MOD_SRC_LINE) or, this could be an index into the IPI stream to a
+    /// LF_STRING_ID record (LF_UDT_SRC_LINE).
+    llvm::codeview::TypeIndex FileNameIndex;
+    bool IsIpiIndex;
+
+    uint32_t Line;
+  };
+  llvm::DenseMap<llvm::codeview::TypeIndex, UdtDeclaration> m_udt_declarations;
+  std::once_flag m_cached_udt_declarations;
+
   lldb_private::UniqueCStringMap<uint32_t> m_type_base_names;
+
+  /// mangled name/full function name -> Global ID(s)
+  lldb_private::UniqueCStringMap<uint32_t> m_func_full_names;
+  /// basename -> Global ID(s)
+  lldb_private::UniqueCStringMap<uint32_t> m_func_base_names;
+  /// method basename -> Global ID(s)
+  lldb_private::UniqueCStringMap<uint32_t> m_func_method_names;
 };
 
 } // namespace npdb
diff --git a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
index 8b8eac6e67c01..3a95588326f04 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/PDBASTParser.cpp
@@ -407,8 +407,8 @@ lldb::TypeSP PDBASTParser::CreateLLDBTypeFromPDBType(const PDBSymbol &type) {
     // symbols in PDB for types with const or volatile modifiers, but we need
     // to create only one declaration for them all.
     Type::ResolveState type_resolve_state;
-    CompilerType clang_type =
-        m_ast.GetTypeForIdentifier<clang::CXXRecordDecl>(name, decl_context);
+    CompilerType clang_type = m_ast.GetTypeForIdentifier<clang::CXXRecordDecl>(
+        m_ast.getASTContext(), name, decl_context);
     if (!clang_type.IsValid()) {
       auto access = GetAccessibilityForUdt(*udt);
 
@@ -479,8 +479,8 @@ lldb::TypeSP PDBASTParser::CreateLLDBTypeFromPDBType(const PDBSymbol &type) {
     uint64_t bytes = enum_type->getLength();
 
     // Check if such an enum already exists in the current context
-    CompilerType ast_enum =
-        m_ast.GetTypeForIdentifier<clang::EnumDecl>(name, decl_context);
+    CompilerType ast_enum = m_ast.GetTypeForIdentifier<clang::EnumDecl>(
+        m_ast.getASTContext(), name, decl_context);
     if (!ast_enum.IsValid()) {
       auto underlying_type_up = enum_type->getUnderlyingType();
       if (!underlying_type_up)
@@ -557,7 +557,8 @@ lldb::TypeSP PDBASTParser::CreateLLDBTypeFromPDBType(const PDBSymbol &type) {
 
     // Check if such a typedef already exists in the current context
     CompilerType ast_typedef =
-        m_ast.GetTypeForIdentifier<clang::TypedefNameDecl>(name, decl_ctx);
+        m_ast.GetTypeForIdentifier<clang::TypedefNameDecl>(
+            m_ast.getASTContext(), name, decl_ctx);
     if (!ast_typedef.IsValid()) {
       CompilerType target_ast_type = target_type->GetFullCompilerType();
 
diff --git a/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp b/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp
index b23f64210cc80..04a25e4d35d54 100644
--- a/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp
+++ b/lldb/source/Plugins/SystemRuntime/MacOSX/SystemRuntimeMacOSX.cpp
@@ -544,9 +544,9 @@ ThreadSP SystemRuntimeMacOSX::GetExtendedBacktraceThread(ThreadSP real_thread,
     if (!thread_extended_info->ForEach(extract_frame_pc))
       return {};
 
-    originating_thread_sp =
-        std::make_shared<HistoryThread>(*m_process, real_thread->GetIndexID(),
-                                        app_specific_backtrace_pcs, true);
+    originating_thread_sp = std::make_shared<HistoryThread>(
+        *m_process, real_thread->GetIndexID(), app_specific_backtrace_pcs,
+        HistoryPCType::Calls);
     originating_thread_sp->SetQueueName(type.AsCString());
   }
   return originating_thread_sp;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 6addf4fc9239e..39aacdb58e694 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -27,6 +27,7 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Mangle.h"
+#include "clang/AST/QualTypeNames.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/VTableBuilder.h"
@@ -161,8 +162,7 @@ void addOverridesForMethod(clang::CXXMethodDecl *decl) {
   auto find_overridden_methods =
       [&decls, decl](const clang::CXXBaseSpecifier *specifier,
                      clang::CXXBasePath &path) {
-        if (auto *base_record = llvm::dyn_cast<clang::CXXRecordDecl>(
-                specifier->getType()->castAs<clang::RecordType>()->getDecl())) {
+        if (auto *base_record = specifier->getType()->getAsCXXRecordDecl()) {
 
           clang::DeclarationName name = decl->getDeclName();
 
@@ -1179,7 +1179,7 @@ CompilerType TypeSystemClang::GetTypeForDecl(clang::NamedDecl *decl) {
 }
 
 CompilerType TypeSystemClang::GetTypeForDecl(TagDecl *decl) {
-  return GetType(getASTContext().getTagDeclType(decl));
+  return GetType(getASTContext().getCanonicalTagType(decl));
 }
 
 CompilerType TypeSystemClang::GetTypeForDecl(ObjCInterfaceDecl *decl) {
@@ -1306,7 +1306,7 @@ CompilerType TypeSystemClang::CreateRecordType(
   if (decl_ctx)
     decl_ctx->addDecl(decl);
 
-  return GetType(ast.getTagDeclType(decl));
+  return GetType(ast.getCanonicalTagType(decl));
 }
 
 namespace {
@@ -1674,7 +1674,6 @@ TypeSystemClang::CreateClassTemplateSpecializationDecl(
   class_template_specialization_decl->setInstantiationOf(class_template_decl);
   class_template_specialization_decl->setTemplateArgs(
       TemplateArgumentList::CreateCopy(ast, args));
-  ast.getTypeDeclType(class_template_specialization_decl, nullptr);
   class_template_specialization_decl->setDeclName(
       class_template_decl->getDeclName());
 
@@ -1696,7 +1695,7 @@ CompilerType TypeSystemClang::CreateClassTemplateSpecializationType(
     ClassTemplateSpecializationDecl *class_template_specialization_decl) {
   if (class_template_specialization_decl) {
     ASTContext &ast = getASTContext();
-    return GetType(ast.getTagDeclType(class_template_specialization_decl));
+    return GetType(ast.getCanonicalTagType(class_template_specialization_decl));
   }
   return CompilerType();
 }
@@ -1792,9 +1791,7 @@ bool TypeSystemClang::RecordHasFields(const RecordDecl *record_decl) {
     for (base_class = cxx_record_decl->bases_begin(),
         base_class_end = cxx_record_decl->bases_end();
          base_class != base_class_end; ++base_class) {
-      const CXXRecordDecl *base_class_decl = cast<CXXRecordDecl>(
-          base_class->getType()->getAs<RecordType>()->getDecl());
-      if (RecordHasFields(base_class_decl))
+      if (RecordHasFields(base_class->getType()->getAsCXXRecordDecl()))
         return true;
     }
   }
@@ -2290,9 +2287,9 @@ CompilerType TypeSystemClang::CreateStructForIdentifier(
         &type_fields,
     bool packed) {
   CompilerType type;
-  if (!type_name.empty() &&
-      (type = GetTypeForIdentifier<clang::CXXRecordDecl>(type_name))
-          .IsValid()) {
+  if (!type_name.empty() && (type = GetTypeForIdentifier<clang::CXXRecordDecl>(
+                                 getASTContext(), type_name))
+                                .IsValid()) {
     lldbassert(0 && "Trying to create a type for an existing name");
     return type;
   }
@@ -2316,7 +2313,9 @@ CompilerType TypeSystemClang::GetOrCreateStructForIdentifier(
         &type_fields,
     bool packed) {
   CompilerType type;
-  if ((type = GetTypeForIdentifier<clang::CXXRecordDecl>(type_name)).IsValid())
+  if ((type = GetTypeForIdentifier<clang::CXXRecordDecl>(getASTContext(),
+                                                         type_name))
+          .IsValid())
     return type;
 
   return CreateStructForIdentifier(type_name, type_fields, packed);
@@ -2355,7 +2354,7 @@ CompilerType TypeSystemClang::CreateEnumerationType(
 
   enum_decl->setAccess(AS_public); // TODO respect what's in the debug info
 
-  return GetType(ast.getTagDeclType(enum_decl));
+  return GetType(ast.getCanonicalTagType(enum_decl));
 }
 
 CompilerType TypeSystemClang::GetIntTypeFromBitSize(size_t bit_size,
@@ -2471,7 +2470,7 @@ bool TypeSystemClang::GetCompleteDecl(clang::ASTContext *ast,
 
     ast_source->CompleteType(tag_decl);
 
-    return !tag_decl->getTypeForDecl()->isIncompleteType();
+    return !ast->getCanonicalTagType(tag_decl)->isIncompleteType();
   } else if (clang::ObjCInterfaceDecl *objc_interface_decl =
                  llvm::dyn_cast<clang::ObjCInterfaceDecl>(decl)) {
     if (objc_interface_decl->getDefinition())
@@ -2575,7 +2574,6 @@ RemoveWrappingTypes(QualType type, ArrayRef<clang::Type::TypeClass> mask = {}) {
       break;
     case clang::Type::Auto:
     case clang::Type::Decltype:
-    case clang::Type::Elaborated:
     case clang::Type::Paren:
     case clang::Type::SubstTemplateTypeParm:
     case clang::Type::TemplateSpecialization:
@@ -2607,10 +2605,11 @@ TypeSystemClang::GetDeclContextForType(clang::QualType type) {
     return GetDeclContextForType(
         llvm::cast<clang::ObjCObjectPointerType>(qual_type.getTypePtr())
             ->getPointeeType());
-  case clang::Type::Record:
-    return llvm::cast<clang::RecordType>(qual_type)->getDecl();
   case clang::Type::Enum:
-    return llvm::cast<clang::EnumType>(qual_type)->getDecl();
+  case clang::Type::Record:
+    return llvm::cast<clang::TagType>(qual_type)
+        ->getOriginalDecl()
+        ->getDefinitionOrSelf();
   default:
     break;
   }
@@ -2790,7 +2789,7 @@ static bool GetCompleteQualType(clang::ASTContext *ast,
     if (ast->getTargetInfo().getCXXABI().isMicrosoft()) {
       auto *MPT = qual_type.getTypePtr()->castAs<clang::MemberPointerType>();
       if (auto *RD = MPT->getMostRecentCXXRecordDecl())
-        GetCompleteRecordType(ast, QualType(RD->getTypeForDecl(), 0),
+        GetCompleteRecordType(ast, ast->getCanonicalTagType(RD),
                               allow_completion);
 
       return !qual_type.getTypePtr()->isIncompleteType();
@@ -2859,7 +2858,8 @@ bool TypeSystemClang::IsAnonymousType(lldb::opaque_compiler_type_t type) {
     if (const clang::RecordType *record_type =
             llvm::dyn_cast_or_null<clang::RecordType>(
                 qual_type.getTypePtrOrNull())) {
-      if (const clang::RecordDecl *record_decl = record_type->getDecl()) {
+      if (const clang::RecordDecl *record_decl =
+              record_type->getOriginalDecl()) {
         return record_decl->isAnonymousStructOrUnion();
       }
     }
@@ -3099,8 +3099,8 @@ TypeSystemClang::IsHomogeneousAggregate(lldb::opaque_compiler_type_t type,
       const clang::RecordType *record_type =
           llvm::cast<clang::RecordType>(qual_type.getTypePtr());
       if (record_type) {
-        const clang::RecordDecl *record_decl = record_type->getDecl();
-        if (record_decl) {
+        if (const clang::RecordDecl *record_decl =
+                record_type->getOriginalDecl()->getDefinition()) {
           // We are looking for a structure that contains only floating point
           // types
           clang::RecordDecl::field_iterator field_pos,
@@ -3280,7 +3280,10 @@ bool TypeSystemClang::IsEnumerationType(lldb::opaque_compiler_type_t type,
         GetCanonicalQualType(type)->getCanonicalTypeInternal());
 
     if (enum_type) {
-      IsIntegerType(enum_type->getDecl()->getIntegerType().getAsOpaquePtr(),
+      IsIntegerType(enum_type->getOriginalDecl()
+                        ->getDefinitionOrSelf()
+                        ->getIntegerType()
+                        .getAsOpaquePtr(),
                     is_signed);
       return true;
     }
@@ -3505,8 +3508,7 @@ bool TypeSystemClang::IsDefined(lldb::opaque_compiler_type_t type) {
   const clang::TagType *tag_type =
       llvm::dyn_cast<clang::TagType>(qual_type.getTypePtr());
   if (tag_type) {
-    clang::TagDecl *tag_decl = tag_type->getDecl();
-    if (tag_decl)
+    if (clang::TagDecl *tag_decl = tag_type->getOriginalDecl()->getDefinition())
       return tag_decl->isCompleteDefinition();
     return false;
   } else {
@@ -3565,21 +3567,14 @@ bool TypeSystemClang::IsPolymorphicClass(lldb::opaque_compiler_type_t type) {
     switch (type_class) {
     case clang::Type::Record:
       if (GetCompleteType(type)) {
-        const clang::RecordType *record_type =
-            llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-        const clang::RecordDecl *record_decl = record_type->getDecl();
-        if (record_decl) {
-          const clang::CXXRecordDecl *cxx_record_decl =
-              llvm::dyn_cast<clang::CXXRecordDecl>(record_decl);
-          if (cxx_record_decl) {
-            // We can't just call is isPolymorphic() here because that just
-            // means the current class has virtual functions, it doesn't check
-            // if any inherited classes have virtual functions. The doc string
-            // in SBType::IsPolymorphicClass() says it is looking for both
-            // if the class has virtual methods or if any bases do, so this
-            // should be more correct.
-            return cxx_record_decl->isDynamicClass();
-          }
+        if (const auto *cxx_record_decl = qual_type->getAsCXXRecordDecl()) {
+          // We can't just call is isPolymorphic() here because that just
+          // means the current class has virtual functions, it doesn't check
+          // if any inherited classes have virtual functions. The doc string
+          // in SBType::IsPolymorphicClass() says it is looking for both
+          // if the class has virtual methods or if any bases do, so this
+          // should be more correct.
+          return cxx_record_decl->isDynamicClass();
         }
       }
       break;
@@ -3766,7 +3761,7 @@ bool TypeSystemClang::IsBeingDefined(lldb::opaque_compiler_type_t type) {
   clang::QualType qual_type(GetCanonicalQualType(type));
   const clang::TagType *tag_type = llvm::dyn_cast<clang::TagType>(qual_type);
   if (tag_type)
-    return tag_type->isBeingDefined();
+    return tag_type->getOriginalDecl()->isEntityBeingDefined();
   return false;
 }
 
@@ -3974,7 +3969,8 @@ TypeSystemClang::GetTypeInfo(lldb::opaque_compiler_type_t type,
     if (pointee_or_element_clang_type)
       pointee_or_element_clang_type->SetCompilerType(
           weak_from_this(), llvm::cast<clang::EnumType>(qual_type)
-                                ->getDecl()
+                                ->getOriginalDecl()
+                                ->getDefinitionOrSelf()
                                 ->getIntegerType()
                                 .getAsOpaquePtr());
     return eTypeIsEnumeration | eTypeHasValue;
@@ -4154,7 +4150,6 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
   case clang::Type::Auto:
   case clang::Type::CountAttributed:
   case clang::Type::Decltype:
-  case clang::Type::Elaborated:
   case clang::Type::Paren:
   case clang::Type::TypeOf:
   case clang::Type::TypeOfExpr:
@@ -4214,7 +4209,7 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
   case clang::Type::Record: {
     const clang::RecordType *record_type =
         llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-    const clang::RecordDecl *record_decl = record_type->getDecl();
+    const clang::RecordDecl *record_decl = record_type->getOriginalDecl();
     if (record_decl->isUnion())
       return lldb::eTypeClassUnion;
     else if (record_decl->isStruct())
@@ -4280,6 +4275,8 @@ TypeSystemClang::GetTypeClass(lldb::opaque_compiler_type_t type) {
     break;
   case clang::Type::HLSLInlineSpirv:
     break;
+  case clang::Type::SubstBuiltinTemplatePack:
+    break;
   }
   // We don't know hot to display this type...
   return lldb::eTypeClassOther;
@@ -4412,17 +4409,10 @@ TypeSystemClang::GetNumMemberFunctions(lldb::opaque_compiler_type_t type) {
     clang::QualType qual_type = RemoveWrappingTypes(GetCanonicalQualType(type));
     switch (qual_type->getTypeClass()) {
     case clang::Type::Record:
-      if (GetCompleteQualType(&getASTContext(), qual_type)) {
-        const clang::RecordType *record_type =
-            llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-        const clang::RecordDecl *record_decl = record_type->getDecl();
-        assert(record_decl);
-        const clang::CXXRecordDecl *cxx_record_decl =
-            llvm::dyn_cast<clang::CXXRecordDecl>(record_decl);
-        if (cxx_record_decl)
+      if (GetCompleteQualType(&getASTContext(), qual_type))
+        if (const auto *cxx_record_decl = qual_type->getAsCXXRecordDecl())
           num_functions = std::distance(cxx_record_decl->method_begin(),
                                         cxx_record_decl->method_end());
-      }
       break;
 
     case clang::Type::ObjCObjectPointer: {
@@ -4477,13 +4467,7 @@ TypeSystemClang::GetMemberFunctionAtIndex(lldb::opaque_compiler_type_t type,
     switch (qual_type->getTypeClass()) {
     case clang::Type::Record:
       if (GetCompleteQualType(&getASTContext(), qual_type)) {
-        const clang::RecordType *record_type =
-            llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-        const clang::RecordDecl *record_decl = record_type->getDecl();
-        assert(record_decl);
-        const clang::CXXRecordDecl *cxx_record_decl =
-            llvm::dyn_cast<clang::CXXRecordDecl>(record_decl);
-        if (cxx_record_decl) {
+        if (const auto *cxx_record_decl = qual_type->getAsCXXRecordDecl()) {
           auto method_iter = cxx_record_decl->method_begin();
           auto method_end = cxx_record_decl->method_end();
           if (idx <
@@ -4703,9 +4687,9 @@ CompilerType TypeSystemClang::CreateTypedef(
     clang::TagDecl *tdecl = nullptr;
     if (!qual_type.isNull()) {
       if (const clang::RecordType *rt = qual_type->getAs<clang::RecordType>())
-        tdecl = rt->getDecl();
+        tdecl = rt->getOriginalDecl();
       if (const clang::EnumType *et = qual_type->getAs<clang::EnumType>())
-        tdecl = et->getDecl();
+        tdecl = et->getOriginalDecl();
     }
 
     // Check whether this declaration is an anonymous struct, union, or enum,
@@ -4717,7 +4701,10 @@ CompilerType TypeSystemClang::CreateTypedef(
     decl->setAccess(clang::AS_public); // TODO respect proper access specifier
 
     // Get a uniqued clang::QualType for the typedef decl type
-    return GetType(clang_ast.getTypedefType(decl));
+    NestedNameSpecifier Qualifier =
+        clang::TypeName::getFullyQualifiedDeclaredContext(clang_ast, decl);
+    return GetType(
+        clang_ast.getTypedefType(ElaboratedTypeKeyword::None, Qualifier, decl));
   }
   return CompilerType();
 }
@@ -4869,7 +4856,6 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type,
   case clang::Type::Auto:
   case clang::Type::CountAttributed:
   case clang::Type::Decltype:
-  case clang::Type::Elaborated:
   case clang::Type::Paren:
   case clang::Type::Typedef:
   case clang::Type::TypeOf:
@@ -5155,6 +5141,8 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type,
     break;
   case clang::Type::HLSLInlineSpirv:
     break;
+  case clang::Type::SubstBuiltinTemplatePack:
+    break;
   }
   count = 0;
   return lldb::eEncodingInvalid;
@@ -5171,7 +5159,6 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
   case clang::Type::Auto:
   case clang::Type::CountAttributed:
   case clang::Type::Decltype:
-  case clang::Type::Elaborated:
   case clang::Type::Paren:
   case clang::Type::Typedef:
   case clang::Type::TypeOf:
@@ -5324,6 +5311,8 @@ lldb::Format TypeSystemClang::GetFormat(lldb::opaque_compiler_type_t type) {
     break;
   case clang::Type::HLSLInlineSpirv:
     break;
+  case clang::Type::SubstBuiltinTemplatePack:
+    break;
   }
   // We don't know hot to display this type...
   return lldb::eFormatBytes;
@@ -5380,8 +5369,8 @@ TypeSystemClang::GetNumChildren(lldb::opaque_compiler_type_t type,
     if (GetCompleteQualType(&getASTContext(), qual_type)) {
       const clang::RecordType *record_type =
           llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-      const clang::RecordDecl *record_decl = record_type->getDecl();
-      assert(record_decl);
+      const clang::RecordDecl *record_decl =
+          record_type->getOriginalDecl()->getDefinitionOrSelf();
       const clang::CXXRecordDecl *cxx_record_decl =
           llvm::dyn_cast<clang::CXXRecordDecl>(record_decl);
 
@@ -5577,7 +5566,8 @@ void TypeSystemClang::ForEachEnumerator(
   const clang::EnumType *enum_type =
       llvm::dyn_cast<clang::EnumType>(GetCanonicalQualType(type));
   if (enum_type) {
-    const clang::EnumDecl *enum_decl = enum_type->getDecl();
+    const clang::EnumDecl *enum_decl =
+        enum_type->getOriginalDecl()->getDefinitionOrSelf();
     if (enum_decl) {
       CompilerType integer_type = GetType(enum_decl->getIntegerType());
 
@@ -5608,7 +5598,8 @@ uint32_t TypeSystemClang::GetNumFields(lldb::opaque_compiler_type_t type) {
       const clang::RecordType *record_type =
           llvm::dyn_cast<clang::RecordType>(qual_type.getTypePtr());
       if (record_type) {
-        clang::RecordDecl *record_decl = record_type->getDecl();
+        clang::RecordDecl *record_decl =
+            record_type->getOriginalDecl()->getDefinition();
         if (record_decl) {
           count = std::distance(record_decl->field_begin(),
                                 record_decl->field_end());
@@ -5722,7 +5713,8 @@ CompilerType TypeSystemClang::GetFieldAtIndex(lldb::opaque_compiler_type_t type,
     if (GetCompleteType(type)) {
       const clang::RecordType *record_type =
           llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-      const clang::RecordDecl *record_decl = record_type->getDecl();
+      const clang::RecordDecl *record_decl =
+          record_type->getOriginalDecl()->getDefinitionOrSelf();
       uint32_t field_idx = 0;
       clang::RecordDecl::field_iterator field, field_end;
       for (field = record_decl->field_begin(),
@@ -5908,7 +5900,7 @@ CompilerType TypeSystemClang::GetDirectBaseClassAtIndex(
                   llvm::cast<clang::CXXRecordDecl>(
                       base_class->getType()
                           ->castAs<clang::RecordType>()
-                          ->getDecl());
+                          ->getOriginalDecl());
               if (base_class->isVirtual())
                 *bit_offset_ptr =
                     record_layout.getVBaseClassOffset(base_class_decl)
@@ -6003,7 +5995,7 @@ CompilerType TypeSystemClang::GetVirtualBaseClassAtIndex(
                   llvm::cast<clang::CXXRecordDecl>(
                       base_class->getType()
                           ->castAs<clang::RecordType>()
-                          ->getDecl());
+                          ->getOriginalDecl());
               *bit_offset_ptr =
                   record_layout.getVBaseClassOffset(base_class_decl)
                       .getQuantity() *
@@ -6033,7 +6025,8 @@ TypeSystemClang::GetStaticFieldWithName(lldb::opaque_compiler_type_t type,
 
     const clang::RecordType *record_type =
         llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-    const clang::RecordDecl *record_decl = record_type->getDecl();
+    const clang::RecordDecl *record_decl =
+        record_type->getOriginalDecl()->getDefinitionOrSelf();
 
     clang::DeclarationName decl_name(&getASTContext().Idents.get(name));
     for (NamedDecl *decl : record_decl->lookup(decl_name)) {
@@ -6263,8 +6256,8 @@ llvm::Expected<CompilerType> TypeSystemClang::GetChildCompilerTypeAtIndex(
     if (idx_is_valid && GetCompleteType(type)) {
       const clang::RecordType *record_type =
           llvm::cast<clang::RecordType>(parent_qual_type.getTypePtr());
-      const clang::RecordDecl *record_decl = record_type->getDecl();
-      assert(record_decl);
+      const clang::RecordDecl *record_decl =
+          record_type->getOriginalDecl()->getDefinitionOrSelf();
       const clang::ASTRecordLayout &record_layout =
           getASTContext().getASTRecordLayout(record_decl);
       uint32_t child_idx = 0;
@@ -6283,7 +6276,10 @@ llvm::Expected<CompilerType> TypeSystemClang::GetChildCompilerTypeAtIndex(
           // Skip empty base classes
           if (omit_empty_base_classes) {
             base_class_decl = llvm::cast<clang::CXXRecordDecl>(
-                base_class->getType()->getAs<clang::RecordType>()->getDecl());
+                                  base_class->getType()
+                                      ->getAs<clang::RecordType>()
+                                      ->getOriginalDecl())
+                                  ->getDefinitionOrSelf();
             if (!TypeSystemClang::RecordHasFields(base_class_decl))
               continue;
           }
@@ -6291,7 +6287,10 @@ llvm::Expected<CompilerType> TypeSystemClang::GetChildCompilerTypeAtIndex(
           if (idx == child_idx) {
             if (base_class_decl == nullptr)
               base_class_decl = llvm::cast<clang::CXXRecordDecl>(
-                  base_class->getType()->getAs<clang::RecordType>()->getDecl());
+                                    base_class->getType()
+                                        ->getAs<clang::RecordType>()
+                                        ->getOriginalDecl())
+                                    ->getDefinitionOrSelf();
 
             if (base_class->isVirtual()) {
               bool handled = false;
@@ -6752,7 +6751,8 @@ size_t TypeSystemClang::GetIndexOfChildMemberWithName(
       if (GetCompleteType(type)) {
         const clang::RecordType *record_type =
             llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-        const clang::RecordDecl *record_decl = record_type->getDecl();
+        const clang::RecordDecl *record_decl =
+            record_type->getOriginalDecl()->getDefinitionOrSelf();
 
         assert(record_decl);
         uint32_t child_idx = 0;
@@ -6817,10 +6817,10 @@ size_t TypeSystemClang::GetIndexOfChildMemberWithName(
                   return 0;
                 } else {
                   child_indexes.push_back(child_idx);
-                  parent_record_decl = llvm::cast<clang::RecordDecl>(
-                      elem.Base->getType()
-                          ->castAs<clang::RecordType>()
-                          ->getDecl());
+                  parent_record_decl = elem.Base->getType()
+                                           ->castAs<clang::RecordType>()
+                                           ->getOriginalDecl()
+                                           ->getDefinitionOrSelf();
                 }
               }
               for (clang::DeclContext::lookup_iterator I = path->Decls, E;
@@ -6954,7 +6954,8 @@ TypeSystemClang::GetIndexOfChildWithName(lldb::opaque_compiler_type_t type,
       if (GetCompleteType(type)) {
         const clang::RecordType *record_type =
             llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-        const clang::RecordDecl *record_decl = record_type->getDecl();
+        const clang::RecordDecl *record_decl =
+            record_type->getOriginalDecl()->getDefinitionOrSelf();
 
         assert(record_decl);
         uint32_t child_idx = 0;
@@ -6973,7 +6974,8 @@ TypeSystemClang::GetIndexOfChildWithName(lldb::opaque_compiler_type_t type,
                 llvm::cast<clang::CXXRecordDecl>(
                     base_class->getType()
                         ->castAs<clang::RecordType>()
-                        ->getDecl());
+                        ->getOriginalDecl())
+                    ->getDefinitionOrSelf();
             if (omit_empty_base_classes &&
                 !TypeSystemClang::RecordHasFields(base_class_decl))
               continue;
@@ -7092,14 +7094,17 @@ TypeSystemClang::GetDirectNestedTypeWithName(lldb::opaque_compiler_type_t type,
       return CompilerType();
     const clang::RecordType *record_type =
         llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-    const clang::RecordDecl *record_decl = record_type->getDecl();
+    const clang::RecordDecl *record_decl =
+        record_type->getOriginalDecl()->getDefinitionOrSelf();
 
     clang::DeclarationName decl_name(&getASTContext().Idents.get(name));
     for (NamedDecl *decl : record_decl->lookup(decl_name)) {
       if (auto *tag_decl = dyn_cast<clang::TagDecl>(decl))
-        return GetType(getASTContext().getTagDeclType(tag_decl));
+        return GetType(getASTContext().getCanonicalTagType(tag_decl));
       if (auto *typedef_decl = dyn_cast<clang::TypedefNameDecl>(decl))
-        return GetType(getASTContext().getTypedefType(typedef_decl));
+        return GetType(getASTContext().getTypedefType(
+            ElaboratedTypeKeyword::None, /*Qualifier=*/std::nullopt,
+            typedef_decl));
     }
     break;
   }
@@ -7116,7 +7121,7 @@ bool TypeSystemClang::IsTemplateType(lldb::opaque_compiler_type_t type) {
   const clang::Type *clang_type = ClangUtil::GetQualType(ct).getTypePtr();
   if (auto *cxx_record_decl = dyn_cast<clang::TagType>(clang_type))
     return isa<clang::ClassTemplateSpecializationDecl>(
-        cxx_record_decl->getDecl());
+        cxx_record_decl->getOriginalDecl());
   return false;
 }
 
@@ -7319,7 +7324,7 @@ clang::EnumDecl *TypeSystemClang::GetAsEnumDecl(const CompilerType &type) {
   const clang::EnumType *enutype =
       llvm::dyn_cast<clang::EnumType>(ClangUtil::GetCanonicalQualType(type));
   if (enutype)
-    return enutype->getDecl();
+    return enutype->getOriginalDecl()->getDefinitionOrSelf();
   return nullptr;
 }
 
@@ -7327,7 +7332,7 @@ clang::RecordDecl *TypeSystemClang::GetAsRecordDecl(const CompilerType &type) {
   const clang::RecordType *record_type =
       llvm::dyn_cast<clang::RecordType>(ClangUtil::GetCanonicalQualType(type));
   if (record_type)
-    return record_type->getDecl();
+    return record_type->getOriginalDecl()->getDefinitionOrSelf();
   return nullptr;
 }
 
@@ -7409,7 +7414,7 @@ clang::FieldDecl *TypeSystemClang::AddFieldToRecordType(
       if (const clang::TagType *TagT =
               field->getType()->getAs<clang::TagType>()) {
         if (clang::RecordDecl *Rec =
-                llvm::dyn_cast<clang::RecordDecl>(TagT->getDecl()))
+                llvm::dyn_cast<clang::RecordDecl>(TagT->getOriginalDecl()))
           if (!Rec->getDeclName()) {
             Rec->setAnonymousStructOrUnion(true);
             field->setImplicit();
@@ -7494,7 +7499,8 @@ void TypeSystemClang::BuildIndirectFields(const CompilerType &type) {
       if (!field_record_type)
         continue;
 
-      clang::RecordDecl *field_record_decl = field_record_type->getDecl();
+      clang::RecordDecl *field_record_decl =
+          field_record_type->getOriginalDecl()->getDefinition();
 
       if (!field_record_decl)
         continue;
@@ -7636,7 +7642,8 @@ void TypeSystemClang::SetIntegerInitializerForVariable(
   // If the variable is an enum type, take the underlying integer type as
   // the type of the integer literal.
   if (const EnumType *enum_type = qt->getAs<EnumType>()) {
-    const EnumDecl *enum_decl = enum_type->getDecl();
+    const EnumDecl *enum_decl =
+        enum_type->getOriginalDecl()->getDefinitionOrSelf();
     qt = enum_decl->getIntegerType();
   }
   // Bools are handled separately because the clang AST printer handles bools
@@ -8296,7 +8303,7 @@ bool TypeSystemClang::SetHasExternalStorage(lldb::opaque_compiler_type_t type,
 
   case clang::Type::Enum: {
     clang::EnumDecl *enum_decl =
-        llvm::cast<clang::EnumType>(qual_type)->getDecl();
+        llvm::cast<clang::EnumType>(qual_type)->getOriginalDecl();
     if (enum_decl) {
       enum_decl->setHasExternalLexicalStorage(has_extern);
       enum_decl->setHasExternalVisibleStorage(has_extern);
@@ -8334,7 +8341,7 @@ bool TypeSystemClang::StartTagDeclarationDefinition(const CompilerType &type) {
   if (!qual_type.isNull()) {
     const clang::TagType *tag_type = qual_type->getAs<clang::TagType>();
     if (tag_type) {
-      clang::TagDecl *tag_decl = tag_type->getDecl();
+      clang::TagDecl *tag_decl = tag_type->getOriginalDecl();
       if (tag_decl) {
         tag_decl->startDefinition();
         return true;
@@ -8369,7 +8376,8 @@ bool TypeSystemClang::CompleteTagDeclarationDefinition(
   // the definition.
   const clang::TagType *tag_type = qual_type->getAs<clang::TagType>();
   if (tag_type) {
-    clang::TagDecl *tag_decl = tag_type->getDecl();
+    clang::TagDecl *tag_decl =
+        tag_type->getOriginalDecl()->getDefinitionOrSelf();
 
     if (auto *cxx_record_decl = llvm::dyn_cast<CXXRecordDecl>(tag_decl)) {
       // If we have a move constructor declared but no copy constructor we
@@ -8404,7 +8412,8 @@ bool TypeSystemClang::CompleteTagDeclarationDefinition(
 
   if (!enutype)
     return false;
-  clang::EnumDecl *enum_decl = enutype->getDecl();
+  clang::EnumDecl *enum_decl =
+      enutype->getOriginalDecl()->getDefinitionOrSelf();
 
   if (enum_decl->isCompleteDefinition())
     return true;
@@ -8462,17 +8471,19 @@ clang::EnumConstantDecl *TypeSystemClang::AddEnumerationValueToEnumerationType(
   clang::EnumConstantDecl *enumerator_decl =
       clang::EnumConstantDecl::CreateDeserialized(getASTContext(),
                                                   GlobalDeclID());
-  enumerator_decl->setDeclContext(enutype->getDecl());
+  clang::EnumDecl *enum_decl =
+      enutype->getOriginalDecl()->getDefinitionOrSelf();
+  enumerator_decl->setDeclContext(enum_decl);
   if (name && name[0])
     enumerator_decl->setDeclName(&getASTContext().Idents.get(name));
   enumerator_decl->setType(clang::QualType(enutype, 0));
   enumerator_decl->setInitVal(getASTContext(), value);
-  SetMemberOwningModule(enumerator_decl, enutype->getDecl());
+  SetMemberOwningModule(enumerator_decl, enum_decl);
 
   if (!enumerator_decl)
     return nullptr;
 
-  enutype->getDecl()->addDecl(enumerator_decl);
+  enum_decl->addDecl(enumerator_decl);
 
   VerifyDecl(enumerator_decl);
   return enumerator_decl;
@@ -8496,7 +8507,8 @@ CompilerType TypeSystemClang::GetEnumerationIntegerType(CompilerType type) {
   if (!enum_type)
     return CompilerType();
 
-  return GetType(enum_type->getDecl()->getIntegerType());
+  return GetType(
+      enum_type->getOriginalDecl()->getDefinitionOrSelf()->getIntegerType());
 }
 
 CompilerType
@@ -8509,7 +8521,7 @@ TypeSystemClang::CreateMemberPointerType(const CompilerType &type,
       return CompilerType();
     return ast->GetType(ast->getASTContext().getMemberPointerType(
         ClangUtil::GetQualType(pointee_type),
-        /*Qualifier=*/nullptr,
+        /*Qualifier=*/std::nullopt,
         ClangUtil::GetQualType(type)->getAsCXXRecordDecl()));
   }
   return CompilerType();
@@ -8587,8 +8599,8 @@ static bool DumpEnumValue(const clang::QualType &qual_type, Stream &s,
                           uint32_t bitfield_bit_size) {
   const clang::EnumType *enutype =
       llvm::cast<clang::EnumType>(qual_type.getTypePtr());
-  const clang::EnumDecl *enum_decl = enutype->getDecl();
-  assert(enum_decl);
+  const clang::EnumDecl *enum_decl =
+      enutype->getOriginalDecl()->getDefinitionOrSelf();
   lldb::offset_t offset = byte_offset;
   bool qual_type_is_signed = qual_type->isSignedIntegerOrEnumerationType();
   const uint64_t enum_svalue =
@@ -8694,15 +8706,7 @@ bool TypeSystemClang::DumpTypeValue(
   } else {
     clang::QualType qual_type(GetQualType(type));
 
-    const clang::Type::TypeClass type_class = qual_type->getTypeClass();
-
-    if (type_class == clang::Type::Elaborated) {
-      qual_type = llvm::cast<clang::ElaboratedType>(qual_type)->getNamedType();
-      return DumpTypeValue(qual_type.getAsOpaquePtr(), s, format, data, byte_offset, byte_size,
-                           bitfield_bit_size, bitfield_bit_offset, exe_scope);
-    }
-
-    switch (type_class) {
+    switch (qual_type->getTypeClass()) {
     case clang::Type::Typedef: {
       clang::QualType typedef_qual_type =
           llvm::cast<clang::TypedefType>(qual_type)
@@ -8872,7 +8876,7 @@ void TypeSystemClang::DumpTypeDescription(lldb::opaque_compiler_type_t type,
       GetCompleteType(type);
 
       auto *record_type = llvm::cast<clang::RecordType>(qual_type.getTypePtr());
-      const clang::RecordDecl *record_decl = record_type->getDecl();
+      const clang::RecordDecl *record_decl = record_type->getOriginalDecl();
       if (level == eDescriptionLevelVerbose)
         record_decl->dump(llvm_ostrm);
       else {
@@ -8884,7 +8888,7 @@ void TypeSystemClang::DumpTypeDescription(lldb::opaque_compiler_type_t type,
     default: {
       if (auto *tag_type =
               llvm::dyn_cast<clang::TagType>(qual_type.getTypePtr())) {
-        if (clang::TagDecl *tag_decl = tag_type->getDecl()) {
+        if (clang::TagDecl *tag_decl = tag_type->getOriginalDecl()) {
           if (level == eDescriptionLevelVerbose)
             tag_decl->dump(llvm_ostrm);
           else
@@ -8924,7 +8928,7 @@ void TypeSystemClang::DumpTypeName(const CompilerType &type) {
 
     case clang::Type::Enum: {
       clang::EnumDecl *enum_decl =
-          llvm::cast<clang::EnumType>(qual_type)->getDecl();
+          llvm::cast<clang::EnumType>(qual_type)->getOriginalDecl();
       if (enum_decl) {
         printf("enum %s", enum_decl->getName().str().c_str());
       }
@@ -8960,13 +8964,6 @@ void TypeSystemClang::DumpTypeName(const CompilerType &type) {
                                            ->getDeducedType()
                                            .getAsOpaquePtr()));
 
-    case clang::Type::Elaborated:
-      printf("elaborated ");
-      return DumpTypeName(CompilerType(
-          type.GetTypeSystem(), llvm::cast<clang::ElaboratedType>(qual_type)
-                                    ->getNamedType()
-                                    .getAsOpaquePtr()));
-
     case clang::Type::Paren:
       printf("paren ");
       return DumpTypeName(CompilerType(
@@ -9796,8 +9793,8 @@ bool TypeSystemClang::IsForcefullyCompleted(lldb::opaque_compiler_type_t type) {
     const clang::RecordType *record_type =
         llvm::dyn_cast<clang::RecordType>(qual_type.getTypePtr());
     if (record_type) {
-      const clang::RecordDecl *record_decl = record_type->getDecl();
-      assert(record_decl);
+      const clang::RecordDecl *record_decl =
+          record_type->getOriginalDecl()->getDefinitionOrSelf();
       if (std::optional<ClangASTMetadata> metadata = GetMetadata(record_decl))
         return metadata->IsForcefullyCompleted();
     }
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 70d613d47f93a..709f89590ba3b 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -260,7 +260,7 @@ class TypeSystemClang : public TypeSystem {
 
   template <typename RecordDeclType>
   CompilerType
-  GetTypeForIdentifier(llvm::StringRef type_name,
+  GetTypeForIdentifier(const clang::ASTContext &Ctx, llvm::StringRef type_name,
                        clang::DeclContext *decl_context = nullptr) {
     CompilerType compiler_type;
     if (type_name.empty())
@@ -278,11 +278,10 @@ class TypeSystemClang : public TypeSystem {
       return compiler_type;
 
     clang::NamedDecl *named_decl = *result.begin();
-    if (const RecordDeclType *record_decl =
-            llvm::dyn_cast<RecordDeclType>(named_decl))
+    if (const auto *type_decl = llvm::dyn_cast<clang::TypeDecl>(named_decl);
+        llvm::isa_and_nonnull<RecordDeclType>(type_decl))
       compiler_type = CompilerType(
-          weak_from_this(),
-          clang::QualType(record_decl->getTypeForDecl(), 0).getAsOpaquePtr());
+          weak_from_this(), Ctx.getTypeDeclType(type_decl).getAsOpaquePtr());
 
     return compiler_type;
   }
diff --git a/lldb/source/Protocol/MCP/CMakeLists.txt b/lldb/source/Protocol/MCP/CMakeLists.txt
index f1b1098e064a5..a73e7e6a7cab1 100644
--- a/lldb/source/Protocol/MCP/CMakeLists.txt
+++ b/lldb/source/Protocol/MCP/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_lldb_library(lldbProtocolMCP NO_PLUGIN_DEPENDENCIES
   MCPError.cpp
   Protocol.cpp
+  Server.cpp
   Tool.cpp
 
   LINK_COMPONENTS
diff --git a/lldb/source/Protocol/MCP/MCPError.cpp b/lldb/source/Protocol/MCP/MCPError.cpp
index c610e882abf51..e140d11e12cfe 100644
--- a/lldb/source/Protocol/MCP/MCPError.cpp
+++ b/lldb/source/Protocol/MCP/MCPError.cpp
@@ -25,10 +25,10 @@ std::error_code MCPError::convertToErrorCode() const {
   return llvm::inconvertibleErrorCode();
 }
 
-lldb_protocol::mcp::Error MCPError::toProtcolError() const {
+lldb_protocol::mcp::Error MCPError::toProtocolError() const {
   lldb_protocol::mcp::Error error;
-  error.error.code = m_error_code;
-  error.error.message = m_message;
+  error.code = m_error_code;
+  error.message = m_message;
   return error;
 }
 
diff --git a/lldb/source/Protocol/MCP/Protocol.cpp b/lldb/source/Protocol/MCP/Protocol.cpp
index d579b88037e63..65ddfaee70160 100644
--- a/lldb/source/Protocol/MCP/Protocol.cpp
+++ b/lldb/source/Protocol/MCP/Protocol.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Protocol/MCP/Protocol.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/JSON.h"
 
 using namespace llvm;
@@ -26,8 +27,45 @@ static bool mapRaw(const json::Value &Params, StringLiteral Prop,
   return true;
 }
 
+static llvm::json::Value toJSON(const Id &Id) {
+  if (const int64_t *I = std::get_if<int64_t>(&Id))
+    return json::Value(*I);
+  if (const std::string *S = std::get_if<std::string>(&Id))
+    return json::Value(*S);
+  llvm_unreachable("unexpected type in protocol::Id");
+}
+
+static bool mapId(const llvm::json::Value &V, StringLiteral Prop, Id &Id,
+                  llvm::json::Path P) {
+  const auto *O = V.getAsObject();
+  if (!O) {
+    P.report("expected object");
+    return false;
+  }
+
+  const auto *E = O->get(Prop);
+  if (!E) {
+    P.field(Prop).report("not found");
+    return false;
+  }
+
+  if (auto S = E->getAsString()) {
+    Id = S->str();
+    return true;
+  }
+
+  if (auto I = E->getAsInteger()) {
+    Id = *I;
+    return true;
+  }
+
+  P.report("expected string or number");
+  return false;
+}
+
 llvm::json::Value toJSON(const Request &R) {
-  json::Object Result{{"jsonrpc", "2.0"}, {"id", R.id}, {"method", R.method}};
+  json::Object Result{
+      {"jsonrpc", "2.0"}, {"id", toJSON(R.id)}, {"method", R.method}};
   if (R.params)
     Result.insert({"params", R.params});
   return Result;
@@ -35,47 +73,75 @@ llvm::json::Value toJSON(const Request &R) {
 
 bool fromJSON(const llvm::json::Value &V, Request &R, llvm::json::Path P) {
   llvm::json::ObjectMapper O(V, P);
-  if (!O || !O.map("id", R.id) || !O.map("method", R.method))
-    return false;
-  return mapRaw(V, "params", R.params, P);
-}
-
-llvm::json::Value toJSON(const ErrorInfo &EI) {
-  llvm::json::Object Result{{"code", EI.code}, {"message", EI.message}};
-  if (!EI.data.empty())
-    Result.insert({"data", EI.data});
-  return Result;
+  return O && mapId(V, "id", R.id, P) && O.map("method", R.method) &&
+         mapRaw(V, "params", R.params, P);
 }
 
-bool fromJSON(const llvm::json::Value &V, ErrorInfo &EI, llvm::json::Path P) {
-  llvm::json::ObjectMapper O(V, P);
-  return O && O.map("code", EI.code) && O.map("message", EI.message) &&
-         O.mapOptional("data", EI.data);
+bool operator==(const Request &a, const Request &b) {
+  return a.id == b.id && a.method == b.method && a.params == b.params;
 }
 
 llvm::json::Value toJSON(const Error &E) {
-  return json::Object{{"jsonrpc", "2.0"}, {"id", E.id}, {"error", E.error}};
+  llvm::json::Object Result{{"code", E.code}, {"message", E.message}};
+  if (E.data)
+    Result.insert({"data", *E.data});
+  return Result;
 }
 
 bool fromJSON(const llvm::json::Value &V, Error &E, llvm::json::Path P) {
   llvm::json::ObjectMapper O(V, P);
-  return O && O.map("id", E.id) && O.map("error", E.error);
+  return O && O.map("code", E.code) && O.map("message", E.message) &&
+         mapRaw(V, "data", E.data, P);
+}
+
+bool operator==(const Error &a, const Error &b) {
+  return a.code == b.code && a.message == b.message && a.data == b.data;
 }
 
 llvm::json::Value toJSON(const Response &R) {
-  llvm::json::Object Result{{"jsonrpc", "2.0"}, {"id", R.id}};
-  if (R.result)
-    Result.insert({"result", R.result});
-  if (R.error)
-    Result.insert({"error", R.error});
+  llvm::json::Object Result{{"jsonrpc", "2.0"}, {"id", toJSON(R.id)}};
+
+  if (const Error *error = std::get_if<Error>(&R.result))
+    Result.insert({"error", *error});
+  if (const json::Value *result = std::get_if<json::Value>(&R.result))
+    Result.insert({"result", *result});
   return Result;
 }
 
 bool fromJSON(const llvm::json::Value &V, Response &R, llvm::json::Path P) {
-  llvm::json::ObjectMapper O(V, P);
-  if (!O || !O.map("id", R.id) || !O.map("error", R.error))
+  const json::Object *E = V.getAsObject();
+  if (!E) {
+    P.report("expected object");
+    return false;
+  }
+
+  const json::Value *result = E->get("result");
+  const json::Value *raw_error = E->get("error");
+
+  if (result && raw_error) {
+    P.report("'result' and 'error' fields are mutually exclusive");
     return false;
-  return mapRaw(V, "result", R.result, P);
+  }
+
+  if (!result && !raw_error) {
+    P.report("'result' or 'error' fields are required'");
+    return false;
+  }
+
+  if (result) {
+    R.result = std::move(*result);
+  } else {
+    Error error;
+    if (!fromJSON(*raw_error, error, P))
+      return false;
+    R.result = std::move(error);
+  }
+
+  return mapId(V, "id", R.id, P);
+}
+
+bool operator==(const Response &a, const Response &b) {
+  return a.id == b.id && a.result == b.result;
 }
 
 llvm::json::Value toJSON(const Notification &N) {
@@ -97,6 +163,10 @@ bool fromJSON(const llvm::json::Value &V, Notification &N, llvm::json::Path P) {
   return true;
 }
 
+bool operator==(const Notification &a, const Notification &b) {
+  return a.method == b.method && a.params == b.params;
+}
+
 llvm::json::Value toJSON(const ToolCapability &TC) {
   return llvm::json::Object{{"listChanged", TC.listChanged}};
 }
@@ -235,24 +305,16 @@ bool fromJSON(const llvm::json::Value &V, Message &M, llvm::json::Path P) {
     return true;
   }
 
-  if (O->get("error")) {
-    Error E;
-    if (!fromJSON(V, E, P))
-      return false;
-    M = std::move(E);
-    return true;
-  }
-
-  if (O->get("result")) {
-    Response R;
+  if (O->get("method")) {
+    Request R;
     if (!fromJSON(V, R, P))
       return false;
     M = std::move(R);
     return true;
   }
 
-  if (O->get("method")) {
-    Request R;
+  if (O->get("result") || O->get("error")) {
+    Response R;
     if (!fromJSON(V, R, P))
       return false;
     M = std::move(R);
diff --git a/lldb/source/Protocol/MCP/Server.cpp b/lldb/source/Protocol/MCP/Server.cpp
new file mode 100644
index 0000000000000..a9c1482e3e378
--- /dev/null
+++ b/lldb/source/Protocol/MCP/Server.cpp
@@ -0,0 +1,234 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Protocol/MCP/Server.h"
+#include "lldb/Protocol/MCP/MCPError.h"
+
+using namespace lldb_protocol::mcp;
+using namespace llvm;
+
+Server::Server(std::string name, std::string version)
+    : m_name(std::move(name)), m_version(std::move(version)) {
+  AddRequestHandlers();
+}
+
+void Server::AddRequestHandlers() {
+  AddRequestHandler("initialize", std::bind(&Server::InitializeHandler, this,
+                                            std::placeholders::_1));
+  AddRequestHandler("tools/list", std::bind(&Server::ToolsListHandler, this,
+                                            std::placeholders::_1));
+  AddRequestHandler("tools/call", std::bind(&Server::ToolsCallHandler, this,
+                                            std::placeholders::_1));
+  AddRequestHandler("resources/list", std::bind(&Server::ResourcesListHandler,
+                                                this, std::placeholders::_1));
+  AddRequestHandler("resources/read", std::bind(&Server::ResourcesReadHandler,
+                                                this, std::placeholders::_1));
+}
+
+llvm::Expected<Response> Server::Handle(Request request) {
+  auto it = m_request_handlers.find(request.method);
+  if (it != m_request_handlers.end()) {
+    llvm::Expected<Response> response = it->second(request);
+    if (!response)
+      return response;
+    response->id = request.id;
+    return *response;
+  }
+
+  return llvm::make_error<MCPError>(
+      llvm::formatv("no handler for request: {0}", request.method).str());
+}
+
+void Server::Handle(Notification notification) {
+  auto it = m_notification_handlers.find(notification.method);
+  if (it != m_notification_handlers.end()) {
+    it->second(notification);
+    return;
+  }
+}
+
+llvm::Expected<std::optional<Message>>
+Server::HandleData(llvm::StringRef data) {
+  auto message = llvm::json::parse<Message>(/*JSON=*/data);
+  if (!message)
+    return message.takeError();
+
+  if (const Request *request = std::get_if<Request>(&(*message))) {
+    llvm::Expected<Response> response = Handle(*request);
+
+    // Handle failures by converting them into an Error message.
+    if (!response) {
+      Error protocol_error;
+      llvm::handleAllErrors(
+          response.takeError(),
+          [&](const MCPError &err) { protocol_error = err.toProtocolError(); },
+          [&](const llvm::ErrorInfoBase &err) {
+            protocol_error.code = MCPError::kInternalError;
+            protocol_error.message = err.message();
+          });
+      Response error_response;
+      error_response.id = request->id;
+      error_response.result = std::move(protocol_error);
+      return error_response;
+    }
+
+    return *response;
+  }
+
+  if (const Notification *notification =
+          std::get_if<Notification>(&(*message))) {
+    Handle(*notification);
+    return std::nullopt;
+  }
+
+  if (std::get_if<Response>(&(*message)))
+    return llvm::createStringError("unexpected MCP message: response");
+
+  llvm_unreachable("all message types handled");
+}
+
+void Server::AddTool(std::unique_ptr<Tool> tool) {
+  std::lock_guard<std::mutex> guard(m_mutex);
+
+  if (!tool)
+    return;
+  m_tools[tool->GetName()] = std::move(tool);
+}
+
+void Server::AddResourceProvider(
+    std::unique_ptr<ResourceProvider> resource_provider) {
+  std::lock_guard<std::mutex> guard(m_mutex);
+
+  if (!resource_provider)
+    return;
+  m_resource_providers.push_back(std::move(resource_provider));
+}
+
+void Server::AddRequestHandler(llvm::StringRef method, RequestHandler handler) {
+  std::lock_guard<std::mutex> guard(m_mutex);
+  m_request_handlers[method] = std::move(handler);
+}
+
+void Server::AddNotificationHandler(llvm::StringRef method,
+                                    NotificationHandler handler) {
+  std::lock_guard<std::mutex> guard(m_mutex);
+  m_notification_handlers[method] = std::move(handler);
+}
+
+llvm::Expected<Response> Server::InitializeHandler(const Request &request) {
+  Response response;
+  response.result = llvm::json::Object{
+      {"protocolVersion", mcp::kProtocolVersion},
+      {"capabilities", GetCapabilities()},
+      {"serverInfo",
+       llvm::json::Object{{"name", m_name}, {"version", m_version}}}};
+  return response;
+}
+
+llvm::Expected<Response> Server::ToolsListHandler(const Request &request) {
+  Response response;
+
+  llvm::json::Array tools;
+  for (const auto &tool : m_tools)
+    tools.emplace_back(toJSON(tool.second->GetDefinition()));
+
+  response.result = llvm::json::Object{{"tools", std::move(tools)}};
+
+  return response;
+}
+
+llvm::Expected<Response> Server::ToolsCallHandler(const Request &request) {
+  Response response;
+
+  if (!request.params)
+    return llvm::createStringError("no tool parameters");
+
+  const json::Object *param_obj = request.params->getAsObject();
+  if (!param_obj)
+    return llvm::createStringError("no tool parameters");
+
+  const json::Value *name = param_obj->get("name");
+  if (!name)
+    return llvm::createStringError("no tool name");
+
+  llvm::StringRef tool_name = name->getAsString().value_or("");
+  if (tool_name.empty())
+    return llvm::createStringError("no tool name");
+
+  auto it = m_tools.find(tool_name);
+  if (it == m_tools.end())
+    return llvm::createStringError(llvm::formatv("no tool \"{0}\"", tool_name));
+
+  ToolArguments tool_args;
+  if (const json::Value *args = param_obj->get("arguments"))
+    tool_args = *args;
+
+  llvm::Expected<TextResult> text_result = it->second->Call(tool_args);
+  if (!text_result)
+    return text_result.takeError();
+
+  response.result = toJSON(*text_result);
+
+  return response;
+}
+
+llvm::Expected<Response> Server::ResourcesListHandler(const Request &request) {
+  Response response;
+
+  llvm::json::Array resources;
+
+  std::lock_guard<std::mutex> guard(m_mutex);
+  for (std::unique_ptr<ResourceProvider> &resource_provider_up :
+       m_resource_providers) {
+    for (const Resource &resource : resource_provider_up->GetResources())
+      resources.push_back(resource);
+  }
+  response.result = llvm::json::Object{{"resources", std::move(resources)}};
+
+  return response;
+}
+
+llvm::Expected<Response> Server::ResourcesReadHandler(const Request &request) {
+  Response response;
+
+  if (!request.params)
+    return llvm::createStringError("no resource parameters");
+
+  const json::Object *param_obj = request.params->getAsObject();
+  if (!param_obj)
+    return llvm::createStringError("no resource parameters");
+
+  const json::Value *uri = param_obj->get("uri");
+  if (!uri)
+    return llvm::createStringError("no resource uri");
+
+  llvm::StringRef uri_str = uri->getAsString().value_or("");
+  if (uri_str.empty())
+    return llvm::createStringError("no resource uri");
+
+  std::lock_guard<std::mutex> guard(m_mutex);
+  for (std::unique_ptr<ResourceProvider> &resource_provider_up :
+       m_resource_providers) {
+    llvm::Expected<ResourceResult> result =
+        resource_provider_up->ReadResource(uri_str);
+    if (result.errorIsA<UnsupportedURI>()) {
+      llvm::consumeError(result.takeError());
+      continue;
+    }
+    if (!result)
+      return result.takeError();
+
+    Response response;
+    response.result = std::move(*result);
+    return response;
+  }
+
+  return make_error<MCPError>(
+      llvm::formatv("no resource handler for uri: {0}", uri_str).str(),
+      MCPError::kResourceNotFound);
+}
diff --git a/lldb/source/Symbol/ObjectFile.cpp b/lldb/source/Symbol/ObjectFile.cpp
index 21daf7476b522..7efce2a035505 100644
--- a/lldb/source/Symbol/ObjectFile.cpp
+++ b/lldb/source/Symbol/ObjectFile.cpp
@@ -379,6 +379,7 @@ AddressClass ObjectFile::GetAddressClass(addr_t file_addr) {
           case eSectionTypeELFDynamicSymbols:
           case eSectionTypeELFRelocationEntries:
           case eSectionTypeELFDynamicLinkInfo:
+          case eSectionTypeWasmName:
           case eSectionTypeOther:
             return AddressClass::eUnknown;
           case eSectionTypeAbsoluteAddress:
diff --git a/lldb/source/Symbol/Symbol.cpp b/lldb/source/Symbol/Symbol.cpp
index d6689a647062a..40497dbccc5c3 100644
--- a/lldb/source/Symbol/Symbol.cpp
+++ b/lldb/source/Symbol/Symbol.cpp
@@ -392,45 +392,8 @@ bool Symbol::Compare(ConstString name, SymbolType type) const {
   return false;
 }
 
-#define ENUM_TO_CSTRING(x)                                                     \
-  case eSymbolType##x:                                                         \
-    return #x;
-
 const char *Symbol::GetTypeAsString() const {
-  switch (m_type) {
-    ENUM_TO_CSTRING(Invalid);
-    ENUM_TO_CSTRING(Absolute);
-    ENUM_TO_CSTRING(Code);
-    ENUM_TO_CSTRING(Resolver);
-    ENUM_TO_CSTRING(Data);
-    ENUM_TO_CSTRING(Trampoline);
-    ENUM_TO_CSTRING(Runtime);
-    ENUM_TO_CSTRING(Exception);
-    ENUM_TO_CSTRING(SourceFile);
-    ENUM_TO_CSTRING(HeaderFile);
-    ENUM_TO_CSTRING(ObjectFile);
-    ENUM_TO_CSTRING(CommonBlock);
-    ENUM_TO_CSTRING(Block);
-    ENUM_TO_CSTRING(Local);
-    ENUM_TO_CSTRING(Param);
-    ENUM_TO_CSTRING(Variable);
-    ENUM_TO_CSTRING(VariableType);
-    ENUM_TO_CSTRING(LineEntry);
-    ENUM_TO_CSTRING(LineHeader);
-    ENUM_TO_CSTRING(ScopeBegin);
-    ENUM_TO_CSTRING(ScopeEnd);
-    ENUM_TO_CSTRING(Additional);
-    ENUM_TO_CSTRING(Compiler);
-    ENUM_TO_CSTRING(Instrumentation);
-    ENUM_TO_CSTRING(Undefined);
-    ENUM_TO_CSTRING(ObjCClass);
-    ENUM_TO_CSTRING(ObjCMetaClass);
-    ENUM_TO_CSTRING(ObjCIVar);
-    ENUM_TO_CSTRING(ReExported);
-  default:
-    break;
-  }
-  return "<unknown SymbolType>";
+  return GetTypeAsString(static_cast<lldb::SymbolType>(m_type));
 }
 
 void Symbol::CalculateSymbolContext(SymbolContext *sc) {
@@ -774,6 +737,79 @@ bool Symbol::operator==(const Symbol &rhs) const {
   return true;
 }
 
+#define ENUM_TO_CSTRING(x)                                                     \
+  case eSymbolType##x:                                                         \
+    return #x;
+
+const char *Symbol::GetTypeAsString(lldb::SymbolType symbol_type) {
+  switch (symbol_type) {
+    ENUM_TO_CSTRING(Invalid);
+    ENUM_TO_CSTRING(Absolute);
+    ENUM_TO_CSTRING(Code);
+    ENUM_TO_CSTRING(Resolver);
+    ENUM_TO_CSTRING(Data);
+    ENUM_TO_CSTRING(Trampoline);
+    ENUM_TO_CSTRING(Runtime);
+    ENUM_TO_CSTRING(Exception);
+    ENUM_TO_CSTRING(SourceFile);
+    ENUM_TO_CSTRING(HeaderFile);
+    ENUM_TO_CSTRING(ObjectFile);
+    ENUM_TO_CSTRING(CommonBlock);
+    ENUM_TO_CSTRING(Block);
+    ENUM_TO_CSTRING(Local);
+    ENUM_TO_CSTRING(Param);
+    ENUM_TO_CSTRING(Variable);
+    ENUM_TO_CSTRING(VariableType);
+    ENUM_TO_CSTRING(LineEntry);
+    ENUM_TO_CSTRING(LineHeader);
+    ENUM_TO_CSTRING(ScopeBegin);
+    ENUM_TO_CSTRING(ScopeEnd);
+    ENUM_TO_CSTRING(Additional);
+    ENUM_TO_CSTRING(Compiler);
+    ENUM_TO_CSTRING(Instrumentation);
+    ENUM_TO_CSTRING(Undefined);
+    ENUM_TO_CSTRING(ObjCClass);
+    ENUM_TO_CSTRING(ObjCMetaClass);
+    ENUM_TO_CSTRING(ObjCIVar);
+    ENUM_TO_CSTRING(ReExported);
+  }
+  return "<unknown SymbolType>";
+}
+
+lldb::SymbolType Symbol::GetTypeFromString(const char *str) {
+  std::string str_lower = llvm::StringRef(str).lower();
+  return llvm::StringSwitch<lldb::SymbolType>(str_lower)
+      .Case("absolute", eSymbolTypeAbsolute)
+      .Case("code", eSymbolTypeCode)
+      .Case("resolver", eSymbolTypeResolver)
+      .Case("data", eSymbolTypeData)
+      .Case("trampoline", eSymbolTypeTrampoline)
+      .Case("runtime", eSymbolTypeRuntime)
+      .Case("exception", eSymbolTypeException)
+      .Case("sourcefile", eSymbolTypeSourceFile)
+      .Case("headerfile", eSymbolTypeHeaderFile)
+      .Case("objectfile", eSymbolTypeObjectFile)
+      .Case("commonblock", eSymbolTypeCommonBlock)
+      .Case("block", eSymbolTypeBlock)
+      .Case("local", eSymbolTypeLocal)
+      .Case("param", eSymbolTypeParam)
+      .Case("variable", eSymbolTypeVariable)
+      .Case("variableType", eSymbolTypeVariableType)
+      .Case("lineentry", eSymbolTypeLineEntry)
+      .Case("lineheader", eSymbolTypeLineHeader)
+      .Case("scopebegin", eSymbolTypeScopeBegin)
+      .Case("scopeend", eSymbolTypeScopeEnd)
+      .Case("additional,", eSymbolTypeAdditional)
+      .Case("compiler", eSymbolTypeCompiler)
+      .Case("instrumentation", eSymbolTypeInstrumentation)
+      .Case("undefined", eSymbolTypeUndefined)
+      .Case("objcclass", eSymbolTypeObjCClass)
+      .Case("objcmetaclass", eSymbolTypeObjCMetaClass)
+      .Case("objcivar", eSymbolTypeObjCIVar)
+      .Case("reexported", eSymbolTypeReExported)
+      .Default(eSymbolTypeInvalid);
+}
+
 namespace llvm {
 namespace json {
 
@@ -804,36 +840,8 @@ bool fromJSON(const llvm::json::Value &value, lldb_private::JSONSymbol &symbol,
 bool fromJSON(const llvm::json::Value &value, lldb::SymbolType &type,
               llvm::json::Path path) {
   if (auto str = value.getAsString()) {
-    type = llvm::StringSwitch<lldb::SymbolType>(*str)
-               .Case("absolute", eSymbolTypeAbsolute)
-               .Case("code", eSymbolTypeCode)
-               .Case("resolver", eSymbolTypeResolver)
-               .Case("data", eSymbolTypeData)
-               .Case("trampoline", eSymbolTypeTrampoline)
-               .Case("runtime", eSymbolTypeRuntime)
-               .Case("exception", eSymbolTypeException)
-               .Case("sourcefile", eSymbolTypeSourceFile)
-               .Case("headerfile", eSymbolTypeHeaderFile)
-               .Case("objectfile", eSymbolTypeObjectFile)
-               .Case("commonblock", eSymbolTypeCommonBlock)
-               .Case("block", eSymbolTypeBlock)
-               .Case("local", eSymbolTypeLocal)
-               .Case("param", eSymbolTypeParam)
-               .Case("variable", eSymbolTypeVariable)
-               .Case("variableType", eSymbolTypeVariableType)
-               .Case("lineentry", eSymbolTypeLineEntry)
-               .Case("lineheader", eSymbolTypeLineHeader)
-               .Case("scopebegin", eSymbolTypeScopeBegin)
-               .Case("scopeend", eSymbolTypeScopeEnd)
-               .Case("additional,", eSymbolTypeAdditional)
-               .Case("compiler", eSymbolTypeCompiler)
-               .Case("instrumentation", eSymbolTypeInstrumentation)
-               .Case("undefined", eSymbolTypeUndefined)
-               .Case("objcclass", eSymbolTypeObjCClass)
-               .Case("objcmetaClass", eSymbolTypeObjCMetaClass)
-               .Case("objcivar", eSymbolTypeObjCIVar)
-               .Case("reexporte", eSymbolTypeReExported)
-               .Default(eSymbolTypeInvalid);
+    llvm::StringRef str_ref = str.value_or("");
+    type = Symbol::GetTypeFromString(str_ref.data());
 
     if (type == eSymbolTypeInvalid) {
       path.report("invalid symbol type");
diff --git a/lldb/source/Target/ExecutionContext.cpp b/lldb/source/Target/ExecutionContext.cpp
index 3f54ea55c9c81..9d232e420f71c 100644
--- a/lldb/source/Target/ExecutionContext.cpp
+++ b/lldb/source/Target/ExecutionContext.cpp
@@ -12,7 +12,9 @@
 #include "lldb/Target/StackFrame.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/State.h"
+#include <mutex>
 
 using namespace lldb_private;
 
@@ -125,19 +127,47 @@ ExecutionContext::ExecutionContext(const ExecutionContextRef *exe_ctx_ref_ptr,
   }
 }
 
-ExecutionContext::ExecutionContext(const ExecutionContextRef *exe_ctx_ref_ptr,
-                                   std::unique_lock<std::recursive_mutex> &lock)
-    : m_target_sp(), m_process_sp(), m_thread_sp(), m_frame_sp() {
-  if (exe_ctx_ref_ptr) {
-    m_target_sp = exe_ctx_ref_ptr->GetTargetSP();
-    if (m_target_sp) {
-      lock = std::unique_lock<std::recursive_mutex>(m_target_sp->GetAPIMutex());
+llvm::Expected<StoppedExecutionContext>
+lldb_private::GetStoppedExecutionContext(
+    const lldb::ExecutionContextRefSP &exe_ctx_ref_ptr) {
+  return GetStoppedExecutionContext(exe_ctx_ref_ptr.get());
+}
 
-      m_process_sp = exe_ctx_ref_ptr->GetProcessSP();
-      m_thread_sp = exe_ctx_ref_ptr->GetThreadSP();
-      m_frame_sp = exe_ctx_ref_ptr->GetFrameSP();
-    }
-  }
+llvm::Expected<StoppedExecutionContext>
+lldb_private::GetStoppedExecutionContext(
+    const ExecutionContextRef *exe_ctx_ref_ptr) {
+  if (!exe_ctx_ref_ptr)
+    return llvm::createStringError(
+        "StoppedExecutionContext created with an empty ExecutionContextRef");
+
+  lldb::TargetSP target_sp = exe_ctx_ref_ptr->GetTargetSP();
+  if (!target_sp)
+    return llvm::createStringError(
+        "StoppedExecutionContext created with a null target");
+
+  auto api_lock =
+      std::unique_lock<std::recursive_mutex>(target_sp->GetAPIMutex());
+
+  auto process_sp = exe_ctx_ref_ptr->GetProcessSP();
+  if (!process_sp)
+    return llvm::createStringError(
+        "StoppedExecutionContext created with a null process");
+
+  ProcessRunLock::ProcessRunLocker stop_locker;
+  if (!stop_locker.TryLock(&process_sp->GetRunLock()))
+    return llvm::createStringError(
+        "attempted to create a StoppedExecutionContext with a running process");
+
+  auto thread_sp = exe_ctx_ref_ptr->GetThreadSP();
+  auto frame_sp = exe_ctx_ref_ptr->GetFrameSP();
+  return StoppedExecutionContext(target_sp, process_sp, thread_sp, frame_sp,
+                                 std::move(api_lock), std::move(stop_locker));
+}
+
+std::unique_lock<std::recursive_mutex> StoppedExecutionContext::AllowResume() {
+  Clear();
+  m_stop_locker = ProcessRunLock::ProcessRunLocker();
+  return std::move(m_api_lock);
 }
 
 ExecutionContext::ExecutionContext(ExecutionContextScope *exe_scope_ptr)
diff --git a/lldb/source/Target/RegisterContextUnwind.cpp b/lldb/source/Target/RegisterContextUnwind.cpp
index 9e9e2d86958f3..bcf1297f2114f 100644
--- a/lldb/source/Target/RegisterContextUnwind.cpp
+++ b/lldb/source/Target/RegisterContextUnwind.cpp
@@ -2026,7 +2026,6 @@ bool RegisterContextUnwind::ReadFrameAddress(
             "Got an invalid CFA register value - reg %s (%d), value 0x%" PRIx64,
             cfa_reg.GetName(), cfa_reg.GetAsKind(eRegisterKindLLDB),
             cfa_reg_contents);
-        cfa_reg_contents = LLDB_INVALID_ADDRESS;
         return false;
       }
       address = cfa_reg_contents + fa.GetOffset();
diff --git a/lldb/source/Target/StackFrame.cpp b/lldb/source/Target/StackFrame.cpp
index d97a814952186..40049d40aa413 100644
--- a/lldb/source/Target/StackFrame.cpp
+++ b/lldb/source/Target/StackFrame.cpp
@@ -61,8 +61,9 @@ StackFrame::StackFrame(const ThreadSP &thread_sp, user_id_t frame_idx,
                        const SymbolContext *sc_ptr)
     : m_thread_wp(thread_sp), m_frame_index(frame_idx),
       m_concrete_frame_index(unwind_frame_index), m_reg_context_sp(),
-      m_id(pc, cfa, nullptr), m_frame_code_addr(pc), m_sc(), m_flags(),
-      m_frame_base(), m_frame_base_error(), m_cfa_is_valid(cfa_is_valid),
+      m_id(pc, cfa, nullptr, thread_sp->GetProcess().get()),
+      m_frame_code_addr(pc), m_sc(), m_flags(), m_frame_base(),
+      m_frame_base_error(), m_cfa_is_valid(cfa_is_valid),
       m_stack_frame_kind(kind),
       m_behaves_like_zeroth_frame(behaves_like_zeroth_frame),
       m_variable_list_sp(), m_variable_list_value_objects(),
@@ -71,7 +72,7 @@ StackFrame::StackFrame(const ThreadSP &thread_sp, user_id_t frame_idx,
   // recursive functions properly aren't confused with one another on a history
   // stack.
   if (IsHistorical() && !m_cfa_is_valid) {
-    m_id.SetCFA(m_frame_index);
+    m_id.SetCFA(m_frame_index, thread_sp->GetProcess().get());
   }
 
   if (sc_ptr != nullptr) {
@@ -87,7 +88,8 @@ StackFrame::StackFrame(const ThreadSP &thread_sp, user_id_t frame_idx,
                        const SymbolContext *sc_ptr)
     : m_thread_wp(thread_sp), m_frame_index(frame_idx),
       m_concrete_frame_index(unwind_frame_index),
-      m_reg_context_sp(reg_context_sp), m_id(pc, cfa, nullptr),
+      m_reg_context_sp(reg_context_sp),
+      m_id(pc, cfa, nullptr, thread_sp->GetProcess().get()),
       m_frame_code_addr(pc), m_sc(), m_flags(), m_frame_base(),
       m_frame_base_error(), m_cfa_is_valid(true),
       m_stack_frame_kind(StackFrame::Kind::Regular),
@@ -115,7 +117,7 @@ StackFrame::StackFrame(const ThreadSP &thread_sp, user_id_t frame_idx,
       m_concrete_frame_index(unwind_frame_index),
       m_reg_context_sp(reg_context_sp),
       m_id(pc_addr.GetLoadAddress(thread_sp->CalculateTarget().get()), cfa,
-           nullptr),
+           nullptr, thread_sp->GetProcess().get()),
       m_frame_code_addr(pc_addr), m_sc(), m_flags(), m_frame_base(),
       m_frame_base_error(), m_cfa_is_valid(true),
       m_stack_frame_kind(StackFrame::Kind::Regular),
@@ -1995,7 +1997,9 @@ void StackFrame::UpdatePreviousFrameFromCurrentFrame(StackFrame &curr_frame) {
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
   assert(GetStackID() ==
          curr_frame.GetStackID());     // TODO: remove this after some testing
-  m_id.SetPC(curr_frame.m_id.GetPC()); // Update the Stack ID PC value
+  m_id.SetPC(
+      curr_frame.m_id.GetPC(),
+      curr_frame.CalculateProcess().get()); // Update the Stack ID PC value
   assert(GetThread() == curr_frame.GetThread());
   m_frame_index = curr_frame.m_frame_index;
   m_concrete_frame_index = curr_frame.m_concrete_frame_index;
diff --git a/lldb/source/Target/StackFrameRecognizer.cpp b/lldb/source/Target/StackFrameRecognizer.cpp
index 9d5116cb909d1..d9dbed8f2d434 100644
--- a/lldb/source/Target/StackFrameRecognizer.cpp
+++ b/lldb/source/Target/StackFrameRecognizer.cpp
@@ -152,7 +152,7 @@ StackFrameRecognizerManager::GetRecognizerForFrame(StackFrameSP frame) {
   Address start_addr = symbol->GetAddress();
   Address current_addr = frame->GetFrameCodeAddress();
 
-  for (auto entry : m_recognizers) {
+  for (const auto &entry : m_recognizers) {
     if (!entry.enabled)
       continue;
 
diff --git a/lldb/source/Target/StackID.cpp b/lldb/source/Target/StackID.cpp
index 410d5b7e820a5..b1795970802ae 100644
--- a/lldb/source/Target/StackID.cpp
+++ b/lldb/source/Target/StackID.cpp
@@ -10,10 +10,28 @@
 #include "lldb/Symbol/Block.h"
 #include "lldb/Symbol/Symbol.h"
 #include "lldb/Symbol/SymbolContext.h"
+#include "lldb/Target/Process.h"
 #include "lldb/Utility/Stream.h"
 
 using namespace lldb_private;
 
+StackID::StackID(lldb::addr_t pc, lldb::addr_t cfa,
+                 SymbolContextScope *symbol_scope, Process *process)
+    : m_pc(pc), m_cfa(cfa), m_symbol_scope(symbol_scope) {
+  if (process) {
+    m_pc = process->FixCodeAddress(m_pc);
+    m_cfa = process->FixDataAddress(m_cfa);
+  }
+}
+
+void StackID::SetPC(lldb::addr_t pc, Process *process) {
+  m_pc = process ? process->FixCodeAddress(pc) : pc;
+}
+
+void StackID::SetCFA(lldb::addr_t cfa, Process *process) {
+  m_cfa = process ? process->FixDataAddress(cfa) : cfa;
+}
+
 void StackID::Dump(Stream *s) {
   s->Printf("StackID (pc = 0x%16.16" PRIx64 ", cfa = 0x%16.16" PRIx64
             ", symbol_scope = %p",
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 4f39f6018e624..fa98c24606492 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -558,10 +558,11 @@ BreakpointSP Target::CreateBreakpoint(lldb::addr_t addr, bool internal,
 
 BreakpointSP Target::CreateBreakpoint(const Address &addr, bool internal,
                                       bool hardware) {
-  SearchFilterSP filter_sp(
-      new SearchFilterForUnconstrainedSearches(shared_from_this()));
-  BreakpointResolverSP resolver_sp(
-      new BreakpointResolverAddress(nullptr, addr));
+  SearchFilterSP filter_sp =
+      std::make_shared<SearchFilterForUnconstrainedSearches>(
+          shared_from_this());
+  BreakpointResolverSP resolver_sp =
+      std::make_shared<BreakpointResolverAddress>(nullptr, addr);
   return CreateBreakpoint(filter_sp, resolver_sp, internal, hardware, false);
 }
 
@@ -569,10 +570,12 @@ lldb::BreakpointSP
 Target::CreateAddressInModuleBreakpoint(lldb::addr_t file_addr, bool internal,
                                         const FileSpec &file_spec,
                                         bool request_hardware) {
-  SearchFilterSP filter_sp(
-      new SearchFilterForUnconstrainedSearches(shared_from_this()));
-  BreakpointResolverSP resolver_sp(new BreakpointResolverAddress(
-      nullptr, file_addr, file_spec));
+  SearchFilterSP filter_sp =
+      std::make_shared<SearchFilterForUnconstrainedSearches>(
+          shared_from_this());
+  BreakpointResolverSP resolver_sp =
+      std::make_shared<BreakpointResolverAddress>(nullptr, file_addr,
+                                                  file_spec);
   return CreateBreakpoint(filter_sp, resolver_sp, internal, request_hardware,
                           false);
 }
@@ -581,7 +584,8 @@ BreakpointSP Target::CreateBreakpoint(
     const FileSpecList *containingModules,
     const FileSpecList *containingSourceFiles, const char *func_name,
     FunctionNameType func_name_type_mask, LanguageType language,
-    lldb::addr_t offset, LazyBool skip_prologue, bool internal, bool hardware) {
+    lldb::addr_t offset, bool offset_is_insn_count, LazyBool skip_prologue,
+    bool internal, bool hardware) {
   BreakpointSP bp_sp;
   if (func_name) {
     SearchFilterSP filter_sp(GetSearchFilterForModuleAndCUList(
@@ -594,7 +598,7 @@ BreakpointSP Target::CreateBreakpoint(
 
     BreakpointResolverSP resolver_sp(new BreakpointResolverName(
         nullptr, func_name, func_name_type_mask, language, Breakpoint::Exact,
-        offset, skip_prologue));
+        offset, offset_is_insn_count, skip_prologue));
     bp_sp = CreateBreakpoint(filter_sp, resolver_sp, internal, hardware, true);
   }
   return bp_sp;
@@ -2996,6 +3000,38 @@ lldb::addr_t Target::GetBreakableLoadAddress(lldb::addr_t addr) {
   return arch_plugin ? arch_plugin->GetBreakableLoadAddress(addr, *this) : addr;
 }
 
+llvm::Expected<lldb::DisassemblerSP>
+Target::ReadInstructions(const Address &start_addr, uint32_t count,
+                         const char *flavor_string) {
+  DataBufferHeap data(GetArchitecture().GetMaximumOpcodeByteSize() * count, 0);
+  bool force_live_memory = true;
+  lldb_private::Status error;
+  lldb::addr_t load_addr = LLDB_INVALID_ADDRESS;
+  const size_t bytes_read =
+      ReadMemory(start_addr, data.GetBytes(), data.GetByteSize(), error,
+                 force_live_memory, &load_addr);
+
+  if (error.Fail())
+    return llvm::createStringError(
+        error.AsCString("Target::ReadInstructions failed to read memory at %s"),
+        start_addr.GetLoadAddress(this));
+
+  const bool data_from_file = load_addr == LLDB_INVALID_ADDRESS;
+  if (!flavor_string || flavor_string[0] == '\0') {
+    // FIXME - we don't have the mechanism in place to do per-architecture
+    // settings.  But since we know that for now we only support flavors on
+    // x86 & x86_64,
+    const llvm::Triple::ArchType arch = GetArchitecture().GetTriple().getArch();
+    if (arch == llvm::Triple::x86 || arch == llvm::Triple::x86_64)
+      flavor_string = GetDisassemblyFlavor();
+  }
+
+  return Disassembler::DisassembleBytes(
+      GetArchitecture(), nullptr, flavor_string, GetDisassemblyCPU(),
+      GetDisassemblyFeatures(), start_addr, data.GetBytes(), bytes_read, count,
+      data_from_file);
+}
+
 SourceManager &Target::GetSourceManager() {
   if (!m_source_manager_up)
     m_source_manager_up = std::make_unique<SourceManager>(shared_from_this());
diff --git a/lldb/source/Utility/ArchSpec.cpp b/lldb/source/Utility/ArchSpec.cpp
index 7c71aaae6bcf2..8f14cf6e39438 100644
--- a/lldb/source/Utility/ArchSpec.cpp
+++ b/lldb/source/Utility/ArchSpec.cpp
@@ -1439,6 +1439,10 @@ bool lldb_private::operator==(const ArchSpec &lhs, const ArchSpec &rhs) {
   return lhs.GetCore() == rhs.GetCore();
 }
 
+bool lldb_private::operator!=(const ArchSpec &lhs, const ArchSpec &rhs) {
+  return !(lhs == rhs);
+}
+
 bool ArchSpec::IsFullySpecifiedTriple() const {
   if (!TripleOSWasSpecified())
     return false;
diff --git a/lldb/source/Utility/XcodeSDK.cpp b/lldb/source/Utility/XcodeSDK.cpp
index eb2047e67c326..2040791882fd0 100644
--- a/lldb/source/Utility/XcodeSDK.cpp
+++ b/lldb/source/Utility/XcodeSDK.cpp
@@ -243,29 +243,6 @@ bool XcodeSDK::SDKSupportsModules(XcodeSDK::Type sdk_type,
   return false;
 }
 
-bool XcodeSDK::SupportsSwift() const {
-  XcodeSDK::Info info = Parse();
-  switch (info.type) {
-  case Type::MacOSX:
-    return info.version.empty() || info.version >= llvm::VersionTuple(10, 10);
-  case Type::iPhoneOS:
-  case Type::iPhoneSimulator:
-    return info.version.empty() || info.version >= llvm::VersionTuple(8);
-  case Type::AppleTVSimulator:
-  case Type::AppleTVOS:
-    return info.version.empty() || info.version >= llvm::VersionTuple(9);
-  case Type::WatchSimulator:
-  case Type::watchOS:
-    return info.version.empty() || info.version >= llvm::VersionTuple(2);
-  case Type::XROS:
-  case Type::XRSimulator:
-  case Type::Linux:
-    return true;
-  default:
-    return false;
-  }
-}
-
 bool XcodeSDK::SDKSupportsModules(XcodeSDK::Type desired_type,
                                   const FileSpec &sdk_path) {
   ConstString last_path_component = sdk_path.GetFilename();
diff --git a/lldb/test/API/commands/expression/TestRegisterExpressionEndian.py b/lldb/test/API/commands/expression/TestRegisterExpressionEndian.py
new file mode 100644
index 0000000000000..d6de8731385b6
--- /dev/null
+++ b/lldb/test/API/commands/expression/TestRegisterExpressionEndian.py
@@ -0,0 +1,128 @@
+""" Check that registers written to memory for expression evaluation are
+    written using the target's endian not the host's.
+"""
+
+from enum import Enum
+from textwrap import dedent
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test.decorators import *
+from lldbsuite.test.gdbclientutils import *
+from lldbsuite.test.lldbgdbclient import GDBRemoteTestBase
+
+
+class Endian(Enum):
+    BIG = 0
+    LITTLE = 1
+
+
+class Responder(MockGDBServerResponder):
+    def __init__(self, doc, endian):
+        super().__init__()
+        self.target_xml = doc
+        self.endian = endian
+
+    def qXferRead(self, obj, annex, offset, length):
+        if annex == "target.xml":
+            return self.target_xml, False
+        return (None,)
+
+    def readRegister(self, regnum):
+        return "E01"
+
+    def readRegisters(self):
+        # 64 bit pc value.
+        data = ["00", "00", "00", "00", "00", "00", "12", "34"]
+        if self.endian == Endian.LITTLE:
+            data.reverse()
+        return "".join(data)
+
+
+class TestXMLRegisterFlags(GDBRemoteTestBase):
+    def do_endian_test(self, endian):
+        architecture, pc_reg_name, yaml_file, data, machine = {
+            Endian.BIG: ("s390x", "pswa", "s390x.yaml", "ELFDATA2MSB", "EM_S390"),
+            Endian.LITTLE: (
+                "aarch64",
+                "pc",
+                "aarch64.yaml",
+                "ELFDATA2LSB",
+                "EM_AARCH64",
+            ),
+        }[endian]
+
+        self.server.responder = Responder(
+            dedent(
+                f"""\
+            <?xml version="1.0"?>
+              <target version="1.0">
+                <architecture>{architecture}</architecture>
+                <feature>
+                  <reg name="{pc_reg_name}" bitsize="64"/>
+                </feature>
+            </target>"""
+            ),
+            endian,
+        )
+
+        # We need to have a program file, so that we have a full type system,
+        # so that we can do the casts later.
+        obj_path = self.getBuildArtifact("main.o")
+        yaml_path = self.getBuildArtifact(yaml_file)
+        with open(yaml_path, "w") as f:
+            f.write(
+                dedent(
+                    f"""\
+                --- !ELF
+                FileHeader:
+                  Class:    ELFCLASS64
+                  Data:     {data}
+                  Type:     ET_REL
+                  Machine:  {machine}
+                ...
+                """
+                )
+            )
+        self.yaml2obj(yaml_path, obj_path)
+        target = self.dbg.CreateTarget(obj_path)
+
+        process = self.connect(target)
+        lldbutil.expect_state_changes(
+            self, self.dbg.GetListener(), process, [lldb.eStateStopped]
+        )
+
+        # If expressions convert register values into target endian, the
+        # result of register read, expr and casts should be the same.
+        pc_value = "0x0000000000001234"
+        self.expect(
+            "register read pc",
+            substrs=[pc_value],
+        )
+        self.expect("expr --format hex -- $pc", substrs=[pc_value])
+
+        pc = (
+            process.thread[0]
+            .frame[0]
+            .GetRegisters()
+            .GetValueAtIndex(0)
+            .GetChildMemberWithName("pc")
+        )
+        ull = target.FindTypes("unsigned long long").GetTypeAtIndex(0)
+        pc_ull = pc.Cast(ull)
+
+        self.assertEqual(pc.GetValue(), pc_ull.GetValue())
+        self.assertEqual(pc.GetValueAsAddress(), pc_ull.GetValueAsAddress())
+        self.assertEqual(pc.GetValueAsSigned(), pc_ull.GetValueAsSigned())
+        self.assertEqual(pc.GetValueAsUnsigned(), pc_ull.GetValueAsUnsigned())
+
+    @skipIfXmlSupportMissing
+    @skipIfRemote
+    @skipIfLLVMTargetMissing("AArch64")
+    def test_little_endian_target(self):
+        self.do_endian_test(Endian.LITTLE)
+
+    @skipIfXmlSupportMissing
+    @skipIfRemote
+    @skipIfLLVMTargetMissing("SystemZ")
+    def test_big_endian_target(self):
+        self.do_endian_test(Endian.BIG)
diff --git a/lldb/test/API/commands/settings/TestSettings.py b/lldb/test/API/commands/settings/TestSettings.py
index bc864942055c0..95b07f8326f2c 100644
--- a/lldb/test/API/commands/settings/TestSettings.py
+++ b/lldb/test/API/commands/settings/TestSettings.py
@@ -972,6 +972,123 @@ def test_settings_set_exists(self):
         # A known option should fail if its argument is invalid.
         self.expect("settings set auto-confirm bogus", error=True)
 
+    def test_settings_show_defaults(self):
+        # boolean
+        self.expect(
+            "settings show --defaults auto-one-line-summaries",
+            matching=False,
+            substrs=["(default: true)"],
+        )
+        self.runCmd("settings set auto-one-line-summaries false")
+        self.expect(
+            "settings show --defaults auto-one-line-summaries",
+            substrs=["= false (default: true)"],
+        )
+        # unsigned
+        self.expect(
+            "settings show --defaults stop-line-count-before",
+            matching=False,
+            patterns=[r"\(default: \d+\)"],
+        )
+        self.runCmd("settings set stop-line-count-before 99")
+        self.expect(
+            "settings show --defaults stop-line-count-before",
+            patterns=[r"= 99 \(default: \d+\)"],
+        )
+        # string
+        self.expect(
+            "settings show --defaults prompt",
+            matching=False,
+            patterns=[r'\(default: ".+"\)'],
+        )
+        self.runCmd("settings set prompt '<LlDb> '")
+        self.expect(
+            "settings show --defaults prompt",
+            patterns=[r'= "<LlDb> " \(default: ".+"\)'],
+        )
+        # enum
+        self.expect(
+            "settings show --defaults stop-disassembly-display",
+            matching=False,
+            patterns=[r"\(default: .+\)"],
+        )
+        self.runCmd("settings set stop-disassembly-display no-source")
+        self.expect(
+            "settings show --defaults stop-disassembly-display",
+            patterns=[r"= no-source \(default: .+\)"],
+        )
+        # regex
+        self.expect(
+            "settings show --defaults target.process.thread.step-avoid-regexp",
+            matching=False,
+            patterns=[r"\(default: .+\)"],
+        )
+        self.runCmd("settings set target.process.thread.step-avoid-regexp dotstar")
+        self.expect(
+            "settings show --defaults target.process.thread.step-avoid-regexp",
+            patterns=[r"= dotstar \(default: .+\)"],
+        )
+        # format-string
+        self.expect(
+            "settings show --defaults disassembly-format",
+            matching=False,
+            patterns=[r'\(default: ".+"\)'],
+        )
+        self.runCmd("settings set disassembly-format dollar")
+        self.expect(
+            "settings show --defaults disassembly-format",
+            patterns=[r'= "dollar" \(default: ".+"\)'],
+        )
+        # arrays
+        self.expect(
+            "settings show --defaults target.unset-env-vars",
+            matching=False,
+            substrs=["(default: empty)"],
+        )
+        self.runCmd("settings set target.unset-env-vars PATH")
+        self.expect(
+            "settings show --defaults target.unset-env-vars",
+            substrs=["(default: empty)", '[0]: "PATH"'],
+        )
+        # dictionaries
+        self.runCmd("settings clear target.env-vars")
+        self.expect(
+            "settings show --defaults target.env-vars",
+            matching=False,
+            substrs=["(default: empty)"],
+        )
+        self.runCmd("settings set target.env-vars THING=value")
+        self.expect(
+            "settings show --defaults target.env-vars",
+            substrs=["(default: empty)", "THING=value"],
+        )
+        pwd = os.getcwd()
+        # file list
+        self.expect(
+            "settings show --defaults target.exec-search-paths",
+            matching=False,
+            substrs=["(default: empty)"],
+        )
+        self.runCmd(f"settings set target.exec-search-paths {pwd}")
+        self.expect(
+            "settings show --defaults target.exec-search-paths",
+            substrs=["(default: empty)", f"[0]: {pwd}"],
+        )
+        # path map
+        self.expect(
+            "settings show --defaults target.source-map",
+            matching=False,
+            substrs=["(default: empty)"],
+        )
+        self.runCmd(f"settings set target.source-map /abc {pwd}")
+        self.expect(
+            "settings show --defaults target.source-map",
+            patterns=[
+                r"\(default: empty\)",
+                rf'\[0\] "[/\\]abc" -> "{re.escape(pwd)}"',
+            ],
+        )
+
     def get_setting_json(self, setting_path=None):
         settings_data = self.dbg.GetSetting(setting_path)
         stream = lldb.SBStream()
diff --git a/lldb/test/API/functionalities/asan/TestMemoryHistory.py b/lldb/test/API/functionalities/asan/TestMemoryHistory.py
index 1568140b355dc..66f6e3e7502c1 100644
--- a/lldb/test/API/functionalities/asan/TestMemoryHistory.py
+++ b/lldb/test/API/functionalities/asan/TestMemoryHistory.py
@@ -41,18 +41,16 @@ def setUp(self):
         self.line_free = line_number("main.c", "// free line")
         self.line_breakpoint = line_number("main.c", "// break line")
 
-    # Test line numbers: rdar://126237493
-    # for libsanitizers and remove `skip_line_numbers` parameter
-    def check_traces(self, skip_line_numbers=False):
+    def check_traces(self):
         self.expect(
             "memory history 'pointer'",
             substrs=[
                 "Memory deallocated by Thread",
                 "a.out`f2",
-                "main.c" if skip_line_numbers else f"main.c:{self.line_free}",
+                f"main.c:{self.line_free}",
                 "Memory allocated by Thread",
                 "a.out`f1",
-                "main.c" if skip_line_numbers else f"main.c:{self.line_malloc}",
+                f"main.c:{self.line_malloc}",
             ],
         )
 
@@ -76,7 +74,7 @@ def libsanitizers_traces_tests(self):
         self.runCmd("env SanitizersAllocationTraces=all")
 
         self.run_to_breakpoint(target)
-        self.check_traces(skip_line_numbers=True)
+        self.check_traces()
 
     def libsanitizers_asan_tests(self):
         target = self.createTestTarget()
@@ -84,7 +82,7 @@ def libsanitizers_asan_tests(self):
         self.runCmd("env SanitizersAddress=1 MallocSanitizerZone=1")
 
         self.run_to_breakpoint(target)
-        self.check_traces(skip_line_numbers=True)
+        self.check_traces()
 
         self.runCmd("continue")
 
@@ -94,7 +92,7 @@ def libsanitizers_asan_tests(self):
             "Process should be stopped due to ASan report",
             substrs=["stopped", "stop reason = Use of deallocated memory"],
         )
-        self.check_traces(skip_line_numbers=True)
+        self.check_traces()
 
         # do the same using SB API
         process = self.dbg.GetSelectedTarget().process
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/TestDataFormatterStdString.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/TestDataFormatterStdString.py
index 49c0c814f077a..fec20bae997ef 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/TestDataFormatterStdString.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/generic/string/TestDataFormatterStdString.py
@@ -217,9 +217,6 @@ def test_unavailable_summary_libcxx(self):
         self.build(dictionary={"USE_LIBCPP": 1})
         self.do_test_summary_unavailable()
 
-    @expectedFailureAll(
-        bugnumber="libstdc++ std::string summary provider doesn't output a user-friendly message for invalid strings."
-    )
     @add_test_categories(["libstdcxx"])
     def test_unavailable_summary_libstdcxx(self):
         self.build(dictionary={"USE_LIBSTDCPP": 1})
diff --git a/lldb/test/API/functionalities/process_crash_info/TestProcessCrashInfo.py b/lldb/test/API/functionalities/process_crash_info/TestProcessCrashInfo.py
index af05c2f3a0f62..c143da2bf94fb 100644
--- a/lldb/test/API/functionalities/process_crash_info/TestProcessCrashInfo.py
+++ b/lldb/test/API/functionalities/process_crash_info/TestProcessCrashInfo.py
@@ -22,6 +22,15 @@ def tearDown(self):
         self.runCmd("settings clear auto-confirm")
         TestBase.tearDown(self)
 
+    def containsLibmallocError(self, output):
+        for error in [
+            "pointer being freed was not allocated",
+            "not an allocated block",
+        ]:
+            if error in output:
+                return True
+        return False
+
     @skipIfAsan  # The test process intentionally double-frees.
     @skipUnlessDarwin
     def test_cli(self):
@@ -33,15 +42,15 @@ def test_cli(self):
 
         self.expect("process launch", patterns=["Process .* launched: .*a.out"])
 
-        self.expect(
-            "process status --verbose",
-            patterns=[
-                "Extended Crash Information",
-                "Crash-Info Annotations",
-                "pointer being freed was not allocated",
-            ],
+        result = lldb.SBCommandReturnObject()
+        self.dbg.GetCommandInterpreter().HandleCommand(
+            "process status --verbose", result
         )
 
+        self.assertIn("Extended Crash Information", result.GetOutput())
+        self.assertIn("Crash-Info Annotations", result.GetOutput())
+        self.assertTrue(self.containsLibmallocError(result.GetOutput()))
+
     @skipIfAsan  # The test process intentionally hits a memory bug.
     @skipUnlessDarwin
     def test_api(self):
@@ -67,7 +76,7 @@ def test_api(self):
 
         self.assertTrue(crash_info.IsValid())
 
-        self.assertIn("pointer being freed was not allocated", stream.GetData())
+        self.assertTrue(self.containsLibmallocError(stream.GetData()))
 
     # dyld leaves permanent crash_info records when testing on device.
     @skipIfDarwinEmbedded
diff --git a/lldb/test/API/functionalities/thread/state/TestThreadStates.py b/lldb/test/API/functionalities/thread/state/TestThreadStates.py
index 4dbe230c0ce85..6eeff07955733 100644
--- a/lldb/test/API/functionalities/thread/state/TestThreadStates.py
+++ b/lldb/test/API/functionalities/thread/state/TestThreadStates.py
@@ -30,7 +30,6 @@ def test_state_after_breakpoint(self):
     @expectedFailureAll(
         oslist=lldbplatformutil.getDarwinOSTriples(), bugnumber="llvm.org/pr23669"
     )
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr24660")
     def test_state_after_continue(self):
         """Test thread state after continue."""
         self.build()
diff --git a/lldb/test/API/lang/c/step-target/TestStepTarget.py b/lldb/test/API/lang/c/step-target/TestStepTarget.py
index 2da0a7894655d..e5bd64d8927af 100644
--- a/lldb/test/API/lang/c/step-target/TestStepTarget.py
+++ b/lldb/test/API/lang/c/step-target/TestStepTarget.py
@@ -83,14 +83,16 @@ def test_with_end_line_deeper(self):
     @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr32343")
     def test_with_command_and_block(self):
         """Test stepping over vrs. hitting breakpoints & subsequent stepping in various forms."""
+        self.do_command_and_block()
+        self.do_command_and_block(True)
 
+    def do_command_and_block(self, use_regexp_step=False):
         thread = self.get_to_start()
 
-        result = lldb.SBCommandReturnObject()
-        self.dbg.GetCommandInterpreter().HandleCommand(
-            'thread step-in -t "lotsOfArgs" -e block', result
-        )
-        self.assertTrue(result.Succeeded(), "thread step-in command succeeded.")
+        if use_regexp_step:
+            self.expect("s lotsOfArgs")
+        else:
+            self.expect('thread step-in -t "lotsOfArgs" -e block')
 
         frame = thread.frames[0]
         self.assertEqual(frame.name, "lotsOfArgs", "Stepped to lotsOfArgs.")
@@ -98,14 +100,16 @@ def test_with_command_and_block(self):
     @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr32343")
     def test_with_command_and_block_and_bad_name(self):
         """Test stepping over vrs. hitting breakpoints & subsequent stepping in various forms."""
+        self.do_with_command_and_block_and_bad_name()
+        self.do_with_command_and_block_and_bad_name(True)
 
+    def do_with_command_and_block_and_bad_name(self, use_regexp_step=False):
         thread = self.get_to_start()
 
-        result = lldb.SBCommandReturnObject()
-        self.dbg.GetCommandInterpreter().HandleCommand(
-            'thread step-in -t "lotsOfArgsssss" -e block', result
-        )
-        self.assertTrue(result.Succeeded(), "thread step-in command succeeded.")
+        if use_regexp_step:
+            self.expect("s lotsOfArgsssss")
+        else:
+            self.expect('thread step-in -t "lotsOfArgsssss" -e block')
 
         frame = thread.frames[0]
 
diff --git a/lldb/test/API/lang/cpp/expr-definition-in-dylib/TestExprDefinitionInDylib.py b/lldb/test/API/lang/cpp/expr-definition-in-dylib/TestExprDefinitionInDylib.py
index 02c34b3132722..70d7fd096c7cf 100644
--- a/lldb/test/API/lang/cpp/expr-definition-in-dylib/TestExprDefinitionInDylib.py
+++ b/lldb/test/API/lang/cpp/expr-definition-in-dylib/TestExprDefinitionInDylib.py
@@ -31,3 +31,10 @@ def test(self):
         )
 
         self.expect_expr("f.method()", result_value="-72", result_type="int")
+        self.expect_expr("Foo()", result_type="Foo")
+
+        # FIXME: mangled name lookup for ABI-tagged ctors fails because
+        # the debug-info AST doesn't have ABI-tag information.
+        self.expect(
+            "expr Bar()", error=True, substrs=["error: Couldn't look up symbols"]
+        )
diff --git a/lldb/test/API/lang/cpp/expr-definition-in-dylib/lib.cpp b/lldb/test/API/lang/cpp/expr-definition-in-dylib/lib.cpp
index ad148cebb00d1..1a08817f5cda1 100644
--- a/lldb/test/API/lang/cpp/expr-definition-in-dylib/lib.cpp
+++ b/lldb/test/API/lang/cpp/expr-definition-in-dylib/lib.cpp
@@ -1,3 +1,13 @@
 #include "lib.h"
 
+#include <cstdio>
+
 int Foo::method() { return -72; }
+
+Foo::Foo() { std::puts(__func__); }
+
+Foo::~Foo() { std::puts(__func__); }
+
+Bar::Bar() { std::puts(__func__); }
+
+Bar::~Bar() { std::puts(__func__); }
diff --git a/lldb/test/API/lang/cpp/expr-definition-in-dylib/lib.h b/lldb/test/API/lang/cpp/expr-definition-in-dylib/lib.h
index 9568db2166ec4..5ec227946cba0 100644
--- a/lldb/test/API/lang/cpp/expr-definition-in-dylib/lib.h
+++ b/lldb/test/API/lang/cpp/expr-definition-in-dylib/lib.h
@@ -3,6 +3,13 @@
 
 struct Foo {
   int method();
+  Foo();
+  ~Foo();
+};
+
+struct Bar {
+  [[gnu::abi_tag("Ctor")]] Bar();
+  [[gnu::abi_tag("Dtor")]] ~Bar();
 };
 
 #endif // LIB_H_IN
diff --git a/lldb/test/API/lang/cpp/expr-definition-in-dylib/main.cpp b/lldb/test/API/lang/cpp/expr-definition-in-dylib/main.cpp
index 2fddb2b7b3e74..4d6bece21ecac 100644
--- a/lldb/test/API/lang/cpp/expr-definition-in-dylib/main.cpp
+++ b/lldb/test/API/lang/cpp/expr-definition-in-dylib/main.cpp
@@ -2,5 +2,6 @@
 
 int main() {
   Foo f;
+  Bar b;
   return f.method();
 }
diff --git a/lldb/test/API/lang/cpp/lambdas/TestLambdas.py b/lldb/test/API/lang/cpp/lambdas/TestLambdas.py
index c8308c16011e0..112f375c0ef2f 100644
--- a/lldb/test/API/lang/cpp/lambdas/TestLambdas.py
+++ b/lldb/test/API/lang/cpp/lambdas/TestLambdas.py
@@ -1,4 +1,12 @@
 from lldbsuite.test import lldbinline
 from lldbsuite.test import decorators
 
-lldbinline.MakeInlineTest(__file__, globals())
+lldbinline.MakeInlineTest(
+    __file__,
+    globals(),
+    [
+        decorators.expectedFailureAll(
+            bugnumber="https://github.com/llvm/llvm-project/issues/149477"
+        )
+    ],
+)
diff --git a/lldb/test/API/lang/objc/failing-description/Makefile b/lldb/test/API/lang/objc/failing-description/Makefile
new file mode 100644
index 0000000000000..c8d46cd4cd21a
--- /dev/null
+++ b/lldb/test/API/lang/objc/failing-description/Makefile
@@ -0,0 +1,3 @@
+OBJC_SOURCES := main.m
+LD_EXTRAS := -lobjc -framework Foundation
+include Makefile.rules
diff --git a/lldb/test/API/lang/objc/failing-description/TestObjCFailingDescription.py b/lldb/test/API/lang/objc/failing-description/TestObjCFailingDescription.py
new file mode 100644
index 0000000000000..a06ff577e7f7d
--- /dev/null
+++ b/lldb/test/API/lang/objc/failing-description/TestObjCFailingDescription.py
@@ -0,0 +1,17 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.m"))
+        self.expect(
+            "expr -O -- bad", substrs=["error:", "expression interrupted", "(Bad *) 0x"]
+        )
+        self.expect(
+            "dwim-print -O -- bad",
+            substrs=["error:", "expression interrupted", "_lookHere = NO"],
+        )
diff --git a/lldb/test/API/lang/objc/failing-description/main.m b/lldb/test/API/lang/objc/failing-description/main.m
new file mode 100644
index 0000000000000..7416e0a76544d
--- /dev/null
+++ b/lldb/test/API/lang/objc/failing-description/main.m
@@ -0,0 +1,21 @@
+#import <Foundation/Foundation.h>
+
+@interface Bad : NSObject
+@end
+
+@implementation Bad {
+  BOOL _lookHere;
+}
+
+- (NSString *)description {
+  int *i = NULL;
+  *i = 0;
+  return @"surprise";
+}
+@end
+
+int main() {
+  Bad *bad = [Bad new];
+  printf("break here\n");
+  return 0;
+}
diff --git a/lldb/test/API/lang/objc/struct-description/Makefile b/lldb/test/API/lang/objc/struct-description/Makefile
new file mode 100644
index 0000000000000..d0aadc1af9e58
--- /dev/null
+++ b/lldb/test/API/lang/objc/struct-description/Makefile
@@ -0,0 +1,2 @@
+OBJC_SOURCES := main.m
+include Makefile.rules
diff --git a/lldb/test/API/lang/objc/struct-description/TestObjCStructDescription.py b/lldb/test/API/lang/objc/struct-description/TestObjCStructDescription.py
new file mode 100644
index 0000000000000..d152b2690c663
--- /dev/null
+++ b/lldb/test/API/lang/objc/struct-description/TestObjCStructDescription.py
@@ -0,0 +1,18 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestCase(TestBase):
+    def test(self):
+        self.build()
+        lldbutil.run_to_source_breakpoint(self, "break here", lldb.SBFileSpec("main.m"))
+        self.expect(
+            "vo pair",
+            substrs=["error:", "not a pointer type", "(Pair) pair = (f = 2, e = 3)"],
+        )
+        self.expect(
+            "expr -O -- pair",
+            substrs=["error:", "not a pointer type", "(Pair)  (f = 2, e = 3)"],
+        )
diff --git a/lldb/test/API/lang/objc/struct-description/main.m b/lldb/test/API/lang/objc/struct-description/main.m
new file mode 100644
index 0000000000000..5031a28437e60
--- /dev/null
+++ b/lldb/test/API/lang/objc/struct-description/main.m
@@ -0,0 +1,12 @@
+#include <stdio.h>
+
+typedef struct Pair {
+  int f;
+  int e;
+} Pair;
+
+int main() {
+  Pair pair = {2, 3};
+  printf("break here\n");
+  return 0;
+}
diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index 7ab9749f6266d..dcff8687fb4ca 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -353,6 +353,7 @@ def delete_module_cache(path):
 # Some steps required to initialize the tests dynamically link with python.dll
 # and need to know the location of the Python libraries. This ensures that we
 # use the same version of Python that was used to build lldb to run our tests.
+config.environment["PYTHONHOME"] = config.python_root_dir
 config.environment["PATH"] = os.path.pathsep.join(
     (config.python_root_dir, config.environment.get("PATH", ""))
 )
diff --git a/lldb/test/API/python_api/find_in_memory/address_ranges_helper.py b/lldb/test/API/python_api/find_in_memory/address_ranges_helper.py
index dcceca6d8a5c5..102f2b0edd4c6 100644
--- a/lldb/test/API/python_api/find_in_memory/address_ranges_helper.py
+++ b/lldb/test/API/python_api/find_in_memory/address_ranges_helper.py
@@ -55,27 +55,34 @@ def GetRangeFromAddrValue(test_base, addr, shrink=False):
     return lldb.SBAddressRange(start, size)
 
 
-def IsWithinRange(addr, size, range, target):
-    start_addr = range.GetBaseAddress().GetLoadAddress(target)
-    end_addr = start_addr + range.GetByteSize()
-    addr = addr.GetValueAsUnsigned()
-    return addr >= start_addr and addr + size <= end_addr
-
-
 def GetHeapRanges(test_base, shrink=False):
     frame = test_base.thread.GetSelectedFrame()
 
     ex = frame.EvaluateExpression("heap_pointer1")
     test_base.assertTrue(ex.IsValid())
-    range = GetRangeFromAddrValue(test_base, ex, shrink)
-    addr_ranges = lldb.SBAddressRangeList()
-    addr_ranges.Append(range)
+    range1 = GetRangeFromAddrValue(test_base, ex, shrink)
+    range1_start = range1.GetBaseAddress().GetLoadAddress(test_base.target)
+    range1_end = range1_start + range1.GetByteSize()
 
     ex = frame.EvaluateExpression("heap_pointer2")
     test_base.assertTrue(ex.IsValid())
-    size = len(DOUBLE_INSTANCE_PATTERN_HEAP)
-    if not IsWithinRange(ex, size, addr_ranges[0], test_base.target):
-        addr_ranges.Append(GetRangeFromAddrValue(test_base, ex, shrink))
+    range2 = GetRangeFromAddrValue(test_base, ex, shrink)
+    range2_start = range2.GetBaseAddress().GetLoadAddress(test_base.target)
+    range2_end = range2_start + range2.GetByteSize()
+
+    addr_ranges = lldb.SBAddressRangeList()
+
+    if range1_end < range2_start or range2_end < range1_start:
+        # The ranges do not overlap; add them both.
+        addr_ranges.Append(range1)
+        addr_ranges.Append(range2)
+    else:
+        # Merge overlapping ranges.
+        base = min(range1_start, range2_start)
+        end = max(range1_end, range2_end)
+        start = lldb.SBAddress(base, test_base.target)
+        size = end - base
+        addr_ranges.Append(lldb.SBAddressRange(start, size))
 
     return addr_ranges
 
diff --git a/lldb/test/API/python_api/run_locker/TestRunLocker.py b/lldb/test/API/python_api/run_locker/TestRunLocker.py
index b7b4941214e86..d525bbf6b406f 100644
--- a/lldb/test/API/python_api/run_locker/TestRunLocker.py
+++ b/lldb/test/API/python_api/run_locker/TestRunLocker.py
@@ -107,4 +107,6 @@ def runlocker_test(self, stop_at_entry):
             "script var = lldb.frame.EvaluateExpression('SomethingToCall()'); var.GetError().GetCString()",
             result,
         )
-        self.assertIn("sbframe object is not valid", result.GetOutput())
+        self.assertIn(
+            "can't evaluate expressions when the process is running", result.GetOutput()
+        )
diff --git a/lldb/test/API/python_api/sbmodule/Makefile b/lldb/test/API/python_api/sbmodule/Makefile
index 10495940055b6..81df9049e13cd 100644
--- a/lldb/test/API/python_api/sbmodule/Makefile
+++ b/lldb/test/API/python_api/sbmodule/Makefile
@@ -1,3 +1,12 @@
-C_SOURCES := main.c
+C_SOURCES := main.c a.c b.c
+MAKE_DSYM := NO
+
+all: a.out 
+
+a.out: main.o libfoo.a
+	$(LD) $(LDFLAGS) $^ -o $@
+
+libfoo.a: a.o b.o
+	$(AR) $(ARFLAGS) $@ $^
 
 include Makefile.rules
diff --git a/lldb/test/API/python_api/sbmodule/TestSBModule.py b/lldb/test/API/python_api/sbmodule/TestSBModule.py
index c04e2fa55e8cf..096eadd410cfe 100644
--- a/lldb/test/API/python_api/sbmodule/TestSBModule.py
+++ b/lldb/test/API/python_api/sbmodule/TestSBModule.py
@@ -18,6 +18,45 @@ def tearDown(self):
         if self.background_pid:
             os.kill(self.background_pid, signal.SIGKILL)
 
+    @skipIfRemote
+    def test_GetObjectName(self):
+        """Test the SBModule::GetObjectName() method"""
+        self.build()
+        exe = self.getBuildArtifact("a.out")
+        libfoo_path = self.getBuildArtifact("libfoo.a")
+        target_exe = self.dbg.CreateTarget(exe)
+        self.assertTrue(target_exe.IsValid(), "Target for a.out is valid")
+
+        # Test that the executable module has no object name (usually the first module in the target)
+        exe_module = target_exe.GetModuleAtIndex(0)
+        self.assertTrue(exe_module.IsValid(), "Executable module is valid")
+        self.assertIsNone(
+            exe_module.GetObjectName(), "a.out should have no object name"
+        )
+
+        # check archive member names
+        module_specs = lldb.SBModuleSpecList.GetModuleSpecifications(libfoo_path)
+        self.assertGreater(
+            module_specs.GetSize(), 0, "Archive should have at least one module spec"
+        )
+        found = set()
+        expected = {"a.o", "b.o"}
+        for i in range(module_specs.GetSize()):
+            spec = module_specs.GetSpecAtIndex(i)
+            obj_name = spec.GetObjectName()
+            self.assertIsInstance(obj_name, str)
+            self.assertIn(obj_name, expected, f"Unexpected object name: {obj_name}")
+            # create a module from the arhive using the sepc
+            module = lldb.SBModule(spec)
+            self.assertTrue(module.IsValid(), "Module is valid")
+            self.assertTrue(module.IsValid(), f"Module for {obj_name} is valid")
+            self.assertEqual(
+                module.GetObjectName(), obj_name, f"Object name for {obj_name} matches"
+            )
+            found.add(obj_name)
+
+        self.assertEqual(found, expected, "Did not find all expected archive members")
+
     @skipUnlessDarwin
     @skipIfRemote
     def test_module_is_file_backed(self):
diff --git a/lldb/test/API/python_api/sbmodule/a.c b/lldb/test/API/python_api/sbmodule/a.c
new file mode 100644
index 0000000000000..49f5fffe53b49
--- /dev/null
+++ b/lldb/test/API/python_api/sbmodule/a.c
@@ -0,0 +1,11 @@
+int __a_global = 1;
+
+int a(int arg) {
+  int result = arg + __a_global;
+  return result;
+}
+
+int aa(int arg1) {
+  int result1 = arg1 - __a_global;
+  return result1;
+}
diff --git a/lldb/test/API/python_api/sbmodule/b.c b/lldb/test/API/python_api/sbmodule/b.c
new file mode 100644
index 0000000000000..4c7011b682be1
--- /dev/null
+++ b/lldb/test/API/python_api/sbmodule/b.c
@@ -0,0 +1,11 @@
+static int __b_global = 2;
+char __extra[4096]; // Make sure sizeof b.o differs from a.o
+int b(int arg) {
+  int result = arg + __b_global;
+  return result;
+}
+
+int bb(int arg1) {
+  int result2 = arg1 - __b_global;
+  return result2;
+}
diff --git a/lldb/test/API/python_api/sbmodule/main.c b/lldb/test/API/python_api/sbmodule/main.c
index 101d495698f45..574d57d5fd0bb 100644
--- a/lldb/test/API/python_api/sbmodule/main.c
+++ b/lldb/test/API/python_api/sbmodule/main.c
@@ -1,3 +1,5 @@
+extern int a(int);
+extern int b(int);
 int main() {
   while (1) // break here
     ;
diff --git a/lldb/test/API/python_api/sbstructureddata/TestStructuredDataAPI.py b/lldb/test/API/python_api/sbstructureddata/TestStructuredDataAPI.py
index 21256d6c6e743..99f88d3da794a 100644
--- a/lldb/test/API/python_api/sbstructureddata/TestStructuredDataAPI.py
+++ b/lldb/test/API/python_api/sbstructureddata/TestStructuredDataAPI.py
@@ -10,7 +10,6 @@
 
 import json
 
-
 class TestStructuredDataAPI(TestBase):
     NO_DEBUG_INFO_TESTCASE = True
 
@@ -130,6 +129,52 @@ class MyRandomClass:
         self.assertSuccess(example.SetFromJSON("null"))
         self.assertEqual(example.GetType(), lldb.eStructuredDataTypeNull)
 
+        example = lldb.SBStructuredData()
+        example.SetUnsignedIntegerValue(1)
+        self.assertEqual(example.GetType(), lldb.eStructuredDataTypeInteger)
+        self.assertEqual(example.GetIntegerValue(), 1)
+
+        example.SetSignedIntegerValue(-42)
+        self.assertEqual(example.GetType(), lldb.eStructuredDataTypeSignedInteger)
+        self.assertEqual(example.GetSignedIntegerValue(), -42)
+
+        example.SetFloatValue(4.19)
+        self.assertEqual(example.GetType(), lldb.eStructuredDataTypeFloat)
+        self.assertEqual(example.GetFloatValue(), 4.19)
+
+        example.SetStringValue("Bonjour, 123!")
+        self.assertEqual(example.GetType(), lldb.eStructuredDataTypeString)
+        self.assertEqual(example.GetStringValue(42), "Bonjour, 123!")
+
+        value = lldb.SBStructuredData()
+        example.SetValueForKey("Hello", value)
+        self.assertEqual(example.GetSize(), 0)
+
+        nested_obj = lldb.SBStructuredData()
+        nested_obj.SetStringValue("World")
+        example.SetValueForKey("Hello", nested_obj)
+        self.assertEqual(example.GetType(), lldb.eStructuredDataTypeDictionary)
+        nested_obj = None
+        nested_obj = example.GetValueForKey("Hello")
+        self.assertTrue(nested_obj.IsValid())
+        self.assertEqual(nested_obj.GetType(), lldb.eStructuredDataTypeString)
+        self.assertEqual(nested_obj.GetStringValue(42), "World")
+
+        example.SetBooleanValue(True)
+        self.assertEqual(example.GetType(), lldb.eStructuredDataTypeBoolean)
+        self.assertTrue(example.GetBooleanValue())
+
+        rnd_obj = MyRandomClass()
+        stp = lldb.SBScriptObject(rnd_obj, lldb.eScriptLanguagePython)
+        self.assertEqual(stp.ptr, rnd_obj)
+
+        example.SetGenericValue(stp)
+        self.assertEqual(example.GetType(), lldb.eStructuredDataTypeGeneric)
+
+        my_random_class = example.GetGenericValue()
+        self.assertTrue(my_random_class)
+        self.assertEqual(my_random_class.payload, MyRandomClass.payload)
+
         example_arr = [1, 2.3, "4", {"5": False}]
         arr_str = json.dumps(example_arr)
         s.Clear()
diff --git a/lldb/test/API/python_api/value/TestValueAPI.py b/lldb/test/API/python_api/value/TestValueAPI.py
index 0da57346212d0..907992bf05c04 100644
--- a/lldb/test/API/python_api/value/TestValueAPI.py
+++ b/lldb/test/API/python_api/value/TestValueAPI.py
@@ -83,7 +83,7 @@ def test(self):
 
         fmt = lldbutil.BasicFormatter()
         cvf = lldbutil.ChildVisitingFormatter(indent_child=2)
-        rdf = lldbutil.RecursiveDecentFormatter(indent_child=2)
+        rdf = lldbutil.RecursiveDescentFormatter(indent_child=2)
         if self.TraceOn():
             print(fmt.format(days_of_week))
             print(cvf.format(days_of_week))
diff --git a/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py b/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py
index 55557e6e0030e..c54e21c1b973a 100644
--- a/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py
+++ b/lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py
@@ -153,7 +153,7 @@ def test_commands(self):
         breakpoint_ids = self.set_function_breakpoints(functions)
         self.assertEqual(len(breakpoint_ids), len(functions), "expect one breakpoint")
         self.continue_to_breakpoints(breakpoint_ids)
-        output = self.collect_console(timeout_secs=10, pattern=stopCommands[-1])
+        output = self.collect_console(timeout=10, pattern=stopCommands[-1])
         self.verify_commands("stopCommands", output, stopCommands)
 
         # Continue after launch and hit the "pause()" call and stop the target.
@@ -163,7 +163,7 @@ def test_commands(self):
         time.sleep(0.5)
         self.dap_server.request_pause()
         self.dap_server.wait_for_stopped()
-        output = self.collect_console(timeout_secs=10, pattern=stopCommands[-1])
+        output = self.collect_console(timeout=10, pattern=stopCommands[-1])
         self.verify_commands("stopCommands", output, stopCommands)
 
         # Continue until the program exits
@@ -172,7 +172,7 @@ def test_commands(self):
         # "exitCommands" that were run after the second breakpoint was hit
         # and the "terminateCommands" due to the debugging session ending
         output = self.collect_console(
-            timeout_secs=10.0,
+            timeout=10.0,
             pattern=terminateCommands[0],
         )
         self.verify_commands("exitCommands", output, exitCommands)
@@ -223,7 +223,7 @@ def test_terminate_commands(self):
         # "terminateCommands"
         self.dap_server.request_disconnect(terminateDebuggee=True)
         output = self.collect_console(
-            timeout_secs=1.0,
+            timeout=1.0,
             pattern=terminateCommands[0],
         )
         self.verify_commands("terminateCommands", output, terminateCommands)
diff --git a/lldb/test/API/tools/lldb-dap/breakpoint-assembly/TestDAP_breakpointAssembly.py b/lldb/test/API/tools/lldb-dap/breakpoint-assembly/TestDAP_breakpointAssembly.py
index 674bfe4199b4a..fab109c93a17b 100644
--- a/lldb/test/API/tools/lldb-dap/breakpoint-assembly/TestDAP_breakpointAssembly.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint-assembly/TestDAP_breakpointAssembly.py
@@ -2,7 +2,6 @@
 Test lldb-dap setBreakpoints request in assembly source references.
 """
 
-
 from lldbsuite.test.decorators import *
 from dap_server import Source
 import lldbdap_testcase
@@ -52,7 +51,7 @@ def test_break_on_invalid_source_reference(self):
 
         # Verify that setting a breakpoint on an invalid source reference fails
         response = self.dap_server.request_setBreakpoints(
-            Source(source_reference=-1), [1]
+            Source.build(source_reference=-1), [1]
         )
         self.assertIsNotNone(response)
         breakpoints = response["body"]["breakpoints"]
@@ -69,7 +68,7 @@ def test_break_on_invalid_source_reference(self):
 
         # Verify that setting a breakpoint on a source reference that is not created fails
         response = self.dap_server.request_setBreakpoints(
-            Source(source_reference=200), [1]
+            Source.build(source_reference=200), [1]
         )
         self.assertIsNotNone(response)
         breakpoints = response["body"]["breakpoints"]
@@ -83,3 +82,79 @@ def test_break_on_invalid_source_reference(self):
             break_point["message"],
             "Invalid sourceReference.",
         )
+
+    @skipIfWindows
+    def test_persistent_assembly_breakpoint(self):
+        """Tests that assembly breakpoints are working persistently across sessions"""
+        self.build()
+        program = self.getBuildArtifact("a.out")
+        self.create_debug_adapter()
+
+        # Run the first session and set a persistent assembly breakpoint
+        try:
+            self.dap_server.request_initialize()
+            self.dap_server.request_launch(program)
+
+            assmebly_func_breakpoints = self.set_function_breakpoints(["assembly_func"])
+            self.continue_to_breakpoints(assmebly_func_breakpoints)
+
+            assembly_func_frame = self.get_stackFrames()[0]
+            source_reference = assembly_func_frame["source"]["sourceReference"]
+
+            # Set an assembly breakpoint in the middle of the assembly function
+            persistent_breakpoint_line = 4
+            persistent_breakpoint_ids = self.set_source_breakpoints_assembly(
+                source_reference, [persistent_breakpoint_line]
+            )
+
+            self.assertEqual(
+                len(persistent_breakpoint_ids),
+                1,
+                "Expected one assembly breakpoint to be set",
+            )
+
+            persistent_breakpoint_source = self.dap_server.resolved_breakpoints[
+                persistent_breakpoint_ids[0]
+            ]["source"]
+            self.assertIn(
+                "adapterData",
+                persistent_breakpoint_source,
+                "Expected assembly breakpoint to have persistent information",
+            )
+            self.assertIn(
+                "persistenceData",
+                persistent_breakpoint_source["adapterData"],
+                "Expected assembly breakpoint to have persistent information",
+            )
+
+            self.continue_to_breakpoints(persistent_breakpoint_ids)
+        finally:
+            self.dap_server.request_disconnect(terminateDebuggee=True)
+            self.dap_server.terminate()
+
+        # Restart the session and verify the breakpoint is still there
+        self.create_debug_adapter()
+        try:
+            self.dap_server.request_initialize()
+            self.dap_server.request_launch(program)
+            new_session_breakpoints_ids = self.set_source_breakpoints_from_source(
+                Source(persistent_breakpoint_source),
+                [persistent_breakpoint_line],
+            )
+
+            self.assertEqual(
+                len(new_session_breakpoints_ids),
+                1,
+                "Expected one breakpoint to be set in the new session",
+            )
+
+            self.continue_to_breakpoints(new_session_breakpoints_ids)
+            current_line = self.get_stackFrames()[0]["line"]
+            self.assertEqual(
+                current_line,
+                persistent_breakpoint_line,
+                "Expected to hit the persistent assembly breakpoint at the same line",
+            )
+        finally:
+            self.dap_server.request_disconnect(terminateDebuggee=True)
+            self.dap_server.terminate()
diff --git a/lldb/test/API/tools/lldb-dap/breakpoint-events/TestDAP_breakpointEvents.py b/lldb/test/API/tools/lldb-dap/breakpoint-events/TestDAP_breakpointEvents.py
index 8f03244bc6572..151ad761a5044 100644
--- a/lldb/test/API/tools/lldb-dap/breakpoint-events/TestDAP_breakpointEvents.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint-events/TestDAP_breakpointEvents.py
@@ -58,7 +58,7 @@ def test_breakpoint_events(self):
         # Set breakpoints and verify that they got set correctly
         dap_breakpoint_ids = []
         response = self.dap_server.request_setBreakpoints(
-            Source(main_source_path), [main_bp_line]
+            Source.build(path=main_source_path), [main_bp_line]
         )
         self.assertTrue(response["success"])
         breakpoints = response["body"]["breakpoints"]
@@ -70,7 +70,7 @@ def test_breakpoint_events(self):
             )
 
         response = self.dap_server.request_setBreakpoints(
-            Source(foo_source_path), [foo_bp1_line]
+            Source.build(path=foo_source_path), [foo_bp1_line]
         )
         self.assertTrue(response["success"])
         breakpoints = response["body"]["breakpoints"]
diff --git a/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py
index 2e860ff5d5e17..3309800c1dd10 100644
--- a/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py
+++ b/lldb/test/API/tools/lldb-dap/breakpoint/TestDAP_setBreakpoints.py
@@ -2,7 +2,6 @@
 Test lldb-dap setBreakpoints request
 """
 
-
 from dap_server import Source
 import shutil
 from lldbsuite.test.decorators import *
@@ -58,7 +57,7 @@ def test_source_map(self):
 
         # breakpoint in main.cpp
         response = self.dap_server.request_setBreakpoints(
-            Source(new_main_path), [main_line]
+            Source.build(path=new_main_path), [main_line]
         )
         breakpoints = response["body"]["breakpoints"]
         self.assertEqual(len(breakpoints), 1)
@@ -70,7 +69,7 @@ def test_source_map(self):
 
         # 2nd breakpoint, which is from a dynamically loaded library
         response = self.dap_server.request_setBreakpoints(
-            Source(new_other_path), [other_line]
+            Source.build(path=new_other_path), [other_line]
         )
         breakpoints = response["body"]["breakpoints"]
         breakpoint = breakpoints[0]
@@ -85,7 +84,7 @@ def test_source_map(self):
 
         # 2nd breakpoint again, which should be valid at this point
         response = self.dap_server.request_setBreakpoints(
-            Source(new_other_path), [other_line]
+            Source.build(path=new_other_path), [other_line]
         )
         breakpoints = response["body"]["breakpoints"]
         breakpoint = breakpoints[0]
@@ -129,7 +128,9 @@ def test_set_and_clear(self):
         self.build_and_launch(program)
 
         # Set 3 breakpoints and verify that they got set correctly
-        response = self.dap_server.request_setBreakpoints(Source(self.main_path), lines)
+        response = self.dap_server.request_setBreakpoints(
+            Source.build(path=self.main_path), lines
+        )
         line_to_id = {}
         breakpoints = response["body"]["breakpoints"]
         self.assertEqual(
@@ -154,7 +155,9 @@ def test_set_and_clear(self):
         lines.remove(second_line)
         # Set 2 breakpoints and verify that the previous breakpoints that were
         # set above are still set.
-        response = self.dap_server.request_setBreakpoints(Source(self.main_path), lines)
+        response = self.dap_server.request_setBreakpoints(
+            Source.build(path=self.main_path), lines
+        )
         breakpoints = response["body"]["breakpoints"]
         self.assertEqual(
             len(breakpoints),
@@ -199,7 +202,9 @@ def test_set_and_clear(self):
         # Now clear all breakpoints for the source file by passing down an
         # empty lines array
         lines = []
-        response = self.dap_server.request_setBreakpoints(Source(self.main_path), lines)
+        response = self.dap_server.request_setBreakpoints(
+            Source.build(path=self.main_path), lines
+        )
         breakpoints = response["body"]["breakpoints"]
         self.assertEqual(
             len(breakpoints),
@@ -219,7 +224,9 @@ def test_set_and_clear(self):
         # Now set a breakpoint again in the same source file and verify it
         # was added.
         lines = [second_line]
-        response = self.dap_server.request_setBreakpoints(Source(self.main_path), lines)
+        response = self.dap_server.request_setBreakpoints(
+            Source.build(path=self.main_path), lines
+        )
         if response:
             breakpoints = response["body"]["breakpoints"]
             self.assertEqual(
@@ -270,7 +277,9 @@ def test_clear_breakpoints_unset_breakpoints(self):
         self.build_and_launch(program)
 
         # Set one breakpoint and verify that it got set correctly.
-        response = self.dap_server.request_setBreakpoints(Source(self.main_path), lines)
+        response = self.dap_server.request_setBreakpoints(
+            Source.build(path=self.main_path), lines
+        )
         line_to_id = {}
         breakpoints = response["body"]["breakpoints"]
         self.assertEqual(
@@ -286,7 +295,9 @@ def test_clear_breakpoints_unset_breakpoints(self):
         # Now clear all breakpoints for the source file by not setting the
         # lines array.
         lines = None
-        response = self.dap_server.request_setBreakpoints(Source(self.main_path), lines)
+        response = self.dap_server.request_setBreakpoints(
+            Source.build(path=self.main_path), lines
+        )
         breakpoints = response["body"]["breakpoints"]
         self.assertEqual(len(breakpoints), 0, "expect no source breakpoints")
 
@@ -362,7 +373,7 @@ def test_column_breakpoints(self):
         # Set two breakpoints on the loop line at different columns.
         columns = [13, 39]
         response = self.dap_server.request_setBreakpoints(
-            Source(self.main_path),
+            Source.build(path=self.main_path),
             [loop_line, loop_line],
             list({"column": c} for c in columns),
         )
diff --git a/lldb/test/API/tools/lldb-dap/cancel/TestDAP_cancel.py b/lldb/test/API/tools/lldb-dap/cancel/TestDAP_cancel.py
index 824ed8fe3bb97..e722fcea9283a 100644
--- a/lldb/test/API/tools/lldb-dap/cancel/TestDAP_cancel.py
+++ b/lldb/test/API/tools/lldb-dap/cancel/TestDAP_cancel.py
@@ -10,16 +10,14 @@
 
 
 class TestDAP_cancel(lldbdap_testcase.DAPTestCaseBase):
-    def send_async_req(self, command: str, arguments={}) -> int:
-        seq = self.dap_server.sequence
-        self.dap_server.send_packet(
+    def send_async_req(self, command: str, arguments: dict = {}) -> int:
+        return self.dap_server.send_packet(
             {
                 "type": "request",
                 "command": command,
                 "arguments": arguments,
             }
         )
-        return seq
 
     def async_blocking_request(self, duration: float) -> int:
         """
@@ -54,18 +52,18 @@ def test_pending_request(self):
         pending_seq = self.async_blocking_request(duration=self.DEFAULT_TIMEOUT / 2)
         cancel_seq = self.async_cancel(requestId=pending_seq)
 
-        blocking_resp = self.dap_server.recv_packet(filter_type=["response"])
+        blocking_resp = self.dap_server.receive_response(blocking_seq)
         self.assertEqual(blocking_resp["request_seq"], blocking_seq)
         self.assertEqual(blocking_resp["command"], "evaluate")
         self.assertEqual(blocking_resp["success"], True)
 
-        pending_resp = self.dap_server.recv_packet(filter_type=["response"])
+        pending_resp = self.dap_server.receive_response(pending_seq)
         self.assertEqual(pending_resp["request_seq"], pending_seq)
         self.assertEqual(pending_resp["command"], "evaluate")
         self.assertEqual(pending_resp["success"], False)
         self.assertEqual(pending_resp["message"], "cancelled")
 
-        cancel_resp = self.dap_server.recv_packet(filter_type=["response"])
+        cancel_resp = self.dap_server.receive_response(cancel_seq)
         self.assertEqual(cancel_resp["request_seq"], cancel_seq)
         self.assertEqual(cancel_resp["command"], "cancel")
         self.assertEqual(cancel_resp["success"], True)
@@ -80,19 +78,16 @@ def test_inflight_request(self):
 
         blocking_seq = self.async_blocking_request(duration=self.DEFAULT_TIMEOUT / 2)
         # Wait for the sleep to start to cancel the inflight request.
-        self.collect_console(
-            timeout_secs=self.DEFAULT_TIMEOUT,
-            pattern="starting sleep",
-        )
+        self.collect_console(pattern="starting sleep")
         cancel_seq = self.async_cancel(requestId=blocking_seq)
 
-        blocking_resp = self.dap_server.recv_packet(filter_type=["response"])
+        blocking_resp = self.dap_server.receive_response(blocking_seq)
         self.assertEqual(blocking_resp["request_seq"], blocking_seq)
         self.assertEqual(blocking_resp["command"], "evaluate")
         self.assertEqual(blocking_resp["success"], False)
         self.assertEqual(blocking_resp["message"], "cancelled")
 
-        cancel_resp = self.dap_server.recv_packet(filter_type=["response"])
+        cancel_resp = self.dap_server.receive_response(cancel_seq)
         self.assertEqual(cancel_resp["request_seq"], cancel_seq)
         self.assertEqual(cancel_resp["command"], "cancel")
         self.assertEqual(cancel_resp["success"], True)
diff --git a/lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py b/lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py
index ea6b2ea7f28ab..e61d2480ea4bb 100644
--- a/lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py
+++ b/lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py
@@ -1,8 +1,8 @@
-import os
+"""
+Test lldb-dap command hooks
+"""
 
-import dap_server
 import lldbdap_testcase
-from lldbsuite.test import lldbtest, lldbutil
 from lldbsuite.test.decorators import *
 
 
@@ -23,7 +23,7 @@ def test_command_directive_quiet_on_success(self):
             exitCommands=["?" + command_quiet, command_not_quiet],
         )
         full_output = self.collect_console(
-            timeout_secs=1.0,
+            timeout=1.0,
             pattern=command_not_quiet,
         )
         self.assertNotIn(command_quiet, full_output)
@@ -51,7 +51,7 @@ def do_test_abort_on_error(
             expectFailure=True,
         )
         full_output = self.collect_console(
-            timeout_secs=1.0,
+            timeout=1.0,
             pattern=command_abort_on_error,
         )
         self.assertNotIn(command_quiet, full_output)
@@ -81,9 +81,6 @@ def test_command_directive_abort_on_error_attach_commands(self):
             expectFailure=True,
         )
         self.assertFalse(resp["success"], "expected 'attach' failure")
-        full_output = self.collect_console(
-            timeout_secs=1.0,
-            pattern=command_abort_on_error,
-        )
+        full_output = self.collect_console(pattern=command_abort_on_error)
         self.assertNotIn(command_quiet, full_output)
         self.assertIn(command_abort_on_error, full_output)
diff --git a/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py b/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
index 04897acfcf85d..954a3a4b7d14e 100644
--- a/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
+++ b/lldb/test/API/tools/lldb-dap/completions/TestDAP_completions.py
@@ -189,7 +189,7 @@ def test_variable_completions(self):
             self.dap_server.get_completions("str"),
             [
                 {"text": "struct", "label": "struct"},
-                {"text": "str1", "label": "str1 -- string &"},
+                {"text": "str1", "label": "str1 -- std::string &"},
             ],
         )
 
diff --git a/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py b/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py
index 811843dfdf7af..ceddaeb50cd3b 100644
--- a/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py
+++ b/lldb/test/API/tools/lldb-dap/console/TestDAP_console.py
@@ -139,9 +139,7 @@ def test_exit_status_message_sigterm(self):
         process.wait()
 
         # Get the console output
-        console_output = self.collect_console(
-            timeout_secs=10.0, pattern="exited with status"
-        )
+        console_output = self.collect_console(pattern="exited with status")
 
         # Verify the exit status message is printed.
         self.assertRegex(
@@ -156,9 +154,7 @@ def test_exit_status_message_ok(self):
         self.continue_to_exit()
 
         # Get the console output
-        console_output = self.collect_console(
-            timeout_secs=10.0, pattern="exited with status"
-        )
+        console_output = self.collect_console(pattern="exited with status")
 
         # Verify the exit status message is printed.
         self.assertIn(
@@ -177,9 +173,7 @@ def test_diagnositcs(self):
             f"target create --core  {core}", context="repl"
         )
 
-        diagnostics = self.collect_important(
-            timeout_secs=self.DEFAULT_TIMEOUT, pattern="minidump file"
-        )
+        diagnostics = self.collect_important(pattern="minidump file")
 
         self.assertIn(
             "warning: unable to retrieve process ID from minidump file",
diff --git a/lldb/test/API/tools/lldb-dap/instruction-breakpoint/TestDAP_instruction_breakpoint.py b/lldb/test/API/tools/lldb-dap/instruction-breakpoint/TestDAP_instruction_breakpoint.py
index b8b266beaf182..8bb9ea2be5a9f 100644
--- a/lldb/test/API/tools/lldb-dap/instruction-breakpoint/TestDAP_instruction_breakpoint.py
+++ b/lldb/test/API/tools/lldb-dap/instruction-breakpoint/TestDAP_instruction_breakpoint.py
@@ -34,7 +34,7 @@ def instruction_breakpoint_test(self):
 
         # Set source breakpoint 1
         response = self.dap_server.request_setBreakpoints(
-            Source(self.main_path), [main_line]
+            Source.build(path=self.main_path), [main_line]
         )
         breakpoints = response["body"]["breakpoints"]
         self.assertEqual(len(breakpoints), 1)
diff --git a/lldb/test/API/tools/lldb-dap/io/TestDAP_io.py b/lldb/test/API/tools/lldb-dap/io/TestDAP_io.py
index b72b98de412b4..af5c62a8c4eb5 100644
--- a/lldb/test/API/tools/lldb-dap/io/TestDAP_io.py
+++ b/lldb/test/API/tools/lldb-dap/io/TestDAP_io.py
@@ -8,6 +8,9 @@
 import lldbdap_testcase
 import dap_server
 
+EXIT_FAILURE = 1
+EXIT_SUCCESS = 0
+
 
 class TestDAP_io(lldbdap_testcase.DAPTestCaseBase):
     def launch(self):
@@ -41,40 +44,44 @@ def test_eof_immediately(self):
         """
         process = self.launch()
         process.stdin.close()
-        self.assertEqual(process.wait(timeout=5.0), 0)
+        self.assertEqual(process.wait(timeout=5.0), EXIT_SUCCESS)
 
     def test_invalid_header(self):
         """
-        lldb-dap handles invalid message headers.
+        lldb-dap returns a failure exit code when the input stream is closed
+        with a malformed request header.
         """
         process = self.launch()
-        process.stdin.write(b"not the corret message header")
+        process.stdin.write(b"not the correct message header")
         process.stdin.close()
-        self.assertEqual(process.wait(timeout=5.0), 1)
+        self.assertEqual(process.wait(timeout=5.0), EXIT_FAILURE)
 
     def test_partial_header(self):
         """
-        lldb-dap handles parital message headers.
+        lldb-dap returns a failure exit code when the input stream is closed
+        with an incomplete message header is in the message buffer.
         """
         process = self.launch()
         process.stdin.write(b"Content-Length: ")
         process.stdin.close()
-        self.assertEqual(process.wait(timeout=5.0), 1)
+        self.assertEqual(process.wait(timeout=5.0), EXIT_FAILURE)
 
     def test_incorrect_content_length(self):
         """
-        lldb-dap handles malformed content length headers.
+        lldb-dap returns a failure exit code when reading malformed content
+        length headers.
         """
         process = self.launch()
         process.stdin.write(b"Content-Length: abc")
         process.stdin.close()
-        self.assertEqual(process.wait(timeout=5.0), 1)
+        self.assertEqual(process.wait(timeout=5.0), EXIT_FAILURE)
 
     def test_partial_content_length(self):
         """
-        lldb-dap handles partial messages.
+        lldb-dap returns a failure exit code when the input stream is closed
+        with a partial message in the message buffer.
         """
         process = self.launch()
         process.stdin.write(b"Content-Length: 10\r\n\r\n{")
         process.stdin.close()
-        self.assertEqual(process.wait(timeout=5.0), 1)
+        self.assertEqual(process.wait(timeout=5.0), EXIT_FAILURE)
diff --git a/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py b/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py
index a611cc30c1897..41112a4f5b9e3 100644
--- a/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py
+++ b/lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py
@@ -2,12 +2,9 @@
 Test lldb-dap setBreakpoints request
 """
 
-import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
 import lldbdap_testcase
-import time
 import os
 import re
 
@@ -208,7 +205,7 @@ def test_disableSTDIO(self):
         self.continue_to_exit()
         # Now get the STDOUT and verify our program argument is correct
         output = self.get_stdout()
-        self.assertEqual(output, None, "expect no program output")
+        self.assertEqual(output, "", "expect no program output")
 
     @skipIfWindows
     @skipIfLinux  # shell argument expansion doesn't seem to work on Linux
@@ -409,14 +406,14 @@ def test_commands(self):
         # Get output from the console. This should contain both the
         # "stopCommands" that were run after the first breakpoint was hit
         self.continue_to_breakpoints(breakpoint_ids)
-        output = self.get_console(timeout=self.DEFAULT_TIMEOUT)
+        output = self.get_console()
         self.verify_commands("stopCommands", output, stopCommands)
 
         # Continue again and hit the second breakpoint.
         # Get output from the console. This should contain both the
         # "stopCommands" that were run after the second breakpoint was hit
         self.continue_to_breakpoints(breakpoint_ids)
-        output = self.get_console(timeout=self.DEFAULT_TIMEOUT)
+        output = self.get_console()
         self.verify_commands("stopCommands", output, stopCommands)
 
         # Continue until the program exits
@@ -424,13 +421,12 @@ def test_commands(self):
         # Get output from the console. This should contain both the
         # "exitCommands" that were run after the second breakpoint was hit
         # and the "terminateCommands" due to the debugging session ending
-        output = self.collect_console(
-            timeout_secs=1.0,
-            pattern=terminateCommands[0],
-        )
+        output = self.collect_console(pattern=terminateCommands[0])
         self.verify_commands("exitCommands", output, exitCommands)
         self.verify_commands("terminateCommands", output, terminateCommands)
 
+    # Flakey on 32-bit Arm Linux.
+    @skipIf(oslist=["linux"], archs=["arm$"])
     def test_extra_launch_commands(self):
         """
         Tests the "launchCommands" with extra launching settings
@@ -478,21 +474,21 @@ def test_extra_launch_commands(self):
         self.verify_commands("launchCommands", output, launchCommands)
         # Verify the "stopCommands" here
         self.continue_to_next_stop()
-        output = self.get_console(timeout=self.DEFAULT_TIMEOUT)
+        output = self.get_console()
         self.verify_commands("stopCommands", output, stopCommands)
 
         # Continue and hit the second breakpoint.
         # Get output from the console. This should contain both the
         # "stopCommands" that were run after the first breakpoint was hit
         self.continue_to_next_stop()
-        output = self.get_console(timeout=self.DEFAULT_TIMEOUT)
+        output = self.get_console()
         self.verify_commands("stopCommands", output, stopCommands)
 
         # Continue until the program exits
         self.continue_to_exit()
         # Get output from the console. This should contain both the
         # "exitCommands" that were run after the second breakpoint was hit
-        output = self.get_console(timeout=self.DEFAULT_TIMEOUT)
+        output = self.get_console()
         self.verify_commands("exitCommands", output, exitCommands)
 
     def test_failing_launch_commands(self):
@@ -556,10 +552,7 @@ def test_terminate_commands(self):
         # Once it's disconnected the console should contain the
         # "terminateCommands"
         self.dap_server.request_disconnect(terminateDebuggee=True)
-        output = self.collect_console(
-            timeout_secs=1.0,
-            pattern=terminateCommands[0],
-        )
+        output = self.collect_console(pattern=terminateCommands[0])
         self.verify_commands("terminateCommands", output, terminateCommands)
 
     @skipIfWindows
diff --git a/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py b/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py
index 2c3c1fe9cae6b..f51056d7020c6 100644
--- a/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py
+++ b/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py
@@ -126,6 +126,8 @@ def test_readMemory(self):
 
         self.continue_to_exit()
 
+    # Flakey on 32-bit Arm Linux.
+    @skipIf(oslist=["linux"], archs=["arm$"])
     def test_writeMemory(self):
         """
         Tests the 'writeMemory' request
diff --git a/lldb/test/API/tools/lldb-dap/module-event/TestDAP_module_event.py b/lldb/test/API/tools/lldb-dap/module-event/TestDAP_module_event.py
index 64ed4154b035d..bb835af12f5ef 100644
--- a/lldb/test/API/tools/lldb-dap/module-event/TestDAP_module_event.py
+++ b/lldb/test/API/tools/lldb-dap/module-event/TestDAP_module_event.py
@@ -23,15 +23,15 @@ def test_module_event(self):
         self.continue_to_breakpoints(breakpoint_ids)
 
         # We're now stopped at breakpoint 1 before the dlopen. Flush all the module events.
-        event = self.dap_server.wait_for_event("module", 0.25)
+        event = self.dap_server.wait_for_event(["module"], 0.25)
         while event is not None:
-            event = self.dap_server.wait_for_event("module", 0.25)
+            event = self.dap_server.wait_for_event(["module"], 0.25)
 
         # Continue to the second breakpoint, before the dlclose.
         self.continue_to_breakpoints(breakpoint_ids)
 
         # Make sure we got a module event for libother.
-        event = self.dap_server.wait_for_event("module", 5)
+        event = self.dap_server.wait_for_event(["module"], 5)
         self.assertIsNotNone(event, "didn't get a module event")
         module_name = event["body"]["module"]["name"]
         module_id = event["body"]["module"]["id"]
@@ -42,7 +42,7 @@ def test_module_event(self):
         self.continue_to_breakpoints(breakpoint_ids)
 
         # Make sure we got a module event for libother.
-        event = self.dap_server.wait_for_event("module", 5)
+        event = self.dap_server.wait_for_event(["module"], 5)
         self.assertIsNotNone(event, "didn't get a module event")
         reason = event["body"]["reason"]
         self.assertEqual(reason, "removed")
@@ -56,7 +56,7 @@ def test_module_event(self):
         self.assertEqual(module_data["name"], "", "expects empty name.")
 
         # Make sure we do not send another event
-        event = self.dap_server.wait_for_event("module", 3)
+        event = self.dap_server.wait_for_event(["module"], 3)
         self.assertIsNone(event, "expects no events.")
 
         self.continue_to_exit()
diff --git a/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py b/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py
index c9091df64f487..74743d9182ab4 100644
--- a/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py
+++ b/lldb/test/API/tools/lldb-dap/module/TestDAP_module.py
@@ -1,11 +1,9 @@
 """
-Test lldb-dap setBreakpoints request
+Test lldb-dap module request
 """
 
-import dap_server
 from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
-from lldbsuite.test import lldbutil
 import lldbdap_testcase
 import re
 
@@ -55,7 +53,7 @@ def check_symbols_loaded_with_size():
 
         if expect_debug_info_size:
             self.assertTrue(
-                self.waitUntil(check_symbols_loaded_with_size),
+                self.wait_until(check_symbols_loaded_with_size),
                 "expect has debug info size",
             )
 
@@ -68,7 +66,7 @@ def check_symbols_loaded_with_size():
         # Collect all the module names we saw as events.
         module_new_names = []
         module_changed_names = []
-        module_event = self.dap_server.wait_for_event("module", 1)
+        module_event = self.dap_server.wait_for_event(["module"], 1)
         while module_event is not None:
             reason = module_event["body"]["reason"]
             if reason == "new":
@@ -76,7 +74,7 @@ def check_symbols_loaded_with_size():
             elif reason == "changed":
                 module_changed_names.append(module_event["body"]["module"]["name"])
 
-            module_event = self.dap_server.wait_for_event("module", 1)
+            module_event = self.dap_server.wait_for_event(["module"], 1)
 
         # Make sure we got an event for every active module.
         self.assertNotEqual(len(module_new_names), 0)
diff --git a/lldb/test/API/tools/lldb-dap/moduleSymbols/Makefile b/lldb/test/API/tools/lldb-dap/moduleSymbols/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/moduleSymbols/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/tools/lldb-dap/moduleSymbols/TestDAP_moduleSymbols.py b/lldb/test/API/tools/lldb-dap/moduleSymbols/TestDAP_moduleSymbols.py
new file mode 100644
index 0000000000000..2336b9f2a5a1a
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/moduleSymbols/TestDAP_moduleSymbols.py
@@ -0,0 +1,40 @@
+"""
+Test lldb-dap moduleSymbols request
+"""
+
+import lldbdap_testcase
+from lldbsuite.test.decorators import *
+
+
+class TestDAP_moduleSymbols(lldbdap_testcase.DAPTestCaseBase):
+    # On windows LLDB doesn't recognize symbols in a.out.
+    @skipIfWindows
+    def test_moduleSymbols(self):
+        """
+        Test that the moduleSymbols request returns correct symbols from the module.
+        """
+        program = self.getBuildArtifact("a.out")
+        self.build_and_launch(program)
+
+        symbol_names = []
+        i = 0
+        while True:
+            next_symbol = self.dap_server.request_moduleSymbols(
+                moduleName="a.out", startIndex=i, count=1
+            )
+            self.assertIn("symbols", next_symbol["body"])
+            result_symbols = next_symbol["body"]["symbols"]
+            self.assertLessEqual(len(result_symbols), 1)
+            if len(result_symbols) == 0:
+                break
+
+            self.assertIn("name", result_symbols[0])
+            symbol_names.append(result_symbols[0]["name"])
+            i += 1
+            if i >= 1000:
+                break
+
+        self.assertGreater(len(symbol_names), 0)
+        self.assertIn("main", symbol_names)
+        self.assertIn("func1", symbol_names)
+        self.assertIn("func2", symbol_names)
diff --git a/lldb/test/API/tools/lldb-dap/moduleSymbols/main.c b/lldb/test/API/tools/lldb-dap/moduleSymbols/main.c
new file mode 100644
index 0000000000000..b038b10480b80
--- /dev/null
+++ b/lldb/test/API/tools/lldb-dap/moduleSymbols/main.c
@@ -0,0 +1,9 @@
+int func1() { return 42; }
+
+int func2() { return 84; }
+
+int main() {
+  func1();
+  func2();
+  return 0;
+}
diff --git a/lldb/test/API/tools/lldb-dap/output/TestDAP_output.py b/lldb/test/API/tools/lldb-dap/output/TestDAP_output.py
index 0425b55a5e552..fe978a9a73351 100644
--- a/lldb/test/API/tools/lldb-dap/output/TestDAP_output.py
+++ b/lldb/test/API/tools/lldb-dap/output/TestDAP_output.py
@@ -29,7 +29,7 @@ def test_output(self):
         self.continue_to_breakpoints(breakpoint_ids)
 
         # Ensure partial messages are still sent.
-        output = self.collect_stdout(timeout_secs=1.0, pattern="abcdef")
+        output = self.collect_stdout(timeout=1.0, pattern="abcdef")
         self.assertTrue(output and len(output) > 0, "expect program stdout")
 
         self.continue_to_exit()
@@ -37,14 +37,14 @@ def test_output(self):
         # Disconnecting from the server to ensure any pending IO is flushed.
         self.dap_server.request_disconnect()
 
-        output += self.get_stdout(timeout=self.DEFAULT_TIMEOUT)
+        output += self.get_stdout()
         self.assertTrue(output and len(output) > 0, "expect program stdout")
         self.assertIn(
             "abcdefghi\r\nhello world\r\nfinally\0\0",
             output,
             "full stdout not found in: " + repr(output),
         )
-        console = self.get_console(timeout=self.DEFAULT_TIMEOUT)
+        console = self.get_console()
         self.assertTrue(console and len(console) > 0, "expect dap messages")
         self.assertIn(
             "out\0\0\r\nerr\0\0\r\n", console, f"full console message not found"
diff --git a/lldb/test/API/tools/lldb-dap/progress/TestDAP_Progress.py b/lldb/test/API/tools/lldb-dap/progress/TestDAP_Progress.py
index b47d52968f8a1..3f57dfb66024d 100755
--- a/lldb/test/API/tools/lldb-dap/progress/TestDAP_Progress.py
+++ b/lldb/test/API/tools/lldb-dap/progress/TestDAP_Progress.py
@@ -21,7 +21,7 @@ def verify_progress_events(
         expected_not_in_message=None,
         only_verify_first_update=False,
     ):
-        self.dap_server.wait_for_event("progressEnd")
+        self.dap_server.wait_for_event(["progressEnd"])
         self.assertTrue(len(self.dap_server.progress_events) > 0)
         start_found = False
         update_found = False
diff --git a/lldb/test/Shell/Expr/TestLambdaExprImport.test b/lldb/test/Shell/Expr/TestLambdaExprImport.test
new file mode 100644
index 0000000000000..c57ce06453fe2
--- /dev/null
+++ b/lldb/test/Shell/Expr/TestLambdaExprImport.test
@@ -0,0 +1,26 @@
+# Test that we can successfully ASTImport clang::LambdaExpr nodes.
+# Currently this is not supported in MinimalImport mode (which LLDB
+# uses always).
+
+# RUN: split-file %s %t
+# RUN: %clang_host -g -gdwarf %t/main.cpp -o %t.out
+# RUN: %lldb -o "settings set interpreter.stop-command-source-on-error false" \
+# RUN:       -x -b -s %t/commands.input %t.out 2>&1 \
+# RUN:       | FileCheck %s
+
+#--- main.cpp
+
+int main() {
+  __builtin_debugtrap();
+}
+
+#--- commands.input
+
+run
+expression --top-level -- void method(int x) { [x=x] { ; }; }
+target dump typesystem
+
+# CHECK:      expression
+# CHECK:      target dump typesystem
+# CHECK-NOT:  FunctionDecl
+# CHECK-NOT:  LambdaExpr
diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-foreign-type-units.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-foreign-type-units.cpp
index 415b4850a244c..9251930d7d13c 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-foreign-type-units.cpp
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/dwp-foreign-type-units.cpp
@@ -56,8 +56,8 @@
 // DWPMAIN-NEXT: struct CustomType {
 // DWPMAIN-NEXT:     typedef int IntegerType;
 // DWPMAIN-NEXT:     typedef double FloatType;
-// DWPMAIN-NEXT:     CustomType::IntegerType x;
-// DWPMAIN-NEXT:     CustomType::FloatType y;
+// DWPMAIN-NEXT:     IntegerType x;
+// DWPMAIN-NEXT:     FloatType y;
 // DWPMAIN-NEXT: }
 
 // Next we check when we make the .dwp file with %t.foo.dwo first so it will
@@ -78,8 +78,8 @@
 // DWPFOO-NEXT: struct CustomType {
 // DWPFOO-NEXT:     typedef unsigned int IntegerType;
 // DWPFOO-NEXT:     typedef float FloatType;
-// DWPFOO-NEXT:     CustomType::IntegerType x;
-// DWPFOO-NEXT:     CustomType::FloatType y;
+// DWPFOO-NEXT:     IntegerType x;
+// DWPFOO-NEXT:     FloatType y;
 // DWPFOO-NEXT: }
 
 struct CustomType {
diff --git a/lldb/test/Shell/SymbolFile/NativePDB/find-functions.cpp b/lldb/test/Shell/SymbolFile/NativePDB/find-functions.cpp
index 5ebef61bdbfef..3ef7a4c94c287 100644
--- a/lldb/test/Shell/SymbolFile/NativePDB/find-functions.cpp
+++ b/lldb/test/Shell/SymbolFile/NativePDB/find-functions.cpp
@@ -6,24 +6,52 @@
 
 // RUN: lldb-test symbols --find=function --name=main --function-flags=full %t.exe \
 // RUN:     | FileCheck %s --check-prefix=FIND-MAIN
+// RUN: lldb-test symbols --find=function --name=main --function-flags=method %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-NO-FUNCTION
+// RUN: lldb-test symbols --find=function --name=main --function-flags=base %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-MAIN
 
 // RUN: lldb-test symbols --find=function --name=static_fn --function-flags=full %t.exe \
 // RUN:     | FileCheck %s --check-prefix=FIND-STATIC
+// RUN: lldb-test symbols --find=function --name=static_fn --function-flags=method %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-NO-FUNCTION
+// RUN: lldb-test symbols --find=function --name=static_fn --function-flags=base %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-STATIC
 
 // RUN: lldb-test symbols --find=function --name=varargs_fn --function-flags=full %t.exe \
 // RUN:     | FileCheck %s --check-prefix=FIND-VAR
+// RUN: lldb-test symbols --find=function --name=varargs_fn --function-flags=method %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-NO-FUNCTION
+// RUN: lldb-test symbols --find=function --name=varargs_fn --function-flags=base %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-VAR
 
 // RUN: lldb-test symbols --find=function --name=Struct::simple_method --function-flags=full %t.exe \
 // RUN:     | FileCheck %s --check-prefix=FIND-SIMPLE
+// RUN: lldb-test symbols --find=function --name=Struct::simple_method --function-flags=method %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-SIMPLE
+// RUN: lldb-test symbols --find=function --name=Struct::simple_method --function-flags=base %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-SIMPLE
 
 // RUN: lldb-test symbols --find=function --name=Struct::virtual_method --function-flags=full %t.exe \
 // RUN:     | FileCheck %s --check-prefix=FIND-VIRTUAL
+// RUN: lldb-test symbols --find=function --name=Struct::virtual_method --function-flags=method %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-VIRTUAL
+// RUN: lldb-test symbols --find=function --name=Struct::virtual_method --function-flags=base %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-VIRTUAL
 
 // RUN: lldb-test symbols --find=function --name=Struct::static_method --function-flags=full %t.exe \
 // RUN:     | FileCheck %s --check-prefix=FIND-STATIC-METHOD
+// RUN: lldb-test symbols --find=function --name=Struct::static_method --function-flags=method %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-NO-FUNCTION
+// RUN: lldb-test symbols --find=function --name=Struct::static_method --function-flags=base %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-STATIC-METHOD
 
 // RUN: lldb-test symbols --find=function --name=Struct::overloaded_method --function-flags=full %t.exe \
-// RUN:     | FileCheck %s --check-prefix=FIND-OVERLOAD
+// RUN:     | FileCheck %s --check-prefix=FIND-OVERLOAD-FULL
+// RUN: lldb-test symbols --find=function --name=Struct::overloaded_method --function-flags=method %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-OVERLOAD-METHOD
+// RUN: lldb-test symbols --find=function --name=Struct::overloaded_method --function-flags=base %t.exe \
+// RUN:     | FileCheck %s --check-prefix=FIND-OVERLOAD-BASE
 
 struct Struct {
   int simple_method() {
@@ -51,7 +79,28 @@ struct Struct {
   }
 };
 
+class Class {
+public:
+  bool overloaded_method() {
+    return false;
+  }
+  bool overloaded_method(int i) {
+    return i > 0;
+  }
+  static int overloaded_method(bool b) {
+    return b ? 1 : 2;
+  }
+};
+
+char overloaded_method() {
+  return 0;
+}
+char overloaded_method(int i) {
+  return 0;
+}
+
 Struct s;
+Class c;
 
 static int static_fn() {
   return 42;
@@ -63,12 +112,16 @@ int varargs_fn(int x, int y, ...) {
 
 int main(int argc, char **argv) {
   return static_fn() + varargs_fn(argc, argc) + s.simple_method() +
-  Struct::static_method() + s.virtual_method() + s.overloaded_method();
+  Struct::static_method() + s.virtual_method() + s.overloaded_method() + 
+  Class::overloaded_method(false) + c.overloaded_method(1) + c.overloaded_method() 
+  + overloaded_method() + overloaded_method(1);
 }
 
 // FIND-MAIN:      Function: id = {{.*}}, name = "main"
 // FIND-MAIN-NEXT: FuncType: id = {{.*}}, compiler_type = "int (int, char **)"
 
+// FIND-NO-FUNCTION: Found 0 functions
+
 // FIND-STATIC:      Function: id = {{.*}}, name = "{{.*}}static_fn{{.*}}"
 // FIND-STATIC-NEXT: FuncType: id = {{.*}}, compiler_type = "int (void)"
 
@@ -84,7 +137,29 @@ int main(int argc, char **argv) {
 // FIND-STATIC-METHOD:      Function: id = {{.*}}, name = "{{.*}}Struct::static_method{{.*}}"
 // FIND-STATIC-METHOD-NEXT: FuncType: id = {{.*}}, compiler_type = "int (void)"
 
-// FIND-OVERLOAD: Function: id = {{.*}}, name = "{{.*}}Struct::overloaded_method{{.*}}"
-// FIND-OVERLOAD: FuncType: id = {{.*}}, compiler_type = "int (void)"
-// FIND-OVERLOAD: FuncType: id = {{.*}}, compiler_type = "int (char)"
-// FIND-OVERLOAD: FuncType: id = {{.*}}, compiler_type = "int (char, int, ...)"
+// FIND-OVERLOAD-FULL-NOT: "Class::overloaded_method"
+// FIND-OVERLOAD-FULL-NOT: "overloaded_method"
+// FIND-OVERLOAD-FULL-DAG: Function: id = {{.*}}, name = "{{.*}}Struct::overloaded_method{{.*}}"
+// FIND-OVERLOAD-FULL-DAG: FuncType: id = {{.*}}, compiler_type = "int (void)"
+// FIND-OVERLOAD-FULL-DAG: FuncType: id = {{.*}}, compiler_type = "int (char)"
+// FIND-OVERLOAD-FULL-DAG: FuncType: id = {{.*}}, compiler_type = "int (char, int, ...)"
+
+// FIND-OVERLOAD-BASE-DAG: Function: id = {{.*}}, name = "{{.*}}Struct::overloaded_method{{.*}}"
+// FIND-OVERLOAD-BASE-DAG: FuncType: id = {{.*}}, compiler_type = "int (void)"
+// FIND-OVERLOAD-BASE-DAG: FuncType: id = {{.*}}, compiler_type = "int (char)"
+// FIND-OVERLOAD-BASE-DAG: FuncType: id = {{.*}}, compiler_type = "int (char, int, ...)"
+// FIND-OVERLOAD-BASE-DAG: Function: id = {{.*}}, name = "Class::overloaded_method"
+// FIND-OVERLOAD-BASE-DAG: FuncType: id = {{.*}}, compiler_type = "_Bool (void)"
+// FIND-OVERLOAD-BASE-DAG: FuncType: id = {{.*}}, compiler_type = "_Bool (int)"
+// FIND-OVERLOAD-BASE-DAG: FuncType: id = {{.*}}, compiler_type = "int (_Bool)"
+// FIND-OVERLOAD-BASE-DAG: Function: id = {{.*}}, name = "overloaded_method"
+// FIND-OVERLOAD-BASE-DAG: FuncType: id = {{.*}}, compiler_type = "char (void)"
+// FIND-OVERLOAD-BASE-DAG: FuncType: id = {{.*}}, compiler_type = "char (int)"
+
+// FIND-OVERLOAD-METHOD-NOT: "overloaded_method"
+// FIND-OVERLOAD-METHOD-DAG: Function: id = {{.*}}, name = "{{.*}}Struct::overloaded_method{{.*}}"
+// FIND-OVERLOAD-METHOD-DAG: FuncType: id = {{.*}}, compiler_type = "int (void)"
+// FIND-OVERLOAD-METHOD-DAG: FuncType: id = {{.*}}, compiler_type = "int (char)"
+// FIND-OVERLOAD-METHOD-DAG: Function: id = {{.*}}, name = "Class::overloaded_method"
+// FIND-OVERLOAD-METHOD-DAG: FuncType: id = {{.*}}, compiler_type = "_Bool (void)"
+// FIND-OVERLOAD-METHOD-DAG: FuncType: id = {{.*}}, compiler_type = "_Bool (int)"
diff --git a/lldb/test/Shell/SymbolFile/NativePDB/unknown-udt-decl.ll b/lldb/test/Shell/SymbolFile/NativePDB/unknown-udt-decl.ll
new file mode 100644
index 0000000000000..af787897dd8cf
--- /dev/null
+++ b/lldb/test/Shell/SymbolFile/NativePDB/unknown-udt-decl.ll
@@ -0,0 +1,56 @@
+; Test that the declaration for UDTs won't be "<unknown>" or "\<unknown>".
+; Rustc sets the location of some builtin types to this string.
+
+; REQUIRES: system-windows
+; RUN: %build --compiler=clang-cl --nodefaultlib -o %t.exe -- %s
+; RUN: lldb-test symbols %t.exe | FileCheck %s
+
+; there shouldn't be a declaration (would be between size and compiler_type)
+; CHECK: Type{{.*}} , name = "Foo", size = 1, compiler_type = {{.*}} struct Foo {
+
+; This is edited output from clang  simulates rustc behavior (see !17)
+; Source:
+; struct Foo {};
+;
+; int main() { Foo f; }
+
+
+; ModuleID = 'main.cpp'
+source_filename = "main.cpp"
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.44.35207"
+
+%struct.Foo = type { i8 }
+
+; Function Attrs: mustprogress noinline norecurse nounwind optnone uwtable
+define dso_local noundef i32 @main() #0 !dbg !9 {
+  %1 = alloca %struct.Foo, align 1
+    #dbg_declare(ptr %1, !14, !DIExpression(), !16)
+  ret i32 0, !dbg !16
+}
+
+attributes #0 = { mustprogress noinline norecurse nounwind optnone uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 20.1.6", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "main.cpp", directory: "F:\\Dev\\rust-dbg-test", checksumkind: CSK_MD5, checksum: "b8942260dadf9ec35328889f05afb954")
+!2 = !{i32 2, !"CodeView", i32 1}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 2}
+!5 = !{i32 8, !"PIC Level", i32 2}
+!6 = !{i32 7, !"uwtable", i32 2}
+!7 = !{i32 1, !"MaxTLSAlign", i32 65536}
+!8 = !{!"clang version 20.1.6"}
+!9 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 3, type: !10, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !13)
+!10 = !DISubroutineType(types: !11)
+!11 = !{!12}
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !{}
+!14 = !DILocalVariable(name: "f", scope: !9, file: !1, line: 3, type: !15)
+!15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Foo", file: !17, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !13, identifier: ".?AUFoo@@")
+!16 = !DILocation(line: 3, scope: !9)
+; This is how rustc emits some types
+!17 = !DIFile(filename: "<unknown>", directory: "")
diff --git a/lldb/test/Shell/SymbolFile/PDB/ast-restore.test b/lldb/test/Shell/SymbolFile/PDB/ast-restore.test
index a91364bbbee63..f127acda90d22 100644
--- a/lldb/test/Shell/SymbolFile/PDB/ast-restore.test
+++ b/lldb/test/Shell/SymbolFile/PDB/ast-restore.test
@@ -3,13 +3,19 @@ RUN: %build --compiler=msvc --nodefaultlib --output=%t.exe %S/Inputs/AstRestoreT
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=ENUM %s
 RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=ENUM %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=GLOBAL %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=GLOBAL %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=BASE %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=BASE %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=CLASS %s
 RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=CLASS %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=INNER %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=INNER %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=TEMPLATE %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=TEMPLATE %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=FOO %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=FOO %s
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=MAIN %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix=MAIN %s
 
 ENUM: Module: {{.*}}
 ENUM: namespace N0 {
diff --git a/lldb/test/Shell/SymbolFile/PDB/calling-conventions-arm.test b/lldb/test/Shell/SymbolFile/PDB/calling-conventions-arm.test
index 7dabf9157d47e..8b0b5b665274d 100644
--- a/lldb/test/Shell/SymbolFile/PDB/calling-conventions-arm.test
+++ b/lldb/test/Shell/SymbolFile/PDB/calling-conventions-arm.test
@@ -1,7 +1,12 @@
 REQUIRES: target-windows, lld, (target-arm || target-aarch64)
+
 RUN: %build --compiler=clang-cl --arch=32 --nodefaultlib --output=%t.exe %S/Inputs/CallingConventionsTest.cpp
+RUN: lldb-test symbols -dump-ast %t.exe | FileCheck %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck %s
+
 RUN: %build --compiler=clang-cl --arch=64 --nodefaultlib --output=%t.exe %S/Inputs/CallingConventionsTest.cpp
 RUN: lldb-test symbols -dump-ast %t.exe | FileCheck %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t.exe | FileCheck %s
 
 CHECK: Module: {{.*}}
 CHECK-DAG: int (*FuncCCallPtr)();
diff --git a/lldb/test/Shell/SymbolFile/PDB/calling-conventions-x86.test b/lldb/test/Shell/SymbolFile/PDB/calling-conventions-x86.test
index 065c8b69b06dd..ceb7ad18919da 100644
--- a/lldb/test/Shell/SymbolFile/PDB/calling-conventions-x86.test
+++ b/lldb/test/Shell/SymbolFile/PDB/calling-conventions-x86.test
@@ -1,8 +1,12 @@
 REQUIRES: system-windows, lld, (target-x86 || target-x86_64) 
-RUN: %build --compiler=clang-cl --arch=32 --nodefaultlib --output=%t.exe %S/Inputs/CallingConventionsTest.cpp \
-RUN:  && lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix 32BIT-CHECK %s
-RUN: %build --compiler=clang-cl --arch=64 --nodefaultlib --output=%t.exe %S/Inputs/CallingConventionsTest.cpp \
-RUN:  && lldb-test symbols -dump-ast %t.exe | FileCheck --check-prefix 64BIT-CHECK %s
+
+RUN: %build --compiler=clang-cl --arch=32 --nodefaultlib --output=%t-32.exe %S/Inputs/CallingConventionsTest.cpp
+RUN: lldb-test symbols -dump-ast %t-32.exe | FileCheck --check-prefix 32BIT-CHECK %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t-32.exe | FileCheck --check-prefix 32BIT-CHECK %s
+
+RUN: %build --compiler=clang-cl --arch=64 --nodefaultlib --output=%t-64.exe %S/Inputs/CallingConventionsTest.cpp
+RUN: lldb-test symbols -dump-ast %t-64.exe | FileCheck --check-prefix 64BIT-CHECK %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols -dump-ast %t-64.exe | FileCheck --check-prefix 64BIT-CHECK %s
 
 64BIT-CHECK: Module: {{.*}}
 64BIT-CHECK-DAG: int (*FuncCCallPtr)();
diff --git a/lldb/test/Shell/SymbolFile/PDB/class-layout.test b/lldb/test/Shell/SymbolFile/PDB/class-layout.test
index e9a7d1c0daa9e..eca910e997e40 100644
--- a/lldb/test/Shell/SymbolFile/PDB/class-layout.test
+++ b/lldb/test/Shell/SymbolFile/PDB/class-layout.test
@@ -12,9 +12,19 @@ RUN: lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix
 RUN: lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=BASE %s
 RUN: lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=FRIEND %s
 RUN: lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=CLASS %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=ENUM %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=UNION %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=STRUCT %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=COMPLEX %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=LIST %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=UNNAMED-STRUCT %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=BASE %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=FRIEND %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/ClassLayoutTest.cpp.exe | FileCheck --check-prefix=CLASS %s
 
 CHECK: Module [[MOD:.*]]
-CHECK: SymbolFile pdb ([[MOD]])
+CHECK: SymbolFile {{(native-)?}}pdb ([[MOD]])
 CHECK: {{^[0-9A-F]+}}:   CompileUnit{{[{]0x[0-9a-f]+[}]}}, language = "c++", file = '{{.*}}\ClassLayoutTest.cpp'
 
 ENUM:  name = "Enum", size = 4,  decl = ClassLayoutTest.cpp:5
diff --git a/lldb/test/Shell/SymbolFile/PDB/enums-layout.test b/lldb/test/Shell/SymbolFile/PDB/enums-layout.test
index 6f861c6d65adf..9766d6f8b0324 100644
--- a/lldb/test/Shell/SymbolFile/PDB/enums-layout.test
+++ b/lldb/test/Shell/SymbolFile/PDB/enums-layout.test
@@ -7,6 +7,12 @@ RUN: lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-
 RUN: lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-prefix=UCHAR-ENUM %s
 RUN: lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-prefix=CLASS-ENUM %s
 RUN: lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-prefix=STRUCT-ENUM %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-prefix=ENUM %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-prefix=CONST-ENUM %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-prefix=EMPTY-ENUM %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-prefix=UCHAR-ENUM %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-prefix=CLASS-ENUM %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 lldb-test symbols %t.dir/SimpleTypesTest.cpp.enums.exe | FileCheck --check-prefix=STRUCT-ENUM %s
 
 ; FIXME: PDB does not have information about scoped enumeration (Enum class) so the  
 ; compiler type used is the same as the one for unscoped enumeration.
diff --git a/lldb/test/Shell/SymbolFile/PDB/vbases.test b/lldb/test/Shell/SymbolFile/PDB/vbases.test
index b58e3edc3cc80..e5ab80b0205a4 100644
--- a/lldb/test/Shell/SymbolFile/PDB/vbases.test
+++ b/lldb/test/Shell/SymbolFile/PDB/vbases.test
@@ -1,6 +1,7 @@
 REQUIRES: target-windows, lld
 RUN: %build --compiler=clang-cl --output=%t.exe %S/Inputs/VBases.cpp
 RUN: %lldb -b -s %S/Inputs/VBases.script -- %t.exe | FileCheck %s
+RUN: env LLDB_USE_NATIVE_PDB_READER=1 %lldb -b -s %S/Inputs/VBases.script -- %t.exe | FileCheck %s
 
 CHECK: {
 CHECK:   A = (a = '\x01')
diff --git a/lldb/test/Shell/Symtab/Inputs/simple.wasm.yaml b/lldb/test/Shell/Symtab/Inputs/simple.wasm.yaml
new file mode 100644
index 0000000000000..088d6163d6b0b
--- /dev/null
+++ b/lldb/test/Shell/Symtab/Inputs/simple.wasm.yaml
@@ -0,0 +1,254 @@
+# clang -target wasm32 -nostdlib -Wl,--no-entry -Wl,--export-all -O0 -g -o simple.wasm simple.c
+# char* str = "data str";
+#
+# int add(int a, int b) {
+#   return a + b;
+# }
+#
+# int main() {
+#   int i = 1;
+#   int j = 2;
+#   return add(i, j);
+# }
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:     []
+      - Index:           1
+        ParamTypes:
+          - I32
+          - I32
+        ReturnTypes:
+          - I32
+      - Index:           2
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1, 2, 1 ]
+  - Type:            TABLE
+    Tables:
+      - Index:           0
+        ElemType:        FUNCREF
+        Limits:
+          Flags:           [ HAS_MAX ]
+          Minimum:         0x1
+          Maximum:         0x1
+  - Type:            MEMORY
+    Memories:
+      - Minimum:         0x2
+  - Type:            GLOBAL
+    Globals:
+      - Index:           0
+        Type:            I32
+        Mutable:         true
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           66576
+      - Index:           1
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           1036
+      - Index:           2
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           1024
+      - Index:           3
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           1040
+      - Index:           4
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           1040
+      - Index:           5
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           66576
+      - Index:           6
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           1024
+      - Index:           7
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           66576
+      - Index:           8
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           131072
+      - Index:           9
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           0
+      - Index:           10
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           1
+      - Index:           11
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           65536
+  - Type:            EXPORT
+    Exports:
+      - Name:            memory
+        Kind:            MEMORY
+        Index:           0
+      - Name:            __wasm_call_ctors
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            add
+        Kind:            FUNCTION
+        Index:           1
+      - Name:            __original_main
+        Kind:            FUNCTION
+        Index:           2
+      - Name:            main
+        Kind:            FUNCTION
+        Index:           3
+      - Name:            str
+        Kind:            GLOBAL
+        Index:           1
+      - Name:            __main_void
+        Kind:            FUNCTION
+        Index:           2
+      - Name:            __indirect_function_table
+        Kind:            TABLE
+        Index:           0
+      - Name:            __dso_handle
+        Kind:            GLOBAL
+        Index:           2
+      - Name:            __data_end
+        Kind:            GLOBAL
+        Index:           3
+      - Name:            __stack_low
+        Kind:            GLOBAL
+        Index:           4
+      - Name:            __stack_high
+        Kind:            GLOBAL
+        Index:           5
+      - Name:            __global_base
+        Kind:            GLOBAL
+        Index:           6
+      - Name:            __heap_base
+        Kind:            GLOBAL
+        Index:           7
+      - Name:            __heap_end
+        Kind:            GLOBAL
+        Index:           8
+      - Name:            __memory_base
+        Kind:            GLOBAL
+        Index:           9
+      - Name:            __table_base
+        Kind:            GLOBAL
+        Index:           10
+      - Name:            __wasm_first_page_end
+        Kind:            GLOBAL
+        Index:           11
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            0B
+      - Index:           1
+        Locals:
+          - Type:            I32
+            Count:           1
+        Body:            23808080800041106B21022002200036020C20022001360208200228020C20022802086A0F0B
+      - Index:           2
+        Locals:
+          - Type:            I32
+            Count:           2
+        Body:            23808080800041106B210020002480808080002000410036020C2000410136020820004102360204200028020820002802041081808080002101200041106A24808080800020010F0B
+      - Index:           3
+        Locals:          []
+        Body:            1082808080000F0B
+  - Type:            DATA
+    Segments:
+      - SectionOffset:   7
+        InitFlags:       0
+        Offset:
+          Opcode:          I32_CONST
+          Value:           1024
+        Content:         '646174612073747200'
+      - SectionOffset:   22
+        InitFlags:       0
+        Offset:
+          Opcode:          I32_CONST
+          Value:           1036
+        Content:         '00040000'
+  - Type:            CUSTOM
+    Name:            name
+    FunctionNames:
+      - Index:           0
+        Name:            __wasm_call_ctors
+      - Index:           1
+        Name:            add
+      - Index:           2
+        Name:            __original_main
+      - Index:           3
+        Name:            main
+    GlobalNames:
+      - Index:           0
+        Name:            __stack_pointer
+    DataSegmentNames:
+      - Index:           0
+        Name:            .rodata
+      - Index:           1
+        Name:            .data
+  - Type:            CUSTOM
+    HeaderSecSizeEncodingLen: 2
+    Name:            producers
+    Languages:
+      - Name:            C11
+        Version:         ''
+    Tools:
+      - Name:            clang
+        Version:         '22.0.0git'
+  - Type:            CUSTOM
+    Name:            target_features
+    Features:
+      - Prefix:          USED
+        Name:            bulk-memory
+      - Prefix:          USED
+        Name:            bulk-memory-opt
+      - Prefix:          USED
+        Name:            call-indirect-overlong
+      - Prefix:          USED
+        Name:            multivalue
+      - Prefix:          USED
+        Name:            mutable-globals
+      - Prefix:          USED
+        Name:            nontrapping-fptoint
+      - Prefix:          USED
+        Name:            reference-types
+      - Prefix:          USED
+        Name:            sign-ext
+...
diff --git a/lldb/test/Shell/Symtab/symtab-wasm.test b/lldb/test/Shell/Symtab/symtab-wasm.test
new file mode 100644
index 0000000000000..4170d9aba9eea
--- /dev/null
+++ b/lldb/test/Shell/Symtab/symtab-wasm.test
@@ -0,0 +1,15 @@
+# RUN: yaml2obj %S/Inputs/simple.wasm.yaml -o %t.wasm
+# RUN: %lldb %t.wasm -o 'image dump symtab' -o 'image dump sections' | FileCheck %s
+
+CHECK: Code 0x0000000000000002 0x0000000000000002 0x00000000 __wasm_call_ctors
+CHECK: Code 0x0000000000000005 0x0000000000000029 0x00000000 add
+CHECK: Code 0x000000000000002f 0x000000000000004c 0x00000000 __original_main
+CHECK: Code 0x000000000000007c 0x0000000000000009 0x00000000 main
+CHECK: Data 0x0000000000000233 0x0000000000000009 0x00000000 .rodata
+CHECK: Data 0x0000000000000242 0x0000000000000004 0x00000000 .data
+
+CHECK: 0x0000000000000001 code       {{.*}} 0x000001a1 0x00000085 0x00000000 symtab-wasm.test.tmp.wasm.code
+CHECK: 0x0000000000000003 data       {{.*}} 0x0000022c 0x0000001a 0x00000000 symtab-wasm.test.tmp.wasm.data
+CHECK: 0x0000000000000040 wasm-name  {{.*}} 0x00000251 0x00000059 0x00000000 symtab-wasm.test.tmp.wasm.name
+CHECK: 0x0000000000000100 data       {{.*}} 0x00000233 0x00000009 0x00000000 symtab-wasm.test.tmp.wasm.data..rodata
+CHECK: 0x0000000000000200 data       {{.*}} 0x00000242 0x00000004 0x00000000 symtab-wasm.test.tmp.wasm.data..data
diff --git a/lldb/tools/debugserver/source/debugserver.cpp b/lldb/tools/debugserver/source/debugserver.cpp
index f41a9e00ec948..2c32fe98cf1ba 100644
--- a/lldb/tools/debugserver/source/debugserver.cpp
+++ b/lldb/tools/debugserver/source/debugserver.cpp
@@ -792,6 +792,39 @@ void show_usage_and_exit(int exit_code) {
                DEBUGSERVER_PROGRAM_NAME);
   RNBLogSTDERR("  %s /path/file --attach=<process_name>\n",
                DEBUGSERVER_PROGRAM_NAME);
+  RNBLogSTDERR("\n");
+  RNBLogSTDERR("  -a | --attach <pid>\n");
+  RNBLogSTDERR("  -w | --waitfor <name>\n");
+  RNBLogSTDERR("  -A | --arch <arch>\n");
+  RNBLogSTDERR("  -g | --debug\n");
+  RNBLogSTDERR("  -K | --kill-on-error\n");
+  RNBLogSTDERR("  -v | --verbose\n");
+  RNBLogSTDERR("  -V | --version\n");
+  RNBLogSTDERR("  -k | --lockdown\n");
+  RNBLogSTDERR("  -t | --applist\n");
+  RNBLogSTDERR("  -l | --log-file\n");
+  RNBLogSTDERR("  -f | --log-flags\n");
+  RNBLogSTDERR("  -x | --launch <auto|posix-spawn|fork-exec|springboard>\n");
+  RNBLogSTDERR("  -d | --waitfor-duration <seconds>\n");
+  RNBLogSTDERR("  -i | --waitfor-interval <usecs>\n");
+  RNBLogSTDERR("  -r | --native-regs\n");
+  RNBLogSTDERR("  -s | --studio-path <path>\n");
+  RNBLogSTDERR("  -I | --stdin-path <path>\n");
+  RNBLogSTDERR("  -O | --stdout-path <path>\n");
+  RNBLogSTDERR("  -E | --stderr-path <path>\n");
+  RNBLogSTDERR("  -n | --no-stdio\n");
+  RNBLogSTDERR("  -S | --setsid\n");
+  RNBLogSTDERR("  -D | --disable-aslr\n");
+  RNBLogSTDERR("  -W | --working-dir <dir>\n");
+  RNBLogSTDERR("  -p | --platform <arg?>\n");
+  RNBLogSTDERR("  -u | --unix-socket <unix socket name>\n");
+  RNBLogSTDERR("  -2 | --fd <file descriptor number>\n");
+  RNBLogSTDERR("  -P | --named-pipe <pipe>\n");
+  RNBLogSTDERR("  -R | --reverse-connect\n");
+  RNBLogSTDERR("  -e | --env <env>\n");
+  RNBLogSTDERR("  -F | --forward-env <env>\n");
+  RNBLogSTDERR("  -U | --unmask-signals\n");
+
   exit(exit_code);
 }
 
diff --git a/lldb/tools/driver/CMakeLists.txt b/lldb/tools/driver/CMakeLists.txt
index 5c0cac484cc94..446bf68d09622 100644
--- a/lldb/tools/driver/CMakeLists.txt
+++ b/lldb/tools/driver/CMakeLists.txt
@@ -11,7 +11,7 @@ if(APPLE)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-sectcreate,__TEXT,__info_plist,${CMAKE_CURRENT_BINARY_DIR}/lldb-Info.plist")
 endif()
 
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")                               
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   remove_definitions("-D_XOPEN_SOURCE=700")                                    
   add_definitions("-D_ALL_SOURCE")                                             
 endif()
diff --git a/lldb/tools/lldb-dap/Breakpoint.cpp b/lldb/tools/lldb-dap/Breakpoint.cpp
index b4e593eb83d27..c8039576b29bd 100644
--- a/lldb/tools/lldb-dap/Breakpoint.cpp
+++ b/lldb/tools/lldb-dap/Breakpoint.cpp
@@ -8,10 +8,14 @@
 
 #include "Breakpoint.h"
 #include "DAP.h"
+#include "LLDBUtils.h"
+#include "Protocol/DAPTypes.h"
 #include "ProtocolUtils.h"
 #include "lldb/API/SBAddress.h"
 #include "lldb/API/SBBreakpointLocation.h"
+#include "lldb/API/SBFileSpec.h"
 #include "lldb/API/SBLineEntry.h"
+#include "lldb/API/SBModule.h"
 #include "lldb/API/SBMutex.h"
 #include "llvm/ADT/StringExtras.h"
 #include <cstddef>
@@ -21,6 +25,22 @@
 
 using namespace lldb_dap;
 
+static std::optional<protocol::PersistenceData>
+GetPersistenceDataForSymbol(lldb::SBSymbol &symbol) {
+  protocol::PersistenceData persistence_data;
+  lldb::SBModule module = symbol.GetStartAddress().GetModule();
+  if (!module.IsValid())
+    return std::nullopt;
+
+  lldb::SBFileSpec file_spec = module.GetFileSpec();
+  if (!file_spec.IsValid())
+    return std::nullopt;
+
+  persistence_data.module_path = GetSBFileSpecPath(file_spec);
+  persistence_data.symbol_name = symbol.GetName();
+  return persistence_data;
+}
+
 void Breakpoint::SetCondition() { m_bp.SetCondition(m_condition.c_str()); }
 
 void Breakpoint::SetHitCondition() {
@@ -73,7 +93,7 @@ protocol::Breakpoint Breakpoint::ToProtocolBreakpoint() {
       const auto column = line_entry.GetColumn();
       if (column != LLDB_INVALID_COLUMN_NUMBER)
         breakpoint.column = column;
-    } else {
+    } else if (source) {
       // Assembly breakpoint.
       auto symbol = bp_addr.GetSymbol();
       if (symbol.IsValid()) {
@@ -82,6 +102,15 @@ protocol::Breakpoint Breakpoint::ToProtocolBreakpoint() {
                 .ReadInstructions(symbol.GetStartAddress(), bp_addr, nullptr)
                 .GetSize() +
             1;
+
+        // Add persistent data so that the breakpoint can be resolved
+        // in future sessions.
+        std::optional<protocol::PersistenceData> persistence_data =
+            GetPersistenceDataForSymbol(symbol);
+        if (persistence_data) {
+          source->adapterData =
+              protocol::SourceLLDBData{std::move(persistence_data)};
+        }
       }
     }
 
diff --git a/lldb/tools/lldb-dap/CMakeLists.txt b/lldb/tools/lldb-dap/CMakeLists.txt
index 4cddfb1bea1c2..7db334ca56bcf 100644
--- a/lldb/tools/lldb-dap/CMakeLists.txt
+++ b/lldb/tools/lldb-dap/CMakeLists.txt
@@ -45,6 +45,7 @@ add_lldb_library(lldbDAP
   Handler/LaunchRequestHandler.cpp
   Handler/LocationsRequestHandler.cpp
   Handler/ModulesRequestHandler.cpp
+  Handler/ModuleSymbolsRequestHandler.cpp
   Handler/NextRequestHandler.cpp
   Handler/PauseRequestHandler.cpp
   Handler/ReadMemoryRequestHandler.cpp
@@ -66,7 +67,8 @@ add_lldb_library(lldbDAP
   Handler/ThreadsRequestHandler.cpp
   Handler/VariablesRequestHandler.cpp
   Handler/WriteMemoryRequestHandler.cpp
-  
+
+  Protocol/DAPTypes.cpp
   Protocol/ProtocolBase.cpp
   Protocol/ProtocolEvents.cpp
   Protocol/ProtocolTypes.cpp
diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp
index cbd3b14463e25..b1ad38d983893 100644
--- a/lldb/tools/lldb-dap/DAP.cpp
+++ b/lldb/tools/lldb-dap/DAP.cpp
@@ -23,13 +23,14 @@
 #include "Transport.h"
 #include "lldb/API/SBBreakpoint.h"
 #include "lldb/API/SBCommandInterpreter.h"
-#include "lldb/API/SBCommandReturnObject.h"
 #include "lldb/API/SBEvent.h"
 #include "lldb/API/SBLanguageRuntime.h"
 #include "lldb/API/SBListener.h"
 #include "lldb/API/SBProcess.h"
 #include "lldb/API/SBStream.h"
-#include "lldb/Utility/IOObject.h"
+#include "lldb/Host/JSONTransport.h"
+#include "lldb/Host/MainLoop.h"
+#include "lldb/Host/MainLoopBase.h"
 #include "lldb/Utility/Status.h"
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
@@ -52,7 +53,7 @@
 #include <cstdarg>
 #include <cstdint>
 #include <cstdio>
-#include <fstream>
+#include <functional>
 #include <future>
 #include <memory>
 #include <mutex>
@@ -120,11 +121,12 @@ static std::string capitalize(llvm::StringRef str) {
 llvm::StringRef DAP::debug_adapter_path = "";
 
 DAP::DAP(Log *log, const ReplMode default_repl_mode,
-         std::vector<std::string> pre_init_commands, Transport &transport)
+         std::vector<std::string> pre_init_commands,
+         llvm::StringRef client_name, DAPTransport &transport, MainLoop &loop)
     : log(log), transport(transport), broadcaster("lldb-dap"),
       progress_event_reporter(
           [&](const ProgressEvent &event) { SendJSON(event.ToJSON()); }),
-      repl_mode(default_repl_mode) {
+      repl_mode(default_repl_mode), m_client_name(client_name), m_loop(loop) {
   configuration.preInitCommands = std::move(pre_init_commands);
   RegisterRequests();
 }
@@ -257,36 +259,49 @@ void DAP::SendJSON(const llvm::json::Value &json) {
   llvm::json::Path::Root root;
   if (!fromJSON(json, message, root)) {
     DAP_LOG_ERROR(log, root.getError(), "({1}) encoding failed: {0}",
-                  transport.GetClientName());
+                  m_client_name);
     return;
   }
   Send(message);
 }
 
 void DAP::Send(const Message &message) {
-  // FIXME: After all the requests have migrated from LegacyRequestHandler >
-  // RequestHandler<> this should be handled in RequestHandler<>::operator().
-  if (auto *resp = std::get_if<Response>(&message);
-      resp && debugger.InterruptRequested()) {
-    // Clear the interrupt request.
-    debugger.CancelInterruptRequest();
-
-    // If the debugger was interrupted, convert this response into a 'cancelled'
-    // response because we might have a partial result.
-    Response cancelled{/*request_seq=*/resp->request_seq,
-                       /*command=*/resp->command,
-                       /*success=*/false,
-                       /*message=*/eResponseMessageCancelled,
-                       /*body=*/std::nullopt};
-    if (llvm::Error err = transport.Write(cancelled))
-      DAP_LOG_ERROR(log, std::move(err), "({1}) write failed: {0}",
-                    transport.GetClientName());
+  if (const protocol::Event *event = std::get_if<protocol::Event>(&message)) {
+    if (llvm::Error err = transport.Send(*event))
+      DAP_LOG_ERROR(log, std::move(err), "({0}) sending event failed",
+                    m_client_name);
     return;
   }
 
-  if (llvm::Error err = transport.Write(message))
-    DAP_LOG_ERROR(log, std::move(err), "({1}) write failed: {0}",
-                  transport.GetClientName());
+  if (const Request *req = std::get_if<Request>(&message)) {
+    if (llvm::Error err = transport.Send(*req))
+      DAP_LOG_ERROR(log, std::move(err), "({0}) sending request failed",
+                    m_client_name);
+    return;
+  }
+
+  if (const Response *resp = std::get_if<Response>(&message)) {
+    // FIXME: After all the requests have migrated from LegacyRequestHandler >
+    // RequestHandler<> this should be handled in RequestHandler<>::operator().
+    // If the debugger was interrupted, convert this response into a
+    // 'cancelled' response because we might have a partial result.
+    llvm::Error err =
+        (debugger.InterruptRequested())
+            ? transport.Send({/*request_seq=*/resp->request_seq,
+                              /*command=*/resp->command,
+                              /*success=*/false,
+                              /*message=*/eResponseMessageCancelled,
+                              /*body=*/std::nullopt})
+            : transport.Send(*resp);
+    if (err) {
+      DAP_LOG_ERROR(log, std::move(err), "({0}) sending response failed",
+                    m_client_name);
+      return;
+    }
+    return;
+  }
+
+  llvm_unreachable("Unexpected message type");
 }
 
 // "OutputEvent": {
@@ -551,6 +566,9 @@ lldb::SBThread DAP::GetLLDBThread(const llvm::json::Object &arguments) {
 }
 
 lldb::SBFrame DAP::GetLLDBFrame(uint64_t frame_id) {
+  if (frame_id == LLDB_DAP_INVALID_FRAME_ID)
+    return lldb::SBFrame();
+
   lldb::SBProcess process = target.GetProcess();
   // Upper 32 bits is the thread index ID
   lldb::SBThread thread =
@@ -560,8 +578,8 @@ lldb::SBFrame DAP::GetLLDBFrame(uint64_t frame_id) {
 }
 
 lldb::SBFrame DAP::GetLLDBFrame(const llvm::json::Object &arguments) {
-  const auto frame_id =
-      GetInteger<uint64_t>(arguments, "frameId").value_or(UINT64_MAX);
+  const auto frame_id = GetInteger<uint64_t>(arguments, "frameId")
+                            .value_or(LLDB_DAP_INVALID_FRAME_ID);
   return GetLLDBFrame(frame_id);
 }
 
@@ -754,7 +772,6 @@ void DAP::RunTerminateCommands() {
 }
 
 lldb::SBTarget DAP::CreateTarget(lldb::SBError &error) {
-  // Grab the name of the program we need to debug and create a target using
   // the given program as an argument. Executable file can be a source of target
   // architecture and platform, if they differ from the host. Setting exe path
   // in launch info is useless because Target.Launch() will not change
@@ -794,7 +811,7 @@ void DAP::SetTarget(const lldb::SBTarget target) {
 
 bool DAP::HandleObject(const Message &M) {
   TelemetryDispatcher dispatcher(&debugger);
-  dispatcher.Set("client_name", transport.GetClientName().str());
+  dispatcher.Set("client_name", m_client_name.str());
   if (const auto *req = std::get_if<Request>(&M)) {
     {
       std::lock_guard<std::mutex> guard(m_active_request_mutex);
@@ -820,8 +837,8 @@ bool DAP::HandleObject(const Message &M) {
 
     dispatcher.Set("error",
                    llvm::Twine("unhandled-command:" + req->command).str());
-    DAP_LOG(log, "({0}) error: unhandled command '{1}'",
-            transport.GetClientName(), req->command);
+    DAP_LOG(log, "({0}) error: unhandled command '{1}'", m_client_name,
+            req->command);
     return false; // Fail
   }
 
@@ -917,9 +934,7 @@ llvm::Error DAP::Disconnect(bool terminateDebuggee) {
   }
 
   SendTerminatedEvent();
-
-  disconnecting = true;
-
+  TerminateLoop();
   return ToError(error);
 }
 
@@ -935,91 +950,121 @@ void DAP::ClearCancelRequest(const CancelArguments &args) {
 }
 
 template <typename T>
-static std::optional<T> getArgumentsIfRequest(const Message &pm,
+static std::optional<T> getArgumentsIfRequest(const Request &req,
                                               llvm::StringLiteral command) {
-  auto *const req = std::get_if<Request>(&pm);
-  if (!req || req->command != command)
+  if (req.command != command)
     return std::nullopt;
 
   T args;
   llvm::json::Path::Root root;
-  if (!fromJSON(req->arguments, args, root))
+  if (!fromJSON(req.arguments, args, root))
     return std::nullopt;
 
   return args;
 }
 
-llvm::Error DAP::Loop() {
-  // Can't use \a std::future<llvm::Error> because it doesn't compile on
-  // Windows.
-  std::future<lldb::SBError> queue_reader =
-      std::async(std::launch::async, [&]() -> lldb::SBError {
-        llvm::set_thread_name(transport.GetClientName() + ".transport_handler");
-        auto cleanup = llvm::make_scope_exit([&]() {
-          // Ensure we're marked as disconnecting when the reader exits.
-          disconnecting = true;
-          m_queue_cv.notify_all();
-        });
-
-        while (!disconnecting) {
-          llvm::Expected<Message> next =
-              transport.Read<protocol::Message>(std::chrono::seconds(1));
-          if (next.errorIsA<TransportEOFError>()) {
-            consumeError(next.takeError());
-            break;
-          }
+void DAP::Received(const protocol::Event &event) {
+  // no-op, no supported events from the client to the server as of DAP v1.68.
+}
 
-          // If the read timed out, continue to check if we should disconnect.
-          if (next.errorIsA<TransportTimeoutError>()) {
-            consumeError(next.takeError());
-            continue;
-          }
+void DAP::Received(const protocol::Request &request) {
+  if (request.command == "disconnect")
+    m_disconnecting = true;
 
-          if (llvm::Error err = next.takeError()) {
-            lldb::SBError errWrapper;
-            errWrapper.SetErrorString(llvm::toString(std::move(err)).c_str());
-            return errWrapper;
-          }
+  const std::optional<CancelArguments> cancel_args =
+      getArgumentsIfRequest<CancelArguments>(request, "cancel");
+  if (cancel_args) {
+    {
+      std::lock_guard<std::mutex> guard(m_cancelled_requests_mutex);
+      if (cancel_args->requestId)
+        m_cancelled_requests.insert(*cancel_args->requestId);
+    }
 
-          if (const protocol::Request *req =
-                  std::get_if<protocol::Request>(&*next);
-              req && req->command == "disconnect")
-            disconnecting = true;
-
-          const std::optional<CancelArguments> cancel_args =
-              getArgumentsIfRequest<CancelArguments>(*next, "cancel");
-          if (cancel_args) {
-            {
-              std::lock_guard<std::mutex> guard(m_cancelled_requests_mutex);
-              if (cancel_args->requestId)
-                m_cancelled_requests.insert(*cancel_args->requestId);
-            }
+    // If a cancel is requested for the active request, make a best
+    // effort attempt to interrupt.
+    std::lock_guard<std::mutex> guard(m_active_request_mutex);
+    if (m_active_request && cancel_args->requestId == m_active_request->seq) {
+      DAP_LOG(log, "({0}) interrupting inflight request (command={1} seq={2})",
+              m_client_name, m_active_request->command, m_active_request->seq);
+      debugger.RequestInterrupt();
+    }
+  }
 
-            // If a cancel is requested for the active request, make a best
-            // effort attempt to interrupt.
-            std::lock_guard<std::mutex> guard(m_active_request_mutex);
-            if (m_active_request &&
-                cancel_args->requestId == m_active_request->seq) {
-              DAP_LOG(
-                  log,
-                  "({0}) interrupting inflight request (command={1} seq={2})",
-                  transport.GetClientName(), m_active_request->command,
-                  m_active_request->seq);
-              debugger.RequestInterrupt();
-            }
-          }
+  std::lock_guard<std::mutex> guard(m_queue_mutex);
+  DAP_LOG(log, "({0}) queued (command={1} seq={2})", m_client_name,
+          request.command, request.seq);
+  m_queue.push_back(request);
+  m_queue_cv.notify_one();
+}
 
-          {
-            std::lock_guard<std::mutex> guard(m_queue_mutex);
-            m_queue.push_back(std::move(*next));
-          }
-          m_queue_cv.notify_one();
-        }
+void DAP::Received(const protocol::Response &response) {
+  std::lock_guard<std::mutex> guard(m_queue_mutex);
+  DAP_LOG(log, "({0}) queued (command={1} seq={2})", m_client_name,
+          response.command, response.request_seq);
+  m_queue.push_back(response);
+  m_queue_cv.notify_one();
+}
+
+void DAP::OnError(llvm::Error error) {
+  DAP_LOG_ERROR(log, std::move(error), "({1}) received error: {0}",
+                m_client_name);
+  TerminateLoop(/*failed=*/true);
+}
+
+void DAP::OnClosed() {
+  DAP_LOG(log, "({0}) received EOF", m_client_name);
+  TerminateLoop();
+}
 
-        return lldb::SBError();
-      });
+void DAP::TerminateLoop(bool failed) {
+  std::lock_guard<std::mutex> guard(m_queue_mutex);
+  if (m_disconnecting)
+    return; // Already disconnecting.
 
-  auto cleanup = llvm::make_scope_exit([&]() {
+  m_error_occurred = failed;
+  m_disconnecting = true;
+  m_loop.AddPendingCallback(
+      [](MainLoopBase &loop) { loop.RequestTermination(); });
+}
+
+void DAP::TransportHandler() {
+  auto scope_guard = llvm::make_scope_exit([this] {
+    std::lock_guard<std::mutex> guard(m_queue_mutex);
+    // Ensure we're marked as disconnecting when the reader exits.
+    m_disconnecting = true;
+    m_queue_cv.notify_all();
+  });
+
+  auto handle = transport.RegisterMessageHandler(m_loop, *this);
+  if (!handle) {
+    DAP_LOG_ERROR(log, handle.takeError(),
+                  "({1}) registering message handler failed: {0}",
+                  m_client_name);
+    std::lock_guard<std::mutex> guard(m_queue_mutex);
+    m_error_occurred = true;
+    return;
+  }
+
+  if (Status status = m_loop.Run(); status.Fail()) {
+    DAP_LOG_ERROR(log, status.takeError(), "({1}) MainLoop run failed: {0}",
+                  m_client_name);
+    std::lock_guard<std::mutex> guard(m_queue_mutex);
+    m_error_occurred = true;
+    return;
+  }
+}
+
+llvm::Error DAP::Loop() {
+  {
+    // Reset disconnect flag once we start the loop.
+    std::lock_guard<std::mutex> guard(m_queue_mutex);
+    m_disconnecting = false;
+  }
+
+  auto thread = std::thread(std::bind(&DAP::TransportHandler, this));
+
+  auto cleanup = llvm::make_scope_exit([this]() {
+    // FIXME: Merge these into the MainLoop handler.
     out.Stop();
     err.Stop();
     StopEventHandlers();
@@ -1027,9 +1072,9 @@ llvm::Error DAP::Loop() {
 
   while (true) {
     std::unique_lock<std::mutex> lock(m_queue_mutex);
-    m_queue_cv.wait(lock, [&] { return disconnecting || !m_queue.empty(); });
+    m_queue_cv.wait(lock, [&] { return m_disconnecting || !m_queue.empty(); });
 
-    if (disconnecting && m_queue.empty())
+    if (m_disconnecting && m_queue.empty())
       break;
 
     Message next = m_queue.front();
@@ -1043,7 +1088,15 @@ llvm::Error DAP::Loop() {
                                      "unhandled packet");
   }
 
-  return ToError(queue_reader.get());
+  m_loop.AddPendingCallback(
+      [](MainLoopBase &loop) { loop.RequestTermination(); });
+  thread.join();
+
+  if (m_error_occurred)
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "DAP Loop terminated due to an internal "
+                                   "error, see DAP Logs for more information.");
+  return llvm::Error::success();
 }
 
 lldb::SBError DAP::WaitForProcessToStop(std::chrono::seconds seconds) {
@@ -1208,6 +1261,27 @@ protocol::Capabilities DAP::GetCapabilities() {
   return capabilities;
 }
 
+protocol::Capabilities DAP::GetCustomCapabilities() {
+  protocol::Capabilities capabilities;
+
+  // Add all custom capabilities here.
+  const llvm::DenseSet<AdapterFeature> all_custom_features = {
+      protocol::eAdapterFeatureSupportsModuleSymbolsRequest,
+  };
+
+  for (auto &kv : request_handlers) {
+    llvm::SmallDenseSet<AdapterFeature, 1> features =
+        kv.second->GetSupportedFeatures();
+
+    for (auto &feature : features) {
+      if (all_custom_features.contains(feature))
+        capabilities.supportedFeatures.insert(feature);
+    }
+  }
+
+  return capabilities;
+}
+
 void DAP::StartEventThread() {
   event_thread = std::thread(&DAP::EventThread, this);
 }
@@ -1282,7 +1356,7 @@ void DAP::ProgressEventThread() {
 // them prevent multiple threads from writing simultaneously so no locking
 // is required.
 void DAP::EventThread() {
-  llvm::set_thread_name(transport.GetClientName() + ".event_handler");
+  llvm::set_thread_name("lldb.DAP.client." + m_client_name + ".event_handler");
   lldb::SBEvent event;
   lldb::SBListener listener = debugger.GetListener();
   broadcaster.AddListener(listener, eBroadcastBitStopEventThread);
@@ -1314,7 +1388,7 @@ void DAP::EventThread() {
               if (llvm::Error err = SendThreadStoppedEvent(*this))
                 DAP_LOG_ERROR(log, std::move(err),
                               "({1}) reporting thread stopped: {0}",
-                              transport.GetClientName());
+                              m_client_name);
             }
             break;
           case lldb::eStateRunning:
@@ -1406,11 +1480,15 @@ void DAP::EventThread() {
             // avoids sending paths that should be source mapped. Note that
             // CreateBreakpoint doesn't apply source mapping and certain
             // implementation ignore the source part of this event anyway.
-            llvm::json::Value source_bp = bp.ToProtocolBreakpoint();
-            source_bp.getAsObject()->erase("source");
+            protocol::Breakpoint protocol_bp = bp.ToProtocolBreakpoint();
+
+            // "source" is not needed here, unless we add adapter data to be
+            // saved by the client.
+            if (protocol_bp.source && !protocol_bp.source->adapterData)
+              protocol_bp.source = std::nullopt;
 
             llvm::json::Object body;
-            body.try_emplace("breakpoint", source_bp);
+            body.try_emplace("breakpoint", protocol_bp);
             body.try_emplace("reason", "changed");
 
             llvm::json::Object bp_event = CreateEventObject("breakpoint");
@@ -1491,8 +1569,9 @@ std::vector<protocol::Breakpoint> DAP::SetSourceBreakpoints(
 
       protocol::Breakpoint response_breakpoint =
           iv->second.ToProtocolBreakpoint();
-      response_breakpoint.source = source;
 
+      if (!response_breakpoint.source)
+        response_breakpoint.source = source;
       if (!response_breakpoint.line &&
           src_bp.GetLine() != LLDB_INVALID_LINE_NUMBER)
         response_breakpoint.line = src_bp.GetLine();
@@ -1559,6 +1638,7 @@ void DAP::RegisterRequests() {
   // Custom requests
   RegisterRequest<CompileUnitsRequestHandler>();
   RegisterRequest<ModulesRequestHandler>();
+  RegisterRequest<ModuleSymbolsRequestHandler>();
 
   // Testing requests
   RegisterRequest<TestGetTargetBreakpointsRequestHandler>();
diff --git a/lldb/tools/lldb-dap/DAP.h b/lldb/tools/lldb-dap/DAP.h
index af4aabaafaae8..04f70f76a09cd 100644
--- a/lldb/tools/lldb-dap/DAP.h
+++ b/lldb/tools/lldb-dap/DAP.h
@@ -31,6 +31,8 @@
 #include "lldb/API/SBMutex.h"
 #include "lldb/API/SBTarget.h"
 #include "lldb/API/SBThread.h"
+#include "lldb/Host/MainLoop.h"
+#include "lldb/Utility/Status.h"
 #include "lldb/lldb-types.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
@@ -76,12 +78,16 @@ enum DAPBroadcasterBits {
 
 enum class ReplMode { Variable = 0, Command, Auto };
 
-struct DAP {
+using DAPTransport =
+    lldb_private::Transport<protocol::Request, protocol::Response,
+                            protocol::Event>;
+
+struct DAP final : private DAPTransport::MessageHandler {
   /// Path to the lldb-dap binary itself.
   static llvm::StringRef debug_adapter_path;
 
   Log *log;
-  Transport &transport;
+  DAPTransport &transport;
   lldb::SBFile in;
   OutputRedirector out;
   OutputRedirector err;
@@ -112,7 +118,6 @@ struct DAP {
   /// The focused thread for this DAP session.
   lldb::tid_t focus_tid = LLDB_INVALID_THREAD_ID;
 
-  bool disconnecting = false;
   llvm::once_flag terminated_event_flag;
   bool stop_at_entry = false;
   bool is_attach = false;
@@ -175,8 +180,11 @@ struct DAP {
   ///     allocated.
   /// \param[in] transport
   ///     Transport for this debug session.
+  /// \param[in] loop
+  ///     Main loop associated with this instance.
   DAP(Log *log, const ReplMode default_repl_mode,
-      std::vector<std::string> pre_init_commands, Transport &transport);
+      std::vector<std::string> pre_init_commands, llvm::StringRef client_name,
+      DAPTransport &transport, lldb_private::MainLoop &loop);
 
   ~DAP();
 
@@ -315,7 +323,7 @@ struct DAP {
   lldb::SBTarget CreateTarget(lldb::SBError &error);
 
   /// Set given target object as a current target for lldb-dap and start
-  /// listeing for its breakpoint events.
+  /// listening for its breakpoint events.
   void SetTarget(const lldb::SBTarget target);
 
   bool HandleObject(const protocol::Message &M);
@@ -359,6 +367,9 @@ struct DAP {
   /// The set of capabilities supported by this adapter.
   protocol::Capabilities GetCapabilities();
 
+  /// The set of custom capabilities supported by this adapter.
+  protocol::Capabilities GetCustomCapabilities();
+
   /// Debuggee will continue from stopped state.
   void WillContinue() { variables.Clear(); }
 
@@ -418,12 +429,21 @@ struct DAP {
       const std::optional<std::vector<protocol::SourceBreakpoint>>
           &breakpoints);
 
+  void Received(const protocol::Event &) override;
+  void Received(const protocol::Request &) override;
+  void Received(const protocol::Response &) override;
+  void OnError(llvm::Error) override;
+  void OnClosed() override;
+
 private:
   std::vector<protocol::Breakpoint> SetSourceBreakpoints(
       const protocol::Source &source,
       const std::optional<std::vector<protocol::SourceBreakpoint>> &breakpoints,
       SourceBreakpointMap &existing_breakpoints);
 
+  void TransportHandler();
+  void TerminateLoop(bool failed = false);
+
   /// Registration of request handler.
   /// @{
   void RegisterRequests();
@@ -442,6 +462,8 @@ struct DAP {
   std::thread progress_event_thread;
   /// @}
 
+  const llvm::StringRef m_client_name;
+
   /// List of addresses mapped by sourceReference.
   std::vector<lldb::addr_t> m_source_references;
   std::mutex m_source_references_mutex;
@@ -450,6 +472,11 @@ struct DAP {
   std::deque<protocol::Message> m_queue;
   std::mutex m_queue_mutex;
   std::condition_variable m_queue_cv;
+  bool m_disconnecting = false;
+  bool m_error_occurred = false;
+
+  // Loop for managing reading from the client.
+  lldb_private::MainLoop &m_loop;
 
   std::mutex m_cancelled_requests_mutex;
   llvm::SmallSet<int64_t, 4> m_cancelled_requests;
diff --git a/lldb/tools/lldb-dap/EventHelper.cpp b/lldb/tools/lldb-dap/EventHelper.cpp
index 364cc7ab4ef8c..bfb05a387d04d 100644
--- a/lldb/tools/lldb-dap/EventHelper.cpp
+++ b/lldb/tools/lldb-dap/EventHelper.cpp
@@ -38,25 +38,37 @@ static void SendThreadExitedEvent(DAP &dap, lldb::tid_t tid) {
   dap.SendJSON(llvm::json::Value(std::move(event)));
 }
 
-void SendTargetBasedCapabilities(DAP &dap) {
+/// Get capabilities based on the configured target.
+static llvm::DenseSet<AdapterFeature> GetTargetBasedCapabilities(DAP &dap) {
+  llvm::DenseSet<AdapterFeature> capabilities;
   if (!dap.target.IsValid())
-    return;
-
-  protocol::CapabilitiesEventBody body;
+    return capabilities;
 
   const llvm::StringRef target_triple = dap.target.GetTriple();
   if (target_triple.starts_with("x86"))
-    body.capabilities.supportedFeatures.insert(
-        protocol::eAdapterFeatureStepInTargetsRequest);
+    capabilities.insert(protocol::eAdapterFeatureStepInTargetsRequest);
 
   // We only support restarting launch requests not attach requests.
   if (dap.last_launch_request)
-    body.capabilities.supportedFeatures.insert(
-        protocol::eAdapterFeatureRestartRequest);
+    capabilities.insert(protocol::eAdapterFeatureRestartRequest);
+
+  return capabilities;
+}
+
+void SendExtraCapabilities(DAP &dap) {
+  protocol::Capabilities capabilities = dap.GetCustomCapabilities();
+  llvm::DenseSet<AdapterFeature> target_capabilities =
+      GetTargetBasedCapabilities(dap);
+
+  capabilities.supportedFeatures.insert(target_capabilities.begin(),
+                                        target_capabilities.end());
+
+  protocol::CapabilitiesEventBody body;
+  body.capabilities = std::move(capabilities);
 
   // Only notify the client if supportedFeatures changed.
   if (!body.capabilities.supportedFeatures.empty())
-    dap.Send(protocol::Event{"capabilities", body});
+    dap.Send(protocol::Event{"capabilities", std::move(body)});
 }
 
 // "ProcessEvent": {
diff --git a/lldb/tools/lldb-dap/EventHelper.h b/lldb/tools/lldb-dap/EventHelper.h
index 72ad5308a2b0c..592c1b81c46af 100644
--- a/lldb/tools/lldb-dap/EventHelper.h
+++ b/lldb/tools/lldb-dap/EventHelper.h
@@ -17,8 +17,8 @@ struct DAP;
 
 enum LaunchMethod { Launch, Attach, AttachForSuspendedLaunch };
 
-/// Update capabilities based on the configured target.
-void SendTargetBasedCapabilities(DAP &dap);
+/// Sends target based capabilities and lldb-dap custom capabilities.
+void SendExtraCapabilities(DAP &dap);
 
 void SendProcessEvent(DAP &dap, LaunchMethod launch_method);
 
diff --git a/lldb/tools/lldb-dap/Handler/CompletionsHandler.cpp b/lldb/tools/lldb-dap/Handler/CompletionsHandler.cpp
index c72fc5686cd5b..de9a15dcb73f4 100644
--- a/lldb/tools/lldb-dap/Handler/CompletionsHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/CompletionsHandler.cpp
@@ -8,156 +8,46 @@
 
 #include "DAP.h"
 #include "JSONUtils.h"
+#include "Protocol/ProtocolRequests.h"
+#include "Protocol/ProtocolTypes.h"
 #include "RequestHandler.h"
 #include "lldb/API/SBStringList.h"
 
-namespace lldb_dap {
+using namespace llvm;
+using namespace lldb_dap;
+using namespace lldb_dap::protocol;
 
-// "CompletionsRequest": {
-//   "allOf": [ { "$ref": "#/definitions/Request" }, {
-//     "type": "object",
-//     "description": "Returns a list of possible completions for a given caret
-//     position and text.\nThe CompletionsRequest may only be called if the
-//     'supportsCompletionsRequest' capability exists and is true.",
-//     "properties": {
-//       "command": {
-//         "type": "string",
-//         "enum": [ "completions" ]
-//       },
-//       "arguments": {
-//         "$ref": "#/definitions/CompletionsArguments"
-//       }
-//     },
-//     "required": [ "command", "arguments"  ]
-//   }]
-// },
-// "CompletionsArguments": {
-//   "type": "object",
-//   "description": "Arguments for 'completions' request.",
-//   "properties": {
-//     "frameId": {
-//       "type": "integer",
-//       "description": "Returns completions in the scope of this stack frame.
-//       If not specified, the completions are returned for the global scope."
-//     },
-//     "text": {
-//       "type": "string",
-//       "description": "One or more source lines. Typically this is the text a
-//       user has typed into the debug console before he asked for completion."
-//     },
-//     "column": {
-//       "type": "integer",
-//       "description": "The character position for which to determine the
-//       completion proposals."
-//     },
-//     "line": {
-//       "type": "integer",
-//       "description": "An optional line for which to determine the completion
-//       proposals. If missing the first line of the text is assumed."
-//     }
-//   },
-//   "required": [ "text", "column" ]
-// },
-// "CompletionsResponse": {
-//   "allOf": [ { "$ref": "#/definitions/Response" }, {
-//     "type": "object",
-//     "description": "Response to 'completions' request.",
-//     "properties": {
-//       "body": {
-//         "type": "object",
-//         "properties": {
-//           "targets": {
-//             "type": "array",
-//             "items": {
-//               "$ref": "#/definitions/CompletionItem"
-//             },
-//             "description": "The possible completions for ."
-//           }
-//         },
-//         "required": [ "targets" ]
-//       }
-//     },
-//     "required": [ "body" ]
-//   }]
-// },
-// "CompletionItem": {
-//   "type": "object",
-//   "description": "CompletionItems are the suggestions returned from the
-//   CompletionsRequest.", "properties": {
-//     "label": {
-//       "type": "string",
-//       "description": "The label of this completion item. By default this is
-//       also the text that is inserted when selecting this completion."
-//     },
-//     "text": {
-//       "type": "string",
-//       "description": "If text is not falsy then it is inserted instead of the
-//       label."
-//     },
-//     "sortText": {
-//       "type": "string",
-//       "description": "A string that should be used when comparing this item
-//       with other items. When `falsy` the label is used."
-//     },
-//     "type": {
-//       "$ref": "#/definitions/CompletionItemType",
-//       "description": "The item's type. Typically the client uses this
-//       information to render the item in the UI with an icon."
-//     },
-//     "start": {
-//       "type": "integer",
-//       "description": "This value determines the location (in the
-//       CompletionsRequest's 'text' attribute) where the completion text is
-//       added.\nIf missing the text is added at the location specified by the
-//       CompletionsRequest's 'column' attribute."
-//     },
-//     "length": {
-//       "type": "integer",
-//       "description": "This value determines how many characters are
-//       overwritten by the completion text.\nIf missing the value 0 is assumed
-//       which results in the completion text being inserted."
-//     }
-//   },
-//   "required": [ "label" ]
-// },
-// "CompletionItemType": {
-//   "type": "string",
-//   "description": "Some predefined types for the CompletionItem. Please note
-//   that not all clients have specific icons for all of them.", "enum": [
-//   "method", "function", "constructor", "field", "variable", "class",
-//   "interface", "module", "property", "unit", "value", "enum", "keyword",
-//   "snippet", "text", "color", "file", "reference", "customcolor" ]
-// }
-void CompletionsRequestHandler::operator()(
-    const llvm::json::Object &request) const {
-  llvm::json::Object response;
-  FillResponse(request, response);
-  llvm::json::Object body;
-  const auto *arguments = request.getObject("arguments");
+namespace lldb_dap {
 
+/// Returns a list of possible completions for a given caret position and text.
+///
+/// Clients should only call this request if the corresponding capability
+/// `supportsCompletionsRequest` is true.
+Expected<CompletionsResponseBody>
+CompletionsRequestHandler::Run(const CompletionsArguments &args) const {
   // If we have a frame, try to set the context for variable completions.
-  lldb::SBFrame frame = dap.GetLLDBFrame(*arguments);
+  lldb::SBFrame frame = dap.GetLLDBFrame(args.frameId);
   if (frame.IsValid()) {
     frame.GetThread().GetProcess().SetSelectedThread(frame.GetThread());
     frame.GetThread().SetSelectedFrame(frame.GetFrameID());
   }
 
-  std::string text = GetString(arguments, "text").value_or("").str();
-  auto original_column =
-      GetInteger<int64_t>(arguments, "column").value_or(text.size());
-  auto original_line = GetInteger<int64_t>(arguments, "line").value_or(1);
+  std::string text = args.text;
+  auto original_column = args.column;
+  auto original_line = args.line;
   auto offset = original_column - 1;
   if (original_line > 1) {
-    llvm::SmallVector<::llvm::StringRef, 2> lines;
-    llvm::StringRef(text).split(lines, '\n');
+    SmallVector<StringRef, 2> lines;
+    StringRef(text).split(lines, '\n');
     for (int i = 0; i < original_line - 1; i++) {
       offset += lines[i].size();
     }
   }
-  llvm::json::Array targets;
+
+  std::vector<CompletionItem> targets;
 
   bool had_escape_prefix =
-      llvm::StringRef(text).starts_with(dap.configuration.commandEscapePrefix);
+      StringRef(text).starts_with(dap.configuration.commandEscapePrefix);
   ReplMode completion_mode = dap.DetectReplMode(frame, text, true);
 
   // Handle the offset change introduced by stripping out the
@@ -165,10 +55,7 @@ void CompletionsRequestHandler::operator()(
   if (had_escape_prefix) {
     if (offset <
         static_cast<int64_t>(dap.configuration.commandEscapePrefix.size())) {
-      body.try_emplace("targets", std::move(targets));
-      response.try_emplace("body", std::move(body));
-      dap.SendJSON(llvm::json::Value(std::move(response)));
-      return;
+      return CompletionsResponseBody{std::move(targets)};
     }
     offset -= dap.configuration.commandEscapePrefix.size();
   }
@@ -198,27 +85,25 @@ void CompletionsRequestHandler::operator()(
       std::string match = matches.GetStringAtIndex(i);
       std::string description = descriptions.GetStringAtIndex(i);
 
-      llvm::json::Object item;
-      llvm::StringRef match_ref = match;
-      for (llvm::StringRef commit_point : {".", "->"}) {
+      CompletionItem item;
+      StringRef match_ref = match;
+      for (StringRef commit_point : {".", "->"}) {
         if (match_ref.contains(commit_point)) {
           match_ref = match_ref.rsplit(commit_point).second;
         }
       }
-      EmplaceSafeString(item, "text", match_ref);
+      item.text = match_ref;
 
       if (description.empty())
-        EmplaceSafeString(item, "label", match);
+        item.label = match;
       else
-        EmplaceSafeString(item, "label", match + " -- " + description);
+        item.label = match + " -- " + description;
 
       targets.emplace_back(std::move(item));
     }
   }
 
-  body.try_emplace("targets", std::move(targets));
-  response.try_emplace("body", std::move(body));
-  dap.SendJSON(llvm::json::Value(std::move(response)));
+  return CompletionsResponseBody{std::move(targets)};
 }
 
 } // namespace lldb_dap
diff --git a/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp
index e7735a705d0aa..1bfe7b7f6ef5c 100644
--- a/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/ConfigurationDoneRequestHandler.cpp
@@ -9,6 +9,7 @@
 #include "DAP.h"
 #include "EventHelper.h"
 #include "LLDBUtils.h"
+#include "Protocol/ProtocolEvents.h"
 #include "Protocol/ProtocolRequests.h"
 #include "ProtocolUtils.h"
 #include "RequestHandler.h"
@@ -44,7 +45,10 @@ ConfigurationDoneRequestHandler::Run(const ConfigurationDoneArguments &) const {
   // Waiting until 'configurationDone' to send target based capabilities in case
   // the launch or attach scripts adjust the target. The initial dummy target
   // may have different capabilities than the final target.
-  SendTargetBasedCapabilities(dap);
+
+  /// Also send here custom capabilities to the client, which is consumed by the
+  /// lldb-dap specific editor extension.
+  SendExtraCapabilities(dap);
 
   // Clients can request a baseline of currently existing threads after
   // we acknowledge the configurationDone request.
diff --git a/lldb/tools/lldb-dap/Handler/DataBreakpointInfoRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/DataBreakpointInfoRequestHandler.cpp
index 8cb25d0603449..87b93fc999ecd 100644
--- a/lldb/tools/lldb-dap/Handler/DataBreakpointInfoRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/DataBreakpointInfoRequestHandler.cpp
@@ -23,7 +23,7 @@ llvm::Expected<protocol::DataBreakpointInfoResponseBody>
 DataBreakpointInfoRequestHandler::Run(
     const protocol::DataBreakpointInfoArguments &args) const {
   protocol::DataBreakpointInfoResponseBody response;
-  lldb::SBFrame frame = dap.GetLLDBFrame(args.frameId.value_or(UINT64_MAX));
+  lldb::SBFrame frame = dap.GetLLDBFrame(args.frameId);
   lldb::SBValue variable = dap.variables.FindVariable(
       args.variablesReference.value_or(0), args.name);
   std::string addr, size;
diff --git a/lldb/tools/lldb-dap/Handler/ModuleSymbolsRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/ModuleSymbolsRequestHandler.cpp
new file mode 100644
index 0000000000000..4a9d256cfa975
--- /dev/null
+++ b/lldb/tools/lldb-dap/Handler/ModuleSymbolsRequestHandler.cpp
@@ -0,0 +1,90 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DAP.h"
+#include "DAPError.h"
+#include "Protocol/DAPTypes.h"
+#include "RequestHandler.h"
+#include "lldb/API/SBAddress.h"
+#include "lldb/API/SBFileSpec.h"
+#include "lldb/API/SBModule.h"
+#include "lldb/API/SBModuleSpec.h"
+#include "lldb/Utility/UUID.h"
+#include "llvm/Support/Error.h"
+#include <cstddef>
+
+using namespace lldb_dap::protocol;
+namespace lldb_dap {
+
+llvm::Expected<ModuleSymbolsResponseBody>
+ModuleSymbolsRequestHandler::Run(const ModuleSymbolsArguments &args) const {
+  ModuleSymbolsResponseBody response;
+
+  lldb::SBModuleSpec module_spec;
+  if (!args.moduleId.empty()) {
+    llvm::SmallVector<uint8_t, 20> uuid_bytes;
+    if (!lldb_private::UUID::DecodeUUIDBytesFromString(args.moduleId,
+                                                       uuid_bytes)
+             .empty())
+      return llvm::make_error<DAPError>("invalid module ID");
+
+    module_spec.SetUUIDBytes(uuid_bytes.data(), uuid_bytes.size());
+  }
+
+  if (!args.moduleName.empty()) {
+    lldb::SBFileSpec file_spec;
+    file_spec.SetFilename(args.moduleName.c_str());
+    module_spec.SetFileSpec(file_spec);
+  }
+
+  // Empty request, return empty response.
+  if (!module_spec.IsValid())
+    return response;
+
+  std::vector<Symbol> &symbols = response.symbols;
+  lldb::SBModule module = dap.target.FindModule(module_spec);
+  if (!module.IsValid())
+    return llvm::make_error<DAPError>("module not found");
+
+  const size_t num_symbols = module.GetNumSymbols();
+  const size_t start_index = args.startIndex.value_or(0);
+  const size_t end_index =
+      std::min(start_index + args.count.value_or(num_symbols), num_symbols);
+  for (size_t i = start_index; i < end_index; ++i) {
+    lldb::SBSymbol symbol = module.GetSymbolAtIndex(i);
+    if (!symbol.IsValid())
+      continue;
+
+    Symbol dap_symbol;
+    dap_symbol.id = symbol.GetID();
+    dap_symbol.type = symbol.GetType();
+    dap_symbol.isDebug = symbol.IsDebug();
+    dap_symbol.isSynthetic = symbol.IsSynthetic();
+    dap_symbol.isExternal = symbol.IsExternal();
+
+    lldb::SBAddress start_address = symbol.GetStartAddress();
+    if (start_address.IsValid()) {
+      lldb::addr_t file_address = start_address.GetFileAddress();
+      if (file_address != LLDB_INVALID_ADDRESS)
+        dap_symbol.fileAddress = file_address;
+
+      lldb::addr_t load_address = start_address.GetLoadAddress(dap.target);
+      if (load_address != LLDB_INVALID_ADDRESS)
+        dap_symbol.loadAddress = load_address;
+    }
+
+    dap_symbol.size = symbol.GetSize();
+    if (const char *symbol_name = symbol.GetName())
+      dap_symbol.name = symbol_name;
+    symbols.push_back(std::move(dap_symbol));
+  }
+
+  return response;
+}
+
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-dap/Handler/RequestHandler.h b/lldb/tools/lldb-dap/Handler/RequestHandler.h
index 16f8062f97d7b..977a247996750 100644
--- a/lldb/tools/lldb-dap/Handler/RequestHandler.h
+++ b/lldb/tools/lldb-dap/Handler/RequestHandler.h
@@ -243,14 +243,17 @@ class BreakpointLocationsRequestHandler
                                  uint32_t end_line) const;
 };
 
-class CompletionsRequestHandler : public LegacyRequestHandler {
+class CompletionsRequestHandler
+    : public RequestHandler<protocol::CompletionsArguments,
+                            llvm::Expected<protocol::CompletionsResponseBody>> {
 public:
-  using LegacyRequestHandler::LegacyRequestHandler;
+  using RequestHandler::RequestHandler;
   static llvm::StringLiteral GetCommand() { return "completions"; }
   FeatureSet GetSupportedFeatures() const override {
     return {protocol::eAdapterFeatureCompletionsRequest};
   }
-  void operator()(const llvm::json::Object &request) const override;
+  llvm::Expected<protocol::CompletionsResponseBody>
+  Run(const protocol::CompletionsArguments &args) const override;
 };
 
 class ContinueRequestHandler
@@ -594,6 +597,20 @@ class CancelRequestHandler : public RequestHandler<protocol::CancelArguments,
   llvm::Error Run(const protocol::CancelArguments &args) const override;
 };
 
+class ModuleSymbolsRequestHandler
+    : public RequestHandler<
+          protocol::ModuleSymbolsArguments,
+          llvm::Expected<protocol::ModuleSymbolsResponseBody>> {
+public:
+  using RequestHandler::RequestHandler;
+  static llvm::StringLiteral GetCommand() { return "__lldb_moduleSymbols"; }
+  FeatureSet GetSupportedFeatures() const override {
+    return {protocol::eAdapterFeatureSupportsModuleSymbolsRequest};
+  }
+  llvm::Expected<protocol::ModuleSymbolsResponseBody>
+  Run(const protocol::ModuleSymbolsArguments &args) const override;
+};
+
 /// A request used in testing to get the details on all breakpoints that are
 /// currently set in the target. This helps us to test "setBreakpoints" and
 /// "setFunctionBreakpoints" requests to verify we have the correct set of
diff --git a/lldb/tools/lldb-dap/Handler/SetBreakpointsRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/SetBreakpointsRequestHandler.cpp
index 5d336af740c99..142351fd62179 100644
--- a/lldb/tools/lldb-dap/Handler/SetBreakpointsRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/SetBreakpointsRequestHandler.cpp
@@ -9,7 +9,6 @@
 #include "DAP.h"
 #include "Protocol/ProtocolRequests.h"
 #include "RequestHandler.h"
-#include <vector>
 
 namespace lldb_dap {
 
diff --git a/lldb/tools/lldb-dap/Protocol/DAPTypes.cpp b/lldb/tools/lldb-dap/Protocol/DAPTypes.cpp
new file mode 100644
index 0000000000000..a14ed9e521f48
--- /dev/null
+++ b/lldb/tools/lldb-dap/Protocol/DAPTypes.cpp
@@ -0,0 +1,69 @@
+#include "Protocol/DAPTypes.h"
+#include "lldb/API/SBSymbol.h"
+#include "lldb/lldb-enumerations.h"
+
+using namespace llvm;
+
+namespace lldb_dap::protocol {
+
+bool fromJSON(const llvm::json::Value &Params, PersistenceData &PD,
+              llvm::json::Path P) {
+  json::ObjectMapper O(Params, P);
+  return O && O.mapOptional("module_path", PD.module_path) &&
+         O.mapOptional("symbol_name", PD.symbol_name);
+}
+
+llvm::json::Value toJSON(const PersistenceData &PD) {
+  json::Object result{
+      {"module_path", PD.module_path},
+      {"symbol_name", PD.symbol_name},
+  };
+
+  return result;
+}
+
+bool fromJSON(const llvm::json::Value &Params, SourceLLDBData &SLD,
+              llvm::json::Path P) {
+  json::ObjectMapper O(Params, P);
+  return O && O.mapOptional("persistenceData", SLD.persistenceData);
+}
+
+llvm::json::Value toJSON(const SourceLLDBData &SLD) {
+  json::Object result;
+  if (SLD.persistenceData)
+    result.insert({"persistenceData", SLD.persistenceData});
+  return result;
+}
+
+bool fromJSON(const llvm::json::Value &Params, Symbol &DS, llvm::json::Path P) {
+  json::ObjectMapper O(Params, P);
+  std::string type_str;
+  if (!(O && O.map("id", DS.id) && O.map("isDebug", DS.isDebug) &&
+        O.map("isSynthetic", DS.isSynthetic) &&
+        O.map("isExternal", DS.isExternal) && O.map("type", type_str) &&
+        O.map("fileAddress", DS.fileAddress) &&
+        O.mapOptional("loadAddress", DS.loadAddress) &&
+        O.map("size", DS.size) && O.map("name", DS.name)))
+    return false;
+
+  DS.type = lldb::SBSymbol::GetTypeFromString(type_str.c_str());
+  return true;
+}
+
+llvm::json::Value toJSON(const Symbol &DS) {
+  json::Object result{
+      {"id", DS.id},
+      {"isDebug", DS.isDebug},
+      {"isSynthetic", DS.isSynthetic},
+      {"isExternal", DS.isExternal},
+      {"type", lldb::SBSymbol::GetTypeAsString(DS.type)},
+      {"fileAddress", DS.fileAddress},
+      {"loadAddress", DS.loadAddress},
+      {"size", DS.size},
+      {"name", DS.name},
+  };
+
+  return result;
+}
+
+} // namespace lldb_dap::protocol
diff --git a/lldb/tools/lldb-dap/Protocol/DAPTypes.h b/lldb/tools/lldb-dap/Protocol/DAPTypes.h
new file mode 100644
index 0000000000000..7fccf1359a737
--- /dev/null
+++ b/lldb/tools/lldb-dap/Protocol/DAPTypes.h
@@ -0,0 +1,85 @@
+//===-- ProtocolTypes.h ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains private DAP types used in the protocol.
+//
+// Each struct has a toJSON and fromJSON function, that converts between
+// the struct and a JSON representation. (See JSON.h)
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_TOOLS_LLDB_DAP_PROTOCOL_DAP_TYPES_H
+#define LLDB_TOOLS_LLDB_DAP_PROTOCOL_DAP_TYPES_H
+
+#include "lldb/lldb-types.h"
+#include "llvm/Support/JSON.h"
+#include <optional>
+#include <string>
+
+namespace lldb_dap::protocol {
+
+/// Data used to help lldb-dap resolve breakpoints persistently across different
+/// sessions. This information is especially useful for assembly breakpoints,
+/// because `sourceReference` can change across sessions. For regular source
+/// breakpoints the path and line are the same For each session.
+struct PersistenceData {
+  /// The source module path.
+  std::string module_path;
+
+  /// The symbol name of the Source.
+  std::string symbol_name;
+};
+bool fromJSON(const llvm::json::Value &, PersistenceData &, llvm::json::Path);
+llvm::json::Value toJSON(const PersistenceData &);
+
+/// Custom source data used by lldb-dap.
+/// This data should help lldb-dap identify sources correctly across different
+/// sessions.
+struct SourceLLDBData {
+  /// Data that helps lldb resolve this source persistently across different
+  /// sessions.
+  std::optional<PersistenceData> persistenceData;
+};
+bool fromJSON(const llvm::json::Value &, SourceLLDBData &, llvm::json::Path);
+llvm::json::Value toJSON(const SourceLLDBData &);
+
+struct Symbol {
+  /// The symbol id, usually the original symbol table index.
+  uint32_t id;
+
+  /// True if this symbol is debug information in a symbol.
+  bool isDebug;
+
+  /// True if this symbol is not actually in the symbol table, but synthesized
+  /// from other info in the object file.
+  bool isSynthetic;
+
+  /// True if this symbol is globally visible.
+  bool isExternal;
+
+  /// The symbol type.
+  lldb::SymbolType type;
+
+  /// The symbol file address.
+  lldb::addr_t fileAddress;
+
+  /// The symbol load address.
+  std::optional<lldb::addr_t> loadAddress;
+
+  /// The symbol size.
+  lldb::addr_t size;
+
+  /// The symbol name.
+  std::string name;
+};
+bool fromJSON(const llvm::json::Value &, Symbol &, llvm::json::Path);
+llvm::json::Value toJSON(const Symbol &);
+
+} // namespace lldb_dap::protocol
+
+#endif
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp
index bc4fee4aa8b8d..9cd9028d879e9 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolBase.cpp
@@ -98,6 +98,10 @@ bool fromJSON(json::Value const &Params, Request &R, json::Path P) {
   return mapRaw(Params, "arguments", R.arguments, P);
 }
 
+bool operator==(const Request &a, const Request &b) {
+  return a.seq == b.seq && a.command == b.command && a.arguments == b.arguments;
+}
+
 json::Value toJSON(const Response &R) {
   json::Object Result{{"type", "response"},
                       {"seq", 0},
@@ -177,6 +181,11 @@ bool fromJSON(json::Value const &Params, Response &R, json::Path P) {
          mapRaw(Params, "body", R.body, P);
 }
 
+bool operator==(const Response &a, const Response &b) {
+  return a.request_seq == b.request_seq && a.command == b.command &&
+         a.success == b.success && a.message == b.message && a.body == b.body;
+}
+
 json::Value toJSON(const ErrorMessage &EM) {
   json::Object Result{{"id", EM.id}, {"format", EM.format}};
 
@@ -248,6 +257,10 @@ bool fromJSON(json::Value const &Params, Event &E, json::Path P) {
   return mapRaw(Params, "body", E.body, P);
 }
 
+bool operator==(const Event &a, const Event &b) {
+  return a.event == b.event && a.body == b.body;
+}
+
 bool fromJSON(const json::Value &Params, Message &PM, json::Path P) {
   json::ObjectMapper O(Params, P);
   if (!O)
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolBase.h b/lldb/tools/lldb-dap/Protocol/ProtocolBase.h
index 81496380d412f..0a9ef538a7398 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolBase.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolBase.h
@@ -52,6 +52,7 @@ struct Request {
 };
 llvm::json::Value toJSON(const Request &);
 bool fromJSON(const llvm::json::Value &, Request &, llvm::json::Path);
+bool operator==(const Request &, const Request &);
 
 /// A debug adapter initiated event.
 struct Event {
@@ -63,6 +64,7 @@ struct Event {
 };
 llvm::json::Value toJSON(const Event &);
 bool fromJSON(const llvm::json::Value &, Event &, llvm::json::Path);
+bool operator==(const Event &, const Event &);
 
 enum ResponseMessage : unsigned {
   /// The request was cancelled
@@ -101,6 +103,7 @@ struct Response {
 };
 bool fromJSON(const llvm::json::Value &, Response &, llvm::json::Path);
 llvm::json::Value toJSON(const Response &);
+bool operator==(const Response &, const Response &);
 
 /// A structured message object. Used to return errors from requests.
 struct ErrorMessage {
@@ -140,6 +143,7 @@ llvm::json::Value toJSON(const ErrorMessage &);
 using Message = std::variant<Request, Response, Event>;
 bool fromJSON(const llvm::json::Value &, Message &, llvm::json::Path);
 llvm::json::Value toJSON(const Message &);
+bool operator==(const Message &, const Message &);
 
 inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Message &V) {
   OS << toJSON(V);
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
index 29855ca50e9e0..eab7211e18973 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
@@ -329,6 +329,17 @@ json::Value toJSON(const ContinueResponseBody &CRB) {
   return std::move(Body);
 }
 
+bool fromJSON(const json::Value &Params, CompletionsArguments &CA,
+              json::Path P) {
+  json::ObjectMapper O(Params, P);
+  return O && O.map("text", CA.text) && O.map("column", CA.column) &&
+         O.mapOptional("frameId", CA.frameId) && O.mapOptional("line", CA.line);
+}
+
+json::Value toJSON(const CompletionsResponseBody &CRB) {
+  return json::Object{{"targets", CRB.targets}};
+}
+
 bool fromJSON(const json::Value &Params, SetVariableArguments &SVA,
               json::Path P) {
   json::ObjectMapper O(Params, P);
@@ -598,4 +609,19 @@ json::Value toJSON(const WriteMemoryResponseBody &WMR) {
   return result;
 }
 
+bool fromJSON(const llvm::json::Value &Params, ModuleSymbolsArguments &Args,
+              llvm::json::Path P) {
+  json::ObjectMapper O(Params, P);
+  return O && O.map("moduleId", Args.moduleId) &&
+         O.map("moduleName", Args.moduleName) &&
+         O.mapOptional("startIndex", Args.startIndex) &&
+         O.mapOptional("count", Args.count);
+}
+
+llvm::json::Value toJSON(const ModuleSymbolsResponseBody &DGMSR) {
+  json::Object result;
+  result.insert({"symbols", DGMSR.symbols});
+  return result;
+}
+
 } // namespace lldb_dap::protocol
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
index c45ee10e77d1c..0848ee53b4410 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
@@ -310,6 +310,8 @@ bool fromJSON(const llvm::json::Value &, LaunchRequestArguments &,
 using LaunchResponse = VoidResponse;
 
 #define LLDB_DAP_INVALID_PORT -1
+/// An invalid 'frameId' default value.
+#define LLDB_DAP_INVALID_FRAME_ID UINT64_MAX
 
 /// lldb-dap specific attach arguments.
 struct AttachRequestArguments {
@@ -376,6 +378,35 @@ struct ContinueResponseBody {
 };
 llvm::json::Value toJSON(const ContinueResponseBody &);
 
+/// Arguments for `completions` request.
+struct CompletionsArguments {
+  /// Returns completions in the scope of this stack frame. If not specified,
+  /// the completions are returned for the global scope.
+  uint64_t frameId = LLDB_DAP_INVALID_FRAME_ID;
+
+  /// One or more source lines. Typically this is the text users have typed into
+  /// the debug console before they asked for completion.
+  std::string text;
+
+  /// The position within `text` for which to determine the completion
+  /// proposals. It is measured in UTF-16 code units and the client capability
+  /// `columnsStartAt1` determines whether it is 0- or 1-based.
+  int64_t column = 0;
+
+  /// A line for which to determine the completion proposals. If missing the
+  /// first line of the text is assumed.
+  int64_t line = 0;
+};
+bool fromJSON(const llvm::json::Value &, CompletionsArguments &,
+              llvm::json::Path);
+
+/// Response to `completions` request.
+struct CompletionsResponseBody {
+  /// The possible completions for a given caret position and text.
+  std::vector<CompletionItem> targets;
+};
+llvm::json::Value toJSON(const CompletionsResponseBody &);
+
 /// Arguments for `configurationDone` request.
 using ConfigurationDoneArguments = EmptyArguments;
 
@@ -455,7 +486,7 @@ struct ScopesArguments {
   /// Retrieve the scopes for the stack frame identified by `frameId`. The
   /// `frameId` must have been obtained in the current suspended state. See
   /// 'Lifetime of Object References' in the Overview section for details.
-  uint64_t frameId = LLDB_INVALID_FRAME_ID;
+  uint64_t frameId = LLDB_DAP_INVALID_FRAME_ID;
 };
 bool fromJSON(const llvm::json::Value &, ScopesArguments &, llvm::json::Path);
 
@@ -541,7 +572,7 @@ using StepInResponse = VoidResponse;
 /// Arguments for `stepInTargets` request.
 struct StepInTargetsArguments {
   /// The stack frame for which to retrieve the possible step-in targets.
-  uint64_t frameId = LLDB_INVALID_FRAME_ID;
+  uint64_t frameId = LLDB_DAP_INVALID_FRAME_ID;
 };
 bool fromJSON(const llvm::json::Value &, StepInTargetsArguments &,
               llvm::json::Path);
@@ -690,7 +721,7 @@ struct DataBreakpointInfoArguments {
   /// When `name` is an expression, evaluate it in the scope of this stack
   /// frame. If not specified, the expression is evaluated in the global scope.
   /// When `asAddress` is true, the `frameId` is ignored.
-  std::optional<uint64_t> frameId;
+  uint64_t frameId = LLDB_DAP_INVALID_FRAME_ID;
 
   /// If specified, a debug adapter should return information for the range of
   /// memory extending `bytes` number of bytes from the address or variable
@@ -981,6 +1012,30 @@ struct WriteMemoryResponseBody {
 };
 llvm::json::Value toJSON(const WriteMemoryResponseBody &);
 
+struct ModuleSymbolsArguments {
+  /// The module UUID for which to retrieve symbols.
+  std::string moduleId;
+
+  /// The module path.
+  std::string moduleName;
+
+  /// The index of the first symbol to return; if omitted, start at the
+  /// beginning.
+  std::optional<uint32_t> startIndex;
+
+  /// The number of symbols to return; if omitted, all symbols are returned.
+  std::optional<uint32_t> count;
+};
+bool fromJSON(const llvm::json::Value &, ModuleSymbolsArguments &,
+              llvm::json::Path);
+
+/// Response to `getModuleSymbols` request.
+struct ModuleSymbolsResponseBody {
+  /// The symbols for the specified module.
+  std::vector<Symbol> symbols;
+};
+llvm::json::Value toJSON(const ModuleSymbolsResponseBody &);
+
 } // namespace lldb_dap::protocol
 
 #endif
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
index fe8bb5252cc23..dc8edaadcd9bb 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.cpp
@@ -46,7 +46,8 @@ bool fromJSON(const json::Value &Params, Source &S, json::Path P) {
   json::ObjectMapper O(Params, P);
   return O && O.map("name", S.name) && O.map("path", S.path) &&
          O.map("presentationHint", S.presentationHint) &&
-         O.map("sourceReference", S.sourceReference);
+         O.map("sourceReference", S.sourceReference) &&
+         O.map("adapterData", S.adapterData);
 }
 
 llvm::json::Value toJSON(Source::PresentationHint hint) {
@@ -71,6 +72,8 @@ llvm::json::Value toJSON(const Source &S) {
     result.insert({"sourceReference", *S.sourceReference});
   if (S.presentationHint)
     result.insert({"presentationHint", *S.presentationHint});
+  if (S.adapterData)
+    result.insert({"adapterData", *S.adapterData});
 
   return result;
 }
@@ -197,6 +200,123 @@ bool fromJSON(const llvm::json::Value &Params, ChecksumAlgorithm &CA,
   return true;
 }
 
+bool fromJSON(const json::Value &Params, CompletionItemType &CIT,
+              json::Path P) {
+  auto raw_item_type = Params.getAsString();
+  if (!raw_item_type) {
+    P.report("expected a string");
+    return false;
+  }
+
+  std::optional<CompletionItemType> item_type =
+      StringSwitch<std::optional<CompletionItemType>>(*raw_item_type)
+          .Case("method", eCompletionItemTypeMethod)
+          .Case("function", eCompletionItemTypeFunction)
+          .Case("constructor", eCompletionItemTypeConstructor)
+          .Case("field", eCompletionItemTypeField)
+          .Case("variable", eCompletionItemTypeVariable)
+          .Case("class", eCompletionItemTypeClass)
+          .Case("interface", eCompletionItemTypeInterface)
+          .Case("module", eCompletionItemTypeModule)
+          .Case("property", eCompletionItemTypeProperty)
+          .Case("unit", eCompletionItemTypeUnit)
+          .Case("value", eCompletionItemTypeValue)
+          .Case("enum", eCompletionItemTypeEnum)
+          .Case("keyword", eCompletionItemTypeKeyword)
+          .Case("snippet", eCompletionItemTypeSnippet)
+          .Case("text", eCompletionItemTypeText)
+          .Case("color", eCompletionItemTypeColor)
+          .Case("file", eCompletionItemTypeFile)
+          .Case("reference", eCompletionItemTypeReference)
+          .Case("customcolor", eCompletionItemTypeCustomColor)
+          .Default(std::nullopt);
+
+  if (!item_type) {
+    P.report("unexpected value");
+    return false;
+  }
+
+  CIT = *item_type;
+  return true;
+}
+
+json::Value toJSON(const CompletionItemType &CIT) {
+  switch (CIT) {
+  case eCompletionItemTypeMethod:
+    return "method";
+  case eCompletionItemTypeFunction:
+    return "function";
+  case eCompletionItemTypeConstructor:
+    return "constructor";
+  case eCompletionItemTypeField:
+    return "field";
+  case eCompletionItemTypeVariable:
+    return "variable";
+  case eCompletionItemTypeClass:
+    return "class";
+  case eCompletionItemTypeInterface:
+    return "interface";
+  case eCompletionItemTypeModule:
+    return "module";
+  case eCompletionItemTypeProperty:
+    return "property";
+  case eCompletionItemTypeUnit:
+    return "unit";
+  case eCompletionItemTypeValue:
+    return "value";
+  case eCompletionItemTypeEnum:
+    return "enum";
+  case eCompletionItemTypeKeyword:
+    return "keyword";
+  case eCompletionItemTypeSnippet:
+    return "snippet";
+  case eCompletionItemTypeText:
+    return "text";
+  case eCompletionItemTypeColor:
+    return "color";
+  case eCompletionItemTypeFile:
+    return "file";
+  case eCompletionItemTypeReference:
+    return "reference";
+  case eCompletionItemTypeCustomColor:
+    return "customcolor";
+  }
+  llvm_unreachable("unhandled CompletionItemType.");
+}
+
+bool fromJSON(const json::Value &Params, CompletionItem &CI, json::Path P) {
+  json::ObjectMapper O(Params, P);
+  return O && O.map("label", CI.label) && O.mapOptional("text", CI.text) &&
+         O.mapOptional("sortText", CI.sortText) &&
+         O.mapOptional("detail", CI.detail) && O.mapOptional("type", CI.type) &&
+         O.mapOptional("start", CI.start) &&
+         O.mapOptional("length", CI.length) &&
+         O.mapOptional("selectionStart", CI.selectionStart) &&
+         O.mapOptional("selectionLength", CI.selectionLength);
+}
+json::Value toJSON(const CompletionItem &CI) {
+  json::Object result{{"label", CI.label}};
+
+  if (!CI.text.empty())
+    result.insert({"text", CI.text});
+  if (!CI.sortText.empty())
+    result.insert({"sortText", CI.sortText});
+  if (!CI.detail.empty())
+    result.insert({"detail", CI.detail});
+  if (CI.type)
+    result.insert({"type", CI.type});
+  if (CI.start)
+    result.insert({"start", CI.start});
+  if (CI.length)
+    result.insert({"length", CI.length});
+  if (CI.selectionStart)
+    result.insert({"selectionStart", CI.selectionStart});
+  if (CI.selectionLength)
+    result.insert({"selectionLength", CI.selectionLength});
+
+  return result;
+}
+
 json::Value toJSON(const BreakpointModeApplicability &BMA) {
   switch (BMA) {
   case eBreakpointModeApplicabilitySource:
@@ -332,6 +452,8 @@ static llvm::StringLiteral ToString(AdapterFeature feature) {
     return "supportsWriteMemoryRequest";
   case eAdapterFeatureTerminateDebuggee:
     return "supportTerminateDebuggee";
+  case eAdapterFeatureSupportsModuleSymbolsRequest:
+    return "supportsModuleSymbolsRequest";
   }
   llvm_unreachable("unhandled adapter feature.");
 }
@@ -403,6 +525,8 @@ bool fromJSON(const llvm::json::Value &Params, AdapterFeature &feature,
                 eAdapterFeatureValueFormattingOptions)
           .Case("supportsWriteMemoryRequest", eAdapterFeatureWriteMemoryRequest)
           .Case("supportTerminateDebuggee", eAdapterFeatureTerminateDebuggee)
+          .Case("supportsModuleSymbolsRequest",
+                eAdapterFeatureSupportsModuleSymbolsRequest)
           .Default(std::nullopt);
 
   if (!parsedFeature) {
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
index 89122c8f66307..7077df90a85b5 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolTypes.h
@@ -20,6 +20,7 @@
 #ifndef LLDB_TOOLS_LLDB_DAP_PROTOCOL_PROTOCOL_TYPES_H
 #define LLDB_TOOLS_LLDB_DAP_PROTOCOL_PROTOCOL_TYPES_H
 
+#include "Protocol/DAPTypes.h"
 #include "lldb/lldb-defines.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/Support/JSON.h"
@@ -108,6 +109,84 @@ enum ChecksumAlgorithm : unsigned {
 bool fromJSON(const llvm::json::Value &, ChecksumAlgorithm &, llvm::json::Path);
 llvm::json::Value toJSON(const ChecksumAlgorithm &);
 
+/// Some predefined types for the CompletionItem. Please note that not all
+/// clients have specific icons for all of them.
+enum CompletionItemType : unsigned {
+  eCompletionItemTypeMethod,
+  eCompletionItemTypeFunction,
+  eCompletionItemTypeConstructor,
+  eCompletionItemTypeField,
+  eCompletionItemTypeVariable,
+  eCompletionItemTypeClass,
+  eCompletionItemTypeInterface,
+  eCompletionItemTypeModule,
+  eCompletionItemTypeProperty,
+  eCompletionItemTypeUnit,
+  eCompletionItemTypeValue,
+  eCompletionItemTypeEnum,
+  eCompletionItemTypeKeyword,
+  eCompletionItemTypeSnippet,
+  eCompletionItemTypeText,
+  eCompletionItemTypeColor,
+  eCompletionItemTypeFile,
+  eCompletionItemTypeReference,
+  eCompletionItemTypeCustomColor,
+};
+bool fromJSON(const llvm::json::Value &, CompletionItemType &,
+              llvm::json::Path);
+llvm::json::Value toJSON(const CompletionItemType &);
+
+/// `CompletionItems` are the suggestions returned from the `completions`
+/// request.
+struct CompletionItem {
+  /// The label of this completion item. By default this is also the text that
+  /// is inserted when selecting this completion.
+  std::string label;
+
+  /// If text is returned and not an empty string, then it is inserted instead
+  /// of the label.
+  std::string text;
+
+  /// A string that should be used when comparing this item with other items. If
+  /// not returned or an empty string, the `label` is used instead.
+  std::string sortText;
+
+  /// A human-readable string with additional information about this item, like
+  /// type or symbol information.
+  std::string detail;
+
+  /// The item's type. Typically the client uses this information to render the
+  /// item in the UI with an icon.
+  std::optional<CompletionItemType> type;
+
+  /// Start position (within the `text` attribute of the `completions`
+  /// request) where the completion text is added. The position is measured in
+  /// UTF-16 code units and the client capability `columnsStartAt1` determines
+  /// whether it is 0- or 1-based. If the start position is omitted the text
+  /// is added at the location specified by the `column` attribute of the
+  /// `completions` request.
+  int64_t start = 0;
+
+  /// Length determines how many characters are overwritten by the completion
+  /// text and it is measured in UTF-16 code units. If missing the value 0 is
+  /// assumed which results in the completion text being inserted.
+  int64_t length = 0;
+
+  /// Determines the start of the new selection after the text has been
+  /// inserted (or replaced). `selectionStart` is measured in UTF-16 code
+  /// units and must be in the range 0 and length of the completion text. If
+  /// omitted the selection starts at the end of the completion text.
+  int64_t selectionStart = 0;
+
+  /// Determines the length of the new selection after the text has been
+  /// inserted (or replaced) and it is measured in UTF-16 code units. The
+  /// selection can not extend beyond the bounds of the completion text. If
+  /// omitted the length is assumed to be 0.
+  int64_t selectionLength = 0;
+};
+bool fromJSON(const llvm::json::Value &, CompletionItem &, llvm::json::Path);
+llvm::json::Value toJSON(const CompletionItem &);
+
 /// Describes one or more type of breakpoint a BreakpointMode applies to. This
 /// is a non-exhaustive enumeration and may expand as future breakpoint types
 /// are added.
@@ -241,8 +320,11 @@ enum AdapterFeature : unsigned {
   /// The debug adapter supports the `terminateDebuggee` attribute on the
   /// `disconnect` request.
   eAdapterFeatureTerminateDebuggee,
+  /// The debug adapter supports the `supportsModuleSymbols` request.
+  /// This request is a custom request of lldb-dap.
+  eAdapterFeatureSupportsModuleSymbolsRequest,
   eAdapterFeatureFirst = eAdapterFeatureANSIStyling,
-  eAdapterFeatureLast = eAdapterFeatureTerminateDebuggee,
+  eAdapterFeatureLast = eAdapterFeatureSupportsModuleSymbolsRequest,
 };
 bool fromJSON(const llvm::json::Value &, AdapterFeature &, llvm::json::Path);
 llvm::json::Value toJSON(const AdapterFeature &);
@@ -336,7 +418,12 @@ struct Source {
   /// skipped on stepping.
   std::optional<PresentationHint> presentationHint;
 
-  // unsupported keys: origin, sources, adapterData, checksums
+  /// Additional data that a debug adapter might want to loop through the
+  /// client. The client should leave the data intact and persist it across
+  /// sessions. The client should not interpret the data.
+  std::optional<SourceLLDBData> adapterData;
+
+  // unsupported keys: origin, sources, checksums
 };
 bool fromJSON(const llvm::json::Value &, Source::PresentationHint &,
               llvm::json::Path);
diff --git a/lldb/tools/lldb-dap/SourceBreakpoint.cpp b/lldb/tools/lldb-dap/SourceBreakpoint.cpp
index 5fce9fe0ddbb3..843a5eb09c7ae 100644
--- a/lldb/tools/lldb-dap/SourceBreakpoint.cpp
+++ b/lldb/tools/lldb-dap/SourceBreakpoint.cpp
@@ -10,7 +10,9 @@
 #include "BreakpointBase.h"
 #include "DAP.h"
 #include "JSONUtils.h"
+#include "ProtocolUtils.h"
 #include "lldb/API/SBBreakpoint.h"
+#include "lldb/API/SBFileSpec.h"
 #include "lldb/API/SBFileSpecList.h"
 #include "lldb/API/SBFrame.h"
 #include "lldb/API/SBInstruction.h"
@@ -46,41 +48,20 @@ llvm::Error SourceBreakpoint::SetBreakpoint(const protocol::Source &source) {
 
   if (source.sourceReference) {
     // Breakpoint set by assembly source.
-    std::optional<lldb::addr_t> raw_addr =
-        m_dap.GetSourceReferenceAddress(*source.sourceReference);
-    if (!raw_addr)
-      return llvm::createStringError(llvm::inconvertibleErrorCode(),
-                                     "Invalid sourceReference.");
-
-    lldb::SBAddress source_address(*raw_addr, m_dap.target);
-    if (!source_address.IsValid())
-      return llvm::createStringError(llvm::inconvertibleErrorCode(),
-                                     "Invalid sourceReference.");
-
-    lldb::SBSymbol symbol = source_address.GetSymbol();
-    if (!symbol.IsValid()) {
-      // FIXME: Support assembly breakpoints without a valid symbol.
-      return llvm::createStringError(llvm::inconvertibleErrorCode(),
-                                     "Breakpoints in assembly without a valid "
-                                     "symbol are not supported yet.");
+    if (source.adapterData && source.adapterData->persistenceData) {
+      // Prefer use the adapter persitence data, because this could be a
+      // breakpoint from a previous session where the `sourceReference` is not
+      // valid anymore.
+      if (llvm::Error error = CreateAssemblyBreakpointWithPersistenceData(
+              *source.adapterData->persistenceData))
+        return error;
+    } else {
+      if (llvm::Error error = CreateAssemblyBreakpointWithSourceReference(
+              *source.sourceReference))
+        return error;
     }
-
-    lldb::SBInstructionList inst_list =
-        m_dap.target.ReadInstructions(symbol.GetStartAddress(), m_line);
-    if (inst_list.GetSize() < m_line)
-      return llvm::createStringError(llvm::inconvertibleErrorCode(),
-                                     "Invalid instruction list size.");
-
-    lldb::SBAddress address =
-        inst_list.GetInstructionAtIndex(m_line - 1).GetAddress();
-
-    m_bp = m_dap.target.BreakpointCreateBySBAddress(address);
   } else {
-    // Breakpoint set by a regular source file.
-    const auto source_path = source.path.value_or("");
-    lldb::SBFileSpecList module_list;
-    m_bp = m_dap.target.BreakpointCreateByLocation(source_path.c_str(), m_line,
-                                                   m_column, 0, module_list);
+    CreatePathBreakpoint(source);
   }
 
   if (!m_log_message.empty())
@@ -97,6 +78,60 @@ void SourceBreakpoint::UpdateBreakpoint(const SourceBreakpoint &request_bp) {
   BreakpointBase::UpdateBreakpoint(request_bp);
 }
 
+void SourceBreakpoint::CreatePathBreakpoint(const protocol::Source &source) {
+  const auto source_path = source.path.value_or("");
+  lldb::SBFileSpecList module_list;
+  m_bp = m_dap.target.BreakpointCreateByLocation(source_path.c_str(), m_line,
+                                                 m_column, 0, module_list);
+}
+
+llvm::Error SourceBreakpoint::CreateAssemblyBreakpointWithSourceReference(
+    int64_t source_reference) {
+  std::optional<lldb::addr_t> raw_addr =
+      m_dap.GetSourceReferenceAddress(source_reference);
+  if (!raw_addr)
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "Invalid sourceReference.");
+
+  lldb::SBAddress source_address(*raw_addr, m_dap.target);
+  if (!source_address.IsValid())
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "Invalid sourceReference.");
+
+  lldb::SBSymbol symbol = source_address.GetSymbol();
+  if (!symbol.IsValid()) {
+    // FIXME: Support assembly breakpoints without a valid symbol.
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "Breakpoints in assembly without a valid "
+                                   "symbol are not supported yet.");
+  }
+
+  lldb::SBInstructionList inst_list =
+      m_dap.target.ReadInstructions(symbol.GetStartAddress(), m_line);
+  if (inst_list.GetSize() < m_line)
+    return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                   "Invalid instruction list size.");
+
+  lldb::SBAddress address =
+      inst_list.GetInstructionAtIndex(m_line - 1).GetAddress();
+
+  m_bp = m_dap.target.BreakpointCreateBySBAddress(address);
+  return llvm::Error::success();
+}
+
+llvm::Error SourceBreakpoint::CreateAssemblyBreakpointWithPersistenceData(
+    const protocol::PersistenceData &persistence_data) {
+  lldb::SBFileSpec file_spec(persistence_data.module_path.c_str());
+  lldb::SBFileSpecList comp_unit_list;
+  lldb::SBFileSpecList file_spec_list;
+  file_spec_list.Append(file_spec);
+  m_bp = m_dap.target.BreakpointCreateByName(
+      persistence_data.symbol_name.c_str(), lldb::eFunctionNameTypeFull,
+      lldb::eLanguageTypeUnknown, m_line - 1, true, file_spec_list,
+      comp_unit_list);
+  return llvm::Error::success();
+}
+
 lldb::SBError SourceBreakpoint::AppendLogMessagePart(llvm::StringRef part,
                                                      bool is_expr) {
   if (is_expr) {
diff --git a/lldb/tools/lldb-dap/SourceBreakpoint.h b/lldb/tools/lldb-dap/SourceBreakpoint.h
index 857ac4286d59d..34054a8dcfd5f 100644
--- a/lldb/tools/lldb-dap/SourceBreakpoint.h
+++ b/lldb/tools/lldb-dap/SourceBreakpoint.h
@@ -11,6 +11,7 @@
 
 #include "Breakpoint.h"
 #include "DAPForward.h"
+#include "Protocol/DAPTypes.h"
 #include "Protocol/ProtocolTypes.h"
 #include "lldb/API/SBError.h"
 #include "llvm/ADT/StringRef.h"
@@ -50,6 +51,12 @@ class SourceBreakpoint : public Breakpoint {
   uint32_t GetColumn() const { return m_column; }
 
 protected:
+  void CreatePathBreakpoint(const protocol::Source &source);
+  llvm::Error
+  CreateAssemblyBreakpointWithSourceReference(int64_t source_reference);
+  llvm::Error CreateAssemblyBreakpointWithPersistenceData(
+      const protocol::PersistenceData &persistence_data);
+
   // logMessage part can be either a raw text or an expression.
   struct LogMessagePart {
     LogMessagePart(llvm::StringRef text, bool is_expr)
diff --git a/lldb/tools/lldb-dap/Transport.cpp b/lldb/tools/lldb-dap/Transport.cpp
index d602920da34e3..8f71f88cae1f7 100644
--- a/lldb/tools/lldb-dap/Transport.cpp
+++ b/lldb/tools/lldb-dap/Transport.cpp
@@ -14,7 +14,8 @@
 using namespace llvm;
 using namespace lldb;
 using namespace lldb_private;
-using namespace lldb_dap;
+
+namespace lldb_dap {
 
 Transport::Transport(llvm::StringRef client_name, lldb_dap::Log *log,
                      lldb::IOObjectSP input, lldb::IOObjectSP output)
@@ -24,3 +25,5 @@ Transport::Transport(llvm::StringRef client_name, lldb_dap::Log *log,
 void Transport::Log(llvm::StringRef message) {
   DAP_LOG(m_log, "({0}) {1}", m_client_name, message);
 }
+
+} // namespace lldb_dap
diff --git a/lldb/tools/lldb-dap/Transport.h b/lldb/tools/lldb-dap/Transport.h
index 51f62e718a0d0..4a9dd76c2303e 100644
--- a/lldb/tools/lldb-dap/Transport.h
+++ b/lldb/tools/lldb-dap/Transport.h
@@ -15,6 +15,7 @@
 #define LLDB_TOOLS_LLDB_DAP_TRANSPORT_H
 
 #include "DAPForward.h"
+#include "Protocol/ProtocolBase.h"
 #include "lldb/Host/JSONTransport.h"
 #include "lldb/lldb-forward.h"
 #include "llvm/ADT/StringRef.h"
@@ -23,17 +24,15 @@ namespace lldb_dap {
 
 /// A transport class that performs the Debug Adapter Protocol communication
 /// with the client.
-class Transport : public lldb_private::HTTPDelimitedJSONTransport {
+class Transport final
+    : public lldb_private::HTTPDelimitedJSONTransport<
+          protocol::Request, protocol::Response, protocol::Event> {
 public:
   Transport(llvm::StringRef client_name, lldb_dap::Log *log,
             lldb::IOObjectSP input, lldb::IOObjectSP output);
   virtual ~Transport() = default;
 
-  virtual void Log(llvm::StringRef message) override;
-
-  /// Returns the name of this transport client, for example `stdin/stdout` or
-  /// `client_1`.
-  llvm::StringRef GetClientName() { return m_client_name; }
+  void Log(llvm::StringRef message) override;
 
 private:
   llvm::StringRef m_client_name;
diff --git a/lldb/tools/lldb-dap/package-lock.json b/lldb/tools/lldb-dap/package-lock.json
index 1969b196accc6..26db1ce6df2fd 100644
--- a/lldb/tools/lldb-dap/package-lock.json
+++ b/lldb/tools/lldb-dap/package-lock.json
@@ -1,20 +1,24 @@
 {
   "name": "lldb-dap",
-  "version": "0.2.15",
+  "version": "0.2.16",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "lldb-dap",
-      "version": "0.2.15",
+      "version": "0.2.16",
       "license": "Apache 2.0 License with LLVM exceptions",
       "devDependencies": {
         "@types/node": "^18.19.41",
+        "@types/tabulator-tables": "^6.2.10",
         "@types/vscode": "1.75.0",
+        "@types/vscode-webview": "^1.57.5",
         "@vscode/debugprotocol": "^1.68.0",
         "@vscode/vsce": "^3.2.2",
+        "esbuild": "^0.25.9",
         "prettier": "^3.4.2",
         "prettier-plugin-curly": "^0.3.1",
+        "tabulator-tables": "^6.3.1",
         "typescript": "^5.7.3"
       },
       "engines": {
@@ -318,6 +322,448 @@
         "node": ">=6.9.0"
       }
     },
+    "node_modules/@esbuild/aix-ppc64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.9.tgz",
+      "integrity": "sha512-OaGtL73Jck6pBKjNIe24BnFE6agGl+6KxDtTfHhy1HmhthfKouEcOhqpSL64K4/0WCtbKFLOdzD/44cJ4k9opA==",
+      "cpu": [
+        "ppc64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "aix"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/android-arm": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.9.tgz",
+      "integrity": "sha512-5WNI1DaMtxQ7t7B6xa572XMXpHAaI/9Hnhk8lcxF4zVN4xstUgTlvuGDorBguKEnZO70qwEcLpfifMLoxiPqHQ==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/android-arm64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.9.tgz",
+      "integrity": "sha512-IDrddSmpSv51ftWslJMvl3Q2ZT98fUSL2/rlUXuVqRXHCs5EUF1/f+jbjF5+NG9UffUDMCiTyh8iec7u8RlTLg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/android-x64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.9.tgz",
+      "integrity": "sha512-I853iMZ1hWZdNllhVZKm34f4wErd4lMyeV7BLzEExGEIZYsOzqDWDf+y082izYUE8gtJnYHdeDpN/6tUdwvfiw==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "android"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/darwin-arm64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.9.tgz",
+      "integrity": "sha512-XIpIDMAjOELi/9PB30vEbVMs3GV1v2zkkPnuyRRURbhqjyzIINwj+nbQATh4H9GxUgH1kFsEyQMxwiLFKUS6Rg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/darwin-x64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.9.tgz",
+      "integrity": "sha512-jhHfBzjYTA1IQu8VyrjCX4ApJDnH+ez+IYVEoJHeqJm9VhG9Dh2BYaJritkYK3vMaXrf7Ogr/0MQ8/MeIefsPQ==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/freebsd-arm64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.9.tgz",
+      "integrity": "sha512-z93DmbnY6fX9+KdD4Ue/H6sYs+bhFQJNCPZsi4XWJoYblUqT06MQUdBCpcSfuiN72AbqeBFu5LVQTjfXDE2A6Q==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/freebsd-x64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.9.tgz",
+      "integrity": "sha512-mrKX6H/vOyo5v71YfXWJxLVxgy1kyt1MQaD8wZJgJfG4gq4DpQGpgTB74e5yBeQdyMTbgxp0YtNj7NuHN0PoZg==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/linux-arm": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.9.tgz",
+      "integrity": "sha512-HBU2Xv78SMgaydBmdor38lg8YDnFKSARg1Q6AT0/y2ezUAKiZvc211RDFHlEZRFNRVhcMamiToo7bDx3VEOYQw==",
+      "cpu": [
+        "arm"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/linux-arm64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.9.tgz",
+      "integrity": "sha512-BlB7bIcLT3G26urh5Dmse7fiLmLXnRlopw4s8DalgZ8ef79Jj4aUcYbk90g8iCa2467HX8SAIidbL7gsqXHdRw==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/linux-ia32": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.9.tgz",
+      "integrity": "sha512-e7S3MOJPZGp2QW6AK6+Ly81rC7oOSerQ+P8L0ta4FhVi+/j/v2yZzx5CqqDaWjtPFfYz21Vi1S0auHrap3Ma3A==",
+      "cpu": [
+        "ia32"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/linux-loong64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.9.tgz",
+      "integrity": "sha512-Sbe10Bnn0oUAB2AalYztvGcK+o6YFFA/9829PhOCUS9vkJElXGdphz0A3DbMdP8gmKkqPmPcMJmJOrI3VYB1JQ==",
+      "cpu": [
+        "loong64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/linux-mips64el": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.9.tgz",
+      "integrity": "sha512-YcM5br0mVyZw2jcQeLIkhWtKPeVfAerES5PvOzaDxVtIyZ2NUBZKNLjC5z3/fUlDgT6w89VsxP2qzNipOaaDyA==",
+      "cpu": [
+        "mips64el"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/linux-ppc64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.9.tgz",
+      "integrity": "sha512-++0HQvasdo20JytyDpFvQtNrEsAgNG2CY1CLMwGXfFTKGBGQT3bOeLSYE2l1fYdvML5KUuwn9Z8L1EWe2tzs1w==",
+      "cpu": [
+        "ppc64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/linux-riscv64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.9.tgz",
+      "integrity": "sha512-uNIBa279Y3fkjV+2cUjx36xkx7eSjb8IvnL01eXUKXez/CBHNRw5ekCGMPM0BcmqBxBcdgUWuUXmVWwm4CH9kg==",
+      "cpu": [
+        "riscv64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/linux-s390x": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.9.tgz",
+      "integrity": "sha512-Mfiphvp3MjC/lctb+7D287Xw1DGzqJPb/J2aHHcHxflUo+8tmN/6d4k6I2yFR7BVo5/g7x2Monq4+Yew0EHRIA==",
+      "cpu": [
+        "s390x"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/linux-x64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.9.tgz",
+      "integrity": "sha512-iSwByxzRe48YVkmpbgoxVzn76BXjlYFXC7NvLYq+b+kDjyyk30J0JY47DIn8z1MO3K0oSl9fZoRmZPQI4Hklzg==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/netbsd-arm64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.9.tgz",
+      "integrity": "sha512-9jNJl6FqaUG+COdQMjSCGW4QiMHH88xWbvZ+kRVblZsWrkXlABuGdFJ1E9L7HK+T0Yqd4akKNa/lO0+jDxQD4Q==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "netbsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/netbsd-x64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.9.tgz",
+      "integrity": "sha512-RLLdkflmqRG8KanPGOU7Rpg829ZHu8nFy5Pqdi9U01VYtG9Y0zOG6Vr2z4/S+/3zIyOxiK6cCeYNWOFR9QP87g==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "netbsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/openbsd-arm64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.9.tgz",
+      "integrity": "sha512-YaFBlPGeDasft5IIM+CQAhJAqS3St3nJzDEgsgFixcfZeyGPCd6eJBWzke5piZuZ7CtL656eOSYKk4Ls2C0FRQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "openbsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/openbsd-x64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.9.tgz",
+      "integrity": "sha512-1MkgTCuvMGWuqVtAvkpkXFmtL8XhWy+j4jaSO2wxfJtilVCi0ZE37b8uOdMItIHz4I6z1bWWtEX4CJwcKYLcuA==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "openbsd"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/openharmony-arm64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.9.tgz",
+      "integrity": "sha512-4Xd0xNiMVXKh6Fa7HEJQbrpP3m3DDn43jKxMjxLLRjWnRsfxjORYJlXPO4JNcXtOyfajXorRKY9NkOpTHptErg==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "openharmony"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/sunos-x64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.9.tgz",
+      "integrity": "sha512-WjH4s6hzo00nNezhp3wFIAfmGZ8U7KtrJNlFMRKxiI9mxEK1scOMAaa9i4crUtu+tBr+0IN6JCuAcSBJZfnphw==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "sunos"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/win32-arm64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.9.tgz",
+      "integrity": "sha512-mGFrVJHmZiRqmP8xFOc6b84/7xa5y5YvR1x8djzXpJBSv/UsNK6aqec+6JDjConTgvvQefdGhFDAs2DLAds6gQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/win32-ia32": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.9.tgz",
+      "integrity": "sha512-b33gLVU2k11nVx1OhX3C8QQP6UHQK4ZtN56oFWvVXvz2VkDoe6fbG8TOgHFxEvqeqohmRnIHe5A1+HADk4OQww==",
+      "cpu": [
+        "ia32"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "node_modules/@esbuild/win32-x64": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.9.tgz",
+      "integrity": "sha512-PPOl1mi6lpLNQxnGoyAfschAodRFYXJ+9fs6WHXz7CSWKbOqiMZsubC+BQsVKuul+3vKLuwTHsS2c2y9EoKwxQ==",
+      "cpu": [
+        "x64"
+      ],
+      "dev": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@isaacs/cliui": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -399,6 +845,13 @@
         "undici-types": "~5.26.4"
       }
     },
+    "node_modules/@types/tabulator-tables": {
+      "version": "6.2.10",
+      "resolved": "https://registry.npmjs.org/@types/tabulator-tables/-/tabulator-tables-6.2.10.tgz",
+      "integrity": "sha512-g6o0gG3lu/ozmxPw9rLY1p57T6rvV8OhbJKyzWwPwjdnN3JuSQ3gWxb06v2+dl2tdoqNXTvlylipSSKpS8UzzQ==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/vscode": {
       "version": "1.75.0",
       "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.75.0.tgz",
@@ -406,6 +859,13 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/vscode-webview": {
+      "version": "1.57.5",
+      "resolved": "https://registry.npmjs.org/@types/vscode-webview/-/vscode-webview-1.57.5.tgz",
+      "integrity": "sha512-iBAUYNYkz+uk1kdsq05fEcoh8gJmwT3lqqFPN7MGyjQ3HVloViMdo7ZJ8DFIP8WOK74PjOEilosqAyxV2iUFUw==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@vscode/debugprotocol": {
       "version": "1.68.0",
       "resolved": "https://registry.npmjs.org/@vscode/debugprotocol/-/debugprotocol-1.68.0.tgz",
@@ -1169,6 +1629,48 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/esbuild": {
+      "version": "0.25.9",
+      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.9.tgz",
+      "integrity": "sha512-CRbODhYyQx3qp7ZEwzxOk4JBqmD/seJrzPa/cGjY1VtIn5E09Oi9/dB4JwctnfZ8Q8iT7rioVv5k/FNT/uf54g==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "bin": {
+        "esbuild": "bin/esbuild"
+      },
+      "engines": {
+        "node": ">=18"
+      },
+      "optionalDependencies": {
+        "@esbuild/aix-ppc64": "0.25.9",
+        "@esbuild/android-arm": "0.25.9",
+        "@esbuild/android-arm64": "0.25.9",
+        "@esbuild/android-x64": "0.25.9",
+        "@esbuild/darwin-arm64": "0.25.9",
+        "@esbuild/darwin-x64": "0.25.9",
+        "@esbuild/freebsd-arm64": "0.25.9",
+        "@esbuild/freebsd-x64": "0.25.9",
+        "@esbuild/linux-arm": "0.25.9",
+        "@esbuild/linux-arm64": "0.25.9",
+        "@esbuild/linux-ia32": "0.25.9",
+        "@esbuild/linux-loong64": "0.25.9",
+        "@esbuild/linux-mips64el": "0.25.9",
+        "@esbuild/linux-ppc64": "0.25.9",
+        "@esbuild/linux-riscv64": "0.25.9",
+        "@esbuild/linux-s390x": "0.25.9",
+        "@esbuild/linux-x64": "0.25.9",
+        "@esbuild/netbsd-arm64": "0.25.9",
+        "@esbuild/netbsd-x64": "0.25.9",
+        "@esbuild/openbsd-arm64": "0.25.9",
+        "@esbuild/openbsd-x64": "0.25.9",
+        "@esbuild/openharmony-arm64": "0.25.9",
+        "@esbuild/sunos-x64": "0.25.9",
+        "@esbuild/win32-arm64": "0.25.9",
+        "@esbuild/win32-ia32": "0.25.9",
+        "@esbuild/win32-x64": "0.25.9"
+      }
+    },
     "node_modules/escape-string-regexp": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
@@ -2557,6 +3059,13 @@
         "node": ">=4"
       }
     },
+    "node_modules/tabulator-tables": {
+      "version": "6.3.1",
+      "resolved": "https://registry.npmjs.org/tabulator-tables/-/tabulator-tables-6.3.1.tgz",
+      "integrity": "sha512-qFW7kfadtcaISQIibKAIy0f3eeIXUVi8242Vly1iJfMD79kfEGzfczNuPBN/80hDxHzQJXYbmJ8VipI40hQtfA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/tar-fs": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
diff --git a/lldb/tools/lldb-dap/package.json b/lldb/tools/lldb-dap/package.json
index d677a81cc7974..f11b64aa72ae4 100644
--- a/lldb/tools/lldb-dap/package.json
+++ b/lldb/tools/lldb-dap/package.json
@@ -29,11 +29,15 @@
   ],
   "devDependencies": {
     "@types/node": "^18.19.41",
+    "@types/tabulator-tables": "^6.2.10",
     "@types/vscode": "1.75.0",
+    "@types/vscode-webview": "^1.57.5",
     "@vscode/debugprotocol": "^1.68.0",
     "@vscode/vsce": "^3.2.2",
+    "esbuild": "^0.25.9",
     "prettier": "^3.4.2",
     "prettier-plugin-curly": "^0.3.1",
+    "tabulator-tables": "^6.3.1",
     "typescript": "^5.7.3"
   },
   "activationEvents": [
@@ -42,8 +46,11 @@
   ],
   "main": "./out/extension",
   "scripts": {
-    "vscode:prepublish": "tsc -p ./",
-    "watch": "tsc -watch -p ./",
+    "bundle-symbols-table-view": "npx tsc -p src-ts/webview --noEmit && npx esbuild src-ts/webview/symbols-table-view.ts --bundle --format=iife --outdir=./out/webview",
+    "bundle-tabulator": "cp node_modules/tabulator-tables/dist/js/tabulator.min.js ./out/webview/ && cp node_modules/tabulator-tables/dist/css/tabulator_midnight.min.css ./out/webview/ && cp node_modules/tabulator-tables/dist/css/tabulator_simple.min.css ./out/webview/",
+    "bundle-webview": "npm run bundle-symbols-table-view && npm run bundle-tabulator",
+    "vscode:prepublish": "npm run bundle-webview && tsc -p ./",
+    "watch": "npm run bundle-webview && tsc -watch -p ./",
     "format": "npx prettier './src-ts/' --write",
     "package": "rm -rf ./out/lldb-dap.vsix && vsce package --out ./out/lldb-dap.vsix",
     "publish": "vsce publish",
@@ -259,6 +266,15 @@
       {
         "command": "lldb-dap.modules.copyProperty",
         "title": "Copy Value"
+      },
+      {
+        "command": "lldb-dap.modules.showSymbols",
+        "title": "Show Module Symbols"
+      },
+      {
+        "category": "lldb-dap",
+        "command": "lldb-dap.debug.showSymbols",
+        "title": "Show Symbols of a Module"
       }
     ],
     "menus": {
@@ -266,12 +282,24 @@
         {
           "command": "lldb-dap.modules.copyProperty",
           "when": "false"
+        },
+        {
+          "command": "lldb-dap.modules.showSymbols",
+          "when": "false"
+        },
+        {
+          "command": "lldb-dap.debug.showSymbols",
+          "when": "debuggersAvailable && debugType == 'lldb-dap' && lldb-dap.supportsModuleSymbolsRequest"
         }
       ],
       "view/item/context": [
         {
           "command": "lldb-dap.modules.copyProperty",
           "when": "view == lldb-dap.modules && viewItem == property"
+        },
+        {
+          "command": "lldb-dap.modules.showSymbols",
+          "when": "view == lldb-dap.modules && viewItem == module && lldb-dap.supportsModuleSymbolsRequest"
         }
       ]
     },
diff --git a/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts b/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
index 6e94400b09155..157aa2ac76a1f 100644
--- a/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
+++ b/lldb/tools/lldb-dap/src-ts/debug-adapter-factory.ts
@@ -217,7 +217,15 @@ export class LLDBDapDescriptorFactory
   constructor(
     private readonly logger: vscode.LogOutputChannel,
     private logFilePath: LogFilePathProvider,
-  ) {}
+  ) {
+    vscode.commands.registerCommand(
+      "lldb-dap.createDebugAdapterDescriptor",
+      (
+        session: vscode.DebugSession,
+        executable: vscode.DebugAdapterExecutable | undefined,
+      ) => this.createDebugAdapterDescriptor(session, executable),
+    );
+  }
 
   async createDebugAdapterDescriptor(
     session: vscode.DebugSession,
diff --git a/lldb/tools/lldb-dap/src-ts/debug-configuration-provider.ts b/lldb/tools/lldb-dap/src-ts/debug-configuration-provider.ts
index 8c04ec2bdc9d3..1ae87116141f1 100644
--- a/lldb/tools/lldb-dap/src-ts/debug-configuration-provider.ts
+++ b/lldb/tools/lldb-dap/src-ts/debug-configuration-provider.ts
@@ -69,6 +69,10 @@ const configurations: Record<string, DefaultConfig> = {
   terminateCommands: { type: "stringArray", default: [] },
 };
 
+export function getDefaultConfigKey(key: string): string | number | boolean | string[] | undefined {
+  return configurations[key]?.default;
+}
+
 export class LLDBDapConfigurationProvider
   implements vscode.DebugConfigurationProvider
 {
@@ -76,7 +80,29 @@ export class LLDBDapConfigurationProvider
     private readonly server: LLDBDapServer,
     private readonly logger: vscode.LogOutputChannel,
     private readonly logFilePath: LogFilePathProvider,
-  ) {}
+  ) {
+    vscode.commands.registerCommand(
+      "lldb-dap.resolveDebugConfiguration",
+      (
+        folder: vscode.WorkspaceFolder | undefined,
+        debugConfiguration: vscode.DebugConfiguration,
+        token?: vscode.CancellationToken,
+      ) => this.resolveDebugConfiguration(folder, debugConfiguration, token),
+    );
+    vscode.commands.registerCommand(
+      "lldb-dap.resolveDebugConfigurationWithSubstitutedVariables",
+      (
+        folder: vscode.WorkspaceFolder | undefined,
+        debugConfiguration: vscode.DebugConfiguration,
+        token?: vscode.CancellationToken,
+      ) =>
+        this.resolveDebugConfigurationWithSubstitutedVariables(
+          folder,
+          debugConfiguration,
+          token,
+        ),
+    );
+  }
 
   async resolveDebugConfiguration(
     folder: vscode.WorkspaceFolder | undefined,
diff --git a/lldb/tools/lldb-dap/src-ts/debug-session-tracker.ts b/lldb/tools/lldb-dap/src-ts/debug-session-tracker.ts
index 7d7f73dbff92d..6e89d441bbcf0 100644
--- a/lldb/tools/lldb-dap/src-ts/debug-session-tracker.ts
+++ b/lldb/tools/lldb-dap/src-ts/debug-session-tracker.ts
@@ -1,11 +1,17 @@
 import { DebugProtocol } from "@vscode/debugprotocol";
 import * as vscode from "vscode";
 
+export interface LLDBDapCapabilities extends DebugProtocol.Capabilities {
+  /** The debug adapter supports the `moduleSymbols` request. */
+  supportsModuleSymbolsRequest?: boolean;
+}
+
 /** A helper type for mapping event types to their corresponding data type. */
 // prettier-ignore
 interface EventMap {
   "module": DebugProtocol.ModuleEvent;
   "exited": DebugProtocol.ExitedEvent;
+  "capabilities": DebugProtocol.CapabilitiesEvent;
 }
 
 /** A type assertion to check if a ProtocolMessage is an event or if it is a specific event. */
@@ -39,6 +45,9 @@ export class DebugSessionTracker
   private modulesChanged = new vscode.EventEmitter<
     vscode.DebugSession | undefined
   >();
+  private sessionReceivedCapabilities =
+      new vscode.EventEmitter<[ vscode.DebugSession, LLDBDapCapabilities ]>();
+  private sessionExited = new vscode.EventEmitter<vscode.DebugSession>();
 
   /**
    * Fired when modules are changed for any active debug session.
@@ -48,6 +57,15 @@ export class DebugSessionTracker
   onDidChangeModules: vscode.Event<vscode.DebugSession | undefined> =
     this.modulesChanged.event;
 
+  /** Fired when a debug session is initialized. */
+  onDidReceiveSessionCapabilities:
+      vscode.Event<[ vscode.DebugSession, LLDBDapCapabilities ]> =
+      this.sessionReceivedCapabilities.event;
+
+  /** Fired when a debug session is exiting. */
+  onDidExitSession: vscode.Event<vscode.DebugSession> =
+    this.sessionExited.event;
+
   constructor(private logger: vscode.LogOutputChannel) {
     this.onDidChangeModules(this.moduleChangedListener, this);
     vscode.debug.onDidChangeActiveDebugSession((session) =>
@@ -146,6 +164,10 @@ export class DebugSessionTracker
       this.logger.info(
         `Session "${session.name}" exited with code ${exitCode}`,
       );
+
+      this.sessionExited.fire(session);
+    } else if (isEvent(message, "capabilities")) {
+      this.sessionReceivedCapabilities.fire([ session, message.body.capabilities ]);
     }
   }
 }
diff --git a/lldb/tools/lldb-dap/src-ts/extension.ts b/lldb/tools/lldb-dap/src-ts/extension.ts
index 4b7a35e6944c6..7119cba972fa4 100644
--- a/lldb/tools/lldb-dap/src-ts/extension.ts
+++ b/lldb/tools/lldb-dap/src-ts/extension.ts
@@ -12,6 +12,7 @@ import {
   ModuleProperty,
 } from "./ui/modules-data-provider";
 import { LogFilePathProvider } from "./logging";
+import { SymbolsProvider } from "./ui/symbols-provider";
 
 /**
  * This class represents the extension and manages its life cycle. Other extensions
@@ -19,6 +20,7 @@ import { LogFilePathProvider } from "./logging";
  */
 export class LLDBDapExtension extends DisposableContext {
   constructor(
+    context: vscode.ExtensionContext,
     logger: vscode.LogOutputChannel,
     logFilePath: LogFilePathProvider,
     outputChannel: vscode.OutputChannel,
@@ -52,10 +54,12 @@ export class LLDBDapExtension extends DisposableContext {
       vscode.window.registerUriHandler(new LaunchUriHandler()),
     );
 
-    vscode.commands.registerCommand(
+    this.pushSubscription(vscode.commands.registerCommand(
       "lldb-dap.modules.copyProperty",
       (node: ModuleProperty) => vscode.env.clipboard.writeText(node.value),
-    );
+    ));
+
+    this.pushSubscription(new SymbolsProvider(sessionTracker, context));
   }
 }
 
@@ -67,7 +71,7 @@ export async function activate(context: vscode.ExtensionContext) {
   outputChannel.info("LLDB-DAP extension activating...");
   const logFilePath = new LogFilePathProvider(context, outputChannel);
   context.subscriptions.push(
-    new LLDBDapExtension(outputChannel, logFilePath, outputChannel),
+    new LLDBDapExtension(context, outputChannel, logFilePath, outputChannel),
   );
   outputChannel.info("LLDB-DAP extension activated");
 }
diff --git a/lldb/tools/lldb-dap/src-ts/index.d.ts b/lldb/tools/lldb-dap/src-ts/index.d.ts
new file mode 100644
index 0000000000000..d4618f44dee7b
--- /dev/null
+++ b/lldb/tools/lldb-dap/src-ts/index.d.ts
@@ -0,0 +1,14 @@
+export {};
+
+/// The symbol type we get from the lldb-dap server
+export declare interface SymbolType {
+  id: number;
+  isDebug: boolean;
+  isSynthetic: boolean;
+  isExternal: boolean;
+  type: string;
+  fileAddress: number;
+  loadAddress?: number;
+  size: number;
+  name: string;
+}
diff --git a/lldb/tools/lldb-dap/src-ts/lldb-dap-server.ts b/lldb/tools/lldb-dap/src-ts/lldb-dap-server.ts
index 03a0fc9d0fef3..5f9d8efdcb3a3 100644
--- a/lldb/tools/lldb-dap/src-ts/lldb-dap-server.ts
+++ b/lldb/tools/lldb-dap/src-ts/lldb-dap-server.ts
@@ -12,6 +12,13 @@ export class LLDBDapServer implements vscode.Disposable {
   private serverProcess?: child_process.ChildProcessWithoutNullStreams;
   private serverInfo?: Promise<{ host: string; port: number }>;
 
+  constructor() {
+    vscode.commands.registerCommand(
+      "lldb-dap.getServerProcess",
+      () => this.serverProcess,
+    );
+  }
+
   /**
    * Starts the server with the provided options. The server will be restarted or reused as
    * necessary.
@@ -39,8 +46,7 @@ export class LLDBDapServer implements vscode.Disposable {
       const process = child_process.spawn(dapPath, dapArgs, options);
       process.on("error", (error) => {
         reject(error);
-        this.serverProcess = undefined;
-        this.serverInfo = undefined;
+        this.cleanUp(process);
       });
       process.on("exit", (code, signal) => {
         let errorMessage = "Server process exited early";
@@ -50,8 +56,7 @@ export class LLDBDapServer implements vscode.Disposable {
           errorMessage += ` due to signal ${signal}`;
         }
         reject(new Error(errorMessage));
-        this.serverProcess = undefined;
-        this.serverInfo = undefined;
+        this.cleanUp(process);
       });
       process.stdout.setEncoding("utf8").on("data", (data) => {
         const connection = /connection:\/\/\[([^\]]+)\]:(\d+)/.exec(
@@ -126,7 +131,16 @@ Restarting the server will interrupt any existing debug sessions and start a new
       return;
     }
     this.serverProcess.kill();
-    this.serverProcess = undefined;
-    this.serverInfo = undefined;
+    this.cleanUp(this.serverProcess);
+  }
+
+  cleanUp(process: child_process.ChildProcessWithoutNullStreams) {
+    // If the following don't equal, then the fields have already been updated
+    // (either a new process has started, or the fields were already cleaned
+    // up), and so the cleanup should be skipped.
+    if (this.serverProcess === process) {
+      this.serverProcess = undefined;
+      this.serverInfo = undefined;
+    }
   }
 }
diff --git a/lldb/tools/lldb-dap/src-ts/ui/modules-data-provider.ts b/lldb/tools/lldb-dap/src-ts/ui/modules-data-provider.ts
index d0fb9270c734f..96343cb0a8da6 100644
--- a/lldb/tools/lldb-dap/src-ts/ui/modules-data-provider.ts
+++ b/lldb/tools/lldb-dap/src-ts/ui/modules-data-provider.ts
@@ -19,6 +19,7 @@ class ModuleItem extends vscode.TreeItem {
   constructor(module: DebugProtocol.Module) {
     super(module.name, vscode.TreeItemCollapsibleState.Collapsed);
     this.description = module.symbolStatus;
+    this.contextValue = "module";
   }
 
   static getProperties(module: DebugProtocol.Module): ModuleProperty[] {
diff --git a/lldb/tools/lldb-dap/src-ts/ui/symbols-provider.ts b/lldb/tools/lldb-dap/src-ts/ui/symbols-provider.ts
new file mode 100644
index 0000000000000..84b9387ffe49f
--- /dev/null
+++ b/lldb/tools/lldb-dap/src-ts/ui/symbols-provider.ts
@@ -0,0 +1,127 @@
+import * as vscode from "vscode";
+import { DebugProtocol } from "@vscode/debugprotocol";
+
+import { DebugSessionTracker } from "../debug-session-tracker";
+import { DisposableContext } from "../disposable-context";
+
+import { SymbolType } from "..";
+import { getSymbolsTableHTMLContent } from "./symbols-webview-html";
+import { getDefaultConfigKey } from "../debug-configuration-provider";
+
+export class SymbolsProvider extends DisposableContext {
+  constructor(
+    private readonly tracker: DebugSessionTracker,
+    private readonly extensionContext: vscode.ExtensionContext,
+  ) {
+    super();
+
+    this.pushSubscription(vscode.commands.registerCommand(
+      "lldb-dap.debug.showSymbols",
+      () => {
+        const session = vscode.debug.activeDebugSession;
+        if (!session) return;
+
+        this.SelectModuleAndShowSymbols(session);
+      },
+    ));
+
+    this.pushSubscription(vscode.commands.registerCommand(
+      "lldb-dap.modules.showSymbols",
+      (moduleItem: DebugProtocol.Module) => {
+        const session = vscode.debug.activeDebugSession;
+        if (!session) return;
+
+        this.showSymbolsForModule(session, moduleItem);
+      },
+    ));
+
+    this.tracker.onDidReceiveSessionCapabilities(([ _session, capabilities ]) => {
+      if (capabilities.supportsModuleSymbolsRequest) {
+        vscode.commands.executeCommand(
+            "setContext", "lldb-dap.supportsModuleSymbolsRequest", true);
+      }
+    });
+
+    this.tracker.onDidExitSession((_session) => {
+      vscode.commands.executeCommand("setContext", "lldb-dap.supportsModuleSymbolsRequest", false);
+    });
+  }
+
+  private async SelectModuleAndShowSymbols(session: vscode.DebugSession) {
+    const modules = this.tracker.debugSessionModules(session);
+    if (!modules || modules.length === 0) {
+      return;
+    }
+
+    // Let the user select a module to show symbols for
+    const selectedModule = await vscode.window.showQuickPick(modules.map(m => new ModuleQuickPickItem(m)), {
+        placeHolder: "Select a module to show symbols for"
+    });
+    if (!selectedModule) {
+      return;
+    }
+
+    this.showSymbolsForModule(session, selectedModule.module);
+  }
+
+  private async showSymbolsForModule(session: vscode.DebugSession, module: DebugProtocol.Module) {
+    try {
+      const symbols = await this.getSymbolsForModule(session, module.id.toString());
+      this.showSymbolsInNewTab(module.name.toString(), symbols);
+    } catch (error) {
+      if (error instanceof Error) {
+        vscode.window.showErrorMessage("Failed to retrieve symbols: " + error.message);
+      } else {
+        vscode.window.showErrorMessage("Failed to retrieve symbols due to an unknown error.");
+      }
+      
+      return;
+    }
+  }
+
+  private async getSymbolsForModule(session: vscode.DebugSession, moduleId: string): Promise<SymbolType[]> {
+    const symbols_response: { symbols: Array<SymbolType> } = await session.customRequest("__lldb_moduleSymbols", { moduleId, moduleName: '' });
+    return symbols_response?.symbols || [];
+  }
+
+  private async showSymbolsInNewTab(moduleName: string, symbols: SymbolType[]) {
+    const panel = vscode.window.createWebviewPanel(
+      "lldb-dap.symbols",
+      `Symbols for ${moduleName}`,
+      vscode.ViewColumn.Active,
+      {
+        enableScripts: true,
+        localResourceRoots: [
+          this.getExtensionResourcePath()
+        ]
+      }
+    );
+
+    let tabulatorJsFilename = "tabulator_simple.min.css";
+    if (vscode.window.activeColorTheme.kind === vscode.ColorThemeKind.Dark || vscode.window.activeColorTheme.kind === vscode.ColorThemeKind.HighContrast) {
+      tabulatorJsFilename = "tabulator_midnight.min.css";
+    }
+    const tabulatorCssPath = panel.webview.asWebviewUri(vscode.Uri.joinPath(this.getExtensionResourcePath(), tabulatorJsFilename));
+    const tabulatorJsPath = panel.webview.asWebviewUri(vscode.Uri.joinPath(this.getExtensionResourcePath(), "tabulator.min.js"));
+    const symbolsTableScriptPath = panel.webview.asWebviewUri(vscode.Uri.joinPath(this.getExtensionResourcePath(), "symbols-table-view.js"));
+
+    panel.webview.html = getSymbolsTableHTMLContent(tabulatorJsPath, tabulatorCssPath, symbolsTableScriptPath);
+    panel.webview.postMessage({ command: "updateSymbols", symbols: symbols });
+  }
+
+  private getExtensionResourcePath(): vscode.Uri {
+    return vscode.Uri.joinPath(this.extensionContext.extensionUri, "out", "webview");
+  }
+}
+
+class ModuleQuickPickItem implements vscode.QuickPickItem {
+  constructor(public readonly module: DebugProtocol.Module) {}
+
+  get label(): string {
+    return this.module.name;
+  }
+
+  get description(): string {
+    return this.module.id.toString();
+  }
+}
diff --git a/lldb/tools/lldb-dap/src-ts/ui/symbols-webview-html.ts b/lldb/tools/lldb-dap/src-ts/ui/symbols-webview-html.ts
new file mode 100644
index 0000000000000..88e24f3108787
--- /dev/null
+++ b/lldb/tools/lldb-dap/src-ts/ui/symbols-webview-html.ts
@@ -0,0 +1,51 @@
+import * as vscode from "vscode";
+
+export function getSymbolsTableHTMLContent(tabulatorJsPath: vscode.Uri, tabulatorCssPath: vscode.Uri, symbolsTableScriptPath: vscode.Uri): string {
+    return `<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <link href="${tabulatorCssPath}" rel="stylesheet">
+    <style>
+      .tabulator {
+        background-color: var(--vscode-editor-background);
+        color: var(--vscode-editor-foreground);
+      }
+
+      .tabulator .tabulator-header .tabulator-col {
+        background-color: var(--vscode-editor-background);
+        color: var(--vscode-editor-foreground);
+      }
+
+      .tabulator-row {
+        background-color: var(--vscode-editor-background);
+        color: var(--vscode-editor-foreground);
+      }
+
+      .tabulator-row.tabulator-row-even {
+        background-color: var(--vscode-editor-background);
+        color: var(--vscode-editor-foreground);
+      }
+
+      .tabulator-row.tabulator-selected {
+        background-color: var(--vscode-editor-background);
+        color: var(--vscode-editor-foreground);
+      }
+
+      .tabulator-cell {
+        text-overflow: clip !important;
+      }
+
+      #symbols-table {
+        width: 100%;
+        height: 100vh;
+      }
+    </style>
+</head>
+<body>
+    <div id="symbols-table"></div>
+    <script src="${tabulatorJsPath}"></script>
+    <script src="${symbolsTableScriptPath}"></script>
+</body>
+</html>`;
+}
\ No newline at end of file
diff --git a/lldb/tools/lldb-dap/src-ts/webview/symbols-table-view.ts b/lldb/tools/lldb-dap/src-ts/webview/symbols-table-view.ts
new file mode 100644
index 0000000000000..8454378abef16
--- /dev/null
+++ b/lldb/tools/lldb-dap/src-ts/webview/symbols-table-view.ts
@@ -0,0 +1,114 @@
+import type { CellComponent, ColumnDefinition } from "tabulator-tables";
+import type { SymbolType } from ".."
+
+/// SVG from https://github.com/olifolkerd/tabulator/blob/master/src/js/modules/Format/defaults/formatters/tickCross.js
+/// but with the default font color.
+/// hopefully in the future we can set the color as parameter: https://github.com/olifolkerd/tabulator/pull/4791
+const TICK_ELEMENT = `<svg enable-background="new 0 0 24 24" height="14" width="14" viewBox="0 0 24 24" xml:space="preserve" ><path fill="var(--vscode-editor-foreground)" clip-rule="evenodd" d="M21.652,3.211c-0.293-0.295-0.77-0.295-1.061,0L9.41,14.34  c-0.293,0.297-0.771,0.297-1.062,0L3.449,9.351C3.304,9.203,3.114,9.13,2.923,9.129C2.73,9.128,2.534,9.201,2.387,9.351  l-2.165,1.946C0.078,11.445,0,11.63,0,11.823c0,0.194,0.078,0.397,0.223,0.544l4.94,5.184c0.292,0.296,0.771,0.776,1.062,1.07  l2.124,2.141c0.292,0.293,0.769,0.293,1.062,0l14.366-14.34c0.293-0.294,0.293-0.777,0-1.071L21.652,3.211z" fill-rule="evenodd"/></svg>`;
+
+function getTabulatorHexaFormatter(padding: number): (cell: CellComponent) => string {
+  return (cell: CellComponent) => {
+    const val = cell.getValue();
+    if (val === undefined || val === null) {
+      return "";
+    }
+
+    return val !== undefined ? "0x" + val.toString(16).toLowerCase().padStart(padding, "0") : "";
+  };
+}
+
+const SYMBOL_TABLE_COLUMNS: ColumnDefinition[] = [
+  { title: "ID", field: "id", headerTooltip: true, sorter: "number", widthGrow: 0.6 },
+  {
+    title: "Name",
+    field: "name",
+    headerTooltip: true,
+    sorter: "string",
+    widthGrow: 2.5,
+    minWidth: 200,
+    tooltip : (_event: MouseEvent, cell: CellComponent) => {
+      const rowData = cell.getRow().getData();
+      return rowData.name;
+    }
+  },
+  {
+    title: "Debug",
+    field: "isDebug",
+    headerTooltip: true,
+    hozAlign: "center",
+    widthGrow: 0.8,
+    formatter: "tickCross",
+    formatterParams: {
+      tickElement: TICK_ELEMENT,
+      crossElement: false,
+    }
+  },
+  {
+    title: "Synthetic",
+    field: "isSynthetic",
+    headerTooltip: true,
+    hozAlign: "center",
+    widthGrow: 0.8,
+    formatter: "tickCross",
+    formatterParams: {
+      tickElement: TICK_ELEMENT,
+      crossElement: false,
+    }
+  },
+  {
+    title: "External",
+    field: "isExternal",
+    headerTooltip: true,
+    hozAlign: "center",
+    widthGrow: 0.8,
+    formatter: "tickCross",
+    formatterParams: {
+      tickElement: TICK_ELEMENT,
+      crossElement: false,
+    }
+  },
+  { title: "Type", field: "type", sorter: "string" },
+  {
+    title: "File Address",
+    field: "fileAddress",
+    headerTooltip: true,
+    sorter: "number",
+    widthGrow : 1.25,
+    formatter: getTabulatorHexaFormatter(16),
+  },
+  {
+    title: "Load Address",
+    field: "loadAddress",
+    headerTooltip: true,
+    sorter: "number",
+    widthGrow : 1.25,
+    formatter: getTabulatorHexaFormatter(16),
+  },
+  { title: "Size", field: "size", headerTooltip: true, sorter: "number", formatter: getTabulatorHexaFormatter(8) },
+];
+
+const vscode = acquireVsCodeApi();
+const previousState: any = vscode.getState();
+
+declare const Tabulator: any; // HACK: real definition comes from tabulator.min.js
+const SYMBOLS_TABLE = new Tabulator("#symbols-table", {
+  height: "100vh",
+  columns: SYMBOL_TABLE_COLUMNS,
+  layout: "fitColumns",
+  data: previousState?.symbols || [],
+});
+
+function updateSymbolsTable(symbols: SymbolType[]) {
+  SYMBOLS_TABLE.setData(symbols);
+}
+
+window.addEventListener("message", (event: MessageEvent<any>) => {
+  const message = event.data;
+  switch (message.command) {
+    case "updateSymbols":
+      vscode.setState({ symbols: message.symbols });
+      updateSymbolsTable(message.symbols);
+      break;
+  }
+});
+
diff --git a/lldb/tools/lldb-dap/src-ts/webview/tsconfig.json b/lldb/tools/lldb-dap/src-ts/webview/tsconfig.json
new file mode 100644
index 0000000000000..cfe64fc4b989f
--- /dev/null
+++ b/lldb/tools/lldb-dap/src-ts/webview/tsconfig.json
@@ -0,0 +1,15 @@
+{
+	"compilerOptions": {
+		"moduleResolution": "node",
+		"module": "esnext",
+		"outDir": "out",
+		"rootDir": ".",
+		"sourceMap": true,
+		"strict": true,
+		"noEmit": true,
+		"target": "es2017"
+	},
+	"include": [
+		"./"
+	],
+}
diff --git a/lldb/tools/lldb-dap/tool/lldb-dap.cpp b/lldb/tools/lldb-dap/tool/lldb-dap.cpp
index 8bba4162aa7bf..b74085f25f4e2 100644
--- a/lldb/tools/lldb-dap/tool/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/tool/lldb-dap.cpp
@@ -39,6 +39,7 @@
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/Threading.h"
+#include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <condition_variable>
 #include <cstdio>
@@ -284,7 +285,7 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
   });
   std::condition_variable dap_sessions_condition;
   std::mutex dap_sessions_mutex;
-  std::map<IOObject *, DAP *> dap_sessions;
+  std::map<MainLoop *, DAP *> dap_sessions;
   unsigned int clientCount = 0;
   auto handle = listener->Accept(g_loop, [=, &dap_sessions_condition,
                                           &dap_sessions_mutex, &dap_sessions,
@@ -300,8 +301,10 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
     std::thread client([=, &dap_sessions_condition, &dap_sessions_mutex,
                         &dap_sessions]() {
       llvm::set_thread_name(client_name + ".runloop");
+      MainLoop loop;
       Transport transport(client_name, log, io, io);
-      DAP dap(log, default_repl_mode, pre_init_commands, transport);
+      DAP dap(log, default_repl_mode, pre_init_commands, client_name, transport,
+              loop);
 
       if (auto Err = dap.ConfigureIO()) {
         llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(),
@@ -311,7 +314,7 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
 
       {
         std::scoped_lock<std::mutex> lock(dap_sessions_mutex);
-        dap_sessions[io.get()] = &dap;
+        dap_sessions[&loop] = &dap;
       }
 
       if (auto Err = dap.Loop()) {
@@ -322,7 +325,7 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
 
       DAP_LOG(log, "({0}) client disconnected", client_name);
       std::unique_lock<std::mutex> lock(dap_sessions_mutex);
-      dap_sessions.erase(io.get());
+      dap_sessions.erase(&loop);
       std::notify_all_at_thread_exit(dap_sessions_condition, std::move(lock));
     });
     client.detach();
@@ -344,13 +347,14 @@ serveConnection(const Socket::SocketProtocol &protocol, const std::string &name,
   bool client_failed = false;
   {
     std::scoped_lock<std::mutex> lock(dap_sessions_mutex);
-    for (auto [sock, dap] : dap_sessions) {
+    for (auto [loop, dap] : dap_sessions) {
       if (llvm::Error error = dap->Disconnect()) {
         client_failed = true;
-        llvm::errs() << "DAP client " << dap->transport.GetClientName()
-                     << " disconnected failed: "
-                     << llvm::toString(std::move(error)) << "\n";
+        llvm::WithColor::error() << "DAP client disconnected failed: "
+                                 << llvm::toString(std::move(error)) << "\n";
       }
+      loop->AddPendingCallback(
+          [](MainLoopBase &loop) { loop.RequestTermination(); });
     }
   }
 
@@ -550,8 +554,10 @@ int main(int argc, char *argv[]) {
       stdout_fd, File::eOpenOptionWriteOnly, NativeFile::Unowned);
 
   constexpr llvm::StringLiteral client_name = "stdio";
+  MainLoop loop;
   Transport transport(client_name, log.get(), input, output);
-  DAP dap(log.get(), default_repl_mode, pre_init_commands, transport);
+  DAP dap(log.get(), default_repl_mode, pre_init_commands, client_name,
+          transport, loop);
 
   // stdout/stderr redirection to the IDE's console
   if (auto Err = dap.ConfigureIO(stdout, stderr)) {
diff --git a/lldb/tools/lldb-dap/tsconfig.json b/lldb/tools/lldb-dap/tsconfig.json
index 2092148888904..06a484a1fc263 100644
--- a/lldb/tools/lldb-dap/tsconfig.json
+++ b/lldb/tools/lldb-dap/tsconfig.json
@@ -1,5 +1,6 @@
 {
 	"compilerOptions": {
+		"moduleResolution": "node",
 		"module": "commonjs",
 		"outDir": "out",
 		"rootDir": "src-ts",
@@ -12,5 +13,6 @@
 	],
 	"exclude": [
 		"node_modules",
+		"src-ts/webview",
 	]
 }
diff --git a/lldb/tools/lldb-rpc-gen/lldb-rpc-gen.cpp b/lldb/tools/lldb-rpc-gen/lldb-rpc-gen.cpp
index 9b48796a30be5..47b3093fe5212 100644
--- a/lldb/tools/lldb-rpc-gen/lldb-rpc-gen.cpp
+++ b/lldb/tools/lldb-rpc-gen/lldb-rpc-gen.cpp
@@ -121,11 +121,10 @@ class SBVisitor : public RecursiveASTVisitor<SBVisitor> {
   ///   - Certain inconvenient classes
   ///   - Records without definitions (forward declarations)
   bool ShouldSkipRecord(CXXRecordDecl *Decl) {
-    const Type *DeclType = Decl->getTypeForDecl();
-    QualType CanonicalType = DeclType->getCanonicalTypeInternal();
     return !Manager.isInMainFile(Decl->getBeginLoc()) ||
            !Decl->hasDefinition() || Decl->getDefinition() != Decl ||
-           lldb_rpc_gen::TypeIsDisallowedClass(CanonicalType);
+           lldb_rpc_gen::TypeIsDisallowedClass(
+               Context.getCanonicalTagType(Decl));
   }
 
   /// Check the support level for a type
diff --git a/lldb/unittests/Core/CMakeLists.txt b/lldb/unittests/Core/CMakeLists.txt
index a6820bdc7d7fa..6e609a63ad9b6 100644
--- a/lldb/unittests/Core/CMakeLists.txt
+++ b/lldb/unittests/Core/CMakeLists.txt
@@ -15,6 +15,7 @@ add_lldb_unittest(LLDBCoreTests
   SourceManagerTest.cpp
   TelemetryTest.cpp
   UniqueCStringMapTest.cpp
+  Value.cpp
 
   LINK_COMPONENTS
     Support
diff --git a/lldb/unittests/Core/MangledTest.cpp b/lldb/unittests/Core/MangledTest.cpp
index a6e6e75fde145..cbc0c5d951b99 100644
--- a/lldb/unittests/Core/MangledTest.cpp
+++ b/lldb/unittests/Core/MangledTest.cpp
@@ -889,10 +889,10 @@ TEST_P(DemanglingInfoCorrectnessTestFixutre, Correctness) {
   EXPECT_THAT_EXPECTED(qualifiers, llvm::Succeeded());
   reconstructed_name += *qualifiers;
 
-  // TODO: should retrieve suffix using the plugin too.
-  auto suffix = tracked_name.slice(OB->NameInfo.QualifiersRange.second,
-                                   llvm::StringRef::npos);
-  reconstructed_name += suffix;
+  auto suffix =
+      CPlusPlusLanguage::GetDemangledFunctionSuffix(tracked_name, OB->NameInfo);
+  EXPECT_THAT_EXPECTED(suffix, llvm::Succeeded());
+  reconstructed_name += *suffix;
 
   EXPECT_EQ(reconstructed_name, demangled);
 }
diff --git a/lldb/unittests/Core/Value.cpp b/lldb/unittests/Core/Value.cpp
new file mode 100644
index 0000000000000..d18a700c00ec0
--- /dev/null
+++ b/lldb/unittests/Core/Value.cpp
@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Core/Value.h"
+#include "Plugins/Platform/MacOSX/PlatformMacOSX.h"
+#include "Plugins/TypeSystem/Clang/TypeSystemClang.h"
+#include "TestingSupport/SubsystemRAII.h"
+#include "TestingSupport/Symbol/ClangTestUtils.h"
+
+#include "lldb/Utility/DataExtractor.h"
+
+#include "gtest/gtest.h"
+
+using namespace lldb_private;
+using namespace lldb_private::clang_utils;
+
+TEST(ValueTest, GetValueAsData) {
+  SubsystemRAII<FileSystem, HostInfo, PlatformMacOSX> subsystems;
+  auto holder = std::make_unique<clang_utils::TypeSystemClangHolder>("test");
+  auto *clang = holder->GetAST();
+
+  Value v(Scalar(42));
+  DataExtractor extractor;
+
+  // no compiler type
+  Status status = v.GetValueAsData(nullptr, extractor, nullptr);
+  ASSERT_TRUE(status.Fail());
+
+  // with compiler type
+  v.SetCompilerType(clang->GetBasicType(lldb::BasicType::eBasicTypeChar));
+
+  status = v.GetValueAsData(nullptr, extractor, nullptr);
+  ASSERT_TRUE(status.Success());
+}
diff --git a/lldb/unittests/DAP/CMakeLists.txt b/lldb/unittests/DAP/CMakeLists.txt
index 156cd625546bd..716159b454231 100644
--- a/lldb/unittests/DAP/CMakeLists.txt
+++ b/lldb/unittests/DAP/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_lldb_unittest(DAPTests
   DAPErrorTest.cpp
   DAPTest.cpp
+  DAPTypesTest.cpp
   FifoFilesTest.cpp
   Handler/DisconnectTest.cpp
   Handler/ContinueTest.cpp
diff --git a/lldb/unittests/DAP/DAPTest.cpp b/lldb/unittests/DAP/DAPTest.cpp
index 40ffaf87c9c45..d5a9591ad0a43 100644
--- a/lldb/unittests/DAP/DAPTest.cpp
+++ b/lldb/unittests/DAP/DAPTest.cpp
@@ -9,11 +9,9 @@
 #include "DAP.h"
 #include "Protocol/ProtocolBase.h"
 #include "TestBase.h"
-#include "Transport.h"
 #include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
-#include <chrono>
-#include <memory>
 #include <optional>
 
 using namespace llvm;
@@ -21,6 +19,7 @@ using namespace lldb;
 using namespace lldb_dap;
 using namespace lldb_dap_tests;
 using namespace lldb_dap::protocol;
+using namespace testing;
 
 class DAPTest : public TransportBase {};
 
@@ -29,11 +28,13 @@ TEST_F(DAPTest, SendProtocolMessages) {
       /*log=*/nullptr,
       /*default_repl_mode=*/ReplMode::Auto,
       /*pre_init_commands=*/{},
-      /*transport=*/*to_dap,
+      /*client_name=*/"test_client",
+      /*transport=*/*transport,
+      /*loop=*/loop,
   };
   dap.Send(Event{/*event=*/"my-event", /*body=*/std::nullopt});
-  ASSERT_THAT_EXPECTED(
-      from_dap->Read<protocol::Message>(std::chrono::milliseconds(1)),
-      HasValue(testing::VariantWith<Event>(testing::FieldsAre(
-          /*event=*/"my-event", /*body=*/std::nullopt))));
+  loop.AddPendingCallback(
+      [](lldb_private::MainLoopBase &loop) { loop.RequestTermination(); });
+  EXPECT_CALL(client, Received(IsEvent("my-event", std::nullopt)));
+  ASSERT_THAT_ERROR(dap.Loop(), llvm::Succeeded());
 }
diff --git a/lldb/unittests/DAP/DAPTypesTest.cpp b/lldb/unittests/DAP/DAPTypesTest.cpp
new file mode 100644
index 0000000000000..f398c54b724a0
--- /dev/null
+++ b/lldb/unittests/DAP/DAPTypesTest.cpp
@@ -0,0 +1,60 @@
+//===-- DAPTypesTest.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Protocol/DAPTypes.h"
+#include "TestingSupport/TestUtilities.h"
+#include "lldb/lldb-enumerations.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+#include <optional>
+
+using namespace llvm;
+using namespace lldb;
+using namespace lldb_dap;
+using namespace lldb_dap::protocol;
+using lldb_private::roundtripJSON;
+
+TEST(DAPTypesTest, SourceLLDBData) {
+  SourceLLDBData source_data;
+  source_data.persistenceData =
+      PersistenceData{"module_path123", "symbol_name456"};
+
+  llvm::Expected<SourceLLDBData> deserialized_data = roundtripJSON(source_data);
+  ASSERT_THAT_EXPECTED(deserialized_data, llvm::Succeeded());
+
+  EXPECT_EQ(source_data.persistenceData->module_path,
+            deserialized_data->persistenceData->module_path);
+  EXPECT_EQ(source_data.persistenceData->symbol_name,
+            deserialized_data->persistenceData->symbol_name);
+}
+
+TEST(DAPTypesTest, DAPSymbol) {
+  Symbol symbol;
+  symbol.id = 42;
+  symbol.isDebug = true;
+  symbol.isExternal = false;
+  symbol.isSynthetic = true;
+  symbol.type = lldb::eSymbolTypeTrampoline;
+  symbol.fileAddress = 0x12345678;
+  symbol.loadAddress = 0x87654321;
+  symbol.size = 64;
+  symbol.name = "testSymbol";
+
+  llvm::Expected<Symbol> deserialized_symbol = roundtripJSON(symbol);
+  ASSERT_THAT_EXPECTED(deserialized_symbol, llvm::Succeeded());
+
+  EXPECT_EQ(symbol.id, deserialized_symbol->id);
+  EXPECT_EQ(symbol.isDebug, deserialized_symbol->isDebug);
+  EXPECT_EQ(symbol.isExternal, deserialized_symbol->isExternal);
+  EXPECT_EQ(symbol.isSynthetic, deserialized_symbol->isSynthetic);
+  EXPECT_EQ(symbol.type, deserialized_symbol->type);
+  EXPECT_EQ(symbol.fileAddress, deserialized_symbol->fileAddress);
+  EXPECT_EQ(symbol.loadAddress, deserialized_symbol->loadAddress);
+  EXPECT_EQ(symbol.size, deserialized_symbol->size);
+  EXPECT_EQ(symbol.name, deserialized_symbol->name);
+}
diff --git a/lldb/unittests/DAP/Handler/DisconnectTest.cpp b/lldb/unittests/DAP/Handler/DisconnectTest.cpp
index 0546aeb154d50..c6ff1f90b01d5 100644
--- a/lldb/unittests/DAP/Handler/DisconnectTest.cpp
+++ b/lldb/unittests/DAP/Handler/DisconnectTest.cpp
@@ -23,18 +23,15 @@ using namespace lldb;
 using namespace lldb_dap;
 using namespace lldb_dap_tests;
 using namespace lldb_dap::protocol;
+using testing::_;
 
 class DisconnectRequestHandlerTest : public DAPTestBase {};
 
 TEST_F(DisconnectRequestHandlerTest, DisconnectTriggersTerminated) {
   DisconnectRequestHandler handler(*dap);
-  EXPECT_FALSE(dap->disconnecting);
   ASSERT_THAT_ERROR(handler.Run(std::nullopt), Succeeded());
-  EXPECT_TRUE(dap->disconnecting);
-  std::vector<Message> messages = DrainOutput();
-  EXPECT_THAT(messages,
-              testing::Contains(testing::VariantWith<Event>(testing::FieldsAre(
-                  /*event=*/"terminated", /*body=*/testing::_))));
+  EXPECT_CALL(client, Received(IsEvent("terminated", _)));
+  RunOnce();
 }
 
 TEST_F(DisconnectRequestHandlerTest, DisconnectTriggersTerminateCommands) {
@@ -47,17 +44,14 @@ TEST_F(DisconnectRequestHandlerTest, DisconnectTriggersTerminateCommands) {
 
   DisconnectRequestHandler handler(*dap);
 
-  EXPECT_FALSE(dap->disconnecting);
   dap->configuration.terminateCommands = {"?script print(1)",
                                           "script print(2)"};
   EXPECT_EQ(dap->target.GetProcess().GetState(), lldb::eStateStopped);
   ASSERT_THAT_ERROR(handler.Run(std::nullopt), Succeeded());
-  EXPECT_TRUE(dap->disconnecting);
-  std::vector<Message> messages = DrainOutput();
-  EXPECT_THAT(messages, testing::ElementsAre(
-                            OutputMatcher("Running terminateCommands:\n"),
-                            OutputMatcher("(lldb) script print(2)\n"),
-                            OutputMatcher("2\n"),
-                            testing::VariantWith<Event>(testing::FieldsAre(
-                                /*event=*/"terminated", /*body=*/testing::_))));
+  EXPECT_CALL(client, Received(Output("1\n")));
+  EXPECT_CALL(client, Received(Output("2\n"))).Times(2);
+  EXPECT_CALL(client, Received(Output("(lldb) script print(2)\n")));
+  EXPECT_CALL(client, Received(Output("Running terminateCommands:\n")));
+  EXPECT_CALL(client, Received(IsEvent("terminated", _)));
+  RunOnce();
 }
diff --git a/lldb/unittests/DAP/ProtocolTypesTest.cpp b/lldb/unittests/DAP/ProtocolTypesTest.cpp
index 3a1b9402b40c9..c5d47fcb08da4 100644
--- a/lldb/unittests/DAP/ProtocolTypesTest.cpp
+++ b/lldb/unittests/DAP/ProtocolTypesTest.cpp
@@ -108,8 +108,9 @@ TEST(ProtocolTypesTest, Breakpoint) {
   breakpoint.id = 42;
   breakpoint.verified = true;
   breakpoint.message = "Breakpoint set successfully";
-  breakpoint.source = Source{"test.cpp", "/path/to/test.cpp", 123,
-                             Source::eSourcePresentationHintNormal};
+  breakpoint.source =
+      Source{"test.cpp", "/path/to/test.cpp", 123,
+             Source::eSourcePresentationHintNormal, std::nullopt};
   breakpoint.line = 10;
   breakpoint.column = 5;
   breakpoint.endLine = 15;
@@ -567,8 +568,9 @@ TEST(ProtocolTypesTest, DisassembledInstruction) {
   instruction.instructionBytes = "0F 1F 00";
   instruction.instruction = "mov eax, ebx";
   instruction.symbol = "main";
-  instruction.location = Source{"test.cpp", "/path/to/test.cpp", 123,
-                                Source::eSourcePresentationHintNormal};
+  instruction.location =
+      Source{"test.cpp", "/path/to/test.cpp", 123,
+             Source::eSourcePresentationHintNormal, std::nullopt};
   instruction.line = 10;
   instruction.column = 5;
   instruction.endLine = 15;
@@ -1002,3 +1004,72 @@ TEST(ProtocolTypesTest, VariablesResponseBody) {
   ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
   EXPECT_EQ(pp(*expected), pp(response));
 }
+
+TEST(ProtocolTypesTest, CompletionItem) {
+  CompletionItem item;
+  item.label = "label";
+  item.text = "text";
+  item.sortText = "sortText";
+  item.detail = "detail";
+  item.type = eCompletionItemTypeConstructor;
+  item.start = 1;
+  item.length = 3;
+  item.selectionStart = 4;
+  item.selectionLength = 8;
+
+  const StringRef json = R"({
+  "detail": "detail",
+  "label": "label",
+  "length": 3,
+  "selectionLength": 8,
+  "selectionStart": 4,
+  "sortText": "sortText",
+  "start": 1,
+  "text": "text",
+  "type": "constructor"
+})";
+
+  EXPECT_EQ(pp(Value(item)), json);
+  EXPECT_THAT_EXPECTED(json::parse(json), HasValue(Value(item)));
+}
+
+TEST(ProtocolTypesTest, CompletionsArguments) {
+  llvm::Expected<CompletionsArguments> expected =
+      parse<CompletionsArguments>(R"({
+    "column": 8,
+    "frameId": 7,
+    "line": 9,
+    "text": "abc"
+  })");
+  ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
+  EXPECT_EQ(expected->frameId, 7u);
+  EXPECT_EQ(expected->text, "abc");
+  EXPECT_EQ(expected->column, 8);
+  EXPECT_EQ(expected->line, 9);
+
+  // Check required keys.
+  EXPECT_THAT_EXPECTED(parse<CompletionsArguments>(R"({})"),
+                       FailedWithMessage("missing value at (root).text"));
+  EXPECT_THAT_EXPECTED(parse<CompletionsArguments>(R"({"text":"abc"})"),
+                       FailedWithMessage("missing value at (root).column"));
+}
+
+TEST(ProtocolTypesTest, CompletionsResponseBody) {
+  CompletionItem item;
+  item.label = "label";
+  item.text = "text";
+  item.detail = "detail";
+  CompletionsResponseBody response{{item}};
+
+  Expected<json::Value> expected = json::parse(R"({
+      "targets": [
+        {
+          "detail": "detail",
+          "label": "label",
+          "text": "text"
+        }
+      ]
+    })");
+  ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
+  EXPECT_EQ(pp(*expected), pp(response));
+}
diff --git a/lldb/unittests/DAP/TestBase.cpp b/lldb/unittests/DAP/TestBase.cpp
index 94b9559b9ca70..54ac27da694e6 100644
--- a/lldb/unittests/DAP/TestBase.cpp
+++ b/lldb/unittests/DAP/TestBase.cpp
@@ -7,17 +7,19 @@
 //===----------------------------------------------------------------------===//
 
 #include "TestBase.h"
-#include "Protocol/ProtocolBase.h"
+#include "DAPLog.h"
 #include "TestingSupport/TestUtilities.h"
 #include "lldb/API/SBDefines.h"
 #include "lldb/API/SBStructuredData.h"
-#include "lldb/Host/File.h"
+#include "lldb/Host/MainLoop.h"
 #include "lldb/Host/Pipe.h"
-#include "lldb/lldb-forward.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
+#include <cstdio>
 #include <memory>
+#include <system_error>
 
 using namespace llvm;
 using namespace lldb;
@@ -25,36 +27,36 @@ using namespace lldb_dap;
 using namespace lldb_dap::protocol;
 using namespace lldb_dap_tests;
 using lldb_private::File;
-using lldb_private::NativeFile;
+using lldb_private::FileSpec;
+using lldb_private::FileSystem;
+using lldb_private::MainLoop;
 using lldb_private::Pipe;
 
-void TransportBase::SetUp() {
-  PipePairTest::SetUp();
-  to_dap = std::make_unique<Transport>(
-      "to_dap", nullptr,
-      std::make_shared<NativeFile>(input.GetReadFileDescriptor(),
-                                   File::eOpenOptionReadOnly,
-                                   NativeFile::Unowned),
-      std::make_shared<NativeFile>(output.GetWriteFileDescriptor(),
-                                   File::eOpenOptionWriteOnly,
-                                   NativeFile::Unowned));
-  from_dap = std::make_unique<Transport>(
-      "from_dap", nullptr,
-      std::make_shared<NativeFile>(output.GetReadFileDescriptor(),
-                                   File::eOpenOptionReadOnly,
-                                   NativeFile::Unowned),
-      std::make_shared<NativeFile>(input.GetWriteFileDescriptor(),
-                                   File::eOpenOptionWriteOnly,
-                                   NativeFile::Unowned));
+Expected<MainLoop::ReadHandleUP>
+TestTransport::RegisterMessageHandler(MainLoop &loop, MessageHandler &handler) {
+  Expected<lldb::FileUP> dummy_file = FileSystem::Instance().Open(
+      FileSpec(FileSystem::DEV_NULL), File::eOpenOptionReadWrite);
+  if (!dummy_file)
+    return dummy_file.takeError();
+  m_dummy_file = std::move(*dummy_file);
+  lldb_private::Status status;
+  auto handle = loop.RegisterReadObject(
+      m_dummy_file, [](lldb_private::MainLoopBase &) {}, status);
+  if (status.Fail())
+    return status.takeError();
+  return handle;
 }
 
 void DAPTestBase::SetUp() {
   TransportBase::SetUp();
+  std::error_code EC;
+  log = std::make_unique<Log>("-", EC);
   dap = std::make_unique<DAP>(
-      /*log=*/nullptr,
+      /*log=*/log.get(),
       /*default_repl_mode=*/ReplMode::Auto,
       /*pre_init_commands=*/std::vector<std::string>(),
-      /*transport=*/*to_dap);
+      /*client_name=*/"test_client",
+      /*transport=*/*transport, /*loop=*/loop);
 }
 
 void DAPTestBase::TearDown() {
@@ -72,7 +74,7 @@ void DAPTestBase::SetUpTestSuite() {
 }
 void DAPTestBase::TeatUpTestSuite() { SBDebugger::Terminate(); }
 
-bool DAPTestBase::GetDebuggerSupportsTarget(llvm::StringRef platform) {
+bool DAPTestBase::GetDebuggerSupportsTarget(StringRef platform) {
   EXPECT_TRUE(dap->debugger);
 
   lldb::SBStructuredData data = dap->debugger.GetBuildConfiguration()
@@ -81,7 +83,7 @@ bool DAPTestBase::GetDebuggerSupportsTarget(llvm::StringRef platform) {
   for (size_t i = 0; i < data.GetSize(); i++) {
     char buf[100] = {0};
     size_t size = data.GetItemAtIndex(i).GetStringValue(buf, sizeof(buf));
-    if (llvm::StringRef(buf, size) == platform)
+    if (StringRef(buf, size) == platform)
       return true;
   }
 
@@ -91,6 +93,24 @@ bool DAPTestBase::GetDebuggerSupportsTarget(llvm::StringRef platform) {
 void DAPTestBase::CreateDebugger() {
   dap->debugger = lldb::SBDebugger::Create();
   ASSERT_TRUE(dap->debugger);
+  dap->target = dap->debugger.GetDummyTarget();
+
+  Expected<lldb::FileUP> dev_null = FileSystem::Instance().Open(
+      FileSpec(FileSystem::DEV_NULL), File::eOpenOptionReadWrite);
+  ASSERT_THAT_EXPECTED(dev_null, Succeeded());
+  lldb::FileSP dev_null_sp = std::move(*dev_null);
+
+  std::FILE *dev_null_stream = dev_null_sp->GetStream();
+  ASSERT_THAT_ERROR(dap->ConfigureIO(dev_null_stream, dev_null_stream),
+                    Succeeded());
+
+  dap->debugger.SetInputFile(dap->in);
+  auto out_fd = dap->out.GetWriteFileDescriptor();
+  ASSERT_THAT_EXPECTED(out_fd, Succeeded());
+  dap->debugger.SetOutputFile(lldb::SBFile(*out_fd, "w", false));
+  auto err_fd = dap->out.GetWriteFileDescriptor();
+  ASSERT_THAT_EXPECTED(err_fd, Succeeded());
+  dap->debugger.SetErrorFile(lldb::SBFile(*err_fd, "w", false));
 }
 
 void DAPTestBase::LoadCore() {
@@ -114,18 +134,3 @@ void DAPTestBase::LoadCore() {
   SBProcess process = dap->target.LoadCore(this->core->TmpName.data());
   ASSERT_TRUE(process);
 }
-
-std::vector<Message> DAPTestBase::DrainOutput() {
-  std::vector<Message> msgs;
-  output.CloseWriteFileDescriptor();
-  while (true) {
-    Expected<Message> next =
-        from_dap->Read<protocol::Message>(std::chrono::milliseconds(1));
-    if (!next) {
-      consumeError(next.takeError());
-      break;
-    }
-    msgs.push_back(*next);
-  }
-  return msgs;
-}
diff --git a/lldb/unittests/DAP/TestBase.h b/lldb/unittests/DAP/TestBase.h
index 50884b1d7feb9..c19eead4e37e7 100644
--- a/lldb/unittests/DAP/TestBase.h
+++ b/lldb/unittests/DAP/TestBase.h
@@ -8,35 +8,109 @@
 
 #include "DAP.h"
 #include "Protocol/ProtocolBase.h"
-#include "TestingSupport/Host/PipeTestUtilities.h"
-#include "Transport.h"
+#include "TestingSupport/Host/JSONTransportTestUtilities.h"
+#include "TestingSupport/SubsystemRAII.h"
+#include "lldb/Host/FileSystem.h"
+#include "lldb/Host/HostInfo.h"
+#include "lldb/Host/MainLoop.h"
+#include "lldb/Host/MainLoopBase.h"
+#include "lldb/lldb-forward.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Testing/Support/Error.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include <memory>
 
 namespace lldb_dap_tests {
 
+class TestTransport final
+    : public lldb_private::Transport<lldb_dap::protocol::Request,
+                                     lldb_dap::protocol::Response,
+                                     lldb_dap::protocol::Event> {
+public:
+  using Message = lldb_private::Transport<lldb_dap::protocol::Request,
+                                          lldb_dap::protocol::Response,
+                                          lldb_dap::protocol::Event>::Message;
+
+  TestTransport(lldb_private::MainLoop &loop, MessageHandler &handler)
+      : m_loop(loop), m_handler(handler) {}
+
+  llvm::Error Send(const lldb_dap::protocol::Event &e) override {
+    m_loop.AddPendingCallback([this, e](lldb_private::MainLoopBase &) {
+      this->m_handler.Received(e);
+    });
+    return llvm::Error::success();
+  }
+
+  llvm::Error Send(const lldb_dap::protocol::Request &r) override {
+    m_loop.AddPendingCallback([this, r](lldb_private::MainLoopBase &) {
+      this->m_handler.Received(r);
+    });
+    return llvm::Error::success();
+  }
+
+  llvm::Error Send(const lldb_dap::protocol::Response &r) override {
+    m_loop.AddPendingCallback([this, r](lldb_private::MainLoopBase &) {
+      this->m_handler.Received(r);
+    });
+    return llvm::Error::success();
+  }
+
+  llvm::Expected<lldb_private::MainLoop::ReadHandleUP>
+  RegisterMessageHandler(lldb_private::MainLoop &loop,
+                         MessageHandler &handler) override;
+
+  void Log(llvm::StringRef message) override {
+    log_messages.emplace_back(message);
+  }
+
+  std::vector<std::string> log_messages;
+
+private:
+  lldb_private::MainLoop &m_loop;
+  MessageHandler &m_handler;
+  lldb::FileSP m_dummy_file;
+};
+
 /// A base class for tests that need transport configured for communicating DAP
 /// messages.
-class TransportBase : public PipePairTest {
+class TransportBase : public testing::Test {
 protected:
-  std::unique_ptr<lldb_dap::Transport> to_dap;
-  std::unique_ptr<lldb_dap::Transport> from_dap;
+  lldb_private::SubsystemRAII<lldb_private::FileSystem, lldb_private::HostInfo>
+      subsystems;
+  lldb_private::MainLoop loop;
+  std::unique_ptr<TestTransport> transport;
+  MockMessageHandler<lldb_dap::protocol::Request, lldb_dap::protocol::Response,
+                     lldb_dap::protocol::Event>
+      client;
 
-  void SetUp() override;
+  void SetUp() override {
+    transport = std::make_unique<TestTransport>(loop, client);
+  }
 };
 
+/// A matcher for a DAP event.
+template <typename M1, typename M2>
+inline testing::Matcher<const lldb_dap::protocol::Event &>
+IsEvent(const M1 &m1, const M2 &m2) {
+  return testing::AllOf(testing::Field(&lldb_dap::protocol::Event::event, m1),
+                        testing::Field(&lldb_dap::protocol::Event::body, m2));
+}
+
 /// Matches an "output" event.
-inline auto OutputMatcher(const llvm::StringRef output,
-                          const llvm::StringRef category = "console") {
-  return testing::VariantWith<lldb_dap::protocol::Event>(testing::FieldsAre(
-      /*event=*/"output", /*body=*/testing::Optional<llvm::json::Value>(
-          llvm::json::Object{{"category", category}, {"output", output}})));
+inline auto Output(llvm::StringRef o, llvm::StringRef cat = "console") {
+  return IsEvent("output",
+                 testing::Optional(llvm::json::Value(
+                     llvm::json::Object{{"category", cat}, {"output", o}})));
 }
 
 /// A base class for tests that interact with a `lldb_dap::DAP` instance.
 class DAPTestBase : public TransportBase {
 protected:
+  std::unique_ptr<lldb_dap::Log> log;
   std::unique_ptr<lldb_dap::DAP> dap;
   std::optional<llvm::sys::fs::TempFile> core;
   std::optional<llvm::sys::fs::TempFile> binary;
@@ -53,9 +127,11 @@ class DAPTestBase : public TransportBase {
   void CreateDebugger();
   void LoadCore();
 
-  /// Closes the DAP output pipe and returns the remaining protocol messages in
-  /// the buffer.
-  std::vector<lldb_dap::protocol::Message> DrainOutput();
+  void RunOnce() {
+    loop.AddPendingCallback(
+        [](lldb_private::MainLoopBase &loop) { loop.RequestTermination(); });
+    ASSERT_THAT_ERROR(dap->Loop(), llvm::Succeeded());
+  }
 };
 
 } // namespace lldb_dap_tests
diff --git a/lldb/unittests/Expression/DWARFExpressionTest.cpp b/lldb/unittests/Expression/DWARFExpressionTest.cpp
index 3008fc0d9ca17..5a5d3aba0e207 100644
--- a/lldb/unittests/Expression/DWARFExpressionTest.cpp
+++ b/lldb/unittests/Expression/DWARFExpressionTest.cpp
@@ -125,8 +125,8 @@ class MockRegisterContext : public RegisterContext {
   }
 
 private:
-  RegisterInfo m_reg_info;
-  RegisterValue m_reg_value;
+  RegisterInfo m_reg_info{};
+  RegisterValue m_reg_value{};
 };
 } // namespace
 
diff --git a/lldb/unittests/Host/JSONTransportTest.cpp b/lldb/unittests/Host/JSONTransportTest.cpp
index 2f0846471688c..445674f402252 100644
--- a/lldb/unittests/Host/JSONTransportTest.cpp
+++ b/lldb/unittests/Host/JSONTransportTest.cpp
@@ -1,4 +1,4 @@
-//===-- JSONTransportTest.cpp ---------------------------------------------===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,16 +7,143 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Host/JSONTransport.h"
+#include "TestingSupport/Host/JSONTransportTestUtilities.h"
 #include "TestingSupport/Host/PipeTestUtilities.h"
 #include "lldb/Host/File.h"
+#include "lldb/Host/MainLoop.h"
+#include "lldb/Host/MainLoopBase.h"
+#include "lldb/Utility/Log.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <chrono>
+#include <cstddef>
+#include <memory>
+#include <string>
 
 using namespace llvm;
 using namespace lldb_private;
+using testing::_;
+using testing::HasSubstr;
+using testing::InSequence;
 
 namespace {
-template <typename T> class JSONTransportTest : public PipePairTest {
+
+namespace test_protocol {
+
+struct Req {
+  std::string name;
+};
+json::Value toJSON(const Req &T) { return json::Object{{"req", T.name}}; }
+bool fromJSON(const json::Value &V, Req &T, json::Path P) {
+  json::ObjectMapper O(V, P);
+  return O && O.map("req", T.name);
+}
+bool operator==(const Req &a, const Req &b) { return a.name == b.name; }
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Req &V) {
+  OS << toJSON(V);
+  return OS;
+}
+void PrintTo(const Req &message, std::ostream *os) {
+  std::string O;
+  llvm::raw_string_ostream OS(O);
+  OS << message;
+  *os << O;
+}
+
+struct Resp {
+  std::string name;
+};
+json::Value toJSON(const Resp &T) { return json::Object{{"resp", T.name}}; }
+bool fromJSON(const json::Value &V, Resp &T, json::Path P) {
+  json::ObjectMapper O(V, P);
+  return O && O.map("resp", T.name);
+}
+bool operator==(const Resp &a, const Resp &b) { return a.name == b.name; }
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Resp &V) {
+  OS << toJSON(V);
+  return OS;
+}
+void PrintTo(const Resp &message, std::ostream *os) {
+  std::string O;
+  llvm::raw_string_ostream OS(O);
+  OS << message;
+  *os << O;
+}
+
+struct Evt {
+  std::string name;
+};
+json::Value toJSON(const Evt &T) { return json::Object{{"evt", T.name}}; }
+bool fromJSON(const json::Value &V, Evt &T, json::Path P) {
+  json::ObjectMapper O(V, P);
+  return O && O.map("evt", T.name);
+}
+bool operator==(const Evt &a, const Evt &b) { return a.name == b.name; }
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Evt &V) {
+  OS << toJSON(V);
+  return OS;
+}
+void PrintTo(const Evt &message, std::ostream *os) {
+  std::string O;
+  llvm::raw_string_ostream OS(O);
+  OS << message;
+  *os << O;
+}
+
+using Message = std::variant<Req, Resp, Evt>;
+json::Value toJSON(const Message &msg) {
+  return std::visit([](const auto &msg) { return toJSON(msg); }, msg);
+}
+bool fromJSON(const json::Value &V, Message &msg, json::Path P) {
+  const json::Object *O = V.getAsObject();
+  if (!O) {
+    P.report("expected object");
+    return false;
+  }
+  if (O->get("req")) {
+    Req R;
+    if (!fromJSON(V, R, P))
+      return false;
+
+    msg = std::move(R);
+    return true;
+  }
+  if (O->get("resp")) {
+    Resp R;
+    if (!fromJSON(V, R, P))
+      return false;
+
+    msg = std::move(R);
+    return true;
+  }
+  if (O->get("evt")) {
+    Evt E;
+    if (!fromJSON(V, E, P))
+      return false;
+
+    msg = std::move(E);
+    return true;
+  }
+  P.report("unknown message type");
+  return false;
+}
+
+} // namespace test_protocol
+
+template <typename T, typename Req, typename Resp, typename Evt>
+class JSONTransportTest : public PipePairTest {
+
 protected:
-  std::unique_ptr<JSONTransport> transport;
+  MockMessageHandler<Req, Resp, Evt> message_handler;
+  std::unique_ptr<T> transport;
+  MainLoop loop;
 
   void SetUp() override {
     PipePairTest::SetUp();
@@ -28,79 +155,231 @@ template <typename T> class JSONTransportTest : public PipePairTest {
                                      File::eOpenOptionWriteOnly,
                                      NativeFile::Unowned));
   }
+
+  /// Run the transport MainLoop and return any messages received.
+  Error
+  Run(bool close_input = true,
+      std::chrono::milliseconds timeout = std::chrono::milliseconds(5000)) {
+    if (close_input) {
+      input.CloseWriteFileDescriptor();
+      EXPECT_CALL(message_handler, OnClosed()).WillOnce([this]() {
+        loop.RequestTermination();
+      });
+    }
+    loop.AddCallback(
+        [](MainLoopBase &loop) {
+          loop.RequestTermination();
+          FAIL() << "timeout";
+        },
+        timeout);
+    auto handle = transport->RegisterMessageHandler(loop, message_handler);
+    if (!handle)
+      return handle.takeError();
+
+    return loop.Run().takeError();
+  }
+
+  template <typename... Ts> void Write(Ts... args) {
+    std::string message;
+    for (const auto &arg : {args...})
+      message += Encode(arg);
+    EXPECT_THAT_EXPECTED(input.Write(message.data(), message.size()),
+                         Succeeded());
+  }
+
+  virtual std::string Encode(const json::Value &) = 0;
 };
 
-class HTTPDelimitedJSONTransportTest
-    : public JSONTransportTest<HTTPDelimitedJSONTransport> {
+class TestHTTPDelimitedJSONTransport final
+    : public HTTPDelimitedJSONTransport<test_protocol::Req, test_protocol::Resp,
+                                        test_protocol::Evt> {
 public:
-  using JSONTransportTest::JSONTransportTest;
+  using HTTPDelimitedJSONTransport::HTTPDelimitedJSONTransport;
+
+  void Log(llvm::StringRef message) override {
+    log_messages.emplace_back(message);
+  }
+
+  std::vector<std::string> log_messages;
 };
 
-class JSONRPCTransportTest : public JSONTransportTest<JSONRPCTransport> {
+class HTTPDelimitedJSONTransportTest
+    : public JSONTransportTest<TestHTTPDelimitedJSONTransport,
+                               test_protocol::Req, test_protocol::Resp,
+                               test_protocol::Evt> {
 public:
   using JSONTransportTest::JSONTransportTest;
+
+  std::string Encode(const json::Value &V) override {
+    std::string msg;
+    raw_string_ostream OS(msg);
+    OS << formatv("{0}", V);
+    return formatv("Content-Length: {0}\r\nContent-type: "
+                   "text/json\r\n\r\n{1}",
+                   msg.size(), msg)
+        .str();
+  }
 };
 
-struct JSONTestType {
-  std::string str;
+class TestJSONRPCTransport final
+    : public JSONRPCTransport<test_protocol::Req, test_protocol::Resp,
+                              test_protocol::Evt> {
+public:
+  using JSONRPCTransport::JSONRPCTransport;
+
+  void Log(llvm::StringRef message) override {
+    log_messages.emplace_back(message);
+  }
+
+  std::vector<std::string> log_messages;
 };
 
-llvm::json::Value toJSON(const JSONTestType &T) {
-  return llvm::json::Object{{"str", T.str}};
-}
+class JSONRPCTransportTest
+    : public JSONTransportTest<TestJSONRPCTransport, test_protocol::Req,
+                               test_protocol::Resp, test_protocol::Evt> {
+public:
+  using JSONTransportTest::JSONTransportTest;
+
+  std::string Encode(const json::Value &V) override {
+    std::string msg;
+    raw_string_ostream OS(msg);
+    OS << formatv("{0}\n", V);
+    return msg;
+  }
+};
 
-bool fromJSON(const llvm::json::Value &V, JSONTestType &T, llvm::json::Path P) {
-  llvm::json::ObjectMapper O(V, P);
-  return O && O.map("str", T.str);
-}
 } // namespace
 
+// Failing on Windows, see https://github.com/llvm/llvm-project/issues/153446.
+#ifndef _WIN32
+using namespace test_protocol;
+
 TEST_F(HTTPDelimitedJSONTransportTest, MalformedRequests) {
-  std::string malformed_header = "COnTent-LenGth: -1{}\r\n\r\nnotjosn";
+  std::string malformed_header =
+      "COnTent-LenGth: -1\r\nContent-Type: text/json\r\n\r\nnotjosn";
   ASSERT_THAT_EXPECTED(
       input.Write(malformed_header.data(), malformed_header.size()),
       Succeeded());
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      FailedWithMessage(
-          "expected 'Content-Length: ' and got 'COnTent-LenGth: '"));
+
+  EXPECT_CALL(message_handler, OnError(_)).WillOnce([](llvm::Error err) {
+    ASSERT_THAT_ERROR(std::move(err),
+                      FailedWithMessage("invalid content length: -1"));
+  });
+  ASSERT_THAT_ERROR(Run(), Succeeded());
 }
 
 TEST_F(HTTPDelimitedJSONTransportTest, Read) {
-  std::string json = R"json({"str": "foo"})json";
-  std::string message =
-      formatv("Content-Length: {0}\r\n\r\n{1}", json.size(), json).str();
-  ASSERT_THAT_EXPECTED(input.Write(message.data(), message.size()),
-                       Succeeded());
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      HasValue(testing::FieldsAre(/*str=*/"foo")));
+  Write(Req{"foo"});
+  EXPECT_CALL(message_handler, Received(Req{"foo"}));
+  ASSERT_THAT_ERROR(Run(), Succeeded());
 }
 
-TEST_F(HTTPDelimitedJSONTransportTest, ReadWithEOF) {
+TEST_F(HTTPDelimitedJSONTransportTest, ReadMultipleMessagesInSingleWrite) {
+  InSequence seq;
+  Write(Message{Req{"one"}}, Message{Evt{"two"}}, Message{Resp{"three"}});
+  EXPECT_CALL(message_handler, Received(Req{"one"}));
+  EXPECT_CALL(message_handler, Received(Evt{"two"}));
+  EXPECT_CALL(message_handler, Received(Resp{"three"}));
+  ASSERT_THAT_ERROR(Run(), Succeeded());
+}
+
+TEST_F(HTTPDelimitedJSONTransportTest, ReadAcrossMultipleChunks) {
+  std::string long_str = std::string(
+      HTTPDelimitedJSONTransport<Req, Resp, Evt>::kReadBufferSize * 2, 'x');
+  Write(Req{long_str});
+  EXPECT_CALL(message_handler, Received(Req{long_str}));
+  ASSERT_THAT_ERROR(Run(), Succeeded());
+}
+
+TEST_F(HTTPDelimitedJSONTransportTest, ReadPartialMessage) {
+  std::string message = Encode(Req{"foo"});
+  auto split_at = message.size() / 2;
+  std::string part1 = message.substr(0, split_at);
+  std::string part2 = message.substr(split_at);
+
+  EXPECT_CALL(message_handler, Received(Req{"foo"}));
+
+  ASSERT_THAT_EXPECTED(input.Write(part1.data(), part1.size()), Succeeded());
+  loop.AddPendingCallback(
+      [](MainLoopBase &loop) { loop.RequestTermination(); });
+  ASSERT_THAT_ERROR(Run(/*close_stdin=*/false), Succeeded());
+  ASSERT_THAT_EXPECTED(input.Write(part2.data(), part2.size()), Succeeded());
   input.CloseWriteFileDescriptor();
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      Failed<TransportEOFError>());
+  ASSERT_THAT_ERROR(Run(), Succeeded());
+}
+
+TEST_F(HTTPDelimitedJSONTransportTest, ReadWithZeroByteWrites) {
+  std::string message = Encode(Req{"foo"});
+  auto split_at = message.size() / 2;
+  std::string part1 = message.substr(0, split_at);
+  std::string part2 = message.substr(split_at);
+
+  EXPECT_CALL(message_handler, Received(Req{"foo"}));
+
+  ASSERT_THAT_EXPECTED(input.Write(part1.data(), part1.size()), Succeeded());
+
+  // Run the main loop once for the initial read.
+  loop.AddPendingCallback(
+      [](MainLoopBase &loop) { loop.RequestTermination(); });
+  ASSERT_THAT_ERROR(Run(/*close_stdin=*/false), Succeeded());
+
+  // zero-byte write.
+  ASSERT_THAT_EXPECTED(input.Write(part1.data(), 0),
+                       Succeeded()); // zero-byte write.
+  loop.AddPendingCallback(
+      [](MainLoopBase &loop) { loop.RequestTermination(); });
+  ASSERT_THAT_ERROR(Run(/*close_stdin=*/false), Succeeded());
+
+  // Write the remaining part of the message.
+  ASSERT_THAT_EXPECTED(input.Write(part2.data(), part2.size()), Succeeded());
+  ASSERT_THAT_ERROR(Run(), Succeeded());
+}
+
+TEST_F(HTTPDelimitedJSONTransportTest, ReadWithEOF) {
+  ASSERT_THAT_ERROR(Run(), Succeeded());
 }
 
+TEST_F(HTTPDelimitedJSONTransportTest, ReaderWithUnhandledData) {
+  std::string json = R"json({"str": "foo"})json";
+  std::string message =
+      formatv("Content-Length: {0}\r\nContent-type: text/json\r\n\r\n{1}",
+              json.size(), json)
+          .str();
+
+  EXPECT_CALL(message_handler, OnError(_)).WillOnce([](llvm::Error err) {
+    // The error should indicate that there are unhandled contents.
+    ASSERT_THAT_ERROR(std::move(err),
+                      Failed<TransportUnhandledContentsError>());
+  });
+
+  // Write an incomplete message and close the handle.
+  ASSERT_THAT_EXPECTED(input.Write(message.data(), message.size() - 1),
+                       Succeeded());
+  ASSERT_THAT_ERROR(Run(), Succeeded());
+}
 
 TEST_F(HTTPDelimitedJSONTransportTest, InvalidTransport) {
-  transport = std::make_unique<HTTPDelimitedJSONTransport>(nullptr, nullptr);
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      Failed<TransportInvalidError>());
+  transport =
+      std::make_unique<TestHTTPDelimitedJSONTransport>(nullptr, nullptr);
+  ASSERT_THAT_ERROR(Run(/*close_input=*/false),
+                    FailedWithMessage("IO object is not valid."));
 }
 
 TEST_F(HTTPDelimitedJSONTransportTest, Write) {
-  ASSERT_THAT_ERROR(transport->Write(JSONTestType{"foo"}), Succeeded());
+  ASSERT_THAT_ERROR(transport->Send(Req{"foo"}), Succeeded());
+  ASSERT_THAT_ERROR(transport->Send(Resp{"bar"}), Succeeded());
+  ASSERT_THAT_ERROR(transport->Send(Evt{"baz"}), Succeeded());
   output.CloseWriteFileDescriptor();
   char buf[1024];
   Expected<size_t> bytes_read =
       output.Read(buf, sizeof(buf), std::chrono::milliseconds(1));
   ASSERT_THAT_EXPECTED(bytes_read, Succeeded());
   ASSERT_EQ(StringRef(buf, *bytes_read), StringRef("Content-Length: 13\r\n\r\n"
-                                                   R"json({"str":"foo"})json"));
+                                                   R"({"req":"foo"})"
+                                                   "Content-Length: 14\r\n\r\n"
+                                                   R"({"resp":"bar"})"
+                                                   "Content-Length: 13\r\n\r\n"
+                                                   R"({"evt":"baz"})"));
 }
 
 TEST_F(JSONRPCTransportTest, MalformedRequests) {
@@ -108,74 +387,94 @@ TEST_F(JSONRPCTransportTest, MalformedRequests) {
   ASSERT_THAT_EXPECTED(
       input.Write(malformed_header.data(), malformed_header.size()),
       Succeeded());
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      llvm::Failed());
+  EXPECT_CALL(message_handler, OnError(_)).WillOnce([](llvm::Error err) {
+    ASSERT_THAT_ERROR(std::move(err),
+                      FailedWithMessage(HasSubstr("Invalid JSON value")));
+  });
+  ASSERT_THAT_ERROR(Run(), Succeeded());
 }
 
 TEST_F(JSONRPCTransportTest, Read) {
-  std::string json = R"json({"str": "foo"})json";
-  std::string message = formatv("{0}\n", json).str();
-  ASSERT_THAT_EXPECTED(input.Write(message.data(), message.size()),
-                       Succeeded());
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      HasValue(testing::FieldsAre(/*str=*/"foo")));
+  Write(Message{Req{"foo"}});
+  EXPECT_CALL(message_handler, Received(Req{"foo"}));
+  ASSERT_THAT_ERROR(Run(), Succeeded());
 }
 
-TEST_F(JSONRPCTransportTest, ReadWithEOF) {
+TEST_F(JSONRPCTransportTest, ReadMultipleMessagesInSingleWrite) {
+  InSequence seq;
+  Write(Message{Req{"one"}}, Message{Evt{"two"}}, Message{Resp{"three"}});
+  EXPECT_CALL(message_handler, Received(Req{"one"}));
+  EXPECT_CALL(message_handler, Received(Evt{"two"}));
+  EXPECT_CALL(message_handler, Received(Resp{"three"}));
+  ASSERT_THAT_ERROR(Run(), Succeeded());
+}
+
+TEST_F(JSONRPCTransportTest, ReadAcrossMultipleChunks) {
+  // Use a string longer than the chunk size to ensure we split the message
+  // across the chunk boundary.
+  std::string long_str =
+      std::string(JSONTransport<Req, Resp, Evt>::kReadBufferSize * 2, 'x');
+  Write(Req{long_str});
+  EXPECT_CALL(message_handler, Received(Req{long_str}));
+  ASSERT_THAT_ERROR(Run(), Succeeded());
+}
+
+TEST_F(JSONRPCTransportTest, ReadPartialMessage) {
+  std::string message = R"({"req": "foo"})"
+                        "\n";
+  std::string part1 = message.substr(0, 7);
+  std::string part2 = message.substr(7);
+
+  EXPECT_CALL(message_handler, Received(Req{"foo"}));
+
+  ASSERT_THAT_EXPECTED(input.Write(part1.data(), part1.size()), Succeeded());
+  loop.AddPendingCallback(
+      [](MainLoopBase &loop) { loop.RequestTermination(); });
+  ASSERT_THAT_ERROR(Run(/*close_input=*/false), Succeeded());
+
+  ASSERT_THAT_EXPECTED(input.Write(part2.data(), part2.size()), Succeeded());
   input.CloseWriteFileDescriptor();
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      Failed<TransportEOFError>());
+  ASSERT_THAT_ERROR(Run(), Succeeded());
+}
+
+TEST_F(JSONRPCTransportTest, ReadWithEOF) {
+  ASSERT_THAT_ERROR(Run(), Succeeded());
+}
+
+TEST_F(JSONRPCTransportTest, ReaderWithUnhandledData) {
+  std::string message = R"json({"req": "foo")json";
+  // Write an incomplete message and close the handle.
+  ASSERT_THAT_EXPECTED(input.Write(message.data(), message.size() - 1),
+                       Succeeded());
+
+  EXPECT_CALL(message_handler, OnError(_)).WillOnce([](llvm::Error err) {
+    ASSERT_THAT_ERROR(std::move(err),
+                      Failed<TransportUnhandledContentsError>());
+  });
+  ASSERT_THAT_ERROR(Run(), Succeeded());
 }
 
 TEST_F(JSONRPCTransportTest, Write) {
-  ASSERT_THAT_ERROR(transport->Write(JSONTestType{"foo"}), Succeeded());
+  ASSERT_THAT_ERROR(transport->Send(Req{"foo"}), Succeeded());
+  ASSERT_THAT_ERROR(transport->Send(Resp{"bar"}), Succeeded());
+  ASSERT_THAT_ERROR(transport->Send(Evt{"baz"}), Succeeded());
   output.CloseWriteFileDescriptor();
   char buf[1024];
   Expected<size_t> bytes_read =
       output.Read(buf, sizeof(buf), std::chrono::milliseconds(1));
   ASSERT_THAT_EXPECTED(bytes_read, Succeeded());
-  ASSERT_EQ(StringRef(buf, *bytes_read), StringRef(R"json({"str":"foo"})json"
+  ASSERT_EQ(StringRef(buf, *bytes_read), StringRef(R"({"req":"foo"})"
+                                                   "\n"
+                                                   R"({"resp":"bar"})"
+                                                   "\n"
+                                                   R"({"evt":"baz"})"
                                                    "\n"));
 }
 
 TEST_F(JSONRPCTransportTest, InvalidTransport) {
-  transport = std::make_unique<JSONRPCTransport>(nullptr, nullptr);
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      Failed<TransportInvalidError>());
-}
-
-#ifndef _WIN32
-TEST_F(HTTPDelimitedJSONTransportTest, ReadWithTimeout) {
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      Failed<TransportTimeoutError>());
+  transport = std::make_unique<TestJSONRPCTransport>(nullptr, nullptr);
+  ASSERT_THAT_ERROR(Run(/*close_input=*/false),
+                    FailedWithMessage("IO object is not valid."));
 }
 
-TEST_F(JSONRPCTransportTest, ReadWithTimeout) {
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      Failed<TransportTimeoutError>());
-}
-
-// Windows CRT _read checks that the file descriptor is valid and calls a
-// handler if not. This handler is normally a breakpoint, which looks like a
-// crash when not handled by a debugger.
-// https://learn.microsoft.com/en-us/%20cpp/c-runtime-library/reference/read?view=msvc-170
-TEST_F(HTTPDelimitedJSONTransportTest, ReadAfterClosed) {
-  input.CloseReadFileDescriptor();
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      llvm::Failed());
-}
-
-TEST_F(JSONRPCTransportTest, ReadAfterClosed) {
-  input.CloseReadFileDescriptor();
-  ASSERT_THAT_EXPECTED(
-      transport->Read<JSONTestType>(std::chrono::milliseconds(1)),
-      llvm::Failed());
-}
 #endif
diff --git a/lldb/unittests/Host/MainLoopTest.cpp b/lldb/unittests/Host/MainLoopTest.cpp
index 448ad9cfd20f0..0bc291c26b9c9 100644
--- a/lldb/unittests/Host/MainLoopTest.cpp
+++ b/lldb/unittests/Host/MainLoopTest.cpp
@@ -80,6 +80,8 @@ TEST_F(MainLoopTest, ReadSocketObject) {
   ASSERT_EQ(1u, callback_count);
 }
 
+// Flakey, see https://github.com/llvm/llvm-project/issues/152677.
+#ifndef _WIN32
 TEST_F(MainLoopTest, ReadPipeObject) {
   Pipe pipe;
 
@@ -142,6 +144,7 @@ TEST_F(MainLoopTest, MultipleReadsPipeObject) {
   ASSERT_EQ(5u, callback_count);
   async_writer.wait();
 }
+#endif
 
 TEST_F(MainLoopTest, PipeDelayBetweenRegisterAndRun) {
   Pipe pipe;
diff --git a/lldb/unittests/Instruction/ARM64/TestAArch64Emulator.cpp b/lldb/unittests/Instruction/ARM64/TestAArch64Emulator.cpp
index 08296da564976..c6bfe0fa68759 100644
--- a/lldb/unittests/Instruction/ARM64/TestAArch64Emulator.cpp
+++ b/lldb/unittests/Instruction/ARM64/TestAArch64Emulator.cpp
@@ -62,7 +62,7 @@ struct Arch64EmulatorTester : public EmulateInstructionARM64 {
       reg_value.SetUInt64(tester->gpr.pc);
       return true;
     case gpr_cpsr_arm64:
-      reg_value.SetUInt64(tester->gpr.cpsr);
+      reg_value.SetUInt32(tester->gpr.cpsr);
       return true;
     default:
       return false;
@@ -97,7 +97,7 @@ struct Arch64EmulatorTester : public EmulateInstructionARM64 {
       tester->gpr.pc = reg_value.GetAsUInt64();
       return true;
     case gpr_cpsr_arm64:
-      tester->gpr.cpsr = reg_value.GetAsUInt64();
+      tester->gpr.cpsr = reg_value.GetAsUInt32();
       return true;
     default:
       return false;
@@ -112,7 +112,7 @@ struct Arch64EmulatorTester : public EmulateInstructionARM64 {
     assert(addr - tester->memory_offset + length <= sizeof(tester->memory));
     if (addr >= tester->memory_offset &&
         addr - tester->memory_offset + length <= sizeof(tester->memory)) {
-      memcpy(dst, tester->memory + addr - tester->memory_offset, length);
+      memcpy(dst, tester->memory + (addr - tester->memory_offset), length);
       return length;
     }
     return 0;
diff --git a/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp b/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp
index 6eeb4f54952b9..957fb3f600499 100644
--- a/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp
+++ b/lldb/unittests/Language/CPlusPlus/CPlusPlusLanguageTest.cpp
@@ -397,3 +397,33 @@ TEST(CPlusPlusLanguage, CPlusPlusNameParser) {
   // Don't crash.
   CPlusPlusNameParser((const char *)nullptr);
 }
+
+TEST(CPlusPlusLanguage, DoesNotMatchCxx) {
+  // Test that a symbol name that is NOT C++ does not match C++.
+
+  SubsystemRAII<CPlusPlusLanguage> lang;
+  Language *CPlusPlusLang =
+      Language::FindPlugin(lldb::eLanguageTypeC_plus_plus);
+
+  EXPECT_TRUE(CPlusPlusLang != nullptr);
+
+  Mangled swiftSymbol("$sS");
+  EXPECT_FALSE(CPlusPlusLang->SymbolNameFitsToLanguage(swiftSymbol));
+}
+
+TEST(CPlusPlusLanguage, MatchesCxx) {
+  // Test that a symbol name that is C++ does match C++ (both Itanium and MSVC).
+
+  SubsystemRAII<CPlusPlusLanguage> lang;
+  Language *CPlusPlusLang =
+      Language::FindPlugin(lldb::eLanguageTypeC_plus_plus);
+
+  EXPECT_TRUE(CPlusPlusLang != nullptr);
+
+  Mangled itaniumSymbol("_Z3Foo");
+  EXPECT_TRUE(CPlusPlusLang->SymbolNameFitsToLanguage(itaniumSymbol));
+  Mangled itaniumExtensionSymbol("___Z3Bar_block_invoke");
+  EXPECT_TRUE(CPlusPlusLang->SymbolNameFitsToLanguage(itaniumExtensionSymbol));
+  Mangled msvcSymbol("??x@@3AH");
+  EXPECT_TRUE(CPlusPlusLang->SymbolNameFitsToLanguage(msvcSymbol));
+}
diff --git a/lldb/unittests/Protocol/ProtocolMCPTest.cpp b/lldb/unittests/Protocol/ProtocolMCPTest.cpp
index 1dca0e5fc5bb8..ea19922522ffe 100644
--- a/lldb/unittests/Protocol/ProtocolMCPTest.cpp
+++ b/lldb/unittests/Protocol/ProtocolMCPTest.cpp
@@ -149,9 +149,7 @@ TEST(ProtocolMCPTest, MessageWithRequest) {
   const Request &deserialized_request =
       std::get<Request>(*deserialized_message);
 
-  EXPECT_EQ(request.id, deserialized_request.id);
-  EXPECT_EQ(request.method, deserialized_request.method);
-  EXPECT_EQ(request.params, deserialized_request.params);
+  EXPECT_EQ(request, deserialized_request);
 }
 
 TEST(ProtocolMCPTest, MessageWithResponse) {
@@ -168,8 +166,7 @@ TEST(ProtocolMCPTest, MessageWithResponse) {
   const Response &deserialized_response =
       std::get<Response>(*deserialized_message);
 
-  EXPECT_EQ(response.id, deserialized_response.id);
-  EXPECT_EQ(response.result, deserialized_response.result);
+  EXPECT_EQ(response, deserialized_response);
 }
 
 TEST(ProtocolMCPTest, MessageWithNotification) {
@@ -186,49 +183,28 @@ TEST(ProtocolMCPTest, MessageWithNotification) {
   const Notification &deserialized_notification =
       std::get<Notification>(*deserialized_message);
 
-  EXPECT_EQ(notification.method, deserialized_notification.method);
-  EXPECT_EQ(notification.params, deserialized_notification.params);
+  EXPECT_EQ(notification, deserialized_notification);
 }
 
-TEST(ProtocolMCPTest, MessageWithError) {
-  ErrorInfo error_info;
-  error_info.code = -32603;
-  error_info.message = "Internal error";
-
+TEST(ProtocolMCPTest, MessageWithErrorResponse) {
   Error error;
-  error.id = 3;
-  error.error = error_info;
+  error.code = -32603;
+  error.message = "Internal error";
+
+  Response error_response;
+  error_response.id = 3;
+  error_response.result = error;
 
-  Message message = error;
+  Message message = error_response;
 
   llvm::Expected<Message> deserialized_message = roundtripJSON(message);
   ASSERT_THAT_EXPECTED(deserialized_message, llvm::Succeeded());
 
-  ASSERT_TRUE(std::holds_alternative<Error>(*deserialized_message));
-  const Error &deserialized_error = std::get<Error>(*deserialized_message);
-
-  EXPECT_EQ(error.id, deserialized_error.id);
-  EXPECT_EQ(error.error.code, deserialized_error.error.code);
-  EXPECT_EQ(error.error.message, deserialized_error.error.message);
-}
-
-TEST(ProtocolMCPTest, ResponseWithError) {
-  ErrorInfo error_info;
-  error_info.code = -32700;
-  error_info.message = "Parse error";
-
-  Response response;
-  response.id = 4;
-  response.error = error_info;
-
-  llvm::Expected<Response> deserialized_response = roundtripJSON(response);
-  ASSERT_THAT_EXPECTED(deserialized_response, llvm::Succeeded());
+  ASSERT_TRUE(std::holds_alternative<Response>(*deserialized_message));
+  const Response &deserialized_error =
+      std::get<Response>(*deserialized_message);
 
-  EXPECT_EQ(response.id, deserialized_response->id);
-  EXPECT_FALSE(deserialized_response->result.has_value());
-  ASSERT_TRUE(deserialized_response->error.has_value());
-  EXPECT_EQ(response.error->code, deserialized_response->error->code);
-  EXPECT_EQ(response.error->message, deserialized_response->error->message);
+  EXPECT_EQ(error_response, deserialized_error);
 }
 
 TEST(ProtocolMCPTest, Resource) {
diff --git a/lldb/unittests/ProtocolServer/ProtocolMCPServerTest.cpp b/lldb/unittests/ProtocolServer/ProtocolMCPServerTest.cpp
index dc3b63ab65466..18112428950ce 100644
--- a/lldb/unittests/ProtocolServer/ProtocolMCPServerTest.cpp
+++ b/lldb/unittests/ProtocolServer/ProtocolMCPServerTest.cpp
@@ -8,23 +8,33 @@
 
 #include "Plugins/Platform/MacOSX/PlatformRemoteMacOSX.h"
 #include "Plugins/Protocol/MCP/ProtocolServerMCP.h"
-#include "TestingSupport/Host/SocketTestUtilities.h"
+#include "TestingSupport/Host/JSONTransportTestUtilities.h"
 #include "TestingSupport/SubsystemRAII.h"
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/ProtocolServer.h"
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Host/JSONTransport.h"
+#include "lldb/Host/MainLoop.h"
+#include "lldb/Host/MainLoopBase.h"
 #include "lldb/Host/Socket.h"
+#include "lldb/Host/common/TCPSocket.h"
 #include "lldb/Protocol/MCP/MCPError.h"
 #include "lldb/Protocol/MCP/Protocol.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Testing/Support/Error.h"
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include <chrono>
+#include <condition_variable>
+#include <mutex>
 
 using namespace llvm;
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_protocol::mcp;
+using testing::_;
 
 namespace {
 class TestProtocolServerMCP : public lldb_private::mcp::ProtocolServerMCP {
@@ -37,11 +47,18 @@ class TestProtocolServerMCP : public lldb_private::mcp::ProtocolServerMCP {
   using ProtocolServerMCP::ProtocolServerMCP;
 };
 
-class TestJSONTransport : public lldb_private::JSONRPCTransport {
+using Message = typename Transport<Request, Response, Notification>::Message;
+
+class TestJSONTransport final
+    : public lldb_private::JSONRPCTransport<Request, Response, Notification> {
 public:
   using JSONRPCTransport::JSONRPCTransport;
-  using JSONRPCTransport::ReadImpl;
-  using JSONRPCTransport::WriteImpl;
+
+  void Log(llvm::StringRef message) override {
+    log_messages.emplace_back(message);
+  }
+
+  std::vector<std::string> log_messages;
 };
 
 /// Test tool that returns it argument as text.
@@ -49,7 +66,7 @@ class TestTool : public Tool {
 public:
   using Tool::Tool;
 
-  virtual llvm::Expected<TextResult> Call(const ToolArguments &args) override {
+  llvm::Expected<TextResult> Call(const ToolArguments &args) override {
     std::string argument;
     if (const json::Object *args_obj =
             std::get<json::Value>(args).getAsObject()) {
@@ -67,7 +84,7 @@ class TestTool : public Tool {
 class TestResourceProvider : public ResourceProvider {
   using ResourceProvider::ResourceProvider;
 
-  virtual std::vector<Resource> GetResources() const override {
+  std::vector<Resource> GetResources() const override {
     std::vector<Resource> resources;
 
     Resource resource;
@@ -80,7 +97,7 @@ class TestResourceProvider : public ResourceProvider {
     return resources;
   }
 
-  virtual llvm::Expected<ResourceResult>
+  llvm::Expected<ResourceResult>
   ReadResource(llvm::StringRef uri) const override {
     if (uri != "lldb://foo/bar")
       return llvm::make_error<UnsupportedURI>(uri.str());
@@ -101,7 +118,7 @@ class ErrorTool : public Tool {
 public:
   using Tool::Tool;
 
-  virtual llvm::Expected<TextResult> Call(const ToolArguments &args) override {
+  llvm::Expected<TextResult> Call(const ToolArguments &args) override {
     return llvm::createStringError("error");
   }
 };
@@ -111,7 +128,7 @@ class FailTool : public Tool {
 public:
   using Tool::Tool;
 
-  virtual llvm::Expected<TextResult> Call(const ToolArguments &args) override {
+  llvm::Expected<TextResult> Call(const ToolArguments &args) override {
     TextResult text_result;
     text_result.content.emplace_back(TextContent{{"failed"}});
     text_result.isError = true;
@@ -127,18 +144,34 @@ class ProtocolServerMCPTest : public ::testing::Test {
   lldb::IOObjectSP m_io_sp;
   std::unique_ptr<TestJSONTransport> m_transport_up;
   std::unique_ptr<TestProtocolServerMCP> m_server_up;
+  MainLoop loop;
+  MockMessageHandler<Request, Response, Notification> message_handler;
 
   static constexpr llvm::StringLiteral k_localhost = "localhost";
 
   llvm::Error Write(llvm::StringRef message) {
-    return m_transport_up->WriteImpl(llvm::formatv("{0}\n", message).str());
+    std::string output = llvm::formatv("{0}\n", message).str();
+    size_t bytes_written = output.size();
+    return m_io_sp->Write(output.data(), bytes_written).takeError();
   }
 
-  llvm::Expected<std::string> Read() {
-    return m_transport_up->ReadImpl(std::chrono::milliseconds(100));
+  void CloseInput() {
+    EXPECT_THAT_ERROR(m_io_sp->Close().takeError(), Succeeded());
   }
 
-  void SetUp() {
+  /// Run the transport MainLoop and return any messages received.
+  llvm::Error
+  Run(std::chrono::milliseconds timeout = std::chrono::milliseconds(200)) {
+    loop.AddCallback([](MainLoopBase &loop) { loop.RequestTermination(); },
+                     timeout);
+    auto handle = m_transport_up->RegisterMessageHandler(loop, message_handler);
+    if (!handle)
+      return handle.takeError();
+
+    return loop.Run().takeError();
+  }
+
+  void SetUp() override {
     // Create a debugger.
     ArchSpec arch("arm64-apple-macosx-");
     Platform::SetHostPlatform(
@@ -170,7 +203,7 @@ class ProtocolServerMCPTest : public ::testing::Test {
     m_transport_up = std::make_unique<TestJSONTransport>(m_io_sp, m_io_sp);
   }
 
-  void TearDown() {
+  void TearDown() override {
     // Stop the server.
     ASSERT_THAT_ERROR(m_server_up->Stop(), llvm::Succeeded());
   }
@@ -178,44 +211,47 @@ class ProtocolServerMCPTest : public ::testing::Test {
 
 } // namespace
 
-TEST_F(ProtocolServerMCPTest, Intialization) {
+TEST_F(ProtocolServerMCPTest, Initialization) {
   llvm::StringLiteral request =
-      R"json({"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"lldb-unit","version":"0.1.0"}},"jsonrpc":"2.0","id":0})json";
+      R"json({"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"lldb-unit","version":"0.1.0"}},"jsonrpc":"2.0","id":1})json";
   llvm::StringLiteral response =
-      R"json( {"id":0,"jsonrpc":"2.0","result":{"capabilities":{"resources":{"listChanged":false,"subscribe":false},"tools":{"listChanged":true}},"protocolVersion":"2024-11-05","serverInfo":{"name":"lldb-mcp","version":"0.1.0"}}})json";
-
-  ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
+      R"json({"id":1,"jsonrpc":"2.0","result":{"capabilities":{"resources":{"listChanged":false,"subscribe":false},"tools":{"listChanged":true}},"protocolVersion":"2024-11-05","serverInfo":{"name":"lldb-mcp","version":"0.1.0"}}})json";
 
-  llvm::Expected<std::string> response_str = Read();
-  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
-
-  llvm::Expected<json::Value> response_json = json::parse(*response_str);
-  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
-
-  llvm::Expected<json::Value> expected_json = json::parse(response);
-  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
-
-  EXPECT_EQ(*response_json, *expected_json);
+  ASSERT_THAT_ERROR(Write(request), Succeeded());
+  llvm::Expected<Response> expected_resp = json::parse<Response>(response);
+  ASSERT_THAT_EXPECTED(expected_resp, llvm::Succeeded());
+  EXPECT_CALL(message_handler, Received(*expected_resp));
+  EXPECT_THAT_ERROR(Run(), Succeeded());
 }
 
 TEST_F(ProtocolServerMCPTest, ToolsList) {
   llvm::StringLiteral request =
-      R"json({"method":"tools/list","params":{},"jsonrpc":"2.0","id":1})json";
-  llvm::StringLiteral response =
-      R"json({"id":1,"jsonrpc":"2.0","result":{"tools":[{"description":"test tool","inputSchema":{"type":"object"},"name":"test"},{"description":"Run an lldb command.","inputSchema":{"properties":{"arguments":{"type":"string"},"debugger_id":{"type":"number"}},"required":["debugger_id"],"type":"object"},"name":"lldb_command"}]}})json";
+      R"json({"method":"tools/list","params":{},"jsonrpc":"2.0","id":"one"})json";
+
+  ToolDefinition test_tool;
+  test_tool.name = "test";
+  test_tool.description = "test tool";
+  test_tool.inputSchema = json::Object{{"type", "object"}};
+
+  ToolDefinition lldb_command_tool;
+  lldb_command_tool.description = "Run an lldb command.";
+  lldb_command_tool.name = "lldb_command";
+  lldb_command_tool.inputSchema = json::Object{
+      {"type", "object"},
+      {"properties",
+       json::Object{{"arguments", json::Object{{"type", "string"}}},
+                    {"debugger_id", json::Object{{"type", "number"}}}}},
+      {"required", json::Array{"debugger_id"}}};
+  Response response;
+  response.id = "one";
+  response.result = json::Object{
+      {"tools",
+       json::Array{std::move(test_tool), std::move(lldb_command_tool)}},
+  };
 
   ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
-
-  llvm::Expected<std::string> response_str = Read();
-  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
-
-  llvm::Expected<json::Value> response_json = json::parse(*response_str);
-  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
-
-  llvm::Expected<json::Value> expected_json = json::parse(response);
-  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
-
-  EXPECT_EQ(*response_json, *expected_json);
+  EXPECT_CALL(message_handler, Received(response));
+  EXPECT_THAT_ERROR(Run(), Succeeded());
 }
 
 TEST_F(ProtocolServerMCPTest, ResourcesList) {
@@ -225,17 +261,10 @@ TEST_F(ProtocolServerMCPTest, ResourcesList) {
       R"json({"id":2,"jsonrpc":"2.0","result":{"resources":[{"description":"description","mimeType":"application/json","name":"name","uri":"lldb://foo/bar"}]}})json";
 
   ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
-
-  llvm::Expected<std::string> response_str = Read();
-  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
-
-  llvm::Expected<json::Value> response_json = json::parse(*response_str);
-  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
-
-  llvm::Expected<json::Value> expected_json = json::parse(response);
-  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
-
-  EXPECT_EQ(*response_json, *expected_json);
+  llvm::Expected<Response> expected_resp = json::parse<Response>(response);
+  ASSERT_THAT_EXPECTED(expected_resp, llvm::Succeeded());
+  EXPECT_CALL(message_handler, Received(*expected_resp));
+  EXPECT_THAT_ERROR(Run(), Succeeded());
 }
 
 TEST_F(ProtocolServerMCPTest, ToolsCall) {
@@ -245,17 +274,10 @@ TEST_F(ProtocolServerMCPTest, ToolsCall) {
       R"json({"id":11,"jsonrpc":"2.0","result":{"content":[{"text":"foo","type":"text"}],"isError":false}})json";
 
   ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
-
-  llvm::Expected<std::string> response_str = Read();
-  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
-
-  llvm::Expected<json::Value> response_json = json::parse(*response_str);
-  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
-
-  llvm::Expected<json::Value> expected_json = json::parse(response);
-  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
-
-  EXPECT_EQ(*response_json, *expected_json);
+  llvm::Expected<Response> expected_resp = json::parse<Response>(response);
+  ASSERT_THAT_EXPECTED(expected_resp, llvm::Succeeded());
+  EXPECT_CALL(message_handler, Received(*expected_resp));
+  EXPECT_THAT_ERROR(Run(), Succeeded());
 }
 
 TEST_F(ProtocolServerMCPTest, ToolsCallError) {
@@ -267,17 +289,10 @@ TEST_F(ProtocolServerMCPTest, ToolsCallError) {
       R"json({"error":{"code":-32603,"message":"error"},"id":11,"jsonrpc":"2.0"})json";
 
   ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
-
-  llvm::Expected<std::string> response_str = Read();
-  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
-
-  llvm::Expected<json::Value> response_json = json::parse(*response_str);
-  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
-
-  llvm::Expected<json::Value> expected_json = json::parse(response);
-  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
-
-  EXPECT_EQ(*response_json, *expected_json);
+  llvm::Expected<Response> expected_resp = json::parse<Response>(response);
+  ASSERT_THAT_EXPECTED(expected_resp, llvm::Succeeded());
+  EXPECT_CALL(message_handler, Received(*expected_resp));
+  EXPECT_THAT_ERROR(Run(), Succeeded());
 }
 
 TEST_F(ProtocolServerMCPTest, ToolsCallFail) {
@@ -289,17 +304,10 @@ TEST_F(ProtocolServerMCPTest, ToolsCallFail) {
       R"json({"id":11,"jsonrpc":"2.0","result":{"content":[{"text":"failed","type":"text"}],"isError":true}})json";
 
   ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
-
-  llvm::Expected<std::string> response_str = Read();
-  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
-
-  llvm::Expected<json::Value> response_json = json::parse(*response_str);
-  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
-
-  llvm::Expected<json::Value> expected_json = json::parse(response);
-  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
-
-  EXPECT_EQ(*response_json, *expected_json);
+  llvm::Expected<Response> expected_resp = json::parse<Response>(response);
+  ASSERT_THAT_EXPECTED(expected_resp, llvm::Succeeded());
+  EXPECT_CALL(message_handler, Received(*expected_resp));
+  EXPECT_THAT_ERROR(Run(), Succeeded());
 }
 
 TEST_F(ProtocolServerMCPTest, NotificationInitialized) {
diff --git a/lldb/unittests/TestingSupport/Host/JSONTransportTestUtilities.h b/lldb/unittests/TestingSupport/Host/JSONTransportTestUtilities.h
new file mode 100644
index 0000000000000..5a9eb8e59f2b6
--- /dev/null
+++ b/lldb/unittests/TestingSupport/Host/JSONTransportTestUtilities.h
@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_UNITTESTS_TESTINGSUPPORT_HOST_NATIVEPROCESSTESTUTILS_H
+#define LLDB_UNITTESTS_TESTINGSUPPORT_HOST_NATIVEPROCESSTESTUTILS_H
+
+#include "lldb/Host/JSONTransport.h"
+#include "gmock/gmock.h"
+
+template <typename Req, typename Resp, typename Evt>
+class MockMessageHandler final
+    : public lldb_private::Transport<Req, Resp, Evt>::MessageHandler {
+public:
+  MOCK_METHOD(void, Received, (const Evt &), (override));
+  MOCK_METHOD(void, Received, (const Req &), (override));
+  MOCK_METHOD(void, Received, (const Resp &), (override));
+  MOCK_METHOD(void, OnError, (llvm::Error), (override));
+  MOCK_METHOD(void, OnClosed, (), (override));
+};
+
+#endif
diff --git a/lldb/unittests/TestingSupport/TestUtilities.h b/lldb/unittests/TestingSupport/TestUtilities.h
index db62881872fef..cc93a68a6a431 100644
--- a/lldb/unittests/TestingSupport/TestUtilities.h
+++ b/lldb/unittests/TestingSupport/TestUtilities.h
@@ -11,11 +11,11 @@
 
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Utility/DataBuffer.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
-#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/raw_ostream.h"
 #include <string>
 
 #define ASSERT_NO_ERROR(x)                                                     \
@@ -61,12 +61,10 @@ class TestFile {
 };
 
 template <typename T> static llvm::Expected<T> roundtripJSON(const T &input) {
-  llvm::json::Value value = toJSON(input);
-  llvm::json::Path::Root root;
-  T output;
-  if (!fromJSON(value, output, root))
-    return root.getError();
-  return output;
+  std::string encoded;
+  llvm::raw_string_ostream OS(encoded);
+  OS << toJSON(input);
+  return llvm::json::parse<T>(encoded);
 }
 } // namespace lldb_private
 
diff --git a/lldb/unittests/Utility/XcodeSDKTest.cpp b/lldb/unittests/Utility/XcodeSDKTest.cpp
index 4db6a50fcf860..de9f91a04d53e 100644
--- a/lldb/unittests/Utility/XcodeSDKTest.cpp
+++ b/lldb/unittests/Utility/XcodeSDKTest.cpp
@@ -102,17 +102,6 @@ TEST(XcodeSDKTest, SDKSupportsModules) {
 }
 #endif
 
-TEST(XcodeSDKTest, SDKSupportsSwift) {
-  EXPECT_TRUE(XcodeSDK("iPhoneSimulator12.0.sdk").SupportsSwift());
-  EXPECT_TRUE(XcodeSDK("iPhoneSimulator12.0.Internal.sdk").SupportsSwift());
-  EXPECT_FALSE(XcodeSDK("iPhoneSimulator7.2.sdk").SupportsSwift());
-  EXPECT_TRUE(XcodeSDK("MacOSX10.10.sdk").SupportsSwift());
-  EXPECT_FALSE(XcodeSDK("MacOSX10.9.sdk").SupportsSwift());
-  EXPECT_TRUE(XcodeSDK("Linux.sdk").SupportsSwift());
-  EXPECT_TRUE(XcodeSDK("MacOSX.sdk").SupportsSwift());
-  EXPECT_FALSE(XcodeSDK("EverythingElse.sdk").SupportsSwift());
-}
-
 TEST(XcodeSDKTest, GetCanonicalNameAndConstruct) {
   XcodeSDK::Info info;
   info.type = XcodeSDK::Type::MacOSX;
diff --git a/lldb/utils/lui/lldbutil.py b/lldb/utils/lui/lldbutil.py
index 6cbf4a302f65f..140317af3670b 100644
--- a/lldb/utils/lui/lldbutil.py
+++ b/lldb/utils/lui/lldbutil.py
@@ -1040,8 +1040,8 @@ def format(self, value, buffer=None):
         return output.getvalue()
 
 
-class RecursiveDecentFormatter(BasicFormatter):
-    """The recursive decent formatter prints the value and the decendents.
+class RecursiveDescentFormatter(BasicFormatter):
+    """The recursive descent formatter prints the value and the descendents.
 
     The constructor takes two keyword args: indent_level, which defaults to 0,
     and indent_child, which defaults to 2.  The current indentation level is
@@ -1058,7 +1058,6 @@ def format(self, value, buffer=None):
             output = io.StringIO()
         else:
             output = buffer
-
         BasicFormatter.format(self, value, buffer=output, indent=self.lindent)
         new_indent = self.lindent + self.cindent
         for child in value:
@@ -1066,7 +1065,7 @@ def format(self, value, buffer=None):
                 BasicFormatter.format(self, child, buffer=output, indent=new_indent)
             else:
                 if child.GetNumChildren() > 0:
-                    rdf = RecursiveDecentFormatter(indent_level=new_indent)
+                    rdf = RecursiveDescentFormatter(indent_level=new_indent)
                     rdf.format(child, buffer=output)
                 else:
                     BasicFormatter.format(self, child, buffer=output, indent=new_indent)
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 4c70b98a32f92..a90be4f6235e6 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -110,7 +110,7 @@ endif()
 # one for llvm+clang+... using the same sources.
 # These projects will be included when "all" is included in LLVM_ENABLE_PROJECTS.
 set(LLVM_ALL_PROJECTS "bolt;clang;clang-tools-extra;compiler-rt;cross-project-tests;libclc;lld;lldb;mlir;openmp;polly")
-if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   # Disallow 'openmp' as a LLVM PROJECT on AIX as the supported way is to use
   # LLVM_ENABLE_RUNTIMES.
   list(REMOVE_ITEM LLVM_ALL_PROJECTS openmp)
@@ -149,7 +149,7 @@ endforeach()
 # As we migrate runtimes to using the bootstrapping build, the set of default runtimes
 # should grow as we remove those runtimes from LLVM_ENABLE_PROJECTS above.
 set(LLVM_DEFAULT_RUNTIMES "libcxx;libcxxabi;libunwind")
-set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;libcxx;compiler-rt;openmp;llvm-libgcc;offload;flang-rt;libclc;libsycl")
+set(LLVM_SUPPORTED_RUNTIMES "libc;libunwind;libcxxabi;libcxx;compiler-rt;openmp;llvm-libgcc;offload;flang-rt;libclc;libsycl;orc-rt")
 set(LLVM_ENABLE_RUNTIMES "" CACHE STRING
   "Semicolon-separated list of runtimes to build, or \"all\" (${LLVM_DEFAULT_RUNTIMES}). Supported runtimes are ${LLVM_SUPPORTED_RUNTIMES}.")
 if(LLVM_ENABLE_RUNTIMES STREQUAL "all")
@@ -187,28 +187,29 @@ if ("lldb" IN_LIST LLVM_ENABLE_PROJECTS)
 endif ()
 
 if ("libc" IN_LIST LLVM_ENABLE_PROJECTS)
-  message(WARNING "Using LLVM_ENABLE_PROJECTS=libc is deprecated.  Please use "
+  message(WARNING "Using LLVM_ENABLE_PROJECTS=libc is deprecated now, and will "
+    "become a fatal error in a future release. Please use "
     "-DLLVM_ENABLE_RUNTIMES=libc or see the instructions at "
     "https://libc.llvm.org/ for building the runtimes.")
 endif()
 
 if ("compiler-rt" IN_LIST LLVM_ENABLE_PROJECTS)
   message(WARNING "Using LLVM_ENABLE_PROJECTS=compiler-rt is deprecated now, and will "
-    "become a fatal error in the LLVM 21 release.  Please use "
+    "become a fatal error in a future release.  Please use "
     "-DLLVM_ENABLE_RUNTIMES=compiler-rt or see the instructions at "
     "https://compiler-rt.llvm.org/ for building the runtimes.")
 endif()
 
 if ("offload" IN_LIST LLVM_ENABLE_PROJECTS)
   message(WARNING "Using LLVM_ENABLE_PROJECTS=offload is deprecated now, and will "
-    "become a fatal error in the LLVM 21 release.  Please use "
+    "become a fatal error in a future release.  Please use "
     "-DLLVM_ENABLE_RUNTIMES=offload or see the instructions at "
     "https://openmp.llvm.org/ for building the runtimes.")
 endif()
 
 if ("openmp" IN_LIST LLVM_ENABLE_PROJECTS)
   message(WARNING "Using LLVM_ENABLE_PROJECTS=openmp is deprecated now, and will "
-    "become a fatal error in the LLVM 21 release.  Please use "
+    "become a fatal error in a future release.  Please use "
     "-DLLVM_ENABLE_RUNTIMES=openmp or see the instructions at "
     "https://openmp.llvm.org/ for building the runtimes.")
 endif()
@@ -221,7 +222,7 @@ endif ()
 
 if ("libclc" IN_LIST LLVM_ENABLE_PROJECTS)
   message(WARNING "Using LLVM_ENABLE_PROJECTS=libclc is deprecated now, and will "
-    "become a fatal error in the LLVM 21 release.  Please use "
+    "become a fatal error in a future release.  Please use "
     "-DLLVM_ENABLE_RUNTIMES=libclc or see the instructions at "
     "https://libclc.llvm.org/ for building the runtimes.")
 endif()
@@ -587,9 +588,6 @@ set(LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING "DISABLED" CACHE STRING
   "Enhance Debugify's line number coverage tracking; enabling this is ABI-breaking. Can be DISABLED, COVERAGE, or COVERAGE_AND_ORIGIN.")
 set_property(CACHE LLVM_ENABLE_DEBUGLOC_COVERAGE_TRACKING PROPERTY STRINGS DISABLED COVERAGE COVERAGE_AND_ORIGIN)
 
-option(LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS
-  "Add additional fields to DILocations to support Key Instructions" ON)
-
 set(WINDOWS_PREFER_FORWARD_SLASH_DEFAULT OFF)
 if (MINGW)
   # Cygwin doesn't identify itself as Windows, and thus gets path::Style::posix
@@ -684,7 +682,7 @@ if (NOT CMAKE_SYSTEM_NAME MATCHES "OS390")
   option(LLVM_ENABLE_PIC "Build Position-Independent Code" ON)
 endif()
 option(LLVM_ENABLE_MODULES "Compile with C++ modules enabled." OFF)
-if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "Darwin")
   option(LLVM_ENABLE_MODULE_DEBUGGING "Compile with -gmodules." ON)
 else()
   option(LLVM_ENABLE_MODULE_DEBUGGING "Compile with -gmodules." OFF)
@@ -788,7 +786,7 @@ option(LLVM_USE_SPLIT_DWARF
 
 # Define an option controlling whether we should build for 32-bit on 64-bit
 # platforms, where supported.
-if( CMAKE_SIZEOF_VOID_P EQUAL 8 AND NOT (WIN32 OR ${CMAKE_SYSTEM_NAME} MATCHES "AIX"))
+if( CMAKE_SIZEOF_VOID_P EQUAL 8 AND NOT (WIN32 OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX"))
   # TODO: support other platforms and toolchains.
   option(LLVM_BUILD_32_BITS "Build 32 bits executables and libraries." OFF)
 endif()
@@ -1258,7 +1256,7 @@ endif()
 
 # Build with _XOPEN_SOURCE on AIX, as stray macros in _ALL_SOURCE mode tend to
 # break things. In this case we need to enable the large-file API as well.
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
           add_compile_definitions(_XOPEN_SOURCE=700)
           add_compile_definitions(_LARGE_FILE_API)
           add_compile_options(-pthread)
@@ -1286,7 +1284,7 @@ if (CMAKE_SYSTEM_NAME MATCHES "OS390")
 endif()
 
 # Build with _FILE_OFFSET_BITS=64 on Solaris to match g++ >= 9.
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "SunOS")
           add_compile_definitions(_FILE_OFFSET_BITS=64)
 endif()
 
@@ -1304,10 +1302,10 @@ if(LLVM_TARGET_IS_CROSSCOMPILE_HOST)
 # (this is a variable that CrossCompile sets on recursive invocations)
 endif()
 
-if( ${CMAKE_SYSTEM_NAME} MATCHES SunOS )
+if( "${CMAKE_SYSTEM_NAME}" MATCHES SunOS )
    # special hack for Solaris to handle crazy system sys/regset.h
    include_directories("${LLVM_MAIN_INCLUDE_DIR}/llvm/Support/Solaris")
-endif( ${CMAKE_SYSTEM_NAME} MATCHES SunOS )
+endif( "${CMAKE_SYSTEM_NAME}" MATCHES SunOS )
 
 # Make sure we don't get -rdynamic in every binary. For those that need it,
 # use EXPORT_SYMBOLS argument.
@@ -1378,6 +1376,17 @@ if( LLVM_INCLUDE_TOOLS )
   add_subdirectory(tools)
 endif()
 
+# We need to setup KillTheDoctor before setting up the runtimes build because
+# the runtimes build needs to add a KillTheDoctor dep for compiler-rt on
+# Windows.
+if (LLVM_INCLUDE_TESTS)
+  if (WIN32)
+    # This utility is used to prevent crashing tests from calling Dr. Watson on
+    # Windows.
+    add_subdirectory(utils/KillTheDoctor)
+  endif()
+endif()
+
 if( LLVM_INCLUDE_RUNTIMES )
   add_subdirectory(runtimes)
 endif()
@@ -1402,12 +1411,6 @@ if( LLVM_INCLUDE_TESTS )
   add_subdirectory(test)
   add_subdirectory(unittests)
 
-  if (WIN32)
-    # This utility is used to prevent crashing tests from calling Dr. Watson on
-    # Windows.
-    add_subdirectory(utils/KillTheDoctor)
-  endif()
-
   umbrella_lit_testsuite_end(check-all)
   get_property(LLVM_ALL_LIT_DEPENDS GLOBAL PROPERTY LLVM_ALL_LIT_DEPENDS)
   get_property(LLVM_ALL_ADDITIONAL_TEST_DEPENDS
diff --git a/llvm/benchmarks/CMakeLists.txt b/llvm/benchmarks/CMakeLists.txt
index 1078efa55f497..e32aba5ebff43 100644
--- a/llvm/benchmarks/CMakeLists.txt
+++ b/llvm/benchmarks/CMakeLists.txt
@@ -11,3 +11,6 @@ add_benchmark(FormatVariadicBM FormatVariadicBM.cpp PARTIAL_SOURCES_INTENDED)
 add_benchmark(GetIntrinsicInfoTableEntriesBM GetIntrinsicInfoTableEntriesBM.cpp PARTIAL_SOURCES_INTENDED)
 add_benchmark(SandboxIRBench SandboxIRBench.cpp PARTIAL_SOURCES_INTENDED)
 
+add_benchmark(RuntimeLibcallsBench RuntimeLibcalls.cpp PARTIAL_SOURCES_INTENDED)
+
+
diff --git a/llvm/benchmarks/RuntimeLibcalls.cpp b/llvm/benchmarks/RuntimeLibcalls.cpp
new file mode 100644
index 0000000000000..9ac77bb74a3df
--- /dev/null
+++ b/llvm/benchmarks/RuntimeLibcalls.cpp
@@ -0,0 +1,121 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/RuntimeLibcalls.h"
+#include "benchmark/benchmark.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/TargetParser/Triple.h"
+#include <random>
+#include <string>
+using namespace llvm;
+
+static constexpr unsigned MaxFuncNameSize = 53;
+
+static std::vector<StringRef> getLibcallNameStringRefs() {
+  std::vector<StringRef> Names(RTLIB::NumLibcallImpls);
+  // Keep the strlens on the StringRef construction out of the benchmark loop.
+  for (RTLIB::LibcallImpl LC : RTLIB::libcall_impls())
+    Names[LC] = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(LC);
+
+  return Names;
+}
+
+static std::vector<std::string> getRandomFuncNames() {
+  std::mt19937_64 Rng;
+  std::uniform_int_distribution<> StringLengthDistribution(1, MaxFuncNameSize);
+  std::uniform_int_distribution<> CharDistribution(1, 255);
+  int NumTestFuncs = 1 << 10;
+  std::vector<std::string> TestFuncNames(NumTestFuncs);
+
+  for (std::string &TestFuncName : TestFuncNames) {
+    for (int I = 0, E = StringLengthDistribution(Rng); I != E; ++I)
+      TestFuncName += static_cast<char>(CharDistribution(Rng));
+  }
+
+  return TestFuncNames;
+}
+
+#ifdef SYMBOL_TEST_DATA_FILE
+static std::vector<std::string> readSymbolsFromFile(StringRef InputFile) {
+  auto BufOrError = MemoryBuffer::getFileOrSTDIN(InputFile, /*IsText=*/true);
+  if (!BufOrError) {
+    reportFatalUsageError("failed to open \'" + Twine(InputFile) +
+                          "\': " + BufOrError.getError().message());
+  }
+
+  // Hackily figure out if there's a prefix on the symbol names - llvm-nm
+  // appears to not have a flag to skip this.
+  llvm::Triple HostTriple(LLVM_HOST_TRIPLE);
+  std::string DummyDatalayout = "e";
+  DummyDatalayout += DataLayout::getManglingComponent(HostTriple);
+
+  DataLayout DL(DummyDatalayout);
+  char GlobalPrefix = DL.getGlobalPrefix();
+
+  std::vector<std::string> Lines;
+  for (line_iterator LineIt(**BufOrError, /*SkipBlanks=*/true);
+       !LineIt.is_at_eof(); ++LineIt) {
+    StringRef SymbolName = *LineIt;
+    SymbolName.consume_front(StringRef(&GlobalPrefix, 1));
+
+    Lines.push_back(SymbolName.str());
+  }
+  return Lines;
+}
+#endif
+
+static void BM_LookupRuntimeLibcallByNameKnownCalls(benchmark::State &State) {
+  std::vector<StringRef> Names = getLibcallNameStringRefs();
+
+  for (auto _ : State) {
+    for (StringRef Name : Names) {
+      benchmark::DoNotOptimize(
+          RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName(Name).empty());
+    }
+  }
+}
+
+static void BM_LookupRuntimeLibcallByNameRandomCalls(benchmark::State &State) {
+  std::vector<std::string> TestFuncNames = getRandomFuncNames();
+
+  for (auto _ : State) {
+    for (const std::string &Name : TestFuncNames) {
+      benchmark::DoNotOptimize(
+          RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName(StringRef(Name))
+              .empty());
+    }
+  }
+}
+
+#ifdef SYMBOL_TEST_DATA_FILE
+// This isn't fully representative, it doesn't include any anonymous functions.
+// nm -n --no-demangle --format=just-symbols sample-binary > sample.txt
+static void BM_LookupRuntimeLibcallByNameSampleData(benchmark::State &State) {
+  std::vector<std::string> TestFuncNames =
+      readSymbolsFromFile(SYMBOL_TEST_DATA_FILE);
+  for (auto _ : State) {
+    for (const std::string &Name : TestFuncNames) {
+      benchmark::DoNotOptimize(
+          RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName(StringRef(Name))
+              .empty());
+    }
+  }
+}
+#endif
+
+BENCHMARK(BM_LookupRuntimeLibcallByNameKnownCalls);
+BENCHMARK(BM_LookupRuntimeLibcallByNameRandomCalls);
+
+#ifdef SYMBOL_TEST_DATA_FILE
+BENCHMARK(BM_LookupRuntimeLibcallByNameSampleData);
+#endif
+
+BENCHMARK_MAIN();
diff --git a/llvm/cmake/config-ix.cmake b/llvm/cmake/config-ix.cmake
index a8e4e5a63244d..ed2bfa6df68f4 100644
--- a/llvm/cmake/config-ix.cmake
+++ b/llvm/cmake/config-ix.cmake
@@ -69,14 +69,14 @@ endif()
 
 # Do checks with _XOPEN_SOURCE and large-file API on AIX, because we will build
 # with those too.
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
           list(APPEND CMAKE_REQUIRED_DEFINITIONS "-D_XOPEN_SOURCE=700")
           list(APPEND CMAKE_REQUIRED_DEFINITIONS "-D_LARGE_FILE_API")
 endif()
 
 # Do checks with _FILE_OFFSET_BITS=64 on Solaris, because we will build
 # with those too.
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "SunOS")
           list(APPEND CMAKE_REQUIRED_DEFINITIONS "-D_FILE_OFFSET_BITS=64")
 endif()
 
@@ -122,7 +122,7 @@ if(APPLE)
     HAVE_CRASHREPORTER_INFO)
 endif()
 
-if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
+if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
   check_include_file(linux/magic.h HAVE_LINUX_MAGIC_H)
   if(NOT HAVE_LINUX_MAGIC_H)
     # older kernels use split files
@@ -411,7 +411,7 @@ endif()
 
 CHECK_STRUCT_HAS_MEMBER("struct stat" st_mtimespec.tv_nsec
     "sys/types.h;sys/stat.h" HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC)
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
 # The st_mtim.tv_nsec member of a `stat` structure is not reliable on some AIX
 # environments.
   set(HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC 0)
diff --git a/llvm/cmake/config.guess b/llvm/cmake/config.guess
index 96cc554f181ab..27b55bd166edc 100644
--- a/llvm/cmake/config.guess
+++ b/llvm/cmake/config.guess
@@ -1003,7 +1003,7 @@ EOF
     ppcle:Linux:*:*)
 	echo powerpcle-unknown-linux-gnu
 	exit ;;
-    riscv32:Linux:*:* | riscv64:Linux:*:*)
+    riscv32:Linux:*:* | riscv64:Linux:*:* | riscv32be:Linux:*:* | riscv64be:Linux:*:*)
 	LIBC=gnu
 	eval $set_cc_for_build
 	# Do not check for __GLIBC__ because uclibc defines it too
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index 83772ed8d2b13..835750e2d2a13 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -99,7 +99,7 @@ function(llvm_update_compile_flags name)
 endfunction()
 
 function(add_llvm_symbol_exports target_name export_file)
-  if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  if("${CMAKE_SYSTEM_NAME}" MATCHES "Darwin")
     set(native_export_file "${target_name}.exports")
     add_custom_command(OUTPUT ${native_export_file}
       COMMAND sed -e "s/^/_/" < ${export_file} > ${native_export_file}
@@ -108,7 +108,7 @@ function(add_llvm_symbol_exports target_name export_file)
       COMMENT "Creating export file for ${target_name}")
     set_property(TARGET ${target_name} APPEND_STRING PROPERTY
                  LINK_FLAGS " -Wl,-exported_symbols_list,\"${CMAKE_CURRENT_BINARY_DIR}/${native_export_file}\"")
-  elseif(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  elseif("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
     # FIXME: `-Wl,-bE:` bypasses whatever handling there is in the build
     # compiler driver to defer to the specified export list.
     set(native_export_file "${export_file}")
@@ -268,7 +268,7 @@ if (NOT DEFINED LLVM_LINKER_DETECTED AND NOT WIN32)
     endif()
   endif()
 
-  if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  if("${CMAKE_SYSTEM_NAME}" MATCHES "Darwin")
     include(CheckLinkerFlag)
     # Linkers that support Darwin allow a setting to internalize all symbol exports,
     # aiding in reducing binary size and often is applicable for executables.
@@ -315,11 +315,11 @@ function(add_link_opts target_name)
   # linker in a context where the optimizations are not important.
   if (NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
     if(NOT LLVM_NO_DEAD_STRIP)
-      if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+      if("${CMAKE_SYSTEM_NAME}" MATCHES "Darwin")
         # ld64's implementation of -dead_strip breaks tools that use plugins.
         set_property(TARGET ${target_name} APPEND_STRING PROPERTY
                      LINK_FLAGS " -Wl,-dead_strip")
-      elseif(${CMAKE_SYSTEM_NAME} MATCHES "SunOS" AND LLVM_LINKER_IS_SOLARISLD)
+      elseif("${CMAKE_SYSTEM_NAME}" MATCHES "SunOS" AND LLVM_LINKER_IS_SOLARISLD)
         # Support for ld -z discard-unused=sections was only added in
         # Solaris 11.4.  GNU ld ignores it, but warns every time.
         check_linker_flag(CXX "-Wl,-z,discard-unused=sections" LINKER_SUPPORTS_Z_DISCARD_UNUSED)
@@ -333,7 +333,7 @@ function(add_link_opts target_name)
                      LINK_FLAGS " -Wl,--gc-sections")
       endif()
     else() #LLVM_NO_DEAD_STRIP
-      if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+      if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
         set_property(TARGET ${target_name} APPEND_STRING PROPERTY
                      LINK_FLAGS " -Wl,-bnogc")
       endif()
@@ -345,7 +345,7 @@ function(add_link_opts target_name)
                  LINK_FLAGS " -Wl,-no_warn_duplicate_libraries")
   endif()
 
-  if(ARG_SUPPORT_PLUGINS AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  if(ARG_SUPPORT_PLUGINS AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
     set_property(TARGET ${target_name} APPEND_STRING PROPERTY
                  LINK_FLAGS " -Wl,-brtl")
   endif()
@@ -667,7 +667,7 @@ function(llvm_add_library name)
     # that are used across shared objects which we can't hide.
     if (LLVM_BUILD_LLVM_DYLIB_VIS AND NOT BUILD_SHARED_LIBS AND NOT APPLE AND
         (NOT (WIN32 OR CYGWIN) OR ((MINGW OR CYGWIN) AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
-        NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX") AND
+        NOT ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX") AND
         NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET)
 
       set_target_properties(${name} PROPERTIES
@@ -1094,7 +1094,7 @@ macro(add_llvm_executable name)
     llvm_update_compile_flags(${name})
   endif()
 
-  if (ARG_SUPPORT_PLUGINS AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  if (ARG_SUPPORT_PLUGINS AND NOT "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
     set(LLVM_NO_DEAD_STRIP On)
   endif()
 
@@ -1417,7 +1417,7 @@ function(export_executable_symbols target)
     # CMake doesn't set CMAKE_EXE_EXPORTS_${lang}_FLAG on Solaris, so
     # ENABLE_EXPORTS has no effect.  While Solaris ld defaults to -rdynamic
     # behaviour, GNU ld needs it.
-    if (APPLE OR ${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
+    if (APPLE OR "${CMAKE_SYSTEM_NAME}" STREQUAL "SunOS")
       set_property(TARGET ${target} APPEND_STRING PROPERTY
         LINK_FLAGS " -rdynamic")
     endif()
@@ -2540,7 +2540,7 @@ function(llvm_setup_rpath name)
   if (APPLE)
     set(_install_name_dir INSTALL_NAME_DIR "@rpath")
     set(_install_rpath "@loader_path/../lib${LLVM_LIBDIR_SUFFIX}" ${extra_libdir})
-  elseif(${CMAKE_SYSTEM_NAME} MATCHES "AIX" AND BUILD_SHARED_LIBS)
+  elseif("${CMAKE_SYSTEM_NAME}" MATCHES "AIX" AND BUILD_SHARED_LIBS)
     # $ORIGIN is not interpreted at link time by aix ld.
     # Since BUILD_SHARED_LIBS is only recommended for use by developers,
     # hardcode the rpath to build/install lib dir first in this mode.
@@ -2549,7 +2549,7 @@ function(llvm_setup_rpath name)
   elseif(UNIX)
     set(_build_rpath "\$ORIGIN/../lib${LLVM_LIBDIR_SUFFIX}" ${extra_libdir})
     set(_install_rpath "\$ORIGIN/../lib${LLVM_LIBDIR_SUFFIX}")
-    if(${CMAKE_SYSTEM_NAME} MATCHES "(FreeBSD|DragonFly)")
+    if("${CMAKE_SYSTEM_NAME}" MATCHES "(FreeBSD|DragonFly)")
       set_property(TARGET ${name} APPEND_STRING PROPERTY
                    LINK_FLAGS " -Wl,-z,origin ")
     endif()
@@ -2567,7 +2567,7 @@ function(llvm_setup_rpath name)
   # On AIX, the tool chain doesn't support modifying rpaths/libpaths for XCOFF
   # on install at the moment, so BUILD_WITH_INSTALL_RPATH is required.
   if("${CMAKE_BUILD_RPATH}" STREQUAL "")
-    if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin|AIX")
+    if("${CMAKE_SYSTEM_NAME}" MATCHES "Darwin|AIX")
       set_property(TARGET ${name} PROPERTY BUILD_WITH_INSTALL_RPATH ON)
     else()
       set_property(TARGET ${name} APPEND PROPERTY BUILD_RPATH "${_build_rpath}")
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index c126b0d073322..d3c6e9f6a168c 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -175,6 +175,24 @@ if(LLVM_ENABLE_EXPENSIVE_CHECKS)
   endif()
 endif()
 
+CHECK_CXX_SOURCE_COMPILES("
+#include <iosfwd>
+#if !defined(__GLIBCXX__)
+#error Not libstdc++
+#endif
+int main() { return 0; }
+" LLVM_USES_LIBSTDCXX)
+
+option(GLIBCXX_USE_CXX11_ABI "Use new libstdc++ CXX11 ABI" ON)
+
+if (LLVM_USES_LIBSTDCXX)
+  if (GLIBCXX_USE_CXX11_ABI)
+    add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=1)
+  else()
+    add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0)
+  endif()
+endif()
+
 if (LLVM_ENABLE_STRICT_FIXED_SIZE_VECTORS)
   add_compile_definitions(STRICT_FIXED_SIZE_VECTORS)
 endif()
@@ -212,10 +230,6 @@ endif()
 # LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE (non-cached) is expected to be
 # 1 or 0 here, assuming referenced in #cmakedefine01.
 
-if(LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS)
-  add_compile_definitions(EXPERIMENTAL_KEY_INSTRUCTIONS)
-endif()
-
 if( LLVM_REVERSE_ITERATION )
   set( LLVM_ENABLE_REVERSE_ITERATION 1 )
 endif()
@@ -232,7 +246,7 @@ if(WIN32 OR CYGWIN)
 elseif(FUCHSIA OR UNIX)
   set(LLVM_ON_WIN32 0)
   set(LLVM_ON_UNIX 1)
-  if(APPLE OR ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  if(APPLE OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
     set(LLVM_HAVE_LINK_VERSION_SCRIPT 0)
   else()
     set(LLVM_HAVE_LINK_VERSION_SCRIPT 1)
@@ -253,7 +267,7 @@ set(EXEEXT ${CMAKE_EXECUTABLE_SUFFIX})
 set(LTDL_SHLIB_EXT ${CMAKE_SHARED_LIBRARY_SUFFIX})
 
 # We use *.dylib rather than *.so on darwin, but we stick with *.so on AIX.
-if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(LLVM_PLUGIN_EXT ${CMAKE_SHARED_MODULE_SUFFIX})
 else()
   set(LLVM_PLUGIN_EXT ${CMAKE_SHARED_LIBRARY_SUFFIX})
@@ -264,7 +278,7 @@ if(APPLE)
   set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,-flat_namespace -Wl,-undefined -Wl,dynamic_lookup")
 endif()
 
-if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "Linux")
   # RHEL7 has ar and ranlib being non-deterministic by default. The D flag forces determinism,
   # however only GNU version of ar and ranlib (2.27) have this option.
   # RHEL DTS7 is also affected by this, which uses GNU binutils 2.28
@@ -296,7 +310,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
   endif()
 endif()
 
-if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   # -fPIC does not enable the large code model for GCC on AIX but does for XL.
   if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang")
     append("-mcmodel=large" CMAKE_CXX_FLAGS CMAKE_C_FLAGS)
@@ -332,7 +346,7 @@ endif()
 # by dlclose(). We need that since the CLI API relies on cross-references
 # between global objects which became horribly broken when one of the libraries
 # is unloaded.
-if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "Linux")
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,nodelete")
   set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,-z,nodelete")
 endif()
@@ -458,13 +472,13 @@ if( LLVM_ENABLE_PIC )
   # to SEGV (GCC PR target/96607).
   # clang with -O3 -fPIC generates code that SEGVs.
   # Both can be worked around by compiling with -O instead.
-  if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS" AND LLVM_NATIVE_ARCH STREQUAL "Sparc")
+  if("${CMAKE_SYSTEM_NAME}" STREQUAL "SunOS" AND LLVM_NATIVE_ARCH STREQUAL "Sparc")
     llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELEASE "-O[23]" "-O")
     llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O[23]" "-O")
   endif()
 endif()
 
-if((NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX")) AND
+if((NOT ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")) AND
    (NOT (WIN32 OR CYGWIN) OR ((MINGW OR CYGWIN) AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")))
   # GCC for MinGW does nothing about -fvisibility-inlines-hidden, but warns
   # about use of the attributes. As long as we don't use the attributes (to
@@ -712,7 +726,7 @@ endif ()
 if ( LLVM_COMPILER_IS_GCC_COMPATIBLE AND LLVM_ENABLE_MODULES )
   set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
   set(module_flags "-fmodules -fmodules-cache-path=${PROJECT_BINARY_DIR}/module.cache")
-  if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+  if ("${CMAKE_SYSTEM_NAME}" MATCHES "Darwin")
     # On Darwin -fmodules does not imply -fcxx-modules.
     set(module_flags "${module_flags} -fcxx-modules")
   endif()
@@ -1127,7 +1141,7 @@ endif()
 # But MinSizeRel seems to add that automatically, so maybe disable these
 # flags instead if LLVM_NO_DEAD_STRIP is set.
 if(NOT CYGWIN AND NOT MSVC)
-  if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND
+  if(NOT "${CMAKE_SYSTEM_NAME}" MATCHES "Darwin" AND
      NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
     if (CMAKE_CXX_COMPILER_ID MATCHES "XL")
       append("-qfuncsect" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
@@ -1319,9 +1333,14 @@ endif()
 # linking (due to incompatibility). With MSVC, note that the plugin has to
 # explicitly link against (exactly one) tool so we can't unilaterally turn on
 # LLVM_ENABLE_PLUGINS when it's enabled.
+if("${CMAKE_SYSTEM_NAME}" MATCHES AIX)
+  set(LLVM_EXPORT_SYMBOLS_FOR_PLUGINS_OPTION OFF)
+else()
+  set(LLVM_EXPORT_SYMBOLS_FOR_PLUGINS_OPTION ON)
+endif()
 CMAKE_DEPENDENT_OPTION(LLVM_EXPORT_SYMBOLS_FOR_PLUGINS
        "Export symbols from LLVM tools so that plugins can import them" OFF
-       "NOT ${CMAKE_SYSTEM_NAME} MATCHES AIX" ${LLVM_EXPORT_SYMBOLS_FOR_PLUGINS_AIX_default})
+       "LLVM_EXPORT_SYMBOLS_FOR_PLUGINS_OPTION" ${LLVM_EXPORT_SYMBOLS_FOR_PLUGINS_AIX_default})
 if(BUILD_SHARED_LIBS AND LLVM_EXPORT_SYMBOLS_FOR_PLUGINS)
   message(FATAL_ERROR "BUILD_SHARED_LIBS not compatible with LLVM_EXPORT_SYMBOLS_FOR_PLUGINS")
 endif()
diff --git a/llvm/cmake/modules/LLVMProcessSources.cmake b/llvm/cmake/modules/LLVMProcessSources.cmake
index cf358a88f5fb6..0670d60bf2afd 100644
--- a/llvm/cmake/modules/LLVMProcessSources.cmake
+++ b/llvm/cmake/modules/LLVMProcessSources.cmake
@@ -58,21 +58,6 @@ function(llvm_process_sources OUT_VAR)
   set(sources ${ARG_UNPARSED_ARGUMENTS})
   llvm_check_source_file_list(${sources})
 
-  # Don't generate __SHORT_FILE__ on VS builds as it can prevent build parallelisation.
-  if(NOT CMAKE_GENERATOR MATCHES "Visual Studio")
-    foreach(fn ${sources})
-      get_filename_component(suf ${fn} EXT)
-      if("${suf}" STREQUAL ".cpp" OR "${suf}" STREQUAL ".c")
-        get_filename_component(short_name ${fn} NAME)
-        set_property(
-            SOURCE ${fn}
-            APPEND
-            PROPERTY COMPILE_DEFINITIONS __SHORT_FILE__="${short_name}")
-      endif()
-    endforeach()
-  endif()
-
-
   # This adds .td and .h files to the Visual Studio solution:
   add_td_sources(sources)
   find_all_header_files(hdrs "${ARG_ADDITIONAL_HEADER_DIRS}")
diff --git a/llvm/docs/AArch64SME.rst b/llvm/docs/AArch64SME.rst
index ac8ce32ddb9e6..47ed7bc68726d 100644
--- a/llvm/docs/AArch64SME.rst
+++ b/llvm/docs/AArch64SME.rst
@@ -18,8 +18,8 @@ This document describes how the SME ACLE attributes map to LLVM IR
 attributes and how LLVM lowers these attributes to implement the rules and
 requirements of the ABI.
 
-Below we describe the LLVM IR attributes and their relation to the C/C++
-level ACLE attributes:
+Below, we describe the LLVM IR attributes and their relation to the
+C/C++-level ACLE attributes:
 
 ``aarch64_pstate_sm_enabled``
     is used for functions with ``__arm_streaming``
@@ -51,8 +51,8 @@ level ACLE attributes:
 
 Clang must ensure that the above attributes are added both to the
 function's declaration/definition as well as to their call-sites. This is
-important for calls to attributed function pointers, where there is no
-definition or declaration available.
+important for calls to attributed function pointers, where no
+definition or declaration is available.
 
 
 2. Handling PSTATE.SM
@@ -77,7 +77,7 @@ and almost all parts of CodeGen we can assume that the runtime value for
 ``vscale`` does not. If we let the compiler insert the appropriate ``smstart``
 and ``smstop`` instructions around call boundaries, then the effects on SVE
 state can be mitigated. By limiting the state changes to a very brief window
-around the call we can control how the operations are scheduled and how live
+around the call, we can control how the operations are scheduled and how live
 values remain preserved between state transitions.
 
 In order to control PSTATE.SM at this level of granularity, we use function and
@@ -89,7 +89,7 @@ Restrictions on attributes
 
 * It is undefined behaviour to pass or return (pointers to) scalable vector
   objects to/from functions which may use a different SVE vector length.
-  This includes functions with a non-streaming interface, but marked with
+  This includes functions with a non-streaming interface but marked with
   ``aarch64_pstate_sm_body``.
 
 * It is not allowed for a function to be decorated with both
@@ -100,7 +100,7 @@ Restrictions on attributes
   ``aarch64_new_za``, ``aarch64_in_za``, ``aarch64_out_za``, ``aarch64_inout_za``,
   ``aarch64_preserves_za``.
 
-These restrictions also apply in the higher level SME ACLE, which means we can
+These restrictions also apply in the higher-level SME ACLE, which means we can
 emit diagnostics in Clang to signal users about incorrect behaviour.
 
 
@@ -224,7 +224,7 @@ The ``COND_SMSTART/COND_SMSTOP`` nodes additionally take ``CurrentState`` and
 
 When ``CurrentState`` and ``ExpectedState`` can be evaluated at compile-time
 (i.e. they are both constants) then an unconditional ``smstart/smstop``
-instruction is emitted. Otherwise the node is matched to a Pseudo instruction
+instruction is emitted. Otherwise, the node is matched to a Pseudo instruction
 which expands to a compare/branch and a ``smstart/smstop``. This is necessary to
 implement transitions from ``SC -> N`` and ``SC -> S``.
 
@@ -236,7 +236,7 @@ streaming compatible, the compiler has to insert a SMSTOP before the call and
 insert a SMSTOP after the call.
 
 If the function that is called is an intrinsic with no side-effects which in
-turn is lowered to a function call (e.g. ``@llvm.cos()``), then the call to
+turn is lowered to a function call (e.g., ``@llvm.cos()``), then the call to
 ``@llvm.cos()`` is not part of any Chain; it can be scheduled freely.
 
 Lowering of a Callsite creates a small chain of nodes which:
@@ -297,11 +297,11 @@ To ensure we use the correct SVE vector length to allocate the locals with, we
 can use the streaming vector-length to allocate the stack-slots through the
 ``ADDSVL`` instruction, even when the CPU is not yet in streaming mode.
 
-This only works for locals and not callee-save slots, since LLVM doesn't support
+This works only for locals and not callee-save slots, since LLVM doesn't support
 mixing two different scalable vector lengths in one stack frame. That means that the
 case where a function is marked ``arm_locally_streaming`` and needs to spill SVE
 callee-saves in the prologue is currently unsupported.  However, it is unlikely
-for this to happen without user intervention, because ``arm_locally_streaming``
+for this to happen without user intervention because ``arm_locally_streaming``
 functions cannot take or return vector-length-dependent values. This would otherwise
 require forcing both the SVE PCS using '``aarch64_sve_pcs``' combined with using
 ``arm_locally_streaming`` in order to encounter this problem. This combination
@@ -330,7 +330,7 @@ attributed with ``arm_locally_streaming``:
       return array[N - 1] + arg;
     }
 
-should use ADDSVL for allocating the stack space and should avoid clobbering
+should use ``ADDSVL`` for allocating the stack space and should avoid clobbering
 the return/argument values.
 
 .. code-block:: none
@@ -381,17 +381,17 @@ Preventing the use of illegal instructions in Streaming Mode
 * When executing a program in normal mode (PSTATE.SM=0), a subset of SME
   instructions are invalid.
 
-* Streaming-compatible functions must only use instructions that are valid when
+* Streaming-compatible functions must use only instructions that are valid when
   either PSTATE.SM=0 or PSTATE.SM=1.
 
 The value of PSTATE.SM is not controlled by the feature flags, but rather by the
-function attributes. This means that we can compile for '``+sme``' and the compiler
+function attributes. This means that we can compile for '``+sme``', and the compiler
 will code-generate any instructions, even if they are not legal under the requested
 streaming mode. The compiler needs to use the function attributes to ensure the
 compiler doesn't do transformations under the assumption that certain operations
 are available at runtime.
 
-We made a conscious choice not to model this with feature flags, because we
+We made a conscious choice not to model this with feature flags because we
 still want to support inline-asm in either mode (with the user placing
 smstart/smstop manually), and this became rather complicated to implement at the
 individual instruction level (see `D120261 <https://reviews.llvm.org/D120261>`_
@@ -408,7 +408,7 @@ auto-vectorization with a subset of streaming-compatible instructions, but that
 requires changes to the CostModel, Legalization and SelectionDAG lowering.
 
 We will also emit diagnostics in Clang to prevent the use of
-non-streaming(-compatible) operations, e.g. through ACLE intrinsics, when a
+non-streaming(-compatible) operations, e.g., through ACLE intrinsics, when a
 function is decorated with the streaming mode attributes.
 
 
@@ -456,7 +456,7 @@ AArch64 Predicate-as-Counter Type
 :Overview:
 
 The predicate-as-counter type represents the type of a predicate-as-counter
-value held in a AArch64 SVE predicate register. Such a value contains
+value held in an AArch64 SVE predicate register. Such a value contains
 information about the number of active lanes, the element width and a bit that
 tells whether the generated mask should be inverted. ACLE intrinsics should be
 used to move the predicate-as-counter value to/from a predicate vector.
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 5343d66b083c7..b6d61a62f50ff 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1771,6 +1771,10 @@ The AMDGPU backend supports the following LLVM IR attributes.
                                                       using dedicated instructions, but may not send the DEALLOC_VGPRS
                                                       message. If a shader has this attribute, then all its callees must
                                                       match its value.
+                                                      An amd_cs_chain CC function with this enabled has an extra symbol
+                                                      prefixed with "_dvgpr$" with the value of the function symbol,
+                                                      offset by one less than the number of dynamic VGPR blocks required
+                                                      by the function encoded in bits 5..3.
 
      ================================================ ==========================================================
 
@@ -5401,7 +5405,21 @@ The fields used by CP for code objects before V3 also match those specified in
 
                                                        Used by CP to set up
                                                        ``COMPUTE_PGM_RSRC1.FP16_OVFL``.
-     28:27   2 bits                                  Reserved, must be 0.
+     27      1 bit    RESERVED                       GFX6-GFX120*
+                                                       Reserved, must be 0.
+                      FLAT_SCRATCH_IS_NV             GFX125*
+                                                       0 - Use the NV ISA as indication
+                                                       that scratch is NV. 1 - Force
+                                                       scratch to NV = 1, even if
+                                                       ISA.NV == 0 if the address falls
+                                                       into scratch space (not global).
+                                                       This allows global.NV = 0 and
+                                                       scratch.NV = 1 for flat ops. Other
+                                                       threads use the ISA bit value.
+
+                                                       Used by CP to set up
+                                                       ``COMPUTE_PGM_RSRC1.FLAT_SCRATCH_IS_NV``.
+     28      1 bit    RESERVED                       Reserved, must be 0.
      29      1 bit    WGP_MODE                       GFX6-GFX9
                                                        Reserved, must be 0.
                                                      GFX10-GFX12
@@ -5483,15 +5501,16 @@ The fields used by CP for code objects before V3 also match those specified in
 
                                                      Used by CP to set up
                                                      ``COMPUTE_PGM_RSRC2.SCRATCH_EN``.
-     5:1     5 bits  USER_SGPR_COUNT                 The total number of SGPR
-                                                     user data
-                                                     registers requested. This
-                                                     number must be greater than
-                                                     or equal to the number of user
-                                                     data registers enabled.
+     5:1     5 bits  USER_SGPR_COUNT                 GFX6-GFX120*
+                                                       The total number of SGPR
+                                                       user data
+                                                       registers requested. This
+                                                       number must be greater than
+                                                       or equal to the number of user
+                                                       data registers enabled.
 
-                                                     Used by CP to set up
-                                                     ``COMPUTE_PGM_RSRC2.USER_SGPR``.
+                                                       Used by CP to set up
+                                                       ``COMPUTE_PGM_RSRC2.USER_SGPR``.
      6       1 bit   ENABLE_TRAP_HANDLER             GFX6-GFX11
                                                        Must be 0.
 
@@ -5500,8 +5519,25 @@ The fields used by CP for code objects before V3 also match those specified in
                                                        which is set by the CP if
                                                        the runtime has installed a
                                                        trap handler.
-                                                     GFX12
-                                                       Reserved, must be 0.
+                     ENABLE_DYNAMIC_VGPR             GFX120*
+                                                       Enables dynamic VGPR mode, where
+                                                       each wave allocates one VGPR chunk
+                                                       at launch and can request for
+                                                       additional space to use during
+                                                       execution in SQ.
+
+                                                       Used by CP to set up
+                                                       ``COMPUTE_PGM_RSRC2.DYNAMIC_VGPR``.
+     6:1     6 bits  USER_SGPR_COUNT                 GFX125*
+                                                       The total number of SGPR
+                                                       user data
+                                                       registers requested. This
+                                                       number must be greater than
+                                                       or equal to the number of user
+                                                       data registers enabled.
+
+                                                       Used by CP to set up
+                                                       ``COMPUTE_PGM_RSRC2.USER_SGPR``.
      7       1 bit   ENABLE_SGPR_WORKGROUP_ID_X      Enable the setup of the
                                                      system SGPR register for
                                                      the work-group id in the X
@@ -5594,10 +5630,12 @@ The fields used by CP for code objects before V3 also match those specified in
 
                                                      GFX6
                                                        roundup(lds-size / (64 * 4))
-                                                     GFX7-GFX11
+                                                     GFX7-GFX12
                                                        roundup(lds-size / (128 * 4))
                                                      GFX950
                                                        roundup(lds-size / (320 * 4))
+                                                     GFX125*
+                                                       roundup(lds-size / (256 * 4))
 
      24      1 bit   ENABLE_EXCEPTION_IEEE_754_FP    Wavefront starts execution
                      _INVALID_OPERATION              with specified exceptions
@@ -5716,7 +5754,30 @@ The fields used by CP for code objects before V3 also match those specified in
                                                      with a granularity of 128 bytes.
      12      1 bit   RESERVED                        Reserved, must be 0.
      13      1 bit   GLG_EN                          If 1, group launch guarantee will be enabled for this dispatch
-     30:14   17 bits RESERVED                        Reserved, must be 0.
+     16:14   3 bits  RESERVED                        GFX120*
+                                                       Reserved, must be 0.
+                     NAMED_BAR_CNT                   GFX125*
+                                                       Number of named barriers to alloc for each workgroup, in granularity of
+                                                       4. Range is from 0-4 allocating 0, 4, 8, 12, 16.
+     17      1 bit   RESERVED                        GFX120*
+                                                       Reserved, must be 0.
+                     ENABLE_DYNAMIC_VGPR             GFX125*
+                                                       Enables dynamic VGPR mode, where each wave allocates one VGPR chunk
+                                                       at launch and can request for additional space to use during
+                                                       execution in SQ.
+
+                                                       Used by CP to set up ``COMPUTE_PGM_RSRC3.DYNAMIC_VGPR``.
+     20:18   3 bits  RESERVED                        GFX120*
+                                                       Reserved, must be 0.
+                     TCP_SPLIT                       GFX125*
+                                                       Desired LDS/VC split of TCP. 0: no preference 1: LDS=0, VC=448kB
+                                                       2: LDS=64kB, VC=384kB 3: LDS=128kB, VC=320kB 4: LDS=192kB, VC=256kB
+                                                       5: LDS=256kB, VC=192kB 6: LDS=320kB, VC=128kB 7: LDS=384kB, VC=64kB
+     21      1 bit   RESERVED                        GFX120*
+                                                       Reserved, must be 0.
+                     ENABLE_DIDT_THROTTLE            GFX125*
+                                                       Enable DIDT throttling for all ACE pipes
+     30:22   9 bits  RESERVED                        Reserved, must be 0.
      31      1 bit   IMAGE_OP                        If 1, the kernel execution contains image instructions. If executed as
                                                      part of a graphics pipeline, image read instructions will stall waiting
                                                      for any necessary ``WAIT_SYNC`` fence to be performed in order to
@@ -18201,6 +18262,8 @@ terminated by an ``.end_amdhsa_kernel`` directive.
                                                               (wavefrontsize64)
      ``.amdhsa_uses_dynamic_stack``                           0                   GFX6-GFX12   Controls USES_DYNAMIC_STACK in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
+     ``.amdhsa_named_barrier_count``                          0                   GFX1250+     Controls NAMED_BAR_CNT in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx12-table`.
      ``.amdhsa_system_sgpr_private_segment_wavefront_offset`` 0                   GFX6-GFX10   Controls ENABLE_PRIVATE_SEGMENT in
                                                                                   (except      :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
                                                                                   GFX942)
diff --git a/llvm/docs/CMake.rst b/llvm/docs/CMake.rst
index 17be41b20a12a..30b71bffaf76f 100644
--- a/llvm/docs/CMake.rst
+++ b/llvm/docs/CMake.rst
@@ -9,7 +9,7 @@ Introduction
 ============
 
 `CMake <http://www.cmake.org/>`_ is a cross-platform build-generator tool. CMake
-does not build the project, it generates the files needed by your build tool
+does not build the project; it generates the files needed by your build tool
 (GNU make, Visual Studio, etc.) for building LLVM.
 
 If **you are a new contributor**, please start with the :doc:`GettingStarted`
@@ -23,7 +23,7 @@ and then go back to the `Quick start`_ section once you know what you are doing.
 you already have experience with CMake, this is the recommended starting point.
 
 This page is geared towards users of the LLVM CMake build. If you're looking for
-information about modifying the LLVM CMake build system you may want to see the
+information about modifying the LLVM CMake build system, you may want to see the
 :doc:`CMakePrimer` page. It has a basic overview of the CMake language.
 
 .. _Quick start:
@@ -37,7 +37,7 @@ We use here the command-line, non-interactive CMake interface.
    CMake. Version 3.20.0 is the minimum required.
 
 #. Open a shell. Your development tools must be reachable from this shell
-   through the PATH environment variable.
+   through the ``PATH`` environment variable.
 
 #. Create a build directory. Building LLVM in the source
    directory is not supported. cd to this directory:
@@ -70,7 +70,7 @@ We use here the command-line, non-interactive CMake interface.
    components are built; see the `Frequently Used LLVM-related
    variables`_ below.
 
-#. After CMake has finished running, proceed to use IDE project files, or start
+#. After CMake has finished running, use IDE project files, or start
    the build from the build directory:
 
    .. code-block:: console
@@ -106,8 +106,7 @@ We use here the command-line, non-interactive CMake interface.
 Basic CMake usage
 =================
 
-This section explains basic aspects of CMake
-which you may need in your day-to-day usage.
+This section explains basic aspects of CMake for daily use.
 
 CMake comes with extensive documentation, in the form of html files, and as
 online help accessible via the ``cmake`` executable itself. Execute ``cmake
@@ -115,11 +114,11 @@ online help accessible via the ``cmake`` executable itself. Execute ``cmake
 
 CMake allows you to specify a build tool (e.g., GNU make, Visual Studio,
 or Xcode). If not specified on the command line, CMake tries to guess which
-build tool to use, based on your environment. Once it has identified your
+build tool to use based on your environment. Once it has identified your
 build tool, CMake uses the corresponding *Generator* to create files for your
 build tool (e.g., Makefiles or Visual Studio or Xcode project files). You can
 explicitly specify the generator with the command line option ``-G "Name of the
-generator"``. To see a list of the available generators on your system, execute
+generator"``. To see a list of the available generators on your system, execute:
 
 .. code-block:: console
 
@@ -127,7 +126,7 @@ generator"``. To see a list of the available generators on your system, execute
 
 This will list the generator names at the end of the help text.
 
-Generators' names are case-sensitive, and may contain spaces. For this reason,
+Generators' names are case-sensitive and may contain spaces. For this reason,
 you should enter them exactly as they are listed in the ``cmake --help``
 output, in quotes. For example, to generate project files specifically for
 Visual Studio 12, you can execute:
@@ -136,7 +135,7 @@ Visual Studio 12, you can execute:
 
   $ cmake -G "Visual Studio 12" path/to/llvm/source/root
 
-For a given development platform there can be more than one adequate
+A given development platform can have more than one adequate
 generator. If you use Visual Studio, "NMake Makefiles" is a generator you can use
 for building with NMake. By default, CMake chooses the most specific generator
 supported by your development environment. If you want an alternative generator,
@@ -205,17 +204,17 @@ used variables that control features of LLVM and enabled subprojects.
   **MinSizeRel**              For Size      No         No         When disk space matters
   =========================== ============= ========== ========== ==========================
 
-  * Optimizations make LLVM/Clang run faster, but can be an impediment for
+  * Optimizations make LLVM/Clang run faster but can be an impediment for
     step-by-step debugging.
   * Builds with debug information can use a lot of RAM and disk space and is
     usually slower to run. You can improve RAM usage by using ``lld``, see
     the :ref:`LLVM_USE_LINKER <llvm_use_linker>` option.
   * Assertions are internal checks to help you find bugs. They typically slow
-    down LLVM and Clang when enabled, but can be useful during development.
+    down LLVM and Clang when enabled but can be useful during development.
     You can manually set :ref:`LLVM_ENABLE_ASSERTIONS <llvm_enable_assertions>`
     to override the default from `CMAKE_BUILD_TYPE`.
 
-  If you are using an IDE such as Visual Studio or Xcode, you should use
+  If you are using an IDE such as Visual Studio or Xcode, use
   the IDE settings to set the build type.
 
   Note: on Windows (building with MSVC or clang-cl), CMake's **RelWithDebInfo**
@@ -244,11 +243,11 @@ LLVM variables that are frequently used to control that. The full
 description is in `LLVM-related variables`_ below.
 
 **LLVM_ENABLE_PROJECTS**:STRING
-  Control which projects are enabled. For example you may want to work on clang
+  Control which projects are enabled. For example, you may want to work on clang
   or lldb by specifying ``-DLLVM_ENABLE_PROJECTS="clang;lldb"``.
 
 **LLVM_ENABLE_RUNTIMES**:STRING
-  Control which runtimes are enabled. For example you may want to work on
+  Control which runtimes are enabled. For example, you may want to work on
   libc++ or libc++abi by specifying ``-DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi"``.
 
 **LLVM_LIBDIR_SUFFIX**:STRING
@@ -264,13 +263,13 @@ description is in `LLVM-related variables`_ below.
   32GB machine, specify ``-G Ninja -DLLVM_PARALLEL_LINK_JOBS=2``.
 
 **LLVM_TARGETS_TO_BUILD**:STRING
-  Control which targets are enabled. For example you may only need to enable
+  Control which targets are enabled. For example, you may only need to enable
   your native target with, for example, ``-DLLVM_TARGETS_TO_BUILD=X86``.
 
 .. _llvm_use_linker:
 
 **LLVM_USE_LINKER**:STRING
-  Override the system's default linker. For instance use ``lld`` with
+  Override the system's default linker. For instance, use ``lld`` with
   ``-DLLVM_USE_LINKER=lld``.
 
 Rarely-used CMake variables
@@ -282,7 +281,7 @@ manual, or execute ``cmake --help-variable VARIABLE_NAME``.
 
 **CMAKE_CXX_STANDARD**:STRING
   Sets the C++ standard to conform to when building LLVM.  Possible values are
-  17 and 20.  LLVM Requires C++17 or higher.  This defaults to 17.
+  17 and 20.  LLVM requires C++17 or higher.  This defaults to 17.
 
 **CMAKE_INSTALL_BINDIR**:PATH
   The path to install executables, relative to the *CMAKE_INSTALL_PREFIX*.
@@ -306,7 +305,7 @@ LLVM-related variables
 -----------------------
 
 These variables provide fine control over the build of LLVM and
-enabled sub-projects. Nearly all of these variable names begin with
+its enabled sub-projects. Nearly all of these variable names begin with
 ``LLVM_``.
 
 .. _LLVM-related variables BUILD_SHARED_LIBS:
@@ -317,7 +316,7 @@ enabled sub-projects. Nearly all of these variable names begin with
   Windows, shared libraries may be used when building with MinGW, including
   mingw-w64, but not when building with the Microsoft toolchain.
 
-  .. note:: BUILD_SHARED_LIBS is only recommended for use by LLVM developers.
+  .. note:: ``BUILD_SHARED_LIBS`` is only recommended for use by LLVM developers.
             If you want to build LLVM as a shared library, you should use the
             ``LLVM_BUILD_LLVM_DYLIB`` option.
 
@@ -332,7 +331,7 @@ enabled sub-projects. Nearly all of these variable names begin with
 
 **LLVM_ADDITIONAL_BUILD_TYPES**:LIST
   Adding a semicolon separated list of additional build types to this flag
-  allows for them to be specified as values in CMAKE_BUILD_TYPE without
+  allows for them to be specified as values in ``CMAKE_BUILD_TYPE`` without
   encountering a fatal error during the configuration process.
 
 **LLVM_APPEND_VC_REV**:BOOL
@@ -359,12 +358,12 @@ enabled sub-projects. Nearly all of these variable names begin with
   Adds benchmarks to the list of default targets. Defaults to OFF.
 
 **LLVM_BUILD_DOCS**:BOOL
-  Adds all *enabled* documentation targets (i.e. Doxgyen and Sphinx targets) as
+  Adds all *enabled* documentation targets (i.e. Doxygen and Sphinx targets) as
   dependencies of the default build targets.  This results in all of the (enabled)
   documentation targets being as part of a normal build.  If the ``install``
-  target is run then this also enables all built documentation targets to be
+  target is run, then this also enables all built documentation targets to be
   installed. Defaults to OFF.  To enable a particular documentation target, see
-  LLVM_ENABLE_SPHINX and LLVM_ENABLE_DOXYGEN.
+  ``LLVM_ENABLE_SPHINX`` and ``LLVM_ENABLE_DOXYGEN``.
 
 **LLVM_BUILD_EXAMPLES**:BOOL
   Build LLVM examples. Defaults to OFF. Targets for building each example are
@@ -375,7 +374,7 @@ enabled sub-projects. Nearly all of these variable names begin with
   If enabled, `source-based code coverage
   <https://clang.llvm.org/docs/SourceBasedCodeCoverage.html>`_ instrumentation
   is enabled while building llvm. If CMake can locate the code coverage
-  scripts and the llvm-cov and llvm-profdata tools that pair to your compiler,
+  scripts and the llvm-cov and llvm-profdata tools that pair with your compiler,
   the build will also generate the `generate-coverage-report` target to generate
   the code coverage report for LLVM, and the `clear-profile-data` utility target
   to delete captured profile data. See documentation for
@@ -385,10 +384,10 @@ enabled sub-projects. Nearly all of these variable names begin with
 **LLVM_BUILD_LLVM_DYLIB**:BOOL
   If enabled, the target for building the libLLVM shared library is added.
   This library contains all of LLVM's components in a single shared library.
-  Defaults to OFF. This cannot be used in conjunction with BUILD_SHARED_LIBS.
-  Tools will only be linked to the libLLVM shared library if LLVM_LINK_LLVM_DYLIB
+  Defaults to OFF. This cannot be used in conjunction with ``BUILD_SHARED_LIBS``.
+  Tools will only be linked to the libLLVM shared library if ``LLVM_LINK_LLVM_DYLIB``
   is also ON.
-  The components in the library can be customised by setting LLVM_DYLIB_COMPONENTS
+  The components in the library can be customised by setting ``LLVM_DYLIB_COMPONENTS``
   to a list of the desired components.
   This option is not available on Windows.
 
@@ -410,8 +409,8 @@ enabled sub-projects. Nearly all of these variable names begin with
   If enabled and the ``ccache`` program is available, then LLVM will be
   built using ``ccache`` to speed up rebuilds of LLVM and its components.
   Defaults to OFF.  The size and location of the cache maintained
-  by ``ccache`` can be adjusted via the LLVM_CCACHE_MAXSIZE and LLVM_CCACHE_DIR
-  options, which are passed to the CCACHE_MAXSIZE and CCACHE_DIR environment
+  by ``ccache`` can be adjusted via the ``LLVM_CCACHE_MAXSIZE`` and ``LLVM_CCACHE_DIR``
+  options, which are passed to the ``CCACHE_MAXSIZE`` and ``CCACHE_DIR`` environment
   variables, respectively.
 
 **LLVM_CODE_COVERAGE_TARGETS**:STRING
@@ -425,9 +424,9 @@ enabled sub-projects. Nearly all of these variable names begin with
   coverage reports will include all sources identified by the tooling.
 
 **LLVM_CREATE_XCODE_TOOLCHAIN**:BOOL
-  macOS Only: If enabled CMake will generate a target named
+  macOS Only: If enabled, CMake will generate a target named
   'install-xcode-toolchain'. This target will create a directory at
-  $CMAKE_INSTALL_PREFIX/Toolchains containing an xctoolchain directory which can
+  ``$CMAKE_INSTALL_PREFIX/Toolchains`` containing an xctoolchain directory which can
   be used to override the default system tools.
 
 **LLVM_DEFAULT_TARGET_TRIPLE**:STRING
@@ -519,8 +518,8 @@ enabled sub-projects. Nearly all of these variable names begin with
   Indicates whether the LLVM Interpreter will be linked with the Foreign Function
   Interface library (libffi) in order to enable calling external functions.
   If the library or its headers are installed in a custom
-  location, you can also set the variables FFI_INCLUDE_DIR and
-  FFI_LIBRARY_DIR to the directories where ffi.h and libffi.so can be found,
+  location, you can also set the variables ``FFI_INCLUDE_DIR`` and
+  ``FFI_LIBRARY_DIR`` to the directories where ``ffi.h`` and ``libffi.so`` can be found,
   respectively. Defaults to OFF.
 
 **LLVM_ENABLE_HTTPLIB**:BOOL
@@ -536,7 +535,7 @@ enabled sub-projects. Nearly all of these variable names begin with
   configured manually to explicitly control the generation of those targets.
 
 **LLVM_ENABLE_LIBCXX**:BOOL
-  If the host compiler and linker supports the stdlib flag, -stdlib=libc++ is
+  If the host compiler and linker support the stdlib flag, ``-stdlib=libc++`` is
   passed to invocations of both so that the project is built using libc++
   instead of stdlibc++. Defaults to OFF.
 
@@ -615,11 +614,11 @@ enabled sub-projects. Nearly all of these variable names begin with
   .. note::
     The list should not have duplicates with ``LLVM_ENABLE_PROJECTS``.
 
-  The full list is:
-
-  ``libc;libunwind;libcxxabi;libcxx;compiler-rt;openmp;llvm-libgcc;offload``
+  To list all possible runtimes, include an invalid name. For example
+  ``-DLLVM_ENABLE_RUNTIMES=notaruntime``. The resulting CMake error will list
+  the possible runtime names.
 
-  To enable all of them, use:
+  To enable all of the runtimes, use:
 
   ``LLVM_ENABLE_RUNTIMES=all``
 
@@ -643,7 +642,7 @@ enabled sub-projects. Nearly all of these variable names begin with
 
 **LLVM_ENABLE_Z3_SOLVER**:BOOL
   If enabled, the Z3 constraint solver is activated for the Clang static analyzer.
-  A recent version of the z3 library needs to be available on the system.
+  A recent version of the z3 library must be available on the system.
 
 **LLVM_ENABLE_ZLIB**:STRING
   Used to decide if LLVM tools should support compression/decompression with
@@ -672,7 +671,7 @@ enabled sub-projects. Nearly all of these variable names begin with
   These variables specify the path to the source directory for the external
   LLVM projects Clang, lld, and Polly, respectively, relative to the top-level
   source directory.  If the in-tree subdirectory for an external project
-  exists (e.g., llvm/tools/clang for Clang), then the corresponding variable
+  exists (e.g., ``llvm/tools/clang`` for Clang), then the corresponding variable
   will not be used.  If the variable for an external project does not point
   to a valid path, then that project will not be built.
 
@@ -722,17 +721,17 @@ enabled sub-projects. Nearly all of these variable names begin with
 
 **LLVM_INSTALL_OCAMLDOC_HTML_DIR**:STRING
   The path to install OCamldoc-generated HTML documentation to. This path can
-  either be absolute or relative to the CMAKE_INSTALL_PREFIX. Defaults to
+  either be absolute or relative to the ``CMAKE_INSTALL_PREFIX``. Defaults to
   ``${CMAKE_INSTALL_DOCDIR}/llvm/ocaml-html``.
 
 **LLVM_INSTALL_SPHINX_HTML_DIR**:STRING
   The path to install Sphinx-generated HTML documentation to. This path can
-  either be absolute or relative to the CMAKE_INSTALL_PREFIX. Defaults to
+  either be absolute or relative to the ``CMAKE_INSTALL_PREFIX``. Defaults to
   ``${CMAKE_INSTALL_DOCDIR}/llvm/html``.
 
 **LLVM_INSTALL_UTILS**:BOOL
   If enabled, utility binaries like ``FileCheck`` and ``not`` will be installed
-  to CMAKE_INSTALL_PREFIX.
+  to ``CMAKE_INSTALL_PREFIX``.
 
 **LLVM_INSTALL_DOXYGEN_HTML_DIR**:STRING
   The path to install Doxygen-generated HTML documentation to. This path can
@@ -753,7 +752,7 @@ enabled sub-projects. Nearly all of these variable names begin with
     $ D:\llvm-project> cmake ... -DLLVM_INTEGRATED_CRT_ALLOC=D:\git\rpmalloc
 
   This option needs to be used along with the static CRT, ie. if building the
-  Release target, add -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded.
+  Release target, add ``-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded``.
   Note that rpmalloc is also supported natively in-tree, see option below.
 
 **LLVM_ENABLE_RPMALLOC**:BOOL
@@ -764,7 +763,7 @@ enabled sub-projects. Nearly all of these variable names begin with
 
 **LLVM_LINK_LLVM_DYLIB**:BOOL
   If enabled, tools will be linked with the libLLVM shared library. Defaults
-  to OFF. Setting LLVM_LINK_LLVM_DYLIB to ON also sets LLVM_BUILD_LLVM_DYLIB
+  to OFF. Setting ``LLVM_LINK_LLVM_DYLIB`` to ON also sets ``LLVM_BUILD_LLVM_DYLIB``
   to ON.
   This option is not available on Windows.
 
@@ -779,8 +778,8 @@ enabled sub-projects. Nearly all of these variable names begin with
 **LLVM_LIT_TOOLS_DIR**:PATH
   The path to GnuWin32 tools for tests. Valid on Windows host.  Defaults to
   the empty string, in which case lit will look for tools needed for tests
-  (e.g. ``grep``, ``sort``, etc.) in your %PATH%. If GnuWin32 is not in your
-  %PATH%, then you can set this variable to the GnuWin32 directory so that
+  (e.g. ``grep``, ``sort``, etc.) in your ``%PATH%``. If GnuWin32 is not in your
+  ``%PATH%``, then you can set this variable to the GnuWin32 directory so that
   lit can find tools needed for tests in that directory.
 
 **LLVM_NATIVE_TOOL_DIR**:STRING
@@ -798,9 +797,9 @@ enabled sub-projects. Nearly all of these variable names begin with
   set to non-standard values.
 
 **LLVM_OPTIMIZED_TABLEGEN**:BOOL
-  If enabled and building a debug or asserts build the CMake build system will
+  If enabled and building a debug or asserts build, the CMake build system will
   generate a Release build tree to build a fully optimized tablegen for use
-  during the build. Enabling this option can significantly speed up build times
+  during the build. Enabling this option can significantly speed up build times,
   especially when building LLVM in Debug configurations.
 
 **LLVM_PARALLEL_{COMPILE,LINK,TABLEGEN}_JOBS**:STRING
@@ -809,7 +808,7 @@ enabled sub-projects. Nearly all of these variable names begin with
   determined by the number of logical CPUs.
 
 **LLVM_PROFDATA_FILE**:PATH
-  Path to a profdata file to pass into clang's -fprofile-instr-use flag. This
+  Path to a profdata file to pass into clang's ``-fprofile-instr-use`` flag. This
   can only be specified if you're building with clang.
 
 **LLVM_RAM_PER_{COMPILE,LINK,TABLEGEN}_JOB**:STRING
@@ -833,7 +832,7 @@ enabled sub-projects. Nearly all of these variable names begin with
 
 **LLVM_STATIC_LINK_CXX_STDLIB**:BOOL
   Statically link to the C++ standard library if possible. This uses the flag
-  "-static-libstdc++", but a Clang host compiler will statically link to libc++
+  ``-static-libstdc++``, but a Clang host compiler will statically link to libc++
   if used in conjunction with the **LLVM_ENABLE_LIBCXX** flag. Defaults to OFF.
 
 **LLVM_TABLEGEN**:STRING
@@ -851,8 +850,8 @@ enabled sub-projects. Nearly all of these variable names begin with
   Semicolon-separated list of targets to build, or *all* for building all
   targets. Case-sensitive. Defaults to *all*. Example:
   ``-DLLVM_TARGETS_TO_BUILD="X86;PowerPC"``.
-  The full list, as of March 2023, is:
-  ``AArch64;AMDGPU;ARM;AVR;BPF;Hexagon;Lanai;LoongArch;Mips;MSP430;NVPTX;PowerPC;RISCV;Sparc;SystemZ;VE;WebAssembly;X86;XCore``
+  The full list, as of August 2025, is:
+  ``AArch64;AMDGPU;ARM;AVR;BPF;Hexagon;Lanai;LoongArch;Mips;MSP430;NVPTX;PowerPC;RISCV;Sparc;SPIRV;SystemZ;VE;WebAssembly;X86;XCore``
 
   You can also specify ``host`` or ``Native`` to automatically detect and
   include the target corresponding to the host machine's architecture, or
@@ -870,16 +869,16 @@ enabled sub-projects. Nearly all of these variable names begin with
   the default set of UBSan flags.
 
 **LLVM_UNREACHABLE_OPTIMIZE**:BOOL
-  This flag controls the behavior of `llvm_unreachable()` in release build
+  This flag controls the behavior of ``llvm_unreachable()`` in release build
   (when assertions are disabled in general). When ON (default) then
-  `llvm_unreachable()` is considered "undefined behavior" and optimized as
+  ``llvm_unreachable()`` is considered "undefined behavior" and optimized as
   such. When OFF it is instead replaced with a guaranteed "trap".
 
 **LLVM_USE_INTEL_JITEVENTS**:BOOL
   Enable building support for Intel JIT Events API. Defaults to OFF.
 
 **LLVM_USE_LINKER**:STRING
-  Add ``-fuse-ld={name}`` to the link invocation. The possible value depend on
+  Add ``-fuse-ld={name}`` to the link invocation. The possible values depend on
   your compiler, for clang the value can be an absolute path to your custom
   linker, otherwise clang will prefix the name with ``ld.`` and apply its usual
   search. For example to link LLVM with the Gold linker, cmake can be invoked
@@ -929,7 +928,7 @@ enabled sub-projects. Nearly all of these variable names begin with
   to ON.
 
 **SPHINX_WARNINGS_AS_ERRORS**:BOOL
-  If enabled then sphinx documentation warnings will be treated as
+  If enabled, then sphinx documentation warnings will be treated as
   errors. Defaults to ON.
 
 Advanced variables
@@ -955,12 +954,12 @@ things to go wrong.  They are also unstable across LLVM versions.
 CMake Caches
 ============
 
-Recently LLVM and Clang have been adding some more complicated build system
+Recently, LLVM and Clang have been adding some more complicated build system
 features. Utilizing these new features often involves a complicated chain of
 CMake variables passed on the command line. Clang provides a collection of CMake
 cache scripts to make these features more approachable.
 
-CMake cache files are utilized using CMake's -C flag:
+CMake cache files are utilized using CMake's ``-C`` flag:
 
 .. code-block:: console
 
@@ -974,15 +973,15 @@ A few notes about CMake Caches:
 
 - Order of command line arguments is important
 
-  - -D arguments specified before -C are set before the cache is processed and
+  - ``-D`` arguments specified before -C are set before the cache is processed and
     can be read inside the cache file
-  - -D arguments specified after -C are set after the cache is processed and
+  - ``-D`` arguments specified after -C are set after the cache is processed and
     are unset inside the cache file
 
-- All -D arguments will override cache file settings
+- All ``-D`` arguments will override cache file settings
 - CMAKE_TOOLCHAIN_FILE is evaluated after both the cache file and the command
   line arguments
-- It is recommended that all -D options should be specified *before* -C
+- It is recommended that all ``-D`` options should be specified *before* -C
 
 For more information about some of the advanced build configurations supported
 via Cache files see :doc:`AdvancedBuilds`.
@@ -1055,7 +1054,7 @@ and uses them to build a simple application ``simple-tool``.
 
 The ``find_package(...)`` directive when used in CONFIG mode (as in the above
 example) will look for the ``LLVMConfig.cmake`` file in various locations (see
-cmake manual for details).  It creates a ``LLVM_DIR`` cache entry to save the
+cmake manual for details).  It creates an ``LLVM_DIR`` cache entry to save the
 directory where ``LLVMConfig.cmake`` is found or allows the user to specify the
 directory (e.g. by passing ``-DLLVM_DIR=/usr/lib/cmake/llvm`` to
 the ``cmake`` command or by setting it directly in ``ccmake`` or ``cmake-gui``).
@@ -1079,11 +1078,11 @@ or you wish to build directly against the LLVM build tree you can use
 ``LLVM_DIR`` as previously mentioned.
 
 The ``LLVMConfig.cmake`` file sets various useful variables. Notable variables
-include
+include:
 
 ``LLVM_CMAKE_DIR``
   The path to the LLVM CMake directory (i.e. the directory containing
-  LLVMConfig.cmake).
+  ``LLVMConfig.cmake``).
 
 ``LLVM_DEFINITIONS``
   A list of preprocessor defines that should be used when building against LLVM.
@@ -1123,7 +1122,7 @@ and will be removed in a future version of LLVM.
 Developing LLVM passes out of source
 ------------------------------------
 
-It is possible to develop LLVM passes out of LLVM's source tree (i.e. against an
+You can develop LLVM passes out of LLVM's source tree (i.e. against an
 installed or built LLVM). An example of a project layout is provided below.
 
 .. code-block:: none
diff --git a/llvm/docs/CodingStandards.rst b/llvm/docs/CodingStandards.rst
index 2dc3d7797b133..e48af0ee3054f 100644
--- a/llvm/docs/CodingStandards.rst
+++ b/llvm/docs/CodingStandards.rst
@@ -46,11 +46,11 @@ Languages, Libraries, and Standards
 Most source code in LLVM and other LLVM projects using these coding standards
 is C++ code. There are some places where C code is used either due to
 environment restrictions, historical restrictions, or due to third-party source
-code imported into the tree. Generally, our preference is for standards
-conforming, modern, and portable C++ code as the implementation language of
+code imported into the tree. Generally, our preference is for
+standards-conforming, modern, and portable C++ code as the implementation language of
 choice.
 
-For automation, build-systems, and utility scripts, Python is preferred and
+For automation, build systems, and utility scripts, Python is preferred and
 is widely used in the LLVM repository already.
 
 C++ Standard Versions
@@ -236,7 +236,7 @@ Comment Formatting
 
 In general, prefer C++-style comments (``//`` for normal comments, ``///`` for
 ``doxygen`` documentation comments).  There are a few cases when it is
-useful to use C-style (``/* */``) comments however:
+useful to use C-style (``/* */``) comments, however:
 
 #. When writing C code to be compatible with C89.
 
@@ -263,7 +263,7 @@ useful to use C-style (``/* */``) comments however:
 Commenting out large blocks of code is discouraged, but if you really have to do
 this (for documentation purposes or as a suggestion for debug printing), use
 ``#if 0`` and ``#endif``. These nest properly and are better behaved in general
-than C style comments.
+than C-style comments.
 
 Doxygen Use in Documentation Comments
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -365,7 +365,7 @@ Clear diagnostic messages are important to help users identify and fix issues in
 their inputs. Use succinct but correct English prose that gives the user the
 context needed to understand what went wrong. Also, to match error message
 styles commonly produced by other tools, start the first sentence with a
-lower-case letter, and finish the last sentence without a period, if it would
+lowercase letter, and finish the last sentence without a period, if it would
 end in one otherwise. Sentences which end with different punctuation, such as
 "did you forget ';'?", should still do so.
 
@@ -703,7 +703,7 @@ uses a more moderate stance. Use ``auto`` if and only if it makes the code more
 readable or easier to maintain. Don't "almost always" use ``auto``, but do use
 ``auto`` with initializers like ``cast<Foo>(...)`` or other places where the
 type is already obvious from the context. Another time when ``auto`` works well
-for these purposes is when the type would have been abstracted away anyways,
+for these purposes is when the type would have been abstracted away anyway,
 often behind a container's typedef such as ``std::vector<T>::iterator``.
 
 Similarly, C++14 adds generic lambda expressions where parameter types can be
@@ -754,7 +754,7 @@ Beware of non-deterministic sorting order of equal elements
 elements is not guaranteed to be preserved. Thus using ``std::sort`` for a
 container having equal elements may result in non-deterministic behavior.
 To uncover such instances of non-determinism, LLVM has introduced a new
-llvm::sort wrapper function. For an EXPENSIVE_CHECKS build this will randomly
+``llvm::sort`` wrapper function. For an ``EXPENSIVE_CHECKS`` build this will randomly
 shuffle the container before sorting. Default to using ``llvm::sort`` instead
 of ``std::sort``.
 
@@ -829,7 +829,7 @@ prototyped function or method, you don't need it.  In fact, for most cases, you
 simply don't need the definition of a class. And not ``#include``\ing speeds up
 compilation.
 
-It is easy to try to go too overboard on this recommendation, however.  You
+It is easy to try to go overboard on this recommendation, however.  You
 **must** include all of the header files that you are using --- you can include
 them either directly or indirectly through another header file.  To make sure
 that you don't accidentally forget to include a header file in your module
@@ -857,7 +857,7 @@ your private interface remains private and undisturbed by outsiders.
 Use Namespace Qualifiers to Implement Previously Declared Functions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-When providing an out of line implementation of a function in a source file, do
+When providing an out-of-line implementation of a function in a source file, do
 not open namespace blocks in the source file. Instead, use namespace qualifiers
 to help ensure that your definition matches an existing declaration. Do this:
 
@@ -1154,7 +1154,7 @@ In general, names should be in camel case (e.g. ``TextFileReader`` and
 
 * **Function names** should be verb phrases (as they represent actions), and
   command-like function should be imperative.  The name should be camel case,
-  and start with a lower-case letter (e.g. ``openFile()`` or ``isFoo()``).
+  and start with a lowercase letter (e.g. ``openFile()`` or ``isFoo()``).
 
 * **Enum declarations** (e.g. ``enum Foo {...}``) are types, so they should
   follow the naming conventions for types.  A common use for enums is as a
@@ -1179,7 +1179,7 @@ In general, names should be in camel case (e.g. ``TextFileReader`` and
       };
 
 As an exception, classes that mimic STL classes can have member names in STL's
-style of lower-case words separated by underscores (e.g. ``begin()``,
+style of lowercase words separated by underscores (e.g. ``begin()``,
 ``push_back()``, and ``empty()``). Classes that provide multiple
 iterators should add a singular prefix to ``begin()`` and ``end()``
 (e.g. ``global_begin()`` and ``use_begin()``).
@@ -1417,7 +1417,7 @@ please write the loop in the first form and add a comment indicating that you
 did it intentionally.
 
 Why do we prefer the second form (when correct)?  Writing the loop in the first
-form has two problems. First it may be less efficient than evaluating it at the
+form has two problems. First, it may be less efficient than evaluating it at the
 start of the loop.  In this case, the cost is probably minor --- a few extra
 loads every time through the loop.  However, if the base expression is more
 complex, then the cost can rise quickly.  I've seen loops where the end
diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst
index eb90e950a3770..15c249d8e6d31 100644
--- a/llvm/docs/CommandGuide/lit.rst
+++ b/llvm/docs/CommandGuide/lit.rst
@@ -399,6 +399,11 @@ ADDITIONAL OPTIONS
  Show all features used in the test suite (in ``XFAIL``, ``UNSUPPORTED`` and
  ``REQUIRES``) and exit.
 
+.. option:: --update-tests
+
+ Pass failing tests to functions in the ``lit_config.test_updaters`` list to
+ check whether any of them know how to update the test to make it pass.
+
 EXIT STATUS
 -----------
 
diff --git a/llvm/docs/ContentAddressableStorage.md b/llvm/docs/ContentAddressableStorage.md
new file mode 100644
index 0000000000000..a42cbe731bf25
--- /dev/null
+++ b/llvm/docs/ContentAddressableStorage.md
@@ -0,0 +1,134 @@
+# Content Addressable Storage
+
+## Introduction to CAS
+
+Content Addressable Storage, or `CAS`, is a storage system that assigns
+unique addresses to the data stored. It is very useful for data deduplicaton
+and creating unique identifiers.
+
+Unlike other kinds of storage systems, like file systems, CAS is immutable. It
+is more reliable to model a computation by representing the inputs and outputs
+of the computation using objects stored in CAS.
+
+The basic unit of the CAS library is a CASObject, where it contains:
+
+* Data: arbitrary data
+* References: references to other CASObject
+
+It can be conceptually modeled as something like:
+
+```
+struct CASObject {
+  ArrayRef<char> Data;
+  ArrayRef<CASObject*> Refs;
+}
+```
+
+With this abstraction, it is possible to compose `CASObject`s into a DAG that is
+capable of representing complicated data structures, while still allowing data
+deduplication. Note you can compare two DAGs by just comparing the CASObject
+hash of two root nodes.
+
+
+## LLVM CAS Library User Guide
+
+The CAS-like storage provided in LLVM is `llvm::cas::ObjectStore`.
+To reference a CASObject, there are few different abstractions provided
+with different trade-offs:
+
+### ObjectRef
+
+`ObjectRef` is a lightweight reference to a CASObject stored in the CAS.
+This is the most commonly used abstraction and it is cheap to copy/pass
+along. It has following properties:
+
+* `ObjectRef` is only meaningful within the `ObjectStore` that created the ref.
+`ObjectRef` created by different `ObjectStore` cannot be cross-referenced or
+compared.
+* `ObjectRef` doesn't guarantee the existence of the CASObject it points to. An
+explicit load is required before accessing the data stored in CASObject.
+This load can also fail, for reasons like (but not limited to): object does
+not exist, corrupted CAS storage, operation timeout, etc.
+* If two `ObjectRef` are equal, it is guaranteed that the object they point to
+are identical (if they exist). If they are not equal, the underlying objects are
+guaranteed to be not the same.
+
+### ObjectProxy
+
+`ObjectProxy` represents a loaded CASObject. With an `ObjectProxy`, the
+underlying stored data and references can be accessed without the need
+of error handling. The class APIs also provide convenient methods to
+access underlying data. The lifetime of the underlying data is equal to
+the lifetime of the instance of `ObjectStore` unless explicitly copied.
+
+### CASID
+
+`CASID` is the hash identifier for CASObjects. It owns the underlying
+storage for hash value so it can be expensive to copy and compare depending
+on the hash algorithm. `CASID` is generally only useful in rare situations
+like printing raw hash value or exchanging hash values between different
+CAS instances with the same hashing schema.
+
+### ObjectStore
+
+`ObjectStore` is the CAS-like object storage. It provides API to save
+and load CASObjects, for example:
+
+```
+ObjectRef A, B, C;
+Expected<ObjectRef> Stored = ObjectStore.store("data", {A, B});
+Expected<ObjectProxy> Loaded = ObjectStore.getProxy(C);
+```
+
+It also provides APIs to convert between `ObjectRef`, `ObjectProxy` and
+`CASID`.
+
+### ActionCache
+
+`ActionCache` is a key value storage can be used to associate two CASIDs. It is
+usually used with an `ObjectStore` to map an input CASObject to an output CASObject
+with their CASIDs.
+
+`ActionCache` has APIs like following:
+
+```
+CASID A, B;
+Error E = ActionCache.put(A, B);
+Expected<std::optional<CASID>> Result = ActionCache.get(A);
+```
+
+
+## CAS Library Implementation Guide
+
+The LLVM ObjectStore API was designed so that it is easy to add
+customized CAS implementations that are interchangeable with the builtin
+ones.
+
+To add your own implementation, you just need to add a subclass to
+`llvm::cas::ObjectStore` and implement all its pure virtual methods.
+To be interchangeable with LLVM ObjectStore, the new CAS implementation
+needs to conform to following contracts:
+
+* Different CASObjects stored in the ObjectStore need to have a different hash
+and result in a different `ObjectRef`. Similarly, the same CASObject should have
+the same hash and the same `ObjectRef`. Note: two different CASObjects with
+identical data but different references are considered different objects.
+* `ObjectRef`s are only comparable within the same `ObjectStore` instance, and
+can be used to determine the equality of the underlying CASObjects.
+* The loaded objects from the ObjectStore need to have a lifetime at least as
+long as the ObjectStore itself so it is always legal to access the loaded data
+without holding on the `ObjectProxy` until the `ObjectStore` is destroyed.
+
+
+If not specified, the behavior can be implementation defined. For example,
+`ObjectRef` can be used to point to a loaded CASObject so
+`ObjectStore` never fails to load. It is also legal to use a stricter model
+than required. For example, the underlying value inside `ObjectRef` can be
+the unique indentities of the objects across multiple `ObjectStore` instances,
+but comparing such `ObjectRef` from different `ObjectStore` is still illegal.
+
+For CAS library implementers, there is also an `ObjectHandle` class that
+is an internal representation of a loaded CASObject reference.
+`ObjectProxy` is just a pair of `ObjectHandle` and `ObjectStore`, and
+just like `ObjectRef`, `ObjectHandle` is only useful when paired with
+the `ObjectStore` that knows about the loaded CASObject.
diff --git a/llvm/docs/GlobalISel/Legalizer.rst b/llvm/docs/GlobalISel/Legalizer.rst
index 0e256ac7781cc..74e83aef80ab2 100644
--- a/llvm/docs/GlobalISel/Legalizer.rst
+++ b/llvm/docs/GlobalISel/Legalizer.rst
@@ -15,7 +15,7 @@ A legal instruction is defined as:
 * operating on **vregs that can be loaded and stored** -- if necessary, the
   target can select a ``G_LOAD``/``G_STORE`` of each gvreg operand.
 
-As opposed to SelectionDAG, there are no legalization phases.  In particular,
+Unlike SelectionDAG, there are no legalization phases.  In particular,
 'type' and 'operation' legalization are not separate.
 
 Legalization is iterative, and all state is contained in GMIR.  To maintain the
@@ -44,7 +44,7 @@ have a ``G_FOO`` instruction of the form::
   %1:_(s32) = G_CONSTANT i32 1
   %2:_(s32) = G_FOO %0:_(s32), %1:_(s32)
 
-it's impossible to say that G_FOO is legal iff %1 is a ``G_CONSTANT`` with
+it's impossible to say that ``G_FOO`` is legal iff %1 is a ``G_CONSTANT`` with
 value ``1``. However, the following::
 
   %2:_(s32) = G_FOO %0:_(s32), i32 1
@@ -93,8 +93,8 @@ legality contains:
 
 .. rubric:: Footnotes
 
-.. [#legalizer-legacy-footnote] An API is broadly similar to
-   SelectionDAG/TargetLowering is available but is not recommended as a more
+.. [#legalizer-legacy-footnote] An API that is broadly similar to
+   SelectionDAG/TargetLowering is available, but is not recommended as a more
    powerful API is available.
 
 Rule Processing and Declaring Rules
@@ -108,10 +108,10 @@ legalized as a result of the rules. If the ruleset is exhausted without
 satisfying any rule, then it is considered unsupported.
 
 When it doesn't declare the instruction legal, each pass over the rules may
-request that one type changes to another type. Sometimes this can cause multiple
+request that one type be changed to another type. Sometimes this can cause multiple
 types to change but we avoid this as much as possible as making multiple changes
 can make it difficult to avoid infinite loops where, for example, narrowing one
-type causes another to be too small and widening that type causes the first one
+type causes another to be too small, and widening that type causes the first one
 to be too big.
 
 In general, it's advisable to declare instructions legal as close to the top of
@@ -130,7 +130,7 @@ and the instruction::
 
   %2:_(s7) = G_ADD %0:_(s7), %1:_(s7)
 
-this doesn't meet the predicate for the :ref:`.legalFor() <legalfor>` as ``s7``
+This doesn't meet the predicate for the :ref:`.legalFor() <legalfor>` as ``s7``
 is not one of the listed types so it falls through to the
 :ref:`.clampScalar() <clampscalar>`. It does meet the predicate for this rule
 as the type is smaller than the ``s32`` and this rule instructs the legalizer
@@ -148,7 +148,7 @@ processing by the legalizer.
 Rule Actions
 """"""""""""
 
-There are various rule factories that append rules to a ruleset but they have a
+There are various rule factories that append rules to a ruleset, but they have a
 few actions in common:
 
 .. _legalfor:
@@ -202,7 +202,7 @@ few actions in common:
 Rule Predicates
 """""""""""""""
 
-The rule factories also have predicates in common:
+The rule factories also have the following predicates in common:
 
 * ``legal()``, ``lower()``, etc. are always satisfied.
 
@@ -269,81 +269,81 @@ Consumer Type Set
   The set of types which is the union of all possible types consumed by at
   least one legal instruction.
 
-Both sets are often identical but there's no guarantee of that. For example,
+Both sets are often identical, but there's no guarantee of that. For example,
 it's not uncommon to be unable to consume s64 but still be able to produce it
 for a few specific instructions.
 
 Minimum Rules For Scalars
 """""""""""""""""""""""""
 
-* G_ANYEXT must be legal for all inputs from the producer type set and all larger
+* ``G_ANYEXT`` must be legal for all inputs from the producer type set and all larger
   outputs from the consumer type set.
-* G_TRUNC must be legal for all inputs from the producer type set and all
+* ``G_TRUNC`` must be legal for all inputs from the producer type set and all
   smaller outputs from the consumer type set.
 
-G_ANYEXT, and G_TRUNC have mandatory legality since the GMIR requires a means to
+``G_ANYEXT`` and ``G_TRUNC`` have mandatory legality since the GMIR requires a means to
 connect operations with different type sizes. They are usually trivial to support
-since G_ANYEXT doesn't define the value of the additional bits and G_TRUNC is
-discarding bits. The other conversions can be lowered into G_ANYEXT/G_TRUNC
+since ``G_ANYEXT`` doesn't define the value of the additional bits and ``G_TRUNC`` is
+discarding bits. The other conversions can be lowered into ``G_ANYEXT``/``G_TRUNC``
 with some additional operations that are subject to further legalization. For
-example, G_SEXT can lower to::
+example, ``G_SEXT`` can lower to::
 
   %1 = G_ANYEXT %0
   %2 = G_CONSTANT ...
   %3 = G_SHL %1, %2
   %4 = G_ASHR %3, %2
 
-and the G_CONSTANT/G_SHL/G_ASHR can further lower to other operations or target
-instructions. Similarly, G_FPEXT has no legality requirement since it can lower
-to a G_ANYEXT followed by a target instruction.
+and the ``G_CONSTANT``/``G_SHL``/``G_ASHR`` can further lower to other operations or target
+instructions. Similarly, ``G_FPEXT`` has no legality requirement since it can lower
+to a ``G_ANYEXT`` followed by a target instruction.
 
-G_MERGE_VALUES and G_UNMERGE_VALUES do not have legality requirements since the
-former can lower to G_ANYEXT and some other legalizable instructions, while the
-latter can lower to some legalizable instructions followed by G_TRUNC.
+``G_MERGE_VALUES`` and ``G_UNMERGE_VALUES`` do not have legality requirements since the
+former can lower to ``G_ANYEXT`` and some other legalizable instructions, while the
+latter can lower to some legalizable instructions followed by ``G_TRUNC``.
 
 Minimum Legality For Vectors
 """"""""""""""""""""""""""""
 
 Within the vector types, there aren't any defined conversions in LLVM IR as
 vectors are often converted by reinterpreting the bits or by decomposing the
-vector and reconstituting it as a different type. As such, G_BITCAST is the
+vector and reconstituting it as a different type. As such, ``G_BITCAST`` is the
 only operation to account for. We generally don't require that it's legal
-because it can usually be lowered to COPY (or to nothing using
-replaceAllUses()). However, there are situations where G_BITCAST is non-trivial
+because it can usually be lowered to ``COPY`` (or to nothing using
+``replaceAllUses()``). However, there are situations where ``G_BITCAST`` is non-trivial
 (e.g. little-endian vectors of big-endian data such as on big-endian MIPS MSA and
-big-endian ARM NEON, see `_i_bitcast`). To account for this G_BITCAST must be
+big-endian ARM NEON, see `_i_bitcast`). To account for this, ``G_BITCAST`` must be
 legal for all type combinations that change the bit pattern in the value.
 
-There are no legality requirements for G_BUILD_VECTOR, or G_BUILD_VECTOR_TRUNC
+There are no legality requirements for ``G_BUILD_VECTOR``, or ``G_BUILD_VECTOR_TRUNC``
 since these can be handled by:
 * Declaring them legal.
 * Scalarizing them.
-* Lowering them to G_TRUNC+G_ANYEXT and some legalizable instructions.
+* Lowering them to ``G_TRUNC``+``G_ANYEXT`` and some legalizable instructions.
 * Lowering them to target instructions which are legal by definition.
 
-The same reasoning also allows G_UNMERGE_VALUES to lack legality requirements
+The same reasoning also allows ``G_UNMERGE_VALUES`` to lack legality requirements
 for vector inputs.
 
 Minimum Legality for Pointers
 """""""""""""""""""""""""""""
 
-There are no minimum rules for pointers since G_INTTOPTR and G_PTRTOINT can
-be selected to a COPY from register class to another by the legalizer.
+There are no minimum rules for pointers since ``G_INTTOPTR`` and ``G_PTRTOINT`` can
+be selected to a ``COPY`` from register class to another by the legalizer.
 
 Minimum Legality For Operations
 """""""""""""""""""""""""""""""
 
-The rules for G_ANYEXT, G_MERGE_VALUES, G_BITCAST, G_BUILD_VECTOR,
-G_BUILD_VECTOR_TRUNC, G_CONCAT_VECTORS, G_UNMERGE_VALUES, G_PTRTOINT, and
-G_INTTOPTR have already been noted above. In addition to those, the following
+The rules for ``G_ANYEXT``, ``G_MERGE_VALUES``, ``G_BITCAST``, ``G_BUILD_VECTOR``,
+``G_BUILD_VECTOR_TRUNC``, ``G_CONCAT_VECTORS``, ``G_UNMERGE_VALUES``, ``G_PTRTOINT``, and
+``G_INTTOPTR`` have already been noted above. In addition to those, the following
 operations have requirements:
 
-* For every type that can be produced by any instruction, G_IMPLICIT_DEF must be
-  legal.
-* G_PHI must be legal for all types in the producer and consumer typesets. This
+* ``G_IMPLICIT_DEF`` must be legal for every type that can be produced
+   by any instruction.
+* ``G_PHI`` must be legal for all types in the producer and consumer typesets. This
   is usually trivial as it requires no code to be selected.
-* At least one G_FRAME_INDEX must be legal
-* At least one G_BLOCK_ADDR must be legal
+* At least one ``G_FRAME_INDEX`` must be legal
+* At least one ``G_BLOCK_ADDR`` must be legal
 
-There are many other operations you'd expect to have legality requirements but
+There are many other operations you'd expect to have legality requirements, but
 they can be lowered to target instructions which are legal by definition.
diff --git a/llvm/docs/HowToCrossCompileBuiltinsOnArm.rst b/llvm/docs/HowToCrossCompileBuiltinsOnArm.rst
index 31ead45d052fe..d7759ad8edd06 100644
--- a/llvm/docs/HowToCrossCompileBuiltinsOnArm.rst
+++ b/llvm/docs/HowToCrossCompileBuiltinsOnArm.rst
@@ -25,12 +25,16 @@ using as many of the LLVM tools as we can, but it is possible to use GNU
 equivalents.
 
 You will need:
- * A build of LLVM for the llvm-tools and ``llvm-config``.
+ * A build of LLVM for the llvm-tools and LLVM CMake files.
  * A clang executable with support for the ``ARM`` target.
- * compiler-rt sources.
+ * ``compiler-rt`` sources.
  * The ``qemu-arm`` user mode emulator.
  * An ``arm-linux-gnueabihf`` sysroot.
 
+.. note::
+  An existing sysroot is required because some of the builtins include C library
+  headers and a sysroot is the easiest way to get those.
+
 In this example we will be using ``ninja`` as the build tool.
 
 See https://compiler-rt.llvm.org/ for information about the dependencies
@@ -52,78 +56,94 @@ toolchain from https://developer.arm.com/open-source/gnu-toolchain/gnu-a/downloa
 Building compiler-rt builtins for Arm
 =====================================
 
-We will be doing a standalone build of compiler-rt using the following cmake
-options::
+We will be doing a standalone build of compiler-rt. The command is shown below.
+Shell variables are used to simplify some of the options::
+
+  LLVM_TOOLCHAIN=<path-to-llvm-install>/
+  TARGET_TRIPLE=arm-none-linux-gnueabihf
+  GCC_TOOLCHAIN=<path-to-gcc-toolchain>
+  SYSROOT=${GCC_TOOLCHAIN}/${TARGET_TRIPLE}/libc
+  COMPILE_FLAGS="-march=armv7-a"
 
-  cmake path/to/compiler-rt \
+  cmake ../llvm-project/compiler-rt \
     -G Ninja \
-    -DCMAKE_AR=/path/to/llvm-ar \
-    -DCMAKE_ASM_COMPILER_TARGET="arm-linux-gnueabihf" \
-    -DCMAKE_ASM_FLAGS="build-c-flags" \
-    -DCMAKE_C_COMPILER=/path/to/clang \
-    -DCMAKE_C_COMPILER_TARGET="arm-linux-gnueabihf" \
-    -DCMAKE_C_FLAGS="build-c-flags" \
+    -DCMAKE_AR=${LLVM_TOOLCHAIN}/bin/llvm-ar \
+    -DCMAKE_NM=${LLVM_TOOLCHAIN}/bin/llvm-nm \
+    -DCMAKE_RANLIB=${LLVM_TOOLCHAIN}/bin/llvm-ranlib \
+    -DLLVM_CMAKE_DIR="${LLVM_TOOLCHAIN}/lib/cmake/llvm" \
+    -DCMAKE_SYSROOT="${SYSROOT}" \
+    -DCMAKE_ASM_COMPILER_TARGET="${TARGET_TRIPLE}" \
+    -DCMAKE_ASM_FLAGS="${COMPILE_FLAGS}" \
+    -DCMAKE_C_COMPILER_TARGET="${TARGET_TRIPLE}" \
+    -DCMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN=${GCC_TOOLCHAIN} \
+    -DCMAKE_C_COMPILER=${LLVM_TOOLCHAIN}/bin/clang \
+    -DCMAKE_C_FLAGS="${COMPILE_FLAGS}" \
+    -DCMAKE_CXX_COMPILER_TARGET="${TARGET_TRIPLE}" \
+    -DCMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN=${GCC_TOOLCHAIN} \
+    -DCMAKE_CXX_COMPILER=${LLVM_TOOLCHAIN}/bin/clang \
+    -DCMAKE_CXX_FLAGS="${COMPILE_FLAGS}" \
     -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \
-    -DCMAKE_NM=/path/to/llvm-nm \
-    -DCMAKE_RANLIB=/path/to/llvm-ranlib \
     -DCOMPILER_RT_BUILD_BUILTINS=ON \
     -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \
     -DCOMPILER_RT_BUILD_MEMPROF=OFF \
     -DCOMPILER_RT_BUILD_PROFILE=OFF \
+    -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \
     -DCOMPILER_RT_BUILD_SANITIZERS=OFF \
     -DCOMPILER_RT_BUILD_XRAY=OFF \
+    -DCOMPILER_RT_BUILD_ORC=OFF \
+    -DCOMPILER_RT_BUILD_CRT=OFF \
     -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \
-    -DLLVM_CONFIG_PATH=/path/to/llvm-config
+    -DCOMPILER_RT_EMULATOR="qemu-arm -L ${SYSROOT}" \
+    -DCOMPILER_RT_INCLUDE_TESTS=ON \
+    -DCOMPILER_RT_TEST_COMPILER=${LLVM_TOOLCHAIN}/bin/clang \
+    -DCOMPILER_RT_TEST_COMPILER_CFLAGS="--target=${TARGET_TRIPLE} ${COMPILE_FLAGS} --gcc-toolchain=${GCC_TOOLCHAIN} --sysroot=${SYSROOT} -fuse-ld=lld"
 
-The ``build-c-flags`` need to be sufficient to pass the C-make compiler check,
-compile compiler-rt, and if you are running the tests, compile and link the
-tests. When cross-compiling with clang we will need to pass sufficient
-information to generate code for the Arm architecture we are targeting.
+.. note::
+  The command above also enables tests. Enabling tests is not required, more details
+  in the testing section.
 
-We will need to select:
- * The Arm target and Armv7-A architecture with ``--target=arm-linux-gnueabihf -march=armv7a``.
- * Whether to generate Arm (the default) or Thumb instructions (``-mthumb``).
+``CMAKE_<LANGUAGE>_<OPTION>`` options are set so that the correct ``--target``,
+``--sysroot``, ``--gcc-toolchain`` and ``-march`` options will be given to the
+compilers.
 
-When using a GCC ``arm-linux-gnueabihf`` toolchain the following flags are
-needed to pick up the includes and libraries:
+The combination of these settings needs to be enough to pass CMake's compiler
+checks, compile compiler-rt and build the test cases.
 
- * ``--gcc-toolchain=/path/to/dir/toolchain``
- * ``--sysroot=/path/to/toolchain/arm-linux-gnueabihf/libc``
+The flags need to select:
+ * The Arm target (``--target arm-none-linux-gnueabihf``)
+ * The Arm architecture level (``-march=armv7-a``)
+ * Whether to generate Arm (``-marm``, the default) or Thumb (``-mthumb``) instructions.
 
-In this example we will be adding all of the command line options to both
-``CMAKE_C_FLAGS`` and ``CMAKE_ASM_FLAGS``. There are cmake flags to pass some of
-these options individually which can be used to simplify the ``build-c-flags``::
+It is possible to pass all these flags to CMake using ``CMAKE_<LANGUAGE>_FLAGS``,
+but the command above uses standard CMake options instead. If you need to
+add flags that CMake cannot generate automatically, add them to
+``CMAKE_<LANGUAGE>_FLAGS``.
 
- -DCMAKE_C_COMPILER_TARGET="arm-linux-gnueabihf"
- -DCMAKE_ASM_COMPILER_TARGET="arm-linux-gnueabihf"
- -DCMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN=/path/to/dir/toolchain
- -DCMAKE_SYSROOT=/path/to/dir/toolchain/arm-linux-gnueabihf/libc
+When CMake has finished, build with Ninja::
 
-Once cmake has completed the builtins can be built with ``ninja builtins``
+  ninja builtins
 
 Testing compiler-rt builtins using qemu-arm
 ===========================================
 
-To test the builtins library we need to add a few more cmake flags to enable
-testing and set up the compiler and flags for test case. We must also tell
-cmake that we wish to run the tests on ``qemu-arm``::
+The following options are required to enable tests::
+
+ -DCOMPILER_RT_EMULATOR="qemu-arm -L ${SYSROOT}" \
+ -DCOMPILER_RT_INCLUDE_TESTS=ON \
+ -DCOMPILER_RT_TEST_COMPILER=${LLVM_TOOLCHAIN}/bin/clang \
+ -DCOMPILER_RT_TEST_COMPILER_CFLAGS="--target=${TARGET_TRIPLE} -march=armv7-a --gcc-toolchain=${GCC_TOOLCHAIN} --sysroot=${SYSROOT} -fuse-ld=lld"
 
- -DCOMPILER_RT_EMULATOR="qemu-arm -L /path/to/armhf/sysroot"
- -DCOMPILER_RT_INCLUDE_TESTS=ON
- -DCOMPILER_RT_TEST_COMPILER="/path/to/clang"
- -DCOMPILER_RT_TEST_COMPILER_CFLAGS="test-c-flags"
+This tells compiler-rt that we want to run tests on ``qemu-arm``. If you do not
+want to run tests, remove these options from the CMake command.
 
-The ``/path/to/armhf/sysroot`` should be the same as the one passed to
-``--sysroot`` in the ``build-c-flags``.
+Note that ``COMPILER_RT_TEST_COMPILER_CFLAGS`` contains the equivalent of the
+options CMake generated for us with the first command. We must pass them
+manually here because standard options like ``CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN``
+do not apply here.
 
-The ``test-c-flags`` need to include the target, architecture, gcc-toolchain,
-sysroot and Arm/Thumb state. The additional cmake defines such as
-``CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN`` do not apply when building the tests. If
-you have put all of these in ``build-c-flags`` then these can be repeated. If you
-wish to use lld to link the tests then add ``-fuse-ld=lld``.
+When CMake has finished, run the tests::
 
-Once cmake has completed the tests can be built and run using
-``ninja check-builtins``
+  ninja check-builtins
 
 Troubleshooting
 ===============
@@ -133,9 +153,10 @@ The cmake try compile stage fails
 At an early stage cmake will attempt to compile and link a simple C program to
 test if the toolchain is working.
 
-This stage can often fail at link time if the ``--sysroot=`` and
+This stage can often fail at link time if the ``--sysroot=``, ``--target`` or
 ``--gcc-toolchain=`` options are not passed to the compiler. Check the
-``CMAKE_C_FLAGS`` and ``CMAKE_C_COMPILER_TARGET`` flags.
+``CMAKE_<LANGUAGE>_FLAGS`` and ``CMAKE_<LANGAUGE>_COMPILER_TARGET`` flags along
+with any of the specific CMake sysroot and toolchain options.
 
 It can be useful to build a simple example outside of cmake with your toolchain
 to make sure it is working. For example::
@@ -179,10 +200,10 @@ The flags used to build the tests are not the same as those used to build the
 builtins. The c flags are provided by ``COMPILER_RT_TEST_COMPILE_CFLAGS`` and
 the ``CMAKE_C_COMPILER_TARGET``, ``CMAKE_ASM_COMPILER_TARGET``,
 ``CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN`` and ``CMAKE_SYSROOT`` flags are not
-applied.
+applied to tests.
 
 Make sure that ``COMPILER_RT_TEST_COMPILE_CFLAGS`` contains all the necessary
-information.
+flags.
 
 
 Modifications for other Targets
@@ -206,13 +227,13 @@ You will need to use an ``arm-linux-gnueabi`` GNU toolchain for soft-float.
 AArch64 Target
 --------------
 The instructions for Arm can be used for AArch64 by substituting AArch64
-equivalents for the sysroot, emulator and target.
+equivalents for the sysroot, emulator and target::
 
-* ``-DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu``
-* ``-DCOMPILER_RT_EMULATOR="qemu-aarch64 -L /path/to/aarch64/sysroot``
+ -DCMAKE_C_COMPILER_TARGET=aarch64-linux-gnu
+ -DCOMPILER_RT_EMULATOR="qemu-aarch64 -L /path/to/aarch64/sysroot
 
-The CMAKE_C_FLAGS and COMPILER_RT_TEST_COMPILER_CFLAGS may also need:
-``"--sysroot=/path/to/aarch64/sysroot --gcc-toolchain=/path/to/gcc-toolchain"``
+You will also have to update any use of the target triple in compiler flags.
+For instance in ``CMAKE_C_FLAGS`` and ``COMPILER_RT_TEST_COMPILER_CFLAGS``.
 
 Armv6-m, Armv7-m and Armv7E-M targets
 -------------------------------------
@@ -221,7 +242,7 @@ but more difficult. The main problems are:
 
 * There is not a ``qemu-arm`` user-mode emulator for bare-metal systems.
   ``qemu-system-arm`` can be used but this is significantly more difficult
-  to setup.
+  to setup. This document does not explain how to do this.
 * The targets to compile compiler-rt have the suffix ``-none-eabi``. This uses
   the BareMetal driver in clang and by default will not find the libraries
   needed to pass the cmake compiler check.
@@ -235,31 +256,68 @@ into a binary and execute the tests correctly but it will not catch if the
 builtins use instructions that are supported on Armv7-A but not Armv6-M,
 Armv7-M and Armv7E-M.
 
-To get the cmake compile test to pass you will need to pass the libraries
-needed to successfully link the cmake test via ``CMAKE_CFLAGS``::
-
- -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \
- -DCOMPILER_RT_OS_DIR="baremetal" \
- -DCOMPILER_RT_BUILD_BUILTINS=ON \
- -DCOMPILER_RT_BUILD_SANITIZERS=OFF \
- -DCOMPILER_RT_BUILD_XRAY=OFF \
- -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \
- -DCOMPILER_RT_BUILD_PROFILE=OFF \
- -DCMAKE_C_COMPILER=${host_install_dir}/bin/clang \
- -DCMAKE_C_COMPILER_TARGET="your *-none-eabi target" \
- -DCMAKE_ASM_COMPILER_TARGET="your *-none-eabi target" \
- -DCMAKE_AR=/path/to/llvm-ar \
- -DCMAKE_NM=/path/to/llvm-nm \
- -DCMAKE_RANLIB=/path/to/llvm-ranlib \
- -DCOMPILER_RT_BAREMETAL_BUILD=ON \
- -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \
- -DLLVM_CONFIG_PATH=/path/to/llvm-config \
- -DCMAKE_C_FLAGS="build-c-flags" \
- -DCMAKE_ASM_FLAGS="build-c-flags" \
- -DCOMPILER_RT_EMULATOR="qemu-arm -L /path/to/armv7-A/sysroot" \
- -DCOMPILER_RT_INCLUDE_TESTS=ON \
- -DCOMPILER_RT_TEST_COMPILER="/path/to/clang" \
- -DCOMPILER_RT_TEST_COMPILER_CFLAGS="test-c-flags"
+This requires a second ``arm-none-eabi`` toolchain for building the builtins.
+Using a bare-metal toolchain ensures that the target and C library details are
+specific to bare-metal instead of using Linux settings. This means that some
+tests may behave differently compared to real hardware, but at least the content
+of the builtins library is correct.
+
+Below is an example that builds the builtins for Armv7-M, but runs the tests
+as Armv7-A. It is presented in full, but is very similar to the earlier
+command for Armv7-A build and test::
+
+  LLVM_TOOLCHAIN=<path to llvm install>/
+
+  # For the builtins.
+  TARGET_TRIPLE=arm-none-eabi
+  GCC_TOOLCHAIN=<path to arm-none-eabi toolchain>/
+  SYSROOT=${GCC_TOOLCHAIN}/${TARGET_TRIPLE}/libc
+  COMPILE_FLAGS="-march=armv7-m -mfpu=vfpv2"
+
+  # For the test cases.
+  A_PROFILE_TARGET_TRIPLE=arm-none-linux-gnueabihf
+  A_PROFILE_GCC_TOOLCHAIN=<path to arm-none-linux-gnueabihf toolchain>/
+  A_PROFILE_SYSROOT=${A_PROFILE_GCC_TOOLCHAIN}/${A_PROFILE_TARGET_TRIPLE}/libc
+
+  cmake ../llvm-project/compiler-rt \
+    -G Ninja \
+    -DCMAKE_AR=${LLVM_TOOLCHAIN}/bin/llvm-ar \
+    -DCMAKE_NM=${LLVM_TOOLCHAIN}/bin/llvm-nm \
+    -DCMAKE_RANLIB=${LLVM_TOOLCHAIN}/bin/llvm-ranlib \
+    -DLLVM_CMAKE_DIR="${LLVM_TOOLCHAIN}/lib/cmake/llvm" \
+    -DCMAKE_SYSROOT="${SYSROOT}" \
+    -DCMAKE_ASM_COMPILER_TARGET="${TARGET_TRIPLE}" \
+    -DCMAKE_ASM_FLAGS="${COMPILE_FLAGS}" \
+    -DCMAKE_C_COMPILER_TARGET="${TARGET_TRIPLE}" \
+    -DCMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN=${GCC_TOOLCHAIN} \
+    -DCMAKE_C_COMPILER=${LLVM_TOOLCHAIN}/bin/clang \
+    -DCMAKE_C_FLAGS="${COMPILE_FLAGS}" \
+    -DCMAKE_CXX_COMPILER_TARGET="${TARGET_TRIPLE}" \
+    -DCMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN=${GCC_TOOLCHAIN} \
+    -DCMAKE_CXX_COMPILER=${LLVM_TOOLCHAIN}/bin/clang \
+    -DCMAKE_CXX_FLAGS="${COMPILE_FLAGS}" \
+    -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \
+    -DCOMPILER_RT_BUILD_BUILTINS=ON \
+    -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \
+    -DCOMPILER_RT_BUILD_MEMPROF=OFF \
+    -DCOMPILER_RT_BUILD_PROFILE=OFF \
+    -DCOMPILER_RT_BUILD_CTX_PROFILE=OFF \
+    -DCOMPILER_RT_BUILD_SANITIZERS=OFF \
+    -DCOMPILER_RT_BUILD_XRAY=OFF \
+    -DCOMPILER_RT_BUILD_ORC=OFF \
+    -DCOMPILER_RT_BUILD_CRT=OFF \
+    -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \
+    -DCOMPILER_RT_EMULATOR="qemu-arm -L ${A_PROFILE_SYSROOT}" \
+    -DCOMPILER_RT_INCLUDE_TESTS=ON \
+    -DCOMPILER_RT_TEST_COMPILER=${LLVM_TOOLCHAIN}/bin/clang \
+    -DCOMPILER_RT_TEST_COMPILER_CFLAGS="--target=${A_PROFILE_TARGET_TRIPLE} -march=armv7-a --gcc-toolchain=${A_PROFILE_GCC_TOOLCHAIN} --sysroot=${A_PROFILE_SYSROOT} -fuse-ld=lld" \
+    -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY \
+    -DCOMPILER_RT_OS_DIR="baremetal" \
+    -DCOMPILER_RT_BAREMETAL_BUILD=ON
+
+.. note::
+  The sysroot used for compiling the tests is ``arm-linux-gnueabihf``, not
+  ``arm-none-eabi`` which is used when compiling the builtins.
 
 The Armv6-M builtins will use the soft-float ABI. When compiling the tests for
 Armv7-A we must include ``"-mthumb -mfloat-abi=soft -mfpu=none"`` in the
@@ -270,19 +328,3 @@ mismatches between the M-profile objects from compiler-rt and the A-profile
 objects from the test. The lld linker does not check the profile
 BuildAttribute so it can be used to link the tests by adding ``-fuse-ld=lld`` to the
 ``COMPILER_RT_TEST_COMPILER_CFLAGS``.
-
-Alternative using a cmake cache
--------------------------------
-If you wish to build, but not test compiler-rt for Armv6-M, Armv7-M or Armv7E-M
-the easiest way is to use the ``BaremetalARM.cmake`` recipe in ``clang/cmake/caches``.
-
-You will need a bare metal sysroot such as that provided by the GNU ARM Embedded
-toolchain.
-
-The libraries can be built with the cmake options::
-
- -DBAREMETAL_ARMV6M_SYSROOT=/path/to/bare/metal/toolchain/arm-none-eabi \
- -DBAREMETAL_ARMV7M_SYSROOT=/path/to/bare/metal/toolchain/arm-none-eabi \
- -DBAREMETAL_ARMV7EM_SYSROOT=/path/to/bare/metal/toolchain/arm-none-eabi \
- -C /path/to/llvm/source/tools/clang/cmake/caches/BaremetalARM.cmake \
- /path/to/llvm
diff --git a/llvm/docs/HowToReleaseLLVM.rst b/llvm/docs/HowToReleaseLLVM.rst
index dd4bb08b81d1c..f3792e367de0d 100644
--- a/llvm/docs/HowToReleaseLLVM.rst
+++ b/llvm/docs/HowToReleaseLLVM.rst
@@ -101,8 +101,8 @@ release process to begin.  Specifically, it involves:
 
 * Tagging release candidates for the release team to begin testing.
 
-Create Release Branch
-^^^^^^^^^^^^^^^^^^^^^
+Create Release Branch and Update LLVM Version
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Branch the Git trunk using the following procedure:
 
@@ -114,14 +114,16 @@ Branch the Git trunk using the following procedure:
 #. Verify that the current git trunk is in decent shape by
    examining nightly tester and buildbot results.
 
-#. Bump the version in trunk to N.0.0git and tag the commit with llvmorg-N-init.
+#. Bump the version in trunk to N.0.0git with the script in
+   ``llvm/utils/release/bump-version.py``, and tag the commit with llvmorg-N-init.
    If ``X`` is the version to be released, then ``N`` is ``X + 1``.
 
 ::
 
   $ git tag -sa llvmorg-N-init
 
-#. Clear the release notes in trunk.
+4. Clear the release notes in trunk with the script in
+   ``llvm/utils/release/clear-release-notes.py``.
 
 #. Create the release branch from the last known good revision from before the
    version bump.  The branch's name is release/X.x where ``X`` is the major version
@@ -133,12 +135,6 @@ Branch the Git trunk using the following procedure:
 #. All tags and branches need to be created in both the llvm/llvm-project and
    llvm/llvm-test-suite repos.
 
-Update LLVM Version
-^^^^^^^^^^^^^^^^^^^
-
-After creating the LLVM release branch, update the release branches'
-version with the script in ``llvm/utils/release/bump-version.py``.
-
 Tagging the LLVM Release Candidates
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 3a3a74f323a26..b122fd352eaf5 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -392,7 +392,7 @@ added in the future:
     sequence in place of a call site. This convention forces the call
     arguments into registers but allows them to be dynamically
     allocated. This can currently only be used with calls to
-    llvm.experimental.patchpoint because only this intrinsic records
+    ``llvm.experimental.patchpoint`` because only this intrinsic records
     the location of its arguments in a side table. See :doc:`StackMaps`.
 "``preserve_mostcc``" - The `PreserveMost` calling convention
     This calling convention attempts to make the code in the caller as
@@ -413,7 +413,9 @@ added in the future:
     - On AArch64 the callee preserves all general purpose registers, except
       X0-X8 and X16-X18. Not allowed with ``nest``.
 
-    - On RISC-V the callee preserve x5-x31 except x6, x7 and x28 registers.
+    - On RISC-V the callee preserves x5-x31 except x6, x7 and x28 registers.
+
+    - On LoongArch the callee preserves r4-r31 except r12-r15 and r20-r21 registers.
 
     The idea behind this convention is to support calls to runtime functions
     that have a hot path and a cold path. The hot path is usually a small piece
@@ -575,7 +577,7 @@ DLL storage classes:
     and the function or variable name. On XCOFF targets, ``dllexport`` indicates
     that the symbol will be made visible to other modules using "exported"
     visibility and thus placed by the linker in the loader section symbol table.
-    Since this storage class exists for defining a dll interface, the compiler,
+    Since this storage class exists for defining a DLL interface, the compiler,
     assembler and linker know it is externally referenced and must refrain from
     deleting the symbol.
 
@@ -610,7 +612,7 @@ model is not supported, or if a better choice of model can be made.
 A model can also be specified in an alias, but then it only governs how
 the alias is accessed. It will not have any effect on the aliasee.
 
-For platforms without linker support of ELF TLS model, the -femulated-tls
+For platforms without linker support of ELF TLS model, the ``-femulated-tls``
 flag can be used to generate GCC-compatible emulated TLS code.
 
 .. _runtime_preemption_model:
@@ -1887,7 +1889,7 @@ Attribute Groups
 
 Attribute groups are groups of attributes that are referenced by objects within
 the IR. They are important for keeping ``.ll`` files readable, because a lot of
-functions will use the same set of attributes. In the degenerative case of a
+functions will use the same set of attributes. In the degenerate case of a
 ``.ll`` file that corresponds to a single ``.c`` file, the single attribute
 group will capture the important command line flags used to build that file.
 
@@ -1946,8 +1948,8 @@ For example:
     ``::operator::delete``. Matching malloc/realloc/free calls within a family
     can be optimized, but mismatched ones will be left alone.
 ``allockind("KIND")``
-    Describes the behavior of an allocation function. The KIND string contains comma
-    separated entries from the following options:
+    Describes the behavior of an allocation function. The KIND string contains
+    comma-separated entries from the following options:
 
     * "alloc": the function returns a new block of memory or null.
     * "realloc": the function returns a new block of memory or null. If the
@@ -2047,7 +2049,7 @@ For example:
     even if this attribute says the frame pointer can be eliminated.
     The allowed string values are:
 
-     * ``"none"`` (default) - the frame pointer can be eliminated, and it's
+     * ``"none"`` (default) - the frame pointer can be eliminated, and its
        register can be used for other purposes.
      * ``"reserved"`` - the frame pointer register must either be updated to
        point to a valid frame record for the current function, or not be
@@ -2201,7 +2203,7 @@ For example:
 
     A ``nofree`` function is explicitly allowed to free memory which it
     allocated or (if not ``nosync``) arrange for another thread to free
-    memory on it's behalf.  As a result, perhaps surprisingly, a ``nofree``
+    memory on its behalf.  As a result, perhaps surprisingly, a ``nofree``
     function can return a pointer to a previously deallocated
     :ref:`allocated object<allocatedobjects>`.
 ``noimplicitfloat``
@@ -2232,14 +2234,14 @@ For example:
     may make calls to the function faster, at the cost of extra program
     startup time if the function is not called during program startup.
 ``noprofile``
-    This function attribute prevents instrumentation based profiling, used for
+    This function attribute prevents instrumentation-based profiling, used for
     coverage or profile based optimization, from being added to a function. It
     also blocks inlining if the caller and callee have different values of this
     attribute.
 ``skipprofile``
-    This function attribute prevents instrumentation based profiling, used for
+    This function attribute prevents instrumentation-based profiling, used for
     coverage or profile based optimization, from being added to a function. This
-    attribute does not restrict inlining, so instrumented instruction could end
+    attribute does not restrict inlining, so instrumented instructions could end
     up in this function.
 ``noredzone``
     This attribute indicates that the code generator should not use a
@@ -2339,7 +2341,7 @@ For example:
 
      * ``"prologue-short-redirect"`` - This style of patchable
        function is intended to support patching a function prologue to
-       redirect control away from the function in a thread safe
+       redirect control away from the function in a thread-safe
        manner.  It guarantees that the first instruction of the
        function will be large enough to accommodate a short jump
        instruction, and will be sufficiently aligned to allow being
@@ -2584,7 +2586,7 @@ For example:
 ``uwtable[(sync|async)]``
     This attribute indicates that the ABI being targeted requires that
     an unwind table entry be produced for this function even if we can
-    show that no exceptions passes by it. This is normally the case for
+    show that no exceptions pass by it. This is normally the case for
     the ELF x86-64 abi, but it can be disabled for some compilation
     units. The optional parameter describes what kind of unwind tables
     to generate: ``sync`` for normal unwind tables, ``async`` for asynchronous
@@ -2599,7 +2601,7 @@ For example:
 ``shadowcallstack``
     This attribute indicates that the ShadowCallStack checks are enabled for
     the function. The instrumentation checks that the return address for the
-    function has not changed between the function prolog and epilog. It is
+    function has not changed between the function prologue and epilogue. It is
     currently x86_64-specific.
 
 .. _langref_mustprogress:
@@ -2807,7 +2809,7 @@ operand bundle tag.  These operand bundles represent an alternate
 "safe" continuation for the call site they're attached to, and can be
 used by a suitable runtime to deoptimize the compiled frame at the
 specified call site.  There can be at most one ``"deopt"`` operand
-bundle attached to a call site.  Exact details of deoptimization is
+bundle attached to a call site.  Exact details of deoptimization are
 out of scope for the language reference, but it usually involves
 rewriting a compiled frame into a set of interpreted frames.
 
@@ -2896,7 +2898,7 @@ generated code.  For more details, see :ref:`GC Transitions
 
 The bundle contains an arbitrary list of Values which need to be passed
 to GC transition code. They will be lowered and passed as operands to
-the appropriate GC_TRANSITION nodes in the selection DAG. It is assumed
+the appropriate ``GC_TRANSITION`` nodes in the selection DAG. It is assumed
 that these arguments must be available before and after (but not
 necessarily during) the execution of the callee.
 
@@ -3334,7 +3336,7 @@ by the minus sign character ('-'). The canonical forms are:
 
 This information is passed along to the backend so that it generates
 code for the proper architecture. It's possible to override this on the
-command line with the ``-mtriple`` command line option.
+command line with the ``-mtriple`` command-line option.
 
 
 .. _allocatedobjects:
@@ -3641,8 +3643,8 @@ to support the somewhat common pattern in C of intentionally storing to an
 invalid pointer to crash the program. In the future, it might make sense to
 allow frontends to control this behavior.
 
-IR-level volatile loads and stores cannot safely be optimized into llvm.memcpy
-or llvm.memmove intrinsics even when those intrinsics are flagged volatile.
+IR-level volatile loads and stores cannot safely be optimized into ``llvm.memcpy``
+or ``llvm.memmove`` intrinsics even when those intrinsics are flagged volatile.
 Likewise, the backend should never split or merge target-legal volatile
 load/store instructions. Similarly, IR-level volatile loads and stores cannot
 change from integer to floating-point or vice versa.
@@ -4289,7 +4291,7 @@ X86_amx Type
 :Overview:
 
 The x86_amx type represents a value held in an AMX tile register on an x86
-machine. The operations allowed on it are quite limited. Only few intrinsics
+machine. The operations allowed on it are quite limited. Only a few intrinsics
 are allowed: stride load and store, zero and dot product. No instruction is
 allowed for this type. There are no arguments, arrays, pointers, vectors
 or constants of this type.
@@ -5058,14 +5060,14 @@ Addresses of Basic Blocks
 The '``blockaddress``' constant computes the address of the specified
 basic block in the specified function.
 
-It always has an ``ptr addrspace(P)`` type, where ``P`` is the address space
+It always has a ``ptr addrspace(P)`` type, where ``P`` is the address space
 of the function containing ``%block`` (usually ``addrspace(0)``).
 
 Taking the address of the entry block is illegal.
 
 This value only has defined behavior when used as an operand to the
 ':ref:`indirectbr <i_indirectbr>`' or for comparisons against null. Pointer
-equality tests between labels addresses results in undefined behavior ---
+equality tests between label addresses results in undefined behavior ---
 though, again, comparison against null is ok, and no label is equal to the null
 pointer. This may be passed around as an opaque pointer sized value as long as
 the bits are not inspected. This allows ``ptrtoint`` and arithmetic to be
@@ -5098,7 +5100,7 @@ The target function may not have ``extern_weak`` linkage.
   to the function.
 - ``dso_local_equivalent`` can be implemented with a stub that tail-calls the
   function. Many targets support relocations that resolve at link time to either
-  a function or a stub for it, depending on if the function is defined within the
+  a function or a stub for it, depending on whether the function is defined within the
   linkage unit; LLVM will use this when available. (This is commonly called a
   "PLT stub".) On other targets, the stub may need to be emitted explicitly.
 
@@ -5175,6 +5177,8 @@ The following is the syntax for constant expressions:
     Perform the :ref:`trunc operation <i_trunc>` on constants.
 ``ptrtoint (CST to TYPE)``
     Perform the :ref:`ptrtoint operation <i_ptrtoint>` on constants.
+``ptrtoaddr (CST to TYPE)``
+    Perform the :ref:`ptrtoaddr operation <i_ptrtoaddr>` on constants.
 ``inttoptr (CST to TYPE)``
     Perform the :ref:`inttoptr operation <i_inttoptr>` on constants.
     This one is *really* dangerous!
@@ -5318,7 +5322,7 @@ the '``unwind``' keyword, the behavior is undefined.
 
 If multiple keywords appear, the '``sideeffect``' keyword must come
 first, the '``alignstack``' keyword second, the '``inteldialect``' keyword
-third and the '``unwind``' keyword last.
+third, and the '``unwind``' keyword last.
 
 Inline Asm Constraint String
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -5481,7 +5485,7 @@ followed by two letters (e.g. "``^wc``"), or "``{``" register-name "``}``"
 The one and two letter constraint codes are typically chosen to be the same as
 GCC's constraint codes.
 
-A single constraint may include one or more than constraint code in it, leaving
+A single constraint may include one or more constraint codes in it, leaving
 it up to LLVM to choose which one to use. This is included mainly for
 compatibility with the translation of GCC inline asm coming from clang.
 
@@ -5580,9 +5584,13 @@ AArch64:
 AMDGPU:
 
 - ``r``: A 32 or 64-bit integer register.
-- ``[0-9]v``: The 32-bit VGPR register, number 0-9.
-- ``[0-9]s``: The 32-bit SGPR register, number 0-9.
-- ``[0-9]a``: The 32-bit AGPR register, number 0-9.
+- ``s``: SGPR register or tuple
+- ``v``: VGPR register or tuple
+- ``a``: AGPR register or tuple. Only valid on gfx908+.
+- ``VA``: VGPR or AGPR register or tuple. Only valid on gfx90a+.
+- ``v[0-9]``: The 32-bit VGPR register, number 0-9.
+- ``s[0-9]``: The 32-bit SGPR register, number 0-9.
+- ``a[0-9]``: The 32-bit AGPR register, number 0-9.
 - ``I``: An integer inline constant in the range from -16 to 64.
 - ``J``: A 16-bit signed integer constant.
 - ``A``: An integer or a floating-point inline constant.
@@ -6022,7 +6030,7 @@ Inline Asm Metadata
 The call instructions that wrap inline asm nodes may have a
 "``!srcloc``" MDNode attached to it that contains a list of constant
 integers. If present, the code generator will use the integer as the
-location cookie value when report errors through the ``LLVMContext``
+location cookie value when reporting errors through the ``LLVMContext``
 error reporting mechanisms. This allows a front-end to correlate backend
 errors that occur with inline asm back to the source code that produced
 it. For example:
@@ -6203,7 +6211,7 @@ Unlike instructions, global objects (functions and global variables) may have
 multiple metadata attachments with the same identifier.
 
 A transformation is required to drop any metadata attachment that it
-does not know or know it can't preserve. Currently there is an
+does not recognize or cannot preserve. Currently there is an
 exception for metadata attachment to globals for ``!func_sanitize``,
 ``!type``, ``!absolute_symbol`` and ``!associated`` which can't be
 unconditionally dropped unless the global is itself deleted.
@@ -6442,19 +6450,19 @@ descriptors <DISubrange>` or :ref:`subrange descriptors
 <DISubrangeType>`, each representing the range of subscripts at that
 level of indexing. The ``DIFlagVector`` flag to ``flags:`` indicates
 that an array type is a native packed vector. The optional
-``dataLocation`` is a DIExpression that describes how to get from an
+``dataLocation`` is a ``DIExpression`` that describes how to get from an
 object's address to the actual raw data, if they aren't
 equivalent. This is only supported for array types, particularly to
 describe Fortran arrays, which have an array descriptor in addition to
-the array data. Alternatively it can also be DIVariable which has the
+the array data. Alternatively it can also be ``DIVariable`` which has the
 address of the actual raw data. The Fortran language supports pointer
 arrays which can be attached to actual arrays, this attachment between
 pointer and pointee is called association.  The optional
-``associated`` is a DIExpression that describes whether the pointer
+``associated`` is a ``DIExpression`` that describes whether the pointer
 array is currently associated.  The optional ``allocated`` is a
-DIExpression that describes whether the allocatable array is currently
-allocated.  The optional ``rank`` is a DIExpression that describes the
-rank (number of dimensions) of fortran assumed rank array (rank is
+``DIExpression`` that describes whether the allocatable array is currently
+allocated.  The optional ``rank`` is a ``DIExpression`` that describes the
+rank (number of dimensions) of Fortran assumed rank array (rank is
 known at runtime).  The optional ``bitStride`` is an unsigned constant
 that describes the number of bits occupied by an element of the array;
 this is only needed if it differs from the element type's natural
@@ -6746,161 +6754,23 @@ parameter, and it will be included in the ``retainedNodes:`` field of its
                           type: !3)
     !2 = !DILocalVariable(name: "y", scope: !5, file: !2, line: 7, type: !3)
 
-.. _DIExpression:
-
 DIExpression
 """"""""""""
 
 ``DIExpression`` nodes represent expressions that are inspired by the DWARF
-expression language. They are used in :ref:`debug records <debugrecords>`
-(such as ``#dbg_declare`` and ``#dbg_value``) to describe how the
-referenced LLVM variable relates to the source language variable. Debug
-expressions are interpreted left-to-right: start by pushing the value/address
-operand of the record onto a stack, then repeatedly push and evaluate
-opcodes from the DIExpression until the final variable description is produced.
-
-The current supported opcode vocabulary is limited:
-
-- ``DW_OP_deref`` dereferences the top of the expression stack.
-- ``DW_OP_plus`` pops the last two entries from the expression stack, adds
-  them together and appends the result to the expression stack.
-- ``DW_OP_minus`` pops the last two entries from the expression stack, subtracts
-  the last entry from the second last entry and appends the result to the
-  expression stack.
-- ``DW_OP_plus_uconst, 93`` adds ``93`` to the working expression.
-- ``DW_OP_LLVM_fragment, 16, 8`` specifies the offset and size (``16`` and ``8``
-  here, respectively) of the variable fragment from the working expression. Note
-  that contrary to DW_OP_bit_piece, the offset is describing the location
-  within the described source variable.
-- ``DW_OP_LLVM_convert, 16, DW_ATE_signed`` specifies a bit size and encoding
-  (``16`` and ``DW_ATE_signed`` here, respectively) to which the top of the
-  expression stack is to be converted. Maps into a ``DW_OP_convert`` operation
-  that references a base type constructed from the supplied values.
-- ``DW_OP_LLVM_extract_bits_sext, 16, 8,`` specifies the offset and size
-  (``16`` and ``8`` here, respectively) of bits that are to be extracted and
-  sign-extended from the value at the top of the expression stack. If the top of
-  the expression stack is a memory location then these bits are extracted from
-  the value pointed to by that memory location. Maps into a ``DW_OP_shl``
-  followed by ``DW_OP_shra``.
-- ``DW_OP_LLVM_extract_bits_zext`` behaves similarly to
-  ``DW_OP_LLVM_extract_bits_sext``, but zero-extends instead of sign-extending.
-  Maps into a ``DW_OP_shl`` followed by ``DW_OP_shr``.
-- ``DW_OP_LLVM_tag_offset, tag_offset`` specifies that a memory tag should be
-  optionally applied to the pointer. The memory tag is derived from the
-  given tag offset in an implementation-defined manner.
-- ``DW_OP_swap`` swaps top two stack entries.
-- ``DW_OP_xderef`` provides extended dereference mechanism. The entry at the top
-  of the stack is treated as an address. The second stack entry is treated as an
-  address space identifier.
-- ``DW_OP_stack_value`` marks a constant value.
-- ``DW_OP_LLVM_entry_value, N`` refers to the value a register had upon
-  function entry. When targeting DWARF, a ``DBG_VALUE(reg, ...,
-  DIExpression(DW_OP_LLVM_entry_value, 1, ...)`` is lowered to
-  ``DW_OP_entry_value [reg], ...``, which pushes the value ``reg`` had upon
-  function entry onto the DWARF expression stack.
-
-  The next ``(N - 1)`` operations will be part of the ``DW_OP_entry_value``
-  block argument. For example, ``!DIExpression(DW_OP_LLVM_entry_value, 1,
-  DW_OP_plus_uconst, 123, DW_OP_stack_value)`` specifies an expression where
-  the entry value of ``reg`` is pushed onto the stack, and is added with 123.
-  Due to framework limitations ``N`` must be 1, in other words,
-  ``DW_OP_entry_value`` always refers to the value/address operand of the
-  instruction.
-
-  Because ``DW_OP_LLVM_entry_value`` is defined in terms of registers, it is
-  usually used in MIR, but it is also allowed in LLVM IR when targeting a
-  :ref:`swiftasync <swiftasync>` argument. The operation is introduced by:
-
-    - ``LiveDebugValues`` pass, which applies it to function parameters that
-      are unmodified throughout the function. Support is limited to simple
-      register location descriptions, or as indirect locations (e.g.,
-      parameters passed-by-value to a callee via a pointer to a temporary copy
-      made in the caller).
-    - ``AsmPrinter`` pass when a call site parameter value
-      (``DW_AT_call_site_parameter_value``) is represented as entry value of
-      the parameter.
-    - ``CoroSplit`` pass, which may move variables from allocas into a
-      coroutine frame. If the coroutine frame is a
-      :ref:`swiftasync <swiftasync>` argument, the variable is described with
-      an ``DW_OP_LLVM_entry_value`` operation.
-
-- ``DW_OP_LLVM_arg, N`` is used in debug intrinsics that refer to more than one
-  value, such as one that calculates the sum of two registers. This is always
-  used in combination with an ordered list of values, such that
-  ``DW_OP_LLVM_arg, N`` refers to the ``N``\ :sup:`th` element in that list. For
-  example, ``!DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1, DW_OP_minus,
-  DW_OP_stack_value)`` used with the list ``(%reg1, %reg2)`` would evaluate to
-  ``%reg1 - reg2``. This list of values should be provided by the containing
-  intrinsic/instruction.
-- ``DW_OP_breg`` (or ``DW_OP_bregx``) represents a content on the provided
-  signed offset of the specified register. The opcode is only generated by the
-  ``AsmPrinter`` pass to describe call site parameter value which requires an
-  expression over two registers.
-- ``DW_OP_push_object_address`` pushes the address of the object which can then
-  serve as a descriptor in subsequent calculation. This opcode can be used to
-  calculate bounds of fortran allocatable array which has array descriptors.
-- ``DW_OP_over`` duplicates the entry currently second in the stack at the top
-  of the stack. This opcode can be used to calculate bounds of fortran assumed
-  rank array which has rank known at run time and current dimension number is
-  implicitly first element of the stack.
-- ``DW_OP_LLVM_implicit_pointer`` It specifies the dereferenced value. It can
-  be used to represent pointer variables which are optimized out but the value
-  it points to is known. This operator is required as it is different than DWARF
-  operator DW_OP_implicit_pointer in representation and specification (number
-  and types of operands) and later can not be used as multiple level.
-
-.. code-block:: text
+expression language. They are used in :ref:`debug records <debug_records>`
+(such as ``#dbg_declare`` and ``#dbg_value``) to describe how the referenced
+LLVM variable relates to the source language variable.
 
-    IR for "*ptr = 4;"
-    --------------
-      #dbg_value(i32 4, !17, !DIExpression(DW_OP_LLVM_implicit_pointer), !20)
-    !17 = !DILocalVariable(name: "ptr1", scope: !12, file: !3, line: 5,
-                           type: !18)
-    !18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64)
-    !19 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-    !20 = !DILocation(line: 10, scope: !12)
-
-    IR for "**ptr = 4;"
-    --------------
-      #dbg_value(i32 4, !17,
-        !DIExpression(DW_OP_LLVM_implicit_pointer, DW_OP_LLVM_implicit_pointer),
-        !21)
-    !17 = !DILocalVariable(name: "ptr1", scope: !12, file: !3, line: 5,
-                           type: !18)
-    !18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64)
-    !19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !20, size: 64)
-    !20 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-    !21 = !DILocation(line: 10, scope: !12)
-
-DWARF specifies three kinds of simple location descriptions: Register, memory,
-and implicit location descriptions.  Note that a location description is
-defined over certain ranges of a program, i.e the location of a variable may
-change over the course of the program. Register and memory location
-descriptions describe the *concrete location* of a source variable (in the
-sense that a debugger might modify its value), whereas *implicit locations*
-describe merely the actual *value* of a source variable which might not exist
-in registers or in memory (see ``DW_OP_stack_value``).
-
-A ``#dbg_declare`` record describes an indirect value (the address) of a
-source variable. The first operand of the record must be an address of some
-kind. A DIExpression operand to the record refines this address to produce a
-concrete location for the source variable.
-
-A ``#dbg_value`` record describes the direct value of a source variable.
-The first operand of the record may be a direct or indirect value. A
-DIExpression operand to the record refines the first operand to produce a
-direct value. For example, if the first operand is an indirect value, it may be
-necessary to insert ``DW_OP_deref`` into the DIExpression in order to produce a
-valid debug record.
+See :ref:`diexpression` for details.
 
 .. note::
 
-   A DIExpression is interpreted in the same way regardless of which kind of
-   debug record it's attached to.
-
-   DIExpressions are always printed and parsed inline; they can never be
+   ``DIExpression``\s are always printed and parsed inline; they can never be
    referenced by an ID (e.g. ``!1``).
 
+Some examples of expressions:
+
 .. code-block:: text
 
     !DIExpression(DW_OP_deref)
@@ -6938,7 +6808,7 @@ DIArgList
 ``DIArgList`` nodes hold a list of constant or SSA value references. These are
 used in :ref:`debug records <debugrecords>` in combination with a
 ``DIExpression`` that uses the
-``DW_OP_LLVM_arg`` operator. Because a DIArgList may refer to local values
+``DW_OP_LLVM_arg`` operator. Because a ``DIArgList`` may refer to local values
 within a function, it must only be used as a function argument, must always be
 inlined, and cannot appear in named metadata.
 
@@ -6956,7 +6826,7 @@ These flags encode various properties of DINodes.
 
 The `ExportSymbols` flag marks a class, struct or union whose members
 may be referenced as if they were defined in the containing class or
-union. This flag is used to decide whether the DW_AT_export_symbols can
+union. This flag is used to decide whether the ``DW_AT_export_symbols`` can
 be used for the structure type.
 
 DIObjCProperty
@@ -7441,7 +7311,7 @@ For example, in the code below, the call instruction may only target the
 
 ``callback`` metadata may be attached to a function declaration, or definition.
 (Call sites are excluded only due to the lack of a use case.) For ease of
-exposition, we'll refer to the function annotated w/ metadata as a broker
+exposition, we'll refer to the function annotated with metadata as a broker
 function. The metadata describes how the arguments of a call to the broker are
 in turn passed to the callback function specified by the metadata. Thus, the
 ``callback`` metadata provides a partial description of a call site inside the
@@ -7533,7 +7403,7 @@ sections that the user does not want removed after linking.
 
 ``unpredictable`` metadata may be attached to any branch or switch
 instruction. It can be used to express the unpredictability of control
-flow. Similar to the llvm.expect intrinsic, it may be used to alter
+flow. Similar to the ``llvm.expect`` intrinsic, it may be used to alter
 optimizations related to compare and branch instructions. The metadata
 is treated as a boolean value; if it exists, it signals that the branch
 or switch that it is attached to is completely unpredictable.
@@ -7610,7 +7480,7 @@ loop is transformed to a different loop before an explicitly requested
 other transformations impossible. Mandatory loop canonicalizations such
 as loop rotation are still applied.
 
-It is recommended to use this metadata in addition to any llvm.loop.*
+It is recommended to use this metadata in addition to any ``llvm.loop.*``
 transformation directive. Also, any loop should have at most one
 directive applied to it (and a sequence of transformations built using
 followup-attributes). Otherwise, which transformation will be applied
@@ -7956,10 +7826,17 @@ the non-distributed fallback version will have. See
 '``llvm.loop.distribute.followup_all``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The attributes in this metadata is added to all followup loops of the
+The attributes in this metadata are added to all followup loops of the
 loop distribution pass. See
 :ref:`Transformation Metadata <transformation-metadata>` for details.
 
+'``llvm.loop.isdistributed``' Metadata
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If a loop was successfully processed by the loop distribution pass,
+this metadata is added (i.e. has been distributed).  See
+:ref:`Transformation Metadata <transformation-metadata>` for details.
+
 '``llvm.licm.disable``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -7971,7 +7848,7 @@ performed on this loop. The metadata has a single operand which is the string
 
    !0 = !{!"llvm.licm.disable"}
 
-Note that although it operates per loop it isn't given the llvm.loop prefix
+Note that although it operates per loop it isn't given the ``llvm.loop`` prefix
 as it is not affected by the ``llvm.loop.disable_nonforced`` metadata.
 
 '``llvm.access.group``' Metadata
@@ -8035,8 +7912,8 @@ undefined.
 Note that if not all memory access instructions belong to an access
 group referred to by ``llvm.loop.parallel_accesses``, then the loop must
 not be considered trivially parallel. Additional
-memory dependence analysis is required to make that determination. As a fail
-safe mechanism, this causes loops that were originally parallel to be considered
+memory dependence analysis is required to make that determination. As a
+fail-safe mechanism, this causes loops that were originally parallel to be considered
 sequential (if optimization passes that are unaware of the parallel semantics
 insert new memory instructions into the loop body).
 
@@ -8168,8 +8045,8 @@ Examples:
 
    !0 = !{}
 
-The invariant.group metadata must be dropped when replacing one pointer by
-another based on aliasing information. This is because invariant.group is tied
+The ``invariant.group`` metadata must be dropped when replacing one pointer by
+another based on aliasing information. This is because ``invariant.group`` is tied
 to the SSA value of the pointer operand.
 
 .. code-block:: llvm
@@ -8205,7 +8082,7 @@ compatibility, globals carrying this metadata should:
 - Be in ``@llvm.compiler.used``.
 - If the referenced global variable is in a comdat, be in the same comdat.
 
-``!associated`` can not express many-to-one relationship. A global variable with
+``!associated`` can not express a many-to-one relationship. A global variable with
 the metadata should generally not be referenced by a function: the function may
 be inlined into other functions, leading to more references to the metadata.
 Ideally we would want to keep metadata alive as long as any inline location is
@@ -8266,12 +8143,12 @@ VP
 
 VP (value profile) metadata can be attached to instructions that have
 value profile information. Currently this is indirect calls (where it
-records the hottest callees) and calls to memory intrinsics such as memcpy,
+records the hottest callees) and calls to memory intrinsics, such as memcpy,
 memmove, and memset (where it records the hottest byte lengths).
 
-Each VP metadata node contains "VP" string, then a uint32_t value for the value
-profiling kind, a uint64_t value for the total number of times the instruction
-is executed, followed by uint64_t value and execution count pairs.
+Each VP metadata node contains "VP" string, then a ``uint32_t`` value for the value
+profiling kind, a ``uint64_t`` value for the total number of times the instruction
+is executed, followed by ``uint64_t`` value and execution count pairs.
 The value profiling kind is 0 for indirect call targets and 1 for memory
 operations. For indirect call targets, each profile value is a hash
 of the callee function name, and for memory operations each value is the
@@ -8470,8 +8347,8 @@ Example:
 
 This is intended for use on targets with a notion of generic address
 spaces, which at runtime resolve to different physical memory
-spaces. The interpretation of the address space values is target
-specific. The behavior is undefined if the runtime memory address does
+spaces. The interpretation of the address space values is target specific.
+The behavior is undefined if the runtime memory address does
 resolve to an object defined in one of the indicated address spaces.
 
 
@@ -8482,7 +8359,7 @@ Information about the module as a whole is difficult to convey to LLVM's
 subsystems. The LLVM IR isn't sufficient to transmit this information.
 The ``llvm.module.flags`` named metadata exists in order to facilitate
 this. These flags are in the form of key / value pairs --- much like a
-dictionary --- making it easy for any subsystem who cares about a flag to
+dictionary --- making it easy for any subsystem that cares about a flag to
 look it up.
 
 The ``llvm.module.flags`` metadata contains a list of metadata triplets.
@@ -8742,7 +8619,7 @@ Automatic Linker Flags Named Metadata
 
 Some targets support embedding of flags to the linker inside individual object
 files. Typically this is used in conjunction with language extensions which
-allow source files to contain linker command line options, and have these
+allow source files to contain linker command-line options, and have these
 automatically be transmitted to the linker via object files.
 
 These flags are encoded in the IR using named metadata with the name
@@ -11733,7 +11610,7 @@ size of the '<value>' type. Note that this default alignment assumption is
 different from the alignment used for the load/store instructions when align
 isn't specified.
 
-A ``atomicrmw`` instruction can also take an optional
+An ``atomicrmw`` instruction can also take an optional
 ":ref:`syncscope <syncscope>`" argument.
 
 Semantics:
@@ -12504,7 +12381,7 @@ Semantics:
 """"""""""
 
 The '``ptrtoint``' instruction converts ``value`` to integer type
-``ty2`` by interpreting the all pointer representation bits as an integer
+``ty2`` by interpreting all the pointer representation bits as an integer
 (equivalent to a ``bitcast``) and either truncating or zero extending that value
 to the size of the integer type.
 If ``value`` is smaller than ``ty2`` then a zero extension is done. If
@@ -12523,6 +12400,59 @@ Example:
       %Y = ptrtoint ptr %P to i64                        ; yields zero extension on 32-bit architecture
       %Z = ptrtoint <4 x ptr> %P to <4 x i64>; yields vector zero extension for a vector of addresses on 32-bit architecture
 
+.. _i_ptrtoaddr:
+
+'``ptrtoaddr .. to``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      <result> = ptrtoaddr <ty> <value> to <ty2>             ; yields ty2
+
+Overview:
+"""""""""
+
+The '``ptrtoaddr``' instruction converts the pointer or a vector of
+pointers ``value`` to the underlying integer address (or vector of addresses) of
+type ``ty2``. This is different from :ref:`ptrtoint <i_ptrtoint>` in that it
+only operates on the index bits of the pointer and ignores all other bits, and
+does not capture the provenance of the pointer.
+
+Arguments:
+""""""""""
+
+The '``ptrtoaddr``' instruction takes a ``value`` to cast, which must be
+a value of type :ref:`pointer <t_pointer>` or a vector of pointers, and a
+type to cast it to ``ty2``, which must be must be the :ref:`integer <t_integer>`
+type (or vector of integers) matching the pointer index width of the address
+space of ``ty``.
+
+Semantics:
+""""""""""
+
+The '``ptrtoaddr``' instruction converts ``value`` to integer type ``ty2`` by
+interpreting the lowest index-width pointer representation bits as an integer.
+If the address size and the pointer representation size are the same and
+``value`` and ``ty2`` are the same size, then nothing is done (*no-op cast*)
+other than a type change.
+
+The ``ptrtoaddr`` instruction always :ref:`captures the address but not the provenance <pointercapture>`
+of the pointer argument.
+
+Example:
+""""""""
+This example assumes pointers in address space 1 are 64 bits in size with an
+address width of 32 bits (``p1:64:64:64:32`` :ref:`datalayout string<langref_datalayout>`)
+
+.. code-block:: llvm
+
+      %X = ptrtoaddr ptr addrspace(1) %P to i32              ; extracts low 32 bits of pointer
+      %Y = ptrtoaddr <4 x ptr addrspace(1)> %P to <4 x i32>  ; yields vector of low 32 bits for each pointer
+
+
 .. _i_inttoptr:
 
 '``inttoptr .. to``' Instruction
@@ -13483,7 +13413,7 @@ ensures that each ``catchpad`` has exactly one predecessor block, and it always
 terminates in a ``catchswitch``.
 
 The ``args`` correspond to whatever information the personality routine
-requires to know if this is an appropriate handler for the exception. Control
+requires to determine if this is an appropriate handler for the exception. Control
 will transfer to the ``catchpad`` if this is the first appropriate handler for
 the exception.
 
@@ -13827,7 +13757,7 @@ Semantics:
 The '``llvm.va_copy``' intrinsic works just like the ``va_copy`` macro
 available in C. In a target-dependent way, it copies the source
 ``va_list`` element into the destination ``va_list`` element. This
-intrinsic is necessary because the `` llvm.va_start`` intrinsic may be
+intrinsic is necessary because the ``llvm.va_start`` intrinsic may be
 arbitrarily complex and require, for example, memory allocation.
 
 Accurate Garbage Collection Intrinsics
@@ -14018,7 +13948,7 @@ types of the 'call parameters' arguments.
 
 The '#call args' operand is the number of arguments to the actual
 call.  It must exactly match the number of arguments passed in the
-'call parameters' variable length section.
+'call parameters' variable-length section.
 
 The 'flags' operand is used to specify extra information about the
 statepoint. This is currently only used to mark certain statepoints
@@ -14139,7 +14069,7 @@ so constructed.
 
 The third argument is an index which specify the (potentially) derived pointer
 being relocated.  It is legal for this index to be the same as the second
-argument if-and-only-if a base pointer is being relocated.
+argument if and only if a base pointer is being relocated.
 
 Semantics:
 """"""""""
@@ -14835,7 +14765,7 @@ Overview:
 """""""""
 
 The '``llvm.instrprof.increment``' intrinsic can be emitted by a
-frontend for use with instrumentation based profiling. These will be
+frontend for use with instrumentation-based profiling. These will be
 lowered by the ``-instrprof`` pass to generate execution counts of a
 program at runtime.
 
@@ -15038,7 +14968,7 @@ Overview:
 """""""""
 
 The '``llvm.instrprof.value.profile``' intrinsic can be emitted by a
-frontend for use with instrumentation based profiling. This will be
+frontend for use with instrumentation-based profiling. This will be
 lowered by the ``-instrprof`` pass to find out the target values,
 instrumented expressions take in a program at runtime.
 
@@ -15685,7 +15615,7 @@ external functions.
 Syntax:
 """""""
 
-This is an overloaded intrinsic. You can use llvm.memmove on any integer
+This is an overloaded intrinsic. You can use ``llvm.memmove`` on any integer
 bit width and for different address space. Not all targets support all
 bit widths however.
 
@@ -15746,7 +15676,7 @@ otherwise the behavior is undefined.
 Syntax:
 """""""
 
-This is an overloaded intrinsic. You can use llvm.memset on any integer
+This is an overloaded intrinsic. You can use ``llvm.memset`` on any integer
 bit width and for different address spaces. However, not all targets
 support all bit widths.
 
@@ -17935,7 +17865,7 @@ operate on a per-element basis and the element order is not affected.
 Syntax:
 """""""
 
-This is an overloaded intrinsic. You can use llvm.ctpop on any integer
+This is an overloaded intrinsic. You can use ``llvm.ctpop`` on any integer
 bit width, or on any vector with integer elements. Not all targets
 support all bit widths or vector types, however.
 
@@ -18455,7 +18385,7 @@ Overview:
 """""""""
 
 The '``llvm.umul.with.overflow``' family of intrinsic functions perform
-a unsigned multiplication of the two arguments, and indicate whether an
+an unsigned multiplication of the two arguments, and indicate whether an
 overflow occurred during the unsigned multiplication.
 
 Arguments:
@@ -20622,7 +20552,7 @@ Semantics:
 
 The '``llvm.experimental.vector.histogram.*``' intrinsics are used to perform
 updates on potentially overlapping values in memory. The intrinsics represent
-the follow sequence of operations:
+the following sequence of operations:
 
 1. Gather load from the ``ptrs`` operand, with element type matching that of
    the ``inc`` operand.
@@ -26355,7 +26285,7 @@ This is an overloaded intrinsic.
 Overview:
 """""""""
 
-Predicated llvm.is.fpclass :ref:`llvm.is.fpclass <llvm.is.fpclass>`
+Predicated ``llvm.is.fpclass`` :ref:`llvm.is.fpclass <llvm.is.fpclass>`
 
 Arguments:
 """"""""""
@@ -26370,7 +26300,7 @@ operation.
 Semantics:
 """"""""""
 
-The '``llvm.vp.is.fpclass``' intrinsic performs llvm.is.fpclass (:ref:`llvm.is.fpclass <llvm.is.fpclass>`).
+The '``llvm.vp.is.fpclass``' intrinsic performs ``llvm.is.fpclass`` (:ref:`llvm.is.fpclass <llvm.is.fpclass>`).
 
 
 Examples:
@@ -26730,7 +26660,7 @@ Syntax:
 
 ::
 
-      declare void @llvm.lifetime.start(i64 <size>, ptr captures(none) <ptr>)
+      declare void @llvm.lifetime.start(ptr captures(none) <ptr>)
 
 Overview:
 """""""""
@@ -26741,11 +26671,8 @@ object's lifetime.
 Arguments:
 """"""""""
 
-The first argument is a constant integer, which is ignored and will be removed
-in the future.
-
-The second argument is either a pointer to an ``alloca`` instruction or
-a ``poison`` value.
+The argument is either a pointer to an ``alloca`` instruction or a ``poison``
+value.
 
 Semantics:
 """"""""""
@@ -26774,7 +26701,7 @@ Syntax:
 
 ::
 
-      declare void @llvm.lifetime.end(i64 <size>, ptr captures(none) <ptr>)
+      declare void @llvm.lifetime.end(ptr captures(none) <ptr>)
 
 Overview:
 """""""""
@@ -26785,11 +26712,8 @@ The '``llvm.lifetime.end``' intrinsic specifies the end of a
 Arguments:
 """"""""""
 
-The first argument is a constant integer, which is ignored and will be removed
-in the future.
-
-The second argument is either a pointer to an ``alloca`` instruction or
-a ``poison`` value.
+The argument is either a pointer to an ``alloca`` instruction or a ``poison``
+value.
 
 Semantics:
 """"""""""
@@ -28440,7 +28364,7 @@ environment.  The rounding mode argument is only intended as information
 to the compiler.
 
 If the runtime floating-point environment is using the default rounding mode
-then the results will be the same as the llvm.lrint intrinsic.
+then the results will be the same as the ``llvm.lrint`` intrinsic.
 
 
 '``llvm.experimental.constrained.llrint``' Intrinsic
@@ -28488,7 +28412,7 @@ environment.  The rounding mode argument is only intended as information
 to the compiler.
 
 If the runtime floating-point environment is using the default rounding mode
-then the results will be the same as the llvm.llrint intrinsic.
+then the results will be the same as the ``llvm.llrint`` intrinsic.
 
 
 '``llvm.experimental.constrained.nearbyint``' Intrinsic
@@ -28949,7 +28873,7 @@ was only valid within a single iteration.
 
 .. code-block:: llvm
 
-  ; This examples shows two possible positions for noalias.decl and how they impact the semantics:
+  ; This example shows two possible positions for noalias.decl and how they impact the semantics:
   ; If it is outside the loop (Version 1), then %a and %b are noalias across *all* iterations.
   ; If it is inside the loop (Version 2), then %a and %b are noalias only within *one* iteration.
   declare void @decl_in_loop(ptr %a.base, ptr %b.base) {
@@ -30404,7 +30328,7 @@ has externally observable side effects.
 Syntax:
 """""""
 
-This is an overloaded intrinsic. You can use llvm.is.constant with any argument type.
+This is an overloaded intrinsic. You can use ``llvm.is.constant`` with any argument type.
 
 ::
 
diff --git a/llvm/docs/MIRLangRef.rst b/llvm/docs/MIRLangRef.rst
index a505c1ea4b0aa..3f4c3cde9b3aa 100644
--- a/llvm/docs/MIRLangRef.rst
+++ b/llvm/docs/MIRLangRef.rst
@@ -27,7 +27,7 @@ data serialization language, and the full YAML language spec can be read at
 `yaml.org
 <http://www.yaml.org/spec/1.2/spec.html#Introduction>`_.
 
-A MIR file is split up into a series of `YAML documents`_. The first document
+A MIR file is split into a series of `YAML documents`_. The first document
 can contain an optional embedded LLVM IR module, and the rest of the documents
 contain the serialized machine functions.
 
@@ -65,22 +65,22 @@ after the name with a comma.
 
    ``llc -stop-after=dead-mi-elimination,1 bug-trigger.ll -o test.mir``
 
-After generating the input MIR file, you'll have to add a run line that uses
+After generating the input MIR file, you'll have to add a ``RUN`` line that uses
 the ``-run-pass`` option to it. In order to test the post register allocation
 pseudo instruction expansion pass on X86-64, a run line like the one shown
 below can be used:
 
     ``# RUN: llc -o - %s -mtriple=x86_64-- -run-pass=postrapseudos | FileCheck %s``
 
-The MIR files are target dependent, so they have to be placed in the target
-specific test directories (``lib/CodeGen/TARGETNAME``). They also need to
-specify a target triple or a target architecture either in the run line or in
+The MIR files are target dependent, so they have to be placed in the
+target-specific test directories (``lib/CodeGen/TARGETNAME``). They also need to
+specify a target triple or a target architecture either in the ``RUN`` line or in
 the embedded LLVM IR module.
 
 Simplifying MIR files
 ^^^^^^^^^^^^^^^^^^^^^
 
-The MIR code coming out of ``-stop-after``/``-stop-before`` is very verbose;
+The MIR code coming out of ``-stop-after``/``-stop-before`` is very verbose.
 Tests are more accessible and future proof when simplified:
 
 - Use the ``-simplify-mir`` option with llc.
@@ -113,12 +113,12 @@ Tests are more accessible and future proof when simplified:
   If the test doesn't depend on (good) alias analysis the references can be
   dropped: `:: (load 8)`
 
-- MIR blocks can reference IR blocks for debug printing, profile information
+- MIR blocks can reference IR blocks for debug printing, profile information,
   or debug locations. Example: `bb.42.myblock` in MIR references the IR block
   `myblock`. It is usually possible to drop the `.myblock` reference and simply
   use `bb.42`.
 
-- If there are no memory operands or blocks referencing the IR then the
+- If there are no memory operands or blocks referencing the IR, then the
   IR function can be replaced by a parameterless dummy function like
   `define @func() { ret void }`.
 
@@ -143,7 +143,7 @@ can serialize:
 - The ``MCSymbol`` machine operands don't support temporary or local symbols.
 
 - A lot of the state in ``MachineModuleInfo`` isn't serialized - only the CFI
-  instructions and the variable debug information from MMI is serialized right
+  instructions and the variable debug information from MMI are serialized right
   now.
 
 These limitations impose restrictions on what you can test with the MIR format.
@@ -182,7 +182,7 @@ Machine Functions
 -----------------
 
 The remaining YAML documents contain the machine functions. This is an example
-of such YAML document:
+of such a YAML document:
 
 .. code-block:: text
 
@@ -299,7 +299,7 @@ instructions:
     bb.2.else:
       <instructions>
 
-The branch weights can be specified in brackets after the successor blocks.
+The branch weights can be specified in parentheses after the successor blocks.
 The example below defines a block that has two successors with branch weights
 of 32 and 16:
 
@@ -314,7 +314,7 @@ Live In Registers
 ^^^^^^^^^^^^^^^^^
 
 The machine basic block's live in registers have to be specified before any of
-the instructions:
+its instructions:
 
 .. code-block:: text
 
@@ -322,14 +322,14 @@ the instructions:
       liveins: $edi, $esi
 
 The list of live in registers and successors can be empty. The language also
-allows multiple live in register and successor lists - they are combined into
+allows multiple live in register and successor lists; they are combined into
 one list by the parser.
 
 Miscellaneous Attributes
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
 The attributes ``IsAddressTaken``, ``IsLandingPad``,
-``IsInlineAsmBrIndirectTarget`` and ``Alignment`` can be specified in brackets
+``IsInlineAsmBrIndirectTarget`` and ``Alignment`` can be specified in parentheses
 after the block's definition:
 
 .. code-block:: text
@@ -417,7 +417,7 @@ and ``}`` are bundled with the first instruction.
 Registers
 ---------
 
-Registers are one of the key primitives in the machine instructions
+Registers are one of the key primitives in the machine instruction
 serialization language. They are primarily used in the
 :ref:`register machine operands <register-operands>`,
 but they can also be used in a number of other places, like the
@@ -503,9 +503,9 @@ will be printed as ``%subreg.sub_32``:
 
     %1:gpr64 = SUBREG_TO_REG 0, %0, %subreg.sub_32
 
-For integers > 64bit, we use a special machine operand, ``MO_CImmediate``,
+For integers > 64 bits, we use a special machine operand, ``MO_CImmediate``,
 which stores the immediate in a ``ConstantInt`` using an ``APInt`` (LLVM's
-arbitrary precision integers).
+arbitrary-precision integers).
 
 .. TODO: Describe the FPIMM immediate operands.
 
@@ -626,7 +626,7 @@ For a CPI with the index 0 and offset -12:
     %1:gr64 = MOV64ri %const.0 - 12
 
 A constant pool entry is bound to a LLVM IR ``Constant`` or a target-specific
-``MachineConstantPoolValue``. When serializing all the function's constants the
+``MachineConstantPoolValue``. When serializing all the function's constants, the
 following format is used:
 
 .. code-block:: text
@@ -695,7 +695,7 @@ and the offset 8:
 Jump-table Index Operands
 ^^^^^^^^^^^^^^^^^^^^^^^^^
 
-A jump-table index operand with the index 0 is printed as following:
+A jump-table index operand with the index 0 is printed as follows:
 
 .. code-block:: text
 
@@ -711,7 +711,7 @@ A machine jump-table entry contains a list of ``MachineBasicBlocks``. When seria
         - id:             <index>
           blocks:         [ <bbreference>, <bbreference>, ... ]
 
-where ``<kind>`` is describing how the jump table is represented and emitted (plain address, relocations, PIC, etc.), and each ``<index>`` is a 32-bit unsigned integer and ``blocks`` contains a list of :ref:`machine basic block references <block-references>`.
+where ``<kind>`` describes how the jump table is represented and emitted (plain address, relocations, PIC, etc.), and each ``<index>`` is a 32-bit unsigned integer and ``blocks`` contains a list of :ref:`machine basic block references <block-references>`.
 
 Example:
 
@@ -741,7 +741,7 @@ Example:
 MCSymbol Operands
 ^^^^^^^^^^^^^^^^^
 
-A MCSymbol operand is holding a pointer to a ``MCSymbol``. For the limitations
+A MCSymbol operand holds a pointer to a ``MCSymbol``. For the limitations
 of this operand in MIR, see :ref:`limitations <limitations>`.
 
 The syntax is:
@@ -754,7 +754,7 @@ Debug Instruction Reference Operands
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 A debug instruction reference operand is a pair of indices, referring to an
-instruction and an operand within that instruction respectively; see
+instruction and an operand within that instruction, respectively; see
 :ref:`Instruction referencing locations <instruction-referencing-locations>`.
 
 The example below uses a reference to Instruction 1, Operand 0:
@@ -766,7 +766,7 @@ The example below uses a reference to Instruction 1, Operand 0:
 CFIIndex Operands
 ^^^^^^^^^^^^^^^^^
 
-A CFI Index operand is holding an index into a per-function side-table,
+A CFI Index operand holds an index into a per-function side-table,
 ``MachineFunction::getFrameInstructions()``, which references all the frame
 instructions in a ``MachineFunction``. A ``CFI_INSTRUCTION`` may look like it
 contains multiple operands, but the only operand it contains is the CFI Index.
@@ -842,7 +842,7 @@ Comments can be added or customized by overriding InstrInfo's hook
 Debug-Info constructs
 ---------------------
 
-Most of the debugging information in a MIR file is to be found in the metadata
+Most of the debugging information in a MIR file is found in the metadata
 of the embedded module. Within a machine function, that metadata is referred to
 by various constructs to describe source locations and variable locations.
 
diff --git a/llvm/docs/MergeFunctions.rst b/llvm/docs/MergeFunctions.rst
index 02344bca6f45c..c27f603aa97c5 100644
--- a/llvm/docs/MergeFunctions.rst
+++ b/llvm/docs/MergeFunctions.rst
@@ -7,7 +7,7 @@ MergeFunctions pass, how it works
 
 Introduction
 ============
-Sometimes code contains equal functions, or functions that does exactly the same
+Sometimes code contains equal functions, or functions that do exactly the same
 thing even though they are non-equal on the IR level (e.g.: multiplication on 2
 and 'shl 1'). It could happen due to several reasons: mainly, the usage of
 templates and automatic code generators. Though, sometimes the user itself could
@@ -16,7 +16,7 @@ write the same thing twice :-)
 The main purpose of this pass is to recognize such functions and merge them.
 
 This document is the extension to pass comments and describes the pass logic. It
-describes the algorithm that is used in order to compare functions and
+describes the algorithm used to compare functions and
 explains how we could combine equal functions correctly to keep the module
 valid.
 
@@ -58,7 +58,7 @@ It's especially important to understand chapter 3 of tutorial:
 
 :doc:`tutorial/LangImpl03`
 
-The reader should also know how passes work in LLVM. They could use this
+The reader should also know how passes work in LLVM. They can use this
 article as a reference and start point here:
 
 :doc:`WritingAnLLVMPass`
@@ -68,7 +68,7 @@ debugging and bug-fixing.
 
 Narrative structure
 -------------------
-The article consists of three parts. The first part explains pass functionality
+This article consists of three parts. The first part explains pass functionality
 on the top-level. The second part describes the comparison procedure itself.
 The third part describes the merging process.
 
@@ -130,7 +130,7 @@ access lookup? The answer is: "yes".
 
 Random-access
 """""""""""""
-How it could this be done? Just convert each function to a number, and gather
+How can this be done? Just convert each function to a number, and gather
 all of them in a special hash-table. Functions with equal hashes are equal.
 Good hashing means, that every function part must be taken into account. That
 means we have to convert every function part into some number, and then add it
@@ -190,17 +190,17 @@ The algorithm is pretty simple:
 
 1. Put all module's functions into the *worklist*.
 
-2. Scan *worklist*'s functions twice: first enumerate only strong functions and
+2. Scan *worklist*'s functions twice: first, enumerate only strong functions and
 then only weak ones:
 
    2.1. Loop body: take a function from *worklist*  (call it *FCur*) and try to
    insert it into *FnTree*: check whether *FCur* is equal to one of functions
    in *FnTree*. If there *is* an equal function in *FnTree*
-   (call it *FExists*): merge function *FCur* with *FExists*. Otherwise add
+   (call it *FExists*): merge function *FCur* with *FExists*. Otherwise, add
    the function from the *worklist* to *FnTree*.
 
 3. Once the *worklist* scanning and merging operations are complete, check the
-*Deferred* list. If it is not empty: refill the *worklist* contents with
+*Deferred* list. If it is not empty, refill the *worklist* contents with
 *Deferred* list and redo step 2, if the *Deferred* list is empty, then exit
 from method.
 
@@ -249,14 +249,14 @@ Below, we will use the following operations:
 
 The rest of the article is based on *MergeFunctions.cpp* source code
 (found in *<llvm_dir>/lib/Transforms/IPO/MergeFunctions.cpp*). We would like
-to ask reader to keep this file open, so we could use it as a reference
+to ask the reader to keep this file open, so we could use it as a reference
 for further explanations.
 
 Now, we're ready to proceed to the next chapter and see how it works.
 
 Functions comparison
 ====================
-At first, let's define how exactly we compare complex objects.
+First, let's define exactly how we compare complex objects.
 
 Complex object comparison (function, basic-block, etc) is mostly based on its
 sub-object comparison results. It is similar to the next "tree" objects
@@ -307,7 +307,7 @@ to those we met later in function body (value we met first would be *less*).
 This is done by “``FunctionComparator::cmpValues(const Value*, const Value*)``”
 method (will be described a bit later).
 
-4. Function body comparison. As it written in method comments:
+4. Function body comparison. As written in method comments:
 
 “We do a CFG-ordered walk since the actual ordering of the blocks in the linked
 list is immaterial. Our walk starts at the entry block for both functions, then
@@ -477,7 +477,7 @@ Of course, we can combine insertion and comparison:
     = sn_mapR.insert(std::make_pair(Right, sn_mapR.size()));
   return cmpNumbers(LeftRes.first->second, RightRes.first->second);
 
-Let's look, how whole method could be implemented.
+Let's look at how the whole method could be implemented.
 
 1. We have to start with the bad news. Consider function self and
 cross-referencing cases:
@@ -519,7 +519,7 @@ the result of numbers comparison:
    if (LeftRes.first->second < RightRes.first->second) return -1;
    return 1;
 
-Now when *cmpValues* returns 0, we can proceed the comparison procedure.
+Now, when *cmpValues* returns 0, we can proceed with the comparison procedure.
 Otherwise, if we get (-1 or 1), we need to pass this result to the top level,
 and finish comparison procedure.
 
@@ -549,7 +549,7 @@ losslessly bitcasted to each other. The further explanation is modification of
    2.1.3.1. If types are vectors, compare their bitwidth using the
    *cmpNumbers*. If result is not 0, return it.
 
-   2.1.3.2. Different types, but not a vectors:
+   2.1.3.2. Different types, but not vectors:
 
    * if both of them are pointers, good for us, we can proceed to step 3.
    * if one of types is pointer, return result of *isPointer* flags
@@ -654,7 +654,7 @@ O(N*N) to O(log(N)).
 
 Merging process, mergeTwoFunctions
 ==================================
-Once *MergeFunctions* detected that current function (*G*) is equal to one that
+Once *MergeFunctions* detects that current function (*G*) is equal to one that
 were analyzed before (function *F*) it calls ``mergeTwoFunctions(Function*,
 Function*)``.
 
@@ -664,7 +664,7 @@ Operation affects ``FnTree`` contents with next way: *F* will stay in
 functions that calls *G* would be put into ``Deferred`` set and removed from
 ``FnTree``, and analyzed again.
 
-The approach is next:
+The approach is as follows:
 
 1. Most wished case: when we can use alias and both of *F* and *G* are weak. We
 make both of them with aliases to the third strong function *H*. Actually *H*
@@ -691,12 +691,12 @@ ok: we can use alias to *F* instead of *G* or change call instructions itself.
 
 HasGlobalAliases, removeUsers
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-First consider the case when we have global aliases of one function name to
+First, consider the case when we have global aliases of one function name to
 another. Our purpose is  make both of them with aliases to the third strong
 function. Though if we keep *F* alive and without major changes we can leave it
 in ``FnTree``. Try to combine these two goals.
 
-Do stub replacement of *F* itself with an alias to *F*.
+Do a stub replacement of *F* itself with an alias to *F*.
 
 1. Create stub function *H*, with the same name and attributes like function
 *F*. It takes maximum alignment of *F* and *G*.
@@ -725,7 +725,7 @@ also have alias to *F*.
 
 No global aliases, replaceDirectCallers
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-If global aliases are not supported. We call ``replaceDirectCallers``. Just
+If global aliases are not supported, we call ``replaceDirectCallers``. Just
 go through all calls of *G* and replace it with calls of *F*. If you look into
 the method you will see that it scans all uses of *G* too, and if use is callee
 (if user is call instruction and *G* is used as what to be called), we replace
diff --git a/llvm/docs/NVPTXUsage.rst b/llvm/docs/NVPTXUsage.rst
index d28eb6860c33a..629bf2ea5afb4 100644
--- a/llvm/docs/NVPTXUsage.rst
+++ b/llvm/docs/NVPTXUsage.rst
@@ -92,6 +92,12 @@ Function Attributes
     dimension. Specifying a different cluster dimension at launch will result in
     a runtime error or kernel launch failure. Only supported for Hopper+.
 
+``"nvvm.blocksareclusters"``
+    This attribute implies that the grid launch configuration for the corresponding
+    kernel function is specifying the number of clusters instead of the number of thread
+    blocks. This attribute is only allowed for kernel functions and requires
+    ``nvvm.reqntid`` and ``nvvm.cluster_dim`` attributes.
+
 .. _address_spaces:
 
 Address Spaces
@@ -971,6 +977,10 @@ Syntax:
   declare void  @llvm.nvvm.prefetch.L1(ptr %ptr)
   declare void  @llvm.nvvm.prefetch.L2(ptr %ptr)
   
+  declare void  @llvm.nvvm.prefetch.tensormap.p0(ptr %ptr)
+  declare void  @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) %const_ptr)
+  declare void  @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101) %param_ptr)  
+  
   declare void  @llvm.nvvm.prefetch.global.L2.evict.normal(ptr addrspace(1) %global_ptr)
   declare void  @llvm.nvvm.prefetch.global.L2.evict.last(ptr addrspace(1) %global_ptr)
 
@@ -983,7 +993,10 @@ The '``@llvm.nvvm.prefetch.*``' and '``@llvm.nvvm.prefetchu.*``' intrinsic
 correspond to the '``prefetch.*``;' and '``prefetchu.*``' family of PTX instructions. 
 The '``prefetch.*``' instructions bring the cache line containing the
 specified address in global or local memory address space into the 
-specified cache level (L1 or L2). The '`prefetchu.*``' instruction brings the cache line 
+specified cache level (L1 or L2). If the '``.tensormap``' qualifier is specified then the 
+prefetch instruction brings the cache line containing the specified address in the 
+'``.const``' or '``.param memory``' state space for subsequent use by the '``cp.async.bulk.tensor``' 
+instruction. The '`prefetchu.*``' instruction brings the cache line 
 containing the specified generic address into the specified uniform cache level.
 If no address space is specified, it is assumed to be generic address. The intrinsic 
 uses and eviction priority which can be accessed by the '``.level::eviction_priority``' modifier.
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index a29e06cdf838d..f9f3e39727a5a 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -531,6 +531,10 @@ The current vendor extensions supported are:
 ``XAndesVDot``
   LLVM implements `version 5.0.0 of the Andes Vector Dot Product Extension specification <https://github.com/andestech/andes-v5-isa/releases/download/ast-v5_4_0-release/AndeStar_V5_ISA_Spec_UM165-v1.5.08-20250317.pdf>`__ by Andes Technology. All instructions are prefixed with `nds.` as described in the specification.
 
+``XSMTVDot``
+  SpacemiT defines `Intrinsic Matrix Extension (IME) specification <https://github.com/space-mit/riscv-ime-extension-spec/releases/tag/v0429>`__.
+  LLVM implement the hardware-adapted subset for SpacemiT X60, defined in the `feature document <https://developer.spacemit.com/documentation?token=BWbGwbx7liGW21kq9lucSA6Vnpb#2.1>`__ by SpacemiT. All instructions are prefixed with `smt.` as described in the implementation guide. Note that this implemented subset is `version 1.0.0 of the SpacemiT Vector Dot Product Extension specification`, which is strictly a subset of the full IME specification to reflect the capabilities of SpacemiT X60 hardware correctly.
+
 Experimental C Intrinsics
 =========================
 
diff --git a/llvm/docs/Reference.rst b/llvm/docs/Reference.rst
index 35a6f59ecbf35..7d0fdd78dc96d 100644
--- a/llvm/docs/Reference.rst
+++ b/llvm/docs/Reference.rst
@@ -17,6 +17,7 @@ LLVM and API reference documentation.
    CalleeTypeMetadata
    CIBestPractices
    CommandGuide/index
+   ContentAddressableStorage
    ConvergenceAndUniformity
    ConvergentOperations
    Coroutines
@@ -244,3 +245,6 @@ Additional Topics
 :doc:`MLGO`
    Facilities for ML-Guided Optimization, such as collecting IR corpora from a
    build, interfacing with ML models, an exposing features for training.
+
+:doc:`ContentAddressableStorage`
+   A reference guide for using LLVM's CAS library.
diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md
index 0c49fc86a4232..3b90c964ac53d 100644
--- a/llvm/docs/ReleaseNotes.md
+++ b/llvm/docs/ReleaseNotes.md
@@ -1,6 +1,9 @@
 <!-- This document is written in Markdown and uses extra directives provided by
 MyST (https://myst-parser.readthedocs.io/en/latest/). -->
 
+<!-- If you want to modify sections/contents permanently, you should modify both
+ReleaseNotes.md and ReleaseNotesTemplate.txt. -->
+
 LLVM {{env.config.release}} Release Notes
 =========================================
 
@@ -56,6 +59,10 @@ Makes programs 10x faster by doing Special New Thing.
 Changes to the LLVM IR
 ----------------------
 
+* The `ptrtoaddr` instruction was introduced. This instruction returns the
+  address component of a pointer type variable but unlike `ptrtoint` does not
+  capture provenance ([#125687](https://github.com/llvm/llvm-project/pull/125687)).
+
 Changes to LLVM infrastructure
 ------------------------------
 
@@ -73,6 +80,7 @@ Changes to Vectorizers
 
 * Added initial support for copyable elements in SLP, which models copyable
   elements as add <element>, 0, i.e. uses identity constants for missing lanes.
+* SLP vectorizer supports initial recognition of FMA/FMAD pattern
 
 Changes to the AArch64 Backend
 ------------------------------
@@ -104,6 +112,14 @@ Changes to the PowerPC Backend
 Changes to the RISC-V Backend
 -----------------------------
 
+* The loop vectorizer now performs tail folding by default on RISC-V, which
+  removes the need for a scalar epilogue loop. To restore the previous behaviour
+  use `-prefer-predicate-over-epilogue=scalar-epilogue`.
+* `llvm-objdump` now has basic support for switching between disassembling code
+  and data using mapping symbols such as `$x` and `$d`. Switching architectures
+  using `$x` with an architecture string suffix is not yet supported.
+* Ssctr and Smctr extensions are no longer experimental.
+
 Changes to the WebAssembly Backend
 ----------------------------------
 
diff --git a/llvm/docs/ReleaseNotesTemplate.txt b/llvm/docs/ReleaseNotesTemplate.txt
new file mode 100644
index 0000000000000..ea1e4906fbe2e
--- /dev/null
+++ b/llvm/docs/ReleaseNotesTemplate.txt
@@ -0,0 +1,163 @@
+<!-- This document is written in Markdown and uses extra directives provided by
+MyST (https://myst-parser.readthedocs.io/en/latest/). -->
+
+<!-- If you want to modify sections/contents permanently, you should modify both
+ReleaseNotes.md and ReleaseNotesTemplate.txt. -->
+
+LLVM {{env.config.release}} Release Notes
+=========================================
+
+```{contents}
+```
+
+````{only} PreRelease
+```{warning} These are in-progress notes for the upcoming LLVM {{env.config.release}}
+             release. Release notes for previous releases can be found on
+             [the Download Page](https://releases.llvm.org/download.html).
+```
+````
+
+Introduction
+============
+
+This document contains the release notes for the LLVM Compiler Infrastructure,
+release {{env.config.release}}.  Here we describe the status of LLVM, including
+major improvements from the previous release, improvements in various subprojects
+of LLVM, and some of the current users of the code.  All LLVM releases may be
+downloaded from the [LLVM releases web site](https://llvm.org/releases/).
+
+For more information about LLVM, including information about the latest
+release, please check out the [main LLVM web site](https://llvm.org/).  If you
+have questions or comments, the [Discourse forums](https://discourse.llvm.org)
+is a good place to ask them.
+
+Note that if you are reading this file from a Git checkout or the main
+LLVM web page, this document applies to the *next* release, not the current
+one.  To see the release notes for a specific release, please see the
+[releases page](https://llvm.org/releases/).
+
+Non-comprehensive list of changes in this release
+=================================================
+
+<!-- For small 1-3 sentence descriptions, just add an entry at the end of
+this list. If your description won't fit comfortably in one bullet
+point (e.g. maybe you would like to give an example of the
+functionality, or simply have a lot to talk about), see the comment below
+for adding a new subsection. -->
+
+* ...
+
+<!-- If you would like to document a larger change, then you can add a
+subsection about it right here. You can copy the following boilerplate:
+
+Special New Feature
+-------------------
+
+Makes programs 10x faster by doing Special New Thing.
+-->
+
+Changes to the LLVM IR
+----------------------
+
+Changes to LLVM infrastructure
+------------------------------
+
+Changes to building LLVM
+------------------------
+
+Changes to TableGen
+-------------------
+
+Changes to Interprocedural Optimizations
+----------------------------------------
+
+Changes to Vectorizers
+----------------------
+
+Changes to the AArch64 Backend
+------------------------------
+
+Changes to the AMDGPU Backend
+-----------------------------
+
+Changes to the ARM Backend
+--------------------------
+
+Changes to the AVR Backend
+--------------------------
+
+Changes to the DirectX Backend
+------------------------------
+
+Changes to the Hexagon Backend
+------------------------------
+
+Changes to the LoongArch Backend
+--------------------------------
+
+Changes to the MIPS Backend
+---------------------------
+
+Changes to the PowerPC Backend
+------------------------------
+
+Changes to the RISC-V Backend
+-----------------------------
+
+Changes to the WebAssembly Backend
+----------------------------------
+
+Changes to the Windows Target
+-----------------------------
+
+Changes to the X86 Backend
+--------------------------
+
+Changes to the OCaml bindings
+-----------------------------
+
+Changes to the Python bindings
+------------------------------
+
+Changes to the C API
+--------------------
+
+Changes to the CodeGen infrastructure
+-------------------------------------
+
+Changes to the Metadata Info
+----------------------------
+
+Changes to the Debug Info
+-------------------------
+
+Changes to the LLVM tools
+-------------------------
+
+Changes to LLDB
+---------------
+
+Changes to BOLT
+---------------
+
+Changes to Sanitizers
+---------------------
+
+Other Changes
+-------------
+
+External Open Source Projects Using LLVM {{env.config.release}}
+===============================================================
+
+Additional Information
+======================
+
+A wide variety of additional information is available on the
+[LLVM web page](https://llvm.org/), in particular in the
+[documentation](https://llvm.org/docs/) section.  The web page also contains
+versions of the API documentation which is up-to-date with the Git version of
+the source code.  You can access versions of these documents specific to this
+release by going into the `llvm/docs/` directory in the LLVM tree.
+
+If you have any questions or comments about LLVM, please feel free to contact
+us via the [Discourse forums](https://discourse.llvm.org).
diff --git a/llvm/docs/SourceLevelDebugging.rst b/llvm/docs/SourceLevelDebugging.rst
index dfc8c53edbb8e..f057b2d2d2e36 100644
--- a/llvm/docs/SourceLevelDebugging.rst
+++ b/llvm/docs/SourceLevelDebugging.rst
@@ -34,7 +34,7 @@ important ones are:
   the source-level-language.
 
 * Source-level languages are often **widely** different from one another.
-  LLVM should not put any restrictions of the flavor of the source-language,
+  LLVM should not put any restrictions on the flavor of the source-language,
   and the debugging information should work with any language.
 
 * With code generator support, it should be possible to use an LLVM compiler
@@ -74,10 +74,10 @@ from and inspired by DWARF, but it is feasible to translate into other target
 debug info formats such as STABS.
 
 SamplePGO (also known as `AutoFDO <https://gcc.gnu.org/wiki/AutoFDO>`_)
-is a variant of profile guided optimizations which uses hardware sampling based
+is a variant of profile-guided optimizations which uses hardware sampling based
 profilers to collect branch frequency data with low overhead in production
 environments. It relies on debug information to associate profile information
-to LLVM IR which is then used to guide optimization heuristics. Maintaining
+with LLVM IR which is then used to guide optimization heuristics. Maintaining
 deterministic and distinct source locations is necessary to maximize the
 accuracy of mapping hardware sample counts to LLVM IR. For example, DWARF
 `discriminators <https://wiki.dwarfstd.org/Path_Discriminators.md>`_ allow
@@ -160,15 +160,15 @@ which can have a value, including at least:
    There is no special provision for "true" constants in LLVM today, and
    they are instead treated as local or global variables.
 
-A variable is represented by a `local variable <LangRef.html#dilocalvariable>`_
-or `global variable <LangRef.html#diglobalvariable>`_ metadata node.
+A variable is represented by a :ref:`local variable <dilocalvariable>` or
+:ref:`global variable <diglobalvariable>` metadata node.
 
 A "variable fragment" (or just "fragment") is a contiguous span of bits of a
 variable.
 
-A :ref:`debug record <debug_records>` which refers to a ``DIExpression`` ending
-with a ``DW_OP_LLVM_fragment`` operation describes a fragment of the variable
-it refers to.
+A :ref:`debug record <debug_records>` which refers to a :ref:`diexpression`
+ending with a ``DW_OP_LLVM_fragment`` operation describes a fragment of the
+variable it refers to.
 
 The operands of the ``DW_OP_LLVM_fragment`` operation encode the bit offset of
 the fragment relative to the start of the variable, and the size of the
@@ -205,16 +205,16 @@ debugger to interpret the information.
 To provide basic functionality, the LLVM debugger does have to make some
 assumptions about the source-level language being debugged, though it keeps
 these to a minimum.  The only common features that the LLVM debugger assumes
-exist are `source files <LangRef.html#difile>`_, and `program objects
-<LangRef.html#diglobalvariable>`_.  These abstract objects are used by a
-debugger to form stack traces, show information about local variables, etc.
+exist are :ref:`source files <difile>`, and :ref:`program objects
+<diglobalvariable>`.  These abstract objects are used by a debugger to form
+stack traces, show information about local variables, etc.
 
 This section of the documentation first describes the representation aspects
 common to any source-language.  :ref:`ccxx_frontend` describes the data layout
 conventions used by the C and C++ front-ends.
 
-Debug information descriptors are `specialized metadata nodes
-<LangRef.html#specialized-metadata>`_, first-class subclasses of ``Metadata``.
+Debug information descriptors are :ref:`specialized metadata nodes
+<specialized-metadata>`, first-class subclasses of ``Metadata``.
 
 There are two models for defining the values of source variables at different
 states of the program and tracking these values through optimization and code
@@ -229,7 +229,7 @@ document.
 .. _debug_records:
 
 Debug Records
-----------------------------
+-------------
 
 Debug records define the value that a source variable has during execution of
 the program; they appear interleaved with instructions, although they are not
@@ -256,14 +256,13 @@ comma-separated arguments in parentheses, as with a `call`.
 
     #dbg_declare([Value|MDNode], DILocalVariable, DIExpression, DILocation)
 
-This record provides information about a local element (e.g., variable).
-The first argument is an SSA ``ptr`` value corresponding to a variable address,
-and is typically a static alloca in the function entry block.  The second
-argument is a `local variable <LangRef.html#dilocalvariable>`_ containing a
-description of the variable.  The third argument is a `complex expression
-<LangRef.html#diexpression>`_. The fourth argument is a `source location
-<LangRef.html#dilocation>`_. A ``#dbg_declare`` record describes the
-*address* of a source variable.
+This record provides information about a local element (e.g., variable). The
+first argument is an SSA ``ptr`` value corresponding to a variable address, and
+is typically a static ``alloca`` in the function entry block.  The second
+argument is a :ref:`local variable <dilocalvariable>` containing a description
+of the variable.  The third argument is a :ref:`complex expression
+<diexpression>`. The fourth argument is a :ref:`source location <dilocation>`.
+A ``#dbg_declare`` record describes the *address* of a source variable.
 
 .. code-block:: llvm
 
@@ -299,11 +298,10 @@ must agree on the memory location.
     #dbg_value([Value|DIArgList|MDNode], DILocalVariable, DIExpression, DILocation)
 
 This record provides information when a user source variable is set to a new
-value.  The first argument is the new value. The second argument is a `local
-variable <LangRef.html#dilocalvariable>`_ containing a description of the
-variable.  The third argument is a `complex expression
-<LangRef.html#diexpression>`_. The fourth argument is a `source location
-<LangRef.html#dilocation>`_.
+value.  The first argument is the new value. The second argument is a
+:ref:`local variable <dilocalvariable>` containing a description of the
+variable.  The third argument is a :ref:`complex expression <diexpression>`.
+The fourth argument is a :ref:`source location <dilocation>`.
 
 A ``#dbg_value`` record describes the *value* of a source variable
 directly, not its address.  Note that the value operand of this intrinsic may
@@ -311,7 +309,7 @@ be indirect (i.e, a pointer to the source variable), provided that interpreting
 the complex expression derives the direct value.
 
 ``#dbg_assign``
-^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^
 .. toctree::
    :hidden:
 
@@ -333,15 +331,21 @@ performs the assignment, and the destination address.
 
 The first three arguments are the same as for a ``#dbg_value``. The fourth
 argument is a ``DIAssignID`` used to reference a store. The fifth is the
-destination of the store, the sixth is a `complex
-expression <LangRef.html#diexpression>`_ that modfies it, and the seventh is a
-`source location <LangRef.html#dilocation>`_.
+destination of the store, the sixth is a :ref:`complex expression
+<diexpression>` that modifies it, and the seventh is a :ref:`source location
+<dilocation>`.
 
 See :doc:`AssignmentTracking` for more info.
 
 Debugger intrinsic functions
 ----------------------------
 
+.. warning::
+
+  These intrinsics are deprecated, please use :ref:`debug records
+  <debug_records>` instead. For more details see `RemoveDIs
+  <RemoveDIsDebugInfo.html>`_.
+
 .. _format_common_intrinsics:
 
 In intrinsic-mode, LLVM uses several intrinsic functions (name prefixed with "``llvm.dbg``") to
@@ -394,12 +398,231 @@ This intrinsic is equivalent to ``#dbg_assign``:
 
 .. code-block:: llvm
 
-      #dbg_assign(i32 %i, !1, !DIExpression(), !2, 
+      #dbg_assign(i32 %i, !1, !DIExpression(), !2,
                   ptr %i.addr, !DIExpression(), !3)
     call void @llvm.dbg.assign(
       metadata i32 %i, metadata !1, metadata !DIExpression(), metadata !2,
       metadata ptr %i.addr, metadata !DIExpression(), metadata !3), !dbg !3
 
+.. _diexpression:
+
+DIExpression
+------------
+
+Debug expressions are represented as :ref:`specialized-metadata`.
+
+Debug expressions are interpreted left-to-right: start by pushing the
+value/address operand of the record onto a stack, then repeatedly push and
+evaluate opcodes from the ``DIExpression`` until the final variable description
+is produced.
+
+The opcodes available in these expressions are described in
+:ref:`dwarf-opcodes` and :ref:`internal-opcodes`.
+
+DWARF specifies three kinds of simple location descriptions: register, memory,
+and implicit location descriptions.  Note that a location description is
+defined over certain ranges of a program, i.e the location of a variable may
+change over the course of the program. Register and memory location
+descriptions describe the *concrete location* of a source variable (in the
+sense that a debugger might modify its value), whereas *implicit locations*
+describe merely the actual *value* of a source variable which might not exist
+in registers or in memory (see ``DW_OP_stack_value``).
+
+A ``#dbg_declare`` record describes an indirect value (the address) of a source
+variable. The first operand of the record must be an address of some kind. A
+``DIExpression`` operand to the record refines this address to produce a
+concrete location for the source variable.
+
+A ``#dbg_value`` record describes the direct value of a source variable. The
+first operand of the record may be a direct or indirect value. A
+``DIExpression`` operand to the record refines the first operand to produce a
+direct value. For example, if the first operand is an indirect value, it may be
+necessary to insert ``DW_OP_deref`` into the ``DIExpression`` in order to
+produce a valid debug record.
+
+.. note::
+
+   A ``DIExpression`` is interpreted in the same way regardless of which kind
+   of debug record it's attached to.
+
+   ``DIExpression``\s are always printed and parsed inline; they can never be
+   referenced by an ID (e.g. ``!1``).
+
+.. _dwarf-opcodes:
+
+DWARF Opcodes
+^^^^^^^^^^^^^
+
+When possible LLVM reuses DWARF opcodes and gives them identical semantics in
+LLVM expressions as in DWARF expressions. The current supported opcode
+vocabulary is limited, but includes at least:
+
+- ``DW_OP_deref`` dereferences the top of the expression stack.
+- ``DW_OP_plus`` pops the last two entries from the expression stack, adds
+  them together and pushes the result to the expression stack.
+- ``DW_OP_minus`` pops the last two entries from the expression stack, subtracts
+  the last entry from the second last entry and appends the result to the
+  expression stack.
+- ``DW_OP_plus_uconst, 93`` adds ``93`` to the value on top of the stack.
+- ``DW_OP_swap`` swaps top two stack entries.
+- ``DW_OP_xderef`` provides extended dereference mechanism. The entry at the top
+  of the stack is treated as an address. The second stack entry is treated as an
+  address space identifier. The two entries are popped and then an
+  implementation defined value is pushed on the stack.
+- ``DW_OP_stack_value`` may appear at most once in an expression, and must be
+  the last opcode if ``DW_OP_LLVM_fragment`` is not present, or the second last
+  opcode if ``DW_OP_LLVM_fragment`` is present. It pops the top value of the
+  expression stack and makes an implicit value location with that value.
+- ``DW_OP_breg`` (or ``DW_OP_bregx``) represents a content on the provided
+  signed offset of the specified register. The opcode is only generated by the
+  ``AsmPrinter`` pass to describe call site parameter value which requires an
+  expression over two registers.
+- ``DW_OP_push_object_address`` pushes the address of the object which can then
+  serve as a descriptor in subsequent calculation. This opcode can be used to
+  calculate bounds of a Fortran allocatable array which has array descriptors.
+- ``DW_OP_over`` duplicates the entry currently second in the stack at the top
+  of the stack. This opcode can be used to calculate bounds of a Fortran
+  assumed rank array which has rank known at run time and current dimension
+  number is implicitly first element of the stack.
+
+.. _internal-opcodes:
+
+Internal Opcodes
+^^^^^^^^^^^^^^^^
+
+Where the DWARF equivalent is not suitable, or no DWARF equivalent exists, LLVM
+defines internal-only opcodes which have no direct analog in DWARF.
+
+.. note::
+
+   Some opcodes do not influence the final DWARF expression directly, instead
+   encoding information logically belonging to the debug records which use
+   them.
+
+- ``DW_OP_LLVM_fragment, <offset>, <size>`` may appear at most once in an
+  expression, and must be the last opcode. It specifies the bit offset and bit
+  size of the variable fragment being described by the record or intrinsic
+  using the expression. Note that contrary to ``DW_OP_bit_piece``, the offset
+  is describing the location within the described source variable. At DWARF
+  generation time all fragments for the same variable are collected together
+  and DWARF ``DW_OP_piece`` and ``DW_OP_bit_piece`` opcodes are used to
+  describe a composite with pieces corresponding to the fragments. (This does
+  not affect the semantics of the expression containing it.) -
+  ``DW_OP_LLVM_convert, 16, DW_ATE_signed`` specifies a bit size and encoding
+  (``16`` and ``DW_ATE_signed`` here, respectively) to which the top of the
+  expression stack is to be converted. Maps into a ``DW_OP_convert`` operation
+  that references a base type constructed from the supplied values. -
+  ``DW_OP_LLVM_tag_offset, tag_offset`` specifies that a memory tag should be
+  optionally applied to the pointer. The memory tag is derived from the given
+  tag offset in an implementation-defined manner. (This does not affect the
+  semantics of the expression containing it.) - ``DW_OP_LLVM_entry_value, N``
+  evaluates a sub-expression as-if it were evaluated upon entry to the current
+  call frame.
+
+  The sub-expression replaces the operations which comprise it, i.e. all such
+  operations are evaluated only in the frame entry context.
+
+  The sub-expression begins with the operation which immediately precedes
+  ``DW_OP_LLVM_entry_value, N`` in the ``DIExpression``. If no such operation
+  exists (i.e. the expression begins with ``DW_OP_LLVM_entry_value, N``), the
+  implicit operation which pushes the first debug argument of the containing
+  marker/pseudo is used instead. The value ``N`` must always be at least ``1``,
+  as this first operation cannot be omitted and is counted in ``N``.
+
+  The rest of the sub-expression comprises the ``(N - 1)`` operations following
+  ``DW_OP_LLVM_entry_value, N`` in the ``DIExpression``.
+
+  Due to framework limitations:
+
+    - ``N`` must not be greater than ``1``. In other words, ``N`` must equal
+      ``1``, and the sub-expression comprises only the operation immediately
+      preceding ``DW_OP_LLVM_entry_value, N``.
+    - ``DW_OP_LLVM_entry_value, N`` must be either the first operation of a
+      ``DIExpression`` or the second operation if the expression begins with
+      ``DW_OP_LLVM_arg, 0``.
+    - The first operation must refer to a register value.
+
+  Taken together, these limitations mean that ``DW_OP_LLVM_entry_value`` can
+  only currently be used to push the value a single register had on entry to
+  the current stack frame.
+
+  For example, ``!DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_entry_value, 1,
+  DW_OP_LLVM_arg, 1, DW_OP_plus, DW_OP_stack_value)`` specifies an expression
+  where the entry value of the first argument to the ``DIExpression`` is added
+  to the non-entry value of the second argument, and the result is used as the
+  value for an implicit value location.
+
+  When targeting DWARF, a ``DBG_VALUE(reg, ...,
+  DIExpression(DW_OP_LLVM_entry_value, 1, ...)`` is lowered to
+  ``DW_OP_entry_value [reg], ...``, which pushes the value ``reg`` had upon
+  frame entry onto the DWARF expression stack.
+
+  Because ``DW_OP_LLVM_entry_value`` is currently limited to registers, it is
+  usually used in MIR, but it is also allowed in LLVM IR when targeting a
+  :ref:`swiftasync <swiftasync>` argument. The operation is introduced by:
+
+    - ``LiveDebugValues`` pass, which applies it to function parameters that
+      are unmodified throughout the function. Support is limited to simple
+      register location descriptions, or as indirect locations (e.g.,
+      parameters passed-by-value to a callee via a pointer to a temporary copy
+      made in the caller).
+    - ``AsmPrinter`` pass when a call site parameter value
+      (``DW_AT_call_site_parameter_value``) is represented as entry value of
+      the parameter.
+    - ``CoroSplit`` pass, which may move variables from ``alloca``\s into a
+      coroutine frame. If the coroutine frame is a
+      :ref:`swiftasync <swiftasync>` argument, the variable is described with
+      an ``DW_OP_LLVM_entry_value`` operation.
+
+- ``DW_OP_LLVM_implicit_pointer`` It specifies the dereferenced value. It can
+  be used to represent pointer variables which are optimized out but the value
+  it points to is known. This operator is required as it is different than
+  DWARF operator ``DW_OP_implicit_pointer`` in representation and specification
+  (number and types of operands) and later can not be used as multiple level.
+
+  Examples using ``DW_OP_LLVM_implicit_pointer``:
+
+  .. code-block:: text
+
+      IR for "*ptr = 4;"
+      --------------
+        #dbg_value(i32 4, !17, !DIExpression(DW_OP_LLVM_implicit_pointer), !20)
+      !17 = !DILocalVariable(name: "ptr", scope: !12, file: !3, line: 5,
+                             type: !18)
+      !18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64)
+      !19 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+      !20 = !DILocation(line: 10, scope: !12)
+
+      IR for "**ptr = 4;"
+      --------------
+        #dbg_value(i32 4, !17,
+          !DIExpression(DW_OP_LLVM_implicit_pointer, DW_OP_LLVM_implicit_pointer),
+          !21)
+      !17 = !DILocalVariable(name: "ptr", scope: !12, file: !3, line: 5,
+                             type: !18)
+      !18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !19, size: 64)
+      !19 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !20, size: 64)
+      !20 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+      !21 = !DILocation(line: 10, scope: !12)
+
+- ``DW_OP_LLVM_arg, N`` is used in debug intrinsics that refer to more than one
+  value, such as one that calculates the sum of two registers. This is always
+  used in combination with an ordered list of values, such that
+  ``DW_OP_LLVM_arg, N`` refers to the ``N``\ :sup:`th` element in that list.
+  For example, ``!DIExpression(DW_OP_LLVM_arg, 0, DW_OP_LLVM_arg, 1,
+  DW_OP_minus, DW_OP_stack_value)`` used with the list ``(%reg1, %reg2)`` would
+  evaluate to an implicit value location that has the value of
+  ``%reg1 - reg2``. This list of values should be provided by the containing
+  intrinsic/instruction.
+- ``DW_OP_LLVM_extract_bits_sext, 16, 8,`` specifies the offset and size
+  (``16`` and ``8`` here, respectively) of bits that are to be extracted and
+  sign-extended from the value at the top of the expression stack. If the top of
+  the expression stack is a memory location then these bits are extracted from
+  the value pointed to by that memory location. Maps into a ``DW_OP_shl``
+  followed by ``DW_OP_shra``.
+- ``DW_OP_LLVM_extract_bits_zext`` behaves similarly to
+  ``DW_OP_LLVM_extract_bits_sext``, but zero-extends instead of sign-extending.
+  Maps into a ``DW_OP_shl`` followed by ``DW_OP_shr``.
 
 Object lifetimes and scoping
 ============================
@@ -413,7 +636,7 @@ it is non-trivial to model in LLVM, because it has no notion of scoping in this
 sense, and does not want to be tied to a language's scoping rules.
 
 In order to handle this, the LLVM debug format uses the metadata attached to
-llvm instructions to encode line number and scoping information.  Consider the
+LLVM instructions to encode line number and scoping information.  Consider the
 following C fragment, for example:
 
 .. code-block:: c
@@ -506,13 +729,13 @@ scope information for the variable ``X``.
                               isLocal: false, isDefinition: true, scopeLine: 1,
                               isOptimized: false, retainedNodes: !2)
 
-Here ``!13`` is metadata providing `location information
-<LangRef.html#dilocation>`_.  In this example, scope is encoded by ``!4``, a
-`subprogram descriptor <LangRef.html#disubprogram>`_.  This way the location
-information parameter to the records indicates that the variable ``X`` is
-declared at line number 2 at a function level scope in function ``foo``.
+Here ``!13`` is metadata providing :ref:`location information <dilocation>`.
+In this example, scope is encoded by ``!4``, a :ref:`subprogram descriptor
+<disubprogram>`.  This way the location information parameter to the records
+indicates that the variable ``X`` is declared at line number 2 at a function
+level scope in function ``foo``.
 
-Now lets take another example.
+Now, let's take another example.
 
 .. code-block:: llvm
 
@@ -532,14 +755,14 @@ Here ``!18`` indicates that ``Z`` is declared at line number 5 and column
 number 11 inside of lexical scope ``!17``.  The lexical scope itself resides
 inside of subprogram ``!4`` described above.
 
-The scope information attached with each instruction provides a straightforward
+The scope information attached to each instruction provides a straightforward
 way to find instructions covered by a scope.
 
 Object lifetime in optimized code
 =================================
 
 In the example above, every variable assignment uniquely corresponds to a
-memory store to the variable's position on the stack. However in heavily
+memory store to the variable's position on the stack. However, in heavily
 optimized code LLVM promotes most variables into SSA values, which can
 eventually be placed in physical registers or memory locations. To track SSA
 values through compilation, when objects are promoted to SSA values a
@@ -628,10 +851,10 @@ perhaps, be optimized into the following code:
   }
 
 What ``#dbg_value`` records should be placed to represent the original variable
-locations in this code? Unfortunately the second, third and fourth
-#dbg_values for ``!1`` in the source function have had their operands
-(%tval, %fval, %merge) optimized out. Assuming we cannot recover them, we
-might consider this placement of #dbg_values:
+locations in this code? Unfortunately the second, third, and fourth
+``#dbg_value``\s for ``!1`` in the source function have had their operands
+(``%tval``, ``%fval``, ``%merge``) optimized out. Assuming we cannot recover
+them, we might consider this placement of ``#dbg_value``\s:
 
 .. code-block:: llvm
 
@@ -651,7 +874,7 @@ However, this will cause ``!3`` to have the return value of ``@gazonk()`` at
 the same time as ``!1`` has the constant value zero -- a pair of assignments
 that never occurred in the unoptimized program. To avoid this, we must terminate
 the range that ``!1`` has the constant value assignment by inserting an poison
-#dbg_value before the #dbg_value for ``!3``:
+``#dbg_value`` before the ``#dbg_value`` for ``!3``:
 
 .. code-block:: llvm
 
@@ -668,7 +891,7 @@ the range that ``!1`` has the constant value assignment by inserting an poison
     ret i32 %toret
   }
 
-There are a few other #dbg_value configurations that mean it terminates
+There are a few other ``#dbg_value`` configurations that mean it terminates
 dominating location definitions without adding a new location. The complete
 list is:
 
@@ -678,17 +901,17 @@ list is:
 * There are no location operands (empty ``DIArgList``) and the ``DIExpression``
   is empty.
 
-This class of #dbg_value that kills variable locations is called a "kill
-#dbg_value" or "kill location", and for legacy reasons the term "undef
-#dbg_value" may be used in existing code. The ``DbgVariableIntrinsic`` methods
-``isKillLocation`` and ``setKillLocation`` should be used where possible rather
-than inspecting location operands directly to check or set whether a #dbg_value
-is a kill location.
+This class of ``#dbg_value`` that kills variable locations is called a "kill
+``#dbg_value``" or "kill location", and for legacy reasons the term "``undef
+#dbg_value``" may be used in existing code. The ``DbgVariableIntrinsic``
+methods ``isKillLocation`` and ``setKillLocation`` should be used where
+possible rather than inspecting location operands directly to check or set
+whether a ``#dbg_value`` is a kill location.
 
-In general, if any #dbg_value has its operand optimized out and cannot be
-recovered, then a kill #dbg_value is necessary to terminate earlier variable
-locations. Additional kill #dbg_values may be necessary when the debugger can
-observe re-ordering of assignments.
+In general, if any ``#dbg_value`` has its operand optimized out and cannot be
+recovered, then a kill ``#dbg_value`` is necessary to terminate earlier
+variable locations. Additional kill ``#dbg_values`` may be necessary when the
+debugger can observe re-ordering of assignments.
 
 How variable location metadata is transformed during CodeGen
 ============================================================
@@ -696,7 +919,7 @@ How variable location metadata is transformed during CodeGen
 LLVM preserves debug information throughout mid-level and backend passes,
 ultimately producing a mapping between source-level information and
 instruction ranges. This
-is relatively straightforwards for line number information, as mapping
+is relatively straightforward for line number information, as mapping
 instructions to line numbers is a simple association. For variable locations
 however the story is more complex. As each ``#dbg_value`` record
 represents a source-level assignment of a value to a source variable, the
@@ -710,13 +933,13 @@ location fidelity are:
 2. Register allocation
 3. Block layout
 
-each of which are discussed below. In addition, instruction scheduling can
+each of which is discussed below. In addition, instruction scheduling can
 significantly change the ordering of the program, and occurs in a number of
 different passes.
 
 Some variable locations are not transformed during CodeGen. Stack locations
-specified by ``#dbg_declare`` are valid and unchanging for the entire
-duration of the function, and are recorded in a simple MachineFunction table.
+specified by ``#dbg_declare`` are valid and unchanging for the entire duration
+of the function, and are recorded in a simple ``MachineFunction`` table.
 Location changes in the prologue and epilogue of a function are also ignored:
 frame setup and destruction may take several instructions, require a
 disproportionate amount of debugging information in the output binary to
@@ -727,21 +950,20 @@ Variable locations in Instruction Selection and MIR
 
 Instruction selection creates a MIR function from an IR function, and just as
 it transforms ``intermediate`` instructions into machine instructions, so must
-``intermediate`` variable locations become machine variable locations.
-Within IR, variable locations are always identified by a Value, but in MIR
-there can be different types of variable locations. In addition, some IR
-locations become unavailable, for example if the operation of multiple IR
-instructions are combined into one machine instruction (such as
-multiply-and-accumulate) then intermediate Values are lost. To track variable
-locations through instruction selection, they are first separated into
-locations that do not depend on code generation (constants, stack locations,
-allocated virtual registers) and those that do. For those that do, debug
-metadata is attached to SDNodes in SelectionDAGs. After instruction selection
-has occurred and a MIR function is created, if the SDNode associated with debug
-metadata is allocated a virtual register, that virtual register is used as the
-variable location. If the SDNode is folded into a machine instruction or
-otherwise transformed into a non-register, the variable location becomes
-unavailable.
+``intermediate`` variable locations become machine variable locations. Within
+IR, variable locations are always identified by a Value, but in MIR there can
+be different types of variable locations. In addition, some IR locations become
+unavailable, for example if the operation of multiple IR instructions are
+combined into one machine instruction (such as multiply-and-accumulate) then
+intermediate Values are lost. To track variable locations through instruction
+selection, they are first separated into locations that do not depend on code
+generation (constants, stack locations, allocated virtual registers) and those
+that do. For those that do, debug metadata is attached to ``SDNode``\s in
+``SelectionDAG``\s. After instruction selection has occurred and a MIR function
+is created, if the ``SDNode`` associated with debug metadata is allocated a
+virtual register, that virtual register is used as the variable location. If
+the ``SDNode`` is folded into a machine instruction or otherwise transformed
+into a non-register, the variable location becomes unavailable.
 
 Locations that are unavailable are treated as if they have been optimized out:
 in IR the location would be assigned ``undef`` by a debug record, and in MIR
@@ -761,9 +983,9 @@ And has the following operands:
  * The first operand can record the variable location as a register,
    a frame index, an immediate, or the base address register if the original
    debug record referred to memory. ``$noreg`` indicates the variable
-   location is undefined, equivalent to an ``undef`` #dbg_value operand.
+   location is undefined, equivalent to an ``undef #dbg_value`` operand.
  * The type of the second operand indicates whether the variable location is
-   directly referred to by the DBG_VALUE, or whether it is indirect. The
+   directly referred to by the ``DBG_VALUE``, or whether it is indirect. The
    ``$noreg`` register signifies the former, an immediate operand (0) the
    latter.
  * Operand 3 is the Variable field of the original debug record.
@@ -782,13 +1004,12 @@ And has the following operands:
    location operands, which may take any of the same values as the first
    operand of the ``DBG_VALUE`` instruction above. These variable location
    operands are inserted into the final DWARF Expression in positions indicated
-   by the DW_OP_LLVM_arg operator in the `DIExpression
-   <LangRef.html#diexpression>`_.
+   by the ``DW_OP_LLVM_arg`` operator in the :ref:`diexpression`.
 
-The position at which the DBG_VALUEs are inserted should correspond to the
-positions of their matching ``#dbg_value`` records in the IR block.  As
-with optimization, LLVM aims to preserve the order in which variable
-assignments occurred in the source program. However SelectionDAG performs some
+The position at which the ``DBG_VALUE``\s are inserted should correspond to the
+positions of their matching ``#dbg_value`` records in the IR block.  As with
+optimization, LLVM aims to preserve the order in which variable assignments
+occurred in the source program. However, ``SelectionDAG`` performs some
 instruction scheduling, which can reorder assignments (discussed below).
 Function parameter locations are moved to the beginning of the function if
 they're not already, to ensure they're immediately available on function entry.
@@ -855,20 +1076,20 @@ If one compiles this IR with ``llc -o - -start-after=codegen-prepare -stop-after
     $eax = COPY %8, debug-location !5
     RET 0, $eax, debug-location !5
 
-Observe first that there is a DBG_VALUE instruction for every ``#dbg_value``
+Observe first that there is a ``DBG_VALUE`` instruction for every ``#dbg_value``
 record in the source IR, ensuring no source level assignments go missing.
 Then consider the different ways in which variable locations have been recorded:
 
-* For the first #dbg_value an immediate operand is used to record a zero value.
-* The #dbg_value of the PHI instruction leads to a DBG_VALUE of virtual register
+* For the first ``#dbg_value`` an immediate operand is used to record a zero value.
+* The ``#dbg_value`` of the PHI instruction leads to a ``DBG_VALUE`` of virtual register
   ``%0``.
 * The first GEP has its effect folded into the first load instruction
   (as a 4-byte offset), but the variable location is salvaged by folding
-  the GEPs effect into the DIExpression.
+  the GEPs effect into the ``DIExpression``.
 * The second GEP is also folded into the corresponding load. However, it is
   insufficiently simple to be salvaged, and is emitted as a ``$noreg``
-  DBG_VALUE, indicating that the variable takes on an undefined location.
-* The final #dbg_value has its Value placed in virtual register ``%1``.
+  ``DBG_VALUE``, indicating that the variable takes on an undefined location.
+* The final ``#dbg_value`` has its Value placed in virtual register ``%1``.
 
 Instruction Scheduling
 ----------------------
@@ -880,7 +1101,7 @@ case the instruction sequence could be completely reversed. In such
 circumstances LLVM follows the principle applied to optimizations, that it is
 better for the debugger not to display any state than a misleading state.
 Thus, whenever instructions are advanced in order of execution, any
-corresponding DBG_VALUE is kept in its original position, and if an instruction
+corresponding ``DBG_VALUE`` is kept in its original position, and if an instruction
 is delayed then the variable is given an undefined location for the duration
 of the delay. To illustrate, consider this pseudo-MIR:
 
@@ -893,7 +1114,7 @@ of the delay. To illustrate, consider this pseudo-MIR:
   %7:gr32 = SUB32rr %6, %5, implicit-def dead $eflags
   DBG_VALUE %7, $noreg, !5, !6
 
-Imagine that the SUB32rr were moved forward to give us the following MIR:
+Imagine that the ``SUB32rr`` were moved forward to give us the following MIR:
 
 .. code-block:: text
 
@@ -905,13 +1126,13 @@ Imagine that the SUB32rr were moved forward to give us the following MIR:
   DBG_VALUE %7, $noreg, !5, !6
 
 In this circumstance LLVM would leave the MIR as shown above. Were we to move
-the DBG_VALUE of virtual register %7 upwards with the SUB32rr, we would re-order
+the ``DBG_VALUE`` of virtual register %7 upwards with the ``SUB32rr``, we would re-order
 assignments and introduce a new state of the program. Whereas with the solution
 above, the debugger will see one fewer combination of variable values, because
 ``!3`` and ``!5`` will change value at the same time. This is preferred over
 misrepresenting the original program.
 
-In comparison, if one sunk the MOV32rm, LLVM would produce the following:
+In comparison, if one sunk the ``MOV32rm``, LLVM would produce the following:
 
 .. code-block:: text
 
@@ -924,10 +1145,10 @@ In comparison, if one sunk the MOV32rm, LLVM would produce the following:
   DBG_VALUE %1, $noreg, !1, !2
 
 Here, to avoid presenting a state in which the first assignment to ``!1``
-disappears, the DBG_VALUE at the top of the block assigns the variable the
+disappears, the ``DBG_VALUE`` at the top of the block assigns the variable the
 undefined location, until its value is available at the end of the block where
-an additional DBG_VALUE is added. Were any other DBG_VALUE for ``!1`` to occur
-in the instructions that the MOV32rm was sunk past, the DBG_VALUE for ``%1``
+an additional ``DBG_VALUE`` is added. Were any other ``DBG_VALUE`` for ``!1`` to occur
+in the instructions that the ``MOV32rm`` was sunk past, the ``DBG_VALUE`` for ``%1``
 would be dropped and the debugger would never observe it in the variable. This
 accurately reflects that the value is not available during the corresponding
 portion of the original program.
@@ -936,31 +1157,31 @@ Variable locations during Register Allocation
 ---------------------------------------------
 
 To avoid debug instructions interfering with the register allocator, the
-LiveDebugVariables pass extracts variable locations from a MIR function and
-deletes the corresponding DBG_VALUE instructions. Some localized copy
+``LiveDebugVariables`` pass extracts variable locations from a MIR function and
+deletes the corresponding ``DBG_VALUE`` instructions. Some localized copy
 propagation is performed within blocks. After register allocation, the
-VirtRegRewriter pass re-inserts DBG_VALUE instructions in their original
-positions, translating virtual register references into their physical
-machine locations. To avoid encoding incorrect variable locations, in this
-pass any DBG_VALUE of a virtual register that is not live, is replaced by
-the undefined location. The LiveDebugVariables may insert redundant DBG_VALUEs
-because of virtual register rewriting. These will be subsequently removed by
-the RemoveRedundantDebugValues pass.
-
-LiveDebugValues expansion of variable locations
------------------------------------------------
+``VirtRegRewriter`` pass re-inserts ``DBG_VALUE`` instructions in their
+original positions, translating virtual register references into their physical
+machine locations. To avoid encoding incorrect variable locations, in this pass
+any ``DBG_VALUE`` of a virtual register that is not live, is replaced by the
+undefined location. The ``LiveDebugVariables`` may insert redundant
+``DBG_VALUE``\s because of virtual register rewriting. These will be
+subsequently removed by the ``RemoveRedundantDebugValues`` pass.
+
+``LiveDebugValues`` expansion of variable locations
+---------------------------------------------------
 
 After all optimizations have run and shortly before emission, the
-LiveDebugValues pass runs to achieve two aims:
+``LiveDebugValue``\s pass runs to achieve two aims:
 
 * To propagate the location of variables through copies and register spills,
 * For every block, to record every valid variable location in that block.
 
-After this pass the DBG_VALUE instruction changes meaning: rather than
+After this pass the ``DBG_VALUE`` instruction changes meaning: rather than
 corresponding to a source-level assignment where the variable may change value,
 it asserts the location of a variable in a block, and loses effect outside the
 block. Propagating variable locations through copies and spills is
-straightforwards: determining the variable location in every basic block
+straightforward: determining the variable location in every basic block
 requires the consideration of control flow. Consider the following IR, which
 presents several difficulties:
 
@@ -998,7 +1219,7 @@ Here the difficulties are:
 
 As mentioned above, the ``#dbg_value`` records essentially form an
 imperative program embedded in the IR, with each record defining a variable
-location. This *could* be converted to an SSA form by mem2reg, in the same way
+location. This *could* be converted to an SSA form by ``mem2reg``, in the same way
 that it uses use-def chains to identify control flow merges and insert phi
 nodes for IR Values. However, because debug variable locations are defined for
 every machine instruction, in effect every IR instruction uses every variable
@@ -1009,21 +1230,21 @@ Examining the example above, variable ``!30`` is assigned ``%input`` on both
 conditional paths through the function, while ``!23`` is assigned differing
 constant values on either path. Where control flow merges in ``%bb1`` we would
 want ``!30`` to keep its location (``%input``), but ``!23`` to become undefined
-as we cannot determine at runtime what value it should have in %bb1 without
-inserting a PHI node. mem2reg does not insert the PHI node to avoid changing
-codegen when debugging is enabled, and does not insert the other #dbg_values
+as we cannot determine at runtime what value it should have in ``%bb1`` without
+inserting a PHI node. ``mem2reg`` does not insert the PHI node to avoid changing
+CodeGen when debugging is enabled, and does not insert the other ``#dbg_values``
 to avoid adding very large numbers of records.
 
-Instead, LiveDebugValues determines variable locations when control
+Instead, ``LiveDebugValue``\s determines variable locations when control
 flow merges. A dataflow analysis is used to propagate locations between blocks:
 when control flow merges, if a variable has the same location in all
 predecessors then that location is propagated into the successor. If the
 predecessor locations disagree, the location becomes undefined.
 
-Once LiveDebugValues has run, every block should have all valid variable
-locations described by DBG_VALUE instructions within the block. Very little
+Once ``LiveDebugValue``\s has run, every block should have all valid variable
+locations described by ``DBG_VALUE`` instructions within the block. Very little
 effort is then required by supporting classes (such as
-DbgEntityHistoryCalculator) to build a map of each instruction to every
+``DbgEntityHistoryCalculator``) to build a map of each instruction to every
 valid variable location, without the need to consider control flow. From
 the example above, it is otherwise difficult to determine that the location
 of variable ``!30`` should flow "up" into block ``%bb1``, but that the location
@@ -1057,7 +1278,7 @@ helper functions in ``lib/IR/DIBuilder.cpp``.
 C/C++ source file information
 -----------------------------
 
-``llvm::Instruction`` provides easy access to metadata attached with an
+``llvm::Instruction`` provides easy access to metadata attached to an
 instruction.  One can extract line number information encoded in LLVM IR using
 ``Instruction::getDebugLoc()`` and ``DILocation::getLine()``.
 
@@ -1070,7 +1291,7 @@ instruction.  One can extract line number information encoded in LLVM IR using
     bool ImplicitCode = Loc->isImplicitCode();
   }
 
-When the flag ImplicitCode is true then it means that the Instruction has been
+When the flag ``ImplicitCode`` is true then it means that the Instruction has been
 added by the front-end but doesn't correspond to source code written by the user. For example
 
 .. code-block:: c++
@@ -1080,8 +1301,8 @@ added by the front-end but doesn't correspond to source code written by the user
     ...
   }
 
-At the end of the scope the MyObject's destructor is called but it isn't written
-explicitly. This information is useful to avoid to have counters on brackets when
+At the end of the scope the ``MyObject``'s destructor is called but it isn't written
+explicitly. This information is useful to avoid having counters on brackets when
 making code coverage.
 
 C/C++ global variable information
@@ -1147,11 +1368,11 @@ a C/C++ front-end would generate the following descriptors:
   !8 = !{!"clang version 4.0.0"}
 
 
-The align value in DIGlobalVariable description specifies variable alignment in
-case it was forced by C11 _Alignas(), C++11 alignas() keywords or compiler
-attribute __attribute__((aligned ())). In other case (when this field is missing)
+The align value in ``DIGlobalVariable`` description specifies variable alignment in
+case it was forced by C11 ``_Alignas()``, C++11 ``alignas()`` keywords or compiler
+attribute ``__attribute__((aligned ()))``. In other case (when this field is missing)
 alignment is considered default. This is used when producing DWARF output
-for DW_AT_alignment value.
+for ``DW_AT_alignment`` value.
 
 C/C++ function information
 --------------------------
@@ -1189,7 +1410,7 @@ C++ specific debug information
 C++ special member functions information
 ----------------------------------------
 
-DWARF v5 introduces attributes defined to enhance debugging information of C++ programs. LLVM can generate (or omit) these appropriate DWARF attributes. In C++ a special member function Ctors, Dtors, Copy/Move Ctors, assignment operators can be declared with C++11 keyword deleted. This is represented in LLVM using spFlags value DISPFlagDeleted.
+DWARF v5 introduces attributes defined to enhance debugging information of C++ programs. LLVM can generate (or omit) these appropriate DWARF attributes. In C++ a special member function Ctors, Dtors, Copy/Move Ctors, assignment operators can be declared with C++11 keyword deleted. This is represented in LLVM using ``spFlags`` value ``DISPFlagDeleted``.
 
 Given a class declaration with copy constructor declared as deleted:
 
@@ -1200,7 +1421,7 @@ Given a class declaration with copy constructor declared as deleted:
      foo(const foo&) = deleted;
   };
 
-A C++ frontend would generate following:
+A C++ frontend would generate the following:
 
 .. code-block:: text
 
@@ -1222,7 +1443,7 @@ Fortran specific debug information
 Fortran function information
 ----------------------------
 
-There are a few DWARF attributes defined to support client debugging of Fortran programs.  LLVM can generate (or omit) the appropriate DWARF attributes for the prefix-specs of ELEMENTAL, PURE, IMPURE, RECURSIVE, and NON_RECURSIVE.  This is done by using the spFlags values: DISPFlagElemental, DISPFlagPure, and DISPFlagRecursive.
+There are a few DWARF attributes defined to support client debugging of Fortran programs.  LLVM can generate (or omit) the appropriate DWARF attributes for the prefix-specs of ELEMENTAL, PURE, IMPURE, RECURSIVE, and NON_RECURSIVE.  This is done by using the ``spFlags`` values: ``DISPFlagElemental``, ``DISPFlagPure``, and ``DISPFlagRecursive``.
 
 .. code-block:: fortran
 
@@ -1247,7 +1468,7 @@ and this will materialize an additional DWARF attribute as:
      ...
      DW_AT_elemental [DW_FORM_flag_present]  (true)
 
-There are a few DWARF tags defined to represent Fortran specific constructs i.e DW_TAG_string_type for representing Fortran character(n). In LLVM this is represented as DIStringType.
+There are a few DWARF tags defined to represent Fortran specific constructs i.e ``DW_TAG_string_type`` for representing Fortran character(n). In LLVM, this is represented as ``DIStringType``.
 
 .. code-block:: fortran
 
@@ -1260,7 +1481,7 @@ a Fortran front-end would generate the following descriptors:
   !DILocalVariable(name: "string", arg: 1, scope: !10, file: !3, line: 4, type: !15)
   !DIStringType(name: "character(*)!2", stringLength: !16, stringLengthExpression: !DIExpression(), size: 32)
 
-A fortran deferred-length character can also contain the information of raw storage of the characters in addition to the length of the string. This information is encoded in the  stringLocationExpression field. Based on this information, DW_AT_data_location attribute is emitted in a DW_TAG_string_type debug info.
+A fortran deferred-length character can also contain the information of raw storage of the characters in addition to the length of the string. This information is encoded in the  stringLocationExpression field. Based on this information, ``DW_AT_data_location`` attribute is emitted in a ``DW_TAG_string_type`` debug info.
 
   !DIStringType(name: "character(*)!2", stringLengthExpression: !DIExpression(), stringLocationExpression: !DIExpression(DW_OP_push_object_address, DW_OP_deref), size: 32)
 
@@ -1300,28 +1521,28 @@ calls. This descriptor results in the following DWARF tag:
 Debugging information format
 ============================
 
-Debugging Information Extension for Objective C Properties
+Debugging Information Extension for Objective-C Properties
 ----------------------------------------------------------
 
 Introduction
 ^^^^^^^^^^^^
 
-Objective C provides a simpler way to declare and define accessor methods using
+Objective-C provides a simpler way to declare and define accessor methods using
 declared properties.  The language provides features to declare a property and
 to let compiler synthesize accessor methods.
 
-The debugger lets developer inspect Objective C interfaces and their instance
+The debugger lets developers inspect Objective-C interfaces and their instance
 variables and class variables.  However, the debugger does not know anything
-about the properties defined in Objective C interfaces.  The debugger consumes
+about the properties defined in Objective-C interfaces.  The debugger consumes
 information generated by compiler in DWARF format.  The format does not support
-encoding of Objective C properties.  This proposal describes DWARF extensions to
-encode Objective C properties, which the debugger can use to let developers
-inspect Objective C properties.
+encoding of Objective-C properties.  This proposal describes DWARF extensions to
+encode Objective-C properties, which the debugger can use to let developers
+inspect Objective-C properties.
 
 Proposal
 ^^^^^^^^
 
-Objective C properties exist separately from class members.  A property can be
+Objective-C properties exist separately from class members.  A property can be
 defined only by "setter" and "getter" selectors, and be calculated anew on each
 access.  Or a property can just be a direct access to some declared ivar.
 Finally it can have an ivar "automatically synthesized" for it by the compiler,
@@ -1397,7 +1618,7 @@ don't need to know this convention, since we are given the name of the ivar
 directly.
 
 Also, it is common practice in ObjC to have different property declarations in
-the @interface and @implementation - e.g. to provide a read-only property in
+the ``@interface`` and ``@implementation`` - e.g. to provide a read-only property in
 the interface, and a read-write interface in the implementation.  In that case,
 the compiler should emit whichever property declaration will be in force in the
 current translation unit.
@@ -1624,24 +1845,24 @@ The BUCKETS are an array of offsets to DATA for each hash:
 
 So for ``bucket[3]`` in the example above, we have an offset into the table
 0x000034f0 which points to a chain of entries for the bucket.  Each bucket must
-contain a next pointer, full 32 bit hash value, the string itself, and the data
+contain a next pointer, full 32-bit hash value, the string itself, and the data
 for the current string value.
 
 .. code-block:: none
 
               .------------.
   0x000034f0: | 0x00003500 | next pointer
-              | 0x12345678 | 32 bit hash
+              | 0x12345678 | 32-bit hash
               | "erase"    | string value
               | data[n]    | HashData for this bucket
               |------------|
   0x00003500: | 0x00003550 | next pointer
-              | 0x29273623 | 32 bit hash
+              | 0x29273623 | 32-bit hash
               | "dump"     | string value
               | data[n]    | HashData for this bucket
               |------------|
   0x00003550: | 0x00000000 | next pointer
-              | 0x82638293 | 32 bit hash
+              | 0x82638293 | 32-bit hash
               | "main"     | string value
               | data[n]    | HashData for this bucket
               `------------'
@@ -1650,17 +1871,17 @@ The problem with this layout for debuggers is that we need to optimize for the
 negative lookup case where the symbol we're searching for is not present.  So
 if we were to lookup "``printf``" in the table above, we would make a 32-bit
 hash for "``printf``", it might match ``bucket[3]``.  We would need to go to
-the offset 0x000034f0 and start looking to see if our 32 bit hash matches.  To
+the offset 0x000034f0 and start looking to see if our 32-bit hash matches.  To
 do so, we need to read the next pointer, then read the hash, compare it, and
 skip to the next bucket.  Each time we are skipping many bytes in memory and
-touching new pages just to do the compare on the full 32 bit hash.  All of
+touching new pages just to do the compare on the full 32-bit hash.  All of
 these accesses then tell us that we didn't have a match.
 
 Name Hash Tables
 """"""""""""""""
 
-To solve the issues mentioned above we have structured the hash tables a bit
-differently: a header, buckets, an array of all unique 32 bit hash values,
+To solve the issues mentioned above, we have structured the hash tables a bit
+differently: a header, buckets, an array of all unique 32-bit hash values,
 followed by an array of hash value data offsets, one for each hash value, then
 the data for all hash values:
 
@@ -1679,11 +1900,11 @@ the data for all hash values:
   `-------------'
 
 The ``BUCKETS`` in the name tables are an index into the ``HASHES`` array.  By
-making all of the full 32 bit hash values contiguous in memory, we allow
+making all of the full 32-bit hash values contiguous in memory, we allow
 ourselves to efficiently check for a match while touching as little memory as
-possible.  Most often checking the 32 bit hash values is as far as the lookup
+possible.  Most often checking the 32-bit hash values is as far as the lookup
 goes.  If it does match, it usually is a match with no collisions.  So for a
-table with "``n_buckets``" buckets, and "``n_hashes``" unique 32 bit hash
+table with "``n_buckets``" buckets, and "``n_hashes``" unique 32-bit hash
 values, we can clarify the contents of the ``BUCKETS``, ``HASHES`` and
 ``OFFSETS`` as:
 
@@ -1698,16 +1919,16 @@ values, we can clarify the contents of the ``BUCKETS``, ``HASHES`` and
   |  HEADER.header_data_len | uint32_t
   |  HEADER_DATA            | HeaderData
   |-------------------------|
-  |  BUCKETS                | uint32_t[n_buckets] // 32 bit hash indexes
+  |  BUCKETS                | uint32_t[n_buckets] // 32-bit hash indexes
   |-------------------------|
-  |  HASHES                 | uint32_t[n_hashes] // 32 bit hash values
+  |  HASHES                 | uint32_t[n_hashes] // 32-bit hash values
   |-------------------------|
-  |  OFFSETS                | uint32_t[n_hashes] // 32 bit offsets to hash value data
+  |  OFFSETS                | uint32_t[n_hashes] // 32-bit offsets to hash value data
   |-------------------------|
   |  ALL HASH DATA          |
   `-------------------------'
 
-So taking the exact same data from the standard hash example above we end up
+So taking the exact same data from the standard hash example above, we end up
 with:
 
 .. code-block:: none
@@ -1761,7 +1982,7 @@ with:
               |            |
               |------------|
   0x000034f0: | 0x00001203 | .debug_str ("erase")
-              | 0x00000004 | A 32 bit array count - number of HashData with name "erase"
+              | 0x00000004 | A 32-bit array count - number of HashData with name "erase"
               | 0x........ | HashData[0]
               | 0x........ | HashData[1]
               | 0x........ | HashData[2]
@@ -1769,18 +1990,18 @@ with:
               | 0x00000000 | String offset into .debug_str (terminate data for hash)
               |------------|
   0x00003500: | 0x00001203 | String offset into .debug_str ("collision")
-              | 0x00000002 | A 32 bit array count - number of HashData with name "collision"
+              | 0x00000002 | A 32-bit array count - number of HashData with name "collision"
               | 0x........ | HashData[0]
               | 0x........ | HashData[1]
               | 0x00001203 | String offset into .debug_str ("dump")
-              | 0x00000003 | A 32 bit array count - number of HashData with name "dump"
+              | 0x00000003 | A 32-bit array count - number of HashData with name "dump"
               | 0x........ | HashData[0]
               | 0x........ | HashData[1]
               | 0x........ | HashData[2]
               | 0x00000000 | String offset into .debug_str (terminate data for hash)
               |------------|
   0x00003550: | 0x00001203 | String offset into .debug_str ("main")
-              | 0x00000009 | A 32 bit array count - number of HashData with name "main"
+              | 0x00000009 | A 32-bit array count - number of HashData with name "main"
               | 0x........ | HashData[0]
               | 0x........ | HashData[1]
               | 0x........ | HashData[2]
@@ -1795,13 +2016,13 @@ with:
 
 So we still have all of the same data, we just organize it more efficiently for
 debugger lookup.  If we repeat the same "``printf``" lookup from above, we
-would hash "``printf``" and find it matches ``BUCKETS[3]`` by taking the 32 bit
+would hash "``printf``" and find it matches ``BUCKETS[3]`` by taking the 32-bit
 hash value and modulo it by ``n_buckets``.  ``BUCKETS[3]`` contains "6" which
 is the index into the ``HASHES`` table.  We would then compare any consecutive
-32 bit hashes values in the ``HASHES`` array as long as the hashes would be in
+32-bit hash values in the ``HASHES`` array as long as the hashes would be in
 ``BUCKETS[3]``.  We do this by verifying that each subsequent hash value modulo
 ``n_buckets`` is still 3.  In the case of a failed lookup we would access the
-memory for ``BUCKETS[3]``, and then compare a few consecutive 32 bit hashes
+memory for ``BUCKETS[3]``, and then compare a few consecutive 32-bit hashes
 before we know that we have no match.  We don't end up marching through
 multiple words of memory and we really keep the number of processor data cache
 lines being accessed as small as possible.
@@ -1842,10 +2063,10 @@ header is:
     HeaderData header_data;     // Implementation specific header data
   };
 
-The header starts with a 32 bit "``magic``" value which must be ``'HASH'``
+The header starts with a 32-bit "``magic``" value which must be ``'HASH'``
 encoded as an ASCII integer.  This allows the detection of the start of the
 hash table and also allows the table's byte order to be determined so the table
-can be correctly extracted.  The "``magic``" value is followed by a 16 bit
+can be correctly extracted.  The "``magic``" value is followed by a 16-bit
 ``version`` number which allows the table to be revised and modified in the
 future.  The current version number is 1. ``hash_function`` is a ``uint16_t``
 enumeration that specifies which hash function was used to produce this table.
@@ -1858,8 +2079,8 @@ The current values for the hash function enumerations include:
     eHashFunctionDJB = 0u, // Daniel J Bernstein hash function
   };
 
-``bucket_count`` is a 32 bit unsigned integer that represents how many buckets
-are in the ``BUCKETS`` array.  ``hashes_count`` is the number of unique 32 bit
+``bucket_count`` is a 32-bit unsigned integer that represents how many buckets
+are in the ``BUCKETS`` array.  ``hashes_count`` is the number of unique 32-bit
 hash values that are in the ``HASHES`` array, and is the same number of offsets
 are contained in the ``OFFSETS`` array.  ``header_data_len`` specifies the size
 in bytes of the ``HeaderData`` that is filled in by specialized versions of
@@ -1875,12 +2096,12 @@ The header is followed by the buckets, hashes, offsets, and hash value data.
   struct FixedTable
   {
     uint32_t buckets[Header.bucket_count];  // An array of hash indexes into the "hashes[]" array below
-    uint32_t hashes [Header.hashes_count];  // Every unique 32 bit hash for the entire table is in this table
+    uint32_t hashes [Header.hashes_count];  // Every unique 32-bit hash for the entire table is in this table
     uint32_t offsets[Header.hashes_count];  // An offset that corresponds to each item in the "hashes[]" array above
   };
 
-``buckets`` is an array of 32 bit indexes into the ``hashes`` array.  The
-``hashes`` array contains all of the 32 bit hash values for all names in the
+``buckets`` is an array of 32-bit indexes into the ``hashes`` array.  The
+``hashes`` array contains all of the 32-bit hash values for all names in the
 hash table.  Each hash in the ``hashes`` table has an offset in the ``offsets``
 array that points to the data for the hash value.
 
@@ -1966,23 +2187,23 @@ array to be:
   HeaderData.atoms[0].type = eAtomTypeDIEOffset;
   HeaderData.atoms[0].form = DW_FORM_data4;
 
-This defines the contents to be the DIE offset (eAtomTypeDIEOffset) that is
-encoded as a 32 bit value (DW_FORM_data4).  This allows a single name to have
+This defines the contents to be the DIE offset (``eAtomTypeDIEOffset``) that is
+encoded as a 32-bit value (``DW_FORM_data4``).  This allows a single name to have
 multiple matching DIEs in a single file, which could come up with an inlined
 function for instance.  Future tables could include more information about the
 DIE such as flags indicating if the DIE is a function, method, block,
 or inlined.
 
-The KeyType for the DWARF table is a 32 bit string table offset into the
+The KeyType for the DWARF table is a 32-bit string table offset into the
 ".debug_str" table.  The ".debug_str" is the string table for the DWARF which
 may already contain copies of all of the strings.  This helps make sure, with
 help from the compiler, that we reuse the strings between all of the DWARF
 sections and keeps the hash table size down.  Another benefit to having the
-compiler generate all strings as DW_FORM_strp in the debug info, is that
+compiler generate all strings as ``DW_FORM_strp`` in the debug info, is that
 DWARF parsing can be made much faster.
 
 After a lookup is made, we get an offset into the hash data.  The hash data
-needs to be able to deal with 32 bit hash collisions, so the chunk of data
+needs to be able to deal with 32-bit hash collisions, so the chunk of data
 at the offset in the hash data consists of a triple:
 
 .. code-block:: c
@@ -1992,7 +2213,7 @@ at the offset in the hash data consists of a triple:
   HashData[hash_data_count]
 
 If "str_offset" is zero, then the bucket contents are done. 99.9% of the
-hash data chunks contain a single item (no 32 bit hash collision):
+hash data chunks contain a single item (no 32-bit hash collision):
 
 .. code-block:: none
 
@@ -2025,7 +2246,7 @@ If there are collisions, you will have multiple valid string offsets:
   `------------'
 
 Current testing with real world C++ binaries has shown that there is around 1
-32 bit hash collision per 100,000 name entries.
+32-bit hash collision per 100,000 name entries.
 
 Contents
 ^^^^^^^^
@@ -2114,7 +2335,7 @@ We get a few type DIEs:
                   AT_type( {0x00000067} ( int ) )
                   AT_byte_size( 0x08 )
 
-The DW_TAG_pointer_type is not included because it does not have a ``DW_AT_name``.
+The ``DW_TAG_pointer_type`` is not included because it does not have a ``DW_AT_name``.
 
 "``.apple_namespaces``" section should contain all ``DW_TAG_namespace`` DIEs.
 If we run into a namespace that has no name this is an anonymous namespace, and
diff --git a/llvm/docs/TestingGuide.rst b/llvm/docs/TestingGuide.rst
index b1819c795bab4..f79d7eacd5b6c 100644
--- a/llvm/docs/TestingGuide.rst
+++ b/llvm/docs/TestingGuide.rst
@@ -834,7 +834,7 @@ Besides replacing LLVM tool names, the following substitutions are performed in
    Invokes the Clang driver.
 
 ``%clang_cpp``
-   Invokes the Clang driver for C++.
+   Invokes the Clang driver as the preprocessor.
 
 ``%clang_cl``
    Invokes the CL-compatible Clang driver.
diff --git a/llvm/docs/TransformMetadata.rst b/llvm/docs/TransformMetadata.rst
index 3c0e10b3eb7a5..1f9f50f1f33bc 100644
--- a/llvm/docs/TransformMetadata.rst
+++ b/llvm/docs/TransformMetadata.rst
@@ -346,6 +346,18 @@ It is recommended to add ``llvm.loop.disable_nonforced`` to
 fallback version (which is likely never executed) is further optimized
 which would increase the code size.
 
+Attributes defined in ``llvm.loop.isdistributed`` are added to successfully
+distributed loops to prevent subsequent reprocessing.
+
+As an example, the following instructs a loop to be ignored during
+loop distribution.
+
+.. code-block:: llvm
+
+    !4 = distinct !{!4, !5, !6}
+    !5 = !{!"llvm.loop.mustprogress"}
+    !6 = !{!"llvm.loop.isdistributed", i32 1}
+
 Versioning LICM
 ---------------
 
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index d645646289025..d02cf984167c9 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -111,6 +111,7 @@ typedef enum {
   LLVMFPTrunc        = 37,
   LLVMFPExt          = 38,
   LLVMPtrToInt       = 39,
+  LLVMPtrToAddr      = 69,
   LLVMIntToPtr       = 40,
   LLVMBitCast        = 41,
   LLVMAddrSpaceCast  = 60,
@@ -2693,6 +2694,14 @@ LLVM_C_ABI void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes);
 LLVM_C_ABI void LLVMGlobalSetMetadata(LLVMValueRef Global, unsigned Kind,
                                       LLVMMetadataRef MD);
 
+/**
+ * Adds a metadata attachment.
+ *
+ * @see llvm::GlobalObject::addMetadata()
+ */
+LLVM_C_ABI void LLVMGlobalAddMetadata(LLVMValueRef Global, unsigned Kind,
+                                      LLVMMetadataRef MD);
+
 /**
  * Erases a metadata attachment of the given kind if it exists.
  *
@@ -2707,6 +2716,14 @@ LLVM_C_ABI void LLVMGlobalEraseMetadata(LLVMValueRef Global, unsigned Kind);
  */
 LLVM_C_ABI void LLVMGlobalClearMetadata(LLVMValueRef Global);
 
+/**
+ * Add debuginfo metadata to this global.
+ *
+ * @see llvm::GlobalVariable::addDebugInfo()
+ */
+LLVM_C_ABI void LLVMGlobalAddDebugInfo(LLVMValueRef Global,
+                                       LLVMMetadataRef GVE);
+
 /**
  * Retrieves an array of metadata entries representing the metadata attached to
  * this value. The caller is responsible for freeing this array by calling
diff --git a/llvm/include/llvm/ADT/APFloat.h b/llvm/include/llvm/ADT/APFloat.h
index e1589544787cf..6bc62845fddf4 100644
--- a/llvm/include/llvm/ADT/APFloat.h
+++ b/llvm/include/llvm/ADT/APFloat.h
@@ -460,12 +460,6 @@ class IEEEFloat final {
   LLVM_ABI opStatus convertToInteger(MutableArrayRef<integerPart>, unsigned int,
                                      bool, roundingMode, bool *) const;
   LLVM_ABI opStatus convertFromAPInt(const APInt &, bool, roundingMode);
-  LLVM_ABI opStatus convertFromSignExtendedInteger(const integerPart *,
-                                                   unsigned int, bool,
-                                                   roundingMode);
-  LLVM_ABI opStatus convertFromZeroExtendedInteger(const integerPart *,
-                                                   unsigned int, bool,
-                                                   roundingMode);
   LLVM_ABI Expected<opStatus> convertFromString(StringRef, roundingMode);
   LLVM_ABI APInt bitcastToAPInt() const;
   LLVM_ABI double convertToDouble() const;
@@ -605,33 +599,10 @@ class IEEEFloat final {
                          unsigned FormatMaxPadding = 3,
                          bool TruncateZero = true) const;
 
-  /// If this value has an exact multiplicative inverse, store it in inv and
-  /// return true.
-  LLVM_ABI bool getExactInverse(APFloat *inv) const;
-
-  // If this is an exact power of two, return the exponent while ignoring the
-  // sign bit. If it's not an exact power of 2, return INT_MIN
   LLVM_ABI LLVM_READONLY int getExactLog2Abs() const;
 
-  // If this is an exact power of two, return the exponent. If it's not an exact
-  // power of 2, return INT_MIN
-  LLVM_READONLY
-  int getExactLog2() const {
-    return isNegative() ? INT_MIN : getExactLog2Abs();
-  }
-
-  /// Returns the exponent of the internal representation of the APFloat.
-  ///
-  /// Because the radix of APFloat is 2, this is equivalent to floor(log2(x)).
-  /// For special APFloat values, this returns special error codes:
-  ///
-  ///   NaN -> \c IEK_NaN
-  ///   0   -> \c IEK_Zero
-  ///   Inf -> \c IEK_Inf
-  ///
   LLVM_ABI friend int ilogb(const IEEEFloat &Arg);
 
-  /// Returns: X * 2^Exp for integral exponents.
   LLVM_ABI friend IEEEFloat scalbn(IEEEFloat X, int Exp, roundingMode);
 
   LLVM_ABI friend IEEEFloat frexp(const IEEEFloat &X, int &Exp, roundingMode);
@@ -824,6 +795,19 @@ class DoubleAPFloat final {
 
   opStatus addWithSpecial(const DoubleAPFloat &LHS, const DoubleAPFloat &RHS,
                           DoubleAPFloat &Out, roundingMode RM);
+  opStatus convertToSignExtendedInteger(MutableArrayRef<integerPart> Input,
+                                        unsigned int Width, bool IsSigned,
+                                        roundingMode RM, bool *IsExact) const;
+
+  // Convert an unsigned integer Src to a floating point number,
+  // rounding according to RM.  The sign of the floating point number is not
+  // modified.
+  opStatus convertFromUnsignedParts(const integerPart *Src,
+                                    unsigned int SrcCount, roundingMode RM);
+
+  // Handle overflow.  Sign is preserved.  We either become infinity or
+  // the largest finite number.
+  opStatus handleOverflow(roundingMode RM);
 
 public:
   LLVM_ABI DoubleAPFloat(const fltSemantics &S);
@@ -880,14 +864,6 @@ class DoubleAPFloat final {
                                      roundingMode RM, bool *IsExact) const;
   LLVM_ABI opStatus convertFromAPInt(const APInt &Input, bool IsSigned,
                                      roundingMode RM);
-  LLVM_ABI opStatus convertFromSignExtendedInteger(const integerPart *Input,
-                                                   unsigned int InputSize,
-                                                   bool IsSigned,
-                                                   roundingMode RM);
-  LLVM_ABI opStatus convertFromZeroExtendedInteger(const integerPart *Input,
-                                                   unsigned int InputSize,
-                                                   bool IsSigned,
-                                                   roundingMode RM);
   LLVM_ABI unsigned int convertToHexString(char *DST, unsigned int HexDigits,
                                            bool UpperCase,
                                            roundingMode RM) const;
@@ -902,11 +878,9 @@ class DoubleAPFloat final {
                          unsigned FormatMaxPadding,
                          bool TruncateZero = true) const;
 
-  LLVM_ABI bool getExactInverse(APFloat *inv) const;
-
-  LLVM_ABI LLVM_READONLY int getExactLog2() const;
   LLVM_ABI LLVM_READONLY int getExactLog2Abs() const;
 
+  LLVM_ABI friend int ilogb(const DoubleAPFloat &X);
   LLVM_ABI friend DoubleAPFloat scalbn(const DoubleAPFloat &X, int Exp,
                                        roundingMode);
   LLVM_ABI friend DoubleAPFloat frexp(const DoubleAPFloat &X, int &Exp,
@@ -1067,6 +1041,8 @@ class APFloat : public APFloatBase {
   explicit APFloat(DoubleAPFloat F, const fltSemantics &S)
       : U(std::move(F), S) {}
 
+  // Compares the absolute value of this APFloat with another.  Both operands
+  // must be finite non-zero.
   cmpResult compareAbsoluteValue(const APFloat &RHS) const {
     assert(&getSemantics() == &RHS.getSemantics() &&
            "Should only compare APFloats with the same semantics");
@@ -1345,30 +1321,34 @@ class APFloat : public APFloatBase {
 
   LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM,
                             bool *losesInfo);
+  // Convert a floating point number to an integer according to the
+  // rounding mode.  We provide deterministic values in case of an invalid
+  // operation exception, namely zero for NaNs and the minimal or maximal value
+  // respectively for underflow or overflow.
+  // The *IsExact output tells whether the result is exact, in the sense that
+  // converting it back to the original floating point type produces the
+  // original value.  This is almost equivalent to result==opOK, except for
+  // negative zeroes.
   opStatus convertToInteger(MutableArrayRef<integerPart> Input,
                             unsigned int Width, bool IsSigned, roundingMode RM,
                             bool *IsExact) const {
     APFLOAT_DISPATCH_ON_SEMANTICS(
         convertToInteger(Input, Width, IsSigned, RM, IsExact));
   }
+  // Same as convertToInteger(integerPart*, ...), except the result is returned
+  // in an APSInt, whose initial bit-width and signed-ness are used to determine
+  // the precision of the conversion.
   LLVM_ABI opStatus convertToInteger(APSInt &Result, roundingMode RM,
                                      bool *IsExact) const;
+
+  // Convert a two's complement integer Input to a floating point number,
+  // rounding according to RM.  IsSigned is true if the integer is signed,
+  // in which case it must be sign-extended.
   opStatus convertFromAPInt(const APInt &Input, bool IsSigned,
                             roundingMode RM) {
     APFLOAT_DISPATCH_ON_SEMANTICS(convertFromAPInt(Input, IsSigned, RM));
   }
-  opStatus convertFromSignExtendedInteger(const integerPart *Input,
-                                          unsigned int InputSize, bool IsSigned,
-                                          roundingMode RM) {
-    APFLOAT_DISPATCH_ON_SEMANTICS(
-        convertFromSignExtendedInteger(Input, InputSize, IsSigned, RM));
-  }
-  opStatus convertFromZeroExtendedInteger(const integerPart *Input,
-                                          unsigned int InputSize, bool IsSigned,
-                                          roundingMode RM) {
-    APFLOAT_DISPATCH_ON_SEMANTICS(
-        convertFromZeroExtendedInteger(Input, InputSize, IsSigned, RM));
-  }
+
   LLVM_ABI Expected<opStatus> convertFromString(StringRef, roundingMode);
   APInt bitcastToAPInt() const {
     APFLOAT_DISPATCH_ON_SEMANTICS(bitcastToAPInt());
@@ -1419,6 +1399,8 @@ class APFloat : public APFloatBase {
     return Res == cmpGreaterThan || Res == cmpEqual;
   }
 
+  // IEEE comparison with another floating point number (NaNs compare unordered,
+  // 0==-0).
   cmpResult compare(const APFloat &RHS) const {
     assert(&getSemantics() == &RHS.getSemantics() &&
            "Should only compare APFloats with the same semantics");
@@ -1505,22 +1487,26 @@ class APFloat : public APFloatBase {
   LLVM_DUMP_METHOD void dump() const;
 #endif
 
-  bool getExactInverse(APFloat *inv) const {
-    APFLOAT_DISPATCH_ON_SEMANTICS(getExactInverse(inv));
-  }
+  /// If this value is normal and has an exact, normal, multiplicative inverse,
+  /// store it in inv and return true.
+  bool getExactInverse(APFloat *Inv) const;
 
+  // If this is an exact power of two, return the exponent while ignoring the
+  // sign bit. If it's not an exact power of 2, return INT_MIN
   LLVM_READONLY
   int getExactLog2Abs() const {
     APFLOAT_DISPATCH_ON_SEMANTICS(getExactLog2Abs());
   }
 
+  // If this is an exact power of two, return the exponent. If it's not an exact
+  // power of 2, return INT_MIN
   LLVM_READONLY
   int getExactLog2() const {
-    APFLOAT_DISPATCH_ON_SEMANTICS(getExactLog2());
+    return isNegative() ? INT_MIN : getExactLog2Abs();
   }
 
   LLVM_ABI friend hash_code hash_value(const APFloat &Arg);
-  friend int ilogb(const APFloat &Arg) { return ilogb(Arg.getIEEE()); }
+  friend int ilogb(const APFloat &Arg);
   friend APFloat scalbn(APFloat X, int Exp, roundingMode RM);
   friend APFloat frexp(const APFloat &X, int &Exp, roundingMode RM);
   friend IEEEFloat;
@@ -1535,6 +1521,25 @@ static_assert(sizeof(APFloat) == sizeof(detail::IEEEFloat),
 /// These additional declarations are required in order to compile LLVM with IBM
 /// xlC compiler.
 LLVM_ABI hash_code hash_value(const APFloat &Arg);
+
+/// Returns the exponent of the internal representation of the APFloat.
+///
+/// Because the radix of APFloat is 2, this is equivalent to floor(log2(x)).
+/// For special APFloat values, this returns special error codes:
+///
+///   NaN -> \c IEK_NaN
+///   0   -> \c IEK_Zero
+///   Inf -> \c IEK_Inf
+///
+inline int ilogb(const APFloat &Arg) {
+  if (APFloat::usesLayout<detail::IEEEFloat>(Arg.getSemantics()))
+    return ilogb(Arg.U.IEEE);
+  if (APFloat::usesLayout<detail::DoubleAPFloat>(Arg.getSemantics()))
+    return ilogb(Arg.U.Double);
+  llvm_unreachable("Unexpected semantics");
+}
+
+/// Returns: X * 2^Exp for integral exponents.
 inline APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM) {
   if (APFloat::usesLayout<detail::IEEEFloat>(X.getSemantics()))
     return APFloat(scalbn(X.U.IEEE, Exp, RM), X.getSemantics());
diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index 295506393a1c4..e68dc8bd14d96 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -2294,6 +2294,12 @@ LLVM_ABI APInt mulhs(const APInt &C1, const APInt &C2);
 /// Returns the high N bits of the multiplication result.
 LLVM_ABI APInt mulhu(const APInt &C1, const APInt &C2);
 
+/// Performs (2*N)-bit multiplication on sign-extended operands.
+LLVM_ABI APInt mulsExtended(const APInt &C1, const APInt &C2);
+
+/// Performs (2*N)-bit multiplication on zero-extended operands.
+LLVM_ABI APInt muluExtended(const APInt &C1, const APInt &C2);
+
 /// Compute X^N for N>=0.
 /// 0^0 is supported and returns 1.
 LLVM_ABI APInt pow(const APInt &X, int64_t N);
@@ -2404,6 +2410,35 @@ LLVM_ABI std::optional<unsigned> GetMostSignificantDifferentBit(const APInt &A,
 /// A.getBitwidth() or NewBitWidth must be a whole multiples of the other.
 LLVM_ABI APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth,
                             bool MatchAllBits = false);
+
+/// Perform a funnel shift left.
+///
+/// Concatenate Hi and Lo (Hi is the most significant bits of the wide value),
+/// the combined value is shifted left by Shift (modulo the bit width of the
+/// original arguments), and the most significant bits are extracted to produce
+/// a result that is the same size as the original arguments.
+///
+/// Examples:
+/// (1) fshl(i8 255, i8 0, i8 15) = 128 (0b10000000)
+/// (2) fshl(i8 15, i8 15, i8 11) = 120 (0b01111000)
+/// (3) fshl(i8 0, i8 255, i8 8)  = 0   (0b00000000)
+/// (4) fshl(i8 255, i8 0, i8 15) = fshl(i8 255, i8 0, i8 7) // 15 % 8
+LLVM_ABI APInt fshl(const APInt &Hi, const APInt &Lo, const APInt &Shift);
+
+/// Perform a funnel shift right.
+///
+/// Concatenate Hi and Lo (Hi is the most significant bits of the wide value),
+/// the combined value is shifted right by Shift (modulo the bit width of the
+/// original arguments), and the least significant bits are extracted to produce
+/// a result that is the same size as the original arguments.
+///
+/// Examples:
+/// (1) fshr(i8 255, i8 0, i8 15) = 254 (0b11111110)
+/// (2) fshr(i8 15, i8 15, i8 11) = 225 (0b11100001)
+/// (3) fshr(i8 0, i8 255, i8 8)  = 255 (0b11111111)
+/// (4) fshr(i8 255, i8 0, i8 9)  = fshr(i8 255, i8 0, i8 1) // 9 % 8
+LLVM_ABI APInt fshr(const APInt &Hi, const APInt &Lo, const APInt &Shift);
+
 } // namespace APIntOps
 
 // See friend declaration above. This additional declaration is required in
diff --git a/llvm/include/llvm/ADT/DenseMap.h b/llvm/include/llvm/ADT/DenseMap.h
index ea5eac4cc63f0..ab1bc6356dcb9 100644
--- a/llvm/include/llvm/ADT/DenseMap.h
+++ b/llvm/include/llvm/ADT/DenseMap.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/EpochTracker.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/Support/AlignOf.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
@@ -256,22 +257,12 @@ class DenseMapBase : public DebugEpochBase {
   std::pair<iterator, bool> try_emplace(KeyT &&Key, Ts &&...Args) {
     BucketT *TheBucket;
     if (LookupBucketFor(Key, TheBucket))
-      return std::make_pair(makeIterator(TheBucket,
-                                         shouldReverseIterate<KeyT>()
-                                             ? getBuckets()
-                                             : getBucketsEnd(),
-                                         *this, true),
-                            false); // Already in map.
+      return {makeInsertIterator(TheBucket), false}; // Already in map.
 
     // Otherwise, insert the new element.
     TheBucket =
         InsertIntoBucket(TheBucket, std::move(Key), std::forward<Ts>(Args)...);
-    return std::make_pair(makeIterator(TheBucket,
-                                       shouldReverseIterate<KeyT>()
-                                           ? getBuckets()
-                                           : getBucketsEnd(),
-                                       *this, true),
-                          true);
+    return {makeInsertIterator(TheBucket), true};
   }
 
   // Inserts key,value pair into the map if the key isn't already in the map.
@@ -281,21 +272,11 @@ class DenseMapBase : public DebugEpochBase {
   std::pair<iterator, bool> try_emplace(const KeyT &Key, Ts &&...Args) {
     BucketT *TheBucket;
     if (LookupBucketFor(Key, TheBucket))
-      return std::make_pair(makeIterator(TheBucket,
-                                         shouldReverseIterate<KeyT>()
-                                             ? getBuckets()
-                                             : getBucketsEnd(),
-                                         *this, true),
-                            false); // Already in map.
+      return {makeInsertIterator(TheBucket), false}; // Already in map.
 
     // Otherwise, insert the new element.
     TheBucket = InsertIntoBucket(TheBucket, Key, std::forward<Ts>(Args)...);
-    return std::make_pair(makeIterator(TheBucket,
-                                       shouldReverseIterate<KeyT>()
-                                           ? getBuckets()
-                                           : getBucketsEnd(),
-                                       *this, true),
-                          true);
+    return {makeInsertIterator(TheBucket), true};
   }
 
   /// Alternate version of insert() which allows a different, and possibly
@@ -308,22 +289,12 @@ class DenseMapBase : public DebugEpochBase {
                                       const LookupKeyT &Val) {
     BucketT *TheBucket;
     if (LookupBucketFor(Val, TheBucket))
-      return std::make_pair(makeIterator(TheBucket,
-                                         shouldReverseIterate<KeyT>()
-                                             ? getBuckets()
-                                             : getBucketsEnd(),
-                                         *this, true),
-                            false); // Already in map.
+      return {makeInsertIterator(TheBucket), false}; // Already in map.
 
     // Otherwise, insert the new element.
     TheBucket = InsertIntoBucketWithLookup(TheBucket, std::move(KV.first),
                                            std::move(KV.second), Val);
-    return std::make_pair(makeIterator(TheBucket,
-                                       shouldReverseIterate<KeyT>()
-                                           ? getBuckets()
-                                           : getBucketsEnd(),
-                                       *this, true),
-                          true);
+    return {makeInsertIterator(TheBucket), true};
   }
 
   /// insert - Range insertion of pairs.
@@ -454,28 +425,28 @@ class DenseMapBase : public DebugEpochBase {
     return NextPowerOf2(NumEntries * 4 / 3 + 1);
   }
 
-  void moveFromOldBuckets(BucketT *OldBucketsBegin, BucketT *OldBucketsEnd) {
+  void moveFromOldBuckets(iterator_range<BucketT *> OldBuckets) {
     initEmpty();
 
     // Insert all the old elements.
     const KeyT EmptyKey = getEmptyKey();
     const KeyT TombstoneKey = getTombstoneKey();
-    for (BucketT *B = OldBucketsBegin, *E = OldBucketsEnd; B != E; ++B) {
-      if (!KeyInfoT::isEqual(B->getFirst(), EmptyKey) &&
-          !KeyInfoT::isEqual(B->getFirst(), TombstoneKey)) {
+    for (BucketT &B : OldBuckets) {
+      if (!KeyInfoT::isEqual(B.getFirst(), EmptyKey) &&
+          !KeyInfoT::isEqual(B.getFirst(), TombstoneKey)) {
         // Insert the key/value into the new table.
         BucketT *DestBucket;
-        bool FoundVal = LookupBucketFor(B->getFirst(), DestBucket);
+        bool FoundVal = LookupBucketFor(B.getFirst(), DestBucket);
         (void)FoundVal; // silence warning.
         assert(!FoundVal && "Key already in new map?");
-        DestBucket->getFirst() = std::move(B->getFirst());
-        ::new (&DestBucket->getSecond()) ValueT(std::move(B->getSecond()));
+        DestBucket->getFirst() = std::move(B.getFirst());
+        ::new (&DestBucket->getSecond()) ValueT(std::move(B.getSecond()));
         incrementNumEntries();
 
         // Free the value.
-        B->getSecond().~ValueT();
+        B.getSecond().~ValueT();
       }
-      B->getFirst().~KeyT();
+      B.getFirst().~KeyT();
     }
   }
 
@@ -544,6 +515,13 @@ class DenseMapBase : public DebugEpochBase {
     return const_iterator(P, E, Epoch, NoAdvance);
   }
 
+  iterator makeInsertIterator(BucketT *TheBucket) {
+    return makeIterator(TheBucket,
+                        shouldReverseIterate<KeyT>() ? getBuckets()
+                                                     : getBucketsEnd(),
+                        *this, true);
+  }
+
   unsigned getNumEntries() const {
     return static_cast<const DerivedT *>(this)->getNumEntries();
   }
@@ -811,10 +789,12 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
     this->insert(I, E);
   }
 
-  DenseMap(std::initializer_list<typename BaseT::value_type> Vals) {
-    init(Vals.size());
-    this->insert(Vals.begin(), Vals.end());
-  }
+  template <typename RangeT>
+  DenseMap(llvm::from_range_t, const RangeT &Range)
+      : DenseMap(adl_begin(Range), adl_end(Range)) {}
+
+  DenseMap(std::initializer_list<typename BaseT::value_type> Vals)
+      : DenseMap(Vals.begin(), Vals.end()) {}
 
   ~DenseMap() {
     this->destroyAll();
@@ -867,7 +847,8 @@ class DenseMap : public DenseMapBase<DenseMap<KeyT, ValueT, KeyInfoT, BucketT>,
       return;
     }
 
-    this->moveFromOldBuckets(OldBuckets, OldBuckets + OldNumBuckets);
+    this->moveFromOldBuckets(
+        llvm::make_range(OldBuckets, OldBuckets + OldNumBuckets));
 
     // Free the old table.
     deallocate_buffer(OldBuckets, sizeof(BucketT) * OldNumBuckets,
@@ -952,6 +933,9 @@ class SmallDenseMap
   struct LargeRep {
     BucketT *Buckets;
     unsigned NumBuckets;
+    iterator_range<BucketT *> buckets() {
+      return llvm::make_range(Buckets, Buckets + NumBuckets);
+    }
   };
 
   /// A "union" of an inline bucket array and the struct representing
@@ -981,6 +965,10 @@ class SmallDenseMap
     this->insert(I, E);
   }
 
+  template <typename RangeT>
+  SmallDenseMap(llvm::from_range_t, const RangeT &Range)
+      : SmallDenseMap(adl_begin(Range), adl_end(Range)) {}
+
   SmallDenseMap(std::initializer_list<typename BaseT::value_type> Vals)
       : SmallDenseMap(Vals.begin(), Vals.end()) {}
 
@@ -1027,8 +1015,7 @@ class SmallDenseMap
       return;
     }
     if (!Small && !RHS.Small) {
-      std::swap(getLargeRep()->Buckets, RHS.getLargeRep()->Buckets);
-      std::swap(getLargeRep()->NumBuckets, RHS.getLargeRep()->NumBuckets);
+      std::swap(*getLargeRep(), *RHS.getLargeRep());
       return;
     }
 
@@ -1129,7 +1116,7 @@ class SmallDenseMap
         Small = false;
         new (getLargeRep()) LargeRep(allocateBuckets(AtLeast));
       }
-      this->moveFromOldBuckets(TmpBegin, TmpEnd);
+      this->moveFromOldBuckets(llvm::make_range(TmpBegin, TmpEnd));
       return;
     }
 
@@ -1141,8 +1128,7 @@ class SmallDenseMap
       new (getLargeRep()) LargeRep(allocateBuckets(AtLeast));
     }
 
-    this->moveFromOldBuckets(OldRep.Buckets,
-                             OldRep.Buckets + OldRep.NumBuckets);
+    this->moveFromOldBuckets(OldRep.buckets());
 
     // Free the old table.
     deallocate_buffer(OldRep.Buckets, sizeof(BucketT) * OldRep.NumBuckets,
diff --git a/llvm/include/llvm/ADT/DenseSet.h b/llvm/include/llvm/ADT/DenseSet.h
index 5642f1400bc39..e6051f79d5710 100644
--- a/llvm/include/llvm/ADT/DenseSet.h
+++ b/llvm/include/llvm/ADT/DenseSet.h
@@ -104,95 +104,68 @@ class DenseSetImpl {
 
   void swap(DenseSetImpl &RHS) { TheMap.swap(RHS.TheMap); }
 
-  // Iterators.
-
-  class ConstIterator;
-
-  class Iterator {
-    typename MapTy::iterator I;
+private:
+  template <bool IsConst> class DenseSetIterator {
     friend class DenseSetImpl;
-    friend class ConstIterator;
-
-  public:
-    using difference_type = typename MapTy::iterator::difference_type;
-    using value_type = ValueT;
-    using pointer = value_type *;
-    using reference = value_type &;
-    using iterator_category = std::forward_iterator_tag;
 
-    Iterator() = default;
-    Iterator(const typename MapTy::iterator &i) : I(i) {}
-
-    ValueT &operator*() { return I->getFirst(); }
-    const ValueT &operator*() const { return I->getFirst(); }
-    ValueT *operator->() { return &I->getFirst(); }
-    const ValueT *operator->() const { return &I->getFirst(); }
-
-    Iterator &operator++() {
-      ++I;
-      return *this;
-    }
-    Iterator operator++(int) {
-      auto T = *this;
-      ++I;
-      return T;
-    }
-    friend bool operator==(const Iterator &X, const Iterator &Y) {
-      return X.I == Y.I;
-    }
-    friend bool operator!=(const Iterator &X, const Iterator &Y) {
-      return X.I != Y.I;
-    }
-  };
+    using MapIteratorT =
+        std::conditional_t<IsConst, typename MapTy::const_iterator,
+                           typename MapTy::iterator>;
 
-  class ConstIterator {
-    typename MapTy::const_iterator I;
-    friend class DenseSetImpl;
-    friend class Iterator;
+    MapIteratorT I;
 
   public:
-    using difference_type = typename MapTy::const_iterator::difference_type;
-    using value_type = ValueT;
-    using pointer = const value_type *;
-    using reference = const value_type &;
+    using difference_type = typename MapIteratorT::difference_type;
     using iterator_category = std::forward_iterator_tag;
+    using value_type = ValueT;
+    using pointer =
+        std::conditional_t<IsConst, const value_type *, value_type *>;
+    using reference =
+        std::conditional_t<IsConst, const value_type &, value_type &>;
 
-    ConstIterator() = default;
-    ConstIterator(const Iterator &B) : I(B.I) {}
-    ConstIterator(const typename MapTy::const_iterator &i) : I(i) {}
+    DenseSetIterator() = default;
+    DenseSetIterator(MapIteratorT I) : I(I) {}
 
-    const ValueT &operator*() const { return I->getFirst(); }
-    const ValueT *operator->() const { return &I->getFirst(); }
+    // Allow conversion from iterator to const_iterator.
+    template <bool C = IsConst, typename = std::enable_if_t<C>>
+    DenseSetIterator(const DenseSetIterator<false> &Other) : I(Other.I) {}
 
-    ConstIterator &operator++() {
+    reference operator*() const { return I->getFirst(); }
+    pointer operator->() const { return &I->getFirst(); }
+
+    DenseSetIterator &operator++() {
       ++I;
       return *this;
     }
-    ConstIterator operator++(int) {
+    DenseSetIterator operator++(int) {
       auto T = *this;
       ++I;
       return T;
     }
-    friend bool operator==(const ConstIterator &X, const ConstIterator &Y) {
-      return X.I == Y.I;
+
+    friend bool operator==(const DenseSetIterator &LHS,
+                           const DenseSetIterator &RHS) {
+      return LHS.I == RHS.I;
     }
-    friend bool operator!=(const ConstIterator &X, const ConstIterator &Y) {
-      return X.I != Y.I;
+    friend bool operator!=(const DenseSetIterator &LHS,
+                           const DenseSetIterator &RHS) {
+      return LHS.I != RHS.I;
     }
   };
 
-  using iterator = Iterator;
-  using const_iterator = ConstIterator;
+public:
+  using iterator = DenseSetIterator<false>;
+  using const_iterator = DenseSetIterator<true>;
 
-  iterator begin() { return Iterator(TheMap.begin()); }
-  iterator end() { return Iterator(TheMap.end()); }
+  iterator begin() { return iterator(TheMap.begin()); }
+  iterator end() { return iterator(TheMap.end()); }
 
-  const_iterator begin() const { return ConstIterator(TheMap.begin()); }
-  const_iterator end() const { return ConstIterator(TheMap.end()); }
+  const_iterator begin() const { return const_iterator(TheMap.begin()); }
+  const_iterator end() const { return const_iterator(TheMap.end()); }
 
-  iterator find(const_arg_type_t<ValueT> V) { return Iterator(TheMap.find(V)); }
+  iterator find(const_arg_type_t<ValueT> V) { return iterator(TheMap.find(V)); }
   const_iterator find(const_arg_type_t<ValueT> V) const {
-    return ConstIterator(TheMap.find(V));
+    return const_iterator(TheMap.find(V));
   }
 
   /// Check if the set contains the given element.
@@ -206,15 +179,15 @@ class DenseSetImpl {
   /// getHashValue(LookupKeyT) and isEqual(LookupKeyT, KeyT) for each key type
   /// used.
   template <class LookupKeyT> iterator find_as(const LookupKeyT &Val) {
-    return Iterator(TheMap.find_as(Val));
+    return iterator(TheMap.find_as(Val));
   }
   template <class LookupKeyT>
   const_iterator find_as(const LookupKeyT &Val) const {
-    return ConstIterator(TheMap.find_as(Val));
+    return const_iterator(TheMap.find_as(Val));
   }
 
-  void erase(Iterator I) { return TheMap.erase(I.I); }
-  void erase(ConstIterator CI) { return TheMap.erase(CI.I); }
+  void erase(iterator I) { return TheMap.erase(I.I); }
+  void erase(const_iterator CI) { return TheMap.erase(CI.I); }
 
   std::pair<iterator, bool> insert(const ValueT &V) {
     detail::DenseSetEmpty Empty;
diff --git a/llvm/include/llvm/ADT/MapVector.h b/llvm/include/llvm/ADT/MapVector.h
index 015605fd98dff..24976535716d1 100644
--- a/llvm/include/llvm/ADT/MapVector.h
+++ b/llvm/include/llvm/ADT/MapVector.h
@@ -123,9 +123,9 @@ class MapVector {
       It->second = Vector.size();
       Vector.emplace_back(std::piecewise_construct, std::forward_as_tuple(Key),
                           std::forward_as_tuple(std::forward<Ts>(Args)...));
-      return std::make_pair(std::prev(end()), true);
+      return {std::prev(end()), true};
     }
-    return std::make_pair(begin() + It->second, false);
+    return {begin() + It->second, false};
   }
   template <typename... Ts>
   std::pair<iterator, bool> try_emplace(KeyT &&Key, Ts &&...Args) {
@@ -135,9 +135,9 @@ class MapVector {
       Vector.emplace_back(std::piecewise_construct,
                           std::forward_as_tuple(std::move(Key)),
                           std::forward_as_tuple(std::forward<Ts>(Args)...));
-      return std::make_pair(std::prev(end()), true);
+      return {std::prev(end()), true};
     }
-    return std::make_pair(begin() + It->second, false);
+    return {begin() + It->second, false};
   }
 
   std::pair<iterator, bool> insert(const std::pair<KeyT, ValueT> &KV) {
diff --git a/llvm/include/llvm/ADT/SetVector.h b/llvm/include/llvm/ADT/SetVector.h
index 0692071adb391..85d4f2d5ee28a 100644
--- a/llvm/include/llvm/ADT/SetVector.h
+++ b/llvm/include/llvm/ADT/SetVector.h
@@ -313,9 +313,8 @@ class SetVector {
   bool set_union(const STy &S) {
     bool Changed = false;
 
-    for (typename STy::const_iterator SI = S.begin(), SE = S.end(); SI != SE;
-         ++SI)
-      if (insert(*SI))
+    for (const auto &Elem : S)
+      if (insert(Elem))
         Changed = true;
 
     return Changed;
@@ -326,9 +325,8 @@ class SetVector {
   ///       SetVector interface is inconsistent with DenseSet.
   template <class STy>
   void set_subtract(const STy &S) {
-    for (typename STy::const_iterator SI = S.begin(), SE = S.end(); SI != SE;
-         ++SI)
-      remove(*SI);
+    for (const auto &Elem : S)
+      remove(Elem);
   }
 
   void swap(SetVector<T, Vector, Set, N> &RHS) {
diff --git a/llvm/include/llvm/ADT/SmallPtrSet.h b/llvm/include/llvm/ADT/SmallPtrSet.h
index 892040ef4cab1..16ad3973e054d 100644
--- a/llvm/include/llvm/ADT/SmallPtrSet.h
+++ b/llvm/include/llvm/ADT/SmallPtrSet.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/ADL.h"
 #include "llvm/ADT/EpochTracker.h"
 #include "llvm/ADT/STLForwardCompat.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ReverseIteration.h"
@@ -46,7 +47,7 @@ namespace llvm {
 /// sets are often small.  In this case, no memory allocation is used, and only
 /// light-weight and cache-efficient scanning is used.
 ///
-/// Large sets use a classic exponentially-probed hash table.  Empty buckets are
+/// Large sets use a classic quadratically-probed hash table.  Empty buckets are
 /// represented with an illegal pointer value (-1) to allow null pointers to be
 /// inserted.  Tombstones are represented with another illegal pointer value
 /// (-2), to allow deletion.  The hash table is resized when the table is 3/4 or
@@ -61,10 +62,10 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   /// CurArraySize - The allocated size of CurArray, always a power of two.
   unsigned CurArraySize;
 
-  /// Number of elements in CurArray that contain a value or are a tombstone.
+  /// Number of elements in CurArray that contain a value.
   /// If small, all these elements are at the beginning of CurArray and the rest
   /// is uninitialized.
-  unsigned NumNonEmpty;
+  unsigned NumEntries;
   /// Number of tombstones in CurArray.
   unsigned NumTombstones;
   /// Whether the set is in small representation.
@@ -78,7 +79,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
                                SmallPtrSetImplBase &&that);
 
   explicit SmallPtrSetImplBase(const void **SmallStorage, unsigned SmallSize)
-      : CurArray(SmallStorage), CurArraySize(SmallSize), NumNonEmpty(0),
+      : CurArray(SmallStorage), CurArraySize(SmallSize), NumEntries(0),
         NumTombstones(0), IsSmall(true) {
     assert(llvm::has_single_bit(SmallSize) &&
            "Initial size must be a power of two!");
@@ -95,7 +96,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   SmallPtrSetImplBase &operator=(const SmallPtrSetImplBase &) = delete;
 
   [[nodiscard]] bool empty() const { return size() == 0; }
-  size_type size() const { return NumNonEmpty - NumTombstones; }
+  size_type size() const { return NumEntries; }
   size_type capacity() const { return CurArraySize; }
 
   void clear() {
@@ -109,26 +110,26 @@ class SmallPtrSetImplBase : public DebugEpochBase {
       memset(CurArray, -1, CurArraySize * sizeof(void *));
     }
 
-    NumNonEmpty = 0;
+    NumEntries = 0;
     NumTombstones = 0;
   }
 
-  void reserve(size_type NumEntries) {
+  void reserve(size_type NewNumEntries) {
     incrementEpoch();
     // Do nothing if we're given zero as a reservation size.
-    if (NumEntries == 0)
+    if (NewNumEntries == 0)
       return;
-    // No need to expand if we're small and NumEntries will fit in the space.
-    if (isSmall() && NumEntries <= CurArraySize)
+    // No need to expand if we're small and NewNumEntries will fit in the space.
+    if (isSmall() && NewNumEntries <= CurArraySize)
       return;
     // insert_imp_big will reallocate if stores is more than 75% full, on the
     // /final/ insertion.
-    if (!isSmall() && ((NumEntries - 1) * 4) < (CurArraySize * 3))
+    if (!isSmall() && ((NewNumEntries - 1) * 4) < (CurArraySize * 3))
       return;
     // We must Grow -- find the size where we'd be 75% full, then round up to
     // the next power of two.
-    size_type NewSize = NumEntries + (NumEntries / 3);
-    NewSize = 1 << (Log2_32_Ceil(NewSize) + 1);
+    size_type NewSize = NewNumEntries + (NewNumEntries / 3);
+    NewSize = llvm::bit_ceil(NewSize);
     // Like insert_imp_big, always allocate at least 128 elements.
     NewSize = std::max(128u, NewSize);
     Grow(NewSize);
@@ -144,7 +145,23 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   }
 
   const void **EndPointer() const {
-    return isSmall() ? CurArray + NumNonEmpty : CurArray + CurArraySize;
+    return isSmall() ? CurArray + NumEntries : CurArray + CurArraySize;
+  }
+
+  iterator_range<const void **> small_buckets() {
+    return make_range(CurArray, CurArray + NumEntries);
+  }
+
+  iterator_range<const void *const *> small_buckets() const {
+    return {CurArray, CurArray + NumEntries};
+  }
+
+  iterator_range<const void **> buckets() {
+    return make_range(CurArray, EndPointer());
+  }
+
+  iterator_range<const void *const *> buckets() const {
+    return make_range(CurArray, EndPointer());
   }
 
   /// insert_imp - This returns true if the pointer was new to the set, false if
@@ -153,18 +170,16 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   std::pair<const void *const *, bool> insert_imp(const void *Ptr) {
     if (isSmall()) {
       // Check to see if it is already in the set.
-      for (const void **APtr = CurArray, **E = CurArray + NumNonEmpty;
-           APtr != E; ++APtr) {
-        const void *Value = *APtr;
-        if (Value == Ptr)
-          return std::make_pair(APtr, false);
+      for (const void *&Bucket : small_buckets()) {
+        if (Bucket == Ptr)
+          return {&Bucket, false};
       }
 
       // Nope, there isn't.  If we stay small, just 'pushback' now.
-      if (NumNonEmpty < CurArraySize) {
-        CurArray[NumNonEmpty++] = Ptr;
+      if (NumEntries < CurArraySize) {
+        CurArray[NumEntries++] = Ptr;
         incrementEpoch();
-        return std::make_pair(CurArray + (NumNonEmpty - 1), true);
+        return {CurArray + (NumEntries - 1), true};
       }
       // Otherwise, hit the big set case, which will call grow.
     }
@@ -177,10 +192,9 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   /// in.
   bool erase_imp(const void * Ptr) {
     if (isSmall()) {
-      for (const void **APtr = CurArray, **E = CurArray + NumNonEmpty;
-           APtr != E; ++APtr) {
-        if (*APtr == Ptr) {
-          *APtr = CurArray[--NumNonEmpty];
+      for (const void *&Bucket : small_buckets()) {
+        if (Bucket == Ptr) {
+          Bucket = CurArray[--NumEntries];
           incrementEpoch();
           return true;
         }
@@ -194,6 +208,7 @@ class SmallPtrSetImplBase : public DebugEpochBase {
 
     *const_cast<const void **>(Bucket) = getTombstoneMarker();
     NumTombstones++;
+    --NumEntries;
     // Treat this consistently from an API perspective, even if we don't
     // actually invalidate iterators here.
     incrementEpoch();
@@ -206,11 +221,9 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   const void *const * find_imp(const void * Ptr) const {
     if (isSmall()) {
       // Linear search for the item.
-      for (const void *const *APtr = CurArray, *const *E =
-                                                   CurArray + NumNonEmpty;
-           APtr != E; ++APtr)
-        if (*APtr == Ptr)
-          return APtr;
+      for (const void *const &Bucket : small_buckets())
+        if (Bucket == Ptr)
+          return &Bucket;
       return EndPointer();
     }
 
@@ -223,10 +236,8 @@ class SmallPtrSetImplBase : public DebugEpochBase {
   bool contains_imp(const void *Ptr) const {
     if (isSmall()) {
       // Linear search for the item.
-      const void *const *APtr = CurArray;
-      const void *const *E = CurArray + NumNonEmpty;
-      for (; APtr != E; ++APtr)
-        if (*APtr == Ptr)
+      for (const void *const &Bucket : small_buckets())
+        if (Bucket == Ptr)
           return true;
       return false;
     }
@@ -389,7 +400,7 @@ class SmallPtrSetImpl : public SmallPtrSetImplBase {
   /// the element equal to Ptr.
   std::pair<iterator, bool> insert(PtrType Ptr) {
     auto p = insert_imp(PtrTraits::getAsVoidPointer(Ptr));
-    return std::make_pair(makeIterator(p.first), p.second);
+    return {makeIterator(p.first), p.second};
   }
 
   /// Insert the given pointer with an iterator hint that is ignored. This is
@@ -424,12 +435,13 @@ class SmallPtrSetImpl : public SmallPtrSetImplBase {
   bool remove_if(UnaryPredicate P) {
     bool Removed = false;
     if (isSmall()) {
-      const void **APtr = CurArray, **E = CurArray + NumNonEmpty;
+      auto Buckets = small_buckets();
+      const void **APtr = Buckets.begin(), **E = Buckets.end();
       while (APtr != E) {
         PtrType Ptr = PtrTraits::getFromVoidPointer(const_cast<void *>(*APtr));
         if (P(Ptr)) {
           *APtr = *--E;
-          --NumNonEmpty;
+          --NumEntries;
           incrementEpoch();
           Removed = true;
         } else {
@@ -439,14 +451,14 @@ class SmallPtrSetImpl : public SmallPtrSetImplBase {
       return Removed;
     }
 
-    for (const void **APtr = CurArray, **E = EndPointer(); APtr != E; ++APtr) {
-      const void *Value = *APtr;
-      if (Value == getTombstoneMarker() || Value == getEmptyMarker())
+    for (const void *&Bucket : buckets()) {
+      if (Bucket == getTombstoneMarker() || Bucket == getEmptyMarker())
         continue;
-      PtrType Ptr = PtrTraits::getFromVoidPointer(const_cast<void *>(Value));
+      PtrType Ptr = PtrTraits::getFromVoidPointer(const_cast<void *>(Bucket));
       if (P(Ptr)) {
-        *APtr = getTombstoneMarker();
+        Bucket = getTombstoneMarker();
         ++NumTombstones;
+        --NumEntries;
         incrementEpoch();
         Removed = true;
       }
diff --git a/llvm/include/llvm/ADT/SmallSet.h b/llvm/include/llvm/ADT/SmallSet.h
index eb434bcb71717..4c60b15224a45 100644
--- a/llvm/include/llvm/ADT/SmallSet.h
+++ b/llvm/include/llvm/ADT/SmallSet.h
@@ -268,8 +268,17 @@ class SmallSet {
 
 /// If this set is of pointer values, transparently switch over to using
 /// SmallPtrSet for performance.
+///
+/// We use this middleman class DeprecatedSmallSet so that the deprecation
+/// warning works.  Placing LLVM_DEPRECATED just before SmallSet below won't
+/// work.
+template <typename PointerType, unsigned N>
+class LLVM_DEPRECATED("Use SmallPtrSet instead", "SmallPtrSet")
+    DeprecatedSmallSet : public SmallPtrSet<PointerType, N> {};
+
 template <typename PointeeType, unsigned N>
-class SmallSet<PointeeType*, N> : public SmallPtrSet<PointeeType*, N> {};
+class SmallSet<PointeeType *, N> : public DeprecatedSmallSet<PointeeType *, N> {
+};
 
 /// Equality comparison for SmallSet.
 ///
diff --git a/llvm/include/llvm/ADT/SparseSet.h b/llvm/include/llvm/ADT/SparseSet.h
index d9ded9875d377..a8ebc9f786486 100644
--- a/llvm/include/llvm/ADT/SparseSet.h
+++ b/llvm/include/llvm/ADT/SparseSet.h
@@ -258,10 +258,10 @@ class SparseSet {
     unsigned Idx = ValIndexOf(Val);
     iterator I = findIndex(Idx);
     if (I != end())
-      return std::make_pair(I, false);
+      return {I, false};
     Sparse[Idx] = size();
     Dense.push_back(Val);
-    return std::make_pair(end() - 1, true);
+    return {end() - 1, true};
   }
 
   /// array subscript - If an element already exists with this key, return it.
diff --git a/llvm/include/llvm/ADT/StringMap.h b/llvm/include/llvm/ADT/StringMap.h
index f839edf031006..dbc0485967ac6 100644
--- a/llvm/include/llvm/ADT/StringMap.h
+++ b/llvm/include/llvm/ADT/StringMap.h
@@ -89,6 +89,10 @@ class StringMapImpl {
   /// setup the map as empty.
   LLVM_ABI void init(unsigned Size);
 
+  iterator_range<StringMapEntryBase **> buckets() {
+    return make_range(TheTable, TheTable + NumBuckets);
+  }
+
 public:
   static constexpr uintptr_t TombstoneIntVal =
       static_cast<uintptr_t>(-1)
@@ -198,8 +202,7 @@ class LLVM_ALLOCATORHOLDER_EMPTYBASE StringMap
     // to default values.  This is a copy of clear(), but avoids unnecessary
     // work not required in the destructor.
     if (!empty()) {
-      for (unsigned I = 0, E = NumBuckets; I != E; ++I) {
-        StringMapEntryBase *Bucket = TheTable[I];
+      for (StringMapEntryBase *Bucket : buckets()) {
         if (Bucket && Bucket != getTombstoneVal()) {
           static_cast<MapEntryTy *>(Bucket)->Destroy(getAllocator());
         }
@@ -377,8 +380,8 @@ class LLVM_ALLOCATORHOLDER_EMPTYBASE StringMap
     unsigned BucketNo = LookupBucketFor(Key, FullHashValue);
     StringMapEntryBase *&Bucket = TheTable[BucketNo];
     if (Bucket && Bucket != getTombstoneVal())
-      return std::make_pair(iterator(TheTable + BucketNo, false),
-                            false); // Already exists in map.
+      return {iterator(TheTable + BucketNo, false),
+              false}; // Already exists in map.
 
     if (Bucket == getTombstoneVal())
       --NumTombstones;
@@ -388,7 +391,7 @@ class LLVM_ALLOCATORHOLDER_EMPTYBASE StringMap
     assert(NumItems + NumTombstones <= NumBuckets);
 
     BucketNo = RehashTable(BucketNo);
-    return std::make_pair(iterator(TheTable + BucketNo, false), true);
+    return {iterator(TheTable + BucketNo, false), true};
   }
 
   // clear - Empties out the StringMap
@@ -398,8 +401,7 @@ class LLVM_ALLOCATORHOLDER_EMPTYBASE StringMap
 
     // Zap all values, resetting the keys back to non-present (not tombstone),
     // which is safe because we're removing all elements.
-    for (unsigned I = 0, E = NumBuckets; I != E; ++I) {
-      StringMapEntryBase *&Bucket = TheTable[I];
+    for (StringMapEntryBase *&Bucket : buckets()) {
       if (Bucket && Bucket != getTombstoneVal()) {
         static_cast<MapEntryTy *>(Bucket)->Destroy(getAllocator());
       }
diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h
index 0ced1c0379a3b..16aca4d45892d 100644
--- a/llvm/include/llvm/ADT/StringRef.h
+++ b/llvm/include/llvm/ADT/StringRef.h
@@ -38,6 +38,8 @@ namespace llvm {
   LLVM_ABI bool getAsSignedInteger(StringRef Str, unsigned Radix,
                                    long long &Result);
 
+  LLVM_ABI unsigned getAutoSenseRadix(StringRef &Str);
+
   LLVM_ABI bool consumeUnsignedInteger(StringRef &Str, unsigned Radix,
                                        unsigned long long &Result);
   LLVM_ABI bool consumeSignedInteger(StringRef &Str, unsigned Radix,
diff --git a/llvm/include/llvm/Analysis/DXILResource.h b/llvm/include/llvm/Analysis/DXILResource.h
index 9d07bc0013ca7..88ac0a11fe5a2 100644
--- a/llvm/include/llvm/Analysis/DXILResource.h
+++ b/llvm/include/llvm/Analysis/DXILResource.h
@@ -360,9 +360,11 @@ class ResourceInfo {
              std::tie(RHS.RecordID, RHS.Space, RHS.LowerBound, RHS.Size);
     }
     bool overlapsWith(const ResourceBinding &RHS) const {
+      if (Space != RHS.Space)
+        return false;
       if (Size == UINT32_MAX)
         return LowerBound < RHS.LowerBound;
-      return Space == RHS.Space && LowerBound + Size - 1 >= RHS.LowerBound;
+      return LowerBound + Size - 1 >= RHS.LowerBound;
     }
   };
 
diff --git a/llvm/include/llvm/Analysis/Delinearization.h b/llvm/include/llvm/Analysis/Delinearization.h
index eb775babd6061..434cfb61699d6 100644
--- a/llvm/include/llvm/Analysis/Delinearization.h
+++ b/llvm/include/llvm/Analysis/Delinearization.h
@@ -112,6 +112,35 @@ void delinearize(ScalarEvolution &SE, const SCEV *Expr,
                  SmallVectorImpl<const SCEV *> &Subscripts,
                  SmallVectorImpl<const SCEV *> &Sizes, const SCEV *ElementSize);
 
+/// Compute the dimensions of fixed size array from \Expr and save the results
+/// in \p Sizes.
+bool findFixedSizeArrayDimensions(ScalarEvolution &SE, const SCEV *Expr,
+                                  SmallVectorImpl<uint64_t> &Sizes,
+                                  const SCEV *ElementSize);
+
+/// Split this SCEVAddRecExpr into two vectors of SCEVs representing the
+/// subscripts and sizes of an access to a fixed size array. This is a special
+/// case of delinearization for fixed size arrays.
+///
+/// The delinearization is a 2 step process: the first step estimates the sizes
+/// of each dimension of the array. The second step computes the access
+/// functions for the delinearized array:
+///
+/// 1. Compute the array size
+/// 2. Compute the access function: same as normal delinearization
+///
+/// Different from the normal delinearization, this function assumes that NO
+/// terms exist in the \p Expr. In other words, it assumes that the all step
+/// values are constant.
+///
+/// This function is intended to replace getIndexExpressionsFromGEP and
+/// tryDelinearizeFixedSizeImpl. They rely on the GEP source element type so
+/// that they will be removed in the future.
+bool delinearizeFixedSizeArray(ScalarEvolution &SE, const SCEV *Expr,
+                               SmallVectorImpl<const SCEV *> &Subscripts,
+                               SmallVectorImpl<const SCEV *> &Sizes,
+                               const SCEV *ElementSize);
+
 /// Gathers the individual index expressions from a GEP instruction.
 ///
 /// This function optimistically assumes the GEP references into a fixed size
diff --git a/llvm/include/llvm/Analysis/DependenceAnalysis.h b/llvm/include/llvm/Analysis/DependenceAnalysis.h
index f98bd684149f9..f66c79d915665 100644
--- a/llvm/include/llvm/Analysis/DependenceAnalysis.h
+++ b/llvm/include/llvm/Analysis/DependenceAnalysis.h
@@ -47,994 +47,927 @@
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
-  class AAResults;
-  template <typename T> class ArrayRef;
-  class Loop;
-  class LoopInfo;
-  class SCEVConstant;
-  class raw_ostream;
-
-  /// Dependence - This class represents a dependence between two memory
-  /// memory references in a function. It contains minimal information and
-  /// is used in the very common situation where the compiler is unable to
-  /// determine anything beyond the existence of a dependence; that is, it
-  /// represents a confused dependence (see also FullDependence). In most
-  /// cases (for output, flow, and anti dependences), the dependence implies
-  /// an ordering, where the source must precede the destination; in contrast,
-  /// input dependences are unordered.
-  ///
-  /// When a dependence graph is built, each Dependence will be a member of
-  /// the set of predecessor edges for its destination instruction and a set
-  /// if successor edges for its source instruction. These sets are represented
-  /// as singly-linked lists, with the "next" fields stored in the dependence
-  /// itelf.
-  class LLVM_ABI Dependence {
-  protected:
-    Dependence(Dependence &&) = default;
-    Dependence &operator=(Dependence &&) = default;
-
-  public:
-    Dependence(Instruction *Source, Instruction *Destination,
-               const SCEVUnionPredicate &A)
-        : Src(Source), Dst(Destination), Assumptions(A) {}
-    virtual ~Dependence() = default;
-
-    /// Dependence::DVEntry - Each level in the distance/direction vector
-    /// has a direction (or perhaps a union of several directions), and
-    /// perhaps a distance.
-    struct DVEntry {
-      enum : unsigned char {
-        NONE = 0,
-        LT = 1,
-        EQ = 2,
-        LE = 3,
-        GT = 4,
-        NE = 5,
-        GE = 6,
-        ALL = 7
-      };
-      unsigned char Direction : 3; // Init to ALL, then refine.
-      bool Scalar    : 1; // Init to true.
-      bool PeelFirst : 1; // Peeling the first iteration will break dependence.
-      bool PeelLast  : 1; // Peeling the last iteration will break the dependence.
-      bool Splitable : 1; // Splitting the loop will break dependence.
-      const SCEV *Distance = nullptr; // NULL implies no distance available.
-      DVEntry()
-          : Direction(ALL), Scalar(true), PeelFirst(false), PeelLast(false),
-            Splitable(false) {}
+class AAResults;
+template <typename T> class ArrayRef;
+class Loop;
+class LoopInfo;
+class SCEVConstant;
+class raw_ostream;
+
+/// Dependence - This class represents a dependence between two memory
+/// memory references in a function. It contains minimal information and
+/// is used in the very common situation where the compiler is unable to
+/// determine anything beyond the existence of a dependence; that is, it
+/// represents a confused dependence (see also FullDependence). In most
+/// cases (for output, flow, and anti dependences), the dependence implies
+/// an ordering, where the source must precede the destination; in contrast,
+/// input dependences are unordered.
+///
+/// When a dependence graph is built, each Dependence will be a member of
+/// the set of predecessor edges for its destination instruction and a set
+/// if successor edges for its source instruction. These sets are represented
+/// as singly-linked lists, with the "next" fields stored in the dependence
+/// itelf.
+class LLVM_ABI Dependence {
+protected:
+  Dependence(Dependence &&) = default;
+  Dependence &operator=(Dependence &&) = default;
+
+public:
+  Dependence(Instruction *Source, Instruction *Destination,
+             const SCEVUnionPredicate &A)
+      : Src(Source), Dst(Destination), Assumptions(A) {}
+  virtual ~Dependence() = default;
+
+  /// Dependence::DVEntry - Each level in the distance/direction vector
+  /// has a direction (or perhaps a union of several directions), and
+  /// perhaps a distance.
+  struct DVEntry {
+    enum : unsigned char {
+      NONE = 0,
+      LT = 1,
+      EQ = 2,
+      LE = 3,
+      GT = 4,
+      NE = 5,
+      GE = 6,
+      ALL = 7
     };
+    unsigned char Direction : 3; // Init to ALL, then refine.
+    bool Scalar : 1;             // Init to true.
+    bool PeelFirst : 1; // Peeling the first iteration will break dependence.
+    bool PeelLast : 1;  // Peeling the last iteration will break the dependence.
+    bool Splitable : 1; // Splitting the loop will break dependence.
+    const SCEV *Distance = nullptr; // NULL implies no distance available.
+    DVEntry()
+        : Direction(ALL), Scalar(true), PeelFirst(false), PeelLast(false),
+          Splitable(false) {}
+  };
 
-    /// getSrc - Returns the source instruction for this dependence.
-    ///
-    Instruction *getSrc() const { return Src; }
+  /// getSrc - Returns the source instruction for this dependence.
+  Instruction *getSrc() const { return Src; }
 
-    /// getDst - Returns the destination instruction for this dependence.
-    ///
-    Instruction *getDst() const { return Dst; }
+  /// getDst - Returns the destination instruction for this dependence.
+  Instruction *getDst() const { return Dst; }
 
-    /// isInput - Returns true if this is an input dependence.
-    ///
-    bool isInput() const;
+  /// isInput - Returns true if this is an input dependence.
+  bool isInput() const;
 
-    /// isOutput - Returns true if this is an output dependence.
-    ///
-    bool isOutput() const;
+  /// isOutput - Returns true if this is an output dependence.
+  bool isOutput() const;
 
-    /// isFlow - Returns true if this is a flow (aka true) dependence.
-    ///
-    bool isFlow() const;
+  /// isFlow - Returns true if this is a flow (aka true) dependence.
+  bool isFlow() const;
 
-    /// isAnti - Returns true if this is an anti dependence.
-    ///
-    bool isAnti() const;
+  /// isAnti - Returns true if this is an anti dependence.
+  bool isAnti() const;
 
-    /// isOrdered - Returns true if dependence is Output, Flow, or Anti
-    ///
-    bool isOrdered() const { return isOutput() || isFlow() || isAnti(); }
+  /// isOrdered - Returns true if dependence is Output, Flow, or Anti
+  bool isOrdered() const { return isOutput() || isFlow() || isAnti(); }
 
-    /// isUnordered - Returns true if dependence is Input
-    ///
-    bool isUnordered() const { return isInput(); }
+  /// isUnordered - Returns true if dependence is Input
+  bool isUnordered() const { return isInput(); }
 
-    /// isLoopIndependent - Returns true if this is a loop-independent
-    /// dependence.
-    virtual bool isLoopIndependent() const { return true; }
+  /// isLoopIndependent - Returns true if this is a loop-independent
+  /// dependence.
+  virtual bool isLoopIndependent() const { return true; }
 
-    /// isConfused - Returns true if this dependence is confused
-    /// (the compiler understands nothing and makes worst-case
-    /// assumptions).
-    virtual bool isConfused() const { return true; }
+  /// isConfused - Returns true if this dependence is confused
+  /// (the compiler understands nothing and makes worst-case assumptions).
+  virtual bool isConfused() const { return true; }
 
-    /// isConsistent - Returns true if this dependence is consistent
-    /// (occurs every time the source and destination are executed).
-    virtual bool isConsistent() const { return false; }
+  /// isConsistent - Returns true if this dependence is consistent
+  /// (occurs every time the source and destination are executed).
+  virtual bool isConsistent() const { return false; }
 
-    /// getLevels - Returns the number of common loops surrounding the
-    /// source and destination of the dependence.
-    virtual unsigned getLevels() const { return 0; }
+  /// getLevels - Returns the number of common loops surrounding the
+  /// source and destination of the dependence.
+  virtual unsigned getLevels() const { return 0; }
 
-    /// getDirection - Returns the direction associated with a particular
-    /// level.
-    virtual unsigned getDirection(unsigned Level) const { return DVEntry::ALL; }
+  /// getDirection - Returns the direction associated with a particular level.
+  virtual unsigned getDirection(unsigned Level) const { return DVEntry::ALL; }
 
-    /// getDistance - Returns the distance (or NULL) associated with a
-    /// particular level.
-    virtual const SCEV *getDistance(unsigned Level) const { return nullptr; }
+  /// getDistance - Returns the distance (or NULL) associated with a particular
+  /// level.
+  virtual const SCEV *getDistance(unsigned Level) const { return nullptr; }
 
-    /// Check if the direction vector is negative. A negative direction
-    /// vector means Src and Dst are reversed in the actual program.
-    virtual bool isDirectionNegative() const { return false; }
+  /// Check if the direction vector is negative. A negative direction
+  /// vector means Src and Dst are reversed in the actual program.
+  virtual bool isDirectionNegative() const { return false; }
+
+  /// If the direction vector is negative, normalize the direction
+  /// vector to make it non-negative. Normalization is done by reversing
+  /// Src and Dst, plus reversing the dependence directions and distances
+  /// in the vector.
+  virtual bool normalize(ScalarEvolution *SE) { return false; }
 
-    /// If the direction vector is negative, normalize the direction
-    /// vector to make it non-negative. Normalization is done by reversing
-    /// Src and Dst, plus reversing the dependence directions and distances
-    /// in the vector.
-    virtual bool normalize(ScalarEvolution *SE) { return false; }
+  /// isPeelFirst - Returns true if peeling the first iteration from
+  /// this loop will break this dependence.
+  virtual bool isPeelFirst(unsigned Level) const { return false; }
 
-    /// isPeelFirst - Returns true if peeling the first iteration from
-    /// this loop will break this dependence.
-    virtual bool isPeelFirst(unsigned Level) const { return false; }
+  /// isPeelLast - Returns true if peeling the last iteration from
+  /// this loop will break this dependence.
+  virtual bool isPeelLast(unsigned Level) const { return false; }
 
-    /// isPeelLast - Returns true if peeling the last iteration from
-    /// this loop will break this dependence.
-    virtual bool isPeelLast(unsigned Level) const { return false; }
+  /// isSplitable - Returns true if splitting this loop will break the
+  /// dependence.
+  virtual bool isSplitable(unsigned Level) const { return false; }
 
-    /// isSplitable - Returns true if splitting this loop will break
-    /// the dependence.
-    virtual bool isSplitable(unsigned Level) const { return false; }
+  /// isScalar - Returns true if a particular level is scalar; that is,
+  /// if no subscript in the source or destination mention the induction
+  /// variable associated with the loop at this level.
+  virtual bool isScalar(unsigned Level) const;
+
+  /// getNextPredecessor - Returns the value of the NextPredecessor field.
+  const Dependence *getNextPredecessor() const { return NextPredecessor; }
+
+  /// getNextSuccessor - Returns the value of the NextSuccessor field.
+  const Dependence *getNextSuccessor() const { return NextSuccessor; }
+
+  /// setNextPredecessor - Sets the value of the NextPredecessor
+  /// field.
+  void setNextPredecessor(const Dependence *pred) { NextPredecessor = pred; }
+
+  /// setNextSuccessor - Sets the value of the NextSuccessor field.
+  void setNextSuccessor(const Dependence *succ) { NextSuccessor = succ; }
+
+  /// getRuntimeAssumptions - Returns the runtime assumptions under which this
+  /// Dependence relation is valid.
+  SCEVUnionPredicate getRuntimeAssumptions() const { return Assumptions; }
+
+  /// dump - For debugging purposes, dumps a dependence to OS.
+  void dump(raw_ostream &OS) const;
+
+protected:
+  Instruction *Src, *Dst;
+
+private:
+  SCEVUnionPredicate Assumptions;
+  const Dependence *NextPredecessor = nullptr, *NextSuccessor = nullptr;
+  friend class DependenceInfo;
+};
+
+/// FullDependence - This class represents a dependence between two memory
+/// references in a function. It contains detailed information about the
+/// dependence (direction vectors, etc.) and is used when the compiler is
+/// able to accurately analyze the interaction of the references; that is,
+/// it is not a confused dependence (see Dependence). In most cases
+/// (for output, flow, and anti dependences), the dependence implies an
+/// ordering, where the source must precede the destination; in contrast,
+/// input dependences are unordered.
+class LLVM_ABI FullDependence final : public Dependence {
+public:
+  FullDependence(Instruction *Source, Instruction *Destination,
+                 const SCEVUnionPredicate &Assumes,
+                 bool PossiblyLoopIndependent, unsigned Levels);
+
+  /// isLoopIndependent - Returns true if this is a loop-independent
+  /// dependence.
+  bool isLoopIndependent() const override { return LoopIndependent; }
+
+  /// isConfused - Returns true if this dependence is confused
+  /// (the compiler understands nothing and makes worst-case
+  /// assumptions).
+  bool isConfused() const override { return false; }
+
+  /// isConsistent - Returns true if this dependence is consistent
+  /// (occurs every time the source and destination are executed).
+  bool isConsistent() const override { return Consistent; }
+
+  /// getLevels - Returns the number of common loops surrounding the
+  /// source and destination of the dependence.
+  unsigned getLevels() const override { return Levels; }
+
+  /// getDirection - Returns the direction associated with a particular
+  /// level.
+  unsigned getDirection(unsigned Level) const override;
+
+  /// getDistance - Returns the distance (or NULL) associated with a
+  /// particular level.
+  const SCEV *getDistance(unsigned Level) const override;
+
+  /// Check if the direction vector is negative. A negative direction
+  /// vector means Src and Dst are reversed in the actual program.
+  bool isDirectionNegative() const override;
+
+  /// If the direction vector is negative, normalize the direction
+  /// vector to make it non-negative. Normalization is done by reversing
+  /// Src and Dst, plus reversing the dependence directions and distances
+  /// in the vector.
+  bool normalize(ScalarEvolution *SE) override;
+
+  /// isPeelFirst - Returns true if peeling the first iteration from
+  /// this loop will break this dependence.
+  bool isPeelFirst(unsigned Level) const override;
+
+  /// isPeelLast - Returns true if peeling the last iteration from
+  /// this loop will break this dependence.
+  bool isPeelLast(unsigned Level) const override;
+
+  /// isSplitable - Returns true if splitting the loop will break
+  /// the dependence.
+  bool isSplitable(unsigned Level) const override;
+
+  /// isScalar - Returns true if a particular level is scalar; that is,
+  /// if no subscript in the source or destination mention the induction
+  /// variable associated with the loop at this level.
+  bool isScalar(unsigned Level) const override;
+
+private:
+  unsigned short Levels;
+  bool LoopIndependent;
+  bool Consistent; // Init to true, then refine.
+  std::unique_ptr<DVEntry[]> DV;
+  friend class DependenceInfo;
+};
+
+/// DependenceInfo - This class is the main dependence-analysis driver.
+class DependenceInfo {
+public:
+  DependenceInfo(Function *F, AAResults *AA, ScalarEvolution *SE, LoopInfo *LI)
+      : AA(AA), SE(SE), LI(LI), F(F) {}
+
+  /// Handle transitive invalidation when the cached analysis results go away.
+  LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA,
+                           FunctionAnalysisManager::Invalidator &Inv);
+
+  /// depends - Tests for a dependence between the Src and Dst instructions.
+  /// Returns NULL if no dependence; otherwise, returns a Dependence (or a
+  /// FullDependence) with as much information as can be gleaned. By default,
+  /// the dependence test collects a set of runtime assumptions that cannot be
+  /// solved at compilation time. By default UnderRuntimeAssumptions is false
+  /// for a safe approximation of the dependence relation that does not
+  /// require runtime checks.
+  LLVM_ABI std::unique_ptr<Dependence>
+  depends(Instruction *Src, Instruction *Dst,
+          bool UnderRuntimeAssumptions = false);
+
+  /// getSplitIteration - Give a dependence that's splittable at some
+  /// particular level, return the iteration that should be used to split
+  /// the loop.
+  ///
+  /// Generally, the dependence analyzer will be used to build
+  /// a dependence graph for a function (basically a map from instructions
+  /// to dependences). Looking for cycles in the graph shows us loops
+  /// that cannot be trivially vectorized/parallelized.
+  ///
+  /// We can try to improve the situation by examining all the dependences
+  /// that make up the cycle, looking for ones we can break.
+  /// Sometimes, peeling the first or last iteration of a loop will break
+  /// dependences, and there are flags for those possibilities.
+  /// Sometimes, splitting a loop at some other iteration will do the trick,
+  /// and we've got a flag for that case. Rather than waste the space to
+  /// record the exact iteration (since we rarely know), we provide
+  /// a method that calculates the iteration. It's a drag that it must work
+  /// from scratch, but wonderful in that it's possible.
+  ///
+  /// Here's an example:
+  ///
+  ///    for (i = 0; i < 10; i++)
+  ///        A[i] = ...
+  ///        ... = A[11 - i]
+  ///
+  /// There's a loop-carried flow dependence from the store to the load,
+  /// found by the weak-crossing SIV test. The dependence will have a flag,
+  /// indicating that the dependence can be broken by splitting the loop.
+  /// Calling getSplitIteration will return 5.
+  /// Splitting the loop breaks the dependence, like so:
+  ///
+  ///    for (i = 0; i <= 5; i++)
+  ///        A[i] = ...
+  ///        ... = A[11 - i]
+  ///    for (i = 6; i < 10; i++)
+  ///        A[i] = ...
+  ///        ... = A[11 - i]
+  ///
+  /// breaks the dependence and allows us to vectorize/parallelize
+  /// both loops.
+  LLVM_ABI const SCEV *getSplitIteration(const Dependence &Dep, unsigned Level);
+
+  Function *getFunction() const { return F; }
+
+  /// getRuntimeAssumptions - Returns all the runtime assumptions under which
+  /// the dependence test is valid.
+  LLVM_ABI SCEVUnionPredicate getRuntimeAssumptions() const;
+
+private:
+  AAResults *AA;
+  ScalarEvolution *SE;
+  LoopInfo *LI;
+  Function *F;
+  SmallVector<const SCEVPredicate *, 4> Assumptions;
+
+  /// Subscript - This private struct represents a pair of subscripts from
+  /// a pair of potentially multi-dimensional array references. We use a
+  /// vector of them to guide subscript partitioning.
+  struct Subscript {
+    const SCEV *Src;
+    const SCEV *Dst;
+    enum ClassificationKind { ZIV, SIV, RDIV, MIV, NonLinear } Classification;
+    SmallBitVector Loops;
+    SmallBitVector GroupLoops;
+    SmallBitVector Group;
+  };
 
-    /// isScalar - Returns true if a particular level is scalar; that is,
-    /// if no subscript in the source or destination mention the induction
-    /// variable associated with the loop at this level.
-    virtual bool isScalar(unsigned Level) const;
+  struct CoefficientInfo {
+    const SCEV *Coeff;
+    const SCEV *PosPart;
+    const SCEV *NegPart;
+    const SCEV *Iterations;
+  };
 
-    /// getNextPredecessor - Returns the value of the NextPredecessor
-    /// field.
-    const Dependence *getNextPredecessor() const { return NextPredecessor; }
+  struct BoundInfo {
+    const SCEV *Iterations;
+    const SCEV *Upper[8];
+    const SCEV *Lower[8];
+    unsigned char Direction;
+    unsigned char DirSet;
+  };
 
-    /// getNextSuccessor - Returns the value of the NextSuccessor
-    /// field.
-    const Dependence *getNextSuccessor() const { return NextSuccessor; }
+  /// Constraint - This private class represents a constraint, as defined
+  /// in the paper
+  ///
+  ///           Practical Dependence Testing
+  ///           Goff, Kennedy, Tseng
+  ///           PLDI 1991
+  ///
+  /// There are 5 kinds of constraint, in a hierarchy.
+  ///   1) Any - indicates no constraint, any dependence is possible.
+  ///   2) Line - A line ax + by = c, where a, b, and c are parameters,
+  ///             representing the dependence equation.
+  ///   3) Distance - The value d of the dependence distance;
+  ///   4) Point - A point <x, y> representing the dependence from
+  ///              iteration x to iteration y.
+  ///   5) Empty - No dependence is possible.
+  class Constraint {
+  private:
+    enum ConstraintKind { Empty, Point, Distance, Line, Any } Kind;
+    ScalarEvolution *SE;
+    const SCEV *A;
+    const SCEV *B;
+    const SCEV *C;
+    const Loop *AssociatedLoop;
 
-    /// setNextPredecessor - Sets the value of the NextPredecessor
-    /// field.
-    void setNextPredecessor(const Dependence *pred) { NextPredecessor = pred; }
+  public:
+    /// isEmpty - Return true if the constraint is of kind Empty.
+    bool isEmpty() const { return Kind == Empty; }
 
-    /// setNextSuccessor - Sets the value of the NextSuccessor
-    /// field.
-    void setNextSuccessor(const Dependence *succ) { NextSuccessor = succ; }
+    /// isPoint - Return true if the constraint is of kind Point.
+    bool isPoint() const { return Kind == Point; }
 
-    /// getRuntimeAssumptions - Returns the runtime assumptions under which this
-    /// Dependence relation is valid.
-    SCEVUnionPredicate getRuntimeAssumptions() const { return Assumptions; }
+    /// isDistance - Return true if the constraint is of kind Distance.
+    bool isDistance() const { return Kind == Distance; }
 
-    /// dump - For debugging purposes, dumps a dependence to OS.
-    ///
-    void dump(raw_ostream &OS) const;
+    /// isLine - Return true if the constraint is of kind Line.
+    /// Since Distance's can also be represented as Lines, we also return
+    /// true if the constraint is of kind Distance.
+    bool isLine() const { return Kind == Line || Kind == Distance; }
 
-  protected:
-    Instruction *Src, *Dst;
+    /// isAny - Return true if the constraint is of kind Any;
+    bool isAny() const { return Kind == Any; }
 
-  private:
-    SCEVUnionPredicate Assumptions;
-    const Dependence *NextPredecessor = nullptr, *NextSuccessor = nullptr;
-    friend class DependenceInfo;
-  };
+    /// getX - If constraint is a point <X, Y>, returns X.
+    /// Otherwise assert.
+    LLVM_ABI const SCEV *getX() const;
 
-  /// FullDependence - This class represents a dependence between two memory
-  /// references in a function. It contains detailed information about the
-  /// dependence (direction vectors, etc.) and is used when the compiler is
-  /// able to accurately analyze the interaction of the references; that is,
-  /// it is not a confused dependence (see Dependence). In most cases
-  /// (for output, flow, and anti dependences), the dependence implies an
-  /// ordering, where the source must precede the destination; in contrast,
-  /// input dependences are unordered.
-  class LLVM_ABI FullDependence final : public Dependence {
-  public:
-    FullDependence(Instruction *Source, Instruction *Destination,
-                   const SCEVUnionPredicate &Assumes,
-                   bool PossiblyLoopIndependent, unsigned Levels);
-
-    /// isLoopIndependent - Returns true if this is a loop-independent
-    /// dependence.
-    bool isLoopIndependent() const override { return LoopIndependent; }
-
-    /// isConfused - Returns true if this dependence is confused
-    /// (the compiler understands nothing and makes worst-case
-    /// assumptions).
-    bool isConfused() const override { return false; }
-
-    /// isConsistent - Returns true if this dependence is consistent
-    /// (occurs every time the source and destination are executed).
-    bool isConsistent() const override { return Consistent; }
-
-    /// getLevels - Returns the number of common loops surrounding the
-    /// source and destination of the dependence.
-    unsigned getLevels() const override { return Levels; }
-
-    /// getDirection - Returns the direction associated with a particular
-    /// level.
-    unsigned getDirection(unsigned Level) const override;
-
-    /// getDistance - Returns the distance (or NULL) associated with a
-    /// particular level.
-    const SCEV *getDistance(unsigned Level) const override;
-
-    /// Check if the direction vector is negative. A negative direction
-    /// vector means Src and Dst are reversed in the actual program.
-    bool isDirectionNegative() const override;
-
-    /// If the direction vector is negative, normalize the direction
-    /// vector to make it non-negative. Normalization is done by reversing
-    /// Src and Dst, plus reversing the dependence directions and distances
-    /// in the vector.
-    bool normalize(ScalarEvolution *SE) override;
-
-    /// isPeelFirst - Returns true if peeling the first iteration from
-    /// this loop will break this dependence.
-    bool isPeelFirst(unsigned Level) const override;
-
-    /// isPeelLast - Returns true if peeling the last iteration from
-    /// this loop will break this dependence.
-    bool isPeelLast(unsigned Level) const override;
-
-    /// isSplitable - Returns true if splitting the loop will break
-    /// the dependence.
-    bool isSplitable(unsigned Level) const override;
-
-    /// isScalar - Returns true if a particular level is scalar; that is,
-    /// if no subscript in the source or destination mention the induction
-    /// variable associated with the loop at this level.
-    bool isScalar(unsigned Level) const override;
+    /// getY - If constraint is a point <X, Y>, returns Y.
+    /// Otherwise assert.
+    LLVM_ABI const SCEV *getY() const;
 
-  private:
-    unsigned short Levels;
-    bool LoopIndependent;
-    bool Consistent; // Init to true, then refine.
-    std::unique_ptr<DVEntry[]> DV;
-    friend class DependenceInfo;
-  };
+    /// getA - If constraint is a line AX + BY = C, returns A.
+    /// Otherwise assert.
+    LLVM_ABI const SCEV *getA() const;
 
-  /// DependenceInfo - This class is the main dependence-analysis driver.
-  ///
-  class DependenceInfo {
-  public:
-    DependenceInfo(Function *F, AAResults *AA, ScalarEvolution *SE,
-                   LoopInfo *LI)
-        : AA(AA), SE(SE), LI(LI), F(F) {}
-
-    /// Handle transitive invalidation when the cached analysis results go away.
-    LLVM_ABI bool invalidate(Function &F, const PreservedAnalyses &PA,
-                             FunctionAnalysisManager::Invalidator &Inv);
-
-    /// depends - Tests for a dependence between the Src and Dst instructions.
-    /// Returns NULL if no dependence; otherwise, returns a Dependence (or a
-    /// FullDependence) with as much information as can be gleaned. By default,
-    /// the dependence test collects a set of runtime assumptions that cannot be
-    /// solved at compilation time. By default UnderRuntimeAssumptions is false
-    /// for a safe approximation of the dependence relation that does not
-    /// require runtime checks.
-    LLVM_ABI std::unique_ptr<Dependence>
-    depends(Instruction *Src, Instruction *Dst,
-            bool UnderRuntimeAssumptions = false);
-
-    /// getSplitIteration - Give a dependence that's splittable at some
-    /// particular level, return the iteration that should be used to split
-    /// the loop.
-    ///
-    /// Generally, the dependence analyzer will be used to build
-    /// a dependence graph for a function (basically a map from instructions
-    /// to dependences). Looking for cycles in the graph shows us loops
-    /// that cannot be trivially vectorized/parallelized.
-    ///
-    /// We can try to improve the situation by examining all the dependences
-    /// that make up the cycle, looking for ones we can break.
-    /// Sometimes, peeling the first or last iteration of a loop will break
-    /// dependences, and there are flags for those possibilities.
-    /// Sometimes, splitting a loop at some other iteration will do the trick,
-    /// and we've got a flag for that case. Rather than waste the space to
-    /// record the exact iteration (since we rarely know), we provide
-    /// a method that calculates the iteration. It's a drag that it must work
-    /// from scratch, but wonderful in that it's possible.
-    ///
-    /// Here's an example:
-    ///
-    ///    for (i = 0; i < 10; i++)
-    ///        A[i] = ...
-    ///        ... = A[11 - i]
-    ///
-    /// There's a loop-carried flow dependence from the store to the load,
-    /// found by the weak-crossing SIV test. The dependence will have a flag,
-    /// indicating that the dependence can be broken by splitting the loop.
-    /// Calling getSplitIteration will return 5.
-    /// Splitting the loop breaks the dependence, like so:
-    ///
-    ///    for (i = 0; i <= 5; i++)
-    ///        A[i] = ...
-    ///        ... = A[11 - i]
-    ///    for (i = 6; i < 10; i++)
-    ///        A[i] = ...
-    ///        ... = A[11 - i]
-    ///
-    /// breaks the dependence and allows us to vectorize/parallelize
-    /// both loops.
-    LLVM_ABI const SCEV *getSplitIteration(const Dependence &Dep,
-                                           unsigned Level);
-
-    Function *getFunction() const { return F; }
-
-    /// getRuntimeAssumptions - Returns all the runtime assumptions under which
-    /// the dependence test is valid.
-    LLVM_ABI SCEVUnionPredicate getRuntimeAssumptions() const;
-
-  private:
-    AAResults *AA;
-    ScalarEvolution *SE;
-    LoopInfo *LI;
-    Function *F;
-    SmallVector<const SCEVPredicate *, 4> Assumptions;
-
-    /// Subscript - This private struct represents a pair of subscripts from
-    /// a pair of potentially multi-dimensional array references. We use a
-    /// vector of them to guide subscript partitioning.
-    struct Subscript {
-      const SCEV *Src;
-      const SCEV *Dst;
-      enum ClassificationKind { ZIV, SIV, RDIV, MIV, NonLinear } Classification;
-      SmallBitVector Loops;
-      SmallBitVector GroupLoops;
-      SmallBitVector Group;
-    };
-
-    struct CoefficientInfo {
-      const SCEV *Coeff;
-      const SCEV *PosPart;
-      const SCEV *NegPart;
-      const SCEV *Iterations;
-    };
-
-    struct BoundInfo {
-      const SCEV *Iterations;
-      const SCEV *Upper[8];
-      const SCEV *Lower[8];
-      unsigned char Direction;
-      unsigned char DirSet;
-    };
+    /// getB - If constraint is a line AX + BY = C, returns B.
+    /// Otherwise assert.
+    LLVM_ABI const SCEV *getB() const;
 
-    /// Constraint - This private class represents a constraint, as defined
-    /// in the paper
-    ///
-    ///           Practical Dependence Testing
-    ///           Goff, Kennedy, Tseng
-    ///           PLDI 1991
-    ///
-    /// There are 5 kinds of constraint, in a hierarchy.
-    ///   1) Any - indicates no constraint, any dependence is possible.
-    ///   2) Line - A line ax + by = c, where a, b, and c are parameters,
-    ///             representing the dependence equation.
-    ///   3) Distance - The value d of the dependence distance;
-    ///   4) Point - A point <x, y> representing the dependence from
-    ///              iteration x to iteration y.
-    ///   5) Empty - No dependence is possible.
-    class Constraint {
-    private:
-      enum ConstraintKind { Empty, Point, Distance, Line, Any } Kind;
-      ScalarEvolution *SE;
-      const SCEV *A;
-      const SCEV *B;
-      const SCEV *C;
-      const Loop *AssociatedLoop;
-
-    public:
-      /// isEmpty - Return true if the constraint is of kind Empty.
-      bool isEmpty() const { return Kind == Empty; }
-
-      /// isPoint - Return true if the constraint is of kind Point.
-      bool isPoint() const { return Kind == Point; }
-
-      /// isDistance - Return true if the constraint is of kind Distance.
-      bool isDistance() const { return Kind == Distance; }
-
-      /// isLine - Return true if the constraint is of kind Line.
-      /// Since Distance's can also be represented as Lines, we also return
-      /// true if the constraint is of kind Distance.
-      bool isLine() const { return Kind == Line || Kind == Distance; }
-
-      /// isAny - Return true if the constraint is of kind Any;
-      bool isAny() const { return Kind == Any; }
-
-      /// getX - If constraint is a point <X, Y>, returns X.
-      /// Otherwise assert.
-      LLVM_ABI const SCEV *getX() const;
-
-      /// getY - If constraint is a point <X, Y>, returns Y.
-      /// Otherwise assert.
-      LLVM_ABI const SCEV *getY() const;
-
-      /// getA - If constraint is a line AX + BY = C, returns A.
-      /// Otherwise assert.
-      LLVM_ABI const SCEV *getA() const;
-
-      /// getB - If constraint is a line AX + BY = C, returns B.
-      /// Otherwise assert.
-      LLVM_ABI const SCEV *getB() const;
-
-      /// getC - If constraint is a line AX + BY = C, returns C.
-      /// Otherwise assert.
-      LLVM_ABI const SCEV *getC() const;
-
-      /// getD - If constraint is a distance, returns D.
-      /// Otherwise assert.
-      LLVM_ABI const SCEV *getD() const;
-
-      /// getAssociatedLoop - Returns the loop associated with this constraint.
-      LLVM_ABI const Loop *getAssociatedLoop() const;
-
-      /// setPoint - Change a constraint to Point.
-      LLVM_ABI void setPoint(const SCEV *X, const SCEV *Y,
-                             const Loop *CurrentLoop);
-
-      /// setLine - Change a constraint to Line.
-      LLVM_ABI void setLine(const SCEV *A, const SCEV *B, const SCEV *C,
-                            const Loop *CurrentLoop);
-
-      /// setDistance - Change a constraint to Distance.
-      LLVM_ABI void setDistance(const SCEV *D, const Loop *CurrentLoop);
-
-      /// setEmpty - Change a constraint to Empty.
-      LLVM_ABI void setEmpty();
-
-      /// setAny - Change a constraint to Any.
-      LLVM_ABI void setAny(ScalarEvolution *SE);
-
-      /// dump - For debugging purposes. Dumps the constraint
-      /// out to OS.
-      LLVM_ABI void dump(raw_ostream &OS) const;
-    };
+    /// getC - If constraint is a line AX + BY = C, returns C.
+    /// Otherwise assert.
+    LLVM_ABI const SCEV *getC() const;
 
-    /// establishNestingLevels - Examines the loop nesting of the Src and Dst
-    /// instructions and establishes their shared loops. Sets the variables
-    /// CommonLevels, SrcLevels, and MaxLevels.
-    /// The source and destination instructions needn't be contained in the same
-    /// loop. The routine establishNestingLevels finds the level of most deeply
-    /// nested loop that contains them both, CommonLevels. An instruction that's
-    /// not contained in a loop is at level = 0. MaxLevels is equal to the level
-    /// of the source plus the level of the destination, minus CommonLevels.
-    /// This lets us allocate vectors MaxLevels in length, with room for every
-    /// distinct loop referenced in both the source and destination subscripts.
-    /// The variable SrcLevels is the nesting depth of the source instruction.
-    /// It's used to help calculate distinct loops referenced by the destination.
-    /// Here's the map from loops to levels:
-    ///            0 - unused
-    ///            1 - outermost common loop
-    ///          ... - other common loops
-    /// CommonLevels - innermost common loop
-    ///          ... - loops containing Src but not Dst
-    ///    SrcLevels - innermost loop containing Src but not Dst
-    ///          ... - loops containing Dst but not Src
-    ///    MaxLevels - innermost loop containing Dst but not Src
-    /// Consider the follow code fragment:
-    ///    for (a = ...) {
-    ///      for (b = ...) {
-    ///        for (c = ...) {
-    ///          for (d = ...) {
-    ///            A[] = ...;
-    ///          }
-    ///        }
-    ///        for (e = ...) {
-    ///          for (f = ...) {
-    ///            for (g = ...) {
-    ///              ... = A[];
-    ///            }
-    ///          }
-    ///        }
-    ///      }
-    ///    }
-    /// If we're looking at the possibility of a dependence between the store
-    /// to A (the Src) and the load from A (the Dst), we'll note that they
-    /// have 2 loops in common, so CommonLevels will equal 2 and the direction
-    /// vector for Result will have 2 entries. SrcLevels = 4 and MaxLevels = 7.
-    /// A map from loop names to level indices would look like
-    ///     a - 1
-    ///     b - 2 = CommonLevels
-    ///     c - 3
-    ///     d - 4 = SrcLevels
-    ///     e - 5
-    ///     f - 6
-    ///     g - 7 = MaxLevels
-    void establishNestingLevels(const Instruction *Src,
-                                const Instruction *Dst);
-
-    unsigned CommonLevels, SrcLevels, MaxLevels;
-
-    /// mapSrcLoop - Given one of the loops containing the source, return
-    /// its level index in our numbering scheme.
-    unsigned mapSrcLoop(const Loop *SrcLoop) const;
-
-    /// mapDstLoop - Given one of the loops containing the destination,
-    /// return its level index in our numbering scheme.
-    unsigned mapDstLoop(const Loop *DstLoop) const;
-
-    /// isLoopInvariant - Returns true if Expression is loop invariant
-    /// in LoopNest.
-    bool isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const;
-
-    /// Makes sure all subscript pairs share the same integer type by
-    /// sign-extending as necessary.
-    /// Sign-extending a subscript is safe because getelementptr assumes the
-    /// array subscripts are signed.
-    void unifySubscriptType(ArrayRef<Subscript *> Pairs);
-
-    /// removeMatchingExtensions - Examines a subscript pair.
-    /// If the source and destination are identically sign (or zero)
-    /// extended, it strips off the extension in an effort to
-    /// simplify the actual analysis.
-    void removeMatchingExtensions(Subscript *Pair);
-
-    /// collectCommonLoops - Finds the set of loops from the LoopNest that
-    /// have a level <= CommonLevels and are referred to by the SCEV Expression.
-    void collectCommonLoops(const SCEV *Expression,
-                            const Loop *LoopNest,
-                            SmallBitVector &Loops) const;
-
-    /// checkSrcSubscript - Examines the SCEV Src, returning true iff it's
-    /// linear. Collect the set of loops mentioned by Src.
-    bool checkSrcSubscript(const SCEV *Src,
-                           const Loop *LoopNest,
-                           SmallBitVector &Loops);
-
-    /// checkDstSubscript - Examines the SCEV Dst, returning true iff it's
-    /// linear. Collect the set of loops mentioned by Dst.
-    bool checkDstSubscript(const SCEV *Dst,
-                           const Loop *LoopNest,
-                           SmallBitVector &Loops);
-
-    /// isKnownPredicate - Compare X and Y using the predicate Pred.
-    /// Basically a wrapper for SCEV::isKnownPredicate,
-    /// but tries harder, especially in the presence of sign and zero
-    /// extensions and symbolics.
-    bool isKnownPredicate(ICmpInst::Predicate Pred,
-                          const SCEV *X,
-                          const SCEV *Y) const;
-
-    /// isKnownLessThan - Compare to see if S is less than Size
-    /// Another wrapper for isKnownNegative(S - max(Size, 1)) with some extra
-    /// checking if S is an AddRec and we can prove lessthan using the loop
-    /// bounds.
-    bool isKnownLessThan(const SCEV *S, const SCEV *Size) const;
-
-    /// isKnownNonNegative - Compare to see if S is known not to be negative
-    /// Uses the fact that S comes from Ptr, which may be an inbound GEP,
-    /// Proving there is no wrapping going on.
-    bool isKnownNonNegative(const SCEV *S, const Value *Ptr) const;
-
-    /// collectUpperBound - All subscripts are the same type (on my machine,
-    /// an i64). The loop bound may be a smaller type. collectUpperBound
-    /// find the bound, if available, and zero extends it to the Type T.
-    /// (I zero extend since the bound should always be >= 0.)
-    /// If no upper bound is available, return NULL.
-    const SCEV *collectUpperBound(const Loop *l, Type *T) const;
-
-    /// collectConstantUpperBound - Calls collectUpperBound(), then
-    /// attempts to cast it to SCEVConstant. If the cast fails,
-    /// returns NULL.
-    const SCEVConstant *collectConstantUpperBound(const Loop *l, Type *T) const;
-
-    /// classifyPair - Examines the subscript pair (the Src and Dst SCEVs)
-    /// and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear.
-    /// Collects the associated loops in a set.
-    Subscript::ClassificationKind classifyPair(const SCEV *Src,
-                                           const Loop *SrcLoopNest,
-                                           const SCEV *Dst,
-                                           const Loop *DstLoopNest,
-                                           SmallBitVector &Loops);
-
-    /// testZIV - Tests the ZIV subscript pair (Src and Dst) for dependence.
-    /// Returns true if any possible dependence is disproved.
-    /// If there might be a dependence, returns false.
-    /// If the dependence isn't proven to exist,
-    /// marks the Result as inconsistent.
-    bool testZIV(const SCEV *Src,
-                 const SCEV *Dst,
-                 FullDependence &Result) const;
-
-    /// testSIV - Tests the SIV subscript pair (Src and Dst) for dependence.
-    /// Things of the form [c1 + a1*i] and [c2 + a2*j], where
-    /// i and j are induction variables, c1 and c2 are loop invariant,
-    /// and a1 and a2 are constant.
-    /// Returns true if any possible dependence is disproved.
-    /// If there might be a dependence, returns false.
-    /// Sets appropriate direction vector entry and, when possible,
-    /// the distance vector entry.
-    /// If the dependence isn't proven to exist,
-    /// marks the Result as inconsistent.
-    bool testSIV(const SCEV *Src,
-                 const SCEV *Dst,
-                 unsigned &Level,
-                 FullDependence &Result,
-                 Constraint &NewConstraint,
-                 const SCEV *&SplitIter) const;
-
-    /// testRDIV - Tests the RDIV subscript pair (Src and Dst) for dependence.
-    /// Things of the form [c1 + a1*i] and [c2 + a2*j]
-    /// where i and j are induction variables, c1 and c2 are loop invariant,
-    /// and a1 and a2 are constant.
-    /// With minor algebra, this test can also be used for things like
-    /// [c1 + a1*i + a2*j][c2].
-    /// Returns true if any possible dependence is disproved.
-    /// If there might be a dependence, returns false.
-    /// Marks the Result as inconsistent.
-    bool testRDIV(const SCEV *Src,
-                  const SCEV *Dst,
-                  FullDependence &Result) const;
+    /// getD - If constraint is a distance, returns D.
+    /// Otherwise assert.
+    LLVM_ABI const SCEV *getD() const;
 
-    /// testMIV - Tests the MIV subscript pair (Src and Dst) for dependence.
-    /// Returns true if dependence disproved.
-    /// Can sometimes refine direction vectors.
-    bool testMIV(const SCEV *Src,
-                 const SCEV *Dst,
-                 const SmallBitVector &Loops,
-                 FullDependence &Result) const;
-
-    /// strongSIVtest - Tests the strong SIV subscript pair (Src and Dst)
-    /// for dependence.
-    /// Things of the form [c1 + a*i] and [c2 + a*i],
-    /// where i is an induction variable, c1 and c2 are loop invariant,
-    /// and a is a constant
-    /// Returns true if any possible dependence is disproved.
-    /// If there might be a dependence, returns false.
-    /// Sets appropriate direction and distance.
-    bool strongSIVtest(const SCEV *Coeff,
-                       const SCEV *SrcConst,
-                       const SCEV *DstConst,
-                       const Loop *CurrentLoop,
-                       unsigned Level,
-                       FullDependence &Result,
-                       Constraint &NewConstraint) const;
-
-    /// weakCrossingSIVtest - Tests the weak-crossing SIV subscript pair
-    /// (Src and Dst) for dependence.
-    /// Things of the form [c1 + a*i] and [c2 - a*i],
-    /// where i is an induction variable, c1 and c2 are loop invariant,
-    /// and a is a constant.
-    /// Returns true if any possible dependence is disproved.
-    /// If there might be a dependence, returns false.
-    /// Sets appropriate direction entry.
-    /// Set consistent to false.
-    /// Marks the dependence as splitable.
-    bool weakCrossingSIVtest(const SCEV *SrcCoeff,
-                             const SCEV *SrcConst,
-                             const SCEV *DstConst,
-                             const Loop *CurrentLoop,
-                             unsigned Level,
-                             FullDependence &Result,
-                             Constraint &NewConstraint,
-                             const SCEV *&SplitIter) const;
-
-    /// ExactSIVtest - Tests the SIV subscript pair
-    /// (Src and Dst) for dependence.
-    /// Things of the form [c1 + a1*i] and [c2 + a2*i],
-    /// where i is an induction variable, c1 and c2 are loop invariant,
-    /// and a1 and a2 are constant.
-    /// Returns true if any possible dependence is disproved.
-    /// If there might be a dependence, returns false.
-    /// Sets appropriate direction entry.
-    /// Set consistent to false.
-    bool exactSIVtest(const SCEV *SrcCoeff,
-                      const SCEV *DstCoeff,
-                      const SCEV *SrcConst,
-                      const SCEV *DstConst,
-                      const Loop *CurrentLoop,
-                      unsigned Level,
-                      FullDependence &Result,
-                      Constraint &NewConstraint) const;
-
-    /// weakZeroSrcSIVtest - Tests the weak-zero SIV subscript pair
-    /// (Src and Dst) for dependence.
-    /// Things of the form [c1] and [c2 + a*i],
-    /// where i is an induction variable, c1 and c2 are loop invariant,
-    /// and a is a constant. See also weakZeroDstSIVtest.
-    /// Returns true if any possible dependence is disproved.
-    /// If there might be a dependence, returns false.
-    /// Sets appropriate direction entry.
-    /// Set consistent to false.
-    /// If loop peeling will break the dependence, mark appropriately.
-    bool weakZeroSrcSIVtest(const SCEV *DstCoeff,
-                            const SCEV *SrcConst,
-                            const SCEV *DstConst,
-                            const Loop *CurrentLoop,
-                            unsigned Level,
-                            FullDependence &Result,
-                            Constraint &NewConstraint) const;
-
-    /// weakZeroDstSIVtest - Tests the weak-zero SIV subscript pair
-    /// (Src and Dst) for dependence.
-    /// Things of the form [c1 + a*i] and [c2],
-    /// where i is an induction variable, c1 and c2 are loop invariant,
-    /// and a is a constant. See also weakZeroSrcSIVtest.
-    /// Returns true if any possible dependence is disproved.
-    /// If there might be a dependence, returns false.
-    /// Sets appropriate direction entry.
-    /// Set consistent to false.
-    /// If loop peeling will break the dependence, mark appropriately.
-    bool weakZeroDstSIVtest(const SCEV *SrcCoeff,
-                            const SCEV *SrcConst,
-                            const SCEV *DstConst,
-                            const Loop *CurrentLoop,
-                            unsigned Level,
-                            FullDependence &Result,
-                            Constraint &NewConstraint) const;
-
-    /// exactRDIVtest - Tests the RDIV subscript pair for dependence.
-    /// Things of the form [c1 + a*i] and [c2 + b*j],
-    /// where i and j are induction variable, c1 and c2 are loop invariant,
-    /// and a and b are constants.
-    /// Returns true if any possible dependence is disproved.
-    /// Marks the result as inconsistent.
-    /// Works in some cases that symbolicRDIVtest doesn't,
-    /// and vice versa.
-    bool exactRDIVtest(const SCEV *SrcCoeff,
-                       const SCEV *DstCoeff,
-                       const SCEV *SrcConst,
-                       const SCEV *DstConst,
-                       const Loop *SrcLoop,
-                       const Loop *DstLoop,
-                       FullDependence &Result) const;
+    /// getAssociatedLoop - Returns the loop associated with this constraint.
+    LLVM_ABI const Loop *getAssociatedLoop() const;
 
-    /// symbolicRDIVtest - Tests the RDIV subscript pair for dependence.
-    /// Things of the form [c1 + a*i] and [c2 + b*j],
-    /// where i and j are induction variable, c1 and c2 are loop invariant,
-    /// and a and b are constants.
-    /// Returns true if any possible dependence is disproved.
-    /// Marks the result as inconsistent.
-    /// Works in some cases that exactRDIVtest doesn't,
-    /// and vice versa. Can also be used as a backup for
-    /// ordinary SIV tests.
-    bool symbolicRDIVtest(const SCEV *SrcCoeff,
-                          const SCEV *DstCoeff,
-                          const SCEV *SrcConst,
-                          const SCEV *DstConst,
-                          const Loop *SrcLoop,
-                          const Loop *DstLoop) const;
-
-    /// gcdMIVtest - Tests an MIV subscript pair for dependence.
-    /// Returns true if any possible dependence is disproved.
-    /// Marks the result as inconsistent.
-    /// Can sometimes disprove the equal direction for 1 or more loops.
-    //  Can handle some symbolics that even the SIV tests don't get,
-    /// so we use it as a backup for everything.
-    bool gcdMIVtest(const SCEV *Src,
-                    const SCEV *Dst,
-                    FullDependence &Result) const;
-
-    /// banerjeeMIVtest - Tests an MIV subscript pair for dependence.
-    /// Returns true if any possible dependence is disproved.
-    /// Marks the result as inconsistent.
-    /// Computes directions.
-    bool banerjeeMIVtest(const SCEV *Src,
-                         const SCEV *Dst,
-                         const SmallBitVector &Loops,
-                         FullDependence &Result) const;
-
-    /// collectCoefficientInfo - Walks through the subscript,
-    /// collecting each coefficient, the associated loop bounds,
-    /// and recording its positive and negative parts for later use.
-    CoefficientInfo *collectCoeffInfo(const SCEV *Subscript,
-                                      bool SrcFlag,
-                                      const SCEV *&Constant) const;
-
-    /// getPositivePart - X^+ = max(X, 0).
-    ///
-    const SCEV *getPositivePart(const SCEV *X) const;
-
-    /// getNegativePart - X^- = min(X, 0).
-    ///
-    const SCEV *getNegativePart(const SCEV *X) const;
-
-    /// getLowerBound - Looks through all the bounds info and
-    /// computes the lower bound given the current direction settings
-    /// at each level.
-    const SCEV *getLowerBound(BoundInfo *Bound) const;
-
-    /// getUpperBound - Looks through all the bounds info and
-    /// computes the upper bound given the current direction settings
-    /// at each level.
-    const SCEV *getUpperBound(BoundInfo *Bound) const;
-
-    /// exploreDirections - Hierarchically expands the direction vector
-    /// search space, combining the directions of discovered dependences
-    /// in the DirSet field of Bound. Returns the number of distinct
-    /// dependences discovered. If the dependence is disproved,
-    /// it will return 0.
-    unsigned exploreDirections(unsigned Level,
-                               CoefficientInfo *A,
-                               CoefficientInfo *B,
-                               BoundInfo *Bound,
-                               const SmallBitVector &Loops,
-                               unsigned &DepthExpanded,
-                               const SCEV *Delta) const;
-
-    /// testBounds - Returns true iff the current bounds are plausible.
-    bool testBounds(unsigned char DirKind,
-                    unsigned Level,
-                    BoundInfo *Bound,
-                    const SCEV *Delta) const;
-
-    /// findBoundsALL - Computes the upper and lower bounds for level K
-    /// using the * direction. Records them in Bound.
-    void findBoundsALL(CoefficientInfo *A,
-                       CoefficientInfo *B,
-                       BoundInfo *Bound,
-                       unsigned K) const;
-
-    /// findBoundsLT - Computes the upper and lower bounds for level K
-    /// using the < direction. Records them in Bound.
-    void findBoundsLT(CoefficientInfo *A,
-                      CoefficientInfo *B,
-                      BoundInfo *Bound,
-                      unsigned K) const;
-
-    /// findBoundsGT - Computes the upper and lower bounds for level K
-    /// using the > direction. Records them in Bound.
-    void findBoundsGT(CoefficientInfo *A,
-                      CoefficientInfo *B,
-                      BoundInfo *Bound,
-                      unsigned K) const;
-
-    /// findBoundsEQ - Computes the upper and lower bounds for level K
-    /// using the = direction. Records them in Bound.
-    void findBoundsEQ(CoefficientInfo *A,
-                      CoefficientInfo *B,
-                      BoundInfo *Bound,
-                      unsigned K) const;
-
-    /// intersectConstraints - Updates X with the intersection
-    /// of the Constraints X and Y. Returns true if X has changed.
-    bool intersectConstraints(Constraint *X,
-                              const Constraint *Y);
-
-    /// propagate - Review the constraints, looking for opportunities
-    /// to simplify a subscript pair (Src and Dst).
-    /// Return true if some simplification occurs.
-    /// If the simplification isn't exact (that is, if it is conservative
-    /// in terms of dependence), set consistent to false.
-    bool propagate(const SCEV *&Src,
-                   const SCEV *&Dst,
-                   SmallBitVector &Loops,
-                   SmallVectorImpl<Constraint> &Constraints,
-                   bool &Consistent);
-
-    /// propagateDistance - Attempt to propagate a distance
-    /// constraint into a subscript pair (Src and Dst).
-    /// Return true if some simplification occurs.
-    /// If the simplification isn't exact (that is, if it is conservative
-    /// in terms of dependence), set consistent to false.
-    bool propagateDistance(const SCEV *&Src,
-                           const SCEV *&Dst,
-                           Constraint &CurConstraint,
-                           bool &Consistent);
-
-    /// propagatePoint - Attempt to propagate a point
-    /// constraint into a subscript pair (Src and Dst).
-    /// Return true if some simplification occurs.
-    bool propagatePoint(const SCEV *&Src,
-                        const SCEV *&Dst,
-                        Constraint &CurConstraint);
-
-    /// propagateLine - Attempt to propagate a line
-    /// constraint into a subscript pair (Src and Dst).
-    /// Return true if some simplification occurs.
-    /// If the simplification isn't exact (that is, if it is conservative
-    /// in terms of dependence), set consistent to false.
-    bool propagateLine(const SCEV *&Src,
-                       const SCEV *&Dst,
-                       Constraint &CurConstraint,
-                       bool &Consistent);
-
-    /// findCoefficient - Given a linear SCEV,
-    /// return the coefficient corresponding to specified loop.
-    /// If there isn't one, return the SCEV constant 0.
-    /// For example, given a*i + b*j + c*k, returning the coefficient
-    /// corresponding to the j loop would yield b.
-    const SCEV *findCoefficient(const SCEV *Expr,
-                                const Loop *TargetLoop) const;
-
-    /// zeroCoefficient - Given a linear SCEV,
-    /// return the SCEV given by zeroing out the coefficient
-    /// corresponding to the specified loop.
-    /// For example, given a*i + b*j + c*k, zeroing the coefficient
-    /// corresponding to the j loop would yield a*i + c*k.
-    const SCEV *zeroCoefficient(const SCEV *Expr,
-                                const Loop *TargetLoop) const;
-
-    /// addToCoefficient - Given a linear SCEV Expr,
-    /// return the SCEV given by adding some Value to the
-    /// coefficient corresponding to the specified TargetLoop.
-    /// For example, given a*i + b*j + c*k, adding 1 to the coefficient
-    /// corresponding to the j loop would yield a*i + (b+1)*j + c*k.
-    const SCEV *addToCoefficient(const SCEV *Expr,
-                                 const Loop *TargetLoop,
-                                 const SCEV *Value)  const;
-
-    /// updateDirection - Update direction vector entry
-    /// based on the current constraint.
-    void updateDirection(Dependence::DVEntry &Level,
-                         const Constraint &CurConstraint) const;
-
-    /// Given a linear access function, tries to recover subscripts
-    /// for each dimension of the array element access.
-    bool tryDelinearize(Instruction *Src, Instruction *Dst,
-                        SmallVectorImpl<Subscript> &Pair);
-
-    /// Tries to delinearize \p Src and \p Dst access functions for a fixed size
-    /// multi-dimensional array. Calls tryDelinearizeFixedSizeImpl() to
-    /// delinearize \p Src and \p Dst separately,
-    bool tryDelinearizeFixedSize(Instruction *Src, Instruction *Dst,
-                                 const SCEV *SrcAccessFn,
-                                 const SCEV *DstAccessFn,
-                                 SmallVectorImpl<const SCEV *> &SrcSubscripts,
-                                 SmallVectorImpl<const SCEV *> &DstSubscripts);
-
-    /// Tries to delinearize access function for a multi-dimensional array with
-    /// symbolic runtime sizes.
-    /// Returns true upon success and false otherwise.
-    bool tryDelinearizeParametricSize(
-        Instruction *Src, Instruction *Dst, const SCEV *SrcAccessFn,
-        const SCEV *DstAccessFn, SmallVectorImpl<const SCEV *> &SrcSubscripts,
-        SmallVectorImpl<const SCEV *> &DstSubscripts);
-
-    /// checkSubscript - Helper function for checkSrcSubscript and
-    /// checkDstSubscript to avoid duplicate code
-    bool checkSubscript(const SCEV *Expr, const Loop *LoopNest,
-                        SmallBitVector &Loops, bool IsSrc);
-  }; // class DependenceInfo
-
-  /// AnalysisPass to compute dependence information in a function
-  class DependenceAnalysis : public AnalysisInfoMixin<DependenceAnalysis> {
-  public:
-    typedef DependenceInfo Result;
-    LLVM_ABI Result run(Function &F, FunctionAnalysisManager &FAM);
+    /// setPoint - Change a constraint to Point.
+    LLVM_ABI void setPoint(const SCEV *X, const SCEV *Y,
+                           const Loop *CurrentLoop);
 
-  private:
-    LLVM_ABI static AnalysisKey Key;
-    friend struct AnalysisInfoMixin<DependenceAnalysis>;
-  }; // class DependenceAnalysis
+    /// setLine - Change a constraint to Line.
+    LLVM_ABI void setLine(const SCEV *A, const SCEV *B, const SCEV *C,
+                          const Loop *CurrentLoop);
 
-  /// Printer pass to dump DA results.
-  struct DependenceAnalysisPrinterPass
-      : public PassInfoMixin<DependenceAnalysisPrinterPass> {
-    DependenceAnalysisPrinterPass(raw_ostream &OS,
-                                  bool NormalizeResults = false)
-        : OS(OS), NormalizeResults(NormalizeResults) {}
+    /// setDistance - Change a constraint to Distance.
+    LLVM_ABI void setDistance(const SCEV *D, const Loop *CurrentLoop);
 
-    LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+    /// setEmpty - Change a constraint to Empty.
+    LLVM_ABI void setEmpty();
 
-    static bool isRequired() { return true; }
+    /// setAny - Change a constraint to Any.
+    LLVM_ABI void setAny(ScalarEvolution *SE);
 
-  private:
-    raw_ostream &OS;
-    bool NormalizeResults;
-  }; // class DependenceAnalysisPrinterPass
+    /// dump - For debugging purposes. Dumps the constraint
+    /// out to OS.
+    LLVM_ABI void dump(raw_ostream &OS) const;
+  };
 
-  /// Legacy pass manager pass to access dependence information
-  class LLVM_ABI DependenceAnalysisWrapperPass : public FunctionPass {
-  public:
-    static char ID; // Class identification, replacement for typeinfo
-    DependenceAnalysisWrapperPass();
+  /// establishNestingLevels - Examines the loop nesting of the Src and Dst
+  /// instructions and establishes their shared loops. Sets the variables
+  /// CommonLevels, SrcLevels, and MaxLevels.
+  /// The source and destination instructions needn't be contained in the same
+  /// loop. The routine establishNestingLevels finds the level of most deeply
+  /// nested loop that contains them both, CommonLevels. An instruction that's
+  /// not contained in a loop is at level = 0. MaxLevels is equal to the level
+  /// of the source plus the level of the destination, minus CommonLevels.
+  /// This lets us allocate vectors MaxLevels in length, with room for every
+  /// distinct loop referenced in both the source and destination subscripts.
+  /// The variable SrcLevels is the nesting depth of the source instruction.
+  /// It's used to help calculate distinct loops referenced by the destination.
+  /// Here's the map from loops to levels:
+  ///            0 - unused
+  ///            1 - outermost common loop
+  ///          ... - other common loops
+  /// CommonLevels - innermost common loop
+  ///          ... - loops containing Src but not Dst
+  ///    SrcLevels - innermost loop containing Src but not Dst
+  ///          ... - loops containing Dst but not Src
+  ///    MaxLevels - innermost loop containing Dst but not Src
+  /// Consider the follow code fragment:
+  ///    for (a = ...) {
+  ///      for (b = ...) {
+  ///        for (c = ...) {
+  ///          for (d = ...) {
+  ///            A[] = ...;
+  ///          }
+  ///        }
+  ///        for (e = ...) {
+  ///          for (f = ...) {
+  ///            for (g = ...) {
+  ///              ... = A[];
+  ///            }
+  ///          }
+  ///        }
+  ///      }
+  ///    }
+  /// If we're looking at the possibility of a dependence between the store
+  /// to A (the Src) and the load from A (the Dst), we'll note that they
+  /// have 2 loops in common, so CommonLevels will equal 2 and the direction
+  /// vector for Result will have 2 entries. SrcLevels = 4 and MaxLevels = 7.
+  /// A map from loop names to level indices would look like
+  ///     a - 1
+  ///     b - 2 = CommonLevels
+  ///     c - 3
+  ///     d - 4 = SrcLevels
+  ///     e - 5
+  ///     f - 6
+  ///     g - 7 = MaxLevels
+  void establishNestingLevels(const Instruction *Src, const Instruction *Dst);
+
+  unsigned CommonLevels, SrcLevels, MaxLevels;
+
+  /// mapSrcLoop - Given one of the loops containing the source, return
+  /// its level index in our numbering scheme.
+  unsigned mapSrcLoop(const Loop *SrcLoop) const;
+
+  /// mapDstLoop - Given one of the loops containing the destination,
+  /// return its level index in our numbering scheme.
+  unsigned mapDstLoop(const Loop *DstLoop) const;
+
+  /// isLoopInvariant - Returns true if Expression is loop invariant
+  /// in LoopNest.
+  bool isLoopInvariant(const SCEV *Expression, const Loop *LoopNest) const;
+
+  /// Makes sure all subscript pairs share the same integer type by
+  /// sign-extending as necessary.
+  /// Sign-extending a subscript is safe because getelementptr assumes the
+  /// array subscripts are signed.
+  void unifySubscriptType(ArrayRef<Subscript *> Pairs);
+
+  /// removeMatchingExtensions - Examines a subscript pair.
+  /// If the source and destination are identically sign (or zero)
+  /// extended, it strips off the extension in an effort to
+  /// simplify the actual analysis.
+  void removeMatchingExtensions(Subscript *Pair);
+
+  /// collectCommonLoops - Finds the set of loops from the LoopNest that
+  /// have a level <= CommonLevels and are referred to by the SCEV Expression.
+  void collectCommonLoops(const SCEV *Expression, const Loop *LoopNest,
+                          SmallBitVector &Loops) const;
+
+  /// checkSrcSubscript - Examines the SCEV Src, returning true iff it's
+  /// linear. Collect the set of loops mentioned by Src.
+  bool checkSrcSubscript(const SCEV *Src, const Loop *LoopNest,
+                         SmallBitVector &Loops);
+
+  /// checkDstSubscript - Examines the SCEV Dst, returning true iff it's
+  /// linear. Collect the set of loops mentioned by Dst.
+  bool checkDstSubscript(const SCEV *Dst, const Loop *LoopNest,
+                         SmallBitVector &Loops);
+
+  /// isKnownPredicate - Compare X and Y using the predicate Pred.
+  /// Basically a wrapper for SCEV::isKnownPredicate,
+  /// but tries harder, especially in the presence of sign and zero
+  /// extensions and symbolics.
+  bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *X,
+                        const SCEV *Y) const;
+
+  /// isKnownLessThan - Compare to see if S is less than Size
+  /// Another wrapper for isKnownNegative(S - max(Size, 1)) with some extra
+  /// checking if S is an AddRec and we can prove lessthan using the loop
+  /// bounds.
+  bool isKnownLessThan(const SCEV *S, const SCEV *Size) const;
+
+  /// isKnownNonNegative - Compare to see if S is known not to be negative
+  /// Uses the fact that S comes from Ptr, which may be an inbound GEP,
+  /// Proving there is no wrapping going on.
+  bool isKnownNonNegative(const SCEV *S, const Value *Ptr) const;
+
+  /// collectUpperBound - All subscripts are the same type (on my machine,
+  /// an i64). The loop bound may be a smaller type. collectUpperBound
+  /// find the bound, if available, and zero extends it to the Type T.
+  /// (I zero extend since the bound should always be >= 0.)
+  /// If no upper bound is available, return NULL.
+  const SCEV *collectUpperBound(const Loop *l, Type *T) const;
+
+  /// collectConstantUpperBound - Calls collectUpperBound(), then
+  /// attempts to cast it to SCEVConstant. If the cast fails,
+  /// returns NULL.
+  const SCEVConstant *collectConstantUpperBound(const Loop *l, Type *T) const;
+
+  /// classifyPair - Examines the subscript pair (the Src and Dst SCEVs)
+  /// and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear.
+  /// Collects the associated loops in a set.
+  Subscript::ClassificationKind
+  classifyPair(const SCEV *Src, const Loop *SrcLoopNest, const SCEV *Dst,
+               const Loop *DstLoopNest, SmallBitVector &Loops);
+
+  /// testZIV - Tests the ZIV subscript pair (Src and Dst) for dependence.
+  /// Returns true if any possible dependence is disproved.
+  /// If there might be a dependence, returns false.
+  /// If the dependence isn't proven to exist,
+  /// marks the Result as inconsistent.
+  bool testZIV(const SCEV *Src, const SCEV *Dst, FullDependence &Result) const;
+
+  /// testSIV - Tests the SIV subscript pair (Src and Dst) for dependence.
+  /// Things of the form [c1 + a1*i] and [c2 + a2*j], where
+  /// i and j are induction variables, c1 and c2 are loop invariant,
+  /// and a1 and a2 are constant.
+  /// Returns true if any possible dependence is disproved.
+  /// If there might be a dependence, returns false.
+  /// Sets appropriate direction vector entry and, when possible,
+  /// the distance vector entry.
+  /// If the dependence isn't proven to exist,
+  /// marks the Result as inconsistent.
+  bool testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level,
+               FullDependence &Result, Constraint &NewConstraint,
+               const SCEV *&SplitIter) const;
+
+  /// testRDIV - Tests the RDIV subscript pair (Src and Dst) for dependence.
+  /// Things of the form [c1 + a1*i] and [c2 + a2*j]
+  /// where i and j are induction variables, c1 and c2 are loop invariant,
+  /// and a1 and a2 are constant.
+  /// With minor algebra, this test can also be used for things like
+  /// [c1 + a1*i + a2*j][c2].
+  /// Returns true if any possible dependence is disproved.
+  /// If there might be a dependence, returns false.
+  /// Marks the Result as inconsistent.
+  bool testRDIV(const SCEV *Src, const SCEV *Dst, FullDependence &Result) const;
+
+  /// testMIV - Tests the MIV subscript pair (Src and Dst) for dependence.
+  /// Returns true if dependence disproved.
+  /// Can sometimes refine direction vectors.
+  bool testMIV(const SCEV *Src, const SCEV *Dst, const SmallBitVector &Loops,
+               FullDependence &Result) const;
+
+  /// strongSIVtest - Tests the strong SIV subscript pair (Src and Dst)
+  /// for dependence.
+  /// Things of the form [c1 + a*i] and [c2 + a*i],
+  /// where i is an induction variable, c1 and c2 are loop invariant,
+  /// and a is a constant
+  /// Returns true if any possible dependence is disproved.
+  /// If there might be a dependence, returns false.
+  /// Sets appropriate direction and distance.
+  bool strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
+                     const SCEV *DstConst, const Loop *CurrentLoop,
+                     unsigned Level, FullDependence &Result,
+                     Constraint &NewConstraint) const;
+
+  /// weakCrossingSIVtest - Tests the weak-crossing SIV subscript pair
+  /// (Src and Dst) for dependence.
+  /// Things of the form [c1 + a*i] and [c2 - a*i],
+  /// where i is an induction variable, c1 and c2 are loop invariant,
+  /// and a is a constant.
+  /// Returns true if any possible dependence is disproved.
+  /// If there might be a dependence, returns false.
+  /// Sets appropriate direction entry.
+  /// Set consistent to false.
+  /// Marks the dependence as splitable.
+  bool weakCrossingSIVtest(const SCEV *SrcCoeff, const SCEV *SrcConst,
+                           const SCEV *DstConst, const Loop *CurrentLoop,
+                           unsigned Level, FullDependence &Result,
+                           Constraint &NewConstraint,
+                           const SCEV *&SplitIter) const;
+
+  /// ExactSIVtest - Tests the SIV subscript pair
+  /// (Src and Dst) for dependence.
+  /// Things of the form [c1 + a1*i] and [c2 + a2*i],
+  /// where i is an induction variable, c1 and c2 are loop invariant,
+  /// and a1 and a2 are constant.
+  /// Returns true if any possible dependence is disproved.
+  /// If there might be a dependence, returns false.
+  /// Sets appropriate direction entry.
+  /// Set consistent to false.
+  bool exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
+                    const SCEV *SrcConst, const SCEV *DstConst,
+                    const Loop *CurrentLoop, unsigned Level,
+                    FullDependence &Result, Constraint &NewConstraint) const;
+
+  /// weakZeroSrcSIVtest - Tests the weak-zero SIV subscript pair
+  /// (Src and Dst) for dependence.
+  /// Things of the form [c1] and [c2 + a*i],
+  /// where i is an induction variable, c1 and c2 are loop invariant,
+  /// and a is a constant. See also weakZeroDstSIVtest.
+  /// Returns true if any possible dependence is disproved.
+  /// If there might be a dependence, returns false.
+  /// Sets appropriate direction entry.
+  /// Set consistent to false.
+  /// If loop peeling will break the dependence, mark appropriately.
+  bool weakZeroSrcSIVtest(const SCEV *DstCoeff, const SCEV *SrcConst,
+                          const SCEV *DstConst, const Loop *CurrentLoop,
+                          unsigned Level, FullDependence &Result,
+                          Constraint &NewConstraint) const;
+
+  /// weakZeroDstSIVtest - Tests the weak-zero SIV subscript pair
+  /// (Src and Dst) for dependence.
+  /// Things of the form [c1 + a*i] and [c2],
+  /// where i is an induction variable, c1 and c2 are loop invariant,
+  /// and a is a constant. See also weakZeroSrcSIVtest.
+  /// Returns true if any possible dependence is disproved.
+  /// If there might be a dependence, returns false.
+  /// Sets appropriate direction entry.
+  /// Set consistent to false.
+  /// If loop peeling will break the dependence, mark appropriately.
+  bool weakZeroDstSIVtest(const SCEV *SrcCoeff, const SCEV *SrcConst,
+                          const SCEV *DstConst, const Loop *CurrentLoop,
+                          unsigned Level, FullDependence &Result,
+                          Constraint &NewConstraint) const;
+
+  /// exactRDIVtest - Tests the RDIV subscript pair for dependence.
+  /// Things of the form [c1 + a*i] and [c2 + b*j],
+  /// where i and j are induction variable, c1 and c2 are loop invariant,
+  /// and a and b are constants.
+  /// Returns true if any possible dependence is disproved.
+  /// Marks the result as inconsistent.
+  /// Works in some cases that symbolicRDIVtest doesn't,
+  /// and vice versa.
+  bool exactRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
+                     const SCEV *SrcConst, const SCEV *DstConst,
+                     const Loop *SrcLoop, const Loop *DstLoop,
+                     FullDependence &Result) const;
+
+  /// symbolicRDIVtest - Tests the RDIV subscript pair for dependence.
+  /// Things of the form [c1 + a*i] and [c2 + b*j],
+  /// where i and j are induction variable, c1 and c2 are loop invariant,
+  /// and a and b are constants.
+  /// Returns true if any possible dependence is disproved.
+  /// Marks the result as inconsistent.
+  /// Works in some cases that exactRDIVtest doesn't,
+  /// and vice versa. Can also be used as a backup for
+  /// ordinary SIV tests.
+  bool symbolicRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
+                        const SCEV *SrcConst, const SCEV *DstConst,
+                        const Loop *SrcLoop, const Loop *DstLoop) const;
+
+  /// gcdMIVtest - Tests an MIV subscript pair for dependence.
+  /// Returns true if any possible dependence is disproved.
+  /// Marks the result as inconsistent.
+  /// Can sometimes disprove the equal direction for 1 or more loops.
+  //  Can handle some symbolics that even the SIV tests don't get,
+  /// so we use it as a backup for everything.
+  bool gcdMIVtest(const SCEV *Src, const SCEV *Dst,
+                  FullDependence &Result) const;
 
-    bool runOnFunction(Function &F) override;
-    void releaseMemory() override;
-    void getAnalysisUsage(AnalysisUsage &) const override;
-    void print(raw_ostream &, const Module * = nullptr) const override;
-    DependenceInfo &getDI() const;
+  /// banerjeeMIVtest - Tests an MIV subscript pair for dependence.
+  /// Returns true if any possible dependence is disproved.
+  /// Marks the result as inconsistent.
+  /// Computes directions.
+  bool banerjeeMIVtest(const SCEV *Src, const SCEV *Dst,
+                       const SmallBitVector &Loops,
+                       FullDependence &Result) const;
 
-  private:
-    std::unique_ptr<DependenceInfo> info;
-  }; // class DependenceAnalysisWrapperPass
+  /// collectCoeffInfo - Walks through the subscript, collecting each
+  /// coefficient, the associated loop bounds, and recording its positive and
+  /// negative parts for later use.
+  CoefficientInfo *collectCoeffInfo(const SCEV *Subscript, bool SrcFlag,
+                                    const SCEV *&Constant) const;
 
-  /// createDependenceAnalysisPass - This creates an instance of the
-  /// DependenceAnalysis wrapper pass.
-  LLVM_ABI FunctionPass *createDependenceAnalysisWrapperPass();
+  /// Given \p Expr of the form
+  ///
+  ///   c_0*X_0*i_0 + c_1*X_1*i_1 + ...c_n*X_n*i_n + C
+  ///
+  /// compute
+  ///
+  ///   RunningGCD = gcd(RunningGCD, c_0, c_1, ..., c_n)
+  ///
+  /// where c_0, c_1, ..., and c_n are the constant values. The result is stored
+  /// in \p RunningGCD. Also, the initial value of \p RunningGCD affects the
+  /// result. If we find a term like (c_k * X_k * i_k), where i_k is the
+  /// induction variable of \p CurLoop, c_k is stored in \p CurLoopCoeff and not
+  /// included in the GCD computation. Returns false if we fail to find a
+  /// constant coefficient for some loop, e.g., when a term like (X+Y)*i is
+  /// present. Otherwise returns true.
+  bool accumulateCoefficientsGCD(const SCEV *Expr, const Loop *CurLoop,
+                                 const SCEV *&CurLoopCoeff,
+                                 APInt &RunningGCD) const;
+
+  /// getPositivePart - X^+ = max(X, 0).
+  const SCEV *getPositivePart(const SCEV *X) const;
+
+  /// getNegativePart - X^- = min(X, 0).
+  const SCEV *getNegativePart(const SCEV *X) const;
+
+  /// getLowerBound - Looks through all the bounds info and
+  /// computes the lower bound given the current direction settings
+  /// at each level.
+  const SCEV *getLowerBound(BoundInfo *Bound) const;
+
+  /// getUpperBound - Looks through all the bounds info and
+  /// computes the upper bound given the current direction settings
+  /// at each level.
+  const SCEV *getUpperBound(BoundInfo *Bound) const;
+
+  /// exploreDirections - Hierarchically expands the direction vector
+  /// search space, combining the directions of discovered dependences
+  /// in the DirSet field of Bound. Returns the number of distinct
+  /// dependences discovered. If the dependence is disproved,
+  /// it will return 0.
+  unsigned exploreDirections(unsigned Level, CoefficientInfo *A,
+                             CoefficientInfo *B, BoundInfo *Bound,
+                             const SmallBitVector &Loops,
+                             unsigned &DepthExpanded, const SCEV *Delta) const;
+
+  /// testBounds - Returns true iff the current bounds are plausible.
+  bool testBounds(unsigned char DirKind, unsigned Level, BoundInfo *Bound,
+                  const SCEV *Delta) const;
+
+  /// findBoundsALL - Computes the upper and lower bounds for level K
+  /// using the * direction. Records them in Bound.
+  void findBoundsALL(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound,
+                     unsigned K) const;
+
+  /// findBoundsLT - Computes the upper and lower bounds for level K
+  /// using the < direction. Records them in Bound.
+  void findBoundsLT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound,
+                    unsigned K) const;
+
+  /// findBoundsGT - Computes the upper and lower bounds for level K
+  /// using the > direction. Records them in Bound.
+  void findBoundsGT(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound,
+                    unsigned K) const;
+
+  /// findBoundsEQ - Computes the upper and lower bounds for level K
+  /// using the = direction. Records them in Bound.
+  void findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B, BoundInfo *Bound,
+                    unsigned K) const;
+
+  /// intersectConstraints - Updates X with the intersection
+  /// of the Constraints X and Y. Returns true if X has changed.
+  bool intersectConstraints(Constraint *X, const Constraint *Y);
+
+  /// propagate - Review the constraints, looking for opportunities
+  /// to simplify a subscript pair (Src and Dst).
+  /// Return true if some simplification occurs.
+  /// If the simplification isn't exact (that is, if it is conservative
+  /// in terms of dependence), set consistent to false.
+  bool propagate(const SCEV *&Src, const SCEV *&Dst, SmallBitVector &Loops,
+                 SmallVectorImpl<Constraint> &Constraints, bool &Consistent);
+
+  /// propagateDistance - Attempt to propagate a distance
+  /// constraint into a subscript pair (Src and Dst).
+  /// Return true if some simplification occurs.
+  /// If the simplification isn't exact (that is, if it is conservative
+  /// in terms of dependence), set consistent to false.
+  bool propagateDistance(const SCEV *&Src, const SCEV *&Dst,
+                         Constraint &CurConstraint, bool &Consistent);
+
+  /// propagatePoint - Attempt to propagate a point
+  /// constraint into a subscript pair (Src and Dst).
+  /// Return true if some simplification occurs.
+  bool propagatePoint(const SCEV *&Src, const SCEV *&Dst,
+                      Constraint &CurConstraint);
+
+  /// propagateLine - Attempt to propagate a line
+  /// constraint into a subscript pair (Src and Dst).
+  /// Return true if some simplification occurs.
+  /// If the simplification isn't exact (that is, if it is conservative
+  /// in terms of dependence), set consistent to false.
+  bool propagateLine(const SCEV *&Src, const SCEV *&Dst,
+                     Constraint &CurConstraint, bool &Consistent);
+
+  /// findCoefficient - Given a linear SCEV,
+  /// return the coefficient corresponding to specified loop.
+  /// If there isn't one, return the SCEV constant 0.
+  /// For example, given a*i + b*j + c*k, returning the coefficient
+  /// corresponding to the j loop would yield b.
+  const SCEV *findCoefficient(const SCEV *Expr, const Loop *TargetLoop) const;
+
+  /// zeroCoefficient - Given a linear SCEV,
+  /// return the SCEV given by zeroing out the coefficient
+  /// corresponding to the specified loop.
+  /// For example, given a*i + b*j + c*k, zeroing the coefficient
+  /// corresponding to the j loop would yield a*i + c*k.
+  const SCEV *zeroCoefficient(const SCEV *Expr, const Loop *TargetLoop) const;
+
+  /// addToCoefficient - Given a linear SCEV Expr,
+  /// return the SCEV given by adding some Value to the
+  /// coefficient corresponding to the specified TargetLoop.
+  /// For example, given a*i + b*j + c*k, adding 1 to the coefficient
+  /// corresponding to the j loop would yield a*i + (b+1)*j + c*k.
+  const SCEV *addToCoefficient(const SCEV *Expr, const Loop *TargetLoop,
+                               const SCEV *Value) const;
+
+  /// updateDirection - Update direction vector entry
+  /// based on the current constraint.
+  void updateDirection(Dependence::DVEntry &Level,
+                       const Constraint &CurConstraint) const;
+
+  /// Given a linear access function, tries to recover subscripts
+  /// for each dimension of the array element access.
+  bool tryDelinearize(Instruction *Src, Instruction *Dst,
+                      SmallVectorImpl<Subscript> &Pair);
+
+  /// Tries to delinearize \p Src and \p Dst access functions for a fixed size
+  /// multi-dimensional array. Calls tryDelinearizeFixedSizeImpl() to
+  /// delinearize \p Src and \p Dst separately,
+  bool tryDelinearizeFixedSize(Instruction *Src, Instruction *Dst,
+                               const SCEV *SrcAccessFn, const SCEV *DstAccessFn,
+                               SmallVectorImpl<const SCEV *> &SrcSubscripts,
+                               SmallVectorImpl<const SCEV *> &DstSubscripts);
+
+  /// Tries to delinearize access function for a multi-dimensional array with
+  /// symbolic runtime sizes.
+  /// Returns true upon success and false otherwise.
+  bool
+  tryDelinearizeParametricSize(Instruction *Src, Instruction *Dst,
+                               const SCEV *SrcAccessFn, const SCEV *DstAccessFn,
+                               SmallVectorImpl<const SCEV *> &SrcSubscripts,
+                               SmallVectorImpl<const SCEV *> &DstSubscripts);
+
+  /// checkSubscript - Helper function for checkSrcSubscript and
+  /// checkDstSubscript to avoid duplicate code
+  bool checkSubscript(const SCEV *Expr, const Loop *LoopNest,
+                      SmallBitVector &Loops, bool IsSrc);
+}; // class DependenceInfo
+
+/// AnalysisPass to compute dependence information in a function
+class DependenceAnalysis : public AnalysisInfoMixin<DependenceAnalysis> {
+public:
+  typedef DependenceInfo Result;
+  LLVM_ABI Result run(Function &F, FunctionAnalysisManager &FAM);
+
+private:
+  LLVM_ABI static AnalysisKey Key;
+  friend struct AnalysisInfoMixin<DependenceAnalysis>;
+}; // class DependenceAnalysis
+
+/// Printer pass to dump DA results.
+struct DependenceAnalysisPrinterPass
+    : public PassInfoMixin<DependenceAnalysisPrinterPass> {
+  DependenceAnalysisPrinterPass(raw_ostream &OS, bool NormalizeResults = false)
+      : OS(OS), NormalizeResults(NormalizeResults) {}
+
+  LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+
+  static bool isRequired() { return true; }
+
+private:
+  raw_ostream &OS;
+  bool NormalizeResults;
+}; // class DependenceAnalysisPrinterPass
+
+/// Legacy pass manager pass to access dependence information
+class LLVM_ABI DependenceAnalysisWrapperPass : public FunctionPass {
+public:
+  static char ID; // Class identification, replacement for typeinfo
+  DependenceAnalysisWrapperPass();
+
+  bool runOnFunction(Function &F) override;
+  void releaseMemory() override;
+  void getAnalysisUsage(AnalysisUsage &) const override;
+  void print(raw_ostream &, const Module * = nullptr) const override;
+  DependenceInfo &getDI() const;
+
+private:
+  std::unique_ptr<DependenceInfo> info;
+}; // class DependenceAnalysisWrapperPass
+
+/// createDependenceAnalysisPass - This creates an instance of the
+/// DependenceAnalysis wrapper pass.
+LLVM_ABI FunctionPass *createDependenceAnalysisWrapperPass();
 
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Analysis/GenericDomTreeUpdaterImpl.h b/llvm/include/llvm/Analysis/GenericDomTreeUpdaterImpl.h
index 896b68c5021b3..6bfad783b529b 100644
--- a/llvm/include/llvm/Analysis/GenericDomTreeUpdaterImpl.h
+++ b/llvm/include/llvm/Analysis/GenericDomTreeUpdaterImpl.h
@@ -383,7 +383,7 @@ void GenericDomTreeUpdater<DerivedT, DomTreeT, PostDomTreeT>::
   // field of all the elements of Edges.
   // I.e., forall elt in Edges, it exists BB in NewBBs
   // such as BB == elt.NewBB.
-  SmallSet<BasicBlockT *, 32> NewBBs;
+  SmallPtrSet<BasicBlockT *, 32> NewBBs;
   for (auto &Edge : Edges)
     NewBBs.insert(Edge.NewBB);
   // For each element in Edges, remember whether or not element
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 1dc73205a0ebb..f9e6da6d0846a 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -35,6 +35,8 @@ enum class RecurKind {
   // clang-format off
   None,     ///< Not a recurrence.
   Add,      ///< Sum of integers.
+  Sub,      ///< Subtraction of integers
+  AddChainWithSubs, ///< A chain of adds and subs
   Mul,      ///< Product of integers.
   Or,       ///< Bitwise or logical OR of integers.
   And,      ///< Bitwise or logical AND of integers.
diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h
index 93b0a8d3cef04..ae86f353f16d9 100644
--- a/llvm/include/llvm/Analysis/InlineCost.h
+++ b/llvm/include/llvm/Analysis/InlineCost.h
@@ -336,7 +336,11 @@ LLVM_ABI std::optional<InlineCostFeatures> getInliningCostFeatures(
     ProfileSummaryInfo *PSI = nullptr,
     OptimizationRemarkEmitter *ORE = nullptr);
 
-/// Minimal filter to detect invalid constructs for inlining.
+/// Check if it is mechanically possible to inline the function \p Callee, based
+/// on the contents of the function.
+///
+/// See also \p CanInlineCallSite as an additional precondition necessary to
+/// perform a valid inline in a particular use context.
 LLVM_ABI InlineResult isInlineViable(Function &Callee);
 
 // This pass is used to annotate instructions during the inline process for
diff --git a/llvm/include/llvm/Analysis/LoopInfo.h b/llvm/include/llvm/Analysis/LoopInfo.h
index a7a6a2753709c..f80744e70f7ad 100644
--- a/llvm/include/llvm/Analysis/LoopInfo.h
+++ b/llvm/include/llvm/Analysis/LoopInfo.h
@@ -59,11 +59,12 @@ class LLVM_ABI Loop : public LoopBase<BasicBlock, Loop> {
   };
 
   /// Return true if the specified value is loop invariant.
-  bool isLoopInvariant(const Value *V) const;
+  bool isLoopInvariant(const Value *V, bool HasCoroSuspendInst = false) const;
 
   /// Return true if all the operands of the specified instruction are loop
   /// invariant.
-  bool hasLoopInvariantOperands(const Instruction *I) const;
+  bool hasLoopInvariantOperands(const Instruction *I,
+                                bool HasCoroSuspendInst = false) const;
 
   /// If the given value is an instruction inside of the loop and it can be
   /// hoisted, do so to make it trivially loop-invariant.
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
index 011d5994dc670..75b2bc1c067ec 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h
@@ -116,6 +116,18 @@ struct is_specific_cst {
 /// Match an SCEV constant with a plain unsigned integer.
 inline cst_pred_ty<is_specific_cst> m_scev_SpecificInt(uint64_t V) { return V; }
 
+struct is_specific_signed_cst {
+  int64_t CV;
+  is_specific_signed_cst(int64_t C) : CV(C) {}
+  bool isValue(const APInt &C) const { return C.trySExtValue() == CV; }
+};
+
+/// Match an SCEV constant with a plain signed integer (sign-extended value will
+/// be matched)
+inline cst_pred_ty<is_specific_signed_cst> m_scev_SpecificSInt(int64_t V) {
+  return V;
+}
+
 struct bind_cst_ty {
   const APInt *&CR;
 
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index aa4550de455e0..c4ba8e9857dc4 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -961,12 +961,10 @@ class TargetTransformInfo {
       TTI::TargetCostKind CostKind, bool ForPoisonSrc = true,
       ArrayRef<Value *> VL = {}) const;
 
-  /// Estimate the overhead of scalarizing an instructions unique
-  /// non-constant operands. The (potentially vector) types to use for each of
-  /// argument are passes via Tys.
+  /// Estimate the overhead of scalarizing operands with the given types. The
+  /// (potentially vector) types to use for each of argument are passes via Tys.
   LLVM_ABI InstructionCost getOperandsScalarizationOverhead(
-      ArrayRef<const Value *> Args, ArrayRef<Type *> Tys,
-      TTI::TargetCostKind CostKind) const;
+      ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const;
 
   /// If target has efficient vector element load/store instructions, it can
   /// return true here so that insertion/extraction costs are not added to
@@ -1512,6 +1510,14 @@ class TargetTransformInfo {
                                               TTI::TargetCostKind CostKind,
                                               unsigned Index = -1) const;
 
+  /// \return The expected cost of inserting or extracting a lane that is \p
+  /// Index elements from the end of a vector, i.e. the mathematical expression
+  /// for the lane is (VF - 1 - Index). This is required for scalable vectors
+  /// where the exact lane index is unknown at compile time.
+  LLVM_ABI InstructionCost getIndexedVectorInstrCostFromEnd(
+      unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+      unsigned Index) const;
+
   /// \return The expected cost of aggregate inserts and extracts. This is
   /// used when the instruction is not available; a typical use case is to
   /// provision the cost of vectorization/scalarization in vectorizer passes.
@@ -1675,13 +1681,14 @@ class TargetTransformInfo {
 
   /// \returns The cost of the address computation. For most targets this can be
   /// merged into the instruction indexing mode. Some targets might want to
-  /// distinguish between address computation for memory operations on vector
-  /// types and scalar types. Such targets should override this function.
-  /// The 'SE' parameter holds pointer for the scalar evolution object which
-  /// is used in order to get the Ptr step value in case of constant stride.
-  /// The 'Ptr' parameter holds SCEV of the access pointer.
-  LLVM_ABI InstructionCost getAddressComputationCost(
-      Type *Ty, ScalarEvolution *SE = nullptr, const SCEV *Ptr = nullptr) const;
+  /// distinguish between address computation for memory operations with vector
+  /// pointer types and scalar pointer types. Such targets should override this
+  /// function. \p SE holds the pointer for the scalar evolution object which
+  /// was used in order to get the Ptr step value. \p Ptr holds the SCEV of the
+  /// access pointer.
+  LLVM_ABI InstructionCost
+  getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,
+                            TTI::TargetCostKind CostKind) const;
 
   /// \returns The cost, if any, of keeping values of the given types alive
   /// over a callsite.
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index abdbca04488db..43813d2f3acb5 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -459,8 +459,7 @@ class TargetTransformInfoImplBase {
   }
 
   virtual InstructionCost
-  getOperandsScalarizationOverhead(ArrayRef<const Value *> Args,
-                                   ArrayRef<Type *> Tys,
+  getOperandsScalarizationOverhead(ArrayRef<Type *> Tys,
                                    TTI::TargetCostKind CostKind) const {
     return 0;
   }
@@ -731,6 +730,13 @@ class TargetTransformInfoImplBase {
         return 0;
       break;
     }
+    case Instruction::PtrToAddr: {
+      unsigned DstSize = Dst->getScalarSizeInBits();
+      assert(DstSize == DL.getAddressSizeInBits(Src));
+      if (DL.isLegalInteger(DstSize))
+        return 0;
+      break;
+    }
     case Instruction::PtrToInt: {
       unsigned DstSize = Dst->getScalarSizeInBits();
       if (DL.isLegalInteger(DstSize) &&
@@ -802,6 +808,13 @@ class TargetTransformInfoImplBase {
     return 1;
   }
 
+  virtual InstructionCost
+  getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
+                                   TTI::TargetCostKind CostKind,
+                                   unsigned Index) const {
+    return 1;
+  }
+
   virtual InstructionCost
   getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
                             const APInt &DemandedDstElts,
@@ -878,9 +891,6 @@ class TargetTransformInfoImplBase {
     switch (ICA.getID()) {
     default:
       break;
-    case Intrinsic::experimental_vector_histogram_add:
-      // For now, we want explicit support from the target for histograms.
-      return InstructionCost::getInvalid();
     case Intrinsic::allow_runtime_check:
     case Intrinsic::allow_ubsan_check:
     case Intrinsic::annotation:
@@ -933,8 +943,10 @@ class TargetTransformInfoImplBase {
   // Assume that we have a register of the right size for the type.
   virtual unsigned getNumberOfParts(Type *Tp) const { return 1; }
 
-  virtual InstructionCost getAddressComputationCost(Type *Tp, ScalarEvolution *,
-                                                    const SCEV *) const {
+  virtual InstructionCost getAddressComputationCost(Type *PtrTy,
+                                                    ScalarEvolution *,
+                                                    const SCEV *,
+                                                    TTI::TargetCostKind) const {
     return 0;
   }
 
@@ -1439,6 +1451,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
                                                Op2Info, Operands, I);
     }
     case Instruction::IntToPtr:
+    case Instruction::PtrToAddr:
     case Instruction::PtrToInt:
     case Instruction::SIToFP:
     case Instruction::UIToFP:
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 02990a3cb44f7..15ff129deda13 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -359,11 +359,6 @@ GetPointerBaseWithConstantOffset(const Value *Ptr, int64_t &Offset,
                                           AllowNonInbounds);
 }
 
-/// Returns true if the GEP is based on a pointer to a string (array of
-// \p CharSize integers) and is indexing into this string.
-LLVM_ABI bool isGEPBasedOnPointerToString(const GEPOperator *GEP,
-                                          unsigned CharSize = 8);
-
 /// Represents offset+length into a ConstantDataArray.
 struct ConstantDataArraySlice {
   /// ConstantDataArray pointer. nullptr indicates a zeroinitializer (a valid
diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h
index a2311d2ac285d..e6a0eae9da30c 100644
--- a/llvm/include/llvm/AsmParser/LLToken.h
+++ b/llvm/include/llvm/AsmParser/LLToken.h
@@ -319,6 +319,7 @@ enum Kind {
   kw_fptoui,
   kw_fptosi,
   kw_inttoptr,
+  kw_ptrtoaddr,
   kw_ptrtoint,
   kw_bitcast,
   kw_addrspacecast,
diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index 89abca02efefa..f74c9775cb3f3 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/DXILABI.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/SwapByteOrder.h"
 #include "llvm/TargetParser/Triple.h"
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def
index 48b33478d5045..b561125fe37a4 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -625,6 +625,9 @@ HANDLE_DW_AT(0x3e0a, LLVM_ptrauth_authentication_mode, 0, LLVM)
 HANDLE_DW_AT(0x3e0b, LLVM_num_extra_inhabitants, 0, LLVM)
 HANDLE_DW_AT(0x3e0c, LLVM_stmt_sequence, 0, LLVM)
 HANDLE_DW_AT(0x3e0d, LLVM_coro_suspend_idx, 0, LLVM)
+// The DWARF v6 working draft defines DW_AT_alloc_type; use this LLVM-private ID
+// until that is released as an official standard.
+HANDLE_DW_AT(0x3e0e, LLVM_alloc_type, 0, LLVM)
 
 // Apple extensions.
 
diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def
index 05b79eae573f7..8dcc2926e1b59 100644
--- a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def
+++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def
@@ -61,6 +61,7 @@ ELF_RELOC(R_AARCH64_LD64_GOT_LO12_NC,                0x138)
 ELF_RELOC(R_AARCH64_LD64_GOTPAGE_LO15,               0x139)
 ELF_RELOC(R_AARCH64_PLT32,                           0x13a)
 ELF_RELOC(R_AARCH64_GOTPCREL32,                      0x13b)
+ELF_RELOC(R_AARCH64_PATCHINST,                       0x13c)
 // General dynamic TLS relocations
 ELF_RELOC(R_AARCH64_TLSGD_ADR_PREL21,                0x200)
 ELF_RELOC(R_AARCH64_TLSGD_ADR_PAGE21,                0x201)
diff --git a/llvm/include/llvm/BinaryFormat/MsgPackDocument.h b/llvm/include/llvm/BinaryFormat/MsgPackDocument.h
index 26fff8d5f8d30..f09feabb10284 100644
--- a/llvm/include/llvm/BinaryFormat/MsgPackDocument.h
+++ b/llvm/include/llvm/BinaryFormat/MsgPackDocument.h
@@ -213,6 +213,7 @@ class DocNode {
   LLVM_ABI DocNode &operator=(unsigned Val);
   LLVM_ABI DocNode &operator=(int64_t Val);
   LLVM_ABI DocNode &operator=(uint64_t Val);
+  LLVM_ABI DocNode &operator=(double Val);
 
 private:
   // Private constructor setting KindAndDoc, used by methods in Document.
diff --git a/llvm/include/llvm/BinaryFormat/SFrame.h b/llvm/include/llvm/BinaryFormat/SFrame.h
index 0c6c4d176eaec..095db18b9c254 100644
--- a/llvm/include/llvm/BinaryFormat/SFrame.h
+++ b/llvm/include/llvm/BinaryFormat/SFrame.h
@@ -104,14 +104,8 @@ template <endianness E> struct Header {
   detail::packed<uint32_t, E> FREOff;
 };
 
-template <endianness E> struct FuncDescEntry {
-  detail::packed<int32_t, E> StartAddress;
-  detail::packed<uint32_t, E> Size;
-  detail::packed<uint32_t, E> StartFREOff;
-  detail::packed<uint32_t, E> NumFREs;
+template <endianness E> struct FDEInfo {
   detail::packed<uint8_t, E> Info;
-  detail::packed<uint8_t, E> RepSize;
-  detail::packed<uint16_t, E> Padding2;
 
   uint8_t getPAuthKey() const { return (Info >> 5) & 1; }
   FDEType getFDEType() const { return static_cast<FDEType>((Info >> 4) & 1); }
@@ -125,6 +119,16 @@ template <endianness E> struct FuncDescEntry {
   }
 };
 
+template <endianness E> struct FuncDescEntry {
+  detail::packed<int32_t, E> StartAddress;
+  detail::packed<uint32_t, E> Size;
+  detail::packed<uint32_t, E> StartFREOff;
+  detail::packed<uint32_t, E> NumFREs;
+  FDEInfo<E> Info;
+  detail::packed<uint8_t, E> RepSize;
+  detail::packed<uint16_t, E> Padding2;
+};
+
 template <endianness E> struct FREInfo {
   detail::packed<uint8_t, E> Info;
 
@@ -169,6 +173,7 @@ LLVM_ABI ArrayRef<EnumEntry<FREType>> getFRETypes();
 LLVM_ABI ArrayRef<EnumEntry<FDEType>> getFDETypes();
 LLVM_ABI ArrayRef<EnumEntry<AArch64PAuthKey>> getAArch64PAuthKeys();
 LLVM_ABI ArrayRef<EnumEntry<FREOffset>> getFREOffsets();
+LLVM_ABI ArrayRef<EnumEntry<BaseReg>> getBaseRegisters();
 
 } // namespace sframe
 } // namespace llvm
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index dc78eb4164acf..1c7d3462b6bae 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -456,7 +456,8 @@ enum CastOpcodes {
   CAST_PTRTOINT = 9,
   CAST_INTTOPTR = 10,
   CAST_BITCAST = 11,
-  CAST_ADDRSPACECAST = 12
+  CAST_ADDRSPACECAST = 12,
+  CAST_PTRTOADDR = 13,
 };
 
 /// UnaryOpcodes - These are values used in the bitcode files to encode which
diff --git a/llvm/include/llvm/CAS/ActionCache.h b/llvm/include/llvm/CAS/ActionCache.h
new file mode 100644
index 0000000000000..69ee4dde1974a
--- /dev/null
+++ b/llvm/include/llvm/CAS/ActionCache.h
@@ -0,0 +1,102 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declaration of the ActionCache class, which is the
+/// base class for ActionCache implementations.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_ACTIONCACHE_H
+#define LLVM_CAS_ACTIONCACHE_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CAS/CASID.h"
+#include "llvm/CAS/CASReference.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm::cas {
+
+class ObjectStore;
+class CASID;
+class ObjectProxy;
+
+/// A key for caching an operation.
+/// It is implemented as a bag of bytes and provides a convenient constructor
+/// for CAS types.
+class CacheKey {
+public:
+  StringRef getKey() const { return Key; }
+
+  CacheKey(const CASID &ID);
+  CacheKey(const ObjectProxy &Proxy);
+  CacheKey(const ObjectStore &CAS, const ObjectRef &Ref);
+
+private:
+  std::string Key;
+};
+
+/// A cache from a key (that describes an action) to the result of performing
+/// that action.
+///
+/// Actions are expected to be pure. Storing mappings from one action to
+/// multiple results will result in error (cache poisoning).
+class ActionCache {
+  virtual void anchor();
+
+public:
+  /// Get a previously computed result for \p ActionKey.
+  ///
+  /// \param CanBeDistributed is a hint to the underlying implementation that if
+  /// it is true, the lookup is profitable to be done on a distributed caching
+  /// level, not just locally. The implementation is free to ignore this flag.
+  Expected<std::optional<CASID>> get(const CacheKey &ActionKey,
+                                     bool CanBeDistributed = false) const {
+    return getImpl(arrayRefFromStringRef(ActionKey.getKey()), CanBeDistributed);
+  }
+
+  /// Cache \p Result for the \p ActionKey computation.
+  ///
+  /// \param CanBeDistributed is a hint to the underlying implementation that if
+  /// it is true, the association is profitable to be done on a distributed
+  /// caching level, not just locally. The implementation is free to ignore this
+  /// flag.
+  Error put(const CacheKey &ActionKey, const CASID &Result,
+            bool CanBeDistributed = false) {
+    assert(Result.getContext().getHashSchemaIdentifier() ==
+               getContext().getHashSchemaIdentifier() &&
+           "Hash schema mismatch");
+    return putImpl(arrayRefFromStringRef(ActionKey.getKey()), Result,
+                   CanBeDistributed);
+  }
+
+  virtual ~ActionCache() = default;
+
+protected:
+  // Implementation detail for \p get method.
+  virtual Expected<std::optional<CASID>>
+  getImpl(ArrayRef<uint8_t> ResolvedKey, bool CanBeDistributed) const = 0;
+
+  // Implementation details for \p put method.
+  virtual Error putImpl(ArrayRef<uint8_t> ResolvedKey, const CASID &Result,
+                        bool CanBeDistributed) = 0;
+
+  ActionCache(const CASContext &Context) : Context(Context) {}
+
+  const CASContext &getContext() const { return Context; }
+
+private:
+  const CASContext &Context;
+};
+
+/// Create an action cache in memory.
+std::unique_ptr<ActionCache> createInMemoryActionCache();
+
+} // end namespace llvm::cas
+
+#endif // LLVM_CAS_ACTIONCACHE_H
diff --git a/llvm/include/llvm/CAS/BuiltinCASContext.h b/llvm/include/llvm/CAS/BuiltinCASContext.h
new file mode 100644
index 0000000000000..e9a226a423e5a
--- /dev/null
+++ b/llvm/include/llvm/CAS/BuiltinCASContext.h
@@ -0,0 +1,89 @@
+//===- BuiltinCASContext.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_BUILTINCASCONTEXT_H
+#define LLVM_CAS_BUILTINCASCONTEXT_H
+
+#include "llvm/CAS/CASID.h"
+#include "llvm/Support/BLAKE3.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm::cas::builtin {
+
+/// Current hash type for the builtin CAS.
+///
+/// FIXME: This should be configurable via an enum to allow configuring the hash
+/// function. The enum should be sent into \a createInMemoryCAS() and \a
+/// createOnDiskCAS().
+///
+/// This is important (at least) for future-proofing, when we want to make new
+/// CAS instances use BLAKE7, but still know how to read/write BLAKE3.
+///
+/// Even just for BLAKE3, it would be useful to have these values:
+///
+///     BLAKE3     => 32B hash from BLAKE3
+///     BLAKE3_16B => 16B hash from BLAKE3 (truncated)
+///
+/// ... where BLAKE3_16 uses \a TruncatedBLAKE3<16>.
+///
+/// Motivation for a truncated hash is that it's cheaper to store. It's not
+/// clear if we always (or ever) need the full 32B, and for an ephemeral
+/// in-memory CAS, we almost certainly don't need it.
+///
+/// Note that the cost is linear in the number of objects for the builtin CAS,
+/// since we're using internal offsets and/or pointers as an optimization.
+///
+/// However, it's possible we'll want to hook up a local builtin CAS to, e.g.,
+/// a distributed generic hash map to use as an ActionCache. In that scenario,
+/// the transitive closure of the structured objects that are the results of
+/// the cached actions would need to be serialized into the map, something
+/// like:
+///
+///     "action:<schema>:<key>" -> "0123"
+///     "object:<schema>:0123"  -> "3,4567,89AB,CDEF,9,some data"
+///     "object:<schema>:4567"  -> ...
+///     "object:<schema>:89AB"  -> ...
+///     "object:<schema>:CDEF"  -> ...
+///
+/// These references would be full cost.
+using HasherT = BLAKE3;
+using HashType = decltype(HasherT::hash(std::declval<ArrayRef<uint8_t> &>()));
+
+/// CASContext for LLVM builtin CAS using BLAKE3 hash type.
+class BuiltinCASContext : public CASContext {
+  void printIDImpl(raw_ostream &OS, const CASID &ID) const final;
+  void anchor() override;
+
+public:
+  /// Get the name of the hash for any table identifiers.
+  ///
+  /// FIXME: This should be configurable via an enum, with at the following
+  /// values:
+  ///
+  ///     "BLAKE3"    => 32B hash from BLAKE3
+  ///     "BLAKE3.16" => 16B hash from BLAKE3 (truncated)
+  ///
+  /// Enum can be sent into \a createInMemoryCAS() and \a createOnDiskCAS().
+  static StringRef getHashName() { return "BLAKE3"; }
+  StringRef getHashSchemaIdentifier() const final {
+    static const std::string ID =
+        ("llvm.cas.builtin.v2[" + getHashName() + "]").str();
+    return ID;
+  }
+
+  static const BuiltinCASContext &getDefaultContext();
+
+  BuiltinCASContext() = default;
+
+  static Expected<HashType> parseID(StringRef PrintedDigest);
+  static void printID(ArrayRef<uint8_t> Digest, raw_ostream &OS);
+};
+
+} // namespace llvm::cas::builtin
+
+#endif // LLVM_CAS_BUILTINCASCONTEXT_H
diff --git a/llvm/include/llvm/CAS/BuiltinObjectHasher.h b/llvm/include/llvm/CAS/BuiltinObjectHasher.h
new file mode 100644
index 0000000000000..c9b004216f796
--- /dev/null
+++ b/llvm/include/llvm/CAS/BuiltinObjectHasher.h
@@ -0,0 +1,82 @@
+//===- BuiltinObjectHasher.h ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_BUILTINOBJECTHASHER_H
+#define LLVM_CAS_BUILTINOBJECTHASHER_H
+
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/Endian.h"
+
+namespace llvm::cas {
+
+/// Hasher for stored objects in builtin CAS.
+template <class HasherT> class BuiltinObjectHasher {
+public:
+  using HashT = decltype(HasherT::hash(std::declval<ArrayRef<uint8_t> &>()));
+
+  static HashT hashObject(const ObjectStore &CAS, ArrayRef<ObjectRef> Refs,
+                          ArrayRef<char> Data) {
+    BuiltinObjectHasher H;
+    H.updateSize(Refs.size());
+    for (const ObjectRef &Ref : Refs)
+      H.updateRef(CAS, Ref);
+    H.updateArray(Data);
+    return H.finish();
+  }
+
+  static HashT hashObject(ArrayRef<ArrayRef<uint8_t>> Refs,
+                          ArrayRef<char> Data) {
+    BuiltinObjectHasher H;
+    H.updateSize(Refs.size());
+    for (const ArrayRef<uint8_t> &Ref : Refs)
+      H.updateID(Ref);
+    H.updateArray(Data);
+    return H.finish();
+  }
+
+private:
+  HashT finish() { return Hasher.final(); }
+
+  void updateRef(const ObjectStore &CAS, ObjectRef Ref) {
+    updateID(CAS.getID(Ref));
+  }
+
+  void updateID(const CASID &ID) { updateID(ID.getHash()); }
+
+  void updateID(ArrayRef<uint8_t> Hash) {
+    // NOTE: Does not hash the size of the hash. That's a CAS implementation
+    // detail that shouldn't leak into the UUID for an object.
+    assert(Hash.size() == sizeof(HashT) &&
+           "Expected object ref to match the hash size");
+    Hasher.update(Hash);
+  }
+
+  void updateArray(ArrayRef<uint8_t> Bytes) {
+    updateSize(Bytes.size());
+    Hasher.update(Bytes);
+  }
+
+  void updateArray(ArrayRef<char> Bytes) {
+    updateArray(ArrayRef(reinterpret_cast<const uint8_t *>(Bytes.data()),
+                         Bytes.size()));
+  }
+
+  void updateSize(uint64_t Size) {
+    Size = support::endian::byte_swap(Size, endianness::little);
+    Hasher.update(
+        ArrayRef(reinterpret_cast<const uint8_t *>(&Size), sizeof(Size)));
+  }
+
+  BuiltinObjectHasher() = default;
+  ~BuiltinObjectHasher() = default;
+  HasherT Hasher;
+};
+
+} // namespace llvm::cas
+
+#endif // LLVM_CAS_BUILTINOBJECTHASHER_H
diff --git a/llvm/include/llvm/CAS/CASID.h b/llvm/include/llvm/CAS/CASID.h
new file mode 100644
index 0000000000000..882099451c5cc
--- /dev/null
+++ b/llvm/include/llvm/CAS/CASID.h
@@ -0,0 +1,147 @@
+//===- llvm/CAS/CASID.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_CASID_H
+#define LLVM_CAS_CASID_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+
+class raw_ostream;
+
+namespace cas {
+
+class CASID;
+
+/// Context for CAS identifiers.
+class CASContext {
+  virtual void anchor();
+
+public:
+  virtual ~CASContext() = default;
+
+  /// Get an identifer for the schema used by this CAS context. Two CAS
+  /// instances should return \c true for this identifier if and only if their
+  /// CASIDs are safe to compare by hash. This is used by \a
+  /// CASID::equalsImpl().
+  virtual StringRef getHashSchemaIdentifier() const = 0;
+
+protected:
+  /// Print \p ID to \p OS.
+  virtual void printIDImpl(raw_ostream &OS, const CASID &ID) const = 0;
+
+  friend class CASID;
+};
+
+/// Unique identifier for a CAS object.
+///
+/// Locally, stores an internal CAS identifier that's specific to a single CAS
+/// instance. It's guaranteed not to change across the view of that CAS, but
+/// might change between runs.
+///
+/// It also has \a CASIDContext pointer to allow comparison of these
+/// identifiers. If two CASIDs are from the same CASIDContext, they can be
+/// compared directly. If they are, then \a
+/// CASIDContext::getHashSchemaIdentifier() is compared to see if they can be
+/// compared by hash, in which case the result of \a getHash() is compared.
+class CASID {
+public:
+  void dump() const;
+
+  friend raw_ostream &operator<<(raw_ostream &OS, const CASID &ID) {
+    ID.print(OS);
+    return OS;
+  }
+
+  /// Print CASID.
+  void print(raw_ostream &OS) const {
+    return getContext().printIDImpl(OS, *this);
+  }
+
+  /// Return a printable string for CASID.
+  std::string toString() const;
+
+  ArrayRef<uint8_t> getHash() const {
+    return arrayRefFromStringRef<uint8_t>(Hash);
+  }
+
+  friend bool operator==(const CASID &LHS, const CASID &RHS) {
+    if (LHS.Context == RHS.Context)
+      return LHS.Hash == RHS.Hash;
+
+    // EmptyKey or TombstoneKey.
+    if (!LHS.Context || !RHS.Context)
+      return false;
+
+    // CASIDs are equal when they have the same hash schema and same hash value.
+    return LHS.Context->getHashSchemaIdentifier() ==
+               RHS.Context->getHashSchemaIdentifier() &&
+           LHS.Hash == RHS.Hash;
+  }
+
+  friend bool operator!=(const CASID &LHS, const CASID &RHS) {
+    return !(LHS == RHS);
+  }
+
+  friend hash_code hash_value(const CASID &ID) {
+    ArrayRef<uint8_t> Hash = ID.getHash();
+    return hash_combine_range(Hash.begin(), Hash.end());
+  }
+
+  const CASContext &getContext() const {
+    assert(Context && "Tombstone or empty key for DenseMap?");
+    return *Context;
+  }
+
+  static CASID getDenseMapEmptyKey() {
+    return CASID(nullptr, DenseMapInfo<StringRef>::getEmptyKey());
+  }
+  static CASID getDenseMapTombstoneKey() {
+    return CASID(nullptr, DenseMapInfo<StringRef>::getTombstoneKey());
+  }
+
+  CASID() = delete;
+
+  /// Create CASID from CASContext and raw hash bytes.
+  static CASID create(const CASContext *Context, StringRef Hash) {
+    return CASID(Context, Hash);
+  }
+
+private:
+  CASID(const CASContext *Context, StringRef Hash)
+      : Context(Context), Hash(Hash) {}
+
+  const CASContext *Context;
+  SmallString<32> Hash;
+};
+
+} // namespace cas
+
+template <> struct DenseMapInfo<cas::CASID> {
+  static cas::CASID getEmptyKey() { return cas::CASID::getDenseMapEmptyKey(); }
+
+  static cas::CASID getTombstoneKey() {
+    return cas::CASID::getDenseMapTombstoneKey();
+  }
+
+  static unsigned getHashValue(cas::CASID ID) {
+    return (unsigned)hash_value(ID);
+  }
+
+  static bool isEqual(cas::CASID LHS, cas::CASID RHS) { return LHS == RHS; }
+};
+
+} // namespace llvm
+
+#endif // LLVM_CAS_CASID_H
diff --git a/llvm/include/llvm/CAS/CASReference.h b/llvm/include/llvm/CAS/CASReference.h
new file mode 100644
index 0000000000000..ccbfb25079e8d
--- /dev/null
+++ b/llvm/include/llvm/CAS/CASReference.h
@@ -0,0 +1,195 @@
+//===- llvm/CAS/CASReference.h ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_CASREFERENCE_H
+#define LLVM_CAS_CASREFERENCE_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+
+class raw_ostream;
+
+namespace cas {
+
+class ObjectStore;
+class ObjectHandle;
+class ObjectRef;
+
+/// Base class for references to things in \a ObjectStore.
+class ReferenceBase {
+protected:
+  struct DenseMapEmptyTag {};
+  struct DenseMapTombstoneTag {};
+  static constexpr uint64_t getDenseMapEmptyRef() { return -1ULL; }
+  static constexpr uint64_t getDenseMapTombstoneRef() { return -2ULL; }
+
+public:
+  /// Get an internal reference.
+  uint64_t getInternalRef(const ObjectStore &ExpectedCAS) const {
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+    assert(CAS == &ExpectedCAS && "Extracting reference for the wrong CAS");
+#endif
+    return InternalRef;
+  }
+
+  /// Helper functions for DenseMapInfo.
+  unsigned getDenseMapHash() const {
+    return static_cast<unsigned>(llvm::hash_value(InternalRef));
+  }
+  bool isDenseMapEmpty() const { return InternalRef == getDenseMapEmptyRef(); }
+  bool isDenseMapTombstone() const {
+    return InternalRef == getDenseMapTombstoneRef();
+  }
+  bool isDenseMapSentinel() const {
+    return isDenseMapEmpty() || isDenseMapTombstone();
+  }
+
+protected:
+  void print(raw_ostream &OS, const ObjectHandle &This) const;
+  void print(raw_ostream &OS, const ObjectRef &This) const;
+
+  bool hasSameInternalRef(const ReferenceBase &RHS) const {
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+    assert(
+        (isDenseMapSentinel() || RHS.isDenseMapSentinel() || CAS == RHS.CAS) &&
+        "Cannot compare across CAS instances");
+#endif
+    return InternalRef == RHS.InternalRef;
+  }
+
+protected:
+  friend class ObjectStore;
+  ReferenceBase(const ObjectStore *CAS, uint64_t InternalRef, bool IsHandle)
+      : InternalRef(InternalRef) {
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+    this->CAS = CAS;
+#endif
+    assert(InternalRef != getDenseMapEmptyRef() && "Reserved for DenseMapInfo");
+    assert(InternalRef != getDenseMapTombstoneRef() &&
+           "Reserved for DenseMapInfo");
+  }
+  explicit ReferenceBase(DenseMapEmptyTag)
+      : InternalRef(getDenseMapEmptyRef()) {}
+  explicit ReferenceBase(DenseMapTombstoneTag)
+      : InternalRef(getDenseMapTombstoneRef()) {}
+
+private:
+  uint64_t InternalRef;
+
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+  const ObjectStore *CAS = nullptr;
+#endif
+};
+
+/// Reference to an object in an \a ObjectStore instance.
+///
+/// If you have an ObjectRef, you know the object exists, and you can point at
+/// it from new nodes with \a ObjectStore::store(), but you don't know anything
+/// about it. "Loading" the object is a separate step that may not have
+/// happened yet, and which can fail (due to filesystem corruption) or
+/// introduce latency (if downloading from a remote store).
+///
+/// \a ObjectStore::store() takes a list of these, and these are returned by \a
+/// ObjectStore::forEachRef() and \a ObjectStore::readRef(), which are accessors
+/// for nodes, and \a ObjectStore::getReference().
+///
+/// \a ObjectStore::load() will load the referenced object, and returns \a
+/// ObjectHandle, a variant that knows what kind of entity it is. \a
+/// ObjectStore::getReferenceKind() can expect the type of reference without
+/// asking for unloaded objects to be loaded.
+class ObjectRef : public ReferenceBase {
+  struct DenseMapTag {};
+
+public:
+  friend bool operator==(const ObjectRef &LHS, const ObjectRef &RHS) {
+    return LHS.hasSameInternalRef(RHS);
+  }
+  friend bool operator!=(const ObjectRef &LHS, const ObjectRef &RHS) {
+    return !(LHS == RHS);
+  }
+
+  static ObjectRef getDenseMapEmptyKey() {
+    return ObjectRef(DenseMapEmptyTag{});
+  }
+  static ObjectRef getDenseMapTombstoneKey() {
+    return ObjectRef(DenseMapTombstoneTag{});
+  }
+
+  /// Print internal ref and/or CASID. Only suitable for debugging.
+  void print(raw_ostream &OS) const { return ReferenceBase::print(OS, *this); }
+
+  LLVM_DUMP_METHOD void dump() const;
+
+private:
+  friend class ObjectStore;
+  friend class ReferenceBase;
+  using ReferenceBase::ReferenceBase;
+  ObjectRef(const ObjectStore &CAS, uint64_t InternalRef)
+      : ReferenceBase(&CAS, InternalRef, /*IsHandle=*/false) {
+    assert(InternalRef != -1ULL && "Reserved for DenseMapInfo");
+    assert(InternalRef != -2ULL && "Reserved for DenseMapInfo");
+  }
+  explicit ObjectRef(DenseMapEmptyTag T) : ReferenceBase(T) {}
+  explicit ObjectRef(DenseMapTombstoneTag T) : ReferenceBase(T) {}
+  explicit ObjectRef(ReferenceBase) = delete;
+};
+
+/// Handle to a loaded object in a \a ObjectStore instance.
+///
+/// ObjectHandle encapulates a *loaded* object in the CAS. You need one
+/// of these to inspect the content of an object: to look at its stored
+/// data and references.
+class ObjectHandle : public ReferenceBase {
+public:
+  friend bool operator==(const ObjectHandle &LHS, const ObjectHandle &RHS) {
+    return LHS.hasSameInternalRef(RHS);
+  }
+  friend bool operator!=(const ObjectHandle &LHS, const ObjectHandle &RHS) {
+    return !(LHS == RHS);
+  }
+
+  /// Print internal ref and/or CASID. Only suitable for debugging.
+  void print(raw_ostream &OS) const { return ReferenceBase::print(OS, *this); }
+
+  LLVM_DUMP_METHOD void dump() const;
+
+private:
+  friend class ObjectStore;
+  friend class ReferenceBase;
+  using ReferenceBase::ReferenceBase;
+  explicit ObjectHandle(ReferenceBase) = delete;
+  ObjectHandle(const ObjectStore &CAS, uint64_t InternalRef)
+      : ReferenceBase(&CAS, InternalRef, /*IsHandle=*/true) {}
+};
+
+} // namespace cas
+
+template <> struct DenseMapInfo<cas::ObjectRef> {
+  static cas::ObjectRef getEmptyKey() {
+    return cas::ObjectRef::getDenseMapEmptyKey();
+  }
+
+  static cas::ObjectRef getTombstoneKey() {
+    return cas::ObjectRef::getDenseMapTombstoneKey();
+  }
+
+  static unsigned getHashValue(cas::ObjectRef Ref) {
+    return Ref.getDenseMapHash();
+  }
+
+  static bool isEqual(cas::ObjectRef LHS, cas::ObjectRef RHS) {
+    return LHS == RHS;
+  }
+};
+
+} // namespace llvm
+
+#endif // LLVM_CAS_CASREFERENCE_H
diff --git a/llvm/include/llvm/CAS/ObjectStore.h b/llvm/include/llvm/CAS/ObjectStore.h
new file mode 100644
index 0000000000000..6db5dd3904095
--- /dev/null
+++ b/llvm/include/llvm/CAS/ObjectStore.h
@@ -0,0 +1,298 @@
+//===- llvm/CAS/ObjectStore.h -----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_OBJECTSTORE_H
+#define LLVM_CAS_OBJECTSTORE_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CAS/CASID.h"
+#include "llvm/CAS/CASReference.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include <cstddef>
+
+namespace llvm {
+
+class MemoryBuffer;
+template <typename T> class unique_function;
+
+namespace cas {
+
+class ObjectStore;
+class ObjectProxy;
+
+/// Content-addressable storage for objects.
+///
+/// Conceptually, objects are stored in a "unique set".
+///
+/// - Objects are immutable ("value objects") that are defined by their
+///   content. They are implicitly deduplicated by content.
+/// - Each object has a unique identifier (UID) that's derived from its content,
+///   called a \a CASID.
+///     - This UID is a fixed-size (strong) hash of the transitive content of a
+///       CAS object.
+///     - It's comparable between any two CAS instances that have the same \a
+///       CASIDContext::getHashSchemaIdentifier().
+///     - The UID can be printed (e.g., \a CASID::toString()) and it can parsed
+///       by the same or a different CAS instance with \a
+///       ObjectStore::parseID().
+/// - An object can be looked up by content or by UID.
+///     - \a store() is "get-or-create"  methods, writing an object if it
+///       doesn't exist yet, and return a ref to it in any case.
+///     - \a loadObject(const CASID&) looks up an object by its UID.
+/// - Objects can reference other objects, forming an arbitrary DAG.
+///
+/// The \a ObjectStore interface has a few ways of referencing objects:
+///
+/// - \a ObjectRef encapsulates a reference to something in the CAS. It is an
+///   opaque type that references an object inside a specific CAS. It is
+///   implementation defined if the underlying object exists or not for an
+///   ObjectRef, and it can used to speed up CAS lookup as an implementation
+///   detail. However, you don't know anything about the underlying objects.
+///   "Loading" the object is a separate step that may not have happened
+///   yet, and which can fail (e.g. due to filesystem corruption) or introduce
+///   latency (if downloading from a remote store).
+/// - \a ObjectHandle encapulates a *loaded* object in the CAS. You need one of
+///   these to inspect the content of an object: to look at its stored
+///   data and references. This is internal to CAS implementation and not
+///   availble from CAS public APIs.
+/// - \a CASID: the UID for an object in the CAS, obtained through \a
+///   ObjectStore::getID() or \a ObjectStore::parseID(). This is a valid CAS
+///   identifier, but may reference an object that is unknown to this CAS
+///   instance.
+/// - \a ObjectProxy pairs an ObjectHandle (subclass) with a ObjectStore, and
+///   wraps access APIs to avoid having to pass extra parameters. It is the
+///   object used for accessing underlying data and refs by CAS users.
+///
+/// Both ObjectRef and ObjectHandle are lightweight, wrapping a `uint64_t` and
+/// are only valid with the associated ObjectStore instance.
+///
+/// There are a few options for accessing content of objects, with different
+/// lifetime tradeoffs:
+///
+/// - \a getData() accesses data without exposing lifetime at all.
+/// - \a getMemoryBuffer() returns a \a MemoryBuffer whose lifetime
+///   is independent of the CAS (it can live longer).
+/// - \a getDataString() return StringRef with lifetime is guaranteed to last as
+///   long as \a ObjectStore.
+/// - \a readRef() and \a forEachRef() iterate through the references in an
+///   object. There is no lifetime assumption.
+class ObjectStore {
+  friend class ObjectProxy;
+  void anchor();
+
+public:
+  /// Get a \p CASID from a \p ID, which should have been generated by \a
+  /// CASID::print(). This succeeds as long as \a validateID() would pass. The
+  /// object may be unknown to this CAS instance.
+  ///
+  /// TODO: Remove, and update callers to use \a validateID() or \a
+  /// extractHashFromID().
+  virtual Expected<CASID> parseID(StringRef ID) = 0;
+
+  /// Store object into ObjectStore.
+  virtual Expected<ObjectRef> store(ArrayRef<ObjectRef> Refs,
+                                    ArrayRef<char> Data) = 0;
+  /// Get an ID for \p Ref.
+  virtual CASID getID(ObjectRef Ref) const = 0;
+
+  /// Get an existing reference to the object called \p ID.
+  ///
+  /// Returns \c None if the object is not stored in this CAS.
+  virtual std::optional<ObjectRef> getReference(const CASID &ID) const = 0;
+
+  /// \returns true if the object is directly available from the local CAS, for
+  /// implementations that have this kind of distinction.
+  virtual Expected<bool> isMaterialized(ObjectRef Ref) const = 0;
+
+  /// Validate the underlying object referred by CASID.
+  virtual Error validate(const CASID &ID) = 0;
+
+protected:
+  /// Load the object referenced by \p Ref.
+  ///
+  /// Errors if the object cannot be loaded.
+  /// \returns \c std::nullopt if the object is missing from the CAS.
+  virtual Expected<std::optional<ObjectHandle>> loadIfExists(ObjectRef Ref) = 0;
+
+  /// Like \c loadIfExists but returns an error if the object is missing.
+  Expected<ObjectHandle> load(ObjectRef Ref);
+
+  /// Get the size of some data.
+  virtual uint64_t getDataSize(ObjectHandle Node) const = 0;
+
+  /// Methods for handling objects. CAS implementations need to override to
+  /// provide functions to access stored CAS objects and references.
+  virtual Error forEachRef(ObjectHandle Node,
+                           function_ref<Error(ObjectRef)> Callback) const = 0;
+  virtual ObjectRef readRef(ObjectHandle Node, size_t I) const = 0;
+  virtual size_t getNumRefs(ObjectHandle Node) const = 0;
+  virtual ArrayRef<char> getData(ObjectHandle Node,
+                                 bool RequiresNullTerminator = false) const = 0;
+
+  /// Get ObjectRef from open file.
+  virtual Expected<ObjectRef>
+  storeFromOpenFileImpl(sys::fs::file_t FD,
+                        std::optional<sys::fs::file_status> Status);
+
+  /// Get a lifetime-extended StringRef pointing at \p Data.
+  ///
+  /// Depending on the CAS implementation, this may involve in-memory storage
+  /// overhead.
+  StringRef getDataString(ObjectHandle Node) {
+    return toStringRef(getData(Node));
+  }
+
+  /// Get a lifetime-extended MemoryBuffer pointing at \p Data.
+  ///
+  /// Depending on the CAS implementation, this may involve in-memory storage
+  /// overhead.
+  std::unique_ptr<MemoryBuffer>
+  getMemoryBuffer(ObjectHandle Node, StringRef Name = "",
+                  bool RequiresNullTerminator = true);
+
+  /// Read all the refs from object in a SmallVector.
+  virtual void readRefs(ObjectHandle Node,
+                        SmallVectorImpl<ObjectRef> &Refs) const;
+
+  /// Allow ObjectStore implementations to create internal handles.
+#define MAKE_CAS_HANDLE_CONSTRUCTOR(HandleKind)                                \
+  HandleKind make##HandleKind(uint64_t InternalRef) const {                    \
+    return HandleKind(*this, InternalRef);                                     \
+  }
+  MAKE_CAS_HANDLE_CONSTRUCTOR(ObjectHandle)
+  MAKE_CAS_HANDLE_CONSTRUCTOR(ObjectRef)
+#undef MAKE_CAS_HANDLE_CONSTRUCTOR
+
+public:
+  /// Helper functions to store object and returns a ObjectProxy.
+  Expected<ObjectProxy> createProxy(ArrayRef<ObjectRef> Refs, StringRef Data);
+
+  /// Store object from StringRef.
+  Expected<ObjectRef> storeFromString(ArrayRef<ObjectRef> Refs,
+                                      StringRef String) {
+    return store(Refs, arrayRefFromStringRef<char>(String));
+  }
+
+  /// Default implementation reads \p FD and calls \a storeNode(). Does not
+  /// take ownership of \p FD; the caller is responsible for closing it.
+  ///
+  /// If \p Status is sent in it is to be treated as a hint. Implementations
+  /// must protect against the file size potentially growing after the status
+  /// was taken (i.e., they cannot assume that an mmap will be null-terminated
+  /// where \p Status implies).
+  ///
+  /// Returns the \a CASID and the size of the file.
+  Expected<ObjectRef>
+  storeFromOpenFile(sys::fs::file_t FD,
+                    std::optional<sys::fs::file_status> Status = std::nullopt) {
+    return storeFromOpenFileImpl(FD, Status);
+  }
+
+  static Error createUnknownObjectError(const CASID &ID);
+
+  /// Create ObjectProxy from CASID. If the object doesn't exist, get an error.
+  Expected<ObjectProxy> getProxy(const CASID &ID);
+  /// Create ObjectProxy from ObjectRef. If the object can't be loaded, get an
+  /// error.
+  Expected<ObjectProxy> getProxy(ObjectRef Ref);
+
+  /// \returns \c std::nullopt if the object is missing from the CAS.
+  Expected<std::optional<ObjectProxy>> getProxyIfExists(ObjectRef Ref);
+
+  /// Read the data from \p Data into \p OS.
+  uint64_t readData(ObjectHandle Node, raw_ostream &OS, uint64_t Offset = 0,
+                    uint64_t MaxBytes = -1ULL) const {
+    ArrayRef<char> Data = getData(Node);
+    assert(Offset < Data.size() && "Expected valid offset");
+    Data = Data.drop_front(Offset).take_front(MaxBytes);
+    OS << toStringRef(Data);
+    return Data.size();
+  }
+
+  /// Validate the whole node tree.
+  Error validateTree(ObjectRef Ref);
+
+  /// Get CASContext
+  const CASContext &getContext() const { return Context; }
+
+  virtual ~ObjectStore() = default;
+
+protected:
+  ObjectStore(const CASContext &Context) : Context(Context) {}
+
+private:
+  const CASContext &Context;
+};
+
+/// Reference to an abstract hierarchical node, with data and references.
+/// Reference is passed by value and is expected to be valid as long as the \a
+/// ObjectStore is.
+class ObjectProxy {
+public:
+  ObjectStore &getCAS() const { return *CAS; }
+  CASID getID() const { return CAS->getID(Ref); }
+  ObjectRef getRef() const { return Ref; }
+  size_t getNumReferences() const { return CAS->getNumRefs(H); }
+  ObjectRef getReference(size_t I) const { return CAS->readRef(H, I); }
+
+  operator CASID() const { return getID(); }
+  CASID getReferenceID(size_t I) const {
+    std::optional<CASID> ID = getCAS().getID(getReference(I));
+    assert(ID && "Expected reference to be first-class object");
+    return *ID;
+  }
+
+  /// Visit each reference in order, returning an error from \p Callback to
+  /// stop early.
+  Error forEachReference(function_ref<Error(ObjectRef)> Callback) const {
+    return CAS->forEachRef(H, Callback);
+  }
+
+  std::unique_ptr<MemoryBuffer>
+  getMemoryBuffer(StringRef Name = "",
+                  bool RequiresNullTerminator = true) const;
+
+  /// Get the content of the node. Valid as long as the CAS is valid.
+  StringRef getData() const { return CAS->getDataString(H); }
+
+  friend bool operator==(const ObjectProxy &Proxy, ObjectRef Ref) {
+    return Proxy.getRef() == Ref;
+  }
+  friend bool operator==(ObjectRef Ref, const ObjectProxy &Proxy) {
+    return Proxy.getRef() == Ref;
+  }
+  friend bool operator!=(const ObjectProxy &Proxy, ObjectRef Ref) {
+    return !(Proxy.getRef() == Ref);
+  }
+  friend bool operator!=(ObjectRef Ref, const ObjectProxy &Proxy) {
+    return !(Proxy.getRef() == Ref);
+  }
+
+public:
+  ObjectProxy() = delete;
+
+  static ObjectProxy load(ObjectStore &CAS, ObjectRef Ref, ObjectHandle Node) {
+    return ObjectProxy(CAS, Ref, Node);
+  }
+
+private:
+  ObjectProxy(ObjectStore &CAS, ObjectRef Ref, ObjectHandle H)
+      : CAS(&CAS), Ref(Ref), H(H) {}
+
+  ObjectStore *CAS;
+  ObjectRef Ref;
+  ObjectHandle H;
+};
+
+std::unique_ptr<ObjectStore> createInMemoryCAS();
+
+} // namespace cas
+} // namespace llvm
+
+#endif // LLVM_CAS_OBJECTSTORE_H
diff --git a/llvm/include/llvm/CGData/CodeGenData.h b/llvm/include/llvm/CGData/CodeGenData.h
index 38b96b72ccac6..e44497a408245 100644
--- a/llvm/include/llvm/CGData/CodeGenData.h
+++ b/llvm/include/llvm/CGData/CodeGenData.h
@@ -285,6 +285,9 @@ enum CGDataVersion {
   // Version 3 adds the total size of the Names in the stable function map so
   // we can skip reading them into the memory for non-assertion builds.
   Version3 = 3,
+  // Version 4 adjusts the structure of stable function merging map for
+  // efficient lazy loading support.
+  Version4 = 4,
   CurrentVersion = CG_DATA_INDEX_VERSION
 };
 const uint64_t Version = CGDataVersion::CurrentVersion;
diff --git a/llvm/include/llvm/CGData/CodeGenData.inc b/llvm/include/llvm/CGData/CodeGenData.inc
index 94de4c0b017a2..d5fbe2fb97718 100644
--- a/llvm/include/llvm/CGData/CodeGenData.inc
+++ b/llvm/include/llvm/CGData/CodeGenData.inc
@@ -49,4 +49,4 @@ CG_DATA_SECT_ENTRY(CG_merge, CG_DATA_QUOTE(CG_DATA_MERGE_COMMON),
 #endif
 
 /* Indexed codegen data format version (start from 1). */
-#define CG_DATA_INDEX_VERSION 3
+#define CG_DATA_INDEX_VERSION 4
diff --git a/llvm/include/llvm/CGData/StableFunctionMap.h b/llvm/include/llvm/CGData/StableFunctionMap.h
index bcb72e8216973..8c7e83f4da1b1 100644
--- a/llvm/include/llvm/CGData/StableFunctionMap.h
+++ b/llvm/include/llvm/CGData/StableFunctionMap.h
@@ -20,6 +20,9 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/IR/StructuralHash.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <mutex>
+#include <unordered_map>
 
 namespace llvm {
 
@@ -72,11 +75,37 @@ struct StableFunctionMap {
           IndexOperandHashMap(std::move(IndexOperandHashMap)) {}
   };
 
-  using HashFuncsMapType =
-      DenseMap<stable_hash, SmallVector<std::unique_ptr<StableFunctionEntry>>>;
+  using StableFunctionEntries =
+      SmallVector<std::unique_ptr<StableFunctionEntry>>;
+
+  /// In addition to the deserialized StableFunctionEntry, the struct stores
+  /// the offsets of corresponding serialized stable function entries, and a
+  /// once flag for safe lazy loading in a multithreaded environment.
+  struct EntryStorage {
+    /// The actual storage of deserialized stable function entries. If the map
+    /// is lazily loaded, this will be empty until the first access by the
+    /// corresponding function hash.
+    StableFunctionEntries Entries;
+
+  private:
+    /// This is used to deserialize the entry lazily. Each element is the
+    /// corresponding serialized stable function entry's offset in the memory
+    /// buffer (StableFunctionMap::Buffer).
+    /// The offsets are only populated when loading the map lazily, otherwise
+    /// it is empty.
+    SmallVector<uint64_t> Offsets;
+    std::once_flag LazyLoadFlag;
+    friend struct StableFunctionMap;
+    friend struct StableFunctionMapRecord;
+  };
+
+  // Note: DenseMap requires value type to be copyable even if only using
+  // in-place insertion. Use STL instead. This also affects the
+  // deletion-while-iteration in finalize().
+  using HashFuncsMapType = std::unordered_map<stable_hash, EntryStorage>;
 
   /// Get the HashToFuncs map for serialization.
-  const HashFuncsMapType &getFunctionMap() const { return HashToFuncs; }
+  const HashFuncsMapType &getFunctionMap() const;
 
   /// Get the NameToId vector for serialization.
   ArrayRef<std::string> getNames() const { return IdToName; }
@@ -99,6 +128,19 @@ struct StableFunctionMap {
   /// \returns true if there is no stable function entry.
   bool empty() const { return size() == 0; }
 
+  /// \returns true if there is an entry for the given function hash.
+  /// This does not trigger lazy loading.
+  bool contains(HashFuncsMapType::key_type FunctionHash) const {
+    return HashToFuncs.count(FunctionHash) > 0;
+  }
+
+  /// \returns the stable function entries for the given function hash. If the
+  /// map is lazily loaded, it will deserialize the entries if it is not already
+  /// done, other requests to the same hash at the same time will be blocked
+  /// until the entries are deserialized.
+  const StableFunctionEntries &
+  at(HashFuncsMapType::key_type FunctionHash) const;
+
   enum SizeType {
     UniqueHashCount,        // The number of unique hashes in HashToFuncs.
     TotalFunctionCount,     // The number of total functions in HashToFuncs.
@@ -119,17 +161,31 @@ struct StableFunctionMap {
   /// `StableFunctionEntry` is ready for insertion.
   void insert(std::unique_ptr<StableFunctionEntry> FuncEntry) {
     assert(!Finalized && "Cannot insert after finalization");
-    HashToFuncs[FuncEntry->Hash].emplace_back(std::move(FuncEntry));
+    HashToFuncs[FuncEntry->Hash].Entries.emplace_back(std::move(FuncEntry));
   }
 
+  void deserializeLazyLoadingEntry(HashFuncsMapType::iterator It) const;
+
+  /// Eagerly deserialize all the unloaded entries in the lazy loading map.
+  void deserializeLazyLoadingEntries() const;
+
+  bool isLazilyLoaded() const { return (bool)Buffer; }
+
   /// A map from a stable_hash to a vector of functions with that hash.
-  HashFuncsMapType HashToFuncs;
+  mutable HashFuncsMapType HashToFuncs;
   /// A vector of strings to hold names.
   SmallVector<std::string> IdToName;
   /// A map from StringRef (name) to an ID.
   StringMap<unsigned> NameToId;
   /// True if the function map is finalized with minimal content.
   bool Finalized = false;
+  /// The memory buffer that contains the serialized stable function map for
+  /// lazy loading.
+  /// Non-empty only if this StableFunctionMap is created from a MemoryBuffer
+  /// (i.e. by IndexedCodeGenDataReader::read()) and lazily deserialized.
+  std::shared_ptr<MemoryBuffer> Buffer;
+  /// Whether to read stable function names from the buffer.
+  bool ReadStableFunctionMapNames = true;
 
   friend struct StableFunctionMapRecord;
 };
diff --git a/llvm/include/llvm/CGData/StableFunctionMapRecord.h b/llvm/include/llvm/CGData/StableFunctionMapRecord.h
index a75cb12a70ba6..2d8b573a3cb46 100644
--- a/llvm/include/llvm/CGData/StableFunctionMapRecord.h
+++ b/llvm/include/llvm/CGData/StableFunctionMapRecord.h
@@ -24,6 +24,26 @@
 
 namespace llvm {
 
+/// The structure of the serialized stable function map is as follows:
+/// - Number of unique function/module names
+/// - Total size of unique function/module names for opt-in skipping
+/// - Unique function/module names
+/// - Padding to align to 4 bytes
+/// - Number of StableFunctionEntries
+/// - Hashes of each StableFunctionEntry
+/// - Fixed-size fields for each StableFunctionEntry (the order is consistent
+///   with the hashes above):
+///   - FunctionNameId
+///   - ModuleNameId
+///   - InstCount
+///   - Relative offset to the beginning of IndexOperandHashes for this entry
+/// - Total size of variable-sized IndexOperandHashes for lazy-loading support
+/// - Variable-sized IndexOperandHashes for each StableFunctionEntry:
+///   - Number of IndexOperandHashes
+///   - Contents of each IndexOperandHashes
+///     - InstIndex
+///     - OpndIndex
+///     - OpndHash
 struct StableFunctionMapRecord {
   std::unique_ptr<StableFunctionMap> FunctionMap;
 
@@ -40,13 +60,25 @@ struct StableFunctionMapRecord {
                                  const StableFunctionMap *FunctionMap,
                                  std::vector<CGDataPatchItem> &PatchItems);
 
+  /// A static helper function to deserialize the stable function map entry.
+  /// Ptr should be pointing to the start of the fixed-sized fields of the
+  /// entry when passed in.
+  LLVM_ABI static void deserializeEntry(const unsigned char *Ptr,
+                                        stable_hash Hash,
+                                        StableFunctionMap *FunctionMap);
+
   /// Serialize the stable function map to a raw_ostream.
   LLVM_ABI void serialize(raw_ostream &OS,
                           std::vector<CGDataPatchItem> &PatchItems) const;
 
   /// Deserialize the stable function map from a raw_ostream.
-  LLVM_ABI void deserialize(const unsigned char *&Ptr,
-                            bool ReadStableFunctionMapNames = true);
+  LLVM_ABI void deserialize(const unsigned char *&Ptr);
+
+  /// Lazily deserialize the stable function map from `Buffer` starting at
+  /// `Offset`. The individual stable function entry would be read lazily from
+  /// `Buffer` when the function map is accessed.
+  LLVM_ABI void lazyDeserialize(std::shared_ptr<MemoryBuffer> Buffer,
+                                uint64_t Offset);
 
   /// Serialize the stable function map to a YAML stream.
   LLVM_ABI void serializeYAML(yaml::Output &YOS) const;
@@ -70,6 +102,18 @@ struct StableFunctionMapRecord {
     yaml::Output YOS(OS);
     serializeYAML(YOS);
   }
+
+  /// Set whether to read stable function names from the buffer.
+  /// Has no effect if the function map is read from a YAML stream.
+  void setReadStableFunctionMapNames(bool Read) {
+    assert(
+        FunctionMap->empty() &&
+        "Cannot change ReadStableFunctionMapNames after the map is populated");
+    FunctionMap->ReadStableFunctionMapNames = Read;
+  }
+
+private:
+  void deserialize(const unsigned char *&Ptr, bool Lazy);
 };
 
 } // namespace llvm
diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h
index 362cc30bbd06a..98b52579d03b7 100644
--- a/llvm/include/llvm/CodeGen/Analysis.h
+++ b/llvm/include/llvm/CodeGen/Analysis.h
@@ -55,6 +55,13 @@ inline unsigned ComputeLinearIndex(Type *Ty,
   return ComputeLinearIndex(Ty, Indices.begin(), Indices.end(), CurIndex);
 }
 
+/// Given an LLVM IR type, compute non-aggregate subtypes. Optionally also
+/// compute their offsets.
+void ComputeValueTypes(const DataLayout &DL, Type *Ty,
+                       SmallVectorImpl<Type *> &Types,
+                       SmallVectorImpl<TypeSize> *Offsets = nullptr,
+                       TypeSize StartingOffset = TypeSize::getZero());
+
 /// ComputeValueVTs - Given an LLVM IR type, compute a sequence of
 /// EVTs that represent all the individual underlying
 /// non-aggregate types that comprise it.
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 1fcedcd35ee8b..0a10b51f97c63 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -18,6 +18,7 @@
 
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -347,6 +348,21 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return Cost;
   }
 
+  /// Filter out constant and duplicated entries in \p Ops and return a vector
+  /// containing the types from \p Tys corresponding to the remaining operands.
+  static SmallVector<Type *, 4>
+  filterConstantAndDuplicatedOperands(ArrayRef<const Value *> Ops,
+                                      ArrayRef<Type *> Tys) {
+    SmallPtrSet<const Value *, 4> UniqueOperands;
+    SmallVector<Type *, 4> FilteredTys;
+    for (const auto &[Op, Ty] : zip_equal(Ops, Tys)) {
+      if (isa<Constant>(Op) || !UniqueOperands.insert(Op).second)
+        continue;
+      FilteredTys.push_back(Ty);
+    }
+    return FilteredTys;
+  }
+
 protected:
   explicit BasicTTIImplBase(const TargetMachine *TM, const DataLayout &DL)
       : BaseT(DL) {}
@@ -935,29 +951,21 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                              CostKind);
   }
 
-  /// Estimate the overhead of scalarizing an instructions unique
-  /// non-constant operands. The (potentially vector) types to use for each of
+  /// Estimate the overhead of scalarizing an instruction's
+  /// operands. The (potentially vector) types to use for each of
   /// argument are passes via Tys.
   InstructionCost getOperandsScalarizationOverhead(
-      ArrayRef<const Value *> Args, ArrayRef<Type *> Tys,
-      TTI::TargetCostKind CostKind) const override {
-    assert(Args.size() == Tys.size() && "Expected matching Args and Tys");
-
+      ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const override {
     InstructionCost Cost = 0;
-    SmallPtrSet<const Value*, 4> UniqueOperands;
-    for (int I = 0, E = Args.size(); I != E; I++) {
+    for (Type *Ty : Tys) {
       // Disregard things like metadata arguments.
-      const Value *A = Args[I];
-      Type *Ty = Tys[I];
       if (!Ty->isIntOrIntVectorTy() && !Ty->isFPOrFPVectorTy() &&
           !Ty->isPtrOrPtrVectorTy())
         continue;
 
-      if (!isa<Constant>(A) && UniqueOperands.insert(A).second) {
-        if (auto *VecTy = dyn_cast<VectorType>(Ty))
-          Cost += getScalarizationOverhead(VecTy, /*Insert*/ false,
-                                           /*Extract*/ true, CostKind);
-      }
+      if (auto *VecTy = dyn_cast<VectorType>(Ty))
+        Cost += getScalarizationOverhead(VecTy, /*Insert*/ false,
+                                         /*Extract*/ true, CostKind);
     }
 
     return Cost;
@@ -974,7 +982,8 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     InstructionCost Cost = getScalarizationOverhead(
         RetTy, /*Insert*/ true, /*Extract*/ false, CostKind);
     if (!Args.empty())
-      Cost += getOperandsScalarizationOverhead(Args, Tys, CostKind);
+      Cost += getOperandsScalarizationOverhead(
+          filterConstantAndDuplicatedOperands(Args, Tys), CostKind);
     else
       // When no information on arguments is provided, we add the cost
       // associated with one argument as a heuristic.
@@ -1444,6 +1453,20 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
                                        Op1);
   }
 
+  InstructionCost
+  getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
+                                   TTI::TargetCostKind CostKind,
+                                   unsigned Index) const override {
+    unsigned NewIndex = -1;
+    if (auto *FVTy = dyn_cast<FixedVectorType>(Val)) {
+      assert(Index < FVTy->getNumElements() &&
+             "Unexpected index from end of vector");
+      NewIndex = FVTy->getNumElements() - 1 - Index;
+    }
+    return thisT()->getVectorInstrCost(Opcode, Val, CostKind, NewIndex, nullptr,
+                                       nullptr);
+  }
+
   InstructionCost
   getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF,
                             const APInt &DemandedDstElts,
@@ -2105,6 +2128,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     }
     case Intrinsic::get_active_lane_mask:
     case Intrinsic::experimental_vector_match:
+    case Intrinsic::experimental_vector_histogram_add:
+    case Intrinsic::experimental_vector_histogram_uadd_sat:
+    case Intrinsic::experimental_vector_histogram_umax:
+    case Intrinsic::experimental_vector_histogram_umin:
       return thisT()->getTypeBasedIntrinsicInstrCost(ICA, CostKind);
     case Intrinsic::modf:
     case Intrinsic::sincos:
@@ -2152,8 +2179,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
               /*Insert=*/true, /*Extract=*/false, CostKind);
         }
       }
-      ScalarizationCost +=
-          getOperandsScalarizationOverhead(Args, ICA.getArgTypes(), CostKind);
+      ScalarizationCost += getOperandsScalarizationOverhead(
+          filterConstantAndDuplicatedOperands(Args, ICA.getArgTypes()),
+          CostKind);
     }
 
     IntrinsicCostAttributes Attrs(IID, RetTy, ICA.getArgTypes(), FMF, I,
@@ -2457,6 +2485,51 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       return thisT()->getShuffleCost(TTI::SK_Reverse, cast<VectorType>(RetTy),
                                      cast<VectorType>(ICA.getArgTypes()[0]), {},
                                      CostKind, 0, cast<VectorType>(RetTy));
+    case Intrinsic::experimental_vector_histogram_add:
+    case Intrinsic::experimental_vector_histogram_uadd_sat:
+    case Intrinsic::experimental_vector_histogram_umax:
+    case Intrinsic::experimental_vector_histogram_umin: {
+      FixedVectorType *PtrsTy = dyn_cast<FixedVectorType>(ICA.getArgTypes()[0]);
+      Type *EltTy = ICA.getArgTypes()[1];
+
+      // Targets with scalable vectors must handle this on their own.
+      if (!PtrsTy)
+        return InstructionCost::getInvalid();
+
+      Align Alignment = thisT()->DL.getABITypeAlign(EltTy);
+      InstructionCost Cost = 0;
+      Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, PtrsTy,
+                                          CostKind, 1, nullptr, nullptr);
+      Cost += thisT()->getMemoryOpCost(Instruction::Load, EltTy, Alignment, 0,
+                                       CostKind);
+      switch (IID) {
+      default:
+        llvm_unreachable("Unhandled histogram update operation.");
+      case Intrinsic::experimental_vector_histogram_add:
+        Cost +=
+            thisT()->getArithmeticInstrCost(Instruction::Add, EltTy, CostKind);
+        break;
+      case Intrinsic::experimental_vector_histogram_uadd_sat: {
+        IntrinsicCostAttributes UAddSat(Intrinsic::uadd_sat, EltTy, {EltTy});
+        Cost += thisT()->getIntrinsicInstrCost(UAddSat, CostKind);
+        break;
+      }
+      case Intrinsic::experimental_vector_histogram_umax: {
+        IntrinsicCostAttributes UMax(Intrinsic::umax, EltTy, {EltTy});
+        Cost += thisT()->getIntrinsicInstrCost(UMax, CostKind);
+        break;
+      }
+      case Intrinsic::experimental_vector_histogram_umin: {
+        IntrinsicCostAttributes UMin(Intrinsic::umin, EltTy, {EltTy});
+        Cost += thisT()->getIntrinsicInstrCost(UMin, CostKind);
+        break;
+      }
+      }
+      Cost += thisT()->getMemoryOpCost(Instruction::Store, EltTy, Alignment, 0,
+                                       CostKind);
+      Cost *= PtrsTy->getNumElements();
+      return Cost;
+    }
     case Intrinsic::get_active_lane_mask: {
       Type *ArgTy = ICA.getArgTypes()[0];
       EVT ResVT = getTLI()->getValueType(DL, RetTy, true);
@@ -2977,8 +3050,9 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
     return LT.first.getValue();
   }
 
-  InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *,
-                                            const SCEV *) const override {
+  InstructionCost
+  getAddressComputationCost(Type *PtrTy, ScalarEvolution *, const SCEV *,
+                            TTI::TargetCostKind) const override {
     return 0;
   }
 
diff --git a/llvm/include/llvm/CodeGen/CallingConvLower.h b/llvm/include/llvm/CodeGen/CallingConvLower.h
index a114a0c40a083..c7281ff1a99e6 100644
--- a/llvm/include/llvm/CodeGen/CallingConvLower.h
+++ b/llvm/include/llvm/CodeGen/CallingConvLower.h
@@ -154,9 +154,9 @@ struct ForwardedRegister {
 
 /// CCAssignFn - This function assigns a location for Val, updating State to
 /// reflect the change.  It returns 'true' if it failed to handle Val.
-typedef bool CCAssignFn(unsigned ValNo, MVT ValVT,
-                        MVT LocVT, CCValAssign::LocInfo LocInfo,
-                        ISD::ArgFlagsTy ArgFlags, CCState &State);
+typedef bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT,
+                        CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                        Type *OrigTy, CCState &State);
 
 /// CCCustomFn - This function assigns a location for Val, possibly updating
 /// all args to reflect changes and indicates if it handled it. It must set
@@ -290,6 +290,7 @@ class CCState {
   /// and argument flags.
   LLVM_ABI void AnalyzeCallOperands(SmallVectorImpl<MVT> &ArgVTs,
                                     SmallVectorImpl<ISD::ArgFlagsTy> &Flags,
+                                    SmallVectorImpl<Type *> &OrigTys,
                                     CCAssignFn Fn);
 
   /// The function will invoke AnalyzeCallOperands.
@@ -310,7 +311,7 @@ class CCState {
 
   /// AnalyzeCallResult - Same as above except it's specialized for calls which
   /// produce a single value.
-  LLVM_ABI void AnalyzeCallResult(MVT VT, CCAssignFn Fn);
+  LLVM_ABI void AnalyzeCallResult(MVT VT, Type *OrigTy, CCAssignFn Fn);
 
   /// getFirstUnallocated - Return the index of the first unallocated register
   /// in the set, or Regs.size() if they are all allocated.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 75c051712ae43..a8bde824527a5 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -50,14 +50,12 @@ class LLVM_ABI CallLowering {
   struct BaseArgInfo {
     Type *Ty;
     SmallVector<ISD::ArgFlagsTy, 4> Flags;
-    bool IsFixed;
 
     BaseArgInfo(Type *Ty,
-                ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(),
-                bool IsFixed = true)
-        : Ty(Ty), Flags(Flags), IsFixed(IsFixed) {}
+                ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>())
+        : Ty(Ty), Flags(Flags) {}
 
-    BaseArgInfo() : Ty(nullptr), IsFixed(false) {}
+    BaseArgInfo() : Ty(nullptr) {}
   };
 
   struct ArgInfo : public BaseArgInfo {
@@ -81,8 +79,8 @@ class LLVM_ABI CallLowering {
 
     ArgInfo(ArrayRef<Register> Regs, Type *Ty, unsigned OrigIndex,
             ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(),
-            bool IsFixed = true, const Value *OrigValue = nullptr)
-        : BaseArgInfo(Ty, Flags, IsFixed), Regs(Regs), OrigValue(OrigValue),
+            const Value *OrigValue = nullptr)
+        : BaseArgInfo(Ty, Flags), Regs(Regs), OrigValue(OrigValue),
           OrigArgIndex(OrigIndex) {
       if (!Regs.empty() && Flags.empty())
         this->Flags.push_back(ISD::ArgFlagsTy());
@@ -93,9 +91,8 @@ class LLVM_ABI CallLowering {
     }
 
     ArgInfo(ArrayRef<Register> Regs, const Value &OrigValue, unsigned OrigIndex,
-            ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>(),
-            bool IsFixed = true)
-      : ArgInfo(Regs, OrigValue.getType(), OrigIndex, Flags, IsFixed, &OrigValue) {}
+            ArrayRef<ISD::ArgFlagsTy> Flags = ArrayRef<ISD::ArgFlagsTy>())
+        : ArgInfo(Regs, OrigValue.getType(), OrigIndex, Flags, &OrigValue) {}
 
     ArgInfo() = default;
   };
@@ -201,7 +198,7 @@ class LLVM_ABI CallLowering {
                            CCValAssign::LocInfo LocInfo, const ArgInfo &Info,
                            ISD::ArgFlagsTy Flags, CCState &State) {
       if (getAssignFn(State.isVarArg())(ValNo, ValVT, LocVT, LocInfo, Flags,
-                                        State))
+                                        Info.Ty, State))
         return true;
       StackSize = State.getStackSize();
       return false;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index da829046cc421..6dba689e8af71 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -66,6 +66,7 @@ struct PtrAddChain {
   int64_t Imm;
   Register Base;
   const RegisterBank *Bank;
+  unsigned Flags;
 };
 
 struct RegisterImmPair {
@@ -726,6 +727,23 @@ class CombinerHelper {
   bool matchUMulHToLShr(MachineInstr &MI) const;
   void applyUMulHToLShr(MachineInstr &MI) const;
 
+  // Combine trunc(smin(smax(x, C1), C2)) -> truncssat_s(x)
+  // or      trunc(smax(smin(x, C2), C1)) -> truncssat_s(x).
+  bool matchTruncSSatS(MachineInstr &MI, Register &MatchInfo) const;
+  void applyTruncSSatS(MachineInstr &MI, Register &MatchInfo) const;
+
+  // Combine trunc(smin(smax(x, 0), C)) -> truncssat_u(x)
+  // or      trunc(smax(smin(x, C), 0)) -> truncssat_u(x)
+  // or      trunc(umin(smax(x, 0), C)) -> truncssat_u(x)
+  bool matchTruncSSatU(MachineInstr &MI, Register &MatchInfo) const;
+  void applyTruncSSatU(MachineInstr &MI, Register &MatchInfo) const;
+
+  // Combine trunc(umin(x, C)) -> truncusat_u(x).
+  bool matchTruncUSatU(MachineInstr &MI, MachineInstr &MinMI) const;
+
+  // Combine truncusat_u(fptoui(x)) -> fptoui_sat(x)
+  bool matchTruncUSatUToFPTOUISat(MachineInstr &MI, MachineInstr &SrcMI) const;
+
   /// Try to transform \p MI by using all of the above
   /// combine functions. Returns true if changed.
   bool tryCombine(MachineInstr &MI) const;
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
index 4292c0b31c750..5faf57fd06228 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h
@@ -874,6 +874,9 @@ class GCastOp : public GenericMachineInstr {
     case TargetOpcode::G_SEXT:
     case TargetOpcode::G_SITOFP:
     case TargetOpcode::G_TRUNC:
+    case TargetOpcode::G_TRUNC_SSAT_S:
+    case TargetOpcode::G_TRUNC_SSAT_U:
+    case TargetOpcode::G_TRUNC_USAT_U:
     case TargetOpcode::G_UITOFP:
     case TargetOpcode::G_ZEXT:
     case TargetOpcode::G_ANYEXT:
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index 6fd05c8fddd5f..3d7ccd55ee042 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -486,6 +486,10 @@ class IRTranslator : public MachineFunctionPass {
   bool translatePtrToInt(const User &U, MachineIRBuilder &MIRBuilder) {
     return translateCast(TargetOpcode::G_PTRTOINT, U, MIRBuilder);
   }
+  bool translatePtrToAddr(const User &U, MachineIRBuilder &MIRBuilder) {
+    // FIXME: this is not correct for pointers with addr width != pointer width
+    return translatePtrToInt(U, MIRBuilder);
+  }
   bool translateTrunc(const User &U, MachineIRBuilder &MIRBuilder) {
     return translateCast(TargetOpcode::G_TRUNC, U, MIRBuilder);
   }
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h b/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h
index cee779a5fd5d1..4b7506e013762 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LoadStoreOpt.h
@@ -162,7 +162,7 @@ class LLVM_ABI LoadStoreOpt : public MachineFunctionPass {
   DenseMap<unsigned, BitVector> LegalStoreSizes;
   bool IsPreLegalizer = false;
   /// Contains instructions to be erased at the end of a block scan.
-  SmallSet<MachineInstr *, 16> InstsToErase;
+  SmallPtrSet<MachineInstr *, 16> InstsToErase;
 
 public:
   LoadStoreOpt();
diff --git a/llvm/include/llvm/CodeGen/LivePhysRegs.h b/llvm/include/llvm/CodeGen/LivePhysRegs.h
index 2a719571fde2d..4af5b00014cb7 100644
--- a/llvm/include/llvm/CodeGen/LivePhysRegs.h
+++ b/llvm/include/llvm/CodeGen/LivePhysRegs.h
@@ -165,10 +165,17 @@ class LivePhysRegs {
   void dump() const;
 
 private:
+  /// Adds a register, taking the lane mask into consideration.
+  void addRegMaskPair(const MachineBasicBlock::RegisterMaskPair &Pair);
+
   /// Adds live-in registers from basic block \p MBB, taking associated
   /// lane masks into consideration.
   void addBlockLiveIns(const MachineBasicBlock &MBB);
 
+  /// Adds live-out registers from basic block \p MBB, taking associated
+  /// lane masks into consideration.
+  void addBlockLiveOuts(const MachineBasicBlock &MBB);
+
   /// Adds pristine registers. Pristine registers are callee saved registers
   /// that are unused in the function.
   void addPristines(const MachineFunction &MF);
diff --git a/llvm/include/llvm/CodeGen/LiveVariables.h b/llvm/include/llvm/CodeGen/LiveVariables.h
index 974bf9eaa0376..dbf736ad65a99 100644
--- a/llvm/include/llvm/CodeGen/LiveVariables.h
+++ b/llvm/include/llvm/CodeGen/LiveVariables.h
@@ -165,10 +165,8 @@ class LiveVariables {
   MachineInstr *FindLastRefOrPartRef(Register Reg);
 
   /// FindLastPartialDef - Return the last partial def of the specified
-  /// register. Also returns the sub-registers that're defined by the
-  /// instruction.
-  MachineInstr *FindLastPartialDef(Register Reg,
-                                   SmallSet<Register, 4> &PartDefRegs);
+  /// register.
+  MachineInstr *FindLastPartialDef(Register Reg);
 
   /// analyzePHINodes - Gather information about the PHI nodes in here. In
   /// particular, we want to map the variable information of a virtual
diff --git a/llvm/include/llvm/CodeGen/MIRYamlMapping.h b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
index 0f3f945905d3a..a91c26ee1122a 100644
--- a/llvm/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/llvm/include/llvm/CodeGen/MIRYamlMapping.h
@@ -634,6 +634,22 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::CalledGlobal)
 namespace llvm {
 namespace yaml {
 
+// Struct representing one save/restore point in the 'savePoint'/'restorePoint'
+// list
+struct SaveRestorePointEntry {
+  StringValue Point;
+
+  bool operator==(const SaveRestorePointEntry &Other) const {
+    return Point == Other.Point;
+  }
+};
+
+template <> struct MappingTraits<SaveRestorePointEntry> {
+  static void mapping(IO &YamlIO, SaveRestorePointEntry &Entry) {
+    YamlIO.mapRequired("point", Entry.Point);
+  }
+};
+
 template <> struct MappingTraits<MachineJumpTable> {
   static void mapping(IO &YamlIO, MachineJumpTable &JT) {
     YamlIO.mapRequired("kind", JT.Kind);
@@ -642,6 +658,14 @@ template <> struct MappingTraits<MachineJumpTable> {
   }
 };
 
+} // namespace yaml
+} // namespace llvm
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::SaveRestorePointEntry)
+
+namespace llvm {
+namespace yaml {
+
 /// Serializable representation of MachineFrameInfo.
 ///
 /// Doesn't serialize attributes like 'StackAlignment', 'IsStackRealignable' and
@@ -669,8 +693,8 @@ struct MachineFrameInfo {
   bool HasTailCall = false;
   bool IsCalleeSavedInfoValid = false;
   unsigned LocalFrameSize = 0;
-  StringValue SavePoint;
-  StringValue RestorePoint;
+  std::vector<SaveRestorePointEntry> SavePoints;
+  std::vector<SaveRestorePointEntry> RestorePoints;
 
   bool operator==(const MachineFrameInfo &Other) const {
     return IsFrameAddressTaken == Other.IsFrameAddressTaken &&
@@ -691,7 +715,8 @@ struct MachineFrameInfo {
            HasMustTailInVarArgFunc == Other.HasMustTailInVarArgFunc &&
            HasTailCall == Other.HasTailCall &&
            LocalFrameSize == Other.LocalFrameSize &&
-           SavePoint == Other.SavePoint && RestorePoint == Other.RestorePoint &&
+           SavePoints == Other.SavePoints &&
+           RestorePoints == Other.RestorePoints &&
            IsCalleeSavedInfoValid == Other.IsCalleeSavedInfoValid;
   }
 };
@@ -723,10 +748,8 @@ template <> struct MappingTraits<MachineFrameInfo> {
     YamlIO.mapOptional("isCalleeSavedInfoValid", MFI.IsCalleeSavedInfoValid,
                        false);
     YamlIO.mapOptional("localFrameSize", MFI.LocalFrameSize, (unsigned)0);
-    YamlIO.mapOptional("savePoint", MFI.SavePoint,
-                       StringValue()); // Don't print it out when it's empty.
-    YamlIO.mapOptional("restorePoint", MFI.RestorePoint,
-                       StringValue()); // Don't print it out when it's empty.
+    YamlIO.mapOptional("savePoint", MFI.SavePoints);
+    YamlIO.mapOptional("restorePoint", MFI.RestorePoints);
   }
 };
 
diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
index 9e3d9196cc184..94139b64a3e30 100644
--- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
@@ -553,8 +553,8 @@ class MachineBasicBlock
         LiveRegI = (*BlockI)->livein_begin();
         if (!advanceToValidPosition())
           return;
-        if (LiveRegI->PhysReg == ExceptionPointer ||
-            LiveRegI->PhysReg == ExceptionSelector)
+        if ((*BlockI)->isEHPad() && (LiveRegI->PhysReg == ExceptionPointer ||
+                                     LiveRegI->PhysReg == ExceptionSelector))
           ++(*this);
       }
     }
diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
index 403e5eda949f1..e666001035deb 100644
--- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
@@ -332,10 +332,10 @@ class MachineFrameInfo {
   /// stack objects like arguments so we can't treat them as immutable.
   bool HasTailCall = false;
 
-  /// Not null, if shrink-wrapping found a better place for the prologue.
-  MachineBasicBlock *Save = nullptr;
-  /// Not null, if shrink-wrapping found a better place for the epilogue.
-  MachineBasicBlock *Restore = nullptr;
+  /// Not empty, if shrink-wrapping found a better place for the prologue.
+  SmallVector<MachineBasicBlock *, 4> SavePoints;
+  /// Not empty, if shrink-wrapping found a better place for the epilogue.
+  SmallVector<MachineBasicBlock *, 4> RestorePoints;
 
   /// Size of the UnsafeStack Frame
   uint64_t UnsafeStackSize = 0;
@@ -825,10 +825,16 @@ class MachineFrameInfo {
 
   void setCalleeSavedInfoValid(bool v) { CSIValid = v; }
 
-  MachineBasicBlock *getSavePoint() const { return Save; }
-  void setSavePoint(MachineBasicBlock *NewSave) { Save = NewSave; }
-  MachineBasicBlock *getRestorePoint() const { return Restore; }
-  void setRestorePoint(MachineBasicBlock *NewRestore) { Restore = NewRestore; }
+  ArrayRef<MachineBasicBlock *> getSavePoints() const { return SavePoints; }
+  void setSavePoints(ArrayRef<MachineBasicBlock *> NewSavePoints) {
+    SavePoints.assign(NewSavePoints.begin(), NewSavePoints.end());
+  }
+  ArrayRef<MachineBasicBlock *> getRestorePoints() const {
+    return RestorePoints;
+  }
+  void setRestorePoints(ArrayRef<MachineBasicBlock *> NewRestorePoints) {
+    RestorePoints.assign(NewRestorePoints.begin(), NewRestorePoints.end());
+  }
 
   uint64_t getUnsafeStackSize() const { return UnsafeStackSize; }
   void setUnsafeStackSize(uint64_t Size) { UnsafeStackSize = Size; }
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index 69b7a3f570c89..ef783f276b7d4 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -286,7 +286,7 @@ struct LandingPadInfo {
 class LLVM_ABI MachineFunction {
   Function &F;
   const TargetMachine &Target;
-  const TargetSubtargetInfo *STI;
+  const TargetSubtargetInfo &STI;
   MCContext &Ctx;
 
   // RegInfo - Information about each register in use in the function.
@@ -759,13 +759,13 @@ class LLVM_ABI MachineFunction {
 
   /// getSubtarget - Return the subtarget for which this machine code is being
   /// compiled.
-  const TargetSubtargetInfo &getSubtarget() const { return *STI; }
+  const TargetSubtargetInfo &getSubtarget() const { return STI; }
 
   /// getSubtarget - This method returns a pointer to the specified type of
   /// TargetSubtargetInfo.  In debug builds, it verifies that the object being
   /// returned is of the correct type.
   template<typename STC> const STC &getSubtarget() const {
-    return *static_cast<const STC *>(STI);
+    return static_cast<const STC &>(STI);
   }
 
   /// getRegInfo - Return information about the registers currently in use.
diff --git a/llvm/include/llvm/CodeGen/MachinePipeliner.h b/llvm/include/llvm/CodeGen/MachinePipeliner.h
index e50443d25cc60..c90ff4f3daa47 100644
--- a/llvm/include/llvm/CodeGen/MachinePipeliner.h
+++ b/llvm/include/llvm/CodeGen/MachinePipeliner.h
@@ -830,7 +830,7 @@ class SMSchedule {
     return ScheduledInstrs[cycle];
   }
 
-  SmallSet<SUnit *, 8>
+  SmallPtrSet<SUnit *, 8>
   computeUnpipelineableNodes(SwingSchedulerDAG *SSD,
                              TargetInstrInfo::PipelinerLoopInfo *PLI);
 
diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index be90250b068f6..c468f2f676281 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -1098,9 +1098,9 @@ struct ConstantInt_match {
                                       BindVal ? *BindVal : Discard);
   }
 };
-/// Match any interger constants or splat of an integer constant.
+/// Match any integer constants or splat of an integer constant.
 inline ConstantInt_match m_ConstInt() { return ConstantInt_match(nullptr); }
-/// Match any interger constants or splat of an integer constant; return the
+/// Match any integer constants or splat of an integer constant; return the
 /// specific constant or constant splat value.
 inline ConstantInt_match m_ConstInt(APInt &V) { return ConstantInt_match(&V); }
 
diff --git a/llvm/include/llvm/CodeGen/ScheduleDAG.h b/llvm/include/llvm/CodeGen/ScheduleDAG.h
index 122b7be96b46a..aee1514581485 100644
--- a/llvm/include/llvm/CodeGen/ScheduleDAG.h
+++ b/llvm/include/llvm/CodeGen/ScheduleDAG.h
@@ -237,7 +237,7 @@ class TargetRegisterInfo;
   };
 
   /// Keep record of which SUnit are in the same cluster group.
-  typedef SmallSet<SUnit *, 8> ClusterInfo;
+  typedef SmallPtrSet<SUnit *, 8> ClusterInfo;
   constexpr unsigned InvalidClusterId = ~0u;
 
   /// Return whether the input cluster ID's are the same and valid.
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 6f2ad33094dce..dc00db9daa3b6 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -238,8 +238,6 @@ class SelectionDAG {
   LLVMContext *Context;
   CodeGenOptLevel OptLevel;
 
-  bool DivergentTarget = false;
-
   UniformityInfo *UA = nullptr;
   FunctionLoweringInfo * FLI = nullptr;
 
@@ -473,16 +471,14 @@ class SelectionDAG {
                      Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
                      UniformityInfo *UA, ProfileSummaryInfo *PSIin,
                      BlockFrequencyInfo *BFIin, MachineModuleInfo &MMI,
-                     FunctionVarLocs const *FnVarLocs, bool HasDivergency);
+                     FunctionVarLocs const *FnVarLocs);
 
   void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
             MachineFunctionAnalysisManager &AM,
             const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA,
             ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin,
-            MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs,
-            bool HasDivergency) {
-    init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, MMI, FnVarLocs,
-         HasDivergency);
+            MachineModuleInfo &MMI, FunctionVarLocs const *FnVarLocs) {
+    init(NewMF, NewORE, nullptr, LibraryInfo, UA, PSIin, BFIin, MMI, FnVarLocs);
     MFAM = &AM;
   }
 
@@ -1260,6 +1256,9 @@ class SelectionDAG {
   /// stack arguments from being clobbered.
   LLVM_ABI SDValue getStackArgumentTokenFactor(SDValue Chain);
 
+  std::pair<SDValue, SDValue> getMemcmp(SDValue Chain, const SDLoc &dl,
+                                        SDValue Dst, SDValue Src, SDValue Size,
+                                        const CallInst *CI);
   /* \p CI if not null is the memset call being lowered.
    * \p OverrideTailCall is an optional parameter that can be used to override
    * the tail call optimization decision. */
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
index a6a3928230c3d..5241a51dd8cd8 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h
@@ -57,7 +57,9 @@ class SelectionDAGISel {
   AssumptionCache *AC = nullptr;
   GCFunctionInfo *GFI = nullptr;
   SSPLayoutInfo *SP = nullptr;
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
   TargetTransformInfo *TTI = nullptr;
+#endif
   CodeGenOptLevel OptLevel;
   const TargetInstrInfo *TII;
   const TargetLowering *TLI;
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h
index 463f0ec350d9c..fd00f813bc9c3 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h
@@ -23,6 +23,7 @@
 
 namespace llvm {
 
+class CallInst;
 class SelectionDAG;
 
 //===----------------------------------------------------------------------===//
@@ -118,8 +119,7 @@ class SelectionDAGTargetInfo {
   virtual std::pair<SDValue, SDValue>
   EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
                           SDValue Op1, SDValue Op2, SDValue Op3,
-                          MachinePointerInfo Op1PtrInfo,
-                          MachinePointerInfo Op2PtrInfo) const {
+                          const CallInst *CI) const {
     return std::make_pair(SDValue(), SDValue());
   }
 
diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h
index a28c7a99fb3b5..f197c7f1645ec 100644
--- a/llvm/include/llvm/CodeGen/TargetCallingConv.h
+++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h
@@ -54,6 +54,8 @@ namespace ISD {
     unsigned IsInConsecutiveRegs : 1;
     unsigned IsCopyElisionCandidate : 1; ///< Argument copy elision candidate
     unsigned IsPointer : 1;
+    /// Whether this is part of a variable argument list (non-fixed).
+    unsigned IsVarArg : 1;
 
     unsigned ByValOrByRefSize = 0; ///< Byval or byref struct size
 
@@ -67,7 +69,7 @@ namespace ISD {
           IsSwiftError(0), IsCFGuardTarget(0), IsHva(0), IsHvaStart(0),
           IsSecArgPass(0), MemAlign(0), OrigAlign(0),
           IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0),
-          IsCopyElisionCandidate(0), IsPointer(0) {
+          IsCopyElisionCandidate(0), IsPointer(0), IsVarArg(0) {
       static_assert(sizeof(*this) == 4 * sizeof(unsigned), "flags are too big");
     }
 
@@ -145,6 +147,9 @@ namespace ISD {
     bool isPointer()  const { return IsPointer; }
     void setPointer() { IsPointer = 1; }
 
+    bool isVarArg() const { return IsVarArg; }
+    void setVarArg() { IsVarArg = 1; }
+
     Align getNonZeroMemAlign() const {
       return decodeMaybeAlign(MemAlign).valueOrOne();
     }
@@ -198,9 +203,16 @@ namespace ISD {
   ///
   struct InputArg {
     ArgFlagsTy Flags;
+    /// Legalized type of this argument part.
     MVT VT = MVT::Other;
+    /// Usually the non-legalized type of the argument, which is the EVT
+    /// corresponding to the OrigTy IR type. However, for post-legalization
+    /// libcalls, this will be a legalized type.
     EVT ArgVT;
-    bool Used = false;
+    /// Original IR type of the argument. For aggregates, this is the type of
+    /// an individual aggregate element, not the whole aggregate.
+    Type *OrigTy;
+    bool Used;
 
     /// Index original Function's argument.
     unsigned OrigArgIndex;
@@ -212,13 +224,10 @@ namespace ISD {
     /// registers, we got 4 InputArgs with PartOffsets 0, 4, 8 and 12.
     unsigned PartOffset;
 
-    InputArg() = default;
-    InputArg(ArgFlagsTy flags, EVT vt, EVT argvt, bool used,
-             unsigned origIdx, unsigned partOffs)
-      : Flags(flags), Used(used), OrigArgIndex(origIdx), PartOffset(partOffs) {
-      VT = vt.getSimpleVT();
-      ArgVT = argvt;
-    }
+    InputArg(ArgFlagsTy Flags, MVT VT, EVT ArgVT, Type *OrigTy, bool Used,
+             unsigned OrigArgIndex, unsigned PartOffset)
+        : Flags(Flags), VT(VT), ArgVT(ArgVT), OrigTy(OrigTy), Used(Used),
+          OrigArgIndex(OrigArgIndex), PartOffset(PartOffset) {}
 
     bool isOrigArg() const {
       return OrigArgIndex != NoArgIndex;
@@ -236,11 +245,14 @@ namespace ISD {
   ///
   struct OutputArg {
     ArgFlagsTy Flags;
+    // Legalized type of this argument part.
     MVT VT;
+    /// Non-legalized type of the argument. This is the EVT corresponding to
+    /// the OrigTy IR type.
     EVT ArgVT;
-
-    /// IsFixed - Is this a "fixed" value, ie not passed through a vararg "...".
-    bool IsFixed = false;
+    /// Original IR type of the argument. For aggregates, this is the type of
+    /// an individual aggregate element, not the whole aggregate.
+    Type *OrigTy;
 
     /// Index original Function's argument.
     unsigned OrigArgIndex;
@@ -250,14 +262,10 @@ namespace ISD {
     /// registers, we got 4 OutputArgs with PartOffsets 0, 4, 8 and 12.
     unsigned PartOffset;
 
-    OutputArg() = default;
-    OutputArg(ArgFlagsTy flags, MVT vt, EVT argvt, bool isfixed,
-              unsigned origIdx, unsigned partOffs)
-        : Flags(flags), IsFixed(isfixed), OrigArgIndex(origIdx),
-          PartOffset(partOffs) {
-      VT = vt;
-      ArgVT = argvt;
-    }
+    OutputArg(ArgFlagsTy Flags, MVT VT, EVT ArgVT, Type *OrigTy,
+              unsigned OrigArgIndex, unsigned PartOffset)
+        : Flags(Flags), VT(VT), ArgVT(ArgVT), OrigTy(OrigTy),
+          OrigArgIndex(OrigArgIndex), PartOffset(PartOffset) {}
   };
 
 } // end namespace ISD
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 01f8fb5ed061f..4480ced637456 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -299,9 +299,12 @@ class LLVM_ABI TargetLoweringBase {
 
   class ArgListEntry {
   public:
-    Value *Val = nullptr;
-    SDValue Node = SDValue();
-    Type *Ty = nullptr;
+    Value *Val;
+    SDValue Node;
+    /// Original unlegalized argument type.
+    Type *OrigTy;
+    /// Same as OrigTy, or partially legalized for soft float libcalls.
+    Type *Ty;
     bool IsSExt : 1;
     bool IsZExt : 1;
     bool IsNoExt : 1;
@@ -320,12 +323,17 @@ class LLVM_ABI TargetLoweringBase {
     MaybeAlign Alignment = std::nullopt;
     Type *IndirectType = nullptr;
 
-    ArgListEntry()
-        : IsSExt(false), IsZExt(false), IsNoExt(false), IsInReg(false),
-          IsSRet(false), IsNest(false), IsByVal(false), IsByRef(false),
-          IsInAlloca(false), IsPreallocated(false), IsReturned(false),
-          IsSwiftSelf(false), IsSwiftAsync(false), IsSwiftError(false),
-          IsCFGuardTarget(false) {}
+    ArgListEntry(Value *Val, SDValue Node, Type *Ty)
+        : Val(Val), Node(Node), OrigTy(Ty), Ty(Ty), IsSExt(false),
+          IsZExt(false), IsNoExt(false), IsInReg(false), IsSRet(false),
+          IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false),
+          IsPreallocated(false), IsReturned(false), IsSwiftSelf(false),
+          IsSwiftAsync(false), IsSwiftError(false), IsCFGuardTarget(false) {}
+
+    explicit ArgListEntry(Value *Val, SDValue Node = SDValue())
+        : ArgListEntry(Val, Node, Val->getType()) {}
+
+    ArgListEntry(SDValue Node, Type *Ty) : ArgListEntry(nullptr, Node, Ty) {}
 
     LLVM_ABI void setAttributes(const CallBase *Call, unsigned ArgIdx);
   };
@@ -547,10 +555,10 @@ class LLVM_ABI TargetLoweringBase {
   // intermediate results in f32 precision and range.
   virtual bool softPromoteHalfType() const { return false; }
 
-  // Return true if, for soft-promoted half, the half type should be passed
-  // passed to and returned from functions as f32. The default behavior is to
-  // pass as i16. If soft-promoted half is not used, this function is ignored
-  // and values are always passed and returned as f32.
+  // Return true if, for soft-promoted half, the half type should be passed to
+  // and returned from functions as f32. The default behavior is to pass as
+  // i16. If soft-promoted half is not used, this function is ignored and
+  // values are always passed and returned as f32.
   virtual bool useFPRegsForHalfType() const { return false; }
 
   // There are two general methods for expanding a BUILD_VECTOR node:
@@ -3202,10 +3210,12 @@ class LLVM_ABI TargetLoweringBase {
   /// \p Shuffles is the shufflevector list to DE-interleave the loaded vector.
   /// \p Indices is the corresponding indices for each shufflevector.
   /// \p Factor is the interleave factor.
+  /// \p GapMask is a mask with zeros for components / fields that may not be
+  /// accessed.
   virtual bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
                                     ArrayRef<ShuffleVectorInst *> Shuffles,
-                                    ArrayRef<unsigned> Indices,
-                                    unsigned Factor) const {
+                                    ArrayRef<unsigned> Indices, unsigned Factor,
+                                    const APInt &GapMask) const {
     return false;
   }
 
@@ -3550,15 +3560,25 @@ class LLVM_ABI TargetLoweringBase {
 
   /// Get the libcall routine name for the specified libcall.
   const char *getLibcallName(RTLIB::Libcall Call) const {
-    return Libcalls.getLibcallName(Call);
+    // FIXME: Return StringRef
+    return Libcalls.getLibcallName(Call).data();
   }
 
   /// Get the libcall routine name for the specified libcall implementation
-  const char *getLibcallImplName(RTLIB::LibcallImpl Call) const {
-    return Libcalls.getLibcallImplName(Call);
+  static StringRef getLibcallImplName(RTLIB::LibcallImpl Call) {
+    return RTLIB::RuntimeLibcallsInfo::getLibcallImplName(Call);
   }
 
-  const char *getMemcpyName() const { return Libcalls.getMemcpyName(); }
+  const char *getMemcpyName() const {
+    // FIXME: Return StringRef
+    return Libcalls.getMemcpyName().data();
+  }
+
+  /// Check if this is valid libcall for the current module, otherwise
+  /// RTLIB::Unsupported.
+  RTLIB::LibcallImpl getSupportedLibcallImpl(StringRef FuncName) const {
+    return Libcalls.getSupportedLibcallImpl(FuncName);
+  }
 
   /// Get the comparison predicate that's to be used to test the result of the
   /// comparison libcall against zero. This should only be used with
@@ -4666,6 +4686,9 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
   /// implementation.
   struct CallLoweringInfo {
     SDValue Chain;
+    /// Original unlegalized return type.
+    Type *OrigRetTy = nullptr;
+    /// Same as OrigRetTy, or partially legalized for soft float libcalls.
     Type *RetTy = nullptr;
     bool RetSExt           : 1;
     bool RetZExt           : 1;
@@ -4720,6 +4743,14 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
     // setCallee with target/module-specific attributes
     CallLoweringInfo &setLibCallee(CallingConv::ID CC, Type *ResultType,
                                    SDValue Target, ArgListTy &&ArgsList) {
+      return setLibCallee(CC, ResultType, ResultType, Target,
+                          std::move(ArgsList));
+    }
+
+    CallLoweringInfo &setLibCallee(CallingConv::ID CC, Type *ResultType,
+                                   Type *OrigResultType, SDValue Target,
+                                   ArgListTy &&ArgsList) {
+      OrigRetTy = OrigResultType;
       RetTy = ResultType;
       Callee = Target;
       CallConv = CC;
@@ -4734,7 +4765,7 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
     CallLoweringInfo &setCallee(CallingConv::ID CC, Type *ResultType,
                                 SDValue Target, ArgListTy &&ArgsList,
                                 AttributeSet ResultAttrs = {}) {
-      RetTy = ResultType;
+      RetTy = OrigRetTy = ResultType;
       IsInReg = ResultAttrs.hasAttribute(Attribute::InReg);
       RetSExt = ResultAttrs.hasAttribute(Attribute::SExt);
       RetZExt = ResultAttrs.hasAttribute(Attribute::ZExt);
@@ -4750,7 +4781,7 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
     CallLoweringInfo &setCallee(Type *ResultType, FunctionType *FTy,
                                 SDValue Target, ArgListTy &&ArgsList,
                                 const CallBase &Call) {
-      RetTy = ResultType;
+      RetTy = OrigRetTy = ResultType;
 
       IsInReg = Call.hasRetAttr(Attribute::InReg);
       DoesNotReturn =
@@ -4886,11 +4917,10 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
       return *this;
     }
 
-    MakeLibCallOptions &setTypeListBeforeSoften(ArrayRef<EVT> OpsVT, EVT RetVT,
-                                                bool Value = true) {
+    MakeLibCallOptions &setTypeListBeforeSoften(ArrayRef<EVT> OpsVT, EVT RetVT) {
       OpsVTBeforeSoften = OpsVT;
       RetVTBeforeSoften = RetVT;
-      IsSoften = Value;
+      IsSoften = true;
       return *this;
     }
 
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h
index 1847fa8323480..2cf4a8ec6a37f 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVBinaryReader.h
@@ -159,7 +159,8 @@ class LVBinaryReader : public LVReader {
   LVAddress WasmCodeSectionOffset = 0;
 
   // Loads all info for the architecture of the provided object file.
-  Error loadGenericTargetInfo(StringRef TheTriple, StringRef TheFeatures);
+  Error loadGenericTargetInfo(StringRef TheTriple, StringRef TheFeatures,
+                              StringRef TheCPU);
 
   virtual void mapRangeAddress(const object::ObjectFile &Obj) {}
   virtual void mapRangeAddress(const object::ObjectFile &Obj,
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
index 03165895b85d0..567a58e7729c2 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
@@ -153,12 +153,20 @@ class ThreadSafeModule {
 using GVPredicate = std::function<bool(const GlobalValue &)>;
 using GVModifier = std::function<void(GlobalValue &)>;
 
-/// Clones teh given module onto the given context.
+/// Clones the given module onto the given context.
 LLVM_ABI ThreadSafeModule
 cloneToContext(const ThreadSafeModule &TSMW, ThreadSafeContext TSCtx,
                GVPredicate ShouldCloneDef = GVPredicate(),
                GVModifier UpdateClonedDefSource = GVModifier());
 
+/// Clone the given module onto the given context.
+/// The caller is responsible for ensuring that the source module and its
+/// LLVMContext will not be concurrently accessed during the clone.
+LLVM_ABI ThreadSafeModule
+cloneExternalModuleToContext(const Module &M, ThreadSafeContext TSCtx,
+                             GVPredicate ShouldCloneDef = GVPredicate(),
+                             GVModifier UpdateClonedDefSource = GVModifier());
+
 /// Clones the given module on to a new context.
 LLVM_ABI ThreadSafeModule cloneToNewContext(
     const ThreadSafeModule &TSMW, GVPredicate ShouldCloneDef = GVPredicate(),
diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
index e44612af071bc..87777fddc9157 100644
--- a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
+++ b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignature.h
@@ -42,10 +42,9 @@ struct RootConstants {
   dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
 };
 
-enum class DescriptorType : uint8_t { SRV = 0, UAV, CBuffer };
 // Models RootDescriptor : CBV | SRV | UAV, by collecting like parameters
 struct RootDescriptor {
-  DescriptorType Type;
+  dxil::ResourceClass Type;
   Register Reg;
   uint32_t Space = 0;
   dxbc::ShaderVisibility Visibility = dxbc::ShaderVisibility::All;
@@ -60,13 +59,16 @@ struct RootDescriptor {
     assert(Version == llvm::dxbc::RootSignatureVersion::V1_1 &&
            "Specified an invalid root signature version");
     switch (Type) {
-    case DescriptorType::CBuffer:
-    case DescriptorType::SRV:
+    case dxil::ResourceClass::CBuffer:
+    case dxil::ResourceClass::SRV:
       Flags = dxbc::RootDescriptorFlags::DataStaticWhileSetAtExecute;
       break;
-    case DescriptorType::UAV:
+    case dxil::ResourceClass::UAV:
       Flags = dxbc::RootDescriptorFlags::DataVolatile;
       break;
+    case dxil::ResourceClass::Sampler:
+      llvm_unreachable(
+          "ResourceClass::Sampler is not valid for RootDescriptors");
     }
   }
 };
@@ -82,9 +84,8 @@ struct DescriptorTable {
 static const uint32_t NumDescriptorsUnbounded = 0xffffffff;
 static const uint32_t DescriptorTableOffsetAppend = 0xffffffff;
 // Models DTClause : CBV | SRV | UAV | Sampler, by collecting like parameters
-using ClauseType = llvm::dxil::ResourceClass;
 struct DescriptorTableClause {
-  ClauseType Type;
+  dxil::ResourceClass Type;
   Register Reg;
   uint32_t NumDescriptors = 1;
   uint32_t Space = 0;
@@ -94,7 +95,7 @@ struct DescriptorTableClause {
   void setDefaultFlags(dxbc::RootSignatureVersion Version) {
     if (Version == dxbc::RootSignatureVersion::V1_0) {
       Flags = dxbc::DescriptorRangeFlags::DescriptorsVolatile;
-      if (Type != ClauseType::Sampler)
+      if (Type != dxil::ResourceClass::Sampler)
         Flags |= dxbc::DescriptorRangeFlags::DataVolatile;
       return;
     }
@@ -102,14 +103,14 @@ struct DescriptorTableClause {
     assert(Version == dxbc::RootSignatureVersion::V1_1 &&
            "Specified an invalid root signature version");
     switch (Type) {
-    case ClauseType::CBuffer:
-    case ClauseType::SRV:
+    case dxil::ResourceClass::CBuffer:
+    case dxil::ResourceClass::SRV:
       Flags = dxbc::DescriptorRangeFlags::DataStaticWhileSetAtExecute;
       break;
-    case ClauseType::UAV:
+    case dxil::ResourceClass::UAV:
       Flags = dxbc::DescriptorRangeFlags::DataVolatile;
       break;
-    case ClauseType::Sampler:
+    case dxil::ResourceClass::Sampler:
       Flags = dxbc::DescriptorRangeFlags::None;
       break;
     }
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index ce1cedc188fbf..8ea50e7e8d416 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -242,7 +242,7 @@ ENUM(MotionExpectation, Present);
 // V5.2: [15.9.1] `task-dependence-type` modifier
 ENUM(DependenceType, Depobj, In, Inout, Inoutset, Mutexinoutset, Out, Sink,
      Source);
-ENUM(Prescriptiveness, Strict);
+ENUM(Prescriptiveness, Strict, Fallback);
 
 template <typename I, typename E> //
 struct LoopIterationT {
@@ -574,6 +574,15 @@ struct DynamicAllocatorsT {
   using EmptyTrait = std::true_type;
 };
 
+template <typename T, typename I, typename E> //
+struct DynGroupprivateT {
+  ENUM(AccessGroup, Cgroup);
+  using Prescriptiveness = type::Prescriptiveness;
+  using Size = E;
+  using TupleTrait = std::true_type;
+  std::tuple<OPT(AccessGroup), OPT(Prescriptiveness), Size> t;
+};
+
 // V5.2: [5.8.4] `enter` clause
 template <typename T, typename I, typename E> //
 struct EnterT {
@@ -1263,11 +1272,12 @@ template <typename T, typename I, typename E>
 using TupleClausesT =
     std::variant<AffinityT<T, I, E>, AlignedT<T, I, E>, AllocateT<T, I, E>,
                  DefaultmapT<T, I, E>, DeviceT<T, I, E>, DistScheduleT<T, I, E>,
-                 DoacrossT<T, I, E>, FromT<T, I, E>, GrainsizeT<T, I, E>,
-                 IfT<T, I, E>, InitT<T, I, E>, InReductionT<T, I, E>,
-                 LastprivateT<T, I, E>, LinearT<T, I, E>, MapT<T, I, E>,
-                 NumTasksT<T, I, E>, OrderT<T, I, E>, ReductionT<T, I, E>,
-                 ScheduleT<T, I, E>, TaskReductionT<T, I, E>, ToT<T, I, E>>;
+                 DoacrossT<T, I, E>, DynGroupprivateT<T, I, E>, FromT<T, I, E>,
+                 GrainsizeT<T, I, E>, IfT<T, I, E>, InitT<T, I, E>,
+                 InReductionT<T, I, E>, LastprivateT<T, I, E>, LinearT<T, I, E>,
+                 MapT<T, I, E>, NumTasksT<T, I, E>, OrderT<T, I, E>,
+                 ReductionT<T, I, E>, ScheduleT<T, I, E>,
+                 TaskReductionT<T, I, E>, ToT<T, I, E>>;
 
 template <typename T, typename I, typename E>
 using UnionClausesT = std::variant<DependT<T, I, E>>;
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 79f25bb05f20e..1f1d7d06b4269 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -178,6 +178,9 @@ def OMPC_Doacross : Clause<[Spelling<"doacross">]> {
 def OMPC_DynamicAllocators : Clause<[Spelling<"dynamic_allocators">]> {
   let clangClass = "OMPDynamicAllocatorsClause";
 }
+def OMPC_DynGroupprivate : Clause<[Spelling<"dyn_groupprivate">]> {
+  let flangClass = "OmpDynGroupprivateClause";
+}
 def OMPC_Enter : Clause<[Spelling<"enter">]> {
   let flangClass = "OmpEnterClause";
 }
@@ -873,6 +876,14 @@ def OMP_For : Directive<[Spelling<"for">]> {
   let category = CA_Executable;
   let languages = [L_C];
 }
+def OMP_Groupprivate : Directive<[Spelling<"groupprivate">]> {
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_DeviceType>,
+  ];
+  let association = AS_None;
+  let category = CA_Declarative;
+  let languages = [L_C, L_Fortran];
+}
 def OMP_Interchange : Directive<[Spelling<"interchange">]> {
   let allowedOnceClauses = [
     VersionedClause<OMPC_Permutation>,
@@ -1092,6 +1103,7 @@ def OMP_Target : Directive<[Spelling<"target">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_InReduction, 50>,
@@ -1246,6 +1258,7 @@ def OMP_TaskYield : Directive<[Spelling<"taskyield">]> {
 def OMP_Teams : Directive<[Spelling<"teams">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_OMPX_Attribute>,
     VersionedClause<OMPC_Private>,
@@ -1309,6 +1322,17 @@ def OMP_EndWorkshare : Directive<[Spelling<"end workshare">]> {
   let category = OMP_Workshare.category;
   let languages = [L_Fortran];
 }
+def OMP_Workdistribute : Directive<[Spelling<"workdistribute">]> {
+  let association = AS_Block;
+  let category = CA_Executable;
+  let languages = [L_Fortran];
+}
+def OMP_EndWorkdistribute : Directive<[Spelling<"end workdistribute">]> {
+  let leafConstructs = OMP_Workdistribute.leafConstructs;
+  let association = OMP_Workdistribute.association;
+  let category = OMP_Workdistribute.category;
+  let languages = [L_Fortran];
+}
 
 //===----------------------------------------------------------------------===//
 // Definitions of OpenMP compound directives
@@ -1507,6 +1531,7 @@ def OMP_target_loop : Directive<[Spelling<"target loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_IsDevicePtr>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
@@ -1968,6 +1993,7 @@ def OMP_TargetParallel : Directive<[Spelling<"target parallel">]> {
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Default>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -1996,6 +2022,7 @@ def OMP_TargetParallelDo : Directive<[Spelling<"target parallel do">]> {
     VersionedClause<OMPC_Allocator>,
     VersionedClause<OMPC_Default>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2033,6 +2060,7 @@ def OMP_TargetParallelDoSimd
     VersionedClause<OMPC_DefaultMap>,
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2066,6 +2094,7 @@ def OMP_TargetParallelFor : Directive<[Spelling<"target parallel for">]> {
     VersionedClause<OMPC_DefaultMap>,
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2103,6 +2132,7 @@ def OMP_TargetParallelForSimd
     VersionedClause<OMPC_DefaultMap>,
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2138,6 +2168,7 @@ def OMP_target_parallel_loop : Directive<[Spelling<"target parallel loop">]> {
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2170,6 +2201,7 @@ def OMP_TargetSimd : Directive<[Spelling<"target simd">]> {
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2205,6 +2237,7 @@ def OMP_TargetTeams : Directive<[Spelling<"target teams">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2234,6 +2267,7 @@ def OMP_TargetTeamsDistribute
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2266,6 +2300,7 @@ def OMP_TargetTeamsDistributeParallelDo
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2303,6 +2338,7 @@ def OMP_TargetTeamsDistributeParallelDoSimd
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2347,6 +2383,7 @@ def OMP_TargetTeamsDistributeParallelFor
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Device>,
     VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2385,6 +2422,7 @@ def OMP_TargetTeamsDistributeParallelForSimd
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Device>,
     VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2422,6 +2460,7 @@ def OMP_TargetTeamsDistributeSimd
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2452,12 +2491,42 @@ def OMP_TargetTeamsDistributeSimd
   let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TargetTeamsWorkdistribute : Directive<[Spelling<"target teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_Depend>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_HasDeviceAddr, 51>,
+    VersionedClause<OMPC_If>,
+    VersionedClause<OMPC_IsDevicePtr>,
+    VersionedClause<OMPC_Map>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+    VersionedClause<OMPC_UsesAllocators, 50>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_DefaultMap>,
+    VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_NoWait>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_OMPX_DynCGroupMem>,
+    VersionedClause<OMPC_OMPX_Bare>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+  let languages = [L_Fortran];
+}
 def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_DefaultMap>,
     VersionedClause<OMPC_Depend>,
     VersionedClause<OMPC_Device>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_HasDeviceAddr, 51>,
     VersionedClause<OMPC_If>,
@@ -2522,6 +2591,7 @@ def OMP_TeamsDistribute : Directive<[Spelling<"teams distribute">]> {
     VersionedClause<OMPC_Collapse>,
     VersionedClause<OMPC_Default>,
     VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_LastPrivate>,
     VersionedClause<OMPC_NumTeams>,
@@ -2543,6 +2613,7 @@ def OMP_TeamsDistributeParallelDo
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
     VersionedClause<OMPC_Copyin>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_If>,
     VersionedClause<OMPC_LastPrivate>,
@@ -2571,6 +2642,7 @@ def OMP_TeamsDistributeParallelDoSimd
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_If>,
     VersionedClause<OMPC_LastPrivate>,
@@ -2606,6 +2678,7 @@ def OMP_TeamsDistributeParallelFor
     VersionedClause<OMPC_Copyin>,
     VersionedClause<OMPC_Default>,
     VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_If>,
     VersionedClause<OMPC_LastPrivate>,
@@ -2632,6 +2705,7 @@ def OMP_TeamsDistributeParallelForSimd
     VersionedClause<OMPC_Collapse>,
     VersionedClause<OMPC_Default>,
     VersionedClause<OMPC_DistSchedule>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_If>,
     VersionedClause<OMPC_LastPrivate>,
@@ -2659,6 +2733,7 @@ def OMP_TeamsDistributeSimd : Directive<[Spelling<"teams distribute simd">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Aligned>,
     VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_If, 50>,
     VersionedClause<OMPC_LastPrivate>,
@@ -2682,9 +2757,29 @@ def OMP_TeamsDistributeSimd : Directive<[Spelling<"teams distribute simd">]> {
   let leafConstructs = [OMP_Teams, OMP_Distribute, OMP_Simd];
   let category = CA_Executable;
 }
+def OMP_TeamsWorkdistribute : Directive<[Spelling<"teams workdistribute">]> {
+  let allowedClauses = [
+    VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_FirstPrivate>,
+    VersionedClause<OMPC_OMPX_Attribute>,
+    VersionedClause<OMPC_Private>,
+    VersionedClause<OMPC_Reduction>,
+    VersionedClause<OMPC_Shared>,
+  ];
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_Default>,
+    VersionedClause<OMPC_If, 52>,
+    VersionedClause<OMPC_NumTeams>,
+    VersionedClause<OMPC_ThreadLimit>,
+  ];
+  let leafConstructs = [OMP_Teams, OMP_Workdistribute];
+  let category = CA_Executable;
+  let languages = [L_Fortran];
+}
 def OMP_teams_loop : Directive<[Spelling<"teams loop">]> {
   let allowedClauses = [
     VersionedClause<OMPC_Allocate>,
+    VersionedClause<OMPC_DynGroupprivate, 61>,
     VersionedClause<OMPC_FirstPrivate>,
     VersionedClause<OMPC_LastPrivate>,
     VersionedClause<OMPC_OMPX_Attribute>,
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 206ad4a4ef85f..1050e3d8b08dd 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -31,6 +31,7 @@
 
 namespace llvm {
 class CanonicalLoopInfo;
+class ScanInfo;
 struct TargetRegionEntryInfo;
 class OffloadEntriesInfoManager;
 class OpenMPIRBuilder;
@@ -707,6 +708,9 @@ class OpenMPIRBuilder {
   LLVM_ABI InsertPointOrErrorTy createCancellationPoint(
       const LocationDescription &Loc, omp::Directive CanceledDirective);
 
+  /// Creates a ScanInfo object, allocates and returns the pointer.
+  LLVM_ABI Expected<ScanInfo *> scanInfoInitialize();
+
   /// Generator for '#omp parallel'
   ///
   /// \param Loc The insert and source location description.
@@ -750,6 +754,44 @@ class OpenMPIRBuilder {
                       LoopBodyGenCallbackTy BodyGenCB, Value *TripCount,
                       const Twine &Name = "loop");
 
+  /// Generator for the control flow structure of an OpenMP canonical loops if
+  /// the parent directive has an `inscan` modifier specified.
+  /// If the `inscan` modifier is specified, the region of the parent is
+  /// expected to have a `scan` directive. Based on the clauses in
+  /// scan directive, the body of the loop is split into two loops: Input loop
+  /// and Scan Loop. Input loop contains the code generated for input phase of
+  /// scan and Scan loop contains the code generated for scan phase of scan.
+  /// From the bodyGen callback of these loops, `createScan` would be called
+  /// when a scan directive is encountered from the loop body. `createScan`
+  /// based on whether 1. inclusive or exclusive scan is specified and, 2. input
+  /// loop or scan loop is generated, lowers the body of the for loop
+  /// accordingly.
+  ///
+  /// \param Loc       The insert and source location description.
+  /// \param BodyGenCB Callback that will generate the loop body code.
+  /// \param Start     Value of the loop counter for the first iterations.
+  /// \param Stop      Loop counter values past this will stop the loop.
+  /// \param Step      Loop counter increment after each iteration; negative
+  ///                  means counting down.
+  /// \param IsSigned  Whether Start, Stop and Step are signed integers.
+  /// \param InclusiveStop Whether \p Stop itself is a valid value for the loop
+  ///                      counter.
+  /// \param ComputeIP Insertion point for instructions computing the trip
+  ///                  count. Can be used to ensure the trip count is available
+  ///                  at the outermost loop of a loop nest. If not set,
+  ///                  defaults to the preheader of the generated loop.
+  /// \param Name      Base name used to derive BB and instruction names.
+  /// \param ScanRedInfo Pointer to the ScanInfo objected created using
+  ///                  `ScanInfoInitialize`.
+  ///
+  /// \returns A vector containing Loop Info of Input Loop and Scan Loop.
+  LLVM_ABI Expected<SmallVector<llvm::CanonicalLoopInfo *>>
+  createCanonicalScanLoops(const LocationDescription &Loc,
+                           LoopBodyGenCallbackTy BodyGenCB, Value *Start,
+                           Value *Stop, Value *Step, bool IsSigned,
+                           bool InclusiveStop, InsertPointTy ComputeIP,
+                           const Twine &Name, ScanInfo *ScanRedInfo);
+
   /// Calculate the trip count of a canonical loop.
   ///
   /// This allows specifying user-defined loop counter values using increment,
@@ -818,13 +860,17 @@ class OpenMPIRBuilder {
   ///                  at the outermost loop of a loop nest. If not set,
   ///                  defaults to the preheader of the generated loop.
   /// \param Name      Base name used to derive BB and instruction names.
+  /// \param InScan    Whether loop has a scan reduction specified.
+  /// \param ScanRedInfo  Pointer to the ScanInfo objected created using
+  ///                  `ScanInfoInitialize`.
   ///
   /// \returns An object representing the created control flow structure which
   ///          can be used for loop-associated directives.
   LLVM_ABI Expected<CanonicalLoopInfo *> createCanonicalLoop(
       const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
       Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
-      InsertPointTy ComputeIP = {}, const Twine &Name = "loop");
+      InsertPointTy ComputeIP = {}, const Twine &Name = "loop",
+      bool InScan = false, ScanInfo *ScanRedInfo = nullptr);
 
   /// Collapse a loop nest into a single loop.
   ///
@@ -1556,6 +1602,47 @@ class OpenMPIRBuilder {
       ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
       Function *ReduceFn, AttributeList FuncAttrs);
 
+  /// Helper function for CreateCanonicalScanLoops to create InputLoop
+  /// in the firstGen and Scan Loop in the SecondGen
+  /// \param InputLoopGen Callback for generating the loop for input phase
+  /// \param ScanLoopGen  Callback for generating the loop for scan phase
+  /// \param ScanRedInfo  Pointer to the ScanInfo objected created using
+  ///                     `ScanInfoInitialize`.
+  ///
+  /// \return error if any produced, else return success.
+  Error emitScanBasedDirectiveIR(
+      llvm::function_ref<Error()> InputLoopGen,
+      llvm::function_ref<Error(LocationDescription Loc)> ScanLoopGen,
+      ScanInfo *ScanRedInfo);
+
+  /// Creates the basic blocks required for scan reduction.
+  /// \param ScanRedInfo Pointer to the ScanInfo objected created using
+  ///                    `ScanInfoInitialize`.
+  void createScanBBs(ScanInfo *ScanRedInfo);
+
+  /// Dynamically allocates the buffer needed for scan reduction.
+  /// \param AllocaIP    The IP where possibly-shared pointer of buffer needs to
+  ///                    be declared.
+  /// \param ScanVars    Scan Variables.
+  /// \param ScanRedInfo Pointer to the ScanInfo objected created using
+  ///                    `ScanInfoInitialize`.
+  ///
+  /// \return error if any produced, else return success.
+  Error emitScanBasedDirectiveDeclsIR(InsertPointTy AllocaIP,
+                                      ArrayRef<llvm::Value *> ScanVars,
+                                      ArrayRef<llvm::Type *> ScanVarsType,
+                                      ScanInfo *ScanRedInfo);
+
+  /// Copies the result back to the reduction variable.
+  /// \param ReductionInfos Array type containing the ReductionOps.
+  /// \param ScanRedInfo    Pointer to the ScanInfo objected created using
+  ///                       `ScanInfoInitialize`.
+  ///
+  /// \return error if any produced, else return success.
+  Error emitScanBasedDirectiveFinalsIR(
+      ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+      ScanInfo *ScanInfo);
+
   /// This function emits a helper that gathers Reduce lists from the first
   /// lane of every active warp to lanes in the first warp.
   ///
@@ -2184,6 +2271,9 @@ class OpenMPIRBuilder {
   /// free'd.
   std::forward_list<CanonicalLoopInfo> LoopInfos;
 
+  /// Collection of owned ScanInfo objects that eventually need to be free'd.
+  std::forward_list<ScanInfo> ScanInfos;
+
   /// Add a new region that will be outlined later.
   void addOutlineInfo(OutlineInfo &&OI) { OutlineInfos.emplace_back(OI); }
 
@@ -2639,6 +2729,49 @@ class OpenMPIRBuilder {
                                              FinalizeCallbackTy FiniCB,
                                              Value *Filter);
 
+  /// This function performs the scan reduction of the values updated in
+  /// the input phase. The reduction logic needs to be emitted between input
+  /// and scan loop returned by `CreateCanonicalScanLoops`. The following
+  /// is the code that is generated, `buffer` and `span` are expected to be
+  /// populated before executing the generated code.
+  /// \code{c}
+  /// for (int k = 0; k != ceil(log2(span)); ++k) {
+  ///   i=pow(2,k)
+  ///   for (size cnt = last_iter; cnt >= i; --cnt)
+  ///     buffer[cnt] op= buffer[cnt-i];
+  /// }
+  /// \endcode
+  /// \param Loc The insert and source location description.
+  /// \param ReductionInfos Array type containing the ReductionOps.
+  /// \param ScanRedInfo    Pointer to the ScanInfo objected created using
+  ///                       `ScanInfoInitialize`.
+  ///
+  /// \returns The insertion position *after* the masked.
+  LLVM_ABI InsertPointOrErrorTy emitScanReduction(
+      const LocationDescription &Loc,
+      ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+      ScanInfo *ScanRedInfo);
+
+  /// This directive split and directs the control flow to input phase
+  ///  blocks or scan phase blocks based on 1. whether input loop or scan loop
+  ///  is executed, 2. whether exclusive or inclusive scan is used.
+  ///
+  /// \param Loc The insert and source location description.
+  /// \param AllocaIP    The IP where the temporary buffer for scan reduction
+  //                     needs to be allocated.
+  /// \param ScanVars    Scan Variables.
+  /// \param IsInclusive Whether it is an inclusive or exclusive scan.
+  /// \param ScanRedInfo Pointer to the ScanInfo objected created using
+  ///                    `ScanInfoInitialize`.
+  ///
+  /// \returns The insertion position *after* the scan.
+  LLVM_ABI InsertPointOrErrorTy createScan(const LocationDescription &Loc,
+                                           InsertPointTy AllocaIP,
+                                           ArrayRef<llvm::Value *> ScanVars,
+                                           ArrayRef<llvm::Type *> ScanVarsType,
+                                           bool IsInclusive,
+                                           ScanInfo *ScanRedInfo);
+
   /// Generator for '#omp critical'
   ///
   /// \param Loc The insert and source location description.
@@ -3286,7 +3419,8 @@ class OpenMPIRBuilder {
   emitAtomicUpdate(InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
                    AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
                    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX,
-                   bool IsXBinopExpr);
+                   bool IsXBinopExpr, bool IsIgnoreDenormalMode,
+                   bool IsFineGrainedMemory, bool IsRemoteMemory);
 
   /// Emit the binary op. described by \p RMWOp, using \p Src1 and \p Src2 .
   ///
@@ -3359,7 +3493,9 @@ class OpenMPIRBuilder {
   LLVM_ABI InsertPointOrErrorTy createAtomicUpdate(
       const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
       Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-      AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr);
+      AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
+      bool IsIgnoreDenormalMode = false, bool IsFineGrainedMemory = false,
+      bool IsRemoteMemory = false);
 
   /// Emit atomic update for constructs: --- Only Scalar data types
   /// V = X; X = X BinOp Expr ,
@@ -3394,7 +3530,9 @@ class OpenMPIRBuilder {
       const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
       AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
       AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
-      bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr);
+      bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
+      bool IsIgnoreDenormalMode = false, bool IsFineGrainedMemory = false,
+      bool IsRemoteMemory = false);
 
   /// Emit atomic compare for constructs: --- Only scalar data types
   /// cond-expr-stmt:
@@ -3774,6 +3912,93 @@ class CanonicalLoopInfo {
   LLVM_ABI void invalidate();
 };
 
+/// ScanInfo holds the information to assist in lowering of Scan reduction.
+/// Before lowering, the body of the for loop specifying scan reduction is
+/// expected to have the following structure
+///
+///     Loop Body Entry
+///        |
+///     Code before the scan directive
+///        |
+///     Scan Directive
+///        |
+///     Code after the scan directive
+///        |
+///     Loop Body Exit
+/// When `createCanonicalScanLoops` is executed, the bodyGen callback of it
+/// transforms the body to:
+///
+///     Loop Body Entry
+///        |
+///     OMPScanDispatch
+///
+///     OMPBeforeScanBlock
+///        |
+///     OMPScanLoopExit
+///        |
+///     Loop Body Exit
+///
+/// The insert point is updated to the first insert point of OMPBeforeScanBlock.
+/// It dominates the control flow of code generated until
+/// scan directive is encountered and OMPAfterScanBlock dominates the
+/// control flow of code generated after scan is encountered. The successor
+/// of OMPScanDispatch can be OMPBeforeScanBlock or OMPAfterScanBlock based
+/// on 1.whether it is in Input phase or Scan Phase , 2. whether it is an
+/// exclusive or inclusive scan. This jump is added when `createScan` is
+/// executed. If input loop is being generated, if it is inclusive scan,
+/// `OMPAfterScanBlock` succeeds `OMPScanDispatch` , if exclusive,
+/// `OMPBeforeScanBlock` succeeds `OMPDispatch` and vice versa for scan loop. At
+/// the end of the input loop, temporary buffer is populated and at the
+/// beginning of the scan loop, temporary buffer is read. After scan directive
+/// is encountered, insertion point is updated to `OMPAfterScanBlock` as it is
+/// expected to dominate the code after the scan directive. Both Before and
+/// After scan blocks are succeeded by `OMPScanLoopExit`.
+/// Temporary buffer allocations are done in `ScanLoopInit` block before the
+/// lowering of for-loop. The results are copied back to reduction variable in
+/// `ScanLoopFinish` block.
+class ScanInfo {
+public:
+  /// Dominates the body of the loop before scan directive
+  llvm::BasicBlock *OMPBeforeScanBlock = nullptr;
+
+  /// Dominates the body of the loop before scan directive
+  llvm::BasicBlock *OMPAfterScanBlock = nullptr;
+
+  /// Controls the flow to before or after scan blocks
+  llvm::BasicBlock *OMPScanDispatch = nullptr;
+
+  /// Exit block of loop body
+  llvm::BasicBlock *OMPScanLoopExit = nullptr;
+
+  /// Block before loop body where scan initializations are done
+  llvm::BasicBlock *OMPScanInit = nullptr;
+
+  /// Block after loop body where scan finalizations are done
+  llvm::BasicBlock *OMPScanFinish = nullptr;
+
+  /// If true, it indicates Input phase is lowered; else it indicates
+  /// ScanPhase is lowered
+  bool OMPFirstScanLoop = false;
+
+  /// Maps the private reduction variable to the pointer of the temporary
+  /// buffer
+  llvm::SmallDenseMap<llvm::Value *, llvm::Value *> *ScanBuffPtrs;
+
+  /// Keeps track of value of iteration variable for input/scan loop to be
+  /// used for Scan directive lowering
+  llvm::Value *IV;
+
+  /// Stores the span of canonical loop being lowered to be used for temporary
+  /// buffer allocation or Finalization.
+  llvm::Value *Span;
+
+  ScanInfo() {
+    ScanBuffPtrs = new llvm::SmallDenseMap<llvm::Value *, llvm::Value *>();
+  }
+
+  ~ScanInfo() { delete (ScanBuffPtrs); }
+};
+
 } // end namespace llvm
 
 #endif // LLVM_FRONTEND_OPENMP_OMPIRBUILDER_H
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index f974cfc78c8dd..5f0b6e9521395 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -470,18 +470,18 @@ __OMP_RTL(__kmpc_target_deinit, false, Void,)
 __OMP_RTL(__kmpc_kernel_prepare_parallel, false, Void, VoidPtr)
 __OMP_RTL(__kmpc_parallel_51, false, Void, IdentPtr, Int32, Int32, Int32, Int32,
           VoidPtr, VoidPtr, VoidPtrPtr, SizeTy)
-__OMP_RTL(__kmpc_for_static_loop_4, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int32)
-__OMP_RTL(__kmpc_for_static_loop_4u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int32)
-__OMP_RTL(__kmpc_for_static_loop_8, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int64)
-__OMP_RTL(__kmpc_for_static_loop_8u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int64)
-__OMP_RTL(__kmpc_distribute_static_loop_4, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32)
-__OMP_RTL(__kmpc_distribute_static_loop_4u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32)
-__OMP_RTL(__kmpc_distribute_static_loop_8, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64)
-__OMP_RTL(__kmpc_distribute_static_loop_8u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64)
-__OMP_RTL(__kmpc_distribute_for_static_loop_4, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int32, Int32)
-__OMP_RTL(__kmpc_distribute_for_static_loop_4u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int32, Int32)
-__OMP_RTL(__kmpc_distribute_for_static_loop_8, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int64, Int64)
-__OMP_RTL(__kmpc_distribute_for_static_loop_8u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int64, Int64)
+__OMP_RTL(__kmpc_for_static_loop_4, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int32, Int8)
+__OMP_RTL(__kmpc_for_static_loop_4u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int32, Int8)
+__OMP_RTL(__kmpc_for_static_loop_8, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int64, Int8)
+__OMP_RTL(__kmpc_for_static_loop_8u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int64, Int8)
+__OMP_RTL(__kmpc_distribute_static_loop_4, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int8)
+__OMP_RTL(__kmpc_distribute_static_loop_4u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int8)
+__OMP_RTL(__kmpc_distribute_static_loop_8, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int8)
+__OMP_RTL(__kmpc_distribute_static_loop_8u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int8)
+__OMP_RTL(__kmpc_distribute_for_static_loop_4, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int32, Int32, Int8)
+__OMP_RTL(__kmpc_distribute_for_static_loop_4u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int32, Int32, Int32, Int32, Int8)
+__OMP_RTL(__kmpc_distribute_for_static_loop_8, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int64, Int64, Int8)
+__OMP_RTL(__kmpc_distribute_for_static_loop_8u, false, Void, IdentPtr, VoidPtr, VoidPtr, Int64, Int64, Int64, Int64, Int8)
 __OMP_RTL(__kmpc_kernel_parallel, false, Int1, VoidPtrPtr)
 __OMP_RTL(__kmpc_kernel_end_parallel, false, Void, )
 __OMP_RTL(__kmpc_serialized_parallel, false, Void, IdentPtr, Int32)
@@ -674,22 +674,22 @@ __OMP_RTL_ATTRS(__kmpc_cancel_barrier, BarrierAttrs, SExt,
                 ParamAttrs(ReadOnlyPtrAttrs, SExt))
 __OMP_RTL_ATTRS(__kmpc_distribute_for_static_loop_4, AlwaysInlineAttrs, AttributeSet(),
                 ParamAttrs(ReadOnlyPtrAttrs, AttributeSet(), AttributeSet(),
-                           SExt, SExt, SExt, SExt))
+                           SExt, SExt, SExt, SExt, ZExt))
 __OMP_RTL_ATTRS(__kmpc_distribute_for_static_loop_4u, AlwaysInlineAttrs, AttributeSet(),
                 ParamAttrs(ReadOnlyPtrAttrs, AttributeSet(), AttributeSet(),
-                           ZExt, ZExt, ZExt, ZExt))
+                           ZExt, ZExt, ZExt, ZExt, ZExt))
 __OMP_RTL_ATTRS(__kmpc_distribute_static_loop_4, AlwaysInlineAttrs, AttributeSet(),
                 ParamAttrs(ReadOnlyPtrAttrs, AttributeSet(), AttributeSet(),
-                           SExt, SExt))
+                           SExt, SExt, ZExt))
 __OMP_RTL_ATTRS(__kmpc_distribute_static_loop_4u, AlwaysInlineAttrs, AttributeSet(),
                 ParamAttrs(ReadOnlyPtrAttrs, AttributeSet(), AttributeSet(),
-                           ZExt, ZExt))
+                           ZExt, ZExt, ZExt))
 __OMP_RTL_ATTRS(__kmpc_for_static_loop_4, AlwaysInlineAttrs, AttributeSet(),
                 ParamAttrs(ReadOnlyPtrAttrs, AttributeSet(), AttributeSet(),
-                           SExt, SExt, SExt))
+                           SExt, SExt, SExt, ZExt))
 __OMP_RTL_ATTRS(__kmpc_for_static_loop_4u, AlwaysInlineAttrs, AttributeSet(),
                 ParamAttrs(ReadOnlyPtrAttrs, AttributeSet(), AttributeSet(),
-                           ZExt, ZExt, ZExt))
+                           ZExt, ZExt, ZExt, ZExt))
 __OMP_RTL_ATTRS(__kmpc_error, AttributeSet(), AttributeSet(),
                 ParamAttrs(AttributeSet(), SExt))
 __OMP_RTL_ATTRS(__kmpc_flush, BarrierAttrs, AttributeSet(),
diff --git a/llvm/include/llvm/IR/BasicBlock.h b/llvm/include/llvm/IR/BasicBlock.h
index c24f01fe26cc8..533808e0666d5 100644
--- a/llvm/include/llvm/IR/BasicBlock.h
+++ b/llvm/include/llvm/IR/BasicBlock.h
@@ -68,7 +68,7 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
   // Allow Function to renumber blocks.
   friend class Function;
   /// Per-function unique number.
-  unsigned Number = -1u;
+  unsigned Number = ~0u;
 
   friend class BlockAddress;
   friend class SymbolTableListTraits<BasicBlock>;
diff --git a/llvm/include/llvm/IR/ConstantRange.h b/llvm/include/llvm/IR/ConstantRange.h
index 9a6a9db65688a..f4d4f1f555fa4 100644
--- a/llvm/include/llvm/IR/ConstantRange.h
+++ b/llvm/include/llvm/IR/ConstantRange.h
@@ -380,8 +380,10 @@ class [[nodiscard]] ConstantRange {
   /// Return a new range in the specified integer type, which must be
   /// strictly smaller than the current type.  The returned range will
   /// correspond to the possible range of values if the source range had been
-  /// truncated to the specified type.
-  LLVM_ABI ConstantRange truncate(uint32_t BitWidth) const;
+  /// truncated to the specified type with wrap type \p NoWrapKind.
+  /// Note that the result of trunc nuw is exact.
+  LLVM_ABI ConstantRange truncate(uint32_t BitWidth,
+                                  unsigned NoWrapKind = 0) const;
 
   /// Make this range have the bit width given by \p BitWidth. The
   /// value is zero extended, truncated, or left alone to make it that width.
diff --git a/llvm/include/llvm/IR/Constants.h b/llvm/include/llvm/IR/Constants.h
index 9c9fc8892bdbf..e06e6adbc3130 100644
--- a/llvm/include/llvm/IR/Constants.h
+++ b/llvm/include/llvm/IR/Constants.h
@@ -1158,6 +1158,8 @@ class ConstantExpr : public Constant {
   LLVM_ABI static Constant *getXor(Constant *C1, Constant *C2);
   LLVM_ABI static Constant *getTrunc(Constant *C, Type *Ty,
                                      bool OnlyIfReduced = false);
+  LLVM_ABI static Constant *getPtrToAddr(Constant *C, Type *Ty,
+                                         bool OnlyIfReduced = false);
   LLVM_ABI static Constant *getPtrToInt(Constant *C, Type *Ty,
                                         bool OnlyIfReduced = false);
   LLVM_ABI static Constant *getIntToPtr(Constant *C, Type *Ty,
diff --git a/llvm/include/llvm/IR/DebugInfo.h b/llvm/include/llvm/IR/DebugInfo.h
index c529a86309a94..220e4c1cad4f7 100644
--- a/llvm/include/llvm/IR/DebugInfo.h
+++ b/llvm/include/llvm/IR/DebugInfo.h
@@ -195,38 +195,8 @@ inline AssignmentInstRange getAssignmentInsts(const DbgVariableRecord *DVR) {
   return getAssignmentInsts(DVR->getAssignID());
 }
 
-//
-// Utilities for enumerating llvm.dbg.assign intrinsic from an assignment ID.
-//
-/// High level: this is an iterator for llvm.dbg.assign intrinsics.
-/// Implementation details: this is a wrapper around Value's User iterator that
-/// dereferences to a DbgAssignIntrinsic ptr rather than a User ptr.
-class DbgAssignIt
-    : public iterator_adaptor_base<DbgAssignIt, Value::user_iterator,
-                                   typename std::iterator_traits<
-                                       Value::user_iterator>::iterator_category,
-                                   DbgAssignIntrinsic *, std::ptrdiff_t,
-                                   DbgAssignIntrinsic **,
-                                   DbgAssignIntrinsic *&> {
-public:
-  DbgAssignIt(Value::user_iterator It) : iterator_adaptor_base(It) {}
-  DbgAssignIntrinsic *operator*() const { return cast<DbgAssignIntrinsic>(*I); }
-};
-/// A range of llvm.dbg.assign intrinsics.
-using AssignmentMarkerRange = iterator_range<DbgAssignIt>;
-/// Return a range of dbg.assign intrinsics which use \ID as an operand.
-/// Iterators invalidated by deleting an intrinsic contained in this range.
-LLVM_ABI AssignmentMarkerRange getAssignmentMarkers(DIAssignID *ID);
-/// Return a range of dbg.assign intrinsics for which \p Inst performs the
+/// Return a range of dbg_assign records for which \p Inst performs the
 /// assignment they encode.
-/// Iterators invalidated by deleting an intrinsic contained in this range.
-inline AssignmentMarkerRange getAssignmentMarkers(const Instruction *Inst) {
-  if (auto *ID = Inst->getMetadata(LLVMContext::MD_DIAssignID))
-    return getAssignmentMarkers(cast<DIAssignID>(ID));
-  else
-    return make_range(Value::user_iterator(), Value::user_iterator());
-}
-
 inline SmallVector<DbgVariableRecord *>
 getDVRAssignmentMarkers(const Instruction *Inst) {
   if (auto *ID = Inst->getMetadata(LLVMContext::MD_DIAssignID))
@@ -253,11 +223,6 @@ LLVM_ABI void deleteAll(Function *F);
 ///
 /// Result contains a zero-sized fragment if there's no intersect.
 LLVM_ABI bool
-calculateFragmentIntersect(const DataLayout &DL, const Value *Dest,
-                           uint64_t SliceOffsetInBits, uint64_t SliceSizeInBits,
-                           const DbgAssignIntrinsic *DbgAssign,
-                           std::optional<DIExpression::FragmentInfo> &Result);
-LLVM_ABI bool
 calculateFragmentIntersect(const DataLayout &DL, const Value *Dest,
                            uint64_t SliceOffsetInBits, uint64_t SliceSizeInBits,
                            const DbgVariableRecord *DVRAssign,
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index f1f0c18949c35..a4e9d1639bb2b 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -2506,10 +2506,8 @@ class DISubprogram : public DILocalScope {
 class DILocation : public MDNode {
   friend class LLVMContextImpl;
   friend class MDNode;
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
   uint64_t AtomGroup : 61;
   uint64_t AtomRank : 3;
-#endif
 
   DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
              unsigned Column, uint64_t AtomGroup, uint8_t AtomRank,
@@ -2539,20 +2537,8 @@ class DILocation : public MDNode {
   }
 
 public:
-  uint64_t getAtomGroup() const {
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
-    return AtomGroup;
-#else
-    return 0;
-#endif
-  }
-  uint8_t getAtomRank() const {
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
-    return AtomRank;
-#else
-    return 0;
-#endif
-  }
+  uint64_t getAtomGroup() const { return AtomGroup; }
+  uint8_t getAtomRank() const { return AtomRank; }
 
   const DILocation *getWithoutAtom() const {
     if (!getAtomGroup() && !getAtomRank())
diff --git a/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h b/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h
index 8aac9d5b49dbb..448a6e913eb86 100644
--- a/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h
+++ b/llvm/include/llvm/IR/GenericFloatingPointPredicateUtils.h
@@ -135,6 +135,12 @@ template <typename ContextT> class GenericFloatingPointPredicateUtils {
       if (Mode.Input != DenormalMode::IEEE)
         return {Invalid, fcAllFlags, fcAllFlags};
 
+      auto ExactClass = [IsFabs, Src](FPClassTest Mask) {
+        if (IsFabs)
+          Mask = llvm::inverse_fabs(Mask);
+        return exactClass(Src, Mask);
+      };
+
       switch (Pred) {
       case FCmpInst::FCMP_OEQ: // Match x == 0.0
         return exactClass(Src, fcZero);
@@ -151,26 +157,24 @@ template <typename ContextT> class GenericFloatingPointPredicateUtils {
       case FCmpInst::FCMP_UNO:
         return exactClass(Src, fcNan);
       case FCmpInst::FCMP_OGT: // x > 0
-        return exactClass(Src, fcPosSubnormal | fcPosNormal | fcPosInf);
+        return ExactClass(fcPosSubnormal | fcPosNormal | fcPosInf);
       case FCmpInst::FCMP_UGT: // isnan(x) || x > 0
-        return exactClass(Src, fcPosSubnormal | fcPosNormal | fcPosInf | fcNan);
+        return ExactClass(fcPosSubnormal | fcPosNormal | fcPosInf | fcNan);
       case FCmpInst::FCMP_OGE: // x >= 0
-        return exactClass(Src, fcPositive | fcNegZero);
+        return ExactClass(fcPositive | fcNegZero);
       case FCmpInst::FCMP_UGE: // isnan(x) || x >= 0
-        return exactClass(Src, fcPositive | fcNegZero | fcNan);
+        return ExactClass(fcPositive | fcNegZero | fcNan);
       case FCmpInst::FCMP_OLT: // x < 0
-        return exactClass(Src, fcNegSubnormal | fcNegNormal | fcNegInf);
+        return ExactClass(fcNegSubnormal | fcNegNormal | fcNegInf);
       case FCmpInst::FCMP_ULT: // isnan(x) || x < 0
-        return exactClass(Src, fcNegSubnormal | fcNegNormal | fcNegInf | fcNan);
+        return ExactClass(fcNegSubnormal | fcNegNormal | fcNegInf | fcNan);
       case FCmpInst::FCMP_OLE: // x <= 0
-        return exactClass(Src, fcNegative | fcPosZero);
+        return ExactClass(fcNegative | fcPosZero);
       case FCmpInst::FCMP_ULE: // isnan(x) || x <= 0
-        return exactClass(Src, fcNegative | fcPosZero | fcNan);
+        return ExactClass(fcNegative | fcPosZero | fcNan);
       default:
         llvm_unreachable("all compare types are handled");
       }
-
-      return {Invalid, fcAllFlags, fcAllFlags};
     }
 
     const bool IsDenormalRHS = (OrigClass & fcSubnormal) == OrigClass;
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index d106dedf814e5..783f8f6d2478c 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -812,15 +812,10 @@ class IRBuilderBase {
   LLVM_ABI CallInst *CreateFPMinimumReduce(Value *Src);
 
   /// Create a lifetime.start intrinsic.
-  ///
-  /// If the pointer isn't i8* it will be converted.
-  LLVM_ABI CallInst *CreateLifetimeStart(Value *Ptr,
-                                         ConstantInt *Size = nullptr);
+  LLVM_ABI CallInst *CreateLifetimeStart(Value *Ptr);
 
   /// Create a lifetime.end intrinsic.
-  ///
-  /// If the pointer isn't i8* it will be converted.
-  LLVM_ABI CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr);
+  LLVM_ABI CallInst *CreateLifetimeEnd(Value *Ptr);
 
   /// Create a call to invariant.start intrinsic.
   ///
@@ -2192,7 +2187,10 @@ class IRBuilderBase {
     return CreateCast(Instruction::FPExt, V, DestTy, Name, FPMathTag,
                       FMFSource);
   }
-
+  Value *CreatePtrToAddr(Value *V, const Twine &Name = "") {
+    return CreateCast(Instruction::PtrToInt, V,
+                      BB->getDataLayout().getAddressType(V->getType()), Name);
+  }
   Value *CreatePtrToInt(Value *V, Type *DestTy,
                         const Twine &Name = "") {
     return CreateCast(Instruction::PtrToInt, V, DestTy, Name);
diff --git a/llvm/include/llvm/IR/InstVisitor.h b/llvm/include/llvm/IR/InstVisitor.h
index 6d5398bb7a4cd..8e4dc647e5230 100644
--- a/llvm/include/llvm/IR/InstVisitor.h
+++ b/llvm/include/llvm/IR/InstVisitor.h
@@ -183,6 +183,7 @@ class InstVisitor {
   RetTy visitUIToFPInst(UIToFPInst &I)            { DELEGATE(CastInst);}
   RetTy visitSIToFPInst(SIToFPInst &I)            { DELEGATE(CastInst);}
   RetTy visitPtrToIntInst(PtrToIntInst &I)        { DELEGATE(CastInst);}
+  RetTy visitPtrToAddrInst(PtrToAddrInst &I)      { DELEGATE(CastInst);}
   RetTy visitIntToPtrInst(IntToPtrInst &I)        { DELEGATE(CastInst);}
   RetTy visitBitCastInst(BitCastInst &I)          { DELEGATE(CastInst);}
   RetTy visitAddrSpaceCastInst(AddrSpaceCastInst &I) { DELEGATE(CastInst);}
diff --git a/llvm/include/llvm/IR/Instruction.def b/llvm/include/llvm/IR/Instruction.def
index a5ad92f58f94e..face6a93ec7d5 100644
--- a/llvm/include/llvm/IR/Instruction.def
+++ b/llvm/include/llvm/IR/Instruction.def
@@ -190,35 +190,36 @@ HANDLE_CAST_INST(43, UIToFP  , UIToFPInst  )  // UInt -> floating point
 HANDLE_CAST_INST(44, SIToFP  , SIToFPInst  )  // SInt -> floating point
 HANDLE_CAST_INST(45, FPTrunc , FPTruncInst )  // Truncate floating point
 HANDLE_CAST_INST(46, FPExt   , FPExtInst   )  // Extend floating point
-HANDLE_CAST_INST(47, PtrToInt, PtrToIntInst)  // Pointer -> Integer
-HANDLE_CAST_INST(48, IntToPtr, IntToPtrInst)  // Integer -> Pointer
-HANDLE_CAST_INST(49, BitCast , BitCastInst )  // Type cast
-HANDLE_CAST_INST(50, AddrSpaceCast, AddrSpaceCastInst)  // addrspace cast
-  LAST_CAST_INST(50)
+HANDLE_CAST_INST(47, PtrToInt, PtrToIntInst)  // Pointer -> Integer (bitcast)
+HANDLE_CAST_INST(48, PtrToAddr, PtrToAddrInst) // Pointer -> Address
+HANDLE_CAST_INST(49, IntToPtr, IntToPtrInst)  // Integer -> Pointer
+HANDLE_CAST_INST(50, BitCast , BitCastInst )  // Type cast
+HANDLE_CAST_INST(51, AddrSpaceCast, AddrSpaceCastInst)  // addrspace cast
+  LAST_CAST_INST(51)
 
- FIRST_FUNCLETPAD_INST(51)
-HANDLE_FUNCLETPAD_INST(51, CleanupPad, CleanupPadInst)
-HANDLE_FUNCLETPAD_INST(52, CatchPad  , CatchPadInst)
-  LAST_FUNCLETPAD_INST(52)
+ FIRST_FUNCLETPAD_INST(52)
+HANDLE_FUNCLETPAD_INST(52, CleanupPad, CleanupPadInst)
+HANDLE_FUNCLETPAD_INST(53, CatchPad  , CatchPadInst)
+  LAST_FUNCLETPAD_INST(53)
 
 // Other operators...
- FIRST_OTHER_INST(53)
-HANDLE_OTHER_INST(53, ICmp   , ICmpInst   )  // Integer comparison instruction
-HANDLE_OTHER_INST(54, FCmp   , FCmpInst   )  // Floating point comparison instr.
-HANDLE_OTHER_INST(55, PHI    , PHINode    )  // PHI node instruction
-HANDLE_OTHER_INST(56, Call   , CallInst   )  // Call a function
-HANDLE_OTHER_INST(57, Select , SelectInst )  // select instruction
-HANDLE_USER_INST (58, UserOp1, Instruction)  // May be used internally in a pass
-HANDLE_USER_INST (59, UserOp2, Instruction)  // Internal to passes only
-HANDLE_OTHER_INST(60, VAArg  , VAArgInst  )  // vaarg instruction
-HANDLE_OTHER_INST(61, ExtractElement, ExtractElementInst)// extract from vector
-HANDLE_OTHER_INST(62, InsertElement, InsertElementInst)  // insert into vector
-HANDLE_OTHER_INST(63, ShuffleVector, ShuffleVectorInst)  // shuffle two vectors.
-HANDLE_OTHER_INST(64, ExtractValue, ExtractValueInst)// extract from aggregate
-HANDLE_OTHER_INST(65, InsertValue, InsertValueInst)  // insert into aggregate
-HANDLE_OTHER_INST(66, LandingPad, LandingPadInst)  // Landing pad instruction.
-HANDLE_OTHER_INST(67, Freeze, FreezeInst) // Freeze instruction.
-  LAST_OTHER_INST(67)
+ FIRST_OTHER_INST(54)
+HANDLE_OTHER_INST(54, ICmp   , ICmpInst   )  // Integer comparison instruction
+HANDLE_OTHER_INST(55, FCmp   , FCmpInst   )  // Floating point comparison instr.
+HANDLE_OTHER_INST(56, PHI    , PHINode    )  // PHI node instruction
+HANDLE_OTHER_INST(57, Call   , CallInst   )  // Call a function
+HANDLE_OTHER_INST(58, Select , SelectInst )  // select instruction
+HANDLE_USER_INST (59, UserOp1, Instruction)  // May be used internally in a pass
+HANDLE_USER_INST (60, UserOp2, Instruction)  // Internal to passes only
+HANDLE_OTHER_INST(61, VAArg  , VAArgInst  )  // vaarg instruction
+HANDLE_OTHER_INST(62, ExtractElement, ExtractElementInst)// extract from vector
+HANDLE_OTHER_INST(63, InsertElement, InsertElementInst)  // insert into vector
+HANDLE_OTHER_INST(64, ShuffleVector, ShuffleVectorInst)  // shuffle two vectors.
+HANDLE_OTHER_INST(65, ExtractValue, ExtractValueInst)// extract from aggregate
+HANDLE_OTHER_INST(66, InsertValue, InsertValueInst)  // insert into aggregate
+HANDLE_OTHER_INST(67, LandingPad, LandingPadInst)  // Landing pad instruction.
+HANDLE_OTHER_INST(68, Freeze, FreezeInst) // Freeze instruction.
+  LAST_OTHER_INST(68)
 
 #undef  FIRST_TERM_INST
 #undef HANDLE_TERM_INST
diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h
index 5d25804a684ac..2eb4fd36c5b7d 100644
--- a/llvm/include/llvm/IR/Instruction.h
+++ b/llvm/include/llvm/IR/Instruction.h
@@ -584,9 +584,10 @@ class Instruction : public User,
   dropUBImplyingAttrsAndUnknownMetadata(ArrayRef<unsigned> KnownIDs = {});
 
   /// Drop any attributes or metadata that can cause immediate undefined
-  /// behavior. Retain other attributes/metadata on a best-effort basis.
-  /// This should be used when speculating instructions.
-  LLVM_ABI void dropUBImplyingAttrsAndMetadata();
+  /// behavior. Retain other attributes/metadata on a best-effort basis, as well
+  /// as those passed in `Keep`. This should be used when speculating
+  /// instructions.
+  LLVM_ABI void dropUBImplyingAttrsAndMetadata(ArrayRef<unsigned> Keep = {});
 
   /// Return true if this instruction has UB-implying attributes
   /// that can cause immediate undefined behavior.
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index 6f69b68f628fc..95a0a7fd2f97e 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -4949,6 +4949,46 @@ class PtrToIntInst : public CastInst {
   }
 };
 
+/// This class represents a cast from a pointer to an address (non-capturing
+/// ptrtoint).
+class PtrToAddrInst : public CastInst {
+protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+
+  /// Clone an identical PtrToAddrInst.
+  PtrToAddrInst *cloneImpl() const;
+
+public:
+  /// Constructor with insert-before-instruction semantics
+  PtrToAddrInst(Value *S,                  ///< The value to be converted
+                Type *Ty,                  ///< The type to convert to
+                const Twine &NameStr = "", ///< A name for the new instruction
+                InsertPosition InsertBefore =
+                    nullptr ///< Where to insert the new instruction
+  );
+
+  /// Gets the pointer operand.
+  Value *getPointerOperand() { return getOperand(0); }
+  /// Gets the pointer operand.
+  const Value *getPointerOperand() const { return getOperand(0); }
+  /// Gets the operand index of the pointer operand.
+  static unsigned getPointerOperandIndex() { return 0U; }
+
+  /// Returns the address space of the pointer operand.
+  unsigned getPointerAddressSpace() const {
+    return getPointerOperand()->getType()->getPointerAddressSpace();
+  }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static bool classof(const Instruction *I) {
+    return I->getOpcode() == PtrToAddr;
+  }
+  static bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+};
+
 //===----------------------------------------------------------------------===//
 //                             BitCastInst Class
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 2e1389633d7e4..eb0440f500735 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -987,6 +987,13 @@ template <typename Derived> class MemIntrinsicBase : public IntrinsicInst {
   const Use &getLengthUse() const { return getArgOperandUse(ARG_LENGTH); }
   Use &getLengthUse() { return getArgOperandUse(ARG_LENGTH); }
 
+  std::optional<APInt> getLengthInBytes() const {
+    ConstantInt *C = dyn_cast<ConstantInt>(getLength());
+    if (!C)
+      return std::nullopt;
+    return C->getValue();
+  }
+
   /// This is just like getRawDest, but it strips off any cast
   /// instructions (including addrspacecast) that feed it, giving the
   /// original input.  The returned value is guaranteed to be a pointer.
@@ -1022,6 +1029,10 @@ template <typename Derived> class MemIntrinsicBase : public IntrinsicInst {
            "setLength called with value of wrong type!");
     setArgOperand(ARG_LENGTH, L);
   }
+
+  void setLength(uint64_t L) {
+    setLength(ConstantInt::get(getLength()->getType(), L));
+  }
 };
 
 /// Common base class for all memory transfer intrinsics. Simply provides
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index 48735b06d3f53..9577d0141f168 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -104,12 +104,6 @@ namespace Intrinsic {
   LLVM_ABI Function *getOrInsertDeclaration(Module *M, ID id,
                                             ArrayRef<Type *> Tys = {});
 
-  LLVM_DEPRECATED("Use getOrInsertDeclaration instead",
-                  "getOrInsertDeclaration")
-  inline Function *getDeclaration(Module *M, ID id, ArrayRef<Type *> Tys = {}) {
-    return getOrInsertDeclaration(M, id, Tys);
-  }
-
   /// Look up the Function declaration of the intrinsic \p id in the Module
   /// \p M and return it if it exists. Otherwise, return nullptr. This version
   /// supports non-overloaded intrinsics.
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 1a23276bc6ffb..e0ee12391b31d 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -1644,16 +1644,12 @@ def int_ucmp : DefaultAttrsIntrinsic<
 
 //===------------------------- Memory Use Markers -------------------------===//
 //
-def int_lifetime_start  : DefaultAttrsIntrinsic<[],
-                                    [llvm_i64_ty, llvm_anyptr_ty],
-                                    [IntrArgMemOnly,
-                                     NoCapture<ArgIndex<1>>,
-                                     ImmArg<ArgIndex<0>>]>;
-def int_lifetime_end    : DefaultAttrsIntrinsic<[],
-                                    [llvm_i64_ty, llvm_anyptr_ty],
-                                    [IntrArgMemOnly,
-                                     NoCapture<ArgIndex<1>>,
-                                     ImmArg<ArgIndex<0>>]>;
+def int_lifetime_start
+    : DefaultAttrsIntrinsic<[], [llvm_anyptr_ty],
+                            [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
+def int_lifetime_end
+    : DefaultAttrsIntrinsic<[], [llvm_anyptr_ty],
+                            [IntrArgMemOnly, NoCapture<ArgIndex<0>>]>;
 def int_invariant_start : DefaultAttrsIntrinsic<[llvm_ptr_ty],
                                     [llvm_i64_ty, llvm_anyptr_ty],
                                     [IntrArgMemOnly,
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index ca6e2128812f7..6d53bf8b172d8 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -296,22 +296,22 @@ let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
   def int_aarch64_neon_sqrdmlah : AdvSIMD_3IntArg_Intrinsic;
   def int_aarch64_neon_sqrdmlsh : AdvSIMD_3IntArg_Intrinsic;
 
-  // Vector Polynominal Multiply
-  def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
-
-  // Vector Long Multiply
-  def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
-  def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
-  def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
-
-  // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
-  // it with a v16i8.
-  def int_aarch64_neon_pmull64 :
-        DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
-
-  // Vector Extending Multiply
-  def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
-    let IntrProperties = [IntrNoMem, Commutative];
+  let IntrProperties = [IntrNoMem, Commutative] in {
+    // Vector Polynominal Multiply
+    def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
+
+    // Vector Long Multiply
+    def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
+    def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
+    def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
+
+    // 64-bit polynomial multiply really returns an i128, which is not legal.
+    // Fake it with a v16i8.
+    def int_aarch64_neon_pmull64
+      : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty]>;
+
+    // Vector Extending Multiply
+    def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic;
   }
 
   // Vector Saturating Doubling Long Multiply
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 90cfd8cedd51b..abd83c7c4d4a7 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -290,11 +290,29 @@ def int_amdgcn_s_barrier_signal_isfirst : ClangBuiltin<"__builtin_amdgcn_s_barri
   Intrinsic<[llvm_i1_ty], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrConvergent,
                                 IntrWillReturn, IntrNoCallback, IntrNoFree]>;
 
+// void @llvm.amdgcn.s.barrier.init(ptr addrspace(3) %barrier, i32 %memberCnt)
+// The %barrier and %memberCnt argument must be uniform, otherwise behavior is undefined.
+def int_amdgcn_s_barrier_init : ClangBuiltin<"__builtin_amdgcn_s_barrier_init">,
+  Intrinsic<[], [local_ptr_ty, llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, IntrConvergent,
+                                IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+
+// void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) %barrier)
+// The %barrier argument must be uniform, otherwise behavior is undefined.
+def int_amdgcn_s_barrier_join : ClangBuiltin<"__builtin_amdgcn_s_barrier_join">,
+  Intrinsic<[], [local_ptr_ty], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn,
+                                IntrNoCallback, IntrNoFree]>;
+
 // void @llvm.amdgcn.s.barrier.wait(i16 %barrierType)
 def int_amdgcn_s_barrier_wait : ClangBuiltin<"__builtin_amdgcn_s_barrier_wait">,
   Intrinsic<[], [llvm_i16_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrConvergent,
                                 IntrWillReturn, IntrNoCallback, IntrNoFree]>;
 
+
+// void @llvm.amdgcn.s.barrier.leave(i16 %barrierType)
+def int_amdgcn_s_barrier_leave : ClangBuiltin<"__builtin_amdgcn_s_barrier_leave">,
+  Intrinsic<[], [llvm_i16_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrConvergent,
+                                IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+
 // uint32_t @llvm.amdgcn.s.get.barrier.state(i32 %barrierId)
 // The %barrierType argument must be uniform, otherwise behavior is undefined.
 def int_amdgcn_s_get_barrier_state : ClangBuiltin<"__builtin_amdgcn_s_get_barrier_state">,
@@ -2671,6 +2689,18 @@ def int_amdgcn_cs_chain:
             ],
             [IntrConvergent, IntrNoReturn, ImmArg<ArgIndex<4>>]>;
 
+// Run a function with all the lanes enabled. Only direct calls are allowed. The
+// first argument is the callee, which must have the `amdgpu_gfx_whole_wave`
+// calling convention and must not be variadic. The remaining arguments to the
+// callee are taken from the arguments passed to the intrinsic. Lanes that are
+// inactive at the point of the call will receive poison. The return value is
+// the return value of the callee for the active lanes (there is no return
+// value in the inactive ones).
+def int_amdgcn_call_whole_wave:
+  Intrinsic<[llvm_any_ty],    // The return type of the callee.
+            [llvm_anyptr_ty,  // The callee.
+             llvm_vararg_ty], // The arguments to the callee.
+            [IntrConvergent]>;
 
 //===----------------------------------------------------------------------===//
 // CI+ Intrinsics
@@ -3087,7 +3117,7 @@ class AMDGPULoadIntrinsic<LLVMType ptr_ty>:
   Intrinsic<
     [llvm_any_ty],
     [ptr_ty],
-    [IntrReadMem, IntrWillReturn, IntrConvergent, NoCapture<ArgIndex<0>>, IntrNoCallback, IntrNoFree],
+    [IntrReadMem, IntrArgMemOnly, IntrWillReturn, IntrConvergent, NoCapture<ArgIndex<0>>, IntrNoCallback, IntrNoFree],
     "",
     [SDNPMemOperand]
   >;
diff --git a/llvm/include/llvm/IR/IntrinsicsLoongArch.td b/llvm/include/llvm/IR/IntrinsicsLoongArch.td
index f5e3d412666a2..84026aa9d3624 100644
--- a/llvm/include/llvm/IR/IntrinsicsLoongArch.td
+++ b/llvm/include/llvm/IR/IntrinsicsLoongArch.td
@@ -19,6 +19,10 @@ let TargetPrefix = "loongarch" in {
 class MaskedAtomicRMW<LLVMType itype>
     : Intrinsic<[itype], [llvm_anyptr_ty, itype, itype, itype],
                 [IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
+// T @llvm.<name>.T.<p>(any*, T, T, T, T imm);
+class LoongArchMaskedAtomicRMWFiveArg<LLVMType itype>
+    : Intrinsic<[itype], [llvm_anyptr_ty, itype, itype, itype, itype],
+                [IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<4>>]>;
 
 // We define 32-bit and 64-bit variants of the above, where T stands for i32
 // or i64 respectively:
@@ -31,9 +35,9 @@ multiclass MaskedAtomicRMWIntrinsics {
 
 multiclass MaskedAtomicRMWFiveOpIntrinsics {
   // i32 @llvm.<name>.i32.<p>(any*, i32, i32, i32, i32 imm);
-  def _i32 : MaskedAtomicRMWFiveArg<llvm_i32_ty>;
+  def _i32 : LoongArchMaskedAtomicRMWFiveArg<llvm_i32_ty>;
   // i64 @llvm.<name>.i64.<p>(any*, i64, i64, i64, i64 imm);
-  def _i64 : MaskedAtomicRMWFiveArg<llvm_i64_ty>;
+  def _i64 : LoongArchMaskedAtomicRMWFiveArg<llvm_i64_ty>;
 }
 
 defm int_loongarch_masked_atomicrmw_xchg : MaskedAtomicRMWIntrinsics;
diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 967d1663f237b..7b40841e45d0d 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -128,15 +128,16 @@
 //   * llvm.nvvm.swap.lo.hi.b64      --> llvm.fshl(x, x, 32)
 //   * llvm.nvvm.atomic.load.inc.32  --> atomicrmw uinc_wrap
 //   * llvm.nvvm.atomic.load.dec.32  --> atomicrmw udec_wrap
-// * llvm.nvvm.barrier0              --> llvm.nvvm.barrier.cta.sync.aligned.all(0)
-// * llvm.nvvm.barrier.n             --> llvm.nvvm.barrier.cta.sync.aligned.all(x)
-// * llvm.nvvm.bar.sync              --> llvm.nvvm.barrier.cta.sync.aligned.all(x)
-// * llvm.nvvm.barrier               --> llvm.nvvm.barrier.cta.sync.aligned(x, y)
-// * llvm.nvvm.barrier.sync          --> llvm.nvvm.barrier.cta.sync.all(x)
-// * llvm.nvvm.barrier.sync.cnt      --> llvm.nvvm.barrier.cta.sync(x, y)
+//   * llvm.nvvm.barrier0            --> llvm.nvvm.barrier.cta.sync.aligned.all(0)
+//   * llvm.nvvm.barrier.n           --> llvm.nvvm.barrier.cta.sync.aligned.all(x)
+//   * llvm.nvvm.bar.sync            --> llvm.nvvm.barrier.cta.sync.aligned.all(x)
+//   * llvm.nvvm.barrier             --> llvm.nvvm.barrier.cta.sync.aligned(x, y)
+//   * llvm.nvvm.barrier.sync        --> llvm.nvvm.barrier.cta.sync.all(x)
+//   * llvm.nvvm.barrier.sync.cnt    --> llvm.nvvm.barrier.cta.sync(x, y)
 
 def llvm_global_ptr_ty  : LLVMQualPointerType<1>;         // (global)ptr
 def llvm_shared_ptr_ty  : LLVMQualPointerType<3>;         // (shared)ptr
+def llvm_constant_ptr_ty: LLVMQualPointerType<4>;         // (const)ptr
 def llvm_local_ptr_ty   : LLVMQualPointerType<5>;         // (local)ptr
 def llvm_tmem_ptr_ty    : LLVMQualPointerType<6>;         // (tensor memory)ptr
 def llvm_shared_cluster_ptr_ty : LLVMQualPointerType<7>;  // (shared_cluster)ptr
@@ -170,172 +171,248 @@ class StrJoin<string sep, list<string> str_list> {
 // Geom: m<M>n<N>k<K>. E.g. m8n32k16
 // Frag: [a|b|c|d] ([x1|x2|x4] for ldmatrix)
 // PtxEltType: PTX type for the element.
-class WMMA_REGS<string Geom, string Frag, string PtxEltType> {
+class WMMA_REGS<string Geom, string Frag, string PtxEltType, bit IsSparse = false> {
   string geom = Geom;
   string frag = Frag;
   string ptx_elt_type = PtxEltType;
   string gft = Geom#":"#Frag#":"#ptx_elt_type;
   string gf = Geom#":"#Frag;
   string ft = frag#":"#ptx_elt_type;
-  list<LLVMType> regs = !cond(
-    // mma fp ops use smaller fragments than wmma fp ops
-    !eq(gft,"m8n8k4:a:f16") : !listsplat(llvm_v2f16_ty, 2),
-    !eq(gft,"m8n8k4:b:f16") : !listsplat(llvm_v2f16_ty, 2),
-    !eq(gft,"m16n8k8:a:f16") : !listsplat(llvm_v2f16_ty, 2),
-    !eq(gft,"m16n8k8:b:f16") : [llvm_v2f16_ty],
-    !eq(gft,"m16n8k8:c:f16") : !listsplat(llvm_v2f16_ty, 2),
-    !eq(gft,"m16n8k8:d:f16") : !listsplat(llvm_v2f16_ty, 2),
-    !eq(gft,"m16n8k8:c:f32") : !listsplat(llvm_float_ty, 4),
-    !eq(gft,"m16n8k8:d:f32") : !listsplat(llvm_float_ty, 4),
-    !eq(gft,"m16n8k16:a:f16") : !listsplat(llvm_v2f16_ty, 4),
-    !eq(gft,"m16n8k16:b:f16") : !listsplat(llvm_v2f16_ty, 2),
-    !eq(gft,"m16n8k16:c:f16") : !listsplat(llvm_v2f16_ty, 2),
-    !eq(gft,"m16n8k16:d:f16") : !listsplat(llvm_v2f16_ty, 2),
-    !eq(gft,"m16n8k16:c:f32") : !listsplat(llvm_float_ty, 4),
-    !eq(gft,"m16n8k16:d:f32") : !listsplat(llvm_float_ty, 4),
-    !eq(gft,"m16n8k4:c:f32") : !listsplat(llvm_float_ty, 4),
-    !eq(gft,"m16n8k4:d:f32") : !listsplat(llvm_float_ty, 4),
-
-    // wmma fp16 -> fp16/fp32 @  m16n16k16/m8n32k16/m32n8k16
-    // All other supported geometries use the same fragment format for f32 and
-    // f16, so we only need to consider {fragment, type}.
-    !eq(ft,"a:f16") : !listsplat(llvm_v2f16_ty, 8),
-    !eq(ft,"b:f16") : !listsplat(llvm_v2f16_ty, 8),
-    !eq(ft,"c:f16") : !listsplat(llvm_v2f16_ty, 4),
-    !eq(ft,"d:f16") : !listsplat(llvm_v2f16_ty, 4),
-    !eq(ft,"c:f32") : !listsplat(llvm_float_ty, 8),
-    !eq(ft,"d:f32") : !listsplat(llvm_float_ty, 8),
-
-    // wmma tf32 -> s32 @ m16n16k8
-    !eq(gft,"m16n16k8:a:tf32") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n16k8:b:tf32") : !listsplat(llvm_i32_ty, 4),
-
-    // mma tf32 -> s32 @ m16n16k8/m16n8k8
-    !eq(gft,"m16n8k4:a:tf32") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k4:b:tf32") : [llvm_i32_ty],
-    !eq(gft,"m16n8k8:a:tf32") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k8:b:tf32") : !listsplat(llvm_i32_ty, 2),
-
-    !eq(gft,"m8n8k4:a:f64") : [llvm_double_ty],
-    !eq(gft,"m8n8k4:b:f64") : [llvm_double_ty],
-    !eq(gft,"m8n8k4:c:f64") : !listsplat(llvm_double_ty, 2),
-    !eq(gft,"m8n8k4:d:f64") : !listsplat(llvm_double_ty, 2),
-
-    // wmma bf16 -> s32 @ m16n16k16/m8n32k16/m32n8k16
-    !eq(gft,"m16n16k16:a:bf16") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n16k16:b:bf16") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m8n32k16:a:bf16") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m8n32k16:b:bf16") : !listsplat(llvm_i32_ty, 8),
-    !eq(gft,"m32n8k16:a:bf16") : !listsplat(llvm_i32_ty, 8),
-    !eq(gft,"m32n8k16:b:bf16") : !listsplat(llvm_i32_ty, 2),
-
-    // mma bf16 -> s32 @ m16n8k16/m16n8k8
-    !eq(gft,"m16n8k16:a:bf16") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k16:b:bf16") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k8:a:bf16") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k8:b:bf16") : [llvm_i32_ty],
-
-    // wmma u8/s8 -> s32 @ m16n16k16/m8n32k16/m32n8k16
-    !eq(gft,"m16n16k16:a:u8") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n16k16:a:s8") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n16k16:b:u8") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n16k16:b:s8") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n16k16:c:s32") : !listsplat(llvm_i32_ty, 8),
-    !eq(gft,"m16n16k16:d:s32") : !listsplat(llvm_i32_ty, 8),
-
-    !eq(gft,"m8n32k16:a:u8") : [llvm_i32_ty],
-    !eq(gft,"m8n32k16:a:s8") : [llvm_i32_ty],
-    !eq(gft,"m8n32k16:b:u8") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m8n32k16:b:s8") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m8n32k16:c:s32") : !listsplat(llvm_i32_ty, 8),
-    !eq(gft,"m8n32k16:d:s32") : !listsplat(llvm_i32_ty, 8),
-
-    !eq(gft,"m32n8k16:a:u8") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m32n8k16:a:s8") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m32n8k16:b:u8") : [llvm_i32_ty],
-    !eq(gft,"m32n8k16:b:s8") : [llvm_i32_ty],
-    !eq(gft,"m32n8k16:c:s32") : !listsplat(llvm_i32_ty, 8),
-    !eq(gft,"m32n8k16:d:s32") : !listsplat(llvm_i32_ty, 8),
-
-    // mma u8/s8 -> s32 @ m8n8k16/m16n8k16/m16n8k32
-    !eq(gft,"m8n8k16:a:u8") : [llvm_i32_ty],
-    !eq(gft,"m8n8k16:a:s8") : [llvm_i32_ty],
-    !eq(gft,"m8n8k16:b:u8") : [llvm_i32_ty],
-    !eq(gft,"m8n8k16:b:s8") : [llvm_i32_ty],
-    !eq(gft,"m8n8k16:c:s32") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m8n8k16:d:s32") : !listsplat(llvm_i32_ty, 2),
-
-    !eq(gft,"m16n8k16:a:u8") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k16:a:s8") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k16:b:u8") : [llvm_i32_ty],
-    !eq(gft,"m16n8k16:b:s8") : [llvm_i32_ty],
-    !eq(gft,"m16n8k16:c:s32") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k16:d:s32") : !listsplat(llvm_i32_ty, 4),
-
-    !eq(gft,"m16n8k32:a:u8") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k32:a:s8") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k32:b:u8") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k32:b:s8") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k32:c:s32") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k32:d:s32") : !listsplat(llvm_i32_ty, 4),
-
-    // wmma/mma u4/s4 -> s32 @ m8n8k32 (u4/s4)
-    !eq(gft,"m8n8k32:a:u4") : [llvm_i32_ty],
-    !eq(gft,"m8n8k32:a:s4") : [llvm_i32_ty],
-    !eq(gft,"m8n8k32:b:u4") : [llvm_i32_ty],
-    !eq(gft,"m8n8k32:b:s4") : [llvm_i32_ty],
-    !eq(gft,"m8n8k32:c:s32") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m8n8k32:d:s32") : !listsplat(llvm_i32_ty, 2),
-
-    !eq(gft,"m16n8k32:a:u4") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k32:a:s4") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k32:b:u4") : [llvm_i32_ty],
-    !eq(gft,"m16n8k32:b:s4") : [llvm_i32_ty],
-    !eq(gft,"m16n8k32:c:s32") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k32:d:s32") : !listsplat(llvm_i32_ty, 4),
-
-    !eq(gft,"m16n8k64:a:u4") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k64:a:s4") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k64:b:u4") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k64:b:s4") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k64:c:s32") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k64:d:s32") : !listsplat(llvm_i32_ty, 4),
-
-    // wmma/mma b1 -> s32 @ m8n8k128(b1)
-    !eq(gft,"m8n8k128:a:b1") : [llvm_i32_ty],
-    !eq(gft,"m8n8k128:b:b1") : [llvm_i32_ty],
-    !eq(gft,"m8n8k128:c:s32") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m8n8k128:d:s32") : !listsplat(llvm_i32_ty, 2),
-
-    !eq(gft,"m16n8k128:a:b1") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k128:b:b1") : [llvm_i32_ty],
-    !eq(gft,"m16n8k128:c:s32") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k128:d:s32") : !listsplat(llvm_i32_ty, 4),
-
-    !eq(gft,"m16n8k256:a:b1") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k256:b:b1") : !listsplat(llvm_i32_ty, 2),
-    !eq(gft,"m16n8k256:c:s32") : !listsplat(llvm_i32_ty, 4),
-    !eq(gft,"m16n8k256:d:s32") : !listsplat(llvm_i32_ty, 4),
-
-    // ldmatrix b16 -> s32 @ m8n8
-    !eq(gf,"m8n8:x1") : !listsplat(llvm_i32_ty, 1),
-    !eq(gf,"m8n8:x2") : !listsplat(llvm_i32_ty, 2),
-    !eq(gf,"m8n8:x4") : !listsplat(llvm_i32_ty, 4),
-
-    // ldmatrix b8, b8x16.b6x16_p32, b8x16.b4x16_p64 -> s32 @ m16n16
-    !eq(gf,"m16n16:x1") : !listsplat(llvm_i32_ty, 2),
-    !eq(gf,"m16n16:x2") : !listsplat(llvm_i32_ty, 4),
-
-    // ldmatrix b8x16.b6x16_p32, b8x16.b4x16_p64 -> s32 @ m8n16
-    !eq(gf,"m8n16:x1") : !listsplat(llvm_i32_ty, 1),
-    !eq(gf,"m8n16:x2") : !listsplat(llvm_i32_ty, 2),
-    !eq(gf,"m8n16:x4") : !listsplat(llvm_i32_ty, 4),
-
-    // stmatrix b8 -> s32 @ m16n8
-    !eq(gf,"m16n8:x1") : !listsplat(llvm_i32_ty, 1),
-    !eq(gf,"m16n8:x2") : !listsplat(llvm_i32_ty, 2),
-    !eq(gf,"m16n8:x4") : !listsplat(llvm_i32_ty, 4),
-
+  list<LLVMType> regs = !if(!eq(IsSparse, true),
+    !cond(
+      // mma sparse ops use other fragments for some arguments
+      !eq(gft,"m16n8k16:a:bf16") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k16:a:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k16:b:bf16") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k16:b:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k16:c:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k16:c:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k16:d:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k16:d:f32") : !listsplat(llvm_float_ty, 4),
+
+      !eq(gft,"m16n8k32:a:bf16") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k32:a:f16") : !listsplat(llvm_v2f16_ty, 4),
+      !eq(gft,"m16n8k32:b:bf16") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k32:b:f16") : !listsplat(llvm_v2f16_ty, 4),
+      !eq(gft,"m16n8k32:c:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k32:c:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k32:d:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k32:d:f32") : !listsplat(llvm_float_ty, 4),
+
+      !eq(gft,"m16n8k16:a:tf32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k16:b:tf32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k16:c:tf32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k16:d:tf32") : !listsplat(llvm_float_ty, 4),
+
+      !eq(gft,"m16n8k8:a:tf32") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k8:b:tf32") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k8:c:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k8:d:f32") : !listsplat(llvm_float_ty, 4),
+
+      !eq(gft,"m16n8k32:a:u8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k32:a:s8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k32:b:u8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k32:b:s8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k32:c:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k32:d:s32") : !listsplat(llvm_i32_ty, 4),
+
+      !eq(gft,"m16n8k64:a:u8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:a:s8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:a:e4m3") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:a:e5m2") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:a:e3m2") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:a:e2m3") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:a:e2m1") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:b:u8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:b:s8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:b:e4m3") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:b:e5m2") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:b:e3m2") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:b:e2m3") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:b:e2m1") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:c:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k64:c:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k64:d:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k64:d:f32") : !listsplat(llvm_float_ty, 4),
+
+      !eq(gft,"m16n8k64:a:u4") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k64:a:s4") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k64:b:u4") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k64:b:s4") : !listsplat(llvm_i32_ty, 2),
+
+      !eq(gft,"m16n8k64:c:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:d:s32") : !listsplat(llvm_i32_ty, 4),
+
+      !eq(gft,"m16n8k128:a:u4") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k128:a:s4") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k128:a:e2m1") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k128:b:u4") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k128:b:s4") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k128:b:e2m1") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k128:c:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k128:c:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k128:d:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k128:d:f32") : !listsplat(llvm_float_ty, 4),
+    ),
+    !cond(
+      // mma fp ops use smaller fragments than wmma fp ops
+      !eq(gft,"m8n8k4:a:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m8n8k4:b:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k8:a:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k8:b:f16") : [llvm_v2f16_ty],
+      !eq(gft,"m16n8k8:c:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k8:d:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k8:c:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k8:d:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k16:a:f16") : !listsplat(llvm_v2f16_ty, 4),
+      !eq(gft,"m16n8k16:b:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k16:c:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k16:d:f16") : !listsplat(llvm_v2f16_ty, 2),
+      !eq(gft,"m16n8k16:c:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k16:d:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k4:c:f32") : !listsplat(llvm_float_ty, 4),
+      !eq(gft,"m16n8k4:d:f32") : !listsplat(llvm_float_ty, 4),
+
+      // wmma fp16 -> fp16/fp32 @  m16n16k16/m8n32k16/m32n8k16
+      // All other supported geometries use the same fragment format for f32 and
+      // f16, so we only need to consider {fragment, type}.
+      !eq(ft,"a:f16") : !listsplat(llvm_v2f16_ty, 8),
+      !eq(ft,"b:f16") : !listsplat(llvm_v2f16_ty, 8),
+      !eq(ft,"c:f16") : !listsplat(llvm_v2f16_ty, 4),
+      !eq(ft,"d:f16") : !listsplat(llvm_v2f16_ty, 4),
+      !eq(ft,"c:f32") : !listsplat(llvm_float_ty, 8),
+      !eq(ft,"d:f32") : !listsplat(llvm_float_ty, 8),
+
+      // wmma tf32 -> s32 @ m16n16k8
+      !eq(gft,"m16n16k8:a:tf32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n16k8:b:tf32") : !listsplat(llvm_i32_ty, 4),
+
+      // mma tf32 -> s32 @ m16n16k8/m16n8k8
+      !eq(gft,"m16n8k4:a:tf32") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k4:b:tf32") : [llvm_i32_ty],
+      !eq(gft,"m16n8k8:a:tf32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k8:b:tf32") : !listsplat(llvm_i32_ty, 2),
+
+      !eq(gft,"m8n8k4:a:f64") : [llvm_double_ty],
+      !eq(gft,"m8n8k4:b:f64") : [llvm_double_ty],
+      !eq(gft,"m8n8k4:c:f64") : !listsplat(llvm_double_ty, 2),
+      !eq(gft,"m8n8k4:d:f64") : !listsplat(llvm_double_ty, 2),
+
+      // wmma bf16 -> s32 @ m16n16k16/m8n32k16/m32n8k16
+      !eq(gft,"m16n16k16:a:bf16") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n16k16:b:bf16") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m8n32k16:a:bf16") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m8n32k16:b:bf16") : !listsplat(llvm_i32_ty, 8),
+      !eq(gft,"m32n8k16:a:bf16") : !listsplat(llvm_i32_ty, 8),
+      !eq(gft,"m32n8k16:b:bf16") : !listsplat(llvm_i32_ty, 2),
+
+      // mma bf16 -> s32 @ m16n8k16/m16n8k8
+      !eq(gft,"m16n8k16:a:bf16") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k16:b:bf16") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k8:a:bf16") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k8:b:bf16") : [llvm_i32_ty],
+
+      // wmma u8/s8 -> s32 @ m16n16k16/m8n32k16/m32n8k16
+      !eq(gft,"m16n16k16:a:u8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n16k16:a:s8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n16k16:b:u8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n16k16:b:s8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n16k16:c:s32") : !listsplat(llvm_i32_ty, 8),
+      !eq(gft,"m16n16k16:d:s32") : !listsplat(llvm_i32_ty, 8),
+
+      !eq(gft,"m8n32k16:a:u8") : [llvm_i32_ty],
+      !eq(gft,"m8n32k16:a:s8") : [llvm_i32_ty],
+      !eq(gft,"m8n32k16:b:u8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m8n32k16:b:s8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m8n32k16:c:s32") : !listsplat(llvm_i32_ty, 8),
+      !eq(gft,"m8n32k16:d:s32") : !listsplat(llvm_i32_ty, 8),
+
+      !eq(gft,"m32n8k16:a:u8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m32n8k16:a:s8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m32n8k16:b:u8") : [llvm_i32_ty],
+      !eq(gft,"m32n8k16:b:s8") : [llvm_i32_ty],
+      !eq(gft,"m32n8k16:c:s32") : !listsplat(llvm_i32_ty, 8),
+      !eq(gft,"m32n8k16:d:s32") : !listsplat(llvm_i32_ty, 8),
+
+      // mma u8/s8 -> s32 @ m8n8k16/m16n8k16/m16n8k32
+      !eq(gft,"m8n8k16:a:u8") : [llvm_i32_ty],
+      !eq(gft,"m8n8k16:a:s8") : [llvm_i32_ty],
+      !eq(gft,"m8n8k16:b:u8") : [llvm_i32_ty],
+      !eq(gft,"m8n8k16:b:s8") : [llvm_i32_ty],
+      !eq(gft,"m8n8k16:c:s32") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m8n8k16:d:s32") : !listsplat(llvm_i32_ty, 2),
+
+      !eq(gft,"m16n8k16:a:u8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k16:a:s8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k16:b:u8") : [llvm_i32_ty],
+      !eq(gft,"m16n8k16:b:s8") : [llvm_i32_ty],
+      !eq(gft,"m16n8k16:c:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k16:d:s32") : !listsplat(llvm_i32_ty, 4),
+
+      !eq(gft,"m16n8k32:a:u8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k32:a:s8") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k32:b:u8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k32:b:s8") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k32:c:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k32:d:s32") : !listsplat(llvm_i32_ty, 4),
+
+      // wmma/mma u4/s4 -> s32 @ m8n8k32 (u4/s4)
+      !eq(gft,"m8n8k32:a:u4") : [llvm_i32_ty],
+      !eq(gft,"m8n8k32:a:s4") : [llvm_i32_ty],
+      !eq(gft,"m8n8k32:b:u4") : [llvm_i32_ty],
+      !eq(gft,"m8n8k32:b:s4") : [llvm_i32_ty],
+      !eq(gft,"m8n8k32:c:s32") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m8n8k32:d:s32") : !listsplat(llvm_i32_ty, 2),
+
+      !eq(gft,"m16n8k32:a:u4") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k32:a:s4") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k32:b:u4") : [llvm_i32_ty],
+      !eq(gft,"m16n8k32:b:s4") : [llvm_i32_ty],
+      !eq(gft,"m16n8k32:c:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k32:d:s32") : !listsplat(llvm_i32_ty, 4),
+
+      !eq(gft,"m16n8k64:a:u4") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:a:s4") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:b:u4") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k64:b:s4") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k64:c:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k64:d:s32") : !listsplat(llvm_i32_ty, 4),
+
+      // wmma/mma b1 -> s32 @ m8n8k128(b1)
+      !eq(gft,"m8n8k128:a:b1") : [llvm_i32_ty],
+      !eq(gft,"m8n8k128:b:b1") : [llvm_i32_ty],
+      !eq(gft,"m8n8k128:c:s32") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m8n8k128:d:s32") : !listsplat(llvm_i32_ty, 2),
+
+      !eq(gft,"m16n8k128:a:b1") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k128:b:b1") : [llvm_i32_ty],
+      !eq(gft,"m16n8k128:c:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k128:d:s32") : !listsplat(llvm_i32_ty, 4),
+
+      !eq(gft,"m16n8k256:a:b1") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k256:b:b1") : !listsplat(llvm_i32_ty, 2),
+      !eq(gft,"m16n8k256:c:s32") : !listsplat(llvm_i32_ty, 4),
+      !eq(gft,"m16n8k256:d:s32") : !listsplat(llvm_i32_ty, 4),
+
+      // ldmatrix b16 -> s32 @ m8n8
+      !eq(gf,"m8n8:x1") : !listsplat(llvm_i32_ty, 1),
+      !eq(gf,"m8n8:x2") : !listsplat(llvm_i32_ty, 2),
+      !eq(gf,"m8n8:x4") : !listsplat(llvm_i32_ty, 4),
+
+      // ldmatrix b8, b8x16.b6x16_p32, b8x16.b4x16_p64 -> s32 @ m16n16
+      !eq(gf,"m16n16:x1") : !listsplat(llvm_i32_ty, 2),
+      !eq(gf,"m16n16:x2") : !listsplat(llvm_i32_ty, 4),
+
+      // ldmatrix b8x16.b6x16_p32, b8x16.b4x16_p64 -> s32 @ m8n16
+      !eq(gf,"m8n16:x1") : !listsplat(llvm_i32_ty, 1),
+      !eq(gf,"m8n16:x2") : !listsplat(llvm_i32_ty, 2),
+      !eq(gf,"m8n16:x4") : !listsplat(llvm_i32_ty, 4),
+
+      // stmatrix b8 -> s32 @ m16n8
+      !eq(gf,"m16n8:x1") : !listsplat(llvm_i32_ty, 1),
+      !eq(gf,"m16n8:x2") : !listsplat(llvm_i32_ty, 2),
+      !eq(gf,"m16n8:x4") : !listsplat(llvm_i32_ty, 4),
+    )
   );
 }
 
@@ -362,6 +439,12 @@ class WMMA_NAME_LDST<string Op, WMMA_REGS Frag, string Layout, int WithStride> {
 
 class MMA_SIGNATURE<WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D> {
   list<WMMA_REGS> id_frags = !cond(
+     // FP8/F8F6F4 ops are identified by A,B inputs & accomulator & result type.
+     !or(!eq(A.ptx_elt_type, "e4m3"),
+         !eq(A.ptx_elt_type, "e5m2"),
+         !eq(A.ptx_elt_type, "e3m2"),
+         !eq(A.ptx_elt_type, "e2m3"),
+         !eq(A.ptx_elt_type, "e2m1")): [D, A, B, C],
      // FP16 ops are identified by accumulator & result type.
      !eq(A.ptx_elt_type, "f16") : [D, C],
      // other ops are identified by input types.
@@ -397,6 +480,19 @@ class MMA_NAME<string ALayout, string BLayout, int Satfinite, string b1op,
                   # signature;
 }
 
+class MMA_SP_NAME<string Metadata, string Kind, int Satfinite,
+                  WMMA_REGS A, WMMA_REGS B,
+                  WMMA_REGS C, WMMA_REGS D> {
+  string signature = MMA_SIGNATURE<A, B, C, D>.ret;
+  string record = "int_nvvm_mma"
+                  # "_" # !subst("::", "_", Metadata)
+                  # "_" # A.geom
+                  # "_row_col"
+                  # !if(!ne(Kind, ""), !strconcat("_", !subst("::", "_", Kind)), "")
+                  # !if(Satfinite, "_satfinite", "")
+                  # signature;
+}
+
 class LDMATRIX_NAME<WMMA_REGS Frag, int Trans> {
   string intr = "llvm.nvvm.ldmatrix.sync.aligned"
                 # "." # Frag.geom
@@ -424,21 +520,22 @@ class STMATRIX_NAME<WMMA_REGS Frag, int Trans> {
 //   TypeN: PTX type of the corresponding fragment's element.
 //   TypeB and TypeD may be empty if it must match that of TypeA or TypeC.
 class MMA_OPS<list<string> Geom, list<string> TypeA, list<string> TypeB,
-            list<string> TypeC, list<string> TypeD> {
+            list<string> TypeC, list<string> TypeD, bit IsSparse = false> {
   list<list<WMMA_REGS>> ret =
      !foldl([]<list<WMMA_REGS>>, Geom, t1, geom, !listconcat(t1,
      !foldl([]<list<WMMA_REGS>>, TypeA, t2, type_a, !listconcat(t2,
      !foldl([]<list<WMMA_REGS>>, !if(!size(TypeB), TypeB, [type_a]), t3, type_b, !listconcat(t3,
      !foldl([]<list<WMMA_REGS>>, TypeC, t4, type_c, !listconcat(t4,
      !foldl([]<list<WMMA_REGS>>, !if(!size(TypeD), TypeD, [type_c]), t5, type_d, !listconcat(t5,
-            [[WMMA_REGS<geom, "a", type_a>,
-              WMMA_REGS<geom, "b", type_b>,
-              WMMA_REGS<geom, "c", type_c>,
-              WMMA_REGS<geom, "d", type_d>]]))))))))));
+            [[WMMA_REGS<geom, "a", type_a, IsSparse>,
+              WMMA_REGS<geom, "b", type_b, IsSparse>,
+              WMMA_REGS<geom, "c", type_c, IsSparse>,
+              WMMA_REGS<geom, "d", type_d, IsSparse>]]))))))))));
    // Debugging aid for readable representation of the list above.
    list<list<string>> ops = !foreach(x, ret, [x[0].gft, x[1].gft, x[2].gft, x[3].gft]);
 }
 
+
 class MMA_LDST_OPS<list<string> Geom, list<string> Frags, list<string> Types> {
   list<WMMA_REGS> ret =
      !foldl([]<WMMA_REGS>, Geom, t1, geom, !listconcat(t1,
@@ -522,6 +619,30 @@ class NVVM_MMA_OPS {
             tf32_mma_ops, bf16_mma_ops, f64_mma_ops,
             fp_mma_ops, int_mma_ops, subint_mma_ops, bit_mma_ops);
 
+  list<list<WMMA_REGS>> bf16_mma_sp_ops = MMA_OPS<
+            ["m16n8k16", "m16n8k32"],
+            ["bf16"], [], ["f32"], [], true>.ret;
+  list<list<WMMA_REGS>> tf32_mma_sp_ops = MMA_OPS<
+            ["m16n8k8", "m16n8k16"],
+            ["tf32"], [], ["f32"], [], true>.ret;
+  list<list<WMMA_REGS>> fp_mma_sp_ops = MMA_OPS<
+            ["m16n8k16", "m16n8k32"],
+            ["f16"], [], ["f16", "f32"], ["f16", "f32"], true>.ret;
+  list<list<WMMA_REGS>> fp8_mma_sp_ops = MMA_OPS<
+            ["m16n8k64"],
+            ["e4m3", "e5m2", "e3m2", "e2m3", "e2m1"],
+            ["e4m3", "e5m2", "e3m2", "e2m3", "e2m1"],
+            ["f16", "f32"], ["f16", "f32"], true>.ret;
+  list<list<WMMA_REGS>> subint_mma_sp_ops = MMA_OPS<
+            ["m16n8k64", "m16n8k128"],
+            ["s4", "u4"], ["s4", "u4"], ["s32"], [], true>.ret;
+  list<list<WMMA_REGS>> int_mma_sp_ops = MMA_OPS<
+            ["m16n8k32", "m16n8k64"],
+            ["s8", "u8"], ["s8", "u8"], ["s32"], [], true>.ret;
+  list<list<WMMA_REGS>> all_mma_sp_ops = !listconcat(
+            bf16_mma_sp_ops, tf32_mma_sp_ops, fp_mma_sp_ops, fp8_mma_sp_ops,
+            subint_mma_sp_ops, int_mma_sp_ops);
+
   list<WMMA_REGS> ldst_ab_ops = MMA_LDST_OPS<
             ["m16n16k16", "m32n8k16", "m8n32k16"],
             ["a", "b"], ["f16", "u8", "s8", "bf16"]>.ret;
@@ -728,6 +849,68 @@ class NVVM_STMATRIX_SUPPORTED<WMMA_REGS frag, bit trans> {
   );
 }
 
+
+// Returns true if this combination of layout/kind/satf for MMA.SP ops is supported;
+// false otherwise.
+// E.g.
+// if NVVM_MMA_SP_SUPPORTED<...>.ret then
+//   def : FOO<>; // The record will only be defined for supported ops.
+//
+class NVVM_MMA_SP_SUPPORTED<list<WMMA_REGS> frags, string metadata,
+                            string kind, int satf> {
+  // MMA.SP ops check both layouts.
+  string a_type = frags[0].ptx_elt_type;
+  string b_type = frags[1].ptx_elt_type;
+  string c_type = frags[2].ptx_elt_type;
+  string d_type = frags[3].ptx_elt_type;
+  string geom = frags[0].geom;
+
+  bit is_int = !or(!eq(a_type, "s8"),
+                   !eq(a_type, "u8"),
+                   !eq(a_type, "s4"),
+                   !eq(a_type, "u4"));
+
+  bit ret = !cond(
+
+    // Limit satf to valid types
+    !and(!eq(satf, 1),
+         !eq(is_int, 0)): false,
+
+    // f16/bf16/tf32 requires A and B to be the same type.
+    !and(!or(!eq(a_type, "f16"),
+             !eq(a_type, "bf16"),
+             !eq(a_type, "tf32")),
+         !ne(a_type, b_type)): false,
+
+    // m16n8k16 and m16n8k32 requires C and D to be the same type.
+    !and(!or(!eq(geom, "m16n8k16"),
+             !eq(geom, "m16n8k32")),
+         !ne(c_type, d_type)): false,
+
+    !and(!eq(kind, ""),
+         !or(!eq(a_type, "e3m2"),
+             !eq(a_type, "e2m3"),
+             !eq(a_type, "e2m1"),
+             !eq(b_type, "e3m2"),
+             !eq(b_type, "e2m3"),
+             !eq(b_type, "e2m1"))): false,
+
+    !and(!eq(kind, ""),
+         !eq(geom, "m16n8k64"),
+         !or(!eq(c_type, "f16"),
+             !eq(d_type, "f16"))): false,
+
+    !and(!ne(kind, ""),
+         !or(!eq(metadata, "sp"),
+             !ne(geom, "m16n8k64"),
+             !eq(is_int, 1))): false,
+
+    // All other are OK.
+    true: true
+  );
+}
+
+
 class SHFL_INFO<bit sync, string mode, string type, bit return_pred> {
   string Suffix = !if(sync, "sync_", "")
                   # mode # "_"
@@ -792,38 +975,49 @@ class NVVMBuiltin :
            "NVVMBuiltin must be a NVVM intrinsic starting with 'int_nvvm_'";
 }
 
+class PureIntrinsic<list<LLVMType> ret_types,
+                    list<LLVMType> param_types = [],
+                    list<IntrinsicProperty> intr_properties = [],
+                    string name = ""> :
+  DefaultAttrsIntrinsic<ret_types, param_types,
+                        intr_properties # [IntrNoMem, IntrSpeculatable], name> {}
+
 let TargetPrefix = "nvvm" in {
 
+  //
   // PRMT - permute
-
-  let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
-    def int_nvvm_prmt : NVVMBuiltin,
-      DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>;
-
-    foreach mode = ["f4e", "b4e"] in
-      def int_nvvm_prmt_ # mode :
-          DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>;
-
-    // Note: these variants also have 2 source operands but only one will ever
-    // be used so we eliminate the other operand in the IR (0 is used as the
-    // placeholder in the backend).
-    foreach mode = ["rc8", "ecl", "ecr", "rc16"] in
-      def int_nvvm_prmt_ # mode :
-          DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty]>;
-  }
-
+  //
+  def int_nvvm_prmt : NVVMBuiltin,
+    PureIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>;
+
+  foreach mode = ["f4e", "b4e"] in
+    def int_nvvm_prmt_ # mode :
+        PureIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>;
+
+  // Note: these variants also have 2 source operands but only one will ever
+  // be used so we eliminate the other operand in the IR (0 is used as the
+  // placeholder in the backend).
+  foreach mode = ["rc8", "ecl", "ecr", "rc16"] in
+    def int_nvvm_prmt_ # mode :
+        PureIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty]>;
+
+  //
+  // Nanosleep
+  //
   def int_nvvm_nanosleep : NVVMBuiltin,
       DefaultAttrsIntrinsic<[], [llvm_i32_ty],
                             [IntrConvergent, IntrNoMem, IntrHasSideEffects]>;
 
+  //
   // Performance Monitor Events (pm events) intrinsics
+  //
   def int_nvvm_pm_event_mask : NVVMBuiltin,
       DefaultAttrsIntrinsic<[], [llvm_i16_ty],
                 [IntrConvergent, IntrNoMem, IntrHasSideEffects,
                  ImmArg<ArgIndex<0>>]>;
-//
-// Min Max
-//
+  //
+  // Min Max
+  //
   let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative] in {
     foreach operation = ["min", "max"] in {
       def int_nvvm_f # operation # _d : NVVMBuiltin,
@@ -852,9 +1046,9 @@ let TargetPrefix = "nvvm" in {
     } // operation
   }
 
-//
-// Multiplication
-//
+  //
+  // Multiplication
+  //
   let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative] in {
     foreach sign = ["", "u"] in {
       def int_nvvm_mulhi_ # sign # s : NVVMBuiltin,
@@ -880,9 +1074,9 @@ let TargetPrefix = "nvvm" in {
     }
   }
 
-//
-// Div
-//
+  //
+  // Div
+  //
   let IntrProperties = [IntrNoMem] in {
     foreach ftz = ["", "_ftz"] in {
       def int_nvvm_div_approx # ftz # _f : NVVMBuiltin,
@@ -902,90 +1096,79 @@ let TargetPrefix = "nvvm" in {
     }
   }
 
-//
-// Sad
-//
-  let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
-    foreach sign = ["", "u"] in {
-      def int_nvvm_sad_ # sign # s : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty]>;
+  //
+  // Sad - Sum of Absolute Differences
+  //
+  foreach sign = ["", "u"] in {
+    def int_nvvm_sad_ # sign # s : NVVMBuiltin,
+        PureIntrinsic<[llvm_i16_ty], [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty]>;
 
-      def int_nvvm_sad_ # sign # i : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>;
+    def int_nvvm_sad_ # sign # i : NVVMBuiltin,
+        PureIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>;
 
-      def int_nvvm_sad_ # sign # ll : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty]>;
-    }
+    def int_nvvm_sad_ # sign # ll : NVVMBuiltin,
+        PureIntrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty, llvm_i64_ty]>;
   }
 
-//
-// Floor  Ceil
-//
-  let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
-    foreach op = ["floor", "ceil"] in {
-      foreach ftz = ["", "_ftz"] in
-        def int_nvvm_ # op # ftz # _f : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty]>;
-      def int_nvvm_ # op # _d : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
-    }
+  //
+  // Floor  Ceil
+  //
+  foreach op = ["floor", "ceil"] in {
+    foreach ftz = ["", "_ftz"] in
+      def int_nvvm_ # op # ftz # _f : NVVMBuiltin,
+          PureIntrinsic<[llvm_float_ty], [llvm_float_ty]>;
+    def int_nvvm_ # op # _d : NVVMBuiltin,
+        PureIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
   }
 
-//
-// Abs
-//
+  //
+  // Abs
+  //
   foreach ftz = ["", "_ftz"] in
     def int_nvvm_fabs # ftz :
-      DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>],
-                            [IntrNoMem, IntrSpeculatable]>;
+      PureIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>;
 
-//
-// Abs, Neg bf16, bf16x2
-//
+  //
+  // Neg bf16, bf16x2
+  //
   def int_nvvm_neg_bf16 : NVVMBuiltin,
-    DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_bfloat_ty], [IntrNoMem]>;
+    PureIntrinsic<[llvm_bfloat_ty], [llvm_bfloat_ty]>;
   def int_nvvm_neg_bf16x2 : NVVMBuiltin,
-    DefaultAttrsIntrinsic<[llvm_v2bf16_ty], [llvm_v2bf16_ty], [IntrNoMem]>;
+    PureIntrinsic<[llvm_v2bf16_ty], [llvm_v2bf16_ty]>;
 
-//
-// Round
-//
-  let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
-    foreach ftz = ["", "_ftz"] in
-      def int_nvvm_round # ftz # _f : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty]>;
+  //
+  // Round
+  //
+  foreach ftz = ["", "_ftz"] in
+    def int_nvvm_round # ftz # _f : NVVMBuiltin,
+        PureIntrinsic<[llvm_float_ty], [llvm_float_ty]>;
 
-    def int_nvvm_round_d : NVVMBuiltin,
-        DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
-  }
+  def int_nvvm_round_d : NVVMBuiltin,
+      PureIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
 
-//
-// Trunc
-//
-  let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
-    foreach ftz = ["", "_ftz"] in
-      def int_nvvm_trunc # ftz # _f : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty]>;
+  //
+  // Trunc
+  //
+  foreach ftz = ["", "_ftz"] in
+    def int_nvvm_trunc # ftz # _f : NVVMBuiltin,
+        PureIntrinsic<[llvm_float_ty], [llvm_float_ty]>;
 
-    def int_nvvm_trunc_d : NVVMBuiltin,
-        DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
-  }
+  def int_nvvm_trunc_d : NVVMBuiltin,
+      PureIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
 
-//
-// Saturate
-//
-  let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
-    foreach ftz = ["", "_ftz"] in
-      def int_nvvm_saturate # ftz # _f : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty]>;
+  //
+  // Saturate
+  //
+  foreach ftz = ["", "_ftz"] in
+    def int_nvvm_saturate # ftz # _f : NVVMBuiltin,
+        PureIntrinsic<[llvm_float_ty], [llvm_float_ty]>;
 
-    def int_nvvm_saturate_d : NVVMBuiltin,
-        DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
-  }
+  def int_nvvm_saturate_d : NVVMBuiltin,
+      PureIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
 
-//
-// Exp2  Log2
-//
+  //
+  // Exp2  Log2
+  //
   let IntrProperties = [IntrNoMem] in {
     foreach ftz = ["", "_ftz"] in
       def int_nvvm_ex2_approx # ftz # _f : NVVMBuiltin,
@@ -1006,53 +1189,51 @@ let TargetPrefix = "nvvm" in {
         DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
   }
 
-//
-// Sin  Cos
-//
+  //
+  // Sin  Cos
+  //
   foreach op = ["sin", "cos"] in
     foreach ftz = ["", "_ftz"] in
       def int_nvvm_ # op # _approx # ftz # _f : NVVMBuiltin,
           DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty], [IntrNoMem]>;
 
-//
-// Fma
-//
-  let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
-    foreach variant = ["", "_sat", "_relu"] in {
-      foreach ftz = ["", "_ftz"] in {
-        def int_nvvm_fma_rn # ftz # variant # _f16 :
-          DefaultAttrsIntrinsic<[llvm_half_ty],
-            [llvm_half_ty, llvm_half_ty, llvm_half_ty]>;
-
-        def int_nvvm_fma_rn # ftz # variant # _f16x2 :
-          DefaultAttrsIntrinsic<[llvm_v2f16_ty],
-            [llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty]>;
-
-        def int_nvvm_fma_rn # ftz # variant # _bf16 : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_bfloat_ty],
-            [llvm_bfloat_ty, llvm_bfloat_ty, llvm_bfloat_ty]>;
-
-        def int_nvvm_fma_rn # ftz # variant # _bf16x2 : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_v2bf16_ty],
-            [llvm_v2bf16_ty, llvm_v2bf16_ty, llvm_v2bf16_ty]>;
-      } // ftz
-    } // variant
+  //
+  // Fma
+  //
+  foreach variant = ["", "_sat", "_relu"] in {
+    foreach ftz = ["", "_ftz"] in {
+      def int_nvvm_fma_rn # ftz # variant # _f16 :
+        PureIntrinsic<[llvm_half_ty],
+          [llvm_half_ty, llvm_half_ty, llvm_half_ty]>;
 
-    foreach rnd = ["rn", "rz", "rm", "rp"] in {
-      foreach ftz = ["", "_ftz"] in
-        def int_nvvm_fma_ # rnd # ftz # _f : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_float_ty],
-            [llvm_float_ty, llvm_float_ty, llvm_float_ty]>;
+      def int_nvvm_fma_rn # ftz # variant # _f16x2 :
+        PureIntrinsic<[llvm_v2f16_ty],
+          [llvm_v2f16_ty, llvm_v2f16_ty, llvm_v2f16_ty]>;
 
-      def int_nvvm_fma_ # rnd # _d : NVVMBuiltin,
-        DefaultAttrsIntrinsic<[llvm_double_ty],
-          [llvm_double_ty, llvm_double_ty, llvm_double_ty]>;
-    }
+      def int_nvvm_fma_rn # ftz # variant # _bf16 : NVVMBuiltin,
+        PureIntrinsic<[llvm_bfloat_ty],
+          [llvm_bfloat_ty, llvm_bfloat_ty, llvm_bfloat_ty]>;
+
+      def int_nvvm_fma_rn # ftz # variant # _bf16x2 : NVVMBuiltin,
+        PureIntrinsic<[llvm_v2bf16_ty],
+          [llvm_v2bf16_ty, llvm_v2bf16_ty, llvm_v2bf16_ty]>;
+    } // ftz
+  } // variant
+
+  foreach rnd = ["rn", "rz", "rm", "rp"] in {
+    foreach ftz = ["", "_ftz"] in
+      def int_nvvm_fma_ # rnd # ftz # _f : NVVMBuiltin,
+        PureIntrinsic<[llvm_float_ty],
+          [llvm_float_ty, llvm_float_ty, llvm_float_ty]>;
+
+    def int_nvvm_fma_ # rnd # _d : NVVMBuiltin,
+      PureIntrinsic<[llvm_double_ty],
+        [llvm_double_ty, llvm_double_ty, llvm_double_ty]>;
   }
 
-//
-// Rcp
-//
+  //
+  // Rcp
+  //
   let IntrProperties = [IntrNoMem] in {
     foreach rnd = ["rn", "rz", "rm", "rp"] in {
       foreach ftz = ["", "_ftz"] in
@@ -1069,9 +1250,9 @@ let TargetPrefix = "nvvm" in {
         DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty]>;
   }
 
-//
-// Sqrt
-//
+  //
+  // Sqrt
+  //
   let IntrProperties = [IntrNoMem] in {
     foreach rnd = ["rn", "rz", "rm", "rp"] in {
       foreach ftz = ["", "_ftz"] in
@@ -1090,9 +1271,9 @@ let TargetPrefix = "nvvm" in {
           DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty]>;
   }
 
-//
-// Rsqrt
-//
+  //
+  // Rsqrt
+  //
   let IntrProperties = [IntrNoMem] in {
     foreach ftz = ["", "_ftz"] in {
       def int_nvvm_rsqrt_approx # ftz # _f : NVVMBuiltin,
@@ -1102,208 +1283,206 @@ let TargetPrefix = "nvvm" in {
     }
   }
 
-//
-// Add
-//
+  //
+  // Add
+  //
   let IntrProperties = [IntrNoMem, IntrSpeculatable, Commutative] in {
     foreach rnd = ["rn", "rz", "rm", "rp"] in {
       foreach ftz = ["", "_ftz"] in
         def int_nvvm_add_ # rnd # ftz # _f : NVVMBuiltin,
           DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty]>;
 
-    def int_nvvm_add_ # rnd # _d : NVVMBuiltin,
-        DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty]>;
+      def int_nvvm_add_ # rnd # _d : NVVMBuiltin,
+          DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty]>;
     }
   }
 
-//
-// Dot Product
-//
+  //
+  // Dot Product
+  //
   foreach a_type = ["s", "u"] in {
     foreach b_type = ["s", "u"] in {
       def int_nvvm_idp4a_ # a_type # _ # b_type :
-          DefaultAttrsIntrinsic<[llvm_i32_ty],
-              [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-              [IntrNoMem, IntrSpeculatable]>;
+          PureIntrinsic<[llvm_i32_ty],
+              [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>;
       def int_nvvm_idp2a_ # a_type # _ # b_type :
-          DefaultAttrsIntrinsic<[llvm_i32_ty],
+          PureIntrinsic<[llvm_i32_ty],
             [llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i32_ty],
-            [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<2>>]>;
+            [ImmArg<ArgIndex<2>>]>;
     }
   }
 
-//
-// Funnel-shift
-//
+  //
+  // Funnel-shift
+  //
   foreach direction = ["l", "r"] in
     def int_nvvm_fsh # direction # _clamp :
-      DefaultAttrsIntrinsic<[llvm_anyint_ty],
-        [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
-        [IntrNoMem, IntrSpeculatable]>;
+      PureIntrinsic<[llvm_anyint_ty],
+                    [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
 
-//
-// FLO - Find Leading One
-//
+  //
+  // FLO - Find Leading One
+  //
   foreach sign = ["s", "u"] in
     def int_nvvm_flo_ # sign :
-      DefaultAttrsIntrinsic<[llvm_i32_ty],
-        [llvm_anyint_ty, llvm_i1_ty],
-        [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<1>>]>;
+      PureIntrinsic<[llvm_i32_ty], [llvm_anyint_ty, llvm_i1_ty],
+                    [ImmArg<ArgIndex<1>>]>;
 
-//
-// szext
-//
+  //
+  // szext
+  //
   foreach ext = ["sext", "zext"] in
     foreach mode = ["wrap", "clamp"] in
       def int_nvvm_ # ext # _ # mode :
-        DefaultAttrsIntrinsic<[llvm_i32_ty],
-          [llvm_i32_ty, llvm_i32_ty],
-          [IntrNoMem, IntrSpeculatable]>;
+        PureIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty]>;
 
-//
-// BMSK - bit mask
-//
+  //
+  // BMSK - bit mask
+  //
   foreach mode = ["wrap", "clamp"] in
     def int_nvvm_bmsk_ # mode :
-      DefaultAttrsIntrinsic<[llvm_i32_ty],
-        [llvm_i32_ty, llvm_i32_ty],
-        [IntrNoMem, IntrSpeculatable]>;
+      PureIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty]>;
 
-//
-// Convert
-//
-  let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
-    def int_nvvm_lohi_i2d : NVVMBuiltin,
-        DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty, llvm_i32_ty]>;
-
-    def int_nvvm_d2i_lo : NVVMBuiltin,
-        DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty]>;
-    def int_nvvm_d2i_hi : NVVMBuiltin,
-        DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty]>;
-
-    foreach rnd = ["rn", "rz", "rm", "rp"] in {
-      foreach ftz = ["", "_ftz"] in
-        def int_nvvm_d2f_ # rnd # ftz : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty]>;
+  //
+  // FNS - Find the n-th set bit
+  //
+  def int_nvvm_fns : NVVMBuiltin,
+      PureIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>;
+
+  //
+  // Convert
+  //
+  // TODO: All these intrinsics are defined as PureIntrinsic, this attaches the
+  //       IntrSpeculatable property to them. Consider if some of these should
+  //       have this attribute removed as they may be too expensive.
+  //
+  def int_nvvm_lohi_i2d : NVVMBuiltin,
+      PureIntrinsic<[llvm_double_ty], [llvm_i32_ty, llvm_i32_ty]>;
+
+  def int_nvvm_d2i_lo : NVVMBuiltin,
+      PureIntrinsic<[llvm_i32_ty], [llvm_double_ty]>;
+  def int_nvvm_d2i_hi : NVVMBuiltin,
+      PureIntrinsic<[llvm_i32_ty], [llvm_double_ty]>;
+
+  foreach rnd = ["rn", "rz", "rm", "rp"] in {
+    foreach ftz = ["", "_ftz"] in
+      def int_nvvm_d2f_ # rnd # ftz : NVVMBuiltin,
+          PureIntrinsic<[llvm_float_ty], [llvm_double_ty]>;
 
-      foreach sign = ["", "u"] in {
+    foreach sign = ["", "u"] in {
 
-        def int_nvvm_d2 # sign # i_ # rnd : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty]>;
+      def int_nvvm_d2 # sign # i_ # rnd : NVVMBuiltin,
+          PureIntrinsic<[llvm_i32_ty], [llvm_double_ty]>;
 
-        def int_nvvm_ # sign # i2d_ # rnd : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i32_ty]>;
+      def int_nvvm_ # sign # i2d_ # rnd : NVVMBuiltin,
+        PureIntrinsic<[llvm_double_ty], [llvm_i32_ty]>;
 
-        foreach ftz = ["", "_ftz"] in
-          def int_nvvm_f2 # sign # i_ # rnd # ftz : NVVMBuiltin,
-              DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty]>;
+      foreach ftz = ["", "_ftz"] in
+        def int_nvvm_f2 # sign # i_ # rnd # ftz : NVVMBuiltin,
+            PureIntrinsic<[llvm_i32_ty], [llvm_float_ty]>;
 
-        def int_nvvm_ # sign # i2f_ # rnd : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i32_ty]>;
+      def int_nvvm_ # sign # i2f_ # rnd : NVVMBuiltin,
+          PureIntrinsic<[llvm_float_ty], [llvm_i32_ty]>;
 
-        foreach ftz = ["", "_ftz"] in
-          def int_nvvm_f2 # sign # ll_ # rnd # ftz : NVVMBuiltin,
-              DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_float_ty]>;
+      foreach ftz = ["", "_ftz"] in
+        def int_nvvm_f2 # sign # ll_ # rnd # ftz : NVVMBuiltin,
+            PureIntrinsic<[llvm_i64_ty], [llvm_float_ty]>;
 
-        def int_nvvm_d2 # sign # ll_ # rnd : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_double_ty]>;
+      def int_nvvm_d2 # sign # ll_ # rnd : NVVMBuiltin,
+        PureIntrinsic<[llvm_i64_ty], [llvm_double_ty]>;
 
-        def int_nvvm_ # sign # ll2f_ # rnd : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_i64_ty]>;
+      def int_nvvm_ # sign # ll2f_ # rnd : NVVMBuiltin,
+          PureIntrinsic<[llvm_float_ty], [llvm_i64_ty]>;
 
-        def int_nvvm_ # sign # ll2d_ # rnd : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_double_ty], [llvm_i64_ty]>;
+      def int_nvvm_ # sign # ll2d_ # rnd : NVVMBuiltin,
+          PureIntrinsic<[llvm_double_ty], [llvm_i64_ty]>;
 
-      } // sign
-    } // rnd
+    } // sign
+  } // rnd
 
-    foreach ftz = ["", "_ftz"] in {
-      def int_nvvm_f2h_rn # ftz : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_float_ty]>;
+  foreach ftz = ["", "_ftz"] in {
+    def int_nvvm_f2h_rn # ftz : NVVMBuiltin,
+        PureIntrinsic<[llvm_i16_ty], [llvm_float_ty]>;
 
-      def int_nvvm_bf2h_rn # ftz : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_bfloat_ty]>;
-    }
+    def int_nvvm_bf2h_rn # ftz : NVVMBuiltin,
+        PureIntrinsic<[llvm_i16_ty], [llvm_bfloat_ty]>;
+  }
 
-    foreach rnd = ["rn", "rz"] in {
-      foreach relu = ["", "_relu"] in {
-        def int_nvvm_ff2bf16x2_ # rnd # relu : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_v2bf16_ty], [llvm_float_ty, llvm_float_ty]>;
+  foreach rnd = ["rn", "rz"] in {
+    foreach relu = ["", "_relu"] in {
+      def int_nvvm_ff2bf16x2_ # rnd # relu : NVVMBuiltin,
+          PureIntrinsic<[llvm_v2bf16_ty], [llvm_float_ty, llvm_float_ty]>;
 
-        def int_nvvm_ff2f16x2_ # rnd # relu : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty]>;
+      def int_nvvm_ff2f16x2_ # rnd # relu : NVVMBuiltin,
+          PureIntrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty]>;
 
-        def int_nvvm_f2bf16_ # rnd # relu : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_float_ty]>;
-      }
+      def int_nvvm_f2bf16_ # rnd # relu : NVVMBuiltin,
+          PureIntrinsic<[llvm_bfloat_ty], [llvm_float_ty]>;
     }
+  }
 
-    foreach satfinite = ["", "_satfinite"] in {
-      def int_nvvm_f2tf32_rna # satfinite : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty]>;
+  foreach satfinite = ["", "_satfinite"] in {
+    def int_nvvm_f2tf32_rna # satfinite : NVVMBuiltin,
+        PureIntrinsic<[llvm_i32_ty], [llvm_float_ty]>;
 
-      foreach rnd = ["rn", "rz"] in
-        foreach relu = ["", "_relu"] in
-          def int_nvvm_f2tf32_ # rnd # relu # satfinite : NVVMBuiltin,
-              DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_float_ty]>;
-    }
+    foreach rnd = ["rn", "rz"] in
+      foreach relu = ["", "_relu"] in
+        def int_nvvm_f2tf32_ # rnd # relu # satfinite : NVVMBuiltin,
+            PureIntrinsic<[llvm_i32_ty], [llvm_float_ty]>;
+  }
 
-    foreach type = ["e4m3x2", "e5m2x2"] in {
-      foreach relu = ["", "_relu"] in {
-        def int_nvvm_ff_to_ # type # _rn # relu : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty]>;
+  foreach type = ["e4m3x2", "e5m2x2"] in {
+    foreach relu = ["", "_relu"] in {
+      def int_nvvm_ff_to_ # type # _rn # relu : NVVMBuiltin,
+          PureIntrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty]>;
 
-        def int_nvvm_f16x2_to_ # type # _rn # relu : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_v2f16_ty]>;
+      def int_nvvm_f16x2_to_ # type # _rn # relu : NVVMBuiltin,
+          PureIntrinsic<[llvm_i16_ty], [llvm_v2f16_ty]>;
 
-        def int_nvvm_ # type # _to_f16x2_rn # relu : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_i16_ty]>;
-      }
+      def int_nvvm_ # type # _to_f16x2_rn # relu : NVVMBuiltin,
+          PureIntrinsic<[llvm_v2f16_ty], [llvm_i16_ty]>;
     }
+  }
 
-    // FP4 conversions.
-    foreach relu = ["", "_relu"] in {
-      def int_nvvm_ff_to_e2m1x2_rn # relu # _satfinite : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty]>;
+  // FP4 conversions.
+  foreach relu = ["", "_relu"] in {
+    def int_nvvm_ff_to_e2m1x2_rn # relu # _satfinite : NVVMBuiltin,
+        PureIntrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty]>;
 
-      def int_nvvm_e2m1x2_to_f16x2_rn # relu : NVVMBuiltin,
-          DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_i16_ty]>;
-    }
+    def int_nvvm_e2m1x2_to_f16x2_rn # relu : NVVMBuiltin,
+        PureIntrinsic<[llvm_v2f16_ty], [llvm_i16_ty]>;
+  }
 
-    // FP6 conversions.
-    foreach type = ["e2m3x2", "e3m2x2"] in {
-      foreach relu = ["", "_relu"] in {
-        def int_nvvm_ff_to_ # type # _rn # relu # _satfinite : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty]>;
+  // FP6 conversions.
+  foreach type = ["e2m3x2", "e3m2x2"] in {
+    foreach relu = ["", "_relu"] in {
+      def int_nvvm_ff_to_ # type # _rn # relu # _satfinite : NVVMBuiltin,
+          PureIntrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty]>;
 
-        def int_nvvm_ # type # _to_f16x2_rn # relu : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_v2f16_ty], [llvm_i16_ty]>;
-      }
+      def int_nvvm_ # type # _to_f16x2_rn # relu : NVVMBuiltin,
+          PureIntrinsic<[llvm_v2f16_ty], [llvm_i16_ty]>;
     }
+  }
 
-    // UE8M0x2 conversions.
-    foreach rmode = ["_rz", "_rp"] in {
-      foreach satmode = ["", "_satfinite"] in {
-        defvar suffix = rmode # satmode;
-        def int_nvvm_ff_to_ue8m0x2 # suffix : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty]>;
+  // UE8M0x2 conversions.
+  foreach rmode = ["_rz", "_rp"] in {
+    foreach satmode = ["", "_satfinite"] in {
+      defvar suffix = rmode # satmode;
+      def int_nvvm_ff_to_ue8m0x2 # suffix : NVVMBuiltin,
+          PureIntrinsic<[llvm_i16_ty], [llvm_float_ty, llvm_float_ty]>;
 
-        def int_nvvm_bf16x2_to_ue8m0x2 # suffix : NVVMBuiltin,
-            DefaultAttrsIntrinsic<[llvm_i16_ty], [llvm_v2bf16_ty]>;
+      def int_nvvm_bf16x2_to_ue8m0x2 # suffix : NVVMBuiltin,
+          PureIntrinsic<[llvm_i16_ty], [llvm_v2bf16_ty]>;
 
-      }
     }
+  }
 
-    def int_nvvm_ue8m0x2_to_bf16x2 : NVVMBuiltin,
-        DefaultAttrsIntrinsic<[llvm_v2bf16_ty], [llvm_i16_ty]>;
-
-  } // IntrProperties = [IntrNoMem, IntrSpeculatable]
-
-// FNS
-  def int_nvvm_fns : NVVMBuiltin,
-      DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
-                [IntrNoMem]>;
+  def int_nvvm_ue8m0x2_to_bf16x2 : NVVMBuiltin,
+      PureIntrinsic<[llvm_v2bf16_ty], [llvm_i16_ty]>;
 
+  //
+  // Atomic operations
+  //
   class SCOPED_ATOMIC2_impl<LLVMType elty>
         : Intrinsic<[elty],
           [llvm_anyptr_ty, LLVMMatchType<0>],
@@ -1336,7 +1515,9 @@ let TargetPrefix = "nvvm" in {
   defm int_nvvm_atomic_and_gen_i  : PTXAtomicWithScope2<llvm_anyint_ty>;
   defm int_nvvm_atomic_cas_gen_i  : PTXAtomicWithScope3<llvm_anyint_ty>;
 
-// Bar.Sync
+  //
+  // Bar.Sync
+  //
   def int_nvvm_barrier0_popc : ClangBuiltin<"__nvvm_bar0_popc">,
       Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent, IntrNoCallback]>;
   def int_nvvm_barrier0_and : ClangBuiltin<"__nvvm_bar0_and">,
@@ -1360,62 +1541,65 @@ let TargetPrefix = "nvvm" in {
     }
   }
 
-  // barrier.cluster.[wait, arrive, arrive.relaxed]
-  def int_nvvm_barrier_cluster_arrive :
-      Intrinsic<[], [], [IntrConvergent, IntrNoCallback]>;
-  def int_nvvm_barrier_cluster_arrive_relaxed :
-      Intrinsic<[], [], [IntrConvergent, IntrNoCallback]>;
-  def int_nvvm_barrier_cluster_wait :
-      Intrinsic<[], [], [IntrConvergent, IntrNoCallback]>;
-
-  // 'aligned' versions of the above barrier.cluster.* intrinsics
-  def int_nvvm_barrier_cluster_arrive_aligned :
-      Intrinsic<[], [], [IntrConvergent, IntrNoCallback]>;
-  def int_nvvm_barrier_cluster_arrive_relaxed_aligned :
-      Intrinsic<[], [], [IntrConvergent, IntrNoCallback]>;
-  def int_nvvm_barrier_cluster_wait_aligned :
-      Intrinsic<[], [], [IntrConvergent, IntrNoCallback]>;
+  let IntrProperties = [IntrConvergent, IntrNoCallback] in {
+    // barrier.cluster.[wait, arrive, arrive.relaxed]
+    def int_nvvm_barrier_cluster_arrive : Intrinsic<[]>;
+    def int_nvvm_barrier_cluster_arrive_relaxed : Intrinsic<[]>;
+    def int_nvvm_barrier_cluster_wait : Intrinsic<[]>;
+
+    // 'aligned' versions of the above barrier.cluster.* intrinsics
+    def int_nvvm_barrier_cluster_arrive_aligned : Intrinsic<[]>;
+    def int_nvvm_barrier_cluster_arrive_relaxed_aligned : Intrinsic<[]>;
+    def int_nvvm_barrier_cluster_wait_aligned : Intrinsic<[]>;
+  }
 
+  //
   // Membar
-  def int_nvvm_membar_cta : NVVMBuiltin, Intrinsic<[], [], [IntrNoCallback]>;
-  def int_nvvm_membar_gl : NVVMBuiltin, Intrinsic<[], [], [IntrNoCallback]>;
-  def int_nvvm_membar_sys : NVVMBuiltin, Intrinsic<[], [], [IntrNoCallback]>;
-  def int_nvvm_fence_sc_cluster : Intrinsic<[], [], [IntrNoCallback]>;
-
-// Proxy fence (uni-directional)
-foreach scope = ["cta", "cluster", "gpu", "sys"] in {
-
-  def int_nvvm_fence_proxy_tensormap_generic_release_ # scope :
-        Intrinsic<[], [], [IntrNoCallback],
-        "llvm.nvvm.fence.proxy.tensormap_generic.release." # scope>;
-
-  // The imm-arg 'size' can only be 128.
-  def int_nvvm_fence_proxy_tensormap_generic_acquire_ # scope :
-        Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty],
-                  [IntrNoCallback, IntrArgMemOnly, ImmArg<ArgIndex<1>>,
-                   Range<ArgIndex<1>, 128, 129>],
-                  "llvm.nvvm.fence.proxy.tensormap_generic.acquire." # scope>;
-}
+  //
+  let IntrProperties = [IntrNoCallback] in {
+    def int_nvvm_membar_cta : NVVMBuiltin, Intrinsic<[]>;
+    def int_nvvm_membar_gl : NVVMBuiltin, Intrinsic<[]>;
+    def int_nvvm_membar_sys : NVVMBuiltin, Intrinsic<[]>;
+    def int_nvvm_fence_sc_cluster : Intrinsic<[]>;
+  }
 
+  //
+  // Proxy fence (uni-directional)
+  //
+  foreach scope = ["cta", "cluster", "gpu", "sys"] in {
+
+    def int_nvvm_fence_proxy_tensormap_generic_release_ # scope :
+          Intrinsic<[], [], [IntrNoCallback],
+          "llvm.nvvm.fence.proxy.tensormap_generic.release." # scope>;
+
+    // The imm-arg 'size' can only be 128.
+    def int_nvvm_fence_proxy_tensormap_generic_acquire_ # scope :
+          Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty],
+                    [IntrNoCallback, IntrArgMemOnly, ImmArg<ArgIndex<1>>,
+                    Range<ArgIndex<1>, 128, 129>],
+                    "llvm.nvvm.fence.proxy.tensormap_generic.acquire." # scope>;
+  }
+
+//
 // Async Copy
+//
 let IntrProperties = [IntrConvergent, IntrNoCallback] in {
   def int_nvvm_cp_async_mbarrier_arrive : NVVMBuiltin,
-      Intrinsic<[],[llvm_ptr_ty]>;
+      Intrinsic<[], [llvm_ptr_ty]>;
   def int_nvvm_cp_async_mbarrier_arrive_shared : NVVMBuiltin,
-      Intrinsic<[],[llvm_shared_ptr_ty]>;
+      Intrinsic<[], [llvm_shared_ptr_ty]>;
   def int_nvvm_cp_async_mbarrier_arrive_noinc : NVVMBuiltin,
-      Intrinsic<[],[llvm_ptr_ty]>;
+      Intrinsic<[], [llvm_ptr_ty]>;
   def int_nvvm_cp_async_mbarrier_arrive_noinc_shared : NVVMBuiltin,
-      Intrinsic<[],[llvm_shared_ptr_ty]>;
+      Intrinsic<[], [llvm_shared_ptr_ty]>;
 }
 
 multiclass CP_ASYNC_SHARED_GLOBAL {
-  def NAME : Intrinsic<[], [llvm_shared_ptr_ty, llvm_global_ptr_ty],
-        [IntrArgMemOnly, IntrNoCallback, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>,
-        WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>]>;
-  def _s : Intrinsic<[], [llvm_shared_ptr_ty, llvm_global_ptr_ty, llvm_i32_ty],
-        [IntrArgMemOnly, IntrNoCallback, NoAlias<ArgIndex<0>>, NoAlias<ArgIndex<1>>,
-        WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>]>;
+  let IntrProperties = [IntrArgMemOnly, IntrNoCallback, NoAlias<ArgIndex<0>>,
+                        NoAlias<ArgIndex<1>>, WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<1>>] in {
+    def NAME : Intrinsic<[], [llvm_shared_ptr_ty, llvm_global_ptr_ty]>;
+    def _s : Intrinsic<[], [llvm_shared_ptr_ty, llvm_global_ptr_ty, llvm_i32_ty]>;
+  }
 }
 
 defm int_nvvm_cp_async_ca_shared_global_4 : CP_ASYNC_SHARED_GLOBAL;
@@ -1423,17 +1607,15 @@ defm int_nvvm_cp_async_ca_shared_global_8 : CP_ASYNC_SHARED_GLOBAL;
 defm int_nvvm_cp_async_ca_shared_global_16 : CP_ASYNC_SHARED_GLOBAL;
 defm int_nvvm_cp_async_cg_shared_global_16 : CP_ASYNC_SHARED_GLOBAL;
 
-def int_nvvm_cp_async_commit_group : NVVMBuiltin, Intrinsic<[], [], []>;
+def int_nvvm_cp_async_commit_group : NVVMBuiltin, Intrinsic<[]>;
 
 def int_nvvm_cp_async_wait_group : NVVMBuiltin,
     Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>;
 
-def int_nvvm_cp_async_wait_all : NVVMBuiltin,
-    Intrinsic<[], [], []>;
+def int_nvvm_cp_async_wait_all : NVVMBuiltin, Intrinsic<[]>;
 
 // cp.async.bulk variants of the commit/wait group
-def int_nvvm_cp_async_bulk_commit_group :
-    Intrinsic<[], [], []>;
+def int_nvvm_cp_async_bulk_commit_group : Intrinsic<[]>;
 
 def int_nvvm_cp_async_bulk_wait_group :
     Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>;
@@ -1456,29 +1638,30 @@ def int_nvvm_mbarrier_inval_shared : NVVMBuiltin,
     [IntrConvergent, IntrWriteMem, IntrArgMemOnly, IntrNoCallback,
     WriteOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>]>;
 
-def int_nvvm_mbarrier_arrive : NVVMBuiltin,
-    Intrinsic<[llvm_i64_ty], [llvm_ptr_ty], [IntrConvergent, IntrNoCallback]>;
-def int_nvvm_mbarrier_arrive_shared : NVVMBuiltin,
-    Intrinsic<[llvm_i64_ty], [llvm_shared_ptr_ty], [IntrConvergent, IntrNoCallback]>;
-def int_nvvm_mbarrier_arrive_noComplete : NVVMBuiltin,
-    Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrConvergent, IntrNoCallback]>;
-def int_nvvm_mbarrier_arrive_noComplete_shared : NVVMBuiltin,
-    Intrinsic<[llvm_i64_ty], [llvm_shared_ptr_ty,
-    llvm_i32_ty], [IntrConvergent, IntrNoCallback]>;
-
-def int_nvvm_mbarrier_arrive_drop : NVVMBuiltin,
-    Intrinsic<[llvm_i64_ty], [llvm_ptr_ty], [IntrConvergent, IntrNoCallback]>;
-def int_nvvm_mbarrier_arrive_drop_shared : NVVMBuiltin,
-    Intrinsic<[llvm_i64_ty], [llvm_shared_ptr_ty], [IntrConvergent, IntrNoCallback]>;
-def int_nvvm_mbarrier_arrive_drop_noComplete : NVVMBuiltin,
-    Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i32_ty], [IntrConvergent, IntrNoCallback]>;
-def int_nvvm_mbarrier_arrive_drop_noComplete_shared : NVVMBuiltin,
-    Intrinsic<[llvm_i64_ty], [llvm_shared_ptr_ty, llvm_i32_ty], [IntrConvergent, IntrNoCallback]>;
-
-def int_nvvm_mbarrier_test_wait : NVVMBuiltin,
-    Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrConvergent, IntrNoCallback]>;
-def int_nvvm_mbarrier_test_wait_shared : NVVMBuiltin,
-    Intrinsic<[llvm_i1_ty], [llvm_shared_ptr_ty, llvm_i64_ty], [IntrConvergent, IntrNoCallback]>;
+let IntrProperties = [IntrConvergent, IntrNoCallback] in {
+  def int_nvvm_mbarrier_arrive : NVVMBuiltin,
+      Intrinsic<[llvm_i64_ty], [llvm_ptr_ty]>;
+  def int_nvvm_mbarrier_arrive_shared : NVVMBuiltin,
+      Intrinsic<[llvm_i64_ty], [llvm_shared_ptr_ty]>;
+  def int_nvvm_mbarrier_arrive_noComplete : NVVMBuiltin,
+      Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i32_ty]>;
+  def int_nvvm_mbarrier_arrive_noComplete_shared : NVVMBuiltin,
+      Intrinsic<[llvm_i64_ty], [llvm_shared_ptr_ty, llvm_i32_ty]>;
+
+  def int_nvvm_mbarrier_arrive_drop : NVVMBuiltin,
+      Intrinsic<[llvm_i64_ty], [llvm_ptr_ty]>;
+  def int_nvvm_mbarrier_arrive_drop_shared : NVVMBuiltin,
+      Intrinsic<[llvm_i64_ty], [llvm_shared_ptr_ty]>;
+  def int_nvvm_mbarrier_arrive_drop_noComplete : NVVMBuiltin,
+      Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i32_ty]>;
+  def int_nvvm_mbarrier_arrive_drop_noComplete_shared : NVVMBuiltin,
+      Intrinsic<[llvm_i64_ty], [llvm_shared_ptr_ty, llvm_i32_ty]>;
+
+  def int_nvvm_mbarrier_test_wait : NVVMBuiltin,
+      Intrinsic<[llvm_i1_ty], [llvm_ptr_ty, llvm_i64_ty]>;
+  def int_nvvm_mbarrier_test_wait_shared : NVVMBuiltin,
+      Intrinsic<[llvm_i1_ty], [llvm_shared_ptr_ty, llvm_i64_ty]>;
+}
 
 def int_nvvm_mbarrier_pending_count : NVVMBuiltin,
     Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem, IntrConvergent, IntrNoCallback]>;
@@ -1503,9 +1686,8 @@ let IntrProperties = [IntrReadMem, IntrArgMemOnly, IntrNoCallback, IntrWillRetur
 //    space when lowered during ISel.
 //
 def int_nvvm_internal_addrspace_wrap :
-  DefaultAttrsIntrinsic<[llvm_anyptr_ty], [llvm_anyptr_ty],
-                        [IntrNoMem, IntrSpeculatable, NoUndef<ArgIndex<0>>,
-                         NoUndef<RetIndex>]>;
+  PureIntrinsic<[llvm_anyptr_ty], [llvm_anyptr_ty],
+                [NoUndef<ArgIndex<0>>, NoUndef<RetIndex>]>;
 
 // Move intrinsics, used in nvvm internally
 
@@ -1519,36 +1701,26 @@ let IntrProperties = [IntrNoMem] in {
 }
 
 // For getting the handle from a texture or surface variable
-let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
-  def int_nvvm_texsurf_handle
-    : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_metadata_ty, llvm_anyptr_ty]>;
-  def int_nvvm_texsurf_handle_internal
-    : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
-}
+def int_nvvm_texsurf_handle
+  : PureIntrinsic<[llvm_i64_ty], [llvm_metadata_ty, llvm_anyptr_ty]>;
+def int_nvvm_texsurf_handle_internal
+  : PureIntrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
 
 /// Error / Warn
 def int_nvvm_compiler_error : Intrinsic<[], [llvm_anyptr_ty]>;
 def int_nvvm_compiler_warn : Intrinsic<[], [llvm_anyptr_ty]>;
 
-def int_nvvm_reflect : NVVMBuiltin,
-  Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>;
+def int_nvvm_reflect : NVVMBuiltin, PureIntrinsic<[llvm_i32_ty], [llvm_ptr_ty]>;
 
 // isspacep.{const, global, local, shared}
 foreach space = ["const", "global", "local", "shared", "shared_cluster"] in
   def int_nvvm_isspacep_ # space : NVVMBuiltin,
-    DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_ptr_ty],
-              [IntrNoMem, IntrSpeculatable, NoCapture<ArgIndex<0>>]>;
-
-// Environment register read
-foreach i = 0...31 in
-  def int_nvvm_read_ptx_sreg_envreg # i : NVVMBuiltin,
-    DefaultAttrsIntrinsic<[llvm_i32_ty], [],
-              [IntrNoMem, IntrSpeculatable, NoUndef<RetIndex>]>;
+    PureIntrinsic<[llvm_i1_ty], [llvm_ptr_ty], [NoCapture<ArgIndex<0>>]>;
 
 //
 // Texture Fetch
 //
-let IntrProperties = [IntrReadMem] in {
+let IntrProperties = [IntrReadMem, IntrNoCallback, IntrNoFree, IntrWillReturn] in {
   foreach is_unified = [true, false] in {
     defvar mode = !if(is_unified, "_unified", "");
     defvar addr_args = !if(is_unified, [llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty]);
@@ -1557,76 +1729,63 @@ let IntrProperties = [IntrReadMem] in {
       foreach is_array = [true, false] in {
         defvar array = !if(is_array, "_array", "");
         defvar array_args = !if(is_array, [llvm_i32_ty], []<LLVMType>);
+        defvar base_args = !listconcat(addr_args, array_args);
 
         def int_nvvm_tex # mode # _1d # array # _ # vec.Name # _s32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_i32_ty, 1))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_i32_ty, 1)>;
         def int_nvvm_tex # mode # _1d # array # _ # vec.Name # _f32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_float_ty, 1))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 1)>;
         def int_nvvm_tex # mode # _1d # array # _level_ # vec.Name # _f32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_float_ty, 2))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 2)>;
         def int_nvvm_tex # mode # _1d # array # _grad_ # vec.Name # _f32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_float_ty, 3))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 3)>;
 
         def int_nvvm_tex # mode # _2d # array # _ # vec.Name # _s32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_i32_ty, 2))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_i32_ty, 2)>;
         def int_nvvm_tex # mode # _2d # array # _ # vec.Name # _f32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_float_ty, 2))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 2)>;
         def int_nvvm_tex # mode # _2d # array # _level_ # vec.Name # _f32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_float_ty, 3))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 3)>;
         def int_nvvm_tex # mode # _2d # array # _grad_ # vec.Name # _f32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_float_ty, 6))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 6)>;
 
         if !not(is_array) then {
           def int_nvvm_tex # mode # _3d_ # vec.Name # _s32
-            : Intrinsic<vec.Types,
-                        !listconcat(addr_args, !listsplat(llvm_i32_ty, 3))>;
+            : Intrinsic<vec.Types, base_args # !listsplat(llvm_i32_ty, 3)>;
           def int_nvvm_tex # mode # _3d_ # vec.Name # _f32
-            : Intrinsic<vec.Types,
-                        !listconcat(addr_args, !listsplat(llvm_float_ty, 3))>;
+            : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 3)>;
           def int_nvvm_tex # mode # _3d_level_ # vec.Name # _f32
-            : Intrinsic<vec.Types,
-                        !listconcat(addr_args, !listsplat(llvm_float_ty, 4))>;
+            : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 4)>;
           def int_nvvm_tex # mode # _3d_grad_ # vec.Name # _f32
-            : Intrinsic<vec.Types,
-                        !listconcat(addr_args, !listsplat(llvm_float_ty, 9))>;
+            : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 9)>;
         }
 
         def int_nvvm_tex # mode # _cube # array # _ # vec.Name # _f32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_float_ty, 3))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 3)>;
         def int_nvvm_tex # mode # _cube # array # _level_ # vec.Name # _f32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, array_args, !listsplat(llvm_float_ty, 4))>;
+          : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 4)>;
 
         if is_unified then
           def int_nvvm_tex # mode # _cube # array # _grad_ # vec.Name # _f32
-            : Intrinsic<vec.Types,
-                        !listconcat(addr_args, array_args, !listsplat(llvm_float_ty, 9))>;
+            : Intrinsic<vec.Types, base_args # !listsplat(llvm_float_ty, 9)>;
       } // is_array
 
       foreach comp = ["r", "g", "b", "a"] in {
         def int_nvvm_tld4 # mode # _ # comp # _2d_ # vec.Name # _f32
-          : Intrinsic<vec.Types,
-                      !listconcat(addr_args, !listsplat(llvm_float_ty, 2))>;
+          : Intrinsic<vec.Types, addr_args # !listsplat(llvm_float_ty, 2)>;
       } // comp
     } // vec
   } // is_unified
 } // IntrProperties = [IntrReadMem]
 
 //=== Surface Load
-let IntrProperties = [IntrReadMem] in {
-  foreach clamp = ["clamp", "trap", "zero"] in {
-    foreach vec = [TV_I8, TV_I16, TV_I32, TV_I64,
-                  TV_V2I8, TV_V2I16, TV_V2I32, TV_V2I64,
-                  TV_V4I8, TV_V4I16, TV_V4I32] in {
+foreach clamp = ["clamp", "trap", "zero"] in {
+  foreach vec = [TV_I8, TV_I16, TV_I32, TV_I64,
+                TV_V2I8, TV_V2I16, TV_V2I32, TV_V2I64,
+                TV_V4I8, TV_V4I16, TV_V4I32] in {
+
+    let IntrProperties = [IntrNoCallback, IntrNoFree, IntrReadMem]
+                         # !if(!ne(clamp, "trap"), [IntrWillReturn], []<IntrinsicProperty>) in {
 
       def int_nvvm_suld_1d_ # vec.Name # _ # clamp
         : Intrinsic<vec.Types,
@@ -1647,47 +1806,50 @@ let IntrProperties = [IntrReadMem] in {
       def int_nvvm_suld_3d_ # vec.Name # _ # clamp
         : Intrinsic<vec.Types,
                     [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty]>;
-    } // vec
-  } // clamp
-} // IntrProperties = [IntrReadMem]
+    }
+  } // vec
+} // clamp
 
 //===- Texture Query ------------------------------------------------------===//
 
 foreach query = ["channel_order", "channel_data_type", "width", "height",
                  "depth", "array_size", "num_samples", "num_mipmap_levels"] in
   def int_nvvm_txq_ # query : NVVMBuiltin,
-    Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
+    DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
 
 //===- Surface Query ------------------------------------------------------===//
 
 foreach query = ["channel_order", "channel_data_type", "width", "height",
                  "depth", "array_size"] in
   def int_nvvm_suq_ # query : NVVMBuiltin,
-    Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
+    DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;
 
 //===- Handle Query -------------------------------------------------------===//
 
 foreach type = ["sampler", "surface", "texture"] in
   def int_nvvm_istypep_ # type : NVVMBuiltin,
-    Intrinsic<[llvm_i1_ty], [llvm_i64_ty], [IntrNoMem]>;
+    DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i64_ty], [IntrNoMem]>;
 
 //===- Surface Stores -----------------------------------------------------===//
 
 multiclass SurfaceStoreIntrinsics<string clamp, TexVector vec> {
-  def _1d_ # vec.Name # _ # clamp : NVVMBuiltin,
-      Intrinsic<[], !listconcat([llvm_i64_ty, llvm_i32_ty], vec.Types)>;
+  let IntrProperties = [IntrNoCallback, IntrNoFree, IntrWriteMem] #
+                       !if(!ne(clamp, "trap"), [IntrWillReturn], []<IntrinsicProperty>) in {
+    def _1d_ # vec.Name # _ # clamp : NVVMBuiltin,
+        Intrinsic<[], [llvm_i64_ty, llvm_i32_ty] # vec.Types>;
 
-  def _1d_array_ # vec.Name # _ # clamp : NVVMBuiltin,
-      Intrinsic<[], !listconcat([llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], vec.Types)>;
+    def _1d_array_ # vec.Name # _ # clamp : NVVMBuiltin,
+        Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty] # vec.Types>;
 
-  def _2d_ # vec.Name # _ # clamp : NVVMBuiltin,
-      Intrinsic<[], !listconcat([llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], vec.Types)>;
+    def _2d_ # vec.Name # _ # clamp : NVVMBuiltin,
+        Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty] # vec.Types>;
 
-  def _2d_array_ # vec.Name # _ # clamp : NVVMBuiltin,
-      Intrinsic<[], !listconcat([llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], vec.Types)>;
+    def _2d_array_ # vec.Name # _ # clamp : NVVMBuiltin,
+        Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty] # vec.Types>;
 
-  def _3d_ # vec.Name # _ # clamp : NVVMBuiltin,
-      Intrinsic<[], !listconcat([llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], vec.Types)>;
+    def _3d_ # vec.Name # _ # clamp : NVVMBuiltin,
+        Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty] # vec.Types>;
+  }
 }
 
 // Unformatted
@@ -1703,23 +1865,17 @@ foreach vec = [TV_I8, TV_I16, TV_I32,
                TV_V4I8, TV_V4I16, TV_V4I32] in
   defm int_nvvm_sust_p : SurfaceStoreIntrinsics<"trap", vec>;
 
+//
 // Accessing special registers.
-
+//
 class PTXReadSRegIntrinsicNB_r32<list<IntrinsicProperty> properties = []>
-  : DefaultAttrsIntrinsic<[llvm_i32_ty], [],
-      !listconcat([IntrNoMem, IntrSpeculatable, NoUndef<RetIndex>], properties)>;
+  : PureIntrinsic<[llvm_i32_ty], [], [NoUndef<RetIndex>] # properties>;
 
 class PTXReadSRegIntrinsic_r32<list<IntrinsicProperty> properties = []>
-  : PTXReadSRegIntrinsicNB_r32<properties>,
-    NVVMBuiltin;
+  : PTXReadSRegIntrinsicNB_r32<properties>, NVVMBuiltin;
 
 multiclass PTXReadSRegIntrinsic_v4i32<list<list<IntrinsicProperty>> properties = [[], [], [], []]> {
   assert !eq(!size(properties), 4), "properties must be a list of 4 lists";
-// FIXME: Do we need the 128-bit integer type version?
-//    def _r64   : Intrinsic<[llvm_i128_ty],   [], [IntrNoMem, IntrSpeculatable]>;
-
-// FIXME: Enable this once v4i32 support is enabled in back-end.
-//    def _v4i16 : Intrinsic<[llvm_v4i32_ty], [], [IntrNoMem, IntrSpeculatable]>;
   defvar suffixes = ["_x", "_y", "_z", "_w"];
   foreach i = !range(suffixes) in
     def suffixes[i] : PTXReadSRegIntrinsic_r32<properties[i]>;
@@ -1736,30 +1892,20 @@ multiclass PTXReadSRegIntrinsicNB_v4i32<list<list<IntrinsicProperty>> properties
 
 // Intrinsics to read registers with non-constant values. E.g. the values that
 // do change over the kernel lifetime. Such reads should not be CSE'd.
-class PTXReadNCSRegIntrinsic_r32
-  : Intrinsic<[llvm_i32_ty], [], [IntrInaccessibleMemOnly, IntrNoCallback, NoUndef<RetIndex>]>,
-    NVVMBuiltin;
-class PTXReadNCSRegIntrinsic_r64
-  : Intrinsic<[llvm_i64_ty], [], [IntrInaccessibleMemOnly, IntrNoCallback, NoUndef<RetIndex>]>,
+class PTXReadNCSRegIntrinsic<LLVMType ty>
+  : Intrinsic<[ty], [], [IntrInaccessibleMemOnly, IntrNoCallback,
+                         IntrNoFree, IntrWillReturn, NoUndef<RetIndex>]>,
     NVVMBuiltin;
 
-defm int_nvvm_read_ptx_sreg_tid
-  : PTXReadSRegIntrinsic_v4i32<[[Range<RetIndex, 0, MAX_BLOCK_SIZE_X>],
-                                [Range<RetIndex, 0, MAX_BLOCK_SIZE_Y>],
-                                [Range<RetIndex, 0, MAX_BLOCK_SIZE_Z>],
-                                [Range<RetIndex, 0, 1>]]>;
-
-defm int_nvvm_read_ptx_sreg_ntid
-  : PTXReadSRegIntrinsic_v4i32<[[Range<RetIndex, 1, !add(MAX_BLOCK_SIZE_X, 1)>],
-                                [Range<RetIndex, 1, !add(MAX_BLOCK_SIZE_Y, 1)>],
-                                [Range<RetIndex, 1, !add(MAX_BLOCK_SIZE_Z, 1)>],
-                                [Range<RetIndex, 0, 1>]]>;
+defvar MAX_BLOCK_ID_RANGE = [[Range<RetIndex, 0, MAX_BLOCK_SIZE_X>],
+                             [Range<RetIndex, 0, MAX_BLOCK_SIZE_Y>],
+                             [Range<RetIndex, 0, MAX_BLOCK_SIZE_Z>],
+                             [Range<RetIndex, 0, 1>]];
 
-def int_nvvm_read_ptx_sreg_laneid
-  : PTXReadSRegIntrinsic_r32<[Range<RetIndex, 0, WARP_SIZE>]>;
-
-def int_nvvm_read_ptx_sreg_warpid : PTXReadSRegIntrinsic_r32;
-def int_nvvm_read_ptx_sreg_nwarpid : PTXReadSRegIntrinsic_r32;
+defvar MAX_BLOCK_NID_RANGE = [[Range<RetIndex, 1, !add(MAX_BLOCK_SIZE_X, 1)>],
+                              [Range<RetIndex, 1, !add(MAX_BLOCK_SIZE_Y, 1)>],
+                              [Range<RetIndex, 1, !add(MAX_BLOCK_SIZE_Z, 1)>],
+                              [Range<RetIndex, 0, 1>]];
 
 defvar MAX_GRID_ID_RANGE = [[Range<RetIndex, 0, MAX_GRID_SIZE_X>],
                             [Range<RetIndex, 0, MAX_GRID_SIZE_Y>],
@@ -1771,11 +1917,17 @@ defvar MAX_GRID_NID_RANGE = [[Range<RetIndex, 1, !add(MAX_GRID_SIZE_X, 1)>],
                              [Range<RetIndex, 1, !add(MAX_GRID_SIZE_Z, 1)>],
                              [Range<RetIndex, 0, 1>]];
 
-defm int_nvvm_read_ptx_sreg_ctaid
-  : PTXReadSRegIntrinsic_v4i32<MAX_GRID_ID_RANGE>;
+defm int_nvvm_read_ptx_sreg_tid : PTXReadSRegIntrinsic_v4i32<MAX_BLOCK_ID_RANGE>;
+defm int_nvvm_read_ptx_sreg_ntid : PTXReadSRegIntrinsic_v4i32<MAX_BLOCK_NID_RANGE>;
 
-defm int_nvvm_read_ptx_sreg_nctaid
-  : PTXReadSRegIntrinsic_v4i32<MAX_GRID_NID_RANGE>;
+def int_nvvm_read_ptx_sreg_laneid
+  : PTXReadSRegIntrinsic_r32<[Range<RetIndex, 0, WARP_SIZE>]>;
+
+def int_nvvm_read_ptx_sreg_warpid : PTXReadSRegIntrinsic_r32;
+def int_nvvm_read_ptx_sreg_nwarpid : PTXReadSRegIntrinsic_r32;
+
+defm int_nvvm_read_ptx_sreg_ctaid : PTXReadSRegIntrinsic_v4i32<MAX_GRID_ID_RANGE>;
+defm int_nvvm_read_ptx_sreg_nctaid : PTXReadSRegIntrinsic_v4i32<MAX_GRID_NID_RANGE>;
 
 def int_nvvm_read_ptx_sreg_smid : PTXReadSRegIntrinsic_r32;
 def int_nvvm_read_ptx_sreg_nsmid : PTXReadSRegIntrinsic_r32;
@@ -1787,19 +1939,23 @@ def int_nvvm_read_ptx_sreg_lanemask_lt : PTXReadSRegIntrinsic_r32;
 def int_nvvm_read_ptx_sreg_lanemask_ge : PTXReadSRegIntrinsic_r32;
 def int_nvvm_read_ptx_sreg_lanemask_gt : PTXReadSRegIntrinsic_r32;
 
-def int_nvvm_read_ptx_sreg_clock : PTXReadNCSRegIntrinsic_r32;
-def int_nvvm_read_ptx_sreg_clock64 : PTXReadNCSRegIntrinsic_r64;
+def int_nvvm_read_ptx_sreg_clock : PTXReadNCSRegIntrinsic<llvm_i32_ty>;
+def int_nvvm_read_ptx_sreg_clock64 : PTXReadNCSRegIntrinsic<llvm_i64_ty>;
 
-def int_nvvm_read_ptx_sreg_globaltimer : PTXReadNCSRegIntrinsic_r64;
+def int_nvvm_read_ptx_sreg_globaltimer : PTXReadNCSRegIntrinsic<llvm_i64_ty>;
+def int_nvvm_read_ptx_sreg_globaltimer_lo : PTXReadNCSRegIntrinsic<llvm_i32_ty>;
 
-def int_nvvm_read_ptx_sreg_pm0 : PTXReadNCSRegIntrinsic_r32;
-def int_nvvm_read_ptx_sreg_pm1 : PTXReadNCSRegIntrinsic_r32;
-def int_nvvm_read_ptx_sreg_pm2 : PTXReadNCSRegIntrinsic_r32;
-def int_nvvm_read_ptx_sreg_pm3 : PTXReadNCSRegIntrinsic_r32;
+def int_nvvm_read_ptx_sreg_pm0 : PTXReadNCSRegIntrinsic<llvm_i32_ty>;
+def int_nvvm_read_ptx_sreg_pm1 : PTXReadNCSRegIntrinsic<llvm_i32_ty>;
+def int_nvvm_read_ptx_sreg_pm2 : PTXReadNCSRegIntrinsic<llvm_i32_ty>;
+def int_nvvm_read_ptx_sreg_pm3 : PTXReadNCSRegIntrinsic<llvm_i32_ty>;
 
 def int_nvvm_read_ptx_sreg_warpsize
   : PTXReadSRegIntrinsic_r32<[Range<RetIndex, WARP_SIZE, !add(WARP_SIZE, 1)>]>;
 
+foreach i = 0...31 in
+  def int_nvvm_read_ptx_sreg_envreg # i : PTXReadSRegIntrinsic_r32;
+
 // sm90+, PTX7.8+
 
 // Note: Since clusters are subdivisions of the grid, we conservatively use the
@@ -1807,14 +1963,10 @@ def int_nvvm_read_ptx_sreg_warpsize
 // practice, the clusterid will likely be much smaller. The CUDA programming
 // guide recommends 8 as a maximum portable value and H100s support 16.
 
-defm int_nvvm_read_ptx_sreg_clusterid
-  : PTXReadSRegIntrinsicNB_v4i32<MAX_GRID_ID_RANGE>;
-defm int_nvvm_read_ptx_sreg_nclusterid
-  : PTXReadSRegIntrinsicNB_v4i32<MAX_GRID_NID_RANGE>;
-defm int_nvvm_read_ptx_sreg_cluster_ctaid
-  : PTXReadSRegIntrinsicNB_v4i32<MAX_GRID_ID_RANGE>;
-defm int_nvvm_read_ptx_sreg_cluster_nctaid
-  : PTXReadSRegIntrinsicNB_v4i32<MAX_GRID_NID_RANGE>;
+defm int_nvvm_read_ptx_sreg_clusterid : PTXReadSRegIntrinsicNB_v4i32<MAX_GRID_ID_RANGE>;
+defm int_nvvm_read_ptx_sreg_nclusterid : PTXReadSRegIntrinsicNB_v4i32<MAX_GRID_NID_RANGE>;
+defm int_nvvm_read_ptx_sreg_cluster_ctaid : PTXReadSRegIntrinsicNB_v4i32<MAX_GRID_ID_RANGE>;
+defm int_nvvm_read_ptx_sreg_cluster_nctaid : PTXReadSRegIntrinsicNB_v4i32<MAX_GRID_NID_RANGE>;
 
 def int_nvvm_read_ptx_sreg_cluster_ctarank : PTXReadSRegIntrinsicNB_r32;
 def int_nvvm_read_ptx_sreg_cluster_nctarank : PTXReadSRegIntrinsicNB_r32;
@@ -1842,13 +1994,13 @@ let IntrProperties = [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback] i
 //
 // VOTE
 //
-
 let IntrProperties = [IntrInaccessibleMemOnly, IntrConvergent, IntrNoCallback] in {
   def int_nvvm_vote_all : NVVMBuiltin, Intrinsic<[llvm_i1_ty], [llvm_i1_ty]>;
   def int_nvvm_vote_any : NVVMBuiltin, Intrinsic<[llvm_i1_ty], [llvm_i1_ty]>;
   def int_nvvm_vote_uni : NVVMBuiltin, Intrinsic<[llvm_i1_ty], [llvm_i1_ty]>;
   def int_nvvm_vote_ballot : NVVMBuiltin, Intrinsic<[llvm_i32_ty], [llvm_i1_ty]>;
 }
+
 //
 // VOTE.SYNC
 //
@@ -2001,6 +2153,53 @@ foreach layout_a = ["row", "col"] in {
   } // layout_b
 } // layout_a
 
+// MMA.SP
+class NVVM_MMA_SP<WMMA_REGS A, WMMA_REGS B, WMMA_REGS C, WMMA_REGS D>
+  : Intrinsic<D.regs,
+              !listconcat(A.regs, B.regs, C.regs, [llvm_i32_ty], [llvm_i32_ty])> {
+    int pos = !size(!listconcat(A.regs, B.regs, C.regs, [llvm_i32_ty]));
+
+    // The range [0;num_threads) is for the sparsity selector that indicates the threads
+    // which contribute metadata.
+    int num_threads = !if(!or(!and(!eq(A.geom, "m16n8k32"), !eq(A.ptx_elt_type, "bf16")),
+                              !and(!eq(A.geom, "m16n8k32"), !eq(A.ptx_elt_type, "f16")),
+                              !and(!eq(A.geom, "m16n8k16"), !eq(A.ptx_elt_type, "tf32")),
+                              !and(!eq(A.geom, "m16n8k32"), !eq(A.ptx_elt_type, "u8")),
+                              !and(!eq(A.geom, "m16n8k32"), !eq(A.ptx_elt_type, "s8")),
+                              !and(!eq(A.geom, "m16n8k64"), !eq(A.ptx_elt_type, "u4")),
+                              !and(!eq(A.geom, "m16n8k64"), !eq(A.ptx_elt_type, "s4"))),
+                          2,
+                          !if(!and(!eq(A.geom, "m16n8k64"),
+                                  !or(!eq(A.ptx_elt_type, "u8"),
+                                      !eq(A.ptx_elt_type, "s8"),
+                                      !eq(A.ptx_elt_type, "e4m3"),
+                                      !eq(A.ptx_elt_type, "e5m2"),
+                                      !eq(A.ptx_elt_type, "e3m2"),
+                                      !eq(A.ptx_elt_type, "e2m3"),
+                                      !eq(A.ptx_elt_type, "e2m1"))),
+                              1,
+                              !if(!and(!eq(A.geom, "m16n8k128"),
+                                       !or(!eq(A.ptx_elt_type, "s4"),
+                                           !eq(A.ptx_elt_type, "u4"))),
+                                  1, 4)));
+    let IntrProperties = [IntrNoMem, IntrNoCallback, ImmArg<ArgIndex<pos>>,
+                          Range<ArgIndex<pos>, 0, num_threads>];
+}
+
+foreach metadata = ["sp", "sp::ordered_metadata"] in {
+  foreach kind = ["", "kind::f8f6f4"] in {
+    foreach satf = [0, 1] in {
+      foreach op = NVVM_MMA_OPS.all_mma_sp_ops in {
+        if NVVM_MMA_SP_SUPPORTED<op, metadata, kind, satf>.ret then {
+          def MMA_SP_NAME<metadata, kind, satf,
+                          op[0], op[1], op[2], op[3]>.record
+            : NVVM_MMA_SP<op[0], op[1], op[2], op[3]>;
+        }
+      } // op
+    } // satf
+  } // kind
+} // metadata
+
 // LDMATRIX
 class NVVM_LDMATRIX<WMMA_REGS Frag, int Transposed>
   : Intrinsic<Frag.regs, [llvm_anyptr_ty],
@@ -2051,8 +2250,7 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable, NoCapture<ArgIndex<0>>] in {
 }
 
 def int_nvvm_is_explicit_cluster
-  : DefaultAttrsIntrinsic<[llvm_i1_ty], [],
-              [IntrNoMem, IntrSpeculatable, NoUndef<RetIndex>],
+  : PureIntrinsic<[llvm_i1_ty], [], [NoUndef<RetIndex>],
               "llvm.nvvm.is_explicit_cluster">;
 
 // Setmaxnreg inc/dec intrinsics
@@ -2212,15 +2410,17 @@ def int_nvvm_cp_async_bulk_tensor_prefetch_tile_gather4_2d
 // Intrinsics for Prefetch and Prefetchu
 let IntrProperties = [IntrArgMemOnly, ReadOnly<ArgIndex<0>>, NoCapture<ArgIndex<0>>] in {
   foreach level = ["L1", "L2"] in {
-    def int_nvvm_prefetch_ # level : Intrinsic<[], [llvm_ptr_ty]>;
-    def int_nvvm_prefetch_global_ # level : Intrinsic<[], [llvm_global_ptr_ty]>;
-    def int_nvvm_prefetch_local_ # level : Intrinsic<[], [llvm_local_ptr_ty]>;
+    def int_nvvm_prefetch_ # level : DefaultAttrsIntrinsic<[], [llvm_ptr_ty]>;
+    def int_nvvm_prefetch_global_ # level : DefaultAttrsIntrinsic<[], [llvm_global_ptr_ty]>;
+    def int_nvvm_prefetch_local_ # level : DefaultAttrsIntrinsic<[], [llvm_local_ptr_ty]>;
   }
 
+  def int_nvvm_prefetch_tensormap : DefaultAttrsIntrinsic<[], [llvm_anyptr_ty]>;
+  
   foreach eviction_priority = ["evict_normal", "evict_last"] in
-    def int_nvvm_prefetch_global_L2_ # eviction_priority : Intrinsic<[], [llvm_global_ptr_ty]>;
+    def int_nvvm_prefetch_global_L2_ # eviction_priority : DefaultAttrsIntrinsic<[], [llvm_global_ptr_ty]>;
 
-  def int_nvvm_prefetchu_L1 : Intrinsic<[], [llvm_ptr_ty]>;
+  def int_nvvm_prefetchu_L1 : DefaultAttrsIntrinsic<[], [llvm_ptr_ty]>;
 }
 
 // applypriority
@@ -2455,13 +2655,12 @@ def int_nvvm_clusterlaunchcontrol_try_cancel_async_multicast_shared
 // clusterlaunchcontrol.query_cancel.is_canceled
 
 def int_nvvm_clusterlaunchcontrol_query_cancel_is_canceled
-    : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_i128_ty], [IntrNoMem, IntrSpeculatable],
-                            "llvm.nvvm.clusterlaunchcontrol.query_cancel.is_canceled">;
+    : PureIntrinsic<[llvm_i1_ty], [llvm_i128_ty], [],
+                    "llvm.nvvm.clusterlaunchcontrol.query_cancel.is_canceled">;
 
-foreach dim = ["x", "y", "z"] in {
-def int_nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_ # dim
-    : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i128_ty], [IntrNoMem, IntrSpeculatable],
-                            "llvm.nvvm.clusterlaunchcontrol.query_cancel.get_first_ctaid." # dim>;
-}
+foreach dim = ["x", "y", "z"] in
+  def int_nvvm_clusterlaunchcontrol_query_cancel_get_first_ctaid_ # dim
+    : PureIntrinsic<[llvm_i32_ty], [llvm_i128_ty], [],
+                    "llvm.nvvm.clusterlaunchcontrol.query_cancel.get_first_ctaid." # dim>;
 
 } // let TargetPrefix = "nvvm"
diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
index 7dd9ff7f08b8b..94afa94bfb1ee 100644
--- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td
@@ -676,6 +676,13 @@ let TargetPrefix = "ppc" in {  // All intrinsics start with "llvm.ppc.".
   def int_ppc_addg6s: ClangBuiltin<"__builtin_addg6s">,
     DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
 
+  // BCD Format conversion intrinsics 
+  def int_ppc_bcdcopysign : ClangBuiltin<"__builtin_ppc_bcdcopysign">,
+    DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; 
+  def int_ppc_bcdsetsign : ClangBuiltin<"__builtin_ppc_bcdsetsign">,
+    DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], 
+    [IntrNoMem, ImmArg<ArgIndex<1>>]>;
+
   def int_ppc_bcdadd : ClangBuiltin<"__builtin_ppc_bcdadd">,
     DefaultAttrsIntrinsic<
     [llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty],
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td
index 99f975faeb85e..4b8a9da9688a2 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCV.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td
@@ -15,8 +15,7 @@
 
 // Atomic Intrinsics have multiple versions for different access widths, which
 // all follow one of the following signatures (depending on how many arguments
-// they require). We carefully instantiate only specific versions of these for
-// specific integer widths, rather than using `llvm_anyint_ty`.
+// they require).
 //
 // In fact, as these intrinsics take `llvm_anyptr_ty`, the given names are the
 // canonical names, and the intrinsics used in the code will have a name
@@ -25,52 +24,39 @@
 
 let TargetPrefix = "riscv" in {
 
-  // T @llvm.<name>.T.<p>(any*, T, T, T imm);
-  class MaskedAtomicRMWFourArg<LLVMType itype>
-      : Intrinsic<[itype], [llvm_anyptr_ty, itype, itype, itype],
+  // T @llvm.<name>.<i>.<p>(any*, T, T, T imm);
+  class RISCVMaskedAtomicRMWFourArg
+      : Intrinsic<[llvm_anyint_ty], [llvm_anyptr_ty, LLVMMatchType<0>,
+                                     LLVMMatchType<0>, LLVMMatchType<0>],
                   [IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<3>>]>;
-  // T @llvm.<name>.T.<p>(any*, T, T, T, T imm);
-  class MaskedAtomicRMWFiveArg<LLVMType itype>
-      : Intrinsic<[itype], [llvm_anyptr_ty, itype, itype, itype, itype],
+  // T @llvm.<name>.<i>.<p>(any*, T, T, T, T imm);
+  class RISCVMaskedAtomicRMWFiveArg
+      : Intrinsic<[llvm_anyint_ty], [llvm_anyptr_ty, LLVMMatchType<0>,
+                                     LLVMMatchType<0>, LLVMMatchType<0>,
+                                     LLVMMatchType<0>],
                   [IntrArgMemOnly, NoCapture<ArgIndex<0>>, ImmArg<ArgIndex<4>>]>;
 
-  // We define 32-bit and 64-bit variants of the above, where T stands for i32
-  // or i64 respectively:
-  multiclass MaskedAtomicRMWFourArgIntrinsics {
-    // i32 @llvm.<name>.i32.<p>(any*, i32, i32, i32 imm);
-    def _i32 : MaskedAtomicRMWFourArg<llvm_i32_ty>;
-    // i64 @llvm.<name>.i32.<p>(any*, i64, i64, i64 imm);
-    def _i64 : MaskedAtomicRMWFourArg<llvm_i64_ty>;
-  }
-
-  multiclass MaskedAtomicRMWFiveArgIntrinsics {
-    // i32 @llvm.<name>.i32.<p>(any*, i32, i32, i32, i32 imm);
-    def _i32 : MaskedAtomicRMWFiveArg<llvm_i32_ty>;
-    // i64 @llvm.<name>.i64.<p>(any*, i64, i64, i64, i64 imm);
-    def _i64 : MaskedAtomicRMWFiveArg<llvm_i64_ty>;
-  }
-
   // These intrinsics are intended only for internal compiler use (i.e. as
   // part of AtomicExpandpass via the emitMaskedAtomic*Intrinsic hooks). Their
   // names and semantics could change in the future.
 
-  // @llvm.riscv.masked.atomicrmw.*.{i32,i64}.<p>(
+  // @llvm.riscv.masked.atomicrmw.*.<i>.<p>(
   //   ptr addr, ixlen oparg, ixlen mask, ixlenimm ordering)
-  defm int_riscv_masked_atomicrmw_xchg : MaskedAtomicRMWFourArgIntrinsics;
-  defm int_riscv_masked_atomicrmw_add : MaskedAtomicRMWFourArgIntrinsics;
-  defm int_riscv_masked_atomicrmw_sub : MaskedAtomicRMWFourArgIntrinsics;
-  defm int_riscv_masked_atomicrmw_nand : MaskedAtomicRMWFourArgIntrinsics;
-  defm int_riscv_masked_atomicrmw_umax : MaskedAtomicRMWFourArgIntrinsics;
-  defm int_riscv_masked_atomicrmw_umin : MaskedAtomicRMWFourArgIntrinsics;
+  def int_riscv_masked_atomicrmw_xchg : RISCVMaskedAtomicRMWFourArg;
+  def int_riscv_masked_atomicrmw_add : RISCVMaskedAtomicRMWFourArg;
+  def int_riscv_masked_atomicrmw_sub : RISCVMaskedAtomicRMWFourArg;
+  def int_riscv_masked_atomicrmw_nand : RISCVMaskedAtomicRMWFourArg;
+  def int_riscv_masked_atomicrmw_umax : RISCVMaskedAtomicRMWFourArg;
+  def int_riscv_masked_atomicrmw_umin : RISCVMaskedAtomicRMWFourArg;
   // Signed min and max need an extra operand to do sign extension with.
-  // @llvm.riscv.masked.atomicrmw.{max,min}.{i32,i64}.<p>(
+  // @llvm.riscv.masked.atomicrmw.{max,min}.<i>.<p>(
   //   ptr addr, ixlen oparg, ixlen mask, ixlen shamt, ixlenimm ordering)
-  defm int_riscv_masked_atomicrmw_max : MaskedAtomicRMWFiveArgIntrinsics;
-  defm int_riscv_masked_atomicrmw_min : MaskedAtomicRMWFiveArgIntrinsics;
+  def int_riscv_masked_atomicrmw_max : RISCVMaskedAtomicRMWFiveArg;
+  def int_riscv_masked_atomicrmw_min : RISCVMaskedAtomicRMWFiveArg;
 
-  // @llvm.riscv.masked.cmpxchg.{i32,i64}.<p>(
+  // @llvm.riscv.masked.cmpxchg.<i>.<p>(
   //   ptr addr, ixlen cmpval, ixlen newval, ixlen mask, ixlenimm ordering)
-  defm int_riscv_masked_cmpxchg : MaskedAtomicRMWFiveArgIntrinsics;
+  def int_riscv_masked_cmpxchg : RISCVMaskedAtomicRMWFiveArg;
 
 } // TargetPrefix = "riscv"
 
@@ -79,33 +65,33 @@ let TargetPrefix = "riscv" in {
 
 let TargetPrefix = "riscv" in {
 
-  class BitManipGPRIntrinsics
-      : DefaultAttrsIntrinsic<[llvm_any_ty],
+  class RISCVBitManipGPRIntrinsics
+      : DefaultAttrsIntrinsic<[llvm_anyint_ty],
                               [LLVMMatchType<0>],
                               [IntrNoMem, IntrSpeculatable]>;
-  class BitManipGPRGPRIntrinsics
-      : DefaultAttrsIntrinsic<[llvm_any_ty],
+  class RISCVBitManipGPRGPRIntrinsics
+      : DefaultAttrsIntrinsic<[llvm_anyint_ty],
                               [LLVMMatchType<0>, LLVMMatchType<0>],
                               [IntrNoMem, IntrSpeculatable]>;
 
   // Zbb
-  def int_riscv_orc_b : BitManipGPRIntrinsics;
+  def int_riscv_orc_b : RISCVBitManipGPRIntrinsics;
 
   // Zbc or Zbkc
-  def int_riscv_clmul  : BitManipGPRGPRIntrinsics;
-  def int_riscv_clmulh : BitManipGPRGPRIntrinsics;
+  def int_riscv_clmul  : RISCVBitManipGPRGPRIntrinsics;
+  def int_riscv_clmulh : RISCVBitManipGPRGPRIntrinsics;
 
   // Zbc
-  def int_riscv_clmulr : BitManipGPRGPRIntrinsics;
+  def int_riscv_clmulr : RISCVBitManipGPRGPRIntrinsics;
 
   // Zbkb
-  def int_riscv_brev8 : BitManipGPRIntrinsics;
-  def int_riscv_zip   : BitManipGPRIntrinsics;
-  def int_riscv_unzip : BitManipGPRIntrinsics;
+  def int_riscv_brev8 : RISCVBitManipGPRIntrinsics;
+  def int_riscv_zip   : RISCVBitManipGPRIntrinsics;
+  def int_riscv_unzip : RISCVBitManipGPRIntrinsics;
 
   // Zbkx
-  def int_riscv_xperm4  : BitManipGPRGPRIntrinsics;
-  def int_riscv_xperm8  : BitManipGPRGPRIntrinsics;
+  def int_riscv_xperm4  : RISCVBitManipGPRGPRIntrinsics;
+  def int_riscv_xperm8  : RISCVBitManipGPRGPRIntrinsics;
 } // TargetPrefix = "riscv"
 
 //===----------------------------------------------------------------------===//
@@ -115,11 +101,11 @@ let TargetPrefix = "riscv" in {
 
   // Zimop
   def int_riscv_mopr
-      : DefaultAttrsIntrinsic<[llvm_any_ty],
+      : DefaultAttrsIntrinsic<[llvm_anyint_ty],
                               [LLVMMatchType<0>, LLVMMatchType<0>],
                               [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<1>>]>;
   def int_riscv_moprr
-      : DefaultAttrsIntrinsic<[llvm_any_ty],
+      : DefaultAttrsIntrinsic<[llvm_anyint_ty],
                               [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
                               [IntrNoMem, IntrSpeculatable, ImmArg<ArgIndex<2>>]>;
 } // TargetPrefix = "riscv"
@@ -135,7 +121,7 @@ defvar NoScalarOperand = 0xF;
 defvar NoVLOperand = 0x1F;
 
 class RISCVVIntrinsic {
-  // These intrinsics may accept illegal integer values in their llvm_any_ty
+  // These intrinsics may accept illegal integer values in their llvm_anyint_ty
   // operand, so they have to be extended.
   Intrinsic IntrinsicID = !cast<Intrinsic>(NAME);
   bits<4> ScalarOperand = NoScalarOperand;
@@ -1717,7 +1703,7 @@ let TargetPrefix = "riscv" in {
                                    llvm_anyint_ty],
                                   [NoCapture<ArgIndex<0>>, IntrReadMem]>;
 
-    // Input: (pointer, offset, mask, vl)
+    // Input: (pointer, stride, mask, vl)
     def int_riscv_sseg # nf # _load_mask
           : DefaultAttrsIntrinsic<!listconcat([llvm_anyvector_ty],
                                               !listsplat(LLVMMatchType<0>,
@@ -1736,6 +1722,17 @@ let TargetPrefix = "riscv" in {
                                               [llvm_anyptr_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                                llvm_anyint_ty]),
                                   [NoCapture<ArgIndex<nf>>, IntrWriteMem]>;
+
+    // Input: (<stored values>..., pointer, stride, mask, vl)
+    def int_riscv_sseg # nf # _store_mask
+          : DefaultAttrsIntrinsic<[],
+                                  !listconcat([llvm_anyvector_ty],
+                                              !listsplat(LLVMMatchType<0>,
+                                                          !add(nf, -1)),
+                                              [llvm_anyptr_ty, llvm_anyint_ty,
+                                               LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                                               llvm_anyint_ty]),
+                                  [NoCapture<ArgIndex<nf>>, IntrWriteMem]>;
   }
 
 } // TargetPrefix = "riscv"
diff --git a/llvm/include/llvm/IR/IntrinsicsRISCVXTHead.td b/llvm/include/llvm/IR/IntrinsicsRISCVXTHead.td
index 5af10a3e197aa..c851f34a41c7d 100644
--- a/llvm/include/llvm/IR/IntrinsicsRISCVXTHead.td
+++ b/llvm/include/llvm/IR/IntrinsicsRISCVXTHead.td
@@ -12,7 +12,7 @@
 
 let TargetPrefix = "riscv" in {
 
-  class TH_VdotTernaryWideMasked
+  class RISCV_TH_VdotTernaryWideMasked
         : DefaultAttrsIntrinsic< [llvm_anyvector_ty],
                      [LLVMMatchType<0>, llvm_any_ty, llvm_anyvector_ty,
                       LLVMScalarOrSameVectorWidth<2, llvm_i1_ty>,
@@ -22,13 +22,13 @@ let TargetPrefix = "riscv" in {
     let VLOperand = 4;
   }
 
-  multiclass TH_VdotTernaryWide {
+  multiclass RISCV_TH_VdotTernaryWide {
     def "int_riscv_" # NAME : RISCVTernaryWideUnMasked;
-    def "int_riscv_" # NAME # "_mask" : TH_VdotTernaryWideMasked;
+    def "int_riscv_" # NAME # "_mask" : RISCV_TH_VdotTernaryWideMasked;
   }
 
-  defm th_vmaqa    : TH_VdotTernaryWide;
-  defm th_vmaqau   : TH_VdotTernaryWide;
-  defm th_vmaqasu  : TH_VdotTernaryWide;
-  defm th_vmaqaus  : TH_VdotTernaryWide;
+  defm th_vmaqa    : RISCV_TH_VdotTernaryWide;
+  defm th_vmaqau   : RISCV_TH_VdotTernaryWide;
+  defm th_vmaqasu  : RISCV_TH_VdotTernaryWide;
+  defm th_vmaqaus  : RISCV_TH_VdotTernaryWide;
 }
diff --git a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
index 11bfd733a8854..cc4929a1ff8da 100644
--- a/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
+++ b/llvm/include/llvm/IR/NVVMIntrinsicUtils.h
@@ -414,6 +414,178 @@ inline DenormalMode GetNVVMDenormMode(bool ShouldFTZ) {
   return DenormalMode::getIEEE();
 }
 
+inline bool FAddShouldFTZ(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_add_rm_ftz_f:
+  case Intrinsic::nvvm_add_rn_ftz_f:
+  case Intrinsic::nvvm_add_rp_ftz_f:
+  case Intrinsic::nvvm_add_rz_ftz_f:
+    return true;
+
+  case Intrinsic::nvvm_add_rm_f:
+  case Intrinsic::nvvm_add_rn_f:
+  case Intrinsic::nvvm_add_rp_f:
+  case Intrinsic::nvvm_add_rz_f:
+  case Intrinsic::nvvm_add_rm_d:
+  case Intrinsic::nvvm_add_rn_d:
+  case Intrinsic::nvvm_add_rp_d:
+  case Intrinsic::nvvm_add_rz_d:
+    return false;
+  }
+  llvm_unreachable("Checking FTZ flag for invalid NVVM add intrinsic");
+}
+
+inline APFloat::roundingMode GetFAddRoundingMode(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_add_rm_f:
+  case Intrinsic::nvvm_add_rm_d:
+  case Intrinsic::nvvm_add_rm_ftz_f:
+    return APFloat::rmTowardNegative;
+  case Intrinsic::nvvm_add_rn_f:
+  case Intrinsic::nvvm_add_rn_d:
+  case Intrinsic::nvvm_add_rn_ftz_f:
+    return APFloat::rmNearestTiesToEven;
+  case Intrinsic::nvvm_add_rp_f:
+  case Intrinsic::nvvm_add_rp_d:
+  case Intrinsic::nvvm_add_rp_ftz_f:
+    return APFloat::rmTowardPositive;
+  case Intrinsic::nvvm_add_rz_f:
+  case Intrinsic::nvvm_add_rz_d:
+  case Intrinsic::nvvm_add_rz_ftz_f:
+    return APFloat::rmTowardZero;
+  }
+  llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM add");
+}
+
+inline bool FMulShouldFTZ(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_mul_rm_ftz_f:
+  case Intrinsic::nvvm_mul_rn_ftz_f:
+  case Intrinsic::nvvm_mul_rp_ftz_f:
+  case Intrinsic::nvvm_mul_rz_ftz_f:
+    return true;
+
+  case Intrinsic::nvvm_mul_rm_f:
+  case Intrinsic::nvvm_mul_rn_f:
+  case Intrinsic::nvvm_mul_rp_f:
+  case Intrinsic::nvvm_mul_rz_f:
+  case Intrinsic::nvvm_mul_rm_d:
+  case Intrinsic::nvvm_mul_rn_d:
+  case Intrinsic::nvvm_mul_rp_d:
+  case Intrinsic::nvvm_mul_rz_d:
+    return false;
+  }
+  llvm_unreachable("Checking FTZ flag for invalid NVVM mul intrinsic");
+}
+
+inline APFloat::roundingMode GetFMulRoundingMode(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_mul_rm_f:
+  case Intrinsic::nvvm_mul_rm_d:
+  case Intrinsic::nvvm_mul_rm_ftz_f:
+    return APFloat::rmTowardNegative;
+  case Intrinsic::nvvm_mul_rn_f:
+  case Intrinsic::nvvm_mul_rn_d:
+  case Intrinsic::nvvm_mul_rn_ftz_f:
+    return APFloat::rmNearestTiesToEven;
+  case Intrinsic::nvvm_mul_rp_f:
+  case Intrinsic::nvvm_mul_rp_d:
+  case Intrinsic::nvvm_mul_rp_ftz_f:
+    return APFloat::rmTowardPositive;
+  case Intrinsic::nvvm_mul_rz_f:
+  case Intrinsic::nvvm_mul_rz_d:
+  case Intrinsic::nvvm_mul_rz_ftz_f:
+    return APFloat::rmTowardZero;
+  }
+  llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM mul");
+}
+
+inline bool FDivShouldFTZ(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_div_rm_ftz_f:
+  case Intrinsic::nvvm_div_rn_ftz_f:
+  case Intrinsic::nvvm_div_rp_ftz_f:
+  case Intrinsic::nvvm_div_rz_ftz_f:
+    return true;
+
+  case Intrinsic::nvvm_div_rm_f:
+  case Intrinsic::nvvm_div_rn_f:
+  case Intrinsic::nvvm_div_rp_f:
+  case Intrinsic::nvvm_div_rz_f:
+  case Intrinsic::nvvm_div_rm_d:
+  case Intrinsic::nvvm_div_rn_d:
+  case Intrinsic::nvvm_div_rp_d:
+  case Intrinsic::nvvm_div_rz_d:
+    return false;
+  }
+  llvm_unreachable("Checking FTZ flag for invalid NVVM div intrinsic");
+}
+
+inline APFloat::roundingMode GetFDivRoundingMode(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_div_rm_f:
+  case Intrinsic::nvvm_div_rm_d:
+  case Intrinsic::nvvm_div_rm_ftz_f:
+    return APFloat::rmTowardNegative;
+  case Intrinsic::nvvm_div_rn_f:
+  case Intrinsic::nvvm_div_rn_d:
+  case Intrinsic::nvvm_div_rn_ftz_f:
+    return APFloat::rmNearestTiesToEven;
+  case Intrinsic::nvvm_div_rp_f:
+  case Intrinsic::nvvm_div_rp_d:
+  case Intrinsic::nvvm_div_rp_ftz_f:
+    return APFloat::rmTowardPositive;
+  case Intrinsic::nvvm_div_rz_f:
+  case Intrinsic::nvvm_div_rz_d:
+  case Intrinsic::nvvm_div_rz_ftz_f:
+    return APFloat::rmTowardZero;
+  }
+  llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM div");
+}
+
+inline bool FMAShouldFTZ(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_fma_rm_ftz_f:
+  case Intrinsic::nvvm_fma_rn_ftz_f:
+  case Intrinsic::nvvm_fma_rp_ftz_f:
+  case Intrinsic::nvvm_fma_rz_ftz_f:
+    return true;
+
+  case Intrinsic::nvvm_fma_rm_f:
+  case Intrinsic::nvvm_fma_rn_f:
+  case Intrinsic::nvvm_fma_rp_f:
+  case Intrinsic::nvvm_fma_rz_f:
+  case Intrinsic::nvvm_fma_rm_d:
+  case Intrinsic::nvvm_fma_rn_d:
+  case Intrinsic::nvvm_fma_rp_d:
+  case Intrinsic::nvvm_fma_rz_d:
+    return false;
+  }
+  llvm_unreachable("Checking FTZ flag for invalid NVVM fma intrinsic");
+}
+
+inline APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID) {
+  switch (IntrinsicID) {
+  case Intrinsic::nvvm_fma_rm_f:
+  case Intrinsic::nvvm_fma_rm_d:
+  case Intrinsic::nvvm_fma_rm_ftz_f:
+    return APFloat::rmTowardNegative;
+  case Intrinsic::nvvm_fma_rn_f:
+  case Intrinsic::nvvm_fma_rn_d:
+  case Intrinsic::nvvm_fma_rn_ftz_f:
+    return APFloat::rmNearestTiesToEven;
+  case Intrinsic::nvvm_fma_rp_f:
+  case Intrinsic::nvvm_fma_rp_d:
+  case Intrinsic::nvvm_fma_rp_ftz_f:
+    return APFloat::rmTowardPositive;
+  case Intrinsic::nvvm_fma_rz_f:
+  case Intrinsic::nvvm_fma_rz_d:
+  case Intrinsic::nvvm_fma_rz_ftz_f:
+    return APFloat::rmTowardZero;
+  }
+  llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM fma");
+}
+
 } // namespace nvvm
 } // namespace llvm
 #endif // LLVM_IR_NVVMINTRINSICUTILS_H
diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h
index 8344eaec807b3..10816c0e62c29 100644
--- a/llvm/include/llvm/IR/Operator.h
+++ b/llvm/include/llvm/IR/Operator.h
@@ -595,6 +595,37 @@ struct OperandTraits<PtrToIntOperator>
 
 DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PtrToIntOperator, Value)
 
+class PtrToAddrOperator
+    : public ConcreteOperator<Operator, Instruction::PtrToAddr> {
+  friend class PtrToAddr;
+  friend class ConstantExpr;
+
+public:
+  /// Transparently provide more efficient getOperand methods.
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  Value *getPointerOperand() { return getOperand(0); }
+  const Value *getPointerOperand() const { return getOperand(0); }
+
+  static unsigned getPointerOperandIndex() {
+    return 0U; // get index for modifying correct operand
+  }
+
+  /// Method to return the pointer operand as a PointerType.
+  Type *getPointerOperandType() const { return getPointerOperand()->getType(); }
+
+  /// Method to return the address space of the pointer operand.
+  unsigned getPointerAddressSpace() const {
+    return cast<PointerType>(getPointerOperandType())->getAddressSpace();
+  }
+};
+
+template <>
+struct OperandTraits<PtrToAddrOperator>
+    : public FixedNumOperandTraits<PtrToAddrOperator, 1> {};
+
+DEFINE_TRANSPARENT_OPERAND_ACCESSORS(PtrToAddrOperator, Value)
+
 class BitCastOperator
     : public ConcreteOperator<Operator, Instruction::BitCast> {
   friend class BitCastInst;
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 27c5d5ca08cd6..d68be03d4ba7d 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -1013,12 +1013,13 @@ struct bind_const_intval_ty {
   bind_const_intval_ty(uint64_t &V) : VR(V) {}
 
   template <typename ITy> bool match(ITy *V) const {
-    if (const auto *CV = dyn_cast<ConstantInt>(V))
-      if (CV->getValue().ule(UINT64_MAX)) {
-        VR = CV->getZExtValue();
-        return true;
-      }
-    return false;
+    const APInt *ConstInt;
+    if (!apint_match(ConstInt, /*AllowPoison=*/false).match(V))
+      return false;
+    if (ConstInt->getActiveBits() > 64)
+      return false;
+    VR = ConstInt->getZExtValue();
+    return true;
   }
 };
 
@@ -1327,6 +1328,45 @@ inline BinaryOp_match<LHS, RHS, Instruction::AShr> m_AShr(const LHS &L,
   return BinaryOp_match<LHS, RHS, Instruction::AShr>(L, R);
 }
 
+template <typename LHS_t, unsigned Opcode> struct ShiftLike_match {
+  LHS_t L;
+  uint64_t &R;
+
+  ShiftLike_match(const LHS_t &LHS, uint64_t &RHS) : L(LHS), R(RHS) {}
+
+  template <typename OpTy> bool match(OpTy *V) const {
+    if (auto *Op = dyn_cast<BinaryOperator>(V)) {
+      if (Op->getOpcode() == Opcode)
+        return m_ConstantInt(R).match(Op->getOperand(1)) &&
+               L.match(Op->getOperand(0));
+    }
+    // Interpreted as shiftop V, 0
+    R = 0;
+    return L.match(V);
+  }
+};
+
+/// Matches shl L, ConstShAmt or L itself (R will be set to zero in this case).
+template <typename LHS>
+inline ShiftLike_match<LHS, Instruction::Shl> m_ShlOrSelf(const LHS &L,
+                                                          uint64_t &R) {
+  return ShiftLike_match<LHS, Instruction::Shl>(L, R);
+}
+
+/// Matches lshr L, ConstShAmt or L itself (R will be set to zero in this case).
+template <typename LHS>
+inline ShiftLike_match<LHS, Instruction::LShr> m_LShrOrSelf(const LHS &L,
+                                                            uint64_t &R) {
+  return ShiftLike_match<LHS, Instruction::LShr>(L, R);
+}
+
+/// Matches ashr L, ConstShAmt or L itself (R will be set to zero in this case).
+template <typename LHS>
+inline ShiftLike_match<LHS, Instruction::AShr> m_AShrOrSelf(const LHS &L,
+                                                            uint64_t &R) {
+  return ShiftLike_match<LHS, Instruction::AShr>(L, R);
+}
+
 template <typename LHS_t, typename RHS_t, unsigned Opcode,
           unsigned WrapFlags = 0, bool Commutable = false>
 struct OverflowingBinaryOp_match {
diff --git a/llvm/include/llvm/IR/ProfDataUtils.h b/llvm/include/llvm/IR/ProfDataUtils.h
index ca56e4aa81575..404875285beae 100644
--- a/llvm/include/llvm/IR/ProfDataUtils.h
+++ b/llvm/include/llvm/IR/ProfDataUtils.h
@@ -144,6 +144,33 @@ LLVM_ABI bool extractProfTotalWeight(const Instruction &I,
 LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
                                bool IsExpected);
 
+/// downscale the given weights preserving the ratio. If the maximum value is
+/// not already known and not provided via \param KnownMaxCount , it will be
+/// obtained from \param Weights.
+LLVM_ABI SmallVector<uint32_t>
+downscaleWeights(ArrayRef<uint64_t> Weights,
+                 std::optional<uint64_t> KnownMaxCount = std::nullopt);
+
+/// Calculate what to divide by to scale counts.
+///
+/// Given the maximum count, calculate a divisor that will scale all the
+/// weights to strictly less than std::numeric_limits<uint32_t>::max().
+inline uint64_t calculateCountScale(uint64_t MaxCount) {
+  return MaxCount < std::numeric_limits<uint32_t>::max()
+             ? 1
+             : MaxCount / std::numeric_limits<uint32_t>::max() + 1;
+}
+
+/// Scale an individual branch count.
+///
+/// Scale a 64-bit weight down to 32-bits using \c Scale.
+///
+inline uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale) {
+  uint64_t Scaled = Count / Scale;
+  assert(Scaled <= std::numeric_limits<uint32_t>::max() && "overflow 32-bits");
+  return Scaled;
+}
+
 /// Specify that the branch weights for this terminator cannot be known at
 /// compile time. This should only be called by passes, and never as a default
 /// behavior in e.g. MDBuilder. The goal is to use this info to validate passes
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h
index f39e2e3c26900..cf3e6c7f0b3a2 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.h
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.h
@@ -77,17 +77,17 @@ struct RuntimeLibcallsInfo {
 
   /// Get the libcall routine name for the specified libcall.
   // FIXME: This should be removed. Only LibcallImpl should have a name.
-  const char *getLibcallName(RTLIB::Libcall Call) const {
+  StringRef getLibcallName(RTLIB::Libcall Call) const {
     return getLibcallImplName(LibcallImpls[Call]);
   }
 
   /// Get the libcall routine name for the specified libcall implementation.
-  // FIXME: Change to return StringRef
-  static const char *getLibcallImplName(RTLIB::LibcallImpl CallImpl) {
+  static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl) {
     if (CallImpl == RTLIB::Unsupported)
-      return nullptr;
-    return RuntimeLibcallImplNameTable[RuntimeLibcallNameOffsetTable[CallImpl]]
-        .data();
+      return StringRef();
+    return StringRef(RuntimeLibcallImplNameTable.getCString(
+                         RuntimeLibcallNameOffsetTable[CallImpl]),
+                     RuntimeLibcallNameSizeTable[CallImpl]);
   }
 
   /// Return the lowering's selection of implementation call for \p Call
@@ -119,9 +119,10 @@ struct RuntimeLibcallsInfo {
 
   /// Return a function name compatible with RTLIB::MEMCPY, or nullptr if fully
   /// unsupported.
-  const char *getMemcpyName() const {
-    if (const char *Memcpy = getLibcallName(RTLIB::MEMCPY))
-      return Memcpy;
+  StringRef getMemcpyName() const {
+    RTLIB::LibcallImpl Memcpy = getLibcallImpl(RTLIB::MEMCPY);
+    if (Memcpy != RTLIB::Unsupported)
+      return getLibcallImplName(Memcpy);
 
     // Fallback to memmove if memcpy isn't available.
     return getLibcallName(RTLIB::MEMMOVE);
@@ -132,11 +133,41 @@ struct RuntimeLibcallsInfo {
     return ImplToLibcall[Impl];
   }
 
+  /// Check if a function name is a recognized runtime call of any kind. This
+  /// does not consider if this call is available for any current compilation,
+  /// just that it is a known call somewhere. This returns the set of all
+  /// LibcallImpls which match the name; multiple implementations with the same
+  /// name may exist but differ in interpretation based on the target context.
+  ///
+  /// Generated by tablegen.
+  LLVM_ABI static inline iota_range<RTLIB::LibcallImpl>
+  lookupLibcallImplName(StringRef Name){
+  // Inlining the early exit on the string name appears to be worthwhile when
+  // querying a real set of symbols
+#define GET_LOOKUP_LIBCALL_IMPL_NAME_BODY
+#include "llvm/IR/RuntimeLibcalls.inc"
+#undef GET_LOOKUP_LIBCALL_IMPL_NAME_BODY
+  }
+
   /// Check if this is valid libcall for the current module, otherwise
   /// RTLIB::Unsupported.
-  LLVM_ABI RTLIB::LibcallImpl getSupportedLibcallImpl(StringRef FuncName) const;
+  LLVM_ABI RTLIB::LibcallImpl
+      getSupportedLibcallImpl(StringRef FuncName) const {
+    for (RTLIB::LibcallImpl Impl : lookupLibcallImplName(FuncName)) {
+      // FIXME: This should not depend on looking up ImplToLibcall, only the
+      // list of libcalls for the module.
+      RTLIB::LibcallImpl Recognized = LibcallImpls[ImplToLibcall[Impl]];
+      if (Recognized != RTLIB::Unsupported)
+        return Recognized;
+    }
+
+    return RTLIB::Unsupported;
+  }
 
 private:
+  LLVM_ABI static iota_range<RTLIB::LibcallImpl>
+  lookupLibcallImplNameImpl(StringRef Name);
+
   /// Stores the implementation choice for each each libcall.
   RTLIB::LibcallImpl LibcallImpls[RTLIB::UNKNOWN_LIBCALL + 1] = {
       RTLIB::Unsupported};
@@ -153,17 +184,16 @@ struct RuntimeLibcallsInfo {
   LLVM_ABI static const char RuntimeLibcallImplNameTableStorage[];
   LLVM_ABI static const StringTable RuntimeLibcallImplNameTable;
   LLVM_ABI static const uint16_t RuntimeLibcallNameOffsetTable[];
+  LLVM_ABI static const uint8_t RuntimeLibcallNameSizeTable[];
 
   /// Map from a concrete LibcallImpl implementation to its RTLIB::Libcall kind.
   LLVM_ABI static const RTLIB::Libcall ImplToLibcall[RTLIB::NumLibcallImpls];
 
-  /// Check if a function name is a recognized runtime call of any kind. This
-  /// does not consider if this call is available for any current compilation,
-  /// just that it is a known call somewhere. This returns the set of all
-  /// LibcallImpls which match the name; multiple implementations with the same
-  /// name may exist but differ in interpretation based on the target context.
-  LLVM_ABI static iterator_range<ArrayRef<uint16_t>::const_iterator>
-  getRecognizedLibcallImpls(StringRef FuncName);
+  /// Utility function for tablegenerated lookup function. Return a range of
+  /// enum values that apply for the function name at \p NameOffsetEntry with
+  /// the value \p StrOffset.
+  static inline iota_range<RTLIB::LibcallImpl>
+  libcallImplNameHit(uint16_t NameOffsetEntry, uint16_t StrOffset);
 
   static bool darwinHasSinCosStret(const Triple &TT) {
     if (!TT.isOSDarwin())
@@ -182,6 +212,14 @@ struct RuntimeLibcallsInfo {
     return true;
   }
 
+  static bool hasAEABILibcalls(const Triple &TT) {
+    return TT.isTargetAEABI() || TT.isTargetGNUAEABI() ||
+           TT.isTargetMuslAEABI() || TT.isAndroid();
+  }
+
+  LLVM_READONLY
+  static bool isAAPCS_ABI(const Triple &TT, StringRef ABIName);
+
   static bool darwinHasExp10(const Triple &TT);
 
   /// Return true if the target has sincosf/sincos/sincosl functions
@@ -196,7 +234,9 @@ struct RuntimeLibcallsInfo {
 
   /// Generated by tablegen.
   void setTargetRuntimeLibcallSets(const Triple &TT,
-                                   FloatABI::ABIType FloatABI);
+                                   ExceptionHandling ExceptionModel,
+                                   FloatABI::ABIType FloatABI, EABI ABIType,
+                                   StringRef ABIName);
 
   /// Set default libcall names. If a target wants to opt-out of a libcall it
   /// should be placed here.
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td
index df472d4b9cfee..24ea92cc6865d 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.td
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.td
@@ -21,6 +21,7 @@ def isOSOpenBSD : RuntimeLibcallPredicate<"TT.isOSOpenBSD()">;
 def isNotOSOpenBSD : RuntimeLibcallPredicate<"!TT.isOSOpenBSD()">;
 def isOSWindows : RuntimeLibcallPredicate<"TT.isOSWindows()">;
 def isNotOSWindows : RuntimeLibcallPredicate<"!TT.isOSWindows()">;
+def isNotOSLinux : RuntimeLibcallPredicate<[{!TT.isOSLinux()}]>;
 def isNotOSMSVCRT : RuntimeLibcallPredicate<"!TT.isOSMSVCRT()">;
 def isPS : RuntimeLibcallPredicate<"TT.isPS()">;
 def isNotOSWindowsOrIsCygwinMinGW
@@ -28,6 +29,12 @@ def isNotOSWindowsOrIsCygwinMinGW
 def isWindowsMSVCEnvironment : RuntimeLibcallPredicate<
   [{TT.isWindowsMSVCEnvironment()}]>;
 
+def isNotOSLinuxAndNotOSOpenBSD : RuntimeLibcallPredicate<
+  [{!TT.isOSLinux() && !TT.isOSOpenBSD()}]>;
+
+def isWindowsMSVCOrItaniumEnvironment : RuntimeLibcallPredicate<
+  [{TT.isWindowsMSVCEnvironment() || TT.isWindowsItaniumEnvironment()}]>;
+
 def isGNUEnvironment : RuntimeLibcallPredicate<"TT.isGNUEnvironment()">;
 def darwinHasSinCosStret : RuntimeLibcallPredicate<"darwinHasSinCosStret(TT)">;
 def darwinHasExp10 : RuntimeLibcallPredicate<"darwinHasExp10(TT)">;
@@ -39,6 +46,14 @@ def hasSinCos : RuntimeLibcallPredicate<"hasSinCos(TT)">;
 // FIXME: Way to combine predicates
 def hasSinCos_f32_f64 : RuntimeLibcallPredicate<"hasSinCos_f32_f64(TT)">;
 
+def ExceptionModelIsNotNone : RuntimeLibcallPredicate<
+  [{ExceptionModel != ExceptionHandling::None}]
+>;
+
+def ExceptionModelIsSjLj : RuntimeLibcallPredicate<
+  [{ExceptionModel == ExceptionHandling::SjLj}]
+>;
+
 //--------------------------------------------------------------------
 // Declare all kinds of used libcalls
 //--------------------------------------------------------------------
@@ -276,6 +291,7 @@ foreach FPTy = ["F32", "F64", "F128", "PPCF128"] in {
 }
 
 // Memory
+def MEMCMP : RuntimeLibcall;
 def MEMCPY : RuntimeLibcall;
 def MEMMOVE : RuntimeLibcall;
 def MEMSET : RuntimeLibcall;
@@ -405,6 +421,17 @@ multiclass LibmLongDoubleLibCall<string libcall_basename = !toupper(NAME),
 def SC_MEMCPY : RuntimeLibcall;
 def SC_MEMMOVE : RuntimeLibcall;
 def SC_MEMSET : RuntimeLibcall;
+def SC_MEMCHR: RuntimeLibcall;
+
+// AArch64 SME ABI calls
+def SMEABI_SME_STATE : RuntimeLibcall;
+def SMEABI_TPIDR2_SAVE : RuntimeLibcall;
+def SMEABI_ZA_DISABLE : RuntimeLibcall;
+def SMEABI_TPIDR2_RESTORE : RuntimeLibcall;
+def SMEABI_GET_CURRENT_VG : RuntimeLibcall;
+def SMEABI_SME_STATE_SIZE : RuntimeLibcall;
+def SMEABI_SME_SAVE : RuntimeLibcall;
+def SMEABI_SME_RESTORE : RuntimeLibcall;
 
 // ARM EABI calls
 def AEABI_MEMCPY4 : RuntimeLibcall; // Align 4
@@ -451,6 +478,16 @@ def OBJC_RETAIN_AUTORELEASE : RuntimeLibcall;
 def OBJC_SYNC_ENTER : RuntimeLibcall;
 def OBJC_SYNC_EXIT : RuntimeLibcall;
 
+//--------------------------------------------------------------------
+// Global variable references
+//--------------------------------------------------------------------
+//
+// TODO: These are not libcalls and probably should be distinguished
+// in some way from callable functions.
+// --------------------------------------------------------------------
+
+def STACK_CHECK_GUARD : RuntimeLibcall;
+
 //--------------------------------------------------------------------
 // Define implementation default libcalls
 //--------------------------------------------------------------------
@@ -695,10 +732,6 @@ foreach MemSize = [ 1, 2, 4, 8, 16 ] in {
     !cast<RuntimeLibcall>("MEMSET_ELEMENT_UNORDERED_ATOMIC_"#MemSize)>;
 }
 
-// Exception handling
-def _Unwind_Resume : RuntimeLibcallImpl<UNWIND_RESUME>;
-def __cxa_end_cleanup : RuntimeLibcallImpl<CXA_END_CLEANUP>;
-
 // Atomic '__sync_*' libcalls.
 foreach lc = LibCalls__sync in {
   def __#!tolower(!cast<string>(lc)) : RuntimeLibcallImpl<lc>;
@@ -1002,9 +1035,20 @@ defm sincos : LibmLongDoubleLibCall;
 
 def bzero : RuntimeLibcallImpl<BZERO>;
 def __bzero : RuntimeLibcallImpl<BZERO>;
-def _Unwind_SjLj_Resume : RuntimeLibcallImpl<UNWIND_RESUME>;
-def _Unwind_SjLj_Register : RuntimeLibcallImpl<UNWIND_REGISTER>;
-def _Unwind_SjLj_Unregister : RuntimeLibcallImpl<UNWIND_UNREGISTER>;
+
+// Exception handling
+defset list<RuntimeLibcallImpl> DefaultExceptionHandlingLibcalls = {
+  def _Unwind_Resume : RuntimeLibcallImpl<UNWIND_RESUME>;
+  def __cxa_end_cleanup : RuntimeLibcallImpl<CXA_END_CLEANUP>;
+}
+
+defset list<RuntimeLibcallImpl> SjLjExceptionHandlingLibcalls = {
+  def _Unwind_SjLj_Resume : RuntimeLibcallImpl<UNWIND_RESUME>;
+  def _Unwind_SjLj_Register : RuntimeLibcallImpl<UNWIND_REGISTER>;
+  def _Unwind_SjLj_Unregister : RuntimeLibcallImpl<UNWIND_UNREGISTER>;
+}
+
+// Only used on wasm?
 def _Unwind_CallPersonality : RuntimeLibcallImpl<UNWIND_CALL_PERSONALITY>;
 
 // Used on OpenBSD
@@ -1076,10 +1120,32 @@ defset list<RuntimeLibcallImpl> LibmF128FiniteLibcalls = {
   def __powf128_finite : RuntimeLibcallImpl<POW_FINITE_F128>;
 }
 
+//--------------------------------------------------------------------
+// Global variable references
+//--------------------------------------------------------------------
+
+def __stack_chk_guard : RuntimeLibcallImpl<STACK_CHECK_GUARD>;
+
+// Name used on OpenBSD
+def __guard_local : RuntimeLibcallImpl<STACK_CHECK_GUARD>;
+
+// Name used with Windows MSVC
+def __security_cookie : RuntimeLibcallImpl<STACK_CHECK_GUARD>;
+
+// Name used on AIX
+def __ssp_canary_word : RuntimeLibcallImpl<STACK_CHECK_GUARD>;
+
 //===----------------------------------------------------------------------===//
 // Common Libcall Sets
 //===----------------------------------------------------------------------===//
 
+defvar ExceptionModelCalls = (add
+  LibcallImpls<(add DefaultExceptionHandlingLibcalls),
+               ExceptionModelIsNotNone>,
+  LibcallImpls<(add SjLjExceptionHandlingLibcalls),
+               ExceptionModelIsSjLj>
+);
+
 // FIXME: Should move to explicit opt-in to different sets of libcalls
 // instead of trying to remove from a default set. We have
 // unreasonable defaults like reporting f80 calls on most targets when
@@ -1100,26 +1166,33 @@ defvar DefaultRuntimeLibcallImpls_f128 =
     !filter(entry, AllDefaultRuntimeLibcallImpls,
             !match(!cast<string>(entry.Provides), "_F128"));
 
-defvar DefaultRuntimeLibcallImpls =
+// FIXME: Ideally we would just use dags everywhere, but for the
+// arm64ec case we need iterable lists so we can add the # prefix
+defvar DefaultRuntimeLibcallImplsBaseList =
 !listremove(
   !listremove(
     !listremove(AllDefaultRuntimeLibcallImpls, Int128RTLibcalls),
                 DefaultRuntimeLibcallImpls_f80),
                 DefaultRuntimeLibcallImpls_ppcf128);
 
+defvar DefaultRuntimeLibcallImpls =
+  (add DefaultRuntimeLibcallImplsBaseList,
+       ExceptionModelCalls);
+
 /// Default set of libcall impls for 32-bit architectures.
-defvar DefaultLibcallImpls32 = DefaultRuntimeLibcallImpls;
+defvar DefaultLibcallImpls32 = (add DefaultRuntimeLibcallImpls);
 
 /// Default set of libcall impls for 64-bit architectures.
-defvar DefaultLibcallImpls64 = !listconcat(DefaultRuntimeLibcallImpls,
-                                           Int128RTLibcalls);
+defvar DefaultLibcallImpls64 = (add DefaultRuntimeLibcallImpls,
+                                    Int128RTLibcalls);
 
 defvar DarwinSinCosStret = LibcallImpls<(add __sincosf_stret, __sincos_stret),
                                         darwinHasSinCosStret>;
 defvar DarwinExp10 = LibcallImpls<(add __exp10f, __exp10), darwinHasExp10>;
 
 defvar SecurityCheckCookieIfWinMSVC =
-    LibcallImpls<(add __security_check_cookie), isWindowsMSVCEnvironment>;
+    LibcallImpls<(add __security_check_cookie, __security_cookie),
+                 isWindowsMSVCOrItaniumEnvironment>;
 
 defvar LibmHasSinCosF32 = LibcallImpls<(add sincosf), hasSinCos>;
 defvar LibmHasSinCosF64 =  LibcallImpls<(add sincos), hasSinCos>;
@@ -1148,8 +1221,12 @@ defvar WindowsExclusions = !listconcat(WindowsMathRemovals, MostPowI);
 
 // Targets which support windows should start with these as a base and
 // add in calls for other OSes
-defvar Win32DefaultLibcallImpls = !listremove(DefaultLibcallImpls32, WindowsExclusions);
-defvar Win64DefaultLibcallImpls = !listremove(DefaultLibcallImpls64, WindowsExclusions);
+defvar WinDefaultLibcallImplsBaseList =
+  !listremove(DefaultRuntimeLibcallImplsBaseList,
+              WindowsExclusions);
+
+defvar WinDefaultLibcallImpls = (add WinDefaultLibcallImplsBaseList,
+                                     ExceptionModelCalls);
 
 defvar LibmHasFrexpF32 = LibcallImpls<(add frexpf), isNotOSWindowsOrIsCygwinMinGW>;
 defvar LibmHasLdexpF32 = LibcallImpls<(add ldexpf), isNotOSWindowsOrIsCygwinMinGW>;
@@ -1161,10 +1238,13 @@ defvar LibmHasFrexpF128 = LibcallImpls<(add frexp_f128), isNotOSWindowsOrIsCygwi
 defvar LibmHasLdexpF128 = LibcallImpls<(add ldexp_f128), isNotOSWindowsOrIsCygwinMinGW>;
 
 defvar has__stack_chk_fail = LibcallImpls<(add __stack_chk_fail), isNotOSOpenBSD>;
+defvar has__stack_chk_guard =
+    LibcallImpls<(add __stack_chk_guard), isNotOSOpenBSD>;
 defvar has__stack_smash_handler = LibcallImpls<(add __stack_smash_handler), isOSOpenBSD>;
+defvar has___guard_local = LibcallImpls<(add __guard_local), isOSOpenBSD>;
 
-defvar DefaultStackProtector = (add has__stack_chk_fail,
-                                    has__stack_smash_handler);
+defvar DefaultStackProtector = (add has__stack_chk_fail, has__stack_chk_guard,
+    has__stack_smash_handler, has___guard_local);
 
 //===----------------------------------------------------------------------===//
 // Objective-C Runtime Libcalls
@@ -1223,8 +1303,35 @@ defset list<RuntimeLibcallImpl> AArch64LibcallImpls = {
   def __arm_sc_memcpy : RuntimeLibcallImpl<SC_MEMCPY>;
   def __arm_sc_memmove : RuntimeLibcallImpl<SC_MEMMOVE>;
   def __arm_sc_memset : RuntimeLibcallImpl<SC_MEMSET>;
+  def __arm_sc_memchr : RuntimeLibcallImpl<SC_MEMCHR>;
 } // End AArch64LibcallImpls
 
+def __arm_sme_state : RuntimeLibcallImpl<SMEABI_SME_STATE>;
+def __arm_tpidr2_save : RuntimeLibcallImpl<SMEABI_TPIDR2_SAVE>;
+def __arm_za_disable : RuntimeLibcallImpl<SMEABI_ZA_DISABLE>;
+def __arm_tpidr2_restore : RuntimeLibcallImpl<SMEABI_TPIDR2_RESTORE>;
+def __arm_get_current_vg : RuntimeLibcallImpl<SMEABI_GET_CURRENT_VG>;
+def __arm_sme_state_size : RuntimeLibcallImpl<SMEABI_SME_STATE_SIZE>;
+def __arm_sme_save : RuntimeLibcallImpl<SMEABI_SME_SAVE>;
+def __arm_sme_restore : RuntimeLibcallImpl<SMEABI_SME_RESTORE>;
+
+def SMEABI_LibCalls_PreserveMost_From_X0 : LibcallsWithCC<(add
+  __arm_tpidr2_save,
+  __arm_za_disable,
+  __arm_tpidr2_restore),
+  SMEABI_PreserveMost_From_X0>;
+
+def SMEABI_LibCalls_PreserveMost_From_X1 : LibcallsWithCC<(add
+  __arm_get_current_vg,
+  __arm_sme_state_size,
+  __arm_sme_save,
+  __arm_sme_restore),
+  SMEABI_PreserveMost_From_X1>;
+
+def SMEABI_LibCalls_PreserveMost_From_X2 : LibcallsWithCC<(add
+  __arm_sme_state),
+  SMEABI_PreserveMost_From_X2>;
+
 def isAArch64_ExceptArm64EC
     : RuntimeLibcallPredicate<"(TT.isAArch64() && !TT.isWindowsArm64EC())">;
 def isWindowsArm64EC : RuntimeLibcallPredicate<"TT.isWindowsArm64EC()">;
@@ -1233,7 +1340,7 @@ def isAArch64_ILP64  : RuntimeLibcallPredicate<"TT.isAArch64(64)">;
 
 def AArch64SystemLibrary : SystemRuntimeLibrary<
   isAArch64_ExceptArm64EC,
-  (add Win64DefaultLibcallImpls,
+  (add WinDefaultLibcallImpls,
        LibcallImpls<(add __powisf2, __powidf2), isNotOSMSVCRT>,
        LibmHasFrexpF32, LibmHasLdexpF32,
        LibmHasFrexpF128, LibmHasLdexpF128,
@@ -1244,12 +1351,15 @@ def AArch64SystemLibrary : SystemRuntimeLibrary<
        LibmHasSinCosF32, LibmHasSinCosF64, LibmHasSinCosF128,
        DefaultLibmExp10,
        DefaultStackProtector,
-       SecurityCheckCookieIfWinMSVC)
+       SecurityCheckCookieIfWinMSVC,
+       SMEABI_LibCalls_PreserveMost_From_X0,
+       SMEABI_LibCalls_PreserveMost_From_X1,
+       SMEABI_LibCalls_PreserveMost_From_X2)
 >;
 
 // Prepend a # to every name
 defset list<RuntimeLibcallImpl> WinArm64ECDefaultRuntimeLibcallImpls = {
-  foreach libcall = Win64DefaultLibcallImpls in {
+  foreach libcall = WinDefaultLibcallImplsBaseList in {
     def arm64ec_#libcall : DuplicateLibcallImplWithPrefix<libcall, "#">;
   }
 
@@ -1260,12 +1370,34 @@ defset list<RuntimeLibcallImpl> WinArm64ECDefaultRuntimeLibcallImpls = {
 
 def arm64ec___stack_chk_fail : DuplicateLibcallImplWithPrefix<__stack_chk_fail, "#">;
 
+defset list<RuntimeLibcallImpl> WinArm64ECDefaultExceptionHandlingLibcalls = {
+  foreach libcall = DefaultExceptionHandlingLibcalls in {
+    def arm64ec_#libcall : DuplicateLibcallImplWithPrefix<libcall, "#">;
+  }
+}
+
+defset list<RuntimeLibcallImpl> WinArm64ECSjLjExceptionHandlingLibcalls = {
+  foreach libcall = SjLjExceptionHandlingLibcalls in {
+    def arm64ec_#libcall : DuplicateLibcallImplWithPrefix<libcall, "#">;
+  }
+}
+
+defvar ExceptionModelCallsArm64EC = (add
+  LibcallImpls<(add WinArm64ECDefaultExceptionHandlingLibcalls),
+               ExceptionModelIsNotNone>,
+  LibcallImpls<(add WinArm64ECSjLjExceptionHandlingLibcalls),
+               ExceptionModelIsSjLj>
+);
+
 def WindowsARM64ECSystemLibrary
     : SystemRuntimeLibrary<isWindowsArm64EC,
                            (add WinArm64ECDefaultRuntimeLibcallImpls,
                                 arm64ec___stack_chk_fail,
-                                LibcallImpls<(add __security_check_cookie_arm64ec),
-                                              isWindowsMSVCEnvironment>)>;
+                                __stack_chk_guard,
+                                LibcallImpls<(add __security_check_cookie_arm64ec,
+                                                  __security_cookie),
+                                              isWindowsMSVCEnvironment>,
+                                ExceptionModelCallsArm64EC)>;
 
 //===----------------------------------------------------------------------===//
 // AMDGPU Runtime Libcalls
@@ -1280,6 +1412,9 @@ def AMDGPUSystemLibrary : SystemRuntimeLibrary<isAMDGPU, (add)>;
 // ARM Runtime Libcalls
 //===----------------------------------------------------------------------===//
 
+def isTargetAEABIAndAAPCS_ABI : RuntimeLibcallPredicate<
+  [{TT.isTargetAEABI() && isAAPCS_ABI(TT, ABIName)}]>;
+
 // if (isTargetMachO()) {
 // if (Subtarget->isThumb() && Subtarget->hasVFP2Base() &&
 //  Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) {
@@ -1473,13 +1608,33 @@ def __udivmodsi4 : RuntimeLibcallImpl<UDIVREM_I32>;
 // a __gnu_ prefix (which is the default).
 // isTargetAEABI()
 def __aeabi_f2h : RuntimeLibcallImpl<FPROUND_F32_F16>; // CallingConv::ARM_AAPCS
-//def __aeabi_d2h : RuntimeLibcallImpl<FPROUND_F64_F16>; // CallingConv::ARM_AAPCS
 def __aeabi_h2f : RuntimeLibcallImpl<FPEXT_F16_F32>; // CallingConv::ARM_AAPCS
 
 // !isTargetMachO()
 def __gnu_f2h_ieee : RuntimeLibcallImpl<FPROUND_F32_F16>;
 def __gnu_h2f_ieee : RuntimeLibcallImpl<FPEXT_F16_F32>;
 
+// In EABI, these functions have an __aeabi_ prefix, but in GNUEABI
+// they have a __gnu_ prefix (which is the default).
+def EABIHalfConvertCalls : LibcallImpls<(add __aeabi_f2h, __aeabi_h2f),
+                                        isTargetAEABIAndAAPCS_ABI> {
+  let CallingConv = ARM_AAPCS;
+}
+
+// The half <-> float conversion functions are always soft-float on
+// non-watchos platforms, but are needed for some targets which use a
+// hard-float calling convention by default.
+def ARMHalfConvertLibcallCallingConv : LibcallCallingConv<
+  [{TT.isWatchABI() ? DefaultCC :
+    (isAAPCS_ABI(TT, ABIName) ? CallingConv::ARM_AAPCS : CallingConv::ARM_APCS)}]
+>;
+
+def GNUEABIHalfConvertCalls :
+  LibcallImpls<(add __gnu_f2h_ieee, __gnu_h2f_ieee),
+    RuntimeLibcallPredicate<[{!TT.isOSBinFormatMachO() &&
+                              !TT.isTargetAEABI()}]>> {
+  let CallingConv = ARMHalfConvertLibcallCallingConv;
+}
 
 def WindowARMDivRemCalls : LibcallImpls<
   (add __rt_sdiv, __rt_sdiv64, __rt_udiv, __rt_udiv64),
@@ -1498,8 +1653,101 @@ def WindowARMFPIntCasts : LibcallImpls<
 def AEABIDivRemCalls : LibcallImpls<
   (add __aeabi_idivmod, __aeabi_ldivmod,
        __aeabi_uidivmod, __aeabi_uldivmod),
-  RuntimeLibcallPredicate<[{TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
-    TT.isTargetMuslAEABI()}]>> {
+  RuntimeLibcallPredicate<[{hasAEABILibcalls(TT)}]>> {
+  let CallingConv = ARM_AAPCS;
+}
+
+def AEABICalls : LibcallImpls<
+  (add
+      // Double-precision floating-point arithmetic helper functions
+      // RTABI chapter 4.1.2, Table 2
+      __aeabi_dadd,
+      __aeabi_ddiv,
+      __aeabi_dmul,
+      __aeabi_dsub,
+
+      // Double-precision floating-point comparison helper functions
+      // RTABI chapter 4.1.2, Table 3
+      __aeabi_dcmpeq__oeq,
+      __aeabi_dcmpeq__une,
+      __aeabi_dcmplt,
+      __aeabi_dcmple,
+      __aeabi_dcmpge,
+      __aeabi_dcmpgt,
+       __aeabi_dcmpun,
+
+      // Single-precision floating-point arithmetic helper functions
+      // RTABI chapter 4.1.2, Table 4
+      __aeabi_fadd,
+      __aeabi_fdiv,
+      __aeabi_fmul,
+      __aeabi_fsub,
+
+      // Single-precision floating-point comparison helper functions
+      // RTABI chapter 4.1.2, Table 5
+      __aeabi_fcmpeq__oeq,
+      __aeabi_fcmpeq__une,
+      __aeabi_fcmplt,
+      __aeabi_fcmple,
+      __aeabi_fcmpge,
+      __aeabi_fcmpgt,
+      __aeabi_fcmpun,
+
+      // Floating-point to integer conversions.
+      // RTABI chapter 4.1.2, Table 6
+      __aeabi_d2iz,
+      __aeabi_d2uiz,
+      __aeabi_d2lz,
+      __aeabi_d2ulz,
+      __aeabi_f2iz,
+      __aeabi_f2uiz,
+      __aeabi_f2lz,
+      __aeabi_f2ulz,
+
+      // Conversions between floating types.
+      // RTABI chapter 4.1.2, Table 7
+      __aeabi_d2f,
+      __aeabi_f2d,
+      //__aeabi_h2f added separately
+      //__aeabi_f2h added separately
+      __aeabi_d2h,
+
+      // Integer to floating-point conversions.
+      // RTABI chapter 4.1.2, Table 8
+      __aeabi_i2d,
+      __aeabi_ui2d,
+      __aeabi_l2d,
+      __aeabi_ul2d,
+      __aeabi_i2f,
+      __aeabi_ui2f,
+      __aeabi_l2f,
+      __aeabi_ul2f,
+
+      // Long long helper functions
+      // RTABI chapter 4.2, Table 9
+      __aeabi_lmul,
+      __aeabi_llsl,
+      __aeabi_llsr,
+      __aeabi_lasr,
+
+      // Integer division functions
+      // RTABI chapter 4.3.1
+      __aeabi_idiv,
+      __aeabi_uidiv),
+  RuntimeLibcallPredicate<[{hasAEABILibcalls(TT) && isAAPCS_ABI(TT, ABIName)}]>> {
+  let CallingConv = ARM_AAPCS;
+}
+
+// EABI dependent RTLIB, Memory operations
+// RTABI chapter 4.3.4
+def AEABI45MemCalls : LibcallImpls<
+  (add __aeabi_memcpy,  __aeabi_memcpy4, __aeabi_memcpy8,
+       __aeabi_memmove, __aeabi_memmove4, __aeabi_memmove8,
+       __aeabi_memset, __aeabi_memset4, __aeabi_memset8,
+       __aeabi_memclr, __aeabi_memclr4, __aeabi_memclr8),
+  RuntimeLibcallPredicate<[{(EABIVersion == EABI::EABI4 ||
+                             EABIVersion == EABI::EABI5) &&
+                             hasAEABILibcalls(TT) && isAAPCS_ABI(TT, ABIName)}]>> {
   let CallingConv = ARM_AAPCS;
 }
 
@@ -1507,7 +1755,7 @@ def isARMOrThumb : RuntimeLibcallPredicate<"TT.isARM() || TT.isThumb()">;
 
 def ARMSystemLibrary
     : SystemRuntimeLibrary<isARMOrThumb,
-      (add Win32DefaultLibcallImpls,
+      (add WinDefaultLibcallImpls,
            LibcallImpls<(add __powisf2, __powidf2), isNotOSMSVCRT>,
            LibmHasFrexpF32, LibmHasLdexpF32,
            LibmHasFrexpF128, LibmHasLdexpF128,
@@ -1519,6 +1767,11 @@ def ARMSystemLibrary
            LibmHasSinCosF32, LibmHasSinCosF64, LibmHasSinCosF128,
            DefaultLibmExp10,
 
+           AEABICalls,
+           AEABI45MemCalls,
+           EABIHalfConvertCalls,
+           GNUEABIHalfConvertCalls,
+
            // Use divmod compiler-rt calls for iOS 5.0 and later.
            LibcallImpls<(add __divmodsi4, __udivmodsi4),
                         RuntimeLibcallPredicate<[{TT.isOSBinFormatMachO() &&
@@ -1633,7 +1886,7 @@ def HexagonSystemLibrary
     __umoddi3, __divdf3, __muldf3, __divsi3, __subdf3, sqrtf,
     __divdi3, __umodsi3, __moddi3, __modsi3), HexagonLibcalls,
     LibmHasSinCosF32, LibmHasSinCosF64, LibmHasSinCosF128,
-    exp10f, exp10, exp10l_f128, __stack_chk_fail)>;
+    exp10f, exp10, exp10l_f128, __stack_chk_fail, __stack_chk_guard)>;
 
 //===----------------------------------------------------------------------===//
 // Lanai Runtime Libcalls
@@ -1644,7 +1897,7 @@ def isLanai : RuntimeLibcallPredicate<"TT.getArch() == Triple::lanai">;
 // Use fast calling convention for library functions.
 def LanaiSystemLibrary
     : SystemRuntimeLibrary<isLanai, (add DefaultRuntimeLibcallImpls,
-                                         __stack_chk_fail)> {
+                                        __stack_chk_fail, __stack_chk_guard)> {
   let DefaultLibcallCallingConv = FASTCC;
 }
 
@@ -1932,6 +2185,9 @@ def MSP430SystemLibrary
       __mspabi_remul,
       LibcallsWithCC<(add __mspabi_remull), MSP430_BUILTIN>,
 
+      // May not be used depending on subtarget
+      LibcallsWithCC<(add __mspabi_mpyll), MSP430_BUILTIN>,
+
       // Bitwise Operations - EABI Table 10
       // TODO: __mspabi_[srli/srai/slli] ARE implemented in libgcc
       __mspabi_srll,
@@ -1939,7 +2195,8 @@ def MSP430SystemLibrary
       __mspabi_slll,
       // __mspabi_[srlll/srall/sllll/rlli/rlll] are NOT implemented in libgcc
 
-      __stack_chk_fail
+      __stack_chk_fail,
+      __stack_chk_guard
   )
 >;
 
@@ -1990,12 +2247,14 @@ defset list<RuntimeLibcallImpl> PPCRuntimeLibcalls = {
 }
 
 defset list<RuntimeLibcallImpl> PPC64AIXCallList = {
+  def ___memcmp64 : RuntimeLibcallImpl<MEMCMP>;
   def ___memmove64 : RuntimeLibcallImpl<MEMCPY>;
   def ___memset64 : RuntimeLibcallImpl<MEMSET>;
   def ___bzero64 : RuntimeLibcallImpl<BZERO>;
 }
 
 defset list<RuntimeLibcallImpl> PPC32AIXCallList = {
+  def ___memcmp : RuntimeLibcallImpl<MEMCMP>;
   def ___memmove : RuntimeLibcallImpl<MEMMOVE>;
   def ___memset : RuntimeLibcallImpl<MEMSET>;
   def ___bzero : RuntimeLibcallImpl<BZERO>;
@@ -2031,7 +2290,11 @@ def PPCSystemLibrary
            LibmHasSinCosPPCF128,
            AvailableIf<memcpy, isNotAIX>,
            LibcallImpls<(add Int128RTLibcalls), isPPC64>,
-           DefaultStackProtector)>;
+           has__stack_chk_fail,
+           has__stack_smash_handler,
+           has___guard_local,
+           AvailableIf<__ssp_canary_word, isAIX>,
+           AvailableIf<__stack_chk_guard, isNotAIX>)>;
 
 //===----------------------------------------------------------------------===//
 // RISCV Runtime Libcalls
@@ -2114,7 +2377,10 @@ def SPARCSystemLibrary
        LibcallImpls<(add _Q_qtoll, _Q_qtoull, _Q_lltoq, _Q_ulltoq), isSPARC32>,
        LibcallImpls<(add SPARC64_MulDivCalls, Int128RTLibcalls), isSPARC64>,
        LibmHasSinCosF32, LibmHasSinCosF64, LibmHasSinCosF128,
-       DefaultStackProtector)
+       has__stack_chk_fail,
+       has__stack_smash_handler,
+       has___guard_local,
+       AvailableIf<__stack_chk_guard, isNotOSLinuxAndNotOSOpenBSD>)
 >;
 
 //===----------------------------------------------------------------------===//
@@ -2161,7 +2427,8 @@ defvar X86_F128_Libcalls = LibcallImpls<(add LibmF128Libcalls, LibmF128FiniteLib
 defvar SinCosF32F64Libcalls = LibcallImpls<(add sincosf, sincos), hasSinCos_f32_f64>;
 
 defvar X86CommonLibcalls =
-  (add DarwinSinCosStret, DarwinExp10,
+  (add WinDefaultLibcallImpls,
+       DarwinSinCosStret, DarwinExp10,
        X86_F128_Libcalls,
        LibmHasSinCosF80, // FIXME: Depends on long double
        SinCosF32F64Libcalls,
@@ -2177,7 +2444,7 @@ defvar X86CommonLibcalls =
        // hack for one test relying on it.
        __powitf2_f128,
        DefaultStackProtector
-);
+     );
 
 defvar Windows32DivRemMulCalls =
   LibcallsWithCC<(add WindowsDivRemMulLibcalls), X86_STDCALL,
@@ -2186,11 +2453,11 @@ defvar Windows32DivRemMulCalls =
 def X86_32SystemLibrary
     : SystemRuntimeLibrary<isX86_32,
       (add X86CommonLibcalls,
-           Windows32DivRemMulCalls, Win32DefaultLibcallImpls)>;
+           Windows32DivRemMulCalls)>;
 
 def X86_64SystemLibrary
     : SystemRuntimeLibrary<isX86_64,
-      (add X86CommonLibcalls, Win64DefaultLibcallImpls, Int128RTLibcalls)>;
+      (add X86CommonLibcalls, Int128RTLibcalls)>;
 
 //===----------------------------------------------------------------------===//
 // XCore Runtime Libcalls
@@ -2321,8 +2588,9 @@ def WasmSystemLibrary
       (add DefaultRuntimeLibcallImpls, Int128RTLibcalls,
            CompilerRTOnlyInt64Libcalls, CompilerRTOnlyInt128Libcalls,
            exp10f, exp10,
+           _Unwind_CallPersonality,
            emscripten_return_address,
-           __stack_chk_fail)>;
+           __stack_chk_fail, __stack_chk_guard)>;
 
 //===----------------------------------------------------------------------===//
 // Legacy Default Runtime Libcalls
diff --git a/llvm/include/llvm/IR/RuntimeLibcallsImpl.td b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
index 601c291daf89d..b5752c1b69ad8 100644
--- a/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
+++ b/llvm/include/llvm/IR/RuntimeLibcallsImpl.td
@@ -36,6 +36,9 @@ def ARM_AAPCS : LibcallCallingConv<[{CallingConv::ARM_AAPCS}]>;
 def ARM_AAPCS_VFP : LibcallCallingConv<[{CallingConv::ARM_AAPCS_VFP}]>;
 def X86_STDCALL : LibcallCallingConv<[{CallingConv::X86_StdCall}]>;
 def AVR_BUILTIN : LibcallCallingConv<[{CallingConv::AVR_BUILTIN}]>;
+def SMEABI_PreserveMost_From_X0 : LibcallCallingConv<[{CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0}]>;
+def SMEABI_PreserveMost_From_X1 : LibcallCallingConv<[{CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1}]>;
+def SMEABI_PreserveMost_From_X2 : LibcallCallingConv<[{CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2}]>;
 
 /// Abstract definition for functionality the compiler may need to
 /// emit a call to. Emits the RTLIB::Libcall enum - This enum defines
diff --git a/llvm/include/llvm/IR/VPIntrinsics.def b/llvm/include/llvm/IR/VPIntrinsics.def
index 4a71097226f18..cd7343ff8df56 100644
--- a/llvm/include/llvm/IR/VPIntrinsics.def
+++ b/llvm/include/llvm/IR/VPIntrinsics.def
@@ -278,24 +278,28 @@ END_REGISTER_VP(vp_fshr, VP_FSHR)
 
 // llvm.vp.sadd.sat(x,y,mask,vlen)
 BEGIN_REGISTER_VP(vp_sadd_sat, 2, 3, VP_SADDSAT, -1)
+VP_PROPERTY_BINARYOP
 VP_PROPERTY_FUNCTIONAL_INTRINSIC(sadd_sat)
 VP_PROPERTY_FUNCTIONAL_SDOPC(SADDSAT)
 END_REGISTER_VP(vp_sadd_sat, VP_SADDSAT)
 
 // llvm.vp.uadd.sat(x,y,mask,vlen)
 BEGIN_REGISTER_VP(vp_uadd_sat, 2, 3, VP_UADDSAT, -1)
+VP_PROPERTY_BINARYOP
 VP_PROPERTY_FUNCTIONAL_INTRINSIC(uadd_sat)
 VP_PROPERTY_FUNCTIONAL_SDOPC(UADDSAT)
 END_REGISTER_VP(vp_uadd_sat, VP_UADDSAT)
 
 // llvm.vp.ssub.sat(x,y,mask,vlen)
 BEGIN_REGISTER_VP(vp_ssub_sat, 2, 3, VP_SSUBSAT, -1)
+VP_PROPERTY_BINARYOP
 VP_PROPERTY_FUNCTIONAL_INTRINSIC(ssub_sat)
 VP_PROPERTY_FUNCTIONAL_SDOPC(SSUBSAT)
 END_REGISTER_VP(vp_ssub_sat, VP_SSUBSAT)
 
 // llvm.vp.usub.sat(x,y,mask,vlen)
 BEGIN_REGISTER_VP(vp_usub_sat, 2, 3, VP_USUBSAT, -1)
+VP_PROPERTY_BINARYOP
 VP_PROPERTY_FUNCTIONAL_INTRINSIC(usub_sat)
 VP_PROPERTY_FUNCTIONAL_SDOPC(USUBSAT)
 END_REGISTER_VP(vp_usub_sat, VP_USUBSAT)
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 31801daa126ad..a99b0ff188ea8 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -85,6 +85,7 @@ LLVM_ABI void initializeCycleInfoWrapperPassPass(PassRegistry &);
 LLVM_ABI void initializeDAEPass(PassRegistry &);
 LLVM_ABI void initializeDAHPass(PassRegistry &);
 LLVM_ABI void initializeDCELegacyPassPass(PassRegistry &);
+LLVM_ABI void initializeDSELegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeDXILMetadataAnalysisWrapperPassPass(PassRegistry &);
 LLVM_ABI void initializeDXILMetadataAnalysisWrapperPrinterPass(PassRegistry &);
 LLVM_ABI void initializeDXILResourceBindingWrapperPassPass(PassRegistry &);
@@ -127,6 +128,7 @@ LLVM_ABI void initializeGCEmptyBasicBlocksPass(PassRegistry &);
 LLVM_ABI void initializeGCMachineCodeAnalysisPass(PassRegistry &);
 LLVM_ABI void initializeGCModuleInfoPass(PassRegistry &);
 LLVM_ABI void initializeGVNLegacyPassPass(PassRegistry &);
+LLVM_ABI void initializeGlobalDCELegacyPassPass(PassRegistry &);
 LLVM_ABI void initializeGlobalMergeFuncPassWrapperPass(PassRegistry &);
 LLVM_ABI void initializeGlobalMergePass(PassRegistry &);
 LLVM_ABI void initializeGlobalsAAWrapperPassPass(PassRegistry &);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index f82a43967e67a..630bdf96c18df 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -38,6 +38,7 @@
 #include "llvm/Support/Valgrind.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/AlwaysInliner.h"
+#include "llvm/Transforms/IPO/GlobalDCE.h"
 #include "llvm/Transforms/InstCombine/InstCombine.h"
 #include "llvm/Transforms/ObjCARC.h"
 #include "llvm/Transforms/Scalar.h"
@@ -77,12 +78,14 @@ struct ForcePassLinking {
     (void)llvm::createDXILResourceTypeWrapperPassPass();
     (void)llvm::createDeadArgEliminationPass();
     (void)llvm::createDeadCodeEliminationPass();
+    (void)llvm::createDeadStoreEliminationPass();
     (void)llvm::createDependenceAnalysisWrapperPass();
     (void)llvm::createDomOnlyPrinterWrapperPassPass();
     (void)llvm::createDomPrinterWrapperPassPass();
     (void)llvm::createDomOnlyViewerWrapperPassPass();
     (void)llvm::createDomViewerWrapperPassPass();
     (void)llvm::createAlwaysInlinerLegacyPass();
+    (void)llvm::createGlobalDCEPass();
     (void)llvm::createGlobalMergeFuncPass();
     (void)llvm::createGlobalsAAWrapperPass();
     (void)llvm::createInstSimplifyLegacyPass();
diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h
index 311d9ca8321c7..1625355323692 100644
--- a/llvm/include/llvm/MC/MCAsmBackend.h
+++ b/llvm/include/llvm/MC/MCAsmBackend.h
@@ -166,12 +166,8 @@ class LLVM_ABI MCAsmBackend {
   // Return false to use default handling. Otherwise, set `Size` to the number
   // of padding bytes.
   virtual bool relaxAlign(MCFragment &F, unsigned &Size) { return false; }
-  virtual bool relaxDwarfLineAddr(MCFragment &, bool &WasRelaxed) const {
-    return false;
-  }
-  virtual bool relaxDwarfCFA(MCFragment &, bool &WasRelaxed) const {
-    return false;
-  }
+  virtual bool relaxDwarfLineAddr(MCFragment &) const { return false; }
+  virtual bool relaxDwarfCFA(MCFragment &) const { return false; }
 
   // Defined by linker relaxation targets to possibly emit LEB128 relocations
   // and set Value at the relocated location.
diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h
index afd38d2e6d106..1316d8669239d 100644
--- a/llvm/include/llvm/MC/MCAssembler.h
+++ b/llvm/include/llvm/MC/MCAssembler.h
@@ -112,15 +112,11 @@ class MCAssembler {
 
   /// Perform relaxation on a single fragment.
   bool relaxFragment(MCFragment &F);
-  bool relaxInstruction(MCFragment &F);
-  bool relaxLEB(MCFragment &F);
-  bool relaxBoundaryAlign(MCBoundaryAlignFragment &BF);
-  bool relaxDwarfLineAddr(MCFragment &F);
-  bool relaxDwarfCallFrameFragment(MCFragment &F);
-  bool relaxCVInlineLineTable(MCCVInlineLineTableFragment &DF);
-  bool relaxCVDefRange(MCCVDefRangeFragment &DF);
-  bool relaxFill(MCFillFragment &F);
-  bool relaxOrg(MCOrgFragment &F);
+  void relaxInstruction(MCFragment &F);
+  void relaxLEB(MCFragment &F);
+  void relaxBoundaryAlign(MCBoundaryAlignFragment &BF);
+  void relaxDwarfLineAddr(MCFragment &F);
+  void relaxDwarfCallFrameFragment(MCFragment &F);
 
 public:
   /// Construct a new assembler instance.
diff --git a/llvm/include/llvm/MC/MCDecoder.h b/llvm/include/llvm/MC/MCDecoder.h
new file mode 100644
index 0000000000000..70762a4a5ebae
--- /dev/null
+++ b/llvm/include/llvm/MC/MCDecoder.h
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Disassembler decoder helper functions.
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_MC_MCDECODER_H
+#define LLVM_MC_MCDECODER_H
+
+#include "llvm/MC/MCDisassembler/MCDisassembler.h"
+#include "llvm/Support/MathExtras.h"
+#include <cassert>
+
+namespace llvm::MCD {
+
+// Helper to propagate SoftFail status. Returns false if the status is Fail;
+// callers are expected to early-exit in that condition. (Note, the '&' operator
+// is correct to propagate the values of this enum; see comment on 'enum
+// DecodeStatus'.)
+inline bool Check(MCDisassembler::DecodeStatus &Out,
+                  MCDisassembler::DecodeStatus In) {
+  Out = static_cast<MCDisassembler::DecodeStatus>(Out & In);
+  return Out != MCDisassembler::Fail;
+}
+
+// Extracts a given span of bits from the instruction bits and return it as an
+// integer.
+template <typename IntType>
+#if defined(_MSC_VER) && !defined(__clang__)
+__declspec(noinline)
+#endif
+inline std::enable_if_t<std::is_integral_v<IntType>, IntType>
+fieldFromInstruction(const IntType &Insn, unsigned StartBit, unsigned NumBits) {
+  assert(StartBit + NumBits <= 64 && "Cannot support >64-bit extractions!");
+  assert(StartBit + NumBits <= (sizeof(IntType) * 8) &&
+         "Instruction field out of bounds!");
+  const IntType Mask = maskTrailingOnes<IntType>(NumBits);
+  return (Insn >> StartBit) & Mask;
+}
+
+template <typename InsnType>
+inline std::enable_if_t<!std::is_integral_v<InsnType>, uint64_t>
+fieldFromInstruction(const InsnType &Insn, unsigned StartBit,
+                     unsigned NumBits) {
+  return Insn.extractBitsAsZExtValue(NumBits, StartBit);
+}
+
+// Helper function for inserting bits extracted from an encoded instruction into
+// an integer-typed field.
+template <typename IntType>
+static std::enable_if_t<std::is_integral_v<IntType>, void>
+insertBits(IntType &field, IntType bits, unsigned startBit, unsigned numBits) {
+  // Check that no bit beyond numBits is set, so that a simple bitwise |
+  // is sufficient.
+  assert((~(((IntType)1 << numBits) - 1) & bits) == 0 &&
+         "bits has more than numBits bits set");
+  assert(startBit + numBits <= sizeof(IntType) * 8);
+  (void)numBits;
+  field |= bits << startBit;
+}
+
+} // namespace llvm::MCD
+
+#endif // LLVM_MC_MCDECODER_H
diff --git a/llvm/include/llvm/MC/MCDecoderOps.h b/llvm/include/llvm/MC/MCDecoderOps.h
index 50fcf2a40e26d..dea5b6931250f 100644
--- a/llvm/include/llvm/MC/MCDecoderOps.h
+++ b/llvm/include/llvm/MC/MCDecoderOps.h
@@ -1,11 +1,11 @@
-//===------------ llvm/MC/MCDecoderOps.h - Decoder driver -------*- C++ -*-===//
+//===----------------------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-// Disassembler decoder state machine driver.
+// Disassembler decoder state machine ops.
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_MC_MCDECODEROPS_H
 #define LLVM_MC_MCDECODEROPS_H
diff --git a/llvm/include/llvm/MC/MCMachObjectWriter.h b/llvm/include/llvm/MC/MCMachObjectWriter.h
index 170e2e74c671a..41416a2f9c698 100644
--- a/llvm/include/llvm/MC/MCMachObjectWriter.h
+++ b/llvm/include/llvm/MC/MCMachObjectWriter.h
@@ -17,6 +17,7 @@
 #include "llvm/MC/MCLinkerOptimizationHint.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSectionMachO.h"
+#include "llvm/MC/MCSymbolMachO.h"
 #include "llvm/MC/StringTableBuilder.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/EndianStream.h"
@@ -110,7 +111,7 @@ class LLVM_ABI MachObjectWriter final : public MCObjectWriter {
 private:
   /// Helper struct for containing some precomputed information on symbols.
   struct MachSymbolData {
-    const MCSymbol *Symbol;
+    const MCSymbolMachO *Symbol;
     uint64_t StringIndex;
     uint8_t SectionIndex;
 
@@ -119,7 +120,7 @@ class LLVM_ABI MachObjectWriter final : public MCObjectWriter {
   };
 
   struct IndirectSymbolData {
-    MCSymbol *Symbol;
+    MCSymbolMachO *Symbol;
     MCSection *Section;
   };
 
diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h
index d69560ca2d652..ed7f462c3c598 100644
--- a/llvm/include/llvm/MC/MCObjectFileInfo.h
+++ b/llvm/include/llvm/MC/MCObjectFileInfo.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_MC_MCOBJECTFILEINFO_H
 #define LLVM_MC_MCOBJECTFILEINFO_H
 
+#include "llvm/BinaryFormat/SFrame.h"
 #include "llvm/BinaryFormat/Swift.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/Support/Compiler.h"
@@ -50,6 +51,9 @@ class LLVM_ABI MCObjectFileInfo {
   /// Compact unwind encoding indicating that we should emit only an EH frame.
   unsigned CompactUnwindDwarfEHFrameOnly = 0;
 
+  /// SFrame ABI architecture byte.
+  std::optional<sframe::ABI> SFrameABIArch = {};
+
   /// Section directive for standard text.
   MCSection *TextSection = nullptr;
 
@@ -177,6 +181,9 @@ class LLVM_ABI MCObjectFileInfo {
   /// It is initialized on demand so it can be overwritten (with uniquing).
   MCSection *EHFrameSection = nullptr;
 
+  /// SFrame section.
+  MCSection *SFrameSection = nullptr;
+
   /// Section containing metadata on function stack sizes.
   MCSection *StackSizesSection = nullptr;
 
@@ -269,6 +276,7 @@ class LLVM_ABI MCObjectFileInfo {
     return CompactUnwindDwarfEHFrameOnly;
   }
 
+  std::optional<sframe::ABI> getSFrameABIArch() const { return SFrameABIArch; }
   virtual unsigned getTextSectionAlignment() const { return 4; }
   MCSection *getTextSection() const { return TextSection; }
   MCSection *getDataSection() const { return DataSection; }
@@ -450,6 +458,7 @@ class LLVM_ABI MCObjectFileInfo {
   MCSection *getTOCBaseSection() const { return TOCBaseSection; }
 
   MCSection *getEHFrameSection() const { return EHFrameSection; }
+  MCSection *getSFrameSection() const { return SFrameSection; }
 
   bool isPositionIndependent() const { return PositionIndependent; }
 
diff --git a/llvm/include/llvm/MC/MCSFrame.h b/llvm/include/llvm/MC/MCSFrame.h
new file mode 100644
index 0000000000000..8f182a86d1ab1
--- /dev/null
+++ b/llvm/include/llvm/MC/MCSFrame.h
@@ -0,0 +1,32 @@
+//===- MCSFrame.h - Machine Code SFrame support ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the declaration of MCSFrameEmitter to support emitting
+// sframe unwinding info from .cfi_* directives. It relies on FDEs and CIEs
+// created for Dwarf frame info, but emits that info in a different format.
+//
+// See https://sourceware.org/binutils/docs-2.41/sframe-spec.html
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_MC_MCSFRAME_H
+#define LLVM_MC_MCSFRAME_H
+
+namespace llvm {
+
+class MCObjectStreamer;
+
+class MCSFrameEmitter {
+public:
+  // Emit the sframe section.
+  //
+  // \param Streamer - Emit into this stream.
+  static void emit(MCObjectStreamer &Streamer);
+};
+
+} // end namespace llvm
+#endif // LLVM_MC_MCSFRAME_H
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index e976f471e78e2..12389d623e588 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -307,7 +307,6 @@ class MCFillFragment : public MCFragment {
   uint64_t Value;
   /// The number of bytes to insert.
   const MCExpr &NumValues;
-  uint64_t Size = 0;
 
   /// Source location of the directive that this fragment was created for.
   SMLoc Loc;
@@ -321,8 +320,6 @@ class MCFillFragment : public MCFragment {
   uint64_t getValue() const { return Value; }
   uint8_t getValueSize() const { return ValueSize; }
   const MCExpr &getNumValues() const { return NumValues; }
-  uint64_t getSize() const { return Size; }
-  void setSize(uint64_t Value) { Size = Value; }
 
   SMLoc getLoc() const { return Loc; }
 
@@ -371,16 +368,12 @@ class MCOrgFragment : public MCFragment {
   /// Source location of the directive that this fragment was created for.
   SMLoc Loc;
 
-  uint64_t Size = 0;
-
 public:
   MCOrgFragment(const MCExpr &Offset, int8_t Value, SMLoc Loc)
       : MCFragment(FT_Org), Value(Value), Offset(&Offset), Loc(Loc) {}
 
   const MCExpr &getOffset() const { return *Offset; }
   uint8_t getValue() const { return Value; }
-  uint64_t getSize() const { return Size; }
-  void setSize(uint64_t Value) { Size = Value; }
 
   SMLoc getLoc() const { return Loc; }
 
@@ -534,6 +527,10 @@ class LLVM_ABI MCSection {
   Align Alignment;
   /// The section index in the assemblers section list.
   unsigned Ordinal = 0;
+  // If not -1u, the first linker-relaxable fragment's order within the
+  // subsection. When present, the offset between two locations crossing this
+  // fragment may not be fully resolved.
+  unsigned FirstLinkerRelaxable = -1u;
 
   /// Whether this section has had instructions emitted into it.
   bool HasInstructions : 1;
@@ -543,10 +540,6 @@ class LLVM_ABI MCSection {
   bool IsText : 1;
   bool IsBss : 1;
 
-  /// Whether the section contains linker-relaxable fragments. If true, the
-  /// offset between two locations may not be fully resolved.
-  bool LinkerRelaxable : 1;
-
   MCFragment DummyFragment;
 
   // Mapping from subsection number to fragment list. At layout time, the
@@ -601,8 +594,9 @@ class LLVM_ABI MCSection {
   bool isRegistered() const { return IsRegistered; }
   void setIsRegistered(bool Value) { IsRegistered = Value; }
 
-  bool isLinkerRelaxable() const { return LinkerRelaxable; }
-  void setLinkerRelaxable() { LinkerRelaxable = true; }
+  unsigned firstLinkerRelaxable() const { return FirstLinkerRelaxable; }
+  bool isLinkerRelaxable() const { return FirstLinkerRelaxable != -1u; }
+  void setFirstLinkerRelaxable(unsigned Order) { FirstLinkerRelaxable = Order; }
 
   MCFragment &getDummyFragment() { return DummyFragment; }
 
diff --git a/llvm/include/llvm/MC/MCSymbol.h b/llvm/include/llvm/MC/MCSymbol.h
index ddc560ec52501..e31d0374baf4a 100644
--- a/llvm/include/llvm/MC/MCSymbol.h
+++ b/llvm/include/llvm/MC/MCSymbol.h
@@ -42,11 +42,10 @@ class raw_ostream;
 class MCSymbol {
 protected:
   // A symbol can be regular, equated to an expression, or a common symbol.
-  enum Contents : uint8_t {
-    SymContentsUnset,
-    SymContentsVariable,
-    SymContentsCommon,
-    SymContentsTargetCommon, // Index stores the section index
+  enum Kind : uint8_t {
+    Regular,
+    Equated,
+    Common,
   };
 
   // Special sentinel value for the absolute pseudo fragment.
@@ -65,6 +64,10 @@ class MCSymbol {
   /// relative to, if any.
   mutable MCFragment *Fragment = nullptr;
 
+  /// The symbol kind. Use an unsigned bitfield to achieve better bitpacking
+  /// with MSVC.
+  unsigned kind : 2;
+
   /// True if this symbol is named.  A named symbol will have a pointer to the
   /// name allocated in the bytes immediately prior to the MCSymbol.
   unsigned HasName : 1;
@@ -95,10 +98,6 @@ class MCSymbol {
   /// Used to detect cyclic dependency like `a = a + 1` and `a = b; b = a`.
   unsigned IsResolving : 1;
 
-  /// This is actually a Contents enumerator, but is unsigned to avoid sign
-  /// extension and achieve better bitpacking with MSVC.
-  unsigned SymbolContents : 3;
-
   /// The alignment of the symbol if it is 'common'.
   ///
   /// Internally, this is stored as log2(align) + 1.
@@ -145,9 +144,9 @@ class MCSymbol {
   };
 
   MCSymbol(const MCSymbolTableEntry *Name, bool isTemporary)
-      : IsTemporary(isTemporary), IsRedefinable(false), IsRegistered(false),
-        IsExternal(false), IsPrivateExtern(false), IsWeakExternal(false),
-        IsUsedInReloc(false), IsResolving(0), SymbolContents(SymContentsUnset),
+      : kind(Kind::Regular), IsTemporary(isTemporary), IsRedefinable(false),
+        IsRegistered(false), IsExternal(false), IsPrivateExtern(false),
+        IsWeakExternal(false), IsUsedInReloc(false), IsResolving(0),
         CommonAlignLog2(0), Flags(0) {
     Offset = 0;
     HasName = !!Name;
@@ -212,11 +211,11 @@ class MCSymbol {
   /// Prepare this symbol to be redefined.
   void redefineIfPossible() {
     if (IsRedefinable) {
-      if (SymbolContents == SymContentsVariable) {
+      if (kind == Kind::Equated) {
         Value = nullptr;
-        SymbolContents = SymContentsUnset;
+        kind = Kind::Regular;
       }
-      setUndefined();
+      Fragment = nullptr;
       IsRedefinable = false;
     }
   }
@@ -260,17 +259,12 @@ class MCSymbol {
     Fragment = F;
   }
 
-  /// Mark the symbol as undefined.
-  void setUndefined() { Fragment = nullptr; }
-
   /// @}
   /// \name Variable Symbols
   /// @{
 
   /// isVariable - Check if this is a variable symbol.
-  bool isVariable() const {
-    return SymbolContents == SymContentsVariable;
-  }
+  bool isVariable() const { return kind == Equated; }
 
   /// Get the expression of the variable symbol.
   const MCExpr *getVariableValue() const {
@@ -293,12 +287,12 @@ class MCSymbol {
   }
 
   uint64_t getOffset() const {
-    assert(SymbolContents == SymContentsUnset &&
+    assert(kind == Kind::Regular &&
            "Cannot get offset for a common/variable symbol");
     return Offset;
   }
   void setOffset(uint64_t Value) {
-    assert(SymbolContents == SymContentsUnset &&
+    assert(kind == Kind::Regular &&
            "Cannot set offset for a common/variable symbol");
     Offset = Value;
   }
@@ -314,10 +308,10 @@ class MCSymbol {
   /// \param Size - The size of the symbol.
   /// \param Alignment - The alignment of the symbol.
   /// \param Target - Is the symbol a target-specific common-like symbol.
-  void setCommon(uint64_t Size, Align Alignment, bool Target = false) {
+  void setCommon(uint64_t Size, Align Alignment) {
     assert(getOffset() == 0);
     CommonSize = Size;
-    SymbolContents = Target ? SymContentsTargetCommon : SymContentsCommon;
+    kind = Kind::Common;
 
     unsigned Log2Align = encode(Alignment);
     assert(Log2Align < (1U << NumCommonAlignmentBits) &&
@@ -335,29 +329,19 @@ class MCSymbol {
   ///
   /// \param Size - The size of the symbol.
   /// \param Alignment - The alignment of the symbol.
-  /// \param Target - Is the symbol a target-specific common-like symbol.
   /// \return True if symbol was already declared as a different type
-  bool declareCommon(uint64_t Size, Align Alignment, bool Target = false) {
+  bool declareCommon(uint64_t Size, Align Alignment) {
     assert(isCommon() || getOffset() == 0);
     if(isCommon()) {
-      if (CommonSize != Size || getCommonAlignment() != Alignment ||
-          isTargetCommon() != Target)
+      if (CommonSize != Size || getCommonAlignment() != Alignment)
         return true;
     } else
-      setCommon(Size, Alignment, Target);
+      setCommon(Size, Alignment);
     return false;
   }
 
   /// Is this a 'common' symbol.
-  bool isCommon() const {
-    return SymbolContents == SymContentsCommon ||
-           SymbolContents == SymContentsTargetCommon;
-  }
-
-  /// Is this a target-specific common-like symbol.
-  bool isTargetCommon() const {
-    return SymbolContents == SymContentsTargetCommon;
-  }
+  bool isCommon() const { return kind == Kind::Common; }
 
   MCFragment *getFragment() const {
     if (Fragment || !isVariable() || isWeakExternal())
@@ -368,10 +352,6 @@ class MCSymbol {
     return Fragment;
   }
 
-  // For ELF, use MCSymbolELF::setBinding instead.
-  bool isExternal() const { return IsExternal; }
-  void setExternal(bool Value) const { IsExternal = Value; }
-
   // COFF-specific
   bool isWeakExternal() const { return IsWeakExternal; }
 
diff --git a/llvm/include/llvm/MC/MCSymbolCOFF.h b/llvm/include/llvm/MC/MCSymbolCOFF.h
index 0742cf7b406c7..fe1e8f6836420 100644
--- a/llvm/include/llvm/MC/MCSymbolCOFF.h
+++ b/llvm/include/llvm/MC/MCSymbolCOFF.h
@@ -33,6 +33,9 @@ class MCSymbolCOFF : public MCSymbol {
   MCSymbolCOFF(const MCSymbolTableEntry *Name, bool isTemporary)
       : MCSymbol(Name, isTemporary) {}
 
+  bool isExternal() const { return IsExternal; }
+  void setExternal(bool Value) const { IsExternal = Value; }
+
   uint16_t getType() const {
     return Type;
   }
diff --git a/llvm/include/llvm/MC/MCSymbolMachO.h b/llvm/include/llvm/MC/MCSymbolMachO.h
index b2b4442e0d666..2b8bfe0dcac5e 100644
--- a/llvm/include/llvm/MC/MCSymbolMachO.h
+++ b/llvm/include/llvm/MC/MCSymbolMachO.h
@@ -46,6 +46,8 @@ class MCSymbolMachO : public MCSymbol {
   MCSymbolMachO(const MCSymbolTableEntry *Name, bool isTemporary)
       : MCSymbol(Name, isTemporary) {}
 
+  bool isExternal() const { return IsExternal; }
+  void setExternal(bool Value) const { IsExternal = Value; }
   bool isPrivateExtern() const { return IsPrivateExtern; }
   void setPrivateExtern(bool Value) { IsPrivateExtern = Value; }
 
diff --git a/llvm/include/llvm/MC/MCSymbolWasm.h b/llvm/include/llvm/MC/MCSymbolWasm.h
index bd13896ac0712..5c9f14e5e6d64 100644
--- a/llvm/include/llvm/MC/MCSymbolWasm.h
+++ b/llvm/include/llvm/MC/MCSymbolWasm.h
@@ -37,6 +37,8 @@ class MCSymbolWasm : public MCSymbol {
   MCSymbolWasm(const MCSymbolTableEntry *Name, bool isTemporary)
       : MCSymbol(Name, isTemporary) {}
 
+  bool isExternal() const { return IsExternal; }
+  void setExternal(bool Value) const { IsExternal = Value; }
   const MCExpr *getSize() const { return SymbolSize; }
   void setSize(const MCExpr *SS) { SymbolSize = SS; }
 
diff --git a/llvm/include/llvm/MC/MCSymbolXCOFF.h b/llvm/include/llvm/MC/MCSymbolXCOFF.h
index d86b2d7527fbe..e1d61e2bb76fb 100644
--- a/llvm/include/llvm/MC/MCSymbolXCOFF.h
+++ b/llvm/include/llvm/MC/MCSymbolXCOFF.h
@@ -37,6 +37,8 @@ class MCSymbolXCOFF : public MCSymbol {
     return Name;
   }
 
+  bool isExternal() const { return IsExternal; }
+  void setExternal(bool Value) const { IsExternal = Value; }
   void setStorageClass(XCOFF::StorageClass SC) {
     StorageClass = SC;
   };
diff --git a/llvm/include/llvm/MCA/Instruction.h b/llvm/include/llvm/MCA/Instruction.h
index 4c683aa3421de..3cdbf84748c79 100644
--- a/llvm/include/llvm/MCA/Instruction.h
+++ b/llvm/include/llvm/MCA/Instruction.h
@@ -382,6 +382,10 @@ class ReadState {
   bool isReadZero() const { return IsZero; }
   void setReadZero() { IsZero = true; }
   void setPRF(unsigned ID) { PRFID = ID; }
+
+#ifndef NDEBUG
+  void dump() const;
+#endif
 };
 
 /// A sequence of cycles.
diff --git a/llvm/include/llvm/ObjCopy/ConfigManager.h b/llvm/include/llvm/ObjCopy/ConfigManager.h
index e7b3775f0e9f1..4b596c604ea3a 100644
--- a/llvm/include/llvm/ObjCopy/ConfigManager.h
+++ b/llvm/include/llvm/ObjCopy/ConfigManager.h
@@ -11,6 +11,7 @@
 
 #include "llvm/ObjCopy/COFF/COFFConfig.h"
 #include "llvm/ObjCopy/CommonConfig.h"
+#include "llvm/ObjCopy/DXContainer/DXContainerConfig.h"
 #include "llvm/ObjCopy/ELF/ELFConfig.h"
 #include "llvm/ObjCopy/MachO/MachOConfig.h"
 #include "llvm/ObjCopy/MultiFormatConfig.h"
@@ -36,6 +37,8 @@ struct LLVM_ABI ConfigManager : public MultiFormatConfig {
 
   Expected<const XCOFFConfig &> getXCOFFConfig() const override;
 
+  Expected<const DXContainerConfig &> getDXContainerConfig() const override;
+
   // All configs.
   CommonConfig Common;
   ELFConfig ELF;
@@ -43,6 +46,7 @@ struct LLVM_ABI ConfigManager : public MultiFormatConfig {
   MachOConfig MachO;
   WasmConfig Wasm;
   XCOFFConfig XCOFF;
+  DXContainerConfig DXContainer;
 };
 
 } // namespace objcopy
diff --git a/llvm/include/llvm/ObjCopy/DXContainer/DXContainerConfig.h b/llvm/include/llvm/ObjCopy/DXContainer/DXContainerConfig.h
new file mode 100644
index 0000000000000..953e0d7189708
--- /dev/null
+++ b/llvm/include/llvm/ObjCopy/DXContainer/DXContainerConfig.h
@@ -0,0 +1,23 @@
+//===- DXContainerConfig.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJCOPY_DXCONTAINER_DXCONTAINERCONFIG_H
+#define LLVM_OBJCOPY_DXCONTAINER_DXCONTAINERCONFIG_H
+
+namespace llvm {
+namespace objcopy {
+
+// DXContainer specific configuration for copying/stripping a single file. This
+// is defined, following convention elsewhere, as the return type of
+// `getDXContainerConfig`, which reports an error for an unsupported option.
+struct DXContainerConfig {};
+
+} // namespace objcopy
+} // namespace llvm
+
+#endif // LLVM_OBJCOPY_DXCONTAINER_DXCONTAINERCONFIG_H
diff --git a/llvm/include/llvm/ObjCopy/DXContainer/DXContainerObjcopy.h b/llvm/include/llvm/ObjCopy/DXContainer/DXContainerObjcopy.h
new file mode 100644
index 0000000000000..a410cfec18ea4
--- /dev/null
+++ b/llvm/include/llvm/ObjCopy/DXContainer/DXContainerObjcopy.h
@@ -0,0 +1,36 @@
+//===- DXContainerObjcopy.h -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJCOPY_DXCONTAINER_DXCONTAINEROBJCOPY_H
+#define LLVM_OBJCOPY_DXCONTAINER_DXCONTAINEROBJCOPY_H
+
+namespace llvm {
+class Error;
+class raw_ostream;
+
+namespace object {
+class DXContainerObjectFile;
+} // end namespace object
+
+namespace objcopy {
+struct CommonConfig;
+struct DXContainerConfig;
+
+namespace dxbc {
+/// Apply the transformations described by \p Config and \p DXContainerConfig
+/// to \p In and writes the result into \p Out.
+/// \returns any Error encountered whilst performing the operation.
+Error executeObjcopyOnBinary(const CommonConfig &Config,
+                             const DXContainerConfig &,
+                             object::DXContainerObjectFile &In,
+                             raw_ostream &Out);
+} // end namespace dxbc
+} // end namespace objcopy
+} // end namespace llvm
+
+#endif // LLVM_OBJCOPY_DXCONTAINER_DXCONTAINEROBJCOPY_H
diff --git a/llvm/include/llvm/ObjCopy/MultiFormatConfig.h b/llvm/include/llvm/ObjCopy/MultiFormatConfig.h
index 180f2f82a908b..bb93f64aa2788 100644
--- a/llvm/include/llvm/ObjCopy/MultiFormatConfig.h
+++ b/llvm/include/llvm/ObjCopy/MultiFormatConfig.h
@@ -20,6 +20,7 @@ struct COFFConfig;
 struct MachOConfig;
 struct WasmConfig;
 struct XCOFFConfig;
+struct DXContainerConfig;
 
 class MultiFormatConfig {
 public:
@@ -31,6 +32,7 @@ class MultiFormatConfig {
   virtual Expected<const MachOConfig &> getMachOConfig() const = 0;
   virtual Expected<const WasmConfig &> getWasmConfig() const = 0;
   virtual Expected<const XCOFFConfig &> getXCOFFConfig() const = 0;
+  virtual Expected<const DXContainerConfig &> getDXContainerConfig() const = 0;
 };
 
 } // namespace objcopy
diff --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h
index ad1b2361ff064..9bc1918852335 100644
--- a/llvm/include/llvm/Object/DXContainer.h
+++ b/llvm/include/llvm/Object/DXContainer.h
@@ -603,8 +603,12 @@ class LLVM_ABI DXContainerObjectFile : public ObjectFile {
   }
 
 public:
+  const DXContainer &getDXContainer() const { return Container; }
+
   static bool classof(const Binary *v) { return v->isDXContainer(); }
 
+  const dxbc::Header &getHeader() const { return Container.getHeader(); }
+
   Expected<StringRef> getSymbolName(DataRefImpl) const override;
   Expected<uint64_t> getSymbolAddress(DataRefImpl Symb) const override;
   uint64_t getSymbolValueImpl(DataRefImpl Symb) const override;
diff --git a/llvm/include/llvm/Object/SFrameParser.h b/llvm/include/llvm/Object/SFrameParser.h
index 245e7ba40d0b8..3ce5d70142a9f 100644
--- a/llvm/include/llvm/Object/SFrameParser.h
+++ b/llvm/include/llvm/Object/SFrameParser.h
@@ -10,6 +10,7 @@
 #define LLVM_OBJECT_SFRAME_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/fallible_iterator.h"
 #include "llvm/BinaryFormat/SFrame.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
@@ -19,6 +20,8 @@ namespace llvm {
 namespace object {
 
 template <endianness E> class SFrameParser {
+  class FallibleFREIterator;
+
 public:
   static Expected<SFrameParser> create(ArrayRef<uint8_t> Contents,
                                        uint64_t SectionAddress);
@@ -42,6 +45,25 @@ template <endianness E> class SFrameParser {
   // objects returned by the `fdes()` function.
   uint64_t getAbsoluteStartAddress(typename FDERange::iterator FDE) const;
 
+  // Returns the offset (in the SFrame section) of the given FDE, which must be
+  // one of the objects returned by the `fdes()` function.
+  uint64_t offsetOf(typename FDERange::iterator FDE) const;
+
+  struct FrameRowEntry {
+    uint32_t StartAddress;
+    sframe::FREInfo<endianness::native> Info;
+    SmallVector<int32_t, 3> Offsets;
+  };
+
+  using fre_iterator = fallible_iterator<FallibleFREIterator>;
+  iterator_range<fre_iterator> fres(const sframe::FuncDescEntry<E> &FDE,
+                                    Error &Err) const;
+
+  std::optional<int32_t> getCFAOffset(const FrameRowEntry &FRE) const;
+  std::optional<int32_t> getRAOffset(const FrameRowEntry &FRE) const;
+  std::optional<int32_t> getFPOffset(const FrameRowEntry &FRE) const;
+  ArrayRef<int32_t> getExtraOffsets(const FrameRowEntry &FRE) const;
+
 private:
   ArrayRef<uint8_t> Data;
   uint64_t SectionAddress;
@@ -54,6 +76,39 @@ template <endianness E> class SFrameParser {
   uint64_t getFDEBase() const {
     return sizeof(Header) + Header.AuxHdrLen + Header.FDEOff;
   }
+
+  uint64_t getFREBase() const {
+    return getFDEBase() + Header.NumFDEs * sizeof(sframe::FuncDescEntry<E>);
+  }
+};
+
+template <endianness E> class SFrameParser<E>::FallibleFREIterator {
+public:
+  // NB: This iterator starts out in the before_begin() state. It must be
+  // ++'ed to reach the first element.
+  FallibleFREIterator(ArrayRef<uint8_t> Data, sframe::FREType FREType,
+                      uint32_t Idx, uint32_t Size, uint64_t Offset)
+      : Data(Data), FREType(FREType), Idx(Idx), Size(Size), Offset(Offset) {}
+
+  Error inc();
+  const FrameRowEntry &operator*() const { return FRE; }
+
+  friend bool operator==(const FallibleFREIterator &LHS,
+                         const FallibleFREIterator &RHS) {
+    assert(LHS.Data.data() == RHS.Data.data());
+    assert(LHS.Data.size() == RHS.Data.size());
+    assert(LHS.FREType == RHS.FREType);
+    assert(LHS.Size == RHS.Size);
+    return LHS.Idx == RHS.Idx;
+  }
+
+private:
+  ArrayRef<uint8_t> Data;
+  sframe::FREType FREType;
+  uint32_t Idx;
+  uint32_t Size;
+  uint64_t Offset;
+  FrameRowEntry FRE;
 };
 
 extern template class LLVM_TEMPLATE_ABI SFrameParser<endianness::big>;
diff --git a/llvm/include/llvm/ProfileData/IndexedMemProfData.h b/llvm/include/llvm/ProfileData/IndexedMemProfData.h
index dae0928d50ca7..09c24483619ad 100644
--- a/llvm/include/llvm/ProfileData/IndexedMemProfData.h
+++ b/llvm/include/llvm/ProfileData/IndexedMemProfData.h
@@ -18,10 +18,9 @@
 #include "llvm/ProfileData/DataAccessProf.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/MemProf.h"
+#include "llvm/Support/BLAKE3.h"
 #include "llvm/Support/Compiler.h"
-
-#include <functional>
-#include <optional>
+#include "llvm/Support/HashBuilder.h"
 
 namespace llvm {
 namespace memprof {
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index f53602424b583..85a9efe73855b 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -526,24 +526,27 @@ class InstrProfSymtab {
   // so it doesn't use a StringSet for function names.
   StringSet<> VTableNames;
   // A map from MD5 keys to function name strings.
-  std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
+  mutable std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
   // A map from MD5 keys to function define. We only populate this map
   // when build the Symtab from a Module.
-  std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
+  mutable std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
   // A map from MD5 to the global variable. This map is only populated when
   // building the symtab from a module. Use separate container instances for
   // `MD5FuncMap` and `MD5VTableMap`.
   // TODO: Unify the container type and the lambda function 'mapName' inside
   // add{Func,VTable}WithName.
-  DenseMap<uint64_t, GlobalVariable *> MD5VTableMap;
+  mutable DenseMap<uint64_t, GlobalVariable *> MD5VTableMap;
   // A map from function runtime address to function name MD5 hash.
   // This map is only populated and used by raw instr profile reader.
-  AddrHashMap AddrToMD5Map;
+  mutable AddrHashMap AddrToMD5Map;
 
   AddrIntervalMap::Allocator VTableAddrMapAllocator;
   // This map is only populated and used by raw instr profile reader.
   AddrIntervalMap VTableAddrMap;
-  bool Sorted = false;
+
+  // "dirty" flag for the rest of the mutable state. lookup APIs (like
+  // getFunction) need the mutable state to be sorted.
+  mutable bool Sorted = false;
 
   static StringRef getExternalSymbol() { return "** External Symbol **"; }
 
@@ -565,8 +568,10 @@ class InstrProfSymtab {
   // If the symtab is created by a series of calls to \c addFuncName, \c
   // finalizeSymtab needs to be called before looking up function names.
   // This is required because the underlying map is a vector (for space
-  // efficiency) which needs to be sorted.
-  inline void finalizeSymtab();
+  // efficiency) which needs to be sorted. The API is `const` because it's part
+  // of the implementation detail of `const` APIs that need to first ensure this
+  // property of ordering on the other mutable state.
+  inline void finalizeSymtab() const;
 
 public:
   InstrProfSymtab() : VTableAddrMap(VTableAddrMapAllocator) {}
@@ -660,6 +665,10 @@ class InstrProfSymtab {
     return Error::success();
   }
 
+  const std::vector<std::pair<uint64_t, Function *>> &getIDToNameMap() const {
+    return MD5FuncMap;
+  }
+
   const StringSet<> &getVTableNames() const { return VTableNames; }
 
   /// Map a function address to its name's MD5 hash. This interface
@@ -676,24 +685,25 @@ class InstrProfSymtab {
   }
 
   /// Return a function's hash, or 0, if the function isn't in this SymTab.
-  LLVM_ABI uint64_t getFunctionHashFromAddress(uint64_t Address);
+  LLVM_ABI uint64_t getFunctionHashFromAddress(uint64_t Address) const;
 
   /// Return a vtable's hash, or 0 if the vtable doesn't exist in this SymTab.
-  LLVM_ABI uint64_t getVTableHashFromAddress(uint64_t Address);
+  LLVM_ABI uint64_t getVTableHashFromAddress(uint64_t Address) const;
 
   /// Return function's PGO name from the function name's symbol
   /// address in the object file. If an error occurs, return
   /// an empty string.
-  LLVM_ABI StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize);
+  LLVM_ABI StringRef getFuncName(uint64_t FuncNameAddress,
+                                 size_t NameSize) const;
 
   /// Return name of functions or global variables from the name's md5 hash
   /// value. If not found, return an empty string.
-  inline StringRef getFuncOrVarName(uint64_t ValMD5Hash);
+  inline StringRef getFuncOrVarName(uint64_t ValMD5Hash) const;
 
   /// Just like getFuncOrVarName, except that it will return literal string
   /// 'External Symbol' if the function or global variable is external to
   /// this symbol table.
-  inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash);
+  inline StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash) const;
 
   /// True if Symbol is the value used to represent external symbols.
   static bool isExternalSymbol(const StringRef &Symbol) {
@@ -701,11 +711,11 @@ class InstrProfSymtab {
   }
 
   /// Return function from the name's md5 hash. Return nullptr if not found.
-  inline Function *getFunction(uint64_t FuncMD5Hash);
+  inline Function *getFunction(uint64_t FuncMD5Hash) const;
 
   /// Return the global variable corresponding to md5 hash. Return nullptr if
   /// not found.
-  inline GlobalVariable *getGlobalVariable(uint64_t MD5Hash);
+  inline GlobalVariable *getGlobalVariable(uint64_t MD5Hash) const;
 
   /// Return the name section data.
   inline StringRef getNameData() const { return Data; }
@@ -748,7 +758,7 @@ Error InstrProfSymtab::create(const FuncNameIterRange &FuncIterRange,
   return Error::success();
 }
 
-void InstrProfSymtab::finalizeSymtab() {
+void InstrProfSymtab::finalizeSymtab() const {
   if (Sorted)
     return;
   llvm::sort(MD5NameMap, less_first());
@@ -758,14 +768,14 @@ void InstrProfSymtab::finalizeSymtab() {
   Sorted = true;
 }
 
-StringRef InstrProfSymtab::getFuncOrVarNameIfDefined(uint64_t MD5Hash) {
-  StringRef ret = getFuncOrVarName(MD5Hash);
-  if (ret.empty())
+StringRef InstrProfSymtab::getFuncOrVarNameIfDefined(uint64_t MD5Hash) const {
+  StringRef Ret = getFuncOrVarName(MD5Hash);
+  if (Ret.empty())
     return InstrProfSymtab::getExternalSymbol();
-  return ret;
+  return Ret;
 }
 
-StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) {
+StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) const {
   finalizeSymtab();
   auto Result = llvm::lower_bound(MD5NameMap, MD5Hash,
                                   [](const std::pair<uint64_t, StringRef> &LHS,
@@ -775,7 +785,7 @@ StringRef InstrProfSymtab::getFuncOrVarName(uint64_t MD5Hash) {
   return StringRef();
 }
 
-Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
+Function *InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) const {
   finalizeSymtab();
   auto Result = llvm::lower_bound(MD5FuncMap, FuncMD5Hash,
                                   [](const std::pair<uint64_t, Function *> &LHS,
@@ -785,7 +795,7 @@ Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) {
   return nullptr;
 }
 
-GlobalVariable *InstrProfSymtab::getGlobalVariable(uint64_t MD5Hash) {
+GlobalVariable *InstrProfSymtab::getGlobalVariable(uint64_t MD5Hash) const {
   return MD5VTableMap.lookup(MD5Hash);
 }
 
diff --git a/llvm/include/llvm/ProfileData/InstrProfWriter.h b/llvm/include/llvm/ProfileData/InstrProfWriter.h
index f339fe2c2a9eb..1b24425e68a9e 100644
--- a/llvm/include/llvm/ProfileData/InstrProfWriter.h
+++ b/llvm/include/llvm/ProfileData/InstrProfWriter.h
@@ -226,8 +226,6 @@ class InstrProfWriter {
   void addRecord(StringRef Name, uint64_t Hash, InstrProfRecord &&I,
                  uint64_t Weight, function_ref<void(Error)> Warn);
   bool shouldEncodeData(const ProfilingData &PD);
-  /// Add \p Trace using reservoir sampling.
-  void addTemporalProfileTrace(TemporalProfTraceTy Trace);
 
   /// Add a memprof record for a function identified by its \p Id.
   void addMemProfRecord(const GlobalValue::GUID Id,
diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index 9df4efd8e0853..e4bc9ee6f6ad1 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -14,24 +14,20 @@
 #ifndef LLVM_PROFILEDATA_MEMPROF_H
 #define LLVM_PROFILEDATA_MEMPROF_H
 
-#include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/ProfileData/MemProfData.inc"
-#include "llvm/Support/BLAKE3.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/EndianStream.h"
-#include "llvm/Support/HashBuilder.h"
 #include "llvm/Support/raw_ostream.h"
 
 #include <bitset>
 #include <cstdint>
-#include <optional>
 
 namespace llvm {
 namespace yaml {
diff --git a/llvm/include/llvm/ProfileData/MemProfRadixTree.h b/llvm/include/llvm/ProfileData/MemProfRadixTree.h
index 35df7a8f16a6e..de1ddf86d8135 100644
--- a/llvm/include/llvm/ProfileData/MemProfRadixTree.h
+++ b/llvm/include/llvm/ProfileData/MemProfRadixTree.h
@@ -13,10 +13,13 @@
 #ifndef LLVM_PROFILEDATA_MEMPROFRADIXTREE_H
 #define LLVM_PROFILEDATA_MEMPROFRADIXTREE_H
 
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ProfileData/IndexedMemProfData.h"
 #include "llvm/ProfileData/MemProf.h"
 #include "llvm/Support/Compiler.h"
 
+#include <optional>
+
 namespace llvm {
 namespace memprof {
 namespace detail {
diff --git a/llvm/include/llvm/SandboxIR/Instruction.h b/llvm/include/llvm/SandboxIR/Instruction.h
index 4e3ff19d47787..e1c1ca039a8a0 100644
--- a/llvm/include/llvm/SandboxIR/Instruction.h
+++ b/llvm/include/llvm/SandboxIR/Instruction.h
@@ -2278,6 +2278,8 @@ class CastInst : public UnaryInstruction {
       return Opcode::FPToSI;
     case llvm::Instruction::FPExt:
       return Opcode::FPExt;
+    case llvm::Instruction::PtrToAddr:
+      return Opcode::PtrToAddr;
     case llvm::Instruction::PtrToInt:
       return Opcode::PtrToInt;
     case llvm::Instruction::IntToPtr:
@@ -2364,6 +2366,8 @@ class FPToUIInst final : public CastInstImpl<Instruction::Opcode::FPToUI> {};
 class FPToSIInst final : public CastInstImpl<Instruction::Opcode::FPToSI> {};
 class IntToPtrInst final : public CastInstImpl<Instruction::Opcode::IntToPtr> {
 };
+class PtrToAddrInst final
+    : public CastInstImpl<Instruction::Opcode::PtrToAddr> {};
 class PtrToIntInst final : public CastInstImpl<Instruction::Opcode::PtrToInt> {
 };
 class BitCastInst final : public CastInstImpl<Instruction::Opcode::BitCast> {};
diff --git a/llvm/include/llvm/SandboxIR/Values.def b/llvm/include/llvm/SandboxIR/Values.def
index a55abbd20f4c0..72683e4c4d3f8 100644
--- a/llvm/include/llvm/SandboxIR/Values.def
+++ b/llvm/include/llvm/SandboxIR/Values.def
@@ -118,6 +118,7 @@ DEF_INSTR(Cast,   OPCODES(\
                           OP(FPToUI)        \
                           OP(FPToSI)        \
                           OP(FPExt)         \
+                          OP(PtrToAddr)     \
                           OP(PtrToInt)      \
                           OP(IntToPtr)      \
                           OP(SIToFP)        \
diff --git a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
index 8f367390c531c..fb9d68428cf18 100644
--- a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
+++ b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
@@ -83,26 +83,32 @@ enum : uint8_t {
 
 // Compute program resource register 1. Must match hardware definition.
 // GFX6+.
-#define COMPUTE_PGM_RSRC1(NAME, SHIFT, WIDTH) \
-  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_ ## NAME, SHIFT, WIDTH)
+#define COMPUTE_PGM_RSRC1(NAME, SHIFT, WIDTH)                                  \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_##NAME, SHIFT, WIDTH)
 // [GFX6-GFX8].
-#define COMPUTE_PGM_RSRC1_GFX6_GFX8(NAME, SHIFT, WIDTH) \
-  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX8_ ## NAME, SHIFT, WIDTH)
+#define COMPUTE_PGM_RSRC1_GFX6_GFX8(NAME, SHIFT, WIDTH)                        \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX8_##NAME, SHIFT, WIDTH)
 // [GFX6-GFX9].
-#define COMPUTE_PGM_RSRC1_GFX6_GFX9(NAME, SHIFT, WIDTH) \
-  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX9_ ## NAME, SHIFT, WIDTH)
+#define COMPUTE_PGM_RSRC1_GFX6_GFX9(NAME, SHIFT, WIDTH)                        \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX9_##NAME, SHIFT, WIDTH)
 // [GFX6-GFX11].
 #define COMPUTE_PGM_RSRC1_GFX6_GFX11(NAME, SHIFT, WIDTH)                       \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX11_##NAME, SHIFT, WIDTH)
+// [GFX6-GFX120].
+#define COMPUTE_PGM_RSRC1_GFX6_GFX120(NAME, SHIFT, WIDTH)                      \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX120_##NAME, SHIFT, WIDTH)
 // GFX9+.
-#define COMPUTE_PGM_RSRC1_GFX9_PLUS(NAME, SHIFT, WIDTH) \
-  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX9_PLUS_ ## NAME, SHIFT, WIDTH)
+#define COMPUTE_PGM_RSRC1_GFX9_PLUS(NAME, SHIFT, WIDTH)                        \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX9_PLUS_##NAME, SHIFT, WIDTH)
 // GFX10+.
-#define COMPUTE_PGM_RSRC1_GFX10_PLUS(NAME, SHIFT, WIDTH) \
-  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX10_PLUS_ ## NAME, SHIFT, WIDTH)
+#define COMPUTE_PGM_RSRC1_GFX10_PLUS(NAME, SHIFT, WIDTH)                       \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX10_PLUS_##NAME, SHIFT, WIDTH)
 // GFX12+.
 #define COMPUTE_PGM_RSRC1_GFX12_PLUS(NAME, SHIFT, WIDTH)                       \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX12_PLUS_##NAME, SHIFT, WIDTH)
+// [GFX125].
+#define COMPUTE_PGM_RSRC1_GFX125(NAME, SHIFT, WIDTH)                           \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX125_##NAME, SHIFT, WIDTH)
 enum : int32_t {
   COMPUTE_PGM_RSRC1(GRANULATED_WORKITEM_VGPR_COUNT, 0, 6),
   COMPUTE_PGM_RSRC1(GRANULATED_WAVEFRONT_SGPR_COUNT, 6, 4),
@@ -121,8 +127,10 @@ enum : int32_t {
   COMPUTE_PGM_RSRC1(CDBG_USER, 25, 1),
   COMPUTE_PGM_RSRC1_GFX6_GFX8(RESERVED0, 26, 1),
   COMPUTE_PGM_RSRC1_GFX9_PLUS(FP16_OVFL, 26, 1),
-  COMPUTE_PGM_RSRC1(RESERVED1, 27, 2),
-  COMPUTE_PGM_RSRC1_GFX6_GFX9(RESERVED2, 29, 3),
+  COMPUTE_PGM_RSRC1_GFX6_GFX120(RESERVED1, 27, 1),
+  COMPUTE_PGM_RSRC1_GFX125(FLAT_SCRATCH_IS_NV, 27, 1),
+  COMPUTE_PGM_RSRC1(RESERVED2, 28, 1),
+  COMPUTE_PGM_RSRC1_GFX6_GFX9(RESERVED3, 29, 3),
   COMPUTE_PGM_RSRC1_GFX10_PLUS(WGP_MODE, 29, 1),
   COMPUTE_PGM_RSRC1_GFX10_PLUS(MEM_ORDERED, 30, 1),
   COMPUTE_PGM_RSRC1_GFX10_PLUS(FWD_PROGRESS, 31, 1),
@@ -131,19 +139,29 @@ enum : int32_t {
 
 // Compute program resource register 2. Must match hardware definition.
 // GFX6+.
-#define COMPUTE_PGM_RSRC2(NAME, SHIFT, WIDTH) \
-  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC2_ ## NAME, SHIFT, WIDTH)
+#define COMPUTE_PGM_RSRC2(NAME, SHIFT, WIDTH)                                  \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC2_##NAME, SHIFT, WIDTH)
 // [GFX6-GFX11].
 #define COMPUTE_PGM_RSRC2_GFX6_GFX11(NAME, SHIFT, WIDTH)                       \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC2_GFX6_GFX11_##NAME, SHIFT, WIDTH)
+// [GFX6-GFX120].
+#define COMPUTE_PGM_RSRC2_GFX6_GFX120(NAME, SHIFT, WIDTH)                      \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC2_GFX6_GFX120_##NAME, SHIFT, WIDTH)
 // GFX12+.
 #define COMPUTE_PGM_RSRC2_GFX12_PLUS(NAME, SHIFT, WIDTH)                       \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC2_GFX12_PLUS_##NAME, SHIFT, WIDTH)
+// [GFX120].
+#define COMPUTE_PGM_RSRC2_GFX120(NAME, SHIFT, WIDTH)                           \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC2_GFX120_##NAME, SHIFT, WIDTH)
+// [GFX125].
+#define COMPUTE_PGM_RSRC2_GFX125(NAME, SHIFT, WIDTH)                           \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC2_GFX125_##NAME, SHIFT, WIDTH)
 enum : int32_t {
   COMPUTE_PGM_RSRC2(ENABLE_PRIVATE_SEGMENT, 0, 1),
-  COMPUTE_PGM_RSRC2(USER_SGPR_COUNT, 1, 5),
+  COMPUTE_PGM_RSRC2_GFX6_GFX120(USER_SGPR_COUNT, 1, 5),
   COMPUTE_PGM_RSRC2_GFX6_GFX11(ENABLE_TRAP_HANDLER, 6, 1),
-  COMPUTE_PGM_RSRC2_GFX12_PLUS(RESERVED1, 6, 1),
+  COMPUTE_PGM_RSRC2_GFX120(ENABLE_DYNAMIC_VGPR, 6, 1),
+  COMPUTE_PGM_RSRC2_GFX125(USER_SGPR_COUNT, 1, 6),
   COMPUTE_PGM_RSRC2(ENABLE_SGPR_WORKGROUP_ID_X, 7, 1),
   COMPUTE_PGM_RSRC2(ENABLE_SGPR_WORKGROUP_ID_Y, 8, 1),
   COMPUTE_PGM_RSRC2(ENABLE_SGPR_WORKGROUP_ID_Z, 9, 1),
@@ -178,23 +196,29 @@ enum : int32_t {
 // Compute program resource register 3 for GFX10+. Must match hardware
 // definition.
 // GFX10+.
-#define COMPUTE_PGM_RSRC3_GFX10_PLUS(NAME, SHIFT, WIDTH) \
-  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX10_PLUS_ ## NAME, SHIFT, WIDTH)
+#define COMPUTE_PGM_RSRC3_GFX10_PLUS(NAME, SHIFT, WIDTH)                       \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX10_PLUS_##NAME, SHIFT, WIDTH)
 // [GFX10].
 #define COMPUTE_PGM_RSRC3_GFX10(NAME, SHIFT, WIDTH)                            \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX10_##NAME, SHIFT, WIDTH)
 // [GFX10-GFX11].
 #define COMPUTE_PGM_RSRC3_GFX10_GFX11(NAME, SHIFT, WIDTH)                      \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX10_GFX11_##NAME, SHIFT, WIDTH)
+// [GFX10-GFX120].
+#define COMPUTE_PGM_RSRC3_GFX10_GFX120(NAME, SHIFT, WIDTH)                     \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX10_GFX120_##NAME, SHIFT, WIDTH)
 // GFX11+.
-#define COMPUTE_PGM_RSRC3_GFX11_PLUS(NAME, SHIFT, WIDTH) \
-  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX11_PLUS_ ## NAME, SHIFT, WIDTH)
+#define COMPUTE_PGM_RSRC3_GFX11_PLUS(NAME, SHIFT, WIDTH)                       \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX11_PLUS_##NAME, SHIFT, WIDTH)
 // [GFX11].
 #define COMPUTE_PGM_RSRC3_GFX11(NAME, SHIFT, WIDTH)                            \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX11_##NAME, SHIFT, WIDTH)
 // GFX12+.
 #define COMPUTE_PGM_RSRC3_GFX12_PLUS(NAME, SHIFT, WIDTH)                       \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX12_PLUS_##NAME, SHIFT, WIDTH)
+// [GFX125].
+#define COMPUTE_PGM_RSRC3_GFX125(NAME, SHIFT, WIDTH)                           \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_GFX125_##NAME, SHIFT, WIDTH)
 enum : int32_t {
   COMPUTE_PGM_RSRC3_GFX10_GFX11(SHARED_VGPR_COUNT, 0, 4),
   COMPUTE_PGM_RSRC3_GFX12_PLUS(RESERVED0, 0, 4),
@@ -206,8 +230,13 @@ enum : int32_t {
   COMPUTE_PGM_RSRC3_GFX10_PLUS(RESERVED2, 12, 1),
   COMPUTE_PGM_RSRC3_GFX10_GFX11(RESERVED3, 13, 1),
   COMPUTE_PGM_RSRC3_GFX12_PLUS(GLG_EN, 13, 1),
-  COMPUTE_PGM_RSRC3_GFX10_PLUS(RESERVED4, 14, 17),
-  COMPUTE_PGM_RSRC3_GFX10(RESERVED5, 31, 1),
+  COMPUTE_PGM_RSRC3_GFX10_GFX120(RESERVED4, 14, 8),
+  COMPUTE_PGM_RSRC3_GFX125(NAMED_BAR_CNT, 14, 3),
+  COMPUTE_PGM_RSRC3_GFX125(ENABLE_DYNAMIC_VGPR, 17, 1),
+  COMPUTE_PGM_RSRC3_GFX125(TCP_SPLIT, 18, 3),
+  COMPUTE_PGM_RSRC3_GFX125(ENABLE_DIDT_THROTTLE, 21, 1),
+  COMPUTE_PGM_RSRC3_GFX10_PLUS(RESERVED5, 22, 9),
+  COMPUTE_PGM_RSRC3_GFX10(RESERVED6, 31, 1),
   COMPUTE_PGM_RSRC3_GFX11_PLUS(IMAGE_OP, 31, 1),
 };
 #undef COMPUTE_PGM_RSRC3_GFX10_PLUS
diff --git a/llvm/include/llvm/Support/Atomic.h b/llvm/include/llvm/Support/Atomic.h
index a8445fddc1a85..c2d9ae2da231c 100644
--- a/llvm/include/llvm/Support/Atomic.h
+++ b/llvm/include/llvm/Support/Atomic.h
@@ -17,6 +17,7 @@
 #ifndef LLVM_SUPPORT_ATOMIC_H
 #define LLVM_SUPPORT_ATOMIC_H
 
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataTypes.h"
 
 // Windows will at times define MemoryFence.
@@ -26,16 +27,15 @@
 
 namespace llvm {
   namespace sys {
-    void MemoryFence();
+  LLVM_ABI void MemoryFence();
 
 #ifdef _MSC_VER
-    typedef long cas_flag;
+  typedef long cas_flag;
 #else
-    typedef uint32_t cas_flag;
+  typedef uint32_t cas_flag;
 #endif
-    cas_flag CompareAndSwap(volatile cas_flag* ptr,
-                            cas_flag new_value,
-                            cas_flag old_value);
+  LLVM_ABI cas_flag CompareAndSwap(volatile cas_flag *ptr, cas_flag new_value,
+                                   cas_flag old_value);
   }
 }
 
diff --git a/llvm/include/llvm/Support/DXILABI.h b/llvm/include/llvm/Support/DXILABI.h
index b479f7c73eba3..b25b3632f6c3b 100644
--- a/llvm/include/llvm/Support/DXILABI.h
+++ b/llvm/include/llvm/Support/DXILABI.h
@@ -17,6 +17,7 @@
 #ifndef LLVM_SUPPORT_DXILABI_H
 #define LLVM_SUPPORT_DXILABI_H
 
+#include "llvm/ADT/StringRef.h"
 #include <cstdint>
 
 namespace llvm {
@@ -99,6 +100,8 @@ enum class SamplerFeedbackType : uint32_t {
 const unsigned MinWaveSize = 4;
 const unsigned MaxWaveSize = 128;
 
+LLVM_ABI StringRef getResourceClassName(ResourceClass RC);
+
 } // namespace dxil
 } // namespace llvm
 
diff --git a/llvm/include/llvm/Support/Debug.h b/llvm/include/llvm/Support/Debug.h
index 55420891fe7d5..b73f2d7c8b852 100644
--- a/llvm/include/llvm/Support/Debug.h
+++ b/llvm/include/llvm/Support/Debug.h
@@ -15,7 +15,7 @@
 // LLVM_DEBUG() requires the DEBUG_TYPE macro to be defined. Set it to "foo"
 // specify that your debug code belongs to class "foo". Be careful that you only
 // do this after including Debug.h and not around any #include of headers.
-// Headers should define and undef the macro acround the code that needs to use
+// Headers should define and undef the macro around the code that needs to use
 // the LLVM_DEBUG() macro. Then, on the command line, you can specify
 // '-debug-only=foo' to enable JUST the debug information for the foo class.
 //
@@ -42,7 +42,7 @@ class raw_ostream;
 /// An optional level can be provided to control the verbosity of the output.
 /// If the provided level is not 0 and user specified a level below the provided
 /// level, return false.
-bool isCurrentDebugType(const char *Type, int Level = 0);
+LLVM_ABI bool isCurrentDebugType(const char *Type, int Level = 0);
 
 /// setCurrentDebugType - Set the current debug type, as if the -debug-only=X
 /// option were specified.  Note that DebugFlag also needs to be set to true for
@@ -52,13 +52,13 @@ bool isCurrentDebugType(const char *Type, int Level = 0);
 /// user specified a level at least as high as the provided level.
 /// 0 is a special level that acts as an opt-out for this specific debug type
 /// without affecting the other debug output.
-void setCurrentDebugType(const char *Type);
+LLVM_ABI void setCurrentDebugType(const char *Type);
 
 /// setCurrentDebugTypes - Set the current debug type, as if the
 /// -debug-only=X,Y,Z option were specified. Note that DebugFlag
 /// also needs to be set to true for debug output to be produced.
 ///
-void setCurrentDebugTypes(const char **Types, unsigned Count);
+LLVM_ABI void setCurrentDebugTypes(const char **Types, unsigned Count);
 
 /// DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug
 /// information.  If the '-debug' option is specified on the commandline, and if
@@ -71,7 +71,7 @@ void setCurrentDebugTypes(const char **Types, unsigned Count);
 /// is not specified, or is specified as "bitset".
 #define DEBUG_WITH_TYPE(TYPE, ...)                                             \
   do {                                                                         \
-    if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(TYPE)) {               \
+    if (::llvm::DebugFlag && ::llvm::isCurrentDebugType(TYPE, 1)) {            \
       __VA_ARGS__;                                                             \
     }                                                                          \
   } while (false)
diff --git a/llvm/include/llvm/Support/DebugLog.h b/llvm/include/llvm/Support/DebugLog.h
index a3312950da94e..ead5dd2a4e8bd 100644
--- a/llvm/include/llvm/Support/DebugLog.h
+++ b/llvm/include/llvm/Support/DebugLog.h
@@ -56,30 +56,30 @@ namespace llvm {
   DEBUGLOG_WITH_STREAM_AND_TYPE(llvm::dbgs(), LEVEL, DEBUG_TYPE)
 #define LDBG_LOG_LEVEL_1() LDBG_LOG_LEVEL(1)
 
+// We want the filename without the full path. We are using the __FILE__ macro
+// and a constexpr function to strip the path prefix. We can avoid the frontend
+// repeated evaluation of __FILE__ by using the __FILE_NAME__ when defined
+// (gcc and clang do) which contains the file name already.
+#if defined(__FILE_NAME__)
+#define __LLVM_FILE_NAME__ __FILE_NAME__
+#else
+#define __LLVM_FILE_NAME__ ::llvm::impl::getShortFileName(__FILE__)
+#endif
+
 #define DEBUGLOG_WITH_STREAM_TYPE_FILE_AND_LINE(STREAM, LEVEL, TYPE, FILE,     \
                                                 LINE)                          \
   for (bool _c =                                                               \
            (::llvm::DebugFlag && ::llvm::isCurrentDebugType(TYPE, LEVEL));     \
        _c; _c = false)                                                         \
-    for (::llvm::impl::RAIINewLineStream NewLineStream{(STREAM)}; _c;          \
-         _c = false)                                                           \
-  ::llvm::impl::raw_ldbg_ostream{                                              \
-      ::llvm::impl::computePrefix(TYPE, FILE, LINE, LEVEL), NewLineStream}     \
-      .asLvalue()
+    for (::llvm::impl::raw_ldbg_ostream LdbgOS{                                \
+             ::llvm::impl::computePrefix(TYPE, FILE, LINE, LEVEL), (STREAM)};  \
+         _c; _c = false)                                                       \
+  ::llvm::impl::RAIINewLineStream{LdbgOS}.asLvalue()
 
 #define DEBUGLOG_WITH_STREAM_TYPE_AND_FILE(STREAM, LEVEL, TYPE, FILE)          \
   DEBUGLOG_WITH_STREAM_TYPE_FILE_AND_LINE(STREAM, LEVEL, TYPE, FILE, __LINE__)
-// When __SHORT_FILE__ is not defined, the File is the full path,
-// otherwise __SHORT_FILE__ is defined in CMake to provide the file name
-// without the path prefix.
-#if defined(__SHORT_FILE__)
 #define DEBUGLOG_WITH_STREAM_AND_TYPE(STREAM, LEVEL, TYPE)                     \
-  DEBUGLOG_WITH_STREAM_TYPE_AND_FILE(STREAM, LEVEL, TYPE, __SHORT_FILE__)
-#else
-#define DEBUGLOG_WITH_STREAM_AND_TYPE(STREAM, LEVEL, TYPE)                     \
-  DEBUGLOG_WITH_STREAM_TYPE_AND_FILE(STREAM, LEVEL, TYPE,                      \
-                                     ::llvm::impl::getShortFileName(__FILE__))
-#endif
+  DEBUGLOG_WITH_STREAM_TYPE_AND_FILE(STREAM, LEVEL, TYPE, __LLVM_FILE_NAME__)
 
 namespace impl {
 
@@ -88,22 +88,22 @@ namespace impl {
 class LLVM_ABI raw_ldbg_ostream final : public raw_ostream {
   std::string Prefix;
   raw_ostream &Os;
-  bool HasPendingNewline;
+  bool ShouldPrefixNextString;
 
   /// Split the line on newlines and insert the prefix before each
   /// newline. Forward everything to the underlying stream.
   void write_impl(const char *Ptr, size_t Size) final {
     auto Str = StringRef(Ptr, Size);
-    // Handle the initial prefix.
-    if (!Str.empty())
-      writeWithPrefix(StringRef());
-
     auto Eol = Str.find('\n');
+    // Handle `\n` occurring in the string, ensure to print the prefix at the
+    // beginning of each line.
     while (Eol != StringRef::npos) {
+      // Take the line up to the newline (including the newline).
       StringRef Line = Str.take_front(Eol + 1);
       if (!Line.empty())
         writeWithPrefix(Line);
-      HasPendingNewline = true;
+      // We printed a newline, record here to print a prefix.
+      ShouldPrefixNextString = true;
       Str = Str.drop_front(Eol + 1);
       Eol = Str.find('\n');
     }
@@ -112,24 +112,21 @@ class LLVM_ABI raw_ldbg_ostream final : public raw_ostream {
   }
   void emitPrefix() { Os.write(Prefix.c_str(), Prefix.size()); }
   void writeWithPrefix(StringRef Str) {
-    flushEol();
+    if (ShouldPrefixNextString) {
+      emitPrefix();
+      ShouldPrefixNextString = false;
+    }
     Os.write(Str.data(), Str.size());
   }
 
 public:
   explicit raw_ldbg_ostream(std::string Prefix, raw_ostream &Os,
-                            bool HasPendingNewline = true)
+                            bool ShouldPrefixNextString = true)
       : Prefix(std::move(Prefix)), Os(Os),
-        HasPendingNewline(HasPendingNewline) {
+        ShouldPrefixNextString(ShouldPrefixNextString) {
     SetUnbuffered();
   }
-  ~raw_ldbg_ostream() final { flushEol(); }
-  void flushEol() {
-    if (HasPendingNewline) {
-      emitPrefix();
-      HasPendingNewline = false;
-    }
-  }
+  ~raw_ldbg_ostream() final {}
 
   /// Forward the current_pos method to the underlying stream.
   uint64_t current_pos() const final { return Os.tell(); }
@@ -148,6 +145,7 @@ class RAIINewLineStream final : public raw_ostream {
   ~RAIINewLineStream() { Os << '\n'; }
   void write_impl(const char *Ptr, size_t Size) final { Os.write(Ptr, Size); }
   uint64_t current_pos() const final { return Os.tell(); }
+  RAIINewLineStream &asLvalue() { return *this; }
 };
 
 /// Remove the path prefix from the file name.
diff --git a/llvm/include/llvm/Support/FileSystem.h b/llvm/include/llvm/Support/FileSystem.h
index 31fedc37bf776..a21b0a272d2b0 100644
--- a/llvm/include/llvm/Support/FileSystem.h
+++ b/llvm/include/llvm/Support/FileSystem.h
@@ -1171,6 +1171,12 @@ LLVM_ABI Expected<file_t>
 openNativeFileForRead(const Twine &Name, OpenFlags Flags = OF_None,
                       SmallVectorImpl<char> *RealPath = nullptr);
 
+/// An enumeration for the lock kind.
+enum class LockKind {
+  Exclusive, // Exclusive/writer lock
+  Shared     // Shared/reader lock
+};
+
 /// Try to locks the file during the specified time.
 ///
 /// This function implements advisory locking on entire file. If it returns
@@ -1184,6 +1190,7 @@ openNativeFileForRead(const Twine &Name, OpenFlags Flags = OF_None,
 /// @param Timeout Time in milliseconds that the process should wait before
 ///                reporting lock failure. Zero value means try to get lock only
 ///                once.
+/// @param Kind    The kind of the lock used (exclusive/shared).
 /// @returns errc::success if lock is successfully obtained,
 /// errc::no_lock_available if the file cannot be locked, or platform-specific
 /// error_code otherwise.
@@ -1194,12 +1201,15 @@ openNativeFileForRead(const Twine &Name, OpenFlags Flags = OF_None,
 /// descriptor.
 LLVM_ABI std::error_code
 tryLockFile(int FD,
-            std::chrono::milliseconds Timeout = std::chrono::milliseconds(0));
+            std::chrono::milliseconds Timeout = std::chrono::milliseconds(0),
+            LockKind Kind = LockKind::Exclusive);
 
 /// Lock the file.
 ///
 /// This function acts as @ref tryLockFile but it waits infinitely.
-LLVM_ABI std::error_code lockFile(int FD);
+/// \param FD file descriptor to use for locking.
+/// \param Kind of lock to used (exclusive/shared).
+LLVM_ABI std::error_code lockFile(int FD, LockKind Kind = LockKind::Exclusive);
 
 /// Unlock the file.
 ///
@@ -1342,6 +1352,11 @@ class mapped_file_region {
   LLVM_ABI size_t size() const;
   LLVM_ABI char *data() const;
 
+  /// Write changes to disk and synchronize. Equivalent to POSIX msync. This
+  /// will wait for flushing memory-mapped region back to disk and can be very
+  /// slow.
+  LLVM_ABI std::error_code sync() const;
+
   /// Get a const view of the data. Modifying this memory has undefined
   /// behavior.
   LLVM_ABI const char *const_data() const;
diff --git a/llvm/include/llvm/Support/GraphWriter.h b/llvm/include/llvm/Support/GraphWriter.h
index 39a4c0befbb89..a8784edfcf896 100644
--- a/llvm/include/llvm/Support/GraphWriter.h
+++ b/llvm/include/llvm/Support/GraphWriter.h
@@ -61,8 +61,8 @@ enum Name {
 LLVM_ABI bool DisplayGraph(StringRef Filename, bool wait = true,
                            GraphProgram::Name program = GraphProgram::DOT);
 
-template<typename GraphType>
-class GraphWriter {
+template <typename GraphType, typename Derived> class GraphWriterBase {
+protected:
   raw_ostream &O;
   const GraphType &G;
   bool RenderUsingHTML = false;
@@ -75,9 +75,15 @@ class GraphWriter {
   DOTTraits DTraits;
 
   static_assert(std::is_pointer_v<NodeRef>,
-                "FIXME: Currently GraphWriter requires the NodeRef type to be "
-                "a pointer.\nThe pointer usage should be moved to "
-                "DOTGraphTraits, and removed from GraphWriter itself.");
+                "FIXME: Currently GraphWriterBase requires the NodeRef type to "
+                "be a pointer.\nThe pointer usage should be moved to "
+                "DOTGraphTraits, and removed from GraphWriterBase itself.");
+
+  // Cast the 'this' pointer to the derived type and return a reference.
+  Derived &getDerived() { return *static_cast<Derived *>(this); }
+  const Derived &getDerived() const {
+    return *static_cast<const Derived *>(this);
+  }
 
   // Writes the edge labels of the node to O and returns true if there are any
   // edge labels not equal to the empty string "".
@@ -118,23 +124,24 @@ class GraphWriter {
   }
 
 public:
-  GraphWriter(raw_ostream &o, const GraphType &g, bool SN) : O(o), G(g) {
+  GraphWriterBase(raw_ostream &o, const GraphType &g, bool SN) : O(o), G(g) {
     DTraits = DOTTraits(SN);
     RenderUsingHTML = DTraits.renderNodesUsingHTML();
   }
+  virtual ~GraphWriterBase() {}
 
   void writeGraph(const std::string &Title = "") {
     // Output the header for the graph...
-    writeHeader(Title);
+    getDerived().writeHeader(Title);
 
     // Emit all of the nodes in the graph...
-    writeNodes();
+    getDerived().writeNodes();
 
     // Output any customizations on the graph
-    DOTGraphTraits<GraphType>::addCustomGraphFeatures(G, *this);
+    DOTGraphTraits<GraphType>::addCustomGraphFeatures(G, getDerived());
 
     // Output the end of the graph
-    writeFooter();
+    getDerived().writeFooter();
   }
 
   void writeHeader(const std::string &Title) {
@@ -166,8 +173,8 @@ class GraphWriter {
   void writeNodes() {
     // Loop over the graph, printing it out...
     for (const auto Node : nodes<GraphType>(G))
-      if (!isNodeHidden(Node))
-        writeNode(Node);
+      if (!getDerived().isNodeHidden(Node))
+        getDerived().writeNode(Node);
   }
 
   bool isNodeHidden(NodeRef Node) { return DTraits.isNodeHidden(Node, G); }
@@ -302,9 +309,9 @@ class GraphWriter {
       if (DTraits.getEdgeSourceLabel(Node, EI).empty())
         edgeidx = -1;
 
-      emitEdge(static_cast<const void*>(Node), edgeidx,
-               static_cast<const void*>(TargetNode), DestPort,
-               DTraits.getEdgeAttributes(Node, EI, G));
+      getDerived().emitEdge(static_cast<const void *>(Node), edgeidx,
+                            static_cast<const void *>(TargetNode), DestPort,
+                            DTraits.getEdgeAttributes(Node, EI, G));
     }
   }
 
@@ -357,10 +364,17 @@ class GraphWriter {
   }
 };
 
-template<typename GraphType>
+template <typename GraphType>
+class GraphWriter : public GraphWriterBase<GraphType, GraphWriter<GraphType>> {
+public:
+  GraphWriter(raw_ostream &o, const GraphType &g, bool SN)
+      : GraphWriterBase<GraphType, GraphWriter<GraphType>>(o, g, SN) {}
+  ~GraphWriter() override {}
+};
+
+template <typename GraphType>
 raw_ostream &WriteGraph(raw_ostream &O, const GraphType &G,
-                        bool ShortNames = false,
-                        const Twine &Title = "") {
+                        bool ShortNames = false, const Twine &Title = "") {
   // Start the graph emission process...
   GraphWriter<GraphType> W(O, G, ShortNames);
 
diff --git a/llvm/include/llvm/Support/MathExtras.h b/llvm/include/llvm/Support/MathExtras.h
index 7bbf1b3aab761..c9dae7ae7346d 100644
--- a/llvm/include/llvm/Support/MathExtras.h
+++ b/llvm/include/llvm/Support/MathExtras.h
@@ -74,6 +74,10 @@ constexpr float ef          = 0x1.5bf0a8P+1F, // (2.71828183) https://oeis.org/A
                 sqrt3f      = 0x1.bb67aeP+0F, // (1.73205081) https://oeis.org/A002194
                 inv_sqrt3f  = 0x1.279a74P-1F, // (.577350269)
                 phif        = 0x1.9e377aP+0F; // (1.61803399) https://oeis.org/A001622
+// These string literals are taken from below:
+// https://github.com/bminor/glibc/blob/8543577b04ded6d979ffcc5a818930e4d74d0645/math/math.h#L1215-L1229
+constexpr const char *pis     = "3.141592653589793238462643383279502884",
+                     *inv_pis = "0.318309886183790671537767526745028724";
 // clang-format on
 } // namespace numbers
 
diff --git a/llvm/include/llvm/Support/ScopedPrinter.h b/llvm/include/llvm/Support/ScopedPrinter.h
index e6c4cc4a4ea13..a08cc8fd31fd2 100644
--- a/llvm/include/llvm/Support/ScopedPrinter.h
+++ b/llvm/include/llvm/Support/ScopedPrinter.h
@@ -107,6 +107,17 @@ std::string enumToString(T Value, ArrayRef<EnumEntry<TEnum>> EnumValues) {
   return utohexstr(Value, true);
 }
 
+/// Retrieves the Value's enum name.
+///
+/// Returns an empty StringRef when an invalid value is provided.
+template <typename T, typename TEnum>
+StringRef enumToStringRef(T Value, ArrayRef<EnumEntry<TEnum>> EnumValues) {
+  for (const EnumEntry<TEnum> &EnumItem : EnumValues)
+    if (EnumItem.Value == Value)
+      return EnumItem.AltName;
+  return "";
+}
+
 class LLVM_ABI ScopedPrinter {
 public:
   enum class ScopedPrinterKind {
diff --git a/llvm/include/llvm/Support/SwapByteOrder.h b/llvm/include/llvm/Support/SwapByteOrder.h
index e6d4c26838c6a..e9d528111609d 100644
--- a/llvm/include/llvm/Support/SwapByteOrder.h
+++ b/llvm/include/llvm/Support/SwapByteOrder.h
@@ -26,7 +26,7 @@ namespace sys {
 constexpr bool IsBigEndianHost =
     llvm::endianness::native == llvm::endianness::big;
 
-static const bool IsLittleEndianHost = !IsBigEndianHost;
+constexpr bool IsLittleEndianHost = !IsBigEndianHost;
 
 inline unsigned char      getSwappedBytes(unsigned char      C) { return llvm::byteswap(C); }
 inline   signed char      getSwappedBytes( signed  char      C) { return llvm::byteswap(C); }
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index b619de39a8c75..9564b581c5ebb 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -560,6 +560,13 @@ def right_identity_neg_zero_fp: GICombineRule<
   (apply (GIReplaceReg $dst, $x))
 >;
 
+def right_identity_neg_zero_fp_nsz: GICombineRule<
+  (defs root:$dst),
+  (match (G_FADD $dst, $x, $y, (MIFlags FmNsz)):$root,
+    [{ return Helper.matchConstantFPOp(${y}, 0.0); }]),
+  (apply (GIReplaceReg $dst, $x))
+>;
+
 // Fold x op 1 -> x
 def right_identity_one_int: GICombineRule<
   (defs root:$dst),
@@ -1243,6 +1250,35 @@ def mulh_to_lshr : GICombineRule<
 
 def mulh_combines : GICombineGroup<[mulh_to_lshr]>;
 
+def trunc_ssats : GICombineRule<
+  (defs root:$root, register_matchinfo:$matchinfo),
+  (match (G_TRUNC $dst, $src):$root,
+         [{ return Helper.matchTruncSSatS(*${root}, ${matchinfo}); }]),
+  (apply [{ Helper.applyTruncSSatS(*${root}, ${matchinfo}); }])>;
+
+def trunc_ssatu : GICombineRule<
+  (defs root:$root, register_matchinfo:$matchinfo),
+  (match (G_TRUNC $dst, $src):$root,
+    [{ return Helper.matchTruncSSatU(*${root}, ${matchinfo}); }]),
+  (apply [{ Helper.applyTruncSSatU(*${root}, ${matchinfo}); }])>;
+
+def trunc_usatu : GICombineRule<
+  (defs root:$root),
+  (match (G_UMIN $min, $x, $y):$Min, 
+         (G_TRUNC $dst, $min):$root,
+    [{ return Helper.matchTruncUSatU(*${root}, *${Min}); }]),
+  (apply (G_TRUNC_USAT_U $dst, $x))>;
+
+def truncusatu_to_fptouisat : GICombineRule<
+  (defs root:$root),
+  (match (G_FPTOUI $src, $x):$Src,
+         (G_TRUNC_USAT_U $dst, $src):$root,
+    [{ return Helper.matchTruncUSatUToFPTOUISat(*${root}, *${Src}); }]),
+  (apply (G_FPTOUI_SAT $dst, $x))
+>;
+
+def truncsat_combines : GICombineGroup<[trunc_ssats, trunc_ssatu, trunc_usatu, truncusatu_to_fptouisat]>;
+
 def redundant_neg_operands: GICombineRule<
   (defs root:$root, build_fn_matchinfo:$matchinfo),
   (match (wip_match_opcode G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMAD, G_FMA):$root,
@@ -2003,7 +2039,7 @@ def identity_combines : GICombineGroup<[select_same_val, right_identity_zero,
                                         trunc_buildvector_fold,
                                         trunc_lshr_buildvector_fold,
                                         bitcast_bitcast_fold, fptrunc_fpext_fold,
-                                        right_identity_neg_zero_fp,
+                                        right_identity_neg_zero_fp, right_identity_neg_zero_fp_nsz,
                                         right_identity_neg_one_fp]>;
 
 def const_combines : GICombineGroup<[constant_fold_fp_ops, const_ptradd_to_i2p,
@@ -2067,7 +2103,7 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
     fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
     simplify_neg_minmax, combine_concat_vector,
     sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
-    combine_use_vector_truncate, merge_combines, overflow_combines]>;
+    combine_use_vector_truncate, merge_combines, overflow_combines, truncsat_combines]>;
 
 // A combine group used to for prelegalizer combiners at -O0. The combines in
 // this group have been selected based on experiments to balance code size and
diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index b65a63b5108dc..c0d480294dd8b 100644
--- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -51,6 +51,9 @@ def : GINodeEquiv<G_SEXT, sext>;
 def : GINodeEquiv<G_SEXT_INREG, sext_inreg>;
 def : GINodeEquiv<G_ZEXT, zext>;
 def : GINodeEquiv<G_TRUNC, trunc>;
+def : GINodeEquiv<G_TRUNC_SSAT_S, truncssat_s>;
+def : GINodeEquiv<G_TRUNC_SSAT_U, truncssat_u>;
+def : GINodeEquiv<G_TRUNC_USAT_U, truncusat_u>;
 def : GINodeEquiv<G_BITCAST, bitconvert>;
 // G_INTTOPTR - SelectionDAG has no equivalent.
 // G_PTRTOINT - SelectionDAG has no equivalent.
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index 4c83f8a580aa0..495b59ee916cf 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -250,6 +250,7 @@ class RegisterWithSubRegs<string n, list<Register> subregs> : Register<n> {
 class DAGOperand {
   string OperandNamespace = "MCOI";
   string DecoderMethod = "";
+  bit hasCompleteDecoder = true;
 }
 
 // RegisterClass - Now that all of the registers are defined, and aliases
@@ -1003,7 +1004,6 @@ class Operand<ValueType ty> : DAGOperand {
   ValueType Type = ty;
   string PrintMethod = "printOperand";
   string EncoderMethod = "";
-  bit hasCompleteDecoder = true;
   string OperandType = "OPERAND_UNKNOWN";
   dag MIOperandInfo = (ops);
 
diff --git a/llvm/include/llvm/Target/TargetCallingConv.td b/llvm/include/llvm/Target/TargetCallingConv.td
index d0533cad927a7..75d8ae44935b1 100644
--- a/llvm/include/llvm/Target/TargetCallingConv.td
+++ b/llvm/include/llvm/Target/TargetCallingConv.td
@@ -97,6 +97,12 @@ class CCIfVarArg<CCAction A> : CCIf<"State.isVarArg()", A> {}
 /// CCIfNotVarArg - If the current function is not vararg - apply the action
 class CCIfNotVarArg<CCAction A> : CCIf<"!State.isVarArg()", A> {}
 
+/// Apply the action if argument is fixed (not vararg).
+class CCIfArgFixed<CCAction A> : CCIf<"!ArgFlags.isVarArg()", A>;
+
+/// Apply the action if argument is vararg (not fixed).
+class CCIfArgVarArg<CCAction A> : CCIf<"ArgFlags.isVarArg()", A>;
+
 /// CCIfPtrAddrSpace - If the top-level parent of the current argument has
 /// pointer type in the specified address-space.
 class CCIfPtrAddrSpace<int AS, CCAction A>
diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.h b/llvm/include/llvm/TargetParser/ARMTargetParser.h
index 90eae9ef78da6..919598c894cd5 100644
--- a/llvm/include/llvm/TargetParser/ARMTargetParser.h
+++ b/llvm/include/llvm/TargetParser/ARMTargetParser.h
@@ -270,9 +270,9 @@ LLVM_ABI ProfileKind parseArchProfile(StringRef Arch);
 LLVM_ABI unsigned parseArchVersion(StringRef Arch);
 
 LLVM_ABI void fillValidCPUArchList(SmallVectorImpl<StringRef> &Values);
-LLVM_ABI StringRef computeDefaultTargetABI(const Triple &TT);
-
-LLVM_ABI ARMABI computeTargetABI(const Triple &TT, StringRef ABIName = "");
+LLVM_ABI LLVM_READONLY StringRef computeDefaultTargetABI(const Triple &TT);
+LLVM_ABI LLVM_READONLY ARMABI computeTargetABI(const Triple &TT,
+                                               StringRef ABIName = "");
 
 /// Get the (LLVM) name of the minimum ARM CPU for the arch we are targeting.
 ///
diff --git a/llvm/include/llvm/TargetParser/Host.h b/llvm/include/llvm/TargetParser/Host.h
index 40a9b6cc13902..b44b9b9a4d069 100644
--- a/llvm/include/llvm/TargetParser/Host.h
+++ b/llvm/include/llvm/TargetParser/Host.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_TARGETPARSER_HOST_H
 #define LLVM_TARGETPARSER_HOST_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/Compiler.h"
 #include <string>
 
@@ -63,6 +64,8 @@ namespace detail {
 /// Helper functions to extract HostCPUName from /proc/cpuinfo on linux.
 LLVM_ABI StringRef getHostCPUNameForPowerPC(StringRef ProcCpuinfoContent);
 LLVM_ABI StringRef getHostCPUNameForARM(StringRef ProcCpuinfoContent);
+LLVM_ABI StringRef getHostCPUNameForARM(uint64_t PrimaryCpuInfo,
+                                        ArrayRef<uint64_t> UniqueCpuInfos);
 LLVM_ABI StringRef getHostCPUNameForS390x(StringRef ProcCpuinfoContent);
 LLVM_ABI StringRef getHostCPUNameForRISCV(StringRef ProcCpuinfoContent);
 LLVM_ABI StringRef getHostCPUNameForSPARC(StringRef ProcCpuinfoContent);
diff --git a/llvm/include/llvm/TargetParser/XtensaTargetParser.def b/llvm/include/llvm/TargetParser/XtensaTargetParser.def
new file mode 100644
index 0000000000000..b0afe6f161770
--- /dev/null
+++ b/llvm/include/llvm/TargetParser/XtensaTargetParser.def
@@ -0,0 +1,79 @@
+//===- XtensaTargetParser.def - Xtensa target parsing defines ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file provides defines to build up the Xtensa target parser's logic.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef XTENSA_FEATURE
+#define XTENSA_FEATURE(ID, STR)
+#endif
+
+XTENSA_FEATURE(XF_DENSITY,                 "density")
+XTENSA_FEATURE(XF_FP,                      "fp")
+XTENSA_FEATURE(XF_WINDOWED,                "windowed")
+XTENSA_FEATURE(XF_BOOLEAN,                 "bool")
+XTENSA_FEATURE(XF_LOOP,                    "loop")
+XTENSA_FEATURE(XF_SEXT,                    "sext")
+XTENSA_FEATURE(XF_NSA,                     "nsa")
+XTENSA_FEATURE(XF_CLAMPS,                  "clamps")
+XTENSA_FEATURE(XF_MINMAX,                  "minmax")
+XTENSA_FEATURE(XF_MAC16,                   "mac16")
+XTENSA_FEATURE(XF_MUL32,                   "mul32")
+XTENSA_FEATURE(XF_MUL32HIGH,               "mul32high")
+XTENSA_FEATURE(XF_DIV32,                   "div32")
+XTENSA_FEATURE(XF_MUL16,                   "mul16")
+XTENSA_FEATURE(XF_DFPACCEL,                "dfpaccel")
+XTENSA_FEATURE(XF_S32C1I,                  "s32c1i")
+XTENSA_FEATURE(XF_THREADPTR,               "threadptr")
+XTENSA_FEATURE(XF_EXTENDEDL32R,            "extendedl32r")
+XTENSA_FEATURE(XF_DATACACHE,               "dcache")
+XTENSA_FEATURE(XF_DEBUG,                   "debug")
+XTENSA_FEATURE(XF_EXCEPTION,               "exception")
+XTENSA_FEATURE(XF_HIGHPRIINTERRUPTS,       "highpriinterrupts")
+XTENSA_FEATURE(XF_HIGHPRIINTERRUPTSLEVEL3, "highpriinterruptslevel3")
+XTENSA_FEATURE(XF_HIGHPRIINTERRUPTSLEVEL4, "highpriinterruptslevel4")
+XTENSA_FEATURE(XF_HIGHPRIINTERRUPTSLEVEL5, "highpriinterruptslevel5")
+XTENSA_FEATURE(XF_HIGHPRIINTERRUPTSLEVEL6, "highpriinterruptslevel6")
+XTENSA_FEATURE(XF_HIGHPRIINTERRUPTSLEVEL7, "highpriinterruptslevel7")
+XTENSA_FEATURE(XF_COPROCESSOR,             "coprocessor")
+XTENSA_FEATURE(XF_INTERRUPT,               "interrupt")
+XTENSA_FEATURE(XF_RVECTOR,                 "rvector")
+XTENSA_FEATURE(XF_TIMERS1,                 "timers1")
+XTENSA_FEATURE(XF_TIMERS2,                 "timers2")
+XTENSA_FEATURE(XF_TIMERS3,                 "timers3")
+XTENSA_FEATURE(XF_PRID,                    "prid")
+XTENSA_FEATURE(XF_REGPROTECT,              "regprotect")
+XTENSA_FEATURE(XF_MISCSR,                  "miscsr")
+
+#undef XTENSA_FEATURE
+
+#ifndef XTENSA_CPU
+#define XTENSA_CPU(ENUM, NAME, FEATURES)
+#endif
+
+XTENSA_CPU(INVALID, {"invalid"}, XF_INVALID)
+XTENSA_CPU(GENERIC, {"generic"}, XF_NONE)
+XTENSA_CPU(ESP8266, {"esp8266"},
+               (XF_DENSITY | XF_NSA | XF_MUL16 | XF_MUL32 | XF_EXTENDEDL32R | XF_DEBUG | XF_EXCEPTION |
+                XF_HIGHPRIINTERRUPTS | XF_HIGHPRIINTERRUPTSLEVEL3 | XF_INTERRUPT | XF_RVECTOR | XF_TIMERS1 |
+                XF_REGPROTECT | XF_PRID))
+XTENSA_CPU(ESP32,   {"esp32"},
+               (XF_DENSITY | XF_FP | XF_LOOP | XF_MAC16 | XF_WINDOWED | XF_BOOLEAN | XF_SEXT | XF_NSA |
+                XF_CLAMPS | XF_MINMAX | XF_MUL32 | XF_MUL32HIGH | XF_MUL16 | XF_DFPACCEL | XF_S32C1I |
+                XF_THREADPTR | XF_DIV32 | XF_DATACACHE | XF_DEBUG | XF_EXCEPTION | XF_HIGHPRIINTERRUPTS |
+                XF_HIGHPRIINTERRUPTSLEVEL7 | XF_COPROCESSOR | XF_INTERRUPT | XF_RVECTOR | XF_TIMERS3 | XF_PRID |
+				XF_REGPROTECT | XF_MISCSR))
+
+#undef XTENSA_CPU
+
+#ifndef XTENSA_CPU_ALIAS
+#define XTENSA_CPU_ALIAS(NAME, ALTNMAME)
+#endif
+
+#undef XTENSA_CPU_ALIAS
diff --git a/llvm/include/llvm/TargetParser/XtensaTargetParser.h b/llvm/include/llvm/TargetParser/XtensaTargetParser.h
new file mode 100644
index 0000000000000..828b4079ef328
--- /dev/null
+++ b/llvm/include/llvm/TargetParser/XtensaTargetParser.h
@@ -0,0 +1,79 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements a target parser to recognise Xtensa hardware features.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TARGETPARSER_XTENSATARGETPARSER_H
+#define LLVM_TARGETPARSER_XTENSATARGETPARSER_H
+
+#include "llvm/TargetParser/Triple.h"
+#include <vector>
+
+namespace llvm {
+class StringRef;
+
+namespace Xtensa {
+
+enum CPUKind : unsigned {
+#define XTENSA_CPU(ENUM, NAME, FEATURES) CK_##ENUM,
+#include "XtensaTargetParser.def"
+};
+
+enum XtensaFeatureKind : uint64_t {
+  XF_INVALID = 0,
+  XF_NONE = 1,
+  XF_FP = 1 << 1,
+  XF_WINDOWED = 1 << 2,
+  XF_BOOLEAN = 1 << 3,
+  XF_DENSITY = 1 << 4,
+  XF_LOOP = 1 << 5,
+  XF_SEXT = 1 << 6,
+  XF_NSA = 1 << 7,
+  XF_CLAMPS = 1 << 8,
+  XF_MINMAX = 1 << 9,
+  XF_MAC16 = 1 << 10,
+  XF_MUL32 = 1 << 11,
+  XF_MUL32HIGH = 1 << 12,
+  XF_DIV32 = 1 << 13,
+  XF_MUL16 = 1 << 14,
+  XF_DFPACCEL = 1 << 15,
+  XF_S32C1I = 1 << 16,
+  XF_THREADPTR = 1 << 17,
+  XF_EXTENDEDL32R = 1 << 18,
+  XF_DATACACHE = 1 << 19,
+  XF_DEBUG = 1 << 20,
+  XF_EXCEPTION = 1 << 21,
+  XF_HIGHPRIINTERRUPTS = 1 << 22,
+  XF_HIGHPRIINTERRUPTSLEVEL3 = 1 << 23,
+  XF_HIGHPRIINTERRUPTSLEVEL4 = 1 << 24,
+  XF_HIGHPRIINTERRUPTSLEVEL5 = 1 << 25,
+  XF_HIGHPRIINTERRUPTSLEVEL6 = 1 << 26,
+  XF_HIGHPRIINTERRUPTSLEVEL7 = 1 << 27,
+  XF_COPROCESSOR = 1 << 28,
+  XF_INTERRUPT = 1 << 29,
+  XF_RVECTOR = 1 << 30,
+  XF_TIMERS1 = 1ULL << 31,
+  XF_TIMERS2 = 1ULL << 32,
+  XF_TIMERS3 = 1ULL << 33,
+  XF_PRID = 1ULL << 34,
+  XF_REGPROTECT = 1ULL << 35,
+  XF_MISCSR = 1ULL << 36
+};
+
+CPUKind parseCPUKind(StringRef CPU);
+StringRef getBaseName(StringRef CPU);
+void getCPUFeatures(StringRef CPU, SmallVectorImpl<StringRef> &Features);
+void fillValidCPUList(SmallVectorImpl<StringRef> &Values);
+
+} // namespace Xtensa
+} // namespace llvm
+
+#endif // LLVM_SUPPORT_XTENSATARGETPARSER_H
diff --git a/llvm/include/llvm/TextAPI/Architecture.def b/llvm/include/llvm/TextAPI/Architecture.def
index 58ef31b25fe0b..53877cf0b60ce 100644
--- a/llvm/include/llvm/TextAPI/Architecture.def
+++ b/llvm/include/llvm/TextAPI/Architecture.def
@@ -13,33 +13,38 @@
 ///
 /// X86 architectures sorted by cpu type and sub type id.
 ///
-ARCHINFO(i386, MachO::CPU_TYPE_I386, MachO::CPU_SUBTYPE_I386_ALL, 32)
-ARCHINFO(x86_64, MachO::CPU_TYPE_X86_64, MachO::CPU_SUBTYPE_X86_64_ALL, 64)
-ARCHINFO(x86_64h, MachO::CPU_TYPE_X86_64, MachO::CPU_SUBTYPE_X86_64_H, 64)
+ARCHINFO(i386, i386, MachO::CPU_TYPE_I386, MachO::CPU_SUBTYPE_I386_ALL, 32)
+ARCHINFO(x86_64, x86_64, MachO::CPU_TYPE_X86_64, MachO::CPU_SUBTYPE_X86_64_ALL, 64)
+ARCHINFO(x86_64h, x86_64h, MachO::CPU_TYPE_X86_64, MachO::CPU_SUBTYPE_X86_64_H, 64)
 
 
 ///
 /// ARM architectures sorted by cpu sub type id.
 ///
-ARCHINFO(armv4t, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V4T, 32)
-ARCHINFO(armv6, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V6, 32)
-ARCHINFO(armv5, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V5TEJ, 32)
-ARCHINFO(armv7, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7, 32)
-ARCHINFO(armv7s, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7S, 32)
-ARCHINFO(armv7k, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7K, 32)
-ARCHINFO(armv6m, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V6M, 32)
-ARCHINFO(armv7m, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7M, 32)
-ARCHINFO(armv7em, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7EM, 32)
+ARCHINFO(armv4t, armv4t, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V4T, 32)
+ARCHINFO(armv6, armv6, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V6, 32)
+ARCHINFO(armv5, armv5, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V5TEJ, 32)
+ARCHINFO(armv7, armv7, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7, 32)
+ARCHINFO(armv7s, armv7s, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7S, 32)
+ARCHINFO(armv7k, armv7k, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7K, 32)
+ARCHINFO(armv6m, armv6m, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V6M, 32)
+ARCHINFO(armv7m, armv7m, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7M, 32)
+ARCHINFO(armv7em, armv7em, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7EM, 32)
 
 
 ///
 /// ARM64 architectures sorted by cpu sub type id.
 ///
-ARCHINFO(arm64, MachO::CPU_TYPE_ARM64, MachO::CPU_SUBTYPE_ARM64_ALL, 64)
-ARCHINFO(arm64e, MachO::CPU_TYPE_ARM64, MachO::CPU_SUBTYPE_ARM64E, 64)
+ARCHINFO(arm64, arm64, MachO::CPU_TYPE_ARM64, MachO::CPU_SUBTYPE_ARM64_ALL, 64)
+ARCHINFO(arm64e, arm64e, MachO::CPU_TYPE_ARM64, MachO::CPU_SUBTYPE_ARM64E, 64)
 
 
 ///
 /// ARM64_32 architectures sorted by cpu sub type id
 ///
-ARCHINFO(arm64_32, MachO::CPU_TYPE_ARM64_32, MachO::CPU_SUBTYPE_ARM64_32_V8, 32)
+ARCHINFO(arm64_32, arm64_32, MachO::CPU_TYPE_ARM64_32, MachO::CPU_SUBTYPE_ARM64_32_V8, 32)
+
+///
+/// RISCV32 architectures sorted by cpu sub type id
+///
+ARCHINFO(riscv32, riscv32, MachO::CPU_TYPE_RISCV, MachO::CPU_SUBTYPE_RISCV_ALL, 32)
diff --git a/llvm/include/llvm/TextAPI/Architecture.h b/llvm/include/llvm/TextAPI/Architecture.h
index 7a7f5416fe7c7..2ca199489f1d3 100644
--- a/llvm/include/llvm/TextAPI/Architecture.h
+++ b/llvm/include/llvm/TextAPI/Architecture.h
@@ -26,7 +26,7 @@ namespace MachO {
 
 /// Defines the architecture slices that are supported by Text-based Stub files.
 enum Architecture : uint8_t {
-#define ARCHINFO(Arch, Type, SubType, NumBits) AK_##Arch,
+#define ARCHINFO(Arch, Name, Type, SubType, NumBits) AK_##Arch,
 #include "llvm/TextAPI/Architecture.def"
 #undef ARCHINFO
   AK_unknown, // this has to go last.
diff --git a/llvm/include/llvm/Transforms/IPO/GlobalDCE.h b/llvm/include/llvm/Transforms/IPO/GlobalDCE.h
index f8f775b8ea96e..fb263599a5c13 100644
--- a/llvm/include/llvm/Transforms/IPO/GlobalDCE.h
+++ b/llvm/include/llvm/Transforms/IPO/GlobalDCE.h
@@ -32,6 +32,7 @@ class GlobalVariable;
 class Metadata;
 class Module;
 class Value;
+class ModulePass;
 
 /// Pass to remove unused function declarations.
 class GlobalDCEPass : public PassInfoMixin<GlobalDCEPass> {
@@ -80,6 +81,7 @@ class GlobalDCEPass : public PassInfoMixin<GlobalDCEPass> {
   void ComputeDependencies(Value *V, SmallPtrSetImpl<GlobalValue *> &U);
 };
 
+LLVM_ABI ModulePass *createGlobalDCEPass();
 }
 
 #endif // LLVM_TRANSFORMS_IPO_GLOBALDCE_H
diff --git a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
index 1fececbef2adc..1efe3ea127fd0 100644
--- a/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
+++ b/llvm/include/llvm/Transforms/IPO/SampleContextTracker.h
@@ -187,8 +187,8 @@ class SampleContextTracker {
 
 #ifndef NDEBUG
   // Get a context string from root to current node.
-  std::string getContextString(const FunctionSamples &FSamples) const;
-  std::string getContextString(ContextTrieNode *Node) const;
+  LLVM_ABI std::string getContextString(const FunctionSamples &FSamples) const;
+  LLVM_ABI std::string getContextString(ContextTrieNode *Node) const;
 #endif
   // Dump the internal context profile trie.
   LLVM_ABI void dump();
diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
index 5084e53b24397..ced446dacb6cc 100644
--- a/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
+++ b/llvm/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
@@ -104,8 +104,8 @@ class PGOMemOPSizeOpt : public PassInfoMixin<PGOMemOPSizeOpt> {
   LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &MAM);
 };
 
-LLVM_ABI void setProfMetadata(Module *M, Instruction *TI,
-                              ArrayRef<uint64_t> EdgeCounts, uint64_t MaxCount);
+LLVM_ABI void setProfMetadata(Instruction *TI, ArrayRef<uint64_t> EdgeCounts,
+                              uint64_t MaxCount);
 
 LLVM_ABI void setIrrLoopHeaderMetadata(Module *M, Instruction *TI,
                                        uint64_t Count);
diff --git a/llvm/include/llvm/Transforms/Scalar.h b/llvm/include/llvm/Transforms/Scalar.h
index 1398f171b0f78..8e68b6a57e51f 100644
--- a/llvm/include/llvm/Transforms/Scalar.h
+++ b/llvm/include/llvm/Transforms/Scalar.h
@@ -33,6 +33,13 @@ class Pass;
 //
 LLVM_ABI FunctionPass *createDeadCodeEliminationPass();
 
+//===----------------------------------------------------------------------===//
+//
+// DeadStoreElimination - This pass deletes stores that are post-dominated by
+// must-aliased stores and are not loaded used between the stores.
+//
+LLVM_ABI FunctionPass *createDeadStoreEliminationPass();
+
 //===----------------------------------------------------------------------===//
 //
 // SROA - Replace aggregates or pieces of aggregates with scalar SSA values.
diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h
index 6b56230a6e1d4..cfa06a5be79fd 100644
--- a/llvm/include/llvm/Transforms/Utils/Cloning.h
+++ b/llvm/include/llvm/Transforms/Utils/Cloning.h
@@ -279,6 +279,9 @@ class InlineFunctionInfo {
   /// `InlinedCalls` above is used.
   SmallVector<CallBase *, 8> InlinedCallSites;
 
+  Value *ConvergenceControlToken = nullptr;
+  Instruction *CallSiteEHPad = nullptr;
+
   /// Update profile for callee as well as cloned version. We need to do this
   /// for regular inlining, but not for inlining from sample profile loader.
   bool UpdateProfile;
@@ -287,9 +290,36 @@ class InlineFunctionInfo {
     StaticAllocas.clear();
     InlinedCalls.clear();
     InlinedCallSites.clear();
+    ConvergenceControlToken = nullptr;
+    CallSiteEHPad = nullptr;
   }
 };
 
+/// Check if it is legal to perform inlining of the function called by \p CB
+/// into the caller at this particular use, and sets fields in \p IFI.
+///
+/// This does not consider whether it is possible for the function callee itself
+/// to be inlined; for that see isInlineViable.
+LLVM_ABI InlineResult CanInlineCallSite(const CallBase &CB,
+                                        InlineFunctionInfo &IFI);
+
+/// This should generally not be used, use InlineFunction instead.
+///
+/// Perform mechanical inlining of \p CB into the caller.
+///
+/// This does not perform any legality or profitability checks for the
+/// inlining. This assumes that CanInlineCallSite was already called, populated
+/// \p IFI, and returned InlineResult::success.
+///
+/// Also assumes that isInlineViable returned InlineResult::success for the
+/// called function.
+LLVM_ABI void InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
+                                 bool MergeAttributes = false,
+                                 AAResults *CalleeAAR = nullptr,
+                                 bool InsertLifetime = true,
+                                 Function *ForwardVarArgsTo = nullptr,
+                                 OptimizationRemarkEmitter *ORE = nullptr);
+
 /// This function inlines the called function into the basic
 /// block of the caller.  This returns false if it is not possible to inline
 /// this call.  The program is still in a well defined state if this occurs
@@ -328,7 +358,8 @@ LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
                                      bool MergeAttributes = false,
                                      AAResults *CalleeAAR = nullptr,
                                      bool InsertLifetime = true,
-                                     Function *ForwardVarArgsTo = nullptr);
+                                     Function *ForwardVarArgsTo = nullptr,
+                                     OptimizationRemarkEmitter *ORE = nullptr);
 
 /// Clones a loop \p OrigLoop.  Returns the loop and the blocks in \p
 /// Blocks.
diff --git a/llvm/include/llvm/Transforms/Utils/Instrumentation.h b/llvm/include/llvm/Transforms/Utils/Instrumentation.h
index 962d9e734a40a..93ab8c693607f 100644
--- a/llvm/include/llvm/Transforms/Utils/Instrumentation.h
+++ b/llvm/include/llvm/Transforms/Utils/Instrumentation.h
@@ -169,26 +169,6 @@ struct SanitizerCoverageOptions {
   SanitizerCoverageOptions() = default;
 };
 
-/// Calculate what to divide by to scale counts.
-///
-/// Given the maximum count, calculate a divisor that will scale all the
-/// weights to strictly less than std::numeric_limits<uint32_t>::max().
-static inline uint64_t calculateCountScale(uint64_t MaxCount) {
-  return MaxCount < std::numeric_limits<uint32_t>::max()
-             ? 1
-             : MaxCount / std::numeric_limits<uint32_t>::max() + 1;
-}
-
-/// Scale an individual branch count.
-///
-/// Scale a 64-bit weight down to 32-bits using \c Scale.
-///
-static inline uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale) {
-  uint64_t Scaled = Count / Scale;
-  assert(Scaled <= std::numeric_limits<uint32_t>::max() && "overflow 32-bits");
-  return Scaled;
-}
-
 // Use to ensure the inserted instrumentation has a DebugLocation; if none is
 // attached to the source instruction, try to use a DILocation with offset 0
 // scoped to surrounding function (if it has a DebugLocation).
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index e4d2f9d191707..96e3d3d47f2d0 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -185,7 +185,8 @@ LLVM_ABI bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *,
                           TargetLibraryInfo *, Loop *, MemorySSAUpdater &,
                           ScalarEvolution *, ICFLoopSafetyInfo *,
                           SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *,
-                          bool, bool AllowSpeculation);
+                          bool, bool AllowSpeculation,
+                          bool HasCoroSuspendInst = false);
 
 /// Return true if the induction variable \p IV in a Loop whose latch is
 /// \p LatchBlock would become dead if the exit test \p Cond were removed.
@@ -370,6 +371,9 @@ LLVM_ABI bool canSinkOrHoistInst(Instruction &I, AAResults *AA,
 /// Returns the llvm.vector.reduce intrinsic that corresponds to the recurrence
 /// kind.
 LLVM_ABI constexpr Intrinsic::ID getReductionIntrinsicID(RecurKind RK);
+/// Returns the llvm.vector.reduce min/max intrinsic that corresponds to the
+/// intrinsic op.
+LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicID(Intrinsic::ID IID);
 
 /// Returns the arithmetic instruction opcode used when expanding a reduction.
 LLVM_ABI unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID);
diff --git a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
index c243e236901d5..3df3495f84470 100644
--- a/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
+++ b/llvm/include/llvm/Transforms/Utils/PredicateInfo.h
@@ -30,7 +30,7 @@
 /// %cmp = icmp eq i32, %x, 50
 /// br i1 %cmp, label %true, label %false
 /// true:
-/// %x.0 = call \@llvm.ssa_copy.i32(i32 %x)
+/// %x.0 = bitcast i32 %x to %x
 /// ret i32 %x.0
 /// false:
 /// ret i32 1
@@ -70,7 +70,7 @@ class raw_ostream;
 enum PredicateType { PT_Branch, PT_Assume, PT_Switch };
 
 /// Constraint for a predicate of the form "cmp Pred Op, OtherOp", where Op
-/// is the value the constraint applies to (the ssa.copy result).
+/// is the value the constraint applies to (the bitcast result).
 struct PredicateConstraint {
   CmpInst::Predicate Predicate;
   Value *OtherOp;
@@ -177,7 +177,6 @@ class PredicateInfo {
 public:
   LLVM_ABI PredicateInfo(Function &, DominatorTree &, AssumptionCache &,
                          BumpPtrAllocator &);
-  LLVM_ABI ~PredicateInfo();
 
   LLVM_ABI void verifyPredicateInfo() const;
 
@@ -200,10 +199,6 @@ class PredicateInfo {
   // the Predicate Info, they belong to the ValueInfo structs in the ValueInfos
   // vector.
   DenseMap<const Value *, const PredicateBase *> PredicateMap;
-  // The set of ssa_copy declarations we created with our custom mangling.
-  SmallSet<AssertingVH<Function>, 20> CreatedDeclarations;
-  // Cache of ssa.copy declaration for a given type.
-  SmallDenseMap<Type *, Function *> DeclarationCache;
 };
 
 /// Printer pass for \c PredicateInfo.
diff --git a/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h b/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h
index 746926e5bee33..52fe3a6f4baf4 100644
--- a/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h
+++ b/llvm/include/llvm/Transforms/Utils/SSAUpdaterImpl.h
@@ -366,7 +366,7 @@ class SSAUpdaterImpl {
         continue;
 
       // Look for an existing PHI.
-      FindExistingPHI(Info->BB, BlockList);
+      FindExistingPHI(Info->BB);
       if (Info->AvailableVal)
         continue;
 
@@ -412,11 +412,11 @@ class SSAUpdaterImpl {
 
   /// FindExistingPHI - Look through the PHI nodes in a block to see if any of
   /// them match what is needed.
-  void FindExistingPHI(BlkT *BB, BlockListTy *BlockList) {
+  void FindExistingPHI(BlkT *BB) {
     SmallVector<BBInfo *, 20> TaggedBlocks;
     for (auto &SomePHI : BB->phis()) {
       if (CheckIfPHIMatches(&SomePHI, TaggedBlocks)) {
-        RecordMatchingPHIs(BlockList);
+        RecordMatchingPHIs(TaggedBlocks);
         break;
       }
     }
@@ -424,7 +424,7 @@ class SSAUpdaterImpl {
 
   /// CheckIfPHIMatches - Check if a PHI node matches the placement and values
   /// in the BBMap.
-  bool CheckIfPHIMatches(PhiT *PHI, SmallVectorImpl<BBInfo *> &TaggedBlocks) {
+  bool CheckIfPHIMatches(PhiT *PHI, BlockListTy &TaggedBlocks) {
     // Match failed: clear all the PHITag values. Only need to clear visited
     // blocks.
     auto Cleanup = make_scope_exit([&]() {
@@ -484,15 +484,15 @@ class SSAUpdaterImpl {
 
   /// RecordMatchingPHIs - For each PHI node that matches, record it in both
   /// the BBMap and the AvailableVals mapping.
-  void RecordMatchingPHIs(BlockListTy *BlockList) {
-    for (typename BlockListTy::iterator I = BlockList->begin(),
-           E = BlockList->end(); I != E; ++I)
-      if (PhiT *PHI = (*I)->PHITag) {
-        BlkT *BB = PHI->getParent();
-        ValT PHIVal = Traits::GetPHIValue(PHI);
-        (*AvailableVals)[BB] = PHIVal;
-        BBMap[BB]->AvailableVal = PHIVal;
-      }
+  void RecordMatchingPHIs(BlockListTy &TaggedBlocks) {
+    for (BBInfo *Block : TaggedBlocks) {
+      PhiT *PHI = Block->PHITag;
+      assert(PHI && "PHITag didn't set?");
+      BlkT *BB = PHI->getParent();
+      ValT PHIVal = Traits::GetPHIValue(PHI);
+      (*AvailableVals)[BB] = PHIVal;
+      BBMap[BB]->AvailableVal = PHIVal;
+    }
   }
 };
 
diff --git a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 43ff084816d18..48ee93acbe008 100644
--- a/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -493,6 +493,9 @@ class LoopVectorizationLegality {
   /// and we only need to check individual instructions.
   bool canVectorizeInstrs();
 
+  /// Check if an individual instruction is vectorizable.
+  bool canVectorizeInstr(Instruction &I);
+
   /// When we vectorize loops we may change the order in which
   /// we read and write from memory. This method checks if it is
   /// legal to vectorize the code, considering only memory constrains.
diff --git a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
index ec70350691abe..b289520fa83af 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SandboxVectorizer/SeedCollector.h
@@ -191,7 +191,8 @@ class SeedContainer {
 
   ScalarEvolution &SE;
 
-  template <typename LoadOrStoreT> KeyT getKey(LoadOrStoreT *LSI) const;
+  template <typename LoadOrStoreT>
+  KeyT getKey(LoadOrStoreT *LSI, bool AllowDiffTypes) const;
 
 public:
   SeedContainer(ScalarEvolution &SE) : SE(SE) {}
@@ -267,7 +268,8 @@ class SeedContainer {
     bool operator!=(const iterator &Other) const { return !(*this == Other); }
   };
   using const_iterator = BundleMapT::const_iterator;
-  template <typename LoadOrStoreT> void insert(LoadOrStoreT *LSI);
+  template <typename LoadOrStoreT>
+  void insert(LoadOrStoreT *LSI, bool AllowDiffTypes);
   // To support constant-time erase, these just mark the element used, rather
   // than actually removing them from the bundle.
   LLVM_ABI bool erase(Instruction *I);
@@ -291,9 +293,9 @@ class SeedContainer {
 
 // Explicit instantiations
 extern template LLVM_TEMPLATE_ABI void
-SeedContainer::insert<LoadInst>(LoadInst *);
+SeedContainer::insert<LoadInst>(LoadInst *, bool);
 extern template LLVM_TEMPLATE_ABI void
-SeedContainer::insert<StoreInst>(StoreInst *);
+SeedContainer::insert<StoreInst>(StoreInst *, bool);
 
 class SeedCollector {
   SeedContainer StoreSeeds;
@@ -308,7 +310,8 @@ class SeedCollector {
 
 public:
   LLVM_ABI SeedCollector(BasicBlock *BB, ScalarEvolution &SE,
-                         bool CollectStores, bool CollectLoads);
+                         bool CollectStores, bool CollectLoads,
+                         bool AllowDiffTypes = false);
   LLVM_ABI ~SeedCollector();
 
   iterator_range<SeedContainer::iterator> getStoreSeeds() {
diff --git a/llvm/include/module.modulemap b/llvm/include/module.modulemap
index ac360b25d7529..f9597b569ce15 100644
--- a/llvm/include/module.modulemap
+++ b/llvm/include/module.modulemap
@@ -107,6 +107,12 @@ module LLVM_BinaryFormat {
     textual header "llvm/BinaryFormat/MsgPack.def"
 }
 
+module LLVM_CAS {
+  requires cplusplus
+  umbrella "llvm/CAS"
+  module * { export * }
+}
+
 module LLVM_Config {
   requires cplusplus
   umbrella "llvm/Config"
@@ -415,6 +421,7 @@ module LLVM_Utils {
     textual header "llvm/TargetParser/X86TargetParser.def"
     textual header "llvm/TargetParser/LoongArchTargetParser.def"
     textual header "llvm/TargetParser/PPCTargetParser.def"
+    textual header "llvm/TargetParser/XtensaTargetParser.def"
   }
 
   // This part of the module is usable from both C and C++ code.
diff --git a/llvm/lib/Analysis/CallPrinter.cpp b/llvm/lib/Analysis/CallPrinter.cpp
index 672dae1642cb3..69df122133b88 100644
--- a/llvm/lib/Analysis/CallPrinter.cpp
+++ b/llvm/lib/Analysis/CallPrinter.cpp
@@ -15,7 +15,6 @@
 
 #include "llvm/Analysis/CallPrinter.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/HeatUtils.h"
@@ -70,7 +69,7 @@ class CallGraphDOTInfo {
 
     for (Function &F : M->getFunctionList()) {
       uint64_t localSumFreq = 0;
-      SmallSet<Function *, 16> Callers;
+      SmallPtrSet<Function *, 16> Callers;
       for (User *U : F.users())
         if (isa<CallInst>(U))
           Callers.insert(cast<Instruction>(U)->getFunction());
@@ -99,7 +98,7 @@ class CallGraphDOTInfo {
 
       bool FoundParallelEdge = true;
       while (FoundParallelEdge) {
-        SmallSet<Function *, 16> Visited;
+        SmallPtrSet<Function *, 16> Visited;
         FoundParallelEdge = false;
         for (auto CI = Node->begin(), CE = Node->end(); CI != CE; CI++) {
           if (!(Visited.insert(CI->second->getFunction())).second) {
diff --git a/llvm/lib/Analysis/CaptureTracking.cpp b/llvm/lib/Analysis/CaptureTracking.cpp
index 076f4176c0219..71ca5131ee128 100644
--- a/llvm/lib/Analysis/CaptureTracking.cpp
+++ b/llvm/lib/Analysis/CaptureTracking.cpp
@@ -16,7 +16,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/CaptureTracking.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
@@ -359,6 +358,12 @@ UseCaptureInfo llvm::DetermineUseCaptureKind(const Use &U, const Value *Base) {
   case Instruction::AddrSpaceCast:
     // The original value is not captured via this if the new value isn't.
     return UseCaptureInfo::passthrough();
+  case Instruction::PtrToAddr:
+    // We treat ptrtoaddr as a location-independent capture of the address even
+    // if it is ultimately not used. Continuing recursive analysis after
+    // ptrtoaddr would be possible, but we'd need logic to do that correctly,
+    // which is not the same as the current pointer following logic.
+    return CaptureComponents::Address;
   case Instruction::ICmp: {
     unsigned Idx = U.getOperandNo();
     unsigned OtherIdx = 1 - Idx;
@@ -399,7 +404,7 @@ void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker,
 
   SmallVector<const Use *, 20> Worklist;
   Worklist.reserve(getDefaultMaxUsesToExploreForCaptureTracking());
-  SmallSet<const Use *, 20> Visited;
+  SmallPtrSet<const Use *, 20> Visited;
 
   auto AddUses = [&](const Value *V) {
     for (const Use &U : V->uses()) {
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 4969528a1b29b..2148431c1acce 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1485,6 +1485,9 @@ Constant *llvm::ConstantFoldCastOperand(unsigned Opcode, Constant *C,
   switch (Opcode) {
   default:
     llvm_unreachable("Missing case");
+  case Instruction::PtrToAddr:
+    // TODO: Add some of the ptrtoint folds here as well.
+    break;
   case Instruction::PtrToInt:
     if (auto *CE = dyn_cast<ConstantExpr>(C)) {
       Constant *FoldedValue = nullptr;
@@ -1844,6 +1847,62 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::nvvm_sqrt_rn_ftz_f:
     return !Call->isStrictFP();
 
+  // NVVM add intrinsics with explicit rounding modes
+  case Intrinsic::nvvm_add_rm_d:
+  case Intrinsic::nvvm_add_rn_d:
+  case Intrinsic::nvvm_add_rp_d:
+  case Intrinsic::nvvm_add_rz_d:
+  case Intrinsic::nvvm_add_rm_f:
+  case Intrinsic::nvvm_add_rn_f:
+  case Intrinsic::nvvm_add_rp_f:
+  case Intrinsic::nvvm_add_rz_f:
+  case Intrinsic::nvvm_add_rm_ftz_f:
+  case Intrinsic::nvvm_add_rn_ftz_f:
+  case Intrinsic::nvvm_add_rp_ftz_f:
+  case Intrinsic::nvvm_add_rz_ftz_f:
+
+  // NVVM div intrinsics with explicit rounding modes
+  case Intrinsic::nvvm_div_rm_d:
+  case Intrinsic::nvvm_div_rn_d:
+  case Intrinsic::nvvm_div_rp_d:
+  case Intrinsic::nvvm_div_rz_d:
+  case Intrinsic::nvvm_div_rm_f:
+  case Intrinsic::nvvm_div_rn_f:
+  case Intrinsic::nvvm_div_rp_f:
+  case Intrinsic::nvvm_div_rz_f:
+  case Intrinsic::nvvm_div_rm_ftz_f:
+  case Intrinsic::nvvm_div_rn_ftz_f:
+  case Intrinsic::nvvm_div_rp_ftz_f:
+  case Intrinsic::nvvm_div_rz_ftz_f:
+
+  // NVVM mul intrinsics with explicit rounding modes
+  case Intrinsic::nvvm_mul_rm_d:
+  case Intrinsic::nvvm_mul_rn_d:
+  case Intrinsic::nvvm_mul_rp_d:
+  case Intrinsic::nvvm_mul_rz_d:
+  case Intrinsic::nvvm_mul_rm_f:
+  case Intrinsic::nvvm_mul_rn_f:
+  case Intrinsic::nvvm_mul_rp_f:
+  case Intrinsic::nvvm_mul_rz_f:
+  case Intrinsic::nvvm_mul_rm_ftz_f:
+  case Intrinsic::nvvm_mul_rn_ftz_f:
+  case Intrinsic::nvvm_mul_rp_ftz_f:
+  case Intrinsic::nvvm_mul_rz_ftz_f:
+
+  // NVVM fma intrinsics with explicit rounding modes
+  case Intrinsic::nvvm_fma_rm_d:
+  case Intrinsic::nvvm_fma_rn_d:
+  case Intrinsic::nvvm_fma_rp_d:
+  case Intrinsic::nvvm_fma_rz_d:
+  case Intrinsic::nvvm_fma_rm_f:
+  case Intrinsic::nvvm_fma_rn_f:
+  case Intrinsic::nvvm_fma_rp_f:
+  case Intrinsic::nvvm_fma_rz_f:
+  case Intrinsic::nvvm_fma_rm_ftz_f:
+  case Intrinsic::nvvm_fma_rn_ftz_f:
+  case Intrinsic::nvvm_fma_rp_ftz_f:
+  case Intrinsic::nvvm_fma_rz_ftz_f:
+
   // Sign operations are actually bitwise operations, they do not raise
   // exceptions even for SNANs.
   case Intrinsic::fabs:
@@ -3319,6 +3378,96 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
 
         return ConstantFP::get(Ty->getContext(), Res);
       }
+
+      case Intrinsic::nvvm_add_rm_f:
+      case Intrinsic::nvvm_add_rn_f:
+      case Intrinsic::nvvm_add_rp_f:
+      case Intrinsic::nvvm_add_rz_f:
+      case Intrinsic::nvvm_add_rm_d:
+      case Intrinsic::nvvm_add_rn_d:
+      case Intrinsic::nvvm_add_rp_d:
+      case Intrinsic::nvvm_add_rz_d:
+      case Intrinsic::nvvm_add_rm_ftz_f:
+      case Intrinsic::nvvm_add_rn_ftz_f:
+      case Intrinsic::nvvm_add_rp_ftz_f:
+      case Intrinsic::nvvm_add_rz_ftz_f: {
+
+        bool IsFTZ = nvvm::FAddShouldFTZ(IntrinsicID);
+        APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
+        APFloat B = IsFTZ ? FTZPreserveSign(Op2V) : Op2V;
+
+        APFloat::roundingMode RoundMode =
+            nvvm::GetFAddRoundingMode(IntrinsicID);
+
+        APFloat Res = A;
+        APFloat::opStatus Status = Res.add(B, RoundMode);
+
+        if (!Res.isNaN() &&
+            (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+          Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+          return ConstantFP::get(Ty->getContext(), Res);
+        }
+        return nullptr;
+      }
+
+      case Intrinsic::nvvm_mul_rm_f:
+      case Intrinsic::nvvm_mul_rn_f:
+      case Intrinsic::nvvm_mul_rp_f:
+      case Intrinsic::nvvm_mul_rz_f:
+      case Intrinsic::nvvm_mul_rm_d:
+      case Intrinsic::nvvm_mul_rn_d:
+      case Intrinsic::nvvm_mul_rp_d:
+      case Intrinsic::nvvm_mul_rz_d:
+      case Intrinsic::nvvm_mul_rm_ftz_f:
+      case Intrinsic::nvvm_mul_rn_ftz_f:
+      case Intrinsic::nvvm_mul_rp_ftz_f:
+      case Intrinsic::nvvm_mul_rz_ftz_f: {
+
+        bool IsFTZ = nvvm::FMulShouldFTZ(IntrinsicID);
+        APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
+        APFloat B = IsFTZ ? FTZPreserveSign(Op2V) : Op2V;
+
+        APFloat::roundingMode RoundMode =
+            nvvm::GetFMulRoundingMode(IntrinsicID);
+
+        APFloat Res = A;
+        APFloat::opStatus Status = Res.multiply(B, RoundMode);
+
+        if (!Res.isNaN() &&
+            (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+          Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+          return ConstantFP::get(Ty->getContext(), Res);
+        }
+        return nullptr;
+      }
+
+      case Intrinsic::nvvm_div_rm_f:
+      case Intrinsic::nvvm_div_rn_f:
+      case Intrinsic::nvvm_div_rp_f:
+      case Intrinsic::nvvm_div_rz_f:
+      case Intrinsic::nvvm_div_rm_d:
+      case Intrinsic::nvvm_div_rn_d:
+      case Intrinsic::nvvm_div_rp_d:
+      case Intrinsic::nvvm_div_rz_d:
+      case Intrinsic::nvvm_div_rm_ftz_f:
+      case Intrinsic::nvvm_div_rn_ftz_f:
+      case Intrinsic::nvvm_div_rp_ftz_f:
+      case Intrinsic::nvvm_div_rz_ftz_f: {
+        bool IsFTZ = nvvm::FDivShouldFTZ(IntrinsicID);
+        APFloat A = IsFTZ ? FTZPreserveSign(Op1V) : Op1V;
+        APFloat B = IsFTZ ? FTZPreserveSign(Op2V) : Op2V;
+        APFloat::roundingMode RoundMode =
+            nvvm::GetFDivRoundingMode(IntrinsicID);
+
+        APFloat Res = A;
+        APFloat::opStatus Status = Res.divide(B, RoundMode);
+        if (!Res.isNaN() &&
+            (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+          Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+          return ConstantFP::get(Ty->getContext(), Res);
+        }
+        return nullptr;
+      }
       }
 
       if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy())
@@ -3730,6 +3879,38 @@ static Constant *ConstantFoldScalarCall3(StringRef Name,
           V.fusedMultiplyAdd(C2, C3, APFloat::rmNearestTiesToEven);
           return ConstantFP::get(Ty->getContext(), V);
         }
+
+        case Intrinsic::nvvm_fma_rm_f:
+        case Intrinsic::nvvm_fma_rn_f:
+        case Intrinsic::nvvm_fma_rp_f:
+        case Intrinsic::nvvm_fma_rz_f:
+        case Intrinsic::nvvm_fma_rm_d:
+        case Intrinsic::nvvm_fma_rn_d:
+        case Intrinsic::nvvm_fma_rp_d:
+        case Intrinsic::nvvm_fma_rz_d:
+        case Intrinsic::nvvm_fma_rm_ftz_f:
+        case Intrinsic::nvvm_fma_rn_ftz_f:
+        case Intrinsic::nvvm_fma_rp_ftz_f:
+        case Intrinsic::nvvm_fma_rz_ftz_f: {
+          bool IsFTZ = nvvm::FMAShouldFTZ(IntrinsicID);
+          APFloat A = IsFTZ ? FTZPreserveSign(C1) : C1;
+          APFloat B = IsFTZ ? FTZPreserveSign(C2) : C2;
+          APFloat C = IsFTZ ? FTZPreserveSign(C3) : C3;
+
+          APFloat::roundingMode RoundMode =
+              nvvm::GetFMARoundingMode(IntrinsicID);
+
+          APFloat Res = A;
+          APFloat::opStatus Status = Res.fusedMultiplyAdd(B, C, RoundMode);
+
+          if (!Res.isNaN() &&
+              (Status == APFloat::opOK || Status == APFloat::opInexact)) {
+            Res = IsFTZ ? FTZPreserveSign(Res) : Res;
+            return ConstantFP::get(Ty->getContext(), Res);
+          }
+          return nullptr;
+        }
+
         case Intrinsic::amdgcn_cubeid:
         case Intrinsic::amdgcn_cubema:
         case Intrinsic::amdgcn_cubesc:
@@ -4008,7 +4189,7 @@ static Constant *ConstantFoldFixedVectorCall(
       MulVector[I] = Elt0->getSExtValue() * Elt1->getSExtValue();
     }
     for (unsigned I = 0; I < Result.size(); I++) {
-      int32_t IAdd = MulVector[I * 2] + MulVector[I * 2 + 1];
+      int64_t IAdd = (int64_t)MulVector[I * 2] + (int64_t)MulVector[I * 2 + 1];
       Result[I] = ConstantInt::get(Ty, IAdd);
     }
 
diff --git a/llvm/lib/Analysis/DXILResource.cpp b/llvm/lib/Analysis/DXILResource.cpp
index 629fa7cddb9d4..3a70666029248 100644
--- a/llvm/lib/Analysis/DXILResource.cpp
+++ b/llvm/lib/Analysis/DXILResource.cpp
@@ -20,6 +20,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/Support/DXILABI.h"
 #include "llvm/Support/FormatVariadic.h"
 #include <cstdint>
 #include <optional>
@@ -29,20 +30,6 @@
 using namespace llvm;
 using namespace dxil;
 
-static StringRef getResourceClassName(ResourceClass RC) {
-  switch (RC) {
-  case ResourceClass::SRV:
-    return "SRV";
-  case ResourceClass::UAV:
-    return "UAV";
-  case ResourceClass::CBuffer:
-    return "CBuffer";
-  case ResourceClass::Sampler:
-    return "Sampler";
-  }
-  llvm_unreachable("Unhandled ResourceClass");
-}
-
 static StringRef getResourceKindName(ResourceKind RK) {
   switch (RK) {
   case ResourceKind::Texture1D:
@@ -612,7 +599,12 @@ void ResourceTypeInfo::print(raw_ostream &OS, const DataLayout &DL) const {
 
 GlobalVariable *ResourceInfo::createSymbol(Module &M, StructType *Ty) {
   assert(!Symbol && "Symbol has already been created");
-  Symbol = new GlobalVariable(M, Ty, /*isConstant=*/true,
+  Type *ResTy = Ty;
+  int64_t Size = Binding.Size;
+  if (Size != 1)
+    // unbounded arrays are represented as zero-sized arrays in LLVM IR
+    ResTy = ArrayType::get(Ty, Size == ~0u ? 0 : Size);
+  Symbol = new GlobalVariable(M, ResTy, /*isConstant=*/true,
                               GlobalValue::ExternalLinkage,
                               /*Initializer=*/nullptr, Name);
   return Symbol;
diff --git a/llvm/lib/Analysis/Delinearization.cpp b/llvm/lib/Analysis/Delinearization.cpp
index 329bd35530c72..762d9191aab1e 100644
--- a/llvm/lib/Analysis/Delinearization.cpp
+++ b/llvm/lib/Analysis/Delinearization.cpp
@@ -24,6 +24,7 @@
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -32,6 +33,11 @@ using namespace llvm;
 #define DL_NAME "delinearize"
 #define DEBUG_TYPE DL_NAME
 
+static cl::opt<bool> UseFixedSizeArrayHeuristic(
+    "delinearize-use-fixed-size-array-heuristic", cl::init(false), cl::Hidden,
+    cl::desc("When printing analysis, use the heuristic for fixed-size arrays "
+             "if the default delinearizetion fails."));
+
 // Return true when S contains at least an undef value.
 static inline bool containsUndefs(const SCEV *S) {
   return SCEVExprContains(S, [](const SCEV *S) {
@@ -480,6 +486,184 @@ void llvm::delinearize(ScalarEvolution &SE, const SCEV *Expr,
   });
 }
 
+static std::optional<APInt> tryIntoAPInt(const SCEV *S) {
+  if (const auto *Const = dyn_cast<SCEVConstant>(S))
+    return Const->getAPInt();
+  return std::nullopt;
+}
+
+/// Collects the absolute values of constant steps for all induction variables.
+/// Returns true if we can prove that all step recurrences are constants and \p
+/// Expr is divisible by \p ElementSize. Each step recurrence is stored in \p
+/// Steps after divided by \p ElementSize.
+static bool collectConstantAbsSteps(ScalarEvolution &SE, const SCEV *Expr,
+                                    SmallVectorImpl<uint64_t> &Steps,
+                                    uint64_t ElementSize) {
+  // End of recursion. The constant value also must be a multiple of
+  // ElementSize.
+  if (const auto *Const = dyn_cast<SCEVConstant>(Expr)) {
+    const uint64_t Mod = Const->getAPInt().urem(ElementSize);
+    return Mod == 0;
+  }
+
+  const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Expr);
+  if (!AR || !AR->isAffine())
+    return false;
+
+  const SCEV *Step = AR->getStepRecurrence(SE);
+  std::optional<APInt> StepAPInt = tryIntoAPInt(Step);
+  if (!StepAPInt)
+    return false;
+
+  APInt Q;
+  uint64_t R;
+  APInt::udivrem(StepAPInt->abs(), ElementSize, Q, R);
+  if (R != 0)
+    return false;
+
+  // Bail out when the step is too large.
+  std::optional<uint64_t> StepVal = Q.tryZExtValue();
+  if (!StepVal)
+    return false;
+
+  Steps.push_back(*StepVal);
+  return collectConstantAbsSteps(SE, AR->getStart(), Steps, ElementSize);
+}
+
+bool llvm::findFixedSizeArrayDimensions(ScalarEvolution &SE, const SCEV *Expr,
+                                        SmallVectorImpl<uint64_t> &Sizes,
+                                        const SCEV *ElementSize) {
+  if (!ElementSize)
+    return false;
+
+  std::optional<APInt> ElementSizeAPInt = tryIntoAPInt(ElementSize);
+  if (!ElementSizeAPInt || *ElementSizeAPInt == 0)
+    return false;
+
+  std::optional<uint64_t> ElementSizeConst = ElementSizeAPInt->tryZExtValue();
+
+  // Early exit when ElementSize is not a positive constant.
+  if (!ElementSizeConst)
+    return false;
+
+  if (!collectConstantAbsSteps(SE, Expr, Sizes, *ElementSizeConst) ||
+      Sizes.empty()) {
+    Sizes.clear();
+    return false;
+  }
+
+  // At this point, Sizes contains the absolute step recurrences for all
+  // induction variables. Each step recurrence must be a multiple of the size of
+  // the array element. Assuming that the each value represents the size of an
+  // array for each dimension, attempts to restore the length of each dimension
+  // by dividing the step recurrence by the next smaller value. For example, if
+  // we have the following AddRec SCEV:
+  //
+  //   AddRec: {{{0,+,2048}<%for.i>,+,256}<%for.j>,+,8}<%for.k> (ElementSize=8)
+  //
+  // Then Sizes will become [256, 32, 1] after sorted. We don't know the size of
+  // the outermost dimension, the next dimension will be computed as 256 / 32 =
+  // 8, and the last dimension will be computed as 32 / 1 = 32. Thus it results
+  // in like Arr[UnknownSize][8][32] with elements of size 8 bytes, where Arr is
+  // a base pointer.
+  //
+  // TODO: Catch more cases, e.g., when a step recurrence is not divisible by
+  // the next smaller one, like A[i][3*j].
+  llvm::sort(Sizes.rbegin(), Sizes.rend());
+  Sizes.erase(llvm::unique(Sizes), Sizes.end());
+
+  // The last element in Sizes should be ElementSize. At this point, all values
+  // in Sizes are assumed to be divided by ElementSize, so replace it with 1.
+  assert(Sizes.back() != 0 && "Unexpected zero size in Sizes.");
+  Sizes.back() = 1;
+
+  for (unsigned I = 0; I + 1 < Sizes.size(); I++) {
+    uint64_t PrevSize = Sizes[I + 1];
+    if (Sizes[I] % PrevSize) {
+      Sizes.clear();
+      return false;
+    }
+    Sizes[I] /= PrevSize;
+  }
+
+  // Finally, the last element in Sizes should be ElementSize.
+  Sizes.back() = *ElementSizeConst;
+  return true;
+}
+
+/// Splits the SCEV into two vectors of SCEVs representing the subscripts and
+/// sizes of an array access, assuming that the array is a fixed size array.
+///
+/// E.g., if we have the code like as follows:
+///
+///  double A[42][8][32];
+///  for i
+///    for j
+///      for k
+///        use A[i][j][k]
+///
+/// The access function will be represented as an AddRec SCEV like:
+///
+///  AddRec: {{{0,+,2048}<%for.i>,+,256}<%for.j>,+,8}<%for.k> (ElementSize=8)
+///
+/// Then findFixedSizeArrayDimensions infers the size of each dimension of the
+/// array based on the fact that the value of the step recurrence is a multiple
+/// of the size of the corresponding array element. In the above example, it
+/// results in the following:
+///
+///  CHECK: ArrayDecl[UnknownSize][8][32] with elements of 8 bytes.
+///
+/// Finally each subscript will be computed as follows:
+///
+///  CHECK: ArrayRef[{0,+,1}<%for.i>][{0,+,1}<%for.j>][{0,+,1}<%for.k>]
+///
+/// Note that this function doesn't check the range of possible values for each
+/// subscript, so the caller should perform additional boundary checks if
+/// necessary.
+///
+/// Also note that this function doesn't guarantee that the original array size
+/// is restored "correctly". For example, in the following case:
+///
+///  double A[42][4][64];
+///  double B[42][8][32];
+///  for i
+///    for j
+///      for k
+///        use A[i][j][k]
+///        use B[i][2*j][k]
+///
+/// The access function for both accesses will be the same:
+///
+///  AddRec: {{{0,+,2048}<%for.i>,+,512}<%for.j>,+,8}<%for.k> (ElementSize=8)
+///
+/// The array sizes for both A and B will be computed as
+/// ArrayDecl[UnknownSize][4][64], which matches for A, but not for B.
+///
+/// TODO: At the moment, this function can handle only simple cases. For
+/// example, we cannot handle a case where a step recurrence is not divisible
+/// by the next smaller step recurrence, e.g., A[i][3*j].
+bool llvm::delinearizeFixedSizeArray(ScalarEvolution &SE, const SCEV *Expr,
+                                     SmallVectorImpl<const SCEV *> &Subscripts,
+                                     SmallVectorImpl<const SCEV *> &Sizes,
+                                     const SCEV *ElementSize) {
+
+  // First step: find the fixed array size.
+  SmallVector<uint64_t, 4> ConstSizes;
+  if (!findFixedSizeArrayDimensions(SE, Expr, ConstSizes, ElementSize)) {
+    Sizes.clear();
+    return false;
+  }
+
+  // Convert the constant size to SCEV.
+  for (uint64_t Size : ConstSizes)
+    Sizes.push_back(SE.getConstant(Expr->getType(), Size));
+
+  // Second step: compute the access functions for each subscript.
+  computeAccessFunctions(SE, Expr, Subscripts, Sizes);
+
+  return !Subscripts.empty();
+}
+
 bool llvm::getIndexExpressionsFromGEP(ScalarEvolution &SE,
                                       const GetElementPtrInst *GEP,
                                       SmallVectorImpl<const SCEV *> &Subscripts,
@@ -560,35 +744,50 @@ namespace {
 
 void printDelinearization(raw_ostream &O, Function *F, LoopInfo *LI,
                           ScalarEvolution *SE) {
-  O << "Delinearization on function " << F->getName() << ":\n";
+  O << "Printing analysis 'Delinearization' for function '" << F->getName()
+    << "':";
   for (Instruction &Inst : instructions(F)) {
     // Only analyze loads and stores.
-    if (!isa<StoreInst>(&Inst) && !isa<LoadInst>(&Inst) &&
-        !isa<GetElementPtrInst>(&Inst))
+    if (!isa<StoreInst>(&Inst) && !isa<LoadInst>(&Inst))
       continue;
 
     const BasicBlock *BB = Inst.getParent();
-    // Delinearize the memory access as analyzed in all the surrounding loops.
+    Loop *L = LI->getLoopFor(BB);
+    // Only delinearize the memory access in the innermost loop.
     // Do not analyze memory accesses outside loops.
-    for (Loop *L = LI->getLoopFor(BB); L != nullptr; L = L->getParentLoop()) {
-      const SCEV *AccessFn = SE->getSCEVAtScope(getPointerOperand(&Inst), L);
+    if (!L)
+      continue;
 
-      const SCEVUnknown *BasePointer =
-          dyn_cast<SCEVUnknown>(SE->getPointerBase(AccessFn));
-      // Do not delinearize if we cannot find the base pointer.
-      if (!BasePointer)
-        break;
-      AccessFn = SE->getMinusSCEV(AccessFn, BasePointer);
+    const SCEV *AccessFn = SE->getSCEVAtScope(getPointerOperand(&Inst), L);
 
-      O << "\n";
-      O << "Inst:" << Inst << "\n";
-      O << "In Loop with Header: " << L->getHeader()->getName() << "\n";
-      O << "AccessFunction: " << *AccessFn << "\n";
-
-      SmallVector<const SCEV *, 3> Subscripts, Sizes;
-      delinearize(*SE, AccessFn, Subscripts, Sizes, SE->getElementSize(&Inst));
-      if (Subscripts.size() == 0 || Sizes.size() == 0 ||
-          Subscripts.size() != Sizes.size()) {
+    const SCEVUnknown *BasePointer =
+        dyn_cast<SCEVUnknown>(SE->getPointerBase(AccessFn));
+    // Do not delinearize if we cannot find the base pointer.
+    if (!BasePointer)
+      break;
+    AccessFn = SE->getMinusSCEV(AccessFn, BasePointer);
+
+    O << "\n";
+    O << "Inst:" << Inst << "\n";
+    O << "In Loop with Header: " << L->getHeader()->getName() << "\n";
+    O << "AccessFunction: " << *AccessFn << "\n";
+
+    SmallVector<const SCEV *, 3> Subscripts, Sizes;
+
+    auto IsDelinearizationFailed = [&]() {
+      return Subscripts.size() == 0 || Sizes.size() == 0 ||
+             Subscripts.size() != Sizes.size();
+    };
+
+    delinearize(*SE, AccessFn, Subscripts, Sizes, SE->getElementSize(&Inst));
+    if (UseFixedSizeArrayHeuristic && IsDelinearizationFailed()) {
+      Subscripts.clear();
+      Sizes.clear();
+      delinearizeFixedSizeArray(*SE, AccessFn, Subscripts, Sizes,
+                                SE->getElementSize(&Inst));
+    }
+
+      if (IsDelinearizationFailed()) {
         O << "failed to delinearize\n";
         continue;
       }
@@ -604,7 +803,6 @@ void printDelinearization(raw_ostream &O, Function *F, LoopInfo *LI,
       for (int i = 0; i < Size; i++)
         O << "[" << *Subscripts[i] << "]";
       O << "\n";
-    }
   }
 }
 
diff --git a/llvm/lib/Analysis/DemandedBits.cpp b/llvm/lib/Analysis/DemandedBits.cpp
index 6694d5cc06c8c..e0881751aef7e 100644
--- a/llvm/lib/Analysis/DemandedBits.cpp
+++ b/llvm/lib/Analysis/DemandedBits.cpp
@@ -76,6 +76,26 @@ void DemandedBits::determineLiveOperandBits(
           computeKnownBits(V2, Known2, DL, &AC, UserI, &DT);
         }
       };
+  auto GetShiftedRange = [&](uint64_t Min, uint64_t Max, bool ShiftLeft) {
+    auto ShiftF = [ShiftLeft](const APInt &Mask, unsigned ShiftAmnt) {
+      return ShiftLeft ? Mask.shl(ShiftAmnt) : Mask.lshr(ShiftAmnt);
+    };
+    AB = APInt::getZero(BitWidth);
+    uint64_t LoopRange = Max - Min;
+    APInt Mask = AOut;
+    APInt Shifted = AOut; // AOut | (AOut << 1) | ... | (AOut << (ShiftAmnt - 1)
+    for (unsigned ShiftAmnt = 1; ShiftAmnt <= LoopRange; ShiftAmnt <<= 1) {
+      if (LoopRange & ShiftAmnt) {
+        // Account for (LoopRange - ShiftAmnt, LoopRange]
+        Mask |= ShiftF(Shifted, LoopRange - ShiftAmnt + 1);
+        // Clears the low bit.
+        LoopRange -= ShiftAmnt;
+      }
+      // [0, ShiftAmnt) -> [0, ShiftAmnt * 2)
+      Shifted |= ShiftF(Shifted, ShiftAmnt);
+    }
+    AB = ShiftF(Mask, Min);
+  };
 
   switch (UserI->getOpcode()) {
   default: break;
@@ -183,6 +203,17 @@ void DemandedBits::determineLiveOperandBits(
           AB |= APInt::getHighBitsSet(BitWidth, ShiftAmt+1);
         else if (S->hasNoUnsignedWrap())
           AB |= APInt::getHighBitsSet(BitWidth, ShiftAmt);
+      } else {
+        ComputeKnownBits(BitWidth, UserI->getOperand(1), nullptr);
+        uint64_t Min = Known.getMinValue().getLimitedValue(BitWidth - 1);
+        uint64_t Max = Known.getMaxValue().getLimitedValue(BitWidth - 1);
+        // similar to Lshr case
+        GetShiftedRange(Min, Max, /*ShiftLeft=*/false);
+        const auto *S = cast<ShlOperator>(UserI);
+        if (S->hasNoSignedWrap())
+          AB |= APInt::getHighBitsSet(BitWidth, Max + 1);
+        else if (S->hasNoUnsignedWrap())
+          AB |= APInt::getHighBitsSet(BitWidth, Max);
       }
     }
     break;
@@ -197,6 +228,24 @@ void DemandedBits::determineLiveOperandBits(
         // (they must be zero).
         if (cast<LShrOperator>(UserI)->isExact())
           AB |= APInt::getLowBitsSet(BitWidth, ShiftAmt);
+      } else {
+        ComputeKnownBits(BitWidth, UserI->getOperand(1), nullptr);
+        uint64_t Min = Known.getMinValue().getLimitedValue(BitWidth - 1);
+        uint64_t Max = Known.getMaxValue().getLimitedValue(BitWidth - 1);
+        // Suppose AOut == 0b0000 0001
+        // [min, max] = [1, 3]
+        // iteration 1 shift by 1 mask is 0b0000 0011
+        // iteration 2 shift by 2 mask is 0b0000 1111
+        // iteration 3, shiftAmnt = 4 > max - min, we stop.
+        //
+        // After the iterations we need one more shift by min,
+        // to move from 0b0000 1111 to --> 0b0001 1110.
+        // The loop populates the mask relative to (0,...,max-min),
+        // but we need coverage from (min, max).
+        // This is why the shift by min is needed.
+        GetShiftedRange(Min, Max, /*ShiftLeft=*/true);
+        if (cast<LShrOperator>(UserI)->isExact())
+          AB |= APInt::getLowBitsSet(BitWidth, Max);
       }
     }
     break;
@@ -217,6 +266,26 @@ void DemandedBits::determineLiveOperandBits(
         // (they must be zero).
         if (cast<AShrOperator>(UserI)->isExact())
           AB |= APInt::getLowBitsSet(BitWidth, ShiftAmt);
+      } else {
+        ComputeKnownBits(BitWidth, UserI->getOperand(1), nullptr);
+        uint64_t Min = Known.getMinValue().getLimitedValue(BitWidth - 1);
+        uint64_t Max = Known.getMaxValue().getLimitedValue(BitWidth - 1);
+        GetShiftedRange(Min, Max, /*ShiftLeft=*/true);
+        if (Max &&
+            (AOut & APInt::getHighBitsSet(BitWidth, Max)).getBoolValue()) {
+          // Suppose AOut = 0011 1100
+          // [min, max] = [1, 3]
+          // ShiftAmount = 1 : Mask is 1000 0000
+          // ShiftAmount = 2 : Mask is 1100 0000
+          // ShiftAmount = 3 : Mask is 1110 0000
+          // The Mask with Max covers every case in [min, max],
+          // so we are done
+          AB.setSignBit();
+        }
+        // If the shift is exact, then the low bits are not dead
+        // (they must be zero).
+        if (cast<AShrOperator>(UserI)->isExact())
+          AB |= APInt::getLowBitsSet(BitWidth, Max);
       }
     }
     break;
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index f1473b2694ca4..f33e04e804e3d 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -180,8 +180,8 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA,
   for (inst_iterator SrcI = inst_begin(F), SrcE = inst_end(F); SrcI != SrcE;
        ++SrcI) {
     if (SrcI->mayReadOrWriteMemory()) {
-      for (inst_iterator DstI = SrcI, DstE = inst_end(F);
-           DstI != DstE; ++DstI) {
+      for (inst_iterator DstI = SrcI, DstE = inst_end(F); DstI != DstE;
+           ++DstI) {
         if (DstI->mayReadOrWriteMemory()) {
           OS << "Src:" << *SrcI << " --> Dst:" << *DstI << "\n";
           OS << "  da analyze - ";
@@ -203,7 +203,7 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA,
 
             // Normalize negative direction vectors if required by clients.
             if (NormalizeResults && D->normalize(&SE))
-                OS << "normalized - ";
+              OS << "normalized - ";
             D->dump(OS);
             for (unsigned Level = 1; Level <= D->getLevels(); Level++) {
               if (D->isSplitable(Level)) {
@@ -227,8 +227,8 @@ static void dumpExampleDependence(raw_ostream &OS, DependenceInfo *DA,
 
 void DependenceAnalysisWrapperPass::print(raw_ostream &OS,
                                           const Module *) const {
-  dumpExampleDependence(OS, info.get(),
-                        getAnalysis<ScalarEvolutionWrapperPass>().getSE(), false);
+  dumpExampleDependence(
+      OS, info.get(), getAnalysis<ScalarEvolutionWrapperPass>().getSE(), false);
 }
 
 PreservedAnalyses
@@ -249,33 +249,26 @@ bool Dependence::isInput() const {
   return Src->mayReadFromMemory() && Dst->mayReadFromMemory();
 }
 
-
 // Returns true if this is an output dependence.
 bool Dependence::isOutput() const {
   return Src->mayWriteToMemory() && Dst->mayWriteToMemory();
 }
 
-
 // Returns true if this is an flow (aka true)  dependence.
 bool Dependence::isFlow() const {
   return Src->mayWriteToMemory() && Dst->mayReadFromMemory();
 }
 
-
 // Returns true if this is an anti dependence.
 bool Dependence::isAnti() const {
   return Src->mayReadFromMemory() && Dst->mayWriteToMemory();
 }
 
-
 // Returns true if a particular level is scalar; that is,
 // if no subscript in the source or destination mention the induction
 // variable associated with the loop at this level.
 // Leave this out of line, so it will serve as a virtual method anchor
-bool Dependence::isScalar(unsigned level) const {
-  return false;
-}
-
+bool Dependence::isScalar(unsigned level) const { return false; }
 
 //===----------------------------------------------------------------------===//
 // FullDependence methods
@@ -338,8 +331,7 @@ bool FullDependence::normalize(ScalarEvolution *SE) {
     DV[Level - 1].Direction = RevDirection;
     // Reverse the dependence distance as well.
     if (DV[Level - 1].Distance != nullptr)
-      DV[Level - 1].Distance =
-          SE->getNegativeSCEV(DV[Level - 1].Distance);
+      DV[Level - 1].Distance = SE->getNegativeSCEV(DV[Level - 1].Distance);
   }
 
   LLVM_DEBUG(dbgs() << "After normalizing negative direction vectors:\n";
@@ -355,14 +347,12 @@ unsigned FullDependence::getDirection(unsigned Level) const {
   return DV[Level - 1].Direction;
 }
 
-
 // Returns the distance (or NULL) associated with a particular level.
 const SCEV *FullDependence::getDistance(unsigned Level) const {
   assert(0 < Level && Level <= Levels && "Level out of range");
   return DV[Level - 1].Distance;
 }
 
-
 // Returns true if a particular level is scalar; that is,
 // if no subscript in the source or destination mention the induction
 // variable associated with the loop at this level.
@@ -371,7 +361,6 @@ bool FullDependence::isScalar(unsigned Level) const {
   return DV[Level - 1].Scalar;
 }
 
-
 // Returns true if peeling the first iteration from this loop
 // will break this dependence.
 bool FullDependence::isPeelFirst(unsigned Level) const {
@@ -379,7 +368,6 @@ bool FullDependence::isPeelFirst(unsigned Level) const {
   return DV[Level - 1].PeelFirst;
 }
 
-
 // Returns true if peeling the last iteration from this loop
 // will break this dependence.
 bool FullDependence::isPeelLast(unsigned Level) const {
@@ -387,14 +375,12 @@ bool FullDependence::isPeelLast(unsigned Level) const {
   return DV[Level - 1].PeelLast;
 }
 
-
 // Returns true if splitting this loop will break the dependence.
 bool FullDependence::isSplitable(unsigned Level) const {
   assert(0 < Level && Level <= Levels && "Level out of range");
   return DV[Level - 1].Splitable;
 }
 
-
 //===----------------------------------------------------------------------===//
 // DependenceInfo::Constraint methods
 
@@ -405,7 +391,6 @@ const SCEV *DependenceInfo::Constraint::getX() const {
   return A;
 }
 
-
 // If constraint is a point <X, Y>, returns Y.
 // Otherwise assert.
 const SCEV *DependenceInfo::Constraint::getY() const {
@@ -413,7 +398,6 @@ const SCEV *DependenceInfo::Constraint::getY() const {
   return B;
 }
 
-
 // If constraint is a line AX + BY = C, returns A.
 // Otherwise assert.
 const SCEV *DependenceInfo::Constraint::getA() const {
@@ -422,7 +406,6 @@ const SCEV *DependenceInfo::Constraint::getA() const {
   return A;
 }
 
-
 // If constraint is a line AX + BY = C, returns B.
 // Otherwise assert.
 const SCEV *DependenceInfo::Constraint::getB() const {
@@ -431,7 +414,6 @@ const SCEV *DependenceInfo::Constraint::getB() const {
   return B;
 }
 
-
 // If constraint is a line AX + BY = C, returns C.
 // Otherwise assert.
 const SCEV *DependenceInfo::Constraint::getC() const {
@@ -440,7 +422,6 @@ const SCEV *DependenceInfo::Constraint::getC() const {
   return C;
 }
 
-
 // If constraint is a distance, returns D.
 // Otherwise assert.
 const SCEV *DependenceInfo::Constraint::getD() const {
@@ -448,7 +429,6 @@ const SCEV *DependenceInfo::Constraint::getD() const {
   return SE->getNegativeSCEV(C);
 }
 
-
 // Returns the loop associated with this constraint.
 const Loop *DependenceInfo::Constraint::getAssociatedLoop() const {
   assert((Kind == Distance || Kind == Line || Kind == Point) &&
@@ -499,17 +479,16 @@ LLVM_DUMP_METHOD void DependenceInfo::Constraint::dump(raw_ostream &OS) const {
   else if (isPoint())
     OS << " Point is <" << *getX() << ", " << *getY() << ">\n";
   else if (isDistance())
-    OS << " Distance is " << *getD() <<
-      " (" << *getA() << "*X + " << *getB() << "*Y = " << *getC() << ")\n";
+    OS << " Distance is " << *getD() << " (" << *getA() << "*X + " << *getB()
+       << "*Y = " << *getC() << ")\n";
   else if (isLine())
-    OS << " Line is " << *getA() << "*X + " <<
-      *getB() << "*Y = " << *getC() << "\n";
+    OS << " Line is " << *getA() << "*X + " << *getB() << "*Y = " << *getC()
+       << "\n";
   else
     llvm_unreachable("unknown constraint type in Constraint::dump");
 }
 #endif
 
-
 // Updates X with the intersection
 // of the Constraints X and Y. Returns true if X has changed.
 // Corresponds to Figure 4 from the paper
@@ -591,15 +570,14 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) {
       const SCEV *A1B2 = SE->getMulExpr(X->getA(), Y->getB());
       const SCEV *A2B1 = SE->getMulExpr(Y->getA(), X->getB());
       const SCEVConstant *C1A2_C2A1 =
-        dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1A2, C2A1));
+          dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1A2, C2A1));
       const SCEVConstant *C1B2_C2B1 =
-        dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1B2, C2B1));
+          dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1B2, C2B1));
       const SCEVConstant *A1B2_A2B1 =
-        dyn_cast<SCEVConstant>(SE->getMinusSCEV(A1B2, A2B1));
+          dyn_cast<SCEVConstant>(SE->getMinusSCEV(A1B2, A2B1));
       const SCEVConstant *A2B1_A1B2 =
-        dyn_cast<SCEVConstant>(SE->getMinusSCEV(A2B1, A1B2));
-      if (!C1B2_C2B1 || !C1A2_C2A1 ||
-          !A1B2_A2B1 || !A2B1_A1B2)
+          dyn_cast<SCEVConstant>(SE->getMinusSCEV(A2B1, A1B2));
+      if (!C1B2_C2B1 || !C1A2_C2A1 || !A1B2_A2B1 || !A2B1_A1B2)
         return false;
       APInt Xtop = C1B2_C2B1->getAPInt();
       APInt Xbot = A1B2_A2B1->getAPInt();
@@ -626,8 +604,8 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) {
         ++DeltaSuccesses;
         return true;
       }
-      if (const SCEVConstant *CUB =
-          collectConstantUpperBound(X->getAssociatedLoop(), Prod1->getType())) {
+      if (const SCEVConstant *CUB = collectConstantUpperBound(
+              X->getAssociatedLoop(), Prod1->getType())) {
         const APInt &UpperBound = CUB->getAPInt();
         LLVM_DEBUG(dbgs() << "\t\tupper bound = " << UpperBound << "\n");
         if (Xq.sgt(UpperBound) || Yq.sgt(UpperBound)) {
@@ -636,8 +614,7 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) {
           return true;
         }
       }
-      X->setPoint(SE->getConstant(Xq),
-                  SE->getConstant(Yq),
+      X->setPoint(SE->getConstant(Xq), SE->getConstant(Yq),
                   X->getAssociatedLoop());
       ++DeltaSuccesses;
       return true;
@@ -667,7 +644,6 @@ bool DependenceInfo::intersectConstraints(Constraint *X, const Constraint *Y) {
   return false;
 }
 
-
 //===----------------------------------------------------------------------===//
 // DependenceInfo methods
 
@@ -737,8 +713,7 @@ void Dependence::dump(raw_ostream &OS) const {
 // tbaa, non-overlapping regions etc), then it is known there is no dependecy.
 // Otherwise the underlying objects are checked to see if they point to
 // different identifiable objects.
-static AliasResult underlyingObjectsAlias(AAResults *AA,
-                                          const DataLayout &DL,
+static AliasResult underlyingObjectsAlias(AAResults *AA, const DataLayout &DL,
                                           const MemoryLocation &LocA,
                                           const MemoryLocation &LocB) {
   // Check the original locations (minus size) for noalias, which can happen for
@@ -773,8 +748,7 @@ static AliasResult underlyingObjectsAlias(AAResults *AA,
 
 // Returns true if the load or store can be analyzed. Atomic and volatile
 // operations have properties which this analysis does not understand.
-static
-bool isLoadOrStore(const Instruction *I) {
+static bool isLoadOrStore(const Instruction *I) {
   if (const LoadInst *LI = dyn_cast<LoadInst>(I))
     return LI->isUnordered();
   else if (const StoreInst *SI = dyn_cast<StoreInst>(I))
@@ -782,7 +756,6 @@ bool isLoadOrStore(const Instruction *I) {
   return false;
 }
 
-
 // Examines the loop nesting of the Src and Dst
 // instructions and establishes their shared loops. Sets the variables
 // CommonLevels, SrcLevels, and MaxLevels.
@@ -860,14 +833,12 @@ void DependenceInfo::establishNestingLevels(const Instruction *Src,
   MaxLevels -= CommonLevels;
 }
 
-
 // Given one of the loops containing the source, return
 // its level index in our numbering scheme.
 unsigned DependenceInfo::mapSrcLoop(const Loop *SrcLoop) const {
   return SrcLoop->getLoopDepth();
 }
 
-
 // Given one of the loops containing the destination,
 // return its level index in our numbering scheme.
 unsigned DependenceInfo::mapDstLoop(const Loop *DstLoop) const {
@@ -880,7 +851,6 @@ unsigned DependenceInfo::mapDstLoop(const Loop *DstLoop) const {
     return D;
 }
 
-
 // Returns true if Expression is loop invariant in LoopNest.
 bool DependenceInfo::isLoopInvariant(const SCEV *Expression,
                                      const Loop *LoopNest) const {
@@ -896,8 +866,6 @@ bool DependenceInfo::isLoopInvariant(const SCEV *Expression,
   return SE->isLoopInvariant(Expression, LoopNest->getOutermostLoop());
 }
 
-
-
 // Finds the set of loops from the LoopNest that
 // have a level <= CommonLevels and are referred to by the SCEV Expression.
 void DependenceInfo::collectCommonLoops(const SCEV *Expression,
@@ -924,9 +892,9 @@ void DependenceInfo::unifySubscriptType(ArrayRef<Subscript *> Pairs) {
     IntegerType *SrcTy = dyn_cast<IntegerType>(Src->getType());
     IntegerType *DstTy = dyn_cast<IntegerType>(Dst->getType());
     if (SrcTy == nullptr || DstTy == nullptr) {
-      assert(SrcTy == DstTy && "This function only unify integer types and "
-             "expect Src and Dst share the same type "
-             "otherwise.");
+      assert(SrcTy == DstTy &&
+             "This function only unify integer types and "
+             "expect Src and Dst share the same type otherwise.");
       continue;
     }
     if (SrcTy->getBitWidth() > widestWidthSeen) {
@@ -939,7 +907,6 @@ void DependenceInfo::unifySubscriptType(ArrayRef<Subscript *> Pairs) {
     }
   }
 
-
   assert(widestWidthSeen > 0);
 
   // Now extend each pair to the widest seen.
@@ -949,9 +916,9 @@ void DependenceInfo::unifySubscriptType(ArrayRef<Subscript *> Pairs) {
     IntegerType *SrcTy = dyn_cast<IntegerType>(Src->getType());
     IntegerType *DstTy = dyn_cast<IntegerType>(Dst->getType());
     if (SrcTy == nullptr || DstTy == nullptr) {
-      assert(SrcTy == DstTy && "This function only unify integer types and "
-             "expect Src and Dst share the same type "
-             "otherwise.");
+      assert(SrcTy == DstTy &&
+             "This function only unify integer types and "
+             "expect Src and Dst share the same type otherwise.");
       continue;
     }
     if (SrcTy->getBitWidth() < widestWidthSeen)
@@ -1028,7 +995,6 @@ bool DependenceInfo::checkDstSubscript(const SCEV *Dst, const Loop *LoopNest,
   return checkSubscript(Dst, LoopNest, Loops, false);
 }
 
-
 // Examines the subscript pair (the Src and Dst SCEVs)
 // and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear.
 // Collects the associated loops in a set.
@@ -1049,14 +1015,12 @@ DependenceInfo::classifyPair(const SCEV *Src, const Loop *SrcLoopNest,
     return Subscript::ZIV;
   if (N == 1)
     return Subscript::SIV;
-  if (N == 2 && (SrcLoops.count() == 0 ||
-                 DstLoops.count() == 0 ||
+  if (N == 2 && (SrcLoops.count() == 0 || DstLoops.count() == 0 ||
                  (SrcLoops.count() == 1 && DstLoops.count() == 1)))
     return Subscript::RDIV;
   return Subscript::MIV;
 }
 
-
 // A wrapper around SCEV::isKnownPredicate.
 // Looks for cases where we're interested in comparing for equality.
 // If both X and Y have been identically sign or zero extended,
@@ -1069,12 +1033,9 @@ DependenceInfo::classifyPair(const SCEV *Src, const Loop *SrcLoopNest,
 // involving symbolics.
 bool DependenceInfo::isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *X,
                                       const SCEV *Y) const {
-  if (Pred == CmpInst::ICMP_EQ ||
-      Pred == CmpInst::ICMP_NE) {
-    if ((isa<SCEVSignExtendExpr>(X) &&
-         isa<SCEVSignExtendExpr>(Y)) ||
-        (isa<SCEVZeroExtendExpr>(X) &&
-         isa<SCEVZeroExtendExpr>(Y))) {
+  if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) {
+    if ((isa<SCEVSignExtendExpr>(X) && isa<SCEVSignExtendExpr>(Y)) ||
+        (isa<SCEVZeroExtendExpr>(X) && isa<SCEVZeroExtendExpr>(Y))) {
       const SCEVIntegralCastExpr *CX = cast<SCEVIntegralCastExpr>(X);
       const SCEVIntegralCastExpr *CY = cast<SCEVIntegralCastExpr>(Y);
       const SCEV *Xop = CX->getOperand();
@@ -1111,7 +1072,10 @@ bool DependenceInfo::isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *X,
   }
 }
 
-/// Compare to see if S is less than Size, using isKnownNegative(S - max(Size, 1))
+/// Compare to see if S is less than Size, using
+///
+///    isKnownNegative(S - Size)
+///
 /// with some extra checking if S is an AddRec and we can prove less-than using
 /// the loop bounds.
 bool DependenceInfo::isKnownLessThan(const SCEV *S, const SCEV *Size) const {
@@ -1126,21 +1090,34 @@ bool DependenceInfo::isKnownLessThan(const SCEV *S, const SCEV *Size) const {
   Size = SE->getTruncateOrZeroExtend(Size, MaxType);
 
   // Special check for addrecs using BE taken count
-  const SCEV *Bound = SE->getMinusSCEV(S, Size);
-  if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Bound)) {
-    if (AddRec->isAffine()) {
+  if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(S))
+    if (AddRec->isAffine() && AddRec->hasNoSignedWrap()) {
       const SCEV *BECount = SE->getBackedgeTakenCount(AddRec->getLoop());
-      if (!isa<SCEVCouldNotCompute>(BECount)) {
-        const SCEV *Limit = AddRec->evaluateAtIteration(BECount, *SE);
-        if (SE->isKnownNegative(Limit))
-          return true;
-      }
+      const SCEV *Start = AddRec->getStart();
+      const SCEV *Step = AddRec->getStepRecurrence(*SE);
+      const SCEV *End = AddRec->evaluateAtIteration(BECount, *SE);
+      const SCEV *Diff0 = SE->getMinusSCEV(Start, Size);
+      const SCEV *Diff1 = SE->getMinusSCEV(End, Size);
+
+      // If the value of Step is non-negative and the AddRec is non-wrap, it
+      // reaches its maximum at the last iteration. So it's enouth to check
+      // whether End - Size is negative.
+      if (SE->isKnownNonNegative(Step) && SE->isKnownNegative(Diff1))
+        return true;
+
+      // If the value of Step is non-positive and the AddRec is non-wrap, the
+      // initial value is its maximum.
+      if (SE->isKnownNonPositive(Step) && SE->isKnownNegative(Diff0))
+        return true;
+
+      // Even if we don't know the sign of Step, either Start or End must be
+      // the maximum value of the AddRec since it is non-wrap.
+      if (SE->isKnownNegative(Diff0) && SE->isKnownNegative(Diff1))
+        return true;
     }
-  }
 
   // Check using normal isKnownNegative
-  const SCEV *LimitedBound =
-      SE->getMinusSCEV(S, SE->getSMaxExpr(Size, SE->getOne(Size->getType())));
+  const SCEV *LimitedBound = SE->getMinusSCEV(S, Size);
   return SE->isKnownNegative(LimitedBound);
 }
 
@@ -1178,7 +1155,6 @@ const SCEV *DependenceInfo::collectUpperBound(const Loop *L, Type *T) const {
   return nullptr;
 }
 
-
 // Calls collectUpperBound(), then attempts to cast it to SCEVConstant.
 // If the cast fails, returns NULL.
 const SCEVConstant *DependenceInfo::collectConstantUpperBound(const Loop *L,
@@ -1188,7 +1164,6 @@ const SCEVConstant *DependenceInfo::collectConstantUpperBound(const Loop *L,
   return nullptr;
 }
 
-
 // testZIV -
 // When we have a pair of subscripts of the form [c1] and [c2],
 // where c1 and c2 are both loop invariant, we attack it using
@@ -1218,7 +1193,6 @@ bool DependenceInfo::testZIV(const SCEV *Src, const SCEV *Dst,
   return false; // possibly dependent
 }
 
-
 // strongSIVtest -
 // From the paper, Practical Dependence Testing, Section 4.2.1
 //
@@ -1270,9 +1244,9 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
     LLVM_DEBUG(dbgs() << "\t    UpperBound = " << *UpperBound);
     LLVM_DEBUG(dbgs() << ", " << *UpperBound->getType() << "\n");
     const SCEV *AbsDelta =
-      SE->isKnownNonNegative(Delta) ? Delta : SE->getNegativeSCEV(Delta);
+        SE->isKnownNonNegative(Delta) ? Delta : SE->getNegativeSCEV(Delta);
     const SCEV *AbsCoeff =
-      SE->isKnownNonNegative(Coeff) ? Coeff : SE->getNegativeSCEV(Coeff);
+        SE->isKnownNonNegative(Coeff) ? Coeff : SE->getNegativeSCEV(Coeff);
     const SCEV *Product = SE->getMulExpr(UpperBound, AbsCoeff);
     if (isKnownPredicate(CmpInst::ICMP_SGT, AbsDelta, Product)) {
       // Distance greater than trip count - no dependence
@@ -1286,7 +1260,7 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
   if (isa<SCEVConstant>(Delta) && isa<SCEVConstant>(Coeff)) {
     APInt ConstDelta = cast<SCEVConstant>(Delta)->getAPInt();
     APInt ConstCoeff = cast<SCEVConstant>(Coeff)->getAPInt();
-    APInt Distance  = ConstDelta; // these need to be initialized
+    APInt Distance = ConstDelta; // these need to be initialized
     APInt Remainder = ConstDelta;
     APInt::sdivrem(ConstDelta, ConstCoeff, Distance, Remainder);
     LLVM_DEBUG(dbgs() << "\t    Distance = " << Distance << "\n");
@@ -1307,29 +1281,25 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
     else
       Result.DV[Level].Direction &= Dependence::DVEntry::EQ;
     ++StrongSIVsuccesses;
-  }
-  else if (Delta->isZero()) {
+  } else if (Delta->isZero()) {
     // since 0/X == 0
     Result.DV[Level].Distance = Delta;
     NewConstraint.setDistance(Delta, CurLoop);
     Result.DV[Level].Direction &= Dependence::DVEntry::EQ;
     ++StrongSIVsuccesses;
-  }
-  else {
+  } else {
     if (Coeff->isOne()) {
       LLVM_DEBUG(dbgs() << "\t    Distance = " << *Delta << "\n");
       Result.DV[Level].Distance = Delta; // since X/1 == X
       NewConstraint.setDistance(Delta, CurLoop);
-    }
-    else {
+    } else {
       Result.Consistent = false;
-      NewConstraint.setLine(Coeff,
-                            SE->getNegativeSCEV(Coeff),
+      NewConstraint.setLine(Coeff, SE->getNegativeSCEV(Coeff),
                             SE->getNegativeSCEV(Delta), CurLoop);
     }
 
     // maybe we can get a useful direction
-    bool DeltaMaybeZero     = !SE->isKnownNonZero(Delta);
+    bool DeltaMaybeZero = !SE->isKnownNonZero(Delta);
     bool DeltaMaybePositive = !SE->isKnownNonPositive(Delta);
     bool DeltaMaybeNegative = !SE->isKnownNonNegative(Delta);
     bool CoeffMaybePositive = !SE->isKnownNonPositive(Coeff);
@@ -1353,7 +1323,6 @@ bool DependenceInfo::strongSIVtest(const SCEV *Coeff, const SCEV *SrcConst,
   return false;
 }
 
-
 // weakCrossingSIVtest -
 // From the paper, Practical Dependence Testing, Section 4.2.2
 //
@@ -1447,8 +1416,8 @@ bool DependenceInfo::weakCrossingSIVtest(
   if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) {
     LLVM_DEBUG(dbgs() << "\t    UpperBound = " << *UpperBound << "\n");
     const SCEV *ConstantTwo = SE->getConstant(UpperBound->getType(), 2);
-    const SCEV *ML = SE->getMulExpr(SE->getMulExpr(ConstCoeff, UpperBound),
-                                    ConstantTwo);
+    const SCEV *ML =
+        SE->getMulExpr(SE->getMulExpr(ConstCoeff, UpperBound), ConstantTwo);
     LLVM_DEBUG(dbgs() << "\t    ML = " << *ML << "\n");
     if (isKnownPredicate(CmpInst::ICMP_SGT, Delta, ML)) {
       // Delta too big, no dependence
@@ -1498,7 +1467,6 @@ bool DependenceInfo::weakCrossingSIVtest(
   return false;
 }
 
-
 // Kirch's algorithm, from
 //
 //        Optimizing Supercompilers for Supercomputers
@@ -1519,9 +1487,11 @@ static bool findGCD(unsigned Bits, const APInt &AM, const APInt &BM,
   APInt R = G0;
   APInt::sdivrem(G0, G1, Q, R);
   while (R != 0) {
+    // clang-format off
     APInt A2 = A0 - Q*A1; A0 = A1; A1 = A2;
     APInt B2 = B0 - Q*B1; B0 = B1; B1 = B2;
     G0 = G1; G1 = R;
+    // clang-format on
     APInt::sdivrem(G0, G1, Q, R);
   }
   G = G1;
@@ -1543,8 +1513,7 @@ static APInt floorOfQuotient(const APInt &A, const APInt &B) {
   APInt::sdivrem(A, B, Q, R);
   if (R == 0)
     return Q;
-  if ((A.sgt(0) && B.sgt(0)) ||
-      (A.slt(0) && B.slt(0)))
+  if ((A.sgt(0) && B.sgt(0)) || (A.slt(0) && B.slt(0)))
     return Q;
   else
     return Q - 1;
@@ -1556,13 +1525,68 @@ static APInt ceilingOfQuotient(const APInt &A, const APInt &B) {
   APInt::sdivrem(A, B, Q, R);
   if (R == 0)
     return Q;
-  if ((A.sgt(0) && B.sgt(0)) ||
-      (A.slt(0) && B.slt(0)))
+  if ((A.sgt(0) && B.sgt(0)) || (A.slt(0) && B.slt(0)))
     return Q + 1;
   else
     return Q;
 }
 
+/// Given an affine expression of the form A*k + B, where k is an arbitrary
+/// integer, infer the possible range of k based on the known range of the
+/// affine expression. If we know A*k + B is non-negative, i.e.,
+///
+///   A*k + B >= 0
+///
+/// we can derive the following inequalities for k when A is positive:
+///
+///   k >= -B / A
+///
+/// Since k is an integer, it means k is greater than or equal to the
+/// ceil(-B / A).
+///
+/// If the upper bound of the affine expression \p UB is passed, the following
+/// inequality can be derived as well:
+///
+///   A*k + B <= UB
+///
+/// which leads to:
+///
+///   k <= (UB - B) / A
+///
+/// Again, as k is an integer, it means k is less than or equal to the
+/// floor((UB - B) / A).
+///
+/// The similar logic applies when A is negative, but the inequalities sign flip
+/// while working with them.
+///
+/// Preconditions: \p A is non-zero, and we know A*k + B is non-negative.
+static std::pair<std::optional<APInt>, std::optional<APInt>>
+inferDomainOfAffine(const APInt &A, const APInt &B,
+                    const std::optional<APInt> &UB) {
+  assert(A != 0 && "A must be non-zero");
+  std::optional<APInt> TL, TU;
+  if (A.sgt(0)) {
+    TL = ceilingOfQuotient(-B, A);
+    LLVM_DEBUG(dbgs() << "\t    Possible TL = " << *TL << "\n");
+    // New bound check - modification to Banerjee's e3 check
+    if (UB) {
+      // TODO?: Overflow check for UB - B
+      TU = floorOfQuotient(*UB - B, A);
+      LLVM_DEBUG(dbgs() << "\t    Possible TU = " << *TU << "\n");
+    }
+  } else {
+    TU = floorOfQuotient(-B, A);
+    LLVM_DEBUG(dbgs() << "\t    Possible TU = " << *TU << "\n");
+    // New bound check - modification to Banerjee's e3 check
+    if (UB) {
+      // TODO?: Overflow check for UB - B
+      TL = ceilingOfQuotient(*UB - B, A);
+      LLVM_DEBUG(dbgs() << "\t    Possible TL = " << *TL << "\n");
+    }
+  }
+  return std::make_pair(TL, TU);
+}
+
 // exactSIVtest -
 // When we have a pair of subscripts of the form [c1 + a1*i] and [c2 + a2*i],
 // where i is an induction variable, c1 and c2 are loop invariant, and a1
@@ -1622,14 +1646,12 @@ bool DependenceInfo::exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
   LLVM_DEBUG(dbgs() << "\t    X = " << X << ", Y = " << Y << "\n");
 
   // since SCEV construction normalizes, LM = 0
-  APInt UM(Bits, 1, true);
-  bool UMValid = false;
+  std::optional<APInt> UM;
   // UM is perhaps unavailable, let's check
   if (const SCEVConstant *CUB =
           collectConstantUpperBound(CurLoop, Delta->getType())) {
     UM = CUB->getAPInt();
-    LLVM_DEBUG(dbgs() << "\t    UM = " << UM << "\n");
-    UMValid = true;
+    LLVM_DEBUG(dbgs() << "\t    UM = " << *UM << "\n");
   }
 
   APInt TU(APInt::getSignedMaxValue(Bits));
@@ -1641,44 +1663,33 @@ bool DependenceInfo::exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
   LLVM_DEBUG(dbgs() << "\t    TX = " << TX << "\n");
   LLVM_DEBUG(dbgs() << "\t    TY = " << TY << "\n");
 
-  SmallVector<APInt, 2> TLVec, TUVec;
   APInt TB = BM.sdiv(G);
-  if (TB.sgt(0)) {
-    TLVec.push_back(ceilingOfQuotient(-TX, TB));
-    LLVM_DEBUG(dbgs() << "\t    Possible TL = " << TLVec.back() << "\n");
-    // New bound check - modification to Banerjee's e3 check
-    if (UMValid) {
-      TUVec.push_back(floorOfQuotient(UM - TX, TB));
-      LLVM_DEBUG(dbgs() << "\t    Possible TU = " << TUVec.back() << "\n");
-    }
-  } else {
-    TUVec.push_back(floorOfQuotient(-TX, TB));
-    LLVM_DEBUG(dbgs() << "\t    Possible TU = " << TUVec.back() << "\n");
-    // New bound check - modification to Banerjee's e3 check
-    if (UMValid) {
-      TLVec.push_back(ceilingOfQuotient(UM - TX, TB));
-      LLVM_DEBUG(dbgs() << "\t    Possible TL = " << TLVec.back() << "\n");
-    }
-  }
-
   APInt TA = AM.sdiv(G);
-  if (TA.sgt(0)) {
-    if (UMValid) {
-      TUVec.push_back(floorOfQuotient(UM - TY, TA));
-      LLVM_DEBUG(dbgs() << "\t    Possible TU = " << TUVec.back() << "\n");
-    }
-    // New bound check - modification to Banerjee's e3 check
-    TLVec.push_back(ceilingOfQuotient(-TY, TA));
-    LLVM_DEBUG(dbgs() << "\t    Possible TL = " << TLVec.back() << "\n");
-  } else {
-    if (UMValid) {
-      TLVec.push_back(ceilingOfQuotient(UM - TY, TA));
-      LLVM_DEBUG(dbgs() << "\t    Possible TL = " << TLVec.back() << "\n");
-    }
-    // New bound check - modification to Banerjee's e3 check
-    TUVec.push_back(floorOfQuotient(-TY, TA));
-    LLVM_DEBUG(dbgs() << "\t    Possible TU = " << TUVec.back() << "\n");
-  }
+
+  // At this point, we have the following equations:
+  //
+  //   TA*i0 - TB*i1 = TC
+  //
+  // Also, we know that the all pairs of (i0, i1) can be expressed as:
+  //
+  //   (TX + k*TB, TY + k*TA)
+  //
+  // where k is an arbitrary integer.
+  auto [TL0, TU0] = inferDomainOfAffine(TB, TX, UM);
+  auto [TL1, TU1] = inferDomainOfAffine(TA, TY, UM);
+
+  auto CreateVec = [](const std::optional<APInt> &V0,
+                      const std::optional<APInt> &V1) {
+    SmallVector<APInt, 2> Vec;
+    if (V0)
+      Vec.push_back(*V0);
+    if (V1)
+      Vec.push_back(*V1);
+    return Vec;
+  };
+
+  SmallVector<APInt, 2> TLVec = CreateVec(TL0, TL1);
+  SmallVector<APInt, 2> TUVec = CreateVec(TU0, TU1);
 
   LLVM_DEBUG(dbgs() << "\t    TA = " << TA << "\n");
   LLVM_DEBUG(dbgs() << "\t    TB = " << TB << "\n");
@@ -1733,17 +1744,14 @@ bool DependenceInfo::exactSIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
   return Result.DV[Level].Direction == Dependence::DVEntry::NONE;
 }
 
-
 // Return true if the divisor evenly divides the dividend.
-static
-bool isRemainderZero(const SCEVConstant *Dividend,
-                     const SCEVConstant *Divisor) {
+static bool isRemainderZero(const SCEVConstant *Dividend,
+                            const SCEVConstant *Divisor) {
   const APInt &ConstDividend = Dividend->getAPInt();
   const APInt &ConstDivisor = Divisor->getAPInt();
   return ConstDividend.srem(ConstDivisor) == 0;
 }
 
-
 // weakZeroSrcSIVtest -
 // From the paper, Practical Dependence Testing, Section 4.2.2
 //
@@ -1807,11 +1815,11 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff,
   const SCEVConstant *ConstCoeff = dyn_cast<SCEVConstant>(DstCoeff);
   if (!ConstCoeff)
     return false;
-  const SCEV *AbsCoeff =
-    SE->isKnownNegative(ConstCoeff) ?
-    SE->getNegativeSCEV(ConstCoeff) : ConstCoeff;
+  const SCEV *AbsCoeff = SE->isKnownNegative(ConstCoeff)
+                             ? SE->getNegativeSCEV(ConstCoeff)
+                             : ConstCoeff;
   const SCEV *NewDelta =
-    SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta;
+      SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta;
 
   // check that Delta/SrcCoeff < iteration count
   // really check NewDelta < count*AbsCoeff
@@ -1853,7 +1861,6 @@ bool DependenceInfo::weakZeroSrcSIVtest(const SCEV *DstCoeff,
   return false;
 }
 
-
 // weakZeroDstSIVtest -
 // From the paper, Practical Dependence Testing, Section 4.2.2
 //
@@ -1916,11 +1923,11 @@ bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff,
   const SCEVConstant *ConstCoeff = dyn_cast<SCEVConstant>(SrcCoeff);
   if (!ConstCoeff)
     return false;
-  const SCEV *AbsCoeff =
-    SE->isKnownNegative(ConstCoeff) ?
-    SE->getNegativeSCEV(ConstCoeff) : ConstCoeff;
+  const SCEV *AbsCoeff = SE->isKnownNegative(ConstCoeff)
+                             ? SE->getNegativeSCEV(ConstCoeff)
+                             : ConstCoeff;
   const SCEV *NewDelta =
-    SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta;
+      SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta;
 
   // check that Delta/SrcCoeff < iteration count
   // really check NewDelta < count*AbsCoeff
@@ -1962,7 +1969,6 @@ bool DependenceInfo::weakZeroDstSIVtest(const SCEV *SrcCoeff,
   return false;
 }
 
-
 // exactRDIVtest - Tests the RDIV subscript pair for dependence.
 // Things of the form [c1 + a*i] and [c2 + b*j],
 // where i and j are induction variable, c1 and c2 are loop invariant,
@@ -2004,24 +2010,20 @@ bool DependenceInfo::exactRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
   LLVM_DEBUG(dbgs() << "\t    X = " << X << ", Y = " << Y << "\n");
 
   // since SCEV construction seems to normalize, LM = 0
-  APInt SrcUM(Bits, 1, true);
-  bool SrcUMvalid = false;
+  std::optional<APInt> SrcUM;
   // SrcUM is perhaps unavailable, let's check
   if (const SCEVConstant *UpperBound =
           collectConstantUpperBound(SrcLoop, Delta->getType())) {
     SrcUM = UpperBound->getAPInt();
-    LLVM_DEBUG(dbgs() << "\t    SrcUM = " << SrcUM << "\n");
-    SrcUMvalid = true;
+    LLVM_DEBUG(dbgs() << "\t    SrcUM = " << *SrcUM << "\n");
   }
 
-  APInt DstUM(Bits, 1, true);
-  bool DstUMvalid = false;
+  std::optional<APInt> DstUM;
   // UM is perhaps unavailable, let's check
   if (const SCEVConstant *UpperBound =
           collectConstantUpperBound(DstLoop, Delta->getType())) {
     DstUM = UpperBound->getAPInt();
-    LLVM_DEBUG(dbgs() << "\t    DstUM = " << DstUM << "\n");
-    DstUMvalid = true;
+    LLVM_DEBUG(dbgs() << "\t    DstUM = " << *DstUM << "\n");
   }
 
   APInt TU(APInt::getSignedMaxValue(Bits));
@@ -2033,47 +2035,39 @@ bool DependenceInfo::exactRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
   LLVM_DEBUG(dbgs() << "\t    TX = " << TX << "\n");
   LLVM_DEBUG(dbgs() << "\t    TY = " << TY << "\n");
 
-  SmallVector<APInt, 2> TLVec, TUVec;
   APInt TB = BM.sdiv(G);
-  if (TB.sgt(0)) {
-    TLVec.push_back(ceilingOfQuotient(-TX, TB));
-    LLVM_DEBUG(dbgs() << "\t    Possible TL = " << TLVec.back() << "\n");
-    if (SrcUMvalid) {
-      TUVec.push_back(floorOfQuotient(SrcUM - TX, TB));
-      LLVM_DEBUG(dbgs() << "\t    Possible TU = " << TUVec.back() << "\n");
-    }
-  } else {
-    TUVec.push_back(floorOfQuotient(-TX, TB));
-    LLVM_DEBUG(dbgs() << "\t    Possible TU = " << TUVec.back() << "\n");
-    if (SrcUMvalid) {
-      TLVec.push_back(ceilingOfQuotient(SrcUM - TX, TB));
-      LLVM_DEBUG(dbgs() << "\t    Possible TL = " << TLVec.back() << "\n");
-    }
-  }
-
   APInt TA = AM.sdiv(G);
-  if (TA.sgt(0)) {
-    TLVec.push_back(ceilingOfQuotient(-TY, TA));
-    LLVM_DEBUG(dbgs() << "\t    Possible TL = " << TLVec.back() << "\n");
-    if (DstUMvalid) {
-      TUVec.push_back(floorOfQuotient(DstUM - TY, TA));
-      LLVM_DEBUG(dbgs() << "\t    Possible TU = " << TUVec.back() << "\n");
-    }
-  } else {
-    TUVec.push_back(floorOfQuotient(-TY, TA));
-    LLVM_DEBUG(dbgs() << "\t    Possible TU = " << TUVec.back() << "\n");
-    if (DstUMvalid) {
-      TLVec.push_back(ceilingOfQuotient(DstUM - TY, TA));
-      LLVM_DEBUG(dbgs() << "\t    Possible TL = " << TLVec.back() << "\n");
-    }
-  }
 
-  if (TLVec.empty() || TUVec.empty())
-    return false;
+  // At this point, we have the following equations:
+  //
+  //   TA*i - TB*j = TC
+  //
+  // Also, we know that the all pairs of (i, j) can be expressed as:
+  //
+  //   (TX + k*TB, TY + k*TA)
+  //
+  // where k is an arbitrary integer.
+  auto [TL0, TU0] = inferDomainOfAffine(TB, TX, SrcUM);
+  auto [TL1, TU1] = inferDomainOfAffine(TA, TY, DstUM);
 
   LLVM_DEBUG(dbgs() << "\t    TA = " << TA << "\n");
   LLVM_DEBUG(dbgs() << "\t    TB = " << TB << "\n");
 
+  auto CreateVec = [](const std::optional<APInt> &V0,
+                      const std::optional<APInt> &V1) {
+    SmallVector<APInt, 2> Vec;
+    if (V0)
+      Vec.push_back(*V0);
+    if (V1)
+      Vec.push_back(*V1);
+    return Vec;
+  };
+
+  SmallVector<APInt, 2> TLVec = CreateVec(TL0, TL1);
+  SmallVector<APInt, 2> TUVec = CreateVec(TU0, TU1);
+  if (TLVec.empty() || TUVec.empty())
+    return false;
+
   TL = APIntOps::smax(TLVec.front(), TLVec.back());
   TU = APIntOps::smin(TUVec.front(), TUVec.back());
   LLVM_DEBUG(dbgs() << "\t    TL = " << TL << "\n");
@@ -2084,7 +2078,6 @@ bool DependenceInfo::exactRDIVtest(const SCEV *SrcCoeff, const SCEV *DstCoeff,
   return TL.sgt(TU);
 }
 
-
 // symbolicRDIVtest -
 // In Section 4.5 of the Practical Dependence Testing paper,the authors
 // introduce a special case of Banerjee's Inequalities (also called the
@@ -2167,8 +2160,7 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2,
           return true;
         }
       }
-    }
-    else if (SE->isKnownNonPositive(A2)) {
+    } else if (SE->isKnownNonPositive(A2)) {
       // a1 >= 0 && a2 <= 0
       if (N1 && N2) {
         // make sure that c2 - c1 <= a1*N1 - a2*N2
@@ -2187,8 +2179,7 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2,
         return true;
       }
     }
-  }
-  else if (SE->isKnownNonPositive(A1)) {
+  } else if (SE->isKnownNonPositive(A1)) {
     if (SE->isKnownNonNegative(A2)) {
       // a1 <= 0 && a2 >= 0
       if (N1 && N2) {
@@ -2207,8 +2198,7 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2,
         ++SymbolicRDIVindependence;
         return true;
       }
-    }
-    else if (SE->isKnownNonPositive(A2)) {
+    } else if (SE->isKnownNonPositive(A2)) {
       // a1 <= 0 && a2 <= 0
       if (N1) {
         // make sure that a1*N1 <= c2 - c1
@@ -2233,7 +2223,6 @@ bool DependenceInfo::symbolicRDIVtest(const SCEV *A1, const SCEV *A2,
   return false;
 }
 
-
 // testSIV -
 // When we have a pair of subscripts of the form [c1 + a1*i] and [c2 - a2*i]
 // where i is an induction variable, c1 and c2 are loop invariant, and a1 and
@@ -2260,17 +2249,17 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level,
     Level = mapSrcLoop(CurLoop);
     bool disproven;
     if (SrcCoeff == DstCoeff)
-      disproven = strongSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop,
-                                Level, Result, NewConstraint);
+      disproven = strongSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, Level,
+                                Result, NewConstraint);
     else if (SrcCoeff == SE->getNegativeSCEV(DstCoeff))
       disproven = weakCrossingSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop,
                                       Level, Result, NewConstraint, SplitIter);
     else
       disproven = exactSIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop,
                                Level, Result, NewConstraint);
-    return disproven ||
-      gcdMIVtest(Src, Dst, Result) ||
-      symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop, CurLoop);
+    return disproven || gcdMIVtest(Src, Dst, Result) ||
+           symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop,
+                            CurLoop);
   }
   if (SrcAddRec) {
     const SCEV *SrcConst = SrcAddRec->getStart();
@@ -2278,9 +2267,9 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level,
     const SCEV *DstConst = Dst;
     const Loop *CurLoop = SrcAddRec->getLoop();
     Level = mapSrcLoop(CurLoop);
-    return weakZeroDstSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop,
-                              Level, Result, NewConstraint) ||
-      gcdMIVtest(Src, Dst, Result);
+    return weakZeroDstSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop, Level,
+                              Result, NewConstraint) ||
+           gcdMIVtest(Src, Dst, Result);
   }
   if (DstAddRec) {
     const SCEV *DstConst = DstAddRec->getStart();
@@ -2288,15 +2277,14 @@ bool DependenceInfo::testSIV(const SCEV *Src, const SCEV *Dst, unsigned &Level,
     const SCEV *SrcConst = Src;
     const Loop *CurLoop = DstAddRec->getLoop();
     Level = mapDstLoop(CurLoop);
-    return weakZeroSrcSIVtest(DstCoeff, SrcConst, DstConst,
-                              CurLoop, Level, Result, NewConstraint) ||
-      gcdMIVtest(Src, Dst, Result);
+    return weakZeroSrcSIVtest(DstCoeff, SrcConst, DstConst, CurLoop, Level,
+                              Result, NewConstraint) ||
+           gcdMIVtest(Src, Dst, Result);
   }
   llvm_unreachable("SIV test expected at least one AddRec");
   return false;
 }
 
-
 // testRDIV -
 // When we have a pair of subscripts of the form [c1 + a1*i] and [c2 + a2*j]
 // where i and j are induction variables, c1 and c2 are loop invariant,
@@ -2333,46 +2321,37 @@ bool DependenceInfo::testRDIV(const SCEV *Src, const SCEV *Dst,
     DstConst = DstAddRec->getStart();
     DstCoeff = DstAddRec->getStepRecurrence(*SE);
     DstLoop = DstAddRec->getLoop();
-  }
-  else if (SrcAddRec) {
+  } else if (SrcAddRec) {
     if (const SCEVAddRecExpr *tmpAddRec =
-        dyn_cast<SCEVAddRecExpr>(SrcAddRec->getStart())) {
+            dyn_cast<SCEVAddRecExpr>(SrcAddRec->getStart())) {
       SrcConst = tmpAddRec->getStart();
       SrcCoeff = tmpAddRec->getStepRecurrence(*SE);
       SrcLoop = tmpAddRec->getLoop();
       DstConst = Dst;
       DstCoeff = SE->getNegativeSCEV(SrcAddRec->getStepRecurrence(*SE));
       DstLoop = SrcAddRec->getLoop();
-    }
-    else
+    } else
       llvm_unreachable("RDIV reached by surprising SCEVs");
-  }
-  else if (DstAddRec) {
+  } else if (DstAddRec) {
     if (const SCEVAddRecExpr *tmpAddRec =
-        dyn_cast<SCEVAddRecExpr>(DstAddRec->getStart())) {
+            dyn_cast<SCEVAddRecExpr>(DstAddRec->getStart())) {
       DstConst = tmpAddRec->getStart();
       DstCoeff = tmpAddRec->getStepRecurrence(*SE);
       DstLoop = tmpAddRec->getLoop();
       SrcConst = Src;
       SrcCoeff = SE->getNegativeSCEV(DstAddRec->getStepRecurrence(*SE));
       SrcLoop = DstAddRec->getLoop();
-    }
-    else
+    } else
       llvm_unreachable("RDIV reached by surprising SCEVs");
-  }
-  else
+  } else
     llvm_unreachable("RDIV expected at least one AddRec");
-  return exactRDIVtest(SrcCoeff, DstCoeff,
-                       SrcConst, DstConst,
-                       SrcLoop, DstLoop,
+  return exactRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, SrcLoop, DstLoop,
                        Result) ||
-    gcdMIVtest(Src, Dst, Result) ||
-    symbolicRDIVtest(SrcCoeff, DstCoeff,
-                     SrcConst, DstConst,
-                     SrcLoop, DstLoop);
+         gcdMIVtest(Src, Dst, Result) ||
+         symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, SrcLoop,
+                          DstLoop);
 }
 
-
 // Tests the single-subscript MIV pair (Src and Dst) for dependence.
 // Return true if dependence disproved.
 // Can sometimes refine direction vectors.
@@ -2383,7 +2362,7 @@ bool DependenceInfo::testMIV(const SCEV *Src, const SCEV *Dst,
   LLVM_DEBUG(dbgs() << "    dst = " << *Dst << "\n");
   Result.Consistent = false;
   return gcdMIVtest(Src, Dst, Result) ||
-    banerjeeMIVtest(Src, Dst, Loops, Result);
+         banerjeeMIVtest(Src, Dst, Loops, Result);
 }
 
 // Given a product, e.g., 10*X*Y, returns the first constant operand,
@@ -2397,6 +2376,43 @@ static std::optional<APInt> getConstantPart(const SCEV *Expr) {
   return std::nullopt;
 }
 
+bool DependenceInfo::accumulateCoefficientsGCD(const SCEV *Expr,
+                                               const Loop *CurLoop,
+                                               const SCEV *&CurLoopCoeff,
+                                               APInt &RunningGCD) const {
+  // If RunningGCD is already 1, exit early.
+  // TODO: It might be better to continue the recursion to find CurLoopCoeff.
+  if (RunningGCD == 1)
+    return true;
+
+  const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Expr);
+  if (!AddRec) {
+    assert(isLoopInvariant(Expr, CurLoop) &&
+           "Expected loop invariant expression");
+    return true;
+  }
+
+  assert(AddRec->isAffine() && "Unexpected Expr");
+  const SCEV *Start = AddRec->getStart();
+  const SCEV *Step = AddRec->getStepRecurrence(*SE);
+  if (AddRec->getLoop() == CurLoop) {
+    CurLoopCoeff = Step;
+  } else {
+    std::optional<APInt> ConstCoeff = getConstantPart(Step);
+
+    // If the coefficient is the product of a constant and other stuff, we can
+    // use the constant in the GCD computation.
+    if (!ConstCoeff)
+      return false;
+
+    // TODO: What happens if ConstCoeff is the "most negative" signed number
+    // (e.g. -128 for 8 bit wide APInt)?
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
+  }
+
+  return accumulateCoefficientsGCD(Start, CurLoop, CurLoopCoeff, RunningGCD);
+}
+
 //===----------------------------------------------------------------------===//
 // gcdMIVtest -
 // Tests an MIV subscript pair for dependence.
@@ -2428,7 +2444,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
   // we can't quit the loop just because the GCD == 1.
   const SCEV *Coefficients = Src;
   while (const SCEVAddRecExpr *AddRec =
-         dyn_cast<SCEVAddRecExpr>(Coefficients)) {
+             dyn_cast<SCEVAddRecExpr>(Coefficients)) {
     const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
     // If the coefficient is the product of a constant and other stuff,
     // we can use the constant in the GCD computation.
@@ -2446,7 +2462,7 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
   // we can't quit the loop just because the GCD == 1.
   Coefficients = Dst;
   while (const SCEVAddRecExpr *AddRec =
-         dyn_cast<SCEVAddRecExpr>(Coefficients)) {
+             dyn_cast<SCEVAddRecExpr>(Coefficients)) {
     const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
     // If the coefficient is the product of a constant and other stuff,
     // we can use the constant in the GCD computation.
@@ -2468,16 +2484,14 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
       if (isa<SCEVConstant>(Operand)) {
         assert(!Constant && "Surprised to find multiple constants");
         Constant = cast<SCEVConstant>(Operand);
-      }
-      else if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Operand)) {
+      } else if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Operand)) {
         // Search for constant operand to participate in GCD;
         // If none found; return false.
         std::optional<APInt> ConstOp = getConstantPart(Product);
         if (!ConstOp)
           return false;
         ExtraGCD = APIntOps::GreatestCommonDivisor(ExtraGCD, ConstOp->abs());
-      }
-      else
+      } else
         return false;
     }
   }
@@ -2512,46 +2526,17 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
   bool Improved = false;
   Coefficients = Src;
   while (const SCEVAddRecExpr *AddRec =
-         dyn_cast<SCEVAddRecExpr>(Coefficients)) {
+             dyn_cast<SCEVAddRecExpr>(Coefficients)) {
     Coefficients = AddRec->getStart();
     const Loop *CurLoop = AddRec->getLoop();
     RunningGCD = ExtraGCD;
     const SCEV *SrcCoeff = AddRec->getStepRecurrence(*SE);
     const SCEV *DstCoeff = SE->getMinusSCEV(SrcCoeff, SrcCoeff);
-    const SCEV *Inner = Src;
-    while (RunningGCD != 1 && isa<SCEVAddRecExpr>(Inner)) {
-      AddRec = cast<SCEVAddRecExpr>(Inner);
-      const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
-      if (CurLoop == AddRec->getLoop())
-        ; // SrcCoeff == Coeff
-      else {
-        // If the coefficient is the product of a constant and other stuff,
-        // we can use the constant in the GCD computation.
-        std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
-        if (!ConstCoeff)
-          return false;
-        RunningGCD =
-            APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
-      }
-      Inner = AddRec->getStart();
-    }
-    Inner = Dst;
-    while (RunningGCD != 1 && isa<SCEVAddRecExpr>(Inner)) {
-      AddRec = cast<SCEVAddRecExpr>(Inner);
-      const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
-      if (CurLoop == AddRec->getLoop())
-        DstCoeff = Coeff;
-      else {
-        // If the coefficient is the product of a constant and other stuff,
-        // we can use the constant in the GCD computation.
-        std::optional<APInt> ConstCoeff = getConstantPart(Coeff);
-        if (!ConstCoeff)
-          return false;
-        RunningGCD =
-            APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff->abs());
-      }
-      Inner = AddRec->getStart();
-    }
+
+    if (!accumulateCoefficientsGCD(Src, CurLoop, SrcCoeff, RunningGCD) ||
+        !accumulateCoefficientsGCD(Dst, CurLoop, DstCoeff, RunningGCD))
+      return false;
+
     Delta = SE->getMinusSCEV(SrcCoeff, DstCoeff);
     // If the coefficient is the product of a constant and other stuff,
     // we can use the constant in the GCD computation.
@@ -2578,7 +2563,6 @@ bool DependenceInfo::gcdMIVtest(const SCEV *Src, const SCEV *Dst,
   return false;
 }
 
-
 //===----------------------------------------------------------------------===//
 // banerjeeMIVtest -
 // Use Banerjee's Inequalities to test an MIV subscript pair.
@@ -2652,8 +2636,8 @@ bool DependenceInfo::banerjeeMIVtest(const SCEV *Src, const SCEV *Dst,
   if (testBounds(Dependence::DVEntry::ALL, 0, Bound, Delta)) {
     // Explore the direction vector hierarchy.
     unsigned DepthExpanded = 0;
-    unsigned NewDeps = exploreDirections(1, A, B, Bound,
-                                         Loops, DepthExpanded, Delta);
+    unsigned NewDeps =
+        exploreDirections(1, A, B, Bound, Loops, DepthExpanded, Delta);
     if (NewDeps > 0) {
       bool Improved = false;
       for (unsigned K = 1; K <= CommonLevels; ++K) {
@@ -2670,23 +2654,20 @@ bool DependenceInfo::banerjeeMIVtest(const SCEV *Src, const SCEV *Dst,
       }
       if (Improved)
         ++BanerjeeSuccesses;
-    }
-    else {
+    } else {
       ++BanerjeeIndependence;
       Disproved = true;
     }
-  }
-  else {
+  } else {
     ++BanerjeeIndependence;
     Disproved = true;
   }
-  delete [] Bound;
-  delete [] A;
-  delete [] B;
+  delete[] Bound;
+  delete[] A;
+  delete[] B;
   return Disproved;
 }
 
-
 // Hierarchically expands the direction vector
 // search space, combining the directions of discovered dependences
 // in the DirSet field of Bound. Returns the number of distinct
@@ -2788,27 +2769,26 @@ unsigned DependenceInfo::exploreDirections(unsigned Level, CoefficientInfo *A,
 
     // test bounds for <, *, *, ...
     if (testBounds(Dependence::DVEntry::LT, Level, Bound, Delta))
-      NewDeps += exploreDirections(Level + 1, A, B, Bound,
-                                   Loops, DepthExpanded, Delta);
+      NewDeps += exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded,
+                                   Delta);
 
     // Test bounds for =, *, *, ...
     if (testBounds(Dependence::DVEntry::EQ, Level, Bound, Delta))
-      NewDeps += exploreDirections(Level + 1, A, B, Bound,
-                                   Loops, DepthExpanded, Delta);
+      NewDeps += exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded,
+                                   Delta);
 
     // test bounds for >, *, *, ...
     if (testBounds(Dependence::DVEntry::GT, Level, Bound, Delta))
-      NewDeps += exploreDirections(Level + 1, A, B, Bound,
-                                   Loops, DepthExpanded, Delta);
+      NewDeps += exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded,
+                                   Delta);
 
     Bound[Level].Direction = Dependence::DVEntry::ALL;
     return NewDeps;
-  }
-  else
-    return exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, Delta);
+  } else
+    return exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded,
+                             Delta);
 }
 
-
 // Returns true iff the current bounds are plausible.
 bool DependenceInfo::testBounds(unsigned char DirKind, unsigned Level,
                                 BoundInfo *Bound, const SCEV *Delta) const {
@@ -2822,7 +2802,6 @@ bool DependenceInfo::testBounds(unsigned char DirKind, unsigned Level,
   return true;
 }
 
-
 // Computes the upper and lower bounds for level K
 // using the * direction. Records them in Bound.
 // Wolfe gives the equations
@@ -2840,17 +2819,16 @@ bool DependenceInfo::testBounds(unsigned char DirKind, unsigned Level,
 // and the upper bound is always >= 0.
 void DependenceInfo::findBoundsALL(CoefficientInfo *A, CoefficientInfo *B,
                                    BoundInfo *Bound, unsigned K) const {
-  Bound[K].Lower[Dependence::DVEntry::ALL] = nullptr; // Default value = -infinity.
-  Bound[K].Upper[Dependence::DVEntry::ALL] = nullptr; // Default value = +infinity.
+  Bound[K].Lower[Dependence::DVEntry::ALL] =
+      nullptr; // Default value = -infinity.
+  Bound[K].Upper[Dependence::DVEntry::ALL] =
+      nullptr; // Default value = +infinity.
   if (Bound[K].Iterations) {
-    Bound[K].Lower[Dependence::DVEntry::ALL] =
-      SE->getMulExpr(SE->getMinusSCEV(A[K].NegPart, B[K].PosPart),
-                     Bound[K].Iterations);
-    Bound[K].Upper[Dependence::DVEntry::ALL] =
-      SE->getMulExpr(SE->getMinusSCEV(A[K].PosPart, B[K].NegPart),
-                     Bound[K].Iterations);
-  }
-  else {
+    Bound[K].Lower[Dependence::DVEntry::ALL] = SE->getMulExpr(
+        SE->getMinusSCEV(A[K].NegPart, B[K].PosPart), Bound[K].Iterations);
+    Bound[K].Upper[Dependence::DVEntry::ALL] = SE->getMulExpr(
+        SE->getMinusSCEV(A[K].PosPart, B[K].NegPart), Bound[K].Iterations);
+  } else {
     // If the difference is 0, we won't need to know the number of iterations.
     if (isKnownPredicate(CmpInst::ICMP_EQ, A[K].NegPart, B[K].PosPart))
       Bound[K].Lower[Dependence::DVEntry::ALL] =
@@ -2861,7 +2839,6 @@ void DependenceInfo::findBoundsALL(CoefficientInfo *A, CoefficientInfo *B,
   }
 }
 
-
 // Computes the upper and lower bounds for level K
 // using the = direction. Records them in Bound.
 // Wolfe gives the equations
@@ -2879,18 +2856,19 @@ void DependenceInfo::findBoundsALL(CoefficientInfo *A, CoefficientInfo *B,
 // and the upper bound is always >= 0.
 void DependenceInfo::findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B,
                                   BoundInfo *Bound, unsigned K) const {
-  Bound[K].Lower[Dependence::DVEntry::EQ] = nullptr; // Default value = -infinity.
-  Bound[K].Upper[Dependence::DVEntry::EQ] = nullptr; // Default value = +infinity.
+  Bound[K].Lower[Dependence::DVEntry::EQ] =
+      nullptr; // Default value = -infinity.
+  Bound[K].Upper[Dependence::DVEntry::EQ] =
+      nullptr; // Default value = +infinity.
   if (Bound[K].Iterations) {
     const SCEV *Delta = SE->getMinusSCEV(A[K].Coeff, B[K].Coeff);
     const SCEV *NegativePart = getNegativePart(Delta);
     Bound[K].Lower[Dependence::DVEntry::EQ] =
-      SE->getMulExpr(NegativePart, Bound[K].Iterations);
+        SE->getMulExpr(NegativePart, Bound[K].Iterations);
     const SCEV *PositivePart = getPositivePart(Delta);
     Bound[K].Upper[Dependence::DVEntry::EQ] =
-      SE->getMulExpr(PositivePart, Bound[K].Iterations);
-  }
-  else {
+        SE->getMulExpr(PositivePart, Bound[K].Iterations);
+  } else {
     // If the positive/negative part of the difference is 0,
     // we won't need to know the number of iterations.
     const SCEV *Delta = SE->getMinusSCEV(A[K].Coeff, B[K].Coeff);
@@ -2903,7 +2881,6 @@ void DependenceInfo::findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B,
   }
 }
 
-
 // Computes the upper and lower bounds for level K
 // using the < direction. Records them in Bound.
 // Wolfe gives the equations
@@ -2919,35 +2896,35 @@ void DependenceInfo::findBoundsEQ(CoefficientInfo *A, CoefficientInfo *B,
 // We must be careful to handle the case where the upper bound is unknown.
 void DependenceInfo::findBoundsLT(CoefficientInfo *A, CoefficientInfo *B,
                                   BoundInfo *Bound, unsigned K) const {
-  Bound[K].Lower[Dependence::DVEntry::LT] = nullptr; // Default value = -infinity.
-  Bound[K].Upper[Dependence::DVEntry::LT] = nullptr; // Default value = +infinity.
+  Bound[K].Lower[Dependence::DVEntry::LT] =
+      nullptr; // Default value = -infinity.
+  Bound[K].Upper[Dependence::DVEntry::LT] =
+      nullptr; // Default value = +infinity.
   if (Bound[K].Iterations) {
     const SCEV *Iter_1 = SE->getMinusSCEV(
         Bound[K].Iterations, SE->getOne(Bound[K].Iterations->getType()));
     const SCEV *NegPart =
-      getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff));
+        getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff));
     Bound[K].Lower[Dependence::DVEntry::LT] =
-      SE->getMinusSCEV(SE->getMulExpr(NegPart, Iter_1), B[K].Coeff);
+        SE->getMinusSCEV(SE->getMulExpr(NegPart, Iter_1), B[K].Coeff);
     const SCEV *PosPart =
-      getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff));
+        getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff));
     Bound[K].Upper[Dependence::DVEntry::LT] =
-      SE->getMinusSCEV(SE->getMulExpr(PosPart, Iter_1), B[K].Coeff);
-  }
-  else {
+        SE->getMinusSCEV(SE->getMulExpr(PosPart, Iter_1), B[K].Coeff);
+  } else {
     // If the positive/negative part of the difference is 0,
     // we won't need to know the number of iterations.
     const SCEV *NegPart =
-      getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff));
+        getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff));
     if (NegPart->isZero())
       Bound[K].Lower[Dependence::DVEntry::LT] = SE->getNegativeSCEV(B[K].Coeff);
     const SCEV *PosPart =
-      getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff));
+        getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff));
     if (PosPart->isZero())
       Bound[K].Upper[Dependence::DVEntry::LT] = SE->getNegativeSCEV(B[K].Coeff);
   }
 }
 
-
 // Computes the upper and lower bounds for level K
 // using the > direction. Records them in Bound.
 // Wolfe gives the equations
@@ -2963,45 +2940,45 @@ void DependenceInfo::findBoundsLT(CoefficientInfo *A, CoefficientInfo *B,
 // We must be careful to handle the case where the upper bound is unknown.
 void DependenceInfo::findBoundsGT(CoefficientInfo *A, CoefficientInfo *B,
                                   BoundInfo *Bound, unsigned K) const {
-  Bound[K].Lower[Dependence::DVEntry::GT] = nullptr; // Default value = -infinity.
-  Bound[K].Upper[Dependence::DVEntry::GT] = nullptr; // Default value = +infinity.
+  Bound[K].Lower[Dependence::DVEntry::GT] =
+      nullptr; // Default value = -infinity.
+  Bound[K].Upper[Dependence::DVEntry::GT] =
+      nullptr; // Default value = +infinity.
   if (Bound[K].Iterations) {
     const SCEV *Iter_1 = SE->getMinusSCEV(
         Bound[K].Iterations, SE->getOne(Bound[K].Iterations->getType()));
     const SCEV *NegPart =
-      getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart));
+        getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart));
     Bound[K].Lower[Dependence::DVEntry::GT] =
-      SE->getAddExpr(SE->getMulExpr(NegPart, Iter_1), A[K].Coeff);
+        SE->getAddExpr(SE->getMulExpr(NegPart, Iter_1), A[K].Coeff);
     const SCEV *PosPart =
-      getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart));
+        getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart));
     Bound[K].Upper[Dependence::DVEntry::GT] =
-      SE->getAddExpr(SE->getMulExpr(PosPart, Iter_1), A[K].Coeff);
-  }
-  else {
+        SE->getAddExpr(SE->getMulExpr(PosPart, Iter_1), A[K].Coeff);
+  } else {
     // If the positive/negative part of the difference is 0,
     // we won't need to know the number of iterations.
-    const SCEV *NegPart = getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart));
+    const SCEV *NegPart =
+        getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart));
     if (NegPart->isZero())
       Bound[K].Lower[Dependence::DVEntry::GT] = A[K].Coeff;
-    const SCEV *PosPart = getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart));
+    const SCEV *PosPart =
+        getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart));
     if (PosPart->isZero())
       Bound[K].Upper[Dependence::DVEntry::GT] = A[K].Coeff;
   }
 }
 
-
 // X^+ = max(X, 0)
 const SCEV *DependenceInfo::getPositivePart(const SCEV *X) const {
   return SE->getSMaxExpr(X, SE->getZero(X->getType()));
 }
 
-
 // X^- = min(X, 0)
 const SCEV *DependenceInfo::getNegativePart(const SCEV *X) const {
   return SE->getSMinExpr(X, SE->getZero(X->getType()));
 }
 
-
 // Walks through the subscript,
 // collecting each coefficient, the associated loop bounds,
 // and recording its positive and negative parts for later use.
@@ -3046,7 +3023,6 @@ DependenceInfo::collectCoeffInfo(const SCEV *Subscript, bool SrcFlag,
   return CI;
 }
 
-
 // Looks through all the bounds info and
 // computes the lower bound given the current direction settings
 // at each level. If the lower bound for any level is -inf,
@@ -3062,7 +3038,6 @@ const SCEV *DependenceInfo::getLowerBound(BoundInfo *Bound) const {
   return Sum;
 }
 
-
 // Looks through all the bounds info and
 // computes the upper bound given the current direction settings
 // at each level. If the upper bound at any level is +inf,
@@ -3078,7 +3053,6 @@ const SCEV *DependenceInfo::getUpperBound(BoundInfo *Bound) const {
   return Sum;
 }
 
-
 //===----------------------------------------------------------------------===//
 // Constraint manipulation for Delta test.
 
@@ -3098,7 +3072,6 @@ const SCEV *DependenceInfo::findCoefficient(const SCEV *Expr,
   return findCoefficient(AddRec->getStart(), TargetLoop);
 }
 
-
 // Given a linear SCEV,
 // return the SCEV given by zeroing out the coefficient
 // corresponding to the specified loop.
@@ -3112,12 +3085,10 @@ const SCEV *DependenceInfo::zeroCoefficient(const SCEV *Expr,
   if (AddRec->getLoop() == TargetLoop)
     return AddRec->getStart();
   return SE->getAddRecExpr(zeroCoefficient(AddRec->getStart(), TargetLoop),
-                           AddRec->getStepRecurrence(*SE),
-                           AddRec->getLoop(),
+                           AddRec->getStepRecurrence(*SE), AddRec->getLoop(),
                            AddRec->getNoWrapFlags());
 }
 
-
 // Given a linear SCEV Expr,
 // return the SCEV given by adding some Value to the
 // coefficient corresponding to the specified TargetLoop.
@@ -3128,17 +3099,13 @@ const SCEV *DependenceInfo::addToCoefficient(const SCEV *Expr,
                                              const SCEV *Value) const {
   const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Expr);
   if (!AddRec) // create a new addRec
-    return SE->getAddRecExpr(Expr,
-                             Value,
-                             TargetLoop,
+    return SE->getAddRecExpr(Expr, Value, TargetLoop,
                              SCEV::FlagAnyWrap); // Worst case, with no info.
   if (AddRec->getLoop() == TargetLoop) {
     const SCEV *Sum = SE->getAddExpr(AddRec->getStepRecurrence(*SE), Value);
     if (Sum->isZero())
       return AddRec->getStart();
-    return SE->getAddRecExpr(AddRec->getStart(),
-                             Sum,
-                             AddRec->getLoop(),
+    return SE->getAddRecExpr(AddRec->getStart(), Sum, AddRec->getLoop(),
                              AddRec->getNoWrapFlags());
   }
   if (SE->isLoopInvariant(AddRec, TargetLoop))
@@ -3149,7 +3116,6 @@ const SCEV *DependenceInfo::addToCoefficient(const SCEV *Expr,
       AddRec->getNoWrapFlags());
 }
 
-
 // Review the constraints, looking for opportunities
 // to simplify a subscript pair (Src and Dst).
 // Return true if some simplification occurs.
@@ -3178,7 +3144,6 @@ bool DependenceInfo::propagate(const SCEV *&Src, const SCEV *&Dst,
   return Result;
 }
 
-
 // Attempt to propagate a distance
 // constraint into a subscript pair (Src and Dst).
 // Return true if some simplification occurs.
@@ -3204,7 +3169,6 @@ bool DependenceInfo::propagateDistance(const SCEV *&Src, const SCEV *&Dst,
   return true;
 }
 
-
 // Attempt to propagate a line
 // constraint into a subscript pair (Src and Dst).
 // Return true if some simplification occurs.
@@ -3224,22 +3188,22 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst,
   if (A->isZero()) {
     const SCEVConstant *Bconst = dyn_cast<SCEVConstant>(B);
     const SCEVConstant *Cconst = dyn_cast<SCEVConstant>(C);
-    if (!Bconst || !Cconst) return false;
+    if (!Bconst || !Cconst)
+      return false;
     APInt Beta = Bconst->getAPInt();
     APInt Charlie = Cconst->getAPInt();
     APInt CdivB = Charlie.sdiv(Beta);
     assert(Charlie.srem(Beta) == 0 && "C should be evenly divisible by B");
     const SCEV *AP_K = findCoefficient(Dst, CurLoop);
-    //    Src = SE->getAddExpr(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB)));
     Src = SE->getMinusSCEV(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB)));
     Dst = zeroCoefficient(Dst, CurLoop);
     if (!findCoefficient(Src, CurLoop)->isZero())
       Consistent = false;
-  }
-  else if (B->isZero()) {
+  } else if (B->isZero()) {
     const SCEVConstant *Aconst = dyn_cast<SCEVConstant>(A);
     const SCEVConstant *Cconst = dyn_cast<SCEVConstant>(C);
-    if (!Aconst || !Cconst) return false;
+    if (!Aconst || !Cconst)
+      return false;
     APInt Alpha = Aconst->getAPInt();
     APInt Charlie = Cconst->getAPInt();
     APInt CdivA = Charlie.sdiv(Alpha);
@@ -3249,11 +3213,11 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst,
     Src = zeroCoefficient(Src, CurLoop);
     if (!findCoefficient(Dst, CurLoop)->isZero())
       Consistent = false;
-  }
-  else if (isKnownPredicate(CmpInst::ICMP_EQ, A, B)) {
+  } else if (isKnownPredicate(CmpInst::ICMP_EQ, A, B)) {
     const SCEVConstant *Aconst = dyn_cast<SCEVConstant>(A);
     const SCEVConstant *Cconst = dyn_cast<SCEVConstant>(C);
-    if (!Aconst || !Cconst) return false;
+    if (!Aconst || !Cconst)
+      return false;
     APInt Alpha = Aconst->getAPInt();
     APInt Charlie = Cconst->getAPInt();
     APInt CdivA = Charlie.sdiv(Alpha);
@@ -3264,8 +3228,7 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst,
     Dst = addToCoefficient(Dst, CurLoop, A_K);
     if (!findCoefficient(Dst, CurLoop)->isZero())
       Consistent = false;
-  }
-  else {
+  } else {
     // paper is incorrect here, or perhaps just misleading
     const SCEV *A_K = findCoefficient(Src, CurLoop);
     Src = SE->getMulExpr(Src, A);
@@ -3281,7 +3244,6 @@ bool DependenceInfo::propagateLine(const SCEV *&Src, const SCEV *&Dst,
   return true;
 }
 
-
 // Attempt to propagate a point
 // constraint into a subscript pair (Src and Dst).
 // Return true if some simplification occurs.
@@ -3302,7 +3264,6 @@ bool DependenceInfo::propagatePoint(const SCEV *&Src, const SCEV *&Dst,
   return true;
 }
 
-
 // Update direction vector entry based on the current constraint.
 void DependenceInfo::updateDirection(Dependence::DVEntry &Level,
                                      const Constraint &CurConstraint) const {
@@ -3322,34 +3283,28 @@ void DependenceInfo::updateDirection(Dependence::DVEntry &Level,
     if (!SE->isKnownNonNegative(Level.Distance)) // if may be negative
       NewDirection |= Dependence::DVEntry::GT;
     Level.Direction &= NewDirection;
-  }
-  else if (CurConstraint.isLine()) {
+  } else if (CurConstraint.isLine()) {
     Level.Scalar = false;
     Level.Distance = nullptr;
     // direction should be accurate
-  }
-  else if (CurConstraint.isPoint()) {
+  } else if (CurConstraint.isPoint()) {
     Level.Scalar = false;
     Level.Distance = nullptr;
     unsigned NewDirection = Dependence::DVEntry::NONE;
-    if (!isKnownPredicate(CmpInst::ICMP_NE,
-                          CurConstraint.getY(),
+    if (!isKnownPredicate(CmpInst::ICMP_NE, CurConstraint.getY(),
                           CurConstraint.getX()))
       // if X may be = Y
       NewDirection |= Dependence::DVEntry::EQ;
-    if (!isKnownPredicate(CmpInst::ICMP_SLE,
-                          CurConstraint.getY(),
+    if (!isKnownPredicate(CmpInst::ICMP_SLE, CurConstraint.getY(),
                           CurConstraint.getX()))
       // if Y may be > X
       NewDirection |= Dependence::DVEntry::LT;
-    if (!isKnownPredicate(CmpInst::ICMP_SGE,
-                          CurConstraint.getY(),
+    if (!isKnownPredicate(CmpInst::ICMP_SGE, CurConstraint.getY(),
                           CurConstraint.getX()))
       // if Y may be < X
       NewDirection |= Dependence::DVEntry::GT;
     Level.Direction &= NewDirection;
-  }
-  else
+  } else
     llvm_unreachable("constraint has unexpected kind");
 }
 
@@ -3425,7 +3380,7 @@ bool DependenceInfo::tryDelinearizeFixedSize(
         dyn_cast<SCEVUnknown>(SE->getPointerBase(DstAccessFn));
     assert(SrcBase && DstBase && SrcBase == DstBase &&
            "expected src and dst scev unknowns to be equal");
-    });
+  });
 
   SmallVector<int, 4> SrcSizes;
   SmallVector<int, 4> DstSizes;
@@ -3737,9 +3692,8 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
     Pair[P].Group.resize(Pairs);
     removeMatchingExtensions(&Pair[P]);
     Pair[P].Classification =
-      classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()),
-                   Pair[P].Dst, LI->getLoopFor(Dst->getParent()),
-                   Pair[P].Loops);
+        classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), Pair[P].Dst,
+                     LI->getLoopFor(Dst->getParent()), Pair[P].Loops);
     Pair[P].GroupLoops = Pair[P].Loops;
     Pair[P].Group.set(P);
     LLVM_DEBUG(dbgs() << "    subscript " << P << "\n");
@@ -3814,18 +3768,15 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
     if (Pair[SI].Classification == Subscript::NonLinear) {
       // ignore these, but collect loops for later
       ++NonlinearSubscriptPairs;
-      collectCommonLoops(Pair[SI].Src,
-                         LI->getLoopFor(Src->getParent()),
+      collectCommonLoops(Pair[SI].Src, LI->getLoopFor(Src->getParent()),
                          Pair[SI].Loops);
-      collectCommonLoops(Pair[SI].Dst,
-                         LI->getLoopFor(Dst->getParent()),
+      collectCommonLoops(Pair[SI].Dst, LI->getLoopFor(Dst->getParent()),
                          Pair[SI].Loops);
       Result.Consistent = false;
     } else if (Pair[SI].Classification == Subscript::ZIV) {
       // always separable
       Separable.set(SI);
-    }
-    else {
+    } else {
       // SIV, RDIV, or MIV, so check for coupled group
       bool Done = true;
       for (unsigned SJ = SI + 1; SJ < Pairs; ++SJ) {
@@ -3843,8 +3794,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
         if (Pair[SI].Group.count() == 1) {
           Separable.set(SI);
           ++SeparableSubscriptPairs;
-        }
-        else {
+        } else {
           Coupled.set(SI);
           ++CoupledSubscriptPairs;
         }
@@ -3950,10 +3900,9 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
                           Constraints, Result.Consistent)) {
               LLVM_DEBUG(dbgs() << "\t    Changed\n");
               ++DeltaPropagations;
-              Pair[SJ].Classification =
-                classifyPair(Pair[SJ].Src, LI->getLoopFor(Src->getParent()),
-                             Pair[SJ].Dst, LI->getLoopFor(Dst->getParent()),
-                             Pair[SJ].Loops);
+              Pair[SJ].Classification = classifyPair(
+                  Pair[SJ].Src, LI->getLoopFor(Src->getParent()), Pair[SJ].Dst,
+                  LI->getLoopFor(Dst->getParent()), Pair[SJ].Loops);
               switch (Pair[SJ].Classification) {
               case Subscript::ZIV:
                 LLVM_DEBUG(dbgs() << "ZIV\n");
@@ -3995,8 +3944,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
           LLVM_DEBUG(dbgs() << "MIV test\n");
           if (testMIV(Pair[SJ].Src, Pair[SJ].Dst, Pair[SJ].Loops, Result))
             return nullptr;
-        }
-        else
+        } else
           llvm_unreachable("expected only MIV subscripts at this point");
       }
 
@@ -4052,8 +4000,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
         break;
       }
     }
-  }
-  else {
+  } else {
     // On the other hand, if all directions are equal and there's no
     // loop-independent dependence possible, then no dependence exists.
     bool AllEqual = true;
@@ -4158,9 +4105,8 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep,
     Pair[P].Group.resize(Pairs);
     removeMatchingExtensions(&Pair[P]);
     Pair[P].Classification =
-      classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()),
-                   Pair[P].Dst, LI->getLoopFor(Dst->getParent()),
-                   Pair[P].Loops);
+        classifyPair(Pair[P].Src, LI->getLoopFor(Src->getParent()), Pair[P].Dst,
+                     LI->getLoopFor(Dst->getParent()), Pair[P].Loops);
     Pair[P].GroupLoops = Pair[P].Loops;
     Pair[P].Group.set(P);
   }
@@ -4172,15 +4118,12 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep,
   for (unsigned SI = 0; SI < Pairs; ++SI) {
     if (Pair[SI].Classification == Subscript::NonLinear) {
       // ignore these, but collect loops for later
-      collectCommonLoops(Pair[SI].Src,
-                         LI->getLoopFor(Src->getParent()),
+      collectCommonLoops(Pair[SI].Src, LI->getLoopFor(Src->getParent()),
                          Pair[SI].Loops);
-      collectCommonLoops(Pair[SI].Dst,
-                         LI->getLoopFor(Dst->getParent()),
+      collectCommonLoops(Pair[SI].Dst, LI->getLoopFor(Dst->getParent()),
                          Pair[SI].Loops);
       Result.Consistent = false;
-    }
-    else if (Pair[SI].Classification == Subscript::ZIV)
+    } else if (Pair[SI].Classification == Subscript::ZIV)
       Separable.set(SI);
     else {
       // SIV, RDIV, or MIV, so check for coupled group
@@ -4214,8 +4157,8 @@ const SCEV *DependenceInfo::getSplitIteration(const Dependence &Dep,
     case Subscript::SIV: {
       unsigned Level;
       const SCEV *SplitIter = nullptr;
-      (void) testSIV(Pair[SI].Src, Pair[SI].Dst, Level,
-                     Result, NewConstraint, SplitIter);
+      (void)testSIV(Pair[SI].Src, Pair[SI].Dst, Level, Result, NewConstraint,
+                    SplitIter);
       if (Level == SplitLevel) {
         assert(SplitIter != nullptr);
         return SplitIter;
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 8be5de3bf356f..b8c540ce4b99d 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -40,6 +40,8 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
   switch (Kind) {
   default:
     break;
+  case RecurKind::AddChainWithSubs:
+  case RecurKind::Sub:
   case RecurKind::Add:
   case RecurKind::Mul:
   case RecurKind::Or:
@@ -897,8 +899,11 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
   case Instruction::PHI:
     return InstDesc(I, Prev.getRecKind(), Prev.getExactFPMathInst());
   case Instruction::Sub:
+    return InstDesc(
+        Kind == RecurKind::Sub || Kind == RecurKind::AddChainWithSubs, I);
   case Instruction::Add:
-    return InstDesc(Kind == RecurKind::Add, I);
+    return InstDesc(
+        Kind == RecurKind::Add || Kind == RecurKind::AddChainWithSubs, I);
   case Instruction::Mul:
     return InstDesc(Kind == RecurKind::Mul, I);
   case Instruction::And:
@@ -917,7 +922,8 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
                     I->hasAllowReassoc() ? nullptr : I);
   case Instruction::Select:
     if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul ||
-        Kind == RecurKind::Add || Kind == RecurKind::Mul)
+        Kind == RecurKind::Add || Kind == RecurKind::Mul ||
+        Kind == RecurKind::Sub || Kind == RecurKind::AddChainWithSubs)
       return isConditionalRdxPattern(I);
     if (isFindIVRecurrenceKind(Kind) && SE)
       return isFindIVPattern(Kind, L, OrigPhi, I, *SE);
@@ -1003,6 +1009,17 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
     LLVM_DEBUG(dbgs() << "Found an ADD reduction PHI." << *Phi << "\n");
     return true;
   }
+  if (AddReductionVar(Phi, RecurKind::Sub, TheLoop, FMF, RedDes, DB, AC, DT,
+                      SE)) {
+    LLVM_DEBUG(dbgs() << "Found a SUB reduction PHI." << *Phi << "\n");
+    return true;
+  }
+  if (AddReductionVar(Phi, RecurKind::AddChainWithSubs, TheLoop, FMF, RedDes,
+                      DB, AC, DT, SE)) {
+    LLVM_DEBUG(dbgs() << "Found a chained ADD-SUB reduction PHI." << *Phi
+                      << "\n");
+    return true;
+  }
   if (AddReductionVar(Phi, RecurKind::Mul, TheLoop, FMF, RedDes, DB, AC, DT,
                       SE)) {
     LLVM_DEBUG(dbgs() << "Found a MUL reduction PHI." << *Phi << "\n");
@@ -1201,6 +1218,9 @@ bool RecurrenceDescriptor::isFixedOrderRecurrence(PHINode *Phi, Loop *TheLoop,
 
 unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
   switch (Kind) {
+  case RecurKind::Sub:
+    return Instruction::Sub;
+  case RecurKind::AddChainWithSubs:
   case RecurKind::Add:
     return Instruction::Add;
   case RecurKind::Mul:
@@ -1288,6 +1308,10 @@ RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
     if (isFMulAddIntrinsic(Cur))
       return true;
 
+    if (Cur->getOpcode() == Instruction::Sub &&
+        Kind == RecurKind::AddChainWithSubs)
+      return true;
+
     return Cur->getOpcode() == getOpcode();
   };
 
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index 22f4d08448a22..757f68999691e 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -180,6 +180,10 @@ static cl::opt<bool> DisableGEPConstOperand(
     "disable-gep-const-evaluation", cl::Hidden, cl::init(false),
     cl::desc("Disables evaluation of GetElementPtr with constant operands"));
 
+static cl::opt<bool> InlineAllViableCalls(
+    "inline-all-viable-calls", cl::Hidden, cl::init(false),
+    cl::desc("Inline all viable calls, even if they exceed the inlining "
+             "threshold"));
 namespace llvm {
 std::optional<int> getStringFnAttrAsInt(const Attribute &Attr) {
   if (Attr.isValid()) {
@@ -3272,6 +3276,10 @@ InlineCost llvm::getInlineCost(
     return llvm::InlineCost::getNever(UserDecision->getFailureReason());
   }
 
+  if (InlineAllViableCalls && isInlineViable(*Callee).isSuccess())
+    return llvm::InlineCost::getAlways(
+        "Inlining forced by -inline-all-viable-calls");
+
   LLVM_DEBUG(llvm::dbgs() << "      Analyzing call of " << Callee->getName()
                           << "... (caller:" << Call.getCaller()->getName()
                           << ")\n");
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 922f25de54e9d..c7b0ca97a8e43 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -927,8 +927,13 @@ LazyValueInfoImpl::solveBlockValueCast(CastInst *CI, BasicBlock *BB) {
   // NOTE: We're currently limited by the set of operations that ConstantRange
   // can evaluate symbolically.  Enhancing that set will allows us to analyze
   // more definitions.
-  return ValueLatticeElement::getRange(LHSRange.castOp(CI->getOpcode(),
-                                                       ResultBitWidth));
+  ConstantRange Res = ConstantRange::getEmpty(ResultBitWidth);
+  if (auto *Trunc = dyn_cast<TruncInst>(CI))
+    Res = LHSRange.truncate(ResultBitWidth, Trunc->getNoWrapKind());
+  else
+    Res = LHSRange.castOp(CI->getOpcode(), ResultBitWidth);
+
+  return ValueLatticeElement::getRange(Res);
 }
 
 std::optional<ValueLatticeElement>
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 78d0887d5d87e..9a2c9ba63ec7e 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -276,8 +276,7 @@ static bool AreEquivalentAddressValues(const Value *A, const Value *B) {
   // this function is only used when one address use dominates the
   // other, which means that they'll always either have the same
   // value or one of them will have an undefined value.
-  if (isa<BinaryOperator>(A) || isa<CastInst>(A) || isa<PHINode>(A) ||
-      isa<GetElementPtrInst>(A))
+  if (isa<CastInst>(A) || isa<PHINode>(A) || isa<GetElementPtrInst>(A))
     if (const Instruction *BI = dyn_cast<Instruction>(B))
       if (cast<Instruction>(A)->isIdenticalToWhenDefined(BI))
         return true;
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index a5535339a714f..bceddd0325276 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -936,6 +936,12 @@ class AccessAnalysis {
 static std::optional<int64_t>
 getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy,
                     Value *Ptr, PredicatedScalarEvolution &PSE) {
+  if (isa<ScalableVectorType>(AccessTy)) {
+    LLVM_DEBUG(dbgs() << "LAA: Bad stride - Scalable object: " << *AccessTy
+                      << "\n");
+    return std::nullopt;
+  }
+
   // The access function must stride over the innermost loop.
   if (Lp != AR->getLoop()) {
     LLVM_DEBUG({
@@ -1590,11 +1596,6 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr,
     return 0;
 
   assert(Ptr->getType()->isPointerTy() && "Unexpected non-ptr");
-  if (isa<ScalableVectorType>(AccessTy)) {
-    LLVM_DEBUG(dbgs() << "LAA: Bad stride - Scalable object: " << *AccessTy
-                      << "\n");
-    return std::nullopt;
-  }
 
   const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PtrScev);
   if (Assume && !AR)
@@ -2404,12 +2405,13 @@ bool MemoryDepChecker::areDepsSafe(const DepCandidates &DepCands,
 SmallVector<Instruction *, 4>
 MemoryDepChecker::getInstructionsForAccess(Value *Ptr, bool IsWrite) const {
   MemAccessInfo Access(Ptr, IsWrite);
-  auto &IndexVector = Accesses.find(Access)->second;
-
+  auto I = Accesses.find(Access);
   SmallVector<Instruction *, 4> Insts;
-  transform(IndexVector,
-                 std::back_inserter(Insts),
-                 [&](unsigned Idx) { return this->InstMap[Idx]; });
+  if (I != Accesses.end()) {
+    transform(I->second, std::back_inserter(Insts),
+              [&](unsigned Idx) { return this->InstMap[Idx]; });
+  }
+
   return Insts;
 }
 
diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp
index 518a634cdb363..6ba6073cce950 100644
--- a/llvm/lib/Analysis/LoopInfo.cpp
+++ b/llvm/lib/Analysis/LoopInfo.cpp
@@ -58,14 +58,26 @@ static cl::opt<bool, true>
 // Loop implementation
 //
 
-bool Loop::isLoopInvariant(const Value *V) const {
-  if (const Instruction *I = dyn_cast<Instruction>(V))
-    return !contains(I);
+bool Loop::isLoopInvariant(const Value *V, bool HasCoroSuspendInst) const {
+  if (const Instruction *I = dyn_cast<Instruction>(V)) {
+    // FIXME: this is semantically inconsistent. We're tracking a proper fix in
+    // issue #149604.
+    // If V is a pointer to stack object and L contains a coro.suspend function
+    // call, then V may not be loop invariant because the ramp function and
+    // resume function have different stack frames.
+    if (HasCoroSuspendInst && isa<AllocaInst>(I))
+      return false;
+    else
+      return !contains(I);
+  }
   return true; // All non-instructions are loop invariant
 }
 
-bool Loop::hasLoopInvariantOperands(const Instruction *I) const {
-  return all_of(I->operands(), [this](Value *V) { return isLoopInvariant(V); });
+bool Loop::hasLoopInvariantOperands(const Instruction *I,
+                                    bool HasCoroSuspendInst) const {
+  return all_of(I->operands(), [&](Value *V) {
+    return isLoopInvariant(V, HasCoroSuspendInst);
+  });
 }
 
 bool Loop::makeLoopInvariant(Value *V, bool &Changed, Instruction *InsertPt,
diff --git a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
index 2b0f212bff01a..67c2cfadb6533 100644
--- a/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -150,6 +150,10 @@ static ModRefInfo GetLocation(const Instruction *Inst, MemoryLocation &Loc,
     switch (II->getIntrinsicID()) {
     case Intrinsic::lifetime_start:
     case Intrinsic::lifetime_end:
+      Loc = MemoryLocation::getForArgument(II, 0, TLI);
+      // These intrinsics don't really modify the memory, but returning Mod
+      // will allow them to be handled conservatively.
+      return ModRefInfo::Mod;
     case Intrinsic::invariant_start:
       Loc = MemoryLocation::getForArgument(II, 1, TLI);
       // These intrinsics don't really modify the memory, but returning Mod
@@ -441,11 +445,7 @@ MemDepResult MemoryDependenceResults::getSimplePointerDependencyFrom(
       Intrinsic::ID ID = II->getIntrinsicID();
       switch (ID) {
       case Intrinsic::lifetime_start: {
-        // FIXME: This only considers queries directly on the invariant-tagged
-        // pointer, not on query pointers that are indexed off of them.  It'd
-        // be nice to handle that at some point (the right approach is to use
-        // GetPointerBaseWithConstantOffset).
-        MemoryLocation ArgLoc = MemoryLocation::getAfter(II->getArgOperand(1));
+        MemoryLocation ArgLoc = MemoryLocation::getAfter(II->getArgOperand(0));
         if (BatchAA.isMustAlias(ArgLoc, MemLoc))
           return MemDepResult::getDef(II);
         continue;
diff --git a/llvm/lib/Analysis/MemoryLocation.cpp b/llvm/lib/Analysis/MemoryLocation.cpp
index 28a264093af23..72b643c56a994 100644
--- a/llvm/lib/Analysis/MemoryLocation.cpp
+++ b/llvm/lib/Analysis/MemoryLocation.cpp
@@ -191,7 +191,7 @@ MemoryLocation MemoryLocation::getForArgument(const CallBase *Call,
 
     case Intrinsic::lifetime_start:
     case Intrinsic::lifetime_end: {
-      assert(ArgIdx == 1 && "Invalid argument index");
+      assert(ArgIdx == 0 && "Invalid argument index");
       auto *AI = dyn_cast<AllocaInst>(Arg);
       if (!AI)
         // lifetime of poison value.
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 477e4771e04aa..f60a1e9f22704 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -67,7 +67,6 @@
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
@@ -7284,7 +7283,7 @@ ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops,
                                        bool &Precise) {
   Precise = true;
   // Do a bounded search of the def relation of the requested SCEVs.
-  SmallSet<const SCEV *, 16> Visited;
+  SmallPtrSet<const SCEV *, 16> Visited;
   SmallVector<const SCEV *> Worklist;
   auto pushOp = [&](const SCEV *S) {
     if (!Visited.insert(S).second)
@@ -7435,7 +7434,15 @@ ScalarEvolution::getLoopProperties(const Loop *L) {
       if (auto *SI = dyn_cast<StoreInst>(I))
         return !SI->isSimple();
 
-      return I->mayThrow() || I->mayWriteToMemory();
+      if (I->mayThrow())
+        return true;
+
+      // Non-volatile memset / memcpy do not count as side-effect for forward
+      // progress.
+      if (isa<MemIntrinsic>(I) && !I->isVolatile())
+        return false;
+
+      return I->mayWriteToMemory();
     };
 
     LoopProperties LP = {/* HasNoAbnormalExits */ true,
@@ -14944,6 +14951,29 @@ const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
   if (!AddRec)
     return nullptr;
 
+  // Check if any of the transformed predicates is known to be false. In that
+  // case, it doesn't make sense to convert to a predicated AddRec, as the
+  // versioned loop will never execute.
+  for (const SCEVPredicate *Pred : TransformPreds) {
+    auto *WrapPred = dyn_cast<SCEVWrapPredicate>(Pred);
+    if (!WrapPred || WrapPred->getFlags() != SCEVWrapPredicate::IncrementNSSW)
+      continue;
+
+    const SCEVAddRecExpr *AddRecToCheck = WrapPred->getExpr();
+    const SCEV *ExitCount = getBackedgeTakenCount(AddRecToCheck->getLoop());
+    if (isa<SCEVCouldNotCompute>(ExitCount))
+      continue;
+
+    const SCEV *Step = AddRecToCheck->getStepRecurrence(*this);
+    if (!Step->isOne())
+      continue;
+
+    ExitCount = getTruncateOrSignExtend(ExitCount, Step->getType());
+    const SCEV *Add = getAddExpr(AddRecToCheck->getStart(), ExitCount);
+    if (isKnownPredicate(CmpInst::ICMP_SLT, Add, AddRecToCheck->getStart()))
+      return nullptr;
+  }
+
   // Since the transformation was successful, we can now transfer the SCEV
   // predicates.
   Preds.append(TransformPreds.begin(), TransformPreds.end());
diff --git a/llvm/lib/Analysis/StackLifetime.cpp b/llvm/lib/Analysis/StackLifetime.cpp
index abe4985544e40..1e20fca965ace 100644
--- a/llvm/lib/Analysis/StackLifetime.cpp
+++ b/llvm/lib/Analysis/StackLifetime.cpp
@@ -70,7 +70,7 @@ void StackLifetime::collectMarkers() {
       const IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
       if (!II || !II->isLifetimeStartOrEnd())
         continue;
-      const AllocaInst *AI = dyn_cast<AllocaInst>(II->getArgOperand(1));
+      const AllocaInst *AI = dyn_cast<AllocaInst>(II->getArgOperand(0));
       if (!AI)
         continue;
       auto It = AllocaNumbering.find(AI);
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index c7eb2ec18c679..4ac8f03e6dbf5 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -637,9 +637,8 @@ InstructionCost TargetTransformInfo::getScalarizationOverhead(
 }
 
 InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(
-    ArrayRef<const Value *> Args, ArrayRef<Type *> Tys,
-    TTI::TargetCostKind CostKind) const {
-  return TTIImpl->getOperandsScalarizationOverhead(Args, Tys, CostKind);
+    ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind) const {
+  return TTIImpl->getOperandsScalarizationOverhead(Tys, CostKind);
 }
 
 bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
@@ -1130,6 +1129,15 @@ TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,
   return Cost;
 }
 
+InstructionCost TargetTransformInfo::getIndexedVectorInstrCostFromEnd(
+    unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
+    unsigned Index) const {
+  InstructionCost Cost =
+      TTIImpl->getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind, Index);
+  assert(Cost >= 0 && "TTI should not produce negative costs!");
+  return Cost;
+}
+
 InstructionCost TargetTransformInfo::getInsertExtractValueCost(
     unsigned Opcode, TTI::TargetCostKind CostKind) const {
   assert((Opcode == Instruction::InsertValue ||
@@ -1230,10 +1238,11 @@ unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {
   return TTIImpl->getNumberOfParts(Tp);
 }
 
-InstructionCost
-TargetTransformInfo::getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
-                                               const SCEV *Ptr) const {
-  InstructionCost Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr);
+InstructionCost TargetTransformInfo::getAddressComputationCost(
+    Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,
+    TTI::TargetCostKind CostKind) const {
+  InstructionCost Cost =
+      TTIImpl->getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
   assert(Cost >= 0 && "TTI should not produce negative costs!");
   return Cost;
 }
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 1e70228905c33..7fe129b8456f6 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -19,7 +19,6 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
@@ -6356,27 +6355,6 @@ llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
   return nullptr;
 }
 
-bool llvm::isGEPBasedOnPointerToString(const GEPOperator *GEP,
-                                       unsigned CharSize) {
-  // Make sure the GEP has exactly three arguments.
-  if (GEP->getNumOperands() != 3)
-    return false;
-
-  // Make sure the index-ee is a pointer to array of \p CharSize integers.
-  // CharSize.
-  ArrayType *AT = dyn_cast<ArrayType>(GEP->getSourceElementType());
-  if (!AT || !AT->getElementType()->isIntegerTy(CharSize))
-    return false;
-
-  // Check to make sure that the first operand of the GEP is an integer and
-  // has value 0 so that we are sure we're indexing into the initializer.
-  const ConstantInt *FirstIdx = dyn_cast<ConstantInt>(GEP->getOperand(1));
-  if (!FirstIdx || !FirstIdx->isZero())
-    return false;
-
-  return true;
-}
-
 // If V refers to an initialized global constant, set Slice either to
 // its initializer if the size of its elements equals ElementSize, or,
 // for ElementSize == 8, to its representation as an array of unsiged
@@ -7415,8 +7393,10 @@ static bool canCreateUndefOrPoison(const Operator *Op, UndefPoisonKind Kind,
       case Intrinsic::fshr:
       case Intrinsic::smax:
       case Intrinsic::smin:
+      case Intrinsic::scmp:
       case Intrinsic::umax:
       case Intrinsic::umin:
+      case Intrinsic::ucmp:
       case Intrinsic::ptrmask:
       case Intrinsic::fptoui_sat:
       case Intrinsic::fptosi_sat:
@@ -7785,7 +7765,7 @@ bool llvm::mustExecuteUBIfPoisonOnPathTo(Instruction *Root,
 
   // The set of all recursive users we've visited (which are assumed to all be
   // poison because of said visit)
-  SmallSet<const Value *, 16> KnownPoison;
+  SmallPtrSet<const Value *, 16> KnownPoison;
   SmallVector<const Instruction*, 16> Worklist;
   Worklist.push_back(Root);
   while (!Worklist.empty()) {
@@ -8140,8 +8120,8 @@ static bool programUndefinedIfUndefOrPoison(const Value *V,
 
   // Set of instructions that we have proved will yield poison if Inst
   // does.
-  SmallSet<const Value *, 16> YieldsPoison;
-  SmallSet<const BasicBlock *, 4> Visited;
+  SmallPtrSet<const Value *, 16> YieldsPoison;
+  SmallPtrSet<const BasicBlock *, 4> Visited;
 
   YieldsPoison.insert(V);
   Visited.insert(BB);
@@ -9147,7 +9127,8 @@ static bool matchTwoInputRecurrence(const PHINode *PN, InstTy *&Inst,
     return false;
 
   for (unsigned I = 0; I != 2; ++I) {
-    if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(I))) {
+    if (auto *Operation = dyn_cast<InstTy>(PN->getIncomingValue(I));
+        Operation && Operation->getNumOperands() >= 2) {
       Value *LHS = Operation->getOperand(0);
       Value *RHS = Operation->getOperand(1);
       if (LHS != PN && RHS != PN)
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index b3b4c37475eef..425ea311d653a 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -81,6 +81,7 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
   case Intrinsic::exp:
   case Intrinsic::exp10:
   case Intrinsic::exp2:
+  case Intrinsic::ldexp:
   case Intrinsic::log:
   case Intrinsic::log10:
   case Intrinsic::log2:
@@ -108,6 +109,8 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
   case Intrinsic::canonicalize:
   case Intrinsic::fptosi_sat:
   case Intrinsic::fptoui_sat:
+  case Intrinsic::lround:
+  case Intrinsic::llround:
   case Intrinsic::lrint:
   case Intrinsic::llrint:
   case Intrinsic::ucmp:
@@ -189,6 +192,8 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
   switch (ID) {
   case Intrinsic::fptosi_sat:
   case Intrinsic::fptoui_sat:
+  case Intrinsic::lround:
+  case Intrinsic::llround:
   case Intrinsic::lrint:
   case Intrinsic::llrint:
   case Intrinsic::vp_lrint:
@@ -203,6 +208,7 @@ bool llvm::isVectorIntrinsicWithOverloadTypeAtArg(
   case Intrinsic::vp_is_fpclass:
     return OpdIdx == 0;
   case Intrinsic::powi:
+  case Intrinsic::ldexp:
     return OpdIdx == -1 || OpdIdx == 1;
   default:
     return OpdIdx == -1;
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index 520c6a00a9c07..3d5bd6155536e 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -928,6 +928,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   INSTKEYWORD(fptoui,      FPToUI);
   INSTKEYWORD(fptosi,      FPToSI);
   INSTKEYWORD(inttoptr,    IntToPtr);
+  INSTKEYWORD(ptrtoaddr,   PtrToAddr);
   INSTKEYWORD(ptrtoint,    PtrToInt);
   INSTKEYWORD(bitcast,     BitCast);
   INSTKEYWORD(addrspacecast, AddrSpaceCast);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 13bef1f62f1a9..1bc2906f63b07 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -4273,6 +4273,7 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) {
   case lltok::kw_bitcast:
   case lltok::kw_addrspacecast:
   case lltok::kw_inttoptr:
+  case lltok::kw_ptrtoaddr:
   case lltok::kw_ptrtoint: {
     unsigned Opc = Lex.getUIntVal();
     Type *DestTy = nullptr;
@@ -7310,6 +7311,7 @@ int LLParser::parseInstruction(Instruction *&Inst, BasicBlock *BB,
   case lltok::kw_fptoui:
   case lltok::kw_fptosi:
   case lltok::kw_inttoptr:
+  case lltok::kw_ptrtoaddr:
   case lltok::kw_ptrtoint:
     return parseCast(Inst, PFS, KeywordVal);
   case lltok::kw_fptrunc:
diff --git a/llvm/lib/BinaryFormat/MsgPackDocument.cpp b/llvm/lib/BinaryFormat/MsgPackDocument.cpp
index 11598ee24d6f2..b52f029122444 100644
--- a/llvm/lib/BinaryFormat/MsgPackDocument.cpp
+++ b/llvm/lib/BinaryFormat/MsgPackDocument.cpp
@@ -104,6 +104,10 @@ DocNode &DocNode::operator=(uint64_t Val) {
   *this = getDocument()->getNode(Val);
   return *this;
 }
+DocNode &DocNode::operator=(double Val) {
+  *this = getDocument()->getNode(Val);
+  return *this;
+}
 
 // A level in the document reading stack.
 struct StackLevel {
@@ -293,6 +297,9 @@ void Document::writeToBlob(std::string &Blob) {
     case Type::Binary:
       MPWriter.write(Node.getBinary());
       break;
+    case Type::Float:
+      MPWriter.write(Node.getFloat());
+      break;
     case Type::Empty:
       llvm_unreachable("unhandled empty msgpack node");
     default:
diff --git a/llvm/lib/BinaryFormat/SFrame.cpp b/llvm/lib/BinaryFormat/SFrame.cpp
index f1765d7f3e852..8076a26f27f32 100644
--- a/llvm/lib/BinaryFormat/SFrame.cpp
+++ b/llvm/lib/BinaryFormat/SFrame.cpp
@@ -68,3 +68,11 @@ ArrayRef<EnumEntry<sframe::FREOffset>> sframe::getFREOffsets() {
   };
   return ArrayRef(FREOffsets);
 }
+
+ArrayRef<EnumEntry<sframe::BaseReg>> sframe::getBaseRegisters() {
+  static constexpr EnumEntry<sframe::BaseReg> BaseRegs[] = {
+      {"FP", sframe::BaseReg::FP},
+      {"SP", sframe::BaseReg::SP},
+  };
+  return ArrayRef(BaseRegs);
+}
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index 290d873c632c9..22a0d0ffdbaab 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1283,6 +1283,7 @@ static int getDecodedCastOpcode(unsigned Val) {
   case bitc::CAST_SITOFP  : return Instruction::SIToFP;
   case bitc::CAST_FPTRUNC : return Instruction::FPTrunc;
   case bitc::CAST_FPEXT   : return Instruction::FPExt;
+  case bitc::CAST_PTRTOADDR: return Instruction::PtrToAddr;
   case bitc::CAST_PTRTOINT: return Instruction::PtrToInt;
   case bitc::CAST_INTTOPTR: return Instruction::IntToPtr;
   case bitc::CAST_BITCAST : return Instruction::BitCast;
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 05680fa5c0f5f..a3f825408d0c2 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -647,6 +647,7 @@ static unsigned getEncodedCastOpcode(unsigned Opcode) {
   case Instruction::SIToFP  : return bitc::CAST_SITOFP;
   case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
   case Instruction::FPExt   : return bitc::CAST_FPEXT;
+  case Instruction::PtrToAddr: return bitc::CAST_PTRTOADDR;
   case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
   case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
   case Instruction::BitCast : return bitc::CAST_BITCAST;
diff --git a/llvm/lib/CAS/ActionCache.cpp b/llvm/lib/CAS/ActionCache.cpp
new file mode 100644
index 0000000000000..253ab5d249d3f
--- /dev/null
+++ b/llvm/lib/CAS/ActionCache.cpp
@@ -0,0 +1,22 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/CASID.h"
+#include "llvm/CAS/ObjectStore.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+void ActionCache::anchor() {}
+
+CacheKey::CacheKey(const CASID &ID) : Key(toStringRef(ID.getHash()).str()) {}
+CacheKey::CacheKey(const ObjectProxy &Proxy)
+    : CacheKey(Proxy.getCAS(), Proxy.getRef()) {}
+CacheKey::CacheKey(const ObjectStore &CAS, const ObjectRef &Ref)
+    : Key(toStringRef(CAS.getID(Ref).getHash())) {}
diff --git a/llvm/lib/CAS/ActionCaches.cpp b/llvm/lib/CAS/ActionCaches.cpp
new file mode 100644
index 0000000000000..571c5b3ca5b4b
--- /dev/null
+++ b/llvm/lib/CAS/ActionCaches.cpp
@@ -0,0 +1,101 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file implements the underlying ActionCache implementations.
+///
+//===----------------------------------------------------------------------===//
+
+#include "BuiltinCAS.h"
+#include "llvm/ADT/TrieRawHashMap.h"
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/Support/BLAKE3.h"
+
+#define DEBUG_TYPE "cas-action-caches"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+namespace {
+
+using HasherT = BLAKE3;
+using HashType = decltype(HasherT::hash(std::declval<ArrayRef<uint8_t> &>()));
+
+template <size_t Size> class CacheEntry {
+public:
+  CacheEntry() = default;
+  CacheEntry(ArrayRef<uint8_t> Hash) { llvm::copy(Hash, Value.data()); }
+  CacheEntry(const CacheEntry &Entry) { llvm::copy(Entry.Value, Value.data()); }
+  ArrayRef<uint8_t> getValue() const { return Value; }
+
+private:
+  std::array<uint8_t, Size> Value;
+};
+
+/// Builtin InMemory ActionCache that stores the mapping in memory.
+class InMemoryActionCache final : public ActionCache {
+public:
+  InMemoryActionCache()
+      : ActionCache(builtin::BuiltinCASContext::getDefaultContext()) {}
+
+  Error putImpl(ArrayRef<uint8_t> ActionKey, const CASID &Result,
+                bool CanBeDistributed) final;
+  Expected<std::optional<CASID>> getImpl(ArrayRef<uint8_t> ActionKey,
+                                         bool CanBeDistributed) const final;
+
+private:
+  using DataT = CacheEntry<sizeof(HashType)>;
+  using InMemoryCacheT = ThreadSafeTrieRawHashMap<DataT, sizeof(HashType)>;
+
+  InMemoryCacheT Cache;
+};
+} // end namespace
+
+static Error createResultCachePoisonedError(ArrayRef<uint8_t> KeyHash,
+                                            const CASContext &Context,
+                                            CASID Output,
+                                            ArrayRef<uint8_t> ExistingOutput) {
+  std::string Existing =
+      CASID::create(&Context, toStringRef(ExistingOutput)).toString();
+  SmallString<64> Key;
+  toHex(KeyHash, /*LowerCase=*/true, Key);
+  return createStringError(std::make_error_code(std::errc::invalid_argument),
+                           "cache poisoned for '" + Key + "' (new='" +
+                               Output.toString() + "' vs. existing '" +
+                               Existing + "')");
+}
+
+Expected<std::optional<CASID>>
+InMemoryActionCache::getImpl(ArrayRef<uint8_t> Key,
+                             bool /*CanBeDistributed*/) const {
+  auto Result = Cache.find(Key);
+  if (!Result)
+    return std::nullopt;
+  return CASID::create(&getContext(), toStringRef(Result->Data.getValue()));
+}
+
+Error InMemoryActionCache::putImpl(ArrayRef<uint8_t> Key, const CASID &Result,
+                                   bool /*CanBeDistributed*/) {
+  DataT Expected(Result.getHash());
+  const InMemoryCacheT::value_type &Cached = *Cache.insertLazy(
+      Key, [&](auto ValueConstructor) { ValueConstructor.emplace(Expected); });
+
+  const DataT &Observed = Cached.Data;
+  if (Expected.getValue() == Observed.getValue())
+    return Error::success();
+
+  return createResultCachePoisonedError(Key, getContext(), Result,
+                                        Observed.getValue());
+}
+
+namespace llvm::cas {
+
+std::unique_ptr<ActionCache> createInMemoryActionCache() {
+  return std::make_unique<InMemoryActionCache>();
+}
+
+} // namespace llvm::cas
diff --git a/llvm/lib/CAS/BuiltinCAS.cpp b/llvm/lib/CAS/BuiltinCAS.cpp
new file mode 100644
index 0000000000000..73646ad2c3528
--- /dev/null
+++ b/llvm/lib/CAS/BuiltinCAS.cpp
@@ -0,0 +1,94 @@
+//===- BuiltinCAS.cpp -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "BuiltinCAS.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/CAS/BuiltinObjectHasher.h"
+#include "llvm/Support/Process.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+using namespace llvm::cas::builtin;
+
+static StringRef getCASIDPrefix() { return "llvmcas://"; }
+void BuiltinCASContext::anchor() {}
+
+Expected<HashType> BuiltinCASContext::parseID(StringRef Reference) {
+  if (!Reference.consume_front(getCASIDPrefix()))
+    return createStringError(std::make_error_code(std::errc::invalid_argument),
+                             "invalid cas-id '" + Reference + "'");
+
+  // FIXME: Allow shortened references?
+  if (Reference.size() != 2 * sizeof(HashType))
+    return createStringError(std::make_error_code(std::errc::invalid_argument),
+                             "wrong size for cas-id hash '" + Reference + "'");
+
+  std::string Binary;
+  if (!tryGetFromHex(Reference, Binary))
+    return createStringError(std::make_error_code(std::errc::invalid_argument),
+                             "invalid hash in cas-id '" + Reference + "'");
+
+  assert(Binary.size() == sizeof(HashType));
+  HashType Digest;
+  llvm::copy(Binary, Digest.data());
+  return Digest;
+}
+
+Expected<CASID> BuiltinCAS::parseID(StringRef Reference) {
+  Expected<HashType> Digest = BuiltinCASContext::parseID(Reference);
+  if (!Digest)
+    return Digest.takeError();
+
+  return CASID::create(&getContext(), toStringRef(*Digest));
+}
+
+void BuiltinCASContext::printID(ArrayRef<uint8_t> Digest, raw_ostream &OS) {
+  SmallString<64> Hash;
+  toHex(Digest, /*LowerCase=*/true, Hash);
+  OS << getCASIDPrefix() << Hash;
+}
+
+void BuiltinCASContext::printIDImpl(raw_ostream &OS, const CASID &ID) const {
+  BuiltinCASContext::printID(ID.getHash(), OS);
+}
+
+const BuiltinCASContext &BuiltinCASContext::getDefaultContext() {
+  static BuiltinCASContext DefaultContext;
+  return DefaultContext;
+}
+
+Expected<ObjectRef> BuiltinCAS::store(ArrayRef<ObjectRef> Refs,
+                                      ArrayRef<char> Data) {
+  return storeImpl(BuiltinObjectHasher<HasherT>::hashObject(*this, Refs, Data),
+                   Refs, Data);
+}
+
+Error BuiltinCAS::validate(const CASID &ID) {
+  auto Ref = getReference(ID);
+  if (!Ref)
+    return createUnknownObjectError(ID);
+
+  auto Handle = load(*Ref);
+  if (!Handle)
+    return Handle.takeError();
+
+  auto Proxy = ObjectProxy::load(*this, *Ref, *Handle);
+  SmallVector<ObjectRef> Refs;
+  if (auto E = Proxy.forEachReference([&](ObjectRef Ref) -> Error {
+        Refs.push_back(Ref);
+        return Error::success();
+      }))
+    return E;
+
+  ArrayRef<char> Data(Proxy.getData().data(), Proxy.getData().size());
+  auto Hash = BuiltinObjectHasher<HasherT>::hashObject(*this, Refs, Data);
+  if (!ID.getHash().equals(Hash))
+    return createCorruptObjectError(ID);
+
+  return Error::success();
+}
diff --git a/llvm/lib/CAS/BuiltinCAS.h b/llvm/lib/CAS/BuiltinCAS.h
new file mode 100644
index 0000000000000..3b5374d5e1850
--- /dev/null
+++ b/llvm/lib/CAS/BuiltinCAS.h
@@ -0,0 +1,74 @@
+//===- BuiltinCAS.h ---------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CAS_BUILTINCAS_H
+#define LLVM_LIB_CAS_BUILTINCAS_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CAS/BuiltinCASContext.h"
+#include "llvm/CAS/ObjectStore.h"
+
+namespace llvm::cas {
+class ActionCache;
+namespace builtin {
+
+/// Common base class for builtin CAS implementations using the same CASContext.
+class BuiltinCAS : public ObjectStore {
+public:
+  BuiltinCAS() : ObjectStore(BuiltinCASContext::getDefaultContext()) {}
+
+  Expected<CASID> parseID(StringRef Reference) final;
+
+  Expected<ObjectRef> store(ArrayRef<ObjectRef> Refs,
+                            ArrayRef<char> Data) final;
+  virtual Expected<ObjectRef> storeImpl(ArrayRef<uint8_t> ComputedHash,
+                                        ArrayRef<ObjectRef> Refs,
+                                        ArrayRef<char> Data) = 0;
+
+  virtual Expected<ObjectRef>
+  storeFromNullTerminatedRegion(ArrayRef<uint8_t> ComputedHash,
+                                sys::fs::mapped_file_region Map) {
+    return storeImpl(ComputedHash, {}, ArrayRef(Map.data(), Map.size()));
+  }
+
+  /// Both builtin CAS implementations provide lifetime for free, so this can
+  /// be const, and readData() and getDataSize() can be implemented on top of
+  /// it.
+  virtual ArrayRef<char> getDataConst(ObjectHandle Node) const = 0;
+
+  ArrayRef<char> getData(ObjectHandle Node,
+                         bool RequiresNullTerminator) const final {
+    // BuiltinCAS Objects are always null terminated.
+    return getDataConst(Node);
+  }
+  uint64_t getDataSize(ObjectHandle Node) const final {
+    return getDataConst(Node).size();
+  }
+
+  Error createUnknownObjectError(const CASID &ID) const {
+    return createStringError(std::make_error_code(std::errc::invalid_argument),
+                             "unknown object '" + ID.toString() + "'");
+  }
+
+  Error createCorruptObjectError(const CASID &ID) const {
+    return createStringError(std::make_error_code(std::errc::invalid_argument),
+                             "corrupt object '" + ID.toString() + "'");
+  }
+
+  Error createCorruptStorageError() const {
+    return createStringError(std::make_error_code(std::errc::invalid_argument),
+                             "corrupt storage");
+  }
+
+  Error validate(const CASID &ID) final;
+};
+
+} // end namespace builtin
+} // end namespace llvm::cas
+
+#endif // LLVM_LIB_CAS_BUILTINCAS_H
diff --git a/llvm/lib/CAS/CMakeLists.txt b/llvm/lib/CAS/CMakeLists.txt
new file mode 100644
index 0000000000000..f3d2b41c704bc
--- /dev/null
+++ b/llvm/lib/CAS/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_llvm_component_library(LLVMCAS
+  ActionCache.cpp
+  ActionCaches.cpp
+  BuiltinCAS.cpp
+  InMemoryCAS.cpp
+  ObjectStore.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/CAS
+
+  LINK_COMPONENTS
+  Support
+)
diff --git a/llvm/lib/CAS/InMemoryCAS.cpp b/llvm/lib/CAS/InMemoryCAS.cpp
new file mode 100644
index 0000000000000..255b89c15c4c5
--- /dev/null
+++ b/llvm/lib/CAS/InMemoryCAS.cpp
@@ -0,0 +1,326 @@
+//===- InMemoryCAS.cpp ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "BuiltinCAS.h"
+#include "llvm/ADT/LazyAtomicPointer.h"
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/TrieRawHashMap.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/ThreadSafeAllocator.h"
+#include "llvm/Support/TrailingObjects.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+using namespace llvm::cas::builtin;
+
+namespace {
+
+class InMemoryObject;
+
+/// Index of referenced IDs (map: Hash -> InMemoryObject*). Uses
+/// LazyAtomicPointer to coordinate creation of objects.
+using InMemoryIndexT =
+    ThreadSafeTrieRawHashMap<LazyAtomicPointer<const InMemoryObject>,
+                             sizeof(HashType)>;
+
+/// Values in \a InMemoryIndexT. \a InMemoryObject's point at this to access
+/// their hash.
+using InMemoryIndexValueT = InMemoryIndexT::value_type;
+
+/// Builtin InMemory CAS that stores CAS object in the memory.
+class InMemoryObject {
+public:
+  enum class Kind {
+    /// Node with refs and data.
+    RefNode,
+
+    /// Node with refs and data co-allocated.
+    InlineNode,
+
+    Max = InlineNode,
+  };
+
+  Kind getKind() const { return IndexAndKind.getInt(); }
+  const InMemoryIndexValueT &getIndex() const {
+    assert(IndexAndKind.getPointer());
+    return *IndexAndKind.getPointer();
+  }
+
+  ArrayRef<uint8_t> getHash() const { return getIndex().Hash; }
+
+  InMemoryObject() = delete;
+  InMemoryObject(InMemoryObject &&) = delete;
+  InMemoryObject(const InMemoryObject &) = delete;
+
+protected:
+  InMemoryObject(Kind K, const InMemoryIndexValueT &I) : IndexAndKind(&I, K) {}
+
+private:
+  enum Counts : int {
+    NumKindBits = 2,
+  };
+  PointerIntPair<const InMemoryIndexValueT *, NumKindBits, Kind> IndexAndKind;
+  static_assert((1U << NumKindBits) <= alignof(InMemoryIndexValueT),
+                "Kind will clobber pointer");
+  static_assert(((int)Kind::Max >> NumKindBits) == 0, "Kind will be truncated");
+
+public:
+  ArrayRef<char> getData() const;
+
+  ArrayRef<const InMemoryObject *> getRefs() const;
+};
+
+class InMemoryRefObject final : public InMemoryObject {
+public:
+  static constexpr Kind KindValue = Kind::RefNode;
+  static bool classof(const InMemoryObject *O) {
+    return O->getKind() == KindValue;
+  }
+
+  ArrayRef<const InMemoryObject *> getRefsImpl() const { return Refs; }
+  ArrayRef<const InMemoryObject *> getRefs() const { return Refs; }
+  ArrayRef<char> getDataImpl() const { return Data; }
+  ArrayRef<char> getData() const { return Data; }
+
+  static InMemoryRefObject &create(function_ref<void *(size_t Size)> Allocate,
+                                   const InMemoryIndexValueT &I,
+                                   ArrayRef<const InMemoryObject *> Refs,
+                                   ArrayRef<char> Data) {
+    void *Mem = Allocate(sizeof(InMemoryRefObject));
+    return *new (Mem) InMemoryRefObject(I, Refs, Data);
+  }
+
+private:
+  InMemoryRefObject(const InMemoryIndexValueT &I,
+                    ArrayRef<const InMemoryObject *> Refs, ArrayRef<char> Data)
+      : InMemoryObject(KindValue, I), Refs(Refs), Data(Data) {
+    assert(isAddrAligned(Align(8), this) && "Expected 8-byte alignment");
+    assert(isAddrAligned(Align(8), Data.data()) && "Expected 8-byte alignment");
+    assert(*Data.end() == 0 && "Expected null-termination");
+  }
+
+  ArrayRef<const InMemoryObject *> Refs;
+  ArrayRef<char> Data;
+};
+
+class InMemoryInlineObject final
+    : public InMemoryObject,
+      public TrailingObjects<InMemoryInlineObject, const InMemoryObject *,
+                             char> {
+public:
+  static constexpr Kind KindValue = Kind::InlineNode;
+  static bool classof(const InMemoryObject *O) {
+    return O->getKind() == KindValue;
+  }
+
+  ArrayRef<const InMemoryObject *> getRefs() const { return getRefsImpl(); }
+  ArrayRef<const InMemoryObject *> getRefsImpl() const {
+    return ArrayRef(getTrailingObjects<const InMemoryObject *>(), NumRefs);
+  }
+
+  ArrayRef<char> getData() const { return getDataImpl(); }
+  ArrayRef<char> getDataImpl() const {
+    return ArrayRef(getTrailingObjects<char>(), DataSize);
+  }
+
+  static InMemoryInlineObject &
+  create(function_ref<void *(size_t Size)> Allocate,
+         const InMemoryIndexValueT &I, ArrayRef<const InMemoryObject *> Refs,
+         ArrayRef<char> Data) {
+    void *Mem = Allocate(sizeof(InMemoryInlineObject) +
+                         sizeof(uintptr_t) * Refs.size() + Data.size() + 1);
+    return *new (Mem) InMemoryInlineObject(I, Refs, Data);
+  }
+
+  size_t numTrailingObjects(OverloadToken<const InMemoryObject *>) const {
+    return NumRefs;
+  }
+
+private:
+  InMemoryInlineObject(const InMemoryIndexValueT &I,
+                       ArrayRef<const InMemoryObject *> Refs,
+                       ArrayRef<char> Data)
+      : InMemoryObject(KindValue, I), NumRefs(Refs.size()),
+        DataSize(Data.size()) {
+    auto *BeginRefs = reinterpret_cast<const InMemoryObject **>(this + 1);
+    llvm::copy(Refs, BeginRefs);
+    auto *BeginData = reinterpret_cast<char *>(BeginRefs + NumRefs);
+    llvm::copy(Data, BeginData);
+    BeginData[Data.size()] = 0;
+  }
+  uint32_t NumRefs;
+  uint32_t DataSize;
+};
+
+/// In-memory CAS database and action cache (the latter should be separated).
+class InMemoryCAS : public BuiltinCAS {
+public:
+  Expected<ObjectRef> storeImpl(ArrayRef<uint8_t> ComputedHash,
+                                ArrayRef<ObjectRef> Refs,
+                                ArrayRef<char> Data) final;
+
+  Expected<ObjectRef>
+  storeFromNullTerminatedRegion(ArrayRef<uint8_t> ComputedHash,
+                                sys::fs::mapped_file_region Map) override;
+
+  CASID getID(const InMemoryIndexValueT &I) const {
+    StringRef Hash = toStringRef(I.Hash);
+    return CASID::create(&getContext(), Hash);
+  }
+  CASID getID(const InMemoryObject &O) const { return getID(O.getIndex()); }
+
+  ObjectHandle getObjectHandle(const InMemoryObject &Node) const {
+    assert(!(reinterpret_cast<uintptr_t>(&Node) & 0x1ULL));
+    return makeObjectHandle(reinterpret_cast<uintptr_t>(&Node));
+  }
+
+  Expected<std::optional<ObjectHandle>> loadIfExists(ObjectRef Ref) override {
+    return getObjectHandle(asInMemoryObject(Ref));
+  }
+
+  InMemoryIndexValueT &indexHash(ArrayRef<uint8_t> Hash) {
+    return *Index.insertLazy(
+        Hash, [](auto ValueConstructor) { ValueConstructor.emplace(nullptr); });
+  }
+
+  /// TODO: Consider callers to actually do an insert and to return a handle to
+  /// the slot in the trie.
+  const InMemoryObject *getInMemoryObject(CASID ID) const {
+    assert(ID.getContext().getHashSchemaIdentifier() ==
+               getContext().getHashSchemaIdentifier() &&
+           "Expected ID from same hash schema");
+    if (InMemoryIndexT::const_pointer P = Index.find(ID.getHash()))
+      return P->Data;
+    return nullptr;
+  }
+
+  const InMemoryObject &getInMemoryObject(ObjectHandle OH) const {
+    return *reinterpret_cast<const InMemoryObject *>(
+        (uintptr_t)OH.getInternalRef(*this));
+  }
+
+  const InMemoryObject &asInMemoryObject(ReferenceBase Ref) const {
+    uintptr_t P = Ref.getInternalRef(*this);
+    return *reinterpret_cast<const InMemoryObject *>(P);
+  }
+  ObjectRef toReference(const InMemoryObject &O) const {
+    return makeObjectRef(reinterpret_cast<uintptr_t>(&O));
+  }
+
+  CASID getID(ObjectRef Ref) const final { return getIDImpl(Ref); }
+  CASID getIDImpl(ReferenceBase Ref) const {
+    return getID(asInMemoryObject(Ref));
+  }
+
+  std::optional<ObjectRef> getReference(const CASID &ID) const final {
+    if (const InMemoryObject *Object = getInMemoryObject(ID))
+      return toReference(*Object);
+    return std::nullopt;
+  }
+
+  Expected<bool> isMaterialized(ObjectRef Ref) const final { return true; }
+
+  ArrayRef<char> getDataConst(ObjectHandle Node) const final {
+    return cast<InMemoryObject>(asInMemoryObject(Node)).getData();
+  }
+
+  InMemoryCAS() = default;
+
+private:
+  size_t getNumRefs(ObjectHandle Node) const final {
+    return getInMemoryObject(Node).getRefs().size();
+  }
+  ObjectRef readRef(ObjectHandle Node, size_t I) const final {
+    return toReference(*getInMemoryObject(Node).getRefs()[I]);
+  }
+  Error forEachRef(ObjectHandle Node,
+                   function_ref<Error(ObjectRef)> Callback) const final;
+
+  /// Index of referenced IDs (map: Hash -> InMemoryObject*). Mapped to nullptr
+  /// as a convenient way to store hashes.
+  ///
+  /// - Insert nullptr on lookups.
+  /// - InMemoryObject points back to here.
+  InMemoryIndexT Index;
+
+  ThreadSafeAllocator<BumpPtrAllocator> Objects;
+  ThreadSafeAllocator<SpecificBumpPtrAllocator<sys::fs::mapped_file_region>>
+      MemoryMaps;
+};
+
+} // end anonymous namespace
+
+ArrayRef<char> InMemoryObject::getData() const {
+  if (auto *Derived = dyn_cast<InMemoryRefObject>(this))
+    return Derived->getDataImpl();
+  return cast<InMemoryInlineObject>(this)->getDataImpl();
+}
+
+ArrayRef<const InMemoryObject *> InMemoryObject::getRefs() const {
+  if (auto *Derived = dyn_cast<InMemoryRefObject>(this))
+    return Derived->getRefsImpl();
+  return cast<InMemoryInlineObject>(this)->getRefsImpl();
+}
+
+Expected<ObjectRef>
+InMemoryCAS::storeFromNullTerminatedRegion(ArrayRef<uint8_t> ComputedHash,
+                                           sys::fs::mapped_file_region Map) {
+  // Look up the hash in the index, initializing to nullptr if it's new.
+  ArrayRef<char> Data(Map.data(), Map.size());
+  auto &I = indexHash(ComputedHash);
+
+  // Load or generate.
+  auto Allocator = [&](size_t Size) -> void * {
+    return Objects.Allocate(Size, alignof(InMemoryObject));
+  };
+  auto Generator = [&]() -> const InMemoryObject * {
+    return &InMemoryRefObject::create(Allocator, I, {}, Data);
+  };
+  const InMemoryObject &Node =
+      cast<InMemoryObject>(I.Data.loadOrGenerate(Generator));
+
+  // Save Map if the winning node uses it.
+  if (auto *RefNode = dyn_cast<InMemoryRefObject>(&Node))
+    if (RefNode->getData().data() == Map.data())
+      new (MemoryMaps.Allocate(1)) sys::fs::mapped_file_region(std::move(Map));
+
+  return toReference(Node);
+}
+
+Expected<ObjectRef> InMemoryCAS::storeImpl(ArrayRef<uint8_t> ComputedHash,
+                                           ArrayRef<ObjectRef> Refs,
+                                           ArrayRef<char> Data) {
+  // Look up the hash in the index, initializing to nullptr if it's new.
+  auto &I = indexHash(ComputedHash);
+
+  // Create the node.
+  SmallVector<const InMemoryObject *> InternalRefs;
+  for (ObjectRef Ref : Refs)
+    InternalRefs.push_back(&asInMemoryObject(Ref));
+  auto Allocator = [&](size_t Size) -> void * {
+    return Objects.Allocate(Size, alignof(InMemoryObject));
+  };
+  auto Generator = [&]() -> const InMemoryObject * {
+    return &InMemoryInlineObject::create(Allocator, I, InternalRefs, Data);
+  };
+  return toReference(cast<InMemoryObject>(I.Data.loadOrGenerate(Generator)));
+}
+
+Error InMemoryCAS::forEachRef(ObjectHandle Handle,
+                              function_ref<Error(ObjectRef)> Callback) const {
+  auto &Node = getInMemoryObject(Handle);
+  for (const InMemoryObject *Ref : Node.getRefs())
+    if (Error E = Callback(toReference(*Ref)))
+      return E;
+  return Error::success();
+}
+
+std::unique_ptr<ObjectStore> cas::createInMemoryCAS() {
+  return std::make_unique<InMemoryCAS>();
+}
diff --git a/llvm/lib/CAS/ObjectStore.cpp b/llvm/lib/CAS/ObjectStore.cpp
new file mode 100644
index 0000000000000..e0be50bbe013a
--- /dev/null
+++ b/llvm/lib/CAS/ObjectStore.cpp
@@ -0,0 +1,162 @@
+//===- ObjectStore.cpp ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <optional>
+
+using namespace llvm;
+using namespace llvm::cas;
+
+void CASContext::anchor() {}
+void ObjectStore::anchor() {}
+
+LLVM_DUMP_METHOD void CASID::dump() const { print(dbgs()); }
+LLVM_DUMP_METHOD void ObjectRef::dump() const { print(dbgs()); }
+LLVM_DUMP_METHOD void ObjectHandle::dump() const { print(dbgs()); }
+
+std::string CASID::toString() const {
+  std::string S;
+  raw_string_ostream(S) << *this;
+  return S;
+}
+
+static void printReferenceBase(raw_ostream &OS, StringRef Kind,
+                               uint64_t InternalRef, std::optional<CASID> ID) {
+  OS << Kind << "=" << InternalRef;
+  if (ID)
+    OS << "[" << *ID << "]";
+}
+
+void ReferenceBase::print(raw_ostream &OS, const ObjectHandle &This) const {
+  assert(this == &This);
+  printReferenceBase(OS, "object-handle", InternalRef, std::nullopt);
+}
+
+void ReferenceBase::print(raw_ostream &OS, const ObjectRef &This) const {
+  assert(this == &This);
+
+  std::optional<CASID> ID;
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+  if (CAS)
+    ID = CAS->getID(This);
+#endif
+  printReferenceBase(OS, "object-ref", InternalRef, ID);
+}
+
+Expected<ObjectHandle> ObjectStore::load(ObjectRef Ref) {
+  std::optional<ObjectHandle> Handle;
+  if (Error E = loadIfExists(Ref).moveInto(Handle))
+    return std::move(E);
+  if (!Handle)
+    return createStringError(errc::invalid_argument,
+                             "missing object '" + getID(Ref).toString() + "'");
+  return *Handle;
+}
+
+std::unique_ptr<MemoryBuffer>
+ObjectStore::getMemoryBuffer(ObjectHandle Node, StringRef Name,
+                             bool RequiresNullTerminator) {
+  return MemoryBuffer::getMemBuffer(
+      toStringRef(getData(Node, RequiresNullTerminator)), Name,
+      RequiresNullTerminator);
+}
+
+void ObjectStore::readRefs(ObjectHandle Node,
+                           SmallVectorImpl<ObjectRef> &Refs) const {
+  consumeError(forEachRef(Node, [&Refs](ObjectRef Ref) -> Error {
+    Refs.push_back(Ref);
+    return Error::success();
+  }));
+}
+
+Expected<ObjectProxy> ObjectStore::getProxy(const CASID &ID) {
+  std::optional<ObjectRef> Ref = getReference(ID);
+  if (!Ref)
+    return createUnknownObjectError(ID);
+
+  return getProxy(*Ref);
+}
+
+Expected<ObjectProxy> ObjectStore::getProxy(ObjectRef Ref) {
+  std::optional<ObjectHandle> H;
+  if (Error E = load(Ref).moveInto(H))
+    return std::move(E);
+
+  return ObjectProxy::load(*this, Ref, *H);
+}
+
+Expected<std::optional<ObjectProxy>>
+ObjectStore::getProxyIfExists(ObjectRef Ref) {
+  std::optional<ObjectHandle> H;
+  if (Error E = loadIfExists(Ref).moveInto(H))
+    return std::move(E);
+  if (!H)
+    return std::nullopt;
+  return ObjectProxy::load(*this, Ref, *H);
+}
+
+Error ObjectStore::createUnknownObjectError(const CASID &ID) {
+  return createStringError(std::make_error_code(std::errc::invalid_argument),
+                           "unknown object '" + ID.toString() + "'");
+}
+
+Expected<ObjectProxy> ObjectStore::createProxy(ArrayRef<ObjectRef> Refs,
+                                               StringRef Data) {
+  Expected<ObjectRef> Ref = store(Refs, arrayRefFromStringRef<char>(Data));
+  if (!Ref)
+    return Ref.takeError();
+  return getProxy(*Ref);
+}
+
+Expected<ObjectRef>
+ObjectStore::storeFromOpenFileImpl(sys::fs::file_t FD,
+                                   std::optional<sys::fs::file_status> Status) {
+  // TODO: For the on-disk CAS implementation use cloning to store it as a
+  // standalone file if the file-system supports it and the file is large.
+  uint64_t Size = Status ? Status->getSize() : -1;
+  auto Buffer = MemoryBuffer::getOpenFile(FD, /*Filename=*/"", Size);
+  if (!Buffer)
+    return errorCodeToError(Buffer.getError());
+
+  return store({}, arrayRefFromStringRef<char>((*Buffer)->getBuffer()));
+}
+
+Error ObjectStore::validateTree(ObjectRef Root) {
+  SmallDenseSet<ObjectRef> ValidatedRefs;
+  SmallVector<ObjectRef, 16> RefsToValidate;
+  RefsToValidate.push_back(Root);
+
+  while (!RefsToValidate.empty()) {
+    ObjectRef Ref = RefsToValidate.pop_back_val();
+    auto [I, Inserted] = ValidatedRefs.insert(Ref);
+    if (!Inserted)
+      continue; // already validated.
+    if (Error E = validate(getID(Ref)))
+      return E;
+    Expected<ObjectHandle> Obj = load(Ref);
+    if (!Obj)
+      return Obj.takeError();
+    if (Error E = forEachRef(*Obj, [&RefsToValidate](ObjectRef R) -> Error {
+          RefsToValidate.push_back(R);
+          return Error::success();
+        }))
+      return E;
+  }
+  return Error::success();
+}
+
+std::unique_ptr<MemoryBuffer>
+ObjectProxy::getMemoryBuffer(StringRef Name,
+                             bool RequiresNullTerminator) const {
+  return CAS->getMemoryBuffer(H, Name, RequiresNullTerminator);
+}
diff --git a/llvm/lib/CGData/CodeGenData.cpp b/llvm/lib/CGData/CodeGenData.cpp
index cd012342e1958..b4f08c3d13b0d 100644
--- a/llvm/lib/CGData/CodeGenData.cpp
+++ b/llvm/lib/CGData/CodeGenData.cpp
@@ -186,7 +186,7 @@ Expected<Header> Header::readFromBuffer(const unsigned char *Curr) {
     return make_error<CGDataError>(cgdata_error::unsupported_version);
   H.DataKind = endian::readNext<uint32_t, endianness::little, unaligned>(Curr);
 
-  static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version3,
+  static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version4,
                 "Please update the offset computation below if a new field has "
                 "been added to the header.");
   H.OutlinedHashTreeOffset =
diff --git a/llvm/lib/CGData/CodeGenDataReader.cpp b/llvm/lib/CGData/CodeGenDataReader.cpp
index 0ab35499c8986..fc59be8df525a 100644
--- a/llvm/lib/CGData/CodeGenDataReader.cpp
+++ b/llvm/lib/CGData/CodeGenDataReader.cpp
@@ -26,6 +26,12 @@ static cl::opt<bool> IndexedCodeGenDataReadFunctionMapNames(
              "disabled to save memory and time for final consumption of the "
              "indexed CodeGenData in production."));
 
+cl::opt<bool> IndexedCodeGenDataLazyLoading(
+    "indexed-codegen-data-lazy-loading", cl::init(false), cl::Hidden,
+    cl::desc(
+        "Lazily load indexed CodeGenData. Enable to save memory and time "
+        "for final consumption of the indexed CodeGenData in production."));
+
 namespace llvm {
 
 static Expected<std::unique_ptr<MemoryBuffer>>
@@ -109,11 +115,20 @@ Error IndexedCodeGenDataReader::read() {
       return error(cgdata_error::eof);
     HashTreeRecord.deserialize(Ptr);
   }
+
+  // TODO: lazy loading support for outlined hash tree.
+  std::shared_ptr<MemoryBuffer> SharedDataBuffer = std::move(DataBuffer);
   if (hasStableFunctionMap()) {
     const unsigned char *Ptr = Start + Header.StableFunctionMapOffset;
     if (Ptr >= End)
       return error(cgdata_error::eof);
-    FunctionMapRecord.deserialize(Ptr, IndexedCodeGenDataReadFunctionMapNames);
+    FunctionMapRecord.setReadStableFunctionMapNames(
+        IndexedCodeGenDataReadFunctionMapNames);
+    if (IndexedCodeGenDataLazyLoading)
+      FunctionMapRecord.lazyDeserialize(SharedDataBuffer,
+                                        Header.StableFunctionMapOffset);
+    else
+      FunctionMapRecord.deserialize(Ptr);
   }
 
   return success();
diff --git a/llvm/lib/CGData/StableFunctionMap.cpp b/llvm/lib/CGData/StableFunctionMap.cpp
index 87f1e76afb60b..46e04bd850104 100644
--- a/llvm/lib/CGData/StableFunctionMap.cpp
+++ b/llvm/lib/CGData/StableFunctionMap.cpp
@@ -15,6 +15,7 @@
 
 #include "llvm/CGData/StableFunctionMap.h"
 #include "llvm/ADT/SmallSet.h"
+#include "llvm/CGData/StableFunctionMapRecord.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 
@@ -93,9 +94,10 @@ void StableFunctionMap::insert(const StableFunction &Func) {
 
 void StableFunctionMap::merge(const StableFunctionMap &OtherMap) {
   assert(!Finalized && "Cannot merge after finalization");
+  deserializeLazyLoadingEntries();
   for (auto &[Hash, Funcs] : OtherMap.HashToFuncs) {
-    auto &ThisFuncs = HashToFuncs[Hash];
-    for (auto &Func : Funcs) {
+    auto &ThisFuncs = HashToFuncs[Hash].Entries;
+    for (auto &Func : Funcs.Entries) {
       auto FuncNameId =
           getIdOrCreateForName(*OtherMap.getNameForId(Func->FunctionNameId));
       auto ModuleNameId =
@@ -114,25 +116,63 @@ size_t StableFunctionMap::size(SizeType Type) const {
   case UniqueHashCount:
     return HashToFuncs.size();
   case TotalFunctionCount: {
+    deserializeLazyLoadingEntries();
     size_t Count = 0;
     for (auto &Funcs : HashToFuncs)
-      Count += Funcs.second.size();
+      Count += Funcs.second.Entries.size();
     return Count;
   }
   case MergeableFunctionCount: {
+    deserializeLazyLoadingEntries();
     size_t Count = 0;
     for (auto &[Hash, Funcs] : HashToFuncs)
-      if (Funcs.size() >= 2)
-        Count += Funcs.size();
+      if (Funcs.Entries.size() >= 2)
+        Count += Funcs.Entries.size();
     return Count;
   }
   }
   llvm_unreachable("Unhandled size type");
 }
 
+const StableFunctionMap::StableFunctionEntries &
+StableFunctionMap::at(HashFuncsMapType::key_type FunctionHash) const {
+  auto It = HashToFuncs.find(FunctionHash);
+  if (isLazilyLoaded())
+    deserializeLazyLoadingEntry(It);
+  return It->second.Entries;
+}
+
+void StableFunctionMap::deserializeLazyLoadingEntry(
+    HashFuncsMapType::iterator It) const {
+  assert(isLazilyLoaded() && "Cannot deserialize non-lazily-loaded map");
+  auto &[Hash, Storage] = *It;
+  std::call_once(Storage.LazyLoadFlag,
+                 [this, HashArg = Hash, &StorageArg = Storage]() {
+                   for (auto Offset : StorageArg.Offsets)
+                     StableFunctionMapRecord::deserializeEntry(
+                         reinterpret_cast<const unsigned char *>(Offset),
+                         HashArg, const_cast<StableFunctionMap *>(this));
+                 });
+}
+
+void StableFunctionMap::deserializeLazyLoadingEntries() const {
+  if (!isLazilyLoaded())
+    return;
+  for (auto It = HashToFuncs.begin(); It != HashToFuncs.end(); ++It)
+    deserializeLazyLoadingEntry(It);
+}
+
+const StableFunctionMap::HashFuncsMapType &
+StableFunctionMap::getFunctionMap() const {
+  // Ensure all entries are deserialized before returning the raw map.
+  if (isLazilyLoaded())
+    deserializeLazyLoadingEntries();
+  return HashToFuncs;
+}
+
 using ParamLocs = SmallVector<IndexPair>;
-static void removeIdenticalIndexPair(
-    SmallVector<std::unique_ptr<StableFunctionMap::StableFunctionEntry>> &SFS) {
+static void
+removeIdenticalIndexPair(StableFunctionMap::StableFunctionEntries &SFS) {
   auto &RSF = SFS[0];
   unsigned StableFunctionCount = SFS.size();
 
@@ -159,9 +199,7 @@ static void removeIdenticalIndexPair(
       SF->IndexOperandHashMap->erase(Pair);
 }
 
-static bool isProfitable(
-    const SmallVector<std::unique_ptr<StableFunctionMap::StableFunctionEntry>>
-        &SFS) {
+static bool isProfitable(const StableFunctionMap::StableFunctionEntries &SFS) {
   unsigned StableFunctionCount = SFS.size();
   if (StableFunctionCount < GlobalMergingMinMerges)
     return false;
@@ -202,8 +240,11 @@ static bool isProfitable(
 }
 
 void StableFunctionMap::finalize(bool SkipTrim) {
+  deserializeLazyLoadingEntries();
+  SmallVector<HashFuncsMapType::iterator> ToDelete;
   for (auto It = HashToFuncs.begin(); It != HashToFuncs.end(); ++It) {
-    auto &[StableHash, SFS] = *It;
+    auto &[StableHash, Storage] = *It;
+    auto &SFS = Storage.Entries;
 
     // Group stable functions by ModuleIdentifier.
     llvm::stable_sort(SFS, [&](const std::unique_ptr<StableFunctionEntry> &L,
@@ -236,7 +277,7 @@ void StableFunctionMap::finalize(bool SkipTrim) {
       }
     }
     if (Invalid) {
-      HashToFuncs.erase(It);
+      ToDelete.push_back(It);
       continue;
     }
 
@@ -248,8 +289,10 @@ void StableFunctionMap::finalize(bool SkipTrim) {
     removeIdenticalIndexPair(SFS);
 
     if (!isProfitable(SFS))
-      HashToFuncs.erase(It);
+      ToDelete.push_back(It);
   }
+  for (auto It : ToDelete)
+    HashToFuncs.erase(It);
 
   Finalized = true;
 }
diff --git a/llvm/lib/CGData/StableFunctionMapRecord.cpp b/llvm/lib/CGData/StableFunctionMapRecord.cpp
index 423e068023088..e585995ba6a31 100644
--- a/llvm/lib/CGData/StableFunctionMapRecord.cpp
+++ b/llvm/lib/CGData/StableFunctionMapRecord.cpp
@@ -53,7 +53,7 @@ static SmallVector<const StableFunctionMap::StableFunctionEntry *>
 getStableFunctionEntries(const StableFunctionMap &SFM) {
   SmallVector<const StableFunctionMap::StableFunctionEntry *> FuncEntries;
   for (const auto &P : SFM.getFunctionMap())
-    for (auto &Func : P.second)
+    for (auto &Func : P.second.Entries)
       FuncEntries.emplace_back(Func.get());
 
   llvm::stable_sort(
@@ -107,14 +107,25 @@ void StableFunctionMapRecord::serialize(
   // Write StableFunctionEntries whose pointers are sorted.
   auto FuncEntries = getStableFunctionEntries(*FunctionMap);
   Writer.write<uint32_t>(FuncEntries.size());
-
-  for (const auto *FuncRef : FuncEntries) {
+  for (const auto *FuncRef : FuncEntries)
     Writer.write<stable_hash>(FuncRef->Hash);
+  std::vector<uint64_t> IndexOperandHashesOffsets;
+  IndexOperandHashesOffsets.reserve(FuncEntries.size());
+  for (const auto *FuncRef : FuncEntries) {
     Writer.write<uint32_t>(FuncRef->FunctionNameId);
     Writer.write<uint32_t>(FuncRef->ModuleNameId);
     Writer.write<uint32_t>(FuncRef->InstCount);
-
+    const uint64_t Offset = Writer.OS.tell();
+    IndexOperandHashesOffsets.push_back(Offset);
+    Writer.write<uint64_t>(0);
+  }
+  const uint64_t IndexOperandHashesByteSizeOffset = Writer.OS.tell();
+  Writer.write<uint64_t>(0);
+  for (size_t I = 0; I < FuncEntries.size(); ++I) {
+    const uint64_t Offset = Writer.OS.tell() - IndexOperandHashesOffsets[I];
+    PatchItems.emplace_back(IndexOperandHashesOffsets[I], &Offset, 1);
     // Emit IndexOperandHashes sorted from IndexOperandHashMap.
+    const auto *FuncRef = FuncEntries[I];
     IndexOperandHashVecType IndexOperandHashes =
         getStableIndexOperandHashes(FuncRef);
     Writer.write<uint32_t>(IndexOperandHashes.size());
@@ -124,10 +135,62 @@ void StableFunctionMapRecord::serialize(
       Writer.write<stable_hash>(IndexOperandHash.second);
     }
   }
+  // Write the total size of IndexOperandHashes.
+  const uint64_t IndexOperandHashesByteSize =
+      Writer.OS.tell() - IndexOperandHashesByteSizeOffset - sizeof(uint64_t);
+  PatchItems.emplace_back(IndexOperandHashesByteSizeOffset,
+                          &IndexOperandHashesByteSize, 1);
+}
+
+void StableFunctionMapRecord::deserializeEntry(const unsigned char *Ptr,
+                                               stable_hash Hash,
+                                               StableFunctionMap *FunctionMap) {
+  auto FunctionNameId =
+      endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
+  if (FunctionMap->ReadStableFunctionMapNames)
+    assert(FunctionMap->getNameForId(FunctionNameId) &&
+           "FunctionNameId out of range");
+  auto ModuleNameId =
+      endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
+  if (FunctionMap->ReadStableFunctionMapNames)
+    assert(FunctionMap->getNameForId(ModuleNameId) &&
+           "ModuleNameId out of range");
+  auto InstCount =
+      endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
+
+  // Read IndexOperandHashes to build IndexOperandHashMap
+  auto CurrentPosition = reinterpret_cast<uintptr_t>(Ptr);
+  auto IndexOperandHashesOffset =
+      endian::readNext<uint64_t, endianness::little, unaligned>(Ptr);
+  auto *IndexOperandHashesPtr = reinterpret_cast<const unsigned char *>(
+      CurrentPosition + IndexOperandHashesOffset);
+  auto NumIndexOperandHashes =
+      endian::readNext<uint32_t, endianness::little, unaligned>(
+          IndexOperandHashesPtr);
+  auto IndexOperandHashMap = std::make_unique<IndexOperandHashMapType>();
+  for (unsigned J = 0; J < NumIndexOperandHashes; ++J) {
+    auto InstIndex = endian::readNext<uint32_t, endianness::little, unaligned>(
+        IndexOperandHashesPtr);
+    auto OpndIndex = endian::readNext<uint32_t, endianness::little, unaligned>(
+        IndexOperandHashesPtr);
+    auto OpndHash =
+        endian::readNext<stable_hash, endianness::little, unaligned>(
+            IndexOperandHashesPtr);
+    assert(InstIndex < InstCount && "InstIndex out of range");
+
+    IndexOperandHashMap->try_emplace({InstIndex, OpndIndex}, OpndHash);
+  }
+
+  // Insert a new StableFunctionEntry into the map.
+  auto FuncEntry = std::make_unique<StableFunctionMap::StableFunctionEntry>(
+      Hash, FunctionNameId, ModuleNameId, InstCount,
+      std::move(IndexOperandHashMap));
+
+  FunctionMap->insert(std::move(FuncEntry));
 }
 
 void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr,
-                                          bool ReadStableFunctionMapNames) {
+                                          bool Lazy) {
   // Assert that Ptr is 4-byte aligned
   assert(((uintptr_t)Ptr % 4) == 0);
   // Read Names.
@@ -139,7 +202,7 @@ void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr,
   const auto NamesByteSize =
       endian::readNext<uint64_t, endianness::little, unaligned>(Ptr);
   const auto NamesOffset = reinterpret_cast<uintptr_t>(Ptr);
-  if (ReadStableFunctionMapNames) {
+  if (FunctionMap->ReadStableFunctionMapNames) {
     for (unsigned I = 0; I < NumNames; ++I) {
       StringRef Name(reinterpret_cast<const char *>(Ptr));
       Ptr += Name.size() + 1;
@@ -157,47 +220,51 @@ void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr,
   // Read StableFunctionEntries.
   auto NumFuncs =
       endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
+  auto FixedSizeFieldsOffset =
+      reinterpret_cast<uintptr_t>(Ptr) + NumFuncs * sizeof(stable_hash);
+  constexpr uint32_t FixedSizeFieldsSizePerEntry =
+      // FunctionNameId
+      sizeof(uint32_t) +
+      // ModuleNameId
+      sizeof(uint32_t) +
+      // InstCount
+      sizeof(uint32_t) +
+      // Relative offset to IndexOperandHashes
+      sizeof(uint64_t);
   for (unsigned I = 0; I < NumFuncs; ++I) {
     auto Hash =
         endian::readNext<stable_hash, endianness::little, unaligned>(Ptr);
-    [[maybe_unused]] auto FunctionNameId =
-        endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
-    [[maybe_unused]] auto ModuleNameId =
-        endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
-    // Only validate IDs if we've read the names
-    if (ReadStableFunctionMapNames) {
-      assert(FunctionMap->getNameForId(FunctionNameId) &&
-             "FunctionNameId out of range");
-      assert(FunctionMap->getNameForId(ModuleNameId) &&
-             "ModuleNameId out of range");
+    if (Lazy) {
+      auto It = FunctionMap->HashToFuncs.try_emplace(Hash).first;
+      StableFunctionMap::EntryStorage &Storage = It->second;
+      Storage.Offsets.push_back(FixedSizeFieldsOffset);
+    } else {
+      deserializeEntry(
+          reinterpret_cast<const unsigned char *>(FixedSizeFieldsOffset), Hash,
+          FunctionMap.get());
     }
+    FixedSizeFieldsOffset += FixedSizeFieldsSizePerEntry;
+  }
 
-    auto InstCount =
-        endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
-
-    // Read IndexOperandHashes to build IndexOperandHashMap
-    auto NumIndexOperandHashes =
-        endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
-    auto IndexOperandHashMap = std::make_unique<IndexOperandHashMapType>();
-    for (unsigned J = 0; J < NumIndexOperandHashes; ++J) {
-      auto InstIndex =
-          endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
-      auto OpndIndex =
-          endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
-      auto OpndHash =
-          endian::readNext<stable_hash, endianness::little, unaligned>(Ptr);
-      assert(InstIndex < InstCount && "InstIndex out of range");
-
-      IndexOperandHashMap->try_emplace({InstIndex, OpndIndex}, OpndHash);
-    }
+  // Update Ptr to the end of the serialized map to meet the expectation of
+  // CodeGenDataReader.
+  Ptr = reinterpret_cast<const unsigned char *>(FixedSizeFieldsOffset);
+  auto IndexOperandHashesByteSize =
+      endian::readNext<uint64_t, endianness::little, unaligned>(Ptr);
+  Ptr = reinterpret_cast<const unsigned char *>(
+      reinterpret_cast<uintptr_t>(Ptr) + IndexOperandHashesByteSize);
+}
 
-    // Insert a new StableFunctionEntry into the map.
-    auto FuncEntry = std::make_unique<StableFunctionMap::StableFunctionEntry>(
-        Hash, FunctionNameId, ModuleNameId, InstCount,
-        std::move(IndexOperandHashMap));
+void StableFunctionMapRecord::deserialize(const unsigned char *&Ptr) {
+  deserialize(Ptr, /*Lazy=*/false);
+}
 
-    FunctionMap->insert(std::move(FuncEntry));
-  }
+void StableFunctionMapRecord::lazyDeserialize(
+    std::shared_ptr<MemoryBuffer> Buffer, uint64_t Offset) {
+  const auto *Ptr = reinterpret_cast<const unsigned char *>(
+      reinterpret_cast<uintptr_t>(Buffer->getBufferStart()) + Offset);
+  deserialize(Ptr, /*Lazy=*/true);
+  FunctionMap->Buffer = std::move(Buffer);
 }
 
 void StableFunctionMapRecord::serializeYAML(yaml::Output &YOS) const {
diff --git a/llvm/lib/CMakeLists.txt b/llvm/lib/CMakeLists.txt
index a56183000f085..a9432977718c6 100644
--- a/llvm/lib/CMakeLists.txt
+++ b/llvm/lib/CMakeLists.txt
@@ -9,6 +9,7 @@ add_subdirectory(FileCheck)
 add_subdirectory(InterfaceStub)
 add_subdirectory(IRPrinter)
 add_subdirectory(IRReader)
+add_subdirectory(CAS)
 add_subdirectory(CGData)
 add_subdirectory(CodeGen)
 add_subdirectory(CodeGenTypes)
diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp
index e7b9417de8c9f..2ef96cc4400f7 100644
--- a/llvm/lib/CodeGen/Analysis.cpp
+++ b/llvm/lib/CodeGen/Analysis.cpp
@@ -69,18 +69,10 @@ unsigned llvm::ComputeLinearIndex(Type *Ty,
   return CurIndex + 1;
 }
 
-/// ComputeValueVTs - Given an LLVM IR type, compute a sequence of
-/// EVTs that represent all the individual underlying
-/// non-aggregate types that comprise it.
-///
-/// If Offsets is non-null, it points to a vector to be filled in
-/// with the in-memory offsets of each of the individual values.
-///
-void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL,
-                           Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
-                           SmallVectorImpl<EVT> *MemVTs,
-                           SmallVectorImpl<TypeSize> *Offsets,
-                           TypeSize StartingOffset) {
+void llvm::ComputeValueTypes(const DataLayout &DL, Type *Ty,
+                             SmallVectorImpl<Type *> &Types,
+                             SmallVectorImpl<TypeSize> *Offsets,
+                             TypeSize StartingOffset) {
   assert((Ty->isScalableTy() == StartingOffset.isScalable() ||
           StartingOffset.isZero()) &&
          "Offset/TypeSize mismatch!");
@@ -90,15 +82,13 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL,
     // us to support structs with scalable vectors for operations that don't
     // need offsets.
     const StructLayout *SL = Offsets ? DL.getStructLayout(STy) : nullptr;
-    for (StructType::element_iterator EB = STy->element_begin(),
-                                      EI = EB,
+    for (StructType::element_iterator EB = STy->element_begin(), EI = EB,
                                       EE = STy->element_end();
          EI != EE; ++EI) {
       // Don't compute the element offset if we didn't get a StructLayout above.
       TypeSize EltOffset =
           SL ? SL->getElementOffset(EI - EB) : TypeSize::getZero();
-      ComputeValueVTs(TLI, DL, *EI, ValueVTs, MemVTs, Offsets,
-                      StartingOffset + EltOffset);
+      ComputeValueTypes(DL, *EI, Types, Offsets, StartingOffset + EltOffset);
     }
     return;
   }
@@ -107,21 +97,39 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL,
     Type *EltTy = ATy->getElementType();
     TypeSize EltSize = DL.getTypeAllocSize(EltTy);
     for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i)
-      ComputeValueVTs(TLI, DL, EltTy, ValueVTs, MemVTs, Offsets,
-                      StartingOffset + i * EltSize);
+      ComputeValueTypes(DL, EltTy, Types, Offsets,
+                        StartingOffset + i * EltSize);
     return;
   }
   // Interpret void as zero return values.
   if (Ty->isVoidTy())
     return;
-  // Base case: we can get an EVT for this LLVM IR type.
-  ValueVTs.push_back(TLI.getValueType(DL, Ty));
-  if (MemVTs)
-    MemVTs->push_back(TLI.getMemValueType(DL, Ty));
+  Types.push_back(Ty);
   if (Offsets)
     Offsets->push_back(StartingOffset);
 }
 
+/// ComputeValueVTs - Given an LLVM IR type, compute a sequence of
+/// EVTs that represent all the individual underlying
+/// non-aggregate types that comprise it.
+///
+/// If Offsets is non-null, it points to a vector to be filled in
+/// with the in-memory offsets of each of the individual values.
+///
+void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL,
+                           Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
+                           SmallVectorImpl<EVT> *MemVTs,
+                           SmallVectorImpl<TypeSize> *Offsets,
+                           TypeSize StartingOffset) {
+  SmallVector<Type *> Types;
+  ComputeValueTypes(DL, Ty, Types, Offsets, StartingOffset);
+  for (Type *Ty : Types) {
+    ValueVTs.push_back(TLI.getValueType(DL, Ty));
+    if (MemVTs)
+      MemVTs->push_back(TLI.getMemValueType(DL, Ty));
+  }
+}
+
 void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL,
                            Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
                            SmallVectorImpl<EVT> *MemVTs,
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index c72b6e8ed5f53..23a3543e9ebec 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -3657,6 +3657,7 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV,
     break; // Error
   }
 
+  case Instruction::PtrToAddr:
   case Instruction::PtrToInt: {
     const DataLayout &DL = getDataLayout();
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index f9d7e763e8890..67f526fe91464 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -1292,12 +1292,10 @@ DwarfCompileUnit::getDwarf5OrGNULocationAtom(dwarf::LocationAtom Loc) const {
   }
 }
 
-DIE &DwarfCompileUnit::constructCallSiteEntryDIE(DIE &ScopeDIE,
-                                                 const DISubprogram *CalleeSP,
-                                                 bool IsTail,
-                                                 const MCSymbol *PCAddr,
-                                                 const MCSymbol *CallAddr,
-                                                 unsigned CallReg) {
+DIE &DwarfCompileUnit::constructCallSiteEntryDIE(
+    DIE &ScopeDIE, const DISubprogram *CalleeSP, bool IsTail,
+    const MCSymbol *PCAddr, const MCSymbol *CallAddr, unsigned CallReg,
+    DIType *AllocSiteTy) {
   // Insert a call site entry DIE within ScopeDIE.
   DIE &CallSiteDIE = createAndAddDIE(getDwarf5OrGNUTag(dwarf::DW_TAG_call_site),
                                      ScopeDIE, nullptr);
@@ -1306,7 +1304,7 @@ DIE &DwarfCompileUnit::constructCallSiteEntryDIE(DIE &ScopeDIE,
     // Indirect call.
     addAddress(CallSiteDIE, getDwarf5OrGNUAttr(dwarf::DW_AT_call_target),
                MachineLocation(CallReg));
-  } else {
+  } else if (CalleeSP) {
     DIE *CalleeDIE = getOrCreateSubprogramDIE(CalleeSP);
     assert(CalleeDIE && "Could not create DIE for call site entry origin");
     if (AddLinkageNamesToDeclCallOriginsForTuning(DD) &&
@@ -1351,6 +1349,9 @@ DIE &DwarfCompileUnit::constructCallSiteEntryDIE(DIE &ScopeDIE,
                     getDwarf5OrGNUAttr(dwarf::DW_AT_call_return_pc), PCAddr);
   }
 
+  if (AllocSiteTy)
+    addType(CallSiteDIE, AllocSiteTy, dwarf::DW_AT_LLVM_alloc_type);
+
   return CallSiteDIE;
 }
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 09be22ce35e36..c2f6ca0913818 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -289,7 +289,8 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// the \p CallReg is set to 0.
   DIE &constructCallSiteEntryDIE(DIE &ScopeDIE, const DISubprogram *CalleeSP,
                                  bool IsTail, const MCSymbol *PCAddr,
-                                 const MCSymbol *CallAddr, unsigned CallReg);
+                                 const MCSymbol *CallAddr, unsigned CallReg,
+                                 DIType *AllocSiteTy);
   /// Construct call site parameter DIEs for the \p CallSiteDIE. The \p Params
   /// were collected by the \ref collectCallSiteParameters.
   /// Note: The order of parameters does not matter, since debuggers recognize
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 5ae2d2a3958bd..c27f100775625 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -936,6 +936,8 @@ void DwarfDebug::constructCallSiteEntryDIEs(const DISubprogram &SP,
       if (MI.hasDelaySlot() && !delaySlotSupported(*&MI))
         return;
 
+      DIType *AllocSiteTy = dyn_cast_or_null<DIType>(MI.getHeapAllocMarker());
+
       // If this is a direct call, find the callee's subprogram.
       // In the case of an indirect call find the register that holds
       // the callee.
@@ -950,23 +952,23 @@ void DwarfDebug::constructCallSiteEntryDIEs(const DISubprogram &SP,
         PhysRegCalleeOperand =
             PhysRegCalleeOperand && MCOI.OperandType == MCOI::OPERAND_REGISTER;
       }
-      if (!CalleeOp.isGlobal() && !PhysRegCalleeOperand)
-        continue;
 
       unsigned CallReg = 0;
       const DISubprogram *CalleeSP = nullptr;
       const Function *CalleeDecl = nullptr;
       if (PhysRegCalleeOperand) {
-        CallReg = CalleeOp.getReg();
-        if (!CallReg)
-          continue;
-      } else {
+        CallReg = CalleeOp.getReg(); // might be zero
+      } else if (CalleeOp.isGlobal()) {
         CalleeDecl = dyn_cast<Function>(CalleeOp.getGlobal());
-        if (!CalleeDecl || !CalleeDecl->getSubprogram())
-          continue;
-        CalleeSP = CalleeDecl->getSubprogram();
+        if (CalleeDecl)
+          CalleeSP = CalleeDecl->getSubprogram(); // might be nullptr
       }
 
+      // Omit DIE if we can't tell where the call goes *and* we don't want to
+      // add metadata to it.
+      if (CalleeSP == nullptr && CallReg == 0 && AllocSiteTy == nullptr)
+        continue;
+
       // TODO: Omit call site entries for runtime calls (objc_msgSend, etc).
 
       bool IsTail = TII->isTailCall(MI);
@@ -1000,7 +1002,7 @@ void DwarfDebug::constructCallSiteEntryDIEs(const DISubprogram &SP,
                         << (IsTail ? " [IsTail]" : "") << "\n");
 
       DIE &CallSiteDIE = CU.constructCallSiteEntryDIE(
-          ScopeDIE, CalleeSP, IsTail, PCAddr, CallAddr, CallReg);
+          ScopeDIE, CalleeSP, IsTail, PCAddr, CallAddr, CallReg, AllocSiteTy);
 
       // Optionally emit call-site-param debug info.
       if (emitDebugEntryValues()) {
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 3f3d5dc90711f..278dd6560e736 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -1915,7 +1915,6 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
 
   // TODO: the "order" argument type is "int", not int32. So
   // getInt32Ty may be wrong if the arch uses e.g. 16-bit ints.
-  ConstantInt *SizeVal64 = ConstantInt::get(Type::getInt64Ty(Ctx), Size);
   assert(Ordering != AtomicOrdering::NotAtomic && "expect atomic MO");
   Constant *OrderingVal =
       ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(Ordering));
@@ -2012,7 +2011,7 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
   if (CASExpected) {
     AllocaCASExpected = AllocaBuilder.CreateAlloca(CASExpected->getType());
     AllocaCASExpected->setAlignment(AllocaAlignment);
-    Builder.CreateLifetimeStart(AllocaCASExpected, SizeVal64);
+    Builder.CreateLifetimeStart(AllocaCASExpected);
     Builder.CreateAlignedStore(CASExpected, AllocaCASExpected, AllocaAlignment);
     Args.push_back(AllocaCASExpected);
   }
@@ -2026,7 +2025,7 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
     } else {
       AllocaValue = AllocaBuilder.CreateAlloca(ValueOperand->getType());
       AllocaValue->setAlignment(AllocaAlignment);
-      Builder.CreateLifetimeStart(AllocaValue, SizeVal64);
+      Builder.CreateLifetimeStart(AllocaValue);
       Builder.CreateAlignedStore(ValueOperand, AllocaValue, AllocaAlignment);
       Args.push_back(AllocaValue);
     }
@@ -2036,7 +2035,7 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
   if (!CASExpected && HasResult && !UseSizedLibcall) {
     AllocaResult = AllocaBuilder.CreateAlloca(I->getType());
     AllocaResult->setAlignment(AllocaAlignment);
-    Builder.CreateLifetimeStart(AllocaResult, SizeVal64);
+    Builder.CreateLifetimeStart(AllocaResult);
     Args.push_back(AllocaResult);
   }
 
@@ -2069,7 +2068,7 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
 
   // And then, extract the results...
   if (ValueOperand && !UseSizedLibcall)
-    Builder.CreateLifetimeEnd(AllocaValue, SizeVal64);
+    Builder.CreateLifetimeEnd(AllocaValue);
 
   if (CASExpected) {
     // The final result from the CAS is {load of 'expected' alloca, bool result
@@ -2078,7 +2077,7 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
     Value *V = PoisonValue::get(FinalResultTy);
     Value *ExpectedOut = Builder.CreateAlignedLoad(
         CASExpected->getType(), AllocaCASExpected, AllocaAlignment);
-    Builder.CreateLifetimeEnd(AllocaCASExpected, SizeVal64);
+    Builder.CreateLifetimeEnd(AllocaCASExpected);
     V = Builder.CreateInsertValue(V, ExpectedOut, 0);
     V = Builder.CreateInsertValue(V, Result, 1);
     I->replaceAllUsesWith(V);
@@ -2089,7 +2088,7 @@ bool AtomicExpandImpl::expandAtomicOpToLibcall(
     else {
       V = Builder.CreateAlignedLoad(I->getType(), AllocaResult,
                                     AllocaAlignment);
-      Builder.CreateLifetimeEnd(AllocaResult, SizeVal64);
+      Builder.CreateLifetimeEnd(AllocaResult);
     }
     I->replaceAllUsesWith(V);
   }
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index dcfd9aad70fc5..7292bc2be0df2 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -1787,10 +1787,18 @@ bool BranchFolder::OptimizeBlock(MachineBasicBlock *MBB) {
       // below were performed for EH "FallThrough" blocks.  Therefore, even if
       // that appears not to be happening anymore, we should assume that it is
       // possible and not remove the "!FallThrough()->isEHPad" condition below.
+      //
+      // Similarly, the analyzeBranch call does not consider callbr, which also
+      // introduces the possibility of infinite rotation, as there may be
+      // multiple successors of PrevBB. Thus we check such case by
+      // FallThrough->isInlineAsmBrIndirectTarget().
+      // NOTE: Checking if PrevBB contains callbr is more precise, but much
+      // more expensive.
       MachineBasicBlock *PrevTBB = nullptr, *PrevFBB = nullptr;
       SmallVector<MachineOperand, 4> PrevCond;
-      if (FallThrough != MF.end() &&
-          !FallThrough->isEHPad() &&
+
+      if (FallThrough != MF.end() && !FallThrough->isEHPad() &&
+          !FallThrough->isInlineAsmBrIndirectTarget() &&
           !TII->analyzeBranch(PrevBB, PrevTBB, PrevFBB, PrevCond, true) &&
           PrevBB.isSuccessor(&*FallThrough)) {
         MBB->moveAfter(&MF.back());
diff --git a/llvm/lib/CodeGen/CallingConvLower.cpp b/llvm/lib/CodeGen/CallingConvLower.cpp
index b71e7812296cc..df3433199681b 100644
--- a/llvm/lib/CodeGen/CallingConvLower.cpp
+++ b/llvm/lib/CodeGen/CallingConvLower.cpp
@@ -89,7 +89,7 @@ CCState::AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins,
   for (unsigned i = 0; i != NumArgs; ++i) {
     MVT ArgVT = Ins[i].VT;
     ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
-    if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, *this))
+    if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, Ins[i].OrigTy, *this))
       report_fatal_error("unable to allocate function argument #" + Twine(i));
   }
 }
@@ -102,7 +102,7 @@ bool CCState::CheckReturn(const SmallVectorImpl<ISD::OutputArg> &Outs,
   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
     MVT VT = Outs[i].VT;
     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
-    if (Fn(i, VT, VT, CCValAssign::Full, ArgFlags, *this))
+    if (Fn(i, VT, VT, CCValAssign::Full, ArgFlags, Outs[i].OrigTy, *this))
       return false;
   }
   return true;
@@ -116,7 +116,7 @@ void CCState::AnalyzeReturn(const SmallVectorImpl<ISD::OutputArg> &Outs,
   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
     MVT VT = Outs[i].VT;
     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
-    if (Fn(i, VT, VT, CCValAssign::Full, ArgFlags, *this))
+    if (Fn(i, VT, VT, CCValAssign::Full, ArgFlags, Outs[i].OrigTy, *this))
       report_fatal_error("unable to allocate function return #" + Twine(i));
   }
 }
@@ -129,7 +129,8 @@ void CCState::AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
   for (unsigned i = 0; i != NumOps; ++i) {
     MVT ArgVT = Outs[i].VT;
     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
-    if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, *this)) {
+    if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, Outs[i].OrigTy,
+           *this)) {
 #ifndef NDEBUG
       dbgs() << "Call operand #" << i << " has unhandled type "
              << ArgVT << '\n';
@@ -142,12 +143,13 @@ void CCState::AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
 /// Same as above except it takes vectors of types and argument flags.
 void CCState::AnalyzeCallOperands(SmallVectorImpl<MVT> &ArgVTs,
                                   SmallVectorImpl<ISD::ArgFlagsTy> &Flags,
+                                  SmallVectorImpl<Type *> &OrigTys,
                                   CCAssignFn Fn) {
   unsigned NumOps = ArgVTs.size();
   for (unsigned i = 0; i != NumOps; ++i) {
     MVT ArgVT = ArgVTs[i];
     ISD::ArgFlagsTy ArgFlags = Flags[i];
-    if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, *this)) {
+    if (Fn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, OrigTys[i], *this)) {
 #ifndef NDEBUG
       dbgs() << "Call operand #" << i << " has unhandled type "
              << ArgVT << '\n';
@@ -164,7 +166,7 @@ void CCState::AnalyzeCallResult(const SmallVectorImpl<ISD::InputArg> &Ins,
   for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
     MVT VT = Ins[i].VT;
     ISD::ArgFlagsTy Flags = Ins[i].Flags;
-    if (Fn(i, VT, VT, CCValAssign::Full, Flags, *this)) {
+    if (Fn(i, VT, VT, CCValAssign::Full, Flags, Ins[i].OrigTy, *this)) {
 #ifndef NDEBUG
       dbgs() << "Call result #" << i << " has unhandled type "
              << VT << '\n';
@@ -175,8 +177,8 @@ void CCState::AnalyzeCallResult(const SmallVectorImpl<ISD::InputArg> &Ins,
 }
 
 /// Same as above except it's specialized for calls that produce a single value.
-void CCState::AnalyzeCallResult(MVT VT, CCAssignFn Fn) {
-  if (Fn(0, VT, VT, CCValAssign::Full, ISD::ArgFlagsTy(), *this)) {
+void CCState::AnalyzeCallResult(MVT VT, Type *OrigTy, CCAssignFn Fn) {
+  if (Fn(0, VT, VT, CCValAssign::Full, ISD::ArgFlagsTy(), OrigTy, *this)) {
 #ifndef NDEBUG
     dbgs() << "Call result has unhandled type "
            << VT << '\n';
@@ -213,7 +215,8 @@ void CCState::getRemainingRegParmsForType(SmallVectorImpl<MCRegister> &Regs,
   // location in memory.
   bool HaveRegParm;
   do {
-    if (Fn(0, VT, VT, CCValAssign::Full, Flags, *this)) {
+    Type *OrigTy = EVT(VT).getTypeForEVT(Context);
+    if (Fn(0, VT, VT, CCValAssign::Full, Flags, OrigTy, *this)) {
 #ifndef NDEBUG
       dbgs() << "Call has unhandled type " << VT
              << " while computing remaining regparms\n";
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 9223739fc0098..0e40a92fd8d64 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -377,7 +377,7 @@ class CodeGenPrepare {
   /// to be optimized again.
   /// Note: Consider building time in this pass, when a BB updated, we need
   /// to insert such BB into FreshBBs for huge function.
-  SmallSet<BasicBlock *, 32> FreshBBs;
+  SmallPtrSet<BasicBlock *, 32> FreshBBs;
 
   void releaseMemory() {
     // Clear per function information.
@@ -1105,7 +1105,7 @@ bool CodeGenPrepare::canMergeBlocks(const BasicBlock *BB,
 
 /// Replace all old uses with new ones, and push the updated BBs into FreshBBs.
 static void replaceAllUsesWith(Value *Old, Value *New,
-                               SmallSet<BasicBlock *, 32> &FreshBBs,
+                               SmallPtrSet<BasicBlock *, 32> &FreshBBs,
                                bool IsHuge) {
   auto *OldI = dyn_cast<Instruction>(Old);
   if (OldI) {
@@ -2135,7 +2135,7 @@ static bool isRemOfLoopIncrementWithLoopInvariant(
 //    Rem = rem == RemAmtLoopInvariant ? 0 : Rem;
 static bool foldURemOfLoopIncrement(Instruction *Rem, const DataLayout *DL,
                                     const LoopInfo *LI,
-                                    SmallSet<BasicBlock *, 32> &FreshBBs,
+                                    SmallPtrSet<BasicBlock *, 32> &FreshBBs,
                                     bool IsHuge) {
   Value *AddOffset, *RemAmt, *AddInst;
   PHINode *LoopIncrPN;
@@ -2534,11 +2534,10 @@ static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI,
 ///     %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ]
 ///
 /// If the transform is performed, return true and set ModifiedDT to true.
-static bool despeculateCountZeros(IntrinsicInst *CountZeros,
-                                  LoopInfo &LI,
+static bool despeculateCountZeros(IntrinsicInst *CountZeros, LoopInfo &LI,
                                   const TargetLowering *TLI,
                                   const DataLayout *DL, ModifyDT &ModifiedDT,
-                                  SmallSet<BasicBlock *, 32> &FreshBBs,
+                                  SmallPtrSet<BasicBlock *, 32> &FreshBBs,
                                   bool IsHugeFunc) {
   // If a zero input is undefined, it doesn't make sense to despeculate that.
   if (match(CountZeros->getOperand(1), m_One()))
@@ -4351,7 +4350,7 @@ class AddressingModeCombiner {
                     PhiNodeSet &PhiNodesToMatch) {
     SmallVector<PHIPair, 8> WorkList;
     Matcher.insert({PHI, Candidate});
-    SmallSet<PHINode *, 8> MatchedPHIs;
+    SmallPtrSet<PHINode *, 8> MatchedPHIs;
     MatchedPHIs.insert(PHI);
     WorkList.push_back({PHI, Candidate});
     SmallSet<PHIPair, 8> Visited;
@@ -8635,7 +8634,7 @@ static bool tryUnmergingGEPsAcrossIndirectBr(GetElementPtrInst *GEPI,
 }
 
 static bool optimizeBranch(BranchInst *Branch, const TargetLowering &TLI,
-                           SmallSet<BasicBlock *, 32> &FreshBBs,
+                           SmallPtrSet<BasicBlock *, 32> &FreshBBs,
                            bool IsHugeFunc) {
   // Try and convert
   //  %c = icmp ult %x, 8
diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index 9b2851eb42b40..de95e0aaf2cba 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -67,8 +67,10 @@
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/Support/Allocator.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include <algorithm>
@@ -107,6 +109,42 @@ static bool isNeg(Value *V);
 /// Returns the operand for negation operation.
 static Value *getNegOperand(Value *V);
 
+namespace {
+struct ComplexValue {
+  Value *Real = nullptr;
+  Value *Imag = nullptr;
+
+  bool operator==(const ComplexValue &Other) const {
+    return Real == Other.Real && Imag == Other.Imag;
+  }
+};
+hash_code hash_value(const ComplexValue &Arg) {
+  return hash_combine(DenseMapInfo<Value *>::getHashValue(Arg.Real),
+                      DenseMapInfo<Value *>::getHashValue(Arg.Imag));
+}
+} // end namespace
+typedef SmallVector<struct ComplexValue, 2> ComplexValues;
+
+namespace llvm {
+template <> struct DenseMapInfo<ComplexValue> {
+  static inline ComplexValue getEmptyKey() {
+    return {DenseMapInfo<Value *>::getEmptyKey(),
+            DenseMapInfo<Value *>::getEmptyKey()};
+  }
+  static inline ComplexValue getTombstoneKey() {
+    return {DenseMapInfo<Value *>::getTombstoneKey(),
+            DenseMapInfo<Value *>::getTombstoneKey()};
+  }
+  static unsigned getHashValue(const ComplexValue &Val) {
+    return hash_combine(DenseMapInfo<Value *>::getHashValue(Val.Real),
+                        DenseMapInfo<Value *>::getHashValue(Val.Imag));
+  }
+  static bool isEqual(const ComplexValue &LHS, const ComplexValue &RHS) {
+    return LHS.Real == RHS.Real && LHS.Imag == RHS.Imag;
+  }
+};
+} // end namespace llvm
+
 namespace {
 template <typename T, typename IterT>
 std::optional<T> findCommonBetweenCollections(IterT A, IterT B) {
@@ -145,18 +183,22 @@ struct ComplexDeinterleavingCompositeNode {
 
   ComplexDeinterleavingCompositeNode(ComplexDeinterleavingOperation Op,
                                      Value *R, Value *I)
-      : Operation(Op), Real(R), Imag(I) {}
+      : Operation(Op) {
+    Vals.push_back({R, I});
+  }
+
+  ComplexDeinterleavingCompositeNode(ComplexDeinterleavingOperation Op,
+                                     ComplexValues &Other)
+      : Operation(Op), Vals(Other) {}
 
 private:
   friend class ComplexDeinterleavingGraph;
-  using NodePtr = std::shared_ptr<ComplexDeinterleavingCompositeNode>;
-  using RawNodePtr = ComplexDeinterleavingCompositeNode *;
+  using CompositeNode = ComplexDeinterleavingCompositeNode;
   bool OperandsValid = true;
 
 public:
   ComplexDeinterleavingOperation Operation;
-  Value *Real;
-  Value *Imag;
+  ComplexValues Vals;
 
   // This two members are required exclusively for generating
   // ComplexDeinterleavingOperation::Symmetric operations.
@@ -165,13 +207,13 @@ struct ComplexDeinterleavingCompositeNode {
 
   ComplexDeinterleavingRotation Rotation =
       ComplexDeinterleavingRotation::Rotation_0;
-  SmallVector<RawNodePtr> Operands;
+  SmallVector<CompositeNode *> Operands;
   Value *ReplacementNode = nullptr;
 
-  void addOperand(NodePtr Node) {
-    if (!Node || !Node.get())
+  void addOperand(CompositeNode *Node) {
+    if (!Node)
       OperandsValid = false;
-    Operands.push_back(Node.get());
+    Operands.push_back(Node);
   }
 
   void dump() { dump(dbgs()); }
@@ -184,7 +226,7 @@ struct ComplexDeinterleavingCompositeNode {
       } else
         OS << "nullptr\n";
     };
-    auto PrintNodeRef = [&](RawNodePtr Ptr) {
+    auto PrintNodeRef = [&](CompositeNode *Ptr) {
       if (Ptr)
         OS << Ptr << "\n";
       else
@@ -192,10 +234,12 @@ struct ComplexDeinterleavingCompositeNode {
     };
 
     OS << "- CompositeNode: " << this << "\n";
-    OS << "  Real: ";
-    PrintValue(Real);
-    OS << "  Imag: ";
-    PrintValue(Imag);
+    for (unsigned I = 0; I < Vals.size(); I++) {
+      OS << "  Real(" << I << ") : ";
+      PrintValue(Vals[I].Real);
+      OS << "  Imag(" << I << ") : ";
+      PrintValue(Vals[I].Imag);
+    }
     OS << "  ReplacementNode: ";
     PrintValue(ReplacementNode);
     OS << "  Operation: " << (int)Operation << "\n";
@@ -219,33 +263,35 @@ class ComplexDeinterleavingGraph {
   };
 
   using Addend = std::pair<Value *, bool>;
-  using NodePtr = ComplexDeinterleavingCompositeNode::NodePtr;
-  using RawNodePtr = ComplexDeinterleavingCompositeNode::RawNodePtr;
+  using CompositeNode = ComplexDeinterleavingCompositeNode::CompositeNode;
 
   // Helper struct for holding info about potential partial multiplication
   // candidates
   struct PartialMulCandidate {
     Value *Common;
-    NodePtr Node;
+    CompositeNode *Node;
     unsigned RealIdx;
     unsigned ImagIdx;
     bool IsNodeInverted;
   };
 
   explicit ComplexDeinterleavingGraph(const TargetLowering *TL,
-                                      const TargetLibraryInfo *TLI)
-      : TL(TL), TLI(TLI) {}
+                                      const TargetLibraryInfo *TLI,
+                                      unsigned Factor)
+      : TL(TL), TLI(TLI), Factor(Factor) {}
 
 private:
   const TargetLowering *TL = nullptr;
   const TargetLibraryInfo *TLI = nullptr;
-  SmallVector<NodePtr> CompositeNodes;
-  DenseMap<std::pair<Value *, Value *>, NodePtr> CachedResult;
+  unsigned Factor;
+  SmallVector<CompositeNode *> CompositeNodes;
+  DenseMap<ComplexValues, CompositeNode *> CachedResult;
+  SpecificBumpPtrAllocator<ComplexDeinterleavingCompositeNode> Allocator;
 
   SmallPtrSet<Instruction *, 16> FinalInstructions;
 
   /// Root instructions are instructions from which complex computation starts
-  std::map<Instruction *, NodePtr> RootToNode;
+  std::map<Instruction *, CompositeNode *> RootToNode;
 
   /// Topologically sorted root instructions
   SmallVector<Instruction *, 1> OrderedRoots;
@@ -295,20 +341,35 @@ class ComplexDeinterleavingGraph {
   /// replacement process.
   std::map<PHINode *, PHINode *> OldToNewPHI;
 
-  NodePtr prepareCompositeNode(ComplexDeinterleavingOperation Operation,
-                               Value *R, Value *I) {
+  CompositeNode *prepareCompositeNode(ComplexDeinterleavingOperation Operation,
+                                      Value *R, Value *I) {
     assert(((Operation != ComplexDeinterleavingOperation::ReductionPHI &&
              Operation != ComplexDeinterleavingOperation::ReductionOperation) ||
             (R && I)) &&
            "Reduction related nodes must have Real and Imaginary parts");
-    return std::make_shared<ComplexDeinterleavingCompositeNode>(Operation, R,
-                                                                I);
+    return new (Allocator.Allocate())
+        ComplexDeinterleavingCompositeNode(Operation, R, I);
+  }
+
+  CompositeNode *prepareCompositeNode(ComplexDeinterleavingOperation Operation,
+                                      ComplexValues &Vals) {
+#ifndef NDEBUG
+    for (auto &V : Vals) {
+      assert(
+          ((Operation != ComplexDeinterleavingOperation::ReductionPHI &&
+            Operation != ComplexDeinterleavingOperation::ReductionOperation) ||
+           (V.Real && V.Imag)) &&
+          "Reduction related nodes must have Real and Imaginary parts");
+    }
+#endif
+    return new (Allocator.Allocate())
+        ComplexDeinterleavingCompositeNode(Operation, Vals);
   }
 
-  NodePtr submitCompositeNode(NodePtr Node) {
+  CompositeNode *submitCompositeNode(CompositeNode *Node) {
     CompositeNodes.push_back(Node);
-    if (Node->Real)
-      CachedResult[{Node->Real, Node->Imag}] = Node;
+    if (Node->Vals[0].Real)
+      CachedResult[Node->Vals] = Node;
     return Node;
   }
 
@@ -323,12 +384,12 @@ class ComplexDeinterleavingGraph {
   ///      i: ci - ar * bi
   /// 270: r: cr + ai * bi
   ///      i: ci - ai * br
-  NodePtr identifyPartialMul(Instruction *Real, Instruction *Imag);
+  CompositeNode *identifyPartialMul(Instruction *Real, Instruction *Imag);
 
   /// Identify the other branch of a Partial Mul, taking the CommonOperandI that
   /// is partially known from identifyPartialMul, filling in the other half of
   /// the complex pair.
-  NodePtr
+  CompositeNode *
   identifyNodeWithImplicitAdd(Instruction *I, Instruction *J,
                               std::pair<Value *, Value *> &CommonOperandI);
 
@@ -339,32 +400,38 @@ class ComplexDeinterleavingGraph {
   ///      i: ai + br
   /// 270: r: ar + bi
   ///      i: ai - br
-  NodePtr identifyAdd(Instruction *Real, Instruction *Imag);
-  NodePtr identifySymmetricOperation(Instruction *Real, Instruction *Imag);
-  NodePtr identifyPartialReduction(Value *R, Value *I);
-  NodePtr identifyDotProduct(Value *Inst);
+  CompositeNode *identifyAdd(Instruction *Real, Instruction *Imag);
+  CompositeNode *identifySymmetricOperation(ComplexValues &Vals);
+  CompositeNode *identifyPartialReduction(Value *R, Value *I);
+  CompositeNode *identifyDotProduct(Value *Inst);
 
-  NodePtr identifyNode(Value *R, Value *I);
+  CompositeNode *identifyNode(ComplexValues &Vals);
+
+  CompositeNode *identifyNode(Value *R, Value *I) {
+    ComplexValues Vals;
+    Vals.push_back({R, I});
+    return identifyNode(Vals);
+  }
 
   /// Determine if a sum of complex numbers can be formed from \p RealAddends
   /// and \p ImagAddens. If \p Accumulator is not null, add the result to it.
   /// Return nullptr if it is not possible to construct a complex number.
   /// \p Flags are needed to generate symmetric Add and Sub operations.
-  NodePtr identifyAdditions(std::list<Addend> &RealAddends,
-                            std::list<Addend> &ImagAddends,
-                            std::optional<FastMathFlags> Flags,
-                            NodePtr Accumulator);
+  CompositeNode *identifyAdditions(std::list<Addend> &RealAddends,
+                                   std::list<Addend> &ImagAddends,
+                                   std::optional<FastMathFlags> Flags,
+                                   CompositeNode *Accumulator);
 
   /// Extract one addend that have both real and imaginary parts positive.
-  NodePtr extractPositiveAddend(std::list<Addend> &RealAddends,
-                                std::list<Addend> &ImagAddends);
+  CompositeNode *extractPositiveAddend(std::list<Addend> &RealAddends,
+                                       std::list<Addend> &ImagAddends);
 
   /// Determine if sum of multiplications of complex numbers can be formed from
   /// \p RealMuls and \p ImagMuls. If \p Accumulator is not null, add the result
   /// to it. Return nullptr if it is not possible to construct a complex number.
-  NodePtr identifyMultiplications(std::vector<Product> &RealMuls,
-                                  std::vector<Product> &ImagMuls,
-                                  NodePtr Accumulator);
+  CompositeNode *identifyMultiplications(std::vector<Product> &RealMuls,
+                                         std::vector<Product> &ImagMuls,
+                                         CompositeNode *Accumulator);
 
   /// Go through pairs of multiplication (one Real and one Imag) and find all
   /// possible candidates for partial multiplication and put them into \p
@@ -379,40 +446,42 @@ class ComplexDeinterleavingGraph {
   /// function takes this into consideration and employs a more general approach
   /// to identify complex computations. Initially, it gathers all the addends
   /// and multiplicands and then constructs a complex expression from them.
-  NodePtr identifyReassocNodes(Instruction *I, Instruction *J);
+  CompositeNode *identifyReassocNodes(Instruction *I, Instruction *J);
 
-  NodePtr identifyRoot(Instruction *I);
+  CompositeNode *identifyRoot(Instruction *I);
 
   /// Identifies the Deinterleave operation applied to a vector containing
   /// complex numbers. There are two ways to represent the Deinterleave
   /// operation:
   /// * Using two shufflevectors with even indices for /pReal instruction and
   /// odd indices for /pImag instructions (only for fixed-width vectors)
-  /// * Using two extractvalue instructions applied to `vector.deinterleave2`
-  /// intrinsic (for both fixed and scalable vectors)
-  NodePtr identifyDeinterleave(Instruction *Real, Instruction *Imag);
+  /// * Using N extractvalue instructions applied to `vector.deinterleaveN`
+  /// intrinsics (for both fixed and scalable vectors) where N is a multiple of
+  /// 2.
+  CompositeNode *identifyDeinterleave(ComplexValues &Vals);
 
   /// identifying the operation that represents a complex number repeated in a
   /// Splat vector. There are two possible types of splats: ConstantExpr with
   /// the opcode ShuffleVector and ShuffleVectorInstr. Both should have an
   /// initialization mask with all values set to zero.
-  NodePtr identifySplat(Value *Real, Value *Imag);
+  CompositeNode *identifySplat(ComplexValues &Vals);
 
-  NodePtr identifyPHINode(Instruction *Real, Instruction *Imag);
+  CompositeNode *identifyPHINode(Instruction *Real, Instruction *Imag);
 
   /// Identifies SelectInsts in a loop that has reduction with predication masks
   /// and/or predicated tail folding
-  NodePtr identifySelectNode(Instruction *Real, Instruction *Imag);
+  CompositeNode *identifySelectNode(Instruction *Real, Instruction *Imag);
 
-  Value *replaceNode(IRBuilderBase &Builder, RawNodePtr Node);
+  Value *replaceNode(IRBuilderBase &Builder, CompositeNode *Node);
 
   /// Complete IR modifications after producing new reduction operation:
   /// * Populate the PHINode generated for
   /// ComplexDeinterleavingOperation::ReductionPHI
   /// * Deinterleave the final value outside of the loop and repurpose original
   /// reduction users
-  void processReductionOperation(Value *OperationReplacement, RawNodePtr Node);
-  void processReductionSingle(Value *OperationReplacement, RawNodePtr Node);
+  void processReductionOperation(Value *OperationReplacement,
+                                 CompositeNode *Node);
+  void processReductionSingle(Value *OperationReplacement, CompositeNode *Node);
 
 public:
   void dump() { dump(dbgs()); }
@@ -447,7 +516,7 @@ class ComplexDeinterleaving {
   bool runOnFunction(Function &F);
 
 private:
-  bool evaluateBasicBlock(BasicBlock *B);
+  bool evaluateBasicBlock(BasicBlock *B, unsigned Factor);
 
   const TargetLowering *TL = nullptr;
   const TargetLibraryInfo *TLI = nullptr;
@@ -500,7 +569,15 @@ bool ComplexDeinterleaving::runOnFunction(Function &F) {
 
   bool Changed = false;
   for (auto &B : F)
-    Changed |= evaluateBasicBlock(&B);
+    Changed |= evaluateBasicBlock(&B, 2);
+
+  // TODO: Permit changes for both interleave factors in the same function.
+  if (!Changed) {
+    for (auto &B : F)
+      Changed |= evaluateBasicBlock(&B, 4);
+  }
+
+  // TODO: We can also support interleave factors of 6 and 8 if needed.
 
   return Changed;
 }
@@ -545,8 +622,8 @@ Value *getNegOperand(Value *V) {
   return I->getOperand(1);
 }
 
-bool ComplexDeinterleaving::evaluateBasicBlock(BasicBlock *B) {
-  ComplexDeinterleavingGraph Graph(TL, TLI);
+bool ComplexDeinterleaving::evaluateBasicBlock(BasicBlock *B, unsigned Factor) {
+  ComplexDeinterleavingGraph Graph(TL, TLI, Factor);
   if (Graph.collectPotentialReductions(B))
     Graph.identifyReductionNodes();
 
@@ -561,7 +638,7 @@ bool ComplexDeinterleaving::evaluateBasicBlock(BasicBlock *B) {
   return false;
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyNodeWithImplicitAdd(
     Instruction *Real, Instruction *Imag,
     std::pair<Value *, Value *> &PartialMatch) {
@@ -644,19 +721,20 @@ ComplexDeinterleavingGraph::identifyNodeWithImplicitAdd(
     return nullptr;
   }
 
-  NodePtr CommonNode = identifyNode(PartialMatch.first, PartialMatch.second);
+  CompositeNode *CommonNode =
+      identifyNode(PartialMatch.first, PartialMatch.second);
   if (!CommonNode) {
     LLVM_DEBUG(dbgs() << "  - No CommonNode identified\n");
     return nullptr;
   }
 
-  NodePtr UncommonNode = identifyNode(UncommonRealOp, UncommonImagOp);
+  CompositeNode *UncommonNode = identifyNode(UncommonRealOp, UncommonImagOp);
   if (!UncommonNode) {
     LLVM_DEBUG(dbgs() << "  - No UncommonNode identified\n");
     return nullptr;
   }
 
-  NodePtr Node = prepareCompositeNode(
+  CompositeNode *Node = prepareCompositeNode(
       ComplexDeinterleavingOperation::CMulPartial, Real, Imag);
   Node->Rotation = Rotation;
   Node->addOperand(CommonNode);
@@ -664,11 +742,12 @@ ComplexDeinterleavingGraph::identifyNodeWithImplicitAdd(
   return submitCompositeNode(Node);
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyPartialMul(Instruction *Real,
                                                Instruction *Imag) {
   LLVM_DEBUG(dbgs() << "identifyPartialMul " << *Real << " / " << *Imag
                     << "\n");
+
   // Determine rotation
   auto IsAdd = [](unsigned Op) {
     return Op == Instruction::FAdd || Op == Instruction::Add;
@@ -754,26 +833,28 @@ ComplexDeinterleavingGraph::identifyPartialMul(Instruction *Real,
     return nullptr;
   }
 
-  NodePtr CNode = identifyNodeWithImplicitAdd(CRInst, CIInst, PartialMatch);
+  CompositeNode *CNode =
+      identifyNodeWithImplicitAdd(CRInst, CIInst, PartialMatch);
   if (!CNode) {
     LLVM_DEBUG(dbgs() << "  - No cnode identified\n");
     return nullptr;
   }
 
-  NodePtr UncommonRes = identifyNode(UncommonRealOp, UncommonImagOp);
+  CompositeNode *UncommonRes = identifyNode(UncommonRealOp, UncommonImagOp);
   if (!UncommonRes) {
     LLVM_DEBUG(dbgs() << "  - No UncommonRes identified\n");
     return nullptr;
   }
 
   assert(PartialMatch.first && PartialMatch.second);
-  NodePtr CommonRes = identifyNode(PartialMatch.first, PartialMatch.second);
+  CompositeNode *CommonRes =
+      identifyNode(PartialMatch.first, PartialMatch.second);
   if (!CommonRes) {
     LLVM_DEBUG(dbgs() << "  - No CommonRes identified\n");
     return nullptr;
   }
 
-  NodePtr Node = prepareCompositeNode(
+  CompositeNode *Node = prepareCompositeNode(
       ComplexDeinterleavingOperation::CMulPartial, Real, Imag);
   Node->Rotation = Rotation;
   Node->addOperand(CommonRes);
@@ -782,7 +863,7 @@ ComplexDeinterleavingGraph::identifyPartialMul(Instruction *Real,
   return submitCompositeNode(Node);
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyAdd(Instruction *Real, Instruction *Imag) {
   LLVM_DEBUG(dbgs() << "identifyAdd " << *Real << " / " << *Imag << "\n");
 
@@ -813,18 +894,18 @@ ComplexDeinterleavingGraph::identifyAdd(Instruction *Real, Instruction *Imag) {
     return nullptr;
   }
 
-  NodePtr ResA = identifyNode(AR, AI);
+  CompositeNode *ResA = identifyNode(AR, AI);
   if (!ResA) {
     LLVM_DEBUG(dbgs() << " - AR/AI is not identified as a composite node.\n");
     return nullptr;
   }
-  NodePtr ResB = identifyNode(BR, BI);
+  CompositeNode *ResB = identifyNode(BR, BI);
   if (!ResB) {
     LLVM_DEBUG(dbgs() << " - BR/BI is not identified as a composite node.\n");
     return nullptr;
   }
 
-  NodePtr Node =
+  CompositeNode *Node =
       prepareCompositeNode(ComplexDeinterleavingOperation::CAdd, Real, Imag);
   Node->Rotation = Rotation;
   Node->addOperand(ResA);
@@ -864,52 +945,65 @@ static bool isInstructionPotentiallySymmetric(Instruction *I) {
   }
 }
 
-ComplexDeinterleavingGraph::NodePtr
-ComplexDeinterleavingGraph::identifySymmetricOperation(Instruction *Real,
-                                                       Instruction *Imag) {
-  if (Real->getOpcode() != Imag->getOpcode())
-    return nullptr;
+ComplexDeinterleavingGraph::CompositeNode *
+ComplexDeinterleavingGraph::identifySymmetricOperation(ComplexValues &Vals) {
+  auto *FirstReal = cast<Instruction>(Vals[0].Real);
+  unsigned FirstOpc = FirstReal->getOpcode();
+  for (auto &V : Vals) {
+    auto *Real = cast<Instruction>(V.Real);
+    auto *Imag = cast<Instruction>(V.Imag);
+    if (Real->getOpcode() != FirstOpc || Imag->getOpcode() != FirstOpc)
+      return nullptr;
 
-  if (!isInstructionPotentiallySymmetric(Real) ||
-      !isInstructionPotentiallySymmetric(Imag))
-    return nullptr;
+    if (!isInstructionPotentiallySymmetric(Real) ||
+        !isInstructionPotentiallySymmetric(Imag))
+      return nullptr;
 
-  auto *R0 = Real->getOperand(0);
-  auto *I0 = Imag->getOperand(0);
+    if (isa<FPMathOperator>(FirstReal))
+      if (Real->getFastMathFlags() != FirstReal->getFastMathFlags() ||
+          Imag->getFastMathFlags() != FirstReal->getFastMathFlags())
+        return nullptr;
+  }
 
-  NodePtr Op0 = identifyNode(R0, I0);
-  NodePtr Op1 = nullptr;
+  ComplexValues OpVals;
+  for (auto &V : Vals) {
+    auto *R0 = cast<Instruction>(V.Real)->getOperand(0);
+    auto *I0 = cast<Instruction>(V.Imag)->getOperand(0);
+    OpVals.push_back({R0, I0});
+  }
+
+  CompositeNode *Op0 = identifyNode(OpVals);
+  CompositeNode *Op1 = nullptr;
   if (Op0 == nullptr)
     return nullptr;
 
-  if (Real->isBinaryOp()) {
-    auto *R1 = Real->getOperand(1);
-    auto *I1 = Imag->getOperand(1);
-    Op1 = identifyNode(R1, I1);
+  if (FirstReal->isBinaryOp()) {
+    OpVals.clear();
+    for (auto &V : Vals) {
+      auto *R1 = cast<Instruction>(V.Real)->getOperand(1);
+      auto *I1 = cast<Instruction>(V.Imag)->getOperand(1);
+      OpVals.push_back({R1, I1});
+    }
+    Op1 = identifyNode(OpVals);
     if (Op1 == nullptr)
       return nullptr;
   }
 
-  if (isa<FPMathOperator>(Real) &&
-      Real->getFastMathFlags() != Imag->getFastMathFlags())
-    return nullptr;
-
-  auto Node = prepareCompositeNode(ComplexDeinterleavingOperation::Symmetric,
-                                   Real, Imag);
-  Node->Opcode = Real->getOpcode();
-  if (isa<FPMathOperator>(Real))
-    Node->Flags = Real->getFastMathFlags();
+  auto Node =
+      prepareCompositeNode(ComplexDeinterleavingOperation::Symmetric, Vals);
+  Node->Opcode = FirstReal->getOpcode();
+  if (isa<FPMathOperator>(FirstReal))
+    Node->Flags = FirstReal->getFastMathFlags();
 
   Node->addOperand(Op0);
-  if (Real->isBinaryOp())
+  if (FirstReal->isBinaryOp())
     Node->addOperand(Op1);
 
   return submitCompositeNode(Node);
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyDotProduct(Value *V) {
-
   if (!TL->isComplexDeinterleavingOperationSupported(
           ComplexDeinterleavingOperation::CDot, V->getType())) {
     LLVM_DEBUG(dbgs() << "Target doesn't support complex deinterleaving "
@@ -921,10 +1015,10 @@ ComplexDeinterleavingGraph::identifyDotProduct(Value *V) {
   auto *Inst = cast<Instruction>(V);
   auto *RealUser = cast<Instruction>(*Inst->user_begin());
 
-  NodePtr CN =
+  CompositeNode *CN =
       prepareCompositeNode(ComplexDeinterleavingOperation::CDot, Inst, nullptr);
 
-  NodePtr ANode;
+  CompositeNode *ANode = nullptr;
 
   const Intrinsic::ID PartialReduceInt =
       Intrinsic::experimental_vector_partial_reduce_add;
@@ -1008,7 +1102,7 @@ ComplexDeinterleavingGraph::identifyDotProduct(Value *V) {
   if (Phi->getType() != VTy && RealUser->getType() != VTy)
     return nullptr;
 
-  NodePtr Node = identifyNode(AReal, AImag);
+  CompositeNode *Node = identifyNode(AReal, AImag);
 
   // In the case that a node was identified to figure out the rotation, ensure
   // that trying to identify a node with AReal and AImag post-unwrap results in
@@ -1028,7 +1122,7 @@ ComplexDeinterleavingGraph::identifyDotProduct(Value *V) {
   return submitCompositeNode(CN);
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyPartialReduction(Value *R, Value *I) {
   // Partial reductions don't support non-vector types, so check these first
   if (!isa<VectorType>(R->getType()) || !isa<VectorType>(I->getType()))
@@ -1047,76 +1141,88 @@ ComplexDeinterleavingGraph::identifyPartialReduction(Value *R, Value *I) {
                     Intrinsic::experimental_vector_partial_reduce_add)
     return nullptr;
 
-  if (NodePtr CN = identifyDotProduct(IInst))
+  if (CompositeNode *CN = identifyDotProduct(IInst))
     return CN;
 
   return nullptr;
 }
 
-ComplexDeinterleavingGraph::NodePtr
-ComplexDeinterleavingGraph::identifyNode(Value *R, Value *I) {
-  auto It = CachedResult.find({R, I});
+ComplexDeinterleavingGraph::CompositeNode *
+ComplexDeinterleavingGraph::identifyNode(ComplexValues &Vals) {
+  auto It = CachedResult.find(Vals);
   if (It != CachedResult.end()) {
     LLVM_DEBUG(dbgs() << " - Folding to existing node\n");
     return It->second;
   }
 
-  if (NodePtr CN = identifyPartialReduction(R, I))
-    return CN;
-
-  bool IsReduction = RealPHI == R && (!ImagPHI || ImagPHI == I);
-  if (!IsReduction && R->getType() != I->getType())
-    return nullptr;
+  if (Vals.size() == 1) {
+    assert(Factor == 2 && "Can only handle interleave factors of 2");
+    Value *R = Vals[0].Real;
+    Value *I = Vals[0].Imag;
+    if (CompositeNode *CN = identifyPartialReduction(R, I))
+      return CN;
+    bool IsReduction = RealPHI == R && (!ImagPHI || ImagPHI == I);
+    if (!IsReduction && R->getType() != I->getType())
+      return nullptr;
+  }
 
-  if (NodePtr CN = identifySplat(R, I))
+  if (CompositeNode *CN = identifySplat(Vals))
     return CN;
 
-  auto *Real = dyn_cast<Instruction>(R);
-  auto *Imag = dyn_cast<Instruction>(I);
-  if (!Real || !Imag)
-    return nullptr;
+  for (auto &V : Vals) {
+    auto *Real = dyn_cast<Instruction>(V.Real);
+    auto *Imag = dyn_cast<Instruction>(V.Imag);
+    if (!Real || !Imag)
+      return nullptr;
+  }
 
-  if (NodePtr CN = identifyDeinterleave(Real, Imag))
+  if (CompositeNode *CN = identifyDeinterleave(Vals))
     return CN;
 
-  if (NodePtr CN = identifyPHINode(Real, Imag))
-    return CN;
+  if (Vals.size() == 1) {
+    assert(Factor == 2 && "Can only handle interleave factors of 2");
+    auto *Real = dyn_cast<Instruction>(Vals[0].Real);
+    auto *Imag = dyn_cast<Instruction>(Vals[0].Imag);
+    if (CompositeNode *CN = identifyPHINode(Real, Imag))
+      return CN;
 
-  if (NodePtr CN = identifySelectNode(Real, Imag))
-    return CN;
+    if (CompositeNode *CN = identifySelectNode(Real, Imag))
+      return CN;
 
-  auto *VTy = cast<VectorType>(Real->getType());
-  auto *NewVTy = VectorType::getDoubleElementsVectorType(VTy);
+    auto *VTy = cast<VectorType>(Real->getType());
+    auto *NewVTy = VectorType::getDoubleElementsVectorType(VTy);
 
-  bool HasCMulSupport = TL->isComplexDeinterleavingOperationSupported(
-      ComplexDeinterleavingOperation::CMulPartial, NewVTy);
-  bool HasCAddSupport = TL->isComplexDeinterleavingOperationSupported(
-      ComplexDeinterleavingOperation::CAdd, NewVTy);
+    bool HasCMulSupport = TL->isComplexDeinterleavingOperationSupported(
+        ComplexDeinterleavingOperation::CMulPartial, NewVTy);
+    bool HasCAddSupport = TL->isComplexDeinterleavingOperationSupported(
+        ComplexDeinterleavingOperation::CAdd, NewVTy);
 
-  if (HasCMulSupport && isInstructionPairMul(Real, Imag)) {
-    if (NodePtr CN = identifyPartialMul(Real, Imag))
-      return CN;
-  }
+    if (HasCMulSupport && isInstructionPairMul(Real, Imag)) {
+      if (CompositeNode *CN = identifyPartialMul(Real, Imag))
+        return CN;
+    }
 
-  if (HasCAddSupport && isInstructionPairAdd(Real, Imag)) {
-    if (NodePtr CN = identifyAdd(Real, Imag))
-      return CN;
-  }
+    if (HasCAddSupport && isInstructionPairAdd(Real, Imag)) {
+      if (CompositeNode *CN = identifyAdd(Real, Imag))
+        return CN;
+    }
 
-  if (HasCMulSupport && HasCAddSupport) {
-    if (NodePtr CN = identifyReassocNodes(Real, Imag))
-      return CN;
+    if (HasCMulSupport && HasCAddSupport) {
+      if (CompositeNode *CN = identifyReassocNodes(Real, Imag)) {
+        return CN;
+      }
+    }
   }
 
-  if (NodePtr CN = identifySymmetricOperation(Real, Imag))
+  if (CompositeNode *CN = identifySymmetricOperation(Vals))
     return CN;
 
   LLVM_DEBUG(dbgs() << "  - Not recognised as a valid pattern.\n");
-  CachedResult[{R, I}] = nullptr;
+  CachedResult[Vals] = nullptr;
   return nullptr;
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
                                                  Instruction *Imag) {
   auto IsOperationSupported = [](unsigned Opcode) -> bool {
@@ -1239,7 +1345,7 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
   if (RealAddends.size() != ImagAddends.size())
     return nullptr;
 
-  NodePtr FinalNode;
+  CompositeNode *FinalNode = nullptr;
   if (!RealMuls.empty() || !ImagMuls.empty()) {
     // If there are multiplicands, extract positive addend and use it as an
     // accumulator
@@ -1256,9 +1362,10 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
       return nullptr;
   }
   assert(FinalNode && "FinalNode can not be nullptr here");
+  assert(FinalNode->Vals.size() == 1);
   // Set the Real and Imag fields of the final node and submit it
-  FinalNode->Real = Real;
-  FinalNode->Imag = Imag;
+  FinalNode->Vals[0].Real = Real;
+  FinalNode->Vals[0].Imag = Imag;
   submitCompositeNode(FinalNode);
   return FinalNode;
 }
@@ -1314,10 +1421,10 @@ bool ComplexDeinterleavingGraph::collectPartialMuls(
   return true;
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyMultiplications(
     std::vector<Product> &RealMuls, std::vector<Product> &ImagMuls,
-    NodePtr Accumulator = nullptr) {
+    CompositeNode *Accumulator = nullptr) {
   if (RealMuls.size() != ImagMuls.size())
     return nullptr;
 
@@ -1326,7 +1433,7 @@ ComplexDeinterleavingGraph::identifyMultiplications(
     return nullptr;
 
   // Map to store common instruction to node pointers
-  std::map<Value *, NodePtr> CommonToNode;
+  std::map<Value *, CompositeNode *> CommonToNode;
   std::vector<bool> Processed(Info.size(), false);
   for (unsigned I = 0; I < Info.size(); ++I) {
     if (Processed[I])
@@ -1358,7 +1465,7 @@ ComplexDeinterleavingGraph::identifyMultiplications(
 
   std::vector<bool> ProcessedReal(RealMuls.size(), false);
   std::vector<bool> ProcessedImag(ImagMuls.size(), false);
-  NodePtr Result = Accumulator;
+  CompositeNode *Result = Accumulator;
   for (auto &PMI : Info) {
     if (ProcessedReal[PMI.RealIdx] || ProcessedImag[PMI.ImagIdx])
       continue;
@@ -1381,7 +1488,7 @@ ComplexDeinterleavingGraph::identifyMultiplications(
 
     auto NodeA = It->second;
     auto NodeB = PMI.Node;
-    auto IsMultiplicandReal = PMI.Common == NodeA->Real;
+    auto IsMultiplicandReal = PMI.Common == NodeA->Vals[0].Real;
     // The following table illustrates the relationship between multiplications
     // and rotations. If we consider the multiplication (X + iY) * (U + iV), we
     // can see:
@@ -1423,14 +1530,14 @@ ComplexDeinterleavingGraph::identifyMultiplications(
 
     LLVM_DEBUG({
       dbgs() << "Identified partial multiplication (X, Y) * (U, V):\n";
-      dbgs().indent(4) << "X: " << *NodeA->Real << "\n";
-      dbgs().indent(4) << "Y: " << *NodeA->Imag << "\n";
-      dbgs().indent(4) << "U: " << *NodeB->Real << "\n";
-      dbgs().indent(4) << "V: " << *NodeB->Imag << "\n";
+      dbgs().indent(4) << "X: " << *NodeA->Vals[0].Real << "\n";
+      dbgs().indent(4) << "Y: " << *NodeA->Vals[0].Imag << "\n";
+      dbgs().indent(4) << "U: " << *NodeB->Vals[0].Real << "\n";
+      dbgs().indent(4) << "V: " << *NodeB->Vals[0].Imag << "\n";
       dbgs().indent(4) << "Rotation - " << (int)Rotation * 90 << "\n";
     });
 
-    NodePtr NodeMul = prepareCompositeNode(
+    CompositeNode *NodeMul = prepareCompositeNode(
         ComplexDeinterleavingOperation::CMulPartial, nullptr, nullptr);
     NodeMul->Rotation = Rotation;
     NodeMul->addOperand(NodeA);
@@ -1471,14 +1578,14 @@ ComplexDeinterleavingGraph::identifyMultiplications(
   return Result;
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyAdditions(
     std::list<Addend> &RealAddends, std::list<Addend> &ImagAddends,
-    std::optional<FastMathFlags> Flags, NodePtr Accumulator = nullptr) {
+    std::optional<FastMathFlags> Flags, CompositeNode *Accumulator = nullptr) {
   if (RealAddends.size() != ImagAddends.size())
     return nullptr;
 
-  NodePtr Result;
+  CompositeNode *Result = nullptr;
   // If we have accumulator use it as first addend
   if (Accumulator)
     Result = Accumulator;
@@ -1506,7 +1613,7 @@ ComplexDeinterleavingGraph::identifyAdditions(
       else
         Rotation = ComplexDeinterleavingRotation::Rotation_270;
 
-      NodePtr AddNode;
+      CompositeNode *AddNode = nullptr;
       if (Rotation == ComplexDeinterleavingRotation::Rotation_0 ||
           Rotation == ComplexDeinterleavingRotation::Rotation_180) {
         AddNode = identifyNode(R, I);
@@ -1521,7 +1628,7 @@ ComplexDeinterleavingGraph::identifyAdditions(
           dbgs().indent(4) << "Rotation - " << (int)Rotation * 90 << "\n";
         });
 
-        NodePtr TmpNode;
+        CompositeNode *TmpNode = nullptr;
         if (Rotation == llvm::ComplexDeinterleavingRotation::Rotation_0) {
           TmpNode = prepareCompositeNode(
               ComplexDeinterleavingOperation::Symmetric, nullptr, nullptr);
@@ -1563,7 +1670,7 @@ ComplexDeinterleavingGraph::identifyAdditions(
   return Result;
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::extractPositiveAddend(
     std::list<Addend> &RealAddends, std::list<Addend> &ImagAddends) {
   for (auto ItR = RealAddends.begin(); ItR != RealAddends.end(); ++ItR) {
@@ -1595,10 +1702,13 @@ bool ComplexDeinterleavingGraph::identifyNodes(Instruction *RootI) {
                ComplexDeinterleavingOperation::ReductionOperation ||
            RootNode->Operation ==
                ComplexDeinterleavingOperation::ReductionSingle);
+    assert(RootNode->Vals.size() == 1 &&
+           "Cannot handle reductions involving multiple complex values");
     // Find out which part, Real or Imag, comes later, and only if we come to
     // the latest part, add it to OrderedRoots.
-    auto *R = cast<Instruction>(RootNode->Real);
-    auto *I = RootNode->Imag ? cast<Instruction>(RootNode->Imag) : nullptr;
+    auto *R = cast<Instruction>(RootNode->Vals[0].Real);
+    auto *I = RootNode->Vals[0].Imag ? cast<Instruction>(RootNode->Vals[0].Imag)
+                                     : nullptr;
 
     Instruction *ReplacementAnchor;
     if (I)
@@ -1631,6 +1741,8 @@ bool ComplexDeinterleavingGraph::identifyNodes(Instruction *RootI) {
 
 bool ComplexDeinterleavingGraph::collectPotentialReductions(BasicBlock *B) {
   bool FoundPotentialReduction = false;
+  if (Factor != 2)
+    return false;
 
   auto *Br = dyn_cast<BranchInst>(B->getTerminator());
   if (!Br || Br->getNumSuccessors() != 2)
@@ -1682,6 +1794,8 @@ bool ComplexDeinterleavingGraph::collectPotentialReductions(BasicBlock *B) {
 }
 
 void ComplexDeinterleavingGraph::identifyReductionNodes() {
+  assert(Factor == 2 && "Cannot handle multiple complex values");
+
   SmallVector<bool> Processed(ReductionInfo.size(), false);
   SmallVector<Instruction *> OperationInstruction;
   for (auto &P : ReductionInfo)
@@ -1771,11 +1885,11 @@ void ComplexDeinterleavingGraph::identifyReductionNodes() {
 }
 
 bool ComplexDeinterleavingGraph::checkNodes() {
-
   bool FoundDeinterleaveNode = false;
-  for (NodePtr N : CompositeNodes) {
+  for (CompositeNode *N : CompositeNodes) {
     if (!N->areOperandsValid())
       return false;
+
     if (N->Operation == ComplexDeinterleavingOperation::Deinterleave)
       FoundDeinterleaveNode = true;
   }
@@ -1858,20 +1972,36 @@ bool ComplexDeinterleavingGraph::checkNodes() {
   return !RootToNode.empty();
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyRoot(Instruction *RootI) {
   if (auto *Intrinsic = dyn_cast<IntrinsicInst>(RootI)) {
-    if (Intrinsic->getIntrinsicID() != Intrinsic::vector_interleave2)
+    if (Intrinsic::getInterleaveIntrinsicID(Factor) !=
+        Intrinsic->getIntrinsicID())
       return nullptr;
 
-    auto *Real = dyn_cast<Instruction>(Intrinsic->getOperand(0));
-    auto *Imag = dyn_cast<Instruction>(Intrinsic->getOperand(1));
-    if (!Real || !Imag)
-      return nullptr;
+    ComplexValues Vals;
+    for (unsigned I = 0; I < Factor; I += 2) {
+      auto *Real = dyn_cast<Instruction>(Intrinsic->getOperand(I));
+      auto *Imag = dyn_cast<Instruction>(Intrinsic->getOperand(I + 1));
+      if (!Real || !Imag)
+        return nullptr;
+      Vals.push_back({Real, Imag});
+    }
 
-    return identifyNode(Real, Imag);
+    ComplexDeinterleavingGraph::CompositeNode *Node1 = identifyNode(Vals);
+    if (!Node1)
+      return nullptr;
+    return Node1;
   }
 
+  // TODO: We could also add support for fixed-width interleave factors of 4
+  // and above, but currently for symmetric operations the interleaves and
+  // deinterleaves are already removed by VectorCombine. If we extend this to
+  // permit complex multiplications, reductions, etc. then we should also add
+  // support for fixed-width here.
+  if (Factor != 2)
+    return nullptr;
+
   auto *SVI = dyn_cast<ShuffleVectorInst>(RootI);
   if (!SVI)
     return nullptr;
@@ -1889,23 +2019,53 @@ ComplexDeinterleavingGraph::identifyRoot(Instruction *RootI) {
   return identifyNode(Real, Imag);
 }
 
-ComplexDeinterleavingGraph::NodePtr
-ComplexDeinterleavingGraph::identifyDeinterleave(Instruction *Real,
-                                                 Instruction *Imag) {
-  Instruction *I = nullptr;
-  Value *FinalValue = nullptr;
-  if (match(Real, m_ExtractValue<0>(m_Instruction(I))) &&
-      match(Imag, m_ExtractValue<1>(m_Specific(I))) &&
-      match(I, m_Intrinsic<Intrinsic::vector_deinterleave2>(
-                   m_Value(FinalValue)))) {
-    NodePtr PlaceholderNode = prepareCompositeNode(
-        llvm::ComplexDeinterleavingOperation::Deinterleave, Real, Imag);
-    PlaceholderNode->ReplacementNode = FinalValue;
-    FinalInstructions.insert(Real);
-    FinalInstructions.insert(Imag);
+ComplexDeinterleavingGraph::CompositeNode *
+ComplexDeinterleavingGraph::identifyDeinterleave(ComplexValues &Vals) {
+  Instruction *II = nullptr;
+
+  // Must be at least one complex value.
+  auto CheckExtract = [&](Value *V, unsigned ExpectedIdx,
+                          Instruction *ExpectedInsn) -> ExtractValueInst * {
+    auto *EVI = dyn_cast<ExtractValueInst>(V);
+    if (!EVI || EVI->getNumIndices() != 1 ||
+        EVI->getIndices()[0] != ExpectedIdx ||
+        !isa<Instruction>(EVI->getAggregateOperand()) ||
+        (ExpectedInsn && ExpectedInsn != EVI->getAggregateOperand()))
+      return nullptr;
+    return EVI;
+  };
+
+  for (unsigned Idx = 0; Idx < Vals.size(); Idx++) {
+    ExtractValueInst *RealEVI = CheckExtract(Vals[Idx].Real, Idx * 2, II);
+    if (RealEVI && Idx == 0)
+      II = cast<Instruction>(RealEVI->getAggregateOperand());
+    if (!RealEVI || !CheckExtract(Vals[Idx].Imag, (Idx * 2) + 1, II)) {
+      II = nullptr;
+      break;
+    }
+  }
+
+  if (auto *IntrinsicII = dyn_cast_or_null<IntrinsicInst>(II)) {
+    if (IntrinsicII->getIntrinsicID() !=
+        Intrinsic::getDeinterleaveIntrinsicID(2 * Vals.size()))
+      return nullptr;
+
+    // The remaining should match too.
+    CompositeNode *PlaceholderNode = prepareCompositeNode(
+        llvm::ComplexDeinterleavingOperation::Deinterleave, Vals);
+    PlaceholderNode->ReplacementNode = II->getOperand(0);
+    for (auto &V : Vals) {
+      FinalInstructions.insert(cast<Instruction>(V.Real));
+      FinalInstructions.insert(cast<Instruction>(V.Imag));
+    }
     return submitCompositeNode(PlaceholderNode);
   }
 
+  if (Vals.size() != 1)
+    return nullptr;
+
+  Value *Real = Vals[0].Real;
+  Value *Imag = Vals[0].Imag;
   auto *RealShuffle = dyn_cast<ShuffleVectorInst>(Real);
   auto *ImagShuffle = dyn_cast<ShuffleVectorInst>(Imag);
   if (!RealShuffle || !ImagShuffle) {
@@ -1989,7 +2149,7 @@ ComplexDeinterleavingGraph::identifyDeinterleave(Instruction *Real,
     return nullptr;
   }
 
-  NodePtr PlaceholderNode =
+  CompositeNode *PlaceholderNode =
       prepareCompositeNode(llvm::ComplexDeinterleavingOperation::Deinterleave,
                            RealShuffle, ImagShuffle);
   PlaceholderNode->ReplacementNode = RealShuffle->getOperand(0);
@@ -1998,8 +2158,8 @@ ComplexDeinterleavingGraph::identifyDeinterleave(Instruction *Real,
   return submitCompositeNode(PlaceholderNode);
 }
 
-ComplexDeinterleavingGraph::NodePtr
-ComplexDeinterleavingGraph::identifySplat(Value *R, Value *I) {
+ComplexDeinterleavingGraph::CompositeNode *
+ComplexDeinterleavingGraph::identifySplat(ComplexValues &Vals) {
   auto IsSplat = [](Value *V) -> bool {
     // Fixed-width vector with constants
     if (isa<ConstantDataVector>(V))
@@ -2033,40 +2193,55 @@ ComplexDeinterleavingGraph::identifySplat(Value *R, Value *I) {
     return all_equal(Mask) && Mask[0] == 0;
   };
 
-  if (!IsSplat(R) || !IsSplat(I))
-    return nullptr;
-
-  auto *Real = dyn_cast<Instruction>(R);
-  auto *Imag = dyn_cast<Instruction>(I);
-  if ((!Real && Imag) || (Real && !Imag))
-    return nullptr;
+  // The splats must meet the following requirements:
+  //   1. Must either be all instructions or all values.
+  //   2. Non-constant splats must live in the same block.
+  if (auto *FirstValAsInstruction = dyn_cast<Instruction>(Vals[0].Real)) {
+    BasicBlock *FirstBB = FirstValAsInstruction->getParent();
+    for (auto &V : Vals) {
+      if (!IsSplat(V.Real) || !IsSplat(V.Imag))
+        return nullptr;
 
-  if (Real && Imag) {
-    // Non-constant splats should be in the same basic block
-    if (Real->getParent() != Imag->getParent())
-      return nullptr;
+      auto *Real = dyn_cast<Instruction>(V.Real);
+      auto *Imag = dyn_cast<Instruction>(V.Imag);
+      if (!Real || !Imag || Real->getParent() != FirstBB ||
+          Imag->getParent() != FirstBB)
+        return nullptr;
+    }
+  } else {
+    for (auto &V : Vals) {
+      if (!IsSplat(V.Real) || !IsSplat(V.Imag) || isa<Instruction>(V.Real) ||
+          isa<Instruction>(V.Imag))
+        return nullptr;
+    }
+  }
 
-    FinalInstructions.insert(Real);
-    FinalInstructions.insert(Imag);
+  for (auto &V : Vals) {
+    auto *Real = dyn_cast<Instruction>(V.Real);
+    auto *Imag = dyn_cast<Instruction>(V.Imag);
+    if (Real && Imag) {
+      FinalInstructions.insert(Real);
+      FinalInstructions.insert(Imag);
+    }
   }
-  NodePtr PlaceholderNode =
-      prepareCompositeNode(ComplexDeinterleavingOperation::Splat, R, I);
+  CompositeNode *PlaceholderNode =
+      prepareCompositeNode(ComplexDeinterleavingOperation::Splat, Vals);
   return submitCompositeNode(PlaceholderNode);
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifyPHINode(Instruction *Real,
                                             Instruction *Imag) {
   if (Real != RealPHI || (ImagPHI && Imag != ImagPHI))
     return nullptr;
 
   PHIsFound = true;
-  NodePtr PlaceholderNode = prepareCompositeNode(
+  CompositeNode *PlaceholderNode = prepareCompositeNode(
       ComplexDeinterleavingOperation::ReductionPHI, Real, Imag);
   return submitCompositeNode(PlaceholderNode);
 }
 
-ComplexDeinterleavingGraph::NodePtr
+ComplexDeinterleavingGraph::CompositeNode *
 ComplexDeinterleavingGraph::identifySelectNode(Instruction *Real,
                                                Instruction *Imag) {
   auto *SelectReal = dyn_cast<SelectInst>(Real);
@@ -2096,7 +2271,7 @@ ComplexDeinterleavingGraph::identifySelectNode(Instruction *Real,
   if (!NodeB)
     return nullptr;
 
-  NodePtr PlaceholderNode = prepareCompositeNode(
+  CompositeNode *PlaceholderNode = prepareCompositeNode(
       ComplexDeinterleavingOperation::ReductionSelect, Real, Imag);
   PlaceholderNode->addOperand(NodeA);
   PlaceholderNode->addOperand(NodeB);
@@ -2140,17 +2315,18 @@ static Value *replaceSymmetricNode(IRBuilderBase &B, unsigned Opcode,
 }
 
 Value *ComplexDeinterleavingGraph::replaceNode(IRBuilderBase &Builder,
-                                               RawNodePtr Node) {
+                                               CompositeNode *Node) {
   if (Node->ReplacementNode)
     return Node->ReplacementNode;
 
-  auto ReplaceOperandIfExist = [&](RawNodePtr &Node, unsigned Idx) -> Value * {
+  auto ReplaceOperandIfExist = [&](CompositeNode *Node,
+                                   unsigned Idx) -> Value * {
     return Node->Operands.size() > Idx
                ? replaceNode(Builder, Node->Operands[Idx])
                : nullptr;
   };
 
-  Value *ReplacementNode;
+  Value *ReplacementNode = nullptr;
   switch (Node->Operation) {
   case ComplexDeinterleavingOperation::CDot: {
     Value *Input0 = ReplaceOperandIfExist(Node, 0);
@@ -2186,24 +2362,35 @@ Value *ComplexDeinterleavingGraph::replaceNode(IRBuilderBase &Builder,
     llvm_unreachable("Deinterleave node should already have ReplacementNode");
     break;
   case ComplexDeinterleavingOperation::Splat: {
-    auto *R = dyn_cast<Instruction>(Node->Real);
-    auto *I = dyn_cast<Instruction>(Node->Imag);
+    SmallVector<Value *> Ops;
+    for (auto &V : Node->Vals) {
+      Ops.push_back(V.Real);
+      Ops.push_back(V.Imag);
+    }
+    auto *R = dyn_cast<Instruction>(Node->Vals[0].Real);
+    auto *I = dyn_cast<Instruction>(Node->Vals[0].Imag);
     if (R && I) {
       // Splats that are not constant are interleaved where they are located
-      Instruction *InsertPoint = (I->comesBefore(R) ? R : I)->getNextNode();
+      Instruction *InsertPoint = R;
+      for (auto V : Node->Vals) {
+        if (InsertPoint->comesBefore(cast<Instruction>(V.Real)))
+          InsertPoint = cast<Instruction>(V.Real);
+        if (InsertPoint->comesBefore(cast<Instruction>(V.Imag)))
+          InsertPoint = cast<Instruction>(V.Imag);
+      }
+      InsertPoint = InsertPoint->getNextNode();
       IRBuilder<> IRB(InsertPoint);
-      ReplacementNode = IRB.CreateVectorInterleave({Node->Real, Node->Imag});
+      ReplacementNode = IRB.CreateVectorInterleave(Ops);
     } else {
-      ReplacementNode =
-          Builder.CreateVectorInterleave({Node->Real, Node->Imag});
+      ReplacementNode = Builder.CreateVectorInterleave(Ops);
     }
     break;
   }
   case ComplexDeinterleavingOperation::ReductionPHI: {
     // If Operation is ReductionPHI, a new empty PHINode is created.
     // It is filled later when the ReductionOperation is processed.
-    auto *OldPHI = cast<PHINode>(Node->Real);
-    auto *VTy = cast<VectorType>(Node->Real->getType());
+    auto *OldPHI = cast<PHINode>(Node->Vals[0].Real);
+    auto *VTy = cast<VectorType>(Node->Vals[0].Real->getType());
     auto *NewVTy = VectorType::getDoubleElementsVectorType(VTy);
     auto *NewPHI = PHINode::Create(NewVTy, 0, "", BackEdge->getFirstNonPHIIt());
     OldToNewPHI[OldPHI] = NewPHI;
@@ -2219,8 +2406,8 @@ Value *ComplexDeinterleavingGraph::replaceNode(IRBuilderBase &Builder,
     processReductionOperation(ReplacementNode, Node);
     break;
   case ComplexDeinterleavingOperation::ReductionSelect: {
-    auto *MaskReal = cast<Instruction>(Node->Real)->getOperand(0);
-    auto *MaskImag = cast<Instruction>(Node->Imag)->getOperand(0);
+    auto *MaskReal = cast<Instruction>(Node->Vals[0].Real)->getOperand(0);
+    auto *MaskImag = cast<Instruction>(Node->Vals[0].Imag)->getOperand(0);
     auto *A = replaceNode(Builder, Node->Operands[0]);
     auto *B = replaceNode(Builder, Node->Operands[1]);
     auto *NewMask = Builder.CreateVectorInterleave({MaskReal, MaskImag});
@@ -2236,8 +2423,8 @@ Value *ComplexDeinterleavingGraph::replaceNode(IRBuilderBase &Builder,
 }
 
 void ComplexDeinterleavingGraph::processReductionSingle(
-    Value *OperationReplacement, RawNodePtr Node) {
-  auto *Real = cast<Instruction>(Node->Real);
+    Value *OperationReplacement, CompositeNode *Node) {
+  auto *Real = cast<Instruction>(Node->Vals[0].Real);
   auto *OldPHI = ReductionInfo[Real].first;
   auto *NewPHI = OldToNewPHI[OldPHI];
   auto *VTy = cast<VectorType>(Real->getType());
@@ -2268,9 +2455,9 @@ void ComplexDeinterleavingGraph::processReductionSingle(
 }
 
 void ComplexDeinterleavingGraph::processReductionOperation(
-    Value *OperationReplacement, RawNodePtr Node) {
-  auto *Real = cast<Instruction>(Node->Real);
-  auto *Imag = cast<Instruction>(Node->Imag);
+    Value *OperationReplacement, CompositeNode *Node) {
+  auto *Real = cast<Instruction>(Node->Vals[0].Real);
+  auto *Imag = cast<Instruction>(Node->Vals[0].Imag);
   auto *OldPHIReal = ReductionInfo[Real].first;
   auto *OldPHIImag = ReductionInfo[Imag].first;
   auto *NewPHI = OldToNewPHI[OldPHIReal];
@@ -2314,19 +2501,19 @@ void ComplexDeinterleavingGraph::replaceNodes() {
 
     IRBuilder<> Builder(RootInstruction);
     auto RootNode = RootToNode[RootInstruction];
-    Value *R = replaceNode(Builder, RootNode.get());
+    Value *R = replaceNode(Builder, RootNode);
 
     if (RootNode->Operation ==
         ComplexDeinterleavingOperation::ReductionOperation) {
-      auto *RootReal = cast<Instruction>(RootNode->Real);
-      auto *RootImag = cast<Instruction>(RootNode->Imag);
+      auto *RootReal = cast<Instruction>(RootNode->Vals[0].Real);
+      auto *RootImag = cast<Instruction>(RootNode->Vals[0].Imag);
       ReductionInfo[RootReal].first->removeIncomingValue(BackEdge);
       ReductionInfo[RootImag].first->removeIncomingValue(BackEdge);
       DeadInstrRoots.push_back(RootReal);
       DeadInstrRoots.push_back(RootImag);
     } else if (RootNode->Operation ==
                ComplexDeinterleavingOperation::ReductionSingle) {
-      auto *RootInst = cast<Instruction>(RootNode->Real);
+      auto *RootInst = cast<Instruction>(RootNode->Vals[0].Real);
       auto &Info = ReductionInfo[RootInst];
       Info.first->removeIncomingValue(BackEdge);
       DeadInstrRoots.push_back(Info.second);
diff --git a/llvm/lib/CodeGen/ExpandVectorPredication.cpp b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
index d8e3f5fbb31de..753c656007703 100644
--- a/llvm/lib/CodeGen/ExpandVectorPredication.cpp
+++ b/llvm/lib/CodeGen/ExpandVectorPredication.cpp
@@ -508,8 +508,7 @@ bool CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
     IRBuilder<> Builder(VPI.getParent(), VPI.getIterator());
     Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());
     Value *VScale = Builder.CreateVScale(Int32Ty, "vscale");
-    MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size",
-                               /*NUW*/ true, /*NSW*/ false);
+    MaxEVL = Builder.CreateNUWMul(VScale, FactorConst, "scalable_size");
   } else {
     MaxEVL = ConstantInt::get(Int32Ty, StaticElemCount.getFixedValue(), false);
   }
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 9ba17829d2929..90a18b86c1b1f 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -132,9 +132,10 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
   unsigned i = 0;
   unsigned NumFixedArgs = CB.getFunctionType()->getNumParams();
   for (const auto &Arg : CB.args()) {
-    ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i),
-                    i < NumFixedArgs};
+    ArgInfo OrigArg{ArgRegs[i], *Arg.get(), i, getAttributesForArgIdx(CB, i)};
     setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CB);
+    if (i >= NumFixedArgs)
+      OrigArg.Flags[0].setVarArg();
 
     // If we have an explicit sret argument that is an Instruction, (i.e., it
     // might point to function-local memory), we can't meaningfully tail-call.
@@ -301,7 +302,7 @@ void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
     // double] -> double).
     SplitArgs.emplace_back(OrigArg.Regs[0], SplitVTs[0].getTypeForEVT(Ctx),
                            OrigArg.OrigArgIndex, OrigArg.Flags[0],
-                           OrigArg.IsFixed, OrigArg.OrigValue);
+                           OrigArg.OrigValue);
     return;
   }
 
@@ -313,7 +314,7 @@ void CallLowering::splitToValueTypes(const ArgInfo &OrigArg,
   for (unsigned i = 0, e = SplitVTs.size(); i < e; ++i) {
     Type *SplitTy = SplitVTs[i].getTypeForEVT(Ctx);
     SplitArgs.emplace_back(OrigArg.Regs[i], SplitTy, OrigArg.OrigArgIndex,
-                           OrigArg.Flags[0], OrigArg.IsFixed);
+                           OrigArg.Flags[0]);
     if (NeedsRegBlock)
       SplitArgs.back().Flags[0].setInConsecutiveRegs();
   }
@@ -1098,7 +1099,7 @@ bool CallLowering::checkReturn(CCState &CCInfo,
                                CCAssignFn *Fn) const {
   for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
     MVT VT = MVT::getVT(Outs[I].Ty);
-    if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo))
+    if (Fn(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], Outs[I].Ty, CCInfo))
       return false;
   }
   return true;
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index e84ba91c47c8b..0674f5fd1ae06 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1821,10 +1821,29 @@ bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI,
       return false;
   }
 
+  // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
+  // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
+  // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
+  // largest signed integer that fits into the index type, which is the maximum
+  // size of allocated objects according to the IR Language Reference.
+  unsigned PtrAddFlags = MI.getFlags();
+  unsigned LHSPtrAddFlags = Add2Def->getFlags();
+  bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
+  bool IsInBounds =
+      PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
+  unsigned Flags = 0;
+  if (IsNoUWrap)
+    Flags |= MachineInstr::MIFlag::NoUWrap;
+  if (IsInBounds) {
+    Flags |= MachineInstr::MIFlag::InBounds;
+    Flags |= MachineInstr::MIFlag::NoUSWrap;
+  }
+
   // Pass the combined immediate to the apply function.
   MatchInfo.Imm = AMNew.BaseOffs;
   MatchInfo.Base = Base;
   MatchInfo.Bank = getRegBank(Imm2);
+  MatchInfo.Flags = Flags;
   return true;
 }
 
@@ -1838,6 +1857,7 @@ void CombinerHelper::applyPtrAddImmedChain(MachineInstr &MI,
   Observer.changingInstr(MI);
   MI.getOperand(1).setReg(MatchInfo.Base);
   MI.getOperand(2).setReg(NewOffset.getReg(0));
+  MI.setFlags(MatchInfo.Flags);
   Observer.changedInstr(MI);
 }
 
@@ -4871,14 +4891,34 @@ bool CombinerHelper::matchReassocConstantInnerRHS(GPtrAdd &MI,
   if (!C2)
     return false;
 
+  // If both additions are nuw, the reassociated additions are also nuw.
+  // If the original G_PTR_ADD is additionally nusw, X and C are both not
+  // negative, so BASE+X is between BASE and BASE+(X+C). The new G_PTR_ADDs are
+  // therefore also nusw.
+  // If the original G_PTR_ADD is additionally inbounds (which implies nusw),
+  // the new G_PTR_ADDs are then also inbounds.
+  unsigned PtrAddFlags = MI.getFlags();
+  unsigned AddFlags = RHS->getFlags();
+  bool IsNoUWrap = PtrAddFlags & AddFlags & MachineInstr::MIFlag::NoUWrap;
+  bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::NoUSWrap);
+  bool IsInBounds = IsNoUWrap && (PtrAddFlags & MachineInstr::MIFlag::InBounds);
+  unsigned Flags = 0;
+  if (IsNoUWrap)
+    Flags |= MachineInstr::MIFlag::NoUWrap;
+  if (IsNoUSWrap)
+    Flags |= MachineInstr::MIFlag::NoUSWrap;
+  if (IsInBounds)
+    Flags |= MachineInstr::MIFlag::InBounds;
+
   MatchInfo = [=, &MI](MachineIRBuilder &B) {
     LLT PtrTy = MRI.getType(MI.getOperand(0).getReg());
 
     auto NewBase =
-        Builder.buildPtrAdd(PtrTy, Src1Reg, RHS->getOperand(1).getReg());
+        Builder.buildPtrAdd(PtrTy, Src1Reg, RHS->getOperand(1).getReg(), Flags);
     Observer.changingInstr(MI);
     MI.getOperand(1).setReg(NewBase.getReg(0));
     MI.getOperand(2).setReg(RHS->getOperand(2).getReg());
+    MI.setFlags(Flags);
     Observer.changedInstr(MI);
   };
   return !reassociationCanBreakAddressingModePattern(MI);
@@ -4897,6 +4937,25 @@ bool CombinerHelper::matchReassocConstantInnerLHS(GPtrAdd &MI,
     return false;
 
   auto *LHSPtrAdd = cast<GPtrAdd>(LHS);
+
+  // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
+  // nuw and inbounds (which implies nusw), the offsets are both non-negative,
+  // so the new G_PTR_ADDs are also inbounds.
+  unsigned PtrAddFlags = MI.getFlags();
+  unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
+  bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
+  bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
+                                  MachineInstr::MIFlag::NoUSWrap);
+  bool IsInBounds = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
+                                  MachineInstr::MIFlag::InBounds);
+  unsigned Flags = 0;
+  if (IsNoUWrap)
+    Flags |= MachineInstr::MIFlag::NoUWrap;
+  if (IsNoUSWrap)
+    Flags |= MachineInstr::MIFlag::NoUSWrap;
+  if (IsInBounds)
+    Flags |= MachineInstr::MIFlag::InBounds;
+
   MatchInfo = [=, &MI](MachineIRBuilder &B) {
     // When we change LHSPtrAdd's offset register we might cause it to use a reg
     // before its def. Sink the instruction so the outer PTR_ADD to ensure this
@@ -4907,9 +4966,11 @@ bool CombinerHelper::matchReassocConstantInnerLHS(GPtrAdd &MI,
     auto NewCst = B.buildConstant(MRI.getType(RHSReg), LHSCstOff->Value);
     Observer.changingInstr(MI);
     MI.getOperand(2).setReg(NewCst.getReg(0));
+    MI.setFlags(Flags);
     Observer.changedInstr(MI);
     Observer.changingInstr(*LHSPtrAdd);
     LHSPtrAdd->getOperand(2).setReg(RHSReg);
+    LHSPtrAdd->setFlags(Flags);
     Observer.changedInstr(*LHSPtrAdd);
   };
   return !reassociationCanBreakAddressingModePattern(MI);
@@ -4933,11 +4994,30 @@ bool CombinerHelper::matchReassocFoldConstantsInSubTree(
   if (!C2)
     return false;
 
+  // Reassociating nuw additions preserves nuw. If both original G_PTR_ADDs are
+  // inbounds, reaching the same result in one G_PTR_ADD is also inbounds.
+  // The nusw constraints are satisfied because imm1+imm2 cannot exceed the
+  // largest signed integer that fits into the index type, which is the maximum
+  // size of allocated objects according to the IR Language Reference.
+  unsigned PtrAddFlags = MI.getFlags();
+  unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
+  bool IsNoUWrap = PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::NoUWrap;
+  bool IsInBounds =
+      PtrAddFlags & LHSPtrAddFlags & MachineInstr::MIFlag::InBounds;
+  unsigned Flags = 0;
+  if (IsNoUWrap)
+    Flags |= MachineInstr::MIFlag::NoUWrap;
+  if (IsInBounds) {
+    Flags |= MachineInstr::MIFlag::InBounds;
+    Flags |= MachineInstr::MIFlag::NoUSWrap;
+  }
+
   MatchInfo = [=, &MI](MachineIRBuilder &B) {
     auto NewCst = B.buildConstant(MRI.getType(Src2Reg), *C1 + *C2);
     Observer.changingInstr(MI);
     MI.getOperand(1).setReg(LHSSrc1);
     MI.getOperand(2).setReg(NewCst.getReg(0));
+    MI.setFlags(Flags);
     Observer.changedInstr(MI);
   };
   return !reassociationCanBreakAddressingModePattern(MI);
@@ -5844,6 +5924,96 @@ void CombinerHelper::applyUMulHToLShr(MachineInstr &MI) const {
   MI.eraseFromParent();
 }
 
+bool CombinerHelper::matchTruncSSatS(MachineInstr &MI,
+                                     Register &MatchInfo) const {
+  Register Dst = MI.getOperand(0).getReg();
+  Register Src = MI.getOperand(1).getReg();
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+  unsigned NumDstBits = DstTy.getScalarSizeInBits();
+  unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
+  assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
+
+  if (!LI || !isLegal({TargetOpcode::G_TRUNC_SSAT_S, {DstTy, SrcTy}}))
+    return false;
+
+  APInt SignedMax = APInt::getSignedMaxValue(NumDstBits).sext(NumSrcBits);
+  APInt SignedMin = APInt::getSignedMinValue(NumDstBits).sext(NumSrcBits);
+  return mi_match(Src, MRI,
+                  m_GSMin(m_GSMax(m_Reg(MatchInfo),
+                                  m_SpecificICstOrSplat(SignedMin)),
+                          m_SpecificICstOrSplat(SignedMax))) ||
+         mi_match(Src, MRI,
+                  m_GSMax(m_GSMin(m_Reg(MatchInfo),
+                                  m_SpecificICstOrSplat(SignedMax)),
+                          m_SpecificICstOrSplat(SignedMin)));
+}
+
+void CombinerHelper::applyTruncSSatS(MachineInstr &MI,
+                                     Register &MatchInfo) const {
+  Register Dst = MI.getOperand(0).getReg();
+  Builder.buildTruncSSatS(Dst, MatchInfo);
+  MI.eraseFromParent();
+}
+
+bool CombinerHelper::matchTruncSSatU(MachineInstr &MI,
+                                     Register &MatchInfo) const {
+  Register Dst = MI.getOperand(0).getReg();
+  Register Src = MI.getOperand(1).getReg();
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+  unsigned NumDstBits = DstTy.getScalarSizeInBits();
+  unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
+  assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
+
+  if (!LI || !isLegal({TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
+    return false;
+  APInt UnsignedMax = APInt::getMaxValue(NumDstBits).zext(NumSrcBits);
+  return mi_match(Src, MRI,
+                  m_GSMin(m_GSMax(m_Reg(MatchInfo), m_SpecificICstOrSplat(0)),
+                          m_SpecificICstOrSplat(UnsignedMax))) ||
+         mi_match(Src, MRI,
+                  m_GSMax(m_GSMin(m_Reg(MatchInfo),
+                                  m_SpecificICstOrSplat(UnsignedMax)),
+                          m_SpecificICstOrSplat(0))) ||
+         mi_match(Src, MRI,
+                  m_GUMin(m_GSMax(m_Reg(MatchInfo), m_SpecificICstOrSplat(0)),
+                          m_SpecificICstOrSplat(UnsignedMax)));
+}
+
+void CombinerHelper::applyTruncSSatU(MachineInstr &MI,
+                                     Register &MatchInfo) const {
+  Register Dst = MI.getOperand(0).getReg();
+  Builder.buildTruncSSatU(Dst, MatchInfo);
+  MI.eraseFromParent();
+}
+
+bool CombinerHelper::matchTruncUSatU(MachineInstr &MI,
+                                     MachineInstr &MinMI) const {
+  Register Min = MinMI.getOperand(2).getReg();
+  Register Val = MinMI.getOperand(1).getReg();
+  LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+  LLT SrcTy = MRI.getType(Val);
+  unsigned NumDstBits = DstTy.getScalarSizeInBits();
+  unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
+  assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
+
+  if (!LI || !isLegal({TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
+    return false;
+  APInt UnsignedMax = APInt::getMaxValue(NumDstBits).zext(NumSrcBits);
+  return mi_match(Min, MRI, m_SpecificICstOrSplat(UnsignedMax)) &&
+         !mi_match(Val, MRI, m_GSMax(m_Reg(), m_Reg()));
+}
+
+bool CombinerHelper::matchTruncUSatUToFPTOUISat(MachineInstr &MI,
+                                                MachineInstr &SrcMI) const {
+  LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+  LLT SrcTy = MRI.getType(SrcMI.getOperand(1).getReg());
+
+  return LI &&
+         isLegalOrBeforeLegalizer({TargetOpcode::G_FPTOUI_SAT, {DstTy, SrcTy}});
+}
+
 bool CombinerHelper::matchRedundantNegOperands(MachineInstr &MI,
                                                BuildFnTy &MatchInfo) const {
   unsigned Opc = MI.getOpcode();
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index bbfae570e1e1a..541269ab6bfce 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -13,7 +13,6 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumptionCache.h"
@@ -2209,7 +2208,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
                                                   : TargetOpcode::LIFETIME_END;
 
-    const AllocaInst *AI = dyn_cast<AllocaInst>(CI.getArgOperand(1));
+    const AllocaInst *AI = dyn_cast<AllocaInst>(CI.getArgOperand(0));
     if (!AI || !AI->isStaticAlloca())
       return true;
 
@@ -2522,6 +2521,9 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
       Opc = ID == Intrinsic::vector_reduce_fadd
                 ? TargetOpcode::G_VECREDUCE_SEQ_FADD
                 : TargetOpcode::G_VECREDUCE_SEQ_FMUL;
+      if (!MRI->getType(VecSrc).isVector())
+        Opc = ID == Intrinsic::vector_reduce_fadd ? TargetOpcode::G_FADD
+                                                  : TargetOpcode::G_FMUL;
       MIRBuilder.buildInstr(Opc, {Dst}, {ScalarSrc, VecSrc},
                             MachineInstr::copyFlagsFromInstruction(CI));
       return true;
@@ -2556,6 +2558,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
                          getOrCreateVReg(*ConstantInt::getTrue(CI.getType())));
     return true;
   case Intrinsic::amdgcn_cs_chain:
+  case Intrinsic::amdgcn_call_whole_wave:
     return translateCallBase(CI, MIRBuilder);
   case Intrinsic::fptrunc_round: {
     uint32_t Flags = MachineInstr::copyFlagsFromInstruction(CI);
@@ -2786,11 +2789,14 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
   if (CI.isInlineAsm())
     return translateInlineAsm(CI, MIRBuilder);
 
-  diagnoseDontCall(CI);
-
   Intrinsic::ID ID = F ? F->getIntrinsicID() : Intrinsic::not_intrinsic;
-  if (!F || ID == Intrinsic::not_intrinsic)
-    return translateCallBase(CI, MIRBuilder);
+  if (!F || ID == Intrinsic::not_intrinsic) {
+    if (translateCallBase(CI, MIRBuilder)) {
+      diagnoseDontCall(CI);
+      return true;
+    }
+    return false;
+  }
 
   assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
 
@@ -3513,7 +3519,7 @@ void IRTranslator::finishPendingPhis() {
     Verifier.setCurrentInst(PI);
 #endif // ifndef NDEBUG
 
-    SmallSet<const MachineBasicBlock *, 16> SeenPreds;
+    SmallPtrSet<const MachineBasicBlock *, 16> SeenPreds;
     for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) {
       auto IRPred = PI->getIncomingBlock(i);
       ArrayRef<Register> ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i));
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index d9d3569affa39..008c18837a522 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -5574,12 +5574,19 @@ LegalizerHelper::fewerElementsBitcast(MachineInstr &MI, unsigned int TypeIdx,
 
   unsigned NewElemCount =
       NarrowTy.getSizeInBits() / SrcTy.getScalarSizeInBits();
-  LLT SrcNarrowTy = LLT::fixed_vector(NewElemCount, SrcTy.getElementType());
-
-  // Split the Src and Dst Reg into smaller registers
   SmallVector<Register> SrcVRegs, BitcastVRegs;
-  if (extractGCDType(SrcVRegs, DstTy, SrcNarrowTy, SrcReg) != SrcNarrowTy)
-    return UnableToLegalize;
+  if (NewElemCount == 1) {
+    LLT SrcNarrowTy = SrcTy.getElementType();
+
+    auto Unmerge = MIRBuilder.buildUnmerge(SrcNarrowTy, SrcReg);
+    getUnmergeResults(SrcVRegs, *Unmerge);
+  } else {
+    LLT SrcNarrowTy = LLT::fixed_vector(NewElemCount, SrcTy.getElementType());
+
+    // Split the Src and Dst Reg into smaller registers
+    if (extractGCDType(SrcVRegs, DstTy, SrcNarrowTy, SrcReg) != SrcNarrowTy)
+      return UnableToLegalize;
+  }
 
   // Build new smaller bitcast instructions
   // Not supporting Leftover types for now but will have to
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index 8955dd0370539..e41fd81953f43 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1869,8 +1869,10 @@ static bool canCreateUndefOrPoison(Register Reg, const MachineRegisterInfo &MRI,
   case TargetOpcode::G_FSHR:
   case TargetOpcode::G_SMAX:
   case TargetOpcode::G_SMIN:
+  case TargetOpcode::G_SCMP:
   case TargetOpcode::G_UMAX:
   case TargetOpcode::G_UMIN:
+  case TargetOpcode::G_UCMP:
   case TargetOpcode::G_PTRMASK:
   case TargetOpcode::G_SADDO:
   case TargetOpcode::G_SSUBO:
diff --git a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
index 73f11c1345daf..47640c4aac6df 100644
--- a/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
+++ b/llvm/lib/CodeGen/GlobalMergeFunctions.cpp
@@ -350,9 +350,8 @@ checkConstLocationCompatible(const StableFunctionMap::StableFunctionEntry &SF,
   return true;
 }
 
-static ParamLocsVecTy computeParamInfo(
-    const SmallVector<std::unique_ptr<StableFunctionMap::StableFunctionEntry>>
-        &SFS) {
+static ParamLocsVecTy
+computeParamInfo(const StableFunctionMap::StableFunctionEntries &SFS) {
   std::map<std::vector<stable_hash>, ParamLocs> HashSeqToLocs;
   auto &RSF = *SFS[0];
   unsigned StableFunctionCount = SFS.size();
@@ -396,19 +395,18 @@ bool GlobalMergeFunc::merge(Module &M, const StableFunctionMap *FunctionMap) {
   // Collect stable functions related to the current module.
   DenseMap<stable_hash, SmallVector<std::pair<Function *, FunctionHashInfo>>>
       HashToFuncs;
-  auto &Maps = FunctionMap->getFunctionMap();
   for (auto &F : M) {
     if (!isEligibleFunction(&F))
       continue;
     auto FI = llvm::StructuralHashWithDifferences(F, ignoreOp);
-    if (Maps.contains(FI.FunctionHash))
+    if (FunctionMap->contains(FI.FunctionHash))
       HashToFuncs[FI.FunctionHash].emplace_back(&F, std::move(FI));
   }
 
   for (auto &[Hash, Funcs] : HashToFuncs) {
     std::optional<ParamLocsVecTy> ParamLocsVec;
     SmallVector<FuncMergeInfo> FuncMergeInfos;
-    auto &SFS = Maps.at(Hash);
+    auto &SFS = FunctionMap->at(Hash);
     assert(!SFS.empty());
     auto &RFS = SFS[0];
 
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
index 5e508989ef2da..93f6e39b56ab6 100644
--- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -268,13 +268,16 @@ static Value *getMaskOperand(IntrinsicInst *II) {
   }
 }
 
-// Return the corresponded deinterleaved mask, or nullptr if there is no valid
-// mask.
-static Value *getMask(Value *WideMask, unsigned Factor,
-                      ElementCount LeafValueEC);
-
-static Value *getMask(Value *WideMask, unsigned Factor,
-                      VectorType *LeafValueTy) {
+// Return a pair of
+//  (1) The corresponded deinterleaved mask, or nullptr if there is no valid
+//  mask.
+//  (2) Some mask effectively skips a certain field, and this element is a mask
+//  in which inactive lanes represent fields that are skipped (i.e. "gaps").
+static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
+                                         ElementCount LeafValueEC);
+
+static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
+                                         VectorType *LeafValueTy) {
   return getMask(WideMask, Factor, LeafValueTy->getElementCount());
 }
 
@@ -379,22 +382,25 @@ bool InterleavedAccessImpl::lowerInterleavedLoad(
       replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, Load);
 
   Value *Mask = nullptr;
+  auto GapMask = APInt::getAllOnes(Factor);
   if (LI) {
     LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *Load << "\n");
   } else {
     // Check mask operand. Handle both all-true/false and interleaved mask.
-    Mask = getMask(getMaskOperand(II), Factor, VecTy);
+    std::tie(Mask, GapMask) = getMask(getMaskOperand(II), Factor, VecTy);
     if (!Mask)
       return false;
 
     LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.load or masked.load: "
                       << *Load << "\n");
+    LLVM_DEBUG(dbgs() << "IA: With nominal factor " << Factor
+                      << " and actual factor " << GapMask.popcount() << "\n");
   }
 
   // Try to create target specific intrinsics to replace the load and
   // shuffles.
   if (!TLI->lowerInterleavedLoad(cast<Instruction>(Load), Mask, Shuffles,
-                                 Indices, Factor))
+                                 Indices, Factor, GapMask))
     // If Extracts is not empty, tryReplaceExtracts made changes earlier.
     return !Extracts.empty() || BinOpShuffleChanged;
 
@@ -536,10 +542,15 @@ bool InterleavedAccessImpl::lowerInterleavedStore(
   } else {
     // Check mask operand. Handle both all-true/false and interleaved mask.
     unsigned LaneMaskLen = NumStoredElements / Factor;
-    Mask = getMask(getMaskOperand(II), Factor,
-                   ElementCount::getFixed(LaneMaskLen));
+    APInt GapMask(Factor, 0);
+    std::tie(Mask, GapMask) = getMask(getMaskOperand(II), Factor,
+                                      ElementCount::getFixed(LaneMaskLen));
     if (!Mask)
       return false;
+    // We haven't supported gap mask for stores. Yet it is possible that we
+    // already changed the IR, hence returning true here.
+    if (GapMask.popcount() != Factor)
+      return true;
 
     LLVM_DEBUG(dbgs() << "IA: Found an interleaved vp.store or masked.store: "
                       << *Store << "\n");
@@ -556,34 +567,97 @@ bool InterleavedAccessImpl::lowerInterleavedStore(
   return true;
 }
 
-static Value *getMask(Value *WideMask, unsigned Factor,
-                      ElementCount LeafValueEC) {
+// A wide mask <1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0> could be used to skip the
+// last field in a factor-of-three interleaved store or deinterleaved load (in
+// which case LeafMaskLen is 4). Such (wide) mask is also known as gap mask.
+// This helper function tries to detect this pattern and return the actual
+// factor we're accessing, which is 2 in this example.
+static void getGapMask(const Constant &MaskConst, unsigned Factor,
+                       unsigned LeafMaskLen, APInt &GapMask) {
+  assert(GapMask.getBitWidth() == Factor);
+  for (unsigned F = 0U; F < Factor; ++F) {
+    bool AllZero = true;
+    for (unsigned Idx = 0U; Idx < LeafMaskLen; ++Idx) {
+      Constant *C = MaskConst.getAggregateElement(F + Idx * Factor);
+      if (!C->isZeroValue()) {
+        AllZero = false;
+        break;
+      }
+    }
+    // All mask bits on this field are zero, skipping it.
+    if (AllZero)
+      GapMask.clearBit(F);
+  }
+}
+
+static std::pair<Value *, APInt> getMask(Value *WideMask, unsigned Factor,
+                                         ElementCount LeafValueEC) {
+  auto GapMask = APInt::getAllOnes(Factor);
+
   if (auto *IMI = dyn_cast<IntrinsicInst>(WideMask)) {
     if (unsigned F = getInterleaveIntrinsicFactor(IMI->getIntrinsicID());
-        F && F == Factor && llvm::all_equal(IMI->args())) {
-      return IMI->getArgOperand(0);
+        F && F == Factor) {
+      Value *RefArg = nullptr;
+      // Check if all the intrinsic arguments are the same, except those that
+      // are zeros, which we mark as gaps in the gap mask.
+      for (auto [Idx, Arg] : enumerate(IMI->args())) {
+        if (auto *C = dyn_cast<Constant>(Arg); C && C->isZeroValue()) {
+          GapMask.clearBit(Idx);
+          continue;
+        }
+
+        if (!RefArg)
+          RefArg = Arg;
+        else if (RefArg != Arg)
+          return {nullptr, GapMask};
+      }
+
+      // In a very rare occasion, all the intrinsic arguments might be zeros,
+      // in which case we still want to return an all-zeros constant instead of
+      // nullptr.
+      return {RefArg ? RefArg : IMI->getArgOperand(0), GapMask};
     }
   }
 
+  // Masks that are assembled from bitwise AND.
+  if (auto *AndOp = dyn_cast<BinaryOperator>(WideMask);
+      AndOp && AndOp->getOpcode() == Instruction::And) {
+    auto [MaskLHS, GapMaskLHS] =
+        getMask(AndOp->getOperand(0), Factor, LeafValueEC);
+    auto [MaskRHS, GapMaskRHS] =
+        getMask(AndOp->getOperand(1), Factor, LeafValueEC);
+    if (!MaskLHS || !MaskRHS)
+      return {nullptr, GapMask};
+    // Using IRBuilder here so that any trivial constants could be folded right
+    // away.
+    return {IRBuilder<>(AndOp).CreateAnd(MaskLHS, MaskRHS),
+            GapMaskLHS & GapMaskRHS};
+  }
+
   if (auto *ConstMask = dyn_cast<Constant>(WideMask)) {
     if (auto *Splat = ConstMask->getSplatValue())
       // All-ones or all-zeros mask.
-      return ConstantVector::getSplat(LeafValueEC, Splat);
+      return {ConstantVector::getSplat(LeafValueEC, Splat), GapMask};
 
     if (LeafValueEC.isFixed()) {
       unsigned LeafMaskLen = LeafValueEC.getFixedValue();
+      // First, check if we use a gap mask to skip some of the factors / fields.
+      getGapMask(*ConstMask, Factor, LeafMaskLen, GapMask);
+
       SmallVector<Constant *, 8> LeafMask(LeafMaskLen, nullptr);
       // If this is a fixed-length constant mask, each lane / leaf has to
       // use the same mask. This is done by checking if every group with Factor
       // number of elements in the interleaved mask has homogeneous values.
       for (unsigned Idx = 0U; Idx < LeafMaskLen * Factor; ++Idx) {
+        if (!GapMask[Idx % Factor])
+          continue;
         Constant *C = ConstMask->getAggregateElement(Idx);
         if (LeafMask[Idx / Factor] && LeafMask[Idx / Factor] != C)
-          return nullptr;
+          return {nullptr, GapMask};
         LeafMask[Idx / Factor] = C;
       }
 
-      return ConstantVector::get(LeafMask);
+      return {ConstantVector::get(LeafMask), GapMask};
     }
   }
 
@@ -603,12 +677,13 @@ static Value *getMask(Value *WideMask, unsigned Factor,
       auto *LeafMaskTy =
           VectorType::get(Type::getInt1Ty(SVI->getContext()), LeafValueEC);
       IRBuilder<> Builder(SVI);
-      return Builder.CreateExtractVector(LeafMaskTy, SVI->getOperand(0),
-                                         uint64_t(0));
+      return {Builder.CreateExtractVector(LeafMaskTy, SVI->getOperand(0),
+                                          uint64_t(0)),
+              GapMask};
     }
   }
 
-  return nullptr;
+  return {nullptr, GapMask};
 }
 
 bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
@@ -639,9 +714,16 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
       return false;
 
     // Check mask operand. Handle both all-true/false and interleaved mask.
-    Mask = getMask(getMaskOperand(II), Factor, getDeinterleavedVectorType(DI));
+    APInt GapMask(Factor, 0);
+    std::tie(Mask, GapMask) =
+        getMask(getMaskOperand(II), Factor, getDeinterleavedVectorType(DI));
     if (!Mask)
       return false;
+    // We haven't supported gap mask if it's deinterleaving using intrinsics.
+    // Yet it is possible that we already changed the IR, hence returning true
+    // here.
+    if (GapMask.popcount() != Factor)
+      return true;
 
     LLVM_DEBUG(dbgs() << "IA: Found a vp.load or masked.load with deinterleave"
                       << " intrinsic " << *DI << " and factor = "
@@ -680,10 +762,16 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic(
         II->getIntrinsicID() != Intrinsic::vp_store)
       return false;
     // Check mask operand. Handle both all-true/false and interleaved mask.
-    Mask = getMask(getMaskOperand(II), Factor,
-                   cast<VectorType>(InterleaveValues[0]->getType()));
+    APInt GapMask(Factor, 0);
+    std::tie(Mask, GapMask) =
+        getMask(getMaskOperand(II), Factor,
+                cast<VectorType>(InterleaveValues[0]->getType()));
     if (!Mask)
       return false;
+    // We haven't supported gap mask if it's interleaving using intrinsics. Yet
+    // it is possible that we already changed the IR, hence returning true here.
+    if (GapMask.popcount() != Factor)
+      return true;
 
     LLVM_DEBUG(dbgs() << "IA: Found a vp.store or masked.store with interleave"
                       << " intrinsic " << *IntII << " and factor = "
diff --git a/llvm/lib/CodeGen/LivePhysRegs.cpp b/llvm/lib/CodeGen/LivePhysRegs.cpp
index bc711382420be..5c8f060c09e5f 100644
--- a/llvm/lib/CodeGen/LivePhysRegs.cpp
+++ b/llvm/lib/CodeGen/LivePhysRegs.cpp
@@ -151,23 +151,34 @@ bool LivePhysRegs::available(const MachineRegisterInfo &MRI,
   return true;
 }
 
+/// Adds a register, taking associated lane masks into consideration.
+void LivePhysRegs::addRegMaskPair(
+    const MachineBasicBlock::RegisterMaskPair &Pair) {
+  MCRegister Reg = Pair.PhysReg;
+  LaneBitmask Mask = Pair.LaneMask;
+  MCSubRegIndexIterator S(Reg, TRI);
+  assert(Mask.any() && "Invalid livein mask");
+  if (Mask.all() || !S.isValid()) {
+    addReg(Reg);
+    return;
+  }
+  for (; S.isValid(); ++S) {
+    unsigned SI = S.getSubRegIndex();
+    if ((Mask & TRI->getSubRegIndexLaneMask(SI)).any())
+      addReg(S.getSubReg());
+  }
+}
+
 /// Add live-in registers of basic block \p MBB to \p LiveRegs.
 void LivePhysRegs::addBlockLiveIns(const MachineBasicBlock &MBB) {
-  for (const auto &LI : MBB.liveins()) {
-    MCRegister Reg = LI.PhysReg;
-    LaneBitmask Mask = LI.LaneMask;
-    MCSubRegIndexIterator S(Reg, TRI);
-    assert(Mask.any() && "Invalid livein mask");
-    if (Mask.all() || !S.isValid()) {
-      addReg(Reg);
-      continue;
-    }
-    for (; S.isValid(); ++S) {
-      unsigned SI = S.getSubRegIndex();
-      if ((Mask & TRI->getSubRegIndexLaneMask(SI)).any())
-        addReg(S.getSubReg());
-    }
-  }
+  for (const auto &LI : MBB.liveins())
+    addRegMaskPair(LI);
+}
+
+/// Add live-out registers of basic block \p MBB to \p LiveRegs.
+void LivePhysRegs::addBlockLiveOuts(const MachineBasicBlock &MBB) {
+  for (const auto &LO : MBB.liveouts())
+    addRegMaskPair(LO);
 }
 
 /// Adds all callee saved registers to \p LiveRegs.
@@ -207,9 +218,7 @@ void LivePhysRegs::addPristines(const MachineFunction &MF) {
 }
 
 void LivePhysRegs::addLiveOutsNoPristines(const MachineBasicBlock &MBB) {
-  // To get the live-outs we simply merge the live-ins of all successors.
-  for (const MachineBasicBlock *Succ : MBB.successors())
-    addBlockLiveIns(*Succ);
+  addBlockLiveOuts(MBB);
   if (MBB.isReturnBlock()) {
     // Return blocks are a special case because we currently don't mark up
     // return instructions completely: specifically, there is no explicit
@@ -356,8 +365,8 @@ bool llvm::isPhysRegUsedAfter(Register Reg, MachineBasicBlock::iterator MBI) {
 
   // If we hit the end of the block, check whether Reg is live into a
   //  successor.
-  for (MachineBasicBlock *Succ : MBB->successors())
-    if (Succ->isLiveIn(Reg))
+  for (const auto &LO : MBB->liveouts())
+    if (LO.PhysReg == Reg.asMCReg() && LO.LaneMask.any())
       return true;
 
   return false;
diff --git a/llvm/lib/CodeGen/LiveRangeEdit.cpp b/llvm/lib/CodeGen/LiveRangeEdit.cpp
index a3858efbdc5e1..5514e4eb6cf3e 100644
--- a/llvm/lib/CodeGen/LiveRangeEdit.cpp
+++ b/llvm/lib/CodeGen/LiveRangeEdit.cpp
@@ -190,9 +190,12 @@ SlotIndex LiveRangeEdit::rematerializeAt(MachineBasicBlock &MBB,
   Rematted.insert(RM.ParentVNI);
   ++NumReMaterialization;
 
+  bool EarlyClobber = MI->getOperand(0).isEarlyClobber();
   if (ReplaceIndexMI)
-    return LIS.ReplaceMachineInstrInMaps(*ReplaceIndexMI, *MI).getRegSlot();
-  return LIS.getSlotIndexes()->insertMachineInstrInMaps(*MI, Late).getRegSlot();
+    return LIS.ReplaceMachineInstrInMaps(*ReplaceIndexMI, *MI)
+        .getRegSlot(EarlyClobber);
+  return LIS.getSlotIndexes()->insertMachineInstrInMaps(*MI, Late).getRegSlot(
+      EarlyClobber);
 }
 
 void LiveRangeEdit::eraseVirtReg(Register Reg) {
diff --git a/llvm/lib/CodeGen/LiveRegUnits.cpp b/llvm/lib/CodeGen/LiveRegUnits.cpp
index 34de09dd2944b..0d87062169585 100644
--- a/llvm/lib/CodeGen/LiveRegUnits.cpp
+++ b/llvm/lib/CodeGen/LiveRegUnits.cpp
@@ -91,6 +91,13 @@ static void addBlockLiveIns(LiveRegUnits &LiveUnits,
     LiveUnits.addRegMasked(LI.PhysReg, LI.LaneMask);
 }
 
+/// Add live-out registers of basic block \p MBB to \p LiveUnits.
+static void addBlockLiveOuts(LiveRegUnits &LiveUnits,
+                             const MachineBasicBlock &MBB) {
+  for (const auto &LO : MBB.liveouts())
+    LiveUnits.addRegMasked(LO.PhysReg, LO.LaneMask);
+}
+
 /// Adds all callee saved registers to \p LiveUnits.
 static void addCalleeSavedRegs(LiveRegUnits &LiveUnits,
                                const MachineFunction &MF) {
@@ -137,12 +144,8 @@ void LiveRegUnits::addPristines(const MachineFunction &MF) {
 
 void LiveRegUnits::addLiveOuts(const MachineBasicBlock &MBB) {
   const MachineFunction &MF = *MBB.getParent();
-
   addPristines(MF);
-
-  // To get the live-outs we simply merge the live-ins of all successors.
-  for (const MachineBasicBlock *Succ : MBB.successors())
-    addBlockLiveIns(*this, *Succ);
+  addBlockLiveOuts(*this, MBB);
 
   // For the return block: Add all callee saved registers.
   if (MBB.isReturnBlock()) {
diff --git a/llvm/lib/CodeGen/LiveVariables.cpp b/llvm/lib/CodeGen/LiveVariables.cpp
index 1f23418642bc6..c5dfddaa21e66 100644
--- a/llvm/lib/CodeGen/LiveVariables.cpp
+++ b/llvm/lib/CodeGen/LiveVariables.cpp
@@ -213,11 +213,7 @@ void LiveVariables::HandleVirtRegDef(Register Reg, MachineInstr &MI) {
 }
 
 /// FindLastPartialDef - Return the last partial def of the specified register.
-/// Also returns the sub-registers that're defined by the instruction.
-MachineInstr *
-LiveVariables::FindLastPartialDef(Register Reg,
-                                  SmallSet<Register, 4> &PartDefRegs) {
-  Register LastDefReg = 0;
+MachineInstr *LiveVariables::FindLastPartialDef(Register Reg) {
   unsigned LastDefDist = 0;
   MachineInstr *LastDef = nullptr;
   for (MCPhysReg SubReg : TRI->subregs(Reg)) {
@@ -226,7 +222,6 @@ LiveVariables::FindLastPartialDef(Register Reg,
       continue;
     unsigned Dist = DistanceMap[Def];
     if (Dist > LastDefDist) {
-      LastDefReg  = SubReg;
       LastDef     = Def;
       LastDefDist = Dist;
     }
@@ -235,14 +230,6 @@ LiveVariables::FindLastPartialDef(Register Reg,
   if (!LastDef)
     return nullptr;
 
-  PartDefRegs.insert(LastDefReg);
-  for (MachineOperand &MO : LastDef->all_defs()) {
-    if (MO.getReg() == 0)
-      continue;
-    Register DefReg = MO.getReg();
-    if (TRI->isSubRegister(Reg, DefReg))
-      PartDefRegs.insert_range(TRI->subregs_inclusive(DefReg));
-  }
   return LastDef;
 }
 
@@ -261,27 +248,11 @@ void LiveVariables::HandlePhysRegUse(Register Reg, MachineInstr &MI) {
     // ...
     //    = EAX
     // All of the sub-registers must have been defined before the use of Reg!
-    SmallSet<Register, 4> PartDefRegs;
-    MachineInstr *LastPartialDef = FindLastPartialDef(Reg, PartDefRegs);
+    MachineInstr *LastPartialDef = FindLastPartialDef(Reg);
     // If LastPartialDef is NULL, it must be using a livein register.
     if (LastPartialDef) {
-      LastPartialDef->addOperand(MachineOperand::CreateReg(Reg, true/*IsDef*/,
-                                                           true/*IsImp*/));
-      PhysRegDef[Reg.id()] = LastPartialDef;
-      SmallSet<MCPhysReg, 8> Processed;
-      for (MCPhysReg SubReg : TRI->subregs(Reg)) {
-        if (Processed.count(SubReg))
-          continue;
-        if (PartDefRegs.count(SubReg))
-          continue;
-        // This part of Reg was defined before the last partial def. It's killed
-        // here.
-        LastPartialDef->addOperand(MachineOperand::CreateReg(SubReg,
-                                                             false/*IsDef*/,
-                                                             true/*IsImp*/));
-        PhysRegDef[SubReg] = LastPartialDef;
-        Processed.insert_range(TRI->subregs(SubReg));
-      }
+      LastPartialDef->addOperand(
+          MachineOperand::CreateReg(Reg, /*IsDef=*/true, /*IsImp=*/true));
     }
   } else if (LastDef && !PhysRegUse[Reg.id()] &&
              !LastDef->findRegisterDefOperand(Reg, /*TRI=*/nullptr))
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 3e99e576fa364..bb70e7805e818 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -124,6 +124,11 @@ class MIRParserImpl {
   bool initializeFrameInfo(PerFunctionMIParsingState &PFS,
                            const yaml::MachineFunction &YamlMF);
 
+  bool initializeSaveRestorePoints(
+      PerFunctionMIParsingState &PFS,
+      const std::vector<yaml::SaveRestorePointEntry> &YamlSRPoints,
+      SmallVectorImpl<MachineBasicBlock *> &SaveRestorePoints);
+
   bool initializeCallSiteInfo(PerFunctionMIParsingState &PFS,
                               const yaml::MachineFunction &YamlMF);
 
@@ -529,7 +534,7 @@ void MIRParserImpl::setupDebugValueTracking(
   unsigned MaxInstrNum = 0;
   for (auto &MBB : MF)
     for (auto &MI : MBB)
-      MaxInstrNum = std::max((unsigned)MI.peekDebugInstrNum(), MaxInstrNum);
+      MaxInstrNum = std::max(MI.peekDebugInstrNum(), MaxInstrNum);
   MF.setDebugInstrNumberingCount(MaxInstrNum);
 
   // Load any substitutions.
@@ -867,18 +872,14 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS,
   MFI.setHasTailCall(YamlMFI.HasTailCall);
   MFI.setCalleeSavedInfoValid(YamlMFI.IsCalleeSavedInfoValid);
   MFI.setLocalFrameSize(YamlMFI.LocalFrameSize);
-  if (!YamlMFI.SavePoint.Value.empty()) {
-    MachineBasicBlock *MBB = nullptr;
-    if (parseMBBReference(PFS, MBB, YamlMFI.SavePoint))
-      return true;
-    MFI.setSavePoint(MBB);
-  }
-  if (!YamlMFI.RestorePoint.Value.empty()) {
-    MachineBasicBlock *MBB = nullptr;
-    if (parseMBBReference(PFS, MBB, YamlMFI.RestorePoint))
-      return true;
-    MFI.setRestorePoint(MBB);
-  }
+  SmallVector<MachineBasicBlock *, 4> SavePoints;
+  if (initializeSaveRestorePoints(PFS, YamlMFI.SavePoints, SavePoints))
+    return true;
+  MFI.setSavePoints(SavePoints);
+  SmallVector<MachineBasicBlock *, 4> RestorePoints;
+  if (initializeSaveRestorePoints(PFS, YamlMFI.RestorePoints, RestorePoints))
+    return true;
+  MFI.setRestorePoints(RestorePoints);
 
   std::vector<CalleeSavedInfo> CSIInfo;
   // Initialize the fixed frame objects.
@@ -1093,6 +1094,21 @@ bool MIRParserImpl::initializeConstantPool(PerFunctionMIParsingState &PFS,
   return false;
 }
 
+// Return true if basic block was incorrectly specified in MIR
+bool MIRParserImpl::initializeSaveRestorePoints(
+    PerFunctionMIParsingState &PFS,
+    const std::vector<yaml::SaveRestorePointEntry> &YamlSRPoints,
+    SmallVectorImpl<MachineBasicBlock *> &SaveRestorePoints) {
+  MachineBasicBlock *MBB = nullptr;
+  for (const yaml::SaveRestorePointEntry &Entry : YamlSRPoints) {
+    if (parseMBBReference(PFS, MBB, Entry.Point.Value))
+      return true;
+    SaveRestorePoints.push_back(MBB);
+  }
+
+  return false;
+}
+
 bool MIRParserImpl::initializeJumpTableInfo(PerFunctionMIParsingState &PFS,
     const yaml::MachineJumpTable &YamlJTI) {
   MachineJumpTableInfo *JTI = PFS.MF.getOrCreateJumpTableInfo(YamlJTI.Kind);
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index ce1834a90ca54..7cc91925793a1 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -150,6 +150,10 @@ static void convertMJTI(ModuleSlotTracker &MST, yaml::MachineJumpTable &YamlJTI,
                         const MachineJumpTableInfo &JTI);
 static void convertMFI(ModuleSlotTracker &MST, yaml::MachineFrameInfo &YamlMFI,
                        const MachineFrameInfo &MFI);
+static void
+convertSRPoints(ModuleSlotTracker &MST,
+                std::vector<yaml::SaveRestorePointEntry> &YamlSRPoints,
+                ArrayRef<MachineBasicBlock *> SaveRestorePoints);
 static void convertStackObjects(yaml::MachineFunction &YMF,
                                 const MachineFunction &MF,
                                 ModuleSlotTracker &MST, MFPrintState &State);
@@ -355,14 +359,10 @@ static void convertMFI(ModuleSlotTracker &MST, yaml::MachineFrameInfo &YamlMFI,
   YamlMFI.HasTailCall = MFI.hasTailCall();
   YamlMFI.IsCalleeSavedInfoValid = MFI.isCalleeSavedInfoValid();
   YamlMFI.LocalFrameSize = MFI.getLocalFrameSize();
-  if (MFI.getSavePoint()) {
-    raw_string_ostream StrOS(YamlMFI.SavePoint.Value);
-    StrOS << printMBBReference(*MFI.getSavePoint());
-  }
-  if (MFI.getRestorePoint()) {
-    raw_string_ostream StrOS(YamlMFI.RestorePoint.Value);
-    StrOS << printMBBReference(*MFI.getRestorePoint());
-  }
+  if (!MFI.getSavePoints().empty())
+    convertSRPoints(MST, YamlMFI.SavePoints, MFI.getSavePoints());
+  if (!MFI.getRestorePoints().empty())
+    convertSRPoints(MST, YamlMFI.RestorePoints, MFI.getRestorePoints());
 }
 
 static void convertEntryValueObjects(yaml::MachineFunction &YMF,
@@ -616,6 +616,21 @@ static void convertMCP(yaml::MachineFunction &MF,
   }
 }
 
+static void
+convertSRPoints(ModuleSlotTracker &MST,
+                std::vector<yaml::SaveRestorePointEntry> &YamlSRPoints,
+                ArrayRef<MachineBasicBlock *> SRPoints) {
+  for (const auto &MBB : SRPoints) {
+    SmallString<16> Str;
+    yaml::SaveRestorePointEntry Entry;
+    raw_svector_ostream StrOS(Str);
+    StrOS << printMBBReference(*MBB);
+    Entry.Point = StrOS.str().str();
+    Str.clear();
+    YamlSRPoints.push_back(Entry);
+  }
+}
+
 static void convertMJTI(ModuleSlotTracker &MST, yaml::MachineJumpTable &YamlJTI,
                         const MachineJumpTableInfo &JTI) {
   YamlJTI.Kind = JTI.getEntryKind();
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index c3c5a0f5102d7..08a51b9b0242a 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -1781,9 +1781,6 @@ MachineBasicBlock::livein_iterator MachineBasicBlock::livein_begin() const {
 
 MachineBasicBlock::liveout_iterator MachineBasicBlock::liveout_begin() const {
   const MachineFunction &MF = *getParent();
-  assert(MF.getProperties().hasTracksLiveness() &&
-         "Liveness information is accurate");
-
   const TargetLowering &TLI = *MF.getSubtarget().getTargetLowering();
   MCRegister ExceptionPointer, ExceptionSelector;
   if (MF.getFunction().hasPersonalityFn()) {
diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
index 742de1101faa2..e35983138550f 100644
--- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
@@ -490,7 +490,7 @@ class MachineCopyPropagation {
   SmallSetVector<MachineInstr *, 8> MaybeDeadCopies;
 
   /// Multimap tracking debug users in current BB
-  DenseMap<MachineInstr *, SmallSet<MachineInstr *, 2>> CopyDbgUsers;
+  DenseMap<MachineInstr *, SmallPtrSet<MachineInstr *, 2>> CopyDbgUsers;
 
   CopyTracker Tracker;
 
diff --git a/llvm/lib/CodeGen/MachineDebugify.cpp b/llvm/lib/CodeGen/MachineDebugify.cpp
index 1a20fe586e951..c3d436ded60a8 100644
--- a/llvm/lib/CodeGen/MachineDebugify.cpp
+++ b/llvm/lib/CodeGen/MachineDebugify.cpp
@@ -14,7 +14,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
@@ -87,7 +86,7 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
   // Do this by introducing debug uses of each register definition. If that is
   // not possible (e.g. we have a phi or a meta instruction), emit a constant.
   uint64_t NextImm = 0;
-  SmallSet<DILocalVariable *, 16> VarSet;
+  SmallPtrSet<DILocalVariable *, 16> VarSet;
   const MCInstrDesc &DbgValDesc = TII.get(TargetOpcode::DBG_VALUE);
   for (MachineBasicBlock &MBB : MF) {
     MachineBasicBlock::iterator FirstNonPHIIt = MBB.getFirstNonPHI();
diff --git a/llvm/lib/CodeGen/MachineFrameInfo.cpp b/llvm/lib/CodeGen/MachineFrameInfo.cpp
index e4b993850f73d..a8306b2ef2e5b 100644
--- a/llvm/lib/CodeGen/MachineFrameInfo.cpp
+++ b/llvm/lib/CodeGen/MachineFrameInfo.cpp
@@ -244,6 +244,22 @@ void MachineFrameInfo::print(const MachineFunction &MF, raw_ostream &OS) const{
     }
     OS << "\n";
   }
+  OS << "save/restore points:\n";
+
+  if (!SavePoints.empty()) {
+    OS << "save points:\n";
+
+    for (auto &item : SavePoints)
+      OS << printMBBReference(*item) << "\n";
+  } else
+    OS << "save points are empty\n";
+
+  if (!RestorePoints.empty()) {
+    OS << "restore points:\n";
+    for (auto &item : RestorePoints)
+      OS << printMBBReference(*item) << "\n";
+  } else
+    OS << "restore points are empty\n";
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index ec40f6af3caae..224231cb90fe2 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -154,17 +154,17 @@ void ilist_alloc_traits<MachineBasicBlock>::deleteNode(MachineBasicBlock *MBB) {
   MBB->getParent()->deleteMachineBasicBlock(MBB);
 }
 
-static inline Align getFnStackAlignment(const TargetSubtargetInfo *STI,
-                                           const Function &F) {
+static inline Align getFnStackAlignment(const TargetSubtargetInfo &STI,
+                                        const Function &F) {
   if (auto MA = F.getFnStackAlign())
     return *MA;
-  return STI->getFrameLowering()->getStackAlign();
+  return STI.getFrameLowering()->getStackAlign();
 }
 
 MachineFunction::MachineFunction(Function &F, const TargetMachine &Target,
                                  const TargetSubtargetInfo &STI, MCContext &Ctx,
                                  unsigned FunctionNum)
-    : F(F), Target(Target), STI(&STI), Ctx(Ctx) {
+    : F(F), Target(Target), STI(STI), Ctx(Ctx) {
   FunctionNumber = FunctionNum;
   init();
 }
@@ -195,7 +195,7 @@ void MachineFunction::init() {
 
   // We can realign the stack if the target supports it and the user hasn't
   // explicitly asked us not to.
-  bool CanRealignSP = STI->getFrameLowering()->isStackRealignable() &&
+  bool CanRealignSP = STI.getFrameLowering()->isStackRealignable() &&
                       !F.hasFnAttribute("no-realign-stack");
   bool ForceRealignSP = F.hasFnAttribute(Attribute::StackAlignment) ||
                         F.hasFnAttribute("stackrealign");
@@ -209,11 +209,12 @@ void MachineFunction::init() {
     FrameInfo->ensureMaxAlignment(*F.getFnStackAlign());
 
   ConstantPool = new (Allocator) MachineConstantPool(getDataLayout());
-  Alignment = STI->getTargetLowering()->getMinFunctionAlignment();
+  Alignment = STI.getTargetLowering()->getMinFunctionAlignment();
 
-  if (!F.getAlign() && !F.hasOptSize())
+  // FIXME: Shouldn't use pref alignment if explicit alignment is set on F.
+  if (!F.hasOptSize())
     Alignment = std::max(Alignment,
-                         STI->getTargetLowering()->getPrefFunctionAlignment());
+                         STI.getTargetLowering()->getPrefFunctionAlignment());
 
   // -fsanitize=function and -fsanitize=kcfi instrument indirect function calls
   // to load a type hash before the function label. Ensure functions are aligned
diff --git a/llvm/lib/CodeGen/MachineInstrBundle.cpp b/llvm/lib/CodeGen/MachineInstrBundle.cpp
index 4da0184a0cc76..d9e8484c08d76 100644
--- a/llvm/lib/CodeGen/MachineInstrBundle.cpp
+++ b/llvm/lib/CodeGen/MachineInstrBundle.cpp
@@ -94,6 +94,22 @@ static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI,
   return DebugLoc();
 }
 
+/// Check if target reg is contained in given lists, which are:
+/// LocalDefsV as given list for virtual regs
+/// LocalDefsP as given list for physical regs, in BitVector[RegUnit] form
+static bool containsReg(SmallSetVector<Register, 32> LocalDefsV,
+                        const BitVector &LocalDefsP, Register Reg,
+                        const TargetRegisterInfo *TRI) {
+  if (Reg.isPhysical()) {
+    for (MCRegUnit Unit : TRI->regunits(Reg.asMCReg()))
+      if (!LocalDefsP[Unit])
+        return false;
+
+    return true;
+  }
+  return LocalDefsV.contains(Reg);
+}
+
 /// finalizeBundle - Finalize a machine instruction bundle which includes
 /// a sequence of instructions starting from FirstMI to LastMI (exclusive).
 /// This routine adds a BUNDLE instruction to represent the bundle, it adds
@@ -115,6 +131,7 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB,
   Bundle.prepend(MIB);
 
   SmallSetVector<Register, 32> LocalDefs;
+  BitVector LocalDefsP(TRI->getNumRegUnits());
   SmallSet<Register, 8> DeadDefSet;
   SmallSet<Register, 16> KilledDefSet;
   SmallSetVector<Register, 8> ExternUses;
@@ -130,7 +147,7 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB,
       if (!Reg)
         continue;
 
-      if (LocalDefs.contains(Reg)) {
+      if (containsReg(LocalDefs, LocalDefsP, Reg, TRI)) {
         MO.setIsInternalRead();
         if (MO.isKill()) {
           // Internal def is now killed.
@@ -165,8 +182,10 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB,
         }
       }
 
-      if (!MO.isDead() && Reg.isPhysical())
-        LocalDefs.insert_range(TRI->subregs(Reg));
+      if (!MO.isDead() && Reg.isPhysical()) {
+        for (MCRegUnit Unit : TRI->regunits(Reg.asMCReg()))
+          LocalDefsP.set(Unit);
+      }
     }
 
     // Set FrameSetup/FrameDestroy for the bundle. If any of the instructions
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 90005bd181f3a..3a9651c5cee04 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -3466,9 +3466,9 @@ bool SMSchedule::onlyHasLoopCarriedOutputOrOrderPreds(
 }
 
 /// Determine transitive dependences of unpipelineable instructions
-SmallSet<SUnit *, 8> SMSchedule::computeUnpipelineableNodes(
+SmallPtrSet<SUnit *, 8> SMSchedule::computeUnpipelineableNodes(
     SwingSchedulerDAG *SSD, TargetInstrInfo::PipelinerLoopInfo *PLI) {
-  SmallSet<SUnit *, 8> DoNotPipeline;
+  SmallPtrSet<SUnit *, 8> DoNotPipeline;
   SmallVector<SUnit *, 8> Worklist;
 
   for (auto &SU : SSD->SUnits)
@@ -3498,7 +3498,7 @@ SmallSet<SUnit *, 8> SMSchedule::computeUnpipelineableNodes(
 // and ensure that they are in stage 0.  If unable to do so, return false.
 bool SMSchedule::normalizeNonPipelinedInstructions(
     SwingSchedulerDAG *SSD, TargetInstrInfo::PipelinerLoopInfo *PLI) {
-  SmallSet<SUnit *, 8> DNP = computeUnpipelineableNodes(SSD, PLI);
+  SmallPtrSet<SUnit *, 8> DNP = computeUnpipelineableNodes(SSD, PLI);
 
   int NewLastCycle = INT_MIN;
   for (SUnit &SU : SSD->SUnits) {
diff --git a/llvm/lib/CodeGen/MacroFusion.cpp b/llvm/lib/CodeGen/MacroFusion.cpp
index 975a3fe71abad..1db53017e6cef 100644
--- a/llvm/lib/CodeGen/MacroFusion.cpp
+++ b/llvm/lib/CodeGen/MacroFusion.cpp
@@ -79,7 +79,7 @@ bool llvm::fuseInstructionPair(ScheduleDAGInstrs &DAG, SUnit &FirstSU,
   FirstSU.ParentClusterIdx = Clusters.size();
   SecondSU.ParentClusterIdx = Clusters.size();
 
-  SmallSet<SUnit *, 8> Cluster{{&FirstSU, &SecondSU}};
+  SmallPtrSet<SUnit *, 8> Cluster{{&FirstSU, &SecondSU}};
   Clusters.push_back(Cluster);
 
   // TODO - If we want to chain more than two instructions, we need to create
diff --git a/llvm/lib/CodeGen/PHIElimination.cpp b/llvm/lib/CodeGen/PHIElimination.cpp
index a93a89ecaa96e..34a9d5d0e401f 100644
--- a/llvm/lib/CodeGen/PHIElimination.cpp
+++ b/llvm/lib/CodeGen/PHIElimination.cpp
@@ -30,6 +30,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
@@ -72,6 +73,7 @@ class PHIEliminationImpl {
   LiveIntervals *LIS = nullptr;
   MachineLoopInfo *MLI = nullptr;
   MachineDominatorTree *MDT = nullptr;
+  MachinePostDominatorTree *PDT = nullptr;
 
   /// EliminatePHINodes - Eliminate phi nodes by inserting copy instructions
   /// in predecessor basic blocks.
@@ -123,17 +125,22 @@ class PHIEliminationImpl {
     auto *MLIWrapper = P->getAnalysisIfAvailable<MachineLoopInfoWrapperPass>();
     auto *MDTWrapper =
         P->getAnalysisIfAvailable<MachineDominatorTreeWrapperPass>();
+    auto *PDTWrapper =
+        P->getAnalysisIfAvailable<MachinePostDominatorTreeWrapperPass>();
     LV = LVWrapper ? &LVWrapper->getLV() : nullptr;
     LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
     MLI = MLIWrapper ? &MLIWrapper->getLI() : nullptr;
     MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr;
+    PDT = PDTWrapper ? &PDTWrapper->getPostDomTree() : nullptr;
   }
 
   PHIEliminationImpl(MachineFunction &MF, MachineFunctionAnalysisManager &AM)
       : LV(AM.getCachedResult<LiveVariablesAnalysis>(MF)),
         LIS(AM.getCachedResult<LiveIntervalsAnalysis>(MF)),
         MLI(AM.getCachedResult<MachineLoopAnalysis>(MF)),
-        MDT(AM.getCachedResult<MachineDominatorTreeAnalysis>(MF)), MFAM(&AM) {}
+        MDT(AM.getCachedResult<MachineDominatorTreeAnalysis>(MF)),
+        PDT(AM.getCachedResult<MachinePostDominatorTreeAnalysis>(MF)),
+        MFAM(&AM) {}
 
   bool run(MachineFunction &MF);
 };
@@ -172,6 +179,7 @@ PHIEliminationPass::run(MachineFunction &MF,
   PA.preserve<LiveVariablesAnalysis>();
   PA.preserve<SlotIndexesAnalysis>();
   PA.preserve<MachineDominatorTreeAnalysis>();
+  PA.preserve<MachinePostDominatorTreeAnalysis>();
   PA.preserve<MachineLoopAnalysis>();
   return PA;
 }
@@ -197,6 +205,7 @@ void PHIElimination::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addPreserved<SlotIndexesWrapperPass>();
   AU.addPreserved<LiveIntervalsWrapperPass>();
   AU.addPreserved<MachineDominatorTreeWrapperPass>();
+  AU.addPreserved<MachinePostDominatorTreeWrapperPass>();
   AU.addPreserved<MachineLoopInfoWrapperPass>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
@@ -204,15 +213,8 @@ void PHIElimination::getAnalysisUsage(AnalysisUsage &AU) const {
 bool PHIEliminationImpl::run(MachineFunction &MF) {
   MRI = &MF.getRegInfo();
 
-  MachineDominatorTree *MDT = nullptr;
-  if (P) {
-    auto *MDTWrapper =
-        P->getAnalysisIfAvailable<MachineDominatorTreeWrapperPass>();
-    MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr;
-  } else {
-    MDT = MFAM->getCachedResult<MachineDominatorTreeAnalysis>(MF);
-  }
-  MachineDomTreeUpdater MDTU(MDT, MachineDomTreeUpdater::UpdateStrategy::Lazy);
+  MachineDomTreeUpdater MDTU(MDT, PDT,
+                             MachineDomTreeUpdater::UpdateStrategy::Lazy);
 
   bool Changed = false;
 
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 8de2c48581a1e..96c9cde622b45 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -145,7 +145,7 @@ static bool lowerObjCCall(Function &F, RTLIB::LibcallImpl NewFn,
 
   // FIXME: When RuntimeLibcalls is an analysis, check if the function is really
   // supported, and go through RTLIB::Libcall.
-  const char *NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(NewFn);
+  StringRef NewFnName = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(NewFn);
 
   // If we haven't already looked up this function, check to see if the
   // program already contains a function with this name.
@@ -587,12 +587,14 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
       break;
     case Intrinsic::exp:
     case Intrinsic::exp2:
+    case Intrinsic::log:
       Changed |= forEachCall(F, [&](CallInst *CI) {
         Type *Ty = CI->getArgOperand(0)->getType();
         if (!isa<ScalableVectorType>(Ty))
           return false;
         const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();
         unsigned Op = TL->IntrinsicIDToISD(F.getIntrinsicID());
+        assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
         if (!TL->isOperationExpand(Op, EVT::getEVT(Ty)))
           return false;
         return lowerUnaryVectorIntrinsicAsLoop(M, CI);
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index f66f54682c84c..8fc0748ecc0e3 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -351,8 +351,8 @@ bool PEIImpl::run(MachineFunction &MF) {
   delete RS;
   SaveBlocks.clear();
   RestoreBlocks.clear();
-  MFI.setSavePoint(nullptr);
-  MFI.setRestorePoint(nullptr);
+  MFI.setSavePoints({});
+  MFI.setRestorePoints({});
   return true;
 }
 
@@ -423,16 +423,18 @@ void PEIImpl::calculateCallFrameInfo(MachineFunction &MF) {
 /// callee-saved registers, and placing prolog and epilog code.
 void PEIImpl::calculateSaveRestoreBlocks(MachineFunction &MF) {
   const MachineFrameInfo &MFI = MF.getFrameInfo();
-
   // Even when we do not change any CSR, we still want to insert the
   // prologue and epilogue of the function.
   // So set the save points for those.
 
   // Use the points found by shrink-wrapping, if any.
-  if (MFI.getSavePoint()) {
-    SaveBlocks.push_back(MFI.getSavePoint());
-    assert(MFI.getRestorePoint() && "Both restore and save must be set");
-    MachineBasicBlock *RestoreBlock = MFI.getRestorePoint();
+  if (!MFI.getSavePoints().empty()) {
+    assert(MFI.getSavePoints().size() == 1 &&
+           "Multiple save points are not yet supported!");
+    SaveBlocks.push_back(MFI.getSavePoints().front());
+    assert(MFI.getRestorePoints().size() == 1 &&
+           "Multiple restore points are not yet supported!");
+    MachineBasicBlock *RestoreBlock = MFI.getRestorePoints().front();
     // If RestoreBlock does not have any successor and is not a return block
     // then the end point is unreachable and we do not need to insert any
     // epilogue.
@@ -558,7 +560,11 @@ static void updateLiveness(MachineFunction &MF) {
   SmallPtrSet<MachineBasicBlock *, 8> Visited;
   SmallVector<MachineBasicBlock *, 8> WorkList;
   MachineBasicBlock *Entry = &MF.front();
-  MachineBasicBlock *Save = MFI.getSavePoint();
+
+  assert(MFI.getSavePoints().size() < 2 &&
+         "Multiple save points not yet supported!");
+  MachineBasicBlock *Save =
+      MFI.getSavePoints().empty() ? nullptr : MFI.getSavePoints().front();
 
   if (!Save)
     Save = Entry;
@@ -569,7 +575,10 @@ static void updateLiveness(MachineFunction &MF) {
   }
   Visited.insert(Save);
 
-  MachineBasicBlock *Restore = MFI.getRestorePoint();
+  assert(MFI.getRestorePoints().size() < 2 &&
+         "Multiple restore points not yet supported!");
+  MachineBasicBlock *Restore =
+      MFI.getRestorePoints().empty() ? nullptr : MFI.getRestorePoints().front();
   if (Restore)
     // By construction Restore cannot be visited, otherwise it
     // means there exists a path to Restore that does not go
@@ -1550,7 +1559,7 @@ void PEIImpl::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
       // If this instruction has a FrameIndex operand, we need to
       // use that target machine register info object to eliminate
       // it.
-      TRI.eliminateFrameIndex(MI, SPAdj, i);
+      TRI.eliminateFrameIndex(MI, SPAdj, i, RS);
 
       // Reset the iterator if we were at the beginning of the BB.
       if (AtBeginning) {
diff --git a/llvm/lib/CodeGen/RegAllocFast.cpp b/llvm/lib/CodeGen/RegAllocFast.cpp
index 66a206c9b2344..804480cdd8e6f 100644
--- a/llvm/lib/CodeGen/RegAllocFast.cpp
+++ b/llvm/lib/CodeGen/RegAllocFast.cpp
@@ -474,6 +474,13 @@ int RegAllocFastImpl::getStackSpaceFor(Register VirtReg) {
   const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
   unsigned Size = TRI->getSpillSize(RC);
   Align Alignment = TRI->getSpillAlign(RC);
+
+  const MachineFunction &MF = MRI->getMF();
+  auto &ST = MF.getSubtarget();
+  Align CurrentAlign = ST.getFrameLowering()->getStackAlign();
+  if (Alignment > CurrentAlign && !TRI->canRealignStack(MF))
+    Alignment = CurrentAlign;
+
   int FrameIdx = MFI->CreateSpillStackObject(Size, Alignment);
 
   // Assign the slot.
diff --git a/llvm/lib/CodeGen/RegisterPressure.cpp b/llvm/lib/CodeGen/RegisterPressure.cpp
index ca51b670b46cc..5f3789050b813 100644
--- a/llvm/lib/CodeGen/RegisterPressure.cpp
+++ b/llvm/lib/CodeGen/RegisterPressure.cpp
@@ -1001,7 +1001,7 @@ static void computeMaxPressureDelta(ArrayRef<unsigned> OldMaxPressureVec,
         ++CritIdx;
 
       if (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() == i) {
-        int PDiff = (int)PNew - (int)CriticalPSets[CritIdx].getUnitInc();
+        int PDiff = (int)PNew - CriticalPSets[CritIdx].getUnitInc();
         if (PDiff > 0) {
           Delta.CriticalMax = PressureChange(i);
           Delta.CriticalMax.setUnitInc(PDiff);
@@ -1191,7 +1191,7 @@ getUpwardPressureDelta(const MachineInstr *MI, /*const*/ PressureDiff &PDiff,
         ++CritIdx;
 
       if (CritIdx != CritEnd && CriticalPSets[CritIdx].getPSet() == PSetID) {
-        int CritInc = (int)MNew - (int)CriticalPSets[CritIdx].getUnitInc();
+        int CritInc = (int)MNew - CriticalPSets[CritIdx].getUnitInc();
         if (CritInc > 0 && CritInc <= std::numeric_limits<int16_t>::max()) {
           Delta.CriticalMax = PressureChange(PSetID);
           Delta.CriticalMax.setUnitInc(CritInc);
diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp
index 908ed96172615..6f373a53020d1 100644
--- a/llvm/lib/CodeGen/SafeStack.cpp
+++ b/llvm/lib/CodeGen/SafeStack.cpp
@@ -262,7 +262,7 @@ bool SafeStack::IsMemIntrinsicSafe(const MemIntrinsic *MI, const Use &U,
       return true;
   }
 
-  const auto *Len = dyn_cast<ConstantInt>(MI->getLength());
+  auto Len = MI->getLengthInBytes();
   // Non-constant size => unsafe. FIXME: try SCEV getRange.
   if (!Len) return false;
   return IsAccessSafe(U, Len->getZExtValue(), AllocaPtr, AllocaSize);
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index d70e96938ed9a..27b5a0d37b679 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -401,7 +401,7 @@ namespace {
     SDValue PromoteExtend(SDValue Op);
     bool PromoteLoad(SDValue Op);
 
-    SDValue foldShiftToAvg(SDNode *N);
+    SDValue foldShiftToAvg(SDNode *N, const SDLoc &DL);
     // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
     SDValue foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT);
 
@@ -6070,6 +6070,16 @@ SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
   if (N0 == N1)
     return N0;
 
+  // Fold operation with vscale operands.
+  if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
+    uint64_t C0 = N0->getConstantOperandVal(0);
+    uint64_t C1 = N1->getConstantOperandVal(0);
+    if (Opcode == ISD::UMAX)
+      return C0 > C1 ? N0 : N1;
+    else if (Opcode == ISD::UMIN)
+      return C0 > C1 ? N1 : N0;
+  }
+
   // canonicalize constant to RHS
   if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
       !DAG.isConstantIntBuildVectorOrConstantInt(N1))
@@ -9390,8 +9400,7 @@ SDValue DAGCombiner::mergeTruncStores(StoreSDNode *N) {
   LLVMContext &Context = *DAG.getContext();
   unsigned NumStores = Stores.size();
   unsigned WideNumBits = NumStores * NarrowNumBits;
-  EVT WideVT = EVT::getIntegerVT(Context, WideNumBits);
-  if (WideVT != MVT::i16 && WideVT != MVT::i32 && WideVT != MVT::i64)
+  if (WideNumBits != 16 && WideNumBits != 32 && WideNumBits != 64)
     return SDValue();
 
   // Check if all bytes of the source value that we are looking at are stored
@@ -9445,7 +9454,7 @@ SDValue DAGCombiner::mergeTruncStores(StoreSDNode *N) {
         SourceValue = WideVal;
 
       // Give up if the source value type is smaller than the store size.
-      if (SourceValue.getScalarValueSizeInBits() < WideVT.getScalarSizeInBits())
+      if (SourceValue.getScalarValueSizeInBits() < WideNumBits)
         return SDValue();
     }
 
@@ -9469,6 +9478,8 @@ SDValue DAGCombiner::mergeTruncStores(StoreSDNode *N) {
     OffsetMap[Offset] = ByteOffsetFromBase;
   }
 
+  EVT WideVT = EVT::getIntegerVT(Context, WideNumBits);
+
   assert(FirstOffset != INT64_MAX && "First byte offset must be set");
   assert(FirstStore && "First store must be set");
 
@@ -10982,7 +10993,7 @@ SDValue DAGCombiner::visitSRA(SDNode *N) {
   if (SDValue NarrowLoad = reduceLoadWidth(N))
     return NarrowLoad;
 
-  if (SDValue AVG = foldShiftToAvg(N))
+  if (SDValue AVG = foldShiftToAvg(N, DL))
     return AVG;
 
   return SDValue();
@@ -11255,7 +11266,7 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
   if (SDValue MULH = combineShiftToMULH(N, DL, DAG, TLI))
     return MULH;
 
-  if (SDValue AVG = foldShiftToAvg(N))
+  if (SDValue AVG = foldShiftToAvg(N, DL))
     return AVG;
 
   return SDValue();
@@ -11270,6 +11281,11 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
   unsigned BitWidth = VT.getScalarSizeInBits();
   SDLoc DL(N);
 
+  // fold (fshl/fshr C0, C1, C2) -> C3
+  if (SDValue C =
+          DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1, N2}))
+    return C;
+
   // fold (fshl N0, N1, 0) -> N0
   // fold (fshr N0, N1, 0) -> N1
   if (isPowerOf2_32(BitWidth))
@@ -11771,51 +11787,36 @@ static SDValue combineMinNumMaxNumImpl(const SDLoc &DL, EVT VT, SDValue LHS,
   }
 }
 
-SDValue DAGCombiner::foldShiftToAvg(SDNode *N) {
+// Convert (sr[al] (add n[su]w x, y)) -> (avgfloor[su] x, y)
+SDValue DAGCombiner::foldShiftToAvg(SDNode *N, const SDLoc &DL) {
   const unsigned Opcode = N->getOpcode();
-
-  // Convert (sr[al] (add n[su]w x, y)) -> (avgfloor[su] x, y)
   if (Opcode != ISD::SRA && Opcode != ISD::SRL)
     return SDValue();
 
-  unsigned FloorISD = 0;
-  auto VT = N->getValueType(0);
-  bool IsUnsigned = false;
-
-  // Decide wether signed or unsigned.
-  switch (Opcode) {
-  case ISD::SRA:
-    if (!hasOperation(ISD::AVGFLOORS, VT))
-      return SDValue();
-    FloorISD = ISD::AVGFLOORS;
-    break;
-  case ISD::SRL:
-    IsUnsigned = true;
-    if (!hasOperation(ISD::AVGFLOORU, VT))
-      return SDValue();
-    FloorISD = ISD::AVGFLOORU;
-    break;
-  default:
-    return SDValue();
-  }
+  EVT VT = N->getValueType(0);
+  bool IsUnsigned = Opcode == ISD::SRL;
 
   // Captured values.
   SDValue A, B, Add;
 
   // Match floor average as it is common to both floor/ceil avgs.
-  if (!sd_match(N, m_BinOp(Opcode,
-                           m_AllOf(m_Value(Add), m_Add(m_Value(A), m_Value(B))),
-                           m_One())))
-    return SDValue();
+  if (sd_match(N, m_BinOp(Opcode,
+                          m_AllOf(m_Value(Add), m_Add(m_Value(A), m_Value(B))),
+                          m_One()))) {
+    // Decide whether signed or unsigned.
+    unsigned FloorISD = IsUnsigned ? ISD::AVGFLOORU : ISD::AVGFLOORS;
+    if (!hasOperation(FloorISD, VT))
+      return SDValue();
 
-  // Can't optimize adds that may wrap.
-  if (IsUnsigned && !Add->getFlags().hasNoUnsignedWrap())
-    return SDValue();
+    // Can't optimize adds that may wrap.
+    if ((IsUnsigned && !Add->getFlags().hasNoUnsignedWrap()) ||
+        (!IsUnsigned && !Add->getFlags().hasNoSignedWrap()))
+      return SDValue();
 
-  if (!IsUnsigned && !Add->getFlags().hasNoSignedWrap())
-    return SDValue();
+    return DAG.getNode(FloorISD, DL, N->getValueType(0), {A, B});
+  }
 
-  return DAG.getNode(FloorISD, SDLoc(N), N->getValueType(0), {A, B});
+  return SDValue();
 }
 
 SDValue DAGCombiner::foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT) {
@@ -12842,22 +12843,21 @@ SDValue DAGCombiner::visitMHISTOGRAM(SDNode *N) {
   SDLoc DL(HG);
 
   EVT MemVT = HG->getMemoryVT();
+  EVT DataVT = Index.getValueType();
   MachineMemOperand *MMO = HG->getMemOperand();
   ISD::MemIndexType IndexType = HG->getIndexType();
 
   if (ISD::isConstantSplatVectorAllZeros(Mask.getNode()))
     return Chain;
 
-  SDValue Ops[] = {Chain,          Inc,           Mask, BasePtr, Index,
-                   HG->getScale(), HG->getIntID()};
-  if (refineUniformBase(BasePtr, Index, HG->isIndexScaled(), DAG, DL))
+  if (refineUniformBase(BasePtr, Index, HG->isIndexScaled(), DAG, DL) ||
+      refineIndexType(Index, IndexType, DataVT, DAG)) {
+    SDValue Ops[] = {Chain,          Inc,           Mask, BasePtr, Index,
+                     HG->getScale(), HG->getIntID()};
     return DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), MemVT, DL, Ops,
                                   MMO, IndexType);
+  }
 
-  EVT DataVT = Index.getValueType();
-  if (refineIndexType(Index, IndexType, DataVT, DAG))
-    return DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), MemVT, DL, Ops,
-                                  MMO, IndexType);
   return SDValue();
 }
 
@@ -16342,6 +16342,42 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
                                  DAG, DL);
     }
     break;
+  case ISD::AVGFLOORS:
+  case ISD::AVGFLOORU:
+  case ISD::AVGCEILS:
+  case ISD::AVGCEILU:
+  case ISD::ABDS:
+  case ISD::ABDU:
+    // (trunc (avg a, b)) -> (avg (trunc a), (trunc b))
+    // (trunc (abdu/abds a, b)) -> (abdu/abds (trunc a), (trunc b))
+    if (!LegalOperations && N0.hasOneUse() &&
+        TLI.isOperationLegal(N0.getOpcode(), VT)) {
+      EVT TruncVT = VT;
+      unsigned SrcBits = SrcVT.getScalarSizeInBits();
+      unsigned TruncBits = TruncVT.getScalarSizeInBits();
+
+      SDValue A = N0.getOperand(0);
+      SDValue B = N0.getOperand(1);
+      bool CanFold = false;
+
+      if (N0.getOpcode() == ISD::AVGFLOORU || N0.getOpcode() == ISD::AVGCEILU ||
+          N0.getOpcode() == ISD::ABDU) {
+        APInt UpperBits = APInt::getBitsSetFrom(SrcBits, TruncBits);
+        CanFold = DAG.MaskedValueIsZero(B, UpperBits) &&
+                  DAG.MaskedValueIsZero(A, UpperBits);
+      } else {
+        unsigned NeededBits = SrcBits - TruncBits;
+        CanFold = DAG.ComputeNumSignBits(B) > NeededBits &&
+                  DAG.ComputeNumSignBits(A) > NeededBits;
+      }
+
+      if (CanFold) {
+        SDValue NewA = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, A);
+        SDValue NewB = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, B);
+        return DAG.getNode(N0.getOpcode(), DL, TruncVT, NewA, NewB);
+      }
+    }
+    break;
   }
 
   return SDValue();
@@ -16998,8 +17034,7 @@ SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
   // fadd (G, (fma A, B, (fma (C, D, (fmul (E, F)))))) -->
   // fma A, B, (fma C, D, fma (E, F, G)).
   // This requires reassociation because it changes the order of operations.
-  bool CanReassociate =
-      Options.UnsafeFPMath || N->getFlags().hasAllowReassociation();
+  bool CanReassociate = N->getFlags().hasAllowReassociation();
   if (CanReassociate) {
     SDValue FMA, E;
     if (isFusedOp(N0) && N0.hasOneUse()) {
@@ -17665,7 +17700,7 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
   // If 'unsafe math' or reassoc and nsz, fold lots of things.
   // TODO: break out portions of the transformations below for which Unsafe is
   //       considered and which do not require both nsz and reassoc
-  if (((Options.UnsafeFPMath && Options.NoSignedZerosFPMath) ||
+  if ((Options.NoSignedZerosFPMath ||
        (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros())) &&
       AllowNewConst) {
     // fadd (fadd x, c1), c2 -> fadd x, c1 + c2
@@ -17752,7 +17787,7 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
     }
   } // enable-unsafe-fp-math && AllowNewConst
 
-  if (((Options.UnsafeFPMath && Options.NoSignedZerosFPMath) ||
+  if ((Options.NoSignedZerosFPMath ||
        (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros()))) {
     // Fold fadd(vecreduce(x), vecreduce(y)) -> vecreduce(fadd(x, y))
     if (SDValue SD = reassociateReduction(ISD::VECREDUCE_FADD, ISD::FADD, DL,
@@ -17855,7 +17890,7 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
     }
   }
 
-  if (((Options.UnsafeFPMath && Options.NoSignedZerosFPMath) ||
+  if ((Options.NoSignedZerosFPMath ||
        (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros())) &&
       N1.getOpcode() == ISD::FADD) {
     // X - (X + Y) -> -Y
@@ -17995,7 +18030,6 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
   ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, true);
   EVT VT = N->getValueType(0);
   SDLoc DL(N);
-  const TargetOptions &Options = DAG.getTarget().Options;
   const SDNodeFlags Flags = N->getFlags();
   SelectionDAG::FlagInserter FlagsInserter(DAG, N);
 
@@ -18019,7 +18053,7 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
-  if (Options.UnsafeFPMath || Flags.hasAllowReassociation()) {
+  if (Flags.hasAllowReassociation()) {
     // fmul (fmul X, C1), C2 -> fmul X, C1 * C2
     if (DAG.isConstantFPBuildVectorOrConstantFP(N1) &&
         N0.getOpcode() == ISD::FMUL) {
@@ -18196,8 +18230,7 @@ template <class MatchContextClass> SDValue DAGCombiner::visitFMA(SDNode *N) {
      !DAG.isConstantFPBuildVectorOrConstantFP(N1))
     return matcher.getNode(ISD::FMA, DL, VT, N1, N0, N2);
 
-  bool CanReassociate =
-      Options.UnsafeFPMath || N->getFlags().hasAllowReassociation();
+  bool CanReassociate = N->getFlags().hasAllowReassociation();
   if (CanReassociate) {
     // (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2)
     if (matcher.match(N2, ISD::FMUL) && N0 == N2.getOperand(0) &&
@@ -18292,9 +18325,8 @@ SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
   // TODO: Limit this transform based on optsize/minsize - it always creates at
   //       least 1 extra instruction. But the perf win may be substantial enough
   //       that only minsize should restrict this.
-  bool UnsafeMath = DAG.getTarget().Options.UnsafeFPMath;
   const SDNodeFlags Flags = N->getFlags();
-  if (LegalDAG || (!UnsafeMath && !Flags.hasAllowReciprocal()))
+  if (LegalDAG || !Flags.hasAllowReciprocal())
     return SDValue();
 
   // Skip if current node is a reciprocal/fneg-reciprocal.
@@ -18331,7 +18363,7 @@ SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
 
       // This division is eligible for optimization only if global unsafe math
       // is enabled or if this division allows reciprocal formation.
-      if (UnsafeMath || U->getFlags().hasAllowReciprocal())
+      if (U->getFlags().hasAllowReciprocal())
         Users.insert(U);
     }
   }
@@ -18956,7 +18988,9 @@ SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
     // single-step fp_round we want to fold to.
     // In other words, double rounding isn't the same as rounding.
     // Also, this is a value preserving truncation iff both fp_round's are.
-    if (DAG.getTarget().Options.UnsafeFPMath || N0IsTrunc)
+    if ((N->getFlags().hasAllowContract() &&
+         N0->getFlags().hasAllowContract()) ||
+        N0IsTrunc)
       return DAG.getNode(
           ISD::FP_ROUND, DL, VT, N0.getOperand(0),
           DAG.getIntPtrConstant(NIsTrunc && N0IsTrunc, DL, /*isTarget=*/true));
@@ -25986,7 +26020,10 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
   // Combine an extract of an extract into a single extract_subvector.
   // ext (ext X, C), 0 --> ext X, C
   if (ExtIdx == 0 && V.getOpcode() == ISD::EXTRACT_SUBVECTOR && V.hasOneUse()) {
-    if (TLI.isExtractSubvectorCheap(NVT, V.getOperand(0).getValueType(),
+    // The index has to be a multiple of the new result type's known minimum
+    // vector length.
+    if (V.getConstantOperandVal(1) % NVT.getVectorMinNumElements() == 0 &&
+        TLI.isExtractSubvectorCheap(NVT, V.getOperand(0).getValueType(),
                                     V.getConstantOperandVal(1)) &&
         TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NVT)) {
       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, V.getOperand(0),
@@ -26073,8 +26110,6 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
     EVT ConcatSrcVT = V.getOperand(0).getValueType();
     assert(ConcatSrcVT.getVectorElementType() == NVT.getVectorElementType() &&
            "Concat and extract subvector do not change element type");
-    assert((ExtIdx % ExtNumElts) == 0 &&
-           "Extract index is not a multiple of the input vector length.");
 
     unsigned ConcatSrcNumElts = ConcatSrcVT.getVectorMinNumElements();
     unsigned ConcatOpIdx = ExtIdx / ConcatSrcNumElts;
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index fb9eff942a464..9467ba14cf895 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -729,9 +729,7 @@ bool FastISel::lowerCallOperands(const CallInst *CI, unsigned ArgIdx,
 
     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
 
-    ArgListEntry Entry;
-    Entry.Val = V;
-    Entry.Ty = V->getType();
+    ArgListEntry Entry(V);
     Entry.setAttributes(CI, ArgI);
     Args.push_back(Entry);
   }
@@ -978,9 +976,7 @@ bool FastISel::lowerCallTo(const CallInst *CI, MCSymbol *Symbol,
 
     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
 
-    ArgListEntry Entry;
-    Entry.Val = V;
-    Entry.Ty = V->getType();
+    ArgListEntry Entry(V);
     Entry.setAttributes(CI, ArgI);
     Args.push_back(Entry);
   }
@@ -1012,17 +1008,16 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) {
     MVT RegisterVT = TLI.getRegisterType(CLI.RetTy->getContext(), VT);
     unsigned NumRegs = TLI.getNumRegisters(CLI.RetTy->getContext(), VT);
     for (unsigned i = 0; i != NumRegs; ++i) {
-      ISD::InputArg MyFlags;
-      MyFlags.VT = RegisterVT;
-      MyFlags.ArgVT = VT;
-      MyFlags.Used = CLI.IsReturnValueUsed;
+      ISD::ArgFlagsTy Flags;
       if (CLI.RetSExt)
-        MyFlags.Flags.setSExt();
+        Flags.setSExt();
       if (CLI.RetZExt)
-        MyFlags.Flags.setZExt();
+        Flags.setZExt();
       if (CLI.IsInReg)
-        MyFlags.Flags.setInReg();
-      CLI.Ins.push_back(MyFlags);
+        Flags.setInReg();
+      ISD::InputArg Ret(Flags, RegisterVT, VT, CLI.RetTy, CLI.IsReturnValueUsed,
+                        ISD::InputArg::NoArgIndex, 0);
+      CLI.Ins.push_back(Ret);
     }
   }
 
@@ -1117,7 +1112,6 @@ bool FastISel::lowerCall(const CallInst *CI) {
   Type *RetTy = CI->getType();
 
   ArgListTy Args;
-  ArgListEntry Entry;
   Args.reserve(CI->arg_size());
 
   for (auto i = CI->arg_begin(), e = CI->arg_end(); i != e; ++i) {
@@ -1127,9 +1121,7 @@ bool FastISel::lowerCall(const CallInst *CI) {
     if (V->getType()->isEmptyTy())
       continue;
 
-    Entry.Val = V;
-    Entry.Ty = V->getType();
-
+    ArgListEntry Entry(V);
     // Skip the first return-type Attribute to get to params.
     Entry.setAttributes(CI, i - CI->arg_begin());
     Args.push_back(Entry);
@@ -1148,9 +1140,12 @@ bool FastISel::lowerCall(const CallInst *CI) {
   CLI.setCallee(RetTy, FuncTy, CI->getCalledOperand(), std::move(Args), *CI)
       .setTailCall(IsTailCall);
 
-  diagnoseDontCall(*CI);
+  if (lowerCallTo(CLI)) {
+    diagnoseDontCall(*CI);
+    return true;
+  }
 
-  return lowerCallTo(CLI);
+  return false;
 }
 
 bool FastISel::selectCall(const User *I) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 8c8daef6dccd4..861f76e93f2ce 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -81,12 +81,11 @@ static unsigned countOperands(SDNode *Node, unsigned NumExpUses,
 
 /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
 /// implicit physical register output.
-void InstrEmitter::EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone,
-                                   Register SrcReg, VRBaseMapType &VRBaseMap) {
+void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
+                                   VRBaseMapType &VRBaseMap) {
   Register VRBase;
   if (SrcReg.isVirtual()) {
     // Just use the input register directly!
-    SDValue Op(Node, ResNo);
     if (IsClone)
       VRBaseMap.erase(Op);
     bool isNew = VRBaseMap.insert(std::make_pair(Op, SrcReg)).second;
@@ -99,17 +98,15 @@ void InstrEmitter::EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone,
   // the CopyToReg'd destination register instead of creating a new vreg.
   bool MatchReg = true;
   const TargetRegisterClass *UseRC = nullptr;
-  MVT VT = Node->getSimpleValueType(ResNo);
+  MVT VT = Op.getSimpleValueType();
 
   // Stick to the preferred register classes for legal types.
   if (TLI->isTypeLegal(VT))
-    UseRC = TLI->getRegClassFor(VT, Node->isDivergent());
+    UseRC = TLI->getRegClassFor(VT, Op->isDivergent());
 
-  for (SDNode *User : Node->users()) {
+  for (SDNode *User : Op->users()) {
     bool Match = true;
-    if (User->getOpcode() == ISD::CopyToReg &&
-        User->getOperand(2).getNode() == Node &&
-        User->getOperand(2).getResNo() == ResNo) {
+    if (User->getOpcode() == ISD::CopyToReg && User->getOperand(2) == Op) {
       Register DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
       if (DestReg.isVirtual()) {
         VRBase = DestReg;
@@ -118,10 +115,8 @@ void InstrEmitter::EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone,
         Match = false;
     } else {
       for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
-        SDValue Op = User->getOperand(i);
-        if (Op.getNode() != Node || Op.getResNo() != ResNo)
+        if (User->getOperand(i) != Op)
           continue;
-        MVT VT = Node->getSimpleValueType(Op.getResNo());
         if (VT == MVT::Other || VT == MVT::Glue)
           continue;
         Match = false;
@@ -170,11 +165,11 @@ void InstrEmitter::EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone,
   } else {
     // Create the reg, emit the copy.
     VRBase = MRI->createVirtualRegister(DstRC);
-    BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
-            VRBase).addReg(SrcReg);
+    BuildMI(*MBB, InsertPos, Op.getDebugLoc(), TII->get(TargetOpcode::COPY),
+            VRBase)
+        .addReg(SrcReg);
   }
 
-  SDValue Op(Node, ResNo);
   if (IsClone)
     VRBaseMap.erase(Op);
   bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
@@ -243,7 +238,7 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node,
 
     // Create the result registers for this node and add the result regs to
     // the machine instruction.
-    if (VRBase == 0) {
+    if (!VRBase) {
       assert(RC && "Isn't a register operand!");
       VRBase = MRI->createVirtualRegister(RC);
       MIB.addReg(VRBase, RegState::Define);
@@ -1170,7 +1165,7 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
         continue;
       // This implicitly defined physreg has a use.
       UsedRegs.push_back(Reg);
-      EmitCopyFromReg(Node, i, IsClone, Reg, VRBaseMap);
+      EmitCopyFromReg(SDValue(Node, i), IsClone, Reg, VRBaseMap);
     }
   }
 
@@ -1178,7 +1173,9 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
   if (Node->getValueType(Node->getNumValues()-1) == MVT::Glue) {
     for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
       if (F->getOpcode() == ISD::CopyFromReg) {
-        UsedRegs.push_back(cast<RegisterSDNode>(F->getOperand(1))->getReg());
+        Register Reg = cast<RegisterSDNode>(F->getOperand(1))->getReg();
+        if (Reg.isPhysical())
+          UsedRegs.push_back(Reg);
         continue;
       } else if (F->getOpcode() == ISD::CopyToReg) {
         // Skip CopyToReg nodes that are internal to the glue chain.
@@ -1281,7 +1278,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
   }
   case ISD::CopyFromReg: {
     Register SrcReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
-    EmitCopyFromReg(Node, 0, IsClone, SrcReg, VRBaseMap);
+    EmitCopyFromReg(SDValue(Node, 0), IsClone, SrcReg, VRBaseMap);
     break;
   }
   case ISD::EH_LABEL:
diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
index 16d754cdc2338..b465de8397c78 100644
--- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
+++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.h
@@ -48,8 +48,8 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter {
 
   /// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
   /// implicit physical register output.
-  void EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone,
-                       Register SrcReg, VRBaseMapType &VRBaseMap);
+  void EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
+                       VRBaseMapType &VRBaseMap);
 
   void CreateVirtualRegisters(SDNode *Node,
                               MachineInstrBuilder &MIB,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index ba0ab2383d87a..bcfc2c5dc9f83 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -2181,12 +2181,10 @@ SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
 std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
                                             bool isSigned) {
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
   for (const SDValue &Op : Node->op_values()) {
     EVT ArgVT = Op.getValueType();
     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-    Entry.Node = Op;
-    Entry.Ty = ArgTy;
+    TargetLowering::ArgListEntry Entry(Op, ArgTy);
     Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgTy, isSigned);
     Entry.IsZExt = !Entry.IsSExt;
     Args.push_back(Entry);
@@ -2325,11 +2323,9 @@ SDValue SelectionDAGLegalize::ExpandBitCountingLibCall(
   EVT IntVT =
       EVT::getIntegerVT(*DAG.getContext(), DAG.getLibInfo().getIntSize());
 
-  TargetLowering::ArgListEntry Arg;
   EVT ArgVT = Op.getValueType();
   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-  Arg.Node = Op;
-  Arg.Ty = ArgTy;
+  TargetLowering::ArgListEntry Arg(Op, ArgTy);
   Arg.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgTy, /*IsSigned=*/false);
   Arg.IsZExt = !Arg.IsSExt;
 
@@ -2370,12 +2366,10 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
 
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
   for (const SDValue &Op : Node->op_values()) {
     EVT ArgVT = Op.getValueType();
     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-    Entry.Node = Op;
-    Entry.Ty = ArgTy;
+    TargetLowering::ArgListEntry Entry(Op, ArgTy);
     Entry.IsSExt = isSigned;
     Entry.IsZExt = !isSigned;
     Args.push_back(Entry);
@@ -2383,8 +2377,8 @@ SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
 
   // Also pass the return address of the remainder.
   SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
-  Entry.Node = FIPtr;
-  Entry.Ty = PointerType::getUnqual(RetTy->getContext());
+  TargetLowering::ArgListEntry Entry(
+      FIPtr, PointerType::getUnqual(RetTy->getContext()));
   Entry.IsSExt = isSigned;
   Entry.IsZExt = !isSigned;
   Args.push_back(Entry);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 2cad36eff9c88..83bb1dfe86c6a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -197,7 +197,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
   SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
   TargetLowering::MakeLibCallOptions CallOptions;
   EVT OpVT = N->getOperand(0 + Offset).getValueType();
-  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
+  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
                                                     CallOptions, SDLoc(N),
                                                     Chain);
@@ -218,7 +218,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
   TargetLowering::MakeLibCallOptions CallOptions;
   EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
                    N->getOperand(1 + Offset).getValueType() };
-  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
+  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0));
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
                                                     CallOptions, SDLoc(N),
                                                     Chain);
@@ -558,7 +558,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
   EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
                    N->getOperand(1 + Offset).getValueType(),
                    N->getOperand(2 + Offset).getValueType() };
-  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
+  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0));
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
                                                     GetFPLibCall(N->getValueType(0),
                                                                  RTLIB::FMA_F32,
@@ -642,7 +642,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
   TargetLowering::MakeLibCallOptions CallOptions;
   EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
-  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
+  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
                                                     CallOptions, SDLoc(N),
                                                     Chain);
@@ -658,7 +658,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
   SDValue Op = N->getOperand(0);
   TargetLowering::MakeLibCallOptions CallOptions;
   EVT OpsVT[1] = { N->getOperand(0).getValueType() };
-  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
+  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0));
   SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
                                   CallOptions, SDLoc(N)).first;
   if (N->getValueType(0) == MVT::f32)
@@ -694,7 +694,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
   assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
   TargetLowering::MakeLibCallOptions CallOptions;
   EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
-  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
+  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
                                                     CallOptions, SDLoc(N),
                                                     Chain);
@@ -742,7 +742,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
   TargetLowering::MakeLibCallOptions CallOptions;
   EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
                    N->getOperand(1 + Offset).getValueType() };
-  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
+  CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0));
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
                                                     CallOptions, SDLoc(N),
                                                     Chain);
@@ -779,7 +779,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
 
   // TODO: setTypeListBeforeSoften can't properly express multiple return types,
   // but we only really need to handle the 0th one for softening anyway.
-  CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true)
+  CallOptions.setTypeListBeforeSoften({OpsVT}, VT0)
       .setOpsTypeOverrides(CallOpsTypeOverrides);
 
   auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
@@ -828,7 +828,7 @@ bool DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(
   TargetLowering::MakeLibCallOptions CallOptions;
   // TODO: setTypeListBeforeSoften can't properly express multiple return types,
   // but since both returns have the same type it should be okay.
-  CallOptions.setTypeListBeforeSoften({OpsVT}, VT, true)
+  CallOptions.setTypeListBeforeSoften({OpsVT}, VT)
       .setOpsTypeOverrides(CallOpsTypeOverrides);
 
   auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT, Ops, CallOptions, DL,
@@ -1100,7 +1100,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
                            NVT, N->getOperand(IsStrict ? 1 : 0));
   TargetLowering::MakeLibCallOptions CallOptions;
   CallOptions.setIsSigned(Signed);
-  CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
+  CallOptions.setTypeListBeforeSoften(SVT, RVT);
   std::pair<SDValue, SDValue> Tmp =
       TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
                       Op, CallOptions, dl, Chain);
@@ -1222,7 +1222,7 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
   SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
   Op = GetSoftenedFloat(Op);
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
+  CallOptions.setTypeListBeforeSoften(SVT, RVT);
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
                                                     CallOptions, SDLoc(N),
                                                     Chain);
@@ -1298,7 +1298,7 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
   Op = GetSoftenedFloat(Op);
   SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
   TargetLowering::MakeLibCallOptions CallOptions;
-  CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
+  CallOptions.setTypeListBeforeSoften(SVT, RVT);
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
                                                     CallOptions, dl, Chain);
 
@@ -1453,7 +1453,7 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
   SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
   TargetLowering::MakeLibCallOptions CallOptions;
   EVT OpVT = N->getOperand(0 + Offset).getValueType();
-  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
+  CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
   std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
                                                     CallOptions, SDLoc(N),
                                                     Chain);
@@ -1551,6 +1551,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
   case ISD::VAARG:              ExpandRes_VAARG(N, Lo, Hi); break;
 
   case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
+  case ISD::AssertNoFPClass: ExpandFloatRes_AssertNoFPClass(N, Lo, Hi); break;
   case ISD::FABS:       ExpandFloatRes_FABS(N, Lo, Hi); break;
   case ISD::STRICT_FMINNUM:
   case ISD::FMINNUM:    ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
@@ -1966,6 +1967,13 @@ void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
   Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
 }
 
+void DAGTypeLegalizer::ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo,
+                                                      SDValue &Hi) {
+  // TODO: Handle ppcf128 by preserving AssertNoFPClass for one of the halves.
+  SDLoc dl(N);
+  GetExpandedFloat(N->getOperand(0), Lo, Hi);
+}
+
 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
                                                 SDValue &Hi) {
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
@@ -3559,7 +3567,7 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
     Op = GetSoftenedFloat(Op);
     TargetLowering::MakeLibCallOptions CallOptions;
-    CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
+    CallOptions.setTypeListBeforeSoften(SVT, RVT);
     std::pair<SDValue, SDValue> Tmp =
         TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, SDLoc(N), Chain);
     if (IsStrict)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index a5bd97ace169e..90d62e6da8e94 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -5260,20 +5260,18 @@ void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
                    MachinePointerInfo());
 
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
   for (const SDValue &Op : N->op_values()) {
     EVT ArgVT = Op.getValueType();
     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-    Entry.Node = Op;
-    Entry.Ty = ArgTy;
+    TargetLowering::ArgListEntry Entry(Op, ArgTy);
     Entry.IsSExt = true;
     Entry.IsZExt = false;
     Args.push_back(Entry);
   }
 
   // Also pass the address of the overflow check.
-  Entry.Node = Temp;
-  Entry.Ty = PointerType::getUnqual(PtrTy->getContext());
+  TargetLowering::ArgListEntry Entry(
+      Temp, PointerType::getUnqual(PtrTy->getContext()));
   Entry.IsSExt = true;
   Entry.IsZExt = false;
   Args.push_back(Entry);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 63544e63e1da1..65fd863e55ac9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -681,6 +681,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
       SDNode *N, RTLIB::Libcall LC, std::optional<unsigned> CallRetResNo = {});
 
   // clang-format off
+  void ExpandFloatRes_AssertNoFPClass(SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_FABS      (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_FACOS     (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandFloatRes_FASIN     (SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -908,6 +909,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
   SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
   SDValue ScalarizeVecOp_VSELECT(SDNode *N);
   SDValue ScalarizeVecOp_VSETCC(SDNode *N);
+  SDValue ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N, unsigned OpNo);
   SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo);
   SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo);
   SDValue ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index d2ecc13331e02..2ca98958fde0d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -2223,17 +2223,13 @@ bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
 
   SDLoc DL(Node);
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.IsSExt = false;
-  Entry.IsZExt = false;
 
   unsigned OpNum = 0;
   for (auto &VFParam : OptVFInfo->Shape.Parameters) {
     if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
       EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);
-      Entry.Node = DAG.getBoolConstant(true, DL, MaskVT, VT);
-      Entry.Ty = MaskVT.getTypeForEVT(*Ctx);
-      Args.push_back(Entry);
+      Args.emplace_back(DAG.getBoolConstant(true, DL, MaskVT, VT),
+                        MaskVT.getTypeForEVT(*Ctx));
       continue;
     }
 
@@ -2241,9 +2237,7 @@ bool VectorLegalizer::tryExpandVecMathCall(SDNode *Node, RTLIB::Libcall LC,
     if (VFParam.ParamKind != VFParamKind::Vector)
       return false;
 
-    Entry.Node = Node->getOperand(OpNum++);
-    Entry.Ty = Ty;
-    Args.push_back(Entry);
+    Args.emplace_back(Node->getOperand(OpNum++), Ty);
   }
 
   // Emit a call to the vector function.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index bc2dbfb4cbaae..10e3a5149a5dc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -789,6 +789,10 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
   case ISD::SETCC:
     Res = ScalarizeVecOp_VSETCC(N);
     break;
+  case ISD::STRICT_FSETCC:
+  case ISD::STRICT_FSETCCS:
+    Res = ScalarizeVecOp_VSTRICT_FSETCC(N, OpNo);
+    break;
   case ISD::STORE:
     Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
     break;
@@ -985,6 +989,43 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
   return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
 }
 
+// Similiar to ScalarizeVecOp_VSETCC, with added logic to update chains.
+SDValue DAGTypeLegalizer::ScalarizeVecOp_VSTRICT_FSETCC(SDNode *N,
+                                                        unsigned OpNo) {
+  assert(OpNo == 1 && "Wrong operand for scalarization!");
+  assert(N->getValueType(0).isVector() &&
+         N->getOperand(1).getValueType().isVector() &&
+         "Operand types must be vectors");
+  assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
+
+  EVT VT = N->getValueType(0);
+  SDValue Ch = N->getOperand(0);
+  SDValue LHS = GetScalarizedVector(N->getOperand(1));
+  SDValue RHS = GetScalarizedVector(N->getOperand(2));
+  SDValue CC = N->getOperand(3);
+
+  EVT OpVT = N->getOperand(1).getValueType();
+  EVT NVT = VT.getVectorElementType();
+  SDLoc DL(N);
+  SDValue Res = DAG.getNode(N->getOpcode(), DL, {MVT::i1, MVT::Other},
+                            {Ch, LHS, RHS, CC});
+
+  // Legalize the chain result - switch anything that used the old chain to
+  // use the new one.
+  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
+
+  ISD::NodeType ExtendCode =
+      TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
+
+  Res = DAG.getNode(ExtendCode, DL, NVT, Res);
+  Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
+
+  // Do our own replacement and return SDValue() to tell the caller that we
+  // handled all replacements since caller can only handle a single result.
+  ReplaceValueWith(SDValue(N, 0), Res);
+  return SDValue();
+}
+
 /// If the value to store is a vector that needs to be scalarized, it must be
 /// <1 x ty>. Just store the element.
 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
@@ -3842,13 +3883,45 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
   uint64_t LoEltsMin = Lo.getValueType().getVectorMinNumElements();
   uint64_t IdxVal = Idx->getAsZExtVal();
 
+  unsigned NumResultElts = SubVT.getVectorMinNumElements();
+
   if (IdxVal < LoEltsMin) {
-    assert(IdxVal + SubVT.getVectorMinNumElements() <= LoEltsMin &&
-           "Extracted subvector crosses vector split!");
-    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
-  } else if (SubVT.isScalableVector() ==
-             N->getOperand(0).getValueType().isScalableVector())
-    return DAG.getExtractSubvector(dl, SubVT, Hi, IdxVal - LoEltsMin);
+    // If the extracted elements are all in the low half, do a simple extract.
+    if (IdxVal + NumResultElts <= LoEltsMin)
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
+
+    // Extracted subvector crosses vector split, so we need to blend the two
+    // halves.
+    // TODO: May be able to emit partial extract_subvector.
+    SmallVector<SDValue, 8> Elts;
+    Elts.reserve(NumResultElts);
+
+    DAG.ExtractVectorElements(Lo, Elts, /*Start=*/IdxVal,
+                              /*Count=*/LoEltsMin - IdxVal);
+    DAG.ExtractVectorElements(Hi, Elts, /*Start=*/0,
+                              /*Count=*/SubVT.getVectorNumElements() -
+                                  Elts.size());
+    return DAG.getBuildVector(SubVT, dl, Elts);
+  }
+
+  EVT SrcVT = N->getOperand(0).getValueType();
+  if (SubVT.isScalableVector() == SrcVT.isScalableVector()) {
+    uint64_t ExtractIdx = IdxVal - LoEltsMin;
+    if (ExtractIdx % NumResultElts == 0)
+      return DAG.getExtractSubvector(dl, SubVT, Hi, ExtractIdx);
+
+    // We cannot create an extract_subvector that isn't a multiple of the result
+    // size, which may go out of bounds for the last elements. Shuffle the
+    // desired elements down to 0 and do a simple 0 extract.
+    EVT HiVT = Hi.getValueType();
+    SmallVector<int, 8> Mask(HiVT.getVectorNumElements(), -1);
+    for (int I = 0; I != static_cast<int>(NumResultElts); ++I)
+      Mask[I] = ExtractIdx + I;
+
+    SDValue Shuffle =
+        DAG.getVectorShuffle(HiVT, dl, Hi, DAG.getPOISON(HiVT), Mask);
+    return DAG.getExtractSubvector(dl, SubVT, Shuffle, 0);
+  }
 
   // After this point the DAG node only permits extracting fixed-width
   // subvectors from scalable vectors.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 71a175dfd7b24..3672a91e33a30 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1371,7 +1371,7 @@ void SelectionDAG::init(MachineFunction &NewMF,
                         const TargetLibraryInfo *LibraryInfo,
                         UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
                         BlockFrequencyInfo *BFIin, MachineModuleInfo &MMIin,
-                        FunctionVarLocs const *VarLocs, bool HasDivergency) {
+                        FunctionVarLocs const *VarLocs) {
   MF = &NewMF;
   SDAGISelPass = PassPtr;
   ORE = &NewORE;
@@ -1384,7 +1384,6 @@ void SelectionDAG::init(MachineFunction &NewMF,
   BFI = BFIin;
   MMI = &MMIin;
   FnVarLocs = VarLocs;
-  DivergentTarget = HasDivergency;
 }
 
 SelectionDAG::~SelectionDAG() {
@@ -2331,8 +2330,7 @@ SDValue SelectionDAG::getRegister(Register Reg, EVT VT) {
     return SDValue(E, 0);
 
   auto *N = newSDNode<RegisterSDNode>(Reg, VTs);
-  N->SDNodeBits.IsDivergent =
-      DivergentTarget && TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
+  N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
   CSEMap.InsertNode(N, IP);
   InsertNode(N);
   return SDValue(N, 0);
@@ -2578,18 +2576,12 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
   }
 
   TargetLowering::ArgListTy Args;
-  auto AddArgListEntry = [&](SDValue Node, Type *Ty) {
-    TargetLowering::ArgListEntry Entry{};
-    Entry.Ty = Ty;
-    Entry.Node = Node;
-    Args.push_back(Entry);
-  };
 
   // Pass the arguments.
   for (const SDValue &Op : Node->op_values()) {
     EVT ArgVT = Op.getValueType();
     Type *ArgTy = ArgVT.getTypeForEVT(Ctx);
-    AddArgListEntry(Op, ArgTy);
+    Args.emplace_back(Op, ArgTy);
   }
 
   // Pass the output pointers.
@@ -2601,7 +2593,7 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
     EVT ResVT = Node->getValueType(ResNo);
     SDValue ResultPtr = ST ? ST->getBasePtr() : CreateStackTemporary(ResVT);
     ResultPtrs[ResNo] = ResultPtr;
-    AddArgListEntry(ResultPtr, PointerTy);
+    Args.emplace_back(ResultPtr, PointerTy);
   }
 
   SDLoc DL(Node);
@@ -2610,7 +2602,7 @@ bool SelectionDAG::expandMultipleResultFPLibCall(
   if (VD && VD->isMasked()) {
     EVT MaskVT = TLI->getSetCCResultType(getDataLayout(), Ctx, VT);
     SDValue Mask = getBoolConstant(true, DL, MaskVT, VT);
-    AddArgListEntry(Mask, MaskVT.getTypeForEVT(Ctx));
+    Args.emplace_back(Mask, MaskVT.getTypeForEVT(Ctx));
   }
 
   Type *RetType = CallRetResNo.has_value()
@@ -5462,6 +5454,83 @@ bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
     }
     return true;
 
+  case ISD::EXTRACT_SUBVECTOR: {
+    SDValue Src = Op.getOperand(0);
+    if (Src.getValueType().isScalableVector())
+      break;
+    uint64_t Idx = Op.getConstantOperandVal(1);
+    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+    APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
+    return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, PoisonOnly,
+                                            Depth + 1);
+  }
+
+  case ISD::INSERT_SUBVECTOR: {
+    if (Op.getValueType().isScalableVector())
+      break;
+    SDValue Src = Op.getOperand(0);
+    SDValue Sub = Op.getOperand(1);
+    uint64_t Idx = Op.getConstantOperandVal(2);
+    unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
+    APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
+    APInt DemandedSrcElts = DemandedElts;
+    DemandedSrcElts.clearBits(Idx, Idx + NumSubElts);
+
+    if (!!DemandedSubElts && !isGuaranteedNotToBeUndefOrPoison(
+                                 Sub, DemandedSubElts, PoisonOnly, Depth + 1))
+      return false;
+    if (!!DemandedSrcElts && !isGuaranteedNotToBeUndefOrPoison(
+                                 Src, DemandedSrcElts, PoisonOnly, Depth + 1))
+      return false;
+    return true;
+  }
+
+  case ISD::EXTRACT_VECTOR_ELT: {
+    SDValue Src = Op.getOperand(0);
+    auto *IndexC = dyn_cast<ConstantSDNode>(Op.getOperand(1));
+    EVT SrcVT = Src.getValueType();
+    if (SrcVT.isFixedLengthVector() && IndexC &&
+        IndexC->getAPIntValue().ult(SrcVT.getVectorNumElements())) {
+      APInt DemandedSrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
+                                                  IndexC->getZExtValue());
+      return isGuaranteedNotToBeUndefOrPoison(Src, DemandedSrcElts, PoisonOnly,
+                                              Depth + 1);
+    }
+    break;
+  }
+
+  case ISD::INSERT_VECTOR_ELT: {
+    SDValue InVec = Op.getOperand(0);
+    SDValue InVal = Op.getOperand(1);
+    SDValue EltNo = Op.getOperand(2);
+    EVT VT = InVec.getValueType();
+    auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
+    if (IndexC && VT.isFixedLengthVector() &&
+        IndexC->getAPIntValue().ult(VT.getVectorNumElements())) {
+      if (DemandedElts[IndexC->getZExtValue()] &&
+          !isGuaranteedNotToBeUndefOrPoison(InVal, PoisonOnly, Depth + 1))
+        return false;
+      APInt InVecDemandedElts = DemandedElts;
+      InVecDemandedElts.clearBit(IndexC->getZExtValue());
+      if (!!InVecDemandedElts &&
+          !isGuaranteedNotToBeUndefOrPoison(InVec, InVecDemandedElts,
+                                            PoisonOnly, Depth + 1))
+        return false;
+      return true;
+    }
+    break;
+  }
+
+  case ISD::SCALAR_TO_VECTOR:
+    // Check upper (known undef) elements.
+    if (DemandedElts.ugt(1) && !PoisonOnly)
+      return false;
+    // Check element zero.
+    if (DemandedElts[0] && !isGuaranteedNotToBeUndefOrPoison(
+                               Op.getOperand(0), PoisonOnly, Depth + 1))
+      return false;
+    return true;
+
   case ISD::SPLAT_VECTOR:
     return isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), PoisonOnly,
                                             Depth + 1);
@@ -5484,6 +5553,52 @@ bool SelectionDAG::isGuaranteedNotToBeUndefOrPoison(SDValue Op,
     return true;
   }
 
+  case ISD::SHL:
+  case ISD::SRL:
+  case ISD::SRA:
+    // Shift amount operand is checked by canCreateUndefOrPoison. So it is
+    // enough to check operand 0 if Op can't create undef/poison.
+    return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
+                                   /*ConsiderFlags*/ true, Depth) &&
+           isGuaranteedNotToBeUndefOrPoison(Op.getOperand(0), DemandedElts,
+                                            PoisonOnly, Depth + 1);
+
+  case ISD::BSWAP:
+  case ISD::CTPOP:
+  case ISD::BITREVERSE:
+  case ISD::AND:
+  case ISD::OR:
+  case ISD::XOR:
+  case ISD::ADD:
+  case ISD::SUB:
+  case ISD::MUL:
+  case ISD::SADDSAT:
+  case ISD::UADDSAT:
+  case ISD::SSUBSAT:
+  case ISD::USUBSAT:
+  case ISD::SSHLSAT:
+  case ISD::USHLSAT:
+  case ISD::SMIN:
+  case ISD::SMAX:
+  case ISD::UMIN:
+  case ISD::UMAX:
+  case ISD::ZERO_EXTEND:
+  case ISD::SIGN_EXTEND:
+  case ISD::ANY_EXTEND:
+  case ISD::TRUNCATE:
+  case ISD::VSELECT: {
+    // If Op can't create undef/poison and none of its operands are undef/poison
+    // then Op is never undef/poison. A difference from the more common check
+    // below, outside the switch, is that we handle elementwise operations for
+    // which the DemandedElts mask is valid for all operands here.
+    return !canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly,
+                                   /*ConsiderFlags*/ true, Depth) &&
+           all_of(Op->ops(), [&](SDValue V) {
+             return isGuaranteedNotToBeUndefOrPoison(V, DemandedElts,
+                                                     PoisonOnly, Depth + 1);
+           });
+  }
+
     // TODO: Search for noundef attributes from library functions.
 
     // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
@@ -5549,8 +5664,10 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::ABDS:
   case ISD::SMIN:
   case ISD::SMAX:
+  case ISD::SCMP:
   case ISD::UMIN:
   case ISD::UMAX:
+  case ISD::UCMP:
   case ISD::AND:
   case ISD::XOR:
   case ISD::ROTL:
@@ -5630,6 +5747,11 @@ bool SelectionDAG::canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts,
   case ISD::FDIV:
   case ISD::FREM:
   case ISD::FCOPYSIGN:
+  case ISD::FMA:
+  case ISD::FMAD:
+  case ISD::FP_EXTEND:
+  case ISD::FP_TO_SINT_SAT:
+  case ISD::FP_TO_UINT_SAT:
     // No poison except from flags (which is handled above)
     return false;
 
@@ -6422,6 +6544,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     if (N1.isUndef())
       // sext(undef) = 0, because the top bits will all be the same.
       return getConstant(0, DL, VT);
+
+    // Skip unnecessary sext_inreg pattern:
+    // (sext (trunc x)) -> x iff the upper bits are all signbits.
+    if (OpOpcode == ISD::TRUNCATE) {
+      SDValue OpOp = N1.getOperand(0);
+      if (OpOp.getValueType() == VT) {
+        unsigned NumSignExtBits =
+            VT.getScalarSizeInBits() - N1.getScalarValueSizeInBits();
+        if (ComputeNumSignBits(OpOp) > NumSignExtBits) {
+          transferDbgValues(N1, OpOp);
+          return OpOp;
+        }
+      }
+    }
     break;
   case ISD::ZERO_EXTEND:
     assert(VT.isInteger() && N1.getValueType().isInteger() &&
@@ -7039,6 +7175,45 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
     }
   }
 
+  // Handle fshl/fshr special cases.
+  if (Opcode == ISD::FSHL || Opcode == ISD::FSHR) {
+    auto *C1 = dyn_cast<ConstantSDNode>(Ops[0]);
+    auto *C2 = dyn_cast<ConstantSDNode>(Ops[1]);
+    auto *C3 = dyn_cast<ConstantSDNode>(Ops[2]);
+
+    if (C1 && C2 && C3) {
+      if (C1->isOpaque() || C2->isOpaque() || C3->isOpaque())
+        return SDValue();
+      const APInt &V1 = C1->getAPIntValue(), &V2 = C2->getAPIntValue(),
+                  &V3 = C3->getAPIntValue();
+
+      APInt FoldedVal = Opcode == ISD::FSHL ? APIntOps::fshl(V1, V2, V3)
+                                            : APIntOps::fshr(V1, V2, V3);
+      return getConstant(FoldedVal, DL, VT);
+    }
+  }
+
+  // Handle fma/fmad special cases.
+  if (Opcode == ISD::FMA || Opcode == ISD::FMAD) {
+    assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
+    assert(Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
+           Ops[2].getValueType() == VT && "FMA types must match!");
+    ConstantFPSDNode *C1 = dyn_cast<ConstantFPSDNode>(Ops[0]);
+    ConstantFPSDNode *C2 = dyn_cast<ConstantFPSDNode>(Ops[1]);
+    ConstantFPSDNode *C3 = dyn_cast<ConstantFPSDNode>(Ops[2]);
+    if (C1 && C2 && C3) {
+      APFloat V1 = C1->getValueAPF();
+      const APFloat &V2 = C2->getValueAPF();
+      const APFloat &V3 = C3->getValueAPF();
+      if (Opcode == ISD::FMAD) {
+        V1.multiply(V2, APFloat::rmNearestTiesToEven);
+        V1.add(V3, APFloat::rmNearestTiesToEven);
+      } else
+        V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
+      return getConstantFP(V1, DL, VT);
+    }
+  }
+
   // This is for vector folding only from here on.
   if (!VT.isVector())
     return SDValue();
@@ -7824,6 +7999,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     assert(N2C->getAPIntValue().getBitWidth() ==
                TLI->getVectorIdxWidth(getDataLayout()) &&
            "Constant index for EXTRACT_SUBVECTOR has an invalid size");
+    assert(N2C->getZExtValue() % VT.getVectorMinNumElements() == 0 &&
+           "Extract index is not a multiple of the output vector length");
 
     // Trivial extraction.
     if (VT == N1VT)
@@ -7999,27 +8176,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
          "Operand is DELETED_NODE!");
   // Perform various simplifications.
   switch (Opcode) {
-  case ISD::FMA:
-  case ISD::FMAD: {
-    assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
-    assert(N1.getValueType() == VT && N2.getValueType() == VT &&
-           N3.getValueType() == VT && "FMA types must match!");
-    ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
-    ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
-    ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
-    if (N1CFP && N2CFP && N3CFP) {
-      APFloat  V1 = N1CFP->getValueAPF();
-      const APFloat &V2 = N2CFP->getValueAPF();
-      const APFloat &V3 = N3CFP->getValueAPF();
-      if (Opcode == ISD::FMAD) {
-        V1.multiply(V2, APFloat::rmNearestTiesToEven);
-        V1.add(V3, APFloat::rmNearestTiesToEven);
-      } else
-        V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven);
-      return getConstantFP(V1, DL, VT);
-    }
-    break;
-  }
   case ISD::BUILD_VECTOR: {
     // Attempt to simplify BUILD_VECTOR.
     SDValue Ops[] = {N1, N2, N3};
@@ -8045,12 +8201,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     // Use FoldSetCC to simplify SETCC's.
     if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
       return V;
-    // Vector constant folding.
-    SDValue Ops[] = {N1, N2, N3};
-    if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) {
-      NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
-      return V;
-    }
     break;
   }
   case ISD::SELECT:
@@ -8186,6 +8336,19 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
   }
   }
 
+  // Perform trivial constant folding for arithmetic operators.
+  switch (Opcode) {
+  case ISD::FMA:
+  case ISD::FMAD:
+  case ISD::SETCC:
+  case ISD::FSHL:
+  case ISD::FSHR:
+    if (SDValue SV =
+            FoldConstantArithmetic(Opcode, DL, VT, {N1, N2, N3}, Flags))
+      return SV;
+    break;
+  }
+
   // Memoize node if it doesn't produce a glue result.
   SDNode *N;
   SDVTList VTs = getVTList(VT);
@@ -8874,6 +9037,37 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI,
   }
 }
 
+std::pair<SDValue, SDValue>
+SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0,
+                        SDValue Mem1, SDValue Size, const CallInst *CI) {
+  const char *LibCallName = TLI->getLibcallName(RTLIB::MEMCMP);
+  if (!LibCallName)
+    return {};
+
+  PointerType *PT = PointerType::getUnqual(*getContext());
+  TargetLowering::ArgListTy Args = {
+      {Mem0, PT},
+      {Mem1, PT},
+      {Size, getDataLayout().getIntPtrType(*getContext())}};
+
+  TargetLowering::CallLoweringInfo CLI(*this);
+  bool IsTailCall = false;
+  bool ReturnsFirstArg = CI && funcReturnsFirstArgOfCall(*CI);
+  IsTailCall = CI && CI->isTailCall() &&
+               isInTailCallPosition(*CI, getTarget(), ReturnsFirstArg);
+
+  CLI.setDebugLoc(dl)
+      .setChain(Chain)
+      .setLibCallee(
+          TLI->getLibcallCallingConv(RTLIB::MEMCMP),
+          Type::getInt32Ty(*getContext()),
+          getExternalSymbol(LibCallName, TLI->getPointerTy(getDataLayout())),
+          std::move(Args))
+      .setTailCall(IsTailCall);
+
+  return TLI->LowerCallTo(CLI);
+}
+
 SDValue SelectionDAG::getMemcpy(
     SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size,
     Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI,
@@ -8925,13 +9119,10 @@ SDValue SelectionDAG::getMemcpy(
 
   // Emit a library call.
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Ty = PointerType::getUnqual(*getContext());
-  Entry.Node = Dst; Args.push_back(Entry);
-  Entry.Node = Src; Args.push_back(Entry);
-
-  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
-  Entry.Node = Size; Args.push_back(Entry);
+  Type *PtrTy = PointerType::getUnqual(*getContext());
+  Args.emplace_back(Dst, PtrTy);
+  Args.emplace_back(Src, PtrTy);
+  Args.emplace_back(Size, getDataLayout().getIntPtrType(*getContext()));
   // FIXME: pass in SDLoc
   TargetLowering::CallLoweringInfo CLI(*this);
   bool IsTailCall = false;
@@ -8969,17 +9160,10 @@ SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl,
                                       MachinePointerInfo SrcPtrInfo) {
   // Emit a library call.
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
-  Entry.Node = Dst;
-  Args.push_back(Entry);
-
-  Entry.Node = Src;
-  Args.push_back(Entry);
-
-  Entry.Ty = SizeTy;
-  Entry.Node = Size;
-  Args.push_back(Entry);
+  Type *ArgTy = getDataLayout().getIntPtrType(*getContext());
+  Args.emplace_back(Dst, ArgTy);
+  Args.emplace_back(Src, ArgTy);
+  Args.emplace_back(Size, SizeTy);
 
   RTLIB::Libcall LibraryCall =
       RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(ElemSz);
@@ -9042,13 +9226,10 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
 
   // Emit a library call.
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Ty = PointerType::getUnqual(*getContext());
-  Entry.Node = Dst; Args.push_back(Entry);
-  Entry.Node = Src; Args.push_back(Entry);
-
-  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
-  Entry.Node = Size; Args.push_back(Entry);
+  Type *PtrTy = PointerType::getUnqual(*getContext());
+  Args.emplace_back(Dst, PtrTy);
+  Args.emplace_back(Src, PtrTy);
+  Args.emplace_back(Size, getDataLayout().getIntPtrType(*getContext()));
   // FIXME:  pass in SDLoc
   TargetLowering::CallLoweringInfo CLI(*this);
 
@@ -9086,17 +9267,10 @@ SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl,
                                        MachinePointerInfo SrcPtrInfo) {
   // Emit a library call.
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
-  Entry.Node = Dst;
-  Args.push_back(Entry);
-
-  Entry.Node = Src;
-  Args.push_back(Entry);
-
-  Entry.Ty = SizeTy;
-  Entry.Node = Size;
-  Args.push_back(Entry);
+  Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext());
+  Args.emplace_back(Dst, IntPtrTy);
+  Args.emplace_back(Src, IntPtrTy);
+  Args.emplace_back(Size, SizeTy);
 
   RTLIB::Libcall LibraryCall =
       RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(ElemSz);
@@ -9173,28 +9347,20 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
 
   const char *BzeroName = getTargetLoweringInfo().getLibcallName(RTLIB::BZERO);
 
-  // Helper function to create an Entry from Node and Type.
-  const auto CreateEntry = [](SDValue Node, Type *Ty) {
-    TargetLowering::ArgListEntry Entry;
-    Entry.Node = Node;
-    Entry.Ty = Ty;
-    return Entry;
-  };
-
   bool UseBZero = isNullConstant(Src) && BzeroName;
   // If zeroing out and bzero is present, use it.
   if (UseBZero) {
     TargetLowering::ArgListTy Args;
-    Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
-    Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
+    Args.emplace_back(Dst, PointerType::getUnqual(Ctx));
+    Args.emplace_back(Size, DL.getIntPtrType(Ctx));
     CLI.setLibCallee(
         TLI->getLibcallCallingConv(RTLIB::BZERO), Type::getVoidTy(Ctx),
         getExternalSymbol(BzeroName, TLI->getPointerTy(DL)), std::move(Args));
   } else {
     TargetLowering::ArgListTy Args;
-    Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
-    Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx)));
-    Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
+    Args.emplace_back(Dst, PointerType::getUnqual(Ctx));
+    Args.emplace_back(Src, Src.getValueType().getTypeForEVT(Ctx));
+    Args.emplace_back(Size, DL.getIntPtrType(Ctx));
     CLI.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
                      Dst.getValueType().getTypeForEVT(Ctx),
                      getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
@@ -9223,18 +9389,9 @@ SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,
                                       MachinePointerInfo DstPtrInfo) {
   // Emit a library call.
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Ty = getDataLayout().getIntPtrType(*getContext());
-  Entry.Node = Dst;
-  Args.push_back(Entry);
-
-  Entry.Ty = Type::getInt8Ty(*getContext());
-  Entry.Node = Value;
-  Args.push_back(Entry);
-
-  Entry.Ty = SizeTy;
-  Entry.Node = Size;
-  Args.push_back(Entry);
+  Args.emplace_back(Dst, getDataLayout().getIntPtrType(*getContext()));
+  Args.emplace_back(Value, Type::getInt8Ty(*getContext()));
+  Args.emplace_back(Size, SizeTy);
 
   RTLIB::Libcall LibraryCall =
       RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(ElemSz);
@@ -12211,8 +12368,6 @@ static bool gluePropagatesDivergence(const SDNode *Node) {
 }
 
 bool SelectionDAG::calculateDivergence(SDNode *N) {
-  if (!DivergentTarget)
-    return false;
   if (TLI->isSDNodeAlwaysUniform(N)) {
     assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
            "Conflicting divergence information!");
@@ -12232,8 +12387,6 @@ bool SelectionDAG::calculateDivergence(SDNode *N) {
 }
 
 void SelectionDAG::updateDivergence(SDNode *N) {
-  if (!DivergentTarget)
-    return;
   SmallVector<SDNode *, 16> Worklist(1, N);
   do {
     N = Worklist.pop_back_val();
@@ -13794,20 +13947,16 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
     Ops[I].setInitial(Vals[I]);
     EVT VT = Ops[I].getValueType();
 
-    // Take care of the Node's operands iff target has divergence
     // Skip Chain. It does not carry divergence.
-    if (DivergentTarget && VT != MVT::Other &&
+    if (VT != MVT::Other &&
         (VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
         Ops[I].getNode()->isDivergent()) {
-      // Node is going to be divergent if at least one of its operand is
-      // divergent, unless it belongs to the "AlwaysUniform" exemptions.
       IsDivergent = true;
     }
   }
   Node->NumOperands = Vals.size();
   Node->OperandList = Ops;
-  // Check the divergence of the Node itself.
-  if (DivergentTarget && !TLI->isSDNodeAlwaysUniform(Node)) {
+  if (!TLI->isSDNodeAlwaysUniform(Node)) {
     IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
     Node->SDNodeBits.IsDivergent = IsDivergent;
   }
@@ -13897,10 +14046,7 @@ SDValue SelectionDAG::makeStateFunctionCall(unsigned LibFunc, SDValue Ptr,
                                             const SDLoc &DLoc) {
   assert(InChain.getValueType() == MVT::Other && "Expected token chain");
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Node = Ptr;
-  Entry.Ty = Ptr.getValueType().getTypeForEVT(*getContext());
-  Args.push_back(Entry);
+  Args.emplace_back(Ptr, Ptr.getValueType().getTypeForEVT(*getContext()));
   RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);
   SDValue Callee = getExternalSymbol(TLI->getLibcallName(LC),
                                      TLI->getPointerTy(getDataLayout()));
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index d0815e9f51822..62ba801f69929 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -17,7 +17,6 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
@@ -1837,11 +1836,8 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
                          getValue(CPA->getDiscriminator()));
     }
 
-    if (isa<ConstantPointerNull>(C)) {
-      unsigned AS = V->getType()->getPointerAddressSpace();
-      return DAG.getConstant(0, getCurSDLoc(),
-                             TLI.getPointerTy(DAG.getDataLayout(), AS));
-    }
+    if (isa<ConstantPointerNull>(C))
+      return DAG.getConstant(0, getCurSDLoc(), VT);
 
     if (match(C, m_VScale()))
       return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1));
@@ -2211,9 +2207,9 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
                         MVT::Other, Chains);
   } else if (I.getNumOperands() != 0) {
-    SmallVector<EVT, 4> ValueVTs;
-    ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
-    unsigned NumValues = ValueVTs.size();
+    SmallVector<Type *, 4> Types;
+    ComputeValueTypes(DL, I.getOperand(0)->getType(), Types);
+    unsigned NumValues = Types.size();
     if (NumValues) {
       SDValue RetOp = getValue(I.getOperand(0));
 
@@ -2233,7 +2229,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
       bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg);
 
       for (unsigned j = 0; j != NumValues; ++j) {
-        EVT VT = ValueVTs[j];
+        EVT VT = TLI.getValueType(DL, Types[j]);
 
         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
           VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
@@ -2275,7 +2271,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
         for (unsigned i = 0; i < NumParts; ++i) {
           Outs.push_back(ISD::OutputArg(Flags,
                                         Parts[i].getValueType().getSimpleVT(),
-                                        VT, /*isfixed=*/true, 0, 0));
+                                        VT, Types[j], 0, 0));
           OutVals.push_back(Parts[i]);
         }
       }
@@ -2291,9 +2287,10 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
     assert(SwiftError.getFunctionArg() && "Need a swift error argument");
     ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
     Flags.setSwiftError();
-    Outs.push_back(ISD::OutputArg(
-        Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)),
-        /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0));
+    Outs.push_back(ISD::OutputArg(Flags, /*vt=*/TLI.getPointerTy(DL),
+                                  /*argvt=*/EVT(TLI.getPointerTy(DL)),
+                                  PointerType::getUnqual(*DAG.getContext()),
+                                  /*origidx=*/1, /*partOffs=*/0));
     // Create SDNode for the swifterror virtual register.
     OutVals.push_back(
         DAG.getRegister(SwiftError.getOrCreateVRegUseAt(
@@ -3108,9 +3105,7 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
 
     TargetLowering::ArgListTy Args;
-    TargetLowering::ArgListEntry Entry;
-    Entry.Node = GuardVal;
-    Entry.Ty = FnTy->getParamType(0);
+    TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
     if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
       Entry.IsInReg = true;
     Args.push_back(Entry);
@@ -3207,9 +3202,7 @@ void SelectionDAGBuilder::visitSPDescriptorFailure(
     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
 
     TargetLowering::ArgListTy Args;
-    TargetLowering::ArgListEntry Entry;
-    Entry.Node = GuardVal;
-    Entry.Ty = FnTy->getParamType(0);
+    TargetLowering::ArgListEntry Entry(GuardVal, FnTy->getParamType(0));
     if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg))
       Entry.IsInReg = true;
     Args.push_back(Entry);
@@ -3579,7 +3572,7 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
 
   // Update machine-CFG edges with unique successors.
-  SmallSet<BasicBlock*, 32> Done;
+  SmallPtrSet<BasicBlock *, 32> Done;
   for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
     BasicBlock *BB = I.getSuccessor(i);
     bool Inserted = Done.insert(BB).second;
@@ -3978,6 +3971,11 @@ void SelectionDAGBuilder::visitSIToFP(const User &I) {
   setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
 }
 
+void SelectionDAGBuilder::visitPtrToAddr(const User &I) {
+  // FIXME: this is not correct for pointers with addr width != pointer width
+  visitPtrToInt(I);
+}
+
 void SelectionDAGBuilder::visitPtrToInt(const User &I) {
   // What to do depends on the size of the integer and the size of the pointer.
   // We can either truncate, zero extend, or no-op, accordingly.
@@ -4903,9 +4901,8 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
 // extract the splat value and use it as a uniform base.
 // In all other cases the function returns 'false'.
 static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
-                           ISD::MemIndexType &IndexType, SDValue &Scale,
-                           SelectionDAGBuilder *SDB, const BasicBlock *CurBB,
-                           uint64_t ElemSize) {
+                           SDValue &Scale, SelectionDAGBuilder *SDB,
+                           const BasicBlock *CurBB, uint64_t ElemSize) {
   SelectionDAG& DAG = SDB->DAG;
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   const DataLayout &DL = DAG.getDataLayout();
@@ -4923,7 +4920,6 @@ static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
     ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount();
     EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts);
     Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT);
-    IndexType = ISD::SIGNED_SCALED;
     Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
     return true;
   }
@@ -4953,7 +4949,6 @@ static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index,
 
   Base = SDB->getValue(BasePtr);
   Index = SDB->getValue(IndexVal);
-  IndexType = ISD::SIGNED_SCALED;
 
   Scale =
       DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL));
@@ -4975,9 +4970,8 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
 
   SDValue Base;
   SDValue Index;
-  ISD::MemIndexType IndexType;
   SDValue Scale;
-  bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
+  bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
                                     I.getParent(), VT.getScalarStoreSize());
 
   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
@@ -4987,8 +4981,8 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
   if (!UniformBase) {
     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
     Index = getValue(Ptr);
-    IndexType = ISD::SIGNED_SCALED;
-    Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
+    Scale =
+        DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
   }
 
   EVT IdxVT = Index.getValueType();
@@ -5000,7 +4994,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
 
   SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale };
   SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
-                                         Ops, MMO, IndexType, false);
+                                         Ops, MMO, ISD::SIGNED_SCALED, false);
   DAG.setRoot(Scatter);
   setValue(&I, Scatter);
 }
@@ -5093,9 +5087,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
   SDValue Root = DAG.getRoot();
   SDValue Base;
   SDValue Index;
-  ISD::MemIndexType IndexType;
   SDValue Scale;
-  bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
+  bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
                                     I.getParent(), VT.getScalarStoreSize());
   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
   MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
@@ -5106,8 +5099,8 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
   if (!UniformBase) {
     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
     Index = getValue(Ptr);
-    IndexType = ISD::SIGNED_SCALED;
-    Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
+    Scale =
+        DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
   }
 
   EVT IdxVT = Index.getValueType();
@@ -5118,8 +5111,9 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
   }
 
   SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
-  SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
-                                       Ops, MMO, IndexType, ISD::NON_EXTLOAD);
+  SDValue Gather =
+      DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, Ops, MMO,
+                          ISD::SIGNED_SCALED, ISD::NON_EXTLOAD);
 
   PendingLoads.push_back(Gather.getValue(1));
   setValue(&I, Gather);
@@ -6432,9 +6426,8 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
   SDValue Root = DAG.getRoot();
   SDValue Base;
   SDValue Index;
-  ISD::MemIndexType IndexType;
   SDValue Scale;
-  bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this,
+  bool UniformBase = getUniformBase(Ptr, Base, Index, Scale, this,
                                     I.getParent(), VT.getScalarStoreSize());
 
   unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace();
@@ -6447,7 +6440,6 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
   if (!UniformBase) {
     Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
     Index = getValue(Ptr);
-    IndexType = ISD::SIGNED_SCALED;
     Scale =
         DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
   }
@@ -6463,7 +6455,7 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I,
 
   SDValue Ops[] = {Root, Inc, Mask, Base, Index, Scale, ID};
   SDValue Histogram = DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), VT, sdl,
-                                             Ops, MMO, IndexType);
+                                             Ops, MMO, ISD::SIGNED_SCALED);
 
   setValue(&I, Histogram);
   DAG.setRoot(Histogram);
@@ -7515,10 +7507,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     }
     TargetLowering::ArgListTy Args;
     if (Intrinsic == Intrinsic::ubsantrap) {
-      Args.push_back(TargetLoweringBase::ArgListEntry());
-      Args[0].Val = I.getArgOperand(0);
-      Args[0].Node = getValue(Args[0].Val);
-      Args[0].Ty = Args[0].Val->getType();
+      Value *Arg = I.getArgOperand(0);
+      Args.emplace_back(Arg, getValue(Arg));
     }
 
     TargetLowering::CallLoweringInfo CLI(DAG);
@@ -7598,7 +7588,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     if (TM.getOptLevel() == CodeGenOptLevel::None)
       return;
 
-    const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(I.getArgOperand(1));
+    const AllocaInst *LifetimeObject = dyn_cast<AllocaInst>(I.getArgOperand(0));
     if (!LifetimeObject)
       return;
 
@@ -7947,9 +7937,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     Args.reserve(3);
 
     for (unsigned Idx : {2, 3, 1}) {
-      TargetLowering::ArgListEntry Arg;
-      Arg.Node = getValue(I.getOperand(Idx));
-      Arg.Ty = I.getOperand(Idx)->getType();
+      TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
+                                       I.getOperand(Idx)->getType());
       Arg.setAttributes(&I, Idx);
       Args.push_back(Arg);
     }
@@ -7960,9 +7949,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
 
     // Forward the flags and any additional arguments.
     for (unsigned Idx = 4; Idx < I.arg_size(); ++Idx) {
-      TargetLowering::ArgListEntry Arg;
-      Arg.Node = getValue(I.getOperand(Idx));
-      Arg.Ty = I.getOperand(Idx)->getType();
+      TargetLowering::ArgListEntry Arg(getValue(I.getOperand(Idx)),
+                                       I.getOperand(Idx)->getType());
       Arg.setAttributes(&I, Idx);
       Args.push_back(Arg);
     }
@@ -7984,6 +7972,42 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     HasTailCall = true;
     return;
   }
+  case Intrinsic::amdgcn_call_whole_wave: {
+    TargetLowering::ArgListTy Args;
+
+    // The first argument is the callee. Skip it when assembling the call args.
+    for (unsigned Idx = 1; Idx < I.arg_size(); ++Idx) {
+      TargetLowering::ArgListEntry Arg(getValue(I.getArgOperand(Idx)),
+                                       I.getArgOperand(Idx)->getType());
+      Arg.setAttributes(&I, Idx);
+      Args.push_back(Arg);
+    }
+
+    SDValue ConvControlToken;
+    if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) {
+      auto *Token = Bundle->Inputs[0].get();
+      ConvControlToken = getValue(Token);
+    }
+
+    TargetLowering::CallLoweringInfo CLI(DAG);
+    CLI.setDebugLoc(getCurSDLoc())
+        .setChain(getRoot())
+        .setCallee(CallingConv::AMDGPU_Gfx_WholeWave, I.getType(),
+                   getValue(I.getArgOperand(0)), std::move(Args))
+        .setTailCall(false)
+        .setIsPreallocated(
+            I.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0)
+        .setConvergent(I.isConvergent())
+        .setConvergenceControlToken(ConvControlToken);
+    CLI.CB = &I;
+
+    std::pair<SDValue, SDValue> Result =
+        lowerInvokable(CLI, /*EHPadBB=*/nullptr);
+
+    if (Result.first.getNode())
+      setValue(&I, Result.first);
+    return;
+  }
   case Intrinsic::ptrmask: {
     SDValue Ptr = getValue(I.getOperand(0));
     SDValue Mask = getValue(I.getOperand(1));
@@ -8488,14 +8512,12 @@ void SelectionDAGBuilder::visitVPGather(
       MachinePointerInfo(AS), MachineMemOperand::MOLoad,
       LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
   SDValue Base, Index, Scale;
-  ISD::MemIndexType IndexType;
-  bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
-                                    this, VPIntrin.getParent(),
-                                    VT.getScalarStoreSize());
+  bool UniformBase =
+      getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
+                     VT.getScalarStoreSize());
   if (!UniformBase) {
     Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
     Index = getValue(PtrOperand);
-    IndexType = ISD::SIGNED_SCALED;
     Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
   }
   EVT IdxVT = Index.getValueType();
@@ -8507,7 +8529,7 @@ void SelectionDAGBuilder::visitVPGather(
   LD = DAG.getGatherVP(
       DAG.getVTList(VT, MVT::Other), VT, DL,
       {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO,
-      IndexType);
+      ISD::SIGNED_SCALED);
   PendingLoads.push_back(LD.getValue(1));
   setValue(&VPIntrin, LD);
 }
@@ -8551,16 +8573,13 @@ void SelectionDAGBuilder::visitVPScatter(
       MachinePointerInfo(AS), MachineMemOperand::MOStore,
       LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
   SDValue Base, Index, Scale;
-  ISD::MemIndexType IndexType;
-  bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale,
-                                    this, VPIntrin.getParent(),
-                                    VT.getScalarStoreSize());
+  bool UniformBase =
+      getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
+                     VT.getScalarStoreSize());
   if (!UniformBase) {
     Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()));
     Index = getValue(PtrOperand);
-    IndexType = ISD::SIGNED_SCALED;
-    Scale =
-      DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
+    Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()));
   }
   EVT IdxVT = Index.getValueType();
   EVT EltTy = IdxVT.getVectorElementType();
@@ -8571,7 +8590,7 @@ void SelectionDAGBuilder::visitVPScatter(
   ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL,
                         {getMemoryRoot(), OpValues[0], Base, Index, Scale,
                          OpValues[2], OpValues[3]},
-                        MMO, IndexType);
+                        MMO, ISD::SIGNED_SCALED);
   DAG.setRoot(ST);
   setValue(&VPIntrin, ST);
 }
@@ -8913,7 +8932,6 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
   }
 
   for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) {
-    TargetLowering::ArgListEntry Entry;
     const Value *V = *I;
 
     // Skip empty types
@@ -8921,8 +8939,7 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
       continue;
 
     SDValue ArgNode = getValue(V);
-    Entry.Node = ArgNode; Entry.Ty = V->getType();
-
+    TargetLowering::ArgListEntry Entry(ArgNode, V->getType());
     Entry.setAttributes(&CB, I - CB.arg_begin());
 
     // Use swifterror virtual register as input to the call.
@@ -8946,11 +8963,8 @@ void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee,
   // If call site has a cfguardtarget operand bundle, create and add an
   // additional ArgListEntry.
   if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) {
-    TargetLowering::ArgListEntry Entry;
     Value *V = Bundle->Inputs[0];
-    SDValue ArgNode = getValue(V);
-    Entry.Node = ArgNode;
-    Entry.Ty = V->getType();
+    TargetLowering::ArgListEntry Entry(V, getValue(V));
     Entry.IsCFGuardTarget = true;
     Args.push_back(Entry);
   }
@@ -9091,7 +9105,7 @@ bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) {
   const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
   std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
       DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
-      getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
+      getValue(Size), &I);
   if (Res.first.getNode()) {
     processIntegerCallValue(I, Res.first, true);
     PendingLoads.push_back(Res.second);
@@ -10613,9 +10627,7 @@ void SelectionDAGBuilder::populateCallLoweringInfo(
 
     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
 
-    TargetLowering::ArgListEntry Entry;
-    Entry.Node = getValue(V);
-    Entry.Ty = V->getType();
+    TargetLowering::ArgListEntry Entry(getValue(V), V->getType());
     Entry.setAttributes(Call, ArgI);
     Args.push_back(Entry);
   }
@@ -10975,27 +10987,42 @@ static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
 /// migrated to using LowerCall, this hook should be integrated into SDISel.
 std::pair<SDValue, SDValue>
 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
+  LLVMContext &Context = CLI.RetTy->getContext();
+
   // Handle the incoming return values from the call.
   CLI.Ins.clear();
-  SmallVector<EVT, 4> RetTys;
+  SmallVector<Type *, 4> RetOrigTys;
   SmallVector<TypeSize, 4> Offsets;
   auto &DL = CLI.DAG.getDataLayout();
-  ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
+  ComputeValueTypes(DL, CLI.OrigRetTy, RetOrigTys, &Offsets);
+
+  SmallVector<EVT, 4> RetVTs;
+  if (CLI.RetTy != CLI.OrigRetTy) {
+    assert(RetOrigTys.size() == 1 &&
+           "Only supported for non-aggregate returns");
+    RetVTs.push_back(getValueType(DL, CLI.RetTy));
+  } else {
+    for (Type *Ty : RetOrigTys)
+      RetVTs.push_back(getValueType(DL, Ty));
+  }
 
   if (CLI.IsPostTypeLegalization) {
     // If we are lowering a libcall after legalization, split the return type.
-    SmallVector<EVT, 4> OldRetTys;
+    SmallVector<Type *, 4> OldRetOrigTys;
+    SmallVector<EVT, 4> OldRetVTs;
     SmallVector<TypeSize, 4> OldOffsets;
-    RetTys.swap(OldRetTys);
+    RetOrigTys.swap(OldRetOrigTys);
+    RetVTs.swap(OldRetVTs);
     Offsets.swap(OldOffsets);
 
-    for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
-      EVT RetVT = OldRetTys[i];
+    for (size_t i = 0, e = OldRetVTs.size(); i != e; ++i) {
+      EVT RetVT = OldRetVTs[i];
       uint64_t Offset = OldOffsets[i];
-      MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
-      unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
+      MVT RegisterVT = getRegisterType(Context, RetVT);
+      unsigned NumRegs = getNumRegisters(Context, RetVT);
       unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
-      RetTys.append(NumRegs, RegisterVT);
+      RetOrigTys.append(NumRegs, OldRetOrigTys[i]);
+      RetVTs.append(NumRegs, RegisterVT);
       for (unsigned j = 0; j != NumRegs; ++j)
         Offsets.push_back(TypeSize::getFixed(Offset + j * RegisterVTByteSZ));
     }
@@ -11006,7 +11033,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
 
   bool CanLowerReturn =
       this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
-                           CLI.IsVarArg, Outs, CLI.RetTy->getContext(), CLI.RetTy);
+                           CLI.IsVarArg, Outs, Context, CLI.RetTy);
 
   SDValue DemoteStackSlot;
   int DemoteStackIdx = -100;
@@ -11019,30 +11046,16 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
     MachineFunction &MF = CLI.DAG.getMachineFunction();
     DemoteStackIdx =
         MF.getFrameInfo().CreateStackObject(TySize, Alignment, false);
-    Type *StackSlotPtrType =
-        PointerType::get(CLI.RetTy->getContext(), DL.getAllocaAddrSpace());
+    Type *StackSlotPtrType = PointerType::get(Context, DL.getAllocaAddrSpace());
 
     DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
-    ArgListEntry Entry;
-    Entry.Node = DemoteStackSlot;
-    Entry.Ty = StackSlotPtrType;
-    Entry.IsSExt = false;
-    Entry.IsZExt = false;
-    Entry.IsInReg = false;
+    ArgListEntry Entry(DemoteStackSlot, StackSlotPtrType);
     Entry.IsSRet = true;
-    Entry.IsNest = false;
-    Entry.IsByVal = false;
-    Entry.IsByRef = false;
-    Entry.IsReturned = false;
-    Entry.IsSwiftSelf = false;
-    Entry.IsSwiftAsync = false;
-    Entry.IsSwiftError = false;
-    Entry.IsCFGuardTarget = false;
     Entry.Alignment = Alignment;
     CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
     CLI.NumFixedArgs += 1;
     CLI.getArgs()[0].IndirectType = CLI.RetTy;
-    CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
+    CLI.RetTy = CLI.OrigRetTy = Type::getVoidTy(Context);
 
     // sret demotion isn't compatible with tail-calls, since the sret argument
     // points into the callers stack frame.
@@ -11050,36 +11063,32 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
   } else {
     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
         CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL);
-    for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
+    for (unsigned I = 0, E = RetVTs.size(); I != E; ++I) {
       ISD::ArgFlagsTy Flags;
       if (NeedsRegBlock) {
         Flags.setInConsecutiveRegs();
-        if (I == RetTys.size() - 1)
+        if (I == RetVTs.size() - 1)
           Flags.setInConsecutiveRegsLast();
       }
-      EVT VT = RetTys[I];
-      MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
-                                                     CLI.CallConv, VT);
-      unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
-                                                       CLI.CallConv, VT);
+      EVT VT = RetVTs[I];
+      MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
+      unsigned NumRegs =
+          getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
       for (unsigned i = 0; i != NumRegs; ++i) {
-        ISD::InputArg MyFlags;
-        MyFlags.Flags = Flags;
-        MyFlags.VT = RegisterVT;
-        MyFlags.ArgVT = VT;
-        MyFlags.Used = CLI.IsReturnValueUsed;
+        ISD::InputArg Ret(Flags, RegisterVT, VT, RetOrigTys[I],
+                          CLI.IsReturnValueUsed, ISD::InputArg::NoArgIndex, 0);
         if (CLI.RetTy->isPointerTy()) {
-          MyFlags.Flags.setPointer();
-          MyFlags.Flags.setPointerAddrSpace(
+          Ret.Flags.setPointer();
+          Ret.Flags.setPointerAddrSpace(
               cast<PointerType>(CLI.RetTy)->getAddressSpace());
         }
         if (CLI.RetSExt)
-          MyFlags.Flags.setSExt();
+          Ret.Flags.setSExt();
         if (CLI.RetZExt)
-          MyFlags.Flags.setZExt();
+          Ret.Flags.setZExt();
         if (CLI.IsInReg)
-          MyFlags.Flags.setInReg();
-        CLI.Ins.push_back(MyFlags);
+          Ret.Flags.setInReg();
+        CLI.Ins.push_back(Ret);
       }
     }
   }
@@ -11089,11 +11098,12 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
   if (supportSwiftError()) {
     for (const ArgListEntry &Arg : Args) {
       if (Arg.IsSwiftError) {
-        ISD::InputArg MyFlags;
-        MyFlags.VT = getPointerTy(DL);
-        MyFlags.ArgVT = EVT(getPointerTy(DL));
-        MyFlags.Flags.setSwiftError();
-        CLI.Ins.push_back(MyFlags);
+        ISD::ArgFlagsTy Flags;
+        Flags.setSwiftError();
+        ISD::InputArg Ret(Flags, getPointerTy(DL), EVT(getPointerTy(DL)),
+                          PointerType::getUnqual(Context),
+                          /*Used=*/true, ISD::InputArg::NoArgIndex, 0);
+        CLI.Ins.push_back(Ret);
       }
     }
   }
@@ -11102,18 +11112,24 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
   CLI.Outs.clear();
   CLI.OutVals.clear();
   for (unsigned i = 0, e = Args.size(); i != e; ++i) {
-    SmallVector<EVT, 4> ValueVTs;
-    ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
+    SmallVector<Type *, 4> OrigArgTys;
+    ComputeValueTypes(DL, Args[i].OrigTy, OrigArgTys);
     // FIXME: Split arguments if CLI.IsPostTypeLegalization
     Type *FinalType = Args[i].Ty;
     if (Args[i].IsByVal)
       FinalType = Args[i].IndirectType;
     bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
         FinalType, CLI.CallConv, CLI.IsVarArg, DL);
-    for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
+    for (unsigned Value = 0, NumValues = OrigArgTys.size(); Value != NumValues;
          ++Value) {
-      EVT VT = ValueVTs[Value];
-      Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
+      Type *OrigArgTy = OrigArgTys[Value];
+      Type *ArgTy = OrigArgTy;
+      if (Args[i].Ty != Args[i].OrigTy) {
+        assert(Value == 0 && "Only supported for non-aggregate arguments");
+        ArgTy = Args[i].Ty;
+      }
+
+      EVT VT = getValueType(DL, ArgTy);
       SDValue Op = SDValue(Args[i].Node.getNode(),
                            Args[i].Node.getResNo() + Value);
       ISD::ArgFlagsTy Flags;
@@ -11124,10 +11140,11 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
       const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL));
       Flags.setOrigAlign(OriginalAlignment);
 
-      if (Args[i].Ty->isPointerTy()) {
+      if (i >= CLI.NumFixedArgs)
+        Flags.setVarArg();
+      if (ArgTy->isPointerTy()) {
         Flags.setPointer();
-        Flags.setPointerAddrSpace(
-            cast<PointerType>(Args[i].Ty)->getAddressSpace());
+        Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
       }
       if (Args[i].IsZExt)
         Flags.setZExt();
@@ -11201,10 +11218,9 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
       if (NeedsRegBlock)
         Flags.setInConsecutiveRegs();
 
-      MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
-                                                 CLI.CallConv, VT);
-      unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
-                                                        CLI.CallConv, VT);
+      MVT PartVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
+      unsigned NumParts =
+          getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
       SmallVector<SDValue, 4> Parts(NumParts);
       ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
 
@@ -11221,7 +11237,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
                 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() &&
                  CLI.RetTy->getPointerAddressSpace() ==
                      Args[i].Ty->getPointerAddressSpace())) &&
-               RetTys.size() == NumValues && "unexpected use of 'returned'");
+               RetVTs.size() == NumValues && "unexpected use of 'returned'");
         // Before passing 'returned' to the target lowering code, ensure that
         // either the register MVT and the actual EVT are the same size or that
         // the return value and argument are extended in the same way; in these
@@ -11246,8 +11262,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
         // For scalable vectors the scalable part is currently handled
         // by individual targets, so we just use the known minimum size here.
         ISD::OutputArg MyFlags(
-            Flags, Parts[j].getValueType().getSimpleVT(), VT,
-            i < CLI.NumFixedArgs, i,
+            Flags, Parts[j].getValueType().getSimpleVT(), VT, OrigArgTy, i,
             j * Parts[j].getValueType().getStoreSize().getKnownMinValue());
         if (NumParts > 1 && j == 0)
           MyFlags.Flags.setSplit();
@@ -11303,7 +11318,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
     // hidden sret parameter.
     MVT PtrVT = getPointerTy(DL, DL.getAllocaAddrSpace());
 
-    unsigned NumValues = RetTys.size();
+    unsigned NumValues = RetVTs.size();
     ReturnValues.resize(NumValues);
     SmallVector<SDValue, 4> Chains(NumValues);
 
@@ -11316,7 +11331,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
           DemoteStackSlot, CLI.DAG.getConstant(Offsets[i], CLI.DL, PtrVT),
           CLI.DL, SDNodeFlags::NoUnsignedWrap);
       SDValue L = CLI.DAG.getLoad(
-          RetTys[i], CLI.DL, CLI.Chain, Add,
+          RetVTs[i], CLI.DL, CLI.Chain, Add,
           MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
                                             DemoteStackIdx, Offsets[i]),
           HiddenSRetAlign);
@@ -11334,11 +11349,10 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
     else if (CLI.RetZExt)
       AssertOp = ISD::AssertZext;
     unsigned CurReg = 0;
-    for (EVT VT : RetTys) {
-      MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
-                                                     CLI.CallConv, VT);
-      unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
-                                                       CLI.CallConv, VT);
+    for (EVT VT : RetVTs) {
+      MVT RegisterVT = getRegisterTypeForCallingConv(Context, CLI.CallConv, VT);
+      unsigned NumRegs =
+          getNumRegistersForCallingConv(Context, CLI.CallConv, VT);
 
       ReturnValues.push_back(getCopyFromParts(
           CLI.DAG, CLI.DL, &InVals[CurReg], NumRegs, RegisterVT, VT, nullptr,
@@ -11354,7 +11368,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
   }
 
   SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
-                                CLI.DAG.getVTList(RetTys), ReturnValues);
+                                CLI.DAG.getVTList(RetVTs), ReturnValues);
   return std::make_pair(Res, CLI.Chain);
 }
 
@@ -11625,7 +11639,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
     ISD::ArgFlagsTy Flags;
     Flags.setSRet();
     MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVT);
-    ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, true,
+    ISD::InputArg RetArg(Flags, RegisterVT, ValueVT, F.getReturnType(), true,
                          ISD::InputArg::NoArgIndex, 0);
     Ins.push_back(RetArg);
   }
@@ -11640,8 +11654,8 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
   // Set up the incoming argument description vector.
   for (const Argument &Arg : F.args()) {
     unsigned ArgNo = Arg.getArgNo();
-    SmallVector<EVT, 4> ValueVTs;
-    ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
+    SmallVector<Type *, 4> Types;
+    ComputeValueTypes(DAG.getDataLayout(), Arg.getType(), Types);
     bool isArgValueUsed = !Arg.use_empty();
     unsigned PartBase = 0;
     Type *FinalType = Arg.getType();
@@ -11649,17 +11663,15 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
       FinalType = Arg.getParamByValType();
     bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
         FinalType, F.getCallingConv(), F.isVarArg(), DL);
-    for (unsigned Value = 0, NumValues = ValueVTs.size();
-         Value != NumValues; ++Value) {
-      EVT VT = ValueVTs[Value];
-      Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
+    for (unsigned Value = 0, NumValues = Types.size(); Value != NumValues;
+         ++Value) {
+      Type *ArgTy = Types[Value];
+      EVT VT = TLI->getValueType(DL, ArgTy);
       ISD::ArgFlagsTy Flags;
 
-
-      if (Arg.getType()->isPointerTy()) {
+      if (ArgTy->isPointerTy()) {
         Flags.setPointer();
-        Flags.setPointerAddrSpace(
-            cast<PointerType>(Arg.getType())->getAddressSpace());
+        Flags.setPointerAddrSpace(cast<PointerType>(ArgTy)->getAddressSpace());
       }
       if (Arg.hasAttribute(Attribute::ZExt))
         Flags.setZExt();
@@ -11763,7 +11775,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) {
         // are responsible for handling scalable vector arguments and
         // return values.
         ISD::InputArg MyFlags(
-            Flags, RegisterVT, VT, isArgValueUsed, ArgNo,
+            Flags, RegisterVT, VT, ArgTy, isArgValueUsed, ArgNo,
             PartBase + i * RegisterVT.getStoreSize().getKnownMinValue());
         if (NumRegs > 1 && i == 0)
           MyFlags.Flags.setSplit();
@@ -12737,17 +12749,22 @@ static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) {
   assert(MI->getOpcode() == TargetOpcode::COPY &&
          "start of copy chain MUST be COPY");
   Reg = MI->getOperand(1).getReg();
+
+  // If the copied register in the first copy must be virtual.
+  assert(Reg.isVirtual() && "expected COPY of virtual register");
   MI = MRI.def_begin(Reg)->getParent();
+
   // There may be an optional second copy.
   if (MI->getOpcode() == TargetOpcode::COPY) {
     assert(Reg.isVirtual() && "expected COPY of virtual register");
     Reg = MI->getOperand(1).getReg();
     assert(Reg.isPhysical() && "expected COPY of physical register");
-    MI = MRI.def_begin(Reg)->getParent();
+  } else {
+    // The start of the chain must be an INLINEASM_BR.
+    assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
+           "end of copy chain MUST be INLINEASM_BR");
   }
-  // The start of the chain must be an INLINEASM_BR.
-  assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR &&
-         "end of copy chain MUST be INLINEASM_BR");
+
   return Reg;
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index c251755ee7064..e0835e6310357 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -574,6 +574,7 @@ class SelectionDAGBuilder {
   void visitFPToSI(const User &I);
   void visitUIToFP(const User &I);
   void visitSIToFP(const User &I);
+  void visitPtrToAddr(const User &I);
   void visitPtrToInt(const User &I);
   void visitIntToPtr(const User &I);
   void visitBitCast(const User &I);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 26071ed70c9db..ece50ed95fc49 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -480,10 +480,7 @@ void SelectionDAGISel::initializeAnalysisResults(
   MachineModuleInfo &MMI =
       MAMP.getCachedResult<MachineModuleAnalysis>(*Fn.getParent())->getMMI();
 
-  TTI = &FAM.getResult<TargetIRAnalysis>(Fn);
-
-  CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, MMI, FnVarLocs,
-               TTI->hasBranchDivergence(&Fn));
+  CurDAG->init(*MF, *ORE, MFAM, LibInfo, UA, PSI, BFI, MMI, FnVarLocs);
 
   // Now get the optional analyzes if we want to.
   // This is based on the possibly changed OptLevel (after optnone is taken
@@ -501,6 +498,10 @@ void SelectionDAGISel::initializeAnalysisResults(
     BatchAA = std::nullopt;
 
   SP = &FAM.getResult<SSPLayoutAnalysis>(Fn);
+
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
+  TTI = &FAM.getResult<TargetIRAnalysis>(Fn);
+#endif
 }
 
 void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
@@ -536,10 +537,7 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
   MachineModuleInfo &MMI =
       MFP.getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
 
-  TTI = &MFP.getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn);
-
-  CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, MMI, FnVarLocs,
-               TTI->hasBranchDivergence(&Fn));
+  CurDAG->init(*MF, *ORE, &MFP, LibInfo, UA, PSI, BFI, MMI, FnVarLocs);
 
   // Now get the optional analyzes if we want to.
   // This is based on the possibly changed OptLevel (after optnone is taken
@@ -558,6 +556,10 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) {
     BatchAA = std::nullopt;
 
   SP = &MFP.getAnalysis<StackProtector>().getLayoutInfo();
+
+#if !defined(NDEBUG) && LLVM_ENABLE_ABI_BREAKING_CHECKS
+  TTI = &MFP.getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn);
+#endif
 }
 
 bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 80aeefe8e068a..46a5e44374e1c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -1258,7 +1258,7 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
 
   if (Record.type == RecordType::Spill) {
     unsigned Index = Record.payload.FI;
-    SDValue SpillSlot = DAG.getTargetFrameIndex(Index, getFrameIndexTy());
+    SDValue SpillSlot = DAG.getFrameIndex(Index, getFrameIndexTy());
 
     // All the reloads are independent and are reading memory only modified by
     // statepoints (i.e. no other aliasing stores); informing SelectionDAG of
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index e235d144e85ff..402a012e8e555 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -162,14 +162,17 @@ TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
   TargetLowering::ArgListTy Args;
   Args.reserve(Ops.size());
 
-  TargetLowering::ArgListEntry Entry;
   ArrayRef<Type *> OpsTypeOverrides = CallOptions.OpsTypeOverrides;
   for (unsigned i = 0; i < Ops.size(); ++i) {
     SDValue NewOp = Ops[i];
-    Entry.Node = NewOp;
-    Entry.Ty = i < OpsTypeOverrides.size() && OpsTypeOverrides[i]
+    Type *Ty = i < OpsTypeOverrides.size() && OpsTypeOverrides[i]
                    ? OpsTypeOverrides[i]
-                   : Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
+                   : NewOp.getValueType().getTypeForEVT(*DAG.getContext());
+    TargetLowering::ArgListEntry Entry(NewOp, Ty);
+    if (CallOptions.IsSoften)
+      Entry.OrigTy =
+          CallOptions.OpsVTBeforeSoften[i].getTypeForEVT(*DAG.getContext());
+
     Entry.IsSExt =
         shouldSignExtendTypeInLibCall(Entry.Ty, CallOptions.IsSigned);
     Entry.IsZExt = !Entry.IsSExt;
@@ -189,18 +192,21 @@ TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
       DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout()));
 
   Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
+  Type *OrigRetTy = RetTy;
   TargetLowering::CallLoweringInfo CLI(DAG);
   bool signExtend = shouldSignExtendTypeInLibCall(RetTy, CallOptions.IsSigned);
   bool zeroExtend = !signExtend;
 
-  if (CallOptions.IsSoften &&
-      !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
-    signExtend = zeroExtend = false;
+  if (CallOptions.IsSoften) {
+    OrigRetTy = CallOptions.RetVTBeforeSoften.getTypeForEVT(*DAG.getContext());
+    if (!shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften))
+      signExtend = zeroExtend = false;
   }
 
   CLI.setDebugLoc(dl)
       .setChain(InChain)
-      .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
+      .setLibCallee(getLibcallCallingConv(LC), RetTy, OrigRetTy, Callee,
+                    std::move(Args))
       .setNoReturn(CallOptions.DoesNotReturn)
       .setDiscardResult(!CallOptions.IsReturnValueUsed)
       .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
@@ -420,7 +426,7 @@ void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
   TargetLowering::MakeLibCallOptions CallOptions;
   EVT OpsVT[2] = { OldLHS.getValueType(),
                    OldRHS.getValueType() };
-  CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
+  CallOptions.setTypeListBeforeSoften(OpsVT, RetVT);
   auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
   NewLHS = Call.first;
   NewRHS = DAG.getConstant(0, dl, RetVT);
@@ -5125,10 +5131,11 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           !ISD::isUnsignedIntSetCC(Cond))) &&
         isTypeDesirableForOp(ISD::SETCC, N0.getOperand(0).getValueType())) {
       EVT NewVT = N0.getOperand(0).getValueType();
-      SDValue NewConst = DAG.getConstant(ISD::isSignedIntSetCC(Cond)
-                                             ? C1.sext(NewVT.getSizeInBits())
-                                             : C1.zext(NewVT.getSizeInBits()),
-                                         dl, NewVT);
+      SDValue NewConst = DAG.getConstant(
+          (N0->getFlags().hasNoSignedWrap() && !ISD::isUnsignedIntSetCC(Cond))
+              ? C1.sext(NewVT.getSizeInBits())
+              : C1.zext(NewVT.getSizeInBits()),
+          dl, NewVT);
       return DAG.getSetCC(dl, VT, N0.getOperand(0), NewConst, Cond);
     }
 
@@ -10712,7 +10719,6 @@ SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
   SDLoc dl(GA);
 
   ArgListTy Args;
-  ArgListEntry Entry;
   const GlobalValue *GV =
       cast<GlobalValue>(GA->getGlobal()->stripPointerCastsAndAliases());
   SmallString<32> NameString("__emutls_v.");
@@ -10721,9 +10727,7 @@ SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
   const GlobalVariable *EmuTlsVar =
       GV->getParent()->getNamedGlobal(EmuTlsVarName);
   assert(EmuTlsVar && "Cannot find EmuTlsVar ");
-  Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
-  Entry.Ty = VoidPtrType;
-  Args.push_back(Entry);
+  Args.emplace_back(DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT), VoidPtrType);
 
   SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
 
diff --git a/llvm/lib/CodeGen/ShrinkWrap.cpp b/llvm/lib/CodeGen/ShrinkWrap.cpp
index 41e956caa7b34..938f2d756bc87 100644
--- a/llvm/lib/CodeGen/ShrinkWrap.cpp
+++ b/llvm/lib/CodeGen/ShrinkWrap.cpp
@@ -967,8 +967,14 @@ bool ShrinkWrapImpl::run(MachineFunction &MF) {
                     << "\nRestore: " << printMBBReference(*Restore) << '\n');
 
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  MFI.setSavePoint(Save);
-  MFI.setRestorePoint(Restore);
+  SmallVector<MachineBasicBlock *, 4> SavePoints;
+  SmallVector<MachineBasicBlock *, 4> RestorePoints;
+  if (Save) {
+    SavePoints.push_back(Save);
+    RestorePoints.push_back(Restore);
+  }
+  MFI.setSavePoints(SavePoints);
+  MFI.setRestorePoints(RestorePoints);
   ++NumCandidates;
   return Changed;
 }
diff --git a/llvm/lib/CodeGen/SwiftErrorValueTracking.cpp b/llvm/lib/CodeGen/SwiftErrorValueTracking.cpp
index decffdc7dfe45..4ae9726d7e0c2 100644
--- a/llvm/lib/CodeGen/SwiftErrorValueTracking.cpp
+++ b/llvm/lib/CodeGen/SwiftErrorValueTracking.cpp
@@ -14,7 +14,6 @@
 
 #include "llvm/CodeGen/SwiftErrorValueTracking.h"
 #include "llvm/ADT/PostOrderIterator.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
@@ -179,7 +178,7 @@ void SwiftErrorValueTracking::propagateVRegs() {
       // Check whether we have a single vreg def from all predecessors.
       // Otherwise we need a phi.
       SmallVector<std::pair<MachineBasicBlock *, Register>, 4> VRegs;
-      SmallSet<const MachineBasicBlock *, 8> Visited;
+      SmallPtrSet<const MachineBasicBlock *, 8> Visited;
       for (auto *Pred : MBB->predecessors()) {
         if (!Visited.insert(Pred).second)
           continue;
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 9f525ea4abc3c..9ffced80b07f1 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1008,7 +1008,7 @@ unsigned TargetLoweringBase::getBitWidthForCttzElements(
     CR = CR.subtract(APInt(64, 1));
 
   unsigned EltWidth = RetTy->getScalarSizeInBits();
-  EltWidth = std::min(EltWidth, (unsigned)CR.getActiveBits());
+  EltWidth = std::min(EltWidth, CR.getActiveBits());
   EltWidth = std::max(llvm::bit_ceil(EltWidth), (unsigned)8);
 
   return EltWidth;
@@ -1738,13 +1738,13 @@ void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType,
                          AttributeList attr,
                          SmallVectorImpl<ISD::OutputArg> &Outs,
                          const TargetLowering &TLI, const DataLayout &DL) {
-  SmallVector<EVT, 4> ValueVTs;
-  ComputeValueVTs(TLI, DL, ReturnType, ValueVTs);
-  unsigned NumValues = ValueVTs.size();
+  SmallVector<Type *, 4> Types;
+  ComputeValueTypes(DL, ReturnType, Types);
+  unsigned NumValues = Types.size();
   if (NumValues == 0) return;
 
-  for (unsigned j = 0, f = NumValues; j != f; ++j) {
-    EVT VT = ValueVTs[j];
+  for (Type *Ty : Types) {
+    EVT VT = TLI.getValueType(DL, Ty);
     ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
 
     if (attr.hasRetAttr(Attribute::SExt))
@@ -1772,7 +1772,7 @@ void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType,
       Flags.setZExt();
 
     for (unsigned i = 0; i < NumParts; ++i)
-      Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0));
+      Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, Ty, 0, 0));
   }
 }
 
@@ -1893,6 +1893,7 @@ int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const {
   case SIToFP:         return ISD::SINT_TO_FP;
   case FPTrunc:        return ISD::FP_ROUND;
   case FPExt:          return ISD::FP_EXTEND;
+  case PtrToAddr:      return ISD::BITCAST;
   case PtrToInt:       return ISD::BITCAST;
   case IntToPtr:       return ISD::BITCAST;
   case BitCast:        return ISD::BITCAST;
@@ -1923,6 +1924,8 @@ int TargetLoweringBase::IntrinsicIDToISD(Intrinsic::ID ID) const {
     return ISD::FEXP;
   case Intrinsic::exp2:
     return ISD::FEXP2;
+  case Intrinsic::log:
+    return ISD::FLOG;
   default:
     return ISD::DELETED_NODE;
   }
@@ -2056,29 +2059,37 @@ Value *TargetLoweringBase::getIRStackGuard(IRBuilderBase &IRB) const {
 // Currently only support "standard" __stack_chk_guard.
 // TODO: add LOAD_STACK_GUARD support.
 void TargetLoweringBase::insertSSPDeclarations(Module &M) const {
-  if (!M.getNamedValue("__stack_chk_guard")) {
-    auto *GV = new GlobalVariable(M, PointerType::getUnqual(M.getContext()),
-                                  false, GlobalVariable::ExternalLinkage,
-                                  nullptr, "__stack_chk_guard");
-
-    // FreeBSD has "__stack_chk_guard" defined externally on libc.so
-    if (M.getDirectAccessExternalData() &&
-        !TM.getTargetTriple().isOSCygMing() &&
-        !(TM.getTargetTriple().isPPC64() &&
-          TM.getTargetTriple().isOSFreeBSD()) &&
-        (!TM.getTargetTriple().isOSDarwin() ||
-         TM.getRelocationModel() == Reloc::Static))
-      GV->setDSOLocal(true);
-  }
+  RTLIB::LibcallImpl StackGuardImpl = getLibcallImpl(RTLIB::STACK_CHECK_GUARD);
+  if (StackGuardImpl == RTLIB::Unsupported)
+    return;
+
+  StringRef StackGuardVarName = getLibcallImplName(StackGuardImpl);
+  M.getOrInsertGlobal(
+      StackGuardVarName, PointerType::getUnqual(M.getContext()), [=, &M]() {
+        auto *GV = new GlobalVariable(M, PointerType::getUnqual(M.getContext()),
+                                      false, GlobalVariable::ExternalLinkage,
+                                      nullptr, StackGuardVarName);
+
+        // FreeBSD has "__stack_chk_guard" defined externally on libc.so
+        if (M.getDirectAccessExternalData() &&
+            !TM.getTargetTriple().isOSCygMing() &&
+            !(TM.getTargetTriple().isPPC64() &&
+              TM.getTargetTriple().isOSFreeBSD()) &&
+            (!TM.getTargetTriple().isOSDarwin() ||
+             TM.getRelocationModel() == Reloc::Static))
+          GV->setDSOLocal(true);
+
+        return GV;
+      });
 }
 
 // Currently only support "standard" __stack_chk_guard.
 // TODO: add LOAD_STACK_GUARD support.
 Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const {
-  if (getTargetMachine().getTargetTriple().isOSOpenBSD()) {
-    return M.getNamedValue("__guard_local");
-  }
-  return M.getNamedValue("__stack_chk_guard");
+  RTLIB::LibcallImpl GuardVarImpl = getLibcallImpl(RTLIB::STACK_CHECK_GUARD);
+  if (GuardVarImpl == RTLIB::Unsupported)
+    return nullptr;
+  return M.getNamedValue(getLibcallImplName(GuardVarImpl));
 }
 
 Function *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const {
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index 99ba893d6f096..972bd8f550e8b 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -99,7 +99,7 @@ unsigned VirtRegMap::createSpillSlot(const TargetRegisterClass *RC) {
   // Set preferred alignment if we are still able to realign the stack
   auto &ST = MF->getSubtarget();
   Align CurrentAlign = ST.getFrameLowering()->getStackAlign();
-  if (Alignment > CurrentAlign && !ST.getRegisterInfo()->canRealignStack(*MF)) {
+  if (Alignment > CurrentAlign && !TRI->canRealignStack(*MF)) {
     Alignment = CurrentAlign;
   }
   int SS = MF->getFrameInfo().CreateSpillStackObject(Size, Alignment);
diff --git a/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp b/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp
index 6ddb12ba04349..8052773812a2c 100644
--- a/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp
+++ b/llvm/lib/DWARFLinker/Classic/DWARFLinker.cpp
@@ -109,6 +109,7 @@ static bool isODRAttribute(uint16_t Attr) {
   case dwarf::DW_AT_specification:
   case dwarf::DW_AT_abstract_origin:
   case dwarf::DW_AT_import:
+  case dwarf::DW_AT_LLVM_alloc_type:
     return true;
   }
   llvm_unreachable("Improper attribute.");
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp b/llvm/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp
index 987e63963a068..a201fae84838c 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp
@@ -60,6 +60,20 @@ void DWARFGdbIndex::dumpSymbolTable(raw_ostream &OS) const {
                ", filled slots:",
                SymbolTableOffset, (uint64_t)SymbolTable.size())
      << '\n';
+
+  const auto FindCuVectorId = [&](uint32_t VecOffset) {
+    // Entries in ConstantPoolVectors are sorted by their offset in constant
+    // pool, see how ConstantPoolVectors is populated in parseImpl.
+    const auto *It =
+        llvm::lower_bound(ConstantPoolVectors, VecOffset,
+                          [](const auto &ConstantPoolEntry, uint32_t Offset) {
+                            return ConstantPoolEntry.first < Offset;
+                          });
+    assert(It != ConstantPoolVectors.end() && It->first == VecOffset &&
+           "Invalid symbol table");
+    return It - ConstantPoolVectors.begin();
+  };
+
   uint32_t I = -1;
   for (const SymTableEntry &E : SymbolTable) {
     ++I;
@@ -72,13 +86,7 @@ void DWARFGdbIndex::dumpSymbolTable(raw_ostream &OS) const {
     StringRef Name = ConstantPoolStrings.substr(
         ConstantPoolOffset - StringPoolOffset + E.NameOffset);
 
-    auto CuVector = llvm::find_if(
-        ConstantPoolVectors,
-        [&](const std::pair<uint32_t, SmallVector<uint32_t, 0>> &V) {
-          return V.first == E.VecOffset;
-        });
-    assert(CuVector != ConstantPoolVectors.end() && "Invalid symbol table");
-    uint32_t CuVectorId = CuVector - ConstantPoolVectors.begin();
+    const uint32_t CuVectorId = FindCuVectorId(E.VecOffset);
     OS << format("      String name: %s, CU vector index: %d\n", Name.data(),
                  CuVectorId);
   }
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp
index 80b4185b7c600..0df9137a3bd37 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVBinaryReader.cpp
@@ -275,7 +275,8 @@ void LVBinaryReader::mapVirtualAddress(const object::COFFObjectFile &COFFObj) {
 }
 
 Error LVBinaryReader::loadGenericTargetInfo(StringRef TheTriple,
-                                            StringRef TheFeatures) {
+                                            StringRef TheFeatures,
+                                            StringRef TheCPU) {
   std::string TargetLookupError;
   const Target *TheTarget =
       TargetRegistry::lookupTarget(TheTriple, TargetLookupError);
@@ -298,9 +299,8 @@ Error LVBinaryReader::loadGenericTargetInfo(StringRef TheTriple,
   MAI.reset(AsmInfo);
 
   // Target subtargets.
-  StringRef CPU;
   MCSubtargetInfo *SubtargetInfo(
-      TheTarget->createMCSubtargetInfo(TheTriple, CPU, TheFeatures));
+      TheTarget->createMCSubtargetInfo(TheTriple, TheCPU, TheFeatures));
   if (!SubtargetInfo)
     return createStringError(errc::invalid_argument,
                              "no subtarget info for target " + TheTriple);
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp
index e5895516b5e77..2ff70816b4bf1 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp
@@ -1190,7 +1190,12 @@ Error LVCodeViewReader::loadTargetInfo(const ObjectFile &Obj) {
     FeaturesValue = SubtargetFeatures();
   }
   FeaturesValue = *Features;
-  return loadGenericTargetInfo(TT.str(), FeaturesValue.getString());
+
+  StringRef CPU;
+  if (auto OptCPU = Obj.tryGetCPUName())
+    CPU = *OptCPU;
+
+  return loadGenericTargetInfo(TT.str(), FeaturesValue.getString(), CPU);
 }
 
 Error LVCodeViewReader::loadTargetInfo(const PDBFile &Pdb) {
@@ -1200,8 +1205,9 @@ Error LVCodeViewReader::loadTargetInfo(const PDBFile &Pdb) {
   TT.setOS(Triple::Win32);
 
   StringRef TheFeature = "";
+  StringRef TheCPU = "";
 
-  return loadGenericTargetInfo(TT.str(), TheFeature);
+  return loadGenericTargetInfo(TT.str(), TheFeature, TheCPU);
 }
 
 std::string LVCodeViewReader::getRegisterName(LVSmall Opcode,
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
index 696e2bc948a2e..62134dfdadf46 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVDWARFReader.cpp
@@ -956,10 +956,7 @@ LVElement *LVDWARFReader::getElementForOffset(LVOffset Offset,
 Error LVDWARFReader::loadTargetInfo(const ObjectFile &Obj) {
   // Detect the architecture from the object file. We usually don't need OS
   // info to lookup a target and create register info.
-  Triple TT;
-  TT.setArch(Triple::ArchType(Obj.getArch()));
-  TT.setVendor(Triple::UnknownVendor);
-  TT.setOS(Triple::UnknownOS);
+  Triple TT = Obj.makeTriple();
 
   // Features to be passed to target/subtarget
   Expected<SubtargetFeatures> Features = Obj.getFeatures();
@@ -969,7 +966,12 @@ Error LVDWARFReader::loadTargetInfo(const ObjectFile &Obj) {
     FeaturesValue = SubtargetFeatures();
   }
   FeaturesValue = *Features;
-  return loadGenericTargetInfo(TT.str(), FeaturesValue.getString());
+
+  StringRef CPU;
+  if (auto OptCPU = Obj.tryGetCPUName())
+    CPU = *OptCPU;
+
+  return loadGenericTargetInfo(TT.str(), FeaturesValue.getString(), CPU);
 }
 
 void LVDWARFReader::mapRangeAddress(const ObjectFile &Obj) {
diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
index 12d31f809f882..2f3234683bf55 100644
--- a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp
@@ -28,7 +28,7 @@ namespace aarch32 {
 
 /// Check whether the given target flags are set for this Symbol.
 bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags) {
-  return static_cast<TargetFlagsType>(Sym.getTargetFlags()) & Flags;
+  return Sym.getTargetFlags() & Flags;
 }
 
 /// Encode 22-bit immediate value for branch instructions without J1J2 range
diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp
index 1bafed79d6968..75ae80f610a56 100644
--- a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp
@@ -12,7 +12,6 @@
 #include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h"
 #include "llvm/ExecutionEngine/Orc/MachOBuilder.h"
 
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
@@ -64,7 +63,7 @@ class MachODebugObjectSynthesizerBase
       LLVM_DEBUG({
         dbgs() << "  Preserving debug section " << Sec.getName() << "\n";
       });
-      SmallSet<Block *, 8> PreservedBlocks;
+      SmallPtrSet<Block *, 8> PreservedBlocks;
       for (auto *Sym : Sec.symbols()) {
         bool NewPreservedBlock =
             PreservedBlocks.insert(&Sym->getBlock()).second;
diff --git a/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
index 8e4937d405620..91a31150c13de 100644
--- a/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp
@@ -169,7 +169,7 @@ Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
     std::vector<tpctypes::UInt64Write> PtrUpdates;
     for (auto &SI : StubInits)
       PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress,
-                            static_cast<uint64_t>(SI.second.first.getValue())});
+                            SI.second.first.getValue()});
     return MemAccess.writeUInt64s(PtrUpdates);
   }
   default:
diff --git a/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
index 19c000e2472a8..d460cf6de8a4d 100644
--- a/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
@@ -14,40 +14,39 @@
 namespace llvm {
 namespace orc {
 
-ThreadSafeModule cloneToContext(const ThreadSafeModule &TSM,
-                                ThreadSafeContext TSCtx,
-                                GVPredicate ShouldCloneDef,
-                                GVModifier UpdateClonedDefSource) {
-  assert(TSM && "Can not clone null module");
-
-  if (!ShouldCloneDef)
-    ShouldCloneDef = [](const GlobalValue &) { return true; };
-
-  // First copy the source module into a buffer.
+static std::pair<std::string, SmallVector<char, 1>>
+serializeModule(const Module &M, GVPredicate ShouldCloneDef,
+                GVModifier UpdateClonedDefSource) {
   std::string ModuleName;
   SmallVector<char, 1> ClonedModuleBuffer;
-  TSM.withModuleDo([&](Module &M) {
-    ModuleName = M.getModuleIdentifier();
-    std::set<GlobalValue *> ClonedDefsInSrc;
-    ValueToValueMapTy VMap;
-    auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) {
-      if (ShouldCloneDef(*GV)) {
-        ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV));
-        return true;
-      }
-      return false;
-    });
-
-    if (UpdateClonedDefSource)
-      for (auto *GV : ClonedDefsInSrc)
-        UpdateClonedDefSource(*GV);
-
-    BitcodeWriter BCWriter(ClonedModuleBuffer);
-    BCWriter.writeModule(*Tmp);
-    BCWriter.writeSymtab();
-    BCWriter.writeStrtab();
+
+  ModuleName = M.getModuleIdentifier();
+  std::set<GlobalValue *> ClonedDefsInSrc;
+  ValueToValueMapTy VMap;
+  auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) {
+    if (ShouldCloneDef(*GV)) {
+      ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV));
+      return true;
+    }
+    return false;
   });
 
+  if (UpdateClonedDefSource)
+    for (auto *GV : ClonedDefsInSrc)
+      UpdateClonedDefSource(*GV);
+
+  BitcodeWriter BCWriter(ClonedModuleBuffer);
+  BCWriter.writeModule(*Tmp);
+  BCWriter.writeSymtab();
+  BCWriter.writeStrtab();
+
+  return {std::move(ModuleName), std::move(ClonedModuleBuffer)};
+}
+
+ThreadSafeModule
+deserializeModule(std::string ModuleName,
+                  const SmallVector<char, 1> &ClonedModuleBuffer,
+                  ThreadSafeContext TSCtx) {
   MemoryBufferRef ClonedModuleBufferRef(
       StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()),
       "cloned module buffer");
@@ -63,6 +62,40 @@ ThreadSafeModule cloneToContext(const ThreadSafeModule &TSM,
   return ThreadSafeModule(std::move(M), std::move(TSCtx));
 }
 
+ThreadSafeModule
+cloneExternalModuleToContext(const Module &M, ThreadSafeContext TSCtx,
+                             GVPredicate ShouldCloneDef,
+                             GVModifier UpdateClonedDefSource) {
+
+  if (!ShouldCloneDef)
+    ShouldCloneDef = [](const GlobalValue &) { return true; };
+
+  auto [ModuleName, ClonedModuleBuffer] = serializeModule(
+      M, std::move(ShouldCloneDef), std::move(UpdateClonedDefSource));
+
+  return deserializeModule(std::move(ModuleName), ClonedModuleBuffer,
+                           std::move(TSCtx));
+}
+
+ThreadSafeModule cloneToContext(const ThreadSafeModule &TSM,
+                                ThreadSafeContext TSCtx,
+                                GVPredicate ShouldCloneDef,
+                                GVModifier UpdateClonedDefSource) {
+  assert(TSM && "Can not clone null module");
+
+  if (!ShouldCloneDef)
+    ShouldCloneDef = [](const GlobalValue &) { return true; };
+
+  // First copy the source module into a buffer.
+  auto [ModuleName, ClonedModuleBuffer] = TSM.withModuleDo([&](Module &M) {
+    return serializeModule(M, std::move(ShouldCloneDef),
+                           std::move(UpdateClonedDefSource));
+  });
+
+  return deserializeModule(std::move(ModuleName), ClonedModuleBuffer,
+                           std::move(TSCtx));
+}
+
 ThreadSafeModule cloneToNewContext(const ThreadSafeModule &TSM,
                                    GVPredicate ShouldCloneDef,
                                    GVModifier UpdateClonedDefSource) {
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index 08d6c78bd1eb1..d6268037dea86 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -654,11 +654,10 @@ bool RuntimeDyldELF::resolveLoongArch64ShortBranch(
     if (Loc == GlobalSymbolTable.end())
       return false;
     const auto &SymInfo = Loc->second;
-    Address =
-        uint64_t(Sections[SymInfo.getSectionID()].getLoadAddressWithOffset(
-            SymInfo.getOffset()));
+    Address = Sections[SymInfo.getSectionID()].getLoadAddressWithOffset(
+        SymInfo.getOffset());
   } else {
-    Address = uint64_t(Sections[Value.SectionID].getLoadAddress());
+    Address = Sections[Value.SectionID].getLoadAddress();
   }
   uint64_t Offset = RelI->getOffset();
   uint64_t SourceAddress = Sections[SectionID].getLoadAddressWithOffset(Offset);
diff --git a/llvm/lib/Frontend/HLSL/HLSLBinding.cpp b/llvm/lib/Frontend/HLSL/HLSLBinding.cpp
index d581311f22028..45391460354d0 100644
--- a/llvm/lib/Frontend/HLSL/HLSLBinding.cpp
+++ b/llvm/lib/Frontend/HLSL/HLSLBinding.cpp
@@ -76,7 +76,7 @@ BindingInfo BindingInfoBuilder::calculateBindingInfo(
   // remove duplicates
   Binding *NewEnd = llvm::unique(Bindings);
   if (NewEnd != Bindings.end())
-    Bindings.erase(NewEnd);
+    Bindings.erase(NewEnd, Bindings.end());
 
   BindingInfo Info;
 
diff --git a/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp b/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
index 78c20a6c5c9ff..92c62b83fadb0 100644
--- a/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
+++ b/llvm/lib/Frontend/HLSL/HLSLRootSignature.cpp
@@ -11,30 +11,13 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Frontend/HLSL/HLSLRootSignature.h"
+#include "llvm/Support/DXILABI.h"
 #include "llvm/Support/ScopedPrinter.h"
 
 namespace llvm {
 namespace hlsl {
 namespace rootsig {
 
-template <typename T>
-static std::optional<StringRef> getEnumName(const T Value,
-                                            ArrayRef<EnumEntry<T>> Enums) {
-  for (const auto &EnumItem : Enums)
-    if (EnumItem.Value == Value)
-      return EnumItem.Name;
-  return std::nullopt;
-}
-
-template <typename T>
-static raw_ostream &printEnum(raw_ostream &OS, const T Value,
-                              ArrayRef<EnumEntry<T>> Enums) {
-  auto MaybeName = getEnumName(Value, Enums);
-  if (MaybeName)
-    OS << *MaybeName;
-  return OS;
-}
-
 template <typename T>
 static raw_ostream &printFlags(raw_ostream &OS, const T Value,
                                ArrayRef<EnumEntry<T>> Flags) {
@@ -46,9 +29,9 @@ static raw_ostream &printFlags(raw_ostream &OS, const T Value,
       if (FlagSet)
         OS << " | ";
 
-      auto MaybeFlag = getEnumName(T(Bit), Flags);
-      if (MaybeFlag)
-        OS << *MaybeFlag;
+      StringRef MaybeFlag = enumToStringRef(T(Bit), Flags);
+      if (!MaybeFlag.empty())
+        OS << MaybeFlag;
       else
         OS << "invalid: " << Bit;
 
@@ -70,58 +53,49 @@ static const EnumEntry<RegisterType> RegisterNames[] = {
 };
 
 static raw_ostream &operator<<(raw_ostream &OS, const Register &Reg) {
-  printEnum(OS, Reg.ViewType, ArrayRef(RegisterNames));
-  OS << Reg.Number;
+  OS << enumToStringRef(Reg.ViewType, ArrayRef(RegisterNames)) << Reg.Number;
 
   return OS;
 }
 
 static raw_ostream &operator<<(raw_ostream &OS,
                                const llvm::dxbc::ShaderVisibility &Visibility) {
-  printEnum(OS, Visibility, dxbc::getShaderVisibility());
+  OS << enumToStringRef(Visibility, dxbc::getShaderVisibility());
 
   return OS;
 }
 
 static raw_ostream &operator<<(raw_ostream &OS,
                                const llvm::dxbc::SamplerFilter &Filter) {
-  printEnum(OS, Filter, dxbc::getSamplerFilters());
+  OS << enumToStringRef(Filter, dxbc::getSamplerFilters());
 
   return OS;
 }
 
 static raw_ostream &operator<<(raw_ostream &OS,
                                const dxbc::TextureAddressMode &Address) {
-  printEnum(OS, Address, dxbc::getTextureAddressModes());
+  OS << enumToStringRef(Address, dxbc::getTextureAddressModes());
 
   return OS;
 }
 
 static raw_ostream &operator<<(raw_ostream &OS,
                                const dxbc::ComparisonFunc &CompFunc) {
-  printEnum(OS, CompFunc, dxbc::getComparisonFuncs());
+  OS << enumToStringRef(CompFunc, dxbc::getComparisonFuncs());
 
   return OS;
 }
 
 static raw_ostream &operator<<(raw_ostream &OS,
                                const dxbc::StaticBorderColor &BorderColor) {
-  printEnum(OS, BorderColor, dxbc::getStaticBorderColors());
+  OS << enumToStringRef(BorderColor, dxbc::getStaticBorderColors());
 
   return OS;
 }
 
-static const EnumEntry<dxil::ResourceClass> ResourceClassNames[] = {
-    {"CBV", dxil::ResourceClass::CBuffer},
-    {"SRV", dxil::ResourceClass::SRV},
-    {"UAV", dxil::ResourceClass::UAV},
-    {"Sampler", dxil::ResourceClass::Sampler},
-};
-
-static raw_ostream &operator<<(raw_ostream &OS, const ClauseType &Type) {
-  printEnum(OS, dxil::ResourceClass(llvm::to_underlying(Type)),
-            ArrayRef(ResourceClassNames));
-
+static raw_ostream &operator<<(raw_ostream &OS,
+                               const dxil::ResourceClass &Type) {
+  OS << dxil::getResourceClassName(Type);
   return OS;
 }
 
@@ -179,8 +153,7 @@ raw_ostream &operator<<(raw_ostream &OS, const DescriptorTableClause &Clause) {
 }
 
 raw_ostream &operator<<(raw_ostream &OS, const RootDescriptor &Descriptor) {
-  ClauseType Type = ClauseType(llvm::to_underlying(Descriptor.Type));
-  OS << "Root" << Type << "(" << Descriptor.Reg
+  OS << "Root" << Descriptor.Type << "(" << Descriptor.Reg
      << ", space = " << Descriptor.Space
      << ", visibility = " << Descriptor.Visibility
      << ", flags = " << Descriptor.Flags << ")";
diff --git a/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
index 6d89fa7b1222c..dece8f197aaf7 100644
--- a/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
+++ b/llvm/lib/Frontend/HLSL/RootSignatureMetadata.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Frontend/HLSL/RootSignatureValidations.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Metadata.h"
+#include "llvm/Support/DXILABI.h"
 #include "llvm/Support/ScopedPrinter.h"
 
 using namespace llvm;
@@ -51,20 +52,6 @@ static std::optional<StringRef> extractMdStringValue(MDNode *Node,
   return NodeText->getString();
 }
 
-static const EnumEntry<dxil::ResourceClass> ResourceClassNames[] = {
-    {"CBV", dxil::ResourceClass::CBuffer},
-    {"SRV", dxil::ResourceClass::SRV},
-    {"UAV", dxil::ResourceClass::UAV},
-    {"Sampler", dxil::ResourceClass::Sampler},
-};
-
-static std::optional<StringRef> getResourceName(dxil::ResourceClass Class) {
-  for (const auto &ClassEnum : ResourceClassNames)
-    if (ClassEnum.Value == Class)
-      return ClassEnum.Name;
-  return std::nullopt;
-}
-
 namespace {
 
 // We use the OverloadVisit with std::visit to ensure the compiler catches if a
@@ -133,10 +120,9 @@ MDNode *MetadataBuilder::BuildRootConstants(const RootConstants &Constants) {
 
 MDNode *MetadataBuilder::BuildRootDescriptor(const RootDescriptor &Descriptor) {
   IRBuilder<> Builder(Ctx);
-  std::optional<StringRef> ResName =
-      getResourceName(dxil::ResourceClass(to_underlying(Descriptor.Type)));
-  assert(ResName && "Provided an invalid Resource Class");
-  SmallString<7> Name({"Root", *ResName});
+  StringRef ResName = dxil::getResourceClassName(Descriptor.Type);
+  assert(!ResName.empty() && "Provided an invalid Resource Class");
+  SmallString<7> Name({"Root", ResName});
   Metadata *Operands[] = {
       MDString::get(Ctx, Name),
       ConstantAsMetadata::get(
@@ -174,11 +160,10 @@ MDNode *MetadataBuilder::BuildDescriptorTable(const DescriptorTable &Table) {
 MDNode *MetadataBuilder::BuildDescriptorTableClause(
     const DescriptorTableClause &Clause) {
   IRBuilder<> Builder(Ctx);
-  std::optional<StringRef> ResName =
-      getResourceName(dxil::ResourceClass(to_underlying(Clause.Type)));
-  assert(ResName && "Provided an invalid Resource Class");
+  StringRef ResName = dxil::getResourceClassName(Clause.Type);
+  assert(!ResName.empty() && "Provided an invalid Resource Class");
   Metadata *Operands[] = {
-      MDString::get(Ctx, *ResName),
+      MDString::get(Ctx, ResName),
       ConstantAsMetadata::get(Builder.getInt32(Clause.NumDescriptors)),
       ConstantAsMetadata::get(Builder.getInt32(Clause.Reg.Number)),
       ConstantAsMetadata::get(Builder.getInt32(Clause.Space)),
diff --git a/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp b/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp
index 9d84aa838f476..72308a3de5fd4 100644
--- a/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp
+++ b/llvm/lib/Frontend/HLSL/RootSignatureValidations.cpp
@@ -29,7 +29,7 @@ bool verifyRegisterValue(uint32_t RegisterValue) {
 // This Range is reserverved, therefore invalid, according to the spec
 // https://github.com/llvm/wg-hlsl/blob/main/proposals/0002-root-signature-in-clang.md#all-the-values-should-be-legal
 bool verifyRegisterSpace(uint32_t RegisterSpace) {
-  return !(RegisterSpace >= 0xFFFFFFF0 && RegisterSpace <= 0xFFFFFFFF);
+  return !(RegisterSpace >= 0xFFFFFFF0);
 }
 
 bool verifyRootDescriptorFlag(uint32_t Version, uint32_t FlagsVal) {
diff --git a/llvm/lib/Frontend/OpenMP/OMP.cpp b/llvm/lib/Frontend/OpenMP/OMP.cpp
index 555e2a61e411e..9e625b809de9e 100644
--- a/llvm/lib/Frontend/OpenMP/OMP.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMP.cpp
@@ -190,7 +190,7 @@ bool isCombinedConstruct(Directive D) {
 }
 
 ArrayRef<unsigned> getOpenMPVersions() {
-  static unsigned Versions[]{31, 40, 45, 50, 51, 52, 60};
+  static unsigned Versions[]{31, 40, 45, 50, 51, 52, 60, 61};
   return Versions;
 }
 
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 3aa4f7ae04c33..50ab206e2db8e 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -14,7 +14,6 @@
 
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
 #include "llvm/ADT/SmallBitVector.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/AssumptionCache.h"
@@ -151,6 +150,18 @@ static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType) {
 }
 #endif
 
+/// This is wrapper over IRBuilderBase::restoreIP that also restores the current
+/// debug location to the last instruction in the specified basic block if the
+/// insert point points to the end of the block.
+static void restoreIPandDebugLoc(llvm::IRBuilderBase &Builder,
+                                 llvm::IRBuilderBase::InsertPoint IP) {
+  Builder.restoreIP(IP);
+  llvm::BasicBlock *BB = Builder.GetInsertBlock();
+  llvm::BasicBlock::iterator I = Builder.GetInsertPoint();
+  if (!BB->empty() && I == BB->end())
+    Builder.SetCurrentDebugLocation(BB->back().getStableDebugLoc());
+}
+
 static const omp::GV &getGridValue(const Triple &T, Function *Kernel) {
   if (T.isAMDGPU()) {
     StringRef Features =
@@ -4014,6 +4025,340 @@ OpenMPIRBuilder::createMasked(const LocationDescription &Loc,
                               /*Conditional*/ true, /*hasFinalize*/ true);
 }
 
+static llvm::CallInst *emitNoUnwindRuntimeCall(IRBuilder<> &Builder,
+                                               llvm::FunctionCallee Callee,
+                                               ArrayRef<llvm::Value *> Args,
+                                               const llvm::Twine &Name) {
+  llvm::CallInst *Call = Builder.CreateCall(
+      Callee, Args, SmallVector<llvm::OperandBundleDef, 1>(), Name);
+  Call->setDoesNotThrow();
+  return Call;
+}
+
+// Expects input basic block is dominated by BeforeScanBB.
+// Once Scan directive is encountered, the code after scan directive should be
+// dominated by AfterScanBB. Scan directive splits the code sequence to
+// scan and input phase. Based on whether inclusive or exclusive
+// clause is used in the scan directive and whether input loop or scan loop
+// is lowered, it adds jumps to input and scan phase. First Scan loop is the
+// input loop and second is the scan loop. The code generated handles only
+// inclusive scans now.
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createScan(
+    const LocationDescription &Loc, InsertPointTy AllocaIP,
+    ArrayRef<llvm::Value *> ScanVars, ArrayRef<llvm::Type *> ScanVarsType,
+    bool IsInclusive, ScanInfo *ScanRedInfo) {
+  if (ScanRedInfo->OMPFirstScanLoop) {
+    llvm::Error Err = emitScanBasedDirectiveDeclsIR(AllocaIP, ScanVars,
+                                                    ScanVarsType, ScanRedInfo);
+    if (Err)
+      return Err;
+  }
+  if (!updateToLocation(Loc))
+    return Loc.IP;
+
+  llvm::Value *IV = ScanRedInfo->IV;
+
+  if (ScanRedInfo->OMPFirstScanLoop) {
+    // Emit buffer[i] = red; at the end of the input phase.
+    for (size_t i = 0; i < ScanVars.size(); i++) {
+      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
+      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
+      Type *DestTy = ScanVarsType[i];
+      Value *Val = Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");
+      Value *Src = Builder.CreateLoad(DestTy, ScanVars[i]);
+
+      Builder.CreateStore(Src, Val);
+    }
+  }
+  Builder.CreateBr(ScanRedInfo->OMPScanLoopExit);
+  emitBlock(ScanRedInfo->OMPScanDispatch,
+            Builder.GetInsertBlock()->getParent());
+
+  if (!ScanRedInfo->OMPFirstScanLoop) {
+    IV = ScanRedInfo->IV;
+    // Emit red = buffer[i]; at the entrance to the scan phase.
+    // TODO: if exclusive scan, the red = buffer[i-1] needs to be updated.
+    for (size_t i = 0; i < ScanVars.size(); i++) {
+      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]];
+      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
+      Type *DestTy = ScanVarsType[i];
+      Value *SrcPtr =
+          Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");
+      Value *Src = Builder.CreateLoad(DestTy, SrcPtr);
+      Builder.CreateStore(Src, ScanVars[i]);
+    }
+  }
+
+  // TODO: Update it to CreateBr and remove dead blocks
+  llvm::Value *CmpI = Builder.getInt1(true);
+  if (ScanRedInfo->OMPFirstScanLoop == IsInclusive) {
+    Builder.CreateCondBr(CmpI, ScanRedInfo->OMPBeforeScanBlock,
+                         ScanRedInfo->OMPAfterScanBlock);
+  } else {
+    Builder.CreateCondBr(CmpI, ScanRedInfo->OMPAfterScanBlock,
+                         ScanRedInfo->OMPBeforeScanBlock);
+  }
+  emitBlock(ScanRedInfo->OMPAfterScanBlock,
+            Builder.GetInsertBlock()->getParent());
+  Builder.SetInsertPoint(ScanRedInfo->OMPAfterScanBlock);
+  return Builder.saveIP();
+}
+
+Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
+    InsertPointTy AllocaIP, ArrayRef<Value *> ScanVars,
+    ArrayRef<Type *> ScanVarsType, ScanInfo *ScanRedInfo) {
+
+  Builder.restoreIP(AllocaIP);
+  // Create the shared pointer at alloca IP.
+  for (size_t i = 0; i < ScanVars.size(); i++) {
+    llvm::Value *BuffPtr =
+        Builder.CreateAlloca(Builder.getPtrTy(), nullptr, "vla");
+    (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]] = BuffPtr;
+  }
+
+  // Allocate temporary buffer by master thread
+  auto BodyGenCB = [&](InsertPointTy AllocaIP,
+                       InsertPointTy CodeGenIP) -> Error {
+    Builder.restoreIP(CodeGenIP);
+    Value *AllocSpan =
+        Builder.CreateAdd(ScanRedInfo->Span, Builder.getInt32(1));
+    for (size_t i = 0; i < ScanVars.size(); i++) {
+      Type *IntPtrTy = Builder.getInt32Ty();
+      Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
+      Allocsize = ConstantExpr::getTruncOrBitCast(Allocsize, IntPtrTy);
+      Value *Buff = Builder.CreateMalloc(IntPtrTy, ScanVarsType[i], Allocsize,
+                                         AllocSpan, nullptr, "arr");
+      Builder.CreateStore(Buff, (*(ScanRedInfo->ScanBuffPtrs))[ScanVars[i]]);
+    }
+    return Error::success();
+  };
+  // TODO: Perform finalization actions for variables. This has to be
+  // called for variables which have destructors/finalizers.
+  auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
+
+  Builder.SetInsertPoint(ScanRedInfo->OMPScanInit->getTerminator());
+  llvm::Value *FilterVal = Builder.getInt32(0);
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
+
+  if (!AfterIP)
+    return AfterIP.takeError();
+  Builder.restoreIP(*AfterIP);
+  BasicBlock *InputBB = Builder.GetInsertBlock();
+  if (InputBB->getTerminator())
+    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
+  AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+  if (!AfterIP)
+    return AfterIP.takeError();
+  Builder.restoreIP(*AfterIP);
+
+  return Error::success();
+}
+
+Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
+    ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
+  auto BodyGenCB = [&](InsertPointTy AllocaIP,
+                       InsertPointTy CodeGenIP) -> Error {
+    Builder.restoreIP(CodeGenIP);
+    for (ReductionInfo RedInfo : ReductionInfos) {
+      Value *PrivateVar = RedInfo.PrivateVariable;
+      Value *OrigVar = RedInfo.Variable;
+      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[PrivateVar];
+      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
+
+      Type *SrcTy = RedInfo.ElementType;
+      Value *Val = Builder.CreateInBoundsGEP(SrcTy, Buff, ScanRedInfo->Span,
+                                             "arrayOffset");
+      Value *Src = Builder.CreateLoad(SrcTy, Val);
+
+      Builder.CreateStore(Src, OrigVar);
+      Builder.CreateFree(Buff);
+    }
+    return Error::success();
+  };
+  // TODO: Perform finalization actions for variables. This has to be
+  // called for variables which have destructors/finalizers.
+  auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
+
+  if (ScanRedInfo->OMPScanFinish->getTerminator())
+    Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish->getTerminator());
+  else
+    Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
+
+  llvm::Value *FilterVal = Builder.getInt32(0);
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
+
+  if (!AfterIP)
+    return AfterIP.takeError();
+  Builder.restoreIP(*AfterIP);
+  BasicBlock *InputBB = Builder.GetInsertBlock();
+  if (InputBB->getTerminator())
+    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
+  AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+  if (!AfterIP)
+    return AfterIP.takeError();
+  Builder.restoreIP(*AfterIP);
+  return Error::success();
+}
+
+OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
+    const LocationDescription &Loc,
+    ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
+    ScanInfo *ScanRedInfo) {
+
+  if (!updateToLocation(Loc))
+    return Loc.IP;
+  auto BodyGenCB = [&](InsertPointTy AllocaIP,
+                       InsertPointTy CodeGenIP) -> Error {
+    Builder.restoreIP(CodeGenIP);
+    Function *CurFn = Builder.GetInsertBlock()->getParent();
+    // for (int k = 0; k <= ceil(log2(n)); ++k)
+    llvm::BasicBlock *LoopBB =
+        BasicBlock::Create(CurFn->getContext(), "omp.outer.log.scan.body");
+    llvm::BasicBlock *ExitBB =
+        splitBB(Builder, false, "omp.outer.log.scan.exit");
+    llvm::Function *F = llvm::Intrinsic::getOrInsertDeclaration(
+        Builder.GetInsertBlock()->getModule(),
+        (llvm::Intrinsic::ID)llvm::Intrinsic::log2, Builder.getDoubleTy());
+    llvm::BasicBlock *InputBB = Builder.GetInsertBlock();
+    llvm::Value *Arg =
+        Builder.CreateUIToFP(ScanRedInfo->Span, Builder.getDoubleTy());
+    llvm::Value *LogVal = emitNoUnwindRuntimeCall(Builder, F, Arg, "");
+    F = llvm::Intrinsic::getOrInsertDeclaration(
+        Builder.GetInsertBlock()->getModule(),
+        (llvm::Intrinsic::ID)llvm::Intrinsic::ceil, Builder.getDoubleTy());
+    LogVal = emitNoUnwindRuntimeCall(Builder, F, LogVal, "");
+    LogVal = Builder.CreateFPToUI(LogVal, Builder.getInt32Ty());
+    llvm::Value *NMin1 = Builder.CreateNUWSub(
+        ScanRedInfo->Span,
+        llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
+    Builder.SetInsertPoint(InputBB);
+    Builder.CreateBr(LoopBB);
+    emitBlock(LoopBB, CurFn);
+    Builder.SetInsertPoint(LoopBB);
+
+    PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+    // size pow2k = 1;
+    PHINode *Pow2K = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+    Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
+                         InputBB);
+    Pow2K->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 1),
+                       InputBB);
+    // for (size i = n - 1; i >= 2 ^ k; --i)
+    //   tmp[i] op= tmp[i-pow2k];
+    llvm::BasicBlock *InnerLoopBB =
+        BasicBlock::Create(CurFn->getContext(), "omp.inner.log.scan.body");
+    llvm::BasicBlock *InnerExitBB =
+        BasicBlock::Create(CurFn->getContext(), "omp.inner.log.scan.exit");
+    llvm::Value *CmpI = Builder.CreateICmpUGE(NMin1, Pow2K);
+    Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
+    emitBlock(InnerLoopBB, CurFn);
+    Builder.SetInsertPoint(InnerLoopBB);
+    PHINode *IVal = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+    IVal->addIncoming(NMin1, LoopBB);
+    for (ReductionInfo RedInfo : ReductionInfos) {
+      Value *ReductionVal = RedInfo.PrivateVariable;
+      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
+      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
+      Type *DestTy = RedInfo.ElementType;
+      Value *IV = Builder.CreateAdd(IVal, Builder.getInt32(1));
+      Value *LHSPtr =
+          Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");
+      Value *OffsetIval = Builder.CreateNUWSub(IV, Pow2K);
+      Value *RHSPtr =
+          Builder.CreateInBoundsGEP(DestTy, Buff, OffsetIval, "arrayOffset");
+      Value *LHS = Builder.CreateLoad(DestTy, LHSPtr);
+      Value *RHS = Builder.CreateLoad(DestTy, RHSPtr);
+      llvm::Value *Result;
+      InsertPointOrErrorTy AfterIP =
+          RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.CreateStore(Result, LHSPtr);
+    }
+    llvm::Value *NextIVal = Builder.CreateNUWSub(
+        IVal, llvm::ConstantInt::get(Builder.getInt32Ty(), 1));
+    IVal->addIncoming(NextIVal, Builder.GetInsertBlock());
+    CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);
+    Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
+    emitBlock(InnerExitBB, CurFn);
+    llvm::Value *Next = Builder.CreateNUWAdd(
+        Counter, llvm::ConstantInt::get(Counter->getType(), 1));
+    Counter->addIncoming(Next, Builder.GetInsertBlock());
+    // pow2k <<= 1;
+    llvm::Value *NextPow2K = Builder.CreateShl(Pow2K, 1, "", /*HasNUW=*/true);
+    Pow2K->addIncoming(NextPow2K, Builder.GetInsertBlock());
+    llvm::Value *Cmp = Builder.CreateICmpNE(Next, LogVal);
+    Builder.CreateCondBr(Cmp, LoopBB, ExitBB);
+    Builder.SetInsertPoint(ExitBB->getFirstInsertionPt());
+    return Error::success();
+  };
+
+  // TODO: Perform finalization actions for variables. This has to be
+  // called for variables which have destructors/finalizers.
+  auto FiniCB = [&](InsertPointTy CodeGenIP) { return llvm::Error::success(); };
+
+  llvm::Value *FilterVal = Builder.getInt32(0);
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
+      createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
+
+  if (!AfterIP)
+    return AfterIP.takeError();
+  Builder.restoreIP(*AfterIP);
+  AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+
+  if (!AfterIP)
+    return AfterIP.takeError();
+  Builder.restoreIP(*AfterIP);
+  Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
+  if (Err)
+    return Err;
+
+  return AfterIP;
+}
+
+Error OpenMPIRBuilder::emitScanBasedDirectiveIR(
+    llvm::function_ref<Error()> InputLoopGen,
+    llvm::function_ref<Error(LocationDescription Loc)> ScanLoopGen,
+    ScanInfo *ScanRedInfo) {
+
+  {
+    // Emit loop with input phase:
+    // for (i: 0..<num_iters>) {
+    //   <input phase>;
+    //   buffer[i] = red;
+    // }
+    ScanRedInfo->OMPFirstScanLoop = true;
+    Error Err = InputLoopGen();
+    if (Err)
+      return Err;
+  }
+  {
+    // Emit loop with scan phase:
+    // for (i: 0..<num_iters>) {
+    //   red = buffer[i];
+    //   <scan phase>;
+    // }
+    ScanRedInfo->OMPFirstScanLoop = false;
+    Error Err = ScanLoopGen(Builder.saveIP());
+    if (Err)
+      return Err;
+  }
+  return Error::success();
+}
+
+void OpenMPIRBuilder::createScanBBs(ScanInfo *ScanRedInfo) {
+  Function *Fun = Builder.GetInsertBlock()->getParent();
+  ScanRedInfo->OMPScanDispatch =
+      BasicBlock::Create(Fun->getContext(), "omp.inscan.dispatch");
+  ScanRedInfo->OMPAfterScanBlock =
+      BasicBlock::Create(Fun->getContext(), "omp.after.scan.bb");
+  ScanRedInfo->OMPBeforeScanBlock =
+      BasicBlock::Create(Fun->getContext(), "omp.before.scan.bb");
+  ScanRedInfo->OMPScanLoopExit =
+      BasicBlock::Create(Fun->getContext(), "omp.scan.loop.exit");
+}
 CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
     DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
     BasicBlock *PostInsertBefore, const Twine &Name) {
@@ -4111,6 +4456,76 @@ OpenMPIRBuilder::createCanonicalLoop(const LocationDescription &Loc,
   return CL;
 }
 
+Expected<ScanInfo *> OpenMPIRBuilder::scanInfoInitialize() {
+  ScanInfos.emplace_front();
+  ScanInfo *Result = &ScanInfos.front();
+  return Result;
+}
+
+Expected<SmallVector<llvm::CanonicalLoopInfo *>>
+OpenMPIRBuilder::createCanonicalScanLoops(
+    const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
+    Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
+    InsertPointTy ComputeIP, const Twine &Name, ScanInfo *ScanRedInfo) {
+  LocationDescription ComputeLoc =
+      ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
+  updateToLocation(ComputeLoc);
+
+  SmallVector<CanonicalLoopInfo *> Result;
+
+  Value *TripCount = calculateCanonicalLoopTripCount(
+      ComputeLoc, Start, Stop, Step, IsSigned, InclusiveStop, Name);
+  ScanRedInfo->Span = TripCount;
+  ScanRedInfo->OMPScanInit = splitBB(Builder, true, "scan.init");
+  Builder.SetInsertPoint(ScanRedInfo->OMPScanInit);
+
+  auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
+    Builder.restoreIP(CodeGenIP);
+    ScanRedInfo->IV = IV;
+    createScanBBs(ScanRedInfo);
+    BasicBlock *InputBlock = Builder.GetInsertBlock();
+    Instruction *Terminator = InputBlock->getTerminator();
+    assert(Terminator->getNumSuccessors() == 1);
+    BasicBlock *ContinueBlock = Terminator->getSuccessor(0);
+    Terminator->setSuccessor(0, ScanRedInfo->OMPScanDispatch);
+    emitBlock(ScanRedInfo->OMPBeforeScanBlock,
+              Builder.GetInsertBlock()->getParent());
+    Builder.CreateBr(ScanRedInfo->OMPScanLoopExit);
+    emitBlock(ScanRedInfo->OMPScanLoopExit,
+              Builder.GetInsertBlock()->getParent());
+    Builder.CreateBr(ContinueBlock);
+    Builder.SetInsertPoint(
+        ScanRedInfo->OMPBeforeScanBlock->getFirstInsertionPt());
+    return BodyGenCB(Builder.saveIP(), IV);
+  };
+
+  const auto &&InputLoopGen = [&]() -> Error {
+    Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
+        Builder.saveIP(), BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
+        ComputeIP, Name, true, ScanRedInfo);
+    if (!LoopInfo)
+      return LoopInfo.takeError();
+    Result.push_back(*LoopInfo);
+    Builder.restoreIP((*LoopInfo)->getAfterIP());
+    return Error::success();
+  };
+  const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
+    Expected<CanonicalLoopInfo *> LoopInfo =
+        createCanonicalLoop(Loc, BodyGen, Start, Stop, Step, IsSigned,
+                            InclusiveStop, ComputeIP, Name, true, ScanRedInfo);
+    if (!LoopInfo)
+      return LoopInfo.takeError();
+    Result.push_back(*LoopInfo);
+    Builder.restoreIP((*LoopInfo)->getAfterIP());
+    ScanRedInfo->OMPScanFinish = Builder.GetInsertBlock();
+    return Error::success();
+  };
+  Error Err = emitScanBasedDirectiveIR(InputLoopGen, ScanLoopGen, ScanRedInfo);
+  if (Err)
+    return Err;
+  return Result;
+}
+
 Value *OpenMPIRBuilder::calculateCanonicalLoopTripCount(
     const LocationDescription &Loc, Value *Start, Value *Stop, Value *Step,
     bool IsSigned, bool InclusiveStop, const Twine &Name) {
@@ -4174,7 +4589,8 @@ Value *OpenMPIRBuilder::calculateCanonicalLoopTripCount(
 Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
     const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
     Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
-    InsertPointTy ComputeIP, const Twine &Name) {
+    InsertPointTy ComputeIP, const Twine &Name, bool InScan,
+    ScanInfo *ScanRedInfo) {
   LocationDescription ComputeLoc =
       ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
 
@@ -4185,6 +4601,8 @@ Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
     Builder.restoreIP(CodeGenIP);
     Value *Span = Builder.CreateMul(IV, Step);
     Value *IndVar = Builder.CreateAdd(Span, Start);
+    if (InScan)
+      ScanRedInfo->IV = IndVar;
     return BodyGenCB(Builder.saveIP(), IndVar);
   };
   LocationDescription LoopLoc =
@@ -4550,6 +4968,7 @@ static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder,
   RealArgs.push_back(TripCount);
   if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
     RealArgs.push_back(ConstantInt::get(TripCountTy, 0));
+    RealArgs.push_back(ConstantInt::get(Builder.getInt8Ty(), 0));
     Builder.restoreIP({InsertBlock, std::prev(InsertBlock->end())});
     Builder.CreateCall(RTLFn, RealArgs);
     return;
@@ -4565,6 +4984,7 @@ static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder,
   if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
     RealArgs.push_back(ConstantInt::get(TripCountTy, 0));
   }
+  RealArgs.push_back(ConstantInt::get(Builder.getInt8Ty(), 0));
 
   Builder.CreateCall(RTLFn, RealArgs);
 }
@@ -5026,8 +5446,7 @@ OpenMPIRBuilder::collapseLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
     }
 
     // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
-    CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount,
-                                           {}, /*HasNUW=*/true);
+    CollapsedTripCount = Builder.CreateNUWMul(CollapsedTripCount, OrigTripCount);
   }
 
   // Create the collapsed loop control flow.
@@ -5511,7 +5930,7 @@ void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop,
     createIfVersion(CanonicalLoop, IfCond, VMap, LIA, LI, L, "simd");
   }
 
-  SmallSet<BasicBlock *, 8> Reachable;
+  SmallPtrSet<BasicBlock *, 8> Reachable;
 
   // Get the basic blocks from the loop in which memref instructions
   // can be found.
@@ -6828,7 +7247,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createTargetData(
           BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv);
       if (!AfterIP)
         return AfterIP.takeError();
-      Builder.restoreIP(*AfterIP);
+      restoreIPandDebugLoc(Builder, *AfterIP);
 
       if (IfCond)
         return emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP);
@@ -8586,7 +9005,7 @@ Error OpenMPIRBuilder::emitOffloadingArrays(
     ArrayType *SizeArrayType = ArrayType::get(Int64Ty, Info.NumberOfPtrs);
     Info.RTArgs.SizesArray = Builder.CreateAlloca(
         SizeArrayType, /* ArraySize = */ nullptr, ".offload_sizes");
-    Builder.restoreIP(CodeGenIP);
+    restoreIPandDebugLoc(Builder, CodeGenIP);
   } else {
     auto *SizesArrayInit = ConstantArray::get(
         ArrayType::get(Int64Ty, ConstSizes.size()), ConstSizes);
@@ -8605,7 +9024,7 @@ Error OpenMPIRBuilder::emitOffloadingArrays(
       AllocaInst *Buffer = Builder.CreateAlloca(
           SizeArrayType, /* ArraySize = */ nullptr, ".offload_sizes");
       Buffer->setAlignment(OffloadSizeAlign);
-      Builder.restoreIP(CodeGenIP);
+      restoreIPandDebugLoc(Builder, CodeGenIP);
       Builder.CreateMemCpy(
           Buffer, M.getDataLayout().getPrefTypeAlign(Buffer->getType()),
           SizesArrayGbl, OffloadSizeAlign,
@@ -8615,7 +9034,7 @@ Error OpenMPIRBuilder::emitOffloadingArrays(
 
       Info.RTArgs.SizesArray = Buffer;
     }
-    Builder.restoreIP(CodeGenIP);
+    restoreIPandDebugLoc(Builder, CodeGenIP);
   }
 
   // The map types are always constant so we don't need to generate code to
@@ -8956,7 +9375,8 @@ OpenMPIRBuilder::createAtomicWrite(const LocationDescription &Loc,
 OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
     const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
     Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-    AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr) {
+    AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr,
+    bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
   assert(!isConflictIP(Loc.IP, AllocaIP) && "IPs must not be ambiguous");
   if (!updateToLocation(Loc))
     return Loc.IP;
@@ -8974,9 +9394,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicUpdate(
            "OpenMP atomic does not support LT or GT operations");
   });
 
-  Expected<std::pair<Value *, Value *>> AtomicResult =
-      emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp,
-                       X.IsVolatile, IsXBinopExpr);
+  Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
+      AllocaIP, X.Var, X.ElemTy, Expr, AO, RMWOp, UpdateOp, X.IsVolatile,
+      IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
   if (!AtomicResult)
     return AtomicResult.takeError();
   checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
@@ -9023,7 +9443,8 @@ Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2,
 Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
     InsertPointTy AllocaIP, Value *X, Type *XElemTy, Value *Expr,
     AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp,
-    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr) {
+    AtomicUpdateCallbackTy &UpdateOp, bool VolatileX, bool IsXBinopExpr,
+    bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
   // TODO: handle the case where XElemTy is not byte-sized or not a power of 2
   // or a complex datatype.
   bool emitRMWOp = false;
@@ -9046,7 +9467,20 @@ Expected<std::pair<Value *, Value *>> OpenMPIRBuilder::emitAtomicUpdate(
 
   std::pair<Value *, Value *> Res;
   if (emitRMWOp) {
-    Res.first = Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
+    AtomicRMWInst *RMWInst =
+        Builder.CreateAtomicRMW(RMWOp, X, Expr, llvm::MaybeAlign(), AO);
+    if (T.isAMDGPU()) {
+      if (IsIgnoreDenormalMode)
+        RMWInst->setMetadata("amdgpu.ignore.denormal.mode",
+                             llvm::MDNode::get(Builder.getContext(), {}));
+      if (!IsFineGrainedMemory)
+        RMWInst->setMetadata("amdgpu.no.fine.grained.memory",
+                             llvm::MDNode::get(Builder.getContext(), {}));
+      if (!IsRemoteMemory)
+        RMWInst->setMetadata("amdgpu.no.remote.memory",
+                             llvm::MDNode::get(Builder.getContext(), {}));
+    }
+    Res.first = RMWInst;
     // not needed except in case of postfix captures. Generate anyway for
     // consistency with the else part. Will be removed with any DCE pass.
     // AtomicRMWInst::Xchg does not have a coressponding instruction.
@@ -9178,7 +9612,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
     const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X,
     AtomicOpValue &V, Value *Expr, AtomicOrdering AO,
     AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp,
-    bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr) {
+    bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr,
+    bool IsIgnoreDenormalMode, bool IsFineGrainedMemory, bool IsRemoteMemory) {
   if (!updateToLocation(Loc))
     return Loc.IP;
 
@@ -9197,9 +9632,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createAtomicCapture(
   // If UpdateExpr is 'x' updated with some `expr` not based on 'x',
   // 'x' is simply atomically rewritten with 'expr'.
   AtomicRMWInst::BinOp AtomicOp = (UpdateExpr ? RMWOp : AtomicRMWInst::Xchg);
-  Expected<std::pair<Value *, Value *>> AtomicResult =
-      emitAtomicUpdate(AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp,
-                       X.IsVolatile, IsXBinopExpr);
+  Expected<std::pair<Value *, Value *>> AtomicResult = emitAtomicUpdate(
+      AllocaIP, X.Var, X.ElemTy, Expr, AO, AtomicOp, UpdateOp, X.IsVolatile,
+      IsXBinopExpr, IsIgnoreDenormalMode, IsFineGrainedMemory, IsRemoteMemory);
   if (!AtomicResult)
     return AtomicResult.takeError();
   Value *CapturedVal =
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 71591074cea50..e200f3626e69d 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -1311,14 +1311,15 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
     }
     break;
   case 'l':
-    if (Name.starts_with("lifetime.start") ||
-        Name.starts_with("lifetime.end")) {
-      // Unless remangling is required, do not upgrade the function declaration,
-      // but do upgrade the calls.
-      if (auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F))
-        NewFn = *Result;
-      else
-        NewFn = F;
+    if ((Name.starts_with("lifetime.start") ||
+         Name.starts_with("lifetime.end")) &&
+        F->arg_size() == 2) {
+      Intrinsic::ID IID = Name.starts_with("lifetime.start")
+                              ? Intrinsic::lifetime_start
+                              : Intrinsic::lifetime_end;
+      rename(F);
+      NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID,
+                                                F->getArg(0)->getType());
       return true;
     }
     break;
@@ -5133,21 +5134,20 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
 
   case Intrinsic::lifetime_start:
   case Intrinsic::lifetime_end: {
-    Value *Size = CI->getArgOperand(0);
-    Value *Ptr = CI->getArgOperand(1);
-    if (isa<AllocaInst>(Ptr)) {
+    if (CI->arg_size() != 2) {
       DefaultCase();
       return;
     }
 
+    Value *Ptr = CI->getArgOperand(1);
     // Try to strip pointer casts, such that the lifetime works on an alloca.
     Ptr = Ptr->stripPointerCasts();
     if (isa<AllocaInst>(Ptr)) {
       // Don't use NewFn, as we might have looked through an addrspacecast.
       if (NewFn->getIntrinsicID() == Intrinsic::lifetime_start)
-        NewCall = Builder.CreateLifetimeStart(Ptr, cast<ConstantInt>(Size));
+        NewCall = Builder.CreateLifetimeStart(Ptr);
       else
-        NewCall = Builder.CreateLifetimeEnd(Ptr, cast<ConstantInt>(Size));
+        NewCall = Builder.CreateLifetimeEnd(Ptr);
       break;
     }
 
@@ -5391,7 +5391,7 @@ void llvm::UpgradeNVVMAnnotations(Module &M) {
     return;
 
   SmallVector<MDNode *, 8> NewNodes;
-  SmallSet<const MDNode *, 8> SeenNodes;
+  SmallPtrSet<const MDNode *, 8> SeenNodes;
   for (MDNode *MD : NamedMD->operands()) {
     if (!SeenNodes.insert(MD).second)
       continue;
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index d4ad21e69e848..6b202baf8ccef 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -254,6 +254,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
     return FoldBitCast(V, DestTy);
   case Instruction::AddrSpaceCast:
   case Instruction::IntToPtr:
+  case Instruction::PtrToAddr:
   case Instruction::PtrToInt:
     return nullptr;
   }
diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index e09c139db39c8..b454c9a4cd3ae 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -829,6 +829,7 @@ ConstantRange ConstantRange::castOp(Instruction::CastOps CastOp,
   case Instruction::FPTrunc:
   case Instruction::FPExt:
   case Instruction::IntToPtr:
+  case Instruction::PtrToAddr:
   case Instruction::PtrToInt:
   case Instruction::AddrSpaceCast:
     // Conservatively return getFull set.
@@ -871,7 +872,8 @@ ConstantRange ConstantRange::signExtend(uint32_t DstTySize) const {
   return ConstantRange(Lower.sext(DstTySize), Upper.sext(DstTySize));
 }
 
-ConstantRange ConstantRange::truncate(uint32_t DstTySize) const {
+ConstantRange ConstantRange::truncate(uint32_t DstTySize,
+                                      unsigned NoWrapKind) const {
   assert(getBitWidth() > DstTySize && "Not a value truncation");
   if (isEmptySet())
     return getEmpty(DstTySize);
@@ -885,22 +887,36 @@ ConstantRange ConstantRange::truncate(uint32_t DstTySize) const {
   // We use the non-wrapped set code to analyze the [Lower, MaxValue) part, and
   // then we do the union with [MaxValue, Upper)
   if (isUpperWrapped()) {
-    // If Upper is greater than or equal to MaxValue(DstTy), it covers the whole
-    // truncated range.
-    if (Upper.getActiveBits() > DstTySize || Upper.countr_one() == DstTySize)
+    // If Upper is greater than MaxValue(DstTy), it covers the whole truncated
+    // range.
+    if (Upper.getActiveBits() > DstTySize)
       return getFull(DstTySize);
 
-    Union = ConstantRange(APInt::getMaxValue(DstTySize),Upper.trunc(DstTySize));
-    UpperDiv.setAllBits();
-
-    // Union covers the MaxValue case, so return if the remaining range is just
-    // MaxValue(DstTy).
-    if (LowerDiv == UpperDiv)
-      return Union;
+    // For nuw the two parts are: [0, Upper) \/ [Lower, MaxValue(DstTy)]
+    if (NoWrapKind & TruncInst::NoUnsignedWrap) {
+      Union = ConstantRange(APInt::getZero(DstTySize), Upper.trunc(DstTySize));
+      UpperDiv = APInt::getOneBitSet(getBitWidth(), DstTySize);
+    } else {
+      // If Upper is equal to MaxValue(DstTy), it covers the whole truncated
+      // range.
+      if (Upper.countr_one() == DstTySize)
+        return getFull(DstTySize);
+      Union =
+          ConstantRange(APInt::getMaxValue(DstTySize), Upper.trunc(DstTySize));
+      UpperDiv.setAllBits();
+      // Union covers the MaxValue case, so return if the remaining range is
+      // just MaxValue(DstTy).
+      if (LowerDiv == UpperDiv)
+        return Union;
+    }
   }
 
   // Chop off the most significant bits that are past the destination bitwidth.
   if (LowerDiv.getActiveBits() > DstTySize) {
+    // For trunc nuw if LowerDiv is greater than MaxValue(DstTy), the range is
+    // outside the whole truncated range.
+    if (NoWrapKind & TruncInst::NoUnsignedWrap)
+      return Union;
     // Mask to just the signficant bits and subtract from LowerDiv/UpperDiv.
     APInt Adjust = LowerDiv & APInt::getBitsSetFrom(getBitWidth(), DstTySize);
     LowerDiv -= Adjust;
@@ -912,6 +928,10 @@ ConstantRange ConstantRange::truncate(uint32_t DstTySize) const {
     return ConstantRange(LowerDiv.trunc(DstTySize),
                          UpperDiv.trunc(DstTySize)).unionWith(Union);
 
+  if (!LowerDiv.isZero() && NoWrapKind & TruncInst::NoUnsignedWrap)
+    return ConstantRange(LowerDiv.trunc(DstTySize), APInt::getZero(DstTySize))
+        .unionWith(Union);
+
   // The truncated value wraps around. Check if we can do better than fullset.
   if (UpperDivWidth == DstTySize + 1) {
     // Clear the MSB so that UpperDiv wraps around.
diff --git a/llvm/lib/IR/Constants.cpp b/llvm/lib/IR/Constants.cpp
index a3c725b2af62a..c7e3113a54f22 100644
--- a/llvm/lib/IR/Constants.cpp
+++ b/llvm/lib/IR/Constants.cpp
@@ -1567,6 +1567,7 @@ Constant *ConstantExpr::getWithOperands(ArrayRef<Constant *> Ops, Type *Ty,
   case Instruction::SIToFP:
   case Instruction::FPToUI:
   case Instruction::FPToSI:
+  case Instruction::PtrToAddr:
   case Instruction::PtrToInt:
   case Instruction::IntToPtr:
   case Instruction::BitCast:
@@ -2223,6 +2224,8 @@ Constant *ConstantExpr::getCast(unsigned oc, Constant *C, Type *Ty,
     llvm_unreachable("Invalid cast opcode");
   case Instruction::Trunc:
     return getTrunc(C, Ty, OnlyIfReduced);
+  case Instruction::PtrToAddr:
+    return getPtrToAddr(C, Ty, OnlyIfReduced);
   case Instruction::PtrToInt:
     return getPtrToInt(C, Ty, OnlyIfReduced);
   case Instruction::IntToPtr:
@@ -2280,6 +2283,20 @@ Constant *ConstantExpr::getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced) {
   return getFoldedCast(Instruction::Trunc, C, Ty, OnlyIfReduced);
 }
 
+Constant *ConstantExpr::getPtrToAddr(Constant *C, Type *DstTy,
+                                     bool OnlyIfReduced) {
+  assert(C->getType()->isPtrOrPtrVectorTy() &&
+         "PtrToAddr source must be pointer or pointer vector");
+  assert(DstTy->isIntOrIntVectorTy() &&
+         "PtrToAddr destination must be integer or integer vector");
+  assert(isa<VectorType>(C->getType()) == isa<VectorType>(DstTy));
+  if (isa<VectorType>(C->getType()))
+    assert(cast<VectorType>(C->getType())->getElementCount() ==
+               cast<VectorType>(DstTy)->getElementCount() &&
+           "Invalid cast between a different number of vector elements");
+  return getFoldedCast(Instruction::PtrToAddr, C, DstTy, OnlyIfReduced);
+}
+
 Constant *ConstantExpr::getPtrToInt(Constant *C, Type *DstTy,
                                     bool OnlyIfReduced) {
   assert(C->getType()->isPtrOrPtrVectorTy() &&
@@ -2435,6 +2452,7 @@ bool ConstantExpr::isDesirableCastOp(unsigned Opcode) {
   case Instruction::FPToSI:
     return false;
   case Instruction::Trunc:
+  case Instruction::PtrToAddr:
   case Instruction::PtrToInt:
   case Instruction::IntToPtr:
   case Instruction::BitCast:
@@ -2457,6 +2475,7 @@ bool ConstantExpr::isSupportedCastOp(unsigned Opcode) {
   case Instruction::FPToSI:
     return false;
   case Instruction::Trunc:
+  case Instruction::PtrToAddr:
   case Instruction::PtrToInt:
   case Instruction::IntToPtr:
   case Instruction::BitCast:
@@ -3401,6 +3420,7 @@ Instruction *ConstantExpr::getAsInstruction() const {
 
   switch (getOpcode()) {
   case Instruction::Trunc:
+  case Instruction::PtrToAddr:
   case Instruction::PtrToInt:
   case Instruction::IntToPtr:
   case Instruction::BitCast:
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index f7ef4aa473ef5..8b5965ba45a6d 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -2186,6 +2186,11 @@ void LLVMGlobalSetMetadata(LLVMValueRef Global, unsigned Kind,
   unwrap<GlobalObject>(Global)->setMetadata(Kind, unwrap<MDNode>(MD));
 }
 
+void LLVMGlobalAddMetadata(LLVMValueRef Global, unsigned Kind,
+                           LLVMMetadataRef MD) {
+  unwrap<GlobalObject>(Global)->addMetadata(Kind, *unwrap<MDNode>(MD));
+}
+
 void LLVMGlobalEraseMetadata(LLVMValueRef Global, unsigned Kind) {
   unwrap<GlobalObject>(Global)->eraseMetadata(Kind);
 }
@@ -2194,6 +2199,11 @@ void LLVMGlobalClearMetadata(LLVMValueRef Global) {
   unwrap<GlobalObject>(Global)->clearMetadata();
 }
 
+void LLVMGlobalAddDebugInfo(LLVMValueRef Global, LLVMMetadataRef GVE) {
+  unwrap<GlobalVariable>(Global)->addDebugInfo(
+      unwrap<DIGlobalVariableExpression>(GVE));
+}
+
 /*--.. Operations on global variables ......................................--*/
 
 LLVMValueRef LLVMAddGlobal(LLVMModuleRef M, LLVMTypeRef Ty, const char *Name) {
diff --git a/llvm/lib/IR/DebugInfo.cpp b/llvm/lib/IR/DebugInfo.cpp
index ab8ecee2a81e0..8e523bcf7960e 100644
--- a/llvm/lib/IR/DebugInfo.cpp
+++ b/llvm/lib/IR/DebugInfo.cpp
@@ -1896,29 +1896,8 @@ AssignmentInstRange at::getAssignmentInsts(DIAssignID *ID) {
   return make_range(MapIt->second.begin(), MapIt->second.end());
 }
 
-AssignmentMarkerRange at::getAssignmentMarkers(DIAssignID *ID) {
-  assert(ID && "Expected non-null ID");
-  LLVMContext &Ctx = ID->getContext();
-
-  auto *IDAsValue = MetadataAsValue::getIfExists(Ctx, ID);
-
-  // The ID is only used wrapped in MetadataAsValue(ID), so lets check that
-  // one of those already exists first.
-  if (!IDAsValue)
-    return make_range(Value::user_iterator(), Value::user_iterator());
-
-  return make_range(IDAsValue->user_begin(), IDAsValue->user_end());
-}
-
 void at::deleteAssignmentMarkers(const Instruction *Inst) {
-  auto Range = getAssignmentMarkers(Inst);
-  SmallVector<DbgVariableRecord *> DVRAssigns = getDVRAssignmentMarkers(Inst);
-  if (Range.empty() && DVRAssigns.empty())
-    return;
-  SmallVector<DbgAssignIntrinsic *> ToDelete(Range.begin(), Range.end());
-  for (auto *DAI : ToDelete)
-    DAI->eraseFromParent();
-  for (auto *DVR : DVRAssigns)
+  for (auto *DVR : getDVRAssignmentMarkers(Inst))
     DVR->eraseFromParent();
 }
 
@@ -1936,31 +1915,21 @@ void at::RAUW(DIAssignID *Old, DIAssignID *New) {
 }
 
 void at::deleteAll(Function *F) {
-  SmallVector<DbgAssignIntrinsic *, 12> ToDelete;
-  SmallVector<DbgVariableRecord *, 12> DPToDelete;
   for (BasicBlock &BB : *F) {
     for (Instruction &I : BB) {
-      for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
+      for (DbgVariableRecord &DVR :
+           make_early_inc_range(filterDbgVars(I.getDbgRecordRange())))
         if (DVR.isDbgAssign())
-          DPToDelete.push_back(&DVR);
-      if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(&I))
-        ToDelete.push_back(DAI);
-      else
-        I.setMetadata(LLVMContext::MD_DIAssignID, nullptr);
+          DVR.eraseFromParent();
+
+      I.setMetadata(LLVMContext::MD_DIAssignID, nullptr);
     }
   }
-  for (auto *DAI : ToDelete)
-    DAI->eraseFromParent();
-  for (auto *DVR : DPToDelete)
-    DVR->eraseFromParent();
 }
 
-/// FIXME: Remove this wrapper function and call
-/// DIExpression::calculateFragmentIntersect directly.
-template <typename T>
-bool calculateFragmentIntersectImpl(
+bool at::calculateFragmentIntersect(
     const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits,
-    uint64_t SliceSizeInBits, const T *AssignRecord,
+    uint64_t SliceSizeInBits, const DbgVariableRecord *AssignRecord,
     std::optional<DIExpression::FragmentInfo> &Result) {
   // No overlap if this DbgRecord describes a killed location.
   if (AssignRecord->isKillAddress())
@@ -1989,26 +1958,6 @@ bool calculateFragmentIntersectImpl(
       BitExtractOffsetInBits, VarFrag, Result, OffsetFromLocationInBits);
 }
 
-/// FIXME: Remove this wrapper function and call
-/// DIExpression::calculateFragmentIntersect directly.
-bool at::calculateFragmentIntersect(
-    const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits,
-    uint64_t SliceSizeInBits, const DbgAssignIntrinsic *DbgAssign,
-    std::optional<DIExpression::FragmentInfo> &Result) {
-  return calculateFragmentIntersectImpl(DL, Dest, SliceOffsetInBits,
-                                        SliceSizeInBits, DbgAssign, Result);
-}
-
-/// FIXME: Remove this wrapper function and call
-/// DIExpression::calculateFragmentIntersect directly.
-bool at::calculateFragmentIntersect(
-    const DataLayout &DL, const Value *Dest, uint64_t SliceOffsetInBits,
-    uint64_t SliceSizeInBits, const DbgVariableRecord *DVRAssign,
-    std::optional<DIExpression::FragmentInfo> &Result) {
-  return calculateFragmentIntersectImpl(DL, Dest, SliceOffsetInBits,
-                                        SliceSizeInBits, DVRAssign, Result);
-}
-
 /// Update inlined instructions' DIAssignID metadata. We need to do this
 /// otherwise a function inlined more than once into the same function
 /// will cause DIAssignID to be shared by many instructions.
@@ -2029,8 +1978,6 @@ void at::remapAssignID(DenseMap<DIAssignID *, DIAssignID *> &Map,
   }
   if (auto *ID = I.getMetadata(LLVMContext::MD_DIAssignID))
     I.setMetadata(LLVMContext::MD_DIAssignID, GetNewID(ID));
-  else if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(&I))
-    DAI->setAssignId(GetNewID(DAI->getAssignID()));
 }
 
 /// Collect constant properies (base, size, offset) of \p StoreDest.
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index f1d4549ba015b..96065edca9b51 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -57,15 +57,9 @@ DebugVariable::DebugVariable(const DbgVariableRecord *DVR)
 DILocation::DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
                        unsigned Column, uint64_t AtomGroup, uint8_t AtomRank,
                        ArrayRef<Metadata *> MDs, bool ImplicitCode)
-    : MDNode(C, DILocationKind, Storage, MDs)
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
-      ,
-      AtomGroup(AtomGroup), AtomRank(AtomRank)
-#endif
-{
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
+    : MDNode(C, DILocationKind, Storage, MDs), AtomGroup(AtomGroup),
+      AtomRank(AtomRank) {
   assert(AtomRank <= 7 && "AtomRank number should fit in 3 bits");
-#endif
   if (AtomGroup)
     C.updateDILocationAtomGroupWaterline(AtomGroup + 1);
 
diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp
index 7b799c70a3318..11d33e262fecb 100644
--- a/llvm/lib/IR/Globals.cpp
+++ b/llvm/lib/IR/Globals.cpp
@@ -404,6 +404,7 @@ findBaseObject(const Constant *C, DenseSet<const GlobalAlias *> &Aliases,
       return findBaseObject(CE->getOperand(0), Aliases, Op);
     }
     case Instruction::IntToPtr:
+    case Instruction::PtrToAddr:
     case Instruction::PtrToInt:
     case Instruction::BitCast:
     case Instruction::GetElementPtr:
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 49c6dc7f401ad..614c3a9abb8d0 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -411,28 +411,16 @@ CallInst *IRBuilderBase::CreateFPMinimumReduce(Value *Src) {
   return getReductionIntrinsic(Intrinsic::vector_reduce_fminimum, Src);
 }
 
-CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
+CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr) {
   assert(isa<PointerType>(Ptr->getType()) &&
          "lifetime.start only applies to pointers.");
-  if (!Size)
-    Size = getInt64(-1);
-  else
-    assert(Size->getType() == getInt64Ty() &&
-           "lifetime.start requires the size to be an i64");
-  Value *Ops[] = { Size, Ptr };
-  return CreateIntrinsic(Intrinsic::lifetime_start, {Ptr->getType()}, Ops);
+  return CreateIntrinsic(Intrinsic::lifetime_start, {Ptr->getType()}, {Ptr});
 }
 
-CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
+CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr) {
   assert(isa<PointerType>(Ptr->getType()) &&
          "lifetime.end only applies to pointers.");
-  if (!Size)
-    Size = getInt64(-1);
-  else
-    assert(Size->getType() == getInt64Ty() &&
-           "lifetime.end requires the size to be an i64");
-  Value *Ops[] = { Size, Ptr };
-  return CreateIntrinsic(Intrinsic::lifetime_end, {Ptr->getType()}, Ops);
+  return CreateIntrinsic(Intrinsic::lifetime_end, {Ptr->getType()}, {Ptr});
 }
 
 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index b7cd12ac9691e..5e87b5ff941ad 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -26,9 +26,18 @@
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/ProfDataUtils.h"
 #include "llvm/IR/Type.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 using namespace llvm;
 
+// FIXME: Flag used for an ablation performance test, Issue #147390. Placing it
+// here because referencing IR should be feasible from anywhere. Will be
+// removed after the ablation test.
+cl::opt<bool> ProfcheckDisableMetadataFixes(
+    "profcheck-disable-metadata-fixes", cl::Hidden, cl::init(false),
+    cl::desc(
+        "Disable metadata propagation fixes discovered through Issue #147390"));
+
 InsertPosition::InsertPosition(Instruction *InsertBefore)
     : InsertAt(InsertBefore ? InsertBefore->getIterator()
                             : InstListType::iterator()) {}
@@ -543,14 +552,19 @@ void Instruction::dropUBImplyingAttrsAndUnknownMetadata(
   CB->removeRetAttrs(UBImplyingAttributes);
 }
 
-void Instruction::dropUBImplyingAttrsAndMetadata() {
+void Instruction::dropUBImplyingAttrsAndMetadata(ArrayRef<unsigned> Keep) {
   // !annotation metadata does not impact semantics.
   // !range, !nonnull and !align produce poison, so they are safe to speculate.
   // !noundef and various AA metadata must be dropped, as it generally produces
   // immediate undefined behavior.
-  unsigned KnownIDs[] = {LLVMContext::MD_annotation, LLVMContext::MD_range,
-                         LLVMContext::MD_nonnull, LLVMContext::MD_align};
-  dropUBImplyingAttrsAndUnknownMetadata(KnownIDs);
+  static const unsigned KnownIDs[] = {
+      LLVMContext::MD_annotation, LLVMContext::MD_range,
+      LLVMContext::MD_nonnull, LLVMContext::MD_align};
+  SmallVector<unsigned> KeepIDs;
+  KeepIDs.reserve(Keep.size() + std::size(KnownIDs));
+  append_range(KeepIDs, KnownIDs);
+  append_range(KeepIDs, Keep);
+  dropUBImplyingAttrsAndUnknownMetadata(KeepIDs);
 }
 
 bool Instruction::hasUBImplyingAttrs() const {
@@ -817,6 +831,7 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
   case UIToFP:        return "uitofp";
   case SIToFP:        return "sitofp";
   case IntToPtr:      return "inttoptr";
+  case PtrToAddr:     return "ptrtoaddr";
   case PtrToInt:      return "ptrtoint";
   case BitCast:       return "bitcast";
   case AddrSpaceCast: return "addrspacecast";
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index b8963823f1c67..a1751c0ee3e48 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -2798,6 +2798,7 @@ bool CastInst::isNoopCast(Instruction::CastOps Opcode,
       return false;
     case Instruction::BitCast:
       return true;  // BitCast never modifies bits.
+    case Instruction::PtrToAddr:
     case Instruction::PtrToInt:
       return DL.getIntPtrType(SrcTy)->getScalarSizeInBits() ==
              DestTy->getScalarSizeInBits();
@@ -2855,26 +2856,29 @@ unsigned CastInst::isEliminableCastPair(
   // same reason.
   const unsigned numCastOps =
     Instruction::CastOpsEnd - Instruction::CastOpsBegin;
+  // clang-format off
   static const uint8_t CastResults[numCastOps][numCastOps] = {
-    // T        F  F  U  S  F  F  P  I  B  A  -+
-    // R  Z  S  P  P  I  I  T  P  2  N  T  S   |
-    // U  E  E  2  2  2  2  R  E  I  T  C  C   +- secondOp
-    // N  X  X  U  S  F  F  N  X  N  2  V  V   |
-    // C  T  T  I  I  P  P  C  T  T  P  T  T  -+
-    {  1, 0, 0,99,99, 0, 0,99,99,99, 0, 3, 0}, // Trunc         -+
-    {  8, 1, 9,99,99, 2,17,99,99,99, 2, 3, 0}, // ZExt           |
-    {  8, 0, 1,99,99, 0, 2,99,99,99, 0, 3, 0}, // SExt           |
-    {  0, 0, 0,99,99, 0, 0,99,99,99, 0, 3, 0}, // FPToUI         |
-    {  0, 0, 0,99,99, 0, 0,99,99,99, 0, 3, 0}, // FPToSI         |
-    { 99,99,99, 0, 0,99,99, 0, 0,99,99, 4, 0}, // UIToFP         +- firstOp
-    { 99,99,99, 0, 0,99,99, 0, 0,99,99, 4, 0}, // SIToFP         |
-    { 99,99,99, 0, 0,99,99, 0, 0,99,99, 4, 0}, // FPTrunc        |
-    { 99,99,99, 2, 2,99,99, 8, 2,99,99, 4, 0}, // FPExt          |
-    {  1, 0, 0,99,99, 0, 0,99,99,99, 7, 3, 0}, // PtrToInt       |
-    { 99,99,99,99,99,99,99,99,99,11,99,15, 0}, // IntToPtr       |
-    {  5, 5, 5, 0, 0, 5, 5, 0, 0,16, 5, 1,14}, // BitCast        |
-    {  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,13,12}, // AddrSpaceCast -+
+    // T        F  F  U  S  F  F  P  P  I  B  A  -+
+    // R  Z  S  P  P  I  I  T  P  2  2  N  T  S   |
+    // U  E  E  2  2  2  2  R  E  I  A  T  C  C   +- secondOp
+    // N  X  X  U  S  F  F  N  X  N  D  2  V  V   |
+    // C  T  T  I  I  P  P  C  T  T  R  P  T  T  -+
+    {  1, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // Trunc         -+
+    {  8, 1, 9,99,99, 2,17,99,99,99,99, 2, 3, 0}, // ZExt           |
+    {  8, 0, 1,99,99, 0, 2,99,99,99,99, 0, 3, 0}, // SExt           |
+    {  0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // FPToUI         |
+    {  0, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // FPToSI         |
+    { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // UIToFP         +- firstOp
+    { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // SIToFP         |
+    { 99,99,99, 0, 0,99,99, 0, 0,99,99,99, 4, 0}, // FPTrunc        |
+    { 99,99,99, 2, 2,99,99, 8, 2,99,99,99, 4, 0}, // FPExt          |
+    {  1, 0, 0,99,99, 0, 0,99,99,99,99, 7, 3, 0}, // PtrToInt       |
+    {  1, 0, 0,99,99, 0, 0,99,99,99,99, 0, 3, 0}, // PtrToAddr      |
+    { 99,99,99,99,99,99,99,99,99,11,99,99,15, 0}, // IntToPtr       |
+    {  5, 5, 5, 0, 0, 5, 5, 0, 0,16,16, 5, 1,14}, // BitCast        |
+    {  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,13,12}, // AddrSpaceCast -+
   };
+  // clang-format on
 
   // TODO: This logic could be encoded into the table above and handled in the
   // switch below.
@@ -3046,6 +3050,7 @@ CastInst *CastInst::Create(Instruction::CastOps op, Value *S, Type *Ty,
   case SIToFP:        return new SIToFPInst        (S, Ty, Name, InsertBefore);
   case FPToUI:        return new FPToUIInst        (S, Ty, Name, InsertBefore);
   case FPToSI:        return new FPToSIInst        (S, Ty, Name, InsertBefore);
+  case PtrToAddr:     return new PtrToAddrInst     (S, Ty, Name, InsertBefore);
   case PtrToInt:      return new PtrToIntInst      (S, Ty, Name, InsertBefore);
   case IntToPtr:      return new IntToPtrInst      (S, Ty, Name, InsertBefore);
   case BitCast:
@@ -3347,6 +3352,7 @@ CastInst::castIsValid(Instruction::CastOps op, Type *SrcTy, Type *DstTy) {
   case Instruction::FPToSI:
     return SrcTy->isFPOrFPVectorTy() && DstTy->isIntOrIntVectorTy() &&
            SrcEC == DstEC;
+  case Instruction::PtrToAddr:
   case Instruction::PtrToInt:
     if (SrcEC != DstEC)
       return false;
@@ -3460,6 +3466,12 @@ PtrToIntInst::PtrToIntInst(Value *S, Type *Ty, const Twine &Name,
   assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToInt");
 }
 
+PtrToAddrInst::PtrToAddrInst(Value *S, Type *Ty, const Twine &Name,
+                             InsertPosition InsertBefore)
+    : CastInst(Ty, PtrToAddr, S, Name, InsertBefore) {
+  assert(castIsValid(getOpcode(), S, Ty) && "Illegal PtrToAddr");
+}
+
 IntToPtrInst::IntToPtrInst(Value *S, Type *Ty, const Twine &Name,
                            InsertPosition InsertBefore)
     : CastInst(Ty, IntToPtr, S, Name, InsertBefore) {
@@ -4427,6 +4439,10 @@ PtrToIntInst *PtrToIntInst::cloneImpl() const {
   return new PtrToIntInst(getOperand(0), getType());
 }
 
+PtrToAddrInst *PtrToAddrInst::cloneImpl() const {
+  return new PtrToAddrInst(getOperand(0), getType());
+}
+
 IntToPtrInst *IntToPtrInst::cloneImpl() const {
   return new IntToPtrInst(getOperand(0), getType());
 }
diff --git a/llvm/lib/IR/LLVMContextImpl.h b/llvm/lib/IR/LLVMContextImpl.h
index aa2a60e5038a7..e03f993297e54 100644
--- a/llvm/lib/IR/LLVMContextImpl.h
+++ b/llvm/lib/IR/LLVMContextImpl.h
@@ -312,10 +312,8 @@ template <> struct MDNodeKeyImpl<MDTuple> : MDNodeOpsKey {
 template <> struct MDNodeKeyImpl<DILocation> {
   Metadata *Scope;
   Metadata *InlinedAt;
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
   uint64_t AtomGroup : 61;
   uint64_t AtomRank : 3;
-#endif
   unsigned Line;
   uint16_t Column;
   bool ImplicitCode;
@@ -323,36 +321,24 @@ template <> struct MDNodeKeyImpl<DILocation> {
   MDNodeKeyImpl(unsigned Line, uint16_t Column, Metadata *Scope,
                 Metadata *InlinedAt, bool ImplicitCode, uint64_t AtomGroup,
                 uint8_t AtomRank)
-      : Scope(Scope), InlinedAt(InlinedAt),
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
-        AtomGroup(AtomGroup), AtomRank(AtomRank),
-#endif
-        Line(Line), Column(Column), ImplicitCode(ImplicitCode) {
-  }
+      : Scope(Scope), InlinedAt(InlinedAt), AtomGroup(AtomGroup),
+        AtomRank(AtomRank), Line(Line), Column(Column),
+        ImplicitCode(ImplicitCode) {}
 
   MDNodeKeyImpl(const DILocation *L)
       : Scope(L->getRawScope()), InlinedAt(L->getRawInlinedAt()),
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
         AtomGroup(L->getAtomGroup()), AtomRank(L->getAtomRank()),
-#endif
         Line(L->getLine()), Column(L->getColumn()),
-        ImplicitCode(L->isImplicitCode()) {
-  }
+        ImplicitCode(L->isImplicitCode()) {}
 
   bool isKeyOf(const DILocation *RHS) const {
     return Line == RHS->getLine() && Column == RHS->getColumn() &&
            Scope == RHS->getRawScope() && InlinedAt == RHS->getRawInlinedAt() &&
-           ImplicitCode == RHS->isImplicitCode()
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
-           && AtomGroup == RHS->getAtomGroup() &&
-           AtomRank == RHS->getAtomRank();
-#else
-        ;
-#endif
+           ImplicitCode == RHS->isImplicitCode() &&
+           AtomGroup == RHS->getAtomGroup() && AtomRank == RHS->getAtomRank();
   }
 
   unsigned getHashValue() const {
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
     // Hashing AtomGroup and AtomRank substantially impacts performance whether
     // Key Instructions is enabled or not. We can't detect whether it's enabled
     // here cheaply; avoiding hashing zero values is a good approximation. This
@@ -363,7 +349,6 @@ template <> struct MDNodeKeyImpl<DILocation> {
     if (AtomGroup || AtomRank)
       return hash_combine(Line, Column, Scope, InlinedAt, ImplicitCode,
                           AtomGroup, (uint8_t)AtomRank);
-#endif
     return hash_combine(Line, Column, Scope, InlinedAt, ImplicitCode);
   }
 };
diff --git a/llvm/lib/IR/ProfDataUtils.cpp b/llvm/lib/IR/ProfDataUtils.cpp
index b1b5f67689e6d..d24263f8b3bda 100644
--- a/llvm/lib/IR/ProfDataUtils.cpp
+++ b/llvm/lib/IR/ProfDataUtils.cpp
@@ -270,6 +270,18 @@ void setBranchWeights(Instruction &I, ArrayRef<uint32_t> Weights,
   I.setMetadata(LLVMContext::MD_prof, BranchWeights);
 }
 
+SmallVector<uint32_t> downscaleWeights(ArrayRef<uint64_t> Weights,
+                                       std::optional<uint64_t> KnownMaxCount) {
+  uint64_t MaxCount = KnownMaxCount.has_value() ? KnownMaxCount.value()
+                                                : *llvm::max_element(Weights);
+  assert(MaxCount > 0 && "Bad max count");
+  uint64_t Scale = calculateCountScale(MaxCount);
+  SmallVector<uint32_t> DownscaledWeights;
+  for (const auto &ECI : Weights)
+    DownscaledWeights.push_back(scaleBranchCount(ECI, Scale));
+  return DownscaledWeights;
+}
+
 void scaleProfData(Instruction &I, uint64_t S, uint64_t T) {
   assert(T != 0 && "Caller should guarantee");
   auto *ProfileData = I.getMetadata(LLVMContext::MD_prof);
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index a8e6c7938cf54..3c324f2fe0d84 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -9,6 +9,8 @@
 #include "llvm/IR/RuntimeLibcalls.h"
 #include "llvm/ADT/StringTable.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/xxhash.h"
+#include "llvm/TargetParser/ARMTargetParser.h"
 
 #define DEBUG_TYPE "runtime-libcalls-info"
 
@@ -17,51 +19,11 @@ using namespace RTLIB;
 
 #define GET_INIT_RUNTIME_LIBCALL_NAMES
 #define GET_SET_TARGET_RUNTIME_LIBCALL_SETS
+#define DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME
 #include "llvm/IR/RuntimeLibcalls.inc"
 #undef GET_INIT_RUNTIME_LIBCALL_NAMES
 #undef GET_SET_TARGET_RUNTIME_LIBCALL_SETS
-
-static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
-                               FloatABI::ABIType FloatABIType,
-                               EABI EABIVersion) {
-  static const RTLIB::LibcallImpl AAPCS_Libcalls[] = {
-      RTLIB::__aeabi_dadd,        RTLIB::__aeabi_ddiv,
-      RTLIB::__aeabi_dmul,        RTLIB::__aeabi_dsub,
-      RTLIB::__aeabi_dcmpeq__oeq, RTLIB::__aeabi_dcmpeq__une,
-      RTLIB::__aeabi_dcmplt,      RTLIB::__aeabi_dcmple,
-      RTLIB::__aeabi_dcmpge,      RTLIB::__aeabi_dcmpgt,
-      RTLIB::__aeabi_dcmpun,      RTLIB::__aeabi_fadd,
-      RTLIB::__aeabi_fdiv,        RTLIB::__aeabi_fmul,
-      RTLIB::__aeabi_fsub,        RTLIB::__aeabi_fcmpeq__oeq,
-      RTLIB::__aeabi_fcmpeq__une, RTLIB::__aeabi_fcmplt,
-      RTLIB::__aeabi_fcmple,      RTLIB::__aeabi_fcmpge,
-      RTLIB::__aeabi_fcmpgt,      RTLIB::__aeabi_fcmpun,
-      RTLIB::__aeabi_d2iz,        RTLIB::__aeabi_d2uiz,
-      RTLIB::__aeabi_d2lz,        RTLIB::__aeabi_d2ulz,
-      RTLIB::__aeabi_f2iz,        RTLIB::__aeabi_f2uiz,
-      RTLIB::__aeabi_f2lz,        RTLIB::__aeabi_f2ulz,
-      RTLIB::__aeabi_d2f,         RTLIB::__aeabi_d2h,
-      RTLIB::__aeabi_f2d,         RTLIB::__aeabi_i2d,
-      RTLIB::__aeabi_ui2d,        RTLIB::__aeabi_l2d,
-      RTLIB::__aeabi_ul2d,        RTLIB::__aeabi_i2f,
-      RTLIB::__aeabi_ui2f,        RTLIB::__aeabi_l2f,
-      RTLIB::__aeabi_ul2f,        RTLIB::__aeabi_lmul,
-      RTLIB::__aeabi_llsl,        RTLIB::__aeabi_llsr,
-      RTLIB::__aeabi_lasr,        RTLIB::__aeabi_idiv,
-      RTLIB::__aeabi_idivmod,     RTLIB::__aeabi_uidivmod,
-      RTLIB::__aeabi_ldivmod,     RTLIB::__aeabi_uidiv,
-      RTLIB::__aeabi_uldivmod,    RTLIB::__aeabi_f2h,
-      RTLIB::__aeabi_d2h,         RTLIB::__aeabi_h2f,
-      RTLIB::__aeabi_memcpy,      RTLIB::__aeabi_memmove,
-      RTLIB::__aeabi_memset,      RTLIB::__aeabi_memcpy4,
-      RTLIB::__aeabi_memcpy8,     RTLIB::__aeabi_memmove4,
-      RTLIB::__aeabi_memmove8,    RTLIB::__aeabi_memset4,
-      RTLIB::__aeabi_memset8,     RTLIB::__aeabi_memclr,
-      RTLIB::__aeabi_memclr4,     RTLIB::__aeabi_memclr8};
-
-  for (RTLIB::LibcallImpl Impl : AAPCS_Libcalls)
-    Info.setLibcallImplCallingConv(Impl, CallingConv::ARM_AAPCS);
-}
+#undef DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME
 
 /// Set default libcall names. If a target wants to opt-out of a libcall it
 /// should be placed here.
@@ -69,65 +31,51 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
                                        ExceptionHandling ExceptionModel,
                                        FloatABI::ABIType FloatABI,
                                        EABI EABIVersion, StringRef ABIName) {
-  setTargetRuntimeLibcallSets(TT, FloatABI);
-
-  if (ExceptionModel == ExceptionHandling::SjLj)
-    setLibcallImpl(RTLIB::UNWIND_RESUME, RTLIB::_Unwind_SjLj_Resume);
+  setTargetRuntimeLibcallSets(TT, ExceptionModel, FloatABI, EABIVersion,
+                              ABIName);
 
   if (TT.isARM() || TT.isThumb()) {
-    setARMLibcallNames(*this, TT, FloatABI, EABIVersion);
-    return;
-  }
+    // The half <-> float conversion functions are always soft-float on
+    // non-watchos platforms, but are needed for some targets which use a
+    // hard-float calling convention by default.
+    if (!TT.isWatchABI()) {
+      if (isAAPCS_ABI(TT, ABIName)) {
+        setLibcallImplCallingConv(RTLIB::__truncsfhf2, CallingConv::ARM_AAPCS);
+        setLibcallImplCallingConv(RTLIB::__truncdfhf2, CallingConv::ARM_AAPCS);
+        setLibcallImplCallingConv(RTLIB::__extendhfsf2, CallingConv::ARM_AAPCS);
+      } else {
+        setLibcallImplCallingConv(RTLIB::__truncsfhf2, CallingConv::ARM_APCS);
+        setLibcallImplCallingConv(RTLIB::__truncdfhf2, CallingConv::ARM_APCS);
+        setLibcallImplCallingConv(RTLIB::__extendhfsf2, CallingConv::ARM_APCS);
+      }
+    }
 
-  if (TT.getArch() == Triple::ArchType::msp430) {
-    setLibcallImplCallingConv(RTLIB::__mspabi_mpyll,
-                              CallingConv::MSP430_BUILTIN);
+    return;
   }
 }
 
-RTLIB::LibcallImpl
-RuntimeLibcallsInfo::getSupportedLibcallImpl(StringRef FuncName) const {
-  const ArrayRef<uint16_t> RuntimeLibcallNameOffsets(
-      RuntimeLibcallNameOffsetTable);
-
-  iterator_range<ArrayRef<uint16_t>::const_iterator> Range =
-      getRecognizedLibcallImpls(FuncName);
-
-  for (auto I = Range.begin(); I != Range.end(); ++I) {
-    RTLIB::LibcallImpl Impl =
-        static_cast<RTLIB::LibcallImpl>(I - RuntimeLibcallNameOffsets.begin());
-
-    // FIXME: This should not depend on looking up ImplToLibcall, only the list
-    // of libcalls for the module.
-    RTLIB::LibcallImpl Recognized = LibcallImpls[ImplToLibcall[Impl]];
-    if (Recognized != RTLIB::Unsupported)
-      return Recognized;
+LLVM_ATTRIBUTE_ALWAYS_INLINE
+iota_range<RTLIB::LibcallImpl>
+RuntimeLibcallsInfo::libcallImplNameHit(uint16_t NameOffsetEntry,
+                                        uint16_t StrOffset) {
+  int NumAliases = 1;
+  for (uint16_t Entry : ArrayRef(RuntimeLibcallNameOffsetTable)
+                            .drop_front(NameOffsetEntry + 1)) {
+    if (Entry != StrOffset)
+      break;
+    ++NumAliases;
   }
 
-  return RTLIB::Unsupported;
+  RTLIB::LibcallImpl ImplStart = static_cast<RTLIB::LibcallImpl>(
+      &RuntimeLibcallNameOffsetTable[NameOffsetEntry] -
+      &RuntimeLibcallNameOffsetTable[0]);
+  return enum_seq(ImplStart,
+                  static_cast<RTLIB::LibcallImpl>(ImplStart + NumAliases));
 }
 
-iterator_range<ArrayRef<uint16_t>::const_iterator>
-RuntimeLibcallsInfo::getRecognizedLibcallImpls(StringRef FuncName) {
-  StringTable::Iterator It = lower_bound(RuntimeLibcallImplNameTable, FuncName);
-  if (It == RuntimeLibcallImplNameTable.end() || *It != FuncName)
-    return iterator_range(ArrayRef<uint16_t>());
-
-  uint16_t IndexVal = It.offset().value();
-  const ArrayRef<uint16_t> TableRef(RuntimeLibcallNameOffsetTable);
-
-  ArrayRef<uint16_t>::const_iterator E = TableRef.end();
-  ArrayRef<uint16_t>::const_iterator EntriesBegin =
-      std::lower_bound(TableRef.begin(), E, IndexVal);
-  ArrayRef<uint16_t>::const_iterator EntriesEnd = EntriesBegin;
-
-  while (EntriesEnd != E && *EntriesEnd == IndexVal)
-    ++EntriesEnd;
-
-  assert(EntriesBegin != E &&
-         "libcall found in name table but not offset table");
-
-  return make_range(EntriesBegin, EntriesEnd);
+bool RuntimeLibcallsInfo::isAAPCS_ABI(const Triple &TT, StringRef ABIName) {
+  const ARM::ARMABI TargetABI = ARM::computeTargetABI(TT, ABIName);
+  return TargetABI == ARM::ARM_ABI_AAPCS || TargetABI == ARM::ARM_ABI_AAPCS16;
 }
 
 bool RuntimeLibcallsInfo::darwinHasExp10(const Triple &TT) {
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index 129ca4a57de5a..5928c89029b87 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -747,34 +747,28 @@ const Value *Value::stripAndAccumulateConstantOffsets(
       // means when we construct GEPOffset, we need to use the size
       // of GEP's pointer type rather than the size of the original
       // pointer type.
-      unsigned CurBitWidth = DL.getIndexTypeSizeInBits(V->getType());
-      if (CurBitWidth == BitWidth) {
-        if (!GEP->accumulateConstantOffset(DL, Offset, ExternalAnalysis))
-          return V;
-      } else {
-        APInt GEPOffset(CurBitWidth, 0);
-        if (!GEP->accumulateConstantOffset(DL, GEPOffset, ExternalAnalysis))
-          return V;
+      APInt GEPOffset(DL.getIndexTypeSizeInBits(V->getType()), 0);
+      if (!GEP->accumulateConstantOffset(DL, GEPOffset, ExternalAnalysis))
+        return V;
 
-        // Stop traversal if the pointer offset wouldn't fit in the bit-width
-        // provided by the Offset argument. This can happen due to AddrSpaceCast
-        // stripping.
-        if (GEPOffset.getSignificantBits() > BitWidth)
-          return V;
+      // Stop traversal if the pointer offset wouldn't fit in the bit-width
+      // provided by the Offset argument. This can happen due to AddrSpaceCast
+      // stripping.
+      if (GEPOffset.getSignificantBits() > BitWidth)
+        return V;
 
-        // External Analysis can return a result higher/lower than the value
-        // represents. We need to detect overflow/underflow.
-        APInt GEPOffsetST = GEPOffset.sextOrTrunc(BitWidth);
-        if (!ExternalAnalysis) {
-          Offset += GEPOffsetST;
-        } else {
-          bool Overflow = false;
-          APInt OldOffset = Offset;
-          Offset = Offset.sadd_ov(GEPOffsetST, Overflow);
-          if (Overflow) {
-            Offset = OldOffset;
-            return V;
-          }
+      // External Analysis can return a result higher/lower than the value
+      // represents. We need to detect overflow/underflow.
+      APInt GEPOffsetST = GEPOffset.sextOrTrunc(BitWidth);
+      if (!ExternalAnalysis) {
+        Offset += GEPOffsetST;
+      } else {
+        bool Overflow = false;
+        APInt OldOffset = Offset;
+        Offset = Offset.sadd_ov(GEPOffsetST, Overflow);
+        if (Overflow) {
+          Offset = OldOffset;
+          return V;
         }
       }
       V = GEP->getPointerOperand();
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index ca3f148f881a4..4eb4b58d022e8 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -55,7 +55,6 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
@@ -566,6 +565,8 @@ class Verifier : public InstVisitor<Verifier>, VerifierSupport {
   void visitUIToFPInst(UIToFPInst &I);
   void visitSIToFPInst(SIToFPInst &I);
   void visitIntToPtrInst(IntToPtrInst &I);
+  void checkPtrToAddr(Type *SrcTy, Type *DestTy, const Value &V);
+  void visitPtrToAddrInst(PtrToAddrInst &I);
   void visitPtrToIntInst(PtrToIntInst &I);
   void visitBitCastInst(BitCastInst &I);
   void visitAddrSpaceCastInst(AddrSpaceCastInst &I);
@@ -834,6 +835,7 @@ void Verifier::visitGlobalVariable(const GlobalVariable &GV) {
           &GV);
     Check(GV.getInitializer()->getType()->isSized(),
           "Global variable initializer must be sized", &GV);
+    visitConstantExprsRecursively(GV.getInitializer());
     // If the global has common linkage, it must have a zero initializer and
     // cannot be constant.
     if (GV.hasCommonLinkage()) {
@@ -2610,6 +2612,8 @@ void Verifier::visitConstantExpr(const ConstantExpr *CE) {
     Check(CastInst::castIsValid(Instruction::BitCast, CE->getOperand(0),
                                 CE->getType()),
           "Invalid bitcast", CE);
+  else if (CE->getOpcode() == Instruction::PtrToAddr)
+    checkPtrToAddr(CE->getOperand(0)->getType(), CE->getType(), *CE);
 }
 
 void Verifier::visitConstantPtrAuth(const ConstantPtrAuth *CPA) {
@@ -3532,6 +3536,28 @@ void Verifier::visitFPToSIInst(FPToSIInst &I) {
   visitInstruction(I);
 }
 
+void Verifier::checkPtrToAddr(Type *SrcTy, Type *DestTy, const Value &V) {
+  Check(SrcTy->isPtrOrPtrVectorTy(), "PtrToAddr source must be pointer", V);
+  Check(DestTy->isIntOrIntVectorTy(), "PtrToAddr result must be integral", V);
+  Check(SrcTy->isVectorTy() == DestTy->isVectorTy(), "PtrToAddr type mismatch",
+        V);
+
+  if (SrcTy->isVectorTy()) {
+    auto *VSrc = cast<VectorType>(SrcTy);
+    auto *VDest = cast<VectorType>(DestTy);
+    Check(VSrc->getElementCount() == VDest->getElementCount(),
+          "PtrToAddr vector length mismatch", V);
+  }
+
+  Type *AddrTy = DL.getAddressType(SrcTy);
+  Check(AddrTy == DestTy, "PtrToAddr result must be address width", V);
+}
+
+void Verifier::visitPtrToAddrInst(PtrToAddrInst &I) {
+  checkPtrToAddr(I.getOperand(0)->getType(), I.getType(), I);
+  visitInstruction(I);
+}
+
 void Verifier::visitPtrToIntInst(PtrToIntInst &I) {
   // Get the source and destination types
   Type *SrcTy = I.getOperand(0)->getType();
@@ -3547,7 +3573,7 @@ void Verifier::visitPtrToIntInst(PtrToIntInst &I) {
     auto *VSrc = cast<VectorType>(SrcTy);
     auto *VDest = cast<VectorType>(DestTy);
     Check(VSrc->getElementCount() == VDest->getElementCount(),
-          "PtrToInt Vector width mismatch", &I);
+          "PtrToInt Vector length mismatch", &I);
   }
 
   visitInstruction(I);
@@ -3567,7 +3593,7 @@ void Verifier::visitIntToPtrInst(IntToPtrInst &I) {
     auto *VSrc = cast<VectorType>(SrcTy);
     auto *VDest = cast<VectorType>(DestTy);
     Check(VSrc->getElementCount() == VDest->getElementCount(),
-          "IntToPtr Vector width mismatch", &I);
+          "IntToPtr Vector length mismatch", &I);
   }
   visitInstruction(I);
 }
@@ -4609,7 +4635,7 @@ void Verifier::visitEHPadPredecessors(Instruction &I) {
     }
 
     // The edge may exit from zero or more nested pads.
-    SmallSet<Value *, 8> Seen;
+    SmallPtrSet<Value *, 8> Seen;
     for (;; FromPad = getParentPad(FromPad)) {
       Check(FromPad != ToPad,
             "EH pad cannot handle exceptions raised within it", FromPad, TI);
@@ -4737,7 +4763,7 @@ void Verifier::visitFuncletPadInst(FuncletPadInst &FPI) {
   User *FirstUser = nullptr;
   Value *FirstUnwindPad = nullptr;
   SmallVector<FuncletPadInst *, 8> Worklist({&FPI});
-  SmallSet<FuncletPadInst *, 8> Seen;
+  SmallPtrSet<FuncletPadInst *, 8> Seen;
 
   while (!Worklist.empty()) {
     FuncletPadInst *CurrentPad = Worklist.pop_back_val();
@@ -6612,6 +6638,36 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
           "Value for inactive lanes must be a VGPR function argument", &Call);
     break;
   }
+  case Intrinsic::amdgcn_call_whole_wave: {
+    auto F = dyn_cast<Function>(Call.getArgOperand(0));
+    Check(F, "Indirect whole wave calls are not allowed", &Call);
+
+    CallingConv::ID CC = F->getCallingConv();
+    Check(CC == CallingConv::AMDGPU_Gfx_WholeWave,
+          "Callee must have the amdgpu_gfx_whole_wave calling convention",
+          &Call);
+
+    Check(!F->isVarArg(), "Variadic whole wave calls are not allowed", &Call);
+
+    Check(Call.arg_size() == F->arg_size(),
+          "Call argument count must match callee argument count", &Call);
+
+    // The first argument of the call is the callee, and the first argument of
+    // the callee is the active mask. The rest of the arguments must match.
+    Check(F->arg_begin()->getType()->isIntegerTy(1),
+          "Callee must have i1 as its first argument", &Call);
+    for (auto [CallArg, FuncArg] :
+         drop_begin(zip_equal(Call.args(), F->args()))) {
+      Check(CallArg->getType() == FuncArg.getType(),
+            "Argument types must match", &Call);
+
+      // Check that inreg attributes match between call site and function
+      Check(Call.paramHasAttr(FuncArg.getArgNo(), Attribute::InReg) ==
+                FuncArg.hasInRegAttr(),
+            "Argument inreg attributes must match", &Call);
+    }
+    break;
+  }
   case Intrinsic::amdgcn_s_prefetch_data: {
     Check(
         AMDGPU::isFlatGlobalAddrSpace(
@@ -6770,7 +6826,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
   }
   case Intrinsic::lifetime_start:
   case Intrinsic::lifetime_end: {
-    Value *Ptr = Call.getArgOperand(1);
+    Value *Ptr = Call.getArgOperand(0);
     Check(isa<AllocaInst>(Ptr) || isa<PoisonValue>(Ptr),
           "llvm.lifetime.start/end can only be used on alloca or poison",
           &Call);
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 0323b4d433b87..35d24c17bbd93 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1422,7 +1422,7 @@ SmallVector<const char *> LTO::getRuntimeLibcallSymbols(const Triple &TT) {
 
   for (RTLIB::LibcallImpl Impl : LibcallImpls) {
     if (Impl != RTLIB::Unsupported)
-      LibcallSymbols.push_back(Libcalls.getLibcallImplName(Impl));
+      LibcallSymbols.push_back(Libcalls.getLibcallImplName(Impl).data());
   }
 
   return LibcallSymbols;
diff --git a/llvm/lib/LTO/LTOModule.cpp b/llvm/lib/LTO/LTOModule.cpp
index e0a975806a31d..7dd06118e2a57 100644
--- a/llvm/lib/LTO/LTOModule.cpp
+++ b/llvm/lib/LTO/LTOModule.cpp
@@ -203,8 +203,10 @@ LTOModule::makeLTOModule(MemoryBufferRef Buffer, const TargetOptions &options,
   // find machine architecture for this module
   std::string errMsg;
   const Target *march = TargetRegistry::lookupTarget(Triple, errMsg);
-  if (!march)
+  if (!march) {
+    Context.emitError(errMsg);
     return make_error_code(object::object_error::arch_not_found);
+  }
 
   // construct LTOModule, hand over ownership of module and target
   SubtargetFeatures Features;
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index a466ce5bf0d4c..d6c15de4c4cdc 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -1133,8 +1133,11 @@ void IRLinker::linkNamedMDNodes() {
 
     NamedMDNode *DestNMD = DstM.getOrInsertNamedMetadata(NMD.getName());
     // Add Src elements into Dest node.
-    for (const MDNode *Op : NMD.operands())
-      DestNMD->addOperand(Mapper.mapMDNode(*Op));
+    for (const MDNode *Op : NMD.operands()) {
+      MDNode *MD = Mapper.mapMDNode(*Op);
+      if (!is_contained(DestNMD->operands(), MD))
+        DestNMD->addOperand(MD);
+    }
   }
 }
 
diff --git a/llvm/lib/MC/CMakeLists.txt b/llvm/lib/MC/CMakeLists.txt
index 18a85b3fed08c..1e1d0a6d00601 100644
--- a/llvm/lib/MC/CMakeLists.txt
+++ b/llvm/lib/MC/CMakeLists.txt
@@ -45,6 +45,7 @@ add_llvm_component_library(LLVMMC
   MCSection.cpp
   MCSectionMachO.cpp
   MCStreamer.cpp
+  MCSFrame.cpp
   MCSPIRVStreamer.cpp
   MCSubtargetInfo.cpp
   MCSymbol.cpp
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index 8f3814a1dd62e..759d3e0e14291 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -541,12 +541,12 @@ void ELFWriter::computeSymbolTable(const RevGroupMapTy &RevGroupMap) {
     if (Symbol.isAbsolute()) {
       MSD.SectionIndex = ELF::SHN_ABS;
     } else if (Symbol.isCommon()) {
-      if (Symbol.isTargetCommon()) {
-        MSD.SectionIndex = Symbol.getIndex();
-      } else {
+      auto Shndx = Symbol.getIndex();
+      if (!Shndx) {
         assert(!Local);
-        MSD.SectionIndex = ELF::SHN_COMMON;
+        Shndx = ELF::SHN_COMMON;
       }
+      MSD.SectionIndex = Shndx;
     } else if (Symbol.isUndefined()) {
       if (Symbol.isSignature() && !Symbol.isUsedInReloc()) {
         MSD.SectionIndex = RevGroupMap.lookup(&Symbol);
diff --git a/llvm/lib/MC/GOFFObjectWriter.cpp b/llvm/lib/MC/GOFFObjectWriter.cpp
index 3b629cdb87fc1..d68f4af86752f 100644
--- a/llvm/lib/MC/GOFFObjectWriter.cpp
+++ b/llvm/lib/MC/GOFFObjectWriter.cpp
@@ -17,7 +17,6 @@
 #include "llvm/MC/MCSectionGOFF.h"
 #include "llvm/MC/MCSymbolGOFF.h"
 #include "llvm/MC/MCValue.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/ConvertEBCDIC.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Endian.h"
diff --git a/llvm/lib/MC/MCAsmInfoGOFF.cpp b/llvm/lib/MC/MCAsmInfoGOFF.cpp
index 0a5d1927b1a22..092736b991f93 100644
--- a/llvm/lib/MC/MCAsmInfoGOFF.cpp
+++ b/llvm/lib/MC/MCAsmInfoGOFF.cpp
@@ -62,6 +62,8 @@ static void emitCATTR(raw_ostream &OS, StringRef Name, GOFF::ESDRmode Rmode,
     OS << ',';
     OS << "RMODE(";
     switch (Rmode) {
+    case GOFF::ESD_RMODE_None:
+      llvm_unreachable("");
     case GOFF::ESD_RMODE_24:
       OS << "24";
       break;
@@ -71,8 +73,6 @@ static void emitCATTR(raw_ostream &OS, StringRef Name, GOFF::ESDRmode Rmode,
     case GOFF::ESD_RMODE_64:
       OS << "64";
       break;
-    case GOFF::ESD_RMODE_None:
-      break;
     }
     OS << ')';
   }
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index 9a5e07095fa5a..be8c022f39ad1 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -28,7 +28,6 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbolXCOFF.h"
 #include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/FormattedStream.h"
@@ -2341,6 +2340,9 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
 
   getAssembler().getEmitter().encodeInstruction(Inst, Code, Fixups, STI);
 
+  // RISC-V instructions are always little-endian, even on BE systems.
+  bool ForceLE = getContext().getTargetTriple().isRISCV();
+
   // If we are showing fixups, create symbolic markers in the encoded
   // representation. We do this by making a per-bit map to the fixup item index,
   // then trying to display it as nicely as possible.
@@ -2395,7 +2397,10 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
         unsigned Bit = (Code[i] >> j) & 1;
 
         unsigned FixupBit;
-        if (MAI->isLittleEndian())
+        // RISC-V instructions are always little-endian.
+        // The FixupMap is indexed by actual bit positions in the LE
+        // instruction.
+        if (MAI->isLittleEndian() || ForceLE)
           FixupBit = i * 8 + j;
         else
           FixupBit = i * 8 + (7-j);
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index d172ad170a4d5..b1031d7822604 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -203,7 +203,7 @@ uint64_t MCAssembler::computeFragmentSize(const MCFragment &F) const {
   case MCFragment::FT_CVDefRange:
     return F.getSize();
   case MCFragment::FT_Fill: {
-    auto &FF = cast<MCFillFragment>(F);
+    auto &FF = static_cast<const MCFillFragment &>(F);
     int64_t NumValues = 0;
     if (!FF.getNumValues().evaluateKnownAbsolute(NumValues, *this)) {
       recordError(FF.getLoc(), "expected assembly-time absolute expression");
@@ -353,10 +353,9 @@ const MCSymbol *MCAssembler::getBaseSymbol(const MCSymbol &Symbol) const {
 }
 
 uint64_t MCAssembler::getSectionAddressSize(const MCSection &Sec) const {
-  assert(HasLayout);
-  // The size is the last fragment's end offset.
   const MCFragment &F = *Sec.curFragList()->Tail;
-  return getFragmentOffset(F) + computeFragmentSize(F);
+  assert(HasLayout && F.getKind() == MCFragment::FT_Data);
+  return getFragmentOffset(F) + F.getSize();
 }
 
 uint64_t MCAssembler::getSectionFileSize(const MCSection &Sec) const {
@@ -749,7 +748,7 @@ bool MCAssembler::fixupNeedsRelaxation(const MCFragment &F,
                                                    Resolved);
 }
 
-bool MCAssembler::relaxInstruction(MCFragment &F) {
+void MCAssembler::relaxInstruction(MCFragment &F) {
   assert(getEmitterPtr() &&
          "Expected CodeEmitter defined for relaxInstruction");
   // If this inst doesn't ever need relaxation, ignore it. This occurs when we
@@ -757,14 +756,14 @@ bool MCAssembler::relaxInstruction(MCFragment &F) {
   // previous instruction to one that doesn't need relaxation.
   if (!getBackend().mayNeedRelaxation(F.getOpcode(), F.getOperands(),
                                       *F.getSubtargetInfo()))
-    return false;
+    return;
 
   bool DoRelax = false;
   for (const MCFixup &Fixup : F.getVarFixups())
     if ((DoRelax = fixupNeedsRelaxation(F, Fixup)))
       break;
   if (!DoRelax)
-    return false;
+    return;
 
   ++stats::RelaxedInstructions;
 
@@ -780,12 +779,10 @@ bool MCAssembler::relaxInstruction(MCFragment &F) {
   getEmitter().encodeInstruction(Relaxed, Data, Fixups, *F.getSubtargetInfo());
   F.setVarContents(Data);
   F.setVarFixups(Fixups);
-  return true;
 }
 
-bool MCAssembler::relaxLEB(MCFragment &F) {
-  const unsigned OldSize = F.getVarSize();
-  unsigned PadTo = OldSize;
+void MCAssembler::relaxLEB(MCFragment &F) {
+  unsigned PadTo = F.getVarSize();
   int64_t Value;
   F.clearVarFixups();
   // Use evaluateKnownAbsolute for Mach-O as a hack: .subsections_via_symbols
@@ -819,7 +816,6 @@ bool MCAssembler::relaxLEB(MCFragment &F) {
   else
     Size = encodeULEB128(Value, Data, PadTo);
   F.setVarContents({reinterpret_cast<char *>(Data), Size});
-  return OldSize != Size;
 }
 
 /// Check if the branch crosses the boundary.
@@ -859,11 +855,11 @@ static bool needPadding(uint64_t StartAddr, uint64_t Size,
          isAgainstBoundary(StartAddr, Size, BoundaryAlignment);
 }
 
-bool MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
+void MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
   // BoundaryAlignFragment that doesn't need to align any fragment should not be
   // relaxed.
   if (!BF.getLastFragment())
-    return false;
+    return;
 
   uint64_t AlignedOffset = getFragmentOffset(BF);
   uint64_t AlignedSize = 0;
@@ -878,18 +874,15 @@ bool MCAssembler::relaxBoundaryAlign(MCBoundaryAlignFragment &BF) {
                          ? offsetToAlignment(AlignedOffset, BoundaryAlignment)
                          : 0U;
   if (NewSize == BF.getSize())
-    return false;
+    return;
   BF.setSize(NewSize);
-  return true;
 }
 
-bool MCAssembler::relaxDwarfLineAddr(MCFragment &F) {
-  bool WasRelaxed;
-  if (getBackend().relaxDwarfLineAddr(F, WasRelaxed))
-    return WasRelaxed;
+void MCAssembler::relaxDwarfLineAddr(MCFragment &F) {
+  if (getBackend().relaxDwarfLineAddr(F))
+    return;
 
   MCContext &Context = getContext();
-  auto OldSize = F.getVarSize();
   int64_t AddrDelta;
   bool Abs = F.getDwarfAddrDelta().evaluateKnownAbsolute(AddrDelta, *this);
   assert(Abs && "We created a line delta with an invalid expression");
@@ -899,13 +892,11 @@ bool MCAssembler::relaxDwarfLineAddr(MCFragment &F) {
                           F.getDwarfLineDelta(), AddrDelta, Data);
   F.setVarContents(Data);
   F.clearVarFixups();
-  return OldSize != Data.size();
 }
 
-bool MCAssembler::relaxDwarfCallFrameFragment(MCFragment &F) {
-  bool WasRelaxed;
-  if (getBackend().relaxDwarfCFA(F, WasRelaxed))
-    return WasRelaxed;
+void MCAssembler::relaxDwarfCallFrameFragment(MCFragment &F) {
+  if (getBackend().relaxDwarfCFA(F))
+    return;
 
   MCContext &Context = getContext();
   int64_t Value;
@@ -914,69 +905,49 @@ bool MCAssembler::relaxDwarfCallFrameFragment(MCFragment &F) {
     reportError(F.getDwarfAddrDelta().getLoc(),
                 "invalid CFI advance_loc expression");
     F.setDwarfAddrDelta(MCConstantExpr::create(0, Context));
-    return false;
+    return;
   }
 
-  auto OldSize = F.getVarContents().size();
   SmallVector<char, 8> Data;
   MCDwarfFrameEmitter::encodeAdvanceLoc(Context, Value, Data);
   F.setVarContents(Data);
   F.clearVarFixups();
-  return OldSize != Data.size();
-}
-
-bool MCAssembler::relaxCVInlineLineTable(MCCVInlineLineTableFragment &F) {
-  unsigned OldSize = F.getVarContents().size();
-  getContext().getCVContext().encodeInlineLineTable(*this, F);
-  return OldSize != F.getVarContents().size();
-}
-
-bool MCAssembler::relaxCVDefRange(MCCVDefRangeFragment &F) {
-  unsigned OldSize = F.getVarContents().size();
-  getContext().getCVContext().encodeDefRange(*this, F);
-  return OldSize != F.getVarContents().size();
-}
-
-bool MCAssembler::relaxFill(MCFillFragment &F) {
-  uint64_t Size = computeFragmentSize(F);
-  if (F.getSize() == Size)
-    return false;
-  F.setSize(Size);
-  return true;
-}
-
-bool MCAssembler::relaxOrg(MCOrgFragment &F) {
-  uint64_t Size = computeFragmentSize(F);
-  if (F.getSize() == Size)
-    return false;
-  F.setSize(Size);
-  return true;
 }
 
 bool MCAssembler::relaxFragment(MCFragment &F) {
-  switch(F.getKind()) {
+  auto Size = computeFragmentSize(F);
+  switch (F.getKind()) {
   default:
     return false;
   case MCFragment::FT_Relaxable:
     assert(!getRelaxAll() && "Did not expect a FT_Relaxable in RelaxAll mode");
-    return relaxInstruction(F);
+    relaxInstruction(F);
+    break;
   case MCFragment::FT_LEB:
-    return relaxLEB(F);
+    relaxLEB(F);
+    break;
   case MCFragment::FT_Dwarf:
-    return relaxDwarfLineAddr(F);
+    relaxDwarfLineAddr(F);
+    break;
   case MCFragment::FT_DwarfFrame:
-    return relaxDwarfCallFrameFragment(F);
+    relaxDwarfCallFrameFragment(F);
+    break;
   case MCFragment::FT_BoundaryAlign:
-    return relaxBoundaryAlign(cast<MCBoundaryAlignFragment>(F));
+    relaxBoundaryAlign(static_cast<MCBoundaryAlignFragment &>(F));
+    break;
   case MCFragment::FT_CVInlineLines:
-    return relaxCVInlineLineTable(cast<MCCVInlineLineTableFragment>(F));
+    getContext().getCVContext().encodeInlineLineTable(
+        *this, static_cast<MCCVInlineLineTableFragment &>(F));
+    break;
   case MCFragment::FT_CVDefRange:
-    return relaxCVDefRange(cast<MCCVDefRangeFragment>(F));
+    getContext().getCVContext().encodeDefRange(
+        *this, static_cast<MCCVDefRangeFragment &>(F));
+    break;
   case MCFragment::FT_Fill:
-    return relaxFill(cast<MCFillFragment>(F));
   case MCFragment::FT_Org:
-    return relaxOrg(static_cast<MCOrgFragment &>(F));
+    return F.getNext()->Offset - F.Offset != Size;
   }
+  return computeFragmentSize(F) != Size;
 }
 
 void MCAssembler::layoutSection(MCSection &Sec) {
@@ -1025,7 +996,7 @@ unsigned MCAssembler::relaxOnce(unsigned FirstStable) {
     for (;;) {
       bool Changed = false;
       for (MCFragment &F : Sec)
-        if (relaxFragment(F))
+        if (F.getKind() != MCFragment::FT_Data && relaxFragment(F))
           Changed = true;
 
       if (!Changed)
diff --git a/llvm/lib/MC/MCContext.cpp b/llvm/lib/MC/MCContext.cpp
index 5e364e9ad50c0..5d9ddc2f1306c 100644
--- a/llvm/lib/MC/MCContext.cpp
+++ b/llvm/lib/MC/MCContext.cpp
@@ -42,7 +42,6 @@
 #include "llvm/MC/MCSymbolXCOFF.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/SectionKind.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -327,7 +326,6 @@ MCSymbol *MCContext::cloneSymbol(MCSymbol &Sym) {
 
   // Ensure the original symbol is not emitted to the symbol table.
   Sym.IsTemporary = true;
-  Sym.setExternal(false);
   return NewSym;
 }
 
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 275e76e37f9fc..2881d7cfab4ba 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -29,7 +29,6 @@
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/LEB128.h"
 #include <cassert>
diff --git a/llvm/lib/MC/MCFragment.cpp b/llvm/lib/MC/MCFragment.cpp
index 6cbdf749a6943..21da79bb0aa30 100644
--- a/llvm/lib/MC/MCFragment.cpp
+++ b/llvm/lib/MC/MCFragment.cpp
@@ -68,6 +68,8 @@ LLVM_DUMP_METHOD void MCFragment::dump() const {
       OS << "\n  Fixup @" << F.getOffset() << " Value:";
       F.getValue()->print(OS, nullptr);
       OS << " Kind:" << F.getKind();
+      if (F.isLinkerRelaxable())
+        OS << " LinkerRelaxable";
     }
   };
 
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index 6226b02a0615b..2b7a248e6d109 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -149,7 +149,7 @@ void MCMachOStreamer::emitEHSymAttributes(const MCSymbol *Symbol,
                                           MCSymbol *EHSymbol) {
   auto *Sym = static_cast<const MCSymbolMachO *>(Symbol);
   getAssembler().registerSymbol(*Symbol);
-  if (Symbol->isExternal())
+  if (Sym->isExternal())
     emitSymbolAttribute(EHSymbol, MCSA_Global);
   if (Sym->isWeakDefinition())
     emitSymbolAttribute(EHSymbol, MCSA_WeakDefinition);
@@ -372,12 +372,13 @@ void MCMachOStreamer::emitSymbolDesc(MCSymbol *Symbol, unsigned DescValue) {
 
 void MCMachOStreamer::emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                        Align ByteAlignment) {
+  auto &Sym = static_cast<MCSymbolMachO &>(*Symbol);
   // FIXME: Darwin 'as' does appear to allow redef of a .comm by itself.
   assert(Symbol->isUndefined() && "Cannot define a symbol twice!");
 
-  getAssembler().registerSymbol(*Symbol);
-  Symbol->setExternal(true);
-  Symbol->setCommon(Size, ByteAlignment);
+  getAssembler().registerSymbol(Sym);
+  Sym.setExternal(true);
+  Sym.setCommon(Size, ByteAlignment);
 }
 
 void MCMachOStreamer::emitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
@@ -460,7 +461,8 @@ void MCMachOStreamer::finishImpl() {
 }
 
 void MCMachOStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *&SRE) {
-  const MCSymbol *S = &SRE->getSymbol();
+  auto *S =
+      static_cast<MCSymbolMachO *>(const_cast<MCSymbol *>(&SRE->getSymbol()));
   if (getAssembler().registerSymbol(*S))
     S->setExternal(true);
 }
diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp
index 4ac73ab34aeda..d505ac6dd4bf2 100644
--- a/llvm/lib/MC/MCObjectFileInfo.cpp
+++ b/llvm/lib/MC/MCObjectFileInfo.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/BinaryFormat/SFrame.h"
 #include "llvm/BinaryFormat/Wasm.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
@@ -23,7 +24,6 @@
 #include "llvm/MC/MCSectionSPIRV.h"
 #include "llvm/MC/MCSectionWasm.h"
 #include "llvm/MC/MCSectionXCOFF.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/TargetParser/Triple.h"
 
 using namespace llvm;
@@ -380,6 +380,19 @@ void MCObjectFileInfo::initELFMCObjectFileInfo(const Triple &T, bool Large) {
   unsigned EHSectionType = T.getArch() == Triple::x86_64
                                ? ELF::SHT_X86_64_UNWIND
                                : ELF::SHT_PROGBITS;
+  switch (T.getArch()) {
+  case Triple::x86_64:
+    SFrameABIArch = sframe::ABI::AMD64EndianLittle;
+    break;
+  case Triple::aarch64:
+    SFrameABIArch = sframe::ABI::AArch64EndianLittle;
+    break;
+  case Triple::aarch64_be:
+    SFrameABIArch = sframe::ABI::AArch64EndianBig;
+    break;
+  default:
+    break;
+  }
 
   // Solaris requires different flags for .eh_frame to seemingly every other
   // platform.
@@ -537,6 +550,9 @@ void MCObjectFileInfo::initELFMCObjectFileInfo(const Triple &T, bool Large) {
   EHFrameSection =
       Ctx->getELFSection(".eh_frame", EHSectionType, EHSectionFlags);
 
+  SFrameSection =
+      Ctx->getELFSection(".sframe", ELF::SHT_GNU_SFRAME, ELF::SHF_ALLOC);
+
   CallGraphSection = Ctx->getELFSection(".callgraph", ELF::SHT_PROGBITS, 0);
 
   StackSizesSection = Ctx->getELFSection(".stack_sizes", ELF::SHT_PROGBITS, 0);
@@ -1064,6 +1080,7 @@ void MCObjectFileInfo::initMCObjectFileInfo(MCContext &MCCtx, bool PIC,
   CompactUnwindDwarfEHFrameOnly = 0;
 
   EHFrameSection = nullptr;             // Created on demand.
+  SFrameSection = nullptr;              // Created on demand.
   CompactUnwindSection = nullptr;       // Used only by selected targets.
   DwarfAccelNamesSection = nullptr;     // Used only by selected targets.
   DwarfAccelObjCSection = nullptr;      // Used only by selected targets.
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 8c279586bb4d3..59265bc8595ba 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -17,6 +17,7 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCSFrame.h"
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -30,7 +31,7 @@ MCObjectStreamer::MCObjectStreamer(MCContext &Context,
     : MCStreamer(Context),
       Assembler(std::make_unique<MCAssembler>(
           Context, std::move(TAB), std::move(Emitter), std::move(OW))),
-      EmitEHFrame(true), EmitDebugFrame(false) {
+      EmitEHFrame(true), EmitDebugFrame(false), EmitSFrame(false) {
   assert(Assembler->getBackendPtr() && Assembler->getEmitterPtr());
   IsObj = true;
   setAllowAutoPadding(Assembler->getBackend().allowAutoPadding());
@@ -186,6 +187,10 @@ void MCObjectStreamer::emitFrames(MCAsmBackend *MAB) {
 
   if (EmitDebugFrame)
     MCDwarfFrameEmitter::Emit(*this, MAB, false);
+
+  if (EmitSFrame || (getContext().getTargetOptions() &&
+                     getContext().getTargetOptions()->EmitSFrameUnwind))
+    MCSFrameEmitter::emit(*this);
 }
 
 void MCObjectStreamer::visitUsedSymbol(const MCSymbol &Sym) {
@@ -443,7 +448,7 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst,
     // MCAssembler::relaxAlign.
     auto *Sec = F->getParent();
     if (!Sec->isLinkerRelaxable())
-      Sec->setLinkerRelaxable();
+      Sec->setFirstLinkerRelaxable(F->getLayoutOrder());
     // Do not add data after a linker-relaxable instruction. The difference
     // between a new label and a label at or before the linker-relaxable
     // instruction cannot be resolved at assemble-time.
@@ -461,11 +466,23 @@ void MCObjectStreamer::emitInstToFragment(const MCInst &Inst,
   getAssembler().getEmitter().encodeInstruction(Inst, Data, Fixups, STI);
 
   F->Kind = MCFragment::FT_Relaxable;
-  F->STI = &STI;
-  F->HasInstructions = true;
+  F->setHasInstructions(STI);
+
   F->setVarContents(Data);
-  F->setVarFixups(Fixups);
   F->setInst(Inst);
+
+  bool MarkedLinkerRelaxable = false;
+  for (auto &Fixup : Fixups) {
+    if (!Fixup.isLinkerRelaxable() || MarkedLinkerRelaxable)
+      continue;
+    MarkedLinkerRelaxable = true;
+    auto *Sec = F->getParent();
+    if (!Sec->isLinkerRelaxable())
+      Sec->setFirstLinkerRelaxable(F->getLayoutOrder());
+    F->setLinkerRelaxable();
+  }
+  F->setVarFixups(Fixups);
+
   newFragment();
 }
 
diff --git a/llvm/lib/MC/MCParser/AsmParser.cpp b/llvm/lib/MC/MCParser/AsmParser.cpp
index 7782dc1f50055..fb183a10b3d37 100644
--- a/llvm/lib/MC/MCParser/AsmParser.cpp
+++ b/llvm/lib/MC/MCParser/AsmParser.cpp
@@ -1864,11 +1864,13 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
       Lex();
     }
 
-    if (MAI.hasSubsectionsViaSymbols() && CFIStartProcLoc &&
-        Sym->isExternal() && !static_cast<MCSymbolMachO *>(Sym)->isAltEntry())
-      return Error(StartTokLoc, "non-private labels cannot appear between "
-                                ".cfi_startproc / .cfi_endproc pairs") &&
-             Error(*CFIStartProcLoc, "previous .cfi_startproc was here");
+    if (MAI.isMachO() && CFIStartProcLoc) {
+      auto *SymM = static_cast<MCSymbolMachO *>(Sym);
+      if (SymM->isExternal() && !SymM->isAltEntry())
+        return Error(StartTokLoc, "non-private labels cannot appear between "
+                                  ".cfi_startproc / .cfi_endproc pairs") &&
+               Error(*CFIStartProcLoc, "previous .cfi_startproc was here");
+    }
 
     if (discardLTOSymbol(IDVal))
       return false;
diff --git a/llvm/lib/MC/MCParser/COFFMasmParser.cpp b/llvm/lib/MC/MCParser/COFFMasmParser.cpp
index 229b0b8d53892..1bb617b327f1e 100644
--- a/llvm/lib/MC/MCParser/COFFMasmParser.cpp
+++ b/llvm/lib/MC/MCParser/COFFMasmParser.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbolCOFF.h"
 #include "llvm/MC/SectionKind.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/SMLoc.h"
 #include <cstdint>
 #include <utility>
diff --git a/llvm/lib/MC/MCParser/ELFAsmParser.cpp b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
index 6782c4b61863d..513f3b3da7813 100644
--- a/llvm/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/ELFAsmParser.cpp
@@ -22,7 +22,6 @@
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/SectionKind.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/SMLoc.h"
 #include <cassert>
 #include <cstdint>
diff --git a/llvm/lib/MC/MCParser/MasmParser.cpp b/llvm/lib/MC/MCParser/MasmParser.cpp
index 780289eda8e45..2dcfe0f3a420a 100644
--- a/llvm/lib/MC/MCParser/MasmParser.cpp
+++ b/llvm/lib/MC/MCParser/MasmParser.cpp
@@ -39,7 +39,7 @@
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolCOFF.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
@@ -3009,8 +3009,8 @@ bool MasmParser::parseDirectiveEquate(StringRef IDVal, StringRef Name,
     return false;
   }
 
-  MCSymbol *Sym = getContext().getOrCreateSymbol(Var.Name);
-
+  auto *Sym =
+      static_cast<MCSymbolCOFF *>(getContext().getOrCreateSymbol(Var.Name));
   const MCConstantExpr *PrevValue =
       Sym->isVariable()
           ? dyn_cast_or_null<MCConstantExpr>(Sym->getVariableValue())
@@ -4521,7 +4521,8 @@ bool MasmParser::parseDirectiveExtern() {
       KnownType[Name.lower()] = Type;
     }
 
-    MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
+    auto *Sym =
+        static_cast<MCSymbolCOFF *>(getContext().getOrCreateSymbol(Name));
     Sym->setExternal(true);
     getStreamer().emitSymbolAttribute(Sym, MCSA_Extern);
 
diff --git a/llvm/lib/MC/MCParser/WasmAsmParser.cpp b/llvm/lib/MC/MCParser/WasmAsmParser.cpp
index 6c2d2411bc28e..ddfe1e10d9d0a 100644
--- a/llvm/lib/MC/MCParser/WasmAsmParser.cpp
+++ b/llvm/lib/MC/MCParser/WasmAsmParser.cpp
@@ -26,7 +26,6 @@
 #include "llvm/MC/MCSectionWasm.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbolWasm.h"
-#include "llvm/Support/Casting.h"
 #include <optional>
 
 using namespace llvm;
diff --git a/llvm/lib/MC/MCSFrame.cpp b/llvm/lib/MC/MCSFrame.cpp
new file mode 100644
index 0000000000000..ee17b774e4740
--- /dev/null
+++ b/llvm/lib/MC/MCSFrame.cpp
@@ -0,0 +1,175 @@
+//===- lib/MC/MCSFrame.cpp - MCSFrame implementation ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/MC/MCSFrame.h"
+#include "llvm/BinaryFormat/SFrame.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCObjectStreamer.h"
+#include "llvm/MC/MCSection.h"
+#include "llvm/MC/MCSymbol.h"
+#include "llvm/Support/EndianStream.h"
+
+using namespace llvm;
+using namespace sframe;
+
+namespace {
+
+// High-level structure to track info needed to emit a sframe_func_desc_entry
+// and its associated FREs.
+struct SFrameFDE {
+  // Reference to the original dwarf frame to avoid copying.
+  const MCDwarfFrameInfo &DFrame;
+  // Label where this FDE's FREs start.
+  MCSymbol *FREStart;
+
+  SFrameFDE(const MCDwarfFrameInfo &DF, MCSymbol *FRES)
+      : DFrame(DF), FREStart(FRES) {}
+
+  void emit(MCObjectStreamer &S, const MCSymbol *FRESubSectionStart) {
+    MCContext &C = S.getContext();
+
+    // sfde_func_start_address
+    const MCExpr *V = C.getAsmInfo()->getExprForFDESymbol(
+        &(*DFrame.Begin), C.getObjectFileInfo()->getFDEEncoding(), S);
+    S.emitValue(V, sizeof(int32_t));
+
+    // sfde_func_size
+    S.emitAbsoluteSymbolDiff(DFrame.End, DFrame.Begin, sizeof(uint32_t));
+
+    // sfde_func_start_fre_off
+    auto *F = S.getCurrentFragment();
+    const MCExpr *Diff = MCBinaryExpr::createSub(
+        MCSymbolRefExpr::create(FREStart, C),
+        MCSymbolRefExpr::create(FRESubSectionStart, C), C);
+
+    F->addFixup(MCFixup::create(F->getContents().size(), Diff,
+                                MCFixup::getDataKindForSize(4)));
+    S.emitInt32(0);
+
+    // sfde_func_start_num_fres
+    S.emitInt32(0);
+
+    // sfde_func_info word
+    FDEInfo<endianness::native> I;
+    I.setFuncInfo(0 /* No pauth key */, FDEType::PCInc, FREType::Addr1);
+    S.emitInt8(I.Info);
+
+    // sfde_func_rep_size. Not relevant in non-PCMASK fdes.
+    S.emitInt8(0);
+
+    // sfde_func_padding2
+    S.emitInt16(0);
+  }
+};
+
+// Emitting these field-by-field, instead of constructing the actual structures
+// lets Streamer do target endian-fixups for free.
+
+class SFrameEmitterImpl {
+  MCObjectStreamer &Streamer;
+  SmallVector<SFrameFDE> FDEs;
+  ABI SFrameABI;
+  MCSymbol *FDESubSectionStart;
+  MCSymbol *FRESubSectionStart;
+  MCSymbol *FRESubSectionEnd;
+
+public:
+  SFrameEmitterImpl(MCObjectStreamer &Streamer) : Streamer(Streamer) {
+    assert(Streamer.getContext()
+               .getObjectFileInfo()
+               ->getSFrameABIArch()
+               .has_value());
+    FDEs.reserve(Streamer.getDwarfFrameInfos().size());
+    SFrameABI = *Streamer.getContext().getObjectFileInfo()->getSFrameABIArch();
+    FDESubSectionStart = Streamer.getContext().createTempSymbol();
+    FRESubSectionStart = Streamer.getContext().createTempSymbol();
+    FRESubSectionEnd = Streamer.getContext().createTempSymbol();
+  }
+
+  void BuildSFDE(const MCDwarfFrameInfo &DF) {
+    FDEs.emplace_back(DF, Streamer.getContext().createTempSymbol());
+  }
+
+  void emitPreamble() {
+    Streamer.emitInt16(Magic);
+    Streamer.emitInt8(static_cast<uint8_t>(Version::V2));
+    Streamer.emitInt8(static_cast<uint8_t>(Flags::FDEFuncStartPCRel));
+  }
+
+  void emitHeader() {
+    emitPreamble();
+    // sfh_abi_arch
+    Streamer.emitInt8(static_cast<uint8_t>(SFrameABI));
+    // sfh_cfa_fixed_fp_offset
+    Streamer.emitInt8(0);
+    // sfh_cfa_fixed_ra_offset
+    Streamer.emitInt8(0);
+    // sfh_auxhdr_len
+    Streamer.emitInt8(0);
+    // shf_num_fdes
+    Streamer.emitInt32(FDEs.size());
+    // shf_num_fres
+    Streamer.emitInt32(0);
+    // shf_fre_len
+    Streamer.emitAbsoluteSymbolDiff(FRESubSectionEnd, FRESubSectionStart,
+                                    sizeof(int32_t));
+    // shf_fdeoff. With no sfh_auxhdr, these immediately follow this header.
+    Streamer.emitInt32(0);
+    // shf_freoff
+    Streamer.emitAbsoluteSymbolDiff(FRESubSectionStart, FDESubSectionStart,
+                                    sizeof(uint32_t));
+  }
+
+  void emitFDEs() {
+    Streamer.emitLabel(FDESubSectionStart);
+    for (auto &FDE : FDEs) {
+      FDE.emit(Streamer, FRESubSectionStart);
+    }
+  }
+
+  void emitFREs() {
+    Streamer.emitLabel(FRESubSectionStart);
+    for (auto &FDE : FDEs)
+      Streamer.emitLabel(FDE.FREStart);
+    Streamer.emitLabel(FRESubSectionEnd);
+  }
+};
+
+} // end anonymous namespace
+
+void MCSFrameEmitter::emit(MCObjectStreamer &Streamer) {
+  MCContext &Context = Streamer.getContext();
+  // If this target doesn't support sframes, return now. Gas doesn't warn in
+  // this case, but if we want to, it should be done at option-parsing time,
+  // rather than here.
+  if (!Streamer.getContext()
+           .getObjectFileInfo()
+           ->getSFrameABIArch()
+           .has_value())
+    return;
+
+  SFrameEmitterImpl Emitter(Streamer);
+  ArrayRef<MCDwarfFrameInfo> FrameArray = Streamer.getDwarfFrameInfos();
+
+  // Both the header itself and the FDEs include various offsets and counts.
+  // Therefore, all of this must be precomputed.
+  for (const auto &DFrame : FrameArray)
+    Emitter.BuildSFDE(DFrame);
+
+  MCSection *Section = Context.getObjectFileInfo()->getSFrameSection();
+  // Not strictly necessary, but gas always aligns to 8, so match that.
+  Section->ensureMinAlignment(Align(8));
+  Streamer.switchSection(Section);
+  MCSymbol *SectionStart = Context.createTempSymbol();
+  Streamer.emitLabel(SectionStart);
+  Emitter.emitHeader();
+  Emitter.emitFDEs();
+  Emitter.emitFREs();
+}
diff --git a/llvm/lib/MC/MCSection.cpp b/llvm/lib/MC/MCSection.cpp
index 27ca1314074fe..a668e7919b7b9 100644
--- a/llvm/lib/MC/MCSection.cpp
+++ b/llvm/lib/MC/MCSection.cpp
@@ -20,7 +20,7 @@ using namespace llvm;
 
 MCSection::MCSection(StringRef Name, bool IsText, bool IsBss, MCSymbol *Begin)
     : Begin(Begin), HasInstructions(false), IsRegistered(false), IsText(IsText),
-      IsBss(IsBss), LinkerRelaxable(false), Name(Name) {
+      IsBss(IsBss), Name(Name) {
   DummyFragment.setParent(this);
 }
 
@@ -39,6 +39,8 @@ LLVM_DUMP_METHOD void MCSection::dump(
   raw_ostream &OS = errs();
 
   OS << "MCSection Name:" << getName();
+  if (isLinkerRelaxable())
+    OS << " FirstLinkerRelaxable:" << firstLinkerRelaxable();
   for (auto &F : *this) {
     OS << '\n';
     F.dump();
diff --git a/llvm/lib/MC/MCSymbol.cpp b/llvm/lib/MC/MCSymbol.cpp
index 8192896eeb6bb..b86873824cb00 100644
--- a/llvm/lib/MC/MCSymbol.cpp
+++ b/llvm/lib/MC/MCSymbol.cpp
@@ -20,6 +20,10 @@
 
 using namespace llvm;
 
+// There are numerous MCSymbol objects, so keeping sizeof(MCSymbol) small is
+// crucial for minimizing peak memory usage.
+static_assert(sizeof(MCSymbol) <= 24, "Keep the base symbol small");
+
 // Only the address of this fragment is ever actually used.
 static MCFragment SentinelFragment;
 
@@ -44,13 +48,12 @@ void *MCSymbol::operator new(size_t s, const MCSymbolTableEntry *Name,
 }
 
 void MCSymbol::setVariableValue(const MCExpr *Value) {
-  assert(Value && "Invalid variable value!");
-  assert((SymbolContents == SymContentsUnset ||
-          SymbolContents == SymContentsVariable) &&
-         "Cannot give common/offset symbol a variable value");
+  assert(Value && "Invalid equated expression");
+  assert((kind == Kind::Regular || kind == Kind::Equated) &&
+         "Cannot equate a common symbol");
   this->Value = Value;
-  SymbolContents = SymContentsVariable;
-  setUndefined();
+  kind = Kind::Equated;
+  Fragment = nullptr;
 }
 
 void MCSymbol::print(raw_ostream &OS, const MCAsmInfo *MAI) const {
diff --git a/llvm/lib/MC/MCWasmStreamer.cpp b/llvm/lib/MC/MCWasmStreamer.cpp
index 9c8b22480fb7f..070b3d9f8d2c8 100644
--- a/llvm/lib/MC/MCWasmStreamer.cpp
+++ b/llvm/lib/MC/MCWasmStreamer.cpp
@@ -22,7 +22,6 @@
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolWasm.h"
 #include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 
 namespace llvm {
diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index a45936bebf0c1..2e632deaf3276 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -30,7 +30,6 @@
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/MCWinCOFFObjectWriter.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/SMLoc.h"
diff --git a/llvm/lib/MC/MCXCOFFStreamer.cpp b/llvm/lib/MC/MCXCOFFStreamer.cpp
index a0e3dbaca3db1..4bf14c11068cb 100644
--- a/llvm/lib/MC/MCXCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCXCOFFStreamer.cpp
@@ -21,7 +21,6 @@
 #include "llvm/MC/MCSymbolXCOFF.h"
 #include "llvm/MC/MCXCOFFObjectWriter.h"
 #include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/Casting.h"
 
 using namespace llvm;
 
@@ -129,14 +128,14 @@ void MCXCOFFStreamer::emitXCOFFCInfoSym(StringRef Name, StringRef Metadata) {
 
 void MCXCOFFStreamer::emitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                        Align ByteAlignment) {
-  auto *Sym = static_cast<MCSymbolXCOFF *>(Symbol);
+  auto &Sym = static_cast<MCSymbolXCOFF &>(*Symbol);
   getAssembler().registerSymbol(*Symbol);
-  Symbol->setExternal(Sym->getStorageClass() != XCOFF::C_HIDEXT);
+  Sym.setExternal(Sym.getStorageClass() != XCOFF::C_HIDEXT);
   Symbol->setCommon(Size, ByteAlignment);
 
   // Default csect align is 4, but common symbols have explicit alignment values
   // and we should honor it.
-  Sym->getRepresentedCsect()->setAlignment(ByteAlignment);
+  Sym.getRepresentedCsect()->setAlignment(ByteAlignment);
 
   // Emit the alignment and storage for the variable to the section.
   emitValueToAlignment(ByteAlignment);
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index eb59e39d73eda..39542bfbdd8e3 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -383,7 +383,7 @@ const MCSymbol &MachObjectWriter::findAliasedSymbol(const MCSymbol &Sym) const {
 }
 
 void MachObjectWriter::writeNlist(MachSymbolData &MSD, const MCAssembler &Asm) {
-  auto *Symbol = static_cast<const MCSymbolMachO *>(MSD.Symbol);
+  auto *Symbol = MSD.Symbol;
   const auto &Data = static_cast<const MCSymbolMachO &>(*Symbol);
   auto *AliasedSymbol =
       static_cast<const MCSymbolMachO *>(&findAliasedSymbol(*Symbol));
@@ -602,15 +602,16 @@ void MachObjectWriter::computeSymbolTable(
   // match 'as'. Even though it doesn't matter for correctness, this is
   // important for letting us diff .o files.
   for (const MCSymbol &Symbol : Asm.symbols()) {
+    auto &Sym = static_cast<const MCSymbolMachO &>(Symbol);
     // Ignore non-linker visible symbols.
-    if (!static_cast<const MCSymbolMachO &>(Symbol).isSymbolLinkerVisible())
+    if (!Sym.isSymbolLinkerVisible())
       continue;
 
-    if (!Symbol.isExternal() && !Symbol.isUndefined())
+    if (!Sym.isExternal() && !Sym.isUndefined())
       continue;
 
     MachSymbolData MSD;
-    MSD.Symbol = &Symbol;
+    MSD.Symbol = &Sym;
     MSD.StringIndex = StringTable.getOffset(Symbol.getName());
 
     if (Symbol.isUndefined()) {
@@ -628,15 +629,16 @@ void MachObjectWriter::computeSymbolTable(
 
   // Now add the data for local symbols.
   for (const MCSymbol &Symbol : Asm.symbols()) {
+    auto &Sym = static_cast<const MCSymbolMachO &>(Symbol);
     // Ignore non-linker visible symbols.
-    if (!static_cast<const MCSymbolMachO &>(Symbol).isSymbolLinkerVisible())
+    if (!Sym.isSymbolLinkerVisible())
       continue;
 
-    if (Symbol.isExternal() || Symbol.isUndefined())
+    if (Sym.isExternal() || Sym.isUndefined())
       continue;
 
     MachSymbolData MSD;
-    MSD.Symbol = &Symbol;
+    MSD.Symbol = &Sym;
     MSD.StringIndex = StringTable.getOffset(Symbol.getName());
 
     if (Symbol.isAbsolute()) {
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index af009a48193bd..15590b31fd07f 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -528,8 +528,12 @@ void WasmObjectWriter::recordRelocation(const MCFragment &F,
   // be negative and don't wrap.
   FixedValue = 0;
 
-  unsigned Type =
-      TargetObjectWriter->getRelocType(Target, Fixup, FixupSection, IsLocRel);
+  unsigned Type;
+  if (mc::isRelocRelocation(Fixup.getKind()))
+    Type = Fixup.getKind() - FirstLiteralRelocationKind;
+  else
+    Type =
+        TargetObjectWriter->getRelocType(Target, Fixup, FixupSection, IsLocRel);
 
   // Absolute offset within a section or a function.
   // Currently only supported for metadata sections.
@@ -1561,7 +1565,7 @@ uint64_t WasmObjectWriter::writeOneObject(MCAssembler &Asm,
                  << toString(WS.getType().value_or(wasm::WASM_SYMBOL_TYPE_DATA))
                  << " '" << S << "'"
                  << " isDefined=" << S.isDefined() << " isExternal="
-                 << S.isExternal() << " isTemporary=" << S.isTemporary()
+                 << WS.isExternal() << " isTemporary=" << S.isTemporary()
                  << " isWeak=" << WS.isWeak() << " isHidden=" << WS.isHidden()
                  << " isVariable=" << WS.isVariable() << "\n");
 
diff --git a/llvm/lib/MC/WinCOFFObjectWriter.cpp b/llvm/lib/MC/WinCOFFObjectWriter.cpp
index 0cc5ff5d66d8b..8f850c0228f37 100644
--- a/llvm/lib/MC/WinCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/WinCOFFObjectWriter.cpp
@@ -168,13 +168,13 @@ class llvm::WinCOFFWriter {
 private:
   MCContext &getContext() const { return OWriter.getContext(); }
   COFFSymbol *createSymbol(StringRef Name);
-  COFFSymbol *GetOrCreateCOFFSymbol(const MCSymbol *Symbol);
+  COFFSymbol *getOrCreateCOFFSymbol(const MCSymbol &Sym);
   COFFSection *createSection(StringRef Name);
 
   void defineSection(MCSectionCOFF const &Sec);
 
   COFFSymbol *getLinkedSymbol(const MCSymbol &Symbol);
-  void defineSymbol(const MCSymbol &Symbol);
+  void defineSymbol(const MCSymbolCOFF &Symbol);
 
   void SetSymbolName(COFFSymbol &S);
   void SetSectionName(COFFSection &S);
@@ -244,10 +244,10 @@ COFFSymbol *WinCOFFWriter::createSymbol(StringRef Name) {
   return Symbols.back().get();
 }
 
-COFFSymbol *WinCOFFWriter::GetOrCreateCOFFSymbol(const MCSymbol *Symbol) {
-  COFFSymbol *&Ret = SymbolMap[Symbol];
+COFFSymbol *WinCOFFWriter::getOrCreateCOFFSymbol(const MCSymbol &Sym) {
+  COFFSymbol *&Ret = SymbolMap[&Sym];
   if (!Ret)
-    Ret = createSymbol(Symbol->getName());
+    Ret = createSymbol(Sym.getName());
   return Ret;
 }
 
@@ -303,7 +303,7 @@ void WinCOFFWriter::defineSection(const MCSectionCOFF &MCSec) {
   // Create a COMDAT symbol if needed.
   if (MCSec.getSelection() != COFF::IMAGE_COMDAT_SELECT_ASSOCIATIVE) {
     if (const MCSymbol *S = MCSec.getCOMDATSymbol()) {
-      COFFSymbol *COMDATSymbol = GetOrCreateCOFFSymbol(S);
+      COFFSymbol *COMDATSymbol = getOrCreateCOFFSymbol(*S);
       if (COMDATSymbol->Section)
         report_fatal_error("two sections have the same comdat");
       COMDATSymbol->Section = Section;
@@ -339,7 +339,8 @@ void WinCOFFWriter::defineSection(const MCSectionCOFF &MCSec) {
   }
 }
 
-static uint64_t getSymbolValue(const MCSymbol &Symbol, const MCAssembler &Asm) {
+static uint64_t getSymbolValue(const MCSymbolCOFF &Symbol,
+                               const MCAssembler &Asm) {
   if (Symbol.isCommon() && Symbol.isExternal())
     return Symbol.getCommonSize();
 
@@ -354,21 +355,20 @@ COFFSymbol *WinCOFFWriter::getLinkedSymbol(const MCSymbol &Symbol) {
   if (!Symbol.isVariable())
     return nullptr;
 
-  const MCSymbolRefExpr *SymRef =
-      dyn_cast<MCSymbolRefExpr>(Symbol.getVariableValue());
+  const auto *SymRef = dyn_cast<MCSymbolRefExpr>(Symbol.getVariableValue());
   if (!SymRef)
     return nullptr;
 
-  const MCSymbol &Aliasee = SymRef->getSymbol();
+  auto &Aliasee = static_cast<const MCSymbolCOFF &>(SymRef->getSymbol());
   if (Aliasee.isUndefined() || Aliasee.isExternal())
-    return GetOrCreateCOFFSymbol(&Aliasee);
+    return getOrCreateCOFFSymbol(Aliasee);
   else
     return nullptr;
 }
 
 /// This function takes a symbol data object from the assembler
 /// and creates the associated COFF symbol staging object.
-void WinCOFFWriter::defineSymbol(const MCSymbol &MCSym) {
+void WinCOFFWriter::defineSymbol(const MCSymbolCOFF &MCSym) {
   const MCSymbol *Base = Asm->getBaseSymbol(MCSym);
   COFFSection *Sec = nullptr;
   MCSectionCOFF *MCSec = nullptr;
@@ -380,7 +380,7 @@ void WinCOFFWriter::defineSymbol(const MCSymbol &MCSym) {
   if (Mode == NonDwoOnly && MCSec && isDwoSection(*MCSec))
     return;
 
-  COFFSymbol *Sym = GetOrCreateCOFFSymbol(&MCSym);
+  COFFSymbol *Sym = getOrCreateCOFFSymbol(MCSym);
   COFFSymbol *Local = nullptr;
   if (static_cast<const MCSymbolCOFF &>(MCSym)
           .getWeakExternalCharacteristics()) {
@@ -819,13 +819,14 @@ void WinCOFFWriter::executePostLayoutBinding() {
     defineSection(static_cast<const MCSectionCOFF &>(Section));
   }
 
-  if (Mode != DwoOnly)
-    for (const MCSymbol &Symbol : Asm->symbols())
+  if (Mode != DwoOnly) {
+    for (const MCSymbol &Symbol : Asm->symbols()) {
+      auto &Sym = static_cast<const MCSymbolCOFF &>(Symbol);
       // Define non-temporary or temporary static (private-linkage) symbols
-      if (!Symbol.isTemporary() ||
-          static_cast<const MCSymbolCOFF &>(Symbol).getClass() ==
-              COFF::IMAGE_SYM_CLASS_STATIC)
-        defineSymbol(Symbol);
+      if (!Sym.isTemporary() || Sym.getClass() == COFF::IMAGE_SYM_CLASS_STATIC)
+        defineSymbol(Sym);
+    }
+  }
 
   UseBigObj = Sections.size() > COFF::MaxNumberOfSections16;
   Header.NumberOfSections = Sections.size();
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index 13917ba53933f..fce6b2ac5cf8d 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -20,7 +20,6 @@
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/MCXCOFFObjectWriter.h"
 #include "llvm/MC/StringTableBuilder.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
diff --git a/llvm/lib/MCA/Instruction.cpp b/llvm/lib/MCA/Instruction.cpp
index d4adfce597139..79667080ecbbc 100644
--- a/llvm/lib/MCA/Instruction.cpp
+++ b/llvm/lib/MCA/Instruction.cpp
@@ -128,6 +128,13 @@ void WriteState::dump() const {
 }
 #endif
 
+#ifndef NDEBUG
+void ReadState::dump() const {
+  dbgs() << "{ OpIdx=" << RD->OpIndex << ", RegID " << getRegisterID()
+         << ", Cycles Left=" << CyclesLeft << " }";
+}
+#endif
+
 const CriticalDependency &Instruction::computeCriticalRegDep() {
   if (CriticalRegDep.Cycles)
     return CriticalRegDep;
diff --git a/llvm/lib/ObjCopy/CMakeLists.txt b/llvm/lib/ObjCopy/CMakeLists.txt
index 2d6bee94875f0..960f1d1fae2d0 100644
--- a/llvm/lib/ObjCopy/CMakeLists.txt
+++ b/llvm/lib/ObjCopy/CMakeLists.txt
@@ -16,6 +16,9 @@ source_group("Header Files\\wasm" REGULAR_EXPRESSION
 source_group("Header Files\\XCOFF" REGULAR_EXPRESSION
   XCOFF/.*[.]h
 )
+source_group("Header Files\\DXContainer" REGULAR_EXPRESSION
+  DXContainer/.*[.]h
+)
 source_group("Source Files" REGULAR_EXPRESSION
   .*[.]cpp
 )
@@ -34,9 +37,16 @@ source_group("Source Files\\wasm" REGULAR_EXPRESSION
 source_group("Source Files\\XCOFF" REGULAR_EXPRESSION
   XCOFF/.*[.]cpp
 )
+source_group("Source Files\\DXContainer" REGULAR_EXPRESSION
+  DXContainer/.*[.]cpp
+)
 
 add_llvm_component_library(LLVMObjCopy
   Archive.cpp
+  DXContainer/DXContainerObjcopy.cpp
+  DXContainer/DXContainerObject.cpp
+  DXContainer/DXContainerReader.cpp
+  DXContainer/DXContainerWriter.cpp
   CommonConfig.cpp
   ObjCopy.cpp
   ConfigManager.cpp
@@ -62,11 +72,13 @@ add_llvm_component_library(LLVMObjCopy
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ObjCopy
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ObjCopy/COFF
+  ${LLVM_MAIN_INCLUDE_DIR}/llvm/ObjCopy/DXContainer
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ObjCopy/ELF
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ObjCopy/MachO
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ObjCopy/wasm
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/ObjCopy/XCOFF
   COFF
+  DXContainer
   ELF
   MachO
   wasm
diff --git a/llvm/lib/ObjCopy/ConfigManager.cpp b/llvm/lib/ObjCopy/ConfigManager.cpp
index 9a81b51c4d277..2b17d728aad3f 100644
--- a/llvm/lib/ObjCopy/ConfigManager.cpp
+++ b/llvm/lib/ObjCopy/ConfigManager.cpp
@@ -107,3 +107,27 @@ Expected<const XCOFFConfig &> ConfigManager::getXCOFFConfig() const {
 
   return XCOFF;
 }
+
+Expected<const DXContainerConfig &>
+ConfigManager::getDXContainerConfig() const {
+  // All other flags are either supported or not applicable for DXContainer
+  // object files and will be silently ignored.
+  if (!Common.AddGnuDebugLink.empty() || !Common.SplitDWO.empty() ||
+      !Common.AllocSectionsPrefix.empty() ||
+      Common.DiscardMode != DiscardType::None || !Common.AddSection.empty() ||
+      !Common.DumpSection.empty() || !Common.KeepSection.empty() ||
+      !Common.OnlySection.empty() || !Common.SectionsToRename.empty() ||
+      !Common.SetSectionAlignment.empty() || !Common.SetSectionFlags.empty() ||
+      !Common.SetSectionType.empty() || Common.ExtractDWO ||
+      Common.OnlyKeepDebug || Common.StripAllGNU || Common.StripDWO ||
+      Common.StripDebug || Common.StripNonAlloc || Common.StripSections ||
+      Common.StripUnneeded || Common.DecompressDebugSections ||
+      Common.GapFill != 0 || Common.PadTo != 0 ||
+      Common.ChangeSectionLMAValAll != 0 ||
+      !Common.ChangeSectionAddress.empty()) {
+    return createStringError(
+        llvm::errc::invalid_argument,
+        "no flags are supported yet, only basic copying is allowed");
+  }
+  return DXContainer;
+}
diff --git a/llvm/lib/ObjCopy/DXContainer/DXContainerObjcopy.cpp b/llvm/lib/ObjCopy/DXContainer/DXContainerObjcopy.cpp
new file mode 100644
index 0000000000000..375e382ddb044
--- /dev/null
+++ b/llvm/lib/ObjCopy/DXContainer/DXContainerObjcopy.cpp
@@ -0,0 +1,59 @@
+//===- DXContainerObjcopy.cpp ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ObjCopy/DXContainer/DXContainerObjcopy.h"
+#include "DXContainerReader.h"
+#include "DXContainerWriter.h"
+#include "llvm/ObjCopy/CommonConfig.h"
+#include "llvm/ObjCopy/DXContainer/DXContainerConfig.h"
+
+namespace llvm {
+namespace objcopy {
+namespace dxbc {
+
+using namespace object;
+
+static Error handleArgs(const CommonConfig &Config, Object &Obj) {
+  std::function<bool(const Part &)> RemovePred = [](const Part &) {
+    return false;
+  };
+
+  if (!Config.ToRemove.empty())
+    RemovePred = [&Config](const Part &P) {
+      return Config.ToRemove.matches(P.Name);
+    };
+
+  if (auto E = Obj.removeParts(RemovePred))
+    return E;
+
+  Obj.recomputeHeader();
+  return Error::success();
+}
+
+Error executeObjcopyOnBinary(const CommonConfig &Config,
+                             const DXContainerConfig &,
+                             DXContainerObjectFile &In, raw_ostream &Out) {
+  DXContainerReader Reader(In);
+  Expected<std::unique_ptr<Object>> ObjOrErr = Reader.create();
+  if (!ObjOrErr)
+    return createFileError(Config.InputFilename, ObjOrErr.takeError());
+  Object *Obj = ObjOrErr->get();
+  assert(Obj && "Unable to deserialize DXContainer object");
+
+  if (Error E = handleArgs(Config, *Obj))
+    return E;
+
+  DXContainerWriter Writer(*Obj, Out);
+  if (Error E = Writer.write())
+    return createFileError(Config.OutputFilename, std::move(E));
+  return Error::success();
+}
+
+} // end namespace dxbc
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/llvm/lib/ObjCopy/DXContainer/DXContainerObject.cpp b/llvm/lib/ObjCopy/DXContainer/DXContainerObject.cpp
new file mode 100644
index 0000000000000..b845478de16d8
--- /dev/null
+++ b/llvm/lib/ObjCopy/DXContainer/DXContainerObject.cpp
@@ -0,0 +1,29 @@
+//===- DXContainerObject.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DXContainerObject.h"
+
+namespace llvm {
+namespace objcopy {
+namespace dxbc {
+
+Error Object::removeParts(PartPred ToRemove) {
+  erase_if(Parts, ToRemove);
+  return Error::success();
+}
+
+void Object::recomputeHeader() {
+  Header.FileSize = headerSize();
+  Header.PartCount = Parts.size();
+  for (const Part &P : Parts)
+    Header.FileSize += P.size();
+}
+
+} // end namespace dxbc
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/llvm/lib/ObjCopy/DXContainer/DXContainerObject.h b/llvm/lib/ObjCopy/DXContainer/DXContainerObject.h
new file mode 100644
index 0000000000000..cbb09f5ec8e0d
--- /dev/null
+++ b/llvm/lib/ObjCopy/DXContainer/DXContainerObject.h
@@ -0,0 +1,52 @@
+//===- DXContainerObject.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_OBJCOPY_DXCONTAINER_DXCONTAINEROBJECT_H
+#define LLVM_LIB_OBJCOPY_DXCONTAINER_DXCONTAINEROBJECT_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Object/DXContainer.h"
+#include <vector>
+
+namespace llvm {
+namespace objcopy {
+namespace dxbc {
+
+using namespace object;
+
+struct Part {
+  StringRef Name;
+  ArrayRef<uint8_t> Data;
+
+  size_t size() const {
+    return sizeof(::llvm::dxbc::PartHeader) // base header
+           + Data.size();                   // contents size
+  }
+};
+
+using PartPred = llvm::function_ref<bool(const Part &)>;
+
+struct Object {
+  ::llvm::dxbc::Header Header;
+  SmallVector<Part> Parts;
+
+  size_t headerSize() const {
+    return sizeof(::llvm::dxbc::Header)       // base header
+           + sizeof(uint32_t) * Parts.size(); // part offset values
+  }
+
+  Error removeParts(PartPred ToRemove);
+  void recomputeHeader();
+};
+
+} // end namespace dxbc
+} // end namespace objcopy
+} // end namespace llvm
+
+#endif // LLVM_LIB_OBJCOPY_DXCONTAINER_DXCONTAINEROBJECT_H
diff --git a/llvm/lib/ObjCopy/DXContainer/DXContainerReader.cpp b/llvm/lib/ObjCopy/DXContainer/DXContainerReader.cpp
new file mode 100644
index 0000000000000..61ee94c5cae95
--- /dev/null
+++ b/llvm/lib/ObjCopy/DXContainer/DXContainerReader.cpp
@@ -0,0 +1,38 @@
+//===- DXContainerReader.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DXContainerReader.h"
+
+namespace llvm {
+namespace objcopy {
+namespace dxbc {
+
+using namespace object;
+
+Expected<std::unique_ptr<Object>> DXContainerReader::create() const {
+  auto Obj = std::make_unique<Object>();
+  Obj->Header = DXContainerObj.getHeader();
+  for (const SectionRef &Part : DXContainerObj.sections()) {
+    DataRefImpl PartDRI = Part.getRawDataRefImpl();
+    Expected<StringRef> Name = DXContainerObj.getSectionName(PartDRI);
+    if (auto E = Name.takeError())
+      return E;
+    assert(Name->size() == 4 &&
+           "Valid DXIL Part name consists of 4 characters");
+    Expected<ArrayRef<uint8_t>> Data =
+        DXContainerObj.getSectionContents(PartDRI);
+    if (auto E = Data.takeError())
+      return E;
+    Obj->Parts.push_back({*Name, *Data});
+  }
+  return std::move(Obj);
+}
+
+} // end namespace dxbc
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/llvm/lib/ObjCopy/DXContainer/DXContainerReader.h b/llvm/lib/ObjCopy/DXContainer/DXContainerReader.h
new file mode 100644
index 0000000000000..54058ff133adb
--- /dev/null
+++ b/llvm/lib/ObjCopy/DXContainer/DXContainerReader.h
@@ -0,0 +1,34 @@
+//===- DXContainerReader.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_OBJCOPY_DXCONTAINER_DXCONTAINERREADER_H
+#define LLVM_LIB_OBJCOPY_DXCONTAINER_DXCONTAINERREADER_H
+
+#include "DXContainerObject.h"
+
+namespace llvm {
+namespace objcopy {
+namespace dxbc {
+
+using namespace object;
+
+class DXContainerReader {
+public:
+  explicit DXContainerReader(const DXContainerObjectFile &Obj)
+      : DXContainerObj(Obj) {}
+  Expected<std::unique_ptr<Object>> create() const;
+
+private:
+  const DXContainerObjectFile &DXContainerObj;
+};
+
+} // end namespace dxbc
+} // end namespace objcopy
+} // end namespace llvm
+
+#endif // LLVM_LIB_OBJCOPY_DXCONTAINER_DXCONTAINERREADER_H
diff --git a/llvm/lib/ObjCopy/DXContainer/DXContainerWriter.cpp b/llvm/lib/ObjCopy/DXContainer/DXContainerWriter.cpp
new file mode 100644
index 0000000000000..6c26ec4cff646
--- /dev/null
+++ b/llvm/lib/ObjCopy/DXContainer/DXContainerWriter.cpp
@@ -0,0 +1,58 @@
+//===- DXContainerWriter.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DXContainerWriter.h"
+
+namespace llvm {
+namespace objcopy {
+namespace dxbc {
+
+using namespace object;
+
+size_t DXContainerWriter::finalize() {
+  assert(Offsets.empty() &&
+         "Attempted to finalize writer with already computed offsets");
+  Offsets.reserve(Obj.Parts.size());
+  size_t Offset = Obj.headerSize();
+  for (const Part &P : Obj.Parts) {
+    Offsets.push_back(Offset);
+    Offset += P.size();
+  }
+  return Obj.Header.FileSize;
+}
+
+Error DXContainerWriter::write() {
+  size_t TotalSize = finalize();
+  Out.reserveExtraSpace(TotalSize);
+
+  llvm::dxbc::Header Header = Obj.Header;
+  if (sys::IsBigEndianHost)
+    Header.swapBytes();
+  Out.write(reinterpret_cast<const char *>(&Header),
+            sizeof(::llvm::dxbc::Header));
+  if (sys::IsBigEndianHost)
+    for (auto &O : Offsets)
+      sys::swapByteOrder(O);
+  Out.write(reinterpret_cast<const char *>(Offsets.data()),
+            Offsets.size() * sizeof(uint32_t));
+
+  for (const Part &P : Obj.Parts) {
+    Out.write(reinterpret_cast<const char *>(P.Name.data()), 4);
+    uint32_t Size = P.Data.size();
+    if (sys::IsBigEndianHost)
+      sys::swapByteOrder(Size);
+    Out.write(reinterpret_cast<const char *>(&Size), sizeof(uint32_t));
+    Out.write(reinterpret_cast<const char *>(P.Data.data()), P.Data.size());
+  }
+
+  return Error::success();
+}
+
+} // end namespace dxbc
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/llvm/lib/ObjCopy/DXContainer/DXContainerWriter.h b/llvm/lib/ObjCopy/DXContainer/DXContainerWriter.h
new file mode 100644
index 0000000000000..c785c1663b39c
--- /dev/null
+++ b/llvm/lib/ObjCopy/DXContainer/DXContainerWriter.h
@@ -0,0 +1,39 @@
+//===- DXContainerWriter.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_OBJCOPY_DXCONTAINER_DXCONTAINERWRITER_H
+#define LLVM_LIB_OBJCOPY_DXCONTAINER_DXCONTAINERWRITER_H
+
+#include "DXContainerObject.h"
+
+namespace llvm {
+namespace objcopy {
+namespace dxbc {
+
+using namespace object;
+
+class DXContainerWriter {
+public:
+  explicit DXContainerWriter(const Object &Obj, raw_ostream &Out)
+      : Obj(Obj), Out(Out) {}
+  Error write();
+
+private:
+  const Object &Obj;
+  raw_ostream &Out;
+
+  SmallVector<uint32_t> Offsets;
+
+  size_t finalize();
+};
+
+} // end namespace dxbc
+} // end namespace objcopy
+} // end namespace llvm
+
+#endif // LLVM_LIB_OBJCOPY_DXCONTAINER_DXCONTAINERWRITER_H
diff --git a/llvm/lib/ObjCopy/MachO/MachOWriter.cpp b/llvm/lib/ObjCopy/MachO/MachOWriter.cpp
index 89c1df8699298..07514dd2f8d6a 100644
--- a/llvm/lib/ObjCopy/MachO/MachOWriter.cpp
+++ b/llvm/lib/ObjCopy/MachO/MachOWriter.cpp
@@ -301,7 +301,7 @@ void MachOWriter::writeSymbolTable() {
       O.LoadCommands[*O.SymTabCommandIndex]
           .MachOLoadCommand.symtab_command_data;
 
-  char *SymTable = (char *)Buf->getBufferStart() + SymTabCommand.symoff;
+  char *SymTable = Buf->getBufferStart() + SymTabCommand.symoff;
   for (auto &Symbol : O.SymTable.Symbols) {
     SymbolEntry *Sym = Symbol.get();
     uint32_t Nstrx = LayoutBuilder.getStringTableBuilder().getOffset(Sym->Name);
@@ -319,7 +319,7 @@ void MachOWriter::writeRebaseInfo() {
   const MachO::dyld_info_command &DyLdInfoCommand =
       O.LoadCommands[*O.DyLdInfoCommandIndex]
           .MachOLoadCommand.dyld_info_command_data;
-  char *Out = (char *)Buf->getBufferStart() + DyLdInfoCommand.rebase_off;
+  char *Out = Buf->getBufferStart() + DyLdInfoCommand.rebase_off;
   assert((DyLdInfoCommand.rebase_size == O.Rebases.Opcodes.size()) &&
          "Incorrect rebase opcodes size");
   memcpy(Out, O.Rebases.Opcodes.data(), O.Rebases.Opcodes.size());
@@ -331,7 +331,7 @@ void MachOWriter::writeBindInfo() {
   const MachO::dyld_info_command &DyLdInfoCommand =
       O.LoadCommands[*O.DyLdInfoCommandIndex]
           .MachOLoadCommand.dyld_info_command_data;
-  char *Out = (char *)Buf->getBufferStart() + DyLdInfoCommand.bind_off;
+  char *Out = Buf->getBufferStart() + DyLdInfoCommand.bind_off;
   assert((DyLdInfoCommand.bind_size == O.Binds.Opcodes.size()) &&
          "Incorrect bind opcodes size");
   memcpy(Out, O.Binds.Opcodes.data(), O.Binds.Opcodes.size());
@@ -343,7 +343,7 @@ void MachOWriter::writeWeakBindInfo() {
   const MachO::dyld_info_command &DyLdInfoCommand =
       O.LoadCommands[*O.DyLdInfoCommandIndex]
           .MachOLoadCommand.dyld_info_command_data;
-  char *Out = (char *)Buf->getBufferStart() + DyLdInfoCommand.weak_bind_off;
+  char *Out = Buf->getBufferStart() + DyLdInfoCommand.weak_bind_off;
   assert((DyLdInfoCommand.weak_bind_size == O.WeakBinds.Opcodes.size()) &&
          "Incorrect weak bind opcodes size");
   memcpy(Out, O.WeakBinds.Opcodes.data(), O.WeakBinds.Opcodes.size());
@@ -355,7 +355,7 @@ void MachOWriter::writeLazyBindInfo() {
   const MachO::dyld_info_command &DyLdInfoCommand =
       O.LoadCommands[*O.DyLdInfoCommandIndex]
           .MachOLoadCommand.dyld_info_command_data;
-  char *Out = (char *)Buf->getBufferStart() + DyLdInfoCommand.lazy_bind_off;
+  char *Out = Buf->getBufferStart() + DyLdInfoCommand.lazy_bind_off;
   assert((DyLdInfoCommand.lazy_bind_size == O.LazyBinds.Opcodes.size()) &&
          "Incorrect lazy bind opcodes size");
   memcpy(Out, O.LazyBinds.Opcodes.data(), O.LazyBinds.Opcodes.size());
@@ -367,7 +367,7 @@ void MachOWriter::writeExportInfo() {
   const MachO::dyld_info_command &DyLdInfoCommand =
       O.LoadCommands[*O.DyLdInfoCommandIndex]
           .MachOLoadCommand.dyld_info_command_data;
-  char *Out = (char *)Buf->getBufferStart() + DyLdInfoCommand.export_off;
+  char *Out = Buf->getBufferStart() + DyLdInfoCommand.export_off;
   assert((DyLdInfoCommand.export_size == O.Exports.Trie.size()) &&
          "Incorrect export trie size");
   memcpy(Out, O.Exports.Trie.data(), O.Exports.Trie.size());
@@ -397,7 +397,7 @@ void MachOWriter::writeLinkData(std::optional<size_t> LCIndex,
     return;
   const MachO::linkedit_data_command &LinkEditDataCommand =
       O.LoadCommands[*LCIndex].MachOLoadCommand.linkedit_data_command_data;
-  char *Out = (char *)Buf->getBufferStart() + LinkEditDataCommand.dataoff;
+  char *Out = Buf->getBufferStart() + LinkEditDataCommand.dataoff;
   assert((LinkEditDataCommand.datasize == LD.Data.size()) &&
          "Incorrect data size");
   memcpy(Out, LD.Data.data(), LD.Data.size());
@@ -574,7 +574,7 @@ void MachOWriter::writeExportsTrieData() {
   const MachO::linkedit_data_command &ExportsTrieCmd =
       O.LoadCommands[*O.ExportsTrieCommandIndex]
           .MachOLoadCommand.linkedit_data_command_data;
-  char *Out = (char *)Buf->getBufferStart() + ExportsTrieCmd.dataoff;
+  char *Out = Buf->getBufferStart() + ExportsTrieCmd.dataoff;
   assert((ExportsTrieCmd.datasize == O.Exports.Trie.size()) &&
          "Incorrect export trie size");
   memcpy(Out, O.Exports.Trie.data(), O.Exports.Trie.size());
diff --git a/llvm/lib/ObjCopy/ObjCopy.cpp b/llvm/lib/ObjCopy/ObjCopy.cpp
index d9a190d1a2ad1..152600f4f243f 100644
--- a/llvm/lib/ObjCopy/ObjCopy.cpp
+++ b/llvm/lib/ObjCopy/ObjCopy.cpp
@@ -9,6 +9,8 @@
 #include "llvm/ObjCopy/ObjCopy.h"
 #include "llvm/ObjCopy/COFF/COFFConfig.h"
 #include "llvm/ObjCopy/COFF/COFFObjcopy.h"
+#include "llvm/ObjCopy/DXContainer/DXContainerConfig.h"
+#include "llvm/ObjCopy/DXContainer/DXContainerObjcopy.h"
 #include "llvm/ObjCopy/ELF/ELFConfig.h"
 #include "llvm/ObjCopy/ELF/ELFObjcopy.h"
 #include "llvm/ObjCopy/MachO/MachOConfig.h"
@@ -19,6 +21,7 @@
 #include "llvm/ObjCopy/wasm/WasmConfig.h"
 #include "llvm/ObjCopy/wasm/WasmObjcopy.h"
 #include "llvm/Object/COFF.h"
+#include "llvm/Object/DXContainer.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/Error.h"
 #include "llvm/Object/MachO.h"
@@ -78,6 +81,15 @@ Error objcopy::executeObjcopyOnBinary(const MultiFormatConfig &Config,
     return xcoff::executeObjcopyOnBinary(Config.getCommonConfig(), *XCOFFConfig,
                                          *XCOFFBinary, Out);
   }
+  if (auto *DXContainerBinary = dyn_cast<object::DXContainerObjectFile>(&In)) {
+    Expected<const DXContainerConfig &> DXContainerConfig =
+        Config.getDXContainerConfig();
+    if (!DXContainerConfig)
+      return DXContainerConfig.takeError();
+
+    return dxbc::executeObjcopyOnBinary(
+        Config.getCommonConfig(), *DXContainerConfig, *DXContainerBinary, Out);
+  }
   return createStringError(object_error::invalid_file_type,
                            "unsupported object file format");
 }
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index 0f194953787e6..a45b34eb2e706 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -46,14 +46,6 @@ static cl::opt<bool> DisableBitcodeVersionUpgrade(
     "disable-bitcode-version-upgrade", cl::Hidden,
     cl::desc("Disable automatic bitcode upgrade for version mismatch"));
 
-static const char *PreservedSymbols[] = {
-    // There are global variables, so put it here instead of in
-    // RuntimeLibcalls.td.
-    // TODO: Are there similar such variables?
-    "__ssp_canary_word",
-    "__stack_chk_guard",
-};
-
 namespace {
 
 const char *getExpectedProducerName() {
@@ -81,12 +73,16 @@ struct Builder {
   // The StringTableBuilder does not create a copy of any strings added to it,
   // so this provides somewhere to store any strings that we create.
   Builder(SmallVector<char, 0> &Symtab, StringTableBuilder &StrtabBuilder,
-          BumpPtrAllocator &Alloc)
-      : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc) {}
+          BumpPtrAllocator &Alloc, const Triple &TT)
+      : Symtab(Symtab), StrtabBuilder(StrtabBuilder), Saver(Alloc), TT(TT),
+        Libcalls(TT) {}
 
   DenseMap<const Comdat *, int> ComdatMap;
   Mangler Mang;
-  Triple TT;
+  const Triple &TT;
+
+  // FIXME: This shouldn't be here.
+  RTLIB::RuntimeLibcallsInfo Libcalls;
 
   std::vector<storage::Comdat> Comdats;
   std::vector<storage::Module> Mods;
@@ -98,6 +94,10 @@ struct Builder {
 
   std::vector<storage::Str> DependentLibraries;
 
+  bool isPreservedName(StringRef Name) {
+    return Libcalls.getSupportedLibcallImpl(Name) != RTLIB::Unsupported;
+  }
+
   void setStr(storage::Str &S, StringRef Value) {
     S.Offset = StrtabBuilder.add(Value);
     S.Size = Value.size();
@@ -213,19 +213,6 @@ Expected<int> Builder::getComdatIndex(const Comdat *C, const Module *M) {
   return P.first->second;
 }
 
-static StringSet<> buildPreservedSymbolsSet(const Triple &TT) {
-  StringSet<> PreservedSymbolSet;
-  PreservedSymbolSet.insert(std::begin(PreservedSymbols),
-                            std::end(PreservedSymbols));
-  // FIXME: Do we need to pass in ABI fields from TargetOptions?
-  RTLIB::RuntimeLibcallsInfo Libcalls(TT);
-  for (RTLIB::LibcallImpl Impl : Libcalls.getLibcallImpls()) {
-    if (Impl != RTLIB::Unsupported)
-      PreservedSymbolSet.insert(Libcalls.getLibcallImplName(Impl));
-  }
-  return PreservedSymbolSet;
-}
-
 Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
                          const SmallPtrSet<GlobalValue *, 4> &Used,
                          ModuleSymbolTable::Symbol Msym) {
@@ -279,13 +266,10 @@ Error Builder::addSymbol(const ModuleSymbolTable &Msymtab,
     return Error::success();
   }
 
-  setStr(Sym.IRName, GV->getName());
-
-  static const StringSet<> PreservedSymbolsSet =
-      buildPreservedSymbolsSet(GV->getParent()->getTargetTriple());
-  bool IsPreservedSymbol = PreservedSymbolsSet.contains(GV->getName());
+  StringRef GVName = GV->getName();
+  setStr(Sym.IRName, GVName);
 
-  if (Used.count(GV) || IsPreservedSymbol)
+  if (Used.count(GV) || isPreservedName(GVName))
     Sym.Flags |= 1 << storage::Symbol::FB_used;
   if (GV->isThreadLocal())
     Sym.Flags |= 1 << storage::Symbol::FB_tls;
@@ -352,7 +336,6 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
   setStr(Hdr.Producer, kExpectedProducerName);
   setStr(Hdr.TargetTriple, IRMods[0]->getTargetTriple().str());
   setStr(Hdr.SourceFileName, IRMods[0]->getSourceFileName());
-  TT = IRMods[0]->getTargetTriple();
 
   for (auto *M : IRMods)
     if (Error Err = addModule(M))
@@ -378,7 +361,8 @@ Error Builder::build(ArrayRef<Module *> IRMods) {
 Error irsymtab::build(ArrayRef<Module *> Mods, SmallVector<char, 0> &Symtab,
                       StringTableBuilder &StrtabBuilder,
                       BumpPtrAllocator &Alloc) {
-  return Builder(Symtab, StrtabBuilder, Alloc).build(Mods);
+  const Triple &TT = Mods[0]->getTargetTriple();
+  return Builder(Symtab, StrtabBuilder, Alloc, TT).build(Mods);
 }
 
 // Upgrade a vector of bitcode modules created by an old version of LLVM by
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index 5db264207ffb7..e09dc947c2779 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -3115,7 +3115,7 @@ void ExportEntry::pushNode(uint64_t offset) {
   }
   State.ChildCount = *Children;
   if (State.ChildCount != 0 && Children + 1 >= Trie.end()) {
-    *E = malformedError("byte for count of childern in export trie data at "
+    *E = malformedError("byte for count of children in export trie data at "
                         "node: 0x" +
                         Twine::utohexstr(offset) +
                         " extends past end of trie data");
@@ -3157,7 +3157,7 @@ void ExportEntry::pushDownUntilBottom() {
     }
     for (const NodeState &node : nodes()) {
       if (node.Start == Trie.begin() + childNodeIndex){
-        *E = malformedError("loop in childern in export trie data at node: 0x" +
+        *E = malformedError("loop in children in export trie data at node: 0x" +
                             Twine::utohexstr(Top.Start - Trie.begin()) +
                             " back to node: 0x" +
                             Twine::utohexstr(childNodeIndex));
diff --git a/llvm/lib/Object/SFrameParser.cpp b/llvm/lib/Object/SFrameParser.cpp
index 5863490634e32..759b579230d9d 100644
--- a/llvm/lib/Object/SFrameParser.cpp
+++ b/llvm/lib/Object/SFrameParser.cpp
@@ -32,14 +32,25 @@ getDataSlice(ArrayRef<uint8_t> Data, uint64_t Offset, uint64_t Size) {
 }
 
 template <typename T>
-static Expected<const T &> getDataSliceAs(ArrayRef<uint8_t> Data,
-                                          uint64_t Offset) {
+static Expected<ArrayRef<T>>
+getDataSliceAsArrayOf(ArrayRef<uint8_t> Data, uint64_t Offset, uint64_t Count) {
   static_assert(std::is_trivial_v<T>);
-  Expected<ArrayRef<uint8_t>> Slice = getDataSlice(Data, Offset, sizeof(T));
+  Expected<ArrayRef<uint8_t>> Slice =
+      getDataSlice(Data, Offset, sizeof(T) * Count);
   if (!Slice)
     return Slice.takeError();
 
-  return *reinterpret_cast<const T *>(Slice->data());
+  return ArrayRef(reinterpret_cast<const T *>(Slice->data()), Count);
+}
+
+template <typename T>
+static Expected<const T &> getDataSliceAs(ArrayRef<uint8_t> Data,
+                                          uint64_t Offset) {
+  Expected<ArrayRef<T>> Array = getDataSliceAsArrayOf<T>(Data, Offset, 1);
+  if (!Array)
+    return Array.takeError();
+
+  return Array->front();
 }
 
 template <endianness E>
@@ -87,17 +98,134 @@ uint64_t SFrameParser<E>::getAbsoluteStartAddress(
   uint64_t Result = SectionAddress + FDE->StartAddress;
 
   if ((getPreamble().Flags.value() & sframe::Flags::FDEFuncStartPCRel) ==
-      sframe::Flags::FDEFuncStartPCRel) {
-    uintptr_t DataPtr = reinterpret_cast<uintptr_t>(Data.data());
-    uintptr_t FDEPtr = reinterpret_cast<uintptr_t>(&*FDE);
+      sframe::Flags::FDEFuncStartPCRel)
+    Result += offsetOf(FDE);
+
+  return Result;
+}
 
-    assert(DataPtr <= FDEPtr && FDEPtr < DataPtr + Data.size() &&
-           "Iterator does not belong to this object!");
+template <endianness E>
+uint64_t SFrameParser<E>::offsetOf(typename FDERange::iterator FDE) const {
+  uintptr_t DataPtr = reinterpret_cast<uintptr_t>(Data.data());
+  uintptr_t FDEPtr = reinterpret_cast<uintptr_t>(&*FDE);
+
+  assert(DataPtr <= FDEPtr && FDEPtr < DataPtr + Data.size() &&
+         "Iterator does not belong to this object!");
+  return FDEPtr - DataPtr;
+}
 
-    Result += FDEPtr - DataPtr;
+template <typename EndianT>
+static Error readArray(ArrayRef<uint8_t> Data, uint64_t Count, uint64_t &Offset,
+                       SmallVectorImpl<int32_t> &Vec) {
+  Expected<ArrayRef<EndianT>> RawArray =
+      getDataSliceAsArrayOf<EndianT>(Data, Offset, Count);
+  if (!RawArray)
+    return RawArray.takeError();
+  Offset += Count * sizeof(EndianT);
+  Vec.resize(Count);
+  llvm::copy(*RawArray, Vec.begin());
+  return Error::success();
+}
+
+template <typename T, endianness E>
+static Error readFRE(ArrayRef<uint8_t> Data, uint64_t &Offset,
+                     typename SFrameParser<E>::FrameRowEntry &FRE) {
+  Expected<sframe::FrameRowEntry<T, E>> RawFRE =
+      getDataSliceAs<sframe::FrameRowEntry<T, E>>(Data, Offset);
+  if (!RawFRE)
+    return RawFRE.takeError();
+
+  Offset += sizeof(*RawFRE);
+  FRE.StartAddress = RawFRE->StartAddress;
+  FRE.Info.Info = RawFRE->Info.Info;
+
+  switch (FRE.Info.getOffsetSize()) {
+  case sframe::FREOffset::B1:
+    return readArray<sframe::detail::packed<int8_t, E>>(
+        Data, FRE.Info.getOffsetCount(), Offset, FRE.Offsets);
+  case sframe::FREOffset::B2:
+    return readArray<sframe::detail::packed<int16_t, E>>(
+        Data, FRE.Info.getOffsetCount(), Offset, FRE.Offsets);
+  case sframe::FREOffset::B4:
+    return readArray<sframe::detail::packed<int32_t, E>>(
+        Data, FRE.Info.getOffsetCount(), Offset, FRE.Offsets);
   }
+  return createError(formatv("unsupported FRE offset size {0} at offset {1:x+}",
+                             static_cast<unsigned>(FRE.Info.getOffsetSize()),
+                             Offset));
+}
 
-  return Result;
+template <endianness E> Error SFrameParser<E>::FallibleFREIterator::inc() {
+  if (++Idx == Size)
+    return Error::success();
+
+  switch (FREType) {
+  case sframe::FREType::Addr1:
+    return readFRE<uint8_t, E>(Data, Offset, FRE);
+  case sframe::FREType::Addr2:
+    return readFRE<uint16_t, E>(Data, Offset, FRE);
+  case sframe::FREType::Addr4:
+    return readFRE<uint32_t, E>(Data, Offset, FRE);
+  }
+  return createError(formatv("unsupported FRE type {0} at offset {1:x+}",
+                             static_cast<unsigned>(FREType), Offset));
+}
+
+template <endianness E>
+iterator_range<typename SFrameParser<E>::fre_iterator>
+SFrameParser<E>::fres(const sframe::FuncDescEntry<E> &FDE, Error &Err) const {
+  uint64_t Offset = getFREBase() + FDE.StartFREOff;
+  fre_iterator BeforeBegin = make_fallible_itr(
+      FallibleFREIterator(Data, FDE.Info.getFREType(), -1, FDE.NumFREs, Offset),
+      Err);
+  fre_iterator End = make_fallible_end(
+      FallibleFREIterator(Data, FDE.Info.getFREType(), FDE.NumFREs, FDE.NumFREs,
+                          /*Offset=*/0));
+  return {++BeforeBegin, End};
+}
+
+static std::optional<int32_t> getOffset(ArrayRef<int32_t> Offsets, size_t Idx) {
+  if (Offsets.size() > Idx)
+    return Offsets[Idx];
+  return std::nullopt;
+}
+
+// The interpretation of offsets is ABI-specific. The implementation of this and
+// the following functions may need to be adjusted when adding support for a new
+// ABI.
+template <endianness E>
+std::optional<int32_t>
+SFrameParser<E>::getCFAOffset(const FrameRowEntry &FRE) const {
+  return getOffset(FRE.Offsets, 0);
+}
+
+template <endianness E>
+std::optional<int32_t>
+SFrameParser<E>::getRAOffset(const FrameRowEntry &FRE) const {
+  if (usesFixedRAOffset())
+    return Header.CFAFixedRAOffset;
+  return getOffset(FRE.Offsets, 1);
+}
+
+template <endianness E>
+std::optional<int32_t>
+SFrameParser<E>::getFPOffset(const FrameRowEntry &FRE) const {
+  if (usesFixedFPOffset())
+    return Header.CFAFixedFPOffset;
+  return getOffset(FRE.Offsets, usesFixedRAOffset() ? 1 : 2);
+}
+
+template <endianness E>
+ArrayRef<int32_t>
+SFrameParser<E>::getExtraOffsets(const FrameRowEntry &FRE) const {
+  size_t UsedOffsets = 1; // CFA
+  if (!usesFixedRAOffset())
+    ++UsedOffsets;
+  if (!usesFixedFPOffset())
+    ++UsedOffsets;
+  if (FRE.Offsets.size() > UsedOffsets)
+    return ArrayRef<int32_t>(FRE.Offsets).drop_front(UsedOffsets);
+  return {};
 }
 
 template class LLVM_EXPORT_TEMPLATE llvm::object::SFrameParser<endianness::big>;
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index f810368a84940..b7edeea082762 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -124,7 +124,6 @@
 #include "llvm/CodeGen/MachineCopyPropagation.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
-#include "llvm/CodeGen/MachineInstrBundle.h"
 #include "llvm/CodeGen/MachineLICM.h"
 #include "llvm/CodeGen/MachineLateInstrsCleanup.h"
 #include "llvm/CodeGen/MachinePassManager.h"
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index cdf4412c6477a..fc2577e6ada5d 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -519,7 +519,7 @@ Error InstrProfSymtab::create(SectionRef &Section) {
   return Error::success();
 }
 
-StringRef InstrProfSymtab::getFuncName(uint64_t Pointer, size_t Size) {
+StringRef InstrProfSymtab::getFuncName(uint64_t Pointer, size_t Size) const {
   if (Pointer < Address)
     return StringRef();
   auto Offset = Pointer - Address;
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 542572975dc89..e1c6315853b3b 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -684,13 +684,13 @@ Error InstrProfSymtab::addFuncWithName(Function &F, StringRef PGOFuncName,
   return Error::success();
 }
 
-uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) {
+uint64_t InstrProfSymtab::getVTableHashFromAddress(uint64_t Address) const {
   // Given a runtime address, look up the hash value in the interval map, and
   // fallback to value 0 if a hash value is not found.
   return VTableAddrMap.lookup(Address, 0);
 }
 
-uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) {
+uint64_t InstrProfSymtab::getFunctionHashFromAddress(uint64_t Address) const {
   finalizeSymtab();
   auto It = partition_point(AddrToMD5Map, [=](std::pair<uint64_t, uint64_t> A) {
     return A.first < Address;
@@ -1607,7 +1607,7 @@ void OverlapStats::dump(raw_fd_ostream &OS) const {
   const char *EntryName =
       (Level == ProgramLevel ? "functions" : "edge counters");
   if (Level == ProgramLevel) {
-    OS << "Profile overlap infomation for base_profile: " << *BaseFilename
+    OS << "Profile overlap information for base_profile: " << *BaseFilename
        << " and test_profile: " << *TestFilename << "\nProgram level:\n";
   } else {
     OS << "Function level:\n"
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 7ca26aa138012..a3473514d4637 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -13,7 +13,6 @@
 
 #include "llvm/ProfileData/InstrProfWriter.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/ProfileSummary.h"
 #include "llvm/ProfileData/DataAccessProf.h"
@@ -331,61 +330,34 @@ void InstrProfWriter::addDataAccessProfData(
   DataAccessProfileData = std::move(DataAccessProfDataIn);
 }
 
-void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
-  assert(Trace.FunctionNameRefs.size() <= MaxTemporalProfTraceLength);
-  assert(!Trace.FunctionNameRefs.empty());
-  if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
-    // Simply append the trace if we have not yet hit our reservoir size limit.
-    TemporalProfTraces.push_back(std::move(Trace));
-  } else {
-    // Otherwise, replace a random trace in the stream.
-    std::uniform_int_distribution<uint64_t> Distribution(
-        0, TemporalProfTraceStreamSize);
-    uint64_t RandomIndex = Distribution(RNG);
-    if (RandomIndex < TemporalProfTraces.size())
-      TemporalProfTraces[RandomIndex] = std::move(Trace);
-  }
-  ++TemporalProfTraceStreamSize;
-}
-
 void InstrProfWriter::addTemporalProfileTraces(
     SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
+  if (TemporalProfTraces.size() > TemporalProfTraceReservoirSize)
+    TemporalProfTraces.truncate(TemporalProfTraceReservoirSize);
   for (auto &Trace : SrcTraces)
     if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
       Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
   llvm::erase_if(SrcTraces, [](auto &T) { return T.FunctionNameRefs.empty(); });
-  // Assume that the source has the same reservoir size as the destination to
-  // avoid needing to record it in the indexed profile format.
-  bool IsDestSampled =
-      (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
-  bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
-  if (!IsDestSampled && IsSrcSampled) {
-    // If one of the traces are sampled, ensure that it belongs to Dest.
-    std::swap(TemporalProfTraces, SrcTraces);
-    std::swap(TemporalProfTraceStreamSize, SrcStreamSize);
-    std::swap(IsDestSampled, IsSrcSampled);
-  }
-  if (!IsSrcSampled) {
-    // If the source stream is not sampled, we add each source trace normally.
-    for (auto &Trace : SrcTraces)
-      addTemporalProfileTrace(std::move(Trace));
+  // If there are no source traces, it is probably because
+  // --temporal-profile-max-trace-length=0 was set to deliberately remove all
+  // traces. In that case, we do not want to increase the stream size
+  if (SrcTraces.empty())
     return;
-  }
-  // Otherwise, we find the traces that would have been removed if we added
-  // the whole source stream.
-  SmallSetVector<uint64_t, 8> IndicesToReplace;
-  for (uint64_t I = 0; I < SrcStreamSize; I++) {
-    std::uniform_int_distribution<uint64_t> Distribution(
-        0, TemporalProfTraceStreamSize);
+  // Add traces until our reservoir is full or we run out of source traces
+  auto SrcTraceIt = SrcTraces.begin();
+  while (TemporalProfTraces.size() < TemporalProfTraceReservoirSize &&
+         SrcTraceIt < SrcTraces.end())
+    TemporalProfTraces.push_back(*SrcTraceIt++);
+  // Our reservoir is full, we need to sample the source stream
+  llvm::shuffle(SrcTraceIt, SrcTraces.end(), RNG);
+  for (uint64_t I = TemporalProfTraces.size();
+       I < SrcStreamSize && SrcTraceIt < SrcTraces.end(); I++) {
+    std::uniform_int_distribution<uint64_t> Distribution(0, I);
     uint64_t RandomIndex = Distribution(RNG);
     if (RandomIndex < TemporalProfTraces.size())
-      IndicesToReplace.insert(RandomIndex);
-    ++TemporalProfTraceStreamSize;
+      TemporalProfTraces[RandomIndex] = *SrcTraceIt++;
   }
-  // Then we insert a random sample of the source traces.
-  llvm::shuffle(SrcTraces.begin(), SrcTraces.end(), RNG);
-  for (const auto &[Index, Trace] : llvm::zip(IndicesToReplace, SrcTraces))
-    TemporalProfTraces[Index] = std::move(Trace);
+  TemporalProfTraceStreamSize += SrcStreamSize;
 }
 
 void InstrProfWriter::mergeRecordsFromWriter(InstrProfWriter &&IPW,
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index d147222fe2ce6..12769a391286c 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -375,16 +375,23 @@ std::error_code SampleProfileReaderText::readImpl() {
       StringRef FName;
       DenseMap<StringRef, uint64_t> TargetCountMap;
       uint32_t Depth, LineOffset, Discriminator;
-      LineType LineTy;
+      LineType LineTy = LineType::BodyProfile;
       uint64_t FunctionHash = 0;
       uint32_t Attributes = 0;
       bool IsFlat = false;
       if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset,
                      Discriminator, FName, TargetCountMap, FunctionHash,
                      Attributes, IsFlat)) {
-        reportError(LineIt.line_number(),
-                    "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
-                        *LineIt);
+        switch (LineTy) {
+        case LineType::Metadata:
+          reportError(LineIt.line_number(),
+                      "Cannot parse metadata: " + *LineIt);
+          break;
+        default:
+          reportError(LineIt.line_number(),
+                      "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " +
+                          *LineIt);
+        }
         return sampleprof_error::malformed;
       }
       if (LineTy != LineType::Metadata && Depth == DepthMetadata) {
diff --git a/llvm/lib/SandboxIR/Context.cpp b/llvm/lib/SandboxIR/Context.cpp
index fe34037d7dc49..70ac68abbcb0d 100644
--- a/llvm/lib/SandboxIR/Context.cpp
+++ b/llvm/lib/SandboxIR/Context.cpp
@@ -256,6 +256,7 @@ Value *Context::getOrCreateValueInternal(llvm::Value *LLVMV, llvm::User *U) {
     case llvm::Instruction::FPToUI:
     case llvm::Instruction::FPToSI:
     case llvm::Instruction::FPExt:
+    case llvm::Instruction::PtrToAddr:
     case llvm::Instruction::PtrToInt:
     case llvm::Instruction::IntToPtr:
     case llvm::Instruction::SIToFP:
diff --git a/llvm/lib/SandboxIR/Instruction.cpp b/llvm/lib/SandboxIR/Instruction.cpp
index 956047cf87b6b..1a81d185acf76 100644
--- a/llvm/lib/SandboxIR/Instruction.cpp
+++ b/llvm/lib/SandboxIR/Instruction.cpp
@@ -1007,6 +1007,9 @@ static llvm::Instruction::CastOps getLLVMCastOp(Instruction::Opcode Opc) {
     return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPToSI);
   case Instruction::Opcode::FPExt:
     return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::FPExt);
+  case Instruction::Opcode::PtrToAddr:
+    return static_cast<llvm::Instruction::CastOps>(
+        llvm::Instruction::PtrToAddr);
   case Instruction::Opcode::PtrToInt:
     return static_cast<llvm::Instruction::CastOps>(llvm::Instruction::PtrToInt);
   case Instruction::Opcode::IntToPtr:
diff --git a/llvm/lib/Support/APFloat.cpp b/llvm/lib/Support/APFloat.cpp
index 46084c5b7fb92..d14abb4bd05b5 100644
--- a/llvm/lib/Support/APFloat.cpp
+++ b/llvm/lib/Support/APFloat.cpp
@@ -2927,51 +2927,6 @@ APFloat::opStatus IEEEFloat::convertFromAPInt(const APInt &Val, bool isSigned,
   return convertFromUnsignedParts(api.getRawData(), partCount, rounding_mode);
 }
 
-/* Convert a two's complement integer SRC to a floating point number,
-   rounding according to ROUNDING_MODE.  ISSIGNED is true if the
-   integer is signed, in which case it must be sign-extended.  */
-APFloat::opStatus
-IEEEFloat::convertFromSignExtendedInteger(const integerPart *src,
-                                          unsigned int srcCount, bool isSigned,
-                                          roundingMode rounding_mode) {
-  opStatus status;
-
-  if (isSigned &&
-      APInt::tcExtractBit(src, srcCount * integerPartWidth - 1)) {
-    integerPart *copy;
-
-    /* If we're signed and negative negate a copy.  */
-    sign = true;
-    copy = new integerPart[srcCount];
-    APInt::tcAssign(copy, src, srcCount);
-    APInt::tcNegate(copy, srcCount);
-    status = convertFromUnsignedParts(copy, srcCount, rounding_mode);
-    delete [] copy;
-  } else {
-    sign = false;
-    status = convertFromUnsignedParts(src, srcCount, rounding_mode);
-  }
-
-  return status;
-}
-
-/* FIXME: should this just take a const APInt reference?  */
-APFloat::opStatus
-IEEEFloat::convertFromZeroExtendedInteger(const integerPart *parts,
-                                          unsigned int width, bool isSigned,
-                                          roundingMode rounding_mode) {
-  unsigned int partCount = partCountForBits(width);
-  APInt api = APInt(width, ArrayRef(parts, partCount));
-
-  sign = false;
-  if (isSigned && APInt::tcExtractBit(parts, width - 1)) {
-    sign = true;
-    api = -api;
-  }
-
-  return convertFromUnsignedParts(api.getRawData(), partCount, rounding_mode);
-}
-
 Expected<APFloat::opStatus>
 IEEEFloat::convertFromHexadecimalString(StringRef s,
                                         roundingMode rounding_mode) {
@@ -4575,35 +4530,6 @@ void IEEEFloat::toString(SmallVectorImpl<char> &Str, unsigned FormatPrecision,
 
 }
 
-bool IEEEFloat::getExactInverse(APFloat *inv) const {
-  // Special floats and denormals have no exact inverse.
-  if (!isFiniteNonZero())
-    return false;
-
-  // Check that the number is a power of two by making sure that only the
-  // integer bit is set in the significand.
-  if (significandLSB() != semantics->precision - 1)
-    return false;
-
-  // Get the inverse.
-  IEEEFloat reciprocal(*semantics, 1ULL);
-  if (reciprocal.divide(*this, rmNearestTiesToEven) != opOK)
-    return false;
-
-  // Avoid multiplication with a denormal, it is not safe on all platforms and
-  // may be slower than a normal division.
-  if (reciprocal.isDenormal())
-    return false;
-
-  assert(reciprocal.isFiniteNonZero() &&
-         reciprocal.significandLSB() == reciprocal.semantics->precision - 1);
-
-  if (inv)
-    *inv = APFloat(reciprocal, *semantics);
-
-  return true;
-}
-
 int IEEEFloat::getExactLog2Abs() const {
   if (!isFinite() || isZero())
     return INT_MIN;
@@ -4949,6 +4875,21 @@ DoubleAPFloat &DoubleAPFloat::operator=(const DoubleAPFloat &RHS) {
   return *this;
 }
 
+// Returns a result such that:
+// 1. abs(Lo) <= ulp(Hi)/2
+// 2. Hi == RTNE(Hi + Lo)
+// 3. Hi + Lo == X + Y
+//
+// Requires that log2(X) >= log2(Y).
+static std::pair<APFloat, APFloat> fastTwoSum(APFloat X, APFloat Y) {
+  if (!X.isFinite())
+    return {X, APFloat::getZero(X.getSemantics(), /*Negative=*/false)};
+  APFloat Hi = X + Y;
+  APFloat Delta = Hi - X;
+  APFloat Lo = Y - Delta;
+  return {Hi, Lo};
+}
+
 // Implement addition, subtraction, multiplication and division based on:
 // "Software for Doubled-Precision Floating-Point Computations",
 // by Seppo Linnainmaa, ACM TOMS vol 7 no 3, September 1981, pages 272-283.
@@ -5218,10 +5159,78 @@ DoubleAPFloat::fusedMultiplyAdd(const DoubleAPFloat &Multiplicand,
 
 APFloat::opStatus DoubleAPFloat::roundToIntegral(APFloat::roundingMode RM) {
   assert(Semantics == &semPPCDoubleDouble && "Unexpected Semantics");
-  APFloat Tmp(semPPCDoubleDoubleLegacy, bitcastToAPInt());
-  auto Ret = Tmp.roundToIntegral(RM);
-  *this = DoubleAPFloat(semPPCDoubleDouble, Tmp.bitcastToAPInt());
-  return Ret;
+  const APFloat &Hi = getFirst();
+  const APFloat &Lo = getSecond();
+
+  APFloat RoundedHi = Hi;
+  const opStatus HiStatus = RoundedHi.roundToIntegral(RM);
+
+  // We can reduce the problem to just the high part if the input:
+  // 1. Represents a non-finite value.
+  // 2. Has a component which is zero.
+  if (!Hi.isFiniteNonZero() || Lo.isZero()) {
+    Floats[0] = std::move(RoundedHi);
+    Floats[1].makeZero(/*Neg=*/false);
+    return HiStatus;
+  }
+
+  // Adjust `Rounded` in the direction of `TieBreaker` if `ToRound` was at a
+  // halfway point.
+  auto RoundToNearestHelper = [](APFloat ToRound, APFloat Rounded,
+                                 APFloat TieBreaker) {
+    // RoundingError tells us which direction we rounded:
+    //   - RoundingError > 0: we rounded up.
+    //   - RoundingError < 0: we rounded down.
+    // Sterbenz' lemma ensures that RoundingError is exact.
+    const APFloat RoundingError = Rounded - ToRound;
+    if (TieBreaker.isNonZero() &&
+        TieBreaker.isNegative() != RoundingError.isNegative() &&
+        abs(RoundingError).isExactlyValue(0.5))
+      Rounded.add(
+          APFloat::getOne(Rounded.getSemantics(), TieBreaker.isNegative()),
+          rmNearestTiesToEven);
+    return Rounded;
+  };
+
+  // Case 1: Hi is not an integer.
+  // Special cases are for rounding modes that are sensitive to ties.
+  if (RoundedHi != Hi) {
+    // We need to consider the case where Hi was between two integers and the
+    // rounding mode broke the tie when, in fact, Lo may have had a different
+    // sign than Hi.
+    if (RM == rmNearestTiesToAway || RM == rmNearestTiesToEven)
+      RoundedHi = RoundToNearestHelper(Hi, RoundedHi, Lo);
+
+    Floats[0] = std::move(RoundedHi);
+    Floats[1].makeZero(/*Neg=*/false);
+    return HiStatus;
+  }
+
+  // Case 2: Hi is an integer.
+  // Special cases are for rounding modes which are rounding towards or away from zero.
+  RoundingMode LoRoundingMode;
+  if (RM == rmTowardZero)
+    // When our input is positive, we want the Lo component rounded toward
+    // negative infinity to get the smallest result magnitude. Likewise,
+    // negative inputs want the Lo component rounded toward positive infinity.
+    LoRoundingMode = isNegative() ? rmTowardPositive : rmTowardNegative;
+  else
+    LoRoundingMode = RM;
+
+  APFloat RoundedLo = Lo;
+  const opStatus LoStatus = RoundedLo.roundToIntegral(LoRoundingMode);
+  if (LoRoundingMode == rmNearestTiesToAway)
+    // We need to consider the case where Lo was between two integers and the
+    // rounding mode broke the tie when, in fact, Hi may have had a different
+    // sign than Lo.
+    RoundedLo = RoundToNearestHelper(Lo, RoundedLo, Hi);
+
+  // We must ensure that the final result has no overlap between the two APFloat values.
+  std::tie(RoundedHi, RoundedLo) = fastTwoSum(RoundedHi, RoundedLo);
+
+  Floats[0] = std::move(RoundedHi);
+  Floats[1] = std::move(RoundedLo);
+  return LoStatus;
 }
 
 void DoubleAPFloat::changeSign() {
@@ -5231,23 +5240,56 @@ void DoubleAPFloat::changeSign() {
 
 APFloat::cmpResult
 DoubleAPFloat::compareAbsoluteValue(const DoubleAPFloat &RHS) const {
-  auto Result = Floats[0].compareAbsoluteValue(RHS.Floats[0]);
-  if (Result != cmpEqual)
-    return Result;
-  Result = Floats[1].compareAbsoluteValue(RHS.Floats[1]);
-  if (Result == cmpLessThan || Result == cmpGreaterThan) {
-    auto Against = Floats[0].isNegative() ^ Floats[1].isNegative();
-    auto RHSAgainst = RHS.Floats[0].isNegative() ^ RHS.Floats[1].isNegative();
-    if (Against && !RHSAgainst)
-      return cmpLessThan;
-    if (!Against && RHSAgainst)
+  // Compare absolute values of the high parts.
+  const cmpResult HiPartCmp = Floats[0].compareAbsoluteValue(RHS.Floats[0]);
+  if (HiPartCmp != cmpEqual)
+    return HiPartCmp;
+
+  // Zero, regardless of sign, is equal.
+  if (Floats[1].isZero() && RHS.Floats[1].isZero())
+    return cmpEqual;
+
+  // At this point, |this->Hi| == |RHS.Hi|.
+  // The magnitude is |Hi+Lo| which is Hi+|Lo| if signs of Hi and Lo are the
+  // same, and Hi-|Lo| if signs are different.
+  const bool ThisIsSubtractive =
+      Floats[0].isNegative() != Floats[1].isNegative();
+  const bool RHSIsSubtractive =
+      RHS.Floats[0].isNegative() != RHS.Floats[1].isNegative();
+
+  // Case 1: The low part of 'this' is zero.
+  if (Floats[1].isZero())
+    // We are comparing |Hi| vs. |Hi| ± |RHS.Lo|.
+    // If RHS is subtractive, its magnitude is smaller.
+    // If RHS is additive, its magnitude is larger.
+    return RHSIsSubtractive ? cmpGreaterThan : cmpLessThan;
+
+  // Case 2: The low part of 'RHS' is zero (and we know 'this' is not).
+  if (RHS.Floats[1].isZero())
+    // We are comparing |Hi| ± |This.Lo| vs. |Hi|.
+    // If 'this' is subtractive, its magnitude is smaller.
+    // If 'this' is additive, its magnitude is larger.
+    return ThisIsSubtractive ? cmpLessThan : cmpGreaterThan;
+
+  // If their natures differ, the additive one is larger.
+  if (ThisIsSubtractive != RHSIsSubtractive)
+    return ThisIsSubtractive ? cmpLessThan : cmpGreaterThan;
+
+  // Case 3: Both are additive (Hi+|Lo|) or both are subtractive (Hi-|Lo|).
+  // The comparison now depends on the magnitude of the low parts.
+  const cmpResult LoPartCmp = Floats[1].compareAbsoluteValue(RHS.Floats[1]);
+
+  if (ThisIsSubtractive) {
+    // Both are subtractive (Hi-|Lo|), so the comparison of |Lo| is inverted.
+    if (LoPartCmp == cmpLessThan)
       return cmpGreaterThan;
-    if (!Against && !RHSAgainst)
-      return Result;
-    if (Against && RHSAgainst)
-      return (cmpResult)(cmpLessThan + cmpGreaterThan - Result);
+    if (LoPartCmp == cmpGreaterThan)
+      return cmpLessThan;
   }
-  return Result;
+
+  // If additive, the comparison of |Lo| is direct.
+  // If equal, they are equal.
+  return LoPartCmp;
 }
 
 APFloat::fltCategory DoubleAPFloat::getCategory() const {
@@ -5436,45 +5478,283 @@ APFloat::opStatus DoubleAPFloat::next(bool nextDown) {
   return opOK;
 }
 
+APFloat::opStatus DoubleAPFloat::convertToSignExtendedInteger(
+    MutableArrayRef<integerPart> Input, unsigned int Width, bool IsSigned,
+    roundingMode RM, bool *IsExact) const {
+  assert(Semantics == &semPPCDoubleDouble && "Unexpected Semantics");
+
+  // If Hi is not finite, or Lo is zero, the value is entirely represented
+  // by Hi. Delegate to the simpler single-APFloat conversion.
+  if (!getFirst().isFiniteNonZero() || getSecond().isZero())
+    return getFirst().convertToInteger(Input, Width, IsSigned, RM, IsExact);
+
+  // First, round the full double-double value to an integral value. This
+  // simplifies the rest of the function, as we no longer need to consider
+  // fractional parts.
+  *IsExact = false;
+  DoubleAPFloat Integral = *this;
+  const opStatus RoundStatus = Integral.roundToIntegral(RM);
+  if (RoundStatus == opInvalidOp)
+    return opInvalidOp;
+  const APFloat &IntegralHi = Integral.getFirst();
+  const APFloat &IntegralLo = Integral.getSecond();
+
+  // If rounding results in either component being zero, the sum is trivial.
+  // Delegate to the simpler single-APFloat conversion.
+  bool HiIsExact;
+  if (IntegralHi.isZero() || IntegralLo.isZero()) {
+    const opStatus HiStatus =
+        IntegralHi.convertToInteger(Input, Width, IsSigned, RM, &HiIsExact);
+    // The conversion from an integer-valued float to an APInt may fail if the
+    // result would be out of range.  Regardless, taking this path is only
+    // possible if rounding occurred during the initial `roundToIntegral`.
+    return HiStatus == opOK ? opInexact : HiStatus;
+  }
+
+  // A negative number cannot be represented by an unsigned integer.
+  // Since a double-double is canonical, if Hi is negative, the sum is negative.
+  if (!IsSigned && IntegralHi.isNegative())
+    return opInvalidOp;
+
+  // Handle the special boundary case where |Hi| is exactly the power of two
+  // that marks the edge of the integer's range (e.g., 2^63 for int64_t). In
+  // this situation, Hi itself won't fit, but the sum Hi + Lo might.
+  // `PositiveOverflowWidth` is the bit number for this boundary (N-1 for
+  // signed, N for unsigned).
+  bool LoIsExact;
+  const int HiExactLog2 = IntegralHi.getExactLog2Abs();
+  const unsigned PositiveOverflowWidth = IsSigned ? Width - 1 : Width;
+  if (HiExactLog2 >= 0 &&
+      static_cast<unsigned>(HiExactLog2) == PositiveOverflowWidth) {
+    // If Hi and Lo have the same sign, |Hi + Lo| > |Hi|, so the sum is
+    // guaranteed to overflow. E.g., for uint128_t, (2^128, 1) overflows.
+    if (IntegralHi.isNegative() == IntegralLo.isNegative())
+      return opInvalidOp;
+
+    // If the signs differ, the sum will fit. We can compute the result using
+    // properties of two's complement arithmetic without a wide intermediate
+    // integer. E.g., for uint128_t, (2^128, -1) should be 2^128 - 1.
+    const opStatus LoStatus = IntegralLo.convertToInteger(
+        Input, Width, /*IsSigned=*/true, RM, &LoIsExact);
+    if (LoStatus == opInvalidOp)
+      return opInvalidOp;
+
+    // Adjust the bit pattern of Lo to account for Hi's value:
+    //  - For unsigned (Hi=2^Width): `2^Width + Lo` in `Width`-bit
+    //    arithmetic is equivalent to just `Lo`. The conversion of `Lo` above
+    //    already produced the correct final bit pattern.
+    //  - For signed (Hi=2^(Width-1)): The sum `2^(Width-1) + Lo` (where Lo<0)
+    //    can be computed by taking the two's complement pattern for `Lo` and
+    //    clearing the sign bit.
+    if (IsSigned && !IntegralHi.isNegative())
+      APInt::tcClearBit(Input.data(), PositiveOverflowWidth);
+    *IsExact = RoundStatus == opOK;
+    return RoundStatus;
+  }
+
+  // Convert Hi into an integer.  This may not fit but that is OK: we know that
+  // Hi + Lo would not fit either in this situation.
+  const opStatus HiStatus = IntegralHi.convertToInteger(
+      Input, Width, IsSigned, rmTowardZero, &HiIsExact);
+  if (HiStatus == opInvalidOp)
+    return HiStatus;
+
+  // Convert Lo into a temporary integer of the same width.
+  APSInt LoResult{Width, /*isUnsigned=*/!IsSigned};
+  const opStatus LoStatus =
+      IntegralLo.convertToInteger(LoResult, rmTowardZero, &LoIsExact);
+  if (LoStatus == opInvalidOp)
+    return LoStatus;
+
+  // Add Lo to Hi. This addition is guaranteed not to overflow because of the
+  // double-double canonicalization rule (`|Lo| <= ulp(Hi)/2`). The only case
+  // where the sum could cross the integer type's boundary is when Hi is a
+  // power of two, which is handled by the special case block above.
+  APInt::tcAdd(Input.data(), LoResult.getRawData(), /*carry=*/0, Input.size());
+
+  *IsExact = RoundStatus == opOK;
+  return RoundStatus;
+}
+
 APFloat::opStatus
 DoubleAPFloat::convertToInteger(MutableArrayRef<integerPart> Input,
                                 unsigned int Width, bool IsSigned,
                                 roundingMode RM, bool *IsExact) const {
-  assert(Semantics == &semPPCDoubleDouble && "Unexpected Semantics");
-  return APFloat(semPPCDoubleDoubleLegacy, bitcastToAPInt())
-      .convertToInteger(Input, Width, IsSigned, RM, IsExact);
-}
+  opStatus FS =
+      convertToSignExtendedInteger(Input, Width, IsSigned, RM, IsExact);
+
+  if (FS == opInvalidOp) {
+    const unsigned DstPartsCount = partCountForBits(Width);
+    assert(DstPartsCount <= Input.size() && "Integer too big");
+
+    unsigned Bits;
+    if (getCategory() == fcNaN)
+      Bits = 0;
+    else if (isNegative())
+      Bits = IsSigned;
+    else
+      Bits = Width - IsSigned;
 
-APFloat::opStatus DoubleAPFloat::convertFromAPInt(const APInt &Input,
-                                                  bool IsSigned,
-                                                  roundingMode RM) {
-  assert(Semantics == &semPPCDoubleDouble && "Unexpected Semantics");
-  APFloat Tmp(semPPCDoubleDoubleLegacy);
-  auto Ret = Tmp.convertFromAPInt(Input, IsSigned, RM);
-  *this = DoubleAPFloat(semPPCDoubleDouble, Tmp.bitcastToAPInt());
-  return Ret;
+    tcSetLeastSignificantBits(Input.data(), DstPartsCount, Bits);
+    if (isNegative() && IsSigned)
+      APInt::tcShiftLeft(Input.data(), DstPartsCount, Width - 1);
+  }
+
+  return FS;
 }
 
-APFloat::opStatus
-DoubleAPFloat::convertFromSignExtendedInteger(const integerPart *Input,
-                                              unsigned int InputSize,
-                                              bool IsSigned, roundingMode RM) {
-  assert(Semantics == &semPPCDoubleDouble && "Unexpected Semantics");
-  APFloat Tmp(semPPCDoubleDoubleLegacy);
-  auto Ret = Tmp.convertFromSignExtendedInteger(Input, InputSize, IsSigned, RM);
-  *this = DoubleAPFloat(semPPCDoubleDouble, Tmp.bitcastToAPInt());
-  return Ret;
+APFloat::opStatus DoubleAPFloat::handleOverflow(roundingMode RM) {
+  switch (RM) {
+  case APFloat::rmTowardZero:
+    makeLargest(/*Neg=*/isNegative());
+    break;
+  case APFloat::rmTowardNegative:
+    if (isNegative())
+      makeInf(/*Neg=*/true);
+    else
+      makeLargest(/*Neg=*/false);
+    break;
+  case APFloat::rmTowardPositive:
+    if (isNegative())
+      makeLargest(/*Neg=*/true);
+    else
+      makeInf(/*Neg=*/false);
+    break;
+  case APFloat::rmNearestTiesToAway:
+  case APFloat::rmNearestTiesToEven:
+    makeInf(/*Neg=*/isNegative());
+    break;
+  default:
+    llvm_unreachable("Invalid rounding mode found");
+  }
+  opStatus S = opInexact;
+  if (!getFirst().isFinite())
+    S = static_cast<opStatus>(S | opOverflow);
+  return S;
+}
+
+APFloat::opStatus DoubleAPFloat::convertFromUnsignedParts(
+    const integerPart *Src, unsigned int SrcCount, roundingMode RM) {
+  // Find the most significant bit of the source integer. APInt::tcMSB returns
+  // UINT_MAX for a zero value.
+  const unsigned SrcMSB = APInt::tcMSB(Src, SrcCount);
+  if (SrcMSB == UINT_MAX) {
+    // The source integer is 0.
+    makeZero(/*Neg=*/false);
+    return opOK;
+  }
+
+  // Create a minimally-sized APInt to represent the source value.
+  const unsigned SrcBitWidth = SrcMSB + 1;
+  APSInt SrcInt{APInt{/*numBits=*/SrcBitWidth,
+                      /*numWords=*/SrcCount, Src},
+                /*isUnsigned=*/true};
+
+  // Stage 1: Initial Approximation.
+  // Convert the source integer SrcInt to the Hi part of the DoubleAPFloat.
+  // We use round-to-nearest because it minimizes the initial error, which is
+  // crucial for the subsequent steps.
+  APFloat Hi{getFirst().getSemantics()};
+  Hi.convertFromAPInt(SrcInt, /*IsSigned=*/false, rmNearestTiesToEven);
+
+  // If the first approximation already overflows, the number is too large.
+  // NOTE: The underlying semantics are *more* conservative when choosing to
+  // overflow because their notion of ULP is much larger. As such, it is always
+  // safe to overflow at the DoubleAPFloat level if the APFloat overflows.
+  if (!Hi.isFinite())
+    return handleOverflow(RM);
+
+  // Stage 2: Exact Error Calculation.
+  // Calculate the exact error of the first approximation: Error = SrcInt - Hi.
+  // This is done by converting Hi back to an integer and subtracting it from
+  // the original source.
+  bool HiAsIntIsExact;
+  // Create an integer representation of Hi. Its width is determined by the
+  // exponent of Hi, ensuring it's just large enough. This width can exceed
+  // SrcBitWidth if the conversion to Hi rounded up to a power of two.
+  // accurately when converted back to an integer.
+  APSInt HiAsInt{static_cast<uint32_t>(ilogb(Hi) + 1), /*isUnsigned=*/true};
+  Hi.convertToInteger(HiAsInt, rmNearestTiesToEven, &HiAsIntIsExact);
+  const APInt Error = SrcInt.zext(HiAsInt.getBitWidth()) - HiAsInt;
+
+  // Stage 3: Error Approximation and Rounding.
+  // Convert the integer error into the Lo part of the DoubleAPFloat. This step
+  // captures the remainder of the original number. The rounding mode for this
+  // conversion (LoRM) may need to be adjusted from the user-requested RM to
+  // ensure the final sum (Hi + Lo) rounds correctly.
+  roundingMode LoRM = RM;
+  // Adjustments are only necessary when the initial approximation Hi was an
+  // overestimate, making the Error negative.
+  if (Error.isNegative()) {
+    if (RM == rmNearestTiesToAway) {
+      // For rmNearestTiesToAway, a tie should round away from zero. Since
+      // SrcInt is positive, this means rounding toward +infinity.
+      // A standard conversion of a negative Error would round ties toward
+      // -infinity, causing the final sum Hi + Lo to be smaller. To
+      // counteract this, we detect the tie case and override the rounding
+      // mode for Lo to rmTowardPositive.
+      const unsigned ErrorActiveBits = Error.getSignificantBits() - 1;
+      const unsigned LoPrecision = getSecond().getSemantics().precision;
+      if (ErrorActiveBits > LoPrecision) {
+        const unsigned RoundingBoundary = ErrorActiveBits - LoPrecision;
+        // A tie occurs when the bits to be truncated are of the form 100...0.
+        // This is detected by checking if the number of trailing zeros is
+        // exactly one less than the number of bits being truncated.
+        if (Error.countTrailingZeros() == RoundingBoundary - 1)
+          LoRM = rmTowardPositive;
+      }
+    } else if (RM == rmTowardZero) {
+      // For rmTowardZero, the final positive result must be truncated (rounded
+      // down). When Hi is an overestimate, Error is negative. A standard
+      // rmTowardZero conversion of Error would make it *less* negative,
+      // effectively rounding the final sum Hi + Lo *up*. To ensure the sum
+      // rounds down correctly, we force Lo to round toward -infinity.
+      LoRM = rmTowardNegative;
+    }
+  }
+
+  APFloat Lo{getSecond().getSemantics()};
+  opStatus Status = Lo.convertFromAPInt(Error, /*IsSigned=*/true, LoRM);
+
+  // Renormalize the pair (Hi, Lo) into a canonical DoubleAPFloat form where the
+  // components do not overlap. fastTwoSum performs this operation.
+  std::tie(Hi, Lo) = fastTwoSum(Hi, Lo);
+  Floats[0] = std::move(Hi);
+  Floats[1] = std::move(Lo);
+
+  // A final check for overflow is needed because fastTwoSum can cause a
+  // carry-out from Lo that pushes Hi to infinity.
+  if (!getFirst().isFinite())
+    return handleOverflow(RM);
+
+  // The largest DoubleAPFloat must be canonical. Values which are larger are
+  // not canonical and are equivalent to overflow.
+  if (getFirst().isFiniteNonZero() && Floats[0].isLargest()) {
+    DoubleAPFloat Largest{*Semantics};
+    Largest.makeLargest(/*Neg=*/false);
+    if (compare(Largest) == APFloat::cmpGreaterThan)
+      return handleOverflow(RM);
+  }
+
+  // The final status of the operation is determined by the conversion of the
+  // error term. If Lo could represent Error exactly, the entire conversion
+  // is exact. Otherwise, it's inexact.
+  return Status;
 }
 
-APFloat::opStatus
-DoubleAPFloat::convertFromZeroExtendedInteger(const integerPart *Input,
-                                              unsigned int InputSize,
-                                              bool IsSigned, roundingMode RM) {
-  assert(Semantics == &semPPCDoubleDouble && "Unexpected Semantics");
-  APFloat Tmp(semPPCDoubleDoubleLegacy);
-  auto Ret = Tmp.convertFromZeroExtendedInteger(Input, InputSize, IsSigned, RM);
-  *this = DoubleAPFloat(semPPCDoubleDouble, Tmp.bitcastToAPInt());
-  return Ret;
+APFloat::opStatus DoubleAPFloat::convertFromAPInt(const APInt &Input,
+                                                  bool IsSigned,
+                                                  roundingMode RM) {
+  const bool NegateInput = IsSigned && Input.isNegative();
+  APInt API = Input;
+  if (NegateInput)
+    API.negate();
+
+  const APFloat::opStatus Status =
+      convertFromUnsignedParts(API.getRawData(), API.getNumWords(), RM);
+  if (NegateInput)
+    changeSign();
+  return Status;
 }
 
 unsigned int DoubleAPFloat::convertToHexString(char *DST,
@@ -5532,25 +5812,30 @@ void DoubleAPFloat::toString(SmallVectorImpl<char> &Str,
       .toString(Str, FormatPrecision, FormatMaxPadding, TruncateZero);
 }
 
-bool DoubleAPFloat::getExactInverse(APFloat *inv) const {
-  assert(Semantics == &semPPCDoubleDouble && "Unexpected Semantics");
-  APFloat Tmp(semPPCDoubleDoubleLegacy, bitcastToAPInt());
-  if (!inv)
-    return Tmp.getExactInverse(nullptr);
-  APFloat Inv(semPPCDoubleDoubleLegacy);
-  auto Ret = Tmp.getExactInverse(&Inv);
-  *inv = APFloat(semPPCDoubleDouble, Inv.bitcastToAPInt());
-  return Ret;
-}
-
-int DoubleAPFloat::getExactLog2() const {
-  // TODO: Implement me
-  return INT_MIN;
-}
-
 int DoubleAPFloat::getExactLog2Abs() const {
-  // TODO: Implement me
-  return INT_MIN;
+  // In order for Hi + Lo to be a power of two, the following must be true:
+  // 1. Hi must be a power of two.
+  // 2. Lo must be zero.
+  if (getSecond().isNonZero())
+    return INT_MIN;
+  return getFirst().getExactLog2Abs();
+}
+
+int ilogb(const DoubleAPFloat &Arg) {
+  const APFloat &Hi = Arg.getFirst();
+  const APFloat &Lo = Arg.getSecond();
+  int IlogbResult = ilogb(Hi);
+  // Zero and non-finite values can delegate to ilogb(Hi).
+  if (Arg.getCategory() != fcNormal)
+    return IlogbResult;
+  // If Lo can't change the binade, we can delegate to ilogb(Hi).
+  if (Lo.isZero() || Hi.isNegative() == Lo.isNegative())
+    return IlogbResult;
+  if (Hi.getExactLog2Abs() == INT_MIN)
+    return IlogbResult;
+  // Numbers of the form 2^a - 2^b or -2^a + 2^b are almost powers of two but
+  // get nudged out of the binade by the low component.
+  return IlogbResult - 1;
 }
 
 DoubleAPFloat scalbn(const DoubleAPFloat &Arg, int Exp,
@@ -5563,10 +5848,101 @@ DoubleAPFloat scalbn(const DoubleAPFloat &Arg, int Exp,
 DoubleAPFloat frexp(const DoubleAPFloat &Arg, int &Exp,
                     APFloat::roundingMode RM) {
   assert(Arg.Semantics == &semPPCDoubleDouble && "Unexpected Semantics");
-  APFloat First = frexp(Arg.Floats[0], Exp, RM);
-  APFloat Second = Arg.Floats[1];
-  if (Arg.getCategory() == APFloat::fcNormal)
-    Second = scalbn(Second, -Exp, RM);
+
+  // Get the unbiased exponent e of the number, where |Arg| = m * 2^e for m in
+  // [1.0, 2.0).
+  Exp = ilogb(Arg);
+
+  // For NaNs, quiet any signaling NaN and return the result, as per standard
+  // practice.
+  if (Exp == APFloat::IEK_NaN) {
+    DoubleAPFloat Quiet{Arg};
+    Quiet.getFirst().makeQuiet();
+    return Quiet;
+  }
+
+  // For infinity, return it unchanged. The exponent remains IEK_Inf.
+  if (Exp == APFloat::IEK_Inf)
+    return Arg;
+
+  // For zero, the fraction is zero and the standard requires the exponent be 0.
+  if (Exp == APFloat::IEK_Zero) {
+    Exp = 0;
+    return Arg;
+  }
+
+  const APFloat &Hi = Arg.getFirst();
+  const APFloat &Lo = Arg.getSecond();
+
+  // frexp requires the fraction's absolute value to be in [0.5, 1.0).
+  // ilogb provides an exponent for an absolute value in [1.0, 2.0).
+  // Increment the exponent to ensure the fraction is in the correct range.
+  ++Exp;
+
+  const bool SignsDisagree = Hi.isNegative() != Lo.isNegative();
+  APFloat Second = Lo;
+  if (Arg.getCategory() == APFloat::fcNormal && Lo.isFiniteNonZero()) {
+    roundingMode LoRoundingMode;
+    // The interpretation of rmTowardZero depends on the sign of the combined
+    // Arg rather than the sign of the component.
+    if (RM == rmTowardZero)
+      LoRoundingMode = Arg.isNegative() ? rmTowardPositive : rmTowardNegative;
+    // For rmNearestTiesToAway, we face a similar problem. If signs disagree,
+    // Lo is a correction *toward* zero relative to Hi. Rounding Lo
+    // "away from zero" based on its own sign would move the value in the
+    // wrong direction. As a safe proxy, we use rmNearestTiesToEven, which is
+    // direction-agnostic. We only need to bother with this if Lo is scaled
+    // down.
+    else if (RM == rmNearestTiesToAway && SignsDisagree && Exp > 0)
+      LoRoundingMode = rmNearestTiesToEven;
+    else
+      LoRoundingMode = RM;
+    Second = scalbn(Lo, -Exp, LoRoundingMode);
+    // The rmNearestTiesToEven proxy is correct most of the time, but it
+    // differs from rmNearestTiesToAway when the scaled value of Lo is an
+    // exact midpoint.
+    // NOTE: This is morally equivalent to roundTiesTowardZero.
+    if (RM == rmNearestTiesToAway && LoRoundingMode == rmNearestTiesToEven) {
+      // Re-scale the result back to check if rounding occurred.
+      const APFloat RecomposedLo = scalbn(Second, Exp, rmNearestTiesToEven);
+      if (RecomposedLo != Lo) {
+        // RoundingError tells us which direction we rounded:
+        //   - RoundingError > 0: we rounded up.
+        //   - RoundingError < 0: we down up.
+        const APFloat RoundingError = RecomposedLo - Lo;
+        // Determine if scalbn(Lo, -Exp) landed exactly on a midpoint.
+        // We do this by checking if the absolute rounding error is exactly
+        // half a ULP of the result.
+        const APFloat UlpOfSecond = harrisonUlp(Second);
+        const APFloat ScaledUlpOfSecond =
+            scalbn(UlpOfSecond, Exp - 1, rmNearestTiesToEven);
+        const bool IsMidpoint = abs(RoundingError) == ScaledUlpOfSecond;
+        const bool RoundedLoAway =
+            Second.isNegative() == RoundingError.isNegative();
+        // The sign of Hi and Lo disagree and we rounded Lo away: we must
+        // decrease the magnitude of Second to increase the magnitude
+        // First+Second.
+        if (IsMidpoint && RoundedLoAway)
+          Second.next(/*nextDown=*/!Second.isNegative());
+      }
+    }
+    // Handle a tricky edge case where Arg is slightly less than a power of two
+    // (e.g., Arg = 2^k - epsilon). In this situation:
+    // 1. Hi is 2^k, and Lo is a small negative value -epsilon.
+    // 2. ilogb(Arg) correctly returns k-1.
+    // 3. Our initial Exp becomes (k-1) + 1 = k.
+    // 4. Scaling Hi (2^k) by 2^-k would yield a magnitude of 1.0 and
+    //    scaling Lo by 2^-k would yield zero. This would make the result 1.0
+    //    which is an invalid fraction, as the required interval is [0.5, 1.0).
+    // We detect this specific case by checking if Hi is a power of two and if
+    // the scaled Lo underflowed to zero. The fix: Increment Exp to k+1. This
+    // adjusts the scale factor, causing Hi to be scaled to 0.5, which is a
+    // valid fraction.
+    if (Second.isZero() && SignsDisagree && Hi.getExactLog2Abs() != INT_MIN)
+      ++Exp;
+  }
+
+  APFloat First = scalbn(Hi, -Exp, RM);
   return DoubleAPFloat(semPPCDoubleDouble, std::move(First), std::move(Second));
 }
 
@@ -5620,6 +5996,64 @@ FPClassTest APFloat::classify() const {
   return isSignaling() ? fcSNan : fcQNan;
 }
 
+bool APFloat::getExactInverse(APFloat *Inv) const {
+  // Only finite, non-zero numbers can have a useful, representable inverse.
+  // This check filters out +/- zero, +/- infinity, and NaN.
+  if (!isFiniteNonZero())
+    return false;
+
+  // Historically, this function rejects subnormal inputs.  One reason why this
+  // might be important is that subnormals may behave differently under FTZ/DAZ
+  // runtime behavior.
+  if (isDenormal())
+    return false;
+
+  // A number has an exact, representable inverse if and only if it is a power
+  // of two.
+  //
+  // Mathematical Rationale:
+  // 1. A binary floating-point number x is a dyadic rational, meaning it can
+  //    be written as x = M / 2^k for integers M (the significand) and k.
+  // 2. The inverse is 1/x = 2^k / M.
+  // 3. For 1/x to also be a dyadic rational (and thus exactly representable
+  //    in binary), its denominator M must also be a power of two.
+  //    Let's say M = 2^m.
+  // 4. Substituting this back into the formula for x, we get
+  //    x = (2^m) / (2^k) = 2^(m-k).
+  //
+  // This proves that x must be a power of two.
+
+  // getExactLog2Abs() returns the integer exponent if the number is a power of
+  // two or INT_MIN if it is not.
+  const int Exp = getExactLog2Abs();
+  if (Exp == INT_MIN)
+    return false;
+
+  // The inverse of +/- 2^Exp is +/- 2^(-Exp). We can compute this by
+  // scaling 1.0 by the negated exponent.
+  APFloat Reciprocal =
+      scalbn(APFloat::getOne(getSemantics(), /*Negative=*/isNegative()), -Exp,
+             rmTowardZero);
+
+  // scalbn might round if the resulting exponent -Exp is outside the
+  // representable range, causing overflow (to infinity) or underflow. We
+  // must verify that the result is still the exact power of two we expect.
+  if (Reciprocal.getExactLog2Abs() != -Exp)
+    return false;
+
+  // Avoid multiplication with a subnormal, it is not safe on all platforms and
+  // may be slower than a normal division.
+  if (Reciprocal.isDenormal())
+    return false;
+
+  assert(Reciprocal.isFiniteNonZero());
+
+  if (Inv)
+    *Inv = std::move(Reciprocal);
+
+  return true;
+}
+
 APFloat::opStatus APFloat::convert(const fltSemantics &ToSemantics,
                                    roundingMode RM, bool *losesInfo) {
   if (&getSemantics() == &ToSemantics) {
@@ -5666,10 +6100,6 @@ void APFloat::Profile(FoldingSetNodeID &NID) const {
   NID.Add(bitcastToAPInt());
 }
 
-/* Same as convertToInteger(integerPart*, ...), except the result is returned in
-   an APSInt, whose initial bit-width and signed-ness are used to determine the
-   precision of the conversion.
- */
 APFloat::opStatus APFloat::convertToInteger(APSInt &result,
                                             roundingMode rounding_mode,
                                             bool *isExact) const {
diff --git a/llvm/lib/Support/APInt.cpp b/llvm/lib/Support/APInt.cpp
index 954af7fff92a8..f6fd5f9ddd633 100644
--- a/llvm/lib/Support/APInt.cpp
+++ b/llvm/lib/Support/APInt.cpp
@@ -1377,7 +1377,7 @@ static void KnuthDiv(uint32_t *u, uint32_t *v, uint32_t *q, uint32_t* r,
     // the true value, and a "borrow" to the left should be remembered.
     int64_t borrow = 0;
     for (unsigned i = 0; i < n; ++i) {
-      uint64_t p = uint64_t(qp) * uint64_t(v[i]);
+      uint64_t p = qp * uint64_t(v[i]);
       int64_t subres = int64_t(u[j+i]) - borrow - Lo_32(p);
       u[j+i] = Lo_32(subres);
       borrow = Hi_32(p) - Hi_32(subres);
@@ -3136,6 +3136,22 @@ APInt APIntOps::mulhu(const APInt &C1, const APInt &C2) {
   return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
 }
 
+APInt APIntOps::mulsExtended(const APInt &C1, const APInt &C2) {
+  assert(C1.getBitWidth() == C2.getBitWidth() && "Unequal bitwidths");
+  unsigned FullWidth = C1.getBitWidth() * 2;
+  APInt C1Ext = C1.sext(FullWidth);
+  APInt C2Ext = C2.sext(FullWidth);
+  return C1Ext * C2Ext;
+}
+
+APInt APIntOps::muluExtended(const APInt &C1, const APInt &C2) {
+  assert(C1.getBitWidth() == C2.getBitWidth() && "Unequal bitwidths");
+  unsigned FullWidth = C1.getBitWidth() * 2;
+  APInt C1Ext = C1.zext(FullWidth);
+  APInt C2Ext = C2.zext(FullWidth);
+  return C1Ext * C2Ext;
+}
+
 APInt APIntOps::pow(const APInt &X, int64_t N) {
   assert(N >= 0 && "negative exponents not supported.");
   APInt Acc = APInt(X.getBitWidth(), 1);
@@ -3153,3 +3169,21 @@ APInt APIntOps::pow(const APInt &X, int64_t N) {
   }
   return Acc;
 }
+
+APInt llvm::APIntOps::fshl(const APInt &Hi, const APInt &Lo,
+                           const APInt &Shift) {
+  assert(Hi.getBitWidth() == Lo.getBitWidth());
+  unsigned ShiftAmt = rotateModulo(Hi.getBitWidth(), Shift);
+  if (ShiftAmt == 0)
+    return Hi;
+  return Hi.shl(ShiftAmt) | Lo.lshr(Hi.getBitWidth() - ShiftAmt);
+}
+
+APInt llvm::APIntOps::fshr(const APInt &Hi, const APInt &Lo,
+                           const APInt &Shift) {
+  assert(Hi.getBitWidth() == Lo.getBitWidth());
+  unsigned ShiftAmt = rotateModulo(Hi.getBitWidth(), Shift);
+  if (ShiftAmt == 0)
+    return Lo;
+  return Hi.shl(Hi.getBitWidth() - ShiftAmt) | Lo.lshr(ShiftAmt);
+}
diff --git a/llvm/lib/Support/BLAKE3/blake3_neon.c b/llvm/lib/Support/BLAKE3/blake3_neon.c
index ee36721f87573..e691635cf41cb 100644
--- a/llvm/lib/Support/BLAKE3/blake3_neon.c
+++ b/llvm/lib/Support/BLAKE3/blake3_neon.c
@@ -54,7 +54,7 @@ INLINE uint32x4_t rot8_128(uint32x4_t x) {
   // return vorrq_u32(vshrq_n_u32(x, 8), vshlq_n_u32(x, 32 - 8));
 #if defined(__clang__)
   return vreinterpretq_u32_u8(__builtin_shufflevector(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), 1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12));
-#elif __GNUC__ * 10000 + __GNUC_MINOR__ * 100 >=40700
+#elif defined(__GNUC__)
   static const uint8x16_t r8 = {1,2,3,0,5,6,7,4,9,10,11,8,13,14,15,12};
   return vreinterpretq_u32_u8(__builtin_shuffle(vreinterpretq_u8_u32(x), vreinterpretq_u8_u32(x), r8));
 #else 
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 10b6101d73277..6646af6db5d3c 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -62,7 +62,7 @@ elseif( CMAKE_HOST_UNIX )
   if( UNIX AND NOT (BEOS OR HAIKU) )
     set(system_libs ${system_libs} m)
   endif()
-  if( UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "SunOS" )
+  if( UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "SunOS" )
     set(system_libs ${system_libs} kstat socket)
   endif()
   if( FUCHSIA )
@@ -130,7 +130,7 @@ endif()
 # unistd.h and it is guarded by _ALL_SOURCE, so we remove the _XOPEN_SOURCE
 # guard here. We should remove the guards all together once AIX cleans up
 # the system headers.
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   remove_definitions("-D_XOPEN_SOURCE=700")
 endif()
 
@@ -182,6 +182,7 @@ add_llvm_component_library(LLVMSupport
   DivisionByConstantInfo.cpp
   DAGDeltaAlgorithm.cpp
   DJB.cpp
+  DXILABI.cpp
   DynamicAPInt.cpp
   ELFAttributes.cpp
   ELFAttrParserCompact.cpp
diff --git a/llvm/lib/Support/ConvertUTF.cpp b/llvm/lib/Support/ConvertUTF.cpp
index bc04c5ab51135..63f92631d5b47 100644
--- a/llvm/lib/Support/ConvertUTF.cpp
+++ b/llvm/lib/Support/ConvertUTF.cpp
@@ -79,7 +79,7 @@
 #  define ConvertUTF_RESTORE_WARNINGS \
     _Pragma("clang diagnostic pop")
 # endif
-#elif defined(__GNUC__) && __GNUC__ > 6
+#elif defined(__GNUC__)
 # define ConvertUTF_DISABLE_WARNINGS \
    _Pragma("GCC diagnostic push")    \
    _Pragma("GCC diagnostic ignored \"-Wimplicit-fallthrough\"")
diff --git a/llvm/lib/Support/DXILABI.cpp b/llvm/lib/Support/DXILABI.cpp
new file mode 100644
index 0000000000000..082e32061bd45
--- /dev/null
+++ b/llvm/lib/Support/DXILABI.cpp
@@ -0,0 +1,33 @@
+//===-- DXILABI.cpp - ABI Sensitive Values for DXIL -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains definitions of various constants and enums that are
+// required to remain stable as per the DXIL format's requirements.
+//
+// Documentation for DXIL can be found in
+// https://github.com/Microsoft/DirectXShaderCompiler/blob/main/docs/DXIL.rst.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/DXILABI.h"
+#include "llvm/Support/ErrorHandling.h"
+using namespace llvm;
+
+StringRef dxil::getResourceClassName(dxil::ResourceClass RC) {
+  switch (RC) {
+  case dxil::ResourceClass::SRV:
+    return "SRV";
+  case dxil::ResourceClass::UAV:
+    return "UAV";
+  case dxil::ResourceClass::CBuffer:
+    return "CBV";
+  case dxil::ResourceClass::Sampler:
+    return "Sampler";
+  }
+  llvm_unreachable("Invalid ResourceClass enum value");
+}
diff --git a/llvm/lib/Support/KnownBits.cpp b/llvm/lib/Support/KnownBits.cpp
index 94a04ab90987a..bd08365a3fcdb 100644
--- a/llvm/lib/Support/KnownBits.cpp
+++ b/llvm/lib/Support/KnownBits.cpp
@@ -888,11 +888,19 @@ KnownBits KnownBits::mul(const KnownBits &LHS, const KnownBits &RHS,
   Res.Zero |= (~BottomKnown).getLoBits(ResultBitsKnown);
   Res.One = BottomKnown.getLoBits(ResultBitsKnown);
 
-  // If we're self-multiplying then bit[1] is guaranteed to be zero.
-  if (NoUndefSelfMultiply && BitWidth > 1) {
-    assert(Res.One[1] == 0 &&
-           "Self-multiplication failed Quadratic Reciprocity!");
-    Res.Zero.setBit(1);
+  if (NoUndefSelfMultiply) {
+    // If X has at least TZ trailing zeroes, then bit (2 * TZ + 1) must be zero.
+    unsigned TwoTZP1 = 2 * TrailZero0 + 1;
+    if (TwoTZP1 < BitWidth)
+      Res.Zero.setBit(TwoTZP1);
+
+    // If X has exactly TZ trailing zeros, then bit (2 * TZ + 2) must also be
+    // zero.
+    if (TrailZero0 < BitWidth && LHS.One[TrailZero0]) {
+      unsigned TwoTZP2 = TwoTZP1 + 1;
+      if (TwoTZP2 < BitWidth)
+        Res.Zero.setBit(TwoTZP2);
+    }
   }
 
   return Res;
diff --git a/llvm/lib/Support/MemoryBuffer.cpp b/llvm/lib/Support/MemoryBuffer.cpp
index 601f11f6d23c8..1c4645ad83641 100644
--- a/llvm/lib/Support/MemoryBuffer.cpp
+++ b/llvm/lib/Support/MemoryBuffer.cpp
@@ -501,8 +501,14 @@ getOpenFileImpl(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize,
     std::unique_ptr<MB> Result(
         new (NamedBufferAlloc(Filename)) MemoryBufferMMapFile<MB>(
             RequiresNullTerminator, FD, MapSize, Offset, EC));
-    if (!EC)
-      return std::move(Result);
+    if (!EC) {
+      // On at least Linux, and possibly on other systems, mmap may return pages
+      // from the page cache that are not properly filled with trailing zeroes,
+      // if some prior user of the page wrote non-zero bytes. Detect this and
+      // don't use mmap in that case.
+      if (!RequiresNullTerminator || *Result->getBufferEnd() == '\0')
+        return std::move(Result);
+    }
   }
 
 #ifdef __MVS__
diff --git a/llvm/lib/Support/PrettyStackTrace.cpp b/llvm/lib/Support/PrettyStackTrace.cpp
index 3f03368c5d22e..26e22161b605b 100644
--- a/llvm/lib/Support/PrettyStackTrace.cpp
+++ b/llvm/lib/Support/PrettyStackTrace.cpp
@@ -114,15 +114,27 @@ static void PrintCurStackTrace(raw_ostream &OS) {
 //  If any clients of llvm try to link to libCrashReporterClient.a themselves,
 //  only one crash info struct will be used.
 extern "C" {
+#ifdef CRASHREPORTER_ANNOTATIONS_INITIALIZER
+// Should be available in CRASHREPORTER_ANNOTATIONS_VERSION > 5
+CRASHREPORTER_ANNOTATIONS_INITIALIZER()
+#else
+// Older CrashReporter annotations layouts
 CRASH_REPORTER_CLIENT_HIDDEN
 struct crashreporter_annotations_t gCRAnnotations
-        __attribute__((section("__DATA," CRASHREPORTER_ANNOTATIONS_SECTION)))
-#if CRASHREPORTER_ANNOTATIONS_VERSION < 5
-        = { CRASHREPORTER_ANNOTATIONS_VERSION, 0, 0, 0, 0, 0, 0 };
-#else
-        = { CRASHREPORTER_ANNOTATIONS_VERSION, 0, 0, 0, 0, 0, 0, 0 };
-#endif
-}
+    __attribute__((section("__DATA," CRASHREPORTER_ANNOTATIONS_SECTION))) = {
+        CRASHREPORTER_ANNOTATIONS_VERSION,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+#if CRASHREPORTER_ANNOTATIONS_VERSION > 4
+        0
+#endif // CRASHREPORTER_ANNOTATIONS_VERSION > 4
+};
+#endif // CRASHREPORTER_ANNOTATIONS_INITIALIZER
+} // extern "C"
 #elif defined(__APPLE__) && HAVE_CRASHREPORTER_INFO
 extern "C" const char *__crashreporter_info__
     __attribute__((visibility("hidden"))) = 0;
diff --git a/llvm/lib/Support/SmallPtrSet.cpp b/llvm/lib/Support/SmallPtrSet.cpp
index 83143a7fe80fa..a602165a0753c 100644
--- a/llvm/lib/Support/SmallPtrSet.cpp
+++ b/llvm/lib/Support/SmallPtrSet.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemAlloc.h"
 #include <algorithm>
@@ -28,7 +29,7 @@ void SmallPtrSetImplBase::shrink_and_clear() {
   // Reduce the number of buckets.
   unsigned Size = size();
   CurArraySize = Size > 16 ? 1 << (Log2_32_Ceil(Size) + 1) : 32;
-  NumNonEmpty = NumTombstones = 0;
+  NumEntries = NumTombstones = 0;
 
   // Install the new array.  Clear all the buckets to empty.
   CurArray = (const void**)safe_malloc(sizeof(void*) * CurArraySize);
@@ -41,7 +42,8 @@ SmallPtrSetImplBase::insert_imp_big(const void *Ptr) {
   if (LLVM_UNLIKELY(size() * 4 >= CurArraySize * 3)) {
     // If more than 3/4 of the array is full, grow.
     Grow(CurArraySize < 64 ? 128 : CurArraySize * 2);
-  } else if (LLVM_UNLIKELY(CurArraySize - NumNonEmpty < CurArraySize / 8)) {
+  } else if (LLVM_UNLIKELY(CurArraySize - NumEntries - NumTombstones <
+                           CurArraySize / 8)) {
     // If fewer of 1/8 of the array is empty (meaning that many are filled with
     // tombstones), rehash.
     Grow(CurArraySize);
@@ -55,8 +57,7 @@ SmallPtrSetImplBase::insert_imp_big(const void *Ptr) {
   // Otherwise, insert it!
   if (*Bucket == getTombstoneMarker())
     --NumTombstones;
-  else
-    ++NumNonEmpty; // Track density.
+  ++NumEntries;
   *Bucket = Ptr;
   incrementEpoch();
   return std::make_pair(Bucket, true);
@@ -110,8 +111,7 @@ const void *const *SmallPtrSetImplBase::FindBucketFor(const void *Ptr) const {
 /// Grow - Allocate a larger backing store for the buckets and move it over.
 ///
 void SmallPtrSetImplBase::Grow(unsigned NewSize) {
-  const void **OldBuckets = CurArray;
-  const void **OldEnd = EndPointer();
+  auto OldBuckets = buckets();
   bool WasSmall = isSmall();
 
   // Install the new array.  Clear all the buckets to empty.
@@ -123,16 +123,14 @@ void SmallPtrSetImplBase::Grow(unsigned NewSize) {
   memset(CurArray, -1, NewSize*sizeof(void*));
 
   // Copy over all valid entries.
-  for (const void **BucketPtr = OldBuckets; BucketPtr != OldEnd; ++BucketPtr) {
+  for (const void *&Bucket : OldBuckets) {
     // Copy over the element if it is valid.
-    const void *Elt = *BucketPtr;
-    if (Elt != getTombstoneMarker() && Elt != getEmptyMarker())
-      *const_cast<void**>(FindBucketFor(Elt)) = const_cast<void*>(Elt);
+    if (Bucket != getTombstoneMarker() && Bucket != getEmptyMarker())
+      *const_cast<void **>(FindBucketFor(Bucket)) = const_cast<void *>(Bucket);
   }
 
   if (!WasSmall)
-    free(OldBuckets);
-  NumNonEmpty -= NumTombstones;
+    free(OldBuckets.begin());
   NumTombstones = 0;
   IsSmall = false;
 }
@@ -193,9 +191,9 @@ void SmallPtrSetImplBase::copyHelper(const SmallPtrSetImplBase &RHS) {
   CurArraySize = RHS.CurArraySize;
 
   // Copy over the contents from the other set
-  std::copy(RHS.CurArray, RHS.EndPointer(), CurArray);
+  llvm::copy(RHS.buckets(), CurArray);
 
-  NumNonEmpty = RHS.NumNonEmpty;
+  NumEntries = RHS.NumEntries;
   NumTombstones = RHS.NumTombstones;
 }
 
@@ -217,7 +215,7 @@ void SmallPtrSetImplBase::moveHelper(const void **SmallStorage,
   if (RHS.isSmall()) {
     // Copy a small RHS rather than moving.
     CurArray = SmallStorage;
-    std::copy(RHS.CurArray, RHS.CurArray + RHS.NumNonEmpty, CurArray);
+    llvm::copy(RHS.small_buckets(), CurArray);
   } else {
     CurArray = RHS.CurArray;
     RHS.CurArray = RHSSmallStorage;
@@ -225,13 +223,13 @@ void SmallPtrSetImplBase::moveHelper(const void **SmallStorage,
 
   // Copy the rest of the trivial members.
   CurArraySize = RHS.CurArraySize;
-  NumNonEmpty = RHS.NumNonEmpty;
+  NumEntries = RHS.NumEntries;
   NumTombstones = RHS.NumTombstones;
   IsSmall = RHS.IsSmall;
 
   // Make the RHS small and empty.
   RHS.CurArraySize = SmallSize;
-  RHS.NumNonEmpty = 0;
+  RHS.NumEntries = 0;
   RHS.NumTombstones = 0;
   RHS.IsSmall = true;
 }
@@ -245,54 +243,42 @@ void SmallPtrSetImplBase::swap(const void **SmallStorage,
   if (!this->isSmall() && !RHS.isSmall()) {
     std::swap(this->CurArray, RHS.CurArray);
     std::swap(this->CurArraySize, RHS.CurArraySize);
-    std::swap(this->NumNonEmpty, RHS.NumNonEmpty);
+    std::swap(this->NumEntries, RHS.NumEntries);
     std::swap(this->NumTombstones, RHS.NumTombstones);
     return;
   }
 
   // FIXME: From here on we assume that both sets have the same small size.
 
-  // If only RHS is small, copy the small elements into LHS and move the pointer
-  // from LHS to RHS.
-  if (!this->isSmall() && RHS.isSmall()) {
-    std::copy(RHS.CurArray, RHS.CurArray + RHS.NumNonEmpty, SmallStorage);
-    std::swap(RHS.CurArraySize, this->CurArraySize);
-    std::swap(this->NumNonEmpty, RHS.NumNonEmpty);
+  // Both a small, just swap the small elements.
+  if (this->isSmall() && RHS.isSmall()) {
+    unsigned MinEntries = std::min(this->NumEntries, RHS.NumEntries);
+    std::swap_ranges(this->CurArray, this->CurArray + MinEntries, RHS.CurArray);
+    if (this->NumEntries > MinEntries) {
+      std::copy(this->CurArray + MinEntries, this->CurArray + this->NumEntries,
+                RHS.CurArray + MinEntries);
+    } else {
+      std::copy(RHS.CurArray + MinEntries, RHS.CurArray + RHS.NumEntries,
+                this->CurArray + MinEntries);
+    }
+    assert(this->CurArraySize == RHS.CurArraySize);
+    std::swap(this->NumEntries, RHS.NumEntries);
     std::swap(this->NumTombstones, RHS.NumTombstones);
-    RHS.CurArray = this->CurArray;
-    RHS.IsSmall = false;
-    this->CurArray = SmallStorage;
-    this->IsSmall = true;
     return;
   }
 
-  // If only LHS is small, copy the small elements into RHS and move the pointer
-  // from RHS to LHS.
-  if (this->isSmall() && !RHS.isSmall()) {
-    std::copy(this->CurArray, this->CurArray + this->NumNonEmpty,
-              RHSSmallStorage);
-    std::swap(RHS.CurArraySize, this->CurArraySize);
-    std::swap(RHS.NumNonEmpty, this->NumNonEmpty);
-    std::swap(RHS.NumTombstones, this->NumTombstones);
-    this->CurArray = RHS.CurArray;
-    this->IsSmall = false;
-    RHS.CurArray = RHSSmallStorage;
-    RHS.IsSmall = true;
-    return;
-  }
-
-  // Both a small, just swap the small elements.
-  assert(this->isSmall() && RHS.isSmall());
-  unsigned MinNonEmpty = std::min(this->NumNonEmpty, RHS.NumNonEmpty);
-  std::swap_ranges(this->CurArray, this->CurArray + MinNonEmpty, RHS.CurArray);
-  if (this->NumNonEmpty > MinNonEmpty) {
-    std::copy(this->CurArray + MinNonEmpty, this->CurArray + this->NumNonEmpty,
-              RHS.CurArray + MinNonEmpty);
-  } else {
-    std::copy(RHS.CurArray + MinNonEmpty, RHS.CurArray + RHS.NumNonEmpty,
-              this->CurArray + MinNonEmpty);
-  }
-  assert(this->CurArraySize == RHS.CurArraySize);
-  std::swap(this->NumNonEmpty, RHS.NumNonEmpty);
-  std::swap(this->NumTombstones, RHS.NumTombstones);
+  // If only one side is small, copy the small elements into the large side and
+  // move the pointer from the large side to the small side.
+  SmallPtrSetImplBase &SmallSide = this->isSmall() ? *this : RHS;
+  SmallPtrSetImplBase &LargeSide = this->isSmall() ? RHS : *this;
+  const void **LargeSideInlineStorage =
+      this->isSmall() ? RHSSmallStorage : SmallStorage;
+  llvm::copy(SmallSide.small_buckets(), LargeSideInlineStorage);
+  std::swap(LargeSide.CurArraySize, SmallSide.CurArraySize);
+  std::swap(LargeSide.NumEntries, SmallSide.NumEntries);
+  std::swap(LargeSide.NumTombstones, SmallSide.NumTombstones);
+  SmallSide.CurArray = LargeSide.CurArray;
+  SmallSide.IsSmall = false;
+  LargeSide.CurArray = LargeSideInlineStorage;
+  LargeSide.IsSmall = true;
 }
diff --git a/llvm/lib/Support/StringRef.cpp b/llvm/lib/Support/StringRef.cpp
index dc758785e40d5..b6a2f8aeadccf 100644
--- a/llvm/lib/Support/StringRef.cpp
+++ b/llvm/lib/Support/StringRef.cpp
@@ -385,7 +385,7 @@ size_t StringRef::count(StringRef Str) const {
   return Count;
 }
 
-static unsigned GetAutoSenseRadix(StringRef &Str) {
+unsigned llvm::getAutoSenseRadix(StringRef &Str) {
   if (Str.empty())
     return 10;
 
@@ -410,7 +410,7 @@ bool llvm::consumeUnsignedInteger(StringRef &Str, unsigned Radix,
                                   unsigned long long &Result) {
   // Autosense radix if not specified.
   if (Radix == 0)
-    Radix = GetAutoSenseRadix(Str);
+    Radix = getAutoSenseRadix(Str);
 
   // Empty strings (after the radix autosense) are invalid.
   if (Str.empty()) return true;
@@ -509,7 +509,7 @@ bool StringRef::consumeInteger(unsigned Radix, APInt &Result) {
 
   // Autosense radix if not specified.
   if (Radix == 0)
-    Radix = GetAutoSenseRadix(Str);
+    Radix = getAutoSenseRadix(Str);
 
   assert(Radix > 1 && Radix <= 36);
 
diff --git a/llvm/lib/Support/Unix/Path.inc b/llvm/lib/Support/Unix/Path.inc
index cc02cae40ec76..2f563e2899b56 100644
--- a/llvm/lib/Support/Unix/Path.inc
+++ b/llvm/lib/Support/Unix/Path.inc
@@ -876,6 +876,12 @@ void mapped_file_region::unmapImpl() {
     ::munmap(Mapping, Size);
 }
 
+std::error_code mapped_file_region::sync() const {
+  if (int Res = ::msync(Mapping, Size, MS_SYNC))
+    return std::error_code(Res, std::generic_category());
+  return std::error_code();
+}
+
 void mapped_file_region::dontNeedImpl() {
   assert(Mode == mapped_file_region::readonly);
   if (!Mapping)
@@ -1224,13 +1230,21 @@ Expected<size_t> readNativeFileSlice(file_t FD, MutableArrayRef<char> Buf,
   return NumRead;
 }
 
-std::error_code tryLockFile(int FD, std::chrono::milliseconds Timeout) {
+std::error_code tryLockFile(int FD, std::chrono::milliseconds Timeout,
+                            LockKind Kind) {
   auto Start = std::chrono::steady_clock::now();
   auto End = Start + Timeout;
   do {
     struct flock Lock;
     memset(&Lock, 0, sizeof(Lock));
-    Lock.l_type = F_WRLCK;
+    switch (Kind) {
+    case LockKind::Exclusive:
+      Lock.l_type = F_WRLCK;
+      break;
+    case LockKind::Shared:
+      Lock.l_type = F_RDLCK;
+      break;
+    }
     Lock.l_whence = SEEK_SET;
     Lock.l_start = 0;
     Lock.l_len = 0;
@@ -1239,15 +1253,24 @@ std::error_code tryLockFile(int FD, std::chrono::milliseconds Timeout) {
     int Error = errno;
     if (Error != EACCES && Error != EAGAIN)
       return std::error_code(Error, std::generic_category());
+    if (Timeout.count() == 0)
+      break;
     usleep(1000);
   } while (std::chrono::steady_clock::now() < End);
   return make_error_code(errc::no_lock_available);
 }
 
-std::error_code lockFile(int FD) {
+std::error_code lockFile(int FD, LockKind Kind) {
   struct flock Lock;
   memset(&Lock, 0, sizeof(Lock));
-  Lock.l_type = F_WRLCK;
+  switch (Kind) {
+  case LockKind::Exclusive:
+    Lock.l_type = F_WRLCK;
+    break;
+  case LockKind::Shared:
+    Lock.l_type = F_RDLCK;
+    break;
+  }
   Lock.l_whence = SEEK_SET;
   Lock.l_start = 0;
   Lock.l_len = 0;
diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc
index fdf9d540a6488..6672d8e0ec777 100644
--- a/llvm/lib/Support/Windows/Path.inc
+++ b/llvm/lib/Support/Windows/Path.inc
@@ -1006,6 +1006,14 @@ void mapped_file_region::unmapImpl() {
 
 void mapped_file_region::dontNeedImpl() {}
 
+std::error_code mapped_file_region::sync() const {
+  if (!::FlushViewOfFile(Mapping, Size))
+    return mapWindowsError(GetLastError());
+  if (!::FlushFileBuffers(FileHandle))
+    return mapWindowsError(GetLastError());
+  return std::error_code();
+}
+
 int mapped_file_region::alignment() {
   SYSTEM_INFO SysInfo;
   ::GetSystemInfo(&SysInfo);
@@ -1329,8 +1337,10 @@ Expected<size_t> readNativeFileSlice(file_t FileHandle,
   return readNativeFileImpl(FileHandle, Buf, &Overlapped);
 }
 
-std::error_code tryLockFile(int FD, std::chrono::milliseconds Timeout) {
-  DWORD Flags = LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY;
+std::error_code tryLockFile(int FD, std::chrono::milliseconds Timeout,
+                            LockKind Kind) {
+  DWORD Flags = Kind == LockKind::Exclusive ? LOCKFILE_EXCLUSIVE_LOCK : 0;
+  Flags |= LOCKFILE_FAIL_IMMEDIATELY;
   OVERLAPPED OV = {};
   file_t File = convertFDToNativeFile(FD);
   auto Start = std::chrono::steady_clock::now();
@@ -1340,6 +1350,8 @@ std::error_code tryLockFile(int FD, std::chrono::milliseconds Timeout) {
       return std::error_code();
     DWORD Error = ::GetLastError();
     if (Error == ERROR_LOCK_VIOLATION) {
+      if (Timeout.count() == 0)
+        break;
       ::Sleep(1);
       continue;
     }
@@ -1348,8 +1360,8 @@ std::error_code tryLockFile(int FD, std::chrono::milliseconds Timeout) {
   return mapWindowsError(ERROR_LOCK_VIOLATION);
 }
 
-std::error_code lockFile(int FD) {
-  DWORD Flags = LOCKFILE_EXCLUSIVE_LOCK;
+std::error_code lockFile(int FD, LockKind Kind) {
+  DWORD Flags = Kind == LockKind::Exclusive ? LOCKFILE_EXCLUSIVE_LOCK : 0;
   OVERLAPPED OV = {};
   file_t File = convertFDToNativeFile(FD);
   if (::LockFileEx(File, Flags, 0, MAXDWORD, MAXDWORD, &OV))
diff --git a/llvm/lib/Support/regcomp.c b/llvm/lib/Support/regcomp.c
index 4ed59823f6f44..f5c47781f3d82 100644
--- a/llvm/lib/Support/regcomp.c
+++ b/llvm/lib/Support/regcomp.c
@@ -305,7 +305,7 @@ llvm_regcomp(llvm_regex_t *preg, const char *pattern, int cflags) {
       return (REG_INVARG);
     len = preg->re_endp - pattern;
   } else {
-    len = strlen((const char *)pattern);
+    len = strlen(pattern);
   }
 
   /* do the mallocs early so failure handling is easy */
diff --git a/llvm/lib/TableGen/Record.cpp b/llvm/lib/TableGen/Record.cpp
index 3f318e2570bf0..3657a15ab1989 100644
--- a/llvm/lib/TableGen/Record.cpp
+++ b/llvm/lib/TableGen/Record.cpp
@@ -1564,9 +1564,24 @@ const Init *BinOpInit::Fold(const Record *CurRec) const {
       case AND: Result = LHSv & RHSv; break;
       case OR:  Result = LHSv | RHSv; break;
       case XOR: Result = LHSv ^ RHSv; break;
-      case SHL: Result = (uint64_t)LHSv << (uint64_t)RHSv; break;
-      case SRA: Result = LHSv >> RHSv; break;
-      case SRL: Result = (uint64_t)LHSv >> (uint64_t)RHSv; break;
+      case SHL:
+        if (RHSv < 0 || RHSv >= 64)
+          PrintFatalError(CurRec->getLoc(),
+                          "Illegal operation: out of bounds shift");
+        Result = (uint64_t)LHSv << (uint64_t)RHSv;
+        break;
+      case SRA:
+        if (RHSv < 0 || RHSv >= 64)
+          PrintFatalError(CurRec->getLoc(),
+                          "Illegal operation: out of bounds shift");
+        Result = LHSv >> (uint64_t)RHSv;
+        break;
+      case SRL:
+        if (RHSv < 0 || RHSv >= 64)
+          PrintFatalError(CurRec->getLoc(),
+                          "Illegal operation: out of bounds shift");
+        Result = (uint64_t)LHSv >> (uint64_t)RHSv;
+        break;
       }
       return IntInit::get(getRecordKeeper(), Result);
     }
@@ -3064,11 +3079,11 @@ const Init *Record::getValueInit(StringRef FieldName) const {
 }
 
 StringRef Record::getValueAsString(StringRef FieldName) const {
-  std::optional<StringRef> S = getValueAsOptionalString(FieldName);
-  if (!S)
-    PrintFatalError(getLoc(), "Record `" + getName() +
-      "' does not have a field named `" + FieldName + "'!\n");
-  return *S;
+  const Init *I = getValueInit(FieldName);
+  if (const auto *SI = dyn_cast<StringInit>(I))
+    return SI->getValue();
+  PrintFatalError(getLoc(), "Record `" + getName() + "', field `" + FieldName +
+                                "' exists but does not have a string value");
 }
 
 std::optional<StringRef>
@@ -3088,24 +3103,16 @@ Record::getValueAsOptionalString(StringRef FieldName) const {
 }
 
 const BitsInit *Record::getValueAsBitsInit(StringRef FieldName) const {
-  const RecordVal *R = getValue(FieldName);
-  if (!R || !R->getValue())
-    PrintFatalError(getLoc(), "Record `" + getName() +
-      "' does not have a field named `" + FieldName + "'!\n");
-
-  if (const auto *BI = dyn_cast<BitsInit>(R->getValue()))
+  const Init *I = getValueInit(FieldName);
+  if (const auto *BI = dyn_cast<BitsInit>(I))
     return BI;
   PrintFatalError(getLoc(), "Record `" + getName() + "', field `" + FieldName +
                                 "' exists but does not have a bits value");
 }
 
 const ListInit *Record::getValueAsListInit(StringRef FieldName) const {
-  const RecordVal *R = getValue(FieldName);
-  if (!R || !R->getValue())
-    PrintFatalError(getLoc(), "Record `" + getName() +
-      "' does not have a field named `" + FieldName + "'!\n");
-
-  if (const auto *LI = dyn_cast<ListInit>(R->getValue()))
+  const Init *I = getValueInit(FieldName);
+  if (const auto *LI = dyn_cast<ListInit>(I))
     return LI;
   PrintFatalError(getLoc(), "Record `" + getName() + "', field `" + FieldName +
                                 "' exists but does not have a list value");
@@ -3127,17 +3134,13 @@ Record::getValueAsListOfDefs(StringRef FieldName) const {
 }
 
 int64_t Record::getValueAsInt(StringRef FieldName) const {
-  const RecordVal *R = getValue(FieldName);
-  if (!R || !R->getValue())
-    PrintFatalError(getLoc(), "Record `" + getName() +
-      "' does not have a field named `" + FieldName + "'!\n");
-
-  if (const auto *II = dyn_cast<IntInit>(R->getValue()))
+  const Init *I = getValueInit(FieldName);
+  if (const auto *II = dyn_cast<IntInit>(I))
     return II->getValue();
-  PrintFatalError(getLoc(), Twine("Record `") + getName() + "', field `" +
-                                FieldName +
-                                "' exists but does not have an int value: " +
-                                R->getValue()->getAsString());
+  PrintFatalError(
+      getLoc(),
+      Twine("Record `") + getName() + "', field `" + FieldName +
+          "' exists but does not have an int value: " + I->getAsString());
 }
 
 std::vector<int64_t>
@@ -3173,67 +3176,47 @@ Record::getValueAsListOfStrings(StringRef FieldName) const {
 }
 
 const Record *Record::getValueAsDef(StringRef FieldName) const {
-  const RecordVal *R = getValue(FieldName);
-  if (!R || !R->getValue())
-    PrintFatalError(getLoc(), "Record `" + getName() +
-      "' does not have a field named `" + FieldName + "'!\n");
-
-  if (const auto *DI = dyn_cast<DefInit>(R->getValue()))
+  const Init *I = getValueInit(FieldName);
+  if (const auto *DI = dyn_cast<DefInit>(I))
     return DI->getDef();
   PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
     FieldName + "' does not have a def initializer!");
 }
 
 const Record *Record::getValueAsOptionalDef(StringRef FieldName) const {
-  const RecordVal *R = getValue(FieldName);
-  if (!R || !R->getValue())
-    PrintFatalError(getLoc(), "Record `" + getName() +
-      "' does not have a field named `" + FieldName + "'!\n");
-
-  if (const auto *DI = dyn_cast<DefInit>(R->getValue()))
+  const Init *I = getValueInit(FieldName);
+  if (const auto *DI = dyn_cast<DefInit>(I))
     return DI->getDef();
-  if (isa<UnsetInit>(R->getValue()))
+  if (isa<UnsetInit>(I))
     return nullptr;
   PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
     FieldName + "' does not have either a def initializer or '?'!");
 }
 
 bool Record::getValueAsBit(StringRef FieldName) const {
-  const RecordVal *R = getValue(FieldName);
-  if (!R || !R->getValue())
-    PrintFatalError(getLoc(), "Record `" + getName() +
-      "' does not have a field named `" + FieldName + "'!\n");
-
-  if (const auto *BI = dyn_cast<BitInit>(R->getValue()))
+  const Init *I = getValueInit(FieldName);
+  if (const auto *BI = dyn_cast<BitInit>(I))
     return BI->getValue();
   PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
     FieldName + "' does not have a bit initializer!");
 }
 
 bool Record::getValueAsBitOrUnset(StringRef FieldName, bool &Unset) const {
-  const RecordVal *R = getValue(FieldName);
-  if (!R || !R->getValue())
-    PrintFatalError(getLoc(), "Record `" + getName() +
-      "' does not have a field named `" + FieldName.str() + "'!\n");
-
-  if (isa<UnsetInit>(R->getValue())) {
+  const Init *I = getValueInit(FieldName);
+  if (isa<UnsetInit>(I)) {
     Unset = true;
     return false;
   }
   Unset = false;
-  if (const auto *BI = dyn_cast<BitInit>(R->getValue()))
+  if (const auto *BI = dyn_cast<BitInit>(I))
     return BI->getValue();
   PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
     FieldName + "' does not have a bit initializer!");
 }
 
 const DagInit *Record::getValueAsDag(StringRef FieldName) const {
-  const RecordVal *R = getValue(FieldName);
-  if (!R || !R->getValue())
-    PrintFatalError(getLoc(), "Record `" + getName() +
-      "' does not have a field named `" + FieldName + "'!\n");
-
-  if (const auto *DI = dyn_cast<DagInit>(R->getValue()))
+  const Init *I = getValueInit(FieldName);
+  if (const auto *DI = dyn_cast<DagInit>(I))
     return DI;
   PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
     FieldName + "' does not have a dag initializer!");
diff --git a/llvm/lib/Target/AArch64/AArch64.h b/llvm/lib/Target/AArch64/AArch64.h
index 5496ebd495a55..8d0ff41fc8c08 100644
--- a/llvm/lib/Target/AArch64/AArch64.h
+++ b/llvm/lib/Target/AArch64/AArch64.h
@@ -60,6 +60,7 @@ FunctionPass *createAArch64CleanupLocalDynamicTLSPass();
 FunctionPass *createAArch64CollectLOHPass();
 FunctionPass *createSMEABIPass();
 FunctionPass *createSMEPeepholeOptPass();
+FunctionPass *createMachineSMEABIPass();
 ModulePass *createSVEIntrinsicOptsPass();
 InstructionSelector *
 createAArch64InstructionSelector(const AArch64TargetMachine &,
@@ -111,6 +112,7 @@ void initializeFalkorMarkStridedAccessesLegacyPass(PassRegistry&);
 void initializeLDTLSCleanupPass(PassRegistry&);
 void initializeSMEABIPass(PassRegistry &);
 void initializeSMEPeepholeOptPass(PassRegistry &);
+void initializeMachineSMEABIPass(PassRegistry &);
 void initializeSVEIntrinsicOptsPass(PassRegistry &);
 void initializeAArch64Arm64ECCallLoweringPass(PassRegistry &);
 } // end namespace llvm
diff --git a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
index ad8368e1692be..1169f26a2ae37 100644
--- a/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Arm64ECCallLowering.cpp
@@ -316,6 +316,12 @@ ThunkArgInfo AArch64Arm64ECCallLowering::canonicalizeThunkType(
                         ThunkArgTranslation::PointerIndirection};
   };
 
+  if (T->isHalfTy()) {
+    // Prefix with `llvm` since MSVC doesn't specify `_Float16`
+    Out << "__llvm_h__";
+    return direct(T);
+  }
+
   if (T->isFloatTy()) {
     Out << "f";
     return direct(T);
@@ -327,8 +333,8 @@ ThunkArgInfo AArch64Arm64ECCallLowering::canonicalizeThunkType(
   }
 
   if (T->isFloatingPointTy()) {
-    report_fatal_error(
-        "Only 32 and 64 bit floating points are supported for ARM64EC thunks");
+    report_fatal_error("Only 16, 32, and 64 bit floating points are supported "
+                       "for ARM64EC thunks");
   }
 
   auto &DL = M->getDataLayout();
@@ -342,8 +348,16 @@ ThunkArgInfo AArch64Arm64ECCallLowering::canonicalizeThunkType(
     uint64_t ElementCnt = T->getArrayNumElements();
     uint64_t ElementSizePerBytes = DL.getTypeSizeInBits(ElementTy) / 8;
     uint64_t TotalSizeBytes = ElementCnt * ElementSizePerBytes;
-    if (ElementTy->isFloatTy() || ElementTy->isDoubleTy()) {
-      Out << (ElementTy->isFloatTy() ? "F" : "D") << TotalSizeBytes;
+    if (ElementTy->isHalfTy() || ElementTy->isFloatTy() ||
+        ElementTy->isDoubleTy()) {
+      if (ElementTy->isHalfTy())
+        // Prefix with `llvm` since MSVC doesn't specify `_Float16`
+        Out << "__llvm_H__";
+      else if (ElementTy->isFloatTy())
+        Out << "F";
+      else if (ElementTy->isDoubleTy())
+        Out << "D";
+      Out << TotalSizeBytes;
       if (Alignment.value() >= 16 && !Ret)
         Out << "a" << Alignment.value();
       if (TotalSizeBytes <= 8) {
@@ -355,8 +369,9 @@ ThunkArgInfo AArch64Arm64ECCallLowering::canonicalizeThunkType(
         return pointerIndirection(T);
       }
     } else if (T->isFloatingPointTy()) {
-      report_fatal_error("Only 32 and 64 bit floating points are supported for "
-                         "ARM64EC thunks");
+      report_fatal_error(
+          "Only 16, 32, and 64 bit floating points are supported "
+          "for ARM64EC thunks");
     }
   }
 
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp b/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
index 787a1a83613c9..cc46159915d76 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
@@ -75,8 +75,10 @@ static bool finishStackBlock(SmallVectorImpl<CCValAssign> &PendingMembers,
     auto &It = PendingMembers[0];
     CCAssignFn *AssignFn =
         TLI->CCAssignFnForCall(State.getCallingConv(), /*IsVarArg=*/false);
+    // FIXME: Get the correct original type.
+    Type *OrigTy = EVT(It.getValVT()).getTypeForEVT(State.getContext());
     if (AssignFn(It.getValNo(), It.getValVT(), It.getValVT(), CCValAssign::Full,
-                 ArgFlags, State))
+                 ArgFlags, OrigTy, State))
       llvm_unreachable("Call operand has unhandled type");
 
     // Return the flags to how they were before.
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.h b/llvm/lib/Target/AArch64/AArch64CallingConvention.h
index 63185a97cba03..7105fa695334b 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.h
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.h
@@ -18,52 +18,63 @@
 namespace llvm {
 bool CC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                       CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                      CCState &State);
+                      Type *OrigTy, CCState &State);
 bool CC_AArch64_Arm64EC_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
                                CCValAssign::LocInfo LocInfo,
-                               ISD::ArgFlagsTy ArgFlags, CCState &State);
+                               ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                               CCState &State);
 bool CC_AArch64_Arm64EC_Thunk(unsigned ValNo, MVT ValVT, MVT LocVT,
                               CCValAssign::LocInfo LocInfo,
-                              ISD::ArgFlagsTy ArgFlags, CCState &State);
+                              ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                              CCState &State);
 bool CC_AArch64_Arm64EC_Thunk_Native(unsigned ValNo, MVT ValVT, MVT LocVT,
                                      CCValAssign::LocInfo LocInfo,
-                                     ISD::ArgFlagsTy ArgFlags, CCState &State);
+                                     ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                                     CCState &State);
 bool CC_AArch64_DarwinPCS_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
                                  CCValAssign::LocInfo LocInfo,
-                                 ISD::ArgFlagsTy ArgFlags, CCState &State);
+                                 ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                                 CCState &State);
 bool CC_AArch64_DarwinPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                           CCValAssign::LocInfo LocInfo,
-                          ISD::ArgFlagsTy ArgFlags, CCState &State);
+                          ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                          CCState &State);
 bool CC_AArch64_DarwinPCS_ILP32_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
-                          CCValAssign::LocInfo LocInfo,
-                          ISD::ArgFlagsTy ArgFlags, CCState &State);
+                                       CCValAssign::LocInfo LocInfo,
+                                       ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                                       CCState &State);
 bool CC_AArch64_Win64PCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                          CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                         CCState &State);
+                         Type *OrigTy, CCState &State);
 bool CC_AArch64_Win64_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
                              CCValAssign::LocInfo LocInfo,
-                             ISD::ArgFlagsTy ArgFlags, CCState &State);
+                             ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                             CCState &State);
 bool CC_AArch64_Win64_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT,
                                     CCValAssign::LocInfo LocInfo,
-                                    ISD::ArgFlagsTy ArgFlags, CCState &State);
+                                    ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                                    CCState &State);
 bool CC_AArch64_Arm64EC_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT,
                                       CCValAssign::LocInfo LocInfo,
-                                      ISD::ArgFlagsTy ArgFlags, CCState &State);
+                                      ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                                      CCState &State);
 bool CC_AArch64_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
                     CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                    CCState &State);
+                    Type *OrigTy, CCState &State);
 bool CC_AArch64_Preserve_None(unsigned ValNo, MVT ValVT, MVT LocVT,
                               CCValAssign::LocInfo LocInfo,
-                              ISD::ArgFlagsTy ArgFlags, CCState &State);
+                              ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                              CCState &State);
 bool RetCC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                          CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                         CCState &State);
+                         Type *OrigTy, CCState &State);
 bool RetCC_AArch64_Arm64EC_Thunk(unsigned ValNo, MVT ValVT, MVT LocVT,
                                  CCValAssign::LocInfo LocInfo,
-                                 ISD::ArgFlagsTy ArgFlags, CCState &State);
+                                 ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                                 CCState &State);
 bool RetCC_AArch64_Arm64EC_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT,
                                          CCValAssign::LocInfo LocInfo,
-                                         ISD::ArgFlagsTy ArgFlags,
+                                         ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
                                          CCState &State);
 } // namespace llvm
 
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
index 99f0af5f6a3f8..5f499e5e9700a 100644
--- a/llvm/lib/Target/AArch64/AArch64Combine.td
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -351,9 +351,10 @@ def AArch64PostLegalizerLowering
 // Post-legalization combines which are primarily optimizations.
 def AArch64PostLegalizerCombiner
     : GICombiner<"AArch64PostLegalizerCombinerImpl",
-                       [copy_prop, cast_of_cast_combines, buildvector_of_truncate,
-                        integer_of_truncate, mutate_anyext_to_zext,
-                        combines_for_extload, combine_indexed_load_store, sext_trunc_sextload,
+                       [copy_prop, cast_of_cast_combines, 
+                        buildvector_of_truncate, integer_of_truncate,
+                        mutate_anyext_to_zext, combines_for_extload, 
+                        combine_indexed_load_store, sext_trunc_sextload,
                         hoist_logic_op_with_same_opcode_hands,
                         redundant_and, xor_of_and_with_same_reg,
                         extractvecelt_pairwise_add, redundant_or,
@@ -367,5 +368,6 @@ def AArch64PostLegalizerCombiner
                         select_to_minmax, or_to_bsp, combine_concat_vector,
                         commute_constant_to_rhs, extract_vec_elt_combines,
                         push_freeze_to_prevent_poison_from_propagating,
-                        combine_mul_cmlt, combine_use_vector_truncate, extmultomull]> {
+                        combine_mul_cmlt, combine_use_vector_truncate, 
+                        extmultomull, truncsat_combines]> {
 }
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 201bfe0a443d6..57dcd68595ff1 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -92,8 +92,9 @@ class AArch64ExpandPseudo : public MachineFunctionPass {
   bool expandCALL_BTI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
   bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator MBBI);
-  MachineBasicBlock *expandRestoreZA(MachineBasicBlock &MBB,
-                                     MachineBasicBlock::iterator MBBI);
+  MachineBasicBlock *
+  expandCommitOrRestoreZASave(MachineBasicBlock &MBB,
+                              MachineBasicBlock::iterator MBBI);
   MachineBasicBlock *expandCondSMToggle(MachineBasicBlock &MBB,
                                         MachineBasicBlock::iterator MBBI);
 };
@@ -528,6 +529,11 @@ bool AArch64ExpandPseudo::expand_DestructiveOp(
       UseRev = true;
     }
     break;
+  case AArch64::Destructive2xRegImmUnpred:
+    // EXT_ZZI_CONSTRUCTIVE Zd, Zs, Imm
+    //      ==> MOVPRFX Zd Zs; EXT_ZZI Zd, Zd, Zs, Imm
+    std::tie(DOPIdx, SrcIdx, Src2Idx) = std::make_tuple(1, 1, 2);
+    break;
   default:
     llvm_unreachable("Unsupported Destructive Operand type");
   }
@@ -548,6 +554,7 @@ bool AArch64ExpandPseudo::expand_DestructiveOp(
     break;
   case AArch64::DestructiveUnaryPassthru:
   case AArch64::DestructiveBinaryImm:
+  case AArch64::Destructive2xRegImmUnpred:
     DOPRegIsUnique = true;
     break;
   case AArch64::DestructiveTernaryCommWithRev:
@@ -674,6 +681,11 @@ bool AArch64ExpandPseudo::expand_DestructiveOp(
         .add(MI.getOperand(SrcIdx))
         .add(MI.getOperand(Src2Idx));
     break;
+  case AArch64::Destructive2xRegImmUnpred:
+    DOP.addReg(MI.getOperand(DOPIdx).getReg(), DOPRegState)
+        .add(MI.getOperand(SrcIdx))
+        .add(MI.getOperand(Src2Idx));
+    break;
   }
 
   if (PRFX) {
@@ -979,10 +991,15 @@ bool AArch64ExpandPseudo::expandStoreSwiftAsyncContext(
   return true;
 }
 
-MachineBasicBlock *
-AArch64ExpandPseudo::expandRestoreZA(MachineBasicBlock &MBB,
-                                     MachineBasicBlock::iterator MBBI) {
+static constexpr unsigned ZERO_ALL_ZA_MASK = 0b11111111;
+
+MachineBasicBlock *AArch64ExpandPseudo::expandCommitOrRestoreZASave(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
   MachineInstr &MI = *MBBI;
+  bool IsRestoreZA = MI.getOpcode() == AArch64::RestoreZAPseudo;
+  assert((MI.getOpcode() == AArch64::RestoreZAPseudo ||
+          MI.getOpcode() == AArch64::CommitZASavePseudo) &&
+         "Expected ZA commit or restore");
   assert((std::next(MBBI) != MBB.end() ||
           MI.getParent()->successors().begin() !=
               MI.getParent()->successors().end()) &&
@@ -990,21 +1007,23 @@ AArch64ExpandPseudo::expandRestoreZA(MachineBasicBlock &MBB,
 
   // Compare TPIDR2_EL0 value against 0.
   DebugLoc DL = MI.getDebugLoc();
-  MachineInstrBuilder Cbz = BuildMI(MBB, MBBI, DL, TII->get(AArch64::CBZX))
-                                .add(MI.getOperand(0));
+  MachineInstrBuilder Branch =
+      BuildMI(MBB, MBBI, DL,
+              TII->get(IsRestoreZA ? AArch64::CBZX : AArch64::CBNZX))
+          .add(MI.getOperand(0));
 
   // Split MBB and create two new blocks:
   //  - MBB now contains all instructions before RestoreZAPseudo.
-  //  - SMBB contains the RestoreZAPseudo instruction only.
-  //  - EndBB contains all instructions after RestoreZAPseudo.
+  //  - SMBB contains the [Commit|RestoreZA]Pseudo instruction only.
+  //  - EndBB contains all instructions after [Commit|RestoreZA]Pseudo.
   MachineInstr &PrevMI = *std::prev(MBBI);
   MachineBasicBlock *SMBB = MBB.splitAt(PrevMI, /*UpdateLiveIns*/ true);
   MachineBasicBlock *EndBB = std::next(MI.getIterator()) == SMBB->end()
                                  ? *SMBB->successors().begin()
                                  : SMBB->splitAt(MI, /*UpdateLiveIns*/ true);
 
-  // Add the SMBB label to the TB[N]Z instruction & create a branch to EndBB.
-  Cbz.addMBB(SMBB);
+  // Add the SMBB label to the CB[N]Z instruction & create a branch to EndBB.
+  Branch.addMBB(SMBB);
   BuildMI(&MBB, DL, TII->get(AArch64::B))
       .addMBB(EndBB);
   MBB.addSuccessor(EndBB);
@@ -1012,11 +1031,30 @@ AArch64ExpandPseudo::expandRestoreZA(MachineBasicBlock &MBB,
   // Replace the pseudo with a call (BL).
   MachineInstrBuilder MIB =
       BuildMI(*SMBB, SMBB->end(), DL, TII->get(AArch64::BL));
-  MIB.addReg(MI.getOperand(1).getReg(), RegState::Implicit);
+  // Copy operands (mainly the regmask) from the pseudo.
   for (unsigned I = 2; I < MI.getNumOperands(); ++I)
     MIB.add(MI.getOperand(I));
-  BuildMI(SMBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
 
+  if (IsRestoreZA) {
+    // Mark the TPIDR2 block pointer (X0) as an implicit use.
+    MIB.addReg(MI.getOperand(1).getReg(), RegState::Implicit);
+  } else /*CommitZA*/ {
+    [[maybe_unused]] auto *TRI =
+        MBB.getParent()->getSubtarget().getRegisterInfo();
+    // Clear TPIDR2_EL0.
+    BuildMI(*SMBB, SMBB->end(), DL, TII->get(AArch64::MSR))
+        .addImm(AArch64SysReg::TPIDR2_EL0)
+        .addReg(AArch64::XZR);
+    bool ZeroZA = MI.getOperand(1).getImm() != 0;
+    if (ZeroZA) {
+      assert(MI.definesRegister(AArch64::ZAB0, TRI) && "should define ZA!");
+      BuildMI(*SMBB, SMBB->end(), DL, TII->get(AArch64::ZERO_M))
+          .addImm(ZERO_ALL_ZA_MASK)
+          .addDef(AArch64::ZAB0, RegState::ImplicitDefine);
+    }
+  }
+
+  BuildMI(SMBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
   MI.eraseFromParent();
   return EndBB;
 }
@@ -1236,14 +1274,20 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
                 .add(MI.getOperand(3));
         transferImpOps(MI, I, I);
       } else {
+        unsigned RegState =
+            getRenamableRegState(MI.getOperand(1).isRenamable()) |
+            getKillRegState(
+                MI.getOperand(1).isKill() &&
+                MI.getOperand(1).getReg() != MI.getOperand(2).getReg() &&
+                MI.getOperand(1).getReg() != MI.getOperand(3).getReg());
         BuildMI(MBB, MBBI, MI.getDebugLoc(),
                 TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::ORRv8i8
                                                     : AArch64::ORRv16i8))
             .addReg(DstReg,
                     RegState::Define |
                         getRenamableRegState(MI.getOperand(0).isRenamable()))
-            .add(MI.getOperand(1))
-            .add(MI.getOperand(1));
+            .addReg(MI.getOperand(1).getReg(), RegState)
+            .addReg(MI.getOperand(1).getReg(), RegState);
         auto I2 =
             BuildMI(MBB, MBBI, MI.getDebugLoc(),
                     TII->get(Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
@@ -1629,8 +1673,9 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
      return expandCALL_BTI(MBB, MBBI);
    case AArch64::StoreSwiftAsyncContext:
      return expandStoreSwiftAsyncContext(MBB, MBBI);
+   case AArch64::CommitZASavePseudo:
    case AArch64::RestoreZAPseudo: {
-     auto *NewMBB = expandRestoreZA(MBB, MBBI);
+     auto *NewMBB = expandCommitOrRestoreZASave(MBB, MBBI);
      if (NewMBB != &MBB)
        NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated.
      return true;
@@ -1641,6 +1686,8 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
        NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated.
      return true;
    }
+   case AArch64::InOutZAUsePseudo:
+   case AArch64::RequiresZASavePseudo:
    case AArch64::COALESCER_BARRIER_FPR16:
    case AArch64::COALESCER_BARRIER_FPR32:
    case AArch64::COALESCER_BARRIER_FPR64:
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index 9d74bb5a8661d..cf344980cbaae 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -267,7 +267,7 @@ class AArch64FastISel final : public FastISel {
 private:
   CCAssignFn *CCAssignFnForCall(CallingConv::ID CC) const;
   bool processCallArgs(CallLoweringInfo &CLI, SmallVectorImpl<MVT> &ArgVTs,
-                       unsigned &NumBytes);
+                       SmallVectorImpl<Type *> &OrigTys, unsigned &NumBytes);
   bool finishCall(CallLoweringInfo &CLI, unsigned NumBytes);
 
 public:
@@ -3011,11 +3011,13 @@ bool AArch64FastISel::fastLowerArguments() {
 
 bool AArch64FastISel::processCallArgs(CallLoweringInfo &CLI,
                                       SmallVectorImpl<MVT> &OutVTs,
+                                      SmallVectorImpl<Type *> &OrigTys,
                                       unsigned &NumBytes) {
   CallingConv::ID CC = CLI.CallConv;
   SmallVector<CCValAssign, 16> ArgLocs;
   CCState CCInfo(CC, false, *FuncInfo.MF, ArgLocs, *Context);
-  CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, CCAssignFnForCall(CC));
+  CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, OrigTys,
+                             CCAssignFnForCall(CC));
 
   // Get a count of how many bytes are to be pushed on the stack.
   NumBytes = CCInfo.getStackSize();
@@ -3194,6 +3196,7 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
 
   // Set up the argument vectors.
   SmallVector<MVT, 16> OutVTs;
+  SmallVector<Type *, 16> OrigTys;
   OutVTs.reserve(CLI.OutVals.size());
 
   for (auto *Val : CLI.OutVals) {
@@ -3207,6 +3210,7 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
       return false;
 
     OutVTs.push_back(VT);
+    OrigTys.push_back(Val->getType());
   }
 
   Address Addr;
@@ -3222,7 +3226,7 @@ bool AArch64FastISel::fastLowerCall(CallLoweringInfo &CLI) {
 
   // Handle the arguments now that we've gotten them.
   unsigned NumBytes;
-  if (!processCallArgs(CLI, OutVTs, NumBytes))
+  if (!processCallArgs(CLI, OutVTs, OrigTys, NumBytes))
     return false;
 
   const AArch64RegisterInfo *RegInfo = Subtarget->getRegisterInfo();
@@ -3574,12 +3578,8 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
     Args.reserve(II->arg_size());
 
     // Populate the argument list.
-    for (auto &Arg : II->args()) {
-      ArgListEntry Entry;
-      Entry.Val = Arg;
-      Entry.Ty = Arg->getType();
-      Args.push_back(Entry);
-    }
+    for (auto &Arg : II->args())
+      Args.emplace_back(Arg);
 
     CallLoweringInfo CLI;
     MCContext &Ctx = MF->getContext();
@@ -4870,12 +4870,8 @@ bool AArch64FastISel::selectFRem(const Instruction *I) {
   Args.reserve(I->getNumOperands());
 
   // Populate the argument list.
-  for (auto &Arg : I->operands()) {
-    ArgListEntry Entry;
-    Entry.Val = Arg;
-    Entry.Ty = Arg->getType();
-    Args.push_back(Entry);
-  }
+  for (auto &Arg : I->operands())
+    Args.emplace_back(Arg);
 
   CallLoweringInfo CLI;
   MCContext &Ctx = MF->getContext();
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 885f2a94f85f5..7725fa4f1ccb1 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -338,7 +338,7 @@ static bool requiresSaveVG(const MachineFunction &MF);
 // Conservatively, returns true if the function is likely to have an SVE vectors
 // on the stack. This function is safe to be called before callee-saves or
 // object offsets have been determined.
-static bool isLikelyToHaveSVEStack(MachineFunction &MF) {
+static bool isLikelyToHaveSVEStack(const MachineFunction &MF) {
   auto *AFI = MF.getInfo<AArch64FunctionInfo>();
   if (AFI->isSVECC())
     return true;
@@ -532,6 +532,7 @@ bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const {
 bool AArch64FrameLowering::hasFPImpl(const MachineFunction &MF) const {
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
+  const AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
 
   // Win64 EH requires a frame pointer if funclets are present, as the locals
   // are accessed off the frame pointer in both the parent function and the
@@ -545,6 +546,29 @@ bool AArch64FrameLowering::hasFPImpl(const MachineFunction &MF) const {
       MFI.hasStackMap() || MFI.hasPatchPoint() ||
       RegInfo->hasStackRealignment(MF))
     return true;
+
+  // If we:
+  //
+  //   1. Have streaming mode changes
+  //     OR:
+  //   2. Have a streaming body with SVE stack objects
+  //
+  // Then the value of VG restored when unwinding to this function may not match
+  // the value of VG used to set up the stack.
+  //
+  // This is a problem as the CFA can be described with an expression of the
+  // form: CFA = SP + NumBytes + VG * NumScalableBytes.
+  //
+  // If the value of VG used in that expression does not match the value used to
+  // set up the stack, an incorrect address for the CFA will be computed, and
+  // unwinding will fail.
+  //
+  // We work around this issue by ensuring the frame-pointer can describe the
+  // CFA in either of these cases.
+  if (AFI.needsDwarfUnwindInfo(MF) &&
+      ((requiresSaveVG(MF) || AFI.getSMEFnAttrs().hasStreamingBody()) &&
+       (!AFI.hasCalculatedStackSizeSVE() || AFI.getStackSizeSVE() > 0)))
+    return true;
   // With large callframes around we may need to use FP to access the scavenging
   // emergency spillslot.
   //
@@ -663,10 +687,6 @@ void AArch64FrameLowering::emitCalleeSavedGPRLocations(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
   MachineFunction &MF = *MBB.getParent();
   MachineFrameInfo &MFI = MF.getFrameInfo();
-  AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
-  SMEAttrs Attrs = AFI->getSMEFnAttrs();
-  bool LocallyStreaming =
-      Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface();
 
   const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
   if (CSI.empty())
@@ -680,14 +700,6 @@ void AArch64FrameLowering::emitCalleeSavedGPRLocations(
 
     assert(!Info.isSpilledToReg() && "Spilling to registers not implemented");
     int64_t Offset = MFI.getObjectOffset(FrameIdx) - getOffsetOfLocalArea();
-
-    // The location of VG will be emitted before each streaming-mode change in
-    // the function. Only locally-streaming functions require emitting the
-    // non-streaming VG location here.
-    if ((LocallyStreaming && FrameIdx == AFI->getStreamingVGIdx()) ||
-        (!LocallyStreaming && Info.getReg() == AArch64::VG))
-      continue;
-
     CFIBuilder.buildOffset(Info.getReg(), Offset);
   }
 }
@@ -707,8 +719,16 @@ void AArch64FrameLowering::emitCalleeSavedSVELocations(
   AArch64FunctionInfo &AFI = *MF.getInfo<AArch64FunctionInfo>();
   CFIInstBuilder CFIBuilder(MBB, MBBI, MachineInstr::FrameSetup);
 
+  std::optional<int64_t> IncomingVGOffsetFromDefCFA;
+  if (requiresSaveVG(MF)) {
+    auto IncomingVG = *find_if(
+        reverse(CSI), [](auto &Info) { return Info.getReg() == AArch64::VG; });
+    IncomingVGOffsetFromDefCFA =
+        MFI.getObjectOffset(IncomingVG.getFrameIdx()) - getOffsetOfLocalArea();
+  }
+
   for (const auto &Info : CSI) {
-    if (!(MFI.getStackID(Info.getFrameIdx()) == TargetStackID::ScalableVector))
+    if (MFI.getStackID(Info.getFrameIdx()) != TargetStackID::ScalableVector)
       continue;
 
     // Not all unwinders may know about SVE registers, so assume the lowest
@@ -722,7 +742,8 @@ void AArch64FrameLowering::emitCalleeSavedSVELocations(
         StackOffset::getScalable(MFI.getObjectOffset(Info.getFrameIdx())) -
         StackOffset::getFixed(AFI.getCalleeSavedStackSize(MFI));
 
-    CFIBuilder.insertCFIInst(createCFAOffset(TRI, Reg, Offset));
+    CFIBuilder.insertCFIInst(
+        createCFAOffset(TRI, Reg, Offset, IncomingVGOffsetFromDefCFA));
   }
 }
 
@@ -783,9 +804,6 @@ static void emitCalleeSavedRestores(MachineBasicBlock &MBB,
         !static_cast<const AArch64RegisterInfo &>(TRI).regNeedsCFI(Reg, Reg))
       continue;
 
-    if (!Info.isRestored())
-      continue;
-
     CFIBuilder.buildRestore(Info.getReg());
   }
 }
@@ -1465,34 +1483,35 @@ bool requiresGetVGCall(MachineFunction &MF) {
 
 static bool requiresSaveVG(const MachineFunction &MF) {
   const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
+  if (!AFI->needsDwarfUnwindInfo(MF) || !AFI->hasStreamingModeChanges())
+    return false;
   // For Darwin platforms we don't save VG for non-SVE functions, even if SME
   // is enabled with streaming mode changes.
-  if (!AFI->hasStreamingModeChanges())
-    return false;
   auto &ST = MF.getSubtarget<AArch64Subtarget>();
   if (ST.isTargetDarwin())
     return ST.hasSVE();
   return true;
 }
 
-bool isVGInstruction(MachineBasicBlock::iterator MBBI) {
+static bool matchLibcall(const TargetLowering &TLI, const MachineOperand &MO,
+                         RTLIB::Libcall LC) {
+  return MO.isSymbol() &&
+         StringRef(TLI.getLibcallName(LC)) == MO.getSymbolName();
+}
+
+bool isVGInstruction(MachineBasicBlock::iterator MBBI,
+                     const TargetLowering &TLI) {
   unsigned Opc = MBBI->getOpcode();
-  if (Opc == AArch64::CNTD_XPiI || Opc == AArch64::RDSVLI_XI ||
-      Opc == AArch64::UBFMXri)
+  if (Opc == AArch64::CNTD_XPiI)
     return true;
 
-  if (requiresGetVGCall(*MBBI->getMF())) {
-    if (Opc == AArch64::ORRXrr)
-      return true;
+  if (!requiresGetVGCall(*MBBI->getMF()))
+    return false;
 
-    if (Opc == AArch64::BL) {
-      auto Op1 = MBBI->getOperand(0);
-      return Op1.isSymbol() &&
-             (StringRef(Op1.getSymbolName()) == "__arm_get_current_vg");
-    }
-  }
+  if (Opc == AArch64::BL)
+    return matchLibcall(TLI, MBBI->getOperand(0), RTLIB::SMEABI_GET_CURRENT_VG);
 
-  return false;
+  return Opc == TargetOpcode::COPY;
 }
 
 // Convert callee-save register save/restore instruction to do stack pointer
@@ -1507,13 +1526,14 @@ static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
   unsigned NewOpc;
 
   // If the function contains streaming mode changes, we expect instructions
-  // to calculate the value of VG before spilling. For locally-streaming
-  // functions, we need to do this for both the streaming and non-streaming
-  // vector length. Move past these instructions if necessary.
+  // to calculate the value of VG before spilling. Move past these instructions
+  // if necessary.
   MachineFunction &MF = *MBB.getParent();
-  if (requiresSaveVG(MF))
-    while (isVGInstruction(MBBI))
+  if (requiresSaveVG(MF)) {
+    auto &TLI = *MF.getSubtarget().getTargetLowering();
+    while (isVGInstruction(MBBI, TLI))
       ++MBBI;
+  }
 
   switch (MBBI->getOpcode()) {
   default:
@@ -2097,11 +2117,12 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
   // Move past the saves of the callee-saved registers, fixing up the offsets
   // and pre-inc if we decided to combine the callee-save and local stack
   // pointer bump above.
+  auto &TLI = *MF.getSubtarget().getTargetLowering();
   while (MBBI != End && MBBI->getFlag(MachineInstr::FrameSetup) &&
          !IsSVECalleeSave(MBBI)) {
     if (CombineSPBump &&
         // Only fix-up frame-setup load/store instructions.
-        (!requiresSaveVG(MF) || !isVGInstruction(MBBI)))
+        (!requiresSaveVG(MF) || !isVGInstruction(MBBI, TLI)))
       fixupCalleeSaveRestoreStackOffset(*MBBI, AFI->getLocalStackSize(),
                                         NeedsWinCFI, &HasWinCFI);
     ++MBBI;
@@ -3468,8 +3489,8 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
     ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const {
   MachineFunction &MF = *MBB.getParent();
+  auto &TLI = *MF.getSubtarget<AArch64Subtarget>().getTargetLowering();
   const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
-  AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
   bool NeedsWinCFI = needsWinCFI(MF);
   DebugLoc DL;
   SmallVector<RegPairInfo, 8> RegPairs;
@@ -3538,59 +3559,44 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
     }
 
     unsigned X0Scratch = AArch64::NoRegister;
+    auto RestoreX0 = make_scope_exit([&] {
+      if (X0Scratch != AArch64::NoRegister)
+        BuildMI(MBB, MI, DL, TII.get(TargetOpcode::COPY), AArch64::X0)
+            .addReg(X0Scratch)
+            .setMIFlag(MachineInstr::FrameSetup);
+    });
+
     if (Reg1 == AArch64::VG) {
       // Find an available register to store value of VG to.
       Reg1 = findScratchNonCalleeSaveRegister(&MBB, true);
       assert(Reg1 != AArch64::NoRegister);
-      SMEAttrs Attrs = AFI->getSMEFnAttrs();
-
-      if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface() &&
-          AFI->getStreamingVGIdx() == std::numeric_limits<int>::max()) {
-        // For locally-streaming functions, we need to store both the streaming
-        // & non-streaming VG. Spill the streaming value first.
-        BuildMI(MBB, MI, DL, TII.get(AArch64::RDSVLI_XI), Reg1)
-            .addImm(1)
-            .setMIFlag(MachineInstr::FrameSetup);
-        BuildMI(MBB, MI, DL, TII.get(AArch64::UBFMXri), Reg1)
-            .addReg(Reg1)
-            .addImm(3)
-            .addImm(63)
-            .setMIFlag(MachineInstr::FrameSetup);
-
-        AFI->setStreamingVGIdx(RPI.FrameIdx);
-      } else if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
+      if (MF.getSubtarget<AArch64Subtarget>().hasSVE()) {
         BuildMI(MBB, MI, DL, TII.get(AArch64::CNTD_XPiI), Reg1)
             .addImm(31)
             .addImm(1)
             .setMIFlag(MachineInstr::FrameSetup);
-        AFI->setVGIdx(RPI.FrameIdx);
       } else {
         const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
-        if (llvm::any_of(
-                MBB.liveins(),
-                [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) {
-                  return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
-                      AArch64::X0, LiveIn.PhysReg);
-                }))
+        if (any_of(MBB.liveins(),
+                   [&STI](const MachineBasicBlock::RegisterMaskPair &LiveIn) {
+                     return STI.getRegisterInfo()->isSuperOrSubRegisterEq(
+                         AArch64::X0, LiveIn.PhysReg);
+                   })) {
           X0Scratch = Reg1;
-
-        if (X0Scratch != AArch64::NoRegister)
-          BuildMI(MBB, MI, DL, TII.get(AArch64::ORRXrr), Reg1)
-              .addReg(AArch64::XZR)
-              .addReg(AArch64::X0, RegState::Undef)
-              .addReg(AArch64::X0, RegState::Implicit)
+          BuildMI(MBB, MI, DL, TII.get(TargetOpcode::COPY), X0Scratch)
+              .addReg(AArch64::X0)
               .setMIFlag(MachineInstr::FrameSetup);
+        }
 
-        const uint32_t *RegMask = TRI->getCallPreservedMask(
-            MF,
-            CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1);
+        RTLIB::Libcall LC = RTLIB::SMEABI_GET_CURRENT_VG;
+        const uint32_t *RegMask =
+            TRI->getCallPreservedMask(MF, TLI.getLibcallCallingConv(LC));
         BuildMI(MBB, MI, DL, TII.get(AArch64::BL))
-            .addExternalSymbol("__arm_get_current_vg")
+            .addExternalSymbol(TLI.getLibcallName(LC))
             .addRegMask(RegMask)
             .addReg(AArch64::X0, RegState::ImplicitDefine)
             .setMIFlag(MachineInstr::FrameSetup);
         Reg1 = AArch64::X0;
-        AFI->setVGIdx(RPI.FrameIdx);
       }
     }
 
@@ -3685,13 +3691,6 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
       if (RPI.isPaired())
         MFI.setStackID(FrameIdxReg2, TargetStackID::ScalableVector);
     }
-
-    if (X0Scratch != AArch64::NoRegister)
-      BuildMI(MBB, MI, DL, TII.get(AArch64::ORRXrr), AArch64::X0)
-          .addReg(AArch64::XZR)
-          .addReg(X0Scratch, RegState::Undef)
-          .addReg(X0Scratch, RegState::Implicit)
-          .setMIFlag(MachineInstr::FrameSetup);
   }
   return true;
 }
@@ -4070,15 +4069,8 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
 
   // Increase the callee-saved stack size if the function has streaming mode
   // changes, as we will need to spill the value of the VG register.
-  // For locally streaming functions, we spill both the streaming and
-  // non-streaming VG value.
-  SMEAttrs Attrs = AFI->getSMEFnAttrs();
-  if (requiresSaveVG(MF)) {
-    if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
-      CSStackSize += 16;
-    else
-      CSStackSize += 8;
-  }
+  if (requiresSaveVG(MF))
+    CSStackSize += 8;
 
   // Determine if a Hazard slot should be used, and increase the CSStackSize by
   // StackHazardSize if so.
@@ -4229,29 +4221,13 @@ bool AArch64FrameLowering::assignCalleeSavedSpillSlots(
 
   // Insert VG into the list of CSRs, immediately before LR if saved.
   if (requiresSaveVG(MF)) {
-    std::vector<CalleeSavedInfo> VGSaves;
-    SMEAttrs Attrs = AFI->getSMEFnAttrs();
-
-    auto VGInfo = CalleeSavedInfo(AArch64::VG);
-    VGInfo.setRestored(false);
-    VGSaves.push_back(VGInfo);
-
-    // Add VG again if the function is locally-streaming, as we will spill two
-    // values.
-    if (Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface())
-      VGSaves.push_back(VGInfo);
-
-    bool InsertBeforeLR = false;
-
-    for (unsigned I = 0; I < CSI.size(); I++)
-      if (CSI[I].getReg() == AArch64::LR) {
-        InsertBeforeLR = true;
-        CSI.insert(CSI.begin() + I, VGSaves.begin(), VGSaves.end());
-        break;
-      }
-
-    if (!InsertBeforeLR)
-      llvm::append_range(CSI, VGSaves);
+    CalleeSavedInfo VGInfo(AArch64::VG);
+    auto It =
+        find_if(CSI, [](auto &Info) { return Info.getReg() == AArch64::LR; });
+    if (It != CSI.end())
+      CSI.insert(It, VGInfo);
+    else
+      CSI.push_back(VGInfo);
   }
 
   Register LastReg = 0;
@@ -5254,46 +5230,11 @@ MachineBasicBlock::iterator tryMergeAdjacentSTG(MachineBasicBlock::iterator II,
 }
 } // namespace
 
-static void emitVGSaveRestore(MachineBasicBlock::iterator II,
-                              const AArch64FrameLowering *TFI) {
-  MachineInstr &MI = *II;
-  MachineBasicBlock *MBB = MI.getParent();
-  MachineFunction *MF = MBB->getParent();
-
-  if (MI.getOpcode() != AArch64::VGSavePseudo &&
-      MI.getOpcode() != AArch64::VGRestorePseudo)
-    return;
-
-  auto *AFI = MF->getInfo<AArch64FunctionInfo>();
-  SMEAttrs FuncAttrs = AFI->getSMEFnAttrs();
-  bool LocallyStreaming =
-      FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
-
-  int64_t VGFrameIdx =
-      LocallyStreaming ? AFI->getStreamingVGIdx() : AFI->getVGIdx();
-  assert(VGFrameIdx != std::numeric_limits<int>::max() &&
-         "Expected FrameIdx for VG");
-
-  CFIInstBuilder CFIBuilder(*MBB, II, MachineInstr::NoFlags);
-  if (MI.getOpcode() == AArch64::VGSavePseudo) {
-    const MachineFrameInfo &MFI = MF->getFrameInfo();
-    int64_t Offset =
-        MFI.getObjectOffset(VGFrameIdx) - TFI->getOffsetOfLocalArea();
-    CFIBuilder.buildOffset(AArch64::VG, Offset);
-  } else {
-    CFIBuilder.buildRestore(AArch64::VG);
-  }
-
-  MI.eraseFromParent();
-}
-
 void AArch64FrameLowering::processFunctionBeforeFrameIndicesReplaced(
     MachineFunction &MF, RegScavenger *RS = nullptr) const {
   for (auto &BB : MF)
     for (MachineBasicBlock::iterator II = BB.begin(); II != BB.end();) {
-      if (requiresSaveVG(MF))
-        emitVGSaveRestore(II++, this);
-      else if (StackTaggingMergeSetTag)
+      if (StackTaggingMergeSetTag)
         II = tryMergeAdjacentSTG(II, this, RS);
     }
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index ad42f4b56caf2..bc786f415b554 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -7617,16 +7617,29 @@ bool AArch64DAGToDAGISel::SelectAnyPredicate(SDValue N) {
 bool AArch64DAGToDAGISel::SelectSMETileSlice(SDValue N, unsigned MaxSize,
                                              SDValue &Base, SDValue &Offset,
                                              unsigned Scale) {
-  // Try to untangle an ADD node into a 'reg + offset'
-  if (CurDAG->isBaseWithConstantOffset(N))
-    if (auto C = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
+  auto MatchConstantOffset = [&](SDValue CN) -> SDValue {
+    if (auto *C = dyn_cast<ConstantSDNode>(CN)) {
       int64_t ImmOff = C->getSExtValue();
-      if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0))) {
-        Base = N.getOperand(0);
-        Offset = CurDAG->getTargetConstant(ImmOff / Scale, SDLoc(N), MVT::i64);
-        return true;
-      }
+      if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0)))
+        return CurDAG->getTargetConstant(ImmOff / Scale, SDLoc(N), MVT::i64);
     }
+    return SDValue();
+  };
+
+  if (SDValue C = MatchConstantOffset(N)) {
+    Base = CurDAG->getConstant(0, SDLoc(N), MVT::i32);
+    Offset = C;
+    return true;
+  }
+
+  // Try to untangle an ADD node into a 'reg + offset'
+  if (CurDAG->isBaseWithConstantOffset(N)) {
+    if (SDValue C = MatchConstantOffset(N.getOperand(1))) {
+      Base = N.getOperand(0);
+      Offset = C;
+      return true;
+    }
+  }
 
   // By default, just match reg + 0.
   Base = N;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 9f6e552ea8916..d70a46b0e8939 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -17,6 +17,7 @@
 #include "AArch64PerfectShuffle.h"
 #include "AArch64RegisterInfo.h"
 #include "AArch64Subtarget.h"
+#include "AArch64TargetMachine.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "Utils/AArch64BaseInfo.h"
 #include "Utils/AArch64SMEAttributes.h"
@@ -1120,7 +1121,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
   setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
 
-  // We combine OR nodes for bitfield operations.
+  // We combine OR nodes for ccmp operations.
   setTargetDAGCombine(ISD::OR);
   // Try to create BICs for vector ANDs.
   setTargetDAGCombine(ISD::AND);
@@ -1769,7 +1770,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
       setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
 
-      if (Subtarget->hasSVEB16B16()) {
+      if (Subtarget->hasSVEB16B16() &&
+          Subtarget->isNonStreamingSVEorSME2Available()) {
         setOperationAction(ISD::FADD, VT, Legal);
         setOperationAction(ISD::FMA, VT, Custom);
         setOperationAction(ISD::FMAXIMUM, VT, Custom);
@@ -1791,7 +1793,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationPromotedToType(Opcode, MVT::nxv8bf16, MVT::nxv8f32);
     }
 
-    if (!Subtarget->hasSVEB16B16()) {
+    if (!Subtarget->hasSVEB16B16() ||
+        !Subtarget->isNonStreamingSVEorSME2Available()) {
       for (auto Opcode : {ISD::FADD, ISD::FMA, ISD::FMAXIMUM, ISD::FMAXNUM,
                           ISD::FMINIMUM, ISD::FMINNUM, ISD::FMUL, ISD::FSUB}) {
         setOperationPromotedToType(Opcode, MVT::nxv2bf16, MVT::nxv2f32);
@@ -1998,6 +2001,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(Op, MVT::f16, Promote);
 }
 
+const AArch64TargetMachine &AArch64TargetLowering::getTM() const {
+  return static_cast<const AArch64TargetMachine &>(getTargetMachine());
+}
+
 void AArch64TargetLowering::addTypeForNEON(MVT VT) {
   assert(VT.isVector() && "VT should be a vector type");
 
@@ -3083,13 +3090,12 @@ AArch64TargetLowering::EmitGetSMESaveSize(MachineInstr &MI,
   AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
   const TargetInstrInfo *TII = Subtarget->getInstrInfo();
   if (FuncInfo->isSMESaveBufferUsed()) {
+    RTLIB::Libcall LC = RTLIB::SMEABI_SME_STATE_SIZE;
     const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
     BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::BL))
-        .addExternalSymbol("__arm_sme_state_size")
+        .addExternalSymbol(getLibcallName(LC))
         .addReg(AArch64::X0, RegState::ImplicitDefine)
-        .addRegMask(TRI->getCallPreservedMask(
-            *MF, CallingConv::
-                     AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1));
+        .addRegMask(TRI->getCallPreservedMask(*MF, getLibcallCallingConv(LC)));
     BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(TargetOpcode::COPY),
             MI.getOperand(0).getReg())
         .addReg(AArch64::X0);
@@ -3101,6 +3107,30 @@ AArch64TargetLowering::EmitGetSMESaveSize(MachineInstr &MI,
   return BB;
 }
 
+MachineBasicBlock *
+AArch64TargetLowering::EmitEntryPStateSM(MachineInstr &MI,
+                                         MachineBasicBlock *BB) const {
+  MachineFunction *MF = BB->getParent();
+  AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
+  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
+  Register ResultReg = MI.getOperand(0).getReg();
+  if (FuncInfo->isPStateSMRegUsed()) {
+    RTLIB::Libcall LC = RTLIB::SMEABI_SME_STATE;
+    const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
+    BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::BL))
+        .addExternalSymbol(getLibcallName(LC))
+        .addReg(AArch64::X0, RegState::ImplicitDefine)
+        .addRegMask(TRI->getCallPreservedMask(*MF, getLibcallCallingConv(LC)));
+    BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(TargetOpcode::COPY), ResultReg)
+        .addReg(AArch64::X0);
+  } else {
+    assert(MI.getMF()->getRegInfo().use_empty(ResultReg) &&
+           "Expected no users of the entry pstate.sm!");
+  }
+  MI.eraseFromParent();
+  return BB;
+}
+
 // Helper function to find the instruction that defined a virtual register.
 // If unable to find such instruction, returns nullptr.
 static const MachineInstr *stripVRegCopies(const MachineRegisterInfo &MRI,
@@ -3216,6 +3246,8 @@ MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
     return EmitAllocateSMESaveBuffer(MI, BB);
   case AArch64::GetSMESaveSize:
     return EmitGetSMESaveSize(MI, BB);
+  case AArch64::EntryPStateSM:
+    return EmitEntryPStateSM(MI, BB);
   case AArch64::F128CSEL:
     return EmitF128CSEL(MI, BB);
   case TargetOpcode::STATEPOINT:
@@ -3320,7 +3352,8 @@ static bool isZerosVector(const SDNode *N) {
 
 /// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
 /// CC
-static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
+static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC,
+                                                  SDValue RHS = {}) {
   switch (CC) {
   default:
     llvm_unreachable("Unknown condition code!");
@@ -3331,9 +3364,9 @@ static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
   case ISD::SETGT:
     return AArch64CC::GT;
   case ISD::SETGE:
-    return AArch64CC::GE;
+    return (RHS && isNullConstant(RHS)) ? AArch64CC::PL : AArch64CC::GE;
   case ISD::SETLT:
-    return AArch64CC::LT;
+    return (RHS && isNullConstant(RHS)) ? AArch64CC::MI : AArch64CC::LT;
   case ISD::SETLE:
     return AArch64CC::LE;
   case ISD::SETUGT:
@@ -3492,6 +3525,13 @@ bool isLegalCmpImmed(APInt C) {
   return isLegalArithImmed(C.abs().getZExtValue());
 }
 
+unsigned numberOfInstrToLoadImm(APInt C) {
+  uint64_t Imm = C.getZExtValue();
+  SmallVector<AArch64_IMM::ImmInsnModel> Insn;
+  AArch64_IMM::expandMOVImm(Imm, 32, Insn);
+  return Insn.size();
+}
+
 static bool isSafeSignedCMN(SDValue Op, SelectionDAG &DAG) {
   // 0 - INT_MIN sign wraps, so no signed wrap means cmn is safe.
   if (Op->getFlags().hasNoSignedWrap())
@@ -3782,7 +3822,7 @@ static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
     SDLoc DL(Val);
     // Determine OutCC and handle FP special case.
     if (isInteger) {
-      OutCC = changeIntCCToAArch64CC(CC);
+      OutCC = changeIntCCToAArch64CC(CC, RHS);
     } else {
       assert(LHS.getValueType().isFloatingPoint());
       AArch64CC::CondCode ExtraCC;
@@ -3961,6 +4001,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
       // CC has already been adjusted.
       RHS = DAG.getConstant(0, DL, VT);
     } else if (!isLegalCmpImmed(C)) {
+      unsigned NumImmForC = numberOfInstrToLoadImm(C);
       // Constant does not fit, try adjusting it by one?
       switch (CC) {
       default:
@@ -3969,43 +4010,49 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
       case ISD::SETGE:
         if (!C.isMinSignedValue()) {
           APInt CMinusOne = C - 1;
-          if (isLegalCmpImmed(CMinusOne)) {
+          if (isLegalCmpImmed(CMinusOne) ||
+              (NumImmForC > numberOfInstrToLoadImm(CMinusOne))) {
             CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
             RHS = DAG.getConstant(CMinusOne, DL, VT);
           }
         }
         break;
       case ISD::SETULT:
-      case ISD::SETUGE:
-        if (!C.isZero()) {
-          APInt CMinusOne = C - 1;
-          if (isLegalCmpImmed(CMinusOne)) {
-            CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
-            RHS = DAG.getConstant(CMinusOne, DL, VT);
-          }
+      case ISD::SETUGE: {
+        // C is not 0 because it is a legal immediate.
+        assert(!C.isZero() && "C should not be zero here");
+        APInt CMinusOne = C - 1;
+        if (isLegalCmpImmed(CMinusOne) ||
+            (NumImmForC > numberOfInstrToLoadImm(CMinusOne))) {
+          CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
+          RHS = DAG.getConstant(CMinusOne, DL, VT);
         }
         break;
+      }
       case ISD::SETLE:
       case ISD::SETGT:
         if (!C.isMaxSignedValue()) {
           APInt CPlusOne = C + 1;
-          if (isLegalCmpImmed(CPlusOne)) {
+          if (isLegalCmpImmed(CPlusOne) ||
+              (NumImmForC > numberOfInstrToLoadImm(CPlusOne))) {
             CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
             RHS = DAG.getConstant(CPlusOne, DL, VT);
           }
         }
         break;
       case ISD::SETULE:
-      case ISD::SETUGT:
+      case ISD::SETUGT: {
         if (!C.isAllOnes()) {
           APInt CPlusOne = C + 1;
-          if (isLegalCmpImmed(CPlusOne)) {
+          if (isLegalCmpImmed(CPlusOne) ||
+              (NumImmForC > numberOfInstrToLoadImm(CPlusOne))) {
             CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
             RHS = DAG.getConstant(CPlusOne, DL, VT);
           }
         }
         break;
       }
+      }
     }
   }
 
@@ -4079,7 +4126,7 @@ static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
 
   if (!Cmp) {
     Cmp = emitComparison(LHS, RHS, CC, DL, DAG);
-    AArch64CC = changeIntCCToAArch64CC(CC);
+    AArch64CC = changeIntCCToAArch64CC(CC, RHS);
   }
   AArch64cc = getCondCode(DAG, AArch64CC);
   return Cmp;
@@ -5174,13 +5221,7 @@ SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
 
   ArgListTy Args;
-  ArgListEntry Entry;
-
-  Entry.Node = Arg;
-  Entry.Ty = ArgTy;
-  Entry.IsSExt = false;
-  Entry.IsZExt = false;
-  Args.push_back(Entry);
+  Args.emplace_back(Arg, ArgTy);
 
   RTLIB::Libcall LC = ArgVT == MVT::f64 ? RTLIB::SINCOS_STRET_F64
                                         : RTLIB::SINCOS_STRET_F32;
@@ -5711,15 +5752,15 @@ static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) {
 SDValue AArch64TargetLowering::getRuntimePStateSM(SelectionDAG &DAG,
                                                   SDValue Chain, SDLoc DL,
                                                   EVT VT) const {
-  SDValue Callee = DAG.getExternalSymbol("__arm_sme_state",
+  RTLIB::Libcall LC = RTLIB::SMEABI_SME_STATE;
+  SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
                                          getPointerTy(DAG.getDataLayout()));
   Type *Int64Ty = Type::getInt64Ty(*DAG.getContext());
   Type *RetTy = StructType::get(Int64Ty, Int64Ty);
   TargetLowering::CallLoweringInfo CLI(DAG);
   ArgListTy Args;
   CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
-      CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X2,
-      RetTy, Callee, std::move(Args));
+      getLibcallCallingConv(LC), RetTy, Callee, std::move(Args));
   std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
   SDValue Mask = DAG.getConstant(/*PSTATE.SM*/ 1, DL, MVT::i64);
   return DAG.getNode(ISD::AND, DL, MVT::i64, CallResult.first.getOperand(0),
@@ -7886,8 +7927,8 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
       else if (ActualMVT == MVT::i16)
         ValVT = MVT::i16;
     }
-    bool Res =
-        AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, CCInfo);
+    bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags,
+                        Ins[i].OrigTy, CCInfo);
     assert(!Res && "Call operand has unhandled type");
     (void)Res;
   }
@@ -8132,19 +8173,26 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
   }
   assert((ArgLocs.size() + ExtraArgLocs) == Ins.size());
 
+  if (Attrs.hasStreamingCompatibleInterface()) {
+    SDValue EntryPStateSM =
+        DAG.getNode(AArch64ISD::ENTRY_PSTATE_SM, DL,
+                    DAG.getVTList(MVT::i64, MVT::Other), {Chain});
+
+    // Copy the value to a virtual register, and save that in FuncInfo.
+    Register EntryPStateSMReg =
+        MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
+    Chain = DAG.getCopyToReg(EntryPStateSM.getValue(1), DL, EntryPStateSMReg,
+                             EntryPStateSM);
+    FuncInfo->setPStateSMReg(EntryPStateSMReg);
+  }
+
   // Insert the SMSTART if this is a locally streaming function and
   // make sure it is Glued to the last CopyFromReg value.
   if (IsLocallyStreaming) {
-    SDValue PStateSM;
-    if (Attrs.hasStreamingCompatibleInterface()) {
-      PStateSM = getRuntimePStateSM(DAG, Chain, DL, MVT::i64);
-      Register Reg = MF.getRegInfo().createVirtualRegister(
-          getRegClassFor(PStateSM.getValueType().getSimpleVT()));
-      FuncInfo->setPStateSMReg(Reg);
-      Chain = DAG.getCopyToReg(Chain, DL, Reg, PStateSM);
+    if (Attrs.hasStreamingCompatibleInterface())
       Chain = changeStreamingMode(DAG, DL, /*Enable*/ true, Chain, Glue,
-                                  AArch64SME::IfCallerIsNonStreaming, PStateSM);
-    } else
+                                  AArch64SME::IfCallerIsNonStreaming);
+    else
       Chain = changeStreamingMode(DAG, DL, /*Enable*/ true, Chain, Glue,
                                   AArch64SME::Always);
 
@@ -8244,53 +8292,54 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
   if (Subtarget->hasCustomCallingConv())
     Subtarget->getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
 
-  // Create a 16 Byte TPIDR2 object. The dynamic buffer
-  // will be expanded and stored in the static object later using a pseudonode.
-  if (Attrs.hasZAState()) {
-    TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj();
-    TPIDR2.FrameIndex = MFI.CreateStackObject(16, Align(16), false);
-    SDValue SVL = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64,
-                              DAG.getConstant(1, DL, MVT::i32));
-
-    SDValue Buffer;
-    if (!Subtarget->isTargetWindows() && !hasInlineStackProbe(MF)) {
-      Buffer = DAG.getNode(AArch64ISD::ALLOCATE_ZA_BUFFER, DL,
-                           DAG.getVTList(MVT::i64, MVT::Other), {Chain, SVL});
-    } else {
-      SDValue Size = DAG.getNode(ISD::MUL, DL, MVT::i64, SVL, SVL);
-      Buffer = DAG.getNode(ISD::DYNAMIC_STACKALLOC, DL,
-                           DAG.getVTList(MVT::i64, MVT::Other),
-                           {Chain, Size, DAG.getConstant(1, DL, MVT::i64)});
-      MFI.CreateVariableSizedObject(Align(16), nullptr);
-    }
-    Chain = DAG.getNode(
-        AArch64ISD::INIT_TPIDR2OBJ, DL, DAG.getVTList(MVT::Other),
-        {/*Chain*/ Buffer.getValue(1), /*Buffer ptr*/ Buffer.getValue(0)});
-  } else if (Attrs.hasAgnosticZAInterface()) {
-    // Call __arm_sme_state_size().
-    SDValue BufferSize =
-        DAG.getNode(AArch64ISD::GET_SME_SAVE_SIZE, DL,
-                    DAG.getVTList(MVT::i64, MVT::Other), Chain);
-    Chain = BufferSize.getValue(1);
-
-    SDValue Buffer;
-    if (!Subtarget->isTargetWindows() && !hasInlineStackProbe(MF)) {
-      Buffer =
-          DAG.getNode(AArch64ISD::ALLOC_SME_SAVE_BUFFER, DL,
-                      DAG.getVTList(MVT::i64, MVT::Other), {Chain, BufferSize});
-    } else {
-      // Allocate space dynamically.
-      Buffer = DAG.getNode(
-          ISD::DYNAMIC_STACKALLOC, DL, DAG.getVTList(MVT::i64, MVT::Other),
-          {Chain, BufferSize, DAG.getConstant(1, DL, MVT::i64)});
-      MFI.CreateVariableSizedObject(Align(16), nullptr);
+  if (!getTM().useNewSMEABILowering() || Attrs.hasAgnosticZAInterface()) {
+    // Old SME ABI lowering (deprecated):
+    // Create a 16 Byte TPIDR2 object. The dynamic buffer
+    // will be expanded and stored in the static object later using a
+    // pseudonode.
+    if (Attrs.hasZAState()) {
+      TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj();
+      TPIDR2.FrameIndex = MFI.CreateStackObject(16, Align(16), false);
+      SDValue SVL = DAG.getNode(AArch64ISD::RDSVL, DL, MVT::i64,
+                                DAG.getConstant(1, DL, MVT::i32));
+      SDValue Buffer;
+      if (!Subtarget->isTargetWindows() && !hasInlineStackProbe(MF)) {
+        Buffer = DAG.getNode(AArch64ISD::ALLOCATE_ZA_BUFFER, DL,
+                             DAG.getVTList(MVT::i64, MVT::Other), {Chain, SVL});
+      } else {
+        SDValue Size = DAG.getNode(ISD::MUL, DL, MVT::i64, SVL, SVL);
+        Buffer = DAG.getNode(ISD::DYNAMIC_STACKALLOC, DL,
+                             DAG.getVTList(MVT::i64, MVT::Other),
+                             {Chain, Size, DAG.getConstant(1, DL, MVT::i64)});
+        MFI.CreateVariableSizedObject(Align(16), nullptr);
+      }
+      Chain = DAG.getNode(
+          AArch64ISD::INIT_TPIDR2OBJ, DL, DAG.getVTList(MVT::Other),
+          {/*Chain*/ Buffer.getValue(1), /*Buffer ptr*/ Buffer.getValue(0)});
+    } else if (Attrs.hasAgnosticZAInterface()) {
+      // Call __arm_sme_state_size().
+      SDValue BufferSize =
+          DAG.getNode(AArch64ISD::GET_SME_SAVE_SIZE, DL,
+                      DAG.getVTList(MVT::i64, MVT::Other), Chain);
+      Chain = BufferSize.getValue(1);
+      SDValue Buffer;
+      if (!Subtarget->isTargetWindows() && !hasInlineStackProbe(MF)) {
+        Buffer = DAG.getNode(AArch64ISD::ALLOC_SME_SAVE_BUFFER, DL,
+                             DAG.getVTList(MVT::i64, MVT::Other),
+                             {Chain, BufferSize});
+      } else {
+        // Allocate space dynamically.
+        Buffer = DAG.getNode(
+            ISD::DYNAMIC_STACKALLOC, DL, DAG.getVTList(MVT::i64, MVT::Other),
+            {Chain, BufferSize, DAG.getConstant(1, DL, MVT::i64)});
+        MFI.CreateVariableSizedObject(Align(16), nullptr);
+      }
+      // Copy the value to a virtual register, and save that in FuncInfo.
+      Register BufferPtr =
+          MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
+      FuncInfo->setSMESaveBufferAddr(BufferPtr);
+      Chain = DAG.getCopyToReg(Chain, DL, BufferPtr, Buffer);
     }
-
-    // Copy the value to a virtual register, and save that in FuncInfo.
-    Register BufferPtr =
-        MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
-    FuncInfo->setSMESaveBufferAddr(BufferPtr);
-    Chain = DAG.getCopyToReg(Chain, DL, BufferPtr, Buffer);
   }
 
   if (CallConv == CallingConv::PreserveNone) {
@@ -8307,6 +8356,15 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
     }
   }
 
+  if (getTM().useNewSMEABILowering()) {
+    // Clear new ZT0 state. TODO: Move this to the SME ABI pass.
+    if (Attrs.isNewZT0())
+      Chain = DAG.getNode(
+          ISD::INTRINSIC_VOID, DL, MVT::Other, Chain,
+          DAG.getConstant(Intrinsic::aarch64_sme_zero_zt, DL, MVT::i32),
+          DAG.getTargetConstant(0, DL, MVT::i32));
+  }
+
   return Chain;
 }
 
@@ -8537,7 +8595,7 @@ static void analyzeCallOperands(const AArch64TargetLowering &TLI,
       if (IsCalleeWin64) {
         UseVarArgCC = true;
       } else {
-        UseVarArgCC = !Outs[i].IsFixed;
+        UseVarArgCC = ArgFlags.isVarArg();
       }
     }
 
@@ -8557,19 +8615,20 @@ static void analyzeCallOperands(const AArch64TargetLowering &TLI,
     // FIXME: CCAssignFnForCall should be called once, for the call and not per
     // argument. This logic should exactly mirror LowerFormalArguments.
     CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CalleeCC, UseVarArgCC);
-    bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
+    bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags,
+                        Outs[i].OrigTy, CCInfo);
     assert(!Res && "Call operand has unhandled type");
     (void)Res;
   }
 }
 
 static SMECallAttrs
-getSMECallAttrs(const Function &Caller,
+getSMECallAttrs(const Function &Caller, const AArch64TargetLowering &TLI,
                 const TargetLowering::CallLoweringInfo &CLI) {
   if (CLI.CB)
-    return SMECallAttrs(*CLI.CB);
+    return SMECallAttrs(*CLI.CB, &TLI);
   if (auto *ES = dyn_cast<ExternalSymbolSDNode>(CLI.Callee))
-    return SMECallAttrs(SMEAttrs(Caller), SMEAttrs(ES->getSymbol()));
+    return SMECallAttrs(SMEAttrs(Caller), SMEAttrs(ES->getSymbol(), TLI));
   return SMECallAttrs(SMEAttrs(Caller), SMEAttrs(SMEAttrs::Normal));
 }
 
@@ -8591,7 +8650,7 @@ bool AArch64TargetLowering::isEligibleForTailCallOptimization(
 
   // SME Streaming functions are not eligible for TCO as they may require
   // the streaming mode or ZA to be restored after returning from the call.
-  SMECallAttrs CallAttrs = getSMECallAttrs(CallerF, CLI);
+  SMECallAttrs CallAttrs = getSMECallAttrs(CallerF, *this, CLI);
   if (CallAttrs.requiresSMChange() || CallAttrs.requiresLazySave() ||
       CallAttrs.requiresPreservingAllZAState() ||
       CallAttrs.caller().hasStreamingBody())
@@ -8834,8 +8893,7 @@ void AArch64TargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
 SDValue AArch64TargetLowering::changeStreamingMode(SelectionDAG &DAG, SDLoc DL,
                                                    bool Enable, SDValue Chain,
                                                    SDValue InGlue,
-                                                   unsigned Condition,
-                                                   SDValue PStateSM) const {
+                                                   unsigned Condition) const {
   MachineFunction &MF = DAG.getMachineFunction();
   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
   FuncInfo->setHasStreamingModeChanges(true);
@@ -8847,9 +8905,16 @@ SDValue AArch64TargetLowering::changeStreamingMode(SelectionDAG &DAG, SDLoc DL,
   SmallVector<SDValue> Ops = {Chain, MSROp};
   unsigned Opcode;
   if (Condition != AArch64SME::Always) {
+    FuncInfo->setPStateSMRegUsed(true);
+    Register PStateReg = FuncInfo->getPStateSMReg();
+    assert(PStateReg.isValid() && "PStateSM Register is invalid");
+    SDValue PStateSM =
+        DAG.getCopyFromReg(Chain, DL, PStateReg, MVT::i64, InGlue);
+    // Use chain and glue from the CopyFromReg.
+    Ops[0] = PStateSM.getValue(1);
+    InGlue = PStateSM.getValue(2);
     SDValue ConditionOp = DAG.getTargetConstant(Condition, DL, MVT::i64);
     Opcode = Enable ? AArch64ISD::COND_SMSTART : AArch64ISD::COND_SMSTOP;
-    assert(PStateSM && "PStateSM should be defined");
     Ops.push_back(ConditionOp);
     Ops.push_back(PStateSM);
   } else {
@@ -8871,22 +8936,19 @@ static SDValue emitSMEStateSaveRestore(const AArch64TargetLowering &TLI,
   MachineFunction &MF = DAG.getMachineFunction();
   AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
   FuncInfo->setSMESaveBufferUsed();
-
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Ty = PointerType::getUnqual(*DAG.getContext());
-  Entry.Node =
-      DAG.getCopyFromReg(Chain, DL, Info->getSMESaveBufferAddr(), MVT::i64);
-  Args.push_back(Entry);
-
-  SDValue Callee =
-      DAG.getExternalSymbol(IsSave ? "__arm_sme_save" : "__arm_sme_restore",
-                            TLI.getPointerTy(DAG.getDataLayout()));
+  Args.emplace_back(
+      DAG.getCopyFromReg(Chain, DL, Info->getSMESaveBufferAddr(), MVT::i64),
+      PointerType::getUnqual(*DAG.getContext()));
+
+  RTLIB::Libcall LC =
+      IsSave ? RTLIB::SMEABI_SME_SAVE : RTLIB::SMEABI_SME_RESTORE;
+  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
+                                         TLI.getPointerTy(DAG.getDataLayout()));
   auto *RetTy = Type::getVoidTy(*DAG.getContext());
   TargetLowering::CallLoweringInfo CLI(DAG);
   CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
-      CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1, RetTy,
-      Callee, std::move(Args));
+      TLI.getLibcallCallingConv(LC), RetTy, Callee, std::move(Args));
   return TLI.LowerCallTo(CLI).second;
 }
 
@@ -8982,7 +9044,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
     unsigned NumArgs = Outs.size();
 
     for (unsigned i = 0; i != NumArgs; ++i) {
-      if (!Outs[i].IsFixed && Outs[i].VT.isScalableVector())
+      if (Outs[i].Flags.isVarArg() && Outs[i].VT.isScalableVector())
         report_fatal_error("Passing SVE types to variadic functions is "
                            "currently not supported");
     }
@@ -9014,14 +9076,28 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
       CallConv = CallingConv::AArch64_SVE_VectorCall;
   }
 
+  // Determine whether we need any streaming mode changes.
+  SMECallAttrs CallAttrs = getSMECallAttrs(MF.getFunction(), *this, CLI);
+  bool UseNewSMEABILowering = getTM().useNewSMEABILowering();
+  bool IsAgnosticZAFunction = CallAttrs.caller().hasAgnosticZAInterface();
+  auto ZAMarkerNode = [&]() -> std::optional<unsigned> {
+    // TODO: Handle agnostic ZA functions.
+    if (!UseNewSMEABILowering || IsAgnosticZAFunction)
+      return std::nullopt;
+    if (!CallAttrs.caller().hasZAState() && !CallAttrs.caller().hasZT0State())
+      return std::nullopt;
+    return CallAttrs.requiresLazySave() ? AArch64ISD::REQUIRES_ZA_SAVE
+                                        : AArch64ISD::INOUT_ZA_USE;
+  }();
+
   if (IsTailCall) {
     // Check if it's really possible to do a tail call.
     IsTailCall = isEligibleForTailCallOptimization(CLI);
 
     // A sibling call is one where we're under the usual C ABI and not planning
     // to change that but can still do a tail call:
-    if (!TailCallOpt && IsTailCall && CallConv != CallingConv::Tail &&
-        CallConv != CallingConv::SwiftTail)
+    if (!ZAMarkerNode && !TailCallOpt && IsTailCall &&
+        CallConv != CallingConv::Tail && CallConv != CallingConv::SwiftTail)
       IsSibCall = true;
 
     if (IsTailCall)
@@ -9073,9 +9149,6 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
     assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
   }
 
-  // Determine whether we need any streaming mode changes.
-  SMECallAttrs CallAttrs = getSMECallAttrs(MF.getFunction(), CLI);
-
   auto DescribeCallsite =
       [&](OptimizationRemarkAnalysis &R) -> OptimizationRemarkAnalysis & {
     R << "call from '" << ore::NV("Caller", MF.getName()) << "' to '";
@@ -9089,7 +9162,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
     return R;
   };
 
-  bool RequiresLazySave = CallAttrs.requiresLazySave();
+  bool RequiresLazySave = !UseNewSMEABILowering && CallAttrs.requiresLazySave();
   bool RequiresSaveAllZA = CallAttrs.requiresPreservingAllZAState();
   if (RequiresLazySave) {
     const TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj();
@@ -9124,15 +9197,8 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
                                     /*IsSave=*/true);
   }
 
-  SDValue PStateSM;
   bool RequiresSMChange = CallAttrs.requiresSMChange();
   if (RequiresSMChange) {
-    if (CallAttrs.caller().hasStreamingInterfaceOrBody())
-      PStateSM = DAG.getConstant(1, DL, MVT::i64);
-    else if (CallAttrs.caller().hasNonStreamingInterface())
-      PStateSM = DAG.getConstant(0, DL, MVT::i64);
-    else
-      PStateSM = getRuntimePStateSM(DAG, Chain, DL, MVT::i64);
     OptimizationRemarkEmitter ORE(&MF.getFunction());
     ORE.emit([&]() {
       auto R = CLI.CB ? OptimizationRemarkAnalysis("sme", "SMETransition",
@@ -9171,10 +9237,20 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
         AArch64ISD::SMSTOP, DL, DAG.getVTList(MVT::Other, MVT::Glue), Chain,
         DAG.getTargetConstant((int32_t)(AArch64SVCR::SVCRZA), DL, MVT::i32));
 
-  // Adjust the stack pointer for the new arguments...
+  // Adjust the stack pointer for the new arguments... and mark ZA uses.
   // These operations are automatically eliminated by the prolog/epilog pass
-  if (!IsSibCall)
+  assert((!IsSibCall || !ZAMarkerNode) && "ZA markers require CALLSEQ_START");
+  if (!IsSibCall) {
     Chain = DAG.getCALLSEQ_START(Chain, IsTailCall ? 0 : NumBytes, 0, DL);
+    if (ZAMarkerNode) {
+      // Note: We need the CALLSEQ_START to glue the ZAMarkerNode to, simply
+      // using a chain can result in incorrect scheduling. The markers refer to
+      // the position just before the CALLSEQ_START (though occur after as
+      // CALLSEQ_START lacks in-glue).
+      Chain = DAG.getNode(*ZAMarkerNode, DL, DAG.getVTList(MVT::Other),
+                          {Chain, Chain.getValue(1)});
+    }
+  }
 
   SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
                                         getPointerTy(DAG.getDataLayout()));
@@ -9441,17 +9517,10 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   SDValue InGlue;
   if (RequiresSMChange) {
-    if (!Subtarget->isTargetDarwin() || Subtarget->hasSVE()) {
-      Chain = DAG.getNode(AArch64ISD::VG_SAVE, DL,
-                          DAG.getVTList(MVT::Other, MVT::Glue), Chain);
-      InGlue = Chain.getValue(1);
-    }
-
-    SDValue NewChain = changeStreamingMode(
-        DAG, DL, CallAttrs.callee().hasStreamingInterface(), Chain, InGlue,
-        getSMToggleCondition(CallAttrs), PStateSM);
-    Chain = NewChain.getValue(0);
-    InGlue = NewChain.getValue(1);
+    Chain =
+        changeStreamingMode(DAG, DL, CallAttrs.callee().hasStreamingInterface(),
+                            Chain, InGlue, getSMToggleCondition(CallAttrs));
+    InGlue = Chain.getValue(1);
   }
 
   // Build a sequence of copy-to-reg nodes chained together with token chain
@@ -9633,20 +9702,12 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
     InGlue = Result.getValue(Result->getNumValues() - 1);
 
   if (RequiresSMChange) {
-    assert(PStateSM && "Expected a PStateSM to be set");
     Result = changeStreamingMode(
         DAG, DL, !CallAttrs.callee().hasStreamingInterface(), Result, InGlue,
-        getSMToggleCondition(CallAttrs), PStateSM);
-
-    if (!Subtarget->isTargetDarwin() || Subtarget->hasSVE()) {
-      InGlue = Result.getValue(1);
-      Result =
-          DAG.getNode(AArch64ISD::VG_RESTORE, DL,
-                      DAG.getVTList(MVT::Other, MVT::Glue), {Result, InGlue});
-    }
+        getSMToggleCondition(CallAttrs));
   }
 
-  if (CallAttrs.requiresEnablingZAAfterCall())
+  if (RequiresLazySave || CallAttrs.requiresEnablingZAAfterCall())
     // Unconditionally resume ZA.
     Result = DAG.getNode(
         AArch64ISD::SMSTART, DL, DAG.getVTList(MVT::Other, MVT::Glue), Result,
@@ -9659,15 +9720,15 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   if (RequiresLazySave) {
     // Conditionally restore the lazy save using a pseudo node.
+    RTLIB::Libcall LC = RTLIB::SMEABI_TPIDR2_RESTORE;
     TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj();
     SDValue RegMask = DAG.getRegisterMask(
-        TRI->SMEABISupportRoutinesCallPreservedMaskFromX0());
+        TRI->getCallPreservedMask(MF, getLibcallCallingConv(LC)));
     SDValue RestoreRoutine = DAG.getTargetExternalSymbol(
-        "__arm_tpidr2_restore", getPointerTy(DAG.getDataLayout()));
+        getLibcallName(LC), getPointerTy(DAG.getDataLayout()));
     SDValue TPIDR2_EL0 = DAG.getNode(
         ISD::INTRINSIC_W_CHAIN, DL, MVT::i64, Result,
         DAG.getConstant(Intrinsic::aarch64_sme_get_tpidr2, DL, MVT::i32));
-
     // Copy the address of the TPIDR2 block into X0 before 'calling' the
     // RESTORE_ZA pseudo.
     SDValue Glue;
@@ -9679,7 +9740,6 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
         DAG.getNode(AArch64ISD::RESTORE_ZA, DL, MVT::Other,
                     {Result, TPIDR2_EL0, DAG.getRegister(AArch64::X0, MVT::i64),
                      RestoreRoutine, RegMask, Result.getValue(1)});
-
     // Finally reset the TPIDR2_EL0 register to 0.
     Result = DAG.getNode(
         ISD::INTRINSIC_VOID, DL, MVT::Other, Result,
@@ -9802,14 +9862,11 @@ AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   // Emit SMSTOP before returning from a locally streaming function
   SMEAttrs FuncAttrs = FuncInfo->getSMEFnAttrs();
   if (FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface()) {
-    if (FuncAttrs.hasStreamingCompatibleInterface()) {
-      Register Reg = FuncInfo->getPStateSMReg();
-      assert(Reg.isValid() && "PStateSM Register is invalid");
-      SDValue PStateSM = DAG.getCopyFromReg(Chain, DL, Reg, MVT::i64);
+    if (FuncAttrs.hasStreamingCompatibleInterface())
       Chain = changeStreamingMode(DAG, DL, /*Enable*/ false, Chain,
                                   /*Glue*/ SDValue(),
-                                  AArch64SME::IfCallerIsNonStreaming, PStateSM);
-    } else
+                                  AArch64SME::IfCallerIsNonStreaming);
+    else
       Chain = changeStreamingMode(DAG, DL, /*Enable*/ false, Chain,
                                   /*Glue*/ SDValue(), AArch64SME::Always);
     Glue = Chain.getValue(1);
@@ -13482,7 +13539,7 @@ static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
   // Look for the first non-undef element.
   const int *FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
 
-  // Benefit form APInt to handle overflow when calculating expected element.
+  // Benefit from APInt to handle overflow when calculating expected element.
   unsigned NumElts = VT.getVectorNumElements();
   unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
   APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1, /*isSigned=*/false,
@@ -13490,7 +13547,7 @@ static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
   // The following shuffle indices must be the successive elements after the
   // first real element.
   bool FoundWrongElt = std::any_of(FirstRealElt + 1, M.end(), [&](int Elt) {
-    return Elt != ExpectedElt++ && Elt != -1;
+    return Elt != ExpectedElt++ && Elt >= 0;
   });
   if (FoundWrongElt)
     return false;
@@ -14742,12 +14799,107 @@ static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
   return ResultSLI;
 }
 
+static SDValue tryLowerToBSL(SDValue N, SelectionDAG &DAG) {
+  EVT VT = N->getValueType(0);
+  assert(VT.isVector() && "Expected vector type in tryLowerToBSL\n");
+  SDLoc DL(N);
+  const auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
+
+  if (VT.isScalableVector() && !Subtarget.hasSVE2())
+    return SDValue();
+
+  SDValue N0 = N->getOperand(0);
+  if (N0.getOpcode() != ISD::AND)
+    return SDValue();
+
+  SDValue N1 = N->getOperand(1);
+  if (N1.getOpcode() != ISD::AND)
+    return SDValue();
+
+  // InstCombine does (not (neg a)) => (add a -1).
+  // Try: (or (and (neg a) b) (and (add a -1) c)) => (bsl (neg a) b c)
+  // Loop over all combinations of AND operands.
+  for (int i = 1; i >= 0; --i) {
+    for (int j = 1; j >= 0; --j) {
+      SDValue O0 = N0->getOperand(i);
+      SDValue O1 = N1->getOperand(j);
+      SDValue Sub, Add, SubSibling, AddSibling;
+
+      // Find a SUB and an ADD operand, one from each AND.
+      if (O0.getOpcode() == ISD::SUB && O1.getOpcode() == ISD::ADD) {
+        Sub = O0;
+        Add = O1;
+        SubSibling = N0->getOperand(1 - i);
+        AddSibling = N1->getOperand(1 - j);
+      } else if (O0.getOpcode() == ISD::ADD && O1.getOpcode() == ISD::SUB) {
+        Add = O0;
+        Sub = O1;
+        AddSibling = N0->getOperand(1 - i);
+        SubSibling = N1->getOperand(1 - j);
+      } else
+        continue;
+
+      if (!ISD::isConstantSplatVectorAllZeros(Sub.getOperand(0).getNode()))
+        continue;
+
+      // Constant ones is always righthand operand of the Add.
+      if (!ISD::isConstantSplatVectorAllOnes(Add.getOperand(1).getNode()))
+        continue;
+
+      if (Sub.getOperand(1) != Add.getOperand(0))
+        continue;
+
+      return DAG.getNode(AArch64ISD::BSP, DL, VT, Sub, SubSibling, AddSibling);
+    }
+  }
+
+  // (or (and a b) (and (not a) c)) => (bsl a b c)
+  // We only have to look for constant vectors here since the general, variable
+  // case can be handled in TableGen.
+  unsigned Bits = VT.getScalarSizeInBits();
+  for (int i = 1; i >= 0; --i)
+    for (int j = 1; j >= 0; --j) {
+      APInt Val1, Val2;
+
+      if (ISD::isConstantSplatVector(N0->getOperand(i).getNode(), Val1) &&
+          ISD::isConstantSplatVector(N1->getOperand(j).getNode(), Val2) &&
+          ~Val1.trunc(Bits) == Val2.trunc(Bits)) {
+        return DAG.getNode(AArch64ISD::BSP, DL, VT, N0->getOperand(i),
+                           N0->getOperand(1 - i), N1->getOperand(1 - j));
+      }
+      BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(i));
+      BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(j));
+      if (!BVN0 || !BVN1)
+        continue;
+
+      bool FoundMatch = true;
+      for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) {
+        ConstantSDNode *CN0 = dyn_cast<ConstantSDNode>(BVN0->getOperand(k));
+        ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(BVN1->getOperand(k));
+        if (!CN0 || !CN1 ||
+            CN0->getAPIntValue().trunc(Bits) !=
+                ~CN1->getAsAPIntVal().trunc(Bits)) {
+          FoundMatch = false;
+          break;
+        }
+      }
+      if (FoundMatch)
+        return DAG.getNode(AArch64ISD::BSP, DL, VT, N0->getOperand(i),
+                           N0->getOperand(1 - i), N1->getOperand(1 - j));
+    }
+
+  return SDValue();
+}
+
 SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
                                              SelectionDAG &DAG) const {
   if (useSVEForFixedLengthVectorVT(Op.getValueType(),
                                    !Subtarget->isNeonAvailable()))
     return LowerToScalableOp(Op, DAG);
 
+  if (SDValue Res = tryLowerToBSL(Op, DAG))
+    return Res;
+
   // Attempt to form a vector S[LR]I from (or (and X, C1), (lsl Y, C2))
   if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
     return Res;
@@ -15777,6 +15929,7 @@ bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
           isREVMask(M, EltSize, NumElts, 32) ||
           isREVMask(M, EltSize, NumElts, 16) ||
           isEXTMask(M, VT, DummyBool, DummyUnsigned) ||
+          isSingletonEXTMask(M, VT, DummyUnsigned) ||
           isTRNMask(M, NumElts, DummyUnsigned) ||
           isUZPMask(M, NumElts, DummyUnsigned) ||
           isZIPMask(M, NumElts, DummyUnsigned) ||
@@ -16289,9 +16442,8 @@ AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(SDValue Op,
     Chain = SP.getValue(1);
     SP = DAG.getNode(ISD::SUB, DL, MVT::i64, SP, Size);
     if (Align)
-      SP =
-          DAG.getNode(ISD::AND, DL, VT, SP.getValue(0),
-                      DAG.getSignedConstant(-(uint64_t)Align->value(), DL, VT));
+      SP = DAG.getNode(ISD::AND, DL, VT, SP.getValue(0),
+                       DAG.getSignedConstant(-Align->value(), DL, VT));
     Chain = DAG.getCopyToReg(Chain, DL, AArch64::SP, SP);
     SDValue Ops[2] = {SP, Chain};
     return DAG.getMergeValues(Ops, DL);
@@ -16328,7 +16480,7 @@ AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(SDValue Op,
   SP = DAG.getNode(ISD::SUB, DL, MVT::i64, SP, Size);
   if (Align)
     SP = DAG.getNode(ISD::AND, DL, VT, SP.getValue(0),
-                     DAG.getSignedConstant(-(uint64_t)Align->value(), DL, VT));
+                     DAG.getSignedConstant(-Align->value(), DL, VT));
   Chain = DAG.getCopyToReg(Chain, DL, AArch64::SP, SP);
 
   Chain = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), DL);
@@ -16356,7 +16508,7 @@ AArch64TargetLowering::LowerInlineDYNAMIC_STACKALLOC(SDValue Op,
   SP = DAG.getNode(ISD::SUB, DL, MVT::i64, SP, Size);
   if (Align)
     SP = DAG.getNode(ISD::AND, DL, VT, SP.getValue(0),
-                     DAG.getSignedConstant(-(uint64_t)Align->value(), DL, VT));
+                     DAG.getSignedConstant(-Align->value(), DL, VT));
 
   // Set the real SP to the new value with a probing loop.
   Chain = DAG.getNode(AArch64ISD::PROBED_ALLOCA, DL, MVT::Other, Chain, SP);
@@ -17259,7 +17411,7 @@ static Function *getStructuredStoreFunction(Module *M, unsigned Factor,
 ///        %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1
 bool AArch64TargetLowering::lowerInterleavedLoad(
     Instruction *Load, Value *Mask, ArrayRef<ShuffleVectorInst *> Shuffles,
-    ArrayRef<unsigned> Indices, unsigned Factor) const {
+    ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const {
   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
          "Invalid interleave factor");
   assert(!Shuffles.empty() && "Empty shufflevector input");
@@ -17269,7 +17421,7 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
   auto *LI = dyn_cast<LoadInst>(Load);
   if (!LI)
     return false;
-  assert(!Mask && "Unexpected mask on a load");
+  assert(!Mask && GapMask.popcount() == Factor && "Unexpected mask on a load");
 
   const DataLayout &DL = LI->getDataLayout();
 
@@ -17992,7 +18144,8 @@ bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(
   case MVT::f64:
     return true;
   case MVT::bf16:
-    return VT.isScalableVector() && Subtarget->hasSVEB16B16();
+    return VT.isScalableVector() && Subtarget->hasSVEB16B16() &&
+           Subtarget->isNonStreamingSVEorSME2Available();
   default:
     break;
   }
@@ -18156,7 +18309,7 @@ bool AArch64TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
   if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, BitSize))
     return true;
 
-  if ((int64_t)Val < 0)
+  if (Val < 0)
     Val = ~Val;
   if (BitSize == 32)
     Val &= (1LL << 32) - 1;
@@ -19419,106 +19572,6 @@ static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
   return FixConv;
 }
 
-static SDValue tryCombineToBSL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
-                               const AArch64TargetLowering &TLI) {
-  EVT VT = N->getValueType(0);
-  SelectionDAG &DAG = DCI.DAG;
-  SDLoc DL(N);
-  const auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
-
-  if (!VT.isVector())
-    return SDValue();
-
-  if (VT.isScalableVector() && !Subtarget.hasSVE2())
-    return SDValue();
-
-  if (VT.isFixedLengthVector() &&
-      (!Subtarget.isNeonAvailable() || TLI.useSVEForFixedLengthVectorVT(VT)))
-    return SDValue();
-
-  SDValue N0 = N->getOperand(0);
-  if (N0.getOpcode() != ISD::AND)
-    return SDValue();
-
-  SDValue N1 = N->getOperand(1);
-  if (N1.getOpcode() != ISD::AND)
-    return SDValue();
-
-  // InstCombine does (not (neg a)) => (add a -1).
-  // Try: (or (and (neg a) b) (and (add a -1) c)) => (bsl (neg a) b c)
-  // Loop over all combinations of AND operands.
-  for (int i = 1; i >= 0; --i) {
-    for (int j = 1; j >= 0; --j) {
-      SDValue O0 = N0->getOperand(i);
-      SDValue O1 = N1->getOperand(j);
-      SDValue Sub, Add, SubSibling, AddSibling;
-
-      // Find a SUB and an ADD operand, one from each AND.
-      if (O0.getOpcode() == ISD::SUB && O1.getOpcode() == ISD::ADD) {
-        Sub = O0;
-        Add = O1;
-        SubSibling = N0->getOperand(1 - i);
-        AddSibling = N1->getOperand(1 - j);
-      } else if (O0.getOpcode() == ISD::ADD && O1.getOpcode() == ISD::SUB) {
-        Add = O0;
-        Sub = O1;
-        AddSibling = N0->getOperand(1 - i);
-        SubSibling = N1->getOperand(1 - j);
-      } else
-        continue;
-
-      if (!ISD::isConstantSplatVectorAllZeros(Sub.getOperand(0).getNode()))
-        continue;
-
-      // Constant ones is always righthand operand of the Add.
-      if (!ISD::isConstantSplatVectorAllOnes(Add.getOperand(1).getNode()))
-        continue;
-
-      if (Sub.getOperand(1) != Add.getOperand(0))
-        continue;
-
-      return DAG.getNode(AArch64ISD::BSP, DL, VT, Sub, SubSibling, AddSibling);
-    }
-  }
-
-  // (or (and a b) (and (not a) c)) => (bsl a b c)
-  // We only have to look for constant vectors here since the general, variable
-  // case can be handled in TableGen.
-  unsigned Bits = VT.getScalarSizeInBits();
-  uint64_t BitMask = Bits == 64 ? -1ULL : ((1ULL << Bits) - 1);
-  for (int i = 1; i >= 0; --i)
-    for (int j = 1; j >= 0; --j) {
-      APInt Val1, Val2;
-
-      if (ISD::isConstantSplatVector(N0->getOperand(i).getNode(), Val1) &&
-          ISD::isConstantSplatVector(N1->getOperand(j).getNode(), Val2) &&
-          (BitMask & ~Val1.getZExtValue()) == Val2.getZExtValue()) {
-        return DAG.getNode(AArch64ISD::BSP, DL, VT, N0->getOperand(i),
-                           N0->getOperand(1 - i), N1->getOperand(1 - j));
-      }
-      BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(i));
-      BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(j));
-      if (!BVN0 || !BVN1)
-        continue;
-
-      bool FoundMatch = true;
-      for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) {
-        ConstantSDNode *CN0 = dyn_cast<ConstantSDNode>(BVN0->getOperand(k));
-        ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(BVN1->getOperand(k));
-        if (!CN0 || !CN1 ||
-            CN0->getZExtValue() != (BitMask & ~CN1->getZExtValue())) {
-          FoundMatch = false;
-          break;
-        }
-      }
-      if (FoundMatch)
-        return DAG.getNode(AArch64ISD::BSP, DL, VT, N0->getOperand(i),
-                           N0->getOperand(1 - i), N1->getOperand(1 - j));
-    }
-
-  return SDValue();
-}
-
 // Given a tree of and/or(csel(0, 1, cc0), csel(0, 1, cc1)), we may be able to
 // convert to csel(ccmp(.., cc0)), depending on cc1:
 
@@ -19600,17 +19653,10 @@ static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                 const AArch64Subtarget *Subtarget,
                                 const AArch64TargetLowering &TLI) {
   SelectionDAG &DAG = DCI.DAG;
-  EVT VT = N->getValueType(0);
 
   if (SDValue R = performANDORCSELCombine(N, DAG))
     return R;
 
-  if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
-    return SDValue();
-
-  if (SDValue Res = tryCombineToBSL(N, DCI, TLI))
-    return Res;
-
   return SDValue();
 }
 
@@ -28194,6 +28240,7 @@ void AArch64TargetLowering::ReplaceNodeResults(
     case Intrinsic::aarch64_sme_in_streaming_mode: {
       SDLoc DL(N);
       SDValue Chain = DAG.getEntryNode();
+
       SDValue RuntimePStateSM =
           getRuntimePStateSM(DAG, Chain, DL, N->getValueType(0));
       Results.push_back(
@@ -28639,9 +28686,13 @@ void AArch64TargetLowering::insertSSPDeclarations(Module &M) const {
   // MSVC CRT provides functionalities for stack protection.
   RTLIB::LibcallImpl SecurityCheckCookieLibcall =
       getLibcallImpl(RTLIB::SECURITY_CHECK_COOKIE);
-  if (SecurityCheckCookieLibcall != RTLIB::Unsupported) {
+
+  RTLIB::LibcallImpl SecurityCookieVar =
+      getLibcallImpl(RTLIB::STACK_CHECK_GUARD);
+  if (SecurityCheckCookieLibcall != RTLIB::Unsupported &&
+      SecurityCookieVar != RTLIB::Unsupported) {
     // MSVC CRT has a global variable holding security cookie.
-    M.getOrInsertGlobal("__security_cookie",
+    M.getOrInsertGlobal(getLibcallImplName(SecurityCookieVar),
                         PointerType::getUnqual(M.getContext()));
 
     // MSVC CRT has a function to validate security cookie.
@@ -28658,13 +28709,6 @@ void AArch64TargetLowering::insertSSPDeclarations(Module &M) const {
   TargetLowering::insertSSPDeclarations(M);
 }
 
-Value *AArch64TargetLowering::getSDagStackGuard(const Module &M) const {
-  // MSVC CRT has a global variable holding security cookie.
-  if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
-    return M.getGlobalVariable("__security_cookie");
-  return TargetLowering::getSDagStackGuard(M);
-}
-
 Function *AArch64TargetLowering::getSSPStackGuardCheck(const Module &M) const {
   // MSVC CRT has a function to validate security cookie.
   RTLIB::LibcallImpl SecurityCheckCookieLibcall =
@@ -29004,7 +29048,7 @@ bool AArch64TargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
 
   // Checks to allow the use of SME instructions
   if (auto *Base = dyn_cast<CallBase>(&Inst)) {
-    auto CallAttrs = SMECallAttrs(*Base);
+    auto CallAttrs = SMECallAttrs(*Base, this);
     if (CallAttrs.requiresSMChange() || CallAttrs.requiresLazySave() ||
         CallAttrs.requiresPreservingZT0() ||
         CallAttrs.requiresPreservingAllZAState())
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 88876570ac811..6c6ae782f779f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -23,6 +23,8 @@
 
 namespace llvm {
 
+class AArch64TargetMachine;
+
 namespace AArch64 {
 /// Possible values of current rounding mode, which is specified in bits
 /// 23:22 of FPCR.
@@ -64,6 +66,8 @@ class AArch64TargetLowering : public TargetLowering {
   explicit AArch64TargetLowering(const TargetMachine &TM,
                                  const AArch64Subtarget &STI);
 
+  const AArch64TargetMachine &getTM() const;
+
   /// Control the following reassociation of operands: (op (op x, c1), y) -> (op
   /// (op x, y), c1) where N0 is (op x, c1) and N1 is y.
   bool isReassocProfitable(SelectionDAG &DAG, SDValue N0,
@@ -173,6 +177,10 @@ class AArch64TargetLowering : public TargetLowering {
   MachineBasicBlock *EmitZTInstr(MachineInstr &MI, MachineBasicBlock *BB,
                                  unsigned Opcode, bool Op0IsDef) const;
   MachineBasicBlock *EmitZero(MachineInstr &MI, MachineBasicBlock *BB) const;
+
+  // Note: The following group of functions are only used as part of the old SME
+  // ABI lowering. They will be removed once -aarch64-new-sme-abi=true is the
+  // default.
   MachineBasicBlock *EmitInitTPIDR2Object(MachineInstr &MI,
                                           MachineBasicBlock *BB) const;
   MachineBasicBlock *EmitAllocateZABuffer(MachineInstr &MI,
@@ -181,6 +189,8 @@ class AArch64TargetLowering : public TargetLowering {
                                                MachineBasicBlock *BB) const;
   MachineBasicBlock *EmitGetSMESaveSize(MachineInstr &MI,
                                         MachineBasicBlock *BB) const;
+  MachineBasicBlock *EmitEntryPStateSM(MachineInstr &MI,
+                                       MachineBasicBlock *BB) const;
 
   /// Replace (0, vreg) discriminator components with the operands of blend
   /// or with (immediate, NoRegister) when possible.
@@ -220,8 +230,8 @@ class AArch64TargetLowering : public TargetLowering {
 
   bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
                             ArrayRef<ShuffleVectorInst *> Shuffles,
-                            ArrayRef<unsigned> Indices,
-                            unsigned Factor) const override;
+                            ArrayRef<unsigned> Indices, unsigned Factor,
+                            const APInt &GapMask) const override;
   bool lowerInterleavedStore(Instruction *Store, Value *Mask,
                              ShuffleVectorInst *SVI,
                              unsigned Factor) const override;
@@ -344,7 +354,6 @@ class AArch64TargetLowering : public TargetLowering {
   Value *getIRStackGuard(IRBuilderBase &IRB) const override;
 
   void insertSSPDeclarations(Module &M) const override;
-  Value *getSDagStackGuard(const Module &M) const override;
   Function *getSSPStackGuardCheck(const Module &M) const override;
 
   /// If the target has a standard location for the unsafe stack pointer,
@@ -523,8 +532,8 @@ class AArch64TargetLowering : public TargetLowering {
   /// node. \p Condition should be one of the enum values from
   /// AArch64SME::ToggleCondition.
   SDValue changeStreamingMode(SelectionDAG &DAG, SDLoc DL, bool Enable,
-                              SDValue Chain, SDValue InGlue, unsigned Condition,
-                              SDValue PStateSM = SDValue()) const;
+                              SDValue Chain, SDValue InGlue,
+                              unsigned Condition) const;
 
   bool isVScaleKnownToBeAPowerOfTwo() const override { return true; }
 
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 5a537f227760f..178dab6897393 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -36,7 +36,12 @@ def DestructiveBinary             : DestructiveInstTypeEnum<5>;
 def DestructiveBinaryComm         : DestructiveInstTypeEnum<6>;
 def DestructiveBinaryCommWithRev  : DestructiveInstTypeEnum<7>;
 def DestructiveTernaryCommWithRev : DestructiveInstTypeEnum<8>;
-def DestructiveUnaryPassthru      : DestructiveInstTypeEnum<9>;
+
+// 3 inputs unpredicated (reg1, reg2, imm).
+// Can be MOVPRFX'd iff reg1 == reg2.
+def Destructive2xRegImmUnpred     : DestructiveInstTypeEnum<9>;
+
+def DestructiveUnaryPassthru      : DestructiveInstTypeEnum<10>;
 
 class FalseLanesEnum<bits<2> val> {
   bits<2> Value = val;
@@ -7362,7 +7367,9 @@ multiclass SIMDDifferentThreeVectorBD<bit U, bits<4> opc, string asm,
       [(set (v8i16 V128:$Rd), (OpNode (v8i8 V64:$Rn), (v8i8 V64:$Rm)))]>;
   def v16i8  : BaseSIMDDifferentThreeVector<U, 0b001, opc,
                                             V128, V128, V128,
-                                            asm#"2", ".8h", ".16b", ".16b", []>;
+                                            asm#"2", ".8h", ".16b", ".16b",
+      [(set (v8i16 V128:$Rd), (OpNode (v8i8 (extract_high_v16i8 (v16i8 V128:$Rn))),
+                                      (v8i8 (extract_high_v16i8 (v16i8 V128:$Rm)))))]>;
   let Predicates = [HasAES] in {
     def v1i64  : BaseSIMDDifferentThreeVector<U, 0b110, opc,
                                               V128, V64, V64,
@@ -7374,10 +7381,6 @@ multiclass SIMDDifferentThreeVectorBD<bit U, bits<4> opc, string asm,
         [(set (v16i8 V128:$Rd), (OpNode (extract_high_v2i64 (v2i64 V128:$Rn)),
                                         (extract_high_v2i64 (v2i64 V128:$Rm))))]>;
   }
-
-  def : Pat<(v8i16 (OpNode (v8i8 (extract_high_v16i8 (v16i8 V128:$Rn))),
-                          (v8i8 (extract_high_v16i8 (v16i8 V128:$Rm))))),
-      (!cast<Instruction>(NAME#"v16i8") V128:$Rn, V128:$Rm)>;
 }
 
 multiclass SIMDLongThreeVectorHS<bit U, bits<4> opc, string asm,
@@ -7402,87 +7405,7 @@ multiclass SIMDLongThreeVectorHS<bit U, bits<4> opc, string asm,
                                       (extract_high_v4i32 (v4i32 V128:$Rm))))]>;
 }
 
-multiclass SIMDLongThreeVectorBHSabdl<bit U, bits<4> opc, string asm,
-                                  SDPatternOperator OpNode = null_frag> {
-  def v8i8_v8i16   : BaseSIMDDifferentThreeVector<U, 0b000, opc,
-                                                  V128, V64, V64,
-                                                  asm, ".8h", ".8b", ".8b",
-      [(set (v8i16 V128:$Rd),
-            (zext (v8i8 (OpNode (v8i8 V64:$Rn), (v8i8 V64:$Rm)))))]>;
-  def v16i8_v8i16  : BaseSIMDDifferentThreeVector<U, 0b001, opc,
-                                                 V128, V128, V128,
-                                                 asm#"2", ".8h", ".16b", ".16b",
-      [(set (v8i16 V128:$Rd),
-            (zext (v8i8 (OpNode (extract_high_v16i8 (v16i8 V128:$Rn)),
-                                (extract_high_v16i8 (v16i8 V128:$Rm))))))]>;
-  def v4i16_v4i32  : BaseSIMDDifferentThreeVector<U, 0b010, opc,
-                                                  V128, V64, V64,
-                                                  asm, ".4s", ".4h", ".4h",
-      [(set (v4i32 V128:$Rd),
-            (zext (v4i16 (OpNode (v4i16 V64:$Rn), (v4i16 V64:$Rm)))))]>;
-  def v8i16_v4i32  : BaseSIMDDifferentThreeVector<U, 0b011, opc,
-                                                  V128, V128, V128,
-                                                  asm#"2", ".4s", ".8h", ".8h",
-      [(set (v4i32 V128:$Rd),
-            (zext (v4i16 (OpNode (extract_high_v8i16 (v8i16 V128:$Rn)),
-                                  (extract_high_v8i16 (v8i16 V128:$Rm))))))]>;
-  def v2i32_v2i64  : BaseSIMDDifferentThreeVector<U, 0b100, opc,
-                                                  V128, V64, V64,
-                                                  asm, ".2d", ".2s", ".2s",
-      [(set (v2i64 V128:$Rd),
-            (zext (v2i32 (OpNode (v2i32 V64:$Rn), (v2i32 V64:$Rm)))))]>;
-  def v4i32_v2i64  : BaseSIMDDifferentThreeVector<U, 0b101, opc,
-                                                  V128, V128, V128,
-                                                  asm#"2", ".2d", ".4s", ".4s",
-      [(set (v2i64 V128:$Rd),
-            (zext (v2i32 (OpNode (extract_high_v4i32 (v4i32 V128:$Rn)),
-                                 (extract_high_v4i32 (v4i32 V128:$Rm))))))]>;
-}
-
-multiclass SIMDLongThreeVectorTiedBHSabal<bit U, bits<4> opc,
-                                          string asm,
-                                          SDPatternOperator OpNode> {
-  def v8i8_v8i16   : BaseSIMDDifferentThreeVectorTied<U, 0b000, opc,
-                                                  V128, V64, V64,
-                                                  asm, ".8h", ".8b", ".8b",
-    [(set (v8i16 V128:$dst),
-          (add (v8i16 V128:$Rd),
-               (zext (v8i8 (OpNode (v8i8 V64:$Rn), (v8i8 V64:$Rm))))))]>;
-  def v16i8_v8i16  : BaseSIMDDifferentThreeVectorTied<U, 0b001, opc,
-                                                 V128, V128, V128,
-                                                 asm#"2", ".8h", ".16b", ".16b",
-    [(set (v8i16 V128:$dst),
-          (add (v8i16 V128:$Rd),
-               (zext (v8i8 (OpNode (extract_high_v16i8 (v16i8 V128:$Rn)),
-                                   (extract_high_v16i8 (v16i8 V128:$Rm)))))))]>;
-  def v4i16_v4i32  : BaseSIMDDifferentThreeVectorTied<U, 0b010, opc,
-                                                  V128, V64, V64,
-                                                  asm, ".4s", ".4h", ".4h",
-    [(set (v4i32 V128:$dst),
-          (add (v4i32 V128:$Rd),
-               (zext (v4i16 (OpNode (v4i16 V64:$Rn), (v4i16 V64:$Rm))))))]>;
-  def v8i16_v4i32  : BaseSIMDDifferentThreeVectorTied<U, 0b011, opc,
-                                                  V128, V128, V128,
-                                                  asm#"2", ".4s", ".8h", ".8h",
-    [(set (v4i32 V128:$dst),
-          (add (v4i32 V128:$Rd),
-               (zext (v4i16 (OpNode (extract_high_v8i16 (v8i16 V128:$Rn)),
-                                    (extract_high_v8i16 (v8i16 V128:$Rm)))))))]>;
-  def v2i32_v2i64  : BaseSIMDDifferentThreeVectorTied<U, 0b100, opc,
-                                                  V128, V64, V64,
-                                                  asm, ".2d", ".2s", ".2s",
-    [(set (v2i64 V128:$dst),
-          (add (v2i64 V128:$Rd),
-               (zext (v2i32 (OpNode (v2i32 V64:$Rn), (v2i32 V64:$Rm))))))]>;
-  def v4i32_v2i64  : BaseSIMDDifferentThreeVectorTied<U, 0b101, opc,
-                                                  V128, V128, V128,
-                                                  asm#"2", ".2d", ".4s", ".4s",
-    [(set (v2i64 V128:$dst),
-          (add (v2i64 V128:$Rd),
-               (zext (v2i32 (OpNode (extract_high_v4i32 (v4i32 V128:$Rn)),
-                                    (extract_high_v4i32 (v4i32 V128:$Rm)))))))]>;
-}
-
+let isCommutable = 1 in
 multiclass SIMDLongThreeVectorBHS<bit U, bits<4> opc, string asm,
                                   SDPatternOperator OpNode = null_frag> {
   def v8i8_v8i16   : BaseSIMDDifferentThreeVector<U, 0b000, opc,
@@ -12564,7 +12487,7 @@ multiclass STOPregister<string asm, string instr> {
 let Predicates = [HasLSUI] in
 class BaseSTOPregisterLSUI<string asm, RegisterClass OP, Register Reg,
                         Instruction inst> :
-      InstAlias<asm # "\t$Rs, [$Rn]", (inst Reg, OP:$Rs, GPR64sp:$Rn), 0>;
+      InstAlias<asm # "\t$Rs, [$Rn]", (inst Reg, OP:$Rs, GPR64sp:$Rn)>;
 
 multiclass STOPregisterLSUI<string asm, string instr> {
   def : BaseSTOPregisterLSUI<asm # "l", GPR32, WZR,
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index fb59c9f131fb2..db028b4b7677c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -20,7 +20,9 @@
 #include "Utils/AArch64BaseInfo.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/CFIInstBuilder.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -83,6 +85,11 @@ static cl::opt<unsigned>
     BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26),
                       cl::desc("Restrict range of B instructions (DEBUG)"));
 
+static cl::opt<unsigned> GatherOptSearchLimit(
+    "aarch64-search-limit", cl::Hidden, cl::init(2048),
+    cl::desc("Restrict range of instructions to search for the "
+             "machine-combiner gather pattern optimization"));
+
 AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
     : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP,
                           AArch64::CATCHRET),
@@ -5078,8 +5085,13 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
         // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
         MCRegister DestRegX = TRI->getMatchingSuperReg(
             DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
-        MCRegister SrcRegX = TRI->getMatchingSuperReg(
-            SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
+        assert(DestRegX.isValid() && "Destination super-reg not valid");
+        MCRegister SrcRegX =
+            SrcReg == AArch64::WZR
+                ? AArch64::XZR
+                : TRI->getMatchingSuperReg(SrcReg, AArch64::sub_32,
+                                           &AArch64::GPR64spRegClass);
+        assert(SrcRegX.isValid() && "Source super-reg not valid");
         // This instruction is reading and writing X registers.  This may upset
         // the register scavenger and machine verifier, so we need to indicate
         // that we are reading an undefined value from SrcRegX, but a proper
@@ -5888,6 +5900,18 @@ static void appendReadRegExpr(SmallVectorImpl<char> &Expr, unsigned RegNum) {
   Expr.push_back(0);
 }
 
+// Convenience function to create a DWARF expression for loading a register from
+// a CFA offset.
+static void appendLoadRegExpr(SmallVectorImpl<char> &Expr,
+                              int64_t OffsetFromDefCFA) {
+  // This assumes the top of the DWARF stack contains the CFA.
+  Expr.push_back(dwarf::DW_OP_dup);
+  // Add the offset to the register.
+  appendConstantExpr(Expr, OffsetFromDefCFA, dwarf::DW_OP_plus);
+  // Dereference the address (loads a 64 bit value)..
+  Expr.push_back(dwarf::DW_OP_deref);
+}
+
 // Convenience function to create a comment for
 //  (+/-) NumBytes (* RegScale)?
 static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment,
@@ -5920,7 +5944,7 @@ static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
   // Build up the expression (Reg + NumBytes + VG * NumVGScaledBytes)
   SmallString<64> Expr;
   unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
-  assert(DwarfReg >= 0 && DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
+  assert(DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
   // Reg + NumBytes
   Expr.push_back(dwarf::DW_OP_breg0 + DwarfReg);
   appendLEB128<LEB128Sign::Signed>(Expr, NumBytes);
@@ -5956,9 +5980,10 @@ MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
   return MCCFIInstruction::cfiDefCfa(nullptr, DwarfReg, (int)Offset.getFixed());
 }
 
-MCCFIInstruction llvm::createCFAOffset(const TargetRegisterInfo &TRI,
-                                       unsigned Reg,
-                                       const StackOffset &OffsetFromDefCFA) {
+MCCFIInstruction
+llvm::createCFAOffset(const TargetRegisterInfo &TRI, unsigned Reg,
+                      const StackOffset &OffsetFromDefCFA,
+                      std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
   int64_t NumBytes, NumVGScaledBytes;
   AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
       OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
@@ -5977,9 +6002,16 @@ MCCFIInstruction llvm::createCFAOffset(const TargetRegisterInfo &TRI,
   assert(NumVGScaledBytes && "Expected scalable offset");
   SmallString<64> OffsetExpr;
   // + VG * NumVGScaledBytes
-  appendOffsetComment(NumVGScaledBytes, Comment, "* VG");
-  appendReadRegExpr(OffsetExpr, TRI.getDwarfRegNum(AArch64::VG, true));
+  StringRef VGRegScale;
+  if (IncomingVGOffsetFromDefCFA) {
+    appendLoadRegExpr(OffsetExpr, *IncomingVGOffsetFromDefCFA);
+    VGRegScale = "* IncomingVG";
+  } else {
+    appendReadRegExpr(OffsetExpr, TRI.getDwarfRegNum(AArch64::VG, true));
+    VGRegScale = "* VG";
+  }
   appendConstantExpr(OffsetExpr, NumVGScaledBytes, dwarf::DW_OP_mul);
+  appendOffsetComment(NumVGScaledBytes, Comment, VGRegScale);
   OffsetExpr.push_back(dwarf::DW_OP_plus);
   if (NumBytes) {
     // + NumBytes
@@ -7412,11 +7444,319 @@ static bool getMiscPatterns(MachineInstr &Root,
   return false;
 }
 
+/// Check if the given instruction forms a gather load pattern that can be
+/// optimized for better Memory-Level Parallelism (MLP). This function
+/// identifies chains of NEON lane load instructions that load data from
+/// different memory addresses into individual lanes of a 128-bit vector
+/// register, then attempts to split the pattern into parallel loads to break
+/// the serial dependency between instructions.
+///
+/// Pattern Matched:
+///   Initial scalar load -> SUBREG_TO_REG (lane 0) -> LD1i* (lane 1) ->
+///   LD1i* (lane 2) -> ... -> LD1i* (lane N-1, Root)
+///
+/// Transformed Into:
+///   Two parallel vector loads using fewer lanes each, followed by ZIP1v2i64
+///   to combine the results, enabling better memory-level parallelism.
+///
+/// Supported Element Types:
+///   - 32-bit elements (LD1i32, 4 lanes total)
+///   - 16-bit elements (LD1i16, 8 lanes total)
+///   - 8-bit elements (LD1i8, 16 lanes total)
+static bool getGatherLanePattern(MachineInstr &Root,
+                                 SmallVectorImpl<unsigned> &Patterns,
+                                 unsigned LoadLaneOpCode, unsigned NumLanes) {
+  const MachineFunction *MF = Root.getMF();
+
+  // Early exit if optimizing for size.
+  if (MF->getFunction().hasMinSize())
+    return false;
+
+  const MachineRegisterInfo &MRI = MF->getRegInfo();
+  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+
+  // The root of the pattern must load into the last lane of the vector.
+  if (Root.getOperand(2).getImm() != NumLanes - 1)
+    return false;
+
+  // Check that we have load into all lanes except lane 0.
+  // For each load we also want to check that:
+  // 1. It has a single non-debug use (since we will be replacing the virtual
+  // register)
+  // 2. That the addressing mode only uses a single pointer operand
+  auto *CurrInstr = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
+  auto Range = llvm::seq<unsigned>(1, NumLanes - 1);
+  SmallSet<unsigned, 16> RemainingLanes(Range.begin(), Range.end());
+  SmallVector<const MachineInstr *, 16> LoadInstrs;
+  while (!RemainingLanes.empty() && CurrInstr &&
+         CurrInstr->getOpcode() == LoadLaneOpCode &&
+         MRI.hasOneNonDBGUse(CurrInstr->getOperand(0).getReg()) &&
+         CurrInstr->getNumOperands() == 4) {
+    RemainingLanes.erase(CurrInstr->getOperand(2).getImm());
+    LoadInstrs.push_back(CurrInstr);
+    CurrInstr = MRI.getUniqueVRegDef(CurrInstr->getOperand(1).getReg());
+  }
+
+  // Check that we have found a match for lanes N-1.. 1.
+  if (!RemainingLanes.empty())
+    return false;
+
+  // Match the SUBREG_TO_REG sequence.
+  if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
+    return false;
+
+  // Verify that the subreg to reg loads an integer into the first lane.
+  auto Lane0LoadReg = CurrInstr->getOperand(2).getReg();
+  unsigned SingleLaneSizeInBits = 128 / NumLanes;
+  if (TRI->getRegSizeInBits(Lane0LoadReg, MRI) != SingleLaneSizeInBits)
+    return false;
+
+  // Verify that it also has a single non debug use.
+  if (!MRI.hasOneNonDBGUse(Lane0LoadReg))
+    return false;
+
+  LoadInstrs.push_back(MRI.getUniqueVRegDef(Lane0LoadReg));
+
+  // If there is any chance of aliasing, do not apply the pattern.
+  // Walk backward through the MBB starting from Root.
+  // Exit early if we've encountered all load instructions or hit the search
+  // limit.
+  auto MBBItr = Root.getIterator();
+  unsigned RemainingSteps = GatherOptSearchLimit;
+  SmallPtrSet<const MachineInstr *, 16> RemainingLoadInstrs;
+  RemainingLoadInstrs.insert(LoadInstrs.begin(), LoadInstrs.end());
+  const MachineBasicBlock *MBB = Root.getParent();
+
+  for (; MBBItr != MBB->begin() && RemainingSteps > 0 &&
+         !RemainingLoadInstrs.empty();
+       --MBBItr, --RemainingSteps) {
+    const MachineInstr &CurrInstr = *MBBItr;
+
+    // Remove this instruction from remaining loads if it's one we're tracking.
+    RemainingLoadInstrs.erase(&CurrInstr);
+
+    // Check for potential aliasing with any of the load instructions to
+    // optimize.
+    if (CurrInstr.isLoadFoldBarrier())
+      return false;
+  }
+
+  // If we hit the search limit without finding all load instructions,
+  // don't match the pattern.
+  if (RemainingSteps == 0 && !RemainingLoadInstrs.empty())
+    return false;
+
+  switch (NumLanes) {
+  case 4:
+    Patterns.push_back(AArch64MachineCombinerPattern::GATHER_LANE_i32);
+    break;
+  case 8:
+    Patterns.push_back(AArch64MachineCombinerPattern::GATHER_LANE_i16);
+    break;
+  case 16:
+    Patterns.push_back(AArch64MachineCombinerPattern::GATHER_LANE_i8);
+    break;
+  default:
+    llvm_unreachable("Got bad number of lanes for gather pattern.");
+  }
+
+  return true;
+}
+
+/// Search for patterns of LD instructions we can optimize.
+static bool getLoadPatterns(MachineInstr &Root,
+                            SmallVectorImpl<unsigned> &Patterns) {
+
+  // The pattern searches for loads into single lanes.
+  switch (Root.getOpcode()) {
+  case AArch64::LD1i32:
+    return getGatherLanePattern(Root, Patterns, Root.getOpcode(), 4);
+  case AArch64::LD1i16:
+    return getGatherLanePattern(Root, Patterns, Root.getOpcode(), 8);
+  case AArch64::LD1i8:
+    return getGatherLanePattern(Root, Patterns, Root.getOpcode(), 16);
+  default:
+    return false;
+  }
+}
+
+/// Generate optimized instruction sequence for gather load patterns to improve
+/// Memory-Level Parallelism (MLP). This function transforms a chain of
+/// sequential NEON lane loads into parallel vector loads that can execute
+/// concurrently.
+static void
+generateGatherLanePattern(MachineInstr &Root,
+                          SmallVectorImpl<MachineInstr *> &InsInstrs,
+                          SmallVectorImpl<MachineInstr *> &DelInstrs,
+                          DenseMap<Register, unsigned> &InstrIdxForVirtReg,
+                          unsigned Pattern, unsigned NumLanes) {
+  MachineFunction &MF = *Root.getParent()->getParent();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
+
+  // Gather the initial load instructions to build the pattern.
+  SmallVector<MachineInstr *, 16> LoadToLaneInstrs;
+  MachineInstr *CurrInstr = &Root;
+  for (unsigned i = 0; i < NumLanes - 1; ++i) {
+    LoadToLaneInstrs.push_back(CurrInstr);
+    CurrInstr = MRI.getUniqueVRegDef(CurrInstr->getOperand(1).getReg());
+  }
+
+  // Sort the load instructions according to the lane.
+  llvm::sort(LoadToLaneInstrs,
+             [](const MachineInstr *A, const MachineInstr *B) {
+               return A->getOperand(2).getImm() > B->getOperand(2).getImm();
+             });
+
+  MachineInstr *SubregToReg = CurrInstr;
+  LoadToLaneInstrs.push_back(
+      MRI.getUniqueVRegDef(SubregToReg->getOperand(2).getReg()));
+  auto LoadToLaneInstrsAscending = llvm::reverse(LoadToLaneInstrs);
+
+  const TargetRegisterClass *FPR128RegClass =
+      MRI.getRegClass(Root.getOperand(0).getReg());
+
+  // Helper lambda to create a LD1 instruction.
+  auto CreateLD1Instruction = [&](MachineInstr *OriginalInstr,
+                                  Register SrcRegister, unsigned Lane,
+                                  Register OffsetRegister,
+                                  bool OffsetRegisterKillState) {
+    auto NewRegister = MRI.createVirtualRegister(FPR128RegClass);
+    MachineInstrBuilder LoadIndexIntoRegister =
+        BuildMI(MF, MIMetadata(*OriginalInstr), TII->get(Root.getOpcode()),
+                NewRegister)
+            .addReg(SrcRegister)
+            .addImm(Lane)
+            .addReg(OffsetRegister, getKillRegState(OffsetRegisterKillState));
+    InstrIdxForVirtReg.insert(std::make_pair(NewRegister, InsInstrs.size()));
+    InsInstrs.push_back(LoadIndexIntoRegister);
+    return NewRegister;
+  };
+
+  // Helper to create load instruction based on the NumLanes in the NEON
+  // register we are rewriting.
+  auto CreateLDRInstruction = [&](unsigned NumLanes, Register DestReg,
+                                  Register OffsetReg,
+                                  bool KillState) -> MachineInstrBuilder {
+    unsigned Opcode;
+    switch (NumLanes) {
+    case 4:
+      Opcode = AArch64::LDRSui;
+      break;
+    case 8:
+      Opcode = AArch64::LDRHui;
+      break;
+    case 16:
+      Opcode = AArch64::LDRBui;
+      break;
+    default:
+      llvm_unreachable(
+          "Got unsupported number of lanes in machine-combiner gather pattern");
+    }
+    // Immediate offset load
+    return BuildMI(MF, MIMetadata(Root), TII->get(Opcode), DestReg)
+        .addReg(OffsetReg)
+        .addImm(0);
+  };
+
+  // Load the remaining lanes into register 0.
+  auto LanesToLoadToReg0 =
+      llvm::make_range(LoadToLaneInstrsAscending.begin() + 1,
+                       LoadToLaneInstrsAscending.begin() + NumLanes / 2);
+  Register PrevReg = SubregToReg->getOperand(0).getReg();
+  for (auto [Index, LoadInstr] : llvm::enumerate(LanesToLoadToReg0)) {
+    const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
+    PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
+                                   OffsetRegOperand.getReg(),
+                                   OffsetRegOperand.isKill());
+    DelInstrs.push_back(LoadInstr);
+  }
+  Register LastLoadReg0 = PrevReg;
+
+  // First load into register 1. Perform an integer load to zero out the upper
+  // lanes in a single instruction.
+  MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
+  MachineInstr *OriginalSplitLoad =
+      *std::next(LoadToLaneInstrsAscending.begin(), NumLanes / 2);
+  Register DestRegForMiddleIndex = MRI.createVirtualRegister(
+      MRI.getRegClass(Lane0Load->getOperand(0).getReg()));
+
+  const MachineOperand &OriginalSplitToLoadOffsetOperand =
+      OriginalSplitLoad->getOperand(3);
+  MachineInstrBuilder MiddleIndexLoadInstr =
+      CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
+                           OriginalSplitToLoadOffsetOperand.getReg(),
+                           OriginalSplitToLoadOffsetOperand.isKill());
+
+  InstrIdxForVirtReg.insert(
+      std::make_pair(DestRegForMiddleIndex, InsInstrs.size()));
+  InsInstrs.push_back(MiddleIndexLoadInstr);
+  DelInstrs.push_back(OriginalSplitLoad);
+
+  // Subreg To Reg instruction for register 1.
+  Register DestRegForSubregToReg = MRI.createVirtualRegister(FPR128RegClass);
+  unsigned SubregType;
+  switch (NumLanes) {
+  case 4:
+    SubregType = AArch64::ssub;
+    break;
+  case 8:
+    SubregType = AArch64::hsub;
+    break;
+  case 16:
+    SubregType = AArch64::bsub;
+    break;
+  default:
+    llvm_unreachable(
+        "Got invalid NumLanes for machine-combiner gather pattern");
+  }
+
+  auto SubRegToRegInstr =
+      BuildMI(MF, MIMetadata(Root), TII->get(SubregToReg->getOpcode()),
+              DestRegForSubregToReg)
+          .addImm(0)
+          .addReg(DestRegForMiddleIndex, getKillRegState(true))
+          .addImm(SubregType);
+  InstrIdxForVirtReg.insert(
+      std::make_pair(DestRegForSubregToReg, InsInstrs.size()));
+  InsInstrs.push_back(SubRegToRegInstr);
+
+  // Load remaining lanes into register 1.
+  auto LanesToLoadToReg1 =
+      llvm::make_range(LoadToLaneInstrsAscending.begin() + NumLanes / 2 + 1,
+                       LoadToLaneInstrsAscending.end());
+  PrevReg = SubRegToRegInstr->getOperand(0).getReg();
+  for (auto [Index, LoadInstr] : llvm::enumerate(LanesToLoadToReg1)) {
+    const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(3);
+    PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
+                                   OffsetRegOperand.getReg(),
+                                   OffsetRegOperand.isKill());
+
+    // Do not add the last reg to DelInstrs - it will be removed later.
+    if (Index == NumLanes / 2 - 2) {
+      break;
+    }
+    DelInstrs.push_back(LoadInstr);
+  }
+  Register LastLoadReg1 = PrevReg;
+
+  // Create the final zip instruction to combine the results.
+  MachineInstrBuilder ZipInstr =
+      BuildMI(MF, MIMetadata(Root), TII->get(AArch64::ZIP1v2i64),
+              Root.getOperand(0).getReg())
+          .addReg(LastLoadReg0)
+          .addReg(LastLoadReg1);
+  InsInstrs.push_back(ZipInstr);
+}
+
 CombinerObjective
 AArch64InstrInfo::getCombinerObjective(unsigned Pattern) const {
   switch (Pattern) {
   case AArch64MachineCombinerPattern::SUBADD_OP1:
   case AArch64MachineCombinerPattern::SUBADD_OP2:
+  case AArch64MachineCombinerPattern::GATHER_LANE_i32:
+  case AArch64MachineCombinerPattern::GATHER_LANE_i16:
+  case AArch64MachineCombinerPattern::GATHER_LANE_i8:
     return CombinerObjective::MustReduceDepth;
   default:
     return TargetInstrInfo::getCombinerObjective(Pattern);
@@ -7446,6 +7786,10 @@ bool AArch64InstrInfo::getMachineCombinerPatterns(
   if (getMiscPatterns(Root, Patterns))
     return true;
 
+  // Load patterns
+  if (getLoadPatterns(Root, Patterns))
+    return true;
+
   return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
                                                      DoRegPressureReduce);
 }
@@ -8701,6 +9045,21 @@ void AArch64InstrInfo::genAlternativeCodeSequence(
     MUL = genFNegatedMAD(MF, MRI, TII, Root, InsInstrs);
     break;
   }
+  case AArch64MachineCombinerPattern::GATHER_LANE_i32: {
+    generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
+                              Pattern, 4);
+    break;
+  }
+  case AArch64MachineCombinerPattern::GATHER_LANE_i16: {
+    generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
+                              Pattern, 8);
+    break;
+  }
+  case AArch64MachineCombinerPattern::GATHER_LANE_i8: {
+    generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
+                              Pattern, 16);
+    break;
+  }
 
   } // end switch (Pattern)
   // Record MUL and ADD/SUB for deletion
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 7c255da333e4b..179574a73aa01 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -172,6 +172,10 @@ enum AArch64MachineCombinerPattern : unsigned {
   FMULv8i16_indexed_OP2,
 
   FNMADD,
+
+  GATHER_LANE_i32,
+  GATHER_LANE_i16,
+  GATHER_LANE_i8
 };
 class AArch64InstrInfo final : public AArch64GenInstrInfo {
   const AArch64RegisterInfo RI;
@@ -642,8 +646,10 @@ bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
 MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg,
                               unsigned Reg, const StackOffset &Offset,
                               bool LastAdjustmentWasScalable = true);
-MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg,
-                                 const StackOffset &OffsetFromDefCFA);
+MCCFIInstruction
+createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg,
+                const StackOffset &OffsetFromDefCFA,
+                std::optional<int64_t> IncomingVGOffsetFromDefCFA);
 
 /// emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg
 /// plus Offset.  This is intended to be used from within the prolog/epilog
@@ -820,7 +826,8 @@ enum DestructiveInstType {
   DestructiveBinaryComm         = TSFLAG_DESTRUCTIVE_INST_TYPE(0x6),
   DestructiveBinaryCommWithRev  = TSFLAG_DESTRUCTIVE_INST_TYPE(0x7),
   DestructiveTernaryCommWithRev = TSFLAG_DESTRUCTIVE_INST_TYPE(0x8),
-  DestructiveUnaryPassthru      = TSFLAG_DESTRUCTIVE_INST_TYPE(0x9),
+  Destructive2xRegImmUnpred     = TSFLAG_DESTRUCTIVE_INST_TYPE(0x9),
+  DestructiveUnaryPassthru      = TSFLAG_DESTRUCTIVE_INST_TYPE(0xa),
 };
 
 enum FalseLaneType {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index ac31236d8f2cf..89f88776d832c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -143,7 +143,7 @@ def HasFuseAES       : Predicate<"Subtarget->hasFuseAES()">,
                                  "fuse-aes">;
 def HasSVE           : Predicate<"Subtarget->isSVEAvailable()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSVE), "sve">;
-def HasSVEB16B16     : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSVEB16B16()">,
+def HasSVEB16B16     : Predicate<"Subtarget->hasSVEB16B16()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSVEB16B16), "sve-b16b16">;
 def HasSVE2          : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2()">,
                                  AssemblerPredicateWithAll<(all_of FeatureSVE2), "sve2">;
@@ -248,6 +248,10 @@ def HasSVE_or_SME
     : Predicate<"Subtarget->isSVEorStreamingSVEAvailable()">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME),
                 "sve or sme">;
+def HasNonStreamingSVE_or_SME2
+    : Predicate<"Subtarget->isNonStreamingSVEorSME2Available()">,
+                AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2),
+                "sve or sme2">;
 def HasNonStreamingSVE_or_SME2p1
     : Predicate<"Subtarget->isSVEAvailable() ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p1())">,
@@ -5707,27 +5711,6 @@ let Predicates = [HasFullFP16] in {
 // Advanced SIMD two vector instructions.
 //===----------------------------------------------------------------------===//
 
-defm UABDL   : SIMDLongThreeVectorBHSabdl<1, 0b0111, "uabdl", abdu>;
-// Match UABDL in log2-shuffle patterns.
-def : Pat<(abs (v8i16 (sub (zext (v8i8 V64:$opA)),
-                           (zext (v8i8 V64:$opB))))),
-          (UABDLv8i8_v8i16 V64:$opA, V64:$opB)>;
-def : Pat<(abs (v8i16 (sub (zext (extract_high_v16i8 (v16i8 V128:$opA))),
-                           (zext (extract_high_v16i8 (v16i8 V128:$opB)))))),
-          (UABDLv16i8_v8i16 V128:$opA, V128:$opB)>;
-def : Pat<(abs (v4i32 (sub (zext (v4i16 V64:$opA)),
-                           (zext (v4i16 V64:$opB))))),
-          (UABDLv4i16_v4i32 V64:$opA, V64:$opB)>;
-def : Pat<(abs (v4i32 (sub (zext (extract_high_v8i16 (v8i16 V128:$opA))),
-                           (zext (extract_high_v8i16 (v8i16 V128:$opB)))))),
-          (UABDLv8i16_v4i32 V128:$opA, V128:$opB)>;
-def : Pat<(abs (v2i64 (sub (zext (v2i32 V64:$opA)),
-                           (zext (v2i32 V64:$opB))))),
-          (UABDLv2i32_v2i64 V64:$opA, V64:$opB)>;
-def : Pat<(abs (v2i64 (sub (zext (extract_high_v4i32 (v4i32 V128:$opA))),
-                           (zext (extract_high_v4i32 (v4i32 V128:$opB)))))),
-          (UABDLv4i32_v2i64 V128:$opA, V128:$opB)>;
-
 defm ABS    : SIMDTwoVectorBHSD<0, 0b01011, "abs", abs>;
 defm CLS    : SIMDTwoVectorBHS<0, 0b00100, "cls", int_aarch64_neon_cls>;
 defm CLZ    : SIMDTwoVectorBHS<1, 0b00100, "clz", ctlz>;
@@ -6055,6 +6038,7 @@ defm MLA      : SIMDThreeSameVectorBHSTied<0, 0b10010, "mla", null_frag>;
 defm MLS      : SIMDThreeSameVectorBHSTied<1, 0b10010, "mls", null_frag>;
 
 defm MUL      : SIMDThreeSameVectorBHS<0, 0b10011, "mul", mul>;
+let isCommutable = 1 in
 defm PMUL     : SIMDThreeSameVectorB<1, 0b10011, "pmul", int_aarch64_neon_pmul>;
 defm SABA     : SIMDThreeSameVectorBHSTied<0, 0b01111, "saba",
       TriOpFrag<(add node:$LHS, (abds node:$MHS, node:$RHS))> >;
@@ -6802,40 +6786,47 @@ def : Pat <(f64 (uint_to_fp (i32
 // Advanced SIMD three different-sized vector instructions.
 //===----------------------------------------------------------------------===//
 
-defm ADDHN  : SIMDNarrowThreeVectorBHS<0,0b0100,"addhn", int_aarch64_neon_addhn>;
-defm SUBHN  : SIMDNarrowThreeVectorBHS<0,0b0110,"subhn", int_aarch64_neon_subhn>;
-defm RADDHN : SIMDNarrowThreeVectorBHS<1,0b0100,"raddhn",int_aarch64_neon_raddhn>;
-defm RSUBHN : SIMDNarrowThreeVectorBHS<1,0b0110,"rsubhn",int_aarch64_neon_rsubhn>;
-defm PMULL  : SIMDDifferentThreeVectorBD<0,0b1110,"pmull", AArch64pmull>;
-defm SABAL  : SIMDLongThreeVectorTiedBHSabal<0,0b0101,"sabal", abds>;
-defm SABDL   : SIMDLongThreeVectorBHSabdl<0, 0b0111, "sabdl", abds>;
+defm ADDHN   : SIMDNarrowThreeVectorBHS<0,0b0100,"addhn", int_aarch64_neon_addhn>;
+defm SUBHN   : SIMDNarrowThreeVectorBHS<0,0b0110,"subhn", int_aarch64_neon_subhn>;
+defm RADDHN  : SIMDNarrowThreeVectorBHS<1,0b0100,"raddhn",int_aarch64_neon_raddhn>;
+defm RSUBHN  : SIMDNarrowThreeVectorBHS<1,0b0110,"rsubhn",int_aarch64_neon_rsubhn>;
+let isCommutable = 1 in
+defm PMULL   : SIMDDifferentThreeVectorBD<0,0b1110,"pmull", AArch64pmull>;
+defm SABAL   : SIMDLongThreeVectorTiedBHS<0,0b0101,"sabal",
+                 TriOpFrag<(add node:$LHS, (zext (abds node:$MHS, node:$RHS)))>>;
+defm SABDL   : SIMDLongThreeVectorBHS<0, 0b0111, "sabdl",
+                 BinOpFrag<(zext (abds node:$LHS, node:$RHS))>>;
 defm SADDL   : SIMDLongThreeVectorBHS<   0, 0b0000, "saddl",
-            BinOpFrag<(add (sext node:$LHS), (sext node:$RHS))>>;
+                BinOpFrag<(add (sext node:$LHS), (sext node:$RHS))>>;
 defm SADDW   : SIMDWideThreeVectorBHS<   0, 0b0001, "saddw",
                  BinOpFrag<(add node:$LHS, (sext node:$RHS))>>;
 defm SMLAL   : SIMDLongThreeVectorTiedBHS<0, 0b1000, "smlal",
-    TriOpFrag<(add node:$LHS, (AArch64smull node:$MHS, node:$RHS))>>;
+                 TriOpFrag<(add node:$LHS, (AArch64smull node:$MHS, node:$RHS))>>;
 defm SMLSL   : SIMDLongThreeVectorTiedBHS<0, 0b1010, "smlsl",
-    TriOpFrag<(sub node:$LHS, (AArch64smull node:$MHS, node:$RHS))>>;
+                 TriOpFrag<(sub node:$LHS, (AArch64smull node:$MHS, node:$RHS))>>;
 defm SMULL   : SIMDLongThreeVectorBHS<0, 0b1100, "smull", AArch64smull>;
 defm SQDMLAL : SIMDLongThreeVectorSQDMLXTiedHS<0, 0b1001, "sqdmlal", saddsat>;
 defm SQDMLSL : SIMDLongThreeVectorSQDMLXTiedHS<0, 0b1011, "sqdmlsl", ssubsat>;
-defm SQDMULL : SIMDLongThreeVectorHS<0, 0b1101, "sqdmull",
-                                     int_aarch64_neon_sqdmull>;
+defm SQDMULL : SIMDLongThreeVectorHS<0, 0b1101, "sqdmull", int_aarch64_neon_sqdmull>;
+let isCommutable = 0 in
 defm SSUBL   : SIMDLongThreeVectorBHS<0, 0b0010, "ssubl",
                  BinOpFrag<(sub (sext node:$LHS), (sext node:$RHS))>>;
 defm SSUBW   : SIMDWideThreeVectorBHS<0, 0b0011, "ssubw",
                  BinOpFrag<(sub node:$LHS, (sext node:$RHS))>>;
-defm UABAL   : SIMDLongThreeVectorTiedBHSabal<1, 0b0101, "uabal", abdu>;
+defm UABAL   : SIMDLongThreeVectorTiedBHS<1, 0b0101, "uabal",
+                 TriOpFrag<(add node:$LHS, (zext (abdu node:$MHS, node:$RHS)))>>;
+defm UABDL   : SIMDLongThreeVectorBHS<1, 0b0111, "uabdl",
+                 BinOpFrag<(zext (abdu node:$LHS, node:$RHS))>>;
 defm UADDL   : SIMDLongThreeVectorBHS<1, 0b0000, "uaddl",
                  BinOpFrag<(add (zanyext node:$LHS), (zanyext node:$RHS))>>;
 defm UADDW   : SIMDWideThreeVectorBHS<1, 0b0001, "uaddw",
                  BinOpFrag<(add node:$LHS, (zanyext node:$RHS))>>;
 defm UMLAL   : SIMDLongThreeVectorTiedBHS<1, 0b1000, "umlal",
-    TriOpFrag<(add node:$LHS, (AArch64umull node:$MHS, node:$RHS))>>;
+                 TriOpFrag<(add node:$LHS, (AArch64umull node:$MHS, node:$RHS))>>;
 defm UMLSL   : SIMDLongThreeVectorTiedBHS<1, 0b1010, "umlsl",
-    TriOpFrag<(sub node:$LHS, (AArch64umull node:$MHS, node:$RHS))>>;
+                 TriOpFrag<(sub node:$LHS, (AArch64umull node:$MHS, node:$RHS))>>;
 defm UMULL   : SIMDLongThreeVectorBHS<1, 0b1100, "umull", AArch64umull>;
+let isCommutable = 0 in
 defm USUBL   : SIMDLongThreeVectorBHS<1, 0b0010, "usubl",
                  BinOpFrag<(sub (zanyext node:$LHS), (zanyext node:$RHS))>>;
 defm USUBW   : SIMDWideThreeVectorBHS<   1, 0b0011, "usubw",
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index 782d62a7e5e13..e69fa32967a79 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -1193,7 +1193,8 @@ AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
       //   USE kill %w1   ; need to clear kill flag when moving STRWui downwards
       //   STRW %w0
       Register Reg = getLdStRegOp(*I).getReg();
-      for (MachineInstr &MI : make_range(std::next(I), Paired))
+      for (MachineInstr &MI :
+           make_range(std::next(I->getIterator()), Paired->getIterator()))
         MI.clearRegisterKills(Reg, TRI);
     }
   }
diff --git a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
index b97d6229b1d01..fd4ef2aa28f8a 100644
--- a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
@@ -8,8 +8,8 @@
 //
 // This pass performs below peephole optimizations on MIR level.
 //
-// 1. MOVi32imm + ANDS?Wrr ==> ANDWri + ANDS?Wri
-//    MOVi64imm + ANDS?Xrr ==> ANDXri + ANDS?Xri
+// 1. MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri
+//    MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri
 //
 // 2. MOVi32imm + ADDWrr ==> ADDWRi + ADDWRi
 //    MOVi64imm + ADDXrr ==> ADDXri + ADDXri
@@ -128,6 +128,7 @@ struct AArch64MIPeepholeOpt : public MachineFunctionPass {
   // Strategy used to split logical immediate bitmasks.
   enum class SplitStrategy {
     Intersect,
+    Disjoint,
   };
   template <typename T>
   bool trySplitLogicalImm(unsigned Opc, MachineInstr &MI,
@@ -163,6 +164,7 @@ INITIALIZE_PASS(AArch64MIPeepholeOpt, "aarch64-mi-peephole-opt",
 template <typename T>
 static bool splitBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc, T &Imm2Enc) {
   T UImm = static_cast<T>(Imm);
+  assert(UImm && (UImm != ~static_cast<T>(0)) && "Invalid immediate!");
 
   // The bitmask immediate consists of consecutive ones.  Let's say there is
   // constant 0b00000000001000000000010000000000 which does not consist of
@@ -190,19 +192,48 @@ static bool splitBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc, T &Imm2Enc) {
   return true;
 }
 
+template <typename T>
+static bool splitDisjointBitmaskImm(T Imm, unsigned RegSize, T &Imm1Enc,
+                                    T &Imm2Enc) {
+  assert(Imm && (Imm != ~static_cast<T>(0)) && "Invalid immediate!");
+
+  // Try to split a bitmask of the form 0b00000000011000000000011110000000 into
+  // two disjoint masks such as 0b00000000011000000000000000000000 and
+  // 0b00000000000000000000011110000000 where the inclusive/exclusive OR of the
+  // new masks match the original mask.
+  unsigned LowestBitSet = llvm::countr_zero(Imm);
+  unsigned LowestGapBitUnset =
+      LowestBitSet + llvm::countr_one(Imm >> LowestBitSet);
+
+  // Create a mask for the least significant group of consecutive ones.
+  assert(LowestGapBitUnset < sizeof(T) * CHAR_BIT && "Undefined behaviour!");
+  T NewImm1 = (static_cast<T>(1) << LowestGapBitUnset) -
+              (static_cast<T>(1) << LowestBitSet);
+  // Create a disjoint mask for the remaining ones.
+  T NewImm2 = Imm & ~NewImm1;
+
+  // Do not split if NewImm2 is not a valid bitmask immediate.
+  if (!AArch64_AM::isLogicalImmediate(NewImm2, RegSize))
+    return false;
+
+  Imm1Enc = AArch64_AM::encodeLogicalImmediate(NewImm1, RegSize);
+  Imm2Enc = AArch64_AM::encodeLogicalImmediate(NewImm2, RegSize);
+  return true;
+}
+
 template <typename T>
 bool AArch64MIPeepholeOpt::trySplitLogicalImm(unsigned Opc, MachineInstr &MI,
                                               SplitStrategy Strategy,
                                               unsigned OtherOpc) {
-  // Try below transformation.
+  // Try below transformations.
   //
-  // MOVi32imm + ANDS?Wrr ==> ANDWri + ANDS?Wri
-  // MOVi64imm + ANDS?Xrr ==> ANDXri + ANDS?Xri
+  // MOVi32imm + (ANDS?|EOR|ORR)Wrr ==> (AND|EOR|ORR)Wri + (ANDS?|EOR|ORR)Wri
+  // MOVi64imm + (ANDS?|EOR|ORR)Xrr ==> (AND|EOR|ORR)Xri + (ANDS?|EOR|ORR)Xri
   //
   // The mov pseudo instruction could be expanded to multiple mov instructions
   // later. Let's try to split the constant operand of mov instruction into two
-  // bitmask immediates. It makes only two AND instructions instead of multiple
-  // mov + and instructions.
+  // bitmask immediates based on the given split strategy. It makes only two
+  // logical instructions instead of multiple mov + logic instructions.
 
   return splitTwoPartImm<T>(
       MI,
@@ -224,6 +255,9 @@ bool AArch64MIPeepholeOpt::trySplitLogicalImm(unsigned Opc, MachineInstr &MI,
         case SplitStrategy::Intersect:
           SplitSucc = splitBitmaskImm(Imm, RegSize, Imm0, Imm1);
           break;
+        case SplitStrategy::Disjoint:
+          SplitSucc = splitDisjointBitmaskImm(Imm, RegSize, Imm0, Imm1);
+          break;
         }
         if (SplitSucc)
           return std::make_pair(Opc, !OtherOpc ? Opc : OtherOpc);
@@ -889,6 +923,22 @@ bool AArch64MIPeepholeOpt::runOnMachineFunction(MachineFunction &MF) {
         Changed |= trySplitLogicalImm<uint64_t>(
             AArch64::ANDXri, MI, SplitStrategy::Intersect, AArch64::ANDSXri);
         break;
+      case AArch64::EORWrr:
+        Changed |= trySplitLogicalImm<uint32_t>(AArch64::EORWri, MI,
+                                                SplitStrategy::Disjoint);
+        break;
+      case AArch64::EORXrr:
+        Changed |= trySplitLogicalImm<uint64_t>(AArch64::EORXri, MI,
+                                                SplitStrategy::Disjoint);
+        break;
+      case AArch64::ORRWrr:
+        Changed |= trySplitLogicalImm<uint32_t>(AArch64::ORRWri, MI,
+                                                SplitStrategy::Disjoint);
+        break;
+      case AArch64::ORRXrr:
+        Changed |= trySplitLogicalImm<uint64_t>(AArch64::ORRXri, MI,
+                                                SplitStrategy::Disjoint);
+        break;
       case AArch64::ORRWrs:
         Changed |= visitORR(MI);
         break;
diff --git a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index 800787cc0b4f5..1fde87e65a34b 100644
--- a/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -213,9 +213,6 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
   /// or return type
   bool IsSVECC = false;
 
-  /// The frame-index for the TPIDR2 object used for lazy saves.
-  TPIDR2Object TPIDR2;
-
   /// Whether this function changes streaming mode within the function.
   bool HasStreamingModeChanges = false;
 
@@ -231,25 +228,26 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
   // on function entry to record the initial pstate of a function.
   Register PStateSMReg = MCRegister::NoRegister;
 
-  // Holds a pointer to a buffer that is large enough to represent
-  // all SME ZA state and any additional state required by the
-  // __arm_sme_save/restore support routines.
-  Register SMESaveBufferAddr = MCRegister::NoRegister;
-
-  // true if SMESaveBufferAddr is used.
-  bool SMESaveBufferUsed = false;
+  // true if PStateSMReg is used.
+  bool PStateSMRegUsed = false;
 
   // Has the PNReg used to build PTRUE instruction.
   // The PTRUE is used for the LD/ST of ZReg pairs in save and restore.
   unsigned PredicateRegForFillSpill = 0;
 
-  // The stack slots where VG values are stored to.
-  int64_t VGIdx = std::numeric_limits<int>::max();
-  int64_t StreamingVGIdx = std::numeric_limits<int>::max();
-
   // Holds the SME function attributes (streaming mode, ZA/ZT0 state).
   SMEAttrs SMEFnAttrs;
 
+  // Note: The following properties are only used for the old SME ABI lowering:
+  /// The frame-index for the TPIDR2 object used for lazy saves.
+  TPIDR2Object TPIDR2;
+  // Holds a pointer to a buffer that is large enough to represent
+  // all SME ZA state and any additional state required by the
+  // __arm_sme_save/restore support routines.
+  Register SMESaveBufferAddr = MCRegister::NoRegister;
+  // true if SMESaveBufferAddr is used.
+  bool SMESaveBufferUsed = false;
+
 public:
   AArch64FunctionInfo(const Function &F, const AArch64Subtarget *STI);
 
@@ -258,6 +256,13 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
         const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
       const override;
 
+  // Old SME ABI lowering state getters/setters:
+  Register getSMESaveBufferAddr() const { return SMESaveBufferAddr; };
+  void setSMESaveBufferAddr(Register Reg) { SMESaveBufferAddr = Reg; };
+  unsigned isSMESaveBufferUsed() const { return SMESaveBufferUsed; };
+  void setSMESaveBufferUsed(bool Used = true) { SMESaveBufferUsed = Used; };
+  TPIDR2Object &getTPIDR2Obj() { return TPIDR2; }
+
   void setPredicateRegForFillSpill(unsigned Reg) {
     PredicateRegForFillSpill = Reg;
   }
@@ -265,26 +270,15 @@ class AArch64FunctionInfo final : public MachineFunctionInfo {
     return PredicateRegForFillSpill;
   }
 
-  Register getSMESaveBufferAddr() const { return SMESaveBufferAddr; };
-  void setSMESaveBufferAddr(Register Reg) { SMESaveBufferAddr = Reg; };
-
-  unsigned isSMESaveBufferUsed() const { return SMESaveBufferUsed; };
-  void setSMESaveBufferUsed(bool Used = true) { SMESaveBufferUsed = Used; };
-
   Register getPStateSMReg() const { return PStateSMReg; };
   void setPStateSMReg(Register Reg) { PStateSMReg = Reg; };
 
-  int64_t getVGIdx() const { return VGIdx; };
-  void setVGIdx(unsigned Idx) { VGIdx = Idx; };
-
-  int64_t getStreamingVGIdx() const { return StreamingVGIdx; };
-  void setStreamingVGIdx(unsigned FrameIdx) { StreamingVGIdx = FrameIdx; };
+  unsigned isPStateSMRegUsed() const { return PStateSMRegUsed; };
+  void setPStateSMRegUsed(bool Used = true) { PStateSMRegUsed = Used; };
 
   bool isSVECC() const { return IsSVECC; };
   void setIsSVECC(bool s) { IsSVECC = s; };
 
-  TPIDR2Object &getTPIDR2Obj() { return TPIDR2; }
-
   void initializeBaseYamlFields(const yaml::AArch64FunctionInfo &YamlMFI);
 
   unsigned getBytesInStackArgArea() const { return BytesInStackArgArea; }
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index adc984ad795af..42eaeca906e66 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -22,7 +22,8 @@ def TuneA320 : SubtargetFeature<"a320", "ARMProcFamily", "CortexA320",
                                    FeatureFuseAES,
                                    FeatureFuseAdrpAdd,
                                    FeaturePostRAScheduler,
-                                   FeatureUseWzrToVecMove]>;
+                                   FeatureUseWzrToVecMove,
+                                   FeatureUseFixedOverScalableIfEqualCost]>;
 
 def TuneA53     : SubtargetFeature<"a53", "ARMProcFamily", "CortexA53",
                                    "Cortex-A53 ARM processors", [
@@ -45,7 +46,8 @@ def TuneA510    : SubtargetFeature<"a510", "ARMProcFamily", "CortexA510",
                                    FeatureFuseAES,
                                    FeatureFuseAdrpAdd,
                                    FeaturePostRAScheduler,
-                                   FeatureUseWzrToVecMove
+                                   FeatureUseWzrToVecMove,
+                                   FeatureUseFixedOverScalableIfEqualCost
                                    ]>;
 
 def TuneA520    : SubtargetFeature<"a520", "ARMProcFamily", "CortexA520",
@@ -53,7 +55,8 @@ def TuneA520    : SubtargetFeature<"a520", "ARMProcFamily", "CortexA520",
                                    FeatureFuseAES,
                                    FeatureFuseAdrpAdd,
                                    FeaturePostRAScheduler,
-                                   FeatureUseWzrToVecMove]>;
+                                   FeatureUseWzrToVecMove,
+                                   FeatureUseFixedOverScalableIfEqualCost]>;
 
 def TuneA520AE  : SubtargetFeature<"a520ae", "ARMProcFamily", "CortexA520",
                                    "Cortex-A520AE ARM processors", [
@@ -318,7 +321,6 @@ def TuneAppleA7  : SubtargetFeature<"apple-a7", "ARMProcFamily", "AppleA7",
                                     FeatureFuseAES, FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
                                     FeatureZCRegMoveGPR64,
-                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing,
                                     FeatureZCZeroingFPWorkaround]>;
 
@@ -332,7 +334,6 @@ def TuneAppleA10 : SubtargetFeature<"apple-a10", "ARMProcFamily", "AppleA10",
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
                                     FeatureZCRegMoveGPR64,
-                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11",
@@ -345,7 +346,6 @@ def TuneAppleA11 : SubtargetFeature<"apple-a11", "ARMProcFamily", "AppleA11",
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
                                     FeatureZCRegMoveGPR64,
-                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12",
@@ -358,7 +358,6 @@ def TuneAppleA12 : SubtargetFeature<"apple-a12", "ARMProcFamily", "AppleA12",
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
                                     FeatureZCRegMoveGPR64,
-                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13",
@@ -371,7 +370,6 @@ def TuneAppleA13 : SubtargetFeature<"apple-a13", "ARMProcFamily", "AppleA13",
                                     FeatureFuseCryptoEOR,
                                     FeatureStorePairSuppress,
                                     FeatureZCRegMoveGPR64,
-                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14",
@@ -389,7 +387,6 @@ def TuneAppleA14 : SubtargetFeature<"apple-a14", "ARMProcFamily", "AppleA14",
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
                                     FeatureZCRegMoveGPR64,
-                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA15 : SubtargetFeature<"apple-a15", "ARMProcFamily", "AppleA15",
@@ -407,7 +404,6 @@ def TuneAppleA15 : SubtargetFeature<"apple-a15", "ARMProcFamily", "AppleA15",
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
                                     FeatureZCRegMoveGPR64,
-                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA16 : SubtargetFeature<"apple-a16", "ARMProcFamily", "AppleA16",
@@ -425,7 +421,6 @@ def TuneAppleA16 : SubtargetFeature<"apple-a16", "ARMProcFamily", "AppleA16",
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
                                     FeatureZCRegMoveGPR64,
-                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleA17 : SubtargetFeature<"apple-a17", "ARMProcFamily", "AppleA17",
@@ -443,7 +438,6 @@ def TuneAppleA17 : SubtargetFeature<"apple-a17", "ARMProcFamily", "AppleA17",
                                     FeatureFuseLiterals,
                                     FeatureStorePairSuppress,
                                     FeatureZCRegMoveGPR64,
-                                    FeatureZCRegMoveFPR64,
                                     FeatureZCZeroing]>;
 
 def TuneAppleM4 : SubtargetFeature<"apple-m4", "ARMProcFamily", "AppleM4",
@@ -460,7 +454,6 @@ def TuneAppleM4 : SubtargetFeature<"apple-m4", "ARMProcFamily", "AppleM4",
                                      FeatureFuseCryptoEOR,
                                      FeatureFuseLiterals,
                                      FeatureZCRegMoveGPR64,
-                                     FeatureZCRegMoveFPR64,
                                      FeatureZCZeroing
                                      ]>;
 
@@ -756,7 +749,6 @@ def ProcessorFeatures {
                                  FeatureSB, FeaturePAuth, FeatureSSBS, FeatureSVE, FeatureSVE2,
                                  FeatureComplxNum, FeatureCRC, FeatureDotProd,
                                  FeatureFPARMv8,FeatureFullFP16, FeatureJS, FeatureLSE,
-                                 FeatureUseFixedOverScalableIfEqualCost,
                                  FeatureRAS, FeatureRCPC, FeatureRDM, FeatureFPAC];
   list<SubtargetFeature> A520 = [HasV9_2aOps, FeaturePerfMon, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVEBitPerm,
@@ -766,7 +758,6 @@ def ProcessorFeatures {
                                  FeatureSVE, FeatureSVE2, FeatureBF16, FeatureComplxNum, FeatureCRC,
                                  FeatureFPARMv8, FeatureFullFP16, FeatureMatMulInt8, FeatureJS,
                                  FeatureNEON, FeatureLSE, FeatureRAS, FeatureRCPC, FeatureRDM,
-                                 FeatureUseFixedOverScalableIfEqualCost,
                                  FeatureDotProd, FeatureFPAC];
   list<SubtargetFeature> A520AE = [HasV9_2aOps, FeaturePerfMon, FeatureAM,
                                  FeatureMTE, FeatureETE, FeatureSVEBitPerm,
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index db27ca978980f..125225df15464 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -39,11 +39,18 @@ def AArch64_save_zt : SDNode<"AArch64ISD::SAVE_ZT", SDTypeProfile<0, 2,
 def AArch64CoalescerBarrier
     : SDNode<"AArch64ISD::COALESCER_BARRIER", SDTypeProfile<1, 1, []>, [SDNPOptInGlue, SDNPOutGlue]>;
 
-def AArch64VGSave : SDNode<"AArch64ISD::VG_SAVE", SDTypeProfile<0, 0, []>,
-                           [SDNPHasChain, SDNPSideEffect, SDNPOptInGlue, SDNPOutGlue]>;
+def AArch64EntryPStateSM
+  : SDNode<"AArch64ISD::ENTRY_PSTATE_SM", SDTypeProfile<1, 0,
+    [SDTCisInt<0>]>, [SDNPHasChain, SDNPSideEffect]>;
 
-def AArch64VGRestore : SDNode<"AArch64ISD::VG_RESTORE", SDTypeProfile<0, 0, []>,
-                              [SDNPHasChain, SDNPSideEffect, SDNPOptInGlue, SDNPOutGlue]>;
+let usesCustomInserter = 1 in {
+  def EntryPStateSM : Pseudo<(outs GPR64:$is_streaming), (ins), []>, Sched<[]> {}
+}
+def : Pat<(i64 (AArch64EntryPStateSM)), (EntryPStateSM)>;
+
+//===----------------------------------------------------------------------===//
+// Old SME ABI lowering ISD nodes/pseudos (deprecated)
+//===----------------------------------------------------------------------===//
 
 def AArch64AllocateZABuffer : SDNode<"AArch64ISD::ALLOCATE_ZA_BUFFER", SDTypeProfile<1, 1,
                               [SDTCisInt<0>, SDTCisInt<1>]>,
@@ -77,6 +84,30 @@ let usesCustomInserter = 1, Defs = [SP] in {
 def : Pat<(i64 (AArch64AllocateSMESaveBuffer GPR64:$size)),
           (AllocateSMESaveBuffer $size)>;
 
+//===----------------------------------------------------------------------===//
+// New SME ABI lowering ISD nodes/pseudos (-aarch64-new-sme-abi)
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 1, isMeta = 1 in {
+  def InOutZAUsePseudo : Pseudo<(outs), (ins), []>, Sched<[]>;
+  def RequiresZASavePseudo : Pseudo<(outs), (ins), []>, Sched<[]>;
+}
+
+def CommitZASavePseudo
+  : Pseudo<(outs),
+           (ins GPR64:$tpidr2_el0, i1imm:$zero_za, i64imm:$commit_routine, variable_ops), []>,
+           Sched<[]>;
+
+def AArch64_inout_za_use
+  : SDNode<"AArch64ISD::INOUT_ZA_USE", SDTypeProfile<0, 0,[]>,
+           [SDNPHasChain, SDNPInGlue]>;
+def : Pat<(AArch64_inout_za_use), (InOutZAUsePseudo)>;
+
+def AArch64_requires_za_save
+  : SDNode<"AArch64ISD::REQUIRES_ZA_SAVE", SDTypeProfile<0, 0,[]>,
+           [SDNPHasChain, SDNPInGlue]>;
+def : Pat<(AArch64_requires_za_save), (RequiresZASavePseudo)>;
+
 //===----------------------------------------------------------------------===//
 // Instruction naming conventions.
 //===----------------------------------------------------------------------===//
@@ -325,16 +356,6 @@ def : Pat<(AArch64_smstart (i32 svcr_op:$pstate)),
 def : Pat<(AArch64_smstop (i32 svcr_op:$pstate)),
           (MSRpstatesvcrImm1 svcr_op:$pstate, 0b0)>;
 
-
-// Pseudo to insert cfi_offset/cfi_restore instructions. Used to save or restore
-// the streaming value of VG around streaming-mode changes in locally-streaming
-// functions.
-def VGSavePseudo : Pseudo<(outs), (ins), []>, Sched<[]>;
-def : Pat<(AArch64VGSave), (VGSavePseudo)>;
-
-def VGRestorePseudo : Pseudo<(outs), (ins), []>, Sched<[]>;
-def : Pat<(AArch64VGRestore), (VGRestorePseudo)>;
-
 //===----------------------------------------------------------------------===//
 // SME2 Instructions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 0c4b4f4c3ed88..eeb47b4d99750 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -1021,7 +1021,9 @@ let Predicates = [HasNonStreamingSVE_or_SME2p2] in {
 let Predicates = [HasSVE_or_SME] in {
   defm INSR_ZR : sve_int_perm_insrs<"insr", AArch64insr>;
   defm INSR_ZV : sve_int_perm_insrv<"insr", AArch64insr>;
-  defm EXT_ZZI : sve_int_perm_extract_i<"ext", AArch64ext>;
+  defm EXT_ZZI : sve_int_perm_extract_i<"ext", AArch64ext, "EXT_ZZI_CONSTRUCTIVE">;
+
+  def EXT_ZZI_CONSTRUCTIVE : UnpredRegImmPseudo<ZPR8, imm0_255>;
 
   defm RBIT_ZPmZ : sve_int_perm_rev_rbit<"rbit", AArch64rbit_mt>;
   defm REVB_ZPmZ : sve_int_perm_rev_revb<"revb", AArch64revb_mt>;
@@ -2131,21 +2133,37 @@ let Predicates = [HasSVE_or_SME] in {
             (LASTB_VPZ_D (PTRUE_D 31), ZPR:$Z1), dsub))>;
 
   // Splice with lane bigger or equal to 0
-  foreach VT = [nxv16i8] in
+  foreach VT = [nxv16i8] in {
     def : Pat<(VT (vector_splice VT:$Z1, VT:$Z2, (i64 (sve_ext_imm_0_255 i32:$index)))),
               (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
+    let AddedComplexity = 1 in
+      def : Pat<(VT (vector_splice VT:$Z1, VT:$Z1, (i64 (sve_ext_imm_0_255 i32:$index)))),
+                (EXT_ZZI_CONSTRUCTIVE  ZPR:$Z1, imm0_255:$index)>;
+  }
 
-  foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in
+  foreach VT = [nxv8i16, nxv8f16, nxv8bf16] in {
     def : Pat<(VT (vector_splice VT:$Z1, VT:$Z2, (i64 (sve_ext_imm_0_127 i32:$index)))),
               (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
+    let AddedComplexity = 1 in
+      def : Pat<(VT (vector_splice VT:$Z1, VT:$Z1, (i64 (sve_ext_imm_0_127 i32:$index)))),
+                (EXT_ZZI_CONSTRUCTIVE  ZPR:$Z1, imm0_255:$index)>;
+  }
 
-  foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in
+  foreach VT = [nxv4i32, nxv4f16, nxv4f32, nxv4bf16] in {
     def : Pat<(VT (vector_splice VT:$Z1, VT:$Z2, (i64 (sve_ext_imm_0_63 i32:$index)))),
               (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
+    let AddedComplexity = 1 in
+      def : Pat<(VT (vector_splice VT:$Z1, VT:$Z1, (i64 (sve_ext_imm_0_63 i32:$index)))),
+                (EXT_ZZI_CONSTRUCTIVE  ZPR:$Z1, imm0_255:$index)>;
+  }
 
-  foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in
+  foreach VT = [nxv2i64, nxv2f16, nxv2f32, nxv2f64, nxv2bf16] in {
     def : Pat<(VT (vector_splice VT:$Z1, VT:$Z2, (i64 (sve_ext_imm_0_31 i32:$index)))),
               (EXT_ZZI  ZPR:$Z1, ZPR:$Z2, imm0_255:$index)>;
+    let AddedComplexity = 1 in
+      def : Pat<(VT (vector_splice VT:$Z1, VT:$Z1, (i64 (sve_ext_imm_0_31 i32:$index)))),
+                (EXT_ZZI_CONSTRUCTIVE  ZPR:$Z1, imm0_255:$index)>;
+  }
 
   defm CMPHS_PPzZZ : sve_int_cmp_0<0b000, "cmphs", SETUGE, SETULE>;
   defm CMPHI_PPzZZ : sve_int_cmp_0<0b001, "cmphi", SETUGT, SETULT>;
@@ -4390,7 +4408,7 @@ def : InstAlias<"pfalse\t$Pd", (PFALSE PPRorPNR8:$Pd), 0>;
 // Non-widening BFloat16 to BFloat16 instructions
 //===----------------------------------------------------------------------===//
 
-let Predicates = [HasSVEB16B16] in {
+let Predicates = [HasSVEB16B16, HasNonStreamingSVE_or_SME2] in {
 defm BFADD_ZZZ : sve_fp_3op_u_zd_bfloat<0b000, "bfadd", AArch64fadd>;
 defm BFSUB_ZZZ : sve_fp_3op_u_zd_bfloat<0b001, "bfsub", AArch64fsub>;
 defm BFMUL_ZZZ : sve_fp_3op_u_zd_bfloat<0b010, "bfmul", AArch64fmul>;
@@ -4423,9 +4441,9 @@ defm BFMLS_ZZZI : sve_fp_fma_by_indexed_elem_bfloat<"bfmls", 0b11, AArch64fmlsid
 defm BFMUL_ZZZI : sve_fp_fmul_by_indexed_elem_bfloat<"bfmul", AArch64fmulidx>;
 
 defm BFCLAMP_ZZZ : sve_fp_clamp_bfloat<"bfclamp", AArch64fclamp>;
-} // End HasSVEB16B16
+} // End HasSVEB16B16, HasNonStreamingSVE_or_SME2
 
-let Predicates = [HasSVEB16B16, UseExperimentalZeroingPseudos] in {
+let Predicates = [HasSVEB16B16, HasNonStreamingSVE_or_SME2, UseExperimentalZeroingPseudos] in {
 defm BFADD_ZPZZ   : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fadd>;
 defm BFSUB_ZPZZ   : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fsub>;
 defm BFMUL_ZPZZ   : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fmul>;
@@ -4433,7 +4451,7 @@ defm BFMAXNM_ZPZZ : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fmaxnm>;
 defm BFMINNM_ZPZZ : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fminnm>;
 defm BFMIN_ZPZZ   : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fmin>;
 defm BFMAX_ZPZZ   : sve_fp_2op_p_zds_zeroing_bfloat<int_aarch64_sve_fmax>;
-} // HasSVEB16B16, UseExperimentalZeroingPseudos
+} // HasSVEB16B16, HasNonStreamingSVE_or_SME2, UseExperimentalZeroingPseudos
 
 let Predicates = [HasSVEBFSCALE] in {
   def BFSCALE_ZPZZ : sve_fp_2op_p_zds_bfscale<0b1001, "bfscale", DestructiveBinary>;
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA320.td b/llvm/lib/Target/AArch64/AArch64SchedA320.td
index 89ed13389daf0..5ec95c707c28f 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedA320.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedA320.td
@@ -847,7 +847,7 @@ def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^[SU]XTB_ZPmZ
                                             "^[SU]XTW_ZPmZ_[D]")>;
 
 // Extract
-def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instrs EXT_ZZI, EXT_ZZI_B)>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instrs EXT_ZZI, EXT_ZZI_CONSTRUCTIVE, EXT_ZZI_B)>;
 
 // Extract narrow saturating
 def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU]QXTN[BT]_ZZ_[BHS]",
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA510.td b/llvm/lib/Target/AArch64/AArch64SchedA510.td
index 9456878946151..356e3fa39c53f 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedA510.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedA510.td
@@ -825,7 +825,7 @@ def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^[SU]XTB_ZPmZ
                                             "^[SU]XTW_ZPmZ_[D]")>;
 
 // Extract
-def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instrs EXT_ZZI, EXT_ZZI_B)>;
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instrs EXT_ZZI, EXT_ZZI_CONSTRUCTIVE, EXT_ZZI_B)>;
 
 // Extract narrow saturating
 def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]QXTN[BT]_ZZ_[BHS]",
@@ -1016,7 +1016,7 @@ def : InstRW<[CortexA510MCWrite<16, 13, CortexA510UnitVALU>], (instrs FADDA_VPZ_
 def : InstRW<[CortexA510MCWrite<8, 5, CortexA510UnitVALU>], (instrs FADDA_VPZ_D)>;
 
 // Floating point compare
-def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FACG[ET]_PPzZZ_[HSD]",
+def : InstRW<[CortexA510MCWrite<4, 2, CortexA510UnitVALU>], (instregex "^FACG[ET]_PPzZZ_[HSD]",
                                             "^FCM(EQ|GE|GT|NE)_PPzZ[0Z]_[HSD]",
                                             "^FCM(LE|LT)_PPzZ0_[HSD]",
                                             "^FCMUO_PPzZZ_[HSD]")>;
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
index 91a707910a7f3..e7982226ff3d1 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN2.td
@@ -1785,7 +1785,7 @@ def : InstRW<[N2Write_2c_1V1], (instregex "^[SU]XTB_ZPmZ_[HSD]",
                                           "^[SU]XTW_ZPmZ_[D]")>;
 
 // Extract
-def : InstRW<[N2Write_2c_1V], (instrs EXT_ZZI, EXT_ZZI_B)>;
+def : InstRW<[N2Write_2c_1V], (instrs EXT_ZZI, EXT_ZZI_CONSTRUCTIVE, EXT_ZZI_B)>;
 
 // Extract narrow saturating
 def : InstRW<[N2Write_4c_1V1], (instregex "^[SU]QXTN[BT]_ZZ_[BHS]$",
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN3.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN3.td
index ecfb1249cfc49..e44d40f8d7020 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseN3.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseN3.td
@@ -1757,7 +1757,7 @@ def : InstRW<[N3Write_2c_1V], (instregex "^[SU]XTB_ZPmZ_[HSD]",
                                          "^[SU]XTW_ZPmZ_[D]")>;
 
 // Extract
-def : InstRW<[N3Write_2c_1V], (instrs EXT_ZZI, EXT_ZZI_B)>;
+def : InstRW<[N3Write_2c_1V], (instrs EXT_ZZI, EXT_ZZI_CONSTRUCTIVE, EXT_ZZI_B)>;
 
 // Extract narrow saturating
 def : InstRW<[N3Write_4c_1V1], (instregex "^[SU]QXTN[BT]_ZZ_[BHS]$",
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
index 368665467859f..44625a2034d9d 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV1.td
@@ -1575,7 +1575,7 @@ def : InstRW<[V1Write_2c_1V1], (instregex "^[SU]XTB_ZPmZ_[HSD]",
                                           "^[SU]XTW_ZPmZ_[D]")>;
 
 // Extract
-def : InstRW<[V1Write_2c_1V01], (instrs EXT_ZZI)>;
+def : InstRW<[V1Write_2c_1V01], (instrs EXT_ZZI, EXT_ZZI_CONSTRUCTIVE)>;
 
 // Extract/insert operation, SIMD and FP scalar form
 def : InstRW<[V1Write_3c_1V1], (instregex "^LAST[AB]_VPZ_[BHSD]$",
diff --git a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td
index b2c3da03b4b84..6261220082029 100644
--- a/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td
+++ b/llvm/lib/Target/AArch64/AArch64SchedNeoverseV2.td
@@ -2272,7 +2272,7 @@ def : InstRW<[V2Write_2c_1V13], (instregex "^[SU]XTB_ZPmZ_[HSD]",
                                            "^[SU]XTW_ZPmZ_[D]")>;
 
 // Extract
-def : InstRW<[V2Write_2c_1V], (instrs EXT_ZZI, EXT_ZZI_B)>;
+def : InstRW<[V2Write_2c_1V], (instrs EXT_ZZI, EXT_ZZI_CONSTRUCTIVE, EXT_ZZI_B)>;
 
 // Extract narrow saturating
 def : InstRW<[V2Write_4c_1V13], (instregex "^[SU]QXTN[BT]_ZZ_[BHS]",
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 8a5b5ba09f3aa..d3b1aa621b61a 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -182,37 +182,25 @@ SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(
   const AArch64Subtarget &STI =
       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
   const AArch64TargetLowering *TLI = STI.getTargetLowering();
-  TargetLowering::ArgListEntry DstEntry;
-  DstEntry.Ty = PointerType::getUnqual(*DAG.getContext());
-  DstEntry.Node = Dst;
   TargetLowering::ArgListTy Args;
-  Args.push_back(DstEntry);
+  Args.emplace_back(Dst, PointerType::getUnqual(*DAG.getContext()));
 
   RTLIB::Libcall NewLC;
   switch (LC) {
   case RTLIB::MEMCPY: {
     NewLC = RTLIB::SC_MEMCPY;
-    TargetLowering::ArgListEntry Entry;
-    Entry.Ty = PointerType::getUnqual(*DAG.getContext());
-    Entry.Node = Src;
-    Args.push_back(Entry);
+    Args.emplace_back(Src, PointerType::getUnqual(*DAG.getContext()));
     break;
   }
   case RTLIB::MEMMOVE: {
     NewLC = RTLIB::SC_MEMMOVE;
-    TargetLowering::ArgListEntry Entry;
-    Entry.Ty = PointerType::getUnqual(*DAG.getContext());
-    Entry.Node = Src;
-    Args.push_back(Entry);
+    Args.emplace_back(Src, PointerType::getUnqual(*DAG.getContext()));
     break;
   }
   case RTLIB::MEMSET: {
     NewLC = RTLIB::SC_MEMSET;
-    TargetLowering::ArgListEntry Entry;
-    Entry.Ty = Type::getInt32Ty(*DAG.getContext());
-    Src = DAG.getZExtOrTrunc(Src, DL, MVT::i32);
-    Entry.Node = Src;
-    Args.push_back(Entry);
+    Args.emplace_back(DAG.getZExtOrTrunc(Src, DL, MVT::i32),
+                      Type::getInt32Ty(*DAG.getContext()));
     break;
   }
   default:
@@ -221,10 +209,7 @@ SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(
 
   EVT PointerVT = TLI->getPointerTy(DAG.getDataLayout());
   SDValue Symbol = DAG.getExternalSymbol(TLI->getLibcallName(NewLC), PointerVT);
-  TargetLowering::ArgListEntry SizeEntry;
-  SizeEntry.Node = Size;
-  SizeEntry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
-  Args.push_back(SizeEntry);
+  Args.emplace_back(Size, DAG.getDataLayout().getIntPtrType(*DAG.getContext()));
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   PointerType *RetTy = PointerType::getUnqual(*DAG.getContext());
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index f136a18421524..a67bd42aa16e0 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -585,8 +585,7 @@ bool AArch64StackTagging::runOnFunction(Function &Fn) {
                                    ClMaxLifetimes);
     if (StandardLifetime) {
       IntrinsicInst *Start = Info.LifetimeStart[0];
-      uint64_t Size =
-          cast<ConstantInt>(Start->getArgOperand(0))->getZExtValue();
+      uint64_t Size = *Info.AI->getAllocationSize(*DL);
       Size = alignTo(Size, kTagGranuleSize);
       tagAlloca(AI, Start->getNextNode(), TagPCall, Size);
 
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index d00e4471e107d..01c0bcc3a6a78 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -212,6 +212,13 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
     return hasSVE() || isStreamingSVEAvailable();
   }
 
+  /// Returns true if the target has access to either the full range of SVE
+  /// instructions, or the streaming-compatible subset of SVE instructions
+  /// available to SME2.
+  bool isNonStreamingSVEorSME2Available() const {
+    return isSVEAvailable() || (isSVEorStreamingSVEAvailable() && hasSME2());
+  }
+
   unsigned getMinVectorRegisterBitWidth() const {
     // Don't assume any minimum vector size when PSTATE.SM may not be 0, because
     // we don't yet support streaming-compatible codegen support that we trust
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 95eab16511e5a..e67bd5869ccd1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -224,6 +224,11 @@ static cl::opt<bool>
                            cl::desc("Enable Machine Pipeliner for AArch64"),
                            cl::init(false), cl::Hidden);
 
+static cl::opt<bool>
+    EnableNewSMEABILowering("aarch64-new-sme-abi",
+                            cl::desc("Enable new lowering for the SME ABI"),
+                            cl::init(false), cl::Hidden);
+
 extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
 LLVMInitializeAArch64Target() {
   // Register the target.
@@ -263,6 +268,7 @@ LLVMInitializeAArch64Target() {
   initializeLDTLSCleanupPass(PR);
   initializeKCFIPass(PR);
   initializeSMEABIPass(PR);
+  initializeMachineSMEABIPass(PR);
   initializeSMEPeepholeOptPass(PR);
   initializeSVEIntrinsicOptsPass(PR);
   initializeAArch64SpeculationHardeningPass(PR);
@@ -367,7 +373,8 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT,
           computeDefaultCPU(TT, CPU), FS, Options,
           getEffectiveRelocModel(TT, RM),
           getEffectiveAArch64CodeModel(TT, CM, JIT), OL),
-      TLOF(createTLOF(getTargetTriple())), isLittle(LittleEndian) {
+      TLOF(createTLOF(getTargetTriple())), isLittle(LittleEndian),
+      UseNewSMEABILowering(EnableNewSMEABILowering) {
   initAsmInfo();
 
   if (TT.isOSBinFormatMachO()) {
@@ -668,10 +675,12 @@ void AArch64PassConfig::addIRPasses() {
     addPass(createInterleavedAccessPass());
   }
 
-  // Expand any functions marked with SME attributes which require special
-  // changes for the calling convention or that require the lazy-saving
-  // mechanism specified in the SME ABI.
-  addPass(createSMEABIPass());
+  if (!EnableNewSMEABILowering) {
+    // Expand any functions marked with SME attributes which require special
+    // changes for the calling convention or that require the lazy-saving
+    // mechanism specified in the SME ABI.
+    addPass(createSMEABIPass());
+  }
 
   // Add Control Flow Guard checks.
   if (TM->getTargetTriple().isOSWindows()) {
@@ -782,6 +791,9 @@ bool AArch64PassConfig::addGlobalInstructionSelect() {
 }
 
 void AArch64PassConfig::addMachineSSAOptimization() {
+  if (EnableNewSMEABILowering && TM->getOptLevel() != CodeGenOptLevel::None)
+    addPass(createMachineSMEABIPass());
+
   if (TM->getOptLevel() != CodeGenOptLevel::None && EnableSMEPeepholeOpt)
     addPass(createSMEPeepholeOptPass());
 
@@ -812,6 +824,9 @@ bool AArch64PassConfig::addILPOpts() {
 }
 
 void AArch64PassConfig::addPreRegAlloc() {
+  if (TM->getOptLevel() == CodeGenOptLevel::None && EnableNewSMEABILowering)
+    addPass(createMachineSMEABIPass());
+
   // Change dead register definitions to refer to the zero register.
   if (TM->getOptLevel() != CodeGenOptLevel::None &&
       EnableDeadRegisterElimination)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.h b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
index b9e522dd6f226..0dd5d95b19b43 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.h
@@ -79,8 +79,12 @@ class AArch64TargetMachine : public CodeGenTargetMachineImpl {
   size_t clearLinkerOptimizationHints(
       const SmallPtrSetImpl<MachineInstr *> &MIs) const override;
 
+  /// Returns true if the new SME ABI lowering should be used.
+  bool useNewSMEABILowering() const { return UseNewSMEABILowering; }
+
 private:
   bool isLittle;
+  bool UseNewSMEABILowering;
 };
 
 // AArch64 little endian target machine.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 9f05add8bc1c1..490f6391c15a0 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -220,20 +220,17 @@ static cl::opt<bool> EnableFixedwidthAutovecInStreamingMode(
 static cl::opt<bool> EnableScalableAutovecInStreamingMode(
     "enable-scalable-autovec-in-streaming-mode", cl::init(false), cl::Hidden);
 
-static bool isSMEABIRoutineCall(const CallInst &CI) {
+static bool isSMEABIRoutineCall(const CallInst &CI,
+                                const AArch64TargetLowering &TLI) {
   const auto *F = CI.getCalledFunction();
-  return F && StringSwitch<bool>(F->getName())
-                  .Case("__arm_sme_state", true)
-                  .Case("__arm_tpidr2_save", true)
-                  .Case("__arm_tpidr2_restore", true)
-                  .Case("__arm_za_disable", true)
-                  .Default(false);
+  return F && SMEAttrs(F->getName(), TLI).isSMEABIRoutine();
 }
 
 /// Returns true if the function has explicit operations that can only be
 /// lowered using incompatible instructions for the selected mode. This also
 /// returns true if the function F may use or modify ZA state.
-static bool hasPossibleIncompatibleOps(const Function *F) {
+static bool hasPossibleIncompatibleOps(const Function *F,
+                                       const AArch64TargetLowering &TLI) {
   for (const BasicBlock &BB : *F) {
     for (const Instruction &I : BB) {
       // Be conservative for now and assume that any call to inline asm or to
@@ -242,7 +239,7 @@ static bool hasPossibleIncompatibleOps(const Function *F) {
       // all native LLVM instructions can be lowered to compatible instructions.
       if (isa<CallInst>(I) && !I.isDebugOrPseudoInst() &&
           (cast<CallInst>(I).isInlineAsm() || isa<IntrinsicInst>(I) ||
-           isSMEABIRoutineCall(cast<CallInst>(I))))
+           isSMEABIRoutineCall(cast<CallInst>(I), TLI)))
         return true;
     }
   }
@@ -290,7 +287,7 @@ bool AArch64TTIImpl::areInlineCompatible(const Function *Caller,
   if (CallAttrs.requiresLazySave() || CallAttrs.requiresSMChange() ||
       CallAttrs.requiresPreservingZT0() ||
       CallAttrs.requiresPreservingAllZAState()) {
-    if (hasPossibleIncompatibleOps(Callee))
+    if (hasPossibleIncompatibleOps(Callee, *getTLI()))
       return false;
   }
 
@@ -357,7 +354,7 @@ AArch64TTIImpl::getInlineCallPenalty(const Function *F, const CallBase &Call,
   // change only once and avoid inlining of G into F.
 
   SMEAttrs FAttrs(*F);
-  SMECallAttrs CallAttrs(Call);
+  SMECallAttrs CallAttrs(Call, getTLI());
 
   if (SMECallAttrs(FAttrs, CallAttrs.callee()).requiresSMChange()) {
     if (F == Call.getCaller()) // (1)
@@ -554,7 +551,17 @@ static bool isUnpackedVectorVT(EVT VecVT) {
          VecVT.getSizeInBits().getKnownMinValue() < AArch64::SVEBitsPerBlock;
 }
 
-static InstructionCost getHistogramCost(const IntrinsicCostAttributes &ICA) {
+static InstructionCost getHistogramCost(const AArch64Subtarget *ST,
+                                        const IntrinsicCostAttributes &ICA) {
+  // We need to know at least the number of elements in the vector of buckets
+  // and the size of each element to update.
+  if (ICA.getArgTypes().size() < 2)
+    return InstructionCost::getInvalid();
+
+  // Only interested in costing for the hardware instruction from SVE2.
+  if (!ST->hasSVE2())
+    return InstructionCost::getInvalid();
+
   Type *BucketPtrsTy = ICA.getArgTypes()[0]; // Type of vector of pointers
   Type *EltTy = ICA.getArgTypes()[1];        // Type of bucket elements
   unsigned TotalHistCnts = 1;
@@ -579,9 +586,11 @@ static InstructionCost getHistogramCost(const IntrinsicCostAttributes &ICA) {
 
     unsigned NaturalVectorWidth = AArch64::SVEBitsPerBlock / LegalEltSize;
     TotalHistCnts = EC / NaturalVectorWidth;
+
+    return InstructionCost(BaseHistCntCost * TotalHistCnts);
   }
 
-  return InstructionCost(BaseHistCntCost * TotalHistCnts);
+  return InstructionCost::getInvalid();
 }
 
 InstructionCost
@@ -597,10 +606,13 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
       return InstructionCost::getInvalid();
 
   switch (ICA.getID()) {
-  case Intrinsic::experimental_vector_histogram_add:
-    if (!ST->hasSVE2())
-      return InstructionCost::getInvalid();
-    return getHistogramCost(ICA);
+  case Intrinsic::experimental_vector_histogram_add: {
+    InstructionCost HistCost = getHistogramCost(ST, ICA);
+    // If the cost isn't valid, we may still be able to scalarize
+    if (HistCost.isValid())
+      return HistCost;
+    break;
+  }
   case Intrinsic::umin:
   case Intrinsic::umax:
   case Intrinsic::smin:
@@ -631,6 +643,13 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
         LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1 : 4;
     if (any_of(ValidSatTys, [&LT](MVT M) { return M == LT.second; }))
       return LT.first * Instrs;
+
+    TypeSize TS = getDataLayout().getTypeSizeInBits(RetTy);
+    uint64_t VectorSize = TS.getKnownMinValue();
+
+    if (ST->isSVEAvailable() && VectorSize >= 128 && isPowerOf2_64(VectorSize))
+      return LT.first * Instrs;
+
     break;
   }
   case Intrinsic::abs: {
@@ -651,6 +670,16 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
       return LT.first;
     break;
   }
+  case Intrinsic::fma:
+  case Intrinsic::fmuladd: {
+    // Given a fma or fmuladd, cost it the same as a fmul instruction which are
+    // usually the same for costs. TODO: Add fp16 and bf16 expansion costs.
+    Type *EltTy = RetTy->getScalarType();
+    if (EltTy->isFloatTy() || EltTy->isDoubleTy() ||
+        (EltTy->isHalfTy() && ST->hasFullFP16()))
+      return getArithmeticInstrCost(Instruction::FMul, RetTy, CostKind);
+    break;
+  }
   case Intrinsic::stepvector: {
     InstructionCost Cost = 1; // Cost of the `index' instruction
     auto LT = getTypeLegalizationCost(RetTy);
@@ -2072,6 +2101,20 @@ instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) {
              : std::nullopt;
 }
 
+static std::optional<Instruction *>
+instCombineSMECntsElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts,
+                       const AArch64Subtarget *ST) {
+  if (!ST->isStreaming())
+    return std::nullopt;
+
+  // In streaming-mode, aarch64_sme_cnts is equivalent to aarch64_sve_cnt
+  // with SVEPredPattern::all
+  Value *Cnt = IC.Builder.CreateElementCount(
+      II.getType(), ElementCount::getScalable(NumElts));
+  Cnt->takeName(&II);
+  return IC.replaceInstUsesWith(II, Cnt);
+}
+
 static std::optional<Instruction *> instCombineSVEPTest(InstCombiner &IC,
                                                         IntrinsicInst &II) {
   Value *PgVal = II.getArgOperand(0);
@@ -2781,6 +2824,14 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
     return instCombineSVECntElts(IC, II, 8);
   case Intrinsic::aarch64_sve_cntb:
     return instCombineSVECntElts(IC, II, 16);
+  case Intrinsic::aarch64_sme_cntsd:
+    return instCombineSMECntsElts(IC, II, 2, ST);
+  case Intrinsic::aarch64_sme_cntsw:
+    return instCombineSMECntsElts(IC, II, 4, ST);
+  case Intrinsic::aarch64_sme_cntsh:
+    return instCombineSMECntsElts(IC, II, 8, ST);
+  case Intrinsic::aarch64_sme_cntsb:
+    return instCombineSMECntsElts(IC, II, 16, ST);
   case Intrinsic::aarch64_sve_ptest_any:
   case Intrinsic::aarch64_sve_ptest_first:
   case Intrinsic::aarch64_sve_ptest_last:
@@ -3961,6 +4012,24 @@ InstructionCost AArch64TTIImpl::getVectorInstrCost(const Instruction &I,
   return getVectorInstrCostHelper(I.getOpcode(), Val, CostKind, Index, &I);
 }
 
+InstructionCost
+AArch64TTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
+                                                 TTI::TargetCostKind CostKind,
+                                                 unsigned Index) const {
+  if (isa<FixedVectorType>(Val))
+    return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,
+                                                   Index);
+
+  // This typically requires both while and lastb instructions in order
+  // to extract the last element. If this is in a loop the while
+  // instruction can at least be hoisted out, although it will consume a
+  // predicate register. The cost should be more expensive than the base
+  // extract cost, which is 2 for most CPUs.
+  return CostKind == TTI::TCK_CodeSize
+             ? 2
+             : ST->getVectorInsertExtractBaseCost() + 1;
+}
+
 InstructionCost AArch64TTIImpl::getScalarizationOverhead(
     VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
     TTI::TargetCostKind CostKind, bool ForPoisonSrc,
@@ -3975,6 +4044,27 @@ InstructionCost AArch64TTIImpl::getScalarizationOverhead(
   return DemandedElts.popcount() * (Insert + Extract) * VecInstCost;
 }
 
+std::optional<InstructionCost> AArch64TTIImpl::getFP16BF16PromoteCost(
+    Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
+    TTI::OperandValueInfo Op2Info, bool IncludeTrunc,
+    std::function<InstructionCost(Type *)> InstCost) const {
+  if (!Ty->getScalarType()->isHalfTy() && !Ty->getScalarType()->isBFloatTy())
+    return std::nullopt;
+  if (Ty->getScalarType()->isHalfTy() && ST->hasFullFP16())
+    return std::nullopt;
+
+  Type *PromotedTy = Ty->getWithNewType(Type::getFloatTy(Ty->getContext()));
+  InstructionCost Cost = getCastInstrCost(Instruction::FPExt, PromotedTy, Ty,
+                                          TTI::CastContextHint::None, CostKind);
+  if (!Op1Info.isConstant() && !Op2Info.isConstant())
+    Cost *= 2;
+  Cost += InstCost(PromotedTy);
+  if (IncludeTrunc)
+    Cost += getCastInstrCost(Instruction::FPTrunc, Ty, PromotedTy,
+                             TTI::CastContextHint::None, CostKind);
+  return Cost;
+}
+
 InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
@@ -3997,6 +4087,18 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
 
+  // Increase the cost for half and bfloat types if not architecturally
+  // supported.
+  if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
+      ISD == ISD::FDIV || ISD == ISD::FREM)
+    if (auto PromotedCost = getFP16BF16PromoteCost(
+            Ty, CostKind, Op1Info, Op2Info, /*IncludeTrunc=*/true,
+            [&](Type *PromotedTy) {
+              return getArithmeticInstrCost(Opcode, PromotedTy, CostKind,
+                                            Op1Info, Op2Info);
+            }))
+      return *PromotedCost;
+
   switch (ISD) {
   default:
     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
@@ -4265,11 +4367,6 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
     [[fallthrough]];
   case ISD::FADD:
   case ISD::FSUB:
-    // Increase the cost for half and bfloat types if not architecturally
-    // supported.
-    if ((Ty->getScalarType()->isHalfTy() && !ST->hasFullFP16()) ||
-        (Ty->getScalarType()->isBFloatTy() && !ST->hasBF16()))
-      return 2 * LT.first;
     if (!Ty->getScalarType()->isFP128Ty())
       return LT.first;
     [[fallthrough]];
@@ -4293,8 +4390,9 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
 }
 
 InstructionCost
-AArch64TTIImpl::getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
-                                          const SCEV *Ptr) const {
+AArch64TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
+                                          const SCEV *Ptr,
+                                          TTI::TargetCostKind CostKind) const {
   // Address computations in vectorized code with non-consecutive addresses will
   // likely result in more instructions compared to scalar code where the
   // computation can more often be merged into the index mode. The resulting
@@ -4302,7 +4400,7 @@ AArch64TTIImpl::getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
   unsigned NumVectorInstToHideOverhead = NeonNonConstStrideOverhead;
   int MaxMergeDistance = 64;
 
-  if (Ty->isVectorTy() && SE &&
+  if (PtrTy->isVectorTy() && SE &&
       !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MaxMergeDistance + 1))
     return NumVectorInstToHideOverhead;
 
@@ -4371,25 +4469,21 @@ InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
   }
 
   if (Opcode == Instruction::FCmp) {
-    // Without dedicated instructions we promote f16 + bf16 compares to f32.
-    if ((!ST->hasFullFP16() && ValTy->getScalarType()->isHalfTy()) ||
-        ValTy->getScalarType()->isBFloatTy()) {
-      Type *PromotedTy =
-          ValTy->getWithNewType(Type::getFloatTy(ValTy->getContext()));
-      InstructionCost Cost =
-          getCastInstrCost(Instruction::FPExt, PromotedTy, ValTy,
-                           TTI::CastContextHint::None, CostKind);
-      if (!Op1Info.isConstant() && !Op2Info.isConstant())
-        Cost *= 2;
-      Cost += getCmpSelInstrCost(Opcode, PromotedTy, CondTy, VecPred, CostKind,
-                                 Op1Info, Op2Info);
-      if (ValTy->isVectorTy())
-        Cost += getCastInstrCost(
-            Instruction::Trunc, VectorType::getInteger(cast<VectorType>(ValTy)),
-            VectorType::getInteger(cast<VectorType>(PromotedTy)),
-            TTI::CastContextHint::None, CostKind);
-      return Cost;
-    }
+    if (auto PromotedCost = getFP16BF16PromoteCost(
+            ValTy, CostKind, Op1Info, Op2Info, /*IncludeTrunc=*/false,
+            [&](Type *PromotedTy) {
+              InstructionCost Cost =
+                  getCmpSelInstrCost(Opcode, PromotedTy, CondTy, VecPred,
+                                     CostKind, Op1Info, Op2Info);
+              if (isa<VectorType>(PromotedTy))
+                Cost += getCastInstrCost(
+                    Instruction::Trunc,
+                    VectorType::getInteger(cast<VectorType>(ValTy)),
+                    VectorType::getInteger(cast<VectorType>(PromotedTy)),
+                    TTI::CastContextHint::None, CostKind);
+              return Cost;
+            }))
+      return *PromotedCost;
 
     auto LT = getTypeLegalizationCost(ValTy);
     // Model unknown fp compares as a libcall.
@@ -4858,32 +4952,18 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
   // Limit to loops with trip counts that are cheap to expand.
   UP.SCEVExpansionBudget = 1;
 
-  // Try to unroll small, single block loops, if they have load/store
-  // dependencies, to expose more parallel memory access streams.
+  // Try to unroll small loops, of few-blocks with low budget, if they have
+  // load/store dependencies, to expose more parallel memory access streams,
+  // or if they do little work inside a block (i.e. load -> X -> store pattern).
   BasicBlock *Header = L->getHeader();
-  if (Header == L->getLoopLatch()) {
+  BasicBlock *Latch = L->getLoopLatch();
+  if (Header == Latch) {
     // Estimate the size of the loop.
     unsigned Size;
-    if (!isLoopSizeWithinBudget(L, TTI, 8, &Size))
+    unsigned Width = 10;
+    if (!isLoopSizeWithinBudget(L, TTI, Width, &Size))
       return;
 
-    SmallPtrSet<Value *, 8> LoadedValues;
-    SmallVector<StoreInst *> Stores;
-    for (auto *BB : L->blocks()) {
-      for (auto &I : *BB) {
-        Value *Ptr = getLoadStorePointerOperand(&I);
-        if (!Ptr)
-          continue;
-        const SCEV *PtrSCEV = SE.getSCEV(Ptr);
-        if (SE.isLoopInvariant(PtrSCEV, L))
-          continue;
-        if (isa<LoadInst>(&I))
-          LoadedValues.insert(&I);
-        else
-          Stores.push_back(cast<StoreInst>(&I));
-      }
-    }
-
     // Try to find an unroll count that maximizes the use of the instruction
     // window, i.e. trying to fetch as many instructions per cycle as possible.
     unsigned MaxInstsPerLine = 16;
@@ -4902,8 +4982,32 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
       UC++;
     }
 
-    if (BestUC == 1 || none_of(Stores, [&LoadedValues](StoreInst *SI) {
-          return LoadedValues.contains(SI->getOperand(0));
+    if (BestUC == 1)
+      return;
+
+    SmallPtrSet<Value *, 8> LoadedValuesPlus;
+    SmallVector<StoreInst *> Stores;
+    for (auto *BB : L->blocks()) {
+      for (auto &I : *BB) {
+        Value *Ptr = getLoadStorePointerOperand(&I);
+        if (!Ptr)
+          continue;
+        const SCEV *PtrSCEV = SE.getSCEV(Ptr);
+        if (SE.isLoopInvariant(PtrSCEV, L))
+          continue;
+        if (isa<LoadInst>(&I)) {
+          LoadedValuesPlus.insert(&I);
+          // Include in-loop 1st users of loaded values.
+          for (auto *U : I.users())
+            if (L->contains(cast<Instruction>(U)))
+              LoadedValuesPlus.insert(U);
+        } else
+          Stores.push_back(cast<StoreInst>(&I));
+      }
+    }
+
+    if (none_of(Stores, [&LoadedValuesPlus](StoreInst *SI) {
+          return LoadedValuesPlus.contains(SI->getOperand(0));
         }))
       return;
 
@@ -4915,7 +5019,6 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
   // Try to runtime-unroll loops with early-continues depending on loop-varying
   // loads; this helps with branch-prediction for the early-continues.
   auto *Term = dyn_cast<BranchInst>(Header->getTerminator());
-  auto *Latch = L->getLoopLatch();
   SmallVector<BasicBlock *> Preds(predecessors(Latch));
   if (!Term || !Term->isConditional() || Preds.size() == 1 ||
       !llvm::is_contained(Preds, Header) ||
@@ -5151,6 +5254,8 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
     return false;
 
   switch (RdxDesc.getRecurrenceKind()) {
+  case RecurKind::Sub:
+  case RecurKind::AddChainWithSubs:
   case RecurKind::Add:
   case RecurKind::FAdd:
   case RecurKind::And:
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 7f45177437237..42ae962b3b426 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -220,6 +220,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
                                      TTI::TargetCostKind CostKind,
                                      unsigned Index) const override;
 
+  InstructionCost
+  getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
+                                   TTI::TargetCostKind CostKind,
+                                   unsigned Index) const override;
+
   InstructionCost
   getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
                          TTI::TargetCostKind CostKind) const override;
@@ -238,8 +243,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
       ArrayRef<const Value *> Args = {},
       const Instruction *CxtI = nullptr) const override;
 
-  InstructionCost getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
-                                            const SCEV *Ptr) const override;
+  InstructionCost
+  getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,
+                            TTI::TargetCostKind CostKind) const override;
 
   InstructionCost getCmpSelInstrCost(
       unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
@@ -435,6 +441,14 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
 
   bool preferPredicatedReductionSelect() const override { return ST->hasSVE(); }
 
+  /// FP16 and BF16 operations are lowered to fptrunc(op(fpext, fpext) if the
+  /// architecture features are not present.
+  std::optional<InstructionCost>
+  getFP16BF16PromoteCost(Type *Ty, TTI::TargetCostKind CostKind,
+                         TTI::OperandValueInfo Op1Info,
+                         TTI::OperandValueInfo Op2Info, bool IncludeTrunc,
+                         std::function<InstructionCost(Type *)> InstCost) const;
+
   InstructionCost
   getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
                              std::optional<FastMathFlags> FMF,
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 1ca61f5c6b349..3641e22e6f76a 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -7909,9 +7909,11 @@ bool AArch64AsmParser::parseDirectiveSEHSavePReg(SMLoc L) {
 }
 
 bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
-  // Expecting 3 AsmToken::Identifier after '.aeabi_subsection', a name and 2
-  // parameters, e.g.: .aeabi_subsection (1)aeabi_feature_and_bits, (2)optional,
-  // (3)uleb128 separated by 2 commas.
+  // Handle parsing of .aeabi_subsection directives
+  // - On first declaration of a subsection, expect exactly three identifiers
+  //   after `.aeabi_subsection`: the subsection name and two parameters.
+  // - When switching to an existing subsection, it is valid to provide only
+  //   the subsection name, or the name together with the two parameters.
   MCAsmParser &Parser = getParser();
 
   // Consume the name (subsection name)
@@ -7925,16 +7927,38 @@ bool AArch64AsmParser::parseDirectiveAeabiSubSectionHeader(SMLoc L) {
     return true;
   }
   Parser.Lex();
-  // consume a comma
+
+  std::unique_ptr<MCELFStreamer::AttributeSubSection> SubsectionExists =
+      getTargetStreamer().getAttributesSubsectionByName(SubsectionName);
+  // Check whether only the subsection name was provided.
+  // If so, the user is trying to switch to a subsection that should have been
+  // declared before.
+  if (Parser.getTok().is(llvm::AsmToken::EndOfStatement)) {
+    if (SubsectionExists) {
+      getTargetStreamer().emitAttributesSubsection(
+          SubsectionName,
+          static_cast<AArch64BuildAttributes::SubsectionOptional>(
+              SubsectionExists->IsOptional),
+          static_cast<AArch64BuildAttributes::SubsectionType>(
+              SubsectionExists->ParameterType));
+      return false;
+    }
+    // If subsection does not exists, report error.
+    else {
+      Error(Parser.getTok().getLoc(),
+            "Could not switch to subsection '" + SubsectionName +
+                "' using subsection name, subsection has not been defined");
+      return true;
+    }
+  }
+
+  // Otherwise, expecting 2 more parameters: consume a comma
   // parseComma() return *false* on success, and call Lex(), no need to call
   // Lex() again.
   if (Parser.parseComma()) {
     return true;
   }
 
-  std::unique_ptr<MCELFStreamer::AttributeSubSection> SubsectionExists =
-      getTargetStreamer().getAttributesSubsectionByName(SubsectionName);
-
   // Consume the first parameter (optionality parameter)
   AArch64BuildAttributes::SubsectionOptional IsOptional;
   // options: optional/required
diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 66136a464f05d..803943fd57c4d 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -89,6 +89,7 @@ add_llvm_target(AArch64CodeGen
   SMEABIPass.cpp
   SMEPeepholeOpt.cpp
   SVEIntrinsicOpts.cpp
+  MachineSMEABIPass.cpp
   AArch64SIMDInstrOpt.cpp
 
   DEPENDS
diff --git a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
index ae984be670fc2..323db2a0728eb 100644
--- a/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
+++ b/llvm/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
@@ -15,6 +15,7 @@
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "TargetInfo/AArch64TargetInfo.h"
 #include "Utils/AArch64BaseInfo.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCRelocationInfo.h"
 #include "llvm/MC/MCInst.h"
@@ -27,6 +28,7 @@
 #include <memory>
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "aarch64-disassembler"
 
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
index 010d0aaa46e7f..79bef76cf4c4f 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp
@@ -125,12 +125,12 @@ struct AArch64OutgoingValueAssigner
     bool UseVarArgsCCForFixed = IsCalleeWin && State.isVarArg();
 
     bool Res;
-    if (Info.IsFixed && !UseVarArgsCCForFixed) {
+    if (!Flags.isVarArg() && !UseVarArgsCCForFixed) {
       if (!IsReturn)
         applyStackPassedSmallTypeDAGHack(OrigVT, ValVT, LocVT);
-      Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
+      Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, Info.Ty, State);
     } else
-      Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, State);
+      Res = AssignFnVarArg(ValNo, ValVT, LocVT, LocInfo, Flags, Info.Ty, State);
 
     StackSize = State.getStackSize();
     return Res;
@@ -361,7 +361,7 @@ struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
     unsigned MaxSize = MemTy.getSizeInBytes() * 8;
     // For varargs, we always want to extend them to 8 bytes, in which case
     // we disable setting a max.
-    if (!Arg.IsFixed)
+    if (Arg.Flags[0].isVarArg())
       MaxSize = 0;
 
     Register ValVReg = Arg.Regs[RegIndex];
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index d9056926ff249..0bceb322726d1 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -1102,82 +1102,6 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
   return true;
 }
 
-static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
-  if (!DstTy.isScalar() || !SrcTy.isScalar())
-    return GenericOpc;
-
-  const unsigned DstSize = DstTy.getSizeInBits();
-  const unsigned SrcSize = SrcTy.getSizeInBits();
-
-  switch (DstSize) {
-  case 32:
-    switch (SrcSize) {
-    case 32:
-      switch (GenericOpc) {
-      case TargetOpcode::G_SITOFP:
-        return AArch64::SCVTFUWSri;
-      case TargetOpcode::G_UITOFP:
-        return AArch64::UCVTFUWSri;
-      case TargetOpcode::G_FPTOSI:
-        return AArch64::FCVTZSUWSr;
-      case TargetOpcode::G_FPTOUI:
-        return AArch64::FCVTZUUWSr;
-      default:
-        return GenericOpc;
-      }
-    case 64:
-      switch (GenericOpc) {
-      case TargetOpcode::G_SITOFP:
-        return AArch64::SCVTFUXSri;
-      case TargetOpcode::G_UITOFP:
-        return AArch64::UCVTFUXSri;
-      case TargetOpcode::G_FPTOSI:
-        return AArch64::FCVTZSUWDr;
-      case TargetOpcode::G_FPTOUI:
-        return AArch64::FCVTZUUWDr;
-      default:
-        return GenericOpc;
-      }
-    default:
-      return GenericOpc;
-    }
-  case 64:
-    switch (SrcSize) {
-    case 32:
-      switch (GenericOpc) {
-      case TargetOpcode::G_SITOFP:
-        return AArch64::SCVTFUWDri;
-      case TargetOpcode::G_UITOFP:
-        return AArch64::UCVTFUWDri;
-      case TargetOpcode::G_FPTOSI:
-        return AArch64::FCVTZSUXSr;
-      case TargetOpcode::G_FPTOUI:
-        return AArch64::FCVTZUUXSr;
-      default:
-        return GenericOpc;
-      }
-    case 64:
-      switch (GenericOpc) {
-      case TargetOpcode::G_SITOFP:
-        return AArch64::SCVTFUXDri;
-      case TargetOpcode::G_UITOFP:
-        return AArch64::UCVTFUXDri;
-      case TargetOpcode::G_FPTOSI:
-        return AArch64::FCVTZSUXDr;
-      case TargetOpcode::G_FPTOUI:
-        return AArch64::FCVTZUUXDr;
-      default:
-        return GenericOpc;
-      }
-    default:
-      return GenericOpc;
-    }
-  default:
-    return GenericOpc;
-  };
-  return GenericOpc;
-}
-
 MachineInstr *
 AArch64InstructionSelector::emitSelect(Register Dst, Register True,
                                        Register False, AArch64CC::CondCode CC,
@@ -1349,7 +1273,9 @@ AArch64InstructionSelector::emitSelect(Register Dst, Register True,
   return &*SelectInst;
 }
 
-static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
+static AArch64CC::CondCode
+changeICMPPredToAArch64CC(CmpInst::Predicate P, Register RHS = {},
+                          MachineRegisterInfo *MRI = nullptr) {
   switch (P) {
   default:
     llvm_unreachable("Unknown condition code!");
@@ -1360,8 +1286,18 @@ static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) {
   case CmpInst::ICMP_SGT:
     return AArch64CC::GT;
   case CmpInst::ICMP_SGE:
+    if (RHS && MRI) {
+      auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS, *MRI);
+      if (ValAndVReg && ValAndVReg->Value == 0)
+        return AArch64CC::PL;
+    }
     return AArch64CC::GE;
   case CmpInst::ICMP_SLT:
+    if (RHS && MRI) {
+      auto ValAndVReg = getIConstantVRegValWithLookThrough(RHS, *MRI);
+      if (ValAndVReg && ValAndVReg->Value == 0)
+        return AArch64CC::MI;
+    }
     return AArch64CC::LT;
   case CmpInst::ICMP_SLE:
     return AArch64CC::LE;
@@ -1697,7 +1633,7 @@ bool AArch64InstructionSelector::selectCompareBranchFedByFCmp(
   emitFPCompare(FCmp.getOperand(2).getReg(), FCmp.getOperand(3).getReg(), MIB,
                 Pred);
   AArch64CC::CondCode CC1, CC2;
-  changeFCMPPredToAArch64CC(static_cast<CmpInst::Predicate>(Pred), CC1, CC2);
+  changeFCMPPredToAArch64CC(Pred, CC1, CC2);
   MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
   MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC1).addMBB(DestMBB);
   if (CC2 != AArch64CC::AL)
@@ -1813,7 +1749,8 @@ bool AArch64InstructionSelector::selectCompareBranchFedByICmp(
   auto &PredOp = ICmp.getOperand(1);
   emitIntegerCompare(ICmp.getOperand(2), ICmp.getOperand(3), PredOp, MIB);
   const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
-      static_cast<CmpInst::Predicate>(PredOp.getPredicate()));
+      static_cast<CmpInst::Predicate>(PredOp.getPredicate()),
+      ICmp.getOperand(3).getReg(), MIB.getMRI());
   MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
   I.eraseFromParent();
   return true;
@@ -2510,8 +2447,8 @@ bool AArch64InstructionSelector::earlySelect(MachineInstr &I) {
     emitIntegerCompare(/*LHS=*/Cmp->getOperand(2),
                        /*RHS=*/Cmp->getOperand(3), PredOp, MIB);
     auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
-    const AArch64CC::CondCode InvCC =
-        changeICMPPredToAArch64CC(CmpInst::getInversePredicate(Pred));
+    const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
+        CmpInst::getInversePredicate(Pred), Cmp->getOperand(3).getReg(), &MRI);
     emitCSINC(/*Dst=*/AddDst, /*Src =*/AddLHS, /*Src2=*/AddLHS, InvCC, MIB);
     I.eraseFromParent();
     return true;
@@ -3511,23 +3448,6 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
     return true;
   }
 
-  case TargetOpcode::G_SITOFP:
-  case TargetOpcode::G_UITOFP:
-  case TargetOpcode::G_FPTOSI:
-  case TargetOpcode::G_FPTOUI: {
-    const LLT DstTy = MRI.getType(I.getOperand(0).getReg()),
-              SrcTy = MRI.getType(I.getOperand(1).getReg());
-    const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy);
-    if (NewOpc == Opcode)
-      return false;
-
-    I.setDesc(TII.get(NewOpc));
-    constrainSelectedInstRegOperands(I, TII, TRI, RBI);
-    I.setFlags(MachineInstr::NoFPExcept);
-
-    return true;
-  }
-
   case TargetOpcode::G_FREEZE:
     return selectCopy(I, TII, MRI, TRI, RBI);
 
@@ -3577,8 +3497,8 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
     auto &PredOp = I.getOperand(1);
     emitIntegerCompare(I.getOperand(2), I.getOperand(3), PredOp, MIB);
     auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
-    const AArch64CC::CondCode InvCC =
-        changeICMPPredToAArch64CC(CmpInst::getInversePredicate(Pred));
+    const AArch64CC::CondCode InvCC = changeICMPPredToAArch64CC(
+        CmpInst::getInversePredicate(Pred), I.getOperand(3).getReg(), &MRI);
     emitCSINC(/*Dst=*/I.getOperand(0).getReg(), /*Src1=*/AArch64::WZR,
               /*Src2=*/AArch64::WZR, InvCC, MIB);
     I.eraseFromParent();
@@ -4931,7 +4851,7 @@ MachineInstr *AArch64InstructionSelector::emitConjunctionRec(
     if (Negate)
       CC = CmpInst::getInversePredicate(CC);
     if (isa<GICmp>(Cmp)) {
-      OutCC = changeICMPPredToAArch64CC(CC);
+      OutCC = changeICMPPredToAArch64CC(CC, RHS, MIB.getMRI());
     } else {
       // Handle special FP cases.
       AArch64CC::CondCode ExtraCC;
@@ -5101,7 +5021,8 @@ bool AArch64InstructionSelector::tryOptSelect(GSelect &I) {
     emitIntegerCompare(CondDef->getOperand(2), CondDef->getOperand(3), PredOp,
                        MIB);
     auto Pred = static_cast<CmpInst::Predicate>(PredOp.getPredicate());
-    CondCode = changeICMPPredToAArch64CC(Pred);
+    CondCode =
+        changeICMPPredToAArch64CC(Pred, CondDef->getOperand(3).getReg(), &MRI);
   } else {
     // Get the condition code for the select.
     auto Pred =
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index e0e1af78770de..210643f6f2f49 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -797,6 +797,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .clampMinNumElements(0, s16, 4)
       .alwaysLegal();
 
+  getActionDefinitionsBuilder({G_TRUNC_SSAT_S, G_TRUNC_SSAT_U, G_TRUNC_USAT_U})
+      .legalFor({{v8s8, v8s16}, {v4s16, v4s32}, {v2s32, v2s64}});
+
   getActionDefinitionsBuilder(G_SEXT_INREG)
       .legalFor({s32, s64})
       .legalFor(PackedVectorAllTypeList)
@@ -965,6 +968,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
                    {s128, s64}});
 
   // Control-flow
+  getActionDefinitionsBuilder(G_BR).alwaysLegal();
   getActionDefinitionsBuilder(G_BRCOND)
     .legalFor({s32})
     .clampScalar(0, s32, s32);
@@ -1255,6 +1259,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
   getActionDefinitionsBuilder(G_BRJT).legalFor({{p0, s64}});
 
+  getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
+
   getActionDefinitionsBuilder(G_DYN_STACKALLOC).custom();
 
   getActionDefinitionsBuilder({G_STACKSAVE, G_STACKRESTORE}).lower();
@@ -1644,6 +1650,11 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
   MachineIRBuilder &MIB = Helper.MIRBuilder;
   MachineRegisterInfo &MRI = *MIB.getMRI();
 
+  auto LowerUnaryOp = [&MI, &MIB](unsigned Opcode) {
+    MIB.buildInstr(Opcode, {MI.getOperand(0)}, {MI.getOperand(2)});
+    MI.eraseFromParent();
+    return true;
+  };
   auto LowerBinOp = [&MI, &MIB](unsigned Opcode) {
     MIB.buildInstr(Opcode, {MI.getOperand(0)},
                    {MI.getOperand(2), MI.getOperand(3)});
@@ -1838,6 +1849,12 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
     return LowerTriOp(AArch64::G_UDOT);
   case Intrinsic::aarch64_neon_sdot:
     return LowerTriOp(AArch64::G_SDOT);
+  case Intrinsic::aarch64_neon_sqxtn:
+    return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_S);
+  case Intrinsic::aarch64_neon_sqxtun:
+    return LowerUnaryOp(TargetOpcode::G_TRUNC_SSAT_U);
+  case Intrinsic::aarch64_neon_uqxtn:
+    return LowerUnaryOp(TargetOpcode::G_TRUNC_USAT_U);
 
   case Intrinsic::vector_reverse:
     // TODO: Add support for vector_reverse
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
index 3ba08c8c1d988..6025f1c9f5f4e 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp
@@ -614,8 +614,7 @@ tryAdjustICmpImmAndPred(Register RHS, CmpInst::Predicate P,
     // x uge c => x ugt c - 1
     //
     // When c is not zero.
-    if (C == 0)
-      return std::nullopt;
+    assert(C != 0 && "C should not be zero here!");
     P = (P == CmpInst::ICMP_ULT) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;
     C -= 1;
     break;
@@ -656,14 +655,13 @@ tryAdjustICmpImmAndPred(Register RHS, CmpInst::Predicate P,
   if (isLegalArithImmed(C))
     return {{C, P}};
 
-  auto IsMaterializableInSingleInstruction = [=](uint64_t Imm) {
+  auto NumberOfInstrToLoadImm = [=](uint64_t Imm) {
     SmallVector<AArch64_IMM::ImmInsnModel> Insn;
     AArch64_IMM::expandMOVImm(Imm, 32, Insn);
-    return Insn.size() == 1;
+    return Insn.size();
   };
 
-  if (!IsMaterializableInSingleInstruction(OriginalC) &&
-      IsMaterializableInSingleInstruction(C))
+  if (NumberOfInstrToLoadImm(OriginalC) > NumberOfInstrToLoadImm(C))
     return {{C, P}};
 
   return std::nullopt;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
index 31954e7954c03..1b919abd222e4 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
@@ -13,6 +13,7 @@
 
 #include "AArch64RegisterBankInfo.h"
 #include "AArch64RegisterInfo.h"
+#include "AArch64Subtarget.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
@@ -492,7 +493,7 @@ static bool isFPIntrinsic(const MachineRegisterInfo &MRI,
 
 bool AArch64RegisterBankInfo::isPHIWithFPConstraints(
     const MachineInstr &MI, const MachineRegisterInfo &MRI,
-    const TargetRegisterInfo &TRI, const unsigned Depth) const {
+    const AArch64RegisterInfo &TRI, const unsigned Depth) const {
   if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
     return false;
 
@@ -506,7 +507,7 @@ bool AArch64RegisterBankInfo::isPHIWithFPConstraints(
 
 bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,
                                                const MachineRegisterInfo &MRI,
-                                               const TargetRegisterInfo &TRI,
+                                               const AArch64RegisterInfo &TRI,
                                                unsigned Depth) const {
   unsigned Op = MI.getOpcode();
   if (Op == TargetOpcode::G_INTRINSIC && isFPIntrinsic(MRI, MI))
@@ -544,7 +545,7 @@ bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,
 
 bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
                                          const MachineRegisterInfo &MRI,
-                                         const TargetRegisterInfo &TRI,
+                                         const AArch64RegisterInfo &TRI,
                                          unsigned Depth) const {
   switch (MI.getOpcode()) {
   case TargetOpcode::G_FPTOSI:
@@ -582,7 +583,7 @@ bool AArch64RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
 
 bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
                                             const MachineRegisterInfo &MRI,
-                                            const TargetRegisterInfo &TRI,
+                                            const AArch64RegisterInfo &TRI,
                                             unsigned Depth) const {
   switch (MI.getOpcode()) {
   case AArch64::G_DUP:
@@ -618,6 +619,19 @@ bool AArch64RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
   return hasFPConstraints(MI, MRI, TRI, Depth);
 }
 
+bool AArch64RegisterBankInfo::prefersFPUse(const MachineInstr &MI,
+                                           const MachineRegisterInfo &MRI,
+                                           const AArch64RegisterInfo &TRI,
+                                           unsigned Depth) const {
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_SITOFP:
+  case TargetOpcode::G_UITOFP:
+    return MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() ==
+           MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
+  }
+  return onlyDefinesFP(MI, MRI, TRI, Depth);
+}
+
 bool AArch64RegisterBankInfo::isLoadFromFPType(const MachineInstr &MI) const {
   // GMemOperation because we also want to match indexed loads.
   auto *MemOp = cast<GMemOperation>(&MI);
@@ -671,8 +685,8 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
 
   const MachineFunction &MF = *MI.getParent()->getParent();
   const MachineRegisterInfo &MRI = MF.getRegInfo();
-  const TargetSubtargetInfo &STI = MF.getSubtarget();
-  const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
+  const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
+  const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
 
   switch (Opc) {
     // G_{F|S|U}REM are not listed because they are not legal.
@@ -826,7 +840,9 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     // Integer to FP conversions don't necessarily happen between GPR -> FPR
     // regbanks. They can also be done within an FPR register.
     Register SrcReg = MI.getOperand(1).getReg();
-    if (getRegBank(SrcReg, MRI, TRI) == &AArch64::FPRRegBank)
+    if (getRegBank(SrcReg, MRI, TRI) == &AArch64::FPRRegBank &&
+        MRI.getType(SrcReg).getSizeInBits() ==
+            MRI.getType(MI.getOperand(0).getReg()).getSizeInBits())
       OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
     else
       OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
@@ -895,13 +911,13 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
                  // instruction.
                  //
                  // Int->FP conversion operations are also captured in
-                 // onlyDefinesFP().
+                 // prefersFPUse().
 
                  if (isPHIWithFPConstraints(UseMI, MRI, TRI))
                    return true;
 
                  return onlyUsesFP(UseMI, MRI, TRI) ||
-                        onlyDefinesFP(UseMI, MRI, TRI);
+                        prefersFPUse(UseMI, MRI, TRI);
                }))
       OpRegBankIdx[0] = PMI_FirstFPR;
     break;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.h
index 3abbc1b68b5be..b2de031772108 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.h
+++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.h
@@ -22,6 +22,7 @@
 namespace llvm {
 
 class TargetRegisterInfo;
+class AArch64RegisterInfo;
 
 class AArch64GenRegisterBankInfo : public RegisterBankInfo {
 protected:
@@ -123,21 +124,26 @@ class AArch64RegisterBankInfo final : public AArch64GenRegisterBankInfo {
   /// \returns true if \p MI is a PHI that its def is used by
   /// any instruction that onlyUsesFP.
   bool isPHIWithFPConstraints(const MachineInstr &MI,
-                             const MachineRegisterInfo &MRI,
-                             const TargetRegisterInfo &TRI,
-                             unsigned Depth = 0) const;
+                              const MachineRegisterInfo &MRI,
+                              const AArch64RegisterInfo &TRI,
+                              unsigned Depth = 0) const;
 
   /// \returns true if \p MI only uses and defines FPRs.
   bool hasFPConstraints(const MachineInstr &MI, const MachineRegisterInfo &MRI,
-                     const TargetRegisterInfo &TRI, unsigned Depth = 0) const;
+                        const AArch64RegisterInfo &TRI,
+                        unsigned Depth = 0) const;
 
   /// \returns true if \p MI only uses FPRs.
   bool onlyUsesFP(const MachineInstr &MI, const MachineRegisterInfo &MRI,
-                  const TargetRegisterInfo &TRI, unsigned Depth = 0) const;
+                  const AArch64RegisterInfo &TRI, unsigned Depth = 0) const;
 
   /// \returns true if \p MI only defines FPRs.
   bool onlyDefinesFP(const MachineInstr &MI, const MachineRegisterInfo &MRI,
-                     const TargetRegisterInfo &TRI, unsigned Depth = 0) const;
+                     const AArch64RegisterInfo &TRI, unsigned Depth = 0) const;
+
+  /// \returns true if \p MI can take both fpr and gpr uses, but prefers fp.
+  bool prefersFPUse(const MachineInstr &MI, const MachineRegisterInfo &MRI,
+                    const AArch64RegisterInfo &TRI, unsigned Depth = 0) const;
 
   /// \returns true if the load \p MI is likely loading from a floating-point
   /// type.
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
index 45ac023236b09..a388216a95098 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
@@ -40,6 +40,7 @@ class AArch64ELFObjectWriter : public MCELFObjectTargetWriter {
                         bool IsPCRel) const override;
   bool needsRelocateWithSymbol(const MCValue &, unsigned Type) const override;
   bool isNonILP32reloc(const MCFixup &Fixup, AArch64::Specifier RefKind) const;
+  void sortRelocs(std::vector<ELFRelocationEntry> &Relocs) override;
 
   bool IsILP32;
 };
@@ -498,6 +499,17 @@ bool AArch64ELFObjectWriter::needsRelocateWithSymbol(const MCValue &Val,
                       Val.getSpecifier());
 }
 
+void AArch64ELFObjectWriter::sortRelocs(
+    std::vector<ELFRelocationEntry> &Relocs) {
+  // PATCHINST relocations should be applied last because they may overwrite the
+  // whole instruction and so should take precedence over other relocations that
+  // modify operands of the original instruction.
+  std::stable_partition(Relocs.begin(), Relocs.end(),
+                        [](const ELFRelocationEntry &R) {
+                          return R.Type != ELF::R_AARCH64_PATCHINST;
+                        });
+}
+
 std::unique_ptr<MCObjectTargetWriter>
 llvm::createAArch64ELFObjectWriter(uint8_t OSABI, bool IsILP32) {
   return std::make_unique<AArch64ELFObjectWriter>(OSABI, IsILP32);
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index 14547e3c4580d..917dbdfbbc9e3 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -35,7 +35,6 @@
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/MCWinCOFFStreamer.h"
 #include "llvm/Support/AArch64BuildAttributes.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/Support/raw_ostream.h"
 
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
index 3c8b5712c1f0c..54b58e948daf2 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp
@@ -1017,14 +1017,22 @@ bool AArch64InstPrinter::printSysAlias(const MCInst *MI,
   else
     return false;
 
+  StringRef Reg = getRegisterName(MI->getOperand(4).getReg());
+  bool NotXZR = Reg != "xzr";
+
+  // If a mandatory is not specified in the TableGen
+  // (i.e. no register operand should be present), and the register value
+  // is not xzr/x31, then disassemble to a SYS alias instead.
+  if (NotXZR && !NeedsReg)
+    return false;
+
   std::string Str = Ins + Name;
   llvm::transform(Str, Str.begin(), ::tolower);
 
   O << '\t' << Str;
-  if (NeedsReg) {
-    O << ", ";
-    printRegName(O, MI->getOperand(4).getReg());
-  }
+
+  if (NeedsReg)
+    O << ", " << Reg;
 
   return true;
 }
diff --git a/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp b/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp
new file mode 100644
index 0000000000000..b58dfdf32e4ab
--- /dev/null
+++ b/llvm/lib/Target/AArch64/MachineSMEABIPass.cpp
@@ -0,0 +1,696 @@
+//===- MachineSMEABIPass.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass implements the SME ABI requirements for ZA state. This includes
+// implementing the lazy ZA state save schemes around calls.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass works by collecting instructions that require ZA to be in a
+// specific state (e.g., "ACTIVE" or "SAVED") and inserting the necessary state
+// transitions to ensure ZA is in the required state before instructions. State
+// transitions represent actions such as setting up or restoring a lazy save.
+// Certain points within a function may also have predefined states independent
+// of any instructions, for example, a "shared_za" function is always entered
+// and exited in the "ACTIVE" state.
+//
+// To handle ZA state across control flow, we make use of edge bundling. This
+// assigns each block an "incoming" and "outgoing" edge bundle (representing
+// incoming and outgoing edges). Initially, these are unique to each block;
+// then, in the process of forming bundles, the outgoing block of a block is
+// joined with the incoming bundle of all successors. The result is that each
+// bundle can be assigned a single ZA state, which ensures the state required by
+// all a blocks' successors is the same, and that each basic block will always
+// be entered with the same ZA state. This eliminates the need for splitting
+// edges to insert state transitions or "phi" nodes for ZA states.
+//
+// See below for a simple example of edge bundling.
+//
+// The following shows a conditionally executed basic block (BB1):
+//
+// if (cond)
+//   BB1
+// BB2
+//
+// Initial Bundles         Joined Bundles
+//
+//   ┌──0──┐                ┌──0──┐
+//   │ BB0 │                │ BB0 │
+//   └──1──┘                └──1──┘
+//      ├───────┐              ├───────┐
+//      ▼       │              ▼       │
+//   ┌──2──┐    │   ─────►  ┌──1──┐    │
+//   │ BB1 │    ▼           │ BB1 │    ▼
+//   └──3──┘ ┌──4──┐        └──1──┘ ┌──1──┐
+//      └───►4 BB2 │           └───►1 BB2 │
+//           └──5──┘                └──2──┘
+//
+// On the left are the initial per-block bundles, and on the right are the
+// joined bundles (which are the result of the EdgeBundles analysis).
+
+#include "AArch64InstrInfo.h"
+#include "AArch64MachineFunctionInfo.h"
+#include "AArch64Subtarget.h"
+#include "MCTargetDesc/AArch64AddressingModes.h"
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/EdgeBundles.h"
+#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "aarch64-machine-sme-abi"
+
+namespace {
+
+enum ZAState {
+  // Any/unknown state (not valid)
+  ANY = 0,
+
+  // ZA is in use and active (i.e. within the accumulator)
+  ACTIVE,
+
+  // A ZA save has been set up or committed (i.e. ZA is dormant or off)
+  LOCAL_SAVED,
+
+  // ZA is off or a lazy save has been set up by the caller
+  CALLER_DORMANT,
+
+  // ZA is off
+  OFF,
+
+  // The number of ZA states (not a valid state)
+  NUM_ZA_STATE
+};
+
+/// A bitmask enum to record live physical registers that the "emit*" routines
+/// may need to preserve. Note: This only tracks registers we may clobber.
+enum LiveRegs : uint8_t {
+  None = 0,
+  NZCV = 1 << 0,
+  W0 = 1 << 1,
+  W0_HI = 1 << 2,
+  X0 = W0 | W0_HI,
+  LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ W0_HI)
+};
+
+/// Holds the virtual registers live physical registers have been saved to.
+struct PhysRegSave {
+  LiveRegs PhysLiveRegs;
+  Register StatusFlags = AArch64::NoRegister;
+  Register X0Save = AArch64::NoRegister;
+};
+
+static bool isLegalEdgeBundleZAState(ZAState State) {
+  switch (State) {
+  case ZAState::ACTIVE:
+  case ZAState::LOCAL_SAVED:
+    return true;
+  default:
+    return false;
+  }
+}
+struct TPIDR2State {
+  int FrameIndex = -1;
+};
+
+StringRef getZAStateString(ZAState State) {
+#define MAKE_CASE(V)                                                           \
+  case V:                                                                      \
+    return #V;
+  switch (State) {
+    MAKE_CASE(ZAState::ANY)
+    MAKE_CASE(ZAState::ACTIVE)
+    MAKE_CASE(ZAState::LOCAL_SAVED)
+    MAKE_CASE(ZAState::CALLER_DORMANT)
+    MAKE_CASE(ZAState::OFF)
+  default:
+    llvm_unreachable("Unexpected ZAState");
+  }
+#undef MAKE_CASE
+}
+
+static bool isZAorZT0RegOp(const TargetRegisterInfo &TRI,
+                           const MachineOperand &MO) {
+  if (!MO.isReg() || !MO.getReg().isPhysical())
+    return false;
+  return any_of(TRI.subregs_inclusive(MO.getReg()), [](const MCPhysReg &SR) {
+    return AArch64::MPR128RegClass.contains(SR) ||
+           AArch64::ZTRRegClass.contains(SR);
+  });
+}
+
+/// Returns the required ZA state needed before \p MI and an iterator pointing
+/// to where any code required to change the ZA state should be inserted.
+static std::pair<ZAState, MachineBasicBlock::iterator>
+getZAStateBeforeInst(const TargetRegisterInfo &TRI, MachineInstr &MI,
+                     bool ZAOffAtReturn) {
+  MachineBasicBlock::iterator InsertPt(MI);
+
+  if (MI.getOpcode() == AArch64::InOutZAUsePseudo)
+    return {ZAState::ACTIVE, std::prev(InsertPt)};
+
+  if (MI.getOpcode() == AArch64::RequiresZASavePseudo)
+    return {ZAState::LOCAL_SAVED, std::prev(InsertPt)};
+
+  if (MI.isReturn())
+    return {ZAOffAtReturn ? ZAState::OFF : ZAState::ACTIVE, InsertPt};
+
+  for (auto &MO : MI.operands()) {
+    if (isZAorZT0RegOp(TRI, MO))
+      return {ZAState::ACTIVE, InsertPt};
+  }
+
+  return {ZAState::ANY, InsertPt};
+}
+
+struct MachineSMEABI : public MachineFunctionPass {
+  inline static char ID = 0;
+
+  MachineSMEABI() : MachineFunctionPass(ID) {}
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override { return "Machine SME ABI pass"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<EdgeBundlesWrapperLegacy>();
+    AU.addPreservedID(MachineLoopInfoID);
+    AU.addPreservedID(MachineDominatorsID);
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  /// Collects the needed ZA state (and live registers) before each instruction
+  /// within the machine function.
+  void collectNeededZAStates(SMEAttrs);
+
+  /// Assigns each edge bundle a ZA state based on the needed states of blocks
+  /// that have incoming or outgoing edges in that bundle.
+  void assignBundleZAStates();
+
+  /// Inserts code to handle changes between ZA states within the function.
+  /// E.g., ACTIVE -> LOCAL_SAVED will insert code required to save ZA.
+  void insertStateChanges();
+
+  // Emission routines for private and shared ZA functions (using lazy saves).
+  void emitNewZAPrologue(MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator MBBI);
+  void emitRestoreLazySave(MachineBasicBlock &MBB,
+                           MachineBasicBlock::iterator MBBI,
+                           LiveRegs PhysLiveRegs);
+  void emitSetupLazySave(MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator MBBI);
+  void emitAllocateLazySaveBuffer(MachineBasicBlock &MBB,
+                                  MachineBasicBlock::iterator MBBI);
+  void emitZAOff(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                 bool ClearTPIDR2);
+
+  void emitStateChange(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                       ZAState From, ZAState To, LiveRegs PhysLiveRegs);
+
+  /// Save live physical registers to virtual registers.
+  PhysRegSave createPhysRegSave(LiveRegs PhysLiveRegs, MachineBasicBlock &MBB,
+                                MachineBasicBlock::iterator MBBI, DebugLoc DL);
+  /// Restore physical registers from a save of their previous values.
+  void restorePhyRegSave(PhysRegSave const &RegSave, MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator MBBI, DebugLoc DL);
+
+  /// Get or create a TPIDR2 block in this function.
+  TPIDR2State getTPIDR2Block();
+
+private:
+  /// Contains the needed ZA state (and live registers) at an instruction.
+  struct InstInfo {
+    ZAState NeededState{ZAState::ANY};
+    MachineBasicBlock::iterator InsertPt;
+    LiveRegs PhysLiveRegs = LiveRegs::None;
+  };
+
+  /// Contains the needed ZA state for each instruction in a block.
+  /// Instructions that do not require a ZA state are not recorded.
+  struct BlockInfo {
+    ZAState FixedEntryState{ZAState::ANY};
+    SmallVector<InstInfo> Insts;
+    LiveRegs PhysLiveRegsAtExit = LiveRegs::None;
+  };
+
+  // All pass state that must be cleared between functions.
+  struct PassState {
+    SmallVector<BlockInfo> Blocks;
+    SmallVector<ZAState> BundleStates;
+    std::optional<TPIDR2State> TPIDR2Block;
+  } State;
+
+  MachineFunction *MF = nullptr;
+  EdgeBundles *Bundles = nullptr;
+  const AArch64Subtarget *Subtarget = nullptr;
+  const AArch64RegisterInfo *TRI = nullptr;
+  const TargetInstrInfo *TII = nullptr;
+  MachineRegisterInfo *MRI = nullptr;
+};
+
+void MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) {
+  assert((SMEFnAttrs.hasZT0State() || SMEFnAttrs.hasZAState()) &&
+         "Expected function to have ZA/ZT0 state!");
+
+  State.Blocks.resize(MF->getNumBlockIDs());
+  for (MachineBasicBlock &MBB : *MF) {
+    BlockInfo &Block = State.Blocks[MBB.getNumber()];
+    if (&MBB == &MF->front()) {
+      // Entry block:
+      Block.FixedEntryState = SMEFnAttrs.hasPrivateZAInterface()
+                                  ? ZAState::CALLER_DORMANT
+                                  : ZAState::ACTIVE;
+    } else if (MBB.isEHPad()) {
+      // EH entry block:
+      Block.FixedEntryState = ZAState::LOCAL_SAVED;
+    }
+
+    LiveRegUnits LiveUnits(*TRI);
+    LiveUnits.addLiveOuts(MBB);
+
+    auto GetPhysLiveRegs = [&] {
+      LiveRegs PhysLiveRegs = LiveRegs::None;
+      if (!LiveUnits.available(AArch64::NZCV))
+        PhysLiveRegs |= LiveRegs::NZCV;
+      // We have to track W0 and X0 separately as otherwise things can get
+      // confused if we attempt to preserve X0 but only W0 was defined.
+      if (!LiveUnits.available(AArch64::W0))
+        PhysLiveRegs |= LiveRegs::W0;
+      if (!LiveUnits.available(AArch64::W0_HI))
+        PhysLiveRegs |= LiveRegs::W0_HI;
+      return PhysLiveRegs;
+    };
+
+    Block.PhysLiveRegsAtExit = GetPhysLiveRegs();
+    auto FirstTerminatorInsertPt = MBB.getFirstTerminator();
+    for (MachineInstr &MI : reverse(MBB)) {
+      MachineBasicBlock::iterator MBBI(MI);
+      LiveUnits.stepBackward(MI);
+      LiveRegs PhysLiveRegs = GetPhysLiveRegs();
+      auto [NeededState, InsertPt] = getZAStateBeforeInst(
+          *TRI, MI, /*ZAOffAtReturn=*/SMEFnAttrs.hasPrivateZAInterface());
+      assert((InsertPt == MBBI ||
+              InsertPt->getOpcode() == AArch64::ADJCALLSTACKDOWN) &&
+             "Unexpected state change insertion point!");
+      // TODO: Do something to avoid state changes where NZCV is live.
+      if (MBBI == FirstTerminatorInsertPt)
+        Block.PhysLiveRegsAtExit = PhysLiveRegs;
+      if (NeededState != ZAState::ANY)
+        Block.Insts.push_back({NeededState, InsertPt, PhysLiveRegs});
+    }
+
+    // Reverse vector (as we had to iterate backwards for liveness).
+    std::reverse(Block.Insts.begin(), Block.Insts.end());
+  }
+}
+
+void MachineSMEABI::assignBundleZAStates() {
+  State.BundleStates.resize(Bundles->getNumBundles());
+  for (unsigned I = 0, E = Bundles->getNumBundles(); I != E; ++I) {
+    LLVM_DEBUG(dbgs() << "Assigning ZA state for edge bundle: " << I << '\n');
+
+    // Attempt to assign a ZA state for this bundle that minimizes state
+    // transitions. Edges within loops are given a higher weight as we assume
+    // they will be executed more than once.
+    // TODO: We should propagate desired incoming/outgoing states through blocks
+    // that have the "ANY" state first to make better global decisions.
+    int EdgeStateCounts[ZAState::NUM_ZA_STATE] = {0};
+    for (unsigned BlockID : Bundles->getBlocks(I)) {
+      LLVM_DEBUG(dbgs() << "- bb." << BlockID);
+
+      const BlockInfo &Block = State.Blocks[BlockID];
+      if (Block.Insts.empty()) {
+        LLVM_DEBUG(dbgs() << " (no state preference)\n");
+        continue;
+      }
+      bool InEdge = Bundles->getBundle(BlockID, /*Out=*/false) == I;
+      bool OutEdge = Bundles->getBundle(BlockID, /*Out=*/true) == I;
+
+      ZAState DesiredIncomingState = Block.Insts.front().NeededState;
+      if (InEdge && isLegalEdgeBundleZAState(DesiredIncomingState)) {
+        EdgeStateCounts[DesiredIncomingState]++;
+        LLVM_DEBUG(dbgs() << " DesiredIncomingState: "
+                          << getZAStateString(DesiredIncomingState));
+      }
+      ZAState DesiredOutgoingState = Block.Insts.back().NeededState;
+      if (OutEdge && isLegalEdgeBundleZAState(DesiredOutgoingState)) {
+        EdgeStateCounts[DesiredOutgoingState]++;
+        LLVM_DEBUG(dbgs() << " DesiredOutgoingState: "
+                          << getZAStateString(DesiredOutgoingState));
+      }
+      LLVM_DEBUG(dbgs() << '\n');
+    }
+
+    ZAState BundleState =
+        ZAState(max_element(EdgeStateCounts) - EdgeStateCounts);
+
+    // Force ZA to be active in bundles that don't have a preferred state.
+    // TODO: Something better here (to avoid extra mode switches).
+    if (BundleState == ZAState::ANY)
+      BundleState = ZAState::ACTIVE;
+
+    LLVM_DEBUG({
+      dbgs() << "Chosen ZA state: " << getZAStateString(BundleState) << '\n'
+             << "Edge counts:";
+      for (auto [State, Count] : enumerate(EdgeStateCounts))
+        dbgs() << " " << getZAStateString(ZAState(State)) << ": " << Count;
+      dbgs() << "\n\n";
+    });
+
+    State.BundleStates[I] = BundleState;
+  }
+}
+
+void MachineSMEABI::insertStateChanges() {
+  for (MachineBasicBlock &MBB : *MF) {
+    const BlockInfo &Block = State.Blocks[MBB.getNumber()];
+    ZAState InState = State.BundleStates[Bundles->getBundle(MBB.getNumber(),
+                                                            /*Out=*/false)];
+
+    ZAState CurrentState = Block.FixedEntryState;
+    if (CurrentState == ZAState::ANY)
+      CurrentState = InState;
+
+    for (auto &Inst : Block.Insts) {
+      if (CurrentState != Inst.NeededState)
+        emitStateChange(MBB, Inst.InsertPt, CurrentState, Inst.NeededState,
+                        Inst.PhysLiveRegs);
+      CurrentState = Inst.NeededState;
+    }
+
+    if (MBB.succ_empty())
+      continue;
+
+    ZAState OutState =
+        State.BundleStates[Bundles->getBundle(MBB.getNumber(), /*Out=*/true)];
+    if (CurrentState != OutState)
+      emitStateChange(MBB, MBB.getFirstTerminator(), CurrentState, OutState,
+                      Block.PhysLiveRegsAtExit);
+  }
+}
+
+TPIDR2State MachineSMEABI::getTPIDR2Block() {
+  if (State.TPIDR2Block)
+    return *State.TPIDR2Block;
+  MachineFrameInfo &MFI = MF->getFrameInfo();
+  State.TPIDR2Block = TPIDR2State{MFI.CreateStackObject(16, Align(16), false)};
+  return *State.TPIDR2Block;
+}
+
+static DebugLoc getDebugLoc(MachineBasicBlock &MBB,
+                            MachineBasicBlock::iterator MBBI) {
+  if (MBBI != MBB.end())
+    return MBBI->getDebugLoc();
+  return DebugLoc();
+}
+
+void MachineSMEABI::emitSetupLazySave(MachineBasicBlock &MBB,
+                                      MachineBasicBlock::iterator MBBI) {
+  DebugLoc DL = getDebugLoc(MBB, MBBI);
+
+  // Get pointer to TPIDR2 block.
+  Register TPIDR2 = MRI->createVirtualRegister(&AArch64::GPR64spRegClass);
+  Register TPIDR2Ptr = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), TPIDR2)
+      .addFrameIndex(getTPIDR2Block().FrameIndex)
+      .addImm(0)
+      .addImm(0);
+  BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), TPIDR2Ptr)
+      .addReg(TPIDR2);
+  // Set TPIDR2_EL0 to point to TPIDR2 block.
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSR))
+      .addImm(AArch64SysReg::TPIDR2_EL0)
+      .addReg(TPIDR2Ptr);
+}
+
+PhysRegSave MachineSMEABI::createPhysRegSave(LiveRegs PhysLiveRegs,
+                                             MachineBasicBlock &MBB,
+                                             MachineBasicBlock::iterator MBBI,
+                                             DebugLoc DL) {
+  PhysRegSave RegSave{PhysLiveRegs};
+  if (PhysLiveRegs & LiveRegs::NZCV) {
+    RegSave.StatusFlags = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::MRS), RegSave.StatusFlags)
+        .addImm(AArch64SysReg::NZCV)
+        .addReg(AArch64::NZCV, RegState::Implicit);
+  }
+  // Note: Preserving X0 is "free" as this is before register allocation, so
+  // the register allocator is still able to optimize these copies.
+  if (PhysLiveRegs & LiveRegs::W0) {
+    RegSave.X0Save = MRI->createVirtualRegister(PhysLiveRegs & LiveRegs::W0_HI
+                                                    ? &AArch64::GPR64RegClass
+                                                    : &AArch64::GPR32RegClass);
+    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), RegSave.X0Save)
+        .addReg(PhysLiveRegs & LiveRegs::W0_HI ? AArch64::X0 : AArch64::W0);
+  }
+  return RegSave;
+}
+
+void MachineSMEABI::restorePhyRegSave(PhysRegSave const &RegSave,
+                                      MachineBasicBlock &MBB,
+                                      MachineBasicBlock::iterator MBBI,
+                                      DebugLoc DL) {
+  if (RegSave.StatusFlags != AArch64::NoRegister)
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSR))
+        .addImm(AArch64SysReg::NZCV)
+        .addReg(RegSave.StatusFlags)
+        .addReg(AArch64::NZCV, RegState::ImplicitDefine);
+
+  if (RegSave.X0Save != AArch64::NoRegister)
+    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY),
+            RegSave.PhysLiveRegs & LiveRegs::W0_HI ? AArch64::X0 : AArch64::W0)
+        .addReg(RegSave.X0Save);
+}
+
+void MachineSMEABI::emitRestoreLazySave(MachineBasicBlock &MBB,
+                                        MachineBasicBlock::iterator MBBI,
+                                        LiveRegs PhysLiveRegs) {
+  auto *TLI = Subtarget->getTargetLowering();
+  DebugLoc DL = getDebugLoc(MBB, MBBI);
+  Register TPIDR2EL0 = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
+  Register TPIDR2 = AArch64::X0;
+
+  // TODO: Emit these within the restore MBB to prevent unnecessary saves.
+  PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);
+
+  // Enable ZA.
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSRpstatesvcrImm1))
+      .addImm(AArch64SVCR::SVCRZA)
+      .addImm(1);
+  // Get current TPIDR2_EL0.
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MRS), TPIDR2EL0)
+      .addImm(AArch64SysReg::TPIDR2_EL0);
+  // Get pointer to TPIDR2 block.
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), TPIDR2)
+      .addFrameIndex(getTPIDR2Block().FrameIndex)
+      .addImm(0)
+      .addImm(0);
+  // (Conditionally) restore ZA state.
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::RestoreZAPseudo))
+      .addReg(TPIDR2EL0)
+      .addReg(TPIDR2)
+      .addExternalSymbol(TLI->getLibcallName(RTLIB::SMEABI_TPIDR2_RESTORE))
+      .addRegMask(TRI->SMEABISupportRoutinesCallPreservedMaskFromX0());
+  // Zero TPIDR2_EL0.
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSR))
+      .addImm(AArch64SysReg::TPIDR2_EL0)
+      .addReg(AArch64::XZR);
+
+  restorePhyRegSave(RegSave, MBB, MBBI, DL);
+}
+
+void MachineSMEABI::emitZAOff(MachineBasicBlock &MBB,
+                              MachineBasicBlock::iterator MBBI,
+                              bool ClearTPIDR2) {
+  DebugLoc DL = getDebugLoc(MBB, MBBI);
+
+  if (ClearTPIDR2)
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSR))
+        .addImm(AArch64SysReg::TPIDR2_EL0)
+        .addReg(AArch64::XZR);
+
+  // Disable ZA.
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSRpstatesvcrImm1))
+      .addImm(AArch64SVCR::SVCRZA)
+      .addImm(0);
+}
+
+void MachineSMEABI::emitAllocateLazySaveBuffer(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
+  MachineFrameInfo &MFI = MF->getFrameInfo();
+
+  DebugLoc DL = getDebugLoc(MBB, MBBI);
+  Register SP = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
+  Register SVL = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
+  Register Buffer = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
+
+  // Calculate SVL.
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::RDSVLI_XI), SVL).addImm(1);
+
+  // 1. Allocate the lazy save buffer.
+  {
+    // TODO This function grows the stack with a subtraction, which doesn't work
+    // on Windows. Some refactoring to share the functionality in
+    // LowerWindowsDYNAMIC_STACKALLOC will be required once the Windows ABI
+    // supports SME
+    assert(!Subtarget->isTargetWindows() &&
+           "Lazy ZA save is not yet supported on Windows");
+    // Get original stack pointer.
+    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), SP)
+        .addReg(AArch64::SP);
+    // Allocate a lazy-save buffer object of the size given, normally SVL * SVL
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSUBXrrr), Buffer)
+        .addReg(SVL)
+        .addReg(SVL)
+        .addReg(SP);
+    BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::SP)
+        .addReg(Buffer);
+    // We have just allocated a variable sized object, tell this to PEI.
+    MFI.CreateVariableSizedObject(Align(16), nullptr);
+  }
+
+  // 2. Setup the TPIDR2 block.
+  {
+    // Note: This case just needs to do `SVL << 48`. It is not implemented as we
+    // generally don't support big-endian SVE/SME.
+    if (!Subtarget->isLittleEndian())
+      reportFatalInternalError(
+          "TPIDR2 block initialization is not supported on big-endian targets");
+
+    // Store buffer pointer and num_za_save_slices.
+    // Bytes 10-15 are implicitly zeroed.
+    BuildMI(MBB, MBBI, DL, TII->get(AArch64::STPXi))
+        .addReg(Buffer)
+        .addReg(SVL)
+        .addFrameIndex(getTPIDR2Block().FrameIndex)
+        .addImm(0);
+  }
+}
+
+void MachineSMEABI::emitNewZAPrologue(MachineBasicBlock &MBB,
+                                      MachineBasicBlock::iterator MBBI) {
+  auto *TLI = Subtarget->getTargetLowering();
+  DebugLoc DL = getDebugLoc(MBB, MBBI);
+
+  // Get current TPIDR2_EL0.
+  Register TPIDR2EL0 = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MRS))
+      .addReg(TPIDR2EL0, RegState::Define)
+      .addImm(AArch64SysReg::TPIDR2_EL0);
+  // If TPIDR2_EL0 is non-zero, commit the lazy save.
+  // NOTE: Functions that only use ZT0 don't need to zero ZA.
+  bool ZeroZA =
+      MF->getInfo<AArch64FunctionInfo>()->getSMEFnAttrs().hasZAState();
+  auto CommitZASave =
+      BuildMI(MBB, MBBI, DL, TII->get(AArch64::CommitZASavePseudo))
+          .addReg(TPIDR2EL0)
+          .addImm(ZeroZA ? 1 : 0)
+          .addExternalSymbol(TLI->getLibcallName(RTLIB::SMEABI_TPIDR2_SAVE))
+          .addRegMask(TRI->SMEABISupportRoutinesCallPreservedMaskFromX0());
+  if (ZeroZA)
+    CommitZASave.addDef(AArch64::ZAB0, RegState::ImplicitDefine);
+  // Enable ZA (as ZA could have previously been in the OFF state).
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSRpstatesvcrImm1))
+      .addImm(AArch64SVCR::SVCRZA)
+      .addImm(1);
+}
+
+void MachineSMEABI::emitStateChange(MachineBasicBlock &MBB,
+                                    MachineBasicBlock::iterator InsertPt,
+                                    ZAState From, ZAState To,
+                                    LiveRegs PhysLiveRegs) {
+
+  // ZA not used.
+  if (From == ZAState::ANY || To == ZAState::ANY)
+    return;
+
+  // If we're exiting from the CALLER_DORMANT state that means this new ZA
+  // function did not touch ZA (so ZA was never turned on).
+  if (From == ZAState::CALLER_DORMANT && To == ZAState::OFF)
+    return;
+
+  // TODO: Avoid setting up the save buffer if there's no transition to
+  // LOCAL_SAVED.
+  if (From == ZAState::CALLER_DORMANT) {
+    assert(MBB.getParent()
+               ->getInfo<AArch64FunctionInfo>()
+               ->getSMEFnAttrs()
+               .hasPrivateZAInterface() &&
+           "CALLER_DORMANT state requires private ZA interface");
+    assert(&MBB == &MBB.getParent()->front() &&
+           "CALLER_DORMANT state only valid in entry block");
+    emitNewZAPrologue(MBB, MBB.getFirstNonPHI());
+    if (To == ZAState::ACTIVE)
+      return; // Nothing more to do (ZA is active after the prologue).
+
+    // Note: "emitNewZAPrologue" zeros ZA, so we may need to setup a lazy save
+    // if "To" is "ZAState::LOCAL_SAVED". It may be possible to improve this
+    // case by changing the placement of the zero instruction.
+    From = ZAState::ACTIVE;
+  }
+
+  if (From == ZAState::ACTIVE && To == ZAState::LOCAL_SAVED)
+    emitSetupLazySave(MBB, InsertPt);
+  else if (From == ZAState::LOCAL_SAVED && To == ZAState::ACTIVE)
+    emitRestoreLazySave(MBB, InsertPt, PhysLiveRegs);
+  else if (To == ZAState::OFF) {
+    assert(From != ZAState::CALLER_DORMANT &&
+           "CALLER_DORMANT to OFF should have already been handled");
+    emitZAOff(MBB, InsertPt, /*ClearTPIDR2=*/From == ZAState::LOCAL_SAVED);
+  } else {
+    dbgs() << "Error: Transition from " << getZAStateString(From) << " to "
+           << getZAStateString(To) << '\n';
+    llvm_unreachable("Unimplemented state transition");
+  }
+}
+
+} // end anonymous namespace
+
+INITIALIZE_PASS(MachineSMEABI, "aarch64-machine-sme-abi", "Machine SME ABI",
+                false, false)
+
+bool MachineSMEABI::runOnMachineFunction(MachineFunction &MF) {
+  if (!MF.getSubtarget<AArch64Subtarget>().hasSME())
+    return false;
+
+  auto *AFI = MF.getInfo<AArch64FunctionInfo>();
+  SMEAttrs SMEFnAttrs = AFI->getSMEFnAttrs();
+  if (!SMEFnAttrs.hasZAState() && !SMEFnAttrs.hasZT0State())
+    return false;
+
+  assert(MF.getRegInfo().isSSA() && "Expected to be run on SSA form!");
+
+  // Reset pass state.
+  State = PassState{};
+  this->MF = &MF;
+  Bundles = &getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
+  Subtarget = &MF.getSubtarget<AArch64Subtarget>();
+  TII = Subtarget->getInstrInfo();
+  TRI = Subtarget->getRegisterInfo();
+  MRI = &MF.getRegInfo();
+
+  collectNeededZAStates(SMEFnAttrs);
+  assignBundleZAStates();
+  insertStateChanges();
+
+  // Allocate save buffer (if needed).
+  if (State.TPIDR2Block) {
+    MachineBasicBlock &EntryBlock = MF.front();
+    emitAllocateLazySaveBuffer(EntryBlock, EntryBlock.getFirstNonPHI());
+  }
+
+  return true;
+}
+
+FunctionPass *llvm::createMachineSMEABIPass() { return new MachineSMEABI(); }
diff --git a/llvm/lib/Target/AArch64/SMEABIPass.cpp b/llvm/lib/Target/AArch64/SMEABIPass.cpp
index 4af4d49306625..2008516885c35 100644
--- a/llvm/lib/Target/AArch64/SMEABIPass.cpp
+++ b/llvm/lib/Target/AArch64/SMEABIPass.cpp
@@ -15,11 +15,16 @@
 #include "AArch64.h"
 #include "Utils/AArch64SMEAttributes.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicsAArch64.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/RuntimeLibcalls.h"
+#include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 
 using namespace llvm;
@@ -33,9 +38,13 @@ struct SMEABI : public FunctionPass {
 
   bool runOnFunction(Function &F) override;
 
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<TargetPassConfig>();
+  }
+
 private:
   bool updateNewStateFunctions(Module *M, Function *F, IRBuilder<> &Builder,
-                               SMEAttrs FnAttrs);
+                               SMEAttrs FnAttrs, const TargetLowering &TLI);
 };
 } // end anonymous namespace
 
@@ -51,14 +60,16 @@ FunctionPass *llvm::createSMEABIPass() { return new SMEABI(); }
 //===----------------------------------------------------------------------===//
 
 // Utility function to emit a call to __arm_tpidr2_save and clear TPIDR2_EL0.
-void emitTPIDR2Save(Module *M, IRBuilder<> &Builder, bool ZT0IsUndef = false) {
+void emitTPIDR2Save(Module *M, IRBuilder<> &Builder, const TargetLowering &TLI,
+                    bool ZT0IsUndef = false) {
   auto &Ctx = M->getContext();
   auto *TPIDR2SaveTy =
       FunctionType::get(Builder.getVoidTy(), {}, /*IsVarArgs=*/false);
   auto Attrs =
       AttributeList().addFnAttribute(Ctx, "aarch64_pstate_sm_compatible");
+  RTLIB::Libcall LC = RTLIB::SMEABI_TPIDR2_SAVE;
   FunctionCallee Callee =
-      M->getOrInsertFunction("__arm_tpidr2_save", TPIDR2SaveTy, Attrs);
+      M->getOrInsertFunction(TLI.getLibcallName(LC), TPIDR2SaveTy, Attrs);
   CallInst *Call = Builder.CreateCall(Callee);
 
   // If ZT0 is undefined (i.e. we're at the entry of a "new_zt0" function), mark
@@ -67,8 +78,7 @@ void emitTPIDR2Save(Module *M, IRBuilder<> &Builder, bool ZT0IsUndef = false) {
   if (ZT0IsUndef)
     Call->addFnAttr(Attribute::get(Ctx, "aarch64_zt0_undef"));
 
-  Call->setCallingConv(
-      CallingConv::AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0);
+  Call->setCallingConv(TLI.getLibcallCallingConv(LC));
 
   // A save to TPIDR2 should be followed by clearing TPIDR2_EL0.
   Function *WriteIntr =
@@ -98,7 +108,8 @@ void emitTPIDR2Save(Module *M, IRBuilder<> &Builder, bool ZT0IsUndef = false) {
 /// interface if it does not share ZA or ZT0.
 ///
 bool SMEABI::updateNewStateFunctions(Module *M, Function *F,
-                                     IRBuilder<> &Builder, SMEAttrs FnAttrs) {
+                                     IRBuilder<> &Builder, SMEAttrs FnAttrs,
+                                     const TargetLowering &TLI) {
   LLVMContext &Context = F->getContext();
   BasicBlock *OrigBB = &F->getEntryBlock();
   Builder.SetInsertPoint(&OrigBB->front());
@@ -124,7 +135,7 @@ bool SMEABI::updateNewStateFunctions(Module *M, Function *F,
 
     // Create a call __arm_tpidr2_save, which commits the lazy save.
     Builder.SetInsertPoint(&SaveBB->back());
-    emitTPIDR2Save(M, Builder, /*ZT0IsUndef=*/FnAttrs.isNewZT0());
+    emitTPIDR2Save(M, Builder, TLI, /*ZT0IsUndef=*/FnAttrs.isNewZT0());
 
     // Enable pstate.za at the start of the function.
     Builder.SetInsertPoint(&OrigBB->front());
@@ -172,10 +183,14 @@ bool SMEABI::runOnFunction(Function &F) {
   if (F.isDeclaration() || F.hasFnAttribute("aarch64_expanded_pstate_za"))
     return false;
 
+  const TargetMachine &TM =
+      getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
+  const TargetLowering &TLI = *TM.getSubtargetImpl(F)->getTargetLowering();
+
   bool Changed = false;
   SMEAttrs FnAttrs(F);
   if (FnAttrs.isNewZA() || FnAttrs.isNewZT0())
-    Changed |= updateNewStateFunctions(M, &F, Builder, FnAttrs);
+    Changed |= updateNewStateFunctions(M, &F, Builder, FnAttrs, TLI);
 
   return Changed;
 }
diff --git a/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp b/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp
index bd28716118880..85cca1de47b78 100644
--- a/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp
+++ b/llvm/lib/Target/AArch64/SMEPeepholeOpt.cpp
@@ -80,16 +80,10 @@ static bool isMatchingStartStopPair(const MachineInstr *MI1,
   if (MI1->getOperand(4).getRegMask() != MI2->getOperand(4).getRegMask())
     return false;
 
-  // This optimisation is unlikely to happen in practice for conditional
-  // smstart/smstop pairs as the virtual registers for pstate.sm will always
-  // be different.
-  // TODO: For this optimisation to apply to conditional smstart/smstop,
-  // this pass will need to do more work to remove redundant calls to
-  // __arm_sme_state.
-
   // Only consider conditional start/stop pairs which read the same register
-  // holding the original value of pstate.sm, as some conditional start/stops
-  // require the state on entry to the function.
+  // holding the original value of pstate.sm. This is somewhat over conservative
+  // as all conditional streaming mode changes only look at the state on entry
+  // to the function.
   if (MI1->getOperand(3).isReg() && MI2->getOperand(3).isReg()) {
     Register Reg1 = MI1->getOperand(3).getReg();
     Register Reg2 = MI2->getOperand(3).getReg();
@@ -134,13 +128,6 @@ bool SMEPeepholeOpt::optimizeStartStopPairs(
 
   bool Changed = false;
   MachineInstr *Prev = nullptr;
-  SmallVector<MachineInstr *, 4> ToBeRemoved;
-
-  // Convenience function to reset the matching of a sequence.
-  auto Reset = [&]() {
-    Prev = nullptr;
-    ToBeRemoved.clear();
-  };
 
   // Walk through instructions in the block trying to find pairs of smstart
   // and smstop nodes that cancel each other out. We only permit a limited
@@ -162,14 +149,10 @@ bool SMEPeepholeOpt::optimizeStartStopPairs(
         // that we marked for deletion in between.
         Prev->eraseFromParent();
         MI.eraseFromParent();
-        for (MachineInstr *TBR : ToBeRemoved)
-          TBR->eraseFromParent();
-        ToBeRemoved.clear();
         Prev = nullptr;
         Changed = true;
         NumSMChangesRemoved += 2;
       } else {
-        Reset();
         Prev = &MI;
       }
       continue;
@@ -185,7 +168,7 @@ bool SMEPeepholeOpt::optimizeStartStopPairs(
     // of streaming mode. If not, the algorithm should reset.
     switch (MI.getOpcode()) {
     default:
-      Reset();
+      Prev = nullptr;
       break;
     case AArch64::COALESCER_BARRIER_FPR16:
     case AArch64::COALESCER_BARRIER_FPR32:
@@ -199,7 +182,7 @@ bool SMEPeepholeOpt::optimizeStartStopPairs(
       // concrete example/test-case.
       if (isSVERegOp(TRI, MRI, MI.getOperand(0)) ||
           isSVERegOp(TRI, MRI, MI.getOperand(1)))
-        Reset();
+        Prev = nullptr;
       break;
     case AArch64::ADJCALLSTACKDOWN:
     case AArch64::ADJCALLSTACKUP:
@@ -207,12 +190,6 @@ bool SMEPeepholeOpt::optimizeStartStopPairs(
     case AArch64::ADDXri:
       // We permit these as they don't generate SVE/NEON instructions.
       break;
-    case AArch64::VGRestorePseudo:
-    case AArch64::VGSavePseudo:
-      // When the smstart/smstop are removed, we should also remove
-      // the pseudos that save/restore the VG value for CFI info.
-      ToBeRemoved.push_back(&MI);
-      break;
     case AArch64::MSRpstatesvcrImm1:
     case AArch64::MSRpstatePseudo:
       llvm_unreachable("Should have been handled");
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index a0320f919e8c5..a3a7d0f74e1bc 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -809,6 +809,11 @@ let hasNoSchedulingInfo = 1 in {
     Pseudo<(outs zprty:$Zd), (ins PPR3bAny:$Pg, zprty:$Zs1, zprty:$Zs2, zprty:$Zs3), []> {
     let FalseLanes = flags;
   }
+
+  class UnpredRegImmPseudo<ZPRRegOp zprty, Operand immty>
+  : SVEPseudo2Instr<NAME, 0>,
+    Pseudo<(outs zprty:$Zd), (ins zprty:$Zs, immty:$imm), []> {
+  }
 }
 
 //
@@ -1885,13 +1890,14 @@ class sve_int_perm_extract_i<string asm>
   let Inst{4-0}   = Zdn;
 
   let Constraints = "$Zdn = $_Zdn";
-  let DestructiveInstType = DestructiveOther;
+  let DestructiveInstType = Destructive2xRegImmUnpred;
   let ElementSize = ElementSizeNone;
   let hasSideEffects = 0;
 }
 
-multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op> {
-  def NAME : sve_int_perm_extract_i<asm>;
+multiclass sve_int_perm_extract_i<string asm, SDPatternOperator op, string Ps> {
+  def NAME : sve_int_perm_extract_i<asm>,
+             SVEPseudo2Instr<Ps, 1>;
 
   def : SVE_3_Op_Imm_Pat<nxv16i8, op, nxv16i8, nxv16i8, i32, imm0_255,
                          !cast<Instruction>(NAME)>;
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.cpp b/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.cpp
index 271094f935e0e..dd6fa167c6f4d 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.cpp
+++ b/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.cpp
@@ -7,17 +7,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "AArch64SMEAttributes.h"
+#include "AArch64ISelLowering.h"
 #include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/RuntimeLibcalls.h"
 #include <cassert>
 
 using namespace llvm;
 
-void SMEAttrs::set(unsigned M, bool Enable) {
-  if (Enable)
-    Bitmask |= M;
-  else
-    Bitmask &= ~M;
-
+void SMEAttrs::validate() const {
   // Streaming Mode Attrs
   assert(!(hasStreamingInterface() && hasStreamingCompatibleInterface()) &&
          "SM_Enabled and SM_Compatible are mutually exclusive");
@@ -77,19 +74,36 @@ SMEAttrs::SMEAttrs(const AttributeList &Attrs) {
     Bitmask |= encodeZT0State(StateValue::New);
 }
 
-void SMEAttrs::addKnownFunctionAttrs(StringRef FuncName) {
+void SMEAttrs::addKnownFunctionAttrs(StringRef FuncName,
+                                     const AArch64TargetLowering &TLI) {
+  RTLIB::LibcallImpl Impl = TLI.getSupportedLibcallImpl(FuncName);
+  if (Impl == RTLIB::Unsupported)
+    return;
   unsigned KnownAttrs = SMEAttrs::Normal;
-  if (FuncName == "__arm_tpidr2_save" || FuncName == "__arm_sme_state")
-    KnownAttrs |= (SMEAttrs::SM_Compatible | SMEAttrs::SME_ABI_Routine);
-  if (FuncName == "__arm_tpidr2_restore")
+  RTLIB::Libcall LC = RTLIB::RuntimeLibcallsInfo::getLibcallFromImpl(Impl);
+  switch (LC) {
+  case RTLIB::SMEABI_SME_STATE:
+  case RTLIB::SMEABI_TPIDR2_SAVE:
+  case RTLIB::SMEABI_GET_CURRENT_VG:
+  case RTLIB::SMEABI_SME_STATE_SIZE:
+  case RTLIB::SMEABI_SME_SAVE:
+  case RTLIB::SMEABI_SME_RESTORE:
+    KnownAttrs |= SMEAttrs::SM_Compatible | SMEAttrs::SME_ABI_Routine;
+    break;
+  case RTLIB::SMEABI_ZA_DISABLE:
+  case RTLIB::SMEABI_TPIDR2_RESTORE:
     KnownAttrs |= SMEAttrs::SM_Compatible | encodeZAState(StateValue::In) |
                   SMEAttrs::SME_ABI_Routine;
-  if (FuncName == "__arm_sc_memcpy" || FuncName == "__arm_sc_memset" ||
-      FuncName == "__arm_sc_memmove" || FuncName == "__arm_sc_memchr")
+    break;
+  case RTLIB::SC_MEMCPY:
+  case RTLIB::SC_MEMMOVE:
+  case RTLIB::SC_MEMSET:
+  case RTLIB::SC_MEMCHR:
     KnownAttrs |= SMEAttrs::SM_Compatible;
-  if (FuncName == "__arm_sme_save" || FuncName == "__arm_sme_restore" ||
-      FuncName == "__arm_sme_state_size")
-    KnownAttrs |= SMEAttrs::SM_Compatible | SMEAttrs::SME_ABI_Routine;
+    break;
+  default:
+    break;
+  }
   set(KnownAttrs);
 }
 
@@ -110,11 +124,11 @@ bool SMECallAttrs::requiresSMChange() const {
   return true;
 }
 
-SMECallAttrs::SMECallAttrs(const CallBase &CB)
+SMECallAttrs::SMECallAttrs(const CallBase &CB, const AArch64TargetLowering *TLI)
     : CallerFn(*CB.getFunction()), CalledFn(SMEAttrs::Normal),
       Callsite(CB.getAttributes()), IsIndirect(CB.isIndirectCall()) {
   if (auto *CalledFunction = CB.getCalledFunction())
-    CalledFn = SMEAttrs(*CalledFunction, SMEAttrs::InferAttrsFromName::Yes);
+    CalledFn = SMEAttrs(*CalledFunction, TLI);
 
   // FIXME: We probably should not allow SME attributes on direct calls but
   // clang duplicates streaming mode attributes at each callsite.
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.h b/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.h
index f1be0ecbee7ed..d26e3cd3a9f76 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.h
+++ b/llvm/lib/Target/AArch64/Utils/AArch64SMEAttributes.h
@@ -13,6 +13,8 @@
 
 namespace llvm {
 
+class AArch64TargetLowering;
+
 class Function;
 class CallBase;
 class AttributeList;
@@ -48,19 +50,27 @@ class SMEAttrs {
     CallSiteFlags_Mask = ZT0_Undef
   };
 
-  enum class InferAttrsFromName { No, Yes };
-
   SMEAttrs() = default;
   SMEAttrs(unsigned Mask) { set(Mask); }
-  SMEAttrs(const Function &F, InferAttrsFromName Infer = InferAttrsFromName::No)
+  SMEAttrs(const Function &F, const AArch64TargetLowering *TLI = nullptr)
       : SMEAttrs(F.getAttributes()) {
-    if (Infer == InferAttrsFromName::Yes)
-      addKnownFunctionAttrs(F.getName());
+    if (TLI)
+      addKnownFunctionAttrs(F.getName(), *TLI);
   }
   SMEAttrs(const AttributeList &L);
-  SMEAttrs(StringRef FuncName) { addKnownFunctionAttrs(FuncName); };
+  SMEAttrs(StringRef FuncName, const AArch64TargetLowering &TLI) {
+    addKnownFunctionAttrs(FuncName, TLI);
+  };
 
-  void set(unsigned M, bool Enable = true);
+  void set(unsigned M, bool Enable = true) {
+    if (Enable)
+      Bitmask |= M;
+    else
+      Bitmask &= ~M;
+#ifndef NDEBUG
+    validate();
+#endif
+  }
 
   // Interfaces to query PSTATE.SM
   bool hasStreamingBody() const { return Bitmask & SM_Body; }
@@ -146,7 +156,9 @@ class SMEAttrs {
   }
 
 private:
-  void addKnownFunctionAttrs(StringRef FuncName);
+  void addKnownFunctionAttrs(StringRef FuncName,
+                             const AArch64TargetLowering &TLI);
+  void validate() const;
 };
 
 /// SMECallAttrs is a utility class to hold the SMEAttrs for a callsite. It has
@@ -163,7 +175,7 @@ class SMECallAttrs {
                SMEAttrs Callsite = SMEAttrs::Normal)
       : CallerFn(Caller), CalledFn(Callee), Callsite(Callsite) {}
 
-  SMECallAttrs(const CallBase &CB);
+  SMECallAttrs(const CallBase &CB, const AArch64TargetLowering *TLI);
 
   SMEAttrs &caller() { return CallerFn; }
   SMEAttrs &callee() { return IsIndirect ? Callsite : CalledFn; }
@@ -194,7 +206,7 @@ class SMECallAttrs {
   }
 
   bool requiresEnablingZAAfterCall() const {
-    return requiresLazySave() || requiresDisablingZABeforeCall();
+    return requiresDisablingZABeforeCall();
   }
 
   bool requiresPreservingAllZAState() const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 007b481f84960..0059a862ba9b2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -439,10 +439,6 @@ struct AMDGPUPrintfRuntimeBindingPass
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
-struct AMDGPUUnifyMetadataPass : PassInfoMixin<AMDGPUUnifyMetadataPass> {
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-};
-
 void initializeSIOptimizeExecMaskingPreRALegacyPass(PassRegistry &);
 extern char &SIOptimizeExecMaskingPreRAID;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index d84f512f4976d..8e4b6365dc06b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1013,6 +1013,14 @@ def FeatureAgentScopeFineGrainedRemoteMemoryAtomics
   "device memory."
 >;
 
+def FeatureEmulatedSystemScopeAtomics
+  : SubtargetFeature<"emulated-system-scope-atomics",
+  "HasEmulatedSystemScopeAtomics",
+  "true",
+  "System scope atomics unsupported by the PCI-e are emulated in HW via CAS "
+  "loop and functional."
+>;
+
 def FeatureDefaultComponentZero : SubtargetFeature<"default-component-zero",
   "HasDefaultComponentZero",
   "true",
@@ -1540,7 +1548,7 @@ def FeatureGFX11 : GCNSubtargetFeatureGeneration<"GFX11",
 
 def FeatureGFX12 : GCNSubtargetFeatureGeneration<"GFX12",
   "gfx12",
-  [FeatureFP64, FeatureAddressableLocalMemorySize65536, FeatureMIMG_R128,
+  [FeatureFP64, FeatureMIMG_R128,
    FeatureFlatAddressSpace, Feature16BitInsts,
    FeatureInv2PiInlineImm, FeatureApertureRegs,
    FeatureCIInsts, FeatureGFX8Insts, FeatureGFX9Insts, FeatureGFX10Insts,
@@ -1969,6 +1977,7 @@ def FeatureISAVersion11_5_3 : FeatureSet<
 
 def FeatureISAVersion12 : FeatureSet<
   [FeatureGFX12,
+   FeatureAddressableLocalMemorySize65536,
    FeatureLDSBankCount32,
    FeatureDLInsts,
    FeatureDot7Insts,
@@ -2011,6 +2020,7 @@ def FeatureISAVersion12_50 : FeatureSet<
   [FeatureGFX12,
    FeatureGFX1250Insts,
    FeatureCUStores,
+   FeatureAddressableLocalMemorySize327680,
    FeatureCuMode,
    Feature64BitLiterals,
    FeatureLDSBankCount32,
@@ -2062,6 +2072,7 @@ def FeatureISAVersion12_50 : FeatureSet<
    FeatureAtomicFMinFMaxF64FlatInsts,
    FeatureFlatBufferGlobalAtomicFaddF64Inst,
    FeatureMemoryAtomicFAddF32DenormalSupport,
+   FeatureEmulatedSystemScopeAtomics,
    FeatureGloballyAddressableScratch,
    FeatureKernargPreload,
    FeatureVmemPrefInsts,
@@ -2603,6 +2614,10 @@ def HasPkMinMax3Insts :
   Predicate<"Subtarget->hasPkMinMax3Insts()">,
   AssemblerPredicate<(any_of FeatureGFX1250Insts)>;
 
+def HasSGetShaderCyclesInst :
+  Predicate<"Subtarget->hasSGetShaderCyclesInst()">,
+  AssemblerPredicate<(any_of FeatureGFX1250Insts)>;
+
 def HasImageInsts : Predicate<"Subtarget->hasImageInsts()">,
   AssemblerPredicate<(all_of FeatureImageInsts)>;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 2a324e5683910..36c0d1cbcea22 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -41,6 +41,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/AMDHSAKernelDescriptor.h"
 #include "llvm/Support/Compiler.h"
@@ -719,6 +720,8 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
                      IsLocal),
         RI.getSymbol(CurrentFnSym->getName(), RIK::RIK_NumSGPR, OutContext,
                      IsLocal),
+        RI.getSymbol(CurrentFnSym->getName(), RIK::RIK_NumNamedBarrier,
+                     OutContext, IsLocal),
         RI.getSymbol(CurrentFnSym->getName(), RIK::RIK_PrivateSegSize,
                      OutContext, IsLocal),
         RI.getSymbol(CurrentFnSym->getName(), RIK::RIK_UsesVCC, OutContext,
@@ -733,6 +736,9 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
                      OutContext, IsLocal));
   }
 
+  // Emit _dvgpr$ symbol when appropriate.
+  emitDVgprSymbol(MF);
+
   if (isVerbose()) {
     MCSectionELF *CommentSection =
         Context.getELFSection(".AMDGPU.csdata", ELF::SHT_PROGBITS, 0);
@@ -803,6 +809,11 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
           " AccumOffset: " + getMCExprStr(AdjustedAccum), false);
     }
 
+    if (AMDGPU::isGFX1250(STM))
+      OutStreamer->emitRawComment(
+          " NamedBarCnt: " + getMCExprStr(CurrentProgramInfo.NamedBarCnt),
+          false);
+
     OutStreamer->emitRawComment(
         " Occupancy: " + getMCExprStr(CurrentProgramInfo.Occupancy), false);
 
@@ -875,6 +886,49 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   return false;
 }
 
+// When appropriate, add a _dvgpr$ symbol, with the value of the function
+// symbol, plus an offset encoding one less than the number of VGPR blocks used
+// by the function in bits 5..3 of the symbol value. A "VGPR block" can be
+// either 16 VGPRs (for a max of 128), or 32 VGPRs (for a max of 256). This is
+// used by a front-end to have functions that are chained rather than called,
+// and a dispatcher that dynamically resizes the VGPR count before dispatching
+// to a function.
+void AMDGPUAsmPrinter::emitDVgprSymbol(MachineFunction &MF) {
+  const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
+  if (MFI.isDynamicVGPREnabled() &&
+      MF.getFunction().getCallingConv() == CallingConv::AMDGPU_CS_Chain) {
+    MCContext &Ctx = MF.getContext();
+    unsigned BlockSize = MFI.getDynamicVGPRBlockSize();
+    MCValue NumVGPRs;
+    if (!CurrentProgramInfo.NumVGPRsForWavesPerEU->evaluateAsRelocatable(
+            NumVGPRs, nullptr) ||
+        !NumVGPRs.isAbsolute()) {
+      llvm_unreachable("unable to resolve NumVGPRs for _dvgpr$ symbol");
+    }
+    // Calculate number of VGPR blocks.
+    // Treat 0 VGPRs as 1 VGPR to avoid underflowing.
+    unsigned NumBlocks =
+        divideCeil(std::max(unsigned(NumVGPRs.getConstant()), 1U), BlockSize);
+
+    if (NumBlocks > 8) {
+      OutContext.reportError({},
+                             "too many DVGPR blocks for _dvgpr$ symbol for '" +
+                                 Twine(CurrentFnSym->getName()) + "'");
+      return;
+    }
+    unsigned EncodedNumBlocks = (NumBlocks - 1) << 3;
+    // Add to function symbol to create _dvgpr$ symbol.
+    const MCExpr *DVgprFuncVal = MCBinaryExpr::createAdd(
+        MCSymbolRefExpr::create(CurrentFnSym, Ctx),
+        MCConstantExpr::create(EncodedNumBlocks, Ctx), Ctx);
+    MCSymbol *DVgprFuncSym =
+        Ctx.getOrCreateSymbol(Twine("_dvgpr$") + CurrentFnSym->getName());
+    OutStreamer->emitAssignment(DVgprFuncSym, DVgprFuncVal);
+    emitVisibility(DVgprFuncSym, MF.getFunction().getVisibility());
+    emitLinkage(&MF.getFunction(), DVgprFuncSym);
+  }
+}
+
 // TODO: Fold this into emitFunctionBodyStart.
 void AMDGPUAsmPrinter::initializeTargetID(const Module &M) {
   // In the beginning all features are either 'Any' or 'NotSupported',
@@ -965,6 +1019,11 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
       MCBinaryExpr::createOr(GetSymRefExpr(RIK::RIK_HasDynSizedStack),
                              GetSymRefExpr(RIK::RIK_HasRecursion), Ctx);
 
+  const MCExpr *BarBlkConst = MCConstantExpr::create(4, Ctx);
+  const MCExpr *AlignToBlk = AMDGPUMCExpr::createAlignTo(
+      GetSymRefExpr(RIK::RIK_NumNamedBarrier), BarBlkConst, Ctx);
+  ProgInfo.NamedBarCnt = MCBinaryExpr::createDiv(AlignToBlk, BarBlkConst, Ctx);
+
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 
   // The calculations related to SGPR/VGPR blocks are
@@ -997,89 +1056,24 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
   const Function &F = MF.getFunction();
 
   // Ensure there are enough SGPRs and VGPRs for wave dispatch, where wave
-  // dispatch registers are function args.
-  unsigned WaveDispatchNumSGPR = 0, WaveDispatchNumVGPR = 0;
-
-  if (isShader(F.getCallingConv())) {
-    bool IsPixelShader =
-        F.getCallingConv() == CallingConv::AMDGPU_PS && !STM.isAmdHsaOS();
-
-    // Calculate the number of VGPR registers based on the SPI input registers
-    uint32_t InputEna = 0;
-    uint32_t InputAddr = 0;
-    unsigned LastEna = 0;
-
-    if (IsPixelShader) {
-      // Note for IsPixelShader:
-      // By this stage, all enabled inputs are tagged in InputAddr as well.
-      // We will use InputAddr to determine whether the input counts against the
-      // vgpr total and only use the InputEnable to determine the last input
-      // that is relevant - if extra arguments are used, then we have to honour
-      // the InputAddr for any intermediate non-enabled inputs.
-      InputEna = MFI->getPSInputEnable();
-      InputAddr = MFI->getPSInputAddr();
-
-      // We only need to consider input args up to the last used arg.
-      assert((InputEna || InputAddr) &&
-             "PSInputAddr and PSInputEnable should "
-             "never both be 0 for AMDGPU_PS shaders");
-      // There are some rare circumstances where InputAddr is non-zero and
-      // InputEna can be set to 0. In this case we default to setting LastEna
-      // to 1.
-      LastEna = InputEna ? llvm::Log2_32(InputEna) + 1 : 1;
-    }
+  // dispatch registers as function args.
+  unsigned WaveDispatchNumSGPR = MFI->getNumWaveDispatchSGPRs(),
+           WaveDispatchNumVGPR = MFI->getNumWaveDispatchVGPRs();
 
-    // FIXME: We should be using the number of registers determined during
-    // calling convention lowering to legalize the types.
-    const DataLayout &DL = F.getDataLayout();
-    unsigned PSArgCount = 0;
-    unsigned IntermediateVGPR = 0;
-    for (auto &Arg : F.args()) {
-      unsigned NumRegs = (DL.getTypeSizeInBits(Arg.getType()) + 31) / 32;
-      if (Arg.hasAttribute(Attribute::InReg)) {
-        WaveDispatchNumSGPR += NumRegs;
-      } else {
-        // If this is a PS shader and we're processing the PS Input args (first
-        // 16 VGPR), use the InputEna and InputAddr bits to define how many
-        // VGPRs are actually used.
-        // Any extra VGPR arguments are handled as normal arguments (and
-        // contribute to the VGPR count whether they're used or not).
-        if (IsPixelShader && PSArgCount < 16) {
-          if ((1 << PSArgCount) & InputAddr) {
-            if (PSArgCount < LastEna)
-              WaveDispatchNumVGPR += NumRegs;
-            else
-              IntermediateVGPR += NumRegs;
-          }
-          PSArgCount++;
-        } else {
-          // If there are extra arguments we have to include the allocation for
-          // the non-used (but enabled with InputAddr) input arguments
-          if (IntermediateVGPR) {
-            WaveDispatchNumVGPR += IntermediateVGPR;
-            IntermediateVGPR = 0;
-          }
-          WaveDispatchNumVGPR += NumRegs;
-        }
-      }
-    }
+  if (WaveDispatchNumSGPR) {
     ProgInfo.NumSGPR = AMDGPUMCExpr::createMax(
-        {ProgInfo.NumSGPR, CreateExpr(WaveDispatchNumSGPR)}, Ctx);
+        {ProgInfo.NumSGPR,
+         MCBinaryExpr::createAdd(CreateExpr(WaveDispatchNumSGPR), ExtraSGPRs,
+                                 Ctx)},
+        Ctx);
+  }
 
+  if (WaveDispatchNumVGPR) {
     ProgInfo.NumArchVGPR = AMDGPUMCExpr::createMax(
         {ProgInfo.NumVGPR, CreateExpr(WaveDispatchNumVGPR)}, Ctx);
 
     ProgInfo.NumVGPR = AMDGPUMCExpr::createTotalNumVGPR(
         ProgInfo.NumAccVGPR, ProgInfo.NumArchVGPR, Ctx);
-  } else if (isKernel(F.getCallingConv()) &&
-             MFI->getNumKernargPreloadedSGPRs()) {
-    // Consider cases where the total number of UserSGPRs with trailing
-    // allocated preload SGPRs, is greater than the number of explicitly
-    // referenced SGPRs.
-    const MCExpr *UserPlusExtraSGPRs = MCBinaryExpr::createAdd(
-        CreateExpr(MFI->getNumUserSGPRs()), ExtraSGPRs, Ctx);
-    ProgInfo.NumSGPR =
-        AMDGPUMCExpr::createMax({ProgInfo.NumSGPR, UserPlusExtraSGPRs}, Ctx);
   }
 
   // Adjust number of registers used to meet default/requested minimum/maximum
@@ -1128,8 +1122,7 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
     Ctx.diagnose(Diag);
   }
 
-  if (MFI->getLDSSize() >
-      static_cast<unsigned>(STM.getAddressableLocalMemorySize())) {
+  if (MFI->getLDSSize() > STM.getAddressableLocalMemorySize()) {
     LLVMContext &Ctx = MF.getFunction().getContext();
     DiagnosticInfoResourceLimit Diag(
         MF.getFunction(), "local memory", MFI->getLDSSize(),
@@ -1168,7 +1161,11 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
   ProgInfo.DX10Clamp = Mode.DX10Clamp;
 
   unsigned LDSAlignShift;
-  if (STM.getFeatureBits().test(FeatureAddressableLocalMemorySize163840)) {
+  if (STM.getFeatureBits().test(FeatureAddressableLocalMemorySize327680)) {
+    // LDS is allocated in 256 dword blocks.
+    LDSAlignShift = 10;
+  } else if (STM.getFeatureBits().test(
+                 FeatureAddressableLocalMemorySize163840)) {
     // LDS is allocated in 320 dword blocks.
     LDSAlignShift = 11;
   } else if (STM.getFeatureBits().test(
@@ -1205,8 +1202,11 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
                               CreateExpr(STM.getWavefrontSize()), Ctx),
       CreateExpr(1ULL << ScratchAlignShift));
 
-  if (getIsaVersion(getGlobalSTI()->getCPU()).Major >= 10) {
+  if (STM.supportsWGP()) {
     ProgInfo.WgpMode = STM.isCuModeEnabled() ? 0 : 1;
+  }
+
+  if (getIsaVersion(getGlobalSTI()->getCPU()).Major >= 10) {
     ProgInfo.MemOrdered = 1;
     ProgInfo.FwdProgress = 1;
   }
@@ -1264,6 +1264,12 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
                 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT);
   }
 
+  if (AMDGPU::isGFX1250(STM))
+    ProgInfo.ComputePGMRSrc3 =
+        SetBits(ProgInfo.ComputePGMRSrc3, ProgInfo.NamedBarCnt,
+                amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
+                amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT);
+
   ProgInfo.Occupancy = AMDGPUMCExpr::createOccupancy(
       STM.computeOccupancy(F, ProgInfo.LDSSize).second,
       ProgInfo.NumSGPRsForWavesPerEU, ProgInfo.NumVGPRsForWavesPerEU,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
index 63589d2b90062..9e854fa554672 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
@@ -54,6 +54,9 @@ class AMDGPUAsmPrinter final : public AsmPrinter {
 
   MCCodeEmitter *DumpCodeInstEmitter = nullptr;
 
+  // When appropriate, add a _dvgpr$ symbol.
+  void emitDVgprSymbol(MachineFunction &MF);
+
   void getSIProgramInfo(SIProgramInfo &Out, const MachineFunction &MF);
   void getAmdKernelCode(AMDGPU::AMDGPUMCKernelCodeT &Out,
                         const SIProgramInfo &KernelInfo,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 3d8d274f06246..d1a5b4e85da46 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -580,6 +580,9 @@ bool AMDGPUCallLowering::lowerFormalArgumentsKernel(
     ++i;
   }
 
+  if (Info->getNumKernargPreloadedSGPRs())
+    Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
+
   TLI.allocateSpecialEntryInputVGPRs(CCInfo, MF, *TRI, *Info);
   TLI.allocateSystemSGPRs(CCInfo, MF, *Info, F.getCallingConv(), false);
   return true;
@@ -743,6 +746,15 @@ bool AMDGPUCallLowering::lowerFormalArguments(
   if (!determineAssignments(Assigner, SplitArgs, CCInfo))
     return false;
 
+  if (IsEntryFunc) {
+    // This assumes the registers are allocated by CCInfo in ascending order
+    // with no gaps.
+    Info->setNumWaveDispatchSGPRs(
+        CCInfo.getFirstUnallocated(AMDGPU::SGPR_32RegClass.getRegisters()));
+    Info->setNumWaveDispatchVGPRs(
+        CCInfo.getFirstUnallocated(AMDGPU::VGPR_32RegClass.getRegisters()));
+  }
+
   FormalArgHandler Handler(B, MRI);
   if (!handleAssignments(Handler, SplitArgs, CCInfo, ArgLocs, B))
     return false;
@@ -1464,9 +1476,22 @@ bool AMDGPUCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
                                    CallLoweringInfo &Info) const {
   if (Function *F = Info.CB->getCalledFunction())
     if (F->isIntrinsic()) {
-      assert(F->getIntrinsicID() == Intrinsic::amdgcn_cs_chain &&
-             "Unexpected intrinsic");
-      return lowerChainCall(MIRBuilder, Info);
+      switch (F->getIntrinsicID()) {
+      case Intrinsic::amdgcn_cs_chain:
+        return lowerChainCall(MIRBuilder, Info);
+      case Intrinsic::amdgcn_call_whole_wave:
+        Info.CallConv = CallingConv::AMDGPU_Gfx_WholeWave;
+
+        // Get the callee from the original instruction, so it doesn't look like
+        // this is an indirect call.
+        Info.Callee = MachineOperand::CreateGA(
+            cast<GlobalValue>(Info.CB->getOperand(0)), /*Offset=*/0);
+        Info.OrigArgs.erase(Info.OrigArgs.begin());
+        Info.IsVarArg = false;
+        break;
+      default:
+        llvm_unreachable("Unexpected intrinsic call");
+      }
     }
 
   if (Info.IsVarArg) {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index a9278c1dc3a6a..24bef82464495 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
@@ -26,6 +27,7 @@
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/PatternMatch.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Support/KnownFPClass.h"
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
index 9587fad1ecd63..b5dac95b57a2d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombine.td
@@ -151,6 +151,25 @@ def zext_of_shift_amount_combines : GICombineGroup<[
   canonicalize_zext_lshr, canonicalize_zext_ashr, canonicalize_zext_shl
 ]>;
 
+// (and/or i64:x, i64:y) -> i64:(merge (and/or lo_32(x), lo_32(y)), (and/or hi_32(x), hi_32(y)))
+// when either x or y is all ones in low or high parts
+class combine_binop_s64_with_s32_mask<Instruction opcode> : GICombineRule<
+  (defs root:$dst),
+  (match (opcode $dst, i64:$x, i64:$y):$dst,
+         [{ return Helper.matchConstantIs32BitMask(${x}.getReg()) ||
+                   Helper.matchConstantIs32BitMask(${y}.getReg()); }]),
+  (apply (G_UNMERGE_VALUES i32:$x_lo, i32:$x_hi, $x),
+         (G_UNMERGE_VALUES i32:$y_lo, i32:$y_hi, $y),
+         (opcode i32:$lo, $x_lo, $y_lo),
+         (opcode i32:$hi, $x_hi, $y_hi),
+         (G_MERGE_VALUES $dst, $lo, $hi))>;
+
+def combine_or_s64_with_s32_mask : combine_binop_s64_with_s32_mask<G_OR>;
+def combine_and_s64_with_s32_mask : combine_binop_s64_with_s32_mask<G_AND>;
+def binop_s64_with_s32_mask_combines : GICombineGroup<[
+  combine_or_s64_with_s32_mask, combine_and_s64_with_s32_mask
+]>;
+
 let Predicates = [Has16BitInsts, NotHasMed3_16] in {
 // For gfx8, expand f16-fmed3-as-f32 into a min/max f16 sequence. This
 // saves one instruction compared to the promotion.
@@ -180,7 +199,8 @@ def gfx8_combines : GICombineGroup<[expand_promoted_fmed3]>;
 def AMDGPUPreLegalizerCombiner: GICombiner<
   "AMDGPUPreLegalizerCombinerImpl",
   [all_combines, combine_fmul_with_select_to_fldexp, clamp_i64_to_i16,
-   foldable_fneg, combine_shuffle_vector_to_build_vector]> {
+   foldable_fneg, combine_shuffle_vector_to_build_vector,
+   binop_s64_with_s32_mask_combines]> {
   let CombineAllMethodName = "tryCombineAllImpl";
 }
 
@@ -188,7 +208,8 @@ def AMDGPUPostLegalizerCombiner: GICombiner<
   "AMDGPUPostLegalizerCombinerImpl",
   [all_combines, gfx6gfx7_combines, gfx8_combines, combine_fmul_with_select_to_fldexp,
    uchar_to_float, cvt_f32_ubyteN, remove_fcanonicalize, foldable_fneg,
-   rcp_sqrt_to_rsq, fdiv_by_sqrt_to_rsq_f16, sign_extension_in_reg, smulu64]> {
+   rcp_sqrt_to_rsq, fdiv_by_sqrt_to_rsq_f16, sign_extension_in_reg, smulu64,
+   binop_s64_with_s32_mask_combines]> {
   let CombineAllMethodName = "tryCombineAllImpl";
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
index bcc19932fd938..d23521c87e202 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
@@ -516,3 +516,18 @@ bool AMDGPUCombinerHelper::matchCombineFmulWithSelectToFldexp(
 
   return true;
 }
+
+bool AMDGPUCombinerHelper::matchConstantIs32BitMask(Register Reg) const {
+  auto Res = getIConstantVRegValWithLookThrough(Reg, MRI);
+  if (!Res)
+    return false;
+
+  const uint64_t Val = Res->Value.getZExtValue();
+  unsigned MaskIdx = 0;
+  unsigned MaskLen = 0;
+  if (!isShiftedMask_64(Val, MaskIdx, MaskLen))
+    return false;
+
+  // Check if low 32 bits or high 32 bits are all ones.
+  return MaskLen >= 32 && ((MaskIdx == 0) || (MaskIdx == 64 - MaskLen));
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
index 96e0b12a168a6..243474a57cfab 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.h
@@ -43,6 +43,8 @@ class AMDGPUCombinerHelper : public CombinerHelper {
   bool matchCombineFmulWithSelectToFldexp(
       MachineInstr &MI, MachineInstr &Sel,
       std::function<void(MachineIRBuilder &)> &MatchInfo) const;
+
+  bool matchConstantIs32BitMask(Register Reg) const;
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUFeatures.td b/llvm/lib/Target/AMDGPU/AMDGPUFeatures.td
index 74d1faeb6f545..d14b5ce80d28e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUFeatures.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUFeatures.td
@@ -30,6 +30,7 @@ class SubtargetFeatureAddressableLocalMemorySize <int Value> : SubtargetFeature<
 def FeatureAddressableLocalMemorySize32768 : SubtargetFeatureAddressableLocalMemorySize<32768>;
 def FeatureAddressableLocalMemorySize65536 : SubtargetFeatureAddressableLocalMemorySize<65536>;
 def FeatureAddressableLocalMemorySize163840 : SubtargetFeatureAddressableLocalMemorySize<163840>;
+def FeatureAddressableLocalMemorySize327680 : SubtargetFeatureAddressableLocalMemorySize<327680>;
 
 class SubtargetFeatureWavefrontSize <int ValueLog2> : SubtargetFeature<
   "wavefrontsize"#!shl(1, ValueLog2),
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index fb83388e5e265..b80e43b27129b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/Support/ErrorHandling.h"
 
 #ifdef EXPENSIVE_CHECKS
@@ -76,6 +77,40 @@ static bool isExtractHiElt(SDValue In, SDValue &Out) {
   return false;
 }
 
+static SDValue createVOP3PSrc32FromLo16(SDValue Lo, SDValue Src,
+                                        llvm::SelectionDAG *CurDAG,
+                                        const GCNSubtarget *Subtarget) {
+  if (!Subtarget->useRealTrue16Insts()) {
+    return Lo;
+  }
+
+  SDValue NewSrc;
+  SDLoc SL(Lo);
+
+  if (Lo->isDivergent()) {
+    SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
+                                                   SL, Lo.getValueType()),
+                            0);
+    const SDValue Ops[] = {
+        CurDAG->getTargetConstant(AMDGPU::VGPR_32RegClassID, SL, MVT::i32), Lo,
+        CurDAG->getTargetConstant(AMDGPU::lo16, SL, MVT::i16), Undef,
+        CurDAG->getTargetConstant(AMDGPU::hi16, SL, MVT::i16)};
+
+    NewSrc = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, SL,
+                                            Src.getValueType(), Ops),
+                     0);
+  } else {
+    // the S_MOV is needed since the Lo could still be a VGPR16.
+    // With S_MOV, isel insert a "sgpr32 = copy vgpr16" and we reply on
+    // the fixvgpr2sgprcopy pass to legalize it
+    NewSrc = SDValue(
+        CurDAG->getMachineNode(AMDGPU::S_MOV_B32, SL, Src.getValueType(), Lo),
+        0);
+  }
+
+  return NewSrc;
+}
+
 // Look through operations that obscure just looking at the low 16-bits of the
 // same register.
 static SDValue stripExtractLoElt(SDValue In) {
@@ -1162,18 +1197,25 @@ void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
 void AMDGPUDAGToDAGISel::SelectMUL_LOHI(SDNode *N) {
   SDLoc SL(N);
   bool Signed = N->getOpcode() == ISD::SMUL_LOHI;
+  SDVTList VTList;
   unsigned Opc;
-  if (Subtarget->hasMADIntraFwdBug())
-    Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
-                 : AMDGPU::V_MAD_U64_U32_gfx11_e64;
-  else
-    Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
+  if (Subtarget->hasMadU64U32NoCarry()) {
+    VTList = CurDAG->getVTList(MVT::i64);
+    Opc = Signed ? AMDGPU::V_MAD_NC_I64_I32_e64 : AMDGPU::V_MAD_NC_U64_U32_e64;
+  } else {
+    VTList = CurDAG->getVTList(MVT::i64, MVT::i1);
+    if (Subtarget->hasMADIntraFwdBug()) {
+      Opc = Signed ? AMDGPU::V_MAD_I64_I32_gfx11_e64
+                   : AMDGPU::V_MAD_U64_U32_gfx11_e64;
+    } else {
+      Opc = Signed ? AMDGPU::V_MAD_I64_I32_e64 : AMDGPU::V_MAD_U64_U32_e64;
+    }
+  }
 
   SDValue Zero = CurDAG->getTargetConstant(0, SL, MVT::i64);
   SDValue Clamp = CurDAG->getTargetConstant(0, SL, MVT::i1);
   SDValue Ops[] = {N->getOperand(0), N->getOperand(1), Zero, Clamp};
-  SDNode *Mad = CurDAG->getMachineNode(
-      Opc, SL, CurDAG->getVTList(MVT::i64, MVT::i1), Ops);
+  SDNode *Mad = CurDAG->getMachineNode(Opc, SL, VTList, Ops);
   if (!SDValue(N, 0).use_empty()) {
     SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, SL, MVT::i32);
     SDNode *Lo = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, SL,
@@ -3212,6 +3254,44 @@ bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
     Src = Src.getOperand(0);
   }
 
+  if (Mods != SISrcMods::NONE)
+    return true;
+
+  // Convert various sign-bit masks on integers to src mods. Currently disabled
+  // for 16-bit types as the codegen replaces the operand without adding a
+  // srcmod. This is intentionally finding the cases where we are performing
+  // float neg and abs on int types, the goal is not to obtain two's complement
+  // neg or abs. Limit converison to select operands via the nonCanonalizing
+  // pattern.
+  // TODO: Add 16-bit support.
+  if (IsCanonicalizing)
+    return true;
+
+  unsigned Opc = Src->getOpcode();
+  EVT VT = Src.getValueType();
+  if ((Opc != ISD::AND && Opc != ISD::OR && Opc != ISD::XOR) ||
+      (VT != MVT::i32 && VT != MVT::i64))
+    return true;
+
+  ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Src->getOperand(1));
+  if (!CRHS)
+    return true;
+
+  // Recognise (xor a, 0x80000000) as NEG SrcMod.
+  // Recognise (and a, 0x7fffffff) as ABS SrcMod.
+  // Recognise (or a, 0x80000000) as NEG+ABS SrcModifiers.
+  if (Opc == ISD::XOR && CRHS->getAPIntValue().isSignMask()) {
+    Mods |= SISrcMods::NEG;
+    Src = Src.getOperand(0);
+  } else if (Opc == ISD::AND && AllowAbs &&
+             CRHS->getAPIntValue().isMaxSignedValue()) {
+    Mods |= SISrcMods::ABS;
+    Src = Src.getOperand(0);
+  } else if (Opc == ISD::OR && AllowAbs && CRHS->getAPIntValue().isSignMask()) {
+    Mods |= SISrcMods::ABS | SISrcMods::NEG;
+    Src = Src.getOperand(0);
+  }
+
   return true;
 }
 
@@ -3374,8 +3454,10 @@ bool AMDGPUDAGToDAGISel::SelectVOP3PMods(SDValue In, SDValue &Src,
       // Really a scalar input. Just select from the low half of the register to
       // avoid packing.
 
-      if (VecSize == 32 || VecSize == Lo.getValueSizeInBits()) {
+      if (VecSize == Lo.getValueSizeInBits()) {
         Src = Lo;
+      } else if (VecSize == 32) {
+        Src = createVOP3PSrc32FromLo16(Lo, Src, CurDAG, Subtarget);
       } else {
         assert(Lo.getValueSizeInBits() == 32 && VecSize == 64);
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 64e68ab7d753c..c048371b11d77 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1512,9 +1512,16 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
   const GlobalValue *GV = G->getGlobal();
 
   if (!MFI->isModuleEntryFunction()) {
+    auto IsNamedBarrier = AMDGPU::isNamedBarrier(*cast<GlobalVariable>(GV));
     if (std::optional<uint32_t> Address =
             AMDGPUMachineFunction::getLDSAbsoluteAddress(*GV)) {
+      if (IsNamedBarrier) {
+        unsigned BarCnt = DL.getTypeAllocSize(GV->getValueType()) / 16;
+        MFI->recordNumNamedBarriers(Address.value(), BarCnt);
+      }
       return DAG.getConstant(*Address, SDLoc(Op), Op.getValueType());
+    } else if (IsNamedBarrier) {
+      llvm_unreachable("named barrier should have an assigned address");
     }
   }
 
@@ -1802,16 +1809,36 @@ std::pair<SDValue, SDValue>
 AMDGPUTargetLowering::splitVector(const SDValue &N, const SDLoc &DL,
                                   const EVT &LoVT, const EVT &HiVT,
                                   SelectionDAG &DAG) const {
+  EVT VT = N.getValueType();
   assert(LoVT.getVectorNumElements() +
                  (HiVT.isVector() ? HiVT.getVectorNumElements() : 1) <=
-             N.getValueType().getVectorNumElements() &&
+             VT.getVectorNumElements() &&
          "More vector elements requested than available!");
   SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, N,
                            DAG.getVectorIdxConstant(0, DL));
-  SDValue Hi = DAG.getNode(
-      HiVT.isVector() ? ISD::EXTRACT_SUBVECTOR : ISD::EXTRACT_VECTOR_ELT, DL,
-      HiVT, N, DAG.getVectorIdxConstant(LoVT.getVectorNumElements(), DL));
-  return std::pair(Lo, Hi);
+
+  unsigned LoNumElts = LoVT.getVectorNumElements();
+
+  if (HiVT.isVector()) {
+    unsigned HiNumElts = HiVT.getVectorNumElements();
+    if ((VT.getVectorNumElements() % HiNumElts) == 0) {
+      // Avoid creating an extract_subvector with an index that isn't a multiple
+      // of the result type.
+      SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, N,
+                               DAG.getConstant(LoNumElts, DL, MVT::i32));
+      return {Lo, Hi};
+    }
+
+    SmallVector<SDValue, 8> Elts;
+    DAG.ExtractVectorElements(N, Elts, /*Start=*/LoNumElts,
+                              /*Count=*/HiNumElts);
+    SDValue Hi = DAG.getBuildVector(HiVT, DL, Elts);
+    return {Lo, Hi};
+  }
+
+  SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, HiVT, N,
+                           DAG.getVectorIdxConstant(LoNumElts, DL));
+  return {Lo, Hi};
 }
 
 SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
@@ -4002,7 +4029,8 @@ SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
   case Intrinsic::amdgcn_rcp_legacy:
   case Intrinsic::amdgcn_rsq_legacy:
   case Intrinsic::amdgcn_rsq_clamp:
-  case Intrinsic::amdgcn_tanh: {
+  case Intrinsic::amdgcn_tanh:
+  case Intrinsic::amdgcn_prng_b32: {
     // FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
     SDValue Src = N->getOperand(1);
     return Src.isUndef() ? Src : SDValue();
@@ -6107,6 +6135,19 @@ unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
   }
 }
 
+bool AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode(
+    SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
+    bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const {
+  unsigned Opcode = Op.getOpcode();
+  switch (Opcode) {
+  case AMDGPUISD::BFE_I32:
+  case AMDGPUISD::BFE_U32:
+    return false;
+  }
+  return TargetLowering::canCreateUndefOrPoisonForTargetNode(
+      Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth);
+}
+
 bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
     SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
     unsigned Depth) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index fd5d5b8dec431..78394ac9cd2dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -323,6 +323,12 @@ class AMDGPUTargetLowering : public TargetLowering {
                                             const MachineRegisterInfo &MRI,
                                             unsigned Depth = 0) const override;
 
+  bool canCreateUndefOrPoisonForTargetNode(SDValue Op,
+                                           const APInt &DemandedElts,
+                                           const SelectionDAG &DAG,
+                                           bool PoisonOnly, bool ConsiderFlags,
+                                           unsigned Depth) const override;
+
   bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts,
                                     const SelectionDAG &DAG, bool SNaN = false,
                                     unsigned Depth = 0) const override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index b7fd131e76056..5d31eed8fe7d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -2368,8 +2368,10 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
   case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn:
   case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn:
     return selectDSBvhStackIntrinsic(I);
+  case Intrinsic::amdgcn_s_barrier_init:
   case Intrinsic::amdgcn_s_barrier_signal_var:
     return selectNamedBarrierInit(I, IntrinsicID);
+  case Intrinsic::amdgcn_s_barrier_join:
   case Intrinsic::amdgcn_s_get_named_barrier_state:
     return selectNamedBarrierInst(I, IntrinsicID);
   case Intrinsic::amdgcn_s_get_barrier_state:
@@ -5521,11 +5523,18 @@ AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root,
 
   Register PtrBase;
   int64_t ConstOffset;
-  std::tie(PtrBase, ConstOffset) =
+  bool IsInBounds;
+  std::tie(PtrBase, ConstOffset, IsInBounds) =
       getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
 
-  if (ConstOffset == 0 || (FlatVariant == SIInstrFlags::FlatScratch &&
-                           !isFlatScratchBaseLegal(Root.getReg())))
+  // Adding the offset to the base address with an immediate in a FLAT
+  // instruction must not change the memory aperture in which the address falls.
+  // Therefore we can only fold offsets from inbounds GEPs into FLAT
+  // instructions.
+  if (ConstOffset == 0 ||
+      (FlatVariant == SIInstrFlags::FlatScratch &&
+       !isFlatScratchBaseLegal(Root.getReg())) ||
+      (FlatVariant == SIInstrFlags::FLAT && !IsInBounds))
     return Default;
 
   unsigned AddrSpace = (*MI->memoperands_begin())->getAddrSpace();
@@ -5577,7 +5586,8 @@ AMDGPUInstructionSelector::selectGlobalSAddr(MachineOperand &Root,
 
   // Match the immediate offset first, which canonically is moved as low as
   // possible.
-  std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI);
+  std::tie(PtrBase, ConstOffset, std::ignore) =
+      getPtrBaseWithConstantOffset(Addr, *MRI);
 
   if (ConstOffset != 0) {
     if (NeedIOffset &&
@@ -5760,7 +5770,8 @@ AMDGPUInstructionSelector::selectScratchSAddr(MachineOperand &Root) const {
 
   // Match the immediate offset first, which canonically is moved as low as
   // possible.
-  std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI);
+  std::tie(PtrBase, ConstOffset, std::ignore) =
+      getPtrBaseWithConstantOffset(Addr, *MRI);
 
   if (ConstOffset != 0 && isFlatScratchBaseLegal(Addr) &&
       TII.isLegalFLATOffset(ConstOffset, AMDGPUAS::PRIVATE_ADDRESS,
@@ -5836,7 +5847,8 @@ AMDGPUInstructionSelector::selectScratchSVAddr(MachineOperand &Root) const {
 
   // Match the immediate offset first, which canonically is moved as low as
   // possible.
-  std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(Addr, *MRI);
+  std::tie(PtrBase, ConstOffset, std::ignore) =
+      getPtrBaseWithConstantOffset(Addr, *MRI);
 
   Register OrigAddr = Addr;
   if (ConstOffset != 0 &&
@@ -5942,7 +5954,8 @@ AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
   const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
   Register PtrBase;
   int64_t ConstOffset;
-  std::tie(PtrBase, ConstOffset) = getPtrBaseWithConstantOffset(VAddr, *MRI);
+  std::tie(PtrBase, ConstOffset, std::ignore) =
+      getPtrBaseWithConstantOffset(VAddr, *MRI);
   if (ConstOffset != 0) {
     if (TII.isLegalMUBUFImmOffset(ConstOffset) &&
         (!STI.privateMemoryResourceIsRangeChecked() ||
@@ -6181,8 +6194,8 @@ AMDGPUInstructionSelector::selectDS1Addr1OffsetImpl(MachineOperand &Root) const
 
   Register PtrBase;
   int64_t Offset;
-  std::tie(PtrBase, Offset) =
-    getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
+  std::tie(PtrBase, Offset, std::ignore) =
+      getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
 
   if (Offset) {
     if (isDSOffsetLegal(PtrBase, Offset)) {
@@ -6243,8 +6256,8 @@ AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
 
   Register PtrBase;
   int64_t Offset;
-  std::tie(PtrBase, Offset) =
-    getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
+  std::tie(PtrBase, Offset, std::ignore) =
+      getPtrBaseWithConstantOffset(Root.getReg(), *MRI);
 
   if (Offset) {
     int64_t OffsetValue0 = Offset;
@@ -6265,22 +6278,25 @@ AMDGPUInstructionSelector::selectDSReadWrite2Impl(MachineOperand &Root,
 }
 
 /// If \p Root is a G_PTR_ADD with a G_CONSTANT on the right hand side, return
-/// the base value with the constant offset. There may be intervening copies
-/// between \p Root and the identified constant. Returns \p Root, 0 if this does
-/// not match the pattern.
-std::pair<Register, int64_t>
+/// the base value with the constant offset, and if the offset computation is
+/// known to be inbounds. There may be intervening copies between \p Root and
+/// the identified constant. Returns \p Root, 0, false if this does not match
+/// the pattern.
+std::tuple<Register, int64_t, bool>
 AMDGPUInstructionSelector::getPtrBaseWithConstantOffset(
-  Register Root, const MachineRegisterInfo &MRI) const {
+    Register Root, const MachineRegisterInfo &MRI) const {
   MachineInstr *RootI = getDefIgnoringCopies(Root, MRI);
   if (RootI->getOpcode() != TargetOpcode::G_PTR_ADD)
-    return {Root, 0};
+    return {Root, 0, false};
 
   MachineOperand &RHS = RootI->getOperand(2);
   std::optional<ValueAndVReg> MaybeOffset =
       getIConstantVRegValWithLookThrough(RHS.getReg(), MRI);
   if (!MaybeOffset)
-    return {Root, 0};
-  return {RootI->getOperand(1).getReg(), MaybeOffset->Value.getSExtValue()};
+    return {Root, 0, false};
+  bool IsInBounds = RootI->getFlag(MachineInstr::MIFlag::InBounds);
+  return {RootI->getOperand(1).getReg(), MaybeOffset->Value.getSExtValue(),
+          IsInBounds};
 }
 
 static void addZeroImm(MachineInstrBuilder &MIB) {
@@ -6358,7 +6374,8 @@ AMDGPUInstructionSelector::parseMUBUFAddress(Register Src) const {
   Register PtrBase;
   int64_t Offset;
 
-  std::tie(PtrBase, Offset) = getPtrBaseWithConstantOffset(Src, *MRI);
+  std::tie(PtrBase, Offset, std::ignore) =
+      getPtrBaseWithConstantOffset(Src, *MRI);
   if (isUInt<32>(Offset)) {
     Data.N0 = PtrBase;
     Data.Offset = Offset;
@@ -6757,6 +6774,8 @@ unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
     switch (IntrID) {
     default:
       llvm_unreachable("not a named barrier op");
+    case Intrinsic::amdgcn_s_barrier_join:
+      return AMDGPU::S_BARRIER_JOIN_IMM;
     case Intrinsic::amdgcn_s_get_named_barrier_state:
       return AMDGPU::S_GET_BARRIER_STATE_IMM;
     };
@@ -6764,6 +6783,8 @@ unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
     switch (IntrID) {
     default:
       llvm_unreachable("not a named barrier op");
+    case Intrinsic::amdgcn_s_barrier_join:
+      return AMDGPU::S_BARRIER_JOIN_M0;
     case Intrinsic::amdgcn_s_get_named_barrier_state:
       return AMDGPU::S_GET_BARRIER_STATE_M0;
     };
@@ -6814,8 +6835,11 @@ bool AMDGPUInstructionSelector::selectNamedBarrierInit(
       BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0).addReg(TmpReg4);
   constrainSelectedInstRegOperands(*CopyMIB, TII, TRI, RBI);
 
+  unsigned Opc = IntrID == Intrinsic::amdgcn_s_barrier_init
+                     ? AMDGPU::S_BARRIER_INIT_M0
+                     : AMDGPU::S_BARRIER_SIGNAL_M0;
   MachineInstrBuilder MIB;
-  MIB = BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_M0));
+  MIB = BuildMI(*MBB, &I, DL, TII.get(Opc));
 
   I.eraseFromParent();
   return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index c9da419846ee5..092439693f399 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -156,6 +156,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
   bool selectNamedBarrierInst(MachineInstr &I, Intrinsic::ID IID) const;
   bool selectSBarrierSignalIsfirst(MachineInstr &I, Intrinsic::ID IID) const;
   bool selectSGetBarrierState(MachineInstr &I, Intrinsic::ID IID) const;
+  bool selectSBarrierLeave(MachineInstr &I) const;
 
   std::pair<Register, unsigned> selectVOP3ModsImpl(Register Src,
                                                    bool IsCanonicalizing = true,
@@ -295,7 +296,7 @@ class AMDGPUInstructionSelector final : public InstructionSelector {
   InstructionSelector::ComplexRendererFns
   selectDSReadWrite2(MachineOperand &Root, unsigned size) const;
 
-  std::pair<Register, int64_t>
+  std::tuple<Register, int64_t, bool>
   getPtrBaseWithConstantOffset(Register Root,
                                const MachineRegisterInfo &MRI) const;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index 511fc6967da31..efcd87e466207 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -34,12 +34,6 @@ class AMDGPUInst <dag outs, dag ins, string asm = "",
   let Pattern = pattern;
   let Itinerary = NullALU;
 
-  // SoftFail is a field the disassembler can use to provide a way for
-  // instructions to not match without killing the whole decode process. It is
-  // mainly used for ARM, but Tablegen expects this field to exist or it fails
-  // to build the decode table.
-  field bits<128> SoftFail = 0; // FIXME: If this is smaller than largest instruction, DecodeEmitter crashes
-
   let DecoderNamespace = Namespace;
 
   let TSFlags{63} = isRegisterLoad;
@@ -450,7 +444,9 @@ def LoadAddress_constant : AddressSpaceList<[ AddrSpaces.Constant,
 def LoadAddress_global : AddressSpaceList<[ AddrSpaces.Global,
                                             AddrSpaces.Constant,
                                             AddrSpaces.Constant32Bit ]>;
-def StoreAddress_global : AddressSpaceList<[ AddrSpaces.Global ]>;
+def StoreAddress_global : AddressSpaceList<[ AddrSpaces.Global,
+                                             AddrSpaces.Constant,
+                                             AddrSpaces.Constant32Bit ]>;
 
 def LoadAddress_flat : AddressSpaceList<[ AddrSpaces.Flat,
                                           AddrSpaces.Global,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
index 523c66c72273c..38718c43a61dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULateCodeGenPrepare.cpp
@@ -21,6 +21,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Transforms/Utils/Local.h"
@@ -545,7 +546,8 @@ class AMDGPULateCodeGenPrepareLegacy : public FunctionPass {
     AU.addRequired<TargetPassConfig>();
     AU.addRequired<AssumptionCacheTracker>();
     AU.addRequired<UniformityInfoWrapperPass>();
-    AU.setPreservesAll();
+    // Invalidates UniformityInfo
+    AU.setPreservesCFG();
   }
 
   bool runOnFunction(Function &F) override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index a6e4a63de4c62..600a13096f55d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -26,6 +26,7 @@
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
+#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
@@ -137,6 +138,14 @@ static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
   };
 }
 
+// Retrieves the scalar type that's the same size as the mem desc
+static LegalizeMutation getScalarTypeFromMemDesc(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
+    return std::make_pair(TypeIdx, LLT::scalar(MemSize));
+  };
+}
+
 // Increase the number of vector elements to reach the next legal RegClass.
 static LegalizeMutation moreElementsToNextExistingRegClass(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
@@ -384,6 +393,16 @@ static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx) {
   };
 }
 
+// If we have a truncating store or an extending load with a data size larger
+// than 32-bits and mem location is a power of 2
+static LegalityPredicate isTruncStoreToSizePowerOf2(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
+    return isWideScalarExtLoadTruncStore(TypeIdx)(Query) &&
+           isPowerOf2_64(MemSize);
+  };
+}
+
 // TODO: Should load to s16 be legal? Most loads extend to 32-bits, but we
 // handle some operations by just promoting the register during
 // selection. There are also d16 loads on GFX9+ which preserve the high bits.
@@ -1635,11 +1654,12 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
               // May need relegalization for the scalars.
               return std::pair(0, EltTy);
             })
-    .minScalar(0, S32)
-    .narrowScalarIf(isWideScalarExtLoadTruncStore(0), changeTo(0, S32))
-    .widenScalarToNextPow2(0)
-    .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
-    .lower();
+        .minScalar(0, S32)
+        .narrowScalarIf(isTruncStoreToSizePowerOf2(0),
+                        getScalarTypeFromMemDesc(0))
+        .widenScalarToNextPow2(0)
+        .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
+        .lower();
   }
 
   // FIXME: Unaligned accesses not lowered.
@@ -5653,7 +5673,7 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
   unsigned SplitSize = 32;
   if (IID == Intrinsic::amdgcn_update_dpp && (Size % 64 == 0) &&
       ST.hasDPALU_DPP() &&
-      AMDGPU::isLegalDPALU_DPPControl(MI.getOperand(4).getImm()))
+      AMDGPU::isLegalDPALU_DPPControl(ST, MI.getOperand(4).getImm()))
     SplitSize = 64;
 
   if (Size == SplitSize) {
@@ -5879,8 +5899,12 @@ AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
   const LLT S32 = LLT::scalar(32);
   MachineRegisterInfo &MRI = *B.getMRI();
 
-  std::tie(BaseReg, ImmOffset) =
-      AMDGPU::getBaseWithConstantOffset(MRI, OrigOffset);
+  // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
+  // being added, so we can only safely match a 32-bit addition with no unsigned
+  // overflow.
+  bool CheckNUW = AMDGPU::isGFX1250(ST);
+  std::tie(BaseReg, ImmOffset) = AMDGPU::getBaseWithConstantOffset(
+      MRI, OrigOffset, /*KnownBits=*/nullptr, CheckNUW);
 
   // If BaseReg is a pointer, convert it to int.
   if (MRI.getType(BaseReg).isPointer())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
index 304e91ec184f1..7dc1ec07cf0f9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerBufferFatPointers.cpp
@@ -225,6 +225,7 @@
 #include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/InstSimplifyFolder.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/Utils/Local.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/AttributeMask.h"
@@ -243,6 +244,7 @@
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/ReplaceConstant.h"
 #include "llvm/IR/ValueHandle.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/AMDGPUAddrSpace.h"
 #include "llvm/Support/Alignment.h"
@@ -599,8 +601,8 @@ bool StoreFatPtrsAsIntsAndExpandMemcpyVisitor::visitStoreInst(StoreInst &SI) {
 
   IRB.SetInsertPoint(&SI);
   Value *IntV = fatPtrsToInts(V, Ty, IntTy, V->getName());
-  for (auto *Dbg : at::getAssignmentMarkers(&SI))
-    Dbg->setValue(IntV);
+  for (auto *Dbg : at::getDVRAssignmentMarkers(&SI))
+    Dbg->setRawLocation(ValueAsMetadata::get(IntV));
 
   SI.setOperand(0, IntV);
   return true;
@@ -1361,6 +1363,7 @@ class SplitPtrStructs : public InstVisitor<SplitPtrStructs, PtrParts> {
   PtrParts visitAtomicCmpXchgInst(AtomicCmpXchgInst &AI);
   PtrParts visitGetElementPtrInst(GetElementPtrInst &GEP);
 
+  PtrParts visitPtrToAddrInst(PtrToAddrInst &PA);
   PtrParts visitPtrToIntInst(PtrToIntInst &PI);
   PtrParts visitIntToPtrInst(IntToPtrInst &IP);
   PtrParts visitAddrSpaceCastInst(AddrSpaceCastInst &I);
@@ -1954,6 +1957,21 @@ PtrParts SplitPtrStructs::visitPtrToIntInst(PtrToIntInst &PI) {
   return {nullptr, nullptr};
 }
 
+PtrParts SplitPtrStructs::visitPtrToAddrInst(PtrToAddrInst &PA) {
+  Value *Ptr = PA.getPointerOperand();
+  if (!isSplitFatPtr(Ptr->getType()))
+    return {nullptr, nullptr};
+  IRB.SetInsertPoint(&PA);
+
+  auto [Rsrc, Off] = getPtrParts(Ptr);
+  Value *Res = IRB.CreateIntCast(Off, PA.getType(), /*isSigned=*/false);
+  copyMetadata(Res, &PA);
+  Res->takeName(&PA);
+  SplitUsers.insert(&PA);
+  PA.replaceAllUsesWith(Res);
+  return {nullptr, nullptr};
+}
+
 PtrParts SplitPtrStructs::visitIntToPtrInst(IntToPtrInst &IP) {
   if (!isSplitFatPtr(IP.getType()))
     return {nullptr, nullptr};
@@ -2350,8 +2368,12 @@ static bool containsBufferFatPointers(const Function &F,
                                       BufferFatPtrToStructTypeMap *TypeMap) {
   bool HasFatPointers = false;
   for (const BasicBlock &BB : F)
-    for (const Instruction &I : BB)
+    for (const Instruction &I : BB) {
       HasFatPointers |= (I.getType() != TypeMap->remapType(I.getType()));
+      // Catch null pointer constants in loads, stores, etc.
+      for (const Value *V : I.operand_values())
+        HasFatPointers |= (V->getType() != TypeMap->remapType(V->getType()));
+    }
   return HasFatPointers;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index 2d8f259007c66..f01d5f6726822 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -190,12 +190,14 @@
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/ReplaceConstant.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -955,6 +957,7 @@ class AMDGPULowerModuleLDS {
       Module &M, LDSUsesInfoTy &LDSUsesInfo,
       VariableFunctionMap &LDSToKernelsThatNeedToAccessItIndirectly) {
     bool Changed = false;
+    const DataLayout &DL = M.getDataLayout();
     // The 1st round: give module-absolute assignments
     int NumAbsolutes = 0;
     std::vector<GlobalVariable *> OrderedGVs;
@@ -976,8 +979,11 @@ class AMDGPULowerModuleLDS {
     }
     OrderedGVs = sortByName(std::move(OrderedGVs));
     for (GlobalVariable *GV : OrderedGVs) {
-      int BarId = ++NumAbsolutes;
       unsigned BarrierScope = llvm::AMDGPU::Barrier::BARRIER_SCOPE_WORKGROUP;
+      unsigned BarId = NumAbsolutes + 1;
+      unsigned BarCnt = DL.getTypeAllocSize(GV->getValueType()) / 16;
+      NumAbsolutes += BarCnt;
+
       // 4 bits for alignment, 5 bits for the barrier num,
       // 3 bits for the barrier scope
       unsigned Offset = 0x802000u | BarrierScope << 9 | BarId << 4;
@@ -1015,12 +1021,11 @@ class AMDGPULowerModuleLDS {
         // create a new GV used only by this kernel and its function.
         auto NewGV = uniquifyGVPerKernel(M, GV, F);
         Changed |= (NewGV != GV);
-        int BarId = (NumAbsolutes + 1);
-        if (Kernel2BarId.contains(F)) {
-          BarId = (Kernel2BarId[F] + 1);
-        }
-        Kernel2BarId[F] = BarId;
         unsigned BarrierScope = llvm::AMDGPU::Barrier::BARRIER_SCOPE_WORKGROUP;
+        unsigned BarId = Kernel2BarId[F];
+        BarId += NumAbsolutes + 1;
+        unsigned BarCnt = DL.getTypeAllocSize(GV->getValueType()) / 16;
+        Kernel2BarId[F] += BarCnt;
         unsigned Offset = 0x802000u | BarrierScope << 9 | BarId << 4;
         recordLDSAbsoluteAddress(&M, NewGV, Offset);
       }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.h b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.h
index 5ddf1ca2ab06d..68b8d4e25a6cc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.h
@@ -22,9 +22,6 @@
 namespace llvm {
 class AsmPrinter;
 class MCContext;
-} // namespace llvm
-
-using namespace llvm;
 
 class AMDGPUMCInstLower {
   MCContext &Ctx;
@@ -68,4 +65,5 @@ static inline const MCExpr *lowerAddrSpaceCast(const TargetMachine &TM,
   return nullptr;
 }
 } // namespace
+} // namespace llvm
 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUMCINSTLOWER_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
index 6390853d53b56..6b3cdf57f3ad6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.cpp
@@ -39,6 +39,8 @@ MCSymbol *MCResourceInfo::getSymbol(StringRef FuncName, ResourceInfoKind RIK,
     return GOCS(".num_agpr");
   case RIK_NumSGPR:
     return GOCS(".numbered_sgpr");
+  case RIK_NumNamedBarrier:
+    return GOCS(".num_named_barrier");
   case RIK_PrivateSegSize:
     return GOCS(".private_seg_size");
   case RIK_UsesVCC:
@@ -66,6 +68,7 @@ void MCResourceInfo::assignMaxRegs(MCContext &OutContext) {
   MCSymbol *MaxVGPRSym = getMaxVGPRSymbol(OutContext);
   MCSymbol *MaxAGPRSym = getMaxAGPRSymbol(OutContext);
   MCSymbol *MaxSGPRSym = getMaxSGPRSymbol(OutContext);
+  MCSymbol *MaxNamedBarrierSym = getMaxNamedBarrierSymbol(OutContext);
 
   auto assignMaxRegSym = [&OutContext](MCSymbol *Sym, int32_t RegCount) {
     const MCExpr *MaxExpr = MCConstantExpr::create(RegCount, OutContext);
@@ -75,6 +78,7 @@ void MCResourceInfo::assignMaxRegs(MCContext &OutContext) {
   assignMaxRegSym(MaxVGPRSym, MaxVGPR);
   assignMaxRegSym(MaxAGPRSym, MaxAGPR);
   assignMaxRegSym(MaxSGPRSym, MaxSGPR);
+  assignMaxRegSym(MaxNamedBarrierSym, MaxNamedBarrier);
 }
 
 void MCResourceInfo::reset() { *this = MCResourceInfo(); }
@@ -97,6 +101,10 @@ MCSymbol *MCResourceInfo::getMaxSGPRSymbol(MCContext &OutContext) {
   return OutContext.getOrCreateSymbol("amdgpu.max_num_sgpr");
 }
 
+MCSymbol *MCResourceInfo::getMaxNamedBarrierSymbol(MCContext &OutContext) {
+  return OutContext.getOrCreateSymbol("amdgpu.max_num_named_barrier");
+}
+
 // Tries to flatten recursive call register resource gathering. Simple cycle
 // avoiding dfs to find the constants in the propagated symbols.
 // Assumes:
@@ -227,6 +235,10 @@ void MCResourceInfo::assignResourceInfoExpr(
         case RIK_NumAGPR:
           ArgExprs.push_back(flattenedCycleMax(CalleeValSym, RIK, OutContext));
           break;
+        case RIK_NumNamedBarrier:
+          ArgExprs.push_back(MCSymbolRefExpr::create(
+              getMaxNamedBarrierSymbol(OutContext), OutContext));
+          break;
         }
       }
     }
@@ -245,11 +257,13 @@ void MCResourceInfo::gatherResourceInfo(
   MCSymbol *MaxAGPRSym = getMaxAGPRSymbol(OutContext);
   MCSymbol *MaxSGPRSym = getMaxSGPRSymbol(OutContext);
   bool IsLocal = MF.getFunction().hasLocalLinkage();
+  MCSymbol *MaxNamedBarrierSym = getMaxNamedBarrierSymbol(OutContext);
 
   if (!AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv())) {
     addMaxVGPRCandidate(FRI.NumVGPR);
     addMaxAGPRCandidate(FRI.NumAGPR);
     addMaxSGPRCandidate(FRI.NumExplicitSGPR);
+    addMaxNamedBarrierCandidate(FRI.NumNamedBarrier);
   }
 
   const TargetMachine &TM = MF.getTarget();
@@ -288,6 +302,7 @@ void MCResourceInfo::gatherResourceInfo(
   SetMaxReg(MaxVGPRSym, FRI.NumVGPR, RIK_NumVGPR);
   SetMaxReg(MaxAGPRSym, FRI.NumAGPR, RIK_NumAGPR);
   SetMaxReg(MaxSGPRSym, FRI.NumExplicitSGPR, RIK_NumSGPR);
+  SetMaxReg(MaxNamedBarrierSym, FRI.NumNamedBarrier, RIK_NumNamedBarrier);
 
   {
     // The expression for private segment size should be: FRI.PrivateSegmentSize
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h
index 297e93bb54e29..b6055166698b3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMCResourceInfo.h
@@ -31,6 +31,7 @@ class MCResourceInfo {
     RIK_NumVGPR,
     RIK_NumAGPR,
     RIK_NumSGPR,
+    RIK_NumNamedBarrier,
     RIK_PrivateSegSize,
     RIK_UsesVCC,
     RIK_UsesFlatScratch,
@@ -43,6 +44,7 @@ class MCResourceInfo {
   int32_t MaxVGPR = 0;
   int32_t MaxAGPR = 0;
   int32_t MaxSGPR = 0;
+  int32_t MaxNamedBarrier = 0;
 
   // Whether the MCResourceInfo has been finalized through finalize(MCContext
   // &). Should only be called once, at the end of AsmPrinting to assign MaxXGPR
@@ -75,6 +77,9 @@ class MCResourceInfo {
   void addMaxSGPRCandidate(int32_t candidate) {
     MaxSGPR = std::max(MaxSGPR, candidate);
   }
+  void addMaxNamedBarrierCandidate(int32_t candidate) {
+    MaxNamedBarrier = std::max(MaxNamedBarrier, candidate);
+  }
 
   MCSymbol *getSymbol(StringRef FuncName, ResourceInfoKind RIK,
                       MCContext &OutContext, bool IsLocal);
@@ -90,6 +95,7 @@ class MCResourceInfo {
   MCSymbol *getMaxVGPRSymbol(MCContext &OutContext);
   MCSymbol *getMaxAGPRSymbol(MCContext &OutContext);
   MCSymbol *getMaxSGPRSymbol(MCContext &OutContext);
+  MCSymbol *getMaxNamedBarrierSymbol(MCContext &OutContext);
 
   /// AMDGPUResourceUsageAnalysis gathers resource usage on a per-function
   /// granularity. However, some resource info has to be assigned the call
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
index 0c82caced3c9b..664a15ca55f53 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
@@ -107,6 +107,8 @@ unsigned AMDGPUMachineFunction::allocateLDSGlobal(const DataLayout &DL,
       if (!BarAddr)
         llvm_unreachable("named barrier should have an assigned address");
       Entry.first->second = BarAddr.value();
+      unsigned BarCnt = DL.getTypeAllocSize(GV.getValueType()) / 16;
+      recordNumNamedBarriers(BarAddr.value(), BarCnt);
       return BarAddr.value();
     }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h
index b1022e48b8d34..fc64e16ffbeb8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h
@@ -49,6 +49,8 @@ class AMDGPUMachineFunction : public MachineFunctionInfo {
   // Flag to check dynamic LDS usage by kernel.
   bool UsesDynamicLDS = false;
 
+  uint32_t NumNamedBarriers = 0;
+
   // Kernels + shaders. i.e. functions called by the hardware and not called
   // by other functions.
   bool IsEntryFunction = false;
@@ -86,6 +88,12 @@ class AMDGPUMachineFunction : public MachineFunctionInfo {
     return GDSSize;
   }
 
+  void recordNumNamedBarriers(uint32_t GVAddr, unsigned BarCnt) {
+    NumNamedBarriers =
+        std::max(NumNamedBarriers, ((GVAddr & 0x1ff) >> 4) + BarCnt - 1);
+  }
+  uint32_t getNumNamedBarriers() const { return NumNamedBarriers; }
+
   bool isEntryFunction() const {
     return IsEntryFunction;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
index aa72c3e61f680..eeb6de512bf5e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMemoryUtils.cpp
@@ -10,7 +10,6 @@
 #include "AMDGPU.h"
 #include "Utils/AMDGPUBaseInfo.h"
 #include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/MemorySSA.h"
@@ -31,28 +30,40 @@ Align getAlign(const DataLayout &DL, const GlobalVariable *GV) {
                                        GV->getValueType());
 }
 
-TargetExtType *isNamedBarrier(const GlobalVariable &GV) {
-  // TODO: Allow arrays and structs, if all members are barriers
-  // in the same scope.
-  // TODO: Disallow other uses of target("amdgcn.named.barrier") including:
-  // - Structs containing barriers in different scope.
-  // - Structs containing a mixture of barriers and other data.
-  // - Globals in other address spaces.
-  // - Allocas.
+// Returns the target extension type of a global variable,
+// which can only be a TargetExtType, an array or single-element struct of it,
+// or their nesting combination.
+// TODO: allow struct of multiple TargetExtType elements of the same type.
+// TODO: Disallow other uses of target("amdgcn.named.barrier") including:
+// - Structs containing barriers in different scope/rank
+// - Structs containing a mixture of barriers and other data.
+// - Globals in other address spaces.
+// - Allocas.
+static TargetExtType *getTargetExtType(const GlobalVariable &GV) {
   Type *Ty = GV.getValueType();
   while (true) {
     if (auto *TTy = dyn_cast<TargetExtType>(Ty))
-      return TTy->getName() == "amdgcn.named.barrier" ? TTy : nullptr;
+      return TTy;
     if (auto *STy = dyn_cast<StructType>(Ty)) {
-      if (STy->getNumElements() == 0)
+      if (STy->getNumElements() != 1)
         return nullptr;
       Ty = STy->getElementType(0);
       continue;
     }
+    if (auto *ATy = dyn_cast<ArrayType>(Ty)) {
+      Ty = ATy->getElementType();
+      continue;
+    }
     return nullptr;
   }
 }
 
+TargetExtType *isNamedBarrier(const GlobalVariable &GV) {
+  if (TargetExtType *Ty = getTargetExtType(GV))
+    return Ty->getName() == "amdgcn.named.barrier" ? Ty : nullptr;
+  return nullptr;
+}
+
 bool isDynamicLDS(const GlobalVariable &GV) {
   // external zero size addrspace(3) without initializer is dynlds.
   const Module *M = GV.getParent();
@@ -352,7 +363,10 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA) {
     case Intrinsic::amdgcn_s_barrier_signal:
     case Intrinsic::amdgcn_s_barrier_signal_var:
     case Intrinsic::amdgcn_s_barrier_signal_isfirst:
+    case Intrinsic::amdgcn_s_barrier_init:
+    case Intrinsic::amdgcn_s_barrier_join:
     case Intrinsic::amdgcn_s_barrier_wait:
+    case Intrinsic::amdgcn_s_barrier_leave:
     case Intrinsic::amdgcn_s_get_barrier_state:
     case Intrinsic::amdgcn_wave_barrier:
     case Intrinsic::amdgcn_sched_barrier:
@@ -381,7 +395,7 @@ bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
                            AAResults *AA) {
   MemorySSAWalker *Walker = MSSA->getWalker();
   SmallVector<MemoryAccess *> WorkList{Walker->getClobberingMemoryAccess(Load)};
-  SmallSet<MemoryAccess *, 8> Visited;
+  SmallPtrSet<MemoryAccess *, 8> Visited;
   MemoryLocation Loc(MemoryLocation::get(Load));
 
   LLVM_DEBUG(dbgs() << "Checking clobbering of: " << *Load << '\n');
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index b6c6d927d0e89..6ddfa386e8ac9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -29,7 +29,6 @@ MODULE_PASS("amdgpu-preload-kernel-arguments", AMDGPUPreloadKernelArgumentsPass(
 MODULE_PASS("amdgpu-printf-runtime-binding", AMDGPUPrintfRuntimeBindingPass())
 MODULE_PASS("amdgpu-remove-incompatible-functions", AMDGPURemoveIncompatibleFunctionsPass(*this))
 MODULE_PASS("amdgpu-sw-lower-lds", AMDGPUSwLowerLDSPass(*this))
-MODULE_PASS("amdgpu-unify-metadata", AMDGPUUnifyMetadataPass())
 #undef MODULE_PASS
 
 #ifndef MODULE_PASS_WITH_PARAMS
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
index 3a3751892c8b6..f8772da6139fe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
@@ -16,7 +16,6 @@
 #include "AMDGPU.h"
 #include "AMDGPUTargetMachine.h"
 #include "Utils/AMDGPUBaseInfo.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
@@ -134,8 +133,8 @@ static std::pair<const Value *, const Type *> getMemoryInstrPtrAndType(
 
 bool AMDGPUPerfHint::isIndirectAccess(const Instruction *Inst) const {
   LLVM_DEBUG(dbgs() << "[isIndirectAccess] " << *Inst << '\n');
-  SmallSet<const Value *, 32> WorkSet;
-  SmallSet<const Value *, 32> Visited;
+  SmallPtrSet<const Value *, 32> WorkSet;
+  SmallPtrSet<const Value *, 32> Visited;
   if (const Value *MO = getMemoryInstrPtrAndType(Inst).first) {
     if (isGlobalAddr(MO))
       WorkSet.insert(MO);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.cpp
index 40094518dce0a..90c4f4e6680c2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernArgProlog.cpp
@@ -109,7 +109,7 @@ AMDGPUPreloadKernArgProlog::AMDGPUPreloadKernArgProlog(MachineFunction &MF)
       TRI(*ST.getRegisterInfo()) {}
 
 bool AMDGPUPreloadKernArgProlog::run() {
-  if (!ST.hasKernargPreload())
+  if (!ST.needsKernArgPreloadProlog())
     return false;
 
   unsigned NumKernArgPreloadSGPRs = MFI.getNumKernargPreloadedSGPRs();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernelArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernelArguments.cpp
index 984c1ee89309e..ffbbf63969427 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernelArguments.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPreloadKernelArguments.cpp
@@ -37,6 +37,11 @@ static cl::opt<unsigned> KernargPreloadCount(
     "amdgpu-kernarg-preload-count",
     cl::desc("How many kernel arguments to preload onto SGPRs"), cl::init(0));
 
+static cl::opt<bool>
+    EnableKernargPreload("amdgpu-kernarg-preload",
+                         cl::desc("Enable preload kernel arguments to SGPRs"),
+                         cl::init(true));
+
 namespace {
 
 class AMDGPUPreloadKernelArgumentsLegacy : public ModulePass {
@@ -275,6 +280,9 @@ AMDGPUPreloadKernelArgumentsLegacy::AMDGPUPreloadKernelArgumentsLegacy(
     : ModulePass(ID), TM(TM) {}
 
 static bool markKernelArgsAsInreg(Module &M, const TargetMachine &TM) {
+  if (!EnableKernargPreload)
+    return false;
+
   SmallVector<Function *, 4> FunctionsToErase;
   bool Changed = false;
   for (auto &F : M) {
@@ -333,6 +341,7 @@ static bool markKernelArgsAsInreg(Module &M, const TargetMachine &TM) {
     Changed |= NumPreloadedExplicitArgs > 0;
   }
 
+  Changed |= !FunctionsToErase.empty();
   // Erase cloned functions if we needed to update the kernel signature to
   // support preloading hidden kernel arguments.
   for (auto *F : FunctionsToErase)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
index 3b06e9b00ac69..0137b3f5943d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrepareAGPRAlloc.cpp
@@ -34,6 +34,8 @@ class AMDGPUPrepareAGPRAllocImpl {
   const SIInstrInfo &TII;
   MachineRegisterInfo &MRI;
 
+  bool isAV64Imm(const MachineOperand &MO) const;
+
 public:
   AMDGPUPrepareAGPRAllocImpl(const GCNSubtarget &ST, MachineRegisterInfo &MRI)
       : TII(*ST.getInstrInfo()), MRI(MRI) {}
@@ -85,11 +87,16 @@ AMDGPUPrepareAGPRAllocPass::run(MachineFunction &MF,
   return PreservedAnalyses::all();
 }
 
+bool AMDGPUPrepareAGPRAllocImpl::isAV64Imm(const MachineOperand &MO) const {
+  return MO.isImm() && TII.isLegalAV64PseudoImm(MO.getImm());
+}
+
 bool AMDGPUPrepareAGPRAllocImpl::run(MachineFunction &MF) {
   if (MRI.isReserved(AMDGPU::AGPR0))
     return false;
 
-  const MCInstrDesc &AVImmPseudo = TII.get(AMDGPU::AV_MOV_B32_IMM_PSEUDO);
+  const MCInstrDesc &AVImmPseudo32 = TII.get(AMDGPU::AV_MOV_B32_IMM_PSEUDO);
+  const MCInstrDesc &AVImmPseudo64 = TII.get(AMDGPU::AV_MOV_B64_IMM_PSEUDO);
 
   bool Changed = false;
   for (MachineBasicBlock &MBB : MF) {
@@ -98,8 +105,19 @@ bool AMDGPUPrepareAGPRAllocImpl::run(MachineFunction &MF) {
            TII.isInlineConstant(MI, 1)) ||
           (MI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 &&
            MI.getOperand(1).isImm())) {
-        MI.setDesc(AVImmPseudo);
+        MI.setDesc(AVImmPseudo32);
+        Changed = true;
+        continue;
+      }
+
+      // TODO: If only half of the value is rewritable, is it worth splitting it
+      // up?
+      if ((MI.getOpcode() == AMDGPU::V_MOV_B64_e64 ||
+           MI.getOpcode() == AMDGPU::V_MOV_B64_PSEUDO) &&
+          isAV64Imm(MI.getOperand(1))) {
+        MI.setDesc(AVImmPseudo64);
         Changed = true;
+        continue;
       }
     }
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 5a6ad405a0261..8c56c21621121 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -724,10 +724,10 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
       .Any({{S32}, {{}, {Vgpr32, SgprV4S32, Vgpr32, Vgpr32, Sgpr32}}});
 
   addRulesForGOpcs({G_PTR_ADD})
-      .Any({{UniP1}, {{SgprP1}, {SgprP1, Sgpr64}}})
-      .Any({{DivP1}, {{VgprP1}, {VgprP1, Vgpr64}}})
-      .Any({{DivP0}, {{VgprP0}, {VgprP0, Vgpr64}}})
-      .Any({{UniP4}, {{SgprP4}, {SgprP4, Sgpr64}}});
+      .Any({{UniPtr32}, {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
+      .Any({{DivPtr32}, {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
+      .Any({{UniPtr64}, {{SgprPtr64}, {SgprPtr64, Sgpr64}}})
+      .Any({{DivPtr64}, {{VgprPtr64}, {VgprPtr64, Vgpr64}}});
 
   addRulesForGOpcs({G_INTTOPTR})
       .Any({{UniPtr32}, {{SgprPtr32}, {Sgpr32}}})
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 868b1a21e3cd5..237929699dd9d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3342,6 +3342,10 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
       assert(OpdMapper.getVRegs(1).empty());
       constrainOpWithReadfirstlane(B, MI, 1);
       return;
+    case Intrinsic::amdgcn_s_barrier_join:
+      constrainOpWithReadfirstlane(B, MI, 1);
+      return;
+    case Intrinsic::amdgcn_s_barrier_init:
     case Intrinsic::amdgcn_s_barrier_signal_var:
       constrainOpWithReadfirstlane(B, MI, 1);
       constrainOpWithReadfirstlane(B, MI, 2);
@@ -5515,6 +5519,10 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_s_sleep_var:
       OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
       break;
+    case Intrinsic::amdgcn_s_barrier_join:
+      OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+      break;
+    case Intrinsic::amdgcn_s_barrier_init:
     case Intrinsic::amdgcn_s_barrier_signal_var:
       OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
       OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURemoveIncompatibleFunctions.cpp b/llvm/lib/Target/AMDGPU/AMDGPURemoveIncompatibleFunctions.cpp
index e2e5c57397d02..d2ec7dd429d8b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURemoveIncompatibleFunctions.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURemoveIncompatibleFunctions.cpp
@@ -195,13 +195,17 @@ bool AMDGPURemoveIncompatibleFunctions::checkFunction(Function &F) {
 
   // Delete FeatureWavefrontSize32 functions for
   // gfx9 and below targets that don't support the mode.
-  // gfx10+ is implied to support both wave32 and 64 features.
+  // gfx10, gfx11, gfx12 are implied to support both wave32 and 64 features.
   // They are not in the feature set. So, we need a separate check
-  if (ST->getGeneration() < AMDGPUSubtarget::GFX10 &&
-      ST->hasFeature(AMDGPU::FeatureWavefrontSize32)) {
+  if (!ST->supportsWave32() && ST->hasFeature(AMDGPU::FeatureWavefrontSize32)) {
     reportFunctionRemoved(F, AMDGPU::FeatureWavefrontSize32);
     return true;
   }
+  // gfx125x only support FeatureWavefrontSize32.
+  if (!ST->supportsWave64() && ST->hasFeature(AMDGPU::FeatureWavefrontSize64)) {
+    reportFunctionRemoved(F, AMDGPU::FeatureWavefrontSize64);
+    return true;
+  }
   return false;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
index 8101c68986241..0ea9add891111 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.cpp
@@ -142,6 +142,8 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
                          MRI.isLiveIn(MFI->getPreloadedReg(
                              AMDGPUFunctionArgInfo::FLAT_SCRATCH_INIT));
 
+  Info.NumNamedBarrier = MFI->getNumNamedBarriers();
+
   // Even if FLAT_SCRATCH is implicitly used, it has no effect if flat
   // instructions aren't used to access the scratch buffer. Inline assembly may
   // need it though.
@@ -241,6 +243,9 @@ AMDGPUResourceUsageAnalysisImpl::analyzeResourceUsage(
         if (!RC || !TRI.isVGPRClass(RC))
           continue;
 
+        if (MI.isCall() || MI.isMetaInstruction())
+          continue;
+
         unsigned Width = divideCeil(TRI.getRegSizeInBits(*RC), 32);
         unsigned HWReg = TRI.getHWRegIndex(Reg);
         int MaxUsed = HWReg + Width - 1;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.h b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.h
index acfff960d3f58..9ae3bb3ca048e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUResourceUsageAnalysis.h
@@ -35,6 +35,7 @@ struct AMDGPUResourceUsageAnalysisImpl {
     int32_t NumVGPR = 0;
     int32_t NumAGPR = 0;
     int32_t NumExplicitSGPR = 0;
+    int32_t NumNamedBarrier = 0;
     uint64_t CalleeSegmentSize = 0;
     uint64_t PrivateSegmentSize = 0;
     bool UsesVCC = false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
index f580f4368110f..8b1d4ba68a444 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteAGPRCopyMFMA.cpp
@@ -14,13 +14,6 @@
 /// MFMA opcode.
 ///
 /// TODO:
-///  - Handle non-tied dst+src2 cases. We need to try to find a copy from an
-///    AGPR from src2, or reassign src2 to an available AGPR (which should work
-///    in the common case of a load).
-///
-///  - Handle multiple MFMA uses of the same register. e.g. chained MFMAs that
-///    can be rewritten as a set
-///
 ///  - Update LiveIntervals incrementally instead of recomputing from scratch
 ///
 //===----------------------------------------------------------------------===//
@@ -42,6 +35,7 @@ using namespace llvm;
 namespace {
 
 class AMDGPURewriteAGPRCopyMFMAImpl {
+  MachineFunction &MF;
   const GCNSubtarget &ST;
   const SIInstrInfo &TII;
   const SIRegisterInfo &TRI;
@@ -49,44 +43,167 @@ class AMDGPURewriteAGPRCopyMFMAImpl {
   VirtRegMap &VRM;
   LiveRegMatrix &LRM;
   LiveIntervals &LIS;
+  const RegisterClassInfo &RegClassInfo;
+
+  bool attemptReassignmentsToAGPR(SmallSetVector<Register, 4> &InterferingRegs,
+                                  MCPhysReg PrefPhysReg) const;
 
 public:
   AMDGPURewriteAGPRCopyMFMAImpl(MachineFunction &MF, VirtRegMap &VRM,
-                                LiveRegMatrix &LRM, LiveIntervals &LIS)
-      : ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
+                                LiveRegMatrix &LRM, LiveIntervals &LIS,
+                                const RegisterClassInfo &RegClassInfo)
+      : MF(MF), ST(MF.getSubtarget<GCNSubtarget>()), TII(*ST.getInstrInfo()),
         TRI(*ST.getRegisterInfo()), MRI(MF.getRegInfo()), VRM(VRM), LRM(LRM),
-        LIS(LIS) {}
+        LIS(LIS), RegClassInfo(RegClassInfo) {}
+
+  bool isRewriteCandidate(const MachineInstr &MI) const {
+    return TII.isMAI(MI) && AMDGPU::getMFMASrcCVDstAGPROp(MI.getOpcode()) != -1;
+  }
 
   /// Compute the register class constraints based on the uses of \p Reg,
-  /// excluding uses from \p ExceptMI. This should be nearly identical to
+  /// excluding MFMA uses from which can be rewritten to change the register
+  /// class constraint. This should be nearly identical to
   /// MachineRegisterInfo::recomputeRegClass.
-  const TargetRegisterClass *
-  recomputeRegClassExcept(Register Reg, const TargetRegisterClass *OldRC,
-                          const TargetRegisterClass *NewRC,
-                          const MachineInstr *ExceptMI) const;
+
+  /// \p RewriteCandidates will collect the set of MFMA instructions that need
+  /// to have the opcode mutated to perform the replacement.
+  ///
+  /// \p RewriteRegs will accumulate the set of register used by those MFMAs
+  /// that need to have the register classes adjusted.
+  bool recomputeRegClassExceptRewritable(
+      Register Reg, SmallVectorImpl<MachineInstr *> &RewriteCandidates,
+      SmallSetVector<Register, 4> &RewriteRegs) const;
 
   bool run(MachineFunction &MF) const;
 };
 
-const TargetRegisterClass *
-AMDGPURewriteAGPRCopyMFMAImpl::recomputeRegClassExcept(
-    Register Reg, const TargetRegisterClass *OldRC,
-    const TargetRegisterClass *NewRC, const MachineInstr *ExceptMI) const {
+bool AMDGPURewriteAGPRCopyMFMAImpl::recomputeRegClassExceptRewritable(
+    Register StartReg, SmallVectorImpl<MachineInstr *> &RewriteCandidates,
+    SmallSetVector<Register, 4> &RewriteRegs) const {
+  SmallVector<Register, 8> Worklist = {StartReg};
+
+  // Recursively visit all transitive MFMA users
+  while (!Worklist.empty()) {
+    Register Reg = Worklist.pop_back_val();
+    const TargetRegisterClass *OldRC = MRI.getRegClass(Reg);
+
+    // Inflate to the equivalent AV_* class.
+    const TargetRegisterClass *NewRC = TRI.getLargestLegalSuperClass(OldRC, MF);
+    if (OldRC == NewRC)
+      return false;
+
+    // Accumulate constraints from all uses.
+    for (MachineOperand &MO : MRI.reg_nodbg_operands(Reg)) {
+      // Apply the effect of the given operand to NewRC.
+      MachineInstr *MI = MO.getParent();
+
+      // We can swap the classes of dst + src2 as a pair to AGPR, so ignore the
+      // effects of rewrite candidates. It just so happens that we can use
+      // either AGPR or VGPR in src0/src1, so don't bother checking the
+      // constraint effects of the individual operands.
+      if (isRewriteCandidate(*MI)) {
+        const MachineOperand *VDst =
+            TII.getNamedOperand(*MI, AMDGPU::OpName::vdst);
+        const MachineOperand *Src2 =
+            TII.getNamedOperand(*MI, AMDGPU::OpName::src2);
+        for (const MachineOperand *Op : {VDst, Src2}) {
+          if (!Op->isReg())
+            continue;
+
+          Register OtherReg = Op->getReg();
+          if (OtherReg.isPhysical())
+            return false;
+
+          if (OtherReg != Reg && RewriteRegs.insert(OtherReg))
+            Worklist.push_back(OtherReg);
+        }
+
+        if (!is_contained(RewriteCandidates, MI)) {
+          LLVM_DEBUG({
+            Register VDstPhysReg = VRM.getPhys(VDst->getReg());
+            dbgs() << "Attempting to replace VGPR MFMA with AGPR version:"
+                   << " Dst=[" << printReg(VDst->getReg()) << " => "
+                   << printReg(VDstPhysReg, &TRI);
+
+            if (Src2->isReg()) {
+              Register Src2PhysReg = VRM.getPhys(Src2->getReg());
+              dbgs() << "], Src2=[" << printReg(Src2->getReg(), &TRI) << " => "
+                     << printReg(Src2PhysReg, &TRI);
+            }
+
+            dbgs() << "]: " << MI;
+          });
+
+          RewriteCandidates.push_back(MI);
+        }
 
-  // Accumulate constraints from all uses.
-  for (MachineOperand &MO : MRI.reg_nodbg_operands(Reg)) {
-    // Apply the effect of the given operand to NewRC.
-    MachineInstr *MI = MO.getParent();
-    if (MI == ExceptMI)
-      continue;
+        continue;
+      }
 
-    unsigned OpNo = &MO - &MI->getOperand(0);
-    NewRC = MI->getRegClassConstraintEffect(OpNo, NewRC, &TII, &TRI);
-    if (!NewRC || NewRC == OldRC)
-      return nullptr;
+      unsigned OpNo = &MO - &MI->getOperand(0);
+      NewRC = MI->getRegClassConstraintEffect(OpNo, NewRC, &TII, &TRI);
+      if (!NewRC || NewRC == OldRC) {
+        LLVM_DEBUG(dbgs() << "User of " << printReg(Reg, &TRI)
+                          << " cannot be reassigned to "
+                          << TRI.getRegClassName(NewRC) << ": " << *MI);
+        return false;
+      }
+    }
   }
 
-  return NewRC;
+  return true;
+}
+
+/// Attempt to reassign the registers in \p InterferingRegs to be AGPRs, with a
+/// preference to use \p PhysReg first. Returns false if the reassignments
+/// cannot be trivially performed.
+bool AMDGPURewriteAGPRCopyMFMAImpl::attemptReassignmentsToAGPR(
+    SmallSetVector<Register, 4> &InterferingRegs, MCPhysReg PrefPhysReg) const {
+  // FIXME: The ordering may matter here, but we're just taking uselistorder
+  // with the special case of ensuring to process the starting instruction
+  // first. We probably should extract the priority advisor out of greedy and
+  // use that ordering.
+  for (Register InterferingReg : InterferingRegs) {
+    LiveInterval &ReassignLI = LIS.getInterval(InterferingReg);
+    const TargetRegisterClass *EquivalentAGPRRegClass =
+        TRI.getEquivalentAGPRClass(MRI.getRegClass(InterferingReg));
+
+    MCPhysReg Assignable = AMDGPU::NoRegister;
+    if (EquivalentAGPRRegClass->contains(PrefPhysReg) &&
+        LRM.checkInterference(ReassignLI, PrefPhysReg) ==
+            LiveRegMatrix::IK_Free) {
+      // First try to assign to the AGPR we were already copying to. This
+      // should be the first assignment we attempt. We have to guard
+      // against the use being a subregister (which doesn't have an exact
+      // class match).
+
+      // TODO: If this does happen to be a subregister use, we should
+      // still try to assign to a subregister of the original copy result.
+      Assignable = PrefPhysReg;
+    } else {
+      ArrayRef<MCPhysReg> AllocOrder =
+          RegClassInfo.getOrder(EquivalentAGPRRegClass);
+      for (MCPhysReg Reg : AllocOrder) {
+        if (LRM.checkInterference(ReassignLI, Reg) == LiveRegMatrix::IK_Free) {
+          Assignable = Reg;
+          break;
+        }
+      }
+    }
+
+    if (!Assignable) {
+      LLVM_DEBUG(dbgs() << "Unable to reassign VGPR "
+                        << printReg(InterferingReg, &TRI)
+                        << " to a free AGPR\n");
+      return false;
+    }
+
+    LLVM_DEBUG(dbgs() << "Reassigning VGPR " << printReg(InterferingReg, &TRI)
+                      << " to " << printReg(Assignable, &TRI) << '\n');
+    LRM.assign(ReassignLI, Assignable);
+  }
+
+  return true;
 }
 
 bool AMDGPURewriteAGPRCopyMFMAImpl::run(MachineFunction &MF) const {
@@ -96,8 +213,10 @@ bool AMDGPURewriteAGPRCopyMFMAImpl::run(MachineFunction &MF) const {
     return false;
 
   // Early exit if no AGPRs were assigned.
-  if (!LRM.isPhysRegUsed(AMDGPU::AGPR0))
+  if (!LRM.isPhysRegUsed(AMDGPU::AGPR0)) {
+    LLVM_DEBUG(dbgs() << "skipping function that did not allocate AGPRs\n");
     return false;
+  }
 
   bool MadeChange = false;
 
@@ -109,73 +228,46 @@ bool AMDGPURewriteAGPRCopyMFMAImpl::run(MachineFunction &MF) const {
 
     // Find AV_* registers assigned to AGPRs.
     const TargetRegisterClass *VirtRegRC = MRI.getRegClass(VReg);
-    if (!TRI.isVectorSuperClass(VirtRegRC))
+    if (!TRI.hasAGPRs(VirtRegRC))
       continue;
 
-    const TargetRegisterClass *AssignedRC = TRI.getPhysRegBaseClass(PhysReg);
-    if (!TRI.isAGPRClass(AssignedRC))
-      continue;
+    const TargetRegisterClass *AssignedRC = VirtRegRC;
+    if (TRI.hasVGPRs(VirtRegRC)) {
+      // If this is an AV register, we have to check if the actual assignment is
+      // to an AGPR
+      AssignedRC = TRI.getPhysRegBaseClass(PhysReg);
+      if (!TRI.isAGPRClass(AssignedRC))
+        continue;
+    }
 
     LiveInterval &LI = LIS.getInterval(VReg);
 
-    // TODO: Test multiple uses
     for (VNInfo *VNI : LI.vnis()) {
-      MachineInstr *DefMI = LIS.getInstructionFromIndex(VNI->def);
-
-      // TODO: Handle SplitKit produced copy bundles for partially defined
-      // registers.
-      if (!DefMI || !DefMI->isFullCopy())
+      if (VNI->isPHIDef() || VNI->isUnused())
         continue;
 
-      Register CopySrcReg = DefMI->getOperand(1).getReg();
-      if (!CopySrcReg.isVirtual())
+      MachineInstr *DefMI = LIS.getInstructionFromIndex(VNI->def);
+      if (!DefMI || !DefMI->isCopy())
         continue;
 
-      LiveInterval &CopySrcLI = LIS.getInterval(CopySrcReg);
-      LiveQueryResult LRQ = CopySrcLI.Query(VNI->def.getRegSlot());
-      MachineInstr *CopySrcMI = LIS.getInstructionFromIndex(LRQ.valueIn()->def);
-      if (!CopySrcMI)
+      Register MFMADstReg = DefMI->getOperand(1).getReg();
+      if (!MFMADstReg.isVirtual())
         continue;
 
-      int AGPROp = AMDGPU::getMFMASrcCVDstAGPROp(CopySrcMI->getOpcode());
-      if (AGPROp == -1)
+      LiveInterval &CopySrcLI = LIS.getInterval(MFMADstReg);
+      LiveQueryResult LRQ = CopySrcLI.Query(VNI->def.getRegSlot());
+      MachineInstr *MFMA = LIS.getInstructionFromIndex(LRQ.valueIn()->def);
+      if (!MFMA || !isRewriteCandidate(*MFMA))
         continue;
 
-      MachineOperand *Src2 =
-          TII.getNamedOperand(*CopySrcMI, AMDGPU::OpName::src2);
-
-      // FIXME: getMinimalPhysRegClass returns a nonsense AV_* subclass instead
-      // of an AGPR or VGPR subclass, so we can't simply use the result on the
-      // assignment.
-
-      LLVM_DEBUG({
-        Register Src2PhysReg = VRM.getPhys(Src2->getReg());
-        dbgs() << "Attempting to replace VGPR MFMA with AGPR version:"
-               << " Dst=[" << printReg(VReg) << " => "
-               << printReg(PhysReg, &TRI) << "], Src2=["
-               << printReg(Src2->getReg(), &TRI) << " => "
-               << printReg(Src2PhysReg, &TRI) << "]: " << *CopySrcMI;
-      });
-
-      // If the inputs are tied and the same register, we can shortcut and
-      // directly replace the register.
-      if (!Src2->isReg() || Src2->getReg() != CopySrcReg ||
-          Src2->getSubReg() != DefMI->getOperand(1).getSubReg()) {
-        LLVM_DEBUG(
-            dbgs()
-            << "Replacing untied VGPR MFMAs with AGPR form not yet handled\n");
-        // TODO: Only handles the tied case for now. If the input operand is a
-        // different register, we need to also reassign it (either by looking
-        // for a compatible copy-from-AGPR, or by seeing if an available AGPR is
-        // compatible with all other uses.
-
-        // If we can't reassign it, we'd need to introduce a different copy
-        // which is likely worse than the copy we'd be saving.
-        continue;
-      }
+      // src2 and dst have the same physical class constraint; try to preserve
+      // the original src2 subclass if one were to exist.
+      SmallVector<MachineInstr *, 4> RewriteCandidates = {MFMA};
+      SmallSetVector<Register, 4> RewriteRegs;
 
-      const TargetRegisterClass *Src2VirtRegRC =
-          MRI.getRegClass(Src2->getReg());
+      // Make sure we reassign the MFMA we found the copy from first. We want
+      // to ensure dst ends up in the physreg we were originally copying to.
+      RewriteRegs.insert(MFMADstReg);
 
       // We've found av = COPY (MFMA), and need to verify that we can trivially
       // rewrite src2 to use the new AGPR. If we can't trivially replace it,
@@ -183,49 +275,69 @@ bool AMDGPURewriteAGPRCopyMFMAImpl::run(MachineFunction &MF) const {
       // first place, as well as need to assign another register, and need to
       // figure out where to put them. The live range splitting is smarter than
       // anything we're doing here, so trust it did something reasonable.
-      const TargetRegisterClass *Src2ExceptRC = recomputeRegClassExcept(
-          Src2->getReg(), Src2VirtRegRC, VirtRegRC, CopySrcMI);
-      if (!Src2ExceptRC)
-        continue;
-
-      const TargetRegisterClass *NewSrc2ConstraintRC =
-          TII.getRegClass(TII.get(AGPROp), Src2->getOperandNo(), &TRI, MF);
-
-      // Try to constrain src2 to the replacement instruction candidate's
-      // register class.
-      const TargetRegisterClass *NewSrc2RC =
-          TRI.getCommonSubClass(Src2ExceptRC, NewSrc2ConstraintRC);
-      if (!NewSrc2RC) {
-        // TODO: This is ignoring ther rewritable uses. e.g. a rewritable MFMA
-        // using a rewritable MFMA can be rewritten as a pair.
-        LLVM_DEBUG(dbgs() << "Other uses of " << printReg(Src2->getReg(), &TRI)
-                          << " are incompatible with replacement class\n");
+      //
+      // Note recomputeRegClassExceptRewritable will consider the constraints of
+      // this MFMA's src2 as well as the src2/dst of any transitive MFMA users.
+      if (!recomputeRegClassExceptRewritable(MFMADstReg, RewriteCandidates,
+                                             RewriteRegs)) {
+        LLVM_DEBUG(dbgs() << "Could not recompute the regclass of dst reg "
+                          << printReg(MFMADstReg, &TRI) << '\n');
         continue;
       }
 
-      MRI.setRegClass(VReg, AssignedRC);
-      MRI.setRegClass(Src2->getReg(), NewSrc2RC);
-
-      CopySrcMI->setDesc(TII.get(AGPROp));
-
-      // TODO: Is replacing too aggressive, fixup these instructions only?
-      MRI.replaceRegWith(CopySrcReg, VReg);
+      // If src2 and dst are different registers, we need to also reassign the
+      // input to an available AGPR if it is compatible with all other uses.
+      //
+      // If we can't reassign it, we'd need to introduce a different copy
+      // which is likely worse than the copy we'd be saving.
+      //
+      // It's likely that the MFMA is used in sequence with other MFMAs; if we
+      // cannot migrate the full use/def chain of MFMAs, we would need to
+      // introduce intermediate copies somewhere. So we only make the
+      // transform if all the interfering MFMAs can also be migrated. Collect
+      // the set of rewritable MFMAs and check if we can assign an AGPR at
+      // that point.
+      //
+      // If any of the MFMAs aren't reassignable, we give up and rollback to
+      // the original register assignments.
+
+      using RecoloringStack =
+          SmallVector<std::pair<const LiveInterval *, MCRegister>, 8>;
+      RecoloringStack TentativeReassignments;
+
+      for (Register RewriteReg : RewriteRegs) {
+        LiveInterval &LI = LIS.getInterval(RewriteReg);
+        TentativeReassignments.push_back({&LI, VRM.getPhys(RewriteReg)});
+        LRM.unassign(LI);
+      }
 
-      LLVM_DEBUG(dbgs() << "Replaced VGPR MFMA with AGPR: " << *CopySrcMI);
+      if (!attemptReassignmentsToAGPR(RewriteRegs, PhysReg)) {
+        // Roll back the register assignments to the original state.
+        for (auto [LI, OldAssign] : TentativeReassignments) {
+          if (VRM.hasPhys(LI->reg()))
+            LRM.unassign(*LI);
+          LRM.assign(*LI, OldAssign);
+        }
 
-      // We left behind an identity copy, so delete it.
-      LIS.RemoveMachineInstrFromMaps(*DefMI);
-      DefMI->eraseFromParent();
+        continue;
+      }
 
-      LRM.unassign(CopySrcLI);
+      // Fixup the register classes of the virtual registers now that we've
+      // committed to the reassignments.
+      for (Register InterferingReg : RewriteRegs) {
+        const TargetRegisterClass *EquivalentAGPRRegClass =
+            TRI.getEquivalentAGPRClass(MRI.getRegClass(InterferingReg));
+        MRI.setRegClass(InterferingReg, EquivalentAGPRRegClass);
+      }
 
-      // We don't need the liveness information anymore, so don't bother
-      // updating the intervals. Just delete the stale information.
-      // TODO: Is it worth preserving these?
-      LIS.removeInterval(CopySrcReg);
-      LIS.removeInterval(VReg);
-      LIS.createAndComputeVirtRegInterval(VReg);
+      for (MachineInstr *RewriteCandidate : RewriteCandidates) {
+        int NewMFMAOp =
+            AMDGPU::getMFMASrcCVDstAGPROp(RewriteCandidate->getOpcode());
+        RewriteCandidate->setDesc(TII.get(NewMFMAOp));
+      }
 
+      // We likely left an identity copy behind after assignment; let
+      // VirtRegRewriter deal with it later.
       MadeChange = true;
     }
   }
@@ -236,6 +348,7 @@ bool AMDGPURewriteAGPRCopyMFMAImpl::run(MachineFunction &MF) const {
 class AMDGPURewriteAGPRCopyMFMALegacy : public MachineFunctionPass {
 public:
   static char ID;
+  RegisterClassInfo RegClassInfo;
 
   AMDGPURewriteAGPRCopyMFMALegacy() : MachineFunctionPass(ID) {
     initializeAMDGPURewriteAGPRCopyMFMALegacyPass(
@@ -281,11 +394,13 @@ bool AMDGPURewriteAGPRCopyMFMALegacy::runOnMachineFunction(
   if (skipFunction(MF.getFunction()))
     return false;
 
+  RegClassInfo.runOnMachineFunction(MF);
+
   auto &VRM = getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
   auto &LRM = getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
   auto &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
 
-  AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS);
+  AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS, RegClassInfo);
   return Impl.run(MF);
 }
 
@@ -295,8 +410,10 @@ AMDGPURewriteAGPRCopyMFMAPass::run(MachineFunction &MF,
   VirtRegMap &VRM = MFAM.getResult<VirtRegMapAnalysis>(MF);
   LiveRegMatrix &LRM = MFAM.getResult<LiveRegMatrixAnalysis>(MF);
   LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+  RegisterClassInfo RegClassInfo;
+  RegClassInfo.runOnMachineFunction(MF);
 
-  AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS);
+  AMDGPURewriteAGPRCopyMFMAImpl Impl(MF, VRM, LRM, LIS, RegClassInfo);
   if (!Impl.run(MF))
     return PreservedAnalyses::all();
   auto PA = getMachineFunctionPassPreservedAnalyses();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 10b8606816243..7be18998ff6b0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -378,6 +378,7 @@ foreach intr = AMDGPUImageDimAtomicIntrinsics in
 def : SourceOfDivergence<intr>;
 
 def : SourceOfDivergence<int_amdgcn_dead>;
+def : SourceOfDivergence<int_amdgcn_call_whole_wave>;
 
 class AlwaysUniform<Intrinsic intr> {
   Intrinsic Intr = intr;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp
index b60ded33a4ac3..56aa3f6db83ad 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSetWavePriority.cpp
@@ -195,7 +195,7 @@ bool AMDGPUSetWavePriority::run(MachineFunction &MF) {
 
   // Lower the priority on edges where control leaves blocks from which
   // the VMEM loads are reachable.
-  SmallSet<MachineBasicBlock *, 16> PriorityLoweringBlocks;
+  SmallPtrSet<MachineBasicBlock *, 16> PriorityLoweringBlocks;
   for (MachineBasicBlock &MBB : MF) {
     if (MBBInfos[&MBB].MayReachVMEMLoad) {
       if (MBB.succ_empty())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index d095fc6cf9549..a1d82128423bb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -209,19 +209,16 @@ std::pair<unsigned, unsigned>
 AMDGPUSubtarget::getWavesPerEU(const Function &F) const {
   // Default/requested minimum/maximum flat work group sizes.
   std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
-  // Minimum number of bytes allocated in the LDS.
-  unsigned LDSBytes = AMDGPU::getIntegerPairAttribute(F, "amdgpu-lds-size",
-                                                      {0, UINT32_MAX}, true)
-                          .first;
-  return getWavesPerEU(FlatWorkGroupSizes, LDSBytes, F);
+  return getWavesPerEU(F, FlatWorkGroupSizes);
 }
 
 std::pair<unsigned, unsigned> AMDGPUSubtarget::getWavesPerEU(
     const Function &F, std::pair<unsigned, unsigned> FlatWorkGroupSizes) const {
   // Minimum number of bytes allocated in the LDS.
-  unsigned LDSBytes = AMDGPU::getIntegerPairAttribute(F, "amdgpu-lds-size",
-                                                      {0, UINT32_MAX}, true)
-                          .first;
+  unsigned LDSBytes =
+      AMDGPU::getIntegerPairAttribute(F, "amdgpu-lds-size", {0, UINT32_MAX},
+                                      /*OnlyFirstRequired=*/true)
+          .first;
   return getWavesPerEU(FlatWorkGroupSizes, LDSBytes, F);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index c1f17033d04a8..e969f9ec88899 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -90,6 +90,7 @@
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/MC/TargetRegistry.h"
+#include "llvm/Passes/CodeGenPassBuilder.h"
 #include "llvm/Passes/PassBuilder.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -125,6 +126,44 @@ using namespace llvm;
 using namespace llvm::PatternMatch;
 
 namespace {
+//===----------------------------------------------------------------------===//
+// AMDGPU CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+class AMDGPUCodeGenPassBuilder
+    : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {
+  using Base = CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine>;
+
+public:
+  AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
+                           const CGPassBuilderOption &Opts,
+                           PassInstrumentationCallbacks *PIC);
+
+  void addIRPasses(AddIRPass &) const;
+  void addCodeGenPrepare(AddIRPass &) const;
+  void addPreISel(AddIRPass &addPass) const;
+  void addILPOpts(AddMachinePass &) const;
+  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
+  Error addInstSelector(AddMachinePass &) const;
+  void addPreRewrite(AddMachinePass &) const;
+  void addMachineSSAOptimization(AddMachinePass &) const;
+  void addPostRegAlloc(AddMachinePass &) const;
+  void addPreEmitPass(AddMachinePass &) const;
+  void addPreEmitRegAlloc(AddMachinePass &) const;
+  Error addRegAssignmentOptimized(AddMachinePass &) const;
+  void addPreRegAlloc(AddMachinePass &) const;
+  void addOptimizedRegAlloc(AddMachinePass &) const;
+  void addPreSched2(AddMachinePass &) const;
+
+  /// Check if a pass is enabled given \p Opt option. The option always
+  /// overrides defaults if explicitly used. Otherwise its default will be used
+  /// given that a pass shall work at an optimization \p Level minimum.
+  bool isPassEnabled(const cl::opt<bool> &Opt,
+                     CodeGenOptLevel Level = CodeGenOptLevel::Default) const;
+  void addEarlyCSEOrGVNPass(AddIRPass &) const;
+  void addStraightLineScalarOptimizationPasses(AddIRPass &) const;
+};
+
 class SGPRRegisterRegAlloc : public RegisterRegAllocBase<SGPRRegisterRegAlloc> {
 public:
   SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
@@ -848,8 +887,6 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
         if (Level == OptimizationLevel::O0)
           return;
 
-        PM.addPass(AMDGPUUnifyMetadataPass());
-
         // We don't want to run internalization at per-module stage.
         if (InternalizeSymbols && !isLTOPreLink(Phase)) {
           PM.addPass(InternalizePass(mustPreserveGV));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index e0f1296ddded8..06a3047196b8a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -18,7 +18,6 @@
 #include "llvm/CodeGen/CodeGenTargetMachineImpl.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/MC/MCStreamer.h"
-#include "llvm/Passes/CodeGenPassBuilder.h"
 #include <optional>
 #include <utility>
 
@@ -158,44 +157,6 @@ class AMDGPUPassConfig : public TargetPassConfig {
   }
 };
 
-//===----------------------------------------------------------------------===//
-// AMDGPU CodeGen Pass Builder interface.
-//===----------------------------------------------------------------------===//
-
-class AMDGPUCodeGenPassBuilder
-    : public CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine> {
-  using Base = CodeGenPassBuilder<AMDGPUCodeGenPassBuilder, GCNTargetMachine>;
-
-public:
-  AMDGPUCodeGenPassBuilder(GCNTargetMachine &TM,
-                           const CGPassBuilderOption &Opts,
-                           PassInstrumentationCallbacks *PIC);
-
-  void addIRPasses(AddIRPass &) const;
-  void addCodeGenPrepare(AddIRPass &) const;
-  void addPreISel(AddIRPass &addPass) const;
-  void addILPOpts(AddMachinePass &) const;
-  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
-  Error addInstSelector(AddMachinePass &) const;
-  void addPreRewrite(AddMachinePass &) const;
-  void addMachineSSAOptimization(AddMachinePass &) const;
-  void addPostRegAlloc(AddMachinePass &) const;
-  void addPreEmitPass(AddMachinePass &) const;
-  void addPreEmitRegAlloc(AddMachinePass &) const;
-  Error addRegAssignmentOptimized(AddMachinePass &) const;
-  void addPreRegAlloc(AddMachinePass &) const;
-  void addOptimizedRegAlloc(AddMachinePass &) const;
-  void addPreSched2(AddMachinePass &) const;
-
-  /// Check if a pass is enabled given \p Opt option. The option always
-  /// overrides defaults if explicitly used. Otherwise its default will be used
-  /// given that a pass shall work at an optimization \p Level minimum.
-  bool isPassEnabled(const cl::opt<bool> &Opt,
-                     CodeGenOptLevel Level = CodeGenOptLevel::Default) const;
-  void addEarlyCSEOrGVNPass(AddIRPass &) const;
-  void addStraightLineScalarOptimizationPasses(AddIRPass &) const;
-};
-
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETMACHINE_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index a0c99b0ef0491..846a0b6280f19 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -991,10 +991,21 @@ bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
     return true;
 
   if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {
-    if (Intrinsic->getIntrinsicID() == Intrinsic::read_register)
+    Intrinsic::ID IID = Intrinsic->getIntrinsicID();
+    switch (IID) {
+    case Intrinsic::read_register:
       return isReadRegisterSourceOfDivergence(Intrinsic);
-
-    return AMDGPU::isIntrinsicSourceOfDivergence(Intrinsic->getIntrinsicID());
+    case Intrinsic::amdgcn_addrspacecast_nonnull: {
+      unsigned SrcAS =
+          Intrinsic->getOperand(0)->getType()->getPointerAddressSpace();
+      unsigned DstAS = Intrinsic->getType()->getPointerAddressSpace();
+      return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
+             DstAS == AMDGPUAS::FLAT_ADDRESS &&
+             ST->hasGloballyAddressableScratch();
+    }
+    default:
+      return AMDGPU::isIntrinsicSourceOfDivergence(IID);
+    }
   }
 
   // Assume all function calls are a source of divergence.
@@ -1008,6 +1019,15 @@ bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
   if (isa<InvokeInst>(V))
     return true;
 
+  // If the target supports globally addressable scratch, the mapping from
+  // scratch memory to the flat aperture changes therefore an address space cast
+  // is no longer uniform.
+  if (auto *CastI = dyn_cast<AddrSpaceCastInst>(V)) {
+    return CastI->getSrcAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS &&
+           CastI->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS &&
+           ST->hasGloballyAddressableScratch();
+  }
+
   return false;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
deleted file mode 100644
index e400491c3860e..0000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
+++ /dev/null
@@ -1,119 +0,0 @@
-//===- AMDGPUUnifyMetadata.cpp - Unify OpenCL metadata --------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// \file
-// This pass that unifies multiple OpenCL metadata due to linking.
-//
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPU.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/PassManager.h"
-#include "llvm/Pass.h"
-
-using namespace llvm;
-
-namespace {
-
-  namespace kOCLMD {
-
-    const char SpirVer[]            = "opencl.spir.version";
-    const char OCLVer[]             = "opencl.ocl.version";
-    const char UsedExt[]            = "opencl.used.extensions";
-    const char UsedOptCoreFeat[]    = "opencl.used.optional.core.features";
-    const char CompilerOptions[]    = "opencl.compiler.options";
-    const char LLVMIdent[]          = "llvm.ident";
-
-  } // end namespace kOCLMD
-
-    /// Unify version metadata.
-    /// \return true if changes are made.
-    /// Assume the named metadata has operands each of which is a pair of
-    /// integer constant, e.g.
-    /// !Name = {!n1, !n2}
-    /// !n1 = {i32 1, i32 2}
-    /// !n2 = {i32 2, i32 0}
-    /// Keep the largest version as the sole operand if PickFirst is false.
-    /// Otherwise pick it from the first value, representing kernel module.
-    bool unifyVersionMD(Module &M, StringRef Name, bool PickFirst) {
-      auto *NamedMD = M.getNamedMetadata(Name);
-      if (!NamedMD || NamedMD->getNumOperands() <= 1)
-        return false;
-      MDNode *MaxMD = nullptr;
-      auto MaxVer = 0U;
-      for (auto *VersionMD : NamedMD->operands()) {
-        assert(VersionMD->getNumOperands() == 2);
-        auto *CMajor = mdconst::extract<ConstantInt>(VersionMD->getOperand(0));
-        auto VersionMajor = CMajor->getZExtValue();
-        auto *CMinor = mdconst::extract<ConstantInt>(VersionMD->getOperand(1));
-        auto VersionMinor = CMinor->getZExtValue();
-        auto Ver = (VersionMajor * 100) + (VersionMinor * 10);
-        if (Ver > MaxVer) {
-          MaxVer = Ver;
-          MaxMD = VersionMD;
-        }
-        if (PickFirst)
-          break;
-      }
-      NamedMD->eraseFromParent();
-      NamedMD = M.getOrInsertNamedMetadata(Name);
-      NamedMD->addOperand(MaxMD);
-      return true;
-    }
-
-  /// Unify version metadata.
-  /// \return true if changes are made.
-  /// Assume the named metadata has operands each of which is a list e.g.
-  /// !Name = {!n1, !n2}
-  /// !n1 = !{!"cl_khr_fp16", {!"cl_khr_fp64"}}
-  /// !n2 = !{!"cl_khr_image"}
-  /// Combine it into a single list with unique operands.
-  bool unifyExtensionMD(Module &M, StringRef Name) {
-    auto *NamedMD = M.getNamedMetadata(Name);
-    if (!NamedMD || NamedMD->getNumOperands() == 1)
-      return false;
-
-    SmallVector<Metadata *, 4> All;
-    for (auto *MD : NamedMD->operands())
-      for (const auto &Op : MD->operands())
-        if (!llvm::is_contained(All, Op.get()))
-          All.push_back(Op.get());
-
-    NamedMD->eraseFromParent();
-    NamedMD = M.getOrInsertNamedMetadata(Name);
-    for (const auto &MD : All)
-      NamedMD->addOperand(MDNode::get(M.getContext(), MD));
-
-    return true;
-  }
-
-  /// Unify multiple OpenCL metadata due to linking.
-  bool unifyMetadataImpl(Module &M) {
-    const char *Vers[] = {kOCLMD::SpirVer, kOCLMD::OCLVer};
-    const char *Exts[] = {kOCLMD::UsedExt, kOCLMD::UsedOptCoreFeat,
-                          kOCLMD::CompilerOptions, kOCLMD::LLVMIdent};
-
-    bool Changed = false;
-
-    for (auto &I : Vers)
-      Changed |= unifyVersionMD(M, I, true);
-
-    for (auto &I : Exts)
-      Changed |= unifyExtensionMD(M, I);
-
-    return Changed;
-  }
-
-  } // end anonymous namespace
-
-  PreservedAnalyses AMDGPUUnifyMetadataPass::run(Module &M,
-                                                 ModuleAnalysisManager &AM) {
-    return unifyMetadataImpl(M) ? PreservedAnalyses::none()
-                                : PreservedAnalyses::all();
-  }
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index ff8efd2debc21..2e21ba4c30b53 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -4933,6 +4933,43 @@ bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
       return false;
   }
 
+  // Packed math FP32 instructions typically accept SGPRs or VGPRs as source
+  // operands. On gfx12+, if a source operand uses SGPRs, the HW can only read
+  // the first SGPR and use it for both the low and high operations.
+  if (isPackedFP32Inst(Opc) && isGFX12Plus()) {
+    int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
+    int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
+    int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
+    int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
+
+    const MCOperand &Src0 = Inst.getOperand(Src0Idx);
+    const MCOperand &Src1 = Inst.getOperand(Src1Idx);
+    unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
+    unsigned OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
+
+    const MCRegisterInfo *TRI = getContext().getRegisterInfo();
+
+    auto VerifyOneSGPR = [OpSel, OpSelHi](unsigned Index) -> bool {
+      unsigned Mask = 1U << Index;
+      return ((OpSel & Mask) == 0) && ((OpSelHi & Mask) == 0);
+    };
+
+    if (Src0.isReg() && isSGPR(Src0.getReg(), TRI) &&
+        !VerifyOneSGPR(/*Index=*/0))
+      return false;
+    if (Src1.isReg() && isSGPR(Src1.getReg(), TRI) &&
+        !VerifyOneSGPR(/*Index=*/1))
+      return false;
+
+    int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
+    if (Src2Idx != -1) {
+      const MCOperand &Src2 = Inst.getOperand(Src2Idx);
+      if (Src2.isReg() && isSGPR(Src2.getReg(), TRI) &&
+          !VerifyOneSGPR(/*Index=*/2))
+        return false;
+    }
+  }
+
   return true;
 }
 
@@ -5015,11 +5052,13 @@ bool AMDGPUAsmParser::validateDPP(const MCInst &Inst,
   if (DppCtrlIdx >= 0) {
     unsigned DppCtrl = Inst.getOperand(DppCtrlIdx).getImm();
 
-    if (!AMDGPU::isLegalDPALU_DPPControl(DppCtrl) &&
-        AMDGPU::isDPALU_DPP(MII.get(Opc))) {
-      // DP ALU DPP is supported for row_newbcast only on GFX9*
+    if (!AMDGPU::isLegalDPALU_DPPControl(getSTI(), DppCtrl) &&
+        AMDGPU::isDPALU_DPP(MII.get(Opc), getSTI())) {
+      // DP ALU DPP is supported for row_newbcast only on GFX9* and row_share
+      // only on GFX12.
       SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl, Operands);
-      Error(S, "DP ALU dpp only supports row_newbcast");
+      Error(S, isGFX12() ? "DP ALU dpp only supports row_share"
+                         : "DP ALU dpp only supports row_newbcast");
       return false;
     }
   }
@@ -5947,6 +5986,7 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
   SMRange VGPRRange;
   const MCExpr *NextFreeVGPR = ZeroExpr;
   const MCExpr *AccumOffset = MCConstantExpr::create(0, getContext());
+  const MCExpr *NamedBarCnt = ZeroExpr;
   uint64_t SharedVGPRCount = 0;
   uint64_t PreloadLength = 0;
   uint64_t PreloadOffset = 0;
@@ -6169,6 +6209,10 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
       if (!isGFX90A())
         return Error(IDRange.Start, "directive requires gfx90a+", IDRange);
       AccumOffset = ExprVal;
+    } else if (ID == ".amdhsa_named_barrier_count") {
+      if (!isGFX1250())
+        return Error(IDRange.Start, "directive requires gfx1250+", IDRange);
+      NamedBarCnt = ExprVal;
     } else if (ID == ".amdhsa_reserve_vcc") {
       if (EvaluatableExpr && !isUInt<1>(Val))
         return OutOfRangeError(ValRange);
@@ -6231,8 +6275,9 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT,
                        ExprVal, ValRange);
     } else if (ID == ".amdhsa_workgroup_processor_mode") {
-      if (IVersion.Major < 10)
-        return Error(IDRange.Start, "directive requires gfx10+", IDRange);
+      if (!supportsWGP(getSTI()))
+        return Error(IDRange.Start,
+                     "directive unsupported on " + getSTI().getCPU(), IDRange);
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
                        COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
                        ValRange);
@@ -6365,12 +6410,24 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
     return TokError("amdgpu_user_sgpr_count smaller than than implied by "
                     "enabled user SGPRs");
 
-  if (!isUInt<COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_WIDTH>(UserSGPRCount))
-    return TokError("too many user SGPRs enabled");
-  AMDGPU::MCKernelDescriptor::bits_set(
-      KD.compute_pgm_rsrc2, MCConstantExpr::create(UserSGPRCount, getContext()),
-      COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT,
-      COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, getContext());
+  if (isGFX1250()) {
+    if (!isUInt<COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_WIDTH>(UserSGPRCount))
+      return TokError("too many user SGPRs enabled");
+    AMDGPU::MCKernelDescriptor::bits_set(
+        KD.compute_pgm_rsrc2,
+        MCConstantExpr::create(UserSGPRCount, getContext()),
+        COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
+        COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT, getContext());
+  } else {
+    if (!isUInt<COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_WIDTH>(
+            UserSGPRCount))
+      return TokError("too many user SGPRs enabled");
+    AMDGPU::MCKernelDescriptor::bits_set(
+        KD.compute_pgm_rsrc2,
+        MCConstantExpr::create(UserSGPRCount, getContext()),
+        COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
+        COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT, getContext());
+  }
 
   int64_t IVal = 0;
   if (!KD.kernarg_size->evaluateAsAbsolute(IVal))
@@ -6408,6 +6465,12 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
                                  getContext());
   }
 
+  if (isGFX1250())
+    MCKernelDescriptor::bits_set(KD.compute_pgm_rsrc3, NamedBarCnt,
+                                 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
+                                 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
+                                 getContext());
+
   if (IVersion.Major >= 10 && IVersion.Major < 12) {
     // SharedVGPRCount < 16 checked by PARSE_ENTRY_BITS
     if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index c466f9cf0f359..dc9dd220130ea 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -114,7 +114,6 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUTargetTransformInfo.cpp
   AMDGPUWaitSGPRHazards.cpp
   AMDGPUUnifyDivergentExitNodes.cpp
-  AMDGPUUnifyMetadata.cpp
   R600MachineCFGStructurizer.cpp
   GCNCreateVOPD.cpp
   GCNDPPCombine.cpp
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index fb7d634e62272..6a2beeed41dfd 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -27,6 +27,7 @@
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInstrDesc.h"
@@ -605,7 +606,7 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
       Bytes = Bytes_.slice(0, MaxInstBytesNum);
     }
 
-    if (isGFX11Plus() && Bytes.size() >= 12 ) {
+    if (isGFX11Plus() && Bytes.size() >= 12) {
       DecoderUInt128 DecW = eat12Bytes(Bytes);
 
       if (isGFX11() &&
@@ -2283,24 +2284,38 @@ Expected<bool> AMDGPUDisassembler::decodeCOMPUTE_PGM_RSRC1(
   CHECK_RESERVED_BITS(COMPUTE_PGM_RSRC1_BULKY);
   CHECK_RESERVED_BITS(COMPUTE_PGM_RSRC1_CDBG_USER);
 
-  if (isGFX9Plus())
+  // Bits [26].
+  if (isGFX9Plus()) {
     PRINT_DIRECTIVE(".amdhsa_fp16_overflow", COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL);
-
-  if (!isGFX9Plus())
+  } else {
     CHECK_RESERVED_BITS_DESC_MSG(COMPUTE_PGM_RSRC1_GFX6_GFX8_RESERVED0,
                                  "COMPUTE_PGM_RSRC1", "must be zero pre-gfx9");
+  }
 
-  CHECK_RESERVED_BITS_DESC(COMPUTE_PGM_RSRC1_RESERVED1, "COMPUTE_PGM_RSRC1");
+  // Bits [27].
+  if (isGFX1250()) {
+    PRINT_PSEUDO_DIRECTIVE_COMMENT("FLAT_SCRATCH_IS_NV",
+                                   COMPUTE_PGM_RSRC1_GFX125_FLAT_SCRATCH_IS_NV);
+  } else {
+    CHECK_RESERVED_BITS_DESC(COMPUTE_PGM_RSRC1_GFX6_GFX120_RESERVED1,
+                             "COMPUTE_PGM_RSRC1");
+  }
 
-  if (!isGFX10Plus())
-    CHECK_RESERVED_BITS_DESC_MSG(COMPUTE_PGM_RSRC1_GFX6_GFX9_RESERVED2,
-                                 "COMPUTE_PGM_RSRC1", "must be zero pre-gfx10");
+  // Bits [28].
+  CHECK_RESERVED_BITS_DESC(COMPUTE_PGM_RSRC1_RESERVED2, "COMPUTE_PGM_RSRC1");
 
+  // Bits [29-31].
   if (isGFX10Plus()) {
-    PRINT_DIRECTIVE(".amdhsa_workgroup_processor_mode",
-                    COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE);
+    // WGP_MODE is not available on GFX1250.
+    if (!isGFX1250()) {
+      PRINT_DIRECTIVE(".amdhsa_workgroup_processor_mode",
+                      COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE);
+    }
     PRINT_DIRECTIVE(".amdhsa_memory_ordered", COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED);
     PRINT_DIRECTIVE(".amdhsa_forward_progress", COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS);
+  } else {
+    CHECK_RESERVED_BITS_DESC(COMPUTE_PGM_RSRC1_GFX6_GFX9_RESERVED3,
+                             "COMPUTE_PGM_RSRC1");
   }
 
   if (isGFX12Plus())
@@ -2422,8 +2437,25 @@ Expected<bool> AMDGPUDisassembler::decodeCOMPUTE_PGM_RSRC3(
                                    "must be zero on gfx10 or gfx11");
     }
 
-    // Bits [14-30].
-    CHECK_RESERVED_BITS_DESC_MSG(COMPUTE_PGM_RSRC3_GFX10_PLUS_RESERVED4,
+    // Bits [14-21].
+    if (isGFX1250()) {
+      PRINT_DIRECTIVE(".amdhsa_named_barrier_count",
+                      COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT);
+      PRINT_PSEUDO_DIRECTIVE_COMMENT(
+          "ENABLE_DYNAMIC_VGPR", COMPUTE_PGM_RSRC3_GFX125_ENABLE_DYNAMIC_VGPR);
+      PRINT_PSEUDO_DIRECTIVE_COMMENT("TCP_SPLIT",
+                                     COMPUTE_PGM_RSRC3_GFX125_TCP_SPLIT);
+      PRINT_PSEUDO_DIRECTIVE_COMMENT(
+          "ENABLE_DIDT_THROTTLE",
+          COMPUTE_PGM_RSRC3_GFX125_ENABLE_DIDT_THROTTLE);
+    } else {
+      CHECK_RESERVED_BITS_DESC_MSG(COMPUTE_PGM_RSRC3_GFX10_GFX120_RESERVED4,
+                                   "COMPUTE_PGM_RSRC3",
+                                   "must be zero on gfx10+");
+    }
+
+    // Bits [22-30].
+    CHECK_RESERVED_BITS_DESC_MSG(COMPUTE_PGM_RSRC3_GFX10_PLUS_RESERVED5,
                                  "COMPUTE_PGM_RSRC3", "must be zero on gfx10+");
 
     // Bits [31].
@@ -2431,7 +2463,7 @@ Expected<bool> AMDGPUDisassembler::decodeCOMPUTE_PGM_RSRC3(
       PRINT_PSEUDO_DIRECTIVE_COMMENT("IMAGE_OP",
                                      COMPUTE_PGM_RSRC3_GFX11_PLUS_IMAGE_OP);
     } else {
-      CHECK_RESERVED_BITS_DESC_MSG(COMPUTE_PGM_RSRC3_GFX10_RESERVED5,
+      CHECK_RESERVED_BITS_DESC_MSG(COMPUTE_PGM_RSRC3_GFX10_RESERVED6,
                                    "COMPUTE_PGM_RSRC3",
                                    "must be zero on gfx10");
     }
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index d5d1074622135..f5d438436b299 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -1274,7 +1274,7 @@ let WaveSizePredicate = isWave64, SubtargetPredicate = isGFX12PlusNot12_50 in {
   defm GLOBAL_LOAD_TR_B64_w64   : FLAT_Global_Load_Pseudo <"global_load_tr_b64_w64", VGPR_32>;
 }
 
-let WaveSizePredicate = isWave32, SubtargetPredicate = isGFX1250Plus in {
+let WaveSizePredicate = isWave32, SubtargetPredicate = HasTransposeLoadF4F6Insts in {
   defm GLOBAL_LOAD_TR6_B96 : FLAT_Global_Load_Pseudo <"global_load_tr6_b96", VReg_96>;
   defm GLOBAL_LOAD_TR4_B64 : FLAT_Global_Load_Pseudo <"global_load_tr4_b64", VReg_64>;
 }
diff --git a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
index f9a907a644373..184929a5a50f6 100644
--- a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp
@@ -421,6 +421,11 @@ MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
           AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::byte_sel)) {
         DPPInst.addImm(ByteSelOpr->getImm());
       }
+      if (MachineOperand *BitOp3 =
+              TII->getNamedOperand(OrigMI, AMDGPU::OpName::bitop3)) {
+        assert(AMDGPU::hasNamedOperand(DPPOp, AMDGPU::OpName::bitop3));
+        DPPInst.add(*BitOp3);
+      }
     }
     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl));
     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask));
@@ -544,11 +549,17 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
     return false;
   }
 
-  if (MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
-      MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp) {
-    auto *DppCtrl = TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl);
-    assert(DppCtrl && DppCtrl->isImm());
-    if (!AMDGPU::isLegalDPALU_DPPControl(DppCtrl->getImm())) {
+  auto *DppCtrl = TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl);
+  assert(DppCtrl && DppCtrl->isImm());
+  unsigned DppCtrlVal = DppCtrl->getImm();
+  if ((MovMI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO ||
+       MovMI.getOpcode() == AMDGPU::V_MOV_B64_dpp)) {
+    if (!ST->hasFeature(AMDGPU::FeatureDPALU_DPP)) {
+      LLVM_DEBUG(dbgs() << "  failed: 64 bit dpp move is unsupported\n");
+      // Split it.
+      return false;
+    }
+    if (!AMDGPU::isLegalDPALU_DPPControl(*ST, DppCtrlVal)) {
       LLVM_DEBUG(dbgs() << "  failed: 64 bit dpp move uses unsupported"
                            " control value\n");
       // Let it split, then control may become legal.
@@ -704,6 +715,20 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
       break;
     }
 
+    if (!ST->hasFeature(AMDGPU::FeatureDPALU_DPP) &&
+        AMDGPU::isDPALU_DPP32BitOpc(OrigOp)) {
+      LLVM_DEBUG(dbgs() << "  " << OrigMI
+                        << "  failed: DPP ALU DPP is not supported\n");
+      break;
+    }
+
+    if (!AMDGPU::isLegalDPALU_DPPControl(*ST, DppCtrlVal) &&
+        AMDGPU::isDPALU_DPP(TII->get(OrigOp), *ST)) {
+      LLVM_DEBUG(dbgs() << "  " << OrigMI
+                        << "  failed: not valid 64-bit DPP control value\n");
+      break;
+    }
+
     LLVM_DEBUG(dbgs() << "  combining: " << OrigMI);
     if (Use == Src0) {
       if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 96cb5ae79534c..a3b64aee297b2 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -1200,6 +1200,14 @@ void GCNHazardRecognizer::fixHazards(MachineInstr *MI) {
   fixShift64HighRegBug(MI);
   fixVALUMaskWriteHazard(MI);
   fixRequiredExportPriority(MI);
+  if (ST.requiresWaitIdleBeforeGetReg())
+    fixGetRegWaitIdle(MI);
+  if (ST.hasDsAtomicAsyncBarrierArriveB64PipeBug())
+    fixDsAtomicAsyncBarrierArriveB64(MI);
+  if (ST.hasScratchBaseForwardingHazard())
+    fixScratchBaseForwardingHazard(MI);
+  if (ST.setRegModeNeedsVNOPs())
+    fixSetRegMode(MI);
 }
 
 static bool isVCmpXWritesExec(const SIInstrInfo &TII, const SIRegisterInfo &TRI,
@@ -1350,6 +1358,9 @@ bool GCNHazardRecognizer::fixSMEMtoVectorWriteHazards(MachineInstr *MI) {
         return (Decoded.DsCnt == 0);
       }
       default:
+        assert((!SIInstrInfo::isWaitcnt(MI.getOpcode()) ||
+                MI.getOpcode() == AMDGPU::S_WAIT_IDLE) &&
+               "unexpected wait count instruction");
         // SOPP instructions cannot mitigate the hazard.
         if (TII->isSOPP(MI))
           return false;
@@ -1731,7 +1742,7 @@ bool GCNHazardRecognizer::fixVALUPartialForwardingHazard(MachineInstr *MI) {
 
   BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
           TII.get(AMDGPU::S_WAITCNT_DEPCTR))
-      .addImm(0x0fff);
+      .addImm(AMDGPU::DepCtr::encodeFieldVaVdst(0));
 
   return true;
 }
@@ -1781,7 +1792,7 @@ bool GCNHazardRecognizer::fixVALUTransUseHazard(MachineInstr *MI) {
     if (SIInstrInfo::isVMEM(I) || SIInstrInfo::isDS(I) ||
         SIInstrInfo::isEXP(I) ||
         (I.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR &&
-         I.getOperand(0).getImm() == 0x0fff))
+         AMDGPU::DepCtr::decodeFieldVaVdst(I.getOperand(0).getImm()) == 0))
       return HazardExpired;
 
     // Track registers writes
@@ -2239,19 +2250,7 @@ int GCNHazardRecognizer::checkFPAtomicToDenormModeHazard(MachineInstr *MI) {
     if (WaitStates >= 3 || SIInstrInfo::isVALU(MI))
       return true;
 
-    switch (MI.getOpcode()) {
-    case AMDGPU::S_WAITCNT:
-    case AMDGPU::S_WAITCNT_VSCNT:
-    case AMDGPU::S_WAITCNT_VMCNT:
-    case AMDGPU::S_WAITCNT_EXPCNT:
-    case AMDGPU::S_WAITCNT_LGKMCNT:
-    case AMDGPU::S_WAIT_IDLE:
-      return true;
-    default:
-      break;
-    }
-
-    return false;
+    return SIInstrInfo::isWaitcnt(MI.getOpcode());
   };
 
   return FPAtomicToDenormModeWaitStates -
@@ -3428,3 +3427,125 @@ bool GCNHazardRecognizer::fixRequiredExportPriority(MachineInstr *MI) {
 
   return true;
 }
+
+bool GCNHazardRecognizer::fixGetRegWaitIdle(MachineInstr *MI) {
+  if (!isSGetReg(MI->getOpcode()))
+    return false;
+
+  const SIInstrInfo *TII = ST.getInstrInfo();
+  switch (getHWReg(TII, *MI)) {
+  default:
+    return false;
+  case AMDGPU::Hwreg::ID_STATUS:
+  case AMDGPU::Hwreg::ID_STATE_PRIV:
+  case AMDGPU::Hwreg::ID_EXCP_FLAG_PRIV:
+  case AMDGPU::Hwreg::ID_EXCP_FLAG_USER:
+    break;
+  }
+
+  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
+          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
+      .addImm(0);
+  return true;
+}
+
+bool GCNHazardRecognizer::fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI) {
+  if (MI->getOpcode() != AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
+    return false;
+
+  const SIInstrInfo *TII = ST.getInstrInfo();
+  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
+          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
+      .addImm(0xFFE3);
+  BuildMI(*MI->getParent(), std::next(MI->getIterator()), MI->getDebugLoc(),
+          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
+      .addImm(0xFFE3);
+
+  return true;
+}
+
+bool GCNHazardRecognizer::fixScratchBaseForwardingHazard(MachineInstr *MI) {
+  // No reason to check this in pre-RA scheduling, SGPRs have to be allocated
+  // for hazard to trigger.
+  if (!IsHazardRecognizerMode)
+    return false;
+
+  const SIRegisterInfo *TRI = ST.getRegisterInfo();
+  const SIInstrInfo *TII = ST.getInstrInfo();
+  // Hazard expires after 10 SGPR writes by SALU or 8 SGPR writes by VALU.
+  const int FlatScrBaseWaitStates = 10;
+
+  bool ReadsFlatScrLo =
+      MI->readsRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_LO, TRI);
+  bool ReadsFlatScrHi =
+      MI->readsRegister(AMDGPU::SRC_FLAT_SCRATCH_BASE_HI, TRI);
+  if (isSGetReg(MI->getOpcode())) {
+    switch (getHWReg(TII, *MI)) {
+    default:
+      break;
+    case AMDGPU::Hwreg::ID_FLAT_SCR_LO:
+      ReadsFlatScrLo = true;
+      break;
+    case AMDGPU::Hwreg::ID_FLAT_SCR_HI:
+      ReadsFlatScrHi = true;
+      break;
+    }
+  }
+
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  auto IsRegDefHazard = [&](Register Reg) -> bool {
+    DenseSet<const MachineBasicBlock *> Visited;
+    auto IsHazardFn = [TRI, Reg](const MachineInstr &MI) {
+      return MI.modifiesRegister(Reg, TRI);
+    };
+
+    // This literally abuses the idea of waitstates. Instead of waitstates it
+    // returns 1 for SGPR written and 0 otherwise.
+    auto IsSGPRDef = [TII, TRI, &MRI](const MachineInstr &MI) -> unsigned {
+      if (!TII->isSALU(MI) && !TII->isVALU(MI))
+        return 0;
+      for (const MachineOperand &MO : MI.all_defs()) {
+        if (TRI->isSGPRReg(MRI, MO.getReg()))
+          return 1;
+      }
+      return 0;
+    };
+
+    auto IsExpiredFn = [=](const MachineInstr &MI, int SgprWrites) {
+      if (MI.getOpcode() == AMDGPU::S_WAITCNT_DEPCTR) {
+        unsigned Wait = MI.getOperand(0).getImm();
+        if (AMDGPU::DepCtr::decodeFieldSaSdst(Wait) == 0 &&
+            AMDGPU::DepCtr::decodeFieldVaSdst(Wait) == 0)
+          return true;
+      }
+      return SgprWrites >= FlatScrBaseWaitStates;
+    };
+
+    return ::getWaitStatesSince(
+               IsHazardFn, MI->getParent(), std::next(MI->getReverseIterator()),
+               0, IsExpiredFn, Visited, IsSGPRDef) < FlatScrBaseWaitStates;
+  };
+
+  if ((!ReadsFlatScrLo || MRI.isConstantPhysReg(AMDGPU::SGPR102) ||
+       !IsRegDefHazard(AMDGPU::SGPR102)) &&
+      (!ReadsFlatScrHi || MRI.isConstantPhysReg(AMDGPU::SGPR103) ||
+       !IsRegDefHazard(AMDGPU::SGPR103)))
+    return false;
+
+  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
+          TII->get(AMDGPU::S_WAITCNT_DEPCTR))
+      .addImm(AMDGPU::DepCtr::encodeFieldVaSdst(
+          AMDGPU::DepCtr::encodeFieldSaSdst(0), 0));
+  return true;
+}
+
+bool GCNHazardRecognizer::fixSetRegMode(MachineInstr *MI) {
+  if (!isSSetReg(MI->getOpcode()) ||
+      MI->getOperand(1).getImm() != AMDGPU::Hwreg::ID_MODE)
+    return false;
+
+  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
+  BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII.get(AMDGPU::V_NOP_e32));
+  return true;
+}
diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
index f796eeaebea89..67beffadc0913 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.h
@@ -110,6 +110,10 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
   bool fixShift64HighRegBug(MachineInstr *MI);
   bool fixVALUMaskWriteHazard(MachineInstr *MI);
   bool fixRequiredExportPriority(MachineInstr *MI);
+  bool fixGetRegWaitIdle(MachineInstr *MI);
+  bool fixDsAtomicAsyncBarrierArriveB64(MachineInstr *MI);
+  bool fixScratchBaseForwardingHazard(MachineInstr *MI);
+  bool fixSetRegMode(MachineInstr *MI);
 
   int checkMAIHazards(MachineInstr *MI);
   int checkMAIHazards908(MachineInstr *MI);
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index 334afd3a2a5b4..ef63acc6355d2 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -368,46 +368,45 @@ static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
 ////////////////////////////////////////////////////////////////////////////////
 // GCNRPTarget
 
-GCNRPTarget::GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP,
-                         bool CombineVGPRSavings)
-    : RP(RP), CombineVGPRSavings(CombineVGPRSavings) {
+GCNRPTarget::GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP)
+    : GCNRPTarget(RP, MF) {
   const Function &F = MF.getFunction();
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
-  setRegLimits(ST.getMaxNumSGPRs(F), ST.getMaxNumVGPRs(F), MF);
+  setTarget(ST.getMaxNumSGPRs(F), ST.getMaxNumVGPRs(F));
 }
 
 GCNRPTarget::GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs,
-                         const MachineFunction &MF, const GCNRegPressure &RP,
-                         bool CombineVGPRSavings)
-    : RP(RP), CombineVGPRSavings(CombineVGPRSavings) {
-  setRegLimits(NumSGPRs, NumVGPRs, MF);
+                         const MachineFunction &MF, const GCNRegPressure &RP)
+    : GCNRPTarget(RP, MF) {
+  setTarget(NumSGPRs, NumVGPRs);
 }
 
 GCNRPTarget::GCNRPTarget(unsigned Occupancy, const MachineFunction &MF,
-                         const GCNRegPressure &RP, bool CombineVGPRSavings)
-    : RP(RP), CombineVGPRSavings(CombineVGPRSavings) {
+                         const GCNRegPressure &RP)
+    : GCNRPTarget(RP, MF) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
   unsigned DynamicVGPRBlockSize =
       MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
-  setRegLimits(ST.getMaxNumSGPRs(Occupancy, /*Addressable=*/false),
-               ST.getMaxNumVGPRs(Occupancy, DynamicVGPRBlockSize), MF);
+  setTarget(ST.getMaxNumSGPRs(Occupancy, /*Addressable=*/false),
+            ST.getMaxNumVGPRs(Occupancy, DynamicVGPRBlockSize));
 }
 
-void GCNRPTarget::setRegLimits(unsigned NumSGPRs, unsigned NumVGPRs,
-                               const MachineFunction &MF) {
+void GCNRPTarget::setTarget(unsigned NumSGPRs, unsigned NumVGPRs) {
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
-  unsigned DynamicVGPRBlockSize =
-      MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
   MaxSGPRs = std::min(ST.getAddressableNumSGPRs(), NumSGPRs);
   MaxVGPRs = std::min(ST.getAddressableNumArchVGPRs(), NumVGPRs);
-  MaxUnifiedVGPRs =
-      ST.hasGFX90AInsts()
-          ? std::min(ST.getAddressableNumVGPRs(DynamicVGPRBlockSize), NumVGPRs)
-          : 0;
+  if (UnifiedRF) {
+    unsigned DynamicVGPRBlockSize =
+        MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
+    MaxUnifiedVGPRs =
+        std::min(ST.getAddressableNumVGPRs(DynamicVGPRBlockSize), NumVGPRs);
+  } else {
+    MaxUnifiedVGPRs = 0;
+  }
 }
 
-bool GCNRPTarget::isSaveBeneficial(Register Reg,
-                                   const MachineRegisterInfo &MRI) const {
+bool GCNRPTarget::isSaveBeneficial(Register Reg) const {
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
   const TargetRegisterClass *RC = MRI.getRegClass(Reg);
   const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo *>(TRI);
@@ -416,16 +415,19 @@ bool GCNRPTarget::isSaveBeneficial(Register Reg,
     return RP.getSGPRNum() > MaxSGPRs;
   unsigned NumVGPRs =
       SRI->isAGPRClass(RC) ? RP.getAGPRNum() : RP.getArchVGPRNum();
-  return isVGPRBankSaveBeneficial(NumVGPRs);
+  // The addressable limit must always be respected.
+  if (NumVGPRs > MaxVGPRs)
+    return true;
+  // For unified RFs, combined VGPR usage limit must be respected as well.
+  return UnifiedRF && RP.getVGPRNum(true) > MaxUnifiedVGPRs;
 }
 
 bool GCNRPTarget::satisfied() const {
-  if (RP.getSGPRNum() > MaxSGPRs)
+  if (RP.getSGPRNum() > MaxSGPRs || RP.getVGPRNum(false) > MaxVGPRs)
     return false;
-  if (RP.getVGPRNum(false) > MaxVGPRs &&
-      (!CombineVGPRSavings || !satisifiesVGPRBanksTarget()))
+  if (UnifiedRF && RP.getVGPRNum(true) > MaxUnifiedVGPRs)
     return false;
-  return satisfiesUnifiedTarget();
+  return true;
 }
 
 ///////////////////////////////////////////////////////////////////////////////
diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
index ea33a229110c1..a9c58bb90ef03 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h
@@ -186,20 +186,22 @@ class GCNRPTarget {
   /// Sets up the target such that the register pressure starting at \p RP does
   /// not show register spilling on function \p MF (w.r.t. the function's
   /// mininum target occupancy).
-  GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP,
-              bool CombineVGPRSavings = false);
+  GCNRPTarget(const MachineFunction &MF, const GCNRegPressure &RP);
 
   /// Sets up the target such that the register pressure starting at \p RP does
   /// not use more than \p NumSGPRs SGPRs and \p NumVGPRs VGPRs on function \p
   /// MF.
   GCNRPTarget(unsigned NumSGPRs, unsigned NumVGPRs, const MachineFunction &MF,
-              const GCNRegPressure &RP, bool CombineVGPRSavings = false);
+              const GCNRegPressure &RP);
 
   /// Sets up the target such that the register pressure starting at \p RP does
   /// not prevent achieving an occupancy of at least \p Occupancy on function
   /// \p MF.
   GCNRPTarget(unsigned Occupancy, const MachineFunction &MF,
-              const GCNRegPressure &RP, bool CombineVGPRSavings = false);
+              const GCNRegPressure &RP);
+
+  /// Changes the target (same semantics as constructor).
+  void setTarget(unsigned NumSGPRs, unsigned NumVGPRs);
 
   const GCNRegPressure &getCurrentRP() const { return RP; }
 
@@ -207,7 +209,7 @@ class GCNRPTarget {
 
   /// Determines whether saving virtual register \p Reg will be beneficial
   /// towards achieving the RP target.
-  bool isSaveBeneficial(Register Reg, const MachineRegisterInfo &MRI) const;
+  bool isSaveBeneficial(Register Reg) const;
 
   /// Saves virtual register \p Reg with lanemask \p Mask.
   void saveReg(Register Reg, LaneBitmask Mask, const MachineRegisterInfo &MRI) {
@@ -227,15 +229,15 @@ class GCNRPTarget {
     if (Target.MaxUnifiedVGPRs) {
       OS << ", " << Target.RP.getVGPRNum(true) << '/' << Target.MaxUnifiedVGPRs
          << " VGPRs (unified)";
-    } else if (Target.CombineVGPRSavings) {
-      OS << ", " << Target.RP.getArchVGPRNum() + Target.RP.getAGPRNum() << '/'
-         << 2 * Target.MaxVGPRs << " VGPRs (combined target)";
     }
     return OS;
   }
 #endif
 
 private:
+  const MachineFunction &MF;
+  const bool UnifiedRF;
+
   /// Current register pressure.
   GCNRegPressure RP;
 
@@ -246,29 +248,10 @@ class GCNRPTarget {
   /// Target number of overall VGPRs for subtargets with unified RFs. Always 0
   /// for subtargets with non-unified RFs.
   unsigned MaxUnifiedVGPRs;
-  /// Whether we consider that the register allocator will be able to swap
-  /// between ArchVGPRs and AGPRs by copying them to a super register class.
-  /// Concretely, this allows savings in one of the VGPR banks to help toward
-  /// savings in the other VGPR bank.
-  bool CombineVGPRSavings;
-
-  inline bool satisifiesVGPRBanksTarget() const {
-    assert(CombineVGPRSavings && "only makes sense with combined savings");
-    return RP.getArchVGPRNum() + RP.getAGPRNum() <= 2 * MaxVGPRs;
-  }
-
-  /// Always satisified when the subtarget doesn't have a unified RF.
-  inline bool satisfiesUnifiedTarget() const {
-    return !MaxUnifiedVGPRs || RP.getVGPRNum(true) <= MaxUnifiedVGPRs;
-  }
-
-  inline bool isVGPRBankSaveBeneficial(unsigned NumVGPRs) const {
-    return NumVGPRs > MaxVGPRs || !satisfiesUnifiedTarget() ||
-           (CombineVGPRSavings && !satisifiesVGPRBanksTarget());
-  }
 
-  void setRegLimits(unsigned MaxSGPRs, unsigned MaxVGPRs,
-                    const MachineFunction &MF);
+  GCNRPTarget(const GCNRegPressure &RP, const MachineFunction &MF)
+      : MF(MF), UnifiedRF(MF.getSubtarget<GCNSubtarget>().hasGFX90AInsts()),
+        RP(RP) {}
 };
 
 ///////////////////////////////////////////////////////////////////////////////
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index 96d5668d5a1b2..254b75b784e75 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1086,7 +1086,8 @@ bool ClusteredLowOccStage::initGCNSchedStage() {
 }
 
 /// Allows to easily filter for this stage's debug output.
-#define REMAT_DEBUG(X) LLVM_DEBUG(dbgs() << "[PreRARemat] "; X;)
+#define REMAT_PREFIX "[PreRARemat] "
+#define REMAT_DEBUG(X) LLVM_DEBUG(dbgs() << REMAT_PREFIX; X;)
 
 bool PreRARematStage::initGCNSchedStage() {
   // FIXME: This pass will invalidate cached BBLiveInMap and MBBLiveIns for
@@ -1115,10 +1116,15 @@ bool PreRARematStage::initGCNSchedStage() {
   rematerialize();
   if (GCNTrackers)
     DAG.RegionLiveOuts.buildLiveRegMap();
-  REMAT_DEBUG(
-      dbgs() << "Retrying function scheduling with new min. occupancy of "
-             << AchievedOcc << " from rematerializing (original was "
-             << DAG.MinOccupancy << ", target was " << TargetOcc << ")\n");
+  REMAT_DEBUG({
+    dbgs() << "Retrying function scheduling with new min. occupancy of "
+           << AchievedOcc << " from rematerializing (original was "
+           << DAG.MinOccupancy;
+    if (TargetOcc)
+      dbgs() << ", target was " << *TargetOcc;
+    dbgs() << ")\n";
+  });
+
   if (AchievedOcc > DAG.MinOccupancy) {
     DAG.MinOccupancy = AchievedOcc;
     SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
@@ -1540,8 +1546,7 @@ bool ClusteredLowOccStage::shouldRevertScheduling(unsigned WavesAfter) {
 
 bool PreRARematStage::shouldRevertScheduling(unsigned WavesAfter) {
   return GCNSchedStage::shouldRevertScheduling(WavesAfter) ||
-         mayCauseSpilling(WavesAfter) ||
-         (IncreaseOccupancy && WavesAfter < TargetOcc);
+         mayCauseSpilling(WavesAfter) || (TargetOcc && WavesAfter < TargetOcc);
 }
 
 bool ILPInitialScheduleStage::shouldRevertScheduling(unsigned WavesAfter) {
@@ -1687,78 +1692,63 @@ bool PreRARematStage::allUsesAvailableAt(const MachineInstr *InstToRemat,
 }
 
 bool PreRARematStage::canIncreaseOccupancyOrReduceSpill() {
-  REMAT_DEBUG({
-    dbgs() << "Collecting rematerializable instructions in ";
-    MF.getFunction().printAsOperand(dbgs(), false);
-    dbgs() << '\n';
-  });
+  const Function &F = MF.getFunction();
 
   // Maps optimizable regions (i.e., regions at minimum and register-limited
   // occupancy, or regions with spilling) to the target RP we would like to
   // reach.
   DenseMap<unsigned, GCNRPTarget> OptRegions;
-  const Function &F = MF.getFunction();
-  unsigned DynamicVGPRBlockSize =
-      MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
-
-  std::pair<unsigned, unsigned> WavesPerEU = ST.getWavesPerEU(F);
-  const unsigned MaxSGPRsNoSpill = ST.getMaxNumSGPRs(F);
-  const unsigned MaxVGPRsNoSpill = ST.getMaxNumVGPRs(F);
-  const unsigned MaxSGPRsIncOcc =
-      ST.getMaxNumSGPRs(DAG.MinOccupancy + 1, false);
-  const unsigned MaxVGPRsIncOcc =
-      ST.getMaxNumVGPRs(DAG.MinOccupancy + 1, DynamicVGPRBlockSize);
-  IncreaseOccupancy = WavesPerEU.second > DAG.MinOccupancy;
-
-  // Collect optimizable regions. If there is spilling in any region we will
-  // just try to reduce spilling. Otherwise we will try to increase occupancy by
-  // one in the whole function.
-  for (unsigned I = 0, E = DAG.Regions.size(); I != E; ++I) {
-    GCNRegPressure &RP = DAG.Pressure[I];
-    // We allow ArchVGPR or AGPR savings to count as savings of the other kind
-    // of VGPR only when trying to eliminate spilling. We cannot do this when
-    // trying to increase occupancy since VGPR class swaps only occur later in
-    // the register allocator i.e., the scheduler will not be able to reason
-    // about these savings and will not report an increase in the achievable
-    // occupancy, triggering rollbacks.
-    GCNRPTarget Target(MaxSGPRsNoSpill, MaxVGPRsNoSpill, MF, RP,
-                       /*CombineVGPRSavings=*/true);
-    if (!Target.satisfied() && IncreaseOccupancy) {
-      // There is spilling in the region and we were so far trying to increase
-      // occupancy. Strop trying that and focus on reducing spilling.
-      IncreaseOccupancy = false;
-      OptRegions.clear();
-    } else if (IncreaseOccupancy) {
-      // There is no spilling in the region, try to increase occupancy.
-      Target = GCNRPTarget(MaxSGPRsIncOcc, MaxVGPRsIncOcc, MF, RP,
-                           /*CombineVGPRSavings=*/false);
+  unsigned MaxSGPRs = ST.getMaxNumSGPRs(F);
+  unsigned MaxVGPRs = ST.getMaxNumVGPRs(F);
+  auto ResetTargetRegions = [&]() {
+    OptRegions.clear();
+    for (unsigned I = 0, E = DAG.Regions.size(); I != E; ++I) {
+      const GCNRegPressure &RP = DAG.Pressure[I];
+      GCNRPTarget Target(MaxSGPRs, MaxVGPRs, MF, RP);
+      if (!Target.satisfied())
+        OptRegions.insert({I, Target});
     }
-    if (!Target.satisfied())
-      OptRegions.insert({I, Target});
-  }
-  if (OptRegions.empty())
-    return false;
+  };
 
-#ifndef NDEBUG
-  if (IncreaseOccupancy) {
-    REMAT_DEBUG(dbgs() << "Occupancy minimal (" << DAG.MinOccupancy
-                       << ") in regions:\n");
+  ResetTargetRegions();
+  if (!OptRegions.empty() || DAG.MinOccupancy >= MFI.getMaxWavesPerEU()) {
+    // In addition to register usage being above addressable limits, occupancy
+    // below the minimum is considered like "spilling" as well.
+    TargetOcc = std::nullopt;
   } else {
-    REMAT_DEBUG(dbgs() << "Spilling w.r.t. minimum target occupancy ("
-                       << WavesPerEU.first << ") in regions:\n");
-  }
-  for (unsigned I = 0, E = DAG.Regions.size(); I != E; ++I) {
-    if (auto OptIt = OptRegions.find(I); OptIt != OptRegions.end())
-      REMAT_DEBUG(dbgs() << "  [" << I << "] " << OptIt->getSecond() << '\n');
+    // There is no spilling and room to improve occupancy; set up "increased
+    // occupancy targets" for all regions.
+    TargetOcc = DAG.MinOccupancy + 1;
+    unsigned VGPRBlockSize =
+        MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
+    MaxSGPRs = ST.getMaxNumSGPRs(*TargetOcc, false);
+    MaxVGPRs = ST.getMaxNumVGPRs(*TargetOcc, VGPRBlockSize);
+    ResetTargetRegions();
   }
-#endif
-
-  // When we are reducing spilling, the target is the minimum target number of
-  // waves/EU determined by the subtarget. In cases where either one of
-  // "amdgpu-num-sgpr" or "amdgpu-num-vgpr" are set on the function, the current
-  // minimum region occupancy may be higher than the latter.
-  TargetOcc = IncreaseOccupancy ? DAG.MinOccupancy + 1
-                                : std::max(DAG.MinOccupancy, WavesPerEU.first);
+  REMAT_DEBUG({
+    dbgs() << "Analyzing ";
+    MF.getFunction().printAsOperand(dbgs(), false);
+    dbgs() << ": ";
+    if (OptRegions.empty()) {
+      dbgs() << "no objective to achieve, occupancy is maximal at "
+             << MFI.getMaxWavesPerEU();
+    } else if (!TargetOcc) {
+      dbgs() << "reduce spilling (minimum target occupancy is "
+             << MFI.getMinWavesPerEU() << ')';
+    } else {
+      dbgs() << "increase occupancy from " << DAG.MinOccupancy << " to "
+             << TargetOcc;
+    }
+    dbgs() << '\n';
+    for (unsigned I = 0, E = DAG.Regions.size(); I != E; ++I) {
+      if (auto OptIt = OptRegions.find(I); OptIt != OptRegions.end()) {
+        dbgs() << REMAT_PREFIX << "  [" << I << "] " << OptIt->getSecond()
+               << '\n';
+      }
+    }
+  });
+  if (OptRegions.empty())
+    return false;
 
   // Accounts for a reduction in RP in an optimizable region. Returns whether we
   // estimate that we have identified enough rematerialization opportunities to
@@ -1767,7 +1757,7 @@ bool PreRARematStage::canIncreaseOccupancyOrReduceSpill() {
   auto ReduceRPInRegion = [&](auto OptIt, Register Reg, LaneBitmask Mask,
                               bool &Progress) -> bool {
     GCNRPTarget &Target = OptIt->getSecond();
-    if (!Target.isSaveBeneficial(Reg, DAG.MRI))
+    if (!Target.isSaveBeneficial(Reg))
       return false;
     Progress = true;
     Target.saveReg(Reg, Mask, DAG.MRI);
@@ -1876,7 +1866,7 @@ bool PreRARematStage::canIncreaseOccupancyOrReduceSpill() {
     }
   }
 
-  if (IncreaseOccupancy) {
+  if (TargetOcc) {
     // We were trying to increase occupancy but failed, abort the stage.
     REMAT_DEBUG(dbgs() << "Cannot increase occupancy\n");
     Rematerializations.clear();
@@ -1979,7 +1969,9 @@ void PreRARematStage::rematerialize() {
   // All regions impacted by at least one rematerialization must be rescheduled.
   // Maximum pressure must also be recomputed for all regions where it changed
   // non-predictably and checked against the target occupancy.
-  AchievedOcc = TargetOcc;
+  unsigned DynamicVGPRBlockSize =
+      MF.getInfo<SIMachineFunctionInfo>()->getDynamicVGPRBlockSize();
+  AchievedOcc = MFI.getMaxWavesPerEU();
   for (auto &[I, OriginalRP] : ImpactedRegions) {
     bool IsEmptyRegion = DAG.Regions[I].first == DAG.Regions[I].second;
     RescheduleRegions[I] = !IsEmptyRegion;
@@ -2003,9 +1995,8 @@ void PreRARematStage::rematerialize() {
       }
     }
     DAG.Pressure[I] = RP;
-    AchievedOcc = std::min(
-        AchievedOcc, RP.getOccupancy(ST, MF.getInfo<SIMachineFunctionInfo>()
-                                             ->getDynamicVGPRBlockSize()));
+    AchievedOcc =
+        std::min(AchievedOcc, RP.getOccupancy(ST, DynamicVGPRBlockSize));
   }
   REMAT_DEBUG(dbgs() << "Achieved occupancy " << AchievedOcc << "\n");
 }
@@ -2035,7 +2026,7 @@ void PreRARematStage::finalizeGCNSchedStage() {
   // which case we do not want to rollback either (the rescheduling was already
   // reverted in PreRARematStage::shouldRevertScheduling in such cases).
   unsigned MaxOcc = std::max(AchievedOcc, DAG.MinOccupancy);
-  if (!IncreaseOccupancy || MaxOcc >= TargetOcc)
+  if (!TargetOcc || MaxOcc >= *TargetOcc)
     return;
 
   REMAT_DEBUG(dbgs() << "Rolling back all rematerializations\n");
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
index 32139a91c2428..790370ff8ab4d 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -470,15 +470,12 @@ class PreRARematStage : public GCNSchedStage {
   /// After successful stage initialization, indicates which regions should be
   /// rescheduled.
   BitVector RescheduleRegions;
-  /// Target occupancy the stage estimates is reachable through
-  /// rematerialization. Greater than or equal to the pre-stage min occupancy.
-  unsigned TargetOcc;
+  /// The target occupancy the stage is trying to achieve. Empty when the
+  /// objective is spilling reduction.
+  std::optional<unsigned> TargetOcc;
   /// Achieved occupancy *only* through rematerializations (pre-rescheduling).
   /// Smaller than or equal to the target occupancy.
   unsigned AchievedOcc;
-  /// Whether the stage is attempting to increase occupancy in the abscence of
-  /// spilling.
-  bool IncreaseOccupancy;
 
   /// Returns whether remat can reduce spilling or increase function occupancy
   /// by 1 through rematerialization. If it can do one, collects instructions in
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
index 0237a60673a95..931966b6df1df 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.cpp
@@ -137,13 +137,10 @@ GCNSubtarget &GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
   if (LDSBankCount == 0)
     LDSBankCount = 32;
 
-  if (TT.isAMDGCN() && AddressableLocalMemorySize == 0)
+  if (AddressableLocalMemorySize == 0)
     AddressableLocalMemorySize = 32768;
 
-  LocalMemorySize = AddressableLocalMemorySize;
-  if (AMDGPU::isGFX10Plus(*this) &&
-      !getFeatureBits().test(AMDGPU::FeatureCuMode))
-    LocalMemorySize *= 2;
+  LocalMemorySize = AMDGPU::IsaInfo::getLocalMemorySize(this);
 
   HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
   HasSMulHi = getGeneration() >= AMDGPUSubtarget::GFX9;
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 5530886831cae..2a8385df3f934 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -187,6 +187,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   bool HasFlatBufferGlobalAtomicFaddF64Inst = false;
   bool HasDefaultComponentZero = false;
   bool HasAgentScopeFineGrainedRemoteMemoryAtomics = false;
+  bool HasEmulatedSystemScopeAtomics = false;
   bool HasDefaultComponentBroadcast = false;
   bool HasXF32Insts = false;
   /// The maximum number of instructions that may be placed within an S_CLAUSE,
@@ -389,7 +390,11 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   /// the original value.
   bool zeroesHigh16BitsOfDest(unsigned Opcode) const;
 
-  bool supportsWGP() const { return getGeneration() >= GFX10; }
+  bool supportsWGP() const {
+    if (GFX1250Insts)
+      return false;
+    return getGeneration() >= GFX10;
+  }
 
   bool hasIntClamp() const {
     return HasIntClamp;
@@ -950,6 +955,12 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
     return HasAgentScopeFineGrainedRemoteMemoryAtomics;
   }
 
+  /// \return true is HW emulates system scope atomics unsupported by the PCI-e
+  /// via CAS loop.
+  bool hasEmulatedSystemScopeAtomics() const {
+    return HasEmulatedSystemScopeAtomics;
+  }
+
   bool hasDefaultComponentZero() const { return HasDefaultComponentZero; }
 
   bool hasDefaultComponentBroadcast() const {
@@ -1081,7 +1092,7 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   }
 
   bool hasLDSFPAtomicAddF32() const { return GFX8Insts; }
-  bool hasLDSFPAtomicAddF64() const { return GFX90AInsts; }
+  bool hasLDSFPAtomicAddF64() const { return GFX90AInsts || GFX1250Insts; }
 
   /// \returns true if the subtarget has the v_permlanex16_b32 instruction.
   bool hasPermLaneX16() const { return getGeneration() >= GFX10; }
@@ -1334,6 +1345,10 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
 
   bool hasVALUReadSGPRHazard() const { return GFX12Insts && !GFX1250Insts; }
 
+  bool setRegModeNeedsVNOPs() const {
+    return GFX1250Insts && getGeneration() == GFX12;
+  }
+
   /// Return if operations acting on VGPR tuples require even alignment.
   bool needsAlignedVGPRs() const { return GFX90AInsts || GFX1250Insts; }
 
@@ -1555,12 +1570,22 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   // \returns true if the target has V_PK_{MIN|MAX}3_{I|U}16 instructions.
   bool hasPkMinMax3Insts() const { return GFX1250Insts; }
 
+  // \returns ture if target has S_GET_SHADER_CYCLES_U64 instruction.
+  bool hasSGetShaderCyclesInst() const { return GFX1250Insts; }
+
   // \returns true if target has S_SETPRIO_INC_WG instruction.
   bool hasSetPrioIncWgInst() const { return HasSetPrioIncWgInst; }
 
   // \returns true if S_GETPC_B64 zero-extends the result from 48 bits instead
-  // of sign-extending.
-  bool hasGetPCZeroExtension() const { return GFX12Insts; }
+  // of sign-extending. Note that GFX1250 has not only fixed the bug but also
+  // extended VA to 57 bits.
+  bool hasGetPCZeroExtension() const { return GFX12Insts && !GFX1250Insts; }
+
+  // \returns true if the target needs to create a prolog for backward
+  // compatibility when preloading kernel arguments.
+  bool needsKernArgPreloadProlog() const {
+    return hasKernargPreload() && !GFX1250Insts;
+  }
 
   /// \returns SGPR allocation granularity supported by the subtarget.
   unsigned getSGPRAllocGranule() const {
@@ -1711,6 +1736,10 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
   /// unit requirement.
   unsigned getMaxNumVGPRs(const MachineFunction &MF) const;
 
+  bool supportsWave32() const { return getGeneration() >= GFX10; }
+
+  bool supportsWave64() const { return !hasGFX1250Insts(); }
+
   bool isWave32() const {
     return getWavefrontSize() == 32;
   }
@@ -1774,11 +1803,11 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
 
   // \returns true if the subtarget has a hazard requiring an "s_nop 0"
   // instruction before "s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)".
-  bool requiresNopBeforeDeallocVGPRs() const {
-    // Currently all targets that support the dealloc VGPRs message also require
-    // the nop.
-    return true;
-  }
+  bool requiresNopBeforeDeallocVGPRs() const { return !GFX1250Insts; }
+
+  // \returns true if the subtarget needs S_WAIT_ALU 0 before S_GETREG_B32 on
+  // STATUS, STATE_PRIV, EXCP_FLAG_PRIV, or EXCP_FLAG_USER.
+  bool requiresWaitIdleBeforeGetReg() const { return GFX1250Insts; }
 
   bool isDynamicVGPREnabled() const { return DynamicVGPR; }
   unsigned getDynamicVGPRBlockSize() const {
@@ -1790,6 +1819,18 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
     // to the same register.
     return false;
   }
+
+  // DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 shall not be claused with anything
+  // and surronded by S_WAIT_ALU(0xFFE3).
+  bool hasDsAtomicAsyncBarrierArriveB64PipeBug() const {
+    return getGeneration() == GFX12;
+  }
+
+  // Requires s_wait_alu(0) after s102/s103 write and src_flat_scratch_base
+  // read.
+  bool hasScratchBaseForwardingHazard() const {
+    return GFX1250Insts && getGeneration() == GFX12;
+  }
 };
 
 class GCNUserSGPRUsageInfo {
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index ee8683a549a80..aafbdc2e86a9b 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -976,8 +976,10 @@ void AMDGPUInstPrinter::printDPPCtrl(const MCInst *MI, unsigned OpNo,
   unsigned Imm = MI->getOperand(OpNo).getImm();
   const MCInstrDesc &Desc = MII.get(MI->getOpcode());
 
-  if (!AMDGPU::isLegalDPALU_DPPControl(Imm) && AMDGPU::isDPALU_DPP(Desc)) {
-    O << " /* DP ALU dpp only supports row_newbcast */";
+  if (!AMDGPU::isLegalDPALU_DPPControl(STI, Imm) &&
+      AMDGPU::isDPALU_DPP(Desc, STI)) {
+    O << " /* DP ALU dpp only supports "
+      << (isGFX12(STI) ? "row_share" : "row_newbcast") << " */";
     return;
   }
   if (Imm <= DppCtrl::QUAD_PERM_LAST) {
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index f3580842c6ff0..61f673221739a 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -389,6 +389,8 @@ void AMDGPUMCCodeEmitter::encodeInstruction(const MCInst &MI,
        Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_vi) &&
       // Matrix B format operand reuses op_sel_hi.
       !AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::matrix_b_fmt) &&
+      // Matrix B scale operand reuses op_sel_hi.
+      !AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::matrix_b_scale) &&
       // Matrix B reuse operand reuses op_sel_hi.
       !AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::matrix_b_reuse)) {
     Encoding |= getImplicitOpSelHiEncoding(Opcode);
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index 68302f0dd0d64..0bbab29dbda18 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -26,7 +26,6 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/AMDGPUMetadata.h"
 #include "llvm/Support/AMDHSAKernelDescriptor.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FormattedStream.h"
 #include "llvm/TargetParser/TargetParser.h"
@@ -277,10 +276,10 @@ void AMDGPUTargetAsmStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
 
 void AMDGPUTargetAsmStreamer::EmitMCResourceInfo(
     const MCSymbol *NumVGPR, const MCSymbol *NumAGPR,
-    const MCSymbol *NumExplicitSGPR, const MCSymbol *PrivateSegmentSize,
-    const MCSymbol *UsesVCC, const MCSymbol *UsesFlatScratch,
-    const MCSymbol *HasDynamicallySizedStack, const MCSymbol *HasRecursion,
-    const MCSymbol *HasIndirectCall) {
+    const MCSymbol *NumExplicitSGPR, const MCSymbol *NumNamedBarrier,
+    const MCSymbol *PrivateSegmentSize, const MCSymbol *UsesVCC,
+    const MCSymbol *UsesFlatScratch, const MCSymbol *HasDynamicallySizedStack,
+    const MCSymbol *HasRecursion, const MCSymbol *HasIndirectCall) {
 #define PRINT_RES_INFO(ARG)                                                    \
   OS << "\t.set ";                                                             \
   ARG->print(OS, getContext().getAsmInfo());                                   \
@@ -291,6 +290,7 @@ void AMDGPUTargetAsmStreamer::EmitMCResourceInfo(
   PRINT_RES_INFO(NumVGPR);
   PRINT_RES_INFO(NumAGPR);
   PRINT_RES_INFO(NumExplicitSGPR);
+  PRINT_RES_INFO(NumNamedBarrier);
   PRINT_RES_INFO(PrivateSegmentSize);
   PRINT_RES_INFO(UsesVCC);
   PRINT_RES_INFO(UsesFlatScratch);
@@ -396,9 +396,17 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
   EmitMCExpr(KD.kernarg_size);
   OS << '\n';
 
-  PrintField(
-      KD.compute_pgm_rsrc2, amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT,
-      amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, ".amdhsa_user_sgpr_count");
+  if (isGFX1250(STI)) {
+    PrintField(KD.compute_pgm_rsrc2,
+               amdhsa::COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
+               ".amdhsa_user_sgpr_count");
+  } else {
+    PrintField(KD.compute_pgm_rsrc2,
+               amdhsa::COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
+               ".amdhsa_user_sgpr_count");
+  }
 
   if (!hasArchitectedFlatScratch(STI))
     PrintField(
@@ -507,6 +515,12 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
     OS << '\n';
   }
 
+  if (AMDGPU::isGFX1250(STI))
+    PrintField(KD.compute_pgm_rsrc3,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
+               amdhsa::COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
+               ".amdhsa_named_barrier_count");
+
   OS << "\t\t.amdhsa_reserve_vcc ";
   EmitMCExpr(ReserveVCC);
   OS << '\n';
@@ -563,11 +577,12 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
     PrintField(KD.compute_pgm_rsrc3,
                amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT_SHIFT,
                amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, ".amdhsa_tg_split");
-  if (IVersion.Major >= 10) {
+  if (AMDGPU::supportsWGP(STI))
     PrintField(KD.compute_pgm_rsrc1,
                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE_SHIFT,
                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE,
                ".amdhsa_workgroup_processor_mode");
+  if (IVersion.Major >= 10) {
     PrintField(KD.compute_pgm_rsrc1,
                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED_SHIFT,
                amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED,
@@ -885,7 +900,7 @@ void AMDGPUTargetELFStreamer::emitAMDGPULDS(MCSymbol *Symbol, unsigned Size,
   if (!SymbolELF->isBindingSet())
     SymbolELF->setBinding(ELF::STB_GLOBAL);
 
-  if (SymbolELF->declareCommon(Size, Alignment, true)) {
+  if (SymbolELF->declareCommon(Size, Alignment)) {
     report_fatal_error("Symbol: " + Symbol->getName() +
                        " redeclared as different type");
   }
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
index 9c49020850584..22afcdebcdf09 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
@@ -62,10 +62,10 @@ class AMDGPUTargetStreamer : public MCTargetStreamer {
 
   virtual void EmitMCResourceInfo(
       const MCSymbol *NumVGPR, const MCSymbol *NumAGPR,
-      const MCSymbol *NumExplicitSGPR, const MCSymbol *PrivateSegmentSize,
-      const MCSymbol *UsesVCC, const MCSymbol *UsesFlatScratch,
-      const MCSymbol *HasDynamicallySizedStack, const MCSymbol *HasRecursion,
-      const MCSymbol *HasIndirectCall) {};
+      const MCSymbol *NumExplicitSGPR, const MCSymbol *NumNamedBarrier,
+      const MCSymbol *PrivateSegmentSize, const MCSymbol *UsesVCC,
+      const MCSymbol *UsesFlatScratch, const MCSymbol *HasDynamicallySizedStack,
+      const MCSymbol *HasRecursion, const MCSymbol *HasIndirectCall) {};
 
   virtual void EmitMCResourceMaximums(const MCSymbol *MaxVGPR,
                                       const MCSymbol *MaxAGPR,
@@ -141,14 +141,12 @@ class AMDGPUTargetAsmStreamer final : public AMDGPUTargetStreamer {
 
   void emitAMDGPULDS(MCSymbol *Sym, unsigned Size, Align Alignment) override;
 
-  void EmitMCResourceInfo(const MCSymbol *NumVGPR, const MCSymbol *NumAGPR,
-                          const MCSymbol *NumExplicitSGPR,
-                          const MCSymbol *PrivateSegmentSize,
-                          const MCSymbol *UsesVCC,
-                          const MCSymbol *UsesFlatScratch,
-                          const MCSymbol *HasDynamicallySizedStack,
-                          const MCSymbol *HasRecursion,
-                          const MCSymbol *HasIndirectCall) override;
+  void EmitMCResourceInfo(
+      const MCSymbol *NumVGPR, const MCSymbol *NumAGPR,
+      const MCSymbol *NumExplicitSGPR, const MCSymbol *NumNamedBarrier,
+      const MCSymbol *PrivateSegmentSize, const MCSymbol *UsesVCC,
+      const MCSymbol *UsesFlatScratch, const MCSymbol *HasDynamicallySizedStack,
+      const MCSymbol *HasRecursion, const MCSymbol *HasIndirectCall) override;
 
   void EmitMCResourceMaximums(const MCSymbol *MaxVGPR, const MCSymbol *MaxAGPR,
                               const MCSymbol *MaxSGPR) override;
diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
index 531fae3ceff59..ff5321df6452d 100644
--- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td
@@ -2044,9 +2044,9 @@ class VIMAGE_TENSOR_Pseudo<string opName, bit _UpTo2D = 0> :
   let hasSideEffects = 0;
 
   bit UpTo2D = _UpTo2D;
-  let InOperandList = !if(UpTo2D, (ins SReg_128:$vaddr0, SReg_256:$vaddr1, R128A16:$r128, CPol:$cpol),
-                                      (ins SReg_128:$vaddr0, SReg_256:$vaddr1, SReg_128:$vaddr2,
-                                       SReg_128:$vaddr3, R128A16:$r128, CPol:$cpol));
+  let InOperandList = !if(UpTo2D, (ins SReg_128_XNULL:$vaddr0, SReg_256_XNULL:$vaddr1, R128A16:$r128, CPol:$cpol),
+                                      (ins SReg_128_XNULL:$vaddr0, SReg_256_XNULL:$vaddr1, SReg_128_XNULL:$vaddr2,
+                                       SReg_128_XNULL:$vaddr3, R128A16:$r128, CPol:$cpol));
   string AsmOperands = " $vaddr0, $vaddr1"#!if(UpTo2D, "", ", $vaddr2, $vaddr3")#"$r128$cpol";
 }
 
diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
index 3c8bfa629ed3d..2aa54c920a046 100644
--- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -21,6 +21,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/IntrinsicsR600.h"
+#include "llvm/Passes/CodeGenPassBuilder.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Target/AMDGPU/R600MCInstLower.cpp b/llvm/lib/Target/AMDGPU/R600MCInstLower.cpp
index 6d7e834ca3d5d..48b4e7f0d07be 100644
--- a/llvm/lib/Target/AMDGPU/R600MCInstLower.cpp
+++ b/llvm/lib/Target/AMDGPU/R600MCInstLower.cpp
@@ -20,6 +20,8 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 
+using namespace llvm;
+
 namespace {
 class R600MCInstLower : public AMDGPUMCInstLower {
 public:
diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
index eff5b0aad0248..c20487ebd8fc9 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.cpp
@@ -19,6 +19,7 @@
 #include "R600MachineFunctionInfo.h"
 #include "R600MachineScheduler.h"
 #include "R600TargetTransformInfo.h"
+#include "llvm/Passes/CodeGenPassBuilder.h"
 #include "llvm/Transforms/Scalar.h"
 #include <optional>
 
@@ -46,6 +47,21 @@ static MachineSchedRegistry R600SchedRegistry("r600",
                                               "Run R600's custom scheduler",
                                               createR600MachineScheduler);
 
+//===----------------------------------------------------------------------===//
+// R600 CodeGen Pass Builder interface.
+//===----------------------------------------------------------------------===//
+
+class R600CodeGenPassBuilder
+    : public CodeGenPassBuilder<R600CodeGenPassBuilder, R600TargetMachine> {
+public:
+  R600CodeGenPassBuilder(R600TargetMachine &TM, const CGPassBuilderOption &Opts,
+                         PassInstrumentationCallbacks *PIC);
+
+  void addPreISel(AddIRPass &addPass) const;
+  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
+  Error addInstSelector(AddMachinePass &) const;
+};
+
 //===----------------------------------------------------------------------===//
 // R600 Target Machine (R600 -> Cayman)
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/R600TargetMachine.h b/llvm/lib/Target/AMDGPU/R600TargetMachine.h
index eb4cb91cb704d..7985ef136cead 100644
--- a/llvm/lib/Target/AMDGPU/R600TargetMachine.h
+++ b/llvm/lib/Target/AMDGPU/R600TargetMachine.h
@@ -57,21 +57,6 @@ class R600TargetMachine final : public AMDGPUTargetMachine {
   createMachineScheduler(MachineSchedContext *C) const override;
 };
 
-//===----------------------------------------------------------------------===//
-// R600 CodeGen Pass Builder interface.
-//===----------------------------------------------------------------------===//
-
-class R600CodeGenPassBuilder
-    : public CodeGenPassBuilder<R600CodeGenPassBuilder, R600TargetMachine> {
-public:
-  R600CodeGenPassBuilder(R600TargetMachine &TM, const CGPassBuilderOption &Opts,
-                         PassInstrumentationCallbacks *PIC);
-
-  void addPreISel(AddIRPass &addPass) const;
-  void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const;
-  Error addInstSelector(AddMachinePass &) const;
-};
-
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_AMDGPU_R600TARGETMACHINE_H
diff --git a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
index 3be7e5a16c825..f041e94c9161d 100644
--- a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
@@ -16,12 +16,14 @@
 #include "GCNSubtarget.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/UniformityAnalysis.h"
+#include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index deadb7aed0f69..268b153c6c924 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -197,7 +197,7 @@ enum ClassFlags : unsigned {
 
 namespace AMDGPU {
 enum OperandType : unsigned {
-  /// Operands with register or 32-bit immediate
+  /// Operands with register, 32-bit, or 64-bit immediate
   OPERAND_REG_IMM_INT32 = MCOI::OPERAND_FIRST_TARGET,
   OPERAND_REG_IMM_INT64,
   OPERAND_REG_IMM_INT16,
@@ -243,6 +243,10 @@ enum OperandType : unsigned {
   // Operand for SDWA instructions
   OPERAND_SDWA_VOPC_DST,
 
+  // Operand for AV_MOV_B64_IMM_PSEUDO, which is a pair of 32-bit inline
+  // constants.
+  OPERAND_INLINE_C_AV64_PSEUDO,
+
   OPERAND_REG_IMM_FIRST = OPERAND_REG_IMM_INT32,
   OPERAND_REG_IMM_LAST = OPERAND_REG_IMM_V2FP32,
 
@@ -407,7 +411,7 @@ enum CPol {
 
   SCAL = 1 << 11, // Scale offset bit
 
-  ALL = TH | SCOPE,
+  ALL = TH | SCOPE | NV,
 
   // Helper bits
   TH_TYPE_LOAD = 1 << 7,    // TH_LOAD policy
@@ -440,6 +444,7 @@ enum Id { // Message ID, width(4) [3:0].
   ID_EARLY_PRIM_DEALLOC = 8, // added in GFX9, removed in GFX10
   ID_GS_ALLOC_REQ = 9,       // added in GFX9
   ID_GET_DOORBELL = 10,      // added in GFX9, removed in GFX11
+  ID_SAVEWAVE_HAS_TDM = 10,  // added in GFX1250
   ID_GET_DDID = 11,          // added in GFX10, removed in GFX11
   ID_SYSMSG = 15,
 
@@ -513,6 +518,7 @@ enum Id { // HwRegCode, (6) [5:0]
   ID_HW_ID2 = 24,
   ID_POPS_PACKER = 25,
   ID_PERF_SNAPSHOT_DATA_gfx11 = 27,
+  ID_IB_STS2 = 28,
   ID_SHADER_CYCLES = 29,
   ID_SHADER_CYCLES_HI = 30,
   ID_DVGPR_ALLOC_LO = 31,
@@ -536,6 +542,10 @@ enum Id { // HwRegCode, (6) [5:0]
   ID_SQ_PERF_SNAPSHOT_DATA1 = 22,
   ID_SQ_PERF_SNAPSHOT_PC_LO = 23,
   ID_SQ_PERF_SNAPSHOT_PC_HI = 24,
+
+  // GFX1250
+  ID_XNACK_STATE_PRIV = 33,
+  ID_XNACK_MASK_gfx1250 = 34,
 };
 
 enum Offset : unsigned { // Offset, (5) [10:6]
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index f018f77bc83e1..dce4e6f993005 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -460,7 +460,7 @@ static bool hoistAndMergeSGPRInits(unsigned Reg,
   // List of clobbering instructions.
   SmallVector<MachineInstr*, 8> Clobbers;
   // List of instructions marked for deletion.
-  SmallSet<MachineInstr*, 8> MergedInstrs;
+  SmallPtrSet<MachineInstr *, 8> MergedInstrs;
 
   bool Changed = false;
 
@@ -808,7 +808,7 @@ bool SIFixSGPRCopies::run(MachineFunction &MF) {
 void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
   bool AllAGPRUses = true;
   SetVector<const MachineInstr *> worklist;
-  SmallSet<const MachineInstr *, 4> Visited;
+  SmallPtrSet<const MachineInstr *, 4> Visited;
   SetVector<MachineInstr *> PHIOperands;
   worklist.insert(&MI);
   Visited.insert(&MI);
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index 0c653b1b46d65..962c276bc2123 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -2081,7 +2081,9 @@ SIFoldOperandsImpl::isClamp(const MachineInstr &MI) const {
   case AMDGPU::V_MAX_F16_fake16_e64:
   case AMDGPU::V_MAX_F64_e64:
   case AMDGPU::V_MAX_NUM_F64_e64:
-  case AMDGPU::V_PK_MAX_F16: {
+  case AMDGPU::V_PK_MAX_F16:
+  case AMDGPU::V_MAX_BF16_PSEUDO_e64:
+  case AMDGPU::V_PK_MAX_NUM_BF16: {
     if (MI.mayRaiseFPException())
       return nullptr;
 
@@ -2108,8 +2110,10 @@ SIFoldOperandsImpl::isClamp(const MachineInstr &MI) const {
 
     // Having a 0 op_sel_hi would require swizzling the output in the source
     // instruction, which we can't do.
-    unsigned UnsetMods = (Op == AMDGPU::V_PK_MAX_F16) ? SISrcMods::OP_SEL_1
-                                                      : 0u;
+    unsigned UnsetMods =
+        (Op == AMDGPU::V_PK_MAX_F16 || Op == AMDGPU::V_PK_MAX_NUM_BF16)
+            ? SISrcMods::OP_SEL_1
+            : 0u;
     if (Src0Mods != UnsetMods && Src1Mods != UnsetMods)
       return nullptr;
     return Src0;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 63826b782a377..66c1dfc71c2f5 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3106,6 +3106,15 @@ SDValue SITargetLowering::LowerFormalArguments(
   if (!IsKernel) {
     CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, isVarArg);
     CCInfo.AnalyzeFormalArguments(Splits, AssignFn);
+
+    // This assumes the registers are allocated by CCInfo in ascending order
+    // with no gaps.
+    Info->setNumWaveDispatchSGPRs(
+        CCInfo.getFirstUnallocated(AMDGPU::SGPR_32RegClass.getRegisters()));
+    Info->setNumWaveDispatchVGPRs(
+        CCInfo.getFirstUnallocated(AMDGPU::VGPR_32RegClass.getRegisters()));
+  } else if (Info->getNumKernargPreloadedSGPRs()) {
+    Info->setNumWaveDispatchSGPRs(Info->getNumUserSGPRs());
   }
 
   SmallVector<SDValue, 16> Chains;
@@ -6106,6 +6115,7 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
   case MVT::f64:
     return true;
   case MVT::f16:
+  case MVT::bf16:
     return Subtarget->has16BitInsts() && !denormalModeIsFlushAllF64F16(MF);
   default:
     break;
@@ -6611,7 +6621,7 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
   unsigned SplitSize = 32;
   if (IID == Intrinsic::amdgcn_update_dpp && (ValSize % 64 == 0) &&
       ST->hasDPALU_DPP() &&
-      AMDGPU::isLegalDPALU_DPPControl(N->getConstantOperandVal(3)))
+      AMDGPU::isLegalDPALU_DPPControl(*ST, N->getConstantOperandVal(3)))
     SplitSize = 64;
 
   auto createLaneOp = [&DAG, &SL, N, IID](SDValue Src0, SDValue Src1,
@@ -10815,6 +10825,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
     return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other,
                                       Op->getOperand(2), Chain),
                    0);
+  case Intrinsic::amdgcn_s_barrier_init:
   case Intrinsic::amdgcn_s_barrier_signal_var: {
     // these two intrinsics have two operands: barrier pointer and member count
     SDValue Chain = Op->getOperand(0);
@@ -10822,6 +10833,9 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
     SDValue BarOp = Op->getOperand(2);
     SDValue CntOp = Op->getOperand(3);
     SDValue M0Val;
+    unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init
+                       ? AMDGPU::S_BARRIER_INIT_M0
+                       : AMDGPU::S_BARRIER_SIGNAL_M0;
     // extract the BarrierID from bits 4-9 of BarOp
     SDValue BarID;
     BarID = DAG.getNode(ISD::SRL, DL, MVT::i32, BarOp,
@@ -10845,8 +10859,40 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
 
     Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
 
-    auto *NewMI = DAG.getMachineNode(AMDGPU::S_BARRIER_SIGNAL_M0, DL,
-                                     Op->getVTList(), Ops);
+    auto *NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
+    return SDValue(NewMI, 0);
+  }
+  case Intrinsic::amdgcn_s_barrier_join: {
+    // these three intrinsics have one operand: barrier pointer
+    SDValue Chain = Op->getOperand(0);
+    SmallVector<SDValue, 2> Ops;
+    SDValue BarOp = Op->getOperand(2);
+    unsigned Opc;
+
+    if (isa<ConstantSDNode>(BarOp)) {
+      uint64_t BarVal = cast<ConstantSDNode>(BarOp)->getZExtValue();
+      Opc = AMDGPU::S_BARRIER_JOIN_IMM;
+
+      // extract the BarrierID from bits 4-9 of the immediate
+      unsigned BarID = (BarVal >> 4) & 0x3F;
+      SDValue K = DAG.getTargetConstant(BarID, DL, MVT::i32);
+      Ops.push_back(K);
+      Ops.push_back(Chain);
+    } else {
+      Opc = AMDGPU::S_BARRIER_JOIN_M0;
+
+      // extract the BarrierID from bits 4-9 of BarOp, copy to M0[5:0]
+      SDValue M0Val;
+      M0Val = DAG.getNode(ISD::SRL, DL, MVT::i32, BarOp,
+                          DAG.getShiftAmountConstant(4, MVT::i32, DL));
+      M0Val =
+          SDValue(DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, M0Val,
+                                     DAG.getTargetConstant(0x3F, DL, MVT::i32)),
+                  0);
+      Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
+    }
+
+    auto *NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
     return SDValue(NewMI, 0);
   }
   case Intrinsic::amdgcn_s_prefetch_data: {
@@ -10877,6 +10923,13 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
   }
 }
 
+// Return whether the operation has NoUnsignedWrap property.
+static bool isNoUnsignedWrap(SDValue Addr) {
+  return (Addr.getOpcode() == ISD::ADD &&
+          Addr->getFlags().hasNoUnsignedWrap()) ||
+         Addr->getOpcode() == ISD::OR;
+}
+
 bool SITargetLowering::shouldPreservePtrArith(const Function &F,
                                               EVT PtrVT) const {
   return UseSelectionDAGPTRADD && PtrVT == MVT::i64;
@@ -10898,8 +10951,14 @@ SITargetLowering::splitBufferOffsets(SDValue Offset, SelectionDAG &DAG) const {
   if ((C1 = dyn_cast<ConstantSDNode>(N0)))
     N0 = SDValue();
   else if (DAG.isBaseWithConstantOffset(N0)) {
-    C1 = cast<ConstantSDNode>(N0.getOperand(1));
-    N0 = N0.getOperand(0);
+    // On GFX1250+, voffset and immoffset are zero-extended from 32 bits before
+    // being added, so we can only safely match a 32-bit addition with no
+    // unsigned overflow.
+    bool CheckNUW = AMDGPU::isGFX1250(*Subtarget);
+    if (!CheckNUW || isNoUnsignedWrap(N0)) {
+      C1 = cast<ConstantSDNode>(N0.getOperand(1));
+      N0 = N0.getOperand(0);
+    }
   }
 
   if (C1) {
@@ -11481,9 +11540,22 @@ SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
     return FastLowered;
 
   SDLoc SL(Op);
+  EVT VT = Op.getValueType();
   SDValue LHS = Op.getOperand(0);
   SDValue RHS = Op.getOperand(1);
 
+  SDValue LHSExt = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, LHS);
+  SDValue RHSExt = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, RHS);
+
+  if (VT == MVT::bf16) {
+    SDValue ExtDiv =
+        DAG.getNode(ISD::FDIV, SL, MVT::f32, LHSExt, RHSExt, Op->getFlags());
+    return DAG.getNode(ISD::FP_ROUND, SL, MVT::bf16, ExtDiv,
+                       DAG.getTargetConstant(0, SL, MVT::i32));
+  }
+
+  assert(VT == MVT::f16);
+
   // a32.u = opx(V_CVT_F32_F16, a.u); // CVT to F32
   // b32.u = opx(V_CVT_F32_F16, b.u); // CVT to F32
   // r32.u = opx(V_RCP_F32, b32.u); // rcp = 1 / d
@@ -11500,9 +11572,6 @@ SDValue SITargetLowering::LowerFDIV16(SDValue Op, SelectionDAG &DAG) const {
   // We will use ISD::FMA on targets that don't support ISD::FMAD.
   unsigned FMADOpCode =
       isOperationLegal(ISD::FMAD, MVT::f32) ? ISD::FMAD : ISD::FMA;
-
-  SDValue LHSExt = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, LHS);
-  SDValue RHSExt = DAG.getNode(ISD::FP_EXTEND, SL, MVT::f32, RHS);
   SDValue NegRHSExt = DAG.getNode(ISD::FNEG, SL, MVT::f32, RHSExt);
   SDValue Rcp =
       DAG.getNode(AMDGPUISD::RCP, SL, MVT::f32, RHSExt, Op->getFlags());
@@ -15670,7 +15739,7 @@ SDValue SITargetLowering::performFDivCombine(SDNode *N,
   SelectionDAG &DAG = DCI.DAG;
   SDLoc SL(N);
   EVT VT = N->getValueType(0);
-  if (VT != MVT::f16 || !Subtarget->has16BitInsts())
+  if ((VT != MVT::f16 && VT != MVT::bf16) || !Subtarget->has16BitInsts())
     return SDValue();
 
   SDValue LHS = N->getOperand(0);
@@ -16835,6 +16904,11 @@ SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_,
 
   const TargetRegisterClass *RC = nullptr;
   if (Constraint.size() == 1) {
+    // Check if we cannot determine the bit size of the given value type.  This
+    // can happen, for example, in this situation where we have an empty struct
+    // (size 0): `call void asm "", "v"({} poison)`-
+    if (VT == MVT::Other)
+      return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
     const unsigned BitWidth = VT.getSizeInBits();
     switch (Constraint[0]) {
     default:
@@ -16883,13 +16957,26 @@ SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_,
       }
       break;
     }
-    // We actually support i128, i16 and f16 as inline parameters
-    // even if they are not reported as legal
-    if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 ||
-               VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16))
-      return std::pair(0U, RC);
+  } else if (Constraint == "VA" && Subtarget->hasGFX90AInsts()) {
+    const unsigned BitWidth = VT.getSizeInBits();
+    switch (BitWidth) {
+    case 16:
+      RC = &AMDGPU::AV_32RegClass;
+      break;
+    default:
+      RC = TRI->getVectorSuperClassForBitWidth(BitWidth);
+      if (!RC)
+        return std::pair(0U, nullptr);
+      break;
+    }
   }
 
+  // We actually support i128, i16 and f16 as inline parameters
+  // even if they are not reported as legal
+  if (RC && (isTypeLegal(VT) || VT.SimpleTy == MVT::i128 ||
+             VT.SimpleTy == MVT::i16 || VT.SimpleTy == MVT::f16))
+    return std::pair(0U, RC);
+
   auto [Kind, Idx, NumRegs] = AMDGPU::parseAsmConstraintPhysReg(Constraint);
   if (Kind != '\0') {
     if (Kind == 'v') {
@@ -16902,7 +16989,7 @@ SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_,
 
     if (RC) {
       if (NumRegs > 1) {
-        if (Idx >= RC->getNumRegs() || Idx + NumRegs - 1 > RC->getNumRegs())
+        if (Idx >= RC->getNumRegs() || Idx + NumRegs - 1 >= RC->getNumRegs())
           return std::pair(0U, nullptr);
 
         uint32_t Width = NumRegs * 32;
@@ -16974,6 +17061,9 @@ SITargetLowering::getConstraintType(StringRef Constraint) const {
     case 'a':
       return C_RegisterClass;
     }
+  } else if (Constraint.size() == 2) {
+    if (Constraint == "VA")
+      return C_RegisterClass;
   }
   if (isImmConstraint(Constraint)) {
     return C_Other;
@@ -17695,6 +17785,8 @@ static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
     if (Subtarget.supportsAgentScopeFineGrainedRemoteMemoryAtomics() &&
         RMW->hasMetadata("amdgpu.no.remote.memory"))
       return true;
+    if (Subtarget.hasEmulatedSystemScopeAtomics())
+      return true;
   } else if (Subtarget.supportsAgentScopeFineGrainedRemoteMemoryAtomics())
     return true;
 
@@ -17711,23 +17803,9 @@ atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
 
 /// Return if a flat address space atomicrmw can access private memory.
 static bool flatInstrMayAccessPrivate(const Instruction *I) {
-  const MDNode *NoaliasAddrSpaceMD =
-      I->getMetadata(LLVMContext::MD_noalias_addrspace);
-  if (!NoaliasAddrSpaceMD)
-    return true;
-
-  for (unsigned I = 0, E = NoaliasAddrSpaceMD->getNumOperands() / 2; I != E;
-       ++I) {
-    auto *Low = mdconst::extract<ConstantInt>(
-        NoaliasAddrSpaceMD->getOperand(2 * I + 0));
-    if (Low->getValue().uge(AMDGPUAS::PRIVATE_ADDRESS)) {
-      auto *High = mdconst::extract<ConstantInt>(
-          NoaliasAddrSpaceMD->getOperand(2 * I + 1));
-      return High->getValue().ule(AMDGPUAS::PRIVATE_ADDRESS);
-    }
-  }
-
-  return true;
+  const MDNode *MD = I->getMetadata(LLVMContext::MD_noalias_addrspace);
+  return !MD ||
+         !AMDGPU::hasValueInRangeLikeMetadata(*MD, AMDGPUAS::PRIVATE_ADDRESS);
 }
 
 TargetLowering::AtomicExpansionKind
@@ -17762,26 +17840,83 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
 
   auto Op = RMW->getOperation();
   switch (Op) {
-  case AtomicRMWInst::Xchg: {
+  case AtomicRMWInst::Xchg:
     // PCIe supports add and xchg for system atomics.
     return isAtomicRMWLegalXChgTy(RMW)
                ? TargetLowering::AtomicExpansionKind::None
                : TargetLowering::AtomicExpansionKind::CmpXChg;
-  }
   case AtomicRMWInst::Add:
-  case AtomicRMWInst::And:
-  case AtomicRMWInst::UIncWrap:
-  case AtomicRMWInst::UDecWrap:
+    // PCIe supports add and xchg for system atomics.
     return atomicSupportedIfLegalIntType(RMW);
   case AtomicRMWInst::Sub:
+  case AtomicRMWInst::And:
   case AtomicRMWInst::Or:
-  case AtomicRMWInst::Xor: {
-    // Atomic sub/or/xor do not work over PCI express, but atomic add
-    // does. InstCombine transforms these with 0 to or, so undo that.
-    if (HasSystemScope && AMDGPU::isFlatGlobalAddrSpace(AS)) {
-      if (Constant *ConstVal = dyn_cast<Constant>(RMW->getValOperand());
-          ConstVal && ConstVal->isNullValue())
-        return AtomicExpansionKind::Expand;
+  case AtomicRMWInst::Xor:
+  case AtomicRMWInst::Max:
+  case AtomicRMWInst::Min:
+  case AtomicRMWInst::UMax:
+  case AtomicRMWInst::UMin:
+  case AtomicRMWInst::UIncWrap:
+  case AtomicRMWInst::UDecWrap: {
+    if (AMDGPU::isFlatGlobalAddrSpace(AS) ||
+        AS == AMDGPUAS::BUFFER_FAT_POINTER) {
+      if (Subtarget->hasEmulatedSystemScopeAtomics())
+        return atomicSupportedIfLegalIntType(RMW);
+
+      // On most subtargets, for atomicrmw operations other than add/xchg,
+      // whether or not the instructions will behave correctly depends on where
+      // the address physically resides and what interconnect is used in the
+      // system configuration. On some some targets the instruction will nop,
+      // and in others synchronization will only occur at degraded device scope.
+      //
+      // If the allocation is known local to the device, the instructions should
+      // work correctly.
+      if (RMW->hasMetadata("amdgpu.no.remote.memory"))
+        return atomicSupportedIfLegalIntType(RMW);
+
+      // If fine-grained remote memory works at device scope, we don't need to
+      // do anything.
+      if (!HasSystemScope &&
+          Subtarget->supportsAgentScopeFineGrainedRemoteMemoryAtomics())
+        return atomicSupportedIfLegalIntType(RMW);
+
+      // If we are targeting a remote allocated address, it depends what kind of
+      // allocation the address belongs to.
+      //
+      // If the allocation is fine-grained (in host memory, or in PCIe peer
+      // device memory), the operation will fail depending on the target.
+      //
+      // Note fine-grained host memory access does work on APUs or if XGMI is
+      // used, but we do not know if we are targeting an APU or the system
+      // configuration from the ISA version/target-cpu.
+      if (RMW->hasMetadata("amdgpu.no.fine.grained.memory"))
+        return atomicSupportedIfLegalIntType(RMW);
+
+      if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
+          Op == AtomicRMWInst::Xor) {
+        // Atomic sub/or/xor do not work over PCI express, but atomic add
+        // does. InstCombine transforms these with 0 to or, so undo that.
+        if (Constant *ConstVal = dyn_cast<Constant>(RMW->getValOperand());
+            ConstVal && ConstVal->isNullValue())
+          return AtomicExpansionKind::Expand;
+      }
+
+      // If the allocation could be in remote, fine-grained memory, the rmw
+      // instructions may fail. cmpxchg should work, so emit that. On some
+      // system configurations, PCIe atomics aren't supported so cmpxchg won't
+      // even work, so you're out of luck anyway.
+
+      // In summary:
+      //
+      // Cases that may fail:
+      //   - fine-grained pinned host memory
+      //   - fine-grained migratable host memory
+      //   - fine-grained PCIe peer device
+      //
+      // Cases that should work, but may be treated overly conservatively.
+      //   - fine-grained host memory on an APU
+      //   - fine-grained XGMI peer device
+      return AtomicExpansionKind::CmpXChg;
     }
 
     return atomicSupportedIfLegalIntType(RMW);
@@ -17936,19 +18071,6 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
 
     return AtomicExpansionKind::CmpXChg;
   }
-  case AtomicRMWInst::Min:
-  case AtomicRMWInst::Max:
-  case AtomicRMWInst::UMin:
-  case AtomicRMWInst::UMax: {
-    if (AMDGPU::isFlatGlobalAddrSpace(AS) ||
-        AS == AMDGPUAS::BUFFER_FAT_POINTER) {
-      // Always expand system scope min/max atomics.
-      if (HasSystemScope)
-        return AtomicExpansionKind::CmpXChg;
-    }
-
-    return atomicSupportedIfLegalIntType(RMW);
-  }
   case AtomicRMWInst::Nand:
   case AtomicRMWInst::FSub:
   default:
diff --git a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
index d8fe8505bc722..0a68512668c7d 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertHardClauses.cpp
@@ -51,7 +51,7 @@ static cl::opt<unsigned>
 namespace {
 
 enum HardClauseType {
-  // For GFX10:
+  // For GFX10 and GFX1250:
 
   // Texture, buffer, global or scratch memory instructions.
   HARDCLAUSE_VMEM,
@@ -102,7 +102,8 @@ class SIInsertHardClauses {
 
   HardClauseType getHardClauseType(const MachineInstr &MI) {
     if (MI.mayLoad() || (MI.mayStore() && ST->shouldClusterStores())) {
-      if (ST->getGeneration() == AMDGPUSubtarget::GFX10) {
+      if (ST->getGeneration() == AMDGPUSubtarget::GFX10 ||
+          ST->hasGFX1250Insts()) {
         if ((SIInstrInfo::isVMEM(MI) && !SIInstrInfo::isFLAT(MI)) ||
             SIInstrInfo::isSegmentSpecificFLAT(MI)) {
           if (ST->hasNSAClauseBug()) {
@@ -115,7 +116,6 @@ class SIInsertHardClauses {
         if (SIInstrInfo::isFLAT(MI))
           return HARDCLAUSE_FLAT;
       } else {
-        assert(ST->getGeneration() >= AMDGPUSubtarget::GFX11);
         if (SIInstrInfo::isMIMG(MI)) {
           const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI.getOpcode());
           const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 4b48fc4ab8824..e3a2efdd3856f 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -32,11 +32,15 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/Support/DebugCounter.h"
 #include "llvm/TargetParser/TargetParser.h"
+
 using namespace llvm;
 
 #define DEBUG_TYPE "si-insert-waitcnts"
@@ -2010,11 +2014,19 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
     }
   }
 
-  // The subtarget may have an implicit S_WAITCNT 0 before barriers. If it does
-  // not, we need to ensure the subtarget is capable of backing off barrier
-  // instructions in case there are any outstanding memory operations that may
-  // cause an exception. Otherwise, insert an explicit S_WAITCNT 0 here.
-  if (TII->isBarrierStart(MI.getOpcode()) &&
+  // Ensure safety against exceptions from outstanding memory operations while
+  // waiting for a barrier:
+  //
+  //  * Some subtargets safely handle backing off the barrier in hardware
+  //    when an exception occurs.
+  //  * Some subtargets have an implicit S_WAITCNT 0 before barriers, so that
+  //    there can be no outstanding memory operations during the wait.
+  //  * Subtargets with split barriers don't need to back off the barrier; it
+  //    is up to the trap handler to preserve the user barrier state correctly.
+  //
+  // In all other cases, ensure safety by ensuring that there are no outstanding
+  // memory operations.
+  if (MI.getOpcode() == AMDGPU::S_BARRIER &&
       !ST->hasAutoWaitcntBeforeBarrier() && !ST->supportsBackOffBarrier()) {
     Wait = Wait.combined(WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/true));
   }
@@ -2341,6 +2353,7 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
     case AMDGPU::S_MEMREALTIME:
     case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
     case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
+    case AMDGPU::S_BARRIER_LEAVE:
     case AMDGPU::S_GET_BARRIER_STATE_M0:
     case AMDGPU::S_GET_BARRIER_STATE_IMM:
       ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst);
diff --git a/llvm/lib/Target/AMDGPU/SIInstrFormats.td b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
index 89d9b0d32b25b..50964a94d6e58 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrFormats.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
@@ -473,6 +473,7 @@ class VIMAGE_VSAMPLE_Common <bits<8> op> : Enc96 {
   let Inst{4} = r128;
   let Inst{5} = d16;
   let Inst{6} = a16;
+  let Inst{7} = cpol{5}; // nv
   let Inst{21-14} = op;
   let Inst{25-22} = dmask;
   let Inst{39-32} = vdata;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 3f61bbd1d6e85..ba37bdb203a7f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -18,6 +18,7 @@
 #include "GCNSubtarget.h"
 #include "SIMachineFunctionInfo.h"
 #include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/LiveIntervals.h"
@@ -1348,6 +1349,7 @@ bool SIInstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
   case AMDGPU::V_MOV_B64_e32:
   case AMDGPU::V_ACCVGPR_WRITE_B32_e64:
   case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
+  case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
   case AMDGPU::S_MOV_B64_IMM_PSEUDO:
   case AMDGPU::V_MOV_B64_PSEUDO: {
     const MachineOperand &Src0 = MI.getOperand(1);
@@ -2132,6 +2134,25 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
         get(IsAGPR ? AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::V_MOV_B32_e32));
     break;
   }
+  case AMDGPU::AV_MOV_B64_IMM_PSEUDO: {
+    Register Dst = MI.getOperand(0).getReg();
+    if (SIRegisterInfo::isAGPRClass(RI.getPhysRegBaseClass(Dst))) {
+      int64_t Imm = MI.getOperand(1).getImm();
+
+      Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
+      Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
+      BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DstLo)
+          .addImm(SignExtend64<32>(Imm))
+          .addReg(Dst, RegState::Implicit | RegState::Define);
+      BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32_e64), DstHi)
+          .addImm(SignExtend64<32>(Imm >> 32))
+          .addReg(Dst, RegState::Implicit | RegState::Define);
+      MI.eraseFromParent();
+      break;
+    }
+
+    [[fallthrough]];
+  }
   case AMDGPU::V_MOV_B64_PSEUDO: {
     Register Dst = MI.getOperand(0).getReg();
     Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
@@ -2615,9 +2636,9 @@ std::pair<MachineInstr*, MachineInstr*>
 SIInstrInfo::expandMovDPP64(MachineInstr &MI) const {
   assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
 
-  if (ST.hasMovB64() &&
+  if (ST.hasMovB64() && ST.hasFeature(AMDGPU::FeatureDPALU_DPP) &&
       AMDGPU::isLegalDPALU_DPPControl(
-        getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) {
+          ST, getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl)->getImm())) {
     MI.setDesc(get(AMDGPU::V_MOV_B64_dpp));
     return std::pair(&MI, nullptr);
   }
@@ -2904,7 +2925,6 @@ void SIInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
                                        MachineBasicBlock &RestoreBB,
                                        const DebugLoc &DL, int64_t BrOffset,
                                        RegScavenger *RS) const {
-  assert(RS && "RegScavenger required for long branching");
   assert(MBB.empty() &&
          "new block should be inserted for expanding unconditional branch");
   assert(MBB.pred_size() == 1);
@@ -3425,6 +3445,11 @@ bool SIInstrInfo::isFoldableCopy(const MachineInstr &MI) {
   case AMDGPU::V_ACCVGPR_MOV_B32:
   case AMDGPU::AV_MOV_B32_IMM_PSEUDO:
     return true;
+  case AMDGPU::AV_MOV_B64_IMM_PSEUDO:
+    // TODO: We could fold this, but it's a strange case. The immediate value
+    // can't be directly folded into any real use. We would have to spread new
+    // immediate legality checks around and only accept subregister extracts for
+    // profitability.
   default:
     return false;
   }
@@ -3450,9 +3475,9 @@ std::optional<int64_t> SIInstrInfo::extractSubregFromImm(int64_t Imm,
   case AMDGPU::NoSubRegister:
     return Imm;
   case AMDGPU::sub0:
-    return Lo_32(Imm);
+    return SignExtend64<32>(Imm);
   case AMDGPU::sub1:
-    return Hi_32(Imm);
+    return SignExtend64<32>(Imm >> 32);
   case AMDGPU::lo16:
     return SignExtend64<16>(Imm);
   case AMDGPU::hi16:
@@ -3548,54 +3573,93 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI,
     assert(!UseMI.getOperand(0).getSubReg() && "Expected SSA form");
 
     Register DstReg = UseMI.getOperand(0).getReg();
-    unsigned OpSize = getOpSize(UseMI, 0);
-    bool Is16Bit = OpSize == 2;
-    bool Is64Bit = OpSize == 8;
-    bool isVGPRCopy = RI.isVGPR(*MRI, DstReg);
-    unsigned NewOpc = isVGPRCopy ? Is64Bit ? AMDGPU::V_MOV_B64_PSEUDO
-                                           : AMDGPU::V_MOV_B32_e32
-                                 : Is64Bit ? AMDGPU::S_MOV_B64_IMM_PSEUDO
-                                           : AMDGPU::S_MOV_B32;
-
-    std::optional<int64_t> SubRegImm =
-        extractSubregFromImm(Imm, UseMI.getOperand(1).getSubReg());
-
-    APInt Imm(Is64Bit ? 64 : 32, *SubRegImm,
-              /*isSigned=*/true, /*implicitTrunc=*/true);
-
-    if (RI.isAGPR(*MRI, DstReg)) {
-      if (Is64Bit || !isInlineConstant(Imm))
-        return false;
-      NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32_e64;
-    }
+    Register UseSubReg = UseMI.getOperand(1).getSubReg();
+
+    const TargetRegisterClass *DstRC = RI.getRegClassForReg(*MRI, DstReg);
+
+    bool Is16Bit = UseSubReg != AMDGPU::NoSubRegister &&
+                   RI.getSubRegIdxSize(UseSubReg) == 16;
 
     if (Is16Bit) {
-      if (isVGPRCopy)
+      if (RI.hasVGPRs(DstRC))
         return false; // Do not clobber vgpr_hi16
 
-      if (DstReg.isVirtual() && UseMI.getOperand(0).getSubReg() != AMDGPU::lo16)
+      if (DstReg.isVirtual() && UseSubReg != AMDGPU::lo16)
         return false;
-
-      UseMI.getOperand(0).setSubReg(0);
-      if (DstReg.isPhysical()) {
-        DstReg = RI.get32BitRegister(DstReg);
-        UseMI.getOperand(0).setReg(DstReg);
-      }
-      assert(UseMI.getOperand(1).getReg().isVirtual());
     }
 
     MachineFunction *MF = UseMI.getMF();
-    const MCInstrDesc &NewMCID = get(NewOpc);
-    const TargetRegisterClass *NewDefRC = getRegClass(NewMCID, 0, &RI, *MF);
 
-    if (DstReg.isPhysical()) {
-      if (!NewDefRC->contains(DstReg))
-        return false;
-    } else if (!MRI->constrainRegClass(DstReg, NewDefRC))
+    unsigned NewOpc = AMDGPU::INSTRUCTION_LIST_END;
+    MCRegister MovDstPhysReg =
+        DstReg.isPhysical() ? DstReg.asMCReg() : MCRegister();
+
+    std::optional<int64_t> SubRegImm = extractSubregFromImm(Imm, UseSubReg);
+
+    // TODO: Try to fold with AMDGPU::V_MOV_B16_t16_e64
+    for (unsigned MovOp :
+         {AMDGPU::S_MOV_B32, AMDGPU::V_MOV_B32_e32, AMDGPU::S_MOV_B64,
+          AMDGPU::V_MOV_B64_PSEUDO, AMDGPU::V_ACCVGPR_WRITE_B32_e64}) {
+      const MCInstrDesc &MovDesc = get(MovOp);
+
+      const TargetRegisterClass *MovDstRC = getRegClass(MovDesc, 0, &RI, *MF);
+      if (Is16Bit) {
+        // We just need to find a correctly sized register class, so the
+        // subregister index compatibility doesn't matter since we're statically
+        // extracting the immediate value.
+        MovDstRC = RI.getMatchingSuperRegClass(MovDstRC, DstRC, AMDGPU::lo16);
+        if (!MovDstRC)
+          continue;
+
+        if (MovDstPhysReg) {
+          // FIXME: We probably should not do this. If there is a live value in
+          // the high half of the register, it will be corrupted.
+          MovDstPhysReg =
+              RI.getMatchingSuperReg(MovDstPhysReg, AMDGPU::lo16, MovDstRC);
+          if (!MovDstPhysReg)
+            continue;
+        }
+      }
+
+      // Result class isn't the right size, try the next instruction.
+      if (MovDstPhysReg) {
+        if (!MovDstRC->contains(MovDstPhysReg))
+          return false;
+      } else if (!MRI->constrainRegClass(DstReg, MovDstRC)) {
+        // TODO: This will be overly conservative in the case of 16-bit virtual
+        // SGPRs. We could hack up the virtual register uses to use a compatible
+        // 32-bit class.
+        continue;
+      }
+
+      const MCOperandInfo &OpInfo = MovDesc.operands()[1];
+
+      // Ensure the interpreted immediate value is a valid operand in the new
+      // mov.
+      //
+      // FIXME: isImmOperandLegal should have form that doesn't require existing
+      // MachineInstr or MachineOperand
+      if (!RI.opCanUseLiteralConstant(OpInfo.OperandType) &&
+          !isInlineConstant(*SubRegImm, OpInfo.OperandType))
+        break;
+
+      NewOpc = MovOp;
+      break;
+    }
+
+    if (NewOpc == AMDGPU::INSTRUCTION_LIST_END)
       return false;
 
+    if (Is16Bit) {
+      UseMI.getOperand(0).setSubReg(AMDGPU::NoSubRegister);
+      if (MovDstPhysReg)
+        UseMI.getOperand(0).setReg(MovDstPhysReg);
+      assert(UseMI.getOperand(1).getReg().isVirtual());
+    }
+
+    const MCInstrDesc &NewMCID = get(NewOpc);
     UseMI.setDesc(NewMCID);
-    UseMI.getOperand(1).ChangeToImmediate(Imm.getSExtValue());
+    UseMI.getOperand(1).ChangeToImmediate(*SubRegImm);
     UseMI.addImplicitDefUseOperands(*MF);
     return true;
   }
@@ -4240,6 +4304,7 @@ bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
          MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
          MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
          MI.getOpcode() == AMDGPU::S_SETPRIO ||
+         MI.getOpcode() == AMDGPU::S_SETPRIO_INC_WG ||
          changesVGPRIndexingMode(MI);
 }
 
@@ -4266,12 +4331,15 @@ bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const {
   if (MI.memoperands_empty())
     return true;
 
-  // TODO (?): Does this need to be taught how to read noalias.addrspace ?
-
   // See if any memory operand specifies an address space that involves scratch.
   return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) {
     unsigned AS = Memop->getAddrSpace();
-    return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
+    if (AS == AMDGPUAS::FLAT_ADDRESS) {
+      const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace;
+      return !MD || !AMDGPU::hasValueInRangeLikeMetadata(
+                        *MD, AMDGPUAS::PRIVATE_ADDRESS);
+    }
+    return AS == AMDGPUAS::PRIVATE_ADDRESS;
   });
 }
 
@@ -4467,6 +4535,8 @@ bool SIInstrInfo::isInlineConstant(int64_t Imm, uint8_t OperandType) const {
   case AMDGPU::OPERAND_KIMM16:
   case AMDGPU::OPERAND_KIMM64:
     return false;
+  case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
+    return isLegalAV64PseudoImm(Imm);
   case AMDGPU::OPERAND_INPUT_MODS:
   case MCOI::OPERAND_IMMEDIATE:
     // Always embedded in the instruction for free.
@@ -4532,6 +4602,12 @@ bool SIInstrInfo::isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
   return ST.hasVOP3Literal();
 }
 
+bool SIInstrInfo::isLegalAV64PseudoImm(uint64_t Imm) const {
+  // 2 32-bit inline constants packed into one.
+  return AMDGPU::isInlinableLiteral32(Lo_32(Imm), ST.hasInv2PiInlineImm()) &&
+         AMDGPU::isInlinableLiteral32(Hi_32(Imm), ST.hasInv2PiInlineImm());
+}
+
 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
   // GFX90A does not have V_MUL_LEGACY_F32_e32.
   if (Opcode == AMDGPU::V_MUL_LEGACY_F32_e64 && ST.hasGFX90AInsts())
@@ -4857,7 +4933,8 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
 
     int RegClass = Desc.operands()[i].RegClass;
 
-    switch (Desc.operands()[i].OperandType) {
+    const MCOperandInfo &OpInfo = Desc.operands()[i];
+    switch (OpInfo.OperandType) {
     case MCOI::OPERAND_REGISTER:
       if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
         ErrInfo = "Illegal immediate value for operand.";
@@ -4865,15 +4942,31 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
       }
       break;
     case AMDGPU::OPERAND_REG_IMM_INT32:
+    case AMDGPU::OPERAND_REG_IMM_INT64:
+    case AMDGPU::OPERAND_REG_IMM_INT16:
     case AMDGPU::OPERAND_REG_IMM_FP32:
     case AMDGPU::OPERAND_REG_IMM_V2FP32:
+    case AMDGPU::OPERAND_REG_IMM_BF16:
+    case AMDGPU::OPERAND_REG_IMM_FP16:
+    case AMDGPU::OPERAND_REG_IMM_FP64:
+    case AMDGPU::OPERAND_REG_IMM_V2FP16:
+    case AMDGPU::OPERAND_REG_IMM_V2INT16:
+    case AMDGPU::OPERAND_REG_IMM_V2INT32:
+    case AMDGPU::OPERAND_REG_IMM_V2BF16:
       break;
+    case AMDGPU::OPERAND_REG_IMM_NOINLINE_V2FP16:
+      break;
+      break;
+    case AMDGPU::OPERAND_REG_INLINE_C_INT16:
     case AMDGPU::OPERAND_REG_INLINE_C_INT32:
-    case AMDGPU::OPERAND_REG_INLINE_C_FP32:
     case AMDGPU::OPERAND_REG_INLINE_C_INT64:
+    case AMDGPU::OPERAND_REG_INLINE_C_FP32:
     case AMDGPU::OPERAND_REG_INLINE_C_FP64:
-    case AMDGPU::OPERAND_REG_INLINE_C_INT16:
+    case AMDGPU::OPERAND_REG_INLINE_C_BF16:
     case AMDGPU::OPERAND_REG_INLINE_C_FP16:
+    case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
+    case AMDGPU::OPERAND_REG_INLINE_C_V2BF16:
+    case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
     case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
     case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
     case AMDGPU::OPERAND_REG_INLINE_AC_FP64: {
@@ -4889,9 +4982,14 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
         return false;
       }
       break;
+    case AMDGPU::OPERAND_INPUT_MODS:
+    case AMDGPU::OPERAND_SDWA_VOPC_DST:
+    case AMDGPU::OPERAND_KIMM16:
+      break;
     case MCOI::OPERAND_IMMEDIATE:
     case AMDGPU::OPERAND_KIMM32:
     case AMDGPU::OPERAND_KIMM64:
+    case AMDGPU::OPERAND_INLINE_C_AV64_PSEUDO:
       // Check if this operand is an immediate.
       // FrameIndex operands will be replaced by immediates, so they are
       // allowed.
@@ -4899,9 +4997,15 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
         ErrInfo = "Expected immediate, but got non-immediate";
         return false;
       }
-      [[fallthrough]];
+      break;
+    case MCOI::OPERAND_UNKNOWN:
+    case MCOI::OPERAND_MEMORY:
+    case MCOI::OPERAND_PCREL:
+      break;
     default:
-      continue;
+      if (OpInfo.isGenericType())
+        continue;
+      break;
     }
 
     if (!MO.isReg())
@@ -5432,7 +5536,8 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
     }
 
     if (Opcode != AMDGPU::V_MOV_B64_DPP_PSEUDO &&
-        !AMDGPU::isLegalDPALU_DPPControl(DC) && AMDGPU::isDPALU_DPP(Desc)) {
+        !AMDGPU::isLegalDPALU_DPPControl(ST, DC) &&
+        AMDGPU::isDPALU_DPP(Desc, ST)) {
       ErrInfo = "Invalid dpp_ctrl value: "
                 "DP ALU dpp only support row_newbcast";
       return false;
@@ -5534,6 +5639,15 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
     }
   }
 
+  // See SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand for more
+  // information.
+  if (AMDGPU::isPackedFP32Inst(Opcode) && AMDGPU::isGFX12Plus(ST)) {
+    for (unsigned I = 0; I < 3; ++I) {
+      if (!isLegalGFX12PlusPackedMathFP32Operand(MRI, MI, I))
+        return false;
+    }
+  }
+
   return true;
 }
 
@@ -6005,6 +6119,21 @@ bool SIInstrInfo::isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
   const MCOperandInfo OpInfo = MI.getDesc().operands()[OpIdx];
   unsigned Opc = MI.getOpcode();
 
+  // See SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand for more
+  // information.
+  if (AMDGPU::isPackedFP32Inst(MI.getOpcode()) && AMDGPU::isGFX12Plus(ST) &&
+      MO.isReg() && RI.isSGPRReg(MRI, MO.getReg())) {
+    constexpr const AMDGPU::OpName OpNames[] = {
+        AMDGPU::OpName::src0, AMDGPU::OpName::src1, AMDGPU::OpName::src2};
+
+    for (auto [I, OpName] : enumerate(OpNames)) {
+      int SrcIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[I]);
+      if (static_cast<unsigned>(SrcIdx) == OpIdx &&
+          !isLegalGFX12PlusPackedMathFP32Operand(MRI, MI, I, &MO))
+        return false;
+    }
+  }
+
   if (!isLegalRegOperand(MRI, OpInfo, MO))
     return false;
 
@@ -6053,6 +6182,39 @@ bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI,
   return true;
 }
 
+bool SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand(
+    const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
+    const MachineOperand *MO) const {
+  constexpr const unsigned NumOps = 3;
+  constexpr const AMDGPU::OpName OpNames[NumOps * 2] = {
+      AMDGPU::OpName::src0,           AMDGPU::OpName::src1,
+      AMDGPU::OpName::src2,           AMDGPU::OpName::src0_modifiers,
+      AMDGPU::OpName::src1_modifiers, AMDGPU::OpName::src2_modifiers};
+
+  assert(SrcN < NumOps);
+
+  if (!MO) {
+    int SrcIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[SrcN]);
+    if (SrcIdx == -1)
+      return true;
+    MO = &MI.getOperand(SrcIdx);
+  }
+
+  if (!MO->isReg() || !RI.isSGPRReg(MRI, MO->getReg()))
+    return true;
+
+  int ModsIdx =
+      AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpNames[NumOps + SrcN]);
+  if (ModsIdx == -1)
+    return true;
+
+  unsigned Mods = MI.getOperand(ModsIdx).getImm();
+  bool OpSel = Mods & SISrcMods::OP_SEL_0;
+  bool OpSelHi = Mods & SISrcMods::OP_SEL_1;
+
+  return !OpSel && !OpSelHi;
+}
+
 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
                                  const MachineOperand *MO) const {
   const MachineFunction &MF = *MI.getParent()->getParent();
@@ -6122,10 +6284,11 @@ bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
           !Op.isIdenticalTo(*MO))
         return false;
 
-      // Do not fold a frame index into an instruction that already has a frame
-      // index. The frame index handling code doesn't handle fixing up operand
-      // constraints if there are multiple indexes.
-      if (Op.isFI() && MO->isFI())
+      // Do not fold a non-inlineable and non-register operand into an
+      // instruction that already has a frame index. The frame index handling
+      // code could not handle well when a frame index co-exists with another
+      // non-register operand, unless that operand is an inlineable immediate.
+      if (Op.isFI())
         return false;
     }
   } else if (IsInlineConst && ST.hasNoF16PseudoScalarTransInlineConstants() &&
@@ -6389,6 +6552,15 @@ void SIInstrInfo::legalizeOperandsVOP3(MachineRegisterInfo &MRI,
   if ((Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
       !RI.isVGPR(MRI, MI.getOperand(VOP3Idx[2]).getReg()))
     legalizeOpWithMove(MI, VOP3Idx[2]);
+
+  // Fix the register class of packed FP32 instructions on gfx12+. See
+  // SIInstrInfo::isLegalGFX12PlusPackedMathFP32Operand for more information.
+  if (AMDGPU::isPackedFP32Inst(Opc) && AMDGPU::isGFX12Plus(ST)) {
+    for (unsigned I = 0; I < 3; ++I) {
+      if (!isLegalGFX12PlusPackedMathFP32Operand(MRI, MI, /*SrcN=*/I))
+        legalizeOpWithMove(MI, VOP3Idx[I]);
+    }
+  }
 }
 
 Register SIInstrInfo::readlaneVGPRToSGPR(
@@ -9157,7 +9329,7 @@ bool SIInstrInfo::isHighLatencyDef(int Opc) const {
          (isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc) || isFLAT(Opc));
 }
 
-unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI,
+Register SIInstrInfo::isStackAccess(const MachineInstr &MI,
                                     int &FrameIndex) const {
   const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
   if (!Addr || !Addr->isFI())
@@ -9170,7 +9342,7 @@ unsigned SIInstrInfo::isStackAccess(const MachineInstr &MI,
   return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
 }
 
-unsigned SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI,
+Register SIInstrInfo::isSGPRStackAccess(const MachineInstr &MI,
                                         int &FrameIndex) const {
   const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
   assert(Addr && Addr->isFI());
@@ -10073,7 +10245,30 @@ unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
 
 InstructionUniformity
 SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const {
+  const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
   unsigned opcode = MI.getOpcode();
+
+  auto HandleAddrSpaceCast = [this, &MRI](const MachineInstr &MI) {
+    Register Dst = MI.getOperand(0).getReg();
+    Register Src = isa<GIntrinsic>(MI) ? MI.getOperand(2).getReg()
+                                       : MI.getOperand(1).getReg();
+    LLT DstTy = MRI.getType(Dst);
+    LLT SrcTy = MRI.getType(Src);
+    unsigned DstAS = DstTy.getAddressSpace();
+    unsigned SrcAS = SrcTy.getAddressSpace();
+    return SrcAS == AMDGPUAS::PRIVATE_ADDRESS &&
+                   DstAS == AMDGPUAS::FLAT_ADDRESS &&
+                   ST.hasGloballyAddressableScratch()
+               ? InstructionUniformity::NeverUniform
+               : InstructionUniformity::Default;
+  };
+
+  // If the target supports globally addressable scratch, the mapping from
+  // scratch memory to the flat aperture changes therefore an address space cast
+  // is no longer uniform.
+  if (opcode == TargetOpcode::G_ADDRSPACE_CAST)
+    return HandleAddrSpaceCast(MI);
+
   if (auto *GI = dyn_cast<GIntrinsic>(&MI)) {
     auto IID = GI->getIntrinsicID();
     if (AMDGPU::isIntrinsicSourceOfDivergence(IID))
@@ -10082,6 +10277,8 @@ SIInstrInfo::getGenericInstructionUniformity(const MachineInstr &MI) const {
       return InstructionUniformity::AlwaysUniform;
 
     switch (IID) {
+    case Intrinsic::amdgcn_addrspacecast_nonnull:
+      return HandleAddrSpaceCast(MI);
     case Intrinsic::amdgcn_if:
     case Intrinsic::amdgcn_else:
       // FIXME: Uniform if second result
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index e042b59eb0f04..fdbd9ce4a66bf 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -983,19 +983,18 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
     return MI.getDesc().TSFlags & SIInstrFlags::IsNeverUniform;
   }
 
-  // Check to see if opcode is for a barrier start. Pre gfx12 this is just the
-  // S_BARRIER, but after support for S_BARRIER_SIGNAL* / S_BARRIER_WAIT we want
-  // to check for the barrier start (S_BARRIER_SIGNAL*)
-  bool isBarrierStart(unsigned Opcode) const {
+  bool isBarrier(unsigned Opcode) const {
     return Opcode == AMDGPU::S_BARRIER ||
            Opcode == AMDGPU::S_BARRIER_SIGNAL_M0 ||
            Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0 ||
            Opcode == AMDGPU::S_BARRIER_SIGNAL_IMM ||
-           Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM;
-  }
-
-  bool isBarrier(unsigned Opcode) const {
-    return isBarrierStart(Opcode) || Opcode == AMDGPU::S_BARRIER_WAIT ||
+           Opcode == AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM ||
+           Opcode == AMDGPU::S_BARRIER_WAIT ||
+           Opcode == AMDGPU::S_BARRIER_INIT_M0 ||
+           Opcode == AMDGPU::S_BARRIER_INIT_IMM ||
+           Opcode == AMDGPU::S_BARRIER_JOIN_IMM ||
+           Opcode == AMDGPU::S_BARRIER_LEAVE ||
+           Opcode == AMDGPU::S_BARRIER_LEAVE_IMM ||
            Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER;
   }
 
@@ -1051,7 +1050,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
     }
   }
 
-  bool isWaitcnt(unsigned Opcode) const {
+  static bool isWaitcnt(unsigned Opcode) {
     switch (getNonSoftWaitcntOpcode(Opcode)) {
     case AMDGPU::S_WAITCNT:
     case AMDGPU::S_WAITCNT_VSCNT:
@@ -1178,6 +1177,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
   bool isImmOperandLegal(const MachineInstr &MI, unsigned OpNo,
                          const MachineOperand &MO) const;
 
+  /// Check if this immediate value can be used for AV_MOV_B64_IMM_PSEUDO.
+  bool isLegalAV64PseudoImm(uint64_t Imm) const;
+
   /// Return true if this 64-bit VALU instruction has a 32-bit encoding.
   /// This function will return false if you pass it a 32-bit instruction.
   bool hasVALU32BitEncoding(unsigned Opcode) const;
@@ -1287,6 +1289,19 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
                          const MachineOperand &MO) const;
   bool isLegalRegOperand(const MachineInstr &MI, unsigned OpIdx,
                          const MachineOperand &MO) const;
+
+  /// Check if \p MO would be a legal operand for gfx12+ packed math FP32
+  /// instructions. Packed math FP32 instructions typically accept SGPRs or
+  /// VGPRs as source operands. On gfx12+, if a source operand uses SGPRs, the
+  /// HW can only read the first SGPR and use it for both the low and high
+  /// operations.
+  /// \p SrcN can be 0, 1, or 2, representing src0, src1, and src2,
+  /// respectively. If \p MO is nullptr, the operand corresponding to SrcN will
+  /// be used.
+  bool isLegalGFX12PlusPackedMathFP32Operand(
+      const MachineRegisterInfo &MRI, const MachineInstr &MI, unsigned SrcN,
+      const MachineOperand *MO = nullptr) const;
+
   /// Legalize operands in \p MI by either commuting it or inserting a
   /// copy of src1.
   void legalizeOperandsVOP2(MachineRegisterInfo &MRI, MachineInstr &MI) const;
@@ -1389,8 +1404,8 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
     return get(pseudoToMCOpcode(Opcode));
   }
 
-  unsigned isStackAccess(const MachineInstr &MI, int &FrameIndex) const;
-  unsigned isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex) const;
+  Register isStackAccess(const MachineInstr &MI, int &FrameIndex) const;
+  Register isSGPRStackAccess(const MachineInstr &MI, int &FrameIndex) const;
 
   Register isLoadFromStackSlot(const MachineInstr &MI,
                                int &FrameIndex) const override;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index c552f1a2c90e4..0374526e35c44 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1068,6 +1068,13 @@ def SplitBarrier : ImmOperand<i32> {
   let PrintMethod = "printOperand";
 }
 
+// Pseudo-operand type. This is a pair of 32-bit inline constants
+// packed into a single 64-bit value.
+def AV_64_PSEUDO_IMM : Operand<i64> {
+  let OperandNamespace = "AMDGPU";
+  let OperandType = "OPERAND_INLINE_C_AV64_PSEUDO";
+}
+
 def VReg32OrOffClass : AsmOperandClass {
   let Name = "VReg32OrOff";
   let ParserMethod = "parseVReg32OrOff";
@@ -1954,6 +1961,7 @@ class getVOP3DPPSrcForVT<ValueType VT, bit IsFake16 = 1> {
         !eq(VT, v2f16)  : VCSrc_v2f16,
         !eq(VT, v2bf16) : VCSrc_v2bf16,
         !eq(VT, f32)    : VCSrc_f32,
+        !eq(VT, f64)    : VCSrc_f64,
         !eq(VT, v2i32)  : VCSrc_v2b32,
         1               : VCSrc_b32);
 }
@@ -2707,7 +2715,6 @@ class VOPProfile <list<ValueType> _ArgVT, bit _EnableClamp = 0> {
                                isModifierType<Src2VT>.ret,
                                HasOMod);
   field bit HasNeg = HasModifiers;
-  field bit HasMatrixReuse = 0;
   field bit HasMatrixFMT = 0;
   field bit HasMatrixScale = 0;
   field bit HasMatrixReuse = 0;
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index bd5dfa92a8e43..e8b4501226732 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -150,12 +150,34 @@ def AV_MOV_B32_IMM_PSEUDO
   let isReMaterializable = 1;
   let isAsCheapAsAMove = 1;
 
-  // Imprecise, technically if AGPR it's VOP3 and VOP1 for AGPR. But
+  // Imprecise, technically if AGPR it's VOP3 and VOP1 for VGPR. But
   // this tricks the rematerialize logic into working for it.
   let VOP3 = 1;
   let isMoveImm = 1;
   let SchedRW = [Write32Bit];
-  let Size = 4;
+  let Size = 8;
+  let FixedSize = true;
+  let UseNamedOperandTable = 1;
+}
+
+// 64-bit materialize immediate which supports AGPR or VGPR. This has
+// an unusual operand restriction which requires the two halves of the
+// immediate to each be 32-bit inline immediate values.
+//
+// FIXME: This unnecessarily has the even aligned vector register
+// requirement applied.
+def AV_MOV_B64_IMM_PSEUDO
+    : VPseudoInstSI<(outs AV_64:$vdst), (ins AV_64_PSEUDO_IMM:$src0)> {
+  let isReMaterializable = 1;
+  let isAsCheapAsAMove = 1;
+
+  // Imprecise, technically if AGPR it's 2 x VOP3 and 2 x VOP1 for
+  // VGPR. But this tricks the rematerialize logic into working for
+  // it.
+  let VOP3 = 1;
+  let isMoveImm = 1;
+  let SchedRW = [Write32Bit, Write32Bit];
+  let Size = 16; // 2 x v_accwrite_write_b32 in the worst case
   let UseNamedOperandTable = 1;
 }
 
@@ -3056,6 +3078,8 @@ def : GCNPat<
 }
 }  // AddedComplexity = 1
 
+foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+let True16Predicate = p in {
 def : GCNPat<
   (i32 (DivergentUnaryFrag<zext> i16:$src)),
   (V_AND_B32_e64 (S_MOV_B32 (i32 0xffff)), $src)
@@ -3070,7 +3094,28 @@ def : GCNPat<
 
 def : GCNPat<
   (i32 (zext (i16 (bitconvert fp16_zeros_high_16bits:$src)))),
-  (COPY VSrc_b16:$src)>;
+  (COPY VSrc_b16:$src)
+>;
+}
+
+let True16Predicate = UseRealTrue16Insts in {
+def : GCNPat<
+  (i32 (DivergentUnaryFrag<zext> i16:$src)),
+  (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (V_MOV_B16_t16_e64 0, (i16 0), 0), hi16)
+>;
+
+def : GCNPat<
+  (i64 (DivergentUnaryFrag<zext> i16:$src)),
+  (REG_SEQUENCE VReg_64,
+    (INSERT_SUBREG (i32 (V_MOV_B32_e32 (i32 0))), VGPR_16:$src, lo16), sub0,
+    (S_MOV_B32 (i32 0)), sub1)
+>;
+
+def : GCNPat<
+  (i32 (zext (i16 (bitconvert fp16_zeros_high_16bits:$src)))),
+  (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (V_MOV_B16_t16_e64 0, (i16 0), 0), hi16)
+>;
+}
 
 def : GCNPat <
   (i32 (trunc i64:$a)),
diff --git a/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp b/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
index 3f7b0eab6bb8c..73a2d0a56aebe 100644
--- a/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SILateBranchLowering.cpp
@@ -17,6 +17,7 @@
 #include "SIMachineFunctionInfo.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachinePassManager.h"
+#include "llvm/InitializePasses.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index b49c5a997af78..6f2ea8ad1ff01 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -87,6 +87,8 @@ enum InstClassEnum {
   GLOBAL_STORE_SADDR,
   FLAT_LOAD,
   FLAT_STORE,
+  FLAT_LOAD_SADDR,
+  FLAT_STORE_SADDR,
   GLOBAL_LOAD, // GLOBAL_LOAD/GLOBAL_STORE are never used as the InstClass of
   GLOBAL_STORE // any CombineInfo, they are only ever returned by
                // getCommonInstClass.
@@ -354,6 +356,8 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
   case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
   case AMDGPU::FLAT_LOAD_DWORD:
   case AMDGPU::FLAT_STORE_DWORD:
+  case AMDGPU::FLAT_LOAD_DWORD_SADDR:
+  case AMDGPU::FLAT_STORE_DWORD_SADDR:
     return 1;
   case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
@@ -367,6 +371,8 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
   case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
   case AMDGPU::FLAT_LOAD_DWORDX2:
   case AMDGPU::FLAT_STORE_DWORDX2:
+  case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
     return 2;
   case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
@@ -380,6 +386,8 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
   case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
   case AMDGPU::FLAT_LOAD_DWORDX3:
   case AMDGPU::FLAT_STORE_DWORDX3:
+  case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
     return 3;
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
@@ -393,6 +401,8 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
   case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
   case AMDGPU::FLAT_LOAD_DWORDX4:
   case AMDGPU::FLAT_STORE_DWORDX4:
+  case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
     return 4;
   case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
   case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
@@ -575,6 +585,16 @@ static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
   case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
   case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
     return GLOBAL_STORE_SADDR;
+  case AMDGPU::FLAT_LOAD_DWORD_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
+    return FLAT_LOAD_SADDR;
+  case AMDGPU::FLAT_STORE_DWORD_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
+    return FLAT_STORE_SADDR;
   }
 }
 
@@ -661,6 +681,16 @@ static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) {
   case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
   case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
     return AMDGPU::GLOBAL_STORE_DWORD_SADDR;
+  case AMDGPU::FLAT_LOAD_DWORD_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
+    return AMDGPU::FLAT_LOAD_DWORD_SADDR;
+  case AMDGPU::FLAT_STORE_DWORD_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
+    return AMDGPU::FLAT_STORE_DWORD_SADDR;
   }
 }
 
@@ -776,6 +806,14 @@ static AddressRegs getRegs(unsigned Opc, const SIInstrInfo &TII) {
   case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
   case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
   case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORD_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
+  case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
+  case AMDGPU::FLAT_STORE_DWORD_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
+  case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
     Result.SAddr = true;
     [[fallthrough]];
   case AMDGPU::GLOBAL_LOAD_DWORD:
@@ -840,8 +878,12 @@ void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI,
     Offset = I->getOperand(OffsetIdx).getImm();
   }
 
-  if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE)
+  if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
     Format = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::format)->getImm();
+    const AMDGPU::GcnBufferFormatInfo *Info =
+        AMDGPU::getGcnBufferFormatInfo(Format, *LSO.STM);
+    EltSize = Info->BitsPerComp / 8;
+  }
 
   Width = getOpcodeWidth(*I, *LSO.TII);
 
@@ -1049,24 +1091,44 @@ bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI,
 
     const llvm::AMDGPU::GcnBufferFormatInfo *Info0 =
         llvm::AMDGPU::getGcnBufferFormatInfo(CI.Format, STI);
-    if (!Info0)
-      return false;
     const llvm::AMDGPU::GcnBufferFormatInfo *Info1 =
         llvm::AMDGPU::getGcnBufferFormatInfo(Paired.Format, STI);
-    if (!Info1)
-      return false;
 
     if (Info0->BitsPerComp != Info1->BitsPerComp ||
         Info0->NumFormat != Info1->NumFormat)
       return false;
 
-    // TODO: Should be possible to support more formats, but if format loads
-    // are not dword-aligned, the merged load might not be valid.
-    if (Info0->BitsPerComp != 32)
+    // For 8-bit or 16-bit formats there is no 3-component variant.
+    // If NumCombinedComponents is 3, try the 4-component format and use XYZ.
+    // Example:
+    //   tbuffer_load_format_x + tbuffer_load_format_x + tbuffer_load_format_x
+    //   ==> tbuffer_load_format_xyz with format:[BUF_FMT_16_16_16_16_SNORM]
+    unsigned NumCombinedComponents = CI.Width + Paired.Width;
+    if (NumCombinedComponents == 3 && CI.EltSize <= 2)
+      NumCombinedComponents = 4;
+
+    if (getBufferFormatWithCompCount(CI.Format, NumCombinedComponents, STI) ==
+        0)
       return false;
 
-    if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0)
+    // Merge only when the two access ranges are strictly back-to-back,
+    // any gap or overlap can over-write data or leave holes.
+    unsigned ElemIndex0 = CI.Offset / CI.EltSize;
+    unsigned ElemIndex1 = Paired.Offset / Paired.EltSize;
+    if (ElemIndex0 + CI.Width != ElemIndex1 &&
+        ElemIndex1 + Paired.Width != ElemIndex0)
       return false;
+
+    // 1-byte formats require 1-byte alignment.
+    // 2-byte formats require 2-byte alignment.
+    // 4-byte and larger formats require 4-byte alignment.
+    unsigned MergedBytes = CI.EltSize * NumCombinedComponents;
+    unsigned RequiredAlign = std::min(MergedBytes, 4u);
+    unsigned MinOff = std::min(CI.Offset, Paired.Offset);
+    if (MinOff % RequiredAlign != 0)
+      return false;
+
+    return true;
   }
 
   uint32_t EltOffset0 = CI.Offset / CI.EltSize;
@@ -1596,8 +1658,14 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferLoadPair(
   if (Regs.VAddr)
     MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
 
+  // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
+  // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
+  // and use XYZ of XYZW to enable the merge.
+  unsigned NumCombinedComponents = CI.Width + Paired.Width;
+  if (NumCombinedComponents == 3 && CI.EltSize <= 2)
+    NumCombinedComponents = 4;
   unsigned JoinedFormat =
-      getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM);
+      getBufferFormatWithCompCount(CI.Format, NumCombinedComponents, *STM);
 
   // It shouldn't be possible to get this far if the two instructions
   // don't have a single memoperand, because MachineInstr::mayAlias()
@@ -1639,8 +1707,14 @@ MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferStorePair(
   if (Regs.VAddr)
     MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
 
+  // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
+  // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
+  // and use XYZ of XYZW to enable the merge.
+  unsigned NumCombinedComponents = CI.Width + Paired.Width;
+  if (NumCombinedComponents == 3 && CI.EltSize <= 2)
+    NumCombinedComponents = 4;
   unsigned JoinedFormat =
-      getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STM);
+      getBufferFormatWithCompCount(CI.Format, NumCombinedComponents, *STM);
 
   // It shouldn't be possible to get this far if the two instructions
   // don't have a single memoperand, because MachineInstr::mayAlias()
@@ -1875,6 +1949,28 @@ unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI,
     case 4:
       return AMDGPU::FLAT_STORE_DWORDX4;
     }
+  case FLAT_LOAD_SADDR:
+    switch (Width) {
+    default:
+      return 0;
+    case 2:
+      return AMDGPU::FLAT_LOAD_DWORDX2_SADDR;
+    case 3:
+      return AMDGPU::FLAT_LOAD_DWORDX3_SADDR;
+    case 4:
+      return AMDGPU::FLAT_LOAD_DWORDX4_SADDR;
+    }
+  case FLAT_STORE_SADDR:
+    switch (Width) {
+    default:
+      return 0;
+    case 2:
+      return AMDGPU::FLAT_STORE_DWORDX2_SADDR;
+    case 3:
+      return AMDGPU::FLAT_STORE_DWORDX3_SADDR;
+    case 4:
+      return AMDGPU::FLAT_STORE_DWORDX4_SADDR;
+    }
   case MIMG:
     assert(((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == Width) &&
            "No overlaps");
@@ -2353,6 +2449,15 @@ SILoadStoreOptimizer::collectMergeableInsts(
     if (Swizzled != -1 && MI.getOperand(Swizzled).getImm())
       continue;
 
+    if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
+      const MachineOperand *Fmt =
+          TII->getNamedOperand(MI, AMDGPU::OpName::format);
+      if (!AMDGPU::getGcnBufferFormatInfo(Fmt->getImm(), *STM)) {
+        LLVM_DEBUG(dbgs() << "Skip tbuffer with unknown format: " << MI);
+        continue;
+      }
+    }
+
     CombineInfo CI;
     CI.setMI(MI, *this);
     CI.Order = Order++;
@@ -2508,12 +2613,14 @@ SILoadStoreOptimizer::optimizeInstsWithSameBaseAddr(
       OptimizeListAgain |= CI.Width + Paired.Width < 4;
       break;
     case FLAT_LOAD:
+    case FLAT_LOAD_SADDR:
     case GLOBAL_LOAD:
     case GLOBAL_LOAD_SADDR:
       NewMI = mergeFlatLoadPair(CI, Paired, Where->I);
       OptimizeListAgain |= CI.Width + Paired.Width < 4;
       break;
     case FLAT_STORE:
+    case FLAT_STORE_SADDR:
     case GLOBAL_STORE:
     case GLOBAL_STORE_SADDR:
       NewMI = mergeFlatStorePair(CI, Paired, Where->I);
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index f8878f32f829d..e97536d36bab2 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -57,6 +57,7 @@
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
 #include "llvm/Target/TargetMachine.h"
 
 using namespace llvm;
@@ -76,10 +77,11 @@ class SILowerControlFlow {
   LiveIntervals *LIS = nullptr;
   LiveVariables *LV = nullptr;
   MachineDominatorTree *MDT = nullptr;
+  MachinePostDominatorTree *PDT = nullptr;
   MachineRegisterInfo *MRI = nullptr;
   SetVector<MachineInstr*> LoweredEndCf;
   DenseSet<Register> LoweredIf;
-  SmallSet<MachineBasicBlock *, 4> KillBlocks;
+  SmallPtrSet<MachineBasicBlock *, 4> KillBlocks;
   SmallSet<Register, 8> RecomputeRegs;
 
   const TargetRegisterClass *BoolRC = nullptr;
@@ -138,8 +140,8 @@ class SILowerControlFlow {
 
 public:
   SILowerControlFlow(LiveIntervals *LIS, LiveVariables *LV,
-                     MachineDominatorTree *MDT)
-      : LIS(LIS), LV(LV), MDT(MDT) {}
+                     MachineDominatorTree *MDT, MachinePostDominatorTree *PDT)
+      : LIS(LIS), LV(LV), MDT(MDT), PDT(PDT) {}
   bool run(MachineFunction &MF);
 };
 
@@ -159,6 +161,7 @@ class SILowerControlFlowLegacy : public MachineFunctionPass {
     AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
     // Should preserve the same set that TwoAddressInstructions does.
     AU.addPreserved<MachineDominatorTreeWrapperPass>();
+    AU.addPreserved<MachinePostDominatorTreeWrapperPass>();
     AU.addPreserved<SlotIndexesWrapperPass>();
     AU.addPreserved<LiveIntervalsWrapperPass>();
     AU.addPreserved<LiveVariablesWrapperPass>();
@@ -457,7 +460,7 @@ MachineBasicBlock::iterator
 SILowerControlFlow::skipIgnoreExecInstsTrivialSucc(
   MachineBasicBlock &MBB, MachineBasicBlock::iterator It) const {
 
-  SmallSet<const MachineBasicBlock *, 4> Visited;
+  SmallPtrSet<const MachineBasicBlock *, 4> Visited;
   MachineBasicBlock *B = &MBB;
   do {
     if (!Visited.insert(B).second)
@@ -506,13 +509,18 @@ MachineBasicBlock *SILowerControlFlow::emitEndCf(MachineInstr &MI) {
   MachineBasicBlock *SplitBB = &MBB;
   if (NeedBlockSplit) {
     SplitBB = MBB.splitAt(MI, /*UpdateLiveIns*/true, LIS);
-    if (MDT && SplitBB != &MBB) {
-      MachineDomTreeNode *MBBNode = (*MDT)[&MBB];
-      SmallVector<MachineDomTreeNode *> Children(MBBNode->begin(),
-                                                 MBBNode->end());
-      MachineDomTreeNode *SplitBBNode = MDT->addNewBlock(SplitBB, &MBB);
-      for (MachineDomTreeNode *Child : Children)
-        MDT->changeImmediateDominator(Child, SplitBBNode);
+    if (SplitBB != &MBB && (MDT || PDT)) {
+      using DomTreeT = DomTreeBase<MachineBasicBlock>;
+      SmallVector<DomTreeT::UpdateType, 16> DTUpdates;
+      for (MachineBasicBlock *Succ : SplitBB->successors()) {
+        DTUpdates.push_back({DomTreeT::Insert, SplitBB, Succ});
+        DTUpdates.push_back({DomTreeT::Delete, &MBB, Succ});
+      }
+      DTUpdates.push_back({DomTreeT::Insert, &MBB, SplitBB});
+      if (MDT)
+        MDT->applyUpdates(DTUpdates);
+      if (PDT)
+        PDT->applyUpdates(DTUpdates);
     }
     Opcode = OrTermrOpc;
     InsPt = MI;
@@ -727,26 +735,27 @@ bool SILowerControlFlow::removeMBBifRedundant(MachineBasicBlock &MBB) {
   MachineBasicBlock *Succ = *MBB.succ_begin();
   MachineBasicBlock *FallThrough = nullptr;
 
+  using DomTreeT = DomTreeBase<MachineBasicBlock>;
+  SmallVector<DomTreeT::UpdateType, 8> DTUpdates;
+
   while (!MBB.predecessors().empty()) {
     MachineBasicBlock *P = *MBB.pred_begin();
     if (P->getFallThrough(false) == &MBB)
       FallThrough = P;
     P->ReplaceUsesOfBlockWith(&MBB, Succ);
+    DTUpdates.push_back({DomTreeT::Insert, P, Succ});
+    DTUpdates.push_back({DomTreeT::Delete, P, &MBB});
   }
   MBB.removeSuccessor(Succ);
   if (LIS) {
     for (auto &I : MBB.instrs())
       LIS->RemoveMachineInstrFromMaps(I);
   }
-  if (MDT) {
-    // If Succ, the single successor of MBB, is dominated by MBB, MDT needs
-    // updating by changing Succ's idom to the one of MBB; otherwise, MBB must
-    // be a leaf node in MDT and could be erased directly.
-    if (MDT->dominates(&MBB, Succ))
-      MDT->changeImmediateDominator(MDT->getNode(Succ),
-                                    MDT->getNode(&MBB)->getIDom());
-    MDT->eraseNode(&MBB);
-  }
+  if (MDT)
+    MDT->applyUpdates(DTUpdates);
+  if (PDT)
+    PDT->applyUpdates(DTUpdates);
+
   MBB.clear();
   MBB.eraseFromParent();
   if (FallThrough && !FallThrough->isLayoutSuccessor(Succ)) {
@@ -875,7 +884,11 @@ bool SILowerControlFlowLegacy::runOnMachineFunction(MachineFunction &MF) {
   LiveVariables *LV = LVWrapper ? &LVWrapper->getLV() : nullptr;
   auto *MDTWrapper = getAnalysisIfAvailable<MachineDominatorTreeWrapperPass>();
   MachineDominatorTree *MDT = MDTWrapper ? &MDTWrapper->getDomTree() : nullptr;
-  return SILowerControlFlow(LIS, LV, MDT).run(MF);
+  auto *PDTWrapper =
+      getAnalysisIfAvailable<MachinePostDominatorTreeWrapperPass>();
+  MachinePostDominatorTree *PDT =
+      PDTWrapper ? &PDTWrapper->getPostDomTree() : nullptr;
+  return SILowerControlFlow(LIS, LV, MDT, PDT).run(MF);
 }
 
 PreservedAnalyses
@@ -885,13 +898,16 @@ SILowerControlFlowPass::run(MachineFunction &MF,
   LiveVariables *LV = MFAM.getCachedResult<LiveVariablesAnalysis>(MF);
   MachineDominatorTree *MDT =
       MFAM.getCachedResult<MachineDominatorTreeAnalysis>(MF);
+  MachinePostDominatorTree *PDT =
+      MFAM.getCachedResult<MachinePostDominatorTreeAnalysis>(MF);
 
-  bool Changed = SILowerControlFlow(LIS, LV, MDT).run(MF);
+  bool Changed = SILowerControlFlow(LIS, LV, MDT, PDT).run(MF);
   if (!Changed)
     return PreservedAnalyses::all();
 
   auto PA = getMachineFunctionPassPreservedAnalyses();
   PA.preserve<MachineDominatorTreeAnalysis>();
+  PA.preserve<MachinePostDominatorTreeAnalysis>();
   PA.preserve<SlotIndexesAnalysis>();
   PA.preserve<LiveIntervalsAnalysis>();
   PA.preserve<LiveVariablesAnalysis>();
diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
index 9509199a8d5ef..1a91bbd433553 100644
--- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/InitializePasses.h"
 
 using namespace llvm;
 
@@ -209,10 +210,13 @@ void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) {
   // So set the save points for those.
 
   // Use the points found by shrink-wrapping, if any.
-  if (MFI.getSavePoint()) {
-    SaveBlocks.push_back(MFI.getSavePoint());
-    assert(MFI.getRestorePoint() && "Both restore and save must be set");
-    MachineBasicBlock *RestoreBlock = MFI.getRestorePoint();
+  if (!MFI.getSavePoints().empty()) {
+    assert(MFI.getSavePoints().size() == 1 &&
+           "Multiple save points not yet supported!");
+    SaveBlocks.push_back(MFI.getSavePoints().front());
+    assert(MFI.getRestorePoints().size() == 1 &&
+           "Multiple restore points not yet supported!");
+    MachineBasicBlock *RestoreBlock = MFI.getRestorePoints().front();
     // If RestoreBlock does not have any successor and is not a return block
     // then the end point is unreachable and we do not need to insert any
     // epilogue.
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 9a1448f1f95dc..8a1120321af9f 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -33,7 +33,7 @@ using namespace llvm;
 // optimal RC for Opc and Dest of MFMA. In particular, there are high RP cases
 // where it is better to produce the VGPR form (e.g. if there are VGPR users
 // of the MFMA result).
-cl::opt<bool> MFMAVGPRForm(
+static cl::opt<bool> MFMAVGPRForm(
     "amdgpu-mfma-vgpr-form", cl::Hidden,
     cl::desc("Whether to force use VGPR for Opc and Dest of MFMA. If "
              "unspecified, default to compiler heuristics"),
@@ -728,6 +728,8 @@ yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
       MemoryBound(MFI.isMemoryBound()), WaveLimiter(MFI.needsWaveLimiter()),
       HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
       HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
+      NumWaveDispatchSGPRs(MFI.getNumWaveDispatchSGPRs()),
+      NumWaveDispatchVGPRs(MFI.getNumWaveDispatchVGPRs()),
       HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
       Occupancy(MFI.getOccupancy()),
       ScratchRSrcReg(regToString(MFI.getScratchRSrcReg(), TRI)),
@@ -784,6 +786,8 @@ bool SIMachineFunctionInfo::initializeBaseYamlFields(
   WaveLimiter = YamlMFI.WaveLimiter;
   HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
   HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
+  NumWaveDispatchSGPRs = YamlMFI.NumWaveDispatchSGPRs;
+  NumWaveDispatchVGPRs = YamlMFI.NumWaveDispatchVGPRs;
   BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
   ReturnsVoid = YamlMFI.ReturnsVoid;
   IsWholeWaveFunction = YamlMFI.IsWholeWaveFunction;
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index 08b0206d244fb..ca8f8033a2d54 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -270,6 +270,8 @@ struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
   bool WaveLimiter = false;
   bool HasSpilledSGPRs = false;
   bool HasSpilledVGPRs = false;
+  uint16_t NumWaveDispatchSGPRs = 0;
+  uint16_t NumWaveDispatchVGPRs = 0;
   uint32_t HighBitsOf32BitAddress = 0;
 
   // TODO: 10 may be a better default since it's the maximum.
@@ -327,6 +329,8 @@ template <> struct MappingTraits<SIMachineFunctionInfo> {
     YamlIO.mapOptional("waveLimiter", MFI.WaveLimiter, false);
     YamlIO.mapOptional("hasSpilledSGPRs", MFI.HasSpilledSGPRs, false);
     YamlIO.mapOptional("hasSpilledVGPRs", MFI.HasSpilledVGPRs, false);
+    YamlIO.mapOptional("numWaveDispatchSGPRs", MFI.NumWaveDispatchSGPRs, false);
+    YamlIO.mapOptional("numWaveDispatchVGPRs", MFI.NumWaveDispatchVGPRs, false);
     YamlIO.mapOptional("scratchRSrcReg", MFI.ScratchRSrcReg,
                        StringValue("$private_rsrc_reg"));
     YamlIO.mapOptional("frameOffsetReg", MFI.FrameOffsetReg,
@@ -465,6 +469,9 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction,
   unsigned NumUserSGPRs = 0;
   unsigned NumSystemSGPRs = 0;
 
+  unsigned NumWaveDispatchSGPRs = 0;
+  unsigned NumWaveDispatchVGPRs = 0;
+
   bool HasSpilledSGPRs = false;
   bool HasSpilledVGPRs = false;
   bool HasNonSpillStackObjects = false;
@@ -991,6 +998,14 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction,
     return UserSGPRInfo.getNumKernargPreloadSGPRs();
   }
 
+  unsigned getNumWaveDispatchSGPRs() const { return NumWaveDispatchSGPRs; }
+
+  void setNumWaveDispatchSGPRs(unsigned Count) { NumWaveDispatchSGPRs = Count; }
+
+  unsigned getNumWaveDispatchVGPRs() const { return NumWaveDispatchVGPRs; }
+
+  void setNumWaveDispatchVGPRs(unsigned Count) { NumWaveDispatchVGPRs = Count; }
+
   Register getPrivateSegmentWaveByteOffsetSystemSGPR() const {
     return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
   }
diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
index 0e2ac495d2d98..41402bd5278c6 100644
--- a/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
+++ b/llvm/lib/Target/AMDGPU/SIOptimizeVGPRLiveRange.cpp
@@ -80,6 +80,7 @@
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/IR/Dominators.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
index 205a45a045a42..ecfaa5c70e9d3 100644
--- a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp
@@ -23,6 +23,7 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/InitializePasses.h"
 
 using namespace llvm;
 
@@ -130,6 +131,9 @@ void SIPreAllocateWWMRegs::rewriteRegs(MachineFunction &MF) {
         if (VirtReg.isPhysical())
           continue;
 
+        if (!VirtReg.isValid())
+          continue;
+
         if (!VRM->hasPhys(VirtReg))
           continue;
 
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
index 5940f45e74bf2..93ba0a337d7dd 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
@@ -73,6 +73,7 @@ void SIProgramInfo::reset(const MachineFunction &MF) {
 
   NumSGPRsForWavesPerEU = ZeroExpr;
   NumVGPRsForWavesPerEU = ZeroExpr;
+  NamedBarCnt = ZeroExpr;
   Occupancy = ZeroExpr;
   DynamicCallStack = ZeroExpr;
   VCCUsed = ZeroExpr;
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.h b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
index 79099d2182cc8..171c4a313a53b 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
@@ -83,6 +83,9 @@ struct LLVM_EXTERNAL_VISIBILITY SIProgramInfo {
   // Number of VGPRs that meets number of waves per execution unit request.
   const MCExpr *NumVGPRsForWavesPerEU = nullptr;
 
+  // Number of named barriers used by the kernel.
+  const MCExpr *NamedBarCnt = nullptr;
+
   // Final occupancy.
   const MCExpr *Occupancy = nullptr;
 
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
index 81655f5a829fb..0293d4018770f 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -1166,7 +1166,8 @@ class RegOrImmOperand <RegisterClass RegClass, string OperandTypeName>
 }
 
 //===----------------------------------------------------------------------===//
-//  SSrc_* Operands with an SGPR or a 32-bit immediate
+//  SSrc_* Operands with an SGPR, a 32-bit immediate, or 64-bit immediate
+//  if supported by target.
 //===----------------------------------------------------------------------===//
 
 class SrcRegOrImm9<RegisterClass regClass, string operandType>
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 8303410115f93..a003a46191a87 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -484,6 +484,24 @@ def S_BARRIER_SIGNAL_ISFIRST_M0 : SOP1_Pseudo <"s_barrier_signal_isfirst m0", (o
   let isConvergent = 1;
 }
 
+def S_BARRIER_INIT_M0 : SOP1_Pseudo <"s_barrier_init m0", (outs), (ins),
+  "", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_BARRIER_INIT_IMM : SOP1_Pseudo <"s_barrier_init", (outs),
+  (ins SplitBarrier:$src0), "$src0", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_BARRIER_JOIN_M0 : SOP1_Pseudo <"s_barrier_join m0", (outs), (ins),
+  "", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
 } // End Uses = [M0]
 
 def S_BARRIER_SIGNAL_IMM : SOP1_Pseudo <"s_barrier_signal", (outs),
@@ -501,6 +519,12 @@ def S_BARRIER_SIGNAL_ISFIRST_IMM : SOP1_Pseudo <"s_barrier_signal_isfirst", (out
   let isConvergent = 1;
 }
 
+def S_BARRIER_JOIN_IMM : SOP1_Pseudo <"s_barrier_join", (outs),
+  (ins SplitBarrier:$src0), "$src0", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
 } // End has_sdst = 0
 
 def S_GET_BARRIER_STATE_IMM : SOP1_Pseudo <"s_get_barrier_state", (outs SSrc_b32:$sdst),
@@ -1588,6 +1612,17 @@ def S_BARRIER_WAIT : SOPP_Pseudo <"s_barrier_wait", (ins i16imm:$simm16), "$simm
   let isConvergent = 1;
 }
 
+def S_BARRIER_LEAVE : SOPP_Pseudo <"s_barrier_leave", (ins)> {
+  let SchedRW = [WriteBarrier];
+  let simm16 = 0;
+  let fixed_imm = 1;
+  let isConvergent = 1;
+  let Defs = [SCC];
+}
+
+def S_BARRIER_LEAVE_IMM : SOPP_Pseudo <"s_barrier_leave",
+    (ins i16imm:$simm16), "$simm16", [(int_amdgcn_s_barrier_leave timm:$simm16)]>;
+
 def S_WAKEUP : SOPP_Pseudo <"s_wakeup", (ins) > {
   let SubtargetPredicate = isGFX8Plus;
   let simm16 = 0;
@@ -1630,7 +1665,9 @@ def S_WAITCNT_lds_direct : SPseudoInstSI<(outs), (ins)> {
 
 def S_SETHALT : SOPP_Pseudo <"s_sethalt" , (ins i32imm:$simm16), "$simm16",
     [(int_amdgcn_s_sethalt timm:$simm16)]>;
-def S_SETKILL : SOPP_Pseudo <"s_setkill" , (ins i16imm:$simm16), "$simm16">;
+def S_SETKILL : SOPP_Pseudo <"s_setkill" , (ins i16imm:$simm16), "$simm16"> {
+  let SubtargetPredicate = isNotGFX1250Plus;
+}
 
 // On SI the documentation says sleep for approximately 64 * low 2
 // bits, consistent with the reported maximum of 448. On VI the
@@ -1653,6 +1690,12 @@ def S_SETPRIO_INC_WG : SOPP_Pseudo <"s_setprio_inc_wg", (ins i16imm:$simm16), "$
   let SubtargetPredicate = HasSetPrioIncWgInst;
 }
 
+def S_GET_SHADER_CYCLES_U64 : SOP1_64_0 <"s_get_shader_cycles_u64",
+  [(set i64:$sdst, (readcyclecounter))]> {
+  let SubtargetPredicate = HasSGetShaderCyclesInst;
+  let hasSideEffects = 1;
+}
+
 let Uses = [EXEC, M0] in {
 def S_SENDMSG : SOPP_Pseudo <"s_sendmsg" , (ins SendMsg:$simm16), "$simm16",
   [(int_amdgcn_s_sendmsg (i32 timm:$simm16), M0)]> {
@@ -2138,13 +2181,18 @@ defm S_SENDMSG_RTN_B64            : SOP1_Real_gfx11_gfx12<0x04d>;
 defm S_BARRIER_SIGNAL_M0          : SOP1_M0_Real_gfx12<0x04e>;
 defm S_BARRIER_SIGNAL_ISFIRST_M0  : SOP1_M0_Real_gfx12<0x04f>;
 defm S_GET_BARRIER_STATE_M0       : SOP1_M0_Real_gfx12<0x050>;
+defm S_BARRIER_INIT_M0            : SOP1_M0_Real_gfx12<0x051>;
+defm S_BARRIER_JOIN_M0            : SOP1_M0_Real_gfx12<0x052>;
 defm S_BARRIER_SIGNAL_IMM         : SOP1_IMM_Real_gfx12<0x04e>;
 defm S_BARRIER_SIGNAL_ISFIRST_IMM : SOP1_IMM_Real_gfx12<0x04f>;
 defm S_GET_BARRIER_STATE_IMM      : SOP1_IMM_Real_gfx12<0x050>;
+defm S_BARRIER_INIT_IMM           : SOP1_IMM_Real_gfx12<0x051>;
+defm S_BARRIER_JOIN_IMM           : SOP1_IMM_Real_gfx12<0x052>;
 defm S_ALLOC_VGPR                 : SOP1_Real_gfx12<0x053>;
 defm S_SLEEP_VAR                  : SOP1_IMM_Real_gfx12<0x058>;
 
 // GFX1250
+defm S_GET_SHADER_CYCLES_U64      : SOP1_Real_gfx12<0x06>;
 defm S_ADD_PC_I64                 : SOP1_Real_gfx12<0x04b>;
 
 //===----------------------------------------------------------------------===//
@@ -2632,6 +2680,7 @@ multiclass SOPP_Real_32_gfx12<bits<7> op, string name = !tolower(NAME)> {
 }
 
 defm S_BARRIER_WAIT         : SOPP_Real_32_gfx12<0x014>;
+defm S_BARRIER_LEAVE        : SOPP_Real_32_gfx12<0x015>;
 defm S_WAIT_LOADCNT         : SOPP_Real_32_gfx12<0x040>;
 defm S_WAIT_STORECNT        : SOPP_Real_32_gfx12<0x041>;
 defm S_WAIT_SAMPLECNT       : SOPP_Real_32_gfx12<0x042>;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
index e433b85489e6e..c740b5e0f09d1 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
@@ -106,7 +106,7 @@ static constexpr CustomOperand MsgOperands[] = {
   {{"MSG_GET_DDID"},            ID_GET_DDID,                isGFX10},
   {{"MSG_HS_TESSFACTOR"},       ID_HS_TESSFACTOR_GFX11Plus, isGFX11Plus},
   {{"MSG_DEALLOC_VGPRS"},       ID_DEALLOC_VGPRS_GFX11Plus, isGFX11Plus},
-  {{""}},
+  {{"MSG_SAVEWAVE_HAS_TDM"},    ID_SAVEWAVE_HAS_TDM,        isGFX1250},
   {{"MSG_SYSMSG"},              ID_SYSMSG},
   {{"MSG_RTN_GET_DOORBELL"},    ID_RTN_GET_DOORBELL,        isGFX11Plus},
   {{"MSG_RTN_GET_DDID"},        ID_RTN_GET_DDID,            isGFX11Plus},
@@ -195,7 +195,7 @@ static constexpr CustomOperand Operands[] = {
   {{"HW_REG_POPS_PACKER"},   ID_POPS_PACKER, isGFX10},
   {{""}},
   {{"HW_REG_PERF_SNAPSHOT_DATA"}, ID_PERF_SNAPSHOT_DATA_gfx11, isGFX11},
-  {{""}},
+  {{"HW_REG_IB_STS2"}, ID_IB_STS2, isGFX1250},
   {{"HW_REG_SHADER_CYCLES"},    ID_SHADER_CYCLES,    isGFX10_3_GFX11},
   {{"HW_REG_SHADER_CYCLES_HI"}, ID_SHADER_CYCLES_HI, isGFX12Plus},
   {{"HW_REG_DVGPR_ALLOC_LO"},   ID_DVGPR_ALLOC_LO,   isGFX12Plus},
@@ -223,6 +223,10 @@ static constexpr CustomOperand Operands[] = {
   {{"HW_REG_SQ_PERF_SNAPSHOT_PC_LO"}, ID_SQ_PERF_SNAPSHOT_PC_LO, isGFX940},
   {{"HW_REG_SQ_PERF_SNAPSHOT_PC_HI"}, ID_SQ_PERF_SNAPSHOT_PC_HI, isGFX940},
 
+  // GFX1250
+  {{"HW_REG_XNACK_STATE_PRIV"},       ID_XNACK_STATE_PRIV,       isGFX1250},
+  {{"HW_REG_XNACK_MASK"},             ID_XNACK_MASK_gfx1250,     isGFX1250},
+
   // Aliases
   {{"HW_REG_HW_ID"},                  ID_HW_ID1,                 isGFX10},
 };
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 00dcb9b52d4bd..18ee9c16b3ff9 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -21,6 +21,7 @@
 #include "llvm/IR/IntrinsicsAMDGPU.h"
 #include "llvm/IR/IntrinsicsR600.h"
 #include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Metadata.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -1160,17 +1161,28 @@ unsigned getAddressableLocalMemorySize(const MCSubtargetInfo *STI) {
     return 65536;
   if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize163840))
     return 163840;
-  return 0;
+  if (STI->getFeatureBits().test(FeatureAddressableLocalMemorySize327680))
+    return 327680;
+  return 32768;
 }
 
 unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
   // "Per CU" really means "per whatever functional block the waves of a
-  // workgroup must share". For gfx10 in CU mode this is the CU, which contains
+  // workgroup must share".
+
+  // GFX12.5 only supports CU mode, which contains four SIMDs.
+  if (isGFX1250(*STI)) {
+    assert(STI->getFeatureBits().test(FeatureCuMode));
+    return 4;
+  }
+
+  // For gfx10 in CU mode the functional block is the CU, which contains
   // two SIMDs.
   if (isGFX10Plus(*STI) && STI->getFeatureBits().test(FeatureCuMode))
     return 2;
-  // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP contains
-  // two CUs, so a total of four SIMDs.
+
+  // Pre-gfx10 a CU contains four SIMDs. For gfx10 in WGP mode the WGP
+  // contains two CUs, so a total of four SIMDs.
   return 4;
 }
 
@@ -1666,6 +1678,29 @@ getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) {
   return Vals;
 }
 
+bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) {
+  assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!");
+  for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) {
+    auto Low =
+        mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 0))->getValue();
+    auto High =
+        mdconst::extract<ConstantInt>(MD.getOperand(2 * I + 1))->getValue();
+    // There are two types of [A; B) ranges:
+    //  A < B, e.g. [4; 5) which is a range that only includes 4.
+    //  A > B, e.g. [5; 4) which is a range that wraps around and includes
+    //         everything except 4.
+    if (Low.ult(High)) {
+      if (Low.ule(Val) && High.ugt(Val))
+        return true;
+    } else {
+      if (Low.uge(Val) && High.ult(Val))
+        return true;
+    }
+  }
+
+  return false;
+}
+
 unsigned getVmcntBitMask(const IsaVersion &Version) {
   return (1 << (getVmcntBitWidthLo(Version.Major) +
                 getVmcntBitWidthHi(Version.Major))) -
@@ -2406,7 +2441,11 @@ unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler) {
   return 0;
 }
 
-unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) { return 16; }
+unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI) {
+  if (isGFX1250(STI))
+    return 32;
+  return 16;
+}
 
 bool isSI(const MCSubtargetInfo &STI) {
   return STI.hasFeature(AMDGPU::FeatureSouthernIslands);
@@ -2478,6 +2517,12 @@ bool isGFX1250(const MCSubtargetInfo &STI) {
   return STI.getFeatureBits()[AMDGPU::FeatureGFX1250Insts];
 }
 
+bool supportsWGP(const MCSubtargetInfo &STI) {
+  if (isGFX1250(STI))
+    return false;
+  return isGFX10Plus(STI);
+}
+
 bool isNotGFX11Plus(const MCSubtargetInfo &STI) { return !isGFX11Plus(STI); }
 
 bool isNotGFX10Plus(const MCSubtargetInfo &STI) {
@@ -3309,13 +3354,53 @@ bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc) {
   return false;
 }
 
-bool isDPALU_DPP(const MCInstrDesc &OpDesc) {
+bool isDPALU_DPP32BitOpc(unsigned Opc) {
+  switch (Opc) {
+  case AMDGPU::V_MUL_LO_U32_e64:
+  case AMDGPU::V_MUL_LO_U32_e64_dpp:
+  case AMDGPU::V_MUL_LO_U32_e64_dpp_gfx1250:
+  case AMDGPU::V_MUL_HI_U32_e64:
+  case AMDGPU::V_MUL_HI_U32_e64_dpp:
+  case AMDGPU::V_MUL_HI_U32_e64_dpp_gfx1250:
+  case AMDGPU::V_MUL_HI_I32_e64:
+  case AMDGPU::V_MUL_HI_I32_e64_dpp:
+  case AMDGPU::V_MUL_HI_I32_e64_dpp_gfx1250:
+  case AMDGPU::V_MAD_U32_e64:
+  case AMDGPU::V_MAD_U32_e64_dpp:
+  case AMDGPU::V_MAD_U32_e64_dpp_gfx1250:
+    return true;
+  default:
+    return false;
+  }
+}
+
+bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCSubtargetInfo &ST) {
+  if (!ST.hasFeature(AMDGPU::FeatureDPALU_DPP))
+    return false;
+
+  if (isDPALU_DPP32BitOpc(OpDesc.getOpcode()))
+    return ST.hasFeature(AMDGPU::FeatureGFX1250Insts);
+
   return hasAny64BitVGPROperands(OpDesc);
 }
 
 unsigned getLdsDwGranularity(const MCSubtargetInfo &ST) {
-  // Currently this is 128 for all subtargets
-  return 128;
+  return ST.hasFeature(AMDGPU::FeatureAddressableLocalMemorySize327680) ? 256
+                                                                        : 128;
+}
+
+bool isPackedFP32Inst(unsigned Opc) {
+  switch (Opc) {
+  case AMDGPU::V_PK_ADD_F32:
+  case AMDGPU::V_PK_ADD_F32_gfx12:
+  case AMDGPU::V_PK_MUL_F32:
+  case AMDGPU::V_PK_MUL_F32_gfx12:
+  case AMDGPU::V_PK_FMA_F32:
+  case AMDGPU::V_PK_FMA_F32_gfx12:
+    return true;
+  default:
+    return false;
+  }
 }
 
 } // namespace AMDGPU
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 1252e35d81e82..70dfb63cbe040 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -35,6 +35,7 @@ class MCInstrInfo;
 class MCRegisterClass;
 class MCRegisterInfo;
 class MCSubtargetInfo;
+class MDNode;
 class StringRef;
 class Triple;
 class raw_ostream;
@@ -1064,6 +1065,9 @@ SmallVector<unsigned> getIntegerVecAttribute(const Function &F, StringRef Name,
 std::optional<SmallVector<unsigned>>
 getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size);
 
+/// Checks if \p Val is inside \p MD, a !range-like metadata.
+bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val);
+
 /// Represents the counter values to wait for in an s_waitcnt instruction.
 ///
 /// Large values (including the maximum possible integer) can be used to
@@ -1549,6 +1553,7 @@ bool isGFX11Plus(const MCSubtargetInfo &STI);
 bool isGFX12(const MCSubtargetInfo &STI);
 bool isGFX12Plus(const MCSubtargetInfo &STI);
 bool isGFX1250(const MCSubtargetInfo &STI);
+bool supportsWGP(const MCSubtargetInfo &STI);
 bool isNotGFX12Plus(const MCSubtargetInfo &STI);
 bool isNotGFX11Plus(const MCSubtargetInfo &STI);
 bool isGCN3Encoding(const MCSubtargetInfo &STI);
@@ -1709,6 +1714,8 @@ bool isArgPassedInSGPR(const Argument *Arg);
 
 bool isArgPassedInSGPR(const CallBase *CB, unsigned ArgNo);
 
+LLVM_READONLY bool isPackedFP32Inst(unsigned Opc);
+
 LLVM_READONLY
 bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST,
                                       int64_t EncodedOffset);
@@ -1748,15 +1755,22 @@ unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST);
 bool isLegalSMRDImmOffset(const MCSubtargetInfo &ST, int64_t ByteOffset);
 
 LLVM_READNONE
-inline bool isLegalDPALU_DPPControl(unsigned DC) {
-  return DC >= DPP::ROW_NEWBCAST_FIRST && DC <= DPP::ROW_NEWBCAST_LAST;
+inline bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC) {
+  if (isGFX12(ST))
+    return DC >= DPP::ROW_SHARE_FIRST && DC <= DPP::ROW_SHARE_LAST;
+  if (isGFX90A(ST))
+    return DC >= DPP::ROW_NEWBCAST_FIRST && DC <= DPP::ROW_NEWBCAST_LAST;
+  return false;
 }
 
 /// \returns true if an instruction may have a 64-bit VGPR operand.
 bool hasAny64BitVGPROperands(const MCInstrDesc &OpDesc);
 
+/// \returns true if an instruction is a DP ALU DPP without any 64-bit operands.
+bool isDPALU_DPP32BitOpc(unsigned Opc);
+
 /// \returns true if an instruction is a DP ALU DPP.
-bool isDPALU_DPP(const MCInstrDesc &OpDesc);
+bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCSubtargetInfo &ST);
 
 /// \returns true if the intrinsic is divergent
 bool isIntrinsicSourceOfDivergence(unsigned IntrID);
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index b12820782b5e6..11c72751dde58 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -706,7 +706,6 @@ def V_CVT_F16_F8_Fake16_Profile : VOP3_Profile_Fake16<V_CVT_F16_F8_Profile>;
 
 let SubtargetPredicate = isGFX12Plus, OtherPredicates = [HasFP8ConversionInsts],
     mayRaiseFPException = 0, SchedRW = [WriteFloatCvt] in {
-  // FIXME: This differs from downstream due to changes that haven't been upstreamed yet.
   let SubtargetPredicate = isGFX12PlusNot12_50 in
     defm V_CVT_F32_FP8_OP_SEL    : VOP1Inst<"v_cvt_f32_fp8_op_sel", VOPProfile_Base_CVT_F_F8_ByteSel<f32>>;
   let SubtargetPredicate = isGFX125xOnly in
@@ -731,7 +730,6 @@ class Cvt_F_F8_Pat_ByteSel<SDPatternOperator node, VOP3_Pseudo inst, bit HasOpSe
 >;
 
 let OtherPredicates = [HasFP8ConversionInsts] in {
-  // FIXME: This differs from downstream due to changes that haven't been upstreamed yet.
   let SubtargetPredicate = isGFX12PlusNot12_50 in
     def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_fp8, V_CVT_F32_FP8_OP_SEL_e64>;
   let SubtargetPredicate = isGFX125xOnly in {
@@ -740,7 +738,6 @@ let OtherPredicates = [HasFP8ConversionInsts] in {
     def : GCNPat<(int_amdgcn_cvt_f32_fp8_e5m3 i32:$src0, timm:$byte_sel),
                  (V_CVT_F32_FP8_gfx1250_e64 $src0, DSTCLAMP.ENABLE, (as_i32timm $byte_sel))>;
   }
-  // FIXME: This differs from downstream due to changes that haven't been upstreamed yet.
   let SubtargetPredicate = isGFX12Plus in
     def : Cvt_F_F8_Pat_ByteSel<int_amdgcn_cvt_f32_bf8, V_CVT_F32_BF8_OP_SEL_e64>;
 }
@@ -1058,11 +1055,6 @@ multiclass VOP1Only_Real_gfx11_gfx12<bits<9> op> :
 multiclass VOP1_Real_FULL_gfx11_gfx12<bits<9> op> :
   VOP1_Real_FULL<GFX11Gen, op>, VOP1_Real_FULL<GFX12Gen, op>;
 
-multiclass VOP1_Real_NO_DPP_OP_SEL_with_name<GFXGen Gen, bits<9> op,
-                                             string opName, string asmName> :
-  VOP1_Real_e32_with_name<Gen, op, opName, asmName>,
-  VOP3_Real_with_name<Gen, {0, 1, 1, op{6-0}}, opName, asmName>;
-
 multiclass VOP1_Real_FULL_t16_and_fake16_gfx1250<
     bits<9> op, string asmName = !tolower(NAME), string opName = NAME> {
   defm opName#"_t16" :
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index f4b6af647ca1a..329d003cf2506 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -2084,6 +2084,9 @@ multiclass VOP3_Realtriple_gfx11_gfx12<bits<10> op> :
 multiclass VOP3_Real_Base_gfx11_gfx12<bits<10> op> :
   VOP3_Real_Base<GFX11Gen, op>, VOP3_Real_Base<GFX12Gen, op>;
 
+multiclass VOP3_Real_Base_gfx11_gfx12_not_gfx1250<bits<10> op> :
+  VOP3_Real_Base<GFX11Gen, op>, VOP3_Real_Base<GFX12Not12_50Gen, op>;
+
 multiclass VOP3_Realtriple_with_name_gfx11_gfx12<bits<10> op, string opName,
                                                  string asmName> :
   VOP3_Realtriple_with_name<GFX11Gen, op, opName, asmName>,
@@ -2211,9 +2214,9 @@ defm V_MUL_F64             : VOP3_Real_Base_gfx11<0x328>;
 defm V_MIN_F64             : VOP3_Real_Base_gfx11<0x329>;
 defm V_MAX_F64             : VOP3_Real_Base_gfx11<0x32a>;
 defm V_LDEXP_F64           : VOP3_Real_Base_gfx11_gfx12<0x32b>;
-defm V_MUL_LO_U32          : VOP3_Real_Base_gfx11_gfx12<0x32c>;
-defm V_MUL_HI_U32          : VOP3_Real_Base_gfx11_gfx12<0x32d>;
-defm V_MUL_HI_I32          : VOP3_Real_Base_gfx11_gfx12<0x32e>;
+defm V_MUL_LO_U32          : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x32c>;
+defm V_MUL_HI_U32          : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x32d>;
+defm V_MUL_HI_I32          : VOP3_Real_Base_gfx11_gfx12_not_gfx1250<0x32e>;
 defm V_TRIG_PREOP_F64      : VOP3_Real_Base_gfx11_gfx12<0x32f>;
 defm V_LSHLREV_B16         : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x338, "v_lshlrev_b16">;
 defm V_LSHRREV_B16         : VOP3Only_Realtriple_t16_and_fake16_gfx11_gfx12<0x339, "v_lshrrev_b16">;
@@ -2242,6 +2245,10 @@ let AssemblerPredicate = isGFX11Plus in {
 }
 
 // These instructions differ from GFX12 variant by supporting DPP:
+defm V_MUL_LO_U32                    : VOP3Only_Realtriple_gfx1250<0x32c>;
+defm V_MUL_HI_U32                    : VOP3Only_Realtriple_gfx1250<0x32d>;
+defm V_MUL_HI_I32                    : VOP3Only_Realtriple_gfx1250<0x32e>;
+
 defm V_PERM_PK16_B4_U4               : VOP3Only_Real_Base_gfx1250<0x23f>;
 defm V_PERM_PK16_B6_U4               : VOP3Only_Real_Base_gfx1250<0x242>;
 defm V_PERM_PK16_B8_U4               : VOP3Only_Real_Base_gfx1250<0x243>;
diff --git a/llvm/lib/Target/ARC/ARCInstrFormats.td b/llvm/lib/Target/ARC/ARCInstrFormats.td
index d6d2eaffab19b..bd2ed00576177 100644
--- a/llvm/lib/Target/ARC/ARCInstrFormats.td
+++ b/llvm/lib/Target/ARC/ARCInstrFormats.td
@@ -12,7 +12,6 @@
 
 class Encoding64 {
   field bits<64> Inst;
-  field bits<64> SoftFail = 0;
 }
 
 // Address operands
diff --git a/llvm/lib/Target/ARC/Disassembler/ARCDisassembler.cpp b/llvm/lib/Target/ARC/Disassembler/ARCDisassembler.cpp
index 6181017559045..f28c7d8fe92a5 100644
--- a/llvm/lib/Target/ARC/Disassembler/ARCDisassembler.cpp
+++ b/llvm/lib/Target/ARC/Disassembler/ARCDisassembler.cpp
@@ -16,6 +16,7 @@
 #include "MCTargetDesc/ARCMCTargetDesc.h"
 #include "TargetInfo/ARCTargetInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -24,6 +25,7 @@
 #include "llvm/MC/TargetRegistry.h"
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "arc-disassembler"
 
diff --git a/llvm/lib/Target/ARM/ARMCallingConv.h b/llvm/lib/Target/ARM/ARMCallingConv.h
index 7c692f03b4405..b6b2d5928dabc 100644
--- a/llvm/lib/Target/ARM/ARMCallingConv.h
+++ b/llvm/lib/Target/ARM/ARMCallingConv.h
@@ -19,34 +19,35 @@ namespace llvm {
 
 bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                   CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                  CCState &State);
+                  Type *OrigTy, CCState &State);
 bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT,
                       CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                      CCState &State);
+                      Type *OrigTy, CCState &State);
 bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                  CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                 CCState &State);
+                 Type *OrigTy, CCState &State);
 bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
                      CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                     CCState &State);
+                     Type *OrigTy, CCState &State);
 bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                      CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                     CCState &State);
+                     Type *OrigTy, CCState &State);
 bool CC_ARM_Win32_CFGuard_Check(unsigned ValNo, MVT ValVT, MVT LocVT,
                                 CCValAssign::LocInfo LocInfo,
-                                ISD::ArgFlagsTy ArgFlags, CCState &State);
+                                ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                                CCState &State);
 bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                      CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                     CCState &State);
+                     Type *OrigTy, CCState &State);
 bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT,
                          CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                         CCState &State);
+                         Type *OrigTy, CCState &State);
 bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                     CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                    CCState &State);
+                    Type *OrigTy, CCState &State);
 bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT,
                         CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                        CCState &State);
+                        Type *OrigTy, CCState &State);
 
 } // namespace llvm
 
diff --git a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
index ef690838f0f3b..f43ec73db7e1f 100644
--- a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -23,7 +23,6 @@
 #include "Utils/ARMBaseInfo.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
@@ -109,7 +108,7 @@ namespace {
 
     /// NewWaterList - The subset of WaterList that was created since the
     /// previous iteration by inserting unconditional branches.
-    SmallSet<MachineBasicBlock*, 4> NewWaterList;
+    SmallPtrSet<MachineBasicBlock *, 4> NewWaterList;
 
     using water_iterator = std::vector<MachineBasicBlock *>::iterator;
 
diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp
index 7ba2487d2390d..14e1160e70dae 100644
--- a/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -1943,8 +1943,11 @@ bool ARMFastISel::ProcessCallArgs(SmallVectorImpl<Value*> &Args,
                                   unsigned &NumBytes,
                                   bool isVarArg) {
   SmallVector<CCValAssign, 16> ArgLocs;
+  SmallVector<Type *, 16> OrigTys;
+  for (Value *Arg : Args)
+    OrigTys.push_back(Arg->getType());
   CCState CCInfo(CC, isVarArg, *FuncInfo.MF, ArgLocs, *Context);
-  CCInfo.AnalyzeCallOperands(ArgVTs, ArgFlags,
+  CCInfo.AnalyzeCallOperands(ArgVTs, ArgFlags, OrigTys,
                              CCAssignFnForCall(CC, false, isVarArg));
 
   // Check that we can handle all of the arguments. If we can't, then bail out
@@ -2093,7 +2096,8 @@ bool ARMFastISel::FinishCall(MVT RetVT, SmallVectorImpl<Register> &UsedRegs,
   if (RetVT != MVT::isVoid) {
     SmallVector<CCValAssign, 16> RVLocs;
     CCState CCInfo(CC, isVarArg, *FuncInfo.MF, RVLocs, *Context);
-    CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC, true, isVarArg));
+    CCInfo.AnalyzeCallResult(RetVT, I->getType(),
+                             CCAssignFnForCall(CC, true, isVarArg));
 
     // Copy all of the result registers out of their specified physreg.
     if (RVLocs.size() == 2 && RetVT == MVT::f64) {
@@ -2278,7 +2282,7 @@ bool ARMFastISel::ARMEmitLibcall(const Instruction *I, RTLIB::Libcall Call) {
   if (RetVT != MVT::isVoid && RetVT != MVT::i32) {
     SmallVector<CCValAssign, 16> RVLocs;
     CCState CCInfo(CC, false, *FuncInfo.MF, RVLocs, *Context);
-    CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC, true, false));
+    CCInfo.AnalyzeCallResult(RetVT, RetTy, CCAssignFnForCall(CC, true, false));
     if (RVLocs.size() >= 2 && RetVT != MVT::f64)
       return false;
   }
@@ -2389,7 +2393,8 @@ bool ARMFastISel::SelectCall(const Instruction *I,
       RetVT != MVT::i16 && RetVT != MVT::i32) {
     SmallVector<CCValAssign, 16> RVLocs;
     CCState CCInfo(CC, isVarArg, *FuncInfo.MF, RVLocs, *Context);
-    CCInfo.AnalyzeCallResult(RetVT, CCAssignFnForCall(CC, true, isVarArg));
+    CCInfo.AnalyzeCallResult(RetVT, RetTy,
+                             CCAssignFnForCall(CC, true, isVarArg));
     if (RVLocs.size() >= 2 && RetVT != MVT::f64)
       return false;
   }
@@ -2499,6 +2504,7 @@ bool ARMFastISel::SelectCall(const Instruction *I,
   // Set all unused physreg defs as dead.
   static_cast<MachineInstr *>(MIB)->setPhysRegsDeadExcept(UsedRegs, TRI);
 
+  diagnoseDontCall(*CI);
   return true;
 }
 
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index c5bae19e0e02e..12d2d678ff63a 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -587,167 +587,6 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
     }
   }
 
-  // RTLIB
-  if (TM.isAAPCS_ABI() && (TT.isTargetAEABI() || TT.isTargetGNUAEABI() ||
-                           TT.isTargetMuslAEABI() || TT.isAndroid())) {
-    // FIXME: This does not depend on the subtarget and should go directly into
-    // RuntimeLibcalls. This is only here because of missing support for setting
-    // the calling convention of an implementation.
-    // clang-format off
-    static const struct {
-      const RTLIB::Libcall Op;
-      const RTLIB::LibcallImpl Impl;
-    } LibraryCalls[] = {
-      // Double-precision floating-point arithmetic helper functions
-      // RTABI chapter 4.1.2, Table 2
-      { RTLIB::ADD_F64, RTLIB::__aeabi_dadd },
-      { RTLIB::DIV_F64, RTLIB::__aeabi_ddiv },
-      { RTLIB::MUL_F64, RTLIB::__aeabi_dmul },
-      { RTLIB::SUB_F64, RTLIB::__aeabi_dsub },
-
-      // Double-precision floating-point comparison helper functions
-      // RTABI chapter 4.1.2, Table 3
-      { RTLIB::OEQ_F64, RTLIB::__aeabi_dcmpeq__oeq },
-      { RTLIB::UNE_F64, RTLIB::__aeabi_dcmpeq__une },
-      { RTLIB::OLT_F64, RTLIB::__aeabi_dcmplt },
-      { RTLIB::OLE_F64, RTLIB::__aeabi_dcmple },
-      { RTLIB::OGE_F64, RTLIB::__aeabi_dcmpge },
-      { RTLIB::OGT_F64, RTLIB::__aeabi_dcmpgt },
-      { RTLIB::UO_F64,  RTLIB::__aeabi_dcmpun },
-
-      // Single-precision floating-point arithmetic helper functions
-      // RTABI chapter 4.1.2, Table 4
-      { RTLIB::ADD_F32, RTLIB::__aeabi_fadd },
-      { RTLIB::DIV_F32, RTLIB::__aeabi_fdiv },
-      { RTLIB::MUL_F32, RTLIB::__aeabi_fmul },
-      { RTLIB::SUB_F32, RTLIB::__aeabi_fsub },
-
-      // Single-precision floating-point comparison helper functions
-      // RTABI chapter 4.1.2, Table 5
-      { RTLIB::OEQ_F32, RTLIB::__aeabi_fcmpeq__oeq },
-      { RTLIB::UNE_F32, RTLIB::__aeabi_fcmpeq__une },
-      { RTLIB::OLT_F32, RTLIB::__aeabi_fcmplt},
-      { RTLIB::OLE_F32, RTLIB::__aeabi_fcmple },
-      { RTLIB::OGE_F32, RTLIB::__aeabi_fcmpge },
-      { RTLIB::OGT_F32, RTLIB::__aeabi_fcmpgt },
-      { RTLIB::UO_F32,  RTLIB::__aeabi_fcmpun },
-
-      // Floating-point to integer conversions.
-      // RTABI chapter 4.1.2, Table 6
-      { RTLIB::FPTOSINT_F64_I32, RTLIB::__aeabi_d2iz },
-      { RTLIB::FPTOUINT_F64_I32, RTLIB::__aeabi_d2uiz },
-      { RTLIB::FPTOSINT_F64_I64, RTLIB::__aeabi_d2lz },
-      { RTLIB::FPTOUINT_F64_I64, RTLIB::__aeabi_d2ulz },
-      { RTLIB::FPTOSINT_F32_I32, RTLIB::__aeabi_f2iz },
-      { RTLIB::FPTOUINT_F32_I32, RTLIB::__aeabi_f2uiz },
-      { RTLIB::FPTOSINT_F32_I64, RTLIB::__aeabi_f2lz },
-      { RTLIB::FPTOUINT_F32_I64, RTLIB::__aeabi_f2ulz },
-
-      // Conversions between floating types.
-      // RTABI chapter 4.1.2, Table 7
-      { RTLIB::FPROUND_F64_F32, RTLIB::__aeabi_d2f },
-      { RTLIB::FPROUND_F64_F16, RTLIB::__aeabi_d2h },
-      { RTLIB::FPEXT_F32_F64,   RTLIB::__aeabi_f2d },
-
-      // Integer to floating-point conversions.
-      // RTABI chapter 4.1.2, Table 8
-      { RTLIB::SINTTOFP_I32_F64, RTLIB::__aeabi_i2d },
-      { RTLIB::UINTTOFP_I32_F64, RTLIB::__aeabi_ui2d },
-      { RTLIB::SINTTOFP_I64_F64, RTLIB::__aeabi_l2d },
-      { RTLIB::UINTTOFP_I64_F64, RTLIB::__aeabi_ul2d },
-      { RTLIB::SINTTOFP_I32_F32, RTLIB::__aeabi_i2f },
-      { RTLIB::UINTTOFP_I32_F32, RTLIB::__aeabi_ui2f },
-      { RTLIB::SINTTOFP_I64_F32, RTLIB::__aeabi_l2f },
-      { RTLIB::UINTTOFP_I64_F32, RTLIB::__aeabi_ul2f },
-
-      // Long long helper functions
-      // RTABI chapter 4.2, Table 9
-      { RTLIB::MUL_I64, RTLIB::__aeabi_lmul },
-      { RTLIB::SHL_I64, RTLIB::__aeabi_llsl },
-      { RTLIB::SRL_I64, RTLIB::__aeabi_llsr },
-      { RTLIB::SRA_I64, RTLIB::__aeabi_lasr },
-
-      // Integer division functions
-      // RTABI chapter 4.3.1
-      { RTLIB::SDIV_I32, RTLIB::__aeabi_idiv },
-      { RTLIB::SDIV_I64, RTLIB::__aeabi_ldivmod },
-      { RTLIB::UDIV_I32, RTLIB::__aeabi_uidiv },
-      { RTLIB::UDIV_I64, RTLIB::__aeabi_uldivmod },
-    };
-    // clang-format on
-
-    for (const auto &LC : LibraryCalls)
-      setLibcallImpl(LC.Op, LC.Impl);
-
-    // EABI dependent RTLIB
-    if (TM.Options.EABIVersion == EABI::EABI4 ||
-        TM.Options.EABIVersion == EABI::EABI5) {
-      static const struct {
-        const RTLIB::Libcall Op;
-        const RTLIB::LibcallImpl Impl;
-      } MemOpsLibraryCalls[] = {
-          // Memory operations
-          // RTABI chapter 4.3.4
-          {RTLIB::MEMCPY, RTLIB::__aeabi_memcpy},
-          {RTLIB::MEMMOVE, RTLIB::__aeabi_memmove},
-          {RTLIB::MEMSET, RTLIB::__aeabi_memset},
-          {RTLIB::AEABI_MEMCPY4, RTLIB::__aeabi_memcpy4},
-          {RTLIB::AEABI_MEMCPY8, RTLIB::__aeabi_memcpy8},
-          {RTLIB::AEABI_MEMMOVE4, RTLIB::__aeabi_memmove4},
-          {RTLIB::AEABI_MEMMOVE8, RTLIB::__aeabi_memmove8},
-          {RTLIB::AEABI_MEMSET4, RTLIB::__aeabi_memset4},
-          {RTLIB::AEABI_MEMSET8, RTLIB::__aeabi_memset8},
-          {RTLIB::AEABI_MEMCLR, RTLIB::__aeabi_memclr},
-          {RTLIB::AEABI_MEMCLR4, RTLIB::__aeabi_memclr4},
-          {RTLIB::AEABI_MEMCLR8, RTLIB::__aeabi_memclr8},
-      };
-
-      for (const auto &LC : MemOpsLibraryCalls)
-        setLibcallImpl(LC.Op, LC.Impl);
-    }
-  }
-
-  // The half <-> float conversion functions are always soft-float on
-  // non-watchos platforms, but are needed for some targets which use a
-  // hard-float calling convention by default.
-  if (!TT.isWatchABI()) {
-    if (TM.isAAPCS_ABI()) {
-      setLibcallImplCallingConv(RTLIB::__truncsfhf2, CallingConv::ARM_AAPCS);
-      setLibcallImplCallingConv(RTLIB::__truncdfhf2, CallingConv::ARM_AAPCS);
-      setLibcallImplCallingConv(RTLIB::__extendhfsf2, CallingConv::ARM_AAPCS);
-      setLibcallImplCallingConv(RTLIB::__gnu_h2f_ieee, CallingConv::ARM_AAPCS);
-      setLibcallImplCallingConv(RTLIB::__gnu_f2h_ieee, CallingConv::ARM_AAPCS);
-    } else {
-      setLibcallImplCallingConv(RTLIB::__truncsfhf2, CallingConv::ARM_APCS);
-      setLibcallImplCallingConv(RTLIB::__truncdfhf2, CallingConv::ARM_APCS);
-      setLibcallImplCallingConv(RTLIB::__extendhfsf2, CallingConv::ARM_APCS);
-      setLibcallImplCallingConv(RTLIB::__gnu_h2f_ieee, CallingConv::ARM_APCS);
-      setLibcallImplCallingConv(RTLIB::__gnu_f2h_ieee, CallingConv::ARM_APCS);
-    }
-  }
-
-  // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have
-  // a __gnu_ prefix (which is the default).
-  if (TT.isTargetAEABI()) {
-    // FIXME: This does not depend on the subtarget and should go directly into
-    // RuntimeLibcalls. This is only here because of missing support for setting
-    // the calling convention of an implementation.
-    static const struct {
-      const RTLIB::Libcall Op;
-      const RTLIB::LibcallImpl Impl;
-    } LibraryCalls[] = {
-        {RTLIB::FPROUND_F32_F16, RTLIB::__aeabi_f2h},
-        {RTLIB::FPEXT_F16_F32, RTLIB::__aeabi_h2f},
-    };
-
-    for (const auto &LC : LibraryCalls) {
-      setLibcallImpl(LC.Op, LC.Impl);
-    }
-  } else if (!TT.isOSBinFormatMachO()) {
-    setLibcallImpl(RTLIB::FPROUND_F32_F16, RTLIB::__gnu_f2h_ieee);
-    setLibcallImpl(RTLIB::FPEXT_F16_F32, RTLIB::__gnu_h2f_ieee);
-  }
-
   if (Subtarget->isThumb1Only())
     addRegisterClass(MVT::i32, &ARM::tGPRRegClass);
   else
@@ -802,6 +641,12 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
     setOperationAction(ISD::BSWAP, VT, Expand);
   }
 
+  if (!Subtarget->isThumb1Only() && !Subtarget->hasV8_1MMainlineOps())
+    setOperationAction(ISD::SCMP, MVT::i32, Custom);
+
+  if (!Subtarget->hasV8_1MMainlineOps())
+    setOperationAction(ISD::UCMP, MVT::i32, Custom);
+
   setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
   setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
 
@@ -1634,6 +1479,10 @@ bool ARMTargetLowering::useSoftFloat() const {
   return Subtarget->useSoftFloat();
 }
 
+bool ARMTargetLowering::shouldExpandCmpUsingSelects(EVT VT) const {
+  return !Subtarget->isThumb1Only() && VT.getSizeInBits() <= 32;
+}
+
 // FIXME: It might make sense to define the representative register class as the
 // nearest super-register that has a non-null superset. For example, DPR_VFP2 is
 // a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
@@ -3769,10 +3618,7 @@ ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
 
   // call __tls_get_addr.
   ArgListTy Args;
-  ArgListEntry Entry;
-  Entry.Node = Argument;
-  Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext());
-  Args.push_back(Entry);
+  Args.emplace_back(Argument, Type::getInt32Ty(*DAG.getContext()));
 
   // FIXME: is there useful debug info available here?
   TargetLowering::CallLoweringInfo CLI(DAG);
@@ -7396,7 +7242,7 @@ static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
-  if (M.size() != NumElts && M.size() != NumElts*2)
+  if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
     return false;
 
   // If the mask is twice as long as the input vector then we need to check the
@@ -7428,7 +7274,7 @@ static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
-  if (M.size() != NumElts && M.size() != NumElts*2)
+  if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
     return false;
 
   for (unsigned i = 0; i < M.size(); i += NumElts) {
@@ -7531,7 +7377,7 @@ static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
-  if (M.size() != NumElts && M.size() != NumElts*2)
+  if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
     return false;
 
   for (unsigned i = 0; i < M.size(); i += NumElts) {
@@ -7564,7 +7410,7 @@ static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
     return false;
 
   unsigned NumElts = VT.getVectorNumElements();
-  if (M.size() != NumElts && M.size() != NumElts*2)
+  if ((M.size() != NumElts && M.size() != NumElts * 2) || NumElts % 2 != 0)
     return false;
 
   for (unsigned i = 0; i < M.size(); i += NumElts) {
@@ -9991,9 +9837,7 @@ SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
     int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
     SRet = DAG.getFrameIndex(FrameIdx, getPointerTy(DL));
 
-    ArgListEntry Entry;
-    Entry.Node = SRet;
-    Entry.Ty = PointerType::getUnqual(RetTy->getContext());
+    ArgListEntry Entry(SRet, PointerType::getUnqual(RetTy->getContext()));
     Entry.IsSExt = false;
     Entry.IsZExt = false;
     Entry.IsSRet = true;
@@ -10001,12 +9845,7 @@ SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
     RetTy = Type::getVoidTy(*DAG.getContext());
   }
 
-  ArgListEntry Entry;
-  Entry.Node = Arg;
-  Entry.Ty = ArgTy;
-  Entry.IsSExt = false;
-  Entry.IsZExt = false;
-  Args.push_back(Entry);
+  Args.emplace_back(Arg, ArgTy);
 
   RTLIB::Libcall LC =
       (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
@@ -10059,10 +9898,9 @@ SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
   ARMTargetLowering::ArgListTy Args;
 
   for (auto AI : {1, 0}) {
-    ArgListEntry Arg;
-    Arg.Node = Op.getOperand(AI);
-    Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.getContext());
-    Args.push_back(Arg);
+    SDValue Operand = Op.getOperand(AI);
+    Args.emplace_back(Operand,
+                      Operand.getValueType().getTypeForEVT(*DAG.getContext()));
   }
 
   CallLoweringInfo CLI(DAG);
@@ -10612,6 +10450,133 @@ SDValue ARMTargetLowering::LowerFP_TO_BF16(SDValue Op,
   return DAG.getBitcast(MVT::i32, Res);
 }
 
+SDValue ARMTargetLowering::LowerCMP(SDValue Op, SelectionDAG &DAG) const {
+  SDLoc dl(Op);
+  SDValue LHS = Op.getOperand(0);
+  SDValue RHS = Op.getOperand(1);
+
+  // Determine if this is signed or unsigned comparison
+  bool IsSigned = (Op.getOpcode() == ISD::SCMP);
+
+  // Special case for Thumb1 UCMP only
+  if (!IsSigned && Subtarget->isThumb1Only()) {
+    // For Thumb unsigned comparison, use this sequence:
+    // subs r2, r0, r1   ; r2 = LHS - RHS, sets flags
+    // sbc r2, r2        ; r2 = r2 - r2 - !carry
+    // cmp r1, r0        ; compare RHS with LHS
+    // sbc r1, r1        ; r1 = r1 - r1 - !carry
+    // subs r0, r2, r1   ; r0 = r2 - r1 (final result)
+
+    // First subtraction: LHS - RHS
+    SDValue Sub1WithFlags = DAG.getNode(
+        ARMISD::SUBC, dl, DAG.getVTList(MVT::i32, FlagsVT), LHS, RHS);
+    SDValue Sub1Result = Sub1WithFlags.getValue(0);
+    SDValue Flags1 = Sub1WithFlags.getValue(1);
+
+    // SUBE: Sub1Result - Sub1Result - !carry
+    // This gives 0 if LHS >= RHS (unsigned), -1 if LHS < RHS (unsigned)
+    SDValue Sbc1 =
+        DAG.getNode(ARMISD::SUBE, dl, DAG.getVTList(MVT::i32, FlagsVT),
+                    Sub1Result, Sub1Result, Flags1);
+    SDValue Sbc1Result = Sbc1.getValue(0);
+
+    // Second comparison: RHS vs LHS (reverse comparison)
+    SDValue CmpFlags = DAG.getNode(ARMISD::CMP, dl, FlagsVT, RHS, LHS);
+
+    // SUBE: RHS - RHS - !carry
+    // This gives 0 if RHS <= LHS (unsigned), -1 if RHS > LHS (unsigned)
+    SDValue Sbc2 = DAG.getNode(
+        ARMISD::SUBE, dl, DAG.getVTList(MVT::i32, FlagsVT), RHS, RHS, CmpFlags);
+    SDValue Sbc2Result = Sbc2.getValue(0);
+
+    // Final subtraction: Sbc1Result - Sbc2Result (no flags needed)
+    SDValue Result =
+        DAG.getNode(ISD::SUB, dl, MVT::i32, Sbc1Result, Sbc2Result);
+    if (Op.getValueType() != MVT::i32)
+      Result = DAG.getSExtOrTrunc(Result, dl, Op.getValueType());
+
+    return Result;
+  }
+
+  // For the ARM assembly pattern:
+  // subs r0, r0, r1   ; subtract RHS from LHS and set flags
+  // movgt r0, #1      ; if LHS > RHS, set result to 1 (GT for signed, HI for
+  // unsigned) mvnlt r0, #0      ; if LHS < RHS, set result to -1 (LT for
+  // signed, LO for unsigned)
+  //                   ; if LHS == RHS, result remains 0 from the subs
+
+  // Optimization: if RHS is a subtraction against 0, use ADDC instead of SUBC
+  unsigned Opcode = ARMISD::SUBC;
+
+  // Check if RHS is a subtraction against 0: (0 - X)
+  if (RHS.getOpcode() == ISD::SUB) {
+    SDValue SubLHS = RHS.getOperand(0);
+    SDValue SubRHS = RHS.getOperand(1);
+
+    // Check if it's 0 - X
+    if (isNullConstant(SubLHS)) {
+      bool CanUseAdd = false;
+      if (IsSigned) {
+        // For SCMP: only if X is known to never be INT_MIN (to avoid overflow)
+        if (RHS->getFlags().hasNoSignedWrap() || !DAG.computeKnownBits(SubRHS)
+                                                      .getSignedMinValue()
+                                                      .isMinSignedValue()) {
+          CanUseAdd = true;
+        }
+      } else {
+        // For UCMP: only if X is known to never be zero
+        if (DAG.isKnownNeverZero(SubRHS)) {
+          CanUseAdd = true;
+        }
+      }
+
+      if (CanUseAdd) {
+        Opcode = ARMISD::ADDC;
+        RHS = SubRHS; // Replace RHS with X, so we do LHS + X instead of
+                      // LHS - (0 - X)
+      }
+    }
+  }
+
+  // Generate the operation with flags
+  SDValue OpWithFlags;
+  if (Opcode == ARMISD::ADDC) {
+    // Use ADDC: LHS + RHS (where RHS was 0 - X, now X)
+    OpWithFlags = DAG.getNode(ARMISD::ADDC, dl,
+                              DAG.getVTList(MVT::i32, FlagsVT), LHS, RHS);
+  } else {
+    // Use ARMISD::SUBC to generate SUBS instruction (subtract with flags)
+    OpWithFlags = DAG.getNode(ARMISD::SUBC, dl,
+                              DAG.getVTList(MVT::i32, FlagsVT), LHS, RHS);
+  }
+
+  SDValue OpResult = OpWithFlags.getValue(0); // The operation result
+  SDValue Flags = OpWithFlags.getValue(1);    // The flags
+
+  // Constants for conditional moves
+  SDValue One = DAG.getConstant(1, dl, MVT::i32);
+  SDValue MinusOne = DAG.getAllOnesConstant(dl, MVT::i32);
+
+  // Select condition codes based on signed vs unsigned
+  ARMCC::CondCodes GTCond = IsSigned ? ARMCC::GT : ARMCC::HI;
+  ARMCC::CondCodes LTCond = IsSigned ? ARMCC::LT : ARMCC::LO;
+
+  // First conditional move: if greater than, set to 1
+  SDValue GTCondValue = DAG.getConstant(GTCond, dl, MVT::i32);
+  SDValue Result1 = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, OpResult, One,
+                                GTCondValue, Flags);
+
+  // Second conditional move: if less than, set to -1
+  SDValue LTCondValue = DAG.getConstant(LTCond, dl, MVT::i32);
+  SDValue Result2 = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, Result1, MinusOne,
+                                LTCondValue, Flags);
+
+  if (Op.getValueType() != MVT::i32)
+    Result2 = DAG.getSExtOrTrunc(Result2, dl, Op.getValueType());
+
+  return Result2;
+}
+
 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   LLVM_DEBUG(dbgs() << "Lowering node: "; Op.dump());
   switch (Op.getOpcode()) {
@@ -10740,6 +10705,9 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FP_TO_BF16:
     return LowerFP_TO_BF16(Op, DAG);
   case ARMISD::WIN__DBZCHK: return SDValue();
+  case ISD::UCMP:
+  case ISD::SCMP:
+    return LowerCMP(Op, DAG);
   }
 }
 
@@ -20627,12 +20595,10 @@ static TargetLowering::ArgListTy getDivRemArgList(
   bool isSigned = N->getOpcode() == ISD::SDIVREM ||
                   N->getOpcode() == ISD::SREM;
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
   for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
     EVT ArgVT = N->getOperand(i).getValueType();
     Type *ArgTy = ArgVT.getTypeForEVT(*Context);
-    Entry.Node = N->getOperand(i);
-    Entry.Ty = ArgTy;
+    TargetLowering::ArgListEntry Entry(N->getOperand(i), ArgTy);
     Entry.IsSExt = isSigned;
     Entry.IsZExt = !isSigned;
     Args.push_back(Entry);
@@ -20784,9 +20750,8 @@ ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
     Chain = SP.getValue(1);
     SP = DAG.getNode(ISD::SUB, DL, MVT::i32, SP, Size);
     if (Align)
-      SP = DAG.getNode(
-          ISD::AND, DL, MVT::i32, SP.getValue(0),
-          DAG.getSignedConstant(-(uint64_t)Align->value(), DL, MVT::i32));
+      SP = DAG.getNode(ISD::AND, DL, MVT::i32, SP.getValue(0),
+                       DAG.getSignedConstant(-Align->value(), DL, MVT::i32));
     Chain = DAG.getCopyToReg(Chain, DL, ARM::SP, SP);
     SDValue Ops[2] = { SP, Chain };
     return DAG.getMergeValues(Ops, DL);
@@ -21376,20 +21341,6 @@ void ARMTargetLowering::insertSSPDeclarations(Module &M) const {
     F->addParamAttr(0, Attribute::AttrKind::InReg);
 }
 
-Value *ARMTargetLowering::getSDagStackGuard(const Module &M) const {
-  RTLIB::LibcallImpl SecurityCheckCookieLibcall =
-      getLibcallImpl(RTLIB::SECURITY_CHECK_COOKIE);
-  if (SecurityCheckCookieLibcall != RTLIB::Unsupported) {
-    // MSVC CRT has a global variable holding security cookie.
-    //
-    // FIXME: We have a libcall entry for the correlated check function, but not
-    // the global name.
-    return M.getGlobalVariable("__security_cookie");
-  }
-
-  return TargetLowering::getSDagStackGuard(M);
-}
-
 Function *ARMTargetLowering::getSSPStackGuardCheck(const Module &M) const {
   // MSVC CRT has a function to validate security cookie.
   RTLIB::LibcallImpl SecurityCheckCookie =
@@ -21606,7 +21557,7 @@ unsigned ARMTargetLowering::getMaxSupportedInterleaveFactor() const {
 ///        %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
 bool ARMTargetLowering::lowerInterleavedLoad(
     Instruction *Load, Value *Mask, ArrayRef<ShuffleVectorInst *> Shuffles,
-    ArrayRef<unsigned> Indices, unsigned Factor) const {
+    ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const {
   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
          "Invalid interleave factor");
   assert(!Shuffles.empty() && "Empty shufflevector input");
@@ -21616,7 +21567,7 @@ bool ARMTargetLowering::lowerInterleavedLoad(
   auto *LI = dyn_cast<LoadInst>(Load);
   if (!LI)
     return false;
-  assert(!Mask && "Unexpected mask on a load");
+  assert(!Mask && GapMask.popcount() == Factor && "Unexpected mask on a load");
 
   auto *VecTy = cast<FixedVectorType>(Shuffles[0]->getType());
   Type *EltTy = VecTy->getElementType();
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 825145d813fb1..0185c8ddd4928 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -607,6 +607,8 @@ class VectorType;
 
     bool preferZeroCompareBranch() const override { return true; }
 
+    bool shouldExpandCmpUsingSelects(EVT VT) const override;
+
     bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
 
     bool hasAndNotCompare(SDValue V) const override {
@@ -683,8 +685,8 @@ class VectorType;
 
     bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
                               ArrayRef<ShuffleVectorInst *> Shuffles,
-                              ArrayRef<unsigned> Indices,
-                              unsigned Factor) const override;
+                              ArrayRef<unsigned> Indices, unsigned Factor,
+                              const APInt &GapMask) const override;
     bool lowerInterleavedStore(Instruction *Store, Value *Mask,
                                ShuffleVectorInst *SVI,
                                unsigned Factor) const override;
@@ -702,7 +704,6 @@ class VectorType;
     bool useLoadStackGuardNode(const Module &M) const override;
 
     void insertSSPDeclarations(Module &M) const override;
-    Value *getSDagStackGuard(const Module &M) const override;
     Function *getSSPStackGuardCheck(const Module &M) const override;
 
     bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
@@ -904,6 +905,7 @@ class VectorType;
     void LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
                    SelectionDAG &DAG) const;
     SDValue LowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerCMP(SDValue Op, SelectionDAG &DAG) const;
 
     Register getRegisterByName(const char* RegName, LLT VT,
                                const MachineFunction &MF) const override;
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index 934ec52c6f1e4..4345f7a4a0ebc 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -164,10 +164,9 @@ def ARMWrapperPIC    : SDNode<"ARMISD::WrapperPIC",  SDTIntUnaryOp>;
 def ARMWrapperJT     : SDNode<"ARMISD::WrapperJT",   SDTIntUnaryOp>;
 
 def ARMcallseq_start : SDNode<"ISD::CALLSEQ_START", SDT_ARMCallSeqStart,
-                              [SDNPHasChain, SDNPSideEffect, SDNPOutGlue]>;
+                              [SDNPHasChain, SDNPOutGlue]>;
 def ARMcallseq_end   : SDNode<"ISD::CALLSEQ_END",   SDT_ARMCallSeqEnd,
-                              [SDNPHasChain, SDNPSideEffect,
-                               SDNPOptInGlue, SDNPOutGlue]>;
+                              [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 def ARMcopystructbyval : SDNode<"ARMISD::COPY_STRUCT_BYVAL" ,
                                 SDT_ARMStructByVal,
                                 [SDNPHasChain, SDNPInGlue, SDNPOutGlue,
diff --git a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
index b4677a8bfb035..ebfa593fbe9e6 100644
--- a/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMSelectionDAGInfo.cpp
@@ -89,19 +89,15 @@ SDValue ARMSelectionDAGInfo::EmitSpecializedLibcall(
     AlignVariant = ALIGN1;
 
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
-  Entry.Node = Dst;
-  Args.push_back(Entry);
+  Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
+  Args.emplace_back(Dst, IntPtrTy);
   if (AEABILibcall == AEABI_MEMCLR) {
-    Entry.Node = Size;
-    Args.push_back(Entry);
+    Args.emplace_back(Size, IntPtrTy);
   } else if (AEABILibcall == AEABI_MEMSET) {
     // Adjust parameters for memset, EABI uses format (ptr, size, value),
     // GNU library uses (ptr, value, size)
     // See RTABI section 4.3.4
-    Entry.Node = Size;
-    Args.push_back(Entry);
+    Args.emplace_back(Size, IntPtrTy);
 
     // Extend or truncate the argument to be an i32 value for the call.
     if (Src.getValueType().bitsGT(MVT::i32))
@@ -109,16 +105,13 @@ SDValue ARMSelectionDAGInfo::EmitSpecializedLibcall(
     else if (Src.getValueType().bitsLT(MVT::i32))
       Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src);
 
-    Entry.Node = Src;
-    Entry.Ty = Type::getInt32Ty(*DAG.getContext());
+    TargetLowering::ArgListEntry Entry(Src,
+                                       Type::getInt32Ty(*DAG.getContext()));
     Entry.IsSExt = false;
     Args.push_back(Entry);
   } else {
-    Entry.Node = Src;
-    Args.push_back(Entry);
-
-    Entry.Node = Size;
-    Args.push_back(Entry);
+    Args.emplace_back(Src, IntPtrTy);
+    Args.emplace_back(Size, IntPtrTy);
   }
 
   static const RTLIB::Libcall FunctionImpls[4][3] = {
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index 6f37eca2b00a5..6b2854171c819 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1084,9 +1084,10 @@ InstructionCost ARMTTIImpl::getCmpSelInstrCost(
                                               CostKind, Op1Info, Op2Info, I);
 }
 
-InstructionCost ARMTTIImpl::getAddressComputationCost(Type *Ty,
-                                                      ScalarEvolution *SE,
-                                                      const SCEV *Ptr) const {
+InstructionCost
+ARMTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
+                                      const SCEV *Ptr,
+                                      TTI::TargetCostKind CostKind) const {
   // Address computations in vectorized code with non-consecutive addresses will
   // likely result in more instructions compared to scalar code where the
   // computation can more often be merged into the index mode. The resulting
@@ -1095,7 +1096,7 @@ InstructionCost ARMTTIImpl::getAddressComputationCost(Type *Ty,
   int MaxMergeDistance = 64;
 
   if (ST->hasNEON()) {
-    if (Ty->isVectorTy() && SE &&
+    if (PtrTy->isVectorTy() && SE &&
         !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MaxMergeDistance + 1))
       return NumVectorInstToHideOverhead;
 
@@ -1103,7 +1104,7 @@ InstructionCost ARMTTIImpl::getAddressComputationCost(Type *Ty,
     // addressing mode.
     return 1;
   }
-  return BaseT::getAddressComputationCost(Ty, SE, Ptr);
+  return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
 }
 
 bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) const {
@@ -1335,6 +1336,39 @@ InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
 
     if (!Mask.empty()) {
       std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcTy);
+      // Check for LD2/LD4 instructions, which are represented in llvm IR as
+      // deinterleaving-shuffle(load). The shuffle cost could potentially be
+      // free, but we model it with a cost of LT.first so that LD2/LD4 have a
+      // higher cost than just the load.
+      if (Args.size() >= 1 && isa<LoadInst>(Args[0]) &&
+          (LT.second.getScalarSizeInBits() == 8 ||
+           LT.second.getScalarSizeInBits() == 16 ||
+           LT.second.getScalarSizeInBits() == 32) &&
+          LT.second.getSizeInBits() == 128 &&
+          ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
+            ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2)) ||
+           (TLI->getMaxSupportedInterleaveFactor() == 4 &&
+            ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 4))))
+        return ST->getMVEVectorCostFactor(CostKind) *
+               std::max<InstructionCost>(1, LT.first / 4);
+
+      // Check for ST2/ST4 instructions, which are represented in llvm IR as
+      // store(interleaving-shuffle). The shuffle cost could potentially be
+      // free, but we model it with a cost of LT.first so that ST2/ST4 have a
+      // higher cost than just the store.
+      if (CxtI && CxtI->hasOneUse() && isa<StoreInst>(*CxtI->user_begin()) &&
+          (LT.second.getScalarSizeInBits() == 8 ||
+           LT.second.getScalarSizeInBits() == 16 ||
+           LT.second.getScalarSizeInBits() == 32) &&
+          LT.second.getSizeInBits() == 128 &&
+          ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
+            ShuffleVectorInst::isInterleaveMask(
+                Mask, 2, SrcTy->getElementCount().getKnownMinValue() * 2)) ||
+           (TLI->getMaxSupportedInterleaveFactor() == 4 &&
+            ShuffleVectorInst::isInterleaveMask(
+                Mask, 4, SrcTy->getElementCount().getKnownMinValue() * 2))))
+        return ST->getMVEVectorCostFactor(CostKind) * LT.first;
+
       if (LT.second.isVector() &&
           Mask.size() <= LT.second.getVectorNumElements() &&
           (isVREVMask(Mask, LT.second, 16) || isVREVMask(Mask, LT.second, 32) ||
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 522c235a90a8f..cdd8bcb9f7416 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -257,8 +257,9 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
                                      unsigned Index, const Value *Op0,
                                      const Value *Op1) const override;
 
-  InstructionCost getAddressComputationCost(Type *Val, ScalarEvolution *SE,
-                                            const SCEV *Ptr) const override;
+  InstructionCost
+  getAddressComputationCost(Type *Val, ScalarEvolution *SE, const SCEV *Ptr,
+                            TTI::TargetCostKind CostKind) const override;
 
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index ece6c10e828db..0e974838a7c6b 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -3373,12 +3373,12 @@ class ARMOperand : public MCParsedAsmOperand {
 
   void addMSRMaskOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createImm(unsigned(getMSRMask())));
+    Inst.addOperand(MCOperand::createImm(getMSRMask()));
   }
 
   void addBankedRegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    Inst.addOperand(MCOperand::createImm(unsigned(getBankedReg())));
+    Inst.addOperand(MCOperand::createImm(getBankedReg()));
   }
 
   void addProcIFlagsOperands(MCInst &Inst, unsigned N) const {
diff --git a/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp b/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
index 2e47ceeca96b8..19fa03cdc668d 100644
--- a/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
+++ b/llvm/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
@@ -13,6 +13,7 @@
 #include "TargetInfo/ARMTargetInfo.h"
 #include "Utils/ARMBaseInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -31,6 +32,7 @@
 #include <vector>
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "arm-disassembler"
 
@@ -38,94 +40,80 @@ using DecodeStatus = MCDisassembler::DecodeStatus;
 
 namespace {
 
-  // Handles the condition code status of instructions in IT blocks
-  class ITStatus
-  {
-    public:
-      // Returns the condition code for instruction in IT block
-      unsigned getITCC() {
-        unsigned CC = ARMCC::AL;
-        if (instrInITBlock())
-          CC = ITStates.back();
-        return CC;
-      }
-
-      // Advances the IT block state to the next T or E
-      void advanceITState() {
-        ITStates.pop_back();
-      }
-
-      // Returns true if the current instruction is in an IT block
-      bool instrInITBlock() {
-        return !ITStates.empty();
-      }
-
-      // Returns true if current instruction is the last instruction in an IT block
-      bool instrLastInITBlock() {
-        return ITStates.size() == 1;
-      }
-
-      // Called when decoding an IT instruction. Sets the IT state for
-      // the following instructions that for the IT block. Firstcond
-      // corresponds to the field in the IT instruction encoding; Mask
-      // is in the MCOperand format in which 1 means 'else' and 0 'then'.
-      void setITState(char Firstcond, char Mask) {
-        // (3 - the number of trailing zeros) is the number of then / else.
-        unsigned NumTZ = llvm::countr_zero<uint8_t>(Mask);
-        unsigned char CCBits = static_cast<unsigned char>(Firstcond & 0xf);
-        assert(NumTZ <= 3 && "Invalid IT mask!");
-        // push condition codes onto the stack the correct order for the pops
-        for (unsigned Pos = NumTZ+1; Pos <= 3; ++Pos) {
-          unsigned Else = (Mask >> Pos) & 1;
-          ITStates.push_back(CCBits ^ Else);
-        }
-        ITStates.push_back(CCBits);
-      }
+// Handles the condition code status of instructions in IT blocks
+class ITStatus {
+public:
+  // Returns the condition code for instruction in IT block
+  unsigned getITCC() {
+    unsigned CC = ARMCC::AL;
+    if (instrInITBlock())
+      CC = ITStates.back();
+    return CC;
+  }
+
+  // Advances the IT block state to the next T or E
+  void advanceITState() { ITStates.pop_back(); }
+
+  // Returns true if the current instruction is in an IT block
+  bool instrInITBlock() { return !ITStates.empty(); }
+
+  // Returns true if current instruction is the last instruction in an IT block
+  bool instrLastInITBlock() { return ITStates.size() == 1; }
+
+  // Called when decoding an IT instruction. Sets the IT state for
+  // the following instructions that for the IT block. Firstcond
+  // corresponds to the field in the IT instruction encoding; Mask
+  // is in the MCOperand format in which 1 means 'else' and 0 'then'.
+  void setITState(char Firstcond, char Mask) {
+    // (3 - the number of trailing zeros) is the number of then / else.
+    unsigned NumTZ = llvm::countr_zero<uint8_t>(Mask);
+    unsigned char CCBits = static_cast<unsigned char>(Firstcond & 0xf);
+    assert(NumTZ <= 3 && "Invalid IT mask!");
+    // push condition codes onto the stack the correct order for the pops
+    for (unsigned Pos = NumTZ + 1; Pos <= 3; ++Pos) {
+      unsigned Else = (Mask >> Pos) & 1;
+      ITStates.push_back(CCBits ^ Else);
+    }
+    ITStates.push_back(CCBits);
+  }
 
-    private:
-      std::vector<unsigned char> ITStates;
-  };
+private:
+  std::vector<unsigned char> ITStates;
+};
 
-  class VPTStatus
-  {
-    public:
-      unsigned getVPTPred() {
-        unsigned Pred = ARMVCC::None;
-        if (instrInVPTBlock())
-          Pred = VPTStates.back();
-        return Pred;
-      }
+class VPTStatus {
+public:
+  unsigned getVPTPred() {
+    unsigned Pred = ARMVCC::None;
+    if (instrInVPTBlock())
+      Pred = VPTStates.back();
+    return Pred;
+  }
 
-      void advanceVPTState() {
-        VPTStates.pop_back();
-      }
+  void advanceVPTState() { VPTStates.pop_back(); }
 
-      bool instrInVPTBlock() {
-        return !VPTStates.empty();
-      }
+  bool instrInVPTBlock() { return !VPTStates.empty(); }
 
-      bool instrLastInVPTBlock() {
-        return VPTStates.size() == 1;
-      }
+  bool instrLastInVPTBlock() { return VPTStates.size() == 1; }
 
-      void setVPTState(char Mask) {
-        // (3 - the number of trailing zeros) is the number of then / else.
-        unsigned NumTZ = llvm::countr_zero<uint8_t>(Mask);
-        assert(NumTZ <= 3 && "Invalid VPT mask!");
-        // push predicates onto the stack the correct order for the pops
-        for (unsigned Pos = NumTZ+1; Pos <= 3; ++Pos) {
-          bool T = ((Mask >> Pos) & 1) == 0;
-          if (T)
-            VPTStates.push_back(ARMVCC::Then);
-          else
-            VPTStates.push_back(ARMVCC::Else);
-        }
+  void setVPTState(char Mask) {
+    // (3 - the number of trailing zeros) is the number of then / else.
+    unsigned NumTZ = llvm::countr_zero<uint8_t>(Mask);
+    assert(NumTZ <= 3 && "Invalid VPT mask!");
+    // push predicates onto the stack the correct order for the pops
+    for (unsigned Pos = NumTZ + 1; Pos <= 3; ++Pos) {
+      bool T = ((Mask >> Pos) & 1) == 0;
+      if (T)
         VPTStates.push_back(ARMVCC::Then);
-      }
+      else
+        VPTStates.push_back(ARMVCC::Else);
+    }
+    VPTStates.push_back(ARMVCC::Then);
+  }
 
-    private:
-      SmallVector<unsigned char, 4> VPTStates;
-  };
+private:
+  SmallVector<unsigned char, 4> VPTStates;
+};
 
 /// ARM disassembler for all ARM platforms.
 class ARMDisassembler : public MCDisassembler {
@@ -1264,6 +1252,10 @@ DecodeStatus ARMDisassembler::getThumbInstruction(MCInst &MI, uint64_t &Size,
     return Result;
   }
 
+  // Advance IT state to prevent next instruction inheriting
+  // the wrong IT state.
+  if (ITBlock.instrInITBlock())
+    ITBlock.advanceITState();
   Size = 0;
   return MCDisassembler::Fail;
 }
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index cc1c79b7ef9fd..01fe13b343926 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -25,6 +25,7 @@
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCSymbolMachO.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/Debug.h"
@@ -437,13 +438,14 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm,
   // Other relocation types don't want this bit though (branches couldn't encode
   // it if it *was* present, and no other relocations exist) and it can
   // interfere with checking valid expressions.
-  bool IsMachO = getContext().getObjectFileType() == MCContext::IsMachO;
-  if (const auto *SA = Target.getAddSym()) {
-    if (IsMachO && Asm.isThumbFunc(SA) && SA->isExternal() &&
-        (Kind == FK_Data_4 || Kind == ARM::fixup_arm_movw_lo16 ||
-         Kind == ARM::fixup_arm_movt_hi16 || Kind == ARM::fixup_t2_movw_lo16 ||
-         Kind == ARM::fixup_t2_movt_hi16))
-      Value |= 1;
+  if (getContext().getObjectFileType() == MCContext::IsMachO) {
+    if (auto *SA = static_cast<const MCSymbolMachO *>(Target.getAddSym())) {
+      if (Asm.isThumbFunc(SA) && SA->isExternal() &&
+          (Kind == FK_Data_4 || Kind == ARM::fixup_arm_movw_lo16 ||
+           Kind == ARM::fixup_arm_movt_hi16 ||
+           Kind == ARM::fixup_t2_movw_lo16 || Kind == ARM::fixup_t2_movt_hi16))
+        Value |= 1;
+    }
   }
 
   switch (Kind) {
@@ -935,17 +937,6 @@ bool ARMAsmBackend::shouldForceRelocation(const MCFixup &Fixup,
                                           const MCValue &Target) {
   const MCSymbol *Sym = Target.getAddSym();
   const unsigned FixupKind = Fixup.getKind();
-  if (FixupKind == ARM::fixup_arm_thumb_bl) {
-    assert(Sym && "How did we resolve this?");
-
-    // If the symbol is external the linker will handle it.
-    // FIXME: Should we handle it as an optimization?
-
-    // If the symbol is out of range, produce a relocation and hope the
-    // linker can handle it. GNU AS produces an error in this case.
-    if (Sym->isExternal())
-      return true;
-  }
   // Create relocations for unconditional branches to function symbols with
   // different execution mode in ELF binaries.
   if (needsInterworking(*Asm, Sym, Fixup.getKind()))
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
index 8ee3a2d26a599..a5266a9df3183 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
@@ -20,7 +20,6 @@
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCSymbolMachO.h"
 #include "llvm/MC/MCValue.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/ErrorHandling.h"
 
 using namespace llvm;
diff --git a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp
index 0b4e7dfebe369..5eeb4fe995485 100644
--- a/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp
+++ b/llvm/lib/Target/ARM/MVETPAndVPTOptimisationsPass.cpp
@@ -922,7 +922,7 @@ bool MVETPAndVPTOptimisations::ReplaceConstByVPNOTs(MachineBasicBlock &MBB,
   // the function.
   unsigned LastVPTImm = 0;
   Register LastVPTReg = 0;
-  SmallSet<MachineInstr *, 4> DeadInstructions;
+  SmallPtrSet<MachineInstr *, 4> DeadInstructions;
 
   for (MachineInstr &Instr : MBB.instrs()) {
     // Look for predicated MVE instructions.
diff --git a/llvm/lib/Target/AVR/AVRISelLowering.cpp b/llvm/lib/Target/AVR/AVRISelLowering.cpp
index 3955f2a252e76..545bc3af05383 100644
--- a/llvm/lib/Target/AVR/AVRISelLowering.cpp
+++ b/llvm/lib/Target/AVR/AVRISelLowering.cpp
@@ -505,10 +505,9 @@ SDValue AVRTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
   SDValue InChain = DAG.getEntryNode();
 
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
   for (SDValue const &Value : Op->op_values()) {
-    Entry.Node = Value;
-    Entry.Ty = Value.getValueType().getTypeForEVT(*DAG.getContext());
+    TargetLowering::ArgListEntry Entry(
+        Value, Value.getValueType().getTypeForEVT(*DAG.getContext()));
     Entry.IsSExt = IsSigned;
     Entry.IsZExt = !IsSigned;
     Args.push_back(Entry);
@@ -669,7 +668,7 @@ SDValue AVRTargetLowering::getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
       default: {
         // Turn lhs < rhs with lhs constant into rhs >= lhs+1, this allows
         // us to  fold the constant into the cmp instruction.
-        RHS = DAG.getConstant(C->getSExtValue() + 1, DL, VT);
+        RHS = DAG.getSignedConstant(C->getSExtValue() + 1, DL, VT);
         CC = ISD::SETGE;
         break;
       }
@@ -713,7 +712,10 @@ SDValue AVRTargetLowering::getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
     // Turn lhs < rhs with lhs constant into rhs >= lhs+1, this allows us to
     // fold the constant into the cmp instruction.
     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) {
-      RHS = DAG.getConstant(C->getSExtValue() + 1, DL, VT);
+      // Doing a "icmp ugt i16 65535, %0" comparison should have been converted
+      // already to something else. Assert to make sure this assumption holds.
+      assert((!C->isAllOnes()) && "integer overflow in comparison transform");
+      RHS = DAG.getConstant(C->getZExtValue() + 1, DL, VT);
       CC = ISD::SETUGE;
       break;
     }
diff --git a/llvm/lib/Target/AVR/AVRISelLowering.h b/llvm/lib/Target/AVR/AVRISelLowering.h
index 2ae22b2d8af6c..301ce9c34a783 100644
--- a/llvm/lib/Target/AVR/AVRISelLowering.h
+++ b/llvm/lib/Target/AVR/AVRISelLowering.h
@@ -94,6 +94,8 @@ class AVRTargetLowering : public TargetLowering {
     return ShiftLegalizationStrategy::LowerToLibcall;
   }
 
+  bool softPromoteHalfType() const override { return true; }
+
 private:
   SDValue getAVRCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue &AVRcc,
                     SelectionDAG &DAG, SDLoc dl) const;
diff --git a/llvm/lib/Target/AVR/AVRInstrFormats.td b/llvm/lib/Target/AVR/AVRInstrFormats.td
index 407e14a079f3e..e1e65b56370cc 100644
--- a/llvm/lib/Target/AVR/AVRInstrFormats.td
+++ b/llvm/lib/Target/AVR/AVRInstrFormats.td
@@ -19,8 +19,6 @@ class AVRInst<dag outs, dag ins, string asmstr, list<dag> pattern>
   dag InOperandList = ins;
   let AsmString = asmstr;
   let Pattern = pattern;
-
-  field bits<32> SoftFail = 0;
 }
 
 /// A 16-bit AVR instruction.
diff --git a/llvm/lib/Target/AVR/AVRTargetMachine.cpp b/llvm/lib/Target/AVR/AVRTargetMachine.cpp
index b75417a0896a5..fbd148478c894 100644
--- a/llvm/lib/Target/AVR/AVRTargetMachine.cpp
+++ b/llvm/lib/Target/AVR/AVRTargetMachine.cpp
@@ -20,6 +20,7 @@
 #include "AVR.h"
 #include "AVRMachineFunctionInfo.h"
 #include "AVRTargetObjectFile.h"
+#include "AVRTargetTransformInfo.h"
 #include "MCTargetDesc/AVRMCTargetDesc.h"
 #include "TargetInfo/AVRTargetInfo.h"
 
@@ -28,7 +29,7 @@
 namespace llvm {
 
 static const char *AVRDataLayout =
-    "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8-a:8";
+    "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8:16-a:8";
 
 /// Processes a CPU name.
 static StringRef getCPU(StringRef CPU) {
@@ -62,7 +63,9 @@ namespace {
 class AVRPassConfig : public TargetPassConfig {
 public:
   AVRPassConfig(AVRTargetMachine &TM, PassManagerBase &PM)
-      : TargetPassConfig(TM, PM) {}
+      : TargetPassConfig(TM, PM) {
+    EnableLoopTermFold = true;
+  }
 
   AVRTargetMachine &getAVRTargetMachine() const {
     return getTM<AVRTargetMachine>();
@@ -107,6 +110,11 @@ const AVRSubtarget *AVRTargetMachine::getSubtargetImpl(const Function &) const {
   return &SubTarget;
 }
 
+TargetTransformInfo
+AVRTargetMachine::getTargetTransformInfo(const Function &F) const {
+  return TargetTransformInfo(std::make_unique<AVRTTIImpl>(this, F));
+}
+
 MachineFunctionInfo *AVRTargetMachine::createMachineFunctionInfo(
     BumpPtrAllocator &Allocator, const Function &F,
     const TargetSubtargetInfo *STI) const {
diff --git a/llvm/lib/Target/AVR/AVRTargetMachine.h b/llvm/lib/Target/AVR/AVRTargetMachine.h
index 167d0076e9581..9452b3d8cd8a5 100644
--- a/llvm/lib/Target/AVR/AVRTargetMachine.h
+++ b/llvm/lib/Target/AVR/AVRTargetMachine.h
@@ -48,6 +48,8 @@ class AVRTargetMachine : public CodeGenTargetMachineImpl {
   createMachineFunctionInfo(BumpPtrAllocator &Allocator, const Function &F,
                             const TargetSubtargetInfo *STI) const override;
 
+  TargetTransformInfo getTargetTransformInfo(const Function &F) const override;
+
   bool isNoopAddrSpaceCast(unsigned SrcAs, unsigned DestAs) const override {
     // While AVR has different address spaces, they are all represented by
     // 16-bit pointers that can be freely casted between (of course, a pointer
diff --git a/llvm/lib/Target/AVR/AVRTargetTransformInfo.cpp b/llvm/lib/Target/AVR/AVRTargetTransformInfo.cpp
new file mode 100644
index 0000000000000..b1ef38047c07b
--- /dev/null
+++ b/llvm/lib/Target/AVR/AVRTargetTransformInfo.cpp
@@ -0,0 +1,24 @@
+//===-- AVRTargetTransformInfo.cpp - AVR specific TTI ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AVRTargetTransformInfo.h"
+
+using namespace llvm;
+
+bool AVRTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
+                               const TargetTransformInfo::LSRCost &C2) const {
+  // AVR specific here are "instruction number 1st priority".
+  // If we need to emit adds inside the loop to add up base registers, then
+  // we need at least one extra temporary register.
+  unsigned C1NumRegs = C1.NumRegs + (C1.NumBaseAdds != 0);
+  unsigned C2NumRegs = C2.NumRegs + (C2.NumBaseAdds != 0);
+  return std::tie(C1.Insns, C1NumRegs, C1.AddRecCost, C1.NumIVMuls,
+                  C1.NumBaseAdds, C1.ScaleCost, C1.ImmCost, C1.SetupCost) <
+         std::tie(C2.Insns, C2NumRegs, C2.AddRecCost, C2.NumIVMuls,
+                  C2.NumBaseAdds, C2.ScaleCost, C2.ImmCost, C2.SetupCost);
+}
diff --git a/llvm/lib/Target/AVR/AVRTargetTransformInfo.h b/llvm/lib/Target/AVR/AVRTargetTransformInfo.h
new file mode 100644
index 0000000000000..0daeeb8f11cfe
--- /dev/null
+++ b/llvm/lib/Target/AVR/AVRTargetTransformInfo.h
@@ -0,0 +1,51 @@
+//===- AVRTargetTransformInfo.h - AVR specific TTI --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file defines a TargetTransformInfoImplBase conforming object specific
+/// to the AVR target machine. It uses the target's detailed information to
+/// provide more precise answers to certain TTI queries, while letting the
+/// target independent and default TTI implementations handle the rest.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AVR_AVRTARGETTRANSFORMINFO_H
+#define LLVM_LIB_TARGET_AVR_AVRTARGETTRANSFORMINFO_H
+
+#include "AVRSubtarget.h"
+#include "AVRTargetMachine.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/BasicTTIImpl.h"
+#include "llvm/IR/Function.h"
+#include <optional>
+
+namespace llvm {
+
+class AVRTTIImpl final : public BasicTTIImplBase<AVRTTIImpl> {
+  using BaseT = BasicTTIImplBase<AVRTTIImpl>;
+  using TTI = TargetTransformInfo;
+
+  friend BaseT;
+
+  const AVRSubtarget *ST;
+  const AVRTargetLowering *TLI;
+
+  const AVRSubtarget *getST() const { return ST; }
+  const AVRTargetLowering *getTLI() const { return TLI; }
+
+public:
+  explicit AVRTTIImpl(const AVRTargetMachine *TM, const Function &F)
+      : BaseT(TM, F.getDataLayout()), ST(TM->getSubtargetImpl(F)),
+        TLI(ST->getTargetLowering()) {}
+
+  bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
+                     const TargetTransformInfo::LSRCost &C2) const override;
+};
+
+} // end namespace llvm
+
+#endif // LLVM_LIB_TARGET_AVR_AVRTARGETTRANSFORMINFO_H
diff --git a/llvm/lib/Target/AVR/CMakeLists.txt b/llvm/lib/Target/AVR/CMakeLists.txt
index 781dac02c7083..a31c545f48ba3 100644
--- a/llvm/lib/Target/AVR/CMakeLists.txt
+++ b/llvm/lib/Target/AVR/CMakeLists.txt
@@ -29,11 +29,13 @@ add_llvm_target(AVRCodeGen
   AVRSubtarget.cpp
   AVRTargetMachine.cpp
   AVRTargetObjectFile.cpp
+  AVRTargetTransformInfo.cpp
 
   DEPENDS
   intrinsics_gen
 
   LINK_COMPONENTS
+  Analysis
   AVRDesc
   AVRInfo
   AsmPrinter
@@ -44,6 +46,8 @@ add_llvm_target(AVRCodeGen
   SelectionDAG
   Support
   Target
+  TargetParser
+  TransformUtils
 
   ADD_TO_COMPONENT
   AVR
diff --git a/llvm/lib/Target/AVR/Disassembler/AVRDisassembler.cpp b/llvm/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
index c7a584868f4e6..948588cb9a754 100644
--- a/llvm/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
+++ b/llvm/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
@@ -18,6 +18,7 @@
 
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -26,6 +27,7 @@
 #include "llvm/Support/Compiler.h"
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "avr-disassembler"
 
diff --git a/llvm/lib/Target/BPF/BPFInstrFormats.td b/llvm/lib/Target/BPF/BPFInstrFormats.td
index 50cca5148de7c..ccabc5a6ae77b 100644
--- a/llvm/lib/Target/BPF/BPFInstrFormats.td
+++ b/llvm/lib/Target/BPF/BPFInstrFormats.td
@@ -111,7 +111,6 @@ def BPF_FETCH : BPFAtomicFlag<0x1>;
 class InstBPF<dag outs, dag ins, string asmstr, list<dag> pattern>
   : Instruction {
   field bits<64> Inst;
-  field bits<64> SoftFail = 0;
   let Size = 8;
 
   let Namespace = "BPF";
diff --git a/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp b/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp
index 4dfae81e90191..b5bb1c08c5644 100644
--- a/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp
+++ b/llvm/lib/Target/BPF/Disassembler/BPFDisassembler.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
diff --git a/llvm/lib/Target/CMakeLists.txt b/llvm/lib/Target/CMakeLists.txt
index e26e00ea3fdfd..bcc13f942bf96 100644
--- a/llvm/lib/Target/CMakeLists.txt
+++ b/llvm/lib/Target/CMakeLists.txt
@@ -24,7 +24,7 @@ add_llvm_component_library(LLVMTarget
 # that are used across shared objects which we can't hide.
 if (NOT BUILD_SHARED_LIBS AND NOT APPLE AND
     (NOT (WIN32 OR CYGWIN) OR ((MINGW OR CYGWIN) AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) AND
-    NOT (${CMAKE_SYSTEM_NAME} MATCHES "AIX") AND
+    NOT ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX") AND
     NOT DEFINED CMAKE_CXX_VISIBILITY_PRESET)
   # Set default visibility to hidden, so we don't export all the Target classes
   # in libLLVM.so.
diff --git a/llvm/lib/Target/CSKY/CSKYConstantIslandPass.cpp b/llvm/lib/Target/CSKY/CSKYConstantIslandPass.cpp
index e55d9b227d1cd..7885d93cbad98 100644
--- a/llvm/lib/Target/CSKY/CSKYConstantIslandPass.cpp
+++ b/llvm/lib/Target/CSKY/CSKYConstantIslandPass.cpp
@@ -116,7 +116,7 @@ class CSKYConstantIslands : public MachineFunctionPass {
 
   /// NewWaterList - The subset of WaterList that was created since the
   /// previous iteration by inserting unconditional branches.
-  SmallSet<MachineBasicBlock *, 4> NewWaterList;
+  SmallPtrSet<MachineBasicBlock *, 4> NewWaterList;
 
   using water_iterator = std::vector<MachineBasicBlock *>::iterator;
 
diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
index 707017106e61d..e5b4f6eeb7b73 100644
--- a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
@@ -1329,10 +1329,7 @@ SDValue CSKYTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
 
   // Prepare argument list to generate call.
   ArgListTy Args;
-  ArgListEntry Entry;
-  Entry.Node = Load;
-  Entry.Ty = CallTy;
-  Args.push_back(Entry);
+  Args.emplace_back(Load, CallTy);
 
   // Setup call to __tls_get_addr.
   TargetLowering::CallLoweringInfo CLI(DAG);
diff --git a/llvm/lib/Target/CSKY/CSKYInstrFormats.td b/llvm/lib/Target/CSKY/CSKYInstrFormats.td
index 8144a501b3d27..5296d282c689b 100644
--- a/llvm/lib/Target/CSKY/CSKYInstrFormats.td
+++ b/llvm/lib/Target/CSKY/CSKYInstrFormats.td
@@ -24,7 +24,6 @@ class CSKYInst<AddrMode am, int sz, dag outs, dag ins, string asmstr,
   let Namespace = "CSKY";
   int Size = sz;
   AddrMode AM = am;
-  field bits<32> SoftFail = 0;
   let OutOperandList = outs;
   let InOperandList = ins;
   let AsmString = asmstr;
diff --git a/llvm/lib/Target/CSKY/Disassembler/CSKYDisassembler.cpp b/llvm/lib/Target/CSKY/Disassembler/CSKYDisassembler.cpp
index d78d9acc2aa23..749127f4ddc8a 100644
--- a/llvm/lib/Target/CSKY/Disassembler/CSKYDisassembler.cpp
+++ b/llvm/lib/Target/CSKY/Disassembler/CSKYDisassembler.cpp
@@ -15,6 +15,7 @@
 #include "TargetInfo/CSKYTargetInfo.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
diff --git a/llvm/lib/Target/DirectX/CMakeLists.txt b/llvm/lib/Target/DirectX/CMakeLists.txt
index c7c09caf43667..8100f941c8d94 100644
--- a/llvm/lib/Target/DirectX/CMakeLists.txt
+++ b/llvm/lib/Target/DirectX/CMakeLists.txt
@@ -49,6 +49,7 @@ add_llvm_target(DirectXCodeGen
   DirectXInfo
   DirectXPointerTypeAnalysis
   FrontendHLSL
+  IPO
   MC
   ScalarOpts
   SelectionDAG
diff --git a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp
index 26a113d2d5260..a1ef2578f00aa 100644
--- a/llvm/lib/Target/DirectX/DXContainerGlobals.cpp
+++ b/llvm/lib/Target/DirectX/DXContainerGlobals.cpp
@@ -162,8 +162,7 @@ void DXContainerGlobals::addRootSignature(Module &M,
 
   auto &RSA = getAnalysis<RootSignatureAnalysisWrapper>().getRSInfo();
   const Function *EntryFunction = MMI.EntryPropertyVec[0].Entry;
-  const std::optional<mcdxbc::RootSignatureDesc> &RS =
-      RSA.getDescForFunction(EntryFunction);
+  const mcdxbc::RootSignatureDesc *RS = RSA.getDescForFunction(EntryFunction);
 
   if (!RS)
     return;
diff --git a/llvm/lib/Target/DirectX/DXIL.td b/llvm/lib/Target/DirectX/DXIL.td
index 492e0784cedd0..c65ead45e2c7e 100644
--- a/llvm/lib/Target/DirectX/DXIL.td
+++ b/llvm/lib/Target/DirectX/DXIL.td
@@ -1108,11 +1108,11 @@ def RawBufferStore : DXILOp<140, rawBufferStore> {
 def Dot2AddHalf : DXILOp<162, dot2AddHalf> {
   let Doc = "2D half dot product with accumulate to float";
   let intrinsics = [IntrinSelect<int_dx_dot2add>];
-  let arguments = [FloatTy, HalfTy, HalfTy, HalfTy, HalfTy];
-  let result = FloatTy;
-  let overloads = [Overloads<DXIL1_0, []>];
-  let stages = [Stages<DXIL1_0, [all_stages]>];
-  let attributes = [Attributes<DXIL1_0, [ReadNone]>];
+  let arguments = [OverloadTy, HalfTy, HalfTy, HalfTy, HalfTy];
+  let result = OverloadTy;
+  let overloads = [Overloads<DXIL1_4, [FloatTy]>];
+  let stages = [Stages<DXIL1_4, [all_stages]>];
+  let attributes = [Attributes<DXIL1_4, [ReadNone]>];
 }
 
 def Dot4AddI8Packed : DXILOp<163, dot4AddPacked> {
diff --git a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
index 5f331dbd2d826..13e3408815bba 100644
--- a/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
+++ b/llvm/lib/Target/DirectX/DXILFinalizeLinkage.cpp
@@ -20,13 +20,13 @@ using namespace llvm;
 static bool finalizeLinkage(Module &M) {
   bool MadeChange = false;
 
-  // Convert private global variables to internal linkage.
-  for (GlobalVariable &GV : M.globals()) {
-    if (GV.hasPrivateLinkage()) {
+  // Convert private globals and external globals with no usage to internal
+  // linkage.
+  for (GlobalVariable &GV : M.globals())
+    if (GV.hasPrivateLinkage() || (GV.hasExternalLinkage() && GV.use_empty())) {
       GV.setLinkage(GlobalValue::InternalLinkage);
       MadeChange = true;
     }
-  }
 
   SmallVector<Function *> Funcs;
 
diff --git a/llvm/lib/Target/DirectX/DXILForwardHandleAccesses.cpp b/llvm/lib/Target/DirectX/DXILForwardHandleAccesses.cpp
index 73abfe7c48584..695eacbb2fadc 100644
--- a/llvm/lib/Target/DirectX/DXILForwardHandleAccesses.cpp
+++ b/llvm/lib/Target/DirectX/DXILForwardHandleAccesses.cpp
@@ -9,10 +9,13 @@
 #include "DXILForwardHandleAccesses.h"
 #include "DXILShaderFlags.h"
 #include "DirectX.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Analysis/DXILResource.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Dominators.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsDirectX.h"
@@ -70,6 +73,7 @@ static bool forwardHandleAccesses(Function &F, DominatorTree &DT) {
 
   DenseMap<GlobalVariable *, IntrinsicInst *> HandleMap;
   SmallVector<LoadInst *> LoadsToProcess;
+  DenseMap<AllocaInst *, SmallVector<IntrinsicInst *>> LifeTimeIntrinsicMap;
   for (BasicBlock &BB : F)
     for (Instruction &Inst : BB)
       if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
@@ -78,6 +82,14 @@ static bool forwardHandleAccesses(Function &F, DominatorTree &DT) {
         case Intrinsic::dx_resource_handlefromimplicitbinding:
           processHandle(II, HandleMap);
           break;
+        case Intrinsic::lifetime_start:
+        case Intrinsic::lifetime_end:
+          if (II->arg_size() >= 1) {
+            Value *Ptr = II->getArgOperand(0);
+            if (auto *Alloca = dyn_cast<AllocaInst>(Ptr))
+              LifeTimeIntrinsicMap[Alloca].push_back(II);
+          }
+          break;
         default:
           continue;
         }
@@ -87,17 +99,58 @@ static bool forwardHandleAccesses(Function &F, DominatorTree &DT) {
 
   for (LoadInst *LI : LoadsToProcess) {
     Value *V = LI->getPointerOperand();
-    auto *GV = dyn_cast<GlobalVariable>(LI->getPointerOperand());
+    auto *GV = dyn_cast<GlobalVariable>(V);
 
     // If we didn't find the global, we may need to walk through a level of
     // indirection. This generally happens at -O0.
-    if (!GV)
+    if (!GV) {
       if (auto *NestedLI = dyn_cast<LoadInst>(V)) {
         BasicBlock::iterator BBI(NestedLI);
         Value *Loaded = FindAvailableLoadedValue(
             NestedLI, NestedLI->getParent(), BBI, 0, nullptr, nullptr);
         GV = dyn_cast_or_null<GlobalVariable>(Loaded);
+      } else if (auto *NestedAlloca = dyn_cast<AllocaInst>(V)) {
+
+        if (auto It = LifeTimeIntrinsicMap.find(NestedAlloca);
+            It != LifeTimeIntrinsicMap.end()) {
+          llvm::for_each(It->second,
+                         [](IntrinsicInst *II) { II->eraseFromParent(); });
+          LifeTimeIntrinsicMap.erase(It);
+        }
+
+        for (auto *User : NestedAlloca->users()) {
+          auto *Store = dyn_cast<StoreInst>(User);
+          if (!Store)
+            continue;
+
+          Value *StoredVal = Store->getValueOperand();
+          if (!StoredVal)
+            continue;
+
+          // Try direct global match
+          GV = dyn_cast<GlobalVariable>(StoredVal);
+          if (GV)
+            break;
+
+          // If it's a load, check its source
+          if (auto *Load = dyn_cast<LoadInst>(StoredVal)) {
+            GV = dyn_cast<GlobalVariable>(Load->getPointerOperand());
+            if (GV)
+              break;
+
+            // If loading from an unmodified stack copy of the global, reuse the
+            // global's value. Note: we are just repeating what we are doing for
+            // the load case for the alloca store pattern.
+            BasicBlock::iterator BBI(Load);
+            Value *Loaded = FindAvailableLoadedValue(Load, Load->getParent(),
+                                                     BBI, 0, nullptr, nullptr);
+            GV = dyn_cast<GlobalVariable>(Loaded);
+            if (GV)
+              break;
+          }
+        }
       }
+    }
 
     auto It = HandleMap.find(GV);
     if (It == HandleMap.end()) {
diff --git a/llvm/lib/Target/DirectX/DXILOpLowering.cpp b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
index 0ec15a629d0a2..bd421771e8edb 100644
--- a/llvm/lib/Target/DirectX/DXILOpLowering.cpp
+++ b/llvm/lib/Target/DirectX/DXILOpLowering.cpp
@@ -9,6 +9,7 @@
 #include "DXILOpLowering.h"
 #include "DXILConstants.h"
 #include "DXILOpBuilder.h"
+#include "DXILRootSignature.h"
 #include "DXILShaderFlags.h"
 #include "DirectX.h"
 #include "llvm/ADT/SmallVector.h"
@@ -746,7 +747,7 @@ class OpLowerer {
     IRBuilder<> &IRB = OpBuilder.getIRB();
     return replaceFunction(F, [&](CallInst *CI) -> Error {
       IRB.SetInsertPoint(CI);
-      Value *Ptr = CI->getArgOperand(1);
+      Value *Ptr = CI->getArgOperand(0);
       assert(Ptr->getType()->isPointerTy() &&
              "Expected operand of lifetime intrinsic to be a pointer");
 
@@ -918,6 +919,7 @@ PreservedAnalyses DXILOpLowering::run(Module &M, ModuleAnalysisManager &MAM) {
   PA.preserve<DXILResourceAnalysis>();
   PA.preserve<DXILMetadataAnalysis>();
   PA.preserve<ShaderFlagsAnalysis>();
+  PA.preserve<RootSignatureAnalysis>();
   return PA;
 }
 
@@ -945,6 +947,7 @@ class DXILOpLoweringLegacy : public ModulePass {
     AU.addPreserved<DXILResourceWrapperPass>();
     AU.addPreserved<DXILMetadataAnalysisWrapperPass>();
     AU.addPreserved<ShaderFlagsAnalysisWrapper>();
+    AU.addPreserved<RootSignatureAnalysisWrapper>();
   }
 };
 char DXILOpLoweringLegacy::ID = 0;
diff --git a/llvm/lib/Target/DirectX/DXILPostOptimizationValidation.cpp b/llvm/lib/Target/DirectX/DXILPostOptimizationValidation.cpp
index 398dcbb8d1737..be2c7d1ddff3f 100644
--- a/llvm/lib/Target/DirectX/DXILPostOptimizationValidation.cpp
+++ b/llvm/lib/Target/DirectX/DXILPostOptimizationValidation.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "DXILPostOptimizationValidation.h"
+#include "DXILRootSignature.h"
 #include "DXILShaderFlags.h"
 #include "DirectX.h"
 #include "llvm/ADT/SmallString.h"
@@ -17,13 +18,44 @@
 #include "llvm/IR/IntrinsicsDirectX.h"
 #include "llvm/IR/Module.h"
 #include "llvm/InitializePasses.h"
+#include "llvm/Support/DXILABI.h"
 
 #define DEBUG_TYPE "dxil-post-optimization-validation"
 
 using namespace llvm;
 using namespace llvm::dxil;
 
-namespace {
+static ResourceClass toResourceClass(dxbc::DescriptorRangeType RangeType) {
+  using namespace dxbc;
+  switch (RangeType) {
+  case DescriptorRangeType::SRV:
+    return ResourceClass::SRV;
+  case DescriptorRangeType::UAV:
+    return ResourceClass::UAV;
+  case DescriptorRangeType::CBV:
+    return ResourceClass::CBuffer;
+  case DescriptorRangeType::Sampler:
+    return ResourceClass::Sampler;
+  }
+  llvm_unreachable("Unknown DescriptorRangeType");
+}
+
+static ResourceClass toResourceClass(dxbc::RootParameterType Type) {
+  using namespace dxbc;
+  switch (Type) {
+  case RootParameterType::Constants32Bit:
+    return ResourceClass::CBuffer;
+  case RootParameterType::SRV:
+    return ResourceClass::SRV;
+  case RootParameterType::UAV:
+    return ResourceClass::UAV;
+  case RootParameterType::CBV:
+    return ResourceClass::CBuffer;
+  case dxbc::RootParameterType::DescriptorTable:
+    llvm_unreachable("DescriptorTable is not convertible to ResourceClass");
+  }
+  llvm_unreachable("Unknown RootParameterType");
+}
 
 static void reportInvalidDirection(Module &M, DXILResourceMap &DRM) {
   for (const auto &UAV : DRM.uavs()) {
@@ -63,9 +95,7 @@ static void reportOverlappingError(Module &M, ResourceInfo R1,
 }
 
 static void reportOverlappingBinding(Module &M, DXILResourceMap &DRM) {
-  if (DRM.empty())
-    return;
-
+  bool ErrorFound = false;
   for (const auto &ResList :
        {DRM.srvs(), DRM.uavs(), DRM.cbuffers(), DRM.samplers()}) {
     if (ResList.empty())
@@ -77,15 +107,136 @@ static void reportOverlappingBinding(Module &M, DXILResourceMap &DRM) {
       while (RI != ResList.end() &&
              PrevRI->getBinding().overlapsWith(RI->getBinding())) {
         reportOverlappingError(M, *PrevRI, *RI);
+        ErrorFound = true;
         RI++;
       }
       PrevRI = CurrentRI;
     }
   }
+  assert(ErrorFound && "this function should be called only when if "
+                       "DXILResourceBindingInfo::hasOverlapingBinding() is "
+                       "true, yet no overlapping binding was found");
+}
+
+static void
+reportOverlappingRegisters(Module &M,
+                           const llvm::hlsl::BindingInfoBuilder::Binding &R1,
+                           const llvm::hlsl::BindingInfoBuilder::Binding &R2) {
+  SmallString<128> Message;
+
+  raw_svector_ostream OS(Message);
+  OS << "resource " << getResourceClassName(R1.RC) << " (space=" << R1.Space
+     << ", registers=[" << R1.LowerBound << ", " << R1.UpperBound
+     << "]) overlaps with resource " << getResourceClassName(R2.RC)
+     << " (space=" << R2.Space << ", registers=[" << R2.LowerBound << ", "
+     << R2.UpperBound << "])";
+  M.getContext().diagnose(DiagnosticInfoGeneric(Message));
+}
+
+static dxbc::ShaderVisibility
+tripleToVisibility(llvm::Triple::EnvironmentType ET) {
+  switch (ET) {
+  case Triple::Pixel:
+    return dxbc::ShaderVisibility::Pixel;
+  case Triple::Vertex:
+    return dxbc::ShaderVisibility::Vertex;
+  case Triple::Geometry:
+    return dxbc::ShaderVisibility::Geometry;
+  case Triple::Hull:
+    return dxbc::ShaderVisibility::Hull;
+  case Triple::Domain:
+    return dxbc::ShaderVisibility::Domain;
+  case Triple::Mesh:
+    return dxbc::ShaderVisibility::Mesh;
+  case Triple::Compute:
+    return dxbc::ShaderVisibility::All;
+  default:
+    llvm_unreachable("Invalid triple to shader stage conversion");
+  }
+}
+
+static void validateRootSignature(Module &M,
+                                  const mcdxbc::RootSignatureDesc &RSD,
+                                  dxil::ModuleMetadataInfo &MMI) {
+
+  hlsl::BindingInfoBuilder Builder;
+  dxbc::ShaderVisibility Visibility = tripleToVisibility(MMI.ShaderProfile);
+
+  for (const mcdxbc::RootParameterInfo &ParamInfo : RSD.ParametersContainer) {
+    dxbc::ShaderVisibility ParamVisibility =
+        static_cast<dxbc::ShaderVisibility>(ParamInfo.Header.ShaderVisibility);
+    if (ParamVisibility != dxbc::ShaderVisibility::All &&
+        ParamVisibility != Visibility)
+      continue;
+    dxbc::RootParameterType ParamType =
+        static_cast<dxbc::RootParameterType>(ParamInfo.Header.ParameterType);
+    switch (ParamType) {
+    case dxbc::RootParameterType::Constants32Bit: {
+      dxbc::RTS0::v1::RootConstants Const =
+          RSD.ParametersContainer.getConstant(ParamInfo.Location);
+      Builder.trackBinding(dxil::ResourceClass::CBuffer, Const.RegisterSpace,
+                           Const.ShaderRegister, Const.ShaderRegister,
+                           &ParamInfo);
+      break;
+    }
+
+    case dxbc::RootParameterType::SRV:
+    case dxbc::RootParameterType::UAV:
+    case dxbc::RootParameterType::CBV: {
+      dxbc::RTS0::v2::RootDescriptor Desc =
+          RSD.ParametersContainer.getRootDescriptor(ParamInfo.Location);
+      Builder.trackBinding(toResourceClass(static_cast<dxbc::RootParameterType>(
+                               ParamInfo.Header.ParameterType)),
+                           Desc.RegisterSpace, Desc.ShaderRegister,
+                           Desc.ShaderRegister, &ParamInfo);
+
+      break;
+    }
+    case dxbc::RootParameterType::DescriptorTable: {
+      const mcdxbc::DescriptorTable &Table =
+          RSD.ParametersContainer.getDescriptorTable(ParamInfo.Location);
+
+      for (const dxbc::RTS0::v2::DescriptorRange &Range : Table.Ranges) {
+        uint32_t UpperBound =
+            Range.NumDescriptors == ~0U
+                ? Range.BaseShaderRegister
+                : Range.BaseShaderRegister + Range.NumDescriptors - 1;
+        Builder.trackBinding(
+            toResourceClass(
+                static_cast<dxbc::DescriptorRangeType>(Range.RangeType)),
+            Range.RegisterSpace, Range.BaseShaderRegister, UpperBound,
+            &ParamInfo);
+      }
+      break;
+    }
+    }
+  }
+
+  for (const dxbc::RTS0::v1::StaticSampler &S : RSD.StaticSamplers)
+    Builder.trackBinding(dxil::ResourceClass::Sampler, S.RegisterSpace,
+                         S.ShaderRegister, S.ShaderRegister, &S);
+
+  Builder.calculateBindingInfo(
+      [&M](const llvm::hlsl::BindingInfoBuilder &Builder,
+           const llvm::hlsl::BindingInfoBuilder::Binding &ReportedBinding) {
+        const llvm::hlsl::BindingInfoBuilder::Binding &Overlaping =
+            Builder.findOverlapping(ReportedBinding);
+        reportOverlappingRegisters(M, ReportedBinding, Overlaping);
+      });
+}
+
+static mcdxbc::RootSignatureDesc *
+getRootSignature(RootSignatureBindingInfo &RSBI,
+                 dxil::ModuleMetadataInfo &MMI) {
+  if (MMI.EntryPropertyVec.size() == 0)
+    return nullptr;
+  return RSBI.getDescForFunction(MMI.EntryPropertyVec[0].Entry);
 }
 
 static void reportErrors(Module &M, DXILResourceMap &DRM,
-                         DXILResourceBindingInfo &DRBI) {
+                         DXILResourceBindingInfo &DRBI,
+                         RootSignatureBindingInfo &RSBI,
+                         dxil::ModuleMetadataInfo &MMI) {
   if (DRM.hasInvalidCounterDirection())
     reportInvalidDirection(M, DRM);
 
@@ -94,14 +245,19 @@ static void reportErrors(Module &M, DXILResourceMap &DRM,
 
   assert(!DRBI.hasImplicitBinding() && "implicit bindings should be handled in "
                                        "DXILResourceImplicitBinding pass");
+
+  if (mcdxbc::RootSignatureDesc *RSD = getRootSignature(RSBI, MMI))
+    validateRootSignature(M, *RSD, MMI);
 }
-} // namespace
 
 PreservedAnalyses
 DXILPostOptimizationValidation::run(Module &M, ModuleAnalysisManager &MAM) {
   DXILResourceMap &DRM = MAM.getResult<DXILResourceAnalysis>(M);
   DXILResourceBindingInfo &DRBI = MAM.getResult<DXILResourceBindingAnalysis>(M);
-  reportErrors(M, DRM, DRBI);
+  RootSignatureBindingInfo &RSBI = MAM.getResult<RootSignatureAnalysis>(M);
+  ModuleMetadataInfo &MMI = MAM.getResult<DXILMetadataAnalysis>(M);
+
+  reportErrors(M, DRM, DRBI, RSBI, MMI);
   return PreservedAnalyses::all();
 }
 
@@ -113,7 +269,12 @@ class DXILPostOptimizationValidationLegacy : public ModulePass {
         getAnalysis<DXILResourceWrapperPass>().getResourceMap();
     DXILResourceBindingInfo &DRBI =
         getAnalysis<DXILResourceBindingWrapperPass>().getBindingInfo();
-    reportErrors(M, DRM, DRBI);
+    RootSignatureBindingInfo &RSBI =
+        getAnalysis<RootSignatureAnalysisWrapper>().getRSInfo();
+    dxil::ModuleMetadataInfo &MMI =
+        getAnalysis<DXILMetadataAnalysisWrapperPass>().getModuleMetadata();
+
+    reportErrors(M, DRM, DRBI, RSBI, MMI);
     return false;
   }
   StringRef getPassName() const override {
@@ -125,10 +286,13 @@ class DXILPostOptimizationValidationLegacy : public ModulePass {
   void getAnalysisUsage(llvm::AnalysisUsage &AU) const override {
     AU.addRequired<DXILResourceWrapperPass>();
     AU.addRequired<DXILResourceBindingWrapperPass>();
+    AU.addRequired<DXILMetadataAnalysisWrapperPass>();
+    AU.addRequired<RootSignatureAnalysisWrapper>();
     AU.addPreserved<DXILResourceWrapperPass>();
     AU.addPreserved<DXILResourceBindingWrapperPass>();
     AU.addPreserved<DXILMetadataAnalysisWrapperPass>();
     AU.addPreserved<ShaderFlagsAnalysisWrapper>();
+    AU.addPreserved<RootSignatureAnalysisWrapper>();
   }
 };
 char DXILPostOptimizationValidationLegacy::ID = 0;
@@ -139,6 +303,8 @@ INITIALIZE_PASS_BEGIN(DXILPostOptimizationValidationLegacy, DEBUG_TYPE,
 INITIALIZE_PASS_DEPENDENCY(DXILResourceBindingWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DXILResourceTypeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DXILResourceWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DXILMetadataAnalysisWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(RootSignatureAnalysisWrapper)
 INITIALIZE_PASS_END(DXILPostOptimizationValidationLegacy, DEBUG_TYPE,
                     "DXIL Post Optimization Validation", false, false)
 
diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.h b/llvm/lib/Target/DirectX/DXILRootSignature.h
index 254b7ff504633..b990b6c7410ac 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.h
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.h
@@ -43,13 +43,11 @@ class RootSignatureBindingInfo {
 
   iterator end() { return FuncToRsMap.end(); }
 
-  std::optional<mcdxbc::RootSignatureDesc>
-  getDescForFunction(const Function *F) {
+  mcdxbc::RootSignatureDesc *getDescForFunction(const Function *F) {
     const auto FuncRs = find(F);
     if (FuncRs == end())
-      return std::nullopt;
-
-    return FuncRs->second;
+      return nullptr;
+    return &FuncRs->second;
   }
 };
 
diff --git a/llvm/lib/Target/DirectX/DXILWriter/DXILWriterPass.cpp b/llvm/lib/Target/DirectX/DXILWriter/DXILWriterPass.cpp
index 1bd5dd78fedd1..1eb03bfc087e3 100644
--- a/llvm/lib/Target/DirectX/DXILWriter/DXILWriterPass.cpp
+++ b/llvm/lib/Target/DirectX/DXILWriter/DXILWriterPass.cpp
@@ -13,11 +13,15 @@
 #include "DXILWriterPass.h"
 #include "DXILBitcodeWriter.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/InitializePasses.h"
@@ -54,49 +58,81 @@ class WriteDXILPass : public llvm::ModulePass {
 };
 
 static void legalizeLifetimeIntrinsics(Module &M) {
-  for (Function &F : M) {
-    Intrinsic::ID IID = F.getIntrinsicID();
-    if (IID != Intrinsic::lifetime_start && IID != Intrinsic::lifetime_end)
+  LLVMContext &Ctx = M.getContext();
+  Type *I64Ty = IntegerType::get(Ctx, 64);
+  Type *PtrTy = PointerType::get(Ctx, 0);
+  Intrinsic::ID LifetimeIIDs[2] = {Intrinsic::lifetime_start,
+                                   Intrinsic::lifetime_end};
+  for (Intrinsic::ID &IID : LifetimeIIDs) {
+    Function *F = M.getFunction(Intrinsic::getName(IID, {PtrTy}, &M));
+    if (!F)
       continue;
 
-    // Lifetime intrinsics in LLVM 3.7 do not have the memory FnAttr
-    F.removeFnAttr(Attribute::Memory);
-
-    // Lifetime intrinsics in LLVM 3.7 do not have mangled names
-    F.setName(Intrinsic::getBaseName(IID));
-
-    // LLVM 3.7 Lifetime intrinics require an i8* operand, so we insert bitcasts
-    // to ensure that is the case
-    for (auto *User : make_early_inc_range(F.users())) {
-      CallInst *CI = dyn_cast<CallInst>(User);
-      assert(CI && "Expected user of a lifetime intrinsic function to be a "
-                   "lifetime intrinsic call");
-      Value *PtrOperand = CI->getArgOperand(1);
-      PointerType *PtrTy = cast<PointerType>(PtrOperand->getType());
+    // Get or insert an LLVM 3.7-compliant lifetime intrinsic function of the
+    // form `void @llvm.lifetime.[start/end](i64, ptr)` with the NoUnwind
+    // attribute
+    AttributeList Attr;
+    Attr = Attr.addFnAttribute(Ctx, Attribute::NoUnwind);
+    FunctionCallee LifetimeCallee = M.getOrInsertFunction(
+        Intrinsic::getBaseName(IID), Attr, Type::getVoidTy(Ctx), I64Ty, PtrTy);
+
+    // Replace all calls to lifetime intrinsics with calls to the
+    // LLVM 3.7-compliant version of the lifetime intrinsic
+    for (User *U : make_early_inc_range(F->users())) {
+      CallInst *CI = dyn_cast<CallInst>(U);
+      assert(CI &&
+             "Expected user of a lifetime intrinsic function to be a CallInst");
+
+      // LLVM 3.7 lifetime intrinics require an i8* operand, so we insert
+      // a bitcast to ensure that is the case
+      Value *PtrOperand = CI->getArgOperand(0);
+      PointerType *PtrOpPtrTy = cast<PointerType>(PtrOperand->getType());
       Value *NoOpBitCast = CastInst::Create(Instruction::BitCast, PtrOperand,
-                                            PtrTy, "", CI->getIterator());
-      CI->setArgOperand(1, NoOpBitCast);
+                                            PtrOpPtrTy, "", CI->getIterator());
+
+      // LLVM 3.7 lifetime intrinsics have an explicit size operand, whose value
+      // we can obtain from the pointer operand which must be an AllocaInst (as
+      // of https://github.com/llvm/llvm-project/pull/149310)
+      AllocaInst *AI = dyn_cast<AllocaInst>(PtrOperand);
+      assert(AI &&
+             "The pointer operand of a lifetime intrinsic call must be an "
+             "AllocaInst");
+      std::optional<TypeSize> AllocSize =
+          AI->getAllocationSize(CI->getDataLayout());
+      assert(AllocSize.has_value() &&
+             "Expected the allocation size of AllocaInst to be known");
+      CallInst *NewCI = CallInst::Create(
+          LifetimeCallee,
+          {ConstantInt::get(I64Ty, AllocSize.value().getFixedValue()),
+           NoOpBitCast},
+          "", CI->getIterator());
+      for (Attribute ParamAttr : CI->getParamAttributes(0))
+        NewCI->addParamAttr(1, ParamAttr);
+
+      CI->eraseFromParent();
     }
+
+    F->eraseFromParent();
   }
 }
 
 static void removeLifetimeIntrinsics(Module &M) {
-  for (Function &F : make_early_inc_range(M)) {
-    if (Intrinsic::ID IID = F.getIntrinsicID();
-        IID != Intrinsic::lifetime_start && IID != Intrinsic::lifetime_end)
+  Intrinsic::ID LifetimeIIDs[2] = {Intrinsic::lifetime_start,
+                                   Intrinsic::lifetime_end};
+  for (Intrinsic::ID &IID : LifetimeIIDs) {
+    Function *F = M.getFunction(Intrinsic::getBaseName(IID));
+    if (!F)
       continue;
 
-    for (User *U : make_early_inc_range(F.users())) {
-      LifetimeIntrinsic *LI = dyn_cast<LifetimeIntrinsic>(U);
-      assert(LI && "Expected user of lifetime intrinsic function to be "
-                   "a LifetimeIntrinsic instruction");
-      BitCastInst *BCI = dyn_cast<BitCastInst>(LI->getArgOperand(1));
-      assert(BCI && "Expected pointer operand of LifetimeIntrinsic to be a "
-                    "BitCastInst");
-      LI->eraseFromParent();
+    for (User *U : make_early_inc_range(F->users())) {
+      CallInst *CI = dyn_cast<CallInst>(U);
+      assert(CI && "Expected user of lifetime function to be a CallInst");
+      BitCastInst *BCI = dyn_cast<BitCastInst>(CI->getArgOperand(1));
+      assert(BCI && "Expected pointer operand of CallInst to be a BitCastInst");
+      CI->eraseFromParent();
       BCI->eraseFromParent();
     }
-    F.eraseFromParent();
+    F->eraseFromParent();
   }
 }
 
diff --git a/llvm/lib/Target/DirectX/DirectXPassRegistry.def b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
index d5069541642b5..b4b48a166800e 100644
--- a/llvm/lib/Target/DirectX/DirectXPassRegistry.def
+++ b/llvm/lib/Target/DirectX/DirectXPassRegistry.def
@@ -24,6 +24,7 @@ MODULE_ANALYSIS("dxil-root-signature-analysis", dxil::RootSignatureAnalysis())
 #define MODULE_PASS(NAME, CREATE_PASS)
 #endif
 MODULE_PASS("dxil-cbuffer-access", DXILCBufferAccess())
+MODULE_PASS("dxil-finalize-linkage", DXILFinalizeLinkage())
 MODULE_PASS("dxil-data-scalarization", DXILDataScalarization())
 MODULE_PASS("dxil-flatten-arrays", DXILFlattenArrays())
 MODULE_PASS("dxil-intrinsic-expansion", DXILIntrinsicExpansion())
diff --git a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
index 84751d2db2266..f5d5a73c926e9 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetMachine.cpp
@@ -14,6 +14,7 @@
 #include "DirectXTargetMachine.h"
 #include "DXILCBufferAccess.h"
 #include "DXILDataScalarization.h"
+#include "DXILFinalizeLinkage.h"
 #include "DXILFlattenArrays.h"
 #include "DXILForwardHandleAccesses.h"
 #include "DXILIntrinsicExpansion.h"
@@ -45,6 +46,8 @@
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
+#include "llvm/Transforms/IPO/GlobalDCE.h"
+#include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/Scalarizer.h"
 #include <optional>
 
@@ -62,6 +65,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeDirectXTarget() {
   initializeEmbedDXILPassPass(*PR);
   initializeWriteDXILPassPass(*PR);
   initializeDXContainerGlobalsPass(*PR);
+  initializeGlobalDCELegacyPassPass(*PR);
   initializeDXILOpLoweringLegacyPass(*PR);
   initializeDXILResourceAccessLegacyPass(*PR);
   initializeDXILResourceImplicitBindingLegacyPass(*PR);
@@ -72,6 +76,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeDirectXTarget() {
   initializeDXILFinalizeLinkageLegacyPass(*PR);
   initializeDXILPrettyPrinterLegacyPass(*PR);
   initializeDXILForwardHandleAccessesLegacyPass(*PR);
+  initializeDSELegacyPassPass(*PR);
   initializeDXILCBufferAccessLegacyPass(*PR);
 }
 
@@ -103,6 +108,7 @@ class DirectXPassConfig : public TargetPassConfig {
   FunctionPass *createTargetRegisterAllocator(bool) override { return nullptr; }
   void addCodeGenPrepare() override {
     addPass(createDXILFinalizeLinkageLegacyPass());
+    addPass(createGlobalDCEPass());
     addPass(createDXILResourceAccessLegacyPass());
     addPass(createDXILIntrinsicExpansionLegacyPass());
     addPass(createDXILCBufferAccessLegacyPass());
@@ -112,6 +118,7 @@ class DirectXPassConfig : public TargetPassConfig {
     addPass(createScalarizerPass(DxilScalarOptions));
     addPass(createDXILFlattenArraysLegacyPass());
     addPass(createDXILForwardHandleAccessesLegacyPass());
+    addPass(createDeadStoreEliminationPass());
     addPass(createDXILLegalizeLegacyPass());
     addPass(createDXILResourceImplicitBindingLegacyPass());
     addPass(createDXILTranslateMetadataLegacyPass());
diff --git a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
index ffd900c68893f..5153d24070dc9 100644
--- a/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/DirectX/DirectXTargetTransformInfo.cpp
@@ -56,6 +56,8 @@ bool DirectXTTIImpl::isTargetIntrinsicTriviallyScalarizable(
   case Intrinsic::dx_wave_reduce_sum:
   case Intrinsic::dx_wave_reduce_umax:
   case Intrinsic::dx_wave_reduce_usum:
+  case Intrinsic::dx_imad:
+  case Intrinsic::dx_umad:
     return true;
   default:
     return false;
diff --git a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
index 22cff7c80fa01..de10092cbe3c8 100644
--- a/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
+++ b/llvm/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
@@ -13,6 +13,7 @@
 #include "TargetInfo/HexagonTargetInfo.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCExpr.h"
@@ -526,6 +527,9 @@ DecodeStatus HexagonDisassembler::getSingleInstruction(MCInst &MI, MCInst &MCB,
     MI.insert(MI.begin() + 1,
               MCOperand::createExpr(MCConstantExpr::create(-1, getContext())));
     break;
+  case Hexagon::Y4_crswap10:
+    MI.addOperand(MCOperand::createReg(Hexagon::SGP1_0));
+    break;
   default:
     break;
   }
diff --git a/llvm/lib/Target/Hexagon/HexagonCallingConv.td b/llvm/lib/Target/Hexagon/HexagonCallingConv.td
index e0302b85fa948..fd6d873dd4188 100644
--- a/llvm/lib/Target/Hexagon/HexagonCallingConv.td
+++ b/llvm/lib/Target/Hexagon/HexagonCallingConv.td
@@ -6,11 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-class CCIfArgIsVarArg<CCAction A>
-  : CCIf<"State.isVarArg() && "
-         "ValNo >= static_cast<HexagonCCState&>(State)"
-         ".getNumNamedVarArgParams()", A>;
-
 def CC_HexagonStack: CallingConv<[
   CCIfType<[i32,v2i16,v4i8],
     CCAssignToStack<4,4>>,
@@ -28,7 +23,7 @@ def CC_Hexagon_Legacy: CallingConv<[
 
   CCIfByVal<
     CCPassByVal<8,8>>,
-  CCIfArgIsVarArg<
+  CCIfArgVarArg<
     CCDelegateTo<CC_HexagonStack>>,
 
   // Pass split values in pairs, allocate odd register if necessary.
@@ -58,7 +53,7 @@ def CC_Hexagon: CallingConv<[
 
   CCIfByVal<
     CCPassByVal<8,1>>,
-  CCIfArgIsVarArg<
+  CCIfArgVarArg<
     CCDelegateTo<CC_HexagonStack>>,
 
   // Pass split values in pairs, allocate odd register if necessary.
diff --git a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp
index a9201460d8e2e..4ddbe7a53701b 100644
--- a/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonGenInsert.cpp
@@ -17,7 +17,6 @@
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -1273,7 +1272,7 @@ void HexagonGenInsert::selectCandidates() {
 
   for (unsigned R = AllRMs.find_first(); R; R = AllRMs.find_next(R)) {
     using use_iterator = MachineRegisterInfo::use_nodbg_iterator;
-    using InstrSet = SmallSet<const MachineInstr *, 16>;
+    using InstrSet = SmallPtrSet<const MachineInstr *, 16>;
 
     InstrSet UIs;
     // Count as the number of instructions in which R is used, not the
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index facea646d4b68..c54b67ccd8843 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -116,23 +116,6 @@ static cl::opt<bool>
                             cl::desc("Disable minimum alignment of 1 for "
                                      "arguments passed by value on stack"));
 
-namespace {
-
-  class HexagonCCState : public CCState {
-    unsigned NumNamedVarArgParams = 0;
-
-  public:
-    HexagonCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF,
-                   SmallVectorImpl<CCValAssign> &locs, LLVMContext &C,
-                   unsigned NumNamedArgs)
-        : CCState(CC, IsVarArg, MF, locs, C),
-          NumNamedVarArgParams(NumNamedArgs) {}
-    unsigned getNumNamedVarArgParams() const { return NumNamedVarArgParams; }
-  };
-
-} // end anonymous namespace
-
-
 // Implement calling convention for Hexagon.
 
 static bool CC_SkipOdd(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
@@ -497,7 +480,6 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   MachineFrameInfo &MFI = MF.getFrameInfo();
   auto PtrVT = getPointerTy(MF.getDataLayout());
 
-  unsigned NumParams = CLI.CB ? CLI.CB->getFunctionType()->getNumParams() : 0;
   if (GlobalAddressSDNode *GAN = dyn_cast<GlobalAddressSDNode>(Callee))
     Callee = DAG.getTargetGlobalAddress(GAN->getGlobal(), dl, MVT::i32);
 
@@ -506,8 +488,7 @@ HexagonTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   // Analyze operands of the call, assigning locations to each operand.
   SmallVector<CCValAssign, 16> ArgLocs;
-  HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext(),
-                        NumParams);
+  CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
 
   if (Subtarget.useHVXOps())
     CCInfo.AnalyzeCallOperands(Outs, CC_Hexagon_HVX);
@@ -880,9 +861,7 @@ SDValue HexagonTargetLowering::LowerFormalArguments(
 
   // Assign locations to all of the incoming arguments.
   SmallVector<CCValAssign, 16> ArgLocs;
-  HexagonCCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs,
-                        *DAG.getContext(),
-                        MF.getFunction().getFunctionType()->getNumParams());
+  CCState CCInfo(CallConv, TreatAsVarArg, MF, ArgLocs, *DAG.getContext());
 
   if (Subtarget.useHVXOps())
     CCInfo.AnalyzeFormalArguments(Ins, CC_Hexagon_HVX);
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrFormats.td b/llvm/lib/Target/Hexagon/HexagonInstrFormats.td
index 605064986b336..014e48cda5a11 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrFormats.td
+++ b/llvm/lib/Target/Hexagon/HexagonInstrFormats.td
@@ -60,12 +60,6 @@ class InstHexagon<dag outs, dag ins, string asmstr, list<dag> pattern,
   let Itinerary = itin;
   let Size = 4;
 
-  // SoftFail is a field the disassembler can use to provide a way for
-  // instructions to not match without killing the whole decode process. It is
-  // mainly used for ARM, but Tablegen expects this field to exist or it fails
-  // to build the decode table.
-  field bits<32> SoftFail = 0;
-
   // *** Must match MCTargetDesc/HexagonBaseInfo.h ***
 
   // Instruction type according to the ISA.
@@ -287,12 +281,6 @@ class InstDuplex<bits<4> iClass, string cstr = ""> : Instruction,
   let Itinerary = DUPLEX;
   let Size = 4;
 
-  // SoftFail is a field the disassembler can use to provide a way for
-  // instructions to not match without killing the whole decode process. It is
-  // mainly used for ARM, but Tablegen expects this field to exist or it fails
-  // to build the decode table.
-  field bits<32> SoftFail = 0;
-
   // *** Must match MCTargetDesc/HexagonBaseInfo.h ***
 
   let TSFlags{6-0} = Type.Value;
diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
index c34eecd3fcb09..72575f2560a34 100644
--- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -12,7 +12,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/AliasAnalysis.h"
@@ -2289,7 +2288,7 @@ bool HexagonLoopIdiomRecognize::processCopyingStore(Loop *CurLoop,
 // the instructions in Insts are removed.
 bool HexagonLoopIdiomRecognize::coverLoop(Loop *L,
       SmallVectorImpl<Instruction*> &Insts) const {
-  SmallSet<BasicBlock*,8> LoopBlocks;
+  SmallPtrSet<BasicBlock *, 8> LoopBlocks;
   LoopBlocks.insert_range(L->blocks());
 
   SetVector<Instruction *> Worklist(llvm::from_range, Insts);
diff --git a/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp b/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
index 610a81fe45c2f..33aa6e4a26145 100644
--- a/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
@@ -32,14 +32,10 @@ SDValue HexagonSelectionDAGInfo::EmitTargetCodeForMemcpy(
   //
   const TargetLowering &TLI = *DAG.getSubtarget().getTargetLowering();
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
-  Entry.Node = Dst;
-  Args.push_back(Entry);
-  Entry.Node = Src;
-  Args.push_back(Entry);
-  Entry.Node = Size;
-  Args.push_back(Entry);
+  Type *ArgTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
+  Args.emplace_back(Dst, ArgTy);
+  Args.emplace_back(Src, ArgTy);
+  Args.emplace_back(Size, ArgTy);
 
   const char *SpecialMemcpyName = TLI.getLibcallName(
       RTLIB::HEXAGON_MEMCPY_LIKELY_ALIGNED_MIN32BYTES_MULT8BYTES);
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
index ecc1b5d2ebe35..b9cdd6a2a3767 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -15,7 +15,6 @@
 #include "HexagonRegisterInfo.h"
 #include "MCTargetDesc/HexagonMCTargetDesc.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/MachineInstr.h"
@@ -445,8 +444,8 @@ void HexagonSubtarget::adjustSchedDependency(
   const HexagonInstrInfo *QII = getInstrInfo();
 
   // Instructions with .new operands have zero latency.
-  SmallSet<SUnit *, 4> ExclSrc;
-  SmallSet<SUnit *, 4> ExclDst;
+  SmallPtrSet<SUnit *, 4> ExclSrc;
+  SmallPtrSet<SUnit *, 4> ExclDst;
   if (QII->canExecuteInBundle(*SrcInst, *DstInst) &&
       isBestZeroLatency(Src, Dst, QII, ExclSrc, ExclDst)) {
     Dep.setLatency(0);
@@ -630,9 +629,9 @@ static SUnit *getZeroLatency(SUnit *N, SmallVector<SDep, 4> &Deps) {
 // together with a zero latency. Only one dependence should have a zero
 // latency. If there are multiple choices, choose the best, and change
 // the others, if needed.
-bool HexagonSubtarget::isBestZeroLatency(SUnit *Src, SUnit *Dst,
-      const HexagonInstrInfo *TII, SmallSet<SUnit*, 4> &ExclSrc,
-      SmallSet<SUnit*, 4> &ExclDst) const {
+bool HexagonSubtarget::isBestZeroLatency(
+    SUnit *Src, SUnit *Dst, const HexagonInstrInfo *TII,
+    SmallPtrSet<SUnit *, 4> &ExclSrc, SmallPtrSet<SUnit *, 4> &ExclDst) const {
   MachineInstr &SrcInst = *Src->getInstr();
   MachineInstr &DstInst = *Dst->getInstr();
 
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.h b/llvm/lib/Target/Hexagon/HexagonSubtarget.h
index 41555db4ac662..b111471a9696c 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.h
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.h
@@ -366,7 +366,8 @@ class HexagonSubtarget : public HexagonGenSubtargetInfo {
   void restoreLatency(SUnit *Src, SUnit *Dst) const;
   void changeLatency(SUnit *Src, SUnit *Dst, unsigned Lat) const;
   bool isBestZeroLatency(SUnit *Src, SUnit *Dst, const HexagonInstrInfo *TII,
-      SmallSet<SUnit*, 4> &ExclSrc, SmallSet<SUnit*, 4> &ExclDst) const;
+                         SmallPtrSet<SUnit *, 4> &ExclSrc,
+                         SmallPtrSet<SUnit *, 4> &ExclDst) const;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 9fb7d471fd22a..171e2949366ad 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -156,9 +156,10 @@ HexagonTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
   return BaseT::getIntrinsicInstrCost(ICA, CostKind);
 }
 
-InstructionCost HexagonTTIImpl::getAddressComputationCost(Type *Tp,
-                                                          ScalarEvolution *SE,
-                                                          const SCEV *S) const {
+InstructionCost
+HexagonTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
+                                          const SCEV *S,
+                                          TTI::TargetCostKind CostKind) const {
   return 0;
 }
 
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index af8dede723083..dbf16c99c314c 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -111,8 +111,9 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
   InstructionCost
   getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                         TTI::TargetCostKind CostKind) const override;
-  InstructionCost getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
-                                            const SCEV *S) const override;
+  InstructionCost
+  getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *S,
+                            TTI::TargetCostKind CostKind) const override;
   InstructionCost getMemoryOpCost(
       unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
       TTI::TargetCostKind CostKind,
diff --git a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
index 3de6df568c9f4..87d052b9bb679 100644
--- a/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonVectorCombine.cpp
@@ -1677,9 +1677,9 @@ auto HvxIdioms::matchFxpMul(Instruction &In) const -> std::optional<FxpOp> {
     return m_CombineOr(m_LShr(V, S), m_AShr(V, S));
   };
 
-  const APInt *Qn = nullptr;
-  if (Value * T; match(Exp, m_Shr(m_Value(T), m_APInt(Qn)))) {
-    Op.Frac = Qn->getZExtValue();
+  uint64_t Qn = 0;
+  if (Value *T; match(Exp, m_Shr(m_Value(T), m_ConstantInt(Qn)))) {
+    Op.Frac = Qn;
     Exp = T;
   } else {
     Op.Frac = 0;
@@ -1689,9 +1689,9 @@ auto HvxIdioms::matchFxpMul(Instruction &In) const -> std::optional<FxpOp> {
     return std::nullopt;
 
   // Check if there is rounding added.
-  const APInt *C = nullptr;
-  if (Value * T; Op.Frac > 0 && match(Exp, m_Add(m_Value(T), m_APInt(C)))) {
-    uint64_t CV = C->getZExtValue();
+  uint64_t CV;
+  if (Value *T;
+      Op.Frac > 0 && match(Exp, m_Add(m_Value(T), m_ConstantInt(CV)))) {
     if (CV != 0 && !isPowerOf2_64(CV))
       return std::nullopt;
     if (CV != 0)
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
index 039ef4f543a10..645575720e73a 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
@@ -32,7 +32,6 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolELF.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/HexagonAttributes.h"
@@ -145,7 +144,6 @@ void HexagonMCELFStreamer::HexagonMCEmitLocalCommonSymbol(MCSymbol *Symbol,
   getAssembler().registerSymbol(*Symbol);
   auto ELFSymbol = static_cast<const MCSymbolELF *>(Symbol);
   ELFSymbol->setBinding(ELF::STB_LOCAL);
-  ELFSymbol->setExternal(false);
   HexagonMCEmitCommonSymbol(Symbol, Size, ByteAlignment, AccessSize);
 }
 
diff --git a/llvm/lib/Target/Lanai/Disassembler/LanaiDisassembler.cpp b/llvm/lib/Target/Lanai/Disassembler/LanaiDisassembler.cpp
index 5d87c3c4d72cf..c24700b896343 100644
--- a/llvm/lib/Target/Lanai/Disassembler/LanaiDisassembler.cpp
+++ b/llvm/lib/Target/Lanai/Disassembler/LanaiDisassembler.cpp
@@ -16,6 +16,7 @@
 #include "LanaiCondCode.h"
 #include "LanaiInstrInfo.h"
 #include "TargetInfo/LanaiTargetInfo.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCSubtargetInfo.h"
@@ -247,4 +248,4 @@ static DecodeStatus decodePredicateOperand(MCInst &Inst, unsigned Val,
     return MCDisassembler::Fail;
   Inst.addOperand(MCOperand::createImm(Val));
   return MCDisassembler::Success;
-}
+}
\ No newline at end of file
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index d23c5f43ad4ff..f412f66d9d192 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -353,16 +353,15 @@ void LanaiTargetLowering::LowerAsmOperandForConstraint(
 
 #include "LanaiGenCallingConv.inc"
 
-static unsigned NumFixedArgs;
 static bool CC_Lanai32_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
                               CCValAssign::LocInfo LocInfo,
-                              ISD::ArgFlagsTy ArgFlags, CCState &State) {
+                              ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                              CCState &State) {
   // Handle fixed arguments with default CC.
   // Note: Both the default and fast CC handle VarArg the same and hence the
   // calling convention of the function is not considered here.
-  if (ValNo < NumFixedArgs) {
-    return CC_Lanai32(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State);
-  }
+  if (!ArgFlags.isVarArg())
+    return CC_Lanai32(ValNo, ValVT, LocVT, LocInfo, ArgFlags, OrigTy, State);
 
   // Promote i8/i16 args to i32
   if (LocVT == MVT::i8 || LocVT == MVT::i16) {
@@ -603,15 +602,9 @@ SDValue LanaiTargetLowering::LowerCCCCallTo(
   GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
   MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
 
-  NumFixedArgs = 0;
-  if (IsVarArg && G) {
-    const Function *CalleeFn = dyn_cast<Function>(G->getGlobal());
-    if (CalleeFn)
-      NumFixedArgs = CalleeFn->getFunctionType()->getNumParams();
-  }
-  if (NumFixedArgs)
+  if (IsVarArg) {
     CCInfo.AnalyzeCallOperands(Outs, CC_Lanai32_VarArg);
-  else {
+  } else {
     if (CallConv == CallingConv::Fast)
       CCInfo.AnalyzeCallOperands(Outs, CC_Lanai32_Fast);
     else
diff --git a/llvm/lib/Target/Lanai/LanaiInstrFormats.td b/llvm/lib/Target/Lanai/LanaiInstrFormats.td
index 5c21d203edf26..cd36325229b1e 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrFormats.td
+++ b/llvm/lib/Target/Lanai/LanaiInstrFormats.td
@@ -9,7 +9,6 @@
 class InstLanai<dag outs, dag ins, string asmstr, list<dag> pattern>
     : Instruction {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
   let Size = 4;
 
   let Namespace = "Lanai";
diff --git a/llvm/lib/Target/LoongArch/Disassembler/LoongArchDisassembler.cpp b/llvm/lib/Target/LoongArch/Disassembler/LoongArchDisassembler.cpp
index 8c4668ec70c7e..735f798afde24 100644
--- a/llvm/lib/Target/LoongArch/Disassembler/LoongArchDisassembler.cpp
+++ b/llvm/lib/Target/LoongArch/Disassembler/LoongArchDisassembler.cpp
@@ -13,6 +13,7 @@
 #include "MCTargetDesc/LoongArchMCTargetDesc.h"
 #include "TargetInfo/LoongArchTargetInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
diff --git a/llvm/lib/Target/LoongArch/LoongArchCallingConv.td b/llvm/lib/Target/LoongArch/LoongArchCallingConv.td
index 9844163163a55..7dcf65ce2b825 100644
--- a/llvm/lib/Target/LoongArch/LoongArchCallingConv.td
+++ b/llvm/lib/Target/LoongArch/LoongArchCallingConv.td
@@ -21,3 +21,7 @@ def CSR_ILP32D_LP64D
 
 // Needed for implementation of LoongArchRegisterInfo::getNoPreservedMask()
 def CSR_NoRegs : CalleeSavedRegs<(add)>;
+
+def CSR_MostRegs : CalleeSavedRegs<(add CSR_ILP32S_LP64S,
+                                        (sequence "R%u", 4, 11),
+                                        (sequence "R%u", 16, 19))>;
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index a5bf0e57e3053..948362f57904f 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -22,6 +22,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/IR/IRBuilder.h"
@@ -339,6 +340,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
          {MVT::v16i8, MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v8i16, MVT::v4i16,
           MVT::v2i16, MVT::v4i32, MVT::v2i32, MVT::v2i64}) {
       setOperationAction(ISD::TRUNCATE, VT, Custom);
+      setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
     }
   }
 
@@ -376,6 +378,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
       setOperationAction(ISD::ABDS, VT, Legal);
       setOperationAction(ISD::ABDU, VT, Legal);
+      setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
     }
     for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
       setOperationAction(ISD::BITREVERSE, VT, Custom);
@@ -521,10 +524,62 @@ SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
     return lowerFP_TO_BF16(Op, DAG);
   case ISD::BF16_TO_FP:
     return lowerBF16_TO_FP(Op, DAG);
+  case ISD::VECREDUCE_ADD:
+    return lowerVECREDUCE_ADD(Op, DAG);
   }
   return SDValue();
 }
 
+// Lower vecreduce_add using vhaddw instructions.
+// For Example:
+//  call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
+// can be lowered to:
+//  VHADDW_D_W    vr0, vr0, vr0
+//  VHADDW_Q_D    vr0, vr0, vr0
+//  VPICKVE2GR_D  a0,  vr0, 0
+//  ADDI_W        a0,  a0,  0
+SDValue LoongArchTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
+                                                    SelectionDAG &DAG) const {
+
+  SDLoc DL(Op);
+  MVT OpVT = Op.getSimpleValueType();
+  SDValue Val = Op.getOperand(0);
+
+  unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
+  unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
+
+  unsigned LegalVecSize = 128;
+  bool isLASX256Vector =
+      Subtarget.hasExtLASX() && Val.getValueSizeInBits() == 256;
+
+  // Ensure operand type legal or enable it legal.
+  while (!isTypeLegal(Val.getSimpleValueType())) {
+    Val = DAG.WidenVector(Val, DL);
+  }
+
+  // NumEles is designed for iterations count, v4i32 for LSX
+  // and v8i32 for LASX should have the same count.
+  if (isLASX256Vector) {
+    NumEles /= 2;
+    LegalVecSize = 256;
+  }
+
+  for (unsigned i = 1; i < NumEles; i *= 2, EleBits *= 2) {
+    MVT IntTy = MVT::getIntegerVT(EleBits);
+    MVT VecTy = MVT::getVectorVT(IntTy, LegalVecSize / EleBits);
+    Val = DAG.getNode(LoongArchISD::VHADDW, DL, VecTy, Val, Val);
+  }
+
+  if (isLASX256Vector) {
+    SDValue Tmp = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, Val,
+                              DAG.getConstant(2, DL, MVT::i64));
+    Val = DAG.getNode(ISD::ADD, DL, MVT::v4i64, Tmp, Val);
+  }
+
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, Val,
+                     DAG.getConstant(0, DL, Subtarget.getGRLenVT()));
+}
+
 SDValue LoongArchTargetLowering::lowerPREFETCH(SDValue Op,
                                                SelectionDAG &DAG) const {
   unsigned IsData = Op.getConstantOperandVal(4);
@@ -2786,7 +2841,7 @@ SDValue LoongArchTargetLowering::lowerUINT_TO_FP(SDValue Op,
   EVT RetVT = Op.getValueType();
   RTLIB::Libcall LC = RTLIB::getUINTTOFP(OpVT, RetVT);
   MakeLibCallOptions CallOptions;
-  CallOptions.setTypeListBeforeSoften(OpVT, RetVT, true);
+  CallOptions.setTypeListBeforeSoften(OpVT, RetVT);
   SDValue Chain = SDValue();
   SDValue Result;
   std::tie(Result, Chain) =
@@ -2811,7 +2866,7 @@ SDValue LoongArchTargetLowering::lowerSINT_TO_FP(SDValue Op,
   EVT RetVT = Op.getValueType();
   RTLIB::Libcall LC = RTLIB::getSINTTOFP(OpVT, RetVT);
   MakeLibCallOptions CallOptions;
-  CallOptions.setTypeListBeforeSoften(OpVT, RetVT, true);
+  CallOptions.setTypeListBeforeSoften(OpVT, RetVT);
   SDValue Chain = SDValue();
   SDValue Result;
   std::tie(Result, Chain) =
@@ -3037,10 +3092,7 @@ SDValue LoongArchTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
 
   // Prepare argument list to generate call.
   ArgListTy Args;
-  ArgListEntry Entry;
-  Entry.Node = Load;
-  Entry.Ty = CallTy;
-  Args.push_back(Entry);
+  Args.emplace_back(Load, CallTy);
 
   // Setup call to __tls_get_addr.
   TargetLowering::CallLoweringInfo CLI(DAG);
@@ -4107,7 +4159,7 @@ void LoongArchTargetLowering::ReplaceNodeResults(
     LC = RTLIB::getFPTOSINT(Src.getValueType(), VT);
     MakeLibCallOptions CallOptions;
     EVT OpVT = Src.getValueType();
-    CallOptions.setTypeListBeforeSoften(OpVT, VT, true);
+    CallOptions.setTypeListBeforeSoften(OpVT, VT);
     SDValue Chain = SDValue();
     SDValue Result;
     std::tie(Result, Chain) =
@@ -4360,7 +4412,7 @@ void LoongArchTargetLowering::ReplaceNodeResults(
     RTLIB::Libcall LC =
         OpVT == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
     MakeLibCallOptions CallOptions;
-    CallOptions.setTypeListBeforeSoften(OpVT, MVT::i64, true);
+    CallOptions.setTypeListBeforeSoften(OpVT, MVT::i64);
     SDValue Result = makeLibCall(DAG, LC, MVT::i64, Op0, CallOptions, DL).first;
     Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
     Results.push_back(Result);
@@ -6042,17 +6094,20 @@ static MachineBasicBlock *
 emitPseudoXVINSGR2VR(MachineInstr &MI, MachineBasicBlock *BB,
                      const LoongArchSubtarget &Subtarget) {
   unsigned InsOp;
+  unsigned BroadcastOp;
   unsigned HalfSize;
   switch (MI.getOpcode()) {
   default:
     llvm_unreachable("Unexpected opcode");
   case LoongArch::PseudoXVINSGR2VR_B:
     HalfSize = 16;
-    InsOp = LoongArch::VINSGR2VR_B;
+    BroadcastOp = LoongArch::XVREPLGR2VR_B;
+    InsOp = LoongArch::XVEXTRINS_B;
     break;
   case LoongArch::PseudoXVINSGR2VR_H:
     HalfSize = 8;
-    InsOp = LoongArch::VINSGR2VR_H;
+    BroadcastOp = LoongArch::XVREPLGR2VR_H;
+    InsOp = LoongArch::XVEXTRINS_H;
     break;
   }
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
@@ -6066,37 +6121,41 @@ emitPseudoXVINSGR2VR(MachineInstr &MI, MachineBasicBlock *BB,
   Register Elt = MI.getOperand(2).getReg();
   unsigned Idx = MI.getOperand(3).getImm();
 
-  Register ScratchReg1 = XSrc;
-  if (Idx >= HalfSize) {
-    ScratchReg1 = MRI.createVirtualRegister(RC);
-    BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPERMI_D), ScratchReg1)
-        .addReg(XSrc)
-        .addImm(14);
-  }
+  if (XSrc.isVirtual() && MRI.getVRegDef(XSrc)->isImplicitDef() &&
+      Idx < HalfSize) {
+    Register ScratchSubReg1 = MRI.createVirtualRegister(SubRC);
+    Register ScratchSubReg2 = MRI.createVirtualRegister(SubRC);
 
-  Register ScratchSubReg1 = MRI.createVirtualRegister(SubRC);
-  Register ScratchSubReg2 = MRI.createVirtualRegister(SubRC);
-  BuildMI(*BB, MI, DL, TII->get(LoongArch::COPY), ScratchSubReg1)
-      .addReg(ScratchReg1, 0, LoongArch::sub_128);
-  BuildMI(*BB, MI, DL, TII->get(InsOp), ScratchSubReg2)
-      .addReg(ScratchSubReg1)
-      .addReg(Elt)
-      .addImm(Idx >= HalfSize ? Idx - HalfSize : Idx);
+    BuildMI(*BB, MI, DL, TII->get(LoongArch::COPY), ScratchSubReg1)
+        .addReg(XSrc, 0, LoongArch::sub_128);
+    BuildMI(*BB, MI, DL,
+            TII->get(HalfSize == 8 ? LoongArch::VINSGR2VR_H
+                                   : LoongArch::VINSGR2VR_B),
+            ScratchSubReg2)
+        .addReg(ScratchSubReg1)
+        .addReg(Elt)
+        .addImm(Idx);
+
+    BuildMI(*BB, MI, DL, TII->get(LoongArch::SUBREG_TO_REG), XDst)
+        .addImm(0)
+        .addReg(ScratchSubReg2)
+        .addImm(LoongArch::sub_128);
+  } else {
+    Register ScratchReg1 = MRI.createVirtualRegister(RC);
+    Register ScratchReg2 = MRI.createVirtualRegister(RC);
 
-  Register ScratchReg2 = XDst;
-  if (Idx >= HalfSize)
-    ScratchReg2 = MRI.createVirtualRegister(RC);
+    BuildMI(*BB, MI, DL, TII->get(BroadcastOp), ScratchReg1).addReg(Elt);
 
-  BuildMI(*BB, MI, DL, TII->get(LoongArch::SUBREG_TO_REG), ScratchReg2)
-      .addImm(0)
-      .addReg(ScratchSubReg2)
-      .addImm(LoongArch::sub_128);
+    BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPERMI_Q), ScratchReg2)
+        .addReg(ScratchReg1)
+        .addReg(XSrc)
+        .addImm(Idx >= HalfSize ? 48 : 18);
 
-  if (Idx >= HalfSize)
-    BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPERMI_Q), XDst)
+    BuildMI(*BB, MI, DL, TII->get(InsOp), XDst)
         .addReg(XSrc)
         .addReg(ScratchReg2)
-        .addImm(2);
+        .addImm((Idx >= HalfSize ? Idx - HalfSize : Idx) * 17);
+  }
 
   MI.eraseFromParent();
   return BB;
@@ -6654,6 +6713,7 @@ const char *LoongArchTargetLowering::getTargetNodeName(unsigned Opcode) const {
     NODE_NAME_CASE(XVMSKGEZ)
     NODE_NAME_CASE(XVMSKEQZ)
     NODE_NAME_CASE(XVMSKNEZ)
+    NODE_NAME_CASE(VHADDW)
   }
 #undef NODE_NAME_CASE
   return nullptr;
@@ -6729,8 +6789,7 @@ static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State,
 static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
                          unsigned ValNo, MVT ValVT,
                          CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                         CCState &State, bool IsFixed, bool IsRet,
-                         Type *OrigTy) {
+                         CCState &State, bool IsRet, Type *OrigTy) {
   unsigned GRLen = DL.getLargestLegalIntTypeSizeInBits();
   assert((GRLen == 32 || GRLen == 64) && "Unspport GRLen");
   MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64;
@@ -6752,7 +6811,7 @@ static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
   case LoongArchABI::ABI_LP64F:
   case LoongArchABI::ABI_ILP32D:
   case LoongArchABI::ABI_LP64D:
-    UseGPRForFloat = !IsFixed;
+    UseGPRForFloat = ArgFlags.isVarArg();
     break;
   case LoongArchABI::ABI_ILP32S:
   case LoongArchABI::ABI_LP64S:
@@ -6766,7 +6825,8 @@ static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI,
   // will not be passed by registers if the original type is larger than
   // 2*GRLen, so the register alignment rule does not apply.
   unsigned TwoGRLenInBytes = (2 * GRLen) / 8;
-  if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes &&
+  if (ArgFlags.isVarArg() &&
+      ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes &&
       DL.getTypeAllocSize(OrigTy) == TwoGRLenInBytes) {
     unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
     // Skip 'odd' register if necessary.
@@ -6916,7 +6976,7 @@ void LoongArchTargetLowering::analyzeInputArgs(
     LoongArchABI::ABI ABI =
         MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Ins[i].Flags,
-           CCInfo, /*IsFixed=*/true, IsRet, ArgTy)) {
+           CCInfo, IsRet, ArgTy)) {
       LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " << ArgVT
                         << '\n');
       llvm_unreachable("");
@@ -6934,7 +6994,7 @@ void LoongArchTargetLowering::analyzeOutputArgs(
     LoongArchABI::ABI ABI =
         MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Outs[i].Flags,
-           CCInfo, Outs[i].IsFixed, IsRet, OrigTy)) {
+           CCInfo, IsRet, OrigTy)) {
       LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " << ArgVT
                         << "\n");
       llvm_unreachable("");
@@ -7073,7 +7133,8 @@ static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
 
 static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
                              CCValAssign::LocInfo LocInfo,
-                             ISD::ArgFlagsTy ArgFlags, CCState &State) {
+                             ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                             CCState &State) {
   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
     // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, SpLim
     //                        s0    s1  s2  s3  s4  s5  s6  s7  s8
@@ -7126,6 +7187,7 @@ SDValue LoongArchTargetLowering::LowerFormalArguments(
     llvm_unreachable("Unsupported calling convention");
   case CallingConv::C:
   case CallingConv::Fast:
+  case CallingConv::PreserveMost:
     break;
   case CallingConv::GHC:
     if (!MF.getSubtarget().hasFeature(LoongArch::FeatureBasicF) ||
@@ -7647,8 +7709,7 @@ bool LoongArchTargetLowering::CanLowerReturn(
     LoongArchABI::ABI ABI =
         MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
     if (CC_LoongArch(MF.getDataLayout(), ABI, i, Outs[i].VT, CCValAssign::Full,
-                     Outs[i].Flags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true,
-                     nullptr))
+                     Outs[i].Flags, CCInfo, /*IsRet=*/true, nullptr))
       return false;
   }
   return true;
diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 6b49a98f3ae46..40e237b1c69e4 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -177,6 +177,9 @@ enum NodeType : unsigned {
   XVMSKEQZ,
   XVMSKNEZ,
 
+  // Vector Horizontal Addition with Widening‌
+  VHADDW
+
   // Intrinsic operations end =============================================
 };
 } // end namespace LoongArchISD
@@ -330,7 +333,7 @@ class LoongArchTargetLowering : public TargetLowering {
                                    unsigned ValNo, MVT ValVT,
                                    CCValAssign::LocInfo LocInfo,
                                    ISD::ArgFlagsTy ArgFlags, CCState &State,
-                                   bool IsFixed, bool IsRet, Type *OrigTy);
+                                   bool IsRet, Type *OrigTy);
 
   void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo,
                         const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
@@ -386,6 +389,7 @@ class LoongArchTargetLowering : public TargetLowering {
   SDValue lowerFP16_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerBF16_TO_FP(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerVECREDUCE_ADD(SDValue Op, SelectionDAG &DAG) const;
 
   bool isFPImmLegal(const APFloat &Imm, EVT VT,
                     bool ForCodeSize) const override;
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrFormats.td b/llvm/lib/Target/LoongArch/LoongArchInstrFormats.td
index eee297d2e2d91..419e20431c59f 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrFormats.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrFormats.td
@@ -20,11 +20,6 @@ class LAInst<dag outs, dag ins, string opcstr, string opnstr,
              list<dag> pattern = []>
     : Instruction {
   field bits<32> Inst;
-  // SoftFail is a field the disassembler can use to provide a way for
-  // instructions to not match without killing the whole decode process. It is
-  // mainly used for ARM, but Tablegen expects this field to exist or it fails
-  // to build the decode table.
-  field bits<32> SoftFail = 0;
 
   let Namespace = "LoongArch";
   let Size = 4;
diff --git a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
index d8bb16fe9b94d..cf63750461edd 100644
--- a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td
@@ -1186,6 +1186,17 @@ multiclass PatXrXrXr<SDPatternOperator OpNode, string Inst> {
             (!cast<LAInst>(Inst#"_D") LASX256:$xd, LASX256:$xj, LASX256:$xk)>;
 }
 
+multiclass PatXrXrW<SDPatternOperator OpNode, string Inst> {
+  def : Pat<(OpNode(v32i8 LASX256:$vj), (v32i8 LASX256:$vk)),
+            (!cast<LAInst>(Inst#"_H_B") LASX256:$vj, LASX256:$vk)>;
+  def : Pat<(OpNode(v16i16 LASX256:$vj), (v16i16 LASX256:$vk)),
+            (!cast<LAInst>(Inst#"_W_H") LASX256:$vj, LASX256:$vk)>;
+  def : Pat<(OpNode(v8i32 LASX256:$vj), (v8i32 LASX256:$vk)),
+            (!cast<LAInst>(Inst#"_D_W") LASX256:$vj, LASX256:$vk)>;
+  def : Pat<(OpNode(v4i64 LASX256:$vj), (v4i64 LASX256:$vk)),
+            (!cast<LAInst>(Inst#"_Q_D") LASX256:$vj, LASX256:$vk)>;
+}
+
 multiclass PatShiftXrXr<SDPatternOperator OpNode, string Inst> {
   def : Pat<(OpNode (v32i8 LASX256:$xj), (and vsplati8_imm_eq_7,
                                               (v32i8 LASX256:$xk))),
@@ -1513,6 +1524,9 @@ def : Pat<(bswap (v8i32 LASX256:$xj)), (XVSHUF4I_B LASX256:$xj, 0b00011011)>;
 def : Pat<(bswap (v4i64 LASX256:$xj)),
           (XVSHUF4I_W (XVSHUF4I_B LASX256:$xj, 0b00011011), 0b10110001)>;
 
+// XVHADDW_{H_B/W_H/D_W/Q_D}
+defm : PatXrXrW<loongarch_vhaddw, "XVHADDW">;
+
 // XVFADD_{S/D}
 defm : PatXrXrF<fadd, "XVFADD">;
 
@@ -1640,6 +1654,24 @@ defm : PairInsertExtractPatV8<v8f32, f32>;
 defm : PairInsertExtractPatV4<v4i64, GRLenVT>;
 defm : PairInsertExtractPatV4<v4f64, f64>;
 
+def : Pat<(vector_insert v8i32:$xd, (GRLenVT(vector_extract v8i32:$xj, 0)),
+              uimm3:$imm),
+          (XVINSVE0_W v8i32:$xd, v8i32:$xj, uimm3:$imm)>;
+
+def : Pat<(vector_insert v4i64:$xd, (GRLenVT(vector_extract v4i64:$xj, 0)),
+              uimm2:$imm),
+          (XVINSVE0_D v4i64:$xd, v4i64:$xj, uimm2:$imm)>;
+
+def : Pat<(vector_insert v8i32:$xd,
+              (GRLenVT(vector_extract v8i32:$xj, uimm3:$imm1)), uimm3:$imm2),
+          (XVINSVE0_W v8i32:$xd, (XVPICKVE_W v8i32:$xj, uimm3:$imm1),
+              uimm3:$imm2)>;
+
+def : Pat<(vector_insert v4i64:$xd,
+              (GRLenVT(vector_extract v4i64:$xj, uimm2:$imm1)), uimm2:$imm2),
+          (XVINSVE0_D v4i64:$xd, (XVPICKVE_D v4i64:$xj, uimm2:$imm1),
+              uimm2:$imm2)>;
+
 // PseudoXVINSGR2VR_{B/H}
 def : Pat<(vector_insert v32i8:$xd, GRLenVT:$rj, uimm5:$imm),
           (PseudoXVINSGR2VR_B v32i8:$xd, GRLenVT:$rj, uimm5:$imm)>;
diff --git a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
index 3c9defb0366ff..168a60004fc0f 100644
--- a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td
@@ -71,6 +71,8 @@ def loongarch_vsrli : SDNode<"LoongArchISD::VSRLI", SDT_LoongArchV1RUimm>;
 def loongarch_vbsll : SDNode<"LoongArchISD::VBSLL", SDT_LoongArchV1RUimm>;
 def loongarch_vbsrl : SDNode<"LoongArchISD::VBSRL", SDT_LoongArchV1RUimm>;
 
+def loongarch_vhaddw : SDNode<"LoongArchISD::VHADDW", SDT_LoongArchV2R>;
+
 def loongarch_vldrepl
     : SDNode<"LoongArchISD::VLDREPL",
              SDT_LoongArchVLDREPL, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
@@ -1364,6 +1366,17 @@ multiclass PatVrVrVr<SDPatternOperator OpNode, string Inst> {
             (!cast<LAInst>(Inst#"_D") LSX128:$vd, LSX128:$vj, LSX128:$vk)>;
 }
 
+multiclass PatVrVrW<SDPatternOperator OpNode, string Inst> {
+  def : Pat<(OpNode(v16i8 LSX128:$vj), (v16i8 LSX128:$vk)),
+            (!cast<LAInst>(Inst#"_H_B") LSX128:$vj, LSX128:$vk)>;
+  def : Pat<(OpNode(v8i16 LSX128:$vj), (v8i16 LSX128:$vk)),
+            (!cast<LAInst>(Inst#"_W_H") LSX128:$vj, LSX128:$vk)>;
+  def : Pat<(OpNode(v4i32 LSX128:$vj), (v4i32 LSX128:$vk)),
+            (!cast<LAInst>(Inst#"_D_W") LSX128:$vj, LSX128:$vk)>;
+  def : Pat<(OpNode(v2i64 LSX128:$vj), (v2i64 LSX128:$vk)),
+            (!cast<LAInst>(Inst#"_Q_D") LSX128:$vj, LSX128:$vk)>;
+}
+
 multiclass PatShiftVrVr<SDPatternOperator OpNode, string Inst> {
   def : Pat<(OpNode (v16i8 LSX128:$vj), (and vsplati8_imm_eq_7,
                                              (v16i8 LSX128:$vk))),
@@ -1709,6 +1722,9 @@ def : Pat<(bswap (v4i32 LSX128:$vj)), (VSHUF4I_B LSX128:$vj, 0b00011011)>;
 def : Pat<(bswap (v2i64 LSX128:$vj)),
           (VSHUF4I_W (VSHUF4I_B LSX128:$vj, 0b00011011), 0b10110001)>;
 
+// VHADDW_{H_B/W_H/D_W/Q_D}
+defm : PatVrVrW<loongarch_vhaddw, "VHADDW">;
+
 // VFADD_{S/D}
 defm : PatVrVrF<fadd, "VFADD">;
 
diff --git a/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp
index 47fce37ce59f6..9c5f8edfaf66a 100644
--- a/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchRegisterInfo.cpp
@@ -41,6 +41,8 @@ LoongArchRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
 
   if (MF->getFunction().getCallingConv() == CallingConv::GHC)
     return CSR_NoRegs_SaveList;
+  if (MF->getFunction().getCallingConv() == CallingConv::PreserveMost)
+    return CSR_MostRegs_SaveList;
   switch (Subtarget.getTargetABI()) {
   default:
     llvm_unreachable("Unrecognized ABI");
@@ -63,6 +65,8 @@ LoongArchRegisterInfo::getCallPreservedMask(const MachineFunction &MF,
 
   if (CC == CallingConv::GHC)
     return CSR_NoRegs_RegMask;
+  if (CC == CallingConv::PreserveMost)
+    return CSR_MostRegs_RegMask;
   switch (Subtarget.getTargetABI()) {
   default:
     llvm_unreachable("Unrecognized ABI");
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
index ede5477f04bda..efe898c33072e 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
@@ -95,4 +95,13 @@ unsigned LoongArchTTIImpl::getPrefetchDistance() const { return 200; }
 
 bool LoongArchTTIImpl::enableWritePrefetching() const { return true; }
 
+bool LoongArchTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
+  switch (II->getIntrinsicID()) {
+  default:
+    return true;
+  case Intrinsic::vector_reduce_add:
+    return false;
+  }
+}
+
 // TODO: Implement more hooks to provide TTI machinery for LoongArch.
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
index d43d2cb0eb124..e3f16c7804994 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
@@ -53,6 +53,8 @@ class LoongArchTTIImpl : public BasicTTIImplBase<LoongArchTTIImpl> {
   unsigned getPrefetchDistance() const override;
   bool enableWritePrefetching() const override;
 
+  bool shouldExpandReduction(const IntrinsicInst *II) const override;
+
   // TODO: Implement more hooks to provide TTI machinery for LoongArch.
 };
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index fda9d979e1860..0696813040654 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -143,8 +143,6 @@ static void fixupLeb128(MCContext &Ctx, const MCFixup &Fixup, uint8_t *Data,
 void LoongArchAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                      const MCValue &Target, uint8_t *Data,
                                      uint64_t Value, bool IsResolved) {
-  if (IsResolved && shouldForceRelocation(Fixup, Target))
-    IsResolved = false;
   IsResolved = addReloc(F, Fixup, Target, Value, IsResolved);
   if (!Value)
     return; // Doesn't change encoding.
@@ -176,20 +174,6 @@ void LoongArchAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
   }
 }
 
-bool LoongArchAsmBackend::shouldForceRelocation(const MCFixup &Fixup,
-                                                const MCValue &Target) {
-  switch (Fixup.getKind()) {
-  default:
-    return STI.hasFeature(LoongArch::FeatureRelax);
-  case FK_Data_1:
-  case FK_Data_2:
-  case FK_Data_4:
-  case FK_Data_8:
-  case FK_Data_leb128:
-    return !Target.isAbsolute();
-  }
-}
-
 static inline std::pair<MCFixupKind, MCFixupKind>
 getRelocPairForSize(unsigned Size) {
   switch (Size) {
@@ -216,10 +200,19 @@ getRelocPairForSize(unsigned Size) {
 // size, the fixup encodes MaxBytesToEmit in the higher bits and references a
 // per-section marker symbol.
 bool LoongArchAsmBackend::relaxAlign(MCFragment &F, unsigned &Size) {
+  // Alignments before the first linker-relaxable instruction have fixed sizes
+  // and do not require relocations. Alignments after a linker-relaxable
+  // instruction require a relocation, even if the STI specifies norelax.
+  //
+  // firstLinkerRelaxable is the layout order within the subsection, which may
+  // be smaller than the section's order. Therefore, alignments in a
+  // lower-numbered subsection may be unnecessarily treated as linker-relaxable.
+  auto *Sec = F.getParent();
+  if (F.getLayoutOrder() <= Sec->firstLinkerRelaxable())
+    return false;
+
   // Use default handling unless linker relaxation is enabled and the
   // MaxBytesToEmit >= the nop size.
-  if (!F.getSubtargetInfo()->hasFeature(LoongArch::FeatureRelax))
-    return false;
   const unsigned MinNopLen = 4;
   unsigned MaxBytesToEmit = F.getAlignMaxBytesToEmit();
   if (MaxBytesToEmit < MinNopLen)
@@ -254,7 +247,6 @@ bool LoongArchAsmBackend::relaxAlign(MCFragment &F, unsigned &Size) {
       MCFixup::create(0, Expr, FirstLiteralRelocationKind + ELF::R_LARCH_ALIGN);
   F.setVarFixups({Fixup});
   F.setLinkerRelaxable();
-  F.getParent()->setLinkerRelaxable();
   return true;
 }
 
@@ -267,14 +259,10 @@ std::pair<bool, bool> LoongArchAsmBackend::relaxLEB128(MCFragment &F,
   return std::make_pair(true, true);
 }
 
-bool LoongArchAsmBackend::relaxDwarfLineAddr(MCFragment &F,
-                                             bool &WasRelaxed) const {
+bool LoongArchAsmBackend::relaxDwarfLineAddr(MCFragment &F) const {
   MCContext &C = getContext();
-
   int64_t LineDelta = F.getDwarfLineDelta();
   const MCExpr &AddrDelta = F.getDwarfAddrDelta();
-  size_t OldSize = F.getVarSize();
-
   int64_t Value;
   if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
     return false;
@@ -320,15 +308,12 @@ bool LoongArchAsmBackend::relaxDwarfLineAddr(MCFragment &F,
   F.setVarContents(Data);
   F.setVarFixups({MCFixup::create(Offset, &AddrDelta,
                                   MCFixup::getDataKindForSize(PCBytes))});
-  WasRelaxed = OldSize != Data.size();
   return true;
 }
 
-bool LoongArchAsmBackend::relaxDwarfCFA(MCFragment &F, bool &WasRelaxed) const {
+bool LoongArchAsmBackend::relaxDwarfCFA(MCFragment &F) const {
   const MCExpr &AddrDelta = F.getDwarfAddrDelta();
   SmallVector<MCFixup, 2> Fixups;
-  size_t OldSize = F.getVarContents().size();
-
   int64_t Value;
   if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
     return false;
@@ -341,7 +326,6 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCFragment &F, bool &WasRelaxed) const {
   if (Value == 0) {
     F.clearVarContents();
     F.clearVarFixups();
-    WasRelaxed = OldSize != 0;
     return true;
   }
 
@@ -375,8 +359,6 @@ bool LoongArchAsmBackend::relaxDwarfCFA(MCFragment &F, bool &WasRelaxed) const {
   }
   F.setVarContents(Data);
   F.setVarFixups(Fixups);
-
-  WasRelaxed = OldSize != Data.size();
   return true;
 }
 
@@ -447,10 +429,10 @@ bool LoongArchAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
           isPCRelFixupResolved(Target.getSubSym(), F))
         return Fallback();
 
-      // In SecA == SecB case. If the linker relaxation is disabled, the
+      // In SecA == SecB case. If the section is not linker-relaxable, the
       // FixedValue has already been calculated out in evaluateFixup,
       // return true and avoid record relocations.
-      if (&SecA == &SecB && !STI.hasFeature(LoongArch::FeatureRelax))
+      if (&SecA == &SecB && !SecA.isLinkerRelaxable())
         return true;
     }
 
@@ -483,9 +465,16 @@ bool LoongArchAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
     return false;
   }
 
-  IsResolved = Fallback();
   // If linker relaxation is enabled and supported by the current relocation,
-  // append a RELAX relocation.
+  // generate a relocation and then append a RELAX.
+  if (Fixup.isLinkerRelaxable())
+    IsResolved = false;
+  if (IsResolved && Fixup.isPCRel())
+    IsResolved = isPCRelFixupResolved(Target.getAddSym(), F);
+
+  if (!IsResolved)
+    Asm->getWriter().recordRelocation(F, Fixup, Target, FixedValue);
+
   if (Fixup.isLinkerRelaxable()) {
     auto FA = MCFixup::create(Fixup.getOffset(), nullptr, ELF::R_LARCH_RELAX);
     Asm->getWriter().recordRelocation(F, FA, MCValue::get(nullptr),
@@ -497,8 +486,7 @@ bool LoongArchAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
 
 std::unique_ptr<MCObjectTargetWriter>
 LoongArchAsmBackend::createObjectTargetWriter() const {
-  return createLoongArchELFObjectWriter(
-      OSABI, Is64Bit, STI.hasFeature(LoongArch::FeatureRelax));
+  return createLoongArchELFObjectWriter(OSABI, Is64Bit);
 }
 
 MCAsmBackend *llvm::createLoongArchAsmBackend(const Target &T,
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
index 1f1360119edba..8236e300a6ab2 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h
@@ -44,15 +44,13 @@ class LoongArchAsmBackend : public MCAsmBackend {
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
                   uint8_t *Data, uint64_t Value, bool IsResolved) override;
 
-  bool shouldForceRelocation(const MCFixup &Fixup, const MCValue &Target);
-
   std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
 
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   bool relaxAlign(MCFragment &F, unsigned &Size) override;
-  bool relaxDwarfLineAddr(MCFragment &F, bool &WasRelaxed) const override;
-  bool relaxDwarfCFA(MCFragment &F, bool &WasRelaxed) const override;
+  bool relaxDwarfLineAddr(MCFragment &) const override;
+  bool relaxDwarfCFA(MCFragment &) const override;
   std::pair<bool, bool> relaxLEB128(MCFragment &F,
                                     int64_t &Value) const override;
 
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
index 7e021e486836a..7d5456555045b 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchELFObjectWriter.cpp
@@ -21,26 +21,23 @@ using namespace llvm;
 namespace {
 class LoongArchELFObjectWriter : public MCELFObjectTargetWriter {
 public:
-  LoongArchELFObjectWriter(uint8_t OSABI, bool Is64Bit, bool EnableRelax);
+  LoongArchELFObjectWriter(uint8_t OSABI, bool Is64Bit);
 
   ~LoongArchELFObjectWriter() override;
 
   bool needsRelocateWithSymbol(const MCValue &, unsigned Type) const override {
-    return EnableRelax;
+    return true;
   }
 
 protected:
   unsigned getRelocType(const MCFixup &, const MCValue &,
                         bool IsPCRel) const override;
-  bool EnableRelax;
 };
 } // end namespace
 
-LoongArchELFObjectWriter::LoongArchELFObjectWriter(uint8_t OSABI, bool Is64Bit,
-                                                   bool EnableRelax)
+LoongArchELFObjectWriter::LoongArchELFObjectWriter(uint8_t OSABI, bool Is64Bit)
     : MCELFObjectTargetWriter(Is64Bit, OSABI, ELF::EM_LOONGARCH,
-                              /*HasRelocationAddend=*/true),
-      EnableRelax(EnableRelax) {}
+                              /*HasRelocationAddend=*/true) {}
 
 LoongArchELFObjectWriter::~LoongArchELFObjectWriter() {}
 
@@ -103,6 +100,6 @@ unsigned LoongArchELFObjectWriter::getRelocType(const MCFixup &Fixup,
 }
 
 std::unique_ptr<MCObjectTargetWriter>
-llvm::createLoongArchELFObjectWriter(uint8_t OSABI, bool Is64Bit, bool Relax) {
-  return std::make_unique<LoongArchELFObjectWriter>(OSABI, Is64Bit, Relax);
+llvm::createLoongArchELFObjectWriter(uint8_t OSABI, bool Is64Bit) {
+  return std::make_unique<LoongArchELFObjectWriter>(OSABI, Is64Bit);
 }
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.h
index bb05baa9b717c..ab35a0096c8a2 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.h
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchMCTargetDesc.h
@@ -36,7 +36,7 @@ MCAsmBackend *createLoongArchAsmBackend(const Target &T,
                                         const MCTargetOptions &Options);
 
 std::unique_ptr<MCObjectTargetWriter>
-createLoongArchELFObjectWriter(uint8_t OSABI, bool Is64Bit, bool Relax);
+createLoongArchELFObjectWriter(uint8_t OSABI, bool Is64Bit);
 
 } // end namespace llvm
 
diff --git a/llvm/lib/Target/M68k/CMakeLists.txt b/llvm/lib/Target/M68k/CMakeLists.txt
index 7005df4fb8a82..b730f41b22353 100644
--- a/llvm/lib/Target/M68k/CMakeLists.txt
+++ b/llvm/lib/Target/M68k/CMakeLists.txt
@@ -14,7 +14,7 @@ tablegen(LLVM M68kGenDAGISel.inc          -gen-dag-isel)
 tablegen(LLVM M68kGenCallingConv.inc      -gen-callingconv)
 tablegen(LLVM M68kGenAsmWriter.inc        -gen-asm-writer)
 tablegen(LLVM M68kGenAsmMatcher.inc       -gen-asm-matcher)
-tablegen(LLVM M68kGenDisassemblerTable.inc -gen-disassembler)
+tablegen(LLVM M68kGenDisassemblerTables.inc -gen-disassembler)
 
 add_public_tablegen_target(M68kCommonTableGen)
 
diff --git a/llvm/lib/Target/M68k/Disassembler/M68kDisassembler.cpp b/llvm/lib/Target/M68k/Disassembler/M68kDisassembler.cpp
index 4ec18fe6bf544..d3ad65390143b 100644
--- a/llvm/lib/Target/M68k/Disassembler/M68kDisassembler.cpp
+++ b/llvm/lib/Target/M68k/Disassembler/M68kDisassembler.cpp
@@ -19,6 +19,7 @@
 
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -27,6 +28,7 @@
 #include "llvm/Support/ErrorHandling.h"
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "m68k-disassembler"
 
@@ -83,12 +85,6 @@ static DecodeStatus DecodeXR32RegisterClass(MCInst &Inst, uint64_t RegNo,
   return DecodeRegisterClass(Inst, RegNo, Address, Decoder);
 }
 
-static DecodeStatus DecodeXR32RegisterClass(MCInst &Inst, APInt RegNo,
-                                            uint64_t Address,
-                                            const void *Decoder) {
-  return DecodeRegisterClass(Inst, RegNo.getZExtValue(), Address, Decoder);
-}
-
 static DecodeStatus DecodeXR16RegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
@@ -111,25 +107,13 @@ static DecodeStatus DecodeFPCSCRegisterClass(MCInst &Inst, uint64_t RegNo,
 }
 #define DecodeFPICRegisterClass DecodeFPCSCRegisterClass
 
-static DecodeStatus DecodeCCRCRegisterClass(MCInst &Inst, APInt &Insn,
-                                            uint64_t Address,
-                                            const void *Decoder) {
-  llvm_unreachable("unimplemented");
-}
-
-static DecodeStatus DecodeSRCRegisterClass(MCInst &Inst, APInt &Insn,
-                                           uint64_t Address,
-                                           const void *Decoder) {
-  llvm_unreachable("unimplemented");
-}
-
 static DecodeStatus DecodeImm32(MCInst &Inst, uint64_t Imm, uint64_t Address,
                                 const void *Decoder) {
   Inst.addOperand(MCOperand::createImm(M68k::swapWord<uint32_t>(Imm)));
   return DecodeStatus::Success;
 }
 
-#include "M68kGenDisassemblerTable.inc"
+#include "M68kGenDisassemblerTables.inc"
 
 #undef DecodeFPDR32RegisterClass
 #undef DecodeFPDR64RegisterClass
diff --git a/llvm/lib/Target/M68k/M68kISelLowering.cpp b/llvm/lib/Target/M68k/M68kISelLowering.cpp
index 594ea9f48c201..f94380f27e8bd 100644
--- a/llvm/lib/Target/M68k/M68kISelLowering.cpp
+++ b/llvm/lib/Target/M68k/M68kISelLowering.cpp
@@ -51,7 +51,9 @@ M68kTargetLowering::M68kTargetLowering(const M68kTargetMachine &TM,
 
   MVT PtrVT = MVT::i32;
 
-  setBooleanContents(ZeroOrOneBooleanContent);
+  // This is based on M68k SetCC (scc) setting the destination byte to all 1s.
+  // See also getSetCCResultType().
+  setBooleanContents(ZeroOrNegativeOneBooleanContent);
 
   auto *RegInfo = Subtarget.getRegisterInfo();
   setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
@@ -1454,10 +1456,7 @@ SDValue M68kTargetLowering::getTLSGetAddr(GlobalAddressSDNode *GA,
   PointerType *PtrTy = PointerType::get(*DAG.getContext(), 0);
 
   ArgListTy Args;
-  ArgListEntry Entry;
-  Entry.Node = Arg;
-  Entry.Ty = PtrTy;
-  Args.push_back(Entry);
+  Args.emplace_back(Arg, PtrTy);
   return LowerExternalSymbolCall(DAG, SDLoc(GA), "__tls_get_addr",
                                  std::move(Args));
 }
@@ -1667,8 +1666,8 @@ static SDValue getBitTestCondition(SDValue Src, SDValue BitNo, ISD::CondCode CC,
 
   SDValue BTST = DAG.getNode(M68kISD::BTST, DL, MVT::i32, Src, BitNo);
 
-  // NOTE BTST sets CCR.Z flag
-  M68k::CondCode Cond = CC == ISD::SETEQ ? M68k::COND_NE : M68k::COND_EQ;
+  // NOTE BTST sets CCR.Z flag if bit is 0, same as AND with bitmask
+  M68k::CondCode Cond = CC == ISD::SETEQ ? M68k::COND_EQ : M68k::COND_NE;
   return DAG.getNode(M68kISD::SETCC, DL, MVT::i8,
                      DAG.getConstant(Cond, DL, MVT::i8), BTST);
 }
diff --git a/llvm/lib/Target/M68k/M68kInstrArithmetic.td b/llvm/lib/Target/M68k/M68kInstrArithmetic.td
index e2d4e49ddf27b..56b71db2d1d2f 100644
--- a/llvm/lib/Target/M68k/M68kInstrArithmetic.td
+++ b/llvm/lib/Target/M68k/M68kInstrArithmetic.td
@@ -835,7 +835,7 @@ def : Pat<(MxSub 0, i8 :$src), (NEG8d  MxDRD8 :$src)>;
 def : Pat<(MxSub 0, i16:$src), (NEG16d MxDRD16:$src)>;
 def : Pat<(MxSub 0, i32:$src), (NEG32d MxDRD32:$src)>;
 // SExt of i1 values.
-// Although we specify `ZeroOrOneBooleanContent` for boolean content,
+// Although we specify `ZeroOrNegativeOneBooleanContent` for boolean content,
 // we're still adding an AND here as we don't know the origin of the i1 value.
 def : Pat<(sext_inreg i8:$src,  i1), (NEG8d  (AND8di  MxDRD8:$src,  1))>;
 def : Pat<(sext_inreg i16:$src, i1), (NEG16d (AND16di MxDRD16:$src, 1))>;
diff --git a/llvm/lib/Target/M68k/M68kInstrAtomics.td b/llvm/lib/Target/M68k/M68kInstrAtomics.td
index 867afbefe68fe..b2b64ca853220 100644
--- a/llvm/lib/Target/M68k/M68kInstrAtomics.td
+++ b/llvm/lib/Target/M68k/M68kInstrAtomics.td
@@ -67,7 +67,8 @@ class MxCASARIDOp<bits<2> size_encoding, MxType type>
              "cas."#type.Prefix#" $dc, $du, $mem"> {
   let Inst = (ascend
                 (descend 0b00001, size_encoding, 0b011, MxEncAddrMode_p<"mem">.EA),
-                (descend 0b0000000, (operand "$du", 3), 0b000, (operand "$dc", 3))
+                (descend 0b0000000, (operand "$du", 3), 0b000, (operand "$dc", 3)),
+                MxEncAddrMode_p<"mem">.Supplement
               );
   let Constraints = "$out = $dc";
   let mayLoad = 1;
@@ -84,7 +85,8 @@ class MxCASARIIOp<bits<2> size_encoding, MxType type>
              "cas."#type.Prefix#" $dc, $du, $mem"> {
   let Inst = (ascend
                 (descend 0b00001, size_encoding, 0b011, MxEncAddrMode_f<"mem">.EA),
-                (descend 0b0000000, (operand "$du", 3), 0b000, (operand "$dc", 3))
+                (descend 0b0000000, (operand "$du", 3), 0b000, (operand "$dc", 3)),
+                MxEncAddrMode_f<"mem">.Supplement
               );
   let Constraints = "$out = $dc";
   let mayLoad = 1;
@@ -100,8 +102,9 @@ class MxCASALOp<bits<2> size_encoding, MxType type>
              (ins type.ROp:$dc, type.ROp:$du, !cast<MxMemOp>("MxAL"#type.Size):$mem),
              "cas."#type.Prefix#" $dc, $du, $mem"> {
   let Inst = (ascend
-                (descend 0b00001, size_encoding, 0b011, MxEncAddrMode_abs<"mem">.EA),
-                (descend 0b0000000, (operand "$du", 3), 0b000, (operand "$dc", 3))
+                (descend 0b00001, size_encoding, 0b011, MxEncAddrMode_abs<"mem", true>.EA),
+                (descend 0b0000000, (operand "$du", 3), 0b000, (operand "$dc", 3)),
+                MxEncAddrMode_abs<"mem", true>.Supplement
               );
   let Constraints = "$out = $dc";
   let mayLoad = 1;
diff --git a/llvm/lib/Target/M68k/M68kInstrData.td b/llvm/lib/Target/M68k/M68kInstrData.td
index f4ed62720ff99..c5b7ae332822f 100644
--- a/llvm/lib/Target/M68k/M68kInstrData.td
+++ b/llvm/lib/Target/M68k/M68kInstrData.td
@@ -701,18 +701,22 @@ def: Pat<(MxExtLoadi16i8 MxCP_ARID:$src),
           (EXTRACT_SUBREG (MOVZXd32p8 MxARID8:$src), MxSubRegIndex16Lo)>;
 def: Pat<(MxExtLoadi16i8 MxCP_ARII:$src),
           (EXTRACT_SUBREG (MOVZXd32f8 MxARII8:$src), MxSubRegIndex16Lo)>;
+def: Pat<(MxExtLoadi16i8 MxCP_PCD:$src),
+          (EXTRACT_SUBREG (MOVZXd32q8 MxPCD8:$src), MxSubRegIndex16Lo)>;
 
 // i32 <- anyext i8
 def: Pat<(i32 (anyext i8:$src)), (MOVZXd32d8 MxDRD8:$src)>;
 def: Pat<(MxExtLoadi32i8 MxCP_ARI :$src), (MOVZXd32j8 MxARI8 :$src)>;
 def: Pat<(MxExtLoadi32i8 MxCP_ARID:$src), (MOVZXd32p8 MxARID8:$src)>;
 def: Pat<(MxExtLoadi32i8 MxCP_ARII:$src), (MOVZXd32f8 MxARII8:$src)>;
+def: Pat<(MxExtLoadi32i8 MxCP_PCD:$src), (MOVZXd32q8 MxPCD8:$src)>;
 
 // i32 <- anyext i16
 def: Pat<(i32 (anyext i16:$src)), (MOVZXd32d16 MxDRD16:$src)>;
 def: Pat<(MxExtLoadi32i16 MxCP_ARI :$src), (MOVZXd32j16 MxARI16 :$src)>;
 def: Pat<(MxExtLoadi32i16 MxCP_ARID:$src), (MOVZXd32p16 MxARID16:$src)>;
 def: Pat<(MxExtLoadi32i16 MxCP_ARII:$src), (MOVZXd32f16 MxARII16:$src)>;
+def: Pat<(MxExtLoadi32i16 MxCP_PCD:$src), (MOVZXd32q16 MxPCD16:$src)>;
 
 // trunc patterns
 def : Pat<(i16 (trunc i32:$src)),
diff --git a/llvm/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp b/llvm/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp
index 4c5b473982f77..c8094a8eeb361 100644
--- a/llvm/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp
+++ b/llvm/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp
@@ -14,6 +14,7 @@
 #include "MSP430.h"
 #include "TargetInfo/MSP430TargetInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -23,6 +24,7 @@
 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "msp430-disassembler"
 
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index d23504c203dd3..6da5e66be4ad8 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -377,6 +377,7 @@ static void AnalyzeArguments(CCState &State,
   for (unsigned i = 0, e = ArgsParts.size(); i != e; i++) {
     MVT ArgVT = Args[ValNo].VT;
     ISD::ArgFlagsTy ArgFlags = Args[ValNo].Flags;
+    Type *OrigTy = Args[ValNo].OrigTy;
     MVT LocVT = ArgVT;
     CCValAssign::LocInfo LocInfo = CCValAssign::Full;
 
@@ -411,7 +412,8 @@ static void AnalyzeArguments(CCState &State,
       RegsLeft -= 1;
 
       UsedStack = true;
-      CC_MSP430_AssignStack(ValNo++, ArgVT, LocVT, LocInfo, ArgFlags, State);
+      CC_MSP430_AssignStack(ValNo++, ArgVT, LocVT, LocInfo, ArgFlags, OrigTy,
+                            State);
     } else if (Parts <= RegsLeft) {
       for (unsigned j = 0; j < Parts; j++) {
         MCRegister Reg = State.AllocateReg(RegList);
@@ -421,7 +423,8 @@ static void AnalyzeArguments(CCState &State,
     } else {
       UsedStack = true;
       for (unsigned j = 0; j < Parts; j++)
-        CC_MSP430_AssignStack(ValNo++, ArgVT, LocVT, LocInfo, ArgFlags, State);
+        CC_MSP430_AssignStack(ValNo++, ArgVT, LocVT, LocInfo, ArgFlags, OrigTy,
+                              State);
     }
   }
 }
diff --git a/llvm/lib/Target/MSP430/MSP430InstrFormats.td b/llvm/lib/Target/MSP430/MSP430InstrFormats.td
index 36f40d6fc89dd..0e784aa4ff914 100644
--- a/llvm/lib/Target/MSP430/MSP430InstrFormats.td
+++ b/llvm/lib/Target/MSP430/MSP430InstrFormats.td
@@ -31,7 +31,6 @@ def DstMem      : DestMode<1>;   // m
 // Generic MSP430 Format
 class MSP430Inst<dag outs, dag ins, int size, string asmstr> : Instruction {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   let Namespace = "MSP430";
 
diff --git a/llvm/lib/Target/Mips/Disassembler/MipsDisassembler.cpp b/llvm/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
index b3f6cd1609fbb..0c98c4da2ede1 100644
--- a/llvm/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
+++ b/llvm/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
@@ -14,6 +14,7 @@
 #include "TargetInfo/MipsTargetInfo.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -29,6 +30,7 @@
 #include <cstdint>
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "mips-disassembler"
 
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
index 16247bd732685..680d27927ba36 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
@@ -17,7 +17,6 @@
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/MCValue.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
index feeadc5e2297c..a8b7c9e57470d 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
@@ -18,7 +18,6 @@
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSymbolELF.h"
-#include "llvm/Support/Casting.h"
 
 using namespace llvm;
 
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
index 5df70c4675c0c..1e1b9703d8062 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
@@ -26,7 +26,6 @@
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbolELF.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormattedStream.h"
diff --git a/llvm/lib/Target/Mips/MicroMipsInstrFormats.td b/llvm/lib/Target/Mips/MicroMipsInstrFormats.td
index c4c48762dbea4..f848fe6e58912 100644
--- a/llvm/lib/Target/Mips/MicroMipsInstrFormats.td
+++ b/llvm/lib/Target/Mips/MicroMipsInstrFormats.td
@@ -45,7 +45,6 @@ class MicroMipsInst16<dag outs, dag ins, string asmstr, list<dag> pattern,
 {
   let Size = 2;
   field bits<16> Inst;
-  field bits<16> SoftFail = 0;
   bits<6> Opcode = 0x0;
 }
 
diff --git a/llvm/lib/Target/Mips/Mips16InstrFormats.td b/llvm/lib/Target/Mips/Mips16InstrFormats.td
index b180e53855bc4..1a4bbacb48d3e 100644
--- a/llvm/lib/Target/Mips/Mips16InstrFormats.td
+++ b/llvm/lib/Target/Mips/Mips16InstrFormats.td
@@ -62,7 +62,6 @@ class MipsInst16<dag outs, dag ins, string asmstr, list<dag> pattern,
   let Inst{15-11} = Opcode;
 
   let Size=2;
-  field bits<16> SoftFail = 0;
 }
 
 //
@@ -75,7 +74,6 @@ class MipsInst16_32<dag outs, dag ins, string asmstr, list<dag> pattern,
   field bits<32> Inst;
 
   let Size=4;
-  field bits<32> SoftFail = 0;
 }
 
 class MipsInst16_EXTEND<dag outs, dag ins, string asmstr, list<dag> pattern,
diff --git a/llvm/lib/Target/Mips/MipsAsmPrinter.cpp b/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
index 4530fc60e5e21..ae91c97e2a803 100644
--- a/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
+++ b/llvm/lib/Target/Mips/MipsAsmPrinter.cpp
@@ -51,7 +51,6 @@
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/MC/MCSymbolELF.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
diff --git a/llvm/lib/Target/Mips/MipsCCState.cpp b/llvm/lib/Target/Mips/MipsCCState.cpp
index 9e8cd2ea2fd43..86bb3e6e35f3a 100644
--- a/llvm/lib/Target/Mips/MipsCCState.cpp
+++ b/llvm/lib/Target/Mips/MipsCCState.cpp
@@ -12,59 +12,6 @@
 
 using namespace llvm;
 
-bool MipsCCState::isF128SoftLibCall(const char *CallSym) {
-  const char *const LibCalls[] = {
-      "__addtf3",      "__divtf3",     "__eqtf2",       "__extenddftf2",
-      "__extendsftf2", "__fixtfdi",    "__fixtfsi",     "__fixtfti",
-      "__fixunstfdi",  "__fixunstfsi", "__fixunstfti",  "__floatditf",
-      "__floatsitf",   "__floattitf",  "__floatunditf", "__floatunsitf",
-      "__floatuntitf", "__getf2",      "__gttf2",       "__letf2",
-      "__lttf2",       "__multf3",     "__netf2",       "__powitf2",
-      "__subtf3",      "__trunctfdf2", "__trunctfsf2",  "__unordtf2",
-      "ceill",         "copysignl",    "cosl",          "exp2l",
-      "expl",          "floorl",       "fmal",          "fmaxl",
-      "fmodl",         "log10l",       "log2l",         "logl",
-      "nearbyintl",    "powl",         "rintl",         "roundl",
-      "sinl",          "sqrtl",        "truncl"};
-
-  // Check that LibCalls is sorted alphabetically.
-  auto Comp = [](const char *S1, const char *S2) { return strcmp(S1, S2) < 0; };
-  assert(llvm::is_sorted(LibCalls, Comp));
-  return llvm::binary_search(LibCalls, CallSym, Comp);
-}
-
-/// This function returns true if Ty is fp128, {f128} or i128 which was
-/// originally a fp128.
-bool MipsCCState::originalTypeIsF128(const Type *Ty, const char *Func) {
-  if (Ty->isFP128Ty())
-    return true;
-
-  if (Ty->isStructTy() && Ty->getStructNumElements() == 1 &&
-      Ty->getStructElementType(0)->isFP128Ty())
-    return true;
-
-  // If the Ty is i128 and the function being called is a long double emulation
-  // routine, then the original type is f128.
-  // FIXME: This is unsound because these functions could be indirectly called
-  return (Func && Ty->isIntegerTy(128) && isF128SoftLibCall(Func));
-}
-
-/// Return true if the original type was vXfXX.
-bool MipsCCState::originalEVTTypeIsVectorFloat(EVT Ty) {
-  if (Ty.isVector() && Ty.getVectorElementType().isFloatingPoint())
-    return true;
-
-  return false;
-}
-
-/// Return true if the original type was vXfXX / vXfXX.
-bool MipsCCState::originalTypeIsVectorFloat(const Type *Ty) {
-  if (Ty->isVectorTy() && Ty->isFPOrFPVectorTy())
-    return true;
-
-  return false;
-}
-
 MipsCCState::SpecialCallingConvType
 MipsCCState::getSpecialCallingConvForCallee(const SDNode *Callee,
                                             const MipsSubtarget &Subtarget) {
@@ -81,126 +28,3 @@ MipsCCState::getSpecialCallingConvForCallee(const SDNode *Callee,
   }
   return SpecialCallingConv;
 }
-
-void MipsCCState::PreAnalyzeCallResultForF128(
-    const SmallVectorImpl<ISD::InputArg> &Ins,
-    const Type *RetTy, const char *Call) {
-  for (unsigned i = 0; i < Ins.size(); ++i) {
-    OriginalArgWasF128.push_back(
-        originalTypeIsF128(RetTy, Call));
-    OriginalArgWasFloat.push_back(RetTy->isFloatingPointTy());
-  }
-}
-
-/// Identify lowered values that originated from f128 or float arguments and
-/// record this for use by RetCC_MipsN.
-void MipsCCState::PreAnalyzeCallReturnForF128(
-    const SmallVectorImpl<ISD::OutputArg> &Outs, const Type *RetTy) {
-  for (unsigned i = 0; i < Outs.size(); ++i) {
-    OriginalArgWasF128.push_back(
-        originalTypeIsF128(RetTy, nullptr));
-    OriginalArgWasFloat.push_back(
-        RetTy->isFloatingPointTy());
-  }
-}
-
-/// Identify lower values that originated from vXfXX and record
-/// this.
-void MipsCCState::PreAnalyzeCallResultForVectorFloat(
-    const SmallVectorImpl<ISD::InputArg> &Ins, const Type *RetTy) {
-  for (unsigned i = 0; i < Ins.size(); ++i) {
-    OriginalRetWasFloatVector.push_back(originalTypeIsVectorFloat(RetTy));
-  }
-}
-
-/// Identify lowered values that originated from vXfXX arguments and record
-/// this.
-void MipsCCState::PreAnalyzeReturnForVectorFloat(
-    const SmallVectorImpl<ISD::OutputArg> &Outs) {
-  for (unsigned i = 0; i < Outs.size(); ++i) {
-    ISD::OutputArg Out = Outs[i];
-    OriginalRetWasFloatVector.push_back(
-        originalEVTTypeIsVectorFloat(Out.ArgVT));
-  }
-}
-
-void MipsCCState::PreAnalyzeReturnValue(EVT ArgVT) {
-  OriginalRetWasFloatVector.push_back(originalEVTTypeIsVectorFloat(ArgVT));
-}
-
-void MipsCCState::PreAnalyzeCallOperand(const Type *ArgTy, bool IsFixed,
-                                        const char *Func) {
-  OriginalArgWasF128.push_back(originalTypeIsF128(ArgTy, Func));
-  OriginalArgWasFloat.push_back(ArgTy->isFloatingPointTy());
-  OriginalArgWasFloatVector.push_back(ArgTy->isVectorTy());
-  CallOperandIsFixed.push_back(IsFixed);
-}
-
-/// Identify lowered values that originated from f128, float and sret to vXfXX
-/// arguments and record this.
-void MipsCCState::PreAnalyzeCallOperands(
-    const SmallVectorImpl<ISD::OutputArg> &Outs,
-    std::vector<TargetLowering::ArgListEntry> &FuncArgs,
-    const char *Func) {
-  for (unsigned i = 0; i < Outs.size(); ++i) {
-    TargetLowering::ArgListEntry FuncArg = FuncArgs[Outs[i].OrigArgIndex];
-
-    OriginalArgWasF128.push_back(originalTypeIsF128(FuncArg.Ty, Func));
-    OriginalArgWasFloat.push_back(FuncArg.Ty->isFloatingPointTy());
-    OriginalArgWasFloatVector.push_back(FuncArg.Ty->isVectorTy());
-    CallOperandIsFixed.push_back(Outs[i].IsFixed);
-  }
-}
-
-void MipsCCState::PreAnalyzeFormalArgument(const Type *ArgTy,
-                                           ISD::ArgFlagsTy Flags) {
-  // SRet arguments cannot originate from f128 or {f128} returns so we just
-  // push false. We have to handle this specially since SRet arguments
-  // aren't mapped to an original argument.
-  if (Flags.isSRet()) {
-    OriginalArgWasF128.push_back(false);
-    OriginalArgWasFloat.push_back(false);
-    OriginalArgWasFloatVector.push_back(false);
-    return;
-  }
-
-  OriginalArgWasF128.push_back(originalTypeIsF128(ArgTy, nullptr));
-  OriginalArgWasFloat.push_back(ArgTy->isFloatingPointTy());
-
-  // The MIPS vector ABI exhibits a corner case of sorts or quirk; if the
-  // first argument is actually an SRet pointer to a vector, then the next
-  // argument slot is $a2.
-  OriginalArgWasFloatVector.push_back(ArgTy->isVectorTy());
-}
-
-/// Identify lowered values that originated from f128, float and vXfXX arguments
-/// and record this.
-void MipsCCState::PreAnalyzeFormalArgumentsForF128(
-    const SmallVectorImpl<ISD::InputArg> &Ins) {
-  const MachineFunction &MF = getMachineFunction();
-  for (unsigned i = 0; i < Ins.size(); ++i) {
-    Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
-
-    // SRet arguments cannot originate from f128 or {f128} returns so we just
-    // push false. We have to handle this specially since SRet arguments
-    // aren't mapped to an original argument.
-    if (Ins[i].Flags.isSRet()) {
-      OriginalArgWasF128.push_back(false);
-      OriginalArgWasFloat.push_back(false);
-      OriginalArgWasFloatVector.push_back(false);
-      continue;
-    }
-
-    assert(Ins[i].getOrigArgIndex() < MF.getFunction().arg_size());
-    std::advance(FuncArg, Ins[i].getOrigArgIndex());
-
-    OriginalArgWasF128.push_back(
-        originalTypeIsF128(FuncArg->getType(), nullptr));
-    OriginalArgWasFloat.push_back(FuncArg->getType()->isFloatingPointTy());
-
-    // The MIPS vector ABI exhibits a corner case of sorts or quirk; if the
-    // first argument is actually an SRet pointer to a vector, then the next
-    // argument slot is $a2.
-    OriginalArgWasFloatVector.push_back(FuncArg->getType()->isVectorTy());
-  }
-}
diff --git a/llvm/lib/Target/Mips/MipsCCState.h b/llvm/lib/Target/Mips/MipsCCState.h
index 4229da564630d..4c36d42589d7a 100644
--- a/llvm/lib/Target/Mips/MipsCCState.h
+++ b/llvm/lib/Target/Mips/MipsCCState.h
@@ -26,70 +26,7 @@ class MipsCCState : public CCState {
   getSpecialCallingConvForCallee(const SDNode *Callee,
                                  const MipsSubtarget &Subtarget);
 
-  /// This function returns true if CallSym is a long double emulation routine.
-  ///
-  /// FIXME: Changing the ABI based on the callee name is unsound. The lib func
-  /// address could be captured.
-  static bool isF128SoftLibCall(const char *CallSym);
-
-  static bool originalTypeIsF128(const Type *Ty, const char *Func);
-  static bool originalEVTTypeIsVectorFloat(EVT Ty);
-  static bool originalTypeIsVectorFloat(const Type *Ty);
-
-  void PreAnalyzeCallOperand(const Type *ArgTy, bool IsFixed, const char *Func);
-
-  void PreAnalyzeFormalArgument(const Type *ArgTy, ISD::ArgFlagsTy Flags);
-  void PreAnalyzeReturnValue(EVT ArgVT);
-
 private:
-  /// Identify lowered values that originated from f128 arguments and record
-  /// this for use by RetCC_MipsN.
-  void PreAnalyzeCallResultForF128(const SmallVectorImpl<ISD::InputArg> &Ins,
-                                   const Type *RetTy, const char * Func);
-
-  /// Identify lowered values that originated from f128 arguments and record
-  /// this for use by RetCC_MipsN.
-  void PreAnalyzeCallReturnForF128(const SmallVectorImpl<ISD::OutputArg> &Outs, const Type *RetTy);
-
-  /// Identify lowered values that originated from f128 arguments and record
-  /// this.
-  void
-  PreAnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
-                         std::vector<TargetLowering::ArgListEntry> &FuncArgs,
-                         const char *Func);
-
-  /// Identify lowered values that originated from f128 arguments and record
-  /// this for use by RetCC_MipsN.
-  void
-  PreAnalyzeFormalArgumentsForF128(const SmallVectorImpl<ISD::InputArg> &Ins);
-
-  void
-  PreAnalyzeCallResultForVectorFloat(const SmallVectorImpl<ISD::InputArg> &Ins,
-                                     const Type *RetTy);
-
-  void PreAnalyzeFormalArgumentsForVectorFloat(
-      const SmallVectorImpl<ISD::InputArg> &Ins);
-
-  void
-  PreAnalyzeReturnForVectorFloat(const SmallVectorImpl<ISD::OutputArg> &Outs);
-
-  /// Records whether the value has been lowered from an f128.
-  SmallVector<bool, 4> OriginalArgWasF128;
-
-  /// Records whether the value has been lowered from float.
-  SmallVector<bool, 4> OriginalArgWasFloat;
-
-  /// Records whether the value has been lowered from a floating point vector.
-  SmallVector<bool, 4> OriginalArgWasFloatVector;
-
-  /// Records whether the return value has been lowered from a floating point
-  /// vector.
-  SmallVector<bool, 4> OriginalRetWasFloatVector;
-
-  /// Records whether the value was a fixed argument.
-  /// See ISD::OutputArg::IsFixed,
-  SmallVector<bool, 4> CallOperandIsFixed;
-
   // Used to handle MIPS16-specific calling convention tweaks.
   // FIXME: This should probably be a fully fledged calling convention.
   SpecialCallingConvType SpecialCallingConv;
@@ -100,120 +37,6 @@ class MipsCCState : public CCState {
               SpecialCallingConvType SpecialCC = NoSpecialCallingConv)
       : CCState(CC, isVarArg, MF, locs, C), SpecialCallingConv(SpecialCC) {}
 
-  void PreAnalyzeCallOperands(
-      const SmallVectorImpl<ISD::OutputArg> &Outs, CCAssignFn Fn,
-      std::vector<TargetLowering::ArgListEntry> &FuncArgs, const char *Func) {
-    OriginalArgWasF128.clear();
-    OriginalArgWasFloat.clear();
-    OriginalArgWasFloatVector.clear();
-    CallOperandIsFixed.clear();
-    PreAnalyzeCallOperands(Outs, FuncArgs, Func);
-  }
-
-  void
-  AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
-                      CCAssignFn Fn,
-                      std::vector<TargetLowering::ArgListEntry> &FuncArgs,
-                      const char *Func) {
-    PreAnalyzeCallOperands(Outs, Fn, FuncArgs, Func);
-    CCState::AnalyzeCallOperands(Outs, Fn);
-  }
-
-  // The AnalyzeCallOperands in the base class is not usable since we must
-  // provide a means of accessing ArgListEntry::IsFixed. Delete them from this
-  // class. This doesn't stop them being used via the base class though.
-  void AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
-                           CCAssignFn Fn) = delete;
-  void AnalyzeCallOperands(const SmallVectorImpl<MVT> &Outs,
-                           SmallVectorImpl<ISD::ArgFlagsTy> &Flags,
-                           CCAssignFn Fn) = delete;
-
-  void PreAnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins,
-                                 CCAssignFn Fn) {
-    OriginalArgWasFloat.clear();
-    OriginalArgWasF128.clear();
-    OriginalArgWasFloatVector.clear();
-    PreAnalyzeFormalArgumentsForF128(Ins);
-  }
-
-  void AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins,
-                              CCAssignFn Fn) {
-    PreAnalyzeFormalArguments(Ins, Fn);
-    CCState::AnalyzeFormalArguments(Ins, Fn);
-  }
-
-  void PreAnalyzeCallResult(const Type *RetTy, const char *Func) {
-    OriginalArgWasF128.push_back(originalTypeIsF128(RetTy, Func));
-    OriginalArgWasFloat.push_back(RetTy->isFloatingPointTy());
-    OriginalRetWasFloatVector.push_back(originalTypeIsVectorFloat(RetTy));
-  }
-
-  void PreAnalyzeCallResult(const SmallVectorImpl<ISD::InputArg> &Ins,
-                            CCAssignFn Fn, const Type *RetTy,
-                            const char *Func) {
-    OriginalArgWasFloat.clear();
-    OriginalArgWasF128.clear();
-    OriginalArgWasFloatVector.clear();
-    PreAnalyzeCallResultForF128(Ins, RetTy, Func);
-    PreAnalyzeCallResultForVectorFloat(Ins, RetTy);
-  }
-
-  void AnalyzeCallResult(const SmallVectorImpl<ISD::InputArg> &Ins,
-                         CCAssignFn Fn, const Type *RetTy,
-                         const char *Func) {
-    PreAnalyzeCallResult(Ins, Fn, RetTy, Func);
-    CCState::AnalyzeCallResult(Ins, Fn);
-  }
-
-  void PreAnalyzeReturn(const SmallVectorImpl<ISD::OutputArg> &Outs,
-                        CCAssignFn Fn) {
-    const MachineFunction &MF = getMachineFunction();
-    OriginalArgWasFloat.clear();
-    OriginalArgWasF128.clear();
-    OriginalArgWasFloatVector.clear();
-    PreAnalyzeCallReturnForF128(Outs, MF.getFunction().getReturnType());
-    PreAnalyzeReturnForVectorFloat(Outs);
-  }
-
-  void AnalyzeReturn(const SmallVectorImpl<ISD::OutputArg> &Outs,
-                     CCAssignFn Fn) {
-    PreAnalyzeReturn(Outs, Fn);
-    CCState::AnalyzeReturn(Outs, Fn);
-  }
-
-  bool CheckReturn(const SmallVectorImpl<ISD::OutputArg> &ArgsFlags,
-                   CCAssignFn Fn) {
-    const MachineFunction &MF = getMachineFunction();
-    PreAnalyzeCallReturnForF128(ArgsFlags, MF.getFunction().getReturnType());
-    PreAnalyzeReturnForVectorFloat(ArgsFlags);
-    bool Return = CCState::CheckReturn(ArgsFlags, Fn);
-    OriginalArgWasFloat.clear();
-    OriginalArgWasF128.clear();
-    OriginalArgWasFloatVector.clear();
-    return Return;
-  }
-
-  bool CheckCallReturn(const SmallVectorImpl<ISD::OutputArg> &ArgsFlags,
-                   CCAssignFn Fn, const Type *RetTy) {
-    PreAnalyzeCallReturnForF128(ArgsFlags, RetTy);
-    PreAnalyzeReturnForVectorFloat(ArgsFlags);
-    bool Return = CCState::CheckReturn(ArgsFlags, Fn);
-    OriginalArgWasFloat.clear();
-    OriginalArgWasF128.clear();
-    OriginalArgWasFloatVector.clear();
-    return Return;
-  }
-  bool WasOriginalArgF128(unsigned ValNo) { return OriginalArgWasF128[ValNo]; }
-  bool WasOriginalArgFloat(unsigned ValNo) {
-      return OriginalArgWasFloat[ValNo];
-  }
-  bool WasOriginalArgVectorFloat(unsigned ValNo) const {
-    return OriginalArgWasFloatVector[ValNo];
-  }
-  bool WasOriginalRetVectorFloat(unsigned ValNo) const {
-    return OriginalRetWasFloatVector[ValNo];
-  }
-  bool IsCallOperandFixed(unsigned ValNo) { return CallOperandIsFixed[ValNo]; }
   SpecialCallingConvType getSpecialCallingConv() { return SpecialCallingConv; }
 };
 }
diff --git a/llvm/lib/Target/Mips/MipsCallLowering.cpp b/llvm/lib/Target/Mips/MipsCallLowering.cpp
index 555773adf52aa..35194e7e6ba23 100644
--- a/llvm/lib/Target/Mips/MipsCallLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsCallLowering.cpp
@@ -26,62 +26,6 @@ MipsCallLowering::MipsCallLowering(const MipsTargetLowering &TLI)
     : CallLowering(&TLI) {}
 
 namespace {
-struct MipsOutgoingValueAssigner : public CallLowering::OutgoingValueAssigner {
-  /// This is the name of the function being called
-  /// FIXME: Relying on this is unsound
-  const char *Func = nullptr;
-
-  /// Is this a return value, or an outgoing call operand.
-  bool IsReturn;
-
-  MipsOutgoingValueAssigner(CCAssignFn *AssignFn_, const char *Func,
-                            bool IsReturn)
-      : OutgoingValueAssigner(AssignFn_), Func(Func), IsReturn(IsReturn) {}
-
-  bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
-                 CCValAssign::LocInfo LocInfo,
-                 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
-                 CCState &State_) override {
-    MipsCCState &State = static_cast<MipsCCState &>(State_);
-
-    if (IsReturn)
-      State.PreAnalyzeReturnValue(EVT::getEVT(Info.Ty));
-    else
-      State.PreAnalyzeCallOperand(Info.Ty, Info.IsFixed, Func);
-
-    return CallLowering::OutgoingValueAssigner::assignArg(
-        ValNo, OrigVT, ValVT, LocVT, LocInfo, Info, Flags, State);
-  }
-};
-
-struct MipsIncomingValueAssigner : public CallLowering::IncomingValueAssigner {
-  /// This is the name of the function being called
-  /// FIXME: Relying on this is unsound
-  const char *Func = nullptr;
-
-  /// Is this a call return value, or an incoming function argument.
-  bool IsReturn;
-
-  MipsIncomingValueAssigner(CCAssignFn *AssignFn_, const char *Func,
-                            bool IsReturn)
-      : IncomingValueAssigner(AssignFn_), Func(Func), IsReturn(IsReturn) {}
-
-  bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
-                 CCValAssign::LocInfo LocInfo,
-                 const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
-                 CCState &State_) override {
-    MipsCCState &State = static_cast<MipsCCState &>(State_);
-
-    if (IsReturn)
-      State.PreAnalyzeCallResult(Info.Ty, Func);
-    else
-      State.PreAnalyzeFormalArgument(Info.Ty, Flags);
-
-    return CallLowering::IncomingValueAssigner::assignArg(
-        ValNo, OrigVT, ValVT, LocVT, LocInfo, Info, Flags, State);
-  }
-};
-
 class MipsIncomingValueHandler : public CallLowering::IncomingValueHandler {
   const MipsSubtarget &STI;
 
@@ -339,9 +283,7 @@ bool MipsCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
                        F.getContext());
 
     MipsOutgoingValueHandler RetHandler(MIRBuilder, MF.getRegInfo(), Ret);
-    std::string FuncName = F.getName().str();
-    MipsOutgoingValueAssigner Assigner(TLI.CCAssignFnForReturn(),
-                                       FuncName.c_str(), /*IsReturn*/ true);
+    OutgoingValueAssigner Assigner(TLI.CCAssignFnForReturn());
 
     if (!determineAssignments(Assigner, RetInfos, CCInfo))
       return false;
@@ -392,9 +334,7 @@ bool MipsCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
   CCInfo.AllocateStack(ABI.GetCalleeAllocdArgSizeInBytes(F.getCallingConv()),
                        Align(1));
 
-  const std::string FuncName = F.getName().str();
-  MipsIncomingValueAssigner Assigner(TLI.CCAssignFnForCall(), FuncName.c_str(),
-                                     /*IsReturn*/ false);
+  IncomingValueAssigner Assigner(TLI.CCAssignFnForCall());
   if (!determineAssignments(Assigner, ArgInfos, CCInfo))
     return false;
 
@@ -510,11 +450,7 @@ bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   CCInfo.AllocateStack(ABI.GetCalleeAllocdArgSizeInBytes(Info.CallConv),
                        Align(1));
 
-  const char *Call =
-      Info.Callee.isSymbol() ? Info.Callee.getSymbolName() : nullptr;
-
-  MipsOutgoingValueAssigner Assigner(TLI.CCAssignFnForCall(), Call,
-                                     /*IsReturn*/ false);
+  OutgoingValueAssigner Assigner(TLI.CCAssignFnForCall());
   if (!determineAssignments(Assigner, ArgInfos, CCInfo))
     return false;
 
@@ -550,11 +486,8 @@ bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
     CallLowering::splitToValueTypes(Info.OrigRet, ArgInfos, DL,
                                     F.getCallingConv());
 
-    const std::string FuncName = F.getName().str();
     SmallVector<CCValAssign, 8> ArgLocs;
-    MipsIncomingValueAssigner Assigner(TLI.CCAssignFnForReturn(),
-                                       FuncName.c_str(),
-                                       /*IsReturn*/ true);
+    IncomingValueAssigner Assigner(TLI.CCAssignFnForReturn());
     CallReturnHandler RetHandler(MIRBuilder, MF.getRegInfo(), MIB);
 
     MipsCCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs,
diff --git a/llvm/lib/Target/Mips/MipsCallingConv.td b/llvm/lib/Target/Mips/MipsCallingConv.td
index 39e184a6303a5..3501f9fbfd2e7 100644
--- a/llvm/lib/Target/Mips/MipsCallingConv.td
+++ b/llvm/lib/Target/Mips/MipsCallingConv.td
@@ -20,25 +20,15 @@ class CCIfSubtargetNot<string F, CCAction A> : CCIfSubtarget<F, A, "!">;
 
 /// Match if the original argument (before lowering) was a float.
 /// For example, this is true for i32's that were lowered from soft-float.
-class CCIfOrigArgWasFloat<CCAction A>
-    : CCIf<"static_cast<MipsCCState *>(&State)->WasOriginalArgFloat(ValNo)",
-           A>;
+class CCIfOrigArgWasFloat<CCAction A> : CCIf<"OrigTy->isFloatingPointTy()", A>;
 
 /// Match if the original argument (before lowering) was a 128-bit float (i.e.
 /// long double).
-class CCIfOrigArgWasF128<CCAction A>
-    : CCIf<"static_cast<MipsCCState *>(&State)->WasOriginalArgF128(ValNo)", A>;
-
-/// Match if this specific argument is a vararg.
-/// This is slightly different fro CCIfIsVarArg which matches if any argument is
-/// a vararg.
-class CCIfArgIsVarArg<CCAction A>
-    : CCIf<"!static_cast<MipsCCState *>(&State)->IsCallOperandFixed(ValNo)", A>;
+class CCIfOrigArgWasF128<CCAction A> : CCIf<"OrigTy->isFP128Ty()", A>;
 
-/// Match if the return was a floating point vector.
+/// Match if the return was not a floating point vector.
 class CCIfOrigArgWasNotVectorFloat<CCAction A>
-    : CCIf<"!static_cast<MipsCCState *>(&State)"
-                "->WasOriginalRetVectorFloat(ValNo)", A>;
+    : CCIf<"!OrigTy->isVectorTy() || !OrigTy->isFPOrFPVectorTy()", A>;
 
 /// Match if the special calling conv is the specified value.
 class CCIfSpecialCallingConv<string CC, CCAction A>
@@ -344,7 +334,7 @@ def CC_Mips_VarArg : CallingConv<[
 ]>;
 
 def CC_Mips : CallingConv<[
-  CCIfVarArg<CCIfArgIsVarArg<CCDelegateTo<CC_Mips_VarArg>>>,
+  CCIfVarArg<CCIfArgVarArg<CCDelegateTo<CC_Mips_VarArg>>>,
   CCDelegateTo<CC_Mips_FixedArg>
 ]>;
 
diff --git a/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp b/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp
index 8067dbc54170b..8699807b6bf2b 100644
--- a/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp
+++ b/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp
@@ -24,7 +24,6 @@
 #include "MipsMachineFunction.h"
 #include "MipsSubtarget.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
@@ -232,7 +231,7 @@ namespace {
 
     /// NewWaterList - The subset of WaterList that was created since the
     /// previous iteration by inserting unconditional branches.
-    SmallSet<MachineBasicBlock*, 4> NewWaterList;
+    SmallPtrSet<MachineBasicBlock *, 4> NewWaterList;
 
     using water_iterator = std::vector<MachineBasicBlock *>::iterator;
 
diff --git a/llvm/lib/Target/Mips/MipsFastISel.cpp b/llvm/lib/Target/Mips/MipsFastISel.cpp
index f3812d185ec92..1ce8d7e3e2339 100644
--- a/llvm/lib/Target/Mips/MipsFastISel.cpp
+++ b/llvm/lib/Target/Mips/MipsFastISel.cpp
@@ -266,17 +266,19 @@ class MipsFastISel final : public FastISel {
 
 static bool CC_Mips(unsigned ValNo, MVT ValVT, MVT LocVT,
                     CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                    CCState &State) LLVM_ATTRIBUTE_UNUSED;
+                    Type *OrigTy, CCState &State) LLVM_ATTRIBUTE_UNUSED;
 
 static bool CC_MipsO32_FP32(unsigned ValNo, MVT ValVT, MVT LocVT,
                             CCValAssign::LocInfo LocInfo,
-                            ISD::ArgFlagsTy ArgFlags, CCState &State) {
+                            ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                            CCState &State) {
   llvm_unreachable("should not be called");
 }
 
 static bool CC_MipsO32_FP64(unsigned ValNo, MVT ValVT, MVT LocVT,
                             CCValAssign::LocInfo LocInfo,
-                            ISD::ArgFlagsTy ArgFlags, CCState &State) {
+                            ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                            CCState &State) {
   llvm_unreachable("should not be called");
 }
 
@@ -1144,8 +1146,12 @@ bool MipsFastISel::processCallArgs(CallLoweringInfo &CLI,
                                    unsigned &NumBytes) {
   CallingConv::ID CC = CLI.CallConv;
   SmallVector<CCValAssign, 16> ArgLocs;
+  SmallVector<Type *, 16> ArgTys;
+  for (const ArgListEntry &Arg : CLI.Args)
+    ArgTys.push_back(Arg.Val->getType());
   CCState CCInfo(CC, false, *FuncInfo.MF, ArgLocs, *Context);
-  CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, CCAssignFnForCall(CC));
+  CCInfo.AnalyzeCallOperands(OutVTs, CLI.OutFlags, ArgTys,
+                             CCAssignFnForCall(CC));
   // Get a count of how many bytes are to be pushed on the stack.
   NumBytes = CCInfo.getStackSize();
   // This is the minimum argument area used for A0-A3.
@@ -1287,9 +1293,7 @@ bool MipsFastISel::finishCall(CallLoweringInfo &CLI, MVT RetVT,
     SmallVector<CCValAssign, 16> RVLocs;
     MipsCCState CCInfo(CC, false, *FuncInfo.MF, RVLocs, *Context);
 
-    CCInfo.AnalyzeCallResult(CLI.Ins, RetCC_Mips, CLI.RetTy,
-                             CLI.Symbol ? CLI.Symbol->getName().data()
-                                        : nullptr);
+    CCInfo.AnalyzeCallResult(CLI.Ins, RetCC_Mips);
 
     // Only handle a single return value.
     if (RVLocs.size() != 1)
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 881ba8e2f9eff..1491300e37d3e 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -2325,10 +2325,7 @@ lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const
     SDValue TlsGetAddr = DAG.getExternalSymbol("__tls_get_addr", PtrVT);
 
     ArgListTy Args;
-    ArgListEntry Entry;
-    Entry.Node = Argument;
-    Entry.Ty = PtrTy;
-    Args.push_back(Entry);
+    Args.emplace_back(Argument, PtrTy);
 
     TargetLowering::CallLoweringInfo CLI(DAG);
     CLI.setDebugLoc(DL)
@@ -3040,14 +3037,13 @@ SDValue MipsTargetLowering::lowerFP_TO_SINT(SDValue Op,
 
 static bool CC_MipsO32(unsigned ValNo, MVT ValVT, MVT LocVT,
                        CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                       CCState &State, ArrayRef<MCPhysReg> F64Regs) {
+                       Type *OrigTy, CCState &State,
+                       ArrayRef<MCPhysReg> F64Regs) {
   const MipsSubtarget &Subtarget = static_cast<const MipsSubtarget &>(
       State.getMachineFunction().getSubtarget());
 
   static const MCPhysReg IntRegs[] = { Mips::A0, Mips::A1, Mips::A2, Mips::A3 };
 
-  const MipsCCState * MipsState = static_cast<MipsCCState *>(&State);
-
   static const MCPhysReg F32Regs[] = { Mips::F12, Mips::F14 };
 
   static const MCPhysReg FloatVectorIntRegs[] = { Mips::A0, Mips::A2 };
@@ -3089,7 +3085,7 @@ static bool CC_MipsO32(unsigned ValNo, MVT ValVT, MVT LocVT,
                                 State.getFirstUnallocated(F32Regs) != ValNo;
   Align OrigAlign = ArgFlags.getNonZeroOrigAlign();
   bool isI64 = (ValVT == MVT::i32 && OrigAlign == Align(8));
-  bool isVectorFloat = MipsState->WasOriginalArgVectorFloat(ValNo);
+  bool isVectorFloat = OrigTy->isVectorTy() && OrigTy->isFPOrFPVectorTy();
 
   // The MIPS vector ABI for floats passes them in a pair of registers
   if (ValVT == MVT::i32 && isVectorFloat) {
@@ -3160,25 +3156,29 @@ static bool CC_MipsO32(unsigned ValNo, MVT ValVT, MVT LocVT,
   return false;
 }
 
-static bool CC_MipsO32_FP32(unsigned ValNo, MVT ValVT,
-                            MVT LocVT, CCValAssign::LocInfo LocInfo,
-                            ISD::ArgFlagsTy ArgFlags, CCState &State) {
+static bool CC_MipsO32_FP32(unsigned ValNo, MVT ValVT, MVT LocVT,
+                            CCValAssign::LocInfo LocInfo,
+                            ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                            CCState &State) {
   static const MCPhysReg F64Regs[] = { Mips::D6, Mips::D7 };
 
-  return CC_MipsO32(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State, F64Regs);
+  return CC_MipsO32(ValNo, ValVT, LocVT, LocInfo, ArgFlags, OrigTy, State,
+                    F64Regs);
 }
 
-static bool CC_MipsO32_FP64(unsigned ValNo, MVT ValVT,
-                            MVT LocVT, CCValAssign::LocInfo LocInfo,
-                            ISD::ArgFlagsTy ArgFlags, CCState &State) {
+static bool CC_MipsO32_FP64(unsigned ValNo, MVT ValVT, MVT LocVT,
+                            CCValAssign::LocInfo LocInfo,
+                            ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                            CCState &State) {
   static const MCPhysReg F64Regs[] = { Mips::D12_64, Mips::D14_64 };
 
-  return CC_MipsO32(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State, F64Regs);
+  return CC_MipsO32(ValNo, ValVT, LocVT, LocInfo, ArgFlags, OrigTy, State,
+                    F64Regs);
 }
 
 static bool CC_MipsO32(unsigned ValNo, MVT ValVT, MVT LocVT,
                        CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                       CCState &State) LLVM_ATTRIBUTE_UNUSED;
+                       Type *OrigTy, CCState &State) LLVM_ATTRIBUTE_UNUSED;
 
 #include "MipsGenCallingConv.inc"
 
@@ -3392,8 +3392,7 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       MemcpyInByVal ? 0 : ABI.GetCalleeAllocdArgSizeInBytes(CallConv);
   CCInfo.AllocateStack(ReservedArgArea, Align(1));
 
-  CCInfo.AnalyzeCallOperands(Outs, CC_Mips, CLI.getArgs(),
-                             ES ? ES->getSymbol() : nullptr);
+  CCInfo.AnalyzeCallOperands(Outs, CC_Mips);
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned StackSize = CCInfo.getStackSize();
@@ -3688,10 +3687,7 @@ SDValue MipsTargetLowering::LowerCallResult(
   MipsCCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
                      *DAG.getContext());
 
-  const ExternalSymbolSDNode *ES =
-      dyn_cast_or_null<const ExternalSymbolSDNode>(CLI.Callee.getNode());
-  CCInfo.AnalyzeCallResult(Ins, RetCC_Mips, CLI.RetTy,
-                           ES ? ES->getSymbol() : nullptr);
+  CCInfo.AnalyzeCallResult(Ins, RetCC_Mips);
 
   // Copy all of the result registers out of their specified physreg.
   for (unsigned i = 0; i != RVLocs.size(); ++i) {
@@ -3969,7 +3965,7 @@ MipsTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
                                    LLVMContext &Context, const Type *RetTy) const {
   SmallVector<CCValAssign, 16> RVLocs;
   MipsCCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
-  return CCInfo.CheckCallReturn(Outs, RetCC_Mips, RetTy);
+  return CCInfo.CheckReturn(Outs, RetCC_Mips);
 }
 
 bool MipsTargetLowering::shouldSignExtendTypeInLibCall(Type *Ty,
@@ -4408,7 +4404,7 @@ void MipsTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   case 'K': // unsigned 16 bit immediate
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
       EVT Type = Op.getValueType();
-      uint64_t Val = (uint64_t)C->getZExtValue();
+      uint64_t Val = C->getZExtValue();
       if (isUInt<16>(Val)) {
         Result = DAG.getTargetConstant(Val, DL, Type);
         break;
diff --git a/llvm/lib/Target/Mips/MipsInstrFormats.td b/llvm/lib/Target/Mips/MipsInstrFormats.td
index 10529c7d9e192..992b3ae11a178 100644
--- a/llvm/lib/Target/Mips/MipsInstrFormats.td
+++ b/llvm/lib/Target/Mips/MipsInstrFormats.td
@@ -107,8 +107,6 @@ class MipsInst<dag outs, dag ins, string asmstr, list<dag> pattern,
   let TSFlags{6}     = hasFCCRegOperand;
 
   let DecoderNamespace = "Mips";
-
-  field bits<32> SoftFail = 0;
 }
 
 // Mips32/64 Instruction Format
diff --git a/llvm/lib/Target/NVPTX/NVPTX.td b/llvm/lib/Target/NVPTX/NVPTX.td
index 83992606bc419..8a445f82e7001 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.td
+++ b/llvm/lib/Target/NVPTX/NVPTX.td
@@ -97,10 +97,10 @@ foreach sm = [20, 21, 30, 32, 35, 37, 50, 52, 53,
     def SM#sm#a : FeatureSM<""#sm#"a", !add(!mul(sm, 10), 3)>;
 }
 
-foreach version = [32, 40, 41, 42, 43, 50, 60, 61, 62, 63, 64, 65,
-                   70, 71, 72, 73, 74, 75, 76, 77, 78,
-                   80, 81, 82, 83, 84, 85, 86, 87, 88] in
-  def PTX#version: FeaturePTX<version>;
+foreach version = [32, 40, 41, 42, 43, 50, 60, 61, 62, 63, 64, 65, 70, 71, 72,
+                   73, 74, 75, 76, 77, 78, 80, 81, 82, 83, 84, 85, 86, 87, 88,
+                   90] in
+  def PTX#version : FeaturePTX<version>;
 
 //===----------------------------------------------------------------------===//
 // NVPTX supported processors.
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 38912a7f09e30..7391c2d488b57 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -436,9 +436,13 @@ void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F,
 
   if (STI->getSmVersion() >= 90) {
     const auto ClusterDim = getClusterDim(F);
+    const bool BlocksAreClusters = hasBlocksAreClusters(F);
 
     if (!ClusterDim.empty()) {
-      O << ".explicitcluster\n";
+
+      if (!BlocksAreClusters)
+        O << ".explicitcluster\n";
+
       if (ClusterDim[0] != 0) {
         assert(llvm::all_of(ClusterDim, [](unsigned D) { return D != 0; }) &&
                "cluster_dim_x != 0 implies cluster_dim_y and cluster_dim_z "
@@ -452,6 +456,21 @@ void NVPTXAsmPrinter::emitKernelFunctionDirectives(const Function &F,
                "should be 0 as well");
       }
     }
+
+    if (BlocksAreClusters) {
+      LLVMContext &Ctx = F.getContext();
+      if (ReqNTID.empty() || ClusterDim.empty())
+        Ctx.diagnose(DiagnosticInfoUnsupported(
+            F, "blocksareclusters requires reqntid and cluster_dim attributes",
+            F.getSubprogram()));
+      else if (STI->getPTXVersion() < 90)
+        Ctx.diagnose(DiagnosticInfoUnsupported(
+            F, "blocksareclusters requires PTX version >= 9.0",
+            F.getSubprogram()));
+      else
+        O << ".blocksareclusters\n";
+    }
+
     if (const auto Maxclusterrank = getMaxClusterRank(F))
       O << ".maxclusterrank " << *Maxclusterrank << "\n";
   }
@@ -1458,7 +1477,6 @@ void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters(
   // Map the global virtual register number to a register class specific
   // virtual register number starting from 1 with that class.
   const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
-  //unsigned numRegClasses = TRI->getNumRegClasses();
 
   // Emit the Fake Stack Object
   const MachineFrameInfo &MFI = MF.getFrameInfo();
@@ -1479,13 +1497,12 @@ void NVPTXAsmPrinter::setAndEmitFunctionVirtualRegisters(
   // global virtual
   // register number and the per class virtual register number.
   // We use the per class virtual register number in the ptx output.
-  unsigned int numVRs = MRI->getNumVirtRegs();
-  for (unsigned i = 0; i < numVRs; i++) {
-    Register vr = Register::index2VirtReg(i);
-    const TargetRegisterClass *RC = MRI->getRegClass(vr);
-    DenseMap<unsigned, unsigned> &regmap = VRegMapping[RC];
-    int n = regmap.size();
-    regmap.insert(std::make_pair(vr, n + 1));
+  for (unsigned I : llvm::seq(MRI->getNumVirtRegs())) {
+    Register VR = Register::index2VirtReg(I);
+    if (MRI->use_empty(VR) && MRI->def_empty(VR))
+      continue;
+    auto &RCRegMap = VRegMapping[MRI->getRegClass(VR)];
+    RCRegMap[VR] = RCRegMap.size() + 1;
   }
 
   // Emit declaration of the virtual registers or 'physical' registers for
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index 6068035b2ee47..3300ed9a5a81c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -70,7 +70,7 @@ NVPTXDAGToDAGISel::getDivF32Level(const SDNode *N) const {
 }
 
 bool NVPTXDAGToDAGISel::usePrecSqrtF32(const SDNode *N) const {
-  return Subtarget->getTargetLowering()->usePrecSqrtF32(*MF, N);
+  return Subtarget->getTargetLowering()->usePrecSqrtF32(N);
 }
 
 bool NVPTXDAGToDAGISel::useF32FTZ() const {
@@ -82,11 +82,6 @@ bool NVPTXDAGToDAGISel::allowFMA() const {
   return TL->allowFMA(*MF, OptLevel);
 }
 
-bool NVPTXDAGToDAGISel::allowUnsafeFPMath() const {
-  const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
-  return TL->allowUnsafeFPMath(*MF);
-}
-
 bool NVPTXDAGToDAGISel::doRsqrtOpt() const { return EnableRsqrtOpt; }
 
 /// Select - Select instructions not customized! Used for
@@ -1027,6 +1022,72 @@ pickOpcodeForVT(MVT::SimpleValueType VT, std::optional<unsigned> Opcode_i16,
   }
 }
 
+static inline bool isAddLike(const SDValue V) {
+  return V.getOpcode() == ISD::ADD ||
+         (V->getOpcode() == ISD::OR && V->getFlags().hasDisjoint());
+}
+
+static SDValue stripAssertAlign(SDValue N) {
+  if (N.getOpcode() == ISD::AssertAlign)
+    N = N.getOperand(0);
+  return N;
+}
+
+// selectBaseADDR - Match a dag node which will serve as the base address for an
+// ADDR operand pair.
+static SDValue selectBaseADDR(SDValue N, SelectionDAG *DAG) {
+  N = stripAssertAlign(N);
+  if (const auto *GA = dyn_cast<GlobalAddressSDNode>(N))
+    return DAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(N),
+                                       GA->getValueType(0), GA->getOffset(),
+                                       GA->getTargetFlags());
+  if (const auto *ES = dyn_cast<ExternalSymbolSDNode>(N))
+    return DAG->getTargetExternalSymbol(ES->getSymbol(), ES->getValueType(0),
+                                        ES->getTargetFlags());
+  if (const auto *FIN = dyn_cast<FrameIndexSDNode>(N))
+    return DAG->getTargetFrameIndex(FIN->getIndex(), FIN->getValueType(0));
+
+  return N;
+}
+
+static SDValue accumulateOffset(SDValue &Addr, SDLoc DL, SelectionDAG *DAG) {
+  Addr = stripAssertAlign(Addr);
+  APInt AccumulatedOffset(64u, 0);
+  while (isAddLike(Addr)) {
+    const auto *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
+    if (!CN)
+      break;
+
+    const APInt CI = CN->getAPIntValue().sext(64);
+    if (!(CI + AccumulatedOffset).isSignedIntN(32))
+      break;
+
+    AccumulatedOffset += CI;
+    Addr = stripAssertAlign(Addr->getOperand(0));
+  }
+  return DAG->getSignedTargetConstant(AccumulatedOffset.getSExtValue(), DL,
+                                      MVT::i32);
+}
+
+static std::pair<SDValue, SDValue> selectADDR(SDValue Addr, SelectionDAG *DAG) {
+  SDValue Offset = accumulateOffset(Addr, SDLoc(Addr), DAG);
+  SDValue Base = selectBaseADDR(Addr, DAG);
+  return {Base, Offset};
+}
+
+// Select a pair of operands which represent a valid PTX address, this could be
+// one of the following things:
+//  - [var] - Offset is simply set to 0
+//  - [reg] - Offset is simply set to 0
+//  - [reg+immOff]
+//  - [var+immOff]
+// Note that immOff must fit into a 32-bit signed integer.
+bool NVPTXDAGToDAGISel::SelectADDR(SDValue Addr, SDValue &Base,
+                                   SDValue &Offset) {
+  std::tie(Base, Offset) = selectADDR(Addr, CurDAG);
+  return true;
+}
+
 bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
   MemSDNode *LD = cast<MemSDNode>(N);
   assert(LD->readMem() && "Expected load");
@@ -1062,8 +1123,7 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
          FromTypeWidth <= 128 && "Invalid width for load");
 
   // Create the machine instruction DAG
-  SDValue Offset, Base;
-  SelectADDR(N->getOperand(1), Base, Offset);
+  const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
   SDValue Ops[] = {getI32Imm(Ordering, DL),
                    getI32Imm(Scope, DL),
                    getI32Imm(CodeAddrSpace, DL),
@@ -1090,15 +1150,12 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
   return true;
 }
 
-static unsigned getLoadStoreVectorNumElts(SDNode *N) {
+static unsigned getStoreVectorNumElts(SDNode *N) {
   switch (N->getOpcode()) {
-  case NVPTXISD::LoadV2:
   case NVPTXISD::StoreV2:
     return 2;
-  case NVPTXISD::LoadV4:
   case NVPTXISD::StoreV4:
     return 4;
-  case NVPTXISD::LoadV8:
   case NVPTXISD::StoreV8:
     return 8;
   default:
@@ -1111,7 +1168,6 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
   const EVT MemEVT = LD->getMemoryVT();
   if (!MemEVT.isSimple())
     return false;
-  const MVT MemVT = MemEVT.getSimpleVT();
 
   // Address Space Setting
   const auto CodeAddrSpace = getAddrSpace(LD);
@@ -1131,21 +1187,17 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
   // Float  : ISD::NON_EXTLOAD or ISD::EXTLOAD and the type is float
   // Read at least 8 bits (predicates are stored as 8-bit values)
   // The last operand holds the original LoadSDNode::getExtensionType() value
-  const unsigned TotalWidth = MemVT.getSizeInBits();
   const unsigned ExtensionType =
       N->getConstantOperandVal(N->getNumOperands() - 1);
   const unsigned FromType = (ExtensionType == ISD::SEXTLOAD)
                                 ? NVPTX::PTXLdStInstCode::Signed
                                 : NVPTX::PTXLdStInstCode::Untyped;
 
-  const unsigned FromTypeWidth = TotalWidth / getLoadStoreVectorNumElts(N);
+  const unsigned FromTypeWidth = getFromTypeWidthForLoad(LD);
 
   assert(!(EltVT.isVector() && ExtensionType != ISD::NON_EXTLOAD));
-  assert(isPowerOf2_32(FromTypeWidth) && FromTypeWidth >= 8 &&
-         FromTypeWidth <= 128 && TotalWidth <= 256 && "Invalid width for load");
 
-  SDValue Offset, Base;
-  SelectADDR(N->getOperand(1), Base, Offset);
+  const auto [Base, Offset] = selectADDR(N->getOperand(1), CurDAG);
   SDValue Ops[] = {getI32Imm(Ordering, DL),
                    getI32Imm(Scope, DL),
                    getI32Imm(CodeAddrSpace, DL),
@@ -1188,33 +1240,25 @@ bool NVPTXDAGToDAGISel::tryLDG(MemSDNode *LD) {
   const EVT LoadedEVT = LD->getMemoryVT();
   if (!LoadedEVT.isSimple())
     return false;
-  const MVT LoadedVT = LoadedEVT.getSimpleVT();
 
   SDLoc DL(LD);
 
-  const unsigned TotalWidth = LoadedVT.getSizeInBits();
   unsigned ExtensionType;
-  unsigned NumElts;
   if (const auto *Load = dyn_cast<LoadSDNode>(LD)) {
     ExtensionType = Load->getExtensionType();
-    NumElts = 1;
   } else {
     ExtensionType = LD->getConstantOperandVal(LD->getNumOperands() - 1);
-    NumElts = getLoadStoreVectorNumElts(LD);
   }
   const unsigned FromType = (ExtensionType == ISD::SEXTLOAD)
                                 ? NVPTX::PTXLdStInstCode::Signed
                                 : NVPTX::PTXLdStInstCode::Untyped;
 
-  const unsigned FromTypeWidth = TotalWidth / NumElts;
+  const unsigned FromTypeWidth = getFromTypeWidthForLoad(LD);
 
   assert(!(LD->getSimpleValueType(0).isVector() &&
            ExtensionType != ISD::NON_EXTLOAD));
-  assert(isPowerOf2_32(FromTypeWidth) && FromTypeWidth >= 8 &&
-         FromTypeWidth <= 128 && TotalWidth <= 256 && "Invalid width for load");
 
-  SDValue Base, Offset;
-  SelectADDR(LD->getOperand(1), Base, Offset);
+  const auto [Base, Offset] = selectADDR(LD->getOperand(1), CurDAG);
   SDValue Ops[] = {getI32Imm(FromType, DL), getI32Imm(FromTypeWidth, DL), Base,
                    Offset, LD->getChain()};
 
@@ -1251,26 +1295,21 @@ bool NVPTXDAGToDAGISel::tryLDG(MemSDNode *LD) {
   return true;
 }
 
+unsigned NVPTXDAGToDAGISel::getFromTypeWidthForLoad(const MemSDNode *Mem) {
+  auto TotalWidth = Mem->getMemoryVT().getSizeInBits();
+  auto NumElts = Mem->getNumValues() - 1;
+  auto ElementBitWidth = TotalWidth / NumElts;
+  assert(isPowerOf2_32(ElementBitWidth) && ElementBitWidth >= 8 &&
+         ElementBitWidth <= 128 && TotalWidth <= 256 &&
+         "Invalid width for load");
+  return ElementBitWidth;
+}
+
 bool NVPTXDAGToDAGISel::tryLDU(SDNode *N) {
   auto *LD = cast<MemSDNode>(N);
 
-  unsigned NumElts;
-  switch (N->getOpcode()) {
-  default:
-    llvm_unreachable("Unexpected opcode");
-  case ISD::INTRINSIC_W_CHAIN:
-    NumElts = 1;
-    break;
-  case NVPTXISD::LDUV2:
-    NumElts = 2;
-    break;
-  case NVPTXISD::LDUV4:
-    NumElts = 4;
-    break;
-  }
-
   SDLoc DL(N);
-  const unsigned FromTypeWidth = LD->getMemoryVT().getSizeInBits() / NumElts;
+  const unsigned FromTypeWidth = getFromTypeWidthForLoad(LD);
   const MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
 
   // If this is an LDU intrinsic, the address is the third operand. If its an
@@ -1278,8 +1317,7 @@ bool NVPTXDAGToDAGISel::tryLDU(SDNode *N) {
   SDValue Addr =
       LD->getOperand(LD->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 2 : 1);
 
-  SDValue Base, Offset;
-  SelectADDR(Addr, Base, Offset);
+  const auto [Base, Offset] = selectADDR(Addr, CurDAG);
   SDValue Ops[] = {getI32Imm(FromTypeWidth, DL), Base, Offset, LD->getChain()};
 
   std::optional<unsigned> Opcode;
@@ -1339,9 +1377,7 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) {
   assert(isPowerOf2_32(ToTypeWidth) && ToTypeWidth >= 8 && ToTypeWidth <= 128 &&
          "Invalid width for store");
 
-  SDValue Offset, Base;
-  SelectADDR(ST->getBasePtr(), Base, Offset);
-
+  const auto [Base, Offset] = selectADDR(ST->getBasePtr(), CurDAG);
   SDValue Ops[] = {selectPossiblyImm(Value),
                    getI32Imm(Ordering, DL),
                    getI32Imm(Scope, DL),
@@ -1388,7 +1424,7 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
   // - for integer type, always use 'u'
   const unsigned TotalWidth = StoreVT.getSimpleVT().getSizeInBits();
 
-  const unsigned NumElts = getLoadStoreVectorNumElts(ST);
+  const unsigned NumElts = getStoreVectorNumElts(ST);
 
   SmallVector<SDValue, 16> Ops;
   for (auto &V : ST->ops().slice(1, NumElts))
@@ -1399,9 +1435,7 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
   assert(isPowerOf2_32(ToTypeWidth) && ToTypeWidth >= 8 && ToTypeWidth <= 128 &&
          TotalWidth <= 256 && "Invalid width for store");
 
-  SDValue Offset, Base;
-  SelectADDR(Addr, Base, Offset);
-
+  const auto [Base, Offset] = selectADDR(Addr, CurDAG);
   Ops.append({getI32Imm(Ordering, DL), getI32Imm(Scope, DL),
               getI32Imm(CodeAddrSpace, DL), getI32Imm(ToTypeWidth, DL), Base,
               Offset, Chain});
@@ -1708,59 +1742,6 @@ bool NVPTXDAGToDAGISel::tryBF16ArithToFMA(SDNode *N) {
   return true;
 }
 
-static inline bool isAddLike(const SDValue V) {
-  return V.getOpcode() == ISD::ADD ||
-         (V->getOpcode() == ISD::OR && V->getFlags().hasDisjoint());
-}
-
-// selectBaseADDR - Match a dag node which will serve as the base address for an
-// ADDR operand pair.
-static SDValue selectBaseADDR(SDValue N, SelectionDAG *DAG) {
-  if (const auto *GA = dyn_cast<GlobalAddressSDNode>(N))
-    return DAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(N),
-                                       GA->getValueType(0), GA->getOffset(),
-                                       GA->getTargetFlags());
-  if (const auto *ES = dyn_cast<ExternalSymbolSDNode>(N))
-    return DAG->getTargetExternalSymbol(ES->getSymbol(), ES->getValueType(0),
-                                        ES->getTargetFlags());
-  if (const auto *FIN = dyn_cast<FrameIndexSDNode>(N))
-    return DAG->getTargetFrameIndex(FIN->getIndex(), FIN->getValueType(0));
-
-  return N;
-}
-
-static SDValue accumulateOffset(SDValue &Addr, SDLoc DL, SelectionDAG *DAG) {
-  APInt AccumulatedOffset(64u, 0);
-  while (isAddLike(Addr)) {
-    const auto *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
-    if (!CN)
-      break;
-
-    const APInt CI = CN->getAPIntValue().sext(64);
-    if (!(CI + AccumulatedOffset).isSignedIntN(32))
-      break;
-
-    AccumulatedOffset += CI;
-    Addr = Addr->getOperand(0);
-  }
-  return DAG->getSignedTargetConstant(AccumulatedOffset.getSExtValue(), DL,
-                                      MVT::i32);
-}
-
-// Select a pair of operands which represent a valid PTX address, this could be
-// one of the following things:
-//  - [var] - Offset is simply set to 0
-//  - [reg] - Offset is simply set to 0
-//  - [reg+immOff]
-//  - [var+immOff]
-// Note that immOff must fit into a 32-bit signed integer.
-bool NVPTXDAGToDAGISel::SelectADDR(SDValue Addr, SDValue &Base,
-                                   SDValue &Offset) {
-  Offset = accumulateOffset(Addr, SDLoc(Addr), CurDAG);
-  Base = selectBaseADDR(Addr, CurDAG);
-  return true;
-}
-
 SDValue NVPTXDAGToDAGISel::selectPossiblyImm(SDValue V) {
   if (V.getOpcode() == ISD::BITCAST)
     V = V.getOperand(0);
@@ -1774,37 +1755,20 @@ SDValue NVPTXDAGToDAGISel::selectPossiblyImm(SDValue V) {
   return V;
 }
 
-bool NVPTXDAGToDAGISel::ChkMemSDNodeAddressSpace(SDNode *N,
-                                                 unsigned int spN) const {
-  const Value *Src = nullptr;
-  if (MemSDNode *mN = dyn_cast<MemSDNode>(N)) {
-    if (spN == 0 && mN->getMemOperand()->getPseudoValue())
-      return true;
-    Src = mN->getMemOperand()->getValue();
-  }
-  if (!Src)
-    return false;
-  if (auto *PT = dyn_cast<PointerType>(Src->getType()))
-    return (PT->getAddressSpace() == spN);
-  return false;
-}
-
 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
 /// inline asm expressions.
 bool NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand(
     const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
     std::vector<SDValue> &OutOps) {
-  SDValue Op0, Op1;
   switch (ConstraintID) {
   default:
     return true;
-  case InlineAsm::ConstraintCode::m: // memory
-    if (SelectADDR(Op, Op0, Op1)) {
-      OutOps.push_back(Op0);
-      OutOps.push_back(Op1);
-      return false;
-    }
-    break;
+  case InlineAsm::ConstraintCode::m: { // memory
+    const auto [Base, Offset] = selectADDR(Op, CurDAG);
+    OutOps.push_back(Base);
+    OutOps.push_back(Offset);
+    return false;
+  }
   }
   return true;
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index 9e0f88e544980..e2ad55bc1796d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -44,7 +44,6 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
   bool usePrecSqrtF32(const SDNode *N) const;
   bool useF32FTZ() const;
   bool allowFMA() const;
-  bool allowUnsafeFPMath() const;
   bool doRsqrtOpt() const;
 
   NVPTXScopes Scopes{};
@@ -102,8 +101,6 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
   SDValue getPTXCmpMode(const CondCodeSDNode &CondCode);
   SDValue selectPossiblyImm(SDValue V);
 
-  bool ChkMemSDNodeAddressSpace(SDNode *N, unsigned int spN) const;
-
   // Returns the Memory Order and Scope that the PTX memory instruction should
   // use, and inserts appropriate fence instruction before the memory
   // instruction, if needed to implement the instructions memory order. Required
@@ -114,6 +111,7 @@ class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
 
 public:
   static NVPTX::AddressSpace getAddrSpace(const MemSDNode *N);
+  static unsigned getFromTypeWidthForLoad(const MemSDNode *Mem);
 };
 
 class NVPTXDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 15f45a1f35e2f..bb4bb1195f78b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -14,6 +14,7 @@
 #include "NVPTXISelLowering.h"
 #include "MCTargetDesc/NVPTXBaseInfo.h"
 #include "NVPTX.h"
+#include "NVPTXISelDAGToDAG.h"
 #include "NVPTXSubtarget.h"
 #include "NVPTXTargetMachine.h"
 #include "NVPTXTargetObjectFile.h"
@@ -28,7 +29,6 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
-#include "llvm/CodeGen/Register.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
 #include "llvm/CodeGen/TargetCallingConv.h"
@@ -125,10 +125,6 @@ NVPTXTargetLowering::getDivF32Level(const MachineFunction &MF,
   if (UsePrecDivF32.getNumOccurrences() > 0)
     return UsePrecDivF32;
 
-  // Otherwise, use div.approx if fast math is enabled
-  if (allowUnsafeFPMath(MF))
-    return NVPTX::DivPrecisionLevel::Approx;
-
   const SDNodeFlags Flags = N.getFlags();
   if (Flags.hasApproximateFuncs())
     return NVPTX::DivPrecisionLevel::Approx;
@@ -136,16 +132,11 @@ NVPTXTargetLowering::getDivF32Level(const MachineFunction &MF,
   return NVPTX::DivPrecisionLevel::IEEE754;
 }
 
-bool NVPTXTargetLowering::usePrecSqrtF32(const MachineFunction &MF,
-                                         const SDNode *N) const {
+bool NVPTXTargetLowering::usePrecSqrtF32(const SDNode *N) const {
   // If nvptx-prec-sqrtf32 is used on the command-line, always honor it
   if (UsePrecSqrtF32.getNumOccurrences() > 0)
     return UsePrecSqrtF32;
 
-  // Otherwise, use sqrt.approx if fast math is enabled
-  if (allowUnsafeFPMath(MF))
-    return false;
-
   if (N) {
     const SDNodeFlags Flags = N->getFlags();
     if (Flags.hasApproximateFuncs())
@@ -205,7 +196,8 @@ static bool IsPTXVectorType(MVT VT) {
 //    - unsigned int NumElts - The number of elements in the final vector
 //    - EVT EltVT - The type of the elements in the final vector
 static std::optional<std::pair<unsigned int, MVT>>
-getVectorLoweringShape(EVT VectorEVT, bool CanLowerTo256Bit) {
+getVectorLoweringShape(EVT VectorEVT, const NVPTXSubtarget &STI,
+                       unsigned AddressSpace) {
   if (!VectorEVT.isSimple())
     return std::nullopt;
   const MVT VectorVT = VectorEVT.getSimpleVT();
@@ -222,6 +214,8 @@ getVectorLoweringShape(EVT VectorEVT, bool CanLowerTo256Bit) {
   // The size of the PTX virtual register that holds a packed type.
   unsigned PackRegSize;
 
+  bool CanLowerTo256Bit = STI.has256BitVectorLoadStore(AddressSpace);
+
   // We only handle "native" vector sizes for now, e.g. <4 x double> is not
   // legal.  We can (and should) split that into 2 stores of <2 x double> here
   // but I'm leaving that as a TODO for now.
@@ -272,6 +266,8 @@ getVectorLoweringShape(EVT VectorEVT, bool CanLowerTo256Bit) {
     LLVM_FALLTHROUGH;
   case MVT::v2f32: // <1 x f32x2>
   case MVT::v4f32: // <2 x f32x2>
+    if (!STI.hasF32x2Instructions())
+      return std::pair(NumElts, EltVT);
     PackRegSize = 64;
     break;
   }
@@ -287,97 +283,44 @@ getVectorLoweringShape(EVT VectorEVT, bool CanLowerTo256Bit) {
 }
 
 /// ComputePTXValueVTs - For the given Type \p Ty, returns the set of primitive
-/// EVTs that compose it.  Unlike ComputeValueVTs, this will break apart vectors
-/// into their primitive components.
+/// legal-ish MVTs that compose it. Unlike ComputeValueVTs, this will legalize
+/// the types as required by the calling convention (with special handling for
+/// i8s).
 /// NOTE: This is a band-aid for code that expects ComputeValueVTs to return the
 /// same number of types as the Ins/Outs arrays in LowerFormalArguments,
 /// LowerCall, and LowerReturn.
 static void ComputePTXValueVTs(const TargetLowering &TLI, const DataLayout &DL,
+                               LLVMContext &Ctx, CallingConv::ID CallConv,
                                Type *Ty, SmallVectorImpl<EVT> &ValueVTs,
-                               SmallVectorImpl<uint64_t> *Offsets = nullptr,
+                               SmallVectorImpl<uint64_t> &Offsets,
                                uint64_t StartingOffset = 0) {
   SmallVector<EVT, 16> TempVTs;
   SmallVector<uint64_t, 16> TempOffsets;
-
-  // Special case for i128 - decompose to (i64, i64)
-  if (Ty->isIntegerTy(128) || Ty->isFP128Ty()) {
-    ValueVTs.append({MVT::i64, MVT::i64});
-
-    if (Offsets)
-      Offsets->append({StartingOffset + 0, StartingOffset + 8});
-
-    return;
-  }
-
-  // Given a struct type, recursively traverse the elements with custom ComputePTXValueVTs.
-  if (StructType *STy = dyn_cast<StructType>(Ty)) {
-    auto const *SL = DL.getStructLayout(STy);
-    auto ElementNum = 0;
-    for(auto *EI : STy->elements()) {
-      ComputePTXValueVTs(TLI, DL, EI, ValueVTs, Offsets,
-                         StartingOffset + SL->getElementOffset(ElementNum));
-      ++ElementNum;
-    }
-    return;
-  }
-
-  // Given an array type, recursively traverse the elements with custom ComputePTXValueVTs.
-  if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
-    Type *EltTy = ATy->getElementType();
-    uint64_t EltSize = DL.getTypeAllocSize(EltTy);
-    for (int I : llvm::seq<int>(ATy->getNumElements()))
-      ComputePTXValueVTs(TLI, DL, EltTy, ValueVTs, Offsets, StartingOffset + I * EltSize);
-    return;
-  }
-
-  // Will split structs and arrays into member types, but will not split vector
-  // types. We do that manually below.
   ComputeValueVTs(TLI, DL, Ty, TempVTs, &TempOffsets, StartingOffset);
 
-  for (auto [VT, Off] : zip(TempVTs, TempOffsets)) {
-    // Split vectors into individual elements that fit into registers.
-    if (VT.isVector()) {
-      unsigned NumElts = VT.getVectorNumElements();
-      EVT EltVT = VT.getVectorElementType();
-      // Below we must maintain power-of-2 sized vectors because
-      // TargetLoweringBase::getVectorTypeBreakdown() which is invoked in
-      // ComputePTXValueVTs() cannot currently break down non-power-of-2 sized
-      // vectors.
-
-      // If the element type belongs to one of the supported packed vector types
-      // then we can pack multiples of this element into a single register.
-      if (VT == MVT::v2i8) {
-        // We can pack 2 i8s into a single 16-bit register. We only do this for
-        // loads and stores, which is why we have a separate case for it.
-        EltVT = MVT::v2i8;
-        NumElts = 1;
-      } else if (VT == MVT::v3i8) {
-        // We can also pack 3 i8s into 32-bit register, leaving the 4th
-        // element undefined.
-        EltVT = MVT::v4i8;
-        NumElts = 1;
-      } else if (NumElts > 1 && isPowerOf2_32(NumElts)) {
-        // Handle default packed types.
-        for (MVT PackedVT : NVPTX::packed_types()) {
-          const auto NumEltsPerReg = PackedVT.getVectorNumElements();
-          if (NumElts % NumEltsPerReg == 0 &&
-              EltVT == PackedVT.getVectorElementType()) {
-            EltVT = PackedVT;
-            NumElts /= NumEltsPerReg;
-            break;
-          }
-        }
-      }
+  for (const auto [VT, Off] : zip(TempVTs, TempOffsets)) {
+    MVT RegisterVT = TLI.getRegisterTypeForCallingConv(Ctx, CallConv, VT);
+    unsigned NumRegs = TLI.getNumRegistersForCallingConv(Ctx, CallConv, VT);
+
+    // Since we actually can load/store b8, we need to ensure that we'll use
+    // the original sized type for any i8s or i8 vectors.
+    if (VT.getScalarType() == MVT::i8) {
+      if (RegisterVT == MVT::i16)
+        RegisterVT = MVT::i8;
+      else if (RegisterVT == MVT::v2i16)
+        RegisterVT = MVT::v2i8;
+      else
+        assert(RegisterVT == MVT::v4i8 &&
+               "Expected v4i8, v2i16, or i16 for i8 RegisterVT");
+    }
 
-      for (unsigned J : seq(NumElts)) {
-        ValueVTs.push_back(EltVT);
-        if (Offsets)
-          Offsets->push_back(Off + J * EltVT.getStoreSize());
-      }
-    } else {
-      ValueVTs.push_back(VT);
-      if (Offsets)
-        Offsets->push_back(Off);
+    // TODO: This is horribly incorrect for cases where the vector elements are
+    // not a multiple of bytes (ex i1) and legal or i8. However, this problem
+    // has existed for as long as NVPTX has and no one has complained, so we'll
+    // leave it for now.
+    for (unsigned I : seq(NumRegs)) {
+      ValueVTs.push_back(RegisterVT);
+      Offsets.push_back(Off + I * RegisterVT.getStoreSize());
     }
   }
 }
@@ -640,7 +583,9 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   addRegisterClass(MVT::v2f16, &NVPTX::B32RegClass);
   addRegisterClass(MVT::bf16, &NVPTX::B16RegClass);
   addRegisterClass(MVT::v2bf16, &NVPTX::B32RegClass);
-  addRegisterClass(MVT::v2f32, &NVPTX::B64RegClass);
+
+  if (STI.hasF32x2Instructions())
+    addRegisterClass(MVT::v2f32, &NVPTX::B64RegClass);
 
   // Conversion to/from FP16/FP16x2 is always legal.
   setOperationAction(ISD::BUILD_VECTOR, MVT::v2f16, Custom);
@@ -680,6 +625,9 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   // No support for these operations with v2f32.
   setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f32, Expand);
   setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f32, Expand);
+  // Need custom lowering in case the index is dynamic.
+  if (STI.hasF32x2Instructions())
+    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f32, Custom);
 
   // Custom conversions to/from v2i8.
   setOperationAction(ISD::BITCAST, MVT::v2i8, Custom);
@@ -900,6 +848,17 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   if (STI.allowFP16Math() || STI.hasBF16Math())
     setTargetDAGCombine(ISD::SETCC);
 
+  // Vector reduction operations. These may be turned into shuffle or tree
+  // reductions depending on what instructions are available for each type.
+  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
+    MVT EltVT = VT.getVectorElementType();
+    if (EltVT == MVT::f32 || EltVT == MVT::f64) {
+      setOperationAction({ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMIN,
+                          ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM},
+                         VT, Custom);
+    }
+  }
+
   // Promote fp16 arithmetic if fp16 hardware isn't available or the
   // user passed --nvptx-no-fp16-math. The flag is useful because,
   // although sm_53+ GPUs have some sort of FP16 support in
@@ -1143,6 +1102,10 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
     MAKE_CASE(NVPTXISD::BFI)
     MAKE_CASE(NVPTXISD::PRMT)
     MAKE_CASE(NVPTXISD::FCOPYSIGN)
+    MAKE_CASE(NVPTXISD::FMAXNUM3)
+    MAKE_CASE(NVPTXISD::FMINNUM3)
+    MAKE_CASE(NVPTXISD::FMAXIMUM3)
+    MAKE_CASE(NVPTXISD::FMINIMUM3)
     MAKE_CASE(NVPTXISD::DYNAMIC_STACKALLOC)
     MAKE_CASE(NVPTXISD::STACKRESTORE)
     MAKE_CASE(NVPTXISD::STACKSAVE)
@@ -1176,8 +1139,7 @@ SDValue NVPTXTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
                                              bool &UseOneConst,
                                              bool Reciprocal) const {
   if (!(Enabled == ReciprocalEstimate::Enabled ||
-        (Enabled == ReciprocalEstimate::Unspecified &&
-         !usePrecSqrtF32(DAG.getMachineFunction()))))
+        (Enabled == ReciprocalEstimate::Unspecified && !usePrecSqrtF32())))
     return SDValue();
 
   if (ExtraSteps == ReciprocalEstimate::Unspecified)
@@ -1599,7 +1561,8 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     } else {
       SmallVector<EVT, 16> VTs;
       SmallVector<uint64_t, 16> Offsets;
-      ComputePTXValueVTs(*this, DL, Arg.Ty, VTs, &Offsets, VAOffset);
+      ComputePTXValueVTs(*this, DL, Ctx, CLI.CallConv, Arg.Ty, VTs, Offsets,
+                         VAOffset);
       assert(VTs.size() == Offsets.size() && "Size mismatch");
       assert(VTs.size() == ArgOuts.size() && "Size mismatch");
 
@@ -1749,7 +1712,7 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   if (!Ins.empty()) {
     SmallVector<EVT, 16> VTs;
     SmallVector<uint64_t, 16> Offsets;
-    ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets);
+    ComputePTXValueVTs(*this, DL, Ctx, CLI.CallConv, RetTy, VTs, Offsets);
     assert(VTs.size() == Ins.size() && "Bad value decomposition");
 
     const Align RetAlign = getArgumentAlignment(CB, RetTy, 0, DL);
@@ -1929,6 +1892,124 @@ static SDValue getPRMT(SDValue A, SDValue B, uint64_t Selector, SDLoc DL,
   return getPRMT(A, B, DAG.getConstant(Selector, DL, MVT::i32), DL, DAG, Mode);
 }
 
+/// Reduces the elements using the scalar operations provided. The operations
+/// are sorted descending in number of inputs they take. The flags on the
+/// original reduction operation will be propagated to each scalar operation.
+/// Nearby elements are grouped in tree reduction, unlike the shuffle reduction
+/// used in ExpandReductions and SelectionDAG.
+static SDValue buildTreeReduction(
+    const SmallVector<SDValue> &Elements, EVT EltTy,
+    ArrayRef<std::pair<unsigned /*NodeType*/, unsigned /*NumInputs*/>> Ops,
+    const SDLoc &DL, const SDNodeFlags Flags, SelectionDAG &DAG) {
+  // Build the reduction tree at each level, starting with all the elements.
+  SmallVector<SDValue> Level = Elements;
+
+  unsigned OpIdx = 0;
+  while (Level.size() > 1) {
+    // Try to reduce this level using the current operator.
+    const auto [Op, NumInputs] = Ops[OpIdx];
+
+    // Build the next level by partially reducing all elements.
+    SmallVector<SDValue> ReducedLevel;
+    unsigned I = 0, E = Level.size();
+    for (; I + NumInputs <= E; I += NumInputs) {
+      // Reduce elements in groups of [NumInputs], as much as possible.
+      ReducedLevel.push_back(DAG.getNode(
+          Op, DL, EltTy, ArrayRef<SDValue>(Level).slice(I, NumInputs), Flags));
+    }
+
+    if (I < E) {
+      // Handle leftover elements.
+
+      if (ReducedLevel.empty()) {
+        // We didn't reduce anything at this level. We need to pick a smaller
+        // operator.
+        ++OpIdx;
+        assert(OpIdx < Ops.size() && "no smaller operators for reduction");
+        continue;
+      }
+
+      // We reduced some things but there's still more left, meaning the
+      // operator's number of inputs doesn't evenly divide this level size. Move
+      // these elements to the next level.
+      for (; I < E; ++I)
+        ReducedLevel.push_back(Level[I]);
+    }
+
+    // Process the next level.
+    Level = ReducedLevel;
+  }
+
+  return *Level.begin();
+}
+
+// Get scalar reduction opcode
+static ISD::NodeType getScalarOpcodeForReduction(unsigned ReductionOpcode) {
+  switch (ReductionOpcode) {
+  case ISD::VECREDUCE_FMAX:
+    return ISD::FMAXNUM;
+  case ISD::VECREDUCE_FMIN:
+    return ISD::FMINNUM;
+  case ISD::VECREDUCE_FMAXIMUM:
+    return ISD::FMAXIMUM;
+  case ISD::VECREDUCE_FMINIMUM:
+    return ISD::FMINIMUM;
+  default:
+    llvm_unreachable("unhandled reduction opcode");
+  }
+}
+
+/// Get 3-input scalar reduction opcode
+static std::optional<NVPTXISD::NodeType>
+getScalar3OpcodeForReduction(unsigned ReductionOpcode) {
+  switch (ReductionOpcode) {
+  case ISD::VECREDUCE_FMAX:
+    return NVPTXISD::FMAXNUM3;
+  case ISD::VECREDUCE_FMIN:
+    return NVPTXISD::FMINNUM3;
+  case ISD::VECREDUCE_FMAXIMUM:
+    return NVPTXISD::FMAXIMUM3;
+  case ISD::VECREDUCE_FMINIMUM:
+    return NVPTXISD::FMINIMUM3;
+  default:
+    return std::nullopt;
+  }
+}
+
+/// Lower reductions to either a sequence of operations or a tree if
+/// reassociations are allowed. This method will use larger operations like
+/// max3/min3 when the target supports them.
+SDValue NVPTXTargetLowering::LowerVECREDUCE(SDValue Op,
+                                            SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  const SDNodeFlags Flags = Op->getFlags();
+  SDValue Vector = Op.getOperand(0);
+
+  const unsigned Opcode = Op->getOpcode();
+  const EVT EltTy = Vector.getValueType().getVectorElementType();
+
+  // Whether we can use 3-input min/max when expanding the reduction.
+  const bool CanUseMinMax3 =
+      EltTy == MVT::f32 && STI.getSmVersion() >= 100 &&
+      STI.getPTXVersion() >= 88 &&
+      (Opcode == ISD::VECREDUCE_FMAX || Opcode == ISD::VECREDUCE_FMIN ||
+       Opcode == ISD::VECREDUCE_FMAXIMUM || Opcode == ISD::VECREDUCE_FMINIMUM);
+
+  // A list of SDNode opcodes with equivalent semantics, sorted descending by
+  // number of inputs they take.
+  SmallVector<std::pair<unsigned /*Op*/, unsigned /*NumIn*/>, 2> ScalarOps;
+
+  if (auto Opcode3Elem = getScalar3OpcodeForReduction(Opcode);
+      CanUseMinMax3 && Opcode3Elem)
+    ScalarOps.push_back({*Opcode3Elem, 3});
+  ScalarOps.push_back({getScalarOpcodeForReduction(Opcode), 2});
+
+  SmallVector<SDValue> Elements;
+  DAG.ExtractVectorElements(Vector, Elements);
+
+  return buildTreeReduction(Elements, EltTy, ScalarOps, DL, Flags, DAG);
+}
+
 SDValue NVPTXTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
   // Handle bitcasting from v2i8 without hitting the default promotion
   // strategy which goes through stack memory.
@@ -2716,8 +2797,7 @@ static SDValue lowerROT(SDValue Op, SelectionDAG &DAG) {
                      SDLoc(Op), Opcode, DAG);
 }
 
-static SDValue lowerFREM(SDValue Op, SelectionDAG &DAG,
-                         bool AllowUnsafeFPMath) {
+static SDValue lowerFREM(SDValue Op, SelectionDAG &DAG) {
   // Lower (frem x, y) into (sub x, (mul (ftrunc (div x, y)) y)),
   // i.e. "poor man's fmod()". When y is infinite, x is returned. This matches
   // the semantics of LLVM's frem.
@@ -2734,7 +2814,7 @@ static SDValue lowerFREM(SDValue Op, SelectionDAG &DAG,
   SDValue Sub = DAG.getNode(ISD::FSUB, DL, Ty, X, Mul,
                             Flags | SDNodeFlags::AllowContract);
 
-  if (AllowUnsafeFPMath || Flags.hasNoInfs())
+  if (Flags.hasNoInfs())
     return Sub;
 
   // If Y is infinite, return X
@@ -2808,6 +2888,11 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
     return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::CONCAT_VECTORS:
     return LowerCONCAT_VECTORS(Op, DAG);
+  case ISD::VECREDUCE_FMAX:
+  case ISD::VECREDUCE_FMIN:
+  case ISD::VECREDUCE_FMAXIMUM:
+  case ISD::VECREDUCE_FMINIMUM:
+    return LowerVECREDUCE(Op, DAG);
   case ISD::STORE:
     return LowerSTORE(Op, DAG);
   case ISD::LOAD:
@@ -2874,7 +2959,7 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::CTLZ:
     return lowerCTLZCTPOP(Op, DAG);
   case ISD::FREM:
-    return lowerFREM(Op, DAG, allowUnsafeFPMath(DAG.getMachineFunction()));
+    return lowerFREM(Op, DAG);
 
   default:
     llvm_unreachable("Custom lowering not defined for operation");
@@ -3088,8 +3173,8 @@ NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
   if (ValVT != MemVT)
     return SDValue();
 
-  const auto NumEltsAndEltVT = getVectorLoweringShape(
-      ValVT, STI.has256BitVectorLoadStore(N->getAddressSpace()));
+  const auto NumEltsAndEltVT =
+      getVectorLoweringShape(ValVT, STI, N->getAddressSpace());
   if (!NumEltsAndEltVT)
     return SDValue();
   const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
@@ -3257,6 +3342,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
   const DataLayout &DL = DAG.getDataLayout();
+  LLVMContext &Ctx = *DAG.getContext();
   auto PtrVT = getPointerTy(DAG.getDataLayout());
 
   const Function &F = DAG.getMachineFunction().getFunction();
@@ -3328,7 +3414,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
     } else {
       SmallVector<EVT, 16> VTs;
       SmallVector<uint64_t, 16> Offsets;
-      ComputePTXValueVTs(*this, DL, Ty, VTs, &Offsets, 0);
+      ComputePTXValueVTs(*this, DL, Ctx, CallConv, Ty, VTs, Offsets);
       assert(VTs.size() == ArgIns.size() && "Size mismatch");
       assert(VTs.size() == Offsets.size() && "Size mismatch");
 
@@ -3340,7 +3426,7 @@ SDValue NVPTXTargetLowering::LowerFormalArguments(
       for (const unsigned NumElts : VI) {
         // i1 is loaded/stored as i8
         const EVT LoadVT = VTs[I] == MVT::i1 ? MVT::i8 : VTs[I];
-        const EVT VecVT = getVectorizedVT(LoadVT, NumElts, *DAG.getContext());
+        const EVT VecVT = getVectorizedVT(LoadVT, NumElts, Ctx);
 
         SDValue VecAddr = DAG.getObjectPtrOffset(
             dl, ArgSymbol, TypeSize::getFixed(Offsets[I]));
@@ -3385,6 +3471,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   }
 
   const DataLayout &DL = DAG.getDataLayout();
+  LLVMContext &Ctx = *DAG.getContext();
 
   const SDValue RetSymbol = DAG.getExternalSymbol("func_retval0", MVT::i32);
   const auto RetAlign = getFunctionParamOptimizedAlign(&F, RetTy, DL);
@@ -3397,7 +3484,7 @@ NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
 
   SmallVector<EVT, 16> VTs;
   SmallVector<uint64_t, 16> Offsets;
-  ComputePTXValueVTs(*this, DL, RetTy, VTs, &Offsets);
+  ComputePTXValueVTs(*this, DL, Ctx, CallConv, RetTy, VTs, Offsets);
   assert(VTs.size() == OutVals.size() && "Bad return value decomposition");
 
   const auto GetRetVal = [&](unsigned I) -> SDValue {
@@ -3908,6 +3995,18 @@ bool NVPTXTargetLowering::getTgtMemIntrinsic(
     return true;
   }
 
+  case Intrinsic::nvvm_prefetch_tensormap: {
+    auto &DL = I.getDataLayout();
+    Info.opc = ISD::INTRINSIC_VOID;
+    Info.memVT = getPointerTy(DL);
+    Info.ptrVal = I.getArgOperand(0);
+    Info.offset = 0;
+    Info.flags =
+        MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable;
+    Info.align.reset();
+    return true;
+  }
+
   case Intrinsic::nvvm_ldu_global_i:
   case Intrinsic::nvvm_ldu_global_f:
   case Intrinsic::nvvm_ldu_global_p: {
@@ -4716,17 +4815,7 @@ bool NVPTXTargetLowering::allowFMA(MachineFunction &MF,
   if (MF.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast)
     return true;
 
-  return allowUnsafeFPMath(MF);
-}
-
-bool NVPTXTargetLowering::allowUnsafeFPMath(const MachineFunction &MF) const {
-  // Honor TargetOptions flags that explicitly say unsafe math is okay.
-  if (MF.getTarget().Options.UnsafeFPMath)
-    return true;
-
-  // Allow unsafe math if unsafe-fp-math attribute explicitly says so.
-  const Function &F = MF.getFunction();
-  return F.getFnAttribute("unsafe-fp-math").getValueAsBool();
+  return false;
 }
 
 static bool isConstZero(const SDValue &Operand) {
@@ -5112,76 +5201,6 @@ static SDValue PerformFADDCombine(SDNode *N,
   return PerformFADDCombineWithOperands(N, N1, N0, DCI, OptLevel);
 }
 
-static SDValue PerformANDCombine(SDNode *N,
-                                 TargetLowering::DAGCombinerInfo &DCI) {
-  // The type legalizer turns a vector load of i8 values into a zextload to i16
-  // registers, optionally ANY_EXTENDs it (if target type is integer),
-  // and ANDs off the high 8 bits. Since we turn this load into a
-  // target-specific DAG node, the DAG combiner fails to eliminate these AND
-  // nodes. Do that here.
-  SDValue Val = N->getOperand(0);
-  SDValue Mask = N->getOperand(1);
-
-  if (isa<ConstantSDNode>(Val)) {
-    std::swap(Val, Mask);
-  }
-
-  SDValue AExt;
-
-  // Generally, we will see zextload -> IMOV16rr -> ANY_EXTEND -> and
-  if (Val.getOpcode() == ISD::ANY_EXTEND) {
-    AExt = Val;
-    Val = Val->getOperand(0);
-  }
-
-  if (Val->getOpcode() == NVPTXISD::LoadV2 ||
-      Val->getOpcode() == NVPTXISD::LoadV4) {
-    ConstantSDNode *MaskCnst = dyn_cast<ConstantSDNode>(Mask);
-    if (!MaskCnst) {
-      // Not an AND with a constant
-      return SDValue();
-    }
-
-    uint64_t MaskVal = MaskCnst->getZExtValue();
-    if (MaskVal != 0xff) {
-      // Not an AND that chops off top 8 bits
-      return SDValue();
-    }
-
-    MemSDNode *Mem = dyn_cast<MemSDNode>(Val);
-    if (!Mem) {
-      // Not a MemSDNode?!?
-      return SDValue();
-    }
-
-    EVT MemVT = Mem->getMemoryVT();
-    if (MemVT != MVT::v2i8 && MemVT != MVT::v4i8) {
-      // We only handle the i8 case
-      return SDValue();
-    }
-
-    unsigned ExtType = Val->getConstantOperandVal(Val->getNumOperands() - 1);
-    if (ExtType == ISD::SEXTLOAD) {
-      // If for some reason the load is a sextload, the and is needed to zero
-      // out the high 8 bits
-      return SDValue();
-    }
-
-    bool AddTo = false;
-    if (AExt.getNode() != nullptr) {
-      // Re-insert the ext as a zext.
-      Val = DCI.DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
-                            AExt.getValueType(), Val);
-      AddTo = true;
-    }
-
-    // If we get here, the AND is unnecessary.  Just replace it with the load
-    DCI.CombineTo(N, Val, AddTo);
-  }
-
-  return SDValue();
-}
-
 static SDValue PerformREMCombine(SDNode *N,
                                  TargetLowering::DAGCombinerInfo &DCI,
                                  CodeGenOptLevel OptLevel) {
@@ -5853,8 +5872,6 @@ SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
     return PerformADDCombine(N, DCI, OptLevel);
   case ISD::ADDRSPACECAST:
     return combineADDRSPACECAST(N, DCI);
-  case ISD::AND:
-    return PerformANDCombine(N, DCI);
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
     return combineMulWide(N, DCI, OptLevel);
@@ -5926,8 +5943,8 @@ static void replaceLoadVector(SDNode *N, SelectionDAG &DAG,
   if (ResVT != MemVT)
     return;
 
-  const auto NumEltsAndEltVT = getVectorLoweringShape(
-      ResVT, STI.has256BitVectorLoadStore(LD->getAddressSpace()));
+  const auto NumEltsAndEltVT =
+      getVectorLoweringShape(ResVT, STI, LD->getAddressSpace());
   if (!NumEltsAndEltVT)
     return;
   const auto [NumElts, EltVT] = NumEltsAndEltVT.value();
@@ -6479,6 +6496,24 @@ static void computeKnownBitsForPRMT(const SDValue Op, KnownBits &Known,
   }
 }
 
+static void computeKnownBitsForLoadV(const SDValue Op, KnownBits &Known) {
+  MemSDNode *LD = cast<MemSDNode>(Op);
+
+  // We can't do anything without knowing the sign bit.
+  auto ExtType = LD->getConstantOperandVal(LD->getNumOperands() - 1);
+  if (ExtType == ISD::SEXTLOAD)
+    return;
+
+  // ExtLoading to vector types is weird and may not work well with known bits.
+  auto DestVT = LD->getValueType(0);
+  if (DestVT.isVector())
+    return;
+
+  assert(Known.getBitWidth() == DestVT.getSizeInBits());
+  auto ElementBitWidth = NVPTXDAGToDAGISel::getFromTypeWidthForLoad(LD);
+  Known.Zero.setHighBits(Known.getBitWidth() - ElementBitWidth);
+}
+
 void NVPTXTargetLowering::computeKnownBitsForTargetNode(
     const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
     const SelectionDAG &DAG, unsigned Depth) const {
@@ -6488,6 +6523,11 @@ void NVPTXTargetLowering::computeKnownBitsForTargetNode(
   case NVPTXISD::PRMT:
     computeKnownBitsForPRMT(Op, Known, DAG, Depth);
     break;
+  case NVPTXISD::LoadV2:
+  case NVPTXISD::LoadV4:
+  case NVPTXISD::LoadV8:
+    computeKnownBitsForLoadV(Op, Known);
+    break;
   default:
     break;
   }
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index cf72a1e6db89c..27f099e220976 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -64,6 +64,11 @@ enum NodeType : unsigned {
   UNPACK_VECTOR,
 
   FCOPYSIGN,
+  FMAXNUM3,
+  FMINNUM3,
+  FMAXIMUM3,
+  FMINIMUM3,
+
   DYNAMIC_STACKALLOC,
   STACKRESTORE,
   STACKSAVE,
@@ -201,8 +206,7 @@ class NVPTXTargetLowering : public TargetLowering {
 
   // Get whether we should use a precise or approximate 32-bit floating point
   // sqrt instruction.
-  bool usePrecSqrtF32(const MachineFunction &MF,
-                      const SDNode *N = nullptr) const;
+  bool usePrecSqrtF32(const SDNode *N = nullptr) const;
 
   // Get whether we should use instructions that flush floating-point denormals
   // to sign-preserving zero.
@@ -215,7 +219,6 @@ class NVPTXTargetLowering : public TargetLowering {
   unsigned combineRepeatedFPDivisors() const override { return 2; }
 
   bool allowFMA(MachineFunction &MF, CodeGenOptLevel OptLevel) const;
-  bool allowUnsafeFPMath(const MachineFunction &MF) const;
 
   bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
                                   EVT) const override {
@@ -286,6 +289,7 @@ class NVPTXTargetLowering : public TargetLowering {
 
   SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index aac611d4c903a..7b135098bd4c1 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -148,13 +148,16 @@ class OneUse2<SDPatternOperator operator>
     : PatFrag<(ops node:$A, node:$B), (operator node:$A, node:$B), [{ return N->hasOneUse(); }]>;
 
 
-class fpimm_pos_inf<ValueType vt>
-    : FPImmLeaf<vt, [{ return Imm.isPosInfinity(); }]>;
-
 class zeroinitializer<ValueType vt> : 
   PatLeaf<(vt (bitconvert (!cast<ValueType>("i" # vt.Size) 0)))>;
 
 
+def fpimm_pos_inf : FPImmLeaf<fAny, [{ return Imm.isPosInfinity(); }]>;
+def fpimm_0 : FPImmLeaf<fAny, [{ return Imm.isZero(); }]>;
+def fpimm_1 : FPImmLeaf<fAny, [{ return Imm.isExactlyValue(1.0); }]>;
+def fpimm_neg_1 : FPImmLeaf<fAny, [{ return Imm.isExactlyValue(-1.0); }]>;
+
+
 // Operands which can hold a Register or an Immediate.
 //
 // Unfortunately, since most register classes can hold multiple types, we must
@@ -268,7 +271,7 @@ multiclass I3Inst<string op_str, SDPatternOperator op_node, RegTyInfo t,
 // The instructions are named "<OpcStr><Width>" (e.g. "add.s64").
 multiclass I3<string op_str, SDPatternOperator op_node, bit commutative> {
   foreach t = [I16RT, I32RT, I64RT] in
-    defm t.Ty# : I3Inst<op_str # t.Size, op_node, t, commutative>;
+    defm t.Size# : I3Inst<op_str # t.Size, op_node, t, commutative>;
 }
 
 class I16x2<string OpcStr, SDNode OpNode> :
@@ -347,6 +350,36 @@ multiclass FMINIMUMMAXIMUM<string OpcStr, bit NaN, SDNode OpNode> {
                Requires<[hasBF16Math, hasSM<80>, hasPTX<70>]>;
 }
 
+// Template for 3-input minimum/maximum instructions
+// (sm_100+/PTX 8.8 and f32 only)
+//
+// Also defines ftz (flush subnormal inputs and results to sign-preserving
+// zero) variants for fp32 functions.
+multiclass FMINIMUMMAXIMUM3<string OpcStr, bit NaN, SDNode OpNode> {
+  defvar nan_str = !if(NaN, ".NaN", "");
+   def f32rrr :
+     BasicFlagsNVPTXInst<(outs B32:$dst),
+               (ins B32:$a, B32:$b, B32:$c),
+               (ins FTZFlag:$ftz),
+               OpcStr # "$ftz" # nan_str # ".f32",
+               [(set f32:$dst, (OpNode f32:$a, f32:$b, f32:$c))]>,
+               Requires<[hasPTX<88>, hasSM<100>]>;
+   def f32rri :
+     BasicFlagsNVPTXInst<(outs B32:$dst),
+               (ins B32:$a, B32:$b, f32imm:$c),
+               (ins FTZFlag:$ftz),
+               OpcStr # "$ftz" # nan_str # ".f32",
+               [(set f32:$dst, (OpNode f32:$a, f32:$b, fpimm:$c))]>,
+               Requires<[hasPTX<88>, hasSM<100>]>;
+   def f32rii :
+     BasicFlagsNVPTXInst<(outs B32:$dst),
+               (ins B32:$a, f32imm:$b, f32imm:$c),
+               (ins FTZFlag:$ftz),
+               OpcStr # "$ftz" # nan_str # ".f32",
+               [(set f32:$dst, (OpNode f32:$a, fpimm:$b, fpimm:$c))]>,
+               Requires<[hasPTX<88>, hasSM<100>]>;
+}
+
 // Template for instructions which take three FP args.  The
 // instructions are named "<OpcStr>.f<Width>" (e.g. "add.f64").
 //
@@ -731,10 +764,10 @@ def fabs_oneuse : OneUse1<fabs>;
 
 def TESTINF_f32r : BasicNVPTXInst<(outs B1:$p), (ins B32:$a),
                              "testp.infinite.f32",
-                             [(set i1:$p, (seteq (fabs_oneuse f32:$a), fpimm_pos_inf<f32>))]>;
+                             [(set i1:$p, (seteq (fabs_oneuse f32:$a), fpimm_pos_inf))]>;
 def TESTINF_f64r : BasicNVPTXInst<(outs B1:$p), (ins B64:$a),
                              "testp.infinite.f64",
-                             [(set i1:$p, (seteq (fabs_oneuse f64:$a), fpimm_pos_inf<f64>))]>;
+                             [(set i1:$p, (seteq (fabs_oneuse f64:$a), fpimm_pos_inf))]>;
 
 //-----------------------------------
 // Integer Arithmetic
@@ -757,8 +790,8 @@ defm SUBCCC : ADD_SUB_INT_CARRY<"subc.cc", sube, commutative = false>;
 
 defm MULT : I3<"mul.lo.s", mul, commutative = true>;
 
-defm MULTHS : I3<"mul.hi.s", mulhs, commutative = true>;
-defm MULTHU : I3<"mul.hi.u", mulhu, commutative = true>;
+defm MUL_HI_S : I3<"mul.hi.s", mulhs, commutative = true>;
+defm MUL_HI_U : I3<"mul.hi.u", mulhu, commutative = true>;
 
 defm SDIV : I3<"div.s", sdiv, commutative = false>;
 defm UDIV : I3<"div.u", udiv, commutative = false>;
@@ -875,22 +908,6 @@ let Predicates = [hasOptEnabled] in {
 // Floating Point Arithmetic
 //-----------------------------------
 
-// Constant 1.0f
-def f32imm_1 : FPImmLeaf<f32, [{
-  return &Imm.getSemantics() == &llvm::APFloat::IEEEsingle() &&
-         Imm.convertToFloat() == 1.0f;
-}]>;
-// Constant 1.0 (double)
-def f64imm_1 : FPImmLeaf<f64, [{
-  return &Imm.getSemantics() == &llvm::APFloat::IEEEdouble() &&
-         Imm.convertToDouble() == 1.0;
-}]>;
-// Constant -1.0 (double)
-def f64imm_neg1 : FPImmLeaf<f64, [{
-  return &Imm.getSemantics() == &llvm::APFloat::IEEEdouble() &&
-         Imm.convertToDouble() == -1.0;
-}]>;
-
 defm FADD : F3_fma_component<"add", fadd>;
 defm FSUB : F3_fma_component<"sub", fsub>;
 defm FMUL : F3_fma_component<"mul", fmul>;
@@ -900,6 +917,20 @@ defm MAX : FMINIMUMMAXIMUM<"max", /* NaN */ false, fmaxnum>;
 defm MIN_NAN : FMINIMUMMAXIMUM<"min", /* NaN */ true, fminimum>;
 defm MAX_NAN : FMINIMUMMAXIMUM<"max", /* NaN */ true, fmaximum>;
 
+def nvptx_fminnum3 : SDNode<"NVPTXISD::FMINNUM3", SDTFPTernaryOp,
+                            [SDNPCommutative]>;
+def nvptx_fmaxnum3 : SDNode<"NVPTXISD::FMAXNUM3", SDTFPTernaryOp,
+                             [SDNPCommutative]>;
+def nvptx_fminimum3 : SDNode<"NVPTXISD::FMINIMUM3", SDTFPTernaryOp,
+                             [SDNPCommutative]>;
+def nvptx_fmaximum3 : SDNode<"NVPTXISD::FMAXIMUM3", SDTFPTernaryOp,
+                             [SDNPCommutative]>;
+
+defm FMIN3 : FMINIMUMMAXIMUM3<"min", /* NaN */ false, nvptx_fminnum3>;
+defm FMAX3 : FMINIMUMMAXIMUM3<"max", /* NaN */ false, nvptx_fmaxnum3>;
+defm FMINNAN3 : FMINIMUMMAXIMUM3<"min", /* NaN */ true, nvptx_fminimum3>;
+defm FMAXNAN3 : FMINIMUMMAXIMUM3<"max", /* NaN */ true, nvptx_fmaximum3>;
+
 defm FABS  : F2<"abs", fabs>;
 defm FNEG  : F2<"neg", fneg>;
 defm FABS_H: F2_Support_Half<"abs", fabs>;
@@ -950,7 +981,7 @@ def FRCP64r :
   BasicNVPTXInst<(outs B64:$dst),
                  (ins B64:$b),
                  "rcp.rn.f64",
-                 [(set f64:$dst, (fdiv f64imm_1, f64:$b))]>;
+                 [(set f64:$dst, (fdiv fpimm_1, f64:$b))]>;
 def FDIV64rr :
   BasicNVPTXInst<(outs B64:$dst),
                  (ins B64:$a, B64:$b),
@@ -964,7 +995,7 @@ def FDIV64ri :
 
 // fdiv will be converted to rcp
 // fneg (fdiv 1.0, X) => fneg (rcp.rn X)
-def : Pat<(fdiv f64imm_neg1, f64:$b),
+def : Pat<(fdiv fpimm_neg_1, f64:$b),
           (FNEGf64 (FRCP64r $b))>;
 
 //
@@ -977,21 +1008,21 @@ def fdiv_approx : PatFrag<(ops node:$a, node:$b),
 }]>;
 
 
-def FRCP32_approx_r :
+def RCP_APPROX_F32_r :
   BasicFlagsNVPTXInst<(outs B32:$dst),
                  (ins B32:$b), (ins FTZFlag:$ftz),
                  "rcp.approx$ftz.f32",
-                 [(set f32:$dst, (fdiv_approx f32imm_1, f32:$b))]>;
+                 [(set f32:$dst, (fdiv_approx fpimm_1, f32:$b))]>;
 
 //
 // F32 Approximate division
 //
-def FDIV32_approx_rr :
+def DIV_APPROX_F32_rr :
   BasicFlagsNVPTXInst<(outs B32:$dst),
                  (ins B32:$a, B32:$b), (ins FTZFlag:$ftz),
                  "div.approx$ftz.f32",
                  [(set f32:$dst, (fdiv_approx f32:$a, f32:$b))]>;
-def FDIV32_approx_ri :
+def DIV_APPROX_F32_ri :
   BasicFlagsNVPTXInst<(outs B32:$dst),
                  (ins B32:$a, f32imm:$b), (ins FTZFlag:$ftz),
                  "div.approx$ftz.f32",
@@ -1008,8 +1039,8 @@ def fdiv_full : PatFrag<(ops node:$a, node:$b),
 }]>;
 
 
-def : Pat<(fdiv_full f32imm_1, f32:$b),
-          (FRCP32_approx_r $b)>;
+def : Pat<(fdiv_full fpimm_1, f32:$b),
+          (RCP_APPROX_F32_r $b)>;
 
 //
 // F32 Semi-accurate division
@@ -1037,7 +1068,7 @@ def FRCP32r_prec :
   BasicFlagsNVPTXInst<(outs B32:$dst),
                  (ins B32:$b), (ins FTZFlag:$ftz),
                  "rcp.rn$ftz.f32",
-                 [(set f32:$dst, (fdiv_ftz f32imm_1, f32:$b))]>;
+                 [(set f32:$dst, (fdiv_ftz fpimm_1, f32:$b))]>;
 //
 // F32 Accurate division
 //
@@ -1052,7 +1083,7 @@ def FDIV32ri_prec :
                  "div.rn$ftz.f32",
                  [(set f32:$dst, (fdiv_ftz f32:$a, fpimm:$b))]>;
 
-def : Pat<(fdiv f32imm_1, f32:$b), (FRCP32r_prec $b, NoFTZ)>;
+def : Pat<(fdiv fpimm_1, f32:$b), (FRCP32r_prec $b, NoFTZ)>;
 def : Pat<(fdiv f32:$a, f32:$b), (FDIV32rr_prec $a, $b, NoFTZ)>;
 def : Pat<(fdiv f32:$a, fpimm:$b), (FDIV32ri_prec $a, fpimm:$b, NoFTZ)>;
 
@@ -1102,9 +1133,8 @@ defm FMA_F64    : FMA<F64RT,    allow_ftz = false>;
 // sin/cos/tanh
 
 class UnaryOpAllowsApproxFn<SDPatternOperator operator>
-    : PatFrag<(ops node:$A),
-              (operator node:$A), [{
-  return allowUnsafeFPMath() || N->getFlags().hasApproximateFuncs();
+    : PatFrag<(ops node:$A), (operator node:$A), [{
+  return N->getFlags().hasApproximateFuncs();
 }]>;
 
 def SIN_APPROX_f32 :
@@ -1475,23 +1505,28 @@ def MmaCode : Operand<i32> {
 // Get pointer to local stack.
 let hasSideEffects = false in {
   def MOV_DEPOT_ADDR :    NVPTXInst<(outs B32:$d), (ins i32imm:$num),
-                                     "mov.b32 \t$d, __local_depot$num;", []>;
+                                     "mov.b32 \t$d, __local_depot$num;">;
   def MOV_DEPOT_ADDR_64 : NVPTXInst<(outs B64:$d), (ins i32imm:$num),
-                                    "mov.b64 \t$d, __local_depot$num;", []>;
+                                    "mov.b64 \t$d, __local_depot$num;">;
 }
 
-
-// copyPhysreg is hard-coded in NVPTXInstrInfo.cpp
-let hasSideEffects = false, isAsCheapAsAMove = true in {
-  let isMoveReg = true in
+let hasSideEffects = false in {
+  let isMoveReg = true, isAsCheapAsAMove = true in
     class MOVr<RegisterClass RC, string OpStr> :
       BasicNVPTXInst<(outs RC:$dst), (ins RC:$src), "mov." # OpStr>;
 
-  let isMoveImm = true in
+  let isMoveImm = true, isAsCheapAsAMove = true in
     class MOVi<RegTyInfo t, string suffix> :
       BasicNVPTXInst<(outs t.RC:$dst), (ins t.Imm:$src),
               "mov." # suffix,
               [(set t.Ty:$dst, t.ImmNode:$src)]>;
+
+  // We don't want to set isAsCheapAsAMove to true for these instructions as
+  // this would prevent CSE and resulted in regressions (see discussion after
+  // PR-145581 in llvm-project).
+  class MovSymInst<RegTyInfo t> :
+    BasicNVPTXInst<(outs t.RC:$dst), (ins Operand<t.Ty>:$src),
+                   "mov.b" # t.Size>;
 }
 
 def MOV_B1_r : MOVr<B1, "pred">;
@@ -1509,6 +1544,9 @@ def MOV_BF16_i : MOVi<BF16RT, "b16">;
 def MOV_F32_i  : MOVi<F32RT, "b32">;
 def MOV_F64_i  : MOVi<F64RT, "b64">;
 
+def MOV_B32_sym : MovSymInst<I32RT>;
+def MOV_B64_sym : MovSymInst<I64RT>;
+
 
 def to_tglobaladdr : SDNodeXForm<globaladdr, [{
   return CurDAG->getTargetGlobalAddress(N->getGlobal(), SDLoc(N),
@@ -1525,17 +1563,17 @@ def to_tframeindex : SDNodeXForm<frameindex, [{
   return CurDAG->getTargetFrameIndex(N->getIndex(), N->getValueType(0));
 }]>;
 
-def : Pat<(i32 globaladdr:$dst), (MOV_B32_i (to_tglobaladdr $dst))>;
-def : Pat<(i64 globaladdr:$dst), (MOV_B64_i (to_tglobaladdr $dst))>;
+def : Pat<(i32 globaladdr:$dst), (MOV_B32_sym (to_tglobaladdr $dst))>;
+def : Pat<(i64 globaladdr:$dst), (MOV_B64_sym (to_tglobaladdr $dst))>;
 
-def : Pat<(i32 externalsym:$dst), (MOV_B32_i (to_texternsym $dst))>;
-def : Pat<(i64 externalsym:$dst), (MOV_B64_i (to_texternsym $dst))>;
+def : Pat<(i32 externalsym:$dst), (MOV_B32_sym (to_texternsym $dst))>;
+def : Pat<(i64 externalsym:$dst), (MOV_B64_sym (to_texternsym $dst))>;
 
 //---- Copy Frame Index ----
 def LEA_ADDRi :   NVPTXInst<(outs B32:$dst), (ins ADDR:$addr),
-                            "add.u32 \t$dst, ${addr:add};", []>;
+                            "add.u32 \t$dst, ${addr:add};">;
 def LEA_ADDRi64 : NVPTXInst<(outs B64:$dst), (ins ADDR:$addr),
-                            "add.u64 \t$dst, ${addr:add};", []>;
+                            "add.u64 \t$dst, ${addr:add};">;
 
 def : Pat<(i32 frameindex:$fi), (LEA_ADDRi (to_tframeindex $fi), 0)>;
 def : Pat<(i64 frameindex:$fi), (LEA_ADDRi64 (to_tframeindex $fi), 0)>;
@@ -1600,12 +1638,12 @@ foreach is_convergent = [0, 1] in {
       NVPTXInst<(outs),
                 (ins ADDR_base:$addr, CallOperand:$rets, CallOperand:$params, 
                      i32imm:$proto),
-                "call${rets:RetList} $addr, (${params:ParamList}), prototype_$proto;", []>;
+                "call${rets:RetList} $addr, (${params:ParamList}), prototype_$proto;">;
 
     def CALL_UNI # convergent_suffix :
       NVPTXInst<(outs),
                 (ins ADDR_base:$addr, CallOperand:$rets, CallOperand:$params),
-                "call.uni${rets:RetList} $addr, (${params:ParamList});", []>;
+                "call.uni${rets:RetList} $addr, (${params:ParamList});">;
   }
 
   defvar call_inst = !cast<NVPTXInst>("CALL" # convergent_suffix);
@@ -1621,10 +1659,10 @@ foreach is_convergent = [0, 1] in {
 
 def DECLARE_PARAM_array :
   NVPTXInst<(outs), (ins i32imm:$a, i32imm:$align, i32imm:$size),
-            ".param .align $align .b8 \t$a[$size];", []>;
+            ".param .align $align .b8 \t$a[$size];">;
 def DECLARE_PARAM_scalar :
   NVPTXInst<(outs), (ins i32imm:$a, i32imm:$size),
-            ".param .b$size \t$a;", []>;
+            ".param .b$size \t$a;">;
 
 def : Pat<(declare_array_param externalsym:$a, imm:$align, imm:$size),
           (DECLARE_PARAM_array (to_texternsym $a), imm:$align, imm:$size)>;
@@ -1697,7 +1735,7 @@ class LD<NVPTXRegClass regclass>
     (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, AtomicCode:$Sign,
          i32imm:$fromWidth, ADDR:$addr),
     "ld${sem:sem}${scope:scope}${addsp:addsp}.${Sign:sign}$fromWidth "
-    "\t$dst, [$addr];", []>;
+    "\t$dst, [$addr];">;
 
 let mayLoad=1, hasSideEffects=0 in {
   def LD_i16 : LD<B16>;
@@ -1712,7 +1750,7 @@ class ST<DAGOperand O>
          AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$toWidth,
          ADDR:$addr),
     "st${sem:sem}${scope:scope}${addsp:addsp}.b$toWidth"
-    " \t[$addr], $src;", []>;
+    " \t[$addr], $src;">;
 
 let mayStore=1, hasSideEffects=0 in {
   def ST_i16 : ST<RI16>;
@@ -1729,13 +1767,13 @@ multiclass LD_VEC<NVPTXRegClass regclass, bit support_v8 = false> {
     (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
          AtomicCode:$Sign, i32imm:$fromWidth, ADDR:$addr),
     "ld${sem:sem}${scope:scope}${addsp:addsp}.v2.${Sign:sign}$fromWidth "
-    "\t{{$dst1, $dst2}}, [$addr];", []>;
+    "\t{{$dst1, $dst2}}, [$addr];">;
   def _v4 : NVPTXInst<
     (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
     (ins AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp,
          AtomicCode:$Sign, i32imm:$fromWidth, ADDR:$addr),
     "ld${sem:sem}${scope:scope}${addsp:addsp}.v4.${Sign:sign}$fromWidth "
-    "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr];", []>;
+    "\t{{$dst1, $dst2, $dst3, $dst4}}, [$addr];">;
   if support_v8 then
     def _v8 : NVPTXInst<
       (outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4,
@@ -1744,7 +1782,7 @@ multiclass LD_VEC<NVPTXRegClass regclass, bit support_v8 = false> {
            i32imm:$fromWidth, ADDR:$addr),
       "ld${sem:sem}${scope:scope}${addsp:addsp}.v8.${Sign:sign}$fromWidth "
       "\t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, "
-      "[$addr];", []>;
+      "[$addr];">;
 }
 let mayLoad=1, hasSideEffects=0 in {
   defm LDV_i16 : LD_VEC<B16>;
@@ -1759,14 +1797,14 @@ multiclass ST_VEC<DAGOperand O, bit support_v8 = false> {
          AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$fromWidth,
          ADDR:$addr),
     "st${sem:sem}${scope:scope}${addsp:addsp}.v2.b$fromWidth "
-    "\t[$addr], {{$src1, $src2}};", []>;
+    "\t[$addr], {{$src1, $src2}};">;
   def _v4 : NVPTXInst<
     (outs),
     (ins O:$src1, O:$src2, O:$src3, O:$src4,
          AtomicCode:$sem, AtomicCode:$scope, AtomicCode:$addsp, i32imm:$fromWidth,
          ADDR:$addr),
     "st${sem:sem}${scope:scope}${addsp:addsp}.v4.b$fromWidth "
-    "\t[$addr], {{$src1, $src2, $src3, $src4}};", []>;
+    "\t[$addr], {{$src1, $src2, $src3, $src4}};">;
   if support_v8 then
     def _v8 : NVPTXInst<
       (outs),
@@ -1776,7 +1814,7 @@ multiclass ST_VEC<DAGOperand O, bit support_v8 = false> {
            ADDR:$addr),
       "st${sem:sem}${scope:scope}${addsp:addsp}.v8.b$fromWidth "
       "\t[$addr], "
-      "{{$src1, $src2, $src3, $src4, $src5, $src6, $src7, $src8}};", []>;
+      "{{$src1, $src2, $src3, $src4, $src5, $src6, $src7, $src8}};">;
 }
 
 let mayStore=1, hasSideEffects=0 in {
@@ -1971,60 +2009,52 @@ let hasSideEffects = false in {
   def V4I16toI64 : NVPTXInst<(outs B64:$d),
                              (ins B16:$s1, B16:$s2,
                                   B16:$s3, B16:$s4),
-                             "mov.b64 \t$d, {{$s1, $s2, $s3, $s4}};", []>;
+                             "mov.b64 \t$d, {{$s1, $s2, $s3, $s4}};">;
   def V2I16toI32 : NVPTXInst<(outs B32:$d),
                              (ins B16:$s1, B16:$s2),
-                             "mov.b32 \t$d, {{$s1, $s2}};", []>;
+                             "mov.b32 \t$d, {{$s1, $s2}};">;
   def V2I32toI64 : NVPTXInst<(outs B64:$d),
                              (ins B32:$s1, B32:$s2),
-                             "mov.b64 \t$d, {{$s1, $s2}};", []>;
+                             "mov.b64 \t$d, {{$s1, $s2}};">;
   def V2I64toI128 : NVPTXInst<(outs B128:$d),
                               (ins B64:$s1, B64:$s2),
-                              "mov.b128 \t$d, {{$s1, $s2}};", []>;
+                              "mov.b128 \t$d, {{$s1, $s2}};">;
 
   // unpack a larger int register to a set of smaller int registers
   def I64toV4I16 : NVPTXInst<(outs B16:$d1, B16:$d2,
                                    B16:$d3, B16:$d4),
                              (ins B64:$s),
-                             "mov.b64 \t{{$d1, $d2, $d3, $d4}}, $s;", []>;
+                             "mov.b64 \t{{$d1, $d2, $d3, $d4}}, $s;">;
   def I32toV2I16 : NVPTXInst<(outs B16:$d1, B16:$d2),
                              (ins B32:$s),
-                             "mov.b32 \t{{$d1, $d2}}, $s;", []>;
+                             "mov.b32 \t{{$d1, $d2}}, $s;">;
   def I64toV2I32 : NVPTXInst<(outs B32:$d1, B32:$d2),
                              (ins B64:$s),
-                             "mov.b64 \t{{$d1, $d2}}, $s;", []>;
+                             "mov.b64 \t{{$d1, $d2}}, $s;">;
   def I128toV2I64: NVPTXInst<(outs B64:$d1, B64:$d2),
                               (ins B128:$s),
-                              "mov.b128 \t{{$d1, $d2}}, $s;", []>;
+                              "mov.b128 \t{{$d1, $d2}}, $s;">;
 
-  def I32toI16H  : NVPTXInst<(outs B16:$high),
-                             (ins B32:$s),
-                             "{{ .reg .b16 tmp; mov.b32 {tmp, $high}, $s; }}",
-                             []>;
-  def I32toI16L  : NVPTXInst<(outs B16:$low),
-                             (ins B32:$s),
-                             "{{ .reg .b16 tmp; mov.b32 {$low, tmp}, $s; }}",
-                             []>;
-  def I64toI32H  : NVPTXInst<(outs B32:$high),
-                             (ins B64:$s),
-                             "{{ .reg .b32 tmp; mov.b64 {tmp, $high}, $s; }}",
-                             []>;
-  def I64toI32L  : NVPTXInst<(outs B32:$low),
-                             (ins B64:$s),
-                             "{{ .reg .b32 tmp; mov.b64 {$low, tmp}, $s; }}",
-                             []>;
+  def I32toI16H  : NVPTXInst<(outs B16:$high), (ins B32:$s),
+                             "{{ .reg .b16 tmp; mov.b32 {tmp, $high}, $s; }}">;
+  def I32toI16L  : NVPTXInst<(outs B16:$low), (ins B32:$s),
+                             "{{ .reg .b16 tmp; mov.b32 {$low, tmp}, $s; }}">;
+  def I64toI32H  : NVPTXInst<(outs B32:$high), (ins B64:$s),
+                            "{{ .reg .b32 tmp; mov.b64 {tmp, $high}, $s; }}">;
+  def I64toI32L  : NVPTXInst<(outs B32:$low), (ins B64:$s),
+                             "{{ .reg .b32 tmp; mov.b64 {$low, tmp}, $s; }}">;
 
   // PTX 7.1 lets you avoid a temp register and just use _ as a "sink" for the
   // unused high/low part.
   let Predicates = [hasPTX<71>] in {
     def I32toI16H_Sink  : NVPTXInst<(outs B16:$high), (ins B32:$s),
-                              "mov.b32 \t{{_, $high}}, $s;", []>;
+                              "mov.b32 \t{{_, $high}}, $s;">;
     def I32toI16L_Sink  : NVPTXInst<(outs B16:$low), (ins B32:$s),
-                              "mov.b32 \t{{$low, _}}, $s;", []>;
+                              "mov.b32 \t{{$low, _}}, $s;">;
     def I64toI32H_Sink  : NVPTXInst<(outs B32:$high), (ins B64:$s),
-                              "mov.b64 \t{{_, $high}}, $s;", []>;
+                              "mov.b64 \t{{_, $high}}, $s;">;
     def I64toI32L_Sink  : NVPTXInst<(outs B32:$low), (ins B64:$s),
-                              "mov.b64 \t{{$low, _}}, $s;", []>;
+                              "mov.b64 \t{{$low, _}}, $s;">;
   }
 }
 
@@ -2382,10 +2412,6 @@ foreach scope = ["sys", "gpu", "cluster", "cta"] in {
   def atomic_thread_fence_release_#scope: NVPTXFenceInst<scope, "release", hasPTX<87>>;
 }
 
-def fpimm_any_zero : FPImmLeaf<fAny, [{
-  return Imm.isZero();
-}]>;
-
 // Perform substitution if fma only has one use, and also if instruction has
 // nnan instruction flag or if the TM has NoNaNsFPMath
 def NVPTX_fma_oneuse_and_nnan : PatFrag<(ops node:$a, node:$b, node:$c),
@@ -2407,11 +2433,11 @@ class FMARELUInst<RegTyInfo t, bit allow_ftz, PatFrag zero_pat>
                    [(set t.Ty:$dst, (NVPTX_fmaxnum_nsz (NVPTX_fma_oneuse_and_nnan t.Ty:$a, t.Ty:$b, t.Ty:$c), zero_pat))]>;
 
 let Predicates = [useFP16Math, hasPTX<70>, hasSM<80>] in {
-  def FMARELU_F16 : FMARELUInst<F16RT, true, fpimm_any_zero>;
+  def FMARELU_F16 : FMARELUInst<F16RT, true, fpimm_0>;
   def FMARELU_F16X2 : FMARELUInst<F16X2RT, true, zeroinitializer<v2f16>>;
 }
 
 let Predicates = [hasBF16Math, hasPTX<70>, hasSM<80>] in {
-  def FMARELU_BF16 : FMARELUInst<BF16RT, false, fpimm_any_zero>;
+  def FMARELU_BF16 : FMARELUInst<BF16RT, false, fpimm_0>;
   def FMARELU_BF16X2 : FMARELUInst<BF16X2RT, false, zeroinitializer<v2bf16>>;
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index d33719236b172..cba14066f0c0b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -6,38 +6,24 @@
 //
 //===----------------------------------------------------------------------===//
 
-def immFloat0 : PatLeaf<(fpimm), [{
-    float f = (float)N->getValueAPF().convertToFloat();
-    return (f==0.0f);
-}]>;
-
-def immFloat1 : PatLeaf<(fpimm), [{
-    float f = (float)N->getValueAPF().convertToFloat();
-    return (f==1.0f);
-}]>;
-
-def immDouble0 : PatLeaf<(fpimm), [{
-    double d = (double)N->getValueAPF().convertToDouble();
-    return (d==0.0);
-}]>;
-
-def immDouble1 : PatLeaf<(fpimm), [{
-    double d = (double)N->getValueAPF().convertToDouble();
-    return (d==1.0);
-}]>;
-
 def AS_match {
   code generic = [{
-   return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GENERIC);
+   return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_GENERIC;
   }];
   code shared = [{
-   return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_SHARED);
+   return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_SHARED;
   }];
   code shared_cluster = [{
-   return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_SHARED_CLUSTER);
+   return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_SHARED_CLUSTER;
   }];
   code global = [{
-   return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GLOBAL);
+   return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_GLOBAL;
+  }];
+  code const = [{
+   return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_CONST;
+  }];
+  code param = [{
+   return cast<MemSDNode>(N)->getAddressSpace() == llvm::ADDRESS_SPACE_PARAM;
   }];
 }
 
@@ -653,22 +639,22 @@ multiclass CP_ASYNC_BULK_TENSOR_G2S_INTR<int dim, bit is_shared32, string mode>
 
   def "" : NVPTXInst<(outs),
             !con((ins rc:$dst, rc:$mbar, B64:$tmap), dims_dag, im2col_dag, (ins CTAGroupFlags:$cg)),
-            !strconcat(G2S_STRINGS<dim, mode, 0, 0>.inst_name, asm_str, ";"), []>,
+            !strconcat(G2S_STRINGS<dim, mode, 0, 0>.inst_name, asm_str, ";")>,
             Requires<[hasPTX<80>, hasSM<90>]>;
   def _MC : NVPTXInst<(outs),
                   !con((ins rc:$dst, rc:$mbar, B64:$tmap), dims_dag, im2col_dag,
                        (ins B16:$mc, CTAGroupFlags:$cg)),
-                  !strconcat(G2S_STRINGS<dim, mode, 1, 0>.inst_name, asm_str, ", $mc;"), []>,
+                  !strconcat(G2S_STRINGS<dim, mode, 1, 0>.inst_name, asm_str, ", $mc;")>,
                   Requires<[hasPTX<80>, hasSM<90>]>;
   def _CH : NVPTXInst<(outs),
                   !con((ins rc:$dst, rc:$mbar, B64:$tmap), dims_dag, im2col_dag,
                        (ins B64:$ch, CTAGroupFlags:$cg)),
-                  !strconcat(G2S_STRINGS<dim, mode, 0, 1>.inst_name, asm_str, ", $ch;"), []>,
+                  !strconcat(G2S_STRINGS<dim, mode, 0, 1>.inst_name, asm_str, ", $ch;")>,
                   Requires<[hasPTX<80>, hasSM<90>]>;
   def _MC_CH : NVPTXInst<(outs),
                      !con((ins rc:$dst, rc:$mbar, B64:$tmap), dims_dag, im2col_dag,
                           (ins B16:$mc, B64:$ch, CTAGroupFlags:$cg)),
-                     !strconcat(G2S_STRINGS<dim, mode, 1, 1>.inst_name, asm_str, ", $mc, $ch;"), []>,
+                     !strconcat(G2S_STRINGS<dim, mode, 1, 1>.inst_name, asm_str, ", $mc, $ch;")>,
                      Requires<[hasPTX<80>, hasSM<90>]>;
 }
 
@@ -870,11 +856,11 @@ multiclass CP_ASYNC_BULK_TENSOR_REDUCE_INTR<int dim, bit shared32, string mode>
 
   def "" : NVPTXInst<(outs),
             !con((ins rc:$src, B64:$tmap), dims_dag, (ins TMAReductionFlags:$red_op)),
-            !strconcat(prefix, "${red_op}", suffix, asm_str, ";"), []>,
+            !strconcat(prefix, "${red_op}", suffix, asm_str, ";")>,
             Requires<[hasPTX<80>, hasSM<90>]>;
   def _CH : NVPTXInst<(outs),
                   !con((ins rc:$src, B64:$tmap), dims_dag, (ins B64:$ch, TMAReductionFlags:$red_op)),
-                  !strconcat(prefix, "${red_op}", suffix, ".L2::cache_hint", asm_str, ", $ch;"), []>,
+                  !strconcat(prefix, "${red_op}", suffix, ".L2::cache_hint", asm_str, ", $ch;")>,
                   Requires<[hasPTX<80>, hasSM<90>]>;
 }
 
@@ -950,33 +936,47 @@ foreach dim = 3...5 in {
 defm TMA_TENSOR_PF_TILE_GATHER4_2D : TMA_TENSOR_PREFETCH_INTR<5, "tile_gather4",
                                      [hasTMACTAGroupSupport]>;
 
-//Prefetch and Prefetchu 
+//Prefetchu and Prefetch
 
-let Predicates = [hasPTX<80>, hasSM<90>] in {
-  class PREFETCH_INTRS<string InstName> :
-            BasicNVPTXInst<(outs), (ins ADDR:$addr),
-            InstName,
-            [(!cast<Intrinsic>(!strconcat("int_nvvm_",
-            !subst(".", "_", InstName))) addr:$addr)]>;
+defvar frag_pat = (int_nvvm_prefetch_tensormap node:$addr);
 
-  def PREFETCH_L1 : PREFETCH_INTRS<"prefetch.L1">;
-  def PREFETCH_L2 : PREFETCH_INTRS<"prefetch.L2">;
-  def PREFETCH_GLOBAL_L1 : PREFETCH_INTRS<"prefetch.global.L1">;
-  def PREFETCH_LOCAL_L1  : PREFETCH_INTRS<"prefetch.local.L1">;
-  def PREFETCH_GLOBAL_L2 : PREFETCH_INTRS<"prefetch.global.L2">;
-  def PREFETCH_LOCAL_L2 : PREFETCH_INTRS<"prefetch.local.L2">;
-
-  def PREFETCH_GLOBAL_L2_EVICT_NORMAL : BasicNVPTXInst<(outs), (ins ADDR:$addr),
-                                        "prefetch.global.L2::evict_normal",
-                                        [(int_nvvm_prefetch_global_L2_evict_normal addr:$addr)]>;
+multiclass PREFETCH_TENSORMAP_PATFRAG<string suffix, code predicate> {
+  def !tolower(suffix) : PatFrag<!setdagop(frag_pat, ops), frag_pat, predicate>;
+}
 
-  def PREFETCH_GLOBAL_L2_EVICT_LAST   : BasicNVPTXInst<(outs), (ins ADDR:$addr),
-                                        "prefetch.global.L2::evict_last",
-                                        [(int_nvvm_prefetch_global_L2_evict_last addr:$addr)]>;
+defm prefetch_tensormap_ : PREFETCH_TENSORMAP_PATFRAG<"CONST", AS_match.const>;
+defm prefetch_tensormap_ : PREFETCH_TENSORMAP_PATFRAG<"GENERIC", AS_match.generic>;
+defm prefetch_tensormap_ : PREFETCH_TENSORMAP_PATFRAG<"PARAM", AS_match.param>;
 
-  def PREFETCHU_L1 : PREFETCH_INTRS<"prefetchu.L1">;
+multiclass PREFETCH_TENSORMAP_INST<string addrspace_name, PatFrag pattern_frag> {
+  def "" : BasicNVPTXInst<(outs), (ins ADDR:$addr),
+           "prefetch" # addrspace_name # ".tensormap",
+           [(pattern_frag addr:$addr)]>,
+           Requires<[hasPTX<80>, hasSM<90>]>;
 }
 
+defm PREFETCH_CONST_TENSORMAP   : PREFETCH_TENSORMAP_INST<".const", prefetch_tensormap_const>;
+defm PREFETCH_GENERIC_TENSORMAP : PREFETCH_TENSORMAP_INST<"", prefetch_tensormap_generic>;
+defm PREFETCH_PARAM_TENSORMAP   : PREFETCH_TENSORMAP_INST<".param", prefetch_tensormap_param>;
+  
+class PREFETCH_INTRS<string InstName, Intrinsic Intr> :
+          BasicNVPTXInst<(outs), (ins ADDR:$addr),
+          InstName,
+          [(Intr addr:$addr)]>,
+          Requires<[hasPTX<80>, hasSM<90>]>;
+
+def PREFETCHU_L1 : PREFETCH_INTRS<"prefetchu.L1", int_nvvm_prefetchu_L1>;   
+def PREFETCH_L1 : PREFETCH_INTRS<"prefetch.L1", int_nvvm_prefetch_L1>;
+def PREFETCH_L2 : PREFETCH_INTRS<"prefetch.L2", int_nvvm_prefetch_L2>;
+def PREFETCH_GLOBAL_L1 : PREFETCH_INTRS<"prefetch.global.L1", int_nvvm_prefetch_global_L1>;
+def PREFETCH_LOCAL_L1 : PREFETCH_INTRS<"prefetch.local.L1", int_nvvm_prefetch_local_L1>;
+def PREFETCH_GLOBAL_L2 : PREFETCH_INTRS<"prefetch.global.L2", int_nvvm_prefetch_global_L2>;
+def PREFETCH_LOCAL_L2 : PREFETCH_INTRS<"prefetch.local.L2", int_nvvm_prefetch_local_L2>;
+def PREFETCH_GLOBAL_L2_EVICT_NORMAL : PREFETCH_INTRS<"prefetch.global.L2::evict_normal", 
+                                      int_nvvm_prefetch_global_L2_evict_normal>;
+def PREFETCH_GLOBAL_L2_EVICT_LAST : PREFETCH_INTRS<"prefetch.global.L2::evict_last", 
+                                    int_nvvm_prefetch_global_L2_evict_last>;
+
 //Applypriority intrinsics
 class APPLYPRIORITY_L2_INTRS<string addrspace> :
           BasicNVPTXInst<(outs), (ins ADDR:$addr, B64:$size),
@@ -1092,30 +1092,30 @@ let Predicates = [hasPTX<70>, hasSM<80>] in {
 // max(0.0, min(x, 1.0)) is 1.0 while sat(x) is 0.
 // Same story for fmax, fmin.
 
-def : Pat<(int_nvvm_fmin_f immFloat1,
-            (int_nvvm_fmax_f immFloat0, f32:$a)),
+def : Pat<(int_nvvm_fmin_f fpimm_1,
+            (int_nvvm_fmax_f fpimm_0, f32:$a)),
           (CVT_f32_f32 $a, CvtSAT)>;
-def : Pat<(int_nvvm_fmin_f immFloat1,
-            (int_nvvm_fmax_f f32:$a, immFloat0)),
+def : Pat<(int_nvvm_fmin_f fpimm_1,
+            (int_nvvm_fmax_f f32:$a, fpimm_0)),
           (CVT_f32_f32 $a, CvtSAT)>;
 def : Pat<(int_nvvm_fmin_f
-            (int_nvvm_fmax_f immFloat0, f32:$a), immFloat1),
+            (int_nvvm_fmax_f fpimm_0, f32:$a), fpimm_1),
           (CVT_f32_f32 $a, CvtSAT)>;
 def : Pat<(int_nvvm_fmin_f
-            (int_nvvm_fmax_f f32:$a, immFloat0), immFloat1),
+            (int_nvvm_fmax_f f32:$a, fpimm_0), fpimm_1),
           (CVT_f32_f32 $a, CvtSAT)>;
 
-def : Pat<(int_nvvm_fmin_d immDouble1,
-            (int_nvvm_fmax_d immDouble0, f64:$a)),
+def : Pat<(int_nvvm_fmin_d fpimm_1,
+            (int_nvvm_fmax_d fpimm_0, f64:$a)),
           (CVT_f64_f64 $a, CvtSAT)>;
-def : Pat<(int_nvvm_fmin_d immDouble1,
-            (int_nvvm_fmax_d f64:$a, immDouble0)),
+def : Pat<(int_nvvm_fmin_d fpimm_1,
+            (int_nvvm_fmax_d f64:$a, fpimm_0)),
           (CVT_f64_f64 $a, CvtSAT)>;
 def : Pat<(int_nvvm_fmin_d
-            (int_nvvm_fmax_d immDouble0, f64:$a), immDouble1),
+            (int_nvvm_fmax_d fpimm_0, f64:$a), fpimm_1),
           (CVT_f64_f64 $a, CvtSAT)>;
 def : Pat<(int_nvvm_fmin_d
-            (int_nvvm_fmax_d f64:$a, immDouble0), immDouble1),
+            (int_nvvm_fmax_d f64:$a, fpimm_0), fpimm_1),
           (CVT_f64_f64 $a, CvtSAT)>;
 
 
@@ -1309,12 +1309,12 @@ defm INT_NVVM_FMAN : MIN_MAX<"max">;
 // Multiplication
 //
 
-def INT_NVVM_MULHI_S : F_MATH_2<"mul.hi.s16", B16, B16, B16, int_nvvm_mulhi_s>;
-def INT_NVVM_MULHI_US : F_MATH_2<"mul.hi.u16", B16, B16, B16, int_nvvm_mulhi_us>;
-def INT_NVVM_MULHI_I : F_MATH_2<"mul.hi.s32", B32, B32, B32, int_nvvm_mulhi_i>;
-def INT_NVVM_MULHI_UI : F_MATH_2<"mul.hi.u32", B32, B32, B32, int_nvvm_mulhi_ui>;
-def INT_NVVM_MULHI_LL : F_MATH_2<"mul.hi.s64", B64, B64, B64, int_nvvm_mulhi_ll>;
-def INT_NVVM_MULHI_ULL : F_MATH_2<"mul.hi.u64", B64, B64, B64, int_nvvm_mulhi_ull>;
+def : Pat<(int_nvvm_mulhi_s i16:$a, i16:$b), (MUL_HI_S16rr $a, $b)>;
+def : Pat<(int_nvvm_mulhi_us i16:$a, i16:$b), (MUL_HI_U16rr $a, $b)>;
+def : Pat<(int_nvvm_mulhi_i i32:$a, i32:$b), (MUL_HI_S32rr $a, $b)>;
+def : Pat<(int_nvvm_mulhi_ui i32:$a, i32:$b), (MUL_HI_U32rr $a, $b)>;
+def : Pat<(int_nvvm_mulhi_ll i64:$a, i64:$b), (MUL_HI_S64rr $a, $b)>;
+def : Pat<(int_nvvm_mulhi_ull i64:$a, i64:$b), (MUL_HI_U64rr $a, $b)>;
 
 def INT_NVVM_MUL_RN_FTZ_F : F_MATH_2<"mul.rn.ftz.f32", B32, B32, B32, int_nvvm_mul_rn_ftz_f>;
 def INT_NVVM_MUL_RN_F : F_MATH_2<"mul.rn.f32", B32, B32, B32, int_nvvm_mul_rn_f>;
@@ -1337,8 +1337,8 @@ def INT_NVVM_MUL24_UI : F_MATH_2<"mul24.lo.u32", B32, B32, B32, int_nvvm_mul24_u
 // Div
 //
 
-def INT_NVVM_DIV_APPROX_FTZ_F : F_MATH_2<"div.approx.ftz.f32", B32, B32, B32, int_nvvm_div_approx_ftz_f>;
-def INT_NVVM_DIV_APPROX_F : F_MATH_2<"div.approx.f32", B32, B32, B32, int_nvvm_div_approx_f>;
+def : Pat<(int_nvvm_div_approx_ftz_f f32:$a, f32:$b), (DIV_APPROX_F32_rr $a, $b, FTZ)>;
+def : Pat<(int_nvvm_div_approx_f f32:$a, f32:$b), (DIV_APPROX_F32_rr $a, $b, NoFTZ)>;
 
 def INT_NVVM_DIV_RN_FTZ_F : F_MATH_2<"div.rn.ftz.f32", B32, B32, B32, int_nvvm_div_rn_ftz_f>;
 def INT_NVVM_DIV_RN_F     : F_MATH_2<"div.rn.f32", B32, B32, B32, int_nvvm_div_rn_f>;
@@ -1643,13 +1643,13 @@ def : Pat<(int_nvvm_rsqrt_approx_d f64:$a), (RSQRT_APPROX_f64 $a, NoFTZ)>;
 
 // 1.0f / sqrt_approx -> rsqrt_approx
 let Predicates = [doRsqrtOpt] in {
-  def : Pat<(fdiv f32imm_1, (int_nvvm_sqrt_approx_f f32:$a)),
+  def : Pat<(fdiv fpimm_1, (int_nvvm_sqrt_approx_f f32:$a)),
           (RSQRT_APPROX_f32 $a, NoFTZ)>;
-  def : Pat<(fdiv f32imm_1, (int_nvvm_sqrt_approx_ftz_f f32:$a)),
+  def : Pat<(fdiv fpimm_1, (int_nvvm_sqrt_approx_ftz_f f32:$a)),
           (RSQRT_APPROX_f32 $a, FTZ)>;
 
   // same for int_nvvm_sqrt_f when non-precision sqrt is requested
-  def : Pat<(fdiv f32imm_1, (fsqrt_approx f32:$a)),
+  def : Pat<(fdiv fpimm_1, (fsqrt_approx f32:$a)),
           (RSQRT_APPROX_f32 $a)>;
 }
 //
@@ -2211,7 +2211,7 @@ defm INT_PTX_SATOM_XOR  : ATOM2_bitwise_impl<"xor">;
 
 class LDU_G<NVPTXRegClass regclass>
   :  NVPTXInst<(outs regclass:$result), (ins i32imm:$fromWidth, ADDR:$src),
-               "ldu.global.b$fromWidth \t$result, [$src];", []>;
+               "ldu.global.b$fromWidth \t$result, [$src];">;
 
 def LDU_GLOBAL_i16 : LDU_G<B16>;
 def LDU_GLOBAL_i32 : LDU_G<B32>;
@@ -2223,13 +2223,13 @@ def LDU_GLOBAL_i64 : LDU_G<B64>;
 class VLDU_G_ELE_V2<NVPTXRegClass regclass>
   : NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
               (ins i32imm:$fromWidth, ADDR:$src),
-              "ldu.global.v2.b$fromWidth \t{{$dst1, $dst2}}, [$src];", []>;
+              "ldu.global.v2.b$fromWidth \t{{$dst1, $dst2}}, [$src];">;
 
 
 class VLDU_G_ELE_V4<NVPTXRegClass regclass>
   : NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4),
               (ins i32imm:$fromWidth, ADDR:$src),
-               "ldu.global.v4.b$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", []>;
+               "ldu.global.v4.b$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];">;
 
 
 def LDU_GLOBAL_v2i16 : VLDU_G_ELE_V2<B16>;
@@ -2250,9 +2250,8 @@ def LDU_GLOBAL_v4i32 : VLDU_G_ELE_V4<B32>;
 
 class LDG_G<NVPTXRegClass regclass>
   : NVPTXInst<(outs regclass:$result), (ins AtomicCode:$Sign, i32imm:$fromWidth, ADDR:$src),
-               "ld.global.nc.${Sign:sign}$fromWidth \t$result, [$src];", []>;
+               "ld.global.nc.${Sign:sign}$fromWidth \t$result, [$src];">;
 
-def LD_GLOBAL_NC_i8  : LDG_G<B16>;
 def LD_GLOBAL_NC_i16 : LDG_G<B16>;
 def LD_GLOBAL_NC_i32 : LDG_G<B32>;
 def LD_GLOBAL_NC_i64 : LDG_G<B64>;
@@ -2263,19 +2262,19 @@ def LD_GLOBAL_NC_i64 : LDG_G<B64>;
 class VLDG_G_ELE_V2<NVPTXRegClass regclass> :
   NVPTXInst<(outs regclass:$dst1, regclass:$dst2),
             (ins AtomicCode:$Sign, i32imm:$fromWidth, ADDR:$src),
-            "ld.global.nc.v2.${Sign:sign}$fromWidth \t{{$dst1, $dst2}}, [$src];", []>;
+            "ld.global.nc.v2.${Sign:sign}$fromWidth \t{{$dst1, $dst2}}, [$src];">;
 
 
 class VLDG_G_ELE_V4<NVPTXRegClass regclass> :
   NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4), 
             (ins AtomicCode:$Sign, i32imm:$fromWidth, ADDR:$src),
-            "ld.global.nc.v4.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];", []>;
+            "ld.global.nc.v4.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4}}, [$src];">;
 
 class VLDG_G_ELE_V8<NVPTXRegClass regclass> :
   NVPTXInst<(outs regclass:$dst1, regclass:$dst2, regclass:$dst3, regclass:$dst4,
                   regclass:$dst5, regclass:$dst6, regclass:$dst7, regclass:$dst8),
              (ins AtomicCode:$Sign, i32imm:$fromWidth, ADDR:$src),
-             "ld.global.nc.v8.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, [$src];", []>;
+             "ld.global.nc.v8.${Sign:sign}$fromWidth \t{{$dst1, $dst2, $dst3, $dst4, $dst5, $dst6, $dst7, $dst8}}, [$src];">;
 
 // FIXME: 8-bit LDG should be fixed once LDG/LDU nodes are made into proper loads.
 def LD_GLOBAL_NC_v2i16 : VLDG_G_ELE_V2<B16>;
@@ -3520,20 +3519,13 @@ multiclass SULD_1D<string inst, NVPTXRegClass outtype> {
   def _I : SULD_1D_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_1D_I8_CLAMP : SULD_1D<"suld.b.1d.b8.clamp", B16>;
-defm SULD_1D_I16_CLAMP : SULD_1D<"suld.b.1d.b16.clamp", B16>;
-defm SULD_1D_I32_CLAMP : SULD_1D<"suld.b.1d.b32.clamp", B32>;
-defm SULD_1D_I64_CLAMP : SULD_1D<"suld.b.1d.b64.clamp", B64>;
-
-defm SULD_1D_I8_TRAP : SULD_1D<"suld.b.1d.b8.trap", B16>;
-defm SULD_1D_I16_TRAP : SULD_1D<"suld.b.1d.b16.trap", B16>;
-defm SULD_1D_I32_TRAP : SULD_1D<"suld.b.1d.b32.trap", B32>;
-defm SULD_1D_I64_TRAP : SULD_1D<"suld.b.1d.b64.trap", B64>;
-
-defm SULD_1D_I8_ZERO : SULD_1D<"suld.b.1d.b8.zero", B16>;
-defm SULD_1D_I16_ZERO : SULD_1D<"suld.b.1d.b16.zero", B16>;
-defm SULD_1D_I32_ZERO : SULD_1D<"suld.b.1d.b32.zero", B32>;
-defm SULD_1D_I64_ZERO : SULD_1D<"suld.b.1d.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_1D_I8_ # op_upper : SULD_1D<"suld.b.1d.b8." # op, B16>;
+  defm SULD_1D_I16_ # op_upper : SULD_1D<"suld.b.1d.b16." # op, B16>;
+  defm SULD_1D_I32_ # op_upper : SULD_1D<"suld.b.1d.b32." # op, B32>;
+  defm SULD_1D_I64_ # op_upper : SULD_1D<"suld.b.1d.b64." # op, B64>;
+}
 
 class SULD_1D_ARRAY_base<string inst, NVPTXRegClass outtype, dag surf,
                          list<dag> pattern = []>
@@ -3550,20 +3542,13 @@ multiclass SULD_1D_ARRAY<string inst, NVPTXRegClass outtype> {
   def _I : SULD_1D_ARRAY_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_1D_ARRAY_I8_CLAMP : SULD_1D_ARRAY<"suld.b.a1d.b8.clamp", B16>;
-defm SULD_1D_ARRAY_I16_CLAMP : SULD_1D_ARRAY<"suld.b.a1d.b16.clamp", B16>;
-defm SULD_1D_ARRAY_I32_CLAMP : SULD_1D_ARRAY<"suld.b.a1d.b32.clamp", B32>;
-defm SULD_1D_ARRAY_I64_CLAMP : SULD_1D_ARRAY<"suld.b.a1d.b64.clamp", B64>;
-
-defm SULD_1D_ARRAY_I8_TRAP : SULD_1D_ARRAY<"suld.b.a1d.b8.trap", B16>;
-defm SULD_1D_ARRAY_I16_TRAP : SULD_1D_ARRAY<"suld.b.a1d.b16.trap", B16>;
-defm SULD_1D_ARRAY_I32_TRAP : SULD_1D_ARRAY<"suld.b.a1d.b32.trap", B32>;
-defm SULD_1D_ARRAY_I64_TRAP : SULD_1D_ARRAY<"suld.b.a1d.b64.trap", B64>;
-
-defm SULD_1D_ARRAY_I8_ZERO : SULD_1D_ARRAY<"suld.b.a1d.b8.zero", B16>;
-defm SULD_1D_ARRAY_I16_ZERO : SULD_1D_ARRAY<"suld.b.a1d.b16.zero", B16>;
-defm SULD_1D_ARRAY_I32_ZERO : SULD_1D_ARRAY<"suld.b.a1d.b32.zero", B32>;
-defm SULD_1D_ARRAY_I64_ZERO : SULD_1D_ARRAY<"suld.b.a1d.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_1D_ARRAY_I8_ # op_upper : SULD_1D_ARRAY<"suld.b.a1d.b8." # op, B16>;
+  defm SULD_1D_ARRAY_I16_ # op_upper : SULD_1D_ARRAY<"suld.b.a1d.b16." # op, B16>;
+  defm SULD_1D_ARRAY_I32_ # op_upper : SULD_1D_ARRAY<"suld.b.a1d.b32." # op, B32>;
+  defm SULD_1D_ARRAY_I64_ # op_upper : SULD_1D_ARRAY<"suld.b.a1d.b64." # op, B64>;
+}
 
 class SULD_2D_base<string inst, NVPTXRegClass outtype, dag surf,
                    list<dag> pattern = []>
@@ -3579,20 +3564,13 @@ multiclass SULD_2D<string inst, NVPTXRegClass outtype> {
   def _I : SULD_2D_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_2D_I8_CLAMP : SULD_2D<"suld.b.2d.b8.clamp", B16>;
-defm SULD_2D_I16_CLAMP : SULD_2D<"suld.b.2d.b16.clamp", B16>;
-defm SULD_2D_I32_CLAMP : SULD_2D<"suld.b.2d.b32.clamp", B32>;
-defm SULD_2D_I64_CLAMP : SULD_2D<"suld.b.2d.b64.clamp", B64>;
-
-defm SULD_2D_I8_TRAP : SULD_2D<"suld.b.2d.b8.trap", B16>;
-defm SULD_2D_I16_TRAP : SULD_2D<"suld.b.2d.b16.trap", B16>;
-defm SULD_2D_I32_TRAP : SULD_2D<"suld.b.2d.b32.trap", B32>;
-defm SULD_2D_I64_TRAP : SULD_2D<"suld.b.2d.b64.trap", B64>;
-
-defm SULD_2D_I8_ZERO : SULD_2D<"suld.b.2d.b8.zero", B16>;
-defm SULD_2D_I16_ZERO : SULD_2D<"suld.b.2d.b16.zero", B16>;
-defm SULD_2D_I32_ZERO : SULD_2D<"suld.b.2d.b32.zero", B32>;
-defm SULD_2D_I64_ZERO : SULD_2D<"suld.b.2d.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_2D_I8_ # op_upper : SULD_2D<"suld.b.2d.b8." # op, B16>;
+  defm SULD_2D_I16_ # op_upper : SULD_2D<"suld.b.2d.b16." # op, B16>;
+  defm SULD_2D_I32_ # op_upper : SULD_2D<"suld.b.2d.b32." # op, B32>;
+  defm SULD_2D_I64_ # op_upper : SULD_2D<"suld.b.2d.b64." # op, B64>;
+}
 
 class SULD_2D_ARRAY_base<string inst, NVPTXRegClass outtype, dag surf,
                          list<dag> pattern = []>
@@ -3609,20 +3587,13 @@ multiclass SULD_2D_ARRAY<string inst, NVPTXRegClass outtype> {
   def _I : SULD_2D_ARRAY_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_2D_ARRAY_I8_CLAMP : SULD_2D_ARRAY<"suld.b.a2d.b8.clamp", B16>;
-defm SULD_2D_ARRAY_I16_CLAMP : SULD_2D_ARRAY<"suld.b.a2d.b16.clamp", B16>;
-defm SULD_2D_ARRAY_I32_CLAMP : SULD_2D_ARRAY<"suld.b.a2d.b32.clamp", B32>;
-defm SULD_2D_ARRAY_I64_CLAMP : SULD_2D_ARRAY<"suld.b.a2d.b64.clamp", B64>;
-
-defm SULD_2D_ARRAY_I8_TRAP : SULD_2D_ARRAY<"suld.b.a2d.b8.trap", B16>;
-defm SULD_2D_ARRAY_I16_TRAP : SULD_2D_ARRAY<"suld.b.a2d.b16.trap", B16>;
-defm SULD_2D_ARRAY_I32_TRAP : SULD_2D_ARRAY<"suld.b.a2d.b32.trap", B32>;
-defm SULD_2D_ARRAY_I64_TRAP : SULD_2D_ARRAY<"suld.b.a2d.b64.trap", B64>;
-
-defm SULD_2D_ARRAY_I8_ZERO : SULD_2D_ARRAY<"suld.b.a2d.b8.zero", B16>;
-defm SULD_2D_ARRAY_I16_ZERO : SULD_2D_ARRAY<"suld.b.a2d.b16.zero", B16>;
-defm SULD_2D_ARRAY_I32_ZERO : SULD_2D_ARRAY<"suld.b.a2d.b32.zero", B32>;
-defm SULD_2D_ARRAY_I64_ZERO : SULD_2D_ARRAY<"suld.b.a2d.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_2D_ARRAY_I8_ # op_upper : SULD_2D_ARRAY<"suld.b.a2d.b8." # op, B16>;
+  defm SULD_2D_ARRAY_I16_ # op_upper : SULD_2D_ARRAY<"suld.b.a2d.b16." # op, B16>;
+  defm SULD_2D_ARRAY_I32_ # op_upper : SULD_2D_ARRAY<"suld.b.a2d.b32." # op, B32>;
+  defm SULD_2D_ARRAY_I64_ # op_upper : SULD_2D_ARRAY<"suld.b.a2d.b64." # op, B64>;
+}
 
 class SULD_3D_base<string inst, NVPTXRegClass outtype, dag surf,
                    list<dag> pattern = []>
@@ -3639,20 +3610,13 @@ multiclass SULD_3D<string inst, NVPTXRegClass outtype> {
   def _I : SULD_3D_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_3D_I8_CLAMP : SULD_3D<"suld.b.3d.b8.clamp", B16>;
-defm SULD_3D_I16_CLAMP : SULD_3D<"suld.b.3d.b16.clamp", B16>;
-defm SULD_3D_I32_CLAMP : SULD_3D<"suld.b.3d.b32.clamp", B32>;
-defm SULD_3D_I64_CLAMP : SULD_3D<"suld.b.3d.b64.clamp", B64>;
-
-defm SULD_3D_I8_TRAP : SULD_3D<"suld.b.3d.b8.trap", B16>;
-defm SULD_3D_I16_TRAP : SULD_3D<"suld.b.3d.b16.trap", B16>;
-defm SULD_3D_I32_TRAP : SULD_3D<"suld.b.3d.b32.trap", B32>;
-defm SULD_3D_I64_TRAP : SULD_3D<"suld.b.3d.b64.trap", B64>;
-
-defm SULD_3D_I8_ZERO : SULD_3D<"suld.b.3d.b8.zero", B16>;
-defm SULD_3D_I16_ZERO : SULD_3D<"suld.b.3d.b16.zero", B16>;
-defm SULD_3D_I32_ZERO : SULD_3D<"suld.b.3d.b32.zero", B32>;
-defm SULD_3D_I64_ZERO : SULD_3D<"suld.b.3d.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_3D_I8_ # op_upper : SULD_3D<"suld.b.3d.b8." # op, B16>;
+  defm SULD_3D_I16_ # op_upper : SULD_3D<"suld.b.3d.b16." # op, B16>;
+  defm SULD_3D_I32_ # op_upper : SULD_3D<"suld.b.3d.b32." # op, B32>;
+  defm SULD_3D_I64_ # op_upper : SULD_3D<"suld.b.3d.b64." # op, B64>;
+}
 }
 
 let IsSuld = 2 in {
@@ -3672,20 +3636,13 @@ multiclass SULD_1D_V2<string inst, NVPTXRegClass outtype> {
   def _I : SULD_1D_V2_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_1D_V2I8_CLAMP : SULD_1D_V2<"suld.b.1d.v2.b8.clamp", B16>;
-defm SULD_1D_V2I16_CLAMP : SULD_1D_V2<"suld.b.1d.v2.b16.clamp", B16>;
-defm SULD_1D_V2I32_CLAMP : SULD_1D_V2<"suld.b.1d.v2.b32.clamp", B32>;
-defm SULD_1D_V2I64_CLAMP : SULD_1D_V2<"suld.b.1d.v2.b64.clamp", B64>;
-
-defm SULD_1D_V2I8_TRAP : SULD_1D_V2<"suld.b.1d.v2.b8.trap", B16>;
-defm SULD_1D_V2I16_TRAP : SULD_1D_V2<"suld.b.1d.v2.b16.trap", B16>;
-defm SULD_1D_V2I32_TRAP : SULD_1D_V2<"suld.b.1d.v2.b32.trap", B32>;
-defm SULD_1D_V2I64_TRAP : SULD_1D_V2<"suld.b.1d.v2.b64.trap", B64>;
-
-defm SULD_1D_V2I8_ZERO : SULD_1D_V2<"suld.b.1d.v2.b8.zero", B16>;
-defm SULD_1D_V2I16_ZERO : SULD_1D_V2<"suld.b.1d.v2.b16.zero", B16>;
-defm SULD_1D_V2I32_ZERO : SULD_1D_V2<"suld.b.1d.v2.b32.zero", B32>;
-defm SULD_1D_V2I64_ZERO : SULD_1D_V2<"suld.b.1d.v2.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_1D_V2I8_ # op_upper : SULD_1D_V2<"suld.b.1d.v2.b8." # op, B16>;
+  defm SULD_1D_V2I16_ # op_upper : SULD_1D_V2<"suld.b.1d.v2.b16." # op, B16>;
+  defm SULD_1D_V2I32_ # op_upper : SULD_1D_V2<"suld.b.1d.v2.b32." # op, B32>;
+  defm SULD_1D_V2I64_ # op_upper : SULD_1D_V2<"suld.b.1d.v2.b64." # op, B64>;
+}
 
 class SULD_1D_ARRAY_V2_base<string inst, NVPTXRegClass outtype, dag surf,
                             list<dag> pattern = []>
@@ -3702,20 +3659,13 @@ multiclass SULD_1D_ARRAY_V2<string inst, NVPTXRegClass outtype> {
   def _I : SULD_1D_ARRAY_V2_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_1D_ARRAY_V2I8_CLAMP : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b8.clamp", B16>;
-defm SULD_1D_ARRAY_V2I16_CLAMP : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b16.clamp", B16>;
-defm SULD_1D_ARRAY_V2I32_CLAMP : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b32.clamp", B32>;
-defm SULD_1D_ARRAY_V2I64_CLAMP : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b64.clamp", B64>;
-
-defm SULD_1D_ARRAY_V2I8_TRAP : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b8.trap", B16>;
-defm SULD_1D_ARRAY_V2I16_TRAP : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b16.trap", B16>;
-defm SULD_1D_ARRAY_V2I32_TRAP : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b32.trap", B32>;
-defm SULD_1D_ARRAY_V2I64_TRAP : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b64.trap", B64>;
-
-defm SULD_1D_ARRAY_V2I8_ZERO : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b8.zero", B16>;
-defm SULD_1D_ARRAY_V2I16_ZERO : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b16.zero", B16>;
-defm SULD_1D_ARRAY_V2I32_ZERO : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b32.zero", B32>;
-defm SULD_1D_ARRAY_V2I64_ZERO : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_1D_ARRAY_V2I8_ # op_upper : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b8." # op, B16>;
+  defm SULD_1D_ARRAY_V2I16_ # op_upper : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b16." # op, B16>;
+  defm SULD_1D_ARRAY_V2I32_ # op_upper : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b32." # op, B32>;
+  defm SULD_1D_ARRAY_V2I64_ # op_upper : SULD_1D_ARRAY_V2<"suld.b.a1d.v2.b64." # op, B64>;
+}
 
 class SULD_2D_V2_base<string inst, NVPTXRegClass outtype, dag surf,
                       list<dag> pattern = []>
@@ -3732,20 +3682,13 @@ multiclass SULD_2D_V2<string inst, NVPTXRegClass outtype> {
   def _I : SULD_2D_V2_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_2D_V2I8_CLAMP : SULD_2D_V2<"suld.b.2d.v2.b8.clamp", B16>;
-defm SULD_2D_V2I16_CLAMP : SULD_2D_V2<"suld.b.2d.v2.b16.clamp", B16>;
-defm SULD_2D_V2I32_CLAMP : SULD_2D_V2<"suld.b.2d.v2.b32.clamp", B32>;
-defm SULD_2D_V2I64_CLAMP : SULD_2D_V2<"suld.b.2d.v2.b64.clamp", B64>;
-
-defm SULD_2D_V2I8_TRAP : SULD_2D_V2<"suld.b.2d.v2.b8.trap", B16>;
-defm SULD_2D_V2I16_TRAP : SULD_2D_V2<"suld.b.2d.v2.b16.trap", B16>;
-defm SULD_2D_V2I32_TRAP : SULD_2D_V2<"suld.b.2d.v2.b32.trap", B32>;
-defm SULD_2D_V2I64_TRAP : SULD_2D_V2<"suld.b.2d.v2.b64.trap", B64>;
-
-defm SULD_2D_V2I8_ZERO : SULD_2D_V2<"suld.b.2d.v2.b8.zero", B16>;
-defm SULD_2D_V2I16_ZERO : SULD_2D_V2<"suld.b.2d.v2.b16.zero", B16>;
-defm SULD_2D_V2I32_ZERO : SULD_2D_V2<"suld.b.2d.v2.b32.zero", B32>;
-defm SULD_2D_V2I64_ZERO : SULD_2D_V2<"suld.b.2d.v2.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_2D_V2I8_ # op_upper : SULD_2D_V2<"suld.b.2d.v2.b8." # op, B16>;
+  defm SULD_2D_V2I16_ # op_upper : SULD_2D_V2<"suld.b.2d.v2.b16." # op, B16>;
+  defm SULD_2D_V2I32_ # op_upper : SULD_2D_V2<"suld.b.2d.v2.b32." # op, B32>;
+  defm SULD_2D_V2I64_ # op_upper : SULD_2D_V2<"suld.b.2d.v2.b64." # op, B64>;
+}
 
 class SULD_2D_ARRAY_V2_base<string inst, NVPTXRegClass outtype, dag surf,
                             list<dag> pattern = []>
@@ -3762,20 +3705,13 @@ multiclass SULD_2D_ARRAY_V2<string inst, NVPTXRegClass outtype> {
   def _I : SULD_2D_ARRAY_V2_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_2D_ARRAY_V2I8_CLAMP : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b8.clamp", B16>;
-defm SULD_2D_ARRAY_V2I16_CLAMP : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b16.clamp", B16>;
-defm SULD_2D_ARRAY_V2I32_CLAMP : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b32.clamp", B32>;
-defm SULD_2D_ARRAY_V2I64_CLAMP : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b64.clamp", B64>;
-
-defm SULD_2D_ARRAY_V2I8_TRAP : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b8.trap", B16>;
-defm SULD_2D_ARRAY_V2I16_TRAP : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b16.trap", B16>;
-defm SULD_2D_ARRAY_V2I32_TRAP : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b32.trap", B32>;
-defm SULD_2D_ARRAY_V2I64_TRAP : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b64.trap", B64>;
-
-defm SULD_2D_ARRAY_V2I8_ZERO : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b8.zero", B16>;
-defm SULD_2D_ARRAY_V2I16_ZERO : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b16.zero", B16>;
-defm SULD_2D_ARRAY_V2I32_ZERO : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b32.zero", B32>;
-defm SULD_2D_ARRAY_V2I64_ZERO : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_2D_ARRAY_V2I8_ # op_upper : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b8." # op, B16>;
+  defm SULD_2D_ARRAY_V2I16_ # op_upper : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b16." # op, B16>;
+  defm SULD_2D_ARRAY_V2I32_ # op_upper : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b32." # op, B32>;
+  defm SULD_2D_ARRAY_V2I64_ # op_upper : SULD_2D_ARRAY_V2<"suld.b.a2d.v2.b64." # op, B64>;
+}
 
 class SULD_3D_V2_base<string inst, NVPTXRegClass outtype, dag surf,
                       list<dag> pattern = []>
@@ -3792,20 +3728,13 @@ multiclass SULD_3D_V2<string inst, NVPTXRegClass outtype> {
   def _I : SULD_3D_V2_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_3D_V2I8_CLAMP : SULD_3D_V2<"suld.b.3d.v2.b8.clamp", B16>;
-defm SULD_3D_V2I16_CLAMP : SULD_3D_V2<"suld.b.3d.v2.b16.clamp", B16>;
-defm SULD_3D_V2I32_CLAMP : SULD_3D_V2<"suld.b.3d.v2.b32.clamp", B32>;
-defm SULD_3D_V2I64_CLAMP : SULD_3D_V2<"suld.b.3d.v2.b64.clamp", B64>;
-
-defm SULD_3D_V2I8_TRAP : SULD_3D_V2<"suld.b.3d.v2.b8.trap", B16>;
-defm SULD_3D_V2I16_TRAP : SULD_3D_V2<"suld.b.3d.v2.b16.trap", B16>;
-defm SULD_3D_V2I32_TRAP : SULD_3D_V2<"suld.b.3d.v2.b32.trap", B32>;
-defm SULD_3D_V2I64_TRAP : SULD_3D_V2<"suld.b.3d.v2.b64.trap", B64>;
-
-defm SULD_3D_V2I8_ZERO : SULD_3D_V2<"suld.b.3d.v2.b8.zero", B16>;
-defm SULD_3D_V2I16_ZERO : SULD_3D_V2<"suld.b.3d.v2.b16.zero", B16>;
-defm SULD_3D_V2I32_ZERO : SULD_3D_V2<"suld.b.3d.v2.b32.zero", B32>;
-defm SULD_3D_V2I64_ZERO : SULD_3D_V2<"suld.b.3d.v2.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_3D_V2I8_ # op_upper : SULD_3D_V2<"suld.b.3d.v2.b8." # op, B16>;
+  defm SULD_3D_V2I16_ # op_upper : SULD_3D_V2<"suld.b.3d.v2.b16." # op, B16>;
+  defm SULD_3D_V2I32_ # op_upper : SULD_3D_V2<"suld.b.3d.v2.b32." # op, B32>;
+  defm SULD_3D_V2I64_ # op_upper : SULD_3D_V2<"suld.b.3d.v2.b64." # op, B64>;
+}
 
 }
 
@@ -3826,17 +3755,12 @@ multiclass SULD_1D_V4<string inst, NVPTXRegClass outtype> {
   def _I : SULD_1D_V4_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_1D_V4I8_CLAMP : SULD_1D_V4<"suld.b.1d.v4.b8.clamp", B16>;
-defm SULD_1D_V4I16_CLAMP : SULD_1D_V4<"suld.b.1d.v4.b16.clamp", B16>;
-defm SULD_1D_V4I32_CLAMP : SULD_1D_V4<"suld.b.1d.v4.b32.clamp", B32>;
-
-defm SULD_1D_V4I8_TRAP : SULD_1D_V4<"suld.b.1d.v4.b8.trap", B16>;
-defm SULD_1D_V4I16_TRAP : SULD_1D_V4<"suld.b.1d.v4.b16.trap", B16>;
-defm SULD_1D_V4I32_TRAP : SULD_1D_V4<"suld.b.1d.v4.b32.trap", B32>;
-
-defm SULD_1D_V4I8_ZERO : SULD_1D_V4<"suld.b.1d.v4.b8.zero", B16>;
-defm SULD_1D_V4I16_ZERO : SULD_1D_V4<"suld.b.1d.v4.b16.zero", B16>;
-defm SULD_1D_V4I32_ZERO : SULD_1D_V4<"suld.b.1d.v4.b32.zero", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_1D_V4I8_ # op_upper : SULD_1D_V4<"suld.b.1d.v4.b8." # op, B16>;
+  defm SULD_1D_V4I16_ # op_upper : SULD_1D_V4<"suld.b.1d.v4.b16." # op, B16>;
+  defm SULD_1D_V4I32_ # op_upper : SULD_1D_V4<"suld.b.1d.v4.b32." # op, B32>;
+}
 
 class SULD_1D_ARRAY_V4_base<string inst, NVPTXRegClass outtype, dag surf,
                             list<dag> pattern = []>
@@ -3854,17 +3778,12 @@ multiclass SULD_1D_ARRAY_V4<string inst, NVPTXRegClass outtype> {
   def _I : SULD_1D_ARRAY_V4_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_1D_ARRAY_V4I8_CLAMP : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b8.clamp", B16>;
-defm SULD_1D_ARRAY_V4I16_CLAMP : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b16.clamp", B16>;
-defm SULD_1D_ARRAY_V4I32_CLAMP : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b32.clamp", B32>;
-
-defm SULD_1D_ARRAY_V4I8_TRAP : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b8.trap", B16>;
-defm SULD_1D_ARRAY_V4I16_TRAP : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b16.trap", B16>;
-defm SULD_1D_ARRAY_V4I32_TRAP : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b32.trap", B32>;
-
-defm SULD_1D_ARRAY_V4I8_ZERO : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b8.zero", B16>;
-defm SULD_1D_ARRAY_V4I16_ZERO : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b16.zero", B16>;
-defm SULD_1D_ARRAY_V4I32_ZERO : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b32.zero", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_1D_ARRAY_V4I8_ # op_upper : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b8." # op, B16>;
+  defm SULD_1D_ARRAY_V4I16_ # op_upper : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b16." # op, B16>;
+  defm SULD_1D_ARRAY_V4I32_ # op_upper : SULD_1D_ARRAY_V4<"suld.b.a1d.v4.b32." # op, B32>;
+}
 
 class SULD_2D_V4_base<string inst, NVPTXRegClass outtype, dag surf,
                       list<dag> pattern = []>
@@ -3881,17 +3800,12 @@ multiclass SULD_2D_V4<string inst, NVPTXRegClass outtype> {
   def _I : SULD_2D_V4_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_2D_V4I8_CLAMP : SULD_2D_V4<"suld.b.2d.v4.b8.clamp", B16>;
-defm SULD_2D_V4I16_CLAMP : SULD_2D_V4<"suld.b.2d.v4.b16.clamp", B16>;
-defm SULD_2D_V4I32_CLAMP : SULD_2D_V4<"suld.b.2d.v4.b32.clamp", B32>;
-
-defm SULD_2D_V4I8_TRAP : SULD_2D_V4<"suld.b.2d.v4.b8.trap", B16>;
-defm SULD_2D_V4I16_TRAP : SULD_2D_V4<"suld.b.2d.v4.b16.trap", B16>;
-defm SULD_2D_V4I32_TRAP : SULD_2D_V4<"suld.b.2d.v4.b32.trap", B32>;
-
-defm SULD_2D_V4I8_ZERO : SULD_2D_V4<"suld.b.2d.v4.b8.zero", B16>;
-defm SULD_2D_V4I16_ZERO : SULD_2D_V4<"suld.b.2d.v4.b16.zero", B16>;
-defm SULD_2D_V4I32_ZERO : SULD_2D_V4<"suld.b.2d.v4.b32.zero", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_2D_V4I8_ # op_upper : SULD_2D_V4<"suld.b.2d.v4.b8." # op, B16>;
+  defm SULD_2D_V4I16_ # op_upper : SULD_2D_V4<"suld.b.2d.v4.b16." # op, B16>;
+  defm SULD_2D_V4I32_ # op_upper : SULD_2D_V4<"suld.b.2d.v4.b32." # op, B32>;
+}
 
 class SULD_2D_ARRAY_V4_base<string inst, NVPTXRegClass outtype, dag surf,
                             list<dag> pattern = []>
@@ -3909,17 +3823,12 @@ multiclass SULD_2D_ARRAY_V4<string inst, NVPTXRegClass outtype> {
   def _I : SULD_2D_ARRAY_V4_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_2D_ARRAY_V4I8_CLAMP : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b8.clamp", B16>;
-defm SULD_2D_ARRAY_V4I16_CLAMP : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b16.clamp", B16>;
-defm SULD_2D_ARRAY_V4I32_CLAMP : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b32.clamp", B32>;
-
-defm SULD_2D_ARRAY_V4I8_TRAP : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b8.trap", B16>;
-defm SULD_2D_ARRAY_V4I16_TRAP : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b16.trap", B16>;
-defm SULD_2D_ARRAY_V4I32_TRAP : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b32.trap", B32>;
-
-defm SULD_2D_ARRAY_V4I8_ZERO : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b8.zero", B16>;
-defm SULD_2D_ARRAY_V4I16_ZERO : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b16.zero", B16>;
-defm SULD_2D_ARRAY_V4I32_ZERO : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b32.zero", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_2D_ARRAY_V4I8_ # op_upper : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b8." # op, B16>;
+  defm SULD_2D_ARRAY_V4I16_ # op_upper : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b16." # op, B16>;
+  defm SULD_2D_ARRAY_V4I32_ # op_upper : SULD_2D_ARRAY_V4<"suld.b.a2d.v4.b32." # op, B32>;
+}
 
 class SULD_3D_V4_base<string inst, NVPTXRegClass outtype, dag surf,
                       list<dag> pattern = []>
@@ -3936,17 +3845,12 @@ multiclass SULD_3D_V4<string inst, NVPTXRegClass outtype> {
   def _I : SULD_3D_V4_base<inst, outtype, (ins i64imm:$s)>;
 }
 
-defm SULD_3D_V4I8_CLAMP : SULD_3D_V4<"suld.b.3d.v4.b8.clamp", B16>;
-defm SULD_3D_V4I16_CLAMP : SULD_3D_V4<"suld.b.3d.v4.b16.clamp", B16>;
-defm SULD_3D_V4I32_CLAMP : SULD_3D_V4<"suld.b.3d.v4.b32.clamp", B32>;
-
-defm SULD_3D_V4I8_TRAP : SULD_3D_V4<"suld.b.3d.v4.b8.trap", B16>;
-defm SULD_3D_V4I16_TRAP : SULD_3D_V4<"suld.b.3d.v4.b16.trap", B16>;
-defm SULD_3D_V4I32_TRAP : SULD_3D_V4<"suld.b.3d.v4.b32.trap", B32>;
-
-defm SULD_3D_V4I8_ZERO : SULD_3D_V4<"suld.b.3d.v4.b8.zero", B16>;
-defm SULD_3D_V4I16_ZERO : SULD_3D_V4<"suld.b.3d.v4.b16.zero", B16>;
-defm SULD_3D_V4I32_ZERO : SULD_3D_V4<"suld.b.3d.v4.b32.zero", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SULD_3D_V4I8_ # op_upper : SULD_3D_V4<"suld.b.3d.v4.b8." # op, B16>;
+  defm SULD_3D_V4I16_ # op_upper : SULD_3D_V4<"suld.b.3d.v4.b16." # op, B16>;
+  defm SULD_3D_V4I32_ # op_upper : SULD_3D_V4<"suld.b.3d.v4.b32." # op, B32>;
+}
 
 }
 
@@ -4017,20 +3921,13 @@ multiclass SUST_1D<string inst, NVPTXRegClass intype> {
   def _I : SUST_1D_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_I8_CLAMP : SUST_1D<"sust.b.1d.b8.clamp", B16>;
-defm SUST_B_1D_I16_CLAMP : SUST_1D<"sust.b.1d.b16.clamp", B16>;
-defm SUST_B_1D_I32_CLAMP : SUST_1D<"sust.b.1d.b32.clamp", B32>;
-defm SUST_B_1D_I64_CLAMP : SUST_1D<"sust.b.1d.b64.clamp", B64>;
-
-defm SUST_B_1D_I8_TRAP : SUST_1D<"sust.b.1d.b8.trap", B16>;
-defm SUST_B_1D_I16_TRAP : SUST_1D<"sust.b.1d.b16.trap", B16>;
-defm SUST_B_1D_I32_TRAP : SUST_1D<"sust.b.1d.b32.trap", B32>;
-defm SUST_B_1D_I64_TRAP : SUST_1D<"sust.b.1d.b64.trap", B64>;
-
-defm SUST_B_1D_I8_ZERO : SUST_1D<"sust.b.1d.b8.zero", B16>;
-defm SUST_B_1D_I16_ZERO : SUST_1D<"sust.b.1d.b16.zero", B16>;
-defm SUST_B_1D_I32_ZERO : SUST_1D<"sust.b.1d.b32.zero", B32>;
-defm SUST_B_1D_I64_ZERO : SUST_1D<"sust.b.1d.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_1D_I8_ # op_upper : SUST_1D<"sust.b.1d.b8." # op, B16>;
+  defm SUST_B_1D_I16_ # op_upper : SUST_1D<"sust.b.1d.b16." # op, B16>;
+  defm SUST_B_1D_I32_ # op_upper : SUST_1D<"sust.b.1d.b32." # op, B32>;
+  defm SUST_B_1D_I64_ # op_upper : SUST_1D<"sust.b.1d.b64." # op, B64>;
+}
 
 defm SUST_P_1D_I8_TRAP : SUST_1D<"sust.p.1d.b8.trap", B16>;
 defm SUST_P_1D_I16_TRAP : SUST_1D<"sust.p.1d.b16.trap", B16>;
@@ -4048,23 +3945,13 @@ multiclass SUST_1D_V2<string inst, NVPTXRegClass intype> {
   def _I : SUST_1D_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-// int_nvvm_sust_b_1d_v2i8_clamp
-
-defm SUST_B_1D_V2I8_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b8.clamp", B16>;
-defm SUST_B_1D_V2I16_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b16.clamp", B16>;
-defm SUST_B_1D_V2I32_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b32.clamp", B32>;
-defm SUST_B_1D_V2I64_CLAMP : SUST_1D_V2<"sust.b.1d.v2.b64.clamp", B64>;
-
-defm SUST_B_1D_V2I8_TRAP : SUST_1D_V2<"sust.b.1d.v2.b8.trap", B16>;
-defm SUST_B_1D_V2I16_TRAP : SUST_1D_V2<"sust.b.1d.v2.b16.trap", B16>;
-defm SUST_B_1D_V2I32_TRAP : SUST_1D_V2<"sust.b.1d.v2.b32.trap", B32>;
-defm SUST_B_1D_V2I64_TRAP : SUST_1D_V2<"sust.b.1d.v2.b64.trap", B64>;
-
-defm SUST_B_1D_V2I8_ZERO : SUST_1D_V2<"sust.b.1d.v2.b8.zero", B16>;
-defm SUST_B_1D_V2I16_ZERO : SUST_1D_V2<"sust.b.1d.v2.b16.zero", B16>;
-defm SUST_B_1D_V2I32_ZERO : SUST_1D_V2<"sust.b.1d.v2.b32.zero", B32>;
-defm SUST_B_1D_V2I64_ZERO : SUST_1D_V2<"sust.b.1d.v2.b64.zero", B64>;
-
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_1D_V2I8_ # op_upper : SUST_1D_V2<"sust.b.1d.v2.b8." # op, B16>;
+  defm SUST_B_1D_V2I16_ # op_upper : SUST_1D_V2<"sust.b.1d.v2.b16." # op, B16>;
+  defm SUST_B_1D_V2I32_ # op_upper : SUST_1D_V2<"sust.b.1d.v2.b32." # op, B32>;
+  defm SUST_B_1D_V2I64_ # op_upper : SUST_1D_V2<"sust.b.1d.v2.b64." # op, B64>;
+}
 defm SUST_P_1D_V2I8_TRAP : SUST_1D_V2<"sust.p.1d.v2.b8.trap", B16>;
 defm SUST_P_1D_V2I16_TRAP : SUST_1D_V2<"sust.p.1d.v2.b16.trap", B16>;
 defm SUST_P_1D_V2I32_TRAP : SUST_1D_V2<"sust.p.1d.v2.b32.trap", B32>;
@@ -4083,17 +3970,12 @@ multiclass SUST_1D_V4<string inst, NVPTXRegClass intype> {
   def _I : SUST_1D_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_V4I8_CLAMP : SUST_1D_V4<"sust.b.1d.v4.b8.clamp", B16>;
-defm SUST_B_1D_V4I16_CLAMP : SUST_1D_V4<"sust.b.1d.v4.b16.clamp", B16>;
-defm SUST_B_1D_V4I32_CLAMP : SUST_1D_V4<"sust.b.1d.v4.b32.clamp", B32>;
-
-defm SUST_B_1D_V4I8_TRAP : SUST_1D_V4<"sust.b.1d.v4.b8.trap", B16>;
-defm SUST_B_1D_V4I16_TRAP : SUST_1D_V4<"sust.b.1d.v4.b16.trap", B16>;
-defm SUST_B_1D_V4I32_TRAP : SUST_1D_V4<"sust.b.1d.v4.b32.trap", B32>;
-
-defm SUST_B_1D_V4I8_ZERO : SUST_1D_V4<"sust.b.1d.v4.b8.zero", B16>;
-defm SUST_B_1D_V4I16_ZERO : SUST_1D_V4<"sust.b.1d.v4.b16.zero", B16>;
-defm SUST_B_1D_V4I32_ZERO : SUST_1D_V4<"sust.b.1d.v4.b32.zero", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_1D_V4I8_ # op_upper : SUST_1D_V4<"sust.b.1d.v4.b8." # op, B16>;
+  defm SUST_B_1D_V4I16_ # op_upper : SUST_1D_V4<"sust.b.1d.v4.b16." # op, B16>;
+  defm SUST_B_1D_V4I32_ # op_upper : SUST_1D_V4<"sust.b.1d.v4.b32." # op, B32>;
+}
 
 defm SUST_P_1D_V4I8_TRAP : SUST_1D_V4<"sust.p.1d.v4.b8.trap", B16>;
 defm SUST_P_1D_V4I16_TRAP : SUST_1D_V4<"sust.p.1d.v4.b16.trap", B16>;
@@ -4111,20 +3993,13 @@ multiclass SUST_1D_ARRAY<string inst, NVPTXRegClass intype> {
   def _I : SUST_1D_ARRAY_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_ARRAY_I8_CLAMP : SUST_1D_ARRAY<"sust.b.a1d.b8.clamp", B16>;
-defm SUST_B_1D_ARRAY_I16_CLAMP : SUST_1D_ARRAY<"sust.b.a1d.b16.clamp", B16>;
-defm SUST_B_1D_ARRAY_I32_CLAMP : SUST_1D_ARRAY<"sust.b.a1d.b32.clamp", B32>;
-defm SUST_B_1D_ARRAY_I64_CLAMP : SUST_1D_ARRAY<"sust.b.a1d.b64.clamp", B64>;
-
-defm SUST_B_1D_ARRAY_I8_TRAP : SUST_1D_ARRAY<"sust.b.a1d.b8.trap", B16>;
-defm SUST_B_1D_ARRAY_I16_TRAP : SUST_1D_ARRAY<"sust.b.a1d.b16.trap", B16>;
-defm SUST_B_1D_ARRAY_I32_TRAP : SUST_1D_ARRAY<"sust.b.a1d.b32.trap", B32>;
-defm SUST_B_1D_ARRAY_I64_TRAP : SUST_1D_ARRAY<"sust.b.a1d.b64.trap", B64>;
-
-defm SUST_B_1D_ARRAY_I8_ZERO : SUST_1D_ARRAY<"sust.b.a1d.b8.zero", B16>;
-defm SUST_B_1D_ARRAY_I16_ZERO : SUST_1D_ARRAY<"sust.b.a1d.b16.zero", B16>;
-defm SUST_B_1D_ARRAY_I32_ZERO : SUST_1D_ARRAY<"sust.b.a1d.b32.zero", B32>;
-defm SUST_B_1D_ARRAY_I64_ZERO : SUST_1D_ARRAY<"sust.b.a1d.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_1D_ARRAY_I8_ # op_upper : SUST_1D_ARRAY<"sust.b.a1d.b8." # op, B16>;
+  defm SUST_B_1D_ARRAY_I16_ # op_upper : SUST_1D_ARRAY<"sust.b.a1d.b16." # op, B16>;
+  defm SUST_B_1D_ARRAY_I32_ # op_upper : SUST_1D_ARRAY<"sust.b.a1d.b32." # op, B32>;
+  defm SUST_B_1D_ARRAY_I64_ # op_upper : SUST_1D_ARRAY<"sust.b.a1d.b64." # op, B64>;
+}
 
 defm SUST_P_1D_ARRAY_I8_TRAP : SUST_1D_ARRAY<"sust.p.a1d.b8.trap", B16>;
 defm SUST_P_1D_ARRAY_I16_TRAP : SUST_1D_ARRAY<"sust.p.a1d.b16.trap", B16>;
@@ -4144,20 +4019,13 @@ multiclass SUST_1D_ARRAY_V2<string inst, NVPTXRegClass intype> {
   def _I : SUST_1D_ARRAY_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_ARRAY_V2I8_CLAMP : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b8.clamp", B16>;
-defm SUST_B_1D_ARRAY_V2I16_CLAMP : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b16.clamp", B16>;
-defm SUST_B_1D_ARRAY_V2I32_CLAMP : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b32.clamp", B32>;
-defm SUST_B_1D_ARRAY_V2I64_CLAMP : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b64.clamp", B64>;
-
-defm SUST_B_1D_ARRAY_V2I8_TRAP : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b8.trap", B16>;
-defm SUST_B_1D_ARRAY_V2I16_TRAP : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b16.trap", B16>;
-defm SUST_B_1D_ARRAY_V2I32_TRAP : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b32.trap", B32>;
-defm SUST_B_1D_ARRAY_V2I64_TRAP : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b64.trap", B64>;
-
-defm SUST_B_1D_ARRAY_V2I8_ZERO : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b8.zero", B16>;
-defm SUST_B_1D_ARRAY_V2I16_ZERO : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b16.zero", B16>;
-defm SUST_B_1D_ARRAY_V2I32_ZERO : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b32.zero", B32>;
-defm SUST_B_1D_ARRAY_V2I64_ZERO : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_1D_ARRAY_V2I8_ # op_upper : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b8." # op, B16>;
+  defm SUST_B_1D_ARRAY_V2I16_ # op_upper : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b16." # op, B16>;
+  defm SUST_B_1D_ARRAY_V2I32_ # op_upper : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b32." # op, B32>;
+  defm SUST_B_1D_ARRAY_V2I64_ # op_upper : SUST_1D_ARRAY_V2<"sust.b.a1d.v2.b64." # op, B64>;
+}
 
 defm SUST_P_1D_ARRAY_V2I8_TRAP : SUST_1D_ARRAY_V2<"sust.p.a1d.v2.b8.trap", B16>;
 defm SUST_P_1D_ARRAY_V2I16_TRAP : SUST_1D_ARRAY_V2<"sust.p.a1d.v2.b16.trap", B16>;
@@ -4177,33 +4045,16 @@ multiclass SUST_1D_ARRAY_V4<string inst, NVPTXRegClass intype> {
   def _I : SUST_1D_ARRAY_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_1D_ARRAY_V4I8_CLAMP
-  : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b8.clamp", B16>;
-defm SUST_B_1D_ARRAY_V4I16_CLAMP
-  : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b16.clamp", B16>;
-defm SUST_B_1D_ARRAY_V4I32_CLAMP
-  : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b32.clamp", B32>;
-
-defm SUST_B_1D_ARRAY_V4I8_TRAP
-  : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b8.trap", B16>;
-defm SUST_B_1D_ARRAY_V4I16_TRAP
-  : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b16.trap", B16>;
-defm SUST_B_1D_ARRAY_V4I32_TRAP
-  : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b32.trap", B32>;
-
-defm SUST_B_1D_ARRAY_V4I8_ZERO
-  : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b8.zero", B16>;
-defm SUST_B_1D_ARRAY_V4I16_ZERO
-  : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b16.zero", B16>;
-defm SUST_B_1D_ARRAY_V4I32_ZERO
-  : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b32.zero", B32>;
-
-defm SUST_P_1D_ARRAY_V4I8_TRAP
-  : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b8.trap", B16>;
-defm SUST_P_1D_ARRAY_V4I16_TRAP
-  : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b16.trap", B16>;
-defm SUST_P_1D_ARRAY_V4I32_TRAP
-  : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b32.trap", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_1D_ARRAY_V4I8_ # op_upper : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b8." # op, B16>;
+  defm SUST_B_1D_ARRAY_V4I16_ # op_upper : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b16." # op, B16>;
+  defm SUST_B_1D_ARRAY_V4I32_ # op_upper : SUST_1D_ARRAY_V4<"sust.b.a1d.v4.b32." # op, B32>;
+}
+
+defm SUST_P_1D_ARRAY_V4I8_TRAP : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b8.trap", B16>;
+defm SUST_P_1D_ARRAY_V4I16_TRAP : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b16.trap", B16>;
+defm SUST_P_1D_ARRAY_V4I32_TRAP : SUST_1D_ARRAY_V4<"sust.p.a1d.v4.b32.trap", B32>;
 
 class SUST_2D_base<string inst, NVPTXRegClass intype, dag surf, list<dag> pat>
     : NVPTXInst<(outs),
@@ -4217,20 +4068,13 @@ multiclass SUST_2D<string inst, NVPTXRegClass intype> {
   def _I : SUST_2D_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_I8_CLAMP : SUST_2D<"sust.b.2d.b8.clamp", B16>;
-defm SUST_B_2D_I16_CLAMP : SUST_2D<"sust.b.2d.b16.clamp", B16>;
-defm SUST_B_2D_I32_CLAMP : SUST_2D<"sust.b.2d.b32.clamp", B32>;
-defm SUST_B_2D_I64_CLAMP : SUST_2D<"sust.b.2d.b64.clamp", B64>;
-
-defm SUST_B_2D_I8_TRAP : SUST_2D<"sust.b.2d.b8.trap", B16>;
-defm SUST_B_2D_I16_TRAP : SUST_2D<"sust.b.2d.b16.trap", B16>;
-defm SUST_B_2D_I32_TRAP : SUST_2D<"sust.b.2d.b32.trap", B32>;
-defm SUST_B_2D_I64_TRAP : SUST_2D<"sust.b.2d.b64.trap", B64>;
-
-defm SUST_B_2D_I8_ZERO : SUST_2D<"sust.b.2d.b8.zero", B16>;
-defm SUST_B_2D_I16_ZERO : SUST_2D<"sust.b.2d.b16.zero", B16>;
-defm SUST_B_2D_I32_ZERO : SUST_2D<"sust.b.2d.b32.zero", B32>;
-defm SUST_B_2D_I64_ZERO : SUST_2D<"sust.b.2d.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_2D_I8_ # op_upper : SUST_2D<"sust.b.2d.b8." # op, B16>;
+  defm SUST_B_2D_I16_ # op_upper : SUST_2D<"sust.b.2d.b16." # op, B16>;
+  defm SUST_B_2D_I32_ # op_upper : SUST_2D<"sust.b.2d.b32." # op, B32>;
+  defm SUST_B_2D_I64_ # op_upper : SUST_2D<"sust.b.2d.b64." # op, B64>;
+}
 
 defm SUST_P_2D_I8_TRAP : SUST_2D<"sust.p.2d.b8.trap", B16>;
 defm SUST_P_2D_I16_TRAP : SUST_2D<"sust.p.2d.b16.trap", B16>;
@@ -4250,20 +4094,13 @@ multiclass SUST_2D_V2<string inst, NVPTXRegClass intype> {
   def _I : SUST_2D_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_V2I8_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b8.clamp", B16>;
-defm SUST_B_2D_V2I16_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b16.clamp", B16>;
-defm SUST_B_2D_V2I32_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b32.clamp", B32>;
-defm SUST_B_2D_V2I64_CLAMP : SUST_2D_V2<"sust.b.2d.v2.b64.clamp", B64>;
-
-defm SUST_B_2D_V2I8_TRAP : SUST_2D_V2<"sust.b.2d.v2.b8.trap", B16>;
-defm SUST_B_2D_V2I16_TRAP : SUST_2D_V2<"sust.b.2d.v2.b16.trap", B16>;
-defm SUST_B_2D_V2I32_TRAP : SUST_2D_V2<"sust.b.2d.v2.b32.trap", B32>;
-defm SUST_B_2D_V2I64_TRAP : SUST_2D_V2<"sust.b.2d.v2.b64.trap", B64>;
-
-defm SUST_B_2D_V2I8_ZERO : SUST_2D_V2<"sust.b.2d.v2.b8.zero", B16>;
-defm SUST_B_2D_V2I16_ZERO : SUST_2D_V2<"sust.b.2d.v2.b16.zero", B16>;
-defm SUST_B_2D_V2I32_ZERO : SUST_2D_V2<"sust.b.2d.v2.b32.zero", B32>;
-defm SUST_B_2D_V2I64_ZERO : SUST_2D_V2<"sust.b.2d.v2.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_2D_V2I8_ # op_upper : SUST_2D_V2<"sust.b.2d.v2.b8." # op, B16>;
+  defm SUST_B_2D_V2I16_ # op_upper : SUST_2D_V2<"sust.b.2d.v2.b16." # op, B16>;
+  defm SUST_B_2D_V2I32_ # op_upper : SUST_2D_V2<"sust.b.2d.v2.b32." # op, B32>;
+  defm SUST_B_2D_V2I64_ # op_upper : SUST_2D_V2<"sust.b.2d.v2.b64." # op, B64>;
+}
 
 defm SUST_P_2D_V2I8_TRAP : SUST_2D_V2<"sust.p.2d.v2.b8.trap", B16>;
 defm SUST_P_2D_V2I16_TRAP : SUST_2D_V2<"sust.p.2d.v2.b16.trap", B16>;
@@ -4283,17 +4120,12 @@ multiclass SUST_2D_V4<string inst, NVPTXRegClass intype> {
   def _I : SUST_2D_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_V4I8_CLAMP : SUST_2D_V4<"sust.b.2d.v4.b8.clamp", B16>;
-defm SUST_B_2D_V4I16_CLAMP : SUST_2D_V4<"sust.b.2d.v4.b16.clamp", B16>;
-defm SUST_B_2D_V4I32_CLAMP : SUST_2D_V4<"sust.b.2d.v4.b32.clamp", B32>;
-
-defm SUST_B_2D_V4I8_TRAP : SUST_2D_V4<"sust.b.2d.v4.b8.trap", B16>;
-defm SUST_B_2D_V4I16_TRAP : SUST_2D_V4<"sust.b.2d.v4.b16.trap", B16>;
-defm SUST_B_2D_V4I32_TRAP : SUST_2D_V4<"sust.b.2d.v4.b32.trap", B32>;
-
-defm SUST_B_2D_V4I8_ZERO : SUST_2D_V4<"sust.b.2d.v4.b8.zero", B16>;
-defm SUST_B_2D_V4I16_ZERO : SUST_2D_V4<"sust.b.2d.v4.b16.zero", B16>;
-defm SUST_B_2D_V4I32_ZERO : SUST_2D_V4<"sust.b.2d.v4.b32.zero", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_2D_V4I8_ # op_upper : SUST_2D_V4<"sust.b.2d.v4.b8." # op, B16>;
+  defm SUST_B_2D_V4I16_ # op_upper : SUST_2D_V4<"sust.b.2d.v4.b16." # op, B16>;
+  defm SUST_B_2D_V4I32_ # op_upper : SUST_2D_V4<"sust.b.2d.v4.b32." # op, B32>;
+}
 
 defm SUST_P_2D_V4I8_TRAP : SUST_2D_V4<"sust.p.2d.v4.b8.trap", B16>;
 defm SUST_P_2D_V4I16_TRAP : SUST_2D_V4<"sust.p.2d.v4.b16.trap", B16>;
@@ -4313,20 +4145,13 @@ multiclass SUST_2D_ARRAY<string inst, NVPTXRegClass intype> {
   def _I : SUST_2D_ARRAY_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_ARRAY_I8_CLAMP : SUST_2D_ARRAY<"sust.b.a2d.b8.clamp", B16>;
-defm SUST_B_2D_ARRAY_I16_CLAMP : SUST_2D_ARRAY<"sust.b.a2d.b16.clamp", B16>;
-defm SUST_B_2D_ARRAY_I32_CLAMP : SUST_2D_ARRAY<"sust.b.a2d.b32.clamp", B32>;
-defm SUST_B_2D_ARRAY_I64_CLAMP : SUST_2D_ARRAY<"sust.b.a2d.b64.clamp", B64>;
-
-defm SUST_B_2D_ARRAY_I8_TRAP : SUST_2D_ARRAY<"sust.b.a2d.b8.trap", B16>;
-defm SUST_B_2D_ARRAY_I16_TRAP : SUST_2D_ARRAY<"sust.b.a2d.b16.trap", B16>;
-defm SUST_B_2D_ARRAY_I32_TRAP : SUST_2D_ARRAY<"sust.b.a2d.b32.trap", B32>;
-defm SUST_B_2D_ARRAY_I64_TRAP : SUST_2D_ARRAY<"sust.b.a2d.b64.trap", B64>;
-
-defm SUST_B_2D_ARRAY_I8_ZERO : SUST_2D_ARRAY<"sust.b.a2d.b8.zero", B16>;
-defm SUST_B_2D_ARRAY_I16_ZERO : SUST_2D_ARRAY<"sust.b.a2d.b16.zero", B16>;
-defm SUST_B_2D_ARRAY_I32_ZERO : SUST_2D_ARRAY<"sust.b.a2d.b32.zero", B32>;
-defm SUST_B_2D_ARRAY_I64_ZERO : SUST_2D_ARRAY<"sust.b.a2d.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_2D_ARRAY_I8_ # op_upper : SUST_2D_ARRAY<"sust.b.a2d.b8." # op, B16>;
+  defm SUST_B_2D_ARRAY_I16_ # op_upper : SUST_2D_ARRAY<"sust.b.a2d.b16." # op, B16>;
+  defm SUST_B_2D_ARRAY_I32_ # op_upper : SUST_2D_ARRAY<"sust.b.a2d.b32." # op, B32>;
+  defm SUST_B_2D_ARRAY_I64_ # op_upper : SUST_2D_ARRAY<"sust.b.a2d.b64." # op, B64>;
+}
 
 defm SUST_P_2D_ARRAY_I8_TRAP : SUST_2D_ARRAY<"sust.p.a2d.b8.trap", B16>;
 defm SUST_P_2D_ARRAY_I16_TRAP : SUST_2D_ARRAY<"sust.p.a2d.b16.trap", B16>;
@@ -4346,20 +4171,13 @@ multiclass SUST_2D_ARRAY_V2<string inst, NVPTXRegClass intype> {
   def _I : SUST_2D_ARRAY_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_ARRAY_V2I8_CLAMP : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b8.clamp", B16>;
-defm SUST_B_2D_ARRAY_V2I16_CLAMP : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b16.clamp", B16>;
-defm SUST_B_2D_ARRAY_V2I32_CLAMP : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b32.clamp", B32>;
-defm SUST_B_2D_ARRAY_V2I64_CLAMP : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b64.clamp", B64>;
-
-defm SUST_B_2D_ARRAY_V2I8_TRAP : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b8.trap", B16>;
-defm SUST_B_2D_ARRAY_V2I16_TRAP : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b16.trap", B16>;
-defm SUST_B_2D_ARRAY_V2I32_TRAP : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b32.trap", B32>;
-defm SUST_B_2D_ARRAY_V2I64_TRAP : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b64.trap", B64>;
-
-defm SUST_B_2D_ARRAY_V2I8_ZERO : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b8.zero", B16>;
-defm SUST_B_2D_ARRAY_V2I16_ZERO : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b16.zero", B16>;
-defm SUST_B_2D_ARRAY_V2I32_ZERO : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b32.zero", B32>;
-defm SUST_B_2D_ARRAY_V2I64_ZERO : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b64.zero", B64>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_2D_ARRAY_V2I8_ # op_upper : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b8." # op, B16>;
+  defm SUST_B_2D_ARRAY_V2I16_ # op_upper : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b16." # op, B16>;
+  defm SUST_B_2D_ARRAY_V2I32_ # op_upper : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b32." # op, B32>;
+  defm SUST_B_2D_ARRAY_V2I64_ # op_upper : SUST_2D_ARRAY_V2<"sust.b.a2d.v2.b64." # op, B64>;
+}
 
 defm SUST_P_2D_ARRAY_V2I8_TRAP : SUST_2D_ARRAY_V2<"sust.p.a2d.v2.b8.trap", B16>;
 defm SUST_P_2D_ARRAY_V2I16_TRAP : SUST_2D_ARRAY_V2<"sust.p.a2d.v2.b16.trap", B16>;
@@ -4379,17 +4197,12 @@ multiclass SUST_2D_ARRAY_V4<string inst, NVPTXRegClass intype> {
   def _I : SUST_2D_ARRAY_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_2D_ARRAY_V4I8_CLAMP : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b8.clamp", B16>;
-defm SUST_B_2D_ARRAY_V4I16_CLAMP : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b16.clamp", B16>;
-defm SUST_B_2D_ARRAY_V4I32_CLAMP : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b32.clamp", B32>;
-
-defm SUST_B_2D_ARRAY_V4I8_TRAP : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b8.trap", B16>;
-defm SUST_B_2D_ARRAY_V4I16_TRAP : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b16.trap", B16>;
-defm SUST_B_2D_ARRAY_V4I32_TRAP : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b32.trap", B32>;
-
-defm SUST_B_2D_ARRAY_V4I8_ZERO : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b8.zero", B16>;
-defm SUST_B_2D_ARRAY_V4I16_ZERO : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b16.zero", B16>;
-defm SUST_B_2D_ARRAY_V4I32_ZERO : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b32.zero", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_2D_ARRAY_V4I8_ # op_upper : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b8." # op, B16>;
+  defm SUST_B_2D_ARRAY_V4I16_ # op_upper : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b16." # op, B16>;
+  defm SUST_B_2D_ARRAY_V4I32_ # op_upper : SUST_2D_ARRAY_V4<"sust.b.a2d.v4.b32." # op, B32>;
+}
 
 defm SUST_P_2D_ARRAY_V4I8_TRAP : SUST_2D_ARRAY_V4<"sust.p.a2d.v4.b8.trap", B16>;
 defm SUST_P_2D_ARRAY_V4I16_TRAP : SUST_2D_ARRAY_V4<"sust.p.a2d.v4.b16.trap", B16>;
@@ -4409,21 +4222,13 @@ multiclass SUST_3D<string inst, NVPTXRegClass intype> {
   def _I : SUST_3D_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_3D_I8_CLAMP : SUST_3D<"sust.b.3d.b8.clamp", B16>;
-defm SUST_B_3D_I16_CLAMP : SUST_3D<"sust.b.3d.b16.clamp", B16>;
-defm SUST_B_3D_I32_CLAMP : SUST_3D<"sust.b.3d.b32.clamp", B32>;
-defm SUST_B_3D_I64_CLAMP : SUST_3D<"sust.b.3d.b64.clamp", B64>;
-
-defm SUST_B_3D_I8_TRAP : SUST_3D<"sust.b.3d.b8.trap", B16>;
-defm SUST_B_3D_I16_TRAP : SUST_3D<"sust.b.3d.b16.trap", B16>;
-defm SUST_B_3D_I32_TRAP : SUST_3D<"sust.b.3d.b32.trap", B32>;
-defm SUST_B_3D_I64_TRAP : SUST_3D<"sust.b.3d.b64.trap", B64>;
-
-defm SUST_B_3D_I8_ZERO : SUST_3D<"sust.b.3d.b8.zero", B16>;
-defm SUST_B_3D_I16_ZERO : SUST_3D<"sust.b.3d.b16.zero", B16>;
-defm SUST_B_3D_I32_ZERO : SUST_3D<"sust.b.3d.b32.zero", B32>;
-defm SUST_B_3D_I64_ZERO : SUST_3D<"sust.b.3d.b64.zero", B64>;
-
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_3D_I8_ # op_upper : SUST_3D<"sust.b.3d.b8." # op, B16>;
+  defm SUST_B_3D_I16_ # op_upper : SUST_3D<"sust.b.3d.b16." # op, B16>;
+  defm SUST_B_3D_I32_ # op_upper : SUST_3D<"sust.b.3d.b32." # op, B32>;
+  defm SUST_B_3D_I64_ # op_upper : SUST_3D<"sust.b.3d.b64." # op, B64>;
+}
 defm SUST_P_3D_I8_TRAP : SUST_3D<"sust.p.3d.b8.trap", B16>;
 defm SUST_P_3D_I16_TRAP : SUST_3D<"sust.p.3d.b16.trap", B16>;
 defm SUST_P_3D_I32_TRAP : SUST_3D<"sust.p.3d.b32.trap", B32>;
@@ -4442,21 +4247,13 @@ multiclass SUST_3D_V2<string inst, NVPTXRegClass intype> {
   def _I : SUST_3D_V2_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_3D_V2I8_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b8.clamp", B16>;
-defm SUST_B_3D_V2I16_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b16.clamp", B16>;
-defm SUST_B_3D_V2I32_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b32.clamp", B32>;
-defm SUST_B_3D_V2I64_CLAMP : SUST_3D_V2<"sust.b.3d.v2.b64.clamp", B64>;
-
-defm SUST_B_3D_V2I8_TRAP : SUST_3D_V2<"sust.b.3d.v2.b8.trap", B16>;
-defm SUST_B_3D_V2I16_TRAP : SUST_3D_V2<"sust.b.3d.v2.b16.trap", B16>;
-defm SUST_B_3D_V2I32_TRAP : SUST_3D_V2<"sust.b.3d.v2.b32.trap", B32>;
-defm SUST_B_3D_V2I64_TRAP : SUST_3D_V2<"sust.b.3d.v2.b64.trap", B64>;
-
-defm SUST_B_3D_V2I8_ZERO : SUST_3D_V2<"sust.b.3d.v2.b8.zero", B16>;
-defm SUST_B_3D_V2I16_ZERO : SUST_3D_V2<"sust.b.3d.v2.b16.zero", B16>;
-defm SUST_B_3D_V2I32_ZERO : SUST_3D_V2<"sust.b.3d.v2.b32.zero", B32>;
-defm SUST_B_3D_V2I64_ZERO : SUST_3D_V2<"sust.b.3d.v2.b64.zero", B64>;
-
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_3D_V2I8_ # op_upper : SUST_3D_V2<"sust.b.3d.v2.b8." # op, B16>;
+  defm SUST_B_3D_V2I16_ # op_upper : SUST_3D_V2<"sust.b.3d.v2.b16." # op, B16>;
+  defm SUST_B_3D_V2I32_ # op_upper : SUST_3D_V2<"sust.b.3d.v2.b32." # op, B32>;
+  defm SUST_B_3D_V2I64_ # op_upper : SUST_3D_V2<"sust.b.3d.v2.b64." # op, B64>;
+}
 defm SUST_P_3D_V2I8_TRAP : SUST_3D_V2<"sust.p.3d.v2.b8.trap", B16>;
 defm SUST_P_3D_V2I16_TRAP : SUST_3D_V2<"sust.p.3d.v2.b16.trap", B16>;
 defm SUST_P_3D_V2I32_TRAP : SUST_3D_V2<"sust.p.3d.v2.b32.trap", B32>;
@@ -4475,17 +4272,12 @@ multiclass SUST_3D_V4<string inst, NVPTXRegClass intype> {
   def _I : SUST_3D_V4_base<inst, intype, (ins i64imm:$s), []>;
 }
 
-defm SUST_B_3D_V4I8_CLAMP : SUST_3D_V4<"sust.b.3d.v4.b8.clamp", B16>;
-defm SUST_B_3D_V4I16_CLAMP : SUST_3D_V4<"sust.b.3d.v4.b16.clamp", B16>;
-defm SUST_B_3D_V4I32_CLAMP : SUST_3D_V4<"sust.b.3d.v4.b32.clamp", B32>;
-
-defm SUST_B_3D_V4I8_TRAP : SUST_3D_V4<"sust.b.3d.v4.b8.trap", B16>;
-defm SUST_B_3D_V4I16_TRAP : SUST_3D_V4<"sust.b.3d.v4.b16.trap", B16>;
-defm SUST_B_3D_V4I32_TRAP : SUST_3D_V4<"sust.b.3d.v4.b32.trap", B32>;
-
-defm SUST_B_3D_V4I8_ZERO : SUST_3D_V4<"sust.b.3d.v4.b8.zero", B16>;
-defm SUST_B_3D_V4I16_ZERO : SUST_3D_V4<"sust.b.3d.v4.b16.zero", B16>;
-defm SUST_B_3D_V4I32_ZERO : SUST_3D_V4<"sust.b.3d.v4.b32.zero", B32>;
+foreach op = ["clamp", "trap", "zero"] in {
+  defvar op_upper = !toupper(op);
+  defm SUST_B_3D_V4I8_ # op_upper : SUST_3D_V4<"sust.b.3d.v4.b8." # op, B16>;
+  defm SUST_B_3D_V4I16_ # op_upper : SUST_3D_V4<"sust.b.3d.v4.b16." # op, B16>;
+  defm SUST_B_3D_V4I32_ # op_upper : SUST_3D_V4<"sust.b.3d.v4.b32." # op, B32>;
+}
 
 defm SUST_P_3D_V4I8_TRAP : SUST_3D_V4<"sust.p.3d.v4.b8.trap", B16>;
 defm SUST_P_3D_V4I16_TRAP : SUST_3D_V4<"sust.p.3d.v4.b16.trap", B16>;
@@ -4566,10 +4358,12 @@ let hasSideEffects = 1 in {
   def SREG_CLOCK : PTX_READ_SREG_R32<"clock", int_nvvm_read_ptx_sreg_clock>;
   def SREG_CLOCK64 : PTX_READ_SREG_R64<"clock64", int_nvvm_read_ptx_sreg_clock64>;
   def SREG_GLOBALTIMER : PTX_READ_SREG_R64<"globaltimer", int_nvvm_read_ptx_sreg_globaltimer>;
+  def SREG_GLOBALTIMER_LO : PTX_READ_SREG_R32<"globaltimer_lo", int_nvvm_read_ptx_sreg_globaltimer_lo>;
 }
 
 def: Pat <(i64 (readcyclecounter)), (SREG_CLOCK64)>;
 def: Pat <(i64 (readsteadycounter)), (SREG_GLOBALTIMER)>;
+def: Pat <(i32 (readsteadycounter)), (SREG_GLOBALTIMER_LO)>;
 
 def INT_PTX_SREG_PM0 : PTX_READ_SREG_R32<"pm0", int_nvvm_read_ptx_sreg_pm0>;
 def INT_PTX_SREG_PM1 : PTX_READ_SREG_R32<"pm1", int_nvvm_read_ptx_sreg_pm1>;
@@ -4586,10 +4380,15 @@ def INT_PTX_SREG_WARPSIZE :
 // In addition to target-independent fields provided by WMMA_REGS, it adds
 // the fields commonly used to implement specific PTX instruction -- register
 // types and names, constraints, parts of assembly, etc.
-class WMMA_REGINFO<WMMA_REGS r, string op>
-      : WMMA_REGS<r.geom, r.frag, r.ptx_elt_type> {
+class WMMA_REGINFO<WMMA_REGS r, string op, string metadata = "", string kind = "">
+      : WMMA_REGS<r.geom, r.frag, r.ptx_elt_type, !eq(op, "mma.sp")> {
   // NVPTX register types used to carry fragment data.
   NVPTXRegClass regclass = !cond(
+    !eq(ptx_elt_type, "e4m3") : B32,
+    !eq(ptx_elt_type, "e5m2") : B32,
+    !eq(ptx_elt_type, "e3m2") : B32,
+    !eq(ptx_elt_type, "e2m3") : B32,
+    !eq(ptx_elt_type, "e2m1") : B32,
     !eq(ptx_elt_type, "f16") : B32,
     !eq(ptx_elt_type, "f32") : B32,
     !eq(ptx_elt_type, "f64") : B64,
@@ -4622,6 +4421,18 @@ class WMMA_REGINFO<WMMA_REGS r, string op>
   // longer the case, we can concat all per-fragment predicates to enforce that
   // all fragments of the instruction are viable.
   list<Predicate> Predicates = !cond(
+    !or(!eq(ptx_elt_type, "e3m2"),
+        !eq(ptx_elt_type, "e2m3"),
+        !eq(ptx_elt_type, "e2m1"),
+        !ne(kind, "")) : [hasSM120a, hasPTX<87>],
+
+    !or(!eq(ptx_elt_type, "e4m3"),
+        !eq(ptx_elt_type, "e5m2")) : [hasSM<89>, hasPTX<84>],
+
+    !and(!eq(op, "mma.sp"),
+         !ne(metadata, "sp")) : [hasSM<80>, hasPTX<85>],
+    !eq(op, "mma.sp") : [hasSM<80>, hasPTX<71>],
+
     // fp16 -> fp16/fp32 @ m16n16k16
     !and(!eq(geom, "m16n16k16"),
          !or(!eq(ptx_elt_type, "f16"),
@@ -4726,7 +4537,8 @@ class BuildPatternI<Intrinsic Intr, dag Ins> {
   // Build a dag pattern that matches the intrinsic call.
   dag ret = !foreach(tmp, Ins,
                           !subst(ADDR, addr,
-                          !subst(ins, Intr, tmp)));
+                          !subst(ins, Intr,
+                          !subst(i32imm, timm, tmp))));
 }
 
 // Same as above, but uses PatFrag instead of an Intrinsic.
@@ -4960,6 +4772,62 @@ defset list<WMMA_INSTR> MMAs  = {
 } // defset
 }
 
+// MMA SP
+class MMA_SP<WMMA_REGINFO FragA, WMMA_REGINFO FragB,
+             WMMA_REGINFO FragC, WMMA_REGINFO FragD,
+             string Metadata, string Kind, int Satfinite>
+  : WMMA_INSTR<MMA_SP_NAME<Metadata, Kind, Satfinite,
+                           FragA, FragB, FragC, FragD>.record,
+               [FragA.Ins, FragB.Ins, FragC.Ins,
+                (ins B32:$metadata, i32imm:$selector)]>,
+    // Requires does not seem to have effect on Instruction w/o Patterns.
+    // We set it here anyways and propagate to the Pat<> we construct below.
+    Requires<!listconcat(FragA.Predicates,
+                         FragB.Predicates,
+                         FragC.Predicates,
+                         FragD.Predicates)> {
+  let OutOperandList = FragD.Outs;
+  let InOperandList = !con(Args, (ins MmaCode:$ptx));
+  string TypeList = "." # FragD.ptx_elt_type
+                    # "." # FragA.ptx_elt_type
+                    # "." # FragB.ptx_elt_type
+                    # "." # FragC.ptx_elt_type;
+  let AsmString = "mma"
+                  # "." # Metadata
+                  # ".sync.aligned."
+                  # FragA.geom
+                  # ".row.col"
+                  # !if(!ne(Kind, ""), "." # Kind, "")
+                  # !if(Satfinite, ".satfinite", "")
+                  # TypeList # "\n\t\t"
+                  # FragD.regstring # ",\n\t\t"
+                  # FragA.regstring # ",\n\t\t"
+                  # FragB.regstring # ",\n\t\t"
+                  # FragC.regstring # ",\n\t\t"
+                  # "$metadata" # ",\n\t\t"
+                  # "$selector" # ";";
+}
+
+let isConvergent = true in {
+defset list<WMMA_INSTR> MMA_SPs = {
+  foreach metadata = ["sp", "sp::ordered_metadata"] in {
+    foreach kind = ["", "kind::f8f6f4"] in {
+      foreach satf = [0, 1] in {
+        foreach op = NVVM_MMA_OPS.all_mma_sp_ops in {
+          if NVVM_MMA_SP_SUPPORTED<op, metadata, kind, satf>.ret then {
+            def : MMA_SP<WMMA_REGINFO<op[0], "mma.sp", metadata, kind>,
+                         WMMA_REGINFO<op[1], "mma.sp", metadata, kind>,
+                         WMMA_REGINFO<op[2], "mma.sp", metadata, kind>,
+                         WMMA_REGINFO<op[3], "mma.sp", metadata, kind>,
+                         metadata, kind, satf>;
+          }
+        } // op
+      } // satf
+    } // kind
+  } // metadata
+} // defset
+}
+
 //
 // ldmatrix.sync.aligned.m8n8[|.trans][|.shared].b16
 //
@@ -5041,7 +4909,7 @@ class MMA_PAT<WMMA_INSTR wi>
         Requires<wi.Predicates>;
 
 // Build intrinsic->instruction patterns for all MMA instructions.
-foreach mma = !listconcat(MMAs, WMMAs, MMA_LDSTs, LDMATRIXs, STMATRIXs) in
+foreach mma = !listconcat(MMAs, WMMAs, MMA_LDSTs, LDMATRIXs, STMATRIXs, MMA_SPs) in
   def : MMA_PAT<mma>;
 
 multiclass MAPA<string suffix, Intrinsic Intr> {
@@ -5102,27 +4970,23 @@ defm INT_SET_MAXNREG_DEC : SET_MAXNREG<"dec", int_nvvm_setmaxnreg_dec_sync_align
 //
 // WGMMA fence instructions
 //
-let isConvergent = true in {
-def INT_NVVM_WGMMA_FENCE_SYNC_ALIGNED : BasicNVPTXInst<(outs), (ins), "wgmma.fence.sync.aligned",
-                             [(int_nvvm_wgmma_fence_sync_aligned)]>, Requires<[hasSM90a, hasPTX<80>]>;
+let isConvergent = true, Predicates = [hasSM90a, hasPTX<80>] in {
+  def WGMMA_FENCE_SYNC_ALIGNED : NullaryInst<"wgmma.fence.sync.aligned", int_nvvm_wgmma_fence_sync_aligned>;
 
-def INT_NVVM_WGMMA_COMMIT_GROUP_SYNC_ALIGNED : BasicNVPTXInst<(outs), (ins), "wgmma.commit_group.sync.aligned",
-                             [(int_nvvm_wgmma_commit_group_sync_aligned)]>, Requires<[hasSM90a, hasPTX<80>]>;
+  def WGMMA_COMMIT_GROUP_SYNC_ALIGNED : NullaryInst<"wgmma.commit_group.sync.aligned", int_nvvm_wgmma_commit_group_sync_aligned>;
 
-def INT_NVVM_WGMMA_WAIT_GROUP_SYNC_ALIGNED : BasicNVPTXInst<(outs), (ins i64imm:$n), "wgmma.wait_group.sync.aligned",
-                             [(int_nvvm_wgmma_wait_group_sync_aligned timm:$n)]>, Requires<[hasSM90a, hasPTX<80>]>;
-} // isConvergent = true
+  def WGMMA_WAIT_GROUP_SYNC_ALIGNED : BasicNVPTXInst<(outs), (ins i64imm:$n), "wgmma.wait_group.sync.aligned",
+                              [(int_nvvm_wgmma_wait_group_sync_aligned timm:$n)]>;
+}
 
 let Predicates = [hasSM<90>, hasPTX<78>] in {
   def GRIDDEPCONTROL_LAUNCH_DEPENDENTS :
-        BasicNVPTXInst<(outs), (ins), "griddepcontrol.launch_dependents",
-                  [(int_nvvm_griddepcontrol_launch_dependents)]>;
+        NullaryInst<"griddepcontrol.launch_dependents", int_nvvm_griddepcontrol_launch_dependents>;
   def GRIDDEPCONTROL_WAIT :
-        BasicNVPTXInst<(outs), (ins), "griddepcontrol.wait",
-                  [(int_nvvm_griddepcontrol_wait)]>;
+        NullaryInst<"griddepcontrol.wait", int_nvvm_griddepcontrol_wait>;
 }
 
-def INT_EXIT : BasicNVPTXInst<(outs), (ins), "exit", [(int_nvvm_exit)]>;
+def EXIT : NullaryInst<"exit", int_nvvm_exit>;
 
 // Tcgen05 intrinsics
 let isConvergent = true, Predicates = [hasTcgen05Instructions] in {
@@ -5150,9 +5014,7 @@ defm TCGEN05_DEALLOC_CG1: TCGEN05_DEALLOC_INTR<"1", int_nvvm_tcgen05_dealloc_cg1
 defm TCGEN05_DEALLOC_CG2: TCGEN05_DEALLOC_INTR<"2", int_nvvm_tcgen05_dealloc_cg2>;
 
 multiclass TCGEN05_RELINQ_PERMIT_INTR<string num, Intrinsic Intr> {
-  def "" : BasicNVPTXInst<(outs), (ins),
-             "tcgen05.relinquish_alloc_permit.cta_group::" # num # ".sync.aligned",
-             [(Intr)]>;
+  def "" : NullaryInst<"tcgen05.relinquish_alloc_permit.cta_group::" # num # ".sync.aligned", Intr>;
 }
 defm TCGEN05_RELINQ_CG1: TCGEN05_RELINQ_PERMIT_INTR<"1", int_nvvm_tcgen05_relinq_alloc_permit_cg1>;
 defm TCGEN05_RELINQ_CG2: TCGEN05_RELINQ_PERMIT_INTR<"2", int_nvvm_tcgen05_relinq_alloc_permit_cg2>;
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index f4362fe8d9056..e2bbe57c0085c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -412,6 +412,22 @@ static void adjustByValArgAlignment(Argument *Arg, Value *ArgInParamAS,
   }
 }
 
+// Create a call to the nvvm_internal_addrspace_wrap intrinsic and set the
+// alignment of the return value based on the alignment of the argument.
+static CallInst *createNVVMInternalAddrspaceWrap(IRBuilder<> &IRB,
+                                                 Argument &Arg) {
+  CallInst *ArgInParam =
+      IRB.CreateIntrinsic(Intrinsic::nvvm_internal_addrspace_wrap,
+                          {IRB.getPtrTy(ADDRESS_SPACE_PARAM), Arg.getType()},
+                          &Arg, {}, Arg.getName() + ".param");
+
+  if (MaybeAlign ParamAlign = Arg.getParamAlign())
+    ArgInParam->addRetAttr(
+        Attribute::getWithAlignment(ArgInParam->getContext(), *ParamAlign));
+
+  return ArgInParam;
+}
+
 namespace {
 struct ArgUseChecker : PtrUseVisitor<ArgUseChecker> {
   using Base = PtrUseVisitor<ArgUseChecker>;
@@ -515,10 +531,7 @@ void copyByValParam(Function &F, Argument &Arg) {
       Arg.getParamAlign().value_or(DL.getPrefTypeAlign(StructType)));
   Arg.replaceAllUsesWith(AllocA);
 
-  Value *ArgInParam =
-      IRB.CreateIntrinsic(Intrinsic::nvvm_internal_addrspace_wrap,
-                          {IRB.getPtrTy(ADDRESS_SPACE_PARAM), Arg.getType()},
-                          &Arg, {}, Arg.getName());
+  CallInst *ArgInParam = createNVVMInternalAddrspaceWrap(IRB, Arg);
 
   // Be sure to propagate alignment to this load; LLVM doesn't know that NVPTX
   // addrspacecast preserves alignment.  Since params are constant, this load
@@ -549,9 +562,7 @@ static void handleByValParam(const NVPTXTargetMachine &TM, Argument *Arg) {
     SmallVector<Use *, 16> UsesToUpdate(llvm::make_pointer_range(Arg->uses()));
 
     IRBuilder<> IRB(&*FirstInst);
-    Value *ArgInParamAS = IRB.CreateIntrinsic(
-        Intrinsic::nvvm_internal_addrspace_wrap,
-        {IRB.getPtrTy(ADDRESS_SPACE_PARAM), Arg->getType()}, {Arg});
+    CallInst *ArgInParamAS = createNVVMInternalAddrspaceWrap(IRB, *Arg);
 
     for (Use *U : UsesToUpdate)
       convertToParamAS(U, ArgInParamAS, HasCvtaParam, IsGridConstant);
@@ -581,10 +592,7 @@ static void handleByValParam(const NVPTXTargetMachine &TM, Argument *Arg) {
     // argument already in the param address space, we need to use the noop
     // intrinsic, this had the added benefit of preventing other optimizations
     // from folding away this pair of addrspacecasts.
-    auto *ParamSpaceArg =
-        IRB.CreateIntrinsic(Intrinsic::nvvm_internal_addrspace_wrap,
-                            {IRB.getPtrTy(ADDRESS_SPACE_PARAM), Arg->getType()},
-                            Arg, {}, Arg->getName() + ".param");
+    auto *ParamSpaceArg = createNVVMInternalAddrspaceWrap(IRB, *Arg);
 
     // Cast param address to generic address space.
     Value *GenericArg = IRB.CreateAddrSpaceCast(
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp
index e5d680c19d921..a84ceaba991c7 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp
@@ -29,6 +29,12 @@ static cl::opt<bool>
     NoF16Math("nvptx-no-f16-math", cl::Hidden,
               cl::desc("NVPTX Specific: Disable generation of f16 math ops."),
               cl::init(false));
+
+static cl::opt<bool> NoF32x2("nvptx-no-f32x2", cl::Hidden,
+                             cl::desc("NVPTX Specific: Disable generation of "
+                                      "f32x2 instructions and registers."),
+                             cl::init(false));
+
 // Pin the vtable to this file.
 void NVPTXSubtarget::anchor() {}
 
@@ -70,6 +76,10 @@ bool NVPTXSubtarget::allowFP16Math() const {
   return hasFP16Math() && NoF16Math == false;
 }
 
+bool NVPTXSubtarget::hasF32x2Instructions() const {
+  return SmVersion >= 100 && PTXVersion >= 86 && !NoF32x2;
+}
+
 bool NVPTXSubtarget::hasNativeBF16Support(int Opcode) const {
   if (!hasBF16Math())
     return false;
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index 81af55edccadb..acf025b70ce34 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -117,9 +117,7 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
     return HasTcgen05 && PTXVersion >= 86;
   }
   // f32x2 instructions in Blackwell family
-  bool hasF32x2Instructions() const {
-    return SmVersion >= 100 && PTXVersion >= 86;
-  }
+  bool hasF32x2Instructions() const;
 
   // TMA G2S copy with cta_group::1/2 support
   bool hasCpAsyncBulkTensorCTAGroupSupport() const {
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 3ae2d9d5181a3..f4f89613b358d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -564,7 +564,8 @@ bool NVPTXTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
   case Intrinsic::nvvm_isspacep_global:
   case Intrinsic::nvvm_isspacep_local:
   case Intrinsic::nvvm_isspacep_shared:
-  case Intrinsic::nvvm_isspacep_shared_cluster: {
+  case Intrinsic::nvvm_isspacep_shared_cluster:
+  case Intrinsic::nvvm_prefetch_tensormap: {
     OpIndexes.push_back(0);
     return true;
   }
@@ -587,6 +588,11 @@ Value *NVPTXTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
       return ConstantInt::get(II->getType(), *R);
     return nullptr;
   }
+  case Intrinsic::nvvm_prefetch_tensormap: {
+    IRBuilder<> Builder(II);
+    return Builder.CreateUnaryIntrinsic(Intrinsic::nvvm_prefetch_tensormap,
+                                        NewV);
+  }
   }
   return nullptr;
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index 9a6e261c811a0..b32d931bd3074 100644
--- a/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -87,6 +87,13 @@ class NVPTXTTIImpl final : public BasicTTIImplBase<NVPTXTTIImpl> {
   }
   unsigned getMinVectorRegisterBitWidth() const override { return 32; }
 
+  bool shouldExpandReduction(const IntrinsicInst *II) const override {
+    // Turn off ExpandReductions pass for NVPTX, which doesn't have advanced
+    // swizzling operations. Our backend/Selection DAG can expand these
+    // reductions with less movs.
+    return false;
+  }
+
   // We don't want to prevent inlining because of target-cpu and -features
   // attributes that were added to newer versions of LLVM/Clang: There are
   // no incompatible functions in PTX, ptxas will throw errors in such cases.
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
index 6586f925504f1..274b04fdd30b5 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -352,6 +352,10 @@ std::optional<unsigned> getMaxNReg(const Function &F) {
   return getFnAttrParsedInt(F, "nvvm.maxnreg");
 }
 
+bool hasBlocksAreClusters(const Function &F) {
+  return F.hasFnAttribute("nvvm.blocksareclusters");
+}
+
 MaybeAlign getAlign(const CallInst &I, unsigned Index) {
   // First check the alignstack metadata
   if (MaybeAlign StackAlign =
diff --git a/llvm/lib/Target/NVPTX/NVPTXUtilities.h b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
index 4eb452f398220..a070789f85e0b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXUtilities.h
+++ b/llvm/lib/Target/NVPTX/NVPTXUtilities.h
@@ -60,6 +60,8 @@ std::optional<unsigned> getMaxClusterRank(const Function &);
 std::optional<unsigned> getMinCTASm(const Function &);
 std::optional<unsigned> getMaxNReg(const Function &);
 
+bool hasBlocksAreClusters(const Function &);
+
 inline bool isKernelFunction(const Function &F) {
   return F.getCallingConv() == CallingConv::PTX_Kernel;
 }
@@ -80,6 +82,8 @@ inline unsigned promoteScalarArgumentSize(unsigned size) {
     return 32;
   if (size <= 64)
     return 64;
+  if (size <= 128)
+    return 128;
   return size;
 }
 
@@ -87,7 +91,7 @@ bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM);
 
 inline bool shouldPassAsArray(Type *Ty) {
   return Ty->isAggregateType() || Ty->isVectorTy() ||
-         Ty->getScalarSizeInBits() == 128 || Ty->isHalfTy() || Ty->isBFloatTy();
+         Ty->getScalarSizeInBits() >= 128 || Ty->isHalfTy() || Ty->isBFloatTy();
 }
 
 namespace NVPTX {
diff --git a/llvm/lib/Target/PowerPC/CMakeLists.txt b/llvm/lib/Target/PowerPC/CMakeLists.txt
index 3808a26a0b92a..1e39f01fd7aa5 100644
--- a/llvm/lib/Target/PowerPC/CMakeLists.txt
+++ b/llvm/lib/Target/PowerPC/CMakeLists.txt
@@ -25,7 +25,6 @@ add_llvm_target(PowerPCCodeGen
   PPCBranchSelector.cpp
   PPCBranchCoalescing.cpp
   PPCCallingConv.cpp
-  PPCCCState.cpp
   PPCCTRLoops.cpp
   PPCCTRLoopsVerify.cpp
   PPCExpandAtomicPseudoInsts.cpp
diff --git a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
index 71a76142bb389..5e27f06c94f06 100644
--- a/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
+++ b/llvm/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
@@ -8,6 +8,7 @@
 
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include "TargetInfo/PowerPCTargetInfo.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
diff --git a/llvm/lib/Target/PowerPC/PPCCCState.cpp b/llvm/lib/Target/PowerPC/PPCCCState.cpp
deleted file mode 100644
index e08aaf89e8dbe..0000000000000
--- a/llvm/lib/Target/PowerPC/PPCCCState.cpp
+++ /dev/null
@@ -1,33 +0,0 @@
-//===---- PPCCCState.cpp - CCState with PowerPC specific extensions ---------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "PPCCCState.h"
-using namespace llvm;
-
-// Identify lowered values that originated from ppcf128 arguments and record
-// this.
-void PPCCCState::PreAnalyzeCallOperands(
-    const SmallVectorImpl<ISD::OutputArg> &Outs) {
-  for (const auto &I : Outs) {
-    if (I.ArgVT == llvm::MVT::ppcf128)
-      OriginalArgWasPPCF128.push_back(true);
-    else
-      OriginalArgWasPPCF128.push_back(false);
-  }
-}
-
-void PPCCCState::PreAnalyzeFormalArguments(
-    const SmallVectorImpl<ISD::InputArg> &Ins) {
-  for (const auto &I : Ins) {
-    if (I.ArgVT == llvm::MVT::ppcf128) {
-      OriginalArgWasPPCF128.push_back(true);
-    } else {
-      OriginalArgWasPPCF128.push_back(false);
-    }
-  }
-}
diff --git a/llvm/lib/Target/PowerPC/PPCCCState.h b/llvm/lib/Target/PowerPC/PPCCCState.h
deleted file mode 100644
index b0e50b230fb1a..0000000000000
--- a/llvm/lib/Target/PowerPC/PPCCCState.h
+++ /dev/null
@@ -1,73 +0,0 @@
-//===---- PPCCCState.h - CCState with PowerPC specific extensions -----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef PPCCCSTATE_H
-#define PPCCCSTATE_H
-
-#include "PPCISelLowering.h"
-#include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/CallingConvLower.h"
-
-namespace llvm {
-
-class PPCCCState : public CCState {
-public:
-
-  void
-  PreAnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs);
-  void
-  PreAnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins);
-
-private:
-
-  // Records whether the value has been lowered from an ppcf128.
-  SmallVector<bool, 4> OriginalArgWasPPCF128;
-
-public:
-  PPCCCState(CallingConv::ID CC, bool isVarArg, MachineFunction &MF,
-             SmallVectorImpl<CCValAssign> &locs, LLVMContext &C)
-        : CCState(CC, isVarArg, MF, locs, C) {}
-
-  bool WasOriginalArgPPCF128(unsigned ValNo) { return OriginalArgWasPPCF128[ValNo]; }
-  void clearWasPPCF128() { OriginalArgWasPPCF128.clear(); }
-};
-
-class AIXCCState : public CCState {
-private:
-  BitVector IsFixed;
-
-public:
-  AIXCCState(CallingConv::ID CC, bool IsVarArg, MachineFunction &MF,
-             SmallVectorImpl<CCValAssign> &Locs, LLVMContext &C)
-      : CCState(CC, IsVarArg, MF, Locs, C) {}
-
-  void AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins,
-                              CCAssignFn Fn) {
-    // All formal arguments are fixed.
-    IsFixed.resize(Ins.size(), true);
-    CCState::AnalyzeFormalArguments(Ins, Fn);
-  }
-
-  void AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
-                           CCAssignFn Fn) {
-    // Record whether the call operand was a fixed argument.
-    IsFixed.resize(Outs.size(), false);
-    for (unsigned ValNo = 0, E = Outs.size(); ValNo != E; ++ValNo)
-      if (Outs[ValNo].IsFixed)
-        IsFixed.set(ValNo);
-
-    CCState::AnalyzeCallOperands(Outs, Fn);
-  }
-
-  bool isFixed(unsigned ValNo) const { return IsFixed.test(ValNo); }
-};
-
-} // end namespace llvm
-
-#endif
diff --git a/llvm/lib/Target/PowerPC/PPCCTRLoopsVerify.cpp b/llvm/lib/Target/PowerPC/PPCCTRLoopsVerify.cpp
index 46aa27e1450a6..6aa2f0c96d1dc 100644
--- a/llvm/lib/Target/PowerPC/PPCCTRLoopsVerify.cpp
+++ b/llvm/lib/Target/PowerPC/PPCCTRLoopsVerify.cpp
@@ -18,7 +18,6 @@
 #ifndef NDEBUG
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include "PPC.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/ilist_iterator.h"
@@ -93,7 +92,7 @@ static bool clobbersCTR(const MachineInstr &MI) {
 static bool verifyCTRBranch(MachineBasicBlock *MBB,
                             MachineBasicBlock::iterator I) {
   MachineBasicBlock::iterator BI = I;
-  SmallSet<MachineBasicBlock *, 16>   Visited;
+  SmallPtrSet<MachineBasicBlock *, 16> Visited;
   SmallVector<MachineBasicBlock *, 8> Preds;
   bool CheckPreds;
 
diff --git a/llvm/lib/Target/PowerPC/PPCCallingConv.cpp b/llvm/lib/Target/PowerPC/PPCCallingConv.cpp
index b00d70bbe263b..332b5e07d3d5d 100644
--- a/llvm/lib/Target/PowerPC/PPCCallingConv.cpp
+++ b/llvm/lib/Target/PowerPC/PPCCallingConv.cpp
@@ -8,7 +8,6 @@
 
 #include "PPCCallingConv.h"
 #include "PPCSubtarget.h"
-#include "PPCCCState.h"
 using namespace llvm;
 
 inline bool CC_PPC_AnyReg_Error(unsigned &, MVT &, MVT &,
diff --git a/llvm/lib/Target/PowerPC/PPCCallingConv.h b/llvm/lib/Target/PowerPC/PPCCallingConv.h
index ab61472c72eb8..9c47142f78890 100644
--- a/llvm/lib/Target/PowerPC/PPCCallingConv.h
+++ b/llvm/lib/Target/PowerPC/PPCCallingConv.h
@@ -21,28 +21,29 @@ namespace llvm {
 
 bool RetCC_PPC(unsigned ValNo, MVT ValVT, MVT LocVT,
                CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-               CCState &State);
+               Type *OrigTy, CCState &State);
 bool RetCC_PPC64_ELF_FIS(unsigned ValNo, MVT ValVT, MVT LocVT,
                          CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                         CCState &State);
+                         Type *OrigTy, CCState &State);
 bool RetCC_PPC_Cold(unsigned ValNo, MVT ValVT, MVT LocVT,
                     CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                    CCState &State);
+                    Type *OrigTy, CCState &State);
 bool CC_PPC32_SVR4(unsigned ValNo, MVT ValVT, MVT LocVT,
                    CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                   CCState &State);
+                   Type *OrigTy, CCState &State);
 bool CC_PPC64_ELF(unsigned ValNo, MVT ValVT, MVT LocVT,
                   CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                  CCState &State);
+                  Type *OrigTy, CCState &State);
 bool CC_PPC64_ELF_FIS(unsigned ValNo, MVT ValVT, MVT LocVT,
                       CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                      CCState &State);
+                      Type *OrigTy, CCState &State);
 bool CC_PPC32_SVR4_ByVal(unsigned ValNo, MVT ValVT, MVT LocVT,
                          CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                         CCState &State);
+                         Type *OrigTy, CCState &State);
 bool CC_PPC32_SVR4_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
                           CCValAssign::LocInfo LocInfo,
-                          ISD::ArgFlagsTy ArgFlags, CCState &State);
+                          ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                          CCState &State);
 
 } // End llvm namespace
 
diff --git a/llvm/lib/Target/PowerPC/PPCCallingConv.td b/llvm/lib/Target/PowerPC/PPCCallingConv.td
index 899326ad46656..5d4fe06ebdddd 100644
--- a/llvm/lib/Target/PowerPC/PPCCallingConv.td
+++ b/llvm/lib/Target/PowerPC/PPCCallingConv.td
@@ -21,11 +21,9 @@ class CCIfNotSubtarget<string F, CCAction A>
                      F),
           A>;
 class CCIfOrigArgWasNotPPCF128<CCAction A>
-    : CCIf<"!static_cast<PPCCCState *>(&State)->WasOriginalArgPPCF128(ValNo)",
-           A>;
+    : CCIf<"!OrigTy->isPPC_FP128Ty()", A>;
 class CCIfOrigArgWasPPCF128<CCAction A>
-    : CCIf<"static_cast<PPCCCState *>(&State)->WasOriginalArgPPCF128(ValNo)",
-           A>;
+    : CCIf<"OrigTy->isPPC_FP128Ty()", A>;
 
 //===----------------------------------------------------------------------===//
 // Return Value Calling Convention
diff --git a/llvm/lib/Target/PowerPC/PPCFastISel.cpp b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
index e92e00f80c552..0b68ba12e337f 100644
--- a/llvm/lib/Target/PowerPC/PPCFastISel.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
@@ -1374,7 +1374,10 @@ bool PPCFastISel::processCallArgs(SmallVectorImpl<Value *> &Args,
   unsigned LinkageSize = Subtarget->getFrameLowering()->getLinkageSize();
   CCInfo.AllocateStack(LinkageSize, Align(8));
 
-  CCInfo.AnalyzeCallOperands(ArgVTs, ArgFlags, CC_PPC64_ELF_FIS);
+  SmallVector<Type *, 16> ArgTys;
+  for (Value *Arg : Args)
+    ArgTys.push_back(Arg->getType());
+  CCInfo.AnalyzeCallOperands(ArgVTs, ArgFlags, ArgTys, CC_PPC64_ELF_FIS);
 
   // Bail out if we can't handle any of the arguments.
   for (const CCValAssign &VA : ArgLocs) {
@@ -1487,7 +1490,7 @@ bool PPCFastISel::finishCall(MVT RetVT, CallLoweringInfo &CLI, unsigned &NumByte
   if (RetVT != MVT::isVoid) {
     SmallVector<CCValAssign, 16> RVLocs;
     CCState CCInfo(CC, false, *FuncInfo.MF, RVLocs, *Context);
-    CCInfo.AnalyzeCallResult(RetVT, RetCC_PPC64_ELF_FIS);
+    CCInfo.AnalyzeCallResult(RetVT, CLI.RetTy, RetCC_PPC64_ELF_FIS);
     CCValAssign &VA = RVLocs[0];
     assert(RVLocs.size() == 1 && "No support for multi-reg return values!");
     assert(VA.isRegLoc() && "Can only return in registers!");
@@ -1573,7 +1576,7 @@ bool PPCFastISel::fastLowerCall(CallLoweringInfo &CLI) {
       RetVT != MVT::f64) {
     SmallVector<CCValAssign, 16> RVLocs;
     CCState CCInfo(CC, IsVarArg, *FuncInfo.MF, RVLocs, *Context);
-    CCInfo.AnalyzeCallResult(RetVT, RetCC_PPC64_ELF_FIS);
+    CCInfo.AnalyzeCallResult(RetVT, RetTy, RetCC_PPC64_ELF_FIS);
     if (RVLocs.size() > 1)
       return false;
   }
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index c0860fcdaa3b0..2ad3ed21732ed 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -2078,8 +2078,10 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF,
   // tail call might not be in the new RestoreBlock, so real branch instruction
   // won't be generated by emitEpilogue(), because shrink-wrap has chosen new
   // RestoreBlock. So we handle this case here.
-  if (MFI.getSavePoint() && MFI.hasTailCall()) {
-    MachineBasicBlock *RestoreBlock = MFI.getRestorePoint();
+  if (!MFI.getSavePoints().empty() && MFI.hasTailCall()) {
+    assert(MFI.getRestorePoints().size() < 2 &&
+           "MFI can't contain multiple restore points!");
+    MachineBasicBlock *RestoreBlock = MFI.getRestorePoints().front();
     for (MachineBasicBlock &MBB : MF) {
       if (MBB.isReturnBlock() && (&MBB) != RestoreBlock)
         createTailCallBranchInstr(MBB);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 30b5fd6ad1005..7022e9e9dae99 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -14,7 +14,6 @@
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include "MCTargetDesc/PPCPredicates.h"
 #include "PPC.h"
-#include "PPCCCState.h"
 #include "PPCCallingConv.h"
 #include "PPCFrameLowering.h"
 #include "PPCInstrInfo.h"
@@ -30,7 +29,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
@@ -159,8 +157,6 @@ static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int);
 
 static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
 
-static const char AIXSSPCanaryWordName[] = "__ssp_canary_word";
-
 // A faster local-[exec|dynamic] TLS access sequence (enabled with the
 // -maix-small-local-[exec|dynamic]-tls option) can be produced for TLS
 // variables; consistent with the IBM XL compiler, we apply a max size of
@@ -1787,11 +1783,8 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::ADDI_DTPREL_L:   return "PPCISD::ADDI_DTPREL_L";
   case PPCISD::PADDI_DTPREL:
     return "PPCISD::PADDI_DTPREL";
-  case PPCISD::VADD_SPLAT:      return "PPCISD::VADD_SPLAT";
-  case PPCISD::SC:              return "PPCISD::SC";
-  case PPCISD::CLRBHRB:         return "PPCISD::CLRBHRB";
-  case PPCISD::MFBHRBE:         return "PPCISD::MFBHRBE";
-  case PPCISD::RFEBB:           return "PPCISD::RFEBB";
+  case PPCISD::VADD_SPLAT:
+    return "PPCISD::VADD_SPLAT";
   case PPCISD::XXSWAPD:         return "PPCISD::XXSWAPD";
   case PPCISD::SWAP_NO_CHAIN:   return "PPCISD::SWAP_NO_CHAIN";
   case PPCISD::BUILD_FP128:     return "PPCISD::BUILD_FP128";
@@ -3925,9 +3918,6 @@ SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
 
 SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
                                                   SelectionDAG &DAG) const {
-  if (Subtarget.isAIXABI())
-    report_fatal_error("ADJUST_TRAMPOLINE operation is not supported on AIX.");
-
   return Op.getOperand(0);
 }
 
@@ -3984,9 +3974,6 @@ SDValue PPCTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
 
 SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
                                                 SelectionDAG &DAG) const {
-  if (Subtarget.isAIXABI())
-    report_fatal_error("INIT_TRAMPOLINE operation is not supported on AIX.");
-
   SDValue Chain = Op.getOperand(0);
   SDValue Trmp = Op.getOperand(1); // trampoline
   SDValue FPtr = Op.getOperand(2); // nested function
@@ -3994,22 +3981,76 @@ SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
   SDLoc dl(Op);
 
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
+
+  if (Subtarget.isAIXABI()) {
+    // On AIX we create a trampoline descriptor by combining the
+    // entry point and TOC from the global descriptor (FPtr) with the
+    // nest argument as the environment pointer.
+    uint64_t PointerSize = Subtarget.isPPC64() ? 8 : 4;
+    MaybeAlign PointerAlign(PointerSize);
+    auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
+                        ? (MachineMemOperand::MODereferenceable |
+                           MachineMemOperand::MOInvariant)
+                        : MachineMemOperand::MONone;
+
+    uint64_t TOCPointerOffset = 1 * PointerSize;
+    uint64_t EnvPointerOffset = 2 * PointerSize;
+    SDValue SDTOCPtrOffset = DAG.getConstant(TOCPointerOffset, dl, PtrVT);
+    SDValue SDEnvPtrOffset = DAG.getConstant(EnvPointerOffset, dl, PtrVT);
+
+    const Value *TrampolineAddr =
+        cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
+    const Function *Func =
+        cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
+
+    SDValue OutChains[3];
+
+    // Copy the entry point address from the global descriptor to the
+    // trampoline buffer.
+    SDValue LoadEntryPoint =
+        DAG.getLoad(PtrVT, dl, Chain, FPtr, MachinePointerInfo(Func, 0),
+                    PointerAlign, MMOFlags);
+    SDValue EPLoadChain = LoadEntryPoint.getValue(1);
+    OutChains[0] = DAG.getStore(EPLoadChain, dl, LoadEntryPoint, Trmp,
+                                MachinePointerInfo(TrampolineAddr, 0));
+
+    // Copy the TOC pointer from the global descriptor to the trampoline
+    // buffer.
+    SDValue TOCFromDescriptorPtr =
+        DAG.getNode(ISD::ADD, dl, PtrVT, FPtr, SDTOCPtrOffset);
+    SDValue TOCReg = DAG.getLoad(PtrVT, dl, Chain, TOCFromDescriptorPtr,
+                                 MachinePointerInfo(Func, TOCPointerOffset),
+                                 PointerAlign, MMOFlags);
+    SDValue TrampolineTOCPointer =
+        DAG.getNode(ISD::ADD, dl, PtrVT, Trmp, SDTOCPtrOffset);
+    SDValue TOCLoadChain = TOCReg.getValue(1);
+    OutChains[1] =
+        DAG.getStore(TOCLoadChain, dl, TOCReg, TrampolineTOCPointer,
+                     MachinePointerInfo(TrampolineAddr, TOCPointerOffset));
+
+    // Store the nest argument into the environment pointer in the trampoline
+    // buffer.
+    SDValue EnvPointer = DAG.getNode(ISD::ADD, dl, PtrVT, Trmp, SDEnvPtrOffset);
+    OutChains[2] =
+        DAG.getStore(Chain, dl, Nest, EnvPointer,
+                     MachinePointerInfo(TrampolineAddr, EnvPointerOffset));
+
+    SDValue TokenFactor =
+        DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
+    return TokenFactor;
+  }
+
   bool isPPC64 = (PtrVT == MVT::i64);
   Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
 
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-
-  Entry.Ty = IntPtrTy;
-  Entry.Node = Trmp; Args.push_back(Entry);
-
+  Args.emplace_back(Trmp, IntPtrTy);
   // TrampSize == (isPPC64 ? 48 : 40);
-  Entry.Node =
-      DAG.getConstant(isPPC64 ? 48 : 40, dl, Subtarget.getScalarIntVT());
-  Args.push_back(Entry);
-
-  Entry.Node = FPtr; Args.push_back(Entry);
-  Entry.Node = Nest; Args.push_back(Entry);
+  Args.emplace_back(
+      DAG.getConstant(isPPC64 ? 48 : 40, dl, Subtarget.getScalarIntVT()),
+      IntPtrTy);
+  Args.emplace_back(FPtr, IntPtrTy);
+  Args.emplace_back(Nest, IntPtrTy);
 
   // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
   TargetLowering::CallLoweringInfo CLI(DAG);
@@ -4285,17 +4326,13 @@ SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
 
   // Assign locations to all of the incoming arguments.
   SmallVector<CCValAssign, 16> ArgLocs;
-  PPCCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
+  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
                  *DAG.getContext());
 
   // Reserve space for the linkage area on the stack.
   unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
   CCInfo.AllocateStack(LinkageSize, PtrAlign);
-  if (useSoftFloat())
-    CCInfo.PreAnalyzeFormalArguments(Ins);
-
   CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4);
-  CCInfo.clearWasPPCF128();
 
   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
     CCValAssign &VA = ArgLocs[i];
@@ -6017,13 +6054,11 @@ SDValue PPCTargetLowering::LowerCall_32SVR4(
 
   // Assign locations to all of the outgoing arguments.
   SmallVector<CCValAssign, 16> ArgLocs;
-  PPCCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
+  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
 
   // Reserve space for the linkage area on the stack.
   CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(),
                        PtrAlign);
-  if (useSoftFloat())
-    CCInfo.PreAnalyzeCallOperands(Outs);
 
   if (IsVarArg) {
     // Handle fixed and variable vector arguments differently.
@@ -6036,12 +6071,12 @@ SDValue PPCTargetLowering::LowerCall_32SVR4(
       ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
       bool Result;
 
-      if (Outs[i].IsFixed) {
+      if (!ArgFlags.isVarArg()) {
         Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags,
-                               CCInfo);
+                               Outs[i].OrigTy, CCInfo);
       } else {
         Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full,
-                                      ArgFlags, CCInfo);
+                                      ArgFlags, Outs[i].OrigTy, CCInfo);
       }
 
       if (Result) {
@@ -6056,7 +6091,6 @@ SDValue PPCTargetLowering::LowerCall_32SVR4(
     // All arguments are treated the same.
     CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4);
   }
-  CCInfo.clearWasPPCF128();
 
   // Assign locations to all of the outgoing aggregate by value arguments.
   SmallVector<CCValAssign, 16> ByValArgLocs;
@@ -6852,8 +6886,7 @@ static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) {
 
 static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
                    CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                   CCState &S) {
-  AIXCCState &State = static_cast<AIXCCState &>(S);
+                   Type *OrigTy, CCState &State) {
   const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>(
       State.getMachineFunction().getSubtarget());
   const bool IsPPC64 = Subtarget.isPPC64();
@@ -6865,9 +6898,6 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
   if (ValVT == MVT::f128)
     report_fatal_error("f128 is unimplemented on AIX.");
 
-  if (ArgFlags.isNest())
-    report_fatal_error("Nest arguments are unimplemented.");
-
   static const MCPhysReg GPR_32[] = {// 32-bit registers.
                                      PPC::R3, PPC::R4, PPC::R5, PPC::R6,
                                      PPC::R7, PPC::R8, PPC::R9, PPC::R10};
@@ -6882,6 +6912,14 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
 
   const ArrayRef<MCPhysReg> GPRs = IsPPC64 ? GPR_64 : GPR_32;
 
+  if (ArgFlags.isNest()) {
+    MCRegister EnvReg = State.AllocateReg(IsPPC64 ? PPC::X11 : PPC::R11);
+    if (!EnvReg)
+      report_fatal_error("More then one nest argument.");
+    State.addLoc(CCValAssign::getReg(ValNo, ValVT, EnvReg, RegVT, LocInfo));
+    return false;
+  }
+
   if (ArgFlags.isByVal()) {
     const Align ByValAlign(ArgFlags.getNonZeroByValAlign());
     if (ByValAlign > StackAlign)
@@ -7032,7 +7070,7 @@ static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
     // They are passed in VRs if any are available (unlike arguments passed
     // through ellipses) and shadow GPRs (unlike arguments to non-vaarg
     // functions)
-    if (State.isFixed(ValNo)) {
+    if (!ArgFlags.isVarArg()) {
       if (MCRegister VReg = State.AllocateReg(VR)) {
         State.addLoc(CCValAssign::getReg(ValNo, ValVT, VReg, LocVT, LocInfo));
         // Shadow allocate GPRs and stack space even though we pass in a VR.
@@ -7220,7 +7258,7 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo &MFI = MF.getFrameInfo();
   PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
-  AIXCCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
+  CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
 
   const EVT PtrVT = getPointerTy(MF.getDataLayout());
   // Reserve space for the linkage area on the stack.
@@ -7567,8 +7605,8 @@ SDValue PPCTargetLowering::LowerCall_AIX(
 
   MachineFunction &MF = DAG.getMachineFunction();
   SmallVector<CCValAssign, 16> ArgLocs;
-  AIXCCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs,
-                    *DAG.getContext());
+  CCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs,
+                 *DAG.getContext());
 
   // Reserve space for the linkage save area (LSA) on the stack.
   // In both PPC32 and PPC64 there are 6 reserved slots in the LSA:
@@ -8464,10 +8502,11 @@ void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
 
   // Result is a load from the stack slot.  If loading 4 bytes, make sure to
   // add in a bias on big endian.
-  if (Op.getValueType() == MVT::i32 && !i32Stack) {
+  if (Op.getValueType() == MVT::i32 && !i32Stack &&
+      !Subtarget.isLittleEndian()) {
     FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr,
                         DAG.getConstant(4, dl, FIPtr.getValueType()));
-    MPI = MPI.getWithOffset(Subtarget.isLittleEndian() ? 0 : 4);
+    MPI = MPI.getWithOffset(4);
   }
 
   RLI.Chain = Chain;
@@ -9593,12 +9632,14 @@ static bool isValidSplatLoad(const PPCSubtarget &Subtarget, const SDValue &Op,
   return false;
 }
 
-bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN) {
+bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN,
+                     bool IsLittleEndian) {
   assert(BVN.getNumOperands() > 0 && "Unexpected 0-size build vector");
 
   BitMask.clearAllBits();
   EVT VT = BVN.getValueType(0);
-  APInt ConstValue(VT.getSizeInBits(), 0);
+  unsigned VTSize = VT.getSizeInBits();
+  APInt ConstValue(VTSize, 0);
 
   unsigned EltWidth = VT.getScalarSizeInBits();
 
@@ -9608,8 +9649,10 @@ bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN) {
 
     if (!CN)
       return false;
-
-    ConstValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
+    // The elements in a vector register are ordered in reverse byte order
+    // between little-endian and big-endian modes.
+    ConstValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth),
+                          IsLittleEndian ? BitPos : VTSize - EltWidth - BitPos);
     BitPos += EltWidth;
   }
 
@@ -9640,7 +9683,8 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     // we do not convert it to MTVSRBMI.
     // The xxleqv instruction sets a vector with all ones.
     // The xxlxor instruction sets a vector with all zeros.
-    if (isValidMtVsrBmi(BitMask, *BVN) && BitMask != 0 && BitMask != 0xffff) {
+    if (isValidMtVsrBmi(BitMask, *BVN, Subtarget.isLittleEndian()) &&
+        BitMask != 0 && BitMask != 0xffff) {
       SDValue SDConstant = DAG.getTargetConstant(BitMask, dl, MVT::i32);
       MachineSDNode *MSDNode =
           DAG.getMachineNode(PPC::MTVSRBMI, dl, MVT::v16i8, SDConstant);
@@ -14760,9 +14804,9 @@ static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) {
   SDValue Chain = LD->getChain();
   EVT VT = LD->getMemoryVT();
 
-  SmallSet<SDNode *, 16> LoadRoots;
+  SmallPtrSet<SDNode *, 16> LoadRoots;
   SmallVector<SDNode *, 8> Queue(1, Chain.getNode());
-  SmallSet<SDNode *, 16> Visited;
+  SmallPtrSet<SDNode *, 16> Visited;
 
   // First, search up the chain, branching to follow all token-factor operands.
   // If we find a consecutive load, then we're done, otherwise, record all
@@ -18609,24 +18653,6 @@ bool PPCTargetLowering::useLoadStackGuardNode(const Module &M) const {
   return TargetLowering::useLoadStackGuardNode(M);
 }
 
-// Override to disable global variable loading on Linux and insert AIX canary
-// word declaration.
-void PPCTargetLowering::insertSSPDeclarations(Module &M) const {
-  if (Subtarget.isAIXABI()) {
-    M.getOrInsertGlobal(AIXSSPCanaryWordName,
-                        PointerType::getUnqual(M.getContext()));
-    return;
-  }
-  if (!Subtarget.isTargetLinux())
-    return TargetLowering::insertSSPDeclarations(M);
-}
-
-Value *PPCTargetLowering::getSDagStackGuard(const Module &M) const {
-  if (Subtarget.isAIXABI())
-    return M.getGlobalVariable(AIXSSPCanaryWordName);
-  return TargetLowering::getSDagStackGuard(M);
-}
-
 bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
                                      bool ForCodeSize) const {
   if (!VT.isSimple() || !Subtarget.hasVSX())
@@ -19491,12 +19517,10 @@ SDValue PPCTargetLowering::lowerToLibCall(const char *LibCallName, SDValue Op,
       DAG.getExternalSymbol(LibCallName, TLI.getPointerTy(DAG.getDataLayout()));
   bool SignExtend = TLI.shouldSignExtendTypeInLibCall(RetTy, false);
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
   for (const SDValue &N : Op->op_values()) {
     EVT ArgVT = N.getValueType();
     Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-    Entry.Node = N;
-    Entry.Ty = ArgTy;
+    TargetLowering::ArgListEntry Entry(N, ArgTy);
     Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgTy, SignExtend);
     Entry.IsZExt = !Entry.IsSExt;
     Args.push_back(Entry);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 9755f0e272d16..559d583096927 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -430,20 +430,6 @@ namespace llvm {
     /// optimizations due to constant folding.
     VADD_SPLAT,
 
-    /// CHAIN = SC CHAIN, Imm128 - System call.  The 7-bit unsigned
-    /// operand identifies the operating system entry point.
-    SC,
-
-    /// CHAIN = CLRBHRB CHAIN - Clear branch history rolling buffer.
-    CLRBHRB,
-
-    /// GPRC, CHAIN = MFBHRBE CHAIN, Entry, Dummy - Move from branch
-    /// history rolling buffer entry.
-    MFBHRBE,
-
-    /// CHAIN = RFEBB CHAIN, State - Return from event-based branch.
-    RFEBB,
-
     /// VSRC, CHAIN = XXSWAPD CHAIN, VSRC - Occurs only for little
     /// endian.  Maps to an xxswapd instruction that corrects an lxvd2x
     /// or stxvd2x instruction.  The chain is necessary because the
@@ -1155,8 +1141,6 @@ namespace llvm {
 
     /// Override to support customized stack guard loading.
     bool useLoadStackGuardNode(const Module &M) const override;
-    void insertSSPDeclarations(Module &M) const override;
-    Value *getSDagStackGuard(const Module &M) const override;
 
     bool isFPImmLegal(const APFloat &Imm, EVT VT,
                       bool ForCodeSize) const override;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
index 24287a95ecb05..79fe12e8e4b49 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
@@ -1630,9 +1630,11 @@ def BCDCTSQ_rec : VX_VT5_EO5_VB5_XO9_o    <0, 385, "bcdctsq.", []>;
 
 // Decimal Copy-Sign/Set-Sign
 let Defs = [CR6] in
-def BCDCPSGN_rec : VX1_VT5_VA5_VB5<833, "bcdcpsgn.", []>;
+def BCDCPSGN_rec : VX1_VT5_VA5_VB5<833, "bcdcpsgn.",
+    [(set v16i8:$VD, (int_ppc_bcdcopysign v16i8:$VA, v16i8:$VB))]>;
 
-def BCDSETSGN_rec : VX_VT5_EO5_VB5_PS1_XO9_o<31, 385, "bcdsetsgn.", []>;
+def BCDSETSGN_rec : VX_VT5_EO5_VB5_PS1_XO9_o<31, 385, "bcdsetsgn.",
+    [(set v16i8:$VD, (int_ppc_bcdsetsign v16i8:$VB, i32:$PS))]>;
 
 // Decimal Shift/Unsigned-Shift/Shift-and-Round
 def BCDS_rec :  VX_VT5_VA5_VB5_PS1_XO9_o<193, "bcds." , []>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrFormats.td b/llvm/lib/Target/PowerPC/PPCInstrFormats.td
index 2ced3fe80ea99..b4b475b470a56 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrFormats.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrFormats.td
@@ -13,7 +13,6 @@
 class I<bits<6> opcode, dag OOL, dag IOL, string asmstr, InstrItinClass itin>
         : Instruction {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
   let Size = 4;
 
   bit PPC64 = 0;  // Default value, override with isPPC64
@@ -95,7 +94,6 @@ class I2<bits<6> opcode1, bits<6> opcode2, dag OOL, dag IOL, string asmstr,
          InstrItinClass itin>
         : Instruction {
   field bits<64> Inst;
-  field bits<64> SoftFail = 0;
   let Size = 8;
 
   bit PPC64 = 0;  // Default value, override with isPPC64
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index 99ef89a7fdc0c..c2f91ce8e6b96 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -365,16 +365,6 @@ def PPCeh_sjlj_longjmp : SDNode<"PPCISD::EH_SJLJ_LONGJMP",
                                 SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>,
                                 [SDNPHasChain, SDNPSideEffect]>;
 
-def SDT_PPCsc     : SDTypeProfile<0, 1, [SDTCisInt<0>]>;
-def PPCsc         : SDNode<"PPCISD::SC", SDT_PPCsc,
-                           [SDNPHasChain, SDNPSideEffect]>;
-
-def PPCclrbhrb    : SDNode<"PPCISD::CLRBHRB", SDTNone,
-                           [SDNPHasChain, SDNPSideEffect]>;
-def PPCmfbhrbe    : SDNode<"PPCISD::MFBHRBE", SDTIntBinOp, [SDNPHasChain]>;
-def PPCrfebb      : SDNode<"PPCISD::RFEBB", SDT_PPCsc,
-                           [SDNPHasChain, SDNPSideEffect]>;
-
 def PPCvcmp       : SDNode<"PPCISD::VCMP" , SDT_PPCvcmp, []>;
 def PPCvcmp_rec   : SDNode<"PPCISD::VCMP_rec", SDT_PPCvcmp, [SDNPOutGlue]>;
 
@@ -1673,7 +1663,7 @@ let isBranch = 1, isTerminator = 1, Size = 0 in {
 // System call.
 let PPC970_Unit = 7 in {
   def SC     : SCForm<17, 1, 0, (outs), (ins i32imm:$LEV),
-                      "sc $LEV", IIC_BrB, [(PPCsc (i32 imm:$LEV))]>;
+                      "sc $LEV", IIC_BrB, []>;
 }
 
 // We mark SCV as having no scheduling model since it is only meant to be used
@@ -1685,21 +1675,14 @@ let Predicates = [IsISA3_0], hasNoSchedulingInfo = 1 in {
 }
 
 // Branch history rolling buffer.
-def CLRBHRB : XForm_0<31, 430, (outs), (ins), "clrbhrb", IIC_BrB,
-                      [(PPCclrbhrb)]>,
+def CLRBHRB : XForm_0<31, 430, (outs), (ins), "clrbhrb", IIC_BrB, []>,
                       PPC970_DGroup_Single;
-// The $dmy argument used for MFBHRBE is not needed; however, including
-// it avoids automatic generation of PPCFastISel::fastEmit_i(), which
-// interferes with necessary special handling (see PPCFastISel.cpp).
-def MFBHRBE : XFXForm_3p<31, 302, (outs gprc:$RT),
-                         (ins u10imm:$imm, u10imm:$dmy),
-                         "mfbhrbe $RT, $imm", IIC_BrB,
-                         [(set i32:$RT,
-                               (PPCmfbhrbe imm:$imm, imm:$dmy))]>,
+
+def MFBHRBE : XFXForm_3p<31, 302, (outs gprc:$RT), (ins u10imm:$imm),
+                         "mfbhrbe $RT, $imm", IIC_BrB, []>,
                          PPC970_DGroup_First;
 
-def RFEBB : XLForm_S<19, 146, (outs), (ins u1imm:$S), "rfebb $S",
-                     IIC_BrB, [(PPCrfebb (i32 imm:$S))]>,
+def RFEBB : XLForm_S<19, 146, (outs), (ins u1imm:$S), "rfebb $S", IIC_BrB, []>,
                      PPC970_DGroup_Single;
 
 def : InstAlias<"rfebb", (RFEBB 1)>;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrP10.td b/llvm/lib/Target/PowerPC/PPCInstrP10.td
index 98dd8464c0ac8..c4a027d65b665 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrP10.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrP10.td
@@ -117,7 +117,6 @@ def PPClxvrzx : SDNode<"PPCISD::LXVRZX", SDT_PPCLXVRZX,
 class PI<bits<6> pref, bits<6> opcode, dag OOL, dag IOL, string asmstr,
          InstrItinClass itin> : Instruction {
   field bits<64> Inst;
-  field bits<64> SoftFail = 0;
   bit PCRel = 0; // Default value, set by isPCRel.
   let Size = 8;
 
diff --git a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
index 709d7e7e9b47a..3640d2545b5ac 100644
--- a/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
+++ b/llvm/lib/Target/PowerPC/PPCLoopInstrFormPrep.cpp
@@ -80,7 +80,6 @@
 #include "PPCTargetMachine.h"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -264,9 +263,8 @@ namespace {
     bool prepareBasesForCommoningChains(Bucket &BucketChain);
 
     /// Rewrite load/store according to the common chains.
-    bool
-    rewriteLoadStoresForCommoningChains(Loop *L, Bucket &Bucket,
-                                        SmallSet<BasicBlock *, 16> &BBChanged);
+    bool rewriteLoadStoresForCommoningChains(
+        Loop *L, Bucket &Bucket, SmallPtrSet<BasicBlock *, 16> &BBChanged);
 
     /// Collect condition matched(\p isValidCandidate() returns true)
     /// candidates in Loop \p L.
@@ -309,7 +307,7 @@ namespace {
     /// Rewrite load/store instructions in \p BucketChain according to
     /// preparation.
     bool rewriteLoadStores(Loop *L, Bucket &BucketChain,
-                           SmallSet<BasicBlock *, 16> &BBChanged,
+                           SmallPtrSet<BasicBlock *, 16> &BBChanged,
                            PrepForm Form);
 
     /// Rewrite for the base load/store of a chain.
@@ -523,7 +521,7 @@ bool PPCLoopInstrFormPrep::chainCommoning(Loop *L,
   if (Buckets.empty())
     return MadeChange;
 
-  SmallSet<BasicBlock *, 16> BBChanged;
+  SmallPtrSet<BasicBlock *, 16> BBChanged;
 
   for (auto &Bucket : Buckets) {
     if (prepareBasesForCommoningChains(Bucket))
@@ -537,7 +535,7 @@ bool PPCLoopInstrFormPrep::chainCommoning(Loop *L,
 }
 
 bool PPCLoopInstrFormPrep::rewriteLoadStoresForCommoningChains(
-    Loop *L, Bucket &Bucket, SmallSet<BasicBlock *, 16> &BBChanged) {
+    Loop *L, Bucket &Bucket, SmallPtrSet<BasicBlock *, 16> &BBChanged) {
   bool MadeChange = false;
 
   assert(Bucket.Elements.size() ==
@@ -1006,7 +1004,7 @@ bool PPCLoopInstrFormPrep::prepareBaseForUpdateFormChain(Bucket &BucketChain) {
 }
 
 bool PPCLoopInstrFormPrep::rewriteLoadStores(
-    Loop *L, Bucket &BucketChain, SmallSet<BasicBlock *, 16> &BBChanged,
+    Loop *L, Bucket &BucketChain, SmallPtrSet<BasicBlock *, 16> &BBChanged,
     PrepForm Form) {
   bool MadeChange = false;
 
@@ -1089,7 +1087,7 @@ bool PPCLoopInstrFormPrep::updateFormPrep(Loop *L,
   bool MadeChange = false;
   if (Buckets.empty())
     return MadeChange;
-  SmallSet<BasicBlock *, 16> BBChanged;
+  SmallPtrSet<BasicBlock *, 16> BBChanged;
   for (auto &Bucket : Buckets)
     // The base address of each bucket is transformed into a phi and the others
     // are rewritten based on new base.
@@ -1110,7 +1108,7 @@ bool PPCLoopInstrFormPrep::dispFormPrep(Loop *L,
   if (Buckets.empty())
     return MadeChange;
 
-  SmallSet<BasicBlock *, 16> BBChanged;
+  SmallPtrSet<BasicBlock *, 16> BBChanged;
   for (auto &Bucket : Buckets) {
     if (Bucket.Elements.size() < DispFormPrepMinThreshold)
       continue;
diff --git a/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp b/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
index 0ffd35dfa279c..74bce436b53e7 100644
--- a/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
+++ b/llvm/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
@@ -248,6 +248,10 @@ static bool splitMBB(BlockSplitInfo &BSI) {
   }
   addIncomingValuesToPHIs(NewBRTarget, ThisMBB, NewMBB, MRI);
 
+  // Set the call frame size on ThisMBB to the new basic blocks.
+  // See https://reviews.llvm.org/D156113.
+  NewMBB->setCallFrameSize(TII->getCallFrameSizeAt(ThisMBB->back()));
+
   LLVM_DEBUG(dbgs() << "After splitting, ThisMBB:\n"; ThisMBB->dump());
   LLVM_DEBUG(dbgs() << "NewMBB:\n"; NewMBB->dump());
   LLVM_DEBUG(dbgs() << "New branch-to block:\n"; NewBRTarget->dump());
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
index 76dca4794e050..f1230407b1649 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -1102,13 +1102,20 @@ void PPCRegisterInfo::lowerCRBitSpilling(MachineBasicBlock::iterator II,
     SpillsKnownBit = true;
     break;
   default:
+    // When spilling a CR bit, the super register may not be explicitly defined
+    // (i.e. it can be defined by a CR-logical that only defines the subreg) so
+    // we state that the CR field is undef. Also, in order to preserve the kill
+    // flag on the CR bit, we add it as an implicit use.
+
     // On Power10, we can use SETNBC to spill all CR bits. SETNBC will set all
     // bits (specifically, it produces a -1 if the CR bit is set). Ultimately,
     // the bit that is of importance to us is bit 32 (bit 0 of a 32-bit
     // register), and SETNBC will set this.
     if (Subtarget.isISA3_1()) {
       BuildMI(MBB, II, dl, TII.get(LP64 ? PPC::SETNBC8 : PPC::SETNBC), Reg)
-          .addReg(SrcReg, RegState::Undef);
+          .addReg(SrcReg, RegState::Undef)
+          .addReg(SrcReg, RegState::Implicit |
+                              getKillRegState(MI.getOperand(0).isKill()));
       break;
     }
 
@@ -1122,16 +1129,14 @@ void PPCRegisterInfo::lowerCRBitSpilling(MachineBasicBlock::iterator II,
           SrcReg == PPC::CR4LT || SrcReg == PPC::CR5LT ||
           SrcReg == PPC::CR6LT || SrcReg == PPC::CR7LT) {
         BuildMI(MBB, II, dl, TII.get(LP64 ? PPC::SETB8 : PPC::SETB), Reg)
-          .addReg(getCRFromCRBit(SrcReg), RegState::Undef);
+            .addReg(getCRFromCRBit(SrcReg), RegState::Undef)
+            .addReg(SrcReg, RegState::Implicit |
+                                getKillRegState(MI.getOperand(0).isKill()));
         break;
       }
     }
 
     // We need to move the CR field that contains the CR bit we are spilling.
-    // The super register may not be explicitly defined (i.e. it can be defined
-    // by a CR-logical that only defines the subreg) so we state that the CR
-    // field is undef. Also, in order to preserve the kill flag on the CR bit,
-    // we add it as an implicit use.
     BuildMI(MBB, II, dl, TII.get(LP64 ? PPC::MFOCRF8 : PPC::MFOCRF), Reg)
       .addReg(getCRFromCRBit(SrcReg), RegState::Undef)
       .addReg(SrcReg,
diff --git a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp
index 95de9f39b86e8..4039fedd0cb5c 100644
--- a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp
@@ -22,3 +22,9 @@ bool PPCSelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const {
   return Opcode >= PPCISD::FIRST_STRICTFP_OPCODE &&
          Opcode <= PPCISD::LAST_STRICTFP_OPCODE;
 }
+
+std::pair<SDValue, SDValue> PPCSelectionDAGInfo::EmitTargetCodeForMemcmp(
+    SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2,
+    SDValue Op3, const CallInst *CI) const {
+  return DAG.getMemcmp(Chain, dl, Op1, Op2, Op3, CI);
+}
diff --git a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h
index 08e2ddbf1c4ca..1537851a1b610 100644
--- a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h
@@ -20,6 +20,11 @@ class PPCSelectionDAGInfo : public SelectionDAGTargetInfo {
   bool isTargetMemoryOpcode(unsigned Opcode) const override;
 
   bool isTargetStrictFPOpcode(unsigned Opcode) const override;
+
+  std::pair<SDValue, SDValue>
+  EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain,
+                          SDValue Op1, SDValue Op2, SDValue Op3,
+                          const CallInst *CI) const override;
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index d71c42c0a5fc1..9ce44d0ff7fdc 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -903,6 +903,7 @@ struct RISCVOperand final : public MCParsedAsmOperand {
            VK == RISCV::S_QC_ABS20;
   }
 
+  bool isSImm8Unsigned() const { return isSImm<8>() || isUImm<8>(); }
   bool isSImm10Unsigned() const { return isSImm<10>() || isUImm<10>(); }
 
   bool isUImm20LUI() const {
@@ -1199,6 +1200,14 @@ struct RISCVOperand final : public MCParsedAsmOperand {
     addExpr(Inst, getImm(), isRV64Imm());
   }
 
+  void addSImm8UnsignedOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    int64_t Imm;
+    [[maybe_unused]] bool IsConstant = evaluateConstantImm(getImm(), Imm);
+    assert(IsConstant);
+    Inst.addOperand(MCOperand::createImm(SignExtend64<8>(Imm)));
+  }
+
   void addSImm10UnsignedOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     int64_t Imm;
@@ -1547,6 +1556,9 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, 0, (1 << 9) - 8,
         "immediate must be a multiple of 8 bytes in the range");
+  case Match_InvalidSImm8Unsigned:
+    return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 7),
+                                      (1 << 8) - 1);
   case Match_InvalidSImm10:
     return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 9),
                                       (1 << 9) - 1);
@@ -4053,4 +4065,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
 LLVMInitializeRISCVAsmParser() {
   RegisterMCAsmParser<RISCVAsmParser> X(getTheRISCV32Target());
   RegisterMCAsmParser<RISCVAsmParser> Y(getTheRISCV64Target());
+  RegisterMCAsmParser<RISCVAsmParser> A(getTheRISCV32beTarget());
+  RegisterMCAsmParser<RISCVAsmParser> B(getTheRISCV64beTarget());
 }
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index 67cc01e647a04..dbb16fce8390a 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -14,6 +14,7 @@
 #include "MCTargetDesc/RISCVMCTargetDesc.h"
 #include "TargetInfo/RISCVTargetInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -25,6 +26,7 @@
 #include "llvm/Support/Endian.h"
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "riscv-disassembler"
 
@@ -72,6 +74,10 @@ LLVMInitializeRISCVDisassembler() {
                                          createRISCVDisassembler);
   TargetRegistry::RegisterMCDisassembler(getTheRISCV64Target(),
                                          createRISCVDisassembler);
+  TargetRegistry::RegisterMCDisassembler(getTheRISCV32beTarget(),
+                                         createRISCVDisassembler);
+  TargetRegistry::RegisterMCDisassembler(getTheRISCV64beTarget(),
+                                         createRISCVDisassembler);
 }
 
 static DecodeStatus DecodeGPRRegisterClass(MCInst &Inst, uint32_t RegNo,
@@ -552,16 +558,6 @@ static DecodeStatus decodeXqccmpRlistS0(MCInst &Inst, uint32_t Imm,
   return decodeZcmpRlist(Inst, Imm, Address, Decoder);
 }
 
-static DecodeStatus decodeXTHeadMemPair(MCInst &Inst, uint32_t Insn,
-                                        uint64_t Address,
-                                        const MCDisassembler *Decoder);
-
-static DecodeStatus decodeCSSPushPopchk(MCInst &Inst, uint32_t Insn,
-                                        uint64_t Address,
-                                        const MCDisassembler *Decoder);
-
-#include "RISCVGenDisassemblerTables.inc"
-
 static DecodeStatus decodeCSSPushPopchk(MCInst &Inst, uint32_t Insn,
                                         uint64_t Address,
                                         const MCDisassembler *Decoder) {
@@ -602,6 +598,8 @@ static DecodeStatus decodeXTHeadMemPair(MCInst &Inst, uint32_t Insn,
   return S;
 }
 
+#include "RISCVGenDisassemblerTables.inc"
+
 // Add implied SP operand for C.*SP compressed instructions. The SP operand
 // isn't explicitly encoded in the instruction.
 void RISCVDisassembler::addSPOperands(MCInst &MI) const {
@@ -672,8 +670,13 @@ static constexpr FeatureBitset XAndesGroup = {
     RISCV::FeatureVendorXAndesVSIntLoad, RISCV::FeatureVendorXAndesVPackFPH,
     RISCV::FeatureVendorXAndesVDot};
 
+static constexpr FeatureBitset XSMTGroup = {RISCV::FeatureVendorXSMTVDot};
+
 static constexpr DecoderListEntry DecoderList32[]{
     // Vendor Extensions
+    {DecoderTableXCV32, XCVFeatureGroup, "CORE-V extensions"},
+    {DecoderTableXRivos32, XRivosFeatureGroup, "Rivos"},
+    {DecoderTableXqci32, XqciFeatureGroup, "Qualcomm uC Extensions"},
     {DecoderTableXVentana32,
      {RISCV::FeatureVendorXVentanaCondOps},
      "XVentanaCondOps"},
@@ -689,10 +692,8 @@ static constexpr DecoderListEntry DecoderList32[]{
      {RISCV::FeatureVendorXMIPSCBOP},
      "MIPS mips.pref"},
     {DecoderTableXAndes32, XAndesGroup, "Andes extensions"},
+    {DecoderTableXSMT32, XSMTGroup, "SpacemiT extensions"},
     // Standard Extensions
-    {DecoderTableXCV32, XCVFeatureGroup, "CORE-V extensions"},
-    {DecoderTableXqci32, XqciFeatureGroup, "Qualcomm uC Extensions"},
-    {DecoderTableXRivos32, XRivosFeatureGroup, "Rivos"},
     {DecoderTable32, {}, "standard 32-bit instructions"},
     {DecoderTableRV32Only32, {}, "RV32-only standard 32-bit instructions"},
     {DecoderTableZfinx32, {}, "Zfinx (Float in Integer)"},
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
index d2b75a6c17b11..34026ed5d47a4 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
@@ -45,8 +45,8 @@ struct RISCVOutgoingValueAssigner : public CallLowering::OutgoingValueAssigner {
                  CCValAssign::LocInfo LocInfo,
                  const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
                  CCState &State) override {
-    if (RISCVAssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State, Info.IsFixed,
-                      IsRet, Info.Ty))
+    if (RISCVAssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State, IsRet,
+                      Info.Ty))
       return true;
 
     StackSize = State.getStackSize();
@@ -196,8 +196,8 @@ struct RISCVIncomingValueAssigner : public CallLowering::IncomingValueAssigner {
     if (LocVT.isScalableVector())
       MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
 
-    if (RISCVAssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State,
-                      /*IsFixed=*/true, IsRet, Info.Ty))
+    if (RISCVAssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State, IsRet,
+                      Info.Ty))
       return true;
 
     StackSize = State.getStackSize();
@@ -454,7 +454,7 @@ bool RISCVCallLowering::canLowerReturn(MachineFunction &MF,
   for (unsigned I = 0, E = Outs.size(); I < E; ++I) {
     MVT VT = MVT::getVT(Outs[I].Ty);
     if (CC_RISCV(I, VT, VT, CCValAssign::Full, Outs[I].Flags[0], CCInfo,
-                 /*IsFixed=*/true, /*isRet=*/true, nullptr))
+                 /*isRet=*/true, nullptr))
       return false;
   }
   return true;
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index f83c2b6da8923..51ea3fc5f6774 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -736,7 +736,6 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
   }
   case TargetOpcode::G_FCONSTANT: {
     // TODO: Use constant pool for complex constants.
-    // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
     Register DstReg = MI.getOperand(0).getReg();
     const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
     APInt Imm = FPimm.bitcastToAPInt();
@@ -753,8 +752,22 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
       if (!FMV.constrainAllUses(TII, TRI, RBI))
         return false;
     } else {
+      // s64 on rv32
       assert(Size == 64 && !Subtarget->is64Bit() &&
              "Unexpected size or subtarget");
+
+      if (Imm.isNonNegative() && Imm.isZero()) {
+        // Optimize +0.0 to use fcvt.d.w
+        MachineInstrBuilder FCVT =
+            MIB.buildInstr(RISCV::FCVT_D_W, {DstReg}, {Register(RISCV::X0)})
+                .addImm(RISCVFPRndMode::RNE);
+        if (!FCVT.constrainAllUses(TII, TRI, RBI))
+          return false;
+
+        MI.eraseFromParent();
+        return true;
+      }
+
       // Split into two pieces and build through the stack.
       Register GPRRegHigh = MRI->createVirtualRegister(&RISCV::GPRRegClass);
       Register GPRRegLow = MRI->createVirtualRegister(&RISCV::GPRRegClass);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index 95ec42f960105..41a9c92cf99c3 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -32,10 +32,17 @@ static cl::opt<bool> ULEB128Reloc(
     "riscv-uleb128-reloc", cl::init(true), cl::Hidden,
     cl::desc("Emit R_RISCV_SET_ULEB128/E_RISCV_SUB_ULEB128 if appropriate"));
 
+static cl::opt<bool>
+    AlignRvc("riscv-align-rvc", cl::init(true), cl::Hidden,
+             cl::desc("When generating R_RISCV_ALIGN, insert $alignment-2 "
+                      "bytes of NOPs even in norvc code"));
+
 RISCVAsmBackend::RISCVAsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI,
-                                 bool Is64Bit, const MCTargetOptions &Options)
-    : MCAsmBackend(llvm::endianness::little), STI(STI), OSABI(OSABI),
-      Is64Bit(Is64Bit), TargetOptions(Options) {
+                                 bool Is64Bit, bool IsLittleEndian,
+                                 const MCTargetOptions &Options)
+    : MCAsmBackend(IsLittleEndian ? llvm::endianness::little
+                                  : llvm::endianness::big),
+      STI(STI), OSABI(OSABI), Is64Bit(Is64Bit), TargetOptions(Options) {
   RISCVFeatures::validate(STI.getTargetTriple(), STI.getFeatureBits());
 }
 
@@ -306,12 +313,21 @@ void RISCVAsmBackend::relaxInstruction(MCInst &Inst,
 // If conditions are met, compute the padding size and create a fixup encoding
 // the padding size in the addend.
 bool RISCVAsmBackend::relaxAlign(MCFragment &F, unsigned &Size) {
-  // Use default handling unless linker relaxation is enabled and the alignment
-  // is larger than the nop size.
-  const MCSubtargetInfo *STI = F.getSubtargetInfo();
-  if (!STI->hasFeature(RISCV::FeatureRelax))
+  // Alignments before the first linker-relaxable instruction have fixed sizes
+  // and do not require relocations. Alignments after a linker-relaxable
+  // instruction require a relocation, even if the STI specifies norelax.
+  //
+  // firstLinkerRelaxable is the layout order within the subsection, which may
+  // be smaller than the section's order. Therefore, alignments in a
+  // lower-numbered subsection may be unnecessarily treated as linker-relaxable.
+  auto *Sec = F.getParent();
+  if (F.getLayoutOrder() <= Sec->firstLinkerRelaxable())
     return false;
-  unsigned MinNopLen = STI->hasFeature(RISCV::FeatureStdExtZca) ? 2 : 4;
+
+  // Use default handling unless the alignment is larger than the nop size.
+  const MCSubtargetInfo *STI = F.getSubtargetInfo();
+  unsigned MinNopLen =
+      AlignRvc || STI->hasFeature(RISCV::FeatureStdExtZca) ? 2 : 4;
   if (F.getAlignment() <= MinNopLen)
     return false;
 
@@ -321,16 +337,12 @@ bool RISCVAsmBackend::relaxAlign(MCFragment &F, unsigned &Size) {
       MCFixup::create(0, Expr, FirstLiteralRelocationKind + ELF::R_RISCV_ALIGN);
   F.setVarFixups({Fixup});
   F.setLinkerRelaxable();
-  F.getParent()->setLinkerRelaxable();
   return true;
 }
 
-bool RISCVAsmBackend::relaxDwarfLineAddr(MCFragment &F,
-                                         bool &WasRelaxed) const {
+bool RISCVAsmBackend::relaxDwarfLineAddr(MCFragment &F) const {
   int64_t LineDelta = F.getDwarfLineDelta();
   const MCExpr &AddrDelta = F.getDwarfAddrDelta();
-  size_t OldSize = F.getVarSize();
-
   int64_t Value;
   // If the label difference can be resolved, use the default handling, which
   // utilizes a shorter special opcode.
@@ -361,7 +373,7 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCFragment &F,
   } else {
     PCBytes = 2;
     OS << uint8_t(dwarf::DW_LNS_fixed_advance_pc);
-    support::endian::write<uint16_t>(OS, 0, llvm::endianness::little);
+    support::endian::write<uint16_t>(OS, 0, Endian);
   }
   auto Offset = OS.tell() - PCBytes;
 
@@ -376,15 +388,12 @@ bool RISCVAsmBackend::relaxDwarfLineAddr(MCFragment &F,
   F.setVarContents(Data);
   F.setVarFixups({MCFixup::create(Offset, &AddrDelta,
                                   MCFixup::getDataKindForSize(PCBytes))});
-  WasRelaxed = OldSize != Data.size();
   return true;
 }
 
-bool RISCVAsmBackend::relaxDwarfCFA(MCFragment &F, bool &WasRelaxed) const {
+bool RISCVAsmBackend::relaxDwarfCFA(MCFragment &F) const {
   const MCExpr &AddrDelta = F.getDwarfAddrDelta();
   SmallVector<MCFixup, 2> Fixups;
-  size_t OldSize = F.getVarSize();
-
   int64_t Value;
   if (AddrDelta.evaluateAsAbsolute(Value, *Asm))
     return false;
@@ -397,7 +406,6 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCFragment &F, bool &WasRelaxed) const {
   if (Value == 0) {
     F.clearVarContents();
     F.clearVarFixups();
-    WasRelaxed = OldSize != 0;
     return true;
   }
 
@@ -415,23 +423,21 @@ bool RISCVAsmBackend::relaxDwarfCFA(MCFragment &F, bool &WasRelaxed) const {
     AddFixups(0, {ELF::R_RISCV_SET6, ELF::R_RISCV_SUB6});
   } else if (isUInt<8>(Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc1);
-    support::endian::write<uint8_t>(OS, 0, llvm::endianness::little);
+    support::endian::write<uint8_t>(OS, 0, Endian);
     AddFixups(1, {ELF::R_RISCV_SET8, ELF::R_RISCV_SUB8});
   } else if (isUInt<16>(Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc2);
-    support::endian::write<uint16_t>(OS, 0, llvm::endianness::little);
+    support::endian::write<uint16_t>(OS, 0, Endian);
     AddFixups(1, {ELF::R_RISCV_SET16, ELF::R_RISCV_SUB16});
   } else if (isUInt<32>(Value)) {
     OS << uint8_t(dwarf::DW_CFA_advance_loc4);
-    support::endian::write<uint32_t>(OS, 0, llvm::endianness::little);
+    support::endian::write<uint32_t>(OS, 0, Endian);
     AddFixups(1, {ELF::R_RISCV_SET32, ELF::R_RISCV_SUB32});
   } else {
     llvm_unreachable("unsupported CFA encoding");
   }
   F.setVarContents(Data);
   F.setVarFixups(Fixups);
-
-  WasRelaxed = OldSize != Data.size();
   return true;
 }
 
@@ -471,9 +477,12 @@ bool RISCVAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
     Count -= 1;
   }
 
+  // TODO: emit a mapping symbol right here
+
   if (Count % 4 == 2) {
-    // The canonical nop with Zca is c.nop.
-    OS.write(STI->hasFeature(RISCV::FeatureStdExtZca) ? "\x01\0" : "\0\0", 2);
+    // The canonical nop with Zca is c.nop. For .balign 4, we generate a 2-byte
+    // c.nop even in a norvc region.
+    OS.write("\x01\0", 2);
     Count -= 2;
   }
 
@@ -803,6 +812,23 @@ void RISCVAsmBackend::maybeAddVendorReloc(const MCFragment &F,
   Asm->getWriter().recordRelocation(F, VendorFixup, VendorTarget, VendorValue);
 }
 
+static bool relaxableFixupNeedsRelocation(const MCFixupKind Kind) {
+  // Some Fixups are marked as LinkerRelaxable by
+  // `RISCVMCCodeEmitter::getImmOpValue` only because they may be
+  // (assembly-)relaxed into a linker-relaxable instruction. This function
+  // should return `false` for those fixups so they do not get a `R_RISCV_RELAX`
+  // relocation emitted in addition to the relocation.
+  switch (Kind) {
+  default:
+    break;
+  case RISCV::fixup_riscv_rvc_jump:
+  case RISCV::fixup_riscv_rvc_branch:
+  case RISCV::fixup_riscv_jal:
+    return false;
+  }
+  return true;
+}
+
 bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
                                const MCValue &Target, uint64_t &FixedValue,
                                bool IsResolved) {
@@ -845,30 +871,53 @@ bool RISCVAsmBackend::addReloc(const MCFragment &F, const MCFixup &Fixup,
     return false;
   }
 
-  // If linker relaxation is enabled and supported by the current relocation,
-  // generate a relocation and then append a RELAX.
-  if (Fixup.isLinkerRelaxable())
+  // If linker relaxation is enabled and supported by the current fixup, then we
+  // always want to generate a relocation.
+  bool NeedsRelax = Fixup.isLinkerRelaxable() &&
+                    relaxableFixupNeedsRelocation(Fixup.getKind());
+  if (NeedsRelax)
     IsResolved = false;
+
   if (IsResolved && Fixup.isPCRel())
     IsResolved = isPCRelFixupResolved(Target.getAddSym(), F);
 
   if (!IsResolved) {
-    // Some Fixups require a vendor relocation, record it (directly) before we
+    // Some Fixups require a VENDOR relocation, record it (directly) before we
     // add the relocation.
     maybeAddVendorReloc(F, Fixup);
 
     Asm->getWriter().recordRelocation(F, Fixup, Target, FixedValue);
-  }
 
-  if (Fixup.isLinkerRelaxable()) {
-    auto FA = MCFixup::create(Fixup.getOffset(), nullptr, ELF::R_RISCV_RELAX);
-    Asm->getWriter().recordRelocation(F, FA, MCValue::get(nullptr),
-                                      FixedValueA);
+    if (NeedsRelax) {
+      // Some Fixups get a RELAX relocation, record it (directly) after we add
+      // the relocation.
+      MCFixup RelaxFixup =
+          MCFixup::create(Fixup.getOffset(), nullptr, ELF::R_RISCV_RELAX);
+      MCValue RelaxTarget = MCValue::get(nullptr);
+      uint64_t RelaxValue;
+      Asm->getWriter().recordRelocation(F, RelaxFixup, RelaxTarget, RelaxValue);
+    }
   }
 
   return false;
 }
 
+// Data fixups should be swapped for big endian cores.
+// Instruction fixups should not be swapped as RISC-V instructions
+// are always little-endian.
+static bool isDataFixup(unsigned Kind) {
+  switch (Kind) {
+  default:
+    return false;
+
+  case FK_Data_1:
+  case FK_Data_2:
+  case FK_Data_4:
+  case FK_Data_8:
+    return true;
+  }
+}
+
 void RISCVAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
                                  const MCValue &Target, uint8_t *Data,
                                  uint64_t Value, bool IsResolved) {
@@ -892,8 +941,11 @@ void RISCVAsmBackend::applyFixup(const MCFragment &F, const MCFixup &Fixup,
 
   // For each byte of the fragment that the fixup touches, mask in the
   // bits from the fixup value.
+  // For big endian cores, data fixup should be swapped.
+  bool SwapValue = Endian == llvm::endianness::big && isDataFixup(Kind);
   for (unsigned i = 0; i != NumBytes; ++i) {
-    Data[i] |= uint8_t((Value >> (i * 8)) & 0xff);
+    unsigned Idx = SwapValue ? (NumBytes - 1 - i) : i;
+    Data[Idx] |= uint8_t((Value >> (i * 8)) & 0xff);
   }
 }
 
@@ -908,5 +960,6 @@ MCAsmBackend *llvm::createRISCVAsmBackend(const Target &T,
                                           const MCTargetOptions &Options) {
   const Triple &TT = STI.getTargetTriple();
   uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TT.getOS());
-  return new RISCVAsmBackend(STI, OSABI, TT.isArch64Bit(), Options);
+  return new RISCVAsmBackend(STI, OSABI, TT.isArch64Bit(), TT.isLittleEndian(),
+                             Options);
 }
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
index adec1ec699da0..5152d054cb177 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
@@ -35,7 +35,7 @@ class RISCVAsmBackend : public MCAsmBackend {
 
 public:
   RISCVAsmBackend(const MCSubtargetInfo &STI, uint8_t OSABI, bool Is64Bit,
-                  const MCTargetOptions &Options);
+                  bool IsLittleEndian, const MCTargetOptions &Options);
   ~RISCVAsmBackend() override = default;
 
   std::optional<bool> evaluateFixup(const MCFragment &, MCFixup &, MCValue &,
@@ -65,8 +65,8 @@ class RISCVAsmBackend : public MCAsmBackend {
                         const MCSubtargetInfo &STI) const override;
 
   bool relaxAlign(MCFragment &F, unsigned &Size) override;
-  bool relaxDwarfLineAddr(MCFragment &F, bool &WasRelaxed) const override;
-  bool relaxDwarfCFA(MCFragment &F, bool &WasRelaxed) const override;
+  bool relaxDwarfLineAddr(MCFragment &) const override;
+  bool relaxDwarfCFA(MCFragment &) const override;
   std::pair<bool, bool> relaxLEB128(MCFragment &LF,
                                     int64_t &Value) const override;
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index bddea43fbb09c..fcea23a5275c0 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -139,6 +139,9 @@ enum {
   // 3 -> SEW * 4
   DestEEWShift = ElementsDependOnMaskShift + 1,
   DestEEWMask = 3ULL << DestEEWShift,
+
+  ReadsPastVLShift = DestEEWShift + 2,
+  ReadsPastVLMask = 1ULL << ReadsPastVLShift,
 };
 
 // Helper functions to read TSFlags.
@@ -195,6 +198,12 @@ static inline bool elementsDependOnMask(uint64_t TSFlags) {
   return TSFlags & ElementsDependOnMaskMask;
 }
 
+/// \returns true if the instruction may read elements past VL, e.g.
+/// vslidedown/vrgather
+static inline bool readsPastVL(uint64_t TSFlags) {
+  return TSFlags & ReadsPastVLMask;
+}
+
 static inline unsigned getVLOpNum(const MCInstrDesc &Desc) {
   const uint64_t TSFlags = Desc.TSFlags;
   // This method is only called if we expect to have a VL operand, and all
@@ -337,8 +346,11 @@ enum OperandType : unsigned {
   OPERAND_SIMM5_PLUS1,
   OPERAND_SIMM6,
   OPERAND_SIMM6_NONZERO,
+  OPERAND_SIMM8,
+  OPERAND_SIMM8_UNSIGNED,
   OPERAND_SIMM10,
   OPERAND_SIMM10_LSB0000_NONZERO,
+  OPERAND_SIMM10_UNSIGNED,
   OPERAND_SIMM11,
   OPERAND_SIMM12,
   OPERAND_SIMM12_LSB00000,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
index 8c9ab8effa71b..b0c27ce6010f4 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp
@@ -75,7 +75,7 @@ void RISCVInstPrinter::printInst(const MCInst *MI, uint64_t Address,
   if (PrintAliases && !NoAliases)
     Res = RISCVRVC::uncompress(UncompressedMI, *MI, STI);
   if (Res)
-    NewMI = const_cast<MCInst *>(&UncompressedMI);
+    NewMI = &UncompressedMI;
   if (!PrintAliases || NoAliases || !printAliasInstr(NewMI, Address, STI, O))
     printInstruction(NewMI, Address, STI, O);
   printAnnotation(O, Annot);
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
index 090d331d99cab..77f65d814ce7a 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
@@ -21,6 +21,7 @@ using namespace llvm;
 void RISCVMCAsmInfo::anchor() {}
 
 RISCVMCAsmInfo::RISCVMCAsmInfo(const Triple &TT) {
+  IsLittleEndian = TT.isLittleEndian();
   CodePointerSize = CalleeSaveStackSlotSize = TT.isArch64Bit() ? 8 : 4;
   CommentString = "#";
   AlignmentIsInBytes = false;
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index cbeabdddb9371..717fba68b48ed 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -576,8 +576,21 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
          "getImmOpValue expects only expressions or immediates");
   const MCExpr *Expr = MO.getExpr();
   MCExpr::ExprKind Kind = Expr->getKind();
-  unsigned FixupKind = RISCV::fixup_riscv_invalid;
+
+  // `RelaxCandidate` must be set to `true` in two cases:
+  // - The fixup's relocation gets a R_RISCV_RELAX relocation
+  // - The underlying instruction may be relaxed to an instruction that gets a
+  //   `R_RISCV_RELAX` relocation.
+  //
+  // The actual emission of `R_RISCV_RELAX` will be handled in
+  // `RISCVAsmBackend::applyFixup`.
   bool RelaxCandidate = false;
+  auto AsmRelaxToLinkerRelaxableWithFeature = [&](unsigned Feature) -> void {
+    if (!STI.hasFeature(RISCV::FeatureExactAssembly) && STI.hasFeature(Feature))
+      RelaxCandidate = true;
+  };
+
+  unsigned FixupKind = RISCV::fixup_riscv_invalid;
   if (Kind == MCExpr::Specifier) {
     const auto *RVExpr = cast<MCSpecifierExpr>(Expr);
     FixupKind = RVExpr->getSpecifier();
@@ -644,18 +657,26 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
     // FIXME: Sub kind binary exprs have chance of underflow.
     if (MIFrm == RISCVII::InstFormatJ) {
       FixupKind = RISCV::fixup_riscv_jal;
+      AsmRelaxToLinkerRelaxableWithFeature(RISCV::FeatureVendorXqcilb);
     } else if (MIFrm == RISCVII::InstFormatB) {
       FixupKind = RISCV::fixup_riscv_branch;
+      // This might be assembler relaxed to `b<cc>; jal` but we cannot relax
+      // the `jal` again in the assembler.
     } else if (MIFrm == RISCVII::InstFormatCJ) {
       FixupKind = RISCV::fixup_riscv_rvc_jump;
+      AsmRelaxToLinkerRelaxableWithFeature(RISCV::FeatureVendorXqcilb);
     } else if (MIFrm == RISCVII::InstFormatCB) {
       FixupKind = RISCV::fixup_riscv_rvc_branch;
+      // This might be assembler relaxed to `b<cc>; jal` but we cannot relax
+      // the `jal` again in the assembler.
     } else if (MIFrm == RISCVII::InstFormatCI) {
       FixupKind = RISCV::fixup_riscv_rvc_imm;
     } else if (MIFrm == RISCVII::InstFormatI) {
       FixupKind = RISCV::fixup_riscv_12_i;
     } else if (MIFrm == RISCVII::InstFormatQC_EB) {
       FixupKind = RISCV::fixup_riscv_qc_e_branch;
+      // This might be assembler relaxed to `qc.e.b<cc>; jal` but we cannot
+      // relax the `jal` again in the assembler.
     } else if (MIFrm == RISCVII::InstFormatQC_EAI) {
       FixupKind = RISCV::fixup_riscv_qc_e_32;
       RelaxCandidate = true;
@@ -670,9 +691,9 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
   assert(FixupKind != RISCV::fixup_riscv_invalid && "Unhandled expression!");
 
   addFixup(Fixups, 0, Expr, FixupKind);
-  // If linker relaxation is enabled and supported by this relocation, set
-  // a bit so that if fixup is unresolved, a R_RISCV_RELAX relocation will be
-  // appended.
+  // If linker relaxation is enabled and supported by this relocation, set a bit
+  // so that the assembler knows the size of the instruction is not fixed/known,
+  // and the relocation will need a R_RISCV_RELAX relocation.
   if (EnableRelax && RelaxCandidate)
     Fixups.back().setLinkerRelaxable();
   ++MCNumFixups;
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
index 61ecfb278a7d3..d917ef4129791 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
@@ -376,7 +376,8 @@ static MCInstrAnalysis *createRISCVInstrAnalysis(const MCInstrInfo *Info) {
 
 extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
 LLVMInitializeRISCVTargetMC() {
-  for (Target *T : {&getTheRISCV32Target(), &getTheRISCV64Target()}) {
+  for (Target *T : {&getTheRISCV32Target(), &getTheRISCV64Target(),
+                    &getTheRISCV32beTarget(), &getTheRISCV64beTarget()}) {
     TargetRegistry::RegisterMCAsmInfo(*T, createRISCVMCAsmInfo);
     TargetRegistry::RegisterMCObjectFileInfo(*T, createRISCVMCObjectFileInfo);
     TargetRegistry::RegisterMCInstrInfo(*T, createRISCVMCInstrInfo);
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index 2f32e2ac3eba3..83566b1c57782 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -611,6 +611,8 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
 LLVMInitializeRISCVAsmPrinter() {
   RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target());
   RegisterAsmPrinter<RISCVAsmPrinter> Y(getTheRISCV64Target());
+  RegisterAsmPrinter<RISCVAsmPrinter> A(getTheRISCV32beTarget());
+  RegisterAsmPrinter<RISCVAsmPrinter> B(getTheRISCV64beTarget());
 }
 
 void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.cpp b/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
index cb6117eb0917b..78f47794a5b66 100644
--- a/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCallingConv.cpp
@@ -324,7 +324,7 @@ static MCRegister allocateRVVReg(MVT ValVT, unsigned ValNo, CCState &State,
 // Implements the RISC-V calling convention. Returns true upon failure.
 bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT,
                     CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                    CCState &State, bool IsFixed, bool IsRet, Type *OrigTy) {
+                    CCState &State, bool IsRet, Type *OrigTy) {
   const MachineFunction &MF = State.getMachineFunction();
   const DataLayout &DL = MF.getDataLayout();
   const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
@@ -379,12 +379,12 @@ bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT,
     break;
   case RISCVABI::ABI_ILP32F:
   case RISCVABI::ABI_LP64F:
-    UseGPRForF16_F32 = !IsFixed;
+    UseGPRForF16_F32 = ArgFlags.isVarArg();
     break;
   case RISCVABI::ABI_ILP32D:
   case RISCVABI::ABI_LP64D:
-    UseGPRForF16_F32 = !IsFixed;
-    UseGPRForF64 = !IsFixed;
+    UseGPRForF16_F32 = ArgFlags.isVarArg();
+    UseGPRForF64 = ArgFlags.isVarArg();
     break;
   }
 
@@ -465,7 +465,7 @@ bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT,
   // currently if we are using ILP32E calling convention. This behavior may be
   // changed when RV32E/ILP32E is ratified.
   unsigned TwoXLenInBytes = (2 * XLen) / 8;
-  if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
+  if (ArgFlags.isVarArg() && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
       DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
       ABI != RISCVABI::ABI_ILP32E) {
     unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
@@ -620,8 +620,8 @@ bool llvm::CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT,
 // benchmark. But theoretically, it may have benefit for some cases.
 bool llvm::CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT,
                            CCValAssign::LocInfo LocInfo,
-                           ISD::ArgFlagsTy ArgFlags, CCState &State,
-                           bool IsFixed, bool IsRet, Type *OrigTy) {
+                           ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsRet,
+                           Type *OrigTy) {
   const MachineFunction &MF = State.getMachineFunction();
   const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
   const RISCVTargetLowering &TLI = *Subtarget.getTargetLowering();
@@ -741,7 +741,7 @@ bool llvm::CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT,
 
 bool llvm::CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
                         CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                        CCState &State) {
+                        Type *OrigTy, CCState &State) {
   if (ArgFlags.isNest()) {
     report_fatal_error(
         "Attribute 'nest' is not supported in GHC calling convention");
diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.h b/llvm/lib/Target/RISCV/RISCVCallingConv.h
index bf823b78317d5..0847dd6a12b82 100644
--- a/llvm/lib/Target/RISCV/RISCVCallingConv.h
+++ b/llvm/lib/Target/RISCV/RISCVCallingConv.h
@@ -21,19 +21,19 @@ namespace llvm {
 typedef bool RISCVCCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT,
                              CCValAssign::LocInfo LocInfo,
                              ISD::ArgFlagsTy ArgFlags, CCState &State,
-                             bool IsFixed, bool IsRet, Type *OrigTy);
+                             bool IsRet, Type *OrigTy);
 
 bool CC_RISCV(unsigned ValNo, MVT ValVT, MVT LocVT,
               CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-              CCState &State, bool IsFixed, bool IsRet, Type *OrigTy);
+              CCState &State, bool IsRet, Type *OrigTy);
 
 bool CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT,
                      CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                     CCState &State, bool IsFixed, bool IsRet, Type *OrigTy);
+                     CCState &State, bool IsRet, Type *OrigTy);
 
 bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
                   CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-                  CCState &State);
+                  Type *OrigTy, CCState &State);
 
 namespace RISCV {
 
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 171940e149815..3b738e4cc11a1 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -234,7 +234,7 @@ def FeatureStdExtZtso
 def HasStdExtZtso : Predicate<"Subtarget->hasStdExtZtso()">,
                     AssemblerPredicate<(all_of FeatureStdExtZtso),
                         "'Ztso' (Memory Model - Total Store Order)">;
-def NotHasStdExtZtso : Predicate<"!Subtarget->hasStdExtZtso()">;
+def NoStdExtZtso : Predicate<"!Subtarget->hasStdExtZtso()">;
 
 def FeatureStdExtZa64rs
     : RISCVExtension<1, 0, "Reservation Set Size of at Most 64 Bytes">;
@@ -473,7 +473,7 @@ def FeatureStdExtZba
 def HasStdExtZba : Predicate<"Subtarget->hasStdExtZba()">,
                    AssemblerPredicate<(all_of FeatureStdExtZba),
                                       "'Zba' (Address Generation Instructions)">;
-def NotHasStdExtZba : Predicate<"!Subtarget->hasStdExtZba()">;
+def NoStdExtZba : Predicate<"!Subtarget->hasStdExtZba()">;
 
 def FeatureStdExtZbb
     : RISCVExtension<1, 0, "Basic Bit-Manipulation">,
@@ -1055,13 +1055,13 @@ def FeatureStdExtSupm
                      "Indicates User-mode Pointer Masking">;
 
 def FeatureStdExtSmctr
-    : RISCVExperimentalExtension<1, 0,
-                                 "Control Transfer Records Machine Level",
-                                 [FeatureStdExtSscsrind]>;
+    : RISCVExtension<1, 0,
+                     "Control Transfer Records Machine Level",
+                     [FeatureStdExtSscsrind]>;
 def FeatureStdExtSsctr
-    : RISCVExperimentalExtension<1, 0,
-                                 "Control Transfer Records Supervisor Level",
-                                 [FeatureStdExtSscsrind]>;
+    : RISCVExtension<1, 0,
+                     "Control Transfer Records Supervisor Level",
+                     [FeatureStdExtSscsrind]>;
 def HasStdExtSmctrOrSsctr : Predicate<"Subtarget->hasStdExtSmctrOrSsctr()">,
                             AssemblerPredicate<(any_of FeatureStdExtSmctr, FeatureStdExtSsctr),
                                "'Smctr' (Control Transfer Records Machine Level) or "
@@ -1069,7 +1069,7 @@ def HasStdExtSmctrOrSsctr : Predicate<"Subtarget->hasStdExtSmctrOrSsctr()">,
 
 // Packed SIMD Extensions
 def FeatureStdExtP
-    : RISCVExperimentalExtension<0, 14,
+    : RISCVExperimentalExtension<0, 15,
                                  "'Base P' (Packed SIMD)">;
 def HasStdExtP : Predicate<"Subtarget->hasStdExtP()">,
                  AssemblerPredicate<(all_of FeatureStdExtP),
@@ -1408,7 +1408,7 @@ def HasVendorXMIPSCBOP
     : Predicate<"Subtarget->hasVendorXMIPSCBOP()">,
       AssemblerPredicate<(all_of FeatureVendorXMIPSCBOP),
                          "'Xmipscbop' (MIPS hardware prefetch)">;
-def NotHasVendorXMIPSCBOP : Predicate<"!Subtarget->hasVendorXMIPSCBOP()">;
+def NoVendorXMIPSCBOP : Predicate<"!Subtarget->hasVendorXMIPSCBOP()">;
 
 // WCH / Nanjing Qinheng Microelectronics Extension(s)
 
@@ -1642,6 +1642,14 @@ def HasVendorXAndesVDot
       AssemblerPredicate<(all_of FeatureVendorXAndesVDot),
                          "'XAndesVDot' (Andes Vector Dot Product Extension)">;
 
+def FeatureVendorXSMTVDot
+    : RISCVExtension<1, 0, "SpacemiT Vector Dot Product Extension",
+                     [FeatureStdExtZve32f]>;
+def HasVendorXSMTVDot
+    : Predicate<"Subtarget->hasVendorXSMTVDot()">,
+      AssemblerPredicate<(all_of FeatureVendorXSMTVDot),
+                         "'XSMTVDot' (SpacemiT Vector Dot Product Extension)">;
+
 //===----------------------------------------------------------------------===//
 // LLVM specific features and extensions
 //===----------------------------------------------------------------------===//
@@ -1700,6 +1708,18 @@ def TuneNLogNVRGather
 def TunePostRAScheduler : SubtargetFeature<"use-postra-scheduler",
     "UsePostRAScheduler", "true", "Schedule again after register allocation">;
 
+def TuneDisableMISchedLoadClustering : SubtargetFeature<"disable-misched-load-clustering",
+    "EnableMISchedLoadClustering", "false", "Disable load clustering in the machine scheduler">;
+
+def TuneDisableMISchedStoreClustering : SubtargetFeature<"disable-misched-store-clustering",
+    "EnableMISchedStoreClustering", "false", "Disable store clustering in the machine scheduler">;
+
+def TuneDisablePostMISchedLoadClustering : SubtargetFeature<"disable-postmisched-load-clustering",
+    "EnablePostMISchedLoadClustering", "false", "Disable PostRA load clustering in the machine scheduler">;
+
+def TuneDisablePostMISchedStoreClustering : SubtargetFeature<"disable-postmisched-store-clustering",
+    "EnablePostMISchedStoreClustering", "false", "Disable PostRA store clustering in the machine scheduler">;
+
 def TuneDisableLatencySchedHeuristic
     : SubtargetFeature<"disable-latency-sched-heuristic", "DisableLatencySchedHeuristic", "true",
                        "Disable latency scheduling heuristic">;
diff --git a/llvm/lib/Target/RISCV/RISCVGISel.td b/llvm/lib/Target/RISCV/RISCVGISel.td
index 20ade6c39a8cf..791efca09d40e 100644
--- a/llvm/lib/Target/RISCV/RISCVGISel.td
+++ b/llvm/lib/Target/RISCV/RISCVGISel.td
@@ -147,7 +147,7 @@ def : Pat<(i32 (trunc GPR:$src)), (COPY GPR:$src)>;
 def : Pat<(zext_is_sext (i32 GPR:$src)), (ADDIW GPR:$src, 0)>;
 }
 
-let Predicates = [IsRV64, NotHasStdExtZba] in
+let Predicates = [IsRV64, NoStdExtZba] in
 def : Pat<(zext (i32 GPR:$src)), (SRLI (i64 (SLLI GPR:$src, 32)), 32)>;
 
 let Predicates = [IsRV32, NoStdExtZbb, NoStdExtZbkb] in 
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 5998653129209..f9f35f66319b5 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -18,6 +18,7 @@
 #include "RISCVInstrInfo.h"
 #include "RISCVSelectionDAGInfo.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/SDPatternMatch.h"
 #include "llvm/IR/IntrinsicsRISCV.h"
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/Debug.h"
@@ -681,40 +682,86 @@ bool RISCVDAGToDAGISel::trySignedBitfieldInsertInMask(SDNode *Node) {
   if (!Subtarget->hasVendorXqcibm())
     return false;
 
-  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
-  if (!N1C)
+  using namespace SDPatternMatch;
+
+  SDValue X;
+  APInt MaskImm;
+  if (!sd_match(Node, m_Or(m_OneUse(m_Value(X)), m_ConstInt(MaskImm))))
     return false;
 
-  int32_t C1 = N1C->getSExtValue();
-  if (!isShiftedMask_32(C1) || isInt<12>(C1))
+  unsigned ShAmt, Width;
+  if (!MaskImm.isShiftedMask(ShAmt, Width) || MaskImm.isSignedIntN(12))
     return false;
 
-  // INSBI will clobber the input register in N0. Bail out if we need a copy to
-  // preserve this value.
-  SDValue N0 = Node->getOperand(0);
-  if (!N0.hasOneUse())
+  // If Zbs is enabled and it is a single bit set we can use BSETI which
+  // can be compressed to C_BSETI when Xqcibm in enabled.
+  if (Width == 1 && Subtarget->hasStdExtZbs())
     return false;
 
   // If C1 is a shifted mask (but can't be formed as an ORI),
   // use a bitfield insert of -1.
   // Transform (or x, C1)
   //        -> (qc.insbi x, -1, width, shift)
-  const unsigned Leading = llvm::countl_zero((uint32_t)C1);
-  const unsigned Trailing = llvm::countr_zero((uint32_t)C1);
-  const unsigned Width = 32 - Leading - Trailing;
+  SDLoc DL(Node);
+  MVT VT = Node->getSimpleValueType(0);
 
-  // If Zbs is enabled and it is a single bit set we can use BSETI which
-  // can be compressed to C_BSETI when Xqcibm in enabled.
-  if (Width == 1 && Subtarget->hasStdExtZbs())
+  SDValue Ops[] = {X, CurDAG->getSignedTargetConstant(-1, DL, VT),
+                   CurDAG->getTargetConstant(Width, DL, VT),
+                   CurDAG->getTargetConstant(ShAmt, DL, VT)};
+  SDNode *BitIns = CurDAG->getMachineNode(RISCV::QC_INSBI, DL, VT, Ops);
+  ReplaceNode(Node, BitIns);
+  return true;
+}
+
+// Generate a QC_INSB/QC_INSBI from 'or (and X, MaskImm), OrImm' iff the value
+// being inserted only sets known zero bits.
+bool RISCVDAGToDAGISel::tryBitfieldInsertOpFromOrAndImm(SDNode *Node) {
+  // Supported only in Xqcibm for now.
+  if (!Subtarget->hasVendorXqcibm())
+    return false;
+
+  using namespace SDPatternMatch;
+
+  SDValue And;
+  APInt MaskImm, OrImm;
+  if (!sd_match(Node, m_Or(m_OneUse(m_And(m_Value(And), m_ConstInt(MaskImm))),
+                           m_ConstInt(OrImm))))
+    return false;
+
+  // Compute the Known Zero for the AND as this allows us to catch more general
+  // cases than just looking for AND with imm.
+  KnownBits Known = CurDAG->computeKnownBits(Node->getOperand(0));
+
+  // The bits being inserted must only set those bits that are known to be zero.
+  if (!OrImm.isSubsetOf(Known.Zero)) {
+    // FIXME:  It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
+    // currently handle this case.
+    return false;
+  }
+
+  unsigned ShAmt, Width;
+  // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
+  if (!Known.Zero.isShiftedMask(ShAmt, Width))
     return false;
 
+  // QC_INSB(I) dst, src, #width, #shamt.
   SDLoc DL(Node);
   MVT VT = Node->getSimpleValueType(0);
+  SDValue ImmNode;
+  auto Opc = RISCV::QC_INSB;
 
-  SDValue Ops[] = {N0, CurDAG->getSignedTargetConstant(-1, DL, VT),
-                   CurDAG->getTargetConstant(Width, DL, VT),
-                   CurDAG->getTargetConstant(Trailing, DL, VT)};
-  SDNode *BitIns = CurDAG->getMachineNode(RISCV::QC_INSBI, DL, VT, Ops);
+  int32_t LIImm = OrImm.getSExtValue() >> ShAmt;
+
+  if (isInt<5>(LIImm)) {
+    Opc = RISCV::QC_INSBI;
+    ImmNode = CurDAG->getSignedTargetConstant(LIImm, DL, MVT::i32);
+  } else {
+    ImmNode = selectImm(CurDAG, DL, MVT::i32, LIImm, *Subtarget);
+  }
+
+  SDValue Ops[] = {And, ImmNode, CurDAG->getTargetConstant(Width, DL, VT),
+                   CurDAG->getTargetConstant(ShAmt, DL, VT)};
+  SDNode *BitIns = CurDAG->getMachineNode(Opc, DL, VT, Ops);
   ReplaceNode(Node, BitIns);
   return true;
 }
@@ -772,6 +819,49 @@ bool RISCVDAGToDAGISel::trySignedBitfieldInsertInSign(SDNode *Node) {
   return false;
 }
 
+// (xor X, (and (xor X, C1), C2))
+// -> (qc.insbi X, (C1 >> ShAmt), Width, ShAmt)
+// where C2 is a shifted mask with width=Width and shift=ShAmt
+bool RISCVDAGToDAGISel::tryBitfieldInsertOpFromXor(SDNode *Node) {
+
+  if (!Subtarget->hasVendorXqcibm())
+    return false;
+
+  using namespace SDPatternMatch;
+
+  SDValue X;
+  APInt CImm, CMask;
+  if (!sd_match(
+          Node,
+          m_Xor(m_Value(X),
+                m_OneUse(m_And(m_OneUse(m_Xor(m_Deferred(X), m_ConstInt(CImm))),
+                               m_ConstInt(CMask))))))
+    return false;
+
+  unsigned Width, ShAmt;
+  if (!CMask.isShiftedMask(ShAmt, Width))
+    return false;
+
+  int64_t Imm = CImm.getSExtValue();
+  Imm >>= ShAmt;
+
+  SDLoc DL(Node);
+  SDValue ImmNode;
+  auto Opc = RISCV::QC_INSB;
+
+  if (isInt<5>(Imm)) {
+    Opc = RISCV::QC_INSBI;
+    ImmNode = CurDAG->getSignedTargetConstant(Imm, DL, MVT::i32);
+  } else {
+    ImmNode = selectImm(CurDAG, DL, MVT::i32, Imm, *Subtarget);
+  }
+  SDValue Ops[] = {X, ImmNode, CurDAG->getTargetConstant(Width, DL, MVT::i32),
+                   CurDAG->getTargetConstant(ShAmt, DL, MVT::i32)};
+  ReplaceNode(Node, CurDAG->getMachineNode(Opc, DL, MVT::i32, Ops));
+
+  return true;
+}
+
 bool RISCVDAGToDAGISel::tryUnsignedBitfieldExtract(SDNode *Node,
                                                    const SDLoc &DL, MVT VT,
                                                    SDValue X, unsigned Msb,
@@ -1340,6 +1430,9 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     if (trySignedBitfieldInsertInMask(Node))
       return;
 
+    if (tryBitfieldInsertOpFromOrAndImm(Node))
+      return;
+
     if (tryShrinkShlLogicImm(Node))
       return;
 
@@ -1349,6 +1442,9 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     if (tryShrinkShlLogicImm(Node))
       return;
 
+    if (tryBitfieldInsertOpFromXor(Node))
+      return;
+
     break;
   case ISD::AND: {
     auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
@@ -1644,7 +1740,9 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
     // available.
     // Transform (and x, C1)
     //        -> (<bfextract> x, msb, lsb)
-    if (isMask_64(C1) && !isInt<12>(N1C->getSExtValue())) {
+    if (isMask_64(C1) && !isInt<12>(N1C->getSExtValue()) &&
+        !(C1 == 0xffff && Subtarget->hasStdExtZbb()) &&
+        !(C1 == 0xffffffff && Subtarget->hasStdExtZba())) {
       const unsigned Msb = llvm::bit_width(C1) - 1;
       if (tryUnsignedBitfieldExtract(Node, DL, VT, N0, Msb, 0))
         return;
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
index ee3a86e25add0..c329a4c6ec62e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
@@ -75,6 +75,8 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
   bool trySignedBitfieldExtract(SDNode *Node);
   bool trySignedBitfieldInsertInSign(SDNode *Node);
   bool trySignedBitfieldInsertInMask(SDNode *Node);
+  bool tryBitfieldInsertOpFromXor(SDNode *Node);
+  bool tryBitfieldInsertOpFromOrAndImm(SDNode *Node);
   bool tryUnsignedBitfieldExtract(SDNode *Node, const SDLoc &DL, MVT VT,
                                   SDValue X, unsigned Msb, unsigned Lsb);
   bool tryUnsignedBitfieldInsertInZero(SDNode *Node, const SDLoc &DL, MVT VT,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 0077ecf59dd6d..a33224845e2b7 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -109,7 +109,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
 
   switch (ABI) {
   default:
-    report_fatal_error("Don't know how to lower this ABI");
+    reportFatalUsageError("Don't know how to lower this ABI");
   case RISCVABI::ABI_ILP32:
   case RISCVABI::ABI_ILP32E:
   case RISCVABI::ABI_LP64E:
@@ -1800,15 +1800,20 @@ bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   switch (Intrinsic) {
   default:
     return false;
-  case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
-  case Intrinsic::riscv_masked_atomicrmw_add_i32:
-  case Intrinsic::riscv_masked_atomicrmw_sub_i32:
-  case Intrinsic::riscv_masked_atomicrmw_nand_i32:
-  case Intrinsic::riscv_masked_atomicrmw_max_i32:
-  case Intrinsic::riscv_masked_atomicrmw_min_i32:
-  case Intrinsic::riscv_masked_atomicrmw_umax_i32:
-  case Intrinsic::riscv_masked_atomicrmw_umin_i32:
-  case Intrinsic::riscv_masked_cmpxchg_i32:
+  case Intrinsic::riscv_masked_atomicrmw_xchg:
+  case Intrinsic::riscv_masked_atomicrmw_add:
+  case Intrinsic::riscv_masked_atomicrmw_sub:
+  case Intrinsic::riscv_masked_atomicrmw_nand:
+  case Intrinsic::riscv_masked_atomicrmw_max:
+  case Intrinsic::riscv_masked_atomicrmw_min:
+  case Intrinsic::riscv_masked_atomicrmw_umax:
+  case Intrinsic::riscv_masked_atomicrmw_umin:
+  case Intrinsic::riscv_masked_cmpxchg:
+    // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
+    // narrow atomic operation. These will be expanded to an LR/SC loop that
+    // reads/writes to/from an aligned 4 byte location. And, or, shift, etc.
+    // will be used to modify the appropriate part of the 4 byte data and
+    // preserve the rest.
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     Info.memVT = MVT::i32;
     Info.ptrVal = I.getArgOperand(0);
@@ -1844,6 +1849,17 @@ bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
                                /*IsStore*/ true,
                                /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
+  case Intrinsic::riscv_sseg2_store_mask:
+  case Intrinsic::riscv_sseg3_store_mask:
+  case Intrinsic::riscv_sseg4_store_mask:
+  case Intrinsic::riscv_sseg5_store_mask:
+  case Intrinsic::riscv_sseg6_store_mask:
+  case Intrinsic::riscv_sseg7_store_mask:
+  case Intrinsic::riscv_sseg8_store_mask:
+    // Operands are (vec, ..., vec, ptr, offset, mask, vl)
+    return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
+                               /*IsStore*/ true,
+                               /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
   case Intrinsic::riscv_vlm:
     return SetRVVLoadStoreInfo(/*PtrOp*/ 0,
                                /*IsStore*/ false,
@@ -2512,11 +2528,11 @@ static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
       }
       break;
     case ISD::SETUGT:
-      if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isInt<16>(C + 1) &&
-          C != -1) {
+      if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isUInt<16>(C + 1)) {
         // We have a branch immediate instruction for SETUGE but not SETUGT.
-        // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit signed immediate.
-        RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
+        // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit unsigned
+        // immediate.
+        RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
         CC = ISD::SETUGE;
         return;
       }
@@ -7289,7 +7305,8 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
                                             SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   default:
-    report_fatal_error("unimplemented operand");
+    reportFatalInternalError(
+        "Unimplemented RISCVTargetLowering::LowerOperation Case");
   case ISD::PREFETCH:
     return LowerPREFETCH(Op, Subtarget, DAG);
   case ISD::ATOMIC_FENCE:
@@ -7487,7 +7504,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     // vscale as VLENB / 8.
     static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
     if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
-      report_fatal_error("Support for VLEN==32 is incomplete.");
+      reportFatalInternalError("Support for VLEN==32 is incomplete.");
     // We assume VLENB is a multiple of 8. We manually choose the best shift
     // here because SimplifyDemandedBits isn't always able to simplify it.
     uint64_t Val = Op.getConstantOperandVal(0);
@@ -8500,7 +8517,7 @@ SDValue RISCVTargetLowering::emitFlushICache(SelectionDAG &DAG, SDValue InChain,
 SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
                                                   SelectionDAG &DAG) const {
   if (!Subtarget.is64Bit())
-    llvm::report_fatal_error("Trampolines only implemented for RV64");
+    llvm::reportFatalUsageError("Trampolines only implemented for RV64");
 
   // Create an MCCodeEmitter to encode instructions.
   TargetLoweringObjectFile *TLO = getTargetMachine().getObjFileLowering();
@@ -8660,7 +8677,7 @@ SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
 SDValue RISCVTargetLowering::lowerADJUST_TRAMPOLINE(SDValue Op,
                                                     SelectionDAG &DAG) const {
   if (!Subtarget.is64Bit())
-    llvm::report_fatal_error("Trampolines only implemented for RV64");
+    llvm::reportFatalUsageError("Trampolines only implemented for RV64");
 
   return Op.getOperand(0);
 }
@@ -8795,7 +8812,7 @@ SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
 
   switch (getTargetMachine().getCodeModel()) {
   default:
-    report_fatal_error("Unsupported code model for lowering");
+    reportFatalUsageError("Unsupported code model for lowering");
   case CodeModel::Small: {
     // Generate a sequence for accessing addresses within the first 2 GiB of
     // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
@@ -8931,10 +8948,7 @@ SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
 
   // Prepare argument list to generate call.
   ArgListTy Args;
-  ArgListEntry Entry;
-  Entry.Node = Load;
-  Entry.Ty = CallTy;
-  Args.push_back(Entry);
+  Args.emplace_back(Load, CallTy);
 
   // Setup call to __tls_get_addr.
   TargetLowering::CallLoweringInfo CLI(DAG);
@@ -8976,7 +8990,7 @@ SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
 
   if (DAG.getMachineFunction().getFunction().getCallingConv() ==
       CallingConv::GHC)
-    report_fatal_error("In GHC calling convention TLS is not supported");
+    reportFatalUsageError("In GHC calling convention TLS is not supported");
 
   SDValue Addr;
   switch (Model) {
@@ -10825,7 +10839,7 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) ||
         !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) ||
         !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget))
-      report_fatal_error("EGW should be greater than or equal to 4 * SEW.");
+      reportFatalUsageError("EGW should be greater than or equal to 4 * SEW.");
     return Op;
   }
   // EGS * EEW >= 256 bits
@@ -10833,7 +10847,7 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::riscv_vsm3me: {
     if (!isValidEGW(8, Op.getSimpleValueType(), Subtarget) ||
         !isValidEGW(8, Op->getOperand(1).getSimpleValueType(), Subtarget))
-      report_fatal_error("EGW should be greater than or equal to 8 * SEW.");
+      reportFatalUsageError("EGW should be greater than or equal to 8 * SEW.");
     return Op;
   }
   // zvknha(SEW=32)/zvknhb(SEW=[32|64])
@@ -10842,11 +10856,11 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::riscv_vsha2ms: {
     if (Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
         !Subtarget.hasStdExtZvknhb())
-      report_fatal_error("SEW=64 needs Zvknhb to be enabled.");
+      reportFatalUsageError("SEW=64 needs Zvknhb to be enabled.");
     if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) ||
         !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) ||
         !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget))
-      report_fatal_error("EGW should be greater than or equal to 4 * SEW.");
+      reportFatalUsageError("EGW should be greater than or equal to 4 * SEW.");
     return Op;
   }
   case Intrinsic::riscv_sf_vc_v_x:
@@ -11084,69 +11098,118 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
   return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
 }
 
-SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
-                                                 SelectionDAG &DAG) const {
-  unsigned IntNo = Op.getConstantOperandVal(1);
+static SDValue
+lowerFixedVectorSegStoreIntrinsics(unsigned IntNo, SDValue Op,
+                                   const RISCVSubtarget &Subtarget,
+                                   SelectionDAG &DAG) {
+  bool IsStrided;
   switch (IntNo) {
-  default:
-    break;
   case Intrinsic::riscv_seg2_store_mask:
   case Intrinsic::riscv_seg3_store_mask:
   case Intrinsic::riscv_seg4_store_mask:
   case Intrinsic::riscv_seg5_store_mask:
   case Intrinsic::riscv_seg6_store_mask:
   case Intrinsic::riscv_seg7_store_mask:
-  case Intrinsic::riscv_seg8_store_mask: {
-    SDLoc DL(Op);
-    static const Intrinsic::ID VssegInts[] = {
-        Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
-        Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
-        Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
-        Intrinsic::riscv_vsseg8_mask};
+  case Intrinsic::riscv_seg8_store_mask:
+    IsStrided = false;
+    break;
+  case Intrinsic::riscv_sseg2_store_mask:
+  case Intrinsic::riscv_sseg3_store_mask:
+  case Intrinsic::riscv_sseg4_store_mask:
+  case Intrinsic::riscv_sseg5_store_mask:
+  case Intrinsic::riscv_sseg6_store_mask:
+  case Intrinsic::riscv_sseg7_store_mask:
+  case Intrinsic::riscv_sseg8_store_mask:
+    IsStrided = true;
+    break;
+  default:
+    llvm_unreachable("unexpected intrinsic ID");
+  }
 
-    // Operands: (chain, int_id, vec*, ptr, mask, vl)
-    unsigned NF = Op->getNumOperands() - 5;
-    assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
-    MVT XLenVT = Subtarget.getXLenVT();
-    MVT VT = Op->getOperand(2).getSimpleValueType();
-    MVT ContainerVT = getContainerForFixedLengthVector(VT);
-    unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
-                  ContainerVT.getScalarSizeInBits();
-    EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
+  SDLoc DL(Op);
+  static const Intrinsic::ID VssegInts[] = {
+      Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
+      Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
+      Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
+      Intrinsic::riscv_vsseg8_mask};
+  static const Intrinsic::ID VsssegInts[] = {
+      Intrinsic::riscv_vssseg2_mask, Intrinsic::riscv_vssseg3_mask,
+      Intrinsic::riscv_vssseg4_mask, Intrinsic::riscv_vssseg5_mask,
+      Intrinsic::riscv_vssseg6_mask, Intrinsic::riscv_vssseg7_mask,
+      Intrinsic::riscv_vssseg8_mask};
+
+  // Operands: (chain, int_id, vec*, ptr, mask, vl) or
+  // (chain, int_id, vec*, ptr, stride, mask, vl)
+  unsigned NF = Op->getNumOperands() - (IsStrided ? 6 : 5);
+  assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
+  MVT XLenVT = Subtarget.getXLenVT();
+  MVT VT = Op->getOperand(2).getSimpleValueType();
+  MVT ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget);
+  unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
+                ContainerVT.getScalarSizeInBits();
+  EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
+
+  SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
+  SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
+  MVT MaskVT = Mask.getSimpleValueType();
+  MVT MaskContainerVT =
+      ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
+  Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
 
-    SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
-    SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
-    MVT MaskVT = Mask.getSimpleValueType();
-    MVT MaskContainerVT =
-        ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
-    Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
+  SDValue IntID = DAG.getTargetConstant(
+      IsStrided ? VsssegInts[NF - 2] : VssegInts[NF - 2], DL, XLenVT);
+  SDValue Ptr = Op->getOperand(NF + 2);
 
-    SDValue IntID = DAG.getTargetConstant(VssegInts[NF - 2], DL, XLenVT);
-    SDValue Ptr = Op->getOperand(NF + 2);
+  auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(Op);
 
-    auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(Op);
+  SDValue StoredVal = DAG.getUNDEF(VecTupTy);
+  for (unsigned i = 0; i < NF; i++)
+    StoredVal = DAG.getNode(
+        RISCVISD::TUPLE_INSERT, DL, VecTupTy, StoredVal,
+        convertToScalableVector(ContainerVT, FixedIntrinsic->getOperand(2 + i),
+                                DAG, Subtarget),
+        DAG.getTargetConstant(i, DL, MVT::i32));
 
-    SDValue StoredVal = DAG.getUNDEF(VecTupTy);
-    for (unsigned i = 0; i < NF; i++)
-      StoredVal = DAG.getNode(
-          RISCVISD::TUPLE_INSERT, DL, VecTupTy, StoredVal,
-          convertToScalableVector(
-              ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget),
-          DAG.getTargetConstant(i, DL, MVT::i32));
+  SmallVector<SDValue, 10> Ops = {
+      FixedIntrinsic->getChain(),
+      IntID,
+      StoredVal,
+      Ptr,
+      Mask,
+      VL,
+      DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};
+  // Insert the stride operand.
+  if (IsStrided)
+    Ops.insert(std::next(Ops.begin(), 4),
+               Op.getOperand(Op.getNumOperands() - 3));
+
+  return DAG.getMemIntrinsicNode(
+      ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
+      FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
+}
+
+SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
+                                                 SelectionDAG &DAG) const {
+  unsigned IntNo = Op.getConstantOperandVal(1);
+  switch (IntNo) {
+  default:
+    break;
+  case Intrinsic::riscv_seg2_store_mask:
+  case Intrinsic::riscv_seg3_store_mask:
+  case Intrinsic::riscv_seg4_store_mask:
+  case Intrinsic::riscv_seg5_store_mask:
+  case Intrinsic::riscv_seg6_store_mask:
+  case Intrinsic::riscv_seg7_store_mask:
+  case Intrinsic::riscv_seg8_store_mask:
+  case Intrinsic::riscv_sseg2_store_mask:
+  case Intrinsic::riscv_sseg3_store_mask:
+  case Intrinsic::riscv_sseg4_store_mask:
+  case Intrinsic::riscv_sseg5_store_mask:
+  case Intrinsic::riscv_sseg6_store_mask:
+  case Intrinsic::riscv_sseg7_store_mask:
+  case Intrinsic::riscv_sseg8_store_mask:
+    return lowerFixedVectorSegStoreIntrinsics(IntNo, Op, Subtarget, DAG);
 
-    SDValue Ops[] = {
-        FixedIntrinsic->getChain(),
-        IntID,
-        StoredVal,
-        Ptr,
-        Mask,
-        VL,
-        DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};
-
-    return DAG.getMemIntrinsicNode(
-        ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
-        FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
-  }
   case Intrinsic::riscv_sf_vc_xv_se:
     return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XV_SE);
   case Intrinsic::riscv_sf_vc_iv_se:
@@ -14273,7 +14336,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
       LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
     MakeLibCallOptions CallOptions;
     EVT OpVT = Op0.getValueType();
-    CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
+    CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
     SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
     SDValue Result;
     std::tie(Result, Chain) =
@@ -14308,7 +14371,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
         Op0.getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
     MakeLibCallOptions CallOptions;
     EVT OpVT = Op0.getValueType();
-    CallOptions.setTypeListBeforeSoften(OpVT, MVT::i64, true);
+    CallOptions.setTypeListBeforeSoften(OpVT, MVT::i64);
     SDValue Result = makeLibCall(DAG, LC, MVT::i64, Op0, CallOptions, DL).first;
     Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
     Results.push_back(Result);
@@ -16531,8 +16594,10 @@ combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC,
 // (seteq (i64 (sext_inreg (X, i32)), C1')) where C1' is C1 sign extended from
 // bit 31. Same for setne. C1' may be cheaper to materialize and the sext_inreg
 // can become a sext.w instead of a shift pair.
-static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
+static SDValue performSETCCCombine(SDNode *N,
+                                   TargetLowering::DAGCombinerInfo &DCI,
                                    const RISCVSubtarget &Subtarget) {
+  SelectionDAG &DAG = DCI.DAG;
   SDLoc dl(N);
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
@@ -16548,6 +16613,20 @@ static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
           combineVectorSizedSetCCEquality(VT, N0, N1, Cond, dl, DAG, Subtarget))
     return V;
 
+  // (X & -4096) == 0 -> (X >> 12) == 0 if the AND constant can't use ANDI.
+  if (DCI.isAfterLegalizeDAG() && isNullConstant(N1) &&
+      N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
+      isa<ConstantSDNode>(N0.getOperand(1))) {
+    const APInt &AndRHSC =
+        cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
+    if (!isInt<12>(AndRHSC.getSExtValue()) && AndRHSC.isNegatedPowerOf2()) {
+      unsigned ShiftBits = AndRHSC.countr_zero();
+      SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, N0.getOperand(0),
+                                  DAG.getConstant(ShiftBits, dl, VT));
+      return DAG.getSetCC(dl, VT, Shift, N1, Cond);
+    }
+  }
+
   if (OpVT != MVT::i64 || !Subtarget.is64Bit())
     return SDValue();
 
@@ -16582,27 +16661,39 @@ static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG,
 }
 
 static SDValue
-performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG,
+performSIGN_EXTEND_INREGCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
                                 const RISCVSubtarget &Subtarget) {
+  SelectionDAG &DAG = DCI.DAG;
   SDValue Src = N->getOperand(0);
   EVT VT = N->getValueType(0);
   EVT SrcVT = cast<VTSDNode>(N->getOperand(1))->getVT();
   unsigned Opc = Src.getOpcode();
+  SDLoc DL(N);
 
   // Fold (sext_inreg (fmv_x_anyexth X), i16) -> (fmv_x_signexth X)
   // Don't do this with Zhinx. We need to explicitly sign extend the GPR.
   if (Opc == RISCVISD::FMV_X_ANYEXTH && SrcVT.bitsGE(MVT::i16) &&
       Subtarget.hasStdExtZfhmin())
-    return DAG.getNode(RISCVISD::FMV_X_SIGNEXTH, SDLoc(N), VT,
-                       Src.getOperand(0));
+    return DAG.getNode(RISCVISD::FMV_X_SIGNEXTH, DL, VT, Src.getOperand(0));
 
   // Fold (sext_inreg (shl X, Y), i32) -> (sllw X, Y) iff Y u< 32
   if (Opc == ISD::SHL && Subtarget.is64Bit() && SrcVT == MVT::i32 &&
       VT == MVT::i64 && !isa<ConstantSDNode>(Src.getOperand(1)) &&
       DAG.computeKnownBits(Src.getOperand(1)).countMaxActiveBits() <= 5)
-    return DAG.getNode(RISCVISD::SLLW, SDLoc(N), VT, Src.getOperand(0),
+    return DAG.getNode(RISCVISD::SLLW, DL, VT, Src.getOperand(0),
                        Src.getOperand(1));
 
+  // Fold (sext_inreg (setcc), i1) -> (sub 0, (setcc))
+  if (Opc == ISD::SETCC && SrcVT == MVT::i1 && DCI.isAfterLegalizeDAG())
+    return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
+
+  // Fold (sext_inreg (xor (setcc), -1), i1) -> (add (setcc), -1)
+  if (Opc == ISD::XOR && SrcVT == MVT::i1 &&
+      isAllOnesConstant(Src.getOperand(1)) &&
+      Src.getOperand(0).getOpcode() == ISD::SETCC && DCI.isAfterLegalizeDAG())
+    return DAG.getNode(ISD::ADD, DL, VT, Src.getOperand(0),
+                       DAG.getAllOnesConstant(DL, VT));
+
   return SDValue();
 }
 
@@ -17461,7 +17552,7 @@ static SDValue combineOp_VLToVWOp_VL(SDNode *N,
     return SDValue();
 
   SmallVector<SDNode *> Worklist;
-  SmallSet<SDNode *, 8> Inserted;
+  SmallPtrSet<SDNode *, 8> Inserted;
   Worklist.push_back(N);
   Inserted.insert(N);
   SmallVector<CombineResult> CombinesToApply;
@@ -20022,9 +20113,9 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     return SDValue();
   }
   case ISD::SETCC:
-    return performSETCCCombine(N, DAG, Subtarget);
+    return performSETCCCombine(N, DCI, Subtarget);
   case ISD::SIGN_EXTEND_INREG:
-    return performSIGN_EXTEND_INREGCombine(N, DAG, Subtarget);
+    return performSIGN_EXTEND_INREGCombine(N, DCI, Subtarget);
   case ISD::ZERO_EXTEND:
     // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
     // type legalization. This is safe because fp_to_uint produces poison if
@@ -20580,10 +20671,11 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
     // Combine store of vmv.x.s/vfmv.f.s to vse with VL of 1.
     // vfmv.f.s is represented as extract element from 0. Match it late to avoid
     // any illegal types.
-    if (Val.getOpcode() == RISCVISD::VMV_X_S ||
-        (DCI.isAfterLegalizeDAG() &&
-         Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
-         isNullConstant(Val.getOperand(1)))) {
+    if ((Val.getOpcode() == RISCVISD::VMV_X_S ||
+         (DCI.isAfterLegalizeDAG() &&
+          Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+          isNullConstant(Val.getOperand(1)))) &&
+        Val.hasOneUse()) {
       SDValue Src = Val.getOperand(0);
       MVT VecVT = Src.getSimpleValueType();
       // VecVT should be scalable and memory VT should match the element type.
@@ -20673,12 +20765,22 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
         isNullConstant(Src.getOperand(1)) &&
         Src.getOperand(0).getValueType().isScalableVector()) {
       EVT VT = N->getValueType(0);
-      EVT SrcVT = Src.getOperand(0).getValueType();
-      assert(SrcVT.getVectorElementType() == VT.getVectorElementType());
+      SDValue EVSrc = Src.getOperand(0);
+      EVT EVSrcVT = EVSrc.getValueType();
+      assert(EVSrcVT.getVectorElementType() == VT.getVectorElementType());
       // Widths match, just return the original vector.
-      if (SrcVT == VT)
-        return Src.getOperand(0);
-      // TODO: Use insert_subvector/extract_subvector to change widen/narrow?
+      if (EVSrcVT == VT)
+        return EVSrc;
+      SDLoc DL(N);
+      // Width is narrower, using insert_subvector.
+      if (EVSrcVT.getVectorMinNumElements() < VT.getVectorMinNumElements()) {
+        return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
+                           EVSrc,
+                           DAG.getConstant(0, DL, Subtarget.getXLenVT()));
+      }
+      // Width is wider, using extract_subvector.
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, EVSrc,
+                         DAG.getConstant(0, DL, Subtarget.getXLenVT()));
     }
     [[fallthrough]];
   }
@@ -21382,24 +21484,23 @@ unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
     switch (IntNo) {
     default:
       break;
-    case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
-    case Intrinsic::riscv_masked_atomicrmw_add_i64:
-    case Intrinsic::riscv_masked_atomicrmw_sub_i64:
-    case Intrinsic::riscv_masked_atomicrmw_nand_i64:
-    case Intrinsic::riscv_masked_atomicrmw_max_i64:
-    case Intrinsic::riscv_masked_atomicrmw_min_i64:
-    case Intrinsic::riscv_masked_atomicrmw_umax_i64:
-    case Intrinsic::riscv_masked_atomicrmw_umin_i64:
-    case Intrinsic::riscv_masked_cmpxchg_i64:
+    case Intrinsic::riscv_masked_atomicrmw_xchg:
+    case Intrinsic::riscv_masked_atomicrmw_add:
+    case Intrinsic::riscv_masked_atomicrmw_sub:
+    case Intrinsic::riscv_masked_atomicrmw_nand:
+    case Intrinsic::riscv_masked_atomicrmw_max:
+    case Intrinsic::riscv_masked_atomicrmw_min:
+    case Intrinsic::riscv_masked_atomicrmw_umax:
+    case Intrinsic::riscv_masked_atomicrmw_umin:
+    case Intrinsic::riscv_masked_cmpxchg:
       // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
       // narrow atomic operation. These are implemented using atomic
       // operations at the minimum supported atomicrmw/cmpxchg width whose
       // result is then sign extended to XLEN. With +A, the minimum width is
       // 32 for both 64 and 32.
-      assert(Subtarget.getXLen() == 64);
       assert(getMinCmpXchgSizeInBits() == 32);
       assert(Subtarget.hasStdExtA());
-      return 33;
+      return Op.getValueSizeInBits() - 31;
     }
     break;
   }
@@ -22234,8 +22335,8 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case TargetOpcode::STACKMAP:
   case TargetOpcode::PATCHPOINT:
     if (!Subtarget.is64Bit())
-      report_fatal_error("STACKMAP, PATCHPOINT and STATEPOINT are only "
-                         "supported on 64-bit targets");
+      reportFatalUsageError("STACKMAP, PATCHPOINT and STATEPOINT are only "
+                            "supported on 64-bit targets");
     return emitPatchPoint(MI, BB);
   }
 }
@@ -22270,20 +22371,12 @@ void RISCVTargetLowering::analyzeInputArgs(
     MachineFunction &MF, CCState &CCInfo,
     const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
     RISCVCCAssignFn Fn) const {
-  FunctionType *FType = MF.getFunction().getFunctionType();
-
   for (const auto &[Idx, In] : enumerate(Ins)) {
     MVT ArgVT = In.VT;
     ISD::ArgFlagsTy ArgFlags = In.Flags;
 
-    Type *ArgTy = nullptr;
-    if (IsRet)
-      ArgTy = FType->getReturnType();
-    else if (In.isOrigArg())
-      ArgTy = FType->getParamType(In.getOrigArgIndex());
-
-    if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo,
-           /*IsFixed=*/true, IsRet, ArgTy)) {
+    if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, IsRet,
+           In.OrigTy)) {
       LLVM_DEBUG(dbgs() << "InputArg #" << Idx << " has unhandled type "
                         << ArgVT << '\n');
       llvm_unreachable(nullptr);
@@ -22298,10 +22391,9 @@ void RISCVTargetLowering::analyzeOutputArgs(
   for (const auto &[Idx, Out] : enumerate(Outs)) {
     MVT ArgVT = Out.VT;
     ISD::ArgFlagsTy ArgFlags = Out.Flags;
-    Type *OrigTy = CLI ? CLI->getArgs()[Out.OrigArgIndex].Ty : nullptr;
 
-    if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, Out.IsFixed,
-           IsRet, OrigTy)) {
+    if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, IsRet,
+           Out.OrigTy)) {
       LLVM_DEBUG(dbgs() << "OutputArg #" << Idx << " has unhandled type "
                         << ArgVT << "\n");
       llvm_unreachable(nullptr);
@@ -22477,7 +22569,7 @@ SDValue RISCVTargetLowering::LowerFormalArguments(
 
   switch (CallConv) {
   default:
-    report_fatal_error("Unsupported calling convention");
+    reportFatalUsageError("Unsupported calling convention");
   case CallingConv::C:
   case CallingConv::Fast:
   case CallingConv::SPIR_KERNEL:
@@ -22501,17 +22593,17 @@ SDValue RISCVTargetLowering::LowerFormalArguments(
     break;
   case CallingConv::GHC:
     if (Subtarget.hasStdExtE())
-      report_fatal_error("GHC calling convention is not supported on RVE!");
+      reportFatalUsageError("GHC calling convention is not supported on RVE!");
     if (!Subtarget.hasStdExtFOrZfinx() || !Subtarget.hasStdExtDOrZdinx())
-      report_fatal_error("GHC calling convention requires the (Zfinx/F) and "
-                         "(Zdinx/D) instruction set extensions");
+      reportFatalUsageError("GHC calling convention requires the (Zfinx/F) and "
+                            "(Zdinx/D) instruction set extensions");
   }
 
   const Function &Func = MF.getFunction();
   if (Func.hasFnAttribute("interrupt")) {
     if (!Func.arg_empty())
-      report_fatal_error(
-        "Functions with the interrupt attribute cannot have arguments!");
+      reportFatalUsageError(
+          "Functions with the interrupt attribute cannot have arguments!");
 
     StringRef Kind =
       MF.getFunction().getFnAttribute("interrupt").getValueAsString();
@@ -22527,11 +22619,12 @@ SDValue RISCVTargetLowering::LowerFormalArguments(
         "SiFive-CLIC-preemptible-stack-swap",
     };
     if (!llvm::is_contained(SupportedInterruptKinds, Kind))
-      report_fatal_error(
-        "Function interrupt attribute argument not supported!");
+      reportFatalUsageError(
+          "Function interrupt attribute argument not supported!");
 
     if (Kind.starts_with("qci-") && !Subtarget.hasVendorXqciint())
-      report_fatal_error("'qci-*' interrupt kinds require Xqciint extension");
+      reportFatalUsageError(
+          "'qci-*' interrupt kinds require Xqciint extension");
 
     if (Kind.starts_with("SiFive-CLIC-") && !Subtarget.hasVendorXSfmclic())
       reportFatalUsageError(
@@ -22769,7 +22862,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   if (CallConv == CallingConv::GHC) {
     if (Subtarget.hasStdExtE())
-      report_fatal_error("GHC calling convention is not supported on RVE!");
+      reportFatalUsageError("GHC calling convention is not supported on RVE!");
     ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_GHC);
   } else
     analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI,
@@ -22783,8 +22876,8 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
   if (IsTailCall)
     ++NumTailCalls;
   else if (CLI.CB && CLI.CB->isMustTailCall())
-    report_fatal_error("failed to perform tail call elimination on a call "
-                       "site marked musttail");
+    reportFatalInternalError("failed to perform tail call elimination on a "
+                             "call site marked musttail");
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = ArgCCInfo.getStackSize();
@@ -23083,7 +23176,7 @@ bool RISCVTargetLowering::CanLowerReturn(
     MVT VT = Outs[i].VT;
     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
     if (CC_RISCV(i, VT, VT, CCValAssign::Full, ArgFlags, CCInfo,
-                 /*IsFixed=*/true, /*IsRet=*/true, nullptr))
+                 /*IsRet=*/true, Outs[i].OrigTy))
       return false;
   }
   return true;
@@ -23109,7 +23202,7 @@ RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
                     nullptr, CC_RISCV);
 
   if (CallConv == CallingConv::GHC && !RVLocs.empty())
-    report_fatal_error("GHC functions return void only");
+    reportFatalUsageError("GHC functions return void only");
 
   SDValue Glue;
   SmallVector<SDValue, 4> RetOps(1, Chain);
@@ -23175,7 +23268,7 @@ RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   const Function &Func = DAG.getMachineFunction().getFunction();
   if (Func.hasFnAttribute("interrupt")) {
     if (!Func.getReturnType()->isVoidTy())
-      report_fatal_error(
+      reportFatalUsageError(
           "Functions with the interrupt attribute must have void return type!");
 
     MachineFunction &MF = DAG.getMachineFunction();
@@ -23343,6 +23436,12 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
           &RISCV::VRN2M4RegClass}) {
       if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
         return std::make_pair(0U, RC);
+
+      if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
+        MVT ContainerVT = getContainerForFixedLengthVector(VT);
+        if (TRI->isTypeLegalForClass(*RC, ContainerVT))
+          return std::make_pair(0U, RC);
+      }
     }
   } else if (Constraint == "vd") {
     for (const auto *RC :
@@ -23356,10 +23455,24 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
           &RISCV::VRN2M4NoV0RegClass}) {
       if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
         return std::make_pair(0U, RC);
+
+      if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
+        MVT ContainerVT = getContainerForFixedLengthVector(VT);
+        if (TRI->isTypeLegalForClass(*RC, ContainerVT))
+          return std::make_pair(0U, RC);
+      }
     }
   } else if (Constraint == "vm") {
     if (TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.SimpleTy))
       return std::make_pair(0U, &RISCV::VMV0RegClass);
+
+    if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
+      MVT ContainerVT = getContainerForFixedLengthVector(VT);
+      // VT here might be coerced to vector with i8 elements, so we need to
+      // check if this is a M1 register here instead of checking VMV0RegClass.
+      if (TRI->isTypeLegalForClass(RISCV::VRRegClass, ContainerVT))
+        return std::make_pair(0U, &RISCV::VMV0RegClass);
+    }
   } else if (Constraint == "cr") {
     if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
       return std::make_pair(0U, &RISCV::GPRF16CRegClass);
@@ -23679,53 +23792,26 @@ RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
 
 static Intrinsic::ID
 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
-  if (XLen == 32) {
-    switch (BinOp) {
-    default:
-      llvm_unreachable("Unexpected AtomicRMW BinOp");
-    case AtomicRMWInst::Xchg:
-      return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
-    case AtomicRMWInst::Add:
-      return Intrinsic::riscv_masked_atomicrmw_add_i32;
-    case AtomicRMWInst::Sub:
-      return Intrinsic::riscv_masked_atomicrmw_sub_i32;
-    case AtomicRMWInst::Nand:
-      return Intrinsic::riscv_masked_atomicrmw_nand_i32;
-    case AtomicRMWInst::Max:
-      return Intrinsic::riscv_masked_atomicrmw_max_i32;
-    case AtomicRMWInst::Min:
-      return Intrinsic::riscv_masked_atomicrmw_min_i32;
-    case AtomicRMWInst::UMax:
-      return Intrinsic::riscv_masked_atomicrmw_umax_i32;
-    case AtomicRMWInst::UMin:
-      return Intrinsic::riscv_masked_atomicrmw_umin_i32;
-    }
-  }
-
-  if (XLen == 64) {
-    switch (BinOp) {
-    default:
-      llvm_unreachable("Unexpected AtomicRMW BinOp");
-    case AtomicRMWInst::Xchg:
-      return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
-    case AtomicRMWInst::Add:
-      return Intrinsic::riscv_masked_atomicrmw_add_i64;
-    case AtomicRMWInst::Sub:
-      return Intrinsic::riscv_masked_atomicrmw_sub_i64;
-    case AtomicRMWInst::Nand:
-      return Intrinsic::riscv_masked_atomicrmw_nand_i64;
-    case AtomicRMWInst::Max:
-      return Intrinsic::riscv_masked_atomicrmw_max_i64;
-    case AtomicRMWInst::Min:
-      return Intrinsic::riscv_masked_atomicrmw_min_i64;
-    case AtomicRMWInst::UMax:
-      return Intrinsic::riscv_masked_atomicrmw_umax_i64;
-    case AtomicRMWInst::UMin:
-      return Intrinsic::riscv_masked_atomicrmw_umin_i64;
-    }
+  switch (BinOp) {
+  default:
+    llvm_unreachable("Unexpected AtomicRMW BinOp");
+  case AtomicRMWInst::Xchg:
+    return Intrinsic::riscv_masked_atomicrmw_xchg;
+  case AtomicRMWInst::Add:
+    return Intrinsic::riscv_masked_atomicrmw_add;
+  case AtomicRMWInst::Sub:
+    return Intrinsic::riscv_masked_atomicrmw_sub;
+  case AtomicRMWInst::Nand:
+    return Intrinsic::riscv_masked_atomicrmw_nand;
+  case AtomicRMWInst::Max:
+    return Intrinsic::riscv_masked_atomicrmw_max;
+  case AtomicRMWInst::Min:
+    return Intrinsic::riscv_masked_atomicrmw_min;
+  case AtomicRMWInst::UMax:
+    return Intrinsic::riscv_masked_atomicrmw_umax;
+  case AtomicRMWInst::UMin:
+    return Intrinsic::riscv_masked_atomicrmw_umin;
   }
-
-  llvm_unreachable("Unexpected XLen\n");
 }
 
 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
@@ -23750,7 +23836,7 @@ Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
   unsigned XLen = Subtarget.getXLen();
   Value *Ordering =
       Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
-  Type *Tys[] = {AlignedAddr->getType()};
+  Type *Tys[] = {Builder.getIntNTy(XLen), AlignedAddr->getType()};
   Function *LrwOpScwLoop = Intrinsic::getOrInsertDeclaration(
       AI->getModule(),
       getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
@@ -23806,14 +23892,13 @@ Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
   unsigned XLen = Subtarget.getXLen();
   Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
-  Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
+  Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg;
   if (XLen == 64) {
     CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
     NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
     Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
-    CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
   }
-  Type *Tys[] = {AlignedAddr->getType()};
+  Type *Tys[] = {Builder.getIntNTy(XLen), AlignedAddr->getType()};
   Value *Result = Builder.CreateIntrinsic(
       CmpXchgIntrID, Tys, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
   if (XLen == 64)
@@ -24237,7 +24322,12 @@ bool RISCVTargetLowering::splitValueIntoRegisterParts(
     return true;
   }
 
-  if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
+  if ((ValueVT.isScalableVector() || ValueVT.isFixedLengthVector()) &&
+      PartVT.isScalableVector()) {
+    if (ValueVT.isFixedLengthVector()) {
+      ValueVT = getContainerForFixedLengthVector(ValueVT.getSimpleVT());
+      Val = convertToScalableVector(ValueVT, Val, DAG, Subtarget);
+    }
     LLVMContext &Context = *DAG.getContext();
     EVT ValueEltVT = ValueVT.getVectorElementType();
     EVT PartEltVT = PartVT.getVectorElementType();
@@ -24307,12 +24397,17 @@ SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
     return Val;
   }
 
-  if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
+  if ((ValueVT.isScalableVector() || ValueVT.isFixedLengthVector()) &&
+      PartVT.isScalableVector()) {
     LLVMContext &Context = *DAG.getContext();
     SDValue Val = Parts[0];
     EVT ValueEltVT = ValueVT.getVectorElementType();
     EVT PartEltVT = PartVT.getVectorElementType();
     unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinValue();
+    if (ValueVT.isFixedLengthVector())
+      ValueVTBitSize = getContainerForFixedLengthVector(ValueVT.getSimpleVT())
+                           .getSizeInBits()
+                           .getKnownMinValue();
     unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinValue();
     if (PartVTBitSize % ValueVTBitSize == 0) {
       assert(PartVTBitSize >= ValueVTBitSize);
@@ -24330,7 +24425,10 @@ SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
             EVT::getVectorVT(Context, ValueEltVT, Count, /*IsScalable=*/true);
         Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
       }
-      Val = DAG.getExtractSubvector(DL, ValueVT, Val, 0);
+      if (ValueVT.isFixedLengthVector())
+        Val = convertFromScalableVector(ValueVT, Val, DAG, Subtarget);
+      else
+        Val = DAG.getExtractSubvector(DL, ValueVT, Val, 0);
       return Val;
     }
   }
@@ -24437,8 +24535,8 @@ RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
 
   BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
   if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
-    report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
-                             StringRef(RegName) + "\"."));
+    reportFatalUsageError(Twine("Trying to obtain non-reserved register \"" +
+                                StringRef(RegName) + "\"."));
   return Reg;
 }
 
@@ -24691,7 +24789,7 @@ SDValue RISCVTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
   SP = DAG.getNode(ISD::SUB, dl, XLenVT, SP, Size);
   if (Align)
     SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
-                     DAG.getSignedConstant(-(uint64_t)Align->value(), dl, VT));
+                     DAG.getSignedConstant(-Align->value(), dl, VT));
 
   // Set the real SP to the new value with a probing loop.
   Chain = DAG.getNode(RISCVISD::PROBED_ALLOCA, dl, MVT::Other, Chain, SP);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 433b8be5c562e..fb63ebcfaacea 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -431,8 +431,8 @@ class RISCVTargetLowering : public TargetLowering {
 
   bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
                             ArrayRef<ShuffleVectorInst *> Shuffles,
-                            ArrayRef<unsigned> Indices,
-                            unsigned Factor) const override;
+                            ArrayRef<unsigned> Indices, unsigned Factor,
+                            const APInt &GapMask) const override;
 
   bool lowerInterleavedStore(Instruction *Store, Value *Mask,
                              ShuffleVectorInst *SVI,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index d9c6101478064..c2667b0e7c9e4 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -261,17 +261,18 @@ class RVInstCommon<dag outs, dag ins, string opcodestr, string argstr,
   // Indicates the EEW of a vector instruction's destination operand.
   EEW DestEEW = EEWSEWx1;
   let TSFlags{25-24} = DestEEW.Value;
+
+  // Some vector instructions like vslidedown/vrgather will read elements past
+  // VL, and should be marked to make sure RISCVVLOptimizer doesn't reduce its
+  // operands' VLs.
+  bit ReadsPastVL = 0;
+  let TSFlags{26} = ReadsPastVL;
 }
 
 class RVInst<dag outs, dag ins, string opcodestr, string argstr,
              list<dag> pattern, InstFormat format>
     : RVInstCommon<outs, ins, opcodestr, argstr, pattern, format> {
   field bits<32> Inst;
-  // SoftFail is a field the disassembler can use to provide a way for
-  // instructions to not match without killing the whole decode process. It is
-  // mainly used for ARM, but Tablegen expects this field to exist or it fails
-  // to build the decode table.
-  field bits<32> SoftFail = 0;
   let Size = 4;
 }
 
@@ -279,7 +280,6 @@ class RVInst48<dag outs, dag ins, string opcodestr, string argstr,
                list<dag> pattern, InstFormat format>
     : RVInstCommon<outs, ins, opcodestr, argstr, pattern, format> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
   let Size = 6;
 }
 
@@ -287,7 +287,6 @@ class RVInst64<dag outs, dag ins, string opcodestr, string argstr,
                list<dag> pattern, InstFormat format>
     : RVInstCommon<outs, ins, opcodestr, argstr, pattern, format> {
   field bits<64> Inst;
-  field bits<64> SoftFail = 0;
   let Size = 8;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormatsC.td b/llvm/lib/Target/RISCV/RISCVInstrFormatsC.td
index 5e16061dc470f..209c3fae63f45 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormatsC.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormatsC.td
@@ -14,11 +14,6 @@ class RVInst16<dag outs, dag ins, string opcodestr, string argstr,
                list<dag> pattern, InstFormat format>
     : RVInstCommon<outs, ins, opcodestr, argstr, pattern, format> {
   field bits<16> Inst;
-  // SoftFail is a field the disassembler can use to provide a way for
-  // instructions to not match without killing the whole decode process. It is
-  // mainly used for ARM, but Tablegen expects this field to exist or it fails
-  // to build the decode table.
-  field bits<16> SoftFail = 0;
   let Size = 2;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 085064eee896a..7b4a1de167695 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -382,7 +382,7 @@ void RISCVInstrInfo::copyPhysRegVector(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
     const TargetRegisterClass *RegClass) const {
-  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
+  const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
   RISCVVType::VLMUL LMul = RISCVRI::getLMul(RegClass->TSFlags);
   unsigned NF = RISCVRI::getNF(RegClass->TSFlags);
 
@@ -444,13 +444,7 @@ void RISCVInstrInfo::copyPhysRegVector(
     return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
             RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
   };
-  auto FindRegWithEncoding = [TRI](const TargetRegisterClass &RegClass,
-                                   uint16_t Encoding) {
-    MCRegister Reg = RISCV::V0 + Encoding;
-    if (RISCVRI::getLMul(RegClass.TSFlags) == RISCVVType::LMUL_1)
-      return Reg;
-    return TRI->getMatchingSuperReg(Reg, RISCV::sub_vrm1_0, &RegClass);
-  };
+
   while (I != NumRegs) {
     // For non-segment copying, we only do this once as the registers are always
     // aligned.
@@ -470,9 +464,9 @@ void RISCVInstrInfo::copyPhysRegVector(
 
     // Emit actual copying.
     // For reversed copying, the encoding should be decreased.
-    MCRegister ActualSrcReg = FindRegWithEncoding(
+    MCRegister ActualSrcReg = TRI->findVRegWithEncoding(
         RegClass, ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
-    MCRegister ActualDstReg = FindRegWithEncoding(
+    MCRegister ActualDstReg = TRI->findVRegWithEncoding(
         RegClass, ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
 
     auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), ActualDstReg);
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 8bd383033f11c..23f5a848137c4 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1694,6 +1694,16 @@ multiclass SelectCC_GPR_riirr<DAGOperand valty, DAGOperand imm> {
                 valty:$truev, valty:$falsev), []>;
 }
 
+let Predicates = [IsRV32] in {
+def : Pat<(i32 (setlt (i32 GPR:$rs1), 0)), (SRLI GPR:$rs1, 31)>; // compressible
+}
+let Predicates = [IsRV64] in {
+def : Pat<(i64 (seteq (i64 (and GPR:$rs1, 0x0000000080000000)), 0)),
+          (XORI (i64 (SRLIW GPR:$rs1, 31)), 1)>;
+def : Pat<(i64 (setlt (i64 GPR:$rs1), 0)), (SRLI GPR:$rs1, 63)>; // compressible
+def : Pat<(i64 (setlt (sext_inreg GPR:$rs1, i32), 0)), (SRLIW GPR:$rs1, 31)>;
+}
+
 /// Branches and jumps
 
 // Match `riscv_brcc` and lower to the appropriate RISC-V branch instruction.
@@ -2129,14 +2139,14 @@ def ADJCALLSTACKUP   : Pseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2),
 
 /// RV64 patterns
 
-let Predicates = [IsRV64, NotHasStdExtZba] in {
+let Predicates = [IsRV64, NoStdExtZba] in {
 def : Pat<(i64 (and GPR:$rs1, 0xffffffff)), (SRLI (i64 (SLLI GPR:$rs1, 32)), 32)>;
 
 // If we're shifting a 32-bit zero extended value left by 0-31 bits, use 2
 // shifts instead of 3. This can occur when unsigned is used to index an array.
 def : Pat<(i64 (shl (and GPR:$rs1, 0xffffffff), uimm5:$shamt)),
           (SRLI (i64 (SLLI GPR:$rs1, 32)), (ImmSubFrom32 uimm5:$shamt))>;
-}
+} // Predicates = [IsRV64, NoStdExtZba]
 
 class binop_allhusers<SDPatternOperator operator>
     : PatFrag<(ops node:$lhs, node:$rhs),
@@ -2367,6 +2377,7 @@ include "RISCVInstrInfoXqccmp.td"
 include "RISCVInstrInfoXMips.td"
 include "RISCVInstrInfoXRivos.td"
 include "RISCVInstrInfoXAndes.td"
+include "RISCVInstrInfoXSpacemiT.td"
 
 //===----------------------------------------------------------------------===//
 // Global ISel
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
index 5fa7d4160752f..59f5aebf658d8 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
@@ -190,7 +190,7 @@ let Predicates = [HasAtomicLdSt, IsRV64] in {
 
 multiclass AMOPat<string AtomicOp, string BaseInst, ValueType vt = XLenVT,
                   list<Predicate> ExtraPreds = []> {
-let Predicates = !listconcat([HasStdExtA, NotHasStdExtZtso], ExtraPreds) in {
+let Predicates = !listconcat([HasStdExtA, NoStdExtZtso], ExtraPreds) in {
   def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_monotonic"),
                   !cast<RVInst>(BaseInst), vt>;
   def : PatGprGpr<!cast<PatFrag>(AtomicOp#"_acquire"),
@@ -277,13 +277,30 @@ class PseudoMaskedAMOUMinUMax
   let hasSideEffects = 0;
 }
 
+// Ordering constants must be kept in sync with the AtomicOrdering enum in
+// AtomicOrdering.h.
+multiclass PseudoAMOPat<string AtomicOp, Pseudo AMOInst, ValueType vt = XLenVT> {
+  def : Pat<(vt (!cast<PatFrag>(AtomicOp#"_monotonic") GPR:$addr, GPR:$incr)),
+            (AMOInst GPR:$addr, GPR:$incr, 2)>;
+  def : Pat<(vt (!cast<PatFrag>(AtomicOp#"_acquire") GPR:$addr, GPR:$incr)),
+            (AMOInst GPR:$addr, GPR:$incr, 4)>;
+  def : Pat<(vt (!cast<PatFrag>(AtomicOp#"_release") GPR:$addr, GPR:$incr)),
+            (AMOInst GPR:$addr, GPR:$incr, 5)>;
+  def : Pat<(vt (!cast<PatFrag>(AtomicOp#"_acq_rel") GPR:$addr, GPR:$incr)),
+            (AMOInst GPR:$addr, GPR:$incr, 6)>;
+  def : Pat<(vt (!cast<PatFrag>(AtomicOp#"_seq_cst") GPR:$addr, GPR:$incr)),
+            (AMOInst GPR:$addr, GPR:$incr, 7)>;
+}
+
 class PseudoMaskedAMOPat<Intrinsic intrin, Pseudo AMOInst>
-    : Pat<(intrin GPR:$addr, GPR:$incr, GPR:$mask, timm:$ordering),
+    : Pat<(XLenVT (intrin (XLenVT GPR:$addr), (XLenVT GPR:$incr),
+                          (XLenVT GPR:$mask), (XLenVT timm:$ordering))),
           (AMOInst GPR:$addr, GPR:$incr, GPR:$mask, timm:$ordering)>;
 
 class PseudoMaskedAMOMinMaxPat<Intrinsic intrin, Pseudo AMOInst>
-    : Pat<(intrin GPR:$addr, GPR:$incr, GPR:$mask, GPR:$shiftamt,
-           timm:$ordering),
+    : Pat<(XLenVT (intrin (XLenVT GPR:$addr), (XLenVT GPR:$incr),
+                          (XLenVT GPR:$mask), (XLenVT GPR:$shiftamt),
+           (XLenVT timm:$ordering))),
           (AMOInst GPR:$addr, GPR:$incr, GPR:$mask, GPR:$shiftamt,
            timm:$ordering)>;
 
@@ -291,50 +308,40 @@ let Predicates = [HasStdExtA] in {
 
 let Size = 20 in
 def PseudoAtomicLoadNand32 : PseudoAMO;
-// Ordering constants must be kept in sync with the AtomicOrdering enum in
-// AtomicOrdering.h.
-def : Pat<(XLenVT (atomic_load_nand_i32_monotonic GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 2)>;
-def : Pat<(XLenVT (atomic_load_nand_i32_acquire GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 4)>;
-def : Pat<(XLenVT (atomic_load_nand_i32_release GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 5)>;
-def : Pat<(XLenVT (atomic_load_nand_i32_acq_rel GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 6)>;
-def : Pat<(XLenVT (atomic_load_nand_i32_seq_cst GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 7)>;
-
-let Size = 28 in
-def PseudoMaskedAtomicSwap32 : PseudoMaskedAMO;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_xchg_i32,
+defm : PseudoAMOPat<"atomic_load_nand_i32", PseudoAtomicLoadNand32>;
+
+let Size = 28 in {
+  def PseudoMaskedAtomicSwap32 : PseudoMaskedAMO;
+  def PseudoMaskedAtomicLoadAdd32 : PseudoMaskedAMO;
+  def PseudoMaskedAtomicLoadSub32 : PseudoMaskedAMO;
+}
+let Size = 32 in {
+  def PseudoMaskedAtomicLoadNand32 : PseudoMaskedAMO;
+}
+let Size = 44 in {
+  def PseudoMaskedAtomicLoadMax32 : PseudoMaskedAMOMinMax;
+  def PseudoMaskedAtomicLoadMin32 : PseudoMaskedAMOMinMax;
+}
+let Size = 36 in {
+ def PseudoMaskedAtomicLoadUMax32 : PseudoMaskedAMOUMinUMax;
+ def PseudoMaskedAtomicLoadUMin32 : PseudoMaskedAMOUMinUMax;
+}
+
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_xchg,
                          PseudoMaskedAtomicSwap32>;
-let Size = 28 in
-def PseudoMaskedAtomicLoadAdd32 : PseudoMaskedAMO;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_add_i32,
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_add,
                          PseudoMaskedAtomicLoadAdd32>;
-let Size = 28 in
-def PseudoMaskedAtomicLoadSub32 : PseudoMaskedAMO;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_sub_i32,
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_sub,
                          PseudoMaskedAtomicLoadSub32>;
-let Size = 32 in
-def PseudoMaskedAtomicLoadNand32 : PseudoMaskedAMO;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_nand_i32,
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_nand,
                          PseudoMaskedAtomicLoadNand32>;
-let Size = 44 in
-def PseudoMaskedAtomicLoadMax32 : PseudoMaskedAMOMinMax;
-def : PseudoMaskedAMOMinMaxPat<int_riscv_masked_atomicrmw_max_i32,
+def : PseudoMaskedAMOMinMaxPat<int_riscv_masked_atomicrmw_max,
                                PseudoMaskedAtomicLoadMax32>;
-let Size = 44 in
-def PseudoMaskedAtomicLoadMin32 : PseudoMaskedAMOMinMax;
-def : PseudoMaskedAMOMinMaxPat<int_riscv_masked_atomicrmw_min_i32,
+def : PseudoMaskedAMOMinMaxPat<int_riscv_masked_atomicrmw_min,
                                PseudoMaskedAtomicLoadMin32>;
-let Size = 36 in
-def PseudoMaskedAtomicLoadUMax32 : PseudoMaskedAMOUMinUMax;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_umax_i32,
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_umax,
                          PseudoMaskedAtomicLoadUMax32>;
-let Size = 36 in
-def PseudoMaskedAtomicLoadUMin32 : PseudoMaskedAMOUMinUMax;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_umin_i32,
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_umin,
                          PseudoMaskedAtomicLoadUMin32>;
 } // Predicates = [HasStdExtA]
 
@@ -342,35 +349,7 @@ let Predicates = [HasStdExtA, IsRV64] in {
 
 let Size = 20 in
 def PseudoAtomicLoadNand64 : PseudoAMO;
-// Ordering constants must be kept in sync with the AtomicOrdering enum in
-// AtomicOrdering.h.
-def : Pat<(i64 (atomic_load_nand_i64_monotonic GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 2)>;
-def : Pat<(i64 (atomic_load_nand_i64_acquire GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 4)>;
-def : Pat<(i64 (atomic_load_nand_i64_release GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 5)>;
-def : Pat<(i64 (atomic_load_nand_i64_acq_rel GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 6)>;
-def : Pat<(i64 (atomic_load_nand_i64_seq_cst GPR:$addr, GPR:$incr)),
-          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 7)>;
-
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_xchg_i64,
-                         PseudoMaskedAtomicSwap32>;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_add_i64,
-                         PseudoMaskedAtomicLoadAdd32>;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_sub_i64,
-                         PseudoMaskedAtomicLoadSub32>;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_nand_i64,
-                         PseudoMaskedAtomicLoadNand32>;
-def : PseudoMaskedAMOMinMaxPat<int_riscv_masked_atomicrmw_max_i64,
-                               PseudoMaskedAtomicLoadMax32>;
-def : PseudoMaskedAMOMinMaxPat<int_riscv_masked_atomicrmw_min_i64,
-                               PseudoMaskedAtomicLoadMin32>;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_umax_i64,
-                         PseudoMaskedAtomicLoadUMax32>;
-def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_umin_i64,
-                         PseudoMaskedAtomicLoadUMin32>;
+defm : PseudoAMOPat<"atomic_load_nand_i64", PseudoAtomicLoadNand64, i64>;
 } // Predicates = [HasStdExtA, IsRV64]
 
 
@@ -424,15 +403,9 @@ def PseudoMaskedCmpXchg32
   let Size = 32;
 }
 
-def : Pat<(int_riscv_masked_cmpxchg_i32
-            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering),
+def : Pat<(XLenVT (int_riscv_masked_cmpxchg
+            (XLenVT GPR:$addr), (XLenVT GPR:$cmpval), (XLenVT GPR:$newval),
+            (XLenVT GPR:$mask), (XLenVT timm:$ordering))),
           (PseudoMaskedCmpXchg32
             GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering)>;
 } // Predicates = [HasStdExtA]
-
-let Predicates = [HasStdExtA, IsRV64] in {
-def : Pat<(int_riscv_masked_cmpxchg_i64
-            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering),
-          (PseudoMaskedCmpXchg32
-            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering)>;
-} // Predicates = [HasStdExtA, IsRV64]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
index c1f67f77d7b84..fdf0195a2b669 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoM.td
@@ -111,11 +111,11 @@ def : Pat<(srem (sexti32 (i64 GPR:$rs1)), (sexti32 (i64 GPR:$rs2))),
           (REMW GPR:$rs1, GPR:$rs2)>;
 } // Predicates = [HasStdExtM, IsRV64]
 
-let Predicates = [HasStdExtZmmul, IsRV64, NotHasStdExtZba] in {
+let Predicates = [HasStdExtZmmul, IsRV64, NoStdExtZba] in {
 // Special case for calculating the full 64-bit product of a 32x32 unsigned
 // multiply where the inputs aren't known to be zero extended. We can shift the
 // inputs left by 32 and use a MULHU. This saves two SRLIs needed to finish
 // zeroing the upper 32 bits.
 def : Pat<(i64 (mul (and GPR:$rs1, 0xffffffff), (and GPR:$rs2, 0xffffffff))),
           (MULHU (i64 (SLLI GPR:$rs1, 32)), (i64 (SLLI GPR:$rs2, 32)))>;
-} // Predicates = [HasStdExtZmmul, IsRV64, NotHasStdExtZba]
+} // Predicates = [HasStdExtZmmul, IsRV64, NoStdExtZba]
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
index 8297d5050ceda..c342b41e41d01 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td
@@ -18,7 +18,26 @@
 // Operand and SDNode transformation definitions.
 //===----------------------------------------------------------------------===//
 
-def simm10 : RISCVSImmLeafOp<10>;
+def simm10 : RISCVSImmOp<10>;
+
+def SImm8UnsignedAsmOperand : SImmAsmOperand<8, "Unsigned"> {
+  let RenderMethod = "addSImm8UnsignedOperands";
+}
+
+// A 8-bit signed immediate allowing range [-128, 255]
+// but represented as [-128, 255].
+def simm8_unsigned : RISCVOp {
+  let ParserMatchClass = SImm8UnsignedAsmOperand;
+  let EncoderMethod = "getImmOpValue";
+  let DecoderMethod = "decodeSImmOperand<8>";
+  let OperandType = "OPERAND_SIMM8_UNSIGNED";
+  let MCOperandPredicate = [{
+    int64_t Imm;
+    if (!MCOp.evaluateAsConstantImm(Imm))
+      return false;
+    return isInt<8>(Imm);
+  }];
+}
 
 def SImm10UnsignedAsmOperand : SImmAsmOperand<10, "Unsigned"> {
   let RenderMethod = "addSImm10UnsignedOperands";
@@ -30,7 +49,7 @@ def simm10_unsigned : RISCVOp {
   let ParserMatchClass = SImm10UnsignedAsmOperand;
   let EncoderMethod = "getImmOpValue";
   let DecoderMethod = "decodeSImmOperand<10>";
-  let OperandType = "OPERAND_SIMM10";
+  let OperandType = "OPERAND_SIMM10_UNSIGNED";
   let MCOperandPredicate = [{
     int64_t Imm;
     if (!MCOp.evaluateAsConstantImm(Imm))
@@ -43,49 +62,40 @@ def simm10_unsigned : RISCVOp {
 // Instruction class templates
 //===----------------------------------------------------------------------===//
 
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class PLI_i<bits<7> funct7, string opcodestr>
-    : RVInst<(outs GPR:$rd), (ins simm10:$imm10), opcodestr, "$rd, $imm10", [],
+// Common base for pli.b/h/w and plui.h/w
+class RVPLoadImm_i<bits<7> funct7, dag ins, string opcodestr,
+                   string argstr>
+    : RVInst<(outs GPR:$rd), ins, opcodestr, argstr, [],
              InstFormatOther> {
-  bits<10> imm10;
   bits<5> rd;
 
   let Inst{31-25} = funct7;
-  let Inst{24-16} = imm10{8-0};
-  let Inst{15}    = imm10{9};
   let Inst{14-12} = 0b010;
   let Inst{11-7} = rd;
   let Inst{6-0} = OPC_OP_IMM_32.Value;
+
+  let hasSideEffects = 0;
+  let mayLoad = 0;
+  let mayStore = 0;
 }
 
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class PLUI_i<bits<7> funct7, string opcodestr>
-    : RVInst<(outs GPR:$rd), (ins simm10_unsigned:$imm10), opcodestr,
-             "$rd, $imm10", [], InstFormatOther> {
+// Base for pli.h/w.
+class PLI_i<bits<7> funct7, string opcodestr>
+    : RVPLoadImm_i<funct7, (ins simm10:$imm10), opcodestr, "$rd, $imm10"> {
   bits<10> imm10;
-  bits<5> rd;
 
-  let Inst{31-25} = funct7;
-  let Inst{24}    = imm10{0};
-  let Inst{23-15} = imm10{9-1};
-  let Inst{14-12} = 0b010;
-  let Inst{11-7} = rd;
-  let Inst{6-0} = OPC_OP_IMM_32.Value;
+  let Inst{24-16} = imm10{8-0};
+  let Inst{15}    = imm10{9};
 }
 
-let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
-class PLI_B_i<bits<8> funct8, string opcodestr>
-    : RVInst<(outs GPR:$rd), (ins uimm8:$uimm8), opcodestr, "$rd, $uimm8", [],
-             InstFormatOther> {
-  bits<8> uimm8;
-  bits<5> rd;
+// Base for plui.h/w.
+class PLUI_i<bits<7> funct7, string opcodestr>
+    : RVPLoadImm_i<funct7, (ins simm10_unsigned:$imm10), opcodestr,
+                   "$rd, $imm10"> {
+  bits<10> imm10;
 
-  let Inst{31-24} = funct8;
-  let Inst{23-16} = uimm8;
-  let Inst{15}    = 0b0;
-  let Inst{14-12} = 0b010;
-  let Inst{11-7} = rd;
-  let Inst{6-0} = OPC_OP_IMM_32.Value;
+  let Inst{24}    = imm10{0};
+  let Inst{23-15} = imm10{9-1};
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
@@ -98,6 +108,14 @@ class RVPShift_ri<bits<3> f, bits<3> funct3, string opcodestr, Operand ImmType>
   let Inst{27}    = 0b0;
 }
 
+class RVPShiftD_ri<bits<3> f, bits<3> funct3, string opcodestr>
+    : RVPShift_ri<f, funct3, opcodestr, uimm6> {
+  bits<6> shamt;
+
+  let Inst{26} = 0b1;
+  let Inst{25-20} = shamt;
+}
+
 class RVPShiftW_ri<bits<3> f, bits<3> funct3, string opcodestr>
     : RVPShift_ri<f, funct3, opcodestr, uimm5> {
   bits<5> shamt;
@@ -131,59 +149,743 @@ class RVPUnary_ri<bits<2> w, bits<5> uf, string opcodestr>
   let Inst{24-20} = uf;
 }
 
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPBinaryScalar_rr<bits<3> f, bits<2> w, bits<3> funct3, string opcodestr>
+    : RVInstRBase<funct3, OPC_OP_IMM_32, (outs GPR:$rd),
+                  (ins GPR:$rs1, GPR:$rs2), opcodestr, "$rd, $rs1, $rs2"> {
+  let Inst{31} = 0b1;
+  let Inst{30-28} = f;
+  let Inst{27} = 0b1;
+  let Inst{26-25} = w;
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPBinary_rr<bits<4> f, bits<2> w, bits<3> funct3, string opcodestr>
+    : RVInstRBase<funct3, OPC_OP_32, (outs GPR:$rd),
+                  (ins GPR:$rs1, GPR:$rs2), opcodestr, "$rd, $rs1, $rs2"> {
+  let Inst{31} = 0b1;
+  let Inst{30-27} = f;
+  let Inst{26-25} = w;
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
+class RVPTernary_rrr<bits<4> f, bits<2> w, bits<3> funct3, string opcodestr>
+    : RVInstRBase<funct3, OPC_OP_32, (outs GPR:$rd_wb),
+                  (ins GPR:$rd, GPR:$rs1, GPR:$rs2), opcodestr,
+                  "$rd, $rs1, $rs2"> {
+  let Inst{31} = 0b1;
+  let Inst{30-27} = f;
+  let Inst{26-25} = w;
+
+  let Constraints = "$rd = $rd_wb";
+}
+
+// Common base for pli.db/h/w and plui.dh/w
+class RVPPairLoadImm_i<bits<7> funct7, dag ins, string opcodestr,
+                       string argstr>
+    : RVInst<(outs GPRPairRV32:$rd), ins, opcodestr, argstr, [],
+             InstFormatOther> {
+  bits<5> rd;
+
+  let Inst{31-25} = funct7;
+  let Inst{14-12} = 0b010;
+  let Inst{11-8}  = rd{4-1};
+  let Inst{7}     = 0b0;
+  let Inst{6-0}   = OPC_OP_IMM_32.Value;
+
+  let hasSideEffects = 0;
+  let mayLoad = 0;
+  let mayStore = 0;
+}
+
 //===----------------------------------------------------------------------===//
 // Instructions
 //===----------------------------------------------------------------------===//
 
 let Predicates = [HasStdExtP] in {
-let IsSignExtendingOpW = 1 in
-def CLS    : Unary_r<0b011000000011, 0b001, "cls">;
-def ABS    : Unary_r<0b011000000111, 0b001, "abs">;
+  let IsSignExtendingOpW = 1 in
+  def CLS    : Unary_r<0b011000000011, 0b001, "cls">;
+  def ABS    : Unary_r<0b011000000111, 0b001, "abs">;
 } // Predicates = [HasStdExtP]
-let Predicates = [HasStdExtP, IsRV32] in
-def REV_RV32  : Unary_r<0b011010011111, 0b101, "rev">;
+
+let Predicates = [HasStdExtP, IsRV32] in {
+  def REV_RV32  : Unary_r<0b011010011111, 0b101, "rev">;
+} // Predicates = [HasStdExtP, IsRV32]
 
 let Predicates = [HasStdExtP, IsRV64] in {
-def REV16      : Unary_r<0b011010110000, 0b101, "rev16">;
-def REV_RV64   : Unary_r<0b011010111111, 0b101, "rev">;
+  def REV16      : Unary_r<0b011010110000, 0b101, "rev16">;
+  def REV_RV64   : Unary_r<0b011010111111, 0b101, "rev">;
 
-let IsSignExtendingOpW = 1 in {
-def CLSW  : UnaryW_r<0b011000000011, 0b001, "clsw">;
-def ABSW  : UnaryW_r<0b011000000111, 0b001, "absw">;
-}
+  let IsSignExtendingOpW = 1 in {
+    def CLSW  : UnaryW_r<0b011000000011, 0b001, "clsw">;
+    def ABSW  : UnaryW_r<0b011000000111, 0b001, "absw">;
+  }
 } // Predicates = [HasStdExtP, IsRV64]
 
 let Predicates = [HasStdExtP] in {
-def PSLLI_B  : RVPShiftB_ri<0b000, 0b010, "pslli.b">;
-def PSLLI_H  : RVPShiftH_ri<0b000, 0b010, "pslli.h">;
-def PSSLAI_H : RVPShiftH_ri<0b101, 0b010, "psslai.h">;
+  def PSLLI_B  : RVPShiftB_ri<0b000, 0b010, "pslli.b">;
+  def PSLLI_H  : RVPShiftH_ri<0b000, 0b010, "pslli.h">;
+  def PSSLAI_H : RVPShiftH_ri<0b101, 0b010, "psslai.h">;
 } // Predicates = [HasStdExtP]
-let DecoderNamespace = "RV32Only",
-    Predicates = [HasStdExtP, IsRV32] in
-def SSLAI    : RVPShiftW_ri<0b101, 0b010, "sslai">;
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def SSLAI    : RVPShiftW_ri<0b101, 0b010, "sslai">;
+} // Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only"
 let Predicates = [HasStdExtP, IsRV64] in {
-def PSLLI_W  : RVPShiftW_ri<0b000, 0b010, "pslli.w">;
-def PSSLAI_W : RVPShiftW_ri<0b101, 0b010, "psslai.w">;
+  def PSLLI_W  : RVPShiftW_ri<0b000, 0b010, "pslli.w">;
+  def PSSLAI_W : RVPShiftW_ri<0b101, 0b010, "psslai.w">;
 } // Predicates = [HasStdExtP, IsRV64]
 
 let Predicates = [HasStdExtP] in
 def PLI_H : PLI_i<0b1011000, "pli.h">;
 let Predicates = [HasStdExtP, IsRV64] in
 def PLI_W : PLI_i<0b1011001, "pli.w">;
-let Predicates = [HasStdExtP] in
-def PLI_B : PLI_B_i<0b10110100, "pli.b">;
+let Predicates = [HasStdExtP] in {
+  def PLI_B : RVPLoadImm_i<0b1011010, (ins simm8_unsigned:$imm8), "pli.b",
+                           "$rd, $imm8"> {
+    bits<8> imm8;
+
+    let Inst{24}    = 0b0;
+    let Inst{23-16} = imm8;
+    let Inst{15}    = 0b0;
+  }
+}
 
 let Predicates = [HasStdExtP] in {
-def PSEXT_H_B : RVPUnary_ri<0b00, 0b00100, "psext.h.b">;
-def PSABS_H   : RVPUnary_ri<0b00, 0b00111, "psabs.h">;
-def PSABS_B   : RVPUnary_ri<0b10, 0b00111, "psabs.b">;
+  def PSEXT_H_B : RVPUnary_ri<0b00, 0b00100, "psext.h.b">;
+  def PSABS_H   : RVPUnary_ri<0b00, 0b00111, "psabs.h">;
+  def PSABS_B   : RVPUnary_ri<0b10, 0b00111, "psabs.b">;
 } // Predicates = [HasStdExtP]
 let Predicates = [HasStdExtP, IsRV64] in {
-def PSEXT_W_B : RVPUnary_ri<0b01, 0b00100, "psext.w.b">;
-def PSEXT_W_H : RVPUnary_ri<0b01, 0b00101, "psext.w.h">;
+  def PSEXT_W_B : RVPUnary_ri<0b01, 0b00100, "psext.w.b">;
+  def PSEXT_W_H : RVPUnary_ri<0b01, 0b00101, "psext.w.h">;
 } // Predicates = [HasStdExtP, IsRV64]
 
 let Predicates = [HasStdExtP] in
 def PLUI_H : PLUI_i<0b1111000, "plui.h">;
 let Predicates = [HasStdExtP, IsRV64] in
 def PLUI_W : PLUI_i<0b1111001, "plui.w">;
+
+let Predicates = [HasStdExtP] in {
+  def PSLL_HS   : RVPBinaryScalar_rr<0b000, 0b00, 0b010, "psll.hs">;
+  def PSLL_BS   : RVPBinaryScalar_rr<0b000, 0b10, 0b010, "psll.bs">;
+
+  def PADD_HS   : RVPBinaryScalar_rr<0b001, 0b00, 0b010, "padd.hs">;
+  def PADD_BS   : RVPBinaryScalar_rr<0b001, 0b10, 0b010, "padd.bs">;
+
+  def PSSHA_HS  : RVPBinaryScalar_rr<0b110, 0b00, 0b010, "pssha.hs">;
+
+  def PSSHAR_HS : RVPBinaryScalar_rr<0b111, 0b00, 0b010, "psshar.hs">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def SSHA      : RVPBinaryScalar_rr<0b110, 0b01, 0b010, "ssha">;
+
+  def SSHAR     : RVPBinaryScalar_rr<0b111, 0b01, 0b010, "sshar">;
+} // Predicates = [HasStdExtP, IsRV32]
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PSLL_WS   : RVPBinaryScalar_rr<0b000, 0b01, 0b010, "psll.ws">;
+
+  def PADD_WS   : RVPBinaryScalar_rr<0b001, 0b01, 0b010, "padd.ws">;
+
+  def PSSHA_WS  : RVPBinaryScalar_rr<0b110, 0b01, 0b010, "pssha.ws">;
+  def SHA       : RVPBinaryScalar_rr<0b110, 0b11, 0b010, "sha">;
+
+  def PSSHAR_WS : RVPBinaryScalar_rr<0b111, 0b01, 0b010, "psshar.ws">;
+  def SHAR      : RVPBinaryScalar_rr<0b111, 0b11, 0b010, "shar">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+  def PSRLI_B    : RVPShiftB_ri<0b000, 0b100, "psrli.b">;
+  def PSRLI_H    : RVPShiftH_ri<0b000, 0b100, "psrli.h">;
+
+  def PUSATI_H   : RVPShiftH_ri<0b010, 0b100, "pusati.h">;
+
+  def PSRAI_B    : RVPShiftB_ri<0b100, 0b100, "psrai.b">;
+  def PSRAI_H    : RVPShiftH_ri<0b100, 0b100, "psrai.h">;
+
+  def PSRARI_H   : RVPShiftH_ri<0b101, 0b100, "psrari.h">;
+
+  def PSATI_H    : RVPShiftH_ri<0b110, 0b100, "psati.h">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def USATI_RV32 : RVPShiftW_ri<0b010, 0b100, "usati">;
+
+  def SRARI_RV32 : RVPShiftW_ri<0b101, 0b100, "srari">;
+
+  def SATI_RV32  : RVPShiftW_ri<0b110, 0b100, "sati">;
+} // Predicates = [HasStdExtP, IsRV32]
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PSRLI_W    : RVPShiftW_ri<0b000, 0b100, "psrli.w">;
+  def PSRAI_W    : RVPShiftW_ri<0b100, 0b100, "psrai.w">;
+
+  def PUSATI_W   : RVPShiftW_ri<0b010, 0b100, "pusati.w">;
+  def USATI_RV64 : RVPShiftD_ri<0b010, 0b100, "usati">;
+
+  def PSRARI_W   : RVPShiftW_ri<0b101, 0b100, "psrari.w">;
+  def SRARI_RV64 : RVPShiftD_ri<0b101, 0b100, "srari">;
+
+  def PSATI_W    : RVPShiftW_ri<0b110, 0b100, "psati.w">;
+  def SATI_RV64  : RVPShiftD_ri<0b110, 0b100, "sati">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+  def PSRL_HS     : RVPBinaryScalar_rr<0b000, 0b00, 0b100, "psrl.hs">;
+  def PSRL_BS     : RVPBinaryScalar_rr<0b000, 0b10, 0b100, "psrl.bs">;
+
+  def PREDSUM_HS  : RVPBinaryScalar_rr<0b001, 0b00, 0b100, "predsum.hs">;
+  def PREDSUM_BS  : RVPBinaryScalar_rr<0b001, 0b10, 0b100, "predsum.bs">;
+
+  def PREDSUMU_HS : RVPBinaryScalar_rr<0b011, 0b00, 0b100, "predsumu.hs">;
+  def PREDSUMU_BS : RVPBinaryScalar_rr<0b011, 0b10, 0b100, "predsumu.bs">;
+
+  def PSRA_HS     : RVPBinaryScalar_rr<0b100, 0b00, 0b100, "psra.hs">;
+  def PSRA_BS     : RVPBinaryScalar_rr<0b100, 0b10, 0b100, "psra.bs">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PSRL_WS     : RVPBinaryScalar_rr<0b000, 0b01, 0b100, "psrl.ws">;
+
+  def PREDSUM_WS  : RVPBinaryScalar_rr<0b001, 0b01, 0b100, "predsum.ws">;
+
+  def PREDSUMU_WS : RVPBinaryScalar_rr<0b011, 0b01, 0b100, "predsumu.ws">;
+
+  def PSRA_WS     : RVPBinaryScalar_rr<0b100, 0b01, 0b100, "psra.ws">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+  def PADD_H   : RVPBinary_rr<0b0000, 0b00, 0b000, "padd.h">;
+  def PADD_B   : RVPBinary_rr<0b0000, 0b10, 0b000, "padd.b">;
+
+  def PSADD_H  : RVPBinary_rr<0b0010, 0b00, 0b000, "psadd.h">;
+  def PSADD_B  : RVPBinary_rr<0b0010, 0b10, 0b000, "psadd.b">;
+
+  def PAADD_H  : RVPBinary_rr<0b0011, 0b00, 0b000, "paadd.h">;
+  def PAADD_B  : RVPBinary_rr<0b0011, 0b10, 0b000, "paadd.b">;
+
+  def PSADDU_H : RVPBinary_rr<0b0110, 0b00, 0b000, "psaddu.h">;
+  def PSADDU_B : RVPBinary_rr<0b0110, 0b10, 0b000, "psaddu.b">;
+
+  def PAADDU_H : RVPBinary_rr<0b0111, 0b00, 0b000, "paaddu.h">;
+  def PAADDU_B : RVPBinary_rr<0b0111, 0b10, 0b000, "paaddu.b">;
+
+  def PSUB_H   : RVPBinary_rr<0b1000, 0b00, 0b000, "psub.h">;
+  def PSUB_B   : RVPBinary_rr<0b1000, 0b10, 0b000, "psub.b">;
+
+  def PDIF_H   : RVPBinary_rr<0b1001, 0b00, 0b000, "pdif.h">;
+  def PDIF_B   : RVPBinary_rr<0b1001, 0b10, 0b000, "pdif.b">;
+
+  def PSSUB_H  : RVPBinary_rr<0b1010, 0b00, 0b000, "pssub.h">;
+  def PSSUB_B  : RVPBinary_rr<0b1010, 0b10, 0b000, "pssub.b">;
+
+  def PASUB_H  : RVPBinary_rr<0b1011, 0b00, 0b000, "pasub.h">;
+  def PASUB_B  : RVPBinary_rr<0b1011, 0b10, 0b000, "pasub.b">;
+
+  def PDIFU_H  : RVPBinary_rr<0b1101, 0b00, 0b000, "pdifu.h">;
+  def PDIFU_B  : RVPBinary_rr<0b1101, 0b10, 0b000, "pdifu.b">;
+
+  def PSSUBU_H : RVPBinary_rr<0b1110, 0b00, 0b000, "pssubu.h">;
+  def PSSUBU_B : RVPBinary_rr<0b1110, 0b10, 0b000, "pssubu.b">;
+
+  def PASUBU_H : RVPBinary_rr<0b1111, 0b00, 0b000, "pasubu.h">;
+  def PASUBU_B : RVPBinary_rr<0b1111, 0b10, 0b000, "pasubu.b">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def SADD     : RVPBinary_rr<0b0010, 0b01, 0b000, "sadd">;
+
+  def AADD     : RVPBinary_rr<0b0011, 0b01, 0b000, "aadd">;
+
+  def SADDU    : RVPBinary_rr<0b0110, 0b01, 0b000, "saddu">;
+
+  def AADDU    : RVPBinary_rr<0b0111, 0b01, 0b000, "aaddu">;
+
+  def SSUB     : RVPBinary_rr<0b1010, 0b01, 0b000, "ssub">;
+
+  def ASUB     : RVPBinary_rr<0b1011, 0b01, 0b000, "asub">;
+
+  def SSUBU    : RVPBinary_rr<0b1110, 0b01, 0b000, "ssubu">;
+
+  def ASUBU    : RVPBinary_rr<0b1111, 0b01, 0b000, "asubu">;
+} // Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only"
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PADD_W   : RVPBinary_rr<0b0000, 0b01, 0b000, "padd.w">;
+
+  def PSADD_W  : RVPBinary_rr<0b0010, 0b01, 0b000, "psadd.w">;
+
+  def PAADD_W  : RVPBinary_rr<0b0011, 0b01, 0b000, "paadd.w">;
+
+  def PSADDU_W : RVPBinary_rr<0b0110, 0b01, 0b000, "psaddu.w">;
+
+  def PAADDU_W : RVPBinary_rr<0b0111, 0b01, 0b000, "paaddu.w">;
+
+  def PSUB_W   : RVPBinary_rr<0b1000, 0b01, 0b000, "psub.w">;
+
+  def PSSUB_W  : RVPBinary_rr<0b1010, 0b01, 0b000, "pssub.w">;
+
+  def PASUB_W  : RVPBinary_rr<0b1011, 0b01, 0b000, "pasub.w">;
+
+  def PSSUBU_W : RVPBinary_rr<0b1110, 0b01, 0b000, "pssubu.w">;
+
+  def PASUBU_W : RVPBinary_rr<0b1111, 0b01, 0b000, "pasubu.w">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+  def SLX          : RVPBinary_rr<0b0001, 0b11, 0b001, "slx">;
+
+  def PMUL_H_B01   : RVPBinary_rr<0b0010, 0b00, 0b001, "pmul.h.b01">;
+
+  def MVM          : RVPTernary_rrr<0b0101, 0b00, 0b001, "mvm">;
+  def MVMN         : RVPTernary_rrr<0b0101, 0b01, 0b001, "mvmn">;
+  def MERGE        : RVPTernary_rrr<0b0101, 0b10, 0b001, "merge">;
+  def SRX          : RVPTernary_rrr<0b0101, 0b11, 0b001, "srx">;
+
+  def PMULU_H_B01  : RVPBinary_rr<0b0110, 0b00, 0b001, "pmulu.h.b01">;
+  def PDIFSUMU_B   : RVPBinary_rr<0b0110, 0b10, 0b001, "pdifsumu.b">;
+
+  def PDIFSUMAU_B  : RVPTernary_rrr<0b0111, 0b10, 0b001, "pdifsumau.b">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def MUL_H01      : RVPBinary_rr<0b0010, 0b01, 0b001, "mul.h01">;
+
+  def MACC_H01     : RVPTernary_rrr<0b0011, 0b01, 0b001, "macc.h01">;
+
+  def MULU_H01     : RVPBinary_rr<0b0110, 0b01, 0b001, "mulu.h01">;
+
+  def MACCU_H01    : RVPTernary_rrr<0b0111, 0b01, 0b001, "maccu.h01">;
+} // Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only"
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PMUL_W_H01   : RVPBinary_rr<0b0010, 0b01, 0b001, "pmul.w.h01">;
+  def MUL_W01      : RVPBinary_rr<0b0010, 0b11, 0b001, "mul.w01">;
+
+  def PMACC_W_H01  : RVPTernary_rrr<0b0011, 0b01, 0b001, "pmacc.w.h01">;
+  def MACC_W01     : RVPTernary_rrr<0b0011, 0b11, 0b001, "macc.w01">;
+
+  def PMULU_W_H01  : RVPBinary_rr<0b0110, 0b01, 0b001, "pmulu.w.h01">;
+  def MULU_W01     : RVPBinary_rr<0b0110, 0b11, 0b001, "mulu.w01">;
+
+  def PMACCU_W_H01 : RVPTernary_rrr<0b0111, 0b01, 0b001, "pmaccu.w.h01">;
+  def MACCU_W01    : RVPTernary_rrr<0b0111, 0b11, 0b001, "maccu.w01">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+// Note the spec has a 3-bit f field in bits 30:28 with 0 in bit 27.
+// Here we include the 0 in the f field to reduce number of tablegen classes.
+let Predicates = [HasStdExtP] in {
+  def PSH1ADD_H   : RVPBinary_rr<0b0100, 0b00, 0b010, "psh1add.h">;
+
+  def PSSH1SADD_H : RVPBinary_rr<0b0110, 0b00, 0b010, "pssh1sadd.h">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def SSH1SADD    : RVPBinary_rr<0b0110, 0b01, 0b010, "ssh1sadd">;
+} // Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only"
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PSH1ADD_W   : RVPBinary_rr<0b0100, 0b01, 0b010, "psh1add.w">;
+
+  def PSSH1SADD_W : RVPBinary_rr<0b0110, 0b01, 0b010, "pssh1sadd.w">;
+
+  def UNZIP8P     : RVPBinary_rr<0b1100, 0b00, 0b010, "unzip8p">;
+  def UNZIP16P    : RVPBinary_rr<0b1100, 0b01, 0b010, "unzip16p">;
+  def UNZIP8HP    : RVPBinary_rr<0b1100, 0b10, 0b010, "unzip8hp">;
+  def UNZIP16HP   : RVPBinary_rr<0b1100, 0b11, 0b010, "unzip16hp">;
+
+  def ZIP8P       : RVPBinary_rr<0b1110, 0b00, 0b010, "zip8p">;
+  def ZIP16P      : RVPBinary_rr<0b1110, 0b01, 0b010, "zip16p">;
+  def ZIP8HP      : RVPBinary_rr<0b1110, 0b10, 0b010, "zip8hp">;
+  def ZIP16HP     : RVPBinary_rr<0b1110, 0b11, 0b010, "zip16hp">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+  def PMUL_H_B00    : RVPBinary_rr<0b0000, 0b00, 0b011, "pmul.h.b00">;
+
+  def PMUL_H_B11    : RVPBinary_rr<0b0010, 0b00, 0b011, "pmul.h.b11">;
+
+  def PMULU_H_B00   : RVPBinary_rr<0b0100, 0b00, 0b011, "pmulu.h.b00">;
+
+  def PMULU_H_B11   : RVPBinary_rr<0b0110, 0b00, 0b011, "pmulu.h.b11">;
+
+  def PMULSU_H_B00  : RVPBinary_rr<0b1100, 0b00, 0b011, "pmulsu.h.b00">;
+
+  def PMULSU_H_B11  : RVPBinary_rr<0b1110, 0b00, 0b011, "pmulsu.h.b11">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def MUL_H00       : RVPBinary_rr<0b0000, 0b01, 0b011, "mul.h00">;
+
+  def MACC_H00      : RVPTernary_rrr<0b0001, 0b01, 0b011, "macc.h00">;
+
+  def MUL_H11       : RVPBinary_rr<0b0010, 0b01, 0b011, "mul.h11">;
+
+  def MACC_H11      : RVPTernary_rrr<0b0011, 0b01, 0b011, "macc.h11">;
+
+  def MULU_H00      : RVPBinary_rr<0b0100, 0b01, 0b011, "mulu.h00">;
+
+  def MACCU_H00     : RVPTernary_rrr<0b0101, 0b01, 0b011, "maccu.h00">;
+
+  def MULU_H11      : RVPBinary_rr<0b0110, 0b01, 0b011, "mulu.h11">;
+
+  def MACCU_H11     : RVPTernary_rrr<0b0111, 0b01, 0b011, "maccu.h11">;
+
+  def MULSU_H00     : RVPBinary_rr<0b1100, 0b01, 0b011, "mulsu.h00">;
+
+  def MACCSU_H00    : RVPTernary_rrr<0b1101, 0b01, 0b011, "maccsu.h00">;
+
+  def MULSU_H11     : RVPBinary_rr<0b1110, 0b01, 0b011, "mulsu.h11">;
+
+  def MACCSU_H11    : RVPTernary_rrr<0b1111, 0b01, 0b011, "maccsu.h11">;
+} // Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only"
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PMUL_W_H00    : RVPBinary_rr<0b0000, 0b01, 0b011, "pmul.w.h00">;
+  def MUL_W00       : RVPBinary_rr<0b0000, 0b11, 0b011, "mul.w00">;
+
+  def PMACC_W_H00   : RVPTernary_rrr<0b0001, 0b01, 0b011, "pmacc.w.h00">;
+  def MACC_W00      : RVPTernary_rrr<0b0001, 0b11, 0b011, "macc.w00">;
+
+  def PMUL_W_H11    : RVPBinary_rr<0b0010, 0b01, 0b011, "pmul.w.h11">;
+  def MUL_W11       : RVPBinary_rr<0b0010, 0b11, 0b011, "mul.w11">;
+
+  def PMACC_W_H11   : RVPTernary_rrr<0b0011, 0b01, 0b011, "pmacc.w.h11">;
+  def MACC_W11      : RVPTernary_rrr<0b0011, 0b11, 0b011, "macc.w11">;
+
+  def PMULU_W_H00   : RVPBinary_rr<0b0100, 0b01, 0b011, "pmulu.w.h00">;
+  def MULU_W00      : RVPBinary_rr<0b0100, 0b11, 0b011, "mulu.w00">;
+
+  def PMACCU_W_H00  : RVPTernary_rrr<0b0101, 0b01, 0b011, "pmaccu.w.h00">;
+  def MACCU_W00     : RVPTernary_rrr<0b0101, 0b11, 0b011, "maccu.w00">;
+
+  def PMULU_W_H11   : RVPBinary_rr<0b0110, 0b01, 0b011, "pmulu.w.h11">;
+  def MULU_W11      : RVPBinary_rr<0b0110, 0b11, 0b011, "mulu.w11">;
+
+  def PMACCU_W_H11  : RVPTernary_rrr<0b0111, 0b01, 0b011, "pmaccu.w.h11">;
+  def MACCU_W11     : RVPTernary_rrr<0b0111, 0b11, 0b011, "maccu.w11">;
+
+  def PMULSU_W_H00  : RVPBinary_rr<0b1100, 0b01, 0b011, "pmulsu.w.h00">;
+  def MULSU_W00     : RVPBinary_rr<0b1100, 0b11, 0b011, "mulsu.w00">;
+
+  def PMACCSU_W_H00 : RVPTernary_rrr<0b1101, 0b01, 0b011, "pmaccsu.w.h00">;
+  def MACCSU_W00    : RVPTernary_rrr<0b1101, 0b11, 0b011, "maccsu.w00">;
+
+  def PMULSU_W_H11  : RVPBinary_rr<0b1110, 0b01, 0b011, "pmulsu.w.h11">;
+  def MULSU_W11     : RVPBinary_rr<0b1110, 0b11, 0b011, "mulsu.w11">;
+
+  def PMACCSU_W_H11 : RVPTernary_rrr<0b1111, 0b01, 0b011, "pmaccsu.w.h11">;
+  def MACCSU_W11    : RVPTernary_rrr<0b1111, 0b11, 0b011, "maccsu.w11">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+// Note the spec has a 3-bit f field in bits 30:28 with 0 in bit 27.
+// Here we include the 0 in the f field to reduce number of tablegen classes.
+let Predicates = [HasStdExtP] in {
+  def PPACK_H     : RVPBinary_rr<0b0000, 0b00, 0b100, "ppack.h">;
+
+  def PPACKBT_H   : RVPBinary_rr<0b0010, 0b00, 0b100, "ppackbt.h">;
+
+  def PPACKTB_H   : RVPBinary_rr<0b0100, 0b00, 0b100, "ppacktb.h">;
+
+  def PPACKT_H    : RVPBinary_rr<0b0110, 0b00, 0b100, "ppackt.h">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def PACKBT_RV32 : RVPBinary_rr<0b0010, 0b01, 0b100, "packbt">;
+
+  def PACKTB_RV32 : RVPBinary_rr<0b0100, 0b01, 0b100, "packtb">;
+
+  def PACKT_RV32  : RVPBinary_rr<0b0110, 0b01, 0b100, "packt">;
+} // Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only"
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PPACK_W     : RVPBinary_rr<0b0000, 0b01, 0b100, "ppack.w">;
+
+  def PPACKBT_W   : RVPBinary_rr<0b0010, 0b01, 0b100, "ppackbt.w">;
+  def PACKBT_RV64 : RVPBinary_rr<0b0010, 0b11, 0b100, "packbt">;
+
+  def PPACKTB_W   : RVPBinary_rr<0b0100, 0b01, 0b100, "ppacktb.w">;
+  def PACKTB_RV64 : RVPBinary_rr<0b0100, 0b11, 0b100, "packtb">;
+
+  def PPACKT_W    : RVPBinary_rr<0b0110, 0b01, 0b100, "ppackt.w">;
+  def PACKT_RV64  : RVPBinary_rr<0b0110, 0b11, 0b100, "packt">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+  def PM2ADD_H    : RVPBinary_rr<0b0000, 0b00, 0b101, "pm2add.h">;
+  def PM4ADD_B    : RVPBinary_rr<0b0000, 0b10, 0b101, "pm4add.b">;
+
+  def PM2ADDA_H   : RVPTernary_rrr<0b0001, 0b00, 0b101, "pm2adda.h">;
+  def PM4ADDA_B   : RVPTernary_rrr<0b0001, 0b10, 0b101, "pm4adda.b">;
+
+  def PM2ADD_HX   : RVPBinary_rr<0b0010, 0b00, 0b101, "pm2add.hx">;
+
+  def PM2ADDA_HX  : RVPTernary_rrr<0b0011, 0b00, 0b101, "pm2adda.hx">;
+
+  def PM2ADDU_H   : RVPBinary_rr<0b0100, 0b00, 0b101, "pm2addu.h">;
+  def PM4ADDU_B   : RVPBinary_rr<0b0100, 0b10, 0b101, "pm4addu.b">;
+
+  def PM2ADDAU_H  : RVPTernary_rrr<0b0101, 0b00, 0b101, "pm2addau.h">;
+  def PM4ADDAU_B  : RVPTernary_rrr<0b0101, 0b10, 0b101, "pm4addau.b">;
+
+  def PMQ2ADD_H   : RVPBinary_rr<0b0110, 0b00, 0b101, "pmq2add.h">;
+  def PMQR2ADD_H  : RVPBinary_rr<0b0110, 0b10, 0b101, "pmqr2add.h">;
+
+  def PMQ2ADDA_H  : RVPTernary_rrr<0b0111, 0b00, 0b101, "pmq2adda.h">;
+  def PMQR2ADDA_H : RVPTernary_rrr<0b0111, 0b10, 0b101, "pmqr2adda.h">;
+
+  def PM2SUB_H    : RVPBinary_rr<0b1000, 0b00, 0b101, "pm2sub.h">;
+  def PM2SADD_H   : RVPBinary_rr<0b1000, 0b10, 0b101, "pm2sadd.h">;
+
+  def PM2SUBA_H   : RVPTernary_rrr<0b1001, 0b00, 0b101, "pm2suba.h">;
+
+  def PM2SUB_HX   : RVPBinary_rr<0b1010, 0b00, 0b101, "pm2sub.hx">;
+  def PM2SADD_HX  : RVPBinary_rr<0b1010, 0b10, 0b101, "pm2sadd.hx">;
+
+  def PM2SUBA_HX  : RVPTernary_rrr<0b1011, 0b00, 0b101, "pm2suba.hx">;
+
+  def PM2ADDSU_H  : RVPBinary_rr<0b1100, 0b00, 0b101, "pm2addsu.h">;
+  def PM4ADDSU_B  : RVPBinary_rr<0b1100, 0b10, 0b101, "pm4addsu.b">;
+
+  def PM2ADDASU_H : RVPBinary_rr<0b1101, 0b00, 0b101, "pm2addasu.h">;
+  def PM4ADDASU_B : RVPBinary_rr<0b1101, 0b10, 0b101, "pm4addasu.b">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def MQACC_H01  : RVPTernary_rrr<0b1111, 0b00, 0b101, "mqacc.h01">;
+  def MQRACC_H01 : RVPTernary_rrr<0b1111, 0b10, 0b101, "mqracc.h01">;
+} // // Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only"
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PM2ADD_W      : RVPBinary_rr<0b0000, 0b01, 0b101, "pm2add.w">;
+  def PM4ADD_H      : RVPBinary_rr<0b0000, 0b11, 0b101, "pm4add.h">;
+
+  def PM2ADDA_W     : RVPTernary_rrr<0b0001, 0b01, 0b101, "pm2adda.w">;
+  def PM4ADDA_H     : RVPTernary_rrr<0b0001, 0b11, 0b101, "pm4adda.h">;
+
+  def PM2ADD_WX     : RVPBinary_rr<0b0010, 0b01, 0b101, "pm2add.wx">;
+
+  def PM2ADDA_WX    : RVPTernary_rrr<0b0011, 0b01, 0b101, "pm2adda.wx">;
+
+  def PM2ADDU_W     : RVPBinary_rr<0b0100, 0b01, 0b101, "pm2addu.w">;
+  def PM4ADDU_H     : RVPBinary_rr<0b0100, 0b11, 0b101, "pm4addu.h">;
+
+  def PM2ADDAU_W    : RVPTernary_rrr<0b0101, 0b01, 0b101, "pm2addau.w">;
+  def PM4ADDAU_H    : RVPTernary_rrr<0b0101, 0b11, 0b101, "pm4addau.h">;
+
+  def PMQ2ADD_W     : RVPBinary_rr<0b0110, 0b01, 0b101, "pmq2add.w">;
+  def PMQR2ADD_W    : RVPBinary_rr<0b0110, 0b11, 0b101, "pmqr2add.w">;
+
+  def PMQ2ADDA_W    : RVPTernary_rrr<0b0111, 0b01, 0b101, "pmq2adda.w">;
+  def PMQR2ADDA_W   : RVPTernary_rrr<0b0111, 0b11, 0b101, "pmqr2adda.w">;
+
+  def PM2SUB_W      : RVPBinary_rr<0b1000, 0b01, 0b101, "pm2sub.w">;
+
+  def PM2SUBA_W     : RVPTernary_rrr<0b1001, 0b01, 0b101, "pm2suba.w">;
+
+  def PM2SUB_WX     : RVPBinary_rr<0b1010, 0b01, 0b101, "pm2sub.wx">;
+
+  def PM2SUBA_WX    : RVPBinary_rr<0b1011, 0b01, 0b101, "pm2suba.wx">;
+
+  def PM2ADDSU_W    : RVPBinary_rr<0b1100, 0b01, 0b101, "pm2addsu.w">;
+  def PM4ADDSU_H    : RVPBinary_rr<0b1100, 0b11, 0b101, "pm4addsu.h">;
+
+  def PM2ADDASU_W   : RVPTernary_rrr<0b1101, 0b01, 0b101, "pm2addasu.w">;
+  def PM4ADDASU_H   : RVPTernary_rrr<0b1101, 0b11, 0b101, "pm4addasu.h">;
+
+  def PMQACC_W_H01  : RVPTernary_rrr<0b1111, 0b00, 0b101, "pmqacc.w.h01">;
+  def MQACC_W01     : RVPTernary_rrr<0b1111, 0b01, 0b101, "mqacc.w01">;
+
+  def PMQRACC_W_H01 : RVPTernary_rrr<0b1111, 0b10, 0b101, "pmqracc.w.h01">;
+  def MQRACC_W01    : RVPTernary_rrr<0b1111, 0b11, 0b101, "mqracc.w01">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+  def PAS_HX   : RVPBinary_rr<0b0000, 0b00, 0b110, "pas.hx">;
+  def PSA_HX   : RVPBinary_rr<0b0000, 0b10, 0b110, "psa.hx">;
+
+  def PSAS_HX  : RVPBinary_rr<0b0010, 0b00, 0b110, "psas.hx">;
+  def PSSA_HX  : RVPBinary_rr<0b0010, 0b10, 0b110, "pssa.hx">;
+
+  def PAAS_HX  : RVPBinary_rr<0b0011, 0b00, 0b110, "paas.hx">;
+  def PASA_HX  : RVPBinary_rr<0b0011, 0b10, 0b110, "pasa.hx">;
+
+  def PMSEQ_H  : RVPBinary_rr<0b1000, 0b00, 0b110, "pmseq.h">;
+  def PMSEQ_B  : RVPBinary_rr<0b1000, 0b10, 0b110, "pmseq.b">;
+
+  def PMSLT_H  : RVPBinary_rr<0b1010, 0b00, 0b110, "pmslt.h">;
+  def PMSLT_B  : RVPBinary_rr<0b1010, 0b10, 0b110, "pmslt.b">;
+
+  def PMSLTU_H : RVPBinary_rr<0b1011, 0b00, 0b110, "pmsltu.h">;
+  def PMSLTU_B : RVPBinary_rr<0b1011, 0b10, 0b110, "pmsltu.b">;
+
+  def PMIN_H   : RVPBinary_rr<0b1100, 0b00, 0b110, "pmin.h">;
+  def PMIN_B   : RVPBinary_rr<0b1100, 0b10, 0b110, "pmin.b">;
+
+  def PMINU_H  : RVPBinary_rr<0b1101, 0b00, 0b110, "pminu.h">;
+  def PMINU_B  : RVPBinary_rr<0b1101, 0b10, 0b110, "pminu.b">;
+
+  def PMAX_H   : RVPBinary_rr<0b1110, 0b00, 0b110, "pmax.h">;
+  def PMAX_B   : RVPBinary_rr<0b1110, 0b10, 0b110, "pmax.b">;
+
+  def PMAXU_H  : RVPBinary_rr<0b1111, 0b00, 0b110, "pmaxu.h">;
+  def PMAXU_B  : RVPBinary_rr<0b1111, 0b10, 0b110, "pmaxu.b">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def MSEQ  : RVPBinary_rr<0b1000, 0b01, 0b110, "mseq">;
+
+  def MSLT  : RVPBinary_rr<0b1010, 0b01, 0b110, "mslt">;
+
+  def MSLTU : RVPBinary_rr<0b1011, 0b01, 0b110, "msltu">;
+} // Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only"
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PAS_WX   : RVPBinary_rr<0b0000, 0b01, 0b110, "pas.wx">;
+  def PSA_WX   : RVPBinary_rr<0b0000, 0b11, 0b110, "psa.wx">;
+
+  def PSAS_WX  : RVPBinary_rr<0b0010, 0b01, 0b110, "psas.wx">;
+  def PSSA_WX  : RVPBinary_rr<0b0010, 0b11, 0b110, "pssa.wx">;
+
+  def PAAS_WX  : RVPBinary_rr<0b0011, 0b01, 0b110, "paas.wx">;
+  def PASA_WX  : RVPBinary_rr<0b0011, 0b11, 0b110, "pasa.wx">;
+
+  def PMSEQ_W  : RVPBinary_rr<0b1000, 0b01, 0b110, "pmseq.w">;
+
+  def PMSLT_W  : RVPBinary_rr<0b1010, 0b01, 0b110, "pmslt.w">;
+
+  def PMSLTU_W : RVPBinary_rr<0b1011, 0b01, 0b110, "pmsltu.w">;
+
+  def PMIN_W   : RVPBinary_rr<0b1100, 0b01, 0b110, "pmin.w">;
+
+  def PMINU_W  : RVPBinary_rr<0b1101, 0b01, 0b110, "pminu.w">;
+
+  def PMAX_W   : RVPBinary_rr<0b1110, 0b01, 0b110, "pmax.w">;
+
+  def PMAXU_W  : RVPBinary_rr<0b1111, 0b01, 0b110, "pmaxu.w">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP] in {
+  def PMULH_H      : RVPBinary_rr<0b0000, 0b00, 0b111, "pmulh.h">;
+  def PMULHR_H     : RVPBinary_rr<0b0000, 0b10, 0b111, "pmulhr.h">;
+
+  def PMHACC_H     : RVPTernary_rrr<0b0001, 0b00, 0b111, "pmhacc.h">;
+  def PMHRACC_H    : RVPTernary_rrr<0b0001, 0b10, 0b111, "pmhracc.h">;
+
+  def PMULHU_H     : RVPBinary_rr<0b0010, 0b00, 0b111, "pmulhu.h">;
+  def PMULHRU_H    : RVPBinary_rr<0b0010, 0b10, 0b111, "pmulhru.h">;
+
+  def PMHACCU_H    : RVPTernary_rrr<0b0011, 0b00, 0b111, "pmhaccu.h">;
+  def PMHRACCU_H   : RVPTernary_rrr<0b0011, 0b10, 0b111, "pmhraccu.h">;
+
+  def PMULH_H_B0    : RVPBinary_rr<0b0100, 0b00, 0b111, "pmulh.h.b0">;
+  def PMULHSU_H_B0  : RVPBinary_rr<0b0100, 0b10, 0b111, "pmulhsu.h.b0">;
+
+  def PMHACC_H_B0   : RVPTernary_rrr<0b0101, 0b00, 0b111, "pmhacc.h.b0">;
+  def PMHACCSU_H_B0 : RVPTernary_rrr<0b0101, 0b10, 0b111, "pmhaccsu.h.b0">;
+
+  def PMULH_H_B1    : RVPBinary_rr<0b0110, 0b00, 0b111, "pmulh.h.b1">;
+  def PMULHSU_H_B1  : RVPBinary_rr<0b0110, 0b10, 0b111, "pmulhsu.h.b1">;
+
+  def PMHACC_H_B1   : RVPTernary_rrr<0b0111, 0b00, 0b111, "pmhacc.h.b1">;
+  def PMHACCSU_H_B1 : RVPTernary_rrr<0b0111, 0b10, 0b111, "pmhaccsu.h.b1">;
+
+  def PMULHSU_H     : RVPBinary_rr<0b1000, 0b00, 0b111, "pmulhsu.h">;
+  def PMULHRSU_H    : RVPBinary_rr<0b1000, 0b10, 0b111, "pmulhrsu.h">;
+
+  def PMHACCSU_H    : RVPTernary_rrr<0b1001, 0b00, 0b111, "pmhaccsu.h">;
+  def PMHRACCSU_H   : RVPTernary_rrr<0b1001, 0b10, 0b111, "pmhraccsu.h">;
+
+  def PMULQ_H       : RVPBinary_rr<0b1010, 0b00, 0b111, "pmulq.h">;
+  def PMULQR_H      : RVPBinary_rr<0b1010, 0b10, 0b111, "pmulqr.h">;
+} // Predicates = [HasStdExtP]
+let Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in {
+  def MULHR      : RVPBinary_rr<0b0000, 0b11, 0b111, "mulhr">;
+
+  def MHACC      : RVPTernary_rrr<0b0001, 0b01, 0b111, "mhacc">;
+  def MHRACC     : RVPTernary_rrr<0b0001, 0b11, 0b111, "mhracc">;
+
+  def MULHRU     : RVPBinary_rr<0b0010, 0b11, 0b111, "mulhru">;
+
+  def MHACCU     : RVPTernary_rrr<0b0011, 0b01, 0b111, "mhaccu">;
+  def MHRACCU    : RVPTernary_rrr<0b0011, 0b11, 0b111, "mhraccu">;
+
+  def MULH_H0    : RVPBinary_rr<0b0100, 0b01, 0b111, "mulh.h0">;
+  def MULHSU_H0  : RVPBinary_rr<0b0100, 0b11, 0b111, "mulhsu.h0">;
+
+  def MHACC_H0   : RVPTernary_rrr<0b0101, 0b01, 0b111, "mhacc.h0">;
+  def MHACCSU_H0 : RVPTernary_rrr<0b0101, 0b11, 0b111, "mhaccsu.h0">;
+
+  def MULH_H1    : RVPBinary_rr<0b0110, 0b01, 0b111, "mulh.h1">;
+  def MULHSU_H1  : RVPBinary_rr<0b0110, 0b11, 0b111, "mulhsu.h1">;
+
+  def MHACC_H1   : RVPTernary_rrr<0b0111, 0b01, 0b111, "mhacc.h1">;
+  def MHACCSU_H1 : RVPTernary_rrr<0b0111, 0b11, 0b111, "mhaccsu.h1">;
+
+  def MULHRSU  : RVPBinary_rr<0b1000, 0b11, 0b111, "mulhrsu">;
+
+  def MHACCSU    : RVPTernary_rrr<0b1001, 0b01, 0b111, "mhaccsu">;
+  def MHRACCSU   : RVPTernary_rrr<0b1001, 0b11, 0b111, "mhraccsu">;
+
+  def MULQ       : RVPBinary_rr<0b1010, 0b01, 0b111, "mulq">;
+  def MULQR      : RVPBinary_rr<0b1010, 0b11, 0b111, "mulqr">;
+
+  def MQACC_H00  : RVPTernary_rrr<0b1101, 0b00, 0b111, "mqacc.h00">;
+  def MQRACC_H00 : RVPTernary_rrr<0b1101, 0b10, 0b111, "mqracc.h00">;
+
+  def MQACC_H11  : RVPTernary_rrr<0b1111, 0b00, 0b111, "mqacc.h11">;
+  def MQRACC_H11 : RVPTernary_rrr<0b1111, 0b10, 0b111, "mqracc.h11">;
+} // Predicates = [HasStdExtP, IsRV32], DecoderNamespace = "RV32Only" in
+let Predicates = [HasStdExtP, IsRV64] in {
+  def PMULH_W       : RVPBinary_rr<0b0000, 0b01, 0b111, "pmulh.w">;
+  def PMULHR_W      : RVPBinary_rr<0b0000, 0b11, 0b111, "pmulhr.w">;
+
+  def PMHACC_W      : RVPTernary_rrr<0b0001, 0b01, 0b111, "pmhacc.w">;
+  def PMHRACC_W     : RVPTernary_rrr<0b0001, 0b11, 0b111, "pmhracc.w">;
+
+  def PMULHU_W      : RVPBinary_rr<0b0010, 0b01, 0b111, "pmulhu.w">;
+  def PMULHRU_W     : RVPBinary_rr<0b0010, 0b11, 0b111, "pmulhru.w">;
+
+  def PMHACCU_W     : RVPTernary_rrr<0b0011, 0b01, 0b111, "pmhaccu.w">;
+  def PMHRACCU_W    : RVPTernary_rrr<0b0011, 0b11, 0b111, "pmhraccu.w">;
+
+  def PMULH_W_H0    : RVPBinary_rr<0b0100, 0b01, 0b111, "pmulh.w.h0">;
+  def PMULHSU_W_H0  : RVPBinary_rr<0b0100, 0b11, 0b111, "pmulhsu.w.h0">;
+
+  def PMHACC_W_H0   : RVPTernary_rrr<0b0101, 0b01, 0b111, "pmhacc.w.h0">;
+  def PMHACCSU_W_H0 : RVPTernary_rrr<0b0101, 0b11, 0b111, "pmhaccsu.w.h0">;
+
+  def PMULH_W_H1    : RVPBinary_rr<0b0110, 0b01, 0b111, "pmulh.w.h1">;
+  def PMULHSU_W_H1  : RVPBinary_rr<0b0110, 0b11, 0b111, "pmulhsu.w.h1">;
+
+  def PMHACC_W_H1   : RVPTernary_rrr<0b0111, 0b01, 0b111, "pmhacc.w.h1">;
+  def PMHACCSU_W_H1 : RVPTernary_rrr<0b0111, 0b11, 0b111, "pmhaccsu.w.h1">;
+
+  def PMULHSU_W    : RVPBinary_rr<0b1000, 0b01, 0b111, "pmulhsu.w">;
+  def PMULHRSU_W   : RVPBinary_rr<0b1000, 0b11, 0b111, "pmulhrsu.w">;
+
+  def PMHACCSU_W   : RVPTernary_rrr<0b1001, 0b01, 0b111, "pmhaccsu.w">;
+  def PMHRACCSU_W  : RVPTernary_rrr<0b1001, 0b11, 0b111, "pmhraccsu.w">;
+
+  def PMULQ_W      : RVPBinary_rr<0b1010, 0b01, 0b111, "pmulq.w">;
+  def PMULQR_W     : RVPBinary_rr<0b1010, 0b11, 0b111, "pmulqr.w">;
+
+  def PMQACC_W_H00  : RVPTernary_rrr<0b1101, 0b00, 0b111, "pmqacc.w.h00">;
+  def MQACC_W00     : RVPTernary_rrr<0b1101, 0b01, 0b111, "mqacc.w00">;
+  def PMQRACC_W_H00 : RVPTernary_rrr<0b1101, 0b10, 0b111, "pmqracc.w.h00">;
+  def MQRACC_W00    : RVPTernary_rrr<0b1101, 0b11, 0b111, "mqracc.w00">;
+
+  def PMQACC_W_H11  : RVPTernary_rrr<0b1111, 0b00, 0b111, "pmqacc.w.h11">;
+  def MQACC_W11     : RVPTernary_rrr<0b1111, 0b01, 0b111, "mqacc.w11">;
+  def PMQRACC_W_H11 : RVPTernary_rrr<0b1111, 0b10, 0b111, "pmqracc.w.h11">;
+  def MQRACC_W11    : RVPTernary_rrr<0b1111, 0b11, 0b111, "mqracc.w11">;
+} // Predicates = [HasStdExtP, IsRV64]
+
+let Predicates = [HasStdExtP, IsRV32] in {
+  def PLI_DH : RVPPairLoadImm_i<0b0011000, (ins simm10:$imm10), "pli.dh",
+                                "$rd, $imm10"> {
+    bits<10> imm10;
+
+    let Inst{24-16} = imm10{8-0};
+    let Inst{15}    = imm10{9};
+  }
+
+  def PLI_DB : RVPPairLoadImm_i<0b0011010, (ins simm8_unsigned:$imm8), "pli.db",
+                                "$rd, $imm8"> {
+    bits<8> imm8;
+
+    let Inst{24}    = 0b0;
+    let Inst{23-16} = imm8;
+    let Inst{15}    = 0b0;
+  }
+
+  def PLUI_DH : RVPPairLoadImm_i<0b0111000, (ins simm10_unsigned:$imm10),
+                                 "plui.dh", "$rd, $imm10"> {
+    bits<10> imm10;
+
+    let Inst{24}    = imm10{0};
+    let Inst{23-15} = imm10{9-1};
+  }
+}
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
index 33c713833d8b9..8c0ebe6f90a6c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoV.td
@@ -475,8 +475,8 @@ class VALUmVV<bits<6> funct6, RISCVVFormat opv, string opcodestr>
                 opcodestr, "$vd, $vs2, $vs1, $vm">;
 
 // op vd, vs1, vs2, vm (reverse the order of vs1 and vs2)
-class VALUrVV<bits<6> funct6, RISCVVFormat opv, string opcodestr,
-              bit EarlyClobber = 0>
+class VMACVV<bits<6> funct6, RISCVVFormat opv, string opcodestr,
+             bit EarlyClobber = 0>
     : RVInstVV<funct6, opv, (outs VR:$vd_wb),
                 (ins VR:$vd, VR:$vs1, VR:$vs2, VMaskOp:$vm),
                 opcodestr, "$vd, $vs1, $vs2$vm"> {
@@ -505,8 +505,8 @@ class VALUmVX<bits<6> funct6, RISCVVFormat opv, string opcodestr>
                 opcodestr, "$vd, $vs2, $rs1, $vm">;
 
 // op vd, rs1, vs2, vm (reverse the order of rs1 and vs2)
-class VALUrVX<bits<6> funct6, RISCVVFormat opv, string opcodestr,
-              bit EarlyClobber = 0>
+class VMACVX<bits<6> funct6, RISCVVFormat opv, string opcodestr,
+             bit EarlyClobber = 0>
     : RVInstVX<funct6, opv, (outs VR:$vd_wb),
                 (ins VR:$vd, GPR:$rs1, VR:$vs2, VMaskOp:$vm),
                 opcodestr, "$vd, $rs1, $vs2$vm"> {
@@ -549,8 +549,8 @@ class VALUVF<bits<6> funct6, RISCVVFormat opv, string opcodestr>
                 opcodestr, "$vd, $vs2, $rs1$vm">;
 
 // op vd, rs1, vs2, vm (Float) (with mask, reverse the order of rs1 and vs2)
-class VALUrVF<bits<6> funct6, RISCVVFormat opv, string opcodestr,
-              bit EarlyClobber = 0>
+class VMACVF<bits<6> funct6, RISCVVFormat opv, string opcodestr,
+             bit EarlyClobber = 0>
     : RVInstVX<funct6, opv, (outs VR:$vd_wb),
                 (ins VR:$vd, FPR32:$rs1, VR:$vs2, VMaskOp:$vm),
                 opcodestr, "$vd, $rs1, $vs2$vm"> {
@@ -628,17 +628,17 @@ multiclass VALU_MV_V_X<string opcodestr, bits<6> funct6, string vw> {
 }
 
 multiclass VMAC_MV_V_X<string opcodestr, bits<6> funct6> {
-  def V : VALUrVV<funct6, OPMVV, opcodestr # ".vv">,
+  def V : VMACVV<funct6, OPMVV, opcodestr # ".vv">,
           SchedTernaryMC<"WriteVIMulAddV", "ReadVIMulAddV", "ReadVIMulAddV",
                          "ReadVIMulAddV">;
-  def X : VALUrVX<funct6, OPMVX, opcodestr # ".vx">,
+  def X : VMACVX<funct6, OPMVX, opcodestr # ".vx">,
           SchedTernaryMC<"WriteVIMulAddX", "ReadVIMulAddV", "ReadVIMulAddX",
                          "ReadVIMulAddV">;
 }
 
 multiclass VWMAC_MV_X<string opcodestr, bits<6> funct6> {
   let RVVConstraint = WidenV in
-  def X : VALUrVX<funct6, OPMVX, opcodestr # ".vx">,
+  def X : VMACVX<funct6, OPMVX, opcodestr # ".vx", EarlyClobber=1>,
           SchedTernaryMC<"WriteVIWMulAddX", "ReadVIWMulAddV", "ReadVIWMulAddX",
                          "ReadVIWMulAddV">;
 }
@@ -646,7 +646,7 @@ multiclass VWMAC_MV_X<string opcodestr, bits<6> funct6> {
 multiclass VWMAC_MV_V_X<string opcodestr, bits<6> funct6>
    : VWMAC_MV_X<opcodestr, funct6> {
   let RVVConstraint = WidenV in
-  def V : VALUrVV<funct6, OPMVV, opcodestr # ".vv", EarlyClobber=1>,
+  def V : VMACVV<funct6, OPMVV, opcodestr # ".vv", EarlyClobber=1>,
           SchedTernaryMC<"WriteVIWMulAddV", "ReadVIWMulAddV", "ReadVIWMulAddV",
                          "ReadVIWMulAddV">;
 }
@@ -743,20 +743,20 @@ multiclass VWMUL_FV_V_F<string opcodestr, bits<6> funct6> {
 }
 
 multiclass VMAC_FV_V_F<string opcodestr, bits<6> funct6> {
-  def V : VALUrVV<funct6, OPFVV, opcodestr # ".vv">,
+  def V : VMACVV<funct6, OPFVV, opcodestr # ".vv">,
           SchedTernaryMC<"WriteVFMulAddV", "ReadVFMulAddV", "ReadVFMulAddV",
                          "ReadVFMulAddV">;
-  def F : VALUrVF<funct6, OPFVF, opcodestr # ".vf">,
+  def F : VMACVF<funct6, OPFVF, opcodestr # ".vf">,
           SchedTernaryMC<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddF",
                          "ReadVFMulAddV">;
 }
 
 multiclass VWMAC_FV_V_F<string opcodestr, bits<6> funct6> {
   let RVVConstraint = WidenV in {
-  def V : VALUrVV<funct6, OPFVV, opcodestr # ".vv", EarlyClobber=1>,
+  def V : VMACVV<funct6, OPFVV, opcodestr # ".vv", EarlyClobber=1>,
           SchedTernaryMC<"WriteVFWMulAddV", "ReadVFWMulAddV", "ReadVFWMulAddV",
                          "ReadVFWMulAddV">;
-  def F : VALUrVF<funct6, OPFVF, opcodestr # ".vf", EarlyClobber=1>,
+  def F : VMACVF<funct6, OPFVF, opcodestr # ".vf", EarlyClobber=1>,
           SchedTernaryMC<"WriteVFWMulAddF", "ReadVFWMulAddV", "ReadVFWMulAddF",
                          "ReadVFWMulAddV">;
   }
@@ -1703,8 +1703,9 @@ let Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp in {
 defm VSLIDEUP_V : VSLD_IV_X_I<"vslideup", 0b001110, /*slidesUp=*/true>;
 defm VSLIDE1UP_V : VSLD1_MV_X<"vslide1up", 0b001110>;
 } // Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp
+let ReadsPastVL = 1 in
 defm VSLIDEDOWN_V : VSLD_IV_X_I<"vslidedown", 0b001111, /*slidesUp=*/false>;
-let ElementsDependOn = EltDepsVL in
+let ElementsDependOn = EltDepsVL, ReadsPastVL = 1 in
 defm VSLIDE1DOWN_V : VSLD1_MV_X<"vslide1down", 0b001111>;
 } // Predicates = [HasVInstructions]
 
@@ -1712,19 +1713,19 @@ let Predicates = [HasVInstructionsAnyF] in {
 let Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp in {
 defm VFSLIDE1UP_V : VSLD1_FV_F<"vfslide1up", 0b001110>;
 } // Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp
-let ElementsDependOn = EltDepsVL in
+let ElementsDependOn = EltDepsVL, ReadsPastVL = 1 in
 defm VFSLIDE1DOWN_V : VSLD1_FV_F<"vfslide1down", 0b001111>;
 } // Predicates = [HasVInstructionsAnyF]
 
 let Predicates = [HasVInstructions] in {
 // Vector Register Gather Instruction
-let Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather in {
+let Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather, ReadsPastVL = 1 in {
 defm VRGATHER_V : VGTR_IV_V_X_I<"vrgather", 0b001100>;
 def VRGATHEREI16_VV : VALUVV<0b001110, OPIVV, "vrgatherei16.vv">,
                       SchedBinaryMC<"WriteVRGatherEI16VV",
                                     "ReadVRGatherEI16VV_data",
                                     "ReadVRGatherEI16VV_index">;
-} // Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather
+} // Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather, ReadsPastVL = 1
 
 // Vector Compress Instruction
 let Constraints = "@earlyclobber $vd", RVVConstraint = Vcompress, ElementsDependOn = EltDepsVLMask in {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
index c75addd95b149..1fb30a0b73d92 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
@@ -420,7 +420,7 @@ class NDSRVInstVD4DOT<bits<6> funct6, string opcodestr>
 }
 
 class NDSRVInstVBFHCvt<bits<5> vs1, string opcodestr>
-    : RVInst<(outs VR:$vd), (ins VR:$vs2, VMaskOp:$vm),
+    : RVInst<(outs VR:$vd), (ins VR:$vs2),
              opcodestr, "$vd, $vs2", [], InstFormatR> {
   bits<5> vs2;
   bits<5> vd;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
index 0c8487c2f5dbe..889ea98022572 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXMips.td
@@ -129,20 +129,20 @@ class Mips_prefetch_ri<dag outs, dag ins, string opcodestr, string argstr>
 // MIPS extensions
 //===----------------------------------------------------------------------===//
 let Predicates = [HasVendorXMIPSCBOP] ,DecoderNamespace = "Xmipscbop" in {
-  def MIPS_PREFETCH : Mips_prefetch_ri<(outs), (ins GPR:$rs1, uimm9:$imm9, uimm5:$hint),
-                                        "mips.pref", "$hint, ${imm9}(${rs1})">,
-                       Sched<[]>;
+  def MIPS_PREF : Mips_prefetch_ri<(outs), (ins GPR:$rs1, uimm9:$imm9, uimm5:$hint),
+                                    "mips.pref", "$hint, ${imm9}(${rs1})">,
+                   Sched<[]>;
 }
 
 let Predicates = [HasVendorXMIPSCBOP] in {
   // Prefetch Data Write.
   def : Pat<(prefetch (AddrRegImm9  (XLenVT GPR:$rs1), uimm9:$imm9),
              (i32 1), timm, (i32 1)),
-            (MIPS_PREFETCH GPR:$rs1, uimm9:$imm9, 9)>;
+            (MIPS_PREF GPR:$rs1, uimm9:$imm9, 9)>;
   // Prefetch Data Read.
   def : Pat<(prefetch (AddrRegImm9 (XLenVT GPR:$rs1), uimm9:$imm9),
              (i32 0), timm, (i32 1)),
-            (MIPS_PREFETCH GPR:$rs1, uimm9:$imm9, 8)>;
+            (MIPS_PREF GPR:$rs1, uimm9:$imm9, 8)>;
 }
 
 let Predicates = [HasVendorXMIPSCMov], hasSideEffects = 0, mayLoad = 0, mayStore = 0,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td
index ebcf079f300b3..3a6ce3ce1d469 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td
@@ -58,7 +58,7 @@ class CustomRivosXVI<bits<6> funct6, RISCVVFormat opv, dag outs, dag ins,
 
 let Predicates = [HasVendorXRivosVizip], DecoderNamespace = "XRivos",
   Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather,
-  Inst<6-0> = OPC_CUSTOM_2.Value in  {
+  Inst<6-0> = OPC_CUSTOM_2.Value, ReadsPastVL = 1 in  {
 defm RI_VZIPEVEN_V : VALU_IV_V<"ri.vzipeven", 0b001100>;
 defm RI_VZIPODD_V : VALU_IV_V<"ri.vzipodd", 0b011100>;
 defm RI_VZIP2A_V : VALU_IV_V<"ri.vzip2a", 0b000100>;
@@ -126,6 +126,7 @@ def RI_VINSERT : CustomRivosVXI<0b010000, OPMVX, (outs VR:$vd_wb),
                                 (ins VR:$vd, GPR:$rs1, uimm5:$imm),
                                 "ri.vinsert.v.x", "$vd, $rs1, $imm">;
 
+let ReadsPastVL = 1 in
 def RI_VEXTRACT : CustomRivosXVI<0b010111, OPMVV, (outs GPR:$rd),
                                 (ins VR:$vs2, uimm5:$imm),
                                 "ri.vextract.x.v", "$rd, $vs2, $imm">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index a47dfe363c21e..b546339ce99e2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -74,6 +74,7 @@ class RVInstVCCustom2<bits<4> funct6_hi4, bits<3> funct3, dag outs, dag ins,
   let Uses = [VL, VTYPE];
   let RVVConstraint = NoConstraint;
   let ElementsDependOn = EltDepsVLMask;
+  let ReadsPastVL = 1;
 }
 
 class RVInstVCFCustom2<bits<4> funct6_hi4, bits<3> funct3, dag outs, dag ins,
@@ -98,6 +99,7 @@ class RVInstVCFCustom2<bits<4> funct6_hi4, bits<3> funct3, dag outs, dag ins,
   let Uses = [VL, VTYPE];
   let RVVConstraint = NoConstraint;
   let ElementsDependOn = EltDepsVLMask;
+  let ReadsPastVL = 1;
 }
 
 class VCIXInfo<string suffix, VCIXType type, DAGOperand TyRd,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSfmm.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSfmm.td
index 66cb2d53da960..a5ee701386b6d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSfmm.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSfmm.td
@@ -65,6 +65,7 @@ class SFInstTileMemOp<dag outs, dag ins, bits<3> nf, RISCVOpcode opcode,
   let Inst{6-0} = opcode.Value;
 
   let Uses = [VTYPE, VL];
+  let ReadsPastVL = 1;
 }
 
 let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in
@@ -94,6 +95,7 @@ class SFInstTileMoveOp<bits<6> funct6, dag outs, dag ins, string opcodestr,
   let Inst{6-0} = OPC_OP_V.Value;
 
   let Uses = [VTYPE, VL];
+  let ReadsPastVL = 1;
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
@@ -113,6 +115,7 @@ class SFInstMatmulF<dag outs, dag ins, string opcodestr, string argstr>
   let Inst{6-0} = OPC_OP_VE.Value;
 
   let Uses = [VTYPE, VL];
+  let ReadsPastVL = 1;
 }
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in
@@ -135,6 +138,7 @@ class SFInstMatmulF8<bit a, bit b, dag outs, dag ins,
   let Inst{6-0} = OPC_OP_VE.Value;
 
   let Uses = [VTYPE, VL];
+  let ReadsPastVL = 1;
 }
 
 
@@ -167,6 +171,7 @@ class SFInstMatmulI8<bit funct6_1, bit a, bit b, dag outs, dag ins,
   let Inst{6-0} = OPC_OP_VE.Value;
 
   let Uses = [VTYPE, VL];
+  let ReadsPastVL = 1;
 }
 
 class I8Encode<bit encoding, string name> {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSpacemiT.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSpacemiT.td
new file mode 100644
index 0000000000000..0f9b795069b98
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSpacemiT.td
@@ -0,0 +1,141 @@
+//===-- RISCVInstrInfoXSpacemiT.td -------------------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the vendor extensions defined by SpacemiT.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Operand definitions.
+//===----------------------------------------------------------------------===//
+
+class SMTVDotOpcode<bits<7> val> {
+  bits<7> Value = val;
+}
+
+class SMTVEncoding2<bits<2> val> {
+  bits<2> Value = val;
+}
+
+def OPMMA       : SMTVDotOpcode<0b1110001>;
+def OPMMA_SLIDE : SMTVDotOpcode<0b1110011>;
+
+//===----------------------------------------------------------------------===//
+// Vector Dot-Product Sign Encoding
+// Defines the signed/unsigned mixing modes for vector dot-product operations.
+// Encoding format: [1:0] bits
+//   00: UU (Unsigned x Unsigned)
+//   01: US (Unsigned x Signed)
+//   10: SU (Signed x Unsigned)
+//   11: SS (Signed x Signed)
+//===----------------------------------------------------------------------===//
+def SMT_VDot_UU        : SMTVEncoding2<0b00>;
+def SMT_VDot_US        : SMTVEncoding2<0b01>;
+def SMT_VDot_SU        : SMTVEncoding2<0b10>;
+def SMT_VDot_SS        : SMTVEncoding2<0b11>;
+
+//===----------------------------------------------------------------------===//
+// Vector Dot-Product Sliding Window Modes
+// Encoding format: [1:0] bits
+//   00: Slide1 (1-element sliding stride)
+//   01: Slide2 (2-element sliding stride)
+//   10: Slide3 (3-element sliding stride)
+//   11: Reserved
+//
+// Used in sliding-window dot-product operations:
+//   vd = vs1 • vs2.slide{1|2|3}  // • = dot product
+//===----------------------------------------------------------------------===//
+def SMT_VDot_Slide1 : SMTVEncoding2<0b00>;
+def SMT_VDot_Slide2 : SMTVEncoding2<0b01>;
+def SMT_VDot_Slide3 : SMTVEncoding2<0b10>;
+
+//===----------------------------------------------------------------------===//
+// Instruction formats
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+// Base vector dot product (no slide) format.
+class RVInstSMTVDot<SMTVEncoding2 sign, string opcodestr>
+    : RVInst<(outs VRM2:$vd), (ins VR:$vs1, VR:$vs2), opcodestr,
+             "$vd, $vs1, $vs2", [], InstFormatR> {
+  bits<5> vd;
+  bits<5> vs1;
+  bits<5> vs2;
+
+  let Inst{31-25} = OPMMA.Value;
+  let Inst{24-20} = vs2;
+  let Inst{19-15} = vs1;
+  let Inst{14} = 0b0;
+  let Inst{13-12} = sign.Value;
+  let Inst{11-8} = vd{4-1};
+  let Inst{7} = 0b0;
+  let Inst{6-0} = OPC_CUSTOM_1.Value;
+}
+
+// Sliding-window vector dot product format.
+class RVInstSMTVDotSlide<SMTVEncoding2 funct2, SMTVEncoding2 sign, string opcodestr>
+    : RVInst<(outs VRM2:$vd), (ins VRM2:$vs1, VR:$vs2), opcodestr,
+             "$vd, $vs1, $vs2", [], InstFormatR> {
+  bits<5> vd;
+  bits<5> vs1;
+  bits<5> vs2;
+
+  let Inst{31-25} = OPMMA_SLIDE.Value;
+  let Inst{24-20} = vs2;
+  let Inst{19-16} = vs1{4-1};
+  let Inst{15-14} = funct2.Value;
+  let Inst{13-12} = sign.Value;
+  let Inst{11-8} = vd{4-1};
+  let Inst{7} = 0b0;
+  let Inst{6-0} = OPC_CUSTOM_1.Value;
+}
+}
+
+//===----------------------------------------------------------------------===//
+// Instructions
+//===----------------------------------------------------------------------===//
+
+let DecoderNamespace = "XSMT" in {
+
+let Predicates = [HasVendorXSMTVDot], ElementsDependOn = EltDepsVL in {
+// Base vector dot product (no slide) instructions
+// NOTE: Destination registers (vd) MUST be even-numbered (v0, v2, ..., v30)
+//       due to hardware alignment constraints. Using odd registers may cause undefined behavior.
+def SMT_VMADOT   : RVInstSMTVDot<SMT_VDot_SS, "smt.vmadot">;
+def SMT_VMADOTU  : RVInstSMTVDot<SMT_VDot_UU, "smt.vmadotu">;
+def SMT_VMADOTSU : RVInstSMTVDot<SMT_VDot_SU, "smt.vmadotsu">;
+def SMT_VMADOTUS : RVInstSMTVDot<SMT_VDot_US, "smt.vmadotus">;
+
+//===----------------------------------------------------------------------===//
+// Sliding-window Vector Dot Product Instructions
+//
+// The numeric suffix (1, 2, 3) specifies the stride of the sliding window:
+//   1: Window slides by 1 element per operation
+//   2: Window slides by 2 elements per operation
+//   3: Window slides by 3 elements per operation
+//
+// These instructions compute dot products with overlapping operand windows
+// where the window position increments by <N> elements between computations.
+//===----------------------------------------------------------------------===//
+// NOTE: Destination registers (vd) and first source register (vs1) MUST be
+//       even-numbered (v0, v2, ..., v30) due to hardware alignment constraints.
+//       Using odd registers may cause undefined behavior.
+def SMT_VMADOT1   : RVInstSMTVDotSlide<SMT_VDot_Slide1, SMT_VDot_SS, "smt.vmadot1">;
+def SMT_VMADOT1U  : RVInstSMTVDotSlide<SMT_VDot_Slide1, SMT_VDot_UU, "smt.vmadot1u">;
+def SMT_VMADOT1SU : RVInstSMTVDotSlide<SMT_VDot_Slide1, SMT_VDot_SU, "smt.vmadot1su">;
+def SMT_VMADOT1US : RVInstSMTVDotSlide<SMT_VDot_Slide1, SMT_VDot_US, "smt.vmadot1us">;
+def SMT_VMADOT2   : RVInstSMTVDotSlide<SMT_VDot_Slide2, SMT_VDot_SS, "smt.vmadot2">;
+def SMT_VMADOT2U  : RVInstSMTVDotSlide<SMT_VDot_Slide2, SMT_VDot_UU, "smt.vmadot2u">;
+def SMT_VMADOT2SU : RVInstSMTVDotSlide<SMT_VDot_Slide2, SMT_VDot_SU, "smt.vmadot2su">;
+def SMT_VMADOT2US : RVInstSMTVDotSlide<SMT_VDot_Slide2, SMT_VDot_US, "smt.vmadot2us">;
+def SMT_VMADOT3   : RVInstSMTVDotSlide<SMT_VDot_Slide3, SMT_VDot_SS, "smt.vmadot3">;
+def SMT_VMADOT3U  : RVInstSMTVDotSlide<SMT_VDot_Slide3, SMT_VDot_UU, "smt.vmadot3u">;
+def SMT_VMADOT3SU : RVInstSMTVDotSlide<SMT_VDot_Slide3, SMT_VDot_SU, "smt.vmadot3su">;
+def SMT_VMADOT3US : RVInstSMTVDotSlide<SMT_VDot_Slide3, SMT_VDot_US, "smt.vmadot3us">;
+}
+} // DecoderNamespace = "XSMT"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZa.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZa.td
index 77692f7cd3bee..7cf6d5ff762ff 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZa.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZa.td
@@ -70,7 +70,7 @@ defm AMOCAS_Q : AMO_cas_aq_rl<0b00101, 0b100, "amocas.q", GPRPairRV64>;
 
 multiclass AMOCASPat<string AtomicOp, string BaseInst, ValueType vt = XLenVT,
                      list<Predicate> ExtraPreds = []> {
-  let Predicates = !listconcat([HasStdExtZacas, NotHasStdExtZtso], ExtraPreds) in {
+  let Predicates = !listconcat([HasStdExtZacas, NoStdExtZtso], ExtraPreds) in {
     def : Pat<(!cast<PatFrag>(AtomicOp#"_monotonic") (vt GPR:$addr),
                                                      (vt GPR:$cmp),
                                                      (vt GPR:$new)),
@@ -91,7 +91,7 @@ multiclass AMOCASPat<string AtomicOp, string BaseInst, ValueType vt = XLenVT,
                                                    (vt GPR:$cmp),
                                                    (vt GPR:$new)),
               (!cast<RVInst>(BaseInst#"_AQ_RL") GPR:$cmp, GPR:$addr, GPR:$new)>;
-  } // Predicates = !listconcat([HasStdExtZacas, NotHasStdExtZtso], ExtraPreds)
+  } // Predicates = !listconcat([HasStdExtZacas, NoStdExtZtso], ExtraPreds)
   let Predicates = !listconcat([HasStdExtZacas, HasStdExtZtso], ExtraPreds) in {
     def : Pat<(!cast<PatFrag>(AtomicOp#"_monotonic") (vt GPR:$addr),
                                                      (vt GPR:$cmp),
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
index 04ffb05c513f4..2abd3e613a037 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZb.td
@@ -399,7 +399,7 @@ def MAX  : ALU_rr<0b0000101, 0b110, "max", Commutable=1>,
            Sched<[WriteIMinMax, ReadIMinMax, ReadIMinMax]>;
 def MAXU : ALU_rr<0b0000101, 0b111, "maxu", Commutable=1>,
            Sched<[WriteIMinMax, ReadIMinMax, ReadIMinMax]>;
-} // Predicates = [HasStdExtZbb]
+} // Predicates = [HasStdExtZbbOrP]
 
 let Predicates = [HasStdExtZbkbOrP] in
 def PACK  : ALU_rr<0b0000100, 0b100, "pack">,
@@ -629,9 +629,6 @@ def : Pat<(or (shl (zexti8 (XLenVT GPR:$rs2)), (XLenVT 8)),
 def : Pat<(and (or (shl GPR:$rs2, (XLenVT 8)),
                    (zexti8 (XLenVT GPR:$rs1))), 0xFFFF),
           (PACKH GPR:$rs1, GPR:$rs2)>;
-def : Pat<(or (shl (zexti8 (XLenVT GPR:$rs2)), (XLenVT 24)),
-              (shl (zexti8 (XLenVT GPR:$rs1)), (XLenVT 16))),
-          (SLLI (XLenVT (PACKH GPR:$rs1, GPR:$rs2)), (XLenVT 16))>;
 
 def : Pat<(binop_allhusers<or> (shl GPR:$rs2, (XLenVT 8)),
                                (zexti8 (XLenVT GPR:$rs1))),
@@ -642,11 +639,15 @@ let Predicates = [HasStdExtZbkb, IsRV32] in {
 def : Pat<(i32 (or (zexti16 (i32 GPR:$rs1)), (shl GPR:$rs2, (i32 16)))),
           (PACK GPR:$rs1, GPR:$rs2)>;
 
+def : Pat<(or (shl GPR:$rs2, (XLenVT 24)),
+              (shl (zexti8 (XLenVT GPR:$rs1)), (XLenVT 16))),
+          (SLLI (XLenVT (PACKH GPR:$rs1, GPR:$rs2)), (XLenVT 16))>;
+
 // Match a pattern of 2 bytes being inserted into bits [31:16], with bits
 // bits [15:0] coming from a zero extended value. We can use pack with packh for
 // bits [31:16]. If bits [15:0] can also be a packh, it can be matched
 // separately.
-def : Pat<(or (or (shl (zexti8 (XLenVT GPR:$op1rs2)), (XLenVT 24)),
+def : Pat<(or (or (shl GPR:$op1rs2, (XLenVT 24)),
                   (shl (zexti8 (XLenVT GPR:$op1rs1)), (XLenVT 16))),
               (zexti16 (XLenVT GPR:$rs1))),
           (PACK (XLenVT GPR:$rs1),
@@ -657,12 +658,55 @@ let Predicates = [HasStdExtZbkb, IsRV64] in {
 def : Pat<(i64 (or (zexti32 (i64 GPR:$rs1)), (shl GPR:$rs2, (i64 32)))),
           (PACK GPR:$rs1, GPR:$rs2)>;
 
+def : Pat<(or (shl (zexti8 (XLenVT GPR:$rs2)), (XLenVT 24)),
+              (shl (zexti8 (XLenVT GPR:$rs1)), (XLenVT 16))),
+          (SLLI (XLenVT (PACKH GPR:$rs1, GPR:$rs2)), (XLenVT 16))>;
+def : Pat<(binop_allwusers<or> (shl GPR:$rs2, (XLenVT 24)),
+                               (shl (zexti8 (XLenVT GPR:$rs1)), (XLenVT 16))),
+          (SLLI (XLenVT (PACKH GPR:$rs1, GPR:$rs2)), (XLenVT 16))>;
+
 def : Pat<(binop_allwusers<or> (shl GPR:$rs2, (i64 16)),
                                (zexti16 (i64 GPR:$rs1))),
           (PACKW GPR:$rs1, GPR:$rs2)>;
 def : Pat<(i64 (or (sext_inreg (shl GPR:$rs2, (i64 16)), i32),
                    (zexti16 (i64 GPR:$rs1)))),
           (PACKW GPR:$rs1, GPR:$rs2)>;
+
+// Match a pattern of 2 bytes being inserted into bits [31:16], with bits
+// bits [15:0] coming from a zero extended value, and bits [63:32] being
+// ignored. We can use packw with packh for bits [31:16]. If bits [15:0] can
+// also be a packh, it can be matched separately.
+def : Pat<(binop_allwusers<or>
+               (or (shl GPR:$op1rs2, (XLenVT 24)),
+                   (shl (zexti8 (XLenVT GPR:$op1rs1)), (XLenVT 16))),
+               (zexti16 (XLenVT GPR:$rs1))),
+          (PACKW GPR:$rs1, (XLenVT (PACKH GPR:$op1rs1, GPR:$op1rs2)))>;
+// We need to manually reassociate the patterns because of the binop_allwusers.
+def : Pat<(binop_allwusers<or>
+               (or (zexti16 (XLenVT GPR:$rs1)),
+                   (shl (zexti8 (XLenVT GPR:$op1rs1)), (XLenVT 16))),
+               (shl GPR:$op1rs2, (XLenVT 24))),
+          (PACKW GPR:$rs1, (XLenVT (PACKH GPR:$op1rs1, GPR:$op1rs2)))>;
+def : Pat<(binop_allwusers<or>
+               (or (zexti16 (XLenVT GPR:$rs1)),
+                   (shl GPR:$op1rs1, (XLenVT 24))),
+               (shl (zexti8 (XLenVT GPR:$op1rs2)), (XLenVT 16))),
+          (PACKW GPR:$rs1, (XLenVT (PACKH GPR:$op1rs1, GPR:$op1rs2)))>;
+
+def : Pat<(i64 (or (or (zexti16 (XLenVT GPR:$rs1)),
+                       (shl (zexti8 (XLenVT GPR:$op1rs2)), (XLenVT 16))),
+                   (sext_inreg (shl GPR:$op1rs1, (XLenVT 24)), i32))),
+          (PACKW GPR:$rs1, (XLenVT (PACKH GPR:$op1rs1, GPR:$op1rs2)))>;
+
+// Match a pattern of 2 halfwords being inserted into bits [63:32], with bits
+// bits [31:0] coming from a zero extended value. We can use pack with packw for
+// bits [63:32]. If bits [63:31] can also be a packw, it can be matched
+// separately.
+def : Pat<(or (or (shl GPR:$op1rs2, (i64 48)),
+                  (shl (zexti16 (i64 GPR:$op1rs1)), (i64 32))),
+              (zexti32 (i64 GPR:$rs1))),
+          (PACK (XLenVT GPR:$rs1),
+                (XLenVT (PACKW GPR:$op1rs1, GPR:$op1rs2)))>;
 } // Predicates = [HasStdExtZbkb, IsRV64]
 
 let Predicates = [HasStdExtZbb, IsRV32] in
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
index 0723b2f568a79..3ddcb1d615dbb 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
@@ -58,11 +58,11 @@ let Predicates = [HasStdExtZicboz] in {
 def CBO_ZERO : CBO_r<0b000000000100, "cbo.zero">, Sched<[]>;
 } // Predicates = [HasStdExtZicboz]
 
-let Predicates = [HasStdExtZicbop, NotHasVendorXMIPSCBOP] in {
+let Predicates = [HasStdExtZicbop, NoVendorXMIPSCBOP] in {
 def PREFETCH_I : Prefetch_ri<0b00000, "prefetch.i">, Sched<[]>;
 def PREFETCH_R : Prefetch_ri<0b00001, "prefetch.r">, Sched<[]>;
 def PREFETCH_W : Prefetch_ri<0b00011, "prefetch.w">, Sched<[]>;
-} // Predicates = [HasStdExtZicbop]
+} // Predicates = [HasStdExtZicbop, NoVendorXMIPSCBOP]
 
 //===----------------------------------------------------------------------===//
 // Patterns
@@ -70,7 +70,7 @@ def PREFETCH_W : Prefetch_ri<0b00011, "prefetch.w">, Sched<[]>;
 
 def AddrRegImmLsb00000 : ComplexPattern<iPTR, 2, "SelectAddrRegImmLsb00000">;
 
-let Predicates = [HasStdExtZicbop, NotHasVendorXMIPSCBOP] in {
+let Predicates = [HasStdExtZicbop, NoVendorXMIPSCBOP] in {
   def : Pat<(prefetch (AddrRegImmLsb00000 (XLenVT GPR:$rs1), simm12_lsb00000:$imm12),
                       timm, timm, (i32 0)),
             (PREFETCH_I GPR:$rs1, simm12_lsb00000:$imm12)>;
@@ -80,4 +80,4 @@ let Predicates = [HasStdExtZicbop, NotHasVendorXMIPSCBOP] in {
   def : Pat<(prefetch (AddrRegImmLsb00000 (XLenVT GPR:$rs1), simm12_lsb00000:$imm12),
                       (i32 1), timm, (i32 1)),
             (PREFETCH_W GPR:$rs1, simm12_lsb00000:$imm12)>;
-}
+} // Predicates = [HasStdExtZicbop, NoVendorXMIPSCBOP]
diff --git a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
index 726920e4015cf..c7b96f5c3d0c8 100644
--- a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp
@@ -63,6 +63,12 @@ static const Intrinsic::ID FixedVlsegIntrIds[] = {
     Intrinsic::riscv_seg6_load_mask, Intrinsic::riscv_seg7_load_mask,
     Intrinsic::riscv_seg8_load_mask};
 
+static const Intrinsic::ID FixedVlssegIntrIds[] = {
+    Intrinsic::riscv_sseg2_load_mask, Intrinsic::riscv_sseg3_load_mask,
+    Intrinsic::riscv_sseg4_load_mask, Intrinsic::riscv_sseg5_load_mask,
+    Intrinsic::riscv_sseg6_load_mask, Intrinsic::riscv_sseg7_load_mask,
+    Intrinsic::riscv_sseg8_load_mask};
+
 static const Intrinsic::ID ScalableVlsegIntrIds[] = {
     Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
     Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
@@ -197,9 +203,15 @@ static bool getMemOperands(unsigned Factor, VectorType *VTy, Type *XLenTy,
 /// %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1
 bool RISCVTargetLowering::lowerInterleavedLoad(
     Instruction *Load, Value *Mask, ArrayRef<ShuffleVectorInst *> Shuffles,
-    ArrayRef<unsigned> Indices, unsigned Factor) const {
+    ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const {
   assert(Indices.size() == Shuffles.size());
+  assert(GapMask.getBitWidth() == Factor);
 
+  // We only support cases where the skipped fields are the trailing ones.
+  // TODO: Lower to strided load if there is only a single active field.
+  unsigned MaskFactor = GapMask.popcount();
+  if (MaskFactor < 2 || !GapMask.isMask())
+    return false;
   IRBuilder<> Builder(Load);
 
   const DataLayout &DL = Load->getDataLayout();
@@ -208,20 +220,37 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
 
   Value *Ptr, *VL;
   Align Alignment;
-  if (!getMemOperands(Factor, VTy, XLenTy, Load, Ptr, Mask, VL, Alignment))
+  if (!getMemOperands(MaskFactor, VTy, XLenTy, Load, Ptr, Mask, VL, Alignment))
     return false;
 
   Type *PtrTy = Ptr->getType();
   unsigned AS = PtrTy->getPointerAddressSpace();
-  if (!isLegalInterleavedAccessType(VTy, Factor, Alignment, AS, DL))
+  if (!isLegalInterleavedAccessType(VTy, MaskFactor, Alignment, AS, DL))
     return false;
 
-  CallInst *VlsegN = Builder.CreateIntrinsic(
-      FixedVlsegIntrIds[Factor - 2], {VTy, PtrTy, XLenTy}, {Ptr, Mask, VL});
+  CallInst *SegLoad = nullptr;
+  if (MaskFactor < Factor) {
+    // Lower to strided segmented load.
+    unsigned ScalarSizeInBytes = DL.getTypeStoreSize(VTy->getElementType());
+    Value *Stride = ConstantInt::get(XLenTy, Factor * ScalarSizeInBytes);
+    SegLoad = Builder.CreateIntrinsic(FixedVlssegIntrIds[MaskFactor - 2],
+                                      {VTy, PtrTy, XLenTy, XLenTy},
+                                      {Ptr, Stride, Mask, VL});
+  } else {
+    // Lower to normal segmented load.
+    SegLoad = Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2],
+                                      {VTy, PtrTy, XLenTy}, {Ptr, Mask, VL});
+  }
 
   for (unsigned i = 0; i < Shuffles.size(); i++) {
-    Value *SubVec = Builder.CreateExtractValue(VlsegN, Indices[i]);
-    Shuffles[i]->replaceAllUsesWith(SubVec);
+    unsigned FactorIdx = Indices[i];
+    if (FactorIdx >= MaskFactor) {
+      // Replace masked-off factors (that are still extracted) with poison.
+      Shuffles[i]->replaceAllUsesWith(PoisonValue::get(VTy));
+    } else {
+      Value *SubVec = Builder.CreateExtractValue(SegLoad, FactorIdx);
+      Shuffles[i]->replaceAllUsesWith(SubVec);
+    }
   }
 
   return true;
diff --git a/llvm/lib/Target/RISCV/RISCVMacroFusion.td b/llvm/lib/Target/RISCV/RISCVMacroFusion.td
index 875a93d09a2c6..39e099bc947b2 100644
--- a/llvm/lib/Target/RISCV/RISCVMacroFusion.td
+++ b/llvm/lib/Target/RISCV/RISCVMacroFusion.td
@@ -91,3 +91,59 @@ def TuneLDADDFusion
                    CheckIsImmOperand<2>,
                    CheckImmOperand<2, 0>
                  ]>>;
+
+defvar Load = [LB, LH, LW, LD, LBU, LHU, LWU];
+
+// Fuse add(.uw) followed by a load (lb, lh, lw, ld, lbu, lhu, lwu):
+//   add(.uw) rd, rs1, rs2
+//   load rd, imm12(rd)
+def TuneADDLoadFusion
+  : SimpleFusion<"add-load-fusion", "HasADDLoadFusion", "Enable ADD(.UW) + load macrofusion",
+                 CheckOpcode<[ADD, ADD_UW]>,
+                 CheckOpcode<Load>>;
+
+// Fuse AUIPC followed by by a load (lb, lh, lw, ld, lbu, lhu, lwu)
+//   auipc rd, imm20
+//   load rd, imm12(rd)
+def TuneAUIPCLoadFusion
+  : SimpleFusion<"auipc-load-fusion", "HasAUIPCLoadFusion",
+                 "Enable AUIPC + load macrofusion",
+                 CheckOpcode<[AUIPC]>,
+                 CheckOpcode<Load>>;
+
+// Fuse LUI followed by a load (lb, lh, lw, ld, lbu, lhu, lwu)
+//   lui rd, imm[31:12]
+//   load rd, imm12(rd)
+def TuneLUILoadFusion
+  : SimpleFusion<"lui-load-fusion", "HasLUILoadFusion",
+                 "Enable LUI + load macrofusion",
+                 CheckOpcode<[LUI]>,
+                 CheckOpcode<Load>>;
+
+// Bitfield extract fusion: similar to TuneShiftedZExtWFusion
+// but without the immediate restriction
+//   slli rd, rs1, imm12
+//   srli rd, rd, imm12
+def TuneBFExtFusion
+  : SimpleFusion<"bfext-fusion", "HasBFExtFusion",
+                 "Enable SLLI+SRLI (bitfield extract) macrofusion",
+                 CheckOpcode<[SLLI]>,
+                 CheckOpcode<[SRLI]>>;
+
+// Fuse ADDI followed by a load (lb, lh, lw, ld, lbu, lhu, lwu)
+//   addi rd, rs1, imm12
+//   load rd, imm12(rd)
+def TuneADDILoadFusion
+  : SimpleFusion<"addi-load-fusion", "HasADDILoadFusion",
+                 "Enable ADDI + load macrofusion",
+                 CheckOpcode<[ADDI]>,
+                 CheckOpcode<Load>>;
+
+// Fuse shXadd(.uw) followed by a load (lb, lh, lw, ld, lbu, lhu, lwu)
+//   shXadd(.uw) rd, rs1, rs2
+//   load rd, imm12(rd)
+def TuneSHXADDLoadFusion
+  : SimpleFusion<"shxadd-load-fusion", "HasSHXADDLoadFusion",
+                 "Enable SH(1|2|3)ADD(.UW) + load macrofusion",
+                 CheckOpcode<[SH1ADD, SH2ADD, SH3ADD, SH1ADD_UW, SH2ADD_UW, SH3ADD_UW]>,
+                 CheckOpcode<Load>>;
diff --git a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp
index 7a2541a652b58..efea1b422d582 100644
--- a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp
@@ -26,6 +26,7 @@ struct RISCVMoveMerge : public MachineFunctionPass {
 
   RISCVMoveMerge() : MachineFunctionPass(ID) {}
 
+  const RISCVSubtarget *ST;
   const RISCVInstrInfo *TII;
   const TargetRegisterInfo *TRI;
 
@@ -37,15 +38,15 @@ struct RISCVMoveMerge : public MachineFunctionPass {
   // Merge the two instructions indicated into a single pair instruction.
   MachineBasicBlock::iterator
   mergePairedInsns(MachineBasicBlock::iterator I,
-                   MachineBasicBlock::iterator Paired, unsigned Opcode);
+                   MachineBasicBlock::iterator Paired, bool MoveFromSToA);
 
   // Look for C.MV instruction that can be combined with
   // the given instruction into CM.MVA01S or CM.MVSA01. Return the matching
   // instruction if one exists.
   MachineBasicBlock::iterator
-  findMatchingInst(MachineBasicBlock::iterator &MBBI, unsigned InstOpcode,
+  findMatchingInst(MachineBasicBlock::iterator &MBBI, bool MoveFromSToA,
                    const DestSourcePair &RegPair);
-  bool mergeMoveSARegPair(const RISCVSubtarget &STI, MachineBasicBlock &MBB);
+  bool mergeMoveSARegPair(MachineBasicBlock &MBB);
   bool runOnMachineFunction(MachineFunction &Fn) override;
 
   StringRef getPassName() const override { return RISCV_MOVE_MERGE_NAME; }
@@ -58,41 +59,21 @@ char RISCVMoveMerge::ID = 0;
 INITIALIZE_PASS(RISCVMoveMerge, "riscv-move-merge", RISCV_MOVE_MERGE_NAME,
                 false, false)
 
-static bool isMoveFromAToS(unsigned Opcode) {
-  switch (Opcode) {
-  case RISCV::CM_MVA01S:
-  case RISCV::QC_CM_MVA01S:
-    return true;
-  default:
-    return false;
-  }
-}
-
-static unsigned getMoveFromAToSOpcode(const RISCVSubtarget &STI) {
-  if (STI.hasStdExtZcmp())
+static unsigned getMoveFromSToAOpcode(const RISCVSubtarget &ST) {
+  if (ST.hasStdExtZcmp())
     return RISCV::CM_MVA01S;
 
-  if (STI.hasVendorXqccmp())
+  if (ST.hasVendorXqccmp())
     return RISCV::QC_CM_MVA01S;
 
   llvm_unreachable("Unhandled subtarget with paired A to S move.");
 }
 
-static bool isMoveFromSToA(unsigned Opcode) {
-  switch (Opcode) {
-  case RISCV::CM_MVSA01:
-  case RISCV::QC_CM_MVSA01:
-    return true;
-  default:
-    return false;
-  }
-}
-
-static unsigned getMoveFromSToAOpcode(const RISCVSubtarget &STI) {
-  if (STI.hasStdExtZcmp())
+static unsigned getMoveFromAToSOpcode(const RISCVSubtarget &ST) {
+  if (ST.hasStdExtZcmp())
     return RISCV::CM_MVSA01;
 
-  if (STI.hasVendorXqccmp())
+  if (ST.hasVendorXqccmp())
     return RISCV::QC_CM_MVSA01;
 
   llvm_unreachable("Unhandled subtarget with paired S to A move");
@@ -123,20 +104,22 @@ bool RISCVMoveMerge::isCandidateToMergeMVSA01(const DestSourcePair &RegPair) {
 MachineBasicBlock::iterator
 RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I,
                                  MachineBasicBlock::iterator Paired,
-                                 unsigned Opcode) {
+                                 bool MoveFromSToA) {
   const MachineOperand *Sreg1, *Sreg2;
   MachineBasicBlock::iterator E = I->getParent()->end();
   MachineBasicBlock::iterator NextI = next_nodbg(I, E);
   DestSourcePair FirstPair = TII->isCopyInstrImpl(*I).value();
   DestSourcePair PairedRegs = TII->isCopyInstrImpl(*Paired).value();
-  Register ARegInFirstPair = isMoveFromAToS(Opcode)
-                                 ? FirstPair.Destination->getReg()
-                                 : FirstPair.Source->getReg();
 
   if (NextI == Paired)
     NextI = next_nodbg(NextI, E);
   DebugLoc DL = I->getDebugLoc();
 
+  // Make a copy so we can update the kill flag in the MoveFromSToA case. The
+  // copied operand needs to be scoped outside the if since we make a pointer
+  // to it.
+  MachineOperand PairedSource = *PairedRegs.Source;
+
   // The order of S-reg depends on which instruction holds A0, instead of
   // the order of register pair.
   // e,g.
@@ -145,13 +128,26 @@ RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I,
   //
   //   mv a0, s2
   //   mv a1, s1    =>  cm.mva01s s2,s1
-  bool StartWithX10 = ARegInFirstPair == RISCV::X10;
-  if (isMoveFromAToS(Opcode)) {
-    Sreg1 = StartWithX10 ? FirstPair.Source : PairedRegs.Source;
-    Sreg2 = StartWithX10 ? PairedRegs.Source : FirstPair.Source;
+  unsigned Opcode;
+  if (MoveFromSToA) {
+    // We are moving one of the copies earlier so its kill flag may become
+    // invalid. Clear the copied kill flag if there are any reads of the
+    // register between the new location and the old location.
+    for (auto It = std::next(I); It != Paired && PairedSource.isKill(); ++It)
+      if (It->readsRegister(PairedSource.getReg(), TRI))
+        PairedSource.setIsKill(false);
+
+    Opcode = getMoveFromSToAOpcode(*ST);
+    Sreg1 = FirstPair.Source;
+    Sreg2 = &PairedSource;
+    if (FirstPair.Destination->getReg() != RISCV::X10)
+      std::swap(Sreg1, Sreg2);
   } else {
-    Sreg1 = StartWithX10 ? FirstPair.Destination : PairedRegs.Destination;
-    Sreg2 = StartWithX10 ? PairedRegs.Destination : FirstPair.Destination;
+    Opcode = getMoveFromAToSOpcode(*ST);
+    Sreg1 = FirstPair.Destination;
+    Sreg2 = PairedRegs.Destination;
+    if (FirstPair.Source->getReg() != RISCV::X10)
+      std::swap(Sreg1, Sreg2);
   }
 
   BuildMI(*I->getParent(), I, DL, TII->get(Opcode)).add(*Sreg1).add(*Sreg2);
@@ -163,7 +159,7 @@ RISCVMoveMerge::mergePairedInsns(MachineBasicBlock::iterator I,
 
 MachineBasicBlock::iterator
 RISCVMoveMerge::findMatchingInst(MachineBasicBlock::iterator &MBBI,
-                                 unsigned InstOpcode,
+                                 bool MoveFromSToA,
                                  const DestSourcePair &RegPair) {
   MachineBasicBlock::iterator E = MBBI->getParent()->end();
 
@@ -181,26 +177,20 @@ RISCVMoveMerge::findMatchingInst(MachineBasicBlock::iterator &MBBI,
       Register SourceReg = SecondPair->Source->getReg();
       Register DestReg = SecondPair->Destination->getReg();
 
-      if (isMoveFromAToS(InstOpcode) && isCandidateToMergeMVA01S(*SecondPair)) {
-        // If register pair is valid and destination registers are different.
-        if ((RegPair.Destination->getReg() == DestReg))
+      bool IsCandidate = MoveFromSToA ? isCandidateToMergeMVA01S(*SecondPair)
+                                      : isCandidateToMergeMVSA01(*SecondPair);
+      if (IsCandidate) {
+        // Second destination must be different.
+        if (RegPair.Destination->getReg() == DestReg)
           return E;
 
-        //  If paired destination register was modified or used, the source reg
-        //  was modified, there is no possibility of finding matching
-        //  instruction so exit early.
-        if (!ModifiedRegUnits.available(DestReg) ||
-            !UsedRegUnits.available(DestReg) ||
-            !ModifiedRegUnits.available(SourceReg))
-          return E;
-
-        return I;
-      } else if (isMoveFromSToA(InstOpcode) &&
-                 isCandidateToMergeMVSA01(*SecondPair)) {
-        if ((RegPair.Source->getReg() == SourceReg) ||
-            (RegPair.Destination->getReg() == DestReg))
+        // For AtoS the source must also be different.
+        if (!MoveFromSToA && RegPair.Source->getReg() == SourceReg)
           return E;
 
+        // If paired destination register was modified or used, the source reg
+        // was modified, there is no possibility of finding matching
+        // instruction so exit early.
         if (!ModifiedRegUnits.available(DestReg) ||
             !UsedRegUnits.available(DestReg) ||
             !ModifiedRegUnits.available(SourceReg))
@@ -217,8 +207,7 @@ RISCVMoveMerge::findMatchingInst(MachineBasicBlock::iterator &MBBI,
 
 // Finds instructions, which could be represented as C.MV instructions and
 // merged into CM.MVA01S or CM.MVSA01.
-bool RISCVMoveMerge::mergeMoveSARegPair(const RISCVSubtarget &STI,
-                                        MachineBasicBlock &MBB) {
+bool RISCVMoveMerge::mergeMoveSARegPair(MachineBasicBlock &MBB) {
   bool Modified = false;
 
   for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
@@ -227,22 +216,17 @@ bool RISCVMoveMerge::mergeMoveSARegPair(const RISCVSubtarget &STI,
     // can, return Dest/Src register pair.
     auto RegPair = TII->isCopyInstrImpl(*MBBI);
     if (RegPair.has_value()) {
-      unsigned Opcode = 0;
-
-      if (isCandidateToMergeMVA01S(*RegPair))
-        Opcode = getMoveFromAToSOpcode(STI);
-      else if (isCandidateToMergeMVSA01(*RegPair))
-        Opcode = getMoveFromSToAOpcode(STI);
-      else {
+      bool MoveFromSToA = isCandidateToMergeMVA01S(*RegPair);
+      if (!MoveFromSToA && !isCandidateToMergeMVSA01(*RegPair)) {
         ++MBBI;
         continue;
       }
 
       MachineBasicBlock::iterator Paired =
-          findMatchingInst(MBBI, Opcode, RegPair.value());
+          findMatchingInst(MBBI, MoveFromSToA, RegPair.value());
       // If matching instruction can be found merge them.
       if (Paired != E) {
-        MBBI = mergePairedInsns(MBBI, Paired, Opcode);
+        MBBI = mergePairedInsns(MBBI, Paired, MoveFromSToA);
         Modified = true;
         continue;
       }
@@ -256,12 +240,12 @@ bool RISCVMoveMerge::runOnMachineFunction(MachineFunction &Fn) {
   if (skipFunction(Fn.getFunction()))
     return false;
 
-  const RISCVSubtarget *Subtarget = &Fn.getSubtarget<RISCVSubtarget>();
-  if (!(Subtarget->hasStdExtZcmp() || Subtarget->hasVendorXqccmp()))
+  ST = &Fn.getSubtarget<RISCVSubtarget>();
+  if (!ST->hasStdExtZcmp() && !ST->hasVendorXqccmp())
     return false;
 
-  TII = Subtarget->getInstrInfo();
-  TRI = Subtarget->getRegisterInfo();
+  TII = ST->getInstrInfo();
+  TRI = ST->getRegisterInfo();
   // Resize the modified and used register unit trackers.  We do this once
   // per function and then clear the register units each time we optimize a
   // move.
@@ -269,7 +253,7 @@ bool RISCVMoveMerge::runOnMachineFunction(MachineFunction &Fn) {
   UsedRegUnits.init(*TRI);
   bool Modified = false;
   for (auto &MBB : Fn)
-    Modified |= mergeMoveSARegPair(*Subtarget, MBB);
+    Modified |= mergeMoveSARegPair(MBB);
   return Modified;
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index 838edf6c57250..f89d94f41b69f 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -590,12 +590,17 @@ def VENTANA_VEYRON_V1 : RISCVProcessorModel<"veyron-v1",
                                              FeatureStdExtZicboz,
                                              FeatureVendorXVentanaCondOps],
                                              [TuneVentanaVeyron,
+                                              TuneDisableMISchedLoadClustering,
+                                              TuneDisablePostMISchedLoadClustering,
+                                              TuneDisablePostMISchedStoreClustering,
                                               TuneLUIADDIFusion,
                                               TuneAUIPCADDIFusion,
                                               TuneZExtHFusion,
                                               TuneZExtWFusion,
                                               TuneShiftedZExtWFusion,
-                                              TuneLDADDFusion]> {
+                                              TuneADDLoadFusion,
+                                              TuneAUIPCLoadFusion,
+                                              TuneLUILoadFusion]> {
   let MVendorID = 0x61f;
   let MArchID = 0x8000000000010000;
   let MImpID = 0x111;
@@ -668,6 +673,7 @@ def SPACEMIT_X60 : RISCVProcessorModel<"spacemit-x60",
                                         FeatureStdExtZvfh,
                                         FeatureStdExtZvkt,
                                         FeatureStdExtZvl256b,
+                                        FeatureVendorXSMTVDot,
                                         FeatureUnalignedScalarMem]),
                                        [TuneDLenFactor2,
                                         TuneOptimizedNF2SegmentLoadStore,
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 7e58b6f342689..f3966a55ce7d1 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -170,7 +170,7 @@ BitVector RISCVRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
 
   if (MF.getFunction().getCallingConv() == CallingConv::GRAAL) {
     if (Subtarget.hasStdExtE())
-      report_fatal_error("Graal reserved registers do not exist in RVE");
+      reportFatalUsageError("Graal reserved registers do not exist in RVE");
     markSuperRegs(Reserved, RISCV::X23_H);
     markSuperRegs(Reserved, RISCV::X27_H);
   }
@@ -216,7 +216,7 @@ void RISCVRegisterInfo::adjustReg(MachineBasicBlock &MBB,
       const int64_t NumOfVReg = Offset.getScalable() / 8;
       const int64_t FixedOffset = NumOfVReg * VLENB;
       if (!isInt<32>(FixedOffset)) {
-        report_fatal_error(
+        reportFatalUsageError(
             "Frame size outside of the signed 32-bit range not supported");
       }
       Offset = StackOffset::getFixed(FixedOffset + Offset.getFixed());
@@ -389,9 +389,25 @@ void RISCVRegisterInfo::adjustReg(MachineBasicBlock &MBB,
       .setMIFlag(Flag);
 }
 
-// Split a VSPILLx_Mx pseudo into multiple whole register stores separated by
-// LMUL*VLENB bytes.
-void RISCVRegisterInfo::lowerVSPILL(MachineBasicBlock::iterator II) const {
+static std::tuple<RISCVVType::VLMUL, const TargetRegisterClass &, unsigned>
+getSpillReloadInfo(unsigned NumRemaining, uint16_t RegEncoding, bool IsSpill) {
+  if (NumRemaining >= 8 && RegEncoding % 8 == 0)
+    return {RISCVVType::LMUL_8, RISCV::VRM8RegClass,
+            IsSpill ? RISCV::VS8R_V : RISCV::VL8RE8_V};
+  if (NumRemaining >= 4 && RegEncoding % 4 == 0)
+    return {RISCVVType::LMUL_4, RISCV::VRM4RegClass,
+            IsSpill ? RISCV::VS4R_V : RISCV::VL4RE8_V};
+  if (NumRemaining >= 2 && RegEncoding % 2 == 0)
+    return {RISCVVType::LMUL_2, RISCV::VRM2RegClass,
+            IsSpill ? RISCV::VS2R_V : RISCV::VL2RE8_V};
+  return {RISCVVType::LMUL_1, RISCV::VRRegClass,
+          IsSpill ? RISCV::VS1R_V : RISCV::VL1RE8_V};
+}
+
+// Split a VSPILLx_Mx/VSPILLx_Mx pseudo into multiple whole register stores
+// separated by LMUL*VLENB bytes.
+void RISCVRegisterInfo::lowerSegmentSpillReload(MachineBasicBlock::iterator II,
+                                                bool IsSpill) const {
   DebugLoc DL = II->getDebugLoc();
   MachineBasicBlock &MBB = *II->getParent();
   MachineFunction &MF = *MBB.getParent();
@@ -403,47 +419,11 @@ void RISCVRegisterInfo::lowerVSPILL(MachineBasicBlock::iterator II) const {
   auto ZvlssegInfo = RISCV::isRVVSpillForZvlsseg(II->getOpcode());
   unsigned NF = ZvlssegInfo->first;
   unsigned LMUL = ZvlssegInfo->second;
-  assert(NF * LMUL <= 8 && "Invalid NF/LMUL combinations.");
-  unsigned Opcode, SubRegIdx;
-  switch (LMUL) {
-  default:
-    llvm_unreachable("LMUL must be 1, 2, or 4.");
-  case 1:
-    Opcode = RISCV::VS1R_V;
-    SubRegIdx = RISCV::sub_vrm1_0;
-    break;
-  case 2:
-    Opcode = RISCV::VS2R_V;
-    SubRegIdx = RISCV::sub_vrm2_0;
-    break;
-  case 4:
-    Opcode = RISCV::VS4R_V;
-    SubRegIdx = RISCV::sub_vrm4_0;
-    break;
-  }
-  static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
-                "Unexpected subreg numbering");
-  static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
-                "Unexpected subreg numbering");
-  static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
-                "Unexpected subreg numbering");
-
-  Register VL = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-  // Optimize for constant VLEN.
-  if (auto VLEN = STI.getRealVLen()) {
-    const int64_t VLENB = *VLEN / 8;
-    int64_t Offset = VLENB * LMUL;
-    STI.getInstrInfo()->movImm(MBB, II, DL, VL, Offset);
-  } else {
-    BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadVLENB), VL);
-    uint32_t ShiftAmount = Log2_32(LMUL);
-    if (ShiftAmount != 0)
-      BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), VL)
-          .addReg(VL)
-          .addImm(ShiftAmount);
-  }
+  unsigned NumRegs = NF * LMUL;
+  assert(NumRegs <= 8 && "Invalid NF/LMUL combinations.");
 
-  Register SrcReg = II->getOperand(0).getReg();
+  Register Reg = II->getOperand(0).getReg();
+  uint16_t RegEncoding = TRI->getEncodingValue(Reg);
   Register Base = II->getOperand(1).getReg();
   bool IsBaseKill = II->getOperand(1).isKill();
   Register NewBase = MRI.createVirtualRegister(&RISCV::GPRRegClass);
@@ -451,100 +431,63 @@ void RISCVRegisterInfo::lowerVSPILL(MachineBasicBlock::iterator II) const {
   auto *OldMMO = *(II->memoperands_begin());
   LocationSize OldLoc = OldMMO->getSize();
   assert(OldLoc.isPrecise() && OldLoc.getValue().isKnownMultipleOf(NF));
-  TypeSize NewSize = OldLoc.getValue().divideCoefficientBy(NF);
-  auto *NewMMO = MF.getMachineMemOperand(OldMMO, OldMMO->getOffset(), NewSize);
-  for (unsigned I = 0; I < NF; ++I) {
-    // Adding implicit-use of super register to describe we are using part of
-    // super register, that prevents machine verifier complaining when part of
-    // subreg is undef, see comment in MachineVerifier::checkLiveness for more
-    // detail.
-    BuildMI(MBB, II, DL, TII->get(Opcode))
-        .addReg(TRI->getSubReg(SrcReg, SubRegIdx + I))
-        .addReg(Base, getKillRegState(I == NF - 1))
-        .addMemOperand(NewMMO)
-        .addReg(SrcReg, RegState::Implicit);
-    if (I != NF - 1)
+  TypeSize VRegSize = OldLoc.getValue().divideCoefficientBy(NumRegs);
+
+  Register VLENB = 0;
+  unsigned PreHandledNum = 0;
+  unsigned I = 0;
+  while (I != NumRegs) {
+    auto [LMulHandled, RegClass, Opcode] =
+        getSpillReloadInfo(NumRegs - I, RegEncoding, IsSpill);
+    auto [RegNumHandled, _] = RISCVVType::decodeVLMUL(LMulHandled);
+    bool IsLast = I + RegNumHandled == NumRegs;
+    if (PreHandledNum) {
+      Register Step;
+      // Optimize for constant VLEN.
+      if (auto VLEN = STI.getRealVLen()) {
+        int64_t Offset = *VLEN / 8 * PreHandledNum;
+        Step = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+        STI.getInstrInfo()->movImm(MBB, II, DL, Step, Offset);
+      } else {
+        if (!VLENB) {
+          VLENB = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+          BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadVLENB), VLENB);
+        }
+        uint32_t ShiftAmount = Log2_32(PreHandledNum);
+        if (ShiftAmount == 0)
+          Step = VLENB;
+        else {
+          Step = MRI.createVirtualRegister(&RISCV::GPRRegClass);
+          BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), Step)
+              .addReg(VLENB, getKillRegState(IsLast))
+              .addImm(ShiftAmount);
+        }
+      }
+
       BuildMI(MBB, II, DL, TII->get(RISCV::ADD), NewBase)
           .addReg(Base, getKillRegState(I != 0 || IsBaseKill))
-          .addReg(VL, getKillRegState(I == NF - 2));
-    Base = NewBase;
-  }
-  II->eraseFromParent();
-}
+          .addReg(Step, getKillRegState(Step != VLENB || IsLast));
+      Base = NewBase;
+    }
 
-// Split a VSPILLx_Mx pseudo into multiple whole register loads separated by
-// LMUL*VLENB bytes.
-void RISCVRegisterInfo::lowerVRELOAD(MachineBasicBlock::iterator II) const {
-  DebugLoc DL = II->getDebugLoc();
-  MachineBasicBlock &MBB = *II->getParent();
-  MachineFunction &MF = *MBB.getParent();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
-  const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
-  const TargetInstrInfo *TII = STI.getInstrInfo();
-  const TargetRegisterInfo *TRI = STI.getRegisterInfo();
+    MCRegister ActualReg = findVRegWithEncoding(RegClass, RegEncoding);
+    MachineInstrBuilder MIB =
+        BuildMI(MBB, II, DL, TII->get(Opcode))
+            .addReg(ActualReg, getDefRegState(!IsSpill))
+            .addReg(Base, getKillRegState(IsLast))
+            .addMemOperand(MF.getMachineMemOperand(OldMMO, OldMMO->getOffset(),
+                                                   VRegSize * RegNumHandled));
 
-  auto ZvlssegInfo = RISCV::isRVVSpillForZvlsseg(II->getOpcode());
-  unsigned NF = ZvlssegInfo->first;
-  unsigned LMUL = ZvlssegInfo->second;
-  assert(NF * LMUL <= 8 && "Invalid NF/LMUL combinations.");
-  unsigned Opcode, SubRegIdx;
-  switch (LMUL) {
-  default:
-    llvm_unreachable("LMUL must be 1, 2, or 4.");
-  case 1:
-    Opcode = RISCV::VL1RE8_V;
-    SubRegIdx = RISCV::sub_vrm1_0;
-    break;
-  case 2:
-    Opcode = RISCV::VL2RE8_V;
-    SubRegIdx = RISCV::sub_vrm2_0;
-    break;
-  case 4:
-    Opcode = RISCV::VL4RE8_V;
-    SubRegIdx = RISCV::sub_vrm4_0;
-    break;
-  }
-  static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
-                "Unexpected subreg numbering");
-  static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
-                "Unexpected subreg numbering");
-  static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
-                "Unexpected subreg numbering");
-
-  Register VL = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-  // Optimize for constant VLEN.
-  if (auto VLEN = STI.getRealVLen()) {
-    const int64_t VLENB = *VLEN / 8;
-    int64_t Offset = VLENB * LMUL;
-    STI.getInstrInfo()->movImm(MBB, II, DL, VL, Offset);
-  } else {
-    BuildMI(MBB, II, DL, TII->get(RISCV::PseudoReadVLENB), VL);
-    uint32_t ShiftAmount = Log2_32(LMUL);
-    if (ShiftAmount != 0)
-      BuildMI(MBB, II, DL, TII->get(RISCV::SLLI), VL)
-          .addReg(VL)
-          .addImm(ShiftAmount);
-  }
+    // Adding implicit-use of super register to describe we are using part of
+    // super register, that prevents machine verifier complaining when part of
+    // subreg is undef, see comment in MachineVerifier::checkLiveness for more
+    // detail.
+    if (IsSpill)
+      MIB.addReg(Reg, RegState::Implicit);
 
-  Register DestReg = II->getOperand(0).getReg();
-  Register Base = II->getOperand(1).getReg();
-  bool IsBaseKill = II->getOperand(1).isKill();
-  Register NewBase = MRI.createVirtualRegister(&RISCV::GPRRegClass);
-  auto *OldMMO = *(II->memoperands_begin());
-  LocationSize OldLoc = OldMMO->getSize();
-  assert(OldLoc.isPrecise() && OldLoc.getValue().isKnownMultipleOf(NF));
-  TypeSize NewSize = OldLoc.getValue().divideCoefficientBy(NF);
-  auto *NewMMO = MF.getMachineMemOperand(OldMMO, OldMMO->getOffset(), NewSize);
-  for (unsigned I = 0; I < NF; ++I) {
-    BuildMI(MBB, II, DL, TII->get(Opcode),
-            TRI->getSubReg(DestReg, SubRegIdx + I))
-        .addReg(Base, getKillRegState(I == NF - 1))
-        .addMemOperand(NewMMO);
-    if (I != NF - 1)
-      BuildMI(MBB, II, DL, TII->get(RISCV::ADD), NewBase)
-          .addReg(Base, getKillRegState(I != 0 || IsBaseKill))
-          .addReg(VL, getKillRegState(I == NF - 2));
-    Base = NewBase;
+    PreHandledNum = RegNumHandled;
+    RegEncoding += RegNumHandled;
+    I += RegNumHandled;
   }
   II->eraseFromParent();
 }
@@ -568,7 +511,7 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
     Offset += StackOffset::getFixed(MI.getOperand(FIOperandNum + 1).getImm());
 
   if (!isInt<32>(Offset.getFixed())) {
-    report_fatal_error(
+    reportFatalUsageError(
         "Frame offsets outside of the signed 32-bit range not supported");
   }
 
@@ -589,7 +532,7 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
                (Lo12 & 0b11111) != 0) {
       // Prefetch instructions require the offset to be 32 byte aligned.
       MI.getOperand(FIOperandNum + 1).ChangeToImmediate(0);
-    } else if (Opc == RISCV::MIPS_PREFETCH && !isUInt<9>(Val)) {
+    } else if (Opc == RISCV::MIPS_PREF && !isUInt<9>(Val)) {
       // MIPS Prefetch instructions require the offset to be 9 bits encoded.
       MI.getOperand(FIOperandNum + 1).ChangeToImmediate(0);
     } else if ((Opc == RISCV::PseudoRV32ZdinxLD ||
@@ -635,9 +578,7 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   }
 
   // Handle spill/fill of synthetic register classes for segment operations to
-  // ensure correctness in the edge case one gets spilled. There are many
-  // possible optimizations here, but given the extreme rarity of such spills,
-  // we prefer simplicity of implementation for now.
+  // ensure correctness in the edge case one gets spilled.
   switch (MI.getOpcode()) {
   case RISCV::PseudoVSPILL2_M1:
   case RISCV::PseudoVSPILL2_M2:
@@ -650,7 +591,7 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   case RISCV::PseudoVSPILL6_M1:
   case RISCV::PseudoVSPILL7_M1:
   case RISCV::PseudoVSPILL8_M1:
-    lowerVSPILL(II);
+    lowerSegmentSpillReload(II, /*IsSpill=*/true);
     return true;
   case RISCV::PseudoVRELOAD2_M1:
   case RISCV::PseudoVRELOAD2_M2:
@@ -663,7 +604,7 @@ bool RISCVRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   case RISCV::PseudoVRELOAD6_M1:
   case RISCV::PseudoVRELOAD7_M1:
   case RISCV::PseudoVRELOAD8_M1:
-    lowerVRELOAD(II);
+    lowerSegmentSpillReload(II, /*IsSpill=*/false);
     return true;
   }
 
@@ -1052,3 +993,12 @@ bool RISCVRegisterInfo::getRegAllocationHints(
 
   return BaseImplRetVal;
 }
+
+Register
+RISCVRegisterInfo::findVRegWithEncoding(const TargetRegisterClass &RegClass,
+                                        uint16_t Encoding) const {
+  MCRegister Reg = RISCV::V0 + Encoding;
+  if (RISCVRI::getLMul(RegClass.TSFlags) == RISCVVType::LMUL_1)
+    return Reg;
+  return getMatchingSuperReg(Reg, RISCV::sub_vrm1_0, &RegClass);
+}
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h
index b368399e2ad14..2810139bf52ea 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h
@@ -107,8 +107,8 @@ struct RISCVRegisterInfo : public RISCVGenRegisterInfo {
   int64_t getFrameIndexInstrOffset(const MachineInstr *MI,
                                    int Idx) const override;
 
-  void lowerVSPILL(MachineBasicBlock::iterator II) const;
-  void lowerVRELOAD(MachineBasicBlock::iterator II) const;
+  void lowerSegmentSpillReload(MachineBasicBlock::iterator II,
+                               bool IsSpill) const;
 
   Register getFrameRegister(const MachineFunction &MF) const override;
 
@@ -144,6 +144,9 @@ struct RISCVRegisterInfo : public RISCVGenRegisterInfo {
                              const MachineFunction &MF, const VirtRegMap *VRM,
                              const LiveRegMatrix *Matrix) const override;
 
+  Register findVRegWithEncoding(const TargetRegisterClass &RegClass,
+                                uint16_t Encoding) const;
+
   static bool isVRRegClass(const TargetRegisterClass *RC) {
     return RISCVRI::isVRegClass(RC->TSFlags) &&
            RISCVRI::getNF(RC->TSFlags) == 1;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedAndes45.td b/llvm/lib/Target/RISCV/RISCVSchedAndes45.td
index 5ef858a787c7c..8cf15fa26e22d 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedAndes45.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedAndes45.td
@@ -24,7 +24,7 @@ let SchedModel = Andes45Model in {
 
 //===----------------------------------------------------------------------===//
 // Andes 45 series CPU
-//   - 2 Interger Arithmetic and Logical Units (ALU)
+//   - 2 Integer Arithmetic and Logical Units (ALU)
 //   - Multiply / Divide Unit (MDU)
 //   - Load Store Unit (LSU)
 //   - Control and Status Register Unit (CSR)
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSpacemitX60.td b/llvm/lib/Target/RISCV/RISCVSchedSpacemitX60.td
index bf23812c992ba..24ebbc3007cec 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSpacemitX60.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSpacemitX60.td
@@ -13,78 +13,113 @@
 //
 //===----------------------------------------------------------------------===//
 
-class SMX60IsWorstCaseMX<string mx, list<string> MxList> {
-  string LLMUL = LargestLMUL<MxList>.r;
-  bit c = !eq(mx, LLMUL);
-}
+//===----------------------------------------------------------------------===//
+// Helpers
+
+// Maps LMUL string to corresponding value from the Values array
+// LMUL values map to array indices as follows:
+//   MF8 -> Values[0], MF4 -> Values[1], MF2 -> Values[2], M1 -> Values[3],
+//   M2 -> Values[4], M4 -> Values[5], M8 -> Values[6]
+// Shorter lists are allowed, e.g., widening instructions don't work on M8
+class GetLMULValue<list<int> Values, string LMUL> {
+  defvar Index = !cond(
+    !eq(LMUL, "MF8"): 0,
+    !eq(LMUL, "MF4"): 1,
+    !eq(LMUL, "MF2"): 2,
+    !eq(LMUL, "M1"):  3,
+    !eq(LMUL, "M2"):  4,
+    !eq(LMUL, "M4"):  5,
+    !eq(LMUL, "M8"):  6,
+  );
 
-class SMX60IsWorstCaseMXSEW<string mx, int sew, list<string> MxList, bit isF = 0> {
-  string LLMUL = LargestLMUL<MxList>.r;
-  int SSEW = SmallestSEW<mx, isF>.r;
-  bit c = !and(!eq(mx, LLMUL), !eq(sew, SSEW));
+  assert !lt(Index, !size(Values)),
+    "Missing LMUL value for '" # LMUL # "'. " #
+    "Expected at least " # !add(Index, 1) # " elements, but got " #
+    !size(Values) # ".";
+
+  int c = Values[Index];
 }
 
-defvar SMX60VLEN = 256;
-defvar SMX60DLEN = !div(SMX60VLEN, 2);
+// Returns BaseValue for LMUL values before startLMUL, Value for startLMUL,
+// then doubles Value for each subsequent LMUL
+// Example: ConstValueUntilLMULThenDoubleBase<"M1", 2, 4, "M8"> returns:
+//   MF8->2, MF4->2, MF2->2, M1->4, M2->8, M4->16, M8->32
+// This is useful for modeling scheduling parameters that scale with LMUL.
+class ConstValueUntilLMULThenDoubleBase<string startLMUL, int BaseValue, int Value, string currentLMUL> {
+  assert !le(BaseValue, Value), "BaseValue must be less-equal to Value";
+  defvar startPos = GetLMULValue<[0, 1, 2, 3, 4, 5, 6], startLMUL>.c;
+  defvar currentPos = GetLMULValue<[0, 1, 2, 3, 4, 5, 6], currentLMUL>.c;
+
+  // Calculate the difference in positions
+  defvar posDiff = !sub(currentPos, startPos);
 
-class Get1248Latency<string mx> {
+  // Calculate Value * (2^posDiff)
   int c = !cond(
-    !eq(mx, "M2") : 2,
-    !eq(mx, "M4") : 4,
-    !eq(mx, "M8") : 8,
-    true: 1
+    !eq(posDiff, 0) : Value,
+    !eq(posDiff, 1) : !mul(Value, 2),
+    !eq(posDiff, 2) : !mul(Value, 4),
+    !eq(posDiff, 3) : !mul(Value, 8),
+    !eq(posDiff, 4) : !mul(Value, 16),
+    !eq(posDiff, 5) : !mul(Value, 32),
+    !eq(posDiff, 6) : !mul(Value, 64),
+    true : BaseValue
   );
 }
 
-// Used for: logical opsz, shifts, sign ext, merge/move, FP sign/recip/convert, mask ops, slides
-class Get4816Latency<string mx> {
-  int c = !cond(
-    !eq(mx, "M4") : 8,
-    !eq(mx, "M8") : 16,
-    true: 4
-  );
+// Same as the previous function but BaseValue == Value
+class ConstValueUntilLMULThenDouble<string startLMUL, int Value, string currentLMUL> {
+  int c = ConstValueUntilLMULThenDoubleBase<startLMUL, Value, Value, currentLMUL>.c;
+}
+
+// Returns MF8->1, MF4->1, MF2->2, M1->4, M2->8, M4->16, M8->32
+class ConstOneUntilMF4ThenDouble<string mx> {
+  int c = ConstValueUntilLMULThenDouble<"MF4", 1, mx>.c;
 }
 
+// Returns MF8->1, MF4->1, MF2->1, M1->2, M2->4, M4->8, M8->16
+class ConstOneUntilMF2ThenDouble<string mx> {
+  int c = ConstValueUntilLMULThenDouble<"MF2", 1, mx>.c;
+}
+
+// Returns MF8->1, MF4->1, MF2->1, M1->1, M2->2, M4->4, M8->8
+class ConstOneUntilM1ThenDouble<string mx> {
+  int c = ConstValueUntilLMULThenDouble<"M1", 1, mx>.c;
+}
+
+//===----------------------------------------------------------------------===//
+// Latency helper classes
+
 // Used for: arithmetic (add/sub/min/max), saturating/averaging, FP add/sub/min/max
-class Get458Latency<string mx> {
-  int c = !cond(
-    !eq(mx, "M4") : 5,
-    !eq(mx, "M8") : 8,
-    true: 4
-  );
+class Get4458Latency<string mx> {
+  int c = GetLMULValue<[/*MF8=*/4, /*MF4=*/4, /*MF2=*/4, /*M1=*/4, /*M2=*/4, /*M4=*/5, /*M8=*/8], mx>.c;
 }
 
-// Widening scaling pattern (4,4,4,4,5,8,8): plateaus at higher LMULs
-// Used for: widening operations
+// Used for: widening operations (no M8)
 class Get4588Latency<string mx> {
-  int c = !cond(
-    !eq(mx, "M2") : 5,
-    !eq(mx, "M4") : 8,
-    !eq(mx, "M8") : 8, // M8 not supported for most widening, fallback
-    true: 4
-  );
+  int c = GetLMULValue<[/*MF8=*/4, /*MF4=*/4, /*MF2=*/4, /*M1=*/4, /*M2=*/5, /*M4=*/8], mx>.c;
 }
 
 // Used for: mask-producing comparisons, carry ops with mask, FP comparisons
 class Get461018Latency<string mx> {
-  int c = !cond(
-    !eq(mx, "M2") : 6,
-    !eq(mx, "M4") : 10,
-    !eq(mx, "M8") : 18,
-    true: 4
-  );
+  int c = GetLMULValue<[/*MF8=*/4, /*MF4=*/4, /*MF2=*/4, /*M1=*/4, /*M2=*/6, /*M4=*/10, /*M8=*/18], mx>.c;
 }
 
-// Used for: e64 multiply pattern, complex ops
-class Get781632Latency<string mx> {
-  int c = !cond(
-    !eq(mx, "M2") : 8,
-    !eq(mx, "M4") : 16,
-    !eq(mx, "M8") : 32,
-    true: 7
-  );
+//===----------------------------------------------------------------------===//
+
+class SMX60IsWorstCaseMX<string mx, list<string> MxList> {
+  string LLMUL = LargestLMUL<MxList>.r;
+  bit c = !eq(mx, LLMUL);
 }
 
+class SMX60IsWorstCaseMXSEW<string mx, int sew, list<string> MxList, bit isF = 0> {
+  string LLMUL = LargestLMUL<MxList>.r;
+  int SSEW = SmallestSEW<mx, isF>.r;
+  bit c = !and(!eq(mx, LLMUL), !eq(sew, SSEW));
+}
+
+defvar SMX60VLEN = 256;
+defvar SMX60DLEN = !div(SMX60VLEN, 2);
+
 def SpacemitX60Model : SchedMachineModel {
   let IssueWidth        = 2; // dual-issue
   let MicroOpBufferSize = 0; // in-order
@@ -383,12 +418,13 @@ foreach LMul = [1, 2, 4, 8] in {
 foreach mx = SchedMxList in {
   defvar IsWorstCase = SMX60IsWorstCaseMX<mx, SchedMxList>.c;
 
-  let Latency = Get458Latency<mx>.c, ReleaseAtCycles = [4] in {
+  let Latency = Get4458Latency<mx>.c, ReleaseAtCycles = [4] in {
     defm "" : LMULWriteResMX<"WriteVIMinMaxV", [SMX60_VIEU], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVIMinMaxX", [SMX60_VIEU], mx, IsWorstCase>;
   }
 
-  let Latency = Get4816Latency<mx>.c, ReleaseAtCycles = [4] in {
+  defvar VIALULat = ConstValueUntilLMULThenDouble<"M2", 4, mx>.c;
+  let Latency = VIALULat, ReleaseAtCycles = [4] in {
     // Pattern of vadd, vsub, vrsub: 4/4/5/8
     // Pattern of vand, vor, vxor:   4/4/8/16
     // They are grouped together, so we used the worst case 4/4/8/16
@@ -425,7 +461,7 @@ foreach mx = SchedMxList in {
   // Pattern of vmacc, vmadd, vmul, vmulh, etc.: e8/e16 = 4/4/5/8, e32 = 5,5,5,8,
   // e64 = 7,8,16,32. We use the worst-case until we can split the SEW.
   // TODO: change WriteVIMulV, etc to be defined with LMULSEWSchedWrites
-  let Latency = Get781632Latency<mx>.c, ReleaseAtCycles = [7] in {
+  let Latency = ConstValueUntilLMULThenDoubleBase<"M2", 7, 8, mx>.c, ReleaseAtCycles = [7] in {
     defm "" : LMULWriteResMX<"WriteVIMulV", [SMX60_VIEU], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVIMulX", [SMX60_VIEU], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVIMulAddV", [SMX60_VIEU], mx, IsWorstCase>;
@@ -461,15 +497,8 @@ foreach mx = SchedMxList in {
   foreach sew = SchedSEWSet<mx>.val in {
     defvar IsWorstCase = SMX60IsWorstCaseMXSEW<mx, sew, SchedMxList>.c;
 
-    // Slightly reduced for fractional LMULs
-    defvar Multiplier = !cond(
-      !eq(mx, "MF8") : 12,
-      !eq(mx, "MF4") : 12,
-      !eq(mx, "MF2") : 12,
-      true: 24
-    );
-
-    let Latency = !mul(Get1248Latency<mx>.c, Multiplier), ReleaseAtCycles = [12] in {
+    defvar VIDivLat = ConstValueUntilLMULThenDouble<"MF2", 12, mx>.c;
+    let Latency = VIDivLat, ReleaseAtCycles = [12] in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVIDivV", [SMX60_VIEU], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVIDivX", [SMX60_VIEU], mx, sew, IsWorstCase>;
     }
@@ -480,14 +509,8 @@ foreach mx = SchedMxList in {
 foreach mx = SchedMxListW in {
   defvar IsWorstCase = SMX60IsWorstCaseMX<mx, SchedMxListW>.c;
 
-  // Slightly increased for integer LMULs
-  defvar Multiplier = !cond(
-    !eq(mx, "M2") : 2,
-    !eq(mx, "M4") : 2,
-    true: 1
-  );
-
-  let Latency = !mul(Get4816Latency<mx>.c, Multiplier), ReleaseAtCycles = [4] in {
+  defvar VNarrowingLat = ConstValueUntilLMULThenDouble<"M1", 4, mx>.c;
+  let Latency = VNarrowingLat, ReleaseAtCycles = [4] in {
     defm "" : LMULWriteResMX<"WriteVNShiftV", [SMX60_VIEU], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVNShiftX", [SMX60_VIEU], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVNShiftI", [SMX60_VIEU], mx, IsWorstCase>;
@@ -501,16 +524,33 @@ foreach mx = SchedMxListW in {
 foreach mx = SchedMxList in {
   defvar IsWorstCase = SMX60IsWorstCaseMX<mx, SchedMxList>.c;
 
-  defm "" : LMULWriteResMX<"WriteVSALUV", [SMX60_VIEU], mx, IsWorstCase>;
-  defm "" : LMULWriteResMX<"WriteVSALUX", [SMX60_VIEU], mx, IsWorstCase>;
-  defm "" : LMULWriteResMX<"WriteVSALUI", [SMX60_VIEU], mx, IsWorstCase>;
-  defm "" : LMULWriteResMX<"WriteVAALUV", [SMX60_VIEU], mx, IsWorstCase>;
-  defm "" : LMULWriteResMX<"WriteVAALUX", [SMX60_VIEU], mx, IsWorstCase>;
-  defm "" : LMULWriteResMX<"WriteVSMulV", [SMX60_VIEU], mx, IsWorstCase>;
-  defm "" : LMULWriteResMX<"WriteVSMulX", [SMX60_VIEU], mx, IsWorstCase>;
-  defm "" : LMULWriteResMX<"WriteVSShiftV", [SMX60_VIEU], mx, IsWorstCase>;
-  defm "" : LMULWriteResMX<"WriteVSShiftX", [SMX60_VIEU], mx, IsWorstCase>;
-  defm "" : LMULWriteResMX<"WriteVSShiftI", [SMX60_VIEU], mx, IsWorstCase>;
+  let Latency = Get4458Latency<mx>.c, ReleaseAtCycles = [ConstOneUntilM1ThenDouble<mx>.c] in {
+    defm "" : LMULWriteResMX<"WriteVSALUV", [SMX60_VIEU], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSALUX", [SMX60_VIEU], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSALUI", [SMX60_VIEU], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAALUV", [SMX60_VIEU], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVAALUX", [SMX60_VIEU], mx, IsWorstCase>;
+  }
+
+  // Latency of vsmul: e8/e16 = 4/4/5/8, e32 = 5/5/5/8, e64 = 7/8/16/32
+  // We use the worst-case until we can split the SEW.
+  defvar VSMulLat = ConstValueUntilLMULThenDoubleBase<"M2", 7, 8, mx>.c;
+  // Latency of vsmul: e8/e16/e32 = 1/2/4/8, e64 = 4/8/16/32
+  // We use the worst-case until we can split the SEW.
+  defvar VSMulOcc = ConstValueUntilLMULThenDoubleBase<"M1", 1, 4, mx>.c;
+  // TODO: change WriteVSMulV/X to be defined with LMULSEWSchedWrites
+  let Latency = VSMulLat, ReleaseAtCycles = [VSMulOcc] in {
+    defm "" : LMULWriteResMX<"WriteVSMulV", [SMX60_VIEU], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSMulX", [SMX60_VIEU], mx, IsWorstCase>;
+  }
+
+  defvar VSShiftLat = ConstValueUntilLMULThenDouble<"M2", 4, mx>.c;
+  defvar VSShiftOcc = ConstOneUntilMF2ThenDouble<mx>.c;
+  let Latency = VSShiftLat, ReleaseAtCycles = [VSShiftOcc] in {
+    defm "" : LMULWriteResMX<"WriteVSShiftV", [SMX60_VIEU], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSShiftX", [SMX60_VIEU], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVSShiftI", [SMX60_VIEU], mx, IsWorstCase>;
+  }
 }
 
 // 13. Vector Floating-Point Instructions
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 3f2a83f8ce98c..d70b1d0dc8d58 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -38,7 +38,6 @@
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/Scalar.h"
-#include "llvm/Transforms/Vectorize/EVLIndVarSimplify.h"
 #include "llvm/Transforms/Vectorize/LoopIdiomVectorize.h"
 #include <optional>
 using namespace llvm;
@@ -94,16 +93,6 @@ static cl::opt<bool>
                            cl::desc("Enable the loop data prefetch pass"),
                            cl::init(true));
 
-static cl::opt<bool> EnableMISchedLoadStoreClustering(
-    "riscv-misched-load-store-clustering", cl::Hidden,
-    cl::desc("Enable load and store clustering in the machine scheduler"),
-    cl::init(true));
-
-static cl::opt<bool> EnablePostMISchedLoadStoreClustering(
-    "riscv-postmisched-load-store-clustering", cl::Hidden,
-    cl::desc("Enable PostRA load and store clustering in the machine scheduler"),
-    cl::init(true));
-
 static cl::opt<bool> DisableVectorMaskMutation(
     "riscv-disable-vector-mask-mutation",
     cl::desc("Disable the vector mask scheduling mutation"), cl::init(false),
@@ -117,6 +106,8 @@ static cl::opt<bool>
 extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
+  RegisterTargetMachine<RISCVTargetMachine> A(getTheRISCV32beTarget());
+  RegisterTargetMachine<RISCVTargetMachine> B(getTheRISCV64beTarget());
   auto *PR = PassRegistry::getPassRegistry();
   initializeGlobalISel(*PR);
   initializeRISCVO0PreLegalizerCombinerPass(*PR);
@@ -150,21 +141,37 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   initializeRISCVAsmPrinterPass(*PR);
 }
 
-static StringRef computeDataLayout(const Triple &TT,
-                                   const TargetOptions &Options) {
-  StringRef ABIName = Options.MCOptions.getABIName();
-  if (TT.isArch64Bit()) {
-    if (ABIName == "lp64e")
-      return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64";
+static std::string computeDataLayout(const Triple &TT,
+                                     const TargetOptions &Opts) {
+  std::string Ret;
 
-    return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
+  if (TT.isLittleEndian())
+    Ret += "e";
+  else
+    Ret += "E";
+
+  Ret += "-m:e";
+
+  // Pointer and integer sizes.
+  if (TT.isArch64Bit()) {
+    Ret += "-p:64:64-i64:64-i128:128";
+    Ret += "-n32:64";
+  } else {
+    assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported");
+    Ret += "-p:32:32-i64:64";
+    Ret += "-n32";
   }
-  assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported");
 
-  if (ABIName == "ilp32e")
-    return "e-m:e-p:32:32-i64:64-n32-S32";
+  // Stack alignment based on ABI.
+  StringRef ABI = Opts.MCOptions.getABIName();
+  if (ABI == "ilp32e")
+    Ret += "-S32";
+  else if (ABI == "lp64e")
+    Ret += "-S64";
+  else
+    Ret += "-S128";
 
-  return "e-m:e-p:32:32-i64:64-n32-S128";
+  return Ret;
 }
 
 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
@@ -294,15 +301,17 @@ bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
 
 ScheduleDAGInstrs *
 RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
+  const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
   ScheduleDAGMILive *DAG = createSchedLive(C);
-  if (EnableMISchedLoadStoreClustering) {
+
+  if (ST.enableMISchedLoadClustering())
     DAG->addMutation(createLoadClusterDAGMutation(
         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+
+  if (ST.enableMISchedStoreClustering())
     DAG->addMutation(createStoreClusterDAGMutation(
         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
-  }
 
-  const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
   if (!DisableVectorMaskMutation && ST.hasVInstructions())
     DAG->addMutation(createRISCVVectorMaskDAGMutation(DAG->TRI));
 
@@ -311,13 +320,16 @@ RISCVTargetMachine::createMachineScheduler(MachineSchedContext *C) const {
 
 ScheduleDAGInstrs *
 RISCVTargetMachine::createPostMachineScheduler(MachineSchedContext *C) const {
+  const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
   ScheduleDAGMI *DAG = createSchedPostRA(C);
-  if (EnablePostMISchedLoadStoreClustering) {
+
+  if (ST.enablePostMISchedLoadClustering())
     DAG->addMutation(createLoadClusterDAGMutation(
         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
+
+  if (ST.enablePostMISchedStoreClustering())
     DAG->addMutation(createStoreClusterDAGMutation(
         DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
-  }
 
   return DAG;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 67f924aadc8c0..c707fb110b10c 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1431,7 +1431,7 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
   case Intrinsic::ctlz:
   case Intrinsic::ctpop: {
     auto LT = getTypeLegalizationCost(RetTy);
-    if (ST->hasVInstructions() && ST->hasStdExtZvbb() && LT.second.isVector()) {
+    if (ST->hasStdExtZvbb() && LT.second.isVector()) {
       unsigned Op;
       switch (ICA.getID()) {
       case Intrinsic::cttz:
@@ -1629,6 +1629,7 @@ InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
   // scalarized if the legalized Src and Dst are not equal sized.
   const DataLayout &DL = this->getDataLayout();
   if (!SrcLT.second.isVector() || !DstLT.second.isVector() ||
+      !SrcLT.first.isValid() || !DstLT.first.isValid() ||
       !TypeSize::isKnownLE(DL.getTypeSizeInBits(Src),
                            SrcLT.second.getSizeInBits()) ||
       !TypeSize::isKnownLE(DL.getTypeSizeInBits(Dst),
@@ -2414,6 +2415,24 @@ InstructionCost RISCVTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
   return BaseCost + SlideCost;
 }
 
+InstructionCost
+RISCVTTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
+                                               TTI::TargetCostKind CostKind,
+                                               unsigned Index) const {
+  if (isa<FixedVectorType>(Val))
+    return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,
+                                                   Index);
+
+  // TODO: This code replicates what LoopVectorize.cpp used to do when asking
+  // for the cost of extracting the last lane of a scalable vector. It probably
+  // needs a more accurate cost.
+  ElementCount EC = cast<VectorType>(Val)->getElementCount();
+  assert(Index < EC.getKnownMinValue() && "Unexpected reverse index");
+  return getVectorInstrCost(Opcode, Val, CostKind,
+                            EC.getKnownMinValue() - 1 - Index, nullptr,
+                            nullptr);
+}
+
 InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
     TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 05d504cbcb6bb..3236b2a35c853 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -114,6 +114,9 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
   bool enableScalableVectorization() const override {
     return ST->hasVInstructions();
   }
+  bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const override {
+    return ST->hasVInstructions();
+  }
   TailFoldingStyle
   getPreferredTailFoldingStyle(bool IVUpdateMayOverflow) const override {
     return ST->hasVInstructions() ? TailFoldingStyle::DataWithEVL
@@ -240,6 +243,11 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
                                      unsigned Index, const Value *Op0,
                                      const Value *Op1) const override;
 
+  InstructionCost
+  getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
+                                   TTI::TargetCostKind CostKind,
+                                   unsigned Index) const override;
+
   InstructionCost getArithmeticInstrCost(
       unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
       TTI::OperandValueInfo Op1Info = {TTI::OK_AnyValue, TTI::OP_None},
@@ -364,6 +372,8 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
 
     switch (RdxDesc.getRecurrenceKind()) {
     case RecurKind::Add:
+    case RecurKind::Sub:
+    case RecurKind::AddChainWithSubs:
     case RecurKind::And:
     case RecurKind::Or:
     case RecurKind::Xor:
diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
index 37a71e8ec7689..53557049ea33c 100644
--- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
@@ -491,8 +491,42 @@ getOperandLog2EEW(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
   // vfirst find-first-set mask bit
   case RISCV::VCPOP_M:
   case RISCV::VFIRST_M:
+  // Vector Bit-manipulation Instructions (Zvbb)
+  // Vector And-Not
+  case RISCV::VANDN_VV:
+  case RISCV::VANDN_VX:
+  // Vector Reverse Bits in Elements
+  case RISCV::VBREV_V:
+  // Vector Reverse Bits in Bytes
+  case RISCV::VBREV8_V:
+  // Vector Reverse Bytes
+  case RISCV::VREV8_V:
+  // Vector Count Leading Zeros
+  case RISCV::VCLZ_V:
+  // Vector Count Trailing Zeros
+  case RISCV::VCTZ_V:
+  // Vector Population Count
+  case RISCV::VCPOP_V:
+  // Vector Rotate Left
+  case RISCV::VROL_VV:
+  case RISCV::VROL_VX:
+  // Vector Rotate Right
+  case RISCV::VROR_VI:
+  case RISCV::VROR_VV:
+  case RISCV::VROR_VX:
+  // Vector Carry-less Multiplication Instructions (Zvbc)
+  // Vector Carry-less Multiply
+  case RISCV::VCLMUL_VV:
+  case RISCV::VCLMUL_VX:
+  // Vector Carry-less Multiply Return High Half
+  case RISCV::VCLMULH_VV:
+  case RISCV::VCLMULH_VX:
     return MILog2SEW;
 
+  // Vector Widening Shift Left Logical (Zvbb)
+  case RISCV::VWSLL_VI:
+  case RISCV::VWSLL_VX:
+  case RISCV::VWSLL_VV:
   // Vector Widening Integer Add/Subtract
   // Def uses EEW=2*SEW . Operands use EEW=SEW.
   case RISCV::VWADDU_VV:
@@ -503,9 +537,6 @@ getOperandLog2EEW(const MachineOperand &MO, const MachineRegisterInfo *MRI) {
   case RISCV::VWADD_VX:
   case RISCV::VWSUB_VV:
   case RISCV::VWSUB_VX:
-  case RISCV::VWSLL_VI:
-  case RISCV::VWSLL_VX:
-  case RISCV::VWSLL_VV:
   // Vector Widening Integer Multiply Instructions
   // Destination EEW=2*SEW. Source EEW=SEW.
   case RISCV::VWMUL_VV:
@@ -816,13 +847,7 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VLUXEI32_V:
   case RISCV::VLOXEI32_V:
   case RISCV::VLUXEI64_V:
-  case RISCV::VLOXEI64_V: {
-    for (const MachineMemOperand *MMO : MI.memoperands())
-      if (MMO->isVolatile())
-        return false;
-    return true;
-  }
-
+  case RISCV::VLOXEI64_V:
   // Vector Single-Width Integer Add and Subtract
   case RISCV::VADD_VI:
   case RISCV::VADD_VV:
@@ -1020,12 +1045,40 @@ static bool isSupportedInstr(const MachineInstr &MI) {
   case RISCV::VNCLIP_WV:
   case RISCV::VNCLIP_WX:
   case RISCV::VNCLIP_WI:
-
-  // Vector Crypto
+  // Vector Bit-manipulation Instructions (Zvbb)
+  // Vector And-Not
+  case RISCV::VANDN_VV:
+  case RISCV::VANDN_VX:
+  // Vector Reverse Bits in Elements
+  case RISCV::VBREV_V:
+  // Vector Reverse Bits in Bytes
+  case RISCV::VBREV8_V:
+  // Vector Reverse Bytes
+  case RISCV::VREV8_V:
+  // Vector Count Leading Zeros
+  case RISCV::VCLZ_V:
+  // Vector Count Trailing Zeros
+  case RISCV::VCTZ_V:
+  // Vector Population Count
+  case RISCV::VCPOP_V:
+  // Vector Rotate Left
+  case RISCV::VROL_VV:
+  case RISCV::VROL_VX:
+  // Vector Rotate Right
+  case RISCV::VROR_VI:
+  case RISCV::VROR_VV:
+  case RISCV::VROR_VX:
+  // Vector Widening Shift Left Logical
   case RISCV::VWSLL_VI:
   case RISCV::VWSLL_VX:
   case RISCV::VWSLL_VV:
-
+  // Vector Carry-less Multiplication Instructions (Zvbc)
+  // Vector Carry-less Multiply
+  case RISCV::VCLMUL_VV:
+  case RISCV::VCLMUL_VX:
+  // Vector Carry-less Multiply Return High Half
+  case RISCV::VCLMULH_VV:
+  case RISCV::VCLMULH_VX:
   // Vector Mask Instructions
   // Vector Mask-Register Logical Instructions
   // vmsbf.m set-before-first mask bit
@@ -1213,34 +1266,6 @@ static bool isVectorOpUsedAsScalarOp(const MachineOperand &MO) {
   }
 }
 
-/// Return true if MI may read elements past VL.
-static bool mayReadPastVL(const MachineInstr &MI) {
-  const RISCVVPseudosTable::PseudoInfo *RVV =
-      RISCVVPseudosTable::getPseudoInfo(MI.getOpcode());
-  if (!RVV)
-    return true;
-
-  switch (RVV->BaseInstr) {
-  // vslidedown instructions may read elements past VL. They are handled
-  // according to current tail policy.
-  case RISCV::VSLIDEDOWN_VI:
-  case RISCV::VSLIDEDOWN_VX:
-  case RISCV::VSLIDE1DOWN_VX:
-  case RISCV::VFSLIDE1DOWN_VF:
-
-  // vrgather instructions may read the source vector at any index < VLMAX,
-  // regardless of VL.
-  case RISCV::VRGATHER_VI:
-  case RISCV::VRGATHER_VV:
-  case RISCV::VRGATHER_VX:
-  case RISCV::VRGATHEREI16_VV:
-    return true;
-
-  default:
-    return false;
-  }
-}
-
 bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const {
   const MCInstrDesc &Desc = MI.getDesc();
   if (!RISCVII::hasVLOp(Desc.TSFlags) || !RISCVII::hasSEWOp(Desc.TSFlags))
@@ -1261,6 +1286,13 @@ bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const {
     return false;
   }
 
+  for (const MachineMemOperand *MMO : MI.memoperands()) {
+    if (MMO->isVolatile()) {
+      LLVM_DEBUG(dbgs() << "Not a candidate because contains volatile MMO\n");
+      return false;
+    }
+  }
+
   // Some instructions that produce vectors have semantics that make it more
   // difficult to determine whether the VL can be reduced. For example, some
   // instructions, such as reductions, may write lanes past VL to a scalar
@@ -1301,7 +1333,8 @@ RISCVVLOptimizer::getMinimumVLForUser(const MachineOperand &UserOp) const {
     return std::nullopt;
   }
 
-  if (mayReadPastVL(UserMI)) {
+  if (RISCVII::readsPastVL(
+          TII->get(RISCV::getRVVMCOpcode(UserMI.getOpcode())).TSFlags)) {
     LLVM_DEBUG(dbgs() << "    Abort because used by unsafe instruction\n");
     return std::nullopt;
   }
diff --git a/llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp b/llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp
index fc0965d263a8a..7b0afe46971a3 100644
--- a/llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp
+++ b/llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp
@@ -21,10 +21,24 @@ Target &llvm::getTheRISCV64Target() {
   return TheRISCV64Target;
 }
 
+Target &llvm::getTheRISCV32beTarget() {
+  static Target TheRISCV32beTarget;
+  return TheRISCV32beTarget;
+}
+
+Target &llvm::getTheRISCV64beTarget() {
+  static Target TheRISCV64beTarget;
+  return TheRISCV64beTarget;
+}
+
 extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
 LLVMInitializeRISCVTargetInfo() {
   RegisterTarget<Triple::riscv32, /*HasJIT=*/true> X(
       getTheRISCV32Target(), "riscv32", "32-bit RISC-V", "RISCV");
   RegisterTarget<Triple::riscv64, /*HasJIT=*/true> Y(
       getTheRISCV64Target(), "riscv64", "64-bit RISC-V", "RISCV");
+  RegisterTarget<Triple::riscv32be> A(getTheRISCV32beTarget(), "riscv32be",
+                                      "32-bit big endian RISC-V", "RISCV");
+  RegisterTarget<Triple::riscv64be> B(getTheRISCV64beTarget(), "riscv64be",
+                                      "64-bit big endian RISC-V", "RISCV");
 }
diff --git a/llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.h b/llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.h
index ed00a01fa1a2a..9b9fd2cca2fd0 100644
--- a/llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.h
+++ b/llvm/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.h
@@ -15,6 +15,8 @@ class Target;
 
 Target &getTheRISCV32Target();
 Target &getTheRISCV64Target();
+Target &getTheRISCV32beTarget();
+Target &getTheRISCV64beTarget();
 
 } // namespace llvm
 
diff --git a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
index 78a066bef8abc..ed0a1e10562a8 100644
--- a/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
+++ b/llvm/lib/Target/SPIRV/Analysis/SPIRVConvergenceRegionAnalysis.h
@@ -73,7 +73,11 @@ class ConvergenceRegion {
         Entry(std::move(CR.Entry)), Exits(std::move(CR.Exits)),
         Blocks(std::move(CR.Blocks)) {}
 
+  ~ConvergenceRegion() { releaseMemory(); }
+
+  ConvergenceRegion &operator=(ConvergenceRegion &&CR) = delete;
   ConvergenceRegion(const ConvergenceRegion &other) = delete;
+  ConvergenceRegion &operator=(const ConvergenceRegion &other) = delete;
 
   // Returns true if the given basic block belongs to this region, or to one of
   // its subregion.
@@ -101,6 +105,9 @@ class ConvergenceRegionInfo {
 
   ~ConvergenceRegionInfo() { releaseMemory(); }
 
+  ConvergenceRegionInfo(const ConvergenceRegionInfo &LHS) = delete;
+  ConvergenceRegionInfo &operator=(const ConvergenceRegionInfo &LHS) = delete;
+
   ConvergenceRegionInfo(ConvergenceRegionInfo &&LHS)
       : TopLevelRegion(LHS.TopLevelRegion) {
     if (TopLevelRegion != LHS.TopLevelRegion) {
diff --git a/llvm/lib/Target/SPIRV/CMakeLists.txt b/llvm/lib/Target/SPIRV/CMakeLists.txt
index ba09451104479..6660de995e954 100644
--- a/llvm/lib/Target/SPIRV/CMakeLists.txt
+++ b/llvm/lib/Target/SPIRV/CMakeLists.txt
@@ -26,6 +26,7 @@ add_llvm_target(SPIRVCodeGen
   SPIRVGlobalRegistry.cpp
   SPIRVInstrInfo.cpp
   SPIRVInstructionSelector.cpp
+  SPIRVLegalizeImplicitBinding.cpp
   SPIRVStripConvergentIntrinsics.cpp
   SPIRVLegalizePointerCast.cpp
   SPIRVMergeRegionExitTargets.cpp
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
index 0ed97f5b41c51..d6b6079810471 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.cpp
@@ -38,8 +38,15 @@ struct CapabilityEntry {
   Capability::Capability ReqCapability;
 };
 
+struct EnvironmentEntry {
+  OperandCategory::OperandCategory Category;
+  uint32_t Value;
+  Environment::Environment AllowedEnvironment;
+};
+
 using namespace OperandCategory;
 using namespace Extension;
+using namespace Environment;
 using namespace Capability;
 using namespace InstructionSet;
 #define GET_SymbolicOperands_DECL
@@ -48,6 +55,8 @@ using namespace InstructionSet;
 #define GET_ExtensionEntries_IMPL
 #define GET_CapabilityEntries_DECL
 #define GET_CapabilityEntries_IMPL
+#define GET_EnvironmentEntries_DECL
+#define GET_EnvironmentEntries_IMPL
 #define GET_ExtendedBuiltins_DECL
 #define GET_ExtendedBuiltins_IMPL
 #include "SPIRVGenTables.inc"
@@ -133,6 +142,23 @@ getSymbolicOperandCapabilities(SPIRV::OperandCategory::OperandCategory Category,
   return Capabilities;
 }
 
+EnvironmentList getSymbolicOperandAllowedEnvironments(
+    SPIRV::OperandCategory::OperandCategory Category, uint32_t Value) {
+  EnvironmentList Environments;
+  const SPIRV::EnvironmentEntry *Environment =
+      SPIRV::lookupEnvironmentByCategoryAndValue(Category, Value);
+  auto TableEnd = ArrayRef(SPIRV::EnvironmentEntries).end();
+  while (Environment && Environment->Category == Category &&
+         Environment->Value == Value) {
+    Environments.push_back(static_cast<SPIRV::Environment::Environment>(
+        Environment->AllowedEnvironment));
+    if (++Environment == TableEnd)
+      break;
+  }
+
+  return Environments;
+}
+
 CapabilityList
 getCapabilitiesEnabledByExtension(SPIRV::Extension::Extension Extension) {
   const SPIRV::ExtensionEntry *Entry =
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
index b8c467fef8e8e..c2c08f8831307 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVBaseInfo.h
@@ -37,6 +37,11 @@ namespace Capability {
 #include "SPIRVGenTables.inc"
 } // namespace Capability
 
+namespace Environment {
+#define GET_Environment_DECL
+#include "SPIRVGenTables.inc"
+} // namespace Environment
+
 namespace SourceLanguage {
 #define GET_SourceLanguage_DECL
 #include "SPIRVGenTables.inc"
@@ -241,6 +246,7 @@ enum InstFlags {
 
 using CapabilityList = SmallVector<SPIRV::Capability::Capability, 8>;
 using ExtensionList = SmallVector<SPIRV::Extension::Extension, 8>;
+using EnvironmentList = SmallVector<SPIRV::Environment::Environment, 8>;
 
 std::string
 getSymbolicOperandMnemonic(SPIRV::OperandCategory::OperandCategory Category,
@@ -254,6 +260,8 @@ getSymbolicOperandMaxVersion(SPIRV::OperandCategory::OperandCategory Category,
 CapabilityList
 getSymbolicOperandCapabilities(SPIRV::OperandCategory::OperandCategory Category,
                                uint32_t Value);
+EnvironmentList getSymbolicOperandAllowedEnvironments(
+    SPIRV::OperandCategory::OperandCategory Category, uint32_t Value);
 CapabilityList
 getCapabilitiesEnabledByExtension(SPIRV::Extension::Extension Extension);
 ExtensionList
diff --git a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
index 64d301e5ff174..1e3f7fc0070ff 100644
--- a/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/MCTargetDesc/SPIRVInstPrinter.cpp
@@ -96,7 +96,7 @@ void SPIRVInstPrinter::printOpConstantVarOps(const MCInst *MI,
 void SPIRVInstPrinter::recordOpExtInstImport(const MCInst *MI) {
   MCRegister Reg = MI->getOperand(0).getReg();
   auto Name = getSPIRVStringOperand(*MI, 1);
-  auto Set = getExtInstSetFromString(Name);
+  auto Set = getExtInstSetFromString(std::move(Name));
   ExtInstSetIDs.insert({Reg, Set});
 }
 
@@ -210,6 +210,7 @@ void SPIRVInstPrinter::printInst(const MCInst *MI, uint64_t Address,
         case SPIRV::OpConstantF:
           // The last fixed operand along with any variadic operands that follow
           // are part of the variable value.
+          assert(NumFixedOps > 0 && "Expected at least one fixed operand");
           printOpConstantVarOps(MI, NumFixedOps - 1, OS);
           break;
         case SPIRV::OpCooperativeMatrixMulAddKHR: {
@@ -374,9 +375,17 @@ void SPIRVInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
     const MCOperand &Op = MI->getOperand(OpNo);
     if (Op.isReg())
       O << '%' << (getIDFromRegister(Op.getReg().id()) + 1);
-    else if (Op.isImm())
-      O << formatImm(Op.getImm());
-    else if (Op.isDFPImm())
+    else if (Op.isImm()) {
+      int64_t Imm = Op.getImm();
+      // For OpVectorShuffle:
+      // A Component literal may also be FFFFFFFF, which means the corresponding
+      // result component has no source and is undefined.
+      // LLVM representation of poison/undef becomes -1 when lowered to MI.
+      if (MI->getOpcode() == SPIRV::OpVectorShuffle && Imm == -1)
+        O << "0xFFFFFFFF";
+      else
+        O << formatImm(Imm);
+    } else if (Op.isDFPImm())
       O << formatImm((double)Op.getDFPImm());
     else if (Op.isExpr())
       MAI.printExpr(O, *Op.getExpr());
diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h
index 1688fa32ce436..1934e98ca512f 100644
--- a/llvm/lib/Target/SPIRV/SPIRV.h
+++ b/llvm/lib/Target/SPIRV/SPIRV.h
@@ -23,6 +23,7 @@ ModulePass *createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM);
 FunctionPass *createSPIRVStructurizerPass();
 FunctionPass *createSPIRVMergeRegionExitTargetsPass();
 FunctionPass *createSPIRVStripConvergenceIntrinsicsPass();
+ModulePass *createSPIRVLegalizeImplicitBindingPass();
 FunctionPass *createSPIRVLegalizePointerCastPass(SPIRVTargetMachine *TM);
 FunctionPass *createSPIRVRegularizerPass();
 FunctionPass *createSPIRVPreLegalizerCombiner();
@@ -49,6 +50,7 @@ void initializeSPIRVRegularizerPass(PassRegistry &);
 void initializeSPIRVMergeRegionExitTargetsPass(PassRegistry &);
 void initializeSPIRVPrepareFunctionsPass(PassRegistry &);
 void initializeSPIRVStripConvergentIntrinsicsPass(PassRegistry &);
+void initializeSPIRVLegalizeImplicitBindingPass(PassRegistry &);
 } // namespace llvm
 
 #endif // LLVM_LIB_TARGET_SPIRV_SPIRV_H
diff --git a/llvm/lib/Target/SPIRV/SPIRVAPI.cpp b/llvm/lib/Target/SPIRV/SPIRVAPI.cpp
index cfe7ef486381d..d6581b274e00f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAPI.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAPI.cpp
@@ -156,7 +156,7 @@ SPIRVTranslateModule(Module *M, std::string &SpirvObj, std::string &ErrMsg,
     }
   }
   return SPIRVTranslate(M, SpirvObj, ErrMsg, AllowExtNames, OLevel,
-                        TargetTriple);
+                        std::move(TargetTriple));
 }
 
 } // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
index 1ebfde2a603b9..c2a6e51913a0a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp
@@ -50,7 +50,8 @@ class SPIRVAsmPrinter : public AsmPrinter {
 public:
   explicit SPIRVAsmPrinter(TargetMachine &TM,
                            std::unique_ptr<MCStreamer> Streamer)
-      : AsmPrinter(TM, std::move(Streamer), ID), ST(nullptr), TII(nullptr) {}
+      : AsmPrinter(TM, std::move(Streamer), ID), ModuleSectionsEmitted(false),
+        ST(nullptr), TII(nullptr), MAI(nullptr) {}
   static char ID;
   bool ModuleSectionsEmitted;
   const SPIRVSubtarget *ST;
@@ -591,7 +592,9 @@ void SPIRVAsmPrinter::outputAnnotations(const Module &M) {
           cast<GlobalVariable>(CS->getOperand(1)->stripPointerCasts());
 
       StringRef AnnotationString;
-      getConstantStringInfo(GV, AnnotationString);
+      [[maybe_unused]] bool Success =
+          getConstantStringInfo(GV, AnnotationString);
+      assert(Success && "Failed to get annotation string");
       MCInst Inst;
       Inst.setOpcode(SPIRV::OpDecorate);
       Inst.addOperand(MCOperand::createReg(Reg));
diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
index 25cdf72a658a8..2abd9d36f7606 100644
--- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp
@@ -51,7 +51,7 @@ struct IncomingCall {
   IncomingCall(const std::string BuiltinName, const DemangledBuiltin *Builtin,
                const Register ReturnRegister, const SPIRVType *ReturnType,
                const SmallVectorImpl<Register> &Arguments)
-      : BuiltinName(BuiltinName), Builtin(Builtin),
+      : BuiltinName(std::move(BuiltinName)), Builtin(Builtin),
         ReturnRegister(ReturnRegister), ReturnType(ReturnType),
         Arguments(Arguments) {}
 
@@ -802,10 +802,9 @@ static bool buildAtomicCompareExchangeInst(
     MRI->setRegClass(Tmp, GR->getRegClass(SpvDesiredTy));
   GR->assignSPIRVTypeToVReg(SpvDesiredTy, Tmp, MIRBuilder.getMF());
 
-  SPIRVType *IntTy = GR->getOrCreateSPIRVIntegerType(32, MIRBuilder);
   MIRBuilder.buildInstr(Opcode)
       .addDef(Tmp)
-      .addUse(GR->getSPIRVTypeID(IntTy))
+      .addUse(GR->getSPIRVTypeID(SpvDesiredTy))
       .addUse(ObjectPtr)
       .addUse(ScopeReg)
       .addUse(MemSemEqualReg)
@@ -2619,6 +2618,7 @@ static bool generateConvertInst(const StringRef DemangledCall,
                               GR->getSPIRVTypeID(Call->ReturnType));
   }
 
+  assert(Builtin && "Conversion builtin not found.");
   if (Builtin->IsSaturated)
     buildOpDecorate(Call->ReturnRegister, MIRBuilder,
                     SPIRV::Decoration::SaturatedConversion, {});
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
index d9265f498973e..e7da5504b2d58 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp
@@ -12,7 +12,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "SPIRVCommandLine.h"
-#include "llvm/ADT/StringRef.h"
+#include "MCTargetDesc/SPIRVBaseInfo.h"
+#include "llvm/TargetParser/Triple.h"
 #include <algorithm>
 #include <map>
 
@@ -32,6 +33,24 @@ static const std::map<std::string, SPIRV::Extension::Extension, std::less<>>
          SPIRV::Extension::Extension::SPV_EXT_arithmetic_fence},
         {"SPV_EXT_demote_to_helper_invocation",
          SPIRV::Extension::Extension::SPV_EXT_demote_to_helper_invocation},
+        {"SPV_EXT_descriptor_indexing",
+         SPIRV::Extension::Extension::SPV_EXT_descriptor_indexing},
+        {"SPV_EXT_fragment_fully_covered",
+         SPIRV::Extension::Extension::SPV_EXT_fragment_fully_covered},
+        {"SPV_EXT_fragment_invocation_density",
+         SPIRV::Extension::Extension::SPV_EXT_fragment_invocation_density},
+        {"SPV_EXT_fragment_shader_interlock",
+         SPIRV::Extension::Extension::SPV_EXT_fragment_shader_interlock},
+        {"SPV_EXT_mesh_shader",
+         SPIRV::Extension::Extension::SPV_EXT_mesh_shader},
+        {"SPV_EXT_shader_stencil_export",
+         SPIRV::Extension::Extension::SPV_EXT_shader_stencil_export},
+        {"SPV_EXT_shader_viewport_index_layer",
+         SPIRV::Extension::Extension::SPV_EXT_shader_viewport_index_layer},
+        {"SPV_GOOGLE_hlsl_functionality1",
+         SPIRV::Extension::Extension::SPV_GOOGLE_hlsl_functionality1},
+        {"SPV_GOOGLE_user_type",
+         SPIRV::Extension::Extension::SPV_GOOGLE_user_type},
         {"SPV_INTEL_arbitrary_precision_integers",
          SPIRV::Extension::Extension::SPV_INTEL_arbitrary_precision_integers},
         {"SPV_INTEL_cache_controls",
@@ -57,6 +76,19 @@ static const std::map<std::string, SPIRV::Extension::Extension, std::less<>>
          SPIRV::Extension::Extension::SPV_INTEL_memory_access_aliasing},
         {"SPV_INTEL_joint_matrix",
          SPIRV::Extension::Extension::SPV_INTEL_joint_matrix},
+        {"SPV_KHR_16bit_storage",
+         SPIRV::Extension::Extension::SPV_KHR_16bit_storage},
+        {"SPV_KHR_device_group",
+         SPIRV::Extension::Extension::SPV_KHR_device_group},
+        {"SPV_KHR_fragment_shading_rate",
+         SPIRV::Extension::Extension::SPV_KHR_fragment_shading_rate},
+        {"SPV_KHR_multiview", SPIRV::Extension::Extension::SPV_KHR_multiview},
+        {"SPV_KHR_post_depth_coverage",
+         SPIRV::Extension::Extension::SPV_KHR_post_depth_coverage},
+        {"SPV_KHR_shader_draw_parameters",
+         SPIRV::Extension::Extension::SPV_KHR_shader_draw_parameters},
+        {"SPV_KHR_ray_tracing",
+         SPIRV::Extension::Extension::SPV_KHR_ray_tracing},
         {"SPV_KHR_uniform_group_instructions",
          SPIRV::Extension::Extension::SPV_KHR_uniform_group_instructions},
         {"SPV_KHR_no_integer_wrap_decoration",
@@ -89,6 +121,17 @@ static const std::map<std::string, SPIRV::Extension::Extension, std::less<>>
          SPIRV::Extension::Extension::SPV_KHR_cooperative_matrix},
         {"SPV_KHR_non_semantic_info",
          SPIRV::Extension::Extension::SPV_KHR_non_semantic_info},
+        {"SPV_KHR_ray_query", SPIRV::Extension::Extension::SPV_KHR_ray_query},
+        {"SPV_EXT_shader_image_int64",
+         SPIRV::Extension::Extension::SPV_EXT_shader_image_int64},
+        {"SPV_KHR_fragment_shader_barycentric",
+         SPIRV::Extension::Extension::SPV_KHR_fragment_shader_barycentric},
+        {"SPV_KHR_physical_storage_buffer",
+         SPIRV::Extension::Extension::SPV_KHR_physical_storage_buffer},
+        {"SPV_KHR_vulkan_memory_model",
+         SPIRV::Extension::Extension::SPV_KHR_vulkan_memory_model},
+        {"SPV_NV_shader_subgroup_partitioned",
+         SPIRV::Extension::Extension::SPV_NV_shader_subgroup_partitioned},
         {"SPV_INTEL_long_composites",
          SPIRV::Extension::Extension::SPV_INTEL_long_composites},
         {"SPV_INTEL_fp_max_error",
@@ -171,3 +214,23 @@ StringRef SPIRVExtensionsParser::checkExtensions(
   }
   return StringRef();
 }
+
+std::set<SPIRV::Extension::Extension>
+SPIRVExtensionsParser::getValidExtensions(const Triple &TT) {
+  std::set<SPIRV::Extension::Extension> R;
+  SPIRV::Environment::Environment CurrentEnvironment =
+      SPIRV::Environment::Environment::EnvOpenCL;
+  if (TT.getOS() == Triple::Vulkan)
+    CurrentEnvironment = SPIRV::Environment::Environment::EnvVulkan;
+
+  for (const auto &[ExtensionName, ExtensionEnum] : SPIRVExtensionMap) {
+    EnvironmentList AllowedEnv = getSymbolicOperandAllowedEnvironments(
+        SPIRV::OperandCategory::OperandCategory::ExtensionOperand,
+        ExtensionEnum);
+
+    if (llvm::is_contained(AllowedEnv, CurrentEnvironment))
+      R.insert(ExtensionEnum);
+  }
+
+  return R;
+}
diff --git a/llvm/lib/Target/SPIRV/SPIRVCommandLine.h b/llvm/lib/Target/SPIRV/SPIRVCommandLine.h
index 3e3b22bde8603..02e847b322a77 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCommandLine.h
+++ b/llvm/lib/Target/SPIRV/SPIRVCommandLine.h
@@ -21,6 +21,7 @@
 
 namespace llvm {
 class StringRef;
+class Triple;
 
 /// Command line parser for toggling SPIR-V extensions.
 struct SPIRVExtensionsParser
@@ -42,6 +43,11 @@ struct SPIRVExtensionsParser
   static StringRef
   checkExtensions(const std::vector<std::string> &ExtNames,
                   std::set<SPIRV::Extension::Extension> &AllowedExtensions);
+
+  /// Returns the list of extensions that are valid for a particular
+  /// target environment (i.e., OpenCL or Vulkan).
+  static std::set<SPIRV::Extension::Extension>
+  getValidExtensions(const Triple &TT);
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index 2c3e0876b757d..f5a49e2b47363 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -499,7 +499,7 @@ void SPIRVEmitIntrinsics::propagateElemTypeRec(
   std::unordered_set<Value *> Visited;
   DenseMap<Function *, CallInst *> Ptrcasts;
   propagateElemTypeRec(Op, PtrElemTy, CastElemTy, VisitedSubst, Visited,
-                       Ptrcasts);
+                       std::move(Ptrcasts));
 }
 
 void SPIRVEmitIntrinsics::propagateElemTypeRec(
@@ -897,17 +897,16 @@ Type *SPIRVEmitIntrinsics::deduceNestedTypeHelper(
     bool Change = false;
     for (unsigned i = 0; i < U->getNumOperands(); ++i) {
       Value *Op = U->getOperand(i);
+      assert(Op && "Operands should not be null.");
       Type *OpTy = Op->getType();
       Type *Ty = OpTy;
-      if (Op) {
-        if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
-          if (Type *NestedTy =
-                  deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
-            Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
-        } else {
-          Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
-                                      UnknownElemTypeI8);
-        }
+      if (auto *PtrTy = dyn_cast<PointerType>(OpTy)) {
+        if (Type *NestedTy =
+                deduceElementTypeHelper(Op, Visited, UnknownElemTypeI8))
+          Ty = getTypedPointerWrapper(NestedTy, PtrTy->getAddressSpace());
+      } else {
+        Ty = deduceNestedTypeHelper(dyn_cast<User>(Op), OpTy, Visited,
+                                    UnknownElemTypeI8);
       }
       Tys.push_back(Ty);
       Change |= Ty != OpTy;
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
index 7f0d636577869..275463ef4c527 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitNonSemanticDI.cpp
@@ -116,6 +116,7 @@ bool SPIRVEmitNonSemanticDI::emitGlobalDI(MachineFunction &MF) {
       }
     }
     const NamedMDNode *ModuleFlags = M->getNamedMetadata("llvm.module.flags");
+    assert(ModuleFlags && "Expected llvm.module.flags metadata to be present");
     for (const auto *Op : ModuleFlags->operands()) {
       const MDOperand &MaybeStrOp = Op->getOperand(1);
       if (MaybeStrOp.equalsStr("Dwarf Version"))
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index f1436d5b3c047..cfe24c84941a9 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -87,7 +87,7 @@ storageClassRequiresExplictLayout(SPIRV::StorageClass::StorageClass SC) {
 }
 
 SPIRVGlobalRegistry::SPIRVGlobalRegistry(unsigned PointerSize)
-    : PointerSize(PointerSize), Bound(0) {}
+    : PointerSize(PointerSize), Bound(0), CurMF(nullptr) {}
 
 SPIRVType *SPIRVGlobalRegistry::assignIntTypeToVReg(unsigned BitWidth,
                                                     Register VReg,
@@ -474,8 +474,8 @@ Register SPIRVGlobalRegistry::getOrCreateBaseRegister(
   }
   if (Type->getOpcode() == SPIRV::OpTypeFloat) {
     SPIRVType *SpvBaseType = getOrCreateSPIRVFloatType(BitWidth, I, TII);
-    return getOrCreateConstFP(dyn_cast<ConstantFP>(Val)->getValue(), I,
-                              SpvBaseType, TII, ZeroAsNull);
+    return getOrCreateConstFP(cast<ConstantFP>(Val)->getValue(), I, SpvBaseType,
+                              TII, ZeroAsNull);
   }
   assert(Type->getOpcode() == SPIRV::OpTypeInt);
   SPIRVType *SpvBaseType = getOrCreateSPIRVIntegerType(BitWidth, I, TII);
@@ -1069,7 +1069,8 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType(
                                    MIRBuilder);
       };
     }
-    return getOpTypeStruct(SType, MIRBuilder, AccQual, Decorator, EmitIR);
+    return getOpTypeStruct(SType, MIRBuilder, AccQual, std::move(Decorator),
+                           EmitIR);
   }
   if (auto FType = dyn_cast<FunctionType>(Ty)) {
     SPIRVType *RetTy = findSPIRVType(FType->getReturnType(), MIRBuilder,
@@ -1406,8 +1407,9 @@ SPIRVType *SPIRVGlobalRegistry::getOrCreateLayoutType(
   // We need a new OpTypeStruct instruction because decorations will be
   // different from a struct with an explicit layout created from a different
   // entry point.
-  SPIRVType *SPIRVStructType = getOpTypeStruct(
-      ST, MIRBuilder, SPIRV::AccessQualifier::None, Decorator, EmitIr);
+  SPIRVType *SPIRVStructType =
+      getOpTypeStruct(ST, MIRBuilder, SPIRV::AccessQualifier::None,
+                      std::move(Decorator), EmitIr);
   add(Key, SPIRVStructType);
   return SPIRVStructType;
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index e9f5ffa23e220..98c7709acf938 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -220,8 +220,10 @@ class SPIRVInstructionSelector : public InstructionSelector {
   bool selectConst(Register ResVReg, const SPIRVType *ResType,
                    MachineInstr &I) const;
 
-  bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
-                    bool IsSigned) const;
+  bool selectSelect(Register ResVReg, const SPIRVType *ResType,
+                    MachineInstr &I) const;
+  bool selectSelectDefaultArgs(Register ResVReg, const SPIRVType *ResType,
+                               MachineInstr &I, bool IsSigned) const;
   bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
                   bool IsSigned, unsigned Opcode) const;
   bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
@@ -362,6 +364,7 @@ SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
                                                    const RegisterBankInfo &RBI)
     : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
       TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
+      MRI(nullptr),
 #define GET_GLOBALISEL_PREDICATES_INIT
 #include "SPIRVGenGlobalISel.inc"
 #undef GET_GLOBALISEL_PREDICATES_INIT
@@ -509,7 +512,18 @@ bool SPIRVInstructionSelector::select(MachineInstr &I) {
       if (isTypeFoldingSupported(Def->getOpcode()) &&
           Def->getOpcode() != TargetOpcode::G_CONSTANT &&
           Def->getOpcode() != TargetOpcode::G_FCONSTANT) {
-        bool Res = selectImpl(I, *CoverageInfo);
+        bool Res = false;
+        if (Def->getOpcode() == TargetOpcode::G_SELECT) {
+          Register SelectDstReg = Def->getOperand(0).getReg();
+          Res = selectSelect(SelectDstReg, GR.getSPIRVTypeForVReg(SelectDstReg),
+                             *Def);
+          GR.invalidateMachineInstr(Def);
+          Def->removeFromParent();
+          MRI->replaceRegWith(DstReg, SelectDstReg);
+          GR.invalidateMachineInstr(&I);
+          I.removeFromParent();
+        } else
+          Res = selectImpl(I, *CoverageInfo);
         LLVM_DEBUG({
           if (!Res && Def->getOpcode() != TargetOpcode::G_CONSTANT) {
             dbgs() << "Unexpected pattern in ASSIGN_TYPE.\nInstruction: ";
@@ -2564,8 +2578,52 @@ Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
 
 bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
                                             const SPIRVType *ResType,
-                                            MachineInstr &I,
-                                            bool IsSigned) const {
+                                            MachineInstr &I) const {
+  Register SelectFirstArg = I.getOperand(2).getReg();
+  Register SelectSecondArg = I.getOperand(3).getReg();
+  assert(ResType == GR.getSPIRVTypeForVReg(SelectFirstArg) &&
+         ResType == GR.getSPIRVTypeForVReg(SelectSecondArg));
+
+  bool IsFloatTy =
+      GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypeFloat);
+  bool IsPtrTy =
+      GR.isScalarOrVectorOfType(SelectFirstArg, SPIRV::OpTypePointer);
+  bool IsVectorTy = GR.getSPIRVTypeForVReg(SelectFirstArg)->getOpcode() ==
+                    SPIRV::OpTypeVector;
+
+  bool IsScalarBool =
+      GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
+  unsigned Opcode;
+  if (IsVectorTy) {
+    if (IsFloatTy) {
+      Opcode = IsScalarBool ? SPIRV::OpSelectVFSCond : SPIRV::OpSelectVFVCond;
+    } else if (IsPtrTy) {
+      Opcode = IsScalarBool ? SPIRV::OpSelectVPSCond : SPIRV::OpSelectVPVCond;
+    } else {
+      Opcode = IsScalarBool ? SPIRV::OpSelectVISCond : SPIRV::OpSelectVIVCond;
+    }
+  } else {
+    if (IsFloatTy) {
+      Opcode = IsScalarBool ? SPIRV::OpSelectSFSCond : SPIRV::OpSelectVFVCond;
+    } else if (IsPtrTy) {
+      Opcode = IsScalarBool ? SPIRV::OpSelectSPSCond : SPIRV::OpSelectVPVCond;
+    } else {
+      Opcode = IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectVIVCond;
+    }
+  }
+  return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
+      .addDef(ResVReg)
+      .addUse(GR.getSPIRVTypeID(ResType))
+      .addUse(I.getOperand(1).getReg())
+      .addUse(SelectFirstArg)
+      .addUse(SelectSecondArg)
+      .constrainAllUses(TII, TRI, RBI);
+}
+
+bool SPIRVInstructionSelector::selectSelectDefaultArgs(Register ResVReg,
+                                                       const SPIRVType *ResType,
+                                                       MachineInstr &I,
+                                                       bool IsSigned) const {
   // To extend a bool, we need to use OpSelect between constants.
   Register ZeroReg = buildZerosVal(ResType, I);
   Register OneReg = buildOnesVal(IsSigned, ResType, I);
@@ -2597,7 +2655,7 @@ bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
       TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
     }
     SrcReg = createVirtualRegister(TmpType, &GR, MRI, MRI->getMF());
-    selectSelect(SrcReg, TmpType, I, false);
+    selectSelectDefaultArgs(SrcReg, TmpType, I, false);
   }
   return selectOpWithSrcs(ResVReg, ResType, I, {SrcReg}, Opcode);
 }
@@ -2607,7 +2665,7 @@ bool SPIRVInstructionSelector::selectExt(Register ResVReg,
                                          MachineInstr &I, bool IsSigned) const {
   Register SrcReg = I.getOperand(1).getReg();
   if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool))
-    return selectSelect(ResVReg, ResType, I, IsSigned);
+    return selectSelectDefaultArgs(ResVReg, ResType, I, IsSigned);
 
   SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg);
   if (SrcType == ResType)
@@ -3574,7 +3632,7 @@ bool SPIRVInstructionSelector::selectFirstBitSet64Overflow(
 
   // Join all the resulting registers back into the return type in order
   // (ie i32x2, i32x2, i32x1 -> i32x5)
-  return selectOpWithSrcs(ResVReg, ResType, I, PartialRegs,
+  return selectOpWithSrcs(ResVReg, ResType, I, std::move(PartialRegs),
                           SPIRV::OpCompositeConstruct);
 }
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVLegalizeImplicitBinding.cpp b/llvm/lib/Target/SPIRV/SPIRVLegalizeImplicitBinding.cpp
new file mode 100644
index 0000000000000..aea3397ad2fd6
--- /dev/null
+++ b/llvm/lib/Target/SPIRV/SPIRVLegalizeImplicitBinding.cpp
@@ -0,0 +1,158 @@
+//===- SPIRVLegalizeImplicitBinding.cpp - Legalize implicit bindings ----*- C++
+//-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass legalizes the @llvm.spv.resource.handlefromimplicitbinding
+// intrinsic by replacing it with a call to
+// @llvm.spv.resource.handlefrombinding.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SPIRV.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsSPIRV.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Pass.h"
+#include <algorithm>
+#include <vector>
+
+using namespace llvm;
+
+namespace {
+class SPIRVLegalizeImplicitBinding : public ModulePass {
+public:
+  static char ID;
+  SPIRVLegalizeImplicitBinding() : ModulePass(ID) {}
+
+  bool runOnModule(Module &M) override;
+
+private:
+  void collectBindingInfo(Module &M);
+  uint32_t getAndReserveFirstUnusedBinding(uint32_t DescSet);
+  void replaceImplicitBindingCalls(Module &M);
+
+  // A map from descriptor set to a bit vector of used binding numbers.
+  std::vector<BitVector> UsedBindings;
+  // A list of all implicit binding calls, to be sorted by order ID.
+  SmallVector<CallInst *, 16> ImplicitBindingCalls;
+};
+
+struct BindingInfoCollector : public InstVisitor<BindingInfoCollector> {
+  std::vector<BitVector> &UsedBindings;
+  SmallVector<CallInst *, 16> &ImplicitBindingCalls;
+
+  BindingInfoCollector(std::vector<BitVector> &UsedBindings,
+                       SmallVector<CallInst *, 16> &ImplicitBindingCalls)
+      : UsedBindings(UsedBindings), ImplicitBindingCalls(ImplicitBindingCalls) {
+  }
+
+  void visitCallInst(CallInst &CI) {
+    if (CI.getIntrinsicID() == Intrinsic::spv_resource_handlefrombinding) {
+      const uint32_t DescSet =
+          cast<ConstantInt>(CI.getArgOperand(0))->getZExtValue();
+      const uint32_t Binding =
+          cast<ConstantInt>(CI.getArgOperand(1))->getZExtValue();
+
+      if (UsedBindings.size() <= DescSet) {
+        UsedBindings.resize(DescSet + 1);
+        UsedBindings[DescSet].resize(64);
+      }
+      if (UsedBindings[DescSet].size() <= Binding) {
+        UsedBindings[DescSet].resize(2 * Binding + 1);
+      }
+      UsedBindings[DescSet].set(Binding);
+    } else if (CI.getIntrinsicID() ==
+               Intrinsic::spv_resource_handlefromimplicitbinding) {
+      ImplicitBindingCalls.push_back(&CI);
+    }
+  }
+};
+
+void SPIRVLegalizeImplicitBinding::collectBindingInfo(Module &M) {
+  BindingInfoCollector InfoCollector(UsedBindings, ImplicitBindingCalls);
+  InfoCollector.visit(M);
+
+  // Sort the collected calls by their order ID.
+  std::sort(
+      ImplicitBindingCalls.begin(), ImplicitBindingCalls.end(),
+      [](const CallInst *A, const CallInst *B) {
+        const uint32_t OrderIdArgIdx = 0;
+        const uint32_t OrderA =
+            cast<ConstantInt>(A->getArgOperand(OrderIdArgIdx))->getZExtValue();
+        const uint32_t OrderB =
+            cast<ConstantInt>(B->getArgOperand(OrderIdArgIdx))->getZExtValue();
+        return OrderA < OrderB;
+      });
+}
+
+uint32_t SPIRVLegalizeImplicitBinding::getAndReserveFirstUnusedBinding(
+    uint32_t DescSet) {
+  if (UsedBindings.size() <= DescSet) {
+    UsedBindings.resize(DescSet + 1);
+    UsedBindings[DescSet].resize(64);
+  }
+
+  int NewBinding = UsedBindings[DescSet].find_first_unset();
+  if (NewBinding == -1) {
+    NewBinding = UsedBindings[DescSet].size();
+    UsedBindings[DescSet].resize(2 * NewBinding + 1);
+  }
+
+  UsedBindings[DescSet].set(NewBinding);
+  return NewBinding;
+}
+
+void SPIRVLegalizeImplicitBinding::replaceImplicitBindingCalls(Module &M) {
+  for (CallInst *OldCI : ImplicitBindingCalls) {
+    IRBuilder<> Builder(OldCI);
+    const uint32_t DescSet =
+        cast<ConstantInt>(OldCI->getArgOperand(1))->getZExtValue();
+    const uint32_t NewBinding = getAndReserveFirstUnusedBinding(DescSet);
+
+    SmallVector<Value *, 8> Args;
+    Args.push_back(Builder.getInt32(DescSet));
+    Args.push_back(Builder.getInt32(NewBinding));
+
+    // Copy the remaining arguments from the old call.
+    for (uint32_t i = 2; i < OldCI->arg_size(); ++i) {
+      Args.push_back(OldCI->getArgOperand(i));
+    }
+
+    Function *NewFunc = Intrinsic::getOrInsertDeclaration(
+        &M, Intrinsic::spv_resource_handlefrombinding, OldCI->getType());
+    CallInst *NewCI = Builder.CreateCall(NewFunc, Args);
+    NewCI->setCallingConv(OldCI->getCallingConv());
+
+    OldCI->replaceAllUsesWith(NewCI);
+    OldCI->eraseFromParent();
+  }
+}
+
+bool SPIRVLegalizeImplicitBinding::runOnModule(Module &M) {
+  collectBindingInfo(M);
+  if (ImplicitBindingCalls.empty()) {
+    return false;
+  }
+
+  replaceImplicitBindingCalls(M);
+  return true;
+}
+} // namespace
+
+char SPIRVLegalizeImplicitBinding::ID = 0;
+
+INITIALIZE_PASS(SPIRVLegalizeImplicitBinding, "legalize-spirv-implicit-binding",
+                "Legalize SPIR-V implicit bindings", false, false)
+
+ModulePass *llvm::createSPIRVLegalizeImplicitBindingPass() {
+  return new SPIRVLegalizeImplicitBinding();
+}
\ No newline at end of file
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
index ab06fc0b5ff31..8039cf0c432fa 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp
@@ -93,7 +93,7 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
       if (Reqs.isCapabilityAvailable(Cap)) {
         ReqExts.append(getSymbolicOperandExtensions(
             SPIRV::OperandCategory::CapabilityOperand, Cap));
-        return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer};
+        return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
       }
     } else {
       // By SPIR-V specification: "If an instruction, enumerant, or other
@@ -111,7 +111,7 @@ getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category,
         if (i == Sz - 1 || !AvoidCaps.S.contains(Cap)) {
           ReqExts.append(getSymbolicOperandExtensions(
               SPIRV::OperandCategory::CapabilityOperand, Cap));
-          return {true, {Cap}, ReqExts, ReqMinVer, ReqMaxVer};
+          return {true, {Cap}, std::move(ReqExts), ReqMinVer, ReqMaxVer};
         }
       }
     }
@@ -558,7 +558,7 @@ static void collectOtherInstr(MachineInstr &MI, SPIRV::ModuleAnalysisInfo &MAI,
                               bool Append = true) {
   MAI.setSkipEmission(&MI);
   InstrSignature MISign = instrToSignature(MI, MAI, true);
-  auto FoundMI = IS.insert(MISign);
+  auto FoundMI = IS.insert(std::move(MISign));
   if (!FoundMI.second)
     return; // insert failed, so we found a duplicate; don't add it to MAI.MS
   // No duplicates, so add it.
diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
index a0d47cb052b42..41c792a98534f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
+++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h
@@ -54,8 +54,8 @@ struct Requirements {
                std::optional<Capability::Capability> Cap = {},
                ExtensionList Exts = {}, VersionTuple MinVer = VersionTuple(),
                VersionTuple MaxVer = VersionTuple())
-      : IsSatisfiable(IsSatisfiable), Cap(Cap), Exts(Exts), MinVer(MinVer),
-        MaxVer(MaxVer) {}
+      : IsSatisfiable(IsSatisfiable), Cap(Cap), Exts(std::move(Exts)),
+        MinVer(MinVer), MaxVer(MaxVer) {}
   Requirements(Capability::Capability Cap) : Requirements(true, {Cap}) {}
 };
 
@@ -217,7 +217,8 @@ struct SPIRVModuleAnalysis : public ModulePass {
   static char ID;
 
 public:
-  SPIRVModuleAnalysis() : ModulePass(ID) {}
+  SPIRVModuleAnalysis()
+      : ModulePass(ID), ST(nullptr), GR(nullptr), TII(nullptr), MMI(nullptr) {}
 
   bool runOnModule(Module &M) override;
   void getAnalysisUsage(AnalysisUsage &AU) const override;
diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
index 1d38244feeae7..d17528dd882bf 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
@@ -147,7 +147,7 @@ void visit(MachineFunction &MF, MachineBasicBlock &Start,
 // Do a preorder traversal of the CFG starting from the given function's entry
 // point. Calls |op| on each basic block encountered during the traversal.
 void visit(MachineFunction &MF, std::function<void(MachineBasicBlock *)> op) {
-  visit(MF, *MF.begin(), op);
+  visit(MF, *MF.begin(), std::move(op));
 }
 
 bool SPIRVPostLegalizer::runOnMachineFunction(MachineFunction &MF) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index f4b4846f70d7d..1a08c6ac0dcaf 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -99,6 +99,7 @@ addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR,
               SPIRVType *ExtType = GR->getOrCreateSPIRVType(
                   Const->getType(), MIB, SPIRV::AccessQualifier::ReadWrite,
                   true);
+              assert(SrcMI && "Expected source instruction to be valid");
               SrcMI->setDesc(STI.getInstrInfo()->get(SPIRV::OpConstantNull));
               SrcMI->addOperand(MachineOperand::CreateReg(
                   GR->getSPIRVTypeID(ExtType), false));
@@ -440,13 +441,10 @@ void insertAssignInstr(Register Reg, Type *Ty, SPIRVType *SpvType,
   // Tablegen definition assumes SPIRV::ASSIGN_TYPE pseudo-instruction is
   // present after each auto-folded instruction to take a type reference from.
   Register NewReg = MRI.createGenericVirtualRegister(MRI.getType(Reg));
-  if (auto *RC = MRI.getRegClassOrNull(Reg)) {
-    MRI.setRegClass(NewReg, RC);
-  } else {
-    auto RegClass = GR->getRegClass(SpvType);
-    MRI.setRegClass(NewReg, RegClass);
-    MRI.setRegClass(Reg, RegClass);
-  }
+  const auto *RegClass = GR->getRegClass(SpvType);
+  MRI.setRegClass(NewReg, RegClass);
+  MRI.setRegClass(Reg, RegClass);
+
   GR->assignSPIRVTypeToVReg(SpvType, Reg, MIB.getMF());
   // This is to make it convenient for Legalizer to get the SPIRVType
   // when processing the actual MI (i.e. not pseudo one).
diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
index 595424b999439..2b34f61fa2434 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp
@@ -234,7 +234,7 @@ static SmallVector<Metadata *> parseAnnotation(Value *I,
       return SmallVector<Metadata *>{};
     MDs.push_back(MDNode::get(Ctx, MDsItem));
   }
-  return Pos == static_cast<int>(Anno.length()) ? MDs
+  return Pos == static_cast<int>(Anno.length()) ? std::move(MDs)
                                                 : SmallVector<Metadata *>{};
 }
 
@@ -359,18 +359,15 @@ static void lowerExpectAssume(IntrinsicInst *II) {
   }
 }
 
-static bool toSpvOverloadedIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID,
-                                     ArrayRef<unsigned> OpNos) {
-  Function *F = nullptr;
-  if (OpNos.empty()) {
-    F = Intrinsic::getOrInsertDeclaration(II->getModule(), NewID);
-  } else {
-    SmallVector<Type *, 4> Tys;
-    for (unsigned OpNo : OpNos)
-      Tys.push_back(II->getOperand(OpNo)->getType());
-    F = Intrinsic::getOrInsertDeclaration(II->getModule(), NewID, Tys);
-  }
-  II->setCalledFunction(F);
+static bool toSpvLifetimeIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID) {
+  IRBuilder<> Builder(II);
+  auto *Alloca = cast<AllocaInst>(II->getArgOperand(0));
+  std::optional<TypeSize> Size =
+      Alloca->getAllocationSize(Alloca->getDataLayout());
+  Value *SizeVal = Builder.getInt64(Size ? *Size : -1);
+  Builder.CreateIntrinsic(NewID, Alloca->getType(),
+                          {SizeVal, II->getArgOperand(0)});
+  II->eraseFromParent();
   return true;
 }
 
@@ -406,8 +403,8 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
         break;
       case Intrinsic::lifetime_start:
         if (!STI.isShader()) {
-          Changed |= toSpvOverloadedIntrinsic(
-              II, Intrinsic::SPVIntrinsics::spv_lifetime_start, {1});
+          Changed |= toSpvLifetimeIntrinsic(
+              II, Intrinsic::SPVIntrinsics::spv_lifetime_start);
         } else {
           II->eraseFromParent();
           Changed = true;
@@ -415,8 +412,8 @@ bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
         break;
       case Intrinsic::lifetime_end:
         if (!STI.isShader()) {
-          Changed |= toSpvOverloadedIntrinsic(
-              II, Intrinsic::SPVIntrinsics::spv_lifetime_end, {1});
+          Changed |= toSpvLifetimeIntrinsic(
+              II, Intrinsic::SPVIntrinsics::spv_lifetime_end);
         } else {
           II->eraseFromParent();
           Changed = true;
diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
index cdf3c6224d4c8..690493fb426bc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp
@@ -166,7 +166,13 @@ void SPIRVSubtarget::initAvailableExtInstSets() {
 void SPIRVSubtarget::initAvailableExtensions(
     const std::set<SPIRV::Extension::Extension> &AllowedExtIds) {
   AvailableExtensions.clear();
-  AvailableExtensions.insert_range(AllowedExtIds);
+  const std::set<SPIRV::Extension::Extension> &ValidExtensions =
+      SPIRVExtensionsParser::getValidExtensions(TargetTriple);
+
+  for (const auto &Ext : AllowedExtIds) {
+    if (ValidExtensions.count(Ext))
+      AvailableExtensions.insert(Ext);
+  }
 
   accountForAMDShaderTrinaryMinmax();
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
index 614e83ae9b286..d2824ee2d2caf 100644
--- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
+++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td
@@ -109,23 +109,59 @@ def CapabilityEntries : GenericTable {
   let PrimaryKeyName = "lookupCapabilityByCategoryAndValue";
 }
 
+//===----------------------------------------------------------------------===//
+// Lookup table for matching symbolic operands (category + 32-bit value) to
+// SPIR-V environments. If an operand is allows in more than one environment,
+// there will be multiple consecutive entries present in the table.
+//===----------------------------------------------------------------------===//
+
+// Forward-declare classes used in ExtensionEntry
+class Environment;
+
+class EnvironmentEntry<OperandCategory category, bits<32> value,
+                       Environment allowedEnvironment> {
+  OperandCategory Category = category;
+  bits<32> Value = value;
+  Environment AllowedEnvironment = allowedEnvironment;
+}
+
+def EnvironmentEntries : GenericTable {
+  let FilterClass = "EnvironmentEntry";
+  let Fields = ["Category", "Value", "AllowedEnvironment"];
+  string TypeOf_Category = "OperandCategory";
+  string TypeOf_AllowedEnvironment = "Environment";
+  let PrimaryKey = ["Category", "Value"];
+  // Function for looking up a (the first) environment by category + value. Next
+  // environment should be consecutive.
+  let PrimaryKeyName = "lookupEnvironmentByCategoryAndValue";
+}
+
 //===----------------------------------------------------------------------===//
 // Multiclass used to define a SymbolicOperand and at the same time declare
 // required extension and capabilities.
 //===----------------------------------------------------------------------===//
 
-multiclass SymbolicOperandWithRequirements<OperandCategory category, bits<32> value, string mnemonic, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
-    assert !ge(!size(mnemonic), 1), "No mnemonic/string representation provided for symbolic operand with value " # value;
-    def : SymbolicOperand<category, value, mnemonic, minVersion, maxVersion>;
+multiclass SymbolicOperandWithRequirements<
+    OperandCategory category, bits<32> value, string mnemonic,
+    bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions,
+    list<Capability> reqCapabilities, list<Environment> allowedEnvironments> {
+  assert !ge(!size(mnemonic), 1), "No mnemonic/string representation provided "
+                                  "for symbolic operand with value "#value;
+  def : SymbolicOperand<category, value, mnemonic, minVersion, maxVersion>;
+
+  assert !le(!size(reqExtensions), 1),
+      "Too many required extensions for a symbolic/named operand: "#mnemonic;
+  if !eq(!size(reqExtensions), 1) then {
+    def : ExtensionEntry<category, value, reqExtensions[0]>;
+  }
 
-    assert !le(!size(reqExtensions), 1), "Too many required extensions for a symbolic/named operand: " # mnemonic;
-    if !eq(!size(reqExtensions), 1) then {
-        def : ExtensionEntry<category, value, reqExtensions[0]>;
-    }
+  foreach capability = reqCapabilities in {
+    def : CapabilityEntry<category, value, capability>;
+  }
 
-    foreach capability = reqCapabilities in {
-        def : CapabilityEntry<category, value, capability>;
-    }
+  foreach environment = allowedEnvironments in {
+    def : EnvironmentEntry<category, value, environment>;
+  }
 }
 
 //===----------------------------------------------------------------------===//
@@ -175,6 +211,20 @@ def CooperativeMatrixOperandsOperand : OperandCategory;
 def SpecConstantOpOperandsOperand : OperandCategory;
 def MatrixMultiplyAccumulateOperandsOperand : OperandCategory;
 
+//===----------------------------------------------------------------------===//
+// Definition of the Environments
+//===----------------------------------------------------------------------===//
+
+def Environment : GenericEnum, Operand<i32> {
+  let FilterClass = "Environment";
+  let ValueField = "Value";
+}
+
+class Environment<bits<32> value> { bits<32> Value = value; }
+
+def EnvOpenCL : Environment<0>;
+def EnvVulkan : Environment<1>;
+
 //===----------------------------------------------------------------------===//
 // Multiclass used to define Extesions enum values and at the same time
 // SymbolicOperand entries.
@@ -192,135 +242,146 @@ class Extension<string name, bits<32> value> {
   bits<32> Value = value;
 }
 
-multiclass ExtensionOperand<bits<32> value> {
+multiclass ExtensionOperand<bits<32> value,
+                            list<Environment> allowedEnvironments> {
   def NAME : Extension<NAME, value>;
-  defm : SymbolicOperandWithRequirements<ExtensionOperand, value, NAME, 0, 0, [], []>;
-}
-
-defm SPV_AMD_shader_explicit_vertex_parameter : ExtensionOperand<1>;
-defm SPV_AMD_shader_trinary_minmax_extension : ExtensionOperand<2>;
-defm SPV_AMD_gcn_shader : ExtensionOperand<3>;
-defm SPV_KHR_shader_ballot : ExtensionOperand<4>;
-defm SPV_AMD_shader_ballot : ExtensionOperand<5>;
-defm SPV_AMD_gpu_shader_half_float : ExtensionOperand<6>;
-defm SPV_KHR_shader_draw_parameters : ExtensionOperand<7>;
-defm SPV_KHR_subgroup_vote : ExtensionOperand<8>;
-defm SPV_KHR_16bit_storage : ExtensionOperand<9>;
-defm SPV_KHR_device_group : ExtensionOperand<10>;
-defm SPV_KHR_multiview : ExtensionOperand<11>;
-defm SPV_NVX_multiview_per_view_attributes : ExtensionOperand<12>;
-defm SPV_NV_viewport_array2 : ExtensionOperand<13>;
-defm SPV_NV_stereo_view_rendering : ExtensionOperand<14>;
-defm SPV_NV_sample_mask_override_coverage : ExtensionOperand<15>;
-defm SPV_NV_geometry_shader_passthrough : ExtensionOperand<16>;
-defm SPV_AMD_texture_gather_bias_lod : ExtensionOperand<17>;
-defm SPV_KHR_storage_buffer_storage_class : ExtensionOperand<18>;
-defm SPV_KHR_variable_pointers : ExtensionOperand<19>;
-defm SPV_AMD_gpu_shader_int16 : ExtensionOperand<20>;
-defm SPV_KHR_post_depth_coverage : ExtensionOperand<21>;
-defm SPV_KHR_shader_atomic_counter_ops : ExtensionOperand<22>;
-defm SPV_EXT_shader_stencil_export : ExtensionOperand<23>;
-defm SPV_EXT_shader_viewport_index_layer : ExtensionOperand<24>;
-defm SPV_AMD_shader_image_load_store_lod : ExtensionOperand<25>;
-defm SPV_AMD_shader_fragment_mask : ExtensionOperand<26>;
-defm SPV_EXT_fragment_fully_covered : ExtensionOperand<27>;
-defm SPV_AMD_gpu_shader_half_float_fetch : ExtensionOperand<28>;
-defm SPV_GOOGLE_decorate_string : ExtensionOperand<29>;
-defm SPV_GOOGLE_hlsl_functionality1 : ExtensionOperand<30>;
-defm SPV_NV_shader_subgroup_partitioned : ExtensionOperand<31>;
-defm SPV_EXT_descriptor_indexing : ExtensionOperand<32>;
-defm SPV_KHR_8bit_storage : ExtensionOperand<33>;
-defm SPV_KHR_vulkan_memory_model : ExtensionOperand<34>;
-defm SPV_NV_ray_tracing : ExtensionOperand<35>;
-defm SPV_NV_compute_shader_derivatives : ExtensionOperand<36>;
-defm SPV_NV_fragment_shader_barycentric : ExtensionOperand<37>;
-defm SPV_NV_mesh_shader : ExtensionOperand<38>;
-defm SPV_NV_shader_image_footprint : ExtensionOperand<39>;
-defm SPV_NV_shading_rate : ExtensionOperand<40>;
-defm SPV_INTEL_subgroups : ExtensionOperand<41>;
-defm SPV_INTEL_media_block_io : ExtensionOperand<42>;
-defm SPV_EXT_fragment_invocation_density : ExtensionOperand<44>;
-defm SPV_KHR_no_integer_wrap_decoration : ExtensionOperand<45>;
-defm SPV_KHR_float_controls : ExtensionOperand<46>;
-defm SPV_EXT_physical_storage_buffer : ExtensionOperand<47>;
-defm SPV_INTEL_fpga_memory_attributes : ExtensionOperand<48>;
-defm SPV_NV_cooperative_matrix : ExtensionOperand<49>;
-defm SPV_INTEL_shader_integer_functions2 : ExtensionOperand<50>;
-defm SPV_INTEL_fpga_loop_controls : ExtensionOperand<51>;
-defm SPV_EXT_fragment_shader_interlock : ExtensionOperand<52>;
-defm SPV_NV_shader_sm_builtins : ExtensionOperand<53>;
-defm SPV_KHR_shader_clock : ExtensionOperand<54>;
-defm SPV_INTEL_unstructured_loop_controls : ExtensionOperand<55>;
-defm SPV_EXT_demote_to_helper_invocation : ExtensionOperand<56>;
-defm SPV_INTEL_fpga_reg : ExtensionOperand<57>;
-defm SPV_INTEL_blocking_pipes : ExtensionOperand<58>;
-defm SPV_GOOGLE_user_type : ExtensionOperand<59>;
-defm SPV_KHR_physical_storage_buffer : ExtensionOperand<60>;
-defm SPV_INTEL_kernel_attributes : ExtensionOperand<61>;
-defm SPV_KHR_non_semantic_info : ExtensionOperand<62>;
-defm SPV_INTEL_io_pipes : ExtensionOperand<63>;
-defm SPV_KHR_ray_tracing : ExtensionOperand<64>;
-defm SPV_KHR_ray_query : ExtensionOperand<65>;
-defm SPV_INTEL_fpga_memory_accesses : ExtensionOperand<66>;
-defm SPV_INTEL_arbitrary_precision_integers : ExtensionOperand<67>;
-defm SPV_EXT_shader_atomic_float_add : ExtensionOperand<68>;
-defm SPV_KHR_terminate_invocation : ExtensionOperand<69>;
-defm SPV_KHR_fragment_shading_rate : ExtensionOperand<70>;
-defm SPV_EXT_shader_image_int64 : ExtensionOperand<71>;
-defm SPV_INTEL_fp_fast_math_mode : ExtensionOperand<72>;
-defm SPV_INTEL_fpga_cluster_attributes : ExtensionOperand<73>;
-defm SPV_INTEL_loop_fuse : ExtensionOperand<74>;
-defm SPV_EXT_shader_atomic_float_min_max : ExtensionOperand<75>;
-defm SPV_KHR_workgroup_memory_explicit_layout : ExtensionOperand<76>;
-defm SPV_KHR_linkonce_odr : ExtensionOperand<77>;
-defm SPV_KHR_expect_assume : ExtensionOperand<78>;
-defm SPV_INTEL_fpga_dsp_control : ExtensionOperand<79>;
-defm SPV_NV_bindless_texture : ExtensionOperand<80>;
-defm SPV_INTEL_fpga_invocation_pipelining_attributes : ExtensionOperand<81>;
-defm SPV_KHR_subgroup_uniform_control_flow : ExtensionOperand<82>;
-defm SPV_HUAWEI_subpass_shading : ExtensionOperand<83>;
-defm SPV_KHR_integer_dot_product : ExtensionOperand<84>;
-defm SPV_EXT_shader_atomic_float16_add : ExtensionOperand<85>;
-defm SPV_INTEL_runtime_aligned : ExtensionOperand<86>;
-defm SPV_KHR_bit_instructions : ExtensionOperand<87>;
-defm SPV_NV_ray_tracing_motion_blur : ExtensionOperand<88>;
-defm SPV_KHR_uniform_group_instructions : ExtensionOperand<89>;
-defm SPV_KHR_subgroup_rotate : ExtensionOperand<90>;
-defm SPV_INTEL_split_barrier : ExtensionOperand<91>;
-defm SPV_KHR_ray_cull_mask : ExtensionOperand<92>;
-defm SPV_KHR_fragment_shader_barycentric : ExtensionOperand<93>;
-defm SPV_EXT_relaxed_printf_string_address_space : ExtensionOperand<94>;
-defm SPV_EXT_ycbcr_attachments : ExtensionOperand<95>;
-defm SPV_EXT_mesh_shader : ExtensionOperand<96>;
-defm SPV_ARM_core_builtins : ExtensionOperand<97>;
-defm SPV_EXT_opacity_micromap : ExtensionOperand<98>;
-defm SPV_NV_shader_invocation_reorder : ExtensionOperand<99>;
-defm SPV_INTEL_usm_storage_classes : ExtensionOperand<100>;
-defm SPV_INTEL_fpga_latency_control : ExtensionOperand<101>;
-defm SPV_INTEL_fpga_argument_interfaces : ExtensionOperand<102>;
-defm SPV_INTEL_optnone : ExtensionOperand<103>;
-defm SPV_INTEL_function_pointers : ExtensionOperand<104>;
-defm SPV_INTEL_variable_length_array : ExtensionOperand<105>;
-defm SPV_INTEL_bfloat16_conversion : ExtensionOperand<106>;
-defm SPV_INTEL_inline_assembly : ExtensionOperand<107>;
-defm SPV_INTEL_cache_controls : ExtensionOperand<108>;
-defm SPV_INTEL_global_variable_host_access : ExtensionOperand<109>;
-defm SPV_INTEL_global_variable_fpga_decorations : ExtensionOperand<110>;
-defm SPV_KHR_cooperative_matrix : ExtensionOperand<111>;
-defm SPV_EXT_arithmetic_fence : ExtensionOperand<112>;
-defm SPV_EXT_optnone : ExtensionOperand<113>;
-defm SPV_INTEL_joint_matrix : ExtensionOperand<114>;
-defm SPV_INTEL_float_controls2 : ExtensionOperand<115>;
-defm SPV_INTEL_bindless_images : ExtensionOperand<116>;
-defm SPV_INTEL_long_composites : ExtensionOperand<117>;
-defm SPV_INTEL_memory_access_aliasing : ExtensionOperand<118>;
-defm SPV_INTEL_fp_max_error : ExtensionOperand<119>;
-defm SPV_INTEL_ternary_bitwise_function : ExtensionOperand<120>;
-defm SPV_INTEL_subgroup_matrix_multiply_accumulate : ExtensionOperand<121>;
-defm SPV_INTEL_2d_block_io : ExtensionOperand<122>;
-defm SPV_INTEL_int4 : ExtensionOperand<123>;
-defm SPV_KHR_float_controls2 : ExtensionOperand<124>;
-defm SPV_INTEL_tensor_float32_conversion : ExtensionOperand<125>;
+  defm : SymbolicOperandWithRequirements<ExtensionOperand, value, NAME, 0,
+                                         0, [], [], allowedEnvironments>;
+}
+
+defm SPV_AMD_shader_explicit_vertex_parameter
+    : ExtensionOperand<1, [EnvVulkan]>;
+defm SPV_AMD_shader_trinary_minmax_extension : ExtensionOperand<2, [EnvVulkan]>;
+defm SPV_AMD_gcn_shader : ExtensionOperand<3, [EnvVulkan]>;
+defm SPV_KHR_shader_ballot : ExtensionOperand<4, [EnvVulkan]>;
+defm SPV_AMD_shader_ballot : ExtensionOperand<5, [EnvVulkan]>;
+defm SPV_AMD_gpu_shader_half_float : ExtensionOperand<6, [EnvVulkan]>;
+defm SPV_KHR_shader_draw_parameters : ExtensionOperand<7, [EnvVulkan]>;
+defm SPV_KHR_subgroup_vote : ExtensionOperand<8, [EnvVulkan]>;
+defm SPV_KHR_16bit_storage : ExtensionOperand<9, [EnvVulkan]>;
+defm SPV_KHR_device_group : ExtensionOperand<10, [EnvVulkan]>;
+defm SPV_KHR_multiview : ExtensionOperand<11, [EnvVulkan]>;
+defm SPV_NVX_multiview_per_view_attributes : ExtensionOperand<12, [EnvVulkan]>;
+defm SPV_NV_viewport_array2 : ExtensionOperand<13, [EnvVulkan]>;
+defm SPV_NV_stereo_view_rendering : ExtensionOperand<14, [EnvVulkan]>;
+defm SPV_NV_sample_mask_override_coverage : ExtensionOperand<15, [EnvVulkan]>;
+defm SPV_NV_geometry_shader_passthrough : ExtensionOperand<16, [EnvVulkan]>;
+defm SPV_AMD_texture_gather_bias_lod : ExtensionOperand<17, [EnvVulkan]>;
+defm SPV_KHR_storage_buffer_storage_class : ExtensionOperand<18, [EnvVulkan]>;
+defm SPV_KHR_variable_pointers : ExtensionOperand<19, [EnvVulkan]>;
+defm SPV_AMD_gpu_shader_int16 : ExtensionOperand<20, [EnvVulkan]>;
+defm SPV_KHR_post_depth_coverage : ExtensionOperand<21, [EnvVulkan]>;
+defm SPV_KHR_shader_atomic_counter_ops : ExtensionOperand<22, []>;
+defm SPV_EXT_shader_stencil_export : ExtensionOperand<23, [EnvVulkan]>;
+defm SPV_EXT_shader_viewport_index_layer : ExtensionOperand<24, [EnvVulkan]>;
+defm SPV_AMD_shader_image_load_store_lod : ExtensionOperand<25, [EnvVulkan]>;
+defm SPV_AMD_shader_fragment_mask : ExtensionOperand<26, [EnvVulkan]>;
+defm SPV_EXT_fragment_fully_covered : ExtensionOperand<27, [EnvVulkan]>;
+defm SPV_AMD_gpu_shader_half_float_fetch : ExtensionOperand<28, [EnvVulkan]>;
+defm SPV_GOOGLE_decorate_string : ExtensionOperand<29, [EnvVulkan]>;
+defm SPV_GOOGLE_hlsl_functionality1 : ExtensionOperand<30, [EnvVulkan]>;
+defm SPV_NV_shader_subgroup_partitioned : ExtensionOperand<31, [EnvVulkan]>;
+defm SPV_EXT_descriptor_indexing : ExtensionOperand<32, [EnvVulkan]>;
+defm SPV_KHR_8bit_storage : ExtensionOperand<33, [EnvVulkan]>;
+defm SPV_KHR_vulkan_memory_model : ExtensionOperand<34, [EnvVulkan]>;
+defm SPV_NV_ray_tracing : ExtensionOperand<35, [EnvVulkan]>;
+defm SPV_NV_compute_shader_derivatives : ExtensionOperand<36, [EnvVulkan]>;
+defm SPV_NV_fragment_shader_barycentric : ExtensionOperand<37, [EnvVulkan]>;
+defm SPV_NV_mesh_shader : ExtensionOperand<38, [EnvVulkan]>;
+defm SPV_NV_shader_image_footprint : ExtensionOperand<39, [EnvVulkan]>;
+defm SPV_NV_shading_rate : ExtensionOperand<40, [EnvVulkan]>;
+defm SPV_INTEL_subgroups : ExtensionOperand<41, [EnvOpenCL]>;
+defm SPV_INTEL_media_block_io : ExtensionOperand<42, [EnvOpenCL]>;
+defm SPV_EXT_fragment_invocation_density : ExtensionOperand<44, [EnvVulkan]>;
+defm SPV_KHR_no_integer_wrap_decoration : ExtensionOperand<45, [EnvOpenCL]>;
+defm SPV_KHR_float_controls : ExtensionOperand<46, [EnvVulkan, EnvOpenCL]>;
+defm SPV_EXT_physical_storage_buffer : ExtensionOperand<47, [EnvVulkan]>;
+defm SPV_INTEL_fpga_memory_attributes : ExtensionOperand<48, [EnvOpenCL]>;
+defm SPV_NV_cooperative_matrix : ExtensionOperand<49, [EnvVulkan]>;
+defm SPV_INTEL_shader_integer_functions2
+    : ExtensionOperand<50, [EnvVulkan, EnvOpenCL]>;
+defm SPV_INTEL_fpga_loop_controls : ExtensionOperand<51, [EnvOpenCL]>;
+defm SPV_EXT_fragment_shader_interlock : ExtensionOperand<52, [EnvVulkan]>;
+defm SPV_NV_shader_sm_builtins : ExtensionOperand<53, [EnvVulkan]>;
+defm SPV_KHR_shader_clock : ExtensionOperand<54, [EnvVulkan, EnvOpenCL]>;
+defm SPV_INTEL_unstructured_loop_controls : ExtensionOperand<55, [EnvOpenCL]>;
+defm SPV_EXT_demote_to_helper_invocation : ExtensionOperand<56, [EnvVulkan]>;
+defm SPV_INTEL_fpga_reg : ExtensionOperand<57, [EnvOpenCL]>;
+defm SPV_INTEL_blocking_pipes : ExtensionOperand<58, [EnvOpenCL]>;
+defm SPV_GOOGLE_user_type : ExtensionOperand<59, [EnvVulkan]>;
+defm SPV_KHR_physical_storage_buffer : ExtensionOperand<60, [EnvVulkan]>;
+defm SPV_INTEL_kernel_attributes : ExtensionOperand<61, [EnvOpenCL]>;
+defm SPV_KHR_non_semantic_info : ExtensionOperand<62, [EnvVulkan, EnvOpenCL]>;
+defm SPV_INTEL_io_pipes : ExtensionOperand<63, [EnvOpenCL]>;
+defm SPV_KHR_ray_tracing : ExtensionOperand<64, [EnvVulkan]>;
+defm SPV_KHR_ray_query : ExtensionOperand<65, [EnvVulkan]>;
+defm SPV_INTEL_fpga_memory_accesses : ExtensionOperand<66, [EnvOpenCL]>;
+defm SPV_INTEL_arbitrary_precision_integers : ExtensionOperand<67, [EnvOpenCL]>;
+defm SPV_EXT_shader_atomic_float_add
+    : ExtensionOperand<68, [EnvVulkan, EnvOpenCL]>;
+defm SPV_KHR_terminate_invocation : ExtensionOperand<69, [EnvVulkan]>;
+defm SPV_KHR_fragment_shading_rate : ExtensionOperand<70, [EnvVulkan]>;
+defm SPV_EXT_shader_image_int64 : ExtensionOperand<71, [EnvVulkan]>;
+defm SPV_INTEL_fp_fast_math_mode : ExtensionOperand<72, [EnvOpenCL]>;
+defm SPV_INTEL_fpga_cluster_attributes : ExtensionOperand<73, [EnvOpenCL]>;
+defm SPV_INTEL_loop_fuse : ExtensionOperand<74, [EnvOpenCL]>;
+defm SPV_EXT_shader_atomic_float_min_max
+    : ExtensionOperand<75, [EnvVulkan, EnvOpenCL]>;
+defm SPV_KHR_workgroup_memory_explicit_layout
+    : ExtensionOperand<76, [EnvVulkan]>;
+defm SPV_KHR_linkonce_odr : ExtensionOperand<77, [EnvOpenCL]>;
+defm SPV_KHR_expect_assume : ExtensionOperand<78, [EnvVulkan, EnvOpenCL]>;
+defm SPV_INTEL_fpga_dsp_control : ExtensionOperand<79, [EnvOpenCL]>;
+defm SPV_NV_bindless_texture : ExtensionOperand<80, [EnvVulkan]>;
+defm SPV_INTEL_fpga_invocation_pipelining_attributes
+    : ExtensionOperand<81, [EnvOpenCL]>;
+defm SPV_KHR_subgroup_uniform_control_flow : ExtensionOperand<82, [EnvVulkan]>;
+defm SPV_HUAWEI_subpass_shading : ExtensionOperand<83, [EnvVulkan]>;
+defm SPV_KHR_integer_dot_product : ExtensionOperand<84, [EnvVulkan, EnvOpenCL]>;
+defm SPV_EXT_shader_atomic_float16_add
+    : ExtensionOperand<85, [EnvVulkan, EnvOpenCL]>;
+defm SPV_INTEL_runtime_aligned : ExtensionOperand<86, [EnvOpenCL]>;
+defm SPV_KHR_bit_instructions : ExtensionOperand<87, [EnvOpenCL]>;
+defm SPV_NV_ray_tracing_motion_blur : ExtensionOperand<88, [EnvVulkan]>;
+defm SPV_KHR_uniform_group_instructions : ExtensionOperand<89, [EnvOpenCL]>;
+defm SPV_KHR_subgroup_rotate : ExtensionOperand<90, [EnvVulkan, EnvOpenCL]>;
+defm SPV_INTEL_split_barrier : ExtensionOperand<91, [EnvOpenCL]>;
+defm SPV_KHR_ray_cull_mask : ExtensionOperand<92, [EnvVulkan]>;
+defm SPV_KHR_fragment_shader_barycentric : ExtensionOperand<93, [EnvVulkan]>;
+defm SPV_EXT_relaxed_printf_string_address_space
+    : ExtensionOperand<94, [EnvOpenCL]>;
+defm SPV_EXT_mesh_shader : ExtensionOperand<96, [EnvVulkan]>;
+defm SPV_ARM_core_builtins : ExtensionOperand<97, [EnvVulkan]>;
+defm SPV_EXT_opacity_micromap : ExtensionOperand<98, [EnvVulkan]>;
+defm SPV_NV_shader_invocation_reorder : ExtensionOperand<99, [EnvVulkan]>;
+defm SPV_INTEL_usm_storage_classes : ExtensionOperand<100, [EnvOpenCL]>;
+defm SPV_INTEL_fpga_latency_control : ExtensionOperand<101, [EnvOpenCL]>;
+defm SPV_INTEL_fpga_argument_interfaces : ExtensionOperand<102, [EnvOpenCL]>;
+defm SPV_INTEL_optnone : ExtensionOperand<103, [EnvOpenCL]>;
+defm SPV_INTEL_function_pointers : ExtensionOperand<104, [EnvOpenCL]>;
+defm SPV_INTEL_variable_length_array : ExtensionOperand<105, [EnvOpenCL]>;
+defm SPV_INTEL_bfloat16_conversion : ExtensionOperand<106, [EnvOpenCL]>;
+defm SPV_INTEL_inline_assembly : ExtensionOperand<107, [EnvOpenCL]>;
+defm SPV_INTEL_cache_controls : ExtensionOperand<108, [EnvOpenCL]>;
+defm SPV_INTEL_global_variable_host_access : ExtensionOperand<109, [EnvOpenCL]>;
+defm SPV_INTEL_global_variable_fpga_decorations
+    : ExtensionOperand<110, [EnvOpenCL]>;
+defm SPV_KHR_cooperative_matrix : ExtensionOperand<111, [EnvVulkan, EnvOpenCL]>;
+defm SPV_EXT_arithmetic_fence : ExtensionOperand<112, [EnvOpenCL]>;
+defm SPV_EXT_optnone : ExtensionOperand<113, [EnvOpenCL]>;
+defm SPV_INTEL_joint_matrix : ExtensionOperand<114, [EnvOpenCL]>;
+defm SPV_INTEL_float_controls2 : ExtensionOperand<115, [EnvOpenCL]>;
+defm SPV_INTEL_bindless_images : ExtensionOperand<116, [EnvOpenCL]>;
+defm SPV_INTEL_long_composites : ExtensionOperand<117, [EnvOpenCL]>;
+defm SPV_INTEL_memory_access_aliasing : ExtensionOperand<118, [EnvOpenCL]>;
+defm SPV_INTEL_fp_max_error : ExtensionOperand<119, [EnvOpenCL]>;
+defm SPV_INTEL_ternary_bitwise_function : ExtensionOperand<120, [EnvOpenCL]>;
+defm SPV_INTEL_subgroup_matrix_multiply_accumulate
+    : ExtensionOperand<121, [EnvOpenCL]>;
+defm SPV_INTEL_2d_block_io : ExtensionOperand<122, [EnvOpenCL]>;
+defm SPV_INTEL_int4 : ExtensionOperand<123, [EnvOpenCL]>;
+defm SPV_KHR_float_controls2 : ExtensionOperand<124, [EnvVulkan, EnvOpenCL]>;
+defm SPV_INTEL_tensor_float32_conversion : ExtensionOperand<125, [EnvOpenCL]>;
 
 //===----------------------------------------------------------------------===//
 // Multiclass used to define Capabilities enum values and at the same time
@@ -342,7 +403,9 @@ class Capability<string name, bits<32> value> {
 
 multiclass CapabilityOperand<bits<32> value, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def NAME : Capability<NAME, value>;
-  defm : SymbolicOperandWithRequirements<CapabilityOperand, value, NAME, minVersion, maxVersion, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<CapabilityOperand, value, NAME,
+                                         minVersion, maxVersion, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm Matrix : CapabilityOperand<0, 0, 0, [], []>;
@@ -551,7 +614,8 @@ class SourceLanguage<string name, bits<32> value> {
 
 multiclass SourceLanguageOperand<bits<32> value> {
   def : SourceLanguage<NAME, value>;
-  defm : SymbolicOperandWithRequirements<SourceLanguageOperand, value, NAME, 0, 0, [], []>;
+  defm : SymbolicOperandWithRequirements<SourceLanguageOperand, value, NAME, 0,
+                                         0, [], [], []>;
 }
 
 defm Unknown : SourceLanguageOperand<0>;
@@ -580,7 +644,8 @@ class AddressingModel<string name, bits<32> value> {
 
 multiclass AddressingModelOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : AddressingModel<NAME, value>;
-  defm : SymbolicOperandWithRequirements<AddressingModelOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<AddressingModelOperand, value, NAME, 0,
+                                         0, [], reqCapabilities, []>;
 }
 
 defm Logical : AddressingModelOperand<0, []>;
@@ -607,7 +672,8 @@ class ExecutionModel<string name, bits<32> value> {
 
 multiclass ExecutionModelOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : ExecutionModel<NAME, value>;
-  defm : SymbolicOperandWithRequirements<ExecutionModelOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<ExecutionModelOperand, value, NAME, 0,
+                                         0, [], reqCapabilities, []>;
 }
 
 defm Vertex : ExecutionModelOperand<0, [Shader]>;
@@ -645,7 +711,8 @@ class MemoryModel<string name, bits<32> value> {
 
 multiclass MemoryModelOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : MemoryModel<NAME, value>;
-  defm : SymbolicOperandWithRequirements<MemoryModelOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<MemoryModelOperand, value, NAME, 0,
+                                         0, [], reqCapabilities, []>;
 }
 
 defm Simple : MemoryModelOperand<0, [Shader]>;
@@ -672,7 +739,8 @@ class ExecutionMode<string name, bits<32> value> {
 
 multiclass ExecutionModeOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : ExecutionMode<NAME, value>;
-  defm : SymbolicOperandWithRequirements<ExecutionModeOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<ExecutionModeOperand, value, NAME, 0,
+                                         0, [], reqCapabilities, []>;
 }
 
 defm Invocations : ExecutionModeOperand<0, [Geometry]>;
@@ -748,7 +816,8 @@ class StorageClass<string name, bits<32> value> {
 
 multiclass StorageClassOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : StorageClass<NAME, value>;
-  defm : SymbolicOperandWithRequirements<StorageClassOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<StorageClassOperand, value, NAME, 0, 0,
+                                         reqExtensions, reqCapabilities, []>;
 }
 
 defm UniformConstant : StorageClassOperand<0, [], []>;
@@ -794,7 +863,8 @@ class Dim<string name, bits<32> value> {
 
 multiclass DimOperand<bits<32> value, string mnemonic, list<Capability> reqCapabilities> {
   def NAME : Dim<NAME, value>;
-  defm : SymbolicOperandWithRequirements<DimOperand, value, mnemonic, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<DimOperand, value, mnemonic, 0, 0, [],
+                                         reqCapabilities, []>;
 }
 
 defm DIM_1D : DimOperand<0, "1D", [Sampled1D, Image1D]>;
@@ -824,7 +894,8 @@ class SamplerAddressingMode<string name, bits<32> value> {
 
 multiclass SamplerAddressingModeOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : SamplerAddressingMode<NAME, value>;
-  defm : SymbolicOperandWithRequirements<SamplerAddressingModeOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<SamplerAddressingModeOperand, value,
+                                         NAME, 0, 0, [], reqCapabilities, []>;
 }
 
 defm None : SamplerAddressingModeOperand<0, [Kernel]>;
@@ -852,7 +923,8 @@ class SamplerFilterMode<string name, bits<32> value> {
 
 multiclass SamplerFilterModeOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : SamplerFilterMode<NAME, value>;
-  defm : SymbolicOperandWithRequirements<SamplerFilterModeOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<SamplerFilterModeOperand, value, NAME,
+                                         0, 0, [], reqCapabilities, []>;
 }
 
 defm Nearest : SamplerFilterModeOperand<0, [Kernel]>;
@@ -877,7 +949,8 @@ class ImageFormat<string name, bits<32> value> {
 
 multiclass ImageFormatOperand<bits<32> value, list<Capability> reqCapabilities> {
   def NAME : ImageFormat<NAME, value>;
-  defm : SymbolicOperandWithRequirements<ImageFormatOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<ImageFormatOperand, value, NAME, 0,
+                                         0, [], reqCapabilities, []>;
 }
 
 defm Unknown : ImageFormatOperand<0, []>;
@@ -940,7 +1013,8 @@ class ImageChannelOrder<string name, bits<32> value> {
 
 multiclass ImageChannelOrderOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : ImageChannelOrder<NAME, value>;
-  defm : SymbolicOperandWithRequirements<ImageChannelOrderOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<ImageChannelOrderOperand, value, NAME,
+                                         0, 0, [], reqCapabilities, []>;
 }
 
 defm R : ImageChannelOrderOperand<0, [Kernel]>;
@@ -983,7 +1057,8 @@ class ImageChannelDataType<string name, bits<32> value> {
 
 multiclass ImageChannelDataTypeOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : ImageChannelDataType<NAME, value>;
-  defm : SymbolicOperandWithRequirements<ImageChannelDataTypeOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<ImageChannelDataTypeOperand, value,
+                                         NAME, 0, 0, [], reqCapabilities, []>;
 }
 
 defm SnormInt8 : ImageChannelDataTypeOperand<0, []>;
@@ -1023,7 +1098,8 @@ class ImageOperand<string name, bits<32> value> {
 
 multiclass ImageOperandOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : ImageOperand<NAME, value>;
-  defm : SymbolicOperandWithRequirements<ImageOperandOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<ImageOperandOperand, value, NAME, 0,
+                                         0, [], reqCapabilities, []>;
 }
 
 defm None : ImageOperandOperand<0x0, []>;
@@ -1061,7 +1137,8 @@ class FPFastMathMode<string name, bits<32> value> {
 
 multiclass FPFastMathModeOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : FPFastMathMode<NAME, value>;
-  defm : SymbolicOperandWithRequirements<FPFastMathModeOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<FPFastMathModeOperand, value, NAME, 0,
+                                         0, [], reqCapabilities, []>;
 }
 
 defm None : FPFastMathModeOperand<0x0, []>;
@@ -1090,7 +1167,8 @@ class FPRoundingMode<string name, bits<32> value> {
 
 multiclass FPRoundingModeOperand<bits<32> value> {
   def NAME : FPRoundingMode<NAME, value>;
-  defm : SymbolicOperandWithRequirements<FPRoundingModeOperand, value, NAME, 0, 0, [], []>;
+  defm : SymbolicOperandWithRequirements<FPRoundingModeOperand, value, NAME, 0,
+                                         0, [], [], []>;
 }
 
 defm RTE : FPRoundingModeOperand<0>;
@@ -1117,7 +1195,8 @@ class LinkageType<string name, bits<32> value> {
 
 multiclass LinkageTypeOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : LinkageType<NAME, value>;
-  defm : SymbolicOperandWithRequirements<LinkageTypeOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<LinkageTypeOperand, value, NAME, 0,
+                                         0, [], reqCapabilities, []>;
 }
 
 defm Export : LinkageTypeOperand<0, [Linkage]>;
@@ -1143,7 +1222,8 @@ class AccessQualifier<string name, bits<32> value> {
 
 multiclass AccessQualifierOperand<bits<32> value, list<Capability> reqCapabilities> {
   def NAME : AccessQualifier<NAME, value>;
-  defm : SymbolicOperandWithRequirements<AccessQualifierOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<AccessQualifierOperand, value, NAME, 0,
+                                         0, [], reqCapabilities, []>;
 }
 
 defm ReadOnly : AccessQualifierOperand<0, [Kernel]>;
@@ -1170,7 +1250,9 @@ class FunctionParameterAttribute<string name, bits<32> value> {
 
 multiclass FunctionParameterAttributeOperand<bits<32> value, list<Capability> reqCapabilities> {
   def : FunctionParameterAttribute<NAME, value>;
-  defm : SymbolicOperandWithRequirements<FunctionParameterAttributeOperand, value, NAME, 0, 0, [], reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<FunctionParameterAttributeOperand,
+                                         value, NAME, 0, 0, [],
+                                         reqCapabilities, []>;
 }
 
 defm Zext : FunctionParameterAttributeOperand<0, [Kernel]>;
@@ -1202,7 +1284,9 @@ class Decoration<string name, bits<32> value> {
 
 multiclass DecorationOperand<bits<32> value, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : Decoration<NAME, value>;
-  defm : SymbolicOperandWithRequirements<DecorationOperand, value, NAME, minVersion, maxVersion, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<DecorationOperand, value, NAME,
+                                         minVersion, maxVersion, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm RelaxedPrecision : DecorationOperand<0, 0, 0, [], [Shader]>;
@@ -1303,7 +1387,9 @@ class BuiltIn<string name, bits<32> value> {
 
 multiclass BuiltInOperand<bits<32> value, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def NAME : BuiltIn<NAME, value>;
-  defm : SymbolicOperandWithRequirements<BuiltInOperand, value, NAME, minVersion, maxVersion, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<BuiltInOperand, value, NAME,
+                                         minVersion, maxVersion, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm Position : BuiltInOperand<0, 0, 0, [], [Shader]>;
@@ -1417,7 +1503,8 @@ class SelectionControl<string name, bits<32> value> {
 
 multiclass SelectionControlOperand<bits<32> value> {
   def : SelectionControl<NAME, value>;
-  defm : SymbolicOperandWithRequirements<SelectionControlOperand, value, NAME, 0, 0, [], []>;
+  defm : SymbolicOperandWithRequirements<SelectionControlOperand, value, NAME,
+                                         0, 0, [], [], []>;
 }
 
 defm None : SelectionControlOperand<0x0>;
@@ -1443,7 +1530,8 @@ class LoopControl<string name, bits<32> value> {
 
 multiclass LoopControlOperand<bits<32> value> {
   def : LoopControl<NAME, value>;
-  defm : SymbolicOperandWithRequirements<LoopControlOperand, value, NAME, 0, 0, [], []>;
+  defm : SymbolicOperandWithRequirements<LoopControlOperand, value, NAME, 0,
+                                         0, [], [], []>;
 }
 
 defm None : LoopControlOperand<0x0>;
@@ -1476,7 +1564,8 @@ class FunctionControl<string name, bits<32> value> {
 
 multiclass FunctionControlOperand<bits<32> value> {
   def : FunctionControl<NAME, value>;
-  defm : SymbolicOperandWithRequirements<FunctionControlOperand, value, NAME, 0, 0, [], []>;
+  defm : SymbolicOperandWithRequirements<FunctionControlOperand, value, NAME, 0,
+                                         0, [], [], []>;
 }
 
 defm None : FunctionControlOperand<0x0>;
@@ -1506,7 +1595,9 @@ class MemorySemantics<string name, bits<32> value> {
 
 multiclass MemorySemanticsOperand<bits<32> value, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : MemorySemantics<NAME, value>;
-  defm : SymbolicOperandWithRequirements<MemorySemanticsOperand, value, NAME, minVersion, maxVersion, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<MemorySemanticsOperand, value, NAME,
+                                         minVersion, maxVersion, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm None : MemorySemanticsOperand<0x0, 0, 0, [], []>;
@@ -1544,7 +1635,9 @@ class MemoryOperand<string name, bits<32> value> {
 
 multiclass MemoryOperandOperand<bits<32> value, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : MemoryOperand<NAME, value>;
-  defm : SymbolicOperandWithRequirements<MemoryOperandOperand, value, NAME, minVersion, maxVersion, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<MemoryOperandOperand, value, NAME,
+                                         minVersion, maxVersion, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm None : MemoryOperandOperand<0x0, 0, 0, [], []>;
@@ -1577,7 +1670,9 @@ class Scope<string name, bits<32> value> {
 
 multiclass ScopeOperand<bits<32> value, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : Scope<NAME, value>;
-  defm : SymbolicOperandWithRequirements<ScopeOperand, value, NAME, minVersion, maxVersion, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<ScopeOperand, value, NAME, minVersion,
+                                         maxVersion, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm CrossDevice : ScopeOperand<0, 0, 0, [], []>;
@@ -1607,7 +1702,9 @@ class GroupOperation<string name, bits<32> value> {
 
 multiclass GroupOperationOperand<bits<32> value, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def NAME : GroupOperation<NAME, value>;
-  defm : SymbolicOperandWithRequirements<GroupOperationOperand, value, NAME, minVersion, maxVersion, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<GroupOperationOperand, value, NAME,
+                                         minVersion, maxVersion, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm Reduce : GroupOperationOperand<0, 0, 0, [], [Kernel, GroupNonUniformArithmetic, GroupNonUniformBallot]>;
@@ -1638,7 +1735,9 @@ class KernelEnqueueFlags<string name, bits<32> value> {
 
 multiclass KernelEnqueueFlagsOperand<bits<32> value, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : KernelEnqueueFlags<NAME, value>;
-  defm : SymbolicOperandWithRequirements<KernelEnqueueFlagsOperand, value, NAME, minVersion, maxVersion, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<KernelEnqueueFlagsOperand, value, NAME,
+                                         minVersion, maxVersion, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm NoWait : KernelEnqueueFlagsOperand<0, 0, 0, [], [Kernel]>;
@@ -1665,7 +1764,9 @@ class KernelProfilingInfo<string name, bits<32> value> {
 
 multiclass KernelProfilingInfoOperand<bits<32> value, bits<32> minVersion, bits<32> maxVersion, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : KernelProfilingInfo<NAME, value>;
-  defm : SymbolicOperandWithRequirements<KernelProfilingInfoOperand, value, NAME, minVersion, maxVersion, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<KernelProfilingInfoOperand, value,
+                                         NAME, minVersion, maxVersion,
+                                         reqExtensions, reqCapabilities, []>;
 }
 
 defm None : KernelProfilingInfoOperand<0x0, 0, 0, [], []>;
@@ -1690,7 +1791,8 @@ class Opcode<string name, bits<32> value> {
 
 multiclass OpcodeOperand<bits<32> value> {
   def : Opcode<NAME, value>;
-  defm : SymbolicOperandWithRequirements<OpcodeOperand, value, NAME, 0, 0, [], []>;
+  defm : SymbolicOperandWithRequirements<OpcodeOperand, value, NAME, 0,
+                                         0, [], [], []>;
 }
 // TODO: implement other mnemonics.
 defm InBoundsAccessChain : OpcodeOperand<66>;
@@ -1720,7 +1822,9 @@ class CooperativeMatrixLayout<string name, bits<32> value> {
 
 multiclass CooperativeMatrixLayoutOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : CooperativeMatrixLayout<NAME, value>;
-  defm : SymbolicOperandWithRequirements<CooperativeMatrixLayoutOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<CooperativeMatrixLayoutOperand, value,
+                                         NAME, 0, 0, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm RowMajorKHR : CooperativeMatrixLayoutOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
@@ -1747,7 +1851,9 @@ class CooperativeMatrixOperands<string name, bits<32> value> {
 
 multiclass CooperativeMatrixOperandsOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : CooperativeMatrixOperands<NAME, value>;
-  defm : SymbolicOperandWithRequirements<CooperativeMatrixOperandsOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<CooperativeMatrixOperandsOperand,
+                                         value, NAME, 0, 0, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 defm NoneKHR : CooperativeMatrixOperandsOperand<0x0, [SPV_KHR_cooperative_matrix], [CooperativeMatrixKHR]>;
@@ -1780,7 +1886,9 @@ class SpecConstantOpOperands<string name, bits<32> value> {
 
 multiclass SpecConstantOpOperandsOperand<bits<32> value, list<Extension> reqExtensions, list<Capability> reqCapabilities> {
   def : SpecConstantOpOperands<NAME, value>;
-  defm : SymbolicOperandWithRequirements<SpecConstantOpOperandsOperand, value, NAME, 0, 0, reqExtensions, reqCapabilities>;
+  defm : SymbolicOperandWithRequirements<SpecConstantOpOperandsOperand, value,
+                                         NAME, 0, 0, reqExtensions,
+                                         reqCapabilities, []>;
 }
 
 // Conversion
@@ -1868,7 +1976,9 @@ class MatrixMultiplyAccumulateOperands<string name, bits<32> value> {
 
 multiclass  MatrixMultiplyAccumulateOperandsOperand<bits<32> value, list<Extension> reqExtensions> {
   def : MatrixMultiplyAccumulateOperands<NAME, value>;
-  defm : SymbolicOperandWithRequirements<MatrixMultiplyAccumulateOperandsOperand, value, NAME, 0, 0, reqExtensions, []>;
+  defm : SymbolicOperandWithRequirements<
+             MatrixMultiplyAccumulateOperandsOperand, value, NAME, 0, 0,
+             reqExtensions, [], []>;
 }
 
 defm None :  MatrixMultiplyAccumulateOperandsOperand<0x0, [SPV_INTEL_subgroup_matrix_multiply_accumulate]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index d7cf211ba84dc..e0bfb77f4b530 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -226,6 +226,7 @@ void SPIRVPassConfig::addIRPasses() {
 }
 
 void SPIRVPassConfig::addISelPrepare() {
+  addPass(createSPIRVLegalizeImplicitBindingPass());
   addPass(createSPIRVEmitIntrinsicsPass(&getTM<SPIRVTargetMachine>()));
   if (TM.getSubtargetImpl()->isLogicalSPIRV())
     addPass(createSPIRVLegalizePointerCastPass(&getTM<SPIRVTargetMachine>()));
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 416d811ba4e65..820e56b362edc 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -463,8 +463,10 @@ std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) {
     DemangledNameLenStart = NameSpaceStart + 11;
   }
   Start = Name.find_first_not_of("0123456789", DemangledNameLenStart);
-  Name.substr(DemangledNameLenStart, Start - DemangledNameLenStart)
-      .getAsInteger(10, Len);
+  [[maybe_unused]] bool Error =
+      Name.substr(DemangledNameLenStart, Start - DemangledNameLenStart)
+          .getAsInteger(10, Len);
+  assert(!Error && "Failed to parse demangled name length");
   return Name.substr(Start, Len).str();
 }
 
@@ -756,7 +758,7 @@ bool getVacantFunctionName(Module &M, std::string &Name) {
   for (unsigned I = 0; I < MaxIters; ++I) {
     std::string OrdName = Name + Twine(I).str();
     if (!M.getFunction(OrdName)) {
-      Name = OrdName;
+      Name = std::move(OrdName);
       return true;
     }
   }
diff --git a/llvm/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp b/llvm/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp
index fab94fb4d40ca..f1cd9b1ab07ca 100644
--- a/llvm/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp
+++ b/llvm/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp
@@ -14,13 +14,16 @@
 #include "TargetInfo/SparcTargetInfo.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/TargetRegistry.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/Endian.h"
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "sparc-disassembler"
 
@@ -58,7 +61,8 @@ LLVMInitializeSparcDisassembler() {
                                          createSparcDisassembler);
 }
 
-static const unsigned IntRegDecoderTable[] = {
+// clang-format off
+static constexpr unsigned IntRegDecoderTable[] = {
   SP::G0,  SP::G1,  SP::G2,  SP::G3,
   SP::G4,  SP::G5,  SP::G6,  SP::G7,
   SP::O0,  SP::O1,  SP::O2,  SP::O3,
@@ -68,7 +72,7 @@ static const unsigned IntRegDecoderTable[] = {
   SP::I0,  SP::I1,  SP::I2,  SP::I3,
   SP::I4,  SP::I5,  SP::I6,  SP::I7 };
 
-static const unsigned FPRegDecoderTable[] = {
+static constexpr unsigned FPRegDecoderTable[] = {
   SP::F0,   SP::F1,   SP::F2,   SP::F3,
   SP::F4,   SP::F5,   SP::F6,   SP::F7,
   SP::F8,   SP::F9,   SP::F10,  SP::F11,
@@ -78,7 +82,7 @@ static const unsigned FPRegDecoderTable[] = {
   SP::F24,  SP::F25,  SP::F26,  SP::F27,
   SP::F28,  SP::F29,  SP::F30,  SP::F31 };
 
-static const unsigned DFPRegDecoderTable[] = {
+static constexpr unsigned DFPRegDecoderTable[] = {
   SP::D0,   SP::D16,  SP::D1,   SP::D17,
   SP::D2,   SP::D18,  SP::D3,   SP::D19,
   SP::D4,   SP::D20,  SP::D5,   SP::D21,
@@ -88,7 +92,7 @@ static const unsigned DFPRegDecoderTable[] = {
   SP::D12,  SP::D28,  SP::D13,  SP::D29,
   SP::D14,  SP::D30,  SP::D15,  SP::D31 };
 
-static const unsigned QFPRegDecoderTable[] = {
+static constexpr unsigned QFPRegDecoderTable[] = {
   SP::Q0,  SP::Q8,   ~0U,  ~0U,
   SP::Q1,  SP::Q9,   ~0U,  ~0U,
   SP::Q2,  SP::Q10,  ~0U,  ~0U,
@@ -98,29 +102,29 @@ static const unsigned QFPRegDecoderTable[] = {
   SP::Q6,  SP::Q14,  ~0U,  ~0U,
   SP::Q7,  SP::Q15,  ~0U,  ~0U } ;
 
-static const unsigned FCCRegDecoderTable[] = {
+static constexpr unsigned FCCRegDecoderTable[] = {
   SP::FCC0, SP::FCC1, SP::FCC2, SP::FCC3 };
 
-static const unsigned ASRRegDecoderTable[] = {
+static constexpr unsigned ASRRegDecoderTable[] = {
     SP::Y,     SP::ASR1,  SP::ASR2,  SP::ASR3,  SP::ASR4,  SP::ASR5,  SP::ASR6,
     SP::ASR7,  SP::ASR8,  SP::ASR9,  SP::ASR10, SP::ASR11, SP::ASR12, SP::ASR13,
     SP::ASR14, SP::ASR15, SP::ASR16, SP::ASR17, SP::ASR18, SP::ASR19, SP::ASR20,
     SP::ASR21, SP::ASR22, SP::ASR23, SP::ASR24, SP::ASR25, SP::ASR26, SP::ASR27,
     SP::ASR28, SP::ASR29, SP::ASR30, SP::ASR31};
 
-static const unsigned PRRegDecoderTable[] = {
+static constexpr unsigned PRRegDecoderTable[] = {
     SP::TPC,     SP::TNPC,       SP::TSTATE,   SP::TT,       SP::TICK,
     SP::TBA,     SP::PSTATE,     SP::TL,       SP::PIL,      SP::CWP,
     SP::CANSAVE, SP::CANRESTORE, SP::CLEANWIN, SP::OTHERWIN, SP::WSTATE};
 
-static const uint16_t IntPairDecoderTable[] = {
+static constexpr uint16_t IntPairDecoderTable[] = {
   SP::G0_G1, SP::G2_G3, SP::G4_G5, SP::G6_G7,
   SP::O0_O1, SP::O2_O3, SP::O4_O5, SP::O6_O7,
   SP::L0_L1, SP::L2_L3, SP::L4_L5, SP::L6_L7,
   SP::I0_I1, SP::I2_I3, SP::I4_I5, SP::I6_I7,
 };
 
-static const unsigned CPRegDecoderTable[] = {
+static constexpr unsigned CPRegDecoderTable[] = {
   SP::C0,  SP::C1,  SP::C2,  SP::C3,
   SP::C4,  SP::C5,  SP::C6,  SP::C7,
   SP::C8,  SP::C9,  SP::C10, SP::C11,
@@ -131,18 +135,18 @@ static const unsigned CPRegDecoderTable[] = {
   SP::C28, SP::C29, SP::C30, SP::C31
 };
 
-
-static const uint16_t CPPairDecoderTable[] = {
+static constexpr uint16_t CPPairDecoderTable[] = {
   SP::C0_C1,   SP::C2_C3,   SP::C4_C5,   SP::C6_C7,
   SP::C8_C9,   SP::C10_C11, SP::C12_C13, SP::C14_C15,
   SP::C16_C17, SP::C18_C19, SP::C20_C21, SP::C22_C23,
   SP::C24_C25, SP::C26_C27, SP::C28_C29, SP::C30_C31
 };
+// clang-format on
 
 static DecodeStatus DecodeIntRegsRegisterClass(MCInst &Inst, unsigned RegNo,
                                                uint64_t Address,
                                                const MCDisassembler *Decoder) {
-  if (RegNo > 31)
+  if (RegNo >= std::size(IntRegDecoderTable))
     return MCDisassembler::Fail;
   unsigned Reg = IntRegDecoderTable[RegNo];
   Inst.addOperand(MCOperand::createReg(Reg));
@@ -166,7 +170,7 @@ static DecodeStatus DecodePointerLikeRegClass0(MCInst &Inst, unsigned RegNo,
 static DecodeStatus DecodeFPRegsRegisterClass(MCInst &Inst, unsigned RegNo,
                                               uint64_t Address,
                                               const MCDisassembler *Decoder) {
-  if (RegNo > 31)
+  if (RegNo >= std::size(FPRegDecoderTable))
     return MCDisassembler::Fail;
   unsigned Reg = FPRegDecoderTable[RegNo];
   Inst.addOperand(MCOperand::createReg(Reg));
@@ -176,7 +180,7 @@ static DecodeStatus DecodeFPRegsRegisterClass(MCInst &Inst, unsigned RegNo,
 static DecodeStatus DecodeDFPRegsRegisterClass(MCInst &Inst, unsigned RegNo,
                                                uint64_t Address,
                                                const MCDisassembler *Decoder) {
-  if (RegNo > 31)
+  if (RegNo >= std::size(DFPRegDecoderTable))
     return MCDisassembler::Fail;
   unsigned Reg = DFPRegDecoderTable[RegNo];
   Inst.addOperand(MCOperand::createReg(Reg));
@@ -186,7 +190,7 @@ static DecodeStatus DecodeDFPRegsRegisterClass(MCInst &Inst, unsigned RegNo,
 static DecodeStatus DecodeQFPRegsRegisterClass(MCInst &Inst, unsigned RegNo,
                                                uint64_t Address,
                                                const MCDisassembler *Decoder) {
-  if (RegNo > 31)
+  if (RegNo >= std::size(QFPRegDecoderTable))
     return MCDisassembler::Fail;
 
   unsigned Reg = QFPRegDecoderTable[RegNo];
@@ -199,7 +203,7 @@ static DecodeStatus DecodeQFPRegsRegisterClass(MCInst &Inst, unsigned RegNo,
 static DecodeStatus
 DecodeCoprocRegsRegisterClass(MCInst &Inst, unsigned RegNo, uint64_t Address,
                               const MCDisassembler *Decoder) {
-  if (RegNo > 31)
+  if (RegNo >= std::size(CPRegDecoderTable))
     return MCDisassembler::Fail;
   unsigned Reg = CPRegDecoderTable[RegNo];
   Inst.addOperand(MCOperand::createReg(Reg));
@@ -209,7 +213,7 @@ DecodeCoprocRegsRegisterClass(MCInst &Inst, unsigned RegNo, uint64_t Address,
 static DecodeStatus DecodeFCCRegsRegisterClass(MCInst &Inst, unsigned RegNo,
                                                uint64_t Address,
                                                const MCDisassembler *Decoder) {
-  if (RegNo > 3)
+  if (RegNo >= std::size(FCCRegDecoderTable))
     return MCDisassembler::Fail;
   Inst.addOperand(MCOperand::createReg(FCCRegDecoderTable[RegNo]));
   return MCDisassembler::Success;
@@ -218,7 +222,7 @@ static DecodeStatus DecodeFCCRegsRegisterClass(MCInst &Inst, unsigned RegNo,
 static DecodeStatus DecodeASRRegsRegisterClass(MCInst &Inst, unsigned RegNo,
                                                uint64_t Address,
                                                const MCDisassembler *Decoder) {
-  if (RegNo > 31)
+  if (RegNo >= std::size(ASRRegDecoderTable))
     return MCDisassembler::Fail;
   Inst.addOperand(MCOperand::createReg(ASRRegDecoderTable[RegNo]));
   return MCDisassembler::Success;
@@ -238,13 +242,14 @@ static DecodeStatus DecodeIntPairRegisterClass(MCInst &Inst, unsigned RegNo,
                                                const MCDisassembler *Decoder) {
   DecodeStatus S = MCDisassembler::Success;
 
-  if (RegNo > 31)
-    return MCDisassembler::Fail;
-
   if ((RegNo & 1))
     S = MCDisassembler::SoftFail;
 
-  unsigned RegisterPair = IntPairDecoderTable[RegNo/2];
+  RegNo = RegNo / 2;
+  if (RegNo >= std::size(IntPairDecoderTable))
+    return MCDisassembler::Fail;
+
+  unsigned RegisterPair = IntPairDecoderTable[RegNo];
   Inst.addOperand(MCOperand::createReg(RegisterPair));
   return S;
 }
@@ -252,10 +257,11 @@ static DecodeStatus DecodeIntPairRegisterClass(MCInst &Inst, unsigned RegNo,
 static DecodeStatus
 DecodeCoprocPairRegisterClass(MCInst &Inst, unsigned RegNo, uint64_t Address,
                               const MCDisassembler *Decoder) {
-  if (RegNo > 31)
+  RegNo = RegNo / 2;
+  if (RegNo >= std::size(CPPairDecoderTable))
     return MCDisassembler::Fail;
 
-  unsigned RegisterPair = CPPairDecoderTable[RegNo/2];
+  unsigned RegisterPair = CPPairDecoderTable[RegNo];
   Inst.addOperand(MCOperand::createReg(RegisterPair));
   return MCDisassembler::Success;
 }
@@ -283,12 +289,8 @@ static DecodeStatus readInstruction32(ArrayRef<uint8_t> Bytes, uint64_t Address,
   }
 
   Size = 4;
-  Insn = IsLittleEndian
-             ? (Bytes[0] << 0) | (Bytes[1] << 8) | (Bytes[2] << 16) |
-                   (Bytes[3] << 24)
-             : (Bytes[3] << 0) | (Bytes[2] << 8) | (Bytes[1] << 16) |
-                   (Bytes[0] << 24);
-
+  Insn = support::endian::read<uint32_t>(
+      Bytes.data(), IsLittleEndian ? endianness::little : endianness::big);
   return MCDisassembler::Success;
 }
 
@@ -306,13 +308,11 @@ DecodeStatus SparcDisassembler::getInstruction(MCInst &Instr, uint64_t &Size,
   // Calling the auto-generated decoder function.
 
   if (STI.hasFeature(Sparc::FeatureV9))
-  {
-    Result = decodeInstruction(DecoderTableSparcV932, Instr, Insn, Address, this, STI);
-  }
+    Result = decodeInstruction(DecoderTableSparcV932, Instr, Insn, Address,
+                               this, STI);
   else
-  {
     Result = decodeInstruction(DecoderTableSparcV832, Instr, Insn, Address, this, STI);
-  }
+
   if (Result != MCDisassembler::Fail)
     return Result;
 
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 1aa8efe3e9979..d01218f573dc2 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -1179,7 +1179,7 @@ static void fixupVariableFloatArgs(SmallVectorImpl<CCValAssign> &ArgLocs,
     if (!VA.isRegLoc() || (ValTy != MVT::f64 && ValTy != MVT::f128))
       continue;
     // The fixed arguments to a varargs function still go in FP registers.
-    if (Outs[VA.getValNo()].IsFixed)
+    if (!Outs[VA.getValNo()].Flags.isVarArg())
       continue;
 
     // This floating point argument should be reassigned.
@@ -1799,12 +1799,14 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::FCOS , MVT::f64, Expand);
   setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
   setOperationAction(ISD::FREM , MVT::f64, Expand);
-  setOperationAction(ISD::FMA  , MVT::f64, Expand);
+  setOperationAction(ISD::FMA, MVT::f64,
+                     Subtarget->isUA2007() ? Legal : Expand);
   setOperationAction(ISD::FSIN , MVT::f32, Expand);
   setOperationAction(ISD::FCOS , MVT::f32, Expand);
   setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
   setOperationAction(ISD::FREM , MVT::f32, Expand);
-  setOperationAction(ISD::FMA, MVT::f32, Expand);
+  setOperationAction(ISD::FMA, MVT::f32,
+                     Subtarget->isUA2007() ? Legal : Expand);
   setOperationAction(ISD::ROTL , MVT::i32, Expand);
   setOperationAction(ISD::ROTR , MVT::i32, Expand);
   setOperationAction(ISD::BSWAP, MVT::i32, Expand);
@@ -2278,21 +2280,15 @@ SDValue SparcTargetLowering::LowerF128_LibCallArg(SDValue Chain,
   EVT ArgVT = Arg.getValueType();
   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
 
-  ArgListEntry Entry;
-  Entry.Node = Arg;
-  Entry.Ty   = ArgTy;
-
   if (ArgTy->isFP128Ty()) {
     // Create a stack object and pass the pointer to the library function.
     int FI = MFI.CreateStackObject(16, Align(8), false);
     SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
-    Chain = DAG.getStore(Chain, DL, Entry.Node, FIPtr, MachinePointerInfo(),
-                         Align(8));
-
-    Entry.Node = FIPtr;
-    Entry.Ty = PointerType::getUnqual(ArgTy->getContext());
+    Chain = DAG.getStore(Chain, DL, Arg, FIPtr, MachinePointerInfo(), Align(8));
+    Args.emplace_back(FIPtr, PointerType::getUnqual(ArgTy->getContext()));
+  } else {
+    Args.emplace_back(Arg, ArgTy);
   }
-  Args.push_back(Entry);
   return Chain;
 }
 
@@ -2314,11 +2310,9 @@ SparcTargetLowering::LowerF128Op(SDValue Op, SelectionDAG &DAG,
 
   if (RetTy->isFP128Ty()) {
     // Create a Stack Object to receive the return value of type f128.
-    ArgListEntry Entry;
     int RetFI = MFI.CreateStackObject(16, Align(8), false);
     RetPtr = DAG.getFrameIndex(RetFI, PtrVT);
-    Entry.Node = RetPtr;
-    Entry.Ty = PointerType::getUnqual(RetTy->getContext());
+    ArgListEntry Entry(RetPtr, PointerType::getUnqual(RetTy->getContext()));
     if (!Subtarget->is64Bit()) {
       Entry.IsSRet = true;
       Entry.IndirectType = RetTy;
@@ -3550,10 +3544,9 @@ bool SparcTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
   return isCheapToSpeculateCtlz(Ty);
 }
 
-// Override to disable global variable loading on Linux.
-void SparcTargetLowering::insertSSPDeclarations(Module &M) const {
-  if (!Subtarget->isTargetLinux())
-    return TargetLowering::insertSSPDeclarations(M);
+bool SparcTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
+                                                     EVT VT) const {
+  return Subtarget->isUA2007() && !Subtarget->useSoftFloat();
 }
 
 void SparcTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.h b/llvm/lib/Target/Sparc/SparcISelLowering.h
index 0d220f8c3d32e..e7040f770e25b 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.h
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.h
@@ -28,6 +28,8 @@ namespace llvm {
 
     bool useSoftFloat() const override;
 
+    bool softPromoteHalfType() const override { return true; }
+
     /// computeKnownBitsForTargetNode - Determine which of the bits specified
     /// in Mask are known to be either zero or one and return them in the
     /// KnownZero/KnownOne bitsets.
@@ -77,7 +79,6 @@ namespace llvm {
 
     /// Override to support customized stack guard loading.
     bool useLoadStackGuardNode(const Module &M) const override;
-    void insertSSPDeclarations(Module &M) const override;
 
     /// getSetCCResultType - Return the ISD::SETCC ValueType
     EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
@@ -177,6 +178,11 @@ namespace llvm {
 
     bool isCheapToSpeculateCttz(Type *Ty) const override;
 
+    bool enableAggressiveFMAFusion(EVT VT) const override { return true; };
+
+    bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
+                                    EVT VT) const override;
+
     bool shouldInsertFencesForAtomic(const Instruction *I) const override {
       // FIXME: We insert fences for each atomics and generate
       // sub-optimal code for PSO/TSO. (Approximately nobody uses any
diff --git a/llvm/lib/Target/Sparc/SparcInstrFormats.td b/llvm/lib/Target/Sparc/SparcInstrFormats.td
index 79c4cb2128a0f..3d3dfdc52eb5d 100644
--- a/llvm/lib/Target/Sparc/SparcInstrFormats.td
+++ b/llvm/lib/Target/Sparc/SparcInstrFormats.td
@@ -23,7 +23,6 @@ class InstSP<dag outs, dag ins, string asmstr, list<dag> pattern,
   let Pattern = pattern;
 
   let DecoderNamespace = "Sparc";
-  field bits<32> SoftFail = 0;
 
   let Itinerary = itin;
 }
diff --git a/llvm/lib/Target/Sparc/SparcInstrUAOSA.td b/llvm/lib/Target/Sparc/SparcInstrUAOSA.td
index 3a30e552e6db1..ffd4423137e3f 100644
--- a/llvm/lib/Target/Sparc/SparcInstrUAOSA.td
+++ b/llvm/lib/Target/Sparc/SparcInstrUAOSA.td
@@ -66,3 +66,15 @@ defm CXBCOND : F2_56<"cxb", 1>;
 def FPMADDX   : FourOp<"fpmaddx", 0b110111, 0b0000, DFPRegs>;
 def FPMADDXHI : FourOp<"fpmaddxhi", 0b110111, 0b0100, DFPRegs>;
 } // Predicates = [HasOSA2011]
+
+// UA2007 instruction patterns.
+let Predicates = [HasUA2007] in {
+def : Pat<(f32 (any_fma f32:$rs1, f32:$rs2, f32:$add)), (FMADDS $rs1, $rs2, $add)>;
+def : Pat<(f64 (any_fma f64:$rs1, f64:$rs2, f64:$add)), (FMADDD $rs1, $rs2, $add)>;
+def : Pat<(f32 (any_fma f32:$rs1, f32:$rs2, (fneg f32:$sub))), (FMSUBS $rs1, $rs2, $sub)>;
+def : Pat<(f64 (any_fma f64:$rs1, f64:$rs2, (fneg f64:$sub))), (FMSUBD $rs1, $rs2, $sub)>;
+def : Pat<(f32 (fneg (any_fma f32:$rs1, f32:$rs2, f32:$add))), (FNMADDS $rs1, $rs2, $add)>;
+def : Pat<(f64 (fneg (any_fma f64:$rs1, f64:$rs2, f64:$add))), (FNMADDD $rs1, $rs2, $add)>;
+def : Pat<(f32 (fneg (any_fma f32:$rs1, f32:$rs2, (fneg f32:$sub)))), (FNMSUBS $rs1, $rs2, $sub)>;
+def : Pat<(f64 (fneg (any_fma f64:$rs1, f64:$rs2, (fneg f64:$sub)))), (FNMSUBD $rs1, $rs2, $sub)>;
+} // Predicates = [HasUA2007]
diff --git a/llvm/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp b/llvm/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp
index 31b4f1196392c..0b91aba67694f 100644
--- a/llvm/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp
+++ b/llvm/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp
@@ -8,6 +8,7 @@
 
 #include "MCTargetDesc/SystemZMCTargetDesc.h"
 #include "TargetInfo/SystemZTargetInfo.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
diff --git a/llvm/lib/Target/SystemZ/SystemZCallingConv.h b/llvm/lib/Target/SystemZ/SystemZCallingConv.h
index 25f4aacd20166..f5ffbf5c04d60 100644
--- a/llvm/lib/Target/SystemZ/SystemZCallingConv.h
+++ b/llvm/lib/Target/SystemZ/SystemZCallingConv.h
@@ -29,63 +29,6 @@ namespace SystemZ {
   extern const MCPhysReg XPLINK64ArgFPRs[XPLINK64NumArgFPRs];
 } // end namespace SystemZ
 
-class SystemZCCState : public CCState {
-private:
-  /// Records whether the value was a fixed argument.
-  /// See ISD::OutputArg::IsFixed.
-  SmallVector<bool, 4> ArgIsFixed;
-
-  /// Records whether the value was widened from a short vector type.
-  SmallVector<bool, 4> ArgIsShortVector;
-
-  // Check whether ArgVT is a short vector type.
-  bool IsShortVectorType(EVT ArgVT) {
-    return ArgVT.isVector() && ArgVT.getStoreSize() <= 8;
-  }
-
-public:
-  SystemZCCState(CallingConv::ID CC, bool isVarArg, MachineFunction &MF,
-                 SmallVectorImpl<CCValAssign> &locs, LLVMContext &C)
-      : CCState(CC, isVarArg, MF, locs, C) {}
-
-  void AnalyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins,
-                              CCAssignFn Fn) {
-    // Formal arguments are always fixed.
-    ArgIsFixed.clear();
-    for (unsigned i = 0; i < Ins.size(); ++i)
-      ArgIsFixed.push_back(true);
-    // Record whether the call operand was a short vector.
-    ArgIsShortVector.clear();
-    for (unsigned i = 0; i < Ins.size(); ++i)
-      ArgIsShortVector.push_back(IsShortVectorType(Ins[i].ArgVT));
-
-    CCState::AnalyzeFormalArguments(Ins, Fn);
-  }
-
-  void AnalyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs,
-                           CCAssignFn Fn) {
-    // Record whether the call operand was a fixed argument.
-    ArgIsFixed.clear();
-    for (unsigned i = 0; i < Outs.size(); ++i)
-      ArgIsFixed.push_back(Outs[i].IsFixed);
-    // Record whether the call operand was a short vector.
-    ArgIsShortVector.clear();
-    for (unsigned i = 0; i < Outs.size(); ++i)
-      ArgIsShortVector.push_back(IsShortVectorType(Outs[i].ArgVT));
-
-    CCState::AnalyzeCallOperands(Outs, Fn);
-  }
-
-  // This version of AnalyzeCallOperands in the base class is not usable
-  // since we must provide a means of accessing ISD::OutputArg::IsFixed.
-  void AnalyzeCallOperands(const SmallVectorImpl<MVT> &Outs,
-                           SmallVectorImpl<ISD::ArgFlagsTy> &Flags,
-                           CCAssignFn Fn) = delete;
-
-  bool IsFixed(unsigned ValNo) { return ArgIsFixed[ValNo]; }
-  bool IsShortVector(unsigned ValNo) { return ArgIsShortVector[ValNo]; }
-};
-
 // Handle i128 argument types.  These need to be passed by implicit
 // reference.  This could be as simple as the following .td line:
 //    CCIfType<[i128], CCPassIndirect<i64>>,
diff --git a/llvm/lib/Target/SystemZ/SystemZCallingConv.td b/llvm/lib/Target/SystemZ/SystemZCallingConv.td
index 0ad872bcb63a7..2795de5eeeb66 100644
--- a/llvm/lib/Target/SystemZ/SystemZCallingConv.td
+++ b/llvm/lib/Target/SystemZ/SystemZCallingConv.td
@@ -16,17 +16,9 @@ class CCIfSubtarget<string F, CCAction A>
                     "getSubtarget<SystemZSubtarget>().", F),
          A>;
 
-// Match if this specific argument is a fixed (i.e. named) argument.
-class CCIfFixed<CCAction A>
-    : CCIf<"static_cast<SystemZCCState *>(&State)->IsFixed(ValNo)", A>;
-
-// Match if this specific argument is not a fixed (i.e. vararg) argument.
-class CCIfNotFixed<CCAction A>
-    : CCIf<"!(static_cast<SystemZCCState *>(&State)->IsFixed(ValNo))", A>;
-
 // Match if this specific argument was widened from a short vector type.
 class CCIfShortVector<CCAction A>
-    : CCIf<"static_cast<SystemZCCState *>(&State)->IsShortVector(ValNo)", A>;
+    : CCIf<"OrigTy->isVectorTy() && OrigTy->getPrimitiveSizeInBits() <= 64", A>;
 
 
 //===----------------------------------------------------------------------===//
@@ -79,7 +71,7 @@ def CC_SystemZ_GHC : CallingConv<[
   // Pass in STG registers: XMM1, ..., XMM6
   CCIfSubtarget<"hasVector()",
     CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
-             CCIfFixed<CCAssignToReg<[V16, V17, V18, V19, V20, V21]>>>>,
+             CCIfArgFixed<CCAssignToReg<[V16, V17, V18, V19, V20, V21]>>>>,
 
   // Fail otherwise
   CCCustom<"CC_SystemZ_GHC_Error">
@@ -125,8 +117,8 @@ def CC_SystemZ_ELF : CallingConv<[
   // during type legalization.
   CCIfSubtarget<"hasVector()",
     CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
-             CCIfFixed<CCAssignToReg<[V24, V26, V28, V30,
-                                      V25, V27, V29, V31]>>>>,
+             CCIfArgFixed<CCAssignToReg<[V24, V26, V28, V30,
+                                         V25, V27, V29, V31]>>>>,
 
   // However, sub-128 vectors which need to go on the stack occupy just a
   // single 8-byte-aligned 8-byte stack slot.  Pass as i64.
@@ -227,17 +219,17 @@ def CC_SystemZ_XPLINK64 : CallingConv<[
   // Promote f32 to f64 and bitcast to i64, if it needs to be passed in GPRs.
   // Although we assign the f32 vararg to be bitcast, it will first be promoted
   // to an f64 within convertValVTToLocVT().
-  CCIfType<[f32, f64], CCIfNotFixed<CCBitConvertToType<i64>>>,
+  CCIfType<[f32, f64], CCIfArgVarArg<CCBitConvertToType<i64>>>,
   // Pointers are always passed in full 64-bit registers.
   CCIfPtr<CCCustom<"CC_XPLINK64_Pointer">>,
   // long double, can only be passed in GPR2 and GPR3, if available,
   // hence R2Q
-  CCIfType<[f128], CCIfNotFixed<CCCustom<"CC_XPLINK64_Allocate128BitVararg">>>,
+  CCIfType<[f128], CCIfArgVarArg<CCCustom<"CC_XPLINK64_Allocate128BitVararg">>>,
   // Non fixed vector arguments are treated in the same way as long
   // doubles.
   CCIfSubtarget<"hasVector()",
     CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
-      CCIfNotFixed<CCCustom<"CC_XPLINK64_Allocate128BitVararg">>>>,
+      CCIfArgVarArg<CCCustom<"CC_XPLINK64_Allocate128BitVararg">>>>,
 
   // A SwiftSelf is passed in callee-saved R10.
   CCIfSwiftSelf<CCIfType<[i64], CCAssignToReg<[R10D]>>>,
@@ -260,22 +252,24 @@ def CC_SystemZ_XPLINK64 : CallingConv<[
   // during type legalization.
   CCIfSubtarget<"hasVector()",
     CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
-      CCIfFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>>,
+      CCIfArgFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>>,
   CCIfSubtarget<"hasVector()",
     CCIfType<[v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
-              CCIfFixed<CCAssignToRegAndStack<[V24, V25, V26, V27,
-                                               V28, V29, V30, V31], 16, 8>>>>,
+              CCIfArgFixed<CCAssignToRegAndStack<[V24, V25, V26, V27,
+                                                  V28, V29, V30, V31], 16, 8>>>>,
 
   // The first 4 named float and double arguments are passed in registers
   // FPR0-FPR6. The rest will be passed in the user area.
-  CCIfType<[f32, f64], CCIfFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>,
-  CCIfType<[f32], CCIfFixed<CCAssignToRegAndStack<[F0S, F2S, F4S, F6S], 4, 8>>>,
-  CCIfType<[f64], CCIfFixed<CCAssignToRegAndStack<[F0D, F2D, F4D, F6D], 8, 8>>>,
+  CCIfType<[f32, f64], CCIfArgFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>,
+  CCIfType<[f32],
+           CCIfArgFixed<CCAssignToRegAndStack<[F0S, F2S, F4S, F6S], 4, 8>>>,
+  CCIfType<[f64],
+           CCIfArgFixed<CCAssignToRegAndStack<[F0D, F2D, F4D, F6D], 8, 8>>>,
 
   // The first 2 long double arguments are passed in register FPR0/FPR2
   // and FPR4/FPR6. The rest will be passed in the user area.
-  CCIfType<[f128], CCIfFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>,
-  CCIfType<[f128], CCIfFixed<CCAssignToRegAndStack<[F0Q, F4Q], 16, 8>>>,
+  CCIfType<[f128], CCIfArgFixed<CCCustom<"CC_XPLINK64_Shadow_Reg">>>,
+  CCIfType<[f128], CCIfArgFixed<CCAssignToRegAndStack<[F0Q, F4Q], 16, 8>>>,
 
   // Other arguments are passed in 8-byte-aligned 8-byte stack slots.
   CCIfType<[i32, i64, f32, f64], CCAssignToStack<8, 8>>,
diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
index 6297916310803..dcefff99db25b 100644
--- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -20,7 +20,6 @@
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/Module.h"
 #include "llvm/Target/TargetMachine.h"
 
 using namespace llvm;
@@ -574,13 +573,11 @@ void SystemZELFFrameLowering::emitPrologue(MachineFunction &MF,
 
     // Call mcount (Regmask from CC AnyReg since mcount preserves all normal
     // argument registers).
-    FunctionCallee FC = MF.getFunction().getParent()->getOrInsertFunction(
-        "mcount", Type::getVoidTy(MF.getFunction().getContext()));
     const uint32_t *Mask = MF.getSubtarget<SystemZSubtarget>()
                                .getSpecialRegisters()
                                ->getCallPreservedMask(MF, CallingConv::AnyReg);
     BuildMI(MBB, MBBI, DL, ZII->get(SystemZ::CallBRASL))
-        .addGlobalAddress(dyn_cast<Function>(FC.getCallee()))
+        .addExternalSymbol("mcount")
         .addRegMask(Mask);
 
     // Reload return address from 8 bytes above stack pointer.
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index fb0a47dc9dc47..c73dc3021eb42 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1941,7 +1941,7 @@ SDValue SystemZTargetLowering::LowerFormalArguments(
 
   // Assign locations to all of the incoming arguments.
   SmallVector<CCValAssign, 16> ArgLocs;
-  SystemZCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
+  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
   CCInfo.AnalyzeFormalArguments(Ins, CC_SystemZ);
   FuncInfo->setSizeOfFnParams(CCInfo.getStackSize());
 
@@ -2251,7 +2251,7 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI,
 
   // Analyze the operands of the call, assigning locations to each operand.
   SmallVector<CCValAssign, 16> ArgLocs;
-  SystemZCCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
+  CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, Ctx);
   ArgCCInfo.AnalyzeCallOperands(Outs, CC_SystemZ);
 
   // We don't support GuaranteedTailCallOpt, only automatically-detected
@@ -2460,10 +2460,9 @@ std::pair<SDValue, SDValue> SystemZTargetLowering::makeExternalCall(
   TargetLowering::ArgListTy Args;
   Args.reserve(Ops.size());
 
-  TargetLowering::ArgListEntry Entry;
   for (SDValue Op : Ops) {
-    Entry.Node = Op;
-    Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
+    TargetLowering::ArgListEntry Entry(
+        Op, Op.getValueType().getTypeForEVT(*DAG.getContext()));
     Entry.IsSExt = shouldSignExtendTypeInLibCall(Entry.Ty, IsSigned);
     Entry.IsZExt = !Entry.IsSExt;
     Args.push_back(Entry);
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 1866962e17587..707887c59bd65 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -523,7 +523,7 @@ class SystemZTargetLowering : public TargetLowering {
                             bool MathUsed) const override {
     // Form add and sub with overflow intrinsics regardless of any extra
     // users of the math result.
-    return VT == MVT::i32 || VT == MVT::i64;
+    return VT == MVT::i32 || VT == MVT::i64 || VT == MVT::i128;
   }
 
   bool shouldConsiderGEPOffsetSplit() const override { return true; }
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
index 3e9a515bf1153..d0a549518cc45 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -188,7 +188,6 @@ def getTwoOperandOpcode : InstrMapping {
 class InstE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<2, outs, ins, asmstr, pattern> {
   field bits<16> Inst;
-  field bits<16> SoftFail = 0;
 
   let Inst = op;
 }
@@ -196,7 +195,6 @@ class InstE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstI<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<2, outs, ins, asmstr, pattern> {
   field bits<16> Inst;
-  field bits<16> SoftFail = 0;
 
   bits<8> I1;
 
@@ -207,7 +205,6 @@ class InstI<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstIE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> I1;
   bits<4> I2;
@@ -221,7 +218,6 @@ class InstIE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstMII<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> M1;
   bits<12> RI2;
@@ -236,7 +232,6 @@ class InstMII<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRIa<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<16> I2;
@@ -250,7 +245,6 @@ class InstRIa<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRIb<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<16> RI2;
@@ -264,7 +258,6 @@ class InstRIb<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRIc<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> M1;
   bits<16> RI2;
@@ -278,7 +271,6 @@ class InstRIc<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRIEa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<16> I2;
@@ -296,7 +288,6 @@ class InstRIEa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRIEb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -315,7 +306,6 @@ class InstRIEb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRIEc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<8> I2;
@@ -333,7 +323,6 @@ class InstRIEc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRIEd<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R3;
@@ -350,7 +339,6 @@ class InstRIEd<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRIEe<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R3;
@@ -368,7 +356,6 @@ class InstRIEf<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
                bits<8> I3Or = 0, bits<8> I4Or = 0>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -402,7 +389,6 @@ class InstRIEf<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
 class InstRIEg<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> M3;
@@ -419,7 +405,6 @@ class InstRIEg<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRILa<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<32> I2;
@@ -433,7 +418,6 @@ class InstRILa<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRILb<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<32> RI2;
@@ -447,7 +431,6 @@ class InstRILb<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRILc<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> M1;
   bits<32> RI2;
@@ -461,7 +444,6 @@ class InstRILc<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRIS<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<8> I2;
@@ -481,7 +463,6 @@ class InstRIS<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRR<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<2, outs, ins, asmstr, pattern> {
   field bits<16> Inst;
-  field bits<16> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -494,7 +475,6 @@ class InstRR<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRRD<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R3;
@@ -510,7 +490,6 @@ class InstRRD<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRRE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -524,7 +503,6 @@ class InstRRE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRRFa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -541,7 +519,6 @@ class InstRRFa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRRFb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -558,7 +535,6 @@ class InstRRFb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRRFc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -574,7 +550,6 @@ class InstRRFc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRRFd<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -590,7 +565,6 @@ class InstRRFd<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRRFe<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -607,7 +581,6 @@ class InstRRFe<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRRS<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R2;
@@ -628,7 +601,6 @@ class InstRRS<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRXa<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> X2;
@@ -647,7 +619,6 @@ class InstRXa<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRXb<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> M1;
   bits<4> X2;
@@ -666,7 +637,6 @@ class InstRXb<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRXE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> X2;
@@ -689,7 +659,6 @@ class InstRXE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRXF<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R3;
@@ -712,7 +681,6 @@ class InstRXF<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRXYa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> X2;
@@ -734,7 +702,6 @@ class InstRXYa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRXYb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> M1;
   bits<4> X2;
@@ -756,7 +723,6 @@ class InstRXYb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRSa<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R3;
@@ -773,7 +739,6 @@ class InstRSa<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRSb<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> M3;
@@ -790,7 +755,6 @@ class InstRSb<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRSEa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R3;
@@ -809,7 +773,6 @@ class InstRSEa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRSI<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R3;
@@ -824,7 +787,6 @@ class InstRSI<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRSLa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> B1;
   bits<12> D1;
@@ -842,7 +804,6 @@ class InstRSLa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRSLb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> B2;
@@ -862,7 +823,6 @@ class InstRSLb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRSYa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> R3;
@@ -883,7 +843,6 @@ class InstRSYa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstRSYb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> M3;
@@ -904,7 +863,6 @@ class InstRSYb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSI<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> B1;
   bits<12> D1;
@@ -919,7 +877,6 @@ class InstSI<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSIL<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> B1;
   bits<12> D1;
@@ -934,7 +891,6 @@ class InstSIL<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSIY<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> B1;
   bits<20> D1;
@@ -953,7 +909,6 @@ class InstSIY<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSMI<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> M1;
   bits<16> RI2;
@@ -971,7 +926,6 @@ class InstSMI<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSSa<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> B1;
   bits<12> D1;
@@ -990,7 +944,6 @@ class InstSSa<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSSb<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> B1;
   bits<12> D1;
@@ -1011,7 +964,6 @@ class InstSSb<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSSc<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> B1;
   bits<12> D1;
@@ -1032,7 +984,6 @@ class InstSSc<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSSd<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> B1;
@@ -1053,7 +1004,6 @@ class InstSSd<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSSe<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> B2;
@@ -1074,7 +1024,6 @@ class InstSSe<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSSf<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> B1;
   bits<12> D1;
@@ -1093,7 +1042,6 @@ class InstSSf<bits<8> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSSE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> B1;
   bits<12> D1;
@@ -1110,7 +1058,6 @@ class InstSSE<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstSSF<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> B1;
   bits<12> D1;
@@ -1130,7 +1077,6 @@ class InstSSF<bits<12> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstS<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<4, outs, ins, asmstr, pattern> {
   field bits<32> Inst;
-  field bits<32> SoftFail = 0;
 
   bits<4> B2;
   bits<12> D2;
@@ -1143,7 +1089,6 @@ class InstS<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<16> I2;
@@ -1162,7 +1107,6 @@ class InstVRIa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<8> I2;
@@ -1183,7 +1127,6 @@ class InstVRIb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V3;
@@ -1204,7 +1147,6 @@ class InstVRIc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRId<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1229,7 +1171,6 @@ class InstVRId<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIe<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1252,7 +1193,6 @@ class InstVRIe<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIf<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1277,7 +1217,6 @@ class InstVRIf<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIg<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1300,7 +1239,6 @@ class InstVRIg<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIh<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<16> I2;
@@ -1319,7 +1257,6 @@ class InstVRIh<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIi<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<4> R2;
@@ -1340,7 +1277,6 @@ class InstVRIi<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIj<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1362,7 +1298,6 @@ class InstVRIj<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIk<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1387,7 +1322,6 @@ class InstVRIk<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRIl<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1411,7 +1345,6 @@ class InstVRRa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
                bits<4> m4or = 0>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1441,7 +1374,6 @@ class InstVRRb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
                bits<4> m5or = 0>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1470,7 +1402,6 @@ class InstVRRb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
 class InstVRRc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1500,7 +1431,6 @@ class InstVRRd<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
                bits<4> m6or = 0>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1530,7 +1460,6 @@ class InstVRRd<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern,
 class InstVRRe<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1557,7 +1486,6 @@ class InstVRRe<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRRf<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<4> R2;
@@ -1576,7 +1504,6 @@ class InstVRRf<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRRg<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<16> I2;
@@ -1595,7 +1522,6 @@ class InstVRRg<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRRh<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1618,7 +1544,6 @@ class InstVRRh<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRRi<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<5> V2;
@@ -1641,7 +1566,6 @@ class InstVRRi<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRRj<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1666,7 +1590,6 @@ class InstVRRj<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRRk<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1690,7 +1613,6 @@ class InstVRRk<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRSa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<4> B2;
@@ -1713,7 +1635,6 @@ class InstVRSa<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRSb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<4> B2;
@@ -1735,7 +1656,6 @@ class InstVRSb<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRSc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<4> R1;
   bits<4> B2;
@@ -1758,7 +1678,6 @@ class InstVRSc<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRSd<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<4> B2;
@@ -1779,7 +1698,6 @@ class InstVRSd<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRV<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<5> V2;
@@ -1802,7 +1720,6 @@ class InstVRV<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVRX<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<4> X2;
@@ -1824,7 +1741,6 @@ class InstVRX<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
 class InstVSI<bits<16> op, dag outs, dag ins, string asmstr, list<dag> pattern>
   : InstSystemZ<6, outs, ins, asmstr, pattern> {
   field bits<48> Inst;
-  field bits<48> SoftFail = 0;
 
   bits<5> V1;
   bits<4> B2;
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index d76babec73dd4..afe838ac973e6 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -181,8 +181,7 @@ static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
 
 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1,
-    SDValue Src2, SDValue Size, MachinePointerInfo Op1PtrInfo,
-    MachinePointerInfo Op2PtrInfo) const {
+    SDValue Src2, SDValue Size, const CallInst *CI) const {
   SDValue CCReg;
   // Swap operands to invert CC == 1 vs. CC == 2 cases.
   if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) {
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
index c928f343e5710..5a1e0cd108e77 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
@@ -41,8 +41,7 @@ class SystemZSelectionDAGInfo : public SelectionDAGTargetInfo {
   std::pair<SDValue, SDValue>
   EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
                           SDValue Src1, SDValue Src2, SDValue Size,
-                          MachinePointerInfo Op1PtrInfo,
-                          MachinePointerInfo Op2PtrInfo) const override;
+                          const CallInst *CI) const override;
 
   std::pair<SDValue, SDValue>
   EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index f32c9bd2bdea1..2611c291abaa6 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -436,20 +436,6 @@ bool SystemZTTIImpl::isLSRCostLess(
              C2.ScaleCost, C2.SetupCost);
 }
 
-bool SystemZTTIImpl::areInlineCompatible(const Function *Caller,
-                                         const Function *Callee) const {
-  const TargetMachine &TM = getTLI()->getTargetMachine();
-
-  const FeatureBitset &CallerBits =
-      TM.getSubtargetImpl(*Caller)->getFeatureBits();
-  const FeatureBitset &CalleeBits =
-      TM.getSubtargetImpl(*Callee)->getFeatureBits();
-
-  // Support only equal feature bitsets. Restriction should be relaxed in the
-  // future to allow inlining when callee's bits are subset of the caller's.
-  return CallerBits == CalleeBits;
-}
-
 unsigned SystemZTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
   bool Vector = (ClassID == 1);
   if (!Vector)
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index dc5736e8af009..fc681dec1859a 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -65,9 +65,6 @@ class SystemZTTIImpl final : public BasicTTIImplBase<SystemZTTIImpl> {
   bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
                      const TargetTransformInfo::LSRCost &C2) const override;
 
-  bool areInlineCompatible(const Function *Caller,
-                           const Function *Callee) const override;
-
   /// @}
 
   /// \name Vector TTI Implementations
diff --git a/llvm/lib/Target/TargetLoweringObjectFile.cpp b/llvm/lib/Target/TargetLoweringObjectFile.cpp
index 9b03e85ca45bf..28495e7c57195 100644
--- a/llvm/lib/Target/TargetLoweringObjectFile.cpp
+++ b/llvm/lib/Target/TargetLoweringObjectFile.cpp
@@ -151,17 +151,17 @@ void TargetLoweringObjectFile::emitCGProfileMetadata(MCStreamer &Streamer,
   SmallVector<Module::ModuleFlagEntry, 8> ModuleFlags;
   M.getModuleFlagsMetadata(ModuleFlags);
 
-  MDNode *CFGProfile = nullptr;
+  MDNode *CGProfile = nullptr;
 
   for (const auto &MFE : ModuleFlags) {
     StringRef Key = MFE.Key->getString();
     if (Key == "CG Profile") {
-      CFGProfile = cast<MDNode>(MFE.Val);
+      CGProfile = cast<MDNode>(MFE.Val);
       break;
     }
   }
 
-  if (!CFGProfile)
+  if (!CGProfile)
     return;
 
   auto GetSym = [this](const MDOperand &MDO) -> MCSymbol * {
@@ -174,7 +174,7 @@ void TargetLoweringObjectFile::emitCGProfileMetadata(MCStreamer &Streamer,
     return TM->getSymbol(F);
   };
 
-  for (const auto &Edge : CFGProfile->operands()) {
+  for (const auto &Edge : CGProfile->operands()) {
     MDNode *E = cast<MDNode>(Edge);
     const MCSymbol *From = GetSym(E->getOperand(0));
     const MCSymbol *To = GetSym(E->getOperand(1));
diff --git a/llvm/lib/Target/VE/Disassembler/VEDisassembler.cpp b/llvm/lib/Target/VE/Disassembler/VEDisassembler.cpp
index 88200c5fc97eb..d7e1666a7417d 100644
--- a/llvm/lib/Target/VE/Disassembler/VEDisassembler.cpp
+++ b/llvm/lib/Target/VE/Disassembler/VEDisassembler.cpp
@@ -15,6 +15,7 @@
 #include "VE.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -22,6 +23,7 @@
 #include "llvm/Support/Compiler.h"
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "ve-disassembler"
 
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index 9e8f400256198..2cfdc751a55e0 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -1649,14 +1649,11 @@ SDValue VETargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
 
   // Prepare arguments
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-  Entry.Node = Size;
-  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
-  Args.push_back(Entry);
+  Args.emplace_back(Size, Size.getValueType().getTypeForEVT(*DAG.getContext()));
   if (NeedsAlign) {
-    Entry.Node = DAG.getConstant(~(Alignment->value() - 1ULL), DL, VT);
-    Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
-    Args.push_back(Entry);
+    SDValue Align = DAG.getConstant(~(Alignment->value() - 1ULL), DL, VT);
+    Args.emplace_back(Align,
+                      Align.getValueType().getTypeForEVT(*DAG.getContext()));
   }
   Type *RetTy = Type::getVoidTy(*DAG.getContext());
 
diff --git a/llvm/lib/Target/VE/VEInstrFormats.td b/llvm/lib/Target/VE/VEInstrFormats.td
index a2d2ae929dbdc..887c545ece6e8 100644
--- a/llvm/lib/Target/VE/VEInstrFormats.td
+++ b/llvm/lib/Target/VE/VEInstrFormats.td
@@ -55,7 +55,6 @@ class InstVE<dag outs, dag ins, string asmstr, list<dag> pattern>
   let TSFlags{4-2} = !add(VE_VLIndex, VE_VLWithMask);
 
   let DecoderNamespace = "VE";
-  field bits<64> SoftFail = 0;
 }
 
 //-----------------------------------------------------------------------------
diff --git a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index 80df4ed2563b8..45bbf128ce0b7 100644
--- a/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/llvm/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -220,7 +220,6 @@ static MCSymbolWasm *getOrCreateFunctionTableSymbol(MCContext &Ctx,
     Sym = static_cast<MCSymbolWasm *>(Ctx.getOrCreateSymbol(Name));
     Sym->setFunctionTable(Is64);
     // The default function table is synthesized by the linker.
-    Sym->setUndefined();
   }
   return Sym;
 }
diff --git a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
index 2a398d4e6333d..fa6086c7db076 100644
--- a/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
+++ b/llvm/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
@@ -26,7 +26,6 @@
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolWasm.h"
 #include "llvm/MC/TargetRegistry.h"
-#include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/LEB128.h"
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
index eecef31455512..65035df2df5aa 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
@@ -13,6 +13,7 @@
 
 #include "MCTargetDesc/WebAssemblyFixupKinds.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/MC/MCAsmBackend.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCExpr.h"
@@ -36,6 +37,7 @@ class WebAssemblyAsmBackend final : public MCAsmBackend {
       : MCAsmBackend(llvm::endianness::little), Is64Bit(Is64Bit),
         IsEmscripten(IsEmscripten) {}
 
+  std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
   MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
 
   void applyFixup(const MCFragment &, const MCFixup &, const MCValue &Target,
@@ -48,6 +50,18 @@ class WebAssemblyAsmBackend final : public MCAsmBackend {
                     const MCSubtargetInfo *STI) const override;
 };
 
+std::optional<MCFixupKind>
+WebAssemblyAsmBackend::getFixupKind(StringRef Name) const {
+  unsigned Type = llvm::StringSwitch<unsigned>(Name)
+#define WASM_RELOC(NAME, ID) .Case(#NAME, ID)
+#include "llvm/BinaryFormat/WasmRelocs.def"
+#undef WASM_RELOC
+                      .Default(-1u);
+  if (Type != -1u)
+    return static_cast<MCFixupKind>(FirstLiteralRelocationKind + Type);
+  return std::nullopt;
+}
+
 MCFixupKindInfo
 WebAssemblyAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
   const static MCFixupKindInfo Infos[WebAssembly::NumTargetFixupKinds] = {
@@ -61,6 +75,11 @@ WebAssemblyAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_uleb128_i64", 0, 10 * 8, 0},
   };
 
+  // Fixup kinds from raw relocation types and .reloc directives force
+  // relocations and do not use these fields.
+  if (mc::isRelocation(Kind))
+    return {};
+
   if (Kind < FirstTargetFixupKind)
     return MCAsmBackend::getFixupKindInfo(Kind);
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
index ec95e86e4fe3d..2666342d0c7b9 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
@@ -912,6 +912,8 @@ bool WebAssemblyFastISel::selectCall(const Instruction *I) {
 
   if (!IsVoid)
     updateValueMap(Call, ResultReg);
+
+  diagnoseDontCall(*Call);
   return true;
 }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
index b03b35028c69c..fc852d0a12e14 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
@@ -136,6 +136,15 @@ static APInt encodeFunctionSignature(SelectionDAG *DAG, SDLoc &DL,
     if (VT == MVT::f64) {
       return wasm::ValType::F64;
     }
+    if (VT == MVT::externref) {
+      return wasm::ValType::EXTERNREF;
+    }
+    if (VT == MVT::funcref) {
+      return wasm::ValType::FUNCREF;
+    }
+    if (VT == MVT::exnref) {
+      return wasm::ValType::EXNREF;
+    }
     LLVM_DEBUG(errs() << "Unhandled type for llvm.wasm.ref.test.func: " << VT
                       << "\n");
     llvm_unreachable("Unhandled type for llvm.wasm.ref.test.func");
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 3f80b2ab2bd6d..c6c2d0cfccb64 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -246,6 +246,8 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
                    MVT::v2f64})
       setOperationAction(ISD::SPLAT_VECTOR, T, Legal);
 
+    setOperationAction(ISD::AVGCEILU, {MVT::v8i16, MVT::v16i8}, Legal);
+
     // Custom lowering since wasm shifts must have a scalar shift amount
     for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
       for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
@@ -1309,7 +1311,7 @@ WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
       OutVal = FINode;
     }
     // Count the number of fixed args *after* legalization.
-    NumFixedArgs += Out.IsFixed;
+    NumFixedArgs += !Out.Flags.isVarArg();
   }
 
   bool IsVarArg = CLI.IsVarArg;
@@ -1320,18 +1322,21 @@ WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
   // signature They are necessary to match callee and caller signature for
   // indirect call.
   if (CallConv == CallingConv::Swift) {
+    Type *PtrTy = PointerType::getUnqual(*DAG.getContext());
     if (!HasSwiftSelfArg) {
       NumFixedArgs++;
-      ISD::OutputArg Arg;
-      Arg.Flags.setSwiftSelf();
+      ISD::ArgFlagsTy Flags;
+      Flags.setSwiftSelf();
+      ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
       CLI.Outs.push_back(Arg);
       SDValue ArgVal = DAG.getUNDEF(PtrVT);
       CLI.OutVals.push_back(ArgVal);
     }
     if (!HasSwiftErrorArg) {
       NumFixedArgs++;
-      ISD::OutputArg Arg;
-      Arg.Flags.setSwiftError();
+      ISD::ArgFlagsTy Flags;
+      Flags.setSwiftError();
+      ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
       CLI.Outs.push_back(Arg);
       SDValue ArgVal = DAG.getUNDEF(PtrVT);
       CLI.OutVals.push_back(ArgVal);
@@ -1503,7 +1508,7 @@ SDValue WebAssemblyTargetLowering::LowerReturn(
   for (const ISD::OutputArg &Out : Outs) {
     assert(!Out.Flags.isByVal() && "byval is not valid for return values");
     assert(!Out.Flags.isNest() && "nest is not valid for return values");
-    assert(Out.IsFixed && "non-fixed return value is not valid");
+    assert(!Out.Flags.isVarArg() && "non-fixed return value is not valid");
     if (Out.Flags.isInAlloca())
       fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
     if (Out.Flags.isInConsecutiveRegs())
@@ -3383,8 +3388,56 @@ static SDValue TryMatchTrue(SDNode *N, EVT VecVT, SelectionDAG &DAG) {
   return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));
 }
 
+/// Try to convert a i128 comparison to a v16i8 comparison before type
+/// legalization splits it up into chunks
+static SDValue
+combineVectorSizedSetCCEquality(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+                                const WebAssemblySubtarget *Subtarget) {
+
+  SDLoc DL(N);
+  SDValue X = N->getOperand(0);
+  SDValue Y = N->getOperand(1);
+  EVT VT = N->getValueType(0);
+  EVT OpVT = X.getValueType();
+
+  SelectionDAG &DAG = DCI.DAG;
+  if (DCI.DAG.getMachineFunction().getFunction().hasFnAttribute(
+          Attribute::NoImplicitFloat))
+    return SDValue();
+
+  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
+  // We're looking for an oversized integer equality comparison with SIMD
+  if (!OpVT.isScalarInteger() || !OpVT.isByteSized() || OpVT != MVT::i128 ||
+      !Subtarget->hasSIMD128() || !isIntEqualitySetCC(CC))
+    return SDValue();
+
+  // Don't perform this combine if constructing the vector will be expensive.
+  auto IsVectorBitCastCheap = [](SDValue X) {
+    X = peekThroughBitcasts(X);
+    return isa<ConstantSDNode>(X) || X.getOpcode() == ISD::LOAD;
+  };
+
+  if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
+    return SDValue();
+
+  SDValue VecX = DAG.getBitcast(MVT::v16i8, X);
+  SDValue VecY = DAG.getBitcast(MVT::v16i8, Y);
+  SDValue Cmp = DAG.getSetCC(DL, MVT::v16i8, VecX, VecY, CC);
+
+  SDValue Intr =
+      DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
+                  {DAG.getConstant(CC == ISD::SETEQ ? Intrinsic::wasm_alltrue
+                                                    : Intrinsic::wasm_anytrue,
+                                   DL, MVT::i32),
+                   Cmp});
+
+  return DAG.getSetCC(DL, VT, Intr, DAG.getConstant(0, DL, MVT::i32),
+                      ISD::SETNE);
+}
+
 static SDValue performSETCCCombine(SDNode *N,
-                                   TargetLowering::DAGCombinerInfo &DCI) {
+                                   TargetLowering::DAGCombinerInfo &DCI,
+                                   const WebAssemblySubtarget *Subtarget) {
   if (!DCI.isBeforeLegalize())
     return SDValue();
 
@@ -3392,6 +3445,9 @@ static SDValue performSETCCCombine(SDNode *N,
   if (!VT.isScalarInteger())
     return SDValue();
 
+  if (SDValue V = combineVectorSizedSetCCEquality(N, DCI, Subtarget))
+    return V;
+
   SDValue LHS = N->getOperand(0);
   if (LHS->getOpcode() != ISD::BITCAST)
     return SDValue();
@@ -3571,7 +3627,7 @@ WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::BITCAST:
     return performBitcastCombine(N, DCI);
   case ISD::SETCC:
-    return performSETCCCombine(N, DCI);
+    return performSETCCCombine(N, DCI, Subtarget);
   case ISD::VECTOR_SHUFFLE:
     return performVECTOR_SHUFFLECombine(N, DCI);
   case ISD::SIGN_EXTEND:
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index 143298b700928..f06f8d5174e3e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -1144,6 +1144,9 @@ def : Pat<(wasm_shr_u
               (vec.splat (i32 1))),
             (i32 1)),
           (inst $lhs, $rhs)>;
+
+def : Pat<(vec.vt (avgceilu (vec.vt V128:$lhs), (vec.vt V128:$rhs))),
+          (inst $lhs, $rhs)>;
 }
 
 // Widening dot product: i32x4.dot_i16x8_s
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index 4548a7520b3b4..45b0e7dc12263 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -533,8 +533,8 @@ struct StaticLibcallNameMap {
     // different libcalls.
     RTLIB::RuntimeLibcallsInfo RTCI(TT);
     for (RTLIB::Libcall LC : RTLIB::libcalls()) {
-      const char *NameLibcall = RTCI.getLibcallName(LC);
-      if (NameLibcall != nullptr &&
+      StringRef NameLibcall = RTCI.getLibcallName(LC);
+      if (!NameLibcall.empty() &&
           getRuntimeLibcallSignatures().Table[LC] != unsupported) {
         assert(!Map.contains(NameLibcall) &&
                "duplicate libcall names in name map");
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 52e706514226b..08fb7586d215e 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -147,7 +147,8 @@ WebAssemblyTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
 
   Options.AllowOverlappingLoads = true;
 
-  // TODO: Teach WebAssembly backend about load v128.
+  if (ST->hasSIMD128())
+    Options.LoadSizes.push_back(16);
 
   Options.LoadSizes.append({8, 4, 2, 1});
   Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
index 42d1271f29c40..890486778e700 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
@@ -113,7 +113,6 @@ MCSymbolWasm *WebAssembly::getOrCreateFunctionTableSymbol(
     Sym = static_cast<MCSymbolWasm *>(Ctx.getOrCreateSymbol(Name));
     Sym->setFunctionTable(is64);
     // The default function table is synthesized by the linker.
-    Sym->setUndefined();
   }
   // MVP object files can't have symtab entries for tables.
   if (!(Subtarget && Subtarget->hasCallIndirectOverlong()))
diff --git a/llvm/lib/Target/X86/GISel/X86CallLowering.cpp b/llvm/lib/Target/X86/GISel/X86CallLowering.cpp
index c0a6035b792dc..c0b9339e9bc34 100644
--- a/llvm/lib/Target/X86/GISel/X86CallLowering.cpp
+++ b/llvm/lib/Target/X86/GISel/X86CallLowering.cpp
@@ -69,13 +69,13 @@ struct X86OutgoingValueAssigner : public CallLowering::OutgoingValueAssigner {
                  CCValAssign::LocInfo LocInfo,
                  const CallLowering::ArgInfo &Info, ISD::ArgFlagsTy Flags,
                  CCState &State) override {
-    bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, State);
+    bool Res = AssignFn(ValNo, ValVT, LocVT, LocInfo, Flags, Info.Ty, State);
     StackSize = State.getStackSize();
 
     static const MCPhysReg XMMArgRegs[] = {X86::XMM0, X86::XMM1, X86::XMM2,
                                            X86::XMM3, X86::XMM4, X86::XMM5,
                                            X86::XMM6, X86::XMM7};
-    if (!Info.IsFixed)
+    if (Flags.isVarArg())
       NumXMMRegs = State.getFirstUnallocated(XMMArgRegs);
 
     return Res;
@@ -363,7 +363,8 @@ bool X86CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
                                      Info.CallConv, Info.IsVarArg))
     return false;
 
-  bool IsFixed = Info.OrigArgs.empty() ? true : Info.OrigArgs.back().IsFixed;
+  bool IsFixed =
+      Info.OrigArgs.empty() ? true : !Info.OrigArgs.back().Flags[0].isVarArg();
   if (STI.is64Bit() && !IsFixed && !STI.isCallingConvWin64(Info.CallConv)) {
     // From AMD64 ABI document:
     // For calls that may call functions that use varargs or stdargs
diff --git a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.cpp b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.cpp
index 817e88d8a0bc0..e2a1bbf383b3c 100644
--- a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.cpp
+++ b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.cpp
@@ -36,11 +36,31 @@ void X86InstrPostProcess::setMemBarriers(std::unique_ptr<Instruction> &Inst,
   }
 }
 
+void X86InstrPostProcess::useStackEngine(std::unique_ptr<Instruction> &Inst,
+                                         const MCInst &MCI) {
+  // TODO(boomanaiden154): We currently do not handle PUSHF/POPF because we
+  // have not done the necessary benchmarking to see if they are also
+  // optimized by the stack engine.
+  // TODO: We currently just remove all RSP writes from stack operations. This
+  // is not fully correct because we do not model sync uops which will
+  // delay subsequent rsp using non-stack instructions.
+  if (X86::isPOP(MCI.getOpcode()) || X86::isPUSH(MCI.getOpcode())) {
+    auto *StackRegisterDef =
+        llvm::find_if(Inst->getDefs(), [](const WriteState &State) {
+          return State.getRegisterID() == X86::RSP;
+        });
+    assert(
+        StackRegisterDef != Inst->getDefs().end() &&
+        "Expected push instruction to implicitly use stack pointer register.");
+    Inst->getDefs().erase(StackRegisterDef);
+  }
+}
+
 void X86InstrPostProcess::postProcessInstruction(
     std::unique_ptr<Instruction> &Inst, const MCInst &MCI) {
-  // Currently, we only modify certain instructions' IsALoadBarrier and
-  // IsAStoreBarrier flags.
+  // Set IsALoadBarrier and IsAStoreBarrier flags.
   setMemBarriers(Inst, MCI);
+  useStackEngine(Inst, MCI);
 }
 
 } // namespace mca
diff --git a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h
index 4a83ba848dd88..c5459e42dfc9f 100644
--- a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h
+++ b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h
@@ -28,6 +28,11 @@ class X86InstrPostProcess : public InstrPostProcess {
   /// as load and store barriers.
   void setMemBarriers(std::unique_ptr<Instruction> &Inst, const MCInst &MCI);
 
+  /// Called within X86InstrPostPorcess to remove some rsp read operands
+  /// on stack instructions to better simulate the stack engine. We currently
+  /// do not model features of the stack engine like sync uops.
+  void useStackEngine(std::unique_ptr<Instruction> &Inst, const MCInst &MCI);
+
 public:
   X86InstrPostProcess(const MCSubtargetInfo &STI, const MCInstrInfo &MCII)
       : InstrPostProcess(STI, MCII) {}
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
index 0dabd98a38f44..25fcf81587399 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
@@ -17,6 +17,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCSectionMachO.h"
+#include "llvm/MC/MCSymbolMachO.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
@@ -354,8 +355,7 @@ bool X86MachObjectWriter::recordScatteredRelocation(MachObjectWriter *Writer,
   unsigned Type = MachO::GENERIC_RELOC_VANILLA;
 
   // See <reloc.h>.
-  const MCSymbol *A = Target.getAddSym();
-
+  auto *A = static_cast<const MCSymbolMachO *>(Target.getAddSym());
   if (!A->getFragment()) {
     reportError(Fixup.getLoc(),
                 "symbol '" + A->getName() +
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 990b381341f07..9cfe081b87109 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -355,6 +355,9 @@ def FeatureCCMP : SubtargetFeature<"ccmp", "HasCCMP", "true",
                                    "Support conditional cmp & test instructions">;
 def FeatureNF : SubtargetFeature<"nf", "HasNF", "true",
                                  "Support status flags update suppression">;
+// FeatureCF is not enabled by default for APXF and targets that support APXF
+// due to performance reason, though it is part of APXF. Users need to enable it
+// manually.
 def FeatureCF : SubtargetFeature<"cf", "HasCF", "true",
                                  "Support conditional faulting">;
 def FeatureZU : SubtargetFeature<"zu", "HasZU", "true",
@@ -1169,7 +1172,6 @@ def ProcessorFeatures {
                                                   FeaturePPX,
                                                   FeatureNDD,
                                                   FeatureNF,
-                                                  FeatureCF,
                                                   FeatureMOVRS,
                                                   FeatureAMXMOVRS,
                                                   FeatureAMXAVX512,
@@ -1291,7 +1293,9 @@ def ProcessorFeatures {
   list<SubtargetFeature> ADLAdditionalTuning = [TuningPERMFalseDeps,
                                                 TuningPreferMovmskOverVTest,
                                                 TuningFastImmVectorShift];
-  list<SubtargetFeature> ADLTuning = !listconcat(SKLTuning, ADLAdditionalTuning);
+  list<SubtargetFeature> ADLRemoveTuning = [TuningPOPCNTFalseDeps];
+  list<SubtargetFeature> ADLTuning =
+      !listremove(!listconcat(SKLTuning, ADLAdditionalTuning), ADLRemoveTuning);
   list<SubtargetFeature> ADLFeatures =
     !listconcat(TRMFeatures, ADLAdditionalFeatures);
 
diff --git a/llvm/lib/Target/X86/X86CallingConv.h b/llvm/lib/Target/X86/X86CallingConv.h
index 191e0fa619b24..8e37f345ed9e6 100644
--- a/llvm/lib/Target/X86/X86CallingConv.h
+++ b/llvm/lib/Target/X86/X86CallingConv.h
@@ -22,10 +22,10 @@ namespace llvm {
 
 bool RetCC_X86(unsigned ValNo, MVT ValVT, MVT LocVT,
                CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
-               CCState &State);
+               Type *OrigTy, CCState &State);
 
 bool CC_X86(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
-            ISD::ArgFlagsTy ArgFlags, CCState &State);
+            ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State);
 
 } // End llvm namespace
 
diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index 067bd43c8c866..f007886115d35 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -3323,6 +3323,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
     return false;
 
   SmallVector<MVT, 16> OutVTs;
+  SmallVector<Type *, 16> ArgTys;
   SmallVector<Register, 16> ArgRegs;
 
   // If this is a constant i1/i8/i16 argument, promote to i32 to avoid an extra
@@ -3369,6 +3370,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
 
     ArgRegs.push_back(ResultReg);
     OutVTs.push_back(VT);
+    ArgTys.push_back(Val->getType());
   }
 
   // Analyze operands of the call, assigning locations to each operand.
@@ -3379,7 +3381,7 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) {
   if (IsWin64)
     CCInfo.AllocateStack(32, Align(8));
 
-  CCInfo.AnalyzeCallOperands(OutVTs, OutFlags, CC_X86);
+  CCInfo.AnalyzeCallOperands(OutVTs, OutFlags, ArgTys, CC_X86);
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 95ed5908e2314..cba7843d53e3f 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/WinEHFuncInfo.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/EHPersonalities.h"
@@ -2678,7 +2679,7 @@ StackOffset X86FrameLowering::getFrameIndexReference(const MachineFunction &MF,
   // object.
   // We need to factor in additional offsets applied during the prologue to the
   // frame, base, and stack pointer depending on which is used.
-  int Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea();
+  int64_t Offset = MFI.getObjectOffset(FI) - getOffsetOfLocalArea();
   const X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
   unsigned CSSize = X86FI->getCalleeSavedFrameSize();
   uint64_t StackSize = MFI.getStackSize();
@@ -4212,6 +4213,14 @@ void X86FrameLowering::processFunctionBeforeFrameFinalized(
   // emitPrologue if it gets called and emits CFI.
   MF.setHasWinCFI(false);
 
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  // If the frame is big enough that we might need to scavenge a register to
+  // handle huge offsets, reserve a stack slot for that now.
+  if (!isInt<32>(MFI.estimateStackSize(MF))) {
+    int FI = MFI.CreateStackObject(SlotSize, Align(SlotSize), false);
+    RS->addScavengingFrameIndex(FI);
+  }
+
   // If we are using Windows x64 CFI, ensure that the stack is always 8 byte
   // aligned. The format doesn't support misaligned stack adjustments.
   if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI())
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index f366094c31958..19131fbd4102b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2756,8 +2756,10 @@ X86TargetLowering::getPreferredVectorAction(MVT VT) const {
       !Subtarget.hasBWI())
     return TypeSplitVector;
 
+  // Since v8f16 is legal, widen anything over v4f16.
   if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
-      !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16)
+      VT.getVectorNumElements() <= 4 && !Subtarget.hasF16C() &&
+      VT.getVectorElementType() == MVT::f16)
     return TypeSplitVector;
 
   if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
@@ -15419,18 +15421,18 @@ static SDValue lowerShuffleAsLanePermuteAndPermute(
         return SDValue();
     }
 
-    // Avoid returning the same shuffle operation. For example,
-    // t7: v16i16 = vector_shuffle<8,9,10,11,4,5,6,7,0,1,2,3,12,13,14,15> t5,
-    //                             undef:v16i16
-    if (CrossLaneMask == Mask || InLaneMask == Mask)
-      return SDValue();
-
     // Simplify CrossLaneMask based on the actual demanded elements.
     if (V1.hasOneUse())
       for (int i = 0; i != NumElts; ++i)
         if (!DemandedCrossLane[i])
           CrossLaneMask[i] = SM_SentinelUndef;
 
+    // Avoid returning the same shuffle operation. For example,
+    // t7: v16i16 = vector_shuffle<8,9,10,11,4,5,6,7,0,1,2,3,12,13,14,15> t5,
+    //                             undef:v16i16
+    if (CrossLaneMask == Mask || InLaneMask == Mask)
+      return SDValue();
+
     SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask);
     return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT),
                                 InLaneMask);
@@ -22219,9 +22221,8 @@ SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
 
       In = DAG.getBitcast(MVT::i16, In);
       TargetLowering::ArgListTy Args;
-      TargetLowering::ArgListEntry Entry;
-      Entry.Node = In;
-      Entry.Ty = EVT(MVT::i16).getTypeForEVT(*DAG.getContext());
+      TargetLowering::ArgListEntry Entry(
+          In, EVT(MVT::i16).getTypeForEVT(*DAG.getContext()));
       Entry.IsSExt = false;
       Entry.IsZExt = true;
       Args.push_back(Entry);
@@ -22318,9 +22319,8 @@ SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
     Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
 
     TargetLowering::ArgListTy Args;
-    TargetLowering::ArgListEntry Entry;
-    Entry.Node = In;
-    Entry.Ty = EVT(SVT).getTypeForEVT(*DAG.getContext());
+    TargetLowering::ArgListEntry Entry(
+        In, EVT(SVT).getTypeForEVT(*DAG.getContext()));
     Entry.IsSExt = false;
     Entry.IsZExt = true;
     Args.push_back(Entry);
@@ -23185,43 +23185,51 @@ static SDValue LowerVectorAllEqual(const SDLoc &DL, SDValue LHS, SDValue RHS,
 
 // Check whether an AND/OR'd reduction tree is PTEST-able, or if we can fallback
 // to CMP(MOVMSK(PCMPEQB(X,Y))).
-static SDValue MatchVectorAllEqualTest(SDValue LHS, SDValue RHS,
+static SDValue MatchVectorAllEqualTest(SDValue OrigLHS, SDValue OrigRHS,
                                        ISD::CondCode CC, const SDLoc &DL,
                                        const X86Subtarget &Subtarget,
                                        SelectionDAG &DAG,
                                        X86::CondCode &X86CC) {
-  assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Unsupported ISD::CondCode");
+  SDValue Op = OrigLHS;
 
-  bool CmpNull = isNullConstant(RHS);
-  bool CmpAllOnes = isAllOnesConstant(RHS);
-  if (!CmpNull && !CmpAllOnes)
-    return SDValue();
+  bool CmpNull;
+  APInt Mask;
+  if (CC == ISD::SETEQ || CC == ISD::SETNE) {
+    CmpNull = isNullConstant(OrigRHS);
+    if (!CmpNull && !isAllOnesConstant(OrigRHS))
+      return SDValue();
 
-  SDValue Op = LHS;
-  if (!Subtarget.hasSSE2() || !Op->hasOneUse())
-    return SDValue();
+    if (!Subtarget.hasSSE2() || !Op->hasOneUse())
+      return SDValue();
 
-  // Check whether we're masking/truncating an OR-reduction result, in which
-  // case track the masked bits.
-  // TODO: Add CmpAllOnes support.
-  APInt Mask = APInt::getAllOnes(Op.getScalarValueSizeInBits());
-  if (CmpNull) {
-    switch (Op.getOpcode()) {
-    case ISD::TRUNCATE: {
-      SDValue Src = Op.getOperand(0);
-      Mask = APInt::getLowBitsSet(Src.getScalarValueSizeInBits(),
-                                  Op.getScalarValueSizeInBits());
-      Op = Src;
-      break;
-    }
-    case ISD::AND: {
-      if (auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
-        Mask = Cst->getAPIntValue();
-        Op = Op.getOperand(0);
+    // Check whether we're masking/truncating an OR-reduction result, in which
+    // case track the masked bits.
+    // TODO: Add CmpAllOnes support.
+    Mask = APInt::getAllOnes(Op.getScalarValueSizeInBits());
+    if (CmpNull) {
+      switch (Op.getOpcode()) {
+      case ISD::TRUNCATE: {
+        SDValue Src = Op.getOperand(0);
+        Mask = APInt::getLowBitsSet(Src.getScalarValueSizeInBits(),
+                                    Op.getScalarValueSizeInBits());
+        Op = Src;
+        break;
+      }
+      case ISD::AND: {
+        if (auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+          Mask = Cst->getAPIntValue();
+          Op = Op.getOperand(0);
+        }
+        break;
+      }
       }
-      break;
-    }
     }
+  } else if (CC == ISD::SETGT && isAllOnesConstant(OrigRHS)) {
+    CC = ISD::SETEQ;
+    CmpNull = true;
+    Mask = APInt::getSignMask(Op.getScalarValueSizeInBits());
+  } else {
+    return SDValue();
   }
 
   ISD::NodeType LogicOp = CmpNull ? ISD::OR : ISD::AND;
@@ -30049,7 +30057,6 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
   SDValue InChain = DAG.getEntryNode();
 
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
   for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i) {
     EVT ArgVT = Op->getOperand(i).getValueType();
     assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
@@ -30058,13 +30065,9 @@ SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) cons
     int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
     MachinePointerInfo MPI =
         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
-    Entry.Node = StackPtr;
     InChain =
         DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr, MPI, Align(16));
-    Entry.Ty = PointerType::get(*DAG.getContext(), 0);
-    Entry.IsSExt = false;
-    Entry.IsZExt = false;
-    Args.push_back(Entry);
+    Args.emplace_back(StackPtr, PointerType::get(*DAG.getContext(), 0));
   }
 
   SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
@@ -33087,13 +33090,7 @@ static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
 
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-
-  Entry.Node = Arg;
-  Entry.Ty = ArgTy;
-  Entry.IsSExt = false;
-  Entry.IsZExt = false;
-  Args.push_back(Entry);
+  Args.emplace_back(Arg, ArgTy);
 
   bool isF64 = ArgVT == MVT::f64;
   // Only optimize x86_64 for now. i386 is a bit messy. For f32,
@@ -44206,8 +44203,12 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
     }
       // Conversions.
       // TODO: Add more CVT opcodes when we have test coverage.
-    case X86ISD::CVTTP2SI:
     case X86ISD::CVTTP2UI: {
+      if (!Subtarget.hasVLX())
+        break;
+      [[fallthrough]];
+    }
+    case X86ISD::CVTTP2SI: {
       if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f16 &&
           !Subtarget.hasVLX())
         break;
@@ -45163,6 +45164,9 @@ bool X86TargetLowering::canCreateUndefOrPoisonForTargetNode(
   case X86ISD::PCMPEQ:
   case X86ISD::PCMPGT:
     return false;
+  // SSE signbit extraction.
+  case X86ISD::MOVMSK:
+    return false;
   case ISD::INTRINSIC_WO_CHAIN:
     switch (Op->getConstantOperandVal(0)) {
     case Intrinsic::x86_sse2_pmadd_wd:
@@ -56278,14 +56282,16 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
     if (SDValue V = combineVectorSizedSetCCEquality(VT, LHS, RHS, CC, DL, DAG,
                                                     Subtarget))
       return V;
+  }
 
-    if (VT == MVT::i1) {
-      X86::CondCode X86CC;
-      if (SDValue V =
-              MatchVectorAllEqualTest(LHS, RHS, CC, DL, Subtarget, DAG, X86CC))
-        return DAG.getNode(ISD::TRUNCATE, DL, VT, getSETCC(X86CC, V, DL, DAG));
-    }
+  if (VT == MVT::i1) {
+    X86::CondCode X86CC;
+    if (SDValue V =
+            MatchVectorAllEqualTest(LHS, RHS, CC, DL, Subtarget, DAG, X86CC))
+      return DAG.getNode(ISD::TRUNCATE, DL, VT, getSETCC(X86CC, V, DL, DAG));
+  }
 
+  if (CC == ISD::SETNE || CC == ISD::SETEQ) {
     if (OpVT.isScalarInteger()) {
       // cmpeq(or(X,Y),X) --> cmpeq(and(~X,Y),0)
       // cmpne(or(X,Y),X) --> cmpne(and(~X,Y),0)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 547b2210fdbf0..97d3b6e2420d6 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1591,7 +1591,6 @@ namespace llvm {
     bool useLoadStackGuardNode(const Module &M) const override;
     bool useStackGuardXorFP() const override;
     void insertSSPDeclarations(Module &M) const override;
-    Value *getSDagStackGuard(const Module &M) const override;
     Function *getSSPStackGuardCheck(const Module &M) const override;
     SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
                                 const SDLoc &DL) const override;
@@ -1663,8 +1662,8 @@ namespace llvm {
     /// instructions/intrinsics.
     bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
                               ArrayRef<ShuffleVectorInst *> Shuffles,
-                              ArrayRef<unsigned> Indices,
-                              unsigned Factor) const override;
+                              ArrayRef<unsigned> Indices, unsigned Factor,
+                              const APInt &GapMask) const override;
 
     /// Lower interleaved store(s) into target specific
     /// instructions/intrinsics.
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 7c594d04f3c23..1c745a338a617 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -632,15 +632,6 @@ void X86TargetLowering::insertSSPDeclarations(Module &M) const {
   TargetLowering::insertSSPDeclarations(M);
 }
 
-Value *X86TargetLowering::getSDagStackGuard(const Module &M) const {
-  // MSVC CRT has a global variable holding security cookie.
-  if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
-      Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
-    return M.getGlobalVariable("__security_cookie");
-  }
-  return TargetLowering::getSDagStackGuard(M);
-}
-
 Function *X86TargetLowering::getSSPStackGuardCheck(const Module &M) const {
   // MSVC CRT has a function to validate security cookie.
   if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
diff --git a/llvm/lib/Target/X86/X86InterleavedAccess.cpp b/llvm/lib/Target/X86/X86InterleavedAccess.cpp
index 636b072837441..632db7e4326e2 100644
--- a/llvm/lib/Target/X86/X86InterleavedAccess.cpp
+++ b/llvm/lib/Target/X86/X86InterleavedAccess.cpp
@@ -802,7 +802,7 @@ bool X86InterleavedAccessGroup::lowerIntoOptimizedSequence() {
 // Currently, lowering is supported for 4x64 bits with Factor = 4 on AVX.
 bool X86TargetLowering::lowerInterleavedLoad(
     Instruction *Load, Value *Mask, ArrayRef<ShuffleVectorInst *> Shuffles,
-    ArrayRef<unsigned> Indices, unsigned Factor) const {
+    ArrayRef<unsigned> Indices, unsigned Factor, const APInt &GapMask) const {
   assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
          "Invalid interleave factor");
   assert(!Shuffles.empty() && "Empty shufflevector input");
@@ -812,7 +812,7 @@ bool X86TargetLowering::lowerInterleavedLoad(
   auto *LI = dyn_cast<LoadInst>(Load);
   if (!LI)
     return false;
-  assert(!Mask && "Unexpected mask on a load");
+  assert(!Mask && GapMask.popcount() == Factor && "Unexpected mask on a load");
 
   // Create an interleaved access group.
   IRBuilder<> Builder(LI);
diff --git a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
index cf055cf3be0aa..090060eaa65e1 100644
--- a/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
+++ b/llvm/lib/Target/X86/X86LoadValueInjectionLoadHardening.cpp
@@ -491,7 +491,7 @@ X86LoadValueInjectionLoadHardeningPass::getGadgetGraph(
   NumGadgets += GadgetCount;
 
   // Traverse CFG to build the rest of the graph
-  SmallSet<MachineBasicBlock *, 8> BlocksVisited;
+  SmallPtrSet<MachineBasicBlock *, 8> BlocksVisited;
   std::function<void(MachineBasicBlock *, GraphIter, unsigned)> TraverseCFG =
       [&](MachineBasicBlock *MBB, GraphIter GI, unsigned ParentDepth) {
         unsigned LoopDepth = MLI.getLoopDepth(MBB);
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index 3b4e531f25388..2a1c49957bf7a 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -100,7 +100,7 @@ struct BBInfo {
 class X86PreTileConfig : public MachineFunctionPass {
   MachineRegisterInfo *MRI = nullptr;
   const MachineLoopInfo *MLI = nullptr;
-  SmallSet<MachineInstr *, 8> DefVisited;
+  SmallPtrSet<MachineInstr *, 8> DefVisited;
   DenseMap<MachineBasicBlock *, BBInfo> BBVisitedInfo;
   DenseMap<MachineBasicBlock *, SmallVector<MIRef, 8>> ShapeBBs;
 
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 83b11eede829e..595ad3290eedb 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -21,8 +21,8 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/CodeGen/LiveRegMatrix.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TileShapeInfo.h"
@@ -907,7 +907,7 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
 
   // Determine base register and offset.
-  int FIOffset;
+  int64_t FIOffset;
   Register BasePtr;
   if (MI.isReturn()) {
     assert((!hasStackRealignment(MF) ||
@@ -958,11 +958,41 @@ X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   }
 
   if (MI.getOperand(FIOperandNum+3).isImm()) {
-    // Offset is a 32-bit integer.
-    int Imm = (int)(MI.getOperand(FIOperandNum + 3).getImm());
-    int Offset = FIOffset + Imm;
-    assert((!Is64Bit || isInt<32>((long long)FIOffset + Imm)) &&
-           "Requesting 64-bit offset in 32-bit immediate!");
+    const X86InstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
+    const DebugLoc &DL = MI.getDebugLoc();
+    int64_t Imm = MI.getOperand(FIOperandNum + 3).getImm();
+    int64_t Offset = FIOffset + Imm;
+    bool FitsIn32Bits = isInt<32>(Offset);
+    // If the offset will not fit in a 32-bit displacement, then for 64-bit
+    // targets, scavenge a register to hold it. Otherwise...
+    if (Is64Bit && !FitsIn32Bits) {
+      assert(RS && "RegisterScavenger was NULL");
+
+      RS->enterBasicBlockEnd(MBB);
+      RS->backward(std::next(II));
+
+      Register ScratchReg = RS->scavengeRegisterBackwards(
+          X86::GR64RegClass, II, /*RestoreAfter=*/false, /*SPAdj=*/0,
+          /*AllowSpill=*/true);
+      assert(ScratchReg != 0 && "scratch reg was 0");
+      RS->setRegUsed(ScratchReg);
+
+      BuildMI(MBB, II, DL, TII->get(X86::MOV64ri), ScratchReg).addImm(Offset);
+
+      MI.getOperand(FIOperandNum + 3).setImm(0);
+      MI.getOperand(FIOperandNum + 2).setReg(ScratchReg);
+
+      return false;
+    }
+
+    // ... for 32-bit targets, this is a bug!
+    if (!Is64Bit && !FitsIn32Bits) {
+      MI.emitGenericError("64-bit offset calculated but target is 32-bit");
+      // Trap so that the instruction verification pass does not fail if run.
+      BuildMI(MBB, MBBI, DL, TII->get(X86::TRAP));
+      return false;
+    }
+
     if (Offset != 0 || !tryOptimizeLEAtoMOV(II))
       MI.getOperand(FIOperandNum + 3).ChangeToImmediate(Offset);
   } else {
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.h b/llvm/lib/Target/X86/X86RegisterInfo.h
index 19b409ae619d2..2f4c55cfad6d2 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.h
+++ b/llvm/lib/Target/X86/X86RegisterInfo.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_LIB_TARGET_X86_X86REGISTERINFO_H
 #define LLVM_LIB_TARGET_X86_X86REGISTERINFO_H
 
+#include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 
 #define GET_REGINFO_HEADER
@@ -180,6 +181,10 @@ class X86RegisterInfo final : public X86GenRegisterInfo {
   constrainRegClassToNonRex2(const TargetRegisterClass *RC) const;
 
   bool isNonRex2RegClass(const TargetRegisterClass *RC) const;
+
+  bool requiresRegisterScavenging(const MachineFunction &MF) const override {
+    return true;
+  }
 };
 
 } // End llvm namespace
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeClient.td b/llvm/lib/Target/X86/X86SchedSkylakeClient.td
index 8cd52e2a8ebc9..f15a7c7076414 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeClient.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeClient.td
@@ -70,6 +70,12 @@ def SKLPortAny : ProcResGroup<[SKLPort0, SKLPort1, SKLPort2, SKLPort3, SKLPort4,
   let BufferSize=60;
 }
 
+// Skylake can retire up to four (potentially fused) uops per cycle. Set the
+// limit to twice that given we do not model fused uops as only taking up one
+// retirement slot. I could not find any documented sources on how many
+// in-flight micro-ops can be tracked.
+def SKRCU : RetireControlUnit<0, 8>;
+
 // Integer loads are 5 cycles, so ReadAfterLd registers needn't be available until 5
 // cycles after the memory operand.
 def : ReadAdvance<ReadAfterLd, 5>;
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
index 14a51d1ed9ce6..2a793d0205986 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
@@ -70,6 +70,12 @@ def SKXPortAny : ProcResGroup<[SKXPort0, SKXPort1, SKXPort2, SKXPort3, SKXPort4,
   let BufferSize=60;
 }
 
+// Skylake can retire up to four (potentially fused) uops per cycle. Set the
+// limit to twice that given we do not model fused uops as only taking up one
+// retirement slot. I could not find any documented sources on how many
+// in-flight micro-ops can be tracked.
+def SKXRCU : RetireControlUnit<0, 8>;
+
 // Integer loads are 5 cycles, so ReadAfterLd registers needn't be available until 5
 // cycles after the memory operand.
 def : ReadAdvance<ReadAfterLd, 5>;
diff --git a/llvm/lib/Target/X86/X86ScheduleAtom.td b/llvm/lib/Target/X86/X86ScheduleAtom.td
index c92bc97cfb385..133c1a430a96d 100644
--- a/llvm/lib/Target/X86/X86ScheduleAtom.td
+++ b/llvm/lib/Target/X86/X86ScheduleAtom.td
@@ -562,14 +562,7 @@ def AtomWrite0_1_7_4 : SchedWriteRes<[AtomPort0,AtomPort1]> {
   let ReleaseAtCycles = [8,8];
   let NumMicroOps = 4;
 }
-def : InstRW<[AtomWrite0_1_7_4], (instregex "CVTSI642SSrr(_Int)?")>;
-
-def AtomWrite0_1_8_4 : SchedWriteRes<[AtomPort0,AtomPort1]> {
-  let Latency = 8;
-  let ReleaseAtCycles = [8,8];
-  let NumMicroOps = 4;
-}
-def : InstRW<[AtomWrite0_1_7_4], (instregex "CVTSI642SSrm(_Int)?")>;
+def : InstRW<[AtomWrite0_1_7_4], (instregex "CVTSI642SSr(r|m)(_Int)?")>;
 
 def AtomWrite0_1_9 : SchedWriteRes<[AtomPort0,AtomPort1]> {
   let Latency = 9;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 90791fc5788f4..62f95277d0169 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -161,19 +161,26 @@ std::optional<unsigned> X86TTIImpl::getCacheAssociativity(
   llvm_unreachable("Unknown TargetTransformInfo::CacheLevel");
 }
 
+enum ClassIDEnum { GPRClass = 0, VectorClass = 1, ScalarFPClass = 2 };
+
+unsigned X86TTIImpl::getRegisterClassForType(bool Vector, Type *Ty) const {
+  return Vector                          ? VectorClass
+         : Ty && Ty->isFloatingPointTy() ? ScalarFPClass
+                                         : GPRClass;
+}
+
 unsigned X86TTIImpl::getNumberOfRegisters(unsigned ClassID) const {
-  bool Vector = (ClassID == 1);
-  if (Vector && !ST->hasSSE1())
+  if (ClassID == VectorClass && !ST->hasSSE1())
     return 0;
 
-  if (ST->is64Bit()) {
-    if (Vector && ST->hasAVX512())
-      return 32;
-    if (!Vector && ST->hasEGPR())
-      return 32;
-    return 16;
-  }
-  return 8;
+  if (!ST->is64Bit())
+    return 8;
+
+  if ((ClassID == GPRClass && ST->hasEGPR()) ||
+      (ClassID != GPRClass && ST->hasAVX512()))
+    return 32;
+
+  return 16;
 }
 
 bool X86TTIImpl::hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const {
@@ -5488,9 +5495,10 @@ InstructionCost X86TTIImpl::getPointersChainCost(
   return BaseT::getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);
 }
 
-InstructionCost X86TTIImpl::getAddressComputationCost(Type *Ty,
-                                                      ScalarEvolution *SE,
-                                                      const SCEV *Ptr) const {
+InstructionCost
+X86TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
+                                      const SCEV *Ptr,
+                                      TTI::TargetCostKind CostKind) const {
   // Address computations in vectorized code with non-consecutive addresses will
   // likely result in more instructions compared to scalar code where the
   // computation can more often be merged into the index mode. The resulting
@@ -5504,7 +5512,7 @@ InstructionCost X86TTIImpl::getAddressComputationCost(Type *Ty,
   // Even in the case of (loop invariant) stride whose value is not known at
   // compile time, the address computation will not incur more than one extra
   // ADD instruction.
-  if (Ty->isVectorTy() && SE && !ST->hasAVX2()) {
+  if (PtrTy->isVectorTy() && SE && !ST->hasAVX2()) {
     // TODO: AVX2 is the current cut-off because we don't have correct
     //       interleaving costs for prior ISA's.
     if (!BaseT::isStridedAccess(Ptr))
@@ -5513,7 +5521,7 @@ InstructionCost X86TTIImpl::getAddressComputationCost(Type *Ty,
       return 1;
   }
 
-  return BaseT::getAddressComputationCost(Ty, SE, Ptr);
+  return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
 }
 
 InstructionCost
@@ -6525,8 +6533,8 @@ bool X86TTIImpl::areInlineCompatible(const Function *Caller,
 
   for (const Instruction &I : instructions(Callee)) {
     if (const auto *CB = dyn_cast<CallBase>(&I)) {
-      // Having more target features is fine for inline ASM.
-      if (CB->isInlineAsm())
+      // Having more target features is fine for inline ASM and intrinsics.
+      if (CB->isInlineAsm() || CB->getIntrinsicID() != Intrinsic::not_intrinsic)
         continue;
 
       SmallVector<Type *, 8> Types;
@@ -6542,19 +6550,9 @@ bool X86TTIImpl::areInlineCompatible(const Function *Caller,
       if (all_of(Types, IsSimpleTy))
         continue;
 
-      if (Function *NestedCallee = CB->getCalledFunction()) {
-        // Assume that intrinsics are always ABI compatible.
-        if (NestedCallee->isIntrinsic())
-          continue;
-
-        // Do a precise compatibility check.
-        if (!areTypesABICompatible(Caller, NestedCallee, Types))
-          return false;
-      } else {
-        // We don't know the target features of the callee,
-        // assume it is incompatible.
+      // Do a precise compatibility check.
+      if (!areTypesABICompatible(Caller, Callee, Types))
         return false;
-      }
     }
   }
   return true;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index bc06c4746c3c4..133b3668a46c8 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -132,6 +132,7 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
   /// @{
 
   unsigned getNumberOfRegisters(unsigned ClassID) const override;
+  unsigned getRegisterClassForType(bool Vector, Type *Ty) const override;
   bool hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const override;
   TypeSize
   getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const override;
@@ -194,8 +195,9 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
   getPointersChainCost(ArrayRef<const Value *> Ptrs, const Value *Base,
                        const TTI::PointersChainInfo &Info, Type *AccessTy,
                        TTI::TargetCostKind CostKind) const override;
-  InstructionCost getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
-                                            const SCEV *Ptr) const override;
+  InstructionCost
+  getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,
+                            TTI::TargetCostKind CostKind) const override;
 
   std::optional<Instruction *>
   instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const override;
diff --git a/llvm/lib/Target/X86/X86WinEHUnwindV2.cpp b/llvm/lib/Target/X86/X86WinEHUnwindV2.cpp
index e9081a4ae4e72..ea8b88f41bb87 100644
--- a/llvm/lib/Target/X86/X86WinEHUnwindV2.cpp
+++ b/llvm/lib/Target/X86/X86WinEHUnwindV2.cpp
@@ -190,6 +190,7 @@ bool X86WinEHUnwindV2::runOnMachineFunction(MachineFunction &MF) {
         State = FunctionState::FinishedEpilog;
         break;
 
+      case X86::LEA64r:
       case X86::MOV64rr:
       case X86::ADD64ri32:
         if (State == FunctionState::InEpilog) {
@@ -201,51 +202,56 @@ bool X86WinEHUnwindV2::runOnMachineFunction(MachineFunction &MF) {
                 "The epilog is deallocating a stack "
                 "allocation, but the prolog did "
                 "not allocate one");
-          if (HasStackDealloc)
+          if (PoppedRegCount > 0)
             return rejectCurrentFunctionInternalError(
                 MF, Mode,
-                "The epilog is deallocating the stack "
-                "allocation more than once");
-          if (PoppedRegCount > 0)
-            llvm_unreachable(
-                "Should have raised an error: either popping before "
-                "deallocating or deallocating without an allocation");
+                "The epilog is deallocating a stack allocation after popping "
+                "registers");
 
           HasStackDealloc = true;
         } else if (State == FunctionState::FinishedEpilog)
           return rejectCurrentFunctionInternalError(
-              MF, Mode, "Unexpected mov or add instruction after the epilog");
+              MF, Mode,
+              "Unexpected lea, mov or add instruction after the epilog");
         break;
 
       case X86::POP64r:
         if (State == FunctionState::InEpilog) {
-          // After the stack pointer has been adjusted, the epilog must
-          // POP each register in reverse order of the PUSHes in the prolog.
-          PoppedRegCount++;
-          if (HasStackAlloc != HasStackDealloc)
-            return rejectCurrentFunctionInternalError(
-                MF, Mode,
-                "Cannot pop registers before the stack "
-                "allocation has been deallocated");
-          if (PoppedRegCount > PushedRegs.size())
-            return rejectCurrentFunctionInternalError(
-                MF, Mode,
-                "The epilog is popping more registers than the prolog pushed");
-          if (PushedRegs[PushedRegs.size() - PoppedRegCount] !=
-              MI.getOperand(0).getReg())
-            return rejectCurrentFunctionInternalError(
-                MF, Mode,
-                "The epilog is popping a registers in "
-                "a different order than the "
-                "prolog pushed them");
-
-          // Unwind v2 records the size of the epilog not from where we place
-          // SEH_BeginEpilogue (as that contains the instruction to adjust the
-          // stack pointer) but from the first POP instruction (if there is
-          // one).
-          if (!UnwindV2StartLocation) {
-            assert(PoppedRegCount == 1);
-            UnwindV2StartLocation = &MI;
+          Register Reg = MI.getOperand(0).getReg();
+          if (HasStackAlloc && (PoppedRegCount == 0) &&
+              !llvm::is_contained(PushedRegs, Reg)) {
+            // If this is a pop that doesn't correspond to the set of pushed
+            // registers, then assume it was used to adjust the stack pointer.
+            HasStackDealloc = true;
+          } else {
+            // After the stack pointer has been adjusted, the epilog must
+            // POP each register in reverse order of the PUSHes in the prolog.
+            PoppedRegCount++;
+            if (HasStackAlloc != HasStackDealloc)
+              return rejectCurrentFunctionInternalError(
+                  MF, Mode,
+                  "Cannot pop registers before the stack "
+                  "allocation has been deallocated");
+            if (PoppedRegCount > PushedRegs.size())
+              return rejectCurrentFunctionInternalError(
+                  MF, Mode,
+                  "The epilog is popping more registers than the prolog "
+                  "pushed");
+            if (PushedRegs[PushedRegs.size() - PoppedRegCount] != Reg.id())
+              return rejectCurrentFunctionInternalError(
+                  MF, Mode,
+                  "The epilog is popping a registers in "
+                  "a different order than the "
+                  "prolog pushed them");
+
+            // Unwind v2 records the size of the epilog not from where we place
+            // SEH_BeginEpilogue (as that contains the instruction to adjust the
+            // stack pointer) but from the first POP instruction (if there is
+            // one).
+            if (!UnwindV2StartLocation) {
+              assert(PoppedRegCount == 1);
+              UnwindV2StartLocation = &MI;
+            }
           }
         } else if (State == FunctionState::FinishedEpilog)
           // Unexpected instruction after the epilog.
diff --git a/llvm/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp b/llvm/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp
index d36f18238f7a3..6921f44b700c5 100644
--- a/llvm/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp
+++ b/llvm/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp
@@ -15,6 +15,7 @@
 #include "XCore.h"
 #include "XCoreRegisterInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -23,6 +24,7 @@
 #include "llvm/Support/Compiler.h"
 
 using namespace llvm;
+using namespace llvm::MCD;
 
 #define DEBUG_TYPE "xcore-disassembler"
 
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index ef4cfcd69415d..0a96ab236c6ad 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -429,11 +429,7 @@ SDValue XCoreTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   // Lower to a call to __misaligned_load(BasePtr).
   Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(Context);
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-
-  Entry.Ty = IntPtrTy;
-  Entry.Node = BasePtr;
-  Args.push_back(Entry);
+  Args.emplace_back(BasePtr, IntPtrTy);
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
@@ -480,14 +476,8 @@ SDValue XCoreTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
   // Lower to a call to __misaligned_store(BasePtr, Value).
   Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(Context);
   TargetLowering::ArgListTy Args;
-  TargetLowering::ArgListEntry Entry;
-
-  Entry.Ty = IntPtrTy;
-  Entry.Node = BasePtr;
-  Args.push_back(Entry);
-
-  Entry.Node = Value;
-  Args.push_back(Entry);
+  Args.emplace_back(BasePtr, IntPtrTy);
+  Args.emplace_back(Value, IntPtrTy);
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   CLI.setDebugLoc(dl).setChain(Chain).setCallee(
diff --git a/llvm/lib/Target/XCore/XCoreInstrFormats.td b/llvm/lib/Target/XCore/XCoreInstrFormats.td
index deb899ddb1afa..57f9a488cbd27 100644
--- a/llvm/lib/Target/XCore/XCoreInstrFormats.td
+++ b/llvm/lib/Target/XCore/XCoreInstrFormats.td
@@ -19,7 +19,6 @@ class InstXCore<int sz, dag outs, dag ins, string asmstr, list<dag> pattern>
   let AsmString   = asmstr;
   let Pattern = pattern;
   let Size = sz;
-  field bits<32> SoftFail = 0;
 }
 
 // XCore pseudo instructions format
diff --git a/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp b/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
index 1bd92a2b49475..f61115e0139e6 100644
--- a/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
@@ -33,11 +33,10 @@ SDValue XCoreSelectionDAGInfo::EmitTargetCodeForMemcpy(
       DAG.MaskedValueIsZero(Size, APInt(SizeBitWidth, 3))) {
     const TargetLowering &TLI = *DAG.getSubtarget().getTargetLowering();
     TargetLowering::ArgListTy Args;
-    TargetLowering::ArgListEntry Entry;
-    Entry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
-    Entry.Node = Dst; Args.push_back(Entry);
-    Entry.Node = Src; Args.push_back(Entry);
-    Entry.Node = Size; Args.push_back(Entry);
+    Type *ArgTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
+    Args.emplace_back(Dst, ArgTy);
+    Args.emplace_back(Src, ArgTy);
+    Args.emplace_back(Size, ArgTy);
 
     const char *MemcpyAlign4Name = TLI.getLibcallName(RTLIB::MEMCPY_ALIGN_4);
     CallingConv::ID CC = TLI.getLibcallCallingConv(RTLIB::MEMCPY_ALIGN_4);
diff --git a/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp b/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp
index 2f92f8606fb48..396b00f7b8628 100644
--- a/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp
+++ b/llvm/lib/Target/Xtensa/Disassembler/XtensaDisassembler.cpp
@@ -15,6 +15,7 @@
 #include "MCTargetDesc/XtensaMCTargetDesc.h"
 #include "TargetInfo/XtensaTargetInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCDecoder.h"
 #include "llvm/MC/MCDecoderOps.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
 #include "llvm/MC/MCInst.h"
@@ -145,39 +146,40 @@ struct DecodeRegister {
 };
 
 const DecodeRegister SRDecoderTable[] = {
-    {Xtensa::LBEG, 0},          {Xtensa::LEND, 1},
-    {Xtensa::LCOUNT, 2},        {Xtensa::SAR, 3},
-    {Xtensa::BREG, 4},          {Xtensa::LITBASE, 5},
-    {Xtensa::ACCLO, 16},        {Xtensa::ACCHI, 17},
-    {Xtensa::M0, 32},           {Xtensa::M1, 33},
-    {Xtensa::M2, 34},           {Xtensa::M3, 35},
-    {Xtensa::WINDOWBASE, 72},   {Xtensa::WINDOWSTART, 73},
-    {Xtensa::IBREAKENABLE, 96}, {Xtensa::MEMCTL, 97},
-    {Xtensa::DDR, 104},         {Xtensa::IBREAKA0, 128},
-    {Xtensa::IBREAKA1, 129},    {Xtensa::DBREAKA0, 144},
-    {Xtensa::DBREAKA1, 145},    {Xtensa::DBREAKC0, 160},
-    {Xtensa::DBREAKC1, 161},    {Xtensa::CONFIGID0, 176},
-    {Xtensa::EPC1, 177},        {Xtensa::EPC2, 178},
-    {Xtensa::EPC3, 179},        {Xtensa::EPC4, 180},
-    {Xtensa::EPC5, 181},        {Xtensa::EPC6, 182},
-    {Xtensa::EPC7, 183},        {Xtensa::DEPC, 192},
-    {Xtensa::EPS2, 194},        {Xtensa::EPS3, 195},
-    {Xtensa::EPS4, 196},        {Xtensa::EPS5, 197},
-    {Xtensa::EPS6, 198},        {Xtensa::EPS7, 199},
-    {Xtensa::CONFIGID1, 208},   {Xtensa::EXCSAVE1, 209},
-    {Xtensa::EXCSAVE2, 210},    {Xtensa::EXCSAVE3, 211},
-    {Xtensa::EXCSAVE4, 212},    {Xtensa::EXCSAVE5, 213},
-    {Xtensa::EXCSAVE6, 214},    {Xtensa::EXCSAVE7, 215},
-    {Xtensa::CPENABLE, 224},    {Xtensa::INTERRUPT, 226},
-    {Xtensa::INTCLEAR, 227},    {Xtensa::INTENABLE, 228},
-    {Xtensa::PS, 230},          {Xtensa::VECBASE, 231},
-    {Xtensa::EXCCAUSE, 232},    {Xtensa::DEBUGCAUSE, 233},
-    {Xtensa::CCOUNT, 234},      {Xtensa::PRID, 235},
-    {Xtensa::ICOUNT, 236},      {Xtensa::ICOUNTLEVEL, 237},
-    {Xtensa::EXCVADDR, 238},    {Xtensa::CCOMPARE0, 240},
-    {Xtensa::CCOMPARE1, 241},   {Xtensa::CCOMPARE2, 242},
-    {Xtensa::MISC0, 244},       {Xtensa::MISC1, 245},
-    {Xtensa::MISC2, 246},       {Xtensa::MISC3, 247}};
+    {Xtensa::LBEG, 0},         {Xtensa::LEND, 1},
+    {Xtensa::LCOUNT, 2},       {Xtensa::SAR, 3},
+    {Xtensa::BREG, 4},         {Xtensa::LITBASE, 5},
+    {Xtensa::SCOMPARE1, 12},   {Xtensa::ACCLO, 16},
+    {Xtensa::ACCHI, 17},       {Xtensa::M0, 32},
+    {Xtensa::M1, 33},          {Xtensa::M2, 34},
+    {Xtensa::M3, 35},          {Xtensa::WINDOWBASE, 72},
+    {Xtensa::WINDOWSTART, 73}, {Xtensa::IBREAKENABLE, 96},
+    {Xtensa::MEMCTL, 97},      {Xtensa::ATOMCTL, 99},
+    {Xtensa::DDR, 104},        {Xtensa::IBREAKA0, 128},
+    {Xtensa::IBREAKA1, 129},   {Xtensa::DBREAKA0, 144},
+    {Xtensa::DBREAKA1, 145},   {Xtensa::DBREAKC0, 160},
+    {Xtensa::DBREAKC1, 161},   {Xtensa::CONFIGID0, 176},
+    {Xtensa::EPC1, 177},       {Xtensa::EPC2, 178},
+    {Xtensa::EPC3, 179},       {Xtensa::EPC4, 180},
+    {Xtensa::EPC5, 181},       {Xtensa::EPC6, 182},
+    {Xtensa::EPC7, 183},       {Xtensa::DEPC, 192},
+    {Xtensa::EPS2, 194},       {Xtensa::EPS3, 195},
+    {Xtensa::EPS4, 196},       {Xtensa::EPS5, 197},
+    {Xtensa::EPS6, 198},       {Xtensa::EPS7, 199},
+    {Xtensa::CONFIGID1, 208},  {Xtensa::EXCSAVE1, 209},
+    {Xtensa::EXCSAVE2, 210},   {Xtensa::EXCSAVE3, 211},
+    {Xtensa::EXCSAVE4, 212},   {Xtensa::EXCSAVE5, 213},
+    {Xtensa::EXCSAVE6, 214},   {Xtensa::EXCSAVE7, 215},
+    {Xtensa::CPENABLE, 224},   {Xtensa::INTERRUPT, 226},
+    {Xtensa::INTCLEAR, 227},   {Xtensa::INTENABLE, 228},
+    {Xtensa::PS, 230},         {Xtensa::VECBASE, 231},
+    {Xtensa::EXCCAUSE, 232},   {Xtensa::DEBUGCAUSE, 233},
+    {Xtensa::CCOUNT, 234},     {Xtensa::PRID, 235},
+    {Xtensa::ICOUNT, 236},     {Xtensa::ICOUNTLEVEL, 237},
+    {Xtensa::EXCVADDR, 238},   {Xtensa::CCOMPARE0, 240},
+    {Xtensa::CCOMPARE1, 241},  {Xtensa::CCOMPARE2, 242},
+    {Xtensa::MISC0, 244},      {Xtensa::MISC1, 245},
+    {Xtensa::MISC2, 246},      {Xtensa::MISC3, 247}};
 
 static DecodeStatus DecodeSRRegisterClass(MCInst &Inst, uint64_t RegNo,
                                           uint64_t Address,
diff --git a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
index 821cba0fc25c2..080a9c0bdd9e0 100644
--- a/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
+++ b/llvm/lib/Target/Xtensa/MCTargetDesc/XtensaMCTargetDesc.cpp
@@ -200,6 +200,9 @@ bool Xtensa::checkRegister(MCRegister RegNo, const FeatureBitset &FeatureBits,
   case Xtensa::WINDOWBASE:
   case Xtensa::WINDOWSTART:
     return FeatureBits[Xtensa::FeatureWindowed];
+  case Xtensa::ATOMCTL:
+  case Xtensa::SCOMPARE1:
+    return FeatureBits[Xtensa::FeatureWindowed];
   case Xtensa::NoRegister:
     return false;
   }
diff --git a/llvm/lib/Target/Xtensa/Xtensa.td b/llvm/lib/Target/Xtensa/Xtensa.td
index 2c4bacbe8282b..4ef885e19101e 100644
--- a/llvm/lib/Target/Xtensa/Xtensa.td
+++ b/llvm/lib/Target/Xtensa/Xtensa.td
@@ -23,10 +23,8 @@ include "XtensaFeatures.td"
 //===----------------------------------------------------------------------===//
 // Xtensa supported processors.
 //===----------------------------------------------------------------------===//
-class Proc<string Name, list<SubtargetFeature> Features>
-    : Processor<Name, NoItineraries, Features>;
 
-def : Proc<"generic", []>;
+include "XtensaProcessors.td"
 
 //===----------------------------------------------------------------------===//
 // Register File Description
diff --git a/llvm/lib/Target/Xtensa/XtensaFeatures.td b/llvm/lib/Target/Xtensa/XtensaFeatures.td
index 97d5472f3e96c..d6f3ef0f15e32 100644
--- a/llvm/lib/Target/Xtensa/XtensaFeatures.td
+++ b/llvm/lib/Target/Xtensa/XtensaFeatures.td
@@ -73,6 +73,22 @@ def FeatureDiv32 : SubtargetFeature<"div32", "HasDiv32", "true",
 def HasDiv32 : Predicate<"Subtarget->hasDiv32()">,
                AssemblerPredicate<(all_of FeatureDiv32)>;
 
+def FeatureS32C1I : SubtargetFeature<"s32c1i", "HasS32C1I", "true",
+                                     "Enable Xtensa S32C1I option">;
+def HasS32C1I : Predicate<"Subtarget->hasS32C1I()">,
+                AssemblerPredicate<(all_of FeatureS32C1I)>;
+
+// Assume that lock-free native-width atomics are available, even if the target
+// and operating system combination would not usually provide them. The user
+// is responsible for providing any necessary __sync implementations. Code
+// built with this feature is not ABI-compatible with code built without this
+// feature, if atomic variables are exposed across the ABI boundary.
+def FeatureForcedAtomics    : SubtargetFeature<"forced-atomics", "HasForcedAtomics", "true",
+                                               "Assume that lock-free native-width atomics are available">;
+def HasForcedAtomics        : Predicate<"Subtarget->hasForcedAtomics()">,
+                                         AssemblerPredicate<(all_of FeatureForcedAtomics)>;
+def HasAtomicLdSt           : Predicate<"Subtarget->hasS32C1I() || Subtarget->hasForcedAtomics()">;
+
 def FeatureRegionProtection : SubtargetFeature<"regprotect", "HasRegionProtection", "true",
                                                "Enable Xtensa Region Protection option">;
 def HasRegionProtection : Predicate<"Subtarget->hasRegionProtection()">,
diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
index fd42fd2e010ba..f1367037bdf45 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.cpp
@@ -250,6 +250,15 @@ XtensaTargetLowering::XtensaTargetLowering(const TargetMachine &TM,
   // Floating-point truncation and stores need to be done separately.
   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
 
+  if (Subtarget.hasS32C1I()) {
+    setMaxAtomicSizeInBitsSupported(32);
+    setMinCmpXchgSizeInBits(32);
+  } else if (Subtarget.hasForcedAtomics()) {
+    setMaxAtomicSizeInBitsSupported(32);
+  } else {
+    setMaxAtomicSizeInBitsSupported(0);
+  }
+
   // Compute derived properties from the register classes
   computeRegisterProperties(STI.getRegisterInfo());
 }
@@ -344,7 +353,8 @@ static const MCPhysReg IntRegs[] = {Xtensa::A2, Xtensa::A3, Xtensa::A4,
 
 static bool CC_Xtensa_Custom(unsigned ValNo, MVT ValVT, MVT LocVT,
                              CCValAssign::LocInfo LocInfo,
-                             ISD::ArgFlagsTy ArgFlags, CCState &State) {
+                             ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
+                             CCState &State) {
   if (ArgFlags.isByVal()) {
     Align ByValAlign = ArgFlags.getNonZeroByValAlign();
     unsigned ByValSize = ArgFlags.getByValSize();
@@ -1548,6 +1558,11 @@ const char *XtensaTargetLowering::getTargetNodeName(unsigned Opcode) const {
   return nullptr;
 }
 
+TargetLowering::AtomicExpansionKind
+XtensaTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
+  return AtomicExpansionKind::CmpXChg;
+}
+
 //===----------------------------------------------------------------------===//
 // Custom insertion
 //===----------------------------------------------------------------------===//
@@ -1696,6 +1711,23 @@ MachineBasicBlock *XtensaTargetLowering::EmitInstrWithCustomInserter(
 
     return MBB;
   }
+  case Xtensa::ATOMIC_CMP_SWAP_32_P: {
+    MachineOperand &R = MI.getOperand(0);
+    MachineOperand &Addr = MI.getOperand(1);
+    MachineOperand &Cmp = MI.getOperand(2);
+    MachineOperand &Swap = MI.getOperand(3);
+
+    BuildMI(*MBB, MI, DL, TII.get(Xtensa::WSR), Xtensa::SCOMPARE1)
+        .addReg(Cmp.getReg());
+
+    BuildMI(*MBB, MI, DL, TII.get(Xtensa::S32C1I), R.getReg())
+        .addReg(Swap.getReg())
+        .addReg(Addr.getReg())
+        .addImm(0);
+
+    MI.eraseFromParent();
+    return MBB;
+  }
   default:
     llvm_unreachable("Unexpected instr type to insert");
   }
diff --git a/llvm/lib/Target/Xtensa/XtensaISelLowering.h b/llvm/lib/Target/Xtensa/XtensaISelLowering.h
index e6ddf9864932a..d84cbdb6afcef 100644
--- a/llvm/lib/Target/Xtensa/XtensaISelLowering.h
+++ b/llvm/lib/Target/Xtensa/XtensaISelLowering.h
@@ -145,6 +145,12 @@ class XtensaTargetLowering : public TargetLowering {
                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
                       SelectionDAG &DAG) const override;
 
+  bool shouldInsertFencesForAtomic(const Instruction *I) const override {
+    return true;
+  }
+
+  AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
+
   bool decomposeMulByConstant(LLVMContext &Context, EVT VT,
                               SDValue C) const override;
 
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrFormats.td b/llvm/lib/Target/Xtensa/XtensaInstrFormats.td
index e7c51da1e14fa..bd082c1def88d 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrFormats.td
+++ b/llvm/lib/Target/Xtensa/XtensaInstrFormats.td
@@ -30,7 +30,6 @@ class XtensaInst24<dag outs, dag ins, string asmstr, list<dag> pattern,
                    InstrItinClass itin = NoItinerary>
   : XtensaInst<3, outs, ins, asmstr, pattern, itin> {
   field bits<24> Inst;
-  field bits<24> SoftFail = 0;
 }
 
 // Base class for Xtensa 16 bit Format
@@ -38,7 +37,6 @@ class XtensaInst16<dag outs, dag ins, string asmstr, list<dag> pattern,
                    InstrItinClass itin = NoItinerary>
   : XtensaInst<2, outs, ins, asmstr, pattern, itin> {
   field bits<16> Inst;
-  field bits<16> SoftFail = 0;
   let Predicates = [HasDensity];
 }
 
diff --git a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
index 31608f4659365..edcf2473d45cd 100644
--- a/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaInstrInfo.td
@@ -496,6 +496,8 @@ def EXTW : RRR_Inst<0x00, 0x00, 0x00, (outs), (ins),
   let hasSideEffects = 1;
 }
 
+def : Pat<(atomic_fence timm, timm), (MEMW)>;
+
 //===----------------------------------------------------------------------===//
 // Illegal instructions
 //===----------------------------------------------------------------------===//
@@ -1498,6 +1500,46 @@ def RFI : RRR_Inst<0x00, 0x00, 0x00, (outs), (ins uimm4:$imm),
   let t = 0x1;
 }
 
+//===----------------------------------------------------------------------===//
+// S32C1I
+//===----------------------------------------------------------------------===//
+
+let mayStore = 1, mayLoad = 1, Predicates = [HasS32C1I] in {
+  def S32C1I : RRI8_Inst<0x02, (outs AR:$a), (ins AR:$t, mem32:$addr),
+                        "s32c1i\t$t, $addr", []> {
+    bits<12> addr;
+
+    let r = 0x0e;
+    let Uses = [SCOMPARE1];
+    let Constraints = "$a = $t";
+    let imm8{7-0} = addr{11-4};
+    let s{3-0} = addr{3-0};
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Atomic patterns
+//===----------------------------------------------------------------------===//
+
+// Atomic load/store are available under both +s32c1i and +force-atomics.
+// Fences will be inserted for atomic load/stores according to the logic in
+// XtensaTargetLowering.
+let Predicates = [HasAtomicLdSt] in {
+  def : Pat<(i32 (atomic_load_8  addr_ish1:$addr)), (L8UI addr_ish1:$addr)>;
+  def : Pat<(i32 (atomic_load_16 addr_ish2:$addr)), (L16UI addr_ish2:$addr)>;
+  def : Pat<(i32 (atomic_load_32 addr_ish4:$addr)), (L32I addr_ish4:$addr)>;
+
+  def : Pat<(atomic_store_8  AR:$t, addr_ish1:$addr), (S8I AR:$t, addr_ish1:$addr)>;
+  def : Pat<(atomic_store_16 AR:$t, addr_ish2:$addr), (S16I AR:$t, addr_ish2:$addr)>;
+  def : Pat<(atomic_store_32 AR:$t, addr_ish4:$addr), (S32I AR:$t, addr_ish4:$addr)>;
+}
+
+let usesCustomInserter = 1, Predicates = [HasS32C1I] in {
+  def ATOMIC_CMP_SWAP_32_P : Pseudo<(outs AR:$dst), (ins AR:$ptr, AR:$cmp, AR:$swap),
+                                   "!atomic_cmp_swap_32_p, $dst, $ptr, $cmp, $swap",
+                                   [(set AR:$dst, (atomic_cmp_swap_i32 AR:$ptr, AR:$cmp, AR:$swap))]>;
+}
+
 //===----------------------------------------------------------------------===//
 // DSP Instructions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/Xtensa/XtensaProcessors.td b/llvm/lib/Target/Xtensa/XtensaProcessors.td
new file mode 100644
index 0000000000000..0faf07d99aaf7
--- /dev/null
+++ b/llvm/lib/Target/Xtensa/XtensaProcessors.td
@@ -0,0 +1,27 @@
+//===- XtensaProcessors.td - Xtensa Processors -------------*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// Xtensa supported processors.
+//===----------------------------------------------------------------------===//
+class Proc<string Name, list<SubtargetFeature> Features>
+    : Processor<Name, NoItineraries, Features>;
+
+def : Proc<"generic", []>;
+
+def : Proc<"esp32", [FeatureDensity, FeatureSingleFloat, FeatureLoop, FeatureMAC16, FeatureWindowed, FeatureBoolean, FeatureSEXT,
+                     FeatureNSA, FeatureMul16, FeatureMul32, FeatureMul32High, FeatureDFPAccel, FeatureS32C1I, FeatureTHREADPTR, FeatureDiv32,
+                     FeatureDebug, FeatureException, FeatureHighPriInterrupts, FeatureHighPriInterruptsLevel7, FeatureCoprocessor,
+                     FeatureInterrupt, FeatureDataCache, FeatureRelocatableVector, FeatureTimers3, FeaturePRID, FeatureRegionProtection, FeatureMiscSR,
+                     FeatureMINMAX, FeatureCLAMPS]>;
+
+def : Proc<"esp8266", [FeatureDensity, FeatureNSA, FeatureMul16, FeatureMul32, FeatureExtendedL32R, FeatureDebug, FeatureException,
+                       FeatureHighPriInterrupts, FeatureHighPriInterruptsLevel3, FeatureInterrupt, FeatureRelocatableVector, FeatureTimers1,
+					   FeatureRegionProtection, FeaturePRID]>;
diff --git a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
index 596c4105c1118..d1f2c6b8e43a3 100644
--- a/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
+++ b/llvm/lib/Target/Xtensa/XtensaRegisterInfo.td
@@ -84,6 +84,9 @@ def SAR : SRReg<3, "sar", ["SAR","3"]>;
 // Boolean Register
 def BREG : SRReg<4, "br", ["BR","4"]>;
 
+// Expected data value for S32C1I operation
+def SCOMPARE1 : SRReg<12, "scompare1", ["SCOMPARE1", "12"]>;
+
 // Literal base
 def LITBASE : SRReg<5, "litbase", ["LITBASE", "5"]>;
 
@@ -97,6 +100,9 @@ def IBREAKENABLE : SRReg<96, "ibreakenable", ["IBREAKENABLE", "96"]>;
 // Memory Control Register
 def MEMCTL : SRReg<97, "memctl", ["MEMCTL", "97"]>;
 
+// Atomic Operation Control
+def ATOMCTL : SRReg<99, "atomctl", ["ATOMCTL", "99"]>;
+
 def DDR : SRReg<104, "ddr", ["DDR", "104"]>;
 
 // Instuction break address register 0
@@ -218,8 +224,8 @@ def MR23 :  RegisterClass<"Xtensa", [i32], 32, (add M2, M3)>;
 def MR   :  RegisterClass<"Xtensa", [i32], 32, (add MR01, MR23)>;
 
 def SR :  RegisterClass<"Xtensa", [i32], 32, (add
-  LBEG, LEND, LCOUNT, SAR, BREG, LITBASE, ACCLO, ACCHI, MR,
-  WINDOWBASE, WINDOWSTART, IBREAKENABLE, MEMCTL, DDR, IBREAKA0, IBREAKA1,
+  LBEG, LEND, LCOUNT, SAR, BREG, SCOMPARE1, LITBASE, ACCLO, ACCHI, MR,
+  WINDOWBASE, WINDOWSTART, IBREAKENABLE, MEMCTL, ATOMCTL, DDR, IBREAKA0, IBREAKA1,
   DBREAKA0, DBREAKA1, DBREAKC0, DBREAKC1, CONFIGID0, EPC1, EPC2, EPC3, EPC4, EPC5,
   EPC6, EPC7, DEPC, EPS2, EPS3, EPS4, EPS5, EPS6, EPS7, CONFIGID1, EXCSAVE1, EXCSAVE2,
   EXCSAVE3, EXCSAVE4, EXCSAVE5, EXCSAVE6, EXCSAVE7, CPENABLE, INTERRUPT, INTSET, INTCLEAR, INTENABLE,
diff --git a/llvm/lib/Target/Xtensa/XtensaSubtarget.h b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
index fd677a451f3fd..b406534a0ec77 100644
--- a/llvm/lib/Target/Xtensa/XtensaSubtarget.h
+++ b/llvm/lib/Target/Xtensa/XtensaSubtarget.h
@@ -77,6 +77,8 @@ class XtensaSubtarget : public XtensaGenSubtargetInfo {
   bool hasMul32() const { return HasMul32; }
   bool hasMul32High() const { return HasMul32High; }
   bool hasDiv32() const { return HasDiv32; }
+  bool hasS32C1I() const { return HasS32C1I; }
+  bool hasForcedAtomics() const { return HasForcedAtomics; }
   bool hasSingleFloat() const { return HasSingleFloat; }
   bool hasRegionProtection() const { return HasRegionProtection; }
   bool hasRelocatableVector() const { return HasRelocatableVector; }
diff --git a/llvm/lib/Target/Xtensa/XtensaTargetMachine.cpp b/llvm/lib/Target/Xtensa/XtensaTargetMachine.cpp
index 8d2dca6c23721..c9f1ca8b46dab 100644
--- a/llvm/lib/Target/Xtensa/XtensaTargetMachine.cpp
+++ b/llvm/lib/Target/Xtensa/XtensaTargetMachine.cpp
@@ -107,6 +107,7 @@ class XtensaPassConfig : public TargetPassConfig {
   }
 
   bool addInstSelector() override;
+  void addIRPasses() override;
   void addPreEmitPass() override;
 };
 } // end anonymous namespace
@@ -116,6 +117,11 @@ bool XtensaPassConfig::addInstSelector() {
   return false;
 }
 
+void XtensaPassConfig::addIRPasses() {
+  addPass(createAtomicExpandLegacyPass());
+  TargetPassConfig::addIRPasses();
+}
+
 void XtensaPassConfig::addPreEmitPass() { addPass(&BranchRelaxationPassID); }
 
 TargetPassConfig *XtensaTargetMachine::createPassConfig(PassManagerBase &PM) {
diff --git a/llvm/lib/TargetParser/CMakeLists.txt b/llvm/lib/TargetParser/CMakeLists.txt
index 8f8b3a578a1d9..5eecfbf80b2f7 100644
--- a/llvm/lib/TargetParser/CMakeLists.txt
+++ b/llvm/lib/TargetParser/CMakeLists.txt
@@ -9,7 +9,7 @@ if (HAS_WERROR_GLOBAL_CTORS AND NOT LLVM_HAS_NOGLOBAL_CTOR_MUTEX)
 endif()
 
 # Solaris code uses kstat, so specify dependency explicitly for shared builds.
-if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+if ("${CMAKE_SYSTEM_NAME}" MATCHES "SunOS")
   set(system_libs kstat)
 endif()
 
@@ -27,6 +27,7 @@ add_llvm_component_library(LLVMTargetParser
   TargetParser.cpp
   Triple.cpp
   X86TargetParser.cpp
+  XtensaTargetParser.cpp
 
   ADDITIONAL_HEADER_DIRS
   Unix
diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp
index 7e09d30bf3d55..24827537eb19c 100644
--- a/llvm/lib/TargetParser/Host.cpp
+++ b/llvm/lib/TargetParser/Host.cpp
@@ -11,7 +11,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/TargetParser/Host.h"
+#include "llvm/ADT/Bitfields.h"
+#include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -167,35 +170,10 @@ StringRef sys::detail::getHostCPUNameForPowerPC(StringRef ProcCpuinfoContent) {
       .Default(generic);
 }
 
-StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
-  // The cpuid register on arm is not accessible from user space. On Linux,
-  // it is exposed through the /proc/cpuinfo file.
-
-  // Read 32 lines from /proc/cpuinfo, which should contain the CPU part line
-  // in all cases.
-  SmallVector<StringRef, 32> Lines;
-  ProcCpuinfoContent.split(Lines, '\n');
-
-  // Look for the CPU implementer and hardware lines, and store the CPU part
-  // numbers found.
-  StringRef Implementer;
-  StringRef Hardware;
-  SmallVector<StringRef, 32> Parts;
-  for (StringRef Line : Lines) {
-    if (Line.consume_front("CPU implementer"))
-      Implementer = Line.ltrim("\t :");
-    else if (Line.consume_front("Hardware"))
-      Hardware = Line.ltrim("\t :");
-    else if (Line.consume_front("CPU part"))
-      Parts.emplace_back(Line.ltrim("\t :"));
-  }
-
-  // Last `Part' seen, in case we don't analyse all `Parts' parsed.
-  StringRef Part = Parts.empty() ? StringRef() : Parts.back();
-
-  // Remove duplicate `Parts'.
-  llvm::sort(Parts);
-  Parts.erase(llvm::unique(Parts), Parts.end());
+StringRef
+getHostCPUNameForARMFromComponents(StringRef Implementer, StringRef Hardware,
+                                   StringRef Part, ArrayRef<StringRef> Parts,
+                                   function_ref<unsigned()> GetVariant) {
 
   auto MatchBigLittle = [](auto const &Parts, StringRef Big, StringRef Little) {
     if (Parts.size() == 2)
@@ -343,21 +321,17 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
   if (Implementer == "0x53") { // Samsung Electronics Co., Ltd.
     // The Exynos chips have a convoluted ID scheme that doesn't seem to follow
     // any predictive pattern across variants and parts.
-    unsigned Variant = 0, Part = 0;
 
     // Look for the CPU variant line, whose value is a 1 digit hexadecimal
     // number, corresponding to the Variant bits in the CP15/C0 register.
-    for (auto I : Lines)
-      if (I.consume_front("CPU variant"))
-        I.ltrim("\t :").getAsInteger(0, Variant);
+    unsigned Variant = GetVariant();
 
-    // Look for the CPU part line, whose value is a 3 digit hexadecimal
-    // number, corresponding to the PartNum bits in the CP15/C0 register.
-    for (auto I : Lines)
-      if (I.consume_front("CPU part"))
-        I.ltrim("\t :").getAsInteger(0, Part);
+    // Convert the CPU part line, whose value is a 3 digit hexadecimal number,
+    // corresponding to the PartNum bits in the CP15/C0 register.
+    unsigned PartAsInt;
+    Part.getAsInteger(0, PartAsInt);
 
-    unsigned Exynos = (Variant << 12) | Part;
+    unsigned Exynos = (Variant << 12) | PartAsInt;
     switch (Exynos) {
     default:
       // Default by falling through to Exynos M3.
@@ -416,6 +390,78 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
   return "generic";
 }
 
+StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) {
+  // The cpuid register on arm is not accessible from user space. On Linux,
+  // it is exposed through the /proc/cpuinfo file.
+
+  // Read 32 lines from /proc/cpuinfo, which should contain the CPU part line
+  // in all cases.
+  SmallVector<StringRef, 32> Lines;
+  ProcCpuinfoContent.split(Lines, '\n');
+
+  // Look for the CPU implementer and hardware lines, and store the CPU part
+  // numbers found.
+  StringRef Implementer;
+  StringRef Hardware;
+  SmallVector<StringRef, 32> Parts;
+  for (StringRef Line : Lines) {
+    if (Line.consume_front("CPU implementer"))
+      Implementer = Line.ltrim("\t :");
+    else if (Line.consume_front("Hardware"))
+      Hardware = Line.ltrim("\t :");
+    else if (Line.consume_front("CPU part"))
+      Parts.emplace_back(Line.ltrim("\t :"));
+  }
+
+  // Last `Part' seen, in case we don't analyse all `Parts' parsed.
+  StringRef Part = Parts.empty() ? StringRef() : Parts.back();
+
+  // Remove duplicate `Parts'.
+  llvm::sort(Parts);
+  Parts.erase(llvm::unique(Parts), Parts.end());
+
+  auto GetVariant = [&]() {
+    unsigned Variant = 0;
+    for (auto I : Lines)
+      if (I.consume_front("CPU variant"))
+        I.ltrim("\t :").getAsInteger(0, Variant);
+    return Variant;
+  };
+
+  return getHostCPUNameForARMFromComponents(Implementer, Hardware, Part, Parts,
+                                            GetVariant);
+}
+
+StringRef sys::detail::getHostCPUNameForARM(uint64_t PrimaryCpuInfo,
+                                            ArrayRef<uint64_t> UniqueCpuInfos) {
+  // On Windows, the registry provides cached copied of the MIDR_EL1 register.
+  using PartNum = Bitfield::Element<uint16_t, 4, 12>;
+  using Implementer = Bitfield::Element<uint16_t, 24, 8>;
+  using Variant = Bitfield::Element<uint16_t, 20, 4>;
+
+  SmallVector<std::string> PartsHolder;
+  PartsHolder.reserve(UniqueCpuInfos.size());
+  for (auto Info : UniqueCpuInfos)
+    PartsHolder.push_back("0x" + utohexstr(Bitfield::get<PartNum>(Info),
+                                           /*LowerCase*/ true,
+                                           /*Width*/ 3));
+
+  SmallVector<StringRef> Parts;
+  Parts.reserve(PartsHolder.size());
+  for (const auto &Part : PartsHolder)
+    Parts.push_back(Part);
+
+  return getHostCPUNameForARMFromComponents(
+      "0x" + utohexstr(Bitfield::get<Implementer>(PrimaryCpuInfo),
+                       /*LowerCase*/ true,
+                       /*Width*/ 2),
+      /*Hardware*/ "",
+      "0x" + utohexstr(Bitfield::get<PartNum>(PrimaryCpuInfo),
+                       /*LowerCase*/ true,
+                       /*Width*/ 3),
+      Parts, [=]() { return Bitfield::get<Variant>(PrimaryCpuInfo); });
+}
+
 namespace {
 StringRef getCPUNameFromS390Model(unsigned int Id, bool HaveVectorSupport) {
   switch (Id) {
@@ -713,20 +759,20 @@ static StringRef getIntelProcessorTypeAndSubtype(unsigned Family,
   StringRef CPU;
 
   switch (Family) {
-  case 3:
+  case 0x3:
     CPU = "i386";
     break;
-  case 4:
+  case 0x4:
     CPU = "i486";
     break;
-  case 5:
+  case 0x5:
     if (testFeature(X86::FEATURE_MMX)) {
       CPU = "pentium-mmx";
       break;
     }
     CPU = "pentium";
     break;
-  case 6:
+  case 0x6:
     switch (Model) {
     case 0x0f: // Intel Core 2 Duo processor, Intel Core 2 Duo mobile
                // processor, Intel Core 2 Quad processor, Intel Core 2 Quad
@@ -1074,7 +1120,7 @@ static StringRef getIntelProcessorTypeAndSubtype(unsigned Family,
       break;
     }
     break;
-  case 15: {
+  case 0xf: {
     if (testFeature(X86::FEATURE_64BIT)) {
       CPU = "nocona";
       break;
@@ -1086,7 +1132,7 @@ static StringRef getIntelProcessorTypeAndSubtype(unsigned Family,
     CPU = "pentium4";
     break;
   }
-  case 19:
+  case 0x13:
     switch (Model) {
     // Diamond Rapids:
     case 0x01:
@@ -1450,6 +1496,75 @@ StringRef sys::getHostCPUName() {
   return "generic";
 }
 
+#elif defined(_M_ARM64) || defined(_M_ARM64EC)
+
+StringRef sys::getHostCPUName() {
+  constexpr char CentralProcessorKeyName[] =
+      "HARDWARE\\DESCRIPTION\\System\\CentralProcessor";
+  // Sub keys names are simple numbers ("0", "1", etc.) so 10 chars should be
+  // enough for the slash and name.
+  constexpr size_t SubKeyNameMaxSize = ARRAYSIZE(CentralProcessorKeyName) + 10;
+
+  SmallVector<uint64_t> Values;
+  uint64_t PrimaryCpuInfo;
+  char PrimaryPartKeyName[SubKeyNameMaxSize];
+  DWORD PrimaryPartKeyNameSize = 0;
+  HKEY CentralProcessorKey;
+  if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, CentralProcessorKeyName, 0, KEY_READ,
+                    &CentralProcessorKey) == ERROR_SUCCESS) {
+    for (unsigned Index = 0; Index < UINT32_MAX; ++Index) {
+      char SubKeyName[SubKeyNameMaxSize];
+      DWORD SubKeySize = SubKeyNameMaxSize;
+      HKEY SubKey;
+      if ((RegEnumKeyExA(CentralProcessorKey, Index, SubKeyName, &SubKeySize,
+                         nullptr, nullptr, nullptr,
+                         nullptr) == ERROR_SUCCESS) &&
+          (RegOpenKeyExA(CentralProcessorKey, SubKeyName, 0, KEY_READ,
+                         &SubKey) == ERROR_SUCCESS)) {
+        // The "CP 4000" registry key contains a cached copy of the MIDR_EL1
+        // register.
+        uint64_t RegValue;
+        DWORD ActualType;
+        DWORD RegValueSize = sizeof(RegValue);
+        if ((RegQueryValueExA(SubKey, "CP 4000", nullptr, &ActualType,
+                              (PBYTE)&RegValue,
+                              &RegValueSize) == ERROR_SUCCESS) &&
+            (ActualType == REG_QWORD) && RegValueSize == sizeof(RegValue)) {
+          // Assume that the part with the "highest" reg key name is the primary
+          // part (to match the way that Linux's cpuinfo is written). Win32
+          // makes no guarantees about the order of sub keys, so we have to
+          // compare the names.
+          if (PrimaryPartKeyNameSize < SubKeySize ||
+              (PrimaryPartKeyNameSize == SubKeySize &&
+               ::memcmp(SubKeyName, PrimaryPartKeyName, SubKeySize) > 0)) {
+            PrimaryCpuInfo = RegValue;
+            ::memcpy(PrimaryPartKeyName, SubKeyName, SubKeySize + 1);
+            PrimaryPartKeyNameSize = SubKeySize;
+          }
+          if (!llvm::is_contained(Values, RegValue)) {
+            Values.push_back(RegValue);
+          }
+        }
+        RegCloseKey(SubKey);
+      } else {
+        // No more sub keys.
+        break;
+      }
+    }
+    RegCloseKey(CentralProcessorKey);
+  }
+
+  if (Values.empty()) {
+    return "generic";
+  }
+
+  // Win32 makes no guarantees about the order of sub keys, so sort to ensure
+  // reproducibility.
+  llvm::sort(Values);
+
+  return detail::getHostCPUNameForARM(PrimaryCpuInfo, Values);
+}
+
 #elif defined(__APPLE__) && defined(__powerpc__)
 StringRef sys::getHostCPUName() {
   host_basic_info_data_t hostInfo;
diff --git a/llvm/lib/TargetParser/RISCVISAInfo.cpp b/llvm/lib/TargetParser/RISCVISAInfo.cpp
index d6afb8afd98b4..7617058aa102f 100644
--- a/llvm/lib/TargetParser/RISCVISAInfo.cpp
+++ b/llvm/lib/TargetParser/RISCVISAInfo.cpp
@@ -833,19 +833,6 @@ static bool operator<(StringRef LHS, const ImpliedExtsEntry &RHS) {
 #include "llvm/TargetParser/RISCVTargetParserDef.inc"
 
 void RISCVISAInfo::updateImplication() {
-  bool HasE = Exts.count("e") != 0;
-  bool HasI = Exts.count("i") != 0;
-
-  // If not in e extension and i extension does not exist, i extension is
-  // implied
-  if (!HasE && !HasI) {
-    auto Version = findDefaultVersion("i");
-    Exts["i"] = *Version;
-  }
-
-  if (HasE && HasI)
-    Exts.erase("i");
-
   assert(llvm::is_sorted(ImpliedExts) && "Table not sorted by Name");
 
   // This loop may execute over 1 iteration since implication can be layered
@@ -887,6 +874,21 @@ void RISCVISAInfo::updateImplication() {
     auto Version = findDefaultVersion("zcf");
     Exts["zcf"] = *Version;
   }
+
+  // Handle I/E after implications have been resolved, in case either
+  // of them was implied by another extension.
+  bool HasE = Exts.count("e") != 0;
+  bool HasI = Exts.count("i") != 0;
+
+  // If not in e extension and i extension does not exist, i extension is
+  // implied
+  if (!HasE && !HasI) {
+    auto Version = findDefaultVersion("i");
+    Exts["i"] = *Version;
+  }
+
+  if (HasE && HasI)
+    Exts.erase("i");
 }
 
 static constexpr StringLiteral CombineIntoExts[] = {
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 9cd35e35d4bc9..b72096553ad9b 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -143,8 +143,8 @@ constexpr FeatureBitset FeaturesDiamondRapids =
     FeatureCMPCCXADD | FeatureAVXIFMA | FeatureAVXNECONVERT |
     FeatureAVXVNNIINT8 | FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 |
     FeatureSM4 | FeatureEGPR | FeatureZU | FeatureCCMP | FeaturePush2Pop2 |
-    FeaturePPX | FeatureNDD | FeatureNF | FeatureCF | FeatureMOVRS |
-    FeatureAMX_MOVRS | FeatureAMX_AVX512 | FeatureAMX_FP8 | FeatureAMX_TF32 |
+    FeaturePPX | FeatureNDD | FeatureNF | FeatureMOVRS | FeatureAMX_MOVRS |
+    FeatureAMX_AVX512 | FeatureAMX_FP8 | FeatureAMX_TF32 |
     FeatureAMX_TRANSPOSE | FeatureUSERMSR;
 
 // Intel Atom processors.
diff --git a/llvm/lib/TargetParser/XtensaTargetParser.cpp b/llvm/lib/TargetParser/XtensaTargetParser.cpp
new file mode 100644
index 0000000000000..25725f2688cf3
--- /dev/null
+++ b/llvm/lib/TargetParser/XtensaTargetParser.cpp
@@ -0,0 +1,93 @@
+//==-- XtensaTargetParser - Parser for Xtensa features ------------*- C++ -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a target parser to recognise Xtensa hardware features
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/TargetParser/XtensaTargetParser.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSwitch.h"
+
+namespace llvm {
+
+namespace Xtensa {
+struct CPUInfo {
+  StringLiteral Name;
+  CPUKind Kind;
+  uint64_t Features;
+};
+
+struct FeatureName {
+  uint64_t ID;
+  const char *NameCStr;
+  size_t NameLength;
+
+  StringRef getName() const { return StringRef(NameCStr, NameLength); }
+};
+
+const FeatureName XtensaFeatureNames[] = {
+#define XTENSA_FEATURE(ID, NAME) {ID, "+" NAME, sizeof(NAME)},
+#include "llvm/TargetParser/XtensaTargetParser.def"
+};
+
+constexpr CPUInfo XtensaCPUInfo[] = {
+#define XTENSA_CPU(ENUM, NAME, FEATURES) {NAME, CK_##ENUM, FEATURES},
+#include "llvm/TargetParser/XtensaTargetParser.def"
+};
+
+StringRef getBaseName(StringRef CPU) {
+  return llvm::StringSwitch<StringRef>(CPU)
+#define XTENSA_CPU_ALIAS(NAME, ANAME) .Case(ANAME, NAME)
+#include "llvm/TargetParser/XtensaTargetParser.def"
+      .Default(CPU);
+}
+
+StringRef getAliasName(StringRef CPU) {
+  return llvm::StringSwitch<StringRef>(CPU)
+#define XTENSA_CPU_ALIAS(NAME, ANAME) .Case(NAME, ANAME)
+#include "llvm/TargetParser/XtensaTargetParser.def"
+      .Default(CPU);
+}
+
+CPUKind parseCPUKind(StringRef CPU) {
+  CPU = getBaseName(CPU);
+  return llvm::StringSwitch<CPUKind>(CPU)
+#define XTENSA_CPU(ENUM, NAME, FEATURES) .Case(NAME, CK_##ENUM)
+#include "llvm/TargetParser/XtensaTargetParser.def"
+      .Default(CK_INVALID);
+}
+
+// Get all features for the CPU
+void getCPUFeatures(StringRef CPU, std::vector<StringRef> &Features) {
+  CPU = getBaseName(CPU);
+  auto I = llvm::find_if(XtensaCPUInfo,
+                         [&](const CPUInfo &CI) { return CI.Name == CPU; });
+  assert(I != std::end(XtensaCPUInfo) && "CPU not found!");
+  uint64_t Bits = I->Features;
+
+  for (const auto &F : XtensaFeatureNames) {
+    if ((Bits & F.ID) == F.ID)
+      Features.push_back(F.getName());
+  }
+}
+
+// Find all valid CPUs
+void fillValidCPUList(std::vector<StringRef> &Values) {
+  for (const auto &C : XtensaCPUInfo) {
+    if (C.Kind != CK_INVALID) {
+      Values.emplace_back(C.Name);
+      StringRef Name = getAliasName(C.Name);
+      if (Name != C.Name)
+        Values.emplace_back(Name);
+    }
+  }
+}
+
+} // namespace Xtensa
+} // namespace llvm
diff --git a/llvm/lib/TextAPI/Architecture.cpp b/llvm/lib/TextAPI/Architecture.cpp
index 51ca91db13008..3b5306746e1c5 100644
--- a/llvm/lib/TextAPI/Architecture.cpp
+++ b/llvm/lib/TextAPI/Architecture.cpp
@@ -21,7 +21,7 @@ namespace llvm {
 namespace MachO {
 
 Architecture getArchitectureFromCpuType(uint32_t CPUType, uint32_t CPUSubType) {
-#define ARCHINFO(Arch, Type, Subtype, NumBits)                                 \
+#define ARCHINFO(Arch, Name, Type, Subtype, NumBits)                           \
   if (CPUType == (Type) &&                                                     \
       (CPUSubType & ~MachO::CPU_SUBTYPE_MASK) == (Subtype))                    \
     return AK_##Arch;
@@ -33,7 +33,7 @@ Architecture getArchitectureFromCpuType(uint32_t CPUType, uint32_t CPUSubType) {
 
 Architecture getArchitectureFromName(StringRef Name) {
   return StringSwitch<Architecture>(Name)
-#define ARCHINFO(Arch, Type, Subtype, NumBits) .Case(#Arch, AK_##Arch)
+#define ARCHINFO(Arch, Name, Type, Subtype, NumBits) .Case(#Name, AK_##Arch)
 #include "llvm/TextAPI/Architecture.def"
 #undef ARCHINFO
       .Default(AK_unknown);
@@ -41,9 +41,9 @@ Architecture getArchitectureFromName(StringRef Name) {
 
 StringRef getArchitectureName(Architecture Arch) {
   switch (Arch) {
-#define ARCHINFO(Arch, Type, Subtype, NumBits)                                 \
+#define ARCHINFO(Arch, Name, Type, Subtype, NumBits)                           \
   case AK_##Arch:                                                              \
-    return #Arch;
+    return #Name;
 #include "llvm/TextAPI/Architecture.def"
 #undef ARCHINFO
   case AK_unknown:
@@ -57,7 +57,7 @@ StringRef getArchitectureName(Architecture Arch) {
 
 std::pair<uint32_t, uint32_t> getCPUTypeFromArchitecture(Architecture Arch) {
   switch (Arch) {
-#define ARCHINFO(Arch, Type, Subtype, NumBits)                                 \
+#define ARCHINFO(Arch, Name, Type, Subtype, NumBits)                           \
   case AK_##Arch:                                                              \
     return std::make_pair(Type, Subtype);
 #include "llvm/TextAPI/Architecture.def"
@@ -77,7 +77,7 @@ Architecture mapToArchitecture(const Triple &Target) {
 
 bool is64Bit(Architecture Arch) {
   switch (Arch) {
-#define ARCHINFO(Arch, Type, Subtype, NumBits)                                 \
+#define ARCHINFO(Arch, Name, Type, Subtype, NumBits)                           \
   case AK_##Arch:                                                              \
     return NumBits == 64;
 #include "llvm/TextAPI/Architecture.def"
diff --git a/llvm/lib/TextAPI/TextStubCommon.cpp b/llvm/lib/TextAPI/TextStubCommon.cpp
index 0b710b0790b3d..7bf1f9ab4c939 100644
--- a/llvm/lib/TextAPI/TextStubCommon.cpp
+++ b/llvm/lib/TextAPI/TextStubCommon.cpp
@@ -133,7 +133,7 @@ QuotingType ScalarTraits<PlatformSet>::mustQuote(StringRef) {
 
 void ScalarBitSetTraits<ArchitectureSet>::bitset(IO &IO,
                                                  ArchitectureSet &Archs) {
-#define ARCHINFO(arch, type, subtype, numbits)                                 \
+#define ARCHINFO(arch, name, type, subtype, numbits)                           \
   IO.bitSetCase(Archs, #arch, 1U << static_cast<int>(AK_##arch));
 #include "llvm/TextAPI/Architecture.def"
 #undef ARCHINFO
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 40a7f8043034e..40de36d81ddd2 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -83,8 +83,8 @@ static bool foldGuardedFunnelShift(Instruction &I, const DominatorTree &DT) {
     //  == (ShVal0 << ShAmt) | (ShVal1 >> (Width -ShAmt))
     if (match(V, m_OneUse(m_c_Or(
                      m_Shl(m_Value(ShVal0), m_Value(ShAmt)),
-                     m_LShr(m_Value(ShVal1),
-                            m_Sub(m_SpecificInt(Width), m_Deferred(ShAmt))))))) {
+                     m_LShr(m_Value(ShVal1), m_Sub(m_SpecificInt(Width),
+                                                   m_Deferred(ShAmt))))))) {
       return Intrinsic::fshl;
     }
 
@@ -617,7 +617,7 @@ struct LoadOps {
   LoadInst *RootInsert = nullptr;
   bool FoundRoot = false;
   uint64_t LoadSize = 0;
-  const APInt *Shift = nullptr;
+  uint64_t Shift = 0;
   Type *ZextType;
   AAMDNodes AATags;
 };
@@ -627,17 +627,15 @@ struct LoadOps {
 // (ZExt(L1) << shift1) | ZExt(L2) -> ZExt(L3)
 static bool foldLoadsRecursive(Value *V, LoadOps &LOps, const DataLayout &DL,
                                AliasAnalysis &AA) {
-  const APInt *ShAmt2 = nullptr;
+  uint64_t ShAmt2;
   Value *X;
   Instruction *L1, *L2;
 
   // Go to the last node with loads.
-  if (match(V, m_OneUse(m_c_Or(
-                   m_Value(X),
-                   m_OneUse(m_Shl(m_OneUse(m_ZExt(m_OneUse(m_Instruction(L2)))),
-                                  m_APInt(ShAmt2)))))) ||
-      match(V, m_OneUse(m_Or(m_Value(X),
-                             m_OneUse(m_ZExt(m_OneUse(m_Instruction(L2)))))))) {
+  if (match(V,
+            m_OneUse(m_c_Or(m_Value(X), m_OneUse(m_ShlOrSelf(
+                                            m_OneUse(m_ZExt(m_Instruction(L2))),
+                                            ShAmt2)))))) {
     if (!foldLoadsRecursive(X, LOps, DL, AA) && LOps.FoundRoot)
       // Avoid Partial chain merge.
       return false;
@@ -646,11 +644,10 @@ static bool foldLoadsRecursive(Value *V, LoadOps &LOps, const DataLayout &DL,
 
   // Check if the pattern has loads
   LoadInst *LI1 = LOps.Root;
-  const APInt *ShAmt1 = LOps.Shift;
+  uint64_t ShAmt1 = LOps.Shift;
   if (LOps.FoundRoot == false &&
-      (match(X, m_OneUse(m_ZExt(m_Instruction(L1)))) ||
-       match(X, m_OneUse(m_Shl(m_OneUse(m_ZExt(m_OneUse(m_Instruction(L1)))),
-                               m_APInt(ShAmt1)))))) {
+      match(X, m_OneUse(
+                   m_ShlOrSelf(m_OneUse(m_ZExt(m_Instruction(L1))), ShAmt1)))) {
     LI1 = dyn_cast<LoadInst>(L1);
   }
   LoadInst *LI2 = dyn_cast<LoadInst>(L2);
@@ -726,13 +723,6 @@ static bool foldLoadsRecursive(Value *V, LoadOps &LOps, const DataLayout &DL,
   if (IsBigEndian)
     std::swap(ShAmt1, ShAmt2);
 
-  // Find Shifts values.
-  uint64_t Shift1 = 0, Shift2 = 0;
-  if (ShAmt1)
-    Shift1 = ShAmt1->getZExtValue();
-  if (ShAmt2)
-    Shift2 = ShAmt2->getZExtValue();
-
   // First load is always LI1. This is where we put the new load.
   // Use the merged load size available from LI1 for forward loads.
   if (LOps.FoundRoot) {
@@ -747,7 +737,7 @@ static bool foldLoadsRecursive(Value *V, LoadOps &LOps, const DataLayout &DL,
   uint64_t ShiftDiff = IsBigEndian ? LoadSize2 : LoadSize1;
   uint64_t PrevSize =
       DL.getTypeStoreSize(IntegerType::get(LI1->getContext(), LoadSize1));
-  if ((Shift2 - Shift1) != ShiftDiff || (Offset2 - Offset1) != PrevSize)
+  if ((ShAmt2 - ShAmt1) != ShiftDiff || (Offset2 - Offset1) != PrevSize)
     return false;
 
   // Update LOps
@@ -824,7 +814,7 @@ static bool foldConsecutiveLoads(Instruction &I, const DataLayout &DL,
   // Check if shift needed. We need to shift with the amount of load1
   // shift if not zero.
   if (LOps.Shift)
-    NewOp = Builder.CreateShl(NewOp, ConstantInt::get(I.getContext(), *LOps.Shift));
+    NewOp = Builder.CreateShl(NewOp, LOps.Shift);
   I.replaceAllUsesWith(NewOp);
 
   return true;
@@ -860,11 +850,9 @@ static std::optional<PartStore> matchPartStore(Instruction &I,
     return std::nullopt;
 
   uint64_t ValWidth = StoredTy->getPrimitiveSizeInBits();
-  uint64_t ValOffset = 0;
+  uint64_t ValOffset;
   Value *Val;
-  if (!match(StoredVal, m_CombineOr(m_Trunc(m_LShr(m_Value(Val),
-                                                   m_ConstantInt(ValOffset))),
-                                    m_Trunc(m_Value(Val)))))
+  if (!match(StoredVal, m_Trunc(m_LShrOrSelf(m_Value(Val), ValOffset))))
     return std::nullopt;
 
   Value *Ptr = Store->getPointerOperand();
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 332050860e05b..b775c43460190 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1821,7 +1821,7 @@ static void sinkLifetimeStartMarkers(Function &F, coro::Shape &Shape,
       // only used outside the region.
       if (Valid && Lifetimes.size() != 0) {
         auto *NewLifetime = Lifetimes[0]->clone();
-        NewLifetime->replaceUsesOfWith(NewLifetime->getOperand(1), AI);
+        NewLifetime->replaceUsesOfWith(NewLifetime->getOperand(0), AI);
         NewLifetime->insertBefore(DomBB->getTerminator()->getIterator());
 
         // All the outsided lifetime.start markers are no longer necessary.
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index ab906f9fa03a1..180ac9c61e7df 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -2252,6 +2252,10 @@ PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
       UR.CWorklist.insert(CurrentSCC);
       for (Function *Clone : Clones)
         UR.CWorklist.insert(CG.lookupSCC(CG.get(*Clone)));
+    } else if (Shape.ABI == coro::ABI::Async) {
+      // Reprocess the function to inline the tail called return function of
+      // coro.async.end.
+      UR.CWorklist.insert(&C);
     }
   }
 
diff --git a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
index 4e717685555b5..d5d60a320b521 100644
--- a/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
+++ b/llvm/lib/Transforms/Coroutines/SpillUtils.cpp
@@ -264,11 +264,6 @@ struct AllocaUseVisitor : PtrUseVisitor<AllocaUseVisitor> {
   }
 
   void visitIntrinsicInst(IntrinsicInst &II) {
-    // When we found the lifetime markers refers to a
-    // subrange of the original alloca, ignore the lifetime
-    // markers to avoid misleading the analysis.
-    if (!IsOffsetKnown || !Offset.isZero())
-      return Base::visitIntrinsicInst(II);
     switch (II.getIntrinsicID()) {
     default:
       return Base::visitIntrinsicInst(II);
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 01da01239382b..ff4da3378f82d 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -2008,9 +2008,8 @@ struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
     // destination) and second (=source) arguments as we know how they are
     // accessed.
     if (auto *MI = dyn_cast_or_null<MemIntrinsic>(getCtxI())) {
-      ConstantInt *Length = dyn_cast<ConstantInt>(MI->getLength());
       int64_t LengthVal = AA::RangeTy::Unknown;
-      if (Length)
+      if (auto Length = MI->getLengthInBytes())
         LengthVal = Length->getSExtValue();
       unsigned ArgNo = getIRPosition().getCallSiteArgNo();
       ChangeStatus Changed = ChangeStatus::UNCHANGED;
diff --git a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
index da60f521bf084..042578d26818a 100644
--- a/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
+++ b/llvm/lib/Transforms/IPO/ExpandVariadics.cpp
@@ -53,7 +53,6 @@
 
 #include "llvm/Transforms/IPO/ExpandVariadics.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/IR/Constants.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
@@ -226,13 +225,6 @@ class ExpandVariadics : public ModulePass {
                              /*IsVarArgs=*/false);
   }
 
-  static ConstantInt *sizeOfAlloca(LLVMContext &Ctx, const DataLayout &DL,
-                                   AllocaInst *Alloced) {
-    std::optional<TypeSize> AllocaTypeSize = Alloced->getAllocationSize(DL);
-    uint64_t AsInt = AllocaTypeSize ? AllocaTypeSize->getFixedValue() : 0;
-    return ConstantInt::get(Type::getInt64Ty(Ctx), AsInt);
-  }
-
   bool expansionApplicableToFunction(Module &M, Function *F) {
     if (F->isIntrinsic() || !F->isVarArg() ||
         F->hasFnAttribute(Attribute::Naked))
@@ -577,8 +569,7 @@ ExpandVariadics::defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
   AllocaInst *VaListInstance =
       Builder.CreateAlloca(VaListTy, nullptr, "va_start");
 
-  Builder.CreateLifetimeStart(VaListInstance,
-                              sizeOfAlloca(Ctx, DL, VaListInstance));
+  Builder.CreateLifetimeStart(VaListInstance);
 
   Builder.CreateIntrinsic(Intrinsic::vastart, {DL.getAllocaPtrType(Ctx)},
                           {VaListInstance});
@@ -595,8 +586,7 @@ ExpandVariadics::defineVariadicWrapper(Module &M, IRBuilder<> &Builder,
 
   Builder.CreateIntrinsic(Intrinsic::vaend, {DL.getAllocaPtrType(Ctx)},
                           {VaListInstance});
-  Builder.CreateLifetimeEnd(VaListInstance,
-                            sizeOfAlloca(Ctx, DL, VaListInstance));
+  Builder.CreateLifetimeEnd(VaListInstance);
 
   if (Result->getType()->isVoidTy())
     Builder.CreateRetVoid();
@@ -746,7 +736,7 @@ bool ExpandVariadics::expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB,
 
   // Initialize the fields in the struct
   Builder.SetInsertPoint(CB);
-  Builder.CreateLifetimeStart(Alloced, sizeOfAlloca(Ctx, DL, Alloced));
+  Builder.CreateLifetimeStart(Alloced);
   Frame.initializeStructAlloca(DL, Builder, Alloced);
 
   const unsigned NumArgs = FuncType->getNumParams();
@@ -762,7 +752,7 @@ bool ExpandVariadics::expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB,
       Builder.SetCurrentDebugLocation(CB->getStableDebugLoc());
       VaList = Builder.CreateAlloca(VaListTy, nullptr, "va_argument");
       Builder.SetInsertPoint(CB);
-      Builder.CreateLifetimeStart(VaList, sizeOfAlloca(Ctx, DL, VaList));
+      Builder.CreateLifetimeStart(VaList);
     }
     Builder.SetInsertPoint(CB);
     Args.push_back(ABI->initializeVaList(M, Ctx, Builder, VaList, Alloced));
@@ -802,9 +792,9 @@ bool ExpandVariadics::expandCall(Module &M, IRBuilder<> &Builder, CallBase *CB,
   }
 
   if (VaList)
-    Builder.CreateLifetimeEnd(VaList, sizeOfAlloca(Ctx, DL, VaList));
+    Builder.CreateLifetimeEnd(VaList);
 
-  Builder.CreateLifetimeEnd(Alloced, sizeOfAlloca(Ctx, DL, Alloced));
+  Builder.CreateLifetimeEnd(Alloced);
 
   NewCB->setAttributes(PAL);
   NewCB->takeName(CB);
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index 8262c8c3a90f2..8d9a0e7eaef63 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -20,7 +20,6 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AssumptionCache.h"
@@ -273,7 +272,7 @@ MemoryEffects llvm::computeFunctionBodyMemoryAccess(Function &F,
 /// Deduce readonly/readnone/writeonly attributes for the SCC.
 template <typename AARGetterT>
 static void addMemoryAttrs(const SCCNodeSet &SCCNodes, AARGetterT &&AARGetter,
-                           SmallSet<Function *, 8> &Changed) {
+                           SmallPtrSet<Function *, 8> &Changed) {
   MemoryEffects ME = MemoryEffects::none();
   MemoryEffects RecursiveArgME = MemoryEffects::none();
   for (Function *F : SCCNodes) {
@@ -1002,7 +1001,7 @@ determinePointerAccessAttrs(Argument *A,
 
 /// Deduce returned attributes for the SCC.
 static void addArgumentReturnedAttrs(const SCCNodeSet &SCCNodes,
-                                     SmallSet<Function *, 8> &Changed) {
+                                     SmallPtrSet<Function *, 8> &Changed) {
   // Check each function in turn, determining if an argument is always returned.
   for (Function *F : SCCNodes) {
     // We can infer and propagate function attributes only when we know that the
@@ -1238,7 +1237,7 @@ static bool inferInitializes(Argument &A, Function &F) {
 
 /// Deduce nocapture attributes for the SCC.
 static void addArgumentAttrs(const SCCNodeSet &SCCNodes,
-                             SmallSet<Function *, 8> &Changed,
+                             SmallPtrSet<Function *, 8> &Changed,
                              bool SkipInitializes) {
   ArgumentGraph AG;
 
@@ -1510,7 +1509,7 @@ static bool isFunctionMallocLike(Function *F, const SCCNodeSet &SCCNodes) {
 
 /// Deduce noalias attributes for the SCC.
 static void addNoAliasAttrs(const SCCNodeSet &SCCNodes,
-                            SmallSet<Function *, 8> &Changed) {
+                            SmallPtrSet<Function *, 8> &Changed) {
   // Check each function in turn, determining which functions return noalias
   // pointers.
   for (Function *F : SCCNodes) {
@@ -1623,7 +1622,7 @@ static bool isReturnNonNull(Function *F, const SCCNodeSet &SCCNodes,
 
 /// Deduce nonnull attributes for the SCC.
 static void addNonNullAttrs(const SCCNodeSet &SCCNodes,
-                            SmallSet<Function *, 8> &Changed) {
+                            SmallPtrSet<Function *, 8> &Changed) {
   // Speculative that all functions in the SCC return only nonnull
   // pointers.  We may refute this as we analyze functions.
   bool SCCReturnsNonNull = true;
@@ -1680,7 +1679,7 @@ static void addNonNullAttrs(const SCCNodeSet &SCCNodes,
 
 /// Deduce noundef attributes for the SCC.
 static void addNoUndefAttrs(const SCCNodeSet &SCCNodes,
-                            SmallSet<Function *, 8> &Changed) {
+                            SmallPtrSet<Function *, 8> &Changed) {
   // Check each function in turn, determining which functions return noundef
   // values.
   for (Function *F : SCCNodes) {
@@ -1788,13 +1787,13 @@ class AttributeInferer {
     InferenceDescriptors.push_back(AttrInference);
   }
 
-  void run(const SCCNodeSet &SCCNodes, SmallSet<Function *, 8> &Changed);
+  void run(const SCCNodeSet &SCCNodes, SmallPtrSet<Function *, 8> &Changed);
 };
 
 /// Perform all the requested attribute inference actions according to the
 /// attribute predicates stored before.
 void AttributeInferer::run(const SCCNodeSet &SCCNodes,
-                           SmallSet<Function *, 8> &Changed) {
+                           SmallPtrSet<Function *, 8> &Changed) {
   SmallVector<InferenceDescriptor, 4> InferInSCC = InferenceDescriptors;
   // Go through all the functions in SCC and check corresponding attribute
   // assumptions for each of them. Attributes that are invalid for this SCC
@@ -1969,7 +1968,7 @@ static bool InstrBreaksNoSync(Instruction &I, const SCCNodeSet &SCCNodes) {
 ///
 /// Returns true if any changes to function attributes were made.
 static void inferConvergent(const SCCNodeSet &SCCNodes,
-                            SmallSet<Function *, 8> &Changed) {
+                            SmallPtrSet<Function *, 8> &Changed) {
   AttributeInferer AI;
 
   // Request to remove the convergent attribute from all functions in the SCC
@@ -2000,7 +1999,7 @@ static void inferConvergent(const SCCNodeSet &SCCNodes,
 ///
 /// Returns true if any changes to function attributes were made.
 static void inferAttrsFromFunctionBodies(const SCCNodeSet &SCCNodes,
-                                         SmallSet<Function *, 8> &Changed) {
+                                         SmallPtrSet<Function *, 8> &Changed) {
   AttributeInferer AI;
 
   if (!DisableNoUnwindInference)
@@ -2069,7 +2068,7 @@ static void inferAttrsFromFunctionBodies(const SCCNodeSet &SCCNodes,
 }
 
 static void addNoRecurseAttrs(const SCCNodeSet &SCCNodes,
-                              SmallSet<Function *, 8> &Changed) {
+                              SmallPtrSet<Function *, 8> &Changed) {
   // Try and identify functions that do not recurse.
 
   // If the SCC contains multiple nodes we know for sure there is recursion.
@@ -2105,7 +2104,7 @@ static void addNoRecurseAttrs(const SCCNodeSet &SCCNodes,
 
 // Set the noreturn function attribute if possible.
 static void addNoReturnAttrs(const SCCNodeSet &SCCNodes,
-                             SmallSet<Function *, 8> &Changed) {
+                             SmallPtrSet<Function *, 8> &Changed) {
   for (Function *F : SCCNodes) {
     if (!F || !F->hasExactDefinition() || F->hasFnAttribute(Attribute::Naked) ||
         F->doesNotReturn())
@@ -2166,7 +2165,7 @@ static bool allPathsGoThroughCold(Function &F) {
 
 // Set the cold function attribute if possible.
 static void addColdAttrs(const SCCNodeSet &SCCNodes,
-                         SmallSet<Function *, 8> &Changed) {
+                         SmallPtrSet<Function *, 8> &Changed) {
   for (Function *F : SCCNodes) {
     if (!F || !F->hasExactDefinition() || F->hasFnAttribute(Attribute::Naked) ||
         F->hasFnAttribute(Attribute::Cold) || F->hasFnAttribute(Attribute::Hot))
@@ -2213,7 +2212,7 @@ static bool functionWillReturn(const Function &F) {
 
 // Set the willreturn function attribute if possible.
 static void addWillReturn(const SCCNodeSet &SCCNodes,
-                          SmallSet<Function *, 8> &Changed) {
+                          SmallPtrSet<Function *, 8> &Changed) {
   for (Function *F : SCCNodes) {
     if (!F || F->willReturn() || !functionWillReturn(*F))
       continue;
@@ -2239,7 +2238,7 @@ static SCCNodesResult createSCCNodeSet(ArrayRef<Function *> Functions) {
 }
 
 template <typename AARGetterT>
-static SmallSet<Function *, 8>
+static SmallPtrSet<Function *, 8>
 deriveAttrsInPostOrder(ArrayRef<Function *> Functions, AARGetterT &&AARGetter,
                        bool ArgAttrsOnly) {
   SCCNodesResult Nodes = createSCCNodeSet(Functions);
@@ -2248,7 +2247,7 @@ deriveAttrsInPostOrder(ArrayRef<Function *> Functions, AARGetterT &&AARGetter,
   if (Nodes.SCCNodes.empty())
     return {};
 
-  SmallSet<Function *, 8> Changed;
+  SmallPtrSet<Function *, 8> Changed;
   if (ArgAttrsOnly) {
     // ArgAttrsOnly means to only infer attributes that may aid optimizations
     // on the *current* function. "initializes" attribute is to aid
@@ -2328,7 +2327,7 @@ PreservedAnalyses PostOrderFunctionAttrsPass::run(LazyCallGraph::SCC &C,
     // through function attributes.
     for (auto *U : Changed->users()) {
       if (auto *Call = dyn_cast<CallBase>(U)) {
-        if (Call->getCalledFunction() == Changed)
+        if (Call->getCalledOperand() == Changed)
           FAM.invalidate(*Call->getFunction(), FuncPA);
       }
     }
diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
index 45fa9d57e4862..9196a0147c435 100644
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -16,7 +16,6 @@
 #include "llvm/Analysis/ValueLattice.h"
 #include "llvm/Analysis/ValueLatticeUtils.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/Transforms/Scalar/SCCP.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/SCCPSolver.h"
@@ -400,12 +399,6 @@ Constant *InstCostVisitor::visitFreezeInst(FreezeInst &I) {
 Constant *InstCostVisitor::visitCallBase(CallBase &I) {
   assert(LastVisited != KnownConstants.end() && "Invalid iterator!");
 
-  // Look through calls to ssa_copy intrinsics.
-  if (auto *II = dyn_cast<IntrinsicInst>(&I);
-      II && II->getIntrinsicID() == Intrinsic::ssa_copy) {
-    return LastVisited->second;
-  }
-
   Function *F = I.getCalledFunction();
   if (!F || !canConstantFoldCallTo(&I, F))
     return nullptr;
@@ -611,17 +604,15 @@ void FunctionSpecializer::promoteConstantStackValues(Function *F) {
   }
 }
 
-// ssa_copy intrinsics are introduced by the SCCP solver. These intrinsics
-// interfere with the promoteConstantStackValues() optimization.
+// The SCCP solver inserts bitcasts for PredicateInfo. These interfere with the
+// promoteConstantStackValues() optimization.
 static void removeSSACopy(Function &F) {
   for (BasicBlock &BB : F) {
     for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
-      auto *II = dyn_cast<IntrinsicInst>(&Inst);
-      if (!II)
-        continue;
-      if (II->getIntrinsicID() != Intrinsic::ssa_copy)
+      auto *BC = dyn_cast<BitCastInst>(&Inst);
+      if (!BC || BC->getType() != BC->getOperand(0)->getType())
         continue;
-      Inst.replaceAllUsesWith(II->getOperand(0));
+      Inst.replaceAllUsesWith(BC->getOperand(0));
       Inst.eraseFromParent();
     }
   }
diff --git a/llvm/lib/Transforms/IPO/GlobalDCE.cpp b/llvm/lib/Transforms/IPO/GlobalDCE.cpp
index 45fb1f5212b70..c576fbc92f709 100644
--- a/llvm/lib/Transforms/IPO/GlobalDCE.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalDCE.cpp
@@ -21,6 +21,8 @@
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/Utils/CtorUtils.h"
@@ -30,6 +32,35 @@ using namespace llvm;
 
 #define DEBUG_TYPE "globaldce"
 
+namespace {
+class GlobalDCELegacyPass : public ModulePass {
+public:
+  static char ID; // Pass identification, replacement for typeid
+  GlobalDCELegacyPass() : ModulePass(ID) {
+    initializeGlobalDCELegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+  bool runOnModule(Module &M) override {
+    if (skipModule(M))
+      return false;
+    // Note: GlobalDCEPass does not use any analyses, so we're safe to call the
+    // new-pm style pass with a default-initialized analysis manager here
+    ModuleAnalysisManager MAM;
+    auto PA = Impl.run(M, MAM);
+    return !PA.areAllPreserved();
+  }
+
+private:
+  GlobalDCEPass Impl;
+};
+} // namespace
+
+char GlobalDCELegacyPass::ID = 0;
+INITIALIZE_PASS(GlobalDCELegacyPass, "globaldce", "Dead Global Elimination",
+                false, false)
+
+// Public interface to the GlobalDCEPass.
+ModulePass *llvm::createGlobalDCEPass() { return new GlobalDCELegacyPass(); }
+
 static cl::opt<bool>
     ClEnableVFE("enable-vfe", cl::Hidden, cl::init(true),
                 cl::desc("Enable virtual function elimination"));
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index bdda4980c1005..d7edd1288309b 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -1133,9 +1133,6 @@ static bool
 optimizeOnceStoredGlobal(GlobalVariable *GV, Value *StoredOnceVal,
                          const DataLayout &DL,
                          function_ref<TargetLibraryInfo &(Function &)> GetTLI) {
-  // Ignore no-op GEPs and bitcasts.
-  StoredOnceVal = StoredOnceVal->stripPointerCasts();
-
   // If we are dealing with a pointer global that is initialized to null and
   // only has one (non-null) value stored into it, then we can optimize any
   // users of the loaded value (often calls and loads) that would trap if the
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 6554377a4c155..88f5ca08fe8ab 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -459,6 +459,9 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
                              }),
               Calls.end());
 
+          // Report inlining decision BEFORE deleting function contents, so we
+          // can still access e.g. the DebugLoc
+          Advice->recordInliningWithCalleeDeleted();
           // Clear the body and queue the function itself for call graph
           // updating when we finish inlining.
           makeFunctionBodyUnreachable(Callee);
@@ -470,9 +473,7 @@ PreservedAnalyses InlinerPass::run(LazyCallGraph::SCC &InitialC,
           DeadFunctionsInComdats.push_back(&Callee);
         }
       }
-      if (CalleeWasDeleted)
-        Advice->recordInliningWithCalleeDeleted();
-      else
+      if (!CalleeWasDeleted)
         Advice->recordInlining();
     }
 
diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
index 844e27590e501..1185e6378295a 100644
--- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp
+++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp
@@ -284,6 +284,10 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
         Calls->erase_if([&](const std::pair<CallBase *, int> &Call) {
           return Call.first->getCaller() == &Callee;
         });
+
+        // Report inlining decision BEFORE deleting function contents, so we
+        // can still access e.g. the DebugLoc
+        Advice->recordInliningWithCalleeDeleted();
         // Clear the body and queue the function itself for deletion when we
         // finish inlining.
         // Note that after this point, it is an error to do anything other
@@ -295,9 +299,7 @@ PreservedAnalyses ModuleInlinerPass::run(Module &M,
         CalleeWasDeleted = true;
       }
     }
-    if (CalleeWasDeleted)
-      Advice->recordInliningWithCalleeDeleted();
-    else
+    if (!CalleeWasDeleted)
       Advice->recordInlining();
   }
 
diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
index 4387c3827c015..838f97c8f49af 100644
--- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
+++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
@@ -350,12 +350,20 @@ void splitAndWriteThinLTOBitcode(
       });
     }
 
+  auto MustEmitToMergedModule = [](const GlobalValue *GV) {
+    // The __cfi_check definition is filled in by the CrossDSOCFI pass which
+    // runs only in the merged module.
+    return GV->getName() == "__cfi_check";
+  };
+
   ValueToValueMapTy VMap;
   std::unique_ptr<Module> MergedM(
       CloneModule(M, VMap, [&](const GlobalValue *GV) -> bool {
         if (const auto *C = GV->getComdat())
           if (MergedMComdats.count(C))
             return true;
+        if (MustEmitToMergedModule(GV))
+          return true;
         if (auto *F = dyn_cast<Function>(GV))
           return EligibleVirtualFns.count(F);
         if (auto *GVar =
@@ -372,7 +380,7 @@ void splitAndWriteThinLTOBitcode(
   cloneUsedGlobalVariables(M, *MergedM, /*CompilerUsed*/ true);
 
   for (Function &F : *MergedM)
-    if (!F.isDeclaration()) {
+    if (!F.isDeclaration() && !MustEmitToMergedModule(&F)) {
       // Reset the linkage of all functions eligible for virtual constant
       // propagation. The canonical definitions live in the thin LTO module so
       // that they can be imported.
@@ -398,6 +406,8 @@ void splitAndWriteThinLTOBitcode(
     if (const auto *C = GV->getComdat())
       if (MergedMComdats.count(C))
         return false;
+    if (MustEmitToMergedModule(GV))
+      return false;
     return true;
   });
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index d7971e8e3caea..a13d3ceb61320 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1338,16 +1338,51 @@ Value *InstCombinerImpl::foldAndOrOfICmpsUsingRanges(ICmpInst *ICmp1,
       V2 = X;
   }
 
+  // Look through and with a negative power of 2 mask on V1 or V2. This
+  // detects idioms of the form `(x == A) || ((x & Mask) == A + 1)` where A + 1
+  // is aligned to the mask and A + 1 >= |Mask|. This pattern corresponds to a
+  // contiguous range check, which can be folded into an addition and compare.
+  // The same applies for `(x != A) && ((x & Mask) != A + 1)`.
+  auto AreContiguousRangePredicates = [](CmpPredicate Pred1, CmpPredicate Pred2,
+                                         bool IsAnd) {
+    if (IsAnd)
+      return Pred1 == ICmpInst::ICMP_NE && Pred2 == ICmpInst::ICMP_NE;
+    return Pred1 == ICmpInst::ICMP_EQ && Pred2 == ICmpInst::ICMP_EQ;
+  };
+  const APInt *Mask1 = nullptr, *Mask2 = nullptr;
+  bool MatchedAnd1 = false, MatchedAnd2 = false;
+  if (V1 != V2 && AreContiguousRangePredicates(Pred1, Pred2, IsAnd)) {
+    Value *X;
+    if (match(V1, m_OneUse(m_And(m_Value(X), m_NegatedPower2(Mask1)))) &&
+        C1->getBitWidth() == C2->getBitWidth() && *C1 == *C2 + 1 &&
+        C1->uge(Mask1->abs()) && C1->isPowerOf2()) {
+      MatchedAnd1 = true;
+      V1 = X;
+    }
+    if (match(V2, m_OneUse(m_And(m_Value(X), m_NegatedPower2(Mask2)))) &&
+        C1->getBitWidth() == C2->getBitWidth() && *C2 == *C1 + 1 &&
+        C2->uge(Mask2->abs()) && C2->isPowerOf2()) {
+      MatchedAnd2 = true;
+      V2 = X;
+    }
+  }
+
   if (V1 != V2)
     return nullptr;
 
-  ConstantRange CR1 = ConstantRange::makeExactICmpRegion(
-      IsAnd ? ICmpInst::getInverseCmpPredicate(Pred1) : Pred1, *C1);
+  ConstantRange CR1 =
+      MatchedAnd1
+          ? ConstantRange(*C1, *C1 - *Mask1)
+          : ConstantRange::makeExactICmpRegion(
+                IsAnd ? ICmpInst::getInverseCmpPredicate(Pred1) : Pred1, *C1);
   if (Offset1)
     CR1 = CR1.subtract(*Offset1);
 
-  ConstantRange CR2 = ConstantRange::makeExactICmpRegion(
-      IsAnd ? ICmpInst::getInverseCmpPredicate(Pred2) : Pred2, *C2);
+  ConstantRange CR2 =
+      MatchedAnd2
+          ? ConstantRange(*C2, *C2 - *Mask2)
+          : ConstantRange::makeExactICmpRegion(
+                IsAnd ? ICmpInst::getInverseCmpPredicate(Pred2) : Pred2, *C2);
   if (Offset2)
     CR2 = CR2.subtract(*Offset2);
 
@@ -3605,16 +3640,11 @@ static bool matchSubIntegerPackFromVector(Value *V, Value *&Vec,
                                           int64_t &VecOffset,
                                           SmallBitVector &Mask,
                                           const DataLayout &DL) {
-  static const auto m_ConstShlOrSelf = [](const auto &Base, uint64_t &ShlAmt) {
-    ShlAmt = 0;
-    return m_CombineOr(m_Shl(Base, m_ConstantInt(ShlAmt)), Base);
-  };
-
   // First try to match extractelement -> zext -> shl
   uint64_t VecIdx, ShlAmt;
-  if (match(V, m_ConstShlOrSelf(m_ZExtOrSelf(m_ExtractElt(
-                                    m_Value(Vec), m_ConstantInt(VecIdx))),
-                                ShlAmt))) {
+  if (match(V, m_ShlOrSelf(m_ZExtOrSelf(m_ExtractElt(m_Value(Vec),
+                                                     m_ConstantInt(VecIdx))),
+                           ShlAmt))) {
     auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
     if (!VecTy)
       return false;
@@ -3740,6 +3770,82 @@ static Instruction *foldIntegerPackFromVector(Instruction &I,
   return CastInst::Create(Instruction::BitCast, MaskedVec, I.getType());
 }
 
+/// Match \p V as "lshr -> mask -> zext -> shl".
+///
+/// \p Int is the underlying integer being extracted from.
+/// \p Mask is a bitmask identifying which bits of the integer are being
+/// extracted. \p Offset identifies which bit of the result \p V corresponds to
+/// the least significant bit of \p Int
+static bool matchZExtedSubInteger(Value *V, Value *&Int, APInt &Mask,
+                                  uint64_t &Offset, bool &IsShlNUW,
+                                  bool &IsShlNSW) {
+  Value *ShlOp0;
+  uint64_t ShlAmt = 0;
+  if (!match(V, m_OneUse(m_Shl(m_Value(ShlOp0), m_ConstantInt(ShlAmt)))))
+    return false;
+
+  IsShlNUW = cast<BinaryOperator>(V)->hasNoUnsignedWrap();
+  IsShlNSW = cast<BinaryOperator>(V)->hasNoSignedWrap();
+
+  Value *ZExtOp0;
+  if (!match(ShlOp0, m_OneUse(m_ZExt(m_Value(ZExtOp0)))))
+    return false;
+
+  Value *MaskedOp0;
+  const APInt *ShiftedMaskConst = nullptr;
+  if (!match(ZExtOp0, m_CombineOr(m_OneUse(m_And(m_Value(MaskedOp0),
+                                                 m_APInt(ShiftedMaskConst))),
+                                  m_Value(MaskedOp0))))
+    return false;
+
+  uint64_t LShrAmt = 0;
+  if (!match(MaskedOp0,
+             m_CombineOr(m_OneUse(m_LShr(m_Value(Int), m_ConstantInt(LShrAmt))),
+                         m_Value(Int))))
+    return false;
+
+  if (LShrAmt > ShlAmt)
+    return false;
+  Offset = ShlAmt - LShrAmt;
+
+  Mask = ShiftedMaskConst ? ShiftedMaskConst->shl(LShrAmt)
+                          : APInt::getBitsSetFrom(
+                                Int->getType()->getScalarSizeInBits(), LShrAmt);
+
+  return true;
+}
+
+/// Try to fold the join of two scalar integers whose bits are unpacked and
+/// zexted from the same source integer.
+static Value *foldIntegerRepackThroughZExt(Value *Lhs, Value *Rhs,
+                                           InstCombiner::BuilderTy &Builder) {
+
+  Value *LhsInt, *RhsInt;
+  APInt LhsMask, RhsMask;
+  uint64_t LhsOffset, RhsOffset;
+  bool IsLhsShlNUW, IsLhsShlNSW, IsRhsShlNUW, IsRhsShlNSW;
+  if (!matchZExtedSubInteger(Lhs, LhsInt, LhsMask, LhsOffset, IsLhsShlNUW,
+                             IsLhsShlNSW))
+    return nullptr;
+  if (!matchZExtedSubInteger(Rhs, RhsInt, RhsMask, RhsOffset, IsRhsShlNUW,
+                             IsRhsShlNSW))
+    return nullptr;
+  if (LhsInt != RhsInt || LhsOffset != RhsOffset)
+    return nullptr;
+
+  APInt Mask = LhsMask | RhsMask;
+
+  Type *DestTy = Lhs->getType();
+  Value *Res = Builder.CreateShl(
+      Builder.CreateZExt(
+          Builder.CreateAnd(LhsInt, Mask, LhsInt->getName() + ".mask"), DestTy,
+          LhsInt->getName() + ".zext"),
+      ConstantInt::get(DestTy, LhsOffset), "", IsLhsShlNUW && IsRhsShlNUW,
+      IsLhsShlNSW && IsRhsShlNSW);
+  Res->takeName(Lhs);
+  return Res;
+}
+
 // A decomposition of ((X & Mask) * Factor). The NUW / NSW bools
 // track these properities for preservation. Note that we can decompose
 // equivalent select form of this expression (e.g. (!(X & Mask) ? 0 : Mask *
@@ -3841,6 +3947,8 @@ static Value *foldBitmaskMul(Value *Op0, Value *Op1,
 Value *InstCombinerImpl::foldDisjointOr(Value *LHS, Value *RHS) {
   if (Value *Res = foldBitmaskMul(LHS, RHS, Builder))
     return Res;
+  if (Value *Res = foldIntegerRepackThroughZExt(LHS, RHS, Builder))
+    return Res;
 
   return nullptr;
 }
@@ -3973,7 +4081,7 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
                                    /*NSW=*/true, /*NUW=*/true))
       return R;
 
-    if (Value *Res = foldBitmaskMul(I.getOperand(0), I.getOperand(1), Builder))
+    if (Value *Res = foldDisjointOr(I.getOperand(0), I.getOperand(1)))
       return replaceInstUsesWith(I, Res);
 
     if (Value *Res = reassociateDisjointOr(I.getOperand(0), I.getOperand(1)))
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 47e017e17092b..42b65dde67255 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -133,7 +133,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
   // wouldn't be constant), and this must be a noop.
   if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) {
     // Set the size of the copy to 0, it will be deleted on the next iteration.
-    MI->setLength(Constant::getNullValue(MI->getLength()->getType()));
+    MI->setLength((uint64_t)0);
     return MI;
   }
 
@@ -141,7 +141,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
   // (unless the transfer is volatile).
   if (hasUndefSource(MI) && !MI->isVolatile()) {
     // Set the size of the copy to 0, it will be deleted on the next iteration.
-    MI->setLength(Constant::getNullValue(MI->getLength()->getType()));
+    MI->setLength((uint64_t)0);
     return MI;
   }
 
@@ -211,7 +211,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
   }
 
   // Set the size of the copy to 0, it will be deleted on the next iteration.
-  MI->setLength(Constant::getNullValue(MemOpLength->getType()));
+  MI->setLength((uint64_t)0);
   return MI;
 }
 
@@ -229,7 +229,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {
   // wouldn't be constant), and this must be a noop.
   if (!isModSet(AA->getModRefInfoMask(MI->getDest()))) {
     // Set the size of the copy to 0, it will be deleted on the next iteration.
-    MI->setLength(Constant::getNullValue(MI->getLength()->getType()));
+    MI->setLength((uint64_t)0);
     return MI;
   }
 
@@ -238,7 +238,7 @@ Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {
   // value. Change to PoisonValue once #52930 is resolved.
   if (isa<UndefValue>(MI->getValue())) {
     // Set the size of the copy to 0, it will be deleted on the next iteration.
-    MI->setLength(Constant::getNullValue(MI->getLength()->getType()));
+    MI->setLength((uint64_t)0);
     return MI;
   }
 
@@ -267,19 +267,17 @@ Instruction *InstCombinerImpl::SimplifyAnyMemSet(AnyMemSetInst *MI) {
         MI->getContext(), APInt::getSplat(Len * 8, FillC->getValue()));
     StoreInst *S = Builder.CreateStore(FillVal, Dest, MI->isVolatile());
     S->copyMetadata(*MI, LLVMContext::MD_DIAssignID);
-    auto replaceOpForAssignmentMarkers = [FillC, FillVal](auto *DbgAssign) {
+    for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(S)) {
       if (llvm::is_contained(DbgAssign->location_ops(), FillC))
         DbgAssign->replaceVariableLocationOp(FillC, FillVal);
-    };
-    for_each(at::getAssignmentMarkers(S), replaceOpForAssignmentMarkers);
-    for_each(at::getDVRAssignmentMarkers(S), replaceOpForAssignmentMarkers);
+    }
 
     S->setAlignment(Alignment);
     if (MI->isAtomic())
       S->setOrdering(AtomicOrdering::Unordered);
 
     // Set the size of the copy to 0, it will be deleted on the next iteration.
-    MI->setLength(Constant::getNullValue(LenC->getType()));
+    MI->setLength((uint64_t)0);
     return MI;
   }
 
@@ -1532,6 +1530,51 @@ static Instruction *foldBitOrderCrossLogicOp(Value *V,
   return nullptr;
 }
 
+/// Helper to match idempotent binary intrinsics, namely, intrinsics where
+/// `f(f(x, y), y) == f(x, y)` holds.
+static bool isIdempotentBinaryIntrinsic(Intrinsic::ID IID) {
+  switch (IID) {
+  case Intrinsic::smax:
+  case Intrinsic::smin:
+  case Intrinsic::umax:
+  case Intrinsic::umin:
+  case Intrinsic::maximum:
+  case Intrinsic::minimum:
+  case Intrinsic::maximumnum:
+  case Intrinsic::minimumnum:
+  case Intrinsic::maxnum:
+  case Intrinsic::minnum:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/// Attempt to simplify value-accumulating recurrences of kind:
+///   %umax.acc = phi i8 [ %umax, %backedge ], [ %a, %entry ]
+///   %umax = call i8 @llvm.umax.i8(i8 %umax.acc, i8 %b)
+/// And let the idempotent binary intrinsic be hoisted, when the operands are
+/// known to be loop-invariant.
+static Value *foldIdempotentBinaryIntrinsicRecurrence(InstCombinerImpl &IC,
+                                                      IntrinsicInst *II) {
+  PHINode *PN;
+  Value *Init, *OtherOp;
+
+  // A binary intrinsic recurrence with loop-invariant operands is equivalent to
+  // `call @llvm.binary.intrinsic(Init, OtherOp)`.
+  auto IID = II->getIntrinsicID();
+  if (!isIdempotentBinaryIntrinsic(IID) ||
+      !matchSimpleBinaryIntrinsicRecurrence(II, PN, Init, OtherOp) ||
+      !IC.getDominatorTree().dominates(OtherOp, PN))
+    return nullptr;
+
+  auto *InvariantBinaryInst =
+      IC.Builder.CreateBinaryIntrinsic(IID, Init, OtherOp);
+  if (isa<FPMathOperator>(InvariantBinaryInst))
+    cast<Instruction>(InvariantBinaryInst)->copyFastMathFlags(II);
+  return InvariantBinaryInst;
+}
+
 static Value *simplifyReductionOperand(Value *Arg, bool CanReorderLanes) {
   if (!CanReorderLanes)
     return nullptr;
@@ -1708,9 +1751,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   // Intrinsics cannot occur in an invoke or a callbr, so handle them here
   // instead of in visitCallBase.
   if (auto *MI = dyn_cast<AnyMemIntrinsic>(II)) {
-    if (ConstantInt *NumBytes = dyn_cast<ConstantInt>(MI->getLength())) {
+    if (auto NumBytes = MI->getLengthInBytes()) {
       // memmove/cpy/set of zero bytes is a noop.
-      if (NumBytes->isNullValue())
+      if (NumBytes->isZero())
         return eraseInstFromFunction(CI);
 
       // For atomic unordered mem intrinsics if len is not a positive or
@@ -3912,6 +3955,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   if (Value *Reverse = foldReversedIntrinsicOperands(II))
     return replaceInstUsesWith(*II, Reverse);
 
+  if (Value *Res = foldIdempotentBinaryIntrinsicRecurrence(*this, II))
+    return replaceInstUsesWith(*II, Res);
+
   // Some intrinsics (like experimental_gc_statepoint) can be used in invoke
   // context, so it is handled in visitCallBase and we should trigger it.
   return visitCallBase(*II);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index a43a6ee1f58b0..fdef49e310f81 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -1131,11 +1131,10 @@ static bool canEvaluateZExtd(Value *V, Type *Ty, unsigned &BitsToClear,
   case Instruction::Shl: {
     // We can promote shl(x, cst) if we can promote x.  Since shl overwrites the
     // upper bits we can reduce BitsToClear by the shift amount.
-    const APInt *Amt;
-    if (match(I->getOperand(1), m_APInt(Amt))) {
+    uint64_t ShiftAmt;
+    if (match(I->getOperand(1), m_ConstantInt(ShiftAmt))) {
       if (!canEvaluateZExtd(I->getOperand(0), Ty, BitsToClear, IC, CxtI))
         return false;
-      uint64_t ShiftAmt = Amt->getZExtValue();
       BitsToClear = ShiftAmt < BitsToClear ? BitsToClear - ShiftAmt : 0;
       return true;
     }
@@ -1144,11 +1143,11 @@ static bool canEvaluateZExtd(Value *V, Type *Ty, unsigned &BitsToClear,
   case Instruction::LShr: {
     // We can promote lshr(x, cst) if we can promote x.  This requires the
     // ultimate 'and' to clear out the high zero bits we're clearing out though.
-    const APInt *Amt;
-    if (match(I->getOperand(1), m_APInt(Amt))) {
+    uint64_t ShiftAmt;
+    if (match(I->getOperand(1), m_ConstantInt(ShiftAmt))) {
       if (!canEvaluateZExtd(I->getOperand(0), Ty, BitsToClear, IC, CxtI))
         return false;
-      BitsToClear += Amt->getZExtValue();
+      BitsToClear += ShiftAmt;
       if (BitsToClear > V->getType()->getScalarSizeInBits())
         BitsToClear = V->getType()->getScalarSizeInBits();
       return true;
@@ -2073,6 +2072,27 @@ Instruction *InstCombinerImpl::visitIntToPtr(IntToPtrInst &CI) {
     return new IntToPtrInst(P, CI.getType());
   }
 
+  // Replace (inttoptr (add (ptrtoint %Base), %Offset)) with
+  // (getelementptr i8, %Base, %Offset) if the pointer is only used as integer
+  // value.
+  Value *Base;
+  Value *Offset;
+  auto UsesPointerAsInt = [](User *U) {
+    if (isa<ICmpInst, PtrToIntInst>(U))
+      return true;
+    if (auto *P = dyn_cast<PHINode>(U))
+      return P->hasOneUse() && isa<ICmpInst, PtrToIntInst>(*P->user_begin());
+    return false;
+  };
+  if (match(CI.getOperand(0),
+            m_OneUse(m_c_Add(m_PtrToIntSameSize(DL, m_Value(Base)),
+                             m_Value(Offset)))) &&
+      CI.getType()->getPointerAddressSpace() ==
+          Base->getType()->getPointerAddressSpace() &&
+      all_of(CI.users(), UsesPointerAsInt)) {
+    return GetElementPtrInst::Create(Builder.getInt8Ty(), Base, Offset);
+  }
+
   if (Instruction *I = commonCastTransforms(CI))
     return I;
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index cf94d28100488..3a8e043038153 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1320,6 +1320,35 @@ Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) {
   return nullptr;
 }
 
+/// Fold icmp eq (num + mask) & ~mask, num
+///      to
+///      icmp eq (and num, mask), 0
+/// Where mask is a low bit mask.
+Instruction *InstCombinerImpl::foldIsMultipleOfAPowerOfTwo(ICmpInst &Cmp) {
+  Value *Num;
+  CmpPredicate Pred;
+  const APInt *Mask, *Neg;
+
+  if (!match(&Cmp,
+             m_c_ICmp(Pred, m_Value(Num),
+                      m_OneUse(m_c_And(m_OneUse(m_c_Add(m_Deferred(Num),
+                                                        m_LowBitMask(Mask))),
+                                       m_APInt(Neg))))))
+    return nullptr;
+
+  if (*Neg != ~*Mask)
+    return nullptr;
+
+  if (!ICmpInst::isEquality(Pred))
+    return nullptr;
+
+  // Create new icmp eq (num & mask), 0
+  auto *NewAnd = Builder.CreateAnd(Num, *Mask);
+  auto *Zero = Constant::getNullValue(Num->getType());
+
+  return new ICmpInst(Pred, NewAnd, Zero);
+}
+
 /// Fold icmp Pred X, C.
 /// TODO: This code structure does not make sense. The saturating add fold
 /// should be moved to some other helper and extended as noted below (it is also
@@ -1521,11 +1550,11 @@ Instruction *InstCombinerImpl::foldICmpTruncConstant(ICmpInst &Cmp,
   // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] < 0  --> ShOp <  0
   // trunc iN (ShOp >> ShAmtC) to i[N - ShAmtC] > -1 --> ShOp > -1
   Value *ShOp;
-  const APInt *ShAmtC;
+  uint64_t ShAmt;
   bool TrueIfSigned;
   if (isSignBitCheck(Pred, C, TrueIfSigned) &&
-      match(X, m_Shr(m_Value(ShOp), m_APInt(ShAmtC))) &&
-      DstBits == SrcBits - ShAmtC->getZExtValue()) {
+      match(X, m_Shr(m_Value(ShOp), m_ConstantInt(ShAmt))) &&
+      DstBits == SrcBits - ShAmt) {
     return TrueIfSigned ? new ICmpInst(ICmpInst::ICMP_SLT, ShOp,
                                        ConstantInt::getNullValue(SrcTy))
                         : new ICmpInst(ICmpInst::ICMP_SGT, ShOp,
@@ -3291,20 +3320,18 @@ Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp,
     Type *NewCmpTy = V->getType();
     unsigned NewCmpBW = NewCmpTy->getScalarSizeInBits();
     if (shouldChangeType(Ty, NewCmpTy)) {
-      if (CR.getActiveBits() <= NewCmpBW) {
-        ConstantRange SrcCR = CR.truncate(NewCmpBW);
-        CmpInst::Predicate EquivPred;
-        APInt EquivInt;
-        APInt EquivOffset;
-
-        SrcCR.getEquivalentICmp(EquivPred, EquivInt, EquivOffset);
-        return new ICmpInst(
-            EquivPred,
-            EquivOffset.isZero()
-                ? V
-                : Builder.CreateAdd(V, ConstantInt::get(NewCmpTy, EquivOffset)),
-            ConstantInt::get(NewCmpTy, EquivInt));
-      }
+      ConstantRange SrcCR = CR.truncate(NewCmpBW, TruncInst::NoUnsignedWrap);
+      CmpInst::Predicate EquivPred;
+      APInt EquivInt;
+      APInt EquivOffset;
+
+      SrcCR.getEquivalentICmp(EquivPred, EquivInt, EquivOffset);
+      return new ICmpInst(
+          EquivPred,
+          EquivOffset.isZero()
+              ? V
+              : Builder.CreateAdd(V, ConstantInt::get(NewCmpTy, EquivOffset)),
+          ConstantInt::get(NewCmpTy, EquivInt));
     }
   }
 
@@ -7644,6 +7671,9 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) {
   if (Instruction *Res = foldICmpUsingKnownBits(I))
     return Res;
 
+  if (Instruction *Res = foldIsMultipleOfAPowerOfTwo(I))
+    return Res;
+
   // Test if the ICmpInst instruction is used exclusively by a select as
   // part of a minimum or maximum operation. If so, refrain from doing
   // any other folding. This helps out other analyses which understand
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index c67e27e5b3e7c..2340028ce93dc 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -721,6 +721,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
   Instruction *foldICmpUsingKnownBits(ICmpInst &Cmp);
   Instruction *foldICmpWithDominatingICmp(ICmpInst &Cmp);
   Instruction *foldICmpWithConstant(ICmpInst &Cmp);
+  Instruction *foldIsMultipleOfAPowerOfTwo(ICmpInst &Cmp);
   Instruction *foldICmpUsingBoolRange(ICmpInst &I);
   Instruction *foldICmpInstWithConstant(ICmpInst &Cmp);
   Instruction *foldICmpInstWithConstantNotInt(ICmpInst &Cmp);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 0be1034b046b6..4b10586616c29 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -737,6 +737,8 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
       LoadInst *NewLoad = IC.combineLoadToNewType(LI, ST->getTypeAtIndex(0U),
                                                   ".unpack");
       NewLoad->setAAMetadata(LI.getAAMetadata());
+      // Copy invariant metadata from parent load.
+      NewLoad->copyMetadata(LI, LLVMContext::MD_invariant_load);
       return IC.replaceInstUsesWith(LI, IC.Builder.CreateInsertValue(
         PoisonValue::get(T), NewLoad, 0, Name));
     }
@@ -764,6 +766,8 @@ static Instruction *unpackLoadToAggregate(InstCombinerImpl &IC, LoadInst &LI) {
           Name + ".unpack");
       // Propagate AA metadata. It'll still be valid on the narrowed load.
       L->setAAMetadata(LI.getAAMetadata());
+      // Copy invariant metadata from parent load.
+      L->copyMetadata(LI, LLVMContext::MD_invariant_load);
       V = IC.Builder.CreateInsertValue(V, L, i);
     }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index fe0f308223387..b17cf17db1580 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -3042,7 +3042,7 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
     Value *V = LHS;
     unsigned MaskElems = Mask.size();
     auto *SrcTy = cast<FixedVectorType>(V->getType());
-    unsigned VecBitWidth = SrcTy->getPrimitiveSizeInBits().getFixedValue();
+    unsigned VecBitWidth = DL.getTypeSizeInBits(SrcTy);
     unsigned SrcElemBitWidth = DL.getTypeSizeInBits(SrcTy->getElementType());
     assert(SrcElemBitWidth && "vector elements must have a bitwidth");
     unsigned SrcNumElems = SrcTy->getNumElements();
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 5ee3bb1abe86e..1c512ec1e21bb 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -4961,63 +4961,68 @@ Instruction *InstCombinerImpl::visitLandingPadInst(LandingPadInst &LI) {
 Value *
 InstCombinerImpl::pushFreezeToPreventPoisonFromPropagating(FreezeInst &OrigFI) {
   // Try to push freeze through instructions that propagate but don't produce
-  // poison as far as possible.  If an operand of freeze follows three
-  // conditions 1) one-use, 2) does not produce poison, and 3) has all but one
-  // guaranteed-non-poison operands then push the freeze through to the one
-  // operand that is not guaranteed non-poison.  The actual transform is as
-  // follows.
-  //   Op1 = ...                        ; Op1 can be posion
-  //   Op0 = Inst(Op1, NonPoisonOps...) ; Op0 has only one use and only have
-  //                                    ; single guaranteed-non-poison operands
+  // poison as far as possible. If an operand of freeze does not produce poison
+  // then push the freeze through to the operands that are not guaranteed
+  // non-poison. The actual transform is as follows.
+  //   Op1 = ...                        ; Op1 can be poison
+  //   Op0 = Inst(Op1, NonPoisonOps...)
   //   ... = Freeze(Op0)
   // =>
   //   Op1 = ...
   //   Op1.fr = Freeze(Op1)
   //   ... = Inst(Op1.fr, NonPoisonOps...)
-  auto *OrigOp = OrigFI.getOperand(0);
-  auto *OrigOpInst = dyn_cast<Instruction>(OrigOp);
 
-  // While we could change the other users of OrigOp to use freeze(OrigOp), that
-  // potentially reduces their optimization potential, so let's only do this iff
-  // the OrigOp is only used by the freeze.
-  if (!OrigOpInst || !OrigOpInst->hasOneUse() || isa<PHINode>(OrigOp))
-    return nullptr;
+  auto CanPushFreeze = [](Value *V) {
+    if (!isa<Instruction>(V) || isa<PHINode>(V))
+      return false;
 
-  // We can't push the freeze through an instruction which can itself create
-  // poison.  If the only source of new poison is flags, we can simply
-  // strip them (since we know the only use is the freeze and nothing can
-  // benefit from them.)
-  if (canCreateUndefOrPoison(cast<Operator>(OrigOp),
-                             /*ConsiderFlagsAndMetadata*/ false))
-    return nullptr;
+    // We can't push the freeze through an instruction which can itself create
+    // poison.  If the only source of new poison is flags, we can simply
+    // strip them (since we know the only use is the freeze and nothing can
+    // benefit from them.)
+    return !canCreateUndefOrPoison(cast<Operator>(V),
+                                   /*ConsiderFlagsAndMetadata*/ false);
+  };
 
-  // If operand is guaranteed not to be poison, there is no need to add freeze
-  // to the operand. So we first find the operand that is not guaranteed to be
-  // poison.
-  Value *MaybePoisonOperand = nullptr;
-  for (Value *V : OrigOpInst->operands()) {
-    if (isa<MetadataAsValue>(V) || isGuaranteedNotToBeUndefOrPoison(V) ||
-        // Treat identical operands as a single operand.
-        (MaybePoisonOperand && MaybePoisonOperand == V))
+  // Pushing freezes up long instruction chains can be expensive. Instead,
+  // we directly push the freeze all the way to the leaves. However, we leave
+  // deduplication of freezes on the same value for freezeOtherUses().
+  Use *OrigUse = &OrigFI.getOperandUse(0);
+  SmallPtrSet<Instruction *, 8> Visited;
+  SmallVector<Use *, 8> Worklist;
+  Worklist.push_back(OrigUse);
+  while (!Worklist.empty()) {
+    auto *U = Worklist.pop_back_val();
+    Value *V = U->get();
+    if (!CanPushFreeze(V)) {
+      // If we can't push through the original instruction, abort the transform.
+      if (U == OrigUse)
+        return nullptr;
+
+      auto *UserI = cast<Instruction>(U->getUser());
+      Builder.SetInsertPoint(UserI);
+      Value *Frozen = Builder.CreateFreeze(V, V->getName() + ".fr");
+      U->set(Frozen);
       continue;
-    if (!MaybePoisonOperand)
-      MaybePoisonOperand = V;
-    else
-      return nullptr;
-  }
+    }
 
-  OrigOpInst->dropPoisonGeneratingAnnotations();
+    auto *I = cast<Instruction>(V);
+    if (!Visited.insert(I).second)
+      continue;
 
-  // If all operands are guaranteed to be non-poison, we can drop freeze.
-  if (!MaybePoisonOperand)
-    return OrigOp;
+    // reverse() to emit freezes in a more natural order.
+    for (Use &Op : reverse(I->operands())) {
+      Value *OpV = Op.get();
+      if (isa<MetadataAsValue>(OpV) || isGuaranteedNotToBeUndefOrPoison(OpV))
+        continue;
+      Worklist.push_back(&Op);
+    }
 
-  Builder.SetInsertPoint(OrigOpInst);
-  Value *FrozenMaybePoisonOperand = Builder.CreateFreeze(
-      MaybePoisonOperand, MaybePoisonOperand->getName() + ".fr");
+    I->dropPoisonGeneratingAnnotations();
+    this->Worklist.add(I);
+  }
 
-  OrigOpInst->replaceUsesOfWith(MaybePoisonOperand, FrozenMaybePoisonOperand);
-  return OrigOp;
+  return OrigUse->get();
 }
 
 Instruction *InstCombinerImpl::foldFreezeIntoRecurrence(FreezeInst &FI,
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 8da65c597116f..50258af5e26c3 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1211,23 +1211,19 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
       return;
     if (!II.isLifetimeStartOrEnd())
       return;
-    // Found lifetime intrinsic, add ASan instrumentation if necessary.
-    auto *Size = cast<ConstantInt>(II.getArgOperand(0));
-    // If size argument is undefined, don't do anything.
-    if (Size->isMinusOne()) return;
-    // Check that size doesn't saturate uint64_t and can
-    // be stored in IntptrTy.
-    const uint64_t SizeValue = Size->getValue().getLimitedValue();
-    if (SizeValue == ~0ULL ||
-        !ConstantInt::isValueValidForType(IntptrTy, SizeValue))
-      return;
     // Find alloca instruction that corresponds to llvm.lifetime argument.
-    AllocaInst *AI = dyn_cast<AllocaInst>(II.getArgOperand(1));
+    AllocaInst *AI = dyn_cast<AllocaInst>(II.getArgOperand(0));
     // We're interested only in allocas we can handle.
     if (!AI || !ASan.isInterestingAlloca(*AI))
       return;
+
+    std::optional<TypeSize> Size = AI->getAllocationSize(AI->getDataLayout());
+    // Check that size is known and can be stored in IntptrTy.
+    if (!Size || !ConstantInt::isValueValidForType(IntptrTy, *Size))
+      return;
+
     bool DoPoison = (ID == Intrinsic::lifetime_end);
-    AllocaPoisonCall APC = {&II, AI, SizeValue, DoPoison};
+    AllocaPoisonCall APC = {&II, AI, *Size, DoPoison};
     if (AI->isStaticAlloca())
       StaticAllocaPoisonCallVec.push_back(APC);
     else if (ClInstrumentDynamicAllocas)
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index bcb90d6523c72..66cdbfcf998c6 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -1469,22 +1469,6 @@ void HWAddressSanitizer::instrumentStack(memtag::StackInfo &SInfo,
     size_t Size = memtag::getAllocaSizeInBytes(*AI);
     size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
 
-    auto HandleLifetime = [&](IntrinsicInst *II) {
-      // Set the lifetime intrinsic to cover the whole alloca. This reduces the
-      // set of assumptions we need to make about the lifetime. Without this we
-      // would need to ensure that we can track the lifetime pointer to a
-      // constant offset from the alloca, and would still need to change the
-      // size to include the extra alignment we use for the untagging to make
-      // the size consistent.
-      //
-      // The check for standard lifetime below makes sure that we have exactly
-      // one set of start / end in any execution (i.e. the ends are not
-      // reachable from each other), so this will not cause any problems.
-      II->setArgOperand(0, ConstantInt::get(Int64Ty, AlignedSize));
-    };
-    llvm::for_each(Info.LifetimeStart, HandleLifetime);
-    llvm::for_each(Info.LifetimeEnd, HandleLifetime);
-
     AI->replaceUsesWithIf(Replacement, [AILong](const Use &U) {
       auto *User = U.getUser();
       return User != AILong && !isa<LifetimeIntrinsic>(User);
@@ -1590,6 +1574,9 @@ void HWAddressSanitizer::sanitizeFunction(Function &F,
   if (F.empty())
     return;
 
+  if (F.isPresplitCoroutine())
+    return;
+
   NumTotalFuncs++;
 
   OptimizationRemarkEmitter &ORE =
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 7d3c940c0065f..27292d1a66c30 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -2690,6 +2690,54 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     SC.Done(&I);
   }
 
+  // Perform a bitwise OR on the horizontal pairs (or other specified grouping)
+  // of elements.
+  //
+  // For example, suppose we have:
+  //   VectorA: <a1, a2, a3, a4, a5, a6>
+  //   VectorB: <b1, b2, b3, b4, b5, b6>
+  //   ReductionFactor: 3.
+  // The output would be:
+  //   <a1|a2|a3, a4|a5|a6, b1|b2|b3, b4|b5|b6>
+  //
+  // This is convenient for instrumenting horizontal add/sub.
+  // For bitwise OR on "vertical" pairs, see maybeHandleSimpleNomemIntrinsic().
+  Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor,
+                          Value *VectorA, Value *VectorB) {
+    assert(isa<FixedVectorType>(VectorA->getType()));
+    unsigned TotalNumElems =
+        cast<FixedVectorType>(VectorA->getType())->getNumElements();
+
+    if (VectorB) {
+      assert(VectorA->getType() == VectorB->getType());
+      TotalNumElems = TotalNumElems * 2;
+    }
+
+    assert(TotalNumElems % ReductionFactor == 0);
+
+    Value *Or = nullptr;
+
+    IRBuilder<> IRB(&I);
+    for (unsigned i = 0; i < ReductionFactor; i++) {
+      SmallVector<int, 16> Mask;
+      for (unsigned X = 0; X < TotalNumElems; X += ReductionFactor)
+        Mask.push_back(X + i);
+
+      Value *Masked;
+      if (VectorB)
+        Masked = IRB.CreateShuffleVector(VectorA, VectorB, Mask);
+      else
+        Masked = IRB.CreateShuffleVector(VectorA, Mask);
+
+      if (Or)
+        Or = IRB.CreateOr(Or, Masked);
+      else
+        Or = Masked;
+    }
+
+    return Or;
+  }
+
   /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent
   /// fields.
   ///
@@ -2701,7 +2749,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     assert(I.getType()->isVectorTy());
     assert(I.getArgOperand(0)->getType()->isVectorTy());
 
-    FixedVectorType *ParamType =
+    [[maybe_unused]] FixedVectorType *ParamType =
         cast<FixedVectorType>(I.getArgOperand(0)->getType());
     assert((I.arg_size() != 2) ||
            (ParamType == cast<FixedVectorType>(I.getArgOperand(1)->getType())));
@@ -2711,31 +2759,16 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
            2 * ReturnType->getNumElements());
 
     IRBuilder<> IRB(&I);
-    unsigned Width = ParamType->getNumElements() * I.arg_size();
 
     // Horizontal OR of shadow
-    SmallVector<int, 8> EvenMask;
-    SmallVector<int, 8> OddMask;
-    for (unsigned X = 0; X < Width; X += 2) {
-      EvenMask.push_back(X);
-      OddMask.push_back(X + 1);
-    }
-
     Value *FirstArgShadow = getShadow(&I, 0);
-    Value *EvenShadow;
-    Value *OddShadow;
-    if (I.arg_size() == 2) {
-      Value *SecondArgShadow = getShadow(&I, 1);
-      EvenShadow =
-          IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, EvenMask);
-      OddShadow =
-          IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, OddMask);
-    } else {
-      EvenShadow = IRB.CreateShuffleVector(FirstArgShadow, EvenMask);
-      OddShadow = IRB.CreateShuffleVector(FirstArgShadow, OddMask);
-    }
+    Value *SecondArgShadow = nullptr;
+    if (I.arg_size() == 2)
+      SecondArgShadow = getShadow(&I, 1);
+
+    Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, FirstArgShadow,
+                                       SecondArgShadow);
 
-    Value *OrShadow = IRB.CreateOr(EvenShadow, OddShadow);
     OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I));
 
     setShadow(&I, OrShadow);
@@ -2768,23 +2801,14 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
     IRBuilder<> IRB(&I);
 
-    unsigned TotalNumElems = ParamType->getNumElements() * I.arg_size();
     FixedVectorType *ReinterpretShadowTy = nullptr;
     assert(isAligned(Align(ReinterpretElemWidth),
                      ParamType->getPrimitiveSizeInBits()));
     ReinterpretShadowTy = FixedVectorType::get(
         IRB.getIntNTy(ReinterpretElemWidth),
         ParamType->getPrimitiveSizeInBits() / ReinterpretElemWidth);
-    TotalNumElems = ReinterpretShadowTy->getNumElements() * I.arg_size();
 
     // Horizontal OR of shadow
-    SmallVector<int, 8> EvenMask;
-    SmallVector<int, 8> OddMask;
-    for (unsigned X = 0; X < TotalNumElems - 1; X += 2) {
-      EvenMask.push_back(X);
-      OddMask.push_back(X + 1);
-    }
-
     Value *FirstArgShadow = getShadow(&I, 0);
     FirstArgShadow = IRB.CreateBitCast(FirstArgShadow, ReinterpretShadowTy);
 
@@ -2796,22 +2820,15 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         Align(2),
         cast<FixedVectorType>(FirstArgShadow->getType())->getNumElements()));
 
-    Value *EvenShadow;
-    Value *OddShadow;
+    Value *SecondArgShadow = nullptr;
     if (I.arg_size() == 2) {
-      Value *SecondArgShadow = getShadow(&I, 1);
+      SecondArgShadow = getShadow(&I, 1);
       SecondArgShadow = IRB.CreateBitCast(SecondArgShadow, ReinterpretShadowTy);
-
-      EvenShadow =
-          IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, EvenMask);
-      OddShadow =
-          IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, OddMask);
-    } else {
-      EvenShadow = IRB.CreateShuffleVector(FirstArgShadow, EvenMask);
-      OddShadow = IRB.CreateShuffleVector(FirstArgShadow, OddMask);
     }
 
-    Value *OrShadow = IRB.CreateOr(EvenShadow, OddShadow);
+    Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, FirstArgShadow,
+                                       SecondArgShadow);
+
     OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I));
 
     setShadow(&I, OrShadow);
@@ -3219,7 +3236,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   /// Caller guarantees that this intrinsic does not access memory.
   ///
   /// TODO: "horizontal"/"pairwise" intrinsics are often incorrectly matched by
-  ///       by this handler.
+  ///       by this handler. See horizontalReduce().
+  ///
+  /// TODO: permutation intrinsics are also often incorrectly matched.
   [[maybe_unused]] bool
   maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I,
                                   unsigned int trailingFlags) {
@@ -3301,7 +3320,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   void handleLifetimeStart(IntrinsicInst &I) {
     if (!PoisonStack)
       return;
-    AllocaInst *AI = dyn_cast<AllocaInst>(I.getArgOperand(1));
+    AllocaInst *AI = dyn_cast<AllocaInst>(I.getArgOperand(0));
     if (AI)
       LifetimeStartList.push_back(std::make_pair(&I, AI));
   }
@@ -3410,8 +3429,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     return ShadowType;
   }
 
-  /// Doubles the length of a vector shadow (filled with zeros) if necessary to
-  /// match the length of the shadow for the instruction.
+  /// Doubles the length of a vector shadow (extending with zeros) if necessary
+  /// to match the length of the shadow for the instruction.
+  /// If scalar types of the vectors are different, it will use the type of the
+  /// input vector.
   /// This is more type-safe than CreateShadowCast().
   Value *maybeExtendVectorShadowWithZeros(Value *Shadow, IntrinsicInst &I) {
     IRBuilder<> IRB(&I);
@@ -3419,17 +3440,19 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     assert(isa<FixedVectorType>(I.getType()));
 
     Value *FullShadow = getCleanShadow(&I);
-    assert(cast<FixedVectorType>(Shadow->getType())->getNumElements() <=
-           cast<FixedVectorType>(FullShadow->getType())->getNumElements());
-    assert(cast<FixedVectorType>(Shadow->getType())->getScalarType() ==
-           cast<FixedVectorType>(FullShadow->getType())->getScalarType());
+    unsigned ShadowNumElems =
+        cast<FixedVectorType>(Shadow->getType())->getNumElements();
+    unsigned FullShadowNumElems =
+        cast<FixedVectorType>(FullShadow->getType())->getNumElements();
+
+    assert((ShadowNumElems == FullShadowNumElems) ||
+           (ShadowNumElems * 2 == FullShadowNumElems));
 
-    if (Shadow->getType() == FullShadow->getType()) {
+    if (ShadowNumElems == FullShadowNumElems) {
       FullShadow = Shadow;
     } else {
       // TODO: generalize beyond 2x?
-      SmallVector<int, 32> ShadowMask(
-          cast<FixedVectorType>(FullShadow->getType())->getNumElements());
+      SmallVector<int, 32> ShadowMask(FullShadowNumElems);
       std::iota(ShadowMask.begin(), ShadowMask.end(), 0);
 
       // Append zeros
@@ -3624,9 +3647,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOriginForNaryOp(I);
   }
 
-  // Get an MMX-sized vector type.
-  Type *getMMXVectorTy(unsigned EltSizeInBits) {
-    const unsigned X86_MMXSizeInBits = 64;
+  // Get an MMX-sized (64-bit) vector type, or optionally, other sized
+  // vectors.
+  Type *getMMXVectorTy(unsigned EltSizeInBits,
+                       unsigned X86_MMXSizeInBits = 64) {
     assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
            "Illegal MMX vector element size");
     return FixedVectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
@@ -3826,20 +3850,133 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     setOriginForNaryOp(I);
   }
 
-  // Instrument multiply-add intrinsic.
-  void handleVectorPmaddIntrinsic(IntrinsicInst &I,
-                                  unsigned MMXEltSizeInBits = 0) {
-    Type *ResTy =
-        MMXEltSizeInBits ? getMMXVectorTy(MMXEltSizeInBits * 2) : I.getType();
+  // Instrument multiply-add(-accumulate)? intrinsics.
+  //
+  // e.g., Two operands:
+  //         <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a, <8 x i16> %b)
+  //
+  //       Two operands which require an EltSizeInBits override:
+  //         <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> %a, <1 x i64> %b)
+  //
+  //       Three operands:
+  //         <4 x i32> @llvm.x86.avx512.vpdpbusd.128
+  //                       (<4 x i32> %s, <4 x i32> %a, <4 x i32> %b)
+  //         (this is equivalent to multiply-add on %a and %b, followed by
+  //          adding/"accumulating" %s. "Accumulation" stores the result in one
+  //          of the source registers, but this accumulate vs. add distinction
+  //          is lost when dealing with LLVM intrinsics.)
+  void handleVectorPmaddIntrinsic(IntrinsicInst &I, unsigned ReductionFactor,
+                                  unsigned EltSizeInBits = 0) {
     IRBuilder<> IRB(&I);
-    auto *Shadow0 = getShadow(&I, 0);
-    auto *Shadow1 = getShadow(&I, 1);
-    Value *S = IRB.CreateOr(Shadow0, Shadow1);
-    S = IRB.CreateBitCast(S, ResTy);
-    S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
-                       ResTy);
-    S = IRB.CreateBitCast(S, getShadowTy(&I));
-    setShadow(&I, S);
+
+    [[maybe_unused]] FixedVectorType *ReturnType =
+        cast<FixedVectorType>(I.getType());
+    assert(isa<FixedVectorType>(ReturnType));
+
+    // Vectors A and B, and shadows
+    Value *Va = nullptr;
+    Value *Vb = nullptr;
+    Value *Sa = nullptr;
+    Value *Sb = nullptr;
+
+    assert(I.arg_size() == 2 || I.arg_size() == 3);
+    if (I.arg_size() == 2) {
+      Va = I.getOperand(0);
+      Vb = I.getOperand(1);
+
+      Sa = getShadow(&I, 0);
+      Sb = getShadow(&I, 1);
+    } else if (I.arg_size() == 3) {
+      // Operand 0 is the accumulator. We will deal with that below.
+      Va = I.getOperand(1);
+      Vb = I.getOperand(2);
+
+      Sa = getShadow(&I, 1);
+      Sb = getShadow(&I, 2);
+    }
+
+    FixedVectorType *ParamType = cast<FixedVectorType>(Va->getType());
+    assert(ParamType == Vb->getType());
+
+    assert(ParamType->getPrimitiveSizeInBits() ==
+           ReturnType->getPrimitiveSizeInBits());
+
+    if (I.arg_size() == 3) {
+      assert(ParamType == ReturnType);
+      assert(ParamType == I.getArgOperand(0)->getType());
+    }
+
+    FixedVectorType *ImplicitReturnType = ReturnType;
+    // Step 1: instrument multiplication of corresponding vector elements
+    if (EltSizeInBits) {
+      ImplicitReturnType = cast<FixedVectorType>(getMMXVectorTy(
+          EltSizeInBits * 2, ParamType->getPrimitiveSizeInBits()));
+      ParamType = cast<FixedVectorType>(
+          getMMXVectorTy(EltSizeInBits, ParamType->getPrimitiveSizeInBits()));
+
+      Va = IRB.CreateBitCast(Va, ParamType);
+      Vb = IRB.CreateBitCast(Vb, ParamType);
+
+      Sa = IRB.CreateBitCast(Sa, getShadowTy(ParamType));
+      Sb = IRB.CreateBitCast(Sb, getShadowTy(ParamType));
+    } else {
+      assert(ParamType->getNumElements() ==
+             ReturnType->getNumElements() * ReductionFactor);
+    }
+
+    // Multiplying an *initialized* zero by an uninitialized element results in
+    // an initialized zero element.
+    //
+    // This is analogous to bitwise AND, where "AND" of 0 and a poisoned value
+    // results in an unpoisoned value. We can therefore adapt the visitAnd()
+    // instrumentation:
+    //   OutShadow =   (SaNonZero & SbNonZero)
+    //               | (VaNonZero & SbNonZero)
+    //               | (SaNonZero & VbNonZero)
+    //   where non-zero is checked on a per-element basis (not per bit).
+    Value *SZero = Constant::getNullValue(Va->getType());
+    Value *VZero = Constant::getNullValue(Sa->getType());
+    Value *SaNonZero = IRB.CreateICmpNE(Sa, SZero);
+    Value *SbNonZero = IRB.CreateICmpNE(Sb, SZero);
+    Value *VaNonZero = IRB.CreateICmpNE(Va, VZero);
+    Value *VbNonZero = IRB.CreateICmpNE(Vb, VZero);
+
+    Value *SaAndSbNonZero = IRB.CreateAnd(SaNonZero, SbNonZero);
+    Value *VaAndSbNonZero = IRB.CreateAnd(VaNonZero, SbNonZero);
+    Value *SaAndVbNonZero = IRB.CreateAnd(SaNonZero, VbNonZero);
+
+    // Each element of the vector is represented by a single bit (poisoned or
+    // not) e.g., <8 x i1>.
+    Value *And = IRB.CreateOr({SaAndSbNonZero, VaAndSbNonZero, SaAndVbNonZero});
+
+    // Extend <8 x i1> to <8 x i16>.
+    // (The real pmadd intrinsic would have computed intermediate values of
+    // <8 x i32>, but that is irrelevant for our shadow purposes because we
+    // consider each element to be either fully initialized or fully
+    // uninitialized.)
+    And = IRB.CreateSExt(And, Sa->getType());
+
+    // Step 2: instrument horizontal add
+    // We don't need bit-precise horizontalReduce because we only want to check
+    // if each pair of elements is fully zero.
+    // Cast to <4 x i32>.
+    Value *Horizontal = IRB.CreateBitCast(And, ImplicitReturnType);
+
+    // Compute <4 x i1>, then extend back to <4 x i32>.
+    Value *OutShadow = IRB.CreateSExt(
+        IRB.CreateICmpNE(Horizontal,
+                         Constant::getNullValue(Horizontal->getType())),
+        ImplicitReturnType);
+
+    // Cast it back to the required fake return type (<1 x i64>).
+    if (EltSizeInBits)
+      OutShadow = CreateShadowCast(IRB, OutShadow, getShadowTy(&I));
+
+    // Step 3 (if applicable): instrument accumulator
+    if (I.arg_size() == 3)
+      OutShadow = IRB.CreateOr(OutShadow, getShadow(&I, 0));
+
+    setShadow(&I, OutShadow);
     setOriginForNaryOp(I);
   }
 
@@ -4395,58 +4532,102 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     return isFixedFPVectorTy(V->getType());
   }
 
-  // e.g., call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
-  //                           (<16 x float> a, <16 x i32> writethru, i16 mask,
-  //                           i32 rounding)
+  // e.g., <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
+  //                      (<16 x float> a, <16 x i32> writethru, i16 mask,
+  //                       i32 rounding)
+  //
+  // Inconveniently, some similar intrinsics have a different operand order:
+  //       <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
+  //                      (<16 x float> a, i32 rounding, <16 x i16> writethru,
+  //                       i16 mask)
+  //
+  // If the return type has more elements than A, the excess elements are
+  // zeroed (and the corresponding shadow is initialized).
+  //       <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
+  //                      (<4 x float> a, i32 rounding, <8 x i16> writethru,
+  //                       i8 mask)
   //
   // dst[i] = mask[i] ? convert(a[i]) : writethru[i]
   // dst_shadow[i] = mask[i] ? all_or_nothing(a_shadow[i]) : writethru_shadow[i]
   //    where all_or_nothing(x) is fully uninitialized if x has any
   //    uninitialized bits
-  void handleAVX512VectorConvertFPToInt(IntrinsicInst &I) {
+  void handleAVX512VectorConvertFPToInt(IntrinsicInst &I, bool LastMask) {
     IRBuilder<> IRB(&I);
 
     assert(I.arg_size() == 4);
     Value *A = I.getOperand(0);
-    Value *WriteThrough = I.getOperand(1);
-    Value *Mask = I.getOperand(2);
-    Value *RoundingMode = I.getOperand(3);
+    Value *WriteThrough;
+    Value *Mask;
+    Value *RoundingMode;
+    if (LastMask) {
+      WriteThrough = I.getOperand(2);
+      Mask = I.getOperand(3);
+      RoundingMode = I.getOperand(1);
+    } else {
+      WriteThrough = I.getOperand(1);
+      Mask = I.getOperand(2);
+      RoundingMode = I.getOperand(3);
+    }
 
     assert(isFixedFPVector(A));
     assert(isFixedIntVector(WriteThrough));
 
     unsigned ANumElements =
         cast<FixedVectorType>(A->getType())->getNumElements();
-    assert(ANumElements ==
-           cast<FixedVectorType>(WriteThrough->getType())->getNumElements());
+    [[maybe_unused]] unsigned WriteThruNumElements =
+        cast<FixedVectorType>(WriteThrough->getType())->getNumElements();
+    assert(ANumElements == WriteThruNumElements ||
+           ANumElements * 2 == WriteThruNumElements);
 
     assert(Mask->getType()->isIntegerTy());
-    assert(Mask->getType()->getScalarSizeInBits() == ANumElements);
+    unsigned MaskNumElements = Mask->getType()->getScalarSizeInBits();
+    assert(ANumElements == MaskNumElements ||
+           ANumElements * 2 == MaskNumElements);
+
+    assert(WriteThruNumElements == MaskNumElements);
+
+    // Some bits of the mask may be unused, though it's unusual to have partly
+    // uninitialized bits.
     insertCheckShadowOf(Mask, &I);
 
     assert(RoundingMode->getType()->isIntegerTy());
-    // Only four bits of the rounding mode are used, though it's very
+    // Only some bits of the rounding mode are used, though it's very
     // unusual to have uninitialized bits there (more commonly, it's a
     // constant).
     insertCheckShadowOf(RoundingMode, &I);
 
     assert(I.getType() == WriteThrough->getType());
 
+    Value *AShadow = getShadow(A);
+    AShadow = maybeExtendVectorShadowWithZeros(AShadow, I);
+
+    if (ANumElements * 2 == MaskNumElements) {
+      // Ensure that the irrelevant bits of the mask are zero, hence selecting
+      // from the zeroed shadow instead of the writethrough's shadow.
+      Mask =
+          IRB.CreateTrunc(Mask, IRB.getIntNTy(ANumElements), "_ms_mask_trunc");
+      Mask =
+          IRB.CreateZExt(Mask, IRB.getIntNTy(MaskNumElements), "_ms_mask_zext");
+    }
+
     // Convert i16 mask to <16 x i1>
     Mask = IRB.CreateBitCast(
-        Mask, FixedVectorType::get(IRB.getInt1Ty(), ANumElements));
+        Mask, FixedVectorType::get(IRB.getInt1Ty(), MaskNumElements),
+        "_ms_mask_bitcast");
 
-    Value *AShadow = getShadow(A);
-    /// For scalars:
-    /// Since they are converting from floating-point, the output is:
+    /// For floating-point to integer conversion, the output is:
     /// - fully uninitialized if *any* bit of the input is uninitialized
     /// - fully ininitialized if all bits of the input are ininitialized
     /// We apply the same principle on a per-element basis for vectors.
-    AShadow = IRB.CreateSExt(IRB.CreateICmpNE(AShadow, getCleanShadow(A)),
-                             getShadowTy(A));
+    ///
+    /// We use the scalar width of the return type instead of A's.
+    AShadow = IRB.CreateSExt(
+        IRB.CreateICmpNE(AShadow, getCleanShadow(AShadow->getType())),
+        getShadowTy(&I), "_ms_a_shadow");
 
     Value *WriteThroughShadow = getShadow(WriteThrough);
-    Value *Shadow = IRB.CreateSelect(Mask, AShadow, WriteThroughShadow);
+    Value *Shadow = IRB.CreateSelect(Mask, AShadow, WriteThroughShadow,
+                                     "_ms_writethru_select");
 
     setShadow(&I, Shadow);
     setOriginForNaryOp(I);
@@ -5167,6 +5348,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     case Intrinsic::x86_sse_ldmxcsr:
       handleLdmxcsr(I);
       break;
+
+    // Convert Scalar Double Precision Floating-Point Value
+    //   to Unsigned Doubleword Integer
+    // etc.
     case Intrinsic::x86_avx512_vcvtsd2usi64:
     case Intrinsic::x86_avx512_vcvtsd2usi32:
     case Intrinsic::x86_avx512_vcvtss2usi64:
@@ -5207,6 +5392,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       break;
     }
 
+    // Convert Packed Double Precision Floating-Point Values
+    //   to Packed Single Precision Floating-Point Values
     case Intrinsic::x86_sse2_cvtpd2ps:
     case Intrinsic::x86_sse2_cvtps2dq:
     case Intrinsic::x86_sse2_cvtpd2dq:
@@ -5221,6 +5408,20 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       break;
     }
 
+    // Convert Single-Precision FP Value to 16-bit FP Value
+    // <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512
+    //                (<16 x float>, i32, <16 x i16>, i16)
+    //  <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128
+    //                (<4 x float>, i32, <8 x i16>, i8)
+    //  <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256
+    //                (<8 x float>, i32, <8 x i16>, i8)
+    case Intrinsic::x86_avx512_mask_vcvtps2ph_512:
+    case Intrinsic::x86_avx512_mask_vcvtps2ph_256:
+    case Intrinsic::x86_avx512_mask_vcvtps2ph_128:
+      handleAVX512VectorConvertFPToInt(I, /*LastMask=*/true);
+      break;
+
+    // Shift Packed Data (Left Logical, Right Arithmetic, Right Logical)
     case Intrinsic::x86_avx512_psll_w_512:
     case Intrinsic::x86_avx512_psll_d_512:
     case Intrinsic::x86_avx512_psll_q_512:
@@ -5374,21 +5575,185 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       handleVectorSadIntrinsic(I);
       break;
 
+    // Multiply and Add Packed Words
+    //   < 4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>)
+    //   < 8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>)
+    //   <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16>, <32 x i16>)
+    //
+    // Multiply and Add Packed Signed and Unsigned Bytes
+    //   < 8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>)
+    //   <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>)
+    //   <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>)
+    //
+    // These intrinsics are auto-upgraded into non-masked forms:
+    //   < 4 x i32> @llvm.x86.avx512.mask.pmaddw.d.128
+    //                  (<8 x i16>, <8 x i16>, <4 x i32>, i8)
+    //   < 8 x i32> @llvm.x86.avx512.mask.pmaddw.d.256
+    //                  (<16 x i16>, <16 x i16>, <8 x i32>, i8)
+    //   <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512
+    //                  (<32 x i16>, <32 x i16>, <16 x i32>, i16)
+    //   < 8 x i16> @llvm.x86.avx512.mask.pmaddubs.w.128
+    //                  (<16 x i8>, <16 x i8>, <8 x i16>, i8)
+    //   <16 x i16> @llvm.x86.avx512.mask.pmaddubs.w.256
+    //                  (<32 x i8>, <32 x i8>, <16 x i16>, i16)
+    //   <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512
+    //                  (<64 x i8>, <64 x i8>, <32 x i16>, i32)
     case Intrinsic::x86_sse2_pmadd_wd:
     case Intrinsic::x86_avx2_pmadd_wd:
+    case Intrinsic::x86_avx512_pmaddw_d_512:
     case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
     case Intrinsic::x86_avx2_pmadd_ub_sw:
-      handleVectorPmaddIntrinsic(I);
+    case Intrinsic::x86_avx512_pmaddubs_w_512:
+      handleVectorPmaddIntrinsic(I, /*ReductionFactor=*/2);
       break;
 
+    // <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64>, <1 x i64>)
     case Intrinsic::x86_ssse3_pmadd_ub_sw:
-      handleVectorPmaddIntrinsic(I, 8);
+      handleVectorPmaddIntrinsic(I, /*ReductionFactor=*/2, /*EltSize=*/8);
       break;
 
+    // <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64>, <1 x i64>)
     case Intrinsic::x86_mmx_pmadd_wd:
-      handleVectorPmaddIntrinsic(I, 16);
+      handleVectorPmaddIntrinsic(I, /*ReductionFactor=*/2, /*EltSize=*/16);
+      break;
+
+    // AVX Vector Neural Network Instructions: bytes
+    //
+    // Multiply and Add Packed Signed and Unsigned Bytes
+    //   < 4 x i32> @llvm.x86.avx512.vpdpbusd.128
+    //                  (< 4 x i32>, < 4 x i32>, < 4 x i32>)
+    //   < 8 x i32> @llvm.x86.avx512.vpdpbusd.256
+    //                  (< 8 x i32>, < 8 x i32>, < 8 x i32>)
+    //   <16 x i32> @llvm.x86.avx512.vpdpbusd.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>)
+    //
+    // Multiply and Add Unsigned and Signed Bytes With Saturation
+    //   < 4 x i32> @llvm.x86.avx512.vpdpbusds.128
+    //                  (< 4 x i32>, < 4 x i32>, < 4 x i32>)
+    //   < 8 x i32> @llvm.x86.avx512.vpdpbusds.256
+    //                  (< 8 x i32>, < 8 x i32>, < 8 x i32>)
+    //   <16 x i32> @llvm.x86.avx512.vpdpbusds.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>)
+    //
+    //   < 4 x i32> @llvm.x86.avx2.vpdpbssd.128
+    //                  (< 4 x i32>, < 4 x i32>, < 4 x i32>)
+    //   < 8 x i32> @llvm.x86.avx2.vpdpbssd.256
+    //                  (< 8 x i32>, < 8 x i32>, < 8 x i32>)
+    //
+    //   < 4 x i32> @llvm.x86.avx2.vpdpbssds.128
+    //                  (< 4 x i32>, < 4 x i32>, < 4 x i32>)
+    //   < 8 x i32> @llvm.x86.avx2.vpdpbssds.256
+    //                  (< 8 x i32>, < 8 x i32>, < 8 x i32>)
+    //
+    //   <16 x i32> @llvm.x86.avx10.vpdpbssd.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>)
+    //   <16 x i32> @llvm.x86.avx10.vpdpbssds.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>)
+    //
+    // These intrinsics are auto-upgraded into non-masked forms:
+    //   <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128
+    //                  (<4 x i32>, <4 x i32>, <4 x i32>, i8)
+    //   <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128
+    //                  (<4 x i32>, <4 x i32>, <4 x i32>, i8)
+    //   <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256
+    //                  (<8 x i32>, <8 x i32>, <8 x i32>, i8)
+    //   <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256
+    //                  (<8 x i32>, <8 x i32>, <8 x i32>, i8)
+    //   <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>, i16)
+    //   <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>, i16)
+    //
+    //   <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128
+    //                  (<4 x i32>, <4 x i32>, <4 x i32>, i8)
+    //   <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128
+    //                  (<4 x i32>, <4 x i32>, <4 x i32>, i8)
+    //   <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256
+    //                  (<8 x i32>, <8 x i32>, <8 x i32>, i8)
+    //   <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256
+    //                  (<8 x i32>, <8 x i32>, <8 x i32>, i8)
+    //   <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>, i16)
+    //   <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>, i16)
+    case Intrinsic::x86_avx512_vpdpbusd_128:
+    case Intrinsic::x86_avx512_vpdpbusd_256:
+    case Intrinsic::x86_avx512_vpdpbusd_512:
+    case Intrinsic::x86_avx512_vpdpbusds_128:
+    case Intrinsic::x86_avx512_vpdpbusds_256:
+    case Intrinsic::x86_avx512_vpdpbusds_512:
+    case Intrinsic::x86_avx2_vpdpbssd_128:
+    case Intrinsic::x86_avx2_vpdpbssd_256:
+    case Intrinsic::x86_avx2_vpdpbssds_128:
+    case Intrinsic::x86_avx2_vpdpbssds_256:
+    case Intrinsic::x86_avx10_vpdpbssd_512:
+    case Intrinsic::x86_avx10_vpdpbssds_512:
+      handleVectorPmaddIntrinsic(I, /*ReductionFactor=*/4, /*EltSize=*/8);
       break;
 
+    // AVX Vector Neural Network Instructions: words
+    //
+    // Multiply and Add Signed Word Integers
+    //   < 4 x i32> @llvm.x86.avx512.vpdpwssd.128
+    //                  (< 4 x i32>, < 4 x i32>, < 4 x i32>)
+    //   < 8 x i32> @llvm.x86.avx512.vpdpwssd.256
+    //                  (< 8 x i32>, < 8 x i32>, < 8 x i32>)
+    //   <16 x i32> @llvm.x86.avx512.vpdpwssd.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>)
+    //
+    // Multiply and Add Signed Word Integers With Saturation
+    //   < 4 x i32> @llvm.x86.avx512.vpdpwssds.128
+    //                  (< 4 x i32>, < 4 x i32>, < 4 x i32>)
+    //   < 8 x i32> @llvm.x86.avx512.vpdpwssds.256
+    //                  (< 8 x i32>, < 8 x i32>, < 8 x i32>)
+    //   <16 x i32> @llvm.x86.avx512.vpdpwssds.512
+    //                  (<16 x i32>, <16 x i32>, <16 x i32>)
+    //
+    // These intrinsics are auto-upgraded into non-masked forms:
+    //   <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128
+    //                 (<4 x i32>, <4 x i32>, <4 x i32>, i8)
+    //   <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128
+    //                 (<4 x i32>, <4 x i32>, <4 x i32>, i8)
+    //   <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256
+    //                 (<8 x i32>, <8 x i32>, <8 x i32>, i8)
+    //   <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256
+    //                 (<8 x i32>, <8 x i32>, <8 x i32>, i8)
+    //   <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512
+    //                 (<16 x i32>, <16 x i32>, <16 x i32>, i16)
+    //   <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512
+    //                 (<16 x i32>, <16 x i32>, <16 x i32>, i16)
+    //
+    //   <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128
+    //                 (<4 x i32>, <4 x i32>, <4 x i32>, i8)
+    //   <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128
+    //                 (<4 x i32>, <4 x i32>, <4 x i32>, i8)
+    //   <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256
+    //                 (<8 x i32>, <8 x i32>, <8 x i32>, i8)
+    //   <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256
+    //                 (<8 x i32>, <8 x i32>, <8 x i32>, i8)
+    //   <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512
+    //                 (<16 x i32>, <16 x i32>, <16 x i32>, i16)
+    //   <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512
+    //                 (<16 x i32>, <16 x i32>, <16 x i32>, i16)
+    case Intrinsic::x86_avx512_vpdpwssd_128:
+    case Intrinsic::x86_avx512_vpdpwssd_256:
+    case Intrinsic::x86_avx512_vpdpwssd_512:
+    case Intrinsic::x86_avx512_vpdpwssds_128:
+    case Intrinsic::x86_avx512_vpdpwssds_256:
+    case Intrinsic::x86_avx512_vpdpwssds_512:
+      handleVectorPmaddIntrinsic(I, /*ReductionFactor=*/2, /*EltSize=*/16);
+      break;
+
+      // TODO: Dot Product of BF16 Pairs Accumulated Into Packed Single
+      // Precision
+      //   <4 x float> @llvm.x86.avx512bf16.dpbf16ps.128
+      //                   (<4 x float>, <8 x bfloat>, <8 x bfloat>)
+      //   <8 x float> @llvm.x86.avx512bf16.dpbf16ps.256
+      //                   (<8 x float>, <16 x bfloat>, <16 x bfloat>)
+      //   <16 x float> @llvm.x86.avx512bf16.dpbf16ps.512
+      //                   (<16 x float>, <32 x bfloat>, <32 x bfloat>)
+      // handleVectorPmaddIntrinsic() currently only handles integer types.
+
     case Intrinsic::x86_sse_cmp_ss:
     case Intrinsic::x86_sse2_cmp_sd:
     case Intrinsic::x86_sse_comieq_ss:
@@ -5603,10 +5968,34 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       handleAVXVpermi2var(I);
       break;
 
-    case Intrinsic::x86_avx512_mask_cvtps2dq_512: {
-      handleAVX512VectorConvertFPToInt(I);
+    // Packed Shuffle
+    //   llvm.x86.sse.pshuf.w(<1 x i64>, i8)
+    //   llvm.x86.ssse3.pshuf.b(<1 x i64>, <1 x i64>)
+    //   llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>)
+    //   llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>)
+    //   llvm.x86.avx512.pshuf.b.512(<64 x i8>, <64 x i8>)
+    //
+    // The following intrinsics are auto-upgraded:
+    //   llvm.x86.sse2.pshuf.d(<4 x i32>, i8)
+    //   llvm.x86.sse2.gpshufh.w(<8 x i16>, i8)
+    //   llvm.x86.sse2.pshufl.w(<8 x i16>, i8)
+    case Intrinsic::x86_avx2_pshuf_b:
+    case Intrinsic::x86_sse_pshuf_w:
+    case Intrinsic::x86_ssse3_pshuf_b_128:
+    case Intrinsic::x86_ssse3_pshuf_b:
+    case Intrinsic::x86_avx512_pshuf_b_512:
+      handleIntrinsicByApplyingToShadow(I, I.getIntrinsicID(),
+                                        /*trailingVerbatimArgs=*/1);
+      break;
+
+    // Convert Packed Single Precision Floating-Point Values
+    //   to Packed Signed Doubleword Integer Values
+    //
+    // <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512
+    //                (<16 x float>, <16 x i32>, i16, i32)
+    case Intrinsic::x86_avx512_mask_cvtps2dq_512:
+      handleAVX512VectorConvertFPToInt(I, /*LastMask=*/false);
       break;
-    }
 
     // AVX512 PMOV: Packed MOV, with truncation
     // Precisely handled by applying the same intrinsic to the shadow
diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
index 61285810cc529..f5b668678ab7c 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfFlattening.cpp
@@ -58,7 +58,7 @@ void assignProfileData(Function &F, ArrayRef<uint64_t> RawCounters) {
         uint64_t TrueCount, FalseCount = 0;
         if (!PA.getSelectInstrProfile(*SI, TrueCount, FalseCount))
           continue;
-        setProfMetadata(F.getParent(), SI, {TrueCount, FalseCount},
+        setProfMetadata(SI, {TrueCount, FalseCount},
                         std::max(TrueCount, FalseCount));
       }
     if (succ_size(&BB) < 2)
@@ -67,7 +67,7 @@ void assignProfileData(Function &F, ArrayRef<uint64_t> RawCounters) {
     if (!PA.getOutgoingBranchWeights(BB, ProfileHolder, MaxCount))
       continue;
     assert(MaxCount > 0);
-    setProfMetadata(F.getParent(), BB.getTerminator(), ProfileHolder, MaxCount);
+    setProfMetadata(BB.getTerminator(), ProfileHolder, MaxCount);
   }
 }
 
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 6f06a260e238c..d9e850e7a2bf3 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1727,7 +1727,7 @@ void PGOUseFunc::setBranchWeights() {
     }
 
     if (MaxCount)
-      setProfMetadata(M, TI, EdgeCounts, MaxCount);
+      setProfMetadata(TI, EdgeCounts, MaxCount);
     else {
       // A zero MaxCount can come about when we have a BB with a positive
       // count, and whose successor blocks all have 0 count. This can happen
@@ -1801,7 +1801,7 @@ void SelectInstVisitor::annotateOneSelectInst(SelectInst &SI) {
   SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);
   uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);
   if (MaxCount)
-    setProfMetadata(F.getParent(), &SI, SCounts, MaxCount);
+    setProfMetadata(&SI, SCounts, MaxCount);
 }
 
 void SelectInstVisitor::visitSelectInst(SelectInst &SI) {
@@ -2407,13 +2407,9 @@ static std::string getSimpleNodeName(const BasicBlock *Node) {
   return SimpleNodeName;
 }
 
-void llvm::setProfMetadata(Module *M, Instruction *TI,
-                           ArrayRef<uint64_t> EdgeCounts, uint64_t MaxCount) {
-  assert(MaxCount > 0 && "Bad max count");
-  uint64_t Scale = calculateCountScale(MaxCount);
-  SmallVector<unsigned, 4> Weights;
-  for (const auto &ECI : EdgeCounts)
-    Weights.push_back(scaleBranchCount(ECI, Scale));
+void llvm::setProfMetadata(Instruction *TI, ArrayRef<uint64_t> EdgeCounts,
+                           uint64_t MaxCount) {
+  auto Weights = downscaleWeights(EdgeCounts, MaxCount);
 
   LLVM_DEBUG(dbgs() << "Weight is: "; for (const auto &W
                                            : Weights) {
@@ -2434,7 +2430,7 @@ void llvm::setProfMetadata(Module *M, Instruction *TI,
     uint64_t TotalCount =
         std::accumulate(EdgeCounts.begin(), EdgeCounts.end(), (uint64_t)0,
                         [](uint64_t c1, uint64_t c2) { return c1 + c2; });
-    Scale = calculateCountScale(WSum);
+    uint64_t Scale = calculateCountScale(WSum);
     BranchProbability BP(scaleBranchCount(Weights[0], Scale),
                          scaleBranchCount(WSum, Scale));
     std::string BranchProbStr;
diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index ce1d9f1923d05..343bec37018c5 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -432,7 +432,7 @@ bool MemOPSizeOpt::perform(MemOp MO) {
   Updates.clear();
 
   if (MaxCount)
-    setProfMetadata(Func.getParent(), SI, CaseCounts, MaxCount);
+    setProfMetadata(SI, CaseCounts, MaxCount);
 
   LLVM_DEBUG(dbgs() << *BB << "\n");
   LLVM_DEBUG(dbgs() << *DefaultBB << "\n");
diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index 5485998164f1e..0d48a350254ee 100644
--- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -80,6 +80,10 @@ static cl::opt<bool> ClCompoundReadBeforeWrite(
     "tsan-compound-read-before-write", cl::init(false),
     cl::desc("Emit special compound instrumentation for reads-before-writes"),
     cl::Hidden);
+static cl::opt<bool>
+    ClOmitNonCaptured("tsan-omit-by-pointer-capturing", cl::init(true),
+                      cl::desc("Omit accesses due to pointer capturing"),
+                      cl::Hidden);
 
 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
@@ -450,7 +454,8 @@ void ThreadSanitizer::chooseInstructionsToInstrument(
 
     const AllocaInst *AI = findAllocaForValue(Addr);
     // Instead of Addr, we should check whether its base pointer is captured.
-    if (AI && !PointerMayBeCaptured(AI, /*ReturnCaptures=*/true)) {
+    if (AI && !PointerMayBeCaptured(AI, /*ReturnCaptures=*/true) &&
+        ClOmitNonCaptured) {
       // The variable is addressable but not captured, so it cannot be
       // referenced from a different thread and participate in a data race
       // (see llvm/Analysis/CaptureTracking.h for details).
diff --git a/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp
index 46b56737e906e..9471ae3a6c4e9 100644
--- a/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/TypeSanitizer.cpp
@@ -789,6 +789,13 @@ bool TypeSanitizer::instrumentMemInst(Value *V, Instruction *ShadowBase,
   bool NeedsMemMove = false;
   IRBuilder<> IRB(BB, IP);
 
+  auto GetAllocaSize = [&](AllocaInst *AI) {
+    return IRB.CreateMul(
+        IRB.CreateZExtOrTrunc(AI->getArraySize(), IntptrTy),
+        ConstantInt::get(IntptrTy,
+                         DL.getTypeAllocSize(AI->getAllocatedType())));
+  };
+
   if (auto *A = dyn_cast<Argument>(V)) {
     assert(A->hasByValAttr() && "Type reset for non-byval argument?");
 
@@ -811,8 +818,12 @@ bool TypeSanitizer::instrumentMemInst(Value *V, Instruction *ShadowBase,
         }
       }
     } else if (auto *II = dyn_cast<LifetimeIntrinsic>(I)) {
-      Size = II->getArgOperand(0);
-      Dest = II->getArgOperand(1);
+      auto *AI = dyn_cast<AllocaInst>(II->getArgOperand(0));
+      if (!AI)
+        return false;
+
+      Size = GetAllocaSize(AI);
+      Dest = II->getArgOperand(0);
     } else if (auto *AI = dyn_cast<AllocaInst>(I)) {
       // We need to clear the types for new stack allocations (or else we might
       // read stale type information from a previous function execution).
@@ -820,10 +831,7 @@ bool TypeSanitizer::instrumentMemInst(Value *V, Instruction *ShadowBase,
       IRB.SetInsertPoint(&*std::next(BasicBlock::iterator(I)));
       IRB.SetInstDebugLocation(I);
 
-      Size = IRB.CreateMul(
-          IRB.CreateZExtOrTrunc(AI->getArraySize(), IntptrTy),
-          ConstantInt::get(IntptrTy,
-                           DL.getTypeAllocSize(AI->getAllocatedType())));
+      Size = GetAllocaSize(AI);
       Dest = I;
     } else {
       return false;
diff --git a/llvm/lib/Transforms/Scalar/CMakeLists.txt b/llvm/lib/Transforms/Scalar/CMakeLists.txt
index 84a5b02043d01..765059d0c3b20 100644
--- a/llvm/lib/Transforms/Scalar/CMakeLists.txt
+++ b/llvm/lib/Transforms/Scalar/CMakeLists.txt
@@ -95,6 +95,7 @@ add_llvm_component_library(LLVMScalarOpts
   Analysis
   Core
   InstCombine
+  ProfileData
   Support
   TransformUtils
   )
diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
index 938aab5879044..434b55868c99d 100644
--- a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
@@ -60,7 +60,6 @@
 #include "llvm/Transforms/Scalar/DFAJumpThreading.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/CodeMetrics.h"
@@ -447,7 +446,7 @@ struct MainSwitch {
   /// Also, collect select instructions to unfold.
   bool isCandidate(const SwitchInst *SI) {
     std::deque<std::pair<Value *, BasicBlock *>> Q;
-    SmallSet<Value *, 16> SeenValues;
+    SmallPtrSet<Value *, 16> SeenValues;
     SelectInsts.clear();
 
     Value *SICond = SI->getCondition();
@@ -511,7 +510,7 @@ struct MainSwitch {
 
   void addToQueue(Value *Val, BasicBlock *BB,
                   std::deque<std::pair<Value *, BasicBlock *>> &Q,
-                  SmallSet<Value *, 16> &SeenValues) {
+                  SmallPtrSet<Value *, 16> &SeenValues) {
     if (SeenValues.insert(Val).second)
       Q.push_back({Val, BB});
   }
@@ -713,7 +712,7 @@ struct AllSwitchPaths {
 
     // Some blocks have multiple edges to the same successor, and this set
     // is used to prevent a duplicate path from being generated
-    SmallSet<BasicBlock *, 4> Successors;
+    SmallPtrSet<BasicBlock *, 4> Successors;
     for (BasicBlock *Succ : successors(BB)) {
       if (!Successors.insert(Succ).second)
         continue;
@@ -762,7 +761,7 @@ struct AllSwitchPaths {
 
     SmallVector<PHINode *, 8> Stack;
     Stack.push_back(FirstDef);
-    SmallSet<Value *, 16> SeenValues;
+    SmallPtrSet<Value *, 16> SeenValues;
 
     while (!Stack.empty()) {
       PHINode *CurPhi = Stack.pop_back_val();
@@ -955,7 +954,7 @@ struct TransformDFA {
     DuplicateBlockMap DuplicateMap;
     DefMap NewDefs;
 
-    SmallSet<BasicBlock *, 16> BlocksToClean;
+    SmallPtrSet<BasicBlock *, 16> BlocksToClean;
     BlocksToClean.insert_range(successors(SwitchBlock));
 
     for (ThreadingPath &TPath : SwitchPaths->getThreadingPaths()) {
@@ -984,7 +983,7 @@ struct TransformDFA {
   /// the predecessors, and phis in the successor blocks.
   void createExitPath(DefMap &NewDefs, ThreadingPath &Path,
                       DuplicateBlockMap &DuplicateMap,
-                      SmallSet<BasicBlock *, 16> &BlocksToClean,
+                      SmallPtrSet<BasicBlock *, 16> &BlocksToClean,
                       DomTreeUpdater *DTU) {
     APInt NextState = Path.getExitValue();
     const BasicBlock *Determinator = Path.getDeterminatorBB();
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 9b8718035df62..7ad710ddfb273 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -38,6 +38,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -69,6 +70,7 @@
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -543,15 +545,8 @@ static void shortenAssignment(Instruction *Inst, Value *OriginalDest,
   };
 
   // Insert an unlinked dbg.assign intrinsic for the dead fragment after each
-  // overlapping dbg.assign intrinsic. The loop invalidates the iterators
-  // returned by getAssignmentMarkers so save a copy of the markers to iterate
-  // over.
-  auto LinkedRange = at::getAssignmentMarkers(Inst);
-  SmallVector<DbgVariableRecord *> LinkedDVRAssigns =
-      at::getDVRAssignmentMarkers(Inst);
-  SmallVector<DbgAssignIntrinsic *> Linked(LinkedRange.begin(),
-                                           LinkedRange.end());
-  auto InsertAssignForOverlap = [&](auto *Assign) {
+  // overlapping dbg.assign intrinsic.
+  for (DbgVariableRecord *Assign : at::getDVRAssignmentMarkers(Inst)) {
     std::optional<DIExpression::FragmentInfo> NewFragment;
     if (!at::calculateFragmentIntersect(DL, OriginalDest, DeadSliceOffsetInBits,
                                         DeadSliceSizeInBits, Assign,
@@ -561,11 +556,11 @@ static void shortenAssignment(Instruction *Inst, Value *OriginalDest,
       // cautious and unlink the whole assignment from the store.
       Assign->setKillAddress();
       Assign->setAssignId(GetDeadLink());
-      return;
+      continue;
     }
     // No intersect.
     if (NewFragment->SizeInBits == 0)
-      return;
+      continue;
 
     // Fragments overlap: insert a new dbg.assign for this dead part.
     auto *NewAssign = static_cast<decltype(Assign)>(Assign->clone());
@@ -574,9 +569,7 @@ static void shortenAssignment(Instruction *Inst, Value *OriginalDest,
     if (NewFragment)
       SetDeadFragExpr(NewAssign, *NewFragment);
     NewAssign->setKillAddress();
-  };
-  for_each(Linked, InsertAssignForOverlap);
-  for_each(LinkedDVRAssigns, InsertAssignForOverlap);
+  }
 }
 
 /// Update the attributes given that a memory access is updated (the
@@ -683,15 +676,13 @@ static bool tryToShorten(Instruction *DeadI, int64_t &DeadStart,
                     << "\n  KILLER [" << ToRemoveStart << ", "
                     << int64_t(ToRemoveStart + ToRemoveSize) << ")\n");
 
-  Value *DeadWriteLength = DeadIntrinsic->getLength();
-  Value *TrimmedLength = ConstantInt::get(DeadWriteLength->getType(), NewSize);
-  DeadIntrinsic->setLength(TrimmedLength);
+  DeadIntrinsic->setLength(NewSize);
   DeadIntrinsic->setDestAlignment(PrefAlign);
 
   Value *OrigDest = DeadIntrinsic->getRawDest();
   if (!IsOverwriteEnd) {
     Value *Indices[1] = {
-        ConstantInt::get(DeadWriteLength->getType(), ToRemoveSize)};
+        ConstantInt::get(DeadIntrinsic->getLength()->getType(), ToRemoveSize)};
     Instruction *NewDestGEP = GetElementPtrInst::CreateInBounds(
         Type::getInt8Ty(DeadIntrinsic->getContext()), OrigDest, Indices, "",
         DeadI->getIterator());
@@ -1363,7 +1354,7 @@ struct DSEState {
     if (auto *CB = dyn_cast<CallBase>(I)) {
       if (CB->getIntrinsicID() == Intrinsic::lifetime_end)
         return {
-            std::make_pair(MemoryLocation::getForArgument(CB, 1, &TLI), false)};
+            std::make_pair(MemoryLocation::getForArgument(CB, 0, &TLI), false)};
       if (Value *FreedOp = getFreedOperand(CB, &TLI))
         return {std::make_pair(MemoryLocation::getAfter(FreedOp), true)};
     }
@@ -2666,3 +2657,79 @@ PreservedAnalyses DSEPass::run(Function &F, FunctionAnalysisManager &AM) {
   PA.preserve<LoopAnalysis>();
   return PA;
 }
+
+namespace {
+
+/// A legacy pass for the legacy pass manager that wraps \c DSEPass.
+class DSELegacyPass : public FunctionPass {
+public:
+  static char ID; // Pass identification, replacement for typeid
+
+  DSELegacyPass() : FunctionPass(ID) {
+    initializeDSELegacyPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (skipFunction(F))
+      return false;
+
+    AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
+    DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+    const TargetLibraryInfo &TLI =
+        getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+    MemorySSA &MSSA = getAnalysis<MemorySSAWrapperPass>().getMSSA();
+    PostDominatorTree &PDT =
+        getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree();
+    LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+
+    bool Changed = eliminateDeadStores(F, AA, MSSA, DT, PDT, TLI, LI);
+
+#ifdef LLVM_ENABLE_STATS
+    if (AreStatisticsEnabled())
+      for (auto &I : instructions(F))
+        NumRemainingStores += isa<StoreInst>(&I);
+#endif
+
+    return Changed;
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<AAResultsWrapperPass>();
+    AU.addRequired<TargetLibraryInfoWrapperPass>();
+    AU.addPreserved<GlobalsAAWrapperPass>();
+    AU.addRequired<DominatorTreeWrapperPass>();
+    AU.addPreserved<DominatorTreeWrapperPass>();
+    AU.addRequired<PostDominatorTreeWrapperPass>();
+    AU.addRequired<MemorySSAWrapperPass>();
+    AU.addPreserved<PostDominatorTreeWrapperPass>();
+    AU.addPreserved<MemorySSAWrapperPass>();
+    AU.addRequired<LoopInfoWrapperPass>();
+    AU.addPreserved<LoopInfoWrapperPass>();
+    AU.addRequired<AssumptionCacheTracker>();
+  }
+};
+
+} // end anonymous namespace
+
+char DSELegacyPass::ID = 0;
+
+INITIALIZE_PASS_BEGIN(DSELegacyPass, "dse", "Dead Store Elimination", false,
+                      false)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(PostDominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
+INITIALIZE_PASS_END(DSELegacyPass, "dse", "Dead Store Elimination", false,
+                    false)
+
+namespace llvm {
+LLVM_ABI FunctionPass *createDeadStoreEliminationPass() {
+  return new DSELegacyPass();
+}
+} // namespace llvm
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 7704e49c499da..4baa3b3eb8242 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -978,7 +978,7 @@ static bool IsValueFullyAvailableInBlock(
   unsigned NumNewNewSpeculativelyAvailableBBs = 0;
 
 #ifndef NDEBUG
-  SmallSet<BasicBlock *, 32> NewSpeculativelyAvailableBBs;
+  SmallPtrSet<BasicBlock *, 32> NewSpeculativelyAvailableBBs;
   SmallVector<BasicBlock *, 32> AvailableBBs;
 #endif
 
@@ -1222,7 +1222,7 @@ static bool liesBetween(const Instruction *From, Instruction *Between,
                         const Instruction *To, const DominatorTree *DT) {
   if (From->getParent() == Between->getParent())
     return DT->dominates(From, Between);
-  SmallSet<BasicBlock *, 1> Exclusion;
+  SmallPtrSet<BasicBlock *, 1> Exclusion;
   Exclusion.insert(Between->getParent());
   return !isPotentiallyReachable(From, To, &Exclusion, DT);
 }
diff --git a/llvm/lib/Transforms/Scalar/GuardWidening.cpp b/llvm/lib/Transforms/Scalar/GuardWidening.cpp
index 3ba5b79293bcd..d99f1eb9c93cd 100644
--- a/llvm/lib/Transforms/Scalar/GuardWidening.cpp
+++ b/llvm/lib/Transforms/Scalar/GuardWidening.cpp
@@ -642,9 +642,9 @@ Value *GuardWideningImpl::freezeAndPush(Value *Orig,
     return FI;
   }
 
-  SmallSet<Value *, 16> Visited;
+  SmallPtrSet<Value *, 16> Visited;
   SmallVector<Value *, 16> Worklist;
-  SmallSet<Instruction *, 16> DropPoisonFlags;
+  SmallPtrSet<Instruction *, 16> DropPoisonFlags;
   SmallVector<Value *, 16> NeedFreeze;
   DenseMap<Value *, FreezeInst *> CacheOfFreezes;
 
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 334c911191cb8..c32731185afd0 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -28,7 +28,6 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/iterator_range.h"
@@ -1613,7 +1612,7 @@ bool IndVarSimplify::optimizeLoopExits(Loop *L, SCEVExpander &Rewriter) {
     if (CurrMaxExit == MaxBECount)
       SkipLastIter = true;
   };
-  SmallSet<const SCEV *, 8> DominatingExactExitCounts;
+  SmallPtrSet<const SCEV *, 8> DominatingExactExitCounts;
   for (BasicBlock *ExitingBB : ExitingBlocks) {
     const SCEV *ExactExitCount = SE->getExitCount(L, ExitingBB);
     const SCEV *MaxExitCount = SE->getExitCount(
diff --git a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
index 85ee824b67121..a097d338a42ca 100644
--- a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
+++ b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
@@ -434,7 +434,7 @@ bool InferAddressSpacesImpl::rewriteIntrinsicOperands(IntrinsicInst *II,
     NewV = NewV->stripPointerCasts();
     Function *NewDecl = Intrinsic::getOrInsertDeclaration(
         M, II->getIntrinsicID(), {NewV->getType()});
-    II->setArgOperand(1, NewV);
+    II->setArgOperand(0, NewV);
     II->setCalledFunction(NewDecl);
     return true;
   }
@@ -491,7 +491,7 @@ void InferAddressSpacesImpl::collectRewritableIntrinsicOperands(
   }
   case Intrinsic::lifetime_start:
   case Intrinsic::lifetime_end: {
-    appendsFlatAddressExpressionToPostorderStack(II->getArgOperand(1),
+    appendsFlatAddressExpressionToPostorderStack(II->getArgOperand(0),
                                                  PostorderStack, Visited);
     break;
   }
diff --git a/llvm/lib/Transforms/Scalar/InferAlignment.cpp b/llvm/lib/Transforms/Scalar/InferAlignment.cpp
index 0ddc23152d84f..e9bf59c6850a3 100644
--- a/llvm/lib/Transforms/Scalar/InferAlignment.cpp
+++ b/llvm/lib/Transforms/Scalar/InferAlignment.cpp
@@ -58,14 +58,55 @@ bool inferAlignment(Function &F, AssumptionCache &AC, DominatorTree &DT) {
   }
 
   // Compute alignment from known bits.
+  auto InferFromKnownBits = [&](Instruction &I, Value *PtrOp) {
+    KnownBits Known = computeKnownBits(PtrOp, DL, &AC, &I, &DT);
+    unsigned TrailZ =
+        std::min(Known.countMinTrailingZeros(), +Value::MaxAlignmentExponent);
+    return Align(1ull << std::min(Known.getBitWidth() - 1, TrailZ));
+  };
+
+  // Propagate alignment between loads and stores that originate from the
+  // same base pointer.
+  DenseMap<Value *, Align> BestBasePointerAligns;
+  auto InferFromBasePointer = [&](Value *PtrOp, Align LoadStoreAlign) {
+    APInt OffsetFromBase(DL.getIndexTypeSizeInBits(PtrOp->getType()), 0);
+    PtrOp = PtrOp->stripAndAccumulateConstantOffsets(DL, OffsetFromBase, true);
+    // Derive the base pointer alignment from the load/store alignment
+    // and the offset from the base pointer.
+    Align BasePointerAlign =
+        commonAlignment(LoadStoreAlign, OffsetFromBase.getLimitedValue());
+
+    auto [It, Inserted] =
+        BestBasePointerAligns.try_emplace(PtrOp, BasePointerAlign);
+    if (!Inserted) {
+      // If the stored base pointer alignment is better than the
+      // base pointer alignment we derived, we may be able to use it
+      // to improve the load/store alignment. If not, store the
+      // improved base pointer alignment for future iterations.
+      if (It->second > BasePointerAlign) {
+        Align BetterLoadStoreAlign =
+            commonAlignment(It->second, OffsetFromBase.getLimitedValue());
+        return BetterLoadStoreAlign;
+      }
+      It->second = BasePointerAlign;
+    }
+    return LoadStoreAlign;
+  };
+
   for (BasicBlock &BB : F) {
+    // We need to reset the map for each block because alignment information
+    // can only be propagated from instruction A to B if A dominates B.
+    // This is because control flow (and exception throwing) could be dependent
+    // on the address (and its alignment) at runtime. Some sort of dominator
+    // tree approach could be better, but doing a simple forward pass through a
+    // single basic block is correct too.
+    BestBasePointerAligns.clear();
+
     for (Instruction &I : BB) {
       Changed |= tryToImproveAlign(
           DL, &I, [&](Value *PtrOp, Align OldAlign, Align PrefAlign) {
-            KnownBits Known = computeKnownBits(PtrOp, DL, &AC, &I, &DT);
-            unsigned TrailZ = std::min(Known.countMinTrailingZeros(),
-                                       +Value::MaxAlignmentExponent);
-            return Align(1ull << std::min(Known.getBitWidth() - 1, TrailZ));
+            return std::max(InferFromKnownBits(I, PtrOp),
+                            InferFromBasePointer(PtrOp, OldAlign));
           });
     }
   }
diff --git a/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp b/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp
index 7f99cd2060a9d..b0478ee1fd94e 100644
--- a/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpTableToSwitch.cpp
@@ -7,14 +7,22 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Scalar/JumpTableToSwitch.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/ConstantFolding.h"
+#include "llvm/Analysis/CtxProfAnalysis.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/ProfDataUtils.h"
+#include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include <limits>
 
 using namespace llvm;
 
@@ -33,6 +41,8 @@ static cl::opt<unsigned> FunctionSizeThreshold(
              "or equal than this threshold."),
     cl::init(50));
 
+extern cl::opt<bool> ProfcheckDisableMetadataFixes;
+
 #define DEBUG_TYPE "jump-table-to-switch"
 
 namespace {
@@ -90,9 +100,11 @@ static std::optional<JumpTableTy> parseJumpTable(GetElementPtrInst *GEP,
   return JumpTable;
 }
 
-static BasicBlock *expandToSwitch(CallBase *CB, const JumpTableTy &JT,
-                                  DomTreeUpdater &DTU,
-                                  OptimizationRemarkEmitter &ORE) {
+static BasicBlock *
+expandToSwitch(CallBase *CB, const JumpTableTy &JT, DomTreeUpdater &DTU,
+               OptimizationRemarkEmitter &ORE,
+               llvm::function_ref<GlobalValue::GUID(const Function &)>
+                   GetGuidForFunction) {
   const bool IsVoid = CB->getType() == Type::getVoidTy(CB->getContext());
 
   SmallVector<DominatorTree::UpdateType, 8> DTUpdates;
@@ -115,7 +127,30 @@ static BasicBlock *expandToSwitch(CallBase *CB, const JumpTableTy &JT,
   IRBuilder<> BuilderTail(CB);
   PHINode *PHI =
       IsVoid ? nullptr : BuilderTail.CreatePHI(CB->getType(), JT.Funcs.size());
-
+  const auto *ProfMD = CB->getMetadata(LLVMContext::MD_prof);
+
+  SmallVector<uint64_t> BranchWeights;
+  DenseMap<GlobalValue::GUID, uint64_t> GuidToCounter;
+  const bool HadProfile = isValueProfileMD(ProfMD);
+  if (HadProfile) {
+    // The assumptions, coming in, are that the functions in JT.Funcs are
+    // defined in this module (from parseJumpTable).
+    assert(llvm::all_of(
+        JT.Funcs, [](const Function *F) { return F && !F->isDeclaration(); }));
+    BranchWeights.reserve(JT.Funcs.size() + 1);
+    // The first is the default target, which is the unreachable block created
+    // above.
+    BranchWeights.push_back(0U);
+    uint64_t TotalCount = 0;
+    auto Targets = getValueProfDataFromInst(
+        *CB, InstrProfValueKind::IPVK_IndirectCallTarget,
+        std::numeric_limits<uint32_t>::max(), TotalCount);
+
+    for (const auto &[G, C] : Targets) {
+      [[maybe_unused]] auto It = GuidToCounter.insert({G, C});
+      assert(It.second);
+    }
+  }
   for (auto [Index, Func] : llvm::enumerate(JT.Funcs)) {
     BasicBlock *B = BasicBlock::Create(Func->getContext(),
                                        "call." + Twine(Index), &F, Tail);
@@ -123,10 +158,19 @@ static BasicBlock *expandToSwitch(CallBase *CB, const JumpTableTy &JT,
     DTUpdates.push_back({DominatorTree::Insert, B, Tail});
 
     CallBase *Call = cast<CallBase>(CB->clone());
+    // The MD_prof metadata (VP kind), if it existed, can be dropped, it doesn't
+    // make sense on a direct call. Note that the values are used for the branch
+    // weights of the switch.
+    Call->setMetadata(LLVMContext::MD_prof, nullptr);
     Call->setCalledFunction(Func);
     Call->insertInto(B, B->end());
     Switch->addCase(
         cast<ConstantInt>(ConstantInt::get(JT.Index->getType(), Index)), B);
+    GlobalValue::GUID FctID = GetGuidForFunction(*Func);
+    // It'd be OK to _not_ find target functions in GuidToCounter, e.g. suppose
+    // just some of the jump targets are taken (for the given profile).
+    BranchWeights.push_back(FctID == 0U ? 0U
+                                        : GuidToCounter.lookup_or(FctID, 0U));
     BranchInst::Create(Tail, B);
     if (PHI)
       PHI->addIncoming(Call, B);
@@ -136,6 +180,13 @@ static BasicBlock *expandToSwitch(CallBase *CB, const JumpTableTy &JT,
     return OptimizationRemark(DEBUG_TYPE, "ReplacedJumpTableWithSwitch", CB)
            << "expanded indirect call into switch";
   });
+  if (HadProfile && !ProfcheckDisableMetadataFixes) {
+    // At least one of the targets must've been taken.
+    assert(llvm::any_of(BranchWeights, [](uint64_t V) { return V != 0; }));
+    setBranchWeights(*Switch, downscaleWeights(BranchWeights),
+                     /*IsExpected=*/false);
+  } else
+    setExplicitlyUnknownBranchWeights(*Switch);
   if (PHI)
     CB->replaceAllUsesWith(PHI);
   CB->eraseFromParent();
@@ -150,6 +201,15 @@ PreservedAnalyses JumpTableToSwitchPass::run(Function &F,
   PostDominatorTree *PDT = AM.getCachedResult<PostDominatorTreeAnalysis>(F);
   DomTreeUpdater DTU(DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy);
   bool Changed = false;
+  InstrProfSymtab Symtab;
+  if (auto E = Symtab.create(*F.getParent()))
+    F.getContext().emitError(
+        "Could not create indirect call table, likely corrupted IR" +
+        toString(std::move(E)));
+  DenseMap<const Function *, GlobalValue::GUID> FToGuid;
+  for (const auto &[G, FPtr] : Symtab.getIDToNameMap())
+    FToGuid.insert({FPtr, G});
+
   for (BasicBlock &BB : make_early_inc_range(F)) {
     BasicBlock *CurrentBB = &BB;
     while (CurrentBB) {
@@ -170,7 +230,12 @@ PreservedAnalyses JumpTableToSwitchPass::run(Function &F,
         std::optional<JumpTableTy> JumpTable = parseJumpTable(GEP, PtrTy);
         if (!JumpTable)
           continue;
-        SplittedOutTail = expandToSwitch(Call, *JumpTable, DTU, ORE);
+        SplittedOutTail = expandToSwitch(
+            Call, *JumpTable, DTU, ORE, [&](const Function &Fct) {
+              if (Fct.getMetadata(AssignGUIDPass::GUIDMetadataName))
+                return AssignGUIDPass::getGUID(Fct);
+              return FToGuid.lookup_or(&Fct, 0U);
+            });
         Changed = true;
         break;
       }
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index c3f80f901a120..e157cc9212769 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -169,6 +169,8 @@ cl::opt<unsigned> llvm::SetLicmMssaNoAccForPromotionCap(
              "number of accesses allowed to be present in a loop in order to "
              "enable memory promotion."));
 
+extern cl::opt<bool> ProfcheckDisableMetadataFixes;
+
 static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
 static bool isNotUsedOrFoldableInLoop(const Instruction &I, const Loop *CurLoop,
                                       const LoopSafetyInfo *SafetyInfo,
@@ -472,7 +474,7 @@ bool LoopInvariantCodeMotion::runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI,
   if (Preheader)
     Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, AC, TLI, L,
                            MSSAU, SE, &SafetyInfo, Flags, ORE, LoopNestMode,
-                           LicmAllowSpeculation);
+                           LicmAllowSpeculation, HasCoroSuspendInst);
 
   // Now that all loop invariants have been removed from the loop, promote any
   // memory references to scalars that we can.
@@ -857,9 +859,18 @@ class ControlFlowHoister {
     }
 
     // Now finally clone BI.
-    ReplaceInstWithInst(
-        HoistTarget->getTerminator(),
-        BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition()));
+    auto *NewBI =
+        BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition(),
+                           HoistTarget->getTerminator()->getIterator());
+    HoistTarget->getTerminator()->eraseFromParent();
+    // md_prof should also come from the original branch - since the
+    // condition was hoisted, the branch probabilities shouldn't change.
+    if (!ProfcheckDisableMetadataFixes)
+      NewBI->copyMetadata(*BI, {LLVMContext::MD_prof});
+    // FIXME: Issue #152767: debug info should also be the same as the
+    // original branch, **if** the user explicitly indicated that.
+    NewBI->setDebugLoc(HoistTarget->getTerminator()->getDebugLoc());
+
     ++NumClonedBranches;
 
     assert(CurLoop->getLoopPreheader() &&
@@ -881,7 +892,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                        ICFLoopSafetyInfo *SafetyInfo,
                        SinkAndHoistLICMFlags &Flags,
                        OptimizationRemarkEmitter *ORE, bool LoopNestMode,
-                       bool AllowSpeculation) {
+                       bool AllowSpeculation, bool HasCoroSuspendInst) {
   // Verify inputs.
   assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
          CurLoop != nullptr && SafetyInfo != nullptr &&
@@ -914,11 +925,11 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
       // TODO: It may be safe to hoist if we are hoisting to a conditional block
       // and we have accurately duplicated the control flow from the loop header
       // to that block.
-      if (CurLoop->hasLoopInvariantOperands(&I) &&
+      if (CurLoop->hasLoopInvariantOperands(&I, HasCoroSuspendInst) &&
           canSinkOrHoistInst(I, AA, DT, CurLoop, MSSAU, true, Flags, ORE) &&
-          isSafeToExecuteUnconditionally(
-              I, DT, TLI, CurLoop, SafetyInfo, ORE,
-              Preheader->getTerminator(), AC, AllowSpeculation)) {
+          isSafeToExecuteUnconditionally(I, DT, TLI, CurLoop, SafetyInfo, ORE,
+                                         Preheader->getTerminator(), AC,
+                                         AllowSpeculation)) {
         hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
               MSSAU, SE, ORE);
         HoistedInstructions.push_back(&I);
@@ -964,7 +975,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop);
       };
       if ((IsInvariantStart(I) || isGuard(&I)) &&
-          CurLoop->hasLoopInvariantOperands(&I) &&
+          CurLoop->hasLoopInvariantOperands(&I, HasCoroSuspendInst) &&
           MustExecuteWithoutWritesBefore(I)) {
         hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
               MSSAU, SE, ORE);
@@ -1230,11 +1241,16 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
     if (Behavior.doesNotAccessMemory())
       return true;
     if (Behavior.onlyReadsMemory()) {
+      // Might have stale MemoryDef for call that was later inferred to be
+      // read-only.
+      auto *MU = dyn_cast<MemoryUse>(MSSA->getMemoryAccess(CI));
+      if (!MU)
+        return false;
+
       // If we can prove there are no writes to the memory read by the call, we
       // can hoist or sink.
       return !pointerInvalidatedByLoop(
-          MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop, I, Flags,
-          /*InvariantGroup=*/false);
+          MSSA, MU, CurLoop, I, Flags, /*InvariantGroup=*/false);
     }
 
     if (Behavior.onlyWritesMemory()) {
@@ -1688,8 +1704,12 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
       // The check on hasMetadataOtherThanDebugLoc is to prevent us from burning
       // time in isGuaranteedToExecute if we don't actually have anything to
       // drop.  It is a compile time optimization, not required for correctness.
-      !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
-    I.dropUBImplyingAttrsAndMetadata();
+      !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop)) {
+    if (ProfcheckDisableMetadataFixes)
+      I.dropUBImplyingAttrsAndMetadata();
+    else
+      I.dropUBImplyingAttrsAndMetadata({LLVMContext::MD_prof});
+  }
 
   if (isa<PHINode>(I))
     // Move the new node to the end of the phi list in the destination block.
@@ -2856,7 +2876,7 @@ static bool hoistBOAssociation(Instruction &I, Loop &L,
   bool LVInRHS = L.isLoopInvariant(BO->getOperand(0));
   auto *BO0 = dyn_cast<BinaryOperator>(BO->getOperand(LVInRHS));
   if (!BO0 || BO0->getOpcode() != Opcode || !BO0->isAssociative() ||
-      BO0->hasNUsesOrMore(3))
+      BO0->hasNUsesOrMore(BO0->getType()->isIntegerTy() ? 2 : 3))
     return false;
 
   Value *LV = BO0->getOperand(0);
diff --git a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp
index 0ac1a15981770..1099aa335e4c5 100644
--- a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp
@@ -113,6 +113,8 @@ static cl::opt<bool> EnableLoopDistribute(
     cl::desc("Enable the new, experimental LoopDistribution Pass"),
     cl::init(false));
 
+static const char *DistributedMetaData = "llvm.loop.isdistributed";
+
 STATISTIC(NumLoopsDistributed, "Number of loops distributed");
 
 namespace {
@@ -502,8 +504,10 @@ class InstPartitionContainer {
     SmallVector<int, 8> PtrToPartitions(N);
     for (unsigned I = 0; I < N; ++I) {
       Value *Ptr = RtPtrCheck->Pointers[I].PointerValue;
-      auto Instructions =
-          LAI.getInstructionsForAccess(Ptr, RtPtrCheck->Pointers[I].IsWritePtr);
+      auto Instructions = LAI.getInstructionsForAccess(Ptr, /* IsWrite */ true);
+      auto ReadInstructions =
+          LAI.getInstructionsForAccess(Ptr, /* IsWrite */ false);
+      Instructions.append(ReadInstructions.begin(), ReadInstructions.end());
 
       int &Partition = PtrToPartitions[I];
       // First set it to uninitialized.
@@ -824,6 +828,8 @@ class LoopDistributeForLoop {
           {LLVMLoopDistributeFollowupAll, LLVMLoopDistributeFollowupFallback},
           "llvm.loop.distribute.", true);
       LVer.getNonVersionedLoop()->setLoopID(UnversionedLoopID);
+      addStringMetadataToLoop(LVer.getNonVersionedLoop(), DistributedMetaData,
+                              true);
     }
 
     // Create identical copies of the original loop for each partition and hook
@@ -984,6 +990,13 @@ static bool runImpl(Function &F, LoopInfo *LI, DominatorTree *DT,
   for (Loop *L : Worklist) {
     LoopDistributeForLoop LDL(L, &F, LI, DT, SE, LAIs, ORE);
 
+    // Do not reprocess loops we already distributed
+    if (getOptionalBoolLoopAttribute(L, DistributedMetaData).value_or(false)) {
+      LLVM_DEBUG(
+          dbgs() << "LDist: Distributed loop guarded for reprocessing\n");
+      continue;
+    }
+
     // If distribution was forced for the specific loop to be
     // enabled/disabled, follow that.  Otherwise use the global flag.
     if (LDL.isForced().value_or(EnableLoopDistribute))
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index b3bffeb7ea412..56e0569831e83 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -21,7 +21,6 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
@@ -263,6 +262,7 @@ static bool isUniformShape(Value *V) {
     case llvm::Instruction::FPExt:
       return true;
     case llvm::Instruction::AddrSpaceCast:
+    case CastInst::PtrToAddr:
     case CastInst::PtrToInt:
     case CastInst::IntToPtr:
       return false;
@@ -1208,7 +1208,7 @@ class LowerMatrixIntrinsics {
     //
     // For verification, we keep track of where we changed uses to poison in
     // PoisonedInsts and then check that we in fact remove them.
-    SmallSet<Instruction *, 16> PoisonedInsts;
+    SmallPtrSet<Instruction *, 16> PoisonedInsts;
     for (auto *Inst : reverse(ToRemove)) {
       for (Use &U : llvm::make_early_inc_range(Inst->uses())) {
         if (auto *Poisoned = dyn_cast<Instruction>(U.getUser()))
@@ -2166,7 +2166,7 @@ class LowerMatrixIntrinsics {
 
         // If the loads don't alias the lifetime.end, it won't interfere with
         // fusion.
-        MemoryLocation EndLoc = MemoryLocation::getForArgument(End, 1, nullptr);
+        MemoryLocation EndLoc = MemoryLocation::getForArgument(End, 0, nullptr);
         if (!EndLoc.Ptr)
           continue;
         if (AA->isNoAlias(Load0Loc, EndLoc) && AA->isNoAlias(Load1Loc, EndLoc))
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 79721dc5f39f0..e043d072a7638 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -915,7 +915,7 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpyLoad,
   // move the bitcast as well, which we don't handle.
   if (SkippedLifetimeStart) {
     auto *LifetimeArg =
-        dyn_cast<Instruction>(SkippedLifetimeStart->getOperand(1));
+        dyn_cast<Instruction>(SkippedLifetimeStart->getOperand(0));
     if (LifetimeArg && LifetimeArg->getParent() == C->getParent() &&
         C->comesBefore(LifetimeArg))
       return false;
@@ -1010,7 +1010,7 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpyLoad,
       // Lifetime of srcAlloca ends at lifetime.end.
       if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
         if (II->getIntrinsicID() == Intrinsic::lifetime_end &&
-            II->getArgOperand(1) == srcAlloca)
+            II->getArgOperand(0) == srcAlloca)
           break;
       }
 
@@ -1393,7 +1393,7 @@ static bool hasUndefContents(MemorySSA *MSSA, BatchAAResults &AA, Value *V,
   if (auto *II = dyn_cast_or_null<IntrinsicInst>(Def->getMemoryInst()))
     if (II->getIntrinsicID() == Intrinsic::lifetime_start)
       if (auto *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(V)))
-        return II->getArgOperand(1) == Alloca;
+        return II->getArgOperand(0) == Alloca;
 
   return false;
 }
@@ -1530,7 +1530,7 @@ bool MemCpyOptPass::performStackMoveOptzn(Instruction *Load, Instruction *Store,
   // to remove them.
 
   SmallVector<Instruction *, 4> LifetimeMarkers;
-  SmallSet<Instruction *, 4> AAMetadataInstrs;
+  SmallPtrSet<Instruction *, 4> AAMetadataInstrs;
   bool SrcNotDom = false;
 
   auto CaptureTrackingWithModRef =
@@ -1540,7 +1540,7 @@ bool MemCpyOptPass::performStackMoveOptzn(Instruction *Load, Instruction *Store,
     Worklist.push_back(AI);
     unsigned MaxUsesToExplore = getDefaultMaxUsesToExploreForCaptureTracking();
     Worklist.reserve(MaxUsesToExplore);
-    SmallSet<const Use *, 20> Visited;
+    SmallPtrSet<const Use *, 20> Visited;
     while (!Worklist.empty()) {
       Instruction *I = Worklist.pop_back_val();
       for (const Use &U : I->uses()) {
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index 1a52af1045d5c..9d4fb79416596 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -651,7 +651,7 @@ class NewGVN {
   BitVector TouchedInstructions;
 
   DenseMap<const BasicBlock *, std::pair<unsigned, unsigned>> BlockInstRange;
-  mutable DenseMap<const IntrinsicInst *, const Value *> PredicateSwapChoice;
+  mutable DenseMap<const BitCastInst *, const Value *> PredicateSwapChoice;
 
 #ifndef NDEBUG
   // Debugging for how many times each block and instruction got processed.
@@ -819,7 +819,7 @@ class NewGVN {
                                                  BasicBlock *PHIBlock) const;
   const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;
   ExprResult performSymbolicCmpEvaluation(Instruction *) const;
-  ExprResult performSymbolicPredicateInfoEvaluation(IntrinsicInst *) const;
+  ExprResult performSymbolicPredicateInfoEvaluation(BitCastInst *) const;
 
   // Congruence finding.
   bool someEquivalentDominates(const Instruction *, const Instruction *) const;
@@ -841,7 +841,7 @@ class NewGVN {
   unsigned int getRank(const Value *) const;
   bool shouldSwapOperands(const Value *, const Value *) const;
   bool shouldSwapOperandsForPredicate(const Value *, const Value *,
-                                      const IntrinsicInst *I) const;
+                                      const BitCastInst *I) const;
 
   // Reachability handling.
   void updateReachableEdge(BasicBlock *, BasicBlock *);
@@ -1013,9 +1013,9 @@ void NewGVN::deleteExpression(const Expression *E) const {
 
 // If V is a predicateinfo copy, get the thing it is a copy of.
 static Value *getCopyOf(const Value *V) {
-  if (auto *II = dyn_cast<IntrinsicInst>(V))
-    if (II->getIntrinsicID() == Intrinsic::ssa_copy)
-      return II->getOperand(0);
+  if (auto *BC = dyn_cast<BitCastInst>(V))
+    if (BC->getType() == BC->getOperand(0)->getType())
+      return BC->getOperand(0);
   return nullptr;
 }
 
@@ -1535,7 +1535,7 @@ NewGVN::performSymbolicLoadCoercion(Type *LoadType, Value *LoadPtr,
 
   if (auto *II = dyn_cast<IntrinsicInst>(DepInst)) {
     if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
-      auto *LifetimePtr = II->getOperand(1);
+      auto *LifetimePtr = II->getOperand(0);
       if (LoadPtr == lookupOperandLeader(LifetimePtr) ||
           AA->isMustAlias(LoadPtr, LifetimePtr))
         return createConstantExpression(UndefValue::get(LoadType));
@@ -1604,7 +1604,7 @@ const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
 }
 
 NewGVN::ExprResult
-NewGVN::performSymbolicPredicateInfoEvaluation(IntrinsicInst *I) const {
+NewGVN::performSymbolicPredicateInfoEvaluation(BitCastInst *I) const {
   auto *PI = PredInfo->getPredicateInfoFor(I);
   if (!PI)
     return ExprResult::none();
@@ -1647,13 +1647,8 @@ NewGVN::performSymbolicPredicateInfoEvaluation(IntrinsicInst *I) const {
 NewGVN::ExprResult NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
   auto *CI = cast<CallInst>(I);
   if (auto *II = dyn_cast<IntrinsicInst>(I)) {
-    // Intrinsics with the returned attribute are copies of arguments.
-    if (auto *ReturnedValue = II->getReturnedArgOperand()) {
-      if (II->getIntrinsicID() == Intrinsic::ssa_copy)
-        if (auto Res = performSymbolicPredicateInfoEvaluation(II))
-          return Res;
+    if (auto *ReturnedValue = II->getReturnedArgOperand())
       return ExprResult::some(createVariableOrConstant(ReturnedValue));
-    }
   }
 
   // FIXME: Currently the calls which may access the thread id may
@@ -2032,6 +2027,12 @@ NewGVN::performSymbolicEvaluation(Instruction *I,
     E = performSymbolicLoadEvaluation(I);
     break;
   case Instruction::BitCast:
+    // Intrinsics with the returned attribute are copies of arguments.
+    if (I->getType() == I->getOperand(0)->getType())
+      if (auto Res =
+              performSymbolicPredicateInfoEvaluation(cast<BitCastInst>(I)))
+        return Res;
+    [[fallthrough]];
   case Instruction::AddrSpaceCast:
   case Instruction::Freeze:
     return createExpression(I);
@@ -4075,8 +4076,7 @@ bool NewGVN::eliminateInstructions(Function &F) {
               if (DominatingLeader != Def) {
                 // Even if the instruction is removed, we still need to update
                 // flags/metadata due to downstreams users of the leader.
-                if (!match(DefI, m_Intrinsic<Intrinsic::ssa_copy>()))
-                  patchReplacementInstruction(DefI, DominatingLeader);
+                patchReplacementInstruction(DefI, DominatingLeader);
 
                 SmallVector<DbgVariableRecord *> DVRUsers;
                 findDbgUsers(DefI, DVRUsers);
@@ -4116,10 +4116,14 @@ bool NewGVN::eliminateInstructions(Function &F) {
 
           Value *DominatingLeader = EliminationStack.back();
 
-          auto *II = dyn_cast<IntrinsicInst>(DominatingLeader);
-          bool isSSACopy = II && II->getIntrinsicID() == Intrinsic::ssa_copy;
-          if (isSSACopy)
-            DominatingLeader = II->getOperand(0);
+          Instruction *SSACopy = nullptr;
+          if (auto *BC = dyn_cast<BitCastInst>(DominatingLeader)) {
+            if (BC->getType() == BC->getOperand(0)->getType() &&
+                PredInfo->getPredicateInfoFor(DominatingLeader)) {
+              SSACopy = BC;
+              DominatingLeader = BC->getOperand(0);
+            }
+          }
 
           // Don't replace our existing users with ourselves.
           if (U->get() == DominatingLeader)
@@ -4145,12 +4149,12 @@ bool NewGVN::eliminateInstructions(Function &F) {
             ProbablyDead.erase(cast<Instruction>(DominatingLeader));
           // For copy instructions, we use their operand as a leader,
           // which means we remove a user of the copy and it may become dead.
-          if (isSSACopy) {
-            auto It = UseCounts.find(II);
+          if (SSACopy) {
+            auto It = UseCounts.find(SSACopy);
             if (It != UseCounts.end()) {
               unsigned &IIUseCount = It->second;
               if (--IIUseCount == 0)
-                ProbablyDead.insert(II);
+                ProbablyDead.insert(SSACopy);
             }
           }
           ++LeaderUseCount;
@@ -4251,7 +4255,7 @@ bool NewGVN::shouldSwapOperands(const Value *A, const Value *B) const {
 }
 
 bool NewGVN::shouldSwapOperandsForPredicate(const Value *A, const Value *B,
-                                            const IntrinsicInst *I) const {
+                                            const BitCastInst *I) const {
   if (shouldSwapOperands(A, B)) {
     PredicateSwapChoice[I] = B;
     return true;
diff --git a/llvm/lib/Transforms/Scalar/Reassociate.cpp b/llvm/lib/Transforms/Scalar/Reassociate.cpp
index 343da5b2e4704..ba58b8e4eda5e 100644
--- a/llvm/lib/Transforms/Scalar/Reassociate.cpp
+++ b/llvm/lib/Transforms/Scalar/Reassociate.cpp
@@ -878,7 +878,7 @@ static Value *NegateValue(Value *V, Instruction *BI,
 // only that it mostly looks like one.
 static bool isLoadCombineCandidate(Instruction *Or) {
   SmallVector<Instruction *, 8> Worklist;
-  SmallSet<Instruction *, 8> Visited;
+  SmallPtrSet<Instruction *, 8> Visited;
 
   auto Enqueue = [&](Value *V) {
     auto *I = dyn_cast<Instruction>(V);
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index d9805d8b3ea59..74448d872aad2 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -20,7 +20,6 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
@@ -2214,14 +2213,6 @@ static void relocationViaAlloca(
 #endif
 }
 
-/// Implement a unique function which doesn't require we sort the input
-/// vector.  Doing so has the effect of changing the output of a couple of
-/// tests in ways which make them less useful in testing fused safepoints.
-template <typename T> static void unique_unsorted(SmallVectorImpl<T> &Vec) {
-  SmallSet<T, 8> Seen;
-  erase_if(Vec, [&](const T &V) { return !Seen.insert(V).second; });
-}
-
 /// Insert holders so that each Value is obviously live through the entire
 /// lifetime of the call.
 static void insertUseHolderAfter(CallBase *Call, const ArrayRef<Value *> Values,
@@ -2309,8 +2300,9 @@ chainToBasePointerCost(SmallVectorImpl<Instruction *> &Chain,
 
     } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Instr)) {
       // Cost of the address calculation
-      Type *ValTy = GEP->getSourceElementType();
-      Cost += TTI.getAddressComputationCost(ValTy);
+      Cost += TTI.getAddressComputationCost(
+          GEP->getType(), nullptr, nullptr,
+          TargetTransformInfo::TCK_SizeAndLatency);
 
       // And cost of the GEP itself
       // TODO: Use TTI->getGEPCost here (it exists, but appears to be not
@@ -2838,15 +2830,17 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
   }
   PointerToBase.clear();
 
-  // Do all the fixups of the original live variables to their relocated selves
-  SmallVector<Value *, 128> Live;
+  // Do all the fixups of the original live variables to their relocated selves.
+  // A SmallSetVector is used to collect live variables while retaining the
+  // order in which we add them, which is important for reproducible tests.
+  SmallSetVector<Value *, 16> Live;
   for (const PartiallyConstructedSafepointRecord &Info : Records) {
     // We can't simply save the live set from the original insertion.  One of
     // the live values might be the result of a call which needs a safepoint.
     // That Value* no longer exists and we need to use the new gc_result.
     // Thankfully, the live set is embedded in the statepoint (and updated), so
     // we just grab that.
-    llvm::append_range(Live, Info.StatepointToken->gc_live());
+    Live.insert_range(Info.StatepointToken->gc_live());
 #ifndef NDEBUG
     // Do some basic validation checking on our liveness results before
     // performing relocation.  Relocation can and will turn mistakes in liveness
@@ -2866,7 +2860,6 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
     }
 #endif
   }
-  unique_unsorted(Live);
 
 #ifndef NDEBUG
   // Validation check
@@ -2875,7 +2868,7 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
            "must be a gc pointer type");
 #endif
 
-  relocationViaAlloca(F, DT, Live, Records);
+  relocationViaAlloca(F, DT, Live.getArrayRef(), Records);
   return !Records.empty();
 }
 
diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index 8be2f78187a0c..feee794ffeae1 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -20,6 +20,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
@@ -31,6 +32,7 @@
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
@@ -53,12 +55,15 @@ STATISTIC(NumInstReplaced,
 // runSCCP() - Run the Sparse Conditional Constant Propagation algorithm,
 // and return true if the function was modified.
 static bool runSCCP(Function &F, const DataLayout &DL,
-                    const TargetLibraryInfo *TLI, DomTreeUpdater &DTU) {
+                    const TargetLibraryInfo *TLI, DominatorTree &DT,
+                    AssumptionCache &AC) {
   LLVM_DEBUG(dbgs() << "SCCP on function '" << F.getName() << "'\n");
   SCCPSolver Solver(
       DL, [TLI](Function &F) -> const TargetLibraryInfo & { return *TLI; },
       F.getContext());
 
+  Solver.addPredicateInfo(F, DT, AC);
+
   // While we don't do any actual inter-procedural analysis, still track
   // return values so we can infer attributes.
   if (canTrackReturnsInterprocedurally(&F))
@@ -101,6 +106,7 @@ static bool runSCCP(Function &F, const DataLayout &DL,
   }
 
   // Remove unreachable blocks and non-feasible edges.
+  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
   for (BasicBlock *DeadBB : BlocksToErase)
     NumInstRemoved += changeToUnreachable(&*DeadBB->getFirstNonPHIIt(),
                                           /*PreserveLCSSA=*/false, &DTU);
@@ -113,6 +119,8 @@ static bool runSCCP(Function &F, const DataLayout &DL,
     if (!DeadBB->hasAddressTaken())
       DTU.deleteBB(DeadBB);
 
+  Solver.removeSSACopies(F);
+
   Solver.inferReturnAttributes();
 
   return MadeChanges;
@@ -121,9 +129,9 @@ static bool runSCCP(Function &F, const DataLayout &DL,
 PreservedAnalyses SCCPPass::run(Function &F, FunctionAnalysisManager &AM) {
   const DataLayout &DL = F.getDataLayout();
   auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
-  auto *DT = AM.getCachedResult<DominatorTreeAnalysis>(F);
-  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
-  if (!runSCCP(F, DL, &TLI, DTU))
+  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  auto &AC = AM.getResult<AssumptionAnalysis>(F);
+  if (!runSCCP(F, DL, &TLI, DT, AC))
     return PreservedAnalyses::all();
 
   auto PA = PreservedAnalyses();
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 03d9f32aa3f03..9b4f1dc6ddb34 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -320,15 +320,6 @@ static DebugVariable getAggregateVariable(DbgVariableRecord *DVR) {
                        DVR->getDebugLoc().getInlinedAt());
 }
 
-DbgVariableRecord *UnwrapDbgInstPtr(DbgInstPtr P, DbgVariableRecord *Unused) {
-  (void)Unused;
-  return static_cast<DbgVariableRecord *>(cast<DbgRecord *>(P));
-}
-DbgAssignIntrinsic *UnwrapDbgInstPtr(DbgInstPtr P, DbgAssignIntrinsic *Unused) {
-  (void)Unused;
-  return static_cast<DbgAssignIntrinsic *>(cast<Instruction *>(P));
-}
-
 /// Find linked dbg.assign and generate a new one with the correct
 /// FragmentInfo. Link Inst to the new dbg.assign.  If Value is nullptr the
 /// value component is copied from the old dbg.assign to the new.
@@ -348,10 +339,9 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
                              uint64_t SliceSizeInBits, Instruction *OldInst,
                              Instruction *Inst, Value *Dest, Value *Value,
                              const DataLayout &DL) {
-  auto MarkerRange = at::getAssignmentMarkers(OldInst);
   auto DVRAssignMarkerRange = at::getDVRAssignmentMarkers(OldInst);
   // Nothing to do if OldInst has no linked dbg.assign intrinsics.
-  if (MarkerRange.empty() && DVRAssignMarkerRange.empty())
+  if (DVRAssignMarkerRange.empty())
     return;
 
   LLVM_DEBUG(dbgs() << "  migrateDebugInfo\n");
@@ -435,11 +425,10 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
     }
 
     ::Value *NewValue = Value ? Value : DbgAssign->getValue();
-    auto *NewAssign = UnwrapDbgInstPtr(
+    DbgVariableRecord *NewAssign = cast<DbgVariableRecord>(cast<DbgRecord *>(
         DIB.insertDbgAssign(Inst, NewValue, DbgAssign->getVariable(), Expr,
                             Dest, DIExpression::get(Expr->getContext(), {}),
-                            DbgAssign->getDebugLoc()),
-        DbgAssign);
+                            DbgAssign->getDebugLoc())));
 
     // If we've updated the value but the original dbg.assign has an arglist
     // then kill it now - we can't use the requested new value.
@@ -1260,10 +1249,7 @@ class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
       return PI.setAborted(&II);
 
     if (II.isLifetimeStartOrEnd()) {
-      ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0));
-      uint64_t Size = std::min(AllocSize - Offset.getLimitedValue(),
-                               Length->getLimitedValue());
-      insertUse(II, Offset, Size, true);
+      insertUse(II, Offset, AllocSize, true);
       return;
     }
 
@@ -3235,8 +3221,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
       // In theory we should call migrateDebugInfo here. However, we do not
       // emit dbg.assign intrinsics for mem intrinsics storing through non-
       // constant geps, or storing a variable number of bytes.
-      assert(at::getAssignmentMarkers(&II).empty() &&
-             at::getDVRAssignmentMarkers(&II).empty() &&
+      assert(at::getDVRAssignmentMarkers(&II).empty() &&
              "AT: Unexpected link to non-const GEP");
       deleteIfTriviallyDead(OldPtr);
       return false;
@@ -3385,13 +3370,11 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
       Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
       if (IsDest) {
         // Update the address component of linked dbg.assigns.
-        auto UpdateAssignAddress = [&](auto *DbgAssign) {
+        for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(&II)) {
           if (llvm::is_contained(DbgAssign->location_ops(), II.getDest()) ||
               DbgAssign->getAddress() == II.getDest())
             DbgAssign->replaceVariableLocationOp(II.getDest(), AdjustedPtr);
-        };
-        for_each(at::getAssignmentMarkers(&II), UpdateAssignAddress);
-        for_each(at::getDVRAssignmentMarkers(&II), UpdateAssignAddress);
+        }
         II.setDest(AdjustedPtr);
         II.setDestAlignment(SliceAlign);
       } else {
@@ -3428,8 +3411,7 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
 
       // Rewrite the size as needed.
       if (NewEndOffset != EndOffset)
-        II.setLength(ConstantInt::get(II.getLength()->getType(),
-                                      NewEndOffset - NewBeginOffset));
+        II.setLength(NewEndOffset - NewBeginOffset);
       return false;
     }
     // Record this instruction for deletion.
@@ -3614,30 +3596,14 @@ class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
       return true;
     }
 
-    assert(II.getArgOperand(1) == OldPtr);
-    // Lifetime intrinsics are only promotable if they cover the whole alloca.
-    // Therefore, we drop lifetime intrinsics which don't cover the whole
-    // alloca.
-    // (In theory, intrinsics which partially cover an alloca could be
-    // promoted, but PromoteMemToReg doesn't handle that case.)
-    // FIXME: Check whether the alloca is promotable before dropping the
-    // lifetime intrinsics?
-    if (NewBeginOffset != NewAllocaBeginOffset ||
-        NewEndOffset != NewAllocaEndOffset)
-      return true;
-
-    ConstantInt *Size =
-        ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()),
-                         NewEndOffset - NewBeginOffset);
-    // Lifetime intrinsics always expect an i8* so directly get such a pointer
-    // for the new alloca slice.
+    assert(II.getArgOperand(0) == OldPtr);
     Type *PointerTy = IRB.getPtrTy(OldPtr->getType()->getPointerAddressSpace());
     Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
     Value *New;
     if (II.getIntrinsicID() == Intrinsic::lifetime_start)
-      New = IRB.CreateLifetimeStart(Ptr, Size);
+      New = IRB.CreateLifetimeStart(Ptr);
     else
-      New = IRB.CreateLifetimeEnd(Ptr, Size);
+      New = IRB.CreateLifetimeEnd(Ptr);
 
     (void)New;
     LLVM_DEBUG(dbgs() << "          to: " << *New << "\n");
@@ -4005,8 +3971,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
                          Store->getPointerOperand(), Store->getValueOperand(),
                          DL);
       } else {
-        assert(at::getAssignmentMarkers(Store).empty() &&
-               at::getDVRAssignmentMarkers(Store).empty() &&
+        assert(at::getDVRAssignmentMarkers(Store).empty() &&
                "AT: unexpected debug.assign linked to store through "
                "unbounded GEP");
       }
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index c7e4a3e824700..032a3a7792824 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -37,6 +37,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeMergeICmpsLegacyPassPass(Registry);
   initializeNaryReassociateLegacyPassPass(Registry);
   initializePartiallyInlineLibCallsLegacyPassPass(Registry);
+  initializeDSELegacyPassPass(Registry);
   initializeReassociateLegacyPassPass(Registry);
   initializeScalarizeMaskedMemIntrinLegacyPassPass(Registry);
   initializeSROALegacyPassPass(Registry);
diff --git a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 6ffe8413a5079..8b9d06d7e4439 100644
--- a/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/llvm/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -294,6 +294,10 @@ class ConstantOffsetExtractor {
   bool CanTraceInto(bool SignExtended, bool ZeroExtended, BinaryOperator *BO,
                     bool NonNegative);
 
+  /// Analyze XOR instruction to extract disjoint constant bits that behave
+  /// like addition operations for improved address mode folding.
+  APInt extractDisjointBitsFromXor(BinaryOperator *XorInst);
+
   /// The path from the constant offset to the old GEP index. e.g., if the GEP
   /// index is "a * b + (c + 5)". After running function find, UserChain[0] will
   /// be the constant 5, UserChain[1] will be the subexpression "c + 5", and
@@ -596,6 +600,9 @@ APInt ConstantOffsetExtractor::find(Value *V, bool SignExtended,
     // Trace into subexpressions for more hoisting opportunities.
     if (CanTraceInto(SignExtended, ZeroExtended, BO, NonNegative))
       ConstantOffset = findInEitherOperand(BO, SignExtended, ZeroExtended);
+    // Handle XOR with disjoint bits that can be treated as addition.
+    else if (BO->getOpcode() == Instruction::Xor)
+      ConstantOffset = extractDisjointBitsFromXor(BO);
   } else if (isa<TruncInst>(V)) {
     ConstantOffset =
         find(U->getOperand(0), SignExtended, ZeroExtended, NonNegative)
@@ -635,6 +642,13 @@ Value *ConstantOffsetExtractor::applyExts(Value *V) {
 
     Instruction *Ext = I->clone();
     Ext->setOperand(0, Current);
+    // In ConstantOffsetExtractor::find we do not analyze nuw/nsw for trunc, so
+    // we assume that it is ok to redistribute trunc over add/sub/or. But for
+    // example (add (trunc nuw A), (trunc nuw B)) is more poisonous than (trunc
+    // nuw (add A, B))). To make such redistributions legal we drop all the
+    // poison generating flags from cloned trunc instructions here.
+    if (isa<TruncInst>(Ext))
+      Ext->dropPoisonGeneratingFlags();
     Ext->insertBefore(*IP->getParent(), IP);
     Current = Ext;
   }
@@ -708,11 +722,20 @@ Value *ConstantOffsetExtractor::removeConstOffset(unsigned ChainIndex) {
   Value *NextInChain = removeConstOffset(ChainIndex - 1);
   Value *TheOther = BO->getOperand(1 - OpNo);
 
-  // If NextInChain is 0 and not the LHS of a sub, we can simplify the
-  // sub-expression to be just TheOther.
   if (ConstantInt *CI = dyn_cast<ConstantInt>(NextInChain)) {
-    if (CI->isZero() && !(BO->getOpcode() == Instruction::Sub && OpNo == 0))
-      return TheOther;
+    if (CI->isZero()) {
+      // Custom XOR handling for disjoint bits - preserves original XOR
+      // with non-disjoint constant bits.
+      // TODO: The design should be updated to support partial constant
+      // extraction.
+      if (BO->getOpcode() == Instruction::Xor)
+        return BO;
+
+      // If NextInChain is 0 and not the LHS of a sub, we can simplify the
+      // sub-expression to be just TheOther.
+      if (!(BO->getOpcode() == Instruction::Sub && OpNo == 0))
+        return TheOther;
+    }
   }
 
   BinaryOperator::BinaryOps NewOp = BO->getOpcode();
@@ -743,6 +766,67 @@ Value *ConstantOffsetExtractor::removeConstOffset(unsigned ChainIndex) {
   return NewBO;
 }
 
+/// Analyze XOR instruction to extract disjoint constant bits for address
+/// folding
+///
+/// This function identifies bits in an XOR constant operand that are disjoint
+/// from the base operand's known set bits. For these disjoint bits, XOR behaves
+/// identically to addition, allowing us to extract them as constant offsets
+/// that can be folded into addressing modes.
+///
+/// Transformation: `Base ^ Const` becomes `(Base ^ NonDisjointBits) +
+/// DisjointBits` where DisjointBits = Const & KnownZeros(Base)
+///
+/// Example with ptr having known-zero low bit:
+///   Original: `xor %ptr, 3`    ; 3 = 0b11
+///   Analysis: DisjointBits = 3 & KnownZeros(%ptr) = 0b11 & 0b01 = 0b01
+///   Result:   `(xor %ptr, 2) + 1` where 1 can be folded into address mode
+///
+/// \param XorInst The XOR binary operator to analyze
+/// \return APInt containing the disjoint bits that can be extracted as offset,
+///         or zero if no disjoint bits exist
+APInt ConstantOffsetExtractor::extractDisjointBitsFromXor(
+    BinaryOperator *XorInst) {
+  assert(XorInst && XorInst->getOpcode() == Instruction::Xor &&
+         "Expected XOR instruction");
+
+  const unsigned BitWidth = XorInst->getType()->getScalarSizeInBits();
+  Value *BaseOperand;
+  ConstantInt *XorConstant;
+
+  // Match pattern: xor BaseOperand, Constant.
+  if (!match(XorInst, m_Xor(m_Value(BaseOperand), m_ConstantInt(XorConstant))))
+    return APInt::getZero(BitWidth);
+
+  // Compute known bits for the base operand.
+  const SimplifyQuery SQ(DL);
+  const KnownBits BaseKnownBits = computeKnownBits(BaseOperand, SQ);
+  const APInt &ConstantValue = XorConstant->getValue();
+
+  // Identify disjoint bits: constant bits that are known zero in base.
+  const APInt DisjointBits = ConstantValue & BaseKnownBits.Zero;
+
+  // Early exit if no disjoint bits found.
+  if (DisjointBits.isZero())
+    return APInt::getZero(BitWidth);
+
+  // Compute the remaining non-disjoint bits that stay in the XOR.
+  const APInt NonDisjointBits = ConstantValue & ~DisjointBits;
+
+  // FIXME: Enhance XOR constant extraction to handle nested binary operations.
+  // Currently we only extract disjoint bits from the immediate XOR constant,
+  // but we could recursively process cases like:
+  //   xor (add %base, C1), C2  ->  add %base, (C1 ^ disjoint_bits(C2))
+  // This requires careful analysis to ensure the transformation preserves
+  // semantics, particularly around sign extension and overflow behavior.
+
+  // Add the non-disjoint constant to the user chain for later transformation
+  // This will replace the original constant in the XOR with the new
+  // constant.
+  UserChain.push_back(ConstantInt::get(XorInst->getType(), NonDisjointBits));
+  return DisjointBits;
+}
+
 /// A helper function to check if reassociating through an entry in the user
 /// chain would invalidate the GEP's nuw flag.
 static bool allowsPreservingNUW(const User *U) {
@@ -962,19 +1046,31 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
   if (GEP->getType()->isVectorTy())
     return false;
 
+  // If the base of this GEP is a ptradd of a constant, lets pass the constant
+  // along. This ensures that when we have a chain of GEPs the constant
+  // offset from each is accumulated.
+  Value *NewBase;
+  const APInt *BaseOffset;
+  const bool ExtractBase =
+      match(GEP->getPointerOperand(),
+            m_PtrAdd(m_Value(NewBase), m_APInt(BaseOffset)));
+
+  const int64_t BaseByteOffset = ExtractBase ? BaseOffset->getSExtValue() : 0;
+
   // The backend can already nicely handle the case where all indices are
   // constant.
-  if (GEP->hasAllConstantIndices())
+  if (GEP->hasAllConstantIndices() && !ExtractBase)
     return false;
 
   bool Changed = canonicalizeArrayIndicesToIndexSize(GEP);
 
   bool NeedsExtraction;
-  int64_t AccumulativeByteOffset = accumulateByteOffset(GEP, NeedsExtraction);
+  int64_t AccumulativeByteOffset =
+      BaseByteOffset + accumulateByteOffset(GEP, NeedsExtraction);
 
   TargetTransformInfo &TTI = GetTTI(*GEP->getFunction());
 
-  if (!NeedsExtraction) {
+  if (!NeedsExtraction && !ExtractBase) {
     Changed |= reorderGEP(GEP, TTI);
     return Changed;
   }
@@ -998,7 +1094,9 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
 
   // Track information for preserving GEP flags.
   bool AllOffsetsNonNegative = AccumulativeByteOffset >= 0;
-  bool AllNUWPreserved = true;
+  bool AllNUWPreserved = GEP->hasNoUnsignedWrap();
+  bool NewGEPInBounds = GEP->isInBounds();
+  bool NewGEPNUSW = GEP->hasNoUnsignedSignedWrap();
 
   // Remove the constant offset in each sequential index. The resultant GEP
   // computes the variadic base.
@@ -1034,6 +1132,16 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
       }
     }
   }
+  if (ExtractBase) {
+    GEPOperator *Base = cast<GEPOperator>(GEP->getPointerOperand());
+    AllNUWPreserved &= Base->hasNoUnsignedWrap();
+    NewGEPInBounds &= Base->isInBounds();
+    NewGEPNUSW &= Base->hasNoUnsignedSignedWrap();
+    AllOffsetsNonNegative &= BaseByteOffset >= 0;
+
+    GEP->setOperand(0, NewBase);
+    RecursivelyDeleteTriviallyDeadInstructions(Base);
+  }
 
   // Clear the inbounds attribute because the new index may be off-bound.
   // e.g.,
@@ -1061,7 +1169,7 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
 
   // If the initial GEP was NUW and all operations that we reassociate were NUW
   // additions, the resulting GEPs are also NUW.
-  if (GEP->hasNoUnsignedWrap() && AllNUWPreserved) {
+  if (AllNUWPreserved) {
     NewGEPFlags |= GEPNoWrapFlags::noUnsignedWrap();
     // If the initial GEP additionally had NUSW (or inbounds, which implies
     // NUSW), we know that the indices in the initial GEP must all have their
@@ -1069,13 +1177,13 @@ bool SeparateConstOffsetFromGEP::splitGEP(GetElementPtrInst *GEP) {
     // add-operands therefore also don't have their signbit set. Therefore, all
     // indices of the resulting GEPs are non-negative -> we can preserve
     // the inbounds/nusw flag.
-    CanPreserveInBoundsNUSW |= GEP->hasNoUnsignedSignedWrap();
+    CanPreserveInBoundsNUSW |= NewGEPNUSW;
   }
 
   if (CanPreserveInBoundsNUSW) {
-    if (GEP->isInBounds())
+    if (NewGEPInBounds)
       NewGEPFlags |= GEPNoWrapFlags::inBounds();
-    else if (GEP->hasNoUnsignedSignedWrap())
+    else if (NewGEPNUSW)
       NewGEPFlags |= GEPNoWrapFlags::noUnsignedSignedWrap();
   }
 
@@ -1143,11 +1251,13 @@ bool SeparateConstOffsetFromGEP::run(Function &F) {
 
   DL = &F.getDataLayout();
   bool Changed = false;
-  for (BasicBlock &B : F) {
-    if (!DT->isReachableFromEntry(&B))
+
+  ReversePostOrderTraversal<Function *> RPOT(&F);
+  for (BasicBlock *B : RPOT) {
+    if (!DT->isReachableFromEntry(B))
       continue;
 
-    for (Instruction &I : llvm::make_early_inc_range(B))
+    for (Instruction &I : llvm::make_early_inc_range(*B))
       if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(&I))
         Changed |= splitGEP(GEP);
     // No need to split GEP ConstantExprs because all its indices are constant
diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index f6959ca209fd7..9b40fc03da6bb 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -2144,23 +2144,9 @@ void visitDomSubTree(DominatorTree &DT, BasicBlock *BB, CallableT Callable) {
 void postUnswitch(Loop &L, LPMUpdater &U, StringRef LoopName,
                   bool CurrentLoopValid, bool PartiallyInvariant,
                   bool InjectedCondition, ArrayRef<Loop *> NewLoops) {
-  auto RecordLoopAsUnswitched = [&](Loop *TargetLoop, StringRef Tag,
-                                    StringRef DisableTag) {
-    auto &Ctx = TargetLoop->getHeader()->getContext();
-    MDNode *DisableMD = MDNode::get(Ctx, MDString::get(Ctx, DisableTag));
-    MDNode *NewLoopID = makePostTransformationMetadata(
-        Ctx, TargetLoop->getLoopID(), {Tag}, {DisableMD});
-    TargetLoop->setLoopID(NewLoopID);
-  };
-
-  // If we performed a non-trivial unswitch, we have added new cloned loops.
-  // Mark such newly-created loops as visited.
-  if (!NewLoops.empty()) {
-    for (Loop *NL : NewLoops)
-      RecordLoopAsUnswitched(NL, "llvm.loop.unswitch.nontrivial",
-                             "llvm.loop.unswitch.nontrivial.disable");
+  // If we did a non-trivial unswitch, we have added new (cloned) loops.
+  if (!NewLoops.empty())
     U.addSiblingLoops(NewLoops);
-  }
 
   // If the current loop remains valid, we should revisit it to catch any
   // other unswitch opportunities. Otherwise, we need to mark it as deleted.
@@ -2168,12 +2154,24 @@ void postUnswitch(Loop &L, LPMUpdater &U, StringRef LoopName,
     if (PartiallyInvariant) {
       // Mark the new loop as partially unswitched, to avoid unswitching on
       // the same condition again.
-      RecordLoopAsUnswitched(&L, "llvm.loop.unswitch.partial",
-                             "llvm.loop.unswitch.partial.disable");
+      auto &Context = L.getHeader()->getContext();
+      MDNode *DisableUnswitchMD = MDNode::get(
+          Context,
+          MDString::get(Context, "llvm.loop.unswitch.partial.disable"));
+      MDNode *NewLoopID = makePostTransformationMetadata(
+          Context, L.getLoopID(), {"llvm.loop.unswitch.partial"},
+          {DisableUnswitchMD});
+      L.setLoopID(NewLoopID);
     } else if (InjectedCondition) {
       // Do the same for injection of invariant conditions.
-      RecordLoopAsUnswitched(&L, "llvm.loop.unswitch.injection",
-                             "llvm.loop.unswitch.injection.disable");
+      auto &Context = L.getHeader()->getContext();
+      MDNode *DisableUnswitchMD = MDNode::get(
+          Context,
+          MDString::get(Context, "llvm.loop.unswitch.injection.disable"));
+      MDNode *NewLoopID = makePostTransformationMetadata(
+          Context, L.getLoopID(), {"llvm.loop.unswitch.injection"},
+          {DisableUnswitchMD});
+      L.setLoopID(NewLoopID);
     } else
       U.revisitCurrentLoop();
   } else
@@ -2811,9 +2809,9 @@ static BranchInst *turnGuardIntoBranch(IntrinsicInst *GI, Loop &L,
 }
 
 /// Cost multiplier is a way to limit potentially exponential behavior
-/// of loop-unswitch. Cost is multiplied in proportion of 2^number of unswitch
-/// candidates available. Also consider the number of "sibling" loops with
-/// the idea of accounting for previous unswitches that already happened on this
+/// of loop-unswitch. Cost is multipied in proportion of 2^number of unswitch
+/// candidates available. Also accounting for the number of "sibling" loops with
+/// the idea to account for previous unswitches that already happened on this
 /// cluster of loops. There was an attempt to keep this formula simple,
 /// just enough to limit the worst case behavior. Even if it is not that simple
 /// now it is still not an attempt to provide a detailed heuristic size
@@ -3509,9 +3507,8 @@ static bool unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
   SmallVector<NonTrivialUnswitchCandidate, 4> UnswitchCandidates;
   IVConditionInfo PartialIVInfo;
   Instruction *PartialIVCondBranch = nullptr;
-  if (!findOptionMDForLoop(&L, "llvm.loop.unswitch.nontrivial.disable"))
-    collectUnswitchCandidates(UnswitchCandidates, PartialIVInfo,
-                              PartialIVCondBranch, L, LI, AA, MSSAU);
+  collectUnswitchCandidates(UnswitchCandidates, PartialIVInfo,
+                            PartialIVCondBranch, L, LI, AA, MSSAU);
   if (!findOptionMDForLoop(&L, "llvm.loop.unswitch.injection.disable"))
     collectUnswitchCandidatesWithInjections(UnswitchCandidates, PartialIVInfo,
                                             PartialIVCondBranch, L, DT, LI, AA,
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index 44e63a0583d1a..bb7dbc2980f59 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -13,7 +13,6 @@
 #include "llvm/ADT/SCCIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/RegionInfo.h"
@@ -328,7 +327,7 @@ class StructurizeCFG {
   void addPhiValues(BasicBlock *From, BasicBlock *To);
 
   void findUndefBlocks(BasicBlock *PHIBlock,
-                       const SmallSet<BasicBlock *, 8> &Incomings,
+                       const SmallPtrSet<BasicBlock *, 8> &Incomings,
                        SmallVector<BasicBlock *> &UndefBlks) const;
 
   void mergeIfCompatible(EquivalenceClasses<PHINode *> &PhiClasses, PHINode *A,
@@ -762,7 +761,7 @@ void StructurizeCFG::addPhiValues(BasicBlock *From, BasicBlock *To) {
 /// from some blocks as undefined. The function will find out all such blocks
 /// and return in \p UndefBlks.
 void StructurizeCFG::findUndefBlocks(
-    BasicBlock *PHIBlock, const SmallSet<BasicBlock *, 8> &Incomings,
+    BasicBlock *PHIBlock, const SmallPtrSet<BasicBlock *, 8> &Incomings,
     SmallVector<BasicBlock *> &UndefBlks) const {
   //  We may get a post-structured CFG like below:
   //
@@ -788,7 +787,7 @@ void StructurizeCFG::findUndefBlocks(
   // path N->F2->F3->B. For example, the threads take the branch F1->N may
   // always take the branch F2->P2. So, when we are reconstructing a PHI
   // originally in B, we can safely say the incoming value from N is undefined.
-  SmallSet<BasicBlock *, 8> VisitedBlock;
+  SmallPtrSet<BasicBlock *, 8> VisitedBlock;
   SmallVector<BasicBlock *, 8> Stack;
   if (PHIBlock == ParentRegion->getExit()) {
     for (auto P : predecessors(PHIBlock)) {
@@ -884,7 +883,7 @@ void StructurizeCFG::setPhiValues() {
 
     PhiMap &BlkPhis = OldPhiIt->second;
     SmallVector<BasicBlock *> &UndefBlks = UndefBlksMap[To];
-    SmallSet<BasicBlock *, 8> Incomings;
+    SmallPtrSet<BasicBlock *, 8> Incomings;
 
     // Get the undefined blocks shared by all the phi nodes.
     if (!BlkPhis.empty()) {
diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index c8255742c41ba..cad0b4c12b54e 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -377,15 +377,12 @@ bool llvm::MergeBlockSuccessorsIntoGivenBlocks(
 ///
 /// Possible improvements:
 /// - Check fully overlapping fragments and not only identical fragments.
-static bool
-DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
+static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
   SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
   SmallDenseSet<DebugVariable> VariableSet;
   for (auto &I : reverse(*BB)) {
-    for (DbgVariableRecord &DR :
+    for (DbgVariableRecord &DVR :
          reverse(filterDbgVars(I.getDbgRecordRange()))) {
-      DbgVariableRecord &DVR = cast<DbgVariableRecord>(DR);
-
       DebugVariable Key(DVR.getVariable(), DVR.getExpression(),
                         DVR.getDebugLoc()->getInlinedAt());
       auto R = VariableSet.insert(Key);
@@ -416,10 +413,6 @@ DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
   return !ToBeRemoved.empty();
 }
 
-static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
-  return DbgVariableRecordsRemoveRedundantDbgInstrsUsingBackwardScan(BB);
-}
-
 /// Remove redundant dbg.value instructions using a forward scan. This can
 /// remove a dbg.value instruction that is redundant due to indicating that a
 /// variable has the same value as already being indicated by an earlier
@@ -439,14 +432,14 @@ static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB) {
 ///
 /// Possible improvements:
 /// - Keep track of non-overlapping fragments.
-static bool
-DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
-  SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
+static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
+  bool RemovedAny = false;
   SmallDenseMap<DebugVariable,
                 std::pair<SmallVector<Value *, 4>, DIExpression *>, 4>
       VariableMap;
   for (auto &I : *BB) {
-    for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
+    for (DbgVariableRecord &DVR :
+         make_early_inc_range(filterDbgVars(I.getDbgRecordRange()))) {
       if (DVR.getType() == DbgVariableRecord::LocationType::Declare)
         continue;
       DebugVariable Key(DVR.getVariable(), std::nullopt,
@@ -472,55 +465,12 @@ DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
       if (!IsDbgValueKind)
         continue;
       // Found an identical mapping. Remember the instruction for later removal.
-      ToBeRemoved.push_back(&DVR);
-    }
-  }
-
-  for (auto *DVR : ToBeRemoved)
-    DVR->eraseFromParent();
-
-  return !ToBeRemoved.empty();
-}
-
-static bool
-DbgVariableRecordsRemoveUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
-  assert(BB->isEntryBlock() && "expected entry block");
-  SmallVector<DbgVariableRecord *, 8> ToBeRemoved;
-  DenseSet<DebugVariable> SeenDefForAggregate;
-  // Returns the DebugVariable for DVI with no fragment info.
-  auto GetAggregateVariable = [](const DbgVariableRecord &DVR) {
-    return DebugVariable(DVR.getVariable(), std::nullopt,
-                         DVR.getDebugLoc().getInlinedAt());
-  };
-
-  // Remove undef dbg.assign intrinsics that are encountered before
-  // any non-undef intrinsics from the entry block.
-  for (auto &I : *BB) {
-    for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
-      if (!DVR.isDbgValue() && !DVR.isDbgAssign())
-        continue;
-      bool IsDbgValueKind =
-          (DVR.isDbgValue() || at::getAssignmentInsts(&DVR).empty());
-      DebugVariable Aggregate = GetAggregateVariable(DVR);
-      if (!SeenDefForAggregate.contains(Aggregate)) {
-        bool IsKill = DVR.isKillLocation() && IsDbgValueKind;
-        if (!IsKill) {
-          SeenDefForAggregate.insert(Aggregate);
-        } else if (DVR.isDbgAssign()) {
-          ToBeRemoved.push_back(&DVR);
-        }
-      }
+      DVR.eraseFromParent();
+      RemovedAny = true;
     }
   }
 
-  for (DbgVariableRecord *DVR : ToBeRemoved)
-    DVR->eraseFromParent();
-
-  return !ToBeRemoved.empty();
-}
-
-static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
-  return DbgVariableRecordsRemoveRedundantDbgInstrsUsingForwardScan(BB);
+  return RemovedAny;
 }
 
 /// Remove redundant undef dbg.assign intrinsic from an entry block using a
@@ -543,7 +493,34 @@ static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB) {
 /// Possible improvements:
 /// - Keep track of non-overlapping fragments.
 static bool removeUndefDbgAssignsFromEntryBlock(BasicBlock *BB) {
-  return DbgVariableRecordsRemoveUndefDbgAssignsFromEntryBlock(BB);
+  assert(BB->isEntryBlock() && "expected entry block");
+  bool RemovedAny = false;
+  DenseSet<DebugVariableAggregate> SeenDefForAggregate;
+
+  // Remove undef dbg.assign intrinsics that are encountered before
+  // any non-undef intrinsics from the entry block.
+  for (auto &I : *BB) {
+    for (DbgVariableRecord &DVR :
+         make_early_inc_range(filterDbgVars(I.getDbgRecordRange()))) {
+      if (!DVR.isDbgValue() && !DVR.isDbgAssign())
+        continue;
+      bool IsDbgValueKind =
+          (DVR.isDbgValue() || at::getAssignmentInsts(&DVR).empty());
+
+      DebugVariableAggregate Aggregate(&DVR);
+      if (!SeenDefForAggregate.contains(Aggregate)) {
+        bool IsKill = DVR.isKillLocation() && IsDbgValueKind;
+        if (!IsKill) {
+          SeenDefForAggregate.insert(Aggregate);
+        } else if (DVR.isDbgAssign()) {
+          DVR.eraseFromParent();
+          RemovedAny = true;
+        }
+      }
+    }
+  }
+
+  return RemovedAny;
 }
 
 bool llvm::RemoveRedundantDbgInstrs(BasicBlock *BB) {
diff --git a/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp b/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp
index ddd203f3acf71..42b1fdf17f389 100644
--- a/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp
+++ b/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp
@@ -111,15 +111,14 @@ BasicBlock *
 llvm::SplitKnownCriticalEdge(Instruction *TI, unsigned SuccNum,
                              const CriticalEdgeSplittingOptions &Options,
                              const Twine &BBName) {
-  assert(!isa<IndirectBrInst>(TI) &&
-         "Cannot split critical edge from IndirectBrInst");
-
   BasicBlock *TIBB = TI->getParent();
   BasicBlock *DestBB = TI->getSuccessor(SuccNum);
 
-  // Splitting the critical edge to a pad block is non-trivial. Don't do
-  // it in this generic function.
-  if (DestBB->isEHPad()) return nullptr;
+  // Splitting the critical edge to a pad block is non-trivial.
+  // And we cannot split block with IndirectBr as a terminator.
+  // Don't do it in this generic function.
+  if (DestBB->isEHPad() || isa<IndirectBrInst>(TI))
+    return nullptr;
 
   if (Options.IgnoreUnreachableDests &&
       isa<UnreachableInst>(DestBB->getFirstNonPHIOrDbgOrLifetime()))
diff --git a/llvm/lib/Transforms/Utils/CanonicalizeFreezeInLoops.cpp b/llvm/lib/Transforms/Utils/CanonicalizeFreezeInLoops.cpp
index 40010aee9c111..2190dcdc8aeaa 100644
--- a/llvm/lib/Transforms/Utils/CanonicalizeFreezeInLoops.cpp
+++ b/llvm/lib/Transforms/Utils/CanonicalizeFreezeInLoops.cpp
@@ -32,7 +32,6 @@
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/IVDescriptors.h"
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -193,7 +192,7 @@ bool CanonicalizeFreezeInLoopsImpl::run() {
   if (Candidates.empty())
     return false;
 
-  SmallSet<PHINode *, 8> ProcessedPHIs;
+  SmallPtrSet<PHINode *, 8> ProcessedPHIs;
   for (const auto &Info : Candidates) {
     PHINode *PHI = Info.PHI;
     if (!ProcessedPHIs.insert(Info.PHI).second)
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 7a9dd37b72205..bbd1ed6a3ab2d 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -1099,7 +1099,7 @@ static void eraseLifetimeMarkersOnInputs(const SetVector<BasicBlock *> &Blocks,
       // Get the memory operand of the lifetime marker. If the underlying
       // object is a sunk alloca, or is otherwise defined in the extraction
       // region, the lifetime marker must not be erased.
-      Value *Mem = II->getOperand(1)->stripInBoundsOffsets();
+      Value *Mem = II->getOperand(0);
       if (SunkAllocas.count(Mem) || definedInRegion(Blocks, Mem))
         continue;
 
@@ -1115,8 +1115,6 @@ static void eraseLifetimeMarkersOnInputs(const SetVector<BasicBlock *> &Blocks,
 static void insertLifetimeMarkersSurroundingCall(
     Module *M, ArrayRef<Value *> LifetimesStart, ArrayRef<Value *> LifetimesEnd,
     CallInst *TheCall) {
-  LLVMContext &Ctx = M->getContext();
-  auto NegativeOne = ConstantInt::getSigned(Type::getInt64Ty(Ctx), -1);
   Instruction *Term = TheCall->getParent()->getTerminator();
 
   // Emit lifetime markers for the pointers given in \p Objects. Insert the
@@ -1130,7 +1128,7 @@ static void insertLifetimeMarkersSurroundingCall(
 
       Function *Func =
           Intrinsic::getOrInsertDeclaration(M, MarkerFunc, Mem->getType());
-      auto Marker = CallInst::Create(Func, {NegativeOne, Mem});
+      auto Marker = CallInst::Create(Func, Mem);
       if (InsertBefore)
         Marker->insertBefore(TheCall->getIterator());
       else
diff --git a/llvm/lib/Transforms/Utils/ControlFlowUtils.cpp b/llvm/lib/Transforms/Utils/ControlFlowUtils.cpp
index 4b0065d0030cd..0046a00af4338 100644
--- a/llvm/lib/Transforms/Utils/ControlFlowUtils.cpp
+++ b/llvm/lib/Transforms/Utils/ControlFlowUtils.cpp
@@ -12,7 +12,6 @@
 
 #include "llvm/Transforms/Utils/ControlFlowUtils.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Instructions.h"
@@ -276,7 +275,7 @@ std::pair<BasicBlock *, bool> ControlFlowHub::finalize(
     DomTreeUpdater *DTU, SmallVectorImpl<BasicBlock *> &GuardBlocks,
     const StringRef Prefix, std::optional<unsigned> MaxControlFlowBooleans) {
 #ifndef NDEBUG
-  SmallSet<BasicBlock *, 8> Incoming;
+  SmallPtrSet<BasicBlock *, 8> Incoming;
 #endif
   SetVector<BasicBlock *> Outgoing;
 
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index 291e2a536302e..7063cde5263b4 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -706,6 +706,15 @@ bool llvm::checkDebugInfoMetadata(Module &M,
       DILocsBefore, DILocsAfter, InstToDelete, NameOfWrappedPass,
       FileNameFromCU, ShouldWriteIntoJSON, Bugs);
 
+#if LLVM_ENABLE_DEBUGLOC_TRACKING_COVERAGE
+  // If we are tracking DebugLoc coverage, replace each empty DebugLoc with an
+  // annotated location now so that it does not show up in future passes even if
+  // it is propagated to other instructions.
+  for (auto &L : DILocsAfter)
+    if (!L.second)
+      const_cast<Instruction *>(L.first)->setDebugLoc(DebugLoc::getUnknown());
+#endif
+
   bool ResultForVars = checkVars(DIVarsBefore, DIVarsAfter, NameOfWrappedPass,
                                  FileNameFromCU, ShouldWriteIntoJSON, Bugs);
 
diff --git a/llvm/lib/Transforms/Utils/DeclareRuntimeLibcalls.cpp b/llvm/lib/Transforms/Utils/DeclareRuntimeLibcalls.cpp
index 540039b7d2cbd..0642d51cd2c21 100644
--- a/llvm/lib/Transforms/Utils/DeclareRuntimeLibcalls.cpp
+++ b/llvm/lib/Transforms/Utils/DeclareRuntimeLibcalls.cpp
@@ -30,7 +30,7 @@ PreservedAnalyses DeclareRuntimeLibcallsPass::run(Module &M,
     FunctionType *FuncTy =
         FunctionType::get(Type::getVoidTy(Ctx), {}, /*IsVarArgs=*/true);
 
-    const char *FuncName = RTLCI.getLibcallImplName(Impl);
+    StringRef FuncName = RTLCI.getLibcallImplName(Impl);
     M.getOrInsertFunction(FuncName, FuncTy);
   }
 
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index ed3dca2f7c307..f49fbf8807bac 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -77,7 +77,6 @@
 #include <cstdint>
 #include <deque>
 #include <iterator>
-#include <limits>
 #include <optional>
 #include <string>
 #include <utility>
@@ -2361,15 +2360,13 @@ remapIndices(Function &Caller, BasicBlock *StartBB,
 // Updating the contextual profile after an inlining means, at a high level,
 // copying over the data of the callee, **intentionally without any value
 // scaling**, and copying over the callees of the inlined callee.
-llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
-                                        PGOContextualProfile &CtxProf,
-                                        bool MergeAttributes,
-                                        AAResults *CalleeAAR,
-                                        bool InsertLifetime,
-                                        Function *ForwardVarArgsTo) {
+llvm::InlineResult llvm::InlineFunction(
+    CallBase &CB, InlineFunctionInfo &IFI, PGOContextualProfile &CtxProf,
+    bool MergeAttributes, AAResults *CalleeAAR, bool InsertLifetime,
+    Function *ForwardVarArgsTo, OptimizationRemarkEmitter *ORE) {
   if (!CtxProf.isInSpecializedModule())
     return InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime,
-                          ForwardVarArgsTo);
+                          ForwardVarArgsTo, ORE);
 
   auto &Caller = *CB.getCaller();
   auto &Callee = *CB.getCalledFunction();
@@ -2387,7 +2384,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
   const auto NumCalleeCallsites = CtxProf.getNumCallsites(Callee);
 
   auto Ret = InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime,
-                            ForwardVarArgsTo);
+                            ForwardVarArgsTo, ORE);
   if (!Ret.isSuccess())
     return Ret;
 
@@ -2457,20 +2454,8 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
   return Ret;
 }
 
-/// This function inlines the called function into the basic block of the
-/// caller. This returns false if it is not possible to inline this call.
-/// The program is still in a well defined state if this occurs though.
-///
-/// Note that this only does one level of inlining.  For example, if the
-/// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now
-/// exists in the instruction stream.  Similarly this will inline a recursive
-/// function by one level.
-llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
-                                        bool MergeAttributes,
-                                        AAResults *CalleeAAR,
-                                        bool InsertLifetime,
-                                        Function *ForwardVarArgsTo,
-                                        OptimizationRemarkEmitter *ORE) {
+llvm::InlineResult llvm::CanInlineCallSite(const CallBase &CB,
+                                           InlineFunctionInfo &IFI) {
   assert(CB.getParent() && CB.getFunction() && "Instruction not in function!");
 
   // FIXME: we don't inline callbr yet.
@@ -2487,7 +2472,6 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
 
   // The inliner does not know how to inline through calls with operand bundles
   // in general ...
-  Value *ConvergenceControlToken = nullptr;
   if (CB.hasOperandBundles()) {
     for (int i = 0, e = CB.getNumOperandBundles(); i != e; ++i) {
       auto OBUse = CB.getOperandBundleAt(i);
@@ -2503,7 +2487,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
       if (Tag == LLVMContext::OB_kcfi)
         continue;
       if (Tag == LLVMContext::OB_convergencectrl) {
-        ConvergenceControlToken = OBUse.Inputs[0].get();
+        IFI.ConvergenceControlToken = OBUse.Inputs[0].get();
         continue;
       }
 
@@ -2521,28 +2505,22 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
   // fully implements convergence control tokens, there is no mixing of
   // controlled and uncontrolled convergent operations in the whole program.
   if (CB.isConvergent()) {
-    if (!ConvergenceControlToken &&
+    if (!IFI.ConvergenceControlToken &&
         getConvergenceEntry(CalledFunc->getEntryBlock())) {
       return InlineResult::failure(
           "convergent call needs convergencectrl operand");
     }
   }
 
-  // If the call to the callee cannot throw, set the 'nounwind' flag on any
-  // calls that we inline.
-  bool MarkNoUnwind = CB.doesNotThrow();
-
-  BasicBlock *OrigBB = CB.getParent();
-  Function *Caller = OrigBB->getParent();
+  const BasicBlock *OrigBB = CB.getParent();
+  const Function *Caller = OrigBB->getParent();
 
   // GC poses two hazards to inlining, which only occur when the callee has GC:
   //  1. If the caller has no GC, then the callee's GC must be propagated to the
   //     caller.
   //  2. If the caller has a differing GC, it is invalid to inline.
   if (CalledFunc->hasGC()) {
-    if (!Caller->hasGC())
-      Caller->setGC(CalledFunc->getGC());
-    else if (CalledFunc->getGC() != Caller->getGC())
+    if (Caller->hasGC() && CalledFunc->getGC() != Caller->getGC())
       return InlineResult::failure("incompatible GC");
   }
 
@@ -2560,34 +2538,31 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
           ? Caller->getPersonalityFn()->stripPointerCasts()
           : nullptr;
   if (CalledPersonality) {
-    if (!CallerPersonality)
-      Caller->setPersonalityFn(CalledPersonality);
     // If the personality functions match, then we can perform the
     // inlining. Otherwise, we can't inline.
     // TODO: This isn't 100% true. Some personality functions are proper
     //       supersets of others and can be used in place of the other.
-    else if (CalledPersonality != CallerPersonality)
+    if (CallerPersonality && CalledPersonality != CallerPersonality)
       return InlineResult::failure("incompatible personality");
   }
 
   // We need to figure out which funclet the callsite was in so that we may
   // properly nest the callee.
-  Instruction *CallSiteEHPad = nullptr;
   if (CallerPersonality) {
     EHPersonality Personality = classifyEHPersonality(CallerPersonality);
     if (isScopedEHPersonality(Personality)) {
       std::optional<OperandBundleUse> ParentFunclet =
           CB.getOperandBundle(LLVMContext::OB_funclet);
       if (ParentFunclet)
-        CallSiteEHPad = cast<FuncletPadInst>(ParentFunclet->Inputs.front());
+        IFI.CallSiteEHPad = cast<FuncletPadInst>(ParentFunclet->Inputs.front());
 
       // OK, the inlining site is legal.  What about the target function?
 
-      if (CallSiteEHPad) {
+      if (IFI.CallSiteEHPad) {
         if (Personality == EHPersonality::MSVC_CXX) {
           // The MSVC personality cannot tolerate catches getting inlined into
           // cleanup funclets.
-          if (isa<CleanupPadInst>(CallSiteEHPad)) {
+          if (isa<CleanupPadInst>(IFI.CallSiteEHPad)) {
             // Ok, the call site is within a cleanuppad.  Let's check the callee
             // for catchpads.
             for (const BasicBlock &CalledBB : *CalledFunc) {
@@ -2607,13 +2582,34 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
     }
   }
 
+  return InlineResult::success();
+}
+
+/// This function inlines the called function into the basic block of the
+/// caller. This returns false if it is not possible to inline this call.
+/// The program is still in a well defined state if this occurs though.
+///
+/// Note that this only does one level of inlining.  For example, if the
+/// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now
+/// exists in the instruction stream.  Similarly this will inline a recursive
+/// function by one level.
+void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI,
+                              bool MergeAttributes, AAResults *CalleeAAR,
+                              bool InsertLifetime, Function *ForwardVarArgsTo,
+                              OptimizationRemarkEmitter *ORE) {
+  BasicBlock *OrigBB = CB.getParent();
+  Function *Caller = OrigBB->getParent();
+  Function *CalledFunc = CB.getCalledFunction();
+  assert(CalledFunc && !CalledFunc->isDeclaration() &&
+         "CanInlineCallSite should have verified direct call to definition");
+
   // Determine if we are dealing with a call in an EHPad which does not unwind
   // to caller.
   bool EHPadForCallUnwindsLocally = false;
-  if (CallSiteEHPad && isa<CallInst>(CB)) {
+  if (IFI.CallSiteEHPad && isa<CallInst>(CB)) {
     UnwindDestMemoTy FuncletUnwindMap;
     Value *CallSiteUnwindDestToken =
-        getUnwindDestToken(CallSiteEHPad, FuncletUnwindMap);
+        getUnwindDestToken(IFI.CallSiteEHPad, FuncletUnwindMap);
 
     EHPadForCallUnwindsLocally =
         CallSiteUnwindDestToken &&
@@ -2630,6 +2626,30 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
   ClonedCodeInfo InlinedFunctionInfo;
   Function::iterator FirstNewBlock;
 
+  // GC poses two hazards to inlining, which only occur when the callee has GC:
+  //  1. If the caller has no GC, then the callee's GC must be propagated to the
+  //     caller.
+  //  2. If the caller has a differing GC, it is invalid to inline.
+  if (CalledFunc->hasGC()) {
+    if (!Caller->hasGC())
+      Caller->setGC(CalledFunc->getGC());
+    else {
+      assert(CalledFunc->getGC() == Caller->getGC() &&
+             "CanInlineCallSite should have verified compatible GCs");
+    }
+  }
+
+  if (CalledFunc->hasPersonalityFn()) {
+    Constant *CalledPersonality =
+        CalledFunc->getPersonalityFn()->stripPointerCasts();
+    if (!Caller->hasPersonalityFn()) {
+      Caller->setPersonalityFn(CalledPersonality);
+    } else
+      assert(Caller->getPersonalityFn()->stripPointerCasts() ==
+                 CalledPersonality &&
+             "CanInlineCallSite should have verified compatible personality");
+  }
+
   { // Scope to destroy VMap after cloning.
     ValueToValueMapTy VMap;
     struct ByValInit {
@@ -2819,10 +2839,10 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
             IFI.GetAssumptionCache(*Caller).registerAssumption(II);
   }
 
-  if (ConvergenceControlToken) {
+  if (IFI.ConvergenceControlToken) {
     IntrinsicInst *IntrinsicCall = getConvergenceEntry(*FirstNewBlock);
     if (IntrinsicCall) {
-      IntrinsicCall->replaceAllUsesWith(ConvergenceControlToken);
+      IntrinsicCall->replaceAllUsesWith(IFI.ConvergenceControlToken);
       IntrinsicCall->eraseFromParent();
     }
   }
@@ -2869,6 +2889,10 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
     }
   }
 
+  // If the call to the callee cannot throw, set the 'nounwind' flag on any
+  // calls that we inline.
+  bool MarkNoUnwind = CB.doesNotThrow();
+
   SmallVector<Value*,4> VarArgsToForward;
   SmallVector<AttributeSet, 4> VarArgsAttrs;
   for (unsigned i = CalledFunc->getFunctionType()->getNumParams();
@@ -2979,31 +3003,11 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
       if (hasLifetimeMarkers(AI))
         continue;
 
-      // Try to determine the size of the allocation.
-      ConstantInt *AllocaSize = nullptr;
-      if (ConstantInt *AIArraySize =
-          dyn_cast<ConstantInt>(AI->getArraySize())) {
-        auto &DL = Caller->getDataLayout();
-        Type *AllocaType = AI->getAllocatedType();
-        TypeSize AllocaTypeSize = DL.getTypeAllocSize(AllocaType);
-        uint64_t AllocaArraySize = AIArraySize->getLimitedValue();
-
-        // Don't add markers for zero-sized allocas.
-        if (AllocaArraySize == 0)
-          continue;
-
-        // Check that array size doesn't saturate uint64_t and doesn't
-        // overflow when it's multiplied by type size.
-        if (!AllocaTypeSize.isScalable() &&
-            AllocaArraySize != std::numeric_limits<uint64_t>::max() &&
-            std::numeric_limits<uint64_t>::max() / AllocaArraySize >=
-                AllocaTypeSize.getFixedValue()) {
-          AllocaSize = ConstantInt::get(Type::getInt64Ty(AI->getContext()),
-                                        AllocaArraySize * AllocaTypeSize);
-        }
-      }
+      std::optional<TypeSize> Size = AI->getAllocationSize(AI->getDataLayout());
+      if (Size && Size->isZero())
+        continue;
 
-      builder.CreateLifetimeStart(AI, AllocaSize);
+      builder.CreateLifetimeStart(AI);
       for (ReturnInst *RI : Returns) {
         // Don't insert llvm.lifetime.end calls between a musttail or deoptimize
         // call and a return.  The return kills all local allocas.
@@ -3013,7 +3017,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
         if (InlinedDeoptimizeCalls &&
             RI->getParent()->getTerminatingDeoptimizeCall())
           continue;
-        IRBuilder<>(RI).CreateLifetimeEnd(AI, AllocaSize);
+        IRBuilder<>(RI).CreateLifetimeEnd(AI);
       }
     }
   }
@@ -3055,12 +3059,12 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
   // Update the lexical scopes of the new funclets and callsites.
   // Anything that had 'none' as its parent is now nested inside the callsite's
   // EHPad.
-  if (CallSiteEHPad) {
+  if (IFI.CallSiteEHPad) {
     for (Function::iterator BB = FirstNewBlock->getIterator(),
                             E = Caller->end();
          BB != E; ++BB) {
       // Add bundle operands to inlined call sites.
-      PropagateOperandBundles(BB, CallSiteEHPad);
+      PropagateOperandBundles(BB, IFI.CallSiteEHPad);
 
       // It is problematic if the inlinee has a cleanupret which unwinds to
       // caller and we inline it into a call site which doesn't unwind but into
@@ -3076,11 +3080,11 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
 
       if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(I)) {
         if (isa<ConstantTokenNone>(CatchSwitch->getParentPad()))
-          CatchSwitch->setParentPad(CallSiteEHPad);
+          CatchSwitch->setParentPad(IFI.CallSiteEHPad);
       } else {
         auto *FPI = cast<FuncletPadInst>(I);
         if (isa<ConstantTokenNone>(FPI->getParentPad()))
-          FPI->setParentPad(CallSiteEHPad);
+          FPI->setParentPad(IFI.CallSiteEHPad);
       }
     }
   }
@@ -3236,7 +3240,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
       AttributeFuncs::mergeAttributesForInlining(*Caller, *CalledFunc);
 
     // We are now done with the inlining.
-    return InlineResult::success();
+    return;
   }
 
   // Otherwise, we have the normal case, of more than one block to inline or
@@ -3404,6 +3408,19 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
 
   if (MergeAttributes)
     AttributeFuncs::mergeAttributesForInlining(*Caller, *CalledFunc);
+}
 
-  return InlineResult::success();
+llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI,
+                                        bool MergeAttributes,
+                                        AAResults *CalleeAAR,
+                                        bool InsertLifetime,
+                                        Function *ForwardVarArgsTo,
+                                        OptimizationRemarkEmitter *ORE) {
+  llvm::InlineResult Result = CanInlineCallSite(CB, IFI);
+  if (Result.isSuccess()) {
+    InlineFunctionImpl(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime,
+                       ForwardVarArgsTo, ORE);
+  }
+
+  return Result;
 }
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 2619e73f7fc8a..ac344904f90f0 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -275,7 +275,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
       Builder.CreateBr(TheOnlyDest);
       BasicBlock *BB = SI->getParent();
 
-      SmallSet<BasicBlock *, 8> RemovedSuccessors;
+      SmallPtrSet<BasicBlock *, 8> RemovedSuccessors;
 
       // Remove entries from PHI nodes which we no longer branch to...
       BasicBlock *SuccToKeep = TheOnlyDest;
@@ -343,7 +343,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
     if (auto *BA =
           dyn_cast<BlockAddress>(IBI->getAddress()->stripPointerCasts())) {
       BasicBlock *TheOnlyDest = BA->getBasicBlock();
-      SmallSet<BasicBlock *, 8> RemovedSuccessors;
+      SmallPtrSet<BasicBlock *, 8> RemovedSuccessors;
 
       // Insert the new branch.
       Builder.CreateBr(TheOnlyDest);
@@ -481,7 +481,7 @@ bool llvm::wouldInstructionBeTriviallyDead(const Instruction *I,
       return true;
 
     if (II->isLifetimeStartOrEnd()) {
-      auto *Arg = II->getArgOperand(1);
+      auto *Arg = II->getArgOperand(0);
       if (isa<PoisonValue>(Arg))
         return true;
 
@@ -2518,7 +2518,7 @@ unsigned llvm::changeToUnreachable(Instruction *I, bool PreserveLCSSA,
   if (MSSAU)
     MSSAU->changeToUnreachable(I);
 
-  SmallSet<BasicBlock *, 8> UniqueSuccessors;
+  SmallPtrSet<BasicBlock *, 8> UniqueSuccessors;
 
   // Loop over all of the successors, removing BB's entry from any PHI
   // nodes.
diff --git a/llvm/lib/Transforms/Utils/LoopPeel.cpp b/llvm/lib/Transforms/Utils/LoopPeel.cpp
index 27e70c5ddc0fc..735bad1cb1348 100644
--- a/llvm/lib/Transforms/Utils/LoopPeel.cpp
+++ b/llvm/lib/Transforms/Utils/LoopPeel.cpp
@@ -81,6 +81,10 @@ static cl::opt<bool> DisableAdvancedPeeling(
     cl::desc(
         "Disable advance peeling. Issues for convergent targets (D134803)."));
 
+static cl::opt<bool> EnablePeelingForIV(
+    "enable-peeling-for-iv", cl::init(false), cl::Hidden,
+    cl::desc("Enable peeling to convert Phi nodes into IVs"));
+
 static const char *PeeledCountMetaData = "llvm.loop.peeled.count";
 
 // Check whether we are capable of peeling this loop.
@@ -155,45 +159,168 @@ namespace {
 // corresponding calls to g are determined and the code for computing
 // x, y, and a can be removed.
 //
+// Similarly, there are cases where peeling makes Phi nodes loop-inductions
+// (i.e., the value is increased or decreased by a fixed amount on every
+// iteration). For example, consider the following function.
+//
+//   #define N 100
+//   void f(int a[], int b[]) {
+//     int im = N - 1;
+//     for (int i = 0; i < N; i++) {
+//       a[i] = b[i] + b[im];
+//       im = i;
+//     }
+//   }
+//
+// The IR of the loop will look something like the following.
+//
+//   %i = phi i32 [ 0, %entry ], [ %i.next, %for.body ]
+//   %im = phi i32 [ 99, %entry ], [ %i, %for.body ]
+//   ...
+//   %i.next = add nuw nsw i32 %i, 1
+//   ...
+//
+// In this case, %im becomes a loop-induction variable by peeling 1 iteration,
+// because %i is a loop-induction one. The peeling count can be determined by
+// the same algorithm with loop-invariant case. Such peeling is profitable for
+// loop-vectorization.
+//
 // The PhiAnalyzer class calculates how many times a loop should be
 // peeled based on the above analysis of the phi nodes in the loop while
 // respecting the maximum specified.
 class PhiAnalyzer {
 public:
-  PhiAnalyzer(const Loop &L, unsigned MaxIterations);
+  PhiAnalyzer(const Loop &L, unsigned MaxIterations, bool PeelForIV);
 
   // Calculate the sufficient minimum number of iterations of the loop to peel
   // such that phi instructions become determined (subject to allowable limits)
   std::optional<unsigned> calculateIterationsToPeel();
 
 protected:
-  using PeelCounter = std::optional<unsigned>;
+  enum class PeelCounterType {
+    Invariant,
+    Induction,
+  };
+
+  using PeelCounterValue = std::pair<unsigned, PeelCounterType>;
+  using PeelCounter = std::optional<PeelCounterValue>;
   const PeelCounter Unknown = std::nullopt;
 
   // Add 1 respecting Unknown and return Unknown if result over MaxIterations
   PeelCounter addOne(PeelCounter PC) const {
     if (PC == Unknown)
       return Unknown;
-    return (*PC + 1 <= MaxIterations) ? PeelCounter{*PC + 1} : Unknown;
+    auto [Val, Ty] = *PC;
+    return (Val + 1 <= MaxIterations) ? PeelCounter({Val + 1, Ty}) : Unknown;
   }
 
-  // Calculate the number of iterations after which the given value
-  // becomes an invariant.
+  // Return a value representing zero for the given counter type.
+  PeelCounter makeZero(PeelCounterType Ty) const {
+    return PeelCounter({0, Ty});
+  }
+
+  // Calculate the number of iterations after which the given value becomes an
+  // invariant or an induction.
   PeelCounter calculate(const Value &);
 
+  // Auxiliary function to calculate the number of iterations for a comparison
+  // instruction or a binary operator.
+  PeelCounter mergeTwoCounters(const Instruction &CmpOrBinaryOp,
+                               const PeelCounterValue &LHS,
+                               const PeelCounterValue &RHS) const;
+
+  // Returns true if the \p Phi is an induction in the target loop. This is a
+  // lightweight check and possible to detect an IV in some cases.
+  bool isInductionPHI(const PHINode *Phi) const;
+
   const Loop &L;
   const unsigned MaxIterations;
+  const bool PeelForIV;
 
-  // Map of Values to number of iterations to invariance
-  SmallDenseMap<const Value *, PeelCounter> IterationsToInvariance;
+  // Map of Values to number of iterations to invariance or induction
+  SmallDenseMap<const Value *, PeelCounter> IterationsToInvarianceOrInduction;
 };
 
-PhiAnalyzer::PhiAnalyzer(const Loop &L, unsigned MaxIterations)
-    : L(L), MaxIterations(MaxIterations) {
+PhiAnalyzer::PhiAnalyzer(const Loop &L, unsigned MaxIterations, bool PeelForIV)
+    : L(L), MaxIterations(MaxIterations), PeelForIV(PeelForIV) {
   assert(canPeel(&L) && "loop is not suitable for peeling");
   assert(MaxIterations > 0 && "no peeling is allowed?");
 }
 
+/// Test whether \p Phi is an induction variable. Although this can be
+/// determined using SCEV analysis, it is expensive to compute here. Instead,
+/// we perform cheaper checks that may not detect complex cases but are
+/// sufficient for some situations.
+bool PhiAnalyzer::isInductionPHI(const PHINode *Phi) const {
+  // Currently we only support a loop that has single latch.
+  BasicBlock *Latch = L.getLoopLatch();
+  if (Latch == nullptr)
+    return false;
+
+  Value *Cur = Phi->getIncomingValueForBlock(Latch);
+  SmallPtrSet<Value *, 4> Visited;
+  bool VisitBinOp = false;
+
+  // Starting from the incoming value of the Phi, we follow the use-def chain.
+  // We consider Phi to be an IV if we can reach it again by traversing only
+  // add, sub, or cast instructions.
+  while (true) {
+    if (Cur == Phi)
+      break;
+
+    // Avoid infinite loop.
+    if (!Visited.insert(Cur).second)
+      return false;
+
+    auto *I = dyn_cast<Instruction>(Cur);
+    if (!I || !L.contains(I))
+      return false;
+
+    if (auto *Cast = dyn_cast<CastInst>(I)) {
+      Cur = Cast->getOperand(0);
+    } else if (auto *BinOp = dyn_cast<BinaryOperator>(I)) {
+      if (BinOp->getOpcode() != Instruction::Add &&
+          BinOp->getOpcode() != Instruction::Sub)
+        return false;
+      if (!isa<ConstantInt>(BinOp->getOperand(1)))
+        return false;
+
+      VisitBinOp = true;
+      Cur = BinOp->getOperand(0);
+    } else {
+      return false;
+    }
+  }
+
+  // Ignore cases where no binary operations are visited.
+  return VisitBinOp;
+}
+
+/// When either \p LHS or \p RHS is an IV, the result of \p CmpOrBinaryOp is
+/// considered an IV only if it is an addition or a subtraction. Otherwise the
+/// result can be a value that is neither a loop-invariant nor an IV.
+///
+/// If both \p LHS and \p RHS are loop-invariants, then the result of
+/// \CmpOrBinaryOp is also a loop-invariant.
+PhiAnalyzer::PeelCounter
+PhiAnalyzer::mergeTwoCounters(const Instruction &CmpOrBinaryOp,
+                              const PeelCounterValue &LHS,
+                              const PeelCounterValue &RHS) const {
+  auto &[LVal, LTy] = LHS;
+  auto &[RVal, RTy] = RHS;
+  unsigned NewVal = std::max(LVal, RVal);
+
+  if (LTy == PeelCounterType::Induction || RTy == PeelCounterType::Induction) {
+    if (const auto *BinOp = dyn_cast<BinaryOperator>(&CmpOrBinaryOp)) {
+      if (BinOp->getOpcode() == Instruction::Add ||
+          BinOp->getOpcode() == Instruction::Sub)
+        return PeelCounter({NewVal, PeelCounterType::Induction});
+    }
+    return Unknown;
+  }
+  return PeelCounter({NewVal, PeelCounterType::Invariant});
+}
+
 // This function calculates the number of iterations after which the value
 // becomes an invariant. The pre-calculated values are memorized in a map.
 // N.B. This number will be Unknown or <= MaxIterations.
@@ -212,25 +339,34 @@ PhiAnalyzer::PeelCounter PhiAnalyzer::calculate(const Value &V) {
   // If we already know the answer, take it from the map.
   // Otherwise, place Unknown to map to avoid infinite recursion. Such
   // cycles can never stop on an invariant.
-  auto [I, Inserted] = IterationsToInvariance.try_emplace(&V, Unknown);
+  auto [I, Inserted] =
+      IterationsToInvarianceOrInduction.try_emplace(&V, Unknown);
   if (!Inserted)
     return I->second;
 
   if (L.isLoopInvariant(&V))
     // Loop invariant so known at start.
-    return (IterationsToInvariance[&V] = 0);
+    return (IterationsToInvarianceOrInduction[&V] =
+                makeZero(PeelCounterType::Invariant));
   if (const PHINode *Phi = dyn_cast<PHINode>(&V)) {
     if (Phi->getParent() != L.getHeader()) {
       // Phi is not in header block so Unknown.
-      assert(IterationsToInvariance[&V] == Unknown && "unexpected value saved");
+      assert(IterationsToInvarianceOrInduction[&V] == Unknown &&
+             "unexpected value saved");
       return Unknown;
     }
+
+    // If Phi is an induction, register it as a starting point.
+    if (PeelForIV && isInductionPHI(Phi))
+      return (IterationsToInvarianceOrInduction[&V] =
+                  makeZero(PeelCounterType::Induction));
+
     // We need to analyze the input from the back edge and add 1.
     Value *Input = Phi->getIncomingValueForBlock(L.getLoopLatch());
     PeelCounter Iterations = calculate(*Input);
-    assert(IterationsToInvariance[Input] == Iterations &&
+    assert(IterationsToInvarianceOrInduction[Input] == Iterations &&
            "unexpected value saved");
-    return (IterationsToInvariance[Phi] = addOne(Iterations));
+    return (IterationsToInvarianceOrInduction[Phi] = addOne(Iterations));
   }
   if (const Instruction *I = dyn_cast<Instruction>(&V)) {
     if (isa<CmpInst>(I) || I->isBinaryOp()) {
@@ -241,26 +377,30 @@ PhiAnalyzer::PeelCounter PhiAnalyzer::calculate(const Value &V) {
       PeelCounter RHS = calculate(*I->getOperand(1));
       if (RHS == Unknown)
         return Unknown;
-      return (IterationsToInvariance[I] = {std::max(*LHS, *RHS)});
+      return (IterationsToInvarianceOrInduction[I] =
+                  mergeTwoCounters(*I, *LHS, *RHS));
     }
     if (I->isCast())
       // Cast instructions get the value of the operand.
-      return (IterationsToInvariance[I] = calculate(*I->getOperand(0)));
+      return (IterationsToInvarianceOrInduction[I] =
+                  calculate(*I->getOperand(0)));
   }
   // TODO: handle more expressions
 
   // Everything else is Unknown.
-  assert(IterationsToInvariance[&V] == Unknown && "unexpected value saved");
+  assert(IterationsToInvarianceOrInduction[&V] == Unknown &&
+         "unexpected value saved");
   return Unknown;
 }
 
 std::optional<unsigned> PhiAnalyzer::calculateIterationsToPeel() {
   unsigned Iterations = 0;
   for (auto &PHI : L.getHeader()->phis()) {
-    PeelCounter ToInvariance = calculate(PHI);
-    if (ToInvariance != Unknown) {
-      assert(*ToInvariance <= MaxIterations && "bad result in phi analysis");
-      Iterations = std::max(Iterations, *ToInvariance);
+    PeelCounter ToInvarianceOrInduction = calculate(PHI);
+    if (ToInvarianceOrInduction != Unknown) {
+      unsigned Val = ToInvarianceOrInduction->first;
+      assert(Val <= MaxIterations && "bad result in phi analysis");
+      Iterations = std::max(Iterations, Val);
       if (Iterations == MaxIterations)
         break;
     }
@@ -654,14 +794,15 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
   // in TTI.getPeelingPreferences or by the flag -unroll-peel-count.
   unsigned DesiredPeelCount = TargetPeelCount;
 
-  // Here we try to get rid of Phis which become invariants after 1, 2, ..., N
-  // iterations of the loop. For this we compute the number for iterations after
-  // which every Phi is guaranteed to become an invariant, and try to peel the
-  // maximum number of iterations among these values, thus turning all those
-  // Phis into invariants.
+  // Here we try to get rid of Phis which become invariants or inductions after
+  // 1, 2, ..., N iterations of the loop. For this we compute the number for
+  // iterations after which every Phi is guaranteed to become an invariant or an
+  // induction, and try to peel the maximum number of iterations among these
+  // values, thus turning all those Phis into invariants or inductions.
   if (MaxPeelCount > DesiredPeelCount) {
     // Check how many iterations are useful for resolving Phis
-    auto NumPeels = PhiAnalyzer(*L, MaxPeelCount).calculateIterationsToPeel();
+    auto NumPeels = PhiAnalyzer(*L, MaxPeelCount, EnablePeelingForIV)
+                        .calculateIterationsToPeel();
     if (NumPeels)
       DesiredPeelCount = std::max(DesiredPeelCount, *NumPeels);
   }
@@ -680,7 +821,7 @@ void llvm::computePeelCount(Loop *L, unsigned LoopSize,
     if (DesiredPeelCount + AlreadyPeeled <= UnrollPeelMaxCount) {
       LLVM_DEBUG(dbgs() << "Peel " << DesiredPeelCount
                         << " iteration(s) to turn"
-                        << " some Phis into invariants.\n");
+                        << " some Phis into invariants or inductions.\n");
       PP.PeelCount = DesiredPeelCount;
       PP.PeelProfiledIterations = false;
       PP.PeelLast = false;
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
index e7623aaff105d..843364eb34f83 100644
--- a/llvm/lib/Transforms/Utils/LoopUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -914,6 +914,8 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
   switch (RK) {
   default:
     llvm_unreachable("Unexpected recurrence kind");
+  case RecurKind::AddChainWithSubs:
+  case RecurKind::Sub:
   case RecurKind::Add:
     return Intrinsic::vector_reduce_add;
   case RecurKind::Mul:
@@ -954,6 +956,21 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
   }
 }
 
+Intrinsic::ID llvm::getMinMaxReductionIntrinsicID(Intrinsic::ID IID) {
+  switch (IID) {
+  default:
+    llvm_unreachable("Unexpected intrinsic id");
+  case Intrinsic::umin:
+    return Intrinsic::vector_reduce_umin;
+  case Intrinsic::umax:
+    return Intrinsic::vector_reduce_umax;
+  case Intrinsic::smin:
+    return Intrinsic::vector_reduce_smin;
+  case Intrinsic::smax:
+    return Intrinsic::vector_reduce_smax;
+  }
+}
+
 // This is the inverse to getReductionForBinop
 unsigned llvm::getArithmeticReductionInstruction(Intrinsic::ID RdxID) {
   switch (RdxID) {
@@ -1301,6 +1318,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
                                  Builder.getFastMathFlags());
   };
   switch (RdxKind) {
+  case RecurKind::AddChainWithSubs:
+  case RecurKind::Sub:
   case RecurKind::Add:
   case RecurKind::Mul:
   case RecurKind::And:
diff --git a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
index 472c03f7fc6ca..1f59b1782c709 100644
--- a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
+++ b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp
@@ -155,7 +155,7 @@ void StackInfoBuilder::visit(OptimizationRemarkEmitter &ORE,
     return;
   }
   if (auto *II = dyn_cast<LifetimeIntrinsic>(&Inst)) {
-    AllocaInst *AI = dyn_cast<AllocaInst>(II->getArgOperand(1));
+    AllocaInst *AI = dyn_cast<AllocaInst>(II->getArgOperand(0));
     if (!AI ||
         getAllocaInterestingness(*AI) != AllocaInterestingness::kInteresting)
       return;
diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index b22ecbc33e4ff..978d5a25a57c8 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -20,7 +20,6 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -291,6 +290,11 @@ void PredicateInfoBuilder::convertUsesToDFSOrdered(
     Value *Op, SmallVectorImpl<ValueDFS> &DFSOrderedSet) {
   for (auto &U : Op->uses()) {
     if (auto *I = dyn_cast<Instruction>(U.getUser())) {
+      // Lifetime intrinsics must work directly on alloca, do not replace them
+      // with a predicated copy.
+      if (I->isLifetimeStartOrEnd())
+        continue;
+
       ValueDFS VD;
       // Put the phi node uses in the incoming block.
       BasicBlock *IBlock;
@@ -370,6 +374,8 @@ void PredicateInfoBuilder::processAssume(
     Values.push_back(Cond);
     if (auto *Cmp = dyn_cast<CmpInst>(Cond))
       collectCmpOps(Cmp, Values);
+    else if (match(Cond, m_NUWTrunc(m_Value(Op0))))
+      Values.push_back(Op0);
 
     for (Value *V : Values) {
       if (shouldRename(V)) {
@@ -416,6 +422,8 @@ void PredicateInfoBuilder::processBranch(
       Values.push_back(Cond);
       if (auto *Cmp = dyn_cast<CmpInst>(Cond))
         collectCmpOps(Cmp, Values);
+      else if (match(Cond, m_NUWTrunc(m_Value(Op0))))
+        Values.push_back(Op0);
 
       for (Value *V : Values) {
         if (shouldRename(V)) {
@@ -506,23 +514,10 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
     ValInfo->RenamedOp = (RenameStack.end() - Start) == RenameStack.begin()
                              ? OrigOp
                              : (RenameStack.end() - Start - 1)->Def;
-    auto CreateSSACopy = [this](IRBuilderBase &B, Value *Op,
-                                const Twine &Name = "") {
-      auto It = PI.DeclarationCache.try_emplace(Op->getType());
-      if (It.second) {
-        // The number of named values is used to detect if a new declaration
-        // was added. If so, that declaration is tracked so that it can be
-        // removed when the analysis is done. The corner case were a new
-        // declaration results in a name clash and the old name being renamed
-        // is not considered as that represents an invalid module.
-        auto NumDecls = F.getParent()->getNumNamedValues();
-        Function *IF = Intrinsic::getOrInsertDeclaration(
-            F.getParent(), Intrinsic::ssa_copy, Op->getType());
-        if (NumDecls != F.getParent()->getNumNamedValues())
-          PI.CreatedDeclarations.insert(IF);
-        It.first->second = IF;
-      }
-      return B.CreateCall(It.first->second, Op, Name);
+    auto CreateSSACopy = [](Instruction *InsertPt, Value *Op,
+                            const Twine &Name = "") {
+      // Use a no-op bitcast to represent ssa copy.
+      return new BitCastInst(Op, Op->getType(), Name, InsertPt->getIterator());
     };
     // For edge predicates, we can just place the operand in the block before
     // the terminator. For assume, we have to place it right after the assume
@@ -530,9 +525,8 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
     // right before the terminator or after the assume, so that we insert in
     // proper order in the case of multiple predicateinfo in the same block.
     if (isa<PredicateWithEdge>(ValInfo)) {
-      IRBuilder<> B(getBranchTerminator(ValInfo));
-      CallInst *PIC =
-          CreateSSACopy(B, Op, Op->getName() + "." + Twine(Counter++));
+      BitCastInst *PIC = CreateSSACopy(getBranchTerminator(ValInfo), Op,
+                                       Op->getName() + "." + Twine(Counter++));
       PI.PredicateMap.insert({PIC, ValInfo});
       Result.Def = PIC;
     } else {
@@ -541,8 +535,7 @@ Value *PredicateInfoBuilder::materializeStack(unsigned int &Counter,
              "Should not have gotten here without it being an assume");
       // Insert the predicate directly after the assume. While it also holds
       // directly before it, assume(i1 true) is not a useful fact.
-      IRBuilder<> B(PAssume->AssumeInst->getNextNode());
-      CallInst *PIC = CreateSSACopy(B, Op);
+      BitCastInst *PIC = CreateSSACopy(PAssume->AssumeInst->getNextNode(), Op);
       PI.PredicateMap.insert({PIC, ValInfo});
       Result.Def = PIC;
     }
@@ -710,23 +703,6 @@ PredicateInfo::PredicateInfo(Function &F, DominatorTree &DT,
   Builder.buildPredicateInfo();
 }
 
-// Remove all declarations we created . The PredicateInfo consumers are
-// responsible for remove the ssa_copy calls created.
-PredicateInfo::~PredicateInfo() {
-  // Collect function pointers in set first, as SmallSet uses a SmallVector
-  // internally and we have to remove the asserting value handles first.
-  SmallPtrSet<Function *, 20> FunctionPtrs;
-  for (const auto &F : CreatedDeclarations)
-    FunctionPtrs.insert(&*F);
-  CreatedDeclarations.clear();
-
-  for (Function *F : FunctionPtrs) {
-    assert(F->users().empty() &&
-           "PredicateInfo consumer did not remove all SSA copies.");
-    F->eraseFromParent();
-  }
-}
-
 std::optional<PredicateConstraint> PredicateBase::getConstraint() const {
   switch (Type) {
   case PT_Assume:
@@ -741,6 +717,11 @@ std::optional<PredicateConstraint> PredicateBase::getConstraint() const {
                         : ConstantInt::getFalse(Condition->getType())}};
     }
 
+    if (match(Condition, m_NUWTrunc(m_Specific(RenamedOp)))) {
+      return {{TrueEdge ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ,
+               ConstantInt::getNullValue(RenamedOp->getType())}};
+    }
+
     CmpInst *Cmp = dyn_cast<CmpInst>(Condition);
     if (!Cmp) {
       // TODO: Make this an assertion once RenamedOp is fully accurate.
@@ -779,15 +760,16 @@ std::optional<PredicateConstraint> PredicateBase::getConstraint() const {
 
 void PredicateInfo::verifyPredicateInfo() const {}
 
-// Replace ssa_copy calls created by PredicateInfo with their operand.
+// Replace bitcasts created by PredicateInfo with their operand.
 static void replaceCreatedSSACopys(PredicateInfo &PredInfo, Function &F) {
   for (Instruction &Inst : llvm::make_early_inc_range(instructions(F))) {
     const auto *PI = PredInfo.getPredicateInfoFor(&Inst);
-    auto *II = dyn_cast<IntrinsicInst>(&Inst);
-    if (!PI || !II || II->getIntrinsicID() != Intrinsic::ssa_copy)
+    if (!PI)
       continue;
 
-    Inst.replaceAllUsesWith(II->getOperand(0));
+    assert(isa<BitCastInst>(Inst) &&
+           Inst.getType() == Inst.getOperand(0)->getType());
+    Inst.replaceAllUsesWith(Inst.getOperand(0));
     Inst.eraseFromParent();
   }
 }
diff --git a/llvm/lib/Transforms/Utils/ProfileVerify.cpp b/llvm/lib/Transforms/Utils/ProfileVerify.cpp
index 0ffea3f53fef6..41647f7717a43 100644
--- a/llvm/lib/Transforms/Utils/ProfileVerify.cpp
+++ b/llvm/lib/Transforms/Utils/ProfileVerify.cpp
@@ -8,10 +8,8 @@
 
 #include "llvm/Transforms/Utils/ProfileVerify.h"
 #include "llvm/ADT/DynamicAPInt.h"
-#include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
-#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IR/Analysis.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index d96f1d6c23d47..10c162bc6463a 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -136,7 +136,7 @@ class AssignmentTrackingInfo {
   /// \p ToDelete that stores to this alloca.
   void updateForDeletedStore(
       StoreInst *ToDelete, DIBuilder &DIB,
-      SmallSet<DbgVariableRecord *, 8> *DVRAssignsToDelete) const {
+      SmallPtrSet<DbgVariableRecord *, 8> *DVRAssignsToDelete) const {
     // There's nothing to do if the alloca doesn't have any variables using
     // assignment tracking.
     if (DVRAssigns.empty())
@@ -382,7 +382,7 @@ struct PromoteMem2Reg {
   SmallVector<AssignmentTrackingInfo, 8> AllocaATInfo;
   /// A set of dbg.assigns to delete because they've been demoted to
   /// dbg.values. Call cleanUpDbgAssigns to delete them.
-  SmallSet<DbgVariableRecord *, 8> DVRAssignsToDelete;
+  SmallPtrSet<DbgVariableRecord *, 8> DVRAssignsToDelete;
 
   /// The set of basic blocks the renamer has already visited.
   BitVector Visited;
@@ -533,11 +533,10 @@ static void removeIntrinsicUsers(AllocaInst *AI) {
 /// false there were some loads which were not dominated by the single store
 /// and thus must be phi-ed with undef. We fall back to the standard alloca
 /// promotion algorithm in that case.
-static bool
-rewriteSingleStoreAlloca(AllocaInst *AI, AllocaInfo &Info, LargeBlockInfo &LBI,
-                         const DataLayout &DL, DominatorTree &DT,
-                         AssumptionCache *AC,
-                         SmallSet<DbgVariableRecord *, 8> *DVRAssignsToDelete) {
+static bool rewriteSingleStoreAlloca(
+    AllocaInst *AI, AllocaInfo &Info, LargeBlockInfo &LBI, const DataLayout &DL,
+    DominatorTree &DT, AssumptionCache *AC,
+    SmallPtrSet<DbgVariableRecord *, 8> *DVRAssignsToDelete) {
   StoreInst *OnlyStore = Info.OnlyStore;
   Value *ReplVal = OnlyStore->getOperand(0);
   // Loads may either load the stored value or uninitialized memory (undef).
@@ -647,11 +646,10 @@ rewriteSingleStoreAlloca(AllocaInst *AI, AllocaInfo &Info, LargeBlockInfo &LBI,
 ///      use(t);
 ///    *A = 42;
 ///  }
-static bool
-promoteSingleBlockAlloca(AllocaInst *AI, const AllocaInfo &Info,
-                         LargeBlockInfo &LBI, const DataLayout &DL,
-                         DominatorTree &DT, AssumptionCache *AC,
-                         SmallSet<DbgVariableRecord *, 8> *DVRAssignsToDelete) {
+static bool promoteSingleBlockAlloca(
+    AllocaInst *AI, const AllocaInfo &Info, LargeBlockInfo &LBI,
+    const DataLayout &DL, DominatorTree &DT, AssumptionCache *AC,
+    SmallPtrSet<DbgVariableRecord *, 8> *DVRAssignsToDelete) {
   // The trickiest case to handle is when we have large blocks. Because of this,
   // this code is optimized assuming that large blocks happen.  This does not
   // significantly pessimize the small block case.  This uses LargeBlockInfo to
diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index b78c7022b9be0..84485176ad4ff 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -777,10 +777,10 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
 
     for (BasicBlock &BB : F) {
       for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
-        if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
-          if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
+        if (auto *BC = dyn_cast<BitCastInst>(&Inst)) {
+          if (BC->getType() == BC->getOperand(0)->getType()) {
             if (It->second->getPredicateInfoFor(&Inst)) {
-              Value *Op = II->getOperand(0);
+              Value *Op = BC->getOperand(0);
               Inst.replaceAllUsesWith(Op);
               Inst.eraseFromParent();
             }
@@ -1413,6 +1413,15 @@ void SCCPInstVisitor::visitCastInst(CastInst &I) {
   if (ValueState[&I].isOverdefined())
     return;
 
+  if (auto *BC = dyn_cast<BitCastInst>(&I)) {
+    if (BC->getType() == BC->getOperand(0)->getType()) {
+      if (const PredicateBase *PI = getPredicateInfoFor(&I)) {
+        handlePredicate(&I, I.getOperand(0), PI);
+        return;
+      }
+    }
+  }
+
   ValueLatticeElement OpSt = getValueState(I.getOperand(0));
   if (OpSt.isUnknownOrUndef())
     return;
@@ -1433,8 +1442,12 @@ void SCCPInstVisitor::visitCastInst(CastInst &I) {
         OpSt.asConstantRange(I.getSrcTy(), /*UndefAllowed=*/false);
 
     Type *DestTy = I.getDestTy();
-    ConstantRange Res =
-        OpRange.castOp(I.getOpcode(), DestTy->getScalarSizeInBits());
+    ConstantRange Res = ConstantRange::getEmpty(DestTy->getScalarSizeInBits());
+    if (auto *Trunc = dyn_cast<TruncInst>(&I))
+      Res = OpRange.truncate(DestTy->getScalarSizeInBits(),
+                             Trunc->getNoWrapKind());
+    else
+      Res = OpRange.castOp(I.getOpcode(), DestTy->getScalarSizeInBits());
     mergeInValue(LV, &I, ValueLatticeElement::getRange(Res));
   } else
     markOverdefined(&I);
@@ -2001,17 +2014,6 @@ void SCCPInstVisitor::handleCallResult(CallBase &CB) {
   Function *F = CB.getCalledFunction();
 
   if (auto *II = dyn_cast<IntrinsicInst>(&CB)) {
-    if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
-      if (ValueState[&CB].isOverdefined())
-        return;
-
-      Value *CopyOf = CB.getOperand(0);
-      const PredicateBase *PI = getPredicateInfoFor(&CB);
-      assert(PI && "Missing predicate info for ssa.copy");
-      handlePredicate(&CB, CopyOf, PI);
-      return;
-    }
-
     if (II->getIntrinsicID() == Intrinsic::vscale) {
       unsigned BitWidth = CB.getType()->getScalarSizeInBits();
       const ConstantRange Result = getVScaleRange(II->getFunction(), BitWidth);
diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
index 1eb8996fca031..060ca92e559a3 100644
--- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp
@@ -15,7 +15,6 @@
 #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
@@ -1346,7 +1345,7 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
     CanonicalIV->insertBefore(Header->begin());
     rememberInstruction(CanonicalIV);
 
-    SmallSet<BasicBlock *, 4> PredSeen;
+    SmallPtrSet<BasicBlock *, 4> PredSeen;
     Constant *One = ConstantInt::get(Ty, 1);
     for (pred_iterator HPI = HPB; HPI != HPE; ++HPI) {
       BasicBlock *HP = *HPI;
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index deabacc592c7f..7a538ae2c583f 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -291,6 +291,7 @@ class SimplifyCFGOpt {
   bool simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder);
   bool simplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder);
   bool simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder);
+  bool foldCondBranchOnValueKnownInPredecessor(BranchInst *BI);
 
   bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
                                              IRBuilder<> &Builder);
@@ -564,9 +565,23 @@ struct ConstantComparesGatherer {
   /// Number of comparisons matched in the and/or chain
   unsigned UsedICmps = 0;
 
+  /// If the elements in Vals matches the comparisons
+  bool IsEq = false;
+
+  // Used to check if the first matched CompValue shall be the Extra check.
+  bool IgnoreFirstMatch = false;
+  bool MultipleMatches = false;
+
   /// Construct and compute the result for the comparison instruction Cond
   ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) {
     gather(Cond);
+    if (CompValue || !MultipleMatches)
+      return;
+    Extra = nullptr;
+    Vals.clear();
+    UsedICmps = 0;
+    IgnoreFirstMatch = true;
+    gather(Cond);
   }
 
   ConstantComparesGatherer(const ConstantComparesGatherer &) = delete;
@@ -577,10 +592,16 @@ struct ConstantComparesGatherer {
   /// Try to set the current value used for the comparison, it succeeds only if
   /// it wasn't set before or if the new value is the same as the old one
   bool setValueOnce(Value *NewVal) {
-    if (CompValue && CompValue != NewVal)
+    if (IgnoreFirstMatch) {
+      IgnoreFirstMatch = false;
+      return false;
+    }
+    if (CompValue && CompValue != NewVal) {
+      MultipleMatches = true;
       return false;
+    }
     CompValue = NewVal;
-    return (CompValue != nullptr);
+    return true;
   }
 
   /// Try to match Instruction "I" as a comparison against a constant and
@@ -691,7 +712,7 @@ struct ConstantComparesGatherer {
 
       UsedICmps++;
       Vals.push_back(C);
-      return ICI->getOperand(0);
+      return true;
     }
 
     // If we have "x ult 3", for example, then we can add 0,1,2 to the set.
@@ -735,23 +756,23 @@ struct ConstantComparesGatherer {
   /// vector.
   /// One "Extra" case is allowed to differ from the other.
   void gather(Value *V) {
-    bool isEQ = match(V, m_LogicalOr(m_Value(), m_Value()));
-
+    Value *Op0, *Op1;
+    if (match(V, m_LogicalOr(m_Value(Op0), m_Value(Op1))))
+      IsEq = true;
+    else if (match(V, m_LogicalAnd(m_Value(Op0), m_Value(Op1))))
+      IsEq = false;
+    else
+      return;
     // Keep a stack (SmallVector for efficiency) for depth-first traversal
-    SmallVector<Value *, 8> DFT;
-    SmallPtrSet<Value *, 8> Visited;
-
-    // Initialize
-    Visited.insert(V);
-    DFT.push_back(V);
+    SmallVector<Value *, 8> DFT{Op0, Op1};
+    SmallPtrSet<Value *, 8> Visited{V, Op0, Op1};
 
     while (!DFT.empty()) {
       V = DFT.pop_back_val();
 
       if (Instruction *I = dyn_cast<Instruction>(V)) {
         // If it is a || (or && depending on isEQ), process the operands.
-        Value *Op0, *Op1;
-        if (isEQ ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1)))
+        if (IsEq ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1)))
                  : match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
           if (Visited.insert(Op1).second)
             DFT.push_back(Op1);
@@ -762,7 +783,7 @@ struct ConstantComparesGatherer {
         }
 
         // Try to match the current instruction
-        if (matchInstruction(I, isEQ))
+        if (matchInstruction(I, IsEq))
           // Match succeed, continue the loop
           continue;
       }
@@ -810,11 +831,15 @@ Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) {
     if (!SI->getParent()->hasNPredecessorsOrMore(128 / SI->getNumSuccessors()))
       CV = SI->getCondition();
   } else if (BranchInst *BI = dyn_cast<BranchInst>(TI))
-    if (BI->isConditional() && BI->getCondition()->hasOneUse())
+    if (BI->isConditional() && BI->getCondition()->hasOneUse()) {
       if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) {
         if (ICI->isEquality() && getConstantInt(ICI->getOperand(1), DL))
           CV = ICI->getOperand(0);
+      } else if (auto *Trunc = dyn_cast<TruncInst>(BI->getCondition())) {
+        if (Trunc->hasNoUnsignedWrap())
+          CV = Trunc->getOperand(0);
       }
+    }
 
   // Unwrap any lossless ptrtoint cast.
   if (CV) {
@@ -840,11 +865,20 @@ BasicBlock *SimplifyCFGOpt::getValueEqualityComparisonCases(
   }
 
   BranchInst *BI = cast<BranchInst>(TI);
-  ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
-  BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE);
-  Cases.push_back(ValueEqualityComparisonCase(
-      getConstantInt(ICI->getOperand(1), DL), Succ));
-  return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ);
+  Value *Cond = BI->getCondition();
+  ICmpInst::Predicate Pred;
+  ConstantInt *C;
+  if (auto *ICI = dyn_cast<ICmpInst>(Cond)) {
+    Pred = ICI->getPredicate();
+    C = getConstantInt(ICI->getOperand(1), DL);
+  } else {
+    Pred = ICmpInst::ICMP_NE;
+    auto *Trunc = cast<TruncInst>(Cond);
+    C = ConstantInt::get(cast<IntegerType>(Trunc->getOperand(0)->getType()), 0);
+  }
+  BasicBlock *Succ = BI->getSuccessor(Pred == ICmpInst::ICMP_NE);
+  Cases.push_back(ValueEqualityComparisonCase(C, Succ));
+  return BI->getSuccessor(Pred == ICmpInst::ICMP_EQ);
 }
 
 /// Given a vector of bb/value pairs, remove any entries
@@ -1106,7 +1140,10 @@ static void getBranchWeights(Instruction *TI,
   // default weight to be the first entry.
   if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
     assert(Weights.size() == 2);
-    ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
+    auto *ICI = dyn_cast<ICmpInst>(BI->getCondition());
+    if (!ICI)
+      return;
+
     if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
       std::swap(Weights.front(), Weights.back());
   }
@@ -3321,12 +3358,10 @@ bool SimplifyCFGOpt::speculativelyExecuteBB(BranchInst *BI,
     //   %merge = select %cond, %two, %one
     //   store %merge, %x.dest, !DIAssignID !2
     //   dbg.assign %merge, "x", ..., !2
-    auto replaceVariable = [OrigV, S](auto *DbgAssign) {
+    for (DbgVariableRecord *DbgAssign :
+         at::getDVRAssignmentMarkers(SpeculatedStore))
       if (llvm::is_contained(DbgAssign->location_ops(), OrigV))
         DbgAssign->replaceVariableLocationOp(OrigV, S);
-    };
-    for_each(at::getAssignmentMarkers(SpeculatedStore), replaceVariable);
-    for_each(at::getDVRAssignmentMarkers(SpeculatedStore), replaceVariable);
   }
 
   // Metadata can be dependent on the condition we are hoisting above.
@@ -3655,15 +3690,19 @@ foldCondBranchOnValueKnownInPredecessorImpl(BranchInst *BI, DomTreeUpdater *DTU,
   return false;
 }
 
-static bool foldCondBranchOnValueKnownInPredecessor(BranchInst *BI,
-                                                    DomTreeUpdater *DTU,
-                                                    const DataLayout &DL,
-                                                    AssumptionCache *AC) {
+bool SimplifyCFGOpt::foldCondBranchOnValueKnownInPredecessor(BranchInst *BI) {
+  // Note: If BB is a loop header then there is a risk that threading introduces
+  // a non-canonical loop by moving a back edge. So we avoid this optimization
+  // for loop headers if NeedCanonicalLoop is set.
+  if (Options.NeedCanonicalLoop && is_contained(LoopHeaders, BI->getParent()))
+    return false;
+
   std::optional<bool> Result;
   bool EverChanged = false;
   do {
     // Note that None means "we changed things, but recurse further."
-    Result = foldCondBranchOnValueKnownInPredecessorImpl(BI, DTU, DL, AC);
+    Result =
+        foldCondBranchOnValueKnownInPredecessorImpl(BI, DTU, DL, Options.AC);
     EverChanged |= Result == std::nullopt || *Result;
   } while (Result == std::nullopt);
   return EverChanged;
@@ -5084,6 +5123,7 @@ bool SimplifyCFGOpt::simplifyBranchOnICmpChain(BranchInst *BI,
   Value *CompVal = ConstantCompare.CompValue;
   unsigned UsedICmps = ConstantCompare.UsedICmps;
   Value *ExtraCase = ConstantCompare.Extra;
+  bool TrueWhenEqual = ConstantCompare.IsEq;
 
   // If we didn't have a multiply compared value, fail.
   if (!CompVal)
@@ -5093,8 +5133,6 @@ bool SimplifyCFGOpt::simplifyBranchOnICmpChain(BranchInst *BI,
   if (UsedICmps <= 1)
     return false;
 
-  bool TrueWhenEqual = match(Cond, m_LogicalOr(m_Value(), m_Value()));
-
   // There might be duplicate constants in the list, which the switch
   // instruction can't handle, remove them now.
   array_pod_sort(Values.begin(), Values.end(), constantIntSortPredicate);
@@ -8085,7 +8123,7 @@ bool SimplifyCFGOpt::simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
   // If this is a branch on something for which we know the constant value in
   // predecessors (e.g. a phi node in the current block), thread control
   // through this block.
-  if (foldCondBranchOnValueKnownInPredecessor(BI, DTU, DL, Options.AC))
+  if (foldCondBranchOnValueKnownInPredecessor(BI))
     return requestResimplify();
 
   // Scan predecessor blocks for conditional branches.
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 737321daa9109..2d6a748f45079 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -20,6 +20,7 @@
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/Utils/Local.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/AttributeMask.h"
 #include "llvm/IR/DataLayout.h"
@@ -319,10 +320,10 @@ static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> A
     annotateDereferenceableBytes(CI, ArgNos, LenC->getZExtValue());
   } else if (isKnownNonZero(Size, DL)) {
     annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
-    const APInt *X, *Y;
+    uint64_t X, Y;
     uint64_t DerefMin = 1;
-    if (match(Size, m_Select(m_Value(), m_APInt(X), m_APInt(Y)))) {
-      DerefMin = std::min(X->getZExtValue(), Y->getZExtValue());
+    if (match(Size, m_Select(m_Value(), m_ConstantInt(X), m_ConstantInt(Y)))) {
+      DerefMin = std::min(X, Y);
       annotateDereferenceableBytes(CI, ArgNos, DerefMin);
     }
   }
@@ -977,8 +978,14 @@ Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
   // it's not very useful because calling strlen for a pointer of other types is
   // very uncommon.
   if (GEPOperator *GEP = dyn_cast<GEPOperator>(Src)) {
-    // TODO: Handle subobjects.
-    if (!isGEPBasedOnPointerToString(GEP, CharSize))
+    unsigned BW = DL.getIndexTypeSizeInBits(GEP->getType());
+    SmallMapVector<Value *, APInt, 4> VarOffsets;
+    APInt ConstOffset(BW, 0);
+    assert(CharSize % 8 == 0 && "Expected a multiple of 8 sized CharSize");
+    // Check the gep is a single variable offset.
+    if (!GEP->collectOffset(DL, BW, VarOffsets, ConstOffset) ||
+        VarOffsets.size() != 1 || ConstOffset != 0 ||
+        VarOffsets.begin()->second != CharSize / 8)
       return nullptr;
 
     ConstantDataArraySlice Slice;
@@ -1000,10 +1007,8 @@ Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
           return nullptr;
       }
 
-      Value *Offset = GEP->getOperand(2);
+      Value *Offset = VarOffsets.begin()->first;
       KnownBits Known = computeKnownBits(Offset, DL, nullptr, CI, nullptr);
-      uint64_t ArrSize =
-             cast<ArrayType>(GEP->getSourceElementType())->getNumElements();
 
       // If Offset is not provably in the range [0, NullTermIdx], we can still
       // optimize if we can prove that the program has undefined behavior when
@@ -1011,7 +1016,7 @@ Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
       // is a pointer to an object whose memory extent is NullTermIdx+1.
       if ((Known.isNonNegative() && Known.getMaxValue().ule(NullTermIdx)) ||
           (isa<GlobalVariable>(GEP->getOperand(0)) &&
-           NullTermIdx == ArrSize - 1)) {
+           NullTermIdx == Slice.Length - 1)) {
         Offset = B.CreateSExtOrTrunc(Offset, CI->getType());
         return B.CreateSub(ConstantInt::get(CI->getType(), NullTermIdx),
                            Offset);
diff --git a/llvm/lib/Transforms/Utils/SplitModuleByCategory.cpp b/llvm/lib/Transforms/Utils/SplitModuleByCategory.cpp
index 6b18ecee98ec5..c3ac39e5c287f 100644
--- a/llvm/lib/Transforms/Utils/SplitModuleByCategory.cpp
+++ b/llvm/lib/Transforms/Utils/SplitModuleByCategory.cpp
@@ -12,7 +12,6 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
@@ -21,7 +20,6 @@
 #include "llvm/Transforms/Utils/Cloning.h"
 
 #include <map>
-#include <string>
 #include <utility>
 
 using namespace llvm;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index c47fd9421fddd..789047a2a28e7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -793,280 +793,296 @@ static bool canWidenCallReturnType(Type *Ty) {
 }
 
 bool LoopVectorizationLegality::canVectorizeInstrs() {
-  BasicBlock *Header = TheLoop->getHeader();
+  bool DoExtraAnalysis = ORE->allowExtraAnalysis(DEBUG_TYPE);
+  bool Result = true;
 
   // For each block in the loop.
   for (BasicBlock *BB : TheLoop->blocks()) {
     // Scan the instructions in the block and look for hazards.
     for (Instruction &I : *BB) {
-      if (auto *Phi = dyn_cast<PHINode>(&I)) {
-        Type *PhiTy = Phi->getType();
-        // Check that this PHI type is allowed.
-        if (!PhiTy->isIntegerTy() && !PhiTy->isFloatingPointTy() &&
-            !PhiTy->isPointerTy()) {
-          reportVectorizationFailure("Found a non-int non-pointer PHI",
-                                     "loop control flow is not understood by vectorizer",
-                                     "CFGNotUnderstood", ORE, TheLoop);
-          return false;
-        }
+      Result &= canVectorizeInstr(I);
+      if (!DoExtraAnalysis && !Result)
+        return false;
+    }
+  }
 
-        // If this PHINode is not in the header block, then we know that we
-        // can convert it to select during if-conversion. No need to check if
-        // the PHIs in this block are induction or reduction variables.
-        if (BB != Header) {
-          // Non-header phi nodes that have outside uses can be vectorized. Add
-          // them to the list of allowed exits.
-          // Unsafe cyclic dependencies with header phis are identified during
-          // legalization for reduction, induction and fixed order
-          // recurrences.
-          AllowedExit.insert(&I);
-          continue;
-        }
+  if (!PrimaryInduction) {
+    if (Inductions.empty()) {
+      reportVectorizationFailure(
+          "Did not find one integer induction var",
+          "loop induction variable could not be identified",
+          "NoInductionVariable", ORE, TheLoop);
+      return false;
+    }
+    if (!WidestIndTy) {
+      reportVectorizationFailure(
+          "Did not find one integer induction var",
+          "integer loop induction variable could not be identified",
+          "NoIntegerInductionVariable", ORE, TheLoop);
+      return false;
+    }
+    LLVM_DEBUG(dbgs() << "LV: Did not find one integer induction var.\n");
+  }
 
-        // We only allow if-converted PHIs with exactly two incoming values.
-        if (Phi->getNumIncomingValues() != 2) {
-          reportVectorizationFailure("Found an invalid PHI",
-              "loop control flow is not understood by vectorizer",
-              "CFGNotUnderstood", ORE, TheLoop, Phi);
-          return false;
-        }
+  // Now we know the widest induction type, check if our found induction
+  // is the same size. If it's not, unset it here and InnerLoopVectorizer
+  // will create another.
+  if (PrimaryInduction && WidestIndTy != PrimaryInduction->getType())
+    PrimaryInduction = nullptr;
 
-        RecurrenceDescriptor RedDes;
-        if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes, DB, AC,
-                                                 DT, PSE.getSE())) {
-          Requirements->addExactFPMathInst(RedDes.getExactFPMathInst());
-          AllowedExit.insert(RedDes.getLoopExitInstr());
-          Reductions[Phi] = RedDes;
-          continue;
-        }
+  return Result;
+}
 
-        // We prevent matching non-constant strided pointer IVS to preserve
-        // historical vectorizer behavior after a generalization of the
-        // IVDescriptor code.  The intent is to remove this check, but we
-        // have to fix issues around code quality for such loops first.
-        auto IsDisallowedStridedPointerInduction =
-            [](const InductionDescriptor &ID) {
-              if (AllowStridedPointerIVs)
-                return false;
-              return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
-                     ID.getConstIntStepValue() == nullptr;
-            };
-
-        // TODO: Instead of recording the AllowedExit, it would be good to
-        // record the complementary set: NotAllowedExit. These include (but may
-        // not be limited to):
-        // 1. Reduction phis as they represent the one-before-last value, which
-        // is not available when vectorized
-        // 2. Induction phis and increment when SCEV predicates cannot be used
-        // outside the loop - see addInductionPhi
-        // 3. Non-Phis with outside uses when SCEV predicates cannot be used
-        // outside the loop - see call to hasOutsideLoopUser in the non-phi
-        // handling below
-        // 4. FixedOrderRecurrence phis that can possibly be handled by
-        // extraction.
-        // By recording these, we can then reason about ways to vectorize each
-        // of these NotAllowedExit.
-        InductionDescriptor ID;
-        if (InductionDescriptor::isInductionPHI(Phi, TheLoop, PSE, ID) &&
-            !IsDisallowedStridedPointerInduction(ID)) {
-          addInductionPhi(Phi, ID, AllowedExit);
-          Requirements->addExactFPMathInst(ID.getExactFPMathInst());
-          continue;
-        }
+bool LoopVectorizationLegality::canVectorizeInstr(Instruction &I) {
+  BasicBlock *BB = I.getParent();
+  BasicBlock *Header = TheLoop->getHeader();
 
-        if (RecurrenceDescriptor::isFixedOrderRecurrence(Phi, TheLoop, DT)) {
-          AllowedExit.insert(Phi);
-          FixedOrderRecurrences.insert(Phi);
-          continue;
-        }
+  if (auto *Phi = dyn_cast<PHINode>(&I)) {
+    Type *PhiTy = Phi->getType();
+    // Check that this PHI type is allowed.
+    if (!PhiTy->isIntegerTy() && !PhiTy->isFloatingPointTy() &&
+        !PhiTy->isPointerTy()) {
+      reportVectorizationFailure(
+          "Found a non-int non-pointer PHI",
+          "loop control flow is not understood by vectorizer",
+          "CFGNotUnderstood", ORE, TheLoop);
+      return false;
+    }
 
-        // As a last resort, coerce the PHI to a AddRec expression
-        // and re-try classifying it a an induction PHI.
-        if (InductionDescriptor::isInductionPHI(Phi, TheLoop, PSE, ID, true) &&
-            !IsDisallowedStridedPointerInduction(ID)) {
-          addInductionPhi(Phi, ID, AllowedExit);
-          continue;
-        }
+    // If this PHINode is not in the header block, then we know that we
+    // can convert it to select during if-conversion. No need to check if
+    // the PHIs in this block are induction or reduction variables.
+    if (BB != Header) {
+      // Non-header phi nodes that have outside uses can be vectorized. Add
+      // them to the list of allowed exits.
+      // Unsafe cyclic dependencies with header phis are identified during
+      // legalization for reduction, induction and fixed order
+      // recurrences.
+      AllowedExit.insert(&I);
+      return true;
+    }
 
-        reportVectorizationFailure("Found an unidentified PHI",
-            "value that could not be identified as "
-            "reduction is used outside the loop",
-            "NonReductionValueUsedOutsideLoop", ORE, TheLoop, Phi);
-        return false;
-      } // end of PHI handling
-
-      // We handle calls that:
-      //   * Have a mapping to an IR intrinsic.
-      //   * Have a vector version available.
-      auto *CI = dyn_cast<CallInst>(&I);
-
-      if (CI && !getVectorIntrinsicIDForCall(CI, TLI) &&
-          !(CI->getCalledFunction() && TLI &&
-            (!VFDatabase::getMappings(*CI).empty() ||
-             isTLIScalarize(*TLI, *CI)))) {
-        // If the call is a recognized math libary call, it is likely that
-        // we can vectorize it given loosened floating-point constraints.
-        LibFunc Func;
-        bool IsMathLibCall =
-            TLI && CI->getCalledFunction() &&
-            CI->getType()->isFloatingPointTy() &&
-            TLI->getLibFunc(CI->getCalledFunction()->getName(), Func) &&
-            TLI->hasOptimizedCodeGen(Func);
-
-        if (IsMathLibCall) {
-          // TODO: Ideally, we should not use clang-specific language here,
-          // but it's hard to provide meaningful yet generic advice.
-          // Also, should this be guarded by allowExtraAnalysis() and/or be part
-          // of the returned info from isFunctionVectorizable()?
-          reportVectorizationFailure(
-              "Found a non-intrinsic callsite",
-              "library call cannot be vectorized. "
-              "Try compiling with -fno-math-errno, -ffast-math, "
-              "or similar flags",
-              "CantVectorizeLibcall", ORE, TheLoop, CI);
-        } else {
-          reportVectorizationFailure("Found a non-intrinsic callsite",
-                                     "call instruction cannot be vectorized",
-                                     "CantVectorizeLibcall", ORE, TheLoop, CI);
-        }
-        return false;
-      }
+    // We only allow if-converted PHIs with exactly two incoming values.
+    if (Phi->getNumIncomingValues() != 2) {
+      reportVectorizationFailure(
+          "Found an invalid PHI",
+          "loop control flow is not understood by vectorizer",
+          "CFGNotUnderstood", ORE, TheLoop, Phi);
+      return false;
+    }
 
-      // Some intrinsics have scalar arguments and should be same in order for
-      // them to be vectorized (i.e. loop invariant).
-      if (CI) {
-        auto *SE = PSE.getSE();
-        Intrinsic::ID IntrinID = getVectorIntrinsicIDForCall(CI, TLI);
-        for (unsigned Idx = 0; Idx < CI->arg_size(); ++Idx)
-          if (isVectorIntrinsicWithScalarOpAtArg(IntrinID, Idx, TTI)) {
-            if (!SE->isLoopInvariant(PSE.getSCEV(CI->getOperand(Idx)),
-                                     TheLoop)) {
-              reportVectorizationFailure("Found unvectorizable intrinsic",
-                  "intrinsic instruction cannot be vectorized",
-                  "CantVectorizeIntrinsic", ORE, TheLoop, CI);
-              return false;
-            }
-          }
-      }
+    RecurrenceDescriptor RedDes;
+    if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, RedDes, DB, AC, DT,
+                                             PSE.getSE())) {
+      Requirements->addExactFPMathInst(RedDes.getExactFPMathInst());
+      AllowedExit.insert(RedDes.getLoopExitInstr());
+      Reductions[Phi] = RedDes;
+      return true;
+    }
 
-      // If we found a vectorized variant of a function, note that so LV can
-      // make better decisions about maximum VF.
-      if (CI && !VFDatabase::getMappings(*CI).empty())
-        VecCallVariantsFound = true;
-
-      auto CanWidenInstructionTy = [](Instruction const &Inst) {
-        Type *InstTy = Inst.getType();
-        if (!isa<StructType>(InstTy))
-          return canVectorizeTy(InstTy);
-
-        // For now, we only recognize struct values returned from calls where
-        // all users are extractvalue as vectorizable. All element types of the
-        // struct must be types that can be widened.
-        return isa<CallInst>(Inst) && canWidenCallReturnType(InstTy) &&
-               all_of(Inst.users(), IsaPred<ExtractValueInst>);
-      };
+    // We prevent matching non-constant strided pointer IVS to preserve
+    // historical vectorizer behavior after a generalization of the
+    // IVDescriptor code.  The intent is to remove this check, but we
+    // have to fix issues around code quality for such loops first.
+    auto IsDisallowedStridedPointerInduction =
+        [](const InductionDescriptor &ID) {
+          if (AllowStridedPointerIVs)
+            return false;
+          return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
+                 ID.getConstIntStepValue() == nullptr;
+        };
+
+    // TODO: Instead of recording the AllowedExit, it would be good to
+    // record the complementary set: NotAllowedExit. These include (but may
+    // not be limited to):
+    // 1. Reduction phis as they represent the one-before-last value, which
+    // is not available when vectorized
+    // 2. Induction phis and increment when SCEV predicates cannot be used
+    // outside the loop - see addInductionPhi
+    // 3. Non-Phis with outside uses when SCEV predicates cannot be used
+    // outside the loop - see call to hasOutsideLoopUser in the non-phi
+    // handling below
+    // 4. FixedOrderRecurrence phis that can possibly be handled by
+    // extraction.
+    // By recording these, we can then reason about ways to vectorize each
+    // of these NotAllowedExit.
+    InductionDescriptor ID;
+    if (InductionDescriptor::isInductionPHI(Phi, TheLoop, PSE, ID) &&
+        !IsDisallowedStridedPointerInduction(ID)) {
+      addInductionPhi(Phi, ID, AllowedExit);
+      Requirements->addExactFPMathInst(ID.getExactFPMathInst());
+      return true;
+    }
 
-      // Check that the instruction return type is vectorizable.
-      // We can't vectorize casts from vector type to scalar type.
-      // Also, we can't vectorize extractelement instructions.
-      if (!CanWidenInstructionTy(I) ||
-          (isa<CastInst>(I) &&
-           !VectorType::isValidElementType(I.getOperand(0)->getType())) ||
-          isa<ExtractElementInst>(I)) {
-        reportVectorizationFailure("Found unvectorizable type",
-            "instruction return type cannot be vectorized",
-            "CantVectorizeInstructionReturnType", ORE, TheLoop, &I);
-        return false;
-      }
+    if (RecurrenceDescriptor::isFixedOrderRecurrence(Phi, TheLoop, DT)) {
+      AllowedExit.insert(Phi);
+      FixedOrderRecurrences.insert(Phi);
+      return true;
+    }
+
+    // As a last resort, coerce the PHI to a AddRec expression
+    // and re-try classifying it a an induction PHI.
+    if (InductionDescriptor::isInductionPHI(Phi, TheLoop, PSE, ID, true) &&
+        !IsDisallowedStridedPointerInduction(ID)) {
+      addInductionPhi(Phi, ID, AllowedExit);
+      return true;
+    }
 
-      // Check that the stored type is vectorizable.
-      if (auto *ST = dyn_cast<StoreInst>(&I)) {
-        Type *T = ST->getValueOperand()->getType();
-        if (!VectorType::isValidElementType(T)) {
-          reportVectorizationFailure("Store instruction cannot be vectorized",
-                                     "CantVectorizeStore", ORE, TheLoop, ST);
+    reportVectorizationFailure("Found an unidentified PHI",
+                               "value that could not be identified as "
+                               "reduction is used outside the loop",
+                               "NonReductionValueUsedOutsideLoop", ORE, TheLoop,
+                               Phi);
+    return false;
+  } // end of PHI handling
+
+  // We handle calls that:
+  //   * Have a mapping to an IR intrinsic.
+  //   * Have a vector version available.
+  auto *CI = dyn_cast<CallInst>(&I);
+
+  if (CI && !getVectorIntrinsicIDForCall(CI, TLI) &&
+      !(CI->getCalledFunction() && TLI &&
+        (!VFDatabase::getMappings(*CI).empty() || isTLIScalarize(*TLI, *CI)))) {
+    // If the call is a recognized math libary call, it is likely that
+    // we can vectorize it given loosened floating-point constraints.
+    LibFunc Func;
+    bool IsMathLibCall =
+        TLI && CI->getCalledFunction() && CI->getType()->isFloatingPointTy() &&
+        TLI->getLibFunc(CI->getCalledFunction()->getName(), Func) &&
+        TLI->hasOptimizedCodeGen(Func);
+
+    if (IsMathLibCall) {
+      // TODO: Ideally, we should not use clang-specific language here,
+      // but it's hard to provide meaningful yet generic advice.
+      // Also, should this be guarded by allowExtraAnalysis() and/or be part
+      // of the returned info from isFunctionVectorizable()?
+      reportVectorizationFailure(
+          "Found a non-intrinsic callsite",
+          "library call cannot be vectorized. "
+          "Try compiling with -fno-math-errno, -ffast-math, "
+          "or similar flags",
+          "CantVectorizeLibcall", ORE, TheLoop, CI);
+    } else {
+      reportVectorizationFailure("Found a non-intrinsic callsite",
+                                 "call instruction cannot be vectorized",
+                                 "CantVectorizeLibcall", ORE, TheLoop, CI);
+    }
+    return false;
+  }
+
+  // Some intrinsics have scalar arguments and should be same in order for
+  // them to be vectorized (i.e. loop invariant).
+  if (CI) {
+    auto *SE = PSE.getSE();
+    Intrinsic::ID IntrinID = getVectorIntrinsicIDForCall(CI, TLI);
+    for (unsigned Idx = 0; Idx < CI->arg_size(); ++Idx)
+      if (isVectorIntrinsicWithScalarOpAtArg(IntrinID, Idx, TTI)) {
+        if (!SE->isLoopInvariant(PSE.getSCEV(CI->getOperand(Idx)), TheLoop)) {
+          reportVectorizationFailure(
+              "Found unvectorizable intrinsic",
+              "intrinsic instruction cannot be vectorized",
+              "CantVectorizeIntrinsic", ORE, TheLoop, CI);
           return false;
         }
+      }
+  }
 
-        // For nontemporal stores, check that a nontemporal vector version is
-        // supported on the target.
-        if (ST->getMetadata(LLVMContext::MD_nontemporal)) {
-          // Arbitrarily try a vector of 2 elements.
-          auto *VecTy = FixedVectorType::get(T, /*NumElts=*/2);
-          assert(VecTy && "did not find vectorized version of stored type");
-          if (!TTI->isLegalNTStore(VecTy, ST->getAlign())) {
-            reportVectorizationFailure(
-                "nontemporal store instruction cannot be vectorized",
-                "CantVectorizeNontemporalStore", ORE, TheLoop, ST);
-            return false;
-          }
-        }
+  // If we found a vectorized variant of a function, note that so LV can
+  // make better decisions about maximum VF.
+  if (CI && !VFDatabase::getMappings(*CI).empty())
+    VecCallVariantsFound = true;
+
+  auto CanWidenInstructionTy = [](Instruction const &Inst) {
+    Type *InstTy = Inst.getType();
+    if (!isa<StructType>(InstTy))
+      return canVectorizeTy(InstTy);
+
+    // For now, we only recognize struct values returned from calls where
+    // all users are extractvalue as vectorizable. All element types of the
+    // struct must be types that can be widened.
+    return isa<CallInst>(Inst) && canWidenCallReturnType(InstTy) &&
+           all_of(Inst.users(), IsaPred<ExtractValueInst>);
+  };
 
-      } else if (auto *LD = dyn_cast<LoadInst>(&I)) {
-        if (LD->getMetadata(LLVMContext::MD_nontemporal)) {
-          // For nontemporal loads, check that a nontemporal vector version is
-          // supported on the target (arbitrarily try a vector of 2 elements).
-          auto *VecTy = FixedVectorType::get(I.getType(), /*NumElts=*/2);
-          assert(VecTy && "did not find vectorized version of load type");
-          if (!TTI->isLegalNTLoad(VecTy, LD->getAlign())) {
-            reportVectorizationFailure(
-                "nontemporal load instruction cannot be vectorized",
-                "CantVectorizeNontemporalLoad", ORE, TheLoop, LD);
-            return false;
-          }
-        }
+  // Check that the instruction return type is vectorizable.
+  // We can't vectorize casts from vector type to scalar type.
+  // Also, we can't vectorize extractelement instructions.
+  if (!CanWidenInstructionTy(I) ||
+      (isa<CastInst>(I) &&
+       !VectorType::isValidElementType(I.getOperand(0)->getType())) ||
+      isa<ExtractElementInst>(I)) {
+    reportVectorizationFailure("Found unvectorizable type",
+                               "instruction return type cannot be vectorized",
+                               "CantVectorizeInstructionReturnType", ORE,
+                               TheLoop, &I);
+    return false;
+  }
+
+  // Check that the stored type is vectorizable.
+  if (auto *ST = dyn_cast<StoreInst>(&I)) {
+    Type *T = ST->getValueOperand()->getType();
+    if (!VectorType::isValidElementType(T)) {
+      reportVectorizationFailure("Store instruction cannot be vectorized",
+                                 "CantVectorizeStore", ORE, TheLoop, ST);
+      return false;
+    }
 
-        // FP instructions can allow unsafe algebra, thus vectorizable by
-        // non-IEEE-754 compliant SIMD units.
-        // This applies to floating-point math operations and calls, not memory
-        // operations, shuffles, or casts, as they don't change precision or
-        // semantics.
-      } else if (I.getType()->isFloatingPointTy() && (CI || I.isBinaryOp()) &&
-                 !I.isFast()) {
-        LLVM_DEBUG(dbgs() << "LV: Found FP op with unsafe algebra.\n");
-        Hints->setPotentiallyUnsafe();
+    // For nontemporal stores, check that a nontemporal vector version is
+    // supported on the target.
+    if (ST->getMetadata(LLVMContext::MD_nontemporal)) {
+      // Arbitrarily try a vector of 2 elements.
+      auto *VecTy = FixedVectorType::get(T, /*NumElts=*/2);
+      assert(VecTy && "did not find vectorized version of stored type");
+      if (!TTI->isLegalNTStore(VecTy, ST->getAlign())) {
+        reportVectorizationFailure(
+            "nontemporal store instruction cannot be vectorized",
+            "CantVectorizeNontemporalStore", ORE, TheLoop, ST);
+        return false;
       }
+    }
 
-      // Reduction instructions are allowed to have exit users.
-      // All other instructions must not have external users.
-      if (hasOutsideLoopUser(TheLoop, &I, AllowedExit)) {
-        // We can safely vectorize loops where instructions within the loop are
-        // used outside the loop only if the SCEV predicates within the loop is
-        // same as outside the loop. Allowing the exit means reusing the SCEV
-        // outside the loop.
-        if (PSE.getPredicate().isAlwaysTrue()) {
-          AllowedExit.insert(&I);
-          continue;
-        }
-        reportVectorizationFailure("Value cannot be used outside the loop",
-                                   "ValueUsedOutsideLoop", ORE, TheLoop, &I);
+  } else if (auto *LD = dyn_cast<LoadInst>(&I)) {
+    if (LD->getMetadata(LLVMContext::MD_nontemporal)) {
+      // For nontemporal loads, check that a nontemporal vector version is
+      // supported on the target (arbitrarily try a vector of 2 elements).
+      auto *VecTy = FixedVectorType::get(I.getType(), /*NumElts=*/2);
+      assert(VecTy && "did not find vectorized version of load type");
+      if (!TTI->isLegalNTLoad(VecTy, LD->getAlign())) {
+        reportVectorizationFailure(
+            "nontemporal load instruction cannot be vectorized",
+            "CantVectorizeNontemporalLoad", ORE, TheLoop, LD);
         return false;
       }
-    } // next instr.
+    }
+
+    // FP instructions can allow unsafe algebra, thus vectorizable by
+    // non-IEEE-754 compliant SIMD units.
+    // This applies to floating-point math operations and calls, not memory
+    // operations, shuffles, or casts, as they don't change precision or
+    // semantics.
+  } else if (I.getType()->isFloatingPointTy() && (CI || I.isBinaryOp()) &&
+             !I.isFast()) {
+    LLVM_DEBUG(dbgs() << "LV: Found FP op with unsafe algebra.\n");
+    Hints->setPotentiallyUnsafe();
   }
 
-  if (!PrimaryInduction) {
-    if (Inductions.empty()) {
-      reportVectorizationFailure("Did not find one integer induction var",
-          "loop induction variable could not be identified",
-          "NoInductionVariable", ORE, TheLoop);
-      return false;
-    }
-    if (!WidestIndTy) {
-      reportVectorizationFailure("Did not find one integer induction var",
-          "integer loop induction variable could not be identified",
-          "NoIntegerInductionVariable", ORE, TheLoop);
-      return false;
+  // Reduction instructions are allowed to have exit users.
+  // All other instructions must not have external users.
+  if (hasOutsideLoopUser(TheLoop, &I, AllowedExit)) {
+    // We can safely vectorize loops where instructions within the loop are
+    // used outside the loop only if the SCEV predicates within the loop is
+    // same as outside the loop. Allowing the exit means reusing the SCEV
+    // outside the loop.
+    if (PSE.getPredicate().isAlwaysTrue()) {
+      AllowedExit.insert(&I);
+      return true;
     }
-    LLVM_DEBUG(dbgs() << "LV: Did not find one integer induction var.\n");
+    reportVectorizationFailure("Value cannot be used outside the loop",
+                               "ValueUsedOutsideLoop", ORE, TheLoop, &I);
+    return false;
   }
 
-  // Now we know the widest induction type, check if our found induction
-  // is the same size. If it's not, unset it here and InnerLoopVectorizer
-  // will create another.
-  if (PrimaryInduction && WidestIndTy != PrimaryInduction->getType())
-    PrimaryInduction = nullptr;
-
   return true;
 }
 
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 912c893123095..838476dcae661 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -256,13 +256,15 @@ class VPBuilder {
         new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
                           GEPNoWrapFlags::none(), DL, Name));
   }
-  VPInstruction *createInBoundsPtrAdd(VPValue *Ptr, VPValue *Offset,
-                                      DebugLoc DL = DebugLoc::getUnknown(),
-                                      const Twine &Name = "") {
-    return tryInsertInstruction(
-        new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
-                          GEPNoWrapFlags::inBounds(), DL, Name));
+
+  VPInstruction *createNoWrapPtrAdd(VPValue *Ptr, VPValue *Offset,
+                                    GEPNoWrapFlags GEPFlags,
+                                    DebugLoc DL = DebugLoc::getUnknown(),
+                                    const Twine &Name = "") {
+    return tryInsertInstruction(new VPInstruction(
+        VPInstruction::PtrAdd, {Ptr, Offset}, GEPFlags, DL, Name));
   }
+
   VPInstruction *createWidePtrAdd(VPValue *Ptr, VPValue *Offset,
                                   DebugLoc DL = DebugLoc::getUnknown(),
                                   const Twine &Name = "") {
@@ -276,6 +278,20 @@ class VPBuilder {
     return tryInsertInstruction(new VPPhi(IncomingValues, DL, Name));
   }
 
+  VPValue *createElementCount(Type *Ty, ElementCount EC) {
+    VPlan &Plan = *getInsertBlock()->getPlan();
+    VPValue *RuntimeEC =
+        Plan.getOrAddLiveIn(ConstantInt::get(Ty, EC.getKnownMinValue()));
+    if (EC.isScalable()) {
+      VPValue *VScale = createNaryOp(VPInstruction::VScale, {}, Ty);
+      RuntimeEC = EC.getKnownMinValue() == 1
+                      ? VScale
+                      : createOverflowingOp(Instruction::Mul,
+                                            {VScale, RuntimeEC}, {true, false});
+    }
+    return RuntimeEC;
+  }
+
   /// Convert the input value \p Current to the corresponding value of an
   /// induction with \p Start and \p Step values, using \p Start + \p Current *
   /// \p Step.
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 307b5cf1a2c50..a0f306c12754f 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -499,19 +499,18 @@ class InnerLoopVectorizer {
 public:
   InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
                       LoopInfo *LI, DominatorTree *DT,
-                      const TargetLibraryInfo *TLI,
                       const TargetTransformInfo *TTI, AssumptionCache *AC,
-                      OptimizationRemarkEmitter *ORE, ElementCount VecWidth,
+                      ElementCount VecWidth,
                       ElementCount MinProfitableTripCount,
                       unsigned UnrollFactor, LoopVectorizationCostModel *CM,
                       BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
                       GeneratedRTChecks &RTChecks, VPlan &Plan)
-      : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
-        AC(AC), ORE(ORE), VF(VecWidth),
-        MinProfitableTripCount(MinProfitableTripCount), UF(UnrollFactor),
-        Builder(PSE.getSE()->getContext()), Cost(CM), BFI(BFI), PSI(PSI),
-        RTChecks(RTChecks), Plan(Plan),
-        VectorPHVPB(Plan.getVectorLoopRegion()->getSinglePredecessor()) {}
+      : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TTI(TTI), AC(AC),
+        VF(VecWidth), MinProfitableTripCount(MinProfitableTripCount),
+        UF(UnrollFactor), Builder(PSE.getSE()->getContext()), Cost(CM),
+        BFI(BFI), PSI(PSI), RTChecks(RTChecks), Plan(Plan),
+        VectorPHVPBB(cast<VPBasicBlock>(
+            Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
 
   virtual ~InnerLoopVectorizer() = default;
 
@@ -548,9 +547,6 @@ class InnerLoopVectorizer {
 protected:
   friend class LoopVectorizationPlanner;
 
-  /// Returns (and creates if needed) the trip count of the widened loop.
-  Value *getOrCreateVectorTripCount(BasicBlock *InsertBlock);
-
   // Create a check to see if the vector loop should be executed
   Value *createIterationCountCheck(ElementCount VF, unsigned UF) const;
 
@@ -586,18 +582,12 @@ class InnerLoopVectorizer {
   /// Dominator Tree.
   DominatorTree *DT;
 
-  /// Target Library Info.
-  const TargetLibraryInfo *TLI;
-
   /// Target Transform Info.
   const TargetTransformInfo *TTI;
 
   /// Assumption Cache.
   AssumptionCache *AC;
 
-  /// Interface to emit optimization remarks.
-  OptimizationRemarkEmitter *ORE;
-
   /// The vectorization SIMD factor to use. Each vector will have this many
   /// vector elements.
   ElementCount VF;
@@ -619,9 +609,6 @@ class InnerLoopVectorizer {
   /// The scalar-loop preheader.
   BasicBlock *LoopScalarPreHeader = nullptr;
 
-  /// Middle Block between the vector and the scalar.
-  BasicBlock *LoopMiddleBlock = nullptr;
-
   /// Trip count of the original loop.
   Value *TripCount = nullptr;
 
@@ -648,7 +635,7 @@ class InnerLoopVectorizer {
 
   /// The vector preheader block of \p Plan, used as target for check blocks
   /// introduced during skeleton creation.
-  VPBlockBase *VectorPHVPB;
+  VPBasicBlock *VectorPHVPBB;
 };
 
 /// Encapsulate information regarding vectorization of a loop and its epilogue.
@@ -686,14 +673,14 @@ class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
 public:
   InnerLoopAndEpilogueVectorizer(
       Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
-      DominatorTree *DT, const TargetLibraryInfo *TLI,
-      const TargetTransformInfo *TTI, AssumptionCache *AC,
-      OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
-      LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
-      ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan)
-      : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
-                            EPI.MainLoopVF, EPI.MainLoopVF, EPI.MainLoopUF, CM,
-                            BFI, PSI, Checks, Plan),
+      DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC,
+      EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM,
+      BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
+      GeneratedRTChecks &Checks, VPlan &Plan, ElementCount VecWidth,
+      ElementCount MinProfitableTripCount, unsigned UnrollFactor)
+      : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, VecWidth,
+                            MinProfitableTripCount, UnrollFactor, CM, BFI, PSI,
+                            Checks, Plan),
         EPI(EPI) {}
 
   // Override this function to handle the more complex control flow around the
@@ -721,15 +708,17 @@ class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
 /// epilogues.
 class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
 public:
-  EpilogueVectorizerMainLoop(
-      Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
-      DominatorTree *DT, const TargetLibraryInfo *TLI,
-      const TargetTransformInfo *TTI, AssumptionCache *AC,
-      OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
-      LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
-      ProfileSummaryInfo *PSI, GeneratedRTChecks &Check, VPlan &Plan)
-      : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
-                                       EPI, CM, BFI, PSI, Check, Plan) {}
+  EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
+                             LoopInfo *LI, DominatorTree *DT,
+                             const TargetTransformInfo *TTI,
+                             AssumptionCache *AC,
+                             EpilogueLoopVectorizationInfo &EPI,
+                             LoopVectorizationCostModel *CM,
+                             BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
+                             GeneratedRTChecks &Check, VPlan &Plan)
+      : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, EPI, CM,
+                                       BFI, PSI, Check, Plan, EPI.MainLoopVF,
+                                       EPI.MainLoopVF, EPI.MainLoopUF) {}
   /// Implements the interface for creating a vectorized skeleton using the
   /// *main loop* strategy (ie the first pass of vplan execution).
   BasicBlock *createEpilogueVectorizedLoopSkeleton() final;
@@ -750,13 +739,13 @@ class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
 public:
   EpilogueVectorizerEpilogueLoop(
       Loop *OrigLoop, PredicatedScalarEvolution &PSE, LoopInfo *LI,
-      DominatorTree *DT, const TargetLibraryInfo *TLI,
-      const TargetTransformInfo *TTI, AssumptionCache *AC,
-      OptimizationRemarkEmitter *ORE, EpilogueLoopVectorizationInfo &EPI,
-      LoopVectorizationCostModel *CM, BlockFrequencyInfo *BFI,
-      ProfileSummaryInfo *PSI, GeneratedRTChecks &Checks, VPlan &Plan)
-      : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TLI, TTI, AC, ORE,
-                                       EPI, CM, BFI, PSI, Checks, Plan) {
+      DominatorTree *DT, const TargetTransformInfo *TTI, AssumptionCache *AC,
+      EpilogueLoopVectorizationInfo &EPI, LoopVectorizationCostModel *CM,
+      BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI,
+      GeneratedRTChecks &Checks, VPlan &Plan)
+      : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, EPI, CM,
+                                       BFI, PSI, Checks, Plan, EPI.EpilogueVF,
+                                       EPI.EpilogueVF, EPI.EpilogueUF) {
     TripCount = EPI.TripCount;
   }
   /// Implements the interface for creating a vectorized skeleton using the
@@ -835,7 +824,14 @@ namespace llvm {
 Value *createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF,
                        int64_t Step) {
   assert(Ty->isIntegerTy() && "Expected an integer step");
-  return B.CreateElementCount(Ty, VF.multiplyCoefficientBy(Step));
+  ElementCount VFxStep = VF.multiplyCoefficientBy(Step);
+  assert(isPowerOf2_64(VF.getKnownMinValue()) && "must pass power-of-2 VF");
+  if (VF.isScalable() && isPowerOf2_64(Step)) {
+    return B.CreateShl(
+        B.CreateVScale(Ty),
+        ConstantInt::get(Ty, Log2_64(VFxStep.getKnownMinValue())), "", true);
+  }
+  return B.CreateElementCount(Ty, VFxStep);
 }
 
 /// Return the runtime value for VF.
@@ -1701,8 +1697,16 @@ class LoopVectorizationCostModel {
   /// Returns a range containing only operands needing to be extracted.
   SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
                                                    ElementCount VF) const {
-    return SmallVector<Value *, 4>(make_filter_range(
-        Ops, [this, VF](Value *V) { return this->needsExtract(V, VF); }));
+
+    SmallPtrSet<const Value *, 4> UniqueOperands;
+    SmallVector<Value *, 4> Res;
+    for (Value *Op : Ops) {
+      if (isa<Constant>(Op) || !UniqueOperands.insert(Op).second ||
+          !needsExtract(Op, VF))
+        continue;
+      Res.push_back(Op);
+    }
+    return Res;
   }
 
 public:
@@ -2272,65 +2276,15 @@ static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
   return TTI.enableMaskedInterleavedAccessVectorization();
 }
 
-Value *
-InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
-  if (VectorTripCount)
-    return VectorTripCount;
-
-  Value *TC = getTripCount();
-  IRBuilder<> Builder(InsertBlock->getTerminator());
-
-  Type *Ty = TC->getType();
-  // This is where we can make the step a runtime constant.
-  Value *Step = createStepForVF(Builder, Ty, VF, UF);
-
-  // If the tail is to be folded by masking, round the number of iterations N
-  // up to a multiple of Step instead of rounding down. This is done by first
-  // adding Step-1 and then rounding down. Note that it's ok if this addition
-  // overflows: the vector induction variable will eventually wrap to zero given
-  // that it starts at zero and its Step is a power of two; the loop will then
-  // exit, with the last early-exit vector comparison also producing all-true.
-  // For scalable vectors the VF is not guaranteed to be a power of 2, but this
-  // is accounted for in emitIterationCountCheck that adds an overflow check.
-  if (Cost->foldTailByMasking()) {
-    assert(isPowerOf2_32(VF.getKnownMinValue() * UF) &&
-           "VF*UF must be a power of 2 when folding tail by masking");
-    TC = Builder.CreateAdd(TC, Builder.CreateSub(Step, ConstantInt::get(Ty, 1)),
-                           "n.rnd.up");
-  }
-
-  // Now we need to generate the expression for the part of the loop that the
-  // vectorized body will execute. This is equal to N - (N % Step) if scalar
-  // iterations are not required for correctness, or N - Step, otherwise. Step
-  // is equal to the vectorization factor (number of SIMD elements) times the
-  // unroll factor (number of SIMD instructions).
-  Value *R = Builder.CreateURem(TC, Step, "n.mod.vf");
-
-  // There are cases where we *must* run at least one iteration in the remainder
-  // loop.  See the cost model for when this can happen.  If the step evenly
-  // divides the trip count, we set the remainder to be equal to the step. If
-  // the step does not evenly divide the trip count, no adjustment is necessary
-  // since there will already be scalar iterations. Note that the minimum
-  // iterations check ensures that N >= Step.
-  if (Cost->requiresScalarEpilogue(VF.isVector())) {
-    auto *IsZero = Builder.CreateICmpEQ(R, ConstantInt::get(R->getType(), 0));
-    R = Builder.CreateSelect(IsZero, Step, R);
-  }
-
-  VectorTripCount = Builder.CreateSub(TC, R, "n.vec");
-
-  return VectorTripCount;
-}
-
 void InnerLoopVectorizer::introduceCheckBlockInVPlan(BasicBlock *CheckIRBB) {
   // Note: The block with the minimum trip-count check is already connected
   // during earlier VPlan construction.
   VPBlockBase *ScalarPH = Plan.getScalarPreheader();
-  VPBlockBase *PreVectorPH = VectorPHVPB->getSinglePredecessor();
+  VPBlockBase *PreVectorPH = VectorPHVPBB->getSinglePredecessor();
   assert(PreVectorPH->getNumSuccessors() == 2 && "Expected 2 successors");
   assert(PreVectorPH->getSuccessors()[0] == ScalarPH && "Unexpected successor");
   VPIRBasicBlock *CheckVPIRBB = Plan.createVPIRBasicBlock(CheckIRBB);
-  VPBlockUtils::insertOnEdge(PreVectorPH, VectorPHVPB, CheckVPIRBB);
+  VPBlockUtils::insertOnEdge(PreVectorPH, VectorPHVPBB, CheckVPIRBB);
   PreVectorPH = CheckVPIRBB;
   VPBlockUtils::connectBlocks(PreVectorPH, ScalarPH);
   PreVectorPH->swapSuccessors();
@@ -2359,7 +2313,10 @@ Value *InnerLoopVectorizer::createIterationCountCheck(ElementCount VF,
   // Reuse existing vector loop preheader for TC checks.
   // Note that new preheader block is generated for vector loop.
   BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
-  IRBuilder<> Builder(TCCheckBlock->getTerminator());
+  IRBuilder<InstSimplifyFolder> Builder(
+      TCCheckBlock->getContext(),
+      InstSimplifyFolder(TCCheckBlock->getDataLayout()));
+  Builder.SetInsertPoint(TCCheckBlock->getTerminator());
 
   // If tail is to be folded, vector loop takes care of all iterations.
   Value *Count = getTripCount();
@@ -2371,7 +2328,7 @@ Value *InnerLoopVectorizer::createIterationCountCheck(ElementCount VF,
       return createStepForVF(Builder, CountTy, VF, UF);
 
     Value *MinProfTC =
-        createStepForVF(Builder, CountTy, MinProfitableTripCount, 1);
+        Builder.CreateElementCount(CountTy, MinProfitableTripCount);
     if (!VF.isScalable())
       return MinProfTC;
     return Builder.CreateBinaryIntrinsic(
@@ -2437,16 +2394,20 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
 /// VPBB are moved to the end of the newly created VPIRBasicBlock. VPBB must
 /// have a single predecessor, which is rewired to the new VPIRBasicBlock. All
 /// successors of VPBB, if any, are rewired to the new VPIRBasicBlock.
-static void replaceVPBBWithIRVPBB(VPBasicBlock *VPBB, BasicBlock *IRBB) {
+static VPIRBasicBlock *replaceVPBBWithIRVPBB(VPBasicBlock *VPBB,
+                                             BasicBlock *IRBB) {
   VPIRBasicBlock *IRVPBB = VPBB->getPlan()->createVPIRBasicBlock(IRBB);
-  for (auto &R : make_early_inc_range(*VPBB)) {
-    assert((IRVPBB->empty() || IRVPBB->back().isPhi() || !R.isPhi()) &&
-           "Tried to move phi recipe after a non-phi recipe");
+  auto IP = IRVPBB->begin();
+  for (auto &R : make_early_inc_range(VPBB->phis()))
+    R.moveBefore(*IRVPBB, IP);
+
+  for (auto &R :
+       make_early_inc_range(make_range(VPBB->getFirstNonPhi(), VPBB->end())))
     R.moveBefore(*IRVPBB, IRVPBB->end());
-  }
 
   VPBlockUtils::reassociateBlocks(VPBB, IRVPBB);
   // VPBB is now dead and will be cleaned up when the plan gets destroyed.
+  return IRVPBB;
 }
 
 void InnerLoopVectorizer::createVectorLoopSkeleton(StringRef Prefix) {
@@ -2549,7 +2510,7 @@ BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
   // to the scalar loop.
   emitIterationCountCheck(LoopScalarPreHeader);
 
-  replaceVPBBWithIRVPBB(Plan.getScalarPreheader(), LoopScalarPreHeader);
+  replaceVPBBWithIRVPBB(VectorPHVPBB, LoopVectorPreHeader);
   return LoopVectorPreHeader;
 }
 
@@ -2680,19 +2641,6 @@ void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
   // Fix widened non-induction PHIs by setting up the PHI operands.
   fixNonInductionPHIs(State);
 
-  // After vectorization, the exit blocks of the original loop will have
-  // additional predecessors. Invalidate SCEVs for the exit phis in case SE
-  // looked through single-entry phis.
-  SmallVector<BasicBlock *> ExitBlocks;
-  OrigLoop->getExitBlocks(ExitBlocks);
-  for (BasicBlock *Exit : ExitBlocks)
-    for (PHINode &PN : Exit->phis())
-      PSE.getSE()->forgetLcssaPhiWithNewPredecessor(OrigLoop, &PN);
-
-  // Forget the original basic block.
-  PSE.getSE()->forgetLoop(OrigLoop);
-  PSE.getSE()->forgetBlockAndLoopDispositions();
-
   // Don't apply optimizations below when no (vector) loop remains, as they all
   // require one at the moment.
   VPBasicBlock *HeaderVPBB =
@@ -2734,11 +2682,8 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPTransformState &State) {
       PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
       // Make sure the builder has a valid insert point.
       Builder.SetInsertPoint(NewPhi);
-      for (unsigned Idx = 0; Idx < VPPhi->getNumIncoming(); ++Idx) {
-        VPValue *Inc = VPPhi->getIncomingValue(Idx);
-        const VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx);
+      for (const auto &[Inc, VPBB] : VPPhi->incoming_values_and_blocks())
         NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
-      }
     }
   }
 }
@@ -3158,6 +3103,12 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
   if (Group->isReverse())
     return false;
 
+  // TODO: Support interleaved access that requires a gap mask for scalable VFs.
+  bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
+                          StoreAccessWithGapsRequiresMasking;
+  if (VF.isScalable() && NeedsMaskForGaps)
+    return false;
+
   auto *Ty = getLoadStoreType(I);
   const Align Alignment = getLoadStoreAlignment(I);
   unsigned AS = getLoadStoreAddressSpace(I);
@@ -4069,8 +4020,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
       if (VF.isScalar())
         continue;
 
-      VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(),
-                            CM, CM.CostKind);
+      VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind);
       precomputeCosts(*Plan, VF, CostCtx);
       auto Iter = vp_depth_first_deep(Plan->getVectorLoopRegion()->getEntry());
       for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
@@ -4178,7 +4128,7 @@ void LoopVectorizationPlanner::emitInvalidCostRemarks(
 static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
                                 const TargetTransformInfo &TTI) {
   assert(VF.isVector() && "Checking a scalar VF?");
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan);
   DenseSet<VPRecipeBase *> EphemeralRecipes;
   collectEphemeralRecipesForVPlan(Plan, EphemeralRecipes);
   // Set of already visited types.
@@ -4326,8 +4276,7 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() {
 
       // Add on other costs that are modelled in VPlan, but not in the legacy
       // cost model.
-      VPCostContext CostCtx(CM.TTI, *CM.TLI, CM.Legal->getWidestInductionType(),
-                            CM, CM.CostKind);
+      VPCostContext CostCtx(CM.TTI, *CM.TLI, *P, CM, CM.CostKind);
       VPRegionBlock *VectorRegion = P->getVectorLoopRegion();
       assert(VectorRegion && "Expected to have a vector region!");
       for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
@@ -5272,8 +5221,8 @@ LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
   const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, Legal, PSE, TheLoop);
 
   // Get the cost of the scalar memory instruction and address computation.
-  InstructionCost Cost =
-      VF.getFixedValue() * TTI.getAddressComputationCost(PtrTy, SE, PtrSCEV);
+  InstructionCost Cost = VF.getFixedValue() * TTI.getAddressComputationCost(
+                                                  PtrTy, SE, PtrSCEV, CostKind);
 
   // Don't pass *I here, since it is scalar but will actually be part of a
   // vectorized loop where the user of it is a vectorized instruction.
@@ -5344,11 +5293,12 @@ LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
   assert(Legal->isUniformMemOp(*I, VF));
 
   Type *ValTy = getLoadStoreType(I);
+  Type *PtrTy = getLoadStorePointerOperand(I)->getType();
   auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
   const Align Alignment = getLoadStoreAlignment(I);
   unsigned AS = getLoadStoreAddressSpace(I);
   if (isa<LoadInst>(I)) {
-    return TTI.getAddressComputationCost(ValTy) +
+    return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
            TTI.getMemoryOpCost(Instruction::Load, ValTy, Alignment, AS,
                                CostKind) +
            TTI.getShuffleCost(TargetTransformInfo::SK_Broadcast, VectorTy,
@@ -5361,13 +5311,13 @@ LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
   // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
   // the actual generated code, which involves extracting the last element of
   // a scalable vector where the lane to extract is unknown at compile time.
-  return TTI.getAddressComputationCost(ValTy) +
-         TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS,
-                             CostKind) +
-         (IsLoopInvariantStoreValue
-              ? 0
-              : TTI.getVectorInstrCost(Instruction::ExtractElement, VectorTy,
-                                       CostKind, VF.getKnownMinValue() - 1));
+  InstructionCost Cost =
+      TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
+      TTI.getMemoryOpCost(Instruction::Store, ValTy, Alignment, AS, CostKind);
+  if (!IsLoopInvariantStoreValue)
+    Cost += TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
+                                                 VectorTy, CostKind, 0);
+  return Cost;
 }
 
 InstructionCost
@@ -5377,8 +5327,9 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
   auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
   const Align Alignment = getLoadStoreAlignment(I);
   const Value *Ptr = getLoadStorePointerOperand(I);
+  Type *PtrTy = toVectorTy(Ptr->getType(), VF);
 
-  return TTI.getAddressComputationCost(VectorTy) +
+  return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
          TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,
                                     Legal->isMaskRequired(I), Alignment,
                                     CostKind, I);
@@ -5613,11 +5564,12 @@ LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
   // moment.
   if (VF.isScalar()) {
     Type *ValTy = getLoadStoreType(I);
+    Type *PtrTy = getLoadStorePointerOperand(I)->getType();
     const Align Alignment = getLoadStoreAlignment(I);
     unsigned AS = getLoadStoreAddressSpace(I);
 
     TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0));
-    return TTI.getAddressComputationCost(ValTy) +
+    return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
            TTI.getMemoryOpCost(I->getOpcode(), ValTy, Alignment, AS, CostKind,
                                OpInfo, I);
   }
@@ -5666,8 +5618,7 @@ LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
   SmallVector<Type *> Tys;
   for (auto *V : filterExtractingOperands(Ops, VF))
     Tys.push_back(maybeVectorizeType(V->getType(), VF));
-  return Cost + TTI.getOperandsScalarizationOverhead(
-                    filterExtractingOperands(Ops, VF), Tys, CostKind);
+  return Cost + TTI.getOperandsScalarizationOverhead(Tys, CostKind);
 }
 
 void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
@@ -5888,14 +5839,22 @@ void LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) {
       // assumed to be vectors, so we need to extract individual elements from
       // there, execute VF scalar calls, and then gather the result into the
       // vector return value.
-      InstructionCost ScalarCallCost =
-          TTI.getCallInstrCost(ScalarFunc, ScalarRetTy, ScalarTys, CostKind);
-
-      // Compute costs of unpacking argument values for the scalar calls and
-      // packing the return values to a vector.
-      InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF);
+      if (VF.isFixed()) {
+        InstructionCost ScalarCallCost =
+            TTI.getCallInstrCost(ScalarFunc, ScalarRetTy, ScalarTys, CostKind);
+
+        // Compute costs of unpacking argument values for the scalar calls and
+        // packing the return values to a vector.
+        InstructionCost ScalarizationCost = getScalarizationOverhead(CI, VF);
+        ScalarCost = ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;
+      } else {
+        // There is no point attempting to calculate the scalar cost for a
+        // scalable VF as we know it will be Invalid.
+        assert(!getScalarizationOverhead(CI, VF).isValid() &&
+               "Unexpected valid cost for scalarizing scalable vectors");
+        ScalarCost = InstructionCost::getInvalid();
+      }
 
-      ScalarCost = ScalarCallCost * VF.getKnownMinValue() + ScalarizationCost;
       // Honor ForcedScalars and UniformAfterVectorization decisions.
       // TODO: For calls, it might still be more profitable to widen. Use
       // VPlan-based cost model to compare different options.
@@ -5959,21 +5918,11 @@ void LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) {
             // TODO: do we need to figure out the cost of an extract to get the
             // first lane? Or do we hope that it will be folded away?
             ScalarEvolution *SE = PSE.getSE();
-            const auto *SAR =
-                dyn_cast<SCEVAddRecExpr>(SE->getSCEV(ScalarParam));
-
-            if (!SAR || SAR->getLoop() != TheLoop) {
-              ParamsOk = false;
-              break;
-            }
-
-            const SCEVConstant *Step =
-                dyn_cast<SCEVConstant>(SAR->getStepRecurrence(*SE));
-
-            if (!Step ||
-                Step->getAPInt().getSExtValue() != Param.LinearStepOrPos)
+            if (!match(SE->getSCEV(ScalarParam),
+                       m_scev_AffineAddRec(
+                           m_SCEV(), m_scev_SpecificSInt(Param.LinearStepOrPos),
+                           m_SpecificLoop(TheLoop))))
               ParamsOk = false;
-
             break;
           }
           case VFParamKind::GlobalPredicate:
@@ -6976,8 +6925,7 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
 
 InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan,
                                                ElementCount VF) const {
-  VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(), CM,
-                        CM.CostKind);
+  VPCostContext CostCtx(CM.TTI, *CM.TLI, Plan, CM, CM.CostKind);
   InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
 
   // Now compute and add the VPlan-based cost.
@@ -7054,12 +7002,12 @@ static bool planContainsAdditionalSimplifications(VPlan &Plan,
       if (Instruction *UI = GetInstructionForCost(&R)) {
         // If we adjusted the predicate of the recipe, the cost in the legacy
         // cost model may be different.
-        if (auto *WidenCmp = dyn_cast<VPWidenRecipe>(&R)) {
-          if ((WidenCmp->getOpcode() == Instruction::ICmp ||
-               WidenCmp->getOpcode() == Instruction::FCmp) &&
-              WidenCmp->getPredicate() != cast<CmpInst>(UI)->getPredicate())
-            return true;
-        }
+        using namespace VPlanPatternMatch;
+        CmpPredicate Pred;
+        if (match(&R, m_Cmp(Pred, m_VPValue(), m_VPValue())) &&
+            cast<VPRecipeWithIRFlags>(R).getPredicate() !=
+                cast<CmpInst>(UI)->getPredicate())
+          return true;
         SeenInstrs.insert(UI);
       }
     }
@@ -7178,8 +7126,7 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
   // simplifications not accounted for in the legacy cost model. If that's the
   // case, don't trigger the assertion, as the extra simplifications may cause a
   // different VF to be picked by the VPlan-based cost model.
-  VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(), CM,
-                        CM.CostKind);
+  VPCostContext CostCtx(CM.TTI, *CM.TLI, BestPlan, CM, CM.CostKind);
   precomputeCosts(BestPlan, BestFactor.Width, CostCtx);
   // Verify that the VPlan-based and legacy cost models agree, except for VPlans
   // with early exits and plans with additional VPlan simplifications. The
@@ -7317,10 +7264,10 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
     ++LoopsEarlyExitVectorized;
   // TODO: Move to VPlan transform stage once the transition to the VPlan-based
   // cost model is complete for better cost estimates.
-  VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF,
-                           OrigLoop->getHeader()->getContext());
-  VPlanTransforms::runPass(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
+  VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
+  VPlanTransforms::runPass(VPlanTransforms::materializeBuildVectors, BestVPlan);
   VPlanTransforms::runPass(VPlanTransforms::materializeBroadcasts, BestVPlan);
+  VPlanTransforms::runPass(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
   bool HasBranchWeights =
       hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator());
   if (HasBranchWeights) {
@@ -7339,21 +7286,35 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   VPBasicBlock *VectorPH = cast<VPBasicBlock>(BestVPlan.getVectorPreheader());
 
   VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE);
-  VPlanTransforms::simplifyRecipes(BestVPlan, *Legal->getWidestInductionType());
+  VPlanTransforms::simplifyRecipes(BestVPlan);
   VPlanTransforms::removeBranchOnConst(BestVPlan);
   VPlanTransforms::narrowInterleaveGroups(
       BestVPlan, BestVF,
       TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector));
   VPlanTransforms::removeDeadRecipes(BestVPlan);
 
-  VPlanTransforms::convertToConcreteRecipes(BestVPlan,
-                                            *Legal->getWidestInductionType());
+  VPlanTransforms::convertToConcreteRecipes(BestVPlan);
   // Regions are dissolved after optimizing for VF and UF, which completely
   // removes unneeded loop regions first.
   VPlanTransforms::dissolveLoopRegions(BestVPlan);
   // Canonicalize EVL loops after regions are dissolved.
   VPlanTransforms::canonicalizeEVLLoops(BestVPlan);
   VPlanTransforms::materializeBackedgeTakenCount(BestVPlan, VectorPH);
+  VPlanTransforms::materializeVectorTripCount(
+      BestVPlan, VectorPH, CM.foldTailByMasking(),
+      CM.requiresScalarEpilogue(BestVF.isVector()));
+  VPlanTransforms::materializeVFAndVFxUF(BestVPlan, VectorPH, BestVF);
+  VPlanTransforms::simplifyRecipes(BestVPlan);
+
+  // 0. Generate SCEV-dependent code in the entry, including TripCount, before
+  // making any changes to the CFG.
+  DenseMap<const SCEV *, Value *> ExpandedSCEVs =
+      VPlanTransforms::expandSCEVs(BestVPlan, *PSE.getSE());
+  if (!ILV.getTripCount())
+    ILV.setTripCount(BestVPlan.getTripCount()->getLiveInIRValue());
+  else
+    assert(VectorizingEpilogue && "should only re-use the existing trip "
+                                  "count during epilogue vectorization");
 
   // Perform the actual loop transformation.
   VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
@@ -7364,41 +7325,33 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   assert(DT->verify(DominatorTree::VerificationLevel::Fast));
 #endif
 
-  // 0. Generate SCEV-dependent code in the entry, including TripCount, before
-  // making any changes to the CFG.
-  DenseMap<const SCEV *, Value *> ExpandedSCEVs;
-  auto *Entry = cast<VPIRBasicBlock>(BestVPlan.getEntry());
-  State.Builder.SetInsertPoint(Entry->getIRBasicBlock()->getTerminator());
-  for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
-    auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
-    if (!ExpSCEV)
-      continue;
-    ExpSCEV->execute(State);
-    ExpandedSCEVs[ExpSCEV->getSCEV()] = State.get(ExpSCEV, VPLane(0));
-    VPValue *Exp = BestVPlan.getOrAddLiveIn(ExpandedSCEVs[ExpSCEV->getSCEV()]);
-    ExpSCEV->replaceAllUsesWith(Exp);
-    if (BestVPlan.getTripCount() == ExpSCEV)
-      BestVPlan.resetTripCount(Exp);
-    ExpSCEV->eraseFromParent();
-  }
-
-  if (!ILV.getTripCount())
-    ILV.setTripCount(State.get(BestVPlan.getTripCount(), VPLane(0)));
-  else
-    assert(VectorizingEpilogue && "should only re-use the existing trip "
-                                  "count during epilogue vectorization");
-
   // 1. Set up the skeleton for vectorization, including vector pre-header and
   // middle block. The vector loop is created during VPlan execution.
   BasicBlock *EntryBB =
       cast<VPIRBasicBlock>(BestVPlan.getEntry())->getIRBasicBlock();
   State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
-  if (VectorizingEpilogue)
-    VPlanTransforms::removeDeadRecipes(BestVPlan);
+  replaceVPBBWithIRVPBB(BestVPlan.getScalarPreheader(),
+                        State.CFG.PrevBB->getSingleSuccessor());
+  VPlanTransforms::removeDeadRecipes(BestVPlan);
 
   assert(verifyVPlanIsValid(BestVPlan, true /*VerifyLate*/) &&
          "final VPlan is invalid");
 
+  // After vectorization, the exit blocks of the original loop will have
+  // additional predecessors. Invalidate SCEVs for the exit phis in case SE
+  // looked through single-entry phis.
+  ScalarEvolution &SE = *PSE.getSE();
+  for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
+    if (Exit->getNumPredecessors() == 0)
+      continue;
+    for (VPRecipeBase &PhiR : Exit->phis())
+      SE.forgetLcssaPhiWithNewPredecessor(
+          OrigLoop, cast<PHINode>(&cast<VPIRPhi>(PhiR).getInstruction()));
+  }
+  // Forget the original loop and block dispositions.
+  SE.forgetLoop(OrigLoop);
+  SE.forgetBlockAndLoopDispositions();
+
   ILV.printDebugTracesAtStart();
 
   //===------------------------------------------------===//
@@ -7409,11 +7362,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
   //
   //===------------------------------------------------===//
 
-  // 2. Copy and widen instructions from the old loop into the new loop.
-  BestVPlan.prepareToExecute(
-      ILV.getOrCreateVectorTripCount(ILV.LoopVectorPreHeader), State);
-  replaceVPBBWithIRVPBB(VectorPH, State.CFG.PrevBB);
-
   // Move check blocks to their final position.
   // TODO: Move as part of VPIRBB execute and update impacted tests.
   if (BasicBlock *MemCheckBlock = ILV.RTChecks.getMemRuntimeChecks().second)
@@ -7530,10 +7478,6 @@ BasicBlock *EpilogueVectorizerMainLoop::createEpilogueVectorizedLoopSkeleton() {
   EPI.MainLoopIterationCountCheck =
       emitIterationCountCheck(LoopScalarPreHeader, false);
 
-  // Generate the induction variable.
-  EPI.VectorTripCount = getOrCreateVectorTripCount(LoopVectorPreHeader);
-
-  replaceVPBBWithIRVPBB(Plan.getScalarPreheader(), LoopScalarPreHeader);
   return LoopVectorPreHeader;
 }
 
@@ -7560,8 +7504,9 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
   assert(Bypass && "Expected valid bypass basic block.");
   Value *Count = getTripCount();
   MinProfitableTripCount = ElementCount::getFixed(0);
-  Value *CheckMinIters = createIterationCountCheck(
-      ForEpilogue ? EPI.EpilogueVF : VF, ForEpilogue ? EPI.EpilogueUF : UF);
+  Value *CheckMinIters =
+      createIterationCountCheck(ForEpilogue ? EPI.EpilogueVF : EPI.MainLoopVF,
+                                ForEpilogue ? EPI.EpilogueUF : EPI.MainLoopUF);
 
   BasicBlock *const TCCheckBlock = LoopVectorPreHeader;
   if (!ForEpilogue)
@@ -7571,12 +7516,13 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
   LoopVectorPreHeader = SplitBlock(TCCheckBlock, TCCheckBlock->getTerminator(),
                                    static_cast<DominatorTree *>(nullptr), LI,
                                    nullptr, "vector.ph");
-
   if (ForEpilogue) {
     // Save the trip count so we don't have to regenerate it in the
     // vec.epilog.iter.check. This is safe to do because the trip count
     // generated here dominates the vector epilog iter check.
     EPI.TripCount = Count;
+  } else {
+    VectorPHVPBB = replaceVPBBWithIRVPBB(VectorPHVPBB, LoopVectorPreHeader);
   }
 
   BranchInst &BI =
@@ -7610,6 +7556,8 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
   BasicBlock *VecEpilogueIterationCountCheck =
       SplitBlock(LoopVectorPreHeader, LoopVectorPreHeader->begin(), DT, LI,
                  nullptr, "vec.epilog.iter.check", true);
+  VectorPHVPBB = replaceVPBBWithIRVPBB(VectorPHVPBB, LoopVectorPreHeader);
+
   emitMinimumVectorEpilogueIterCountCheck(LoopScalarPreHeader,
                                           VecEpilogueIterationCountCheck);
   AdditionalBypassBlock = VecEpilogueIterationCountCheck;
@@ -7664,7 +7612,6 @@ EpilogueVectorizerEpilogueLoop::createEpilogueVectorizedLoopSkeleton() {
       Phi->removeIncomingValue(MemCheckBlock);
   }
 
-  replaceVPBBWithIRVPBB(Plan.getScalarPreheader(), LoopScalarPreHeader);
   return LoopVectorPreHeader;
 }
 
@@ -7693,11 +7640,11 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
   BranchInst &BI =
       *BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters);
   if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) {
-    // FIXME: See test Transforms/LoopVectorize/branch-weights.ll. I don't
-    // think the MainLoopStep is correct.
-    unsigned MainLoopStep = UF * VF.getKnownMinValue();
+    auto VScale = Cost->getVScaleForTuning();
+    unsigned MainLoopStep =
+        estimateElementCount(EPI.MainLoopVF * EPI.MainLoopUF, VScale);
     unsigned EpilogueLoopStep =
-        EPI.EpilogueUF * EPI.EpilogueVF.getKnownMinValue();
+        estimateElementCount(EPI.EpilogueVF * EPI.EpilogueUF, VScale);
     // We assume the remaining `Count` is equally distributed in
     // [0, MainLoopStep)
     // So the probability for `Count < EpilogueLoopStep` should be
@@ -7815,7 +7762,7 @@ createWidenInductionRecipes(PHINode *Phi, Instruction *PhiOrTrunc,
          "step must be loop invariant");
 
   VPValue *Step =
-      vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep(), SE);
+      vputils::getOrCreateVPValueForSCEVExpr(Plan, IndDesc.getStep());
   if (auto *TruncI = dyn_cast<TruncInst>(PhiOrTrunc)) {
     return new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, &Plan.getVF(),
                                              IndDesc, TruncI,
@@ -7837,8 +7784,7 @@ VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI(
 
   // Check if this is pointer induction. If so, build the recipe for it.
   if (auto *II = Legal->getPointerInductionDescriptor(Phi)) {
-    VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep(),
-                                                           *PSE.getSE());
+    VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep());
     return new VPWidenPointerInductionRecipe(
         Phi, Operands[0], Step, &Plan.getVFxUF(), *II,
         LoopVectorizationPlanner::getDecisionAndClampRange(
@@ -8162,7 +8108,7 @@ void VPRecipeBuilder::collectScaledReductions(VFRange &Range) {
   // extends are intended to be lowered along with the reduction itself.
 
   // Build up a set of partial reduction ops for efficient use checking.
-  SmallSet<User *, 4> PartialReductionOps;
+  SmallPtrSet<User *, 4> PartialReductionOps;
   for (const auto &[PartialRdx, _] : PartialReductionChains)
     PartialReductionOps.insert(PartialRdx.ExtendUser);
 
@@ -8301,7 +8247,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
   VPRecipeBase *Recipe;
   Instruction *Instr = R->getUnderlyingInstr();
   SmallVector<VPValue *, 4> Operands(R->operands());
-  if (auto *PhiR = dyn_cast<VPWidenPHIRecipe>(R)) {
+  if (auto *PhiR = dyn_cast<VPPhi>(R)) {
     VPBasicBlock *Parent = PhiR->getParent();
     [[maybe_unused]] VPRegionBlock *LoopRegionOf =
         Parent->getEnclosingLoopRegion();
@@ -8339,6 +8285,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
     PhiRecipe->addOperand(Operands[1]);
     return PhiRecipe;
   }
+  assert(!R->isPhi() && "only VPPhi nodes expected at this point");
 
   if (isa<TruncInst>(Instr) && (Recipe = tryToOptimizeInductionTruncate(
                                     cast<TruncInst>(Instr), Operands, Range)))
@@ -8437,8 +8384,13 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
     LVer.prepareNoAliasMetadata();
   }
 
+  // Create initial base VPlan0, to serve as common starting point for all
+  // candidates built later for specific VF ranges.
+  auto VPlan0 = VPlanTransforms::buildVPlan0(
+      OrigLoop, *LI, Legal->getWidestInductionType(),
+      getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
+
   auto MaxVFTimes2 = MaxVF * 2;
-  auto VPlan0 = VPlanTransforms::buildPlainCFG(OrigLoop, *LI);
   for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
     VFRange SubRange = {VF, MaxVFTimes2};
     if (auto Plan = tryToBuildVPlanWithVPRecipes(
@@ -8450,11 +8402,9 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
                                  *Plan, CM.getMinimalBitwidths());
       VPlanTransforms::runPass(VPlanTransforms::optimize, *Plan);
       // TODO: try to put it close to addActiveLaneMask().
-      // Discard the plan if it is not EVL-compatible
-      if (CM.foldTailWithEVL() && !HasScalarVF &&
-          !VPlanTransforms::runPass(VPlanTransforms::tryAddExplicitVectorLength,
-                                    *Plan, CM.getMaxSafeElements()))
-        break;
+      if (CM.foldTailWithEVL() && !HasScalarVF)
+        VPlanTransforms::runPass(VPlanTransforms::addExplicitVectorLength,
+                                 *Plan, CM.getMaxSafeElements());
       assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
       VPlans.push_back(std::move(Plan));
     }
@@ -8504,7 +8454,7 @@ static VPInstruction *addResumePhiRecipeForInduction(
 /// \p IVEndValues.
 static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
                                 DenseMap<VPValue *, VPValue *> &IVEndValues) {
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan);
   auto *ScalarPH = Plan.getScalarPreheader();
   auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getPredecessors()[0]);
   VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
@@ -8649,8 +8599,7 @@ static void addExitUsersForFirstOrderRecurrences(VPlan &Plan, VFRange &Range) {
     // the VPIRInstruction modeling the phi.
     for (VPUser *U : FOR->users()) {
       using namespace llvm::VPlanPatternMatch;
-      if (!match(U, m_VPInstruction<VPInstruction::ExtractLastElement>(
-                        m_Specific(FOR))))
+      if (!match(U, m_ExtractLastElement(m_Specific(FOR))))
         continue;
       // For VF vscale x 1, if vscale = 1, we are unable to extract the
       // penultimate value of the recurrence. Instead we rely on the existing
@@ -8679,25 +8628,18 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
   // visit each basic block after having visited its predecessor basic blocks.
   // ---------------------------------------------------------------------------
 
-  // Create initial VPlan skeleton, having a basic block for the pre-header
-  // which contains SCEV expansions that need to happen before the CFG is
-  // modified; a basic block for the vector pre-header, followed by a region for
-  // the vector loop, followed by the middle basic block. The skeleton vector
-  // loop region contains a header and latch basic blocks.
-
   bool RequiresScalarEpilogueCheck =
       LoopVectorizationPlanner::getDecisionAndClampRange(
           [this](ElementCount VF) {
             return !CM.requiresScalarEpilogue(VF.isVector());
           },
           Range);
-  VPlanTransforms::prepareForVectorization(
-      *Plan, Legal->getWidestInductionType(), PSE, RequiresScalarEpilogueCheck,
-      CM.foldTailByMasking(), OrigLoop,
-      getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()),
-      Legal->hasUncountableEarlyExit(), Range);
+  VPlanTransforms::handleEarlyExits(*Plan, Legal->hasUncountableEarlyExit(),
+                                    Range);
+  VPlanTransforms::addMiddleCheck(*Plan, RequiresScalarEpilogueCheck,
+                                  CM.foldTailByMasking());
+
   VPlanTransforms::createLoopRegions(*Plan);
-  VPlanTransforms::createExtractsForLiveOuts(*Plan);
 
   // Don't use getDecisionAndClampRange here, because we don't know the UF
   // so this function is better to be conservative, rather than to split
@@ -8893,8 +8835,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
   // TODO: Enable following transform when the EVL-version of extended-reduction
   // and mulacc-reduction are implemented.
   if (!CM.foldTailWithEVL()) {
-    VPCostContext CostCtx(CM.TTI, *CM.TLI, Legal->getWidestInductionType(), CM,
-                          CM.CostKind);
+    VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, CM.CostKind);
     VPlanTransforms::runPass(VPlanTransforms::convertToAbstractRecipes, *Plan,
                              CostCtx, Range);
   }
@@ -8967,7 +8908,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
     VPlanTransforms::addActiveLaneMask(*Plan, ForControlFlow,
                                        WithoutRuntimeCheck);
   }
-  VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues);
+  VPlanTransforms::optimizeInductionExitUsers(*Plan, IVEndValues, *PSE.getSE());
 
   assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
   return Plan;
@@ -8981,11 +8922,14 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
   assert(!OrigLoop->isInnermost());
   assert(EnableVPlanNativePath && "VPlan-native path is not enabled.");
 
-  auto Plan = VPlanTransforms::buildPlainCFG(OrigLoop, *LI);
-  VPlanTransforms::prepareForVectorization(
-      *Plan, Legal->getWidestInductionType(), PSE, true, false, OrigLoop,
-      getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), false,
-      Range);
+  auto Plan = VPlanTransforms::buildVPlan0(
+      OrigLoop, *LI, Legal->getWidestInductionType(),
+      getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
+  VPlanTransforms::handleEarlyExits(*Plan,
+                                    /*HasUncountableExit*/ false, Range);
+  VPlanTransforms::addMiddleCheck(*Plan, /*RequiresScalarEpilogue*/ true,
+                                  /*TailFolded*/ false);
+
   VPlanTransforms::createLoopRegions(*Plan);
 
   for (ElementCount VF : Range)
@@ -8996,7 +8940,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
           [this](PHINode *P) {
             return Legal->getIntOrFpInductionDescriptor(P);
           },
-          *PSE.getSE(), *TLI))
+          *TLI))
     return nullptr;
 
   // Collect mapping of IR header phis to header phi recipes, to be used in
@@ -9118,6 +9062,16 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
             CurrentLinkI->getFastMathFlags());
         LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
         VecOp = FMulRecipe;
+      } else if (PhiR->isInLoop() && Kind == RecurKind::AddChainWithSubs &&
+                 CurrentLinkI->getOpcode() == Instruction::Sub) {
+        Type *PhiTy = PhiR->getUnderlyingValue()->getType();
+        auto *Zero = Plan->getOrAddLiveIn(ConstantInt::get(PhiTy, 0));
+        VPWidenRecipe *Sub = new VPWidenRecipe(
+            Instruction::Sub, {Zero, CurrentLink->getOperand(1)}, {},
+            VPIRMetadata(), CurrentLinkI->getDebugLoc());
+        Sub->setUnderlyingValue(CurrentLinkI);
+        LinkVPBB->insert(Sub, CurrentLink->getIterator());
+        VecOp = Sub;
       } else {
         if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
           if (isa<VPWidenRecipe>(CurrentLink)) {
@@ -9287,8 +9241,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
       if (FinalReductionResult == U || Parent->getParent())
         continue;
       U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
-      if (match(U, m_VPInstruction<VPInstruction::ExtractLastElement>(
-                       m_VPValue())))
+      if (match(U, m_ExtractLastElement(m_VPValue())))
         cast<VPInstruction>(U)->replaceAllUsesWith(FinalReductionResult);
     }
 
@@ -9411,13 +9364,6 @@ void VPDerivedIVRecipe::execute(VPTransformState &State) {
       State.Builder, Index, getStartValue()->getLiveInIRValue(), Step, Kind,
       cast_if_present<BinaryOperator>(FPBinOp));
   DerivedIV->setName(Name);
-  // If index is the vector trip count, the concrete value will only be set in
-  // prepareToExecute, leading to missed simplifications, e.g. if it is 0.
-  // TODO: Remove the special case for the vector trip count once it is computed
-  // in VPlan and can be used during VPlan simplification.
-  assert((DerivedIV != Index ||
-          getOperand(1) == &getParent()->getPlan()->getVectorTripCount()) &&
-         "IV didn't need transforming?");
   State.set(this, DerivedIV, VPLane(0));
 }
 
@@ -9519,8 +9465,8 @@ static bool processLoopInVPlanNativePath(
 
   {
     GeneratedRTChecks Checks(PSE, DT, LI, TTI, F->getDataLayout(), CM.CostKind);
-    InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width,
-                           VF.Width, 1, &CM, BFI, PSI, Checks, BestPlan);
+    InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, VF.Width, 1, &CM,
+                           BFI, PSI, Checks, BestPlan);
     LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \""
                       << L->getHeader()->getParent()->getName() << "\"\n");
     LVP.executePlan(VF.Width, 1, BestPlan, LB, DT, false);
@@ -9802,6 +9748,9 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
     else if (&*MainScalarPH->begin() != ResumePhi)
       ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->begin());
   }
+  // Add a user to to make sure the resume phi won't get removed.
+  VPBuilder(MainScalarPH)
+      .createNaryOp(VPInstruction::ResumeForEpilogue, ResumePhi);
 }
 
 /// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
@@ -9809,7 +9758,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
 static void
 preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
                                  const SCEV2ValueTy &ExpandedSCEVs,
-                                 const EpilogueLoopVectorizationInfo &EPI) {
+                                 EpilogueLoopVectorizationInfo &EPI) {
   VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
   VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
   Header->setName("vec.epilog.vector.body");
@@ -9827,30 +9776,26 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
       // loop.
       using namespace llvm::PatternMatch;
       PHINode *EPResumeVal = &*L->getLoopPreheader()->phis().begin();
-      assert(EPResumeVal->getType() == IV->getScalarType() &&
-             match(EPResumeVal->getIncomingValueForBlock(
-                       EPI.MainLoopIterationCountCheck),
-                   m_SpecificInt(0)) &&
-             EPResumeVal ==
-                 find_singleton<PHINode>(
-                     L->getLoopPreheader()->phis(),
-                     [&EPI, IV](PHINode &P, bool) -> PHINode * {
-                       if (P.getType() == IV->getScalarType() &&
-                           match(P.getIncomingValueForBlock(
-                                     EPI.MainLoopIterationCountCheck),
-                                 m_SpecificInt(0)) &&
-                           any_of(P.incoming_values(),
-                                  [&EPI](Value *Inc) {
-                                    return Inc == EPI.VectorTripCount;
-                                  }) &&
-                           all_of(P.incoming_values(), [&EPI](Value *Inc) {
-                             return Inc == EPI.VectorTripCount ||
-                                    match(Inc, m_SpecificInt(0));
-                           }))
-                         return &P;
-                       return nullptr;
-                     }) &&
-             "Epilogue resume phis do not match!");
+      for (Value *Inc : EPResumeVal->incoming_values()) {
+        if (match(Inc, m_SpecificInt(0)))
+          continue;
+        assert(!EPI.VectorTripCount &&
+               "Must only have a single non-zero incoming value");
+        EPI.VectorTripCount = Inc;
+      }
+      // If we didn't find a non-zero vector trip count, all incoming values
+      // must be zero, which also means the vector trip count is zero. Pick the
+      // first zero as vector trip count.
+      // TODO: We should not choose VF * UF so the main vector loop is known to
+      // be dead.
+      if (!EPI.VectorTripCount) {
+        assert(
+            EPResumeVal->getNumIncomingValues() > 0 &&
+            all_of(EPResumeVal->incoming_values(),
+                   [](Value *Inc) { return match(Inc, m_SpecificInt(0)); }) &&
+            "all incoming values must be 0");
+        EPI.VectorTripCount = EPResumeVal->getOperand(0);
+      }
       VPValue *VPV = Plan.getOrAddLiveIn(EPResumeVal);
       assert(all_of(IV->users(),
                     [](const VPUser *U) {
@@ -10192,8 +10137,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     // Check if it is profitable to vectorize with runtime checks.
     bool ForceVectorization =
         Hints.getForce() == LoopVectorizeHints::FK_Enabled;
-    VPCostContext CostCtx(CM.TTI, *CM.TLI, CM.Legal->getWidestInductionType(),
-                          CM, CM.CostKind);
+    VPCostContext CostCtx(CM.TTI, *CM.TLI, LVP.getPlanFor(VF.Width), CM,
+                          CM.CostKind);
     if (!ForceVectorization &&
         !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx,
                                      LVP.getPlanFor(VF.Width), SEL,
@@ -10244,8 +10189,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     }
   } else if (IC > 1 && UserIC == 1) {
     // Tell the user interleaving is beneficial, but it explicitly disabled.
-    LLVM_DEBUG(
-        dbgs() << "LV: Interleaving is beneficial but is explicitly disabled.");
+    LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
+                         "disabled.\n");
     IntDiagMsg = {"InterleavingBeneficialButDisabled",
                   "the cost-model indicates that interleaving is beneficial "
                   "but is explicitly disabled or interleave count is set to 1"};
@@ -10316,7 +10261,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
       // interleave it.
       VPlan &BestPlan = LVP.getPlanFor(VF.Width);
       InnerLoopVectorizer Unroller(
-          L, PSE, LI, DT, TLI, TTI, AC, ORE, ElementCount::getFixed(1),
+          L, PSE, LI, DT, TTI, AC, ElementCount::getFixed(1),
           ElementCount::getFixed(1), IC, &CM, BFI, PSI, Checks, BestPlan);
 
       // TODO: Move to general VPlan pipeline once epilogue loops are also
@@ -10351,20 +10296,16 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         preparePlanForMainVectorLoop(*BestMainPlan, BestEpiPlan);
         EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1,
                                           BestEpiPlan);
-        EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TLI, TTI, AC, ORE,
-                                           EPI, &CM, BFI, PSI, Checks,
-                                           *BestMainPlan);
+        EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TTI, AC, EPI, &CM,
+                                           BFI, PSI, Checks, *BestMainPlan);
         auto ExpandedSCEVs = LVP.executePlan(EPI.MainLoopVF, EPI.MainLoopUF,
                                              *BestMainPlan, MainILV, DT, false);
         ++LoopsVectorized;
 
         // Second pass vectorizes the epilogue and adjusts the control flow
         // edges from the first pass.
-        EPI.MainLoopVF = EPI.EpilogueVF;
-        EPI.MainLoopUF = EPI.EpilogueUF;
-        EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TLI, TTI, AC,
-                                                 ORE, EPI, &CM, BFI, PSI,
-                                                 Checks, BestEpiPlan);
+        EpilogueVectorizerEpilogueLoop EpilogILV(
+            L, PSE, LI, DT, TTI, AC, EPI, &CM, BFI, PSI, Checks, BestEpiPlan);
         EpilogILV.setTripCount(MainILV.getTripCount());
         preparePlanForEpilogueVectorLoop(BestEpiPlan, L, ExpandedSCEVs, EPI);
 
@@ -10389,7 +10330,7 @@ bool LoopVectorizePass::processLoop(Loop *L) {
         if (!Checks.hasChecks())
           DisableRuntimeUnroll = true;
       } else {
-        InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width,
+        InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width,
                                VF.MinProfitableTripCount, IC, &CM, BFI, PSI,
                                Checks, BestPlan);
         // TODO: Move to general VPlan pipeline once epilogue loops are also
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 62ab3f522bb6f..37dc41413966d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -525,17 +525,17 @@ static bool isSplat(ArrayRef<Value *> VL) {
 /// instructions, we need to use the converted opcode along with the original
 /// uses.
 /// \param I The instruction to check for commutativity
-/// \param InstWithUses The instruction whose uses are analyzed for special
+/// \param ValWithUses The value whose uses are analyzed for special
 /// patterns
-static bool isCommutative(Instruction *I, Instruction *InstWithUses) {
+static bool isCommutative(Instruction *I, Value *ValWithUses) {
   if (auto *Cmp = dyn_cast<CmpInst>(I))
     return Cmp->isCommutative();
   if (auto *BO = dyn_cast<BinaryOperator>(I))
     return BO->isCommutative() ||
            (BO->getOpcode() == Instruction::Sub &&
-            !InstWithUses->hasNUsesOrMore(UsesLimit) &&
+            !ValWithUses->hasNUsesOrMore(UsesLimit) &&
             all_of(
-                InstWithUses->uses(),
+                ValWithUses->uses(),
                 [](const Use &U) {
                   // Commutative, if icmp eq/ne sub, 0
                   CmpPredicate Pred;
@@ -552,8 +552,8 @@ static bool isCommutative(Instruction *I, Instruction *InstWithUses) {
                           Flag->isOne());
                 })) ||
            (BO->getOpcode() == Instruction::FSub &&
-            !InstWithUses->hasNUsesOrMore(UsesLimit) &&
-            all_of(InstWithUses->uses(), [](const Use &U) {
+            !ValWithUses->hasNUsesOrMore(UsesLimit) &&
+            all_of(ValWithUses->uses(), [](const Use &U) {
               return match(U.getUser(),
                            m_Intrinsic<Intrinsic::fabs>(m_Specific(U.get())));
             }));
@@ -570,6 +570,19 @@ static bool isCommutative(Instruction *I, Instruction *InstWithUses) {
 /// \returns true if the instruction is commutative, false otherwise
 static bool isCommutative(Instruction *I) { return isCommutative(I, I); }
 
+/// \returns number of operands of \p I, considering commutativity. Returns 2
+/// for commutative instrinsics.
+/// \param I The instruction to check for commutativity
+static unsigned getNumberOfPotentiallyCommutativeOps(Instruction *I) {
+  if (isa<IntrinsicInst>(I) && isCommutative(I)) {
+    // IntrinsicInst::isCommutative returns true if swapping the first "two"
+    // arguments to the intrinsic produces the same result.
+    constexpr unsigned IntrinsicNumOperands = 2;
+    return IntrinsicNumOperands;
+  }
+  return I->getNumOperands();
+}
+
 template <typename T>
 static std::optional<unsigned> getInsertExtractIndex(const Value *Inst,
                                                      unsigned Offset) {
@@ -862,6 +875,16 @@ static std::optional<unsigned> getExtractIndex(const Instruction *E) {
 }
 
 namespace llvm {
+/// Checks if the provided value does not require scheduling. It does not
+/// require scheduling if this is not an instruction or it is an instruction
+/// that does not read/write memory and all operands are either not instructions
+/// or phi nodes or instructions from different blocks.
+static bool areAllOperandsNonInsts(Value *V);
+/// Checks if the provided value does not require scheduling. It does not
+/// require scheduling if this is not an instruction or it is an instruction
+/// that does not read/write memory and all users are phi nodes or instructions
+/// from the different blocks.
+static bool isUsedOutsideBlock(Value *V);
 /// Checks if the specified value does not require scheduling. It does not
 /// require scheduling if all operands and all users do not need to be scheduled
 /// in the current basic block.
@@ -1307,6 +1330,7 @@ class InstructionsState {
       : MainOp(MainOp), AltOp(AltOp), HasCopyables(HasCopyables) {}
   static InstructionsState invalid() { return {nullptr, nullptr}; }
 
+  /// Checks if the value is a copyable element.
   bool isCopyableElement(Value *V) const {
     assert(valid() && "InstructionsState is invalid.");
     if (!HasCopyables)
@@ -1338,6 +1362,8 @@ class InstructionsState {
              doesNotNeedToBeScheduled(V);
     // MainOp for copyables always schedulable to correctly identify
     // non-schedulable copyables.
+    if (getMainOp() == V)
+      return false;
     if (isCopyableElement(V)) {
       auto IsNonSchedulableCopyableElement = [this](Value *V) {
         auto *I = dyn_cast<Instruction>(V);
@@ -1355,6 +1381,7 @@ class InstructionsState {
            doesNotNeedToBeScheduled(V);
   }
 
+  /// Checks if the state represents copyable instructions.
   bool areInstructionsWithCopyableElements() const {
     assert(valid() && "InstructionsState is invalid.");
     return HasCopyables;
@@ -1886,6 +1913,7 @@ class BoUpSLP {
   class TreeEntry;
   class ScheduleEntity;
   class ScheduleData;
+  class ScheduleCopyableData;
   class ScheduleBundle;
   class ShuffleCostEstimator;
   class ShuffleInstructionBuilder;
@@ -2246,6 +2274,7 @@ class BoUpSLP {
 
     operator bool() const { return UserTE != nullptr; }
   };
+  friend struct DenseMapInfo<EdgeInfo>;
 
   /// A helper class used for scoring candidates for two consecutive lanes.
   class LookAheadHeuristics {
@@ -2384,6 +2413,11 @@ class BoUpSLP {
       if (C1 && C2)
         return LookAheadHeuristics::ScoreConstants;
 
+      // Consider constants and buildvector compatible.
+      if ((C1 && isa<InsertElementInst>(V2)) ||
+          (C2 && isa<InsertElementInst>(V1)))
+        return LookAheadHeuristics::ScoreConstants;
+
       // Extracts from consecutive indexes of the same vector better score as
       // the extracts could be optimized away.
       Value *EV1;
@@ -3010,10 +3044,9 @@ class BoUpSLP {
       assert(S.valid() && "InstructionsState is invalid.");
       // IntrinsicInst::isCommutative returns true if swapping the first "two"
       // arguments to the intrinsic produces the same result.
-      constexpr unsigned IntrinsicNumOperands = 2;
       Instruction *MainOp = S.getMainOp();
       unsigned NumOperands = MainOp->getNumOperands();
-      ArgSize = isa<IntrinsicInst>(MainOp) ? IntrinsicNumOperands : NumOperands;
+      ArgSize = ::getNumberOfPotentiallyCommutativeOps(MainOp);
       OpsVec.resize(ArgSize);
       unsigned NumLanes = VL.size();
       for (OperandDataVec &Ops : OpsVec)
@@ -3038,7 +3071,7 @@ class BoUpSLP {
         bool IsInverseOperation = false;
         if (S.isCopyableElement(VL[Lane])) {
           // The value is a copyable element.
-          IsInverseOperation = !isCommutative(MainOp);
+          IsInverseOperation = !isCommutative(MainOp, VL[Lane]);
         } else {
           assert(I && "Expected instruction");
           auto [SelectedOp, Ops] = convertTo(I, S);
@@ -3883,6 +3916,7 @@ class BoUpSLP {
     enum CombinedOpcode {
       NotCombinedOp = -1,
       MinMax = Instruction::OtherOpsEnd + 1,
+      FMulAdd,
     };
     CombinedOpcode CombinedOp = NotCombinedOp;
 
@@ -4033,6 +4067,9 @@ class BoUpSLP {
     /// Returns true if any scalar in the list is a copyable element.
     bool hasCopyableElements() const { return !CopyableElements.empty(); }
 
+    /// Returns the state of the operations.
+    const InstructionsState &getOperations() const { return S; }
+
     /// When ReuseReorderShuffleIndices is empty it just returns position of \p
     /// V within vector of Scalars. Otherwise, try to remap on its reuse index.
     unsigned findLaneForValue(Value *V) const {
@@ -4328,7 +4365,10 @@ class BoUpSLP {
     } else {
       // Build a map for gathered scalars to the nodes where they are used.
       bool AllConstsOrCasts = true;
-      for (Value *V : VL)
+      for (Value *V : VL) {
+        if (S && S.areInstructionsWithCopyableElements() &&
+            S.isCopyableElement(V))
+          Last->addCopyableElement(V);
         if (!isConstant(V)) {
           auto *I = dyn_cast<CastInst>(V);
           AllConstsOrCasts &= I && I->getType()->isIntegerTy();
@@ -4336,6 +4376,7 @@ class BoUpSLP {
               !UserTreeIdx.UserTE->isGather())
             ValueToGatherNodes.try_emplace(V).first->getSecond().insert(Last);
         }
+      }
       if (AllConstsOrCasts)
         CastMaxMinBWSizes =
             std::make_pair(std::numeric_limits<unsigned>::max(), 1);
@@ -4514,8 +4555,6 @@ class BoUpSLP {
   bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
                  Instruction *Inst2) {
     assert(Loc1.Ptr && isSimple(Inst1) && "Expected simple first instruction.");
-    if (!isSimple(Inst2))
-      return true;
     // First check if the result is already in the cache.
     AliasCacheKey Key = std::make_pair(Inst1, Inst2);
     auto Res = AliasCache.try_emplace(Key);
@@ -4524,7 +4563,6 @@ class BoUpSLP {
     bool Aliased = isModOrRefSet(BatchAA.getModRefInfo(Inst2, Loc1));
     // Store the result in the cache.
     Res.first->getSecond() = Aliased;
-    AliasCache.try_emplace(std::make_pair(Inst2, Inst1), Aliased);
     return Aliased;
   }
 
@@ -4583,16 +4621,18 @@ class BoUpSLP {
   /// List of hashes of vector of loads, which are known to be non vectorizable.
   DenseSet<size_t> ListOfKnonwnNonVectorizableLoads;
 
-  /// Represents a scheduling entity, either ScheduleData or ScheduleBundle.
-  /// ScheduleData used to gather dependecies for a single instructions, while
-  /// ScheduleBundle represents a batch of instructions, going to be groupped
-  /// together.
+  /// Represents a scheduling entity, either ScheduleData, ScheduleCopyableData
+  /// or ScheduleBundle. ScheduleData used to gather dependecies for a single
+  /// instructions, while ScheduleBundle represents a batch of instructions,
+  /// going to be groupped together. ScheduleCopyableData models extra user for
+  /// "copyable" instructions.
   class ScheduleEntity {
     friend class ScheduleBundle;
     friend class ScheduleData;
+    friend class ScheduleCopyableData;
 
   protected:
-    enum class Kind { ScheduleData, ScheduleBundle };
+    enum class Kind { ScheduleData, ScheduleBundle, ScheduleCopyableData };
     Kind getKind() const { return K; }
     ScheduleEntity(Kind K) : K(K) {}
 
@@ -4611,17 +4651,79 @@ class BoUpSLP {
     void setSchedulingPriority(int Priority) { SchedulingPriority = Priority; }
     int getSchedulingPriority() const { return SchedulingPriority; }
     bool isReady() const {
-      if (auto *SD = dyn_cast<ScheduleData>(this))
+      if (const auto *SD = dyn_cast<ScheduleData>(this))
         return SD->isReady();
+      if (const auto *CD = dyn_cast<ScheduleCopyableData>(this))
+        return CD->isReady();
       return cast<ScheduleBundle>(this)->isReady();
     }
+    /// Returns true if the dependency information has been calculated.
+    /// Note that depenendency validity can vary between instructions within
+    /// a single bundle.
+    bool hasValidDependencies() const {
+      if (const auto *SD = dyn_cast<ScheduleData>(this))
+        return SD->hasValidDependencies();
+      if (const auto *CD = dyn_cast<ScheduleCopyableData>(this))
+        return CD->hasValidDependencies();
+      return cast<ScheduleBundle>(this)->hasValidDependencies();
+    }
+    /// Gets the number of unscheduled dependencies.
+    int getUnscheduledDeps() const {
+      if (const auto *SD = dyn_cast<ScheduleData>(this))
+        return SD->getUnscheduledDeps();
+      if (const auto *CD = dyn_cast<ScheduleCopyableData>(this))
+        return CD->getUnscheduledDeps();
+      return cast<ScheduleBundle>(this)->unscheduledDepsInBundle();
+    }
+    /// Increments the number of unscheduled dependencies.
+    int incrementUnscheduledDeps(int Incr) {
+      if (auto *SD = dyn_cast<ScheduleData>(this))
+        return SD->incrementUnscheduledDeps(Incr);
+      return cast<ScheduleCopyableData>(this)->incrementUnscheduledDeps(Incr);
+    }
+    /// Gets the number of dependencies.
+    int getDependencies() const {
+      if (const auto *SD = dyn_cast<ScheduleData>(this))
+        return SD->getDependencies();
+      return cast<ScheduleCopyableData>(this)->getDependencies();
+    }
+    /// Gets the instruction.
+    Instruction *getInst() const {
+      if (const auto *SD = dyn_cast<ScheduleData>(this))
+        return SD->getInst();
+      return cast<ScheduleCopyableData>(this)->getInst();
+    }
+
     /// Gets/sets if the bundle is scheduled.
     bool isScheduled() const { return IsScheduled; }
     void setScheduled(bool Scheduled) { IsScheduled = Scheduled; }
 
     static bool classof(const ScheduleEntity *) { return true; }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+    void dump(raw_ostream &OS) const {
+      if (const auto *SD = dyn_cast<ScheduleData>(this))
+        return SD->dump(OS);
+      if (const auto *CD = dyn_cast<ScheduleCopyableData>(this))
+        return CD->dump(OS);
+      return cast<ScheduleBundle>(this)->dump(OS);
+    }
+
+    LLVM_DUMP_METHOD void dump() const {
+      dump(dbgs());
+      dbgs() << '\n';
+    }
+#endif // if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   };
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  friend inline raw_ostream &operator<<(raw_ostream &OS,
+                                        const BoUpSLP::ScheduleEntity &SE) {
+    SE.dump(OS);
+    return OS;
+  }
+#endif
+
   /// Contains all scheduling relevant data for an instruction.
   /// A ScheduleData either represents a single instruction or a member of an
   /// instruction bundle (= a group of instructions which is combined into a
@@ -4684,10 +4786,18 @@ class BoUpSLP {
 
     /// Clears all dependency information.
     void clearDependencies() {
-      Dependencies = InvalidDeps;
-      resetUnscheduledDeps();
+      clearDirectDependencies();
       MemoryDependencies.clear();
       ControlDependencies.clear();
+    }
+
+    /// Clears all direct dependencies only, except for control and memory
+    /// dependencies.
+    /// Required for copyable elements to correctly handle control/memory deps
+    /// and avoid extra reclaculation of such deps.
+    void clearDirectDependencies() {
+      Dependencies = InvalidDeps;
+      resetUnscheduledDeps();
       IsScheduled = false;
     }
 
@@ -4777,7 +4887,7 @@ class BoUpSLP {
 
   class ScheduleBundle final : public ScheduleEntity {
     /// The schedule data for the instructions in the bundle.
-    SmallVector<ScheduleData *> Bundle;
+    SmallVector<ScheduleEntity *> Bundle;
     /// True if this bundle is valid.
     bool IsValid = true;
     /// The TreeEntry that this instruction corresponds to.
@@ -4793,7 +4903,7 @@ class BoUpSLP {
 
     /// Verify basic self consistency properties
     void verify() const {
-      for (const ScheduleData *SD : Bundle) {
+      for (const ScheduleEntity *SD : Bundle) {
         if (SD->hasValidDependencies()) {
           assert(SD->getUnscheduledDeps() <= SD->getDependencies() &&
                  "invariant");
@@ -4813,7 +4923,7 @@ class BoUpSLP {
     int unscheduledDepsInBundle() const {
       assert(*this && "bundle must not be empty");
       int Sum = 0;
-      for (const ScheduleData *BundleMember : Bundle) {
+      for (const ScheduleEntity *BundleMember : Bundle) {
         if (BundleMember->getUnscheduledDeps() == ScheduleData::InvalidDeps)
           return ScheduleData::InvalidDeps;
         Sum += BundleMember->getUnscheduledDeps();
@@ -4825,7 +4935,7 @@ class BoUpSLP {
     /// Note that depenendency validity can vary between instructions within
     /// a single bundle.
     bool hasValidDependencies() const {
-      return all_of(Bundle, [](const ScheduleData *SD) {
+      return all_of(Bundle, [](const ScheduleEntity *SD) {
         return SD->hasValidDependencies();
       });
     }
@@ -4839,10 +4949,10 @@ class BoUpSLP {
 
     /// Returns the bundle of scheduling data, associated with the current
     /// instruction.
-    ArrayRef<ScheduleData *> getBundle() { return Bundle; }
-    ArrayRef<const ScheduleData *> getBundle() const { return Bundle; }
+    ArrayRef<ScheduleEntity *> getBundle() { return Bundle; }
+    ArrayRef<const ScheduleEntity *> getBundle() const { return Bundle; }
     /// Adds an instruction to the bundle.
-    void add(ScheduleData *SD) { Bundle.push_back(SD); }
+    void add(ScheduleEntity *SD) { Bundle.push_back(SD); }
 
     /// Gets/sets the associated tree entry.
     void setTreeEntry(TreeEntry *TE) { this->TE = TE; }
@@ -4859,8 +4969,11 @@ class BoUpSLP {
         return;
       }
       OS << '[';
-      interleaveComma(Bundle, OS,
-                      [&](const ScheduleData *SD) { OS << *SD->getInst(); });
+      interleaveComma(Bundle, OS, [&](const ScheduleEntity *SD) {
+        if (isa<ScheduleCopyableData>(SD))
+          OS << "<Copyable>";
+        OS << *SD->getInst();
+      });
       OS << ']';
     }
 
@@ -4879,6 +4992,129 @@ class BoUpSLP {
   }
 #endif
 
+  /// Contains all scheduling relevant data for the copyable instruction.
+  /// It models the virtual instructions, supposed to replace the original
+  /// instructions. E.g., if instruction %0 = load is a part of the bundle [%0,
+  /// %1], where %1 = add, then the ScheduleCopyableData models virtual
+  /// instruction %virt = add %0, 0.
+  class ScheduleCopyableData final : public ScheduleEntity {
+    /// The source schedule data for the instruction.
+    Instruction *Inst = nullptr;
+    /// The edge information for the instruction.
+    const EdgeInfo EI;
+    /// This ScheduleData is in the current scheduling region if this matches
+    /// the current SchedulingRegionID of BlockScheduling.
+    int SchedulingRegionID = 0;
+    /// Bundle, this data is part of.
+    ScheduleBundle &Bundle;
+
+  public:
+    ScheduleCopyableData(int BlockSchedulingRegionID, Instruction *I,
+                         const EdgeInfo &EI, ScheduleBundle &Bundle)
+        : ScheduleEntity(Kind::ScheduleCopyableData), Inst(I), EI(EI),
+          SchedulingRegionID(BlockSchedulingRegionID), Bundle(Bundle) {}
+    static bool classof(const ScheduleEntity *Entity) {
+      return Entity->getKind() == Kind::ScheduleCopyableData;
+    }
+
+    /// Verify basic self consistency properties
+    void verify() {
+      if (hasValidDependencies()) {
+        assert(UnscheduledDeps <= Dependencies && "invariant");
+      } else {
+        assert(UnscheduledDeps == Dependencies && "invariant");
+      }
+
+      if (IsScheduled) {
+        assert(hasValidDependencies() && UnscheduledDeps == 0 &&
+               "unexpected scheduled state");
+      }
+    }
+
+    /// Returns true if the dependency information has been calculated.
+    /// Note that depenendency validity can vary between instructions within
+    /// a single bundle.
+    bool hasValidDependencies() const {
+      return Dependencies != ScheduleData::InvalidDeps;
+    }
+
+    /// Returns true if it is ready for scheduling, i.e. it has no more
+    /// unscheduled depending instructions/bundles.
+    bool isReady() const { return UnscheduledDeps == 0 && !IsScheduled; }
+
+    /// Modifies the number of unscheduled dependencies for this instruction,
+    /// and returns the number of remaining dependencies for the containing
+    /// bundle.
+    int incrementUnscheduledDeps(int Incr) {
+      assert(hasValidDependencies() &&
+             "increment of unscheduled deps would be meaningless");
+      UnscheduledDeps += Incr;
+      assert(UnscheduledDeps >= 0 && "invariant");
+      return UnscheduledDeps;
+    }
+
+    /// Sets the number of unscheduled dependencies to the number of
+    /// dependencies.
+    void resetUnscheduledDeps() { UnscheduledDeps = Dependencies; }
+
+    /// Gets the number of unscheduled dependencies.
+    int getUnscheduledDeps() const { return UnscheduledDeps; }
+    /// Gets the number of dependencies.
+    int getDependencies() const { return Dependencies; }
+    /// Initializes the number of dependencies.
+    void initDependencies() { Dependencies = 0; }
+    /// Increments the number of dependencies.
+    void incDependencies() { Dependencies++; }
+
+    /// Gets scheduling region ID.
+    int getSchedulingRegionID() const { return SchedulingRegionID; }
+
+    /// Gets the instruction.
+    Instruction *getInst() const { return Inst; }
+
+    /// Clears all dependency information.
+    void clearDependencies() {
+      Dependencies = ScheduleData::InvalidDeps;
+      UnscheduledDeps = ScheduleData::InvalidDeps;
+      IsScheduled = false;
+    }
+
+    /// Gets the edge information.
+    const EdgeInfo &getEdgeInfo() const { return EI; }
+
+    /// Gets the bundle.
+    ScheduleBundle &getBundle() { return Bundle; }
+    const ScheduleBundle &getBundle() const { return Bundle; }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+    void dump(raw_ostream &OS) const { OS << "[Copyable]" << *getInst(); }
+
+    LLVM_DUMP_METHOD void dump() const {
+      dump(dbgs());
+      dbgs() << '\n';
+    }
+#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+
+  private:
+    /// true, if it has valid dependency information. These nodes always have
+    /// only single dependency.
+    int Dependencies = ScheduleData::InvalidDeps;
+
+    /// The number of dependencies minus the number of dependencies of scheduled
+    /// instructions. As soon as this is zero, the instruction/bundle gets ready
+    /// for scheduling.
+    /// Note that this is negative as long as Dependencies is not calculated.
+    int UnscheduledDeps = ScheduleData::InvalidDeps;
+  };
+
+#ifndef NDEBUG
+  friend inline raw_ostream &
+  operator<<(raw_ostream &OS, const BoUpSLP::ScheduleCopyableData &SD) {
+    SD.dump(OS);
+    return OS;
+  }
+#endif
+
   friend struct GraphTraits<BoUpSLP *>;
   friend struct DOTGraphTraits<BoUpSLP *>;
 
@@ -4905,6 +5141,10 @@ class BoUpSLP {
     void clear() {
       ScheduledBundles.clear();
       ScheduledBundlesList.clear();
+      ScheduleCopyableDataMap.clear();
+      ScheduleCopyableDataMapByInst.clear();
+      ScheduleCopyableDataMapByInstUser.clear();
+      ScheduleCopyableDataMapByUsers.clear();
       ReadyInsts.clear();
       ScheduleStart = nullptr;
       ScheduleEnd = nullptr;
@@ -4931,7 +5171,7 @@ class BoUpSLP {
         // Avoid lookup if can't possibly be in map.
         return nullptr;
       ScheduleData *SD = ScheduleDataMap.lookup(I);
-      if (SD && isInSchedulingRegion(SD))
+      if (SD && isInSchedulingRegion(*SD))
         return SD;
       return nullptr;
     }
@@ -4940,6 +5180,201 @@ class BoUpSLP {
       return getScheduleData(dyn_cast<Instruction>(V));
     }
 
+    /// Returns the ScheduleCopyableData for the given edge (user tree entry and
+    /// operand number) and value.
+    ScheduleCopyableData *getScheduleCopyableData(const EdgeInfo &EI,
+                                                  const Value *V) const {
+      if (ScheduleCopyableDataMap.empty())
+        return nullptr;
+      auto It = ScheduleCopyableDataMap.find(std::make_pair(EI, V));
+      if (It == ScheduleCopyableDataMap.end())
+        return nullptr;
+      ScheduleCopyableData *SD = It->getSecond().get();
+      if (!isInSchedulingRegion(*SD))
+        return nullptr;
+      return SD;
+    }
+
+    /// Returns the ScheduleCopyableData for the given user \p User, operand
+    /// number and operand \p V.
+    SmallVector<ScheduleCopyableData *>
+    getScheduleCopyableData(const Value *User, unsigned OperandIdx,
+                            const Value *V) {
+      if (ScheduleCopyableDataMapByInstUser.empty())
+        return {};
+      const auto It = ScheduleCopyableDataMapByInstUser.find(
+          std::make_pair(std::make_pair(User, OperandIdx), V));
+      if (It == ScheduleCopyableDataMapByInstUser.end())
+        return {};
+      SmallVector<ScheduleCopyableData *> Res;
+      for (ScheduleCopyableData *SD : It->getSecond()) {
+        if (isInSchedulingRegion(*SD))
+          Res.push_back(SD);
+      }
+      return Res;
+    }
+
+    /// Returns true if all operands of the given instruction \p User are
+    /// replaced by copyable data.
+    /// \param User The user instruction.
+    /// \param Op The operand, which might be replaced by the copyable data.
+    /// \param SLP The SLP tree.
+    /// \param NumOps The number of operands used. If the instruction uses the
+    /// same operand several times, check for the first use, then the second,
+    /// etc.
+    bool areAllOperandsReplacedByCopyableData(Instruction *User,
+                                              Instruction *Op, BoUpSLP &SLP,
+                                              unsigned NumOps) const {
+      assert(NumOps > 0 && "No operands");
+      if (ScheduleCopyableDataMap.empty())
+        return false;
+      SmallDenseMap<TreeEntry *, unsigned> PotentiallyReorderedEntriesCount;
+      SmallDenseMap<const TreeEntry *, unsigned> OrderedEntriesCount;
+      for (const Use &U : User->operands()) {
+        if (U.get() != Op)
+          continue;
+        ArrayRef<TreeEntry *> Entries = SLP.getTreeEntries(User);
+        if (Entries.empty())
+          return false;
+        // Check all tree entries, if they have operands replaced by copyable
+        // data.
+        for (TreeEntry *TE : Entries) {
+          // Check if the user is commutative.
+          // The commutatives are handled later, as their oeprands can be
+          // reordered.
+          // Same applies even for non-commutative cmps, because we can invert
+          // their predicate potentially and, thus, reorder the operands.
+          bool IsCommutativeUser =
+              ::isCommutative(TE->getMatchingMainOpOrAltOp(User), User);
+          EdgeInfo EI(TE, U.getOperandNo());
+          if (!IsCommutativeUser && !isa<CmpInst>(User)) {
+            unsigned &OpCnt =
+                OrderedEntriesCount.try_emplace(TE, 0).first->getSecond();
+            if (!getScheduleCopyableData(EI, Op) && OpCnt < NumOps)
+              return false;
+            // Found copyable operand - continue.
+            ++OpCnt;
+            continue;
+          }
+          ++PotentiallyReorderedEntriesCount.try_emplace(TE, 0)
+                .first->getSecond();
+        }
+      }
+      // Check the commutative/cmp entries.
+      if (!PotentiallyReorderedEntriesCount.empty()) {
+        for (auto &P : PotentiallyReorderedEntriesCount) {
+          auto *It = find(P.first->Scalars, User);
+          assert(It != P.first->Scalars.end() &&
+                 "User is not in the tree entry");
+          int Lane = std::distance(P.first->Scalars.begin(), It);
+          assert(Lane >= 0 && "Lane is not found");
+          if (isa<StoreInst>(User) && !P.first->ReorderIndices.empty())
+            Lane = P.first->ReorderIndices[Lane];
+          assert(Lane < static_cast<int>(P.first->Scalars.size()) &&
+                 "Couldn't find extract lane");
+          SmallVector<unsigned> OpIndices;
+          for (unsigned OpIdx :
+               seq<unsigned>(::getNumberOfPotentiallyCommutativeOps(
+                   P.first->getMainOp()))) {
+            if (P.first->getOperand(OpIdx)[Lane] == Op &&
+                getScheduleCopyableData(EdgeInfo(P.first, OpIdx), Op))
+              --P.getSecond();
+          }
+        }
+        return all_of(PotentiallyReorderedEntriesCount,
+                      [&](const std::pair<const TreeEntry *, unsigned> &P) {
+                        return P.second == NumOps - 1;
+                      });
+      }
+      return true;
+    }
+
+    SmallVector<ScheduleCopyableData *>
+    getScheduleCopyableData(const Instruction *I) const {
+      if (ScheduleCopyableDataMapByInst.empty())
+        return {};
+      const auto It = ScheduleCopyableDataMapByInst.find(I);
+      if (It == ScheduleCopyableDataMapByInst.end())
+        return {};
+      SmallVector<ScheduleCopyableData *> Res;
+      for (ScheduleCopyableData *SD : It->getSecond()) {
+        if (isInSchedulingRegion(*SD))
+          Res.push_back(SD);
+      }
+      return Res;
+    }
+
+    SmallVector<ScheduleCopyableData *>
+    getScheduleCopyableDataUsers(const Instruction *User) const {
+      if (ScheduleCopyableDataMapByUsers.empty())
+        return {};
+      const auto It = ScheduleCopyableDataMapByUsers.find(User);
+      if (It == ScheduleCopyableDataMapByUsers.end())
+        return {};
+      SmallVector<ScheduleCopyableData *> Res;
+      for (ScheduleCopyableData *SD : It->getSecond()) {
+        if (isInSchedulingRegion(*SD))
+          Res.push_back(SD);
+      }
+      return Res;
+    }
+
+    ScheduleCopyableData &addScheduleCopyableData(const EdgeInfo &EI,
+                                                  Instruction *I,
+                                                  int SchedulingRegionID,
+                                                  ScheduleBundle &Bundle) {
+      assert(!getScheduleCopyableData(EI, I) && "already in the map");
+      ScheduleCopyableData *CD =
+          ScheduleCopyableDataMap
+              .try_emplace(std::make_pair(EI, I),
+                           std::make_unique<ScheduleCopyableData>(
+                               SchedulingRegionID, I, EI, Bundle))
+              .first->getSecond()
+              .get();
+      ScheduleCopyableDataMapByInst[I].push_back(CD);
+      if (EI.UserTE) {
+        ArrayRef<Value *> Op = EI.UserTE->getOperand(EI.EdgeIdx);
+        const auto *It = find(Op, I);
+        assert(It != Op.end() && "Lane not set");
+        SmallPtrSet<Instruction *, 4> Visited;
+        do {
+          int Lane = std::distance(Op.begin(), It);
+          assert(Lane >= 0 && "Lane not set");
+          if (isa<StoreInst>(EI.UserTE->Scalars[Lane]) &&
+              !EI.UserTE->ReorderIndices.empty())
+            Lane = EI.UserTE->ReorderIndices[Lane];
+          assert(Lane < static_cast<int>(EI.UserTE->Scalars.size()) &&
+                 "Couldn't find extract lane");
+          auto *In = cast<Instruction>(EI.UserTE->Scalars[Lane]);
+          if (!Visited.insert(In).second) {
+            It = find(make_range(std::next(It), Op.end()), I);
+            continue;
+          }
+          ScheduleCopyableDataMapByInstUser
+              .try_emplace(std::make_pair(std::make_pair(In, EI.EdgeIdx), I))
+              .first->getSecond()
+              .push_back(CD);
+          ScheduleCopyableDataMapByUsers.try_emplace(I)
+              .first->getSecond()
+              .insert(CD);
+          // Remove extra deps for users, becoming non-immediate users of the
+          // instruction. It may happen, if the chain of same copyable elements
+          // appears in the tree.
+          if (In == I) {
+            EdgeInfo UserEI = EI.UserTE->UserTreeIndex;
+            if (ScheduleCopyableData *UserCD =
+                    getScheduleCopyableData(UserEI, In))
+              ScheduleCopyableDataMapByUsers[I].remove(UserCD);
+          }
+          It = find(make_range(std::next(It), Op.end()), I);
+        } while (It != Op.end());
+      } else {
+        ScheduleCopyableDataMapByUsers.try_emplace(I).first->getSecond().insert(
+            CD);
+      }
+      return *CD;
+    }
+
     ArrayRef<ScheduleBundle *> getScheduleBundles(Value *V) const {
       auto *I = dyn_cast<Instruction>(V);
       if (!I)
@@ -4950,34 +5385,44 @@ class BoUpSLP {
       return It->getSecond();
     }
 
-    bool isInSchedulingRegion(ScheduleData *SD) const {
-      return SD->getSchedulingRegionID() == SchedulingRegionID;
-    }
-
-    bool isInSchedulingRegion(const ScheduleBundle &Bundle) const {
-      return all_of(Bundle.getBundle(), [&](const ScheduleData *BundleMember) {
-        return BundleMember->getSchedulingRegionID() == SchedulingRegionID;
-      });
+    /// Returns true if the entity is in the scheduling region.
+    bool isInSchedulingRegion(const ScheduleEntity &SD) const {
+      if (const auto *Data = dyn_cast<ScheduleData>(&SD))
+        return Data->getSchedulingRegionID() == SchedulingRegionID;
+      if (const auto *CD = dyn_cast<ScheduleCopyableData>(&SD))
+        return CD->getSchedulingRegionID() == SchedulingRegionID;
+      return all_of(cast<ScheduleBundle>(SD).getBundle(),
+                    [&](const ScheduleEntity *BundleMember) {
+                      return isInSchedulingRegion(*BundleMember);
+                    });
     }
 
     /// Marks an instruction as scheduled and puts all dependent ready
     /// instructions into the ready-list.
     template <typename ReadyListType>
-    void schedule(ScheduleEntity *Data, ReadyListType &ReadyList) {
-      auto ProcessBundleMember = [&](ScheduleData *BundleMember,
-                                     ScheduleBundle *Bundle) {
+    void schedule(const BoUpSLP &R, const InstructionsState &S,
+                  const EdgeInfo &EI, ScheduleEntity *Data,
+                  ReadyListType &ReadyList) {
+      auto ProcessBundleMember = [&](ScheduleEntity *BundleMember,
+                                     ArrayRef<ScheduleBundle *> Bundles) {
         // Handle the def-use chain dependencies.
 
         // Decrement the unscheduled counter and insert to ready list if ready.
-        auto DecrUnsched = [&](ScheduleData *Data, bool IsControl = false) {
+        auto DecrUnsched = [&](auto *Data, bool IsControl = false) {
           if ((IsControl || Data->hasValidDependencies()) &&
               Data->incrementUnscheduledDeps(-1) == 0) {
             // There are no more unscheduled dependencies after
             // decrementing, so we can put the dependent instruction
             // into the ready list.
-            if (ArrayRef<ScheduleBundle *> Bundles =
-                    getScheduleBundles(Data->getInst());
-                !Bundles.empty()) {
+            SmallVector<ScheduleBundle *, 1> CopyableBundle;
+            ArrayRef<ScheduleBundle *> Bundles;
+            if (auto *CD = dyn_cast<ScheduleCopyableData>(Data)) {
+              CopyableBundle.push_back(&CD->getBundle());
+              Bundles = CopyableBundle;
+            } else {
+              Bundles = getScheduleBundles(Data->getInst());
+            }
+            if (!Bundles.empty()) {
               for (ScheduleBundle *Bundle : Bundles) {
                 if (Bundle->unscheduledDepsInBundle() == 0) {
                   assert(!Bundle->isScheduled() &&
@@ -4991,12 +5436,23 @@ class BoUpSLP {
             }
             assert(!Data->isScheduled() &&
                    "already scheduled bundle gets ready");
+            assert(!isa<ScheduleCopyableData>(Data) &&
+                   "Expected non-copyable data");
             ReadyList.insert(Data);
             LLVM_DEBUG(dbgs() << "SLP:    gets ready: " << *Data << "\n");
           }
         };
 
-        auto DecrUnschedForInst = [&](Instruction *I) {
+        auto DecrUnschedForInst = [&](Instruction *User, unsigned OpIdx,
+                                      Instruction *I) {
+          if (!ScheduleCopyableDataMap.empty()) {
+            SmallVector<ScheduleCopyableData *> CopyableData =
+                getScheduleCopyableData(User, OpIdx, I);
+            for (ScheduleCopyableData *CD : CopyableData)
+              DecrUnsched(CD, /*IsControl=*/false);
+            if (!CopyableData.empty())
+              return;
+          }
           if (ScheduleData *OpSD = getScheduleData(I))
             DecrUnsched(OpSD, /*IsControl=*/false);
         };
@@ -5004,45 +5460,101 @@ class BoUpSLP {
         // If BundleMember is a vector bundle, its operands may have been
         // reordered during buildTree(). We therefore need to get its operands
         // through the TreeEntry.
-        if (Bundle) {
-          // Need to search for the lane since the tree entry can be reordered.
+        if (!Bundles.empty()) {
           auto *In = BundleMember->getInst();
-          int Lane = std::distance(Bundle->getTreeEntry()->Scalars.begin(),
-                                   find(Bundle->getTreeEntry()->Scalars, In));
-          assert(Lane >= 0 && "Lane not set");
-
-          // Since vectorization tree is being built recursively this assertion
-          // ensures that the tree entry has all operands set before reaching
-          // this code. Couple of exceptions known at the moment are extracts
-          // where their second (immediate) operand is not added. Since
-          // immediates do not affect scheduler behavior this is considered
-          // okay.
-          assert(In &&
-                 (isa<ExtractValueInst, ExtractElementInst, CallBase>(In) ||
-                  In->getNumOperands() ==
-                      Bundle->getTreeEntry()->getNumOperands()) &&
-                 "Missed TreeEntry operands?");
-
-          for (unsigned OpIdx :
-               seq<unsigned>(Bundle->getTreeEntry()->getNumOperands()))
-            if (auto *I = dyn_cast<Instruction>(
-                    Bundle->getTreeEntry()->getOperand(OpIdx)[Lane])) {
-              LLVM_DEBUG(dbgs()
-                         << "SLP:   check for readiness (def): " << *I << "\n");
-              DecrUnschedForInst(I);
+          // Count uses of each instruction operand.
+          SmallDenseMap<const Instruction *, unsigned> OperandsUses;
+          unsigned TotalOpCount = 0;
+          if (isa<ScheduleCopyableData>(BundleMember)) {
+            // Copyable data is used only once (uses itself).
+            TotalOpCount = OperandsUses[In] = 1;
+          } else {
+            for (const Use &U : In->operands()) {
+              if (auto *I = dyn_cast<Instruction>(U.get())) {
+                auto Res = OperandsUses.try_emplace(I, 0);
+                ++Res.first->getSecond();
+                ++TotalOpCount;
+              }
+            }
+          }
+          // Decrement the unscheduled counter and insert to ready list if
+          // ready.
+          auto DecrUnschedForInst = [&](Instruction *I, TreeEntry *UserTE,
+                                        unsigned OpIdx) {
+            if (!ScheduleCopyableDataMap.empty()) {
+              const EdgeInfo EI = {UserTE, OpIdx};
+              if (ScheduleCopyableData *CD = getScheduleCopyableData(EI, I)) {
+                DecrUnsched(CD, /*IsControl=*/false);
+                return;
+              }
             }
+            auto It = OperandsUses.find(I);
+            assert(It != OperandsUses.end() && "Operand not found");
+            if (It->second > 0) {
+              --It->getSecond();
+              assert(TotalOpCount > 0 && "No more operands to decrement");
+              --TotalOpCount;
+              if (ScheduleData *OpSD = getScheduleData(I))
+                DecrUnsched(OpSD, /*IsControl=*/false);
+            }
+          };
+
+          for (ScheduleBundle *Bundle : Bundles) {
+            if (ScheduleCopyableDataMap.empty() && TotalOpCount == 0)
+              break;
+            // Need to search for the lane since the tree entry can be
+            // reordered.
+            int Lane = std::distance(Bundle->getTreeEntry()->Scalars.begin(),
+                                     find(Bundle->getTreeEntry()->Scalars, In));
+            assert(Lane >= 0 && "Lane not set");
+            if (isa<StoreInst>(In) &&
+                !Bundle->getTreeEntry()->ReorderIndices.empty())
+              Lane = Bundle->getTreeEntry()->ReorderIndices[Lane];
+            assert(Lane < static_cast<int>(
+                              Bundle->getTreeEntry()->Scalars.size()) &&
+                   "Couldn't find extract lane");
+
+            // Since vectorization tree is being built recursively this
+            // assertion ensures that the tree entry has all operands set before
+            // reaching this code. Couple of exceptions known at the moment are
+            // extracts where their second (immediate) operand is not added.
+            // Since immediates do not affect scheduler behavior this is
+            // considered okay.
+            assert(In &&
+                   (isa<ExtractValueInst, ExtractElementInst, CallBase>(In) ||
+                    In->getNumOperands() ==
+                        Bundle->getTreeEntry()->getNumOperands() ||
+                    Bundle->getTreeEntry()->isCopyableElement(In)) &&
+                   "Missed TreeEntry operands?");
+
+            for (unsigned OpIdx :
+                 seq<unsigned>(Bundle->getTreeEntry()->getNumOperands()))
+              if (auto *I = dyn_cast<Instruction>(
+                      Bundle->getTreeEntry()->getOperand(OpIdx)[Lane])) {
+                LLVM_DEBUG(dbgs() << "SLP:   check for readiness (def): " << *I
+                                  << "\n");
+                DecrUnschedForInst(I, Bundle->getTreeEntry(), OpIdx);
+              }
+          }
         } else {
           // If BundleMember is a stand-alone instruction, no operand reordering
           // has taken place, so we directly access its operands.
-          for (Use &U : BundleMember->getInst()->operands())
+          for (Use &U : BundleMember->getInst()->operands()) {
             if (auto *I = dyn_cast<Instruction>(U.get())) {
               LLVM_DEBUG(dbgs()
                          << "SLP:   check for readiness (def): " << *I << "\n");
-              DecrUnschedForInst(I);
+              DecrUnschedForInst(BundleMember->getInst(), U.getOperandNo(), I);
             }
+          }
         }
         // Handle the memory dependencies.
-        for (ScheduleData *MemoryDep : BundleMember->getMemoryDependencies()) {
+        auto *SD = dyn_cast<ScheduleData>(BundleMember);
+        if (!SD)
+          return;
+        SmallPtrSet<const ScheduleData *, 4> VisitedMemory;
+        for (ScheduleData *MemoryDep : SD->getMemoryDependencies()) {
+          if (!VisitedMemory.insert(MemoryDep).second)
+            continue;
           // There are no more unscheduled dependencies after decrementing,
           // so we can put the dependent instruction into the ready list.
           LLVM_DEBUG(dbgs() << "SLP:   check for readiness (mem): "
@@ -5050,7 +5562,10 @@ class BoUpSLP {
           DecrUnsched(MemoryDep);
         }
         // Handle the control dependencies.
-        for (ScheduleData *Dep : BundleMember->getControlDependencies()) {
+        SmallPtrSet<const ScheduleData *, 4> VisitedControl;
+        for (ScheduleData *Dep : SD->getControlDependencies()) {
+          if (!VisitedControl.insert(Dep).second)
+            continue;
           // There are no more unscheduled dependencies after decrementing,
           // so we can put the dependent instruction into the ready list.
           LLVM_DEBUG(dbgs()
@@ -5061,23 +5576,29 @@ class BoUpSLP {
       if (auto *SD = dyn_cast<ScheduleData>(Data)) {
         SD->setScheduled(/*Scheduled=*/true);
         LLVM_DEBUG(dbgs() << "SLP:   schedule " << *SD << "\n");
-        ProcessBundleMember(SD, nullptr);
+        ProcessBundleMember(SD, {});
       } else {
         ScheduleBundle &Bundle = *cast<ScheduleBundle>(Data);
         Bundle.setScheduled(/*Scheduled=*/true);
         LLVM_DEBUG(dbgs() << "SLP:   schedule " << Bundle << "\n");
-        auto AreAllBundlesScheduled = [&](const ScheduleData *SD) {
-          ArrayRef<ScheduleBundle *> SDBundles =
-              getScheduleBundles(SD->getInst());
-          return !SDBundles.empty() &&
-                 all_of(SDBundles, [&](const ScheduleBundle *SDBundle) {
-                   return SDBundle->isScheduled();
-                 });
-        };
-        for (ScheduleData *SD : Bundle.getBundle()) {
-          if (AreAllBundlesScheduled(SD)) {
+        auto AreAllBundlesScheduled =
+            [&](const ScheduleEntity *SD,
+                ArrayRef<ScheduleBundle *> SDBundles) {
+              if (isa<ScheduleCopyableData>(SD))
+                return true;
+              return !SDBundles.empty() &&
+                     all_of(SDBundles, [&](const ScheduleBundle *SDBundle) {
+                       return SDBundle->isScheduled();
+                     });
+            };
+        for (ScheduleEntity *SD : Bundle.getBundle()) {
+          ArrayRef<ScheduleBundle *> SDBundles;
+          if (!isa<ScheduleCopyableData>(SD))
+            SDBundles = getScheduleBundles(SD->getInst());
+          if (AreAllBundlesScheduled(SD, SDBundles)) {
             SD->setScheduled(/*Scheduled=*/true);
-            ProcessBundleMember(SD, &Bundle);
+            ProcessBundleMember(SD, isa<ScheduleCopyableData>(SD) ? &Bundle
+                                                                  : SDBundles);
           }
         }
       }
@@ -5105,7 +5626,7 @@ class BoUpSLP {
         auto *SD = getScheduleData(I);
         if (!SD)
           continue;
-        assert(isInSchedulingRegion(SD) &&
+        assert(isInSchedulingRegion(*SD) &&
                "primary schedule data not in window?");
         SD->verify();
       }
@@ -5146,8 +5667,11 @@ class BoUpSLP {
 
     /// Build a bundle from the ScheduleData nodes corresponding to the
     /// scalar instruction for each lane.
+    /// \param VL The list of scalar instructions.
+    /// \param S The state of the instructions.
+    /// \param EI The edge in the SLP graph or the user node/operand number.
     ScheduleBundle &buildBundle(ArrayRef<Value *> VL,
-                                const InstructionsState &S);
+                                const InstructionsState &S, const EdgeInfo &EI);
 
     /// Checks if a bundle of instructions can be scheduled, i.e. has no
     /// cyclic dependencies. This is only a dry-run, no instructions are
@@ -5156,7 +5680,7 @@ class BoUpSLP {
     /// std::nullopt if \p VL is allowed to be scheduled.
     std::optional<ScheduleBundle *>
     tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
-                      const InstructionsState &S);
+                      const InstructionsState &S, const EdgeInfo &EI);
 
     /// Allocates schedule data chunk.
     ScheduleData *allocateScheduleDataChunks();
@@ -5174,7 +5698,8 @@ class BoUpSLP {
     /// Updates the dependency information of a bundle and of all instructions/
     /// bundles which depend on the original bundle.
     void calculateDependencies(ScheduleBundle &Bundle, bool InsertInReadyList,
-                               BoUpSLP *SLP);
+                               BoUpSLP *SLP,
+                               ArrayRef<ScheduleData *> ControlDeps = {});
 
     /// Sets all instruction in the scheduling region to un-scheduled.
     void resetSchedule();
@@ -5196,6 +5721,48 @@ class BoUpSLP {
     /// ScheduleData structures are recycled.
     SmallDenseMap<Instruction *, ScheduleData *> ScheduleDataMap;
 
+    /// Attaches ScheduleCopyableData to EdgeInfo (UserTreeEntry + operand
+    /// number) and the operand instruction, represented as copyable element.
+    SmallDenseMap<std::pair<EdgeInfo, const Value *>,
+                  std::unique_ptr<ScheduleCopyableData>>
+        ScheduleCopyableDataMap;
+
+    /// Represents mapping between instruction and all related
+    /// ScheduleCopyableData (for all uses in the tree, represenedt as copyable
+    /// element). The SLP tree may contain several representations of the same
+    /// instruction.
+    SmallDenseMap<const Instruction *, SmallVector<ScheduleCopyableData *>>
+        ScheduleCopyableDataMapByInst;
+
+    /// Represents mapping between user value and operand number, the operand
+    /// value and all related ScheduleCopyableData. The relation is 1:n, because
+    /// the same user may refernce the same operand in different tree entries
+    /// and the operand may be modelled by the different copyable data element.
+    SmallDenseMap<std::pair<std::pair<const Value *, unsigned>, const Value *>,
+                  SmallVector<ScheduleCopyableData *>>
+        ScheduleCopyableDataMapByInstUser;
+
+    /// Represents mapping between instruction and all related
+    /// ScheduleCopyableData. It represents the mapping between the actual
+    /// instruction and the last copyable data element in the chain. E.g., if
+    /// the graph models the following instructions:
+    /// %0 = non-add instruction ...
+    /// ...
+    /// %4 = add %3, 1
+    /// %5 = add %4, 1
+    /// %6 = insertelement poison, %0, 0
+    /// %7 = insertelement %6, %5, 1
+    /// And the graph is modeled as:
+    /// [%5, %0] -> [%4, copyable %0 <0> ] -> [%3, copyable %0 <1> ]
+    ///          -> [1, 0]                 -> [%1, 0]
+    ///
+    /// this map will map %0 only to the copyable element <1>, which is the last
+    /// user (direct user of the actual instruction). <0> uses <1>, so <1> will
+    /// keep the map to <0>, not the %0.
+    SmallDenseMap<const Instruction *,
+                  SmallSetVector<ScheduleCopyableData *, 4>>
+        ScheduleCopyableDataMapByUsers;
+
     /// Attaches ScheduleBundle to Instruction.
     SmallDenseMap<Instruction *, SmallVector<ScheduleBundle *>>
         ScheduledBundles;
@@ -5242,7 +5809,7 @@ class BoUpSLP {
 
   /// Performs the "real" scheduling. Done before vectorization is actually
   /// performed in a basic block.
-  void scheduleBlock(BlockScheduling *BS);
+  void scheduleBlock(const BoUpSLP &R, BlockScheduling *BS);
 
   /// List of users to ignore during scheduling and that don't need extracting.
   const SmallDenseSet<Value *> *UserIgnoreList = nullptr;
@@ -5315,6 +5882,30 @@ class BoUpSLP {
 
 } // end namespace slpvectorizer
 
+template <> struct DenseMapInfo<BoUpSLP::EdgeInfo> {
+  using FirstInfo = DenseMapInfo<BoUpSLP::TreeEntry *>;
+  using SecondInfo = DenseMapInfo<unsigned>;
+  static BoUpSLP::EdgeInfo getEmptyKey() {
+    return BoUpSLP::EdgeInfo(FirstInfo::getEmptyKey(),
+                             SecondInfo::getEmptyKey());
+  }
+
+  static BoUpSLP::EdgeInfo getTombstoneKey() {
+    return BoUpSLP::EdgeInfo(FirstInfo::getTombstoneKey(),
+                             SecondInfo::getTombstoneKey());
+  }
+
+  static unsigned getHashValue(const BoUpSLP::EdgeInfo &Val) {
+    return detail::combineHashValue(FirstInfo::getHashValue(Val.UserTE),
+                                    SecondInfo::getHashValue(Val.EdgeIdx));
+  }
+
+  static bool isEqual(const BoUpSLP::EdgeInfo &LHS,
+                      const BoUpSLP::EdgeInfo &RHS) {
+    return LHS == RHS;
+  }
+};
+
 template <> struct GraphTraits<BoUpSLP *> {
   using TreeEntry = BoUpSLP::TreeEntry;
 
@@ -7191,12 +7782,45 @@ bool BoUpSLP::isProfitableToReorder() const {
     // Check if the tree has only single store and single (unordered) load node,
     // other nodes are phis or geps/binops, combined with phis, and/or single
     // gather load node
-    bool HasPhis = false;
     if (VectorizableTree.front()->hasState() &&
         VectorizableTree.front()->getOpcode() == Instruction::PHI &&
         VectorizableTree.front()->Scalars.size() == TinyVF &&
         VectorizableTree.front()->getNumOperands() > PhiOpsLimit)
       return false;
+    // Single node, which require reorder - skip.
+    if (VectorizableTree.front()->hasState() &&
+        VectorizableTree.front()->getOpcode() == Instruction::Store &&
+        VectorizableTree.front()->ReorderIndices.empty()) {
+      const unsigned ReorderedSplitsCnt =
+          count_if(VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
+            return TE->State == TreeEntry::SplitVectorize &&
+                   !TE->ReorderIndices.empty() && TE->UserTreeIndex.UserTE &&
+                   TE->UserTreeIndex.UserTE->State == TreeEntry::Vectorize &&
+                   ::isCommutative(TE->UserTreeIndex.UserTE->getMainOp());
+          });
+      if (ReorderedSplitsCnt <= 1 &&
+          static_cast<unsigned>(count_if(
+              VectorizableTree, [&](const std::unique_ptr<TreeEntry> &TE) {
+                return ((!TE->isGather() &&
+                         (TE->ReorderIndices.empty() ||
+                          (TE->UserTreeIndex.UserTE &&
+                           TE->UserTreeIndex.UserTE->State ==
+                               TreeEntry::Vectorize &&
+                           !TE->UserTreeIndex.UserTE->ReuseShuffleIndices
+                                .empty()))) ||
+                        (TE->isGather() && TE->ReorderIndices.empty() &&
+                         (!TE->hasState() || TE->isAltShuffle() ||
+                          TE->getOpcode() == Instruction::Load ||
+                          TE->getOpcode() == Instruction::ZExt ||
+                          TE->getOpcode() == Instruction::SExt))) &&
+                       (VectorizableTree.front()->getVectorFactor() > TinyVF ||
+                        !TE->isGather() || none_of(TE->Scalars, [&](Value *V) {
+                          return !isConstant(V) && isVectorized(V);
+                        }));
+              })) >= VectorizableTree.size() - ReorderedSplitsCnt)
+        return false;
+    }
+    bool HasPhis = false;
     bool HasLoad = true;
     unsigned GatherLoads = 0;
     for (const std::unique_ptr<TreeEntry> &TE :
@@ -9768,7 +10392,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
        }))) {
     if (TryPad && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
         S.getMainOp()->isSafeToRemove() &&
-        all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
+        (S.areInstructionsWithCopyableElements() ||
+         all_of(UniqueValues, IsaPred<Instruction, PoisonValue>))) {
       // Find the number of elements, which forms full vectors.
       unsigned PWSz = getFullVectorNumberOfElements(
           TTI, UniqueValues.front()->getType(), UniqueValues.size());
@@ -9785,8 +10410,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
         PaddedUniqueValues.append(
             PWSz - UniqueValues.size(),
             PoisonValue::get(UniqueValues.front()->getType()));
-        // Check that extended with poisons operations are still valid for
-        // vectorization (div/rem are not allowed).
+        // Check that extended with poisons/copyable operations are still valid
+        // for vectorization (div/rem are not allowed).
         if (!S.areInstructionsWithCopyableElements() &&
             !getSameOpcode(PaddedUniqueValues, TLI).valid()) {
           LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
@@ -9948,35 +10573,41 @@ class InstructionsCompatibilityAnalysis {
   unsigned MainOpcode = 0;
   Instruction *MainOp = nullptr;
 
+  /// Checks if the opcode is supported as the main opcode for copyable
+  /// elements.
+  static bool isSupportedOpcode(const unsigned Opcode) {
+    return Opcode == Instruction::Add || Opcode == Instruction::LShr;
+  }
+
   /// Identifies the best candidate value, which represents main opcode
   /// operation.
   /// Currently the best candidate is the Add instruction with the parent
   /// block with the highest DFS incoming number (block, that dominates other).
-  void findAndSetMainInstruction(ArrayRef<Value *> VL) {
+  void findAndSetMainInstruction(ArrayRef<Value *> VL, const BoUpSLP &R) {
     BasicBlock *Parent = nullptr;
     // Checks if the instruction has supported opcode.
-    auto IsSupportedOpcode = [](Instruction *I) {
-      return I && I->getOpcode() == Instruction::Add;
+    auto IsSupportedInstruction = [&](Instruction *I) {
+      return I && isSupportedOpcode(I->getOpcode()) &&
+             (!doesNotNeedToBeScheduled(I) || !R.isVectorized(I));
     };
+    // Exclude operands instructions immediately to improve compile time, it
+    // will be unable to schedule anyway.
     SmallDenseSet<Value *, 8> Operands;
+    SmallMapVector<unsigned, SmallVector<Instruction *>, 4> Candidates;
     for (Value *V : VL) {
       auto *I = dyn_cast<Instruction>(V);
       if (!I)
         continue;
       if (!DT.isReachableFromEntry(I->getParent()))
         continue;
-      if (!MainOp) {
-        MainOp = I;
+      if (Candidates.empty()) {
+        Candidates.try_emplace(I->getOpcode()).first->second.push_back(I);
         Parent = I->getParent();
         Operands.insert(I->op_begin(), I->op_end());
         continue;
       }
       if (Parent == I->getParent()) {
-        if (!IsSupportedOpcode(MainOp))
-          MainOp = I;
-        if (MainOp->getOpcode() == I->getOpcode() &&
-            doesNotNeedToBeScheduled(MainOp) && !doesNotNeedToBeScheduled(I))
-          MainOp = I;
+        Candidates.try_emplace(I->getOpcode()).first->second.push_back(I);
         Operands.insert(I->op_begin(), I->op_end());
         continue;
       }
@@ -9988,24 +10619,35 @@ class InstructionsCompatibilityAnalysis {
                  (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
              "Different nodes should have different DFS numbers");
       if (NodeA->getDFSNumIn() < NodeB->getDFSNumIn()) {
-        MainOp = I;
+        Candidates.clear();
+        Candidates.try_emplace(I->getOpcode()).first->second.push_back(I);
         Parent = I->getParent();
         Operands.clear();
         Operands.insert(I->op_begin(), I->op_end());
       }
     }
-    if (!IsSupportedOpcode(MainOp) || Operands.contains(MainOp)) {
-      MainOp = nullptr;
-      return;
+    unsigned BestOpcodeNum = 0;
+    MainOp = nullptr;
+    for (const auto &P : Candidates) {
+      if (P.second.size() < BestOpcodeNum)
+        continue;
+      for (Instruction *I : P.second) {
+        if (IsSupportedInstruction(I) && !Operands.contains(I)) {
+          MainOp = I;
+          BestOpcodeNum = P.second.size();
+          break;
+        }
+      }
     }
-    MainOpcode = MainOp->getOpcode();
+    if (MainOp)
+      MainOpcode = MainOp->getOpcode();
   }
 
   /// Returns the idempotent value for the \p MainOp with the detected \p
   /// MainOpcode. For Add, returns 0. For Or, it should choose between false and
   /// the operand itself, since V or V == V.
   Value *selectBestIdempotentValue() const {
-    assert(MainOpcode == Instruction::Add && "Unsupported opcode");
+    assert(isSupportedOpcode(MainOpcode) && "Unsupported opcode");
     return ConstantExpr::getBinOpIdentity(MainOpcode, MainOp->getType(),
                                           !MainOp->isCommutative());
   }
@@ -10018,13 +10660,8 @@ class InstructionsCompatibilityAnalysis {
       return {V, V};
     if (!S.isCopyableElement(V))
       return convertTo(cast<Instruction>(V), S).second;
-    switch (MainOpcode) {
-    case Instruction::Add:
-      return {V, selectBestIdempotentValue()};
-    default:
-      break;
-    }
-    llvm_unreachable("Unsupported opcode");
+    assert(isSupportedOpcode(MainOpcode) && "Unsupported opcode");
+    return {V, selectBestIdempotentValue()};
   }
 
   /// Builds operands for the original instructions.
@@ -10198,16 +10835,10 @@ class InstructionsCompatibilityAnalysis {
       return S;
     if (!VectorizeCopyableElements || !TryCopyableElementsVectorization)
       return S;
-    findAndSetMainInstruction(VL);
+    findAndSetMainInstruction(VL, R);
     if (!MainOp)
       return InstructionsState::invalid();
     S = InstructionsState(MainOp, MainOp, /*HasCopyables=*/true);
-    // TODO: Remove this check once support for schulable copyables is landed.
-    if (any_of(VL, [&](Value *V) {
-          return S.isCopyableElement(V) && !S.isNonSchedulable(V);
-        }))
-      return InstructionsState::invalid();
-
     if (!WithProfitabilityCheck)
       return S;
     // Check if it is profitable to vectorize the instruction.
@@ -10243,6 +10874,21 @@ class InstructionsCompatibilityAnalysis {
       }
       if (!Res)
         return InstructionsState::invalid();
+      constexpr TTI::TargetCostKind Kind = TTI::TCK_RecipThroughput;
+      InstructionCost ScalarCost = TTI.getInstructionCost(S.getMainOp(), Kind);
+      InstructionCost VectorCost;
+      FixedVectorType *VecTy =
+          getWidenedType(S.getMainOp()->getType(), VL.size());
+      switch (MainOpcode) {
+      case Instruction::Add:
+      case Instruction::LShr:
+        VectorCost = TTI.getArithmeticInstrCost(MainOpcode, VecTy, Kind);
+        break;
+      default:
+        llvm_unreachable("Unexpected instruction.");
+      }
+      if (VectorCost > ScalarCost)
+        return InstructionsState::invalid();
       return S;
     }
     assert(Operands.size() == 2 && "Unexpected number of operands!");
@@ -10727,7 +11373,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
 
   SetVector<Value *> UniqueValues(llvm::from_range, VL);
   std::optional<ScheduleBundle *> BundlePtr =
-      BS.tryScheduleBundle(UniqueValues.getArrayRef(), this, S);
+      BS.tryScheduleBundle(UniqueValues.getArrayRef(), this, S, UserTreeIdx);
 #ifdef EXPENSIVE_CHECKS
   // Make sure we didn't break any internal invariants
   BS.verify();
@@ -11987,6 +12633,89 @@ void BoUpSLP::reorderGatherNode(TreeEntry &TE) {
   }
 }
 
+/// Check if we can convert fadd/fsub sequence to FMAD.
+/// \returns Cost of the FMAD, if conversion is possible, invalid cost otherwise.
+static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
+                                       const InstructionsState &S,
+                                       DominatorTree &DT, const DataLayout &DL,
+                                       TargetTransformInfo &TTI,
+                                       const TargetLibraryInfo &TLI) {
+  assert(all_of(VL,
+                [](Value *V) {
+                  return V->getType()->getScalarType()->isFloatingPointTy();
+                }) &&
+         "Can only convert to FMA for floating point types");
+  assert(S.isAddSubLikeOp() && "Can only convert to FMA for add/sub");
+
+  auto CheckForContractable = [&](ArrayRef<Value *> VL) {
+    FastMathFlags FMF;
+    FMF.set();
+    for (Value *V : VL) {
+      auto *I = dyn_cast<Instruction>(V);
+      if (!I)
+        continue;
+      if (S.isCopyableElement(I))
+        continue;
+      Instruction *MatchingI = S.getMatchingMainOpOrAltOp(I);
+      if (S.getMainOp() != MatchingI && S.getAltOp() != MatchingI)
+        continue;
+      if (auto *FPCI = dyn_cast<FPMathOperator>(I))
+        FMF &= FPCI->getFastMathFlags();
+    }
+    return FMF.allowContract();
+  };
+  if (!CheckForContractable(VL))
+    return InstructionCost::getInvalid();
+  // fmul also should be contractable
+  InstructionsCompatibilityAnalysis Analysis(DT, DL, TTI, TLI);
+  SmallVector<BoUpSLP::ValueList> Operands = Analysis.buildOperands(S, VL);
+
+  InstructionsState OpS = getSameOpcode(Operands.front(), TLI);
+  if (!OpS.valid())
+    return InstructionCost::getInvalid();
+
+  if (OpS.isAltShuffle() || OpS.getOpcode() != Instruction::FMul)
+    return InstructionCost::getInvalid();
+  if (!CheckForContractable(Operands.front()))
+    return InstructionCost::getInvalid();
+  // Compare the costs.
+  InstructionCost FMulPlusFAddCost = 0;
+  InstructionCost FMACost = 0;
+  constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+  FastMathFlags FMF;
+  FMF.set();
+  for (Value *V : VL) {
+    auto *I = dyn_cast<Instruction>(V);
+    if (!I)
+      continue;
+    if (!S.isCopyableElement(I))
+      if (auto *FPCI = dyn_cast<FPMathOperator>(I))
+        FMF &= FPCI->getFastMathFlags();
+    FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
+  }
+  unsigned NumOps = 0;
+  for (auto [V, Op] : zip(VL, Operands.front())) {
+    if (S.isCopyableElement(V))
+      continue;
+    auto *I = dyn_cast<Instruction>(Op);
+    if (!I || !I->hasOneUse() || OpS.isCopyableElement(I)) {
+      if (auto *OpI = dyn_cast<Instruction>(V))
+        FMACost += TTI.getInstructionCost(OpI, CostKind);
+      if (I)
+        FMACost += TTI.getInstructionCost(I, CostKind);
+      continue;
+    }
+    ++NumOps;
+    if (auto *FPCI = dyn_cast<FPMathOperator>(I))
+      FMF &= FPCI->getFastMathFlags();
+    FMulPlusFAddCost += TTI.getInstructionCost(I, CostKind);
+  }
+  Type *Ty = VL.front()->getType();
+  IntrinsicCostAttributes ICA(Intrinsic::fmuladd, Ty, {Ty, Ty, Ty}, FMF);
+  FMACost += NumOps * TTI.getIntrinsicInstrCost(ICA, CostKind);
+  return FMACost < FMulPlusFAddCost ? FMACost : InstructionCost::getInvalid();
+}
+
 void BoUpSLP::transformNodes() {
   constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
   BaseGraphSize = VectorizableTree.size();
@@ -12355,6 +13084,25 @@ void BoUpSLP::transformNodes() {
       }
       break;
     }
+    case Instruction::FSub:
+    case Instruction::FAdd: {
+      // Check if possible to convert (a*b)+c to fma.
+      if (E.State != TreeEntry::Vectorize ||
+          !E.getOperations().isAddSubLikeOp())
+        break;
+      if (!canConvertToFMA(E.Scalars, E.getOperations(), *DT, *DL, *TTI, *TLI)
+               .isValid())
+        break;
+      // This node is a fmuladd node.
+      E.CombinedOp = TreeEntry::FMulAdd;
+      TreeEntry *FMulEntry = getOperandEntry(&E, 0);
+      if (FMulEntry->UserTreeIndex &&
+          FMulEntry->State == TreeEntry::Vectorize) {
+        // The FMul node is part of the combined fmuladd node.
+        FMulEntry->State = TreeEntry::CombinedVectorize;
+      }
+      break;
+    }
     default:
       break;
     }
@@ -13587,6 +14335,11 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
     }
     return IntrinsicCost;
   };
+  auto GetFMulAddCost = [&, &TTI = *TTI](const InstructionsState &S,
+                                         Instruction *VI) {
+    InstructionCost Cost = canConvertToFMA(VI, S, *DT, *DL, TTI, *TLI);
+    return Cost;
+  };
   switch (ShuffleOrOp) {
   case Instruction::PHI: {
     // Count reused scalars.
@@ -13927,6 +14680,30 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
     };
     return GetCostDiff(GetScalarCost, GetVectorCost);
   }
+  case TreeEntry::FMulAdd: {
+    auto GetScalarCost = [&](unsigned Idx) {
+      if (isa<PoisonValue>(UniqueValues[Idx]))
+        return InstructionCost(TTI::TCC_Free);
+      return GetFMulAddCost(E->getOperations(),
+                            cast<Instruction>(UniqueValues[Idx]));
+    };
+    auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
+      FastMathFlags FMF;
+      FMF.set();
+      for (Value *V : E->Scalars) {
+        if (auto *FPCI = dyn_cast<FPMathOperator>(V)) {
+          FMF &= FPCI->getFastMathFlags();
+          if (auto *FPCIOp = dyn_cast<FPMathOperator>(FPCI->getOperand(0)))
+            FMF &= FPCIOp->getFastMathFlags();
+        }
+      }
+      IntrinsicCostAttributes ICA(Intrinsic::fmuladd, VecTy,
+                                  {VecTy, VecTy, VecTy}, FMF);
+      InstructionCost VecCost = TTI.getIntrinsicInstrCost(ICA, CostKind);
+      return VecCost + CommonCost;
+    };
+    return GetCostDiff(GetScalarCost, GetVectorCost);
+  }
   case Instruction::FNeg:
   case Instruction::Add:
   case Instruction::FAdd:
@@ -13964,8 +14741,16 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       }
       TTI::OperandValueInfo Op1Info = TTI::getOperandInfo(Op1);
       TTI::OperandValueInfo Op2Info = TTI::getOperandInfo(Op2);
-      return TTI->getArithmeticInstrCost(ShuffleOrOp, OrigScalarTy, CostKind,
-                                         Op1Info, Op2Info, Operands);
+      InstructionCost ScalarCost = TTI->getArithmeticInstrCost(
+          ShuffleOrOp, OrigScalarTy, CostKind, Op1Info, Op2Info, Operands);
+      if (auto *I = dyn_cast<Instruction>(UniqueValues[Idx]);
+          I && (ShuffleOrOp == Instruction::FAdd ||
+                ShuffleOrOp == Instruction::FSub)) {
+        InstructionCost IntrinsicCost = GetFMulAddCost(E->getOperations(), I);
+        if (IntrinsicCost.isValid())
+          ScalarCost = IntrinsicCost;
+      }
+      return ScalarCost;
     };
     auto GetVectorCost = [=](InstructionCost CommonCost) {
       if (ShuffleOrOp == Instruction::And && It != MinBWs.end()) {
@@ -14552,6 +15337,31 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
          })))))
     return true;
 
+  // If the tree contains only buildvector, 2 non-buildvectors (with root user
+  // tree node) and other buildvectors, we can skip it.
+  if (!ForReduction && SLPCostThreshold.getNumOccurrences() &&
+      VectorizableTree.front()->State == TreeEntry::SplitVectorize &&
+      VectorizableTree.size() >= Limit &&
+      count_if(ArrayRef(VectorizableTree).drop_front(),
+               [&](const std::unique_ptr<TreeEntry> &TE) {
+                 return !TE->isGather() && TE->UserTreeIndex.UserTE &&
+                        TE->UserTreeIndex.UserTE->Idx == 0;
+               }) == 2)
+    return true;
+
+  // If the tree contains only vectorization of the phi node from the
+  // buildvector - skip it.
+  if (!ForReduction && SLPCostThreshold.getNumOccurrences() &&
+      VectorizableTree.size() > 2 &&
+      VectorizableTree.front()->State == TreeEntry::Vectorize &&
+      VectorizableTree.front()->getOpcode() == Instruction::InsertElement &&
+      VectorizableTree[1]->State == TreeEntry::Vectorize &&
+      VectorizableTree[1]->getOpcode() == Instruction::PHI &&
+      all_of(
+          ArrayRef(VectorizableTree).drop_front(2),
+          [&](const std::unique_ptr<TreeEntry> &TE) { return TE->isGather(); }))
+    return true;
+
   // We can vectorize the tree if its size is greater than or equal to the
   // minimum size specified by the MinTreeSize command line option.
   if (VectorizableTree.size() >= MinTreeSize)
@@ -15097,7 +15907,8 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals,
   for (ExternalUser &EU : ExternalUses) {
     LLVM_DEBUG(dbgs() << "SLP: Computing cost for external use of TreeEntry "
                       << EU.E.Idx << " in lane " << EU.Lane << "\n");
-    LLVM_DEBUG(dbgs() << "  User:" << *EU.User << "\n");
+    LLVM_DEBUG(if (EU.User) dbgs() << "  User:" << *EU.User << "\n";
+               else dbgs() << "  User: nullptr\n");
     LLVM_DEBUG(dbgs() << "  Use: " << EU.Scalar->getNameOrAsOperand() << "\n");
 
     // Uses by ephemeral values are free (because the ephemeral value will be
@@ -19098,7 +19909,7 @@ Value *BoUpSLP::vectorizeTree(
   EntryToLastInstruction.clear();
   // All blocks must be scheduled before any instructions are inserted.
   for (auto &BSIter : BlocksSchedules)
-    scheduleBlock(BSIter.second.get());
+    scheduleBlock(*this, BSIter.second.get());
   // Cache last instructions for the nodes to avoid side effects, which may
   // appear during vectorization, like extra uses, etc.
   for (const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
@@ -19905,24 +20716,29 @@ void BoUpSLP::optimizeGatherSequence() {
   GatherShuffleExtractSeq.clear();
 }
 
-BoUpSLP::ScheduleBundle &
-BoUpSLP::BlockScheduling::buildBundle(ArrayRef<Value *> VL,
-                                      const InstructionsState &S) {
+BoUpSLP::ScheduleBundle &BoUpSLP::BlockScheduling::buildBundle(
+    ArrayRef<Value *> VL, const InstructionsState &S, const EdgeInfo &EI) {
   auto &BundlePtr =
       ScheduledBundlesList.emplace_back(std::make_unique<ScheduleBundle>());
   for (Value *V : VL) {
-    if (doesNotNeedToBeScheduled(V))
+    if (S.isNonSchedulable(V))
       continue;
-    if (S.isCopyableElement(V))
+    auto *I = cast<Instruction>(V);
+    if (S.isCopyableElement(V)) {
+      // Add a copyable element model.
+      ScheduleCopyableData &SD =
+          addScheduleCopyableData(EI, I, SchedulingRegionID, *BundlePtr);
+      // Group the instructions to a bundle.
+      BundlePtr->add(&SD);
       continue;
+    }
     ScheduleData *BundleMember = getScheduleData(V);
     assert(BundleMember && "no ScheduleData for bundle member "
                            "(maybe not in same basic block)");
     // Group the instructions to a bundle.
     BundlePtr->add(BundleMember);
-    ScheduledBundles.try_emplace(cast<Instruction>(V))
-        .first->getSecond()
-        .push_back(BundlePtr.get());
+    ScheduledBundles.try_emplace(I).first->getSecond().push_back(
+        BundlePtr.get());
   }
   assert(BundlePtr && *BundlePtr && "Failed to find schedule bundle");
   return *BundlePtr;
@@ -19932,7 +20748,8 @@ BoUpSLP::BlockScheduling::buildBundle(ArrayRef<Value *> VL,
 // and schedules instructions until the bundle gets ready.
 std::optional<BoUpSLP::ScheduleBundle *>
 BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
-                                            const InstructionsState &S) {
+                                            const InstructionsState &S,
+                                            const EdgeInfo &EI) {
   // No need to schedule PHIs, insertelement, extractelement and extractvalue
   // instructions.
   bool HasCopyables = S.areInstructionsWithCopyableElements();
@@ -19942,33 +20759,83 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
       all_of(VL, [&](Value *V) { return S.isNonSchedulable(V); }))
     return nullptr;
 
-  // TODO Remove once full support for copyables is landed.
-  assert(all_of(VL,
-                [&](Value *V) {
-                  return !S.isCopyableElement(V) || S.isNonSchedulable(V);
-                }) &&
-         "Copyable elements should not be schedulable");
   // Initialize the instruction bundle.
   Instruction *OldScheduleEnd = ScheduleEnd;
   LLVM_DEBUG(dbgs() << "SLP:  bundle: " << *S.getMainOp() << "\n");
 
   auto TryScheduleBundleImpl = [=](bool ReSchedule, ScheduleBundle &Bundle) {
+    // Clear deps or recalculate the region, if the memory instruction is a
+    // copyable. It may have memory deps, which must be recalculated.
+    SmallVector<ScheduleData *> ControlDependentMembers;
+    auto CheckIfNeedToClearDeps = [&](ScheduleBundle &Bundle) {
+      SmallDenseMap<std::pair<Instruction *, Value *>, unsigned> UserOpToNumOps;
+      for (ScheduleEntity *SE : Bundle.getBundle()) {
+        if (ScheduleCopyableData *SD = dyn_cast<ScheduleCopyableData>(SE)) {
+          if (ScheduleData *BundleMember = getScheduleData(SD->getInst());
+              BundleMember && BundleMember->hasValidDependencies()) {
+            BundleMember->clearDirectDependencies();
+            if (RegionHasStackSave ||
+                !isGuaranteedToTransferExecutionToSuccessor(
+                    BundleMember->getInst()))
+              ControlDependentMembers.push_back(BundleMember);
+          }
+          continue;
+        }
+        auto *SD = cast<ScheduleData>(SE);
+        for (const Use &U : SD->getInst()->operands()) {
+          unsigned &NumOps =
+              UserOpToNumOps
+                  .try_emplace(std::make_pair(SD->getInst(), U.get()), 0)
+                  .first->getSecond();
+          ++NumOps;
+          if (auto *Op = dyn_cast<Instruction>(U.get());
+              Op && areAllOperandsReplacedByCopyableData(SD->getInst(), Op,
+                                                         *SLP, NumOps)) {
+            if (ScheduleData *OpSD = getScheduleData(Op)) {
+              OpSD->clearDirectDependencies();
+              if (RegionHasStackSave ||
+                  !isGuaranteedToTransferExecutionToSuccessor(OpSD->getInst()))
+                ControlDependentMembers.push_back(OpSD);
+            }
+          }
+        }
+      }
+    };
     // The scheduling region got new instructions at the lower end (or it is a
     // new region for the first bundle). This makes it necessary to
     // recalculate all dependencies.
     // It is seldom that this needs to be done a second time after adding the
     // initial bundle to the region.
     if (OldScheduleEnd && ScheduleEnd != OldScheduleEnd) {
-      for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
-        if (ScheduleData *SD = getScheduleData(I))
+      for_each(ScheduleDataMap, [&](auto &P) {
+        if (BB != P.first->getParent())
+          return;
+        ScheduleData *SD = P.second;
+        if (isInSchedulingRegion(*SD))
           SD->clearDependencies();
-      }
+      });
+      for_each(ScheduleCopyableDataMapByInst, [&](auto &P) {
+        for_each(P.second, [&](ScheduleCopyableData *SD) {
+          if (isInSchedulingRegion(*SD))
+            SD->clearDependencies();
+        });
+      });
       ReSchedule = true;
     }
+    // Check if the bundle data has deps for copyable elements already. In
+    // this case need to reset deps and recalculate it.
     if (Bundle && !Bundle.getBundle().empty()) {
+      if (S.areInstructionsWithCopyableElements() ||
+          !ScheduleCopyableDataMap.empty())
+        CheckIfNeedToClearDeps(Bundle);
       LLVM_DEBUG(dbgs() << "SLP: try schedule bundle " << Bundle << " in block "
                         << BB->getName() << "\n");
-      calculateDependencies(Bundle, /*InsertInReadyList=*/!ReSchedule, SLP);
+      calculateDependencies(Bundle, /*InsertInReadyList=*/!ReSchedule, SLP,
+                            ControlDependentMembers);
+    } else if (!ControlDependentMembers.empty()) {
+      ScheduleBundle Invalid = ScheduleBundle::invalid();
+      calculateDependencies(Invalid, /*InsertInReadyList=*/!ReSchedule, SLP,
+                            ControlDependentMembers);
     }
 
     if (ReSchedule) {
@@ -19984,7 +20851,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
            !ReadyInsts.empty()) {
       ScheduleEntity *Picked = ReadyInsts.pop_back_val();
       assert(Picked->isReady() && "must be ready to schedule");
-      schedule(Picked, ReadyInsts);
+      schedule(*SLP, S, EI, Picked, ReadyInsts);
       if (Picked == &Bundle)
         break;
     }
@@ -19993,7 +20860,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
   // Make sure that the scheduling region contains all
   // instructions of the bundle.
   for (Value *V : VL) {
-    if (doesNotNeedToBeScheduled(V) || S.isCopyableElement(V))
+    if (S.isNonSchedulable(V))
       continue;
     if (!extendSchedulingRegion(V, S)) {
       // If the scheduling region got new instructions at the lower end (or it
@@ -20010,11 +20877,19 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
 
   bool ReSchedule = false;
   for (Value *V : VL) {
-    if (doesNotNeedToBeScheduled(V) || S.isCopyableElement(V))
+    if (S.isNonSchedulable(V))
       continue;
+    SmallVector<ScheduleCopyableData *> CopyableData =
+        getScheduleCopyableData(cast<Instruction>(V));
+    if (!CopyableData.empty()) {
+      for (ScheduleCopyableData *SD : CopyableData)
+        ReadyInsts.remove(SD);
+    }
     ScheduleData *BundleMember = getScheduleData(V);
-    assert(BundleMember &&
+    assert((BundleMember || S.isCopyableElement(V)) &&
            "no ScheduleData for bundle member (maybe not in same basic block)");
+    if (!BundleMember)
+      continue;
 
     // Make sure we don't leave the pieces of the bundle in the ready list when
     // whole bundle might not be ready.
@@ -20025,20 +20900,25 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
         ReadyInsts.remove(B);
     }
 
-    if (!BundleMember->isScheduled())
+    if (!S.isCopyableElement(V) && !BundleMember->isScheduled())
       continue;
     // A bundle member was scheduled as single instruction before and now
     // needs to be scheduled as part of the bundle. We just get rid of the
     // existing schedule.
+    // A bundle member has deps calculated before it was copyable element - need
+    // to reschedule.
     LLVM_DEBUG(dbgs() << "SLP:  reset schedule because " << *BundleMember
                       << " was already scheduled\n");
     ReSchedule = true;
   }
 
-  ScheduleBundle &Bundle = buildBundle(VL, S);
+  ScheduleBundle &Bundle = buildBundle(VL, S, EI);
   TryScheduleBundleImpl(ReSchedule, Bundle);
   if (!Bundle.isReady()) {
-    for (ScheduleData *BD : Bundle.getBundle()) {
+    for (ScheduleEntity *BD : Bundle.getBundle()) {
+      // Copyable data scheduling is just removed.
+      if (isa<ScheduleCopyableData>(BD))
+        continue;
       if (BD->isReady()) {
         ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(BD->getInst());
         if (Bundles.empty()) {
@@ -20051,10 +20931,66 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
       }
     }
     ScheduledBundlesList.pop_back();
+    SmallVector<ScheduleData *> ControlDependentMembers;
+    SmallPtrSet<Instruction *, 4> Visited;
     for (Value *V : VL) {
-      if (doesNotNeedToBeScheduled(V) || S.isCopyableElement(V))
+      if (S.isNonSchedulable(V))
         continue;
-      ScheduledBundles.find(cast<Instruction>(V))->getSecond().pop_back();
+      auto *I = cast<Instruction>(V);
+      if (S.isCopyableElement(I)) {
+        // Remove the copyable data from the scheduling region and restore
+        // previous mappings.
+        auto KV = std::make_pair(EI, I);
+        assert(ScheduleCopyableDataMap.contains(KV) &&
+               "no ScheduleCopyableData for copyable element");
+        ScheduleCopyableData *SD =
+            ScheduleCopyableDataMapByInst.find(I)->getSecond().pop_back_val();
+        ScheduleCopyableDataMapByUsers[I].remove(SD);
+        if (EI.UserTE) {
+          ArrayRef<Value *> Op = EI.UserTE->getOperand(EI.EdgeIdx);
+          const auto *It = find(Op, I);
+          assert(It != Op.end() && "Lane not set");
+          SmallPtrSet<Instruction *, 4> Visited;
+          do {
+            int Lane = std::distance(Op.begin(), It);
+            assert(Lane >= 0 && "Lane not set");
+            if (isa<StoreInst>(EI.UserTE->Scalars[Lane]) &&
+                !EI.UserTE->ReorderIndices.empty())
+              Lane = EI.UserTE->ReorderIndices[Lane];
+            assert(Lane < static_cast<int>(EI.UserTE->Scalars.size()) &&
+                   "Couldn't find extract lane");
+            auto *In = cast<Instruction>(EI.UserTE->Scalars[Lane]);
+            if (!Visited.insert(In).second) {
+              It = find(make_range(std::next(It), Op.end()), I);
+              break;
+            }
+            ScheduleCopyableDataMapByInstUser
+                [std::make_pair(std::make_pair(In, EI.EdgeIdx), I)]
+                    .pop_back();
+            It = find(make_range(std::next(It), Op.end()), I);
+          } while (It != Op.end());
+          EdgeInfo UserEI = EI.UserTE->UserTreeIndex;
+          if (ScheduleCopyableData *UserCD = getScheduleCopyableData(UserEI, I))
+            ScheduleCopyableDataMapByUsers[I].insert(UserCD);
+        }
+        if (ScheduleCopyableDataMapByUsers[I].empty())
+          ScheduleCopyableDataMapByUsers.erase(I);
+        ScheduleCopyableDataMap.erase(KV);
+        // Need to recalculate dependencies for the actual schedule data.
+        if (ScheduleData *OpSD = getScheduleData(I)) {
+          OpSD->clearDirectDependencies();
+          if (RegionHasStackSave ||
+              !isGuaranteedToTransferExecutionToSuccessor(OpSD->getInst()))
+            ControlDependentMembers.push_back(OpSD);
+        }
+        continue;
+      }
+      ScheduledBundles.find(I)->getSecond().pop_back();
+    }
+    if (!ControlDependentMembers.empty()) {
+      ScheduleBundle Invalid = ScheduleBundle::invalid();
+      calculateDependencies(Invalid, /*InsertInReadyList=*/false, SLP,
+                            ControlDependentMembers);
     }
     return std::nullopt;
   }
@@ -20074,10 +21010,6 @@ bool BoUpSLP::BlockScheduling::extendSchedulingRegion(
     Value *V, const InstructionsState &S) {
   Instruction *I = dyn_cast<Instruction>(V);
   assert(I && "bundle member must be an instruction");
-  assert(!isa<PHINode>(I) && !isVectorLikeInstWithConstOps(I) &&
-         !doesNotNeedToBeScheduled(I) &&
-         "phi nodes/insertelements/extractelements/extractvalues don't need to "
-         "be scheduled");
   if (getScheduleData(I))
     return true;
   if (!ScheduleStart) {
@@ -20147,14 +21079,14 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
   ScheduleData *CurrentLoadStore = PrevLoadStore;
   for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) {
     // No need to allocate data for non-schedulable instructions.
-    if (doesNotNeedToBeScheduled(I))
+    if (isa<PHINode>(I))
       continue;
     ScheduleData *SD = ScheduleDataMap.lookup(I);
     if (!SD) {
       SD = allocateScheduleDataChunks();
       ScheduleDataMap[I] = SD;
     }
-    assert(!isInSchedulingRegion(SD) &&
+    assert(!isInSchedulingRegion(*SD) &&
            "new ScheduleData already in scheduling region");
     SD->init(SchedulingRegionID, I);
 
@@ -20184,34 +21116,128 @@ void BoUpSLP::BlockScheduling::initScheduleData(Instruction *FromI,
   }
 }
 
-void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleBundle &Bundle,
-                                                     bool InsertInReadyList,
-                                                     BoUpSLP *SLP) {
-  SmallVector<ScheduleData *> WorkList;
-  auto ProcessNode = [&](ScheduleData *BundleMember) {
+void BoUpSLP::BlockScheduling::calculateDependencies(
+    ScheduleBundle &Bundle, bool InsertInReadyList, BoUpSLP *SLP,
+    ArrayRef<ScheduleData *> ControlDeps) {
+  SmallVector<ScheduleEntity *> WorkList;
+  auto ProcessNode = [&](ScheduleEntity *SE) {
+    if (auto *CD = dyn_cast<ScheduleCopyableData>(SE)) {
+      if (CD->hasValidDependencies())
+        return;
+      LLVM_DEBUG(dbgs() << "SLP:       update deps of " << *CD << "\n");
+      CD->initDependencies();
+      CD->resetUnscheduledDeps();
+      const EdgeInfo &EI = CD->getEdgeInfo();
+      if (EI.UserTE) {
+        ArrayRef<Value *> Op = EI.UserTE->getOperand(EI.EdgeIdx);
+        const auto *It = find(Op, CD->getInst());
+        assert(It != Op.end() && "Lane not set");
+        SmallPtrSet<Instruction *, 4> Visited;
+        do {
+          int Lane = std::distance(Op.begin(), It);
+          assert(Lane >= 0 && "Lane not set");
+          if (isa<StoreInst>(EI.UserTE->Scalars[Lane]) &&
+              !EI.UserTE->ReorderIndices.empty())
+            Lane = EI.UserTE->ReorderIndices[Lane];
+          assert(Lane < static_cast<int>(EI.UserTE->Scalars.size()) &&
+                 "Couldn't find extract lane");
+          auto *In = cast<Instruction>(EI.UserTE->Scalars[Lane]);
+          if (EI.UserTE->isCopyableElement(In)) {
+            // We may have not have related copyable scheduling data, if the
+            // instruction is non-schedulable.
+            if (ScheduleCopyableData *UseSD =
+                    getScheduleCopyableData(EI.UserTE->UserTreeIndex, In)) {
+              CD->incDependencies();
+              if (!UseSD->isScheduled())
+                CD->incrementUnscheduledDeps(1);
+              if (!UseSD->hasValidDependencies() ||
+                  (InsertInReadyList && UseSD->isReady()))
+                WorkList.push_back(UseSD);
+            }
+          } else if (Visited.insert(In).second) {
+            if (ScheduleData *UseSD = getScheduleData(In)) {
+              CD->incDependencies();
+              if (!UseSD->isScheduled())
+                CD->incrementUnscheduledDeps(1);
+              if (!UseSD->hasValidDependencies() ||
+                  (InsertInReadyList && UseSD->isReady()))
+                WorkList.push_back(UseSD);
+            }
+          }
+          It = find(make_range(std::next(It), Op.end()), CD->getInst());
+        } while (It != Op.end());
+        if (CD->isReady() && CD->getDependencies() == 0 &&
+            (EI.UserTE->hasState() &&
+             (EI.UserTE->getMainOp()->getParent() !=
+                  CD->getInst()->getParent() ||
+              (isa<PHINode>(EI.UserTE->getMainOp()) &&
+               (EI.UserTE->getMainOp()->hasNUsesOrMore(UsesLimit) ||
+                any_of(EI.UserTE->getMainOp()->users(), [&](User *U) {
+                  auto *IU = dyn_cast<Instruction>(U);
+                  if (!IU)
+                    return true;
+                  return IU->getParent() == EI.UserTE->getMainOp()->getParent();
+                })))))) {
+          // If no uses in the block - mark as having pseudo-use, which cannot
+          // be scheduled.
+          // Prevents incorrect def-use tracking between external user and
+          // actual instruction.
+          CD->incDependencies();
+          CD->incrementUnscheduledDeps(1);
+        }
+      }
+      return;
+    }
+    auto *BundleMember = cast<ScheduleData>(SE);
     if (BundleMember->hasValidDependencies())
       return;
     LLVM_DEBUG(dbgs() << "SLP:       update deps of " << *BundleMember << "\n");
     BundleMember->initDependencies();
     BundleMember->resetUnscheduledDeps();
     // Handle def-use chain dependencies.
+    SmallDenseMap<Value *, unsigned> UserToNumOps;
     for (User *U : BundleMember->getInst()->users()) {
+      if (isa<PHINode>(U))
+        continue;
       if (ScheduleData *UseSD = getScheduleData(U)) {
+        // The operand is a copyable element - skip.
+        unsigned &NumOps = UserToNumOps.try_emplace(U, 0).first->getSecond();
+        ++NumOps;
+        if (areAllOperandsReplacedByCopyableData(
+                cast<Instruction>(U), BundleMember->getInst(), *SLP, NumOps))
+          continue;
         BundleMember->incDependencies();
         if (!UseSD->isScheduled())
           BundleMember->incrementUnscheduledDeps(1);
-        WorkList.push_back(UseSD);
+        if (!UseSD->hasValidDependencies() ||
+            (InsertInReadyList && UseSD->isReady()))
+          WorkList.push_back(UseSD);
       }
     }
+    for (ScheduleCopyableData *UseSD :
+         getScheduleCopyableDataUsers(BundleMember->getInst())) {
+      BundleMember->incDependencies();
+      if (!UseSD->isScheduled())
+        BundleMember->incrementUnscheduledDeps(1);
+      if (!UseSD->hasValidDependencies() ||
+          (InsertInReadyList && UseSD->isReady()))
+        WorkList.push_back(UseSD);
+    }
 
+    SmallPtrSet<const Instruction *, 4> Visited;
     auto MakeControlDependent = [&](Instruction *I) {
+      // Do not mark control dependent twice.
+      if (!Visited.insert(I).second)
+        return;
       auto *DepDest = getScheduleData(I);
       assert(DepDest && "must be in schedule window");
       DepDest->addControlDependency(BundleMember);
       BundleMember->incDependencies();
       if (!DepDest->isScheduled())
         BundleMember->incrementUnscheduledDeps(1);
-      WorkList.push_back(DepDest);
+      if (!DepDest->hasValidDependencies() ||
+          (InsertInReadyList && DepDest->isReady()))
+        WorkList.push_back(DepDest);
     };
 
     // Any instruction which isn't safe to speculate at the beginning of the
@@ -20290,7 +21316,7 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleBundle &Bundle,
 
     for (ScheduleData *DepDest = NextLoadStore; DepDest;
          DepDest = DepDest->getNextLoadStore()) {
-      assert(isInSchedulingRegion(DepDest) && "Expected to be in region");
+      assert(isInSchedulingRegion(*DepDest) && "Expected to be in region");
 
       // We have two limits to reduce the complexity:
       // 1) AliasedCheckLimit: It's a small limit to reduce calls to
@@ -20313,7 +21339,9 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleBundle &Bundle,
         BundleMember->incDependencies();
         if (!DepDest->isScheduled())
           BundleMember->incrementUnscheduledDeps(1);
-        WorkList.push_back(DepDest);
+        if (!DepDest->hasValidDependencies() ||
+            (InsertInReadyList && DepDest->isReady()))
+          WorkList.push_back(DepDest);
       }
 
       // Example, explaining the loop break condition: Let's assume our
@@ -20335,13 +21363,25 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleBundle &Bundle,
     }
   };
 
-  WorkList.push_back(Bundle.getBundle().front());
+  assert((Bundle || !ControlDeps.empty()) &&
+         "expected at least one instruction to schedule");
+  if (Bundle)
+    WorkList.push_back(Bundle.getBundle().front());
+  WorkList.append(ControlDeps.begin(), ControlDeps.end());
   SmallPtrSet<ScheduleBundle *, 16> Visited;
   while (!WorkList.empty()) {
-    ScheduleData *SD = WorkList.pop_back_val();
-    ArrayRef<ScheduleBundle *> Bundles = getScheduleBundles(SD->getInst());
+    ScheduleEntity *SD = WorkList.pop_back_val();
+    SmallVector<ScheduleBundle *, 1> CopyableBundle;
+    ArrayRef<ScheduleBundle *> Bundles;
+    if (auto *CD = dyn_cast<ScheduleCopyableData>(SD)) {
+      CopyableBundle.push_back(&CD->getBundle());
+      Bundles = CopyableBundle;
+    } else {
+      Bundles = getScheduleBundles(SD->getInst());
+    }
     if (Bundles.empty()) {
-      ProcessNode(SD);
+      if (!SD->hasValidDependencies())
+        ProcessNode(SD);
       if (InsertInReadyList && SD->isReady()) {
         ReadyInsts.insert(SD);
         LLVM_DEBUG(dbgs() << "SLP:     gets ready on update: " << *SD << "\n");
@@ -20349,7 +21389,7 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleBundle &Bundle,
       continue;
     }
     for (ScheduleBundle *Bundle : Bundles) {
-      if (!Visited.insert(Bundle).second || Bundle->hasValidDependencies())
+      if (Bundle->hasValidDependencies() || !Visited.insert(Bundle).second)
         continue;
       assert(isInSchedulingRegion(*Bundle) &&
              "ScheduleData not in scheduling region");
@@ -20372,23 +21412,40 @@ void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleBundle &Bundle,
 void BoUpSLP::BlockScheduling::resetSchedule() {
   assert(ScheduleStart &&
          "tried to reset schedule on block which has not been scheduled");
-  for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
-    if (ScheduleData *SD = getScheduleData(I)) {
-      assert(isInSchedulingRegion(SD) &&
-             "ScheduleData not in scheduling region");
+  for_each(ScheduleDataMap, [&](auto &P) {
+    if (BB != P.first->getParent())
+      return;
+    ScheduleData *SD = P.second;
+    if (isInSchedulingRegion(*SD)) {
       SD->setScheduled(/*Scheduled=*/false);
       SD->resetUnscheduledDeps();
     }
-    for (ScheduleBundle *Bundle : getScheduleBundles(I)) {
-      assert(isInSchedulingRegion(*Bundle) &&
-             "ScheduleBundle not in scheduling region");
-      Bundle->setScheduled(/*Scheduled=*/false);
+  });
+  for_each(ScheduleCopyableDataMapByInst, [&](auto &P) {
+    for_each(P.second, [&](ScheduleCopyableData *SD) {
+      if (isInSchedulingRegion(*SD)) {
+        SD->setScheduled(/*Scheduled=*/false);
+        SD->resetUnscheduledDeps();
+      }
+    });
+  });
+  for_each(ScheduledBundles, [&](auto &P) {
+    for_each(P.second, [&](ScheduleBundle *Bundle) {
+      if (isInSchedulingRegion(*Bundle))
+        Bundle->setScheduled(/*Scheduled=*/false);
+    });
+  });
+  // Reset schedule data for copyable elements.
+  for (auto &P : ScheduleCopyableDataMap) {
+    if (isInSchedulingRegion(*P.second)) {
+      P.second->setScheduled(/*Scheduled=*/false);
+      P.second->resetUnscheduledDeps();
     }
   }
   ReadyInsts.clear();
 }
 
-void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
+void BoUpSLP::scheduleBlock(const BoUpSLP &R, BlockScheduling *BS) {
   if (!BS->ScheduleStart)
     return;
 
@@ -20426,15 +21483,45 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
         if (!Bundle->hasValidDependencies())
           BS->calculateDependencies(*Bundle, /*InsertInReadyList=*/false, this);
       }
+      SmallVector<ScheduleCopyableData *> SDs = BS->getScheduleCopyableData(I);
+      for (ScheduleCopyableData *SD : reverse(SDs)) {
+        ScheduleBundle &Bundle = SD->getBundle();
+        Bundle.setSchedulingPriority(Idx++);
+        if (!Bundle.hasValidDependencies())
+          BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this);
+      }
       continue;
     }
+    SmallVector<ScheduleCopyableData *> CopyableData =
+        BS->getScheduleCopyableDataUsers(I);
     if (ScheduleData *SD = BS->getScheduleData(I)) {
       [[maybe_unused]] ArrayRef<TreeEntry *> SDTEs = getTreeEntries(I);
       assert((isVectorLikeInstWithConstOps(SD->getInst()) || SDTEs.empty() ||
-              SDTEs.front()->doesNotNeedToSchedule()) &&
+              SDTEs.front()->doesNotNeedToSchedule() ||
+              doesNotNeedToBeScheduled(I)) &&
              "scheduler and vectorizer bundle mismatch");
       SD->setSchedulingPriority(Idx++);
-      continue;
+      if (!SD->hasValidDependencies() &&
+          (!CopyableData.empty() ||
+           any_of(R.ValueToGatherNodes.lookup(I), [&](const TreeEntry *TE) {
+             assert(TE->isGather() && "expected gather node");
+             return TE->hasState() && TE->hasCopyableElements() &&
+                    TE->isCopyableElement(I);
+           }))) {
+        // Need to calculate deps for these nodes to correctly handle copyable
+        // dependencies, even if they were cancelled.
+        // If copyables bundle was cancelled, the deps are cleared and need to
+        // recalculate them.
+        ScheduleBundle Bundle;
+        Bundle.add(SD);
+        BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this);
+      }
+    }
+    for (ScheduleCopyableData *SD : reverse(CopyableData)) {
+      ScheduleBundle &Bundle = SD->getBundle();
+      Bundle.setSchedulingPriority(Idx++);
+      if (!Bundle.hasValidDependencies())
+        BS->calculateDependencies(Bundle, /*InsertInReadyList=*/false, this);
     }
   }
   BS->initialFillReadyList(ReadyInsts);
@@ -20450,9 +21537,12 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
     // Move the scheduled instruction(s) to their dedicated places, if not
     // there yet.
     if (auto *Bundle = dyn_cast<ScheduleBundle>(Picked)) {
-      for (const ScheduleData *BundleMember : Bundle->getBundle()) {
+      for (const ScheduleEntity *BundleMember : Bundle->getBundle()) {
         Instruction *PickedInst = BundleMember->getInst();
-        if (!Scheduled.insert(PickedInst).second)
+        // If copyable must be schedule as part of something else, skip it.
+        bool IsCopyable = Bundle->getTreeEntry()->isCopyableElement(PickedInst);
+        if ((IsCopyable && BS->getScheduleData(PickedInst)) ||
+            (!IsCopyable && !Scheduled.insert(PickedInst).second))
           continue;
         if (PickedInst->getNextNode() != LastScheduledInst)
           PickedInst->moveAfter(LastScheduledInst->getPrevNode());
@@ -20467,7 +21557,8 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
         PickedInst->moveAfter(LastScheduledInst->getPrevNode());
       LastScheduledInst = PickedInst;
     }
-    BS->schedule(Picked, ReadyInsts);
+    auto Invalid = InstructionsState::invalid();
+    BS->schedule(R, Invalid, EdgeInfo(), Picked, ReadyInsts);
   }
 
   // Check that we didn't break any of our invariants.
@@ -20829,9 +21920,11 @@ bool BoUpSLP::collectValuesToDemote(
       return all_of(E.Scalars, [&](Value *V) {
         if (isa<PoisonValue>(V))
           return true;
+        APInt ShiftedBits = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
+        if (E.isCopyableElement(V))
+          return MaskedValueIsZero(V, ShiftedBits, SimplifyQuery(*DL));
         auto *I = cast<Instruction>(V);
         KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL);
-        APInt ShiftedBits = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
         return AmtKnownBits.getMaxValue().ult(BitWidth) &&
                MaskedValueIsZero(I->getOperand(0), ShiftedBits,
                                  SimplifyQuery(*DL));
@@ -23384,7 +24477,7 @@ class HorizontalReduction {
       // correct, replace internal uses with undef, and mark for eventual
       // deletion.
 #ifndef NDEBUG
-      SmallSet<Value *, 4> IgnoreSet;
+      SmallPtrSet<Value *, 4> IgnoreSet;
       for (ArrayRef<Value *> RdxOps : ReductionOps)
         IgnoreSet.insert_range(RdxOps);
 #endif
@@ -23697,6 +24790,8 @@ class HorizontalReduction {
         case RecurKind::FMinimum:
           // res = vv
           break;
+        case RecurKind::Sub:
+        case RecurKind::AddChainWithSubs:
         case RecurKind::Mul:
         case RecurKind::FMul:
         case RecurKind::FMulAdd:
@@ -23836,6 +24931,8 @@ class HorizontalReduction {
     case RecurKind::FMinimum:
       // res = vv
       return VectorizedValue;
+    case RecurKind::Sub:
+    case RecurKind::AddChainWithSubs:
     case RecurKind::Mul:
     case RecurKind::FMul:
     case RecurKind::FMulAdd:
@@ -23940,6 +25037,8 @@ class HorizontalReduction {
       auto *Scale = ConstantVector::get(Vals);
       return Builder.CreateFMul(VectorizedValue, Scale);
     }
+    case RecurKind::Sub:
+    case RecurKind::AddChainWithSubs:
     case RecurKind::Mul:
     case RecurKind::FMul:
     case RecurKind::FMulAdd:
@@ -24204,7 +25303,7 @@ bool SLPVectorizerPass::vectorizeHorReduction(
   Stack.emplace(SelectRoot(), 0);
   SmallPtrSet<Value *, 8> VisitedInstrs;
   bool Res = false;
-  auto &&TryToReduce = [this, &R](Instruction *Inst) -> Value * {
+  auto TryToReduce = [this, &R, TTI = TTI](Instruction *Inst) -> Value * {
     if (R.isAnalyzedReductionRoot(Inst))
       return nullptr;
     if (!isReductionCandidate(Inst))
@@ -24276,6 +25375,12 @@ bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
 
   if (!isa<BinaryOperator, CmpInst>(I) || isa<VectorType>(I->getType()))
     return false;
+  // Skip potential FMA candidates.
+  if ((I->getOpcode() == Instruction::FAdd ||
+       I->getOpcode() == Instruction::FSub) &&
+      canConvertToFMA(I, getSameOpcode(I, *TLI), *DT, *DL, *TTI, *TLI)
+          .isValid())
+    return false;
 
   Value *P = I->getParent();
 
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
index f32d57fa67daa..e414c120b2a8b 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/Legality.cpp
@@ -81,6 +81,7 @@ LegalityAnalysis::notVectorizableBasedOnOpcodesAndTypes(
   case Instruction::Opcode::FPToUI:
   case Instruction::Opcode::FPToSI:
   case Instruction::Opcode::FPExt:
+  case Instruction::Opcode::PtrToAddr:
   case Instruction::Opcode::PtrToInt:
   case Instruction::Opcode::IntToPtr:
   case Instruction::Opcode::SIToFP:
diff --git a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
index 1934326866cdf..e80dc04e15326 100644
--- a/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
+++ b/llvm/lib/Transforms/Vectorize/SandboxVectorizer/SeedCollector.cpp
@@ -73,22 +73,28 @@ ArrayRef<Instruction *> SeedBundle::getSlice(unsigned StartIdx,
 }
 
 template <typename LoadOrStoreT>
-SeedContainer::KeyT SeedContainer::getKey(LoadOrStoreT *LSI) const {
+SeedContainer::KeyT SeedContainer::getKey(LoadOrStoreT *LSI,
+                                          bool AllowDiffTypes) const {
   assert((isa<LoadInst>(LSI) || isa<StoreInst>(LSI)) &&
          "Expected Load or Store!");
   Value *Ptr = Utils::getMemInstructionBase(LSI);
   Instruction::Opcode Op = LSI->getOpcode();
-  Type *Ty = Utils::getExpectedType(LSI);
-  if (auto *VTy = dyn_cast<VectorType>(Ty))
-    Ty = VTy->getElementType();
+  Type *Ty;
+  if (AllowDiffTypes) {
+    Ty = nullptr;
+  } else {
+    Ty = Utils::getExpectedType(LSI);
+    if (auto *VTy = dyn_cast<VectorType>(Ty))
+      Ty = VTy->getElementType();
+  }
   return {Ptr, Ty, Op};
 }
 
 // Explicit instantiations
 template SeedContainer::KeyT
-SeedContainer::getKey<LoadInst>(LoadInst *LSI) const;
+SeedContainer::getKey<LoadInst>(LoadInst *LSI, bool AllowDiffTypes) const;
 template SeedContainer::KeyT
-SeedContainer::getKey<StoreInst>(StoreInst *LSI) const;
+SeedContainer::getKey<StoreInst>(StoreInst *LSI, bool AllowDiffTypes) const;
 
 bool SeedContainer::erase(Instruction *I) {
   assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Expected Load or Store!");
@@ -100,9 +106,10 @@ bool SeedContainer::erase(Instruction *I) {
   return true;
 }
 
-template <typename LoadOrStoreT> void SeedContainer::insert(LoadOrStoreT *LSI) {
+template <typename LoadOrStoreT>
+void SeedContainer::insert(LoadOrStoreT *LSI, bool AllowDiffTypes) {
   // Find the bundle containing seeds for this symbol and type-of-access.
-  auto &BundleVec = Bundles[getKey(LSI)];
+  auto &BundleVec = Bundles[getKey(LSI, AllowDiffTypes)];
   // Fill this vector of bundles front to back so that only the last bundle in
   // the vector may have available space. This avoids iteration to find one with
   // space.
@@ -115,9 +122,10 @@ template <typename LoadOrStoreT> void SeedContainer::insert(LoadOrStoreT *LSI) {
 }
 
 // Explicit instantiations
-template LLVM_EXPORT_TEMPLATE void SeedContainer::insert<LoadInst>(LoadInst *);
-template LLVM_EXPORT_TEMPLATE void
-SeedContainer::insert<StoreInst>(StoreInst *);
+template LLVM_EXPORT_TEMPLATE void SeedContainer::insert<LoadInst>(LoadInst *,
+                                                                   bool);
+template LLVM_EXPORT_TEMPLATE void SeedContainer::insert<StoreInst>(StoreInst *,
+                                                                    bool);
 
 #ifndef NDEBUG
 void SeedContainer::print(raw_ostream &OS) const {
@@ -158,7 +166,8 @@ template bool isValidMemSeed<LoadInst>(LoadInst *LSI);
 template bool isValidMemSeed<StoreInst>(StoreInst *LSI);
 
 SeedCollector::SeedCollector(BasicBlock *BB, ScalarEvolution &SE,
-                             bool CollectStores, bool CollectLoads)
+                             bool CollectStores, bool CollectLoads,
+                             bool AllowDiffTypes)
     : StoreSeeds(SE), LoadSeeds(SE), Ctx(BB->getContext()) {
 
   if (!CollectStores && !CollectLoads)
@@ -175,10 +184,10 @@ SeedCollector::SeedCollector(BasicBlock *BB, ScalarEvolution &SE,
   for (auto &I : *BB) {
     if (StoreInst *SI = dyn_cast<StoreInst>(&I))
       if (CollectStores && isValidMemSeed(SI))
-        StoreSeeds.insert(SI);
+        StoreSeeds.insert(SI, AllowDiffTypes);
     if (LoadInst *LI = dyn_cast<LoadInst>(&I))
       if (CollectLoads && isValidMemSeed(LI))
-        LoadSeeds.insert(LI);
+        LoadSeeds.insert(LI, AllowDiffTypes);
     // Cap compilation time.
     if (totalNumSeedGroups() > SeedGroupsLimit)
       break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 8052e3199ec88..f972efa07eb7e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -246,8 +246,7 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
                                    IRBuilderBase &Builder, VPlan *Plan,
                                    Loop *CurrentParentLoop, Type *CanonicalIVTy)
     : TTI(TTI), VF(VF), CFG(DT), LI(LI), AC(AC), Builder(Builder), Plan(Plan),
-      CurrentParentLoop(CurrentParentLoop), TypeAnalysis(CanonicalIVTy),
-      VPDT(*Plan) {}
+      CurrentParentLoop(CurrentParentLoop), TypeAnalysis(*Plan), VPDT(*Plan) {}
 
 Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
   if (Def->isLiveIn())
@@ -296,27 +295,11 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
   if (hasVectorValue(Def))
     return Data.VPV2Vector[Def];
 
-  auto GetBroadcastInstrs = [this, Def](Value *V) {
-    bool SafeToHoist =
-        !Def->hasDefiningRecipe() ||
-        VPDT.properlyDominates(Def->getDefiningRecipe()->getParent(),
-                               Plan->getVectorPreheader());
-
+  auto GetBroadcastInstrs = [this](Value *V) {
     if (VF.isScalar())
       return V;
-    // Place the code for broadcasting invariant variables in the new preheader.
-    IRBuilder<>::InsertPointGuard Guard(Builder);
-    if (SafeToHoist) {
-      BasicBlock *LoopVectorPreHeader =
-          CFG.VPBB2IRBB[Plan->getVectorPreheader()];
-      if (LoopVectorPreHeader)
-        Builder.SetInsertPoint(LoopVectorPreHeader->getTerminator());
-    }
-
-    // Place the code for broadcasting invariant variables in the new preheader.
     // Broadcast the scalar into all locations in the vector.
     Value *Shuf = Builder.CreateVectorSplat(VF, V, "broadcast");
-
     return Shuf;
   };
 
@@ -372,6 +355,9 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
     set(Def, VectorValue);
   } else {
     assert(!VF.isScalable() && "VF is assumed to be non scalable.");
+    assert(isa<VPInstruction>(Def) &&
+           "Explicit BuildVector recipes must have"
+           "handled packing for non-VPInstructions.");
     // Initialize packing with insertelements to start from poison.
     VectorValue = PoisonValue::get(toVectorizedTy(LastInst->getType(), VF));
     for (unsigned Lane = 0; Lane < VF.getFixedValue(); ++Lane)
@@ -951,28 +937,6 @@ VPlan::~VPlan() {
     delete BackedgeTakenCount;
 }
 
-void VPlan::prepareToExecute(Value *VectorTripCountV, VPTransformState &State) {
-  if (!VectorTripCount.getUnderlyingValue())
-    VectorTripCount.setUnderlyingValue(VectorTripCountV);
-  else
-    assert(VectorTripCount.getUnderlyingValue() == VectorTripCountV &&
-           "VectorTripCount set earlier must much VectorTripCountV");
-
-  IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
-  Type *TCTy = VectorTripCountV->getType();
-  // FIXME: Model VF * UF computation completely in VPlan.
-  unsigned UF = getUF();
-  if (VF.getNumUsers()) {
-    Value *RuntimeVF = getRuntimeVF(Builder, TCTy, State.VF);
-    VF.setUnderlyingValue(RuntimeVF);
-    VFxUF.setUnderlyingValue(
-        UF > 1 ? Builder.CreateMul(RuntimeVF, ConstantInt::get(TCTy, UF))
-               : RuntimeVF);
-  } else {
-    VFxUF.setUnderlyingValue(createStepForVF(Builder, TCTy, State.VF, UF));
-  }
-}
-
 VPIRBasicBlock *VPlan::getExitBlock(BasicBlock *IRBB) const {
   auto Iter = find_if(getExitBlocks(), [IRBB](const VPIRBasicBlock *VPIRBB) {
     return VPIRBB->getIRBasicBlock() == IRBB;
@@ -1054,12 +1018,17 @@ void VPlan::execute(VPTransformState *State) {
 
 InstructionCost VPlan::cost(ElementCount VF, VPCostContext &Ctx) {
   // For now only return the cost of the vector loop region, ignoring any other
-  // blocks, like the preheader or middle blocks.
+  // blocks, like the preheader or middle blocks, expect for checking them for
+  // recipes with invalid costs.
   InstructionCost Cost = getVectorLoopRegion()->cost(VF, Ctx);
 
-  // If any instructions in the middle block are invalid return invalid.
-  // TODO: Remove once no VPlans with VF == vscale x 1 and first-order recurrences are created.
-  if (!getMiddleBlock()->cost(VF, Ctx).isValid())
+  // If the cost of the loop region is invalid or any recipe in the skeleton
+  // outside loop regions are invalid return an invalid cost.
+  if (!Cost.isValid() || any_of(VPBlockUtils::blocksOnly<VPBasicBlock>(
+                                    vp_depth_first_shallow(getEntry())),
+                                [&VF, &Ctx](VPBasicBlock *VPBB) {
+                                  return !VPBB->cost(VF, Ctx).isValid();
+                                }))
     return InstructionCost::getInvalid();
 
   return Cost;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 46dd11425aa2c..d6bc462a0dfab 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -805,6 +805,9 @@ class VPIRFlags {
 
   GEPNoWrapFlags getGEPNoWrapFlags() const { return GEPFlags; }
 
+  /// Returns true if the recipe has a comparison predicate.
+  bool hasPredicate() const { return OpType == OperationType::Cmp; }
+
   /// Returns true if the recipe has fast-math flags.
   bool hasFastMathFlags() const { return OpType == OperationType::FPMathOp; }
 
@@ -898,6 +901,11 @@ struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
   }
 
   void execute(VPTransformState &State) override = 0;
+
+  /// Compute the cost for this recipe for \p VF, using \p Opcode and \p Ctx.
+  std::optional<InstructionCost>
+  getCostForRecipeWithOpcode(unsigned Opcode, ElementCount VF,
+                             VPCostContext &Ctx) const;
 };
 
 /// Helper to access the operand that contains the unroll part for this recipe
@@ -931,6 +939,11 @@ class VPIRMetadata {
   /// Copy constructor for cloning.
   VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
 
+  VPIRMetadata &operator=(const VPIRMetadata &Other) {
+    Metadata = Other.Metadata;
+    return *this;
+  }
+
   /// Add all metadata to \p I.
   void applyMetadata(Instruction &I) const;
 
@@ -938,6 +951,10 @@ class VPIRMetadata {
   void addMetadata(unsigned Kind, MDNode *Node) {
     Metadata.emplace_back(Kind, Node);
   }
+
+  /// Intersect this VPIRMetada object with \p MD, keeping only metadata
+  /// nodes that are common to both.
+  void intersect(const VPIRMetadata &MD);
 };
 
 /// This is a concrete Recipe that models a single VPlan-level instruction.
@@ -1019,7 +1036,11 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
     /// The lane specifies an index into a vector formed by combining all vector
     /// operands (all operands after the first one).
     ExtractLane,
-
+    /// Explicit user for the resume phi of the canonical induction in the main
+    /// VPlan, used by the epilogue vector loop.
+    ResumeForEpilogue,
+    /// Returns the value for vscale.
+    VScale,
   };
 
 private:
@@ -1167,6 +1188,7 @@ class VPInstructionWithType : public VPInstruction {
     switch (VPI->getOpcode()) {
     case VPInstruction::WideIVStep:
     case VPInstruction::StepVector:
+    case VPInstruction::VScale:
       return true;
     default:
       return false;
@@ -1227,6 +1249,31 @@ class VPPhiAccessors {
     return getAsRecipe()->getNumOperands();
   }
 
+  /// Returns an interator range over the incoming values.
+  VPUser::const_operand_range incoming_values() const {
+    return make_range(getAsRecipe()->op_begin(),
+                      getAsRecipe()->op_begin() + getNumIncoming());
+  }
+
+  using const_incoming_blocks_range = iterator_range<mapped_iterator<
+      detail::index_iterator, std::function<const VPBasicBlock *(size_t)>>>;
+
+  /// Returns an iterator range over the incoming blocks.
+  const_incoming_blocks_range incoming_blocks() const {
+    std::function<const VPBasicBlock *(size_t)> GetBlock = [this](size_t Idx) {
+      return getIncomingBlock(Idx);
+    };
+    return map_range(index_range(0, getNumIncoming()), GetBlock);
+  }
+
+  /// Returns an iterator range over pairs of incoming values and corresponding
+  /// incoming blocks.
+  detail::zippy<llvm::detail::zip_first, VPUser::const_operand_range,
+                const_incoming_blocks_range>
+  incoming_values_and_blocks() const {
+    return zip_equal(incoming_values(), incoming_blocks());
+  }
+
   /// Removes the incoming value for \p IncomingBlock, which must be a
   /// predecessor.
   void removeIncomingValueFor(VPBlockBase *IncomingBlock) const;
@@ -1242,12 +1289,24 @@ struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors {
       : VPInstruction(Instruction::PHI, Operands, DL, Name) {}
 
   static inline bool classof(const VPUser *U) {
-    auto *R = dyn_cast<VPInstruction>(U);
-    return R && R->getOpcode() == Instruction::PHI;
+    auto *VPI = dyn_cast<VPInstruction>(U);
+    return VPI && VPI->getOpcode() == Instruction::PHI;
+  }
+
+  static inline bool classof(const VPValue *V) {
+    auto *VPI = dyn_cast<VPInstruction>(V);
+    return VPI && VPI->getOpcode() == Instruction::PHI;
+  }
+
+  static inline bool classof(const VPSingleDefRecipe *SDR) {
+    auto *VPI = dyn_cast<VPInstruction>(SDR);
+    return VPI && VPI->getOpcode() == Instruction::PHI;
   }
 
   VPPhi *clone() override {
-    return new VPPhi(operands(), getDebugLoc(), getName());
+    auto *PhiR = new VPPhi(operands(), getDebugLoc(), getName());
+    PhiR->setUnderlyingValue(getUnderlyingValue());
+    return PhiR;
   }
 
   void execute(VPTransformState &State) override;
@@ -2286,6 +2345,11 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
              VPSlotTracker &SlotTracker) const override;
 #endif
 
+  /// Returns the number of incoming values, also number of incoming blocks.
+  /// Note that at the moment, VPWidenPointerInductionRecipe only has a single
+  /// incoming value, its start value.
+  unsigned getNumIncoming() const override { return 2; }
+
   /// Returns the recurrence kind of the reduction.
   RecurKind getRecurrenceKind() const { return Kind; }
 
@@ -2374,7 +2438,8 @@ class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPSingleDefRecipe {
 /// or stores into one wide load/store and shuffles. The first operand of a
 /// VPInterleave recipe is the address, followed by the stored values, followed
 /// by an optional mask.
-class LLVM_ABI_FOR_TEST VPInterleaveRecipe : public VPRecipeBase {
+class LLVM_ABI_FOR_TEST VPInterleaveRecipe : public VPRecipeBase,
+                                             public VPIRMetadata {
   const InterleaveGroup<Instruction> *IG;
 
   /// Indicates if the interleave group is in a conditional block and requires a
@@ -2388,19 +2453,17 @@ class LLVM_ABI_FOR_TEST VPInterleaveRecipe : public VPRecipeBase {
 public:
   VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Addr,
                      ArrayRef<VPValue *> StoredValues, VPValue *Mask,
-                     bool NeedsMaskForGaps, DebugLoc DL)
-      : VPRecipeBase(VPDef::VPInterleaveSC, {Addr},
-                     DL),
-
+                     bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
+      : VPRecipeBase(VPDef::VPInterleaveSC, {Addr}, DL), VPIRMetadata(MD),
         IG(IG), NeedsMaskForGaps(NeedsMaskForGaps) {
     // TODO: extend the masked interleaved-group support to reversed access.
     assert((!Mask || !IG->isReverse()) &&
            "Reversed masked interleave-group not supported.");
-    for (unsigned i = 0; i < IG->getFactor(); ++i)
-      if (Instruction *I = IG->getMember(i)) {
-        if (I->getType()->isVoidTy())
+    for (unsigned I = 0; I < IG->getFactor(); ++I)
+      if (Instruction *Inst = IG->getMember(I)) {
+        if (Inst->getType()->isVoidTy())
           continue;
-        new VPValue(I, this);
+        new VPValue(Inst, this);
       }
 
     for (auto *SV : StoredValues)
@@ -2414,7 +2477,7 @@ class LLVM_ABI_FOR_TEST VPInterleaveRecipe : public VPRecipeBase {
 
   VPInterleaveRecipe *clone() override {
     return new VPInterleaveRecipe(IG, getAddr(), getStoredValues(), getMask(),
-                                  NeedsMaskForGaps, getDebugLoc());
+                                  NeedsMaskForGaps, *this, getDebugLoc());
   }
 
   VP_CLASSOF_IMPL(VPDef::VPInterleaveSC)
@@ -3064,10 +3127,11 @@ struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
 /// using the address to load from, the explicit vector length and an optional
 /// mask.
 struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
-  VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
+  VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
+                       VPValue *Mask)
       : VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
-                            {L.getAddr(), &EVL}, L.isConsecutive(),
-                            L.isReverse(), L, L.getDebugLoc()),
+                            {Addr, &EVL}, L.isConsecutive(), L.isReverse(), L,
+                            L.getDebugLoc()),
         VPValue(this, &getIngredient()) {
     setMask(Mask);
   }
@@ -3145,11 +3209,11 @@ struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
 /// using the value to store, the address to store to, the explicit vector
 /// length and an optional mask.
 struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
-  VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
+  VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue *Addr, VPValue &EVL,
+                        VPValue *Mask)
       : VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
-                            {S.getAddr(), S.getStoredValue(), &EVL},
-                            S.isConsecutive(), S.isReverse(), S,
-                            S.getDebugLoc()) {
+                            {Addr, S.getStoredValue(), &EVL}, S.isConsecutive(),
+                            S.isReverse(), S, S.getDebugLoc()) {
     setMask(Mask);
   }
 
@@ -3192,22 +3256,20 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
 /// Recipe to expand a SCEV expression.
 class VPExpandSCEVRecipe : public VPSingleDefRecipe {
   const SCEV *Expr;
-  ScalarEvolution &SE;
 
 public:
-  VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE)
-      : VPSingleDefRecipe(VPDef::VPExpandSCEVSC, {}), Expr(Expr), SE(SE) {}
+  VPExpandSCEVRecipe(const SCEV *Expr)
+      : VPSingleDefRecipe(VPDef::VPExpandSCEVSC, {}), Expr(Expr) {}
 
   ~VPExpandSCEVRecipe() override = default;
 
-  VPExpandSCEVRecipe *clone() override {
-    return new VPExpandSCEVRecipe(Expr, SE);
-  }
+  VPExpandSCEVRecipe *clone() override { return new VPExpandSCEVRecipe(Expr); }
 
   VP_CLASSOF_IMPL(VPDef::VPExpandSCEVSC)
 
-  /// Generate a canonical vector induction variable of the vector loop, with
-  void execute(VPTransformState &State) override;
+  void execute(VPTransformState &State) override {
+    llvm_unreachable("SCEV expressions must be expanded before final execute");
+  }
 
   /// Return the cost of this VPExpandSCEVRecipe.
   InstructionCost computeCost(ElementCount VF,
@@ -3956,9 +4018,6 @@ class VPlan {
     VPBB->setPlan(this);
   }
 
-  /// Prepare the plan for execution, setting up the required live-in values.
-  void prepareToExecute(Value *VectorTripCount, VPTransformState &State);
-
   /// Generate the IR code for this VPlan.
   void execute(VPTransformState *State);
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 4c3cdda338708..747c6623aa22a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -21,8 +21,7 @@ using namespace llvm;
 
 #define DEBUG_TYPE "vplan"
 
-VPTypeAnalysis::VPTypeAnalysis(const VPlan &Plan)
-    : Ctx(Plan.getScalarHeader()->getIRBasicBlock()->getContext()) {
+VPTypeAnalysis::VPTypeAnalysis(const VPlan &Plan) : Ctx(Plan.getContext()) {
   if (auto LoopRegion = Plan.getVectorLoopRegion()) {
     if (const auto *CanIV = dyn_cast<VPCanonicalIVPHIRecipe>(
             &LoopRegion->getEntryBasicBlock()->front())) {
@@ -74,6 +73,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
   case Instruction::ExtractElement:
   case Instruction::Freeze:
   case VPInstruction::ReductionStartVector:
+  case VPInstruction::ResumeForEpilogue:
     return inferScalarType(R->getOperand(0));
   case Instruction::Select: {
     Type *ResTy = inferScalarType(R->getOperand(1));
@@ -442,8 +442,9 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
   // assume that each recipe that has in-loop users starts an interval. We
   // record every time that an in-loop value is used, so we have a list of the
   // first and last occurrences of each recipe.
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
-      Plan.getVectorLoopRegion());
+      LoopRegion);
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
     if (!VPBB->getParent())
       break;
@@ -473,10 +474,10 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
         Ends.insert(DefR);
       }
     }
-    if (VPBB == Plan.getVectorLoopRegion()->getExiting()) {
+    if (VPBB == LoopRegion->getExiting()) {
       // VPWidenIntOrFpInductionRecipes are used implicitly at the end of the
       // exiting block, where their increment will get materialized eventually.
-      for (auto &R : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
+      for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
         if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
           EndPoint[&R] = Idx2Recipe.size();
           Ends.insert(&R);
@@ -500,7 +501,7 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
 
   LLVM_DEBUG(dbgs() << "LV(REG): Calculating max register usage:\n");
 
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan);
 
   const auto &TTICapture = TTI;
   auto GetRegUsage = [&TTICapture](Type *Ty, ElementCount VF) -> unsigned {
@@ -555,13 +556,9 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
 
         if (VFs[J].isScalar() ||
             isa<VPCanonicalIVPHIRecipe, VPReplicateRecipe, VPDerivedIVRecipe,
-                VPScalarIVStepsRecipe>(R) ||
+                VPEVLBasedIVPHIRecipe, VPScalarIVStepsRecipe>(R) ||
             (isa<VPInstruction>(R) &&
-             all_of(cast<VPSingleDefRecipe>(R)->users(),
-                    [&](VPUser *U) {
-                      return cast<VPRecipeBase>(U)->usesScalars(
-                          R->getVPSingleValue());
-                    })) ||
+             vputils::onlyScalarValuesUsed(cast<VPSingleDefRecipe>(R))) ||
             (isa<VPReductionPHIRecipe>(R) &&
              (cast<VPReductionPHIRecipe>(R))->isInLoop())) {
           unsigned ClassID = TTI.getRegisterClassForType(
@@ -614,9 +611,7 @@ SmallVector<VPRegisterUsage, 8> llvm::calculateRegisterUsageForPlan(
     for (auto *In : LoopInvariants) {
       // FIXME: The target might use more than one register for the type
       // even in the scalar case.
-      bool IsScalar = all_of(In->users(), [&](VPUser *U) {
-        return cast<VPRecipeBase>(U)->usesScalars(In);
-      });
+      bool IsScalar = vputils::onlyScalarValuesUsed(In);
 
       ElementCount VF = IsScalar ? ElementCount::getFixed(1) : VFs[Idx];
       unsigned ClassID = TTI.getRegisterClassForType(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
index cd86d27cf9122..c6c43698b7483 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h
@@ -58,9 +58,6 @@ class VPTypeAnalysis {
   Type *inferScalarTypeForRecipe(const VPReplicateRecipe *R);
 
 public:
-  VPTypeAnalysis(Type *CanonicalIVTy)
-      : CanonicalIVTy(CanonicalIVTy), Ctx(CanonicalIVTy->getContext()) {}
-
   VPTypeAnalysis(const VPlan &Plan);
 
   /// Infer the type of \p V. Returns the scalar type of \p V.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
index 1b91901e25d08..80b48de57b406 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
@@ -91,17 +91,15 @@ void PlainCFGBuilder::fixHeaderPhis() {
   for (auto *Phi : PhisToFix) {
     assert(IRDef2VPValue.count(Phi) && "Missing VPInstruction for PHINode.");
     VPValue *VPVal = IRDef2VPValue[Phi];
-    assert(isa<VPWidenPHIRecipe>(VPVal) &&
-           "Expected WidenPHIRecipe for phi node.");
-    auto *VPPhi = cast<VPWidenPHIRecipe>(VPVal);
-    assert(VPPhi->getNumOperands() == 0 &&
-           "Expected VPInstruction with no operands.");
+    assert(isa<VPPhi>(VPVal) && "Expected VPPhi for phi node.");
+    auto *PhiR = cast<VPPhi>(VPVal);
+    assert(PhiR->getNumOperands() == 0 && "Expected VPPhi with no operands.");
     assert(isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent())) &&
            "Expected Phi in header block.");
     assert(Phi->getNumOperands() == 2 &&
            "header phi must have exactly 2 operands");
     for (BasicBlock *Pred : predecessors(Phi->getParent()))
-      VPPhi->addOperand(
+      PhiR->addOperand(
           getOrCreateVPOperand(Phi->getIncomingValueForBlock(Pred)));
   }
 }
@@ -204,11 +202,11 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
 
     VPSingleDefRecipe *NewR;
     if (auto *Phi = dyn_cast<PHINode>(Inst)) {
-      // Phi node's operands may have not been visited at this point. We create
+      // Phi node's operands may not have been visited at this point. We create
       // an empty VPInstruction that we will fix once the whole plain CFG has
       // been built.
-      NewR = new VPWidenPHIRecipe(Phi, nullptr, Phi->getDebugLoc(), "vec.phi");
-      VPBB->appendRecipe(NewR);
+      NewR = VPIRBuilder.createScalarPhi({}, Phi->getDebugLoc(), "vec.phi");
+      NewR->setUnderlyingValue(Phi);
       if (isHeaderBB(Phi->getParent(), LI->getLoopFor(Phi->getParent()))) {
         // Header phis need to be fixed after the VPBB for the latch has been
         // created.
@@ -338,12 +336,6 @@ std::unique_ptr<VPlan> PlainCFGBuilder::buildPlainCFG() {
   return std::move(Plan);
 }
 
-std::unique_ptr<VPlan> VPlanTransforms::buildPlainCFG(Loop *TheLoop,
-                                                      LoopInfo &LI) {
-  PlainCFGBuilder Builder(TheLoop, &LI);
-  return Builder.buildPlainCFG();
-}
-
 /// Checks if \p HeaderVPB is a loop header block in the plain CFG; that is, it
 /// has exactly 2 predecessors (preheader and latch), where the block
 /// dominates the latch and the preheader dominates the block. If it is a
@@ -459,10 +451,32 @@ static void addCanonicalIVRecipes(VPlan &Plan, VPBasicBlock *HeaderVPBB,
                        LatchDL);
 }
 
-void VPlanTransforms::prepareForVectorization(
-    VPlan &Plan, Type *InductionTy, PredicatedScalarEvolution &PSE,
-    bool RequiresScalarEpilogueCheck, bool TailFolded, Loop *TheLoop,
-    DebugLoc IVDL, bool HasUncountableEarlyExit, VFRange &Range) {
+/// Creates extracts for values in \p Plan defined in a loop region and used
+/// outside a loop region.
+static void createExtractsForLiveOuts(VPlan &Plan, VPBasicBlock *MiddleVPBB) {
+  VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
+  for (VPBasicBlock *EB : Plan.getExitBlocks()) {
+    if (EB->getSinglePredecessor() != MiddleVPBB)
+      continue;
+
+    for (VPRecipeBase &R : EB->phis()) {
+      auto *ExitIRI = cast<VPIRPhi>(&R);
+      for (unsigned Idx = 0; Idx != ExitIRI->getNumIncoming(); ++Idx) {
+        VPRecipeBase *Inc = ExitIRI->getIncomingValue(Idx)->getDefiningRecipe();
+        if (!Inc)
+          continue;
+        assert(ExitIRI->getNumOperands() == 1 &&
+               ExitIRI->getParent()->getSinglePredecessor() == MiddleVPBB &&
+               "exit values from early exits must be fixed when branch to "
+               "early-exit is added");
+        ExitIRI->extractLastLaneOfFirstOperand(B);
+      }
+    }
+  }
+}
+
+static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL,
+                               PredicatedScalarEvolution &PSE, Loop *TheLoop) {
   VPDominatorTree VPDT;
   VPDT.recalculate(Plan);
 
@@ -488,12 +502,55 @@ void VPlanTransforms::prepareForVectorization(
 
   addCanonicalIVRecipes(Plan, HeaderVPBB, LatchVPBB, InductionTy, IVDL);
 
-  [[maybe_unused]] bool HandledUncountableEarlyExit = false;
+  // Create SCEV and VPValue for the trip count.
+  // We use the symbolic max backedge-taken-count, which works also when
+  // vectorizing loops with uncountable early exits.
+  const SCEV *BackedgeTakenCountSCEV = PSE.getSymbolicMaxBackedgeTakenCount();
+  assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCountSCEV) &&
+         "Invalid backedge-taken count");
+  ScalarEvolution &SE = *PSE.getSE();
+  const SCEV *TripCount = SE.getTripCountFromExitCount(BackedgeTakenCountSCEV,
+                                                       InductionTy, TheLoop);
+  Plan.setTripCount(vputils::getOrCreateVPValueForSCEVExpr(Plan, TripCount));
+
+  VPBasicBlock *ScalarPH = Plan.createVPBasicBlock("scalar.ph");
+  VPBlockUtils::connectBlocks(ScalarPH, Plan.getScalarHeader());
+
+  // The connection order corresponds to the operands of the conditional branch,
+  // with the middle block already connected to the exit block.
+  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
+  // Also connect the entry block to the scalar preheader.
+  // TODO: Also introduce a branch recipe together with the minimum trip count
+  // check.
+  VPBlockUtils::connectBlocks(Plan.getEntry(), ScalarPH);
+  Plan.getEntry()->swapSuccessors();
+
+  createExtractsForLiveOuts(Plan, MiddleVPBB);
+}
+
+std::unique_ptr<VPlan>
+VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy,
+                             DebugLoc IVDL, PredicatedScalarEvolution &PSE) {
+  PlainCFGBuilder Builder(TheLoop, &LI);
+  std::unique_ptr<VPlan> VPlan0 = Builder.buildPlainCFG();
+  addInitialSkeleton(*VPlan0, InductionTy, IVDL, PSE, TheLoop);
+  return VPlan0;
+}
+
+void VPlanTransforms::handleEarlyExits(VPlan &Plan,
+                                       bool HasUncountableEarlyExit,
+                                       VFRange &Range) {
+  auto *MiddleVPBB = cast<VPBasicBlock>(
+      Plan.getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
+  auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
+  VPBlockBase *HeaderVPB = cast<VPBasicBlock>(LatchVPBB->getSuccessors()[1]);
+
   // Disconnect all early exits from the loop leaving it with a single exit from
   // the latch. Early exits that are countable are left for a scalar epilog. The
   // condition of uncountable early exits (currently at most one is supported)
   // is fused into the latch exit, and used to branch from middle block to the
   // early exit destination.
+  [[maybe_unused]] bool HandledUncountableEarlyExit = false;
   for (VPIRBasicBlock *EB : Plan.getExitBlocks()) {
     for (VPBlockBase *Pred : to_vector(EB->getPredecessors())) {
       if (Pred == MiddleVPBB)
@@ -502,7 +559,8 @@ void VPlanTransforms::prepareForVectorization(
         assert(!HandledUncountableEarlyExit &&
                "can handle exactly one uncountable early exit");
         handleUncountableEarlyExit(cast<VPBasicBlock>(Pred), EB, Plan,
-                                   HeaderVPBB, LatchVPBB, Range);
+                                   cast<VPBasicBlock>(HeaderVPB), LatchVPBB,
+                                   Range);
         HandledUncountableEarlyExit = true;
       } else {
         for (VPRecipeBase &R : EB->phis())
@@ -515,36 +573,18 @@ void VPlanTransforms::prepareForVectorization(
 
   assert((!HasUncountableEarlyExit || HandledUncountableEarlyExit) &&
          "missed an uncountable exit that must be handled");
+}
 
-  // Create SCEV and VPValue for the trip count.
-  // We use the symbolic max backedge-taken-count, which works also when
-  // vectorizing loops with uncountable early exits.
-  const SCEV *BackedgeTakenCountSCEV = PSE.getSymbolicMaxBackedgeTakenCount();
-  assert(!isa<SCEVCouldNotCompute>(BackedgeTakenCountSCEV) &&
-         "Invalid loop count");
-  ScalarEvolution &SE = *PSE.getSE();
-  const SCEV *TripCount = SE.getTripCountFromExitCount(BackedgeTakenCountSCEV,
-                                                       InductionTy, TheLoop);
-  Plan.setTripCount(
-      vputils::getOrCreateVPValueForSCEVExpr(Plan, TripCount, SE));
-
-  VPBasicBlock *ScalarPH = Plan.createVPBasicBlock("scalar.ph");
-  VPBlockUtils::connectBlocks(ScalarPH, Plan.getScalarHeader());
-
-  // The connection order corresponds to the operands of the conditional branch,
-  // with the middle block already connected to the exit block.
-  VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
-  // Also connect the entry block to the scalar preheader.
-  // TODO: Also introduce a branch recipe together with the minimum trip count
-  // check.
-  VPBlockUtils::connectBlocks(Plan.getEntry(), ScalarPH);
-  Plan.getEntry()->swapSuccessors();
-
+void VPlanTransforms::addMiddleCheck(VPlan &Plan,
+                                     bool RequiresScalarEpilogueCheck,
+                                     bool TailFolded) {
+  auto *MiddleVPBB = cast<VPBasicBlock>(
+      Plan.getScalarHeader()->getSinglePredecessor()->getPredecessors()[0]);
   // If MiddleVPBB has a single successor then the original loop does not exit
   // via the latch and the single successor must be the scalar preheader.
   // There's no need to add a runtime check to MiddleVPBB.
   if (MiddleVPBB->getNumSuccessors() == 1) {
-    assert(MiddleVPBB->getSingleSuccessor() == ScalarPH &&
+    assert(MiddleVPBB->getSingleSuccessor() == Plan.getScalarPreheader() &&
            "must have ScalarPH as single successor");
     return;
   }
@@ -566,6 +606,7 @@ void VPlanTransforms::prepareForVectorization(
   // the corresponding compare because they may have ended up with different
   // line numbers and we want to avoid awkward line stepping while debugging.
   // E.g., if the compare has got a line number inside the loop.
+  auto *LatchVPBB = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
   DebugLoc LatchDL = LatchVPBB->getTerminator()->getDebugLoc();
   VPBuilder Builder(MiddleVPBB);
   VPValue *Cmp;
@@ -592,30 +633,6 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
   TopRegion->getEntryBasicBlock()->setName("vector.body");
 }
 
-void VPlanTransforms::createExtractsForLiveOuts(VPlan &Plan) {
-  for (VPBasicBlock *EB : Plan.getExitBlocks()) {
-    VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
-    VPBuilder B(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
-
-    if (EB->getSinglePredecessor() != Plan.getMiddleBlock())
-      continue;
-
-    for (VPRecipeBase &R : EB->phis()) {
-      auto *ExitIRI = cast<VPIRPhi>(&R);
-      for (unsigned Idx = 0; Idx != ExitIRI->getNumIncoming(); ++Idx) {
-        VPRecipeBase *Inc = ExitIRI->getIncomingValue(Idx)->getDefiningRecipe();
-        if (!Inc || !Inc->getParent()->getParent())
-          continue;
-        assert(ExitIRI->getNumOperands() == 1 &&
-               ExitIRI->getParent()->getSinglePredecessor() == MiddleVPBB &&
-               "exit values from early exits must be fixed when branch to "
-               "early-exit is added");
-        ExitIRI->extractLastLaneOfFirstOperand(B);
-      }
-    }
-  }
-}
-
 // Likelyhood of bypassing the vectorized loop due to a runtime check block,
 // including memory overlap checks block and wrapping/unit-stride checks block.
 static constexpr uint32_t CheckBypassWeights[] = {1, 127};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
index 4154720cd2608..5ad2ac6b61e05 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanHelpers.h
@@ -351,10 +351,10 @@ struct VPCostContext {
   TargetTransformInfo::TargetCostKind CostKind;
 
   VPCostContext(const TargetTransformInfo &TTI, const TargetLibraryInfo &TLI,
-                Type *CanIVTy, LoopVectorizationCostModel &CM,
+                const VPlan &Plan, LoopVectorizationCostModel &CM,
                 TargetTransformInfo::TargetCostKind CostKind)
-      : TTI(TTI), TLI(TLI), Types(CanIVTy), LLVMCtx(CanIVTy->getContext()),
-        CM(CM), CostKind(CostKind) {}
+      : TTI(TTI), TLI(TLI), Types(Plan), LLVMCtx(Plan.getContext()), CM(CM),
+        CostKind(CostKind) {}
 
   /// Return the cost for \p UI with \p VF using the legacy cost model as
   /// fallback until computing the cost of all recipes migrates to VPlan.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index d133610ef4f75..1ec6ae677374c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -10,11 +10,6 @@
 // tree-based pattern matches on the VPlan values and recipes, based on
 // LLVM's IR pattern matchers.
 //
-// Currently it provides generic matchers for unary and binary VPInstructions,
-// and specialized matchers like m_Not, m_ActiveLaneMask, m_BranchOnCond,
-// m_BranchOnCount to match specific VPInstructions.
-// TODO: Add missing matchers for additional opcodes and recipes as needed.
-//
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
@@ -158,11 +153,7 @@ template <typename LTy, typename RTy> struct match_combine_or {
   match_combine_or(const LTy &Left, const RTy &Right) : L(Left), R(Right) {}
 
   template <typename ITy> bool match(ITy *V) const {
-    if (L.match(V))
-      return true;
-    if (R.match(V))
-      return true;
-    return false;
+    return L.match(V) || R.match(V);
   }
 };
 
@@ -200,15 +191,11 @@ template <typename Ops_t, unsigned Opcode, bool Commutative,
 struct Recipe_match {
   Ops_t Ops;
 
-  Recipe_match() : Ops() {
-    static_assert(std::tuple_size<Ops_t>::value == 0 &&
-                  "constructor can only be used with zero operands");
-  }
-  Recipe_match(Ops_t Ops) : Ops(Ops) {}
-  template <typename A_t, typename B_t>
-  Recipe_match(A_t A, B_t B) : Ops({A, B}) {
-    static_assert(std::tuple_size<Ops_t>::value == 2 &&
-                  "constructor can only be used for binary matcher");
+  template <typename... OpTy> Recipe_match(OpTy... Ops) : Ops(Ops...) {
+    static_assert(std::tuple_size<Ops_t>::value == sizeof...(Ops) &&
+                  "number of operands in constructor doesn't match Ops_t");
+    static_assert((!Commutative || std::tuple_size<Ops_t>::value == 2) &&
+                  "only binary ops can be commutative");
   }
 
   bool match(const VPValue *V) const {
@@ -254,7 +241,6 @@ struct Recipe_match {
     // Check for recipes that do not have opcodes.
     if constexpr (std::is_same<RecipeTy, VPScalarIVStepsRecipe>::value ||
                   std::is_same<RecipeTy, VPCanonicalIVPHIRecipe>::value ||
-                  std::is_same<RecipeTy, VPWidenSelectRecipe>::value ||
                   std::is_same<RecipeTy, VPDerivedIVRecipe>::value ||
                   std::is_same<RecipeTy, VPWidenGEPRecipe>::value)
       return DefR;
@@ -270,202 +256,245 @@ struct Recipe_match {
   }
 };
 
-template <unsigned Opcode, typename... RecipeTys>
-using ZeroOpRecipe_match =
-    Recipe_match<std::tuple<>, Opcode, false, RecipeTys...>;
+template <unsigned Opcode, typename... OpTys>
+using AllRecipe_match =
+    Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ false,
+                 VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe,
+                 VPInstruction, VPWidenSelectRecipe>;
 
-template <typename Op0_t, unsigned Opcode, typename... RecipeTys>
-using UnaryRecipe_match =
-    Recipe_match<std::tuple<Op0_t>, Opcode, false, RecipeTys...>;
+template <unsigned Opcode, typename... OpTys>
+using AllRecipe_commutative_match =
+    Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ true,
+                 VPWidenRecipe, VPReplicateRecipe, VPInstruction>;
 
-template <typename Op0_t, unsigned Opcode>
-using UnaryVPInstruction_match =
-    UnaryRecipe_match<Op0_t, Opcode, VPInstruction>;
+template <unsigned Opcode, typename... OpTys>
+using VPInstruction_match = Recipe_match<std::tuple<OpTys...>, Opcode,
+                                         /*Commutative*/ false, VPInstruction>;
 
-template <unsigned Opcode>
-using ZeroOpVPInstruction_match = ZeroOpRecipe_match<Opcode, VPInstruction>;
-
-template <typename Op0_t, unsigned Opcode>
-using AllUnaryRecipe_match =
-    UnaryRecipe_match<Op0_t, Opcode, VPWidenRecipe, VPReplicateRecipe,
-                      VPWidenCastRecipe, VPInstruction>;
-
-template <typename Op0_t, typename Op1_t, unsigned Opcode, bool Commutative,
-          typename... RecipeTys>
-using BinaryRecipe_match =
-    Recipe_match<std::tuple<Op0_t, Op1_t>, Opcode, Commutative, RecipeTys...>;
-
-template <typename Op0_t, typename Op1_t, unsigned Opcode>
-using BinaryVPInstruction_match =
-    BinaryRecipe_match<Op0_t, Op1_t, Opcode, /*Commutative*/ false,
-                       VPInstruction>;
-
-template <typename Op0_t, typename Op1_t, typename Op2_t, unsigned Opcode,
-          bool Commutative, typename... RecipeTys>
-using TernaryRecipe_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>,
-                                         Opcode, Commutative, RecipeTys...>;
-
-template <typename Op0_t, typename Op1_t, typename Op2_t, unsigned Opcode>
-using TernaryVPInstruction_match =
-    TernaryRecipe_match<Op0_t, Op1_t, Op2_t, Opcode, /*Commutative*/ false,
-                        VPInstruction>;
-
-template <typename Op0_t, typename Op1_t, unsigned Opcode,
-          bool Commutative = false>
-using AllBinaryRecipe_match =
-    BinaryRecipe_match<Op0_t, Op1_t, Opcode, Commutative, VPWidenRecipe,
-                       VPReplicateRecipe, VPWidenCastRecipe, VPInstruction>;
+template <unsigned Opcode, typename... OpTys>
+inline VPInstruction_match<Opcode, OpTys...>
+m_VPInstruction(const OpTys &...Ops) {
+  return VPInstruction_match<Opcode, OpTys...>(Ops...);
+}
 
 /// BuildVector is matches only its opcode, w/o matching its operands as the
 /// number of operands is not fixed.
-inline ZeroOpVPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
-  return ZeroOpVPInstruction_match<VPInstruction::BuildVector>();
-}
-
-template <unsigned Opcode, typename Op0_t>
-inline UnaryVPInstruction_match<Op0_t, Opcode>
-m_VPInstruction(const Op0_t &Op0) {
-  return UnaryVPInstruction_match<Op0_t, Opcode>(Op0);
-}
-
-template <unsigned Opcode, typename Op0_t, typename Op1_t>
-inline BinaryVPInstruction_match<Op0_t, Op1_t, Opcode>
-m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1) {
-  return BinaryVPInstruction_match<Op0_t, Op1_t, Opcode>(Op0, Op1);
-}
-
-template <unsigned Opcode, typename Op0_t, typename Op1_t, typename Op2_t>
-inline TernaryVPInstruction_match<Op0_t, Op1_t, Op2_t, Opcode>
-m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
-  return TernaryVPInstruction_match<Op0_t, Op1_t, Op2_t, Opcode>(
-      {Op0, Op1, Op2});
+inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
+  return m_VPInstruction<VPInstruction::BuildVector>();
 }
 
-template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
-          unsigned Opcode, bool Commutative, typename... RecipeTys>
-using Recipe4Op_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t, Op3_t>,
-                                     Opcode, Commutative, RecipeTys...>;
-
-template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
-          unsigned Opcode>
-using VPInstruction4Op_match =
-    Recipe4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode, /*Commutative*/ false,
-                    VPInstruction>;
-
-template <unsigned Opcode, typename Op0_t, typename Op1_t, typename Op2_t,
-          typename Op3_t>
-inline VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>
-m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2,
-                const Op3_t &Op3) {
-  return VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>(
-      {Op0, Op1, Op2, Op3});
-}
 template <typename Op0_t>
-inline UnaryVPInstruction_match<Op0_t, Instruction::Freeze>
+inline VPInstruction_match<Instruction::Freeze, Op0_t>
 m_Freeze(const Op0_t &Op0) {
   return m_VPInstruction<Instruction::Freeze>(Op0);
 }
 
 template <typename Op0_t>
-inline UnaryVPInstruction_match<Op0_t, VPInstruction::BranchOnCond>
+inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t>
 m_BranchOnCond(const Op0_t &Op0) {
   return m_VPInstruction<VPInstruction::BranchOnCond>(Op0);
 }
 
 template <typename Op0_t>
-inline UnaryVPInstruction_match<Op0_t, VPInstruction::Broadcast>
+inline VPInstruction_match<VPInstruction::Broadcast, Op0_t>
 m_Broadcast(const Op0_t &Op0) {
   return m_VPInstruction<VPInstruction::Broadcast>(Op0);
 }
 
+template <typename Op0_t>
+inline VPInstruction_match<VPInstruction::ExtractLastElement, Op0_t>
+m_ExtractLastElement(const Op0_t &Op0) {
+  return m_VPInstruction<VPInstruction::ExtractLastElement>(Op0);
+}
 template <typename Op0_t, typename Op1_t>
-inline BinaryVPInstruction_match<Op0_t, Op1_t, VPInstruction::ActiveLaneMask>
+inline VPInstruction_match<VPInstruction::ActiveLaneMask, Op0_t, Op1_t>
 m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1) {
   return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1);
 }
 
 template <typename Op0_t, typename Op1_t>
-inline BinaryVPInstruction_match<Op0_t, Op1_t, VPInstruction::BranchOnCount>
+inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t>
 m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) {
   return m_VPInstruction<VPInstruction::BranchOnCount>(Op0, Op1);
 }
 
 template <unsigned Opcode, typename Op0_t>
-inline AllUnaryRecipe_match<Op0_t, Opcode> m_Unary(const Op0_t &Op0) {
-  return AllUnaryRecipe_match<Op0_t, Opcode>(Op0);
+inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) {
+  return AllRecipe_match<Opcode, Op0_t>(Op0);
 }
 
 template <typename Op0_t>
-inline AllUnaryRecipe_match<Op0_t, Instruction::Trunc>
-m_Trunc(const Op0_t &Op0) {
+inline AllRecipe_match<Instruction::Trunc, Op0_t> m_Trunc(const Op0_t &Op0) {
   return m_Unary<Instruction::Trunc, Op0_t>(Op0);
 }
 
 template <typename Op0_t>
-inline AllUnaryRecipe_match<Op0_t, Instruction::ZExt> m_ZExt(const Op0_t &Op0) {
+inline AllRecipe_match<Instruction::ZExt, Op0_t> m_ZExt(const Op0_t &Op0) {
   return m_Unary<Instruction::ZExt, Op0_t>(Op0);
 }
 
 template <typename Op0_t>
-inline AllUnaryRecipe_match<Op0_t, Instruction::SExt> m_SExt(const Op0_t &Op0) {
+inline AllRecipe_match<Instruction::SExt, Op0_t> m_SExt(const Op0_t &Op0) {
   return m_Unary<Instruction::SExt, Op0_t>(Op0);
 }
 
 template <typename Op0_t>
-inline match_combine_or<AllUnaryRecipe_match<Op0_t, Instruction::ZExt>,
-                        AllUnaryRecipe_match<Op0_t, Instruction::SExt>>
+inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>,
+                        AllRecipe_match<Instruction::SExt, Op0_t>>
 m_ZExtOrSExt(const Op0_t &Op0) {
   return m_CombineOr(m_ZExt(Op0), m_SExt(Op0));
 }
 
-template <unsigned Opcode, typename Op0_t, typename Op1_t,
-          bool Commutative = false>
-inline AllBinaryRecipe_match<Op0_t, Op1_t, Opcode, Commutative>
-m_Binary(const Op0_t &Op0, const Op1_t &Op1) {
-  return AllBinaryRecipe_match<Op0_t, Op1_t, Opcode, Commutative>(Op0, Op1);
+template <unsigned Opcode, typename Op0_t, typename Op1_t>
+inline AllRecipe_match<Opcode, Op0_t, Op1_t> m_Binary(const Op0_t &Op0,
+                                                      const Op1_t &Op1) {
+  return AllRecipe_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
 }
 
 template <unsigned Opcode, typename Op0_t, typename Op1_t>
-inline AllBinaryRecipe_match<Op0_t, Op1_t, Opcode, true>
+inline AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>
 m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {
-  return AllBinaryRecipe_match<Op0_t, Op1_t, Opcode, true>(Op0, Op1);
+  return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
+}
+
+template <typename Op0_t, typename Op1_t>
+inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t>
+m_c_Add(const Op0_t &Op0, const Op1_t &Op1) {
+  return m_c_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
 }
 
 template <typename Op0_t, typename Op1_t>
-inline AllBinaryRecipe_match<Op0_t, Op1_t, Instruction::Mul>
-m_Mul(const Op0_t &Op0, const Op1_t &Op1) {
+inline AllRecipe_match<Instruction::Sub, Op0_t, Op1_t> m_Sub(const Op0_t &Op0,
+                                                             const Op1_t &Op1) {
+  return m_Binary<Instruction::Sub, Op0_t, Op1_t>(Op0, Op1);
+}
+
+template <typename Op0_t, typename Op1_t>
+inline AllRecipe_match<Instruction::Mul, Op0_t, Op1_t> m_Mul(const Op0_t &Op0,
+                                                             const Op1_t &Op1) {
   return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
 }
 
 template <typename Op0_t, typename Op1_t>
-inline AllBinaryRecipe_match<Op0_t, Op1_t, Instruction::Mul,
-                             /* Commutative =*/true>
+inline AllRecipe_commutative_match<Instruction::Mul, Op0_t, Op1_t>
 m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) {
-  return m_Binary<Instruction::Mul, Op0_t, Op1_t, true>(Op0, Op1);
+  return m_c_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
 }
 
 /// Match a binary OR operation. Note that while conceptually the operands can
 /// be matched commutatively, \p Commutative defaults to false in line with the
 /// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative
 /// version of the matcher.
-template <typename Op0_t, typename Op1_t, bool Commutative = false>
-inline AllBinaryRecipe_match<Op0_t, Op1_t, Instruction::Or, Commutative>
+template <typename Op0_t, typename Op1_t>
+inline AllRecipe_match<Instruction::Or, Op0_t, Op1_t>
 m_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
-  return m_Binary<Instruction::Or, Op0_t, Op1_t, Commutative>(Op0, Op1);
+  return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
 }
 
 template <typename Op0_t, typename Op1_t>
-inline AllBinaryRecipe_match<Op0_t, Op1_t, Instruction::Or,
-                             /*Commutative*/ true>
+inline AllRecipe_commutative_match<Instruction::Or, Op0_t, Op1_t>
 m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
-  return m_BinaryOr<Op0_t, Op1_t, /*Commutative*/ true>(Op0, Op1);
+  return m_c_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
+}
+
+/// Cmp_match is a variant of BinaryRecipe_match that also binds the comparison
+/// predicate. Opcodes must either be Instruction::ICmp or Instruction::FCmp, or
+/// both.
+template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
+struct Cmp_match {
+  static_assert((sizeof...(Opcodes) == 1 || sizeof...(Opcodes) == 2) &&
+                "Expected one or two opcodes");
+  static_assert(
+      ((Opcodes == Instruction::ICmp || Opcodes == Instruction::FCmp) && ...) &&
+      "Expected a compare instruction opcode");
+
+  CmpPredicate *Predicate = nullptr;
+  Op0_t Op0;
+  Op1_t Op1;
+
+  Cmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1)
+      : Predicate(&Pred), Op0(Op0), Op1(Op1) {}
+  Cmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {}
+
+  bool match(const VPValue *V) const {
+    auto *DefR = V->getDefiningRecipe();
+    return DefR && match(DefR);
+  }
+
+  bool match(const VPRecipeBase *V) const {
+    if ((m_Binary<Opcodes>(Op0, Op1).match(V) || ...)) {
+      if (Predicate)
+        *Predicate = cast<VPRecipeWithIRFlags>(V)->getPredicate();
+      return true;
+    }
+    return false;
+  }
+};
+
+/// SpecificCmp_match is a variant of Cmp_match that matches the comparison
+/// predicate, instead of binding it.
+template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
+struct SpecificCmp_match {
+  const CmpPredicate Predicate;
+  Op0_t Op0;
+  Op1_t Op1;
+
+  SpecificCmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS)
+      : Predicate(Pred), Op0(LHS), Op1(RHS) {}
+
+  bool match(const VPValue *V) const {
+    CmpPredicate CurrentPred;
+    return Cmp_match<Op0_t, Op1_t, Opcodes...>(CurrentPred, Op0, Op1)
+               .match(V) &&
+           CmpPredicate::getMatching(CurrentPred, Predicate);
+  }
+};
+
+template <typename Op0_t, typename Op1_t>
+inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> m_ICmp(const Op0_t &Op0,
+                                                         const Op1_t &Op1) {
+  return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1);
+}
+
+template <typename Op0_t, typename Op1_t>
+inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp>
+m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
+  return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Pred, Op0, Op1);
+}
+
+template <typename Op0_t, typename Op1_t>
+inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>
+m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
+  return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>(MatchPred, Op0,
+                                                            Op1);
+}
+
+template <typename Op0_t, typename Op1_t>
+inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
+m_Cmp(const Op0_t &Op0, const Op1_t &Op1) {
+  return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0,
+                                                                       Op1);
+}
+
+template <typename Op0_t, typename Op1_t>
+inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
+m_Cmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
+  return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
+      Pred, Op0, Op1);
+}
+
+template <typename Op0_t, typename Op1_t>
+inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
+m_SpecificCmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
+  return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
+      MatchPred, Op0, Op1);
 }
 
 template <typename Op0_t, typename Op1_t>
 using GEPLikeRecipe_match =
-    BinaryRecipe_match<Op0_t, Op1_t, Instruction::GetElementPtr, false,
-                       VPWidenRecipe, VPReplicateRecipe, VPWidenGEPRecipe,
-                       VPInstruction>;
+    Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,
+                 /*Commutative*/ false, VPWidenRecipe, VPReplicateRecipe,
+                 VPWidenGEPRecipe, VPInstruction>;
 
 template <typename Op0_t, typename Op1_t>
 inline GEPLikeRecipe_match<Op0_t, Op1_t> m_GetElementPtr(const Op0_t &Op0,
@@ -473,22 +502,17 @@ inline GEPLikeRecipe_match<Op0_t, Op1_t> m_GetElementPtr(const Op0_t &Op0,
   return GEPLikeRecipe_match<Op0_t, Op1_t>(Op0, Op1);
 }
 
-template <typename Op0_t, typename Op1_t, typename Op2_t, unsigned Opcode>
-using AllTernaryRecipe_match =
-    Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, Opcode, false,
-                 VPReplicateRecipe, VPInstruction, VPWidenSelectRecipe>;
-
 template <typename Op0_t, typename Op1_t, typename Op2_t>
-inline AllTernaryRecipe_match<Op0_t, Op1_t, Op2_t, Instruction::Select>
+inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>
 m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
-  return AllTernaryRecipe_match<Op0_t, Op1_t, Op2_t, Instruction::Select>(
+  return AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>(
       {Op0, Op1, Op2});
 }
 
 template <typename Op0_t>
-inline match_combine_or<UnaryVPInstruction_match<Op0_t, VPInstruction::Not>,
-                        AllBinaryRecipe_match<int_pred_ty<is_all_ones>, Op0_t,
-                                              Instruction::Xor, true>>
+inline match_combine_or<VPInstruction_match<VPInstruction::Not, Op0_t>,
+                        AllRecipe_commutative_match<
+                            Instruction::Xor, int_pred_ty<is_all_ones>, Op0_t>>
 m_Not(const Op0_t &Op0) {
   return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),
                      m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));
@@ -496,9 +520,8 @@ m_Not(const Op0_t &Op0) {
 
 template <typename Op0_t, typename Op1_t>
 inline match_combine_or<
-    BinaryVPInstruction_match<Op0_t, Op1_t, VPInstruction::LogicalAnd>,
-    AllTernaryRecipe_match<Op0_t, Op1_t, specific_intval<1>,
-                           Instruction::Select>>
+    VPInstruction_match<VPInstruction::LogicalAnd, Op0_t, Op1_t>,
+    AllRecipe_match<Instruction::Select, Op0_t, Op1_t, specific_intval<1>>>
 m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
   return m_CombineOr(
       m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
@@ -506,15 +529,14 @@ m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
 }
 
 template <typename Op0_t, typename Op1_t>
-inline AllTernaryRecipe_match<Op0_t, specific_intval<1>, Op1_t,
-                              Instruction::Select>
+inline AllRecipe_match<Instruction::Select, Op0_t, specific_intval<1>, Op1_t>
 m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
   return m_Select(Op0, m_True(), Op1);
 }
 
 template <typename Op0_t, typename Op1_t, typename Op2_t>
-using VPScalarIVSteps_match =
-    TernaryRecipe_match<Op0_t, Op1_t, Op2_t, 0, false, VPScalarIVStepsRecipe>;
+using VPScalarIVSteps_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0,
+                                           false, VPScalarIVStepsRecipe>;
 
 template <typename Op0_t, typename Op1_t, typename Op2_t>
 inline VPScalarIVSteps_match<Op0_t, Op1_t, Op2_t>
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
index 3b3bbc312402e..cdadc33e3088e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -227,10 +227,10 @@ void VPPredicator::createSwitchEdgeMasks(VPInstruction *SI) {
 }
 
 void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
-  SmallVector<VPWidenPHIRecipe *> Phis;
+  SmallVector<VPPhi *> Phis;
   for (VPRecipeBase &R : VPBB->phis())
-    Phis.push_back(cast<VPWidenPHIRecipe>(&R));
-  for (VPWidenPHIRecipe *PhiR : Phis) {
+    Phis.push_back(cast<VPPhi>(&R));
+  for (VPPhi *PhiR : Phis) {
     // The non-header Phi is converted into a Blend recipe below,
     // so we don't have to worry about the insertion order and we can just use
     // the builder. At this point we generate the predication tree. There may
@@ -238,14 +238,11 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) {
     // optimizations will clean it up.
 
     SmallVector<VPValue *, 2> OperandsWithMask;
-    unsigned NumIncoming = PhiR->getNumIncoming();
-    for (unsigned In = 0; In < NumIncoming; In++) {
-      const VPBasicBlock *Pred = PhiR->getIncomingBlock(In);
-      OperandsWithMask.push_back(PhiR->getIncomingValue(In));
-      VPValue *EdgeMask = getEdgeMask(Pred, VPBB);
+    for (const auto &[InVPV, InVPBB] : PhiR->incoming_values_and_blocks()) {
+      OperandsWithMask.push_back(InVPV);
+      VPValue *EdgeMask = getEdgeMask(InVPBB, VPBB);
       if (!EdgeMask) {
-        assert(In == 0 && "Both null and non-null edge masks found");
-        assert(all_equal(PhiR->operands()) &&
+        assert(all_equal(PhiR->incoming_values()) &&
                "Distinct incoming values with one having a full mask");
         break;
       }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index e971ba1aac15c..0368f9b8dbb00 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -37,7 +37,6 @@
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include "llvm/Transforms/Utils/LoopVersioning.h"
-#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 #include <cassert>
 
 using namespace llvm;
@@ -326,8 +325,7 @@ VPPartialReductionRecipe::computeCost(ElementCount VF,
   // Pick out opcode, type/ext information and use sub side effects from a widen
   // recipe.
   auto HandleWiden = [&](VPWidenRecipe *Widen) {
-    if (match(Widen,
-              m_Binary<Instruction::Sub>(m_SpecificInt(0), m_VPValue(Op)))) {
+    if (match(Widen, m_Sub(m_SpecificInt(0), m_VPValue(Op)))) {
       Widen = dyn_cast<VPWidenRecipe>(Op->getDefiningRecipe());
     }
     Opcode = Widen->getOpcode();
@@ -452,6 +450,7 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
 
   switch (Opcode) {
   case VPInstruction::StepVector:
+  case VPInstruction::VScale:
     return 0;
   case Instruction::Alloca:
   case Instruction::ExtractValue:
@@ -459,6 +458,8 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) {
   case Instruction::Load:
   case VPInstruction::AnyOf:
   case VPInstruction::BranchOnCond:
+  case VPInstruction::BuildStructVector:
+  case VPInstruction::BuildVector:
   case VPInstruction::CalculateTripCountMinusVF:
   case VPInstruction::CanonicalIVIncrementForPart:
   case VPInstruction::ExplicitVectorLength:
@@ -517,6 +518,7 @@ bool VPInstruction::canGenerateScalarForFirstLane() const {
   case VPInstruction::PtrAdd:
   case VPInstruction::ExplicitVectorLength:
   case VPInstruction::AnyOf:
+  case VPInstruction::Not:
     return true;
   default:
     return false;
@@ -569,7 +571,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
 
   switch (getOpcode()) {
   case VPInstruction::Not: {
-    Value *A = State.get(getOperand(0));
+    bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
+    Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
     return Builder.CreateNot(A, Name);
   }
   case Instruction::ExtractElement: {
@@ -810,10 +813,18 @@ Value *VPInstruction::generate(VPTransformState &State) {
         Value *RdxPart = RdxParts[Part];
         if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK))
           ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
-        else
-          ReducedPartRdx = Builder.CreateBinOp(
-              (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK),
-              RdxPart, ReducedPartRdx, "bin.rdx");
+        else {
+          Instruction::BinaryOps Opcode;
+          // For sub-recurrences, each UF's reduction variable is already
+          // negative, we need to do: reduce.add(-acc_uf0 + -acc_uf1)
+          if (RK == RecurKind::Sub)
+            Opcode = Instruction::Add;
+          else
+            Opcode =
+                (Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK);
+          ReducedPartRdx =
+              Builder.CreateBinOp(Opcode, RdxPart, ReducedPartRdx, "bin.rdx");
+        }
       }
     }
 
@@ -922,33 +933,97 @@ Value *VPInstruction::generate(VPTransformState &State) {
 
     return Res;
   }
+  case VPInstruction::ResumeForEpilogue:
+    return State.get(getOperand(0), true);
   default:
     llvm_unreachable("Unsupported opcode for instruction");
   }
 }
 
+std::optional<InstructionCost> VPRecipeWithIRFlags::getCostForRecipeWithOpcode(
+    unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const {
+  Type *ScalarTy = Ctx.Types.inferScalarType(this);
+  Type *ResultTy = VF.isVector() ? toVectorTy(ScalarTy, VF) : ScalarTy;
+  switch (Opcode) {
+  case Instruction::FNeg:
+    return Ctx.TTI.getArithmeticInstrCost(Opcode, ResultTy, Ctx.CostKind);
+  case Instruction::UDiv:
+  case Instruction::SDiv:
+  case Instruction::SRem:
+  case Instruction::URem:
+  case Instruction::Add:
+  case Instruction::FAdd:
+  case Instruction::Sub:
+  case Instruction::FSub:
+  case Instruction::Mul:
+  case Instruction::FMul:
+  case Instruction::FDiv:
+  case Instruction::FRem:
+  case Instruction::Shl:
+  case Instruction::LShr:
+  case Instruction::AShr:
+  case Instruction::And:
+  case Instruction::Or:
+  case Instruction::Xor: {
+    TargetTransformInfo::OperandValueInfo RHSInfo = {
+        TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None};
+
+    if (VF.isVector()) {
+      // Certain instructions can be cheaper to vectorize if they have a
+      // constant second vector operand. One example of this are shifts on x86.
+      VPValue *RHS = getOperand(1);
+      RHSInfo = Ctx.getOperandInfo(RHS);
+
+      if (RHSInfo.Kind == TargetTransformInfo::OK_AnyValue &&
+          getOperand(1)->isDefinedOutsideLoopRegions())
+        RHSInfo.Kind = TargetTransformInfo::OK_UniformValue;
+    }
+
+    Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+    SmallVector<const Value *, 4> Operands;
+    if (CtxI)
+      Operands.append(CtxI->value_op_begin(), CtxI->value_op_end());
+    return Ctx.TTI.getArithmeticInstrCost(
+        Opcode, ResultTy, Ctx.CostKind,
+        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
+        RHSInfo, Operands, CtxI, &Ctx.TLI);
+  }
+  case Instruction::Freeze:
+    // This opcode is unknown. Assume that it is the same as 'mul'.
+    return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, ResultTy,
+                                          Ctx.CostKind);
+  case Instruction::ExtractValue:
+    return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,
+                                             Ctx.CostKind);
+  case Instruction::ICmp:
+  case Instruction::FCmp: {
+    Type *ScalarOpTy = Ctx.Types.inferScalarType(getOperand(0));
+    Type *OpTy = VF.isVector() ? toVectorTy(ScalarOpTy, VF) : ScalarOpTy;
+    Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
+    return Ctx.TTI.getCmpSelInstrCost(
+        Opcode, OpTy, CmpInst::makeCmpResultType(OpTy), getPredicate(),
+        Ctx.CostKind, {TTI::OK_AnyValue, TTI::OP_None},
+        {TTI::OK_AnyValue, TTI::OP_None}, CtxI);
+  }
+  }
+  return std::nullopt;
+}
+
 InstructionCost VPInstruction::computeCost(ElementCount VF,
                                            VPCostContext &Ctx) const {
   if (Instruction::isBinaryOp(getOpcode())) {
-    Type *ResTy = Ctx.Types.inferScalarType(this);
-    if (!vputils::onlyFirstLaneUsed(this))
-      ResTy = toVectorTy(ResTy, VF);
-
-    if (!getUnderlyingValue()) {
-      switch (getOpcode()) {
-      case Instruction::FMul:
-        return Ctx.TTI.getArithmeticInstrCost(getOpcode(), ResTy, Ctx.CostKind);
-      default:
-        // TODO: Compute cost for VPInstructions without underlying values once
-        // the legacy cost model has been retired.
-        return 0;
-      }
+    if (!getUnderlyingValue() && getOpcode() != Instruction::FMul) {
+      // TODO: Compute cost for VPInstructions without underlying values once
+      // the legacy cost model has been retired.
+      return 0;
     }
 
     assert(!doesGeneratePerAllLanes() &&
            "Should only generate a vector value or single scalar, not scalars "
            "for all lanes.");
-    return Ctx.TTI.getArithmeticInstrCost(getOpcode(), ResTy, Ctx.CostKind);
+    return *getCostForRecipeWithOpcode(
+        getOpcode(),
+        vputils::onlyFirstLaneUsed(this) ? ElementCount::getFixed(1) : VF, Ctx);
   }
 
   switch (getOpcode()) {
@@ -998,6 +1073,12 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
                                   I32Ty, {Arg0Ty, I32Ty, I1Ty});
     return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
   }
+  case VPInstruction::ExtractLastElement: {
+    // Add on the cost of extracting the element.
+    auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
+    return Ctx.TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
+                                                    VecTy, Ctx.CostKind, 0);
+  }
   case VPInstruction::ExtractPenultimateElement:
     if (VF == ElementCount::getScalable(1))
       return InstructionCost::getInvalid();
@@ -1027,6 +1108,8 @@ bool VPInstruction::isSingleScalar() const {
   switch (getOpcode()) {
   case Instruction::PHI:
   case VPInstruction::ExplicitVectorLength:
+  case VPInstruction::ResumeForEpilogue:
+  case VPInstruction::VScale:
     return true;
   default:
     return isScalarCast();
@@ -1076,6 +1159,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
   case Instruction::FCmp:
   case Instruction::ICmp:
   case Instruction::Select:
+  case Instruction::PHI:
   case VPInstruction::AnyOf:
   case VPInstruction::BuildStructVector:
   case VPInstruction::BuildVector:
@@ -1093,6 +1177,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
   case VPInstruction::WidePtrAdd:
   case VPInstruction::StepVector:
   case VPInstruction::ReductionStartVector:
+  case VPInstruction::VScale:
     return false;
   default:
     return true;
@@ -1116,6 +1201,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
   case Instruction::Select:
   case Instruction::Or:
   case Instruction::Freeze:
+  case VPInstruction::Not:
     // TODO: Cover additional opcodes.
     return vputils::onlyFirstLaneUsed(this);
   case VPInstruction::ActiveLaneMask:
@@ -1251,6 +1337,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::ReductionStartVector:
     O << "reduction-start-vector";
     break;
+  case VPInstruction::ResumeForEpilogue:
+    O << "resume-for-epilogue";
+    break;
   default:
     O << Instruction::getOpcodeName(getOpcode());
   }
@@ -1281,6 +1370,12 @@ void VPInstructionWithType::execute(VPTransformState &State) {
     State.set(this, StepVector);
     break;
   }
+  case VPInstruction::VScale: {
+    Value *VScale = State.Builder.CreateVScale(ResultTy);
+    State.set(this, VScale, true);
+    break;
+  }
+
   default:
     llvm_unreachable("opcode not implemented yet");
   }
@@ -1301,6 +1396,9 @@ void VPInstructionWithType::print(raw_ostream &O, const Twine &Indent,
   case VPInstruction::StepVector:
     O << "step-vector " << *ResultTy;
     break;
+  case VPInstruction::VScale:
+    O << "vscale " << *ResultTy;
+    break;
   default:
     assert(Instruction::isCast(getOpcode()) && "unhandled opcode");
     O << Instruction::getOpcodeName(getOpcode()) << " ";
@@ -1434,12 +1532,12 @@ void VPIRPhi::print(raw_ostream &O, const Twine &Indent,
 
   if (getNumOperands() != 0) {
     O << " (extra operand" << (getNumOperands() > 1 ? "s" : "") << ": ";
-    interleaveComma(
-        enumerate(operands()), O, [this, &O, &SlotTracker](auto Op) {
-          Op.value()->printAsOperand(O, SlotTracker);
-          O << " from ";
-          getParent()->getPredecessors()[Op.index()]->printAsOperand(O);
-        });
+    interleaveComma(incoming_values_and_blocks(), O,
+                    [&O, &SlotTracker](auto Op) {
+                      std::get<0>(Op)->printAsOperand(O, SlotTracker);
+                      O << " from ";
+                      std::get<1>(Op)->printAsOperand(O);
+                    });
     O << ")";
   }
 }
@@ -1461,6 +1559,19 @@ void VPIRMetadata::applyMetadata(Instruction &I) const {
     I.setMetadata(Kind, Node);
 }
 
+void VPIRMetadata::intersect(const VPIRMetadata &Other) {
+  SmallVector<std::pair<unsigned, MDNode *>> MetadataIntersection;
+  for (const auto &[KindA, MDA] : Metadata) {
+    for (const auto &[KindB, MDB] : Other.Metadata) {
+      if (KindA == KindB && MDA == MDB) {
+        MetadataIntersection.emplace_back(KindA, MDA);
+        break;
+      }
+    }
+  }
+  Metadata = std::move(MetadataIntersection);
+}
+
 void VPWidenCallRecipe::execute(VPTransformState &State) {
   assert(State.VF.isVector() && "not widening");
   assert(Variant != nullptr && "Can't create vector function.");
@@ -1995,20 +2106,13 @@ void VPWidenRecipe::execute(VPTransformState &State) {
 InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
                                            VPCostContext &Ctx) const {
   switch (Opcode) {
-  case Instruction::FNeg: {
-    Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);
-    return Ctx.TTI.getArithmeticInstrCost(
-        Opcode, VectorTy, Ctx.CostKind,
-        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
-        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None});
-  }
-
   case Instruction::UDiv:
   case Instruction::SDiv:
   case Instruction::SRem:
   case Instruction::URem:
     // More complex computation, let the legacy cost-model handle this for now.
     return Ctx.getLegacyCost(cast<Instruction>(getUnderlyingValue()), VF);
+  case Instruction::FNeg:
   case Instruction::Add:
   case Instruction::FAdd:
   case Instruction::Sub:
@@ -2022,45 +2126,12 @@ InstructionCost VPWidenRecipe::computeCost(ElementCount VF,
   case Instruction::AShr:
   case Instruction::And:
   case Instruction::Or:
-  case Instruction::Xor: {
-    VPValue *RHS = getOperand(1);
-    // Certain instructions can be cheaper to vectorize if they have a constant
-    // second vector operand. One example of this are shifts on x86.
-    TargetTransformInfo::OperandValueInfo RHSInfo = Ctx.getOperandInfo(RHS);
-
-    if (RHSInfo.Kind == TargetTransformInfo::OK_AnyValue &&
-        getOperand(1)->isDefinedOutsideLoopRegions())
-      RHSInfo.Kind = TargetTransformInfo::OK_UniformValue;
-    Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);
-    Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
-
-    SmallVector<const Value *, 4> Operands;
-    if (CtxI)
-      Operands.append(CtxI->value_op_begin(), CtxI->value_op_end());
-    return Ctx.TTI.getArithmeticInstrCost(
-        Opcode, VectorTy, Ctx.CostKind,
-        {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
-        RHSInfo, Operands, CtxI, &Ctx.TLI);
-  }
-  case Instruction::Freeze: {
-    // This opcode is unknown. Assume that it is the same as 'mul'.
-    Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(this), VF);
-    return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy,
-                                          Ctx.CostKind);
-  }
-  case Instruction::ExtractValue: {
-    return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,
-                                             Ctx.CostKind);
-  }
+  case Instruction::Xor:
+  case Instruction::Freeze:
+  case Instruction::ExtractValue:
   case Instruction::ICmp:
-  case Instruction::FCmp: {
-    Instruction *CtxI = dyn_cast_or_null<Instruction>(getUnderlyingValue());
-    Type *VectorTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
-    return Ctx.TTI.getCmpSelInstrCost(
-        Opcode, VectorTy, CmpInst::makeCmpResultType(VectorTy), getPredicate(),
-        Ctx.CostKind, {TTI::OK_AnyValue, TTI::OP_None},
-        {TTI::OK_AnyValue, TTI::OP_None}, CtxI);
-  }
+  case Instruction::FCmp:
+    return *getCostForRecipeWithOpcode(getOpcode(), VF, Ctx);
   default:
     llvm_unreachable("Unsupported opcode for instruction");
   }
@@ -2859,6 +2930,9 @@ static void scalarizeInstruction(const Instruction *Instr,
   RepRecipe->applyFlags(*Cloned);
   RepRecipe->applyMetadata(*Cloned);
 
+  if (RepRecipe->hasPredicate())
+    cast<CmpInst>(Cloned)->setPredicate(RepRecipe->getPredicate());
+
   if (auto DL = RepRecipe->getDebugLoc())
     State.setDebugLocFrom(DL);
 
@@ -2920,9 +2994,7 @@ bool VPReplicateRecipe::shouldPack() const {
   // VPPredInstPHIRecipe. In this case, also pack the scalar values in a vector.
   return any_of(users(), [](const VPUser *U) {
     if (auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
-      return any_of(PredR->users(), [PredR](const VPUser *U) {
-        return !U->usesScalars(PredR);
-      });
+      return !vputils::onlyScalarValuesUsed(PredR);
     return false;
   });
 }
@@ -2934,8 +3006,6 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
   // transform, avoid computing their cost multiple times for now.
   Ctx.SkipCostComputation.insert(UI);
 
-  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
-  Type *ResultTy = Ctx.Types.inferScalarType(this);
   switch (UI->getOpcode()) {
   case Instruction::GetElementPtr:
     // We mark this instruction as zero-cost because the cost of GEPs in
@@ -2943,6 +3013,25 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
     // is scalarized or not. Therefore, we handle GEPs with the memory
     // instruction cost.
     return 0;
+  case Instruction::Call: {
+    if (!isSingleScalar()) {
+      // TODO: Handle remaining call costs here as well.
+      if (VF.isScalable())
+        return InstructionCost::getInvalid();
+      break;
+    }
+
+    auto *CalledFn =
+        cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
+    if (CalledFn->isIntrinsic())
+      break;
+
+    SmallVector<Type *, 4> Tys;
+    for (VPValue *ArgOp : drop_end(operands()))
+      Tys.push_back(Ctx.Types.inferScalarType(ArgOp));
+    Type *ResultTy = Ctx.Types.inferScalarType(this);
+    return Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);
+  }
   case Instruction::Add:
   case Instruction::Sub:
   case Instruction::FAdd:
@@ -2956,16 +3045,13 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
   case Instruction::AShr:
   case Instruction::And:
   case Instruction::Or:
-  case Instruction::Xor: {
-    auto Op2Info = Ctx.getOperandInfo(getOperand(1));
-    SmallVector<const Value *, 4> Operands(UI->operand_values());
-    return Ctx.TTI.getArithmeticInstrCost(
-               UI->getOpcode(), ResultTy, CostKind,
-               {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
-               Op2Info, Operands, UI, &Ctx.TLI) *
+  case Instruction::Xor:
+  case Instruction::ICmp:
+  case Instruction::FCmp:
+    return *getCostForRecipeWithOpcode(getOpcode(), ElementCount::getFixed(1),
+                                       Ctx) *
            (isSingleScalar() ? 1 : VF.getFixedValue());
   }
-  }
 
   return Ctx.getLegacyCost(UI, VF);
 }
@@ -3097,9 +3183,11 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF,
     // Currently, ARM will use the underlying IR to calculate gather/scatter
     // instruction cost.
     const Value *Ptr = getLoadStorePointerOperand(&Ingredient);
+    Type *PtrTy = toVectorTy(Ptr->getType(), VF);
     assert(!Reverse &&
            "Inconsecutive memory access should not have the order.");
-    return Ctx.TTI.getAddressComputationCost(Ty) +
+    return Ctx.TTI.getAddressComputationCost(PtrTy, nullptr, nullptr,
+                                             Ctx.CostKind) +
            Ctx.TTI.getGatherScatterOpCost(Opcode, Ty, Ptr, IsMasked, Alignment,
                                           Ctx.CostKind, &Ingredient);
   }
@@ -3445,6 +3533,8 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
 //   store <12 x i32> %interleaved.vec              ; Write 4 tuples of R,G,B
 void VPInterleaveRecipe::execute(VPTransformState &State) {
   assert(!State.Lane && "Interleave group being replicated.");
+  assert((!NeedsMaskForGaps || !State.VF.isScalable()) &&
+         "Masking gaps for scalable vectors is not yet supported.");
   const InterleaveGroup<Instruction> *Group = IG;
   Instruction *Instr = Group->getInsertPos();
 
@@ -3501,6 +3591,8 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
     } else
       NewLoad = State.Builder.CreateAlignedLoad(VecTy, ResAddr,
                                                 Group->getAlign(), "wide.vec");
+    applyMetadata(*NewLoad);
+    // TODO: Also manage existing metadata using VPIRMetadata.
     Group->addMetadata(NewLoad);
 
     ArrayRef<VPValue *> VPDefs = definedValues();
@@ -3562,8 +3654,6 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
       createBitMaskForGaps(State.Builder, State.VF.getKnownMinValue(), *Group);
   assert(((MaskForGaps != nullptr) == NeedsMaskForGaps) &&
          "Mismatch between NeedsMaskForGaps and MaskForGaps");
-  assert((!MaskForGaps || !State.VF.isScalable()) &&
-         "masking gaps for scalable vectors is not yet supported.");
   ArrayRef<VPValue *> StoredValues = getStoredValues();
   // Collect the stored vector from each member.
   SmallVector<Value *, 4> StoredVecs;
@@ -3605,6 +3695,8 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
     NewStoreInstr =
         State.Builder.CreateAlignedStore(IVec, ResAddr, Group->getAlign());
 
+  applyMetadata(*NewStoreInstr);
+  // TODO: Also manage existing metadata using VPIRMetadata.
   Group->addMetadata(NewStoreInstr);
 }
 
@@ -3714,18 +3806,7 @@ void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent,
     getOperand(4)->printAsOperand(O, SlotTracker);
   }
 }
-#endif
 
-void VPExpandSCEVRecipe::execute(VPTransformState &State) {
-  assert(!State.Lane && "cannot be used in per-lane");
-  const DataLayout &DL = SE.getDataLayout();
-  SCEVExpander Exp(SE, DL, "induction", /*PreserveLCSSA=*/true);
-  Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),
-                                 &*State.Builder.GetInsertPoint());
-  State.set(this, Res, VPLane(0));
-}
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPExpandSCEVRecipe::print(raw_ostream &O, const Twine &Indent,
                                VPSlotTracker &SlotTracker) const {
   O << Indent << "EMIT ";
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 57f8e32029cbf..e0bf241c73fdd 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -34,6 +34,7 @@
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/TypeSize.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 
 using namespace llvm;
 using namespace VPlanPatternMatch;
@@ -42,7 +43,7 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
     VPlanPtr &Plan,
     function_ref<const InductionDescriptor *(PHINode *)>
         GetIntOrFpInductionDescriptor,
-    ScalarEvolution &SE, const TargetLibraryInfo &TLI) {
+    const TargetLibraryInfo &TLI) {
 
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
       Plan->getVectorLoopRegion());
@@ -63,17 +64,20 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
       Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
 
       VPRecipeBase *NewRecipe = nullptr;
-      if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
-        auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
+      if (auto *PhiR = dyn_cast<VPPhi>(&Ingredient)) {
+        auto *Phi = cast<PHINode>(PhiR->getUnderlyingValue());
         const auto *II = GetIntOrFpInductionDescriptor(Phi);
-        if (!II)
-          continue;
-
-        VPValue *Start = Plan->getOrAddLiveIn(II->getStartValue());
-        VPValue *Step =
-            vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
-        NewRecipe = new VPWidenIntOrFpInductionRecipe(
-            Phi, Start, Step, &Plan->getVF(), *II, Ingredient.getDebugLoc());
+        if (!II) {
+          NewRecipe = new VPWidenPHIRecipe(Phi, nullptr, PhiR->getDebugLoc());
+          for (VPValue *Op : PhiR->operands())
+            NewRecipe->addOperand(Op);
+        } else {
+          VPValue *Start = Plan->getOrAddLiveIn(II->getStartValue());
+          VPValue *Step =
+              vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep());
+          NewRecipe = new VPWidenIntOrFpInductionRecipe(
+              Phi, Start, Step, &Plan->getVF(), *II, Ingredient.getDebugLoc());
+        }
       } else {
         assert(isa<VPInstruction>(&Ingredient) &&
                "only VPInstructions expected here");
@@ -514,10 +518,7 @@ static void removeRedundantCanonicalIVs(VPlan &Plan) {
     // everything WidenNewIV's users need. That is, WidenOriginalIV will
     // generate a vector phi or all users of WidenNewIV demand the first lane
     // only.
-    if (any_of(WidenOriginalIV->users(),
-               [WidenOriginalIV](VPUser *U) {
-                 return !U->usesScalars(WidenOriginalIV);
-               }) ||
+    if (!vputils::onlyScalarValuesUsed(WidenOriginalIV) ||
         vputils::onlyFirstLaneUsed(WidenNewIV)) {
       WidenNewIV->replaceAllUsesWith(WidenOriginalIV);
       WidenNewIV->eraseFromParent();
@@ -550,8 +551,22 @@ void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
     // The recipes in the block are processed in reverse order, to catch chains
     // of dead recipes.
     for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
-      if (isDeadRecipe(R))
+      if (isDeadRecipe(R)) {
         R.eraseFromParent();
+        continue;
+      }
+
+      // Check if R is a dead VPPhi <-> update cycle and remove it.
+      auto *PhiR = dyn_cast<VPPhi>(&R);
+      if (!PhiR || PhiR->getNumOperands() != 2 || PhiR->getNumUsers() != 1)
+        continue;
+      VPValue *Incoming = PhiR->getOperand(1);
+      if (*PhiR->user_begin() != Incoming->getDefiningRecipe() ||
+          Incoming->getNumUsers() != 1)
+        continue;
+      PhiR->replaceAllUsesWith(PhiR->getOperand(0));
+      PhiR->eraseFromParent();
+      Incoming->getDefiningRecipe()->eraseFromParent();
     }
   }
 }
@@ -568,8 +583,7 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
       Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
 
   // Truncate base induction if needed.
-  Type *CanonicalIVType = CanonicalIV->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
+  VPTypeAnalysis TypeInfo(Plan);
   Type *ResultTy = TypeInfo.inferScalarType(BaseIV);
   if (TruncI) {
     Type *TruncTy = TruncI->getType();
@@ -702,7 +716,8 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
 /// Check if \p VPV is an untruncated wide induction, either before or after the
 /// increment. If so return the header IV (before the increment), otherwise
 /// return null.
-static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV) {
+static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV,
+                                                  ScalarEvolution &SE) {
   auto *WideIV = dyn_cast<VPWidenInductionRecipe>(VPV);
   if (WideIV) {
     // VPV itself is a wide induction, separately compute the end value for exit
@@ -728,8 +743,7 @@ static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV) {
     VPValue *IVStep = WideIV->getStepValue();
     switch (ID.getInductionOpcode()) {
     case Instruction::Add:
-      return match(VPV, m_c_Binary<Instruction::Add>(m_Specific(WideIV),
-                                                     m_Specific(IVStep)));
+      return match(VPV, m_c_Add(m_Specific(WideIV), m_Specific(IVStep)));
     case Instruction::FAdd:
       return match(VPV, m_c_Binary<Instruction::FAdd>(m_Specific(WideIV),
                                                       m_Specific(IVStep)));
@@ -740,14 +754,13 @@ static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV) {
       // IVStep will be the negated step of the subtraction. Check if Step == -1
       // * IVStep.
       VPValue *Step;
-      if (!match(VPV,
-                 m_Binary<Instruction::Sub>(m_VPValue(), m_VPValue(Step))) ||
-          !Step->isLiveIn() || !IVStep->isLiveIn())
+      if (!match(VPV, m_Sub(m_VPValue(), m_VPValue(Step))))
         return false;
-      auto *StepCI = dyn_cast<ConstantInt>(Step->getLiveInIRValue());
-      auto *IVStepCI = dyn_cast<ConstantInt>(IVStep->getLiveInIRValue());
-      return StepCI && IVStepCI &&
-             StepCI->getValue() == (-1 * IVStepCI->getValue());
+      const SCEV *IVStepSCEV = vputils::getSCEVExprForVPValue(IVStep, SE);
+      const SCEV *StepSCEV = vputils::getSCEVExprForVPValue(Step, SE);
+      return !isa<SCEVCouldNotCompute>(IVStepSCEV) &&
+             !isa<SCEVCouldNotCompute>(StepSCEV) &&
+             IVStepSCEV == SE.getNegativeSCEV(StepSCEV);
     }
     default:
       return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
@@ -764,7 +777,8 @@ static VPWidenInductionRecipe *getOptimizableIVOf(VPValue *VPV) {
 static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
                                                VPTypeAnalysis &TypeInfo,
                                                VPBlockBase *PredVPBB,
-                                               VPValue *Op) {
+                                               VPValue *Op,
+                                               ScalarEvolution &SE) {
   VPValue *Incoming, *Mask;
   if (!match(Op, m_VPInstruction<VPInstruction::ExtractLane>(
                      m_VPInstruction<VPInstruction::FirstActiveLane>(
@@ -772,7 +786,7 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
                      m_VPValue(Incoming))))
     return nullptr;
 
-  auto *WideIV = getOptimizableIVOf(Incoming);
+  auto *WideIV = getOptimizableIVOf(Incoming, SE);
   if (!WideIV)
     return nullptr;
 
@@ -815,16 +829,14 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
 
 /// Attempts to optimize the induction variable exit values for users in the
 /// exit block coming from the latch in the original scalar loop.
-static VPValue *
-optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
-                               VPBlockBase *PredVPBB, VPValue *Op,
-                               DenseMap<VPValue *, VPValue *> &EndValues) {
+static VPValue *optimizeLatchExitInductionUser(
+    VPlan &Plan, VPTypeAnalysis &TypeInfo, VPBlockBase *PredVPBB, VPValue *Op,
+    DenseMap<VPValue *, VPValue *> &EndValues, ScalarEvolution &SE) {
   VPValue *Incoming;
-  if (!match(Op, m_VPInstruction<VPInstruction::ExtractLastElement>(
-                     m_VPValue(Incoming))))
+  if (!match(Op, m_ExtractLastElement(m_VPValue(Incoming))))
     return nullptr;
 
-  auto *WideIV = getOptimizableIVOf(Incoming);
+  auto *WideIV = getOptimizableIVOf(Incoming, SE);
   if (!WideIV)
     return nullptr;
 
@@ -863,9 +875,10 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
 }
 
 void VPlanTransforms::optimizeInductionExitUsers(
-    VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues) {
+    VPlan &Plan, DenseMap<VPValue *, VPValue *> &EndValues,
+    ScalarEvolution &SE) {
   VPBlockBase *MiddleVPBB = Plan.getMiddleBlock();
-  VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
+  VPTypeAnalysis TypeInfo(Plan);
   for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
     for (VPRecipeBase &R : ExitVPBB->phis()) {
       auto *ExitIRI = cast<VPIRPhi>(&R);
@@ -873,11 +886,12 @@ void VPlanTransforms::optimizeInductionExitUsers(
       for (auto [Idx, PredVPBB] : enumerate(ExitVPBB->getPredecessors())) {
         VPValue *Escape = nullptr;
         if (PredVPBB == MiddleVPBB)
-          Escape = optimizeLatchExitInductionUser(
-              Plan, TypeInfo, PredVPBB, ExitIRI->getOperand(Idx), EndValues);
+          Escape = optimizeLatchExitInductionUser(Plan, TypeInfo, PredVPBB,
+                                                  ExitIRI->getOperand(Idx),
+                                                  EndValues, SE);
         else
           Escape = optimizeEarlyExitInductionUser(Plan, TypeInfo, PredVPBB,
-                                                  ExitIRI->getOperand(Idx));
+                                                  ExitIRI->getOperand(Idx), SE);
         if (Escape)
           ExitIRI->setOperand(Idx, Escape);
       }
@@ -967,10 +981,11 @@ static Value *tryToFoldLiveIns(const VPRecipeBase &R, unsigned Opcode,
     return Folder.FoldGEP(IntegerType::getInt8Ty(TypeInfo.getContext()), Ops[0],
                           Ops[1],
                           cast<VPRecipeWithIRFlags>(R).getGEPNoWrapFlags());
-  case Instruction::InsertElement:
-    return Folder.FoldInsertElement(Ops[0], Ops[1], Ops[2]);
+  // An extract of a live-in is an extract of a broadcast, so return the
+  // broadcasted element.
   case Instruction::ExtractElement:
-    return Folder.FoldExtractElement(Ops[0], Ops[1]);
+    assert(!Ops[0]->getType()->isVectorTy() && "Live-ins should be scalar");
+    return Ops[0];
   }
   return nullptr;
 }
@@ -1038,7 +1053,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
 #ifndef NDEBUG
     // Verify that the cached type info is for both A and its users is still
     // accurate by comparing it to freshly computed types.
-    VPTypeAnalysis TypeInfo2(Plan->getCanonicalIV()->getScalarType());
+    VPTypeAnalysis TypeInfo2(*Plan);
     assert(TypeInfo.inferScalarType(A) == TypeInfo2.inferScalarType(A));
     for (VPUser *U : A->users()) {
       auto *R = cast<VPRecipeBase>(U);
@@ -1093,17 +1108,31 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
       return Def->replaceAllUsesWith(A);
 
     // Try to fold Not into compares by adjusting the predicate in-place.
-    if (isa<VPWidenRecipe>(A) && A->getNumUsers() == 1) {
-      auto *WideCmp = cast<VPWidenRecipe>(A);
-      if (WideCmp->getOpcode() == Instruction::ICmp ||
-          WideCmp->getOpcode() == Instruction::FCmp) {
-        WideCmp->setPredicate(
-            CmpInst::getInversePredicate(WideCmp->getPredicate()));
-        Def->replaceAllUsesWith(WideCmp);
-        // If WideCmp doesn't have a debug location, use the one from the
-        // negation, to preserve the location.
-        if (!WideCmp->getDebugLoc() && R.getDebugLoc())
-          WideCmp->setDebugLoc(R.getDebugLoc());
+    CmpPredicate Pred;
+    if (match(A, m_Cmp(Pred, m_VPValue(), m_VPValue()))) {
+      auto *Cmp = cast<VPRecipeWithIRFlags>(A);
+      if (all_of(Cmp->users(), [&Cmp](VPUser *U) {
+            return match(U, m_CombineOr(m_Not(m_Specific(Cmp)),
+                                        m_Select(m_Specific(Cmp), m_VPValue(),
+                                                 m_VPValue())));
+          })) {
+        Cmp->setPredicate(CmpInst::getInversePredicate(Pred));
+        for (VPUser *U : to_vector(Cmp->users())) {
+          auto *R = cast<VPSingleDefRecipe>(U);
+          if (match(R, m_Select(m_Specific(Cmp), m_VPValue(X), m_VPValue(Y)))) {
+            // select (cmp pred), x, y -> select (cmp inv_pred), y, x
+            R->setOperand(1, Y);
+            R->setOperand(2, X);
+          } else {
+            // not (cmp pred) -> cmp inv_pred
+            assert(match(R, m_Not(m_Specific(Cmp))) && "Unexpected user");
+            R->replaceAllUsesWith(Cmp);
+          }
+        }
+        // If Cmp doesn't have a debug location, use the one from the negation,
+        // to preserve the location.
+        if (!Cmp->getDebugLoc() && R.getDebugLoc())
+          Cmp->setDebugLoc(R.getDebugLoc());
       }
     }
   }
@@ -1144,8 +1173,7 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
   }
 
   // Look through ExtractLastElement (BuildVector ....).
-  if (match(&R, m_VPInstruction<VPInstruction::ExtractLastElement>(
-                    m_BuildVector()))) {
+  if (match(&R, m_ExtractLastElement(m_BuildVector()))) {
     auto *BuildVector = cast<VPInstruction>(R.getOperand(0));
     Def->replaceAllUsesWith(
         BuildVector->getOperand(BuildVector->getNumOperands() - 1));
@@ -1161,6 +1189,12 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
     return;
   }
 
+  if (auto *Phi = dyn_cast<VPPhi>(Def)) {
+    if (Phi->getNumOperands() == 1)
+      Phi->replaceAllUsesWith(Phi->getOperand(0));
+    return;
+  }
+
   // Some simplifications can only be applied after unrolling. Perform them
   // below.
   if (!Plan->isUnrolled())
@@ -1193,26 +1227,23 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
     return;
   }
 
-  if (match(Def,
-            m_VPInstruction<VPInstruction::ExtractLastElement>(
-                m_VPInstruction<VPInstruction::Broadcast>(m_VPValue(A))))) {
+  if (match(Def, m_ExtractLastElement(m_Broadcast(m_VPValue(A))))) {
     Def->replaceAllUsesWith(A);
     return;
   }
 
   VPInstruction *OpVPI;
-  if (match(Def, m_VPInstruction<VPInstruction::ExtractLastElement>(
-                     m_VPInstruction(OpVPI))) &&
+  if (match(Def, m_ExtractLastElement(m_VPInstruction(OpVPI))) &&
       OpVPI->isVectorToScalar()) {
     Def->replaceAllUsesWith(OpVPI);
     return;
   }
 }
 
-void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
+void VPlanTransforms::simplifyRecipes(VPlan &Plan) {
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
       Plan.getEntry());
-  VPTypeAnalysis TypeInfo(&CanonicalIVTy);
+  VPTypeAnalysis TypeInfo(Plan);
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
       simplifyRecipe(R, TypeInfo);
@@ -1242,9 +1273,7 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
       // scalar results used. In the latter case, we would introduce extra
       // broadcasts.
       if (!vputils::isSingleScalar(RepOrWidenR) ||
-          any_of(RepOrWidenR->users(), [RepOrWidenR](VPUser *U) {
-            return !U->usesScalars(RepOrWidenR);
-          }))
+          !vputils::onlyScalarValuesUsed(RepOrWidenR))
         continue;
 
       auto *Clone = new VPReplicateRecipe(RepOrWidenR->getUnderlyingInstr(),
@@ -1382,11 +1411,10 @@ static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
 
     // Currently only handle cases where the single user is a header-mask
     // comparison with the backedge-taken-count.
-    if (!match(
-            *WideIV->user_begin(),
-            m_Binary<Instruction::ICmp>(
-                m_Specific(WideIV),
-                m_Broadcast(m_Specific(Plan.getOrCreateBackedgeTakenCount())))))
+    if (!match(*WideIV->user_begin(),
+               m_ICmp(m_Specific(WideIV),
+                      m_Broadcast(
+                          m_Specific(Plan.getOrCreateBackedgeTakenCount())))))
       continue;
 
     // Update IV operands and comparison bound to use new narrower type.
@@ -1419,11 +1447,9 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
     });
 
   auto *CanIV = Plan.getCanonicalIV();
-  if (!match(Cond, m_Binary<Instruction::ICmp>(
-                       m_Specific(CanIV->getBackedgeValue()),
-                       m_Specific(&Plan.getVectorTripCount()))) ||
-      cast<VPRecipeWithIRFlags>(Cond->getDefiningRecipe())->getPredicate() !=
-          CmpInst::ICMP_EQ)
+  if (!match(Cond, m_SpecificICmp(CmpInst::ICMP_EQ,
+                                  m_Specific(CanIV->getBackedgeValue()),
+                                  m_Specific(&Plan.getVectorTripCount()))))
     return false;
 
   // The compare checks CanIV + VFxUF == vector trip count. The vector trip
@@ -1478,12 +1504,25 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
   // The vector loop region only executes once. If possible, completely remove
   // the region, otherwise replace the terminator controlling the latch with
   // (BranchOnCond true).
+  // TODO: VPWidenIntOrFpInductionRecipe is only partially supported; add
+  // support for other non-canonical widen induction recipes (e.g.,
+  // VPWidenPointerInductionRecipe).
   auto *Header = cast<VPBasicBlock>(VectorRegion->getEntry());
-  auto *CanIVTy = Plan.getCanonicalIV()->getScalarType();
-  if (all_of(Header->phis(),
-             IsaPred<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe,
-                     VPFirstOrderRecurrencePHIRecipe, VPPhi>)) {
+  if (all_of(Header->phis(), [](VPRecipeBase &Phi) {
+        if (auto *R = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi))
+          return R->isCanonical();
+        return isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe,
+                   VPFirstOrderRecurrencePHIRecipe, VPPhi>(&Phi);
+      })) {
     for (VPRecipeBase &HeaderR : make_early_inc_range(Header->phis())) {
+      if (auto *R = dyn_cast<VPWidenIntOrFpInductionRecipe>(&HeaderR)) {
+        VPBuilder Builder(Plan.getVectorPreheader());
+        VPValue *StepV = Builder.createNaryOp(VPInstruction::StepVector, {},
+                                              R->getScalarType());
+        HeaderR.getVPSingleValue()->replaceAllUsesWith(StepV);
+        HeaderR.eraseFromParent();
+        continue;
+      }
       auto *Phi = cast<VPPhiAccessors>(&HeaderR);
       HeaderR.getVPSingleValue()->replaceAllUsesWith(Phi->getIncomingValue(0));
       HeaderR.eraseFromParent();
@@ -1499,7 +1538,7 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
 
     VPBlockUtils::connectBlocks(Preheader, Header);
     VPBlockUtils::connectBlocks(ExitingVPBB, Exit);
-    VPlanTransforms::simplifyRecipes(Plan, *CanIVTy);
+    VPlanTransforms::simplifyRecipes(Plan);
   } else {
     // The vector region contains header phis for which we cannot remove the
     // loop region yet.
@@ -1742,7 +1781,8 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
     if (!PhiR)
       continue;
     RecurKind RK = PhiR->getRecurrenceKind();
-    if (RK != RecurKind::Add && RK != RecurKind::Mul)
+    if (RK != RecurKind::Add && RK != RecurKind::Mul && RK != RecurKind::Sub &&
+        RK != RecurKind::AddChainWithSubs)
       continue;
 
     for (VPUser *U : collectUsersRecursively(PhiR))
@@ -1793,8 +1833,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
   // other uses have different types for their operands, making them invalidly
   // typed.
   DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
-  Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
+  VPTypeAnalysis TypeInfo(Plan);
   VPBasicBlock *PH = Plan.getVectorPreheader();
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_deep(Plan.getVectorLoopRegion()))) {
@@ -1822,8 +1861,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
       assert(OldResTy->isIntegerTy() && "only integer types supported");
       (void)OldResSizeInBits;
 
-      LLVMContext &Ctx = CanonicalIVType->getContext();
-      auto *NewResTy = IntegerType::get(Ctx, NewResSizeInBits);
+      auto *NewResTy = IntegerType::get(Plan.getContext(), NewResSizeInBits);
 
       // Any wrapping introduced by shrinking this operation shouldn't be
       // considered undefined behavior. So, we can't unconditionally copy
@@ -1832,7 +1870,7 @@ void VPlanTransforms::truncateToMinimalBitwidths(
         VPW->dropPoisonGeneratingFlags();
 
       if (OldResSizeInBits != NewResSizeInBits &&
-          !match(&R, m_Binary<Instruction::ICmp>(m_VPValue(), m_VPValue()))) {
+          !match(&R, m_ICmp(m_VPValue(), m_VPValue()))) {
         // Extend result to original width.
         auto *Ext =
             new VPWidenCastRecipe(Instruction::ZExt, ResultVPV, OldResTy);
@@ -1841,9 +1879,8 @@ void VPlanTransforms::truncateToMinimalBitwidths(
         Ext->setOperand(0, ResultVPV);
         assert(OldResSizeInBits > NewResSizeInBits && "Nothing to shrink?");
       } else {
-        assert(
-            match(&R, m_Binary<Instruction::ICmp>(m_VPValue(), m_VPValue())) &&
-            "Only ICmps should not need extending the result.");
+        assert(match(&R, m_ICmp(m_VPValue(), m_VPValue())) &&
+               "Only ICmps should not need extending the result.");
       }
 
       assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
@@ -1915,13 +1952,13 @@ void VPlanTransforms::optimize(VPlan &Plan) {
   runPass(removeRedundantCanonicalIVs, Plan);
   runPass(removeRedundantInductionCasts, Plan);
 
-  runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType());
+  runPass(simplifyRecipes, Plan);
   runPass(simplifyBlends, Plan);
   runPass(removeDeadRecipes, Plan);
   runPass(narrowToSingleScalarRecipes, Plan);
   runPass(legalizeAndOptimizeInductions, Plan);
   runPass(removeRedundantExpandSCEVRecipes, Plan);
-  runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType());
+  runPass(simplifyRecipes, Plan);
   runPass(removeBranchOnConst, Plan);
   runPass(removeDeadRecipes, Plan);
 
@@ -2034,11 +2071,11 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
   return LaneMaskPhi;
 }
 
-/// Collect all VPValues representing a header mask through the (ICMP_ULE,
-/// WideCanonicalIV, backedge-taken-count) pattern.
+/// Collect the header mask with the pattern:
+///   (ICMP_ULE, WideCanonicalIV, backedge-taken-count)
 /// TODO: Introduce explicit recipe for header-mask instead of searching
 /// for the header-mask pattern manually.
-static SmallVector<VPValue *> collectAllHeaderMasks(VPlan &Plan) {
+static VPSingleDefRecipe *findHeaderMask(VPlan &Plan) {
   SmallVector<VPValue *> WideCanonicalIVs;
   auto *FoundWidenCanonicalIVUser =
       find_if(Plan.getCanonicalIV()->users(),
@@ -2062,21 +2099,22 @@ static SmallVector<VPValue *> collectAllHeaderMasks(VPlan &Plan) {
       WideCanonicalIVs.push_back(WidenOriginalIV);
   }
 
-  // Walk users of wide canonical IVs and collect to all compares of the form
+  // Walk users of wide canonical IVs and find the single compare of the form
   // (ICMP_ULE, WideCanonicalIV, backedge-taken-count).
-  SmallVector<VPValue *> HeaderMasks;
+  VPSingleDefRecipe *HeaderMask = nullptr;
   for (auto *Wide : WideCanonicalIVs) {
     for (VPUser *U : SmallVector<VPUser *>(Wide->users())) {
-      auto *HeaderMask = dyn_cast<VPInstruction>(U);
-      if (!HeaderMask || !vputils::isHeaderMask(HeaderMask, Plan))
+      auto *VPI = dyn_cast<VPInstruction>(U);
+      if (!VPI || !vputils::isHeaderMask(VPI, Plan))
         continue;
 
-      assert(HeaderMask->getOperand(0) == Wide &&
+      assert(VPI->getOperand(0) == Wide &&
              "WidenCanonicalIV must be the first operand of the compare");
-      HeaderMasks.push_back(HeaderMask);
+      assert(!HeaderMask && "Multiple header masks found?");
+      HeaderMask = VPI;
     }
   }
-  return HeaderMasks;
+  return HeaderMask;
 }
 
 void VPlanTransforms::addActiveLaneMask(
@@ -2092,6 +2130,7 @@ void VPlanTransforms::addActiveLaneMask(
               [](VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); });
   assert(FoundWidenCanonicalIVUser &&
          "Must have widened canonical IV when tail folding!");
+  VPSingleDefRecipe *HeaderMask = findHeaderMask(Plan);
   auto *WideCanonicalIV =
       cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
   VPSingleDefRecipe *LaneMask;
@@ -2105,11 +2144,11 @@ void VPlanTransforms::addActiveLaneMask(
                               "active.lane.mask");
   }
 
-  // Walk users of WideCanonicalIV and replace all compares of the form
-  // (ICMP_ULE, WideCanonicalIV, backedge-taken-count) with an
-  // active-lane-mask.
-  for (VPValue *HeaderMask : collectAllHeaderMasks(Plan))
-    HeaderMask->replaceAllUsesWith(LaneMask);
+  // Walk users of WideCanonicalIV and replace the header mask of the form
+  // (ICMP_ULE, WideCanonicalIV, backedge-taken-count) with an active-lane-mask,
+  // removing the old one to ensure there is always only a single header mask.
+  HeaderMask->replaceAllUsesWith(LaneMask);
+  HeaderMask->eraseFromParent();
 }
 
 /// Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
@@ -2125,6 +2164,8 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
                                        VPRecipeBase &CurRecipe,
                                        VPTypeAnalysis &TypeInfo,
                                        VPValue &AllOneMask, VPValue &EVL) {
+  // FIXME: Don't transform recipes to EVL recipes if they're not masked by the
+  // header mask.
   auto GetNewMask = [&](VPValue *OrigMask) -> VPValue * {
     assert(OrigMask && "Unmasked recipe when folding tail");
     // HeaderMask will be handled using EVL.
@@ -2134,14 +2175,35 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
     return HeaderMask == OrigMask ? nullptr : OrigMask;
   };
 
+  /// Adjust any end pointers so that they point to the end of EVL lanes not VF.
+  auto GetNewAddr = [&CurRecipe, &EVL](VPValue *Addr) -> VPValue * {
+    auto *EndPtr = dyn_cast<VPVectorEndPointerRecipe>(Addr);
+    if (!EndPtr)
+      return Addr;
+    assert(EndPtr->getOperand(1) == &EndPtr->getParent()->getPlan()->getVF() &&
+           "VPVectorEndPointerRecipe with non-VF VF operand?");
+    assert(
+        all_of(EndPtr->users(),
+               [](VPUser *U) {
+                 return cast<VPWidenMemoryRecipe>(U)->isReverse();
+               }) &&
+        "VPVectorEndPointRecipe not used by reversed widened memory recipe?");
+    VPVectorEndPointerRecipe *EVLAddr = EndPtr->clone();
+    EVLAddr->insertBefore(&CurRecipe);
+    EVLAddr->setOperand(1, &EVL);
+    return EVLAddr;
+  };
+
   return TypeSwitch<VPRecipeBase *, VPRecipeBase *>(&CurRecipe)
       .Case<VPWidenLoadRecipe>([&](VPWidenLoadRecipe *L) {
         VPValue *NewMask = GetNewMask(L->getMask());
-        return new VPWidenLoadEVLRecipe(*L, EVL, NewMask);
+        VPValue *NewAddr = GetNewAddr(L->getAddr());
+        return new VPWidenLoadEVLRecipe(*L, NewAddr, EVL, NewMask);
       })
       .Case<VPWidenStoreRecipe>([&](VPWidenStoreRecipe *S) {
         VPValue *NewMask = GetNewMask(S->getMask());
-        return new VPWidenStoreEVLRecipe(*S, EVL, NewMask);
+        VPValue *NewAddr = GetNewAddr(S->getAddr());
+        return new VPWidenStoreEVLRecipe(*S, NewAddr, EVL, NewMask);
       })
       .Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
         VPValue *NewMask = GetNewMask(Red->getCondOp());
@@ -2167,9 +2229,7 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
 
 /// Replace recipes with their EVL variants.
 static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
-  Type *CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
-  LLVMContext &Ctx = CanonicalIVType->getContext();
+  VPTypeAnalysis TypeInfo(Plan);
   VPValue *AllOneMask = Plan.getTrue();
   VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
   VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
@@ -2178,7 +2238,23 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
                 IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
                         VPWidenIntOrFpInductionRecipe>) &&
          "User of VF that we can't transform to EVL.");
-  Plan.getVF().replaceAllUsesWith(&EVL);
+  Plan.getVF().replaceUsesWithIf(&EVL, [](VPUser &U, unsigned Idx) {
+    return isa<VPWidenIntOrFpInductionRecipe, VPScalarIVStepsRecipe>(U);
+  });
+
+  assert(all_of(Plan.getVFxUF().users(),
+                [&Plan](VPUser *U) {
+                  return match(U, m_c_Add(m_Specific(Plan.getCanonicalIV()),
+                                          m_Specific(&Plan.getVFxUF()))) ||
+                         isa<VPWidenPointerInductionRecipe>(U);
+                }) &&
+         "Only users of VFxUF should be VPWidenPointerInductionRecipe and the "
+         "increment of the canonical induction.");
+  Plan.getVFxUF().replaceUsesWithIf(&EVL, [](VPUser &U, unsigned Idx) {
+    // Only replace uses in VPWidenPointerInductionRecipe; The increment of the
+    // canonical induction must not be updated.
+    return isa<VPWidenPointerInductionRecipe>(U);
+  });
 
   // Defer erasing recipes till the end so that we don't invalidate the
   // VPTypeAnalysis cache.
@@ -2193,9 +2269,9 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
     VPValue *MaxEVL = &Plan.getVF();
     // Emit VPScalarCastRecipe in preheader if VF is not a 32 bits integer.
     VPBuilder Builder(LoopRegion->getPreheaderVPBB());
-    MaxEVL = Builder.createScalarZExtOrTrunc(MaxEVL, Type::getInt32Ty(Ctx),
-                                             TypeInfo.inferScalarType(MaxEVL),
-                                             DebugLoc());
+    MaxEVL = Builder.createScalarZExtOrTrunc(
+        MaxEVL, Type::getInt32Ty(Plan.getContext()),
+        TypeInfo.inferScalarType(MaxEVL), DebugLoc());
 
     Builder.setInsertPoint(Header, Header->getFirstNonPhi());
     VPValue *PrevEVL =
@@ -2210,7 +2286,7 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
                        m_VPValue(V1), m_VPValue(V2))))
           continue;
         VPValue *Imm = Plan.getOrAddLiveIn(
-            ConstantInt::getSigned(Type::getInt32Ty(Ctx), -1));
+            ConstantInt::getSigned(Type::getInt32Ty(Plan.getContext()), -1));
         VPWidenIntrinsicRecipe *VPSplice = new VPWidenIntrinsicRecipe(
             Intrinsic::experimental_vp_splice,
             {V1, V2, Imm, AllOneMask, PrevEVL, &EVL},
@@ -2222,47 +2298,51 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
     }
   }
 
+  VPValue *HeaderMask = findHeaderMask(Plan);
+  if (!HeaderMask)
+    return;
+
+  // Replace header masks with a mask equivalent to predicating by EVL:
+  //
+  // icmp ule widen-canonical-iv backedge-taken-count
+  // ->
+  // icmp ult step-vector, EVL
+  VPRecipeBase *EVLR = EVL.getDefiningRecipe();
+  VPBuilder Builder(EVLR->getParent(), std::next(EVLR->getIterator()));
+  Type *EVLType = TypeInfo.inferScalarType(&EVL);
+  VPValue *EVLMask = Builder.createICmp(
+      CmpInst::ICMP_ULT,
+      Builder.createNaryOp(VPInstruction::StepVector, {}, EVLType), &EVL);
+  HeaderMask->replaceAllUsesWith(EVLMask);
+  ToErase.push_back(HeaderMask->getDefiningRecipe());
+
   // Try to optimize header mask recipes away to their EVL variants.
-  for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {
-    // TODO: Split optimizeMaskToEVL out and move into
-    // VPlanTransforms::optimize. transformRecipestoEVLRecipes should be run in
-    // tryToBuildVPlanWithVPRecipes beforehand.
-    for (VPUser *U : collectUsersRecursively(HeaderMask)) {
-      auto *CurRecipe = cast<VPRecipeBase>(U);
-      VPRecipeBase *EVLRecipe =
-          optimizeMaskToEVL(HeaderMask, *CurRecipe, TypeInfo, *AllOneMask, EVL);
-      if (!EVLRecipe)
-        continue;
+  // TODO: Split optimizeMaskToEVL out and move into
+  // VPlanTransforms::optimize. transformRecipestoEVLRecipes should be run in
+  // tryToBuildVPlanWithVPRecipes beforehand.
+  for (VPUser *U : collectUsersRecursively(EVLMask)) {
+    auto *CurRecipe = cast<VPRecipeBase>(U);
+    VPRecipeBase *EVLRecipe =
+        optimizeMaskToEVL(EVLMask, *CurRecipe, TypeInfo, *AllOneMask, EVL);
+    if (!EVLRecipe)
+      continue;
 
-      [[maybe_unused]] unsigned NumDefVal = EVLRecipe->getNumDefinedValues();
-      assert(NumDefVal == CurRecipe->getNumDefinedValues() &&
-             "New recipe must define the same number of values as the "
-             "original.");
-      assert(
-          NumDefVal <= 1 &&
-          "Only supports recipes with a single definition or without users.");
-      EVLRecipe->insertBefore(CurRecipe);
-      if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(EVLRecipe)) {
-        VPValue *CurVPV = CurRecipe->getVPSingleValue();
-        CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue());
-      }
-      ToErase.push_back(CurRecipe);
+    [[maybe_unused]] unsigned NumDefVal = EVLRecipe->getNumDefinedValues();
+    assert(NumDefVal == CurRecipe->getNumDefinedValues() &&
+           "New recipe must define the same number of values as the "
+           "original.");
+    assert(NumDefVal <= 1 &&
+           "Only supports recipes with a single definition or without users.");
+    EVLRecipe->insertBefore(CurRecipe);
+    if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(EVLRecipe)) {
+      VPValue *CurVPV = CurRecipe->getVPSingleValue();
+      CurVPV->replaceAllUsesWith(EVLRecipe->getVPSingleValue());
     }
-
-    // Replace header masks with a mask equivalent to predicating by EVL:
-    //
-    // icmp ule widen-canonical-iv backedge-taken-count
-    // ->
-    // icmp ult step-vector, EVL
-    VPRecipeBase *EVLR = EVL.getDefiningRecipe();
-    VPBuilder Builder(EVLR->getParent(), std::next(EVLR->getIterator()));
-    Type *EVLType = TypeInfo.inferScalarType(&EVL);
-    VPValue *EVLMask = Builder.createICmp(
-        CmpInst::ICMP_ULT,
-        Builder.createNaryOp(VPInstruction::StepVector, {}, EVLType), &EVL);
-    HeaderMask->replaceAllUsesWith(EVLMask);
-    ToErase.push_back(HeaderMask->getDefiningRecipe());
+    ToErase.push_back(CurRecipe);
   }
+  // Remove dead EVL mask.
+  if (EVLMask->getNumUsers() == 0)
+    ToErase.push_back(EVLMask->getDefiningRecipe());
 
   for (VPRecipeBase *R : reverse(ToErase)) {
     SmallVector<VPValue *> PossiblyDead(R->operands());
@@ -2315,16 +2395,9 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
 /// %NextAVL = sub IVSize nuw %AVL, %OpEVL
 /// ...
 ///
-bool VPlanTransforms::tryAddExplicitVectorLength(
+void VPlanTransforms::addExplicitVectorLength(
     VPlan &Plan, const std::optional<unsigned> &MaxSafeElements) {
   VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
-  // The transform updates all users of inductions to work based on EVL, instead
-  // of the VF directly. At the moment, widened pointer inductions cannot be
-  // updated, so bail out if the plan contains any.
-  bool ContainsWidenPointerInductions =
-      any_of(Header->phis(), IsaPred<VPWidenPointerInductionRecipe>);
-  if (ContainsWidenPointerInductions)
-    return false;
 
   auto *CanonicalIVPHI = Plan.getCanonicalIV();
   auto *CanIVTy = CanonicalIVPHI->getScalarType();
@@ -2355,7 +2428,7 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
   Builder.setInsertPoint(CanonicalIVIncrement);
   VPValue *OpVPEVL = VPEVL;
 
-  auto *I32Ty = Type::getInt32Ty(CanIVTy->getContext());
+  auto *I32Ty = Type::getInt32Ty(Plan.getContext());
   OpVPEVL = Builder.createScalarZExtOrTrunc(
       OpVPEVL, CanIVTy, I32Ty, CanonicalIVIncrement->getDebugLoc());
 
@@ -2379,7 +2452,6 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
   CanonicalIVIncrement->setOperand(0, CanonicalIVPHI);
   // TODO: support unroll factor > 1.
   Plan.setUF(1);
-  return true;
 }
 
 void VPlanTransforms::canonicalizeEVLLoops(VPlan &Plan) {
@@ -2412,9 +2484,8 @@ void VPlanTransforms::canonicalizeEVLLoops(VPlan &Plan) {
   // Replace CanonicalIVInc with EVL-PHI increment.
   auto *CanonicalIV = cast<VPPhi>(&*HeaderVPBB->begin());
   VPValue *Backedge = CanonicalIV->getIncomingValue(1);
-  assert(match(Backedge,
-               m_c_Binary<Instruction::Add>(m_Specific(CanonicalIV),
-                                            m_Specific(&Plan.getVFxUF()))) &&
+  assert(match(Backedge, m_c_Add(m_Specific(CanonicalIV),
+                                 m_Specific(&Plan.getVFxUF()))) &&
          "Unexpected canonical iv");
   Backedge->replaceAllUsesWith(EVLIncrement);
 
@@ -2552,12 +2623,22 @@ void VPlanTransforms::createInterleaveGroups(
   VPDominatorTree VPDT;
   VPDT.recalculate(Plan);
   for (const auto *IG : InterleaveGroups) {
+    auto *Start =
+        cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(IG->getMember(0)));
+    VPIRMetadata InterleaveMD(*Start);
     SmallVector<VPValue *, 4> StoredValues;
-    for (unsigned i = 0; i < IG->getFactor(); ++i)
-      if (auto *SI = dyn_cast_or_null<StoreInst>(IG->getMember(i))) {
-        auto *StoreR = cast<VPWidenStoreRecipe>(RecipeBuilder.getRecipe(SI));
+    if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Start))
+      StoredValues.push_back(StoreR->getStoredValue());
+    for (unsigned I = 1; I < IG->getFactor(); ++I) {
+      Instruction *MemberI = IG->getMember(I);
+      if (!MemberI)
+        continue;
+      VPWidenMemoryRecipe *MemoryR =
+          cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(MemberI));
+      if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(MemoryR))
         StoredValues.push_back(StoreR->getStoredValue());
-      }
+      InterleaveMD.intersect(*MemoryR);
+    }
 
     bool NeedsMaskForGaps =
         (IG->requiresScalarEpilogue() && !ScalarEpilogueAllowed) ||
@@ -2567,14 +2648,12 @@ void VPlanTransforms::createInterleaveGroups(
     auto *InsertPos =
         cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(IRInsertPos));
 
-    bool InBounds = false;
+    GEPNoWrapFlags NW = GEPNoWrapFlags::none();
     if (auto *Gep = dyn_cast<GetElementPtrInst>(
             getLoadStorePointerOperand(IRInsertPos)->stripPointerCasts()))
-      InBounds = Gep->isInBounds();
+      NW = Gep->getNoWrapFlags().withoutNoUnsignedWrap();
 
     // Get or create the start address for the interleave group.
-    auto *Start =
-        cast<VPWidenMemoryRecipe>(RecipeBuilder.getRecipe(IG->getMember(0)));
     VPValue *Addr = Start->getAddr();
     VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
     if (AddrDef && !VPDT.properlyDominates(AddrDef, InsertPos)) {
@@ -2594,8 +2673,7 @@ void VPlanTransforms::createInterleaveGroups(
       VPValue *OffsetVPV =
           Plan.getOrAddLiveIn(ConstantInt::get(Plan.getContext(), -Offset));
       VPBuilder B(InsertPos);
-      Addr = InBounds ? B.createInBoundsPtrAdd(InsertPos->getAddr(), OffsetVPV)
-                      : B.createPtrAdd(InsertPos->getAddr(), OffsetVPV);
+      Addr = B.createNoWrapPtrAdd(InsertPos->getAddr(), OffsetVPV, NW);
     }
     // If the group is reverse, adjust the index to refer to the last vector
     // lane instead of the first. We adjust the index from the first vector
@@ -2604,14 +2682,13 @@ void VPlanTransforms::createInterleaveGroups(
     if (IG->isReverse()) {
       auto *ReversePtr = new VPVectorEndPointerRecipe(
           Addr, &Plan.getVF(), getLoadStoreType(IRInsertPos),
-          -(int64_t)IG->getFactor(),
-          InBounds ? GEPNoWrapFlags::inBounds() : GEPNoWrapFlags::none(),
-          InsertPos->getDebugLoc());
+          -(int64_t)IG->getFactor(), NW, InsertPos->getDebugLoc());
       ReversePtr->insertBefore(InsertPos);
       Addr = ReversePtr;
     }
     auto *VPIG = new VPInterleaveRecipe(IG, Addr, StoredValues,
-                                        InsertPos->getMask(), NeedsMaskForGaps, InsertPos->getDebugLoc());
+                                        InsertPos->getMask(), NeedsMaskForGaps,
+                                        InterleaveMD, InsertPos->getDebugLoc());
     VPIG->insertBefore(InsertPos);
 
     unsigned J = 0;
@@ -2699,7 +2776,7 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR,
 
   // Construct the initial value of the vector IV in the vector loop preheader.
   Type *IVIntTy =
-      IntegerType::get(StepTy->getContext(), StepTy->getScalarSizeInBits());
+      IntegerType::get(Plan->getContext(), StepTy->getScalarSizeInBits());
   VPValue *Init = Builder.createNaryOp(VPInstruction::StepVector, {}, IVIntTy);
   if (StepTy->isFloatingPointTy())
     Init = Builder.createWidenCast(Instruction::UIToFP, Init, StepTy);
@@ -2803,13 +2880,12 @@ static void expandVPWidenPointerInduction(VPWidenPointerInductionRecipe *R,
   R->replaceAllUsesWith(PtrAdd);
 
   // Create the backedge value for the scalar pointer phi.
-  Builder.setInsertPoint(R->getParent(), R->getParent()->getFirstNonPhi());
+  VPBasicBlock *ExitingBB = Plan->getVectorLoopRegion()->getExitingBasicBlock();
+  Builder.setInsertPoint(ExitingBB, ExitingBB->getTerminator()->getIterator());
   VF = Builder.createScalarZExtOrTrunc(VF, StepTy, TypeInfo.inferScalarType(VF),
                                        DL);
   VPValue *Inc = Builder.createNaryOp(Instruction::Mul, {Step, VF});
 
-  VPBasicBlock *ExitingBB = Plan->getVectorLoopRegion()->getExitingBasicBlock();
-  Builder.setInsertPoint(ExitingBB, ExitingBB->getTerminator()->getIterator());
   VPValue *InductionGEP =
       Builder.createPtrAdd(ScalarPtrPhi, Inc, DL, "ptr.ind");
   ScalarPtrPhi->addOperand(InductionGEP);
@@ -2827,9 +2903,8 @@ void VPlanTransforms::dissolveLoopRegions(VPlan &Plan) {
     R->dissolveToCFGLoop();
 }
 
-void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan,
-                                               Type &CanonicalIVTy) {
-  VPTypeAnalysis TypeInfo(&CanonicalIVTy);
+void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
+  VPTypeAnalysis TypeInfo(Plan);
   SmallVector<VPRecipeBase *> ToRemove;
   for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
            vp_depth_first_deep(Plan.getEntry()))) {
@@ -3193,8 +3268,7 @@ void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
 
   auto *VectorPreheader = Plan.getVectorPreheader();
   for (VPValue *VPV : VPValues) {
-    if (all_of(VPV->users(),
-               [VPV](VPUser *U) { return U->usesScalars(VPV); }) ||
+    if (vputils::onlyScalarValuesUsed(VPV) ||
         (VPV->isLiveIn() && VPV->getLiveInIRValue() &&
          isa<Constant>(VPV->getLiveInIRValue())))
       continue;
@@ -3267,6 +3341,174 @@ void VPlanTransforms::materializeBackedgeTakenCount(VPlan &Plan,
   BTC->replaceAllUsesWith(TCMO);
 }
 
+void VPlanTransforms::materializeBuildVectors(VPlan &Plan) {
+  if (Plan.hasScalarVFOnly())
+    return;
+
+  VPTypeAnalysis TypeInfo(Plan);
+  VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+  auto VPBBsOutsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(
+      vp_depth_first_shallow(Plan.getEntry()));
+  auto VPBBsInsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(
+      vp_depth_first_shallow(LoopRegion->getEntry()));
+  // Materialize Build(Struct)Vector for all replicating VPReplicateRecipes,
+  // excluding ones in replicate regions. Those are not materialized explicitly
+  // yet. Those vector users are still handled in VPReplicateRegion::execute(),
+  // via shouldPack().
+  // TODO: materialize build vectors for replicating recipes in replicating
+  // regions.
+  // TODO: materialize build vectors for VPInstructions.
+  for (VPBasicBlock *VPBB :
+       concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion)) {
+    for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+      auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
+      auto UsesVectorOrInsideReplicateRegion = [RepR, LoopRegion](VPUser *U) {
+        VPRegionBlock *ParentRegion =
+            cast<VPRecipeBase>(U)->getParent()->getParent();
+        return !U->usesScalars(RepR) || ParentRegion != LoopRegion;
+      };
+      if (!RepR || RepR->isSingleScalar() ||
+          none_of(RepR->users(), UsesVectorOrInsideReplicateRegion))
+        continue;
+
+      Type *ScalarTy = TypeInfo.inferScalarType(RepR);
+      unsigned Opcode = ScalarTy->isStructTy()
+                            ? VPInstruction::BuildStructVector
+                            : VPInstruction::BuildVector;
+      auto *BuildVector = new VPInstruction(Opcode, {RepR});
+      BuildVector->insertAfter(RepR);
+
+      RepR->replaceUsesWithIf(
+          BuildVector, [BuildVector, &UsesVectorOrInsideReplicateRegion](
+                           VPUser &U, unsigned) {
+            return &U != BuildVector && UsesVectorOrInsideReplicateRegion(&U);
+          });
+    }
+  }
+}
+
+void VPlanTransforms::materializeVectorTripCount(VPlan &Plan,
+                                                 VPBasicBlock *VectorPHVPBB,
+                                                 bool TailByMasking,
+                                                 bool RequiresScalarEpilogue) {
+  VPValue &VectorTC = Plan.getVectorTripCount();
+  assert(VectorTC.isLiveIn() && "vector-trip-count must be a live-in");
+  // There's nothing to do if there are no users of the vector trip count or its
+  // IR value has already been set.
+  if (VectorTC.getNumUsers() == 0 || VectorTC.getLiveInIRValue())
+    return;
+
+  VPValue *TC = Plan.getTripCount();
+  Type *TCTy = VPTypeAnalysis(Plan).inferScalarType(TC);
+  VPBuilder Builder(VectorPHVPBB, VectorPHVPBB->begin());
+  VPValue *Step = &Plan.getVFxUF();
+
+  // If the tail is to be folded by masking, round the number of iterations N
+  // up to a multiple of Step instead of rounding down. This is done by first
+  // adding Step-1 and then rounding down. Note that it's ok if this addition
+  // overflows: the vector induction variable will eventually wrap to zero given
+  // that it starts at zero and its Step is a power of two; the loop will then
+  // exit, with the last early-exit vector comparison also producing all-true.
+  // For scalable vectors the VF is not guaranteed to be a power of 2, but this
+  // is accounted for in emitIterationCountCheck that adds an overflow check.
+  if (TailByMasking) {
+    TC = Builder.createNaryOp(
+        Instruction::Add,
+        {TC, Builder.createNaryOp(
+                 Instruction::Sub,
+                 {Step, Plan.getOrAddLiveIn(ConstantInt::get(TCTy, 1))})},
+        DebugLoc::getCompilerGenerated(), "n.rnd.up");
+  }
+
+  // Now we need to generate the expression for the part of the loop that the
+  // vectorized body will execute. This is equal to N - (N % Step) if scalar
+  // iterations are not required for correctness, or N - Step, otherwise. Step
+  // is equal to the vectorization factor (number of SIMD elements) times the
+  // unroll factor (number of SIMD instructions).
+  VPValue *R =
+      Builder.createNaryOp(Instruction::URem, {TC, Step},
+                           DebugLoc::getCompilerGenerated(), "n.mod.vf");
+
+  // There are cases where we *must* run at least one iteration in the remainder
+  // loop.  See the cost model for when this can happen.  If the step evenly
+  // divides the trip count, we set the remainder to be equal to the step. If
+  // the step does not evenly divide the trip count, no adjustment is necessary
+  // since there will already be scalar iterations. Note that the minimum
+  // iterations check ensures that N >= Step.
+  if (RequiresScalarEpilogue) {
+    assert(!TailByMasking &&
+           "requiring scalar epilogue is not supported with fail folding");
+    VPValue *IsZero = Builder.createICmp(
+        CmpInst::ICMP_EQ, R, Plan.getOrAddLiveIn(ConstantInt::get(TCTy, 0)));
+    R = Builder.createSelect(IsZero, Step, R);
+  }
+
+  VPValue *Res = Builder.createNaryOp(
+      Instruction::Sub, {TC, R}, DebugLoc::getCompilerGenerated(), "n.vec");
+  VectorTC.replaceAllUsesWith(Res);
+}
+
+void VPlanTransforms::materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH,
+                                            ElementCount VFEC) {
+  VPBuilder Builder(VectorPH, VectorPH->begin());
+  Type *TCTy = VPTypeAnalysis(Plan).inferScalarType(Plan.getTripCount());
+  VPValue &VF = Plan.getVF();
+  VPValue &VFxUF = Plan.getVFxUF();
+  // Note that after the transform, Plan.getVF and Plan.getVFxUF should not be
+  // used.
+  // TODO: Assert that they aren't used.
+
+  // If there are no users of the runtime VF, compute VFxUF by constant folding
+  // the multiplication of VF and UF.
+  if (VF.getNumUsers() == 0) {
+    VPValue *RuntimeVFxUF =
+        Builder.createElementCount(TCTy, VFEC * Plan.getUF());
+    VFxUF.replaceAllUsesWith(RuntimeVFxUF);
+    return;
+  }
+
+  // For users of the runtime VF, compute it as VF * vscale, and VFxUF as (VF *
+  // vscale) * UF.
+  VPValue *RuntimeVF = Builder.createElementCount(TCTy, VFEC);
+  if (!vputils::onlyScalarValuesUsed(&VF)) {
+    VPValue *BC = Builder.createNaryOp(VPInstruction::Broadcast, RuntimeVF);
+    VF.replaceUsesWithIf(
+        BC, [&VF](VPUser &U, unsigned) { return !U.usesScalars(&VF); });
+  }
+  VF.replaceAllUsesWith(RuntimeVF);
+
+  VPValue *UF = Plan.getOrAddLiveIn(ConstantInt::get(TCTy, Plan.getUF()));
+  VPValue *MulByUF = Builder.createNaryOp(Instruction::Mul, {RuntimeVF, UF});
+  VFxUF.replaceAllUsesWith(MulByUF);
+}
+
+DenseMap<const SCEV *, Value *>
+VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
+  const DataLayout &DL = SE.getDataLayout();
+  SCEVExpander Expander(SE, DL, "induction", /*PreserveLCSSA=*/true);
+
+  auto *Entry = cast<VPIRBasicBlock>(Plan.getEntry());
+  BasicBlock *EntryBB = Entry->getIRBasicBlock();
+  DenseMap<const SCEV *, Value *> ExpandedSCEVs;
+  for (VPRecipeBase &R : make_early_inc_range(*Entry)) {
+    if (isa<VPIRInstruction, VPIRPhi>(&R))
+      continue;
+    auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(&R);
+    if (!ExpSCEV)
+      break;
+    const SCEV *Expr = ExpSCEV->getSCEV();
+    Value *Res =
+        Expander.expandCodeFor(Expr, Expr->getType(), EntryBB->getTerminator());
+    ExpandedSCEVs[ExpSCEV->getSCEV()] = Res;
+    VPValue *Exp = Plan.getOrAddLiveIn(Res);
+    ExpSCEV->replaceAllUsesWith(Exp);
+    if (Plan.getTripCount() == ExpSCEV)
+      Plan.resetTripCount(Exp);
+    ExpSCEV->eraseFromParent();
+  }
+  return ExpandedSCEVs;
+}
+
 /// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
 /// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
 /// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
@@ -3335,9 +3577,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
   if (VF.isScalable() || !VectorLoop)
     return;
 
-  VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
-  Type *CanonicalIVType = CanonicalIV->getScalarType();
-  VPTypeAnalysis TypeInfo(CanonicalIVType);
+  VPTypeAnalysis TypeInfo(Plan);
 
   unsigned FixedVF = VF.getFixedValue();
   SmallVector<VPInterleaveRecipe *> StoreGroups;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index ecaca72e8e7ed..700b94621d5fb 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -54,30 +54,35 @@ struct VPlanTransforms {
       verifyVPlanIsValid(Plan);
   }
 
-  LLVM_ABI_FOR_TEST static std::unique_ptr<VPlan> buildPlainCFG(Loop *TheLoop,
-                                                                LoopInfo &LI);
-
-  /// Prepare the plan for vectorization. It will introduce a dedicated
-  /// VPBasicBlock for the vector pre-header as well as a VPBasicBlock as exit
-  /// block of the main vector loop (middle.block). If a check is needed to
-  /// guard executing the scalar epilogue loop, it will be added to the middle
-  /// block, together with VPBasicBlocks for the scalar preheader and exit
-  /// blocks. \p InductionTy is the type of the canonical induction and used for
-  /// related values, like the trip count expression.  It also creates a VPValue
-  /// expression for the original trip count.
-  LLVM_ABI_FOR_TEST static void prepareForVectorization(
-      VPlan &Plan, Type *InductionTy, PredicatedScalarEvolution &PSE,
-      bool RequiresScalarEpilogueCheck, bool TailFolded, Loop *TheLoop,
-      DebugLoc IVDL, bool HasUncountableExit, VFRange &Range);
+  /// Create a base VPlan0, serving as the common starting point for all later
+  /// candidates. It consists of an initial plain CFG loop with loop blocks from
+  /// \p TheLoop being directly translated to VPBasicBlocks with VPInstruction
+  /// corresponding to the input IR.
+  ///
+  /// The created loop is wrapped in an initial skeleton to facilitate
+  /// vectorization, consisting of a vector pre-header, an exit block for the
+  /// main vector loop (middle.block) and a new block as preheader of the scalar
+  /// loop (scalar.ph). It also adds a canonical IV and its increment, using \p
+  /// InductionTy and \p IVDL, and creates a VPValue expression for the original
+  /// trip count.
+  LLVM_ABI_FOR_TEST static std::unique_ptr<VPlan>
+  buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, DebugLoc IVDL,
+              PredicatedScalarEvolution &PSE);
+
+  /// Update \p Plan to account for all early exits.
+  LLVM_ABI_FOR_TEST static void
+  handleEarlyExits(VPlan &Plan, bool HasUncountableExit, VFRange &Range);
+
+  /// If a check is needed to guard executing the scalar epilogue loop, it will
+  /// be added to the middle block.
+  LLVM_ABI_FOR_TEST static void addMiddleCheck(VPlan &Plan,
+                                               bool RequiresScalarEpilogueCheck,
+                                               bool TailFolded);
 
   /// Replace loops in \p Plan's flat CFG with VPRegionBlocks, turning \p Plan's
   /// flat CFG into a hierarchical CFG.
   LLVM_ABI_FOR_TEST static void createLoopRegions(VPlan &Plan);
 
-  /// Creates extracts for values in \p Plan defined in a loop region and used
-  /// outside a loop region.
-  LLVM_ABI_FOR_TEST static void createExtractsForLiveOuts(VPlan &Plan);
-
   /// Wrap runtime check block \p CheckBlock in a VPIRBB and \p Cond in a
   /// VPValue and connect the block to \p Plan, using the VPValue as branch
   /// condition.
@@ -91,7 +96,7 @@ struct VPlanTransforms {
       VPlanPtr &Plan,
       function_ref<const InductionDescriptor *(PHINode *)>
           GetIntOrFpInductionDescriptor,
-      ScalarEvolution &SE, const TargetLibraryInfo &TLI);
+      const TargetLibraryInfo &TLI);
 
   /// Try to have all users of fixed-order recurrences appear after the recipe
   /// defining their previous value, by either sinking users or hoisting recipes
@@ -113,7 +118,7 @@ struct VPlanTransforms {
   static void clearReductionWrapFlags(VPlan &Plan);
 
   /// Explicitly unroll \p Plan by \p UF.
-  static void unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx);
+  static void unrollByUF(VPlan &Plan, unsigned UF);
 
   /// Replace each VPReplicateRecipe outside on any replicate region in \p Plan
   /// with \p VF single-scalar recipes.
@@ -177,10 +182,9 @@ struct VPlanTransforms {
   /// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.
   /// VPCanonicalIVPHIRecipe is only used to control the loop after
   /// this transformation.
-  /// \returns true if the transformation succeeds, or false if it doesn't.
-  static bool
-  tryAddExplicitVectorLength(VPlan &Plan,
-                             const std::optional<unsigned> &MaxEVLSafeElements);
+  static void
+  addExplicitVectorLength(VPlan &Plan,
+                          const std::optional<unsigned> &MaxEVLSafeElements);
 
   // For each Interleave Group in \p InterleaveGroups replace the Recipes
   // widening its memory instructions with a single VPInterleaveRecipe at its
@@ -221,9 +225,8 @@ struct VPlanTransforms {
   ///    EVLIVInc, TripCount).
   static void canonicalizeEVLLoops(VPlan &Plan);
 
-  /// Lower abstract recipes to concrete ones, that can be codegen'd. Use \p
-  /// CanonicalIVTy as type for all un-typed live-ins in VPTypeAnalysis.
-  static void convertToConcreteRecipes(VPlan &Plan, Type &CanonicalIVTy);
+  /// Lower abstract recipes to concrete ones, that can be codegen'd.
+  static void convertToConcreteRecipes(VPlan &Plan);
 
   /// This function converts initial recipes to the abstract recipes and clamps
   /// \p Range based on cost model for following optimizations and cost
@@ -232,9 +235,8 @@ struct VPlanTransforms {
   static void convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
                                        VFRange &Range);
 
-  /// Perform instcombine-like simplifications on recipes in \p Plan. Use \p
-  /// CanonicalIVTy as type for all un-typed live-ins in VPTypeAnalysis.
-  static void simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy);
+  /// Perform instcombine-like simplifications on recipes in \p Plan.
+  static void simplifyRecipes(VPlan &Plan);
 
   /// Remove BranchOnCond recipes with true or false conditions together with
   /// removing dead edges to their successors.
@@ -245,7 +247,8 @@ struct VPlanTransforms {
   /// one step backwards.
   static void
   optimizeInductionExitUsers(VPlan &Plan,
-                             DenseMap<VPValue *, VPValue *> &EndValues);
+                             DenseMap<VPValue *, VPValue *> &EndValues,
+                             ScalarEvolution &SE);
 
   /// Add explicit broadcasts for live-ins and VPValues defined in \p Plan's entry block if they are used as vectors.
   static void materializeBroadcasts(VPlan &Plan);
@@ -257,11 +260,32 @@ struct VPlanTransforms {
                                      unsigned BestUF,
                                      PredicatedScalarEvolution &PSE);
 
+  /// Materialize vector trip count computations to a set of VPInstructions.
+  static void materializeVectorTripCount(VPlan &Plan,
+                                         VPBasicBlock *VectorPHVPBB,
+                                         bool TailByMasking,
+                                         bool RequiresScalarEpilogue);
+
   /// Materialize the backedge-taken count to be computed explicitly using
   /// VPInstructions.
   static void materializeBackedgeTakenCount(VPlan &Plan,
                                             VPBasicBlock *VectorPH);
 
+  /// Add explicit Build[Struct]Vector recipes that combine multiple scalar
+  /// values into single vectors.
+  static void materializeBuildVectors(VPlan &Plan);
+
+  /// Materialize VF and VFxUF to be computed explicitly using VPInstructions.
+  static void materializeVFAndVFxUF(VPlan &Plan, VPBasicBlock *VectorPH,
+                                    ElementCount VF);
+
+  /// Expand VPExpandSCEVRecipes in \p Plan's entry block. Each
+  /// VPExpandSCEVRecipe is replaced with a live-in wrapping the expanded IR
+  /// value. A mapping from SCEV expressions to their expanded IR value is
+  /// returned.
+  static DenseMap<const SCEV *, Value *> expandSCEVs(VPlan &Plan,
+                                                     ScalarEvolution &SE);
+
   /// Try to convert a plan with interleave groups with VF elements to a plan
   /// with the interleave groups replaced by wide loads and stores processing VF
   /// elements, if all transformed interleave groups access the full vector
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index fc072de8ff78e..4bcde8cd5d42a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -74,8 +74,7 @@ class UnrollState {
   }
 
 public:
-  UnrollState(VPlan &Plan, unsigned UF, LLVMContext &Ctx)
-      : Plan(Plan), UF(UF), TypeInfo(Plan.getCanonicalIV()->getScalarType()) {}
+  UnrollState(VPlan &Plan, unsigned UF) : Plan(Plan), UF(UF), TypeInfo(Plan) {}
 
   void unrollBlock(VPBlockBase *VPB);
 
@@ -374,8 +373,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
         R.addOperand(getValueForPart(Op1, Part));
       continue;
     }
-    if (match(&R, m_VPInstruction<VPInstruction::ExtractLastElement>(
-                      m_VPValue(Op0))) ||
+    if (match(&R, m_ExtractLastElement(m_VPValue(Op0))) ||
         match(&R, m_VPInstruction<VPInstruction::ExtractPenultimateElement>(
                       m_VPValue(Op0)))) {
       addUniformForAllParts(cast<VPSingleDefRecipe>(&R));
@@ -409,7 +407,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
   }
 }
 
-void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx) {
+void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF) {
   assert(UF > 0 && "Unroll factor must be positive");
   Plan.setUF(UF);
   auto Cleanup = make_scope_exit([&Plan]() {
@@ -431,7 +429,7 @@ void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx) {
     return;
   }
 
-  UnrollState Unroller(Plan, UF, Ctx);
+  UnrollState Unroller(Plan, UF);
 
   // Iterate over all blocks in the plan starting from Entry, and unroll
   // recipes inside them. This includes the vector preheader and middle blocks,
@@ -465,15 +463,20 @@ void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx) {
   VPlanTransforms::removeDeadRecipes(Plan);
 }
 
-/// Create a single-scalar clone of \p RepR for lane \p Lane.
-static VPReplicateRecipe *cloneForLane(VPlan &Plan, VPBuilder &Builder,
-                                       Type *IdxTy, VPReplicateRecipe *RepR,
-                                       VPLane Lane) {
+/// Create a single-scalar clone of \p RepR for lane \p Lane. Use \p
+/// Def2LaneDefs to look up scalar definitions for operands of \RepR.
+static VPReplicateRecipe *
+cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy,
+             VPReplicateRecipe *RepR, VPLane Lane,
+             const DenseMap<VPValue *, SmallVector<VPValue *>> &Def2LaneDefs) {
   // Collect the operands at Lane, creating extracts as needed.
   SmallVector<VPValue *> NewOps;
   for (VPValue *Op : RepR->operands()) {
-    if (vputils::isSingleScalar(Op)) {
-      NewOps.push_back(Op);
+    // If Op is a definition that has been unrolled, directly use the clone for
+    // the corresponding lane.
+    auto LaneDefs = Def2LaneDefs.find(Op);
+    if (LaneDefs != Def2LaneDefs.end()) {
+      NewOps.push_back(LaneDefs->second[Lane.getKnownLane()]);
       continue;
     }
     if (Lane.getKind() == VPLane::Kind::ScalableLast) {
@@ -481,6 +484,11 @@ static VPReplicateRecipe *cloneForLane(VPlan &Plan, VPBuilder &Builder,
           Builder.createNaryOp(VPInstruction::ExtractLastElement, {Op}));
       continue;
     }
+    if (vputils::isSingleScalar(Op)) {
+      NewOps.push_back(Op);
+      continue;
+    }
+
     // Look through buildvector to avoid unnecessary extracts.
     if (match(Op, m_BuildVector())) {
       NewOps.push_back(
@@ -513,6 +521,13 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
       vp_depth_first_shallow(Plan.getVectorLoopRegion()->getEntry()));
   auto VPBBsToUnroll =
       concat<VPBasicBlock *>(VPBBsOutsideLoopRegion, VPBBsInsideLoopRegion);
+  // A mapping of current VPValue definitions to collections of new VPValues
+  // defined per lane. Serves to hook-up potential users of current VPValue
+  // definition that are replicated-per-VF later.
+  DenseMap<VPValue *, SmallVector<VPValue *>> Def2LaneDefs;
+  // The removal of current recipes being replaced by new ones needs to be
+  // delayed after Def2LaneDefs is no longer in use.
+  SmallVector<VPRecipeBase *> ToRemove;
   for (VPBasicBlock *VPBB : VPBBsToUnroll) {
     for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
       auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
@@ -524,12 +539,12 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
         if (isa<StoreInst>(RepR->getUnderlyingInstr()) &&
             vputils::isSingleScalar(RepR->getOperand(1))) {
           // Stores to invariant addresses need to store the last lane only.
-          cloneForLane(Plan, Builder, IdxTy, RepR,
-                       VPLane::getLastLaneForVF(VF));
+          cloneForLane(Plan, Builder, IdxTy, RepR, VPLane::getLastLaneForVF(VF),
+                       Def2LaneDefs);
         } else {
           // Create single-scalar version of RepR for all lanes.
           for (unsigned I = 0; I != VF.getKnownMinValue(); ++I)
-            cloneForLane(Plan, Builder, IdxTy, RepR, VPLane(I));
+            cloneForLane(Plan, Builder, IdxTy, RepR, VPLane(I), Def2LaneDefs);
         }
         RepR->eraseFromParent();
         continue;
@@ -537,23 +552,33 @@ void VPlanTransforms::replicateByVF(VPlan &Plan, ElementCount VF) {
       /// Create single-scalar version of RepR for all lanes.
       SmallVector<VPValue *> LaneDefs;
       for (unsigned I = 0; I != VF.getKnownMinValue(); ++I)
-        LaneDefs.push_back(cloneForLane(Plan, Builder, IdxTy, RepR, VPLane(I)));
+        LaneDefs.push_back(
+            cloneForLane(Plan, Builder, IdxTy, RepR, VPLane(I), Def2LaneDefs));
 
+      Def2LaneDefs[RepR] = LaneDefs;
       /// Users that only demand the first lane can use the definition for lane
       /// 0.
       RepR->replaceUsesWithIf(LaneDefs[0], [RepR](VPUser &U, unsigned) {
         return U.onlyFirstLaneUsed(RepR);
       });
 
-      // If needed, create a Build(Struct)Vector recipe to insert the scalar
-      // lane values into a vector.
-      Type *ResTy = RepR->getUnderlyingInstr()->getType();
-      VPValue *VecRes = Builder.createNaryOp(
-          ResTy->isStructTy() ? VPInstruction::BuildStructVector
-                              : VPInstruction::BuildVector,
-          LaneDefs);
-      RepR->replaceAllUsesWith(VecRes);
-      RepR->eraseFromParent();
+      // Update each build vector user that currently has RepR as its only
+      // operand, to have all LaneDefs as its operands.
+      for (VPUser *U : to_vector(RepR->users())) {
+        auto *VPI = dyn_cast<VPInstruction>(U);
+        if (!VPI || (VPI->getOpcode() != VPInstruction::BuildVector &&
+                     VPI->getOpcode() != VPInstruction::BuildStructVector))
+          continue;
+        assert(VPI->getNumOperands() == 1 &&
+               "Build(Struct)Vector must have a single operand before "
+               "replicating by VF");
+        VPI->setOperand(0, LaneDefs[0]);
+        for (VPValue *LaneDef : drop_begin(LaneDefs))
+          VPI->addOperand(LaneDef);
+      }
+      ToRemove.push_back(RepR);
     }
   }
+  for (auto *R : reverse(ToRemove))
+    R->eraseFromParent();
 }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 14f20c65a7034..700a733bf9f2c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -24,8 +24,12 @@ bool vputils::onlyFirstPartUsed(const VPValue *Def) {
                 [Def](const VPUser *U) { return U->onlyFirstPartUsed(Def); });
 }
 
-VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
-                                                ScalarEvolution &SE) {
+bool vputils::onlyScalarValuesUsed(const VPValue *Def) {
+  return all_of(Def->users(),
+                [Def](const VPUser *U) { return U->usesScalars(Def); });
+}
+
+VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr) {
   if (auto *Expanded = Plan.getSCEVExpansion(Expr))
     return Expanded;
   VPValue *Expanded = nullptr;
@@ -40,7 +44,7 @@ VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
     if (U && !isa<Instruction>(U->getValue())) {
       Expanded = Plan.getOrAddLiveIn(U->getValue());
     } else {
-      Expanded = new VPExpandSCEVRecipe(Expr, SE);
+      Expanded = new VPExpandSCEVRecipe(Expr);
       Plan.getEntry()->appendRecipe(Expanded->getDefiningRecipe());
     }
   }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 8dcd57f1b3598..9e1d325a4d8d6 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -25,13 +25,15 @@ bool onlyFirstLaneUsed(const VPValue *Def);
 /// Returns true if only the first part of \p Def is used.
 bool onlyFirstPartUsed(const VPValue *Def);
 
+/// Returns true if only scalar values of \p Def are used by all users.
+bool onlyScalarValuesUsed(const VPValue *Def);
+
 /// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p
 /// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in
 /// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's
 /// pre-header already contains a recipe expanding \p Expr, return it. If not,
 /// create a new one.
-VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
-                                       ScalarEvolution &SE);
+VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr);
 
 /// Return the SCEV expression for \p V. Returns SCEVCouldNotCompute if no
 /// SCEV expression could be constructed.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 14ae4f2204310..e25ffe135418e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -157,7 +157,7 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
           return VerifyEVLUse(*S, S->getNumOperands() - 1);
         })
         .Case<VPWidenStoreEVLRecipe, VPReductionEVLRecipe,
-              VPWidenIntOrFpInductionRecipe>(
+              VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe>(
             [&](const VPRecipeBase *S) { return VerifyEVLUse(*S, 2); })
         .Case<VPScalarIVStepsRecipe>([&](auto *R) {
           if (R->getNumOperands() != 3) {
@@ -183,6 +183,7 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
           case Instruction::ZExt:
           case Instruction::Mul:
           case Instruction::FMul:
+          case VPInstruction::Broadcast:
             // Opcodes above can only use EVL after wide inductions have been
             // expanded.
             if (!VerifyLate) {
@@ -250,17 +251,15 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) {
       for (const VPUser *U : V->users()) {
         auto *UI = cast<VPRecipeBase>(U);
         if (auto *Phi = dyn_cast<VPPhiAccessors>(UI)) {
-          for (unsigned Idx = 0; Idx != Phi->getNumIncoming(); ++Idx) {
-            VPValue *IncomingVPV = Phi->getIncomingValue(Idx);
+          for (const auto &[IncomingVPV, IncomingVPBB] :
+               Phi->incoming_values_and_blocks()) {
             if (IncomingVPV != V)
               continue;
 
-            const VPBasicBlock *IncomingVPBB = Phi->getIncomingBlock(Idx);
             if (VPDT.dominates(VPBB, IncomingVPBB))
               continue;
 
-            errs() << "Incoming def at index " << Idx
-                   << " does not dominate incoming block!\n";
+            errs() << "Incoming def does not dominate incoming block!\n";
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
             VPSlotTracker Tracker(VPBB->getPlan());
             IncomingVPV->getDefiningRecipe()->print(errs(), "  ", Tracker);
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 6345b18b809a6..092a3a87954ff 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -16,6 +16,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
@@ -29,11 +30,13 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include <numeric>
+#include <optional>
 #include <queue>
 #include <set>
 
@@ -74,7 +77,7 @@ class VectorCombine {
                 const DataLayout *DL, TTI::TargetCostKind CostKind,
                 bool TryEarlyFoldsOnly)
       : F(F), Builder(F.getContext(), InstSimplifyFolder(*DL)), TTI(TTI),
-        DT(DT), AA(AA), AC(AC), DL(DL), CostKind(CostKind),
+        DT(DT), AA(AA), AC(AC), DL(DL), CostKind(CostKind), SQ(*DL),
         TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
 
   bool run();
@@ -88,6 +91,7 @@ class VectorCombine {
   AssumptionCache &AC;
   const DataLayout *DL;
   TTI::TargetCostKind CostKind;
+  const SimplifyQuery SQ;
 
   /// If true, only perform beneficial early IR transforms. Do not introduce new
   /// vector operations.
@@ -107,10 +111,8 @@ class VectorCombine {
                              const Instruction &I,
                              ExtractElementInst *&ConvertToShuffle,
                              unsigned PreferredExtractIndex);
-  void foldExtExtCmp(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
-                     Instruction &I);
-  void foldExtExtBinop(ExtractElementInst *Ext0, ExtractElementInst *Ext1,
-                       Instruction &I);
+  Value *foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
+  Value *foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex, Instruction &I);
   bool foldExtractExtract(Instruction &I);
   bool foldInsExtFNeg(Instruction &I);
   bool foldInsExtBinop(Instruction &I);
@@ -133,12 +135,15 @@ class VectorCombine {
   bool foldShuffleOfIntrinsics(Instruction &I);
   bool foldShuffleToIdentity(Instruction &I);
   bool foldShuffleFromReductions(Instruction &I);
+  bool foldShuffleChainsToReduce(Instruction &I);
   bool foldCastFromReductions(Instruction &I);
   bool foldSelectShuffle(Instruction &I, bool FromReduction = false);
   bool foldInterleaveIntrinsics(Instruction &I);
   bool shrinkType(Instruction &I);
+  bool shrinkLoadForShuffles(Instruction &I);
+  bool shrinkPhiOfShuffles(Instruction &I);
 
-  void replaceValue(Value &Old, Value &New) {
+  void replaceValue(Instruction &Old, Value &New, bool Erase = true) {
     LLVM_DEBUG(dbgs() << "VC: Replacing: " << Old << '\n');
     LLVM_DEBUG(dbgs() << "         With: " << New << '\n');
     Old.replaceAllUsesWith(&New);
@@ -147,7 +152,11 @@ class VectorCombine {
       Worklist.pushUsersToWorkList(*NewI);
       Worklist.pushValue(NewI);
     }
-    Worklist.pushValue(&Old);
+    if (Erase && isInstructionTriviallyDead(&Old)) {
+      eraseInstruction(Old);
+    } else {
+      Worklist.push(&Old);
+    }
   }
 
   void eraseInstruction(Instruction &I) {
@@ -158,11 +167,23 @@ class VectorCombine {
 
     // Push remaining users of the operands and then the operand itself - allows
     // further folds that were hindered by OneUse limits.
-    for (Value *Op : Ops)
-      if (auto *OpI = dyn_cast<Instruction>(Op)) {
-        Worklist.pushUsersToWorkList(*OpI);
-        Worklist.pushValue(OpI);
+    SmallPtrSet<Value *, 4> Visited;
+    for (Value *Op : Ops) {
+      if (Visited.insert(Op).second) {
+        if (auto *OpI = dyn_cast<Instruction>(Op)) {
+          if (RecursivelyDeleteTriviallyDeadInstructions(
+                  OpI, nullptr, nullptr, [this](Value *V) {
+                    if (auto I = dyn_cast<Instruction>(V)) {
+                      LLVM_DEBUG(dbgs() << "VC: Erased: " << *I << '\n');
+                      Worklist.remove(I);
+                    }
+                  }))
+            continue;
+          Worklist.pushUsersToWorkList(*OpI);
+          Worklist.pushValue(OpI);
+        }
       }
+    }
   }
 };
 } // namespace
@@ -546,9 +567,8 @@ static Value *createShiftShuffle(Value *Vec, unsigned OldIndex,
 /// the source vector (shift the scalar element) to a NewIndex for extraction.
 /// Return null if the input can be constant folded, so that we are not creating
 /// unnecessary instructions.
-static ExtractElementInst *translateExtract(ExtractElementInst *ExtElt,
-                                            unsigned NewIndex,
-                                            IRBuilderBase &Builder) {
+static Value *translateExtract(ExtractElementInst *ExtElt, unsigned NewIndex,
+                               IRBuilderBase &Builder) {
   // Shufflevectors can only be created for fixed-width vectors.
   Value *X = ExtElt->getVectorOperand();
   if (!isa<FixedVectorType>(X->getType()))
@@ -563,52 +583,43 @@ static ExtractElementInst *translateExtract(ExtractElementInst *ExtElt,
 
   Value *Shuf = createShiftShuffle(X, cast<ConstantInt>(C)->getZExtValue(),
                                    NewIndex, Builder);
-  return dyn_cast<ExtractElementInst>(
-      Builder.CreateExtractElement(Shuf, NewIndex));
+  return Shuf;
 }
 
 /// Try to reduce extract element costs by converting scalar compares to vector
 /// compares followed by extract.
-/// cmp (ext0 V0, C), (ext1 V1, C)
-void VectorCombine::foldExtExtCmp(ExtractElementInst *Ext0,
-                                  ExtractElementInst *Ext1, Instruction &I) {
+/// cmp (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
+Value *VectorCombine::foldExtExtCmp(Value *V0, Value *V1, Value *ExtIndex,
+                                    Instruction &I) {
   assert(isa<CmpInst>(&I) && "Expected a compare");
-  assert(cast<ConstantInt>(Ext0->getIndexOperand())->getZExtValue() ==
-             cast<ConstantInt>(Ext1->getIndexOperand())->getZExtValue() &&
-         "Expected matching constant extract indexes");
 
-  // cmp Pred (extelt V0, C), (extelt V1, C) --> extelt (cmp Pred V0, V1), C
+  // cmp Pred (extelt V0, ExtIndex), (extelt V1, ExtIndex)
+  //   --> extelt (cmp Pred V0, V1), ExtIndex
   ++NumVecCmp;
   CmpInst::Predicate Pred = cast<CmpInst>(&I)->getPredicate();
-  Value *V0 = Ext0->getVectorOperand(), *V1 = Ext1->getVectorOperand();
   Value *VecCmp = Builder.CreateCmp(Pred, V0, V1);
-  Value *NewExt = Builder.CreateExtractElement(VecCmp, Ext0->getIndexOperand());
-  replaceValue(I, *NewExt);
+  return Builder.CreateExtractElement(VecCmp, ExtIndex, "foldExtExtCmp");
 }
 
 /// Try to reduce extract element costs by converting scalar binops to vector
 /// binops followed by extract.
-/// bo (ext0 V0, C), (ext1 V1, C)
-void VectorCombine::foldExtExtBinop(ExtractElementInst *Ext0,
-                                    ExtractElementInst *Ext1, Instruction &I) {
+/// bo (ext0 V0, ExtIndex), (ext1 V1, ExtIndex)
+Value *VectorCombine::foldExtExtBinop(Value *V0, Value *V1, Value *ExtIndex,
+                                      Instruction &I) {
   assert(isa<BinaryOperator>(&I) && "Expected a binary operator");
-  assert(cast<ConstantInt>(Ext0->getIndexOperand())->getZExtValue() ==
-             cast<ConstantInt>(Ext1->getIndexOperand())->getZExtValue() &&
-         "Expected matching constant extract indexes");
 
-  // bo (extelt V0, C), (extelt V1, C) --> extelt (bo V0, V1), C
+  // bo (extelt V0, ExtIndex), (extelt V1, ExtIndex)
+  //   --> extelt (bo V0, V1), ExtIndex
   ++NumVecBO;
-  Value *V0 = Ext0->getVectorOperand(), *V1 = Ext1->getVectorOperand();
-  Value *VecBO =
-      Builder.CreateBinOp(cast<BinaryOperator>(&I)->getOpcode(), V0, V1);
+  Value *VecBO = Builder.CreateBinOp(cast<BinaryOperator>(&I)->getOpcode(), V0,
+                                     V1, "foldExtExtBinop");
 
   // All IR flags are safe to back-propagate because any potential poison
   // created in unused vector elements is discarded by the extract.
   if (auto *VecBOInst = dyn_cast<Instruction>(VecBO))
     VecBOInst->copyIRFlags(&I);
 
-  Value *NewExt = Builder.CreateExtractElement(VecBO, Ext0->getIndexOperand());
-  replaceValue(I, *NewExt);
+  return Builder.CreateExtractElement(VecBO, ExtIndex, "foldExtExtBinop");
 }
 
 /// Match an instruction with extracted vector operands.
@@ -647,25 +658,29 @@ bool VectorCombine::foldExtractExtract(Instruction &I) {
   if (isExtractExtractCheap(Ext0, Ext1, I, ExtractToChange, InsertIndex))
     return false;
 
+  Value *ExtOp0 = Ext0->getVectorOperand();
+  Value *ExtOp1 = Ext1->getVectorOperand();
+
   if (ExtractToChange) {
     unsigned CheapExtractIdx = ExtractToChange == Ext0 ? C1 : C0;
-    ExtractElementInst *NewExtract =
+    Value *NewExtOp =
         translateExtract(ExtractToChange, CheapExtractIdx, Builder);
-    if (!NewExtract)
+    if (!NewExtOp)
       return false;
     if (ExtractToChange == Ext0)
-      Ext0 = NewExtract;
+      ExtOp0 = NewExtOp;
     else
-      Ext1 = NewExtract;
+      ExtOp1 = NewExtOp;
   }
 
-  if (Pred != CmpInst::BAD_ICMP_PREDICATE)
-    foldExtExtCmp(Ext0, Ext1, I);
-  else
-    foldExtExtBinop(Ext0, Ext1, I);
-
+  Value *ExtIndex = ExtractToChange == Ext0 ? Ext1->getIndexOperand()
+                                            : Ext0->getIndexOperand();
+  Value *NewExt = Pred != CmpInst::BAD_ICMP_PREDICATE
+                      ? foldExtExtCmp(ExtOp0, ExtOp1, ExtIndex, I)
+                      : foldExtExtBinop(ExtOp0, ExtOp1, ExtIndex, I);
   Worklist.push(Ext0);
   Worklist.push(Ext1);
+  replaceValue(I, *NewExt);
   return true;
 }
 
@@ -1232,17 +1247,18 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
   // Fold the vector constants in the original vectors into a new base vector to
   // get more accurate cost modelling.
   Value *NewVecC = nullptr;
-  TargetFolder Folder(*DL);
   if (CI)
-    NewVecC = Folder.FoldCmp(CI->getPredicate(), VecCs[0], VecCs[1]);
+    NewVecC = simplifyCmpInst(CI->getPredicate(), VecCs[0], VecCs[1], SQ);
   else if (UO)
     NewVecC =
-        Folder.FoldUnOpFMF(UO->getOpcode(), VecCs[0], UO->getFastMathFlags());
+        simplifyUnOp(UO->getOpcode(), VecCs[0], UO->getFastMathFlags(), SQ);
   else if (BO)
-    NewVecC = Folder.FoldBinOp(BO->getOpcode(), VecCs[0], VecCs[1]);
-  else if (II->arg_size() == 2)
-    NewVecC = Folder.FoldBinaryIntrinsic(II->getIntrinsicID(), VecCs[0],
-                                         VecCs[1], II->getType(), &I);
+    NewVecC = simplifyBinOp(BO->getOpcode(), VecCs[0], VecCs[1], SQ);
+  else if (II)
+    NewVecC = simplifyCall(II, II->getCalledOperand(), VecCs, SQ);
+
+  if (!NewVecC)
+    return false;
 
   // Get cost estimate for the insert element. This cost will factor into
   // both sequences.
@@ -1250,6 +1266,7 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
   InstructionCost NewCost =
       ScalarOpCost + TTI.getVectorInstrCost(Instruction::InsertElement, VecTy,
                                             CostKind, *Index, NewVecC);
+
   for (auto [Idx, Op, VecC, Scalar] : enumerate(Ops, VecCs, ScalarOps)) {
     if (!Scalar || (II && isVectorIntrinsicWithScalarOpAtArg(
                               II->getIntrinsicID(), Idx, &TTI)))
@@ -1294,15 +1311,6 @@ bool VectorCombine::scalarizeOpOrCmp(Instruction &I) {
   if (auto *ScalarInst = dyn_cast<Instruction>(Scalar))
     ScalarInst->copyIRFlags(&I);
 
-  // Create a new base vector if the constant folding failed.
-  if (!NewVecC) {
-    if (CI)
-      NewVecC = Builder.CreateCmp(CI->getPredicate(), VecCs[0], VecCs[1]);
-    else if (UO || BO)
-      NewVecC = Builder.CreateNAryOp(Opcode, VecCs);
-    else
-      NewVecC = Builder.CreateIntrinsic(VecTy, II->getIntrinsicID(), VecCs);
-  }
   Value *Insert = Builder.CreateInsertElement(NewVecC, Scalar, *Index);
   replaceValue(I, *Insert);
   return true;
@@ -1790,7 +1798,8 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) {
     ScalarizedCost +=
         TTI.getMemoryOpCost(Instruction::Load, VecTy->getElementType(),
                             Align(1), LI->getPointerAddressSpace(), CostKind);
-    ScalarizedCost += TTI.getAddressComputationCost(VecTy->getElementType());
+    ScalarizedCost += TTI.getAddressComputationCost(LI->getPointerOperandType(),
+                                                    nullptr, nullptr, CostKind);
   }
 
   LLVM_DEBUG(dbgs() << "Found all extractions of a vector load: " << I
@@ -1804,6 +1813,8 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) {
   // erased in the correct order.
   Worklist.push(LI);
 
+  Type *ElemType = VecTy->getElementType();
+
   // Replace extracts with narrow scalar loads.
   for (User *U : LI->users()) {
     auto *EI = cast<ExtractElementInst>(U);
@@ -1817,14 +1828,20 @@ bool VectorCombine::scalarizeLoadExtract(Instruction &I) {
     Builder.SetInsertPoint(EI);
     Value *GEP =
         Builder.CreateInBoundsGEP(VecTy, Ptr, {Builder.getInt32(0), Idx});
-    auto *NewLoad = cast<LoadInst>(Builder.CreateLoad(
-        VecTy->getElementType(), GEP, EI->getName() + ".scalar"));
+    auto *NewLoad = cast<LoadInst>(
+        Builder.CreateLoad(ElemType, GEP, EI->getName() + ".scalar"));
 
-    Align ScalarOpAlignment = computeAlignmentAfterScalarization(
-        LI->getAlign(), VecTy->getElementType(), Idx, *DL);
+    Align ScalarOpAlignment =
+        computeAlignmentAfterScalarization(LI->getAlign(), ElemType, Idx, *DL);
     NewLoad->setAlignment(ScalarOpAlignment);
 
-    replaceValue(*EI, *NewLoad);
+    if (auto *ConstIdx = dyn_cast<ConstantInt>(Idx)) {
+      size_t Offset = ConstIdx->getZExtValue() * DL->getTypeStoreSize(ElemType);
+      AAMDNodes OldAAMD = LI->getAAMetadata();
+      NewLoad->setAAMetadata(OldAAMD.adjustForAccess(Offset, ElemType, *DL));
+    }
+
+    replaceValue(*EI, *NewLoad, false);
   }
 
   FailureGuard.release();
@@ -1856,15 +1873,15 @@ bool VectorCombine::scalarizeExtExtract(Instruction &I) {
   unsigned ExtCnt = 0;
   bool ExtLane0 = false;
   for (User *U : Ext->users()) {
-    const APInt *Idx;
-    if (!match(U, m_ExtractElt(m_Value(), m_APInt(Idx))))
+    uint64_t Idx;
+    if (!match(U, m_ExtractElt(m_Value(), m_ConstantInt(Idx))))
       return false;
     if (cast<Instruction>(U)->use_empty())
       continue;
     ExtCnt += 1;
-    ExtLane0 |= Idx->isZero();
+    ExtLane0 |= !Idx;
     VectorCost += TTI.getVectorInstrCost(Instruction::ExtractElement, DstTy,
-                                         CostKind, Idx->getZExtValue(), U);
+                                         CostKind, Idx, U);
   }
 
   InstructionCost ScalarCost =
@@ -2817,8 +2834,7 @@ static Value *generateNewInstTree(ArrayRef<InstLane> Item, FixedVectorType *Ty,
     return Value;
   }
   if (auto *CI = dyn_cast<CastInst>(I)) {
-    auto *Value = Builder.CreateCast((Instruction::CastOps)CI->getOpcode(),
-                                     Ops[0], DstTy);
+    auto *Value = Builder.CreateCast(CI->getOpcode(), Ops[0], DstTy);
     propagateIRFlags(Value, ValueList);
     return Value;
   }
@@ -2910,7 +2926,7 @@ bool VectorCombine::foldShuffleToIdentity(Instruction &I) {
       if (!IL.first)
         return true;
       Value *V = IL.first->get();
-      if (auto *I = dyn_cast<Instruction>(V); I && !I->hasOneUse())
+      if (auto *I = dyn_cast<Instruction>(V); I && !I->hasOneUser())
         return false;
       if (V->getValueID() != FrontV->getValueID())
         return false;
@@ -3112,7 +3128,7 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
         Shuffle->getOperand(0), Shuffle->getOperand(1), ConcatMask);
     LLVM_DEBUG(dbgs() << "Created new shuffle: " << *NewShuffle << "\n");
     replaceValue(*Shuffle, *NewShuffle);
-    MadeChanges = true;
+    return true;
   }
 
   // See if we can re-use foldSelectShuffle, getting it to reduce the size of
@@ -3121,6 +3137,267 @@ bool VectorCombine::foldShuffleFromReductions(Instruction &I) {
   return MadeChanges;
 }
 
+/// For a given chain of patterns of the following form:
+///
+/// ```
+///   %1 = shufflevector <n x ty1> %0, <n x ty1> poison <n x ty2> mask
+///
+///   %2 = tail call <n x ty1> llvm.<umin/umax/smin/smax>(<n x ty1> %0, <n x
+///   ty1> %1)
+///     OR
+///   %2 = add/mul/or/and/xor <n x ty1> %0, %1
+///
+///   %3 = shufflevector <n x ty1> %2, <n x ty1> poison <n x ty2> mask
+///   ...
+///   ...
+///   %(i - 1) = tail call <n x ty1> llvm.<umin/umax/smin/smax>(<n x ty1> %(i -
+///   3), <n x ty1> %(i - 2)
+///     OR
+///   %(i - 1) = add/mul/or/and/xor <n x ty1> %(i - 3), %(i - 2)
+///
+///   %(i) = extractelement <n x ty1> %(i - 1), 0
+/// ```
+///
+/// Where:
+///    `mask` follows a partition pattern:
+///
+/// Ex:
+///    [n = 8, p = poison]
+///
+///    4 5 6 7 | p p p p
+///    2 3 | p p p p p p
+///    1 | p p p p p p p
+///
+///    For powers of 2, there's a consistent pattern, but for other cases
+///    the parity of the current half value at each step decides the
+///    next partition half (see `ExpectedParityMask` for more logical details
+///    in generalising this).
+///
+/// Ex:
+///    [n = 6]
+///
+///    3 4 5 | p p p
+///    1 2 | p p p p
+///    1 | p p p p p
+bool VectorCombine::foldShuffleChainsToReduce(Instruction &I) {
+  // Going bottom-up for the pattern.
+  std::queue<Value *> InstWorklist;
+  InstructionCost OrigCost = 0;
+
+  // Common instruction operation after each shuffle op.
+  std::optional<unsigned int> CommonCallOp = std::nullopt;
+  std::optional<Instruction::BinaryOps> CommonBinOp = std::nullopt;
+
+  bool IsFirstCallOrBinInst = true;
+  bool ShouldBeCallOrBinInst = true;
+
+  // This stores the last used instructions for shuffle/common op.
+  //
+  // PrevVecV[0] / PrevVecV[1] store the last two simultaneous
+  // instructions from either shuffle/common op.
+  SmallVector<Value *, 2> PrevVecV(2, nullptr);
+
+  Value *VecOpEE;
+  if (!match(&I, m_ExtractElt(m_Value(VecOpEE), m_Zero())))
+    return false;
+
+  auto *FVT = dyn_cast<FixedVectorType>(VecOpEE->getType());
+  if (!FVT)
+    return false;
+
+  int64_t VecSize = FVT->getNumElements();
+  if (VecSize < 2)
+    return false;
+
+  // Number of levels would be ~log2(n), considering we always partition
+  // by half for this fold pattern.
+  unsigned int NumLevels = Log2_64_Ceil(VecSize), VisitedCnt = 0;
+  int64_t ShuffleMaskHalf = 1, ExpectedParityMask = 0;
+
+  // This is how we generalise for all element sizes.
+  // At each step, if vector size is odd, we need non-poison
+  // values to cover the dominant half so we don't miss out on any element.
+  //
+  // This mask will help us retrieve this as we go from bottom to top:
+  //
+  // Mask Set -> N = N * 2 - 1
+  // Mask Unset -> N = N * 2
+  for (int Cur = VecSize, Mask = NumLevels - 1; Cur > 1;
+       Cur = (Cur + 1) / 2, --Mask) {
+    if (Cur & 1)
+      ExpectedParityMask |= (1ll << Mask);
+  }
+
+  InstWorklist.push(VecOpEE);
+
+  while (!InstWorklist.empty()) {
+    Value *CI = InstWorklist.front();
+    InstWorklist.pop();
+
+    if (auto *II = dyn_cast<IntrinsicInst>(CI)) {
+      if (!ShouldBeCallOrBinInst)
+        return false;
+
+      if (!IsFirstCallOrBinInst &&
+          any_of(PrevVecV, [](Value *VecV) { return VecV == nullptr; }))
+        return false;
+
+      // For the first found call/bin op, the vector has to come from the
+      // extract element op.
+      if (II != (IsFirstCallOrBinInst ? VecOpEE : PrevVecV[0]))
+        return false;
+      IsFirstCallOrBinInst = false;
+
+      if (!CommonCallOp)
+        CommonCallOp = II->getIntrinsicID();
+      if (II->getIntrinsicID() != *CommonCallOp)
+        return false;
+
+      switch (II->getIntrinsicID()) {
+      case Intrinsic::umin:
+      case Intrinsic::umax:
+      case Intrinsic::smin:
+      case Intrinsic::smax: {
+        auto *Op0 = II->getOperand(0);
+        auto *Op1 = II->getOperand(1);
+        PrevVecV[0] = Op0;
+        PrevVecV[1] = Op1;
+        break;
+      }
+      default:
+        return false;
+      }
+      ShouldBeCallOrBinInst ^= 1;
+
+      IntrinsicCostAttributes ICA(
+          *CommonCallOp, II->getType(),
+          {PrevVecV[0]->getType(), PrevVecV[1]->getType()});
+      OrigCost += TTI.getIntrinsicInstrCost(ICA, CostKind);
+
+      // We may need a swap here since it can be (a, b) or (b, a)
+      // and accordingly change as we go up.
+      if (!isa<ShuffleVectorInst>(PrevVecV[1]))
+        std::swap(PrevVecV[0], PrevVecV[1]);
+      InstWorklist.push(PrevVecV[1]);
+      InstWorklist.push(PrevVecV[0]);
+    } else if (auto *BinOp = dyn_cast<BinaryOperator>(CI)) {
+      // Similar logic for bin ops.
+
+      if (!ShouldBeCallOrBinInst)
+        return false;
+
+      if (!IsFirstCallOrBinInst &&
+          any_of(PrevVecV, [](Value *VecV) { return VecV == nullptr; }))
+        return false;
+
+      if (BinOp != (IsFirstCallOrBinInst ? VecOpEE : PrevVecV[0]))
+        return false;
+      IsFirstCallOrBinInst = false;
+
+      if (!CommonBinOp)
+        CommonBinOp = BinOp->getOpcode();
+
+      if (BinOp->getOpcode() != *CommonBinOp)
+        return false;
+
+      switch (*CommonBinOp) {
+      case BinaryOperator::Add:
+      case BinaryOperator::Mul:
+      case BinaryOperator::Or:
+      case BinaryOperator::And:
+      case BinaryOperator::Xor: {
+        auto *Op0 = BinOp->getOperand(0);
+        auto *Op1 = BinOp->getOperand(1);
+        PrevVecV[0] = Op0;
+        PrevVecV[1] = Op1;
+        break;
+      }
+      default:
+        return false;
+      }
+      ShouldBeCallOrBinInst ^= 1;
+
+      OrigCost +=
+          TTI.getArithmeticInstrCost(*CommonBinOp, BinOp->getType(), CostKind);
+
+      if (!isa<ShuffleVectorInst>(PrevVecV[1]))
+        std::swap(PrevVecV[0], PrevVecV[1]);
+      InstWorklist.push(PrevVecV[1]);
+      InstWorklist.push(PrevVecV[0]);
+    } else if (auto *SVInst = dyn_cast<ShuffleVectorInst>(CI)) {
+      // We shouldn't have any null values in the previous vectors,
+      // is so, there was a mismatch in pattern.
+      if (ShouldBeCallOrBinInst ||
+          any_of(PrevVecV, [](Value *VecV) { return VecV == nullptr; }))
+        return false;
+
+      if (SVInst != PrevVecV[1])
+        return false;
+
+      ArrayRef<int> CurMask;
+      if (!match(SVInst, m_Shuffle(m_Specific(PrevVecV[0]), m_Poison(),
+                                   m_Mask(CurMask))))
+        return false;
+
+      // Subtract the parity mask when checking the condition.
+      for (int Mask = 0, MaskSize = CurMask.size(); Mask != MaskSize; ++Mask) {
+        if (Mask < ShuffleMaskHalf &&
+            CurMask[Mask] != ShuffleMaskHalf + Mask - (ExpectedParityMask & 1))
+          return false;
+        if (Mask >= ShuffleMaskHalf && CurMask[Mask] != -1)
+          return false;
+      }
+
+      // Update mask values.
+      ShuffleMaskHalf *= 2;
+      ShuffleMaskHalf -= (ExpectedParityMask & 1);
+      ExpectedParityMask >>= 1;
+
+      OrigCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
+                                     SVInst->getType(), SVInst->getType(),
+                                     CurMask, CostKind);
+
+      VisitedCnt += 1;
+      if (!ExpectedParityMask && VisitedCnt == NumLevels)
+        break;
+
+      ShouldBeCallOrBinInst ^= 1;
+    } else {
+      return false;
+    }
+  }
+
+  // Pattern should end with a shuffle op.
+  if (ShouldBeCallOrBinInst)
+    return false;
+
+  assert(VecSize != -1 && "Expected Match for Vector Size");
+
+  Value *FinalVecV = PrevVecV[0];
+  if (!FinalVecV)
+    return false;
+
+  auto *FinalVecVTy = cast<FixedVectorType>(FinalVecV->getType());
+
+  Intrinsic::ID ReducedOp =
+      (CommonCallOp ? getMinMaxReductionIntrinsicID(*CommonCallOp)
+                    : getReductionForBinop(*CommonBinOp));
+  if (!ReducedOp)
+    return false;
+
+  IntrinsicCostAttributes ICA(ReducedOp, FinalVecVTy, {FinalVecV});
+  InstructionCost NewCost = TTI.getIntrinsicInstrCost(ICA, CostKind);
+
+  if (NewCost >= OrigCost)
+    return false;
+
+  auto *ReducedResult =
+      Builder.CreateIntrinsic(ReducedOp, {FinalVecV->getType()}, {FinalVecV});
+  replaceValue(I, *ReducedResult);
+
+  return true;
+}
+
 /// Determine if its more efficient to fold:
 ///   reduce(trunc(x)) -> trunc(reduce(x)).
 ///   reduce(sext(x))  -> sext(reduce(x)).
@@ -3608,7 +3885,7 @@ bool VectorCombine::foldSelectShuffle(Instruction &I, bool FromReduction) {
   for (int S = 0, E = ReconstructMasks.size(); S != E; S++) {
     Builder.SetInsertPoint(Shuffles[S]);
     Value *NSV = Builder.CreateShuffleVector(NOp0, NOp1, ReconstructMasks[S]);
-    replaceValue(*Shuffles[S], *NSV);
+    replaceValue(*Shuffles[S], *NSV, false);
   }
 
   Worklist.pushValue(NSV0A);
@@ -3861,6 +4138,228 @@ bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) {
   return true;
 }
 
+// Attempt to shrink loads that are only used by shufflevector instructions.
+bool VectorCombine::shrinkLoadForShuffles(Instruction &I) {
+  auto *OldLoad = dyn_cast<LoadInst>(&I);
+  if (!OldLoad || !OldLoad->isSimple())
+    return false;
+
+  auto *OldLoadTy = dyn_cast<FixedVectorType>(OldLoad->getType());
+  if (!OldLoadTy)
+    return false;
+
+  unsigned const OldNumElements = OldLoadTy->getNumElements();
+
+  // Search all uses of load. If all uses are shufflevector instructions, and
+  // the second operands are all poison values, find the minimum and maximum
+  // indices of the vector elements referenced by all shuffle masks.
+  // Otherwise return `std::nullopt`.
+  using IndexRange = std::pair<int, int>;
+  auto GetIndexRangeInShuffles = [&]() -> std::optional<IndexRange> {
+    IndexRange OutputRange = IndexRange(OldNumElements, -1);
+    for (llvm::Use &Use : I.uses()) {
+      // Ensure all uses match the required pattern.
+      User *Shuffle = Use.getUser();
+      ArrayRef<int> Mask;
+
+      if (!match(Shuffle,
+                 m_Shuffle(m_Specific(OldLoad), m_Undef(), m_Mask(Mask))))
+        return std::nullopt;
+
+      // Ignore shufflevector instructions that have no uses.
+      if (Shuffle->use_empty())
+        continue;
+
+      // Find the min and max indices used by the shufflevector instruction.
+      for (int Index : Mask) {
+        if (Index >= 0 && Index < static_cast<int>(OldNumElements)) {
+          OutputRange.first = std::min(Index, OutputRange.first);
+          OutputRange.second = std::max(Index, OutputRange.second);
+        }
+      }
+    }
+
+    if (OutputRange.second < OutputRange.first)
+      return std::nullopt;
+
+    return OutputRange;
+  };
+
+  // Get the range of vector elements used by shufflevector instructions.
+  if (std::optional<IndexRange> Indices = GetIndexRangeInShuffles()) {
+    unsigned const NewNumElements = Indices->second + 1u;
+
+    // If the range of vector elements is smaller than the full load, attempt
+    // to create a smaller load.
+    if (NewNumElements < OldNumElements) {
+      IRBuilder Builder(&I);
+      Builder.SetCurrentDebugLocation(I.getDebugLoc());
+
+      // Calculate costs of old and new ops.
+      Type *ElemTy = OldLoadTy->getElementType();
+      FixedVectorType *NewLoadTy = FixedVectorType::get(ElemTy, NewNumElements);
+      Value *PtrOp = OldLoad->getPointerOperand();
+
+      InstructionCost OldCost = TTI.getMemoryOpCost(
+          Instruction::Load, OldLoad->getType(), OldLoad->getAlign(),
+          OldLoad->getPointerAddressSpace(), CostKind);
+      InstructionCost NewCost =
+          TTI.getMemoryOpCost(Instruction::Load, NewLoadTy, OldLoad->getAlign(),
+                              OldLoad->getPointerAddressSpace(), CostKind);
+
+      using UseEntry = std::pair<ShuffleVectorInst *, std::vector<int>>;
+      SmallVector<UseEntry, 4u> NewUses;
+      unsigned const MaxIndex = NewNumElements * 2u;
+
+      for (llvm::Use &Use : I.uses()) {
+        auto *Shuffle = cast<ShuffleVectorInst>(Use.getUser());
+        ArrayRef<int> OldMask = Shuffle->getShuffleMask();
+
+        // Create entry for new use.
+        NewUses.push_back({Shuffle, OldMask});
+
+        // Validate mask indices.
+        for (int Index : OldMask) {
+          if (Index >= static_cast<int>(MaxIndex))
+            return false;
+        }
+
+        // Update costs.
+        OldCost +=
+            TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, Shuffle->getType(),
+                               OldLoadTy, OldMask, CostKind);
+        NewCost +=
+            TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, Shuffle->getType(),
+                               NewLoadTy, OldMask, CostKind);
+      }
+
+      LLVM_DEBUG(
+          dbgs() << "Found a load used only by shufflevector instructions: "
+                 << I << "\n  OldCost: " << OldCost
+                 << " vs NewCost: " << NewCost << "\n");
+
+      if (OldCost < NewCost || !NewCost.isValid())
+        return false;
+
+      // Create new load of smaller vector.
+      auto *NewLoad = cast<LoadInst>(
+          Builder.CreateAlignedLoad(NewLoadTy, PtrOp, OldLoad->getAlign()));
+      NewLoad->copyMetadata(I);
+
+      // Replace all uses.
+      for (UseEntry &Use : NewUses) {
+        ShuffleVectorInst *Shuffle = Use.first;
+        std::vector<int> &NewMask = Use.second;
+
+        Builder.SetInsertPoint(Shuffle);
+        Builder.SetCurrentDebugLocation(Shuffle->getDebugLoc());
+        Value *NewShuffle = Builder.CreateShuffleVector(
+            NewLoad, PoisonValue::get(NewLoadTy), NewMask);
+
+        replaceValue(*Shuffle, *NewShuffle, false);
+      }
+
+      return true;
+    }
+  }
+  return false;
+}
+
+// Attempt to narrow a phi of shufflevector instructions where the two incoming
+// values have the same operands but different masks. If the two shuffle masks
+// are offsets of one another we can use one branch to rotate the incoming
+// vector and perform one larger shuffle after the phi.
+bool VectorCombine::shrinkPhiOfShuffles(Instruction &I) {
+  auto *Phi = dyn_cast<PHINode>(&I);
+  if (!Phi || Phi->getNumIncomingValues() != 2u)
+    return false;
+
+  Value *Op = nullptr;
+  ArrayRef<int> Mask0;
+  ArrayRef<int> Mask1;
+
+  if (!match(Phi->getOperand(0u),
+             m_OneUse(m_Shuffle(m_Value(Op), m_Poison(), m_Mask(Mask0)))) ||
+      !match(Phi->getOperand(1u),
+             m_OneUse(m_Shuffle(m_Specific(Op), m_Poison(), m_Mask(Mask1)))))
+    return false;
+
+  auto *Shuf = cast<ShuffleVectorInst>(Phi->getOperand(0u));
+
+  // Ensure result vectors are wider than the argument vector.
+  auto *InputVT = cast<FixedVectorType>(Op->getType());
+  auto *ResultVT = cast<FixedVectorType>(Shuf->getType());
+  auto const InputNumElements = InputVT->getNumElements();
+
+  if (InputNumElements >= ResultVT->getNumElements())
+    return false;
+
+  // Take the difference of the two shuffle masks at each index. Ignore poison
+  // values at the same index in both masks.
+  SmallVector<int, 16> NewMask;
+  NewMask.reserve(Mask0.size());
+
+  for (auto [M0, M1] : zip(Mask0, Mask1)) {
+    if (M0 >= 0 && M1 >= 0)
+      NewMask.push_back(M0 - M1);
+    else if (M0 == -1 && M1 == -1)
+      continue;
+    else
+      return false;
+  }
+
+  // Ensure all elements of the new mask are equal. If the difference between
+  // the incoming mask elements is the same, the two must be constant offsets
+  // of one another.
+  if (NewMask.empty() || !all_equal(NewMask))
+    return false;
+
+  // Create new mask using difference of the two incoming masks.
+  int MaskOffset = NewMask[0u];
+  unsigned Index = (InputNumElements - MaskOffset) % InputNumElements;
+  NewMask.clear();
+
+  for (unsigned I = 0u; I < InputNumElements; ++I) {
+    NewMask.push_back(Index);
+    Index = (Index + 1u) % InputNumElements;
+  }
+
+  // Calculate costs for worst cases and compare.
+  auto const Kind = TTI::SK_PermuteSingleSrc;
+  auto OldCost =
+      std::max(TTI.getShuffleCost(Kind, ResultVT, InputVT, Mask0, CostKind),
+               TTI.getShuffleCost(Kind, ResultVT, InputVT, Mask1, CostKind));
+  auto NewCost = TTI.getShuffleCost(Kind, InputVT, InputVT, NewMask, CostKind) +
+                 TTI.getShuffleCost(Kind, ResultVT, InputVT, Mask1, CostKind);
+
+  LLVM_DEBUG(dbgs() << "Found a phi of mergeable shuffles: " << I
+                    << "\n  OldCost: " << OldCost << " vs NewCost: " << NewCost
+                    << "\n");
+
+  if (NewCost > OldCost)
+    return false;
+
+  // Create new shuffles and narrowed phi.
+  auto Builder = IRBuilder(Shuf);
+  Builder.SetCurrentDebugLocation(Shuf->getDebugLoc());
+  auto *PoisonVal = PoisonValue::get(InputVT);
+  auto *NewShuf0 = Builder.CreateShuffleVector(Op, PoisonVal, NewMask);
+  Worklist.push(cast<Instruction>(NewShuf0));
+
+  Builder.SetInsertPoint(Phi);
+  Builder.SetCurrentDebugLocation(Phi->getDebugLoc());
+  auto *NewPhi = Builder.CreatePHI(NewShuf0->getType(), 2u);
+  NewPhi->addIncoming(NewShuf0, Phi->getIncomingBlock(0u));
+  NewPhi->addIncoming(Op, Phi->getIncomingBlock(1u));
+
+  Builder.SetInsertPoint(*NewPhi->getInsertionPointAfterDef());
+  PoisonVal = PoisonValue::get(NewPhi->getType());
+  auto *NewShuf1 = Builder.CreateShuffleVector(NewPhi, PoisonVal, Mask1);
+
+  replaceValue(*Phi, *NewShuf1);
+  return true;
+}
+
 /// This is the entry point for all transforms. Pass manager differences are
 /// handled in the callers of this function.
 bool VectorCombine::run() {
@@ -3873,8 +4372,7 @@ bool VectorCombine::run() {
 
   LLVM_DEBUG(dbgs() << "\n\nVECTORCOMBINE on " << F.getName() << "\n");
 
-  bool MadeChange = false;
-  auto FoldInst = [this, &MadeChange](Instruction &I) {
+  auto FoldInst = [this](Instruction &I) {
     Builder.SetInsertPoint(&I);
     bool IsVectorType = isa<VectorType>(I.getType());
     bool IsFixedVectorType = isa<FixedVectorType>(I.getType());
@@ -3889,10 +4387,12 @@ bool VectorCombine::run() {
     if (IsFixedVectorType) {
       switch (Opcode) {
       case Instruction::InsertElement:
-        MadeChange |= vectorizeLoadInsert(I);
+        if (vectorizeLoadInsert(I))
+          return true;
         break;
       case Instruction::ShuffleVector:
-        MadeChange |= widenSubvectorLoad(I);
+        if (widenSubvectorLoad(I))
+          return true;
         break;
       default:
         break;
@@ -3902,19 +4402,25 @@ bool VectorCombine::run() {
     // This transform works with scalable and fixed vectors
     // TODO: Identify and allow other scalable transforms
     if (IsVectorType) {
-      MadeChange |= scalarizeOpOrCmp(I);
-      MadeChange |= scalarizeLoadExtract(I);
-      MadeChange |= scalarizeExtExtract(I);
-      MadeChange |= scalarizeVPIntrinsic(I);
-      MadeChange |= foldInterleaveIntrinsics(I);
+      if (scalarizeOpOrCmp(I))
+        return true;
+      if (scalarizeLoadExtract(I))
+        return true;
+      if (scalarizeExtExtract(I))
+        return true;
+      if (scalarizeVPIntrinsic(I))
+        return true;
+      if (foldInterleaveIntrinsics(I))
+        return true;
     }
 
     if (Opcode == Instruction::Store)
-      MadeChange |= foldSingleElementStore(I);
+      if (foldSingleElementStore(I))
+        return true;
 
     // If this is an early pipeline invocation of this pass, we are done.
     if (TryEarlyFoldsOnly)
-      return;
+      return false;
 
     // Otherwise, try folds that improve codegen but may interfere with
     // early IR canonicalizations.
@@ -3923,56 +4429,91 @@ bool VectorCombine::run() {
     if (IsFixedVectorType) {
       switch (Opcode) {
       case Instruction::InsertElement:
-        MadeChange |= foldInsExtFNeg(I);
-        MadeChange |= foldInsExtBinop(I);
-        MadeChange |= foldInsExtVectorToShuffle(I);
+        if (foldInsExtFNeg(I))
+          return true;
+        if (foldInsExtBinop(I))
+          return true;
+        if (foldInsExtVectorToShuffle(I))
+          return true;
         break;
       case Instruction::ShuffleVector:
-        MadeChange |= foldPermuteOfBinops(I);
-        MadeChange |= foldShuffleOfBinops(I);
-        MadeChange |= foldShuffleOfSelects(I);
-        MadeChange |= foldShuffleOfCastops(I);
-        MadeChange |= foldShuffleOfShuffles(I);
-        MadeChange |= foldShuffleOfIntrinsics(I);
-        MadeChange |= foldSelectShuffle(I);
-        MadeChange |= foldShuffleToIdentity(I);
+        if (foldPermuteOfBinops(I))
+          return true;
+        if (foldShuffleOfBinops(I))
+          return true;
+        if (foldShuffleOfSelects(I))
+          return true;
+        if (foldShuffleOfCastops(I))
+          return true;
+        if (foldShuffleOfShuffles(I))
+          return true;
+        if (foldShuffleOfIntrinsics(I))
+          return true;
+        if (foldSelectShuffle(I))
+          return true;
+        if (foldShuffleToIdentity(I))
+          return true;
+        break;
+      case Instruction::Load:
+        if (shrinkLoadForShuffles(I))
+          return true;
         break;
       case Instruction::BitCast:
-        MadeChange |= foldBitcastShuffle(I);
+        if (foldBitcastShuffle(I))
+          return true;
         break;
       case Instruction::And:
       case Instruction::Or:
       case Instruction::Xor:
-        MadeChange |= foldBitOpOfCastops(I);
+        if (foldBitOpOfCastops(I))
+          return true;
+        break;
+      case Instruction::PHI:
+        if (shrinkPhiOfShuffles(I))
+          return true;
         break;
       default:
-        MadeChange |= shrinkType(I);
+        if (shrinkType(I))
+          return true;
         break;
       }
     } else {
       switch (Opcode) {
       case Instruction::Call:
-        MadeChange |= foldShuffleFromReductions(I);
-        MadeChange |= foldCastFromReductions(I);
+        if (foldShuffleFromReductions(I))
+          return true;
+        if (foldCastFromReductions(I))
+          return true;
+        break;
+      case Instruction::ExtractElement:
+        if (foldShuffleChainsToReduce(I))
+          return true;
         break;
       case Instruction::ICmp:
       case Instruction::FCmp:
-        MadeChange |= foldExtractExtract(I);
+        if (foldExtractExtract(I))
+          return true;
         break;
       case Instruction::Or:
-        MadeChange |= foldConcatOfBoolMasks(I);
+        if (foldConcatOfBoolMasks(I))
+          return true;
         [[fallthrough]];
       default:
         if (Instruction::isBinaryOp(Opcode)) {
-          MadeChange |= foldExtractExtract(I);
-          MadeChange |= foldExtractedCmps(I);
-          MadeChange |= foldBinopOfReductions(I);
+          if (foldExtractExtract(I))
+            return true;
+          if (foldExtractedCmps(I))
+            return true;
+          if (foldBinopOfReductions(I))
+            return true;
         }
         break;
       }
     }
+    return false;
   };
 
+  bool MadeChange = false;
   for (BasicBlock &BB : F) {
     // Ignore unreachable basic blocks.
     if (!DT.isReachableFromEntry(&BB))
@@ -3981,7 +4522,7 @@ bool VectorCombine::run() {
     for (Instruction &I : make_early_inc_range(BB)) {
       if (I.isDebugOrPseudoInst())
         continue;
-      FoldInst(I);
+      MadeChange |= FoldInst(I);
     }
   }
 
@@ -3995,7 +4536,7 @@ bool VectorCombine::run() {
       continue;
     }
 
-    FoldInst(*I);
+    MadeChange |= FoldInst(*I);
   }
 
   return MadeChange;
diff --git a/llvm/projects/CMakeLists.txt b/llvm/projects/CMakeLists.txt
index f254cf10806d7..f5f7c98826c2d 100644
--- a/llvm/projects/CMakeLists.txt
+++ b/llvm/projects/CMakeLists.txt
@@ -9,6 +9,7 @@ foreach(entry ${entries})
        (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/libcxx) AND
        (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/libcxxabi) AND
        (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/libunwind) AND
+       (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/orc-rt) AND
        (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/test-suite) AND
        (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/openmp) AND
        (NOT ${entry} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}/cross-project-tests) AND
@@ -34,6 +35,7 @@ if(${LLVM_BUILD_RUNTIME})
     add_llvm_external_project(libc)
     add_llvm_external_project(libcxxabi)
     add_llvm_external_project(libcxx)
+    add_llvm_external_project(orc-rt)
   endif()
   if(NOT LLVM_BUILD_EXTERNAL_COMPILER_RT)
     add_llvm_external_project(compiler-rt)
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 94a43b96d2188..86cfd0285aa84 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -482,8 +482,10 @@ if(build_runtimes)
                 llvm-lto
                 llvm-jitlink
                 llvm-nm
+                llvm-strip
                 llvm-objdump
                 llvm-profdata
+                llvm-readobj
                 llvm-size
                 llvm-symbolizer
                 llvm-xray
@@ -499,6 +501,9 @@ if(build_runtimes)
         list(APPEND extra_deps ${dep})
       endif()
     endforeach()
+    if(WIN32)
+      list(APPEND extra_deps KillTheDoctor)
+    endif()
   endif()
 
   # Forward user-provived system configuration to runtimes for requirement introspection.
diff --git a/llvm/test/Analysis/BasicAA/modref.ll b/llvm/test/Analysis/BasicAA/modref.ll
index 1aab28f3f1871..4a91feeafc6cd 100644
--- a/llvm/test/Analysis/BasicAA/modref.ll
+++ b/llvm/test/Analysis/BasicAA/modref.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=gvn,dse -S | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @external(ptr)
 
@@ -71,7 +71,7 @@ define void @test3(i8 %X) {
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:    [[P:%.*]] = alloca i64, align 8
 ; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P]], i32 2
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[P]])
 ; CHECK-NEXT:    store i8 2, ptr [[P2]], align 1
 ; CHECK-NEXT:    call void @external(ptr [[P]])
 ; CHECK-NEXT:    ret void
@@ -81,7 +81,7 @@ define void @test3(i8 %X) {
 
   %P2 = getelementptr i8, ptr %P, i32 2
   store i8 %Y, ptr %P2  ;; Not read by lifetime.end, should be removed.
-  call void @llvm.lifetime.end.p0(i64 1, ptr %P)
+  call void @llvm.lifetime.end.p0(ptr %P)
   store i8 2, ptr %P2
   call void @external(ptr %P)
   ret void
@@ -90,7 +90,7 @@ define void @test3(i8 %X) {
 define void @test3a(i8 %X) {
 ; CHECK-LABEL: @test3a(
 ; CHECK-NEXT:    [[P:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 10, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[P]])
 ; CHECK-NEXT:    ret void
 ;
   %P = alloca i64
@@ -98,7 +98,7 @@ define void @test3a(i8 %X) {
 
   %P2 = getelementptr i8, ptr %P, i32 2
   store i8 %Y, ptr %P2
-  call void @llvm.lifetime.end.p0(i64 10, ptr %P)
+  call void @llvm.lifetime.end.p0(ptr %P)
   ret void
 }
 
diff --git a/llvm/test/Analysis/BasicAA/phi-values-usage.ll b/llvm/test/Analysis/BasicAA/phi-values-usage.ll
index 43df41c916418..680e1df9c68b9 100644
--- a/llvm/test/Analysis/BasicAA/phi-values-usage.ll
+++ b/llvm/test/Analysis/BasicAA/phi-values-usage.ll
@@ -14,7 +14,7 @@ target datalayout = "p:8:8-n8"
 
 declare void @otherfn(ptr)
 declare i32 @__gxx_personality_v0(...)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 @c = external global ptr, align 1
 
 ; This function is one where if we didn't free basicaa after memcpyopt then the
@@ -65,7 +65,7 @@ for.body:                                         ; preds = %for.cond
   br label %for.cond
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 1, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   %1 = load ptr, ptr %d.0, align 1
   store ptr %1, ptr @c, align 1
   ret void
diff --git a/llvm/test/Analysis/CallGraph/ignore-assumelike-calls.ll b/llvm/test/Analysis/CallGraph/ignore-assumelike-calls.ll
index 1c9d20193869e..b93a2a0d505a7 100644
--- a/llvm/test/Analysis/CallGraph/ignore-assumelike-calls.ll
+++ b/llvm/test/Analysis/CallGraph/ignore-assumelike-calls.ll
@@ -29,7 +29,7 @@
 define internal void @used_by_lifetime() {
 entry:
   %a = alloca i8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   ret void
 }
 
@@ -55,6 +55,6 @@ define internal void @other_cast_intrinsic_use() {
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 declare void @llvm.memset.p0.i64(ptr, i8, i64, i1 immarg)
 declare void @llvm.memset.p1.i64(ptr addrspace(1), i8, i64, i1 immarg)
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll b/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
index de1b39db1539c..c208d03ff94b7 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-fp.ll
@@ -34,10 +34,10 @@ define void @fadd() {
 
 define void @fadd_fp16() {
 ; CHECK-BASE-LABEL: 'fadd_fp16'
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fadd half undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fadd <4 x half> undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fadd <8 x half> undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fadd <16 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fadd half undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fadd <4 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fadd <8 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fadd <16 x half> undef, undef
 ; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-FP16-LABEL: 'fadd_fp16'
@@ -84,10 +84,10 @@ define void @fsub() {
 
 define void @fsub_fp16() {
 ; CHECK-BASE-LABEL: 'fsub_fp16'
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fsub half undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fsub <4 x half> undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fsub <8 x half> undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fsub <16 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fsub half undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fsub <4 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fsub <8 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fsub <16 x half> undef, undef
 ; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-FP16-LABEL: 'fsub_fp16'
@@ -134,9 +134,9 @@ define void @fneg_idiom() {
 
 define void @fneg_idiom_fp16() {
 ; CHECK-BASE-LABEL: 'fneg_idiom_fp16'
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fsub half 0xH8000, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fsub <4 x half> splat (half 0xH8000), undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fsub <8 x half> splat (half 0xH8000), undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fsub half 0xH8000, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fsub <4 x half> splat (half 0xH8000), undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fsub <8 x half> splat (half 0xH8000), undef
 ; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-FP16-LABEL: 'fneg_idiom_fp16'
@@ -180,21 +180,13 @@ define void @fneg() {
 }
 
 define void @fneg_fp16() {
-; CHECK-BASE-LABEL: 'fneg_fp16'
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
-;
-; CHECK-FP16-LABEL: 'fneg_fp16'
-; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half undef
-; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> undef
-; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> undef
-; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> undef
-; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> undef
-; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-LABEL: 'fneg_fp16'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> undef
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %F16 = fneg half undef
   %V2F16 = fneg <2 x half> undef
@@ -252,16 +244,16 @@ define void @fmulfneg() {
 
 define void @fmulneg_fp16() {
 ; CHECK-BASE-LABEL: 'fmulneg_fp16'
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16M = fmul half %F16, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16M = fmul <2 x half> %V2F16, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16M = fmul <4 x half> %V4F16, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16M = fmul <8 x half> %V8F16, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16M = fmul <16 x half> %V16F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:1 Lat:3 SizeLat:1 for: %F16M = fmul half %F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16M = fmul <2 x half> %V2F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16M = fmul <4 x half> %V4F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16M = fmul <8 x half> %V8F16, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16M = fmul <16 x half> %V16F16, undef
 ; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-FP16-LABEL: 'fmulneg_fp16'
@@ -338,16 +330,16 @@ define void @fnegfmul() {
 
 define void @fnegfmul_fp16() {
 ; CHECK-BASE-LABEL: 'fnegfmul_fp16'
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16M = fmul half undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half %F16M
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16M = fmul <2 x half> undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> %V2F16M
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16M = fmul <4 x half> undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> %V4F16M
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16M = fmul <8 x half> undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> %V8F16M
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16M = fmul <16 x half> undef, undef
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> %V16F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:1 Lat:3 SizeLat:1 for: %F16M = fmul half undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fneg half %F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16M = fmul <2 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V2F16 = fneg <2 x half> %V2F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16M = fmul <4 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fneg <4 x half> %V4F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16M = fmul <8 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fneg <8 x half> %V8F16M
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16M = fmul <16 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fneg <16 x half> %V16F16M
 ; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-FP16-LABEL: 'fnegfmul_fp16'
@@ -405,12 +397,19 @@ define void @fmul() {
 }
 
 define void @fmul_fp16() {
-; CHECK-LABEL: 'fmul_fp16'
-; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fmul half undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fmul <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fmul <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fmul <16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-BASE-LABEL: 'fmul_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fmul half undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fmul <4 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fmul <8 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fmul <16 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-FP16-LABEL: 'fmul_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = fmul half undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = fmul <4 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = fmul <8 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = fmul <16 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %F16 = fmul half undef, undef
   %V4F16 = fmul <4 x half> undef, undef
@@ -448,12 +447,19 @@ define void @fdiv() {
 }
 
 define void @fdiv_fp16() {
-; CHECK-LABEL: 'fdiv_fp16'
-; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %F16 = fdiv half undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V4F16 = fdiv <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V8F16 = fdiv <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = fdiv <16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-BASE-LABEL: 'fdiv_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %F16 = fdiv half undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4F16 = fdiv <4 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V8F16 = fdiv <8 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V16F16 = fdiv <16 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-FP16-LABEL: 'fdiv_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %F16 = fdiv half undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V4F16 = fdiv <4 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %V8F16 = fdiv <8 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = fdiv <16 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %F16 = fdiv half undef, undef
   %V4F16 = fdiv <4 x half> undef, undef
@@ -491,12 +497,19 @@ define void @frem() {
 }
 
 define void @frem_fp16() {
-; CHECK-LABEL: 'frem_fp16'
-; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %F16 = frem half undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:52 CodeSize:4 Lat:4 SizeLat:4 for: %V4F16 = frem <4 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:108 CodeSize:4 Lat:4 SizeLat:4 for: %V8F16 = frem <8 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:216 CodeSize:4 Lat:4 SizeLat:4 for: %V16F16 = frem <16 x half> undef, undef
-; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; CHECK-BASE-LABEL: 'frem_fp16'
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:13 CodeSize:4 Lat:4 SizeLat:4 for: %F16 = frem half undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:55 CodeSize:4 Lat:4 SizeLat:4 for: %V4F16 = frem <4 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:110 CodeSize:4 Lat:4 SizeLat:4 for: %V8F16 = frem <8 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:220 CodeSize:4 Lat:4 SizeLat:4 for: %V16F16 = frem <16 x half> undef, undef
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-FP16-LABEL: 'frem_fp16'
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %F16 = frem half undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:52 CodeSize:4 Lat:4 SizeLat:4 for: %V4F16 = frem <4 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:108 CodeSize:4 Lat:4 SizeLat:4 for: %V8F16 = frem <8 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:216 CodeSize:4 Lat:4 SizeLat:4 for: %V16F16 = frem <16 x half> undef, undef
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %F16 = frem half undef, undef
   %V4F16 = frem <4 x half> undef, undef
@@ -650,13 +663,13 @@ define void @fcopysign_fp16() {
 
 define void @fma() {
 ; CHECK-LABEL: 'fma'
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F32 = call <2 x float> @llvm.fma.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F64 = call double @llvm.fma.f64(double undef, double undef, double undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = call <2 x float> @llvm.fma.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F64 = call double @llvm.fma.f64(double undef, double undef, double undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:1 Lat:10 SizeLat:10 for: %F128 = call fp128 @llvm.fma.f128(fp128 undef, fp128 undef, fp128 undef)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: %V2F128 = call <2 x fp128> @llvm.fma.v2f128(<2 x fp128> undef, <2 x fp128> undef, <2 x fp128> undef)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -685,10 +698,10 @@ define void @fma_fp16() {
 ; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-FP16-LABEL: 'fma_fp16'
-; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
-; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-; CHECK-FP16-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = call <4 x half> @llvm.fma.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = call <8 x half> @llvm.fma.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = call <16 x half> @llvm.fma.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
 ; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %F16 = call half @llvm.fma.f16(half undef, half undef, half undef)
@@ -700,13 +713,13 @@ define void @fma_fp16() {
 
 define void @fmuladd() {
 ; CHECK-LABEL: 'fmuladd'
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F32 = call float @llvm.fmuladd.f32(float undef, float undef, float undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F32 = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4F32 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8F32 = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %F64 = call double @llvm.fmuladd.f64(double undef, double undef, double undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2F64 = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4F64 = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F32 = call float @llvm.fmuladd.f32(float undef, float undef, float undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> undef, <2 x float> undef, <2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = call <8 x float> @llvm.fmuladd.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F64 = call double @llvm.fmuladd.f64(double undef, double undef, double undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = call <4 x double> @llvm.fmuladd.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:6 SizeLat:2 for: %F128 = call fp128 @llvm.fmuladd.f128(fp128 undef, fp128 undef, fp128 undef)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:2 Lat:6 SizeLat:2 for: %V2F128 = call <2 x fp128> @llvm.fmuladd.v2f128(<2 x fp128> undef, <2 x fp128> undef, <2 x fp128> undef)
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
@@ -729,16 +742,16 @@ define void @fmuladd() {
 define void @fmuladd_fp16() {
 ; CHECK-BASE-LABEL: 'fmuladd_fp16'
 ; CHECK-BASE-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.fmuladd.f16(half undef, half undef, half undef)
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:6 SizeLat:2 for: %V4F16 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:2 Lat:6 SizeLat:2 for: %V8F16 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:2 Lat:6 SizeLat:2 for: %V16F16 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:9 CodeSize:2 Lat:6 SizeLat:2 for: %V4F16 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:18 CodeSize:2 Lat:6 SizeLat:2 for: %V8F16 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:36 CodeSize:2 Lat:6 SizeLat:2 for: %V16F16 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
 ; CHECK-BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-FP16-LABEL: 'fmuladd_fp16'
-; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %F16 = call half @llvm.fmuladd.f16(half undef, half undef, half undef)
-; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V4F16 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
-; CHECK-FP16-NEXT:  Cost Model: Found costs of 1 for: %V8F16 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
-; CHECK-FP16-NEXT:  Cost Model: Found costs of 4 for: %V16F16 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %F16 = call half @llvm.fmuladd.f16(half undef, half undef, half undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = call <4 x half> @llvm.fmuladd.v4f16(<4 x half> undef, <4 x half> undef, <4 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = call <8 x half> @llvm.fmuladd.v8f16(<8 x half> undef, <8 x half> undef, <8 x half> undef)
+; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = call <16 x half> @llvm.fmuladd.v16f16(<16 x half> undef, <16 x half> undef, <16 x half> undef)
 ; CHECK-FP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %F16 = call half @llvm.fmuladd.f32(half undef, half undef, half undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-ssat.ll b/llvm/test/Analysis/CostModel/AArch64/arith-ssat.ll
index 254715ac909e1..0d1f20bdf8176 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-ssat.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-ssat.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s  --check-prefixes=COMMON,BASE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefixes=COMMON,SVE
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
@@ -31,60 +32,128 @@ declare <32 x i8>  @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>)
 declare <64 x i8>  @llvm.sadd.sat.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @add(i32 %arg) {
-; CHECK-LABEL: 'add'
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V3I32 = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> undef, <3 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 6 for: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 6 for: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; COMMON-LABEL: 'add'
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %I64 = call i64 @llvm.sadd.sat.i64(i64 poison, i64 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %I32 = call i32 @llvm.sadd.sat.i32(i32 poison, i32 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V3I32 = call <3 x i32> @llvm.sadd.sat.v3i32(<3 x i32> poison, <3 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 6 for: %I16 = call i16 @llvm.sadd.sat.i16(i16 poison, i16 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 6 for: %I8 = call i8 @llvm.sadd.sat.i8(i8 poison, i8 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
 ;
-  %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-  %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-  %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-  %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-
-  %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
-  %V2I32  = call <2 x i32>  @llvm.sadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-  %V4I32  = call <4 x i32>  @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-  %V8I32  = call <8 x i32>  @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-  %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-  %V3I32  = call <3 x i32>  @llvm.sadd.sat.v3i32(<3 x i32> undef, <3 x i32> undef)
-
-  %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
-  %V2I16  = call <2 x i16>  @llvm.sadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-  %V4I16  = call <4 x i16>  @llvm.sadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-  %V8I16  = call <8 x i16>  @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-  %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-  %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-
-  %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
-  %V2I8  = call <2 x i8>  @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-  %V4I8  = call <4 x i8>  @llvm.sadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-  %V8I8  = call <8 x i8>  @llvm.sadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-  %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-  %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-  %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-
-  ret i32 undef
+  %I64 = call i64 @llvm.sadd.sat.i64(i64 poison, i64 poison)
+  %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+  %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+  %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+
+  %I32 = call i32 @llvm.sadd.sat.i32(i32 poison, i32 poison)
+  %V2I32  = call <2 x i32>  @llvm.sadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+  %V4I32  = call <4 x i32>  @llvm.sadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+  %V8I32  = call <8 x i32>  @llvm.sadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+  %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+  %V3I32  = call <3 x i32>  @llvm.sadd.sat.v3i32(<3 x i32> poison, <3 x i32> poison)
+
+  %I16 = call i16 @llvm.sadd.sat.i16(i16 poison, i16 poison)
+  %V2I16  = call <2 x i16>  @llvm.sadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+  %V4I16  = call <4 x i16>  @llvm.sadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+  %V8I16  = call <8 x i16>  @llvm.sadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+  %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+  %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+
+  %I8 = call i8 @llvm.sadd.sat.i8(i8 poison, i8 poison)
+  %V2I8  = call <2 x i8>  @llvm.sadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+  %V4I8  = call <4 x i8>  @llvm.sadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+  %V8I8  = call <8 x i8>  @llvm.sadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+  %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+  %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+  %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+
+  ret i32 poison
+}
+
+define i32 @add_sve_vscale2(i32 %arg) vscale_range(2,2) {
+; BASE-LABEL: 'add_sve_vscale2'
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
+;
+; SVE-LABEL: 'add_sve_vscale2'
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.sadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.sadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
+;
+  %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+  %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+  %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+
+  %V2I32  = call <2 x i32>  @llvm.sadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+  %V4I32  = call <4 x i32>  @llvm.sadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+  %V8I32  = call <8 x i32>  @llvm.sadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+  %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+
+  %V2I16  = call <2 x i16>  @llvm.sadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+  %V4I16  = call <4 x i16>  @llvm.sadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+  %V8I16  = call <8 x i16>  @llvm.sadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+  %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+  %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+
+  %V2I8  = call <2 x i8>  @llvm.sadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+  %V4I8  = call <4 x i8>  @llvm.sadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+  %V8I8  = call <8 x i8>  @llvm.sadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+  %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+  %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+  %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+
+  ret i32 poison
 }
 
 declare i64        @llvm.ssub.sat.i64(i64, i64)
@@ -114,56 +183,124 @@ declare <32 x i8>  @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>)
 declare <64 x i8>  @llvm.ssub.sat.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @sub(i32 %arg) {
-; CHECK-LABEL: 'sub'
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 6 for: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 6 for: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; COMMON-LABEL: 'sub'
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %I64 = call i64 @llvm.ssub.sat.i64(i64 poison, i64 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %I32 = call i32 @llvm.ssub.sat.i32(i32 poison, i32 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 6 for: %I16 = call i16 @llvm.ssub.sat.i16(i16 poison, i16 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 6 for: %I8 = call i8 @llvm.ssub.sat.i8(i8 poison, i8 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
 ;
-  %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-  %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-  %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-  %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-
-  %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
-  %V2I32  = call <2 x i32>  @llvm.ssub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-  %V4I32  = call <4 x i32>  @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-  %V8I32  = call <8 x i32>  @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-  %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-
-  %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
-  %V2I16  = call <2 x i16>  @llvm.ssub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-  %V4I16  = call <4 x i16>  @llvm.ssub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-  %V8I16  = call <8 x i16>  @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-  %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-  %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-
-  %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
-  %V2I8  = call <2 x i8>  @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-  %V4I8  = call <4 x i8>  @llvm.ssub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-  %V8I8  = call <8 x i8>  @llvm.ssub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-  %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-  %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-  %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-
-  ret i32 undef
+  %I64 = call i64 @llvm.ssub.sat.i64(i64 poison, i64 poison)
+  %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+  %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+  %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+
+  %I32 = call i32 @llvm.ssub.sat.i32(i32 poison, i32 poison)
+  %V2I32  = call <2 x i32>  @llvm.ssub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+  %V4I32  = call <4 x i32>  @llvm.ssub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+  %V8I32  = call <8 x i32>  @llvm.ssub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+  %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+
+  %I16 = call i16 @llvm.ssub.sat.i16(i16 poison, i16 poison)
+  %V2I16  = call <2 x i16>  @llvm.ssub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+  %V4I16  = call <4 x i16>  @llvm.ssub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+  %V8I16  = call <8 x i16>  @llvm.ssub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+  %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+  %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+
+  %I8 = call i8 @llvm.ssub.sat.i8(i8 poison, i8 poison)
+  %V2I8  = call <2 x i8>  @llvm.ssub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+  %V4I8  = call <4 x i8>  @llvm.ssub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+  %V8I8  = call <8 x i8>  @llvm.ssub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+  %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+  %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+  %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+
+  ret i32 poison
+}
+
+define i32 @sub_sve_vscale2(i32 %arg) vscale_range(2,2) {
+; BASE-LABEL: 'sub_sve_vscale2'
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
+;
+; SVE-LABEL: 'sub_sve_vscale2'
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.ssub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.ssub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
+;
+  %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+  %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+  %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+
+  %V2I32  = call <2 x i32>  @llvm.ssub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+  %V4I32  = call <4 x i32>  @llvm.ssub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+  %V8I32  = call <8 x i32>  @llvm.ssub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+  %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+
+  %V2I16  = call <2 x i16>  @llvm.ssub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+  %V4I16  = call <4 x i16>  @llvm.ssub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+  %V8I16  = call <8 x i16>  @llvm.ssub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+  %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+  %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+
+  %V2I8  = call <2 x i8>  @llvm.ssub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+  %V4I8  = call <4 x i8>  @llvm.ssub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+  %V8I8  = call <8 x i8>  @llvm.ssub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+  %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+  %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+  %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+
+  ret i32 poison
 }
diff --git a/llvm/test/Analysis/CostModel/AArch64/arith-usat.ll b/llvm/test/Analysis/CostModel/AArch64/arith-usat.ll
index dba42b1e6ebb0..032a4480d11e6 100644
--- a/llvm/test/Analysis/CostModel/AArch64/arith-usat.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/arith-usat.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=COMMON,BASE
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=aarch64--linux-gnu -mattr=+sve < %s | FileCheck %s --check-prefixes=COMMON,SVE
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
@@ -30,58 +31,126 @@ declare <32 x i8>  @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>)
 declare <64 x i8>  @llvm.uadd.sat.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @add(i32 %arg) {
-; CHECK-LABEL: 'add'
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; COMMON-LABEL: 'add'
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %I64 = call i64 @llvm.uadd.sat.i64(i64 poison, i64 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %I32 = call i32 @llvm.uadd.sat.i32(i32 poison, i32 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %I16 = call i16 @llvm.uadd.sat.i16(i16 poison, i16 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %I8 = call i8 @llvm.uadd.sat.i8(i8 poison, i8 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
 ;
-  %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-  %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-  %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-  %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-
-  %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
-  %V2I32  = call <2 x i32>  @llvm.uadd.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-  %V4I32  = call <4 x i32>  @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-  %V8I32  = call <8 x i32>  @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-  %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-
-  %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
-  %V2I16  = call <2 x i16>  @llvm.uadd.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-  %V4I16  = call <4 x i16>  @llvm.uadd.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-  %V8I16  = call <8 x i16>  @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-  %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-  %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-
-  %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
-  %V2I8  = call <2 x i8>  @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-  %V4I8  = call <4 x i8>  @llvm.uadd.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-  %V8I8  = call <8 x i8>  @llvm.uadd.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-  %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-  %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-  %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-
-  ret i32 undef
+  %I64 = call i64 @llvm.uadd.sat.i64(i64 poison, i64 poison)
+  %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+  %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+  %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+
+  %I32 = call i32 @llvm.uadd.sat.i32(i32 poison, i32 poison)
+  %V2I32  = call <2 x i32>  @llvm.uadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+  %V4I32  = call <4 x i32>  @llvm.uadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+  %V8I32  = call <8 x i32>  @llvm.uadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+  %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+
+  %I16 = call i16 @llvm.uadd.sat.i16(i16 poison, i16 poison)
+  %V2I16  = call <2 x i16>  @llvm.uadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+  %V4I16  = call <4 x i16>  @llvm.uadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+  %V8I16  = call <8 x i16>  @llvm.uadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+  %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+  %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+
+  %I8 = call i8 @llvm.uadd.sat.i8(i8 poison, i8 poison)
+  %V2I8  = call <2 x i8>  @llvm.uadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+  %V4I8  = call <4 x i8>  @llvm.uadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+  %V8I8  = call <8 x i8>  @llvm.uadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+  %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+  %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+  %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+
+  ret i32 poison
+}
+
+define i32 @add_sve_vscale2(i32 %arg) vscale_range(2,2) {
+; BASE-LABEL: 'add_sve_vscale2'
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
+;
+; SVE-LABEL: 'add_sve_vscale2'
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.uadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
+;
+  %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+  %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+  %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+
+  %V2I32  = call <2 x i32>  @llvm.uadd.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+  %V4I32  = call <4 x i32>  @llvm.uadd.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+  %V8I32  = call <8 x i32>  @llvm.uadd.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+  %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+
+  %V2I16  = call <2 x i16>  @llvm.uadd.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+  %V4I16  = call <4 x i16>  @llvm.uadd.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+  %V8I16  = call <8 x i16>  @llvm.uadd.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+  %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+  %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+
+  %V2I8  = call <2 x i8>  @llvm.uadd.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+  %V4I8  = call <4 x i8>  @llvm.uadd.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+  %V8I8  = call <8 x i8>  @llvm.uadd.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+  %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+  %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+  %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+
+  ret i32 poison
 }
 
 declare i64        @llvm.usub.sat.i64(i64, i64)
@@ -111,56 +180,124 @@ declare <32 x i8>  @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>)
 declare <64 x i8>  @llvm.usub.sat.v64i8(<64 x i8>, <64 x i8>)
 
 define i32 @sub(i32 %arg) {
-; CHECK-LABEL: 'sub'
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; COMMON-LABEL: 'sub'
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %I64 = call i64 @llvm.usub.sat.i64(i64 poison, i64 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %I32 = call i32 @llvm.usub.sat.i32(i32 poison, i32 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %I16 = call i16 @llvm.usub.sat.i16(i16 poison, i16 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %I8 = call i8 @llvm.usub.sat.i8(i8 poison, i8 poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; COMMON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
 ;
-  %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-  %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-  %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-  %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
-
-  %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
-  %V2I32  = call <2 x i32>  @llvm.usub.sat.v2i32(<2 x i32> undef, <2 x i32> undef)
-  %V4I32  = call <4 x i32>  @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
-  %V8I32  = call <8 x i32>  @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
-  %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
-
-  %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
-  %V2I16  = call <2 x i16>  @llvm.usub.sat.v2i16(<2 x i16> undef, <2 x i16> undef)
-  %V4I16  = call <4 x i16>  @llvm.usub.sat.v4i16(<4 x i16> undef, <4 x i16> undef)
-  %V8I16  = call <8 x i16>  @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
-  %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
-  %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
-
-  %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
-  %V2I8  = call <2 x i8>  @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
-  %V4I8  = call <4 x i8>  @llvm.usub.sat.v4i8(<4 x i8> undef, <4 x i8> undef)
-  %V8I8  = call <8 x i8>  @llvm.usub.sat.v8i8(<8 x i8> undef, <8 x i8> undef)
-  %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
-  %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
-  %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
-
-  ret i32 undef
+  %I64 = call i64 @llvm.usub.sat.i64(i64 poison, i64 poison)
+  %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+  %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+  %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+
+  %I32 = call i32 @llvm.usub.sat.i32(i32 poison, i32 poison)
+  %V2I32  = call <2 x i32>  @llvm.usub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+  %V4I32  = call <4 x i32>  @llvm.usub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+  %V8I32  = call <8 x i32>  @llvm.usub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+  %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+
+  %I16 = call i16 @llvm.usub.sat.i16(i16 poison, i16 poison)
+  %V2I16  = call <2 x i16>  @llvm.usub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+  %V4I16  = call <4 x i16>  @llvm.usub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+  %V8I16  = call <8 x i16>  @llvm.usub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+  %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+  %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+
+  %I8 = call i8 @llvm.usub.sat.i8(i8 poison, i8 poison)
+  %V2I8  = call <2 x i8>  @llvm.usub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+  %V4I8  = call <4 x i8>  @llvm.usub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+  %V8I8  = call <8 x i8>  @llvm.usub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+  %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+  %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+  %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+
+  ret i32 poison
+}
+
+define i32 @sub_sve_vscale2(i32 %arg) vscale_range(2,2) {
+; BASE-LABEL: 'sub_sve_vscale2'
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 2 for: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of 4 for: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; BASE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
+;
+; SVE-LABEL: 'sub_sve_vscale2'
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V2I32 = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V2I16 = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V4I16 = call <4 x i16> @llvm.usub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V2I8 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 4 for: %V4I8 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V8I8 = call <8 x i8> @llvm.usub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 1 for: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of 2 for: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+; SVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 poison
+;
+  %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> poison, <2 x i64> poison)
+  %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> poison, <4 x i64> poison)
+  %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> poison, <8 x i64> poison)
+
+  %V2I32  = call <2 x i32>  @llvm.usub.sat.v2i32(<2 x i32> poison, <2 x i32> poison)
+  %V4I32  = call <4 x i32>  @llvm.usub.sat.v4i32(<4 x i32> poison, <4 x i32> poison)
+  %V8I32  = call <8 x i32>  @llvm.usub.sat.v8i32(<8 x i32> poison, <8 x i32> poison)
+  %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> poison, <16 x i32> poison)
+
+  %V2I16  = call <2 x i16>  @llvm.usub.sat.v2i16(<2 x i16> poison, <2 x i16> poison)
+  %V4I16  = call <4 x i16>  @llvm.usub.sat.v4i16(<4 x i16> poison, <4 x i16> poison)
+  %V8I16  = call <8 x i16>  @llvm.usub.sat.v8i16(<8 x i16> poison, <8 x i16> poison)
+  %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> poison, <16 x i16> poison)
+  %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> poison, <32 x i16> poison)
+
+  %V2I8  = call <2 x i8>  @llvm.usub.sat.v2i8(<2 x i8> poison, <2 x i8> poison)
+  %V4I8  = call <4 x i8>  @llvm.usub.sat.v4i8(<4 x i8> poison, <4 x i8> poison)
+  %V8I8  = call <8 x i8>  @llvm.usub.sat.v8i8(<8 x i8> poison, <8 x i8> poison)
+  %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> poison, <16 x i8> poison)
+  %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> poison, <32 x i8> poison)
+  %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> poison, <64 x i8> poison)
+
+  ret i32 poison
 }
diff --git a/llvm/test/Analysis/CostModel/AArch64/extract_float.ll b/llvm/test/Analysis/CostModel/AArch64/extract_float.ll
index d2b75faa014d6..c2140218b0c54 100644
--- a/llvm/test/Analysis/CostModel/AArch64/extract_float.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/extract_float.ll
@@ -11,6 +11,7 @@ define double @extract_case1(<2 x double> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = extractelement <2 x double> %a, i32 1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+;
 entry:
   %1 = extractelement <2 x double> %a, i32 0
   %2 = extractelement <2 x double> %a, i32 1
@@ -24,6 +25,7 @@ define double @extract_case2(<2 x double> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %0 = extractelement <2 x double> %a, i32 1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+;
 entry:
   %1 = extractelement <2 x double> %a, i32 1
   %res = fmul double %1, %1
@@ -36,6 +38,7 @@ define double @extract_case3(<2 x double> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x double> %a, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+;
 entry:
   %1 = extractelement <2 x double> %a, i32 0
   %res = fmul double %1, %1
@@ -48,6 +51,7 @@ define double @extract_case4(<2 x double> %a, double %b) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x double> %a, i32 0
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %b
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+;
 entry:
   %1 = extractelement <2 x double> %a, i32 0
   %res = fmul double %1, %b
@@ -60,6 +64,7 @@ define double @extract_case5(<2 x double> %a, double %b) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x double> %a, i32 1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %b
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+;
 entry:
   %1 = extractelement <2 x double> %a, i32 1
   %res = fmul double %1, %b
@@ -74,6 +79,7 @@ define double @extract_case6(<3 x double> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = extractelement <3 x double> %a, i32 1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+;
 entry:
   %1 = extractelement <3 x double> %a, i32 0
   %2 = extractelement <3 x double> %a, i32 1
@@ -90,6 +96,7 @@ define double @extract_case7(<4 x double> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = extractelement <4 x double> %a, i32 2
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul double %0, %1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+;
 entry:
   %1 = extractelement <4 x double> %a, i32 1
   %2 = extractelement <4 x double> %a, i32 2
@@ -108,6 +115,7 @@ define double @extract_case8(<2 x double> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = fmul double %0, %1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = fmul double %3, %4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %5
+;
 entry:
   %1 = extractelement <2 x double> %a, i32 0
   %2 = extractelement <2 x double> %a, i32 1
@@ -129,6 +137,7 @@ define double @extract_case9(<2 x double> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = fmul double %0, %1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = fmul double %3, %4
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %5
+;
 entry:
   %1 = extractelement <2 x double> %a, i32 0
   %2 = extractelement <2 x double> %a, i32 1
@@ -148,6 +157,7 @@ define double @extract_case10(<4 x double> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: call void @foo(double %1)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = fmul double %0, %1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %2
+;
 entry:
   %1 = extractelement <4 x double> %a, i32 0
   %2 = extractelement <4 x double> %a, i32 1
@@ -161,7 +171,7 @@ define half @extract_case11(<2 x half> %a) {
 ; NOFP16-LABEL: 'extract_case11'
 ; NOFP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %0 = extractelement <2 x half> %a, i32 0
 ; NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <2 x half> %a, i32 1
-; NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul half %0, %1
+; NOFP16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = fmul half %0, %1
 ; NOFP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %res
 ;
 ; FULLFP16-LABEL: 'extract_case11'
@@ -169,6 +179,7 @@ define half @extract_case11(<2 x half> %a) {
 ; FULLFP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = extractelement <2 x half> %a, i32 1
 ; FULLFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul half %0, %1
 ; FULLFP16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret half %res
+;
 entry:
   %1 = extractelement <2 x half> %a, i32 0
   %2 = extractelement <2 x half> %a, i32 1
@@ -183,6 +194,7 @@ define float @extract_case12(<2 x float> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %1 = extractelement <2 x float> %a, i32 1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %res = fmul float %0, %1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret float %res
+;
 entry:
   %1 = extractelement <2 x float> %a, i32 0
   %2 = extractelement <2 x float> %a, i32 1
@@ -198,6 +210,7 @@ define double @extract_case13(<2 x double> %a) {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = extractelement <2 x double> %a, i32 1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = fadd double %0, %1
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret double %res
+;
 entry:
   %1 = extractelement <2 x double> %a, i32 0
   %2 = extractelement <2 x double> %a, i32 1
diff --git a/llvm/test/Analysis/CostModel/AArch64/histograms.ll b/llvm/test/Analysis/CostModel/AArch64/histograms.ll
new file mode 100644
index 0000000000000..c0489587551b0
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AArch64/histograms.ll
@@ -0,0 +1,178 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu | FileCheck %s --check-prefix=CHECK-NEON
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve | FileCheck %s --check-prefix=CHECK-SVE
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=throughput 2>&1 -disable-output -S -mtriple=aarch64--linux-gnu -mattr=+sve2 | FileCheck %s --check-prefix=CHECK-SVE2
+
+define void @histograms() {
+; CHECK-NEON-LABEL: 'histograms'
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-SVE-LABEL: 'histograms'
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-SVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-SVE2-LABEL: 'histograms'
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umax.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Invalid cost for instruction: call void @llvm.experimental.vector.histogram.umin.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+; CHECK-SVE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+  call void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+  call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+  call void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+  call void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+  call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+  call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+  call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+  call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+  call void @llvm.experimental.vector.histogram.uadd.sat.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+  call void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+  call void @llvm.experimental.vector.histogram.uadd.sat.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+  call void @llvm.experimental.vector.histogram.uadd.sat.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+  call void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+  call void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+  call void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+  call void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umax.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umax.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umax.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umax.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umin.nxv2p0.i64(<vscale x 2 x ptr> poison, i64 1, <vscale x 2 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umin.nxv4p0.i32(<vscale x 4 x ptr> poison, i32 1, <vscale x 4 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umin.nxv8p0.i16(<vscale x 8 x ptr> poison, i16 1, <vscale x 8 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umin.nxv16p0.i8(<vscale x 16 x ptr> poison, i8 1, <vscale x 16 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr> poison, i64 1, <2 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr> poison, i32 1, <4 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr> poison, i16 1, <8 x i1> poison)
+  call void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr> poison, i8 1, <16 x i1> poison)
+  ret void
+}
+
+declare void @llvm.experimental.vector.histogram.add.nxv2p0.i64(<vscale x 2 x ptr>, i64, <vscale x 2 x i1>)
+declare void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>)
+declare void @llvm.experimental.vector.histogram.add.nxv8p0.i16(<vscale x 8 x ptr>, i16, <vscale x 8 x i1>)
+declare void @llvm.experimental.vector.histogram.add.nxv16p0.i8(<vscale x 16 x ptr>, i8, <vscale x 16 x i1>)
+declare void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr>, i64, <2 x i1>)
+declare void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr>, i32, <4 x i1>)
+declare void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr>, i16, <8 x i1>)
+declare void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr>, i8, <16 x i1>)
+declare void @llvm.experimental.vector.histogram.uadd.sat.nxv2p0.i64(<vscale x 2 x ptr>, i64, <vscale x 2 x i1>)
+declare void @llvm.experimental.vector.histogram.uadd.sat.nxv4p0.i32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>)
+declare void @llvm.experimental.vector.histogram.uadd.sat.nxv8p0.i16(<vscale x 8 x ptr>, i16, <vscale x 8 x i1>)
+declare void @llvm.experimental.vector.histogram.uadd.sat.nxv16p0.i8(<vscale x 16 x ptr>, i8, <vscale x 16 x i1>)
+declare void @llvm.experimental.vector.histogram.uadd.sat.v2p0.i64(<2 x ptr>, i64, <2 x i1>)
+declare void @llvm.experimental.vector.histogram.uadd.sat.v4p0.i32(<4 x ptr>, i32, <4 x i1>)
+declare void @llvm.experimental.vector.histogram.uadd.sat.v8p0.i16(<8 x ptr>, i16, <8 x i1>)
+declare void @llvm.experimental.vector.histogram.uadd.sat.v16p0.i8(<16 x ptr>, i8, <16 x i1>)
+declare void @llvm.experimental.vector.histogram.umax.nxv2p0.i64(<vscale x 2 x ptr>, i64, <vscale x 2 x i1>)
+declare void @llvm.experimental.vector.histogram.umax.nxv4p0.i32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>)
+declare void @llvm.experimental.vector.histogram.umax.nxv8p0.i16(<vscale x 8 x ptr>, i16, <vscale x 8 x i1>)
+declare void @llvm.experimental.vector.histogram.umax.nxv16p0.i8(<vscale x 16 x ptr>, i8, <vscale x 16 x i1>)
+declare void @llvm.experimental.vector.histogram.umax.v2p0.i64(<2 x ptr>, i64, <2 x i1>)
+declare void @llvm.experimental.vector.histogram.umax.v4p0.i32(<4 x ptr>, i32, <4 x i1>)
+declare void @llvm.experimental.vector.histogram.umax.v8p0.i16(<8 x ptr>, i16, <8 x i1>)
+declare void @llvm.experimental.vector.histogram.umax.v16p0.i8(<16 x ptr>, i8, <16 x i1>)
+declare void @llvm.experimental.vector.histogram.umin.nxv2p0.i64(<vscale x 2 x ptr>, i64, <vscale x 2 x i1>)
+declare void @llvm.experimental.vector.histogram.umin.nxv4p0.i32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>)
+declare void @llvm.experimental.vector.histogram.umin.nxv8p0.i16(<vscale x 8 x ptr>, i16, <vscale x 8 x i1>)
+declare void @llvm.experimental.vector.histogram.umin.nxv16p0.i8(<vscale x 16 x ptr>, i8, <vscale x 16 x i1>)
+declare void @llvm.experimental.vector.histogram.umin.v2p0.i64(<2 x ptr>, i64, <2 x i1>)
+declare void @llvm.experimental.vector.histogram.umin.v4p0.i32(<4 x ptr>, i32, <4 x i1>)
+declare void @llvm.experimental.vector.histogram.umin.v8p0.i16(<8 x ptr>, i16, <8 x i1>)
+declare void @llvm.experimental.vector.histogram.umin.v16p0.i8(<16 x ptr>, i8, <16 x i1>)
diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-fadd.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-fadd.ll
index f565924a325a3..c4236d2cf41b8 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-fadd.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-fadd.ll
@@ -26,10 +26,10 @@ define void @strict_fp_reductions() {
 
 define void @strict_fp_reductions_fp16() {
 ; CHECK-NOFP16-LABEL: 'strict_fp_reductions_fp16'
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:5 Lat:10 SizeLat:6 for: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:18 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:23 Lat:46 SizeLat:30 for: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:76 CodeSize:46 Lat:92 SizeLat:60 for: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:5 Lat:10 SizeLat:6 for: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:26 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:54 CodeSize:23 Lat:46 SizeLat:30 for: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:108 CodeSize:46 Lat:92 SizeLat:60 for: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
 ; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-F16-LABEL: 'strict_fp_reductions_fp16'
@@ -40,10 +40,10 @@ define void @strict_fp_reductions_fp16() {
 ; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-BF16-LABEL: 'strict_fp_reductions_fp16'
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:5 Lat:10 SizeLat:6 for: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:18 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:23 Lat:46 SizeLat:30 for: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:76 CodeSize:46 Lat:92 SizeLat:60 for: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:5 Lat:10 SizeLat:6 for: %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:26 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f16 = call half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:54 CodeSize:23 Lat:46 SizeLat:30 for: %fadd_v8f16 = call half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:108 CodeSize:46 Lat:92 SizeLat:60 for: %fadd_v16f16 = call half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
 ; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v2f16 = call half @llvm.vector.reduce.fadd.v2f16(half 0.0, <2 x half> undef)
@@ -55,15 +55,15 @@ define void @strict_fp_reductions_fp16() {
 
 define void @strict_fp_reductions_bf16() {
 ; CHECK-NOFP16-LABEL: 'strict_fp_reductions_bf16'
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:18 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:54 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
 ; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-F16-LABEL: 'strict_fp_reductions_bf16'
-; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:18 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:54 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
 ; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-BF16-LABEL: 'strict_fp_reductions_bf16'
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:26 CodeSize:11 Lat:22 SizeLat:14 for: %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR0000, <4 x bfloat> undef)
 ; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v4f8 = call bfloat @llvm.vector.reduce.fadd.v4f8(bfloat 0.0, <4 x bfloat> undef)
@@ -117,16 +117,16 @@ define void @fast_fp_reductions() {
 
 define void @fast_fp_reductions_fp16() {
 ; CHECK-NOFP16-LABEL: 'fast_fp_reductions_fp16'
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:72 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:72 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:203 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:248 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
 ; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-F16-LABEL: 'fast_fp_reductions_fp16'
@@ -143,16 +143,16 @@ define void @fast_fp_reductions_fp16() {
 ; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-BF16-LABEL: 'fast_fp_reductions_fp16'
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:5 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:30 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:38 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:7 CodeSize:4 Lat:6 SizeLat:4 for: %fadd_v2f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v2f16(half 0xH0000, <2 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_fast = call fast half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v4f16(half 0xH0000, <4 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16 = call fast half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:27 Lat:33 SizeLat:27 for: %fadd_v8f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v8f16(half 0xH0000, <8 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:72 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16 = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:72 CodeSize:44 Lat:52 SizeLat:44 for: %fadd_v16f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:203 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v11f16 = call fast half @llvm.vector.reduce.fadd.v11f16(half 0xH0000, <11 x half> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:248 CodeSize:35 Lat:41 SizeLat:35 for: %fadd_v13f16_reassoc = call reassoc half @llvm.vector.reduce.fadd.v13f16(half 0xH0000, <13 x half> undef)
 ; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v2f16_fast = call fast half @llvm.vector.reduce.fadd.v2f16(half 0.0, <2 x half> undef)
@@ -175,15 +175,15 @@ define void @fast_fp_reductions_fp16() {
 
 define void @fast_fp_reductions_bf16() {
 ; CHECK-NOFP16-LABEL: 'fast_fp_reductions_bf16'
-; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
+; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
 ; CHECK-NOFP16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-F16-LABEL: 'fast_fp_reductions_bf16'
-; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
+; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:28 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
 ; CHECK-F16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-BF16-LABEL: 'fast_fp_reductions_bf16'
-; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
+; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:14 CodeSize:8 Lat:12 SizeLat:8 for: %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4bf16(bfloat 0xR8000, <4 x bfloat> undef)
 ; CHECK-BF16-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %fadd_v4f8 = call reassoc bfloat @llvm.vector.reduce.fadd.v4f8(bfloat -0.0, <4 x bfloat> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-arith-fp.ll b/llvm/test/Analysis/CostModel/AArch64/sve-arith-fp.ll
index dc95eacca28d4..1c40354892191 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-arith-fp.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-arith-fp.ll
@@ -164,3 +164,55 @@ define void @frem() {
 
   ret void
 }
+
+define void @fma() {
+; CHECK-LABEL: 'fma'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = call <vscale x 4 x half> @llvm.fma.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = call <vscale x 8 x half> @llvm.fma.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = call <vscale x 16 x half> @llvm.fma.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = call <vscale x 2 x float> @llvm.fma.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = call <vscale x 8 x float> @llvm.fma.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = call <vscale x 4 x double> @llvm.fma.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %V4F16 = call <vscale x 4 x half> @llvm.fma.v4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
+  %V8F16 = call <vscale x 8 x half> @llvm.fma.v8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
+  %V16F16 = call <vscale x 16 x half> @llvm.fma.v16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
+
+  %V2F32 = call <vscale x 2 x float> @llvm.fma.v2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x float> undef)
+  %V4F32 = call <vscale x 4 x float> @llvm.fma.v4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x float> undef)
+  %V8F32 = call <vscale x 8 x float> @llvm.fma.v8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x float> undef)
+
+  %V2F64 = call <vscale x 2 x double> @llvm.fma.v2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x double> undef)
+  %V4F64 = call <vscale x 4 x double> @llvm.fma.v4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x double> undef)
+
+  ret void
+}
+
+define void @fmuladd() {
+; CHECK-LABEL: 'fmuladd'
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F16 = call <vscale x 4 x half> @llvm.fmuladd.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V8F16 = call <vscale x 8 x half> @llvm.fmuladd.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V16F16 = call <vscale x 16 x half> @llvm.fmuladd.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F32 = call <vscale x 2 x float> @llvm.fmuladd.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V4F32 = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V8F32 = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %V2F64 = call <vscale x 2 x double> @llvm.fmuladd.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %V4F64 = call <vscale x 4 x double> @llvm.fmuladd.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+  %V4F16 = call <vscale x 4 x half> @llvm.fmuladd.v4f16(<vscale x 4 x half> undef, <vscale x 4 x half> undef, <vscale x 4 x half> undef)
+  %V8F16 = call <vscale x 8 x half> @llvm.fmuladd.v8f16(<vscale x 8 x half> undef, <vscale x 8 x half> undef, <vscale x 8 x half> undef)
+  %V16F16 = call <vscale x 16 x half> @llvm.fmuladd.v16f16(<vscale x 16 x half> undef, <vscale x 16 x half> undef, <vscale x 16 x half> undef)
+
+  %V2F32 = call <vscale x 2 x float> @llvm.fmuladd.v2f32(<vscale x 2 x float> undef, <vscale x 2 x float> undef, <vscale x 2 x float> undef)
+  %V4F32 = call <vscale x 4 x float> @llvm.fmuladd.v4f32(<vscale x 4 x float> undef, <vscale x 4 x float> undef, <vscale x 4 x float> undef)
+  %V8F32 = call <vscale x 8 x float> @llvm.fmuladd.v8f32(<vscale x 8 x float> undef, <vscale x 8 x float> undef, <vscale x 8 x float> undef)
+
+  %V2F64 = call <vscale x 2 x double> @llvm.fmuladd.v2f64(<vscale x 2 x double> undef, <vscale x 2 x double> undef, <vscale x 2 x double> undef)
+  %V4F64 = call <vscale x 4 x double> @llvm.fmuladd.v4f64(<vscale x 4 x double> undef, <vscale x 4 x double> undef, <vscale x 4 x double> undef)
+
+  ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
index 7e8d957efc383..609a23bc07938 100644
--- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll
@@ -1277,15 +1277,15 @@ define void @histogram_nxv16i8(<vscale x 16 x ptr> %buckets, <vscale x 16 x i1>
 
 define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) {
 ; CHECK-VSCALE-1-LABEL: 'histogram_v2i64'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:10 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_v2i64'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:10 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v2i64'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:10 CodeSize:8 Lat:10 SizeLat:10 for: call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.v2p0.i64(<2 x ptr> %buckets, i64 1, <2 x i1> %mask)
@@ -1294,15 +1294,15 @@ define void @histogram_v2i64(<2 x ptr> %buckets, <2 x i1> %mask) {
 
 define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) {
 ; CHECK-VSCALE-1-LABEL: 'histogram_v4i32'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:16 Lat:20 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_v4i32'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:16 Lat:20 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v4i32'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:20 CodeSize:16 Lat:20 SizeLat:20 for: call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.v4p0.i32(<4 x ptr> %buckets, i32 1, <4 x i1> %mask)
@@ -1311,15 +1311,15 @@ define void @histogram_v4i32(<4 x ptr> %buckets, <4 x i1> %mask) {
 
 define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) {
 ; CHECK-VSCALE-1-LABEL: 'histogram_v8i16'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:40 CodeSize:32 Lat:40 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_v8i16'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:40 CodeSize:32 Lat:40 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v8i16'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:40 CodeSize:32 Lat:40 SizeLat:40 for: call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.v8p0.i16(<8 x ptr> %buckets, i16 1, <8 x i1> %mask)
@@ -1328,15 +1328,15 @@ define void @histogram_v8i16(<8 x ptr> %buckets, <8 x i1> %mask) {
 
 define void @histogram_v16i8(<16 x ptr> %buckets, <16 x i1> %mask) {
 ; CHECK-VSCALE-1-LABEL: 'histogram_v16i8'
-; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:80 CodeSize:64 Lat:80 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
 ; CHECK-VSCALE-1-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-VSCALE-2-LABEL: 'histogram_v16i8'
-; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:80 CodeSize:64 Lat:80 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
 ; CHECK-VSCALE-2-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; TYPE_BASED_ONLY-LABEL: 'histogram_v16i8'
-; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of Invalid for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
+; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:80 CodeSize:64 Lat:80 SizeLat:80 for: call void @llvm.experimental.vector.histogram.add.v16p0.i8(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
 ; TYPE_BASED_ONLY-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   call void @llvm.experimental.vector.histogram.add.v16p0.i64(<16 x ptr> %buckets, i8 1, <16 x i1> %mask)
diff --git a/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll b/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll
index 6bcf3c705ef6f..f234341b0fd86 100644
--- a/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/vec3-ops.ll
@@ -206,8 +206,8 @@ define void @vec3_float(<3 x float> %a, <3 x float> %b, ptr %src, ptr %dst) {
 define void @vec3_half(<3 x half> %a, <3 x half> %b, ptr %src, ptr %dst) {
 ; CHECK-LABEL: 'vec3_half'
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %l = load <3 x half>, ptr %src, align 1
-; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %add = fadd <3 x half> %l, %b
-; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:3 SizeLat:1 for: %sub = fsub <3 x half> %add, %a
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %add = fadd <3 x half> %l, %b
+; CHECK-NEXT:  Cost Model: Found costs of RThru:4 CodeSize:1 Lat:3 SizeLat:1 for: %sub = fsub <3 x half> %add, %a
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <3 x half> %sub, ptr %dst, align 1
 ; CHECK-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
diff --git a/llvm/test/Analysis/CostModel/ARM/cast.ll b/llvm/test/Analysis/CostModel/ARM/cast.ll
index a0a037444a111..2a57c85046954 100644
--- a/llvm/test/Analysis/CostModel/ARM/cast.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cast.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=thumbv7-apple-ios6.0.0 -mcpu=cortex-a9 < %s  | FileCheck %s --check-prefix=CHECK-NEON
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv7-apple-ios6.0.0 -mcpu=cortex-a9 < %s  | FileCheck %s --check-prefix=CHECK-NEON
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
diff --git a/llvm/test/Analysis/CostModel/ARM/cmps.ll b/llvm/test/Analysis/CostModel/ARM/cmps.ll
index 090fccee81bbe..6b19be542a381 100644
--- a/llvm/test/Analysis/CostModel/ARM/cmps.ll
+++ b/llvm/test/Analysis/CostModel/ARM/cmps.ll
@@ -1,159 +1,83 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R-RECIP
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE-SIZE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN-SIZE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE-SIZE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R-SIZE
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
 define i32 @cmps() {
-; CHECK-MVE-RECIP-LABEL: 'cmps'
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = icmp slt i8 undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %b = icmp ult i16 undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = icmp sge i32 undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = icmp ne i64 undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %e = icmp slt <16 x i8> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %f = icmp ult <8 x i16> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %g = icmp sge <4 x i32> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a7 = fcmp oge half undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a8 = fcmp ogt float undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %p = icmp eq ptr undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %q = icmp eq <4 x ptr> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; CHECK-V8M-MAIN-RECIP-LABEL: 'cmps'
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = icmp slt i8 undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %b = icmp ult i16 undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = icmp sge i32 undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = icmp ne i64 undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g = icmp sge <4 x i32> undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a7 = fcmp oge half undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a8 = fcmp ogt float undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %p = icmp eq ptr undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %q = icmp eq <4 x ptr> undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
-;
-; CHECK-V8M-BASE-RECIP-LABEL: 'cmps'
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = icmp slt i8 undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %b = icmp ult i16 undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = icmp sge i32 undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = icmp ne i64 undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g = icmp sge <4 x i32> undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a7 = fcmp oge half undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a8 = fcmp ogt float undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %p = icmp eq ptr undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %q = icmp eq <4 x ptr> undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
-;
-; CHECK-V8R-RECIP-LABEL: 'cmps'
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = icmp slt i8 undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %b = icmp ult i16 undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = icmp sge i32 undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = icmp ne i64 undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = icmp sge <4 x i32> undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a7 = fcmp oge half undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a8 = fcmp ogt float undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %p = icmp eq ptr undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %q = icmp eq <4 x ptr> undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; CHECK-MVE-SIZE-LABEL: 'cmps'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = icmp slt i8 undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %b = icmp ult i16 undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = icmp sge i32 undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = icmp ne i64 undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = icmp slt <16 x i8> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = icmp ult <8 x i16> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = icmp sge <4 x i32> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a7 = fcmp oge half undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a8 = fcmp ogt float undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %p = icmp eq ptr undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %q = icmp eq <4 x ptr> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
-;
-; CHECK-V8M-MAIN-SIZE-LABEL: 'cmps'
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = icmp slt i8 undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %b = icmp ult i16 undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = icmp sge i32 undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = icmp ne i64 undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g = icmp sge <4 x i32> undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a7 = fcmp oge half undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a8 = fcmp ogt float undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %p = icmp eq ptr undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %q = icmp eq <4 x ptr> undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
-;
-; CHECK-V8M-BASE-SIZE-LABEL: 'cmps'
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = icmp slt i8 undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %b = icmp ult i16 undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = icmp sge i32 undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = icmp ne i64 undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %g = icmp sge <4 x i32> undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a7 = fcmp oge half undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a8 = fcmp ogt float undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %p = icmp eq ptr undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %q = icmp eq <4 x ptr> undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
-;
-; CHECK-V8R-SIZE-LABEL: 'cmps'
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a = icmp slt i8 undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %b = icmp ult i16 undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c = icmp sge i32 undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %d = icmp ne i64 undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %e = icmp slt <16 x i8> undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %f = icmp ult <8 x i16> undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %g = icmp sge <4 x i32> undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a7 = fcmp oge half undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a8 = fcmp ogt float undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a9 = fcmp ogt double undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %a10 = fcmp olt <8 x half> undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %a11 = fcmp oge <4 x float> undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %a12 = fcmp oge <2 x double> undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %p = icmp eq ptr undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %q = icmp eq <4 x ptr> undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 undef
+; CHECK-MVE-LABEL: 'cmps'
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %a = icmp slt i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %b = icmp ult i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %c = icmp sge i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %d = icmp ne i64 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %e = icmp slt <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %f = icmp ult <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %g = icmp sge <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %a11 = fcmp oge <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %q = icmp eq <4 x ptr> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; CHECK-V8M-MAIN-LABEL: 'cmps'
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %a = icmp slt i8 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %b = icmp ult i16 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %c = icmp sge i32 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %d = icmp ne i64 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %e = icmp slt <16 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %f = icmp ult <8 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %g = icmp sge <4 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %a11 = fcmp oge <4 x float> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 6 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %q = icmp eq <4 x ptr> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret i32 undef
+;
+; CHECK-V8M-BASE-LABEL: 'cmps'
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %a = icmp slt i8 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %b = icmp ult i16 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %c = icmp sge i32 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %d = icmp ne i64 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %e = icmp slt <16 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %f = icmp ult <8 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %g = icmp sge <4 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %a11 = fcmp oge <4 x float> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 6 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %q = icmp eq <4 x ptr> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret i32 undef
+;
+; CHECK-V8R-LABEL: 'cmps'
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %a = icmp slt i8 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %b = icmp ult i16 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %c = icmp sge i32 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %d = icmp ne i64 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %e = icmp slt <16 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %f = icmp ult <8 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %g = icmp sge <4 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %a7 = fcmp oge half undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %a8 = fcmp ogt float undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %a9 = fcmp ogt double undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 24 for: %a10 = fcmp olt <8 x half> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %a11 = fcmp oge <4 x float> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 4 for: %a12 = fcmp oge <2 x double> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %p = icmp eq ptr undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %q = icmp eq <4 x ptr> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
 ;
   %a = icmp slt i8 undef, undef
   %b = icmp ult i16 undef, undef
@@ -174,125 +98,65 @@ define i32 @cmps() {
 }
 
 define void @minmax() {
-; CHECK-MVE-RECIP-LABEL: 'minmax'
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c1 = icmp slt i8 undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s1 = select i1 %c1, i8 undef, i8 undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = icmp slt i16 undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2 = select i1 %c2, i16 undef, i16 undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c3 = icmp slt i32 undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s3 = select i1 %c3, i32 undef, i32 undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c5 = icmp slt ptr undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %c6 = icmp slt <4 x ptr> undef, undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
-; CHECK-MVE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; CHECK-V8M-MAIN-RECIP-LABEL: 'minmax'
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c1 = icmp slt i8 undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s1 = select i1 %c1, i8 undef, i8 undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = icmp slt i16 undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2 = select i1 %c2, i16 undef, i16 undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c3 = icmp slt i32 undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s3 = select i1 %c3, i32 undef, i32 undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c5 = icmp slt ptr undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c6 = icmp slt <4 x ptr> undef, undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
-; CHECK-V8M-MAIN-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
-;
-; CHECK-V8M-BASE-RECIP-LABEL: 'minmax'
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c1 = icmp slt i8 undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s1 = select i1 %c1, i8 undef, i8 undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = icmp slt i16 undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2 = select i1 %c2, i16 undef, i16 undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c3 = icmp slt i32 undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s3 = select i1 %c3, i32 undef, i32 undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c5 = icmp slt ptr undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c6 = icmp slt <4 x ptr> undef, undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
-; CHECK-V8M-BASE-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
-;
-; CHECK-V8R-RECIP-LABEL: 'minmax'
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c1 = icmp slt i8 undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s1 = select i1 %c1, i8 undef, i8 undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = icmp slt i16 undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2 = select i1 %c2, i16 undef, i16 undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c3 = icmp slt i32 undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s3 = select i1 %c3, i32 undef, i32 undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c5 = icmp slt ptr undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c6 = icmp slt <4 x ptr> undef, undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
-; CHECK-V8R-RECIP-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; CHECK-MVE-SIZE-LABEL: 'minmax'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c1 = icmp slt i8 undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s1 = select i1 %c1, i8 undef, i8 undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = icmp slt i16 undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s2 = select i1 %c2, i16 undef, i16 undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c3 = icmp slt i32 undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s3 = select i1 %c3, i32 undef, i32 undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c5 = icmp slt ptr undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c6 = icmp slt <4 x ptr> undef, undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
-;
-; CHECK-V8M-MAIN-SIZE-LABEL: 'minmax'
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c1 = icmp slt i8 undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s1 = select i1 %c1, i8 undef, i8 undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = icmp slt i16 undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s2 = select i1 %c2, i16 undef, i16 undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c3 = icmp slt i32 undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s3 = select i1 %c3, i32 undef, i32 undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c5 = icmp slt ptr undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c6 = icmp slt <4 x ptr> undef, undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
-; CHECK-V8M-MAIN-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
-;
-; CHECK-V8M-BASE-SIZE-LABEL: 'minmax'
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c1 = icmp slt i8 undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s1 = select i1 %c1, i8 undef, i8 undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = icmp slt i16 undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s2 = select i1 %c2, i16 undef, i16 undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c3 = icmp slt i32 undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s3 = select i1 %c3, i32 undef, i32 undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c5 = icmp slt ptr undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %c6 = icmp slt <4 x ptr> undef, undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
-; CHECK-V8M-BASE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
-;
-; CHECK-V8R-SIZE-LABEL: 'minmax'
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c1 = icmp slt i8 undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s1 = select i1 %c1, i8 undef, i8 undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c2 = icmp slt i16 undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s2 = select i1 %c2, i16 undef, i16 undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c3 = icmp slt i32 undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s3 = select i1 %c3, i32 undef, i32 undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %c4 = icmp slt <4 x i32> undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c5 = icmp slt ptr undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s5 = select i1 %c5, ptr undef, ptr undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %c6 = icmp slt <4 x ptr> undef, undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
-; CHECK-V8R-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-MVE-LABEL: 'minmax'
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %c1 = icmp slt i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s1 = select i1 %c1, i8 undef, i8 undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %c2 = icmp slt i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s2 = select i1 %c2, i16 undef, i16 undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %c3 = icmp slt i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s3 = select i1 %c3, i32 undef, i32 undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %c4 = icmp slt <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %c5 = icmp slt ptr undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s5 = select i1 %c5, ptr undef, ptr undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %c6 = icmp slt <4 x ptr> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; CHECK-V8M-MAIN-LABEL: 'minmax'
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %c1 = icmp slt i8 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s1 = select i1 %c1, i8 undef, i8 undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %c2 = icmp slt i16 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s2 = select i1 %c2, i16 undef, i16 undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %c3 = icmp slt i32 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s3 = select i1 %c3, i32 undef, i32 undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 0 for: %c4 = icmp slt <4 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %c5 = icmp slt ptr undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s5 = select i1 %c5, ptr undef, ptr undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %c6 = icmp slt <4 x ptr> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:12 Lat:8 SizeLat:8 for: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
+;
+; CHECK-V8M-BASE-LABEL: 'minmax'
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %c1 = icmp slt i8 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s1 = select i1 %c1, i8 undef, i8 undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %c2 = icmp slt i16 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s2 = select i1 %c2, i16 undef, i16 undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %c3 = icmp slt i32 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s3 = select i1 %c3, i32 undef, i32 undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 0 for: %c4 = icmp slt <4 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:20 Lat:16 SizeLat:16 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %c5 = icmp slt ptr undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:2 Lat:1 SizeLat:1 for: %s5 = select i1 %c5, ptr undef, ptr undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %c6 = icmp slt <4 x ptr> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:12 Lat:8 SizeLat:8 for: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
+;
+; CHECK-V8R-LABEL: 'minmax'
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %c1 = icmp slt i8 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %s1 = select i1 %c1, i8 undef, i8 undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %c2 = icmp slt i16 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %s2 = select i1 %c2, i16 undef, i16 undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %c3 = icmp slt i32 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %s3 = select i1 %c3, i32 undef, i32 undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 0 for: %c4 = icmp slt <4 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %s4 = select <4 x i1> %c4, <4 x i32> undef, <4 x i32> undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %c5 = icmp slt ptr undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %s5 = select i1 %c5, ptr undef, ptr undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %c6 = icmp slt <4 x ptr> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 1 for: %s6 = select <4 x i1> %c6, <4 x ptr> undef, <4 x ptr> undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %c1 = icmp slt i8 undef, undef
   %s1 = select i1 %c1, i8 undef, i8 undef
diff --git a/llvm/test/Analysis/CostModel/ARM/control-flow.ll b/llvm/test/Analysis/CostModel/ARM/control-flow.ll
index dcf90285c81fb..af7426b77e1f2 100644
--- a/llvm/test/Analysis/CostModel/ARM/control-flow.ll
+++ b/llvm/test/Analysis/CostModel/ARM/control-flow.ll
@@ -1,151 +1,55 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size  -mtriple=thumbv8m.base   | FileCheck %s --check-prefix=CHECK-T1-SIZE
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size  -mtriple=thumbv8m.main   | FileCheck %s --check-prefix=CHECK-V8M-SIZE
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size  -mtriple=thumbv8.1m.main -mattr=+mve  | FileCheck %s --check-prefix=CHECK-MVE-SIZE
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size  -mtriple=armv8a -mattr=+neon  | FileCheck %s --check-prefix=CHECK-NEON-SIZE
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency    -mtriple=thumbv8m.base   | FileCheck %s --check-prefix=CHECK-T1-LATENCY
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency    -mtriple=thumbv8m.main   | FileCheck %s --check-prefix=CHECK-V8M-LATENCY
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency    -mtriple=thumbv8.1m.main | FileCheck %s --check-prefix=CHECK-V8_1M-LATENCY
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mtriple=thumbv8m.base   | FileCheck %s --check-prefix=CHECK-T1-THROUGHPUT
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mtriple=thumbv8m.main   | FileCheck %s --check-prefix=CHECK-V8M-THROUGHPUT
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mtriple=thumbv8.1m.main | FileCheck %s --check-prefix=CHECK-V8_1M-THROUGHPUT
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mtriple=thumbv8.1m.main -mattr=+mve | FileCheck %s --check-prefix=CHECK-MVE-THROUGHPUT
-; RUN: opt < %s -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mtriple=armv8a -mattr=+neon | FileCheck %s --check-prefix=CHECK-NEON-THROUGHPUT
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output  -mtriple=thumbv8m.base   | FileCheck %s --check-prefix=CHECK-T1
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output  -mtriple=thumbv8m.main   | FileCheck %s --check-prefix=CHECK-V8M
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output  -mtriple=thumbv8.1m.main -mattr=+mve  | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt < %s -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output  -mtriple=armv8a -mattr=+neon  | FileCheck %s --check-prefix=CHECK-NEON
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
 define i32 @simple_loop_cost(i32 %N) {
-; CHECK-T1-SIZE-LABEL: 'simple_loop_cost'
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8M-SIZE-LABEL: 'simple_loop_cost'
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-MVE-SIZE-LABEL: 'simple_loop_cost'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-NEON-SIZE-LABEL: 'simple_loop_cost'
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-T1-LATENCY-LABEL: 'simple_loop_cost'
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8M-LATENCY-LABEL: 'simple_loop_cost'
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8_1M-LATENCY-LABEL: 'simple_loop_cost'
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-T1-THROUGHPUT-LABEL: 'simple_loop_cost'
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8M-THROUGHPUT-LABEL: 'simple_loop_cost'
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8_1M-THROUGHPUT-LABEL: 'simple_loop_cost'
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-MVE-THROUGHPUT-LABEL: 'simple_loop_cost'
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br label %loop
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %res
-;
-; CHECK-NEON-THROUGHPUT-LABEL: 'simple_loop_cost'
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br label %loop
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %res
+; CHECK-T1-LABEL: 'simple_loop_cost'
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: br label %loop
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: ret i32 %res
+;
+; CHECK-V8M-LABEL: 'simple_loop_cost'
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: br label %loop
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: ret i32 %res
+;
+; CHECK-MVE-LABEL: 'simple_loop_cost'
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br label %loop
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %res
+;
+; CHECK-NEON-LABEL: 'simple_loop_cost'
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br label %loop
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %res
 ;
 entry:
   %zero = icmp eq i32 %N, 0
@@ -166,197 +70,69 @@ exit:
 }
 
 define i32 @simple_mul_loop(ptr %A, ptr %B, i32 %N) {
-; CHECK-T1-SIZE-LABEL: 'simple_mul_loop'
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8M-SIZE-LABEL: 'simple_mul_loop'
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-MVE-SIZE-LABEL: 'simple_mul_loop'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-NEON-SIZE-LABEL: 'simple_mul_loop'
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-T1-LATENCY-LABEL: 'simple_mul_loop'
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8M-LATENCY-LABEL: 'simple_mul_loop'
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8_1M-LATENCY-LABEL: 'simple_mul_loop'
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-T1-THROUGHPUT-LABEL: 'simple_mul_loop'
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8M-THROUGHPUT-LABEL: 'simple_mul_loop'
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8_1M-THROUGHPUT-LABEL: 'simple_mul_loop'
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-MVE-THROUGHPUT-LABEL: 'simple_mul_loop'
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br label %loop
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %res
-;
-; CHECK-NEON-THROUGHPUT-LABEL: 'simple_mul_loop'
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br label %loop
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = getelementptr i32, ptr %A, i32 %iv
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i32, ptr %addr.a, align 4
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %load, %load
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = getelementptr i32, ptr %B, i32 %iv
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %res
+; CHECK-T1-LABEL: 'simple_mul_loop'
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: br label %loop
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %addr.a = getelementptr i32, ptr %A, i32 %iv
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load = load i32, ptr %addr.a, align 4
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %mul = mul i32 %load, %load
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %addr.b = getelementptr i32, ptr %B, i32 %iv
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: store i32 %mul, ptr %addr.b, align 4
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: ret i32 %res
+;
+; CHECK-V8M-LABEL: 'simple_mul_loop'
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: br label %loop
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 0 for: %addr.a = getelementptr i32, ptr %A, i32 %iv
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load = load i32, ptr %addr.a, align 4
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %mul = mul i32 %load, %load
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 0 for: %addr.b = getelementptr i32, ptr %B, i32 %iv
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: store i32 %mul, ptr %addr.b, align 4
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: ret i32 %res
+;
+; CHECK-MVE-LABEL: 'simple_mul_loop'
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br label %loop
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %addr.a = getelementptr i32, ptr %A, i32 %iv
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load = load i32, ptr %addr.a, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %mul = mul i32 %load, %load
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %addr.b = getelementptr i32, ptr %B, i32 %iv
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: store i32 %mul, ptr %addr.b, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %res
+;
+; CHECK-NEON-LABEL: 'simple_mul_loop'
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br label %loop
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %addr.a = getelementptr i32, ptr %A, i32 %iv
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load = load i32, ptr %addr.a, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %mul = mul i32 %load, %load
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %addr.b = getelementptr i32, ptr %B, i32 %iv
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: store i32 %mul, ptr %addr.b, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %res
 ;
 entry:
   %zero = icmp eq i32 %N, 0
@@ -382,233 +158,81 @@ exit:
 }
 
 define i32 @simple_mul_ext_lsr_loop(ptr %A, ptr %B, i32 %N) {
-; CHECK-T1-SIZE-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-T1-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8M-SIZE-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8M-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-MVE-SIZE-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-MVE-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-NEON-SIZE-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEON-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-T1-LATENCY-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-T1-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8M-LATENCY-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8_1M-LATENCY-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8_1M-LATENCY-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-T1-THROUGHPUT-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-T1-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8M-THROUGHPUT-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-V8_1M-THROUGHPUT-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br label %loop
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-V8_1M-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %res
-;
-; CHECK-MVE-THROUGHPUT-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br label %loop
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-MVE-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %res
-;
-; CHECK-NEON-THROUGHPUT-LABEL: 'simple_mul_ext_lsr_loop'
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %zero = icmp eq i32 %N, 0
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %zero, label %exit, label %preheader
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br label %loop
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %load = load i16, ptr %addr.a, align 2
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %sext = sext i16 %load to i32
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mul = mul i32 %sext, 7
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: store i32 %mul, ptr %addr.b, align 4
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %iv.next = add nuw i32 %iv, 1
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cmp = icmp ne i32 %iv.next, %N
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: br i1 %cmp, label %loop, label %exit
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
-; CHECK-NEON-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 %res
+; CHECK-T1-LABEL: 'simple_mul_ext_lsr_loop'
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: br label %loop
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load = load i16, ptr %addr.a, align 2
+; CHECK-T1-NEXT:  Cost Model: Found costs of 0 for: %sext = sext i16 %load to i32
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %mul = mul i32 %sext, 7
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: store i32 %mul, ptr %addr.b, align 4
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-T1-NEXT:  Cost Model: Found costs of 0 for: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
+; CHECK-T1-NEXT:  Cost Model: Found costs of 0 for: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-T1-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-T1-NEXT:  Cost Model: Found costs of 1 for: ret i32 %res
+;
+; CHECK-V8M-LABEL: 'simple_mul_ext_lsr_loop'
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: br label %loop
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load = load i16, ptr %addr.a, align 2
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 0 for: %sext = sext i16 %load to i32
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %mul = mul i32 %sext, 7
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: store i32 %mul, ptr %addr.b, align 4
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 0 for: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 0 for: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-V8M-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:0 Lat:0 SizeLat:0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-V8M-NEXT:  Cost Model: Found costs of 1 for: ret i32 %res
+;
+; CHECK-MVE-LABEL: 'simple_mul_ext_lsr_loop'
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br label %loop
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load = load i16, ptr %addr.a, align 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %sext = sext i16 %load to i32
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %mul = mul i32 %sext, 7
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: store i32 %mul, ptr %addr.b, align 4
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %res
+;
+; CHECK-NEON-LABEL: 'simple_mul_ext_lsr_loop'
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %zero = icmp eq i32 %N, 0
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %zero, label %exit, label %preheader
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br label %loop
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %iv = phi i32 [ 0, %preheader ], [ %iv.next, %loop ]
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %addr.a = phi ptr [ %A, %preheader ], [ %addr.a, %loop ]
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %addr.b = phi ptr [ %B, %preheader ], [ %addr.b, %loop ]
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %load = load i16, ptr %addr.a, align 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %sext = sext i16 %load to i32
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %mul = mul i32 %sext, 7
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: store i32 %mul, ptr %addr.b, align 4
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %iv.next = add nuw i32 %iv, 1
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %addr.a.next = getelementptr i16, ptr %addr.a, i32 1
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %addr.b.next = getelementptr i32, ptr %addr.b, i32 1
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 1 for: %cmp = icmp ne i32 %iv.next, %N
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: br i1 %cmp, label %loop, label %exit
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 0 for: %res = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 %res
 ;
 entry:
   %zero = icmp eq i32 %N, 0
diff --git a/llvm/test/Analysis/CostModel/ARM/divrem.ll b/llvm/test/Analysis/CostModel/ARM/divrem.ll
index 9f0c29c8bb0c6..76f80da55af64 100644
--- a/llvm/test/Analysis/CostModel/ARM/divrem.ll
+++ b/llvm/test/Analysis/CostModel/ARM/divrem.ll
@@ -1,67 +1,67 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s  -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv7-apple-ios6.0.0 -mcpu=cortex-a9 | FileCheck %s --check-prefix=CHECK-NEON
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv7-apple-ios6.0.0 -mcpu=cortex-a9 < %s | FileCheck %s --check-prefix=CHECK-NEON
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv8.1m.main-none-eabi -mattr=+mve.fp < %s | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv8m.main-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-MAIN
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=thumbv8m.base-none-eabi < %s | FileCheck %s --check-prefix=CHECK-V8M-BASE
+; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=armv8r-none-eabi -mattr=+neon,+fp-armv8 < %s | FileCheck %s --check-prefix=CHECK-V8R
 
 target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
 
 define void @i8() {
 ; CHECK-NEON-LABEL: 'i8'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i8 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i8 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i8 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i8 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i8 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i8 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i8 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i8 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = sdiv i8 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = udiv i8 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %3 = srem i8 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %4 = urem i8 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %5 = sdiv i8 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %6 = udiv i8 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %7 = srem i8 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %8 = urem i8 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'i8'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i8 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i8 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i8 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i8 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i8 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i8 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i8 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i8 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %1 = sdiv i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %2 = udiv i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i8 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %5 = sdiv i8 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %6 = udiv i8 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i8 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i8 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'i8'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i8 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i8 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i8 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i8 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i8 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i8 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i8 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i8 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %1 = sdiv i8 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %2 = udiv i8 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i8 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i8 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %5 = sdiv i8 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %6 = udiv i8 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i8 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i8 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'i8'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i8 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i8 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i8 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i8 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i8 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i8 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i8 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i8 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %1 = sdiv i8 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %2 = udiv i8 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i8 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i8 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %5 = sdiv i8 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %6 = udiv i8 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i8 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i8 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'i8'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i8 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i8 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %3 = srem i8 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %4 = urem i8 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i8 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i8 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %7 = srem i8 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %8 = urem i8 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %1 = sdiv i8 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %2 = udiv i8 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %3 = srem i8 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %4 = urem i8 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %5 = sdiv i8 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %6 = udiv i8 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %7 = srem i8 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %8 = urem i8 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = sdiv i8 undef, undef
   %2 = udiv i8 undef, undef
@@ -76,59 +76,59 @@ define void @i8() {
 
 define void @i16() {
 ; CHECK-NEON-LABEL: 'i16'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i16 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i16 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i16 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i16 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i16 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i16 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i16 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i16 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = sdiv i16 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = udiv i16 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %3 = srem i16 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %4 = urem i16 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %5 = sdiv i16 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %6 = udiv i16 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %7 = srem i16 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %8 = urem i16 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'i16'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i16 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i16 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i16 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i16 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i16 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i16 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i16 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i16 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %1 = sdiv i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %2 = udiv i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i16 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %5 = sdiv i16 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %6 = udiv i16 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i16 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i16 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'i16'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i16 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i16 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i16 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i16 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i16 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i16 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i16 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i16 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %1 = sdiv i16 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %2 = udiv i16 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i16 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i16 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %5 = sdiv i16 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %6 = udiv i16 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i16 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i16 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'i16'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i16 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i16 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i16 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i16 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i16 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i16 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i16 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i16 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %1 = sdiv i16 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %2 = udiv i16 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i16 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i16 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %5 = sdiv i16 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %6 = udiv i16 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i16 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i16 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'i16'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i16 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i16 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %3 = srem i16 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %4 = urem i16 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i16 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i16 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %7 = srem i16 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %8 = urem i16 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %1 = sdiv i16 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %2 = udiv i16 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %3 = srem i16 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %4 = urem i16 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %5 = sdiv i16 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %6 = udiv i16 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %7 = srem i16 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %8 = urem i16 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = sdiv i16 undef, undef
   %2 = udiv i16 undef, undef
@@ -143,59 +143,59 @@ define void @i16() {
 
 define void @i32() {
 ; CHECK-NEON-LABEL: 'i32'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i32 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i32 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i32 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i32 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i32 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i32 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i32 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i32 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = sdiv i32 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = udiv i32 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %3 = srem i32 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %4 = urem i32 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %5 = sdiv i32 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %6 = udiv i32 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %7 = srem i32 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %8 = urem i32 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'i32'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i32 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i32 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i32 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i32 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i32 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i32 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i32 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i32 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %1 = sdiv i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %2 = udiv i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i32 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %5 = sdiv i32 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %6 = udiv i32 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i32 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i32 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'i32'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i32 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i32 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i32 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i32 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i32 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i32 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i32 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i32 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %1 = sdiv i32 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %2 = udiv i32 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i32 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i32 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %5 = sdiv i32 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %6 = udiv i32 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i32 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i32 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'i32'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i32 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i32 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i32 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i32 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i32 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i32 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i32 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i32 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %1 = sdiv i32 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %2 = udiv i32 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i32 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i32 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %5 = sdiv i32 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %6 = udiv i32 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i32 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i32 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'i32'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sdiv i32 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = udiv i32 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %3 = srem i32 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %4 = urem i32 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %5 = sdiv i32 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %6 = udiv i32 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %7 = srem i32 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %8 = urem i32 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %1 = sdiv i32 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %2 = udiv i32 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %3 = srem i32 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %4 = urem i32 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %5 = sdiv i32 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %6 = udiv i32 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %7 = srem i32 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %8 = urem i32 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = sdiv i32 undef, undef
   %2 = udiv i32 undef, undef
@@ -210,59 +210,59 @@ define void @i32() {
 
 define void @i64() {
 ; CHECK-NEON-LABEL: 'i64'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = sdiv i64 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = udiv i64 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i64 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i64 undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %5 = sdiv i64 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %6 = udiv i64 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i64 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i64 undef, 2
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 4 for: %1 = sdiv i64 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 4 for: %2 = udiv i64 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %3 = srem i64 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %4 = urem i64 undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 4 for: %5 = sdiv i64 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 4 for: %6 = udiv i64 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %7 = srem i64 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %8 = urem i64 undef, 2
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'i64'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i64 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i64 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i64 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i64 undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i64 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i64 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i64 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i64 undef, 2
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 2 for: %1 = sdiv i64 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 2 for: %2 = udiv i64 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i64 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i64 undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 2 for: %5 = sdiv i64 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 2 for: %6 = udiv i64 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i64 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i64 undef, 2
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'i64'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i64 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i64 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i64 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i64 undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i64 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i64 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i64 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i64 undef, 2
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %1 = sdiv i64 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %2 = udiv i64 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i64 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i64 undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %5 = sdiv i64 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %6 = udiv i64 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i64 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i64 undef, 2
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'i64'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i64 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i64 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = srem i64 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = urem i64 undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i64 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i64 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %7 = srem i64 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %8 = urem i64 undef, 2
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %1 = sdiv i64 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %2 = udiv i64 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %3 = srem i64 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %4 = urem i64 undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %5 = sdiv i64 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %6 = udiv i64 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %7 = srem i64 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %8 = urem i64 undef, 2
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'i64'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = sdiv i64 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = udiv i64 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %3 = srem i64 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %4 = urem i64 undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = sdiv i64 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %6 = udiv i64 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %7 = srem i64 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %8 = urem i64 undef, 2
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = sdiv i64 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = udiv i64 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %3 = srem i64 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %4 = urem i64 undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %5 = sdiv i64 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %6 = udiv i64 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %7 = srem i64 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %8 = urem i64 undef, 2
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = sdiv i64 undef, undef
   %2 = udiv i64 undef, undef
@@ -277,39 +277,39 @@ define void @i64() {
 
 define void @f16() {
 ; CHECK-NEON-LABEL: 'f16'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv half undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem half undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv half undef, 0xH4000
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem half undef, 0xH4000
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv half undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem half undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv half undef, 0xH4000
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem half undef, 0xH4000
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'f16'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv half undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem half undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv half undef, 0xH4000
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem half undef, 0xH4000
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %1 = fdiv half undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %2 = frem half undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %3 = fdiv half undef, 0xH4000
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %4 = frem half undef, 0xH4000
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'f16'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv half undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem half undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv half undef, 0xH4000
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem half undef, 0xH4000
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %1 = fdiv half undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %2 = frem half undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %3 = fdiv half undef, 0xH4000
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %4 = frem half undef, 0xH4000
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'f16'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv half undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem half undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv half undef, 0xH4000
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem half undef, 0xH4000
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %1 = fdiv half undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %2 = frem half undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %3 = fdiv half undef, 0xH4000
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %4 = frem half undef, 0xH4000
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'f16'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv half undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem half undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv half undef, 0xH4000
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem half undef, 0xH4000
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv half undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem half undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv half undef, 0xH4000
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem half undef, 0xH4000
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = fdiv half undef, undef
   %2 = frem half undef, undef
@@ -320,39 +320,39 @@ define void @f16() {
 
 define void @f32() {
 ; CHECK-NEON-LABEL: 'f32'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv float undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem float undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv float undef, 2.000000e+00
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem float undef, 2.000000e+00
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv float undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem float undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv float undef, 2.000000e+00
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem float undef, 2.000000e+00
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'f32'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv float undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem float undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv float undef, 2.000000e+00
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem float undef, 2.000000e+00
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %1 = fdiv float undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %2 = frem float undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %3 = fdiv float undef, 2.000000e+00
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %4 = frem float undef, 2.000000e+00
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'f32'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv float undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem float undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv float undef, 2.000000e+00
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem float undef, 2.000000e+00
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %1 = fdiv float undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %2 = frem float undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %3 = fdiv float undef, 2.000000e+00
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: %4 = frem float undef, 2.000000e+00
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'f32'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv float undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem float undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv float undef, 2.000000e+00
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem float undef, 2.000000e+00
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %1 = fdiv float undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %2 = frem float undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %3 = fdiv float undef, 2.000000e+00
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: %4 = frem float undef, 2.000000e+00
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'f32'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv float undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem float undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv float undef, 2.000000e+00
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem float undef, 2.000000e+00
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv float undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem float undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv float undef, 2.000000e+00
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem float undef, 2.000000e+00
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = fdiv float undef, undef
   %2 = frem float undef, undef
@@ -363,39 +363,39 @@ define void @f32() {
 
 define void @f64() {
 ; CHECK-NEON-LABEL: 'f64'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv double undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem double undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv double undef, 2.000000e+00
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem double undef, 2.000000e+00
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv double undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem double undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv double undef, 2.000000e+00
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem double undef, 2.000000e+00
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'f64'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = fdiv double undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %2 = frem double undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %3 = fdiv double undef, 2.000000e+00
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = frem double undef, 2.000000e+00
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %1 = fdiv double undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %2 = frem double undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %3 = fdiv double undef, 2.000000e+00
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 1 for: %4 = frem double undef, 2.000000e+00
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'f64'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv double undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem double undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv double undef, 2.000000e+00
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem double undef, 2.000000e+00
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv double undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %2 = frem double undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %3 = fdiv double undef, 2.000000e+00
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %4 = frem double undef, 2.000000e+00
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'f64'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv double undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem double undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv double undef, 2.000000e+00
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem double undef, 2.000000e+00
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv double undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %2 = frem double undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %3 = fdiv double undef, 2.000000e+00
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %4 = frem double undef, 2.000000e+00
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'f64'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv double undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem double undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %3 = fdiv double undef, 2.000000e+00
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %4 = frem double undef, 2.000000e+00
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv double undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem double undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv double undef, 2.000000e+00
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:2 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem double undef, 2.000000e+00
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = fdiv double undef, undef
   %2 = frem double undef, undef
@@ -406,99 +406,99 @@ define void @f64() {
 
 define void @vi8() {
 ; CHECK-NEON-LABEL: 'vi8'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e1 = sdiv <8 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = udiv <8 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i8> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %f1 = sdiv <4 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %f2 = udiv <4 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %e1 = sdiv <8 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %e2 = udiv <8 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i8> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi8'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i8> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t1 = sdiv <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t2 = udiv <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t3 = srem <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t4 = urem <2 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f1 = sdiv <4 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f2 = udiv <4 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f3 = srem <4 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f4 = urem <4 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e1 = sdiv <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e2 = udiv <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e3 = srem <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e4 = urem <8 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s1 = sdiv <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s2 = udiv <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s3 = srem <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s4 = urem <16 x i8> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi8'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i8> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i8> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vi8'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i8> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i8> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vi8'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e1 = sdiv <8 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = udiv <8 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i8> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %f1 = sdiv <4 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %f2 = udiv <4 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %e1 = sdiv <8 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %e2 = udiv <8 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i8> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %t1 = sdiv <2 x i8> undef, undef
   %t2 = udiv <2 x i8> undef, undef
@@ -521,99 +521,99 @@ define void @vi8() {
 
 define void @vi16() {
 ; CHECK-NEON-LABEL: 'vi16'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i16> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %f1 = sdiv <4 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %f2 = udiv <4 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i16> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi16'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i16> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t1 = sdiv <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t2 = udiv <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t3 = srem <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t4 = urem <2 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f1 = sdiv <4 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f2 = udiv <4 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f3 = srem <4 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f4 = urem <4 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e1 = sdiv <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e2 = udiv <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e3 = srem <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e4 = urem <8 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s1 = sdiv <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s2 = udiv <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s3 = srem <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s4 = urem <16 x i16> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi16'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i16> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i16> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vi16'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i16> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i16> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vi16'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i16> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %f1 = sdiv <4 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %f2 = udiv <4 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i16> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %t1 = sdiv <2 x i16> undef, undef
   %t2 = udiv <2 x i16> undef, undef
@@ -636,99 +636,99 @@ define void @vi16() {
 
 define void @vi32() {
 ; CHECK-NEON-LABEL: 'vi32'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i32> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f1 = sdiv <4 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f2 = udiv <4 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i32> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi32'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i32> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t1 = sdiv <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t2 = udiv <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t3 = srem <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t4 = urem <2 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f1 = sdiv <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f2 = udiv <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f3 = srem <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f4 = urem <4 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e1 = sdiv <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e2 = udiv <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e3 = srem <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e4 = urem <8 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s1 = sdiv <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s2 = udiv <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s3 = srem <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s4 = urem <16 x i32> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi32'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i32> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i32> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vi32'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i32> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i32> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vi32'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i32> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f1 = sdiv <4 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f2 = udiv <4 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i32> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %t1 = sdiv <2 x i32> undef, undef
   %t2 = udiv <2 x i32> undef, undef
@@ -751,99 +751,99 @@ define void @vi32() {
 
 define void @vi64() {
 ; CHECK-NEON-LABEL: 'vi64'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i64> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f1 = sdiv <4 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f2 = udiv <4 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i64> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi64'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %t1 = sdiv <2 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %t2 = udiv <2 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %t3 = srem <2 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %t4 = urem <2 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f1 = sdiv <4 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f2 = udiv <4 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %f3 = srem <4 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %f4 = urem <4 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e1 = sdiv <8 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e2 = udiv <8 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %s1 = sdiv <16 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %s2 = udiv <16 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %s3 = srem <16 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %s4 = urem <16 x i64> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %t1 = sdiv <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %t2 = udiv <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 18 for: %t3 = srem <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 18 for: %t4 = urem <2 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %f1 = sdiv <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %f2 = udiv <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 36 for: %f3 = srem <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 36 for: %f4 = urem <4 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %e1 = sdiv <8 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %e2 = udiv <8 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 72 for: %e3 = srem <8 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 72 for: %e4 = urem <8 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 160 for: %s1 = sdiv <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 160 for: %s2 = udiv <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 144 for: %s3 = srem <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 144 for: %s4 = urem <16 x i64> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi64'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t1 = sdiv <2 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t2 = udiv <2 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f1 = sdiv <4 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f2 = udiv <4 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f3 = srem <4 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f4 = urem <4 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e1 = sdiv <8 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e2 = udiv <8 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e3 = srem <8 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e4 = urem <8 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s1 = sdiv <16 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s2 = udiv <16 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %s3 = srem <16 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %s4 = urem <16 x i64> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t1 = sdiv <2 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t2 = udiv <2 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 6 for: %t3 = srem <2 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 6 for: %t4 = urem <2 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f1 = sdiv <4 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f2 = udiv <4 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 12 for: %f3 = srem <4 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 12 for: %f4 = urem <4 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e1 = sdiv <8 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e2 = udiv <8 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 24 for: %e3 = srem <8 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 24 for: %e4 = urem <8 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s1 = sdiv <16 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s2 = udiv <16 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 48 for: %s3 = srem <16 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 48 for: %s4 = urem <16 x i64> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vi64'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t1 = sdiv <2 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t2 = udiv <2 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f1 = sdiv <4 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f2 = udiv <4 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f3 = srem <4 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f4 = urem <4 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e1 = sdiv <8 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e2 = udiv <8 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e3 = srem <8 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e4 = urem <8 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s1 = sdiv <16 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s2 = udiv <16 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %s3 = srem <16 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %s4 = urem <16 x i64> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t1 = sdiv <2 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t2 = udiv <2 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 6 for: %t3 = srem <2 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 6 for: %t4 = urem <2 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f1 = sdiv <4 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f2 = udiv <4 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 12 for: %f3 = srem <4 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 12 for: %f4 = urem <4 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e1 = sdiv <8 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e2 = udiv <8 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 24 for: %e3 = srem <8 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 24 for: %e4 = urem <8 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s1 = sdiv <16 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s2 = udiv <16 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 48 for: %s3 = srem <16 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 48 for: %s4 = urem <16 x i64> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vi64'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i64> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f1 = sdiv <4 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f2 = udiv <4 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i64> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %t1 = sdiv <2 x i64> undef, undef
   %t2 = udiv <2 x i64> undef, undef
@@ -866,49 +866,49 @@ define void @vi64() {
 
 define void @vf16() {
 ; CHECK-NEON-LABEL: 'vf16'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = frem <2 x half> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = frem <4 x half> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = frem <8 x half> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x half> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf16'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x half> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x half> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf16'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv <2 x half> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem <2 x half> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = fdiv <4 x half> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = frem <4 x half> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %5 = fdiv <8 x half> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %6 = frem <8 x half> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv <2 x half> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %2 = frem <2 x half> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %3 = fdiv <4 x half> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %4 = frem <4 x half> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %5 = fdiv <8 x half> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %6 = frem <8 x half> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vf16'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv <2 x half> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem <2 x half> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = fdiv <4 x half> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = frem <4 x half> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %5 = fdiv <8 x half> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %6 = frem <8 x half> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv <2 x half> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %2 = frem <2 x half> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %3 = fdiv <4 x half> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %4 = frem <4 x half> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %5 = fdiv <8 x half> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %6 = frem <8 x half> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vf16'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = frem <2 x half> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = frem <4 x half> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = frem <8 x half> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x half> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x half> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x half> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x half> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x half> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x half> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = fdiv <2 x half> undef, undef
   %2 = frem <2 x half> undef, undef
@@ -921,49 +921,49 @@ define void @vf16() {
 
 define void @vf32() {
 ; CHECK-NEON-LABEL: 'vf32'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %1 = fdiv <2 x float> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = frem <2 x float> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %3 = fdiv <4 x float> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = frem <4 x float> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %5 = fdiv <8 x float> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = frem <8 x float> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv <2 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x float> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf32'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x float> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x float> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf32'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv <2 x float> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem <2 x float> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = fdiv <4 x float> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = frem <4 x float> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %5 = fdiv <8 x float> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %6 = frem <8 x float> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv <2 x float> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %2 = frem <2 x float> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %3 = fdiv <4 x float> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %4 = frem <4 x float> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %5 = fdiv <8 x float> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %6 = frem <8 x float> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vf32'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv <2 x float> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem <2 x float> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = fdiv <4 x float> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = frem <4 x float> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %5 = fdiv <8 x float> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %6 = frem <8 x float> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv <2 x float> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %2 = frem <2 x float> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %3 = fdiv <4 x float> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %4 = frem <4 x float> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %5 = fdiv <8 x float> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %6 = frem <8 x float> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vf32'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %1 = fdiv <2 x float> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = frem <2 x float> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %3 = fdiv <4 x float> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = frem <4 x float> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %5 = fdiv <8 x float> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = frem <8 x float> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv <2 x float> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x float> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x float> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x float> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x float> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x float> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = fdiv <2 x float> undef, undef
   %2 = frem <2 x float> undef, undef
@@ -976,49 +976,49 @@ define void @vf32() {
 
 define void @vf64() {
 ; CHECK-NEON-LABEL: 'vf64'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = fdiv <2 x double> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %2 = frem <2 x double> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %3 = fdiv <4 x double> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %4 = frem <4 x double> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %5 = fdiv <8 x double> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %6 = frem <8 x double> undef, undef
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv <2 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x double> undef, undef
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf64'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x double> undef, undef
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x double> undef, undef
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf64'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x double> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x double> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x double> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x double> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x double> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x double> undef, undef
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x double> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x double> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x double> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x double> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x double> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x double> undef, undef
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vf64'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x double> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x double> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x double> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x double> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x double> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x double> undef, undef
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x double> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x double> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x double> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x double> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x double> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x double> undef, undef
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vf64'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = fdiv <2 x double> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %2 = frem <2 x double> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %3 = fdiv <4 x double> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %4 = frem <4 x double> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %5 = fdiv <8 x double> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %6 = frem <8 x double> undef, undef
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv <2 x double> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x double> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x double> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x double> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x double> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x double> undef, undef
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = fdiv <2 x double> undef, undef
   %2 = frem <2 x double> undef, undef
@@ -1031,99 +1031,99 @@ define void @vf64() {
 
 define void @vi8_2() {
 ; CHECK-NEON-LABEL: 'vi8_2'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e1 = sdiv <8 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = udiv <8 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i8> undef, splat (i8 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %f1 = sdiv <4 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %f2 = udiv <4 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %e1 = sdiv <8 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %e2 = udiv <8 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i8> undef, splat (i8 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi8_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i8> undef, splat (i8 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t1 = sdiv <2 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t2 = udiv <2 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t3 = srem <2 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t4 = urem <2 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f1 = sdiv <4 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f2 = udiv <4 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f3 = srem <4 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f4 = urem <4 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e1 = sdiv <8 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e2 = udiv <8 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e3 = srem <8 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e4 = urem <8 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s1 = sdiv <16 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s2 = udiv <16 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s3 = srem <16 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s4 = urem <16 x i8> undef, splat (i8 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi8_2'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i8> undef, splat (i8 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i8> undef, splat (i8 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vi8_2'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i8> undef, splat (i8 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i8> undef, splat (i8 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vi8_2'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e1 = sdiv <8 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %e2 = udiv <8 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i8> undef, splat (i8 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %f1 = sdiv <4 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %f2 = udiv <4 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %e1 = sdiv <8 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %e2 = udiv <8 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i8> undef, splat (i8 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %t1 = sdiv <2 x i8> undef, <i8 2, i8 2>
   %t2 = udiv <2 x i8> undef, <i8 2, i8 2>
@@ -1146,99 +1146,99 @@ define void @vi8_2() {
 
 define void @vi16_2() {
 ; CHECK-NEON-LABEL: 'vi16_2'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i16> undef, splat (i16 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %f1 = sdiv <4 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 10 for: %f2 = udiv <4 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i16> undef, splat (i16 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi16_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i16> undef, splat (i16 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t1 = sdiv <2 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t2 = udiv <2 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t3 = srem <2 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t4 = urem <2 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f1 = sdiv <4 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f2 = udiv <4 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f3 = srem <4 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f4 = urem <4 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e1 = sdiv <8 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e2 = udiv <8 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e3 = srem <8 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e4 = urem <8 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s1 = sdiv <16 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s2 = udiv <16 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s3 = srem <16 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s4 = urem <16 x i16> undef, splat (i16 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi16_2'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i16> undef, splat (i16 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i16> undef, splat (i16 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vi16_2'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i16> undef, splat (i16 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i16> undef, splat (i16 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vi16_2'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f1 = sdiv <4 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %f2 = udiv <4 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i16> undef, splat (i16 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %f1 = sdiv <4 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 10 for: %f2 = udiv <4 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i16> undef, splat (i16 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %t1 = sdiv <2 x i16> undef, <i16 2, i16 2>
   %t2 = udiv <2 x i16> undef, <i16 2, i16 2>
@@ -1261,99 +1261,99 @@ define void @vi16_2() {
 
 define void @vi32_2() {
 ; CHECK-NEON-LABEL: 'vi32_2'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i32> undef, splat (i32 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f1 = sdiv <4 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f2 = udiv <4 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i32> undef, splat (i32 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi32_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t1 = sdiv <2 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t2 = udiv <2 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t3 = srem <2 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %t4 = urem <2 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f1 = sdiv <4 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f2 = udiv <4 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f3 = srem <4 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %f4 = urem <4 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e1 = sdiv <8 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e2 = udiv <8 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e3 = srem <8 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %e4 = urem <8 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s1 = sdiv <16 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s2 = udiv <16 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s3 = srem <16 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %s4 = urem <16 x i32> undef, splat (i32 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t1 = sdiv <2 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t2 = udiv <2 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t3 = srem <2 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 10 for: %t4 = urem <2 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f1 = sdiv <4 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f2 = udiv <4 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f3 = srem <4 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %f4 = urem <4 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e1 = sdiv <8 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e2 = udiv <8 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e3 = srem <8 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %e4 = urem <8 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s1 = sdiv <16 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s2 = udiv <16 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s3 = srem <16 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %s4 = urem <16 x i32> undef, splat (i32 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi32_2'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i32> undef, splat (i32 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i32> undef, splat (i32 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vi32_2'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t1 = sdiv <2 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %t2 = udiv <2 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t3 = srem <2 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t4 = urem <2 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f1 = sdiv <4 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %f2 = udiv <4 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f3 = srem <4 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f4 = urem <4 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e1 = sdiv <8 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %e2 = udiv <8 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e3 = srem <8 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e4 = urem <8 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s1 = sdiv <16 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %s2 = udiv <16 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s3 = srem <16 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s4 = urem <16 x i32> undef, splat (i32 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t1 = sdiv <2 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %t2 = udiv <2 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t3 = srem <2 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t4 = urem <2 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f1 = sdiv <4 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %f2 = udiv <4 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f3 = srem <4 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f4 = urem <4 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e1 = sdiv <8 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %e2 = udiv <8 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e3 = srem <8 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e4 = urem <8 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s1 = sdiv <16 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %s2 = udiv <16 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s3 = srem <16 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s4 = urem <16 x i32> undef, splat (i32 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vi32_2'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i32> undef, splat (i32 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f1 = sdiv <4 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f2 = udiv <4 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i32> undef, splat (i32 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %t1 = sdiv <2 x i32> undef, <i32 2, i32 2>
   %t2 = udiv <2 x i32> undef, <i32 2, i32 2>
@@ -1376,99 +1376,99 @@ define void @vi32_2() {
 
 define void @vi64_2() {
 ; CHECK-NEON-LABEL: 'vi64_2'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i64> undef, splat (i64 2)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f1 = sdiv <4 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f2 = udiv <4 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i64> undef, splat (i64 2)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vi64_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %t1 = sdiv <2 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %t2 = udiv <2 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %t3 = srem <2 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %t4 = urem <2 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f1 = sdiv <4 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %f2 = udiv <4 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %f3 = srem <4 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %f4 = urem <4 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e1 = sdiv <8 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %e2 = udiv <8 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e3 = srem <8 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %e4 = urem <8 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %s1 = sdiv <16 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %s2 = udiv <16 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %s3 = srem <16 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 144 for instruction: %s4 = urem <16 x i64> undef, splat (i64 2)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %t1 = sdiv <2 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 20 for: %t2 = udiv <2 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 18 for: %t3 = srem <2 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 18 for: %t4 = urem <2 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %f1 = sdiv <4 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 40 for: %f2 = udiv <4 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 36 for: %f3 = srem <4 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 36 for: %f4 = urem <4 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %e1 = sdiv <8 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 80 for: %e2 = udiv <8 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 72 for: %e3 = srem <8 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 72 for: %e4 = urem <8 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 160 for: %s1 = sdiv <16 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 160 for: %s2 = udiv <16 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 144 for: %s3 = srem <16 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 144 for: %s4 = urem <16 x i64> undef, splat (i64 2)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vi64_2'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t1 = sdiv <2 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t2 = udiv <2 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f1 = sdiv <4 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f2 = udiv <4 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f3 = srem <4 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f4 = urem <4 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e1 = sdiv <8 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e2 = udiv <8 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e3 = srem <8 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e4 = urem <8 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s1 = sdiv <16 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s2 = udiv <16 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %s3 = srem <16 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %s4 = urem <16 x i64> undef, splat (i64 2)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t1 = sdiv <2 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %t2 = udiv <2 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 6 for: %t3 = srem <2 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 6 for: %t4 = urem <2 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f1 = sdiv <4 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %f2 = udiv <4 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 12 for: %f3 = srem <4 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 12 for: %f4 = urem <4 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e1 = sdiv <8 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %e2 = udiv <8 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 24 for: %e3 = srem <8 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 24 for: %e4 = urem <8 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s1 = sdiv <16 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 32 for: %s2 = udiv <16 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 48 for: %s3 = srem <16 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 48 for: %s4 = urem <16 x i64> undef, splat (i64 2)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vi64_2'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t1 = sdiv <2 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %t2 = udiv <2 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t3 = srem <2 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %t4 = urem <2 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f1 = sdiv <4 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %f2 = udiv <4 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f3 = srem <4 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %f4 = urem <4 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e1 = sdiv <8 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %e2 = udiv <8 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e3 = srem <8 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %e4 = urem <8 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s1 = sdiv <16 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %s2 = udiv <16 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %s3 = srem <16 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %s4 = urem <16 x i64> undef, splat (i64 2)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t1 = sdiv <2 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %t2 = udiv <2 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 6 for: %t3 = srem <2 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 6 for: %t4 = urem <2 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f1 = sdiv <4 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %f2 = udiv <4 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 12 for: %f3 = srem <4 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 12 for: %f4 = urem <4 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e1 = sdiv <8 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %e2 = udiv <8 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 24 for: %e3 = srem <8 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 24 for: %e4 = urem <8 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s1 = sdiv <16 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 32 for: %s2 = udiv <16 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 48 for: %s3 = srem <16 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 48 for: %s4 = urem <16 x i64> undef, splat (i64 2)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vi64_2'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t1 = sdiv <2 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t2 = udiv <2 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t3 = srem <2 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %t4 = urem <2 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f1 = sdiv <4 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f2 = udiv <4 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f3 = srem <4 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %f4 = urem <4 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e1 = sdiv <8 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e2 = udiv <8 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e3 = srem <8 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %e4 = urem <8 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s1 = sdiv <16 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s2 = udiv <16 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s3 = srem <16 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %s4 = urem <16 x i64> undef, splat (i64 2)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t1 = sdiv <2 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t2 = udiv <2 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t3 = srem <2 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 40 for: %t4 = urem <2 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f1 = sdiv <4 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f2 = udiv <4 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f3 = srem <4 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 80 for: %f4 = urem <4 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e1 = sdiv <8 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e2 = udiv <8 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e3 = srem <8 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 160 for: %e4 = urem <8 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s1 = sdiv <16 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s2 = udiv <16 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s3 = srem <16 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 320 for: %s4 = urem <16 x i64> undef, splat (i64 2)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %t1 = sdiv <2 x i64> undef, <i64 2, i64 2>
   %t2 = udiv <2 x i64> undef, <i64 2, i64 2>
@@ -1491,49 +1491,49 @@ define void @vi64_2() {
 
 define void @vf16_2() {
 ; CHECK-NEON-LABEL: 'vf16_2'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = frem <2 x half> undef, splat (half 0xH4000)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = frem <4 x half> undef, splat (half 0xH4000)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = frem <8 x half> undef, splat (half 0xH4000)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x half> undef, splat (half 0xH4000)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x half> undef, splat (half 0xH4000)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x half> undef, splat (half 0xH4000)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf16_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x half> undef, splat (half 0xH4000)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x half> undef, splat (half 0xH4000)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x half> undef, splat (half 0xH4000)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x half> undef, splat (half 0xH4000)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x half> undef, splat (half 0xH4000)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x half> undef, splat (half 0xH4000)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf16_2'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem <2 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = frem <4 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %6 = frem <8 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %2 = frem <2 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %4 = frem <4 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %6 = frem <8 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vf16_2'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem <2 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = frem <4 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %6 = frem <8 x half> undef, splat (half 0xH4000)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %2 = frem <2 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %4 = frem <4 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %6 = frem <8 x half> undef, splat (half 0xH4000)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vf16_2'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = frem <2 x half> undef, splat (half 0xH4000)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = frem <4 x half> undef, splat (half 0xH4000)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = frem <8 x half> undef, splat (half 0xH4000)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x half> undef, splat (half 0xH4000)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x half> undef, splat (half 0xH4000)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x half> undef, splat (half 0xH4000)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x half> undef, splat (half 0xH4000)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x half> undef, splat (half 0xH4000)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x half> undef, splat (half 0xH4000)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = fdiv <2 x half> undef, <half 2., half 2.>
   %2 = frem <2 x half> undef, <half 2., half 2.>
@@ -1546,49 +1546,49 @@ define void @vf16_2() {
 
 define void @vf32_2() {
 ; CHECK-NEON-LABEL: 'vf32_2'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf32_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf32_2'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 2 for: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vf32_2'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 2 for: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vf32_2'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:48 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x float> undef, splat (float 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = fdiv <2 x float> undef, <float 2., float 2.>
   %2 = frem <2 x float> undef, <float 2., float 2.>
@@ -1601,49 +1601,49 @@ define void @vf32_2() {
 
 define void @vf64_2() {
 ; CHECK-NEON-LABEL: 'vf64_2'
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-NEON-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-NEON-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-MVE-LABEL: 'vf64_2'
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-MVE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-MVE-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
 ; CHECK-V8M-MAIN-LABEL: 'vf64_2'
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-MAIN-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-MAIN-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8M-BASE-LABEL: 'vf64_2'
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8M-BASE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 4 for: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 8 for: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 16 for: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8M-BASE-NEXT:  Cost Model: Found costs of 1 for: ret void
 ;
 ; CHECK-V8R-LABEL: 'vf64_2'
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
-; CHECK-V8R-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %1 = fdiv <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %2 = frem <2 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %3 = fdiv <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %4 = frem <4 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %5 = fdiv <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %6 = frem <8 x double> undef, splat (double 2.000000e+00)
+; CHECK-V8R-NEXT:  Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
 ;
   %1 = fdiv <2 x double> undef, <double 2., double 2.>
   %2 = frem <2 x double> undef, <double 2., double 2.>
diff --git a/llvm/test/Analysis/CostModel/ARM/mve-shuffle-loadstore.ll b/llvm/test/Analysis/CostModel/ARM/mve-shuffle-loadstore.ll
index 6a327cfed4e4c..ef0b28ea2604a 100644
--- a/llvm/test/Analysis/CostModel/ARM/mve-shuffle-loadstore.ll
+++ b/llvm/test/Analysis/CostModel/ARM/mve-shuffle-loadstore.ll
@@ -7,41 +7,41 @@
 define void @vld2(ptr %p) {
 ; CHECK-LABEL: 'vld2'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = load <4 x i8>, ptr %p, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v4i8_0 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v4i8_1 = shufflevector <4 x i8> %v4i8, <4 x i8> undef, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = load <8 x i8>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v8i8_0 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v8i8_1 = shufflevector <8 x i8> %v8i8, <8 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8 = load <16 x i8>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v16i8_0 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v16i8_1 = shufflevector <16 x i8> %v16i8, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8 = load <32 x i8>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 768 for instruction: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 768 for instruction: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %v32i8_0 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %v32i8_1 = shufflevector <32 x i8> %v32i8, <32 x i8> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = load <4 x i16>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v4i16_0 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v4i16_1 = shufflevector <4 x i16> %v4i16, <4 x i16> undef, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16 = load <8 x i16>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v8i16_0 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v8i16_1 = shufflevector <8 x i16> %v8i16, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16 = load <16 x i16>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v16i16_0 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v16i16_1 = shufflevector <16 x i16> %v16i16, <16 x i16> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16 = load <32 x i16>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 768 for instruction: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 768 for instruction: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %v32i16_0 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %v32i16_1 = shufflevector <32 x i16> %v32i16, <32 x i16> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32 = load <4 x i32>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 0, i32 2>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v4i32_0 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v4i32_1 = shufflevector <4 x i32> %v4i32, <4 x i32> undef, <2 x i32> <i32 1, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32 = load <8 x i32>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v8i32_0 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v8i32_1 = shufflevector <8 x i32> %v8i32, <8 x i32> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32 = load <16 x i32>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 384 for instruction: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v16i32_0 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v16i32_1 = shufflevector <16 x i32> %v16i32, <16 x i32> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i32 = load <32 x i32>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 768 for instruction: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 768 for instruction: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %v32i32_0 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %v32i32_1 = shufflevector <32 x i32> %v32i32, <32 x i32> undef, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i64 = load <4 x i64>, ptr %p, align 32
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v2i64_0 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 0, i32 2>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %v2i64_1 = shufflevector <4 x i64> %v2i64, <4 x i64> undef, <2 x i32> <i32 1, i32 3>
@@ -252,88 +252,171 @@ define void @vld3(ptr %p) {
 }
 
 define void @vld4(ptr %p) {
-; CHECK-LABEL: 'vld4'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8 = load <8 x i8>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 1, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = load <16 x i8>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8 = load <32 x i8>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8 = load <64 x i8>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16 = load <8 x i16>, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 1, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i16 = load <16 x i16>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i16 = load <32 x i16>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i16 = load <64 x i16>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i32 = load <8 x i32>, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 1, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i32 = load <16 x i32>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i32 = load <32 x i32>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i32 = load <64 x i32>, ptr %p, align 256
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i64 = load <8 x i64>, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 0, i32 4>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 1, i32 5>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 2, i32 6>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 3, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i64 = load <16 x i64>, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8i64 = load <32 x i64>, ptr %p, align 256
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16i64 = load <64 x i64>, ptr %p, align 512
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-UF2-LABEL: 'vld4'
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8 = load <8 x i8>, ptr %p, align 8
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = load <16 x i8>, ptr %p, align 8
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8 = load <32 x i8>, ptr %p, align 32
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8 = load <64 x i8>, ptr %p, align 64
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16 = load <8 x i16>, ptr %p, align 8
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i16 = load <16 x i16>, ptr %p, align 32
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i16 = load <32 x i16>, ptr %p, align 64
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i16 = load <64 x i16>, ptr %p, align 128
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i32 = load <8 x i32>, ptr %p, align 32
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i32 = load <16 x i32>, ptr %p, align 64
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i32 = load <32 x i32>, ptr %p, align 128
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i32 = load <64 x i32>, ptr %p, align 256
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i64 = load <8 x i64>, ptr %p, align 64
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i64 = load <16 x i64>, ptr %p, align 128
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8i64 = load <32 x i64>, ptr %p, align 256
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16i64 = load <64 x i64>, ptr %p, align 512
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-UF4-LABEL: 'vld4'
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8 = load <8 x i8>, ptr %p, align 8
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i8_1 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i8_2 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i8_3 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = load <16 x i8>, ptr %p, align 8
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i8_0 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i8_1 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i8_2 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i8_3 = shufflevector <16 x i8> %v4i8, <16 x i8> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i8 = load <32 x i8>, ptr %p, align 32
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v8i8_0 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v8i8_1 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v8i8_2 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v8i8_3 = shufflevector <32 x i8> %v8i8, <32 x i8> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i8 = load <64 x i8>, ptr %p, align 64
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %v16i8_0 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %v16i8_1 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %v16i8_2 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 258 for instruction: %v16i8_3 = shufflevector <64 x i8> %v16i8, <64 x i8> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16 = load <8 x i16>, ptr %p, align 8
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i16_0 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i16_1 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i16_2 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i16_3 = shufflevector <8 x i16> %v2i16, <8 x i16> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i16 = load <16 x i16>, ptr %p, align 32
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i16_0 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i16_1 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i16_2 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i16_3 = shufflevector <16 x i16> %v4i16, <16 x i16> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i16 = load <32 x i16>, ptr %p, align 64
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v8i16_0 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v8i16_1 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v8i16_2 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 130 for instruction: %v8i16_3 = shufflevector <32 x i16> %v8i16, <32 x i16> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i16 = load <64 x i16>, ptr %p, align 128
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %v16i16_0 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %v16i16_1 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %v16i16_2 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 260 for instruction: %v16i16_3 = shufflevector <64 x i16> %v16i16, <64 x i16> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v2i32 = load <8 x i32>, ptr %p, align 32
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i32_0 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i32_1 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i32_2 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v2i32_3 = shufflevector <8 x i32> %v2i32, <8 x i32> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v4i32 = load <16 x i32>, ptr %p, align 64
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i32_0 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i32_1 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i32_2 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %v4i32_3 = shufflevector <16 x i32> %v4i32, <16 x i32> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v8i32 = load <32 x i32>, ptr %p, align 128
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %v8i32_0 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %v8i32_1 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %v8i32_2 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %v8i32_3 = shufflevector <32 x i32> %v8i32, <32 x i32> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v16i32 = load <64 x i32>, ptr %p, align 256
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %v16i32_0 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %v16i32_1 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %v16i32_2 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 264 for instruction: %v16i32_3 = shufflevector <64 x i32> %v16i32, <64 x i32> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v2i64 = load <8 x i64>, ptr %p, align 64
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_0 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 0, i32 4>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_1 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 1, i32 5>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_2 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 2, i32 6>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v2i64_3 = shufflevector <8 x i64> %v2i64, <8 x i64> undef, <2 x i32> <i32 3, i32 7>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v4i64 = load <16 x i64>, ptr %p, align 128
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_0 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_1 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_2 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 640 for instruction: %v4i64_3 = shufflevector <16 x i64> %v4i64, <16 x i64> undef, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v8i64 = load <32 x i64>, ptr %p, align 256
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_0 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_1 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_2 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 1280 for instruction: %v8i64_3 = shufflevector <32 x i64> %v8i64, <32 x i64> undef, <8 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v16i64 = load <64 x i64>, ptr %p, align 512
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_0 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 32, i32 36, i32 40, i32 44, i32 48, i32 52, i32 56, i32 60>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_1 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 33, i32 37, i32 41, i32 45, i32 49, i32 53, i32 57, i32 61>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_2 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 34, i32 38, i32 42, i32 46, i32 50, i32 54, i32 58, i32 62>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2560 for instruction: %v16i64_3 = shufflevector <64 x i64> %v16i64, <64 x i64> undef, <16 x i32> <i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31, i32 35, i32 39, i32 43, i32 47, i32 51, i32 55, i32 59, i32 63>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = load <8 x i8>, ptr %p
   %v2i8_0 = shufflevector <8 x i8> %v2i8, <8 x i8> undef, <2 x i32> <i32 0, i32 4>
@@ -424,29 +507,29 @@ define void @vld4(ptr %p) {
 
 define void @vst2(ptr %p) {
 ; CHECK-LABEL: 'vst2'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8 = shufflevector <2 x i8> undef, <2 x i8> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %v4i8, ptr %p, align 4
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = shufflevector <4 x i8> undef, <4 x i8> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8i8, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16i8, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v32i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <32 x i8> %v32i8, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i16 = shufflevector <2 x i16> undef, <2 x i16> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> %v4i16, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16 = shufflevector <4 x i16> undef, <4 x i16> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v8i16, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <16 x i16> %v16i16, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v32i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <32 x i16> %v32i16, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v4i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32 = shufflevector <2 x i32> undef, <2 x i32> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> %v4i32, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32 = shufflevector <4 x i32> undef, <4 x i32> undef, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <8 x i32> %v8i32, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <16 x i32> %v16i32, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v32i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <32 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <32 x i32> %v32i32, ptr %p, align 128
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v4i64 = shufflevector <2 x i64> undef, <2 x i64> undef, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <4 x i64> %v4i64, ptr %p, align 32
@@ -575,40 +658,75 @@ define void @vst3(ptr %p) {
 
 
 define void @vst4(ptr %p) {
-; CHECK-LABEL: 'vst4'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8i8, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16i8, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v32i8 = shufflevector <32 x i8> undef, <32 x i8> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <32 x i8> %v32i8, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v64i8 = shufflevector <64 x i8> undef, <64 x i8> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <64 x i8> %v64i8, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v8i16, ptr %p, align 8
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <16 x i16> %v16i16, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v32i16 = shufflevector <32 x i16> undef, <32 x i16> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <32 x i16> %v32i16, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v64i16 = shufflevector <64 x i16> undef, <64 x i16> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <64 x i16> %v64i16, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <8 x i32> %v8i32, ptr %p, align 32
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <16 x i32> %v16i32, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v32i32 = shufflevector <32 x i32> undef, <32 x i32> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <32 x i32> %v32i32, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v64i32 = shufflevector <64 x i32> undef, <64 x i32> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <64 x i32> %v64i32, ptr %p, align 256
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i64> %v8i64, ptr %p, align 64
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i64> %v16i64, ptr %p, align 128
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v32i64 = shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <32 x i64> %v32i64, ptr %p, align 256
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2048 for instruction: %v64i64 = shufflevector <64 x i64> undef, <64 x i64> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: store <64 x i64> %v64i64, ptr %p, align 512
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; CHECK-UF2-LABEL: 'vst4'
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8i8, ptr %p, align 8
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16i8, ptr %p, align 8
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v32i8 = shufflevector <32 x i8> undef, <32 x i8> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <32 x i8> %v32i8, ptr %p, align 32
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v64i8 = shufflevector <64 x i8> undef, <64 x i8> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <64 x i8> %v64i8, ptr %p, align 64
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v8i16, ptr %p, align 8
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <16 x i16> %v16i16, ptr %p, align 32
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v32i16 = shufflevector <32 x i16> undef, <32 x i16> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <32 x i16> %v32i16, ptr %p, align 64
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v64i16 = shufflevector <64 x i16> undef, <64 x i16> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <64 x i16> %v64i16, ptr %p, align 128
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <8 x i32> %v8i32, ptr %p, align 32
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <16 x i32> %v16i32, ptr %p, align 64
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v32i32 = shufflevector <32 x i32> undef, <32 x i32> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <32 x i32> %v32i32, ptr %p, align 128
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v64i32 = shufflevector <64 x i32> undef, <64 x i32> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <64 x i32> %v64i32, ptr %p, align 256
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i64> %v8i64, ptr %p, align 64
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i64> %v16i64, ptr %p, align 128
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v32i64 = shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <32 x i64> %v32i64, ptr %p, align 256
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 2048 for instruction: %v64i64 = shufflevector <64 x i64> undef, <64 x i64> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: store <64 x i64> %v64i64, ptr %p, align 512
+; CHECK-UF2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; CHECK-UF4-LABEL: 'vst4'
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i8> %v8i8, ptr %p, align 8
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i8 = shufflevector <16 x i8> undef, <16 x i8> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <16 x i8> %v16i8, ptr %p, align 8
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32i8 = shufflevector <32 x i8> undef, <32 x i8> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <32 x i8> %v32i8, ptr %p, align 32
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v64i8 = shufflevector <64 x i8> undef, <64 x i8> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <64 x i8> %v64i8, ptr %p, align 64
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16 = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: store <8 x i16> %v8i16, ptr %p, align 8
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16 = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <16 x i16> %v16i16, ptr %p, align 32
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16 = shufflevector <32 x i16> undef, <32 x i16> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <32 x i16> %v32i16, ptr %p, align 64
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i16 = shufflevector <64 x i16> undef, <64 x i16> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <64 x i16> %v64i16, ptr %p, align 128
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32 = shufflevector <8 x i32> undef, <8 x i32> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: store <8 x i32> %v8i32, ptr %p, align 32
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <16 x i32> %v16i32, ptr %p, align 64
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i32 = shufflevector <32 x i32> undef, <32 x i32> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <32 x i32> %v32i32, ptr %p, align 128
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v64i32 = shufflevector <64 x i32> undef, <64 x i32> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <64 x i32> %v64i32, ptr %p, align 256
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 256 for instruction: %v8i64 = shufflevector <8 x i64> undef, <8 x i64> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: store <8 x i64> %v8i64, ptr %p, align 64
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 512 for instruction: %v16i64 = shufflevector <16 x i64> undef, <16 x i64> undef, <16 x i32> <i32 0, i32 4, i32 8, i32 12, i32 1, i32 5, i32 9, i32 13, i32 2, i32 6, i32 10, i32 14, i32 3, i32 7, i32 11, i32 15>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: store <16 x i64> %v16i64, ptr %p, align 128
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 1024 for instruction: %v32i64 = shufflevector <32 x i64> undef, <32 x i64> undef, <32 x i32> <i32 0, i32 8, i32 16, i32 24, i32 1, i32 9, i32 17, i32 25, i32 2, i32 10, i32 18, i32 26, i32 3, i32 11, i32 19, i32 27, i32 4, i32 12, i32 20, i32 28, i32 5, i32 13, i32 21, i32 29, i32 6, i32 14, i32 22, i32 30, i32 7, i32 15, i32 23, i32 31>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: store <32 x i64> %v32i64, ptr %p, align 256
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 2048 for instruction: %v64i64 = shufflevector <64 x i64> undef, <64 x i64> undef, <64 x i32> <i32 0, i32 16, i32 32, i32 48, i32 1, i32 17, i32 33, i32 49, i32 2, i32 18, i32 34, i32 50, i32 3, i32 19, i32 35, i32 51, i32 4, i32 20, i32 36, i32 52, i32 5, i32 21, i32 37, i32 53, i32 6, i32 22, i32 38, i32 54, i32 7, i32 23, i32 39, i32 55, i32 8, i32 24, i32 40, i32 56, i32 9, i32 25, i32 41, i32 57, i32 10, i32 26, i32 42, i32 58, i32 11, i32 27, i32 43, i32 59, i32 12, i32 28, i32 44, i32 60, i32 13, i32 29, i32 45, i32 61, i32 14, i32 30, i32 46, i32 62, i32 15, i32 31, i32 47, i32 63>
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: store <64 x i64> %v64i64, ptr %p, align 512
+; CHECK-UF4-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v8i8 = shufflevector <8 x i8> undef, <8 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
   store <8 x i8> %v8i8, ptr %p
@@ -648,6 +766,3 @@ define void @vst4(ptr %p) {
 
   ret void
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-UF2: {{.*}}
-; CHECK-UF4: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/RISCV/cast.ll b/llvm/test/Analysis/CostModel/RISCV/cast.ll
index bdd8540a2c475..e64bce2d9c9e5 100644
--- a/llvm/test/Analysis/CostModel/RISCV/cast.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/cast.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput -intrinsic-cost-strategy=type-based-intrinsic-cost 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV32
-; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput -intrinsic-cost-strategy=type-based-intrinsic-cost 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,RV64
+; RUN: opt < %s -mtriple=riscv32 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput -intrinsic-cost-strategy=type-based-intrinsic-cost 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,ZVE64D
+; RUN: opt < %s -mtriple=riscv64 -mattr=+v,+zfh,+zvfh -passes="print<cost-model>" -cost-kind=throughput -intrinsic-cost-strategy=type-based-intrinsic-cost 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,ZVE64D
+; RUN: opt < %s -mtriple=riscv64 -mattr=+zvl128b,+zve64x -passes="print<cost-model>" -cost-kind=throughput -intrinsic-cost-strategy=type-based-intrinsic-cost 2>&1 -disable-output | FileCheck %s --check-prefixes=CHECK,ZVE64X
 
 define void @sext() {
 ; CHECK-LABEL: 'sext'
@@ -2021,92 +2022,179 @@ define void @trunc() {
 }
 
 define void @fpext() {
-; CHECK-LABEL: 'fpext'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f16_v2f32 = fpext <2 x half> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f16_v2f64 = fpext <2 x half> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2f64 = fpext <2 x float> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f16_v2f32 = call <2 x float> @llvm.vp.fpext.v2f32.v2f16(<2 x half> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f16_v2f64 = call <2 x double> @llvm.vp.fpext.v2f64.v2f16(<2 x half> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2f64 = call <2 x double> @llvm.vp.fpext.v2f64.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f16_v4f32 = fpext <4 x half> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f16_v4f64 = fpext <4 x half> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4f64 = fpext <4 x float> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f16_v4f32 = call <4 x float> @llvm.vp.fpext.v4f32.v4f16(<4 x half> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f16_v4f64 = call <4 x double> @llvm.vp.fpext.v4f64.v4f16(<4 x half> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4f64 = call <4 x double> @llvm.vp.fpext.v4f64.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f16_v8f32 = fpext <8 x half> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8f16_v8f64 = fpext <8 x half> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32_v8f64 = fpext <8 x float> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f16_v8f32 = call <8 x float> @llvm.vp.fpext.v8f32.v8f16(<8 x half> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8f16_v8f64 = call <8 x double> @llvm.vp.fpext.v8f64.v8f16(<8 x half> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f32_v8f64 = call <8 x double> @llvm.vp.fpext.v8f64.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f16_v16f32 = fpext <16 x half> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16f16_v16f64 = fpext <16 x half> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32_v16f64 = fpext <16 x float> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f16_v16f32 = call <16 x float> @llvm.vp.fpext.v16f32.v16f16(<16 x half> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16f16_v16f64 = call <16 x double> @llvm.vp.fpext.v16f64.v16f16(<16 x half> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f32_v16f64 = call <16 x double> @llvm.vp.fpext.v16f64.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32f16_v32f32 = fpext <32 x half> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32f16_v32f64 = fpext <32 x half> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32f32_v32f64 = fpext <32 x float> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32f16_v32f32 = call <32 x float> @llvm.vp.fpext.v32f32.v32f16(<32 x half> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32f16_v32f64 = call <32 x double> @llvm.vp.fpext.v32f64.v32f16(<32 x half> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32f32_v32f64 = call <32 x double> @llvm.vp.fpext.v32f64.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64f16_v64f32 = fpext <64 x half> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64f16_v64f64 = fpext <64 x half> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64f32_v64f64 = fpext <64 x float> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64f16_v64f32 = call <64 x float> @llvm.vp.fpext.v64f32.v64f16(<64 x half> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64f16_v64f64 = call <64 x double> @llvm.vp.fpext.v64f64.v64f16(<64 x half> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64f32_v64f64 = call <64 x double> @llvm.vp.fpext.v64f64.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128f16_v128f32 = fpext <128 x half> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %v128f16_v128f64 = fpext <128 x half> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128f32_v128f64 = fpext <128 x float> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v128f16_v128f32 = call <128 x float> @llvm.vp.fpext.v128f32.v128f16(<128 x half> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_v128f16_v128f64 = call <128 x double> @llvm.vp.fpext.v128f64.v128f16(<128 x half> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128f32_v128f64 = call <128 x double> @llvm.vp.fpext.v128f64.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16_nxv1f32 = fpext <vscale x 1 x half> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16_nxv1f64 = fpext <vscale x 1 x half> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1f64 = fpext <vscale x 1 x float> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f16_nxv1f32 = call <vscale x 1 x float> @llvm.vp.fpext.nxv1f32.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f16_nxv1f64 = call <vscale x 1 x double> @llvm.vp.fpext.nxv1f64.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1f64 = call <vscale x 1 x double> @llvm.vp.fpext.nxv1f64.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16_nxv2f32 = fpext <vscale x 2 x half> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f16_nxv2f64 = fpext <vscale x 2 x half> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2f64 = fpext <vscale x 2 x float> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f16_nxv2f32 = call <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f16_nxv2f64 = call <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2f64 = call <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16_nxv4f32 = fpext <vscale x 4 x half> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4f16_nxv4f64 = fpext <vscale x 4 x half> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f32_nxv4f64 = fpext <vscale x 4 x float> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f16_nxv4f32 = call <vscale x 4 x float> @llvm.vp.fpext.nxv4f32.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4f16_nxv4f64 = call <vscale x 4 x double> @llvm.vp.fpext.nxv4f64.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f32_nxv4f64 = call <vscale x 4 x double> @llvm.vp.fpext.nxv4f64.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f16_nxv8f32 = fpext <vscale x 8 x half> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8f16_nxv8f64 = fpext <vscale x 8 x half> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f32_nxv8f64 = fpext <vscale x 8 x float> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f16_nxv8f32 = call <vscale x 8 x float> @llvm.vp.fpext.nxv8f32.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8f16_nxv8f64 = call <vscale x 8 x double> @llvm.vp.fpext.nxv8f64.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f32_nxv8f64 = call <vscale x 8 x double> @llvm.vp.fpext.nxv8f64.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16f16_nxv16f32 = fpext <vscale x 16 x half> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16f16_nxv16f64 = fpext <vscale x 16 x half> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32_nxv16f64 = fpext <vscale x 16 x float> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16f16_nxv16f32 = call <vscale x 16 x float> @llvm.vp.fpext.nxv16f32.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16f16_nxv16f64 = call <vscale x 16 x double> @llvm.vp.fpext.nxv16f64.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16f32_nxv16f64 = call <vscale x 16 x double> @llvm.vp.fpext.nxv16f64.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32f16_nxv32f32 = fpext <vscale x 32 x half> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32f16_nxv32f64 = fpext <vscale x 32 x half> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32f32_nxv32f64 = fpext <vscale x 32 x float> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32f16_nxv32f32 = call <vscale x 32 x float> @llvm.vp.fpext.nxv32f32.nxv32f16(<vscale x 32 x half> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32f16_nxv32f64 = call <vscale x 32 x double> @llvm.vp.fpext.nxv32f64.nxv32f16(<vscale x 32 x half> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32f32_nxv32f64 = call <vscale x 32 x double> @llvm.vp.fpext.nxv32f64.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64f16_nxv64f32 = fpext <vscale x 64 x half> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %nxv64f16_nxv64f64 = fpext <vscale x 64 x half> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64f32_nxv64f64 = fpext <vscale x 64 x float> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv64f16_nxv64f32 = call <vscale x 64 x float> @llvm.vp.fpext.nxv64f32.nxv64f16(<vscale x 64 x half> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_nxv64f16_nxv64f64 = call <vscale x 64 x double> @llvm.vp.fpext.nxv64f64.nxv64f16(<vscale x 64 x half> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64f32_nxv64f64 = call <vscale x 64 x double> @llvm.vp.fpext.nxv64f64.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ZVE64D-LABEL: 'fpext'
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f16_v2f32 = fpext <2 x half> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f16_v2f64 = fpext <2 x half> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2f64 = fpext <2 x float> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f16_v2f32 = call <2 x float> @llvm.vp.fpext.v2f32.v2f16(<2 x half> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f16_v2f64 = call <2 x double> @llvm.vp.fpext.v2f64.v2f16(<2 x half> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2f64 = call <2 x double> @llvm.vp.fpext.v2f64.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f16_v4f32 = fpext <4 x half> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f16_v4f64 = fpext <4 x half> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4f64 = fpext <4 x float> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f16_v4f32 = call <4 x float> @llvm.vp.fpext.v4f32.v4f16(<4 x half> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f16_v4f64 = call <4 x double> @llvm.vp.fpext.v4f64.v4f16(<4 x half> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4f64 = call <4 x double> @llvm.vp.fpext.v4f64.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f16_v8f32 = fpext <8 x half> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8f16_v8f64 = fpext <8 x half> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32_v8f64 = fpext <8 x float> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f16_v8f32 = call <8 x float> @llvm.vp.fpext.v8f32.v8f16(<8 x half> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8f16_v8f64 = call <8 x double> @llvm.vp.fpext.v8f64.v8f16(<8 x half> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f32_v8f64 = call <8 x double> @llvm.vp.fpext.v8f64.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f16_v16f32 = fpext <16 x half> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16f16_v16f64 = fpext <16 x half> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32_v16f64 = fpext <16 x float> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f16_v16f32 = call <16 x float> @llvm.vp.fpext.v16f32.v16f16(<16 x half> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16f16_v16f64 = call <16 x double> @llvm.vp.fpext.v16f64.v16f16(<16 x half> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f32_v16f64 = call <16 x double> @llvm.vp.fpext.v16f64.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32f16_v32f32 = fpext <32 x half> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32f16_v32f64 = fpext <32 x half> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32f32_v32f64 = fpext <32 x float> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32f16_v32f32 = call <32 x float> @llvm.vp.fpext.v32f32.v32f16(<32 x half> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32f16_v32f64 = call <32 x double> @llvm.vp.fpext.v32f64.v32f16(<32 x half> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32f32_v32f64 = call <32 x double> @llvm.vp.fpext.v32f64.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64f16_v64f32 = fpext <64 x half> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64f16_v64f64 = fpext <64 x half> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64f32_v64f64 = fpext <64 x float> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64f16_v64f32 = call <64 x float> @llvm.vp.fpext.v64f32.v64f16(<64 x half> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64f16_v64f64 = call <64 x double> @llvm.vp.fpext.v64f64.v64f16(<64 x half> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64f32_v64f64 = call <64 x double> @llvm.vp.fpext.v64f64.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128f16_v128f32 = fpext <128 x half> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %v128f16_v128f64 = fpext <128 x half> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128f32_v128f64 = fpext <128 x float> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v128f16_v128f32 = call <128 x float> @llvm.vp.fpext.v128f32.v128f16(<128 x half> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_v128f16_v128f64 = call <128 x double> @llvm.vp.fpext.v128f64.v128f16(<128 x half> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128f32_v128f64 = call <128 x double> @llvm.vp.fpext.v128f64.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f16_nxv1f32 = fpext <vscale x 1 x half> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f16_nxv1f64 = fpext <vscale x 1 x half> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1f64 = fpext <vscale x 1 x float> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f16_nxv1f32 = call <vscale x 1 x float> @llvm.vp.fpext.nxv1f32.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f16_nxv1f64 = call <vscale x 1 x double> @llvm.vp.fpext.nxv1f64.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1f64 = call <vscale x 1 x double> @llvm.vp.fpext.nxv1f64.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f16_nxv2f32 = fpext <vscale x 2 x half> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f16_nxv2f64 = fpext <vscale x 2 x half> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2f64 = fpext <vscale x 2 x float> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f16_nxv2f32 = call <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f16_nxv2f64 = call <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2f64 = call <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f16_nxv4f32 = fpext <vscale x 4 x half> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4f16_nxv4f64 = fpext <vscale x 4 x half> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f32_nxv4f64 = fpext <vscale x 4 x float> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f16_nxv4f32 = call <vscale x 4 x float> @llvm.vp.fpext.nxv4f32.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4f16_nxv4f64 = call <vscale x 4 x double> @llvm.vp.fpext.nxv4f64.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f32_nxv4f64 = call <vscale x 4 x double> @llvm.vp.fpext.nxv4f64.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f16_nxv8f32 = fpext <vscale x 8 x half> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8f16_nxv8f64 = fpext <vscale x 8 x half> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f32_nxv8f64 = fpext <vscale x 8 x float> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f16_nxv8f32 = call <vscale x 8 x float> @llvm.vp.fpext.nxv8f32.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8f16_nxv8f64 = call <vscale x 8 x double> @llvm.vp.fpext.nxv8f64.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f32_nxv8f64 = call <vscale x 8 x double> @llvm.vp.fpext.nxv8f64.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16f16_nxv16f32 = fpext <vscale x 16 x half> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16f16_nxv16f64 = fpext <vscale x 16 x half> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32_nxv16f64 = fpext <vscale x 16 x float> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16f16_nxv16f32 = call <vscale x 16 x float> @llvm.vp.fpext.nxv16f32.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16f16_nxv16f64 = call <vscale x 16 x double> @llvm.vp.fpext.nxv16f64.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16f32_nxv16f64 = call <vscale x 16 x double> @llvm.vp.fpext.nxv16f64.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32f16_nxv32f32 = fpext <vscale x 32 x half> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32f16_nxv32f64 = fpext <vscale x 32 x half> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32f32_nxv32f64 = fpext <vscale x 32 x float> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32f16_nxv32f32 = call <vscale x 32 x float> @llvm.vp.fpext.nxv32f32.nxv32f16(<vscale x 32 x half> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32f16_nxv32f64 = call <vscale x 32 x double> @llvm.vp.fpext.nxv32f64.nxv32f16(<vscale x 32 x half> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32f32_nxv32f64 = call <vscale x 32 x double> @llvm.vp.fpext.nxv32f64.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64f16_nxv64f32 = fpext <vscale x 64 x half> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %nxv64f16_nxv64f64 = fpext <vscale x 64 x half> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64f32_nxv64f64 = fpext <vscale x 64 x float> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv64f16_nxv64f32 = call <vscale x 64 x float> @llvm.vp.fpext.nxv64f32.nxv64f16(<vscale x 64 x half> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_nxv64f16_nxv64f64 = call <vscale x 64 x double> @llvm.vp.fpext.nxv64f64.nxv64f16(<vscale x 64 x half> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64f32_nxv64f64 = call <vscale x 64 x double> @llvm.vp.fpext.nxv64f64.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; ZVE64X-LABEL: 'fpext'
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f16_v2f32 = fpext <2 x half> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f16_v2f64 = fpext <2 x half> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32_v2f64 = fpext <2 x float> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f16_v2f32 = call <2 x float> @llvm.vp.fpext.v2f32.v2f16(<2 x half> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f16_v2f64 = call <2 x double> @llvm.vp.fpext.v2f64.v2f16(<2 x half> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f32_v2f64 = call <2 x double> @llvm.vp.fpext.v2f64.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4f16_v4f32 = fpext <4 x half> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4f16_v4f64 = fpext <4 x half> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4f32_v4f64 = fpext <4 x float> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4f16_v4f32 = call <4 x float> @llvm.vp.fpext.v4f32.v4f16(<4 x half> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4f16_v4f64 = call <4 x double> @llvm.vp.fpext.v4f64.v4f16(<4 x half> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4f32_v4f64 = call <4 x double> @llvm.vp.fpext.v4f64.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8f16_v8f32 = fpext <8 x half> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8f16_v8f64 = fpext <8 x half> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8f32_v8f64 = fpext <8 x float> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8f16_v8f32 = call <8 x float> @llvm.vp.fpext.v8f32.v8f16(<8 x half> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8f16_v8f64 = call <8 x double> @llvm.vp.fpext.v8f64.v8f16(<8 x half> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8f32_v8f64 = call <8 x double> @llvm.vp.fpext.v8f64.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16f16_v16f32 = fpext <16 x half> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16f16_v16f64 = fpext <16 x half> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16f32_v16f64 = fpext <16 x float> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16f16_v16f32 = call <16 x float> @llvm.vp.fpext.v16f32.v16f16(<16 x half> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16f16_v16f64 = call <16 x double> @llvm.vp.fpext.v16f64.v16f16(<16 x half> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16f32_v16f64 = call <16 x double> @llvm.vp.fpext.v16f64.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v32f16_v32f32 = fpext <32 x half> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v32f16_v32f64 = fpext <32 x half> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v32f32_v32f64 = fpext <32 x float> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v32f16_v32f32 = call <32 x float> @llvm.vp.fpext.v32f32.v32f16(<32 x half> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v32f16_v32f64 = call <32 x double> @llvm.vp.fpext.v32f64.v32f16(<32 x half> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v32f32_v32f64 = call <32 x double> @llvm.vp.fpext.v32f64.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v64f16_v64f32 = fpext <64 x half> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v64f16_v64f64 = fpext <64 x half> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v64f32_v64f64 = fpext <64 x float> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v64f16_v64f32 = call <64 x float> @llvm.vp.fpext.v64f32.v64f16(<64 x half> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v64f16_v64f64 = call <64 x double> @llvm.vp.fpext.v64f64.v64f16(<64 x half> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v64f32_v64f64 = call <64 x double> @llvm.vp.fpext.v64f64.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v128f16_v128f32 = fpext <128 x half> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v128f16_v128f64 = fpext <128 x half> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v128f32_v128f64 = fpext <128 x float> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %vp_v128f16_v128f32 = call <128 x float> @llvm.vp.fpext.v128f32.v128f16(<128 x half> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %vp_v128f16_v128f64 = call <128 x double> @llvm.vp.fpext.v128f64.v128f16(<128 x half> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %vp_v128f32_v128f64 = call <128 x double> @llvm.vp.fpext.v128f64.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f16_nxv1f32 = fpext <vscale x 1 x half> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f16_nxv1f64 = fpext <vscale x 1 x half> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1f64 = fpext <vscale x 1 x float> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f16_nxv1f32 = call <vscale x 1 x float> @llvm.vp.fpext.nxv1f32.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f16_nxv1f64 = call <vscale x 1 x double> @llvm.vp.fpext.nxv1f64.nxv1f16(<vscale x 1 x half> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1f64 = call <vscale x 1 x double> @llvm.vp.fpext.nxv1f64.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f16_nxv2f32 = fpext <vscale x 2 x half> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f16_nxv2f64 = fpext <vscale x 2 x half> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2f64 = fpext <vscale x 2 x float> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f16_nxv2f32 = call <vscale x 2 x float> @llvm.vp.fpext.nxv2f32.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f16_nxv2f64 = call <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2f16(<vscale x 2 x half> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2f64 = call <vscale x 2 x double> @llvm.vp.fpext.nxv2f64.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f16_nxv4f32 = fpext <vscale x 4 x half> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f16_nxv4f64 = fpext <vscale x 4 x half> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4f64 = fpext <vscale x 4 x float> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f16_nxv4f32 = call <vscale x 4 x float> @llvm.vp.fpext.nxv4f32.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f16_nxv4f64 = call <vscale x 4 x double> @llvm.vp.fpext.nxv4f64.nxv4f16(<vscale x 4 x half> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4f64 = call <vscale x 4 x double> @llvm.vp.fpext.nxv4f64.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f16_nxv8f32 = fpext <vscale x 8 x half> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f16_nxv8f64 = fpext <vscale x 8 x half> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8f64 = fpext <vscale x 8 x float> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f16_nxv8f32 = call <vscale x 8 x float> @llvm.vp.fpext.nxv8f32.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f16_nxv8f64 = call <vscale x 8 x double> @llvm.vp.fpext.nxv8f64.nxv8f16(<vscale x 8 x half> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8f64 = call <vscale x 8 x double> @llvm.vp.fpext.nxv8f64.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f16_nxv16f32 = fpext <vscale x 16 x half> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f16_nxv16f64 = fpext <vscale x 16 x half> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16f64 = fpext <vscale x 16 x float> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f16_nxv16f32 = call <vscale x 16 x float> @llvm.vp.fpext.nxv16f32.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f16_nxv16f64 = call <vscale x 16 x double> @llvm.vp.fpext.nxv16f64.nxv16f16(<vscale x 16 x half> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16f64 = call <vscale x 16 x double> @llvm.vp.fpext.nxv16f64.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f16_nxv32f32 = fpext <vscale x 32 x half> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f16_nxv32f64 = fpext <vscale x 32 x half> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32f64 = fpext <vscale x 32 x float> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f16_nxv32f32 = call <vscale x 32 x float> @llvm.vp.fpext.nxv32f32.nxv32f16(<vscale x 32 x half> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f16_nxv32f64 = call <vscale x 32 x double> @llvm.vp.fpext.nxv32f64.nxv32f16(<vscale x 32 x half> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32f64 = call <vscale x 32 x double> @llvm.vp.fpext.nxv32f64.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f16_nxv64f32 = fpext <vscale x 64 x half> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f16_nxv64f64 = fpext <vscale x 64 x half> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64f64 = fpext <vscale x 64 x float> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f16_nxv64f32 = call <vscale x 64 x float> @llvm.vp.fpext.nxv64f32.nxv64f16(<vscale x 64 x half> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f16_nxv64f64 = call <vscale x 64 x double> @llvm.vp.fpext.nxv64f64.nxv64f16(<vscale x 64 x half> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64f64 = call <vscale x 64 x double> @llvm.vp.fpext.nxv64f64.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2f16_v2f32 = fpext <2 x half> undef to <2 x float>
   %v2f16_v2f64 = fpext <2 x half> undef to <2 x double>
@@ -2224,92 +2312,179 @@ define void @fpext() {
 }
 
 define void @fptrunc() {
-; CHECK-LABEL: 'fptrunc'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2f16 = fptrunc <2 x float> undef to <2 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f64_v2f16 = fptrunc <2 x double> undef to <2 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2f32 = fptrunc <2 x double> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2f16 = call <2 x half> @llvm.vp.fptrunc.v2f16.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f64_v2f16 = call <2 x half> @llvm.vp.fptrunc.v2f16.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2f32 = call <2 x float> @llvm.vp.fptrunc.v2f32.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4f16 = fptrunc <4 x float> undef to <4 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4f16 = fptrunc <4 x double> undef to <4 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_v4f32 = fptrunc <4 x double> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4f16 = call <4 x half> @llvm.vp.fptrunc.v4f16.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4f16 = call <4 x half> @llvm.vp.fptrunc.v4f16.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f64_v4f32 = call <4 x float> @llvm.vp.fptrunc.v4f32.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_v8f16 = fptrunc <8 x float> undef to <8 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f64_v8f16 = fptrunc <8 x double> undef to <8 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f64_v8f32 = fptrunc <8 x double> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v8f32_v8f16 = call <8 x half> @llvm.vp.fptrunc.v8f16.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f64_v8f16 = call <8 x half> @llvm.vp.fptrunc.v8f16.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f64_v8f32 = call <8 x float> @llvm.vp.fptrunc.v8f32.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f32_v16f16 = fptrunc <16 x float> undef to <16 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f64_v16f16 = fptrunc <16 x double> undef to <16 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f64_v16f32 = fptrunc <16 x double> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v16f32_v16f16 = call <16 x half> @llvm.vp.fptrunc.v16f16.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f64_v16f16 = call <16 x half> @llvm.vp.fptrunc.v16f16.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f64_v16f32 = call <16 x float> @llvm.vp.fptrunc.v16f32.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32f32_v32f16 = fptrunc <32 x float> undef to <32 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32f64_v32f16 = fptrunc <32 x double> undef to <32 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32f64_v32f32 = fptrunc <32 x double> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32f32_v32f16 = call <32 x half> @llvm.vp.fptrunc.v32f16.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v32f64_v32f16 = call <32 x half> @llvm.vp.fptrunc.v32f16.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32f64_v32f32 = call <32 x float> @llvm.vp.fptrunc.v32f32.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64f32_v64f16 = fptrunc <64 x float> undef to <64 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v64f64_v64f16 = fptrunc <64 x double> undef to <64 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64f64_v64f32 = fptrunc <64 x double> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v64f32_v64f16 = call <64 x half> @llvm.vp.fptrunc.v64f16.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v64f64_v64f16 = call <64 x half> @llvm.vp.fptrunc.v64f16.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64f64_v64f32 = call <64 x float> @llvm.vp.fptrunc.v64f32.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v128f32_v128f16 = fptrunc <128 x float> undef to <128 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %v128f64_v128f16 = fptrunc <128 x double> undef to <128 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128f64_v128f32 = fptrunc <128 x double> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v128f32_v128f16 = call <128 x half> @llvm.vp.fptrunc.v128f16.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_v128f64_v128f16 = call <128 x half> @llvm.vp.fptrunc.v128f16.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128f64_v128f32 = call <128 x float> @llvm.vp.fptrunc.v128f32.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1f16 = fptrunc <vscale x 1 x float> undef to <vscale x 1 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64_nxv1f16 = fptrunc <vscale x 1 x double> undef to <vscale x 1 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1f32 = fptrunc <vscale x 1 x double> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1f16 = call <vscale x 1 x half> @llvm.vp.fptrunc.nxv1f16.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f64_nxv1f16 = call <vscale x 1 x half> @llvm.vp.fptrunc.nxv1f16.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1f32 = call <vscale x 1 x float> @llvm.vp.fptrunc.nxv1f32.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv1f16 = fptrunc <vscale x 2 x float> undef to <vscale x 2 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv1f16 = fptrunc <vscale x 2 x double> undef to <vscale x 2 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64_nxv1f32 = fptrunc <vscale x 2 x double> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2f16 = call <vscale x 2 x half> @llvm.vp.fptrunc.nxv2f16.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2f16 = call <vscale x 2 x half> @llvm.vp.fptrunc.nxv2f16.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f64_nxv2f32 = call <vscale x 2 x float> @llvm.vp.fptrunc.nxv2f32.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32_nxv4f16 = fptrunc <vscale x 4 x float> undef to <vscale x 4 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f64_nxv4f16 = fptrunc <vscale x 4 x double> undef to <vscale x 4 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f64_nxv4f32 = fptrunc <vscale x 4 x double> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv4f32_nxv4f16 = call <vscale x 4 x half> @llvm.vp.fptrunc.nxv4f16.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f64_nxv4f16 = call <vscale x 4 x half> @llvm.vp.fptrunc.nxv4f16.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f64_nxv4f32 = call <vscale x 4 x float> @llvm.vp.fptrunc.nxv4f32.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8f32_nxv8f16 = fptrunc <vscale x 8 x float> undef to <vscale x 8 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f64_nxv8f16 = fptrunc <vscale x 8 x double> undef to <vscale x 8 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f64_nxv8f32 = fptrunc <vscale x 8 x double> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8f32_nxv8f16 = call <vscale x 8 x half> @llvm.vp.fptrunc.nxv8f16.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f64_nxv8f16 = call <vscale x 8 x half> @llvm.vp.fptrunc.nxv8f16.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f64_nxv8f32 = call <vscale x 8 x float> @llvm.vp.fptrunc.nxv8f32.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16f32_nxv16f16 = fptrunc <vscale x 16 x float> undef to <vscale x 16 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv16f64_nxv16f16 = fptrunc <vscale x 16 x double> undef to <vscale x 16 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16f64_nxv16f32 = fptrunc <vscale x 16 x double> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16f32_nxv16f16 = call <vscale x 16 x half> @llvm.vp.fptrunc.nxv16f16.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv16f64_nxv16f16 = call <vscale x 16 x half> @llvm.vp.fptrunc.nxv16f16.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16f64_nxv16f32 = call <vscale x 16 x float> @llvm.vp.fptrunc.nxv16f32.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32f32_nxv32f16 = fptrunc <vscale x 32 x float> undef to <vscale x 32 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv32f64_nxv32f16 = fptrunc <vscale x 32 x double> undef to <vscale x 32 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32f64_nxv32f32 = fptrunc <vscale x 32 x double> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv32f32_nxv32f16 = call <vscale x 32 x half> @llvm.vp.fptrunc.nxv32f16.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv32f64_nxv32f16 = call <vscale x 32 x half> @llvm.vp.fptrunc.nxv32f16.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32f64_nxv32f32 = call <vscale x 32 x float> @llvm.vp.fptrunc.nxv32f32.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv64f32_nxv64f16 = fptrunc <vscale x 64 x float> undef to <vscale x 64 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %nxv64f64_nxv64f16 = fptrunc <vscale x 64 x double> undef to <vscale x 64 x half>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64f64_nxv64f32 = fptrunc <vscale x 64 x double> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv64f32_nxv64f16 = call <vscale x 64 x half> @llvm.vp.fptrunc.nxv64f16.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_nxv64f64_nxv64f16 = call <vscale x 64 x half> @llvm.vp.fptrunc.nxv64f16.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64f64_nxv64f32 = call <vscale x 64 x float> @llvm.vp.fptrunc.nxv64f32.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ZVE64D-LABEL: 'fptrunc'
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2f16 = fptrunc <2 x float> undef to <2 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f64_v2f16 = fptrunc <2 x double> undef to <2 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2f32 = fptrunc <2 x double> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2f16 = call <2 x half> @llvm.vp.fptrunc.v2f16.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f64_v2f16 = call <2 x half> @llvm.vp.fptrunc.v2f16.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2f32 = call <2 x float> @llvm.vp.fptrunc.v2f32.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4f16 = fptrunc <4 x float> undef to <4 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4f16 = fptrunc <4 x double> undef to <4 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_v4f32 = fptrunc <4 x double> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4f16 = call <4 x half> @llvm.vp.fptrunc.v4f16.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4f16 = call <4 x half> @llvm.vp.fptrunc.v4f16.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f64_v4f32 = call <4 x float> @llvm.vp.fptrunc.v4f32.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_v8f16 = fptrunc <8 x float> undef to <8 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f64_v8f16 = fptrunc <8 x double> undef to <8 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f64_v8f32 = fptrunc <8 x double> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v8f32_v8f16 = call <8 x half> @llvm.vp.fptrunc.v8f16.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f64_v8f16 = call <8 x half> @llvm.vp.fptrunc.v8f16.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f64_v8f32 = call <8 x float> @llvm.vp.fptrunc.v8f32.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f32_v16f16 = fptrunc <16 x float> undef to <16 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f64_v16f16 = fptrunc <16 x double> undef to <16 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f64_v16f32 = fptrunc <16 x double> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v16f32_v16f16 = call <16 x half> @llvm.vp.fptrunc.v16f16.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f64_v16f16 = call <16 x half> @llvm.vp.fptrunc.v16f16.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f64_v16f32 = call <16 x float> @llvm.vp.fptrunc.v16f32.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32f32_v32f16 = fptrunc <32 x float> undef to <32 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32f64_v32f16 = fptrunc <32 x double> undef to <32 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32f64_v32f32 = fptrunc <32 x double> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32f32_v32f16 = call <32 x half> @llvm.vp.fptrunc.v32f16.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v32f64_v32f16 = call <32 x half> @llvm.vp.fptrunc.v32f16.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32f64_v32f32 = call <32 x float> @llvm.vp.fptrunc.v32f32.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64f32_v64f16 = fptrunc <64 x float> undef to <64 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v64f64_v64f16 = fptrunc <64 x double> undef to <64 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64f64_v64f32 = fptrunc <64 x double> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v64f32_v64f16 = call <64 x half> @llvm.vp.fptrunc.v64f16.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v64f64_v64f16 = call <64 x half> @llvm.vp.fptrunc.v64f16.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64f64_v64f32 = call <64 x float> @llvm.vp.fptrunc.v64f32.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v128f32_v128f16 = fptrunc <128 x float> undef to <128 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %v128f64_v128f16 = fptrunc <128 x double> undef to <128 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128f64_v128f32 = fptrunc <128 x double> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v128f32_v128f16 = call <128 x half> @llvm.vp.fptrunc.v128f16.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_v128f64_v128f16 = call <128 x half> @llvm.vp.fptrunc.v128f16.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128f64_v128f32 = call <128 x float> @llvm.vp.fptrunc.v128f32.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1f16 = fptrunc <vscale x 1 x float> undef to <vscale x 1 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64_nxv1f16 = fptrunc <vscale x 1 x double> undef to <vscale x 1 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1f32 = fptrunc <vscale x 1 x double> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1f16 = call <vscale x 1 x half> @llvm.vp.fptrunc.nxv1f16.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f64_nxv1f16 = call <vscale x 1 x half> @llvm.vp.fptrunc.nxv1f16.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1f32 = call <vscale x 1 x float> @llvm.vp.fptrunc.nxv1f32.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv1f16 = fptrunc <vscale x 2 x float> undef to <vscale x 2 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv1f16 = fptrunc <vscale x 2 x double> undef to <vscale x 2 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64_nxv1f32 = fptrunc <vscale x 2 x double> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2f16 = call <vscale x 2 x half> @llvm.vp.fptrunc.nxv2f16.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2f16 = call <vscale x 2 x half> @llvm.vp.fptrunc.nxv2f16.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f64_nxv2f32 = call <vscale x 2 x float> @llvm.vp.fptrunc.nxv2f32.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32_nxv4f16 = fptrunc <vscale x 4 x float> undef to <vscale x 4 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f64_nxv4f16 = fptrunc <vscale x 4 x double> undef to <vscale x 4 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f64_nxv4f32 = fptrunc <vscale x 4 x double> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv4f32_nxv4f16 = call <vscale x 4 x half> @llvm.vp.fptrunc.nxv4f16.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f64_nxv4f16 = call <vscale x 4 x half> @llvm.vp.fptrunc.nxv4f16.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f64_nxv4f32 = call <vscale x 4 x float> @llvm.vp.fptrunc.nxv4f32.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8f32_nxv8f16 = fptrunc <vscale x 8 x float> undef to <vscale x 8 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f64_nxv8f16 = fptrunc <vscale x 8 x double> undef to <vscale x 8 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f64_nxv8f32 = fptrunc <vscale x 8 x double> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8f32_nxv8f16 = call <vscale x 8 x half> @llvm.vp.fptrunc.nxv8f16.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f64_nxv8f16 = call <vscale x 8 x half> @llvm.vp.fptrunc.nxv8f16.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f64_nxv8f32 = call <vscale x 8 x float> @llvm.vp.fptrunc.nxv8f32.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16f32_nxv16f16 = fptrunc <vscale x 16 x float> undef to <vscale x 16 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv16f64_nxv16f16 = fptrunc <vscale x 16 x double> undef to <vscale x 16 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16f64_nxv16f32 = fptrunc <vscale x 16 x double> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16f32_nxv16f16 = call <vscale x 16 x half> @llvm.vp.fptrunc.nxv16f16.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv16f64_nxv16f16 = call <vscale x 16 x half> @llvm.vp.fptrunc.nxv16f16.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16f64_nxv16f32 = call <vscale x 16 x float> @llvm.vp.fptrunc.nxv16f32.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32f32_nxv32f16 = fptrunc <vscale x 32 x float> undef to <vscale x 32 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv32f64_nxv32f16 = fptrunc <vscale x 32 x double> undef to <vscale x 32 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32f64_nxv32f32 = fptrunc <vscale x 32 x double> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv32f32_nxv32f16 = call <vscale x 32 x half> @llvm.vp.fptrunc.nxv32f16.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv32f64_nxv32f16 = call <vscale x 32 x half> @llvm.vp.fptrunc.nxv32f16.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32f64_nxv32f32 = call <vscale x 32 x float> @llvm.vp.fptrunc.nxv32f32.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv64f32_nxv64f16 = fptrunc <vscale x 64 x float> undef to <vscale x 64 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %nxv64f64_nxv64f16 = fptrunc <vscale x 64 x double> undef to <vscale x 64 x half>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64f64_nxv64f32 = fptrunc <vscale x 64 x double> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv64f32_nxv64f16 = call <vscale x 64 x half> @llvm.vp.fptrunc.nxv64f16.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_nxv64f64_nxv64f16 = call <vscale x 64 x half> @llvm.vp.fptrunc.nxv64f16.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64f64_nxv64f32 = call <vscale x 64 x float> @llvm.vp.fptrunc.nxv64f32.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; ZVE64X-LABEL: 'fptrunc'
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32_v2f16 = fptrunc <2 x float> undef to <2 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f64_v2f16 = fptrunc <2 x double> undef to <2 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f64_v2f32 = fptrunc <2 x double> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f32_v2f16 = call <2 x half> @llvm.vp.fptrunc.v2f16.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f64_v2f16 = call <2 x half> @llvm.vp.fptrunc.v2f16.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f64_v2f32 = call <2 x float> @llvm.vp.fptrunc.v2f32.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4f32_v4f16 = fptrunc <4 x float> undef to <4 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4f64_v4f16 = fptrunc <4 x double> undef to <4 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4f64_v4f32 = fptrunc <4 x double> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4f32_v4f16 = call <4 x half> @llvm.vp.fptrunc.v4f16.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4f64_v4f16 = call <4 x half> @llvm.vp.fptrunc.v4f16.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4f64_v4f32 = call <4 x float> @llvm.vp.fptrunc.v4f32.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8f32_v8f16 = fptrunc <8 x float> undef to <8 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8f64_v8f16 = fptrunc <8 x double> undef to <8 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8f64_v8f32 = fptrunc <8 x double> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8f32_v8f16 = call <8 x half> @llvm.vp.fptrunc.v8f16.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8f64_v8f16 = call <8 x half> @llvm.vp.fptrunc.v8f16.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8f64_v8f32 = call <8 x float> @llvm.vp.fptrunc.v8f32.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16f32_v16f16 = fptrunc <16 x float> undef to <16 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16f64_v16f16 = fptrunc <16 x double> undef to <16 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16f64_v16f32 = fptrunc <16 x double> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16f32_v16f16 = call <16 x half> @llvm.vp.fptrunc.v16f16.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16f64_v16f16 = call <16 x half> @llvm.vp.fptrunc.v16f16.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16f64_v16f32 = call <16 x float> @llvm.vp.fptrunc.v16f32.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v32f32_v32f16 = fptrunc <32 x float> undef to <32 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v32f64_v32f16 = fptrunc <32 x double> undef to <32 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v32f64_v32f32 = fptrunc <32 x double> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v32f32_v32f16 = call <32 x half> @llvm.vp.fptrunc.v32f16.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v32f64_v32f16 = call <32 x half> @llvm.vp.fptrunc.v32f16.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v32f64_v32f32 = call <32 x float> @llvm.vp.fptrunc.v32f32.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v64f32_v64f16 = fptrunc <64 x float> undef to <64 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v64f64_v64f16 = fptrunc <64 x double> undef to <64 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v64f64_v64f32 = fptrunc <64 x double> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v64f32_v64f16 = call <64 x half> @llvm.vp.fptrunc.v64f16.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v64f64_v64f16 = call <64 x half> @llvm.vp.fptrunc.v64f16.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v64f64_v64f32 = call <64 x float> @llvm.vp.fptrunc.v64f32.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v128f32_v128f16 = fptrunc <128 x float> undef to <128 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v128f64_v128f16 = fptrunc <128 x double> undef to <128 x half>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %v128f64_v128f32 = fptrunc <128 x double> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %vp_v128f32_v128f16 = call <128 x half> @llvm.vp.fptrunc.v128f16.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %vp_v128f64_v128f16 = call <128 x half> @llvm.vp.fptrunc.v128f16.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %vp_v128f64_v128f32 = call <128 x float> @llvm.vp.fptrunc.v128f32.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1f16 = fptrunc <vscale x 1 x float> undef to <vscale x 1 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1f16 = fptrunc <vscale x 1 x double> undef to <vscale x 1 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1f32 = fptrunc <vscale x 1 x double> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1f16 = call <vscale x 1 x half> @llvm.vp.fptrunc.nxv1f16.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1f16 = call <vscale x 1 x half> @llvm.vp.fptrunc.nxv1f16.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1f32 = call <vscale x 1 x float> @llvm.vp.fptrunc.nxv1f32.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv1f16 = fptrunc <vscale x 2 x float> undef to <vscale x 2 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv1f16 = fptrunc <vscale x 2 x double> undef to <vscale x 2 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv1f32 = fptrunc <vscale x 2 x double> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2f16 = call <vscale x 2 x half> @llvm.vp.fptrunc.nxv2f16.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2f16 = call <vscale x 2 x half> @llvm.vp.fptrunc.nxv2f16.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2f32 = call <vscale x 2 x float> @llvm.vp.fptrunc.nxv2f32.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4f16 = fptrunc <vscale x 4 x float> undef to <vscale x 4 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4f16 = fptrunc <vscale x 4 x double> undef to <vscale x 4 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4f32 = fptrunc <vscale x 4 x double> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4f16 = call <vscale x 4 x half> @llvm.vp.fptrunc.nxv4f16.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4f16 = call <vscale x 4 x half> @llvm.vp.fptrunc.nxv4f16.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4f32 = call <vscale x 4 x float> @llvm.vp.fptrunc.nxv4f32.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8f16 = fptrunc <vscale x 8 x float> undef to <vscale x 8 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8f16 = fptrunc <vscale x 8 x double> undef to <vscale x 8 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8f32 = fptrunc <vscale x 8 x double> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8f16 = call <vscale x 8 x half> @llvm.vp.fptrunc.nxv8f16.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8f16 = call <vscale x 8 x half> @llvm.vp.fptrunc.nxv8f16.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8f32 = call <vscale x 8 x float> @llvm.vp.fptrunc.nxv8f32.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16f16 = fptrunc <vscale x 16 x float> undef to <vscale x 16 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16f16 = fptrunc <vscale x 16 x double> undef to <vscale x 16 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16f32 = fptrunc <vscale x 16 x double> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16f16 = call <vscale x 16 x half> @llvm.vp.fptrunc.nxv16f16.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16f16 = call <vscale x 16 x half> @llvm.vp.fptrunc.nxv16f16.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16f32 = call <vscale x 16 x float> @llvm.vp.fptrunc.nxv16f32.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32f16 = fptrunc <vscale x 32 x float> undef to <vscale x 32 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32f16 = fptrunc <vscale x 32 x double> undef to <vscale x 32 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32f32 = fptrunc <vscale x 32 x double> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32f16 = call <vscale x 32 x half> @llvm.vp.fptrunc.nxv32f16.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32f16 = call <vscale x 32 x half> @llvm.vp.fptrunc.nxv32f16.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32f32 = call <vscale x 32 x float> @llvm.vp.fptrunc.nxv32f32.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64f16 = fptrunc <vscale x 64 x float> undef to <vscale x 64 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64f16 = fptrunc <vscale x 64 x double> undef to <vscale x 64 x half>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64f32 = fptrunc <vscale x 64 x double> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64f16 = call <vscale x 64 x half> @llvm.vp.fptrunc.nxv64f16.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64f16 = call <vscale x 64 x half> @llvm.vp.fptrunc.nxv64f16.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64f32 = call <vscale x 64 x float> @llvm.vp.fptrunc.nxv64f32.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2f32_v2f16 = fptrunc <2 x float> undef to <2 x half>
   %v2f64_v2f16 = fptrunc <2 x double> undef to <2 x half>
@@ -2427,288 +2602,571 @@ define void @fptrunc() {
 }
 
 define void @fptosi() {
-; CHECK-LABEL: 'fptosi'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32_v2i8 = fptosi <2 x float> undef to <2 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i8 = fptosi <2 x double> undef to <2 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i16 = fptosi <2 x float> undef to <2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f64_v2i16 = fptosi <2 x double> undef to <2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i32 = fptosi <2 x float> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2i32 = fptosi <2 x double> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i64 = fptosi <2 x float> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2i64 = fptosi <2 x double> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f32_v2i1 = fptosi <2 x float> undef to <2 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i1 = fptosi <2 x double> undef to <2 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f32_v2i8 = call <2 x i8> @llvm.vp.fptosi.v2i8.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i8 = call <2 x i8> @llvm.vp.fptosi.v2i8.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i16 = call <2 x i16> @llvm.vp.fptosi.v2i16.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f64_v2i16 = call <2 x i16> @llvm.vp.fptosi.v2i16.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i32 = call <2 x i32> @llvm.vp.fptosi.v2i32.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2i32 = call <2 x i32> @llvm.vp.fptosi.v2i32.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i64 = call <2 x i64> @llvm.vp.fptosi.v2i64.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2i64 = call <2 x i64> @llvm.vp.fptosi.v2i64.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f32_v2i1 = call <2 x i1> @llvm.vp.fptosi.v2i1.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i1 = call <2 x i1> @llvm.vp.fptosi.v2i1.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4i8 = fptosi <4 x float> undef to <4 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f64_v4i8 = fptosi <4 x double> undef to <4 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4i16 = fptosi <4 x float> undef to <4 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4i16 = fptosi <4 x double> undef to <4 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4i32 = fptosi <4 x float> undef to <4 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_v4i32 = fptosi <4 x double> undef to <4 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4i64 = fptosi <4 x float> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4i64 = fptosi <4 x double> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f32_v4i1 = fptosi <4 x float> undef to <4 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f64_v4i1 = fptosi <4 x double> undef to <4 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4i8 = call <4 x i8> @llvm.vp.fptosi.v4i8.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f64_v4i8 = call <4 x i8> @llvm.vp.fptosi.v4i8.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4i16 = call <4 x i16> @llvm.vp.fptosi.v4i16.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4i16 = call <4 x i16> @llvm.vp.fptosi.v4i16.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4i32 = call <4 x i32> @llvm.vp.fptosi.v4i32.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f64_v4i32 = call <4 x i32> @llvm.vp.fptosi.v4i32.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4i64 = call <4 x i64> @llvm.vp.fptosi.v4i64.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4i64 = call <4 x i64> @llvm.vp.fptosi.v4i64.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f32_v4i1 = call <4 x i1> @llvm.vp.fptosi.v4i1.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f64_v4i1 = call <4 x i1> @llvm.vp.fptosi.v4i1.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32_v8i8 = fptosi <8 x float> undef to <8 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64_v8i8 = fptosi <8 x double> undef to <8 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_v8i16 = fptosi <8 x float> undef to <8 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f64_v8i16 = fptosi <8 x double> undef to <8 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32_v8i32 = fptosi <8 x float> undef to <8 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f64_v8i32 = fptosi <8 x double> undef to <8 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32_v8i64 = fptosi <8 x float> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64_v8i64 = fptosi <8 x double> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f32_v8i1 = fptosi <8 x float> undef to <8 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8f64_v8i1 = fptosi <8 x double> undef to <8 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f32_v8i8 = call <8 x i8> @llvm.vp.fptosi.v8i8.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f64_v8i8 = call <8 x i8> @llvm.vp.fptosi.v8i8.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v8f32_v8i16 = call <8 x i16> @llvm.vp.fptosi.v8i16.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f64_v8i16 = call <8 x i16> @llvm.vp.fptosi.v8i16.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f32_v8i32 = call <8 x i32> @llvm.vp.fptosi.v8i32.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f64_v8i32 = call <8 x i32> @llvm.vp.fptosi.v8i32.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f32_v8i64 = call <8 x i64> @llvm.vp.fptosi.v8i64.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f64_v8i64 = call <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f32_v8i1 = call <8 x i1> @llvm.vp.fptosi.v8i1.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8f64_v8i1 = call <8 x i1> @llvm.vp.fptosi.v8i1.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16f32_v16i8 = fptosi <16 x float> undef to <16 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16f64_v16i8 = fptosi <16 x double> undef to <16 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f32_v16i16 = fptosi <16 x float> undef to <16 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f64_v16i16 = fptosi <16 x double> undef to <16 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f32_v16i32 = fptosi <16 x float> undef to <16 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f64_v16i32 = fptosi <16 x double> undef to <16 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32_v16i64 = fptosi <16 x float> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64_v16i64 = fptosi <16 x double> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f32_v16i1 = fptosi <16 x float> undef to <16 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16f64_v16i1 = fptosi <16 x double> undef to <16 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v16f32_v16i8 = call <16 x i8> @llvm.vp.fptosi.v16i8.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v16f64_v16i8 = call <16 x i8> @llvm.vp.fptosi.v16i8.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v16f32_v16i16 = call <16 x i16> @llvm.vp.fptosi.v16i16.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f64_v16i16 = call <16 x i16> @llvm.vp.fptosi.v16i16.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f32_v16i32 = call <16 x i32> @llvm.vp.fptosi.v16i32.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f64_v16i32 = call <16 x i32> @llvm.vp.fptosi.v16i32.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f32_v16i64 = call <16 x i64> @llvm.vp.fptosi.v16i64.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f64_v16i64 = call <16 x i64> @llvm.vp.fptosi.v16i64.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f32_v16i1 = call <16 x i1> @llvm.vp.fptosi.v16i1.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16f64_v16i1 = call <16 x i1> @llvm.vp.fptosi.v16i1.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32f32_v32i8 = fptosi <32 x float> undef to <32 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v32f64_v32i8 = fptosi <32 x double> undef to <32 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32f32_v32i16 = fptosi <32 x float> undef to <32 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32f64_v32i16 = fptosi <32 x double> undef to <32 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32f32_v32i32 = fptosi <32 x float> undef to <32 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32f64_v32i32 = fptosi <32 x double> undef to <32 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32f32_v32i64 = fptosi <32 x float> undef to <32 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32f64_v32i64 = fptosi <32 x double> undef to <32 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32f32_v32i1 = fptosi <32 x float> undef to <32 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32f64_v32i1 = fptosi <32 x double> undef to <32 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v32f32_v32i8 = call <32 x i8> @llvm.vp.fptosi.v32i8.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v32f64_v32i8 = call <32 x i8> @llvm.vp.fptosi.v32i8.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32f32_v32i16 = call <32 x i16> @llvm.vp.fptosi.v32i16.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v32f64_v32i16 = call <32 x i16> @llvm.vp.fptosi.v32i16.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32f32_v32i32 = call <32 x i32> @llvm.vp.fptosi.v32i32.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32f64_v32i32 = call <32 x i32> @llvm.vp.fptosi.v32i32.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32f32_v32i64 = call <32 x i64> @llvm.vp.fptosi.v32i64.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v32f64_v32i64 = call <32 x i64> @llvm.vp.fptosi.v32i64.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v32f32_v32i1 = call <32 x i1> @llvm.vp.fptosi.v32i1.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32f64_v32i1 = call <32 x i1> @llvm.vp.fptosi.v32i1.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v64f32_v64i8 = fptosi <64 x float> undef to <64 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v64f64_v64i8 = fptosi <64 x double> undef to <64 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64f32_v64i16 = fptosi <64 x float> undef to <64 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v64f64_v64i16 = fptosi <64 x double> undef to <64 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64f32_v64i32 = fptosi <64 x float> undef to <64 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64f64_v64i32 = fptosi <64 x double> undef to <64 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64f32_v64i64 = fptosi <64 x float> undef to <64 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v64f64_v64i64 = fptosi <64 x double> undef to <64 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64f32_v64i1 = fptosi <64 x float> undef to <64 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64f64_v64i1 = fptosi <64 x double> undef to <64 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v64f32_v64i8 = call <64 x i8> @llvm.vp.fptosi.v64i8.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v64f64_v64i8 = call <64 x i8> @llvm.vp.fptosi.v64i8.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v64f32_v64i16 = call <64 x i16> @llvm.vp.fptosi.v64i16.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v64f64_v64i16 = call <64 x i16> @llvm.vp.fptosi.v64i16.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v64f32_v64i32 = call <64 x i32> @llvm.vp.fptosi.v64i32.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64f64_v64i32 = call <64 x i32> @llvm.vp.fptosi.v64i32.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64f32_v64i64 = call <64 x i64> @llvm.vp.fptosi.v64i64.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v64f64_v64i64 = call <64 x i64> @llvm.vp.fptosi.v64i64.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64f32_v64i1 = call <64 x i1> @llvm.vp.fptosi.v64i1.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64f64_v64i1 = call <64 x i1> @llvm.vp.fptosi.v64i1.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v128f32_v128i8 = fptosi <128 x float> undef to <128 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v128f64_v128i8 = fptosi <128 x double> undef to <128 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v128f32_v128i16 = fptosi <128 x float> undef to <128 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %v128f64_v128i16 = fptosi <128 x double> undef to <128 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v128f32_v128i32 = fptosi <128 x float> undef to <128 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128f64_v128i32 = fptosi <128 x double> undef to <128 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128f32_v128i64 = fptosi <128 x float> undef to <128 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v128f64_v128i64 = fptosi <128 x double> undef to <128 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v128f32_v128i1 = fptosi <128 x float> undef to <128 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %v128f64_v128i1 = fptosi <128 x double> undef to <128 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v128f32_v128i8 = call <128 x i8> @llvm.vp.fptosi.v128i8.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v128f64_v128i8 = call <128 x i8> @llvm.vp.fptosi.v128i8.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v128f32_v128i16 = call <128 x i16> @llvm.vp.fptosi.v128i16.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_v128f64_v128i16 = call <128 x i16> @llvm.vp.fptosi.v128i16.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v128f32_v128i32 = call <128 x i32> @llvm.vp.fptosi.v128i32.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128f64_v128i32 = call <128 x i32> @llvm.vp.fptosi.v128i32.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128f32_v128i64 = call <128 x i64> @llvm.vp.fptosi.v128i64.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v128f64_v128i64 = call <128 x i64> @llvm.vp.fptosi.v128i64.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v128f32_v128i1 = call <128 x i1> @llvm.vp.fptosi.v128i1.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_v128f64_v128i1 = call <128 x i1> @llvm.vp.fptosi.v128i1.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32_nxv1i8 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64_nxv1i8 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i16 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64_nxv1i16 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i32 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1i32 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i64 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1i64 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f32_nxv1i1 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64_nxv1i1 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f32_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptosi.nxv1i8.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f64_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptosi.nxv1i8.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptosi.nxv1i16.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f64_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptosi.nxv1i16.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptosi.nxv1i32.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptosi.nxv1i32.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptosi.nxv1i64.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptosi.nxv1i64.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f32_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptosi.nxv1i1.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f64_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptosi.nxv1i1.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2i8 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64_nxv2i8 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv2i16 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv2i16 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv2i32 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64_nxv2i32 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2i64 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv2i64 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f32_nxv2i1 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64_nxv2i1 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptosi.nxv2i8.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f64_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptosi.nxv2i8.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptosi.nxv2i16.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptosi.nxv2i16.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptosi.nxv2i32.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f64_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptosi.nxv2i32.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptosi.nxv2i64.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptosi.nxv2i64.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f32_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptosi.nxv2i1.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f64_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptosi.nxv2i1.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32_nxv4i8 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64_nxv4i8 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32_nxv4i16 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f64_nxv4i16 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32_nxv4i32 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f64_nxv4i32 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f32_nxv4i64 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64_nxv4i64 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32_nxv4i1 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4f64_nxv4i1 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f32_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptosi.nxv4i8.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f64_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptosi.nxv4i8.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv4f32_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptosi.nxv4i16.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f64_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptosi.nxv4i16.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f32_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptosi.nxv4i32.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f64_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptosi.nxv4i32.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f32_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptosi.nxv4i64.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f64_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptosi.nxv4i64.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f32_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptosi.nxv4i1.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4f64_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptosi.nxv4i1.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8f32_nxv8i8 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8f64_nxv8i8 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8f32_nxv8i16 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f64_nxv8i16 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32_nxv8i32 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f64_nxv8i32 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f32_nxv8i64 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64_nxv8i64 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f32_nxv8i1 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8f64_nxv8i1 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv8f32_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptosi.nxv8i8.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_nxv8f64_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptosi.nxv8i8.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8f32_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptosi.nxv8i16.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f64_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptosi.nxv8i16.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f32_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptosi.nxv8i32.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f64_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptosi.nxv8i32.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f32_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptosi.nxv8i64.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f64_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptosi.nxv8i64.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f32_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptosi.nxv8i1.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8f64_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptosi.nxv8i1.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16f32_nxv16i8 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv16f64_nxv16i8 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16f32_nxv16i16 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv16f64_nxv16i16 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32_nxv16i32 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16f64_nxv16i32 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32_nxv16i64 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16f64_nxv16i64 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv16f32_nxv16i1 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16f64_nxv16i1 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv16f32_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptosi.nxv16i8.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_nxv16f64_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptosi.nxv16i8.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16f32_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptosi.nxv16i16.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv16f64_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptosi.nxv16i16.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16f32_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptosi.nxv16i32.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16f64_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptosi.nxv16i32.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16f32_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptosi.nxv16i64.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv16f64_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptosi.nxv16i64.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv16f32_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptosi.nxv16i1.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16f64_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptosi.nxv16i1.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32f32_nxv32i8 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %nxv32f64_nxv32i8 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32f32_nxv32i16 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv32f64_nxv32i16 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32f32_nxv32i32 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32f64_nxv32i32 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32f32_nxv32i64 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32f64_nxv32i64 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32f32_nxv32i1 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32f64_nxv32i1 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv32f32_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptosi.nxv32i8.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_nxv32f64_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptosi.nxv32i8.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv32f32_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptosi.nxv32i16.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv32f64_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptosi.nxv32i16.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv32f32_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptosi.nxv32i32.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32f64_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptosi.nxv32i32.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32f32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptosi.nxv32i64.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv32f64_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptosi.nxv32i64.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32f32_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptosi.nxv32i1.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32f64_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptosi.nxv32i1.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv64f32_nxv64i8 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %nxv64f64_nxv64i8 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv64f32_nxv64i16 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %nxv64f64_nxv64i16 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv64f32_nxv64i32 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64f64_nxv64i32 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64f32_nxv64i64 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv64f64_nxv64i64 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv64f32_nxv64i1 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %nxv64f64_nxv64i1 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv64f32_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptosi.nxv64i8.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_nxv64f64_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptosi.nxv64i8.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv64f32_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptosi.nxv64i16.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_nxv64f64_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptosi.nxv64i16.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv64f32_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptosi.nxv64i32.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64f64_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptosi.nxv64i32.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64f32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptosi.nxv64i64.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv64f64_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptosi.nxv64i64.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv64f32_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptosi.nxv64i1.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_nxv64f64_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptosi.nxv64i1.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ZVE64D-LABEL: 'fptosi'
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32_v2i8 = fptosi <2 x float> undef to <2 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i8 = fptosi <2 x double> undef to <2 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i16 = fptosi <2 x float> undef to <2 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f64_v2i16 = fptosi <2 x double> undef to <2 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i32 = fptosi <2 x float> undef to <2 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2i32 = fptosi <2 x double> undef to <2 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i64 = fptosi <2 x float> undef to <2 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2i64 = fptosi <2 x double> undef to <2 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f32_v2i1 = fptosi <2 x float> undef to <2 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i1 = fptosi <2 x double> undef to <2 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f32_v2i8 = call <2 x i8> @llvm.vp.fptosi.v2i8.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i8 = call <2 x i8> @llvm.vp.fptosi.v2i8.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i16 = call <2 x i16> @llvm.vp.fptosi.v2i16.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f64_v2i16 = call <2 x i16> @llvm.vp.fptosi.v2i16.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i32 = call <2 x i32> @llvm.vp.fptosi.v2i32.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2i32 = call <2 x i32> @llvm.vp.fptosi.v2i32.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i64 = call <2 x i64> @llvm.vp.fptosi.v2i64.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2i64 = call <2 x i64> @llvm.vp.fptosi.v2i64.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f32_v2i1 = call <2 x i1> @llvm.vp.fptosi.v2i1.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i1 = call <2 x i1> @llvm.vp.fptosi.v2i1.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4i8 = fptosi <4 x float> undef to <4 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f64_v4i8 = fptosi <4 x double> undef to <4 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4i16 = fptosi <4 x float> undef to <4 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4i16 = fptosi <4 x double> undef to <4 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4i32 = fptosi <4 x float> undef to <4 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_v4i32 = fptosi <4 x double> undef to <4 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4i64 = fptosi <4 x float> undef to <4 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4i64 = fptosi <4 x double> undef to <4 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f32_v4i1 = fptosi <4 x float> undef to <4 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f64_v4i1 = fptosi <4 x double> undef to <4 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4i8 = call <4 x i8> @llvm.vp.fptosi.v4i8.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f64_v4i8 = call <4 x i8> @llvm.vp.fptosi.v4i8.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4i16 = call <4 x i16> @llvm.vp.fptosi.v4i16.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4i16 = call <4 x i16> @llvm.vp.fptosi.v4i16.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4i32 = call <4 x i32> @llvm.vp.fptosi.v4i32.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f64_v4i32 = call <4 x i32> @llvm.vp.fptosi.v4i32.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4i64 = call <4 x i64> @llvm.vp.fptosi.v4i64.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4i64 = call <4 x i64> @llvm.vp.fptosi.v4i64.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f32_v4i1 = call <4 x i1> @llvm.vp.fptosi.v4i1.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f64_v4i1 = call <4 x i1> @llvm.vp.fptosi.v4i1.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32_v8i8 = fptosi <8 x float> undef to <8 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64_v8i8 = fptosi <8 x double> undef to <8 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_v8i16 = fptosi <8 x float> undef to <8 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f64_v8i16 = fptosi <8 x double> undef to <8 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32_v8i32 = fptosi <8 x float> undef to <8 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f64_v8i32 = fptosi <8 x double> undef to <8 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32_v8i64 = fptosi <8 x float> undef to <8 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64_v8i64 = fptosi <8 x double> undef to <8 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f32_v8i1 = fptosi <8 x float> undef to <8 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8f64_v8i1 = fptosi <8 x double> undef to <8 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f32_v8i8 = call <8 x i8> @llvm.vp.fptosi.v8i8.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f64_v8i8 = call <8 x i8> @llvm.vp.fptosi.v8i8.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v8f32_v8i16 = call <8 x i16> @llvm.vp.fptosi.v8i16.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f64_v8i16 = call <8 x i16> @llvm.vp.fptosi.v8i16.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f32_v8i32 = call <8 x i32> @llvm.vp.fptosi.v8i32.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f64_v8i32 = call <8 x i32> @llvm.vp.fptosi.v8i32.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f32_v8i64 = call <8 x i64> @llvm.vp.fptosi.v8i64.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f64_v8i64 = call <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f32_v8i1 = call <8 x i1> @llvm.vp.fptosi.v8i1.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8f64_v8i1 = call <8 x i1> @llvm.vp.fptosi.v8i1.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16f32_v16i8 = fptosi <16 x float> undef to <16 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16f64_v16i8 = fptosi <16 x double> undef to <16 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f32_v16i16 = fptosi <16 x float> undef to <16 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f64_v16i16 = fptosi <16 x double> undef to <16 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f32_v16i32 = fptosi <16 x float> undef to <16 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f64_v16i32 = fptosi <16 x double> undef to <16 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32_v16i64 = fptosi <16 x float> undef to <16 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64_v16i64 = fptosi <16 x double> undef to <16 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f32_v16i1 = fptosi <16 x float> undef to <16 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16f64_v16i1 = fptosi <16 x double> undef to <16 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v16f32_v16i8 = call <16 x i8> @llvm.vp.fptosi.v16i8.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v16f64_v16i8 = call <16 x i8> @llvm.vp.fptosi.v16i8.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v16f32_v16i16 = call <16 x i16> @llvm.vp.fptosi.v16i16.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f64_v16i16 = call <16 x i16> @llvm.vp.fptosi.v16i16.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f32_v16i32 = call <16 x i32> @llvm.vp.fptosi.v16i32.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f64_v16i32 = call <16 x i32> @llvm.vp.fptosi.v16i32.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f32_v16i64 = call <16 x i64> @llvm.vp.fptosi.v16i64.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f64_v16i64 = call <16 x i64> @llvm.vp.fptosi.v16i64.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f32_v16i1 = call <16 x i1> @llvm.vp.fptosi.v16i1.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16f64_v16i1 = call <16 x i1> @llvm.vp.fptosi.v16i1.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32f32_v32i8 = fptosi <32 x float> undef to <32 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v32f64_v32i8 = fptosi <32 x double> undef to <32 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32f32_v32i16 = fptosi <32 x float> undef to <32 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32f64_v32i16 = fptosi <32 x double> undef to <32 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32f32_v32i32 = fptosi <32 x float> undef to <32 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32f64_v32i32 = fptosi <32 x double> undef to <32 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32f32_v32i64 = fptosi <32 x float> undef to <32 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32f64_v32i64 = fptosi <32 x double> undef to <32 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32f32_v32i1 = fptosi <32 x float> undef to <32 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32f64_v32i1 = fptosi <32 x double> undef to <32 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v32f32_v32i8 = call <32 x i8> @llvm.vp.fptosi.v32i8.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v32f64_v32i8 = call <32 x i8> @llvm.vp.fptosi.v32i8.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32f32_v32i16 = call <32 x i16> @llvm.vp.fptosi.v32i16.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v32f64_v32i16 = call <32 x i16> @llvm.vp.fptosi.v32i16.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32f32_v32i32 = call <32 x i32> @llvm.vp.fptosi.v32i32.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32f64_v32i32 = call <32 x i32> @llvm.vp.fptosi.v32i32.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32f32_v32i64 = call <32 x i64> @llvm.vp.fptosi.v32i64.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v32f64_v32i64 = call <32 x i64> @llvm.vp.fptosi.v32i64.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v32f32_v32i1 = call <32 x i1> @llvm.vp.fptosi.v32i1.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32f64_v32i1 = call <32 x i1> @llvm.vp.fptosi.v32i1.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v64f32_v64i8 = fptosi <64 x float> undef to <64 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v64f64_v64i8 = fptosi <64 x double> undef to <64 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64f32_v64i16 = fptosi <64 x float> undef to <64 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v64f64_v64i16 = fptosi <64 x double> undef to <64 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64f32_v64i32 = fptosi <64 x float> undef to <64 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64f64_v64i32 = fptosi <64 x double> undef to <64 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64f32_v64i64 = fptosi <64 x float> undef to <64 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v64f64_v64i64 = fptosi <64 x double> undef to <64 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64f32_v64i1 = fptosi <64 x float> undef to <64 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64f64_v64i1 = fptosi <64 x double> undef to <64 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v64f32_v64i8 = call <64 x i8> @llvm.vp.fptosi.v64i8.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v64f64_v64i8 = call <64 x i8> @llvm.vp.fptosi.v64i8.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v64f32_v64i16 = call <64 x i16> @llvm.vp.fptosi.v64i16.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v64f64_v64i16 = call <64 x i16> @llvm.vp.fptosi.v64i16.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v64f32_v64i32 = call <64 x i32> @llvm.vp.fptosi.v64i32.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64f64_v64i32 = call <64 x i32> @llvm.vp.fptosi.v64i32.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64f32_v64i64 = call <64 x i64> @llvm.vp.fptosi.v64i64.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v64f64_v64i64 = call <64 x i64> @llvm.vp.fptosi.v64i64.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64f32_v64i1 = call <64 x i1> @llvm.vp.fptosi.v64i1.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64f64_v64i1 = call <64 x i1> @llvm.vp.fptosi.v64i1.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v128f32_v128i8 = fptosi <128 x float> undef to <128 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v128f64_v128i8 = fptosi <128 x double> undef to <128 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v128f32_v128i16 = fptosi <128 x float> undef to <128 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %v128f64_v128i16 = fptosi <128 x double> undef to <128 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v128f32_v128i32 = fptosi <128 x float> undef to <128 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128f64_v128i32 = fptosi <128 x double> undef to <128 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128f32_v128i64 = fptosi <128 x float> undef to <128 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v128f64_v128i64 = fptosi <128 x double> undef to <128 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v128f32_v128i1 = fptosi <128 x float> undef to <128 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %v128f64_v128i1 = fptosi <128 x double> undef to <128 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v128f32_v128i8 = call <128 x i8> @llvm.vp.fptosi.v128i8.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v128f64_v128i8 = call <128 x i8> @llvm.vp.fptosi.v128i8.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v128f32_v128i16 = call <128 x i16> @llvm.vp.fptosi.v128i16.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_v128f64_v128i16 = call <128 x i16> @llvm.vp.fptosi.v128i16.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v128f32_v128i32 = call <128 x i32> @llvm.vp.fptosi.v128i32.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128f64_v128i32 = call <128 x i32> @llvm.vp.fptosi.v128i32.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128f32_v128i64 = call <128 x i64> @llvm.vp.fptosi.v128i64.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v128f64_v128i64 = call <128 x i64> @llvm.vp.fptosi.v128i64.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v128f32_v128i1 = call <128 x i1> @llvm.vp.fptosi.v128i1.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_v128f64_v128i1 = call <128 x i1> @llvm.vp.fptosi.v128i1.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32_nxv1i8 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64_nxv1i8 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i16 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64_nxv1i16 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i32 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1i32 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i64 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1i64 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f32_nxv1i1 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64_nxv1i1 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f32_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptosi.nxv1i8.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f64_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptosi.nxv1i8.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptosi.nxv1i16.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f64_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptosi.nxv1i16.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptosi.nxv1i32.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptosi.nxv1i32.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptosi.nxv1i64.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptosi.nxv1i64.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f32_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptosi.nxv1i1.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f64_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptosi.nxv1i1.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2i8 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64_nxv2i8 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv2i16 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv2i16 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv2i32 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64_nxv2i32 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2i64 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv2i64 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f32_nxv2i1 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64_nxv2i1 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptosi.nxv2i8.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f64_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptosi.nxv2i8.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptosi.nxv2i16.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptosi.nxv2i16.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptosi.nxv2i32.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f64_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptosi.nxv2i32.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptosi.nxv2i64.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptosi.nxv2i64.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f32_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptosi.nxv2i1.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f64_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptosi.nxv2i1.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32_nxv4i8 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64_nxv4i8 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32_nxv4i16 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f64_nxv4i16 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32_nxv4i32 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f64_nxv4i32 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f32_nxv4i64 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64_nxv4i64 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32_nxv4i1 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4f64_nxv4i1 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f32_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptosi.nxv4i8.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f64_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptosi.nxv4i8.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv4f32_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptosi.nxv4i16.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f64_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptosi.nxv4i16.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f32_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptosi.nxv4i32.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f64_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptosi.nxv4i32.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f32_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptosi.nxv4i64.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f64_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptosi.nxv4i64.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f32_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptosi.nxv4i1.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4f64_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptosi.nxv4i1.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8f32_nxv8i8 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8f64_nxv8i8 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8f32_nxv8i16 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f64_nxv8i16 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32_nxv8i32 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f64_nxv8i32 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f32_nxv8i64 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64_nxv8i64 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f32_nxv8i1 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8f64_nxv8i1 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv8f32_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptosi.nxv8i8.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_nxv8f64_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptosi.nxv8i8.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8f32_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptosi.nxv8i16.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f64_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptosi.nxv8i16.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f32_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptosi.nxv8i32.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f64_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptosi.nxv8i32.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f32_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptosi.nxv8i64.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f64_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptosi.nxv8i64.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f32_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptosi.nxv8i1.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8f64_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptosi.nxv8i1.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16f32_nxv16i8 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv16f64_nxv16i8 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16f32_nxv16i16 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv16f64_nxv16i16 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32_nxv16i32 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16f64_nxv16i32 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32_nxv16i64 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16f64_nxv16i64 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv16f32_nxv16i1 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16f64_nxv16i1 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv16f32_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptosi.nxv16i8.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_nxv16f64_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptosi.nxv16i8.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16f32_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptosi.nxv16i16.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv16f64_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptosi.nxv16i16.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16f32_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptosi.nxv16i32.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16f64_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptosi.nxv16i32.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16f32_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptosi.nxv16i64.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv16f64_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptosi.nxv16i64.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv16f32_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptosi.nxv16i1.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16f64_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptosi.nxv16i1.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32f32_nxv32i8 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %nxv32f64_nxv32i8 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32f32_nxv32i16 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv32f64_nxv32i16 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32f32_nxv32i32 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32f64_nxv32i32 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32f32_nxv32i64 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32f64_nxv32i64 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32f32_nxv32i1 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32f64_nxv32i1 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv32f32_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptosi.nxv32i8.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_nxv32f64_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptosi.nxv32i8.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv32f32_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptosi.nxv32i16.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv32f64_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptosi.nxv32i16.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv32f32_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptosi.nxv32i32.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32f64_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptosi.nxv32i32.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32f32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptosi.nxv32i64.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv32f64_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptosi.nxv32i64.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32f32_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptosi.nxv32i1.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32f64_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptosi.nxv32i1.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv64f32_nxv64i8 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %nxv64f64_nxv64i8 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv64f32_nxv64i16 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %nxv64f64_nxv64i16 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv64f32_nxv64i32 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64f64_nxv64i32 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64f32_nxv64i64 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv64f64_nxv64i64 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv64f32_nxv64i1 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %nxv64f64_nxv64i1 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv64f32_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptosi.nxv64i8.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_nxv64f64_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptosi.nxv64i8.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv64f32_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptosi.nxv64i16.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_nxv64f64_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptosi.nxv64i16.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv64f32_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptosi.nxv64i32.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64f64_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptosi.nxv64i32.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64f32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptosi.nxv64i64.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv64f64_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptosi.nxv64i64.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv64f32_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptosi.nxv64i1.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_nxv64f64_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptosi.nxv64i1.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; ZVE64X-LABEL: 'fptosi'
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f32_v2i8 = fptosi <2 x float> undef to <2 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f64_v2i8 = fptosi <2 x double> undef to <2 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f32_v2i16 = fptosi <2 x float> undef to <2 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f64_v2i16 = fptosi <2 x double> undef to <2 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f32_v2i32 = fptosi <2 x float> undef to <2 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f64_v2i32 = fptosi <2 x double> undef to <2 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f32_v2i64 = fptosi <2 x float> undef to <2 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i64 = fptosi <2 x double> undef to <2 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %v2f32_v2i1 = fptosi <2 x float> undef to <2 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %v2f64_v2i1 = fptosi <2 x double> undef to <2 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f32_v2i8 = call <2 x i8> @llvm.vp.fptosi.v2i8.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f64_v2i8 = call <2 x i8> @llvm.vp.fptosi.v2i8.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f32_v2i16 = call <2 x i16> @llvm.vp.fptosi.v2i16.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f64_v2i16 = call <2 x i16> @llvm.vp.fptosi.v2i16.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f32_v2i32 = call <2 x i32> @llvm.vp.fptosi.v2i32.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f64_v2i32 = call <2 x i32> @llvm.vp.fptosi.v2i32.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f32_v2i64 = call <2 x i64> @llvm.vp.fptosi.v2i64.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i64 = call <2 x i64> @llvm.vp.fptosi.v2i64.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %vp_v2f32_v2i1 = call <2 x i1> @llvm.vp.fptosi.v2i1.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %vp_v2f64_v2i1 = call <2 x i1> @llvm.vp.fptosi.v2i1.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f32_v4i8 = fptosi <4 x float> undef to <4 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f64_v4i8 = fptosi <4 x double> undef to <4 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f32_v4i16 = fptosi <4 x float> undef to <4 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f64_v4i16 = fptosi <4 x double> undef to <4 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f32_v4i32 = fptosi <4 x float> undef to <4 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f64_v4i32 = fptosi <4 x double> undef to <4 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f32_v4i64 = fptosi <4 x float> undef to <4 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f64_v4i64 = fptosi <4 x double> undef to <4 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %v4f32_v4i1 = fptosi <4 x float> undef to <4 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %v4f64_v4i1 = fptosi <4 x double> undef to <4 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f32_v4i8 = call <4 x i8> @llvm.vp.fptosi.v4i8.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f64_v4i8 = call <4 x i8> @llvm.vp.fptosi.v4i8.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f32_v4i16 = call <4 x i16> @llvm.vp.fptosi.v4i16.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f64_v4i16 = call <4 x i16> @llvm.vp.fptosi.v4i16.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f32_v4i32 = call <4 x i32> @llvm.vp.fptosi.v4i32.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f64_v4i32 = call <4 x i32> @llvm.vp.fptosi.v4i32.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4f32_v4i64 = call <4 x i64> @llvm.vp.fptosi.v4i64.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4f64_v4i64 = call <4 x i64> @llvm.vp.fptosi.v4i64.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %vp_v4f32_v4i1 = call <4 x i1> @llvm.vp.fptosi.v4i1.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %vp_v4f64_v4i1 = call <4 x i1> @llvm.vp.fptosi.v4i1.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f32_v8i8 = fptosi <8 x float> undef to <8 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f64_v8i8 = fptosi <8 x double> undef to <8 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f32_v8i16 = fptosi <8 x float> undef to <8 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f64_v8i16 = fptosi <8 x double> undef to <8 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f32_v8i32 = fptosi <8 x float> undef to <8 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f64_v8i32 = fptosi <8 x double> undef to <8 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f32_v8i64 = fptosi <8 x float> undef to <8 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f64_v8i64 = fptosi <8 x double> undef to <8 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v8f32_v8i1 = fptosi <8 x float> undef to <8 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v8f64_v8i1 = fptosi <8 x double> undef to <8 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f32_v8i8 = call <8 x i8> @llvm.vp.fptosi.v8i8.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f64_v8i8 = call <8 x i8> @llvm.vp.fptosi.v8i8.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f32_v8i16 = call <8 x i16> @llvm.vp.fptosi.v8i16.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f64_v8i16 = call <8 x i16> @llvm.vp.fptosi.v8i16.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f32_v8i32 = call <8 x i32> @llvm.vp.fptosi.v8i32.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f64_v8i32 = call <8 x i32> @llvm.vp.fptosi.v8i32.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8f32_v8i64 = call <8 x i64> @llvm.vp.fptosi.v8i64.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8f64_v8i64 = call <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v8f32_v8i1 = call <8 x i1> @llvm.vp.fptosi.v8i1.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v8f64_v8i1 = call <8 x i1> @llvm.vp.fptosi.v8i1.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f32_v16i8 = fptosi <16 x float> undef to <16 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f64_v16i8 = fptosi <16 x double> undef to <16 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f32_v16i16 = fptosi <16 x float> undef to <16 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f64_v16i16 = fptosi <16 x double> undef to <16 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f32_v16i32 = fptosi <16 x float> undef to <16 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f64_v16i32 = fptosi <16 x double> undef to <16 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f32_v16i64 = fptosi <16 x float> undef to <16 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f64_v16i64 = fptosi <16 x double> undef to <16 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %v16f32_v16i1 = fptosi <16 x float> undef to <16 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %v16f64_v16i1 = fptosi <16 x double> undef to <16 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f32_v16i8 = call <16 x i8> @llvm.vp.fptosi.v16i8.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f64_v16i8 = call <16 x i8> @llvm.vp.fptosi.v16i8.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f32_v16i16 = call <16 x i16> @llvm.vp.fptosi.v16i16.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f64_v16i16 = call <16 x i16> @llvm.vp.fptosi.v16i16.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f32_v16i32 = call <16 x i32> @llvm.vp.fptosi.v16i32.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f64_v16i32 = call <16 x i32> @llvm.vp.fptosi.v16i32.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16f32_v16i64 = call <16 x i64> @llvm.vp.fptosi.v16i64.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16f64_v16i64 = call <16 x i64> @llvm.vp.fptosi.v16i64.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %vp_v16f32_v16i1 = call <16 x i1> @llvm.vp.fptosi.v16i1.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %vp_v16f64_v16i1 = call <16 x i1> @llvm.vp.fptosi.v16i1.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f32_v32i8 = fptosi <32 x float> undef to <32 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f64_v32i8 = fptosi <32 x double> undef to <32 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f32_v32i16 = fptosi <32 x float> undef to <32 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f64_v32i16 = fptosi <32 x double> undef to <32 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f32_v32i32 = fptosi <32 x float> undef to <32 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f64_v32i32 = fptosi <32 x double> undef to <32 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %v32f32_v32i64 = fptosi <32 x float> undef to <32 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %v32f64_v32i64 = fptosi <32 x double> undef to <32 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 319 for instruction: %v32f32_v32i1 = fptosi <32 x float> undef to <32 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 319 for instruction: %v32f64_v32i1 = fptosi <32 x double> undef to <32 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f32_v32i8 = call <32 x i8> @llvm.vp.fptosi.v32i8.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f64_v32i8 = call <32 x i8> @llvm.vp.fptosi.v32i8.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f32_v32i16 = call <32 x i16> @llvm.vp.fptosi.v32i16.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f64_v32i16 = call <32 x i16> @llvm.vp.fptosi.v32i16.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f32_v32i32 = call <32 x i32> @llvm.vp.fptosi.v32i32.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f64_v32i32 = call <32 x i32> @llvm.vp.fptosi.v32i32.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %vp_v32f32_v32i64 = call <32 x i64> @llvm.vp.fptosi.v32i64.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %vp_v32f64_v32i64 = call <32 x i64> @llvm.vp.fptosi.v32i64.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 319 for instruction: %vp_v32f32_v32i1 = call <32 x i1> @llvm.vp.fptosi.v32i1.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 319 for instruction: %vp_v32f64_v32i1 = call <32 x i1> @llvm.vp.fptosi.v32i1.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v64f32_v64i8 = fptosi <64 x float> undef to <64 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v64f64_v64i8 = fptosi <64 x double> undef to <64 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v64f32_v64i16 = fptosi <64 x float> undef to <64 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v64f64_v64i16 = fptosi <64 x double> undef to <64 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %v64f32_v64i32 = fptosi <64 x float> undef to <64 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %v64f64_v64i32 = fptosi <64 x double> undef to <64 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %v64f32_v64i64 = fptosi <64 x float> undef to <64 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %v64f64_v64i64 = fptosi <64 x double> undef to <64 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 639 for instruction: %v64f32_v64i1 = fptosi <64 x float> undef to <64 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 639 for instruction: %v64f64_v64i1 = fptosi <64 x double> undef to <64 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v64f32_v64i8 = call <64 x i8> @llvm.vp.fptosi.v64i8.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v64f64_v64i8 = call <64 x i8> @llvm.vp.fptosi.v64i8.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v64f32_v64i16 = call <64 x i16> @llvm.vp.fptosi.v64i16.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v64f64_v64i16 = call <64 x i16> @llvm.vp.fptosi.v64i16.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %vp_v64f32_v64i32 = call <64 x i32> @llvm.vp.fptosi.v64i32.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %vp_v64f64_v64i32 = call <64 x i32> @llvm.vp.fptosi.v64i32.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %vp_v64f32_v64i64 = call <64 x i64> @llvm.vp.fptosi.v64i64.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %vp_v64f64_v64i64 = call <64 x i64> @llvm.vp.fptosi.v64i64.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 639 for instruction: %vp_v64f32_v64i1 = call <64 x i1> @llvm.vp.fptosi.v64i1.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 639 for instruction: %vp_v64f64_v64i1 = call <64 x i1> @llvm.vp.fptosi.v64i1.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 511 for instruction: %v128f32_v128i8 = fptosi <128 x float> undef to <128 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 511 for instruction: %v128f64_v128i8 = fptosi <128 x double> undef to <128 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %v128f32_v128i16 = fptosi <128 x float> undef to <128 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %v128f64_v128i16 = fptosi <128 x double> undef to <128 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %v128f32_v128i32 = fptosi <128 x float> undef to <128 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %v128f64_v128i32 = fptosi <128 x double> undef to <128 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %v128f32_v128i64 = fptosi <128 x float> undef to <128 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %v128f64_v128i64 = fptosi <128 x double> undef to <128 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1279 for instruction: %v128f32_v128i1 = fptosi <128 x float> undef to <128 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1279 for instruction: %v128f64_v128i1 = fptosi <128 x double> undef to <128 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 511 for instruction: %vp_v128f32_v128i8 = call <128 x i8> @llvm.vp.fptosi.v128i8.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 511 for instruction: %vp_v128f64_v128i8 = call <128 x i8> @llvm.vp.fptosi.v128i8.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %vp_v128f32_v128i16 = call <128 x i16> @llvm.vp.fptosi.v128i16.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %vp_v128f64_v128i16 = call <128 x i16> @llvm.vp.fptosi.v128i16.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %vp_v128f32_v128i32 = call <128 x i32> @llvm.vp.fptosi.v128i32.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %vp_v128f64_v128i32 = call <128 x i32> @llvm.vp.fptosi.v128i32.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %vp_v128f32_v128i64 = call <128 x i64> @llvm.vp.fptosi.v128i64.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %vp_v128f64_v128i64 = call <128 x i64> @llvm.vp.fptosi.v128i64.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1279 for instruction: %vp_v128f32_v128i1 = call <128 x i1> @llvm.vp.fptosi.v128i1.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1279 for instruction: %vp_v128f64_v128i1 = call <128 x i1> @llvm.vp.fptosi.v128i1.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i8 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i8 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i16 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i16 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i32 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i32 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i64 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i64 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i1 = fptosi <vscale x 1 x float> undef to <vscale x 1 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i1 = fptosi <vscale x 1 x double> undef to <vscale x 1 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptosi.nxv1i8.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptosi.nxv1i8.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptosi.nxv1i16.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptosi.nxv1i16.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptosi.nxv1i32.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptosi.nxv1i32.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptosi.nxv1i64.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptosi.nxv1i64.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptosi.nxv1i1.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptosi.nxv1i1.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i8 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i8 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i16 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i16 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i32 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i32 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i64 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i64 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i1 = fptosi <vscale x 2 x float> undef to <vscale x 2 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i1 = fptosi <vscale x 2 x double> undef to <vscale x 2 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptosi.nxv2i8.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptosi.nxv2i8.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptosi.nxv2i16.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptosi.nxv2i16.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptosi.nxv2i32.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptosi.nxv2i32.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptosi.nxv2i64.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptosi.nxv2i64.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptosi.nxv2i1.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptosi.nxv2i1.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i8 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i8 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i16 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i16 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i32 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i32 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i64 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i64 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i1 = fptosi <vscale x 4 x float> undef to <vscale x 4 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i1 = fptosi <vscale x 4 x double> undef to <vscale x 4 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptosi.nxv4i8.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptosi.nxv4i8.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptosi.nxv4i16.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptosi.nxv4i16.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptosi.nxv4i32.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptosi.nxv4i32.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptosi.nxv4i64.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptosi.nxv4i64.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptosi.nxv4i1.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptosi.nxv4i1.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i8 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i8 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i16 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i16 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i32 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i32 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i64 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i64 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i1 = fptosi <vscale x 8 x float> undef to <vscale x 8 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i1 = fptosi <vscale x 8 x double> undef to <vscale x 8 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptosi.nxv8i8.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptosi.nxv8i8.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptosi.nxv8i16.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptosi.nxv8i16.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptosi.nxv8i32.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptosi.nxv8i32.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptosi.nxv8i64.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptosi.nxv8i64.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptosi.nxv8i1.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptosi.nxv8i1.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i8 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i8 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i16 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i16 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i32 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i32 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i64 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i64 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i1 = fptosi <vscale x 16 x float> undef to <vscale x 16 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i1 = fptosi <vscale x 16 x double> undef to <vscale x 16 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptosi.nxv16i8.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptosi.nxv16i8.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptosi.nxv16i16.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptosi.nxv16i16.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptosi.nxv16i32.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptosi.nxv16i32.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptosi.nxv16i64.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptosi.nxv16i64.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptosi.nxv16i1.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptosi.nxv16i1.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i8 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i8 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i16 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i16 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i32 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i32 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i64 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i64 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i1 = fptosi <vscale x 32 x float> undef to <vscale x 32 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i1 = fptosi <vscale x 32 x double> undef to <vscale x 32 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptosi.nxv32i8.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptosi.nxv32i8.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptosi.nxv32i16.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptosi.nxv32i16.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptosi.nxv32i32.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptosi.nxv32i32.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptosi.nxv32i64.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptosi.nxv32i64.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptosi.nxv32i1.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptosi.nxv32i1.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i8 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i8 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i16 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i16 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i32 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i32 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i64 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i64 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i1 = fptosi <vscale x 64 x float> undef to <vscale x 64 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i1 = fptosi <vscale x 64 x double> undef to <vscale x 64 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptosi.nxv64i8.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptosi.nxv64i8.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptosi.nxv64i16.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptosi.nxv64i16.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptosi.nxv64i32.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptosi.nxv64i32.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptosi.nxv64i64.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptosi.nxv64i64.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptosi.nxv64i1.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptosi.nxv64i1.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2f32_v2i8 = fptosi <2 x float> undef to <2 x i8>
   %v2f64_v2i8 = fptosi <2 x double> undef to <2 x i8>
@@ -3022,288 +3480,571 @@ define void @fptosi() {
 }
 
 define void @fptoui() {
-; CHECK-LABEL: 'fptoui'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32_v2i8 = fptoui <2 x float> undef to <2 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i8 = fptoui <2 x double> undef to <2 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i16 = fptoui <2 x float> undef to <2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f64_v2i16 = fptoui <2 x double> undef to <2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i32 = fptoui <2 x float> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2i32 = fptoui <2 x double> undef to <2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i64 = fptoui <2 x float> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2i64 = fptoui <2 x double> undef to <2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f32_v2i1 = fptoui <2 x float> undef to <2 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i1 = fptoui <2 x double> undef to <2 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f32_v2i8 = call <2 x i8> @llvm.vp.fptoui.v2i8.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i8 = call <2 x i8> @llvm.vp.fptoui.v2i8.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i16 = call <2 x i16> @llvm.vp.fptoui.v2i16.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f64_v2i16 = call <2 x i16> @llvm.vp.fptoui.v2i16.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i32 = call <2 x i32> @llvm.vp.fptoui.v2i32.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2i32 = call <2 x i32> @llvm.vp.fptoui.v2i32.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i64 = call <2 x i64> @llvm.vp.fptoui.v2i64.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2i64 = call <2 x i64> @llvm.vp.fptoui.v2i64.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f32_v2i1 = call <2 x i1> @llvm.vp.fptoui.v2i1.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i1 = call <2 x i1> @llvm.vp.fptoui.v2i1.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4i8 = fptoui <4 x float> undef to <4 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f64_v4i8 = fptoui <4 x double> undef to <4 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4i16 = fptoui <4 x float> undef to <4 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4i16 = fptoui <4 x double> undef to <4 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4i32 = fptoui <4 x float> undef to <4 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_v4i32 = fptoui <4 x double> undef to <4 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4i64 = fptoui <4 x float> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4i64 = fptoui <4 x double> undef to <4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f32_v4i1 = fptoui <4 x float> undef to <4 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f64_v4i1 = fptoui <4 x double> undef to <4 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4i8 = call <4 x i8> @llvm.vp.fptoui.v4i8.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f64_v4i8 = call <4 x i8> @llvm.vp.fptoui.v4i8.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4i16 = call <4 x i16> @llvm.vp.fptoui.v4i16.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4i16 = call <4 x i16> @llvm.vp.fptoui.v4i16.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4i32 = call <4 x i32> @llvm.vp.fptoui.v4i32.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f64_v4i32 = call <4 x i32> @llvm.vp.fptoui.v4i32.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4i64 = call <4 x i64> @llvm.vp.fptoui.v4i64.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4i64 = call <4 x i64> @llvm.vp.fptoui.v4i64.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f32_v4i1 = call <4 x i1> @llvm.vp.fptoui.v4i1.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f64_v4i1 = call <4 x i1> @llvm.vp.fptoui.v4i1.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32_v8i8 = fptoui <8 x float> undef to <8 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64_v8i8 = fptoui <8 x double> undef to <8 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_v8i16 = fptoui <8 x float> undef to <8 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f64_v8i16 = fptoui <8 x double> undef to <8 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32_v8i32 = fptoui <8 x float> undef to <8 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f64_v8i32 = fptoui <8 x double> undef to <8 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32_v8i64 = fptoui <8 x float> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64_v8i64 = fptoui <8 x double> undef to <8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f32_v8i1 = fptoui <8 x float> undef to <8 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8f64_v8i1 = fptoui <8 x double> undef to <8 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f32_v8i8 = call <8 x i8> @llvm.vp.fptoui.v8i8.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f64_v8i8 = call <8 x i8> @llvm.vp.fptoui.v8i8.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v8f32_v8i16 = call <8 x i16> @llvm.vp.fptoui.v8i16.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f64_v8i16 = call <8 x i16> @llvm.vp.fptoui.v8i16.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f32_v8i32 = call <8 x i32> @llvm.vp.fptoui.v8i32.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f64_v8i32 = call <8 x i32> @llvm.vp.fptoui.v8i32.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f32_v8i64 = call <8 x i64> @llvm.vp.fptoui.v8i64.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f64_v8i64 = call <8 x i64> @llvm.vp.fptoui.v8i64.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f32_v8i1 = call <8 x i1> @llvm.vp.fptoui.v8i1.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8f64_v8i1 = call <8 x i1> @llvm.vp.fptoui.v8i1.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16f32_v16i8 = fptoui <16 x float> undef to <16 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16f64_v16i8 = fptoui <16 x double> undef to <16 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f32_v16i16 = fptoui <16 x float> undef to <16 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f64_v16i16 = fptoui <16 x double> undef to <16 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f32_v16i32 = fptoui <16 x float> undef to <16 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f64_v16i32 = fptoui <16 x double> undef to <16 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32_v16i64 = fptoui <16 x float> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64_v16i64 = fptoui <16 x double> undef to <16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f32_v16i1 = fptoui <16 x float> undef to <16 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16f64_v16i1 = fptoui <16 x double> undef to <16 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v16f32_v16i8 = call <16 x i8> @llvm.vp.fptoui.v16i8.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v16f64_v16i8 = call <16 x i8> @llvm.vp.fptoui.v16i8.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v16f32_v16i16 = call <16 x i16> @llvm.vp.fptoui.v16i16.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f64_v16i16 = call <16 x i16> @llvm.vp.fptoui.v16i16.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f32_v16i32 = call <16 x i32> @llvm.vp.fptoui.v16i32.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f64_v16i32 = call <16 x i32> @llvm.vp.fptoui.v16i32.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f32_v16i64 = call <16 x i64> @llvm.vp.fptoui.v16i64.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f64_v16i64 = call <16 x i64> @llvm.vp.fptoui.v16i64.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f32_v16i1 = call <16 x i1> @llvm.vp.fptoui.v16i1.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16f64_v16i1 = call <16 x i1> @llvm.vp.fptoui.v16i1.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32f32_v32i8 = fptoui <32 x float> undef to <32 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v32f64_v32i8 = fptoui <32 x double> undef to <32 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32f32_v32i16 = fptoui <32 x float> undef to <32 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32f64_v32i16 = fptoui <32 x double> undef to <32 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32f32_v32i32 = fptoui <32 x float> undef to <32 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32f64_v32i32 = fptoui <32 x double> undef to <32 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32f32_v32i64 = fptoui <32 x float> undef to <32 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32f64_v32i64 = fptoui <32 x double> undef to <32 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32f32_v32i1 = fptoui <32 x float> undef to <32 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32f64_v32i1 = fptoui <32 x double> undef to <32 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v32f32_v32i8 = call <32 x i8> @llvm.vp.fptoui.v32i8.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v32f64_v32i8 = call <32 x i8> @llvm.vp.fptoui.v32i8.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32f32_v32i16 = call <32 x i16> @llvm.vp.fptoui.v32i16.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v32f64_v32i16 = call <32 x i16> @llvm.vp.fptoui.v32i16.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32f32_v32i32 = call <32 x i32> @llvm.vp.fptoui.v32i32.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32f64_v32i32 = call <32 x i32> @llvm.vp.fptoui.v32i32.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32f32_v32i64 = call <32 x i64> @llvm.vp.fptoui.v32i64.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v32f64_v32i64 = call <32 x i64> @llvm.vp.fptoui.v32i64.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v32f32_v32i1 = call <32 x i1> @llvm.vp.fptoui.v32i1.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32f64_v32i1 = call <32 x i1> @llvm.vp.fptoui.v32i1.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v64f32_v64i8 = fptoui <64 x float> undef to <64 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v64f64_v64i8 = fptoui <64 x double> undef to <64 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64f32_v64i16 = fptoui <64 x float> undef to <64 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v64f64_v64i16 = fptoui <64 x double> undef to <64 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64f32_v64i32 = fptoui <64 x float> undef to <64 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64f64_v64i32 = fptoui <64 x double> undef to <64 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64f32_v64i64 = fptoui <64 x float> undef to <64 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v64f64_v64i64 = fptoui <64 x double> undef to <64 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64f32_v64i1 = fptoui <64 x float> undef to <64 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64f64_v64i1 = fptoui <64 x double> undef to <64 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v64f32_v64i8 = call <64 x i8> @llvm.vp.fptoui.v64i8.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v64f64_v64i8 = call <64 x i8> @llvm.vp.fptoui.v64i8.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v64f32_v64i16 = call <64 x i16> @llvm.vp.fptoui.v64i16.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v64f64_v64i16 = call <64 x i16> @llvm.vp.fptoui.v64i16.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v64f32_v64i32 = call <64 x i32> @llvm.vp.fptoui.v64i32.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64f64_v64i32 = call <64 x i32> @llvm.vp.fptoui.v64i32.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64f32_v64i64 = call <64 x i64> @llvm.vp.fptoui.v64i64.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v64f64_v64i64 = call <64 x i64> @llvm.vp.fptoui.v64i64.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64f32_v64i1 = call <64 x i1> @llvm.vp.fptoui.v64i1.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64f64_v64i1 = call <64 x i1> @llvm.vp.fptoui.v64i1.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v128f32_v128i8 = fptoui <128 x float> undef to <128 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v128f64_v128i8 = fptoui <128 x double> undef to <128 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v128f32_v128i16 = fptoui <128 x float> undef to <128 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %v128f64_v128i16 = fptoui <128 x double> undef to <128 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v128f32_v128i32 = fptoui <128 x float> undef to <128 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128f64_v128i32 = fptoui <128 x double> undef to <128 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128f32_v128i64 = fptoui <128 x float> undef to <128 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v128f64_v128i64 = fptoui <128 x double> undef to <128 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v128f32_v128i1 = fptoui <128 x float> undef to <128 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %v128f64_v128i1 = fptoui <128 x double> undef to <128 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v128f32_v128i8 = call <128 x i8> @llvm.vp.fptoui.v128i8.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v128f64_v128i8 = call <128 x i8> @llvm.vp.fptoui.v128i8.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v128f32_v128i16 = call <128 x i16> @llvm.vp.fptoui.v128i16.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_v128f64_v128i16 = call <128 x i16> @llvm.vp.fptoui.v128i16.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v128f32_v128i32 = call <128 x i32> @llvm.vp.fptoui.v128i32.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128f64_v128i32 = call <128 x i32> @llvm.vp.fptoui.v128i32.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128f32_v128i64 = call <128 x i64> @llvm.vp.fptoui.v128i64.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v128f64_v128i64 = call <128 x i64> @llvm.vp.fptoui.v128i64.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v128f32_v128i1 = call <128 x i1> @llvm.vp.fptoui.v128i1.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_v128f64_v128i1 = call <128 x i1> @llvm.vp.fptoui.v128i1.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32_nxv1i8 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64_nxv1i8 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i16 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64_nxv1i16 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i32 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1i32 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i64 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1i64 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f32_nxv1i1 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64_nxv1i1 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f32_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptoui.nxv1i8.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f64_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptoui.nxv1i8.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptoui.nxv1i16.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f64_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptoui.nxv1i16.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptoui.nxv1i32.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptoui.nxv1i32.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptoui.nxv1i64.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptoui.nxv1i64.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f32_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptoui.nxv1i1.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f64_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptoui.nxv1i1.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2i8 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64_nxv2i8 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv2i16 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv2i16 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv2i32 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64_nxv2i32 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2i64 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv2i64 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f32_nxv2i1 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64_nxv2i1 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptoui.nxv2i8.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f64_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptoui.nxv2i8.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptoui.nxv2i16.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptoui.nxv2i16.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptoui.nxv2i32.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f64_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptoui.nxv2i32.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptoui.nxv2i64.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptoui.nxv2i64.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f32_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptoui.nxv2i1.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f64_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptoui.nxv2i1.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32_nxv4i8 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64_nxv4i8 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32_nxv4i16 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f64_nxv4i16 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32_nxv4i32 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f64_nxv4i32 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f32_nxv4i64 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64_nxv4i64 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32_nxv4i1 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4f64_nxv4i1 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f32_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptoui.nxv4i8.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f64_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptoui.nxv4i8.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv4f32_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptoui.nxv4i16.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f64_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptoui.nxv4i16.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f32_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptoui.nxv4i32.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f64_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptoui.nxv4i32.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f32_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptoui.nxv4i64.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f64_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptoui.nxv4i64.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f32_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptoui.nxv4i1.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4f64_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptoui.nxv4i1.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8f32_nxv8i8 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8f64_nxv8i8 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8f32_nxv8i16 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f64_nxv8i16 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32_nxv8i32 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f64_nxv8i32 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f32_nxv8i64 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64_nxv8i64 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f32_nxv8i1 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8f64_nxv8i1 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv8f32_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptoui.nxv8i8.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_nxv8f64_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptoui.nxv8i8.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8f32_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptoui.nxv8i16.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f64_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptoui.nxv8i16.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f32_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptoui.nxv8i32.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f64_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptoui.nxv8i32.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f32_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptoui.nxv8i64.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f64_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptoui.nxv8i64.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f32_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptoui.nxv8i1.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8f64_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptoui.nxv8i1.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16f32_nxv16i8 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv16f64_nxv16i8 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16f32_nxv16i16 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv16f64_nxv16i16 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32_nxv16i32 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16f64_nxv16i32 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32_nxv16i64 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16f64_nxv16i64 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv16f32_nxv16i1 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16f64_nxv16i1 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv16f32_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptoui.nxv16i8.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_nxv16f64_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptoui.nxv16i8.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16f32_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptoui.nxv16i16.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv16f64_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptoui.nxv16i16.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16f32_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptoui.nxv16i32.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16f64_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptoui.nxv16i32.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16f32_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptoui.nxv16i64.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv16f64_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptoui.nxv16i64.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv16f32_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptoui.nxv16i1.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16f64_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptoui.nxv16i1.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32f32_nxv32i8 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %nxv32f64_nxv32i8 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32f32_nxv32i16 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv32f64_nxv32i16 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32f32_nxv32i32 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32f64_nxv32i32 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32f32_nxv32i64 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32f64_nxv32i64 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32f32_nxv32i1 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32f64_nxv32i1 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv32f32_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptoui.nxv32i8.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_nxv32f64_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptoui.nxv32i8.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv32f32_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptoui.nxv32i16.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv32f64_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptoui.nxv32i16.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv32f32_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptoui.nxv32i32.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32f64_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptoui.nxv32i32.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32f32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptoui.nxv32i64.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv32f64_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptoui.nxv32i64.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32f32_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptoui.nxv32i1.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32f64_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptoui.nxv32i1.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv64f32_nxv64i8 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %nxv64f64_nxv64i8 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv64f32_nxv64i16 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %nxv64f64_nxv64i16 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv64f32_nxv64i32 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64f64_nxv64i32 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64f32_nxv64i64 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv64f64_nxv64i64 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i64>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv64f32_nxv64i1 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %nxv64f64_nxv64i1 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv64f32_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptoui.nxv64i8.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_nxv64f64_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptoui.nxv64i8.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv64f32_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptoui.nxv64i16.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_nxv64f64_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptoui.nxv64i16.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv64f32_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptoui.nxv64i32.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64f64_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptoui.nxv64i32.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64f32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptoui.nxv64i64.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv64f64_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptoui.nxv64i64.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv64f32_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptoui.nxv64i1.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_nxv64f64_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptoui.nxv64i1.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ZVE64D-LABEL: 'fptoui'
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f32_v2i8 = fptoui <2 x float> undef to <2 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i8 = fptoui <2 x double> undef to <2 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i16 = fptoui <2 x float> undef to <2 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2f64_v2i16 = fptoui <2 x double> undef to <2 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i32 = fptoui <2 x float> undef to <2 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2i32 = fptoui <2 x double> undef to <2 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f32_v2i64 = fptoui <2 x float> undef to <2 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2f64_v2i64 = fptoui <2 x double> undef to <2 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f32_v2i1 = fptoui <2 x float> undef to <2 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i1 = fptoui <2 x double> undef to <2 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f32_v2i8 = call <2 x i8> @llvm.vp.fptoui.v2i8.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i8 = call <2 x i8> @llvm.vp.fptoui.v2i8.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i16 = call <2 x i16> @llvm.vp.fptoui.v2i16.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2f64_v2i16 = call <2 x i16> @llvm.vp.fptoui.v2i16.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i32 = call <2 x i32> @llvm.vp.fptoui.v2i32.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2i32 = call <2 x i32> @llvm.vp.fptoui.v2i32.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f32_v2i64 = call <2 x i64> @llvm.vp.fptoui.v2i64.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2f64_v2i64 = call <2 x i64> @llvm.vp.fptoui.v2i64.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f32_v2i1 = call <2 x i1> @llvm.vp.fptoui.v2i1.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i1 = call <2 x i1> @llvm.vp.fptoui.v2i1.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4i8 = fptoui <4 x float> undef to <4 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f64_v4i8 = fptoui <4 x double> undef to <4 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4i16 = fptoui <4 x float> undef to <4 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4i16 = fptoui <4 x double> undef to <4 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f32_v4i32 = fptoui <4 x float> undef to <4 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4f64_v4i32 = fptoui <4 x double> undef to <4 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f32_v4i64 = fptoui <4 x float> undef to <4 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4f64_v4i64 = fptoui <4 x double> undef to <4 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f32_v4i1 = fptoui <4 x float> undef to <4 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4f64_v4i1 = fptoui <4 x double> undef to <4 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4i8 = call <4 x i8> @llvm.vp.fptoui.v4i8.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f64_v4i8 = call <4 x i8> @llvm.vp.fptoui.v4i8.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4i16 = call <4 x i16> @llvm.vp.fptoui.v4i16.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4i16 = call <4 x i16> @llvm.vp.fptoui.v4i16.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f32_v4i32 = call <4 x i32> @llvm.vp.fptoui.v4i32.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4f64_v4i32 = call <4 x i32> @llvm.vp.fptoui.v4i32.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f32_v4i64 = call <4 x i64> @llvm.vp.fptoui.v4i64.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4f64_v4i64 = call <4 x i64> @llvm.vp.fptoui.v4i64.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f32_v4i1 = call <4 x i1> @llvm.vp.fptoui.v4i1.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4f64_v4i1 = call <4 x i1> @llvm.vp.fptoui.v4i1.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32_v8i8 = fptoui <8 x float> undef to <8 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64_v8i8 = fptoui <8 x double> undef to <8 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8f32_v8i16 = fptoui <8 x float> undef to <8 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f64_v8i16 = fptoui <8 x double> undef to <8 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f32_v8i32 = fptoui <8 x float> undef to <8 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8f64_v8i32 = fptoui <8 x double> undef to <8 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f32_v8i64 = fptoui <8 x float> undef to <8 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8f64_v8i64 = fptoui <8 x double> undef to <8 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8f32_v8i1 = fptoui <8 x float> undef to <8 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8f64_v8i1 = fptoui <8 x double> undef to <8 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f32_v8i8 = call <8 x i8> @llvm.vp.fptoui.v8i8.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f64_v8i8 = call <8 x i8> @llvm.vp.fptoui.v8i8.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v8f32_v8i16 = call <8 x i16> @llvm.vp.fptoui.v8i16.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f64_v8i16 = call <8 x i16> @llvm.vp.fptoui.v8i16.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f32_v8i32 = call <8 x i32> @llvm.vp.fptoui.v8i32.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8f64_v8i32 = call <8 x i32> @llvm.vp.fptoui.v8i32.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f32_v8i64 = call <8 x i64> @llvm.vp.fptoui.v8i64.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8f64_v8i64 = call <8 x i64> @llvm.vp.fptoui.v8i64.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8f32_v8i1 = call <8 x i1> @llvm.vp.fptoui.v8i1.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8f64_v8i1 = call <8 x i1> @llvm.vp.fptoui.v8i1.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v16f32_v16i8 = fptoui <16 x float> undef to <16 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v16f64_v16i8 = fptoui <16 x double> undef to <16 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16f32_v16i16 = fptoui <16 x float> undef to <16 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f64_v16i16 = fptoui <16 x double> undef to <16 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f32_v16i32 = fptoui <16 x float> undef to <16 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16f64_v16i32 = fptoui <16 x double> undef to <16 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f32_v16i64 = fptoui <16 x float> undef to <16 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16f64_v16i64 = fptoui <16 x double> undef to <16 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16f32_v16i1 = fptoui <16 x float> undef to <16 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16f64_v16i1 = fptoui <16 x double> undef to <16 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v16f32_v16i8 = call <16 x i8> @llvm.vp.fptoui.v16i8.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v16f64_v16i8 = call <16 x i8> @llvm.vp.fptoui.v16i8.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v16f32_v16i16 = call <16 x i16> @llvm.vp.fptoui.v16i16.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f64_v16i16 = call <16 x i16> @llvm.vp.fptoui.v16i16.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f32_v16i32 = call <16 x i32> @llvm.vp.fptoui.v16i32.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16f64_v16i32 = call <16 x i32> @llvm.vp.fptoui.v16i32.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f32_v16i64 = call <16 x i64> @llvm.vp.fptoui.v16i64.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16f64_v16i64 = call <16 x i64> @llvm.vp.fptoui.v16i64.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16f32_v16i1 = call <16 x i1> @llvm.vp.fptoui.v16i1.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16f64_v16i1 = call <16 x i1> @llvm.vp.fptoui.v16i1.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v32f32_v32i8 = fptoui <32 x float> undef to <32 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v32f64_v32i8 = fptoui <32 x double> undef to <32 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32f32_v32i16 = fptoui <32 x float> undef to <32 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v32f64_v32i16 = fptoui <32 x double> undef to <32 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32f32_v32i32 = fptoui <32 x float> undef to <32 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32f64_v32i32 = fptoui <32 x double> undef to <32 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32f32_v32i64 = fptoui <32 x float> undef to <32 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32f64_v32i64 = fptoui <32 x double> undef to <32 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32f32_v32i1 = fptoui <32 x float> undef to <32 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32f64_v32i1 = fptoui <32 x double> undef to <32 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v32f32_v32i8 = call <32 x i8> @llvm.vp.fptoui.v32i8.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v32f64_v32i8 = call <32 x i8> @llvm.vp.fptoui.v32i8.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v32f32_v32i16 = call <32 x i16> @llvm.vp.fptoui.v32i16.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v32f64_v32i16 = call <32 x i16> @llvm.vp.fptoui.v32i16.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32f32_v32i32 = call <32 x i32> @llvm.vp.fptoui.v32i32.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32f64_v32i32 = call <32 x i32> @llvm.vp.fptoui.v32i32.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32f32_v32i64 = call <32 x i64> @llvm.vp.fptoui.v32i64.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v32f64_v32i64 = call <32 x i64> @llvm.vp.fptoui.v32i64.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v32f32_v32i1 = call <32 x i1> @llvm.vp.fptoui.v32i1.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32f64_v32i1 = call <32 x i1> @llvm.vp.fptoui.v32i1.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %v64f32_v64i8 = fptoui <64 x float> undef to <64 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v64f64_v64i8 = fptoui <64 x double> undef to <64 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v64f32_v64i16 = fptoui <64 x float> undef to <64 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v64f64_v64i16 = fptoui <64 x double> undef to <64 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v64f32_v64i32 = fptoui <64 x float> undef to <64 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64f64_v64i32 = fptoui <64 x double> undef to <64 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64f32_v64i64 = fptoui <64 x float> undef to <64 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v64f64_v64i64 = fptoui <64 x double> undef to <64 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64f32_v64i1 = fptoui <64 x float> undef to <64 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64f64_v64i1 = fptoui <64 x double> undef to <64 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_v64f32_v64i8 = call <64 x i8> @llvm.vp.fptoui.v64i8.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v64f64_v64i8 = call <64 x i8> @llvm.vp.fptoui.v64i8.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v64f32_v64i16 = call <64 x i16> @llvm.vp.fptoui.v64i16.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v64f64_v64i16 = call <64 x i16> @llvm.vp.fptoui.v64i16.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v64f32_v64i32 = call <64 x i32> @llvm.vp.fptoui.v64i32.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64f64_v64i32 = call <64 x i32> @llvm.vp.fptoui.v64i32.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64f32_v64i64 = call <64 x i64> @llvm.vp.fptoui.v64i64.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v64f64_v64i64 = call <64 x i64> @llvm.vp.fptoui.v64i64.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64f32_v64i1 = call <64 x i1> @llvm.vp.fptoui.v64i1.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64f64_v64i1 = call <64 x i1> @llvm.vp.fptoui.v64i1.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %v128f32_v128i8 = fptoui <128 x float> undef to <128 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v128f64_v128i8 = fptoui <128 x double> undef to <128 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v128f32_v128i16 = fptoui <128 x float> undef to <128 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %v128f64_v128i16 = fptoui <128 x double> undef to <128 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v128f32_v128i32 = fptoui <128 x float> undef to <128 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128f64_v128i32 = fptoui <128 x double> undef to <128 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128f32_v128i64 = fptoui <128 x float> undef to <128 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v128f64_v128i64 = fptoui <128 x double> undef to <128 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v128f32_v128i1 = fptoui <128 x float> undef to <128 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %v128f64_v128i1 = fptoui <128 x double> undef to <128 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_v128f32_v128i8 = call <128 x i8> @llvm.vp.fptoui.v128i8.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v128f64_v128i8 = call <128 x i8> @llvm.vp.fptoui.v128i8.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v128f32_v128i16 = call <128 x i16> @llvm.vp.fptoui.v128i16.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_v128f64_v128i16 = call <128 x i16> @llvm.vp.fptoui.v128i16.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v128f32_v128i32 = call <128 x i32> @llvm.vp.fptoui.v128i32.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128f64_v128i32 = call <128 x i32> @llvm.vp.fptoui.v128i32.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128f32_v128i64 = call <128 x i64> @llvm.vp.fptoui.v128i64.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v128f64_v128i64 = call <128 x i64> @llvm.vp.fptoui.v128i64.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v128f32_v128i1 = call <128 x i1> @llvm.vp.fptoui.v128i1.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_v128f64_v128i1 = call <128 x i1> @llvm.vp.fptoui.v128i1.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f32_nxv1i8 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64_nxv1i8 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i16 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1f64_nxv1i16 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i32 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1i32 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f32_nxv1i64 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1f64_nxv1i64 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f32_nxv1i1 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1f64_nxv1i1 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f32_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptoui.nxv1i8.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f64_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptoui.nxv1i8.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptoui.nxv1i16.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1f64_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptoui.nxv1i16.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptoui.nxv1i32.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptoui.nxv1i32.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f32_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptoui.nxv1i64.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1f64_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptoui.nxv1i64.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f32_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptoui.nxv1i1.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1f64_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptoui.nxv1i1.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2i8 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64_nxv2i8 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv2i16 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv2i16 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f32_nxv2i32 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2f64_nxv2i32 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f32_nxv2i64 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2f64_nxv2i64 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f32_nxv2i1 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2f64_nxv2i1 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptoui.nxv2i8.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f64_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptoui.nxv2i8.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptoui.nxv2i16.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptoui.nxv2i16.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f32_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptoui.nxv2i32.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2f64_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptoui.nxv2i32.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f32_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptoui.nxv2i64.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2f64_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptoui.nxv2i64.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f32_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptoui.nxv2i1.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2f64_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptoui.nxv2i1.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32_nxv4i8 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64_nxv4i8 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4f32_nxv4i16 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f64_nxv4i16 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f32_nxv4i32 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4f64_nxv4i32 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f32_nxv4i64 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4f64_nxv4i64 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4f32_nxv4i1 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4f64_nxv4i1 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f32_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptoui.nxv4i8.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f64_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptoui.nxv4i8.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv4f32_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptoui.nxv4i16.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f64_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptoui.nxv4i16.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f32_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptoui.nxv4i32.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4f64_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptoui.nxv4i32.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f32_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptoui.nxv4i64.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4f64_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptoui.nxv4i64.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4f32_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptoui.nxv4i1.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4f64_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptoui.nxv4i1.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv8f32_nxv8i8 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %nxv8f64_nxv8i8 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8f32_nxv8i16 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f64_nxv8i16 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f32_nxv8i32 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8f64_nxv8i32 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f32_nxv8i64 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8f64_nxv8i64 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8f32_nxv8i1 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8f64_nxv8i1 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv8f32_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptoui.nxv8i8.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_nxv8f64_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptoui.nxv8i8.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8f32_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptoui.nxv8i16.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f64_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptoui.nxv8i16.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f32_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptoui.nxv8i32.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8f64_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptoui.nxv8i32.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f32_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptoui.nxv8i64.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8f64_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptoui.nxv8i64.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8f32_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptoui.nxv8i1.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8f64_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptoui.nxv8i1.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv16f32_nxv16i8 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %nxv16f64_nxv16i8 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16f32_nxv16i16 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv16f64_nxv16i16 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16f32_nxv16i32 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16f64_nxv16i32 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16f32_nxv16i64 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16f64_nxv16i64 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv16f32_nxv16i1 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16f64_nxv16i1 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv16f32_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptoui.nxv16i8.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_nxv16f64_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptoui.nxv16i8.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv16f32_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptoui.nxv16i16.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv16f64_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptoui.nxv16i16.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16f32_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptoui.nxv16i32.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16f64_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptoui.nxv16i32.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16f32_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptoui.nxv16i64.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv16f64_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptoui.nxv16i64.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv16f32_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptoui.nxv16i1.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16f64_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptoui.nxv16i1.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %nxv32f32_nxv32i8 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %nxv32f64_nxv32i8 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv32f32_nxv32i16 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv32f64_nxv32i16 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32f32_nxv32i32 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32f64_nxv32i32 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32f32_nxv32i64 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32f64_nxv32i64 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32f32_nxv32i1 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32f64_nxv32i1 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %vp_nxv32f32_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptoui.nxv32i8.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_nxv32f64_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptoui.nxv32i8.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv32f32_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptoui.nxv32i16.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv32f64_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptoui.nxv32i16.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv32f32_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptoui.nxv32i32.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32f64_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptoui.nxv32i32.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32f32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptoui.nxv32i64.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv32f64_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptoui.nxv32i64.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32f32_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptoui.nxv32i1.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32f64_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptoui.nxv32i1.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %nxv64f32_nxv64i8 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %nxv64f64_nxv64i8 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv64f32_nxv64i16 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %nxv64f64_nxv64i16 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv64f32_nxv64i32 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64f64_nxv64i32 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64f32_nxv64i64 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv64f64_nxv64i64 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i64>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv64f32_nxv64i1 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %nxv64f64_nxv64i1 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %vp_nxv64f32_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptoui.nxv64i8.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_nxv64f64_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptoui.nxv64i8.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv64f32_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptoui.nxv64i16.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %vp_nxv64f64_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptoui.nxv64i16.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv64f32_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptoui.nxv64i32.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64f64_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptoui.nxv64i32.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64f32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptoui.nxv64i64.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv64f64_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptoui.nxv64i64.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv64f32_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptoui.nxv64i1.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_nxv64f64_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptoui.nxv64i1.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; ZVE64X-LABEL: 'fptoui'
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f32_v2i8 = fptoui <2 x float> undef to <2 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f64_v2i8 = fptoui <2 x double> undef to <2 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f32_v2i16 = fptoui <2 x float> undef to <2 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f64_v2i16 = fptoui <2 x double> undef to <2 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f32_v2i32 = fptoui <2 x float> undef to <2 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v2f64_v2i32 = fptoui <2 x double> undef to <2 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f32_v2i64 = fptoui <2 x float> undef to <2 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2f64_v2i64 = fptoui <2 x double> undef to <2 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %v2f32_v2i1 = fptoui <2 x float> undef to <2 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %v2f64_v2i1 = fptoui <2 x double> undef to <2 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f32_v2i8 = call <2 x i8> @llvm.vp.fptoui.v2i8.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f64_v2i8 = call <2 x i8> @llvm.vp.fptoui.v2i8.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f32_v2i16 = call <2 x i16> @llvm.vp.fptoui.v2i16.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f64_v2i16 = call <2 x i16> @llvm.vp.fptoui.v2i16.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f32_v2i32 = call <2 x i32> @llvm.vp.fptoui.v2i32.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v2f64_v2i32 = call <2 x i32> @llvm.vp.fptoui.v2i32.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f32_v2i64 = call <2 x i64> @llvm.vp.fptoui.v2i64.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2f64_v2i64 = call <2 x i64> @llvm.vp.fptoui.v2i64.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %vp_v2f32_v2i1 = call <2 x i1> @llvm.vp.fptoui.v2i1.v2f32(<2 x float> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %vp_v2f64_v2i1 = call <2 x i1> @llvm.vp.fptoui.v2i1.v2f64(<2 x double> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f32_v4i8 = fptoui <4 x float> undef to <4 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f64_v4i8 = fptoui <4 x double> undef to <4 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f32_v4i16 = fptoui <4 x float> undef to <4 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f64_v4i16 = fptoui <4 x double> undef to <4 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f32_v4i32 = fptoui <4 x float> undef to <4 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v4f64_v4i32 = fptoui <4 x double> undef to <4 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f32_v4i64 = fptoui <4 x float> undef to <4 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4f64_v4i64 = fptoui <4 x double> undef to <4 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %v4f32_v4i1 = fptoui <4 x float> undef to <4 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %v4f64_v4i1 = fptoui <4 x double> undef to <4 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f32_v4i8 = call <4 x i8> @llvm.vp.fptoui.v4i8.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f64_v4i8 = call <4 x i8> @llvm.vp.fptoui.v4i8.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f32_v4i16 = call <4 x i16> @llvm.vp.fptoui.v4i16.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f64_v4i16 = call <4 x i16> @llvm.vp.fptoui.v4i16.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f32_v4i32 = call <4 x i32> @llvm.vp.fptoui.v4i32.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v4f64_v4i32 = call <4 x i32> @llvm.vp.fptoui.v4i32.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4f32_v4i64 = call <4 x i64> @llvm.vp.fptoui.v4i64.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4f64_v4i64 = call <4 x i64> @llvm.vp.fptoui.v4i64.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %vp_v4f32_v4i1 = call <4 x i1> @llvm.vp.fptoui.v4i1.v4f32(<4 x float> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %vp_v4f64_v4i1 = call <4 x i1> @llvm.vp.fptoui.v4i1.v4f64(<4 x double> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f32_v8i8 = fptoui <8 x float> undef to <8 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f64_v8i8 = fptoui <8 x double> undef to <8 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f32_v8i16 = fptoui <8 x float> undef to <8 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f64_v8i16 = fptoui <8 x double> undef to <8 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f32_v8i32 = fptoui <8 x float> undef to <8 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v8f64_v8i32 = fptoui <8 x double> undef to <8 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f32_v8i64 = fptoui <8 x float> undef to <8 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8f64_v8i64 = fptoui <8 x double> undef to <8 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v8f32_v8i1 = fptoui <8 x float> undef to <8 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %v8f64_v8i1 = fptoui <8 x double> undef to <8 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f32_v8i8 = call <8 x i8> @llvm.vp.fptoui.v8i8.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f64_v8i8 = call <8 x i8> @llvm.vp.fptoui.v8i8.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f32_v8i16 = call <8 x i16> @llvm.vp.fptoui.v8i16.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f64_v8i16 = call <8 x i16> @llvm.vp.fptoui.v8i16.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f32_v8i32 = call <8 x i32> @llvm.vp.fptoui.v8i32.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v8f64_v8i32 = call <8 x i32> @llvm.vp.fptoui.v8i32.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8f32_v8i64 = call <8 x i64> @llvm.vp.fptoui.v8i64.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8f64_v8i64 = call <8 x i64> @llvm.vp.fptoui.v8i64.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v8f32_v8i1 = call <8 x i1> @llvm.vp.fptoui.v8i1.v8f32(<8 x float> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %vp_v8f64_v8i1 = call <8 x i1> @llvm.vp.fptoui.v8i1.v8f64(<8 x double> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f32_v16i8 = fptoui <16 x float> undef to <16 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f64_v16i8 = fptoui <16 x double> undef to <16 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f32_v16i16 = fptoui <16 x float> undef to <16 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f64_v16i16 = fptoui <16 x double> undef to <16 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f32_v16i32 = fptoui <16 x float> undef to <16 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v16f64_v16i32 = fptoui <16 x double> undef to <16 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f32_v16i64 = fptoui <16 x float> undef to <16 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16f64_v16i64 = fptoui <16 x double> undef to <16 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %v16f32_v16i1 = fptoui <16 x float> undef to <16 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %v16f64_v16i1 = fptoui <16 x double> undef to <16 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f32_v16i8 = call <16 x i8> @llvm.vp.fptoui.v16i8.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f64_v16i8 = call <16 x i8> @llvm.vp.fptoui.v16i8.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f32_v16i16 = call <16 x i16> @llvm.vp.fptoui.v16i16.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f64_v16i16 = call <16 x i16> @llvm.vp.fptoui.v16i16.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f32_v16i32 = call <16 x i32> @llvm.vp.fptoui.v16i32.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v16f64_v16i32 = call <16 x i32> @llvm.vp.fptoui.v16i32.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16f32_v16i64 = call <16 x i64> @llvm.vp.fptoui.v16i64.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16f64_v16i64 = call <16 x i64> @llvm.vp.fptoui.v16i64.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %vp_v16f32_v16i1 = call <16 x i1> @llvm.vp.fptoui.v16i1.v16f32(<16 x float> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %vp_v16f64_v16i1 = call <16 x i1> @llvm.vp.fptoui.v16i1.v16f64(<16 x double> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f32_v32i8 = fptoui <32 x float> undef to <32 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f64_v32i8 = fptoui <32 x double> undef to <32 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f32_v32i16 = fptoui <32 x float> undef to <32 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f64_v32i16 = fptoui <32 x double> undef to <32 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f32_v32i32 = fptoui <32 x float> undef to <32 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v32f64_v32i32 = fptoui <32 x double> undef to <32 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %v32f32_v32i64 = fptoui <32 x float> undef to <32 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %v32f64_v32i64 = fptoui <32 x double> undef to <32 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 319 for instruction: %v32f32_v32i1 = fptoui <32 x float> undef to <32 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 319 for instruction: %v32f64_v32i1 = fptoui <32 x double> undef to <32 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f32_v32i8 = call <32 x i8> @llvm.vp.fptoui.v32i8.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f64_v32i8 = call <32 x i8> @llvm.vp.fptoui.v32i8.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f32_v32i16 = call <32 x i16> @llvm.vp.fptoui.v32i16.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f64_v32i16 = call <32 x i16> @llvm.vp.fptoui.v32i16.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f32_v32i32 = call <32 x i32> @llvm.vp.fptoui.v32i32.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v32f64_v32i32 = call <32 x i32> @llvm.vp.fptoui.v32i32.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %vp_v32f32_v32i64 = call <32 x i64> @llvm.vp.fptoui.v32i64.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %vp_v32f64_v32i64 = call <32 x i64> @llvm.vp.fptoui.v32i64.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 319 for instruction: %vp_v32f32_v32i1 = call <32 x i1> @llvm.vp.fptoui.v32i1.v32f32(<32 x float> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 319 for instruction: %vp_v32f64_v32i1 = call <32 x i1> @llvm.vp.fptoui.v32i1.v32f64(<32 x double> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v64f32_v64i8 = fptoui <64 x float> undef to <64 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v64f64_v64i8 = fptoui <64 x double> undef to <64 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v64f32_v64i16 = fptoui <64 x float> undef to <64 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v64f64_v64i16 = fptoui <64 x double> undef to <64 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %v64f32_v64i32 = fptoui <64 x float> undef to <64 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %v64f64_v64i32 = fptoui <64 x double> undef to <64 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %v64f32_v64i64 = fptoui <64 x float> undef to <64 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %v64f64_v64i64 = fptoui <64 x double> undef to <64 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 639 for instruction: %v64f32_v64i1 = fptoui <64 x float> undef to <64 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 639 for instruction: %v64f64_v64i1 = fptoui <64 x double> undef to <64 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v64f32_v64i8 = call <64 x i8> @llvm.vp.fptoui.v64i8.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v64f64_v64i8 = call <64 x i8> @llvm.vp.fptoui.v64i8.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v64f32_v64i16 = call <64 x i16> @llvm.vp.fptoui.v64i16.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v64f64_v64i16 = call <64 x i16> @llvm.vp.fptoui.v64i16.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %vp_v64f32_v64i32 = call <64 x i32> @llvm.vp.fptoui.v64i32.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %vp_v64f64_v64i32 = call <64 x i32> @llvm.vp.fptoui.v64i32.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %vp_v64f32_v64i64 = call <64 x i64> @llvm.vp.fptoui.v64i64.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %vp_v64f64_v64i64 = call <64 x i64> @llvm.vp.fptoui.v64i64.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 639 for instruction: %vp_v64f32_v64i1 = call <64 x i1> @llvm.vp.fptoui.v64i1.v64f32(<64 x float> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 639 for instruction: %vp_v64f64_v64i1 = call <64 x i1> @llvm.vp.fptoui.v64i1.v64f64(<64 x double> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 511 for instruction: %v128f32_v128i8 = fptoui <128 x float> undef to <128 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 511 for instruction: %v128f64_v128i8 = fptoui <128 x double> undef to <128 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %v128f32_v128i16 = fptoui <128 x float> undef to <128 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %v128f64_v128i16 = fptoui <128 x double> undef to <128 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %v128f32_v128i32 = fptoui <128 x float> undef to <128 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %v128f64_v128i32 = fptoui <128 x double> undef to <128 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %v128f32_v128i64 = fptoui <128 x float> undef to <128 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %v128f64_v128i64 = fptoui <128 x double> undef to <128 x i64>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1279 for instruction: %v128f32_v128i1 = fptoui <128 x float> undef to <128 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1279 for instruction: %v128f64_v128i1 = fptoui <128 x double> undef to <128 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 511 for instruction: %vp_v128f32_v128i8 = call <128 x i8> @llvm.vp.fptoui.v128i8.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 511 for instruction: %vp_v128f64_v128i8 = call <128 x i8> @llvm.vp.fptoui.v128i8.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %vp_v128f32_v128i16 = call <128 x i16> @llvm.vp.fptoui.v128i16.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 510 for instruction: %vp_v128f64_v128i16 = call <128 x i16> @llvm.vp.fptoui.v128i16.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %vp_v128f32_v128i32 = call <128 x i32> @llvm.vp.fptoui.v128i32.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 508 for instruction: %vp_v128f64_v128i32 = call <128 x i32> @llvm.vp.fptoui.v128i32.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %vp_v128f32_v128i64 = call <128 x i64> @llvm.vp.fptoui.v128i64.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %vp_v128f64_v128i64 = call <128 x i64> @llvm.vp.fptoui.v128i64.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1279 for instruction: %vp_v128f32_v128i1 = call <128 x i1> @llvm.vp.fptoui.v128i1.v128f32(<128 x float> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1279 for instruction: %vp_v128f64_v128i1 = call <128 x i1> @llvm.vp.fptoui.v128i1.v128f64(<128 x double> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i8 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i8 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i16 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i16 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i32 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i32 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i64 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i64 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f32_nxv1i1 = fptoui <vscale x 1 x float> undef to <vscale x 1 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1f64_nxv1i1 = fptoui <vscale x 1 x double> undef to <vscale x 1 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptoui.nxv1i8.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i8 = call <vscale x 1 x i8> @llvm.vp.fptoui.nxv1i8.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptoui.nxv1i16.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i16 = call <vscale x 1 x i16> @llvm.vp.fptoui.nxv1i16.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptoui.nxv1i32.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i32 = call <vscale x 1 x i32> @llvm.vp.fptoui.nxv1i32.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptoui.nxv1i64.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i64 = call <vscale x 1 x i64> @llvm.vp.fptoui.nxv1i64.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f32_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptoui.nxv1i1.nxv1f32(<vscale x 1 x float> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1f64_nxv1i1 = call <vscale x 1 x i1> @llvm.vp.fptoui.nxv1i1.nxv1f64(<vscale x 1 x double> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i8 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i8 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i16 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i16 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i32 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i32 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i64 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i64 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f32_nxv2i1 = fptoui <vscale x 2 x float> undef to <vscale x 2 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2f64_nxv2i1 = fptoui <vscale x 2 x double> undef to <vscale x 2 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptoui.nxv2i8.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i8 = call <vscale x 2 x i8> @llvm.vp.fptoui.nxv2i8.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptoui.nxv2i16.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i16 = call <vscale x 2 x i16> @llvm.vp.fptoui.nxv2i16.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptoui.nxv2i32.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i32 = call <vscale x 2 x i32> @llvm.vp.fptoui.nxv2i32.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptoui.nxv2i64.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i64 = call <vscale x 2 x i64> @llvm.vp.fptoui.nxv2i64.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f32_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptoui.nxv2i1.nxv2f32(<vscale x 2 x float> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2f64_nxv2i1 = call <vscale x 2 x i1> @llvm.vp.fptoui.nxv2i1.nxv2f64(<vscale x 2 x double> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i8 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i8 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i16 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i16 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i32 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i32 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i64 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i64 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f32_nxv4i1 = fptoui <vscale x 4 x float> undef to <vscale x 4 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4f64_nxv4i1 = fptoui <vscale x 4 x double> undef to <vscale x 4 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptoui.nxv4i8.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i8 = call <vscale x 4 x i8> @llvm.vp.fptoui.nxv4i8.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptoui.nxv4i16.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i16 = call <vscale x 4 x i16> @llvm.vp.fptoui.nxv4i16.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptoui.nxv4i32.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i32 = call <vscale x 4 x i32> @llvm.vp.fptoui.nxv4i32.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptoui.nxv4i64.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i64 = call <vscale x 4 x i64> @llvm.vp.fptoui.nxv4i64.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f32_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptoui.nxv4i1.nxv4f32(<vscale x 4 x float> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4f64_nxv4i1 = call <vscale x 4 x i1> @llvm.vp.fptoui.nxv4i1.nxv4f64(<vscale x 4 x double> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i8 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i8 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i16 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i16 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i32 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i32 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i64 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i64 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f32_nxv8i1 = fptoui <vscale x 8 x float> undef to <vscale x 8 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8f64_nxv8i1 = fptoui <vscale x 8 x double> undef to <vscale x 8 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptoui.nxv8i8.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i8 = call <vscale x 8 x i8> @llvm.vp.fptoui.nxv8i8.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptoui.nxv8i16.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i16 = call <vscale x 8 x i16> @llvm.vp.fptoui.nxv8i16.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptoui.nxv8i32.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i32 = call <vscale x 8 x i32> @llvm.vp.fptoui.nxv8i32.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptoui.nxv8i64.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i64 = call <vscale x 8 x i64> @llvm.vp.fptoui.nxv8i64.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f32_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptoui.nxv8i1.nxv8f32(<vscale x 8 x float> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8f64_nxv8i1 = call <vscale x 8 x i1> @llvm.vp.fptoui.nxv8i1.nxv8f64(<vscale x 8 x double> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i8 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i8 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i16 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i16 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i32 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i32 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i64 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i64 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f32_nxv16i1 = fptoui <vscale x 16 x float> undef to <vscale x 16 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16f64_nxv16i1 = fptoui <vscale x 16 x double> undef to <vscale x 16 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptoui.nxv16i8.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i8 = call <vscale x 16 x i8> @llvm.vp.fptoui.nxv16i8.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptoui.nxv16i16.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i16 = call <vscale x 16 x i16> @llvm.vp.fptoui.nxv16i16.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptoui.nxv16i32.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i32 = call <vscale x 16 x i32> @llvm.vp.fptoui.nxv16i32.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptoui.nxv16i64.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i64 = call <vscale x 16 x i64> @llvm.vp.fptoui.nxv16i64.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f32_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptoui.nxv16i1.nxv16f32(<vscale x 16 x float> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16f64_nxv16i1 = call <vscale x 16 x i1> @llvm.vp.fptoui.nxv16i1.nxv16f64(<vscale x 16 x double> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i8 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i8 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i16 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i16 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i32 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i32 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i64 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i64 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f32_nxv32i1 = fptoui <vscale x 32 x float> undef to <vscale x 32 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32f64_nxv32i1 = fptoui <vscale x 32 x double> undef to <vscale x 32 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptoui.nxv32i8.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i8 = call <vscale x 32 x i8> @llvm.vp.fptoui.nxv32i8.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptoui.nxv32i16.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i16 = call <vscale x 32 x i16> @llvm.vp.fptoui.nxv32i16.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptoui.nxv32i32.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i32 = call <vscale x 32 x i32> @llvm.vp.fptoui.nxv32i32.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptoui.nxv32i64.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i64 = call <vscale x 32 x i64> @llvm.vp.fptoui.nxv32i64.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f32_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptoui.nxv32i1.nxv32f32(<vscale x 32 x float> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32f64_nxv32i1 = call <vscale x 32 x i1> @llvm.vp.fptoui.nxv32i1.nxv32f64(<vscale x 32 x double> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i8 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i8 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i8>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i16 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i16 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i16>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i32 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i32 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i32>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i64 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i64 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i64>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f32_nxv64i1 = fptoui <vscale x 64 x float> undef to <vscale x 64 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64f64_nxv64i1 = fptoui <vscale x 64 x double> undef to <vscale x 64 x i1>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptoui.nxv64i8.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i8 = call <vscale x 64 x i8> @llvm.vp.fptoui.nxv64i8.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptoui.nxv64i16.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i16 = call <vscale x 64 x i16> @llvm.vp.fptoui.nxv64i16.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptoui.nxv64i32.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i32 = call <vscale x 64 x i32> @llvm.vp.fptoui.nxv64i32.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptoui.nxv64i64.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i64 = call <vscale x 64 x i64> @llvm.vp.fptoui.nxv64i64.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f32_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptoui.nxv64i1.nxv64f32(<vscale x 64 x float> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64f64_nxv64i1 = call <vscale x 64 x i1> @llvm.vp.fptoui.nxv64i1.nxv64f64(<vscale x 64 x double> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2f32_v2i8 = fptoui <2 x float> undef to <2 x i8>
   %v2f64_v2i8 = fptoui <2 x double> undef to <2 x i8>
@@ -3617,288 +4358,571 @@ define void @fptoui() {
 }
 
 define void @sitofp() {
-; CHECK-LABEL: 'sitofp'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_v2f32 = sitofp <2 x i8> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_v2f64 = sitofp <2 x i8> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16_v2f32 = sitofp <2 x i16> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16_v2f64 = sitofp <2 x i16> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_v2f32 = sitofp <2 x i32> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_v2f32 = sitofp <2 x i64> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_v2f64 = sitofp <2 x i64> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f32 = sitofp <2 x i1> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f64 = sitofp <2 x i1> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi8_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi8_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi16_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi16_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi32_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi32_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi64_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi64_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4f32 = sitofp <4 x i8> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_v4f64 = sitofp <4 x i8> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4f32 = sitofp <4 x i16> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i16_v4f64 = sitofp <4 x i16> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4f32 = sitofp <4 x i32> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4f64 = sitofp <4 x i32> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_v4f32 = sitofp <4 x i64> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_v4f64 = sitofp <4 x i64> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_v4f32 = sitofp <4 x i1> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4f64 = sitofp <4 x i1> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi8_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi8_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi16_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi16_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi32_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi32_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi64_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi64_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi1_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4fi1_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_v8f32 = sitofp <8 x i8> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i8_v8f64 = sitofp <8 x i8> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8f32 = sitofp <8 x i16> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i16_v8f64 = sitofp <8 x i16> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_v8f32 = sitofp <8 x i32> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8f64 = sitofp <8 x i32> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8f32 = sitofp <8 x i64> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_v8f64 = sitofp <8 x i64> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8f32 = sitofp <8 x i1> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8f64 = sitofp <8 x i1> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8fi8_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8fi8_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi16_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8fi16_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi32_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi32_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi64_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi64_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi1_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8fi1_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i8_v16f32 = sitofp <16 x i8> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_v16f64 = sitofp <16 x i8> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16f32 = sitofp <16 x i16> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i16_v16f64 = sitofp <16 x i16> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_v16f32 = sitofp <16 x i32> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16f64 = sitofp <16 x i32> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i64_v16f32 = sitofp <16 x i64> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_v16f64 = sitofp <16 x i64> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16f32 = sitofp <16 x i1> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16f64 = sitofp <16 x i1> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16fi8_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16fi8_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi16_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16fi16_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi32_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi32_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi64_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi64_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi1_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16fi1_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_v32f32 = sitofp <32 x i8> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i8_v32f64 = sitofp <32 x i8> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32f32 = sitofp <32 x i16> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i16_v32f64 = sitofp <32 x i16> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i32_v32f32 = sitofp <32 x i32> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32f64 = sitofp <32 x i32> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i64_v32f32 = sitofp <32 x i64> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i64_v32f64 = sitofp <32 x i64> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32f32 = sitofp <32 x i1> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32f64 = sitofp <32 x i1> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v32fi8_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32fi8_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32fi16_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32fi16_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32fi32_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32fi32_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32fi64_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32fi64_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32fi1_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32fi1_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64i8_v64f32 = sitofp <64 x i8> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64i8_v64f64 = sitofp <64 x i8> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64f32 = sitofp <64 x i16> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64i16_v64f64 = sitofp <64 x i16> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i32_v64f32 = sitofp <64 x i32> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64f64 = sitofp <64 x i32> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64i64_v64f32 = sitofp <64 x i64> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v64i64_v64f64 = sitofp <64 x i64> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64f32 = sitofp <64 x i1> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64f64 = sitofp <64 x i1> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64fi8_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64fi8_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64fi16_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64fi16_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64fi32_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64fi32_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64fi64_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v64fi64_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64fi1_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64fi1_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v128i8_v128f32 = sitofp <128 x i8> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %v128i8_v128f64 = sitofp <128 x i8> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128f32 = sitofp <128 x i16> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %v128i16_v128f64 = sitofp <128 x i16> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v128i32_v128f32 = sitofp <128 x i32> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128f64 = sitofp <128 x i32> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128i64_v128f32 = sitofp <128 x i64> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v128i64_v128f64 = sitofp <128 x i64> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128f32 = sitofp <128 x i1> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128f64 = sitofp <128 x i1> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v128fi8_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_v128fi8_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v128fi16_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_v128fi16_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v128fi32_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128fi32_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128fi64_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v128fi64_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128fi1_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128fi1_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i8_nxv1f32 = sitofp <vscale x 1 x i8> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i8_nxv1f64 = sitofp <vscale x 1 x i8> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16_nxv1f32 = sitofp <vscale x 1 x i16> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i16_nxv1f64 = sitofp <vscale x 1 x i16> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32_nxv1f32 = sitofp <vscale x 1 x i32> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32_nxv1f64 = sitofp <vscale x 1 x i32> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64_nxv1f32 = sitofp <vscale x 1 x i64> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64_nxv1f64 = sitofp <vscale x 1 x i64> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i1_nxv1f32 = sitofp <vscale x 1 x i1> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i1_nxv1f64 = sitofp <vscale x 1 x i1> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi8_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi8_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi16_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi16_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi32_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi32_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi64_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi64_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1fi1_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1fi1_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2f32 = sitofp <vscale x 2 x i8> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i8_nxv2f64 = sitofp <vscale x 2 x i8> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2f32 = sitofp <vscale x 2 x i16> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i16_nxv2f64 = sitofp <vscale x 2 x i16> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2f32 = sitofp <vscale x 2 x i32> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2f64 = sitofp <vscale x 2 x i32> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64_nxv2f32 = sitofp <vscale x 2 x i64> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64_nxv2f64 = sitofp <vscale x 2 x i64> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_nxv2f32 = sitofp <vscale x 2 x i1> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2f64 = sitofp <vscale x 2 x i1> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi8_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi8_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi16_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi16_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi32_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi32_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi64_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi64_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi1_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv2fi1_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i8_nxv4f32 = sitofp <vscale x 4 x i8> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i8_nxv4f64 = sitofp <vscale x 4 x i8> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4f32 = sitofp <vscale x 4 x i16> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i16_nxv4f64 = sitofp <vscale x 4 x i16> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32_nxv4f32 = sitofp <vscale x 4 x i32> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4f64 = sitofp <vscale x 4 x i32> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4f32 = sitofp <vscale x 4 x i64> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64_nxv4f64 = sitofp <vscale x 4 x i64> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4f32 = sitofp <vscale x 4 x i1> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4f64 = sitofp <vscale x 4 x i1> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4fi8_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4fi8_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi16_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4fi16_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi32_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi32_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi64_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi64_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi1_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv4fi1_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i8_nxv8f32 = sitofp <vscale x 8 x i8> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8i8_nxv8f64 = sitofp <vscale x 8 x i8> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8f32 = sitofp <vscale x 8 x i16> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8i16_nxv8f64 = sitofp <vscale x 8 x i16> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32_nxv8f32 = sitofp <vscale x 8 x i32> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8f64 = sitofp <vscale x 8 x i32> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i64_nxv8f32 = sitofp <vscale x 8 x i64> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64_nxv8f64 = sitofp <vscale x 8 x i64> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8f32 = sitofp <vscale x 8 x i1> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8f64 = sitofp <vscale x 8 x i1> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8fi8_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8fi8_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi16_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8fi16_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi32_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi32_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi64_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi64_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv8fi1_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv8fi1_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv16i8_nxv16f32 = sitofp <vscale x 16 x i8> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i8_nxv16f64 = sitofp <vscale x 16 x i8> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16f32 = sitofp <vscale x 16 x i16> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i16_nxv16f64 = sitofp <vscale x 16 x i16> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32_nxv16f32 = sitofp <vscale x 16 x i32> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16f64 = sitofp <vscale x 16 x i32> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i64_nxv16f32 = sitofp <vscale x 16 x i64> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i64_nxv16f64 = sitofp <vscale x 16 x i64> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16f32 = sitofp <vscale x 16 x i1> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16f64 = sitofp <vscale x 16 x i1> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv16fi8_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16fi8_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16fi16_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16fi16_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16fi32_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16fi32_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16fi64_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv16fi64_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv16fi1_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv16fi1_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32i8_nxv32f32 = sitofp <vscale x 32 x i8> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32i8_nxv32f64 = sitofp <vscale x 32 x i8> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32f32 = sitofp <vscale x 32 x i16> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32i16_nxv32f64 = sitofp <vscale x 32 x i16> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i32_nxv32f32 = sitofp <vscale x 32 x i32> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32f64 = sitofp <vscale x 32 x i32> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32i64_nxv32f32 = sitofp <vscale x 32 x i64> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i64_nxv32f64 = sitofp <vscale x 32 x i64> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32f32 = sitofp <vscale x 32 x i1> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32f64 = sitofp <vscale x 32 x i1> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32fi8_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32fi8_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32fi16_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32fi16_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv32fi32_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32fi32_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32fi64_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv32fi64_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv32fi1_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv32fi1_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv64i8_nxv64f32 = sitofp <vscale x 64 x i8> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %nxv64i8_nxv64f64 = sitofp <vscale x 64 x i8> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64f32 = sitofp <vscale x 64 x i16> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %nxv64i16_nxv64f64 = sitofp <vscale x 64 x i16> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv64i32_nxv64f32 = sitofp <vscale x 64 x i32> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64f64 = sitofp <vscale x 64 x i32> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64i64_nxv64f32 = sitofp <vscale x 64 x i64> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv64i64_nxv64f64 = sitofp <vscale x 64 x i64> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64f32 = sitofp <vscale x 64 x i1> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64f64 = sitofp <vscale x 64 x i1> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv64fi8_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_nxv64fi8_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv64fi16_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_nxv64fi16_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv64fi32_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64fi32_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64fi64_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv64fi64_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv64fi1_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_nxv64fi1_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ZVE64D-LABEL: 'sitofp'
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_v2f32 = sitofp <2 x i8> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_v2f64 = sitofp <2 x i8> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16_v2f32 = sitofp <2 x i16> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16_v2f64 = sitofp <2 x i16> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_v2f32 = sitofp <2 x i32> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_v2f32 = sitofp <2 x i64> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_v2f64 = sitofp <2 x i64> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f32 = sitofp <2 x i1> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f64 = sitofp <2 x i1> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi8_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi8_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi16_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi16_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi32_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi32_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi64_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi64_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4f32 = sitofp <4 x i8> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_v4f64 = sitofp <4 x i8> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4f32 = sitofp <4 x i16> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i16_v4f64 = sitofp <4 x i16> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4f32 = sitofp <4 x i32> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4f64 = sitofp <4 x i32> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_v4f32 = sitofp <4 x i64> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_v4f64 = sitofp <4 x i64> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_v4f32 = sitofp <4 x i1> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4f64 = sitofp <4 x i1> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi8_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi8_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi16_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi16_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi32_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi32_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi64_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi64_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi1_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4fi1_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_v8f32 = sitofp <8 x i8> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i8_v8f64 = sitofp <8 x i8> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8f32 = sitofp <8 x i16> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i16_v8f64 = sitofp <8 x i16> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_v8f32 = sitofp <8 x i32> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8f64 = sitofp <8 x i32> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8f32 = sitofp <8 x i64> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_v8f64 = sitofp <8 x i64> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8f32 = sitofp <8 x i1> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8f64 = sitofp <8 x i1> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8fi8_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8fi8_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi16_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8fi16_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi32_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi32_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi64_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi64_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi1_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8fi1_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i8_v16f32 = sitofp <16 x i8> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_v16f64 = sitofp <16 x i8> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16f32 = sitofp <16 x i16> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i16_v16f64 = sitofp <16 x i16> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_v16f32 = sitofp <16 x i32> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16f64 = sitofp <16 x i32> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i64_v16f32 = sitofp <16 x i64> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_v16f64 = sitofp <16 x i64> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16f32 = sitofp <16 x i1> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16f64 = sitofp <16 x i1> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16fi8_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16fi8_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi16_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16fi16_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi32_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi32_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi64_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi64_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi1_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16fi1_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_v32f32 = sitofp <32 x i8> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i8_v32f64 = sitofp <32 x i8> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32f32 = sitofp <32 x i16> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i16_v32f64 = sitofp <32 x i16> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i32_v32f32 = sitofp <32 x i32> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32f64 = sitofp <32 x i32> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i64_v32f32 = sitofp <32 x i64> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i64_v32f64 = sitofp <32 x i64> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32f32 = sitofp <32 x i1> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32f64 = sitofp <32 x i1> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v32fi8_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32fi8_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32fi16_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32fi16_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32fi32_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32fi32_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32fi64_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32fi64_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32fi1_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32fi1_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64i8_v64f32 = sitofp <64 x i8> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64i8_v64f64 = sitofp <64 x i8> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64f32 = sitofp <64 x i16> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64i16_v64f64 = sitofp <64 x i16> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i32_v64f32 = sitofp <64 x i32> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64f64 = sitofp <64 x i32> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64i64_v64f32 = sitofp <64 x i64> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v64i64_v64f64 = sitofp <64 x i64> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64f32 = sitofp <64 x i1> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64f64 = sitofp <64 x i1> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64fi8_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64fi8_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64fi16_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64fi16_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64fi32_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64fi32_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64fi64_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v64fi64_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64fi1_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64fi1_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v128i8_v128f32 = sitofp <128 x i8> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %v128i8_v128f64 = sitofp <128 x i8> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128f32 = sitofp <128 x i16> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %v128i16_v128f64 = sitofp <128 x i16> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v128i32_v128f32 = sitofp <128 x i32> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128f64 = sitofp <128 x i32> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128i64_v128f32 = sitofp <128 x i64> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v128i64_v128f64 = sitofp <128 x i64> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128f32 = sitofp <128 x i1> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128f64 = sitofp <128 x i1> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v128fi8_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_v128fi8_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v128fi16_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_v128fi16_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v128fi32_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128fi32_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128fi64_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v128fi64_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128fi1_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128fi1_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i8_nxv1f32 = sitofp <vscale x 1 x i8> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i8_nxv1f64 = sitofp <vscale x 1 x i8> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16_nxv1f32 = sitofp <vscale x 1 x i16> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i16_nxv1f64 = sitofp <vscale x 1 x i16> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32_nxv1f32 = sitofp <vscale x 1 x i32> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32_nxv1f64 = sitofp <vscale x 1 x i32> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64_nxv1f32 = sitofp <vscale x 1 x i64> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64_nxv1f64 = sitofp <vscale x 1 x i64> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i1_nxv1f32 = sitofp <vscale x 1 x i1> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i1_nxv1f64 = sitofp <vscale x 1 x i1> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi8_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi8_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi16_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi16_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi32_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi32_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi64_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi64_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1fi1_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1fi1_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2f32 = sitofp <vscale x 2 x i8> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i8_nxv2f64 = sitofp <vscale x 2 x i8> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2f32 = sitofp <vscale x 2 x i16> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i16_nxv2f64 = sitofp <vscale x 2 x i16> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2f32 = sitofp <vscale x 2 x i32> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2f64 = sitofp <vscale x 2 x i32> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64_nxv2f32 = sitofp <vscale x 2 x i64> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64_nxv2f64 = sitofp <vscale x 2 x i64> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_nxv2f32 = sitofp <vscale x 2 x i1> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2f64 = sitofp <vscale x 2 x i1> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi8_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi8_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi16_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi16_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi32_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi32_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi64_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi64_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi1_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv2fi1_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i8_nxv4f32 = sitofp <vscale x 4 x i8> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i8_nxv4f64 = sitofp <vscale x 4 x i8> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4f32 = sitofp <vscale x 4 x i16> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i16_nxv4f64 = sitofp <vscale x 4 x i16> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32_nxv4f32 = sitofp <vscale x 4 x i32> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4f64 = sitofp <vscale x 4 x i32> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4f32 = sitofp <vscale x 4 x i64> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64_nxv4f64 = sitofp <vscale x 4 x i64> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4f32 = sitofp <vscale x 4 x i1> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4f64 = sitofp <vscale x 4 x i1> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4fi8_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4fi8_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi16_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4fi16_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi32_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi32_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi64_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi64_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi1_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv4fi1_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i8_nxv8f32 = sitofp <vscale x 8 x i8> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8i8_nxv8f64 = sitofp <vscale x 8 x i8> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8f32 = sitofp <vscale x 8 x i16> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8i16_nxv8f64 = sitofp <vscale x 8 x i16> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32_nxv8f32 = sitofp <vscale x 8 x i32> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8f64 = sitofp <vscale x 8 x i32> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i64_nxv8f32 = sitofp <vscale x 8 x i64> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64_nxv8f64 = sitofp <vscale x 8 x i64> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8f32 = sitofp <vscale x 8 x i1> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8f64 = sitofp <vscale x 8 x i1> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8fi8_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8fi8_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi16_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv8fi16_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi32_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi32_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi64_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv8fi64_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv8fi1_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv8fi1_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv16i8_nxv16f32 = sitofp <vscale x 16 x i8> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i8_nxv16f64 = sitofp <vscale x 16 x i8> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16f32 = sitofp <vscale x 16 x i16> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i16_nxv16f64 = sitofp <vscale x 16 x i16> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32_nxv16f32 = sitofp <vscale x 16 x i32> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16f64 = sitofp <vscale x 16 x i32> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i64_nxv16f32 = sitofp <vscale x 16 x i64> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i64_nxv16f64 = sitofp <vscale x 16 x i64> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16f32 = sitofp <vscale x 16 x i1> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16f64 = sitofp <vscale x 16 x i1> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv16fi8_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16fi8_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16fi16_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16fi16_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16fi32_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16fi32_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16fi64_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv16fi64_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv16fi1_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv16fi1_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32i8_nxv32f32 = sitofp <vscale x 32 x i8> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32i8_nxv32f64 = sitofp <vscale x 32 x i8> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32f32 = sitofp <vscale x 32 x i16> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32i16_nxv32f64 = sitofp <vscale x 32 x i16> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i32_nxv32f32 = sitofp <vscale x 32 x i32> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32f64 = sitofp <vscale x 32 x i32> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32i64_nxv32f32 = sitofp <vscale x 32 x i64> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i64_nxv32f64 = sitofp <vscale x 32 x i64> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32f32 = sitofp <vscale x 32 x i1> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32f64 = sitofp <vscale x 32 x i1> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32fi8_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32fi8_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32fi16_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32fi16_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv32fi32_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32fi32_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32fi64_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv32fi64_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv32fi1_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv32fi1_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv64i8_nxv64f32 = sitofp <vscale x 64 x i8> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %nxv64i8_nxv64f64 = sitofp <vscale x 64 x i8> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64f32 = sitofp <vscale x 64 x i16> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %nxv64i16_nxv64f64 = sitofp <vscale x 64 x i16> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv64i32_nxv64f32 = sitofp <vscale x 64 x i32> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64f64 = sitofp <vscale x 64 x i32> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64i64_nxv64f32 = sitofp <vscale x 64 x i64> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv64i64_nxv64f64 = sitofp <vscale x 64 x i64> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64f32 = sitofp <vscale x 64 x i1> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64f64 = sitofp <vscale x 64 x i1> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv64fi8_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_nxv64fi8_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv64fi16_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_nxv64fi16_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv64fi32_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64fi32_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64fi64_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv64fi64_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv64fi1_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_nxv64fi1_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; ZVE64X-LABEL: 'sitofp'
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_v2f32 = sitofp <2 x i8> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_v2f64 = sitofp <2 x i8> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i16_v2f32 = sitofp <2 x i16> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i16_v2f64 = sitofp <2 x i16> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i32_v2f32 = sitofp <2 x i32> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i32_v2f64 = sitofp <2 x i32> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i64_v2f32 = sitofp <2 x i64> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i64_v2f64 = sitofp <2 x i64> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f32 = sitofp <2 x i1> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f64 = sitofp <2 x i1> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi8_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi8_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi16_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi16_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi32_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi32_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi64_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi64_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f32 = call <2 x float> @llvm.vp.sitofp.v2f32.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f64 = call <2 x double> @llvm.vp.sitofp.v2f64.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i8_v4f32 = sitofp <4 x i8> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i8_v4f64 = sitofp <4 x i8> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i16_v4f32 = sitofp <4 x i16> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i16_v4f64 = sitofp <4 x i16> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i32_v4f32 = sitofp <4 x i32> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i32_v4f64 = sitofp <4 x i32> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i64_v4f32 = sitofp <4 x i64> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i64_v4f64 = sitofp <4 x i64> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_v4f32 = sitofp <4 x i1> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_v4f64 = sitofp <4 x i1> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi8_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi8_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi16_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi16_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi32_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi32_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi64_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi64_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi1_v4f32 = call <4 x float> @llvm.vp.sitofp.v4f32.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi1_v4f64 = call <4 x double> @llvm.vp.sitofp.v4f64.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i8_v8f32 = sitofp <8 x i8> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i8_v8f64 = sitofp <8 x i8> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i16_v8f32 = sitofp <8 x i16> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i16_v8f64 = sitofp <8 x i16> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i32_v8f32 = sitofp <8 x i32> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i32_v8f64 = sitofp <8 x i32> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i64_v8f32 = sitofp <8 x i64> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i64_v8f64 = sitofp <8 x i64> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i1_v8f32 = sitofp <8 x i1> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i1_v8f64 = sitofp <8 x i1> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi8_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi8_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi16_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi16_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi32_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi32_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi64_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi64_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi1_v8f32 = call <8 x float> @llvm.vp.sitofp.v8f32.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi1_v8f64 = call <8 x double> @llvm.vp.sitofp.v8f64.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i8_v16f32 = sitofp <16 x i8> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i8_v16f64 = sitofp <16 x i8> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i16_v16f32 = sitofp <16 x i16> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i16_v16f64 = sitofp <16 x i16> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i32_v16f32 = sitofp <16 x i32> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i32_v16f64 = sitofp <16 x i32> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i64_v16f32 = sitofp <16 x i64> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i64_v16f64 = sitofp <16 x i64> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i1_v16f32 = sitofp <16 x i1> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i1_v16f64 = sitofp <16 x i1> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi8_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi8_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi16_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi16_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi32_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi32_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi64_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi64_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi1_v16f32 = call <16 x float> @llvm.vp.sitofp.v16f32.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi1_v16f64 = call <16 x double> @llvm.vp.sitofp.v16f64.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i8_v32f32 = sitofp <32 x i8> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i8_v32f64 = sitofp <32 x i8> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i16_v32f32 = sitofp <32 x i16> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i16_v32f64 = sitofp <32 x i16> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i32_v32f32 = sitofp <32 x i32> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i32_v32f64 = sitofp <32 x i32> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %v32i64_v32f32 = sitofp <32 x i64> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %v32i64_v32f64 = sitofp <32 x i64> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i1_v32f32 = sitofp <32 x i1> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i1_v32f64 = sitofp <32 x i1> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi8_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi8_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi16_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi16_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi32_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi32_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %vp_v32fi64_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %vp_v32fi64_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi1_v32f32 = call <32 x float> @llvm.vp.sitofp.v32f32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi1_v32f64 = call <32 x double> @llvm.vp.sitofp.v32f64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i8_v64f32 = sitofp <64 x i8> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i8_v64f64 = sitofp <64 x i8> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i16_v64f32 = sitofp <64 x i16> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i16_v64f64 = sitofp <64 x i16> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %v64i32_v64f32 = sitofp <64 x i32> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %v64i32_v64f64 = sitofp <64 x i32> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %v64i64_v64f32 = sitofp <64 x i64> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %v64i64_v64f64 = sitofp <64 x i64> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i1_v64f32 = sitofp <64 x i1> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i1_v64f64 = sitofp <64 x i1> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi8_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi8_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi16_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi16_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %vp_v64fi32_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %vp_v64fi32_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %vp_v64fi64_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %vp_v64fi64_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi1_v64f32 = call <64 x float> @llvm.vp.sitofp.v64f32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi1_v64f64 = call <64 x double> @llvm.vp.sitofp.v64f64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v128i8_v128f32 = sitofp <128 x i8> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v128i8_v128f64 = sitofp <128 x i8> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %v128i16_v128f32 = sitofp <128 x i16> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %v128i16_v128f64 = sitofp <128 x i16> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %v128i32_v128f32 = sitofp <128 x i32> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %v128i32_v128f64 = sitofp <128 x i32> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %v128i64_v128f32 = sitofp <128 x i64> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %v128i64_v128f64 = sitofp <128 x i64> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v128i1_v128f32 = sitofp <128 x i1> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v128i1_v128f64 = sitofp <128 x i1> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v128fi8_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v128fi8_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %vp_v128fi16_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %vp_v128fi16_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %vp_v128fi32_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %vp_v128fi32_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %vp_v128fi64_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %vp_v128fi64_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v128fi1_v128f32 = call <128 x float> @llvm.vp.sitofp.v128f32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v128fi1_v128f64 = call <128 x double> @llvm.vp.sitofp.v128f64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i8_nxv1f32 = sitofp <vscale x 1 x i8> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i8_nxv1f64 = sitofp <vscale x 1 x i8> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i16_nxv1f32 = sitofp <vscale x 1 x i16> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i16_nxv1f64 = sitofp <vscale x 1 x i16> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i32_nxv1f32 = sitofp <vscale x 1 x i32> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i32_nxv1f64 = sitofp <vscale x 1 x i32> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i64_nxv1f32 = sitofp <vscale x 1 x i64> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i64_nxv1f64 = sitofp <vscale x 1 x i64> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i1_nxv1f32 = sitofp <vscale x 1 x i1> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i1_nxv1f64 = sitofp <vscale x 1 x i1> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi8_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi8_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi16_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi16_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi32_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi32_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi64_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi64_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi1_nxv1f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi1_nxv1f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i8_nxv2f32 = sitofp <vscale x 2 x i8> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i8_nxv2f64 = sitofp <vscale x 2 x i8> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i16_nxv2f32 = sitofp <vscale x 2 x i16> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i16_nxv2f64 = sitofp <vscale x 2 x i16> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i32_nxv2f32 = sitofp <vscale x 2 x i32> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i32_nxv2f64 = sitofp <vscale x 2 x i32> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i64_nxv2f32 = sitofp <vscale x 2 x i64> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i64_nxv2f64 = sitofp <vscale x 2 x i64> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i1_nxv2f32 = sitofp <vscale x 2 x i1> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i1_nxv2f64 = sitofp <vscale x 2 x i1> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi8_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi8_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi16_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi16_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi32_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi32_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi64_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi64_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi1_nxv2f32 = call <vscale x 2 x float> @llvm.vp.sitofp.nxv2f32.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi1_nxv2f64 = call <vscale x 2 x double> @llvm.vp.sitofp.nxv2f64.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i8_nxv4f32 = sitofp <vscale x 4 x i8> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i8_nxv4f64 = sitofp <vscale x 4 x i8> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i16_nxv4f32 = sitofp <vscale x 4 x i16> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i16_nxv4f64 = sitofp <vscale x 4 x i16> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i32_nxv4f32 = sitofp <vscale x 4 x i32> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i32_nxv4f64 = sitofp <vscale x 4 x i32> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i64_nxv4f32 = sitofp <vscale x 4 x i64> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i64_nxv4f64 = sitofp <vscale x 4 x i64> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i1_nxv4f32 = sitofp <vscale x 4 x i1> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i1_nxv4f64 = sitofp <vscale x 4 x i1> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi8_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi8_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi16_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi16_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi32_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi32_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi64_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi64_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi1_nxv4f32 = call <vscale x 4 x float> @llvm.vp.sitofp.nxv4f32.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi1_nxv4f64 = call <vscale x 4 x double> @llvm.vp.sitofp.nxv4f64.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i8_nxv8f32 = sitofp <vscale x 8 x i8> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i8_nxv8f64 = sitofp <vscale x 8 x i8> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i16_nxv8f32 = sitofp <vscale x 8 x i16> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i16_nxv8f64 = sitofp <vscale x 8 x i16> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i32_nxv8f32 = sitofp <vscale x 8 x i32> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i32_nxv8f64 = sitofp <vscale x 8 x i32> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i64_nxv8f32 = sitofp <vscale x 8 x i64> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i64_nxv8f64 = sitofp <vscale x 8 x i64> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i1_nxv8f32 = sitofp <vscale x 8 x i1> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i1_nxv8f64 = sitofp <vscale x 8 x i1> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi8_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi8_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi16_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi16_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi32_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi32_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi64_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi64_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi1_nxv8f32 = call <vscale x 1 x float> @llvm.vp.sitofp.nxv1f32.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi1_nxv8f64 = call <vscale x 1 x double> @llvm.vp.sitofp.nxv1f64.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i8_nxv16f32 = sitofp <vscale x 16 x i8> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i8_nxv16f64 = sitofp <vscale x 16 x i8> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i16_nxv16f32 = sitofp <vscale x 16 x i16> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i16_nxv16f64 = sitofp <vscale x 16 x i16> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i32_nxv16f32 = sitofp <vscale x 16 x i32> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i32_nxv16f64 = sitofp <vscale x 16 x i32> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i64_nxv16f32 = sitofp <vscale x 16 x i64> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i64_nxv16f64 = sitofp <vscale x 16 x i64> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i1_nxv16f32 = sitofp <vscale x 16 x i1> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i1_nxv16f64 = sitofp <vscale x 16 x i1> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi8_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi8_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi16_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi16_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi32_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi32_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi64_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi64_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi1_nxv16f32 = call <vscale x 16 x float> @llvm.vp.sitofp.nxv16f32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi1_nxv16f64 = call <vscale x 16 x double> @llvm.vp.sitofp.nxv16f64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i8_nxv32f32 = sitofp <vscale x 32 x i8> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i8_nxv32f64 = sitofp <vscale x 32 x i8> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i16_nxv32f32 = sitofp <vscale x 32 x i16> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i16_nxv32f64 = sitofp <vscale x 32 x i16> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i32_nxv32f32 = sitofp <vscale x 32 x i32> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i32_nxv32f64 = sitofp <vscale x 32 x i32> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i64_nxv32f32 = sitofp <vscale x 32 x i64> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i64_nxv32f64 = sitofp <vscale x 32 x i64> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i1_nxv32f32 = sitofp <vscale x 32 x i1> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i1_nxv32f64 = sitofp <vscale x 32 x i1> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi8_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi8_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi16_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi16_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi32_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi32_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi64_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi64_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi1_nxv32f32 = call <vscale x 32 x float> @llvm.vp.sitofp.nxv32f32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi1_nxv32f64 = call <vscale x 32 x double> @llvm.vp.sitofp.nxv32f64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64f32 = sitofp <vscale x 64 x i8> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64f64 = sitofp <vscale x 64 x i8> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i16_nxv64f32 = sitofp <vscale x 64 x i16> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i16_nxv64f64 = sitofp <vscale x 64 x i16> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i32_nxv64f32 = sitofp <vscale x 64 x i32> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i32_nxv64f64 = sitofp <vscale x 64 x i32> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i64_nxv64f32 = sitofp <vscale x 64 x i64> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i64_nxv64f64 = sitofp <vscale x 64 x i64> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64f32 = sitofp <vscale x 64 x i1> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64f64 = sitofp <vscale x 64 x i1> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi8_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi8_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi16_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi16_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi32_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi32_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi64_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi64_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi1_nxv64f32 = call <vscale x 64 x float> @llvm.vp.sitofp.nxv64f32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi1_nxv64f64 = call <vscale x 64 x double> @llvm.vp.sitofp.nxv64f64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8_v2f32 = sitofp <2 x i8> undef to <2 x float>
   %v2i8_v2f64 = sitofp <2 x i8> undef to <2 x double>
@@ -4212,288 +5236,571 @@ define void @sitofp() {
 }
 
 define void @uitofp() {
-; CHECK-LABEL: 'uitofp'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_v2f32 = uitofp <2 x i8> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_v2f64 = uitofp <2 x i8> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16_v2f32 = uitofp <2 x i16> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16_v2f64 = uitofp <2 x i16> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_v2f32 = uitofp <2 x i32> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_v2f64 = uitofp <2 x i32> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_v2f32 = uitofp <2 x i64> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_v2f64 = uitofp <2 x i64> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f32 = uitofp <2 x i1> undef to <2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f64 = uitofp <2 x i1> undef to <2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi8_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi8_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi16_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi16_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi32_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi32_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi64_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi64_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4f32 = uitofp <4 x i8> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_v4f64 = uitofp <4 x i8> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4f32 = uitofp <4 x i16> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i16_v4f64 = uitofp <4 x i16> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4f32 = uitofp <4 x i32> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4f64 = uitofp <4 x i32> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_v4f32 = uitofp <4 x i64> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_v4f64 = uitofp <4 x i64> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_v4f32 = uitofp <4 x i1> undef to <4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4f64 = uitofp <4 x i1> undef to <4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi8_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi8_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi16_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi16_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi32_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi32_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi64_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi64_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi1_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4fi1_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_v8f32 = uitofp <8 x i8> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i8_v8f64 = uitofp <8 x i8> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8f32 = uitofp <8 x i16> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i16_v8f64 = uitofp <8 x i16> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_v8f32 = uitofp <8 x i32> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8f64 = uitofp <8 x i32> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8f32 = uitofp <8 x i64> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_v8f64 = uitofp <8 x i64> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8f32 = uitofp <8 x i1> undef to <8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8f64 = uitofp <8 x i1> undef to <8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8fi8_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8fi8_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi16_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8fi16_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi32_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi32_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi64_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi64_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi1_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8fi1_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i8_v16f32 = uitofp <16 x i8> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_v16f64 = uitofp <16 x i8> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16f32 = uitofp <16 x i16> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i16_v16f64 = uitofp <16 x i16> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_v16f32 = uitofp <16 x i32> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16f64 = uitofp <16 x i32> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i64_v16f32 = uitofp <16 x i64> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_v16f64 = uitofp <16 x i64> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16f32 = uitofp <16 x i1> undef to <16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16f64 = uitofp <16 x i1> undef to <16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16fi8_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16fi8_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi16_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16fi16_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi32_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi32_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi64_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi64_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi1_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16fi1_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_v32f32 = uitofp <32 x i8> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i8_v32f64 = uitofp <32 x i8> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32f32 = uitofp <32 x i16> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i16_v32f64 = uitofp <32 x i16> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i32_v32f32 = uitofp <32 x i32> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32f64 = uitofp <32 x i32> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i64_v32f32 = uitofp <32 x i64> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i64_v32f64 = uitofp <32 x i64> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32f32 = uitofp <32 x i1> undef to <32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32f64 = uitofp <32 x i1> undef to <32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v32fi8_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32fi8_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32fi16_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32fi16_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32fi32_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32fi32_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32fi64_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32fi64_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32fi1_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32fi1_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64i8_v64f32 = uitofp <64 x i8> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64i8_v64f64 = uitofp <64 x i8> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64f32 = uitofp <64 x i16> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64i16_v64f64 = uitofp <64 x i16> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i32_v64f32 = uitofp <64 x i32> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64f64 = uitofp <64 x i32> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64i64_v64f32 = uitofp <64 x i64> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v64i64_v64f64 = uitofp <64 x i64> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64f32 = uitofp <64 x i1> undef to <64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64f64 = uitofp <64 x i1> undef to <64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64fi8_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64fi8_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64fi16_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64fi16_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64fi32_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64fi32_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64fi64_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v64fi64_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64fi1_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64fi1_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v128i8_v128f32 = uitofp <128 x i8> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %v128i8_v128f64 = uitofp <128 x i8> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128f32 = uitofp <128 x i16> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %v128i16_v128f64 = uitofp <128 x i16> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v128i32_v128f32 = uitofp <128 x i32> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128f64 = uitofp <128 x i32> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128i64_v128f32 = uitofp <128 x i64> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v128i64_v128f64 = uitofp <128 x i64> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128f32 = uitofp <128 x i1> undef to <128 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128f64 = uitofp <128 x i1> undef to <128 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v128fi8_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_v128fi8_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v128fi16_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_v128fi16_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v128fi32_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128fi32_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128fi64_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v128fi64_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128fi1_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128fi1_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i8_nxv1f32 = uitofp <vscale x 1 x i8> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i8_nxv1f64 = uitofp <vscale x 1 x i8> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16_nxv1f32 = uitofp <vscale x 1 x i16> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i16_nxv1f64 = uitofp <vscale x 1 x i16> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32_nxv1f32 = uitofp <vscale x 1 x i32> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32_nxv1f64 = uitofp <vscale x 1 x i32> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64_nxv1f32 = uitofp <vscale x 1 x i64> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64_nxv1f64 = uitofp <vscale x 1 x i64> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i1_nxv1f32 = uitofp <vscale x 1 x i1> undef to <vscale x 1 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i1_nxv1f64 = uitofp <vscale x 1 x i1> undef to <vscale x 1 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi8_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi8_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi16_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi16_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi32_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi32_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi64_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi64_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1fi1_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1fi1_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2f32 = uitofp <vscale x 2 x i8> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i8_nxv2f64 = uitofp <vscale x 2 x i8> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2f32 = uitofp <vscale x 2 x i16> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i16_nxv2f64 = uitofp <vscale x 2 x i16> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2f32 = uitofp <vscale x 2 x i32> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2f64 = uitofp <vscale x 2 x i32> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64_nxv2f32 = uitofp <vscale x 2 x i64> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64_nxv2f64 = uitofp <vscale x 2 x i64> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_nxv2f32 = uitofp <vscale x 2 x i1> undef to <vscale x 2 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2f64 = uitofp <vscale x 2 x i1> undef to <vscale x 2 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi8_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi8_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi16_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi16_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi32_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi32_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi64_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi64_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi1_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv2fi1_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i8_nxv4f32 = uitofp <vscale x 4 x i8> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i8_nxv4f64 = uitofp <vscale x 4 x i8> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4f32 = uitofp <vscale x 4 x i16> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i16_nxv4f64 = uitofp <vscale x 4 x i16> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32_nxv4f32 = uitofp <vscale x 4 x i32> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4f64 = uitofp <vscale x 4 x i32> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4f32 = uitofp <vscale x 4 x i64> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64_nxv4f64 = uitofp <vscale x 4 x i64> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4f32 = uitofp <vscale x 4 x i1> undef to <vscale x 4 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4f64 = uitofp <vscale x 4 x i1> undef to <vscale x 4 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4fi8_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4fi8_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi16_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4fi16_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi32_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi32_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi64_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi64_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi1_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv4fi1_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i8_nxv8f32 = uitofp <vscale x 8 x i8> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8i8_nxv8f64 = uitofp <vscale x 8 x i8> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8f32 = uitofp <vscale x 8 x i16> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8i16_nxv8f64 = uitofp <vscale x 8 x i16> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32_nxv8f32 = uitofp <vscale x 8 x i32> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8f64 = uitofp <vscale x 8 x i32> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i64_nxv8f32 = uitofp <vscale x 8 x i64> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64_nxv8f64 = uitofp <vscale x 8 x i64> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8f32 = uitofp <vscale x 8 x i1> undef to <vscale x 8 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8f64 = uitofp <vscale x 8 x i1> undef to <vscale x 8 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8fi8_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8fi8_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8fi16_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8fi16_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8fi32_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8fi32_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8fi64_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8fi64_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8fi1_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i1(<vscale x 8 x i1> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv8fi1_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i1(<vscale x 8 x i1> undef, <vscale x 8 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv16i8_nxv16f32 = uitofp <vscale x 16 x i8> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i8_nxv16f64 = uitofp <vscale x 16 x i8> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16f32 = uitofp <vscale x 16 x i16> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i16_nxv16f64 = uitofp <vscale x 16 x i16> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32_nxv16f32 = uitofp <vscale x 16 x i32> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16f64 = uitofp <vscale x 16 x i32> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i64_nxv16f32 = uitofp <vscale x 16 x i64> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i64_nxv16f64 = uitofp <vscale x 16 x i64> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16f32 = uitofp <vscale x 16 x i1> undef to <vscale x 16 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16f64 = uitofp <vscale x 16 x i1> undef to <vscale x 16 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv16fi8_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16fi8_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16fi16_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16fi16_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16fi32_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16fi32_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16fi64_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv16fi64_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv16fi1_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv16fi1_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32i8_nxv32f32 = uitofp <vscale x 32 x i8> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32i8_nxv32f64 = uitofp <vscale x 32 x i8> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32f32 = uitofp <vscale x 32 x i16> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32i16_nxv32f64 = uitofp <vscale x 32 x i16> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i32_nxv32f32 = uitofp <vscale x 32 x i32> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32f64 = uitofp <vscale x 32 x i32> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32i64_nxv32f32 = uitofp <vscale x 32 x i64> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i64_nxv32f64 = uitofp <vscale x 32 x i64> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32f32 = uitofp <vscale x 32 x i1> undef to <vscale x 32 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32f64 = uitofp <vscale x 32 x i1> undef to <vscale x 32 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32fi8_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32fi8_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32fi16_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32fi16_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv32fi32_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32fi32_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32fi64_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv32fi64_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv32fi1_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv32fi1_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv64i8_nxv64f32 = uitofp <vscale x 64 x i8> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %nxv64i8_nxv64f64 = uitofp <vscale x 64 x i8> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64f32 = uitofp <vscale x 64 x i16> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %nxv64i16_nxv64f64 = uitofp <vscale x 64 x i16> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv64i32_nxv64f32 = uitofp <vscale x 64 x i32> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64f64 = uitofp <vscale x 64 x i32> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64i64_nxv64f32 = uitofp <vscale x 64 x i64> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv64i64_nxv64f64 = uitofp <vscale x 64 x i64> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64f32 = uitofp <vscale x 64 x i1> undef to <vscale x 64 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64f64 = uitofp <vscale x 64 x i1> undef to <vscale x 64 x double>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv64fi8_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_nxv64fi8_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv64fi16_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_nxv64fi16_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv64fi32_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64fi32_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64fi64_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv64fi64_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv64fi1_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_nxv64fi1_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ZVE64D-LABEL: 'uitofp'
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_v2f32 = uitofp <2 x i8> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i8_v2f64 = uitofp <2 x i8> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16_v2f32 = uitofp <2 x i16> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2i16_v2f64 = uitofp <2 x i16> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_v2f32 = uitofp <2 x i32> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32_v2f64 = uitofp <2 x i32> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_v2f32 = uitofp <2 x i64> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64_v2f64 = uitofp <2 x i64> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f32 = uitofp <2 x i1> undef to <2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f64 = uitofp <2 x i1> undef to <2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi8_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi8_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi16_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v2fi16_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi32_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi32_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi64_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v2fi64_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i8_v4f32 = uitofp <4 x i8> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i8_v4f64 = uitofp <4 x i8> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16_v4f32 = uitofp <4 x i16> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i16_v4f64 = uitofp <4 x i16> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32_v4f32 = uitofp <4 x i32> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i32_v4f64 = uitofp <4 x i32> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64_v4f32 = uitofp <4 x i64> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64_v4f64 = uitofp <4 x i64> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v4i1_v4f32 = uitofp <4 x i1> undef to <4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v4i1_v4f64 = uitofp <4 x i1> undef to <4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi8_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi8_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi16_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi16_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi32_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi32_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_v4fi64_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v4fi64_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v4fi1_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v4fi1_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v8i8_v8f32 = uitofp <8 x i8> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i8_v8f64 = uitofp <8 x i8> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i16_v8f32 = uitofp <8 x i16> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v8i16_v8f64 = uitofp <8 x i16> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32_v8f32 = uitofp <8 x i32> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i32_v8f64 = uitofp <8 x i32> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i64_v8f32 = uitofp <8 x i64> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64_v8f64 = uitofp <8 x i64> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i1_v8f32 = uitofp <8 x i1> undef to <8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v8i1_v8f64 = uitofp <8 x i1> undef to <8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v8fi8_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8fi8_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi16_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v8fi16_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi32_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi32_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_v8fi64_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi64_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v8fi1_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v8fi1_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %v16i8_v16f32 = uitofp <16 x i8> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i8_v16f64 = uitofp <16 x i8> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i16_v16f32 = uitofp <16 x i16> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v16i16_v16f64 = uitofp <16 x i16> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32_v16f32 = uitofp <16 x i32> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i32_v16f64 = uitofp <16 x i32> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i64_v16f32 = uitofp <16 x i64> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i64_v16f64 = uitofp <16 x i64> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16i1_v16f32 = uitofp <16 x i1> undef to <16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v16i1_v16f64 = uitofp <16 x i1> undef to <16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_v16fi8_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16fi8_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi16_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v16fi16_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi32_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi32_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_v16fi64_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi64_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v16fi1_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v16fi1_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %v32i8_v32f32 = uitofp <32 x i8> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i8_v32f64 = uitofp <32 x i8> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i16_v32f32 = uitofp <32 x i16> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v32i16_v32f64 = uitofp <32 x i16> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v32i32_v32f32 = uitofp <32 x i32> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v32i32_v32f64 = uitofp <32 x i32> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v32i64_v32f32 = uitofp <32 x i64> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i64_v32f64 = uitofp <32 x i64> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v32i1_v32f32 = uitofp <32 x i1> undef to <32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v32i1_v32f64 = uitofp <32 x i1> undef to <32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_v32fi8_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32fi8_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32fi16_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v32fi16_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_v32fi32_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v32fi32_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_v32fi64_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32fi64_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v32fi1_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v32fi1_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %v64i8_v64f32 = uitofp <64 x i8> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64i8_v64f64 = uitofp <64 x i8> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %v64i16_v64f32 = uitofp <64 x i16> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v64i16_v64f64 = uitofp <64 x i16> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %v64i32_v64f32 = uitofp <64 x i32> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v64i32_v64f64 = uitofp <64 x i32> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %v64i64_v64f32 = uitofp <64 x i64> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v64i64_v64f64 = uitofp <64 x i64> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %v64i1_v64f32 = uitofp <64 x i1> undef to <64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v64i1_v64f64 = uitofp <64 x i1> undef to <64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_v64fi8_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64fi8_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_v64fi16_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v64fi16_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_v64fi32_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v64fi32_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_v64fi64_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v64fi64_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_v64fi1_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v64fi1_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %v128i8_v128f32 = uitofp <128 x i8> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %v128i8_v128f64 = uitofp <128 x i8> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %v128i16_v128f32 = uitofp <128 x i16> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %v128i16_v128f64 = uitofp <128 x i16> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %v128i32_v128f32 = uitofp <128 x i32> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %v128i32_v128f64 = uitofp <128 x i32> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %v128i64_v128f32 = uitofp <128 x i64> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %v128i64_v128f64 = uitofp <128 x i64> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %v128i1_v128f32 = uitofp <128 x i1> undef to <128 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %v128i1_v128f64 = uitofp <128 x i1> undef to <128 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_v128fi8_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_v128fi8_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_v128fi16_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_v128fi16_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %vp_v128fi32_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_v128fi32_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_v128fi64_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %vp_v128fi64_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_v128fi1_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_v128fi1_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i8_nxv1f32 = uitofp <vscale x 1 x i8> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i8_nxv1f64 = uitofp <vscale x 1 x i8> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16_nxv1f32 = uitofp <vscale x 1 x i16> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv1i16_nxv1f64 = uitofp <vscale x 1 x i16> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32_nxv1f32 = uitofp <vscale x 1 x i32> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32_nxv1f64 = uitofp <vscale x 1 x i32> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64_nxv1f32 = uitofp <vscale x 1 x i64> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64_nxv1f64 = uitofp <vscale x 1 x i64> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i1_nxv1f32 = uitofp <vscale x 1 x i1> undef to <vscale x 1 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv1i1_nxv1f64 = uitofp <vscale x 1 x i1> undef to <vscale x 1 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi8_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi8_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi16_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv1fi16_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi32_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi32_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi64_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv1fi64_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1fi1_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv1fi1_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i8_nxv2f32 = uitofp <vscale x 2 x i8> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i8_nxv2f64 = uitofp <vscale x 2 x i8> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16_nxv2f32 = uitofp <vscale x 2 x i16> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i16_nxv2f64 = uitofp <vscale x 2 x i16> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32_nxv2f32 = uitofp <vscale x 2 x i32> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i32_nxv2f64 = uitofp <vscale x 2 x i32> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64_nxv2f32 = uitofp <vscale x 2 x i64> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64_nxv2f64 = uitofp <vscale x 2 x i64> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv2i1_nxv2f32 = uitofp <vscale x 2 x i1> undef to <vscale x 2 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv2i1_nxv2f64 = uitofp <vscale x 2 x i1> undef to <vscale x 2 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi8_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi8_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi16_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi16_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi32_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi32_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %vp_nxv2fi64_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv2fi64_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv2fi1_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv2fi1_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %nxv4i8_nxv4f32 = uitofp <vscale x 4 x i8> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i8_nxv4f64 = uitofp <vscale x 4 x i8> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i16_nxv4f32 = uitofp <vscale x 4 x i16> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv4i16_nxv4f64 = uitofp <vscale x 4 x i16> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32_nxv4f32 = uitofp <vscale x 4 x i32> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i32_nxv4f64 = uitofp <vscale x 4 x i32> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i64_nxv4f32 = uitofp <vscale x 4 x i64> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64_nxv4f64 = uitofp <vscale x 4 x i64> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i1_nxv4f32 = uitofp <vscale x 4 x i1> undef to <vscale x 4 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv4i1_nxv4f64 = uitofp <vscale x 4 x i1> undef to <vscale x 4 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_nxv4fi8_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4fi8_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi16_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv4fi16_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi32_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi32_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv4fi64_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi64_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv4fi1_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv4fi1_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %nxv8i8_nxv8f32 = uitofp <vscale x 8 x i8> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8i8_nxv8f64 = uitofp <vscale x 8 x i8> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i16_nxv8f32 = uitofp <vscale x 8 x i16> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv8i16_nxv8f64 = uitofp <vscale x 8 x i16> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32_nxv8f32 = uitofp <vscale x 8 x i32> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i32_nxv8f64 = uitofp <vscale x 8 x i32> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i64_nxv8f32 = uitofp <vscale x 8 x i64> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64_nxv8f64 = uitofp <vscale x 8 x i64> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i1_nxv8f32 = uitofp <vscale x 8 x i1> undef to <vscale x 8 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv8i1_nxv8f64 = uitofp <vscale x 8 x i1> undef to <vscale x 8 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %vp_nxv8fi8_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8fi8_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8fi16_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv8fi16_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8fi32_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8fi32_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv8fi64_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8fi64_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv8fi1_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i1(<vscale x 8 x i1> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv8fi1_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i1(<vscale x 8 x i1> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %nxv16i8_nxv16f32 = uitofp <vscale x 16 x i8> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i8_nxv16f64 = uitofp <vscale x 16 x i8> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i16_nxv16f32 = uitofp <vscale x 16 x i16> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv16i16_nxv16f64 = uitofp <vscale x 16 x i16> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32_nxv16f32 = uitofp <vscale x 16 x i32> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv16i32_nxv16f64 = uitofp <vscale x 16 x i32> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %nxv16i64_nxv16f32 = uitofp <vscale x 16 x i64> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i64_nxv16f64 = uitofp <vscale x 16 x i64> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %nxv16i1_nxv16f32 = uitofp <vscale x 16 x i1> undef to <vscale x 16 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv16i1_nxv16f64 = uitofp <vscale x 16 x i1> undef to <vscale x 16 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %vp_nxv16fi8_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16fi8_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16fi16_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv16fi16_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv16fi32_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv16fi32_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %vp_nxv16fi64_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv16fi64_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %vp_nxv16fi1_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv16fi1_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %nxv32i8_nxv32f32 = uitofp <vscale x 32 x i8> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32i8_nxv32f64 = uitofp <vscale x 32 x i8> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %nxv32i16_nxv32f32 = uitofp <vscale x 32 x i16> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv32i16_nxv32f64 = uitofp <vscale x 32 x i16> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv32i32_nxv32f32 = uitofp <vscale x 32 x i32> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv32i32_nxv32f64 = uitofp <vscale x 32 x i32> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %nxv32i64_nxv32f32 = uitofp <vscale x 32 x i64> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i64_nxv32f64 = uitofp <vscale x 32 x i64> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %nxv32i1_nxv32f32 = uitofp <vscale x 32 x i1> undef to <vscale x 32 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv32i1_nxv32f64 = uitofp <vscale x 32 x i1> undef to <vscale x 32 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %vp_nxv32fi8_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32fi8_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %vp_nxv32fi16_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv32fi16_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %vp_nxv32fi32_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv32fi32_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %vp_nxv32fi64_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv32fi64_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %vp_nxv32fi1_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv32fi1_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %nxv64i8_nxv64f32 = uitofp <vscale x 64 x i8> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %nxv64i8_nxv64f64 = uitofp <vscale x 64 x i8> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %nxv64i16_nxv64f32 = uitofp <vscale x 64 x i16> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %nxv64i16_nxv64f64 = uitofp <vscale x 64 x i16> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv64i32_nxv64f32 = uitofp <vscale x 64 x i32> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %nxv64i32_nxv64f64 = uitofp <vscale x 64 x i32> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %nxv64i64_nxv64f32 = uitofp <vscale x 64 x i64> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv64i64_nxv64f64 = uitofp <vscale x 64 x i64> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %nxv64i1_nxv64f32 = uitofp <vscale x 64 x i1> undef to <vscale x 64 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %nxv64i1_nxv64f64 = uitofp <vscale x 64 x i1> undef to <vscale x 64 x double>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %vp_nxv64fi8_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %vp_nxv64fi8_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %vp_nxv64fi16_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %vp_nxv64fi16_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %vp_nxv64fi32_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %vp_nxv64fi32_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %vp_nxv64fi64_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %vp_nxv64fi64_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 67 for instruction: %vp_nxv64fi1_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 135 for instruction: %vp_nxv64fi1_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; ZVE64X-LABEL: 'uitofp'
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_v2f32 = uitofp <2 x i8> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i8_v2f64 = uitofp <2 x i8> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i16_v2f32 = uitofp <2 x i16> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i16_v2f64 = uitofp <2 x i16> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i32_v2f32 = uitofp <2 x i32> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i32_v2f64 = uitofp <2 x i32> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i64_v2f32 = uitofp <2 x i64> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i64_v2f64 = uitofp <2 x i64> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f32 = uitofp <2 x i1> undef to <2 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %v2i1_v2f64 = uitofp <2 x i1> undef to <2 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi8_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi8_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i8(<2 x i8> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi16_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi16_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i16(<2 x i16> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi32_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi32_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i32(<2 x i32> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi64_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi64_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i64(<2 x i64> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f32 = call <2 x float> @llvm.vp.uitofp.v2f32.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %vp_v2fi1_v2f64 = call <2 x double> @llvm.vp.uitofp.v2f64.v2i1(<2 x i1> undef, <2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i8_v4f32 = uitofp <4 x i8> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i8_v4f64 = uitofp <4 x i8> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i16_v4f32 = uitofp <4 x i16> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i16_v4f64 = uitofp <4 x i16> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i32_v4f32 = uitofp <4 x i32> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i32_v4f64 = uitofp <4 x i32> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i64_v4f32 = uitofp <4 x i64> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i64_v4f64 = uitofp <4 x i64> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_v4f32 = uitofp <4 x i1> undef to <4 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %v4i1_v4f64 = uitofp <4 x i1> undef to <4 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi8_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi8_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i8(<4 x i8> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi16_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi16_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i16(<4 x i16> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi32_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi32_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i32(<4 x i32> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi64_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi64_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i64(<4 x i64> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi1_v4f32 = call <4 x float> @llvm.vp.uitofp.v4f32.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %vp_v4fi1_v4f64 = call <4 x double> @llvm.vp.uitofp.v4f64.v4i1(<4 x i1> undef, <4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i8_v8f32 = uitofp <8 x i8> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i8_v8f64 = uitofp <8 x i8> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i16_v8f32 = uitofp <8 x i16> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i16_v8f64 = uitofp <8 x i16> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i32_v8f32 = uitofp <8 x i32> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i32_v8f64 = uitofp <8 x i32> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i64_v8f32 = uitofp <8 x i64> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i64_v8f64 = uitofp <8 x i64> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i1_v8f32 = uitofp <8 x i1> undef to <8 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %v8i1_v8f64 = uitofp <8 x i1> undef to <8 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi8_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi8_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i8(<8 x i8> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi16_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi16_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i16(<8 x i16> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi32_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi32_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i32(<8 x i32> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi64_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi64_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i64(<8 x i64> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi1_v8f32 = call <8 x float> @llvm.vp.uitofp.v8f32.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %vp_v8fi1_v8f64 = call <8 x double> @llvm.vp.uitofp.v8f64.v8i1(<8 x i1> undef, <8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i8_v16f32 = uitofp <16 x i8> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i8_v16f64 = uitofp <16 x i8> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i16_v16f32 = uitofp <16 x i16> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i16_v16f64 = uitofp <16 x i16> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i32_v16f32 = uitofp <16 x i32> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i32_v16f64 = uitofp <16 x i32> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i64_v16f32 = uitofp <16 x i64> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i64_v16f64 = uitofp <16 x i64> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i1_v16f32 = uitofp <16 x i1> undef to <16 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %v16i1_v16f64 = uitofp <16 x i1> undef to <16 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi8_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi8_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i8(<16 x i8> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi16_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi16_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i16(<16 x i16> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi32_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi32_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i32(<16 x i32> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi64_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi64_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i64(<16 x i64> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi1_v16f32 = call <16 x float> @llvm.vp.uitofp.v16f32.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %vp_v16fi1_v16f64 = call <16 x double> @llvm.vp.uitofp.v16f64.v16i1(<16 x i1> undef, <16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i8_v32f32 = uitofp <32 x i8> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i8_v32f64 = uitofp <32 x i8> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i16_v32f32 = uitofp <32 x i16> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i16_v32f64 = uitofp <32 x i16> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i32_v32f32 = uitofp <32 x i32> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i32_v32f64 = uitofp <32 x i32> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %v32i64_v32f32 = uitofp <32 x i64> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %v32i64_v32f64 = uitofp <32 x i64> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i1_v32f32 = uitofp <32 x i1> undef to <32 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %v32i1_v32f64 = uitofp <32 x i1> undef to <32 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi8_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi8_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i8(<32 x i8> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi16_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi16_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i16(<32 x i16> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi32_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi32_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i32(<32 x i32> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %vp_v32fi64_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %vp_v32fi64_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i64(<32 x i64> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi1_v32f32 = call <32 x float> @llvm.vp.uitofp.v32f32.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %vp_v32fi1_v32f64 = call <32 x double> @llvm.vp.uitofp.v32f64.v32i1(<32 x i1> undef, <32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i8_v64f32 = uitofp <64 x i8> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i8_v64f64 = uitofp <64 x i8> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i16_v64f32 = uitofp <64 x i16> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i16_v64f64 = uitofp <64 x i16> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %v64i32_v64f32 = uitofp <64 x i32> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %v64i32_v64f64 = uitofp <64 x i32> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %v64i64_v64f32 = uitofp <64 x i64> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %v64i64_v64f64 = uitofp <64 x i64> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i1_v64f32 = uitofp <64 x i1> undef to <64 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %v64i1_v64f64 = uitofp <64 x i1> undef to <64 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi8_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi8_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i8(<64 x i8> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi16_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi16_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i16(<64 x i16> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %vp_v64fi32_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %vp_v64fi32_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i32(<64 x i32> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %vp_v64fi64_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %vp_v64fi64_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i64(<64 x i64> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi1_v64f32 = call <64 x float> @llvm.vp.uitofp.v64f32.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 127 for instruction: %vp_v64fi1_v64f64 = call <64 x double> @llvm.vp.uitofp.v64f64.v64i1(<64 x i1> undef, <64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v128i8_v128f32 = uitofp <128 x i8> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v128i8_v128f64 = uitofp <128 x i8> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %v128i16_v128f32 = uitofp <128 x i16> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %v128i16_v128f64 = uitofp <128 x i16> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %v128i32_v128f32 = uitofp <128 x i32> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %v128i32_v128f64 = uitofp <128 x i32> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %v128i64_v128f32 = uitofp <128 x i64> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %v128i64_v128f64 = uitofp <128 x i64> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v128i1_v128f32 = uitofp <128 x i1> undef to <128 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %v128i1_v128f64 = uitofp <128 x i1> undef to <128 x double>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v128fi8_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v128fi8_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i8(<128 x i8> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %vp_v128fi16_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 254 for instruction: %vp_v128fi16_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i16(<128 x i16> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %vp_v128fi32_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 252 for instruction: %vp_v128fi32_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i32(<128 x i32> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %vp_v128fi64_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 248 for instruction: %vp_v128fi64_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i64(<128 x i64> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v128fi1_v128f32 = call <128 x float> @llvm.vp.uitofp.v128f32.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 255 for instruction: %vp_v128fi1_v128f64 = call <128 x double> @llvm.vp.uitofp.v128f64.v128i1(<128 x i1> undef, <128 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i8_nxv1f32 = uitofp <vscale x 1 x i8> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i8_nxv1f64 = uitofp <vscale x 1 x i8> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i16_nxv1f32 = uitofp <vscale x 1 x i16> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i16_nxv1f64 = uitofp <vscale x 1 x i16> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i32_nxv1f32 = uitofp <vscale x 1 x i32> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i32_nxv1f64 = uitofp <vscale x 1 x i32> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i64_nxv1f32 = uitofp <vscale x 1 x i64> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i64_nxv1f64 = uitofp <vscale x 1 x i64> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i1_nxv1f32 = uitofp <vscale x 1 x i1> undef to <vscale x 1 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv1i1_nxv1f64 = uitofp <vscale x 1 x i1> undef to <vscale x 1 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi8_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi8_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i8(<vscale x 1 x i8> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi16_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi16_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i16(<vscale x 1 x i16> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi32_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi32_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i32(<vscale x 1 x i32> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi64_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi64_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i64(<vscale x 1 x i64> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi1_nxv1f32 = call <vscale x 1 x float> @llvm.vp.uitofp.nxv1f32.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv1fi1_nxv1f64 = call <vscale x 1 x double> @llvm.vp.uitofp.nxv1f64.nxv1i1(<vscale x 1 x i1> undef, <vscale x 1 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i8_nxv2f32 = uitofp <vscale x 2 x i8> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i8_nxv2f64 = uitofp <vscale x 2 x i8> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i16_nxv2f32 = uitofp <vscale x 2 x i16> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i16_nxv2f64 = uitofp <vscale x 2 x i16> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i32_nxv2f32 = uitofp <vscale x 2 x i32> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i32_nxv2f64 = uitofp <vscale x 2 x i32> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i64_nxv2f32 = uitofp <vscale x 2 x i64> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i64_nxv2f64 = uitofp <vscale x 2 x i64> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i1_nxv2f32 = uitofp <vscale x 2 x i1> undef to <vscale x 2 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv2i1_nxv2f64 = uitofp <vscale x 2 x i1> undef to <vscale x 2 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi8_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi8_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i8(<vscale x 2 x i8> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi16_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi16_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i16(<vscale x 2 x i16> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi32_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi32_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i32(<vscale x 2 x i32> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi64_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi64_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi1_nxv2f32 = call <vscale x 2 x float> @llvm.vp.uitofp.nxv2f32.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv2fi1_nxv2f64 = call <vscale x 2 x double> @llvm.vp.uitofp.nxv2f64.nxv2i1(<vscale x 2 x i1> undef, <vscale x 2 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i8_nxv4f32 = uitofp <vscale x 4 x i8> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i8_nxv4f64 = uitofp <vscale x 4 x i8> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i16_nxv4f32 = uitofp <vscale x 4 x i16> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i16_nxv4f64 = uitofp <vscale x 4 x i16> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i32_nxv4f32 = uitofp <vscale x 4 x i32> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i32_nxv4f64 = uitofp <vscale x 4 x i32> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i64_nxv4f32 = uitofp <vscale x 4 x i64> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i64_nxv4f64 = uitofp <vscale x 4 x i64> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i1_nxv4f32 = uitofp <vscale x 4 x i1> undef to <vscale x 4 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv4i1_nxv4f64 = uitofp <vscale x 4 x i1> undef to <vscale x 4 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi8_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi8_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i8(<vscale x 4 x i8> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi16_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi16_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i16(<vscale x 4 x i16> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi32_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi32_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi64_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi64_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i64(<vscale x 4 x i64> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi1_nxv4f32 = call <vscale x 4 x float> @llvm.vp.uitofp.nxv4f32.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv4fi1_nxv4f64 = call <vscale x 4 x double> @llvm.vp.uitofp.nxv4f64.nxv4i1(<vscale x 4 x i1> undef, <vscale x 4 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i8_nxv8f32 = uitofp <vscale x 8 x i8> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i8_nxv8f64 = uitofp <vscale x 8 x i8> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i16_nxv8f32 = uitofp <vscale x 8 x i16> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i16_nxv8f64 = uitofp <vscale x 8 x i16> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i32_nxv8f32 = uitofp <vscale x 8 x i32> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i32_nxv8f64 = uitofp <vscale x 8 x i32> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i64_nxv8f32 = uitofp <vscale x 8 x i64> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i64_nxv8f64 = uitofp <vscale x 8 x i64> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i1_nxv8f32 = uitofp <vscale x 8 x i1> undef to <vscale x 8 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv8i1_nxv8f64 = uitofp <vscale x 8 x i1> undef to <vscale x 8 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi8_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi8_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i8(<vscale x 8 x i8> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi16_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi16_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i16(<vscale x 8 x i16> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi32_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi32_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i32(<vscale x 8 x i32> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi64_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi64_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i64(<vscale x 8 x i64> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi1_nxv8f32 = call <vscale x 8 x float> @llvm.vp.uitofp.nxv8f32.nxv8i1(<vscale x 8 x i1> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv8fi1_nxv8f64 = call <vscale x 8 x double> @llvm.vp.uitofp.nxv8f64.nxv8i1(<vscale x 8 x i1> undef, <vscale x 8 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i8_nxv16f32 = uitofp <vscale x 16 x i8> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i8_nxv16f64 = uitofp <vscale x 16 x i8> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i16_nxv16f32 = uitofp <vscale x 16 x i16> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i16_nxv16f64 = uitofp <vscale x 16 x i16> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i32_nxv16f32 = uitofp <vscale x 16 x i32> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i32_nxv16f64 = uitofp <vscale x 16 x i32> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i64_nxv16f32 = uitofp <vscale x 16 x i64> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i64_nxv16f64 = uitofp <vscale x 16 x i64> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i1_nxv16f32 = uitofp <vscale x 16 x i1> undef to <vscale x 16 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv16i1_nxv16f64 = uitofp <vscale x 16 x i1> undef to <vscale x 16 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi8_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi8_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i8(<vscale x 16 x i8> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi16_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi16_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i16(<vscale x 16 x i16> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi32_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi32_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i32(<vscale x 16 x i32> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi64_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi64_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i64(<vscale x 16 x i64> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi1_nxv16f32 = call <vscale x 16 x float> @llvm.vp.uitofp.nxv16f32.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv16fi1_nxv16f64 = call <vscale x 16 x double> @llvm.vp.uitofp.nxv16f64.nxv16i1(<vscale x 16 x i1> undef, <vscale x 16 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i8_nxv32f32 = uitofp <vscale x 32 x i8> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i8_nxv32f64 = uitofp <vscale x 32 x i8> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i16_nxv32f32 = uitofp <vscale x 32 x i16> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i16_nxv32f64 = uitofp <vscale x 32 x i16> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i32_nxv32f32 = uitofp <vscale x 32 x i32> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i32_nxv32f64 = uitofp <vscale x 32 x i32> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i64_nxv32f32 = uitofp <vscale x 32 x i64> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i64_nxv32f64 = uitofp <vscale x 32 x i64> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i1_nxv32f32 = uitofp <vscale x 32 x i1> undef to <vscale x 32 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv32i1_nxv32f64 = uitofp <vscale x 32 x i1> undef to <vscale x 32 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi8_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi8_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i8(<vscale x 32 x i8> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi16_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi16_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i16(<vscale x 32 x i16> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi32_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi32_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i32(<vscale x 32 x i32> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi64_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi64_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i64(<vscale x 32 x i64> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi1_nxv32f32 = call <vscale x 32 x float> @llvm.vp.uitofp.nxv32f32.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv32fi1_nxv32f64 = call <vscale x 32 x double> @llvm.vp.uitofp.nxv32f64.nxv32i1(<vscale x 32 x i1> undef, <vscale x 32 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64f32 = uitofp <vscale x 64 x i8> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i8_nxv64f64 = uitofp <vscale x 64 x i8> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i16_nxv64f32 = uitofp <vscale x 64 x i16> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i16_nxv64f64 = uitofp <vscale x 64 x i16> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i32_nxv64f32 = uitofp <vscale x 64 x i32> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i32_nxv64f64 = uitofp <vscale x 64 x i32> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i64_nxv64f32 = uitofp <vscale x 64 x i64> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i64_nxv64f64 = uitofp <vscale x 64 x i64> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64f32 = uitofp <vscale x 64 x i1> undef to <vscale x 64 x float>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %nxv64i1_nxv64f64 = uitofp <vscale x 64 x i1> undef to <vscale x 64 x double>
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi8_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi8_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i8(<vscale x 64 x i8> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi16_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi16_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i16(<vscale x 64 x i16> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi32_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi32_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i32(<vscale x 64 x i32> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi64_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi64_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i64(<vscale x 64 x i64> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi1_nxv64f32 = call <vscale x 64 x float> @llvm.vp.uitofp.nxv64f32.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Invalid cost for instruction: %vp_nxv64fi1_nxv64f64 = call <vscale x 64 x double> @llvm.vp.uitofp.nxv64f64.nxv64i1(<vscale x 64 x i1> undef, <vscale x 64 x i1> undef, i32 undef)
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8_v2f32 = uitofp <2 x i8> undef to <2 x float>
   %v2i8_v2f64 = uitofp <2 x i8> undef to <2 x double>
@@ -4807,32 +6114,59 @@ define void @uitofp() {
 }
 
 define void @oddvec_sizes() {
-; CHECK-LABEL: 'oddvec_sizes'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sext <3 x i8> undef to <3 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = sext <7 x i8> undef to <7 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = sext <15 x i8> undef to <15 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = zext <3 x i8> undef to <3 x i16>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = zext <7 x i8> undef to <7 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %6 = zext <15 x i8> undef to <15 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %7 = trunc <3 x i32> undef to <3 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = trunc <7 x i32> undef to <7 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %9 = trunc <15 x i32> undef to <15 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %10 = bitcast <3 x i32> undef to <3 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %11 = bitcast <7 x i32> undef to <7 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %12 = bitcast <15 x i32> undef to <15 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = sitofp <3 x i32> undef to <3 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = sitofp <7 x i32> undef to <7 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %15 = sitofp <15 x i32> undef to <15 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = uitofp <3 x i32> undef to <3 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = uitofp <7 x i32> undef to <7 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = uitofp <15 x i32> undef to <15 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = fptosi <3 x float> undef to <3 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = fptosi <7 x float> undef to <7 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = fptosi <15 x float> undef to <15 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = fptoui <3 x float> undef to <3 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %23 = fptoui <7 x float> undef to <7 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %24 = fptoui <15 x float> undef to <15 x i32>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ZVE64D-LABEL: 'oddvec_sizes'
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sext <3 x i8> undef to <3 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = sext <7 x i8> undef to <7 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = sext <15 x i8> undef to <15 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = zext <3 x i8> undef to <3 x i16>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = zext <7 x i8> undef to <7 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %6 = zext <15 x i8> undef to <15 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %7 = trunc <3 x i32> undef to <3 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = trunc <7 x i32> undef to <7 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %9 = trunc <15 x i32> undef to <15 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %10 = bitcast <3 x i32> undef to <3 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %11 = bitcast <7 x i32> undef to <7 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %12 = bitcast <15 x i32> undef to <15 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %13 = sitofp <3 x i32> undef to <3 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %14 = sitofp <7 x i32> undef to <7 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %15 = sitofp <15 x i32> undef to <15 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %16 = uitofp <3 x i32> undef to <3 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %17 = uitofp <7 x i32> undef to <7 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %18 = uitofp <15 x i32> undef to <15 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %19 = fptosi <3 x float> undef to <3 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %20 = fptosi <7 x float> undef to <7 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %21 = fptosi <15 x float> undef to <15 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %22 = fptoui <3 x float> undef to <3 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %23 = fptoui <7 x float> undef to <7 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %24 = fptoui <15 x float> undef to <15 x i32>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; ZVE64X-LABEL: 'oddvec_sizes'
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %1 = sext <3 x i8> undef to <3 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %2 = sext <7 x i8> undef to <7 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %3 = sext <15 x i8> undef to <15 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %4 = zext <3 x i8> undef to <3 x i16>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %5 = zext <7 x i8> undef to <7 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %6 = zext <15 x i8> undef to <15 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %7 = trunc <3 x i32> undef to <3 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %8 = trunc <7 x i32> undef to <7 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %9 = trunc <15 x i32> undef to <15 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %10 = bitcast <3 x i32> undef to <3 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %11 = bitcast <7 x i32> undef to <7 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %12 = bitcast <15 x i32> undef to <15 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %13 = sitofp <3 x i32> undef to <3 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %14 = sitofp <7 x i32> undef to <7 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %15 = sitofp <15 x i32> undef to <15 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %16 = uitofp <3 x i32> undef to <3 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %17 = uitofp <7 x i32> undef to <7 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %18 = uitofp <15 x i32> undef to <15 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %19 = fptosi <3 x float> undef to <3 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %20 = fptosi <7 x float> undef to <7 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %21 = fptosi <15 x float> undef to <15 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %22 = fptoui <3 x float> undef to <3 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %23 = fptoui <7 x float> undef to <7 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %24 = fptoui <15 x float> undef to <15 x i32>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   sext <3 x i8> undef to <3 x i16>
   sext <7 x i8> undef to <7 x i32>
@@ -4873,16 +6207,27 @@ define void @oddvec_sizes() {
 ; vector splitting.  We previously crashed on this case due to an
 ; infinite recursion between cast costing and scalarization costing.
 define void @legalization_crash() {
-; CHECK-LABEL: 'legalization_crash'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16700 for instruction: %1 = bitcast <24 x i8> undef to <192 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %2 = trunc <192 x i8> undef to <192 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %3 = zext <192 x i1> undef to <192 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %4 = sext <192 x i1> undef to <192 x i8>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 948 for instruction: %5 = sitofp <192 x i1> undef to <192 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 948 for instruction: %6 = uitofp <192 x i1> undef to <192 x float>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16892 for instruction: %7 = fptosi <192 x float> undef to <192 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 16892 for instruction: %8 = fptoui <192 x float> undef to <192 x i1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; ZVE64D-LABEL: 'legalization_crash'
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16700 for instruction: %1 = bitcast <24 x i8> undef to <192 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %2 = trunc <192 x i8> undef to <192 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %3 = zext <192 x i1> undef to <192 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %4 = sext <192 x i1> undef to <192 x i8>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 948 for instruction: %5 = sitofp <192 x i1> undef to <192 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 948 for instruction: %6 = uitofp <192 x i1> undef to <192 x float>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16892 for instruction: %7 = fptosi <192 x float> undef to <192 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 16892 for instruction: %8 = fptoui <192 x float> undef to <192 x i1>
+; ZVE64D-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; ZVE64X-LABEL: 'legalization_crash'
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16700 for instruction: %1 = bitcast <24 x i8> undef to <192 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %2 = trunc <192 x i8> undef to <192 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %3 = zext <192 x i1> undef to <192 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %4 = sext <192 x i1> undef to <192 x i8>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %5 = sitofp <192 x i1> undef to <192 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %6 = uitofp <192 x i1> undef to <192 x float>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16892 for instruction: %7 = fptosi <192 x float> undef to <192 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 16892 for instruction: %8 = fptoui <192 x float> undef to <192 x i1>
+; ZVE64X-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   bitcast <24 x i8> undef to <192 x i1>
   trunc <192 x i8> undef to <192 x i1>
@@ -4894,6 +6239,3 @@ define void @legalization_crash() {
   fptoui <192 x float> undef to <192 x i1>
   ret void
 }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; RV32: {{.*}}
-; RV64: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll b/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll
index ecc7fc8fdb98a..245e8f70df8d6 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/intrinsic-cost-crash.ll
@@ -23,7 +23,7 @@
 %"class.llvm::Metadata.306.1758.9986.10470.10954.11438.11922.12406.12890.13374.13858.15310.15794.16278.17730.19182.21118.25958.26926.29346.29830.30314.30798.31282.31766.32250.32734.33702.36606.38058.41638" = type { i8, i8, i16, i32 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end(i64, ptr nocapture) #0
+declare void @llvm.lifetime.end(ptr nocapture) #0
 
 ; Function Attrs: nounwind ssp uwtable
 define hidden void @fun(ptr %N, i1 %arg) #1 align 2 {
@@ -42,7 +42,6 @@ for.cond.cleanup:                                 ; preds = %for.body, %entry
 
 for.body:                                         ; preds = %for.body, %for.body.lr.ph
   %indvars.iv190 = phi i64 [ 0, %for.body.lr.ph ], [ %indvars.iv.next191, %for.body ]
-  call void @llvm.lifetime.end(i64 16, ptr nonnull null)
   %indvars.iv.next191 = add nuw nsw i64 %indvars.iv190, 1
   %exitcond193 = icmp eq i64 %indvars.iv.next191, %wide.trip.count192
   br i1 %exitcond193, label %for.cond.cleanup, label %for.body
diff --git a/llvm/test/Analysis/CostModel/X86/free-intrinsics.ll b/llvm/test/Analysis/CostModel/X86/free-intrinsics.ll
index 3a54428bd8291..cef960d3bcd30 100644
--- a/llvm/test/Analysis/CostModel/X86/free-intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/X86/free-intrinsics.ll
@@ -14,8 +14,8 @@ define i32 @trivially_free() {
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a2 = call ptr @llvm.launder.invariant.group.p0(ptr undef)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a3 = call ptr @llvm.strip.invariant.group.p0(ptr undef)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a4 = call i1 @llvm.is.constant.i32(i32 undef)
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(i64 1, ptr %alloca)
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(ptr %alloca)
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(ptr %alloca)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.var.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
@@ -32,8 +32,8 @@ define i32 @trivially_free() {
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a2 = call ptr @llvm.launder.invariant.group.p0(ptr undef)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a3 = call ptr @llvm.strip.invariant.group.p0(ptr undef)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a4 = call i1 @llvm.is.constant.i32(i32 undef)
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(i64 1, ptr %alloca)
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(ptr %alloca)
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(ptr %alloca)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.var.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
@@ -49,8 +49,8 @@ define i32 @trivially_free() {
   %a2 = call ptr @llvm.launder.invariant.group.p0(ptr undef)
   %a3 = call ptr @llvm.strip.invariant.group.p0(ptr undef)
   %a4 = call i1 @llvm.is.constant.i32(i32 undef)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %alloca)
+  call void @llvm.lifetime.start.p0(ptr %alloca)
+  call void @llvm.lifetime.end.p0(ptr %alloca)
   %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 1, i1 1, i1 1)
   %a6 = call ptr @llvm.ptr.annotation.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
   call void @llvm.var.annotation(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
@@ -66,8 +66,8 @@ declare void @llvm.invariant.end.p0(ptr, i64, ptr)
 declare ptr @llvm.launder.invariant.group.p0(ptr)
 declare ptr @llvm.strip.invariant.group.p0(ptr)
 declare i1 @llvm.is.constant.i32(i32)
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 declare i64 @llvm.objectsize.i64.p0(ptr, i1, i1, i1)
 declare ptr @llvm.ptr.annotation.p0(ptr, ptr, ptr, i32, ptr)
 declare void @llvm.var.annotation(ptr, ptr, ptr, i32, ptr)
diff --git a/llvm/test/Analysis/CostModel/free-intrinsics-datalayout.ll b/llvm/test/Analysis/CostModel/free-intrinsics-datalayout.ll
index 96064dc3af60b..2acc8e8cf4e20 100644
--- a/llvm/test/Analysis/CostModel/free-intrinsics-datalayout.ll
+++ b/llvm/test/Analysis/CostModel/free-intrinsics-datalayout.ll
@@ -16,8 +16,8 @@ define i32 @trivially_free() {
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a2 = call ptr @llvm.launder.invariant.group.p0(ptr undef)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a3 = call ptr @llvm.strip.invariant.group.p0(ptr undef)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a4 = call i1 @llvm.is.constant.i32(i32 undef)
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(i64 1, ptr %alloca)
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(ptr %alloca)
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(ptr %alloca)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a7 = call i1 @llvm.allow.ubsan.check(i8 123)
@@ -36,8 +36,8 @@ define i32 @trivially_free() {
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a2 = call ptr @llvm.launder.invariant.group.p0(ptr undef)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a3 = call ptr @llvm.strip.invariant.group.p0(ptr undef)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a4 = call i1 @llvm.is.constant.i32(i32 undef)
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(i64 1, ptr %alloca)
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(ptr %alloca)
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(ptr %alloca)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a7 = call i1 @llvm.allow.ubsan.check(i8 123)
@@ -55,8 +55,8 @@ define i32 @trivially_free() {
   %a2 = call ptr @llvm.launder.invariant.group.p0(ptr undef)
   %a3 = call ptr @llvm.strip.invariant.group.p0(ptr undef)
   %a4 = call i1 @llvm.is.constant.i32(i32 undef)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %alloca)
+  call void @llvm.lifetime.start.p0(ptr %alloca)
+  call void @llvm.lifetime.end.p0(ptr %alloca)
   %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 1, i1 1, i1 1)
   %a6 = call ptr @llvm.ptr.annotation.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
   %a7 = call i1 @llvm.allow.ubsan.check(i8 123)
@@ -74,8 +74,8 @@ declare void @llvm.invariant.end.p0(ptr, i64, ptr)
 declare ptr @llvm.launder.invariant.group.p0(ptr)
 declare ptr @llvm.strip.invariant.group.p0(ptr)
 declare i1 @llvm.is.constant.i32(i32)
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 declare i64 @llvm.objectsize.i64.p0(ptr, i1, i1, i1)
 declare ptr @llvm.ptr.annotation.p0(ptr, ptr, ptr, i32, ptr)
 declare void @llvm.var.annotation(ptr, ptr, ptr, i32, ptr)
diff --git a/llvm/test/Analysis/CostModel/free-intrinsics-no_info.ll b/llvm/test/Analysis/CostModel/free-intrinsics-no_info.ll
index f989ebe592a11..7f002d0ee88e4 100644
--- a/llvm/test/Analysis/CostModel/free-intrinsics-no_info.ll
+++ b/llvm/test/Analysis/CostModel/free-intrinsics-no_info.ll
@@ -14,8 +14,8 @@ define i32 @trivially_free() {
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a2 = call ptr @llvm.launder.invariant.group.p0(ptr undef)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a3 = call ptr @llvm.strip.invariant.group.p0(ptr undef)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a4 = call i1 @llvm.is.constant.i32(i32 undef)
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
-; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(i64 1, ptr %alloca)
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(ptr %alloca)
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(ptr %alloca)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.var.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
@@ -34,8 +34,8 @@ define i32 @trivially_free() {
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a2 = call ptr @llvm.launder.invariant.group.p0(ptr undef)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a3 = call ptr @llvm.strip.invariant.group.p0(ptr undef)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a4 = call i1 @llvm.is.constant.i32(i32 undef)
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
-; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(i64 1, ptr %alloca)
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.start.p0(ptr %alloca)
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.lifetime.end.p0(ptr %alloca)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 true, i1 true, i1 true)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %a6 = call ptr @llvm.ptr.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: call void @llvm.var.annotation.p0.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
@@ -53,8 +53,8 @@ define i32 @trivially_free() {
   %a2 = call ptr @llvm.launder.invariant.group.p0(ptr undef)
   %a3 = call ptr @llvm.strip.invariant.group.p0(ptr undef)
   %a4 = call i1 @llvm.is.constant.i32(i32 undef)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %alloca)
+  call void @llvm.lifetime.start.p0(ptr %alloca)
+  call void @llvm.lifetime.end.p0(ptr %alloca)
   %a5 = call i64 @llvm.objectsize.i64.p0(ptr undef, i1 1, i1 1, i1 1)
   %a6 = call ptr @llvm.ptr.annotation.p0(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
   call void @llvm.var.annotation(ptr undef, ptr undef, ptr undef, i32 undef, ptr undef)
@@ -72,8 +72,8 @@ declare void @llvm.invariant.end.p0(ptr, i64, ptr)
 declare ptr @llvm.launder.invariant.group.p0(ptr)
 declare ptr @llvm.strip.invariant.group.p0(ptr)
 declare i1 @llvm.is.constant.i32(i32)
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 declare i64 @llvm.objectsize.i64.p0(ptr, i1, i1, i1)
 declare ptr @llvm.ptr.annotation.p0(ptr, ptr, ptr, i32, ptr)
 declare void @llvm.var.annotation(ptr, ptr, ptr, i32, ptr)
diff --git a/llvm/test/Analysis/DDG/basic-loopnest.ll b/llvm/test/Analysis/DDG/basic-loopnest.ll
index 325428c13b217..75efff570048b 100644
--- a/llvm/test/Analysis/DDG/basic-loopnest.ll
+++ b/llvm/test/Analysis/DDG/basic-loopnest.ll
@@ -1,5 +1,8 @@
 ; RUN: opt < %s -disable-output "-passes=print<ddg>" 2>&1 | FileCheck %s
 
+; XFAIL: *
+; At the moment, DependenceAnalysis cannot infer `n` to be positive.
+
 
 ; CHECK-LABEL: 'DDG' for loop 'test1.for.cond1.preheader':
 
@@ -378,4 +381,4 @@ for.inc12:                                        ; preds = %for.body4, %test2.f
 
 for.end14:                                        ; preds = %for.inc12, %entry
   ret void
-}
\ No newline at end of file
+}
diff --git a/llvm/test/Analysis/DXILResource/buffer-frombinding.ll b/llvm/test/Analysis/DXILResource/buffer-frombinding.ll
index 2623e6f4d44f1..d08b68d3768af 100644
--- a/llvm/test/Analysis/DXILResource/buffer-frombinding.ll
+++ b/llvm/test/Analysis/DXILResource/buffer-frombinding.ll
@@ -146,7 +146,7 @@ define void @test_typedbuffer() {
   ; CHECK:     Space: 1
   ; CHECK:     Lower Bound: 0
   ; CHECK:     Size: 1
-  ; CHECK:   Class: CBuffer
+  ; CHECK:   Class: CBV
   ; CHECK:   Kind: CBuffer
   ; CHECK:   CBuffer size: 4
 
@@ -159,7 +159,7 @@ define void @test_typedbuffer() {
   ; CHECK:     Space: 1
   ; CHECK:     Lower Bound: 8
   ; CHECK:     Size: 1
-  ; CHECK:   Class: CBuffer
+  ; CHECK:   Class: CBV
   ; CHECK:   Kind: CBuffer
   ; CHECK:   CBuffer size: 4
 
diff --git a/llvm/test/Analysis/Delinearization/a.ll b/llvm/test/Analysis/Delinearization/a.ll
index 02ba3fde695c3..755c9baef9b8f 100644
--- a/llvm/test/Analysis/Delinearization/a.ll
+++ b/llvm/test/Analysis/Delinearization/a.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 ;
 ; void foo(long n, long m, long o, int A[n][m][o]) {
@@ -7,12 +8,15 @@
 ;         A[2*i+3][3*j-4][5*k+7] = 1;
 ; }
 
-; AddRec: {{{(28 + (4 * (-4 + (3 * %m)) * %o) + %A),+,(8 * %m * %o)}<%for.i>,+,(12 * %o)}<%for.j>,+,20}<%for.k>
-; CHECK: Base offset: %A
-; CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of 4 bytes.
-; CHECK: ArrayRef[{3,+,2}<nuw><%for.i>][{-4,+,3}<nw><%for.j>][{7,+,5}<nw><%for.k>]
-
 define void @foo(i64 %n, i64 %m, i64 %o, ptr nocapture %A) #0 {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store i32 1, ptr %arrayidx11.us.us, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}(28 + (4 * (-4 + (3 * %m)) * %o)),+,(8 * %m * %o)}<%for.i>,+,(12 * %o)}<%for.j>,+,20}<%for.k>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%m][%o] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{3,+,2}<nuw><%for.i>][{-4,+,3}<nw><%for.j>][{7,+,5}<nw><%for.k>]
+;
 entry:
   %cmp32 = icmp sgt i64 %n, 0
   br i1 %cmp32, label %for.cond1.preheader.lr.ph, label %for.end17
diff --git a/llvm/test/Analysis/Delinearization/byte_offset.ll b/llvm/test/Analysis/Delinearization/byte_offset.ll
index 6534d3ea6c05b..90b1f03329e44 100644
--- a/llvm/test/Analysis/Delinearization/byte_offset.ll
+++ b/llvm/test/Analysis/Delinearization/byte_offset.ll
@@ -1,8 +1,6 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
-; CHECK: AccessFunction: ({0,+,%i2}<%outer.loop> + %unknown)
-; CHECK-NEXT: failed to delinearize
-
 ; void foo(char A[], long i2, bool c) {
 ;   for (long i = 0; ; ++i) {
 ;     char *tmp = &A[i * i2];
@@ -13,6 +11,12 @@
 ; }
 
 define void @foo(ptr %A, i64 %i2, i64 %arg, i1 %c) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store float 0.000000e+00, ptr %arrayidx, align 4
+; CHECK-NEXT:  In Loop with Header: inner.loop
+; CHECK-NEXT:  AccessFunction: ({0,+,%i2}<%outer.loop> + %unknown)
+; CHECK-NEXT:  failed to delinearize
+;
 entry:
   br label %outer.loop
 
diff --git a/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll b/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll
index 3044a4868260b..c0b1a0b9cddaf 100644
--- a/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll
+++ b/llvm/test/Analysis/Delinearization/constant_functions_multi_dim.ll
@@ -1,23 +1,25 @@
-; RUN: opt -passes='print<delinearization>' -disable-output < %s 2>&1 | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
-; CHECK:      Inst:  %tmp = load float, ptr %arrayidx, align 4
-; CHECK-NEXT: In Loop with Header: for.inc
-; CHECK-NEXT: AccessFunction: {(4 * %N * %call),+,4}<%for.inc>
-; CHECK-NEXT: Base offset: %A
-; CHECK-NEXT: ArrayDecl[UnknownSize][%N] with elements of 4 bytes.
-; CHECK-NEXT: ArrayRef[%call][{0,+,1}<nuw><nsw><%for.inc>]
-
-; CHECK:      Inst:  %tmp5 = load float, ptr %arrayidx4, align 4
-; CHECK-NEXT: In Loop with Header: for.inc
-; CHECK-NEXT: AccessFunction: {(4 * %call1),+,(4 * %N)}<%for.inc>
-; CHECK-NEXT: Base offset: %B
-; CHECK-NEXT: ArrayDecl[UnknownSize][%N] with elements of 4 bytes.
-; CHECK-NEXT: ArrayRef[{0,+,1}<nuw><nsw><%for.inc>][%call1]
-
 ; Function Attrs: noinline nounwind uwtable
 define void @mat_mul(ptr %C, ptr %A, ptr %B, i64 %N) #0 !kernel_arg_addr_space !2 !kernel_arg_access_qual !3 !kernel_arg_type !4 !kernel_arg_base_type !4 !kernel_arg_type_qual !5 {
+; CHECK-LABEL: 'mat_mul'
+; CHECK-NEXT:  Inst: %tmp = load float, ptr %arrayidx, align 4
+; CHECK-NEXT:  In Loop with Header: for.inc
+; CHECK-NEXT:  AccessFunction: {(4 * %N * %call),+,4}<%for.inc>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%N] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[%call][{0,+,1}<nuw><nsw><%for.inc>]
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %tmp5 = load float, ptr %arrayidx4, align 4
+; CHECK-NEXT:  In Loop with Header: for.inc
+; CHECK-NEXT:  AccessFunction: {(4 * %call1),+,(4 * %N)}<%for.inc>
+; CHECK-NEXT:  Base offset: %B
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%N] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.inc>][%call1]
+;
 entry:
   br label %entry.split
 
diff --git a/llvm/test/Analysis/Delinearization/divide_by_one.ll b/llvm/test/Analysis/Delinearization/divide_by_one.ll
index 8c531b78f2bcf..28fe5c50ae779 100644
--- a/llvm/test/Analysis/Delinearization/divide_by_one.ll
+++ b/llvm/test/Analysis/Delinearization/divide_by_one.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-n32"
@@ -10,19 +11,22 @@ target datalayout = "e-m:e-p:32:32-i1:32-i64:64-a:0-n32"
 ;       dst[r * stride + c] = dst[(r + 1) * stride + c - 1];
 ; }
 
-; AddRec: {{(-1 + ((1 + %bs) * %stride)),+,(-1 * %stride)}<%for.cond1.preheader>,+,1}<nw><%for.body3>
-; CHECK: Inst:  %0 = load i8, ptr %arrayidx, align 1
-; CHECK: Base offset: %dst
-; CHECK: ArrayDecl[UnknownSize][%stride] with elements of 1 bytes.
-; CHECK: ArrayRef[{(1 + %bs),+,-1}<nw><%for.cond1.preheader>][{-1,+,1}<nw><%for.body3>]
-
-; AddRec: {{(ptr %bs),+,(-1 * %stride)}<%for.cond1.preheader>,+,1}<nw><%for.body3>
-; CHECK: Inst:  store i8 %0, ptr %arrayidx7, align 1
-; CHECK: Base offset: %dst
-; CHECK: ArrayDecl[UnknownSize][%stride] with elements of 1 bytes.
-; CHECK: ArrayRef[{%bs,+,-1}<nsw><%for.cond1.preheader>][{0,+,1}<nuw><nsw><%for.body3>]
-
 define void @test(ptr nocapture %dst, i32 %stride, i32 %bs) {
+; CHECK-LABEL: 'test'
+; CHECK-NEXT:  Inst: %0 = load i8, ptr %arrayidx, align 1
+; CHECK-NEXT:  In Loop with Header: for.body3
+; CHECK-NEXT:  AccessFunction: {{\{\{}}(-1 + ((1 + %bs) * %stride)),+,(-1 * %stride)}<%for.cond1.preheader>,+,1}<nw><%for.body3>
+; CHECK-NEXT:  Base offset: %dst
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%stride] with elements of 1 bytes.
+; CHECK-NEXT:  ArrayRef[{(1 + %bs),+,-1}<nw><%for.cond1.preheader>][{-1,+,1}<nw><%for.body3>]
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i8 %0, ptr %arrayidx7, align 1
+; CHECK-NEXT:  In Loop with Header: for.body3
+; CHECK-NEXT:  AccessFunction: {{\{\{}}(%stride * %bs),+,(-1 * %stride)}<%for.cond1.preheader>,+,1}<nsw><%for.body3>
+; CHECK-NEXT:  Base offset: %dst
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%stride] with elements of 1 bytes.
+; CHECK-NEXT:  ArrayRef[{%bs,+,-1}<nsw><%for.cond1.preheader>][{0,+,1}<nuw><nsw><%for.body3>]
+;
 entry:
   %cmp20 = icmp sgt i32 %bs, -1
   br i1 %cmp20, label %for.cond1.preheader.lr.ph, label %for.end9
diff --git a/llvm/test/Analysis/Delinearization/fixed_size_array.ll b/llvm/test/Analysis/Delinearization/fixed_size_array.ll
new file mode 100644
index 0000000000000..634850bb4a5a2
--- /dev/null
+++ b/llvm/test/Analysis/Delinearization/fixed_size_array.ll
@@ -0,0 +1,540 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes='print<delinearization>' -disable-output -delinearize-use-fixed-size-array-heuristic 2>&1 | FileCheck %s
+
+; void f(int A[][8][32]) {
+;   for (i = 0; i < 42; i++)
+;    for (j = 0; j < 8; j++)
+;     for (k = 0; k < 32; k++)
+;       A[i][j][k] = 1;
+; }
+
+define void @a_i_j_k(ptr %a) {
+; CHECK-LABEL: 'a_i_j_k'
+; CHECK-NEXT:  Inst: store i32 1, ptr %idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,128}<nw><%for.j.header>,+,4}<nw><%for.k>
+; CHECK-NEXT:  Base offset: %a
+; CHECK-NEXT:  ArrayDecl[UnknownSize][8][32] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i.header>][{0,+,1}<nuw><nsw><%for.j.header>][{0,+,1}<nuw><nsw><%for.k>]
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  br label %for.k
+
+for.k:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k ]
+  %idx = getelementptr [8 x [32 x i32]], ptr %a, i32 %i, i32 %j, i32 %k
+  store i32 1, ptr %idx
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 32
+  br i1 %cmp.k, label %for.k, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 8
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 42
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
+
+; void f(int A[][8][32]) {
+;   for (i = 0; i < 42; i++)
+;    for (j = 0; j < 8; j++)
+;     for (k = 0; k < 32; k++)
+;       A[i][7-j][k] = 1;
+; }
+
+define void @a_i_nj_k(ptr %a) {
+; CHECK-LABEL: 'a_i_nj_k'
+; CHECK-NEXT:  Inst: store i32 1, ptr %idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}896,+,1024}<nuw><nsw><%for.i.header>,+,-128}<nw><%for.j.header>,+,4}<nw><%for.k>
+; CHECK-NEXT:  Base offset: %a
+; CHECK-NEXT:  ArrayDecl[UnknownSize][8][32] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i.header>][{7,+,-1}<nsw><%for.j.header>][{0,+,1}<nuw><nsw><%for.k>]
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  %j.subscript = sub i32 7, %j
+  br label %for.k
+
+for.k:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k ]
+  %idx = getelementptr [8 x [32 x i32]], ptr %a, i32 %i, i32 %j.subscript, i32 %k
+  store i32 1, ptr %idx
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 32
+  br i1 %cmp.k, label %for.k, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 8
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 42
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
+
+; In the following code, the access functions for both stores are represented
+; in the same way in SCEV, so the delinearization results are also the same. We
+; don't have any type information of the underlying objects.
+;
+; void f(int A[][4][64], int B[][8][32]) {
+;   for (i = 0; i < 42; i++)
+;    for (j = 0; j < 4; j++)
+;     for (k = 0; k < 32; k++) {
+;       A[i][j][k] = 1;
+;       B[i][2*j][k] = 1;
+;     }
+; }
+
+define void @a_ijk_b_i2jk(ptr %a, ptr %b) {
+; CHECK-LABEL: 'a_ijk_b_i2jk'
+; CHECK-NEXT:  Inst: store i32 1, ptr %a.idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,256}<nw><%for.j.header>,+,4}<nw><%for.k>
+; CHECK-NEXT:  Base offset: %a
+; CHECK-NEXT:  ArrayDecl[UnknownSize][4][64] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i.header>][{0,+,1}<nuw><nsw><%for.j.header>][{0,+,1}<nuw><nsw><%for.k>]
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 1, ptr %b.idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,256}<nw><%for.j.header>,+,4}<nw><%for.k>
+; CHECK-NEXT:  Base offset: %b
+; CHECK-NEXT:  ArrayDecl[UnknownSize][4][64] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i.header>][{0,+,1}<nuw><nsw><%for.j.header>][{0,+,1}<nuw><nsw><%for.k>]
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  %j2 = shl i32 %j, 1
+  br label %for.k
+
+for.k:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k ]
+  %a.idx = getelementptr [4 x [64 x i32]], ptr %a, i32 %i, i32 %j, i32 %k
+  %b.idx = getelementptr [8 x [32 x i32]], ptr %b, i32 %i, i32 %j2, i32 %k
+  store i32 1, ptr %a.idx
+  store i32 1, ptr %b.idx
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 32
+  br i1 %cmp.k, label %for.k, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 4
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 42
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
+
+; The type information of the underlying object is not available, so the
+; delinearization result is different from the original array size. In this
+; case, the underlying object is a type of int[][8][32], but the
+; delinearization result is like int[][4][64].
+;
+; void f(int A[][8][32]) {
+;   for (i = 0; i < 42; i++)
+;    for (j = 0; j < 3; j++)
+;     for (k = 0; k < 32; k++)
+;       A[i][2*j+1][k] = 1;
+; }
+
+define void @a_i_2j1_k(ptr %a) {
+; CHECK-LABEL: 'a_i_2j1_k'
+; CHECK-NEXT:  Inst: store i32 1, ptr %idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}128,+,1024}<nuw><nsw><%for.i.header>,+,256}<nw><%for.j.header>,+,4}<nw><%for.k>
+; CHECK-NEXT:  Base offset: %a
+; CHECK-NEXT:  ArrayDecl[UnknownSize][4][64] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i.header>][{0,+,1}<nuw><%for.j.header>][{32,+,1}<nw><%for.k>]
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  %j2 = shl i32 %j, 1
+  %j.subscript = add i32 %j2, 1
+  br label %for.k
+
+for.k:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k ]
+  %idx = getelementptr [8 x [32 x i32]], ptr %a, i32 %i, i32 %j.subscript, i32 %k
+  store i32 1, ptr %idx
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 32
+  br i1 %cmp.k, label %for.k, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 3
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 42
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
+
+; Fail to delinearize because the step recurrence value of the i-loop is not
+; divisible by that of the j-loop.
+;
+; void f(int A[][8][32]) {
+;   for (i = 0; i < 42; i++)
+;    for (j = 0; j < 2; j++)
+;     for (k = 0; k < 42; k++)
+;       A[i][3*j][k] = 1;
+; }
+
+define void @a_i_3j_k(ptr %a) {
+; CHECK-LABEL: 'a_i_3j_k'
+; CHECK-NEXT:  Inst: store i32 1, ptr %idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,384}<nw><%for.j.header>,+,4}<nw><%for.k>
+; CHECK-NEXT:  failed to delinearize
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  %j.subscript = mul i32 %j, 3
+  br label %for.k
+
+for.k:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k ]
+  %idx = getelementptr [8 x [32 x i32]], ptr %a, i32 %i, i32 %j.subscript, i32 %k
+  store i32 1, ptr %idx
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 42
+  br i1 %cmp.k, label %for.k, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 2
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 42
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
+
+; Although the step recurrence value of j-loop is not divisible by that of the
+; k-loop, delinearization is possible because we know that the "actual" stride
+; width for the last dimension is 4 instead of 12.
+;
+; void f(int A[][8][32]) {
+;   for (i = 0; i < 42; i++)
+;    for (j = 0; j < 8; j++)
+;     for (k = 0; k < 10; k++)
+;       A[i][j][3*k] = 1;
+; }
+
+define void @a_i_j_3k(ptr %a) {
+; CHECK-LABEL: 'a_i_j_3k'
+; CHECK-NEXT:  Inst: store i32 1, ptr %idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,128}<nw><%for.j.header>,+,12}<nw><%for.k>
+; CHECK-NEXT:  Base offset: %a
+; CHECK-NEXT:  ArrayDecl[UnknownSize][8][32] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i.header>][{0,+,1}<nuw><nsw><%for.j.header>][{0,+,3}<nuw><nsw><%for.k>]
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  br label %for.k
+
+for.k:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k ]
+  %k.subscript = mul i32 %k, 3
+  %idx = getelementptr [8 x [32 x i32]], ptr %a, i32 %i, i32 %j, i32 %k.subscript
+  store i32 1, ptr %idx
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 10
+  br i1 %cmp.k, label %for.k, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 8
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 42
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
+
+; Fail to delinearize because i is used in multiple subscripts that are not adjacent.
+;
+; void f(int A[][8][32]) {
+;   for (i = 0; i < 32; i++)
+;    for (j = 0; j < 2; j++)
+;     for (k = 0; k < 4; k++)
+;       A[i][2*j+k][i] = 1;
+; }
+
+define void @a_i_j2k_i(ptr %a) {
+; CHECK-LABEL: 'a_i_j2k_i'
+; CHECK-NEXT:  Inst: store i32 1, ptr %idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,1028}<%for.i.header>,+,256}<nw><%for.j.header>,+,128}<nw><%for.k>
+; CHECK-NEXT:  failed to delinearize
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  br label %for.k
+
+for.k:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k ]
+  %j2 = shl i32 %j, 1
+  %j2.k = add i32 %j2, %k
+  %idx = getelementptr [8 x [32 x i32]], ptr %a, i32 %i, i32 %j2.k, i32 %i
+  store i32 1, ptr %idx
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 4
+  br i1 %cmp.k, label %for.k, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 2
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 32
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
+
+; Can delinearize, but the result is different from the original array size. In
+; this case, the outermost two dimensions are melded into one.
+;
+; void f(int A[][8][32]) {
+;   for (i = 0; i < 8; i++)
+;    for (j = 0; j < 10; j++)
+;     for (k = 0; k < 10; k++)
+;       A[i][i][j+k] = 1;
+; }
+
+define void @a_i_i_jk(ptr %a) {
+; CHECK-LABEL: 'a_i_i_jk'
+; CHECK-NEXT:  Inst: store i32 1, ptr %idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,1152}<%for.i.header>,+,4}<nw><%for.j.header>,+,4}<nw><%for.k>
+; CHECK-NEXT:  Base offset: %a
+; CHECK-NEXT:  ArrayDecl[UnknownSize][288] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i.header>][{{\{\{}}0,+,1}<nuw><nsw><%for.j.header>,+,1}<nuw><nsw><%for.k>]
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  br label %for.k
+
+for.k:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k ]
+  %jk = add i32 %j, %k
+  %idx = getelementptr [8 x [32 x i32]], ptr %a, i32 %i, i32 %i, i32 %jk
+  store i32 1, ptr %idx
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 10
+  br i1 %cmp.k, label %for.k, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 10
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 8
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
+
+; void f(int A[][8][32]) {
+;   for (i = 0; i < 8; i++)
+;    for (j = 0; j < 4; j++)
+;     for (k = 0; k < 4; k++)
+;       for (l = 0; l < 32; l++)
+;         A[i][j+k][l] = 1;
+; }
+
+define void @a_i_jk_l(ptr %a) {
+; CHECK-LABEL: 'a_i_jk_l'
+; CHECK-NEXT:  Inst: store i32 1, ptr %idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.l
+; CHECK-NEXT:  AccessFunction: {{\{\{\{\{}}0,+,1024}<nuw><nsw><%for.i.header>,+,128}<nw><%for.j.header>,+,128}<nw><%for.k.header>,+,4}<nw><%for.l>
+; CHECK-NEXT:  Base offset: %a
+; CHECK-NEXT:  ArrayDecl[UnknownSize][8][32] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i.header>][{{\{\{}}0,+,1}<nuw><nsw><%for.j.header>,+,1}<nuw><nsw><%for.k.header>][{0,+,1}<nuw><nsw><%for.l>]
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  br label %for.k.header
+
+for.k.header:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k.latch ]
+  %jk = add i32 %j, %k
+  br label %for.l
+
+for.l:
+  %l = phi i32 [ 0, %for.k.header ], [ %l.inc, %for.l ]
+  %idx = getelementptr [8 x [32 x i32]], ptr %a, i32 %i, i32 %jk, i32 %l
+  store i32 1, ptr %idx
+  %l.inc = add i32 %l, 1
+  %cmp.l = icmp slt i32 %l.inc, 32
+  br i1 %cmp.l, label %for.l, label %for.k.latch
+
+for.k.latch:
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 4
+  br i1 %cmp.k, label %for.k.header, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 4
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 8
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
+
+; Reject if the address is not a multiple of the element size.
+;
+; void f(int *A) {
+;   for (i = 0; i < 42; i++)
+;    for (j = 0; j < 8; j++)
+;     for (k = 0; k < 32; k++)
+;       *((int *)((char *)A + i*256 + j*32 + k)) = 1;
+; }
+
+define void @non_divisible_by_element_size(ptr %a) {
+; CHECK-LABEL: 'non_divisible_by_element_size'
+; CHECK-NEXT:  Inst: store i32 1, ptr %idx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,256}<nuw><nsw><%for.i.header>,+,32}<nw><%for.j.header>,+,1}<nw><%for.k>
+; CHECK-NEXT:  failed to delinearize
+;
+entry:
+  br label %for.i.header
+
+for.i.header:
+  %i = phi i32 [ 0, %entry ], [ %i.inc, %for.i.latch ]
+  br label %for.j.header
+
+for.j.header:
+  %j = phi i32 [ 0, %for.i.header ], [ %j.inc, %for.j.latch ]
+  br label %for.k
+
+for.k:
+  %k = phi i32 [ 0, %for.j.header ], [ %k.inc, %for.k ]
+  %idx = getelementptr [8 x [32 x i8]], ptr %a, i32 %i, i32 %j, i32 %k
+  store i32 1, ptr %idx
+  %k.inc = add i32 %k, 1
+  %cmp.k = icmp slt i32 %k.inc, 32
+  br i1 %cmp.k, label %for.k, label %for.j.latch
+
+for.j.latch:
+  %j.inc = add i32 %j, 1
+  %cmp.j = icmp slt i32 %j.inc, 8
+  br i1 %cmp.j, label %for.j.header, label %for.i.latch
+
+for.i.latch:
+  %i.inc = add i32 %i, 1
+  %cmp.i = icmp slt i32 %i.inc, 42
+  br i1 %cmp.i, label %for.i.header, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/Delinearization/gcd_multiply_expr.ll b/llvm/test/Analysis/Delinearization/gcd_multiply_expr.ll
index 95fefed7082f5..3e4545289a450 100644
--- a/llvm/test/Analysis/Delinearization/gcd_multiply_expr.ll
+++ b/llvm/test/Analysis/Delinearization/gcd_multiply_expr.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -aa-pipeline=basic-aa -passes='require<da>,print<delinearization>' -disable-output
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 ;
 ; a, b, c, d, g, h;
 ; char *f;
@@ -26,6 +27,147 @@
 @d = common global i32 0, align 4
 
 define i32 @fn2() {
+; CHECK-LABEL: 'fn2'
+; CHECK-NEXT:  Inst: store i32 %storemerge.i, ptr @a, align 4
+; CHECK-NEXT:  In Loop with Header: for.cond2thread-pre-split.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %9 = load i8, ptr %arrayidx.i, align 1
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: (sext i32 {({(%1 * %2),+,1}<nw><%for.cond2thread-pre-split.i> + %.pr.i),+,8}<nw><%for.body4.i> to i64)
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %conv.i, ptr @c, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %inc.i, ptr @b, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %10 = load i8, ptr %arrayidx.1.i, align 1
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: (sext i32 {({(1 + (%1 * %2)),+,1}<nw><%for.cond2thread-pre-split.i> + %.pr.i),+,8}<nw><%for.body4.i> to i64)
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %conv.1.i, ptr @c, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %inc.1.i, ptr @b, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %11 = load i8, ptr %arrayidx.2.i, align 1
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: (sext i32 {({(2 + (%1 * %2)),+,1}<nw><%for.cond2thread-pre-split.i> + %.pr.i),+,8}<nw><%for.body4.i> to i64)
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %conv.2.i, ptr @c, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %inc.2.i, ptr @b, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %12 = load i8, ptr %arrayidx.3.i, align 1
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: (sext i32 {({(3 + (%1 * %2)),+,1}<nw><%for.cond2thread-pre-split.i> + %.pr.i),+,8}<nw><%for.body4.i> to i64)
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %conv.3.i, ptr @c, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %inc.3.i, ptr @b, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %13 = load i8, ptr %arrayidx.4.i, align 1
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: (sext i32 {({(4 + (%1 * %2)),+,1}<nw><%for.cond2thread-pre-split.i> + %.pr.i),+,8}<nw><%for.body4.i> to i64)
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %conv.4.i, ptr @c, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %inc.4.i, ptr @b, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %14 = load i8, ptr %arrayidx.5.i, align 1
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: (sext i32 {({(5 + (%1 * %2)),+,1}<nw><%for.cond2thread-pre-split.i> + %.pr.i),+,8}<nw><%for.body4.i> to i64)
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %conv.5.i, ptr @c, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %inc.5.i, ptr @b, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %15 = load i8, ptr %arrayidx.6.i, align 1
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: (sext i32 {({(6 + (%1 * %2)),+,1}<nw><%for.cond2thread-pre-split.i> + %.pr.i),+,8}<nw><%for.body4.i> to i64)
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %conv.6.i, ptr @c, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %inc.6.i, ptr @b, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %16 = load i8, ptr %arrayidx.7.i, align 1
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: (sext i32 {({(7 + (%1 * %2)),+,1}<nw><%for.cond2thread-pre-split.i> + %.pr.i),+,8}<nw><%for.body4.i> to i64)
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %conv.7.i, ptr @c, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %inc.7.i, ptr @b, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %21 = load i8, ptr %arrayidx.ur.i, align 1
+; CHECK-NEXT:  In Loop with Header: for.body4.ur.i
+; CHECK-NEXT:  AccessFunction: (sext i32 {({(%1 * %2),+,1}<nw><%for.cond2thread-pre-split.i> + %.ph),+,1}<nw><%for.body4.ur.i> to i64)
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %conv.ur.i, ptr @c, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.ur.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i32 %inc.ur.i, ptr @b, align 4
+; CHECK-NEXT:  In Loop with Header: for.body4.ur.i
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+;
 entry:
   %.pr = load i32, ptr @d, align 4
   %phitmp = icmp eq i32 %.pr, 0
diff --git a/llvm/test/Analysis/Delinearization/himeno_1.ll b/llvm/test/Analysis/Delinearization/himeno_1.ll
index 879284498350a..292dca61d0592 100644
--- a/llvm/test/Analysis/Delinearization/himeno_1.ll
+++ b/llvm/test/Analysis/Delinearization/himeno_1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
 ; #define MR(mt,n,r,c,d)  mt->m[(n) * mt->mrows * mt->mcols * mt->mdeps + (r) * mt->mcols* mt->mdeps + (c) * mt->mdeps + (d)]
@@ -26,14 +27,17 @@
 ;           MR(a,0,i,j,k) = i + j + k;
 ; }
 
-; AddRec: {{{(4 + (4 * (sext i32 %a.deps to i64) * (1 + (sext i32 %a.cols to i64))) + %a.base),+,(4 * (sext i32 %a.deps to i64) * (sext i32 %a.cols to i64))}<%for.i>,+,(4 * (sext i32 %a.deps to i64))}<%for.j>,+,4}<%for.k>
-; CHECK: Base offset: %a.base
-; CHECK: ArrayDecl[UnknownSize][(sext i32 %a.cols to i64)][(sext i32 %a.deps to i64)] with elements of 4 bytes.
-; CHECK: ArrayRef[{1,+,1}<nuw><nsw><%for.i>][{1,+,1}<nuw><nsw><%for.j>][{1,+,1}<nuw><nsw><%for.k>]
-
 %struct.Mat = type { ptr, i32, i32, i32, i32 }
 
 define void @jacobi(i32 %nn, ptr nocapture %a, ptr nocapture %p) nounwind uwtable {
+; CHECK-LABEL: 'jacobi'
+; CHECK-NEXT:  Inst: store float 1.000000e+00, ptr %arrayidx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}(4 + (4 * (sext i32 %a.deps to i64) * (1 + (sext i32 %a.cols to i64))<nsw>)),+,(4 * (sext i32 %a.deps to i64) * (sext i32 %a.cols to i64))}<%for.i>,+,(4 * (sext i32 %a.deps to i64))<nsw>}<%for.j>,+,4}<%for.k>
+; CHECK-NEXT:  Base offset: %a.base
+; CHECK-NEXT:  ArrayDecl[UnknownSize][(sext i32 %a.cols to i64)][(sext i32 %a.deps to i64)] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{1,+,1}<nuw><nsw><%for.i>][{1,+,1}<nuw><nsw><%for.j>][{1,+,1}<nuw><nsw><%for.k>]
+;
 entry:
   %p.rows.ptr = getelementptr inbounds %struct.Mat, ptr %p, i64 0, i32 2
   %p.rows = load i32, ptr %p.rows.ptr
diff --git a/llvm/test/Analysis/Delinearization/himeno_2.ll b/llvm/test/Analysis/Delinearization/himeno_2.ll
index 30b2154abbc83..d210539d67d8b 100644
--- a/llvm/test/Analysis/Delinearization/himeno_2.ll
+++ b/llvm/test/Analysis/Delinearization/himeno_2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
 ; #define MR(mt,n,r,c,d)  mt->m[(n) * mt->mrows * mt->mcols * mt->mdeps + (r) * mt->mcols* mt->mdeps + (c) * mt->mdeps + (d)]
@@ -26,14 +27,17 @@
 ;           MR(a,0,i,j,k) = i + j + k;
 ; }
 
-; AddRec: {{{(4 + (4 * (sext i32 %a.deps to i64) * (1 + (sext i32 %a.cols to i64))) + %a.base),+,(4 * (sext i32 %a.deps to i64) * (sext i32 %a.cols to i64))}<%for.i>,+,(4 * (sext i32 %a.deps to i64))}<%for.j>,+,4}<%for.k>
-; CHECK: Base offset: %a.base
-; CHECK: ArrayDecl[UnknownSize][(sext i32 %a.cols to i64)][(sext i32 %a.deps to i64)] with elements of 4 bytes.
-; CHECK: ArrayRef[{1,+,1}<nuw><nsw><%for.i>][{1,+,1}<nuw><nsw><%for.j>][{1,+,1}<nuw><nsw><%for.k>]
-
 %struct.Mat = type { ptr, i32, i32, i32, i32 }
 
 define void @jacobi(i32 %nn, ptr nocapture %a, ptr nocapture %p) nounwind uwtable {
+; CHECK-LABEL: 'jacobi'
+; CHECK-NEXT:  Inst: store float 1.000000e+00, ptr %arrayidx, align 4
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}(4 + (4 * (sext i32 %a.deps to i64) * (1 + (sext i32 %a.cols to i64))<nsw>)),+,(4 * (sext i32 %a.deps to i64) * (sext i32 %a.cols to i64))}<%for.i>,+,(4 * (sext i32 %a.deps to i64))<nsw>}<%for.j>,+,4}<%for.k>
+; CHECK-NEXT:  Base offset: %a.base
+; CHECK-NEXT:  ArrayDecl[UnknownSize][(sext i32 %a.cols to i64)][(sext i32 %a.deps to i64)] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{1,+,1}<nuw><nsw><%for.i>][{1,+,1}<nuw><nsw><%for.j>][{1,+,1}<nuw><nsw><%for.k>]
+;
 entry:
   %p.rows.ptr = getelementptr inbounds %struct.Mat, ptr %p, i64 0, i32 2
   %p.rows = load i32, ptr %p.rows.ptr
diff --git a/llvm/test/Analysis/Delinearization/iv_times_constant_in_subscript.ll b/llvm/test/Analysis/Delinearization/iv_times_constant_in_subscript.ll
index 84ac1a39cc96e..cbe3ec8a19acd 100644
--- a/llvm/test/Analysis/Delinearization/iv_times_constant_in_subscript.ll
+++ b/llvm/test/Analysis/Delinearization/iv_times_constant_in_subscript.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
 ; Derived from the following code:
@@ -8,13 +9,15 @@
 ;       A[2i+b][2j] = 1.0;
 ; }
 
-; AddRec: {{((%m * %b * 8) + %A),+,(2 * %m * 8)}<%for.i>,+,(2 * 8)}<%for.j>
-; CHECK: Base offset: %A
-; CHECK: ArrayDecl[UnknownSize][%m] with elements of 8 bytes.
-; CHECK: ArrayRef[{%b,+,2}<nsw><%for.i>][{0,+,2}<nuw><%for.j>]
-
-
 define void @foo(i64 %n, i64 %m, i64 %b, ptr %A) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store double 1.000000e+00, ptr %arrayidx, align 8
+; CHECK-NEXT:  In Loop with Header: for.j
+; CHECK-NEXT:  AccessFunction: {{\{\{}}(8 * %m * %b),+,(16 * %m)}<%for.i>,+,16}<%for.j>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%m] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{%b,+,2}<nsw><%for.i>][{0,+,2}<nuw><%for.j>]
+;
 entry:
   br label %for.i
 
diff --git a/llvm/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_3d.ll b/llvm/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_3d.ll
index 4ebac31d5ccf5..3d21d97438462 100644
--- a/llvm/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_3d.ll
+++ b/llvm/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_3d.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output  2>&1 | FileCheck %s
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
@@ -8,12 +9,15 @@
 ;         A[i+3][j-4][k+7] = 1.0;
 ; }
 
-; AddRec: {{{(56 + (8 * (-4 + (3 * %m)) * %o) + %A),+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
-; CHECK: Base offset: %A
-; CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of 8 bytes.
-; CHECK: ArrayRef[{3,+,1}<nuw><%for.i>][{-4,+,1}<nsw><%for.j>][{7,+,1}<nuw><nsw><%for.k>]
-
 define void @foo(i64 %n, i64 %m, i64 %o, ptr %A) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store double 1.000000e+00, ptr %idx, align 8
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}(56 + (8 * (-4 + (3 * %m)) * %o)),+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%m][%o] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{3,+,1}<nuw><%for.i>][{-4,+,1}<nsw><%for.j>][{7,+,1}<nuw><nsw><%for.k>]
+;
 entry:
   br label %for.i
 
diff --git a/llvm/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_nts_3d.ll b/llvm/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_nts_3d.ll
index 0265fb71449f8..3dbd71b1c9ac5 100644
--- a/llvm/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_nts_3d.ll
+++ b/llvm/test/Analysis/Delinearization/multidim_ivs_and_integer_offsets_nts_3d.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
 ; void foo(long n, long m, long o, long p, double A[n][m][o+p]) {
@@ -8,12 +9,15 @@
 ;         A[i+3][j-4][k+7] = 1.0;
 ; }
 
-; AddRec: {{{(56 + (8 * (-4 + (3 * %m)) * (%o + %p)) + %A),+,(8 * (%o + %p) * %m)}<%for.cond4.preheader.lr.ph.us>,+,(8 * (%o + %p))}<%for.body6.lr.ph.us.us>,+,8}<%for.body6.us.us>
-; CHECK: Base offset: %A
-; CHECK: ArrayDecl[UnknownSize][%m][(%o + %p)] with elements of 8 bytes.
-; CHECK: ArrayRef[{3,+,1}<nuw><%for.cond4.preheader.lr.ph.us>][{-4,+,1}<nw><%for.body6.lr.ph.us.us>][{7,+,1}<nw><%for.body6.us.us>]
-
 define void @foo(i64 %n, i64 %m, i64 %o, i64 %p, ptr nocapture %A) nounwind uwtable {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store double 1.000000e+00, ptr %arrayidx10.us.us, align 8
+; CHECK-NEXT:  In Loop with Header: for.body6.us.us
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}(56 + (8 * (-4 + (3 * %m)) * (%o + %p))),+,(8 * (%o + %p) * %m)}<%for.cond4.preheader.lr.ph.us>,+,(8 * (%o + %p))}<%for.body6.lr.ph.us.us>,+,8}<%for.body6.us.us>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%m][(%o + %p)] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{3,+,1}<nuw><%for.cond4.preheader.lr.ph.us>][{-4,+,1}<nw><%for.body6.lr.ph.us.us>][{7,+,1}<nw><%for.body6.us.us>]
+;
 entry:
   %add = add nsw i64 %p, %o
   %cmp22 = icmp sgt i64 %n, 0
diff --git a/llvm/test/Analysis/Delinearization/multidim_ivs_and_parameteric_offsets_3d.ll b/llvm/test/Analysis/Delinearization/multidim_ivs_and_parameteric_offsets_3d.ll
index 51e01010cdb7a..a2d64a5b40bc9 100644
--- a/llvm/test/Analysis/Delinearization/multidim_ivs_and_parameteric_offsets_3d.ll
+++ b/llvm/test/Analysis/Delinearization/multidim_ivs_and_parameteric_offsets_3d.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
 ; void foo(long n, long m, long o, double A[n][m][o], long p, long q, long r) {
@@ -8,12 +9,15 @@
 ;         A[i+p][j+q][k+r] = 1.0;
 ; }
 
-; AddRec: {{{((8 * ((((%m * %p) + %q) * %o) + %r)) + %A),+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
-; CHECK: Base offset: %A
-; CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of 8 bytes.
-; CHECK: ArrayRef[{%p,+,1}<nw><%for.i>][{%q,+,1}<nsw><%for.j>][{%r,+,1}<nsw><%for.k>]
-
 define void @foo(i64 %n, i64 %m, i64 %o, ptr %A, i64 %p, i64 %q, i64 %r) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store double 1.000000e+00, ptr %idx, align 8
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}(8 * ((((%m * %p) + %q) * %o) + %r)),+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%m][%o] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{%p,+,1}<nw><%for.i>][{%q,+,1}<nsw><%for.j>][{%r,+,1}<nsw><%for.k>]
+;
 entry:
   br label %for.i
 
diff --git a/llvm/test/Analysis/Delinearization/multidim_only_ivs_2d.ll b/llvm/test/Analysis/Delinearization/multidim_only_ivs_2d.ll
index b92abfc8d3aba..ac83ba19b252d 100644
--- a/llvm/test/Analysis/Delinearization/multidim_only_ivs_2d.ll
+++ b/llvm/test/Analysis/Delinearization/multidim_only_ivs_2d.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
 ; Derived from the following code:
@@ -8,21 +9,22 @@
 ;       A[i][j] = 1.0;
 ; }
 
-; Inst:  %val = load double, ptr %arrayidx
-; In Loop with Header: for.j
-; AddRec: {{0,+,(%m * 8)}<%for.i>,+,8}<%for.j>
-; Base offset: %A
-; ArrayDecl[UnknownSize][%m] with elements of 8 bytes.
-; ArrayRef[{0,+,1}<nuw><nsw><%for.i>][{0,+,1}<nuw><nsw><%for.j>]
-
-; Inst:  store double %val, ptr %arrayidx
-; In Loop with Header: for.j
-; AddRec: {{%A,+,(8 * %m)}<%for.i>,+,8}<%for.j>
-; CHECK: Base offset: %A
-; CHECK: ArrayDecl[UnknownSize][%m] with elements of 8 bytes.
-; CHECK: ArrayRef[{0,+,1}<nuw><nsw><%for.i>][{0,+,1}<nuw><nsw><%for.j>]
-
 define void @foo(i64 %n, i64 %m, ptr %A) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: %val = load double, ptr %arrayidx, align 8
+; CHECK-NEXT:  In Loop with Header: for.j
+; CHECK-NEXT:  AccessFunction: {{\{\{}}0,+,(8 * %m)}<%for.i>,+,8}<%for.j>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%m] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i>][{0,+,1}<nuw><nsw><%for.j>]
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store double %val, ptr %arrayidx, align 8
+; CHECK-NEXT:  In Loop with Header: for.j
+; CHECK-NEXT:  AccessFunction: {{\{\{}}0,+,(8 * %m)}<%for.i>,+,8}<%for.j>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%m] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i>][{0,+,1}<nuw><nsw><%for.j>]
+;
 entry:
   br label %for.i
 
diff --git a/llvm/test/Analysis/Delinearization/multidim_only_ivs_2d_nested.ll b/llvm/test/Analysis/Delinearization/multidim_only_ivs_2d_nested.ll
index cc4c4eb403d80..262a092794cb1 100644
--- a/llvm/test/Analysis/Delinearization/multidim_only_ivs_2d_nested.ll
+++ b/llvm/test/Analysis/Delinearization/multidim_only_ivs_2d_nested.ll
@@ -1,5 +1,5 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
-; XFAIL: *
 ; We do not recognize anymore variable size arrays.
 
 ; extern void bar(long n, long m, double A[n][m]);
@@ -15,12 +15,13 @@
 ;   }
 ; }
 
-; AddRec: {{%vla.us,+,{8,+,8}<%for.cond7.preheader.lr.ph.split.us.us>}<%for.body9.lr.ph.us.us>,+,8}<%for.body9.us.us>
-; CHECK: Base offset: %vla.us
-; CHECK: ArrayDecl[UnknownSize][{1,+,1}<%for.cond7.preheader.lr.ph.split.us.us>] with elements of sizeof(double) bytes.
-; CHECK: ArrayRef[{0,+,1}<nuw><nsw><%for.body9.lr.ph.us.us>][{0,+,1}<nuw><nsw><%for.body9.us.us>]
-
 define void @foo(i64 %a, i64 %b) nounwind uwtable {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store double 1.000000e+00, ptr %arrayidx10.us.us, align 8
+; CHECK-NEXT:  In Loop with Header: for.body9.us.us
+; CHECK-NEXT:  AccessFunction: {{\{\{}}0,+,{8,+,8}<%for.cond7.preheader.lr.ph.split.us.us>}<%for.body9.lr.ph.us.us>,+,8}<%for.body9.us.us>
+; CHECK-NEXT:  failed to delinearize
+;
 entry:
   %cmp43 = icmp sgt i64 %a, 1
   br i1 %cmp43, label %for.cond1.preheader.lr.ph, label %for.end19
diff --git a/llvm/test/Analysis/Delinearization/multidim_only_ivs_3d.ll b/llvm/test/Analysis/Delinearization/multidim_only_ivs_3d.ll
index 252743b287788..b1405db81a787 100644
--- a/llvm/test/Analysis/Delinearization/multidim_only_ivs_3d.ll
+++ b/llvm/test/Analysis/Delinearization/multidim_only_ivs_3d.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 
 ; void foo(long n, long m, long o, double A[n][m][o]) {
@@ -8,12 +9,15 @@
 ;         A[i][j][k] = 1.0;
 ; }
 
-; AddRec: {{{%A,+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
-; CHECK: Base offset: %A
-; CHECK: ArrayDecl[UnknownSize][%m][%o] with elements of 8 bytes.
-; CHECK: ArrayRef[{0,+,1}<nuw><nsw><%for.i>][{0,+,1}<nuw><nsw><%for.j>][{0,+,1}<nuw><nsw><%for.k>]
-
 define void @foo(i64 %n, i64 %m, i64 %o, ptr %A) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store double 1.000000e+00, ptr %idx, align 8
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,(8 * %m * %o)}<%for.i>,+,(8 * %o)}<%for.j>,+,8}<%for.k>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%m][%o] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i>][{0,+,1}<nuw><nsw><%for.j>][{0,+,1}<nuw><nsw><%for.k>]
+;
 entry:
   br label %for.i
 
diff --git a/llvm/test/Analysis/Delinearization/multidim_only_ivs_3d_cast.ll b/llvm/test/Analysis/Delinearization/multidim_only_ivs_3d_cast.ll
index e8ee7283e4040..6de072ebaee13 100644
--- a/llvm/test/Analysis/Delinearization/multidim_only_ivs_3d_cast.ll
+++ b/llvm/test/Analysis/Delinearization/multidim_only_ivs_3d_cast.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 ; void foo(int n, int m, int o, double A[n][m][o]) {
 ;
@@ -7,15 +8,18 @@
 ;         A[i][j][k] = 1.0;
 ; }
 
-; AddRec: {{{%A,+,(8 * (zext i32 %m to i64) * (zext i32 %o to i64))}<%for.i>,+,(8 * (zext i32 %o to i64))}<%for.j>,+,8}<%for.k>
-; CHECK: Base offset: %A
-; CHECK: ArrayDecl[UnknownSize][(zext i32 %m to i64)][(zext i32 %o to i64)] with elements of 8 bytes.
-; CHECK: ArrayRef[{0,+,1}<nuw><nsw><%for.i>][{0,+,1}<nuw><nsw><%for.j>][{0,+,1}<nuw><nsw><%for.k>]
-
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define void @foo(i32 %n, i32 %m, i32 %o, ptr %A) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store double 1.000000e+00, ptr %idx, align 8
+; CHECK-NEXT:  In Loop with Header: for.k
+; CHECK-NEXT:  AccessFunction: {{\{\{\{}}0,+,(8 * (zext i32 %m to i64) * (zext i32 %o to i64))}<%for.i>,+,(8 * (zext i32 %o to i64))<nuw><nsw>}<%for.j>,+,8}<%for.k>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][(zext i32 %m to i64)][(zext i32 %o to i64)] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i>][{0,+,1}<nuw><nsw><%for.j>][{0,+,1}<nuw><nsw><%for.k>]
+;
 entry:
   %m_zext = zext i32 %m to i64
   %n_zext = zext i32 %o to i64
diff --git a/llvm/test/Analysis/Delinearization/multidim_two_accesses_different_delinearization.ll b/llvm/test/Analysis/Delinearization/multidim_two_accesses_different_delinearization.ll
index ba32401e4bb84..d7148c5216462 100644
--- a/llvm/test/Analysis/Delinearization/multidim_two_accesses_different_delinearization.ll
+++ b/llvm/test/Analysis/Delinearization/multidim_two_accesses_different_delinearization.ll
@@ -1,4 +1,5 @@
-; RUN: opt -aa-pipeline=basic-aa -passes='require<da>,print<delinearization>' -disable-output < %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
@@ -12,6 +13,21 @@ target triple = "x86_64-unknown-linux-gnu"
 ; }
 
 define void @foo(i64 %n, i64 %m, ptr %A) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: store double 1.000000e+00, ptr %arrayidx, align 8
+; CHECK-NEXT:  In Loop with Header: for.j
+; CHECK-NEXT:  AccessFunction: {{\{\{}}0,+,(8 * %m)}<%for.i>,+,8}<%for.j>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%m] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.i>][{0,+,1}<nuw><nsw><%for.j>]
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store double 1.000000e+00, ptr %arrayidx1, align 8
+; CHECK-NEXT:  In Loop with Header: for.j
+; CHECK-NEXT:  AccessFunction: {{\{\{}}0,+,8}<%for.i>,+,(8 * %n)}<%for.j>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%n] with elements of 8 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.j>][{0,+,1}<nuw><nsw><%for.i>]
+;
 entry:
   br label %for.i
 
diff --git a/llvm/test/Analysis/Delinearization/parameter_addrec_product.ll b/llvm/test/Analysis/Delinearization/parameter_addrec_product.ll
index b8d4b3c20ce4e..cbccafd1e6546 100644
--- a/llvm/test/Analysis/Delinearization/parameter_addrec_product.ll
+++ b/llvm/test/Analysis/Delinearization/parameter_addrec_product.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -passes='print<delinearization>' -disable-output < %s 2>&1 | FileCheck %s
 ;
 ;    void foo(float *A, long *p) {
@@ -5,13 +6,29 @@
 ;        for (long j = 0; j < 100; j++)
 ;          A[i * (*p) + j] += i + j;
 ;    }
-;
-; CHECK: ArrayDecl[UnknownSize][%pval] with elements of 4 bytes.
-; CHECK: ArrayRef[{0,+,1}<nuw><nsw><%bb2>][{0,+,1}<nuw><nsw><%bb4>]
-;
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 define void @foo(ptr %A, ptr %p) {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: %pval = load i64, ptr %p, align 8
+; CHECK-NEXT:  In Loop with Header: bb4
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: %tmp11 = load float, ptr %tmp10, align 4
+; CHECK-NEXT:  In Loop with Header: bb4
+; CHECK-NEXT:  AccessFunction: (4 * (({0,+,1}<nuw><nsw><%bb2> * %pval)<nsw> + {0,+,1}<nuw><nsw><%bb4>)<nsw>)<nsw>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%pval] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%bb2>][{0,+,1}<nuw><nsw><%bb4>]
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store float %tmp12, ptr %tmp10, align 4
+; CHECK-NEXT:  In Loop with Header: bb4
+; CHECK-NEXT:  AccessFunction: (4 * (({0,+,1}<nuw><nsw><%bb2> * %pval)<nsw> + {0,+,1}<nuw><nsw><%bb4>)<nsw>)<nsw>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][%pval] with elements of 4 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%bb2>][{0,+,1}<nuw><nsw><%bb4>]
+;
 bb:
   br label %bb2
 
diff --git a/llvm/test/Analysis/Delinearization/terms_with_identity_factor.ll b/llvm/test/Analysis/Delinearization/terms_with_identity_factor.ll
index ab5ae056e02ce..86adcc8a1957c 100644
--- a/llvm/test/Analysis/Delinearization/terms_with_identity_factor.ll
+++ b/llvm/test/Analysis/Delinearization/terms_with_identity_factor.ll
@@ -1,23 +1,27 @@
-; REQUIRES: asserts
-; RUN: opt < %s -passes='print<delinearization>' -disable-output -debug 2>&1 2>&1 | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 2>&1 | FileCheck %s
 ; void foo (int m, int n, char *A) {
 ;    for (int i=0; i < m; i++)
 ;      for(int j=0; j< n; j++)
 ;        A[i*n+j] += 1;
 ;}
 
-; ModuleID = 'delin.cpp'
-;target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
-;target triple = "aarch64--linux-gnu"
-
-; CHECK-LABEL: Delinearization on function foo
-; CHECK: Inst:  %4 = load i8, ptr %arrayidx.us, align 1
-; CHECK: Subscripts
-; CHECK-NEXT: {0,+,1}<nuw><nsw><%for.body3.lr.ph.us>
-; CHECK-NEXT: {0,+,1}<nuw><nsw><%for.body3.us>
-; CHECK: succeeded to delinearize
-
 define void @foo(i32 %m, i32 %n, ptr nocapture %A) #0 {
+; CHECK-LABEL: 'foo'
+; CHECK-NEXT:  Inst: %4 = load i8, ptr %arrayidx.us, align 1
+; CHECK-NEXT:  In Loop with Header: for.body3.us
+; CHECK-NEXT:  AccessFunction: {{\{\{}}0,+,(sext i32 %n to i64)}<nsw><%for.body3.lr.ph.us>,+,1}<nsw><%for.body3.us>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][(sext i32 %n to i64)] with elements of 1 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.body3.lr.ph.us>][{0,+,1}<nuw><nsw><%for.body3.us>]
+; CHECK-EMPTY:
+; CHECK-NEXT:  Inst: store i8 %add4.us, ptr %arrayidx.us, align 1
+; CHECK-NEXT:  In Loop with Header: for.body3.us
+; CHECK-NEXT:  AccessFunction: {{\{\{}}0,+,(sext i32 %n to i64)}<nsw><%for.body3.lr.ph.us>,+,1}<nsw><%for.body3.us>
+; CHECK-NEXT:  Base offset: %A
+; CHECK-NEXT:  ArrayDecl[UnknownSize][(sext i32 %n to i64)] with elements of 1 bytes.
+; CHECK-NEXT:  ArrayRef[{0,+,1}<nuw><nsw><%for.body3.lr.ph.us>][{0,+,1}<nuw><nsw><%for.body3.us>]
+;
 entry:
   br label %entry.split
 
diff --git a/llvm/test/Analysis/Delinearization/type_mismatch.ll b/llvm/test/Analysis/Delinearization/type_mismatch.ll
index a9501519d0316..1b78f24485743 100644
--- a/llvm/test/Analysis/Delinearization/type_mismatch.ll
+++ b/llvm/test/Analysis/Delinearization/type_mismatch.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -passes='print<delinearization>' -disable-output
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes='print<delinearization>' -disable-output 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
 ; Test that SCEV divide code doesn't crash when attempting to create a SCEV
@@ -8,19 +9,25 @@
 
 target datalayout = "e-m:e-p:32:32-i64:64-a:0-v32:32-n16:32"
 
-define fastcc void @test(i1 %arg) {
+define fastcc void @test(i1 %arg, ptr %x) {
+; CHECK-LABEL: 'test'
+; CHECK-NEXT:  Inst: store i8 42, ptr %arrayidx.phi, align 1
+; CHECK-NEXT:  In Loop with Header: for.body11
+; CHECK-NEXT:  AccessFunction: 0
+; CHECK-NEXT:  failed to delinearize
+;
 entry:
-  %0 = load i16, ptr undef, align 2
+  %0 = load i16, ptr %x, align 2
   %conv21 = zext i16 %0 to i32
   br label %for.cond7.preheader
 
 for.cond7.preheader:
-  %p1.022 = phi ptr [ undef, %entry ], [ %add.ptr, %for.end ]
+  %p1.022 = phi ptr [ %x, %entry ], [ %add.ptr, %for.end ]
   br label %for.body11
 
 for.body11:
-  %arrayidx.phi = phi ptr [ %p1.022, %for.cond7.preheader ], [ undef, %for.body11 ]
-  store i8 undef, ptr %arrayidx.phi, align 1
+  %arrayidx.phi = phi ptr [ %p1.022, %for.cond7.preheader ], [ %x, %for.body11 ]
+  store i8 42, ptr %arrayidx.phi, align 1
   br i1 %arg, label %for.body11, label %for.end
 
 for.end:
diff --git a/llvm/test/Analysis/Delinearization/undef.ll b/llvm/test/Analysis/Delinearization/undef.ll
deleted file mode 100644
index 1e62c4579c9c6..0000000000000
--- a/llvm/test/Analysis/Delinearization/undef.ll
+++ /dev/null
@@ -1,38 +0,0 @@
-; RUN: opt < %s -passes='print<delinearization>' -disable-output
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-define void @foo(ptr %Ey, i1 %arg) {
-entry:
-  br i1 %arg, label %for.cond55.preheader, label %for.end324
-
-for.cond55.preheader:
-  %iz.069 = phi i64 [ %inc323, %for.inc322 ], [ 0, %entry ]
-  br i1 %arg, label %for.cond58.preheader, label %for.inc322
-
-for.cond58.preheader:
-  %iy.067 = phi i64 [ %inc320, %for.end ], [ 0, %for.cond55.preheader ]
-  br i1 %arg, label %for.body60, label %for.end
-
-for.body60:
-  %ix.062 = phi i64 [ %inc, %for.body60 ], [ 0, %for.cond58.preheader ]
-  %0 = mul i64 %iz.069, undef
-  %tmp5 = add i64 %iy.067, %0
-  %tmp6 = mul i64 %tmp5, undef
-  %arrayidx69.sum = add i64 undef, %tmp6
-  %arrayidx70 = getelementptr inbounds double, ptr %Ey, i64 %arrayidx69.sum
-  %1 = load double, ptr %arrayidx70, align 8
-  %inc = add nsw i64 %ix.062, 1
-  br i1 false, label %for.body60, label %for.end
-
-for.end:
-  %inc320 = add nsw i64 %iy.067, 1
-  br i1 %arg, label %for.cond58.preheader, label %for.inc322
-
-for.inc322:
-  %inc323 = add nsw i64 %iz.069, 1
-  br i1 %arg, label %for.cond55.preheader, label %for.end324
-
-for.end324:
-  ret void
-}
diff --git a/llvm/test/Analysis/DemandedBits/ashr.ll b/llvm/test/Analysis/DemandedBits/ashr.ll
new file mode 100644
index 0000000000000..6185d4c492d86
--- /dev/null
+++ b/llvm/test/Analysis/DemandedBits/ashr.ll
@@ -0,0 +1,198 @@
+; RUN: opt -S -disable-output -passes="print<demanded-bits>" < %s 2>&1 | FileCheck %s
+
+define i8 @test_ashr_const_amount_4(i32 %a) {
+; CHECK-LABEL: 'test_ashr_const_amount_4'
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr = ashr i32 %a, 4
+; CHECK-DAG:  DemandedBits: 0xff0 for %a in   %ashr = ashr i32 %a, 4
+; CHECK-DAG:  DemandedBits: 0xffffffff for 4 in   %ashr = ashr i32 %a, 4
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr.t = trunc i32 %ashr to i8
+; CHECK-DAG:  DemandedBits: 0xff for %ashr in   %ashr.t = trunc i32 %ashr to i8
+;
+  %ashr = ashr i32 %a, 4
+  %ashr.t = trunc i32 %ashr to i8
+  ret i8 %ashr.t
+}
+
+define i8 @test_ashr_const_amount_5(i32 %a) {
+; CHECK-LABEL: 'test_ashr_const_amount_5'
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr = ashr i32 %a, 5
+; CHECK-DAG:  DemandedBits: 0x1fe0 for %a in   %ashr = ashr i32 %a, 5
+; CHECK-DAG:  DemandedBits: 0xffffffff for 5 in   %ashr = ashr i32 %a, 5
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr.t = trunc i32 %ashr to i8
+; CHECK-DAG:  DemandedBits: 0xff for %ashr in   %ashr.t = trunc i32 %ashr to i8
+;
+  %ashr = ashr i32 %a, 5
+  %ashr.t = trunc i32 %ashr to i8
+  ret i8 %ashr.t
+}
+
+define i8 @test_ashr_const_amount_8(i32 %a) {
+; CHECK-LABEL: 'test_ashr_const_amount_8'
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr = ashr i32 %a, 8
+; CHECK-DAG:  DemandedBits: 0xff00 for %a in   %ashr = ashr i32 %a, 8
+; CHECK-DAG:  DemandedBits: 0xffffffff for 8 in   %ashr = ashr i32 %a, 8
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr.t = trunc i32 %ashr to i8
+; CHECK-DAG:  DemandedBits: 0xff for %ashr in   %ashr.t = trunc i32 %ashr to i8
+;
+  %ashr = ashr i32 %a, 8
+  %ashr.t = trunc i32 %ashr to i8
+  ret i8 %ashr.t
+}
+
+define i8 @test_ashr_const_amount_9(i32 %a) {
+
+; CHECK-LABEL: 'test_ashr_const_amount_9'
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr.t = trunc i32 %ashr to i8
+; CHECK-DAG:  DemandedBits: 0xff for %ashr in   %ashr.t = trunc i32 %ashr to i8
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr = ashr i32 %a, 8
+; CHECK-DAG:  DemandedBits: 0xff00 for %a in   %ashr = ashr i32 %a, 8
+; CHECK-DAG:  DemandedBits: 0xffffffff for 8 in   %ashr = ashr i32 %a, 8
+;
+  %ashr = ashr i32 %a, 8
+  %ashr.t = trunc i32 %ashr to i8
+  ret i8 %ashr.t
+}
+
+define i8 @test_ashr(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_ashr'
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %a in   %ashr = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %ashr = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr.t = trunc i32 %ashr to i8
+; CHECK-DAG:  DemandedBits: 0xff for %ashr in   %ashr.t = trunc i32 %ashr to i8
+;
+  %ashr = ashr i32 %a, %b
+  %ashr.t = trunc i32 %ashr to i8
+  ret i8 %ashr.t
+}
+
+define i8 @test_ashr_range_1(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_ashr_range_1'
+; CHECK-DAG:  DemandedBits: 0xff for   %shl.t = trunc i32 %ashr to i8
+; CHECK-DAG:  DemandedBits: 0xff for %ashr in   %shl.t = trunc i32 %ashr to i8
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0x3 for %b in   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0xffffffff for 3 in   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0xff for   %ashr = ashr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0x7ff for %a in   %ashr = ashr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b2 in   %ashr = ashr i32 %a, %b2
+;
+  %b2 = and i32 %b, 3
+  %ashr = ashr i32 %a, %b2
+  %shl.t = trunc i32 %ashr to i8
+  ret i8 %shl.t
+}
+
+define i32 @test_ashr_range_2(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_ashr_range_2'
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0x3 for %b in   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0xffffffff for 3 in   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %ashr = ashr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for %a in   %ashr = ashr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b2 in   %ashr = ashr i32 %a, %b2
+;
+  %b2 = and i32 %b, 3
+  %ashr = ashr i32 %a, %b2
+  ret i32 %ashr
+}
+
+define i32 @test_ashr_range_3(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_ashr_range_3'
+; CHECK-DAG:  DemandedBits:  0xffff for   %ashr = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits:  0xffffffff for %a in   %ashr = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits:  0xffffffff for %b in   %ashr = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits:  0xffffffff for   %shl = shl i32 %ashr, 16
+; CHECK-DAG:  DemandedBits:  0xffff for %ashr in   %shl = shl i32 %ashr, 16
+; CHECK-DAG:  DemandedBits:  0xffffffff for 16 in   %shl = shl i32 %ashr, 16
+;
+  %ashr = ashr i32 %a, %b
+  %shl = shl i32 %ashr, 16
+  ret i32 %shl
+}
+define i32 @test_ashr_range_4(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_ashr_range_4'
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %shr = lshr i32 %ashr, 8
+; CHECK-DAG:  DemandedBits: 0xffffff00 for %ashr in   %shr = lshr i32 %ashr, 8
+; CHECK-DAG:  DemandedBits: 0xffffffff for 8 in   %shr = lshr i32 %ashr, 8
+; CHECK-DAG:  DemandedBits: 0xffffff00 for   %ashr = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffff00 for %a in   %ashr = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %ashr = ashr i32 %a, %b
+  %ashr = ashr i32 %a, %b
+  %shr = lshr i32 %ashr, 8
+  ret i32 %shr
+}
+
+define i32 @test_ashr_range_5(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_ashr_range_5'
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %2 = and i32 %1, 255
+; CHECK-DAG:  DemandedBits: 0xff for %1 in   %2 = and i32 %1, 255
+; CHECK-DAG:  DemandedBits: 0xffffffff for 255 in   %2 = and i32 %1, 255
+; CHECK-DAG:  DemandedBits: 0xff for   %1 = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %a in   %1 = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %1 = ashr i32 %a, %b
+;
+  %1 = ashr i32 %a, %b
+  %2 = and i32 %1, 255
+  ret i32 %2
+}
+
+define i32 @test_ashr_range_6(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_ashr_range_6'
+; CHECK-DAG:  DemandedBits: 0xffff0000 for   %lshr.1 = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffff0000 for %a in   %lshr.1 = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %lshr.1 = ashr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %lshr.2 = ashr i32 %lshr.1, 16
+; CHECK-DAG:  DemandedBits: 0xffff0000 for %lshr.1 in   %lshr.2 = ashr i32 %lshr.1, 16
+; CHECK-DAG:  DemandedBits: 0xffffffff for 16 in   %lshr.2 = ashr i32 %lshr.1, 16
+;
+  %lshr.1 = ashr i32 %a, %b
+  %lshr.2 = ashr i32 %lshr.1, 16
+  ret i32 %lshr.2
+}
+
+define i8 @test_ashr_var_amount(i32 %a, i32 %b){
+; CHECK-LABEL: 'test_ashr_var_amount'
+; CHECK-DAG: DemandedBits: 0xff for   %4 = ashr i32 %1, %3
+; CHECK-DAG: DemandedBits: 0xffffffff for %1 in   %4 = ashr i32 %1, %3
+; CHECK-DAG: DemandedBits: 0xffffffff for %3 in   %4 = ashr i32 %1, %3
+; CHECK-DAG: DemandedBits: 0xff for   %2 = trunc i32 %1 to i8
+; CHECK-DAG: DemandedBits: 0xff for %1 in   %2 = trunc i32 %1 to i8
+; CHECK-DAG: DemandedBits: 0xffffffff for   %1 = add nsw i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffffffff for %a in   %1 = add nsw i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffffffff for %b in   %1 = add nsw i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffffffff for   %3 = zext i8 %2 to i32
+; CHECK-DAG: DemandedBits: 0xff for %2 in   %3 = zext i8 %2 to i32
+; CHECK-DAG: DemandedBits: 0xff for   %5 = trunc i32 %4 to i8
+; CHECK-DAG: DemandedBits: 0xff for %4 in   %5 = trunc i32 %4 to i8
+;
+  %1 = add nsw i32 %a, %b
+  %2 = trunc i32 %1 to i8
+  %3 = zext i8 %2 to i32
+  %4 = ashr i32 %1, %3
+  %5 = trunc i32 %4 to i8
+  ret i8 %5
+}
+
+define i8 @test_ashr_var_amount_nsw(i32 %a, i32 %b){
+ ; CHECK-LABEL 'test_ashr_var_amount_nsw'
+ ; CHECK-DAG: DemandedBits: 0xff for   %5 = trunc i32 %4 to i8
+ ; CHECK-DAG: DemandedBits: 0xff for %4 in   %5 = trunc i32 %4 to i8
+ ; CHECK-DAG: DemandedBits: 0xffffffff for   %1 = add nsw i32 %a, %b
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %a in   %1 = add nsw i32 %a, %b
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %b in   %1 = add nsw i32 %a, %b
+ ; CHECK-DAG: DemandedBits: 0xff for   %2 = trunc i32 %1 to i8
+ ; CHECK-DAG: DemandedBits: 0xff for %1 in   %2 = trunc i32 %1 to i8
+ ; CHECK-DAG: DemandedBits: 0xffffffff for   %3 = zext i8 %2 to i32
+ ; CHECK-DAG: DemandedBits: 0xff for %2 in   %3 = zext i8 %2 to i32
+ ; CHECK-DAG: DemandedBits: 0xff for   %4 = ashr exact i32 %1, %3
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %1 in   %4 = ashr exact i32 %1, %3
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %3 in   %4 = ashr exact i32 %1, %3
+ ;
+  %1 = add nsw i32 %a, %b
+  %2 = trunc i32 %1 to i8
+  %3 = zext i8 %2 to i32
+  %4 = ashr exact i32 %1, %3
+  %5 = trunc i32 %4 to i8
+  ret i8 %5
+}
diff --git a/llvm/test/Analysis/DemandedBits/lshr.ll b/llvm/test/Analysis/DemandedBits/lshr.ll
new file mode 100644
index 0000000000000..e07f994a1b304
--- /dev/null
+++ b/llvm/test/Analysis/DemandedBits/lshr.ll
@@ -0,0 +1,198 @@
+; RUN: opt -S -disable-output -passes="print<demanded-bits>" < %s 2>&1 | FileCheck %s
+
+define i8 @test_lshr_const_amount_4(i32 %a) {
+; CHECK-LABEL: 'test_lshr_const_amount_4'
+; CHECK-DAG: DemandedBits: 0xff for   %lshr.t = trunc i32 %lshr to i8
+; CHECK-DAG: DemandedBits: 0xff for %lshr in   %lshr.t = trunc i32 %lshr to i8
+; CHECK-DAG: DemandedBits: 0xff for   %lshr = lshr i32 %a, 4
+; CHECK-DAG: DemandedBits: 0xff0 for %a in   %lshr = lshr i32 %a, 4
+; CHECK-DAG: DemandedBits: 0xffffffff for 4 in   %lshr = lshr i32 %a, 4
+;
+  %lshr = lshr i32 %a, 4
+  %lshr.t = trunc i32 %lshr to i8
+  ret i8 %lshr.t
+}
+
+define i8 @test_lshr_const_amount_5(i32 %a) {
+; CHECK-LABEL: 'test_lshr_const_amount_5'
+; CHECK-DAG: DemandedBits: 0xff for   %lshr = lshr i32 %a, 5
+; CHECK-DAG: DemandedBits: 0x1fe0 for %a in   %lshr = lshr i32 %a, 5
+; CHECK-DAG: DemandedBits: 0xffffffff for 5 in   %lshr = lshr i32 %a, 5
+; CHECK-DAG: DemandedBits: 0xff for   %lshr.t = trunc i32 %lshr to i8
+; CHECK-DAG: DemandedBits: 0xff for %lshr in   %lshr.t = trunc i32 %lshr to i8
+;
+  %lshr = lshr i32 %a, 5
+  %lshr.t = trunc i32 %lshr to i8
+  ret i8 %lshr.t
+}
+define i8 @test_lshr_const_amount_8(i32 %a) {
+; CHECK-LABEL: 'test_lshr_const_amount_8'
+; CHECK-DAG: DemandedBits: 0xff for   %lshr.t = trunc i32 %lshr to i8
+; CHECK-DAG: DemandedBits: 0xff for %lshr in   %lshr.t = trunc i32 %lshr to i8
+; CHECK-DAG: DemandedBits: 0xff for   %lshr = lshr i32 %a, 8
+; CHECK-DAG: DemandedBits: 0xff00 for %a in   %lshr = lshr i32 %a, 8
+; CHECK-DAG: DemandedBits: 0xffffffff for 8 in   %lshr = lshr i32 %a, 8
+;
+  %lshr = lshr i32 %a, 8
+  %lshr.t = trunc i32 %lshr to i8
+  ret i8 %lshr.t
+}
+
+define i8 @test_lshr_const_amount_9(i32 %a) {
+; CHECK-LABEL: 'test_lshr_const_amount_9'
+; CHECK-DAG: DemandedBits: 0xff for   %lshr.t = trunc i32 %lshr to i8
+; CHECK-DAG: DemandedBits: 0xff for %lshr in   %lshr.t = trunc i32 %lshr to i8
+; CHECK-DAG: DemandedBits: 0xff for   %lshr = lshr i32 %a, 9
+; CHECK-DAG: DemandedBits: 0x1fe00 for %a in   %lshr = lshr i32 %a, 9
+; CHECK-DAG: DemandedBits: 0xffffffff for 9 in   %lshr = lshr i32 %a, 9
+;
+  %lshr = lshr i32 %a, 9
+  %lshr.t = trunc i32 %lshr to i8
+  ret i8 %lshr.t
+}
+
+define i8 @test_lshr(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_lshr'
+; CHECK-DAG: DemandedBits: 0xff for   %lshr = lshr i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffffffff for %a in   %lshr = lshr i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffffffff for %b in   %lshr = lshr i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xff for   %lshr.t = trunc i32 %lshr to i8
+; CHECK-DAG: DemandedBits: 0xff for %lshr in   %lshr.t = trunc i32 %lshr to i8
+;
+  %lshr = lshr i32 %a, %b
+  %lshr.t = trunc i32 %lshr to i8
+  ret i8 %lshr.t
+}
+
+define i8 @test_lshr_range_1(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_lshr_range_1'
+; CHECK-DAG:  DemandedBits: 0xff for   %shl.t = trunc i32 %lshr to i8
+; CHECK-DAG:  DemandedBits: 0xff for %lshr in   %shl.t = trunc i32 %lshr to i8
+; CHECK-DAG:  DemandedBits: 0xff for   %lshr = lshr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0x7ff for %a in   %lshr = lshr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b2 in   %lshr = lshr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0x3 for %b in   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0xffffffff for 3 in   %b2 = and i32 %b, 3
+;
+  %b2 = and i32 %b, 3
+  %lshr = lshr i32 %a, %b2
+  %shl.t = trunc i32 %lshr to i8
+  ret i8 %shl.t
+}
+
+define i32 @test_lshr_range_2(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_lshr_range_2'
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %lshr = lshr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for %a in   %lshr = lshr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b2 in   %lshr = lshr i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0x3 for %b in   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0xffffffff for 3 in   %b2 = and i32 %b, 3
+;
+  %b2 = and i32 %b, 3
+  %lshr = lshr i32 %a, %b2
+  ret i32 %lshr
+}
+
+define i32 @test_lshr_range_3(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_lshr_range_3'
+; CHECK-DAG:  DemandedBits: 0xffff for   %lshr = lshr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %a in   %lshr = lshr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %lshr = lshr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %shl = shl i32 %lshr, 16
+; CHECK-DAG:  DemandedBits: 0xffff for %lshr in   %shl = shl i32 %lshr, 16
+; CHECK-DAG:  DemandedBits: 0xffffffff for 16 in   %shl = shl i32 %lshr, 16
+;
+  %lshr = lshr i32 %a, %b
+  %shl = shl i32 %lshr, 16
+  ret i32 %shl
+}
+
+define i32 @test_lshr_range_4(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_lshr_range_4'
+; CHECK-DAG:  DemandedBits: 0xffffff00 for   %lshr = lshr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffff00 for %a in   %lshr = lshr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %lshr = lshr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %shr = ashr i32 %lshr, 8
+; CHECK-DAG:  DemandedBits: 0xffffff00 for %lshr in   %shr = ashr i32 %lshr, 8
+; CHECK-DAG:  DemandedBits: 0xffffffff for 8 in   %shr = ashr i32 %lshr, 8
+  %lshr = lshr i32 %a, %b
+  %shr = ashr i32 %lshr, 8
+  ret i32 %shr
+}
+
+define i32 @test_lshr_range_5(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_lshr_range_5'
+; CHECK-DAG:  DemandedBits: 0xff for   %1 = lshr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %a in   %1 = lshr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %1 = lshr i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %2 = and i32 %1, 255
+; CHECK-DAG:  DemandedBits: 0xff for %1 in   %2 = and i32 %1, 255
+; CHECK-DAG:  DemandedBits: 0xffffffff for 255 in   %2 = and i32 %1, 255
+;
+  %1 = lshr i32 %a, %b
+  %2 = and i32 %1, 255
+  ret i32 %2
+}
+
+define i32 @test_lshr_range_6(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_lshr_range_6'
+; CHECK-DAG: DemandedBits: 0xffff0000 for   %lshr.1 = lshr i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffff0000 for %a in   %lshr.1 = lshr i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffffffff for %b in   %lshr.1 = lshr i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffffffff for   %lshr.2 = lshr i32 %lshr.1, 16
+; CHECK-DAG: DemandedBits: 0xffff0000 for %lshr.1 in   %lshr.2 = lshr i32 %lshr.1, 16
+; CHECK-DAG: DemandedBits: 0xffffffff for 16 in   %lshr.2 = lshr i32 %lshr.1, 16
+;
+  %lshr.1 = lshr i32 %a, %b
+  %lshr.2 = lshr i32 %lshr.1, 16
+  ret i32 %lshr.2
+}
+
+
+define i8 @test_lshr_var_amount(i32 %a, i32 %b){
+; CHECK-LABEL: 'test_lshr_var_amount'
+; CHECK-DAG: DemandedBits: 0xff for   %4 = lshr i32 %1, %3
+; CHECK-DAG: DemandedBits: 0xffffffff for %1 in   %4 = lshr i32 %1, %3
+; CHECK-DAG: DemandedBits: 0xffffffff for %3 in   %4 = lshr i32 %1, %3
+; CHECK-DAG: DemandedBits: 0xff for   %5 = trunc i32 %4 to i8
+; CHECK-DAG: DemandedBits: 0xff for %4 in   %5 = trunc i32 %4 to i8
+; CHECK-DAG: DemandedBits: 0xffffffff for   %3 = zext i8 %2 to i32
+; CHECK-DAG: DemandedBits: 0xff for %2 in   %3 = zext i8 %2 to i32
+; CHECK-DAG: DemandedBits: 0xffffffff for   %1 = add nsw i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffffffff for %a in   %1 = add nsw i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xffffffff for %b in   %1 = add nsw i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xff for   %2 = trunc i32 %1 to i8
+; CHECK-DAG: DemandedBits: 0xff for %1 in   %2 = trunc i32 %1 to i8
+;
+  %1 = add nsw i32 %a, %b
+  %2 = trunc i32 %1 to i8
+  %3 = zext i8 %2 to i32
+  %4 = lshr i32 %1, %3
+  %5 = trunc i32 %4 to i8
+  ret i8 %5
+}
+
+define i8 @test_lshr_var_amount_exact(i32 %a, i32 %b){
+ ; CHECK-LABEL 'test_lshr_var_amount_nsw'
+ ; CHECK-DAG: DemandedBits: 0xffffffff for   %1 = add nsw i32 %a, %b
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %a in   %1 = add nsw i32 %a, %b
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %b in   %1 = add nsw i32 %a, %b
+ ; CHECK-DAG: DemandedBits: 0xff for   %2 = trunc i32 %1 to i8
+ ; CHECK-DAG: DemandedBits: 0xff for %1 in   %2 = trunc i32 %1 to i8
+ ; CHECK-DAG: DemandedBits: 0xffffffff for   %3 = zext i8 %2 to i32
+ ; CHECK-DAG: DemandedBits: 0xff for %2 in   %3 = zext i8 %2 to i32
+ ; CHECK-DAG: DemandedBits: 0xff for   %4 = lshr exact i32 %1, %3
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %1 in   %4 = lshr exact i32 %1, %3
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %3 in   %4 = lshr exact i32 %1, %3
+ ; CHECK-DAG: DemandedBits: 0xff for   %5 = trunc i32 %4 to i8
+ ; CHECK-DAG: DemandedBits: 0xff for %4 in   %5 = trunc i32 %4 to i8
+ ;
+  %1 = add nsw i32 %a, %b
+  %2 = trunc i32 %1 to i8
+  %3 = zext i8 %2 to i32
+  %4 = lshr exact i32 %1, %3
+  %5 = trunc i32 %4 to i8
+  ret i8 %5
+}
diff --git a/llvm/test/Analysis/DemandedBits/shl.ll b/llvm/test/Analysis/DemandedBits/shl.ll
index e41f5f4107735..c872d2d854e83 100644
--- a/llvm/test/Analysis/DemandedBits/shl.ll
+++ b/llvm/test/Analysis/DemandedBits/shl.ll
@@ -57,10 +57,142 @@ define i8 @test_shl(i32 %a, i32 %b) {
 ; CHECK-DAG:  DemandedBits: 0xff for %shl.t = trunc i32 %shl to i8
 ; CHECK-DAG:  DemandedBits: 0xff for %shl in %shl.t = trunc i32 %shl to i8
 ; CHECK-DAG:  DemandedBits: 0xff for %shl = shl i32 %a, %b
-; CHECK-DAG:  DemandedBits: 0xffffffff for %a in %shl = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xff for %a in %shl = shl i32 %a, %b
 ; CHECK-DAG:  DemandedBits: 0xffffffff for %b in %shl = shl i32 %a, %b
 ;
   %shl = shl i32 %a, %b
   %shl.t = trunc i32 %shl to i8
   ret i8 %shl.t
 }
+
+define i8 @test_shl_range_1(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_shl_range_1'
+; CHECK-DAG:  DemandedBits: 0xff for   %shl = shl i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xff for %a in   %shl = shl i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b2 in   %shl = shl i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xff for   %shl.t = trunc i32 %shl to i8
+; CHECK-DAG:  DemandedBits: 0xff for %shl in   %shl.t = trunc i32 %shl to i8
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0x3 for %b in   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0xffffffff for 3 in   %b2 = and i32 %b, 3
+;
+  %b2 = and i32 %b, 3
+  %shl = shl i32 %a, %b2
+  %shl.t = trunc i32 %shl to i8
+  ret i8 %shl.t
+}
+
+define i32 @test_shl_range_2(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_shl_range_2'
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0x3 for %b in   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0xffffffff for 3 in   %b2 = and i32 %b, 3
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %shl = shl i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for %a in   %shl = shl i32 %a, %b2
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b2 in   %shl = shl i32 %a, %b2
+;
+  %b2 = and i32 %b, 3
+  %shl = shl i32 %a, %b2
+  ret i32 %shl
+}
+
+define i32 @test_shl_range_3(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_shl_range_3'
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %shr = lshr i32 %shl, 16
+; CHECK-DAG:  DemandedBits: 0xffff0000 for %shl in   %shr = lshr i32 %shl, 16
+; CHECK-DAG:  DemandedBits: 0xffffffff for 16 in   %shr = lshr i32 %shl, 16
+; CHECK-DAG:  DemandedBits: 0xffff0000 for   %shl = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %a in   %shl = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %shl = shl i32 %a, %b
+;
+  %shl = shl i32 %a, %b
+  %shr = lshr i32 %shl, 16
+  ret i32 %shr
+}
+
+define i32 @test_shl_range_4(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_shl_range_4'
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %shr = ashr i32 %shl, 8
+; CHECK-DAG:  DemandedBits: 0xffffff00 for %shl in   %shr = ashr i32 %shl, 8
+; CHECK-DAG:  DemandedBits: 0xffffffff for 8 in   %shr = ashr i32 %shl, 8
+; CHECK-DAG:  DemandedBits: 0xffffff00 for   %shl = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %a in   %shl = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %shl = shl i32 %a, %b
+  %shl = shl i32 %a, %b
+  %shr = ashr i32 %shl, 8
+  ret i32 %shr
+}
+
+define i32 @test_shl_range_5(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_shl_range_5'
+; CHECK-DAG:  DemandedBits: 0xff for   %1 = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xff for %a in   %1 = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %1 = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %2 = and i32 %1, 255
+; CHECK-DAG:  DemandedBits: 0xff for %1 in   %2 = and i32 %1, 255
+; CHECK-DAG:  DemandedBits: 0xffffffff for 255 in   %2 = and i32 %1, 255
+;
+  %1 = shl i32 %a, %b
+  %2 = and i32 %1, 255
+  ret i32 %2
+}
+
+define i32 @test_shl_range_6(i32 %a, i32 %b) {
+; CHECK-LABEL: 'test_shl_range_6'
+; CHECK-DAG:  DemandedBits: 0xffffffff for   %shl.2 = shl i32 %shl.1, 16
+; CHECK-DAG:  DemandedBits: 0xffff for %shl.1 in   %shl.2 = shl i32 %shl.1, 16
+; CHECK-DAG:  DemandedBits: 0xffffffff for 16 in   %shl.2 = shl i32 %shl.1, 16
+; CHECK-DAG:  DemandedBits: 0xffff for   %shl.1 = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffff for %a in   %shl.1 = shl i32 %a, %b
+; CHECK-DAG:  DemandedBits: 0xffffffff for %b in   %shl.1 = shl i32 %a, %b
+;
+  %shl.1 = shl i32 %a, %b
+  %shl.2 = shl i32 %shl.1, 16
+  ret i32 %shl.2
+}
+
+define i8 @test_shl_var_amount(i32 %a, i32 %b){
+; CHECK-LABEL: 'test_shl_var_amount'
+; CHECK-DAG: DemandedBits: 0xff for   %5 = trunc i32 %4 to i8
+; CHECK-DAG: DemandedBits: 0xff for %4 in   %5 = trunc i32 %4 to i8
+; CHECK-DAG: DemandedBits: 0xff for   %4 = shl i32 %1, %3
+; CHECK-DAG: DemandedBits: 0xff for %1 in   %4 = shl i32 %1, %3
+; CHECK-DAG: DemandedBits: 0xffffffff for %3 in   %4 = shl i32 %1, %3
+; CHECK-DAG: DemandedBits: 0xff for   %2 = trunc i32 %1 to i8
+; CHECK-DAG: DemandedBits: 0xff for %1 in   %2 = trunc i32 %1 to i8
+; CHECK-DAG: DemandedBits: 0xffffffff for   %3 = zext i8 %2 to i32
+; CHECK-DAG: DemandedBits: 0xff for %2 in   %3 = zext i8 %2 to i32
+; CHECK-DAG: DemandedBits: 0xff for   %1 = add nsw i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xff for %a in   %1 = add nsw i32 %a, %b
+; CHECK-DAG: DemandedBits: 0xff for %b in   %1 = add nsw i32 %a, %b
+;
+  %1 = add nsw i32 %a, %b
+  %2 = trunc i32 %1 to i8
+  %3 = zext i8 %2 to i32
+  %4 = shl i32 %1, %3
+  %5 = trunc i32 %4 to i8
+  ret i8 %5
+}
+
+define i8 @test_shl_var_amount_nsw(i32 %a, i32 %b){
+ ; CHECK-LABEL 'test_shl_var_amount_nsw'
+ ; CHECK-DAG: DemandedBits: 0xff for   %5 = trunc i32 %4 to i8
+ ; CHECK-DAG: DemandedBits: 0xff for %4 in   %5 = trunc i32 %4 to i8
+ ; CHECK-DAG: DemandedBits: 0xff for   %4 = shl nsw i32 %1, %3
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %1 in   %4 = shl nsw i32 %1, %3
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %3 in   %4 = shl nsw i32 %1, %3
+ ; CHECK-DAG: DemandedBits: 0xffffffff for   %3 = zext i8 %2 to i32
+ ; CHECK-DAG: DemandedBits: 0xff for %2 in   %3 = zext i8 %2 to i32
+ ; CHECK-DAG: DemandedBits: 0xff for   %2 = trunc i32 %1 to i8
+ ; CHECK-DAG: DemandedBits: 0xff for %1 in   %2 = trunc i32 %1 to i8
+ ; CHECK-DAG: DemandedBits: 0xffffffff for   %1 = add nsw i32 %a, %b
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %a in   %1 = add nsw i32 %a, %b
+ ; CHECK-DAG: DemandedBits: 0xffffffff for %b in   %1 = add nsw i32 %a, %b
+ ;
+  %1 = add nsw i32 %a, %b
+  %2 = trunc i32 %1 to i8
+  %3 = zext i8 %2 to i32
+  %4 = shl nsw i32 %1, %3
+  %5 = trunc i32 %4 to i8
+  ret i8 %5
+}
diff --git a/llvm/test/Analysis/DependenceAnalysis/Coupled.ll b/llvm/test/Analysis/DependenceAnalysis/Coupled.ll
index 06bfc5d2e8573..1d4513429a83c 100644
--- a/llvm/test/Analysis/DependenceAnalysis/Coupled.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/Coupled.ll
@@ -719,12 +719,14 @@ for.end:                                          ; preds = %for.body
 ;;    for(int j = 0; j < M; j+=1)
 ;;      A[M*N + M*i + j] = 2;
 
+; FIXME: Currently failing to infer %M being positive.
+
 define void @couple_weakzerosiv(ptr noalias nocapture %A, i64 %N, i64 %M) {
 ; CHECK-LABEL: 'couple_weakzerosiv'
 ; CHECK-NEXT:  Src: store i32 1, ptr %arrayidx.us, align 4 --> Dst: store i32 1, ptr %arrayidx.us, align 4
 ; CHECK-NEXT:    da analyze - none!
 ; CHECK-NEXT:  Src: store i32 1, ptr %arrayidx.us, align 4 --> Dst: store i32 2, ptr %arrayidx9.us, align 4
-; CHECK-NEXT:    da analyze - output [p>]!
+; CHECK-NEXT:    da analyze - output [*|<]!
 ; CHECK-NEXT:  Src: store i32 2, ptr %arrayidx9.us, align 4 --> Dst: store i32 2, ptr %arrayidx9.us, align 4
 ; CHECK-NEXT:    da analyze - none!
 ;
diff --git a/llvm/test/Analysis/DependenceAnalysis/DADelin.ll b/llvm/test/Analysis/DependenceAnalysis/DADelin.ll
index b2e4959a7812e..8f94a455d3724 100644
--- a/llvm/test/Analysis/DependenceAnalysis/DADelin.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/DADelin.ll
@@ -594,14 +594,15 @@ for.end12:                                        ; preds = %for.inc10, %entry
 }
 
 
+; FIXME? It seems that we cannot prove that %N is non-negative...
 define void @nonnegative(ptr nocapture %A, i32 %N) {
 ; CHECK-LABEL: 'nonnegative'
 ; CHECK-NEXT:  Src: store i32 1, ptr %arrayidx, align 4 --> Dst: store i32 1, ptr %arrayidx, align 4
-; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:    da analyze - output [* *]!
 ; CHECK-NEXT:  Src: store i32 1, ptr %arrayidx, align 4 --> Dst: store i32 2, ptr %arrayidx, align 4
-; CHECK-NEXT:    da analyze - consistent output [0 0|<]!
+; CHECK-NEXT:    da analyze - output [* *|<]!
 ; CHECK-NEXT:  Src: store i32 2, ptr %arrayidx, align 4 --> Dst: store i32 2, ptr %arrayidx, align 4
-; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:    da analyze - output [* *]!
 ;
 entry:
   %cmp44 = icmp eq i32 %N, 0
@@ -630,3 +631,81 @@ for.latch:
 exit:
   ret void
 }
+
+; i = 0;
+; do {
+;   a[k * i] = 42;
+;   a[k * (i + 1)] = 42;
+;   i++;
+; } while (i != k);
+;
+; The dependency direction between the two stores depends on the sign of k.
+; Note that the loop guard is omitted intentionally.
+; FIXME: Each store has loop-carried dependencies on itself if k is zero.
+;
+define void @coeff_may_negative(ptr %a, i32 %k) {
+; CHECK-LABEL: 'coeff_may_negative'
+; CHECK-NEXT:  Src: store i8 42, ptr %idx.0, align 1 --> Dst: store i8 42, ptr %idx.0, align 1
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i8 42, ptr %idx.0, align 1 --> Dst: store i8 42, ptr %idx.1, align 1
+; CHECK-NEXT:    da analyze - output [*|<]!
+; CHECK-NEXT:  Src: store i8 42, ptr %idx.1, align 1 --> Dst: store i8 42, ptr %idx.1, align 1
+; CHECK-NEXT:    da analyze - none!
+;
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %i.next = add i32 %i, 1
+  %subscript.0 = mul i32 %i, %k
+  %subscript.1 = mul i32 %i.next, %k
+  %idx.0 = getelementptr i8, ptr %a, i32 %subscript.0
+  %idx.1 = getelementptr i8, ptr %a, i32 %subscript.1
+  store i8 42, ptr %idx.0
+  store i8 42, ptr %idx.1
+  %cond.exit = icmp eq i32 %i.next, %k
+  br i1 %cond.exit, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; i = 0;
+; do {
+;   a[k * i] = 42;
+;   a[k * (i + 1)] = 42;
+;   i++;
+; } while (i != k);
+;
+; Note that the loop guard is omitted intentionally.
+; FIXME: In principle, we can infer that the value of k is non-negative from
+; the nsw flag.
+;
+define void @coeff_positive(ptr %a, i32 %k) {
+; CHECK-LABEL: 'coeff_positive'
+; CHECK-NEXT:  Src: store i8 42, ptr %idx.0, align 1 --> Dst: store i8 42, ptr %idx.0, align 1
+; CHECK-NEXT:    da analyze - none!
+; CHECK-NEXT:  Src: store i8 42, ptr %idx.0, align 1 --> Dst: store i8 42, ptr %idx.1, align 1
+; CHECK-NEXT:    da analyze - output [*|<]!
+; CHECK-NEXT:  Src: store i8 42, ptr %idx.1, align 1 --> Dst: store i8 42, ptr %idx.1, align 1
+; CHECK-NEXT:    da analyze - none!
+;
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %i.next = add nsw i32 %i, 1
+  %subscript.0 = mul i32 %i, %k
+  %subscript.1 = mul i32 %i.next, %k
+  %idx.0 = getelementptr i8, ptr %a, i32 %subscript.0
+  %idx.1 = getelementptr i8, ptr %a, i32 %subscript.1
+  store i8 42, ptr %idx.0
+  store i8 42, ptr %idx.1
+  %cond.exit = icmp eq i32 %i.next, %k
+  br i1 %cond.exit, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json b/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json
index 9b38f2eb6bb4a..07fde84c1541b 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json
+++ b/llvm/test/Analysis/IR2Vec/Inputs/dummy_2D_vocab.json
@@ -47,6 +47,7 @@
         "FPTrunc": [89, 90],
         "FPExt": [91, 92],
         "PtrToInt": [93, 94],
+        "PtrToAddr": [135, 136],
         "IntToPtr": [95, 96],
         "BitCast": [97, 98],
         "AddrSpaceCast": [99, 100],
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/reference_default_vocab_print.txt b/llvm/test/Analysis/IR2Vec/Inputs/reference_default_vocab_print.txt
index 79fcf820d6a57..1b9b3c2acd8a5 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/reference_default_vocab_print.txt
+++ b/llvm/test/Analysis/IR2Vec/Inputs/reference_default_vocab_print.txt
@@ -45,6 +45,7 @@ Key: SIToFP:  [ 87.00  88.00 ]
 Key: FPTrunc:  [ 89.00  90.00 ]
 Key: FPExt:  [ 91.00  92.00 ]
 Key: PtrToInt:  [ 93.00  94.00 ]
+Key: PtrToAddr:  [ 135.00  136.00 ]
 Key: IntToPtr:  [ 95.00  96.00 ]
 Key: BitCast:  [ 97.00  98.00 ]
 Key: AddrSpaceCast:  [ 99.00  100.00 ]
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd1_vocab_print.txt b/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd1_vocab_print.txt
index 584bd315117c8..9673e7f23fa5c 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd1_vocab_print.txt
+++ b/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd1_vocab_print.txt
@@ -45,6 +45,7 @@ Key: SIToFP:  [ 43.50  44.00 ]
 Key: FPTrunc:  [ 44.50  45.00 ]
 Key: FPExt:  [ 45.50  46.00 ]
 Key: PtrToInt:  [ 46.50  47.00 ]
+Key: PtrToAddr:  [ 67.50  68.00 ]
 Key: IntToPtr:  [ 47.50  48.00 ]
 Key: BitCast:  [ 48.50  49.00 ]
 Key: AddrSpaceCast:  [ 49.50  50.00 ]
diff --git a/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd2_vocab_print.txt b/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd2_vocab_print.txt
index 2727c85075efe..1f575d29092dd 100644
--- a/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd2_vocab_print.txt
+++ b/llvm/test/Analysis/IR2Vec/Inputs/reference_wtd2_vocab_print.txt
@@ -45,6 +45,7 @@ Key: SIToFP:  [ 8.70  8.80 ]
 Key: FPTrunc:  [ 8.90  9.00 ]
 Key: FPExt:  [ 9.10  9.20 ]
 Key: PtrToInt:  [ 9.30  9.40 ]
+Key: PtrToAddr:  [ 13.50  13.60 ]
 Key: IntToPtr:  [ 9.50  9.60 ]
 Key: BitCast:  [ 9.70  9.80 ]
 Key: AddrSpaceCast:  [ 9.90  10.00 ]
diff --git a/llvm/test/Analysis/KernelInfo/openmp/nvptx.ll b/llvm/test/Analysis/KernelInfo/openmp/nvptx.ll
index bd46741b24e8c..da5a89897cacb 100644
--- a/llvm/test/Analysis/KernelInfo/openmp/nvptx.ll
+++ b/llvm/test/Analysis/KernelInfo/openmp/nvptx.ll
@@ -417,7 +417,7 @@ define internal noundef range(i32 -1, 1024) i32 @__kmpc_target_init(ptr nofree n
   br label %116
 
 116:                                              ; preds = %110, %128
-  call void @llvm.lifetime.start.p0(i64 noundef 8, ptr noundef nonnull align 8 dereferenceable(8) %3) #20
+  call void @llvm.lifetime.start.p0(ptr noundef nonnull align 8 dereferenceable(8) %3) #20
   tail call void @llvm.nvvm.barrier.sync(i32 noundef 8)
   %117 = call zeroext i1 @__kmpc_kernel_parallel(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) %3) #20
   %118 = load ptr, ptr %3, align 8, !tbaa !93
@@ -446,11 +446,11 @@ define internal noundef range(i32 -1, 1024) i32 @__kmpc_target_init(ptr nofree n
 
 128:                                              ; preds = %126, %120
   tail call void @llvm.nvvm.barrier.sync(i32 noundef 8)
-  call void @llvm.lifetime.end.p0(i64 noundef 8, ptr noundef nonnull %3) #20
+  call void @llvm.lifetime.end.p0(ptr noundef nonnull %3) #20
   br label %116, !llvm.loop !94
 
 129:                                              ; preds = %116
-  call void @llvm.lifetime.end.p0(i64 noundef 8, ptr noundef nonnull %3) #20
+  call void @llvm.lifetime.end.p0(ptr noundef nonnull %3) #20
   br label %130
 
 130:                                              ; preds = %106, %129, %100, %98
@@ -495,7 +495,7 @@ define internal fastcc void @__assert_fail_internal(ptr noundef nonnull derefere
 declare void @llvm.assume(i1 noundef) #9
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #10
+declare void @llvm.lifetime.start.p0(ptr nocapture) #10
 
 ; Function Attrs: convergent nocallback nounwind
 declare void @llvm.nvvm.barrier.sync(i32) #11
@@ -587,7 +587,7 @@ define internal void @__kmpc_kernel_end_parallel() local_unnamed_addr #13 {
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #10
+declare void @llvm.lifetime.end.p0(ptr nocapture) #10
 
 ; Function Attrs: convergent mustprogress nounwind willreturn allockind("free") memory(argmem: readwrite, inaccessiblemem: readwrite)
 declare extern_weak void @free(ptr allocptr nocapture noundef) local_unnamed_addr #14
@@ -595,11 +595,11 @@ declare extern_weak void @free(ptr allocptr nocapture noundef) local_unnamed_add
 ; Function Attrs: convergent mustprogress nounwind
 define internal noundef i32 @_ZN4ompx6printfEPKcz(ptr noundef %0, ...) local_unnamed_addr #15 {
   %2 = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 noundef 8, ptr noundef nonnull align 8 %2) #29
+  call void @llvm.lifetime.start.p0(ptr noundef nonnull align 8 %2) #29
   call void @llvm.va_start.p0(ptr noundef nonnull align 8 %2) #27
   %3 = load ptr, ptr %2, align 8, !tbaa !101
   %4 = call i32 @vprintf(ptr noundef %0, ptr noundef %3) #24
-  call void @llvm.lifetime.end.p0(i64 noundef 8, ptr noundef nonnull %2) #20
+  call void @llvm.lifetime.end.p0(ptr noundef nonnull %2) #20
   ret i32 %4
 }
 
@@ -641,7 +641,7 @@ define internal void @__kmpc_target_deinit() #4 {
   br i1 %14, label %15, label %27
 
 15:                                               ; preds = %11
-  call void @llvm.lifetime.start.p0(i64 noundef 8, ptr noundef nonnull align 8 dereferenceable(8) %1) #29
+  call void @llvm.lifetime.start.p0(ptr noundef nonnull align 8 dereferenceable(8) %1) #29
   %16 = call zeroext i1 @__kmpc_kernel_parallel(ptr noalias nocapture nofree noundef nonnull writeonly align 8 dereferenceable(8) %1) #20
   %17 = load i32, ptr @__omp_rtl_debug_kind, align 4, !tbaa !62
   %18 = load i32, ptr addrspace(4) @__omp_rtl_device_environment, align 8, !tbaa !83
@@ -659,7 +659,7 @@ define internal void @__kmpc_target_deinit() #4 {
 
 26:                                               ; preds = %15
   tail call void @llvm.assume(i1 noundef %23) #23
-  call void @llvm.lifetime.end.p0(i64 noundef 8, ptr noundef nonnull %1) #20
+  call void @llvm.lifetime.end.p0(ptr noundef nonnull %1) #20
   br label %27
 
 27:                                               ; preds = %26, %11, %10, %0
diff --git a/llvm/test/Analysis/LazyValueAnalysis/invalidation.ll b/llvm/test/Analysis/LazyValueAnalysis/invalidation.ll
index 71ea5d2ec1dae..0ad1a337f5235 100644
--- a/llvm/test/Analysis/LazyValueAnalysis/invalidation.ll
+++ b/llvm/test/Analysis/LazyValueAnalysis/invalidation.ll
@@ -17,13 +17,13 @@ target triple = "x86_64-unknown-linux-gnu"
 
 @.str = private unnamed_addr constant [8 x i8] c"a = %l\0A\00", align 1
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @hoo(ptr)
 
 declare i32 @printf(ptr nocapture readonly, ...)
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @goo(i32 %N, ptr %b) {
 entry:
@@ -38,12 +38,12 @@ for.cond:                                         ; preds = %for.body, %entry
   br i1 %cmp, label %for.body, label %for.end
 
 for.body:                                         ; preds = %for.cond
-  call void @llvm.lifetime.start.p0(i64 8, ptr %tmp)
+  call void @llvm.lifetime.start.p0(ptr %tmp)
   call void @hoo(ptr %a.i)
   call void @hoo(ptr %c)
   %tmp1 = load volatile i64, ptr %a.i, align 8
   %call.i = call i32 (ptr, ...) @printf(ptr @.str, i64 %tmp1)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %tmp)
+  call void @llvm.lifetime.end.p0(ptr %tmp)
   %inc = add nsw i32 %i.0, 1
   br label %for.cond
 
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types.ll b/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types.ll
index 1bf80489b1ea0..023a8c056968f 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types.ll
@@ -262,3 +262,301 @@ loop:
 exit:
   ret void
 }
+
+;       i16       i32
+; [ . . 0 0 . . 1 1] [ 1 1 0 0 . . 1 1 ]
+;  ^~~^ gep i8 = 1
+;  ^ ~~ ^ iv.2 = iv + 2
+;       ^ ~~~~~ ^ dependence distance = 4
+;              ^ ~~~~~~~~~~~~~~~~~ ^ 8
+;       ^ ~~~~~~~~~~~~~~~~ ^ 8
+;   ^ ~~~~~~~~~~~~~~~~ ^ iv.next = iv + 8
+;
+; Measurements are in bytes.
+;
+; TODO: Relax the HasSameSize check; the strided accesses are
+; independent, as determined by both the source size and the sink size.
+; This test should report no dependencies.
+define void @different_type_sizes_strided_accesses_independent(ptr %dst) {
+; CHECK-LABEL: 'different_type_sizes_strided_accesses_independent'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            store i16 0, ptr %gep.iv, align 2 ->
+; CHECK-NEXT:            store i32 1, ptr %gep.4.iv, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.4 = getelementptr nuw i8, ptr %dst, i64 4
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.2 = add nuw nsw i64 %iv, 2
+  %gep.iv = getelementptr i8, ptr %dst, i64 %iv.2
+  store i16 0, ptr %gep.iv
+  %gep.4.iv = getelementptr i8, ptr %gep.4, i64 %iv.2
+  store i32 1, ptr %gep.4.iv
+  %iv.next = add nuw nsw i64 %iv, 8
+  %ec = icmp eq i64 %iv.next, 64
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+
+;     i16      i64
+; [ . 0 0 . 1 1 1 1] [ 1 x x 1 1 1 1 1 ]
+;  ^~~^ gep i8 = 1
+;  ^~~^ iv.1 = iv + 1
+;     ^ ~~ ^ dependence distance = 3
+;     ^ ~~~~~~~~~~~~~~~~ ^ 8
+;           ^ ~~~~~~~~~~~~~~~~ ^ 8
+;   ^ ~~~~~~~~~~~~~~~~ ^ iv.next = iv + 8
+;
+; TODO: Relax the HasSameSize check; this test should report a backward
+; loop-carried dependence.
+define void @different_type_sizes_strided_accesses_dependent(ptr %dst) {
+; CHECK-LABEL: 'different_type_sizes_strided_accesses_dependent'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            store i16 0, ptr %gep.iv, align 2 ->
+; CHECK-NEXT:            store i64 1, ptr %gep.3.iv, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.3 = getelementptr nuw i8, ptr %dst, i64 3
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.1 = add nuw nsw i64 %iv, 1
+  %gep.iv = getelementptr i8, ptr %dst, i64 %iv.1
+  store i16 0, ptr %gep.iv
+  %gep.3.iv = getelementptr i8, ptr %gep.3, i64 %iv.1
+  store i64 1, ptr %gep.3.iv
+  %iv.next = add nuw nsw i64 %iv, 8
+  %ec = icmp eq i64 %iv.next, 64
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Variant of the above, where the store size exceeds the dependence
+; distance.
+define void @different_type_sizes_strided_accesses_store_size_exceeds_depdist(ptr %dst) {
+; CHECK-LABEL: 'different_type_sizes_strided_accesses_store_size_exceeds_depdist'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unsafe indirect dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        IndirectUnsafe:
+; CHECK-NEXT:            store i16 0, ptr %gep.iv, align 2 ->
+; CHECK-NEXT:            store i128 1, ptr %gep.10.iv, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.10 = getelementptr nuw i8, ptr %dst, i64 10
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.iv = getelementptr i8, ptr %dst, i64 %iv
+  store i16 0, ptr %gep.iv
+  %gep.10.iv = getelementptr i8, ptr %gep.10, i64 %iv
+  store i128 1, ptr %gep.10.iv
+  %iv.next = add i64 %iv, 8
+  %ec = icmp eq i64 %iv.next, 64
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+
+; Source type-size differs from that of the sink, but when
+; determining backward dependence, only the source size
+; is relevant.
+; TODO: Relax the HasSameSize check; this test should report
+; BackwardVectorizable.
+define void @different_type_sizes_source_size_backwardvectorizible(ptr %dst) {
+; CHECK-LABEL: 'different_type_sizes_source_size_backwardvectorizible'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Unknown data dependence.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Unknown:
+; CHECK-NEXT:            store i16 0, ptr %gep.iv, align 2 ->
+; CHECK-NEXT:            store i32 1, ptr %gep.10.iv, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.10 = getelementptr nuw i8, ptr %dst, i64 10
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.iv = getelementptr i8, ptr %dst, i64 %iv
+  store i16 0, ptr %gep.iv
+  %gep.10.iv = getelementptr i8, ptr %gep.10, i64 %iv
+  store i32 1, ptr %gep.10.iv
+  %iv.next = add i64 %iv, 8
+  %ec = icmp eq i64 %iv.next, 64
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Source type-size differs from that of the sink, and when
+; determining forward dependence, the source size can
+; prevent forwarding.
+define void @different_type_sizes_forward(ptr %dst) {
+; CHECK-LABEL: 'different_type_sizes_forward'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Memory dependences are safe
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        Forward:
+; CHECK-NEXT:            store i32 0, ptr %gep.10.iv, align 4 ->
+; CHECK-NEXT:            store i16 1, ptr %gep.iv, align 2
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %gep.10 = getelementptr nuw i8, ptr %dst, i64 10
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.10.iv = getelementptr i8, ptr %gep.10, i64 %iv
+  store i32 0, ptr %gep.10.iv
+  %gep.iv = getelementptr i8, ptr %dst, i64 %iv
+  store i16 1, ptr %gep.iv
+  %iv.next = add i64 %iv, 8
+  %ec = icmp eq i64 %iv.next, 64
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Same as the above, but here, the store size should not prevent
+; ld->st forwarding.
+; TODO: Relax the HasSameSize check; this test should report a
+; forward dependence.
+define void @different_type_sizes_store_size_cannot_prevent_forwarding(ptr %A, ptr noalias %B) {
+; CHECK-LABEL: 'different_type_sizes_store_size_cannot_prevent_forwarding'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Forward loop carried data dependence that prevents store-to-load forwarding.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        ForwardButPreventsForwarding:
+; CHECK-NEXT:            store i32 0, ptr %gep.A, align 4 ->
+; CHECK-NEXT:            %l = load i16, ptr %gep.A.1, align 2
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %A.1 = getelementptr i32, ptr %A, i64 1
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1022, %entry ], [ %iv.next, %loop ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+  store i32 0, ptr %gep.A
+  %gep.A.1 = getelementptr i32, ptr %A.1, i64 %iv
+  %l = load i16, ptr %gep.A.1
+  store i16 %l, ptr %B
+  %iv.next = add nsw i64 %iv, -1
+  %cmp = icmp eq i64 %iv, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Same as the above, but here, the load size prevents
+; ld->st forwarding.
+define void @different_type_sizes_load_size_prevents_forwarding(ptr %A, ptr noalias %B) {
+; CHECK-LABEL: 'different_type_sizes_load_size_prevents_forwarding'
+; CHECK-NEXT:    loop:
+; CHECK-NEXT:      Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop
+; CHECK-NEXT:  Forward loop carried data dependence that prevents store-to-load forwarding.
+; CHECK-NEXT:      Dependences:
+; CHECK-NEXT:        ForwardButPreventsForwarding:
+; CHECK-NEXT:            store i16 0, ptr %gep.A, align 2 ->
+; CHECK-NEXT:            %l = load i32, ptr %gep.A.1, align 4
+; CHECK-EMPTY:
+; CHECK-NEXT:      Run-time memory checks:
+; CHECK-NEXT:      Grouped accesses:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; CHECK-NEXT:      SCEV assumptions:
+; CHECK-EMPTY:
+; CHECK-NEXT:      Expressions re-written:
+;
+entry:
+  %A.1 = getelementptr i32, ptr %A, i64 1
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1022, %entry ], [ %iv.next, %loop ]
+  %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv
+  store i16 0, ptr %gep.A
+  %gep.A.1 = getelementptr i32, ptr %A.1, i64 %iv
+  %l = load i32, ptr %gep.A.1
+  store i32 %l, ptr %B
+  %iv.next = add nsw i64 %iv, -1
+  %cmp = icmp eq i64 %iv, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/runtime-checks-max-dependences.ll b/llvm/test/Analysis/LoopAccessAnalysis/runtime-checks-max-dependences.ll
new file mode 100644
index 0000000000000..5b17920bde85e
--- /dev/null
+++ b/llvm/test/Analysis/LoopAccessAnalysis/runtime-checks-max-dependences.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes='print<access-info>' -disable-output %s 2>&1 | FileCheck --check-prefix=DEFAULT %s
+; RUN: opt -passes='print<access-info>' -disable-output -max-dependences=5 %s 2>&1 | FileCheck --check-prefix=MAXDEP5 %s
+
+define void @dependences_with_strides(ptr %dst, i64 %stride) {
+; DEFAULT-LABEL: 'dependences_with_strides'
+; DEFAULT-NEXT:    loop:
+; DEFAULT-NEXT:      Memory dependences are safe with run-time checks
+; DEFAULT-NEXT:      Dependences:
+; DEFAULT-NEXT:      Run-time memory checks:
+; DEFAULT-NEXT:      Check 0:
+; DEFAULT-NEXT:        Comparing group GRP0:
+; DEFAULT-NEXT:          %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
+; DEFAULT-NEXT:        Against group GRP1:
+; DEFAULT-NEXT:          %gep.stride = getelementptr i16, ptr %ptr.iv, i64 %stride
+; DEFAULT-NEXT:      Check 1:
+; DEFAULT-NEXT:        Comparing group GRP0:
+; DEFAULT-NEXT:          %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
+; DEFAULT-NEXT:        Against group GRP2:
+; DEFAULT-NEXT:        ptr %dst
+; DEFAULT-NEXT:      Check 2:
+; DEFAULT-NEXT:        Comparing group GRP1:
+; DEFAULT-NEXT:          %gep.stride = getelementptr i16, ptr %ptr.iv, i64 %stride
+; DEFAULT-NEXT:        Against group GRP2:
+; DEFAULT-NEXT:        ptr %dst
+; DEFAULT-NEXT:      Grouped accesses:
+; DEFAULT-NEXT:        Group GRP0:
+; DEFAULT-NEXT:          (Low: %dst High: (4 + %dst))
+; DEFAULT-NEXT:            Member: {%dst,+,2}<nw><%loop>
+; DEFAULT-NEXT:        Group GRP1:
+; DEFAULT-NEXT:          (Low: ((2 * %stride) + %dst) High: (4 + (2 * %stride) + %dst))
+; DEFAULT-NEXT:            Member: {((2 * %stride) + %dst),+,2}<nw><%loop>
+; DEFAULT-NEXT:        Group GRP2:
+; DEFAULT-NEXT:          (Low: %dst High: (2 + %dst))
+; DEFAULT-NEXT:            Member: %dst
+; DEFAULT-EMPTY:
+; DEFAULT-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; DEFAULT-NEXT:      SCEV assumptions:
+; DEFAULT-EMPTY:
+; DEFAULT-NEXT:      Expressions re-written:
+;
+; MAXDEP5-LABEL: 'dependences_with_strides'
+; MAXDEP5-NEXT:    loop:
+; MAXDEP5-NEXT:      Memory dependences are safe with run-time checks
+; MAXDEP5-NEXT:      Too many dependences, not recorded
+; MAXDEP5-NEXT:      Run-time memory checks:
+; MAXDEP5-NEXT:      Check 0:
+; MAXDEP5-NEXT:        Comparing group GRP0:
+; MAXDEP5-NEXT:          %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
+; MAXDEP5-NEXT:        Against group GRP1:
+; MAXDEP5-NEXT:          %gep.stride = getelementptr i16, ptr %ptr.iv, i64 %stride
+; MAXDEP5-NEXT:      Check 1:
+; MAXDEP5-NEXT:        Comparing group GRP0:
+; MAXDEP5-NEXT:          %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
+; MAXDEP5-NEXT:        Against group GRP2:
+; MAXDEP5-NEXT:        ptr %dst
+; MAXDEP5-NEXT:      Check 2:
+; MAXDEP5-NEXT:        Comparing group GRP1:
+; MAXDEP5-NEXT:          %gep.stride = getelementptr i16, ptr %ptr.iv, i64 %stride
+; MAXDEP5-NEXT:        Against group GRP2:
+; MAXDEP5-NEXT:        ptr %dst
+; MAXDEP5-NEXT:      Grouped accesses:
+; MAXDEP5-NEXT:        Group GRP0:
+; MAXDEP5-NEXT:          (Low: %dst High: (4 + %dst))
+; MAXDEP5-NEXT:            Member: {%dst,+,2}<nw><%loop>
+; MAXDEP5-NEXT:        Group GRP1:
+; MAXDEP5-NEXT:          (Low: ((2 * %stride) + %dst) High: (4 + (2 * %stride) + %dst))
+; MAXDEP5-NEXT:            Member: {((2 * %stride) + %dst),+,2}<nw><%loop>
+; MAXDEP5-NEXT:        Group GRP2:
+; MAXDEP5-NEXT:          (Low: %dst High: (2 + %dst))
+; MAXDEP5-NEXT:            Member: %dst
+; MAXDEP5-EMPTY:
+; MAXDEP5-NEXT:      Non vectorizable stores to invariant address were not found in loop.
+; MAXDEP5-NEXT:      SCEV assumptions:
+; MAXDEP5-EMPTY:
+; MAXDEP5-NEXT:      Expressions re-written:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %ptr.iv = phi ptr [ %dst, %entry ], [ %ptr.iv.next, %loop ]
+  store i16 0, ptr %ptr.iv, align 2
+  %gep.stride = getelementptr i16, ptr %ptr.iv, i64 %stride
+  store i16 0, ptr %gep.stride, align 2
+  store i16 0, ptr %dst, align 2
+  store i16 0, ptr %ptr.iv, align 2
+  %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 2
+  %iv.next = add i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv, 1
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Analysis/LoopAccessAnalysis/scalable-vector-regression-tests.ll b/llvm/test/Analysis/LoopAccessAnalysis/scalable-vector-regression-tests.ll
index ffa5b3c868ab0..4dde864b2f1e2 100644
--- a/llvm/test/Analysis/LoopAccessAnalysis/scalable-vector-regression-tests.ll
+++ b/llvm/test/Analysis/LoopAccessAnalysis/scalable-vector-regression-tests.ll
@@ -61,3 +61,29 @@ vector.body:
 end:
   ret void
 }
+
+; CHECK-LABEL: 'regression_test_is_no_wrap_access_scalable_typesize'
+; CHECK: LAA: Found an analyzable loop: loop
+; CHECK: LAA: Bad stride - Scalable object: <vscale x 4 x i32>
+define void @regression_test_is_no_wrap_access_scalable_typesize(ptr %ptr_a, i64 %n, ptr %ptr_b) {
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %2 = shl i64 %iv, 1
+  %3 = add i64 %2, %n
+  %4 = trunc i64 %iv to i32
+  %5 = insertelement <vscale x 4 x i32> zeroinitializer, i32 %4, i64 0
+  %6 = getelementptr i32, ptr %ptr_a, i64 %3
+  store <vscale x 4 x i32> %5, ptr %6, align 4
+  %.reass3 = or i32 %4, 1
+  %7 = insertelement <vscale x 4 x i32> zeroinitializer, i32 %.reass3, i64 0
+  %8 = shufflevector <vscale x 4 x i32> %7, <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> zeroinitializer
+  %9 = getelementptr i32, ptr %ptr_b, i64 %3
+  store <vscale x 4 x i32> %8, ptr %9, align 4
+  %iv.next = add i64 %iv, 1
+  %.not = icmp eq i64 %iv, 16
+  br i1 %.not, label %end, label %loop
+end:
+  ret void
+}
diff --git a/llvm/test/Analysis/MemorySSA/lifetime-simple.ll b/llvm/test/Analysis/MemorySSA/lifetime-simple.ll
index 18d2459b42c40..03b67682392d1 100644
--- a/llvm/test/Analysis/MemorySSA/lifetime-simple.ll
+++ b/llvm/test/Analysis/MemorySSA/lifetime-simple.ll
@@ -9,8 +9,8 @@ entry:
   %P = alloca [32 x i8]
   %Q = call ptr @obscure(ptr %P)
 ; CHECK:  1 = MemoryDef(liveOnEntry)
-; CHECK-NEXT:   call void @llvm.lifetime.start.p0(i64 32, ptr %P)
-  call void @llvm.lifetime.start.p0(i64 32, ptr %P)
+; CHECK-NEXT:   call void @llvm.lifetime.start.p0(ptr %P)
+  call void @llvm.lifetime.start.p0(ptr %P)
 ; CHECK:  MemoryUse(1)
 ; CHECK-NEXT:   %0 = load i8, ptr %P
   %0 = load i8, ptr %P
@@ -18,8 +18,8 @@ entry:
 ; CHECK-NEXT:   store i8 1, ptr %P
   store i8 1, ptr %P
 ; CHECK:  3 = MemoryDef(2)
-; CHECK-NEXT:   call void @llvm.lifetime.end.p0(i64 32, ptr %P)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %P)
+; CHECK-NEXT:   call void @llvm.lifetime.end.p0(ptr %P)
+  call void @llvm.lifetime.end.p0(ptr %P)
 ; CHECK:  MemoryUse(3)
 ; CHECK-NEXT:   %1 = load i8, ptr %P
   %1 = load i8, ptr %P
@@ -28,5 +28,5 @@ entry:
   %2 = load i8, ptr %Q
   ret i8 %1
 }
-declare void @llvm.lifetime.start.p0(i64 %S, ptr nocapture %P) readonly
-declare void @llvm.lifetime.end.p0(i64 %S, ptr nocapture %P)
+declare void @llvm.lifetime.start.p0(ptr nocapture %P) readonly
+declare void @llvm.lifetime.end.p0(ptr nocapture %P)
diff --git a/llvm/test/Analysis/MemorySSA/phi-translation.ll b/llvm/test/Analysis/MemorySSA/phi-translation.ll
index b8244818c6e15..22bbeadc845d5 100644
--- a/llvm/test/Analysis/MemorySSA/phi-translation.ll
+++ b/llvm/test/Analysis/MemorySSA/phi-translation.ll
@@ -465,7 +465,7 @@ end:                                          ; preds = %for.body
 define void @use_clobbered_by_def_in_loop() {
 entry:
   %nodeStack = alloca [12 x i32], align 4
-  call void @llvm.lifetime.start.p0(i64 48, ptr nonnull %nodeStack)
+  call void @llvm.lifetime.start.p0(ptr nonnull %nodeStack)
   br i1 false, label %cleanup, label %while.cond
 
 ; CHECK-LABEL: while.cond:
@@ -502,12 +502,12 @@ while.end:                                        ; preds = %while.cond, %land.r
   br i1 true, label %cleanup, label %while.cond.backedge
 
 cleanup:                                          ; preds = %while.body, %while.end, %entry
-  call void @llvm.lifetime.end.p0(i64 48, ptr nonnull %nodeStack)
+  call void @llvm.lifetime.end.p0(ptr nonnull %nodeStack)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @another_loop_clobber_inc() {
 ; CHECK-LABEL: void @another_loop_clobber_inc
diff --git a/llvm/test/Analysis/MemorySSA/pr43044.ll b/llvm/test/Analysis/MemorySSA/pr43044.ll
index bd767d3b771e6..7ae02f35ff88c 100644
--- a/llvm/test/Analysis/MemorySSA/pr43044.ll
+++ b/llvm/test/Analysis/MemorySSA/pr43044.ll
@@ -4,7 +4,7 @@
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 target triple = "s390x-ibm-linux"
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ; CHECK-LABEL: @func_42()
 define void @func_42() {
diff --git a/llvm/test/Analysis/MemorySSA/pr49859.ll b/llvm/test/Analysis/MemorySSA/pr49859.ll
index 25ef58698d92d..0e97f57a81804 100644
--- a/llvm/test/Analysis/MemorySSA/pr49859.ll
+++ b/llvm/test/Analysis/MemorySSA/pr49859.ll
@@ -11,12 +11,12 @@ entry:
   %n = alloca i8, align 1
   %i = alloca i8, align 1
   %cleanup.dest.slot = alloca i32, align 1
-  call  void @llvm.lifetime.start.p0(i64 1, ptr %sum) #3
+  call  void @llvm.lifetime.start.p0(ptr %sum) #3
   store i8 0, ptr %sum, align 1
-  call  void @llvm.lifetime.start.p0(i64 1, ptr %n) #3
+  call  void @llvm.lifetime.start.p0(ptr %n) #3
   %call = call  i8 @idi(i8 10)
   store i8 %call, ptr %n, align 1
-  call  void @llvm.lifetime.start.p0(i64 1, ptr %i) #3
+  call  void @llvm.lifetime.start.p0(ptr %i) #3
   store i8 0, ptr %i, align 1
   br label %for.cond
 
@@ -61,9 +61,9 @@ for.inc:                                          ; preds = %if.end
 ; CHECK: final.cleanup:
 ; CHECK-NEXT: ; [[NO20:.*]] = MemoryPhi({if.then,[[NO9:.*]]},{for.cond.cleanup,[[NO8:.*]]})
 ; CHECK-NEXT: ; [[NO12:.*]] = MemoryDef([[NO20]])
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 1, ptr %i)
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr %i)
 final.cleanup:                                          ; preds = %if.then, %for.cond.cleanup
-  call  void @llvm.lifetime.end.p0(i64 1, ptr %i) #3
+  call  void @llvm.lifetime.end.p0(ptr %i) #3
   br label %for.end
 
 ; CHECK: for.end:
@@ -71,23 +71,23 @@ final.cleanup:                                          ; preds = %if.then, %for
 ; CHECK-NEXT:  %3 = load i8, ptr %sum, align 1
 for.end:                                          ; preds = %final.cleanup
   %8 = load i8, ptr %sum, align 1
-  call  void @llvm.lifetime.start.p0(i64 1, ptr %res.addr.i)
+  call  void @llvm.lifetime.start.p0(ptr %res.addr.i)
   store i8 %8, ptr %res.addr.i, align 1
   %9 = load i8, ptr %res.addr.i, align 1
   call  void @foo(i8 %9) #3
-  call  void @llvm.lifetime.end.p0(i64 1, ptr %res.addr.i)
-  call  void @llvm.lifetime.end.p0(i64 1, ptr %n) #3
-  call  void @llvm.lifetime.end.p0(i64 1, ptr %sum) #3
+  call  void @llvm.lifetime.end.p0(ptr %res.addr.i)
+  call  void @llvm.lifetime.end.p0(ptr %n) #3
+  call  void @llvm.lifetime.end.p0(ptr %sum) #3
   ret void
 }
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)  #1
+declare void @llvm.lifetime.start.p0(ptr nocapture)  #1
 
 declare i8 @idi(i8)
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)  #1
+declare void @llvm.lifetime.end.p0(ptr nocapture)  #1
 
 ; Function Attrs: nounwind
 declare void @foo(i8)
diff --git a/llvm/test/Analysis/MemorySSA/renamephis.ll b/llvm/test/Analysis/MemorySSA/renamephis.ll
index e297b99eb89a7..a731ef1db8039 100644
--- a/llvm/test/Analysis/MemorySSA/renamephis.ll
+++ b/llvm/test/Analysis/MemorySSA/renamephis.ll
@@ -8,7 +8,7 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @g()
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #0
+declare void @llvm.lifetime.end.p0(ptr nocapture) #0
 
 ; CHECK-LABEL: @f
 define void @f(i1 %arg) align 2 {
diff --git a/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll b/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll
index 39b475dfaba9c..7120eec5982e9 100644
--- a/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll
+++ b/llvm/test/Analysis/ScalarEvolution/add-expr-pointer-operand-sorting.ll
@@ -50,7 +50,7 @@ define i32 @d(i32 %base) {
 ;
 entry:
   %e = alloca [1 x [1 x i8]], align 1
-  call void @llvm.lifetime.start.p0(i64 1, ptr %e) #2
+  call void @llvm.lifetime.start.p0(ptr %e) #2
   br label %for.cond
 
 for.cond:                                         ; preds = %for.cond, %entry
@@ -69,4 +69,4 @@ for.cond:                                         ; preds = %for.cond, %entry
   br label %for.cond
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
diff --git a/llvm/test/Analysis/ScalarEvolution/sdiv.ll b/llvm/test/Analysis/ScalarEvolution/sdiv.ll
index 9eaaf8badd739..acc6ab01978f8 100644
--- a/llvm/test/Analysis/ScalarEvolution/sdiv.ll
+++ b/llvm/test/Analysis/ScalarEvolution/sdiv.ll
@@ -38,7 +38,7 @@ define dso_local void @_Z4loopi(i32 %width) local_unnamed_addr #0 {
 entry:
   %storage = alloca [2 x i32], align 4
   %0 = bitcast ptr %storage to ptr
-  call void @llvm.lifetime.start.p0(i64 8, ptr %storage) #4
+  call void @llvm.lifetime.start.p0(ptr %storage) #4
   call void @llvm.memset.p0.i64(ptr align 4 %0, i8 0, i64 8, i1 false)
   br label %for.cond
 
@@ -48,7 +48,7 @@ for.cond:
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:
-  call void @llvm.lifetime.end.p0(i64 8, ptr %storage) #4
+  call void @llvm.lifetime.end.p0(ptr %storage) #4
   ret void
 
 for.body:
@@ -64,10 +64,10 @@ for.body:
   br label %for.cond
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #2
 
 declare dso_local i32 @_Z3adji(i32) local_unnamed_addr #3
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
diff --git a/llvm/test/Analysis/ScalarEvolution/srem.ll b/llvm/test/Analysis/ScalarEvolution/srem.ll
index 377e58a588b3a..9d4538f381728 100644
--- a/llvm/test/Analysis/ScalarEvolution/srem.ll
+++ b/llvm/test/Analysis/ScalarEvolution/srem.ll
@@ -38,7 +38,7 @@ define dso_local void @_Z4loopi(i32 %width) local_unnamed_addr #0 {
 entry:
   %storage = alloca [2 x i32], align 4
   %0 = bitcast ptr %storage to ptr
-  call void @llvm.lifetime.start.p0(i64 8, ptr %storage) #4
+  call void @llvm.lifetime.start.p0(ptr %storage) #4
   call void @llvm.memset.p0.i64(ptr align 4 %0, i8 0, i64 8, i1 false)
   br label %for.cond
 
@@ -48,7 +48,7 @@ for.cond:
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:
-  call void @llvm.lifetime.end.p0(i64 8, ptr %storage) #4
+  call void @llvm.lifetime.end.p0(ptr %storage) #4
   ret void
 
 for.body:
@@ -64,10 +64,10 @@ for.body:
   br label %for.cond
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #2
 
 declare dso_local i32 @_Z3adji(i32) local_unnamed_addr #3
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
diff --git a/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll b/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll
index 840a5172561dc..36d79f9e44b96 100644
--- a/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll
+++ b/llvm/test/Analysis/ScopedNoAliasAA/alias-scope-merging.ll
@@ -8,10 +8,10 @@ define i8 @test(i8 %input) {
   %dst = alloca i8
   %src = alloca i8
 ; CHECK:   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 1, i1 false), !alias.scope ![[SCOPE:[0-9]+]]
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %src), !noalias !4
+  call void @llvm.lifetime.start.p0(ptr nonnull %src), !noalias !4
   store i8 %input, ptr %src
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %src, i64 1, i1 false), !alias.scope !0
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %src), !noalias !4
+  call void @llvm.lifetime.end.p0(ptr nonnull %src), !noalias !4
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp, i64 1, i1 false), !alias.scope !4
   %ret_value = load i8, ptr %dst
   call void @use(ptr %src)
@@ -23,8 +23,8 @@ define i8 @test(i8 %input) {
 ; CHECK-DAG: ![[CALLEE0_B:[0-9]+]] = distinct !{!{{[0-9]+}}, !{{[0-9]+}}, !"callee0: %b"}
 ; CHECK-DAG: ![[SCOPE]] = !{![[CALLEE0_A]], ![[CALLEE0_B]]}
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
 
 !0 = !{!1, !7}
diff --git a/llvm/test/Analysis/StackSafetyAnalysis/lifetime.ll b/llvm/test/Analysis/StackSafetyAnalysis/lifetime.ll
index 6c3dec9fadac7..51bfa154e957a 100644
--- a/llvm/test/Analysis/StackSafetyAnalysis/lifetime.ll
+++ b/llvm/test/Analysis/StackSafetyAnalysis/lifetime.ll
@@ -11,31 +11,31 @@ entry:
 ; CHECK: %y = alloca i32, align 4
 ; CHECK-NEXT: Alive: <>
   %z = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %z)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %z)
 ; CHECK-NEXT: Alive: <z>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x z>
 
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <z>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <y z>
 
   call void @capture32(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y)
 ; CHECK-NEXT: Alive: <z>
 
   call void @capture32(ptr %z)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %z)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %z)
 ; CHECK-NEXT: Alive: <>
 
   ret void
@@ -48,13 +48,13 @@ entry:
 ; CHECK-NEXT: Alive: <y>
   %x = alloca i32, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x y>
 
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <y>
 
   call void @capture32(ptr %y)
@@ -69,31 +69,31 @@ entry:
   %x = alloca i32, align 4
   %y = alloca i32, align 4
   %z = alloca i64, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x y>
 
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <y>
 
   call void @capture32(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y)
 ; CHECK-NEXT: Alive: <>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %z)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %z)
 ; CHECK-NEXT: Alive: <z>
 
   call void @capture64(ptr %z)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %z)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %z)
 ; CHECK-NEXT: Alive: <>
 
   ret void
@@ -111,31 +111,31 @@ entry:
 ; CHECK-NEXT: Alive: <>
   %z = alloca i64, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x y>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %z)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %z)
 ; CHECK-NEXT: Alive: <x y z>
 
   call void @capture32(ptr %x)
   call void @capture32(ptr %y)
   call void @capture64(ptr %z)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <y z>
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y)
 ; CHECK-NEXT: Alive: <z>
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %z)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %z)
 ; CHECK-NEXT: Alive: <>
 
   ret void
@@ -154,12 +154,12 @@ entry:
   %z = alloca i64, align 8
   %z1 = alloca i64, align 8
   %z2 = alloca i64, align 8
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x1)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x1)
+  call void @llvm.lifetime.start.p0(ptr %x1)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x1)
 ; CHECK-NEXT: Alive: <x1>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x2)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x2)
+  call void @llvm.lifetime.start.p0(ptr %x2)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x2)
 ; CHECK-NEXT: Alive: <x1 x2>
 
   call void @capture64(ptr nonnull %x1)
@@ -171,8 +171,8 @@ entry:
 if.then:                                          ; preds = %entry
 ; CHECK: if.then:
 ; CHECK-NEXT: Alive: <x1 x2>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x1 x2 y>
 
   call void @capture64(ptr nonnull %y)
@@ -181,13 +181,13 @@ if.then:                                          ; preds = %entry
 if.then3:                                         ; preds = %if.then
 ; CHECK: if.then3:
 ; CHECK-NEXT: Alive: <x1 x2 y>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y1)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y1)
+  call void @llvm.lifetime.start.p0(ptr %y1)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y1)
 ; CHECK-NEXT: Alive: <x1 x2 y y1>
 
   call void @capture64(ptr nonnull %y1)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y1)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %y1)
+  call void @llvm.lifetime.end.p0(ptr %y1)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y1)
 ; CHECK-NEXT: Alive: <x1 x2 y>
 
   br label %if.end
@@ -195,13 +195,13 @@ if.then3:                                         ; preds = %if.then
 if.else:                                          ; preds = %if.then
 ; CHECK: if.else:
 ; CHECK-NEXT: Alive: <x1 x2 y>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y2)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y2)
+  call void @llvm.lifetime.start.p0(ptr %y2)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y2)
 ; CHECK-NEXT: Alive: <x1 x2 y y2>
 
   call void @capture64(ptr nonnull %y2)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y2)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %y2)
+  call void @llvm.lifetime.end.p0(ptr %y2)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y2)
 ; CHECK-NEXT: Alive: <x1 x2 y>
 
   br label %if.end
@@ -209,8 +209,8 @@ if.else:                                          ; preds = %if.then
 if.end:                                           ; preds = %if.else, %if.then3
 ; CHECK: if.end:
 ; CHECK-NEXT: Alive: <x1 x2 y>
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x1 x2>
 
   br label %if.end9
@@ -222,8 +222,8 @@ if.else4:                                         ; preds = %entry
 ; CHECK: %z.cast = bitcast ptr %z to ptr
 ; CHECK-NEXT: Alive: <x1 x2>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %z)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %z)
 ; CHECK-NEXT: Alive: <x1 x2 z>
 
   call void @capture64(ptr nonnull %z)
@@ -232,13 +232,13 @@ if.else4:                                         ; preds = %entry
 if.then6:                                         ; preds = %if.else4
 ; CHECK: if.then6:
 ; CHECK-NEXT: Alive: <x1 x2 z>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z1)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %z1)
+  call void @llvm.lifetime.start.p0(ptr %z1)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %z1)
 ; CHECK-NEXT: Alive: <x1 x2 z z1>
 
   call void @capture64(ptr nonnull %z1)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z1)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %z1)
+  call void @llvm.lifetime.end.p0(ptr %z1)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %z1)
 ; CHECK-NEXT: Alive: <x1 x2 z>
 
   br label %if.end8
@@ -246,13 +246,13 @@ if.then6:                                         ; preds = %if.else4
 if.else7:                                         ; preds = %if.else4
 ; CHECK: if.else7:
 ; CHECK-NEXT: Alive: <x1 x2 z>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z2)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %z2)
+  call void @llvm.lifetime.start.p0(ptr %z2)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %z2)
 ; CHECK-NEXT: Alive: <x1 x2 z z2>
 
   call void @capture64(ptr nonnull %z2)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z2)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %z2)
+  call void @llvm.lifetime.end.p0(ptr %z2)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %z2)
 ; CHECK-NEXT: Alive: <x1 x2 z>
 
   br label %if.end8
@@ -260,8 +260,8 @@ if.else7:                                         ; preds = %if.else4
 if.end8:                                          ; preds = %if.else7, %if.then6
 ; CHECK: if.end8:
 ; CHECK-NEXT: Alive: <x1 x2 z>
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %z)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %z)
 ; CHECK-NEXT: Alive: <x1 x2>
 
   br label %if.end9
@@ -269,12 +269,12 @@ if.end8:                                          ; preds = %if.else7, %if.then6
 if.end9:                                          ; preds = %if.end8, %if.end
 ; CHECK: if.end9:
 ; CHECK-NEXT: Alive: <x1 x2>
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x2)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x2)
+  call void @llvm.lifetime.end.p0(ptr %x2)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x2)
 ; CHECK-NEXT: Alive: <x1>
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x1)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x1)
+  call void @llvm.lifetime.end.p0(ptr %x1)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x1)
 ; CHECK-NEXT: Alive: <>
 
   ret void
@@ -287,8 +287,8 @@ entry:
 ; CHECK-NEXT: Alive: <>
   %x = alloca i32, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   call void @capture32(ptr %x)
@@ -297,17 +297,17 @@ entry:
 bb2:                                              ; preds = %entry
 ; CHECK: bb2:
 ; CHECK-NEXT: Alive: <x>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x y>
 
   call void @capture32(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x>
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <>
 
   ret void
@@ -315,8 +315,8 @@ bb2:                                              ; preds = %entry
 bb3:                                              ; preds = %entry
 ; CHECK: bb3:
 ; CHECK-NEXT: Alive: <x>
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <>
 
   ret void
@@ -329,13 +329,13 @@ entry:
 ; CHECK-NEXT: Alive: <>
   %x = alloca i32, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <>
 
   br i1 %d, label %bb2, label %bb3
@@ -343,13 +343,13 @@ entry:
 bb2:                                              ; preds = %entry
 ; CHECK: bb2:
 ; CHECK-NEXT: Alive: <>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <y>
 
   call void @capture32(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y)
 ; CHECK-NEXT: Alive: <>
 
   ret void
@@ -367,13 +367,13 @@ entry:
 ; CHECK-NEXT: Alive: <>
   %x = alloca i32, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <>
 
   br i1 %d, label %bb2, label %bb3
@@ -381,8 +381,8 @@ entry:
 bb2:                                              ; preds = %entry
 ; CHECK: bb2:
 ; CHECK-NEXT: Alive: <>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <y>
 
   call void @capture32(ptr %y)
@@ -401,8 +401,8 @@ entry:
 ; CHECK-NEXT: Alive: <>
   %x = alloca i32, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   call void @capture32(ptr %x)
@@ -411,12 +411,12 @@ entry:
 bb2:                                              ; preds = %entry
 ; CHECK: bb2:
 ; CHECK-NEXT: Alive: <x>
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <y>
 
   call void @capture32(ptr %y)
@@ -436,8 +436,8 @@ entry:
   %x = alloca i32, align 4
   %y = alloca i32, align 4
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   br i1 %d, label %bb2, label %bb3
@@ -445,8 +445,8 @@ entry:
 bb2:                                              ; preds = %entry
 ; CHECK: bb2:
 ; CHECK-NEXT: Alive: <x>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x y>
 
   call void @capture32(ptr %y)
@@ -467,12 +467,12 @@ entry:
   %B.i2 = alloca [100 x i32], align 4
   %A.i = alloca [100 x i32], align 4
   %B.i = alloca [100 x i32], align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %A.i)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %A.i)
+  call void @llvm.lifetime.start.p0(ptr %A.i)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %A.i)
 ; CHECK-NEXT: Alive: <A.i>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %B.i)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %B.i)
+  call void @llvm.lifetime.start.p0(ptr %B.i)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %B.i)
 ; CHECK-NEXT: Alive: <A.i B.i>
 
   call void @capture100x32(ptr %A.i)
@@ -480,30 +480,30 @@ entry:
 ; CHECK-NEXT: Alive: <A.i B.i>
 
   call void @capture100x32(ptr %B.i)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %A.i)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %A.i)
+  call void @llvm.lifetime.end.p0(ptr %A.i)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %A.i)
 ; CHECK-NEXT: Alive: <B.i>
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %B.i)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %B.i)
+  call void @llvm.lifetime.end.p0(ptr %B.i)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %B.i)
 ; CHECK-NEXT: Alive: <>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %A.i1)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %A.i1)
+  call void @llvm.lifetime.start.p0(ptr %A.i1)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %A.i1)
 ; CHECK-NEXT: Alive: <A.i1>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %B.i2)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %B.i2)
+  call void @llvm.lifetime.start.p0(ptr %B.i2)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %B.i2)
 ; CHECK-NEXT: Alive: <A.i1 B.i2>
 
   call void @capture100x32(ptr %A.i1)
   call void @capture100x32(ptr %B.i2)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %A.i1)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %A.i1)
+  call void @llvm.lifetime.end.p0(ptr %A.i1)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %A.i1)
 ; CHECK-NEXT: Alive: <B.i2>
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %B.i2)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %B.i2)
+  call void @llvm.lifetime.end.p0(ptr %B.i2)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %B.i2)
 ; CHECK-NEXT: Alive: <>
 
   ret void
@@ -516,20 +516,20 @@ entry:
 ; CHECK-NEXT: Alive: <>
   %buf1 = alloca i8, i32 100000, align 16
   %buf2 = alloca i8, i32 100000, align 16
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %buf1)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %buf1)
+  call void @llvm.lifetime.start.p0(ptr %buf1)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %buf1)
 ; CHECK-NEXT: Alive: <buf1>
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %buf1)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %buf1)
+  call void @llvm.lifetime.end.p0(ptr %buf1)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %buf1)
 ; CHECK-NEXT: Alive: <>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %buf1)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %buf1)
+  call void @llvm.lifetime.start.p0(ptr %buf1)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %buf1)
 ; CHECK-NEXT: Alive: <buf1>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %buf2)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %buf2)
+  call void @llvm.lifetime.start.p0(ptr %buf2)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %buf2)
 ; CHECK-NEXT: Alive: <buf1 buf2>
 
   call void @capture8(ptr %buf1)
@@ -546,22 +546,22 @@ entry:
   %B.i2 = alloca [100 x i32], align 4
   %A.i = alloca [100 x i32], align 4
   %B.i = alloca [100 x i32], align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %A.i)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %A.i)
+  call void @llvm.lifetime.start.p0(ptr %A.i)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %A.i)
 ; CHECK-NEXT: Alive: <A.i A.i1 B.i2>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %B.i)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %B.i)
+  call void @llvm.lifetime.start.p0(ptr %B.i)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %B.i)
 ; CHECK-NEXT: Alive: <A.i A.i1 B.i B.i2>
 
   call void @capture100x32(ptr %A.i)
   call void @capture100x32(ptr %B.i)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %A.i)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %A.i)
+  call void @llvm.lifetime.end.p0(ptr %A.i)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %A.i)
 ; CHECK-NEXT: Alive: <A.i1 B.i B.i2>
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %B.i)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %B.i)
+  call void @llvm.lifetime.end.p0(ptr %B.i)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %B.i)
 ; CHECK-NEXT: Alive: <A.i1 B.i2>
 
   br label %block2
@@ -583,23 +583,23 @@ entry:
 ; CHECK-NEXT: Alive: <>
   %a.i = alloca [4 x %struct.Klass], align 16
   %b.i = alloca [4 x %struct.Klass], align 16
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %a.i)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %a.i)
+  call void @llvm.lifetime.start.p0(ptr %a.i)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %a.i)
 ; CHECK-NEXT: Alive: <a.i>
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %b.i)
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %b.i)
+  call void @llvm.lifetime.start.p0(ptr %b.i)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %b.i)
 ; CHECK-NEXT: Alive: <a.i b.i>
 
   call void @capture8(ptr %a.i)
   call void @capture8(ptr %b.i)
   %z3 = load i32, ptr %a.i, align 16
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %a.i)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %a.i)
+  call void @llvm.lifetime.end.p0(ptr %a.i)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %a.i)
 ; CHECK-NEXT: Alive: <b.i>
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %b.i)
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %b.i)
+  call void @llvm.lifetime.end.p0(ptr %b.i)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %b.i)
 ; CHECK-NEXT: Alive: <>
 
   ret i32 %z3
@@ -611,8 +611,8 @@ entry:
 ; CHECK: entry:
 ; CHECK-NEXT: Alive: <>
   %x = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   br label %l2
@@ -622,8 +622,8 @@ l2:                                               ; preds = %l2, %entry
 ; MAY-NEXT: Alive: <x>
 ; MUST-NEXT: Alive: <>
   call void @capture8(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <>
 
   br label %l2
@@ -636,8 +636,8 @@ entry:
 ; CHECK-NEXT: Alive: <>
   %x = alloca i8, align 4
   %y = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   br label %l2
@@ -645,17 +645,17 @@ entry:
 l2:                                               ; preds = %l2, %entry
 ; CHECK: l2:
 ; CHECK-NEXT: Alive: <x>
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x y>
 
   call void @capture8(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x>
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   call void @capture8(ptr %x)
@@ -677,24 +677,24 @@ entry:
 if.then:                                          ; preds = %entry
 ; CHECK: if.then:
 ; CHECK-NEXT: Alive: <>
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %a)
-; CHECK: call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
+; CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %a)
 ; CHECK-NEXT: Alive: <a>
   tail call void @capture8(ptr %a)
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
-; CHECK: call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
+; CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %a)
 ; CHECK-NEXT: Alive: <>
   br label %if.end
 
 if.else:                                          ; preds = %entry
 ; CHECK: if.else:
 ; CHECK-NEXT: Alive: <>
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %b)
-; CHECK: call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %b)
+  call void @llvm.lifetime.start.p0(ptr nonnull %b)
+; CHECK: call void @llvm.lifetime.start.p0(ptr nonnull %b)
 ; CHECK-NEXT: Alive: <b>
   tail call void @capture8(ptr %b)
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %b)
-; CHECK: call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
+; CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %b)
 ; CHECK-NEXT: Alive: <>
   br label %if.end
 
@@ -719,8 +719,8 @@ entry:
 if.then:
 ; CHECK: if.then:
 ; CHECK-NEXT: Alive: <>
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <y>
 
   br label %if.end
@@ -730,12 +730,12 @@ if.then:
 if.else:
 ; CHECK: if.else:
 ; CHECK-NEXT: Alive: <>
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <y>
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x y>
 
   br label %if.end
@@ -758,12 +758,12 @@ entry:
   %x = alloca i8, align 4
   %y = alloca i8, align 4
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <y>
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x y>
 
   br label %end
@@ -773,7 +773,7 @@ entry:
 dead:
 ; CHECK: dead:
 ; CHECK-NOT: Alive:
-  call void @llvm.lifetime.start.p0(i64 4, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
 
   br label %end
 ; CHECK: br label %end
@@ -792,20 +792,20 @@ entry:
 ; CHECK: entry:
 ; CHECK-NEXT: Alive: <>
   %x = alloca i8
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
-  call void @llvm.lifetime.end.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <>
 
-  call void @llvm.lifetime.end.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <>
 
   ret void
@@ -827,8 +827,8 @@ if.then:
 ; CHECK: if.then:
 ; MAY-NEXT: Alive: <x y>
 ; MUST-NEXT: Alive: <>
-  call void @llvm.lifetime.end.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y)
 ; MAY-NEXT: Alive: <x>
 ; MUST-NEXT: Alive: <>
 
@@ -840,12 +840,12 @@ if.then:
 if.else:
 ; CHECK: if.else:
 ; CHECK-NEXT: Alive: <>
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <y>
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x y>
 
   br label %if.then
@@ -868,8 +868,8 @@ entry:
   %x = alloca i8, align 4
   %y = alloca i8, align 4
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   br i1 %a, label %if.then, label %if.else
@@ -880,8 +880,8 @@ if.then:
 ; CHECK: if.then:
 ; MAY-NEXT: Alive: <x>
 ; MUST-NEXT: Alive: <>
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; MAY-NEXT: Alive: <x y>
 ; MUST-NEXT: Alive: <y>
 
@@ -893,12 +893,12 @@ if.then:
 if.else:
 ; CHECK: if.else:
 ; CHECK-NEXT: Alive: <x>
-  call void @llvm.lifetime.end.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x>
 
-  call void @llvm.lifetime.end.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT: Alive: <>
 
   br label %if.then
@@ -921,8 +921,8 @@ entry:
   %x = alloca i8, align 4
   %y = alloca i8, align 4
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK-NEXT: Alive: <x>
 
   br i1 %a, label %if.then, label %if.end
@@ -933,8 +933,8 @@ if.then:
 ; CHECK: if.then:
 ; MAY-NEXT: Alive: <x y>
 ; MUST-NEXT: Alive: <x>
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK-NEXT: Alive: <x y>
 
   br i1 %a, label %if.then, label %if.end
@@ -949,8 +949,8 @@ if.end:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr captures(none))
-declare void @llvm.lifetime.end.p0(i64, ptr captures(none))
+declare void @llvm.lifetime.start.p0(ptr captures(none))
+declare void @llvm.lifetime.end.p0(ptr captures(none))
 declare void @capture8(ptr)
 declare void @capture32(ptr)
 declare void @capture64(ptr)
diff --git a/llvm/test/Analysis/StackSafetyAnalysis/local.ll b/llvm/test/Analysis/StackSafetyAnalysis/local.ll
index 02d46c8449bae..6944f38411003 100644
--- a/llvm/test/Analysis/StackSafetyAnalysis/local.ll
+++ b/llvm/test/Analysis/StackSafetyAnalysis/local.ll
@@ -707,9 +707,9 @@ entry:
   %n = load i8, ptr %y
   call void @llvm.memset.p0.i32(ptr nonnull %z, i8 0, i32 1, i1 false)
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %z)
 
   ret void
 }
@@ -731,9 +731,9 @@ entry:
   %y = alloca i8, align 4
   %z = alloca i8, align 4
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %z)
 
   store i8 5, ptr %x
   %n = load i8, ptr %y
@@ -756,13 +756,13 @@ entry:
   %y = alloca i8, align 4
   %z = alloca i8, align 4
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %y)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %z)
 
-  call void @llvm.lifetime.end.p0(i64 1, ptr %x)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %y)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %z)
 
   store i8 5, ptr %x
   %n = load i8, ptr %y
@@ -973,13 +973,13 @@ define void @DoubleLifetime() {
 ; CHECK-EMPTY:
 entry:
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   call void @llvm.memset.p0.i32(ptr %a, i8 1, i32 4, i1 true)
 
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memset.p0.i32(ptr %a, i8 1, i32 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -993,13 +993,13 @@ define void @DoubleLifetime2() {
 ; CHECK-EMPTY:
 entry:
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   %n = load i32, ptr %a
 
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memset.p0.i32(ptr %a, i8 1, i32 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -1013,13 +1013,13 @@ define void @DoubleLifetime3() {
 ; CHECK-EMPTY:
 entry:
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   store i32 5, ptr %a
 
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memset.p0.i32(ptr %a, i8 1, i32 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -1033,9 +1033,9 @@ define void @DoubleLifetime4() {
 ; CHECK-EMPTY:
 entry:
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memset.p0.i32(ptr %a, i8 1, i32 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   call void @unknown_call(ptr %a)
   ret void
 }
@@ -1136,5 +1136,5 @@ entry:
   ret ptr null
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/addrspacecast.mir b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/addrspacecast.mir
new file mode 100644
index 0000000000000..612f7b7ef4ec4
--- /dev/null
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/addrspacecast.mir
@@ -0,0 +1,35 @@
+# NOTE: This file is Generic MIR translation of llvm/test/Analysis/UniformityAnalysis/AMDGPU/addrspacecast.ll test file
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -run-pass=print-machine-uniformity -filetype=null %s 2>&1 | FileCheck %s --check-prefix=UNI
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=print-machine-uniformity -filetype=null %s 2>&1 | FileCheck %s --check-prefix=DIV
+
+# UNI: ALL VALUES UNIFORM
+# DIV: DIVERGENT: %3: %3:_(p0) = G_ADDRSPACE_CAST %2:_(p5)
+# DIV: DIVERGENT: %4: %4:_(p0) = G_INTRINSIC intrinsic(@llvm.amdgcn.addrspacecast.nonnull), %2:_(p5)
+
+--- |
+  define void @foo() {
+    %alloca = alloca i32, align 4, addrspace(5)
+    %cast = addrspacecast ptr addrspace(5) %alloca to ptr
+    store i32 1, ptr %cast, align 4
+    %cast.1 = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p5(ptr addrspace(5) %alloca)
+    store i32 2, ptr %cast.1, align 4
+    ret void
+  }
+...
+---
+name:            foo
+stack:
+  - { id: 0, name: alloca, type: default, offset: 0, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.1 (%ir-block.0):
+    %10:_(s32) = G_CONSTANT i32 1
+    %12:_(s32) = G_CONSTANT i32 2
+    %8:_(p5) = G_FRAME_INDEX %stack.0.alloca
+    %9:_(p0) = G_ADDRSPACE_CAST %8(p5)
+    G_STORE %10(s32), %9(p0) :: (store (s32) into %ir.cast)
+    %11:_(p0) = G_INTRINSIC intrinsic(@llvm.amdgcn.addrspacecast.nonnull), %8(p5)
+    G_STORE %12(s32), %11(p0) :: (store (s32) into %ir.cast.1)
+    SI_RETURN
+...
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/addrspacecast.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/addrspacecast.ll
new file mode 100644
index 0000000000000..e6808448651c8
--- /dev/null
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/addrspacecast.ll
@@ -0,0 +1,14 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes='print<uniformity>' -disable-output %s 2>&1 | FileCheck %s --check-prefix=UNI
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -passes='print<uniformity>' -disable-output %s 2>&1 | FileCheck %s --check-prefix=DIV
+
+; UNI: ALL VALUES UNIFORM
+; DIV: DIVERGENT:   %cast = addrspacecast ptr addrspace(5) %alloca to ptr
+; DIV: DIVERGENT:   %cast.1 = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p5(ptr addrspace(5) %alloca)
+define void @foo() {
+  %alloca = alloca i32, align 4, addrspace(5)
+  %cast = addrspacecast ptr addrspace(5) %alloca to ptr
+  store i32 1, ptr %cast
+  %cast.1 = call ptr @llvm.amdgcn.addrspacecast.nonnull.p0.p5(ptr addrspace(5) %alloca)
+  store i32 2, ptr %cast.1
+  ret void
+}
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
index 099fd2fbc7ca1..d5c6000a1eef6 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
@@ -854,6 +854,15 @@ define amdgpu_cs_chain void @dead(ptr addrspace(1) %out) {
   ret void
 }
 
+; CHECK: DIVERGENT:   %v = call i32 (ptr, ...) @llvm.amdgcn.call.whole.wave.i32.p0(ptr @wwf, i32 15)
+define amdgpu_cs void @call_whole_wave(ptr addrspace(1) %out) {
+  %v = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave.i32.p0(ptr @wwf, i32 15)
+  store i32 %v, ptr addrspace(1) %out
+  ret void
+}
+
+declare amdgpu_gfx_whole_wave i32 @wwf(i1, i32) #0
+
 declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) #1
 declare i32 @llvm.amdgcn.permlane16.i32(i32, i32, i32, i32, i1, i1) #1
 declare i32 @llvm.amdgcn.permlanex16.i32(i32, i32, i32, i32, i1, i1) #1
@@ -931,6 +940,7 @@ declare <8 x half> @llvm.amdgcn.ds.load.tr16.b128.v8f16(ptr addrspace(3))
 declare <8 x bfloat> @llvm.amdgcn.ds.load.tr16.b128.v8bf16(ptr addrspace(3))
 
 declare i32 @llvm.amdgcn.dead.i32()
+declare i32 @llvm.amdgcn.call.whole.wave.i32.p0(ptr, ...) #0
 
 attributes #0 = { nounwind convergent }
 attributes #1 = { nounwind readnone convergent }
diff --git a/llvm/test/Analysis/ValueTracking/pr152700.ll b/llvm/test/Analysis/ValueTracking/pr152700.ll
new file mode 100644
index 0000000000000..91644c543a173
--- /dev/null
+++ b/llvm/test/Analysis/ValueTracking/pr152700.ll
@@ -0,0 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+declare noundef i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
+declare i32 @llvm.umin.i32(i32, i32)
+define i32 @foo(i1 %c, i32 %arg) {
+; CHECK-LABEL: define i32 @foo(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[ARG:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[I:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
+; CHECK-NEXT:    br i1 [[C]], label %[[BB_1:.*]], label %[[BB_2:.*]]
+; CHECK:       [[BB_1]]:
+; CHECK-NEXT:    br label %[[BB_2]]
+; CHECK:       [[BB_2]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[I]], %[[ENTRY]] ], [ 0, %[[BB_1]] ]
+; CHECK-NEXT:    [[RES:%.*]] = call i32 @llvm.umin.i32(i32 [[PHI]], i32 [[ARG]])
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  %i = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
+  br i1 %c, label %bb.1, label %bb.2
+bb.1:
+  br label %bb.2
+bb.2:
+  %phi = phi i32 [ %i, %entry ], [ 0, %bb.1 ]
+  %res = call i32 @llvm.umin.i32(i32 %phi, i32 %arg)
+  ret i32 %res
+}
diff --git a/llvm/test/Assembler/auto_upgrade_intrinsics.ll b/llvm/test/Assembler/auto_upgrade_intrinsics.ll
index d1b535b49ca95..37cb49650f6bd 100644
--- a/llvm/test/Assembler/auto_upgrade_intrinsics.ll
+++ b/llvm/test/Assembler/auto_upgrade_intrinsics.ll
@@ -171,10 +171,10 @@ define void @tests.lifetime.start.end() {
   ; CHECK-LABEL: @tests.lifetime.start.end(
   %a = alloca i8
   call void @llvm.lifetime.start(i64 1, ptr %a)
-  ; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %a)
+  ; CHECK: call void @llvm.lifetime.start.p0(ptr %a)
   store i8 0, ptr %a
   call void @llvm.lifetime.end(i64 1, ptr %a)
-  ; CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr %a)
+  ; CHECK: call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -185,10 +185,10 @@ define void @tests.lifetime.start.end.unnamed() {
   ; CHECK-LABEL: @tests.lifetime.start.end.unnamed(
   %a = alloca ptr
   call void @llvm.lifetime.start.unnamed(i64 1, ptr %a)
-  ; CHECK: call void @llvm.lifetime.start.p0(i64 1, ptr %a)
+  ; CHECK: call void @llvm.lifetime.start.p0(ptr %a)
   store ptr null, ptr %a
   call void @llvm.lifetime.end.unnamed(i64 1, ptr %a)
-  ; CHECK: call void @llvm.lifetime.end.p0(i64 1, ptr %a)
+  ; CHECK: call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -220,5 +220,5 @@ define void @test.prefetch.unnamed(ptr %ptr) {
 ; emitted at the end.
 ; CHECK: declare i32 @llvm.objectsize.i32.p0
 
-; CHECK: declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none))
-; CHECK: declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none))
+; CHECK: declare void @llvm.lifetime.start.p0(ptr captures(none))
+; CHECK: declare void @llvm.lifetime.end.p0(ptr captures(none))
diff --git a/llvm/test/Assembler/autoupgrade-lifetime-intrinsics.ll b/llvm/test/Assembler/autoupgrade-lifetime-intrinsics.ll
index 00ab93470c193..377c00203079f 100644
--- a/llvm/test/Assembler/autoupgrade-lifetime-intrinsics.ll
+++ b/llvm/test/Assembler/autoupgrade-lifetime-intrinsics.ll
@@ -5,8 +5,8 @@ define void @strip_bitcast() {
 ; CHECK-LABEL: define void @strip_bitcast() {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[B:%.*]] = bitcast ptr [[A]] to ptr
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i8
@@ -20,8 +20,8 @@ define void @strip_addrspacecast() {
 ; CHECK-LABEL: define void @strip_addrspacecast() {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[B:%.*]] = addrspacecast ptr [[A]] to ptr addrspace(1)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i8
@@ -35,8 +35,8 @@ define void @strip_gep() {
 ; CHECK-LABEL: define void @strip_gep() {
 ; CHECK-NEXT:    [[A:%.*]] = alloca [2 x i8], align 1
 ; CHECK-NEXT:    [[B:%.*]] = getelementptr [2 x i8], ptr [[A]], i64 0, i64 0
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [2 x i8]
@@ -55,3 +55,8 @@ define void @remove_unanalyzable(ptr %p) {
   call void @llvm.lifetime.end.p0(i64 1, ptr %p)
   ret void
 }
+
+declare void @llvm.lifetime.start.p0(i64, ptr)
+declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p1(i64, ptr addrspace(1))
+declare void @llvm.lifetime.end.p1(i64, ptr addrspace(1))
diff --git a/llvm/test/Assembler/ptrtoaddr-invalid-constexpr.ll b/llvm/test/Assembler/ptrtoaddr-invalid-constexpr.ll
new file mode 100644
index 0000000000000..665deff4cd04b
--- /dev/null
+++ b/llvm/test/Assembler/ptrtoaddr-invalid-constexpr.ll
@@ -0,0 +1,56 @@
+;; Check all requirements on the ptrtoaddr constant expression operands
+;; Most of these invalid cases are detected at parse time but some are only
+;; detected at verification time (see Verifier::visitPtrToAddrInst())
+; RUN: rm -rf %t && split-file --leading-lines %s %t
+
+;--- src_vec_dst_no_vec.ll
+; RUN: not llvm-as %t/src_vec_dst_no_vec.ll -o /dev/null 2>&1 | FileCheck -check-prefix=SRC_VEC_DST_NO_VEC %s --implicit-check-not="error:"
+@g = global i64 ptrtoaddr (<2 x ptr> <ptr @g, ptr @g> to i64)
+; SRC_VEC_DST_NO_VEC: [[#@LINE-1]]:17: error: invalid cast opcode for cast from '<2 x ptr>' to 'i64'
+
+;--- src_no_vec_dst_vec.ll
+; RUN: not llvm-as %t/src_no_vec_dst_vec.ll -o /dev/null 2>&1 | FileCheck -check-prefix=SRC_NO_VEC_DST_VEC %s --implicit-check-not="error:"
+@g = global <2 x i64> ptrtoaddr (ptr @g to <2 x i64>)
+; SRC_NO_VEC_DST_VEC: [[#@LINE-1]]:23: error: invalid cast opcode for cast from 'ptr' to '<2 x i64>'
+
+;--- dst_not_int.ll
+; RUN: not llvm-as %t/dst_not_int.ll -o /dev/null 2>&1 | FileCheck -check-prefix=DST_NOT_INT %s --implicit-check-not="error:"
+@g = global float ptrtoaddr (ptr @g to float)
+; DST_NOT_INT: [[#@LINE-1]]:19: error: invalid cast opcode for cast from 'ptr' to 'float'
+
+;--- dst_not_int_vec.ll
+; RUN: not llvm-as %t/dst_not_int_vec.ll -o /dev/null 2>&1 | FileCheck -check-prefix=DST_NOT_INT_VEC %s --implicit-check-not="error:"
+@g = global <2 x float> ptrtoaddr (<2 x ptr> <ptr @g, ptr @g> to <2 x float>)
+; DST_NOT_INT_VEC: [[#@LINE-1]]:25: error: invalid cast opcode for cast from '<2 x ptr>' to '<2 x float>'
+
+;--- src_not_ptr.ll
+; RUN: not llvm-as %t/src_not_ptr.ll -o /dev/null 2>&1 | FileCheck -check-prefix=SRC_NOT_PTR %s --implicit-check-not="error:"
+@g = global i64 ptrtoaddr (i32 1 to i64)
+; SRC_NOT_PTR: [[#@LINE-1]]:17: error: invalid cast opcode for cast from 'i32' to 'i64'
+
+;--- src_not_ptr_vec.ll
+; RUN: not llvm-as %t/src_not_ptr_vec.ll -o /dev/null 2>&1 | FileCheck -check-prefix=SRC_NOT_PTR_VEC %s --implicit-check-not="error:"
+@g = global <2 x i64> ptrtoaddr (<2 x i32> <i32 1, i32 2> to <2 x i64>)
+; SRC_NOT_PTR_VEC: [[#@LINE-1]]:23: error: invalid cast opcode for cast from '<2 x i32>' to '<2 x i64>'
+
+;--- vec_src_fewer_elems.ll
+; RUN: not llvm-as %t/vec_src_fewer_elems.ll -o /dev/null 2>&1 | FileCheck -check-prefix=VEC_SRC_FEWER_ELEMS %s --implicit-check-not="error:"
+@g = global <4 x i64> ptrtoaddr (<2 x ptr> <ptr @g, ptr @g> to <4 x i64>)
+; VEC_SRC_FEWER_ELEMS: [[#@LINE-1]]:23: error: invalid cast opcode for cast from '<2 x ptr>' to '<4 x i64>'
+
+;--- vec_dst_fewer_elems.ll
+; RUN: not llvm-as %t/vec_dst_fewer_elems.ll -o /dev/null 2>&1 | FileCheck -check-prefix=VEC_DST_FEWER_ELEMS %s --implicit-check-not="error:"
+@g = global <2 x i64> ptrtoaddr (<4 x ptr> <ptr @g, ptr @g, ptr @g, ptr @g> to <2 x i64>)
+; VEC_DST_FEWER_ELEMS: [[#@LINE-1]]:23: error: invalid cast opcode for cast from '<4 x ptr>' to '<2 x i64>'
+
+;--- dst_not_addr_size.ll
+; The following invalid IR is caught by the verifier, not the parser:
+; RUN: llvm-as %t/dst_not_addr_size.ll --disable-output --disable-verify
+; RUN: not llvm-as %t/dst_not_addr_size.ll -o /dev/null 2>&1 | FileCheck -check-prefix=DST_NOT_ADDR_SIZE %s --implicit-check-not="error:"
+; DST_NOT_ADDR_SIZE: assembly parsed, but does not verify as correct!
+@g = global i32 ptrtoaddr (ptr @g to i32)
+; DST_NOT_ADDR_SIZE-NEXT: PtrToAddr result must be address width
+; DST_NOT_ADDR_SIZE-NEXT: i32 ptrtoaddr (ptr @g to i32)
+@g_vec = global <4 x i32> ptrtoaddr (<4 x ptr> <ptr @g, ptr @g, ptr @g, ptr @g> to <4 x i32>)
+; TODO: Verifier.cpp does not visit ConstantVector/ConstantStruct values
+; TODO-DST_NOT_ADDR_SIZE: PtrToAddr result must be address width
diff --git a/llvm/test/Assembler/ptrtoaddr-invalid.ll b/llvm/test/Assembler/ptrtoaddr-invalid.ll
new file mode 100644
index 0000000000000..dff787bc0b0a8
--- /dev/null
+++ b/llvm/test/Assembler/ptrtoaddr-invalid.ll
@@ -0,0 +1,84 @@
+;; Check all requirements on the ptrtoaddr instruction operands
+;; Most of these invalid cases are detected at parse time but some are only
+;; detected at verification time (see Verifier::visitPtrToAddrInst())
+; RUN: rm -rf %t && split-file --leading-lines %s %t
+
+;--- src_vec_dst_no_vec.ll
+; RUN: not llvm-as %t/src_vec_dst_no_vec.ll -o /dev/null 2>&1 | FileCheck -check-prefix=SRC_VEC_DST_NO_VEC %s --implicit-check-not="error:"
+define i64 @bad(<2 x ptr> %p) {
+  %addr = ptrtoaddr <2 x ptr> %p to i64
+  ; SRC_VEC_DST_NO_VEC: [[#@LINE-1]]:21: error: invalid cast opcode for cast from '<2 x ptr>' to 'i64'
+  ret i64 %addr
+}
+
+;--- src_no_vec_dst_vec.ll
+; RUN: not llvm-as %t/src_no_vec_dst_vec.ll -o /dev/null 2>&1 | FileCheck -check-prefix=SRC_NO_VEC_DST_VEC %s --implicit-check-not="error:"
+define <2 x i64> @bad(ptr %p) {
+  %addr = ptrtoaddr ptr %p to <2 x i64>
+  ; SRC_NO_VEC_DST_VEC: [[#@LINE-1]]:21: error: invalid cast opcode for cast from 'ptr' to '<2 x i64>'
+  ret <2 x i64> %addr
+}
+
+;--- dst_not_int.ll
+; RUN: not llvm-as %t/dst_not_int.ll -o /dev/null 2>&1 | FileCheck -check-prefix=DST_NOT_INT %s --implicit-check-not="error:"
+define float @bad(ptr %p) {
+  %addr = ptrtoaddr ptr %p to float
+  ; DST_NOT_INT: [[#@LINE-1]]:21: error: invalid cast opcode for cast from 'ptr' to 'float'
+  ret float %addr
+}
+
+;--- dst_not_int_vec.ll
+; RUN: not llvm-as %t/dst_not_int_vec.ll -o /dev/null 2>&1 | FileCheck -check-prefix=DST_NOT_INT_VEC %s --implicit-check-not="error:"
+define <2 x float> @bad(<2 x ptr> %p) {
+  %addr = ptrtoaddr <2 x ptr> %p to <2 x float>
+  ; DST_NOT_INT_VEC: [[#@LINE-1]]:21: error: invalid cast opcode for cast from '<2 x ptr>' to '<2 x float>'
+  ret <2 x float> %addr
+}
+
+;--- src_not_ptr.ll
+; RUN: not llvm-as %t/src_not_ptr.ll -o /dev/null 2>&1 | FileCheck -check-prefix=SRC_NOT_PTR %s --implicit-check-not="error:"
+define i64 @bad(i32 %p) {
+  %addr = ptrtoaddr i32 %p to i64
+  ; SRC_NOT_PTR: [[#@LINE-1]]:21: error: invalid cast opcode for cast from 'i32' to 'i64'
+  ret i64 %addr
+}
+
+;--- src_not_ptr_vec.ll
+; RUN: not llvm-as %t/src_not_ptr_vec.ll -o /dev/null 2>&1 | FileCheck -check-prefix=SRC_NOT_PTR_VEC %s --implicit-check-not="error:"
+define <2 x i64> @bad(<2 x i32> %p) {
+  %addr = ptrtoaddr <2 x i32> %p to <2 x i64>
+  ; SRC_NOT_PTR_VEC: [[#@LINE-1]]:21: error: invalid cast opcode for cast from '<2 x i32>' to '<2 x i64>'
+  ret <2 x i64> %addr
+}
+
+;--- vec_src_fewer_elems.ll
+; RUN: not llvm-as %t/vec_src_fewer_elems.ll -o /dev/null 2>&1 | FileCheck -check-prefix=VEC_SRC_FEWER_ELEMS %s --implicit-check-not="error:"
+define <4 x i64> @bad(<2 x ptr> %p) {
+  %addr = ptrtoaddr <2 x ptr> %p to <4 x i64>
+  ; VEC_SRC_FEWER_ELEMS: [[#@LINE-1]]:21: error: invalid cast opcode for cast from '<2 x ptr>' to '<4 x i64>'
+  ret <4 x i64> %addr
+}
+
+;--- vec_dst_fewer_elems.ll
+; RUN: not llvm-as %t/vec_dst_fewer_elems.ll -o /dev/null 2>&1 | FileCheck -check-prefix=VEC_DST_FEWER_ELEMS %s --implicit-check-not="error:"
+define <2 x i64> @bad(<4 x ptr> %p) {
+  %addr = ptrtoaddr <4 x ptr> %p to <2 x i64>
+  ; VEC_DST_FEWER_ELEMS: [[#@LINE-1]]:21: error: invalid cast opcode for cast from '<4 x ptr>' to '<2 x i64>'
+  ret <2 x i64> %addr
+}
+
+;--- dst_not_addr_size.ll
+; The following invalid IR is caught by the verifier, not the parser:
+; RUN: llvm-as %t/dst_not_addr_size.ll --disable-output --disable-verify
+; RUN: not llvm-as %t/dst_not_addr_size.ll -o /dev/null 2>&1 | FileCheck -check-prefix=DST_NOT_ADDR_SIZE %s --implicit-check-not="error:"
+; DST_NOT_ADDR_SIZE: assembly parsed, but does not verify as correct!
+define i32 @bad(ptr %p) {
+  %addr = ptrtoaddr ptr %p to i32
+  ; DST_NOT_ADDR_SIZE: PtrToAddr result must be address width
+  ret i32 %addr
+}
+define <4 x i32> @bad_vec(<4 x ptr> %p) {
+  %addr = ptrtoaddr <4 x ptr> %p to <4 x i32>
+  ; DST_NOT_ADDR_SIZE: PtrToAddr result must be address width
+  ret <4 x i32> %addr
+}
diff --git a/llvm/test/Assembler/ptrtoaddr.ll b/llvm/test/Assembler/ptrtoaddr.ll
new file mode 100644
index 0000000000000..f21410b474ab6
--- /dev/null
+++ b/llvm/test/Assembler/ptrtoaddr.ll
@@ -0,0 +1,27 @@
+; RUN: llvm-as < %s | llvm-dis | FileCheck %s
+target datalayout = "p1:64:64:64:32"
+
+@i_as0 = global i32 0
+@global_cast_as0 = global i64 ptrtoaddr (ptr @i_as0 to i64)
+; CHECK: @global_cast_as0 = global i64 ptrtoaddr (ptr @i_as0 to i64)
+@i_as1 = addrspace(1) global i32 0
+@global_cast_as1 = global i32 ptrtoaddr (ptr addrspace(1) @i_as1 to i32)
+; CHECK: @global_cast_as1 = global i32 ptrtoaddr (ptr addrspace(1) @i_as1 to i32)
+
+define i64 @test_as0(ptr %p) {
+  %addr = ptrtoaddr ptr %p to i64
+  ; CHECK: %addr = ptrtoaddr ptr %p to i64
+  ret i64 %addr
+}
+
+define i32 @test_as1(ptr addrspace(1) %p) {
+  %addr = ptrtoaddr ptr addrspace(1) %p to i32
+  ; CHECK: %addr = ptrtoaddr ptr addrspace(1) %p to i32
+  ret i32 %addr
+}
+
+define <2 x i32> @test_vec_as1(<2 x ptr addrspace(1)> %p) {
+  %addr = ptrtoaddr <2 x ptr addrspace(1)> %p to <2 x i32>
+  ; CHECK: %addr = ptrtoaddr <2 x ptr addrspace(1)> %p to <2 x i32>
+  ret <2 x i32> %addr
+}
diff --git a/llvm/test/Bindings/llvm-c/add_globaldebuginfo.ll b/llvm/test/Bindings/llvm-c/add_globaldebuginfo.ll
new file mode 100644
index 0000000000000..da6536a9ce407
--- /dev/null
+++ b/llvm/test/Bindings/llvm-c/add_globaldebuginfo.ll
@@ -0,0 +1,2 @@
+; RUN: llvm-c-test --add-globaldebuginfo < /dev/null
+; This used to trigger an assertion
diff --git a/llvm/test/Bitcode/ptrtoaddr.ll b/llvm/test/Bitcode/ptrtoaddr.ll
new file mode 100644
index 0000000000000..6c5fed22d20bf
--- /dev/null
+++ b/llvm/test/Bitcode/ptrtoaddr.ll
@@ -0,0 +1,27 @@
+; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s
+target datalayout = "p1:64:64:64:32"
+
+@i_as0 = global i32 0
+@global_cast_as0 = global i64 ptrtoaddr (ptr @i_as0 to i64)
+; CHECK: @global_cast_as0 = global i64 ptrtoaddr (ptr @i_as0 to i64)
+@i_as1 = addrspace(1) global i32 0
+@global_cast_as1 = global i32 ptrtoaddr (ptr addrspace(1) @i_as1 to i32)
+; CHECK: @global_cast_as1 = global i32 ptrtoaddr (ptr addrspace(1) @i_as1 to i32)
+
+define i64 @test_as0(ptr %p) {
+  %addr = ptrtoaddr ptr %p to i64
+  ; CHECK: %addr = ptrtoaddr ptr %p to i64
+  ret i64 %addr
+}
+
+define i32 @test_as1(ptr addrspace(1) %p) {
+  %addr = ptrtoaddr ptr addrspace(1) %p to i32
+  ; CHECK: %addr = ptrtoaddr ptr addrspace(1) %p to i32
+  ret i32 %addr
+}
+
+define <2 x i32> @test_vec_as1(<2 x ptr addrspace(1)> %p) {
+  %addr = ptrtoaddr <2 x ptr addrspace(1)> %p to <2 x i32>
+  ; CHECK: %addr = ptrtoaddr <2 x ptr addrspace(1)> %p to <2 x i32>
+  ret <2 x i32> %addr
+}
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 3042b8f8d280e..f6333d68a8ea5 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -30,7 +30,6 @@ llvm_canonicalize_cmake_booleans(
   LLVM_INCLUDE_SPIRV_TOOLS_TESTS
   LLVM_APPEND_VC_REV
   LLVM_HAS_LOGF128
-  LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS
   )
 
 configure_lit_site_cfg(
@@ -119,6 +118,7 @@ set(LLVM_TEST_DEPENDS
           llvm-objdump
           llvm-opt-fuzzer
           llvm-opt-report
+          llvm-offload-wrapper
           llvm-otool
           llvm-pdbutil
           llvm-profdata
diff --git a/llvm/test/CodeGen/AArch64/16bit-float-promotion-with-nofp.ll b/llvm/test/CodeGen/AArch64/16bit-float-promotion-with-nofp.ll
index 5d4f9204e7063..c9560e705280b 100644
--- a/llvm/test/CodeGen/AArch64/16bit-float-promotion-with-nofp.ll
+++ b/llvm/test/CodeGen/AArch64/16bit-float-promotion-with-nofp.ll
@@ -77,7 +77,7 @@ define double @selectcc_f64(double %a, double %b, i32 %d) {
 ; CHECK-LABEL: selectcc_f64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cmp w2, #0
-; CHECK-NEXT:    csel x0, x0, x1, lt
+; CHECK-NEXT:    csel x0, x0, x1, mi
 ; CHECK-NEXT:    ret
 entry:
   %c = icmp slt i32 %d, 0
@@ -89,7 +89,7 @@ define float @selectcc_f32(float %a, float %b, i32 %d) {
 ; CHECK-LABEL: selectcc_f32:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cmp w2, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
 entry:
   %c = icmp slt i32 %d, 0
@@ -101,7 +101,7 @@ define half @selectcc_f16(half %a, half %b, i32 %d) {
 ; CHECK-LABEL: selectcc_f16:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cmp w2, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
 entry:
   %c = icmp slt i32 %d, 0
@@ -113,7 +113,7 @@ define bfloat @selectcc_bf16(bfloat %a, bfloat %b, i32 %d) {
 ; CHECK-LABEL: selectcc_bf16:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cmp w2, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
 entry:
   %c = icmp slt i32 %d, 0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ptradd-reassociation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ptradd-reassociation.mir
index 544cf0f6870aa..764f1c06a9b93 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ptradd-reassociation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ptradd-reassociation.mir
@@ -320,3 +320,245 @@ body:             |
     RET_ReallyLR
 
 ...
+---
+name:            rhs_add_nuw
+alignment:       4
+liveins:
+  - { reg: '$x0' }
+  - { reg: '$x1' }
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: rhs_add_nuw
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[PTR_ADD]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s8))
+    ; CHECK-NEXT: $w0 = COPY [[LOAD]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = G_CONSTANT i64 1
+    %3:_(s64) = nuw G_ADD %1, %2
+    %4:_(p0) = nuw G_PTR_ADD %0, %3(s64)
+    %7:_(s32) = G_LOAD %4(p0) :: (load 1)
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            rhs_add_nuw_nusw
+alignment:       4
+liveins:
+  - { reg: '$x0' }
+  - { reg: '$x1' }
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: rhs_add_nuw_nusw
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw nusw G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw nusw G_PTR_ADD [[PTR_ADD]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s8))
+    ; CHECK-NEXT: $w0 = COPY [[LOAD]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = G_CONSTANT i64 1
+    %3:_(s64) = nuw G_ADD %1, %2
+    %4:_(p0) = nuw nusw G_PTR_ADD %0, %3(s64)
+    %7:_(s32) = G_LOAD %4(p0) :: (load 1)
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            rhs_add_nuw_inbounds
+alignment:       4
+liveins:
+  - { reg: '$x0' }
+  - { reg: '$x1' }
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: rhs_add_nuw_inbounds
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw nusw inbounds G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw nusw inbounds G_PTR_ADD [[PTR_ADD]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s8))
+    ; CHECK-NEXT: $w0 = COPY [[LOAD]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = G_CONSTANT i64 1
+    %3:_(s64) = nuw G_ADD %1, %2
+    %4:_(p0) = nuw nusw inbounds G_PTR_ADD %0, %3(s64)
+    %7:_(s32) = G_LOAD %4(p0) :: (load 1)
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            reassoc_nuw
+alignment:       4
+liveins:
+  - { reg: '$x0' }
+  - { reg: '$x1' }
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: reassoc_nuw
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[PTR_ADD]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s8))
+    ; CHECK-NEXT: $w0 = COPY [[LOAD]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = G_CONSTANT i64 1
+    %3:_(p0) = nuw G_PTR_ADD %0, %2(s64)
+    %4:_(p0) = nuw G_PTR_ADD %3(p0), %1
+    %7:_(s32) = G_LOAD %4(p0) :: (load 1)
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            reassoc_nuw_nusw
+alignment:       4
+liveins:
+  - { reg: '$x0' }
+  - { reg: '$x1' }
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: reassoc_nuw_nusw
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw nusw G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw nusw G_PTR_ADD [[PTR_ADD]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s8))
+    ; CHECK-NEXT: $w0 = COPY [[LOAD]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = G_CONSTANT i64 1
+    %3:_(p0) = nuw nusw G_PTR_ADD %0, %2(s64)
+    %4:_(p0) = nuw nusw G_PTR_ADD %3(p0), %1
+    %7:_(s32) = G_LOAD %4(p0) :: (load 1)
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            reassoc_nuw_inbounds
+alignment:       4
+liveins:
+  - { reg: '$x0' }
+  - { reg: '$x1' }
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: reassoc_nuw_inbounds
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw nusw inbounds G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw nusw inbounds G_PTR_ADD [[PTR_ADD]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s8))
+    ; CHECK-NEXT: $w0 = COPY [[LOAD]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = COPY $x1
+    %2:_(s64) = G_CONSTANT i64 1
+    %3:_(p0) = nuw nusw inbounds G_PTR_ADD %0, %2(s64)
+    %4:_(p0) = nuw nusw inbounds G_PTR_ADD %3(p0), %1
+    %7:_(s32) = G_LOAD %4(p0) :: (load 1)
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            combine_offsets_nuw
+alignment:       4
+liveins:
+  - { reg: '$x0' }
+body:             |
+  bb.1:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: combine_offsets_nuw
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 11
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8))
+    ; CHECK-NEXT: $w0 = COPY [[LOAD]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = G_CONSTANT i64 10
+    %2:_(s64) = G_CONSTANT i64 1
+    %3:_(p0) = nuw G_PTR_ADD %0, %1
+    %4:_(p0) = nuw G_PTR_ADD %3(p0), %2
+    %7:_(s32) = G_LOAD %4(p0) :: (load 1)
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            combine_offsets_nuw_inbounds
+alignment:       4
+liveins:
+  - { reg: '$x0' }
+body:             |
+  bb.1:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: combine_offsets_nuw_inbounds
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 11
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw nusw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s8))
+    ; CHECK-NEXT: $w0 = COPY [[LOAD]](s32)
+    ; CHECK-NEXT: RET_ReallyLR implicit $w0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = G_CONSTANT i64 10
+    %2:_(s64) = G_CONSTANT i64 1
+    %3:_(p0) = nuw nusw inbounds G_PTR_ADD %0, %1
+    %4:_(p0) = nuw nusw inbounds G_PTR_ADD %3(p0), %2
+    %7:_(s32) = G_LOAD %4(p0) :: (load 1)
+    $w0 = COPY %7(s32)
+    RET_ReallyLR implicit $w0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
index b681e3b223117..7872c027aff2b 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-udiv.ll
@@ -155,16 +155,12 @@ define <16 x i8> @combine_vec_udiv_nonuniform4(<16 x i8> %x) {
 ; SDAG:       // %bb.0:
 ; SDAG-NEXT:    movi v1.16b, #171
 ; SDAG-NEXT:    adrp x8, .LCPI4_0
-; SDAG-NEXT:    adrp x9, .LCPI4_1
-; SDAG-NEXT:    ldr q3, [x9, :lo12:.LCPI4_1]
 ; SDAG-NEXT:    umull2 v2.8h, v0.16b, v1.16b
 ; SDAG-NEXT:    umull v1.8h, v0.8b, v1.8b
-; SDAG-NEXT:    and v0.16b, v0.16b, v3.16b
 ; SDAG-NEXT:    uzp2 v1.16b, v1.16b, v2.16b
 ; SDAG-NEXT:    ldr q2, [x8, :lo12:.LCPI4_0]
 ; SDAG-NEXT:    ushr v1.16b, v1.16b, #7
-; SDAG-NEXT:    and v1.16b, v1.16b, v2.16b
-; SDAG-NEXT:    orr v0.16b, v0.16b, v1.16b
+; SDAG-NEXT:    bif v0.16b, v1.16b, v2.16b
 ; SDAG-NEXT:    ret
 ;
 ; GISEL-LABEL: combine_vec_udiv_nonuniform4:
@@ -192,7 +188,6 @@ define <8 x i16> @pr38477(<8 x i16> %a0) {
 ; SDAG-LABEL: pr38477:
 ; SDAG:       // %bb.0:
 ; SDAG-NEXT:    adrp x8, .LCPI5_0
-; SDAG-NEXT:    adrp x9, .LCPI5_4
 ; SDAG-NEXT:    ldr q1, [x8, :lo12:.LCPI5_0]
 ; SDAG-NEXT:    adrp x8, .LCPI5_1
 ; SDAG-NEXT:    ldr q3, [x8, :lo12:.LCPI5_1]
@@ -203,16 +198,13 @@ define <8 x i16> @pr38477(<8 x i16> %a0) {
 ; SDAG-NEXT:    sub v2.8h, v0.8h, v1.8h
 ; SDAG-NEXT:    umull2 v4.4s, v2.8h, v3.8h
 ; SDAG-NEXT:    umull v2.4s, v2.4h, v3.4h
-; SDAG-NEXT:    ldr q3, [x9, :lo12:.LCPI5_4]
-; SDAG-NEXT:    and v0.16b, v0.16b, v3.16b
 ; SDAG-NEXT:    uzp2 v2.8h, v2.8h, v4.8h
 ; SDAG-NEXT:    add v1.8h, v2.8h, v1.8h
 ; SDAG-NEXT:    ldr q2, [x8, :lo12:.LCPI5_2]
 ; SDAG-NEXT:    adrp x8, .LCPI5_3
 ; SDAG-NEXT:    ushl v1.8h, v1.8h, v2.8h
 ; SDAG-NEXT:    ldr q2, [x8, :lo12:.LCPI5_3]
-; SDAG-NEXT:    and v1.16b, v1.16b, v2.16b
-; SDAG-NEXT:    orr v0.16b, v0.16b, v1.16b
+; SDAG-NEXT:    bif v0.16b, v1.16b, v2.16b
 ; SDAG-NEXT:    ret
 ;
 ; GISEL-LABEL: pr38477:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-reductions.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-reductions.ll
index 16762dc4fd3fe..c791e35946f72 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-reductions.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-reductions.ll
@@ -1,34 +1,48 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -O0 -mtriple=aarch64-apple-ios -global-isel -disable-expand-reductions -stop-after=irtranslator %s -o - | FileCheck %s
 
-declare float @llvm.vector.reduce.fadd.v4f32(float, <4 x float>)
-declare double @llvm.vector.reduce.fmul.v4f64(double, <4 x double>)
-
 define float @fadd_seq(float %start, <4 x float> %vec) {
   ; CHECK-LABEL: name: fadd_seq
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q1, $s0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $s0
-  ; CHECK:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
-  ; CHECK:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY1]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_SEQ_FADD:%[0-9]+]]:_(s32) = G_VECREDUCE_SEQ_FADD [[COPY]](s32), [[BITCAST]](<4 x s32>)
-  ; CHECK:   $s0 = COPY [[VECREDUCE_SEQ_FADD]](s32)
-  ; CHECK:   RET_ReallyLR implicit $s0
+  ; CHECK-NEXT:   liveins: $q1, $s0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY1]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_SEQ_FADD:%[0-9]+]]:_(s32) = G_VECREDUCE_SEQ_FADD [[COPY]](s32), [[BITCAST]](<4 x s32>)
+  ; CHECK-NEXT:   $s0 = COPY [[VECREDUCE_SEQ_FADD]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0
   %res = call float @llvm.vector.reduce.fadd.v4f32(float %start, <4 x float> %vec)
   ret float %res
 }
 
+define float @fadd_seq_scalar(float %start, <1 x float> %vec) {
+  ; CHECK-LABEL: name: fadd_seq_scalar
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $d1, $s0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+  ; CHECK-NEXT:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+  ; CHECK-NEXT:   [[FADD:%[0-9]+]]:_(s32) = G_FADD [[COPY]], [[UV]]
+  ; CHECK-NEXT:   $s0 = COPY [[FADD]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0
+  %res = call float @llvm.vector.reduce.fadd.v1f32(float %start, <1 x float> %vec)
+  ret float %res
+}
+
 define float @fadd_fast(float %start, <4 x float> %vec) {
   ; CHECK-LABEL: name: fadd_fast
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q1, $s0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $s0
-  ; CHECK:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
-  ; CHECK:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY1]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_FADD:%[0-9]+]]:_(s32) = reassoc G_VECREDUCE_FADD [[BITCAST]](<4 x s32>)
-  ; CHECK:   [[FADD:%[0-9]+]]:_(s32) = reassoc G_FADD [[COPY]], [[VECREDUCE_FADD]]
-  ; CHECK:   $s0 = COPY [[FADD]](s32)
-  ; CHECK:   RET_ReallyLR implicit $s0
+  ; CHECK-NEXT:   liveins: $q1, $s0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY1]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_FADD:%[0-9]+]]:_(s32) = reassoc G_VECREDUCE_FADD [[BITCAST]](<4 x s32>)
+  ; CHECK-NEXT:   [[FADD:%[0-9]+]]:_(s32) = reassoc G_FADD [[COPY]], [[VECREDUCE_FADD]]
+  ; CHECK-NEXT:   $s0 = COPY [[FADD]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0
   %res = call reassoc float @llvm.vector.reduce.fadd.v4f32(float %start, <4 x float> %vec)
   ret float %res
 }
@@ -36,48 +50,60 @@ define float @fadd_fast(float %start, <4 x float> %vec) {
 define double @fmul_seq(double %start, <4 x double> %vec) {
   ; CHECK-LABEL: name: fmul_seq
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $d0, $q1, $q2
-  ; CHECK:   [[COPY:%[0-9]+]]:_(s64) = COPY $d0
-  ; CHECK:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
-  ; CHECK:   [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
-  ; CHECK:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[COPY1]](<2 x s64>), [[COPY2]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_SEQ_FMUL:%[0-9]+]]:_(s64) = G_VECREDUCE_SEQ_FMUL [[COPY]](s64), [[CONCAT_VECTORS]](<4 x s64>)
-  ; CHECK:   $d0 = COPY [[VECREDUCE_SEQ_FMUL]](s64)
-  ; CHECK:   RET_ReallyLR implicit $d0
+  ; CHECK-NEXT:   liveins: $d0, $q1, $q2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
+  ; CHECK-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[COPY1]](<2 x s64>), [[COPY2]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_SEQ_FMUL:%[0-9]+]]:_(s64) = G_VECREDUCE_SEQ_FMUL [[COPY]](s64), [[CONCAT_VECTORS]](<4 x s64>)
+  ; CHECK-NEXT:   $d0 = COPY [[VECREDUCE_SEQ_FMUL]](s64)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0
   %res = call double @llvm.vector.reduce.fmul.v4f64(double %start, <4 x double> %vec)
   ret double %res
 }
 
+define double @fmul_seq_scalar(double %start, <1 x double> %vec) {
+  ; CHECK-LABEL: name: fmul_seq_scalar
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $d0, $d1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s64) = COPY $d1
+  ; CHECK-NEXT:   [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[COPY1]]
+  ; CHECK-NEXT:   $d0 = COPY [[FMUL]](s64)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0
+  %res = call double @llvm.vector.reduce.fmul.v1f64(double %start, <1 x double> %vec)
+  ret double %res
+}
+
 define double @fmul_fast(double %start, <4 x double> %vec) {
   ; CHECK-LABEL: name: fmul_fast
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $d0, $q1, $q2
-  ; CHECK:   [[COPY:%[0-9]+]]:_(s64) = COPY $d0
-  ; CHECK:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
-  ; CHECK:   [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
-  ; CHECK:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[COPY1]](<2 x s64>), [[COPY2]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_FMUL:%[0-9]+]]:_(s64) = reassoc G_VECREDUCE_FMUL [[CONCAT_VECTORS]](<4 x s64>)
-  ; CHECK:   [[FMUL:%[0-9]+]]:_(s64) = reassoc G_FMUL [[COPY]], [[VECREDUCE_FMUL]]
-  ; CHECK:   $d0 = COPY [[FMUL]](s64)
-  ; CHECK:   RET_ReallyLR implicit $d0
+  ; CHECK-NEXT:   liveins: $d0, $q1, $q2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
+  ; CHECK-NEXT:   [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[COPY1]](<2 x s64>), [[COPY2]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_FMUL:%[0-9]+]]:_(s64) = reassoc G_VECREDUCE_FMUL [[CONCAT_VECTORS]](<4 x s64>)
+  ; CHECK-NEXT:   [[FMUL:%[0-9]+]]:_(s64) = reassoc G_FMUL [[COPY]], [[VECREDUCE_FMUL]]
+  ; CHECK-NEXT:   $d0 = COPY [[FMUL]](s64)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $d0
   %res = call reassoc double @llvm.vector.reduce.fmul.v4f64(double %start, <4 x double> %vec)
   ret double %res
 }
 
-declare float @llvm.vector.reduce.fmax.v4f32(<4 x float>)
-declare float @llvm.vector.reduce.fmin.v4f32(<4 x float>)
-declare float @llvm.vector.reduce.fmaximum.v4f32(<4 x float>)
-declare float @llvm.vector.reduce.fminimum.v4f32(<4 x float>)
-
 define float @fmax(<4 x float> %vec) {
   ; CHECK-LABEL: name: fmax
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
-  ; CHECK:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_FMAX:%[0-9]+]]:_(s32) = G_VECREDUCE_FMAX [[BITCAST]](<4 x s32>)
-  ; CHECK:   $s0 = COPY [[VECREDUCE_FMAX]](s32)
-  ; CHECK:   RET_ReallyLR implicit $s0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_FMAX:%[0-9]+]]:_(s32) = G_VECREDUCE_FMAX [[BITCAST]](<4 x s32>)
+  ; CHECK-NEXT:   $s0 = COPY [[VECREDUCE_FMAX]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0
   %res = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> %vec)
   ret float %res
 }
@@ -85,12 +111,13 @@ define float @fmax(<4 x float> %vec) {
 define float @fmin(<4 x float> %vec) {
   ; CHECK-LABEL: name: fmin
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
-  ; CHECK:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_FMIN:%[0-9]+]]:_(s32) = G_VECREDUCE_FMIN [[BITCAST]](<4 x s32>)
-  ; CHECK:   $s0 = COPY [[VECREDUCE_FMIN]](s32)
-  ; CHECK:   RET_ReallyLR implicit $s0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_FMIN:%[0-9]+]]:_(s32) = G_VECREDUCE_FMIN [[BITCAST]](<4 x s32>)
+  ; CHECK-NEXT:   $s0 = COPY [[VECREDUCE_FMIN]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0
   %res = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> %vec)
   ret float %res
 }
@@ -98,12 +125,13 @@ define float @fmin(<4 x float> %vec) {
 define float @fmin_nnan(<4 x float> %vec) {
   ; CHECK-LABEL: name: fmin_nnan
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
-  ; CHECK:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_FMIN:%[0-9]+]]:_(s32) = nnan G_VECREDUCE_FMIN [[BITCAST]](<4 x s32>)
-  ; CHECK:   $s0 = COPY [[VECREDUCE_FMIN]](s32)
-  ; CHECK:   RET_ReallyLR implicit $s0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_FMIN:%[0-9]+]]:_(s32) = nnan G_VECREDUCE_FMIN [[BITCAST]](<4 x s32>)
+  ; CHECK-NEXT:   $s0 = COPY [[VECREDUCE_FMIN]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0
   %res = call nnan float @llvm.vector.reduce.fmin.v4f32(<4 x float> %vec)
   ret float %res
 }
@@ -111,12 +139,13 @@ define float @fmin_nnan(<4 x float> %vec) {
 define float @fmaximum(<4 x float> %vec) {
   ; CHECK-LABEL: name: fmaximum
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
-  ; CHECK:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_FMAX:%[0-9]+]]:_(s32) = G_VECREDUCE_FMAXIMUM [[BITCAST]](<4 x s32>)
-  ; CHECK:   $s0 = COPY [[VECREDUCE_FMAX]](s32)
-  ; CHECK:   RET_ReallyLR implicit $s0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_FMAXIMUM:%[0-9]+]]:_(s32) = G_VECREDUCE_FMAXIMUM [[BITCAST]](<4 x s32>)
+  ; CHECK-NEXT:   $s0 = COPY [[VECREDUCE_FMAXIMUM]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0
   %res = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> %vec)
   ret float %res
 }
@@ -124,12 +153,13 @@ define float @fmaximum(<4 x float> %vec) {
 define float @fminimum(<4 x float> %vec) {
   ; CHECK-LABEL: name: fminimum
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
-  ; CHECK:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_FMIN:%[0-9]+]]:_(s32) = G_VECREDUCE_FMINIMUM [[BITCAST]](<4 x s32>)
-  ; CHECK:   $s0 = COPY [[VECREDUCE_FMIN]](s32)
-  ; CHECK:   RET_ReallyLR implicit $s0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_FMINIMUM:%[0-9]+]]:_(s32) = G_VECREDUCE_FMINIMUM [[BITCAST]](<4 x s32>)
+  ; CHECK-NEXT:   $s0 = COPY [[VECREDUCE_FMINIMUM]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0
   %res = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> %vec)
   ret float %res
 }
@@ -137,99 +167,91 @@ define float @fminimum(<4 x float> %vec) {
 define float @fminimum_nnan(<4 x float> %vec) {
   ; CHECK-LABEL: name: fminimum_nnan
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
-  ; CHECK:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
-  ; CHECK:   [[VECREDUCE_FMIN:%[0-9]+]]:_(s32) = nnan G_VECREDUCE_FMINIMUM [[BITCAST]](<4 x s32>)
-  ; CHECK:   $s0 = COPY [[VECREDUCE_FMIN]](s32)
-  ; CHECK:   RET_ReallyLR implicit $s0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; CHECK-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
+  ; CHECK-NEXT:   [[VECREDUCE_FMINIMUM:%[0-9]+]]:_(s32) = nnan G_VECREDUCE_FMINIMUM [[BITCAST]](<4 x s32>)
+  ; CHECK-NEXT:   $s0 = COPY [[VECREDUCE_FMINIMUM]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $s0
   %res = call nnan float @llvm.vector.reduce.fminimum.v4f32(<4 x float> %vec)
   ret float %res
 }
 
-declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
-
 define i32 @add(<4 x i32> %vec) {
   ; CHECK-LABEL: name: add
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-  ; CHECK:   [[VECREDUCE_ADD:%[0-9]+]]:_(s32) = G_VECREDUCE_ADD [[COPY]](<4 x s32>)
-  ; CHECK:   $w0 = COPY [[VECREDUCE_ADD]](s32)
-  ; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[VECREDUCE_ADD:%[0-9]+]]:_(s32) = G_VECREDUCE_ADD [[COPY]](<4 x s32>)
+  ; CHECK-NEXT:   $w0 = COPY [[VECREDUCE_ADD]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   %res = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %vec)
   ret i32 %res
 }
 
-declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>)
-
 define i32 @mul(<4 x i32> %vec) {
   ; CHECK-LABEL: name: mul
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-  ; CHECK:   [[VECREDUCE_MUL:%[0-9]+]]:_(s32) = G_VECREDUCE_MUL [[COPY]](<4 x s32>)
-  ; CHECK:   $w0 = COPY [[VECREDUCE_MUL]](s32)
-  ; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[VECREDUCE_MUL:%[0-9]+]]:_(s32) = G_VECREDUCE_MUL [[COPY]](<4 x s32>)
+  ; CHECK-NEXT:   $w0 = COPY [[VECREDUCE_MUL]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   %res = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %vec)
   ret i32 %res
 }
 
-declare i32 @llvm.vector.reduce.and.v4i32(<4 x i32>)
-
 define i32 @and(<4 x i32> %vec) {
   ; CHECK-LABEL: name: and
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-  ; CHECK:   [[VECREDUCE_AND:%[0-9]+]]:_(s32) = G_VECREDUCE_AND [[COPY]](<4 x s32>)
-  ; CHECK:   $w0 = COPY [[VECREDUCE_AND]](s32)
-  ; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[VECREDUCE_AND:%[0-9]+]]:_(s32) = G_VECREDUCE_AND [[COPY]](<4 x s32>)
+  ; CHECK-NEXT:   $w0 = COPY [[VECREDUCE_AND]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   %res = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %vec)
   ret i32 %res
 }
 
-declare i32 @llvm.vector.reduce.or.v4i32(<4 x i32>)
-
 define i32 @or(<4 x i32> %vec) {
   ; CHECK-LABEL: name: or
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-  ; CHECK:   [[VECREDUCE_OR:%[0-9]+]]:_(s32) = G_VECREDUCE_OR [[COPY]](<4 x s32>)
-  ; CHECK:   $w0 = COPY [[VECREDUCE_OR]](s32)
-  ; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[VECREDUCE_OR:%[0-9]+]]:_(s32) = G_VECREDUCE_OR [[COPY]](<4 x s32>)
+  ; CHECK-NEXT:   $w0 = COPY [[VECREDUCE_OR]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   %res = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %vec)
   ret i32 %res
 }
 
-declare i32 @llvm.vector.reduce.xor.v4i32(<4 x i32>)
-
 define i32 @xor(<4 x i32> %vec) {
   ; CHECK-LABEL: name: xor
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-  ; CHECK:   [[VECREDUCE_XOR:%[0-9]+]]:_(s32) = G_VECREDUCE_XOR [[COPY]](<4 x s32>)
-  ; CHECK:   $w0 = COPY [[VECREDUCE_XOR]](s32)
-  ; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[VECREDUCE_XOR:%[0-9]+]]:_(s32) = G_VECREDUCE_XOR [[COPY]](<4 x s32>)
+  ; CHECK-NEXT:   $w0 = COPY [[VECREDUCE_XOR]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   %res = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> %vec)
   ret i32 %res
 }
 
-declare i32 @llvm.vector.reduce.smax.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.smin.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.umax.v4i32(<4 x i32>)
-declare i32 @llvm.vector.reduce.umin.v4i32(<4 x i32>)
-
 define i32 @smax(<4 x i32> %vec) {
   ; CHECK-LABEL: name: smax
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-  ; CHECK:   [[VECREDUCE_SMAX:%[0-9]+]]:_(s32) = G_VECREDUCE_SMAX [[COPY]](<4 x s32>)
-  ; CHECK:   $w0 = COPY [[VECREDUCE_SMAX]](s32)
-  ; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[VECREDUCE_SMAX:%[0-9]+]]:_(s32) = G_VECREDUCE_SMAX [[COPY]](<4 x s32>)
+  ; CHECK-NEXT:   $w0 = COPY [[VECREDUCE_SMAX]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   %res = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %vec)
   ret i32 %res
 }
@@ -237,11 +259,12 @@ define i32 @smax(<4 x i32> %vec) {
 define i32 @smin(<4 x i32> %vec) {
   ; CHECK-LABEL: name: smin
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-  ; CHECK:   [[VECREDUCE_SMIN:%[0-9]+]]:_(s32) = G_VECREDUCE_SMIN [[COPY]](<4 x s32>)
-  ; CHECK:   $w0 = COPY [[VECREDUCE_SMIN]](s32)
-  ; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[VECREDUCE_SMIN:%[0-9]+]]:_(s32) = G_VECREDUCE_SMIN [[COPY]](<4 x s32>)
+  ; CHECK-NEXT:   $w0 = COPY [[VECREDUCE_SMIN]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   %res = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %vec)
   ret i32 %res
 }
@@ -249,11 +272,12 @@ define i32 @smin(<4 x i32> %vec) {
 define i32 @umax(<4 x i32> %vec) {
   ; CHECK-LABEL: name: umax
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-  ; CHECK:   [[VECREDUCE_UMAX:%[0-9]+]]:_(s32) = G_VECREDUCE_UMAX [[COPY]](<4 x s32>)
-  ; CHECK:   $w0 = COPY [[VECREDUCE_UMAX]](s32)
-  ; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[VECREDUCE_UMAX:%[0-9]+]]:_(s32) = G_VECREDUCE_UMAX [[COPY]](<4 x s32>)
+  ; CHECK-NEXT:   $w0 = COPY [[VECREDUCE_UMAX]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   %res = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %vec)
   ret i32 %res
 }
@@ -261,11 +285,12 @@ define i32 @umax(<4 x i32> %vec) {
 define i32 @umin(<4 x i32> %vec) {
   ; CHECK-LABEL: name: umin
   ; CHECK: bb.1 (%ir-block.0):
-  ; CHECK:   liveins: $q0
-  ; CHECK:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
-  ; CHECK:   [[VECREDUCE_UMIN:%[0-9]+]]:_(s32) = G_VECREDUCE_UMIN [[COPY]](<4 x s32>)
-  ; CHECK:   $w0 = COPY [[VECREDUCE_UMIN]](s32)
-  ; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-NEXT:   liveins: $q0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+  ; CHECK-NEXT:   [[VECREDUCE_UMIN:%[0-9]+]]:_(s32) = G_VECREDUCE_UMIN [[COPY]](<4 x s32>)
+  ; CHECK-NEXT:   $w0 = COPY [[VECREDUCE_UMIN]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
   %res = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> %vec)
   ret i32 %res
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-unmerge.mir b/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-unmerge.mir
new file mode 100644
index 0000000000000..a8334dbeee052
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-unmerge.mir
@@ -0,0 +1,45 @@
+# NOTE: Assertions have been autogenerated by utils/update_givaluetracking_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple aarch64 -passes="print<gisel-value-tracking>" %s -o - 2>&1 | FileCheck %s
+
+---
+name:            Scalar
+body:             |
+  bb.1:
+  ; CHECK-LABEL: name: @Scalar
+  ; CHECK-NEXT: %copy_x0:_ KnownBits:???????????????? SignBits:1
+  ; CHECK-NEXT: %maskf:_ KnownBits:0000000000001111 SignBits:12
+  ; CHECK-NEXT: %x0_x1:_ KnownBits:0000000000001111 SignBits:12
+  ; CHECK-NEXT: %and:_ KnownBits:000000000000???? SignBits:12
+  ; CHECK-NEXT: %x0_0:_ KnownBits:000000000000???? SignBits:12
+  ; CHECK-NEXT: %x0_1:_ KnownBits:000000000000???? SignBits:12
+  ; CHECK-NEXT: %result:_ KnownBits:000000000000???? SignBits:12
+   %copy_x0:_(<2 x s16>) = COPY $w0
+   %maskf:_(s16) = G_CONSTANT i16 15
+   %x0_x1:_(<2 x s16>) = G_BUILD_VECTOR %maskf, %maskf
+   %and:_(<2 x s16>) = G_AND %copy_x0, %x0_x1
+   %x0_0:_(s16), %x0_1:_(s16) = G_UNMERGE_VALUES %and
+   %result:_(s16) = COPY %x0_0
+...
+---
+name:            Vector
+body:             |
+  bb.1:
+  ; CHECK-LABEL: name: @Vector
+  ; CHECK-NEXT: %copy_x0:_ KnownBits:???????? SignBits:1
+  ; CHECK-NEXT: %maskff:_ KnownBits:11111111 SignBits:8
+  ; CHECK-NEXT: %maskf:_ KnownBits:00001111 SignBits:4
+  ; CHECK-NEXT: %x0_x1:_ KnownBits:????1111 SignBits:4
+  ; CHECK-NEXT: %and:_ KnownBits:???????? SignBits:1
+  ; CHECK-NEXT: %x0_0:_ KnownBits:0000???? SignBits:4
+  ; CHECK-NEXT: %x0_1:_ KnownBits:???????? SignBits:1
+  ; CHECK-NEXT: %result1:_ KnownBits:0000???? SignBits:4
+  ; CHECK-NEXT: %result2:_ KnownBits:???????? SignBits:1
+   %copy_x0:_(<4 x s8>) = COPY $w0
+   %maskff:_(s8) = G_CONSTANT i8 255
+   %maskf:_(s8) = G_CONSTANT i8 15
+   %x0_x1:_(<4 x s8>) = G_BUILD_VECTOR %maskf, %maskf, %maskff, %maskff
+   %and:_(<4 x s8>) = G_AND %copy_x0, %x0_x1
+   %x0_0:_(<2 x s8>), %x0_1:_(<2 x s8>) = G_UNMERGE_VALUES %and
+   %result1:_(<2 x s8>) = COPY %x0_0
+   %result2:_(<2 x s8>) = COPY %x0_1
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 5c164bf672082..040f97f96ee21 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -321,14 +321,16 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_TRUNC_SSAT_S (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_TRUNC_SSAT_U (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_TRUNC_USAT_U (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_CONSTANT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
@@ -672,8 +674,8 @@
 # DEBUG-NEXT: .. the first uncovered type index: 2, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_BR (opcode {{[0-9]+}}): 0 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_BRJT (opcode {{[0-9]+}}): 2 type indices
 # DEBUG-NEXT: .. the first uncovered type index: 2, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
@@ -855,14 +857,16 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_TRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_DEBUGTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_UBSANTRAP (opcode {{[0-9]+}}): 0 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_VECREDUCE_SEQ_FADD (opcode {{[0-9]+}}): 3 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/opt-and-tbnz-tbz.mir b/llvm/test/CodeGen/AArch64/GlobalISel/opt-and-tbnz-tbz.mir
index 95ae12f6d59db..a5b6ea487aac4 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/opt-and-tbnz-tbz.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/opt-and-tbnz-tbz.mir
@@ -149,7 +149,7 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32 = COPY $w0
   ; CHECK-NEXT:   [[ANDSWri:%[0-9]+]]:gpr32 = ANDSWri [[COPY]], 0, implicit-def $nzcv
-  ; CHECK-NEXT:   Bcc 11, %bb.1, implicit $nzcv
+  ; CHECK-NEXT:   Bcc 4, %bb.1, implicit $nzcv
   ; CHECK-NEXT:   B %bb.0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combine-ptr-add-chain.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combine-ptr-add-chain.mir
index 4ed9322802693..cfce267f40096 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combine-ptr-add-chain.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-combine-ptr-add-chain.mir
@@ -27,3 +27,51 @@ body:             |
     $x0 = COPY %4(p0)
     RET_ReallyLR implicit $x0
 ...
+---
+name: ptradd_chain_nuw
+tracksRegLiveness: true
+legalized: true
+body:             |
+  bb.1:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: ptradd_chain_nuw
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: $x0 = COPY [[PTR_ADD]](p0)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = G_CONSTANT i64 4
+    %2:_(s64) = G_CONSTANT i64 12
+    %3:_(p0) = nuw G_PTR_ADD %0(p0), %1
+    %4:_(p0) = nuw G_PTR_ADD %3(p0), %2
+    $x0 = COPY %4(p0)
+    RET_ReallyLR implicit $x0
+...
+---
+name: ptradd_chain_nuw_inbounds
+tracksRegLiveness: true
+legalized: true
+body:             |
+  bb.1:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: ptradd_chain_nuw_inbounds
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw nusw inbounds G_PTR_ADD [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: $x0 = COPY [[PTR_ADD]](p0)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %0:_(p0) = COPY $x0
+    %1:_(s64) = G_CONSTANT i64 4
+    %2:_(s64) = G_CONSTANT i64 12
+    %3:_(p0) = nuw nusw inbounds G_PTR_ADD %0(p0), %1
+    %4:_(p0) = nuw nusw inbounds G_PTR_ADD %3(p0), %2
+    $x0 = COPY %4(p0)
+    RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-adjust-icmp-imm.mir b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-adjust-icmp-imm.mir
index edc33e340c9b6..3b991c3d910d5 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-adjust-icmp-imm.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/postlegalizer-lowering-adjust-icmp-imm.mir
@@ -661,7 +661,7 @@ body:             |
     ; SELECT-NEXT: %reg0:gpr32common = COPY $w0
     ; SELECT-NEXT: %reg1:gpr32 = COPY $w1
     ; SELECT-NEXT: [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri %reg0, 0, 0, implicit-def $nzcv
-    ; SELECT-NEXT: %select:gpr32 = CSELWr %reg0, %reg1, 11, implicit $nzcv
+    ; SELECT-NEXT: %select:gpr32 = CSELWr %reg0, %reg1, 4, implicit $nzcv
     ; SELECT-NEXT: $w0 = COPY %select
     ; SELECT-NEXT: RET_ReallyLR implicit $w0
     %reg0:_(s32) = COPY $w0
@@ -699,7 +699,7 @@ body:             |
     ; SELECT-NEXT: {{  $}}
     ; SELECT-NEXT: %reg0:gpr64 = COPY $x0
     ; SELECT-NEXT: [[ANDSXri:%[0-9]+]]:gpr64 = ANDSXri %reg0, 8000, implicit-def $nzcv
-    ; SELECT-NEXT: %cmp:gpr32 = CSINCWr $wzr, $wzr, 11, implicit $nzcv
+    ; SELECT-NEXT: %cmp:gpr32 = CSINCWr $wzr, $wzr, 4, implicit $nzcv
     ; SELECT-NEXT: $w0 = COPY %cmp
     ; SELECT-NEXT: RET_ReallyLR implicit $w0
     %reg0:gpr(s64) = COPY $x0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-fp-casts.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-fp-casts.mir
index 0ee0d9587e88b..a58492114003a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-fp-casts.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-fp-casts.mir
@@ -357,31 +357,6 @@ body:             |
     $d0 = COPY %1(s64)
 ...
 
----
-name:            sitofp_s64_s32_fpr_both
-legalized:       true
-regBankSelected: true
-
-registers:
-  - { id: 0, class: fpr }
-  - { id: 1, class: fpr }
-
-body:             |
-  bb.0:
-    liveins: $s0
-
-    ; CHECK-LABEL: name: sitofp_s64_s32_fpr_both
-    ; CHECK: liveins: $s0
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr32 = COPY $s0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr32 = COPY [[COPY]]
-    ; CHECK-NEXT: [[SCVTFUWDri:%[0-9]+]]:fpr64 = nofpexcept SCVTFUWDri [[COPY1]]
-    ; CHECK-NEXT: $d0 = COPY [[SCVTFUWDri]]
-    %0(s32) = COPY $s0
-    %1(s64) = G_SITOFP %0
-    $d0 = COPY %1(s64)
-...
-
 ---
 name:            sitofp_s64_s64_fpr
 legalized:       true
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-tbnz-from-cmp.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-tbnz-from-cmp.mir
index 30db00aa97813..67262c27e2059 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-tbnz-from-cmp.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-tbnz-from-cmp.mir
@@ -166,7 +166,7 @@ body:             |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   %copy:gpr64 = COPY $x0
   ; CHECK-NEXT:   [[ANDSXri:%[0-9]+]]:gpr64 = ANDSXri %copy, 8000, implicit-def $nzcv
-  ; CHECK-NEXT:   Bcc 11, %bb.1, implicit $nzcv
+  ; CHECK-NEXT:   Bcc 4, %bb.1, implicit $nzcv
   ; CHECK-NEXT:   B %bb.0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/store-merging-debug.mir b/llvm/test/CodeGen/AArch64/GlobalISel/store-merging-debug.mir
index 56b3428a1bdf8..82fc3c1b06514 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/store-merging-debug.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/store-merging-debug.mir
@@ -86,8 +86,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/vararg.mir b/llvm/test/CodeGen/AArch64/GlobalISel/vararg.mir
index 437a9e6cb89ac..3f14162d29f8d 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/vararg.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/vararg.mir
@@ -10,7 +10,7 @@
   define i32 @va_start(ptr %a, ...) {
   entry:
     %ap = alloca %struct.__va_list, align 8
-    call void @llvm.lifetime.start.p0(i64 32, ptr %ap)
+    call void @llvm.lifetime.start.p0(ptr %ap)
     call void @llvm.va_start.p0(ptr %ap)
     %vr_offs_p = getelementptr inbounds i8, ptr %ap, i64 28
     %vr_offs = load i32, ptr %vr_offs_p, align 4
diff --git a/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes-limit-size.mir b/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes-limit-size.mir
new file mode 100644
index 0000000000000..17c15124e787e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes-limit-size.mir
@@ -0,0 +1,33 @@
+# RUN: llc -run-pass=machine-combiner -aarch64-search-limit=2 -mcpu=neoverse-n2 -mtriple=aarch64-none-linux-gnu -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            negative_pattern_mbb_too_large
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3, $x4
+
+    ; CHECK-LABEL: name: negative_pattern_mbb_too_large
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
+    ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1
+    ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i32]], %subreg.ssub
+    ; CHECK-NEXT: [[LD_LANE_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, killed [[COPY2]]
+    ; CHECK-NEXT: [[LD_LANE_2:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_1]], 2, killed [[COPY3]]
+    ; CHECK-NEXT: [[LD_LANE_3:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_2]], 3, killed [[COPY4]]
+    ; CHECK-NEXT: $q0 = COPY [[LD_LANE_3]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %4:gpr64common = COPY $x4
+    %5:fpr32 = LDRSroX %0, killed %1, 0, 1
+    %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub
+    %7:fpr128 = LD1i32 %6, 1, killed %2
+    %8:fpr128 = LD1i32 %7, 2, killed %3
+    %9:fpr128 = LD1i32 %8, 3, killed %4
+    $q0 = COPY %9
+    RET_ReallyLR implicit $q0
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes-with-call.mir b/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes-with-call.mir
new file mode 100644
index 0000000000000..6b338d98afb53
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes-with-call.mir
@@ -0,0 +1,45 @@
+# RUN: llc -run-pass=machine-combiner -mcpu=neoverse-n2 -mtriple=aarch64-none-linux-gnu -verify-machineinstrs %s -o - | FileCheck %s
+
+
+--- |
+  @external_func = external global i32
+  define void @negative_pattern_offset_reg_copied_to_physical(i64 %arg0, i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4) {
+  entry:  
+    ret void
+  }
+...
+---
+name:            negative_pattern_offset_reg_copied_to_physical
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3
+
+    ; CHECK-LABEL: name: negative_pattern_offset_reg_copied_to_physical
+    ; CHECK: [[BASE_REG:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[PTR_1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[PTR_2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[PTR_3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[BASE_REG]], killed [[PTR_1]], 0, 1
+    ; CHECK-NEXT: [[LD_LANE_0:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i32]], %subreg.ssub
+    ; CHECK-NEXT: [[LD_LANE_1:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_0]], 1, [[PTR_2]]
+    ; CHECK-NEXT: $x0 = COPY [[PTR_2]]
+    ; CHECK-NEXT: BL @external_func, csr_aarch64_aapcs, implicit-def $lr, implicit $x0, implicit-def $x0
+    ; CHECK-NEXT: [[LD_LANE_2:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_1]], 2, killed [[PTR_2]]
+    ; CHECK-NEXT: [[LD_LANE_3:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_2]], 3, killed [[PTR_3]]
+    ; CHECK-NEXT: [[RESULT:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: $q0 = COPY [[LD_LANE_3]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %5:fpr32 = LDRSroX %0, killed %1, 0, 1
+    %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub
+    %7:fpr128 = LD1i32 %6, 1, %2
+    $x0 = COPY %2
+    BL @external_func, csr_aarch64_aapcs, implicit-def $lr, implicit $x0, implicit-def $x0
+    %8:fpr128 = LD1i32 %7, 2, killed %2
+    %9:fpr128 = LD1i32 %8, 3, killed %3
+    %10:gpr64common = COPY $x0
+    $q0 = COPY %9
+    RET_ReallyLR implicit $q0
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir b/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir
new file mode 100644
index 0000000000000..a7570d2293f8a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-combine-gather-lanes.mir
@@ -0,0 +1,400 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -run-pass=machine-combiner -mcpu=neoverse-n2 -mtriple=aarch64-none-linux-gnu -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            split_loads_to_fpr128
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3, $x4
+
+    ; CHECK-LABEL: name: split_loads_to_fpr128
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
+    ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], [[COPY1]], 0, 1
+    ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, [[LD_i32]], %subreg.ssub
+    ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, [[COPY2]] 
+    ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr32 = LDRSui [[COPY3]], 0
+    ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.ssub
+    ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i32 [[SECOND_REG]], 1, [[COPY4]]
+    ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_1]], [[LD1_1]]
+    ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %4:gpr64common = COPY $x4
+    %5:fpr32 = LDRSroX %0, %1, 0, 1
+    %6:fpr128 = SUBREG_TO_REG 0, %5, %subreg.ssub
+    %7:fpr128 = LD1i32 %6, 1, %2
+    %8:fpr128 = LD1i32 %7, 2, %3
+    %9:fpr128 = LD1i32 %8, 3, %4
+    $q0 = COPY %9
+    RET_ReallyLR implicit $q0
+
+---
+name:            split_loads_to_fpr128_ui
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3, $x4
+
+    ; CHECK-LABEL: name: split_loads_to_fpr128_ui
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
+    ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSui [[COPY]], 0
+    ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i32]], %subreg.ssub
+    ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, killed [[COPY1]] 
+    ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr32 = LDRSui [[COPY2]], 0
+    ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.ssub
+    ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i32 [[SECOND_REG]], 1, killed [[COPY3]]
+    ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_1]], [[LD1_1]]
+    ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %4:gpr64common = COPY $x4
+    %5:fpr32 = LDRSui %0, 0
+    %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub
+    %7:fpr128 = LD1i32 %6, 1, killed %1
+    %8:fpr128 = LD1i32 %7, 2, killed %2
+    %9:fpr128 = LD1i32 %8, 3, killed %3
+    $q0 = COPY %9
+    RET_ReallyLR implicit $q0
+
+---
+name:            split_loads_to_fpr128_i16
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8
+
+    ; CHECK-LABEL: name: split_loads_to_fpr128_i16
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64common = COPY $x5
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr64common = COPY $x6
+    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr64common = COPY $x7
+    ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr64common = COPY $x8
+    ; CHECK-NEXT: [[LD_i16:%[0-9]+]]:fpr16 = LDRHroX [[COPY]], killed [[COPY1]], 0, 1
+    ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i16]], %subreg.hsub
+    ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i16 [[FIRST_REG]], 1, killed [[COPY2]]
+    ; CHECK-NEXT: [[LD0_2:%[0-9]+]]:fpr128 = LD1i16 [[LD0_1]], 2, killed [[COPY3]]
+    ; CHECK-NEXT: [[LD0_3:%[0-9]+]]:fpr128 = LD1i16 [[LD0_2]], 3, killed [[COPY4]]
+    ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr16 = LDRHui [[COPY5]], 0
+    ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.hsub
+    ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i16 [[SECOND_REG]], 1, killed [[COPY6]]
+    ; CHECK-NEXT: [[LD1_2:%[0-9]+]]:fpr128 = LD1i16 [[LD1_1]], 2, killed [[COPY7]]
+    ; CHECK-NEXT: [[LD1_3:%[0-9]+]]:fpr128 = LD1i16 [[LD1_2]], 3, killed [[COPY8]]
+    ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_3]], [[LD1_3]]
+    ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %4:gpr64common = COPY $x4
+    %5:gpr64common = COPY $x5
+    %6:gpr64common = COPY $x6
+    %7:gpr64common = COPY $x7
+    %8:gpr64common = COPY $x8
+    %9:fpr16 = LDRHroX %0, killed %1, 0, 1
+    %10:fpr128 = SUBREG_TO_REG 0, killed %9, %subreg.hsub
+    %11:fpr128 = LD1i16 %10, 1, killed %2
+    %12:fpr128 = LD1i16 %11, 2, killed %3
+    %13:fpr128 = LD1i16 %12, 3, killed %4
+    %14:fpr128 = LD1i16 %13, 4, killed %5
+    %15:fpr128 = LD1i16 %14, 5, killed %6
+    %16:fpr128 = LD1i16 %15, 6, killed %7
+    %17:fpr128 = LD1i16 %16, 7, killed %8
+    $q0 = COPY %17
+    RET_ReallyLR implicit $q0
+
+---
+name:            split_loads_to_fpr128_i16_ui
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8
+
+    ; CHECK-LABEL: name: split_loads_to_fpr128_i16_ui
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64common = COPY $x5
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr64common = COPY $x6
+    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr64common = COPY $x7
+    ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr64common = COPY $x8
+    ; CHECK-NEXT: [[LD_i16:%[0-9]+]]:fpr16 = LDRHui [[COPY]], 0
+    ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i16]], %subreg.hsub
+    ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i16 [[FIRST_REG]], 1, killed [[COPY1]]
+    ; CHECK-NEXT: [[LD0_2:%[0-9]+]]:fpr128 = LD1i16 [[LD0_1]], 2, killed [[COPY2]]
+    ; CHECK-NEXT: [[LD0_3:%[0-9]+]]:fpr128 = LD1i16 [[LD0_2]], 3, killed [[COPY3]]
+    ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr16 = LDRHui [[COPY4]], 0
+    ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.hsub
+    ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i16 [[SECOND_REG]], 1, killed [[COPY5]]
+    ; CHECK-NEXT: [[LD1_2:%[0-9]+]]:fpr128 = LD1i16 [[LD1_1]], 2, killed [[COPY6]]
+    ; CHECK-NEXT: [[LD1_3:%[0-9]+]]:fpr128 = LD1i16 [[LD1_2]], 3, killed [[COPY7]]
+    ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_3]], [[LD1_3]]
+    ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %4:gpr64common = COPY $x4
+    %5:gpr64common = COPY $x5
+    %6:gpr64common = COPY $x6
+    %7:gpr64common = COPY $x7
+    %8:gpr64common = COPY $x8
+    %9:fpr16 = LDRHui %0, 0
+    %10:fpr128 = SUBREG_TO_REG 0, killed %9, %subreg.hsub
+    %11:fpr128 = LD1i16 %10, 1, killed %1
+    %12:fpr128 = LD1i16 %11, 2, killed %2
+    %13:fpr128 = LD1i16 %12, 3, killed %3
+    %14:fpr128 = LD1i16 %13, 4, killed %4
+    %15:fpr128 = LD1i16 %14, 5, killed %5
+    %16:fpr128 = LD1i16 %15, 6, killed %6
+    %17:fpr128 = LD1i16 %16, 7, killed %7
+    $q0 = COPY %17
+    RET_ReallyLR implicit $q0
+
+---
+name:            split_loads_to_fpr128_i8
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x16
+
+    ; CHECK-LABEL: name: split_loads_to_fpr128_i8
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gpr64common = COPY $x5
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:gpr64common = COPY $x6
+    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:gpr64common = COPY $x7
+    ; CHECK-NEXT: [[COPY8:%[0-9]+]]:gpr64common = COPY $x8
+    ; CHECK-NEXT: [[COPY9:%[0-9]+]]:gpr64common = COPY $x9
+    ; CHECK-NEXT: [[COPY10:%[0-9]+]]:gpr64common = COPY $x10
+    ; CHECK-NEXT: [[COPY11:%[0-9]+]]:gpr64common = COPY $x11
+    ; CHECK-NEXT: [[COPY12:%[0-9]+]]:gpr64common = COPY $x12
+    ; CHECK-NEXT: [[COPY13:%[0-9]+]]:gpr64common = COPY $x13
+    ; CHECK-NEXT: [[COPY14:%[0-9]+]]:gpr64common = COPY $x14
+    ; CHECK-NEXT: [[COPY15:%[0-9]+]]:gpr64common = COPY $x15
+    ; CHECK-NEXT: [[COPY16:%[0-9]+]]:gpr64common = COPY $x16
+    ; CHECK-NEXT: [[LD_i8:%[0-9]+]]:fpr8 = LDRBroX [[COPY]], killed [[COPY1]], 0, 0
+    ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i8]], %subreg.bsub
+    ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i8 [[FIRST_REG]], 1, killed [[COPY2]]
+    ; CHECK-NEXT: [[LD0_2:%[0-9]+]]:fpr128 = LD1i8 [[LD0_1]], 2, killed [[COPY3]]
+    ; CHECK-NEXT: [[LD0_3:%[0-9]+]]:fpr128 = LD1i8 [[LD0_2]], 3, killed [[COPY4]]
+    ; CHECK-NEXT: [[LD0_4:%[0-9]+]]:fpr128 = LD1i8 [[LD0_3]], 4, killed [[COPY5]]
+    ; CHECK-NEXT: [[LD0_5:%[0-9]+]]:fpr128 = LD1i8 [[LD0_4]], 5, killed [[COPY6]]
+    ; CHECK-NEXT: [[LD0_6:%[0-9]+]]:fpr128 = LD1i8 [[LD0_5]], 6, killed [[COPY7]]
+    ; CHECK-NEXT: [[LD0_7:%[0-9]+]]:fpr128 = LD1i8 [[LD0_6]], 7, killed [[COPY8]]
+    ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr8 = LDRBui [[COPY9]], 0
+    ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.bsub
+    ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i8 [[SECOND_REG]], 1, killed [[COPY10]]
+    ; CHECK-NEXT: [[LD1_2:%[0-9]+]]:fpr128 = LD1i8 [[LD1_1]], 2, killed [[COPY11]]
+    ; CHECK-NEXT: [[LD1_3:%[0-9]+]]:fpr128 = LD1i8 [[LD1_2]], 3, killed [[COPY12]]
+    ; CHECK-NEXT: [[LD1_4:%[0-9]+]]:fpr128 = LD1i8 [[LD1_3]], 4, killed [[COPY13]]
+    ; CHECK-NEXT: [[LD1_5:%[0-9]+]]:fpr128 = LD1i8 [[LD1_4]], 5, killed [[COPY14]]
+    ; CHECK-NEXT: [[LD1_6:%[0-9]+]]:fpr128 = LD1i8 [[LD1_5]], 6, killed [[COPY15]]
+    ; CHECK-NEXT: [[LD1_7:%[0-9]+]]:fpr128 = LD1i8 [[LD1_6]], 7, killed [[COPY16]]
+    ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_7]], [[LD1_7]]
+    ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %4:gpr64common = COPY $x4
+    %5:gpr64common = COPY $x5
+    %6:gpr64common = COPY $x6
+    %7:gpr64common = COPY $x7
+    %8:gpr64common = COPY $x8
+    %9:gpr64common = COPY $x9
+    %10:gpr64common = COPY $x10
+    %11:gpr64common = COPY $x11
+    %12:gpr64common = COPY $x12
+    %13:gpr64common = COPY $x13
+    %14:gpr64common = COPY $x14
+    %15:gpr64common = COPY $x15
+    %16:gpr64common = COPY $x16
+    %17:fpr8 = LDRBroX %0, killed %1, 0, 0
+    %18:fpr128 = SUBREG_TO_REG 0, killed %17, %subreg.bsub
+    %19:fpr128 = LD1i8 %18, 1, killed %2
+    %20:fpr128 = LD1i8 %19, 2, killed %3
+    %21:fpr128 = LD1i8 %20, 3, killed %4
+    %22:fpr128 = LD1i8 %21, 4, killed %5
+    %23:fpr128 = LD1i8 %22, 5, killed %6
+    %24:fpr128 = LD1i8 %23, 6, killed %7
+    %25:fpr128 = LD1i8 %24, 7, killed %8
+    %26:fpr128 = LD1i8 %25, 8, killed %9
+    %27:fpr128 = LD1i8 %26, 9, killed %10
+    %28:fpr128 = LD1i8 %27, 10, killed %11
+    %29:fpr128 = LD1i8 %28, 11, killed %12
+    %30:fpr128 = LD1i8 %29, 12, killed %13
+    %31:fpr128 = LD1i8 %30, 13, killed %14
+    %32:fpr128 = LD1i8 %31, 14, killed %15
+    %33:fpr128 = LD1i8 %32, 15, killed %16
+    $q0 = COPY %33
+    RET_ReallyLR implicit $q0
+
+---
+name:            negative_pattern_missing_lanes
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: negative_pattern_missing_lanes
+    ; CHECK:      [[LD1:%.*]]:fpr128 = LDRQui $x1, 0
+    ; CHECK-NEXT: [[LD2:%.*]]:fpr128 = LD1i32 [[LD1]]
+  
+    %0:gpr64common = COPY $x0
+    %1:fpr128 = LDRQui $x1, 0
+    %2:fpr128 = LD1i32 %1, 3, %0
+    $q0 = COPY %2
+    RET_ReallyLR implicit $q0
+
+---
+name:            out_of_order_lanes
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3, $x4
+
+    ; CHECK-LABEL: name: out_of_order_lanes
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
+    ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1
+    ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i32]], %subreg.ssub
+    ; CHECK-NEXT: [[LD0_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, killed [[COPY3]] 
+    ; CHECK-NEXT: [[LD1_0:%[0-9]+]]:fpr32 = LDRSui [[COPY2]], 0
+    ; CHECK-NEXT: [[SECOND_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD1_0]], %subreg.ssub
+    ; CHECK-NEXT: [[LD1_1:%[0-9]+]]:fpr128 = LD1i32 [[SECOND_REG]], 1, killed [[COPY4]]
+    ; CHECK-NEXT: [[ZIP:%[0-9]+]]:fpr128 = ZIP1v2i64 [[LD0_1]], [[LD1_1]]
+    ; CHECK-NEXT: $q0 = COPY [[ZIP]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %4:gpr64common = COPY $x4
+    %5:fpr32 = LDRSroX %0, killed %1, 0, 1
+    %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub
+    %7:fpr128 = LD1i32 %6, 2, killed %2
+    %8:fpr128 = LD1i32 %7, 1, killed %3
+    %9:fpr128 = LD1i32 %8, 3, killed %4
+    $q0 = COPY %9
+    RET_ReallyLR implicit $q0
+
+---
+name:            negative_pattern_no_subreg_to_reg
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3
+
+    ; CHECK-LABEL: name: negative_pattern_no_subreg_to_reg
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[INITIAL_VEC:%[0-9]+]]:fpr128 = LDRQui [[COPY]], 0
+    ; CHECK-NEXT: [[LD_LANE_1:%[0-9]+]]:fpr128 = LD1i32 [[INITIAL_VEC]], 1, killed [[COPY1]]
+    ; CHECK-NEXT: [[LD_LANE_2:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_1]], 2, killed [[COPY2]]
+    ; CHECK-NEXT: [[LD_LANE_3:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_2]], 3, killed [[COPY3]]
+    ; CHECK-NEXT: $q0 = COPY [[LD_LANE_3]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %4:fpr128 = LDRQui %0, 0            
+    %5:fpr128 = LD1i32 %4, 1, killed %1 
+    %6:fpr128 = LD1i32 %5, 2, killed %2
+    %7:fpr128 = LD1i32 %6, 3, killed %3
+    $q0 = COPY %7
+    RET_ReallyLR implicit $q0
+
+---
+name:            negative_pattern_multiple_users
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3, $x4
+
+    ; CHECK-LABEL: name: negative_pattern_multiple_users
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gpr64common = COPY $x4
+    ; CHECK-NEXT: [[LD_i32:%[0-9]+]]:fpr32 = LDRSroX [[COPY]], killed [[COPY1]], 0, 1
+    ; CHECK-NEXT: [[FIRST_REG:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LD_i32]], %subreg.ssub
+    ; CHECK-NEXT: [[LD_LANE_1:%[0-9]+]]:fpr128 = LD1i32 [[FIRST_REG]], 1, killed [[COPY2]]
+    ; CHECK-NEXT: [[LD_LANE_2:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_1]], 2, killed [[COPY3]]
+    ; CHECK-NEXT: [[LD_LANE_3:%[0-9]+]]:fpr128 = LD1i32 [[LD_LANE_2]], 3, killed [[COPY4]]
+    ; CHECK-NEXT: $q0 = COPY [[LD_LANE_3]]
+    ; CHECK-NEXT: $q1 = COPY [[LD_LANE_2]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0, implicit $q1
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %4:gpr64common = COPY $x4
+    %5:fpr32 = LDRSroX %0, killed %1, 0, 1
+    %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub
+    %7:fpr128 = LD1i32 %6, 1, killed %2
+    %8:fpr128 = LD1i32 %7, 2, killed %3
+    %9:fpr128 = LD1i32 %8, 3, killed %4
+    $q0 = COPY %9
+    $q1 = COPY %8
+    RET_ReallyLR implicit $q0, implicit $q1
+
+---
+name:            aliasing_store_between_vector_loads
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2, $x3
+
+    ; CHECK-LABEL: name: aliasing_store_between_vector_loads
+    ; CHECK: [[BASE_PTR:%[0-9]+]]:gpr64common = COPY $x0
+    ; CHECK-NEXT: [[OFFSET_PTR:%[0-9]+]]:gpr64common = COPY $x1
+    ; CHECK-NEXT: [[ALIAS_ADDR:%[0-9]+]]:gpr64common = COPY $x2
+    ; CHECK-NEXT: [[OTHER_ADDR:%[0-9]+]]:gpr64common = COPY $x3
+    ; CHECK-NEXT: [[LOAD0:%[0-9]+]]:fpr32 = LDRSroX [[BASE_PTR]], killed [[OFFSET_PTR]], 0, 1
+    ; CHECK-NEXT: [[VEC0:%[0-9]+]]:fpr128 = SUBREG_TO_REG 0, killed [[LOAD0]], %subreg.ssub
+    ; CHECK-NEXT: [[VEC1:%[0-9]+]]:fpr128 = LD1i32 [[VEC0]], 1, [[ALIAS_ADDR]]
+    ; CHECK-NEXT: [[CONST:%[0-9]+]]:gpr32 = MOVi32imm 99
+    ; CHECK-NEXT: STRWui [[CONST]], [[ALIAS_ADDR]], 0
+    ; CHECK-NEXT: [[VEC2:%[0-9]+]]:fpr128 = LD1i32 [[VEC1]], 2, killed [[ALIAS_ADDR]]
+    ; CHECK-NEXT: [[VEC3:%[0-9]+]]:fpr128 = LD1i32 [[VEC2]], 3, killed [[OTHER_ADDR]]
+    ; CHECK-NEXT: $q0 = COPY [[VEC3]]
+    ; CHECK-NEXT: RET_ReallyLR implicit $q0
+    %0:gpr64common = COPY $x0
+    %1:gpr64common = COPY $x1
+    %2:gpr64common = COPY $x2
+    %3:gpr64common = COPY $x3
+    %5:fpr32 = LDRSroX %0, killed %1, 0, 1
+    %6:fpr128 = SUBREG_TO_REG 0, killed %5, %subreg.ssub
+    %7:fpr128 = LD1i32 %6, 1, %2
+    %10:gpr32 = MOVi32imm 99
+    STRWui %10, %2, 0
+    %8:fpr128 = LD1i32 %7, 2, killed %2
+    %9:fpr128 = LD1i32 %8, 3, killed %3
+    $q0 = COPY %9
+    RET_ReallyLR implicit $q0
diff --git a/llvm/test/CodeGen/AArch64/aarch64-histcnt-dag-combine-hang.ll b/llvm/test/CodeGen/AArch64/aarch64-histcnt-dag-combine-hang.ll
new file mode 100644
index 0000000000000..da04c67aa6c5c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-histcnt-dag-combine-hang.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve2 -verify-machineinstrs < %s -o - | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; This test is reduced from a real world example that would cause the DAGCombiner to hang.
+
+define void @histcnt_loop(ptr %0, i64 %1, ptr %2, i64 %3, i64 %4) {
+; CHECK-LABEL: histcnt_loop:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov z0.d, #1 // =0x1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    add x9, x0, x1
+; CHECK-NEXT:  .LBB0_1: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ld1h { z1.d }, p0/z, [x0, x8, lsl #1]
+; CHECK-NEXT:    lsl x10, x8, #1
+; CHECK-NEXT:    add x11, x0, x10
+; CHECK-NEXT:    add x10, x9, x10
+; CHECK-NEXT:    lsl z1.d, z1.d, #1
+; CHECK-NEXT:    ld1h { z4.d }, p0/z, [x11, #1, mul vl]
+; CHECK-NEXT:    ld1h { z5.d }, p0/z, [x10, #1, mul vl]
+; CHECK-NEXT:    histcnt z2.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    ld1h { z3.d }, p0/z, [x2, z1.d]
+; CHECK-NEXT:    mad z2.d, p0/m, z0.d, z3.d
+; CHECK-NEXT:    ld1h { z3.d }, p0/z, [x9, x8, lsl #1]
+; CHECK-NEXT:    add x8, x8, x3
+; CHECK-NEXT:    cmp x4, x8
+; CHECK-NEXT:    st1h { z2.d }, p0, [x2, z1.d]
+; CHECK-NEXT:    lsl z1.d, z4.d, #1
+; CHECK-NEXT:    histcnt z2.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    ld1h { z4.d }, p0/z, [x2, z1.d]
+; CHECK-NEXT:    mad z2.d, p0/m, z0.d, z4.d
+; CHECK-NEXT:    st1h { z2.d }, p0, [x2, z1.d]
+; CHECK-NEXT:    lsl z1.d, z3.d, #1
+; CHECK-NEXT:    histcnt z2.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    ld1h { z3.d }, p0/z, [x2, z1.d]
+; CHECK-NEXT:    mad z2.d, p0/m, z0.d, z3.d
+; CHECK-NEXT:    st1h { z2.d }, p0, [x2, z1.d]
+; CHECK-NEXT:    lsl z1.d, z5.d, #1
+; CHECK-NEXT:    histcnt z2.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    ld1h { z3.d }, p0/z, [x2, z1.d]
+; CHECK-NEXT:    mad z2.d, p0/m, z0.d, z3.d
+; CHECK-NEXT:    st1h { z2.d }, p0, [x2, z1.d]
+; CHECK-NEXT:    b.ne .LBB0_1
+; CHECK-NEXT:  // %bb.2: // %exit
+; CHECK-NEXT:    ret
+entry:
+  br label %loop
+
+loop:
+  %6 = phi i64 [ 0, %entry ], [ %15, %loop ]
+  %7 = getelementptr inbounds nuw i16, ptr %0, i64 %6
+  %8 = getelementptr inbounds nuw i8, ptr %7, i64 %1
+  %9 = load <vscale x 4 x i16>, ptr %7, align 2
+  %10 = load <vscale x 4 x i16>, ptr %8, align 2
+  %11 = zext <vscale x 4 x i16> %9 to <vscale x 4 x i64>
+  %12 = zext <vscale x 4 x i16> %10 to <vscale x 4 x i64>
+  %13 = getelementptr inbounds nuw [16 x i16], ptr %2, i64 0, <vscale x 4 x i64> %11
+  %14 = getelementptr inbounds nuw [16 x i16], ptr %2, i64 0, <vscale x 4 x i64> %12
+  call void @llvm.experimental.vector.histogram.add.nxv4p0.i16(<vscale x 4 x ptr> %13, i16 1, <vscale x 4 x i1> splat (i1 true))
+  call void @llvm.experimental.vector.histogram.add.nxv4p0.i16(<vscale x 4 x ptr> %14, i16 1, <vscale x 4 x i1> splat (i1 true))
+  %15 = add nuw i64 %6, %3
+  %16 = icmp eq i64 %15, %4
+  br i1 %16, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-ldst-no-premature-sp-pop.mir b/llvm/test/CodeGen/AArch64/aarch64-ldst-no-premature-sp-pop.mir
index ba621cf77f9ae..ae1addcc620f5 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-ldst-no-premature-sp-pop.mir
+++ b/llvm/test/CodeGen/AArch64/aarch64-ldst-no-premature-sp-pop.mir
@@ -59,8 +59,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  16
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, type: default, offset: -16, size: 16,
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mov-debug-locs.mir b/llvm/test/CodeGen/AArch64/aarch64-mov-debug-locs.mir
index 16e2de751381a..45fa2be5b59de 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-mov-debug-locs.mir
+++ b/llvm/test/CodeGen/AArch64/aarch64-mov-debug-locs.mir
@@ -157,8 +157,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           
   - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8, 
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sme-za-call-lowering.ll b/llvm/test/CodeGen/AArch64/aarch64-sme-za-call-lowering.ll
new file mode 100644
index 0000000000000..0b8645f66b5f3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-sme-za-call-lowering.ll
@@ -0,0 +1,129 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 -aarch64-streaming-hazard-size=0 -mattr=+sme,+sve -aarch64-new-sme-abi -stop-before=aarch64-machine-sme-abi -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-BEFORE-SMEABI
+; RUN: llc -mtriple=aarch64 -aarch64-streaming-hazard-size=0 -mattr=+sme,+sve -aarch64-new-sme-abi -stop-after=aarch64-machine-sme-abi -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-AFTER-SMEABI
+
+declare void @private_za_callee()
+declare void @shared_za_callee() "aarch64_inout_za"
+declare i64 @shared_za_callee_many_args(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) "aarch64_inout_za"
+
+; Tests a "RequiresZASavePseudo" is placed after the "ADJCALLSTACKDOWN" for private ZA callees.
+define void @requires_za_save() nounwind "aarch64_inout_za" {
+  ; CHECK-BEFORE-SMEABI-LABEL: name: requires_za_save
+  ; CHECK-BEFORE-SMEABI: bb.0 (%ir-block.0):
+  ; CHECK-BEFORE-SMEABI-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-BEFORE-SMEABI-NEXT:   RequiresZASavePseudo
+  ; CHECK-BEFORE-SMEABI-NEXT:   BL @private_za_callee, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+  ; CHECK-BEFORE-SMEABI-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-BEFORE-SMEABI-NEXT:   RET_ReallyLR
+  ;
+  ; CHECK-AFTER-SMEABI-LABEL: name: requires_za_save
+  ; CHECK-AFTER-SMEABI: bb.0 (%ir-block.0):
+  ; CHECK-AFTER-SMEABI-NEXT:   [[RDSVLI_XI:%[0-9]+]]:gpr64 = RDSVLI_XI 1, implicit $vg
+  ; CHECK-AFTER-SMEABI-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $sp
+  ; CHECK-AFTER-SMEABI-NEXT:   [[MSUBXrrr:%[0-9]+]]:gpr64 = MSUBXrrr [[RDSVLI_XI]], [[RDSVLI_XI]], [[COPY]]
+  ; CHECK-AFTER-SMEABI-NEXT:   $sp = COPY [[MSUBXrrr]]
+  ; CHECK-AFTER-SMEABI-NEXT:   STPXi [[MSUBXrrr]], [[RDSVLI_XI]], %stack.0, 0
+  ; CHECK-AFTER-SMEABI-NEXT:   [[ADDXri:%[0-9]+]]:gpr64sp = ADDXri %stack.0, 0, 0
+  ; CHECK-AFTER-SMEABI-NEXT:   [[COPY1:%[0-9]+]]:gpr64 = COPY [[ADDXri]]
+  ; CHECK-AFTER-SMEABI-NEXT:   MSR 56965, [[COPY1]]
+  ; CHECK-AFTER-SMEABI-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-AFTER-SMEABI-NEXT:   RequiresZASavePseudo
+  ; CHECK-AFTER-SMEABI-NEXT:   BL @private_za_callee, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+  ; CHECK-AFTER-SMEABI-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-AFTER-SMEABI-NEXT:   MSRpstatesvcrImm1 2, 1, implicit-def $nzcv
+  ; CHECK-AFTER-SMEABI-NEXT:   [[MRS:%[0-9]+]]:gpr64 = MRS 56965, implicit-def $nzcv
+  ; CHECK-AFTER-SMEABI-NEXT:   $x0 = ADDXri %stack.0, 0, 0
+  ; CHECK-AFTER-SMEABI-NEXT:   RestoreZAPseudo [[MRS]], $x0, &__arm_tpidr2_restore, csr_aarch64_sme_abi_support_routines_preservemost_from_x0
+  ; CHECK-AFTER-SMEABI-NEXT:   MSR 56965, $xzr
+  ; CHECK-AFTER-SMEABI-NEXT:   RET_ReallyLR
+  call void @private_za_callee()
+  ret void
+}
+
+; Tests ZA state markers like "RequiresZASavePseudo" are placed before any streaming mode changes.
+define void @requires_za_save_streaming_mode_change() nounwind "aarch64_inout_za" "aarch64_pstate_sm_enabled" {
+  ; CHECK-BEFORE-SMEABI-LABEL: name: requires_za_save_streaming_mode_change
+  ; CHECK-BEFORE-SMEABI: bb.0 (%ir-block.0):
+  ; CHECK-BEFORE-SMEABI-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-BEFORE-SMEABI-NEXT:   RequiresZASavePseudo
+  ; CHECK-BEFORE-SMEABI-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit $vg, implicit-def $vg, implicit-def $fpmr
+  ; CHECK-BEFORE-SMEABI-NEXT:   BL @private_za_callee, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+  ; CHECK-BEFORE-SMEABI-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-BEFORE-SMEABI-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
+  ; CHECK-BEFORE-SMEABI-NEXT:   RET_ReallyLR
+  ;
+  ; CHECK-AFTER-SMEABI-LABEL: name: requires_za_save_streaming_mode_change
+  ; CHECK-AFTER-SMEABI: bb.0 (%ir-block.0):
+  ; CHECK-AFTER-SMEABI-NEXT:   [[RDSVLI_XI:%[0-9]+]]:gpr64 = RDSVLI_XI 1, implicit $vg
+  ; CHECK-AFTER-SMEABI-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $sp
+  ; CHECK-AFTER-SMEABI-NEXT:   [[MSUBXrrr:%[0-9]+]]:gpr64 = MSUBXrrr [[RDSVLI_XI]], [[RDSVLI_XI]], [[COPY]]
+  ; CHECK-AFTER-SMEABI-NEXT:   $sp = COPY [[MSUBXrrr]]
+  ; CHECK-AFTER-SMEABI-NEXT:   STPXi [[MSUBXrrr]], [[RDSVLI_XI]], %stack.0, 0
+  ; CHECK-AFTER-SMEABI-NEXT:   [[ADDXri:%[0-9]+]]:gpr64sp = ADDXri %stack.0, 0, 0
+  ; CHECK-AFTER-SMEABI-NEXT:   [[COPY1:%[0-9]+]]:gpr64 = COPY [[ADDXri]]
+  ; CHECK-AFTER-SMEABI-NEXT:   MSR 56965, [[COPY1]]
+  ; CHECK-AFTER-SMEABI-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-AFTER-SMEABI-NEXT:   RequiresZASavePseudo
+  ; CHECK-AFTER-SMEABI-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit $vg, implicit-def $vg, implicit-def $fpmr
+  ; CHECK-AFTER-SMEABI-NEXT:   BL @private_za_callee, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+  ; CHECK-AFTER-SMEABI-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-AFTER-SMEABI-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
+  ; CHECK-AFTER-SMEABI-NEXT:   MSRpstatesvcrImm1 2, 1, implicit-def $nzcv
+  ; CHECK-AFTER-SMEABI-NEXT:   [[MRS:%[0-9]+]]:gpr64 = MRS 56965, implicit-def $nzcv
+  ; CHECK-AFTER-SMEABI-NEXT:   $x0 = ADDXri %stack.0, 0, 0
+  ; CHECK-AFTER-SMEABI-NEXT:   RestoreZAPseudo [[MRS]], $x0, &__arm_tpidr2_restore, csr_aarch64_sme_abi_support_routines_preservemost_from_x0
+  ; CHECK-AFTER-SMEABI-NEXT:   MSR 56965, $xzr
+  ; CHECK-AFTER-SMEABI-NEXT:   RET_ReallyLR
+  call void @private_za_callee()
+  ret void
+}
+
+; Tests "InOutZAUsePseudo" is placed after the "ADJCALLSTACKDOWN" for shared ZA callees.
+define void @inout_za_call() nounwind "aarch64_inout_za" {
+  ; CHECK-COMMON-LABEL: name: inout_za_call
+  ; CHECK-COMMON: bb.0 (%ir-block.0):
+  ; CHECK-COMMON-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COMMON-NEXT:   InOutZAUsePseudo
+  ; CHECK-COMMON-NEXT:   BL @shared_za_callee, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+  ; CHECK-COMMON-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COMMON-NEXT:   RET_ReallyLR
+  call void @shared_za_callee()
+  ret void
+}
+
+; Tests ZA state markers like "InOutZAUsePseudo" are placed before any stack argument setup.
+define void @many_args_inout_za_call(i64 %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9) "aarch64_inout_za" {
+  ; CHECK-COMMON-LABEL: name: many_args_inout_za_call
+  ; CHECK-COMMON: bb.0 (%ir-block.10):
+  ; CHECK-COMMON-NEXT:   liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7
+  ; CHECK-COMMON-NEXT: {{  $}}
+  ; CHECK-COMMON-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $x7
+  ; CHECK-COMMON-NEXT:   [[COPY1:%[0-9]+]]:gpr64 = COPY $x6
+  ; CHECK-COMMON-NEXT:   [[COPY2:%[0-9]+]]:gpr64 = COPY $x5
+  ; CHECK-COMMON-NEXT:   [[COPY3:%[0-9]+]]:gpr64 = COPY $x4
+  ; CHECK-COMMON-NEXT:   [[COPY4:%[0-9]+]]:gpr64 = COPY $x3
+  ; CHECK-COMMON-NEXT:   [[COPY5:%[0-9]+]]:gpr64 = COPY $x2
+  ; CHECK-COMMON-NEXT:   [[COPY6:%[0-9]+]]:gpr64 = COPY $x1
+  ; CHECK-COMMON-NEXT:   [[COPY7:%[0-9]+]]:gpr64 = COPY $x0
+  ; CHECK-COMMON-NEXT:   [[LDRXui:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.1, 0 :: (load (s64) from %fixed-stack.1, align 16)
+  ; CHECK-COMMON-NEXT:   [[LDRXui1:%[0-9]+]]:gpr64 = LDRXui %fixed-stack.0, 0 :: (load (s64) from %fixed-stack.0)
+  ; CHECK-COMMON-NEXT:   ADJCALLSTACKDOWN 16, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COMMON-NEXT:   InOutZAUsePseudo
+  ; CHECK-COMMON-NEXT:   [[COPY8:%[0-9]+]]:gpr64sp = COPY $sp
+  ; CHECK-COMMON-NEXT:   STRXui killed [[LDRXui1]], [[COPY8]], 1 :: (store (s64) into stack + 8)
+  ; CHECK-COMMON-NEXT:   STRXui killed [[LDRXui]], [[COPY8]], 0 :: (store (s64) into stack)
+  ; CHECK-COMMON-NEXT:   $x0 = COPY [[COPY7]]
+  ; CHECK-COMMON-NEXT:   $x1 = COPY [[COPY6]]
+  ; CHECK-COMMON-NEXT:   $x2 = COPY [[COPY5]]
+  ; CHECK-COMMON-NEXT:   $x3 = COPY [[COPY4]]
+  ; CHECK-COMMON-NEXT:   $x4 = COPY [[COPY3]]
+  ; CHECK-COMMON-NEXT:   $x5 = COPY [[COPY2]]
+  ; CHECK-COMMON-NEXT:   $x6 = COPY [[COPY1]]
+  ; CHECK-COMMON-NEXT:   $x7 = COPY [[COPY]]
+  ; CHECK-COMMON-NEXT:   BL @shared_za_callee_many_args, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $x3, implicit $x4, implicit $x5, implicit $x6, implicit $x7, implicit-def $sp, implicit-def $x0
+  ; CHECK-COMMON-NEXT:   ADJCALLSTACKUP 16, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-COMMON-NEXT:   [[COPY9:%[0-9]+]]:gpr64all = COPY $x0
+  ; CHECK-COMMON-NEXT:   RET_ReallyLR
+  %ret = call i64 @shared_za_callee_many_args(i64 %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-split-and-bitmask-immediate.ll b/llvm/test/CodeGen/AArch64/aarch64-split-logic-bitmask-immediate.ll
similarity index 71%
rename from llvm/test/CodeGen/AArch64/aarch64-split-and-bitmask-immediate.ll
rename to llvm/test/CodeGen/AArch64/aarch64-split-logic-bitmask-immediate.ll
index 113eb14ca4803..4db9db9185206 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-split-and-bitmask-immediate.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-split-logic-bitmask-immediate.ll
@@ -370,3 +370,175 @@ entry:
   %r = select i1 %c, i64 %a, i64 %ands
   ret i64 %r
 }
+
+; Test EOR.
+define i32 @test1_eor(i32 %a) {
+; CHECK-LABEL: test1_eor:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    eor w8, w0, #0x400
+; CHECK-NEXT:    eor w0, w8, #0x200000
+; CHECK-NEXT:    ret
+entry:
+  %eor = xor i32 %a, 2098176
+  ret i32 %eor
+}
+
+; This constant should not be split because it can be handled by one mov.
+define i32 @test2_eor(i32 %a) {
+; CHECK-LABEL: test2_eor:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #135 // =0x87
+; CHECK-NEXT:    eor w0, w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %eor = xor i32 %a, 135
+  ret i32 %eor
+}
+
+; This constant should not be split because the split immediate is not valid
+; bitmask immediate.
+define i32 @test3_eor(i32 %a) {
+; CHECK-LABEL: test3_eor:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #1024 // =0x400
+; CHECK-NEXT:    movk w8, #33, lsl #16
+; CHECK-NEXT:    eor w0, w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %eor = xor i32 %a, 2163712
+  ret i32 %eor
+}
+
+define i64 @test4_eor(i64 %a) {
+; CHECK-LABEL: test4_eor:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    eor x8, x0, #0x400
+; CHECK-NEXT:    eor x0, x8, #0x200000
+; CHECK-NEXT:    ret
+entry:
+  %eor = xor i64 %a, 2098176
+  ret i64 %eor
+}
+
+define i64 @test5_eor(i64 %a) {
+; CHECK-LABEL: test5_eor:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    eor x8, x0, #0x4000
+; CHECK-NEXT:    eor x0, x8, #0x200000000
+; CHECK-NEXT:    ret
+entry:
+  %eor = xor i64 %a, 8589950976
+  ret i64 %eor
+}
+
+; This constant should not be split because it can be handled by one mov.
+define i64 @test6_eor(i64 %a) {
+; CHECK-LABEL: test6_eor:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #135 // =0x87
+; CHECK-NEXT:    eor x0, x0, x8
+; CHECK-NEXT:    ret
+entry:
+  %eor = xor i64 %a, 135
+  ret i64 %eor
+}
+
+; This constant should not be split because the split immediate is not valid
+; bitmask immediate.
+define i64 @test7_eor(i64 %a) {
+; CHECK-LABEL: test7_eor:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #1024 // =0x400
+; CHECK-NEXT:    movk w8, #33, lsl #16
+; CHECK-NEXT:    eor x0, x0, x8
+; CHECK-NEXT:    ret
+entry:
+  %eor = xor i64 %a, 2163712
+  ret i64 %eor
+}
+
+; Test ORR.
+define i32 @test1_orr(i32 %a) {
+; CHECK-LABEL: test1_orr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    orr w8, w0, #0x400
+; CHECK-NEXT:    orr w0, w8, #0x200000
+; CHECK-NEXT:    ret
+entry:
+  %orr = or i32 %a, 2098176
+  ret i32 %orr
+}
+
+; This constant should not be split because it can be handled by one mov.
+define i32 @test2_orr(i32 %a) {
+; CHECK-LABEL: test2_orr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #135 // =0x87
+; CHECK-NEXT:    orr w0, w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %orr = or i32 %a, 135
+  ret i32 %orr
+}
+
+; This constant should not be split because the split immediate is not valid
+; bitmask immediate.
+define i32 @test3_orr(i32 %a) {
+; CHECK-LABEL: test3_orr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #1024 // =0x400
+; CHECK-NEXT:    movk w8, #33, lsl #16
+; CHECK-NEXT:    orr w0, w0, w8
+; CHECK-NEXT:    ret
+entry:
+  %orr = or i32 %a, 2163712
+  ret i32 %orr
+}
+
+define i64 @test4_orr(i64 %a) {
+; CHECK-LABEL: test4_orr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    orr x8, x0, #0x400
+; CHECK-NEXT:    orr x0, x8, #0x200000
+; CHECK-NEXT:    ret
+entry:
+  %orr = or i64 %a, 2098176
+  ret i64 %orr
+}
+
+define i64 @test5_orr(i64 %a) {
+; CHECK-LABEL: test5_orr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    orr x8, x0, #0x4000
+; CHECK-NEXT:    orr x0, x8, #0x200000000
+; CHECK-NEXT:    ret
+entry:
+  %orr = or i64 %a, 8589950976
+  ret i64 %orr
+}
+
+; This constant should not be split because it can be handled by one mov.
+define i64 @test6_orr(i64 %a) {
+; CHECK-LABEL: test6_orr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #135 // =0x87
+; CHECK-NEXT:    orr x0, x0, x8
+; CHECK-NEXT:    ret
+entry:
+  %orr = or i64 %a, 135
+  ret i64 %orr
+}
+
+; This constant should not be split because the split immediate is not valid
+; bitmask immediate.
+define i64 @test7_orr(i64 %a) {
+; CHECK-LABEL: test7_orr:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #1024 // =0x400
+; CHECK-NEXT:    movk w8, #33, lsl #16
+; CHECK-NEXT:    orr x0, x0, x8
+; CHECK-NEXT:    ret
+entry:
+  %orr = or i64 %a, 2163712
+  ret i64 %orr
+}
diff --git a/llvm/test/CodeGen/AArch64/aarch64st1.mir b/llvm/test/CodeGen/AArch64/aarch64st1.mir
index 22a024d37bc64..bc748f0a1910b 100644
--- a/llvm/test/CodeGen/AArch64/aarch64st1.mir
+++ b/llvm/test/CodeGen/AArch64/aarch64st1.mir
@@ -58,8 +58,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/AArch64/abd-combine.ll b/llvm/test/CodeGen/AArch64/abd-combine.ll
index d0257890d2c43..cdb40ceb46b1e 100644
--- a/llvm/test/CodeGen/AArch64/abd-combine.ll
+++ b/llvm/test/CodeGen/AArch64/abd-combine.ll
@@ -17,12 +17,9 @@ define <8 x i16> @abdu_base(<8 x i16> %src1, <8 x i16> %src2) {
 define <8 x i16> @abdu_const(<8 x i16> %src1) {
 ; CHECK-LABEL: abdu_const:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-NEXT:    uabd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uabd v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    movi v1.4h, #1
+; CHECK-NEXT:    mov v1.d[1], v1.d[0]
+; CHECK-NEXT:    uabd v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -34,12 +31,9 @@ define <8 x i16> @abdu_const(<8 x i16> %src1) {
 define <8 x i16> @abdu_const_lhs(<8 x i16> %src1) {
 ; CHECK-LABEL: abdu_const_lhs:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-NEXT:    uabd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uabd v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    movi v1.4h, #1
+; CHECK-NEXT:    mov v1.d[1], v1.d[0]
+; CHECK-NEXT:    uabd v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %zextsrc1
@@ -318,12 +312,9 @@ define <8 x i16> @abds_base(<8 x i16> %src1, <8 x i16> %src2) {
 define <8 x i16> @abds_const(<8 x i16> %src1) {
 ; CHECK-LABEL: abds_const:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    sabd v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    movi v1.4h, #1
+; CHECK-NEXT:    mov v1.d[1], v1.d[0]
+; CHECK-NEXT:    sabd v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -335,12 +326,9 @@ define <8 x i16> @abds_const(<8 x i16> %src1) {
 define <8 x i16> @abds_const_lhs(<8 x i16> %src1) {
 ; CHECK-LABEL: abds_const_lhs:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    sabd v1.4s, v2.4s, v1.4s
-; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    movi v1.4h, #1
+; CHECK-NEXT:    mov v1.d[1], v1.d[0]
+; CHECK-NEXT:    sabd v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %zextsrc1
@@ -352,11 +340,10 @@ define <8 x i16> @abds_const_lhs(<8 x i16> %src1) {
 define <8 x i16> @abds_const_zero(<8 x i16> %src1) {
 ; CHECK-LABEL: abds_const_zero:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-NEXT:    abs v0.4s, v0.4s
-; CHECK-NEXT:    abs v1.4s, v1.4s
-; CHECK-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    abs v0.4h, v0.4h
+; CHECK-NEXT:    abs v1.4h, v1.4h
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    ret
   %zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
   %sub = sub <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, %zextsrc1
diff --git a/llvm/test/CodeGen/AArch64/arm64-ccmp.ll b/llvm/test/CodeGen/AArch64/arm64-ccmp.ll
index a546ffd7143ad..4fe01e838771d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ccmp.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ccmp.ll
@@ -600,7 +600,7 @@ define i64 @select_noccmp1(i64 %v1, i64 %v2, i64 %v3, i64 %r) {
 ; CHECK-SD-LABEL: select_noccmp1:
 ; CHECK-SD:       ; %bb.0:
 ; CHECK-SD-NEXT:    cmp x0, #0
-; CHECK-SD-NEXT:    ccmp x0, #13, #4, lt
+; CHECK-SD-NEXT:    ccmp x0, #13, #4, mi
 ; CHECK-SD-NEXT:    cset w8, gt
 ; CHECK-SD-NEXT:    cmp x2, #2
 ; CHECK-SD-NEXT:    ccmp x2, #4, #4, lt
@@ -630,7 +630,7 @@ define i64 @select_noccmp2(i64 %v1, i64 %v2, i64 %v3, i64 %r) {
 ; CHECK-SD-LABEL: select_noccmp2:
 ; CHECK-SD:       ; %bb.0:
 ; CHECK-SD-NEXT:    cmp x0, #0
-; CHECK-SD-NEXT:    ccmp x0, #13, #0, ge
+; CHECK-SD-NEXT:    ccmp x0, #13, #0, pl
 ; CHECK-SD-NEXT:    cset w8, gt
 ; CHECK-SD-NEXT:    cmp w8, #0
 ; CHECK-SD-NEXT:    csel x0, xzr, x3, ne
@@ -664,7 +664,7 @@ define i32 @select_noccmp3(i32 %v0, i32 %v1, i32 %v2) {
 ; CHECK-SD-LABEL: select_noccmp3:
 ; CHECK-SD:       ; %bb.0:
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    ccmp w0, #13, #0, ge
+; CHECK-SD-NEXT:    ccmp w0, #13, #0, pl
 ; CHECK-SD-NEXT:    cset w8, gt
 ; CHECK-SD-NEXT:    cmp w0, #22
 ; CHECK-SD-NEXT:    mov w9, #44 ; =0x2c
@@ -937,11 +937,11 @@ define i32 @f128_select_and_olt_oge(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3,
 ; CHECK-SD-NEXT:    stp q2, q3, [sp] ; 32-byte Folded Spill
 ; CHECK-SD-NEXT:    bl ___lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    cset w21, lt
+; CHECK-SD-NEXT:    cset w21, mi
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] ; 32-byte Folded Reload
 ; CHECK-SD-NEXT:    bl ___getf2
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    cset w8, ge
+; CHECK-SD-NEXT:    cset w8, pl
 ; CHECK-SD-NEXT:    tst w8, w21
 ; CHECK-SD-NEXT:    csel w0, w20, w19, ne
 ; CHECK-SD-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
@@ -964,8 +964,8 @@ define i32 @f128_select_and_olt_oge(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3,
 ; CHECK-GI-NEXT:    ldp q1, q0, [sp] ; 32-byte Folded Reload
 ; CHECK-GI-NEXT:    bl ___getf2
 ; CHECK-GI-NEXT:    cmp w21, #0
-; CHECK-GI-NEXT:    ccmp w0, #0, #8, lt
-; CHECK-GI-NEXT:    csel w0, w19, w20, ge
+; CHECK-GI-NEXT:    ccmp w0, #0, #8, mi
+; CHECK-GI-NEXT:    csel w0, w19, w20, pl
 ; CHECK-GI-NEXT:    ldp x29, x30, [sp, #64] ; 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] ; 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldp x22, x21, [sp, #32] ; 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/arm64-copy-phys-zero-reg.mir b/llvm/test/CodeGen/AArch64/arm64-copy-phys-zero-reg.mir
new file mode 100644
index 0000000000000..76b5b76130657
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-copy-phys-zero-reg.mir
@@ -0,0 +1,109 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -o - -mtriple=arm64-apple-ios -run-pass=postrapseudos -simplify-mir -verify-machineinstrs -mattr="-zcm-gpr32,-zcm-gpr64,-zcz" %s \
+# RUN:    | FileCheck --check-prefix=CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ %s
+# RUN: llc -o - -mtriple=arm64-apple-ios -run-pass=postrapseudos -simplify-mir -verify-machineinstrs -mattr="+zcm-gpr32,-zcm-gpr64,-zcz" %s \
+# RUN:    | FileCheck --check-prefix=CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ %s
+# RUN: llc -o - -mtriple=arm64-apple-ios -run-pass=postrapseudos -simplify-mir -verify-machineinstrs -mattr="-zcm-gpr32,+zcm-gpr64,-zcz" %s \
+# RUN:    | FileCheck --check-prefix=CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ %s
+# RUN: llc -o - -mtriple=arm64-apple-ios -run-pass=postrapseudos -simplify-mir -verify-machineinstrs -mattr="+zcm-gpr32,+zcm-gpr64,-zcz" %s \
+# RUN:    | FileCheck --check-prefix=CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ %s
+# RUN: llc -o - -mtriple=arm64-apple-ios -run-pass=postrapseudos -simplify-mir -verify-machineinstrs -mattr="-zcm-gpr32,-zcm-gpr64,+zcz" %s \
+# RUN:    | FileCheck --check-prefix=CHECK-NO-ZCM-ZCZ %s
+# RUN: llc -o - -mtriple=arm64-apple-ios -run-pass=postrapseudos -simplify-mir -verify-machineinstrs -mattr="+zcm-gpr32,+zcm-gpr64,+zcz" %s \
+# RUN:    | FileCheck --check-prefix=CHECK-ZCM-ZCZ %s
+
+--- |
+  define void @f0(i64 noundef %x) { ret void }
+  define void @f1(i64 noundef %x) { ret void }
+  define void @f2(i32 noundef) { ret void }
+...
+---
+name:            f0
+liveins:
+  - { reg: '$x0' }
+body:             |
+  bb.0:
+    liveins: $x0, $lr
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-LABEL: name: f0
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ: liveins: $x0, $lr
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: {{  $}}
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: $w0 = ORRWrr $wzr, $wzr
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-LABEL: name: f0
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ: liveins: $x0, $lr
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: {{  $}}
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: $w0 = ORRWrr $wzr, $wzr
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-LABEL: name: f0
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ: liveins: $x0, $lr
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: {{  $}}
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: $x0 = ORRXrr $xzr, undef $xzr, implicit $wzr
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-LABEL: name: f0
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ: liveins: $x0, $lr
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: {{  $}}
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: $w0 = ORRWrr $wzr, $wzr
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-NO-ZCM-ZCZ-LABEL: name: f0
+    ; CHECK-NO-ZCM-ZCZ: liveins: $x0, $lr
+    ; CHECK-NO-ZCM-ZCZ-NEXT: {{  $}}
+    ; CHECK-NO-ZCM-ZCZ-NEXT: $w0 = MOVZWi 0, 0
+    ; CHECK-NO-ZCM-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-ZCM-ZCZ-LABEL: name: f0
+    ; CHECK-ZCM-ZCZ: liveins: $x0, $lr
+    ; CHECK-ZCM-ZCZ-NEXT: {{  $}}
+    ; CHECK-ZCM-ZCZ-NEXT: $w0 = MOVZWi 0, 0
+    ; CHECK-ZCM-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    $w0 = COPY $wzr
+    BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+...
+---
+name:            f1
+liveins:
+  - { reg: '$x0' }
+body:             |
+  bb.0:
+    liveins: $x0, $lr
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-LABEL: name: f1
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ: liveins: $x0, $lr
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: {{  $}}
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: $x0 = ORRXrr $xzr, $xzr
+    ; CHECK-NO-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-LABEL: name: f1
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ: liveins: $x0, $lr
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: {{  $}}
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: $x0 = ORRXrr $xzr, $xzr
+    ; CHECK-ZCM-GPR32-NO-ZCM-GPR64-NO-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-LABEL: name: f1
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ: liveins: $x0, $lr
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: {{  $}}
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: $x0 = ORRXrr $xzr, $xzr
+    ; CHECK-NO-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-LABEL: name: f1
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ: liveins: $x0, $lr
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: {{  $}}
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: $x0 = ORRXrr $xzr, $xzr
+    ; CHECK-ZCM-GPR32-ZCM-GPR64-NO-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-NO-ZCM-ZCZ-LABEL: name: f1
+    ; CHECK-NO-ZCM-ZCZ: liveins: $x0, $lr
+    ; CHECK-NO-ZCM-ZCZ-NEXT: {{  $}}
+    ; CHECK-NO-ZCM-ZCZ-NEXT: $x0 = MOVZXi 0, 0
+    ; CHECK-NO-ZCM-ZCZ-NEXT: BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ;
+    ; CHECK-ZCM-ZCZ-LABEL: name: f1
+    ; CHECK-ZCM-ZCZ: liveins: $x0, $lr
+    ; CHECK-ZCM-ZCZ-NEXT: {{  $}}
+    ; CHECK-ZCM-ZCZ-NEXT: $x0 = MOVZXi 0, 0
+    ; CHECK-ZCM-ZCZ-NEXT:BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    $x0 = COPY $xzr
+    BL @f2, csr_darwin_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+...
diff --git a/llvm/test/CodeGen/AArch64/arm64-ext.ll b/llvm/test/CodeGen/AArch64/arm64-ext.ll
index 8bf2b826d7101..c3670579c9148 100644
--- a/llvm/test/CodeGen/AArch64/arm64-ext.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-ext.ll
@@ -139,9 +139,8 @@ define <2 x ptr> @test_v2p0(<2 x ptr> %a, <2 x ptr> %b) {
 define <16 x i8> @reverse_vector_s8x16b(<16 x i8> noundef %x) {
 ; CHECK-SD-LABEL: reverse_vector_s8x16b:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    rev64 v1.16b, v0.16b
-; CHECK-SD-NEXT:    ext v0.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    rev64 v0.16b, v0.16b
+; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: reverse_vector_s8x16b:
@@ -161,9 +160,8 @@ entry:
 define <8 x i16> @reverse_vector_s16x8b(<8 x i16> noundef %x) {
 ; CHECK-SD-LABEL: reverse_vector_s16x8b:
 ; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    rev64 v1.8h, v0.8h
-; CHECK-SD-NEXT:    ext v0.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    rev64 v0.8h, v0.8h
+; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: reverse_vector_s16x8b:
diff --git a/llvm/test/CodeGen/AArch64/arm64-fmax.ll b/llvm/test/CodeGen/AArch64/arm64-fmax.ll
index d7d54a6e48a92..85104775339b6 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fmax.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fmax.ll
@@ -60,7 +60,7 @@ define i64 @test_integer(i64  %in) {
 ; CHECK-LABEL: test_integer:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    csel x0, xzr, x0, lt
+; CHECK-NEXT:    csel x0, xzr, x0, mi
 ; CHECK-NEXT:    ret
   %cmp = icmp slt i64 %in, 0
   %val = select i1 %cmp, i64 0, i64 %in
diff --git a/llvm/test/CodeGen/AArch64/arm64-fp128.ll b/llvm/test/CodeGen/AArch64/arm64-fp128.ll
index a75f6419d5a5a..3e4b887fed55d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-fp128.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-fp128.ll
@@ -258,7 +258,7 @@ define i32 @test_br_cc(fp128 %lhs, fp128 %rhs) {
 ; CHECK-SD-NEXT:    mov w8, #29 // =0x1d
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    mov w9, #42 // =0x2a
-; CHECK-SD-NEXT:    csel w0, w9, w8, lt
+; CHECK-SD-NEXT:    csel w0, w9, w8, mi
 ; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-SD-NEXT:    ret
 ;
@@ -271,7 +271,7 @@ define i32 @test_br_cc(fp128 %lhs, fp128 %rhs) {
 ; CHECK-GI-NEXT:    mov w8, #29 // =0x1d
 ; CHECK-GI-NEXT:    mov w9, #42 // =0x2a
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w9, w8, lt
+; CHECK-GI-NEXT:    csel w0, w9, w8, mi
 ; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
   %cond = fcmp olt fp128 %lhs, %rhs
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-3vdiff.ll b/llvm/test/CodeGen/AArch64/arm64-neon-3vdiff.ll
index 256ff94830113..9bafc5b8aea62 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-3vdiff.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-3vdiff.ll
@@ -34,76 +34,38 @@ declare <2 x i32> @llvm.aarch64.neon.raddhn.v2i32(<2 x i64>, <2 x i64>)
 declare <4 x i16> @llvm.aarch64.neon.raddhn.v4i16(<4 x i32>, <4 x i32>)
 declare <8 x i8> @llvm.aarch64.neon.raddhn.v8i8(<8 x i16>, <8 x i16>)
 
-define <8 x i16> @test_vaddl_s8(<8 x i8> %a, <8 x i8> %b) {
-; CHECK-LABEL: test_vaddl_s8:
+define void @test_commutable_vaddl_s8(<8 x i8> %a, <8 x i8> %b, ptr %c) {
+; CHECK-LABEL: test_commutable_vaddl_s8:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    saddl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
 entry:
   %vmovl.i.i = sext <8 x i8> %a to <8 x i16>
   %vmovl.i2.i = sext <8 x i8> %b to <8 x i16>
   %add.i = add <8 x i16> %vmovl.i.i, %vmovl.i2.i
-  ret <8 x i16> %add.i
-}
-
-define <4 x i32> @test_vaddl_s16(<4 x i16> %a, <4 x i16> %b) {
-; CHECK-LABEL: test_vaddl_s16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    saddl v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    ret
-entry:
-  %vmovl.i.i = sext <4 x i16> %a to <4 x i32>
-  %vmovl.i2.i = sext <4 x i16> %b to <4 x i32>
-  %add.i = add <4 x i32> %vmovl.i.i, %vmovl.i2.i
-  ret <4 x i32> %add.i
+  store <8 x i16> %add.i, ptr %c
+  %add.i2 = add <8 x i16> %vmovl.i2.i, %vmovl.i.i
+  %c.gep.1 = getelementptr i8, ptr %c, i64 16
+  store <8 x i16> %add.i2, ptr %c.gep.1
+  ret void
 }
 
-define <2 x i64> @test_vaddl_s32(<2 x i32> %a, <2 x i32> %b) {
-; CHECK-LABEL: test_vaddl_s32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    saddl v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    ret
-entry:
-  %vmovl.i.i = sext <2 x i32> %a to <2 x i64>
-  %vmovl.i2.i = sext <2 x i32> %b to <2 x i64>
-  %add.i = add <2 x i64> %vmovl.i.i, %vmovl.i2.i
-  ret <2 x i64> %add.i
-}
-
-define <8 x i16> @test_vaddl_u8(<8 x i8> %a, <8 x i8> %b) {
-; CHECK-LABEL: test_vaddl_u8:
+define void @test_commutable_vaddl_u8(<8 x i8> %a, <8 x i8> %b, ptr %c) {
+; CHECK-LABEL: test_commutable_vaddl_u8:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    stp q0, q0, [x0]
 ; CHECK-NEXT:    ret
 entry:
   %vmovl.i.i = zext <8 x i8> %a to <8 x i16>
   %vmovl.i2.i = zext <8 x i8> %b to <8 x i16>
   %add.i = add <8 x i16> %vmovl.i.i, %vmovl.i2.i
-  ret <8 x i16> %add.i
-}
-
-define <4 x i32> @test_vaddl_u16(<4 x i16> %a, <4 x i16> %b) {
-; CHECK-LABEL: test_vaddl_u16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uaddl v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    ret
-entry:
-  %vmovl.i.i = zext <4 x i16> %a to <4 x i32>
-  %vmovl.i2.i = zext <4 x i16> %b to <4 x i32>
-  %add.i = add <4 x i32> %vmovl.i.i, %vmovl.i2.i
-  ret <4 x i32> %add.i
-}
-
-define <2 x i64> @test_vaddl_u32(<2 x i32> %a, <2 x i32> %b) {
-; CHECK-LABEL: test_vaddl_u32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    uaddl v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    ret
-entry:
-  %vmovl.i.i = zext <2 x i32> %a to <2 x i64>
-  %vmovl.i2.i = zext <2 x i32> %b to <2 x i64>
-  %add.i = add <2 x i64> %vmovl.i.i, %vmovl.i2.i
-  ret <2 x i64> %add.i
+  store <8 x i16> %add.i, ptr %c
+  %add.i2 = add <8 x i16> %vmovl.i2.i, %vmovl.i.i
+  %c.gep.1 = getelementptr i8, ptr %c, i64 16
+  store <8 x i16> %add.i2, ptr %c.gep.1
+  ret void
 }
 
 define <8 x i16> @test_vaddl_a8(<8 x i8> %a, <8 x i8> %b) {
@@ -2892,9 +2854,9 @@ define <8 x i16> @cmplx_mul_combined_re_im(<8 x i16> noundef %a, i64 %scale.coer
 ; CHECK-GI-LABEL: cmplx_mul_combined_re_im:
 ; CHECK-GI:       // %bb.0: // %entry
 ; CHECK-GI-NEXT:    lsr x9, x0, #16
-; CHECK-GI-NEXT:    adrp x8, .LCPI196_0
+; CHECK-GI-NEXT:    adrp x8, .LCPI192_0
 ; CHECK-GI-NEXT:    rev32 v4.8h, v0.8h
-; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI196_0]
+; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI192_0]
 ; CHECK-GI-NEXT:    fmov d1, x9
 ; CHECK-GI-NEXT:    dup v2.8h, v1.h[0]
 ; CHECK-GI-NEXT:    sqneg v1.8h, v2.8h
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-aba-abd.ll b/llvm/test/CodeGen/AArch64/arm64-neon-aba-abd.ll
index 6c7ddd916abdf..ccd1917ae3d85 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-aba-abd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-aba-abd.ll
@@ -575,3 +575,69 @@ define <4 x i32> @knownbits_sabd_and_mul_mask(<4 x i32> %a0, <4 x i32> %a1) {
   %6 = shufflevector <4 x i32> %5, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 3, i32 3>
   ret <4 x i32> %6
 }
+
+define <4 x i16> @trunc_abdu_foldable(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-SD-LABEL: trunc_abdu_foldable:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    uabd v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: trunc_abdu_foldable:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    uabd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+  %ext_a = zext <4 x i16> %a to <4 x i32>
+  %ext_b = zext <4 x i16> %b to <4 x i32>
+  %abd = call <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32> %ext_a, <4 x i32> %ext_b)
+  %trunc = trunc <4 x i32> %abd to <4 x i16>
+  ret <4 x i16> %trunc
+}
+
+define <4 x i16> @trunc_abds_foldable(<4 x i16> %a, <4 x i16> %b) {
+; CHECK-SD-LABEL: trunc_abds_foldable:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sabd v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: trunc_abds_foldable:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    ret
+  %a32 = sext <4 x i16> %a to <4 x i32>
+  %b32 = sext <4 x i16> %b to <4 x i32>
+  %abd32 = call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %a32, <4 x i32> %b32)
+  %res16 = trunc <4 x i32> %abd32 to <4 x i16>
+  ret <4 x i16> %res16
+}
+
+define <4 x i16> @trunc_abdu_not_foldable(<4 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: trunc_abdu_not_foldable:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    uabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %ext_a = zext <4 x i16> %a to <4 x i32>
+  %abd = call <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32> %ext_a, <4 x i32> %b)
+  %trunc = trunc <4 x i32> %abd to <4 x i16>
+  ret <4 x i16> %trunc
+}
+
+define <4 x i16> @truncate_abds_testcase1(<4 x i16> %a, <4 x i32> %b) {
+; CHECK-LABEL: truncate_abds_testcase1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
+  %a32 = sext <4 x i16> %a to <4 x i32>
+  %abd32 = call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %a32, <4 x i32> %b)
+  %res16 = trunc <4 x i32> %abd32 to <4 x i16>
+  ret <4 x i16> %res16
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll b/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll
index ecf3f69825c0e..0d427c05e3b77 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll
@@ -1608,6 +1608,18 @@ define <16 x i8> @poly_mulv16i8(<16 x i8> %lhs, <16 x i8> %rhs) {
    ret <16 x i8> %prod
 }
 
+define <16 x i8> @commutable_poly_mul(<16 x i8> %lhs, <16 x i8> %rhs) {
+; CHECK-LABEL: commutable_poly_mul:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pmul v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    add v0.16b, v0.16b, v0.16b
+; CHECK-NEXT:    ret
+  %1 = call <16 x i8> @llvm.aarch64.neon.pmul.v16i8(<16 x i8> %lhs, <16 x i8> %rhs)
+  %2 = call <16 x i8> @llvm.aarch64.neon.pmul.v16i8(<16 x i8> %rhs, <16 x i8> %lhs)
+  %3 = add <16 x i8> %1, %2
+  ret <16 x i8> %3
+}
+
 declare <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16>, <4 x i16>)
 declare <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16>, <8 x i16>)
 declare <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32>, <2 x i32>)
diff --git a/llvm/test/CodeGen/AArch64/arm64-trap.ll b/llvm/test/CodeGen/AArch64/arm64-trap.ll
index ffef22782550d..21d4e2d554810 100644
--- a/llvm/test/CodeGen/AArch64/arm64-trap.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-trap.ll
@@ -1,5 +1,6 @@
 ; RUN: llc < %s -mtriple=arm64-eabi | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-eabi -global-isel | FileCheck %s
+
 define void @foo() nounwind {
 ; CHECK-LABEL: foo
 ; CHECK: brk #0x1
diff --git a/llvm/test/CodeGen/AArch64/arm64-vabs.ll b/llvm/test/CodeGen/AArch64/arm64-vabs.ll
index 78881c80ccc10..c408d7fe42000 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vabs.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vabs.ll
@@ -44,6 +44,35 @@ define <2 x i64> @sabdl2d(ptr %A, ptr %B) nounwind {
   ret <2 x i64> %tmp4
 }
 
+define void @commutable_sabdl(ptr %A, ptr %B, ptr %C) nounwind {
+; CHECK-SD-LABEL: commutable_sabdl:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    ldr d1, [x1]
+; CHECK-SD-NEXT:    sabdl.8h v0, v1, v0
+; CHECK-SD-NEXT:    str q0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: commutable_sabdl:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr d0, [x0]
+; CHECK-GI-NEXT:    ldr d1, [x1]
+; CHECK-GI-NEXT:    sabdl.8h v0, v0, v1
+; CHECK-GI-NEXT:    str q0, [x2]
+; CHECK-GI-NEXT:    str q0, [x2]
+; CHECK-GI-NEXT:    ret
+  %tmp1 = load <8 x i8>, ptr %A
+  %tmp2 = load <8 x i8>, ptr %B
+  %tmp3 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+  %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
+  store <8 x i16> %tmp4, ptr %C
+  %tmp5 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp2, <8 x i8> %tmp1)
+  %tmp6 = zext <8 x i8> %tmp5 to <8 x i16>
+  %tmp7 = getelementptr i8, ptr %C, i64 16
+  store <8 x i16> %tmp6, ptr %C
+  ret void
+}
+
 define <8 x i16> @sabdl2_8h(ptr %A, ptr %B) nounwind {
 ; CHECK-SD-LABEL: sabdl2_8h:
 ; CHECK-SD:       // %bb.0:
@@ -155,6 +184,35 @@ define <2 x i64> @uabdl2d(ptr %A, ptr %B) nounwind {
   ret <2 x i64> %tmp4
 }
 
+define void @commutable_uabdl(ptr %A, ptr %B, ptr %C) nounwind {
+; CHECK-SD-LABEL: commutable_uabdl:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    ldr d1, [x1]
+; CHECK-SD-NEXT:    uabdl.8h v0, v1, v0
+; CHECK-SD-NEXT:    str q0, [x2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: commutable_uabdl:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    ldr d0, [x0]
+; CHECK-GI-NEXT:    ldr d1, [x1]
+; CHECK-GI-NEXT:    uabdl.8h v0, v0, v1
+; CHECK-GI-NEXT:    str q0, [x2]
+; CHECK-GI-NEXT:    str q0, [x2]
+; CHECK-GI-NEXT:    ret
+  %tmp1 = load <8 x i8>, ptr %A
+  %tmp2 = load <8 x i8>, ptr %B
+  %tmp3 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
+  %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
+  store <8 x i16> %tmp4, ptr %C
+  %tmp5 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp2, <8 x i8> %tmp1)
+  %tmp6 = zext <8 x i8> %tmp5 to <8 x i16>
+  %tmp7 = getelementptr i8, ptr %C, i64 16
+  store <8 x i16> %tmp6, ptr %C
+  ret void
+}
+
 define <8 x i16> @uabdl2_8h(ptr %A, ptr %B) nounwind {
 ; CHECK-SD-LABEL: uabdl2_8h:
 ; CHECK-SD:       // %bb.0:
@@ -1830,10 +1888,10 @@ define <2 x i128> @uabd_i64(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-GI-NEXT:    subs x10, x11, x13
 ; CHECK-GI-NEXT:    sbc x11, x14, x15
 ; CHECK-GI-NEXT:    cmp x9, #0
-; CHECK-GI-NEXT:    cset w12, lt
+; CHECK-GI-NEXT:    cset w12, mi
 ; CHECK-GI-NEXT:    csel w12, wzr, w12, eq
 ; CHECK-GI-NEXT:    cmp x11, #0
-; CHECK-GI-NEXT:    cset w13, lt
+; CHECK-GI-NEXT:    cset w13, mi
 ; CHECK-GI-NEXT:    csel w13, wzr, w13, eq
 ; CHECK-GI-NEXT:    negs x14, x8
 ; CHECK-GI-NEXT:    ngc x15, x9
diff --git a/llvm/test/CodeGen/AArch64/arm64-vadd.ll b/llvm/test/CodeGen/AArch64/arm64-vadd.ll
index d982dbbb1f69b..f9263185f308e 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vadd.ll
@@ -170,51 +170,6 @@ declare <2 x i32> @llvm.aarch64.neon.raddhn.v2i32(<2 x i64>, <2 x i64>) nounwind
 declare <4 x i16> @llvm.aarch64.neon.raddhn.v4i16(<4 x i32>, <4 x i32>) nounwind readnone
 declare <8 x i8> @llvm.aarch64.neon.raddhn.v8i8(<8 x i16>, <8 x i16>) nounwind readnone
 
-define <8 x i16> @saddl8h(ptr %A, ptr %B) nounwind {
-; CHECK-LABEL: saddl8h:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    saddl v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    ret
-        %tmp1 = load <8 x i8>, ptr %A
-        %tmp2 = load <8 x i8>, ptr %B
-  %tmp3 = sext <8 x i8> %tmp1 to <8 x i16>
-  %tmp4 = sext <8 x i8> %tmp2 to <8 x i16>
-  %tmp5 = add <8 x i16> %tmp3, %tmp4
-        ret <8 x i16> %tmp5
-}
-
-define <4 x i32> @saddl4s(ptr %A, ptr %B) nounwind {
-; CHECK-LABEL: saddl4s:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    saddl v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    ret
-        %tmp1 = load <4 x i16>, ptr %A
-        %tmp2 = load <4 x i16>, ptr %B
-  %tmp3 = sext <4 x i16> %tmp1 to <4 x i32>
-  %tmp4 = sext <4 x i16> %tmp2 to <4 x i32>
-  %tmp5 = add <4 x i32> %tmp3, %tmp4
-        ret <4 x i32> %tmp5
-}
-
-define <2 x i64> @saddl2d(ptr %A, ptr %B) nounwind {
-; CHECK-LABEL: saddl2d:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    saddl v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    ret
-        %tmp1 = load <2 x i32>, ptr %A
-        %tmp2 = load <2 x i32>, ptr %B
-  %tmp3 = sext <2 x i32> %tmp1 to <2 x i64>
-  %tmp4 = sext <2 x i32> %tmp2 to <2 x i64>
-  %tmp5 = add <2 x i64> %tmp3, %tmp4
-        ret <2 x i64> %tmp5
-}
-
 define <8 x i16> @saddl2_8h(<16 x i8> %a, <16 x i8> %b) nounwind  {
 ; CHECK-LABEL: saddl2_8h:
 ; CHECK:       // %bb.0:
@@ -266,52 +221,6 @@ define <2 x i64> @saddl2_2d(<4 x i32> %a, <4 x i32> %b) nounwind  {
   ret <2 x i64> %add.i
 }
 
-define <8 x i16> @uaddl8h(ptr %A, ptr %B) nounwind {
-; CHECK-LABEL: uaddl8h:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
-; CHECK-NEXT:    ret
-  %tmp1 = load <8 x i8>, ptr %A
-  %tmp2 = load <8 x i8>, ptr %B
-  %tmp3 = zext <8 x i8> %tmp1 to <8 x i16>
-  %tmp4 = zext <8 x i8> %tmp2 to <8 x i16>
-  %tmp5 = add <8 x i16> %tmp3, %tmp4
-  ret <8 x i16> %tmp5
-}
-
-define <4 x i32> @uaddl4s(ptr %A, ptr %B) nounwind {
-; CHECK-LABEL: uaddl4s:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    uaddl v0.4s, v0.4h, v1.4h
-; CHECK-NEXT:    ret
-  %tmp1 = load <4 x i16>, ptr %A
-  %tmp2 = load <4 x i16>, ptr %B
-  %tmp3 = zext <4 x i16> %tmp1 to <4 x i32>
-  %tmp4 = zext <4 x i16> %tmp2 to <4 x i32>
-  %tmp5 = add <4 x i32> %tmp3, %tmp4
-  ret <4 x i32> %tmp5
-}
-
-define <2 x i64> @uaddl2d(ptr %A, ptr %B) nounwind {
-; CHECK-LABEL: uaddl2d:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr d0, [x0]
-; CHECK-NEXT:    ldr d1, [x1]
-; CHECK-NEXT:    uaddl v0.2d, v0.2s, v1.2s
-; CHECK-NEXT:    ret
-  %tmp1 = load <2 x i32>, ptr %A
-  %tmp2 = load <2 x i32>, ptr %B
-  %tmp3 = zext <2 x i32> %tmp1 to <2 x i64>
-  %tmp4 = zext <2 x i32> %tmp2 to <2 x i64>
-  %tmp5 = add <2 x i64> %tmp3, %tmp4
-  ret <2 x i64> %tmp5
-}
-
-
 define <8 x i16> @uaddl2_8h(<16 x i8> %a, <16 x i8> %b) nounwind  {
 ; CHECK-LABEL: uaddl2_8h:
 ; CHECK:       // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/arm64-vmovn.ll b/llvm/test/CodeGen/AArch64/arm64-vmovn.ll
index 8e8642f90f133..3bfb40d888b97 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vmovn.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vmovn.ll
@@ -1,114 +1,122 @@
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel=0 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel=1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define <8 x i8> @xtn8b(<8 x i16> %A) nounwind {
-;CHECK-LABEL: xtn8b:
-;CHECK-NOT: ld1
-;CHECK: xtn.8b v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: xtn8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    xtn.8b v0, v0
+; CHECK-NEXT:    ret
   %tmp3 = trunc <8 x i16> %A to <8 x i8>
         ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @xtn4h(<4 x i32> %A) nounwind {
-;CHECK-LABEL: xtn4h:
-;CHECK-NOT: ld1
-;CHECK: xtn.4h v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: xtn4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    xtn.4h v0, v0
+; CHECK-NEXT:    ret
   %tmp3 = trunc <4 x i32> %A to <4 x i16>
         ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @xtn2s(<2 x i64> %A) nounwind {
-;CHECK-LABEL: xtn2s:
-;CHECK-NOT: ld1
-;CHECK: xtn.2s v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: xtn2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    xtn.2s v0, v0
+; CHECK-NEXT:    ret
   %tmp3 = trunc <2 x i64> %A to <2 x i32>
         ret <2 x i32> %tmp3
 }
 
 define <16 x i8> @xtn2_16b(<8 x i8> %ret, <8 x i16> %A) nounwind {
-;CHECK-LABEL: xtn2_16b:
-;CHECK-NOT: ld1
-;CHECK: xtn2.16b v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: xtn2_16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    xtn2.16b v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = trunc <8 x i16> %A to <8 x i8>
         %res = shufflevector <8 x i8> %ret, <8 x i8> %tmp3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
         ret <16 x i8> %res
 }
 
 define <8 x i16> @xtn2_8h(<4 x i16> %ret, <4 x i32> %A) nounwind {
-;CHECK-LABEL: xtn2_8h:
-;CHECK-NOT: ld1
-;CHECK: xtn2.8h v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: xtn2_8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    xtn2.8h v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = trunc <4 x i32> %A to <4 x i16>
         %res = shufflevector <4 x i16> %ret, <4 x i16> %tmp3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
         ret <8 x i16> %res
 }
 
 define <4 x i32> @xtn2_4s(<2 x i32> %ret, <2 x i64> %A) nounwind {
-;CHECK-LABEL: xtn2_4s:
-;CHECK-NOT: ld1
-;CHECK: xtn2.4s v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: xtn2_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    xtn2.4s v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = trunc <2 x i64> %A to <2 x i32>
         %res = shufflevector <2 x i32> %ret, <2 x i32> %tmp3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
         ret <4 x i32> %res
 }
 
 define <8 x i8> @sqxtn8b(<8 x i16> %A) nounwind {
-;CHECK-LABEL: sqxtn8b:
-;CHECK-NOT: ld1
-;CHECK: sqxtn.8b v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtn8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqxtn.8b v0, v0
+; CHECK-NEXT:    ret
         %tmp3 = call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> %A)
         ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @sqxtn4h(<4 x i32> %A) nounwind {
-;CHECK-LABEL: sqxtn4h:
-;CHECK-NOT: ld1
-;CHECK: sqxtn.4h v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtn4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqxtn.4h v0, v0
+; CHECK-NEXT:    ret
         %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> %A)
         ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @sqxtn2s(<2 x i64> %A) nounwind {
-;CHECK-LABEL: sqxtn2s:
-;CHECK-NOT: ld1
-;CHECK: sqxtn.2s v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtn2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqxtn.2s v0, v0
+; CHECK-NEXT:    ret
         %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> %A)
         ret <2 x i32> %tmp3
 }
 
 define <16 x i8> @sqxtn2_16b(<8 x i8> %ret, <8 x i16> %A) nounwind {
-;CHECK-LABEL: sqxtn2_16b:
-;CHECK-NOT: ld1
-;CHECK: sqxtn2.16b v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtn2_16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtn2.16b v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = call <8 x i8> @llvm.aarch64.neon.sqxtn.v8i8(<8 x i16> %A)
         %res = shufflevector <8 x i8> %ret, <8 x i8> %tmp3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
         ret <16 x i8> %res
 }
 
 define <8 x i16> @sqxtn2_8h(<4 x i16> %ret, <4 x i32> %A) nounwind {
-;CHECK-LABEL: sqxtn2_8h:
-;CHECK-NOT: ld1
-;CHECK: sqxtn2.8h v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtn2_8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtn2.8h v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32> %A)
         %res = shufflevector <4 x i16> %ret, <4 x i16> %tmp3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
         ret <8 x i16> %res
 }
 
 define <4 x i32> @sqxtn2_4s(<2 x i32> %ret, <2 x i64> %A) nounwind {
-;CHECK-LABEL: sqxtn2_4s:
-;CHECK-NOT: ld1
-;CHECK: sqxtn2.4s v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtn2_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtn2.4s v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64> %A)
         %res = shufflevector <2 x i32> %ret, <2 x i32> %tmp3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
         ret <4 x i32> %res
@@ -119,57 +127,60 @@ declare <4 x i16> @llvm.aarch64.neon.sqxtn.v4i16(<4 x i32>) nounwind readnone
 declare <2 x i32> @llvm.aarch64.neon.sqxtn.v2i32(<2 x i64>) nounwind readnone
 
 define <8 x i8> @uqxtn8b(<8 x i16> %A) nounwind {
-;CHECK-LABEL: uqxtn8b:
-;CHECK-NOT: ld1
-;CHECK: uqxtn.8b v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: uqxtn8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uqxtn.8b v0, v0
+; CHECK-NEXT:    ret
         %tmp3 = call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> %A)
         ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @uqxtn4h(<4 x i32> %A) nounwind {
-;CHECK-LABEL: uqxtn4h:
-;CHECK-NOT: ld1
-;CHECK: uqxtn.4h v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: uqxtn4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uqxtn.4h v0, v0
+; CHECK-NEXT:    ret
         %tmp3 = call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> %A)
         ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @uqxtn2s(<2 x i64> %A) nounwind {
-;CHECK-LABEL: uqxtn2s:
-;CHECK-NOT: ld1
-;CHECK: uqxtn.2s v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: uqxtn2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uqxtn.2s v0, v0
+; CHECK-NEXT:    ret
         %tmp3 = call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> %A)
         ret <2 x i32> %tmp3
 }
 
 define <16 x i8> @uqxtn2_16b(<8 x i8> %ret, <8 x i16> %A) nounwind {
-;CHECK-LABEL: uqxtn2_16b:
-;CHECK-NOT: ld1
-;CHECK: uqxtn2.16b v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: uqxtn2_16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    uqxtn2.16b v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = call <8 x i8> @llvm.aarch64.neon.uqxtn.v8i8(<8 x i16> %A)
         %res = shufflevector <8 x i8> %ret, <8 x i8> %tmp3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
         ret <16 x i8> %res
 }
 
 define <8 x i16> @uqxtn2_8h(<4 x i16> %ret, <4 x i32> %A) nounwind {
-;CHECK-LABEL: uqxtn2_8h:
-;CHECK-NOT: ld1
-;CHECK: uqxtn2.8h v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: uqxtn2_8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    uqxtn2.8h v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = call <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32> %A)
         %res = shufflevector <4 x i16> %ret, <4 x i16> %tmp3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
         ret <8 x i16> %res
 }
 
 define <4 x i32> @uqxtn2_4s(<2 x i32> %ret, <2 x i64> %A) nounwind {
-;CHECK-LABEL: uqxtn2_4s:
-;CHECK-NOT: ld1
-;CHECK: uqxtn2.4s v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: uqxtn2_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    uqxtn2.4s v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = call <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64> %A)
         %res = shufflevector <2 x i32> %ret, <2 x i32> %tmp3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
         ret <4 x i32> %res
@@ -180,57 +191,60 @@ declare <4 x i16> @llvm.aarch64.neon.uqxtn.v4i16(<4 x i32>) nounwind readnone
 declare <2 x i32> @llvm.aarch64.neon.uqxtn.v2i32(<2 x i64>) nounwind readnone
 
 define <8 x i8> @sqxtun8b(<8 x i16> %A) nounwind {
-;CHECK-LABEL: sqxtun8b:
-;CHECK-NOT: ld1
-;CHECK: sqxtun.8b v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtun8b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqxtun.8b v0, v0
+; CHECK-NEXT:    ret
         %tmp3 = call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> %A)
         ret <8 x i8> %tmp3
 }
 
 define <4 x i16> @sqxtun4h(<4 x i32> %A) nounwind {
-;CHECK-LABEL: sqxtun4h:
-;CHECK-NOT: ld1
-;CHECK: sqxtun.4h v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtun4h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqxtun.4h v0, v0
+; CHECK-NEXT:    ret
         %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> %A)
         ret <4 x i16> %tmp3
 }
 
 define <2 x i32> @sqxtun2s(<2 x i64> %A) nounwind {
-;CHECK-LABEL: sqxtun2s:
-;CHECK-NOT: ld1
-;CHECK: sqxtun.2s v0, v0
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtun2s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqxtun.2s v0, v0
+; CHECK-NEXT:    ret
         %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> %A)
         ret <2 x i32> %tmp3
 }
 
 define <16 x i8> @sqxtun2_16b(<8 x i8> %ret, <8 x i16> %A) nounwind {
-;CHECK-LABEL: sqxtun2_16b:
-;CHECK-NOT: ld1
-;CHECK: sqxtun2.16b v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtun2_16b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtun2.16b v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = call <8 x i8> @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16> %A)
         %res = shufflevector <8 x i8> %ret, <8 x i8> %tmp3, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
         ret <16 x i8> %res
 }
 
 define <8 x i16> @sqxtun2_8h(<4 x i16> %ret, <4 x i32> %A) nounwind {
-;CHECK-LABEL: sqxtun2_8h:
-;CHECK-NOT: ld1
-;CHECK: sqxtun2.8h v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtun2_8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtun2.8h v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32> %A)
         %res = shufflevector <4 x i16> %ret, <4 x i16> %tmp3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
         ret <8 x i16> %res
 }
 
 define <4 x i32> @sqxtun2_4s(<2 x i32> %ret, <2 x i64> %A) nounwind {
-;CHECK-LABEL: sqxtun2_4s:
-;CHECK-NOT: ld1
-;CHECK: sqxtun2.4s v0, v1
-;CHECK-NEXT: ret
+; CHECK-LABEL: sqxtun2_4s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    sqxtun2.4s v0, v1
+; CHECK-NEXT:    ret
         %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64> %A)
         %res = shufflevector <2 x i32> %ret, <2 x i32> %tmp3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
         ret <4 x i32> %res
@@ -240,3 +254,6 @@ declare <8 x i8>  @llvm.aarch64.neon.sqxtun.v8i8(<8 x i16>) nounwind readnone
 declare <4 x i16> @llvm.aarch64.neon.sqxtun.v4i16(<4 x i32>) nounwind readnone
 declare <2 x i32> @llvm.aarch64.neon.sqxtun.v2i32(<2 x i64>) nounwind readnone
 
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-GI: {{.*}}
+; CHECK-SD: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/arm64-vmul.ll b/llvm/test/CodeGen/AArch64/arm64-vmul.ll
index 07400bbb2f58c..d12f7ce33e246 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vmul.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vmul.ll
@@ -3,6 +3,7 @@
 ; RUN: llc -mtriple=aarch64-none-elf -mattr=+aes -global-isel -global-isel-abort=2 2>&1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; CHECK-GI:       warning: Instruction selection used fallback path for pmull8h
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for commutable_pmull8h
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for sqdmulh_1s
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fmls_2s
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for fmls_4s
@@ -78,6 +79,20 @@ define <2 x i64> @smull2d(ptr %A, ptr %B) nounwind {
   ret <2 x i64> %tmp3
 }
 
+define void @commutable_smull(<2 x i32> %A, <2 x i32> %B, ptr %C) {
+; CHECK-LABEL: commutable_smull:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    stp q0, q0, [x0]
+; CHECK-NEXT:    ret
+  %1 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %A, <2 x i32> %B)
+  %2 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %B, <2 x i32> %A)
+  store <2 x i64> %1, ptr %C
+  %3 = getelementptr i8, ptr %C, i64 16
+  store <2 x i64> %2, ptr %3
+  ret void
+}
+
 declare <8 x i16>  @llvm.aarch64.neon.smull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone
@@ -121,6 +136,20 @@ define <2 x i64> @umull2d(ptr %A, ptr %B) nounwind {
   ret <2 x i64> %tmp3
 }
 
+define void @commutable_umull(<2 x i32> %A, <2 x i32> %B, ptr %C) {
+; CHECK-LABEL: commutable_umull:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    stp q0, q0, [x0]
+; CHECK-NEXT:    ret
+  %1 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %A, <2 x i32> %B)
+  %2 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %B, <2 x i32> %A)
+  store <2 x i64> %1, ptr %C
+  %3 = getelementptr i8, ptr %C, i64 16
+  store <2 x i64> %2, ptr %3
+  ret void
+}
+
 declare <8 x i16>  @llvm.aarch64.neon.umull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone
 declare <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone
@@ -212,6 +241,20 @@ define <8 x i16> @pmull8h(ptr %A, ptr %B) nounwind {
   ret <8 x i16> %tmp3
 }
 
+define void @commutable_pmull8h(<8 x i8> %A, <8 x i8> %B, ptr %C) {
+; CHECK-LABEL: commutable_pmull8h:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    pmull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    stp q0, q0, [x0]
+; CHECK-NEXT:    ret
+  %1 = call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %A, <8 x i8> %B)
+  %2 = call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %B, <8 x i8> %A)
+  store <8 x i16> %1, ptr %C
+  %3 = getelementptr i8, ptr %C, i8 16
+  store <8 x i16> %2, ptr %3
+  ret void
+}
+
 declare <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone
 
 define <4 x i16> @sqdmulh_4h(ptr %A, ptr %B) nounwind {
@@ -487,10 +530,10 @@ define void @smlal2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2,
 ; CHECK-GI-LABEL: smlal2d_chain_with_constant:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mvn v3.8b, v2.8b
-; CHECK-GI-NEXT:    adrp x8, .LCPI27_0
+; CHECK-GI-NEXT:    adrp x8, .LCPI30_0
 ; CHECK-GI-NEXT:    smull v1.2d, v1.2s, v3.2s
 ; CHECK-GI-NEXT:    smlal v1.2d, v0.2s, v2.2s
-; CHECK-GI-NEXT:    ldr q0, [x8, :lo12:.LCPI27_0]
+; CHECK-GI-NEXT:    ldr q0, [x8, :lo12:.LCPI30_0]
 ; CHECK-GI-NEXT:    add v0.2d, v1.2d, v0.2d
 ; CHECK-GI-NEXT:    str q0, [x0]
 ; CHECK-GI-NEXT:    ret
@@ -566,8 +609,8 @@ define void @smlsl2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2,
 ;
 ; CHECK-GI-LABEL: smlsl2d_chain_with_constant:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    adrp x8, .LCPI31_0
-; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI31_0]
+; CHECK-GI-NEXT:    adrp x8, .LCPI34_0
+; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI34_0]
 ; CHECK-GI-NEXT:    smlsl v3.2d, v0.2s, v2.2s
 ; CHECK-GI-NEXT:    mvn v0.8b, v2.8b
 ; CHECK-GI-NEXT:    smlsl v3.2d, v1.2s, v0.2s
@@ -829,10 +872,10 @@ define void @umlal2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2,
 ; CHECK-GI-LABEL: umlal2d_chain_with_constant:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mvn v3.8b, v2.8b
-; CHECK-GI-NEXT:    adrp x8, .LCPI43_0
+; CHECK-GI-NEXT:    adrp x8, .LCPI46_0
 ; CHECK-GI-NEXT:    umull v1.2d, v1.2s, v3.2s
 ; CHECK-GI-NEXT:    umlal v1.2d, v0.2s, v2.2s
-; CHECK-GI-NEXT:    ldr q0, [x8, :lo12:.LCPI43_0]
+; CHECK-GI-NEXT:    ldr q0, [x8, :lo12:.LCPI46_0]
 ; CHECK-GI-NEXT:    add v0.2d, v1.2d, v0.2d
 ; CHECK-GI-NEXT:    str q0, [x0]
 ; CHECK-GI-NEXT:    ret
@@ -908,8 +951,8 @@ define void @umlsl2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2,
 ;
 ; CHECK-GI-LABEL: umlsl2d_chain_with_constant:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    adrp x8, .LCPI47_0
-; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI47_0]
+; CHECK-GI-NEXT:    adrp x8, .LCPI50_0
+; CHECK-GI-NEXT:    ldr q3, [x8, :lo12:.LCPI50_0]
 ; CHECK-GI-NEXT:    umlsl v3.2d, v0.2s, v2.2s
 ; CHECK-GI-NEXT:    mvn v0.8b, v2.8b
 ; CHECK-GI-NEXT:    umlsl v3.2d, v1.2s, v0.2s
@@ -3222,6 +3265,20 @@ define <16 x i8> @test_pmull_high_64(<2 x i64> %l, <2 x i64> %r) nounwind {
   ret <16 x i8> %val
 }
 
+define <16 x i8> @test_commutable_pmull_64(i64 %l, i64 %r) nounwind {
+; CHECK-LABEL: test_commutable_pmull_64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov d0, x1
+; CHECK-NEXT:    fmov d1, x0
+; CHECK-NEXT:    pmull v0.1q, v1.1d, v0.1d
+; CHECK-NEXT:    add v0.16b, v0.16b, v0.16b
+; CHECK-NEXT:    ret
+  %1 = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %l, i64 %r)
+  %2 = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %r, i64 %l)
+  %3 = add <16 x i8> %1, %2
+  ret <16 x i8> %3
+}
+
 declare <16 x i8> @llvm.aarch64.neon.pmull64(i64, i64)
 
 define <1 x i64> @test_mul_v1i64(<1 x i64> %lhs, <1 x i64> %rhs) nounwind {
diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-fpr.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-fpr.ll
index f422f96f33495..fa15ab42c2638 100644
--- a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-fpr.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-fpr.ll
@@ -1,7 +1,5 @@
 ; RUN: llc < %s -mtriple=arm64-linux-gnu | FileCheck %s -check-prefixes=NOTCPU-LINUX --match-full-lines
 ; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=generic | FileCheck %s -check-prefixes=NOTCPU-APPLE --match-full-lines
-; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 | FileCheck %s -check-prefixes=CPU --match-full-lines
-; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 -mattr=-zcm-fpr64 | FileCheck %s -check-prefixes=NOTATTR --match-full-lines
 ; RUN: llc < %s -mtriple=arm64-apple-macosx -mattr=+zcm-fpr64 | FileCheck %s -check-prefixes=ATTR --match-full-lines
 
 define void @zero_cycle_regmov_FPR32(float %a, float %b, float %c, float %d) {
@@ -23,22 +21,6 @@ entry:
 ; NOTCPU-APPLE: fmov s0, [[REG1]]
 ; NOTCPU-APPLE: fmov s1, [[REG2]]
 
-; CPU: fmov [[REG2:d[0-9]+]], d3
-; CPU: fmov [[REG1:d[0-9]+]], d2
-; CPU: fmov d0, d2
-; CPU: fmov d1, d3
-; CPU-NEXT: bl {{_?foo_float}}
-; CPU: fmov d0, [[REG1]]
-; CPU: fmov d1, [[REG2]]
-
-; NOTATTR: fmov [[REG2:s[0-9]+]], s3
-; NOTATTR: fmov [[REG1:s[0-9]+]], s2
-; NOTATTR: fmov s0, s2
-; NOTATTR: fmov s1, s3
-; NOTATTR-NEXT: bl {{_?foo_float}}
-; NOTATTR: fmov s0, [[REG1]]
-; NOTATTR: fmov s1, [[REG2]]
-
 ; ATTR: fmov d0, d2
 ; ATTR: fmov d1, d3
 ; ATTR: fmov [[REG2:d[0-9]+]], d3
@@ -72,22 +54,6 @@ entry:
 ; NOTCPU-APPLE: fmov s0, [[REG1]]
 ; NOTCPU-APPLE: fmov s1, [[REG2]]
 
-; CPU: fmov [[REG2:d[0-9]+]], d3
-; CPU: fmov [[REG1:d[0-9]+]], d2
-; CPU: fmov d0, d2
-; CPU: fmov d1, d3
-; CPU-NEXT: bl {{_?foo_half}}
-; CPU: fmov d0, [[REG1]]
-; CPU: fmov d1, [[REG2]]
-
-; NOTATTR: fmov [[REG2:s[0-9]+]], s3
-; NOTATTR: fmov [[REG1:s[0-9]+]], s2
-; NOTATTR: fmov s0, s2
-; NOTATTR: fmov s1, s3
-; NOTATTR-NEXT: bl {{_?foo_half}}
-; NOTATTR: fmov s0, [[REG1]]
-; NOTATTR: fmov s1, [[REG2]]
-
 ; ATTR: fmov d0, d2
 ; ATTR: fmov d1, d3
 ; ATTR: fmov [[REG2:d[0-9]+]], d3
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-dont-call.ll b/llvm/test/CodeGen/AArch64/arm64ec-dont-call.ll
new file mode 100644
index 0000000000000..b53def08ee84b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64ec-dont-call.ll
@@ -0,0 +1,22 @@
+; RUN: not llc -mtriple=arm64ec-windows-msvc -filetype=null %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=arm64ec-windows-msvc -filetype=null -global-isel=1 -global-isel-abort=0 %s 2>&1 | FileCheck %s
+
+define void @baz() #0 {
+  call void @foo()
+  ret void
+}
+
+define void @foo() #1 {
+  ret void
+}
+
+attributes #0 = { noinline optnone }
+attributes #1 = { "dontcall-error"="oh no foo" }
+
+; Regression test for `dontcall-error` for Arm64EC. Since this attribute is
+; checked both by FastISel and SelectionDAGBuilder, and FastISel was bailing for
+; Arm64EC AFTER doing the check, we ended up with duplicate copies of this
+; error.
+
+; CHECK: error: call to #foo marked "dontcall-error": oh no foo
+; CHECK-NOT: error:
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-entry-thunks.ll b/llvm/test/CodeGen/AArch64/arm64ec-entry-thunks.ll
index 6aeeeed94543d..709a17e32f58e 100644
--- a/llvm/test/CodeGen/AArch64/arm64ec-entry-thunks.ll
+++ b/llvm/test/CodeGen/AArch64/arm64ec-entry-thunks.ll
@@ -85,10 +85,10 @@ define i64 @simple_integers(i8, i16, i32, i64) nounwind {
   ret i64 0
 }
 
-; NOTE: Only float and double are supported.
-define double @simple_floats(float, double) nounwind {
-; CHECK-LABEL:    .def    $ientry_thunk$cdecl$d$fd;
-; CHECK:          .section        .wowthk$aa,"xr",discard,$ientry_thunk$cdecl$d$fd
+; NOTE: Only half, float, and double are supported.
+define double @simple_floats(half, float, double) nounwind {
+; CHECK-LABEL:    .def    $ientry_thunk$cdecl$d$__llvm_h__fd;
+; CHECK:          .section        .wowthk$aa,"xr",discard,$ientry_thunk$cdecl$d$__llvm_h__fd
 ; CHECK:          // %bb.0:
 ; CHECK-NEXT:     stp     q6, q7, [sp, #-176]!            // 32-byte Folded Spill
 ; CHECK-NEXT:     .seh_save_any_reg_px    q6, 176
@@ -600,7 +600,7 @@ start:
 ; CHECK-NEXT:     .symidx $ientry_thunk$cdecl$i8$i8i8i8i8
 ; CHECK-NEXT:     .word   1
 ; CHECK-NEXT:     .symidx "#simple_floats"
-; CHECK-NEXT:     .symidx $ientry_thunk$cdecl$d$fd
+; CHECK-NEXT:     .symidx $ientry_thunk$cdecl$d$__llvm_h__fd
 ; CHECK-NEXT:     .word   1
 ; CHECK-NEXT:     .symidx "#has_varargs"
 ; CHECK-NEXT:     .symidx $ientry_thunk$cdecl$v$varargs
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-exit-thunks.ll b/llvm/test/CodeGen/AArch64/arm64ec-exit-thunks.ll
index cba7a8100930f..f829227a47cd7 100644
--- a/llvm/test/CodeGen/AArch64/arm64ec-exit-thunks.ll
+++ b/llvm/test/CodeGen/AArch64/arm64ec-exit-thunks.ll
@@ -93,10 +93,10 @@ declare i64 @simple_integers(i8, i16, i32, i64) nounwind;
 ; CHECK-NEXT:     .seh_endfunclet
 ; CHECK-NEXT:     .seh_endproc
 
-; NOTE: Only float and double are supported.
-declare double @simple_floats(float, double) nounwind;
-; CHECK-LABEL:    .def    $iexit_thunk$cdecl$d$fd;
-; CHECK:          .section        .wowthk$aa,"xr",discard,$iexit_thunk$cdecl$d$fd
+; NOTE: Only half, float, and double are supported.
+declare double @simple_floats(half, float, double) nounwind;
+; CHECK-LABEL:    .def    $iexit_thunk$cdecl$d$__llvm_h__fd;
+; CHECK:          .section        .wowthk$aa,"xr",discard,$iexit_thunk$cdecl$d$__llvm_h__fd
 ; CHECK:          // %bb.0:
 ; CHECK-NEXT:     sub     sp, sp, #48
 ; CHECK-NEXT:     .seh_stackalloc 48
@@ -129,8 +129,8 @@ declare double @simple_floats(float, double) nounwind;
 ; CHECK-NEXT:     adrp    x11, simple_floats
 ; CHECK-NEXT:     add     x11, x11, :lo12:simple_floats
 ; CHECK-NEXT:     ldr     x8, [x8, :lo12:__os_arm64x_check_icall]
-; CHECK-NEXT:     adrp    x10, $iexit_thunk$cdecl$d$fd
-; CHECK-NEXT:     add     x10, x10, :lo12:$iexit_thunk$cdecl$d$fd
+; CHECK-NEXT:     adrp    x10, $iexit_thunk$cdecl$d$__llvm_h__fd
+; CHECK-NEXT:     add     x10, x10, :lo12:$iexit_thunk$cdecl$d$__llvm_h__fd
 ; CHECK-NEXT:     blr     x8
 ; CHECK-NEXT:     .seh_startepilogue
 ; CHECK-NEXT:     ldr     x30, [sp], #16                  // 8-byte Folded Reload
@@ -282,33 +282,36 @@ declare void @has_aligned_sret(ptr align 32 sret(%TSRet)) nounwind;
 ; CHECK:          .seh_endfunclet
 ; CHECK:          .seh_endproc
 
-declare [2 x i8] @small_array([2 x i8], [2 x float]) nounwind;
-; CHECK-LABEL:    .def    $iexit_thunk$cdecl$m2$m2F8;
-; CHECK:          .section        .wowthk$aa,"xr",discard,$iexit_thunk$cdecl$m2$m2F8
+declare [2 x i8] @small_array([2 x i8], [2 x half], [2 x float]) nounwind;
+; CHECK-LABEL:    .def    $iexit_thunk$cdecl$m2$m2__llvm_H__4F8;
+; CHECK:          .section        .wowthk$aa,"xr",discard,$iexit_thunk$cdecl$m2$m2__llvm_H__4F8
 ; CHECK:          // %bb.0:
-; CHECK-NEXT:     sub     sp, sp, #64
-; CHECK-NEXT:     .seh_stackalloc 64
-; CHECK-NEXT:     stp     x29, x30, [sp, #48]             // 16-byte Folded Spill
-; CHECK-NEXT:     .seh_save_fplr  48
-; CHECK-NEXT:     add     x29, sp, #48
-; CHECK-NEXT:     .seh_add_fp     48
+; CHECK-NEXT:     sub     sp, sp, #80
+; CHECK-NEXT:     .seh_stackalloc 80
+; CHECK-NEXT:     stp     x29, x30, [sp, #64]             // 16-byte Folded Spill
+; CHECK-NEXT:     .seh_save_fplr  64
+; CHECK-NEXT:     add     x29, sp, #64
+; CHECK-NEXT:     .seh_add_fp     64
 ; CHECK-NEXT:     .seh_endprologue
-; CHECK-NEXT:     sturb   w1, [x29, #-1]
-; CHECK-NEXT:     adrp    x8, __os_arm64x_dispatch_call_no_redirect
-; CHECK-NEXT:     sturb   w0, [x29, #-2]
-; CHECK-NEXT:     ldr     x16, [x8, :lo12:__os_arm64x_dispatch_call_no_redirect]
-; CHECK-NEXT:     stp     s0, s1, [x29, #-12]
-; CHECK-NEXT:     ldurh   w0, [x29, #-2]
-; CHECK-NEXT:     ldur    x1, [x29, #-12]
-; CHECK-NEXT:     blr     x16
-; CHECK-NEXT:     mov     w0, w8
-; CHECK-NEXT:     sturh   w8, [x29, #-14]
-; CHECK-NEXT:     ubfx    w1, w8, #8, #8
+; CHECK-NEXT:     sturb	w0, [x29, #-2]
+; CHECK-NEXT:     adrp	x8, __os_arm64x_dispatch_call_no_redirect
+; CHECK-NEXT:     sturb	w1, [x29, #-1]
+; CHECK-NEXT:     ldr	x16, [x8, :lo12:__os_arm64x_dispatch_call_no_redirect]
+; CHECK-NEXT:     stur	h0, [x29, #-6]
+; CHECK-NEXT:     ldurh	w0, [x29, #-2]
+; CHECK-NEXT:     stur	h1, [x29, #-4]
+; CHECK-NEXT:     stp	s2, s3, [x29, #-16]
+; CHECK-NEXT:     ldur	w1, [x29, #-6]
+; CHECK-NEXT:     ldur	x2, [x29, #-16]
+; CHECK-NEXT:     blr	x16
+; CHECK-NEXT:     mov	w0, w8
+; CHECK-NEXT:     sturh	w8, [x29, #-18]
+; CHECK-NEXT:     ubfx	w1, w8, #8, #8
 ; CHECK-NEXT:     .seh_startepilogue
-; CHECK-NEXT:     ldp     x29, x30, [sp, #48]             // 16-byte Folded Reload
-; CHECK-NEXT:     .seh_save_fplr  48
-; CHECK-NEXT:     add     sp, sp, #64
-; CHECK-NEXT:     .seh_stackalloc 64
+; CHECK-NEXT:     ldp	x29, x30, [sp, #64]             // 16-byte Folded Reload
+; CHECK-NEXT:     .seh_save_fplr	64
+; CHECK-NEXT:     add	sp, sp, #80
+; CHECK-NEXT:     .seh_stackalloc	80
 ; CHECK-NEXT:     .seh_endepilogue
 ; CHECK-NEXT:     ret
 ; CHECK-NEXT:     .seh_endfunclet
@@ -325,8 +328,8 @@ declare [2 x i8] @small_array([2 x i8], [2 x float]) nounwind;
 ; CHECK-NEXT:     adrp    x11, small_array
 ; CHECK-NEXT:     add     x11, x11, :lo12:small_array
 ; CHECK-NEXT:     ldr     x8, [x8, :lo12:__os_arm64x_check_icall]
-; CHECK-NEXT:     adrp    x10, $iexit_thunk$cdecl$m2$m2F8
-; CHECK-NEXT:     add     x10, x10, :lo12:$iexit_thunk$cdecl$m2$m2F8
+; CHECK-NEXT:     adrp    x10, $iexit_thunk$cdecl$m2$m2__llvm_H__4F8
+; CHECK-NEXT:     add     x10, x10, :lo12:$iexit_thunk$cdecl$m2$m2__llvm_H__4F8
 ; CHECK-NEXT:     blr     x8
 ; CHECK-NEXT:     .seh_startepilogue
 ; CHECK-NEXT:     ldr     x30, [sp], #16                  // 8-byte Folded Reload
@@ -577,7 +580,7 @@ declare <8 x i16> @large_vector(<8 x i16> %0) nounwind;
 ; CHECK-NEXT:     .symidx simple_integers
 ; CHECK-NEXT:     .word   0
 ; CHECK-NEXT:     .symidx simple_floats
-; CHECK-NEXT:     .symidx $iexit_thunk$cdecl$d$fd
+; CHECK-NEXT:     .symidx $iexit_thunk$cdecl$d$__llvm_h__fd
 ; CHECK-NEXT:     .word   4
 ; CHECK-NEXT:     .symidx "#simple_floats$exit_thunk"
 ; CHECK-NEXT:     .symidx simple_floats
@@ -601,7 +604,7 @@ declare <8 x i16> @large_vector(<8 x i16> %0) nounwind;
 ; CHECK-NEXT:     .symidx has_aligned_sret
 ; CHECK-NEXT:     .word   0
 ; CHECK-NEXT:     .symidx small_array
-; CHECK-NEXT:     .symidx $iexit_thunk$cdecl$m2$m2F8
+; CHECK-NEXT:     .symidx $iexit_thunk$cdecl$m2$m2__llvm_H__4F8
 ; CHECK-NEXT:     .word   4
 ; CHECK-NEXT:     .symidx "#small_array$exit_thunk"
 ; CHECK-NEXT:     .symidx small_array
@@ -634,14 +637,14 @@ declare <8 x i16> @large_vector(<8 x i16> %0) nounwind;
 define void @func_caller() nounwind {
   call void @no_op()
   call i64 @simple_integers(i8 0, i16 0, i32 0, i64 0)
-  call double @simple_floats(float 0.0, double 0.0)
+  call double @simple_floats(half 0.0, float 0.0, double 0.0)
   call void (...) @has_varargs()
   %c = alloca i8
   call void @has_sret(ptr sret([100 x i8]) %c)
   %aligned = alloca %TSRet, align 32
   store %TSRet { i64 0, i64 0 }, ptr %aligned, align 32
   call void @has_aligned_sret(ptr align 32 sret(%TSRet) %aligned)
-  call [2 x i8] @small_array([2 x i8] [i8 0, i8 0], [2 x float] [float 0.0, float 0.0])
+  call [2 x i8] @small_array([2 x i8] [i8 0, i8 0], [2 x half] [half 0.0, half 0.0], [2 x float] [float 0.0, float 0.0])
   call [3 x i64] @large_array([3 x i64] [i64 0, i64 0, i64 0], [2 x double] [double 0.0, double 0.0], [2 x [2 x i64]] [[2 x i64] [i64 0, i64 0], [2 x i64] [i64 0, i64 0]])
   call %T2 @simple_struct(%T1 { i16 0 }, %T2 { i32 0, float 0.0 }, %T3 { i64 0, double 0.0 }, %T4 { i64 0, double 0.0, i8 0 })
   call <4 x i8> @small_vector(<4 x i8> <i8 0, i8 0, i8 0, i8 0>)
diff --git a/llvm/test/CodeGen/AArch64/bitcast.ll b/llvm/test/CodeGen/AArch64/bitcast.ll
index d2f72ecacc86c..20f19fddf790a 100644
--- a/llvm/test/CodeGen/AArch64/bitcast.ll
+++ b/llvm/test/CodeGen/AArch64/bitcast.ll
@@ -617,6 +617,31 @@ define <8 x i64> @bitcast_v16i32_v8i64(<16 x i32> %a, <16 x i32> %b){
   ret <8 x i64> %d
 }
 
+define <8 x i32> @scalar_i128(<2 x i128> %a) {
+; CHECK-SD-LABEL: scalar_i128:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmov d1, x2
+; CHECK-SD-NEXT:    fmov d0, x0
+; CHECK-SD-NEXT:    mov v1.d[1], x3
+; CHECK-SD-NEXT:    mov v0.d[1], x1
+; CHECK-SD-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    add v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: scalar_i128:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov v0.d[0], x0
+; CHECK-GI-NEXT:    mov v1.d[0], x2
+; CHECK-GI-NEXT:    mov v0.d[1], x1
+; CHECK-GI-NEXT:    mov v1.d[1], x3
+; CHECK-GI-NEXT:    add v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    add v1.4s, v1.4s, v1.4s
+; CHECK-GI-NEXT:    ret
+  %c = bitcast <2 x i128> %a to <8 x i32>
+  %d = add <8 x i32> %c, %c
+  ret <8 x i32> %d
+}
+
 ; ===== Vectors with Non-Pow 2 Widths =====
 
 define <6 x i16> @bitcast_v3i32_v6i16(<3 x i32> %a, <3 x i32> %b){
diff --git a/llvm/test/CodeGen/AArch64/blr-bti-preserves-operands.mir b/llvm/test/CodeGen/AArch64/blr-bti-preserves-operands.mir
index 6d4f21fdee950..f41e590c870a4 100644
--- a/llvm/test/CodeGen/AArch64/blr-bti-preserves-operands.mir
+++ b/llvm/test/CodeGen/AArch64/blr-bti-preserves-operands.mir
@@ -8,7 +8,7 @@
 # The arguments to the call must become implicit arguments, because the branch
 # only expects to get 1 explicit operand which is the branch target.
 
-# CHECK:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit $sp, implicit $x0, implicit $w1 {
+# CHECK:    BUNDLE implicit-def $lr, implicit-def $sp, implicit $sp, implicit $x0, implicit $w1 {
 # CHECK:      BL @_setjmp, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $w1, implicit-def dead $lr, implicit $sp, implicit-def $sp
 # CHECK:      HINT 36
 # CHECK:    }
diff --git a/llvm/test/CodeGen/AArch64/bsp_implicit_ops.mir b/llvm/test/CodeGen/AArch64/bsp_implicit_ops.mir
index 23ac67cac6416..805d24475081e 100644
--- a/llvm/test/CodeGen/AArch64/bsp_implicit_ops.mir
+++ b/llvm/test/CodeGen/AArch64/bsp_implicit_ops.mir
@@ -96,3 +96,23 @@ body:             |
     $q25 = ORRv16i8 $q3, killed $q3
     RET_ReallyLR implicit $q22
 ...
+---
+name:            DoubleOp
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $q2
+
+    ; CHECK-LABEL: name: DoubleOp
+    ; CHECK: liveins: $q2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: renamable $q0 = MOVIv8i16 1, 0
+    ; CHECK-NEXT: renamable $q1 = ORRv16i8 renamable $q2, renamable $q2
+    ; CHECK-NEXT: renamable $q1 = BSLv16i8 killed renamable $q1, renamable $q2, renamable $q0
+    ; CHECK-NEXT: renamable $q0 = SQADDv8i16 killed renamable $q1, killed renamable $q0
+    ; CHECK-NEXT: RET undef $lr, implicit $q0
+    renamable $q0 = MOVIv8i16 1, 0
+    renamable $q1 = BSPv16i8 killed renamable $q2, renamable $q2, renamable $q0
+    renamable $q0 = SQADDv8i16 killed renamable $q1, killed renamable $q0
+    RET_ReallyLR implicit $q0
+...
diff --git a/llvm/test/CodeGen/AArch64/cfi-fixup-multi-block-prologue.mir b/llvm/test/CodeGen/AArch64/cfi-fixup-multi-block-prologue.mir
index 1ec18b987c1f2..d696776f45d51 100644
--- a/llvm/test/CodeGen/AArch64/cfi-fixup-multi-block-prologue.mir
+++ b/llvm/test/CodeGen/AArch64/cfi-fixup-multi-block-prologue.mir
@@ -80,8 +80,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  30000
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: p, type: default, offset: -30016, size: 30000, alignment: 1,
diff --git a/llvm/test/CodeGen/AArch64/cfi-fixup-multi-section.mir b/llvm/test/CodeGen/AArch64/cfi-fixup-multi-section.mir
index a4c88be375c01..47ede6af42113 100644
--- a/llvm/test/CodeGen/AArch64/cfi-fixup-multi-section.mir
+++ b/llvm/test/CodeGen/AArch64/cfi-fixup-multi-section.mir
@@ -57,8 +57,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16,
@@ -240,8 +240,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16,
diff --git a/llvm/test/CodeGen/AArch64/cfi-fixup.mir b/llvm/test/CodeGen/AArch64/cfi-fixup.mir
index f522df6bb3fa0..b4df731fb266b 100644
--- a/llvm/test/CodeGen/AArch64/cfi-fixup.mir
+++ b/llvm/test/CodeGen/AArch64/cfi-fixup.mir
@@ -65,8 +65,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16,
@@ -248,8 +248,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16,
@@ -403,8 +403,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16,
diff --git a/llvm/test/CodeGen/AArch64/check-sign-bit-before-extension.ll b/llvm/test/CodeGen/AArch64/check-sign-bit-before-extension.ll
index 1d60929f2b94c..0960c4c2a3342 100644
--- a/llvm/test/CodeGen/AArch64/check-sign-bit-before-extension.ll
+++ b/llvm/test/CodeGen/AArch64/check-sign-bit-before-extension.ll
@@ -80,7 +80,7 @@ define i32 @g_i8_sign_extend_inreg(i8 %in, i32 %a, i32 %b) nounwind {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sxtb w8, w0
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w8, w1, w2, lt
+; CHECK-NEXT:    csel w8, w1, w2, mi
 ; CHECK-NEXT:    add w0, w8, w0, uxtb
 ; CHECK-NEXT:    ret
 entry:
@@ -102,7 +102,7 @@ define i32 @g_i16_sign_extend_inreg(i16 %in, i32 %a, i32 %b) nounwind {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sxth w8, w0
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w8, w1, w2, lt
+; CHECK-NEXT:    csel w8, w1, w2, mi
 ; CHECK-NEXT:    add w0, w8, w0, uxth
 ; CHECK-NEXT:    ret
 entry:
@@ -123,7 +123,7 @@ define i64 @g_i32_sign_extend_inreg(i32 %in, i64 %a, i64 %b) nounwind {
 ; CHECK-LABEL: g_i32_sign_extend_inreg:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    csel x8, x1, x2, lt
+; CHECK-NEXT:    csel x8, x1, x2, mi
 ; CHECK-NEXT:    add x0, x8, w0, uxtw
 ; CHECK-NEXT:    ret
 entry:
@@ -170,7 +170,7 @@ define i64 @g_i32_sign_extend_i64(i32 %in, i64 %a, i64 %b) nounwind {
 ; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
 ; CHECK-NEXT:    sxtw x8, w0
 ; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    csel x8, x1, x2, lt
+; CHECK-NEXT:    csel x8, x1, x2, mi
 ; CHECK-NEXT:    add x0, x8, w0, uxtw
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/combine-sdiv.ll b/llvm/test/CodeGen/AArch64/combine-sdiv.ll
index e1ba0e98a6c01..9d0ade2480428 100644
--- a/llvm/test/CodeGen/AArch64/combine-sdiv.ll
+++ b/llvm/test/CodeGen/AArch64/combine-sdiv.ll
@@ -230,14 +230,11 @@ define <16 x i8> @combine_vec_sdiv_by_pow2b_v16i8(<16 x i8> %x) {
 ; CHECK-SD-NEXT:    movi v3.2d, #0x000000000000ff
 ; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI14_0]
 ; CHECK-SD-NEXT:    adrp x8, .LCPI14_1
-; CHECK-SD-NEXT:    movi v4.2d, #0xffffffffffffff00
 ; CHECK-SD-NEXT:    ushl v1.16b, v1.16b, v2.16b
 ; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI14_1]
 ; CHECK-SD-NEXT:    add v1.16b, v0.16b, v1.16b
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v3.16b
 ; CHECK-SD-NEXT:    sshl v1.16b, v1.16b, v2.16b
-; CHECK-SD-NEXT:    and v1.16b, v1.16b, v4.16b
-; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v3.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: combine_vec_sdiv_by_pow2b_v16i8:
@@ -265,21 +262,17 @@ define <16 x i8> @combine_vec_sdiv_by_pow2b_v16i8(<16 x i8> %x) {
 define <8 x i16> @combine_vec_sdiv_by_pow2b_v8i16(<8 x i16> %x) {
 ; CHECK-SD-LABEL: combine_vec_sdiv_by_pow2b_v8i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    adrp x8, .LCPI15_1
+; CHECK-SD-NEXT:    adrp x8, .LCPI15_0
 ; CHECK-SD-NEXT:    cmlt v1.8h, v0.8h, #0
-; CHECK-SD-NEXT:    adrp x9, .LCPI15_3
+; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI15_0]
+; CHECK-SD-NEXT:    adrp x8, .LCPI15_1
+; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v2.8h
 ; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI15_1]
 ; CHECK-SD-NEXT:    adrp x8, .LCPI15_2
-; CHECK-SD-NEXT:    ldr q3, [x9, :lo12:.LCPI15_3]
-; CHECK-SD-NEXT:    ushl v1.8h, v1.8h, v2.8h
-; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI15_2]
-; CHECK-SD-NEXT:    adrp x8, .LCPI15_0
 ; CHECK-SD-NEXT:    add v1.8h, v0.8h, v1.8h
 ; CHECK-SD-NEXT:    sshl v1.8h, v1.8h, v2.8h
-; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI15_0]
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT:    and v1.16b, v1.16b, v3.16b
-; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI15_2]
+; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: combine_vec_sdiv_by_pow2b_v8i16:
@@ -308,28 +301,22 @@ define <8 x i16> @combine_vec_sdiv_by_pow2b_v8i16(<8 x i16> %x) {
 define <16 x i16> @combine_vec_sdiv_by_pow2b_v16i16(<16 x i16> %x) {
 ; CHECK-SD-LABEL: combine_vec_sdiv_by_pow2b_v16i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    adrp x8, .LCPI16_1
+; CHECK-SD-NEXT:    adrp x8, .LCPI16_0
 ; CHECK-SD-NEXT:    cmlt v2.8h, v0.8h, #0
 ; CHECK-SD-NEXT:    cmlt v3.8h, v1.8h, #0
-; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI16_1]
-; CHECK-SD-NEXT:    adrp x8, .LCPI16_2
+; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI16_0]
+; CHECK-SD-NEXT:    adrp x8, .LCPI16_1
 ; CHECK-SD-NEXT:    ushl v2.8h, v2.8h, v4.8h
 ; CHECK-SD-NEXT:    ushl v3.8h, v3.8h, v4.8h
-; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI16_2]
-; CHECK-SD-NEXT:    adrp x8, .LCPI16_0
-; CHECK-SD-NEXT:    ldr q5, [x8, :lo12:.LCPI16_0]
-; CHECK-SD-NEXT:    adrp x8, .LCPI16_3
+; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI16_1]
+; CHECK-SD-NEXT:    adrp x8, .LCPI16_2
 ; CHECK-SD-NEXT:    add v2.8h, v0.8h, v2.8h
 ; CHECK-SD-NEXT:    add v3.8h, v1.8h, v3.8h
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v5.16b
-; CHECK-SD-NEXT:    and v1.16b, v1.16b, v5.16b
 ; CHECK-SD-NEXT:    sshl v2.8h, v2.8h, v4.8h
 ; CHECK-SD-NEXT:    sshl v3.8h, v3.8h, v4.8h
-; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI16_3]
-; CHECK-SD-NEXT:    and v2.16b, v2.16b, v4.16b
-; CHECK-SD-NEXT:    and v3.16b, v3.16b, v4.16b
-; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI16_2]
+; CHECK-SD-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v4.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: combine_vec_sdiv_by_pow2b_v16i16:
@@ -363,42 +350,32 @@ define <16 x i16> @combine_vec_sdiv_by_pow2b_v16i16(<16 x i16> %x) {
 define <32 x i16> @combine_vec_sdiv_by_pow2b_v32i16(<32 x i16> %x) {
 ; CHECK-SD-LABEL: combine_vec_sdiv_by_pow2b_v32i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    adrp x8, .LCPI17_1
+; CHECK-SD-NEXT:    adrp x8, .LCPI17_0
 ; CHECK-SD-NEXT:    cmlt v4.8h, v0.8h, #0
 ; CHECK-SD-NEXT:    cmlt v5.8h, v1.8h, #0
 ; CHECK-SD-NEXT:    cmlt v7.8h, v2.8h, #0
 ; CHECK-SD-NEXT:    cmlt v16.8h, v3.8h, #0
-; CHECK-SD-NEXT:    ldr q6, [x8, :lo12:.LCPI17_1]
-; CHECK-SD-NEXT:    adrp x8, .LCPI17_2
+; CHECK-SD-NEXT:    ldr q6, [x8, :lo12:.LCPI17_0]
+; CHECK-SD-NEXT:    adrp x8, .LCPI17_1
 ; CHECK-SD-NEXT:    ushl v4.8h, v4.8h, v6.8h
 ; CHECK-SD-NEXT:    ushl v5.8h, v5.8h, v6.8h
 ; CHECK-SD-NEXT:    ushl v7.8h, v7.8h, v6.8h
 ; CHECK-SD-NEXT:    ushl v6.8h, v16.8h, v6.8h
-; CHECK-SD-NEXT:    ldr q16, [x8, :lo12:.LCPI17_2]
-; CHECK-SD-NEXT:    adrp x8, .LCPI17_0
+; CHECK-SD-NEXT:    ldr q16, [x8, :lo12:.LCPI17_1]
+; CHECK-SD-NEXT:    adrp x8, .LCPI17_2
 ; CHECK-SD-NEXT:    add v4.8h, v0.8h, v4.8h
 ; CHECK-SD-NEXT:    add v5.8h, v1.8h, v5.8h
-; CHECK-SD-NEXT:    ldr q17, [x8, :lo12:.LCPI17_0]
 ; CHECK-SD-NEXT:    add v7.8h, v2.8h, v7.8h
 ; CHECK-SD-NEXT:    add v6.8h, v3.8h, v6.8h
-; CHECK-SD-NEXT:    adrp x8, .LCPI17_3
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v17.16b
-; CHECK-SD-NEXT:    and v1.16b, v1.16b, v17.16b
-; CHECK-SD-NEXT:    and v2.16b, v2.16b, v17.16b
 ; CHECK-SD-NEXT:    sshl v4.8h, v4.8h, v16.8h
 ; CHECK-SD-NEXT:    sshl v5.8h, v5.8h, v16.8h
-; CHECK-SD-NEXT:    and v3.16b, v3.16b, v17.16b
 ; CHECK-SD-NEXT:    sshl v7.8h, v7.8h, v16.8h
 ; CHECK-SD-NEXT:    sshl v6.8h, v6.8h, v16.8h
-; CHECK-SD-NEXT:    ldr q16, [x8, :lo12:.LCPI17_3]
-; CHECK-SD-NEXT:    and v4.16b, v4.16b, v16.16b
-; CHECK-SD-NEXT:    and v5.16b, v5.16b, v16.16b
-; CHECK-SD-NEXT:    and v7.16b, v7.16b, v16.16b
-; CHECK-SD-NEXT:    and v6.16b, v6.16b, v16.16b
-; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v4.16b
-; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v5.16b
-; CHECK-SD-NEXT:    orr v2.16b, v2.16b, v7.16b
-; CHECK-SD-NEXT:    orr v3.16b, v3.16b, v6.16b
+; CHECK-SD-NEXT:    ldr q16, [x8, :lo12:.LCPI17_2]
+; CHECK-SD-NEXT:    bif v0.16b, v4.16b, v16.16b
+; CHECK-SD-NEXT:    bif v1.16b, v5.16b, v16.16b
+; CHECK-SD-NEXT:    bif v2.16b, v7.16b, v16.16b
+; CHECK-SD-NEXT:    bif v3.16b, v6.16b, v16.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: combine_vec_sdiv_by_pow2b_v32i16:
@@ -904,29 +881,21 @@ define <16 x i8> @non_splat_minus_one_divisor_0(<16 x i8> %A) {
 define <16 x i8> @non_splat_minus_one_divisor_1(<16 x i8> %A) {
 ; CHECK-SD-LABEL: non_splat_minus_one_divisor_1:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    adrp x8, .LCPI26_1
+; CHECK-SD-NEXT:    adrp x8, .LCPI26_0
 ; CHECK-SD-NEXT:    cmlt v1.16b, v0.16b, #0
-; CHECK-SD-NEXT:    adrp x9, .LCPI26_3
+; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI26_0]
+; CHECK-SD-NEXT:    adrp x8, .LCPI26_1
+; CHECK-SD-NEXT:    ushl v1.16b, v1.16b, v2.16b
 ; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI26_1]
 ; CHECK-SD-NEXT:    adrp x8, .LCPI26_2
-; CHECK-SD-NEXT:    ldr q3, [x9, :lo12:.LCPI26_3]
-; CHECK-SD-NEXT:    adrp x9, .LCPI26_5
-; CHECK-SD-NEXT:    ushl v1.16b, v1.16b, v2.16b
-; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI26_2]
-; CHECK-SD-NEXT:    adrp x8, .LCPI26_0
 ; CHECK-SD-NEXT:    add v1.16b, v0.16b, v1.16b
 ; CHECK-SD-NEXT:    sshl v1.16b, v1.16b, v2.16b
-; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI26_0]
-; CHECK-SD-NEXT:    adrp x8, .LCPI26_4
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI26_4]
-; CHECK-SD-NEXT:    and v1.16b, v1.16b, v3.16b
-; CHECK-SD-NEXT:    ldr q3, [x9, :lo12:.LCPI26_5]
-; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI26_2]
+; CHECK-SD-NEXT:    adrp x8, .LCPI26_3
+; CHECK-SD-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    ldr q2, [x8, :lo12:.LCPI26_3]
 ; CHECK-SD-NEXT:    neg v1.16b, v0.16b
-; CHECK-SD-NEXT:    and v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT:    and v1.16b, v1.16b, v3.16b
-; CHECK-SD-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    bit v0.16b, v1.16b, v2.16b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: non_splat_minus_one_divisor_1:
@@ -1464,7 +1433,7 @@ define i32 @combine_i32_sdiv_pow2(i32 %x) {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    add w8, w0, #15
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel w8, w8, w0, lt
+; CHECK-SD-NEXT:    csel w8, w8, w0, mi
 ; CHECK-SD-NEXT:    asr w0, w8, #4
 ; CHECK-SD-NEXT:    ret
 ;
@@ -1483,7 +1452,7 @@ define i32 @combine_i32_sdiv_negpow2(i32 %x) {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    add w8, w0, #255
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel w8, w8, w0, lt
+; CHECK-SD-NEXT:    csel w8, w8, w0, mi
 ; CHECK-SD-NEXT:    neg w0, w8, asr #8
 ; CHECK-SD-NEXT:    ret
 ;
@@ -1502,7 +1471,7 @@ define i64 @combine_i64_sdiv_pow2(i64 %x) {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    add x8, x0, #15
 ; CHECK-SD-NEXT:    cmp x0, #0
-; CHECK-SD-NEXT:    csel x8, x8, x0, lt
+; CHECK-SD-NEXT:    csel x8, x8, x0, mi
 ; CHECK-SD-NEXT:    asr x0, x8, #4
 ; CHECK-SD-NEXT:    ret
 ;
@@ -1521,7 +1490,7 @@ define i64 @combine_i64_sdiv_negpow2(i64 %x) {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    add x8, x0, #255
 ; CHECK-SD-NEXT:    cmp x0, #0
-; CHECK-SD-NEXT:    csel x8, x8, x0, lt
+; CHECK-SD-NEXT:    csel x8, x8, x0, mi
 ; CHECK-SD-NEXT:    neg x0, x8, asr #8
 ; CHECK-SD-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
index 0646ca4948e1d..4f00aed3aa4bc 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
@@ -378,6 +378,116 @@ middle.block:                                     ; preds = %vector.body
 }
 
 
+; Zero initialized double reduction
+;   struct foo {
+;     complex<double> v1, v2;
+;   };
+;
+;complex<double> foo(struct foo *a, struct foo *b, int n) {
+;   complex<double> x = 0.0 + 0.0i;
+;   complex<double> y = 0.0 + 0.0i;
+;   for (int i = 0; i < n; i++) {
+;       struct foo t1 = a[i];
+;       struct foo t2 = b[i];
+;       x += t1.v1 * t2.v1;
+;       y += t1.v2 * t2.v2;
+;   }
+;   return x + y;
+;}
+%struct.foo2 = type { %"class.std::complex", %"class.std::complex" }
+
+define %"class.std::complex" @double_complex_mul_v2f64(ptr noundef readonly captures(none) %src1, ptr noundef readonly captures(none) %src2, i64 noundef %nvec) {
+; CHECK-LABEL: double_complex_mul_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-NEXT:    cntd x8
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    movi v3.2d, #0000000000000000
+; CHECK-NEXT:    rdvl x9, #4
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:  .LBB4_1: // %vector.body
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ld4d { z4.d - z7.d }, p0/z, [x0]
+; CHECK-NEXT:    subs x2, x2, x8
+; CHECK-NEXT:    add x0, x0, x9
+; CHECK-NEXT:    ld4d { z16.d - z19.d }, p0/z, [x1]
+; CHECK-NEXT:    add x1, x1, x9
+; CHECK-NEXT:    fmla z2.d, p0/m, z16.d, z4.d
+; CHECK-NEXT:    fmla z0.d, p0/m, z16.d, z5.d
+; CHECK-NEXT:    fmla z3.d, p0/m, z18.d, z6.d
+; CHECK-NEXT:    fmla z1.d, p0/m, z18.d, z7.d
+; CHECK-NEXT:    fmls z2.d, p0/m, z5.d, z17.d
+; CHECK-NEXT:    fmla z0.d, p0/m, z17.d, z4.d
+; CHECK-NEXT:    fmls z3.d, p0/m, z7.d, z19.d
+; CHECK-NEXT:    fmla z1.d, p0/m, z19.d, z6.d
+; CHECK-NEXT:    b.ne .LBB4_1
+; CHECK-NEXT:  // %bb.2: // %middle.block
+; CHECK-NEXT:    fadd z2.d, z2.d, z3.d
+; CHECK-NEXT:    fadd z1.d, z0.d, z1.d
+; CHECK-NEXT:    faddv d0, p0, z2.d
+; CHECK-NEXT:    faddv d1, p0, z1.d
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $z1
+; CHECK-NEXT:    ret
+entry:
+  %vscale = tail call i64 @llvm.vscale.i64()
+  %inc = shl nuw nsw i64 %vscale, 1
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi1 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %vec.phi1.next, %vector.body ]
+  %vec.phi2 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %vec.phi2.next, %vector.body ]
+  %vec.phi3 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %vec.phi3.next, %vector.body ]
+  %vec.phi4 = phi <vscale x 2 x double> [ zeroinitializer, %entry ], [ %vec.phi4.next, %vector.body ]
+  %gep1 = getelementptr inbounds nuw %struct.foo2, ptr %src1, i64 %index
+  %wide.vec = load <vscale x 8 x double>, ptr %gep1, align 8
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave4.nxv8f64(<vscale x 8 x double> %wide.vec)
+  %ext00 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %ext01 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %ext02 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 2
+  %ext03 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 3
+  %gep2 = getelementptr inbounds nuw %struct.foo2, ptr %src2, i64 %index
+  %wide.vec73 = load <vscale x 8 x double>, ptr %gep2, align 8
+  %strided.vec74 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave4.nxv8f64(<vscale x 8 x double> %wide.vec73)
+  %ext10 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec74, 0
+  %ext11 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec74, 1
+  %ext12 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec74, 2
+  %ext13 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec74, 3
+  %fmul1 = fmul fast <vscale x 2 x double> %ext10, %ext00
+  %fmul2 = fmul fast <vscale x 2 x double> %ext11, %ext00
+  %fmul3 = fmul fast <vscale x 2 x double> %ext10, %ext01
+  %fadd1 = fadd fast <vscale x 2 x double> %fmul1, %vec.phi2
+  %fmul4 = fmul fast <vscale x 2 x double> %ext01, %ext11
+  %vec.phi2.next = fsub fast <vscale x 2 x double> %fadd1, %fmul4
+  %fadd2 = fadd fast <vscale x 2 x double> %fmul3, %vec.phi1
+  %vec.phi1.next = fadd fast <vscale x 2 x double> %fadd2, %fmul2
+  %fmul5 = fmul fast <vscale x 2 x double> %ext12, %ext02
+  %fmul6 = fmul fast <vscale x 2 x double> %ext13, %ext02
+  %fmul7 = fmul fast <vscale x 2 x double> %ext12, %ext03
+  %fadd3 = fadd fast <vscale x 2 x double> %fmul5, %vec.phi4
+  %fmul8 = fmul fast <vscale x 2 x double> %ext03, %ext13
+  %vec.phi4.next = fsub fast <vscale x 2 x double> %fadd3, %fmul8
+  %fadd4 = fadd fast <vscale x 2 x double> %fmul7, %vec.phi3
+  %vec.phi3.next = fadd fast <vscale x 2 x double> %fadd4, %fmul6
+  %index.next = add nuw i64 %index, %inc
+  %cmp = icmp eq i64 %index.next, %nvec
+  br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+  %final1 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %vec.phi1.next)
+  %final2 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %vec.phi2.next)
+  %final3 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %vec.phi3.next)
+  %final4 = tail call fast double @llvm.vector.reduce.fadd.nxv2f64(double 0.000000e+00, <vscale x 2 x double> %vec.phi4.next)
+  %last_fadd1 = fadd fast double %final2, %final4
+  %last_fadd2 = fadd fast double %final1, %final3
+  %.fca.0.0.insert = insertvalue %"class.std::complex" poison, double %last_fadd1, 0, 0
+  %.fca.0.1.insert = insertvalue %"class.std::complex" %.fca.0.0.insert, double %last_fadd2, 0, 1
+  ret %"class.std::complex" %.fca.0.1.insert
+}
+
+
 declare i64 @llvm.vscale.i64()
 declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double>)
 declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-symmetric-fixed.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-symmetric-fixed.ll
new file mode 100644
index 0000000000000..d05b9c6d7662a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-symmetric-fixed.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+complxnum,+sve,+fullfp16 -o - | FileCheck %s
+
+target triple = "aarch64"
+
+define <4 x double> @simple_symmetric_muladd2(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: simple_symmetric_muladd2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, #-7378697629483820647 // =0x9999999999999999
+; CHECK-NEXT:    movk x8, #39322
+; CHECK-NEXT:    movk x8, #16393, lsl #48
+; CHECK-NEXT:    dup v4.2d, x8
+; CHECK-NEXT:    fmla v2.2d, v4.2d, v0.2d
+; CHECK-NEXT:    fmla v3.2d, v4.2d, v1.2d
+; CHECK-NEXT:    mov v0.16b, v2.16b
+; CHECK-NEXT:    mov v1.16b, v3.16b
+; CHECK-NEXT:    ret
+entry:
+  %ext00 = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %ext01 = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %fmul0 = fmul fast <2 x double> %ext00, splat (double 3.200000e+00)
+  %ext10 = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 0, i32 2>
+  %ext11 = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 1, i32 3>
+  %fadd0 = fadd fast <2 x double> %ext10, %fmul0
+  %fmul1 = fmul fast <2 x double> %ext01, splat (double 3.200000e+00)
+  %fadd1 = fadd fast <2 x double> %ext11, %fmul1
+  %interleaved.vec = shufflevector <2 x double> %fadd0, <2 x double> %fadd1, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+  ret <4 x double> %interleaved.vec
+}
+
+define <8 x double> @simple_symmetric_muladd4(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: simple_symmetric_muladd4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov x8, #-7378697629483820647 // =0x9999999999999999
+; CHECK-NEXT:    zip1 v16.2d, v0.2d, v2.2d
+; CHECK-NEXT:    zip2 v0.2d, v0.2d, v2.2d
+; CHECK-NEXT:    movk x8, #39322
+; CHECK-NEXT:    zip1 v2.2d, v1.2d, v3.2d
+; CHECK-NEXT:    zip2 v1.2d, v1.2d, v3.2d
+; CHECK-NEXT:    movk x8, #16393, lsl #48
+; CHECK-NEXT:    zip1 v3.2d, v4.2d, v6.2d
+; CHECK-NEXT:    zip2 v4.2d, v4.2d, v6.2d
+; CHECK-NEXT:    zip1 v17.2d, v5.2d, v7.2d
+; CHECK-NEXT:    zip2 v5.2d, v5.2d, v7.2d
+; CHECK-NEXT:    dup v6.2d, x8
+; CHECK-NEXT:    fmla v3.2d, v6.2d, v16.2d
+; CHECK-NEXT:    fmla v4.2d, v6.2d, v0.2d
+; CHECK-NEXT:    fmla v17.2d, v6.2d, v2.2d
+; CHECK-NEXT:    fmla v5.2d, v6.2d, v1.2d
+; CHECK-NEXT:    zip1 v0.2d, v3.2d, v4.2d
+; CHECK-NEXT:    zip2 v2.2d, v3.2d, v4.2d
+; CHECK-NEXT:    zip1 v1.2d, v17.2d, v5.2d
+; CHECK-NEXT:    zip2 v3.2d, v17.2d, v5.2d
+; CHECK-NEXT:    ret
+entry:
+  %ext00 = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 0, i32 4>
+  %ext01 = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 1, i32 5>
+  %ext02 = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 2, i32 6>
+  %ext03 = shufflevector <8 x double> %a, <8 x double> poison, <2 x i32> <i32 3, i32 7>
+  %fmul0 = fmul fast <2 x double> %ext00, splat (double 3.200000e+00)
+  %ext10 = shufflevector <8 x double> %b, <8 x double> poison, <2 x i32> <i32 0, i32 4>
+  %ext11 = shufflevector <8 x double> %b, <8 x double> poison, <2 x i32> <i32 1, i32 5>
+  %ext12 = shufflevector <8 x double> %b, <8 x double> poison, <2 x i32> <i32 2, i32 6>
+  %ext13 = shufflevector <8 x double> %b, <8 x double> poison, <2 x i32> <i32 3, i32 7>
+  %fadd0 = fadd fast <2 x double> %ext10, %fmul0
+  %fmul1 = fmul fast <2 x double> %ext01, splat (double 3.200000e+00)
+  %fadd1 = fadd fast <2 x double> %ext11, %fmul1
+  %fmul2 = fmul fast <2 x double> %ext02, splat (double 3.200000e+00)
+  %fadd2 = fadd fast <2 x double> %ext12, %fmul2
+  %fmul3 = fmul fast <2 x double> %ext03, splat (double 3.200000e+00)
+  %fadd3 = fadd fast <2 x double> %ext13, %fmul3
+  %interleave.pt1 = shufflevector <2 x double> %fadd0, <2 x double> %fadd1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %interleave.pt2 = shufflevector <2 x double> %fadd2, <2 x double> %fadd3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %interleaved.vec = shufflevector <4 x double> %interleave.pt1, <4 x double> %interleave.pt2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
+  ret <8 x double> %interleaved.vec
+}
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-symmetric-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-symmetric-scalable.ll
new file mode 100644
index 0000000000000..cd1c0c55dc04c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-symmetric-scalable.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s --mattr=+complxnum,+sve,+fullfp16 -o - | FileCheck %s
+
+target triple = "aarch64"
+
+define <vscale x 4 x double> @simple_symmetric_muladd2(<vscale x 4 x double> %a, <vscale x 4 x double> %b) {
+; CHECK-LABEL: simple_symmetric_muladd2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    adrp x8, .LCPI0_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI0_0
+; CHECK-NEXT:    ld1rd { z4.d }, p0/z, [x8]
+; CHECK-NEXT:    fmad z0.d, p0/m, z4.d, z2.d
+; CHECK-NEXT:    fmad z1.d, p0/m, z4.d, z3.d
+; CHECK-NEXT:    ret
+entry:
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %a)
+  %ext00 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %ext01 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %fmul0 = fmul fast <vscale x 2 x double> %ext00, splat (double 3.200000e+00)
+  %strided.vec44 = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %b)
+  %ext10 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec44, 0
+  %ext11 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec44, 1
+  %fadd0 = fadd fast <vscale x 2 x double> %ext10, %fmul0
+  %fmul1 = fmul fast <vscale x 2 x double> %ext01, splat (double 3.200000e+00)
+  %fadd1 = fadd fast <vscale x 2 x double> %ext11, %fmul1
+  %interleaved.vec = tail call <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x double> %fadd0, <vscale x 2 x double> %fadd1)
+  ret <vscale x 4 x double> %interleaved.vec
+}
+
+define <vscale x 4 x double> @simple_symmetric_unary2(<vscale x 4 x double> %a) {
+; CHECK-LABEL: simple_symmetric_unary2:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fneg z0.d, p0/m, z0.d
+; CHECK-NEXT:    fneg z1.d, p0/m, z1.d
+; CHECK-NEXT:    ret
+entry:
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %a)
+  %ext00 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %ext01 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %fneg0 = fneg fast <vscale x 2 x double> %ext00
+  %fneg1 = fneg fast <vscale x 2 x double> %ext01
+  %interleaved.vec = tail call <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x double> %fneg0, <vscale x 2 x double> %fneg1)
+  ret <vscale x 4 x double> %interleaved.vec
+}
+
+define <vscale x 8 x double> @simple_symmetric_muladd4(<vscale x 8 x double> %a, <vscale x 8 x double> %b) {
+; CHECK-LABEL: simple_symmetric_muladd4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    adrp x8, .LCPI2_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI2_0
+; CHECK-NEXT:    ld1rd { z24.d }, p0/z, [x8]
+; CHECK-NEXT:    fmad z0.d, p0/m, z24.d, z4.d
+; CHECK-NEXT:    fmad z1.d, p0/m, z24.d, z5.d
+; CHECK-NEXT:    fmad z2.d, p0/m, z24.d, z6.d
+; CHECK-NEXT:    fmad z3.d, p0/m, z24.d, z7.d
+; CHECK-NEXT:    ret
+entry:
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave4.nxv8f64(<vscale x 8 x double> %a)
+  %ext00 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %ext01 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %ext02 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 2
+  %ext03 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 3
+  %fmul0 = fmul fast <vscale x 2 x double> %ext00, splat (double 3.200000e+00)
+  %strided.vec44 = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave4.nxv8f64(<vscale x 8 x double> %b)
+  %ext10 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec44, 0
+  %ext11 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec44, 1
+  %ext12 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec44, 2
+  %ext13 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec44, 3
+  %fadd0 = fadd fast <vscale x 2 x double> %ext10, %fmul0
+  %fmul1 = fmul fast <vscale x 2 x double> %ext01, splat (double 3.200000e+00)
+  %fadd1 = fadd fast <vscale x 2 x double> %ext11, %fmul1
+  %fmul2 = fmul fast <vscale x 2 x double> %ext02, splat (double 3.200000e+00)
+  %fadd2 = fadd fast <vscale x 2 x double> %ext12, %fmul2
+  %fmul3 = fmul fast <vscale x 2 x double> %ext03, splat (double 3.200000e+00)
+  %fadd3 = fadd fast <vscale x 2 x double> %ext13, %fmul3
+  %interleaved.vec = tail call <vscale x 8 x double> @llvm.vector.interleave4.nxv8f64(<vscale x 2 x double> %fadd0, <vscale x 2 x double> %fadd1, <vscale x 2 x double> %fadd2, <vscale x 2 x double> %fadd3)
+  ret <vscale x 8 x double> %interleaved.vec
+}
+
+
+define <vscale x 8 x double> @simple_symmetric_unary4(<vscale x 8 x double> %a) {
+; CHECK-LABEL: simple_symmetric_unary4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fneg z0.d, p0/m, z0.d
+; CHECK-NEXT:    fneg z1.d, p0/m, z1.d
+; CHECK-NEXT:    fneg z2.d, p0/m, z2.d
+; CHECK-NEXT:    fneg z3.d, p0/m, z3.d
+; CHECK-NEXT:    ret
+entry:
+  %strided.vec = tail call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave4.nxv8f64(<vscale x 8 x double> %a)
+  %ext00 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 0
+  %ext01 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 1
+  %ext02 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 2
+  %ext03 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec, 3
+  %fneg0 = fneg fast <vscale x 2 x double> %ext00
+  %fneg1 = fneg fast <vscale x 2 x double> %ext01
+  %fneg2 = fneg fast <vscale x 2 x double> %ext02
+  %fneg3 = fneg fast <vscale x 2 x double> %ext03
+  %interleaved.vec = tail call <vscale x 8 x double> @llvm.vector.interleave4.nxv8f64(<vscale x 2 x double> %fneg0, <vscale x 2 x double> %fneg1, <vscale x 2 x double> %fneg2, <vscale x 2 x double> %fneg3)
+  ret <vscale x 8 x double> %interleaved.vec
+}
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
index 7686740aec302..13434fabefa78 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
@@ -203,89 +203,93 @@ define <12 x float> @abp90c12(<12 x float> %a, <12 x float> %b, <12 x float> %c)
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    // kill: def $s1 killed $s1 def $q1
 ; CHECK-NEXT:    // kill: def $s3 killed $s3 def $q3
-; CHECK-NEXT:    ldr s17, [sp, #40]
-; CHECK-NEXT:    add x10, sp, #56
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT:    // kill: def $s2 killed $s2 def $q2
+; CHECK-NEXT:    ldr s17, [sp, #32]
+; CHECK-NEXT:    // kill: def $s5 killed $s5 def $q5
 ; CHECK-NEXT:    add x9, sp, #48
+; CHECK-NEXT:    add x10, sp, #64
 ; CHECK-NEXT:    mov v1.s[1], v3.s[0]
-; CHECK-NEXT:    ldr s3, [sp, #32]
-; CHECK-NEXT:    // kill: def $s2 killed $s2 def $q2
 ; CHECK-NEXT:    mov v0.s[1], v2.s[0]
-; CHECK-NEXT:    ld1 { v17.s }[1], [x10]
-; CHECK-NEXT:    // kill: def $s5 killed $s5 def $q5
-; CHECK-NEXT:    ldr s16, [sp, #8]
 ; CHECK-NEXT:    // kill: def $s4 killed $s4 def $q4
-; CHECK-NEXT:    add x10, sp, #24
-; CHECK-NEXT:    ld1 { v3.s }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #72
-; CHECK-NEXT:    // kill: def $s7 killed $s7 def $q7
+; CHECK-NEXT:    add x11, sp, #72
+; CHECK-NEXT:    ld1 { v17.s }[1], [x9]
+; CHECK-NEXT:    ldr s18, [x10]
+; CHECK-NEXT:    add x9, sp, #80
+; CHECK-NEXT:    add x10, sp, #56
 ; CHECK-NEXT:    // kill: def $s6 killed $s6 def $q6
+; CHECK-NEXT:    // kill: def $s7 killed $s7 def $q7
+; CHECK-NEXT:    ldr s16, [sp, #8]
+; CHECK-NEXT:    ldr s3, [sp, #96]
+; CHECK-NEXT:    ld1 { v18.s }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #88
 ; CHECK-NEXT:    ldr s2, [sp]
-; CHECK-NEXT:    ld1 { v16.s }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #112
-; CHECK-NEXT:    ldr s20, [sp, #136]
 ; CHECK-NEXT:    mov v1.s[2], v5.s[0]
-; CHECK-NEXT:    ld1 { v17.s }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #64
-; CHECK-NEXT:    ldr s5, [sp, #96]
-; CHECK-NEXT:    ld1 { v3.s }[2], [x9]
+; CHECK-NEXT:    ldr s5, [sp, #40]
 ; CHECK-NEXT:    mov v0.s[2], v4.s[0]
-; CHECK-NEXT:    add x9, sp, #88
-; CHECK-NEXT:    ldr s4, [sp, #104]
-; CHECK-NEXT:    ldr s19, [sp, #192]
 ; CHECK-NEXT:    ld1 { v5.s }[1], [x10]
-; CHECK-NEXT:    add x10, sp, #80
-; CHECK-NEXT:    ld1 { v17.s }[3], [x9]
-; CHECK-NEXT:    mov v1.s[3], v7.s[0]
-; CHECK-NEXT:    add x9, sp, #120
-; CHECK-NEXT:    ld1 { v3.s }[3], [x10]
-; CHECK-NEXT:    ld1 { v4.s }[1], [x9]
-; CHECK-NEXT:    ldr s7, [sp, #128]
+; CHECK-NEXT:    ldr s19, [x11]
 ; CHECK-NEXT:    add x10, sp, #144
+; CHECK-NEXT:    zip1 v4.2d, v17.2d, v18.2d
+; CHECK-NEXT:    add x11, sp, #160
+; CHECK-NEXT:    ldr s18, [sp, #136]
+; CHECK-NEXT:    ld1 { v19.s }[1], [x9]
 ; CHECK-NEXT:    mov v0.s[3], v6.s[0]
-; CHECK-NEXT:    add x9, sp, #16
+; CHECK-NEXT:    ldr s6, [sp, #128]
+; CHECK-NEXT:    mov v1.s[3], v7.s[0]
+; CHECK-NEXT:    add x9, sp, #24
+; CHECK-NEXT:    ldr s7, [sp, #104]
+; CHECK-NEXT:    ld1 { v16.s }[1], [x9]
+; CHECK-NEXT:    add x9, sp, #112
+; CHECK-NEXT:    ld1 { v6.s }[1], [x10]
+; CHECK-NEXT:    zip1 v5.2d, v5.2d, v19.2d
+; CHECK-NEXT:    add x10, sp, #120
+; CHECK-NEXT:    ld1 { v3.s }[1], [x9]
 ; CHECK-NEXT:    ld1 { v7.s }[1], [x10]
-; CHECK-NEXT:    ld1 { v2.s }[1], [x9]
-; CHECK-NEXT:    add x9, sp, #160
-; CHECK-NEXT:    fmul v6.4s, v17.4s, v1.4s
-; CHECK-NEXT:    fmul v18.4s, v4.4s, v16.4s
-; CHECK-NEXT:    fmul v16.4s, v5.4s, v16.4s
-; CHECK-NEXT:    fmul v1.4s, v3.4s, v1.4s
-; CHECK-NEXT:    add x10, sp, #208
-; CHECK-NEXT:    ld1 { v7.s }[2], [x9]
-; CHECK-NEXT:    add x9, sp, #152
-; CHECK-NEXT:    ld1 { v19.s }[1], [x10]
-; CHECK-NEXT:    ld1 { v20.s }[1], [x9]
+; CHECK-NEXT:    ldr s17, [x11]
 ; CHECK-NEXT:    add x9, sp, #176
-; CHECK-NEXT:    add x10, sp, #184
-; CHECK-NEXT:    fneg v6.4s, v6.4s
-; CHECK-NEXT:    fneg v18.4s, v18.4s
-; CHECK-NEXT:    fmla v16.4s, v2.4s, v4.4s
-; CHECK-NEXT:    fmla v1.4s, v0.4s, v17.4s
-; CHECK-NEXT:    ld1 { v7.s }[3], [x9]
-; CHECK-NEXT:    add x9, sp, #168
-; CHECK-NEXT:    ld1 { v20.s }[2], [x9]
-; CHECK-NEXT:    ldr s4, [sp, #200]
+; CHECK-NEXT:    add x10, sp, #16
+; CHECK-NEXT:    add x11, sp, #168
+; CHECK-NEXT:    ld1 { v17.s }[1], [x9]
+; CHECK-NEXT:    ld1 { v2.s }[1], [x10]
+; CHECK-NEXT:    add x9, sp, #152
+; CHECK-NEXT:    fmul v19.4s, v5.4s, v1.4s
+; CHECK-NEXT:    fmul v20.4s, v7.4s, v16.4s
+; CHECK-NEXT:    fmul v16.4s, v3.4s, v16.4s
+; CHECK-NEXT:    fmul v1.4s, v4.4s, v1.4s
+; CHECK-NEXT:    ld1 { v18.s }[1], [x9]
+; CHECK-NEXT:    ldr s21, [x11]
+; CHECK-NEXT:    zip1 v6.2d, v6.2d, v17.2d
+; CHECK-NEXT:    ldr s17, [sp, #192]
+; CHECK-NEXT:    add x9, sp, #184
+; CHECK-NEXT:    add x10, sp, #208
+; CHECK-NEXT:    ld1 { v21.s }[1], [x9]
 ; CHECK-NEXT:    add x9, sp, #216
-; CHECK-NEXT:    fmla v6.4s, v0.4s, v3.4s
-; CHECK-NEXT:    fmla v18.4s, v2.4s, v5.4s
-; CHECK-NEXT:    ld1 { v4.s }[1], [x9]
-; CHECK-NEXT:    fsub v0.4s, v7.4s, v1.4s
-; CHECK-NEXT:    fsub v1.4s, v19.4s, v16.4s
-; CHECK-NEXT:    ld1 { v20.s }[3], [x10]
-; CHECK-NEXT:    fadd v2.4s, v4.4s, v18.4s
-; CHECK-NEXT:    fadd v3.4s, v20.4s, v6.4s
+; CHECK-NEXT:    fneg v19.4s, v19.4s
+; CHECK-NEXT:    fneg v20.4s, v20.4s
+; CHECK-NEXT:    fmla v16.4s, v2.4s, v7.4s
+; CHECK-NEXT:    fmla v1.4s, v0.4s, v5.4s
+; CHECK-NEXT:    ld1 { v17.s }[1], [x10]
+; CHECK-NEXT:    ldr s5, [sp, #200]
+; CHECK-NEXT:    zip1 v7.2d, v18.2d, v21.2d
+; CHECK-NEXT:    ld1 { v5.s }[1], [x9]
+; CHECK-NEXT:    fmla v19.4s, v0.4s, v4.4s
+; CHECK-NEXT:    fmla v20.4s, v2.4s, v3.4s
+; CHECK-NEXT:    fsub v0.4s, v6.4s, v1.4s
+; CHECK-NEXT:    fsub v1.4s, v17.4s, v16.4s
+; CHECK-NEXT:    fadd v2.4s, v7.4s, v19.4s
+; CHECK-NEXT:    fadd v3.4s, v5.4s, v20.4s
 ; CHECK-NEXT:    ext v4.16b, v0.16b, v1.16b, #12
-; CHECK-NEXT:    ext v5.16b, v3.16b, v2.16b, #12
-; CHECK-NEXT:    trn2 v1.4s, v1.4s, v2.4s
+; CHECK-NEXT:    ext v5.16b, v2.16b, v3.16b, #12
+; CHECK-NEXT:    trn2 v1.4s, v1.4s, v3.4s
 ; CHECK-NEXT:    ext v4.16b, v0.16b, v4.16b, #12
-; CHECK-NEXT:    ext v5.16b, v3.16b, v5.16b, #8
+; CHECK-NEXT:    ext v5.16b, v2.16b, v5.16b, #8
 ; CHECK-NEXT:    rev64 v4.4s, v4.4s
-; CHECK-NEXT:    trn2 v2.4s, v4.4s, v5.4s
-; CHECK-NEXT:    zip2 v4.4s, v0.4s, v3.4s
-; CHECK-NEXT:    zip1 v0.4s, v0.4s, v3.4s
-; CHECK-NEXT:    ext v1.16b, v2.16b, v1.16b, #8
-; CHECK-NEXT:    mov v4.d[1], v2.d[0]
+; CHECK-NEXT:    trn2 v3.4s, v4.4s, v5.4s
+; CHECK-NEXT:    zip2 v4.4s, v0.4s, v2.4s
+; CHECK-NEXT:    zip1 v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    ext v1.16b, v3.16b, v1.16b, #8
+; CHECK-NEXT:    mov v4.d[1], v3.d[0]
 ; CHECK-NEXT:    str q0, [x8]
 ; CHECK-NEXT:    stp q4, q1, [x8, #16]
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/concat-vector.ll b/llvm/test/CodeGen/AArch64/concat-vector.ll
index acf15f1bd1178..e6f27b95d92c8 100644
--- a/llvm/test/CodeGen/AArch64/concat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/concat-vector.ll
@@ -186,8 +186,9 @@ define <16 x i8> @concat_v16s8_v4s8_load(ptr %ptrA, ptr %ptrB, ptr %ptrC, ptr %p
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    ld1 { v0.s }[1], [x1]
-; CHECK-NEXT:    ld1 { v0.s }[2], [x2]
-; CHECK-NEXT:    ld1 { v0.s }[3], [x3]
+; CHECK-NEXT:    ldr s1, [x2]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x3]
+; CHECK-NEXT:    zip1 v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
     %A = load <4 x i8>, ptr %ptrA
     %B = load <4 x i8>, ptr %ptrB
diff --git a/llvm/test/CodeGen/AArch64/csel-cmp-cse.ll b/llvm/test/CodeGen/AArch64/csel-cmp-cse.ll
index e745326323329..07b32ef0830c2 100644
--- a/llvm/test/CodeGen/AArch64/csel-cmp-cse.ll
+++ b/llvm/test/CodeGen/AArch64/csel-cmp-cse.ll
@@ -706,7 +706,7 @@ define i32 @test_ugtsmax_sub_add_i32(i32 %x0, i32 %x1) {
 ; CHECK-NEXT:    add w9, w0, w1
 ; CHECK-NEXT:    cmp w1, #0
 ; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    csel w0, wzr, w8, lt
+; CHECK-NEXT:    csel w0, wzr, w8, mi
 ; CHECK-NEXT:    ret
   %cmp = icmp ugt i32 %x1, 2147483647
   %add = add i32 %x0, %x1
diff --git a/llvm/test/CodeGen/AArch64/dont-shrink-wrap-stack-mayloadorstore.mir b/llvm/test/CodeGen/AArch64/dont-shrink-wrap-stack-mayloadorstore.mir
index b8fd76681c5eb..b912a346c1b66 100644
--- a/llvm/test/CodeGen/AArch64/dont-shrink-wrap-stack-mayloadorstore.mir
+++ b/llvm/test/CodeGen/AArch64/dont-shrink-wrap-stack-mayloadorstore.mir
@@ -7,17 +7,23 @@
  ; RUN: llc -x=mir -simplify-mir -passes='shrink-wrap' -o - %s | FileCheck %s
  ; CHECK:      name:            compiler_pop_stack
  ; CHECK:      frameInfo:
- ; CHECK:      savePoint:       '%bb.1'
- ; CHECK:      restorePoint:    '%bb.7'
+ ; CHECK:        savePoint:
+ ; CHECK-NEXT:     - point:           '%bb.1'
+ ; CHECK:        restorePoint:
+ ; CHECK-NEXT:     - point:           '%bb.7'
  ; CHECK:      name:            compiler_pop_stack_no_memoperands
  ; CHECK:      frameInfo:
- ; CHECK:      savePoint:       '%bb.1'
- ; CHECK:      restorePoint:    '%bb.7'
+ ; CHECK:        savePoint:
+ ; CHECK-NEXT:     - point:           '%bb.1'
+ ; CHECK:        restorePoint:
+ ; CHECK-NEXT:     - point:           '%bb.7'
  ; CHECK:      name:            f
  ; CHECK:      frameInfo:
- ; CHECK:      savePoint:       '%bb.2'
- ; CHECK-NEXT: restorePoint:    '%bb.4'
- ; CHECK-NEXT: stack:
+ ; CHECK:        savePoint:
+ ; CHECK-NEXT:     - point:           '%bb.2'
+ ; CHECK:        restorePoint:
+ ; CHECK-NEXT:     - point:           '%bb.4'
+ ; CHECK:      stack:
 
   target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
   target triple = "aarch64"
diff --git a/llvm/test/CodeGen/AArch64/early-ifcvt-regclass-mismatch.mir b/llvm/test/CodeGen/AArch64/early-ifcvt-regclass-mismatch.mir
index a7f67f8b682c3..1d0d02c0a1d5a 100644
--- a/llvm/test/CodeGen/AArch64/early-ifcvt-regclass-mismatch.mir
+++ b/llvm/test/CodeGen/AArch64/early-ifcvt-regclass-mismatch.mir
@@ -105,8 +105,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
index 615accc7f75e5..37e40f62dd0de 100644
--- a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
+++ b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
@@ -75,8 +75,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     true
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir b/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir
index d61a278d00220..0418ece0839be 100644
--- a/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir
+++ b/llvm/test/CodeGen/AArch64/expand-blr-rvmarker-pseudo.mir
@@ -43,7 +43,7 @@
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $x0, implicit $sp, implicit $xzr, implicit $fp {
+# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $sp, implicit-def dead $x0, implicit-def $fp, implicit $x0, implicit $sp, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BLR $x0, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -66,7 +66,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $sp, implicit $x0, implicit $xzr, implicit $fp {
+# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $sp, implicit-def dead $x0, implicit-def $fp, implicit $sp, implicit $x0, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BL @foo, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -86,7 +86,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def $x0, implicit-def $w0, implicit-def $w0_hi, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $xzr, implicit $fp {
+# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $sp, implicit-def $x0, implicit-def $fp, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BL @foo, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -106,7 +106,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $sp, implicit $w0, implicit $w1, implicit $xzr, implicit $fp {
+# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $sp, implicit-def dead $x0, implicit-def $fp, implicit $sp, implicit $w0, implicit $w1, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BL @foo, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $w0, implicit $w1, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -127,7 +127,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $x8, implicit $sp, implicit $w0, implicit $w1, implicit $xzr, implicit $fp {
+# CHECK-NEXT:     BUNDLE implicit-def $lr, implicit-def $sp, implicit-def dead $x0, implicit-def $fp, implicit $x8, implicit $sp, implicit $w0, implicit $w1, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BLR $x8, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $w0, implicit $w1, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
@@ -149,7 +149,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit-def $fp, implicit-def $w29, implicit-def $w29_hi, implicit $sp, implicit undef $x0, implicit $xzr, implicit $fp {
+# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $sp, implicit-def dead $x0, implicit-def $fp, implicit $sp, implicit undef $x0, implicit $xzr, implicit $fp {
 # CHECK-NEXT:      BL @foo, csr_darwin_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit undef $x0, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      $fp = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT:      BL @objc_retainAutoreleasedReturnValue, implicit-def $lr, implicit internal $sp
@@ -170,7 +170,7 @@ body:             |
 # CHECK:       bb.0:
 # CHECK-NEXT: liveins:
 # CHECK-NEXT: {{  $}}
-# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $w30, implicit-def $w30_hi, implicit-def $sp, implicit-def $wsp, implicit-def $wsp_hi, implicit-def dead $x0, implicit $x0, implicit $sp {
+# CHECK-NEXT:    BUNDLE implicit-def $lr, implicit-def $sp, implicit-def dead $x0, implicit $x0, implicit $sp {
 # CHECK-NEXT:      BLR $x0, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def dead $x0
 # CHECK-NEXT:      BL @attachedcall, implicit-def $lr, implicit internal $sp
 # CHECK-NEXT:   }
diff --git a/llvm/test/CodeGen/AArch64/expand-constructive-zzi.mir b/llvm/test/CodeGen/AArch64/expand-constructive-zzi.mir
new file mode 100644
index 0000000000000..55287712b12c3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/expand-constructive-zzi.mir
@@ -0,0 +1,61 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=aarch64 -mattr=+sve -run-pass=aarch64-expand-pseudo -verify-machineinstrs %s -o - | FileCheck %s
+
+# Test the expansion of constructive binary operations into their
+# destructive counterparts.
+
+
+# EXT_ZZI_CONSTRUCTIVE
+
+---
+name:            test_ext_zzi_unique
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_ext_zzi_unique
+    ; CHECK: BUNDLE implicit-def $z2, implicit killed $z0 {
+    ; CHECK-NEXT:   $z2 = MOVPRFX_ZZ $z0
+    ; CHECK-NEXT:   $z2 = EXT_ZZI internal killed $z2, killed $z0, 1
+    ; CHECK-NEXT: }
+    ; CHECK-NEXT: RET undef $lr, implicit killed $z2
+    $z2 = EXT_ZZI_CONSTRUCTIVE killed $z0, 1
+    RET_ReallyLR implicit killed $z2
+...
+
+---
+name:            test_ext_zzi_already_destructive
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_ext_zzi_already_destructive
+    ; CHECK: $z2 = EXT_ZZI killed $z2, killed $z2, 1
+    ; CHECK-NEXT: RET undef $lr, implicit killed $z2
+    $z2 = EXT_ZZI_CONSTRUCTIVE killed $z2, 1
+    RET_ReallyLR implicit killed $z2
+...
+
+---
+name:            test_ext_zzi_unique_implicit_ops
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_ext_zzi_unique_implicit_ops
+    ; CHECK: BUNDLE implicit-def $z2, implicit-def $q2, implicit killed $z0, implicit killed $q0 {
+    ; CHECK-NEXT:   $z2 = MOVPRFX_ZZ $z0, implicit killed $q0
+    ; CHECK-NEXT:   $z2 = EXT_ZZI internal killed $z2, killed $z0, 1, implicit-def $q2
+    ; CHECK-NEXT: }
+    ; CHECK-NEXT: RET undef $lr, implicit killed $q2
+    $z2 = EXT_ZZI_CONSTRUCTIVE killed $z0, 1, implicit-def $q2, implicit killed $q0
+    RET_ReallyLR implicit killed $q2
+...
+
+---
+name:            test_ext_zzi_undef
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_ext_zzi_undef
+    ; CHECK: BUNDLE implicit-def $z2, implicit undef $z0 {
+    ; CHECK-NEXT:   $z2 = MOVPRFX_ZZ undef $z0
+    ; CHECK-NEXT:   $z2 = EXT_ZZI internal killed $z2, undef $z0, 1
+    ; CHECK-NEXT: }
+    ; CHECK-NEXT: RET undef $lr, implicit killed $z2
+    $z2 = EXT_ZZI_CONSTRUCTIVE undef $z0, 1
+    RET_ReallyLR implicit killed $z2
+...
diff --git a/llvm/test/CodeGen/AArch64/expand-sme-pseudos.mir b/llvm/test/CodeGen/AArch64/expand-sme-pseudos.mir
new file mode 100644
index 0000000000000..6ca9b9b6cb200
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/expand-sme-pseudos.mir
@@ -0,0 +1,101 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -run-pass=aarch64-expand-pseudo -mtriple=aarch64-unknown-linux-gnu -o - %s | FileCheck %s
+
+---
+
+# X0 = TPIDR2 block pointer
+# X8 = TPIDR2_EL0
+name:            restore_za_save
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: restore_za_save
+    ; CHECK: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x0 = IMPLICIT_DEF
+    ; CHECK-NEXT: $x8 = MRS 56965, implicit-def $nzcv
+    ; CHECK-NEXT: CBZX $x8, %bb.1
+    ; CHECK-NEXT: B %bb.2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: .1:
+    ; CHECK-NEXT: successors: %bb.2(0x80000000)
+    ; CHECK-NEXT: liveins: $x8, $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: BL &__arm_tpidr2_restore, csr_aarch64_sme_abi_support_routines_preservemost_from_x0, implicit-def $lr, implicit $sp, implicit $x0
+    ; CHECK-NEXT: B %bb.2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: .2:
+    ; CHECK-NEXT: RET undef $lr
+    $x0 = IMPLICIT_DEF
+    $x8 = MRS 56965, implicit-def $nzcv
+
+    RestoreZAPseudo $x8, $x0, &__arm_tpidr2_restore, csr_aarch64_sme_abi_support_routines_preservemost_from_x0
+
+    RET_ReallyLR
+
+...
+---
+
+# X8 = TPIDR2_EL0
+name:            commit_za_save
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: commit_za_save
+    ; CHECK: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x8 = MRS 56965, implicit-def $nzcv
+    ; CHECK-NEXT: CBNZX $x8, %bb.1
+    ; CHECK-NEXT: B %bb.2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: .1:
+    ; CHECK-NEXT: successors: %bb.2(0x80000000)
+    ; CHECK-NEXT: liveins: $x8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: BL &__arm_tpidr2_save, csr_aarch64_sme_abi_support_routines_preservemost_from_x0, implicit-def $lr, implicit $sp
+    ; CHECK-NEXT: MSR 56965, $xzr
+    ; CHECK-NEXT: B %bb.2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: .2:
+    ; CHECK-NEXT: RET undef $lr
+    $x8 = MRS 56965, implicit-def $nzcv
+
+    CommitZASavePseudo $x8, 0, &__arm_tpidr2_save, csr_aarch64_sme_abi_support_routines_preservemost_from_x0
+
+    RET_ReallyLR
+
+...
+---
+# X8 = TPIDR2_EL0
+name:            commit_za_save_zero_za
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: commit_za_save_zero_za
+    ; CHECK: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x8 = MRS 56965, implicit-def $nzcv
+    ; CHECK-NEXT: CBNZX $x8, %bb.1
+    ; CHECK-NEXT: B %bb.2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: .1:
+    ; CHECK-NEXT: successors: %bb.2(0x80000000)
+    ; CHECK-NEXT: liveins: $x8
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: BL &__arm_tpidr2_save, csr_aarch64_sme_abi_support_routines_preservemost_from_x0, implicit-def $lr, implicit $sp, implicit-def $zab0
+    ; CHECK-NEXT: MSR 56965, $xzr
+    ; CHECK-NEXT: ZERO_M 255, implicit-def $zab0
+    ; CHECK-NEXT: B %bb.2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: .2:
+    ; CHECK-NEXT: RET undef $lr
+    $x8 = MRS 56965, implicit-def $nzcv
+
+    CommitZASavePseudo $x8, 1, &__arm_tpidr2_save, csr_aarch64_sme_abi_support_routines_preservemost_from_x0, implicit-def $zab0
+
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/fadd-combines.ll b/llvm/test/CodeGen/AArch64/fadd-combines.ll
index f7bf92888cd37..76d29a50416e3 100644
--- a/llvm/test/CodeGen/AArch64/fadd-combines.ll
+++ b/llvm/test/CodeGen/AArch64/fadd-combines.ll
@@ -1,12 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -verify-machineinstrs -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define double @test1(double %a, double %b) {
-; CHECK-LABEL: test1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd d1, d1, d1
-; CHECK-NEXT:    fsub d0, d0, d1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test1:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd d1, d1, d1
+; CHECK-SD-NEXT:    fsub d0, d0, d1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov d2, #-2.00000000
+; CHECK-GI-NEXT:    fmul d1, d1, d2
+; CHECK-GI-NEXT:    fadd d0, d0, d1
+; CHECK-GI-NEXT:    ret
   %mul = fmul double %b, -2.000000e+00
   %add1 = fadd double %a, %mul
   ret double %add1
@@ -15,23 +23,38 @@ define double @test1(double %a, double %b) {
 ; DAGCombine will canonicalize 'a - 2.0*b' to 'a + -2.0*b'
 
 define double @test2(double %a, double %b) {
-; CHECK-LABEL: test2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd d1, d1, d1
-; CHECK-NEXT:    fsub d0, d0, d1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test2:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd d1, d1, d1
+; CHECK-SD-NEXT:    fsub d0, d0, d1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test2:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov d2, #2.00000000
+; CHECK-GI-NEXT:    fmul d1, d1, d2
+; CHECK-GI-NEXT:    fsub d0, d0, d1
+; CHECK-GI-NEXT:    ret
   %mul = fmul double %b, 2.000000e+00
   %add1 = fsub double %a, %mul
   ret double %add1
 }
 
 define double @test3(double %a, double %b, double %c) {
-; CHECK-LABEL: test3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmul d0, d0, d1
-; CHECK-NEXT:    fadd d1, d2, d2
-; CHECK-NEXT:    fsub d0, d0, d1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test3:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmul d0, d0, d1
+; CHECK-SD-NEXT:    fadd d1, d2, d2
+; CHECK-SD-NEXT:    fsub d0, d0, d1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test3:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov d3, #2.00000000
+; CHECK-GI-NEXT:    fmul d0, d0, d1
+; CHECK-GI-NEXT:    fmul d1, d2, d3
+; CHECK-GI-NEXT:    fsub d0, d0, d1
+; CHECK-GI-NEXT:    ret
   %mul = fmul double %a, %b
   %mul1 = fmul double %c, 2.000000e+00
   %sub = fsub double %mul, %mul1
@@ -39,12 +62,20 @@ define double @test3(double %a, double %b, double %c) {
 }
 
 define double @test4(double %a, double %b, double %c) {
-; CHECK-LABEL: test4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmul d0, d0, d1
-; CHECK-NEXT:    fadd d1, d2, d2
-; CHECK-NEXT:    fsub d0, d0, d1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test4:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmul d0, d0, d1
+; CHECK-SD-NEXT:    fadd d1, d2, d2
+; CHECK-SD-NEXT:    fsub d0, d0, d1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test4:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov d3, #-2.00000000
+; CHECK-GI-NEXT:    fmul d0, d0, d1
+; CHECK-GI-NEXT:    fmul d1, d2, d3
+; CHECK-GI-NEXT:    fadd d0, d0, d1
+; CHECK-GI-NEXT:    ret
   %mul = fmul double %a, %b
   %mul1 = fmul double %c, -2.000000e+00
   %add2 = fadd double %mul, %mul1
@@ -52,55 +83,95 @@ define double @test4(double %a, double %b, double %c) {
 }
 
 define <4 x float> @fmulnegtwo_vec(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fmulnegtwo_vec:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v1.4s, v1.4s, v1.4s
-; CHECK-NEXT:    fsub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fmulnegtwo_vec:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    fsub v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmulnegtwo_vec:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov s2, #-2.00000000
+; CHECK-GI-NEXT:    fmul v1.4s, v1.4s, v2.s[0]
+; CHECK-GI-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    ret
   %mul = fmul <4 x float> %b, <float -2.0, float -2.0, float -2.0, float -2.0>
   %add = fadd <4 x float> %a, %mul
   ret <4 x float> %add
 }
 
 define <4 x float> @fmulnegtwo_vec_commute(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fmulnegtwo_vec_commute:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v1.4s, v1.4s, v1.4s
-; CHECK-NEXT:    fsub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fmulnegtwo_vec_commute:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    fsub v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmulnegtwo_vec_commute:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov s2, #-2.00000000
+; CHECK-GI-NEXT:    fmul v1.4s, v1.4s, v2.s[0]
+; CHECK-GI-NEXT:    fadd v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    ret
   %mul = fmul <4 x float> %b, <float -2.0, float -2.0, float -2.0, float -2.0>
   %add = fadd <4 x float> %mul, %a
   ret <4 x float> %add
 }
 
 define <4 x float> @fmulnegtwo_vec_undefs(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fmulnegtwo_vec_undefs:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v1.4s, v1.4s, v1.4s
-; CHECK-NEXT:    fsub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fmulnegtwo_vec_undefs:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    fsub v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmulnegtwo_vec_undefs:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov s2, #-2.00000000
+; CHECK-GI-NEXT:    mov v3.s[1], v2.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v2.s[0]
+; CHECK-GI-NEXT:    fmul v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    ret
   %mul = fmul <4 x float> %b, <float undef, float -2.0, float undef, float -2.0>
   %add = fadd <4 x float> %a, %mul
   ret <4 x float> %add
 }
 
 define <4 x float> @fmulnegtwo_vec_commute_undefs(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: fmulnegtwo_vec_commute_undefs:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v1.4s, v1.4s, v1.4s
-; CHECK-NEXT:    fsub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fmulnegtwo_vec_commute_undefs:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    fsub v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmulnegtwo_vec_commute_undefs:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov s2, #-2.00000000
+; CHECK-GI-NEXT:    fmov s3, #-2.00000000
+; CHECK-GI-NEXT:    mov v3.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v3.s[3], v2.s[0]
+; CHECK-GI-NEXT:    fmul v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    fadd v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT:    ret
   %mul = fmul <4 x float> %b, <float -2.0, float undef, float -2.0, float -2.0>
   %add = fadd <4 x float> %mul, %a
   ret <4 x float> %add
 }
 
 define <4 x float> @test6(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: test6:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v1.4s, v1.4s, v1.4s
-; CHECK-NEXT:    fsub v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test6:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v1.4s, v1.4s, v1.4s
+; CHECK-SD-NEXT:    fsub v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test6:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov s2, #2.00000000
+; CHECK-GI-NEXT:    fmul v1.4s, v1.4s, v2.s[0]
+; CHECK-GI-NEXT:    fsub v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    ret
   %mul = fmul <4 x float> %b, <float 2.0, float 2.0, float 2.0, float 2.0>
   %add = fsub <4 x float> %a, %mul
   ret <4 x float> %add
@@ -130,16 +201,26 @@ define double @test7(double %a, double %b) nounwind {
 }
 
 define float @fadd_const_multiuse_fmf(float %x) {
-; CHECK-LABEL: fadd_const_multiuse_fmf:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1109917696 // =0x42280000
-; CHECK-NEXT:    mov w9, #1114374144 // =0x426c0000
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fadd s1, s0, s1
-; CHECK-NEXT:    fadd s0, s0, s2
-; CHECK-NEXT:    fadd s0, s1, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_const_multiuse_fmf:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    mov w8, #1109917696 // =0x42280000
+; CHECK-SD-NEXT:    mov w9, #1114374144 // =0x426c0000
+; CHECK-SD-NEXT:    fmov s1, w8
+; CHECK-SD-NEXT:    fmov s2, w9
+; CHECK-SD-NEXT:    fadd s1, s0, s1
+; CHECK-SD-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_const_multiuse_fmf:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov w8, #1109917696 // =0x42280000
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    fmov s1, #17.00000000
+; CHECK-GI-NEXT:    fadd s1, s0, s1
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %a1 = fadd float %x, 42.0
   %a2 = fadd nsz reassoc float %a1, 17.0
   %a3 = fadd float %a1, %a2
@@ -148,16 +229,26 @@ define float @fadd_const_multiuse_fmf(float %x) {
 
 ; DAGCombiner transforms this into: (x + 17.0) + (x + 59.0).
 define float @fadd_const_multiuse_attr(float %x) {
-; CHECK-LABEL: fadd_const_multiuse_attr:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1109917696 // =0x42280000
-; CHECK-NEXT:    mov w9, #1114374144 // =0x426c0000
-; CHECK-NEXT:    fmov s1, w8
-; CHECK-NEXT:    fmov s2, w9
-; CHECK-NEXT:    fadd s1, s0, s1
-; CHECK-NEXT:    fadd s0, s0, s2
-; CHECK-NEXT:    fadd s0, s1, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_const_multiuse_attr:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    mov w8, #1109917696 // =0x42280000
+; CHECK-SD-NEXT:    mov w9, #1114374144 // =0x426c0000
+; CHECK-SD-NEXT:    fmov s1, w8
+; CHECK-SD-NEXT:    fmov s2, w9
+; CHECK-SD-NEXT:    fadd s1, s0, s1
+; CHECK-SD-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_const_multiuse_attr:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov w8, #1109917696 // =0x42280000
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    fmov s1, #17.00000000
+; CHECK-GI-NEXT:    fadd s1, s0, s1
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %a1 = fadd fast float %x, 42.0
   %a2 = fadd fast float %a1, 17.0
   %a3 = fadd fast float %a1, %a2
@@ -167,12 +258,20 @@ define float @fadd_const_multiuse_attr(float %x) {
 ; PR32939 - https://bugs.llvm.org/show_bug.cgi?id=32939
 
 define double @fmul2_negated(double %a, double %b, double %c) {
-; CHECK-LABEL: fmul2_negated:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd d1, d1, d1
-; CHECK-NEXT:    fmul d1, d1, d2
-; CHECK-NEXT:    fsub d0, d0, d1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fmul2_negated:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd d1, d1, d1
+; CHECK-SD-NEXT:    fmul d1, d1, d2
+; CHECK-SD-NEXT:    fsub d0, d0, d1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmul2_negated:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov d3, #2.00000000
+; CHECK-GI-NEXT:    fmul d1, d1, d3
+; CHECK-GI-NEXT:    fmul d1, d1, d2
+; CHECK-GI-NEXT:    fsub d0, d0, d1
+; CHECK-GI-NEXT:    ret
   %mul = fmul double %b, 2.0
   %mul1 = fmul double %mul, %c
   %sub = fsub double %a, %mul1
@@ -180,12 +279,20 @@ define double @fmul2_negated(double %a, double %b, double %c) {
 }
 
 define <2 x double> @fmul2_negated_vec(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
-; CHECK-LABEL: fmul2_negated_vec:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v1.2d, v1.2d, v1.2d
-; CHECK-NEXT:    fmul v1.2d, v1.2d, v2.2d
-; CHECK-NEXT:    fsub v0.2d, v0.2d, v1.2d
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fmul2_negated_vec:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v1.2d, v1.2d, v1.2d
+; CHECK-SD-NEXT:    fmul v1.2d, v1.2d, v2.2d
+; CHECK-SD-NEXT:    fsub v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmul2_negated_vec:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmov d3, #2.00000000
+; CHECK-GI-NEXT:    fmul v1.2d, v1.2d, v3.d[0]
+; CHECK-GI-NEXT:    fmul v1.2d, v1.2d, v2.2d
+; CHECK-GI-NEXT:    fsub v0.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    ret
   %mul = fmul <2 x double> %b, <double 2.0, double 2.0>
   %mul1 = fmul <2 x double> %mul, %c
   %sub = fsub <2 x double> %a, %mul1
@@ -195,11 +302,18 @@ define <2 x double> @fmul2_negated_vec(<2 x double> %a, <2 x double> %b, <2 x do
 ; ((a*b) + (c*d)) + n1 --> (a*b) + ((c*d) + n1)
 
 define double @fadd_fma_fmul_1(double %a, double %b, double %c, double %d, double %n1) nounwind {
-; CHECK-LABEL: fadd_fma_fmul_1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmadd d2, d2, d3, d4
-; CHECK-NEXT:    fmadd d0, d0, d1, d2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_fma_fmul_1:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmadd d2, d2, d3, d4
+; CHECK-SD-NEXT:    fmadd d0, d0, d1, d2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_fma_fmul_1:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmul d2, d2, d3
+; CHECK-GI-NEXT:    fmadd d0, d0, d1, d2
+; CHECK-GI-NEXT:    fadd d0, d0, d4
+; CHECK-GI-NEXT:    ret
   %m1 = fmul fast double %a, %b
   %m2 = fmul fast double %c, %d
   %a1 = fadd fast double %m1, %m2
@@ -212,11 +326,18 @@ define double @fadd_fma_fmul_1(double %a, double %b, double %c, double %d, doubl
 ; requires reassociation to fuse with c*d.
 
 define float @fadd_fma_fmul_fmf(float %a, float %b, float %c, float %d, float %n0) nounwind {
-; CHECK-LABEL: fadd_fma_fmul_fmf:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmadd s2, s2, s3, s4
-; CHECK-NEXT:    fmadd s0, s0, s1, s2
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_fma_fmul_fmf:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmadd s2, s2, s3, s4
+; CHECK-SD-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_fma_fmul_fmf:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmul s2, s2, s3
+; CHECK-GI-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-GI-NEXT:    fadd s0, s4, s0
+; CHECK-GI-NEXT:    ret
   %m1 = fmul contract float %a, %b
   %m2 = fmul contract float %c, %d
   %a1 = fadd contract float %m1, %m2
@@ -243,14 +364,23 @@ define float @fadd_fma_fmul_2(float %a, float %b, float %c, float %d, float %n0)
 ; The final fadd can be folded with either 1 of the leading fmuls.
 
 define <2 x double> @fadd_fma_fmul_3(<2 x double> %x1, <2 x double> %x2, <2 x double> %x3, <2 x double> %x4, <2 x double> %x5, <2 x double> %x6, <2 x double> %x7, <2 x double> %x8) nounwind {
-; CHECK-LABEL: fadd_fma_fmul_3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmul v2.2d, v2.2d, v3.2d
-; CHECK-NEXT:    fmla v2.2d, v1.2d, v0.2d
-; CHECK-NEXT:    fmla v2.2d, v7.2d, v6.2d
-; CHECK-NEXT:    fmla v2.2d, v5.2d, v4.2d
-; CHECK-NEXT:    mov v0.16b, v2.16b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fadd_fma_fmul_3:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmul v2.2d, v2.2d, v3.2d
+; CHECK-SD-NEXT:    fmla v2.2d, v1.2d, v0.2d
+; CHECK-SD-NEXT:    fmla v2.2d, v7.2d, v6.2d
+; CHECK-SD-NEXT:    fmla v2.2d, v5.2d, v4.2d
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fadd_fma_fmul_3:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmul v2.2d, v2.2d, v3.2d
+; CHECK-GI-NEXT:    fmul v3.2d, v6.2d, v7.2d
+; CHECK-GI-NEXT:    fmla v2.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    fmla v3.2d, v4.2d, v5.2d
+; CHECK-GI-NEXT:    fadd v0.2d, v2.2d, v3.2d
+; CHECK-GI-NEXT:    ret
   %m1 = fmul fast <2 x double> %x1, %x2
   %m2 = fmul fast <2 x double> %x3, %x4
   %m3 = fmul fast <2 x double> %x5, %x6
@@ -316,12 +446,20 @@ define float @fadd_fma_fmul_extra_use_3(float %a, float %b, float %c, float %d,
 }
 
 define float @fmac_sequence_innermost_fmul(float %a, float %b, float %c, float %d, float %e, float %f, float %g) {
-; CHECK-LABEL: fmac_sequence_innermost_fmul:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmadd s0, s0, s1, s6
-; CHECK-NEXT:    fmadd s0, s2, s3, s0
-; CHECK-NEXT:    fmadd s0, s4, s5, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fmac_sequence_innermost_fmul:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmadd s0, s0, s1, s6
+; CHECK-SD-NEXT:    fmadd s0, s2, s3, s0
+; CHECK-SD-NEXT:    fmadd s0, s4, s5, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmac_sequence_innermost_fmul:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmul s2, s2, s3
+; CHECK-GI-NEXT:    fmadd s0, s0, s1, s2
+; CHECK-GI-NEXT:    fmadd s0, s4, s5, s0
+; CHECK-GI-NEXT:    fadd s0, s0, s6
+; CHECK-GI-NEXT:    ret
   %t0 = fmul float %a, %b
   %t1 = fmul contract float %c, %d
   %t2 = fadd contract float %t0, %t1
@@ -332,12 +470,20 @@ define float @fmac_sequence_innermost_fmul(float %a, float %b, float %c, float %
 }
 
 define float @fmac_sequence_innermost_fmul_intrinsics(float %a, float %b, float %c, float %d, float %e, float %f, float %g) {
-; CHECK-LABEL: fmac_sequence_innermost_fmul_intrinsics:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fmadd s0, s0, s1, s6
-; CHECK-NEXT:    fmadd s0, s2, s3, s0
-; CHECK-NEXT:    fmadd s0, s4, s5, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: fmac_sequence_innermost_fmul_intrinsics:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fmadd s0, s0, s1, s6
+; CHECK-SD-NEXT:    fmadd s0, s2, s3, s0
+; CHECK-SD-NEXT:    fmadd s0, s4, s5, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: fmac_sequence_innermost_fmul_intrinsics:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fmul s0, s0, s1
+; CHECK-GI-NEXT:    fmadd s0, s2, s3, s0
+; CHECK-GI-NEXT:    fmadd s0, s4, s5, s0
+; CHECK-GI-NEXT:    fadd s0, s0, s6
+; CHECK-GI-NEXT:    ret
   %t0 = fmul float %a, %b
   %t1 = call float @llvm.fma.f32(float %c, float %d, float %t0)
   %t2 = call float @llvm.fma.f32(float %e, float %f, float %t1)
@@ -349,3 +495,19 @@ declare float @llvm.fma.f32(float, float, float)
 
 declare void @use(double)
 
+
+define float @faddvf32_zero_nsz(float %a) {
+; CHECK-LABEL: faddvf32_zero_nsz:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %b = fadd nsz float %a, 0.0
+  ret float %b
+}
+
+define <2 x double> @faddv2f64_zero_nsz(<2 x double> %a) {
+; CHECK-LABEL: faddv2f64_zero_nsz:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %b = fadd nsz  <2 x double> %a, zeroinitializer
+  ret <2 x double> %b
+}
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-sdiv.ll b/llvm/test/CodeGen/AArch64/fast-isel-sdiv.ll
index 539fe7e7d3c83..7b6780d0e1e1f 100644
--- a/llvm/test/CodeGen/AArch64/fast-isel-sdiv.ll
+++ b/llvm/test/CodeGen/AArch64/fast-isel-sdiv.ll
@@ -1,12 +1,17 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64-linux-gnu                               -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
-; RUN: llc -mtriple=aarch64-linux-gnu -fast-isel -fast-isel-abort=1 -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
+; RUN: llc -mtriple=aarch64-linux-gnu -fast-isel -fast-isel-abort=1 -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-FAST-ISEL
 
 define i32 @sdiv_i32_exact(i32 %a) {
 ; CHECK-LABEL: sdiv_i32_exact:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr w0, w0, #3
 ; CHECK-NEXT:    ret
+;
+; CHECK-FAST-ISEL-LABEL: sdiv_i32_exact:
+; CHECK-FAST-ISEL:       // %bb.0:
+; CHECK-FAST-ISEL-NEXT:    asr w0, w0, #3
+; CHECK-FAST-ISEL-NEXT:    ret
   %1 = sdiv exact i32 %a, 8
   ret i32 %1
 }
@@ -16,9 +21,17 @@ define i32 @sdiv_i32_pos(i32 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add w8, w0, #7
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    csel w8, w8, w0, lt
+; CHECK-NEXT:    csel w8, w8, w0, mi
 ; CHECK-NEXT:    asr w0, w8, #3
 ; CHECK-NEXT:    ret
+;
+; CHECK-FAST-ISEL-LABEL: sdiv_i32_pos:
+; CHECK-FAST-ISEL:       // %bb.0:
+; CHECK-FAST-ISEL-NEXT:    add w8, w0, #7
+; CHECK-FAST-ISEL-NEXT:    cmp w0, #0
+; CHECK-FAST-ISEL-NEXT:    csel w8, w8, w0, lt
+; CHECK-FAST-ISEL-NEXT:    asr w0, w8, #3
+; CHECK-FAST-ISEL-NEXT:    ret
   %1 = sdiv i32 %a, 8
   ret i32 %1
 }
@@ -28,9 +41,17 @@ define i32 @sdiv_i32_neg(i32 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add w8, w0, #7
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    csel w8, w8, w0, lt
+; CHECK-NEXT:    csel w8, w8, w0, mi
 ; CHECK-NEXT:    neg w0, w8, asr #3
 ; CHECK-NEXT:    ret
+;
+; CHECK-FAST-ISEL-LABEL: sdiv_i32_neg:
+; CHECK-FAST-ISEL:       // %bb.0:
+; CHECK-FAST-ISEL-NEXT:    add w8, w0, #7
+; CHECK-FAST-ISEL-NEXT:    cmp w0, #0
+; CHECK-FAST-ISEL-NEXT:    csel w8, w8, w0, lt
+; CHECK-FAST-ISEL-NEXT:    neg w0, w8, asr #3
+; CHECK-FAST-ISEL-NEXT:    ret
   %1 = sdiv i32 %a, -8
   ret i32 %1
 }
@@ -40,6 +61,11 @@ define i64 @sdiv_i64_exact(i64 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    asr x0, x0, #4
 ; CHECK-NEXT:    ret
+;
+; CHECK-FAST-ISEL-LABEL: sdiv_i64_exact:
+; CHECK-FAST-ISEL:       // %bb.0:
+; CHECK-FAST-ISEL-NEXT:    asr x0, x0, #4
+; CHECK-FAST-ISEL-NEXT:    ret
   %1 = sdiv exact i64 %a, 16
   ret i64 %1
 }
@@ -49,9 +75,17 @@ define i64 @sdiv_i64_pos(i64 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add x8, x0, #15
 ; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    csel x8, x8, x0, lt
+; CHECK-NEXT:    csel x8, x8, x0, mi
 ; CHECK-NEXT:    asr x0, x8, #4
 ; CHECK-NEXT:    ret
+;
+; CHECK-FAST-ISEL-LABEL: sdiv_i64_pos:
+; CHECK-FAST-ISEL:       // %bb.0:
+; CHECK-FAST-ISEL-NEXT:    add x8, x0, #15
+; CHECK-FAST-ISEL-NEXT:    cmp x0, #0
+; CHECK-FAST-ISEL-NEXT:    csel x8, x8, x0, lt
+; CHECK-FAST-ISEL-NEXT:    asr x0, x8, #4
+; CHECK-FAST-ISEL-NEXT:    ret
   %1 = sdiv i64 %a, 16
   ret i64 %1
 }
@@ -61,9 +95,17 @@ define i64 @sdiv_i64_neg(i64 %a) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add x8, x0, #15
 ; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    csel x8, x8, x0, lt
+; CHECK-NEXT:    csel x8, x8, x0, mi
 ; CHECK-NEXT:    neg x0, x8, asr #4
 ; CHECK-NEXT:    ret
+;
+; CHECK-FAST-ISEL-LABEL: sdiv_i64_neg:
+; CHECK-FAST-ISEL:       // %bb.0:
+; CHECK-FAST-ISEL-NEXT:    add x8, x0, #15
+; CHECK-FAST-ISEL-NEXT:    cmp x0, #0
+; CHECK-FAST-ISEL-NEXT:    csel x8, x8, x0, lt
+; CHECK-FAST-ISEL-NEXT:    neg x0, x8, asr #4
+; CHECK-FAST-ISEL-NEXT:    ret
   %1 = sdiv i64 %a, -16
   ret i64 %1
 }
diff --git a/llvm/test/CodeGen/AArch64/fcmp-fp128.ll b/llvm/test/CodeGen/AArch64/fcmp-fp128.ll
index 503cb8c533bab..a2b4b61864741 100644
--- a/llvm/test/CodeGen/AArch64/fcmp-fp128.ll
+++ b/llvm/test/CodeGen/AArch64/fcmp-fp128.ll
@@ -98,7 +98,7 @@ define double @olt(fp128 %a, fp128 %b, double %d, double %e) {
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    fcsel d0, d9, d8, lt
+; CHECK-SD-NEXT:    fcsel d0, d9, d8, mi
 ; CHECK-SD-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ret
 ;
@@ -115,7 +115,7 @@ define double @olt(fp128 %a, fp128 %b, double %d, double %e) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    fcsel d0, d8, d9, lt
+; CHECK-GI-NEXT:    fcsel d0, d8, d9, mi
 ; CHECK-GI-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -412,7 +412,7 @@ define double @uge(fp128 %a, fp128 %b, double %d, double %e) {
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    fcsel d0, d9, d8, ge
+; CHECK-SD-NEXT:    fcsel d0, d9, d8, pl
 ; CHECK-SD-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ret
 ;
@@ -429,7 +429,7 @@ define double @uge(fp128 %a, fp128 %b, double %d, double %e) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    fcsel d0, d8, d9, ge
+; CHECK-GI-NEXT:    fcsel d0, d8, d9, pl
 ; CHECK-GI-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -452,7 +452,7 @@ define double @ult(fp128 %a, fp128 %b, double %d, double %e) {
 ; CHECK-SD-NEXT:    bl __getf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    fcsel d0, d9, d8, lt
+; CHECK-SD-NEXT:    fcsel d0, d9, d8, mi
 ; CHECK-SD-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ret
 ;
@@ -469,7 +469,7 @@ define double @ult(fp128 %a, fp128 %b, double %d, double %e) {
 ; CHECK-GI-NEXT:    bl __getf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    fcsel d0, d8, d9, lt
+; CHECK-GI-NEXT:    fcsel d0, d8, d9, mi
 ; CHECK-GI-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/fcmp.ll b/llvm/test/CodeGen/AArch64/fcmp.ll
index f33f57e8751ca..6d673f1204c7f 100644
--- a/llvm/test/CodeGen/AArch64/fcmp.ll
+++ b/llvm/test/CodeGen/AArch64/fcmp.ll
@@ -15,7 +15,7 @@ define fp128 @f128_fp128(fp128 %a, fp128 %b, fp128 %d, fp128 %e) {
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    b.ge .LBB0_2
+; CHECK-SD-NEXT:    b.pl .LBB0_2
 ; CHECK-SD-NEXT:  // %bb.1: // %entry
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:  .LBB0_2: // %entry
@@ -36,9 +36,9 @@ define fp128 @f128_fp128(fp128 %a, fp128 %b, fp128 %d, fp128 %e) {
 ; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    mov d0, v2.d[1]
 ; CHECK-GI-NEXT:    mov d1, v3.d[1]
-; CHECK-GI-NEXT:    fcsel d2, d2, d3, lt
+; CHECK-GI-NEXT:    fcsel d2, d2, d3, mi
 ; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    fcsel d1, d0, d1, lt
+; CHECK-GI-NEXT:    fcsel d1, d0, d1, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
 ; CHECK-GI-NEXT:    fmov x8, d1
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
@@ -71,13 +71,13 @@ define i128 @f128_i128(fp128 %a, fp128 %b, i128 %d, i128 %e) {
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel x20, x22, x20, lt
+; CHECK-SD-NEXT:    csel x20, x22, x20, mi
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    mov w8, w0
 ; CHECK-SD-NEXT:    mov x0, x20
 ; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w8, #0
-; CHECK-SD-NEXT:    csel x1, x21, x19, lt
+; CHECK-SD-NEXT:    csel x1, x21, x19, mi
 ; CHECK-SD-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    add sp, sp, #80
@@ -100,8 +100,8 @@ define i128 @f128_i128(fp128 %a, fp128 %b, i128 %d, i128 %e) {
 ; CHECK-GI-NEXT:    mov x22, x3
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x0, x19, x21, lt
-; CHECK-GI-NEXT:    csel x1, x20, x22, lt
+; CHECK-GI-NEXT:    csel x0, x19, x21, mi
+; CHECK-GI-NEXT:    csel x1, x20, x22, mi
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldr x30, [sp], #48 // 8-byte Folded Reload
@@ -126,7 +126,7 @@ define double @f128_double(fp128 %a, fp128 %b, double %d, double %e) {
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    fcsel d0, d9, d8, lt
+; CHECK-SD-NEXT:    fcsel d0, d9, d8, mi
 ; CHECK-SD-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ret
 ;
@@ -143,7 +143,7 @@ define double @f128_double(fp128 %a, fp128 %b, double %d, double %e) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    fcsel d0, d8, d9, lt
+; CHECK-GI-NEXT:    fcsel d0, d8, d9, mi
 ; CHECK-GI-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -166,7 +166,7 @@ define float @f128_float(fp128 %a, fp128 %b, float %d, float %e) {
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    fcsel s0, s9, s8, lt
+; CHECK-SD-NEXT:    fcsel s0, s9, s8, mi
 ; CHECK-SD-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ret
 ;
@@ -183,7 +183,7 @@ define float @f128_float(fp128 %a, fp128 %b, float %d, float %e) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    fcsel s0, s8, s9, lt
+; CHECK-GI-NEXT:    fcsel s0, s8, s9, mi
 ; CHECK-GI-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
 entry:
@@ -205,7 +205,7 @@ define i32 @f128_i32(fp128 %a, fp128 %b, i32 %d, i32 %e) {
 ; CHECK-SD-NEXT:    mov w20, w0
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel w0, w20, w19, lt
+; CHECK-SD-NEXT:    csel w0, w20, w19, mi
 ; CHECK-SD-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-SD-NEXT:    ret
@@ -222,7 +222,7 @@ define i32 @f128_i32(fp128 %a, fp128 %b, i32 %d, i32 %e) {
 ; CHECK-GI-NEXT:    mov w20, w1
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w19, w20, lt
+; CHECK-GI-NEXT:    csel w0, w19, w20, mi
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    ret
@@ -246,7 +246,7 @@ define half @f128_half(fp128 %a, fp128 %b, half %d, half %e) {
 ; CHECK-SD-NOFP16-NEXT:    bl __lttf2
 ; CHECK-SD-NOFP16-NEXT:    cmp w0, #0
 ; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    fcsel s0, s9, s8, lt
+; CHECK-SD-NOFP16-NEXT:    fcsel s0, s9, s8, mi
 ; CHECK-SD-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-SD-NOFP16-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-SD-NOFP16-NEXT:    ret
@@ -264,7 +264,7 @@ define half @f128_half(fp128 %a, fp128 %b, half %d, half %e) {
 ; CHECK-SD-FP16-NEXT:    bl __lttf2
 ; CHECK-SD-FP16-NEXT:    cmp w0, #0
 ; CHECK-SD-FP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    fcsel h0, h9, h8, lt
+; CHECK-SD-FP16-NEXT:    fcsel h0, h9, h8, mi
 ; CHECK-SD-FP16-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
 ; CHECK-SD-FP16-NEXT:    ret
 ;
@@ -283,7 +283,7 @@ define half @f128_half(fp128 %a, fp128 %b, half %d, half %e) {
 ; CHECK-GI-NEXT:    fmov w9, s9
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    csel w8, w8, w9, lt
+; CHECK-GI-NEXT:    csel w8, w8, w9, mi
 ; CHECK-GI-NEXT:    fmov s0, w8
 ; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-GI-NEXT:    ldp d9, d8, [sp], #32 // 16-byte Folded Reload
@@ -438,7 +438,7 @@ define <2 x fp128> @v2f128_fp128(<2 x fp128> %a, <2 x fp128> %b, <2 x fp128> %d,
 ; CHECK-SD-NEXT:    stp q7, q6, [sp, #64] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    b.ge .LBB12_2
+; CHECK-SD-NEXT:    b.pl .LBB12_2
 ; CHECK-SD-NEXT:  // %bb.1: // %entry
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
@@ -447,7 +447,7 @@ define <2 x fp128> @v2f128_fp128(<2 x fp128> %a, <2 x fp128> %b, <2 x fp128> %d,
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    ldr q1, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    b.ge .LBB12_4
+; CHECK-SD-NEXT:    b.pl .LBB12_4
 ; CHECK-SD-NEXT:  // %bb.3: // %entry
 ; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:  .LBB12_4: // %entry
@@ -476,18 +476,18 @@ define <2 x fp128> @v2f128_fp128(<2 x fp128> %a, <2 x fp128> %b, <2 x fp128> %d,
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #96] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov d0, v2.d[1]
 ; CHECK-GI-NEXT:    mov d1, v3.d[1]
-; CHECK-GI-NEXT:    fcsel d2, d2, d3, lt
+; CHECK-GI-NEXT:    fcsel d2, d2, d3, mi
 ; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    fcsel d3, d0, d1, lt
+; CHECK-GI-NEXT:    fcsel d3, d0, d1, mi
 ; CHECK-GI-NEXT:    ldp q5, q0, [sp, #64] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    mov d1, v0.d[1]
 ; CHECK-GI-NEXT:    mov d4, v5.d[1]
-; CHECK-GI-NEXT:    fcsel d0, d0, d5, lt
+; CHECK-GI-NEXT:    fcsel d0, d0, d5, mi
 ; CHECK-GI-NEXT:    fmov x9, d0
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
 ; CHECK-GI-NEXT:    fmov x8, d3
-; CHECK-GI-NEXT:    fcsel d2, d1, d4, lt
+; CHECK-GI-NEXT:    fcsel d2, d1, d4, mi
 ; CHECK-GI-NEXT:    mov v1.d[0], x9
 ; CHECK-GI-NEXT:    fmov x9, d2
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
@@ -513,7 +513,7 @@ define <3 x fp128> @v3f128_fp128(<3 x fp128> %a, <3 x fp128> %b, <3 x fp128> %d,
 ; CHECK-SD-NEXT:    stp q6, q7, [sp, #64] // 32-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    b.lt .LBB13_2
+; CHECK-SD-NEXT:    b.mi .LBB13_2
 ; CHECK-SD-NEXT:  // %bb.1:
 ; CHECK-SD-NEXT:    ldr q0, [sp, #128]
 ; CHECK-SD-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
@@ -521,7 +521,7 @@ define <3 x fp128> @v3f128_fp128(<3 x fp128> %a, <3 x fp128> %b, <3 x fp128> %d,
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    b.lt .LBB13_4
+; CHECK-SD-NEXT:    b.mi .LBB13_4
 ; CHECK-SD-NEXT:  // %bb.3:
 ; CHECK-SD-NEXT:    ldr q0, [sp, #144]
 ; CHECK-SD-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
@@ -531,7 +531,7 @@ define <3 x fp128> @v3f128_fp128(<3 x fp128> %a, <3 x fp128> %b, <3 x fp128> %d,
 ; CHECK-SD-NEXT:    add x8, sp, #160
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    add x9, sp, #112
-; CHECK-SD-NEXT:    csel x8, x9, x8, lt
+; CHECK-SD-NEXT:    csel x8, x9, x8, mi
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #64] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q2, [x8]
 ; CHECK-SD-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
@@ -571,24 +571,24 @@ define <3 x fp128> @v3f128_fp128(<3 x fp128> %a, <3 x fp128> %b, <3 x fp128> %d,
 ; CHECK-GI-NEXT:    ldr x30, [sp, #160] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    mov d0, v4.d[1]
 ; CHECK-GI-NEXT:    mov d1, v5.d[1]
-; CHECK-GI-NEXT:    fcsel d4, d4, d5, lt
+; CHECK-GI-NEXT:    fcsel d4, d4, d5, mi
 ; CHECK-GI-NEXT:    mov d2, v7.d[1]
 ; CHECK-GI-NEXT:    mov d3, v6.d[1]
 ; CHECK-GI-NEXT:    fmov x8, d4
-; CHECK-GI-NEXT:    fcsel d5, d0, d1, lt
+; CHECK-GI-NEXT:    fcsel d5, d0, d1, mi
 ; CHECK-GI-NEXT:    cmp w20, #0
-; CHECK-GI-NEXT:    fcsel d1, d7, d6, lt
+; CHECK-GI-NEXT:    fcsel d1, d7, d6, mi
 ; CHECK-GI-NEXT:    ldp q7, q0, [sp, #128] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    fcsel d3, d2, d3, lt
+; CHECK-GI-NEXT:    fcsel d3, d2, d3, mi
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #176] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov d2, v0.d[1]
 ; CHECK-GI-NEXT:    mov d6, v7.d[1]
-; CHECK-GI-NEXT:    fcsel d7, d0, d7, lt
+; CHECK-GI-NEXT:    fcsel d7, d0, d7, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
 ; CHECK-GI-NEXT:    fmov x8, d1
 ; CHECK-GI-NEXT:    fmov x9, d7
-; CHECK-GI-NEXT:    fcsel d4, d2, d6, lt
+; CHECK-GI-NEXT:    fcsel d4, d2, d6, mi
 ; CHECK-GI-NEXT:    mov v1.d[0], x8
 ; CHECK-GI-NEXT:    fmov x8, d5
 ; CHECK-GI-NEXT:    mov v2.d[0], x9
@@ -621,7 +621,7 @@ define <2 x double> @v2f128_double(<2 x fp128> %a, <2 x fp128> %b, <2 x double>
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    cset w8, lt
+; CHECK-SD-NEXT:    cset w8, mi
 ; CHECK-SD-NEXT:    sbfx x8, x8, #0, #1
 ; CHECK-SD-NEXT:    fmov d0, x8
 ; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
@@ -630,7 +630,7 @@ define <2 x double> @v2f128_double(<2 x fp128> %a, <2 x fp128> %b, <2 x double>
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    cset w8, lt
+; CHECK-SD-NEXT:    cset w8, mi
 ; CHECK-SD-NEXT:    sbfx x8, x8, #0, #1
 ; CHECK-SD-NEXT:    fmov d0, x8
 ; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
@@ -652,11 +652,11 @@ define <2 x double> @v2f128_double(<2 x fp128> %a, <2 x fp128> %b, <2 x double>
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    cset w19, lt
+; CHECK-GI-NEXT:    cset w19, mi
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    fmov d0, x19
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    cset w8, lt
+; CHECK-GI-NEXT:    cset w8, mi
 ; CHECK-GI-NEXT:    ldp q2, q1, [sp, #32] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
@@ -696,7 +696,7 @@ define <3 x double> @v3f128_double(<3 x fp128> %a, <3 x fp128> %b, <3 x double>
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr q1, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    cset w8, lt
+; CHECK-SD-NEXT:    cset w8, mi
 ; CHECK-SD-NEXT:    sbfx x8, x8, #0, #1
 ; CHECK-SD-NEXT:    fmov d0, x8
 ; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
@@ -704,7 +704,7 @@ define <3 x double> @v3f128_double(<3 x fp128> %a, <3 x fp128> %b, <3 x double>
 ; CHECK-SD-NEXT:    bl __lttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    cset w8, lt
+; CHECK-SD-NEXT:    cset w8, mi
 ; CHECK-SD-NEXT:    sbfx x8, x8, #0, #1
 ; CHECK-SD-NEXT:    fmov d1, x8
 ; CHECK-SD-NEXT:    mov v1.d[1], v0.d[0]
@@ -714,7 +714,7 @@ define <3 x double> @v3f128_double(<3 x fp128> %a, <3 x fp128> %b, <3 x double>
 ; CHECK-SD-NEXT:    ldp q1, q0, [sp, #32] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldp q2, q4, [sp, #64] // 32-byte Folded Reload
-; CHECK-SD-NEXT:    cset w8, lt
+; CHECK-SD-NEXT:    cset w8, mi
 ; CHECK-SD-NEXT:    sbfx x8, x8, #0, #1
 ; CHECK-SD-NEXT:    ldr q3, [sp, #96] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr x30, [sp, #144] // 8-byte Folded Reload
@@ -755,11 +755,11 @@ define <3 x double> @v3f128_double(<3 x fp128> %a, <3 x fp128> %b, <3 x double>
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    cset w21, lt
+; CHECK-GI-NEXT:    cset w21, mi
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    ldp q1, q0, [sp, #32] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    cset w22, lt
+; CHECK-GI-NEXT:    cset w22, mi
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    ldp q0, q2, [sp, #64] // 32-byte Folded Reload
 ; CHECK-GI-NEXT:    sbfx x8, x21, #0, #1
@@ -770,7 +770,7 @@ define <3 x double> @v3f128_double(<3 x fp128> %a, <3 x fp128> %b, <3 x double>
 ; CHECK-GI-NEXT:    ldr x30, [sp, #128] // 8-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v2.d[1], v0.d[0]
 ; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    cset w8, lt
+; CHECK-GI-NEXT:    cset w8, mi
 ; CHECK-GI-NEXT:    mov v3.d[1], v4.d[0]
 ; CHECK-GI-NEXT:    sbfx x8, x8, #0, #1
 ; CHECK-GI-NEXT:    mov v1.d[1], x9
diff --git a/llvm/test/CodeGen/AArch64/fdiv-combine.ll b/llvm/test/CodeGen/AArch64/fdiv-combine.ll
index d8f7f0a306684..91bb8ac714908 100644
--- a/llvm/test/CodeGen/AArch64/fdiv-combine.ll
+++ b/llvm/test/CodeGen/AArch64/fdiv-combine.ll
@@ -11,7 +11,7 @@
 ;   a / D; b / D; c / D;
 ;                =>
 ;   recip = 1.0 / D; a * recip; b * recip; c * recip;
-define void @three_fdiv_float(float %D, float %a, float %b, float %c) #0 {
+define void @three_fdiv_float(float %D, float %a, float %b, float %c) {
 ; CHECK-SD-LABEL: three_fdiv_float:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    fmov s4, #1.00000000
@@ -28,14 +28,14 @@ define void @three_fdiv_float(float %D, float %a, float %b, float %c) #0 {
 ; CHECK-GI-NEXT:    fdiv s2, s3, s0
 ; CHECK-GI-NEXT:    fmov s0, s4
 ; CHECK-GI-NEXT:    b foo_3f
-  %div = fdiv float %a, %D
-  %div1 = fdiv float %b, %D
-  %div2 = fdiv float %c, %D
+  %div = fdiv arcp float %a, %D
+  %div1 = fdiv arcp float %b, %D
+  %div2 = fdiv arcp float %c, %D
   tail call void @foo_3f(float %div, float %div1, float %div2)
   ret void
 }
 
-define void @three_fdiv_double(double %D, double %a, double %b, double %c) #0 {
+define void @three_fdiv_double(double %D, double %a, double %b, double %c) {
 ; CHECK-SD-LABEL: three_fdiv_double:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    fmov d4, #1.00000000
@@ -52,14 +52,14 @@ define void @three_fdiv_double(double %D, double %a, double %b, double %c) #0 {
 ; CHECK-GI-NEXT:    fdiv d2, d3, d0
 ; CHECK-GI-NEXT:    fmov d0, d4
 ; CHECK-GI-NEXT:    b foo_3d
-  %div = fdiv double %a, %D
-  %div1 = fdiv double %b, %D
-  %div2 = fdiv double %c, %D
+  %div = fdiv arcp double %a, %D
+  %div1 = fdiv arcp double %b, %D
+  %div2 = fdiv arcp double %c, %D
   tail call void @foo_3d(double %div, double %div1, double %div2)
   ret void
 }
 
-define void @three_fdiv_4xfloat(<4 x float> %D, <4 x float> %a, <4 x float> %b, <4 x float> %c) #0 {
+define void @three_fdiv_4xfloat(<4 x float> %D, <4 x float> %a, <4 x float> %b, <4 x float> %c) {
 ; CHECK-SD-LABEL: three_fdiv_4xfloat:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    fmov v4.4s, #1.00000000
@@ -76,14 +76,14 @@ define void @three_fdiv_4xfloat(<4 x float> %D, <4 x float> %a, <4 x float> %b,
 ; CHECK-GI-NEXT:    fdiv v2.4s, v3.4s, v0.4s
 ; CHECK-GI-NEXT:    mov v0.16b, v4.16b
 ; CHECK-GI-NEXT:    b foo_3_4xf
-  %div = fdiv <4 x float> %a, %D
-  %div1 = fdiv <4 x float> %b, %D
-  %div2 = fdiv <4 x float> %c, %D
+  %div = fdiv arcp <4 x float> %a, %D
+  %div1 = fdiv arcp <4 x float> %b, %D
+  %div2 = fdiv arcp <4 x float> %c, %D
   tail call void @foo_3_4xf(<4 x float> %div, <4 x float> %div1, <4 x float> %div2)
   ret void
 }
 
-define void @three_fdiv_2xdouble(<2 x double> %D, <2 x double> %a, <2 x double> %b, <2 x double> %c) #0 {
+define void @three_fdiv_2xdouble(<2 x double> %D, <2 x double> %a, <2 x double> %b, <2 x double> %c) {
 ; CHECK-SD-LABEL: three_fdiv_2xdouble:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    fmov v4.2d, #1.00000000
@@ -100,42 +100,42 @@ define void @three_fdiv_2xdouble(<2 x double> %D, <2 x double> %a, <2 x double>
 ; CHECK-GI-NEXT:    fdiv v2.2d, v3.2d, v0.2d
 ; CHECK-GI-NEXT:    mov v0.16b, v4.16b
 ; CHECK-GI-NEXT:    b foo_3_2xd
-  %div = fdiv <2 x double> %a, %D
-  %div1 = fdiv <2 x double> %b, %D
-  %div2 = fdiv <2 x double> %c, %D
+  %div = fdiv arcp <2 x double> %a, %D
+  %div1 = fdiv arcp <2 x double> %b, %D
+  %div2 = fdiv arcp <2 x double> %c, %D
   tail call void @foo_3_2xd(<2 x double> %div, <2 x double> %div1, <2 x double> %div2)
   ret void
 }
 
 ; Following test cases check we never combine two FDIVs if neither of them
 ; calculates a reciprocal.
-define void @two_fdiv_float(float %D, float %a, float %b) #0 {
+define void @two_fdiv_float(float %D, float %a, float %b) {
 ; CHECK-LABEL: two_fdiv_float:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fdiv s3, s1, s0
 ; CHECK-NEXT:    fdiv s1, s2, s0
 ; CHECK-NEXT:    fmov s0, s3
 ; CHECK-NEXT:    b foo_2f
-  %div = fdiv float %a, %D
-  %div1 = fdiv float %b, %D
+  %div = fdiv arcp float %a, %D
+  %div1 = fdiv arcp float %b, %D
   tail call void @foo_2f(float %div, float %div1)
   ret void
 }
 
-define void @two_fdiv_double(double %D, double %a, double %b) #0 {
+define void @two_fdiv_double(double %D, double %a, double %b) {
 ; CHECK-LABEL: two_fdiv_double:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fdiv d3, d1, d0
 ; CHECK-NEXT:    fdiv d1, d2, d0
 ; CHECK-NEXT:    fmov d0, d3
 ; CHECK-NEXT:    b foo_2d
-  %div = fdiv double %a, %D
-  %div1 = fdiv double %b, %D
+  %div = fdiv arcp double %a, %D
+  %div1 = fdiv arcp double %b, %D
   tail call void @foo_2d(double %div, double %div1)
   ret void
 }
 
-define void @splat_three_fdiv_4xfloat(float %D, <4 x float> %a, <4 x float> %b, <4 x float> %c) #0 {
+define void @splat_three_fdiv_4xfloat(float %D, <4 x float> %a, <4 x float> %b, <4 x float> %c) {
 ; CHECK-SD-LABEL: splat_three_fdiv_4xfloat:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
@@ -157,14 +157,14 @@ define void @splat_three_fdiv_4xfloat(float %D, <4 x float> %a, <4 x float> %b,
 ; CHECK-GI-NEXT:    b foo_3_4xf
   %D.ins = insertelement <4 x float> poison, float %D, i64 0
   %splat = shufflevector <4 x float> %D.ins, <4 x float> poison, <4 x i32> zeroinitializer
-  %div = fdiv <4 x float> %a, %splat
-  %div1 = fdiv <4 x float> %b, %splat
-  %div2 = fdiv <4 x float> %c, %splat
+  %div = fdiv arcp <4 x float> %a, %splat
+  %div1 = fdiv arcp <4 x float> %b, %splat
+  %div2 = fdiv arcp <4 x float> %c, %splat
   tail call void @foo_3_4xf(<4 x float> %div, <4 x float> %div1, <4 x float> %div2)
   ret void
 }
 
-define <4 x float> @splat_fdiv_v4f32(float %D, <4 x float> %a) #1 {
+define <4 x float> @splat_fdiv_v4f32(float %D, <4 x float> %a) #0 {
 ; CHECK-SD-LABEL: splat_fdiv_v4f32:
 ; CHECK-SD:       // %bb.0: // %entry
 ; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
@@ -183,11 +183,11 @@ define <4 x float> @splat_fdiv_v4f32(float %D, <4 x float> %a) #1 {
 entry:
   %D.ins = insertelement <4 x float> poison, float %D, i64 0
   %splat = shufflevector <4 x float> %D.ins, <4 x float> poison, <4 x i32> zeroinitializer
-  %div = fdiv <4 x float> %a, %splat
+  %div = fdiv arcp <4 x float> %a, %splat
   ret <4 x float> %div
 }
 
-define <vscale x 4 x float> @splat_fdiv_nxv4f32(float %D, <vscale x 4 x float> %a) #1 {
+define <vscale x 4 x float> @splat_fdiv_nxv4f32(float %D, <vscale x 4 x float> %a) #0 {
 ; CHECK-LABEL: splat_fdiv_nxv4f32:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    fmov s2, #1.00000000
@@ -198,11 +198,11 @@ define <vscale x 4 x float> @splat_fdiv_nxv4f32(float %D, <vscale x 4 x float> %
 entry:
   %D.ins = insertelement <vscale x 4 x float> poison, float %D, i64 0
   %splat = shufflevector <vscale x 4 x float> %D.ins, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-  %div = fdiv <vscale x 4 x float> %a, %splat
+  %div = fdiv arcp <vscale x 4 x float> %a, %splat
   ret <vscale x 4 x float> %div
 }
 
-define void @splat_three_fdiv_nxv4f32(float %D, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) #1 {
+define void @splat_three_fdiv_nxv4f32(float %D, <vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c) #0 {
 ; CHECK-LABEL: splat_three_fdiv_nxv4f32:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    fmov s4, #1.00000000
@@ -215,14 +215,14 @@ define void @splat_three_fdiv_nxv4f32(float %D, <vscale x 4 x float> %a, <vscale
 entry:
   %D.ins = insertelement <vscale x 4 x float> poison, float %D, i64 0
   %splat = shufflevector <vscale x 4 x float> %D.ins, <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-  %div = fdiv <vscale x 4 x float> %a, %splat
-  %div1 = fdiv <vscale x 4 x float> %b, %splat
-  %div2 = fdiv <vscale x 4 x float> %c, %splat
+  %div = fdiv arcp <vscale x 4 x float> %a, %splat
+  %div1 = fdiv arcp <vscale x 4 x float> %b, %splat
+  %div2 = fdiv arcp <vscale x 4 x float> %c, %splat
   tail call void @foo_3_nxv4f32(<vscale x 4 x float> %div, <vscale x 4 x float> %div1, <vscale x 4 x float> %div2)
   ret void
 }
 
-define <vscale x 2 x double> @splat_fdiv_nxv2f64(double %D, <vscale x 2 x double> %a) #1 {
+define <vscale x 2 x double> @splat_fdiv_nxv2f64(double %D, <vscale x 2 x double> %a) #0 {
 ; CHECK-LABEL: splat_fdiv_nxv2f64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
@@ -237,7 +237,7 @@ entry:
   ret <vscale x 2 x double> %div
 }
 
-define void @splat_two_fdiv_nxv2f64(double %D, <vscale x 2 x double> %a, <vscale x 2 x double> %b) #1 {
+define void @splat_two_fdiv_nxv2f64(double %D, <vscale x 2 x double> %a, <vscale x 2 x double> %b) #0 {
 ; CHECK-LABEL: splat_two_fdiv_nxv2f64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    fmov d3, #1.00000000
@@ -249,8 +249,8 @@ define void @splat_two_fdiv_nxv2f64(double %D, <vscale x 2 x double> %a, <vscale
 entry:
   %D.ins = insertelement <vscale x 2 x double> poison, double %D, i64 0
   %splat = shufflevector <vscale x 2 x double> %D.ins, <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
-  %div = fdiv <vscale x 2 x double> %a, %splat
-  %div1 = fdiv <vscale x 2 x double> %b, %splat
+  %div = fdiv arcp <vscale x 2 x double> %a, %splat
+  %div1 = fdiv arcp <vscale x 2 x double> %b, %splat
   tail call void @foo_2_nxv2f64(<vscale x 2 x double> %div, <vscale x 2 x double> %div1)
   ret void
 }
@@ -264,5 +264,4 @@ declare void @foo_2d(double, double)
 declare void @foo_3_nxv4f32(<vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>)
 declare void @foo_2_nxv2f64(<vscale x 2 x double>, <vscale x 2 x double>)
 
-attributes #0 = { "unsafe-fp-math"="true" }
-attributes #1 = { "unsafe-fp-math"="true" "target-features"="+sve" }
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll b/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll
new file mode 100644
index 0000000000000..936cc6a7d366f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fma-fneg-combine.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s --check-prefixes=CHECK
+
+define float @positive_case_fma(float %a0, float %a1, float %a2) {
+; CHECK-LABEL: positive_case_fma:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fnmadd s0, s0, s1, s2
+; CHECK-NEXT:    ret
+  %fma = call float @llvm.fma.f32(float %a0, float %a1, float %a2)
+  %freeze = freeze float %fma
+  %fneg = fneg float %freeze
+  ret float %fneg
+}
+
+define float @negative_case_fma(float %a0, float %a1) {
+; CHECK-LABEL: negative_case_fma:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fnmadd s0, s0, s1, s0
+; CHECK-NEXT:    ret
+  %fma = call float @llvm.fma.f32(float %a0, float %a1, float poison)
+  %freeze = freeze float %fma
+  %fneg = fneg float %freeze
+  ret float %fneg
+}
+
+define float @positive_case_fmad(float %a0, float %a1, float %a2) {
+; CHECK-LABEL: positive_case_fmad:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fnmadd s0, s0, s1, s2
+; CHECK-NEXT:    ret
+  %fma = call float @llvm.fma.f32(float %a0, float %a1, float %a2)
+  %freeze = freeze float %fma
+  %fneg = fneg float %freeze
+  ret float %fneg
+}
+
+define float @negative_case_fmad(float %a0, float %a1) {
+; CHECK-LABEL: negative_case_fmad:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fnmadd s0, s0, s1, s0
+; CHECK-NEXT:    ret
+  %fmad = call float @llvm.fmuladd.f32(float %a0, float %a1, float poison)
+  %freeze = freeze float %fmad
+  %fneg = fneg float %freeze
+  ret float %fneg
+}
+
diff --git a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
index f2a14a9b73fa1..919585a9826b9 100644
--- a/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/fp-intrinsics.ll
@@ -2440,7 +2440,7 @@ define i32 @fcmp_olt_f128(fp128 %a, fp128 %b) #0 {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __lttf2
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"olt", metadata !"fpexcept.strict") #0
@@ -2488,7 +2488,7 @@ define i32 @fcmp_oge_f128(fp128 %a, fp128 %b) #0 {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __getf2
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    cset w0, pl
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"oge", metadata !"fpexcept.strict") #0
@@ -2544,7 +2544,7 @@ define i32 @fcmp_ult_f128(fp128 %a, fp128 %b) #0 {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __getf2
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"ult", metadata !"fpexcept.strict") #0
@@ -2592,7 +2592,7 @@ define i32 @fcmp_uge_f128(fp128 %a, fp128 %b) #0 {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __lttf2
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    cset w0, pl
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %cmp = call i1 @llvm.experimental.constrained.fcmp.f128(fp128 %a, fp128 %b, metadata !"uge", metadata !"fpexcept.strict") #0
@@ -2648,7 +2648,7 @@ define i32 @fcmps_olt_f128(fp128 %a, fp128 %b) #0 {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __lttf2
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"olt", metadata !"fpexcept.strict") #0
@@ -2696,7 +2696,7 @@ define i32 @fcmps_oge_f128(fp128 %a, fp128 %b) #0 {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __getf2
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    cset w0, pl
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"oge", metadata !"fpexcept.strict") #0
@@ -2752,7 +2752,7 @@ define i32 @fcmps_ult_f128(fp128 %a, fp128 %b) #0 {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __getf2
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cset w0, lt
+; CHECK-NEXT:    cset w0, mi
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"ult", metadata !"fpexcept.strict") #0
@@ -2800,7 +2800,7 @@ define i32 @fcmps_uge_f128(fp128 %a, fp128 %b) #0 {
 ; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    bl __lttf2
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    cset w0, pl
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %cmp = call i1 @llvm.experimental.constrained.fcmps.f128(fp128 %a, fp128 %b, metadata !"uge", metadata !"fpexcept.strict") #0
diff --git a/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll b/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll
index c6b8e41f9bdfd..4906e2e15e51c 100644
--- a/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll
+++ b/llvm/test/CodeGen/AArch64/fp-maximumnum-minimumnum.ll
@@ -1431,6 +1431,7 @@ define <9 x half> @max_v9f16(<9 x half> %a, <9 x half> %b) {
 ; FULLFP16-NEXT:    add x9, sp, #16
 ; FULLFP16-NEXT:    // kill: def $h3 killed $h3 def $q3
 ; FULLFP16-NEXT:    // kill: def $h4 killed $h4 def $q4
+; FULLFP16-NEXT:    add x10, sp, #40
 ; FULLFP16-NEXT:    // kill: def $h5 killed $h5 def $q5
 ; FULLFP16-NEXT:    // kill: def $h6 killed $h6 def $q6
 ; FULLFP16-NEXT:    // kill: def $h7 killed $h7 def $q7
@@ -1439,30 +1440,30 @@ define <9 x half> @max_v9f16(<9 x half> %a, <9 x half> %b) {
 ; FULLFP16-NEXT:    ld1 { v1.h }[1], [x9]
 ; FULLFP16-NEXT:    add x9, sp, #24
 ; FULLFP16-NEXT:    mov v0.h[2], v2.h[0]
-; FULLFP16-NEXT:    ldr h2, [sp]
 ; FULLFP16-NEXT:    ld1 { v1.h }[2], [x9]
 ; FULLFP16-NEXT:    add x9, sp, #32
-; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v2.8h
 ; FULLFP16-NEXT:    mov v0.h[3], v3.h[0]
 ; FULLFP16-NEXT:    ld1 { v1.h }[3], [x9]
-; FULLFP16-NEXT:    add x9, sp, #40
-; FULLFP16-NEXT:    ldr h3, [sp, #72]
-; FULLFP16-NEXT:    ld1 { v1.h }[4], [x9]
+; FULLFP16-NEXT:    ldr h2, [x10]
 ; FULLFP16-NEXT:    add x9, sp, #48
+; FULLFP16-NEXT:    ldr h3, [sp, #72]
+; FULLFP16-NEXT:    ld1 { v2.h }[1], [x9]
+; FULLFP16-NEXT:    add x9, sp, #56
 ; FULLFP16-NEXT:    fminnm v3.8h, v3.8h, v3.8h
 ; FULLFP16-NEXT:    mov v0.h[4], v4.h[0]
-; FULLFP16-NEXT:    ld1 { v1.h }[5], [x9]
-; FULLFP16-NEXT:    add x9, sp, #56
-; FULLFP16-NEXT:    fmaxnm v2.8h, v2.8h, v3.8h
-; FULLFP16-NEXT:    mov v0.h[5], v5.h[0]
-; FULLFP16-NEXT:    ld1 { v1.h }[6], [x9]
+; FULLFP16-NEXT:    ld1 { v2.h }[2], [x9]
 ; FULLFP16-NEXT:    add x9, sp, #64
-; FULLFP16-NEXT:    str h2, [x8, #16]
+; FULLFP16-NEXT:    mov v0.h[5], v5.h[0]
+; FULLFP16-NEXT:    ld1 { v2.h }[3], [x9]
+; FULLFP16-NEXT:    zip1 v1.2d, v1.2d, v2.2d
+; FULLFP16-NEXT:    ldr h2, [sp]
 ; FULLFP16-NEXT:    mov v0.h[6], v6.h[0]
-; FULLFP16-NEXT:    ld1 { v1.h }[7], [x9]
+; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v2.8h
 ; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v1.8h
 ; FULLFP16-NEXT:    mov v0.h[7], v7.h[0]
+; FULLFP16-NEXT:    fmaxnm v2.8h, v2.8h, v3.8h
 ; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v0.8h
+; FULLFP16-NEXT:    str h2, [x8, #16]
 ; FULLFP16-NEXT:    fmaxnm v0.8h, v0.8h, v1.8h
 ; FULLFP16-NEXT:    str q0, [x8]
 ; FULLFP16-NEXT:    ret
@@ -2012,6 +2013,7 @@ define <9 x half> @min_v9f16(<9 x half> %a, <9 x half> %b) {
 ; FULLFP16-NEXT:    add x9, sp, #16
 ; FULLFP16-NEXT:    // kill: def $h3 killed $h3 def $q3
 ; FULLFP16-NEXT:    // kill: def $h4 killed $h4 def $q4
+; FULLFP16-NEXT:    add x10, sp, #40
 ; FULLFP16-NEXT:    // kill: def $h5 killed $h5 def $q5
 ; FULLFP16-NEXT:    // kill: def $h6 killed $h6 def $q6
 ; FULLFP16-NEXT:    // kill: def $h7 killed $h7 def $q7
@@ -2020,30 +2022,30 @@ define <9 x half> @min_v9f16(<9 x half> %a, <9 x half> %b) {
 ; FULLFP16-NEXT:    ld1 { v1.h }[1], [x9]
 ; FULLFP16-NEXT:    add x9, sp, #24
 ; FULLFP16-NEXT:    mov v0.h[2], v2.h[0]
-; FULLFP16-NEXT:    ldr h2, [sp]
 ; FULLFP16-NEXT:    ld1 { v1.h }[2], [x9]
 ; FULLFP16-NEXT:    add x9, sp, #32
-; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v2.8h
 ; FULLFP16-NEXT:    mov v0.h[3], v3.h[0]
 ; FULLFP16-NEXT:    ld1 { v1.h }[3], [x9]
-; FULLFP16-NEXT:    add x9, sp, #40
-; FULLFP16-NEXT:    ldr h3, [sp, #72]
-; FULLFP16-NEXT:    ld1 { v1.h }[4], [x9]
+; FULLFP16-NEXT:    ldr h2, [x10]
 ; FULLFP16-NEXT:    add x9, sp, #48
+; FULLFP16-NEXT:    ldr h3, [sp, #72]
+; FULLFP16-NEXT:    ld1 { v2.h }[1], [x9]
+; FULLFP16-NEXT:    add x9, sp, #56
 ; FULLFP16-NEXT:    fminnm v3.8h, v3.8h, v3.8h
 ; FULLFP16-NEXT:    mov v0.h[4], v4.h[0]
-; FULLFP16-NEXT:    ld1 { v1.h }[5], [x9]
-; FULLFP16-NEXT:    add x9, sp, #56
-; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v3.8h
-; FULLFP16-NEXT:    mov v0.h[5], v5.h[0]
-; FULLFP16-NEXT:    ld1 { v1.h }[6], [x9]
+; FULLFP16-NEXT:    ld1 { v2.h }[2], [x9]
 ; FULLFP16-NEXT:    add x9, sp, #64
-; FULLFP16-NEXT:    str h2, [x8, #16]
+; FULLFP16-NEXT:    mov v0.h[5], v5.h[0]
+; FULLFP16-NEXT:    ld1 { v2.h }[3], [x9]
+; FULLFP16-NEXT:    zip1 v1.2d, v1.2d, v2.2d
+; FULLFP16-NEXT:    ldr h2, [sp]
 ; FULLFP16-NEXT:    mov v0.h[6], v6.h[0]
-; FULLFP16-NEXT:    ld1 { v1.h }[7], [x9]
+; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v2.8h
 ; FULLFP16-NEXT:    fminnm v1.8h, v1.8h, v1.8h
 ; FULLFP16-NEXT:    mov v0.h[7], v7.h[0]
+; FULLFP16-NEXT:    fminnm v2.8h, v2.8h, v3.8h
 ; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v0.8h
+; FULLFP16-NEXT:    str h2, [x8, #16]
 ; FULLFP16-NEXT:    fminnm v0.8h, v0.8h, v1.8h
 ; FULLFP16-NEXT:    str q0, [x8]
 ; FULLFP16-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat.ll b/llvm/test/CodeGen/AArch64/fpclamptosat.ll
index 06dc11d413fae..00de1530fb72c 100644
--- a/llvm/test/CodeGen/AArch64/fpclamptosat.ll
+++ b/llvm/test/CodeGen/AArch64/fpclamptosat.ll
@@ -903,7 +903,7 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; CHECK-NEXT:    csinc x8, x1, xzr, lt
 ; CHECK-NEXT:    csel x9, x0, xzr, lt
 ; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    csel x0, xzr, x9, lt
+; CHECK-NEXT:    csel x0, xzr, x9, mi
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
@@ -956,7 +956,7 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; CHECK-NEXT:    csinc x8, x1, xzr, lt
 ; CHECK-NEXT:    csel x9, x0, xzr, lt
 ; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    csel x0, xzr, x9, lt
+; CHECK-NEXT:    csel x0, xzr, x9, mi
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
@@ -1015,7 +1015,7 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; CHECK-NEXT:    csinc x8, x1, xzr, lt
 ; CHECK-NEXT:    csel x9, x0, xzr, lt
 ; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    csel x0, xzr, x9, lt
+; CHECK-NEXT:    csel x0, xzr, x9, mi
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
index 9157bcba59e9b..b09a86723bb56 100644
--- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT
-; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16
+; RUN: llc < %s -mtriple=aarch64 -global-isel=0| FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-SD
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel=0 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-SD
+; RUN: llc < %s -mtriple=aarch64 -global-isel=1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel=1 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-GI
 
 ; i32 saturate
 
@@ -21,15 +23,43 @@ entry:
 }
 
 define <2 x i32> @utest_f64i32(<2 x double> %x) {
-; CHECK-LABEL: utest_f64i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov d1, v0.d[1]
-; CHECK-NEXT:    fcvtzu w8, d0
-; CHECK-NEXT:    fcvtzu w9, d1
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v0.s[1], w9
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utest_f64i32:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-CVT-SD-NEXT:    fcvtzu w8, d0
+; CHECK-CVT-SD-NEXT:    fcvtzu w9, d1
+; CHECK-CVT-SD-NEXT:    fmov s0, w8
+; CHECK-CVT-SD-NEXT:    mov v0.s[1], w9
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utest_f64i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-SD-NEXT:    fcvtzu w8, d0
+; CHECK-FP16-SD-NEXT:    fcvtzu w9, d1
+; CHECK-FP16-SD-NEXT:    fmov s0, w8
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w9
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utest_f64i32:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmhi v2.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utest_f64i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmhi v2.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-FP16-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <2 x double> %x to <2 x i64>
   %0 = icmp ult <2 x i64> %conv, <i64 4294967295, i64 4294967295>
@@ -55,10 +85,57 @@ entry:
 }
 
 define <4 x i32> @stest_f32i32(<4 x float> %x) {
-; CHECK-LABEL: stest_f32i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f32i32:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f32i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f32i32:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    adrp x8, .LCPI3_1
+; CHECK-CVT-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI3_1]
+; CHECK-CVT-GI-NEXT:    adrp x8, .LCPI3_0
+; CHECK-CVT-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v2.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI3_0]
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v0.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f32i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    adrp x8, .LCPI3_1
+; CHECK-FP16-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI3_1]
+; CHECK-FP16-GI-NEXT:    adrp x8, .LCPI3_0
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI3_0]
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <4 x float> %x to <4 x i64>
   %0 = icmp slt <4 x i64> %conv, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
@@ -70,10 +147,43 @@ entry:
 }
 
 define <4 x i32> @utest_f32i32(<4 x float> %x) {
-; CHECK-LABEL: utest_f32i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utest_f32i32:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utest_f32i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utest_f32i32:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmhi v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmhi v4.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v2.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utest_f32i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmhi v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    cmhi v4.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <4 x float> %x to <4 x i64>
   %0 = icmp ult <4 x i64> %conv, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -83,10 +193,51 @@ entry:
 }
 
 define <4 x i32> @ustest_f32i32(<4 x float> %x) {
-; CHECK-LABEL: ustest_f32i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f32i32:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f32i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f32i32:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    cmgt v1.2d, v2.2d, #0
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v0.2d, #0
+; CHECK-CVT-GI-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f32i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    cmgt v1.2d, v2.2d, #0
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v0.2d, #0
+; CHECK-FP16-GI-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <4 x float> %x to <4 x i64>
   %0 = icmp slt <4 x i64> %conv, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -98,11 +249,68 @@ entry:
 }
 
 define <4 x i32> @stest_f16i32(<4 x half> %x) {
-; CHECK-LABEL: stest_f16i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f16i32:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f16i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f16i32:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-GI-NEXT:    adrp x8, .LCPI6_1
+; CHECK-CVT-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI6_1]
+; CHECK-CVT-GI-NEXT:    adrp x8, .LCPI6_0
+; CHECK-CVT-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v2.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI6_0]
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v0.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f16i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT:    adrp x8, .LCPI6_1
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d1, h1
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI6_1]
+; CHECK-FP16-GI-NEXT:    adrp x8, .LCPI6_0
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI6_0]
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <4 x half> %x to <4 x i64>
   %0 = icmp slt <4 x i64> %conv, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
@@ -114,11 +322,54 @@ entry:
 }
 
 define <4 x i32> @utesth_f16i32(<4 x half> %x) {
-; CHECK-LABEL: utesth_f16i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utesth_f16i32:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utesth_f16i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utesth_f16i32:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmhi v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmhi v4.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v2.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utesth_f16i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    fcvt d4, h4
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    cmhi v3.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmhi v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bit v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <4 x half> %x to <4 x i64>
   %0 = icmp ult <4 x i64> %conv, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -128,11 +379,62 @@ entry:
 }
 
 define <4 x i32> @ustest_f16i32(<4 x half> %x) {
-; CHECK-LABEL: ustest_f16i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f16i32:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f16i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f16i32:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    cmgt v1.2d, v2.2d, #0
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v0.2d, #0
+; CHECK-CVT-GI-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f16i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    fcvt d4, h4
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bit v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    cmgt v2.2d, v0.2d, #0
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v1.2d, #0
+; CHECK-FP16-GI-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-FP16-GI-NEXT:    and v1.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <4 x half> %x to <4 x i64>
   %0 = icmp slt <4 x i64> %conv, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -146,15 +448,45 @@ entry:
 ; i16 saturate
 
 define <2 x i16> @stest_f64i16(<2 x double> %x) {
-; CHECK-LABEL: stest_f64i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-NEXT:    movi v1.2s, #127, msl #8
-; CHECK-NEXT:    xtn v0.2s, v0.2d
-; CHECK-NEXT:    smin v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mvni v1.2s, #127, msl #8
-; CHECK-NEXT:    smax v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f64i16:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-CVT-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-CVT-SD-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-CVT-SD-NEXT:    mvni v1.2s, #127, msl #8
+; CHECK-CVT-SD-NEXT:    smax v0.2s, v0.2s, v1.2s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f64i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-FP16-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-FP16-SD-NEXT:    mvni v1.2s, #127, msl #8
+; CHECK-FP16-SD-NEXT:    smax v0.2s, v0.2s, v1.2s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f64i16:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-CVT-GI-NEXT:    mvni v2.2s, #127, msl #8
+; CHECK-CVT-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-CVT-GI-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-CVT-GI-NEXT:    smax v0.2s, v0.2s, v2.2s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f64i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-FP16-GI-NEXT:    mvni v2.2s, #127, msl #8
+; CHECK-FP16-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-FP16-GI-NEXT:    smax v0.2s, v0.2s, v2.2s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i32>
   %0 = icmp slt <2 x i32> %conv, <i32 32767, i32 32767>
@@ -258,10 +590,20 @@ define <8 x i16> @stest_f16i16(<8 x half> %x) {
 ; CHECK-CVT-NEXT:    sqxtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-FP16-LABEL: stest_f16i16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-FP16-SD-LABEL: stest_f16i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f16i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    sqxtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    sqxtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <8 x half> %x to <8 x i32>
   %0 = icmp slt <8 x i32> %conv, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
@@ -283,10 +625,20 @@ define <8 x i16> @utesth_f16i16(<8 x half> %x) {
 ; CHECK-CVT-NEXT:    uqxtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-FP16-LABEL: utesth_f16i16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-FP16-SD-LABEL: utesth_f16i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utesth_f16i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    uqxtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <8 x half> %x to <8 x i32>
   %0 = icmp ult <8 x i32> %conv, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
@@ -306,10 +658,20 @@ define <8 x i16> @ustest_f16i16(<8 x half> %x) {
 ; CHECK-CVT-NEXT:    sqxtun2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-FP16-LABEL: ustest_f16i16:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-FP16-SD-LABEL: ustest_f16i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f16i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    sqxtun v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    sqxtun2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <8 x half> %x to <8 x i32>
   %0 = icmp slt <8 x i32> %conv, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
@@ -323,10 +685,139 @@ entry:
 ; i64 saturate
 
 define <2 x i64> @stest_f64i64(<2 x double> %x) {
-; CHECK-LABEL: stest_f64i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f64i64:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f64i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f64i64:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -40
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -48
+; CHECK-CVT-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-CVT-GI-NEXT:    mov x21, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-CVT-GI-NEXT:    mov x22, #-9223372036854775808 // =0x8000000000000000
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixdfti
+; CHECK-CVT-GI-NEXT:    fmov d0, d8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixdfti
+; CHECK-CVT-GI-NEXT:    cmp x19, x21
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lo
+; CHECK-CVT-GI-NEXT:    cmp x20, #0
+; CHECK-CVT-GI-NEXT:    cset w9, mi
+; CHECK-CVT-GI-NEXT:    csel w8, w8, w9, eq
+; CHECK-CVT-GI-NEXT:    cmp x0, x21
+; CHECK-CVT-GI-NEXT:    cset w9, lo
+; CHECK-CVT-GI-NEXT:    cmp x1, #0
+; CHECK-CVT-GI-NEXT:    cset w10, mi
+; CHECK-CVT-GI-NEXT:    csel w9, w9, w10, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, x21, ne
+; CHECK-CVT-GI-NEXT:    csel x10, x20, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x0, x21, ne
+; CHECK-CVT-GI-NEXT:    csel x11, x1, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, x22
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x10, #1
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, x22
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x11, #1
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, x22, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, x22, ne
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #48 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f64i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -40
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -48
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    mov x21, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-FP16-GI-NEXT:    mov x22, #-9223372036854775808 // =0x8000000000000000
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    fmov d0, d8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    cmp x19, x21
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lo
+; CHECK-FP16-GI-NEXT:    cmp x20, #0
+; CHECK-FP16-GI-NEXT:    cset w9, mi
+; CHECK-FP16-GI-NEXT:    csel w8, w8, w9, eq
+; CHECK-FP16-GI-NEXT:    cmp x0, x21
+; CHECK-FP16-GI-NEXT:    cset w9, lo
+; CHECK-FP16-GI-NEXT:    cmp x1, #0
+; CHECK-FP16-GI-NEXT:    cset w10, mi
+; CHECK-FP16-GI-NEXT:    csel w9, w9, w10, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, x21, ne
+; CHECK-FP16-GI-NEXT:    csel x10, x20, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x0, x21, ne
+; CHECK-FP16-GI-NEXT:    csel x11, x1, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, x22
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x10, #1
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, x22
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x11, #1
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, x22, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, x22, ne
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #48 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 9223372036854775807, i128 9223372036854775807>
@@ -338,34 +829,131 @@ entry:
 }
 
 define <2 x i64> @utest_f64i64(<2 x double> %x) {
-; CHECK-LABEL: utest_f64i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov d0, v0.d[1]
-; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    cmp x1, #0
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, eq
-; CHECK-NEXT:    cmp x20, #0
-; CHECK-NEXT:    csel x9, x19, xzr, eq
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utest_f64i64:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-CVT-SD-NEXT:    bl __fixunsdfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixunsdfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #0
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-CVT-SD-NEXT:    cmp x20, #0
+; CHECK-CVT-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x9
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utest_f64i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-FP16-SD-NEXT:    bl __fixunsdfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixunsdfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #0
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-FP16-SD-NEXT:    cmp x20, #0
+; CHECK-FP16-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utest_f64i64:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-CVT-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixunsdfti
+; CHECK-CVT-GI-NEXT:    fmov d0, d8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixunsdfti
+; CHECK-CVT-GI-NEXT:    cmp x20, #1
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lo
+; CHECK-CVT-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-CVT-GI-NEXT:    cmp x1, #1
+; CHECK-CVT-GI-NEXT:    cset w9, lo
+; CHECK-CVT-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utest_f64i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixunsdfti
+; CHECK-FP16-GI-NEXT:    fmov d0, d8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixunsdfti
+; CHECK-FP16-GI-NEXT:    cmp x20, #1
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lo
+; CHECK-FP16-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-FP16-GI-NEXT:    cmp x1, #1
+; CHECK-FP16-GI-NEXT:    cset w9, lo
+; CHECK-FP16-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <2 x double> %x to <2 x i128>
   %0 = icmp ult <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -375,42 +963,179 @@ entry:
 }
 
 define <2 x i64> @ustest_f64i64(<2 x double> %x) {
-; CHECK-LABEL: ustest_f64i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    mov d0, v0.d[1]
-; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
-; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csel x10, x19, xzr, lt
-; CHECK-NEXT:    csinc x11, x20, xzr, lt
-; CHECK-NEXT:    cmp xzr, x10
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ngcs xzr, x11
-; CHECK-NEXT:    csel x10, x10, xzr, lt
-; CHECK-NEXT:    cmp xzr, x8
-; CHECK-NEXT:    ngcs xzr, x9
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    csel x8, x8, xzr, lt
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f64i64:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixdfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-CVT-SD-NEXT:    bl __fixdfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #1
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, lt
+; CHECK-CVT-SD-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp x20, #1
+; CHECK-CVT-SD-NEXT:    csel x10, x19, xzr, lt
+; CHECK-CVT-SD-NEXT:    csinc x11, x20, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp xzr, x10
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    ngcs xzr, x11
+; CHECK-CVT-SD-NEXT:    csel x10, x10, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp xzr, x8
+; CHECK-CVT-SD-NEXT:    ngcs xzr, x9
+; CHECK-CVT-SD-NEXT:    fmov d0, x10
+; CHECK-CVT-SD-NEXT:    csel x8, x8, xzr, lt
+; CHECK-CVT-SD-NEXT:    fmov d1, x8
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f64i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixdfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-FP16-SD-NEXT:    bl __fixdfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #1
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, lt
+; CHECK-FP16-SD-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp x20, #1
+; CHECK-FP16-SD-NEXT:    csel x10, x19, xzr, lt
+; CHECK-FP16-SD-NEXT:    csinc x11, x20, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp xzr, x10
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ngcs xzr, x11
+; CHECK-FP16-SD-NEXT:    csel x10, x10, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp xzr, x8
+; CHECK-FP16-SD-NEXT:    ngcs xzr, x9
+; CHECK-FP16-SD-NEXT:    fmov d0, x10
+; CHECK-FP16-SD-NEXT:    csel x8, x8, xzr, lt
+; CHECK-FP16-SD-NEXT:    fmov d1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f64i64:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-CVT-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixdfti
+; CHECK-CVT-GI-NEXT:    fmov d0, d8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixdfti
+; CHECK-CVT-GI-NEXT:    cmp x20, #1
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lt
+; CHECK-CVT-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-CVT-GI-NEXT:    cmp x1, #1
+; CHECK-CVT-GI-NEXT:    cset w9, lt
+; CHECK-CVT-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x10, x20, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x11, x1, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w10, gt
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w11, gt
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f64i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    fmov d0, d8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    cmp x20, #1
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lt
+; CHECK-FP16-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-FP16-GI-NEXT:    cmp x1, #1
+; CHECK-FP16-GI-NEXT:    cset w9, lt
+; CHECK-FP16-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x10, x20, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x11, x1, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w10, gt
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w11, gt
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -422,11 +1147,143 @@ entry:
 }
 
 define <2 x i64> @stest_f32i64(<2 x float> %x) {
-; CHECK-LABEL: stest_f32i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f32i64:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f32i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f32i64:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -40
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -48
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-CVT-GI-NEXT:    mov x21, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-CVT-GI-NEXT:    mov x22, #-9223372036854775808 // =0x8000000000000000
+; CHECK-CVT-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixsfti
+; CHECK-CVT-GI-NEXT:    fmov s0, s8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixsfti
+; CHECK-CVT-GI-NEXT:    cmp x19, x21
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lo
+; CHECK-CVT-GI-NEXT:    cmp x20, #0
+; CHECK-CVT-GI-NEXT:    cset w9, mi
+; CHECK-CVT-GI-NEXT:    csel w8, w8, w9, eq
+; CHECK-CVT-GI-NEXT:    cmp x0, x21
+; CHECK-CVT-GI-NEXT:    cset w9, lo
+; CHECK-CVT-GI-NEXT:    cmp x1, #0
+; CHECK-CVT-GI-NEXT:    cset w10, mi
+; CHECK-CVT-GI-NEXT:    csel w9, w9, w10, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, x21, ne
+; CHECK-CVT-GI-NEXT:    csel x10, x20, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x0, x21, ne
+; CHECK-CVT-GI-NEXT:    csel x11, x1, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, x22
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x10, #1
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, x22
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x11, #1
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, x22, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, x22, ne
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #48 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f32i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -40
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -48
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    mov x21, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-FP16-GI-NEXT:    mov x22, #-9223372036854775808 // =0x8000000000000000
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    fmov s0, s8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    cmp x19, x21
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lo
+; CHECK-FP16-GI-NEXT:    cmp x20, #0
+; CHECK-FP16-GI-NEXT:    cset w9, mi
+; CHECK-FP16-GI-NEXT:    csel w8, w8, w9, eq
+; CHECK-FP16-GI-NEXT:    cmp x0, x21
+; CHECK-FP16-GI-NEXT:    cset w9, lo
+; CHECK-FP16-GI-NEXT:    cmp x1, #0
+; CHECK-FP16-GI-NEXT:    cset w10, mi
+; CHECK-FP16-GI-NEXT:    csel w9, w9, w10, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, x21, ne
+; CHECK-FP16-GI-NEXT:    csel x10, x20, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x0, x21, ne
+; CHECK-FP16-GI-NEXT:    csel x11, x1, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, x22
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x10, #1
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, x22
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x11, #1
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, x22, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, x22, ne
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #48 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 9223372036854775807, i128 9223372036854775807>
@@ -438,35 +1295,135 @@ entry:
 }
 
 define <2 x i64> @utest_f32i64(<2 x float> %x) {
-; CHECK-LABEL: utest_f32i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov s0, v0.s[1]
-; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    cmp x1, #0
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, eq
-; CHECK-NEXT:    cmp x20, #0
-; CHECK-NEXT:    csel x9, x19, xzr, eq
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utest_f32i64:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-CVT-SD-NEXT:    bl __fixunssfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixunssfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #0
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-CVT-SD-NEXT:    cmp x20, #0
+; CHECK-CVT-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x9
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utest_f32i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-FP16-SD-NEXT:    bl __fixunssfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixunssfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #0
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-FP16-SD-NEXT:    cmp x20, #0
+; CHECK-FP16-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utest_f32i64:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-CVT-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixunssfti
+; CHECK-CVT-GI-NEXT:    fmov s0, s8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixunssfti
+; CHECK-CVT-GI-NEXT:    cmp x20, #1
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lo
+; CHECK-CVT-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-CVT-GI-NEXT:    cmp x1, #1
+; CHECK-CVT-GI-NEXT:    cset w9, lo
+; CHECK-CVT-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utest_f32i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixunssfti
+; CHECK-FP16-GI-NEXT:    fmov s0, s8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixunssfti
+; CHECK-FP16-GI-NEXT:    cmp x20, #1
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lo
+; CHECK-FP16-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-FP16-GI-NEXT:    cmp x1, #1
+; CHECK-FP16-GI-NEXT:    cset w9, lo
+; CHECK-FP16-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <2 x float> %x to <2 x i128>
   %0 = icmp ult <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -476,43 +1433,183 @@ entry:
 }
 
 define <2 x i64> @ustest_f32i64(<2 x float> %x) {
-; CHECK-LABEL: ustest_f32i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    mov s0, v0.s[1]
-; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csinc x8, x1, xzr, lt
-; CHECK-NEXT:    csel x9, x0, xzr, lt
-; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csel x10, x19, xzr, lt
-; CHECK-NEXT:    csinc x11, x20, xzr, lt
-; CHECK-NEXT:    cmp xzr, x10
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ngcs xzr, x11
-; CHECK-NEXT:    csel x10, x10, xzr, lt
-; CHECK-NEXT:    cmp xzr, x9
-; CHECK-NEXT:    ngcs xzr, x8
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    csel x8, x9, xzr, lt
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f32i64:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixsfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-CVT-SD-NEXT:    bl __fixsfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #1
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csinc x8, x1, xzr, lt
+; CHECK-CVT-SD-NEXT:    csel x9, x0, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp x20, #1
+; CHECK-CVT-SD-NEXT:    csel x10, x19, xzr, lt
+; CHECK-CVT-SD-NEXT:    csinc x11, x20, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp xzr, x10
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    ngcs xzr, x11
+; CHECK-CVT-SD-NEXT:    csel x10, x10, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp xzr, x9
+; CHECK-CVT-SD-NEXT:    ngcs xzr, x8
+; CHECK-CVT-SD-NEXT:    fmov d0, x10
+; CHECK-CVT-SD-NEXT:    csel x8, x9, xzr, lt
+; CHECK-CVT-SD-NEXT:    fmov d1, x8
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f32i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixsfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-FP16-SD-NEXT:    bl __fixsfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #1
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csinc x8, x1, xzr, lt
+; CHECK-FP16-SD-NEXT:    csel x9, x0, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp x20, #1
+; CHECK-FP16-SD-NEXT:    csel x10, x19, xzr, lt
+; CHECK-FP16-SD-NEXT:    csinc x11, x20, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp xzr, x10
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ngcs xzr, x11
+; CHECK-FP16-SD-NEXT:    csel x10, x10, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp xzr, x9
+; CHECK-FP16-SD-NEXT:    ngcs xzr, x8
+; CHECK-FP16-SD-NEXT:    fmov d0, x10
+; CHECK-FP16-SD-NEXT:    csel x8, x9, xzr, lt
+; CHECK-FP16-SD-NEXT:    fmov d1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f32i64:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-CVT-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixsfti
+; CHECK-CVT-GI-NEXT:    fmov s0, s8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixsfti
+; CHECK-CVT-GI-NEXT:    cmp x20, #1
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lt
+; CHECK-CVT-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-CVT-GI-NEXT:    cmp x1, #1
+; CHECK-CVT-GI-NEXT:    cset w9, lt
+; CHECK-CVT-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x10, x20, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x11, x1, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w10, gt
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w11, gt
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f32i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    fmov s0, s8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    cmp x20, #1
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lt
+; CHECK-FP16-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-FP16-GI-NEXT:    cmp x1, #1
+; CHECK-FP16-GI-NEXT:    cset w9, lt
+; CHECK-FP16-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x10, x20, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x11, x1, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w10, gt
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w11, gt
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -524,27 +1621,121 @@ entry:
 }
 
 define <2 x i64> @stest_f16i64(<2 x half> %x) {
-; CHECK-CVT-LABEL: stest_f16i64:
-; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvtzs x8, s0
-; CHECK-CVT-NEXT:    fcvtzs x9, s1
-; CHECK-CVT-NEXT:    fmov d0, x8
-; CHECK-CVT-NEXT:    mov v0.d[1], x9
-; CHECK-CVT-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f16i64:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvtzs x8, s0
+; CHECK-CVT-SD-NEXT:    fcvtzs x9, s1
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-SD-NEXT:    ret
 ;
-; CHECK-FP16-LABEL: stest_f16i64:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-FP16-NEXT:    fcvtzs x8, h0
-; CHECK-FP16-NEXT:    fcvtzs x9, h1
-; CHECK-FP16-NEXT:    fmov d0, x8
-; CHECK-FP16-NEXT:    mov v0.d[1], x9
-; CHECK-FP16-NEXT:    ret
+; CHECK-FP16-SD-LABEL: stest_f16i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    fcvtzs x8, h0
+; CHECK-FP16-SD-NEXT:    fcvtzs x9, h1
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f16i64:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-GI-NEXT:    mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-CVT-GI-NEXT:    mov x16, #-9223372036854775808 // =0x8000000000000000
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcvtzs x9, s0
+; CHECK-CVT-GI-NEXT:    fcvtzs x10, s1
+; CHECK-CVT-GI-NEXT:    asr x11, x9, #63
+; CHECK-CVT-GI-NEXT:    cmp x9, x8
+; CHECK-CVT-GI-NEXT:    cset w12, lo
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    asr x13, x10, #63
+; CHECK-CVT-GI-NEXT:    cset w14, mi
+; CHECK-CVT-GI-NEXT:    csel w12, w12, w14, eq
+; CHECK-CVT-GI-NEXT:    cmp x10, x8
+; CHECK-CVT-GI-NEXT:    cset w14, lo
+; CHECK-CVT-GI-NEXT:    cmp x13, #0
+; CHECK-CVT-GI-NEXT:    cset w15, mi
+; CHECK-CVT-GI-NEXT:    csel w14, w14, w15, eq
+; CHECK-CVT-GI-NEXT:    tst w12, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x9, x8, ne
+; CHECK-CVT-GI-NEXT:    csel x11, x11, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w14, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x10, x8, ne
+; CHECK-CVT-GI-NEXT:    csel x10, x13, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x9, x16
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x11, #1
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    cmp x8, x16
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x10, #1
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x9, x16, ne
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x9
+; CHECK-CVT-GI-NEXT:    csel x8, x8, x16, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x8
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f16i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvtzs x9, h0
+; CHECK-FP16-GI-NEXT:    mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-FP16-GI-NEXT:    mov x16, #-9223372036854775808 // =0x8000000000000000
+; CHECK-FP16-GI-NEXT:    fcvtzs x10, h1
+; CHECK-FP16-GI-NEXT:    asr x11, x9, #63
+; CHECK-FP16-GI-NEXT:    cmp x9, x8
+; CHECK-FP16-GI-NEXT:    cset w12, lo
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w14, mi
+; CHECK-FP16-GI-NEXT:    asr x13, x10, #63
+; CHECK-FP16-GI-NEXT:    csel w12, w12, w14, eq
+; CHECK-FP16-GI-NEXT:    cmp x10, x8
+; CHECK-FP16-GI-NEXT:    cset w14, lo
+; CHECK-FP16-GI-NEXT:    cmp x13, #0
+; CHECK-FP16-GI-NEXT:    cset w15, mi
+; CHECK-FP16-GI-NEXT:    csel w14, w14, w15, eq
+; CHECK-FP16-GI-NEXT:    tst w12, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x9, x8, ne
+; CHECK-FP16-GI-NEXT:    csel x11, x11, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w14, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x10, x8, ne
+; CHECK-FP16-GI-NEXT:    csel x10, x13, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x9, x16
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x11, #1
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    cmp x8, x16
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x10, #1
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x9, x16, ne
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x9
+; CHECK-FP16-GI-NEXT:    csel x8, x8, x16, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x8
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 9223372036854775807, i128 9223372036854775807>
@@ -556,35 +1747,87 @@ entry:
 }
 
 define <2 x i64> @utesth_f16i64(<2 x half> %x) {
-; CHECK-LABEL: utesth_f16i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov h0, v0.h[1]
-; CHECK-NEXT:    bl __fixunshfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $q0
-; CHECK-NEXT:    bl __fixunshfti
-; CHECK-NEXT:    cmp x1, #0
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, eq
-; CHECK-NEXT:    cmp x20, #0
-; CHECK-NEXT:    csel x9, x19, xzr, eq
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utesth_f16i64:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-CVT-SD-NEXT:    bl __fixunshfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixunshfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #0
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-CVT-SD-NEXT:    cmp x20, #0
+; CHECK-CVT-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x9
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utesth_f16i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-FP16-SD-NEXT:    bl __fixunshfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixunshfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #0
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-FP16-SD-NEXT:    cmp x20, #0
+; CHECK-FP16-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utesth_f16i64:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcvtzu x8, s0
+; CHECK-CVT-GI-NEXT:    fcvtzu x9, s1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utesth_f16i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvtzu x8, h0
+; CHECK-FP16-GI-NEXT:    fcvtzu x9, h1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <2 x half> %x to <2 x i128>
   %0 = icmp ult <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -594,43 +1837,159 @@ entry:
 }
 
 define <2 x i64> @ustest_f16i64(<2 x half> %x) {
-; CHECK-LABEL: ustest_f16i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $q0
-; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    mov h0, v0.h[1]
-; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csinc x8, x1, xzr, lt
-; CHECK-NEXT:    csel x9, x0, xzr, lt
-; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csel x10, x19, xzr, lt
-; CHECK-NEXT:    csinc x11, x20, xzr, lt
-; CHECK-NEXT:    cmp xzr, x10
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ngcs xzr, x11
-; CHECK-NEXT:    csel x10, x10, xzr, lt
-; CHECK-NEXT:    cmp xzr, x9
-; CHECK-NEXT:    ngcs xzr, x8
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    csel x8, x9, xzr, lt
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f16i64:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixhfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-CVT-SD-NEXT:    bl __fixhfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #1
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csinc x8, x1, xzr, lt
+; CHECK-CVT-SD-NEXT:    csel x9, x0, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp x20, #1
+; CHECK-CVT-SD-NEXT:    csel x10, x19, xzr, lt
+; CHECK-CVT-SD-NEXT:    csinc x11, x20, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp xzr, x10
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    ngcs xzr, x11
+; CHECK-CVT-SD-NEXT:    csel x10, x10, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp xzr, x9
+; CHECK-CVT-SD-NEXT:    ngcs xzr, x8
+; CHECK-CVT-SD-NEXT:    fmov d0, x10
+; CHECK-CVT-SD-NEXT:    csel x8, x9, xzr, lt
+; CHECK-CVT-SD-NEXT:    fmov d1, x8
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f16i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixhfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-FP16-SD-NEXT:    bl __fixhfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #1
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csinc x8, x1, xzr, lt
+; CHECK-FP16-SD-NEXT:    csel x9, x0, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp x20, #1
+; CHECK-FP16-SD-NEXT:    csel x10, x19, xzr, lt
+; CHECK-FP16-SD-NEXT:    csinc x11, x20, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp xzr, x10
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ngcs xzr, x11
+; CHECK-FP16-SD-NEXT:    csel x10, x10, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp xzr, x9
+; CHECK-FP16-SD-NEXT:    ngcs xzr, x8
+; CHECK-FP16-SD-NEXT:    fmov d0, x10
+; CHECK-FP16-SD-NEXT:    csel x8, x9, xzr, lt
+; CHECK-FP16-SD-NEXT:    fmov d1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f16i64:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcvtzs x8, s0
+; CHECK-CVT-GI-NEXT:    fcvtzs x9, s1
+; CHECK-CVT-GI-NEXT:    asr x10, x8, #63
+; CHECK-CVT-GI-NEXT:    cmp x10, #1
+; CHECK-CVT-GI-NEXT:    asr x11, x9, #63
+; CHECK-CVT-GI-NEXT:    cset w12, lt
+; CHECK-CVT-GI-NEXT:    csel w12, wzr, w12, eq
+; CHECK-CVT-GI-NEXT:    cmp x11, #1
+; CHECK-CVT-GI-NEXT:    cset w13, lt
+; CHECK-CVT-GI-NEXT:    csel w13, wzr, w13, eq
+; CHECK-CVT-GI-NEXT:    tst w12, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x10, x10, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w13, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x11, x11, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w10, gt
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w11, gt
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f16i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvtzs x8, h0
+; CHECK-FP16-GI-NEXT:    fcvtzs x9, h1
+; CHECK-FP16-GI-NEXT:    asr x10, x8, #63
+; CHECK-FP16-GI-NEXT:    cmp x10, #1
+; CHECK-FP16-GI-NEXT:    cset w12, lt
+; CHECK-FP16-GI-NEXT:    asr x11, x9, #63
+; CHECK-FP16-GI-NEXT:    csel w12, wzr, w12, eq
+; CHECK-FP16-GI-NEXT:    cmp x11, #1
+; CHECK-FP16-GI-NEXT:    cset w13, lt
+; CHECK-FP16-GI-NEXT:    csel w13, wzr, w13, eq
+; CHECK-FP16-GI-NEXT:    tst w12, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x10, x10, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w13, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x11, x11, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w10, gt
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w11, gt
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %0 = icmp slt <2 x i128> %conv, <i128 18446744073709551616, i128 18446744073709551616>
@@ -660,15 +2019,43 @@ entry:
 }
 
 define <2 x i32> @utest_f64i32_mm(<2 x double> %x) {
-; CHECK-LABEL: utest_f64i32_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov d1, v0.d[1]
-; CHECK-NEXT:    fcvtzu w8, d0
-; CHECK-NEXT:    fcvtzu w9, d1
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    mov v0.s[1], w9
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utest_f64i32_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-CVT-SD-NEXT:    fcvtzu w8, d0
+; CHECK-CVT-SD-NEXT:    fcvtzu w9, d1
+; CHECK-CVT-SD-NEXT:    fmov s0, w8
+; CHECK-CVT-SD-NEXT:    mov v0.s[1], w9
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utest_f64i32_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-SD-NEXT:    fcvtzu w8, d0
+; CHECK-FP16-SD-NEXT:    fcvtzu w9, d1
+; CHECK-FP16-SD-NEXT:    fmov s0, w8
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w9
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utest_f64i32_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmhi v2.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-CVT-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utest_f64i32_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmhi v2.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-FP16-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <2 x double> %x to <2 x i64>
   %spec.store.select = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %conv, <2 x i64> <i64 4294967295, i64 4294967295>)
@@ -691,10 +2078,57 @@ entry:
 }
 
 define <4 x i32> @stest_f32i32_mm(<4 x float> %x) {
-; CHECK-LABEL: stest_f32i32_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f32i32_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f32i32_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f32i32_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    adrp x8, .LCPI30_1
+; CHECK-CVT-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI30_1]
+; CHECK-CVT-GI-NEXT:    adrp x8, .LCPI30_0
+; CHECK-CVT-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v2.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI30_0]
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v0.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f32i32_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    adrp x8, .LCPI30_1
+; CHECK-FP16-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI30_1]
+; CHECK-FP16-GI-NEXT:    adrp x8, .LCPI30_0
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI30_0]
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <4 x float> %x to <4 x i64>
   %spec.store.select = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %conv, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
@@ -704,10 +2138,43 @@ entry:
 }
 
 define <4 x i32> @utest_f32i32_mm(<4 x float> %x) {
-; CHECK-LABEL: utest_f32i32_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utest_f32i32_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utest_f32i32_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utest_f32i32_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmhi v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmhi v4.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v2.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utest_f32i32_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmhi v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    cmhi v4.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <4 x float> %x to <4 x i64>
   %spec.store.select = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %conv, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
@@ -716,10 +2183,51 @@ entry:
 }
 
 define <4 x i32> @ustest_f32i32_mm(<4 x float> %x) {
-; CHECK-LABEL: ustest_f32i32_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f32i32_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f32i32_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f32i32_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    cmgt v1.2d, v2.2d, #0
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v0.2d, #0
+; CHECK-CVT-GI-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f32i32_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    cmgt v1.2d, v2.2d, #0
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v0.2d, #0
+; CHECK-FP16-GI-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <4 x float> %x to <4 x i64>
   %spec.store.select = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %conv, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
@@ -729,11 +2237,68 @@ entry:
 }
 
 define <4 x i32> @stest_f16i32_mm(<4 x half> %x) {
-; CHECK-LABEL: stest_f16i32_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f16i32_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f16i32_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f16i32_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-GI-NEXT:    adrp x8, .LCPI33_1
+; CHECK-CVT-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI33_1]
+; CHECK-CVT-GI-NEXT:    adrp x8, .LCPI33_0
+; CHECK-CVT-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v2.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI33_0]
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v0.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v2.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f16i32_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT:    adrp x8, .LCPI33_1
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d1, h1
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI33_1]
+; CHECK-FP16-GI-NEXT:    adrp x8, .LCPI33_0
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI33_0]
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bif v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <4 x half> %x to <4 x i64>
   %spec.store.select = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %conv, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
@@ -743,11 +2308,54 @@ entry:
 }
 
 define <4 x i32> @utesth_f16i32_mm(<4 x half> %x) {
-; CHECK-LABEL: utesth_f16i32_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utesth_f16i32_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utesth_f16i32_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utesth_f16i32_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmhi v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmhi v4.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v2.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utesth_f16i32_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    fcvt d4, h4
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    cmhi v3.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmhi v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bit v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <4 x half> %x to <4 x i64>
   %spec.store.select = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %conv, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
@@ -756,11 +2364,62 @@ entry:
 }
 
 define <4 x i32> @ustest_f16i32_mm(<4 x half> %x) {
-; CHECK-LABEL: ustest_f16i32_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f16i32_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f16i32_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f16i32_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    cmgt v4.2d, v1.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    bif v2.16b, v1.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    bif v0.16b, v1.16b, v4.16b
+; CHECK-CVT-GI-NEXT:    cmgt v1.2d, v2.2d, #0
+; CHECK-CVT-GI-NEXT:    cmgt v3.2d, v0.2d, #0
+; CHECK-CVT-GI-NEXT:    and v1.16b, v2.16b, v1.16b
+; CHECK-CVT-GI-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-CVT-GI-NEXT:    uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f16i32_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000ffffffff
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    fcvt d4, h4
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT:    mov v3.d[1], v4.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v1.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    cmgt v4.2d, v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    bif v0.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    bit v1.16b, v2.16b, v4.16b
+; CHECK-FP16-GI-NEXT:    cmgt v2.2d, v0.2d, #0
+; CHECK-FP16-GI-NEXT:    cmgt v3.2d, v1.2d, #0
+; CHECK-FP16-GI-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-FP16-GI-NEXT:    and v1.16b, v1.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <4 x half> %x to <4 x i64>
   %spec.store.select = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %conv, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
@@ -772,15 +2431,45 @@ entry:
 ; i16 saturate
 
 define <2 x i16> @stest_f64i16_mm(<2 x double> %x) {
-; CHECK-LABEL: stest_f64i16_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-NEXT:    movi v1.2s, #127, msl #8
-; CHECK-NEXT:    xtn v0.2s, v0.2d
-; CHECK-NEXT:    smin v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    mvni v1.2s, #127, msl #8
-; CHECK-NEXT:    smax v0.2s, v0.2s, v1.2s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f64i16_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-CVT-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-CVT-SD-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-CVT-SD-NEXT:    mvni v1.2s, #127, msl #8
+; CHECK-CVT-SD-NEXT:    smax v0.2s, v0.2s, v1.2s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f64i16_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-FP16-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-FP16-SD-NEXT:    mvni v1.2s, #127, msl #8
+; CHECK-FP16-SD-NEXT:    smax v0.2s, v0.2s, v1.2s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f64i16_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-GI-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-CVT-GI-NEXT:    mvni v2.2s, #127, msl #8
+; CHECK-CVT-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-CVT-GI-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-CVT-GI-NEXT:    smax v0.2s, v0.2s, v2.2s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f64i16_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    movi v1.2s, #127, msl #8
+; CHECK-FP16-GI-NEXT:    mvni v2.2s, #127, msl #8
+; CHECK-FP16-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    smin v0.2s, v0.2s, v1.2s
+; CHECK-FP16-GI-NEXT:    smax v0.2s, v0.2s, v2.2s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i32>
   %spec.store.select = call <2 x i32> @llvm.smin.v2i32(<2 x i32> %conv, <2 x i32> <i32 32767, i32 32767>)
@@ -874,10 +2563,20 @@ define <8 x i16> @stest_f16i16_mm(<8 x half> %x) {
 ; CHECK-CVT-NEXT:    sqxtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-FP16-LABEL: stest_f16i16_mm:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-FP16-SD-LABEL: stest_f16i16_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f16i16_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    sqxtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    sqxtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <8 x half> %x to <8 x i32>
   %spec.store.select = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %conv, <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>)
@@ -897,10 +2596,20 @@ define <8 x i16> @utesth_f16i16_mm(<8 x half> %x) {
 ; CHECK-CVT-NEXT:    uqxtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-FP16-LABEL: utesth_f16i16_mm:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-FP16-SD-LABEL: utesth_f16i16_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utesth_f16i16_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    uqxtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <8 x half> %x to <8 x i32>
   %spec.store.select = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %conv, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>)
@@ -919,10 +2628,20 @@ define <8 x i16> @ustest_f16i16_mm(<8 x half> %x) {
 ; CHECK-CVT-NEXT:    sqxtun2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-FP16-LABEL: ustest_f16i16_mm:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-FP16-NEXT:    ret
+; CHECK-FP16-SD-LABEL: ustest_f16i16_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f16i16_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    sqxtun v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    sqxtun2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <8 x half> %x to <8 x i32>
   %spec.store.select = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %conv, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>)
@@ -934,10 +2653,139 @@ entry:
 ; i64 saturate
 
 define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
-; CHECK-LABEL: stest_f64i64_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f64i64_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f64i64_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f64i64_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -40
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -48
+; CHECK-CVT-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-CVT-GI-NEXT:    mov x21, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-CVT-GI-NEXT:    mov x22, #-9223372036854775808 // =0x8000000000000000
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixdfti
+; CHECK-CVT-GI-NEXT:    fmov d0, d8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixdfti
+; CHECK-CVT-GI-NEXT:    cmp x19, x21
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lo
+; CHECK-CVT-GI-NEXT:    cmp x20, #0
+; CHECK-CVT-GI-NEXT:    cset w9, mi
+; CHECK-CVT-GI-NEXT:    csel w8, w8, w9, eq
+; CHECK-CVT-GI-NEXT:    cmp x0, x21
+; CHECK-CVT-GI-NEXT:    cset w9, lo
+; CHECK-CVT-GI-NEXT:    cmp x1, #0
+; CHECK-CVT-GI-NEXT:    cset w10, mi
+; CHECK-CVT-GI-NEXT:    csel w9, w9, w10, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, x21, ne
+; CHECK-CVT-GI-NEXT:    csel x10, x20, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x0, x21, ne
+; CHECK-CVT-GI-NEXT:    csel x11, x1, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, x22
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x10, #1
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, x22
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x11, #1
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, x22, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, x22, ne
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #48 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f64i64_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -40
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -48
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    mov x21, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-FP16-GI-NEXT:    mov x22, #-9223372036854775808 // =0x8000000000000000
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    fmov d0, d8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    cmp x19, x21
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lo
+; CHECK-FP16-GI-NEXT:    cmp x20, #0
+; CHECK-FP16-GI-NEXT:    cset w9, mi
+; CHECK-FP16-GI-NEXT:    csel w8, w8, w9, eq
+; CHECK-FP16-GI-NEXT:    cmp x0, x21
+; CHECK-FP16-GI-NEXT:    cset w9, lo
+; CHECK-FP16-GI-NEXT:    cmp x1, #0
+; CHECK-FP16-GI-NEXT:    cset w10, mi
+; CHECK-FP16-GI-NEXT:    csel w9, w9, w10, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, x21, ne
+; CHECK-FP16-GI-NEXT:    csel x10, x20, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x0, x21, ne
+; CHECK-FP16-GI-NEXT:    csel x11, x1, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, x22
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x10, #1
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, x22
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x11, #1
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, x22, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, x22, ne
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #48 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 9223372036854775807, i128 9223372036854775807>)
@@ -947,34 +2795,131 @@ entry:
 }
 
 define <2 x i64> @utest_f64i64_mm(<2 x double> %x) {
-; CHECK-LABEL: utest_f64i64_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov d0, v0.d[1]
-; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    bl __fixunsdfti
-; CHECK-NEXT:    cmp x1, #0
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, eq
-; CHECK-NEXT:    cmp x20, #0
-; CHECK-NEXT:    csel x9, x19, xzr, eq
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utest_f64i64_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-CVT-SD-NEXT:    bl __fixunsdfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixunsdfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #0
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-CVT-SD-NEXT:    cmp x20, #0
+; CHECK-CVT-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x9
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utest_f64i64_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-FP16-SD-NEXT:    bl __fixunsdfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixunsdfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #0
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-FP16-SD-NEXT:    cmp x20, #0
+; CHECK-FP16-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utest_f64i64_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-CVT-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixunsdfti
+; CHECK-CVT-GI-NEXT:    fmov d0, d8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixunsdfti
+; CHECK-CVT-GI-NEXT:    cmp x20, #1
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lo
+; CHECK-CVT-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-CVT-GI-NEXT:    cmp x1, #1
+; CHECK-CVT-GI-NEXT:    cset w9, lo
+; CHECK-CVT-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utest_f64i64_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixunsdfti
+; CHECK-FP16-GI-NEXT:    fmov d0, d8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixunsdfti
+; CHECK-FP16-GI-NEXT:    cmp x20, #1
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lo
+; CHECK-FP16-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-FP16-GI-NEXT:    cmp x1, #1
+; CHECK-FP16-GI-NEXT:    cset w9, lo
+; CHECK-FP16-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <2 x double> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.umin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -983,40 +2928,175 @@ entry:
 }
 
 define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
-; CHECK-LABEL: ustest_f64i64_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    mov d0, v0.d[1]
-; CHECK-NEXT:    bl __fixdfti
-; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
-; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csinc x10, x20, xzr, lt
-; CHECK-NEXT:    csel x11, x19, xzr, lt
-; CHECK-NEXT:    cmp x10, #0
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    csel x10, xzr, x11, lt
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    csel x8, xzr, x8, lt
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f64i64_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixdfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-CVT-SD-NEXT:    bl __fixdfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #1
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, lt
+; CHECK-CVT-SD-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp x20, #1
+; CHECK-CVT-SD-NEXT:    csinc x10, x20, xzr, lt
+; CHECK-CVT-SD-NEXT:    csel x11, x19, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp x10, #0
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x10, xzr, x11, mi
+; CHECK-CVT-SD-NEXT:    cmp x9, #0
+; CHECK-CVT-SD-NEXT:    csel x8, xzr, x8, mi
+; CHECK-CVT-SD-NEXT:    fmov d0, x10
+; CHECK-CVT-SD-NEXT:    fmov d1, x8
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f64i64_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixdfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-FP16-SD-NEXT:    bl __fixdfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #1
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, lt
+; CHECK-FP16-SD-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp x20, #1
+; CHECK-FP16-SD-NEXT:    csinc x10, x20, xzr, lt
+; CHECK-FP16-SD-NEXT:    csel x11, x19, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp x10, #0
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x10, xzr, x11, mi
+; CHECK-FP16-SD-NEXT:    cmp x9, #0
+; CHECK-FP16-SD-NEXT:    csel x8, xzr, x8, mi
+; CHECK-FP16-SD-NEXT:    fmov d0, x10
+; CHECK-FP16-SD-NEXT:    fmov d1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f64i64_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-CVT-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixdfti
+; CHECK-CVT-GI-NEXT:    fmov d0, d8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixdfti
+; CHECK-CVT-GI-NEXT:    cmp x20, #1
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lt
+; CHECK-CVT-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-CVT-GI-NEXT:    cmp x1, #1
+; CHECK-CVT-GI-NEXT:    cset w9, lt
+; CHECK-CVT-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x10, x20, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x11, x1, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w10, gt
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w11, gt
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f64i64_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    fmov d0, d8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    cmp x20, #1
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lt
+; CHECK-FP16-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-FP16-GI-NEXT:    cmp x1, #1
+; CHECK-FP16-GI-NEXT:    cset w9, lt
+; CHECK-FP16-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x10, x20, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x11, x1, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w10, gt
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w11, gt
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x double> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -1026,11 +3106,143 @@ entry:
 }
 
 define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
-; CHECK-LABEL: stest_f32i64_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f32i64_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stest_f32i64_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f32i64_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -40
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -48
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-CVT-GI-NEXT:    mov x21, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-CVT-GI-NEXT:    mov x22, #-9223372036854775808 // =0x8000000000000000
+; CHECK-CVT-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixsfti
+; CHECK-CVT-GI-NEXT:    fmov s0, s8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixsfti
+; CHECK-CVT-GI-NEXT:    cmp x19, x21
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lo
+; CHECK-CVT-GI-NEXT:    cmp x20, #0
+; CHECK-CVT-GI-NEXT:    cset w9, mi
+; CHECK-CVT-GI-NEXT:    csel w8, w8, w9, eq
+; CHECK-CVT-GI-NEXT:    cmp x0, x21
+; CHECK-CVT-GI-NEXT:    cset w9, lo
+; CHECK-CVT-GI-NEXT:    cmp x1, #0
+; CHECK-CVT-GI-NEXT:    cset w10, mi
+; CHECK-CVT-GI-NEXT:    csel w9, w9, w10, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, x21, ne
+; CHECK-CVT-GI-NEXT:    csel x10, x20, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x0, x21, ne
+; CHECK-CVT-GI-NEXT:    csel x11, x1, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, x22
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x10, #1
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, x22
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x11, #1
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, x22, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, x22, ne
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #48 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f32i64_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w21, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset w22, -32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -40
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -48
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    mov x21, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-FP16-GI-NEXT:    mov x22, #-9223372036854775808 // =0x8000000000000000
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    fmov s0, s8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    cmp x19, x21
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lo
+; CHECK-FP16-GI-NEXT:    cmp x20, #0
+; CHECK-FP16-GI-NEXT:    cset w9, mi
+; CHECK-FP16-GI-NEXT:    csel w8, w8, w9, eq
+; CHECK-FP16-GI-NEXT:    cmp x0, x21
+; CHECK-FP16-GI-NEXT:    cset w9, lo
+; CHECK-FP16-GI-NEXT:    cmp x1, #0
+; CHECK-FP16-GI-NEXT:    cset w10, mi
+; CHECK-FP16-GI-NEXT:    csel w9, w9, w10, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, x21, ne
+; CHECK-FP16-GI-NEXT:    csel x10, x20, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x0, x21, ne
+; CHECK-FP16-GI-NEXT:    csel x11, x1, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, x22
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x10, #1
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, x22
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x11, #1
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, x22, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, x22, ne
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #48 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 9223372036854775807, i128 9223372036854775807>)
@@ -1040,35 +3252,135 @@ entry:
 }
 
 define <2 x i64> @utest_f32i64_mm(<2 x float> %x) {
-; CHECK-LABEL: utest_f32i64_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov s0, v0.s[1]
-; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT:    bl __fixunssfti
-; CHECK-NEXT:    cmp x1, #0
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, eq
-; CHECK-NEXT:    cmp x20, #0
-; CHECK-NEXT:    csel x9, x19, xzr, eq
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utest_f32i64_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-CVT-SD-NEXT:    bl __fixunssfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixunssfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #0
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-CVT-SD-NEXT:    cmp x20, #0
+; CHECK-CVT-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x9
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utest_f32i64_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-FP16-SD-NEXT:    bl __fixunssfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixunssfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #0
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-FP16-SD-NEXT:    cmp x20, #0
+; CHECK-FP16-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utest_f32i64_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-CVT-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixunssfti
+; CHECK-CVT-GI-NEXT:    fmov s0, s8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixunssfti
+; CHECK-CVT-GI-NEXT:    cmp x20, #1
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lo
+; CHECK-CVT-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-CVT-GI-NEXT:    cmp x1, #1
+; CHECK-CVT-GI-NEXT:    cset w9, lo
+; CHECK-CVT-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utest_f32i64_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixunssfti
+; CHECK-FP16-GI-NEXT:    fmov s0, s8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixunssfti
+; CHECK-FP16-GI-NEXT:    cmp x20, #1
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lo
+; CHECK-FP16-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-FP16-GI-NEXT:    cmp x1, #1
+; CHECK-FP16-GI-NEXT:    cset w9, lo
+; CHECK-FP16-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <2 x float> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.umin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -1077,41 +3389,179 @@ entry:
 }
 
 define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
-; CHECK-LABEL: ustest_f32i64_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    mov s0, v0.s[1]
-; CHECK-NEXT:    bl __fixsfti
-; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
-; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csinc x10, x20, xzr, lt
-; CHECK-NEXT:    csel x11, x19, xzr, lt
-; CHECK-NEXT:    cmp x10, #0
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    csel x10, xzr, x11, lt
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    csel x8, xzr, x8, lt
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f32i64_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixsfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-CVT-SD-NEXT:    bl __fixsfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #1
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, lt
+; CHECK-CVT-SD-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp x20, #1
+; CHECK-CVT-SD-NEXT:    csinc x10, x20, xzr, lt
+; CHECK-CVT-SD-NEXT:    csel x11, x19, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp x10, #0
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x10, xzr, x11, mi
+; CHECK-CVT-SD-NEXT:    cmp x9, #0
+; CHECK-CVT-SD-NEXT:    csel x8, xzr, x8, mi
+; CHECK-CVT-SD-NEXT:    fmov d0, x10
+; CHECK-CVT-SD-NEXT:    fmov d1, x8
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f32i64_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixsfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-FP16-SD-NEXT:    bl __fixsfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #1
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, lt
+; CHECK-FP16-SD-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp x20, #1
+; CHECK-FP16-SD-NEXT:    csinc x10, x20, xzr, lt
+; CHECK-FP16-SD-NEXT:    csel x11, x19, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp x10, #0
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x10, xzr, x11, mi
+; CHECK-FP16-SD-NEXT:    cmp x9, #0
+; CHECK-FP16-SD-NEXT:    csel x8, xzr, x8, mi
+; CHECK-FP16-SD-NEXT:    fmov d0, x10
+; CHECK-FP16-SD-NEXT:    fmov d1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f32i64_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-CVT-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CVT-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-CVT-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-CVT-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-CVT-GI-NEXT:    bl __fixsfti
+; CHECK-CVT-GI-NEXT:    fmov s0, s8
+; CHECK-CVT-GI-NEXT:    mov x19, x0
+; CHECK-CVT-GI-NEXT:    mov x20, x1
+; CHECK-CVT-GI-NEXT:    bl __fixsfti
+; CHECK-CVT-GI-NEXT:    cmp x20, #1
+; CHECK-CVT-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    cset w8, lt
+; CHECK-CVT-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-CVT-GI-NEXT:    cmp x1, #1
+; CHECK-CVT-GI-NEXT:    cset w9, lt
+; CHECK-CVT-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-CVT-GI-NEXT:    tst w8, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x10, x20, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w9, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x11, x1, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w10, gt
+; CHECK-CVT-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w11, gt
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f32i64_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    fmov s0, s8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    cmp x20, #1
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    cset w8, lt
+; CHECK-FP16-GI-NEXT:    csel w8, wzr, w8, eq
+; CHECK-FP16-GI-NEXT:    cmp x1, #1
+; CHECK-FP16-GI-NEXT:    cset w9, lt
+; CHECK-FP16-GI-NEXT:    csel w9, wzr, w9, eq
+; CHECK-FP16-GI-NEXT:    tst w8, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x19, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x10, x20, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w9, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x0, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x11, x1, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w10, gt
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w11, gt
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x float> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -1121,27 +3571,121 @@ entry:
 }
 
 define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
-; CHECK-CVT-LABEL: stest_f16i64_mm:
-; CHECK-CVT:       // %bb.0: // %entry
-; CHECK-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-CVT-NEXT:    fcvt s0, h0
-; CHECK-CVT-NEXT:    fcvt s1, h1
-; CHECK-CVT-NEXT:    fcvtzs x8, s0
-; CHECK-CVT-NEXT:    fcvtzs x9, s1
-; CHECK-CVT-NEXT:    fmov d0, x8
-; CHECK-CVT-NEXT:    mov v0.d[1], x9
-; CHECK-CVT-NEXT:    ret
+; CHECK-CVT-SD-LABEL: stest_f16i64_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-SD-NEXT:    fcvt s0, h0
+; CHECK-CVT-SD-NEXT:    fcvt s1, h1
+; CHECK-CVT-SD-NEXT:    fcvtzs x8, s0
+; CHECK-CVT-SD-NEXT:    fcvtzs x9, s1
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-SD-NEXT:    ret
 ;
-; CHECK-FP16-LABEL: stest_f16i64_mm:
-; CHECK-FP16:       // %bb.0: // %entry
-; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-FP16-NEXT:    fcvtzs x8, h0
-; CHECK-FP16-NEXT:    fcvtzs x9, h1
-; CHECK-FP16-NEXT:    fmov d0, x8
-; CHECK-FP16-NEXT:    mov v0.d[1], x9
-; CHECK-FP16-NEXT:    ret
+; CHECK-FP16-SD-LABEL: stest_f16i64_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    fcvtzs x8, h0
+; CHECK-FP16-SD-NEXT:    fcvtzs x9, h1
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: stest_f16i64_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-GI-NEXT:    mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-CVT-GI-NEXT:    mov x16, #-9223372036854775808 // =0x8000000000000000
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcvtzs x9, s0
+; CHECK-CVT-GI-NEXT:    fcvtzs x10, s1
+; CHECK-CVT-GI-NEXT:    asr x11, x9, #63
+; CHECK-CVT-GI-NEXT:    cmp x9, x8
+; CHECK-CVT-GI-NEXT:    cset w12, lo
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    asr x13, x10, #63
+; CHECK-CVT-GI-NEXT:    cset w14, mi
+; CHECK-CVT-GI-NEXT:    csel w12, w12, w14, eq
+; CHECK-CVT-GI-NEXT:    cmp x10, x8
+; CHECK-CVT-GI-NEXT:    cset w14, lo
+; CHECK-CVT-GI-NEXT:    cmp x13, #0
+; CHECK-CVT-GI-NEXT:    cset w15, mi
+; CHECK-CVT-GI-NEXT:    csel w14, w14, w15, eq
+; CHECK-CVT-GI-NEXT:    tst w12, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x9, x8, ne
+; CHECK-CVT-GI-NEXT:    csel x11, x11, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w14, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x10, x8, ne
+; CHECK-CVT-GI-NEXT:    csel x10, x13, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x9, x16
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x11, #1
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    cmp x8, x16
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w13, pl
+; CHECK-CVT-GI-NEXT:    cmn x10, #1
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x9, x16, ne
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x9
+; CHECK-CVT-GI-NEXT:    csel x8, x8, x16, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x8
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stest_f16i64_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvtzs x9, h0
+; CHECK-FP16-GI-NEXT:    mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-FP16-GI-NEXT:    mov x16, #-9223372036854775808 // =0x8000000000000000
+; CHECK-FP16-GI-NEXT:    fcvtzs x10, h1
+; CHECK-FP16-GI-NEXT:    asr x11, x9, #63
+; CHECK-FP16-GI-NEXT:    cmp x9, x8
+; CHECK-FP16-GI-NEXT:    cset w12, lo
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w14, mi
+; CHECK-FP16-GI-NEXT:    asr x13, x10, #63
+; CHECK-FP16-GI-NEXT:    csel w12, w12, w14, eq
+; CHECK-FP16-GI-NEXT:    cmp x10, x8
+; CHECK-FP16-GI-NEXT:    cset w14, lo
+; CHECK-FP16-GI-NEXT:    cmp x13, #0
+; CHECK-FP16-GI-NEXT:    cset w15, mi
+; CHECK-FP16-GI-NEXT:    csel w14, w14, w15, eq
+; CHECK-FP16-GI-NEXT:    tst w12, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x9, x8, ne
+; CHECK-FP16-GI-NEXT:    csel x11, x11, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w14, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x10, x8, ne
+; CHECK-FP16-GI-NEXT:    csel x10, x13, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x9, x16
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x11, #1
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    cmp x8, x16
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w13, pl
+; CHECK-FP16-GI-NEXT:    cmn x10, #1
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w13, eq
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x9, x16, ne
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x9
+; CHECK-FP16-GI-NEXT:    csel x8, x8, x16, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x8
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 9223372036854775807, i128 9223372036854775807>)
@@ -1151,35 +3695,87 @@ entry:
 }
 
 define <2 x i64> @utesth_f16i64_mm(<2 x half> %x) {
-; CHECK-LABEL: utesth_f16i64_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov h0, v0.h[1]
-; CHECK-NEXT:    bl __fixunshfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $q0
-; CHECK-NEXT:    bl __fixunshfti
-; CHECK-NEXT:    cmp x1, #0
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, eq
-; CHECK-NEXT:    cmp x20, #0
-; CHECK-NEXT:    csel x9, x19, xzr, eq
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: utesth_f16i64_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-CVT-SD-NEXT:    bl __fixunshfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixunshfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #0
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-CVT-SD-NEXT:    cmp x20, #0
+; CHECK-CVT-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x9
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utesth_f16i64_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-FP16-SD-NEXT:    bl __fixunshfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixunshfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #0
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, eq
+; CHECK-FP16-SD-NEXT:    cmp x20, #0
+; CHECK-FP16-SD-NEXT:    csel x9, x19, xzr, eq
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: utesth_f16i64_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcvtzu x8, s0
+; CHECK-CVT-GI-NEXT:    fcvtzu x9, s1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utesth_f16i64_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvtzu x8, h0
+; CHECK-FP16-GI-NEXT:    fcvtzu x9, h1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptoui <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.umin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -1188,41 +3784,155 @@ entry:
 }
 
 define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
-; CHECK-LABEL: ustest_f16i64_mm:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    // kill: def $h0 killed $h0 killed $q0
-; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    mov h0, v0.h[1]
-; CHECK-NEXT:    bl __fixhfti
-; CHECK-NEXT:    cmp x1, #1
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    csel x8, x0, xzr, lt
-; CHECK-NEXT:    csinc x9, x1, xzr, lt
-; CHECK-NEXT:    cmp x20, #1
-; CHECK-NEXT:    csinc x10, x20, xzr, lt
-; CHECK-NEXT:    csel x11, x19, xzr, lt
-; CHECK-NEXT:    cmp x10, #0
-; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    csel x10, xzr, x11, lt
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    csel x8, xzr, x8, lt
-; CHECK-NEXT:    fmov d0, x10
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: ustest_f16i64_mm:
+; CHECK-CVT-SD:       // %bb.0: // %entry
+; CHECK-CVT-SD-NEXT:    sub sp, sp, #48
+; CHECK-CVT-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-CVT-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-CVT-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-CVT-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-CVT-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-CVT-SD-NEXT:    bl __fixhfti
+; CHECK-CVT-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    mov x19, x0
+; CHECK-CVT-SD-NEXT:    mov x20, x1
+; CHECK-CVT-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-CVT-SD-NEXT:    bl __fixhfti
+; CHECK-CVT-SD-NEXT:    cmp x1, #1
+; CHECK-CVT-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x8, x0, xzr, lt
+; CHECK-CVT-SD-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp x20, #1
+; CHECK-CVT-SD-NEXT:    csinc x10, x20, xzr, lt
+; CHECK-CVT-SD-NEXT:    csel x11, x19, xzr, lt
+; CHECK-CVT-SD-NEXT:    cmp x10, #0
+; CHECK-CVT-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-CVT-SD-NEXT:    csel x10, xzr, x11, mi
+; CHECK-CVT-SD-NEXT:    cmp x9, #0
+; CHECK-CVT-SD-NEXT:    csel x8, xzr, x8, mi
+; CHECK-CVT-SD-NEXT:    fmov d0, x10
+; CHECK-CVT-SD-NEXT:    fmov d1, x8
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-SD-NEXT:    add sp, sp, #48
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: ustest_f16i64_mm:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixhfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-FP16-SD-NEXT:    bl __fixhfti
+; CHECK-FP16-SD-NEXT:    cmp x1, #1
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x8, x0, xzr, lt
+; CHECK-FP16-SD-NEXT:    csinc x9, x1, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp x20, #1
+; CHECK-FP16-SD-NEXT:    csinc x10, x20, xzr, lt
+; CHECK-FP16-SD-NEXT:    csel x11, x19, xzr, lt
+; CHECK-FP16-SD-NEXT:    cmp x10, #0
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    csel x10, xzr, x11, mi
+; CHECK-FP16-SD-NEXT:    cmp x9, #0
+; CHECK-FP16-SD-NEXT:    csel x8, xzr, x8, mi
+; CHECK-FP16-SD-NEXT:    fmov d0, x10
+; CHECK-FP16-SD-NEXT:    fmov d1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: ustest_f16i64_mm:
+; CHECK-CVT-GI:       // %bb.0: // %entry
+; CHECK-CVT-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-GI-NEXT:    fcvt s0, h0
+; CHECK-CVT-GI-NEXT:    fcvt s1, h1
+; CHECK-CVT-GI-NEXT:    fcvtzs x8, s0
+; CHECK-CVT-GI-NEXT:    fcvtzs x9, s1
+; CHECK-CVT-GI-NEXT:    asr x10, x8, #63
+; CHECK-CVT-GI-NEXT:    cmp x10, #1
+; CHECK-CVT-GI-NEXT:    asr x11, x9, #63
+; CHECK-CVT-GI-NEXT:    cset w12, lt
+; CHECK-CVT-GI-NEXT:    csel w12, wzr, w12, eq
+; CHECK-CVT-GI-NEXT:    cmp x11, #1
+; CHECK-CVT-GI-NEXT:    cset w13, lt
+; CHECK-CVT-GI-NEXT:    csel w13, wzr, w13, eq
+; CHECK-CVT-GI-NEXT:    tst w12, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x10, x10, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w13, #0x1
+; CHECK-CVT-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-CVT-GI-NEXT:    csinc x11, x11, xzr, ne
+; CHECK-CVT-GI-NEXT:    cmp x8, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x10, #0
+; CHECK-CVT-GI-NEXT:    cset w10, gt
+; CHECK-CVT-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-CVT-GI-NEXT:    cmp x9, #0
+; CHECK-CVT-GI-NEXT:    cset w12, hi
+; CHECK-CVT-GI-NEXT:    cmp x11, #0
+; CHECK-CVT-GI-NEXT:    cset w11, gt
+; CHECK-CVT-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-CVT-GI-NEXT:    tst w10, #0x1
+; CHECK-CVT-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-CVT-GI-NEXT:    tst w11, #0x1
+; CHECK-CVT-GI-NEXT:    fmov d0, x8
+; CHECK-CVT-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-CVT-GI-NEXT:    mov v0.d[1], x9
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: ustest_f16i64_mm:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvtzs x8, h0
+; CHECK-FP16-GI-NEXT:    fcvtzs x9, h1
+; CHECK-FP16-GI-NEXT:    asr x10, x8, #63
+; CHECK-FP16-GI-NEXT:    cmp x10, #1
+; CHECK-FP16-GI-NEXT:    cset w12, lt
+; CHECK-FP16-GI-NEXT:    asr x11, x9, #63
+; CHECK-FP16-GI-NEXT:    csel w12, wzr, w12, eq
+; CHECK-FP16-GI-NEXT:    cmp x11, #1
+; CHECK-FP16-GI-NEXT:    cset w13, lt
+; CHECK-FP16-GI-NEXT:    csel w13, wzr, w13, eq
+; CHECK-FP16-GI-NEXT:    tst w12, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x10, x10, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w13, #0x1
+; CHECK-FP16-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-FP16-GI-NEXT:    csinc x11, x11, xzr, ne
+; CHECK-FP16-GI-NEXT:    cmp x8, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x10, #0
+; CHECK-FP16-GI-NEXT:    cset w10, gt
+; CHECK-FP16-GI-NEXT:    csel w10, w12, w10, eq
+; CHECK-FP16-GI-NEXT:    cmp x9, #0
+; CHECK-FP16-GI-NEXT:    cset w12, hi
+; CHECK-FP16-GI-NEXT:    cmp x11, #0
+; CHECK-FP16-GI-NEXT:    cset w11, gt
+; CHECK-FP16-GI-NEXT:    csel w11, w12, w11, eq
+; CHECK-FP16-GI-NEXT:    tst w10, #0x1
+; CHECK-FP16-GI-NEXT:    csel x8, x8, xzr, ne
+; CHECK-FP16-GI-NEXT:    tst w11, #0x1
+; CHECK-FP16-GI-NEXT:    fmov d0, x8
+; CHECK-FP16-GI-NEXT:    csel x9, x9, xzr, ne
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %conv = fptosi <2 x half> %x to <2 x i128>
   %spec.store.select = call <2 x i128> @llvm.smin.v2i128(<2 x i128> %conv, <2 x i128> <i128 18446744073709551616, i128 18446744073709551616>)
@@ -1249,3 +3959,5 @@ declare <4 x i64> @llvm.umin.v4i64(<4 x i64>, <4 x i64>)
 declare <2 x i128> @llvm.smin.v2i128(<2 x i128>, <2 x i128>)
 declare <2 x i128> @llvm.smax.v2i128(<2 x i128>, <2 x i128>)
 declare <2 x i128> @llvm.umin.v2i128(<2 x i128>, <2 x i128>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-FP16: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/fptoi.ll b/llvm/test/CodeGen/AArch64/fptoi.ll
index 6a792958ee6be..f6053cee50dae 100644
--- a/llvm/test/CodeGen/AArch64/fptoi.ll
+++ b/llvm/test/CodeGen/AArch64/fptoi.ll
@@ -1,8 +1,8 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16
-; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
-; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
-; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16,CHECK-NOFP16-SD
+; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-SD
+; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16,CHECK-NOFP16-GI
+; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-GI
 
 define i64 @fptos_f64_i64(double %a) {
 ; CHECK-LABEL: fptos_f64_i64:
@@ -55,15 +55,25 @@ entry:
 }
 
 define i16 @fptou_f64_i16(double %a) {
-; CHECK-SD-LABEL: fptou_f64_i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs w0, d0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_f64_i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu w0, d0
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_f64_i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs w0, d0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f64_i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w0, d0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f64_i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w0, d0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_f64_i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w0, d0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui double %a to i16
   ret i16 %c
@@ -80,15 +90,25 @@ entry:
 }
 
 define i8 @fptou_f64_i8(double %a) {
-; CHECK-SD-LABEL: fptou_f64_i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs w0, d0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_f64_i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu w0, d0
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_f64_i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs w0, d0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f64_i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w0, d0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f64_i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w0, d0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_f64_i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w0, d0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui double %a to i8
   ret i8 %c
@@ -173,15 +193,25 @@ entry:
 }
 
 define i16 @fptou_f32_i16(float %a) {
-; CHECK-SD-LABEL: fptou_f32_i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs w0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_f32_i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu w0, s0
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_f32_i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs w0, s0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f32_i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w0, s0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f32_i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w0, s0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_f32_i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w0, s0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui float %a to i16
   ret i16 %c
@@ -198,15 +228,25 @@ entry:
 }
 
 define i8 @fptou_f32_i8(float %a) {
-; CHECK-SD-LABEL: fptou_f32_i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs w0, s0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_f32_i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu w0, s0
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_f32_i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs w0, s0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f32_i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w0, s0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f32_i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w0, s0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_f32_i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w0, s0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui float %a to i8
   ret i8 %c
@@ -241,342 +281,346 @@ entry:
 }
 
 define i64 @fptos_f16_i64(half %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_f16_i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_f16_i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs x0, h0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_f16_i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvtzs x0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_f16_i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs x0, h0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptos_f16_i64:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-NEXT:    fcvtzs x0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_f16_i64:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs x0, h0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptosi half %a to i64
   ret i64 %c
 }
 
 define i64 @fptou_f16_i64(half %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_f16_i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_f16_i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu x0, h0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_f16_i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvtzu x0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_f16_i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu x0, h0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptou_f16_i64:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-NEXT:    fcvtzu x0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptou_f16_i64:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzu x0, h0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptoui half %a to i64
   ret i64 %c
 }
 
 define i32 @fptos_f16_i32(half %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_f16_i32:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_f16_i32:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_f16_i32:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvtzs w0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_f16_i32:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs w0, h0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptos_f16_i32:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-NEXT:    fcvtzs w0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_f16_i32:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptosi half %a to i32
   ret i32 %c
 }
 
 define i32 @fptou_f16_i32(half %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_f16_i32:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzu w0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_f16_i32:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu w0, h0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_f16_i32:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvtzu w0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_f16_i32:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu w0, h0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptou_f16_i32:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-NEXT:    fcvtzu w0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptou_f16_i32:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzu w0, h0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptoui half %a to i32
   ret i32 %c
 }
 
 define i16 @fptos_f16_i16(half %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_f16_i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_f16_i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_f16_i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvtzs w0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_f16_i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs w0, h0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptos_f16_i16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-NEXT:    fcvtzs w0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_f16_i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptosi half %a to i16
   ret i16 %c
 }
 
 define i16 @fptou_f16_i16(half %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_f16_i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_f16_i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_f16_i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvtzu w0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_f16_i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu w0, h0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_f16_i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs w0, s0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f16_i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w0, h0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f16_i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w0, s0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_f16_i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w0, h0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui half %a to i16
   ret i16 %c
 }
 
 define i8 @fptos_f16_i8(half %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_f16_i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_f16_i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_f16_i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvtzs w0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_f16_i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs w0, h0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptos_f16_i8:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-NEXT:    fcvtzs w0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_f16_i8:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs w0, h0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptosi half %a to i8
   ret i8 %c
 }
 
 define i8 @fptou_f16_i8(half %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_f16_i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs w0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_f16_i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs w0, h0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_f16_i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvtzu w0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_f16_i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu w0, h0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_f16_i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs w0, s0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f16_i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w0, h0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f16_i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w0, s0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_f16_i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w0, h0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui half %a to i8
   ret i8 %c
 }
 
 define i128 @fptos_f16_i128(half %a) {
-; CHECK-SD-LABEL: fptos_f16_i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __fixhfti
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_f16_i128:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvtzs x0, s0
-; CHECK-GI-NOFP16-NEXT:    asr x1, x0, #63
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_f16_i128:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs x0, h0
-; CHECK-GI-FP16-NEXT:    asr x1, x0, #63
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_f16_i128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __fixhfti
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_f16_i128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __fixhfti
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_f16_i128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvtzs x0, s0
+; CHECK-NOFP16-GI-NEXT:    asr x1, x0, #63
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_f16_i128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs x0, h0
+; CHECK-FP16-GI-NEXT:    asr x1, x0, #63
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi half %a to i128
   ret i128 %c
 }
 
 define i128 @fptou_f16_i128(half %a) {
-; CHECK-SD-LABEL: fptou_f16_i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __fixunshfti
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_f16_i128:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    mov x1, xzr
-; CHECK-GI-NOFP16-NEXT:    fcvtzu x0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_f16_i128:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu x0, h0
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_f16_i128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __fixunshfti
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f16_i128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __fixunshfti
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f16_i128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    mov x1, xzr
+; CHECK-NOFP16-GI-NEXT:    fcvtzu x0, s0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_f16_i128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu x0, h0
+; CHECK-FP16-GI-NEXT:    mov x1, xzr
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui half %a to i128
   ret i128 %c
 }
 
 define i64 @fptos_f128_i64(fp128 %a) {
-; CHECK-SD-LABEL: fptos_f128_i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __fixtfdi
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_f128_i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __fixtfdi
+; CHECK-NOFP16-SD-LABEL: fptos_f128_i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfdi
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_f128_i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __fixtfdi
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_f128_i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __fixtfdi
+;
+; CHECK-FP16-GI-LABEL: fptos_f128_i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __fixtfdi
 entry:
   %c = fptosi fp128 %a to i64
   ret i64 %c
 }
 
 define i64 @fptou_f128_i64(fp128 %a) {
-; CHECK-SD-LABEL: fptou_f128_i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __fixunstfdi
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_f128_i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __fixunstfdi
+; CHECK-NOFP16-SD-LABEL: fptou_f128_i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __fixunstfdi
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f128_i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __fixunstfdi
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f128_i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __fixunstfdi
+;
+; CHECK-FP16-GI-LABEL: fptou_f128_i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __fixunstfdi
 entry:
   %c = fptoui fp128 %a to i64
   ret i64 %c
 }
 
 define i32 @fptos_f128_i32(fp128 %a) {
-; CHECK-SD-LABEL: fptos_f128_i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_f128_i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __fixtfsi
+; CHECK-NOFP16-SD-LABEL: fptos_f128_i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_f128_i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_f128_i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __fixtfsi
+;
+; CHECK-FP16-GI-LABEL: fptos_f128_i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __fixtfsi
 entry:
   %c = fptosi fp128 %a to i32
   ret i32 %c
 }
 
 define i32 @fptou_f128_i32(fp128 %a) {
-; CHECK-SD-LABEL: fptou_f128_i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __fixunstfsi
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_f128_i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __fixunstfsi
+; CHECK-NOFP16-SD-LABEL: fptou_f128_i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f128_i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __fixunstfsi
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f128_i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __fixunstfsi
+;
+; CHECK-FP16-GI-LABEL: fptou_f128_i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __fixunstfsi
 entry:
   %c = fptoui fp128 %a to i32
   ret i32 %c
@@ -597,23 +641,41 @@ entry:
 }
 
 define i16 @fptou_f128_i16(fp128 %a) {
-; CHECK-SD-LABEL: fptou_f128_i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_f128_i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_f128_i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f128_i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f128_i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_f128_i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui fp128 %a to i16
   ret i16 %c
@@ -634,23 +696,41 @@ entry:
 }
 
 define i8 @fptou_f128_i8(fp128 %a) {
-; CHECK-SD-LABEL: fptou_f128_i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_f128_i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_f128_i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_f128_i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_f128_i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_f128_i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui fp128 %a to i8
   ret i8 %c
@@ -684,6 +764,584 @@ entry:
   ret i128 %c
 }
 
+define void @fptos_store_f64_i64(double %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptos_store_f64_i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs d0, d0
+; CHECK-NOFP16-SD-NEXT:    str d0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_store_f64_i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs d0, d0
+; CHECK-FP16-SD-NEXT:    str d0, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_store_f64_i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs x8, d0
+; CHECK-NOFP16-GI-NEXT:    str x8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_store_f64_i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs x8, d0
+; CHECK-FP16-GI-NEXT:    str x8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptosi double %a to i64
+  store i64 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f64_i64(double %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptou_store_f64_i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzu d0, d0
+; CHECK-NOFP16-SD-NEXT:    str d0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_store_f64_i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu d0, d0
+; CHECK-FP16-SD-NEXT:    str d0, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_store_f64_i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu x8, d0
+; CHECK-NOFP16-GI-NEXT:    str x8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_store_f64_i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu x8, d0
+; CHECK-FP16-GI-NEXT:    str x8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptoui double %a to i64
+  store i64 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f64_i32(double %a, ptr %p) {
+; CHECK-LABEL: fptos_store_f64_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs w8, d0
+; CHECK-NEXT:    str w8, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi double %a to i32
+  store i32 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f64_i32(double %a, ptr %p) {
+; CHECK-LABEL: fptou_store_f64_i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu w8, d0
+; CHECK-NEXT:    str w8, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui double %a to i32
+  store i32 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f64_i16(double %a, ptr %p) {
+; CHECK-LABEL: fptos_store_f64_i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs w8, d0
+; CHECK-NEXT:    strh w8, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi double %a to i16
+  store i16 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f64_i16(double %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptou_store_f64_i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs w8, d0
+; CHECK-NOFP16-SD-NEXT:    strh w8, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_store_f64_i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w8, d0
+; CHECK-FP16-SD-NEXT:    strh w8, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_store_f64_i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w8, d0
+; CHECK-NOFP16-GI-NEXT:    strh w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_store_f64_i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w8, d0
+; CHECK-FP16-GI-NEXT:    strh w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptoui double %a to i16
+  store i16 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f64_i8(double %a, ptr %p) {
+; CHECK-LABEL: fptos_store_f64_i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs w8, d0
+; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi double %a to i8
+  store i8 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f64_i8(double %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptou_store_f64_i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs w8, d0
+; CHECK-NOFP16-SD-NEXT:    strb w8, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_store_f64_i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w8, d0
+; CHECK-FP16-SD-NEXT:    strb w8, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_store_f64_i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w8, d0
+; CHECK-NOFP16-GI-NEXT:    strb w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_store_f64_i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w8, d0
+; CHECK-FP16-GI-NEXT:    strb w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptoui double %a to i8
+  store i8 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f32_i64(float %a, ptr %p) {
+; CHECK-LABEL: fptos_store_f32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzs x8, s0
+; CHECK-NEXT:    str x8, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptosi float %a to i64
+  store i64 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f32_i64(float %a, ptr %p) {
+; CHECK-LABEL: fptou_store_f32_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fcvtzu x8, s0
+; CHECK-NEXT:    str x8, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %c = fptoui float %a to i64
+  store i64 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f32_i32(float %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptos_store_f32_i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str s0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_store_f32_i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-FP16-SD-NEXT:    str s0, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_store_f32_i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs w8, s0
+; CHECK-NOFP16-GI-NEXT:    str w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_store_f32_i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs w8, s0
+; CHECK-FP16-GI-NEXT:    str w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptosi float %a to i32
+  store i32 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f32_i32(float %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptou_store_f32_i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzu s0, s0
+; CHECK-NOFP16-SD-NEXT:    str s0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_store_f32_i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu s0, s0
+; CHECK-FP16-SD-NEXT:    str s0, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_store_f32_i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w8, s0
+; CHECK-NOFP16-GI-NEXT:    str w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_store_f32_i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w8, s0
+; CHECK-FP16-GI-NEXT:    str w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptoui float %a to i32
+  store i32 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f32_i16(float %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptos_store_f32_i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str h0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_store_f32_i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-FP16-SD-NEXT:    str h0, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_store_f32_i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs w8, s0
+; CHECK-NOFP16-GI-NEXT:    strh w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_store_f32_i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs w8, s0
+; CHECK-FP16-GI-NEXT:    strh w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptosi float %a to i16
+  store i16 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f32_i16(float %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptou_store_f32_i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str h0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_store_f32_i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-FP16-SD-NEXT:    str h0, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_store_f32_i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w8, s0
+; CHECK-NOFP16-GI-NEXT:    strh w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_store_f32_i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w8, s0
+; CHECK-FP16-GI-NEXT:    strh w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptoui float %a to i16
+  store i16 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f32_i8(float %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptos_store_f32_i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str b0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_store_f32_i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-FP16-SD-NEXT:    str b0, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_store_f32_i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs w8, s0
+; CHECK-NOFP16-GI-NEXT:    strb w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_store_f32_i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs w8, s0
+; CHECK-FP16-GI-NEXT:    strb w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptosi float %a to i8
+  store i8 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f32_i8(float %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptou_store_f32_i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str b0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_store_f32_i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-FP16-SD-NEXT:    str b0, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_store_f32_i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w8, s0
+; CHECK-NOFP16-GI-NEXT:    strb w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_store_f32_i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w8, s0
+; CHECK-FP16-GI-NEXT:    strb w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptoui float %a to i8
+  store i8 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f16_i64(half %a, ptr %p) {
+; CHECK-NOFP16-LABEL: fptos_store_f16_i64:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-NEXT:    fcvtzs x8, s0
+; CHECK-NOFP16-NEXT:    str x8, [x0]
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_store_f16_i64:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-FP16-NEXT:    str x8, [x0]
+; CHECK-FP16-NEXT:    ret
+entry:
+  %c = fptosi half %a to i64
+  store i64 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f16_i64(half %a, ptr %p) {
+; CHECK-NOFP16-LABEL: fptou_store_f16_i64:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-NEXT:    fcvtzu x8, s0
+; CHECK-NOFP16-NEXT:    str x8, [x0]
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptou_store_f16_i64:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-FP16-NEXT:    str x8, [x0]
+; CHECK-FP16-NEXT:    ret
+entry:
+  %c = fptoui half %a to i64
+  store i64 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f16_i32(half %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptos_store_f16_i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str s0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_store_f16_i32:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs w8, h0
+; CHECK-FP16-NEXT:    str w8, [x0]
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_store_f16_i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvtzs w8, s0
+; CHECK-NOFP16-GI-NEXT:    str w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+entry:
+  %c = fptosi half %a to i32
+  store i32 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f16_i32(half %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptou_store_f16_i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzu s0, s0
+; CHECK-NOFP16-SD-NEXT:    str s0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptou_store_f16_i32:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzu w8, h0
+; CHECK-FP16-NEXT:    str w8, [x0]
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_store_f16_i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w8, s0
+; CHECK-NOFP16-GI-NEXT:    str w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+entry:
+  %c = fptoui half %a to i32
+  store i32 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f16_i16(half %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptos_store_f16_i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str h0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_store_f16_i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs w8, h0
+; CHECK-FP16-NEXT:    strh w8, [x0]
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_store_f16_i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvtzs w8, s0
+; CHECK-NOFP16-GI-NEXT:    strh w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+entry:
+  %c = fptosi half %a to i16
+  store i16 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f16_i16(half %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptou_store_f16_i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str h0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_store_f16_i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w8, h0
+; CHECK-FP16-SD-NEXT:    strh w8, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_store_f16_i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w8, s0
+; CHECK-NOFP16-GI-NEXT:    strh w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_store_f16_i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w8, h0
+; CHECK-FP16-GI-NEXT:    strh w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptoui half %a to i16
+  store i16 %c, ptr %p
+  ret void
+}
+
+define void @fptos_store_f16_i8(half %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptos_store_f16_i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str b0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_store_f16_i8:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs w8, h0
+; CHECK-FP16-NEXT:    strb w8, [x0]
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_store_f16_i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvtzs w8, s0
+; CHECK-NOFP16-GI-NEXT:    strb w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+entry:
+  %c = fptosi half %a to i8
+  store i8 %c, ptr %p
+  ret void
+}
+
+define void @fptou_store_f16_i8(half %a, ptr %p) {
+; CHECK-NOFP16-SD-LABEL: fptou_store_f16_i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs s0, s0
+; CHECK-NOFP16-SD-NEXT:    str b0, [x0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_store_f16_i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs w8, h0
+; CHECK-FP16-SD-NEXT:    strb w8, [x0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_store_f16_i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvtzu w8, s0
+; CHECK-NOFP16-GI-NEXT:    strb w8, [x0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_store_f16_i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu w8, h0
+; CHECK-FP16-GI-NEXT:    strb w8, [x0]
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = fptoui half %a to i8
+  store i8 %c, ptr %p
+  ret void
+}
+
 define <2 x i64> @fptos_v2f64_v2i64(<2 x double> %a) {
 ; CHECK-LABEL: fptos_v2f64_v2i64:
 ; CHECK:       // %bb.0: // %entry
@@ -705,64 +1363,118 @@ entry:
 }
 
 define <3 x i64> @fptos_v3f64_v3i64(<3 x double> %a) {
-; CHECK-SD-LABEL: fptos_v3f64_v3i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f64_v3i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v3f64_v3i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v3f64_v3i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v3f64_v3i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v3f64_v3i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <3 x double> %a to <3 x i64>
   ret <3 x i64> %c
 }
 
 define <3 x i64> @fptou_v3f64_v3i64(<3 x double> %a) {
-; CHECK-SD-LABEL: fptou_v3f64_v3i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f64_v3i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v3f64_v3i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v3f64_v3i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v3f64_v3i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v3f64_v3i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <3 x double> %a to <3 x i64>
   ret <3 x i64> %c
@@ -790,208 +1502,6 @@ entry:
   ret <4 x i64> %c
 }
 
-define <8 x i64> @fptos_v8f64_v8i64(<8 x double> %a) {
-; CHECK-LABEL: fptos_v8f64_v8i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-NEXT:    ret
-entry:
-  %c = fptosi <8 x double> %a to <8 x i64>
-  ret <8 x i64> %c
-}
-
-define <8 x i64> @fptou_v8f64_v8i64(<8 x double> %a) {
-; CHECK-LABEL: fptou_v8f64_v8i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-NEXT:    ret
-entry:
-  %c = fptoui <8 x double> %a to <8 x i64>
-  ret <8 x i64> %c
-}
-
-define <16 x i64> @fptos_v16f64_v16i64(<16 x double> %a) {
-; CHECK-LABEL: fptos_v16f64_v16i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-NEXT:    ret
-entry:
-  %c = fptosi <16 x double> %a to <16 x i64>
-  ret <16 x i64> %c
-}
-
-define <16 x i64> @fptou_v16f64_v16i64(<16 x double> %a) {
-; CHECK-LABEL: fptou_v16f64_v16i64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-NEXT:    ret
-entry:
-  %c = fptoui <16 x double> %a to <16 x i64>
-  ret <16 x i64> %c
-}
-
-define <32 x i64> @fptos_v32f64_v32i64(<32 x double> %a) {
-; CHECK-SD-LABEL: fptos_v32f64_v32i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q17, q16, [sp, #96]
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    ldp q19, q18, [sp, #64]
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    ldp q21, q20, [sp, #32]
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzs v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtzs v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q5, q6, [x8, #80]
-; CHECK-SD-NEXT:    str q16, [x8, #240]
-; CHECK-SD-NEXT:    ldp q22, q16, [sp]
-; CHECK-SD-NEXT:    stp q3, q4, [x8, #48]
-; CHECK-SD-NEXT:    stp q20, q19, [x8, #176]
-; CHECK-SD-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-SD-NEXT:    stp q1, q2, [x8, #16]
-; CHECK-SD-NEXT:    stp q18, q17, [x8, #208]
-; CHECK-SD-NEXT:    fcvtzs v17.2d, v22.2d
-; CHECK-SD-NEXT:    str q0, [x8]
-; CHECK-SD-NEXT:    stp q16, q21, [x8, #144]
-; CHECK-SD-NEXT:    stp q7, q17, [x8, #112]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v32f64_v32i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q0, q1, [x8]
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v6.2d
-; CHECK-GI-NEXT:    str q2, [x8, #32]
-; CHECK-GI-NEXT:    ldp q2, q5, [sp]
-; CHECK-GI-NEXT:    fcvtzs v6.2d, v7.2d
-; CHECK-GI-NEXT:    stp q3, q4, [x8, #48]
-; CHECK-GI-NEXT:    ldp q3, q4, [sp, #32]
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-NEXT:    stp q0, q1, [x8, #80]
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v3.2d
-; CHECK-GI-NEXT:    ldp q1, q3, [sp, #64]
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q6, q2, [x8, #112]
-; CHECK-GI-NEXT:    ldp q2, q6, [sp, #96]
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    stp q5, q0, [x8, #144]
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v6.2d
-; CHECK-GI-NEXT:    stp q4, q1, [x8, #176]
-; CHECK-GI-NEXT:    stp q3, q2, [x8, #208]
-; CHECK-GI-NEXT:    str q0, [x8, #240]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <32 x double> %a to <32 x i64>
-  ret <32 x i64> %c
-}
-
-define <32 x i64> @fptou_v32f64_v32i64(<32 x double> %a) {
-; CHECK-SD-LABEL: fptou_v32f64_v32i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q17, q16, [sp, #96]
-; CHECK-SD-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-SD-NEXT:    ldp q19, q18, [sp, #64]
-; CHECK-SD-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-SD-NEXT:    ldp q21, q20, [sp, #32]
-; CHECK-SD-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzu v16.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzu v17.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzu v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzu v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzu v20.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtzu v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q5, q6, [x8, #80]
-; CHECK-SD-NEXT:    str q16, [x8, #240]
-; CHECK-SD-NEXT:    ldp q22, q16, [sp]
-; CHECK-SD-NEXT:    stp q3, q4, [x8, #48]
-; CHECK-SD-NEXT:    stp q20, q19, [x8, #176]
-; CHECK-SD-NEXT:    fcvtzu v16.2d, v16.2d
-; CHECK-SD-NEXT:    stp q1, q2, [x8, #16]
-; CHECK-SD-NEXT:    stp q18, q17, [x8, #208]
-; CHECK-SD-NEXT:    fcvtzu v17.2d, v22.2d
-; CHECK-SD-NEXT:    str q0, [x8]
-; CHECK-SD-NEXT:    stp q16, q21, [x8, #144]
-; CHECK-SD-NEXT:    stp q7, q17, [x8, #112]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v32f64_v32i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q0, q1, [x8]
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v6.2d
-; CHECK-GI-NEXT:    str q2, [x8, #32]
-; CHECK-GI-NEXT:    ldp q2, q5, [sp]
-; CHECK-GI-NEXT:    fcvtzu v6.2d, v7.2d
-; CHECK-GI-NEXT:    stp q3, q4, [x8, #48]
-; CHECK-GI-NEXT:    ldp q3, q4, [sp, #32]
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-NEXT:    stp q0, q1, [x8, #80]
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v3.2d
-; CHECK-GI-NEXT:    ldp q1, q3, [sp, #64]
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q6, q2, [x8, #112]
-; CHECK-GI-NEXT:    ldp q2, q6, [sp, #96]
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    stp q5, q0, [x8, #144]
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v6.2d
-; CHECK-GI-NEXT:    stp q4, q1, [x8, #176]
-; CHECK-GI-NEXT:    stp q3, q2, [x8, #208]
-; CHECK-GI-NEXT:    str q0, [x8, #240]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <32 x double> %a to <32 x i64>
-  ret <32 x i64> %c
-}
-
 define <2 x i32> @fptos_v2f64_v2i32(<2 x double> %a) {
 ; CHECK-LABEL: fptos_v2f64_v2i32:
 ; CHECK:       // %bb.0: // %entry
@@ -1047,305 +1557,161 @@ entry:
 }
 
 define <4 x i32> @fptos_v4f64_v4i32(<4 x double> %a) {
-; CHECK-SD-LABEL: fptos_v4f64_v4i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v4f64_v4i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v4f64_v4i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v4f64_v4i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v4f64_v4i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v4f64_v4i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <4 x double> %a to <4 x i32>
   ret <4 x i32> %c
 }
 
 define <4 x i32> @fptou_v4f64_v4i32(<4 x double> %a) {
-; CHECK-SD-LABEL: fptou_v4f64_v4i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v4f64_v4i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v4f64_v4i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v4f64_v4i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v4f64_v4i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v4f64_v4i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <4 x double> %a to <4 x i32>
   ret <4 x i32> %c
 }
 
 define <8 x i32> @fptos_v8f64_v8i32(<8 x double> %a) {
-; CHECK-SD-LABEL: fptos_v8f64_v8i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v8f64_v8i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f64_v8i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v8f64_v8i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f64_v8i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v8f64_v8i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x double> %a to <8 x i32>
   ret <8 x i32> %c
 }
 
 define <8 x i32> @fptou_v8f64_v8i32(<8 x double> %a) {
-; CHECK-SD-LABEL: fptou_v8f64_v8i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v8f64_v8i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f64_v8i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v8f64_v8i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f64_v8i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v8f64_v8i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x double> %a to <8 x i32>
   ret <8 x i32> %c
 }
 
-define <16 x i32> @fptos_v16f64_v16i32(<16 x double> %a) {
-; CHECK-SD-LABEL: fptos_v16f64_v16i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-SD-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v16f64_v16i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <16 x double> %a to <16 x i32>
-  ret <16 x i32> %c
-}
-
-define <16 x i32> @fptou_v16f64_v16i32(<16 x double> %a) {
-; CHECK-SD-LABEL: fptou_v16f64_v16i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-SD-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v16f64_v16i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <16 x double> %a to <16 x i32>
-  ret <16 x i32> %c
-}
-
-define <32 x i32> @fptos_v32f64_v32i32(<32 x double> %a) {
-; CHECK-SD-LABEL: fptos_v32f64_v32i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q16, q17, [sp, #96]
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    ldp q18, q19, [sp, #64]
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    ldp q20, q21, [sp, #32]
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    ldp q22, q23, [sp]
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzs v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtzs v23.2d, v23.2d
-; CHECK-SD-NEXT:    fcvtzs v22.2d, v22.2d
-; CHECK-SD-NEXT:    fcvtzs v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-SD-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-SD-NEXT:    uzp1 v5.4s, v20.4s, v21.4s
-; CHECK-SD-NEXT:    uzp1 v4.4s, v22.4s, v23.4s
-; CHECK-SD-NEXT:    uzp1 v6.4s, v18.4s, v19.4s
-; CHECK-SD-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v32f64_v32i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzs v19.2d, v19.2d
-; CHECK-GI-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-GI-NEXT:    fcvtzs v21.2d, v21.2d
-; CHECK-GI-NEXT:    fcvtzs v22.2d, v22.2d
-; CHECK-GI-NEXT:    fcvtzs v23.2d, v23.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
-; CHECK-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
-; CHECK-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <32 x double> %a to <32 x i32>
-  ret <32 x i32> %c
-}
-
-define <32 x i32> @fptou_v32f64_v32i32(<32 x double> %a) {
-; CHECK-SD-LABEL: fptou_v32f64_v32i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q16, q17, [sp, #96]
-; CHECK-SD-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-SD-NEXT:    ldp q18, q19, [sp, #64]
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-SD-NEXT:    ldp q20, q21, [sp, #32]
-; CHECK-SD-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-SD-NEXT:    ldp q22, q23, [sp]
-; CHECK-SD-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzu v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtzu v20.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtzu v23.2d, v23.2d
-; CHECK-SD-NEXT:    fcvtzu v22.2d, v22.2d
-; CHECK-SD-NEXT:    fcvtzu v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtzu v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzu v17.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzu v16.2d, v16.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-SD-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-SD-NEXT:    uzp1 v5.4s, v20.4s, v21.4s
-; CHECK-SD-NEXT:    uzp1 v4.4s, v22.4s, v23.4s
-; CHECK-SD-NEXT:    uzp1 v6.4s, v18.4s, v19.4s
-; CHECK-SD-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v32f64_v32i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtzu v16.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzu v17.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzu v18.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzu v19.2d, v19.2d
-; CHECK-GI-NEXT:    fcvtzu v20.2d, v20.2d
-; CHECK-GI-NEXT:    fcvtzu v21.2d, v21.2d
-; CHECK-GI-NEXT:    fcvtzu v22.2d, v22.2d
-; CHECK-GI-NEXT:    fcvtzu v23.2d, v23.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
-; CHECK-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
-; CHECK-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <32 x double> %a to <32 x i32>
-  ret <32 x i32> %c
-}
-
 define <2 x i16> @fptos_v2f64_v2i16(<2 x double> %a) {
 ; CHECK-LABEL: fptos_v2f64_v2i16:
 ; CHECK:       // %bb.0: // %entry
@@ -1358,17 +1724,29 @@ entry:
 }
 
 define <2 x i16> @fptou_v2f64_v2i16(<2 x double> %a) {
-; CHECK-SD-LABEL: fptou_v2f64_v2i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f64_v2i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f64_v2i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f64_v2i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f64_v2i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f64_v2i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x double> %a to <2 x i16>
   ret <2 x i16> %c
@@ -1392,440 +1770,424 @@ entry:
 }
 
 define <3 x i16> @fptou_v3f64_v3i16(<3 x double> %a) {
-; CHECK-SD-LABEL: fptou_v3f64_v3i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f64_v3i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v3f64_v3i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v3f64_v3i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v3f64_v3i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v3f64_v3i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <3 x double> %a to <3 x i16>
   ret <3 x i16> %c
 }
 
 define <4 x i16> @fptos_v4f64_v4i16(<4 x double> %a) {
-; CHECK-SD-LABEL: fptos_v4f64_v4i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v4f64_v4i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v4f64_v4i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v4f64_v4i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v4f64_v4i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v4f64_v4i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <4 x double> %a to <4 x i16>
   ret <4 x i16> %c
 }
 
 define <4 x i16> @fptou_v4f64_v4i16(<4 x double> %a) {
-; CHECK-SD-LABEL: fptou_v4f64_v4i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v4f64_v4i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v4f64_v4i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v4f64_v4i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v4f64_v4i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v4f64_v4i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <4 x double> %a to <4 x i16>
   ret <4 x i16> %c
 }
 
 define <8 x i16> @fptos_v8f64_v8i16(<8 x double> %a) {
-; CHECK-SD-LABEL: fptos_v8f64_v8i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    adrp x8, .LCPI70_0
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    xtn v6.2s, v3.2d
-; CHECK-SD-NEXT:    xtn v5.2s, v2.2d
-; CHECK-SD-NEXT:    xtn v4.2s, v1.2d
-; CHECK-SD-NEXT:    xtn v3.2s, v0.2d
-; CHECK-SD-NEXT:    ldr q0, [x8, :lo12:.LCPI70_0]
-; CHECK-SD-NEXT:    tbl v0.16b, { v3.16b, v4.16b, v5.16b, v6.16b }, v0.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v8f64_v8i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f64_v8i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    adrp x8, .LCPI84_0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v6.2s, v3.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v5.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v4.2s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v3.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [x8, :lo12:.LCPI84_0]
+; CHECK-NOFP16-SD-NEXT:    tbl v0.16b, { v3.16b, v4.16b, v5.16b, v6.16b }, v0.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v8f64_v8i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    adrp x8, .LCPI84_0
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    xtn v6.2s, v3.2d
+; CHECK-FP16-SD-NEXT:    xtn v5.2s, v2.2d
+; CHECK-FP16-SD-NEXT:    xtn v4.2s, v1.2d
+; CHECK-FP16-SD-NEXT:    xtn v3.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    ldr q0, [x8, :lo12:.LCPI84_0]
+; CHECK-FP16-SD-NEXT:    tbl v0.16b, { v3.16b, v4.16b, v5.16b, v6.16b }, v0.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f64_v8i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v8f64_v8i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x double> %a to <8 x i16>
   ret <8 x i16> %c
 }
 
 define <8 x i16> @fptou_v8f64_v8i16(<8 x double> %a) {
-; CHECK-SD-LABEL: fptou_v8f64_v8i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    adrp x8, .LCPI71_0
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    xtn v6.2s, v3.2d
-; CHECK-SD-NEXT:    xtn v5.2s, v2.2d
-; CHECK-SD-NEXT:    xtn v4.2s, v1.2d
-; CHECK-SD-NEXT:    xtn v3.2s, v0.2d
-; CHECK-SD-NEXT:    ldr q0, [x8, :lo12:.LCPI71_0]
-; CHECK-SD-NEXT:    tbl v0.16b, { v3.16b, v4.16b, v5.16b, v6.16b }, v0.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v8f64_v8i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f64_v8i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    adrp x8, .LCPI85_0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v6.2s, v3.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v5.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v4.2s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v3.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [x8, :lo12:.LCPI85_0]
+; CHECK-NOFP16-SD-NEXT:    tbl v0.16b, { v3.16b, v4.16b, v5.16b, v6.16b }, v0.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v8f64_v8i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    adrp x8, .LCPI85_0
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    xtn v6.2s, v3.2d
+; CHECK-FP16-SD-NEXT:    xtn v5.2s, v2.2d
+; CHECK-FP16-SD-NEXT:    xtn v4.2s, v1.2d
+; CHECK-FP16-SD-NEXT:    xtn v3.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    ldr q0, [x8, :lo12:.LCPI85_0]
+; CHECK-FP16-SD-NEXT:    tbl v0.16b, { v3.16b, v4.16b, v5.16b, v6.16b }, v0.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f64_v8i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v8f64_v8i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x double> %a to <8 x i16>
   ret <8 x i16> %c
 }
 
 define <16 x i16> @fptos_v16f64_v16i16(<16 x double> %a) {
-; CHECK-SD-LABEL: fptos_v16f64_v16i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    adrp x8, .LCPI72_0
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    xtn v19.2s, v3.2d
-; CHECK-SD-NEXT:    xtn v23.2s, v7.2d
-; CHECK-SD-NEXT:    xtn v18.2s, v2.2d
-; CHECK-SD-NEXT:    xtn v22.2s, v6.2d
-; CHECK-SD-NEXT:    xtn v17.2s, v1.2d
-; CHECK-SD-NEXT:    xtn v21.2s, v5.2d
-; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI72_0]
-; CHECK-SD-NEXT:    xtn v16.2s, v0.2d
-; CHECK-SD-NEXT:    xtn v20.2s, v4.2d
-; CHECK-SD-NEXT:    tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
-; CHECK-SD-NEXT:    tbl v1.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v16f64_v16i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v16f64_v16i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    adrp x8, .LCPI86_0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v19.2s, v3.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v23.2s, v7.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v18.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v22.2s, v6.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v17.2s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v21.2s, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI86_0]
+; CHECK-NOFP16-SD-NEXT:    xtn v16.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v20.2s, v4.2d
+; CHECK-NOFP16-SD-NEXT:    tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
+; CHECK-NOFP16-SD-NEXT:    tbl v1.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v16f64_v16i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    adrp x8, .LCPI86_0
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    xtn v19.2s, v3.2d
+; CHECK-FP16-SD-NEXT:    xtn v23.2s, v7.2d
+; CHECK-FP16-SD-NEXT:    xtn v18.2s, v2.2d
+; CHECK-FP16-SD-NEXT:    xtn v22.2s, v6.2d
+; CHECK-FP16-SD-NEXT:    xtn v17.2s, v1.2d
+; CHECK-FP16-SD-NEXT:    xtn v21.2s, v5.2d
+; CHECK-FP16-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI86_0]
+; CHECK-FP16-SD-NEXT:    xtn v16.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    xtn v20.2s, v4.2d
+; CHECK-FP16-SD-NEXT:    tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
+; CHECK-FP16-SD-NEXT:    tbl v1.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v16f64_v16i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v16f64_v16i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <16 x double> %a to <16 x i16>
   ret <16 x i16> %c
 }
 
 define <16 x i16> @fptou_v16f64_v16i16(<16 x double> %a) {
-; CHECK-SD-LABEL: fptou_v16f64_v16i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    adrp x8, .LCPI73_0
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    xtn v19.2s, v3.2d
-; CHECK-SD-NEXT:    xtn v23.2s, v7.2d
-; CHECK-SD-NEXT:    xtn v18.2s, v2.2d
-; CHECK-SD-NEXT:    xtn v22.2s, v6.2d
-; CHECK-SD-NEXT:    xtn v17.2s, v1.2d
-; CHECK-SD-NEXT:    xtn v21.2s, v5.2d
-; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI73_0]
-; CHECK-SD-NEXT:    xtn v16.2s, v0.2d
-; CHECK-SD-NEXT:    xtn v20.2s, v4.2d
-; CHECK-SD-NEXT:    tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
-; CHECK-SD-NEXT:    tbl v1.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v16f64_v16i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v16f64_v16i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    adrp x8, .LCPI87_0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v19.2s, v3.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v23.2s, v7.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v18.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v22.2s, v6.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v17.2s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v21.2s, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI87_0]
+; CHECK-NOFP16-SD-NEXT:    xtn v16.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v20.2s, v4.2d
+; CHECK-NOFP16-SD-NEXT:    tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
+; CHECK-NOFP16-SD-NEXT:    tbl v1.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v16f64_v16i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    adrp x8, .LCPI87_0
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    xtn v19.2s, v3.2d
+; CHECK-FP16-SD-NEXT:    xtn v23.2s, v7.2d
+; CHECK-FP16-SD-NEXT:    xtn v18.2s, v2.2d
+; CHECK-FP16-SD-NEXT:    xtn v22.2s, v6.2d
+; CHECK-FP16-SD-NEXT:    xtn v17.2s, v1.2d
+; CHECK-FP16-SD-NEXT:    xtn v21.2s, v5.2d
+; CHECK-FP16-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI87_0]
+; CHECK-FP16-SD-NEXT:    xtn v16.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    xtn v20.2s, v4.2d
+; CHECK-FP16-SD-NEXT:    tbl v0.16b, { v16.16b, v17.16b, v18.16b, v19.16b }, v1.16b
+; CHECK-FP16-SD-NEXT:    tbl v1.16b, { v20.16b, v21.16b, v22.16b, v23.16b }, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v16f64_v16i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v16f64_v16i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <16 x double> %a to <16 x i16>
   ret <16 x i16> %c
 }
 
-define <32 x i16> @fptos_v32f64_v32i16(<32 x double> %a) {
-; CHECK-SD-LABEL: fptos_v32f64_v32i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset b8, -8
-; CHECK-SD-NEXT:    .cfi_offset b9, -16
-; CHECK-SD-NEXT:    .cfi_offset b10, -24
-; CHECK-SD-NEXT:    .cfi_offset b11, -32
-; CHECK-SD-NEXT:    .cfi_offset b12, -40
-; CHECK-SD-NEXT:    .cfi_offset b13, -48
-; CHECK-SD-NEXT:    .cfi_offset b14, -56
-; CHECK-SD-NEXT:    .cfi_offset b15, -64
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v18.2d, v2.2d
-; CHECK-SD-NEXT:    adrp x8, .LCPI74_0
-; CHECK-SD-NEXT:    fcvtzs v19.2d, v1.2d
-; CHECK-SD-NEXT:    ldp q20, q21, [sp, #160]
-; CHECK-SD-NEXT:    fcvtzs v22.2d, v0.2d
-; CHECK-SD-NEXT:    ldp q23, q24, [sp, #96]
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    ldp q16, q17, [sp, #128]
-; CHECK-SD-NEXT:    xtn v3.2s, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-SD-NEXT:    xtn v2.2s, v18.2d
-; CHECK-SD-NEXT:    ldp q18, q25, [sp, #64]
-; CHECK-SD-NEXT:    xtn v1.2s, v19.2d
-; CHECK-SD-NEXT:    fcvtzs v19.2d, v24.2d
-; CHECK-SD-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-SD-NEXT:    xtn v0.2s, v22.2d
-; CHECK-SD-NEXT:    fcvtzs v22.2d, v23.2d
-; CHECK-SD-NEXT:    xtn v29.2s, v7.2d
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v25.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    xtn v15.2s, v21.2d
-; CHECK-SD-NEXT:    xtn v11.2s, v19.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    xtn v14.2s, v20.2d
-; CHECK-SD-NEXT:    xtn v10.2s, v22.2d
-; CHECK-SD-NEXT:    xtn v13.2s, v17.2d
-; CHECK-SD-NEXT:    xtn v9.2s, v7.2d
-; CHECK-SD-NEXT:    xtn v28.2s, v6.2d
-; CHECK-SD-NEXT:    xtn v8.2s, v18.2d
-; CHECK-SD-NEXT:    xtn v12.2s, v16.2d
-; CHECK-SD-NEXT:    xtn v27.2s, v5.2d
-; CHECK-SD-NEXT:    xtn v26.2s, v4.2d
-; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI74_0]
-; CHECK-SD-NEXT:    tbl v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v4.16b
-; CHECK-SD-NEXT:    tbl v2.16b, { v8.16b, v9.16b, v10.16b, v11.16b }, v4.16b
-; CHECK-SD-NEXT:    tbl v3.16b, { v12.16b, v13.16b, v14.16b, v15.16b }, v4.16b
-; CHECK-SD-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    tbl v1.16b, { v26.16b, v27.16b, v28.16b, v29.16b }, v4.16b
-; CHECK-SD-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v32f64_v32i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzs v19.2d, v19.2d
-; CHECK-GI-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-GI-NEXT:    fcvtzs v21.2d, v21.2d
-; CHECK-GI-NEXT:    fcvtzs v22.2d, v22.2d
-; CHECK-GI-NEXT:    fcvtzs v23.2d, v23.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
-; CHECK-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
-; CHECK-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <32 x double> %a to <32 x i16>
-  ret <32 x i16> %c
-}
-
-define <32 x i16> @fptou_v32f64_v32i16(<32 x double> %a) {
-; CHECK-SD-LABEL: fptou_v32f64_v32i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset b8, -8
-; CHECK-SD-NEXT:    .cfi_offset b9, -16
-; CHECK-SD-NEXT:    .cfi_offset b10, -24
-; CHECK-SD-NEXT:    .cfi_offset b11, -32
-; CHECK-SD-NEXT:    .cfi_offset b12, -40
-; CHECK-SD-NEXT:    .cfi_offset b13, -48
-; CHECK-SD-NEXT:    .cfi_offset b14, -56
-; CHECK-SD-NEXT:    .cfi_offset b15, -64
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v18.2d, v2.2d
-; CHECK-SD-NEXT:    adrp x8, .LCPI75_0
-; CHECK-SD-NEXT:    fcvtzs v19.2d, v1.2d
-; CHECK-SD-NEXT:    ldp q20, q21, [sp, #160]
-; CHECK-SD-NEXT:    fcvtzs v22.2d, v0.2d
-; CHECK-SD-NEXT:    ldp q23, q24, [sp, #96]
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    ldp q16, q17, [sp, #128]
-; CHECK-SD-NEXT:    xtn v3.2s, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-SD-NEXT:    xtn v2.2s, v18.2d
-; CHECK-SD-NEXT:    ldp q18, q25, [sp, #64]
-; CHECK-SD-NEXT:    xtn v1.2s, v19.2d
-; CHECK-SD-NEXT:    fcvtzs v19.2d, v24.2d
-; CHECK-SD-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-SD-NEXT:    xtn v0.2s, v22.2d
-; CHECK-SD-NEXT:    fcvtzs v22.2d, v23.2d
-; CHECK-SD-NEXT:    xtn v29.2s, v7.2d
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v25.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    xtn v15.2s, v21.2d
-; CHECK-SD-NEXT:    xtn v11.2s, v19.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    xtn v14.2s, v20.2d
-; CHECK-SD-NEXT:    xtn v10.2s, v22.2d
-; CHECK-SD-NEXT:    xtn v13.2s, v17.2d
-; CHECK-SD-NEXT:    xtn v9.2s, v7.2d
-; CHECK-SD-NEXT:    xtn v28.2s, v6.2d
-; CHECK-SD-NEXT:    xtn v8.2s, v18.2d
-; CHECK-SD-NEXT:    xtn v12.2s, v16.2d
-; CHECK-SD-NEXT:    xtn v27.2s, v5.2d
-; CHECK-SD-NEXT:    xtn v26.2s, v4.2d
-; CHECK-SD-NEXT:    ldr q4, [x8, :lo12:.LCPI75_0]
-; CHECK-SD-NEXT:    tbl v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v4.16b
-; CHECK-SD-NEXT:    tbl v2.16b, { v8.16b, v9.16b, v10.16b, v11.16b }, v4.16b
-; CHECK-SD-NEXT:    tbl v3.16b, { v12.16b, v13.16b, v14.16b, v15.16b }, v4.16b
-; CHECK-SD-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    tbl v1.16b, { v26.16b, v27.16b, v28.16b, v29.16b }, v4.16b
-; CHECK-SD-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v32f64_v32i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtzu v16.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzu v17.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzu v18.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzu v19.2d, v19.2d
-; CHECK-GI-NEXT:    fcvtzu v20.2d, v20.2d
-; CHECK-GI-NEXT:    fcvtzu v21.2d, v21.2d
-; CHECK-GI-NEXT:    fcvtzu v22.2d, v22.2d
-; CHECK-GI-NEXT:    fcvtzu v23.2d, v23.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
-; CHECK-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
-; CHECK-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <32 x double> %a to <32 x i16>
-  ret <32 x i16> %c
-}
-
 define <2 x i8> @fptos_v2f64_v2i8(<2 x double> %a) {
 ; CHECK-LABEL: fptos_v2f64_v2i8:
 ; CHECK:       // %bb.0: // %entry
@@ -1838,716 +2200,1046 @@ entry:
 }
 
 define <2 x i8> @fptou_v2f64_v2i8(<2 x double> %a) {
-; CHECK-SD-LABEL: fptou_v2f64_v2i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    xtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f64_v2i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    xtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f64_v2i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f64_v2i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f64_v2i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f64_v2i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    xtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x double> %a to <2 x i8>
   ret <2 x i8> %c
 }
 
 define <3 x i8> @fptos_v3f64_v3i8(<3 x double> %a) {
-; CHECK-SD-LABEL: fptos_v3f64_v3i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f64_v3i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fmov x2, d1
-; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 killed $x2
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    fmov x0, d0
-; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-GI-NEXT:    fmov x1, d2
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 killed $x1
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v3f64_v3i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v3f64_v3i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-FP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v3f64_v3i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fmov x2, d1
+; CHECK-NOFP16-GI-NEXT:    // kill: def $w2 killed $w2 killed $x2
+; CHECK-NOFP16-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    fmov x0, d0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NOFP16-GI-NEXT:    fmov x1, d2
+; CHECK-NOFP16-GI-NEXT:    // kill: def $w1 killed $w1 killed $x1
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v3f64_v3i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fmov x2, d1
+; CHECK-FP16-GI-NEXT:    // kill: def $w2 killed $w2 killed $x2
+; CHECK-FP16-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-FP16-GI-NEXT:    fmov x0, d0
+; CHECK-FP16-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-FP16-GI-NEXT:    fmov x1, d2
+; CHECK-FP16-GI-NEXT:    // kill: def $w1 killed $w1 killed $x1
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <3 x double> %a to <3 x i8>
   ret <3 x i8> %c
 }
 
 define <3 x i8> @fptou_v3f64_v3i8(<3 x double> %a) {
-; CHECK-SD-LABEL: fptou_v3f64_v3i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f64_v3i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fmov x2, d1
-; CHECK-GI-NEXT:    // kill: def $w2 killed $w2 killed $x2
-; CHECK-GI-NEXT:    mov d2, v0.d[1]
-; CHECK-GI-NEXT:    fmov x0, d0
-; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
-; CHECK-GI-NEXT:    fmov x1, d2
-; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 killed $x1
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v3f64_v3i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v3f64_v3i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-FP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v3f64_v3i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fmov x2, d1
+; CHECK-NOFP16-GI-NEXT:    // kill: def $w2 killed $w2 killed $x2
+; CHECK-NOFP16-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    fmov x0, d0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NOFP16-GI-NEXT:    fmov x1, d2
+; CHECK-NOFP16-GI-NEXT:    // kill: def $w1 killed $w1 killed $x1
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v3f64_v3i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fmov x2, d1
+; CHECK-FP16-GI-NEXT:    // kill: def $w2 killed $w2 killed $x2
+; CHECK-FP16-GI-NEXT:    mov d2, v0.d[1]
+; CHECK-FP16-GI-NEXT:    fmov x0, d0
+; CHECK-FP16-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-FP16-GI-NEXT:    fmov x1, d2
+; CHECK-FP16-GI-NEXT:    // kill: def $w1 killed $w1 killed $x1
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <3 x double> %a to <3 x i8>
   ret <3 x i8> %c
 }
 
 define <4 x i8> @fptos_v4f64_v4i8(<4 x double> %a) {
-; CHECK-SD-LABEL: fptos_v4f64_v4i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v4f64_v4i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v4f64_v4i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v4f64_v4i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v4f64_v4i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v4f64_v4i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <4 x double> %a to <4 x i8>
   ret <4 x i8> %c
 }
 
 define <4 x i8> @fptou_v4f64_v4i8(<4 x double> %a) {
-; CHECK-SD-LABEL: fptou_v4f64_v4i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v4f64_v4i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v4f64_v4i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v4f64_v4i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v4f64_v4i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v4f64_v4i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <4 x double> %a to <4 x i8>
   ret <4 x i8> %c
 }
 
 define <8 x i8> @fptos_v8f64_v8i8(<8 x double> %a) {
-; CHECK-SD-LABEL: fptos_v8f64_v8i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v8f64_v8i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f64_v8i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v8f64_v8i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f64_v8i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v8f64_v8i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x double> %a to <8 x i8>
   ret <8 x i8> %c
 }
 
 define <8 x i8> @fptou_v8f64_v8i8(<8 x double> %a) {
-; CHECK-SD-LABEL: fptou_v8f64_v8i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v8f64_v8i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f64_v8i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v8f64_v8i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f64_v8i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v8f64_v8i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x double> %a to <8 x i8>
   ret <8 x i8> %c
 }
 
 define <16 x i8> @fptos_v16f64_v16i8(<16 x double> %a) {
-; CHECK-SD-LABEL: fptos_v16f64_v16i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
-; CHECK-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
-; CHECK-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v1.8h, v4.8h, v6.8h
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v16f64_v16i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v16f64_v16i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v4.8h, v6.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v16f64_v16i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v1.8h, v4.8h, v6.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v16f64_v16i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v16f64_v16i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <16 x double> %a to <16 x i8>
   ret <16 x i8> %c
 }
 
 define <16 x i8> @fptou_v16f64_v16i8(<16 x double> %a) {
-; CHECK-SD-LABEL: fptou_v16f64_v16i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
-; CHECK-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
-; CHECK-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v1.8h, v4.8h, v6.8h
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v16f64_v16i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v16f64_v16i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v4.8h, v6.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v16f64_v16i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v1.8h, v4.8h, v6.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v16f64_v16i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v16f64_v16i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <16 x double> %a to <16 x i8>
   ret <16 x i8> %c
 }
 
 define <32 x i8> @fptos_v32f64_v32i8(<32 x double> %a) {
-; CHECK-SD-LABEL: fptos_v32f64_v32i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q16, q17, [sp]
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzs v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtzs v23.2d, v23.2d
-; CHECK-SD-NEXT:    fcvtzs v22.2d, v22.2d
-; CHECK-SD-NEXT:    fcvtzs v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
-; CHECK-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
-; CHECK-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v3.4s, v20.4s, v21.4s
-; CHECK-SD-NEXT:    uzp1 v1.4s, v22.4s, v23.4s
-; CHECK-SD-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
-; CHECK-SD-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
-; CHECK-SD-NEXT:    uzp1 v4.8h, v4.8h, v6.8h
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v2.8h, v7.8h, v5.8h
-; CHECK-SD-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
-; CHECK-SD-NEXT:    uzp1 v1.16b, v2.16b, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v32f64_v32i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzs v19.2d, v19.2d
-; CHECK-GI-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-GI-NEXT:    fcvtzs v21.2d, v21.2d
-; CHECK-GI-NEXT:    fcvtzs v22.2d, v22.2d
-; CHECK-GI-NEXT:    fcvtzs v23.2d, v23.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
-; CHECK-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
-; CHECK-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v32f64_v32i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ldp q16, q17, [sp]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v23.2d, v23.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v22.2d, v22.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v3.4s, v20.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.4s, v22.4s, v23.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v4.8h, v4.8h, v6.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.8h, v7.8h, v5.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v32f64_v32i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ldp q16, q17, [sp]
+; CHECK-FP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-FP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-FP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-FP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v23.2d, v23.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v22.2d, v22.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v3.4s, v20.4s, v21.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v1.4s, v22.4s, v23.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v4.8h, v4.8h, v6.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v2.8h, v7.8h, v5.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
+; CHECK-FP16-SD-NEXT:    uzp1 v1.16b, v2.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v32f64_v32i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ldp q16, q17, [sp]
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v22.2d, v22.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v23.2d, v23.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v32f64_v32i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ldp q16, q17, [sp]
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v22.2d, v22.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v23.2d, v23.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <32 x double> %a to <32 x i8>
   ret <32 x i8> %c
 }
 
 define <32 x i8> @fptou_v32f64_v32i8(<32 x double> %a) {
-; CHECK-SD-LABEL: fptou_v32f64_v32i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q16, q17, [sp]
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzs v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtzs v23.2d, v23.2d
-; CHECK-SD-NEXT:    fcvtzs v22.2d, v22.2d
-; CHECK-SD-NEXT:    fcvtzs v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
-; CHECK-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
-; CHECK-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT:    uzp1 v3.4s, v20.4s, v21.4s
-; CHECK-SD-NEXT:    uzp1 v1.4s, v22.4s, v23.4s
-; CHECK-SD-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
-; CHECK-SD-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
-; CHECK-SD-NEXT:    uzp1 v4.8h, v4.8h, v6.8h
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-SD-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v2.8h, v7.8h, v5.8h
-; CHECK-SD-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
-; CHECK-SD-NEXT:    uzp1 v1.16b, v2.16b, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v32f64_v32i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtzu v16.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzu v17.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzu v18.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzu v19.2d, v19.2d
-; CHECK-GI-NEXT:    fcvtzu v20.2d, v20.2d
-; CHECK-GI-NEXT:    fcvtzu v21.2d, v21.2d
-; CHECK-GI-NEXT:    fcvtzu v22.2d, v22.2d
-; CHECK-GI-NEXT:    fcvtzu v23.2d, v23.2d
-; CHECK-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
-; CHECK-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
-; CHECK-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
-; CHECK-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v32f64_v32i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ldp q16, q17, [sp]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v23.2d, v23.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v22.2d, v22.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v3.4s, v20.4s, v21.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.4s, v22.4s, v23.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v4.8h, v4.8h, v6.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.8h, v7.8h, v5.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.16b, v2.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v32f64_v32i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ldp q16, q17, [sp]
+; CHECK-FP16-SD-NEXT:    fcvtzs v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-FP16-SD-NEXT:    fcvtzs v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-FP16-SD-NEXT:    fcvtzs v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-FP16-SD-NEXT:    fcvtzs v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v21.2d, v21.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v20.2d, v20.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v23.2d, v23.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v22.2d, v22.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v19.2d, v19.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v18.2d, v18.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v17.2d, v17.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT:    uzp1 v6.4s, v6.4s, v7.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v4.4s, v4.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v2.4s, v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v3.4s, v20.4s, v21.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v1.4s, v22.4s, v23.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v7.4s, v16.4s, v17.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v4.8h, v4.8h, v6.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v2.8h, v7.8h, v5.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v4.16b
+; CHECK-FP16-SD-NEXT:    uzp1 v1.16b, v2.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v32f64_v32i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ldp q16, q17, [sp]
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v16.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v17.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v18.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v19.2d, v19.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v20.2d, v20.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v21.2d, v21.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v22.2d, v22.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v23.2d, v23.2d
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v32f64_v32i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ldp q16, q17, [sp]
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ldp q18, q19, [sp, #32]
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ldp q20, q21, [sp, #64]
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ldp q22, q23, [sp, #96]
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v16.2d, v16.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v17.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v18.2d, v18.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v19.2d, v19.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v20.2d, v20.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v21.2d, v21.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v22.2d, v22.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v23.2d, v23.2d
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v1.4s, v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v2.4s, v4.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v3.4s, v6.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v4.4s, v16.4s, v17.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v5.4s, v18.4s, v19.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v6.4s, v20.4s, v21.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v7.4s, v22.4s, v23.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <32 x double> %a to <32 x i8>
   ret <32 x i8> %c
 }
 
 define <2 x i128> @fptos_v2f64_v2i128(<2 x double> %a) {
-; CHECK-SD-LABEL: fptos_v2f64_v2i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    bl __fixdfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    mov d0, v0.d[1]
-; CHECK-SD-NEXT:    bl __fixdfti
-; CHECK-SD-NEXT:    mov x2, x0
-; CHECK-SD-NEXT:    mov x3, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v2f64_v2i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -24
-; CHECK-GI-NEXT:    .cfi_offset b8, -32
-; CHECK-GI-NEXT:    mov d8, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    bl __fixdfti
-; CHECK-GI-NEXT:    fmov d0, d8
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    mov x20, x1
-; CHECK-GI-NEXT:    bl __fixdfti
-; CHECK-GI-NEXT:    mov x2, x0
-; CHECK-GI-NEXT:    mov x3, x1
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f64_v2i128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    bl __fixdfti
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov x19, x0
+; CHECK-NOFP16-SD-NEXT:    mov x20, x1
+; CHECK-NOFP16-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    bl __fixdfti
+; CHECK-NOFP16-SD-NEXT:    mov x2, x0
+; CHECK-NOFP16-SD-NEXT:    mov x3, x1
+; CHECK-NOFP16-SD-NEXT:    mov x0, x19
+; CHECK-NOFP16-SD-NEXT:    mov x1, x20
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f64_v2i128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixdfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-FP16-SD-NEXT:    bl __fixdfti
+; CHECK-FP16-SD-NEXT:    mov x2, x0
+; CHECK-FP16-SD-NEXT:    mov x3, x1
+; CHECK-FP16-SD-NEXT:    mov x0, x19
+; CHECK-FP16-SD-NEXT:    mov x1, x20
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f64_v2i128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-NOFP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    bl __fixdfti
+; CHECK-NOFP16-GI-NEXT:    fmov d0, d8
+; CHECK-NOFP16-GI-NEXT:    mov x19, x0
+; CHECK-NOFP16-GI-NEXT:    mov x20, x1
+; CHECK-NOFP16-GI-NEXT:    bl __fixdfti
+; CHECK-NOFP16-GI-NEXT:    mov x2, x0
+; CHECK-NOFP16-GI-NEXT:    mov x3, x1
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f64_v2i128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    fmov d0, d8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixdfti
+; CHECK-FP16-GI-NEXT:    mov x2, x0
+; CHECK-FP16-GI-NEXT:    mov x3, x1
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x double> %a to <2 x i128>
   ret <2 x i128> %c
 }
 
 define <2 x i128> @fptou_v2f64_v2i128(<2 x double> %a) {
-; CHECK-SD-LABEL: fptou_v2f64_v2i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    bl __fixunsdfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    mov d0, v0.d[1]
-; CHECK-SD-NEXT:    bl __fixunsdfti
-; CHECK-SD-NEXT:    mov x2, x0
-; CHECK-SD-NEXT:    mov x3, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f64_v2i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -24
-; CHECK-GI-NEXT:    .cfi_offset b8, -32
-; CHECK-GI-NEXT:    mov d8, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    bl __fixunsdfti
-; CHECK-GI-NEXT:    fmov d0, d8
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    mov x20, x1
-; CHECK-GI-NEXT:    bl __fixunsdfti
-; CHECK-GI-NEXT:    mov x2, x0
-; CHECK-GI-NEXT:    mov x3, x1
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f64_v2i128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    bl __fixunsdfti
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov x19, x0
+; CHECK-NOFP16-SD-NEXT:    mov x20, x1
+; CHECK-NOFP16-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    bl __fixunsdfti
+; CHECK-NOFP16-SD-NEXT:    mov x2, x0
+; CHECK-NOFP16-SD-NEXT:    mov x3, x1
+; CHECK-NOFP16-SD-NEXT:    mov x0, x19
+; CHECK-NOFP16-SD-NEXT:    mov x1, x20
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f64_v2i128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixunsdfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov d0, v0.d[1]
+; CHECK-FP16-SD-NEXT:    bl __fixunsdfti
+; CHECK-FP16-SD-NEXT:    mov x2, x0
+; CHECK-FP16-SD-NEXT:    mov x3, x1
+; CHECK-FP16-SD-NEXT:    mov x0, x19
+; CHECK-FP16-SD-NEXT:    mov x1, x20
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f64_v2i128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-NOFP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    bl __fixunsdfti
+; CHECK-NOFP16-GI-NEXT:    fmov d0, d8
+; CHECK-NOFP16-GI-NEXT:    mov x19, x0
+; CHECK-NOFP16-GI-NEXT:    mov x20, x1
+; CHECK-NOFP16-GI-NEXT:    bl __fixunsdfti
+; CHECK-NOFP16-GI-NEXT:    mov x2, x0
+; CHECK-NOFP16-GI-NEXT:    mov x3, x1
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f64_v2i128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixunsdfti
+; CHECK-FP16-GI-NEXT:    fmov d0, d8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixunsdfti
+; CHECK-FP16-GI-NEXT:    mov x2, x0
+; CHECK-FP16-GI-NEXT:    mov x3, x1
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x double> %a to <2 x i128>
   ret <2 x i128> %c
 }
 
-define <3 x i128> @fptos_v3f64_v3i128(<3 x double> %a) {
-; CHECK-SD-LABEL: fptos_v3f64_v3i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    .cfi_offset b8, -56
-; CHECK-SD-NEXT:    .cfi_offset b9, -64
-; CHECK-SD-NEXT:    fmov d8, d2
-; CHECK-SD-NEXT:    fmov d9, d1
-; CHECK-SD-NEXT:    bl __fixdfti
-; CHECK-SD-NEXT:    fmov d0, d9
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    bl __fixdfti
-; CHECK-SD-NEXT:    fmov d0, d8
-; CHECK-SD-NEXT:    mov x21, x0
-; CHECK-SD-NEXT:    mov x22, x1
-; CHECK-SD-NEXT:    bl __fixdfti
-; CHECK-SD-NEXT:    mov x4, x0
-; CHECK-SD-NEXT:    mov x5, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    mov x2, x21
-; CHECK-SD-NEXT:    mov x3, x22
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f64_v3i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    .cfi_offset b8, -56
-; CHECK-GI-NEXT:    .cfi_offset b9, -64
-; CHECK-GI-NEXT:    fmov d8, d1
-; CHECK-GI-NEXT:    fmov d9, d2
-; CHECK-GI-NEXT:    bl __fixdfti
-; CHECK-GI-NEXT:    fmov d0, d8
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    mov x20, x1
-; CHECK-GI-NEXT:    bl __fixdfti
-; CHECK-GI-NEXT:    fmov d0, d9
-; CHECK-GI-NEXT:    mov x21, x0
-; CHECK-GI-NEXT:    mov x22, x1
-; CHECK-GI-NEXT:    bl __fixdfti
-; CHECK-GI-NEXT:    mov x4, x0
-; CHECK-GI-NEXT:    mov x5, x1
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    mov x2, x21
-; CHECK-GI-NEXT:    mov x3, x22
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <3 x double> %a to <3 x i128>
-  ret <3 x i128> %c
-}
-
-define <3 x i128> @fptou_v3f64_v3i128(<3 x double> %a) {
-; CHECK-SD-LABEL: fptou_v3f64_v3i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    .cfi_offset b8, -56
-; CHECK-SD-NEXT:    .cfi_offset b9, -64
-; CHECK-SD-NEXT:    fmov d8, d2
-; CHECK-SD-NEXT:    fmov d9, d1
-; CHECK-SD-NEXT:    bl __fixunsdfti
-; CHECK-SD-NEXT:    fmov d0, d9
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    bl __fixunsdfti
-; CHECK-SD-NEXT:    fmov d0, d8
-; CHECK-SD-NEXT:    mov x21, x0
-; CHECK-SD-NEXT:    mov x22, x1
-; CHECK-SD-NEXT:    bl __fixunsdfti
-; CHECK-SD-NEXT:    mov x4, x0
-; CHECK-SD-NEXT:    mov x5, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    mov x2, x21
-; CHECK-SD-NEXT:    mov x3, x22
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f64_v3i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    .cfi_offset b8, -56
-; CHECK-GI-NEXT:    .cfi_offset b9, -64
-; CHECK-GI-NEXT:    fmov d8, d1
-; CHECK-GI-NEXT:    fmov d9, d2
-; CHECK-GI-NEXT:    bl __fixunsdfti
-; CHECK-GI-NEXT:    fmov d0, d8
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    mov x20, x1
-; CHECK-GI-NEXT:    bl __fixunsdfti
-; CHECK-GI-NEXT:    fmov d0, d9
-; CHECK-GI-NEXT:    mov x21, x0
-; CHECK-GI-NEXT:    mov x22, x1
-; CHECK-GI-NEXT:    bl __fixunsdfti
-; CHECK-GI-NEXT:    mov x4, x0
-; CHECK-GI-NEXT:    mov x5, x1
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    mov x2, x21
-; CHECK-GI-NEXT:    mov x3, x22
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <3 x double> %a to <3 x i128>
-  ret <3 x i128> %c
-}
-
 define <2 x i64> @fptos_v2f32_v2i64(<2 x float> %a) {
 ; CHECK-LABEL: fptos_v2f32_v2i64:
 ; CHECK:       // %bb.0: // %entry
@@ -2571,439 +3263,291 @@ entry:
 }
 
 define <3 x i64> @fptos_v3f32_v3i64(<3 x float> %a) {
-; CHECK-SD-LABEL: fptos_v3f32_v3i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v1.2d, v0.2s
-; CHECK-SD-NEXT:    fcvtl2 v0.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v0.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    fmov d0, d3
-; CHECK-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f32_v3i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[2]
-; CHECK-GI-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl v1.2d, v1.2s
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v1.2d
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v3f32_v3i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    fmov d0, d3
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v3f32_v3i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-FP16-SD-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    fmov d0, d3
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v3f32_v3i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[2]
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v3f32_v3i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov v1.s[0], v0.s[2]
+; CHECK-FP16-GI-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <3 x float> %a to <3 x i64>
   ret <3 x i64> %c
 }
 
 define <3 x i64> @fptou_v3f32_v3i64(<3 x float> %a) {
-; CHECK-SD-LABEL: fptou_v3f32_v3i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v1.2d, v0.2s
-; CHECK-SD-NEXT:    fcvtl2 v0.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtzu v3.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzu v2.2d, v0.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    fmov d0, d3
-; CHECK-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f32_v3i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[2]
-; CHECK-GI-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl v1.2d, v1.2s
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v1.2d
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v3f32_v3i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v3.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v2.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    fmov d0, d3
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v3f32_v3i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-FP16-SD-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtzu v3.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzu v2.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    fmov d0, d3
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v3f32_v3i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[2]
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v3f32_v3i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov v1.s[0], v0.s[2]
+; CHECK-FP16-GI-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl v1.2d, v1.2s
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <3 x float> %a to <3 x i64>
   ret <3 x i64> %c
 }
 
 define <4 x i64> @fptos_v4f32_v4i64(<4 x float> %a) {
-; CHECK-SD-LABEL: fptos_v4f32_v4i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v1.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v4f32_v4i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v1.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl2 v2.2d, v0.4s
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v4f32_v4i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v4f32_v4i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v4f32_v4i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v4f32_v4i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <4 x float> %a to <4 x i64>
   ret <4 x i64> %c
 }
 
 define <4 x i64> @fptou_v4f32_v4i64(<4 x float> %a) {
-; CHECK-SD-LABEL: fptou_v4f32_v4i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v1.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-SD-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v4f32_v4i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v1.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl2 v2.2d, v0.4s
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v2.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v4f32_v4i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v4f32_v4i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl2 v1.2d, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v4f32_v4i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v4f32_v4i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <4 x float> %a to <4 x i64>
   ret <4 x i64> %c
 }
 
 define <8 x i64> @fptos_v8f32_v8i64(<8 x float> %a) {
-; CHECK-SD-LABEL: fptos_v8f32_v8i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v2.2d, v0.2s
-; CHECK-SD-NEXT:    fcvtl2 v3.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtl2 v4.2d, v1.4s
-; CHECK-SD-NEXT:    fcvtl v5.2d, v1.2s
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v5.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v8f32_v8i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v2.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl2 v3.2d, v0.4s
-; CHECK-GI-NEXT:    fcvtl v4.2d, v1.2s
-; CHECK-GI-NEXT:    fcvtl2 v5.2d, v1.4s
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v5.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f32_v8i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v4.2d, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtl v5.2d, v1.2s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v8f32_v8i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-FP16-SD-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtl2 v4.2d, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtl v5.2d, v1.2s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f32_v8i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.2d, v1.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v5.2d, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v8f32_v8i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtl v4.2d, v1.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v5.2d, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x float> %a to <8 x i64>
   ret <8 x i64> %c
 }
 
 define <8 x i64> @fptou_v8f32_v8i64(<8 x float> %a) {
-; CHECK-SD-LABEL: fptou_v8f32_v8i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v2.2d, v0.2s
-; CHECK-SD-NEXT:    fcvtl2 v3.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtl2 v4.2d, v1.4s
-; CHECK-SD-NEXT:    fcvtl v5.2d, v1.2s
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzu v1.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtzu v3.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzu v2.2d, v5.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v8f32_v8i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v2.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl2 v3.2d, v0.4s
-; CHECK-GI-NEXT:    fcvtl v4.2d, v1.2s
-; CHECK-GI-NEXT:    fcvtl2 v5.2d, v1.4s
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v5.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f32_v8i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v4.2d, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtl v5.2d, v1.2s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v2.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v8f32_v8i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-FP16-SD-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtl2 v4.2d, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtl v5.2d, v1.2s
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtzu v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    fcvtzu v2.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f32_v8i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.2d, v1.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v5.2d, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v8f32_v8i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v2.2d, v0.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v3.2d, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtl v4.2d, v1.2s
+; CHECK-FP16-GI-NEXT:    fcvtl2 v5.2d, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x float> %a to <8 x i64>
   ret <8 x i64> %c
 }
 
-define <16 x i64> @fptos_v16f32_v16i64(<16 x float> %a) {
-; CHECK-SD-LABEL: fptos_v16f32_v16i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v4.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-SD-NEXT:    fcvtl2 v5.2d, v1.4s
-; CHECK-SD-NEXT:    fcvtl v6.2d, v1.2s
-; CHECK-SD-NEXT:    fcvtl v7.2d, v2.2s
-; CHECK-SD-NEXT:    fcvtl2 v16.2d, v2.4s
-; CHECK-SD-NEXT:    fcvtl2 v17.2d, v3.4s
-; CHECK-SD-NEXT:    fcvtl v18.2d, v3.2s
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v18.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v16f32_v16i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v4.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl2 v5.2d, v0.4s
-; CHECK-GI-NEXT:    fcvtl v6.2d, v1.2s
-; CHECK-GI-NEXT:    fcvtl2 v7.2d, v1.4s
-; CHECK-GI-NEXT:    fcvtl v16.2d, v2.2s
-; CHECK-GI-NEXT:    fcvtl2 v17.2d, v2.4s
-; CHECK-GI-NEXT:    fcvtl v18.2d, v3.2s
-; CHECK-GI-NEXT:    fcvtl2 v19.2d, v3.4s
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzs v5.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzs v6.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzs v7.2d, v19.2d
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <16 x float> %a to <16 x i64>
-  ret <16 x i64> %c
-}
-
-define <16 x i64> @fptou_v16f32_v16i64(<16 x float> %a) {
-; CHECK-SD-LABEL: fptou_v16f32_v16i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v4.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-SD-NEXT:    fcvtl2 v5.2d, v1.4s
-; CHECK-SD-NEXT:    fcvtl v6.2d, v1.2s
-; CHECK-SD-NEXT:    fcvtl v7.2d, v2.2s
-; CHECK-SD-NEXT:    fcvtl2 v16.2d, v2.4s
-; CHECK-SD-NEXT:    fcvtl2 v17.2d, v3.4s
-; CHECK-SD-NEXT:    fcvtl v18.2d, v3.2s
-; CHECK-SD-NEXT:    fcvtzu v1.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtzu v3.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzu v2.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzu v4.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzu v5.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzu v7.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzu v6.2d, v18.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v16f32_v16i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v4.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl2 v5.2d, v0.4s
-; CHECK-GI-NEXT:    fcvtl v6.2d, v1.2s
-; CHECK-GI-NEXT:    fcvtl2 v7.2d, v1.4s
-; CHECK-GI-NEXT:    fcvtl v16.2d, v2.2s
-; CHECK-GI-NEXT:    fcvtl2 v17.2d, v2.4s
-; CHECK-GI-NEXT:    fcvtl v18.2d, v3.2s
-; CHECK-GI-NEXT:    fcvtl2 v19.2d, v3.4s
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v4.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v5.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzu v5.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzu v6.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzu v7.2d, v19.2d
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <16 x float> %a to <16 x i64>
-  ret <16 x i64> %c
-}
-
-define <32 x i64> @fptos_v32f32_v32i64(<32 x float> %a) {
-; CHECK-SD-LABEL: fptos_v32f32_v32i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v16.2d, v7.4s
-; CHECK-SD-NEXT:    fcvtl v7.2d, v7.2s
-; CHECK-SD-NEXT:    fcvtl2 v17.2d, v6.4s
-; CHECK-SD-NEXT:    fcvtl v6.2d, v6.2s
-; CHECK-SD-NEXT:    fcvtl2 v21.2d, v2.4s
-; CHECK-SD-NEXT:    fcvtl v2.2d, v2.2s
-; CHECK-SD-NEXT:    fcvtl2 v19.2d, v4.4s
-; CHECK-SD-NEXT:    fcvtl v4.2d, v4.2s
-; CHECK-SD-NEXT:    fcvtl2 v18.2d, v5.4s
-; CHECK-SD-NEXT:    fcvtl2 v20.2d, v3.4s
-; CHECK-SD-NEXT:    fcvtl v5.2d, v5.2s
-; CHECK-SD-NEXT:    fcvtl v3.2d, v3.2s
-; CHECK-SD-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzs v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzs v20.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-SD-NEXT:    stp q7, q16, [x8, #224]
-; CHECK-SD-NEXT:    fcvtzs v16.2d, v21.2d
-; CHECK-SD-NEXT:    stp q6, q17, [x8, #192]
-; CHECK-SD-NEXT:    fcvtl2 v17.2d, v1.4s
-; CHECK-SD-NEXT:    fcvtl v1.2d, v1.2s
-; CHECK-SD-NEXT:    stp q4, q19, [x8, #128]
-; CHECK-SD-NEXT:    stp q3, q20, [x8, #96]
-; CHECK-SD-NEXT:    stp q2, q16, [x8, #64]
-; CHECK-SD-NEXT:    fcvtl2 v16.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-SD-NEXT:    stp q5, q18, [x8, #160]
-; CHECK-SD-NEXT:    fcvtzs v6.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzs v4.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q1, q6, [x8, #32]
-; CHECK-SD-NEXT:    stp q0, q4, [x8]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v32f32_v32i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v16.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl2 v0.2d, v0.4s
-; CHECK-GI-NEXT:    fcvtl v17.2d, v1.2s
-; CHECK-GI-NEXT:    fcvtl2 v1.2d, v1.4s
-; CHECK-GI-NEXT:    fcvtl v18.2d, v2.2s
-; CHECK-GI-NEXT:    fcvtl2 v2.2d, v2.4s
-; CHECK-GI-NEXT:    fcvtl v19.2d, v3.2s
-; CHECK-GI-NEXT:    fcvtl2 v3.2d, v3.4s
-; CHECK-GI-NEXT:    fcvtl v20.2d, v4.2s
-; CHECK-GI-NEXT:    fcvtl2 v4.2d, v4.4s
-; CHECK-GI-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v18.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q16, q0, [x8]
-; CHECK-GI-NEXT:    fcvtl v0.2d, v5.2s
-; CHECK-GI-NEXT:    fcvtl2 v5.2d, v5.4s
-; CHECK-GI-NEXT:    stp q17, q1, [x8, #32]
-; CHECK-GI-NEXT:    fcvtzs v16.2d, v19.2d
-; CHECK-GI-NEXT:    fcvtl v1.2d, v6.2s
-; CHECK-GI-NEXT:    stp q18, q2, [x8, #64]
-; CHECK-GI-NEXT:    fcvtl2 v6.2d, v6.4s
-; CHECK-GI-NEXT:    fcvtl v2.2d, v7.2s
-; CHECK-GI-NEXT:    fcvtl2 v7.2d, v7.4s
-; CHECK-GI-NEXT:    fcvtzs v17.2d, v20.2d
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-NEXT:    stp q16, q3, [x8, #96]
-; CHECK-GI-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzs v3.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NEXT:    stp q17, q4, [x8, #128]
-; CHECK-GI-NEXT:    stp q0, q5, [x8, #160]
-; CHECK-GI-NEXT:    fcvtzs v0.2d, v7.2d
-; CHECK-GI-NEXT:    stp q1, q3, [x8, #192]
-; CHECK-GI-NEXT:    stp q2, q0, [x8, #224]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <32 x float> %a to <32 x i64>
-  ret <32 x i64> %c
-}
-
-define <32 x i64> @fptou_v32f32_v32i64(<32 x float> %a) {
-; CHECK-SD-LABEL: fptou_v32f32_v32i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v16.2d, v7.4s
-; CHECK-SD-NEXT:    fcvtl v7.2d, v7.2s
-; CHECK-SD-NEXT:    fcvtl2 v17.2d, v6.4s
-; CHECK-SD-NEXT:    fcvtl v6.2d, v6.2s
-; CHECK-SD-NEXT:    fcvtl2 v21.2d, v2.4s
-; CHECK-SD-NEXT:    fcvtl v2.2d, v2.2s
-; CHECK-SD-NEXT:    fcvtl2 v19.2d, v4.4s
-; CHECK-SD-NEXT:    fcvtl v4.2d, v4.2s
-; CHECK-SD-NEXT:    fcvtl2 v18.2d, v5.4s
-; CHECK-SD-NEXT:    fcvtl2 v20.2d, v3.4s
-; CHECK-SD-NEXT:    fcvtl v5.2d, v5.2s
-; CHECK-SD-NEXT:    fcvtl v3.2d, v3.2s
-; CHECK-SD-NEXT:    fcvtzu v16.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtzu v17.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-SD-NEXT:    fcvtzu v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-SD-NEXT:    fcvtzu v18.2d, v18.2d
-; CHECK-SD-NEXT:    fcvtzu v20.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-SD-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-SD-NEXT:    stp q7, q16, [x8, #224]
-; CHECK-SD-NEXT:    fcvtzu v16.2d, v21.2d
-; CHECK-SD-NEXT:    stp q6, q17, [x8, #192]
-; CHECK-SD-NEXT:    fcvtl2 v17.2d, v1.4s
-; CHECK-SD-NEXT:    fcvtl v1.2d, v1.2s
-; CHECK-SD-NEXT:    stp q4, q19, [x8, #128]
-; CHECK-SD-NEXT:    stp q3, q20, [x8, #96]
-; CHECK-SD-NEXT:    stp q2, q16, [x8, #64]
-; CHECK-SD-NEXT:    fcvtl2 v16.2d, v0.4s
-; CHECK-SD-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-SD-NEXT:    stp q5, q18, [x8, #160]
-; CHECK-SD-NEXT:    fcvtzu v6.2d, v17.2d
-; CHECK-SD-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-SD-NEXT:    fcvtzu v4.2d, v16.2d
-; CHECK-SD-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q1, q6, [x8, #32]
-; CHECK-SD-NEXT:    stp q0, q4, [x8]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v32f32_v32i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v16.2d, v0.2s
-; CHECK-GI-NEXT:    fcvtl2 v0.2d, v0.4s
-; CHECK-GI-NEXT:    fcvtl v17.2d, v1.2s
-; CHECK-GI-NEXT:    fcvtl2 v1.2d, v1.4s
-; CHECK-GI-NEXT:    fcvtl v18.2d, v2.2s
-; CHECK-GI-NEXT:    fcvtl2 v2.2d, v2.4s
-; CHECK-GI-NEXT:    fcvtl v19.2d, v3.2s
-; CHECK-GI-NEXT:    fcvtl2 v3.2d, v3.4s
-; CHECK-GI-NEXT:    fcvtl v20.2d, v4.2s
-; CHECK-GI-NEXT:    fcvtl2 v4.2d, v4.4s
-; CHECK-GI-NEXT:    fcvtzu v16.2d, v16.2d
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v17.2d, v17.2d
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v18.2d, v18.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q16, q0, [x8]
-; CHECK-GI-NEXT:    fcvtl v0.2d, v5.2s
-; CHECK-GI-NEXT:    fcvtl2 v5.2d, v5.4s
-; CHECK-GI-NEXT:    stp q17, q1, [x8, #32]
-; CHECK-GI-NEXT:    fcvtzu v16.2d, v19.2d
-; CHECK-GI-NEXT:    fcvtl v1.2d, v6.2s
-; CHECK-GI-NEXT:    stp q18, q2, [x8, #64]
-; CHECK-GI-NEXT:    fcvtl2 v6.2d, v6.4s
-; CHECK-GI-NEXT:    fcvtl v2.2d, v7.2s
-; CHECK-GI-NEXT:    fcvtl2 v7.2d, v7.4s
-; CHECK-GI-NEXT:    fcvtzu v17.2d, v20.2d
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-NEXT:    stp q16, q3, [x8, #96]
-; CHECK-GI-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NEXT:    fcvtzu v3.2d, v6.2d
-; CHECK-GI-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NEXT:    stp q17, q4, [x8, #128]
-; CHECK-GI-NEXT:    stp q0, q5, [x8, #160]
-; CHECK-GI-NEXT:    fcvtzu v0.2d, v7.2d
-; CHECK-GI-NEXT:    stp q1, q3, [x8, #192]
-; CHECK-GI-NEXT:    stp q2, q0, [x8, #224]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <32 x float> %a to <32 x i64>
-  ret <32 x i64> %c
-}
-
 define <2 x i32> @fptos_v2f32_v2i32(<2 x float> %a) {
 ; CHECK-LABEL: fptos_v2f32_v2i32:
 ; CHECK:       // %bb.0: // %entry
@@ -3086,66 +3630,6 @@ entry:
   ret <8 x i32> %c
 }
 
-define <16 x i32> @fptos_v16f32_v16i32(<16 x float> %a) {
-; CHECK-LABEL: fptos_v16f32_v16i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = fptosi <16 x float> %a to <16 x i32>
-  ret <16 x i32> %c
-}
-
-define <16 x i32> @fptou_v16f32_v16i32(<16 x float> %a) {
-; CHECK-LABEL: fptou_v16f32_v16i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = fptoui <16 x float> %a to <16 x i32>
-  ret <16 x i32> %c
-}
-
-define <32 x i32> @fptos_v32f32_v32i32(<32 x float> %a) {
-; CHECK-LABEL: fptos_v32f32_v32i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = fptosi <32 x float> %a to <32 x i32>
-  ret <32 x i32> %c
-}
-
-define <32 x i32> @fptou_v32f32_v32i32(<32 x float> %a) {
-; CHECK-LABEL: fptou_v32f32_v32i32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-NEXT:    fcvtzu v4.4s, v4.4s
-; CHECK-NEXT:    fcvtzu v5.4s, v5.4s
-; CHECK-NEXT:    fcvtzu v6.4s, v6.4s
-; CHECK-NEXT:    fcvtzu v7.4s, v7.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = fptoui <32 x float> %a to <32 x i32>
-  ret <32 x i32> %c
-}
-
 define <2 x i16> @fptos_v2f32_v2i16(<2 x float> %a) {
 ; CHECK-LABEL: fptos_v2f32_v2i16:
 ; CHECK:       // %bb.0: // %entry
@@ -3157,15 +3641,25 @@ entry:
 }
 
 define <2 x i16> @fptou_v2f32_v2i16(<2 x float> %a) {
-; CHECK-SD-LABEL: fptou_v2f32_v2i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v0.2s, v0.2s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f32_v2i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f32_v2i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f32_v2i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f32_v2i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f32_v2i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x float> %a to <2 x i16>
   ret <2 x i16> %c
@@ -3216,167 +3710,161 @@ entry:
 }
 
 define <8 x i16> @fptos_v8f32_v8i16(<8 x float> %a) {
-; CHECK-SD-LABEL: fptos_v8f32_v8i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v8f32_v8i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f32_v8i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v8f32_v8i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f32_v8i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v8f32_v8i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x float> %a to <8 x i16>
   ret <8 x i16> %c
 }
 
 define <8 x i16> @fptou_v8f32_v8i16(<8 x float> %a) {
-; CHECK-SD-LABEL: fptou_v8f32_v8i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v8f32_v8i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f32_v8i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v8f32_v8i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f32_v8i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v8f32_v8i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x float> %a to <8 x i16>
   ret <8 x i16> %c
 }
 
 define <16 x i16> @fptos_v16f32_v16i16(<16 x float> %a) {
-; CHECK-SD-LABEL: fptos_v16f32_v16i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v16f32_v16i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v16f32_v16i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v16f32_v16i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v16f32_v16i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v16f32_v16i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <16 x float> %a to <16 x i16>
   ret <16 x i16> %c
 }
 
 define <16 x i16> @fptou_v16f32_v16i16(<16 x float> %a) {
-; CHECK-SD-LABEL: fptou_v16f32_v16i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v16f32_v16i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v16f32_v16i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v16f32_v16i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v16f32_v16i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v16f32_v16i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <16 x float> %a to <16 x i16>
   ret <16 x i16> %c
 }
 
-define <32 x i16> @fptos_v32f32_v32i16(<32 x float> %a) {
-; CHECK-SD-LABEL: fptos_v32f32_v32i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-SD-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-SD-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-SD-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-SD-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-SD-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v32f32_v32i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-GI-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-GI-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-GI-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <32 x float> %a to <32 x i16>
-  ret <32 x i16> %c
-}
-
-define <32 x i16> @fptou_v32f32_v32i16(<32 x float> %a) {
-; CHECK-SD-LABEL: fptou_v32f32_v32i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-SD-NEXT:    fcvtzu v5.4s, v5.4s
-; CHECK-SD-NEXT:    fcvtzu v4.4s, v4.4s
-; CHECK-SD-NEXT:    fcvtzu v7.4s, v7.4s
-; CHECK-SD-NEXT:    fcvtzu v6.4s, v6.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-SD-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-SD-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v32f32_v32i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-NEXT:    fcvtzu v4.4s, v4.4s
-; CHECK-GI-NEXT:    fcvtzu v5.4s, v5.4s
-; CHECK-GI-NEXT:    fcvtzu v6.4s, v6.4s
-; CHECK-GI-NEXT:    fcvtzu v7.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <32 x float> %a to <32 x i16>
-  ret <32 x i16> %c
-}
-
 define <2 x i8> @fptos_v2f32_v2i8(<2 x float> %a) {
 ; CHECK-LABEL: fptos_v2f32_v2i8:
 ; CHECK:       // %bb.0: // %entry
@@ -3388,63 +3876,111 @@ entry:
 }
 
 define <2 x i8> @fptou_v2f32_v2i8(<2 x float> %a) {
-; CHECK-SD-LABEL: fptou_v2f32_v2i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v0.2s, v0.2s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f32_v2i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f32_v2i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f32_v2i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f32_v2i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f32_v2i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x float> %a to <2 x i8>
   ret <2 x i8> %c
 }
 
 define <3 x i8> @fptos_v3f32_v3i8(<3 x float> %a) {
-; CHECK-SD-LABEL: fptos_v3f32_v3i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f32_v3i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    mov s2, v0.s[2]
-; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    fmov w1, s1
-; CHECK-GI-NEXT:    fmov w2, s2
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v3f32_v3i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v3f32_v3i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-FP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v3f32_v3i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-NOFP16-GI-NEXT:    fmov w0, s0
+; CHECK-NOFP16-GI-NEXT:    fmov w1, s1
+; CHECK-NOFP16-GI-NEXT:    fmov w2, s2
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v3f32_v3i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-FP16-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-FP16-GI-NEXT:    fmov w0, s0
+; CHECK-FP16-GI-NEXT:    fmov w1, s1
+; CHECK-FP16-GI-NEXT:    fmov w2, s2
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <3 x float> %a to <3 x i8>
   ret <3 x i8> %c
 }
 
 define <3 x i8> @fptou_v3f32_v3i8(<3 x float> %a) {
-; CHECK-SD-LABEL: fptou_v3f32_v3i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f32_v3i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NEXT:    mov s2, v0.s[2]
-; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    fmov w1, s1
-; CHECK-GI-NEXT:    fmov w2, s2
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v3f32_v3i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v3f32_v3i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-FP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v3f32_v3i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-NOFP16-GI-NEXT:    fmov w0, s0
+; CHECK-NOFP16-GI-NEXT:    fmov w1, s1
+; CHECK-NOFP16-GI-NEXT:    fmov w2, s2
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v3f32_v3i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-FP16-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-FP16-GI-NEXT:    fmov w0, s0
+; CHECK-FP16-GI-NEXT:    fmov w1, s1
+; CHECK-FP16-GI-NEXT:    fmov w2, s2
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <3 x float> %a to <3 x i8>
   ret <3 x i8> %c
@@ -3462,2267 +3998,1597 @@ entry:
 }
 
 define <4 x i8> @fptou_v4f32_v4i8(<4 x float> %a) {
-; CHECK-SD-LABEL: fptou_v4f32_v4i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v4f32_v4i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v4f32_v4i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v4f32_v4i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v4f32_v4i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v4f32_v4i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <4 x float> %a to <4 x i8>
   ret <4 x i8> %c
 }
 
 define <8 x i8> @fptos_v8f32_v8i8(<8 x float> %a) {
-; CHECK-SD-LABEL: fptos_v8f32_v8i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v8f32_v8i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f32_v8i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v8f32_v8i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f32_v8i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v8f32_v8i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x float> %a to <8 x i8>
   ret <8 x i8> %c
 }
 
 define <8 x i8> @fptou_v8f32_v8i8(<8 x float> %a) {
-; CHECK-SD-LABEL: fptou_v8f32_v8i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v8f32_v8i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f32_v8i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v8f32_v8i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f32_v8i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v8f32_v8i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x float> %a to <8 x i8>
   ret <8 x i8> %c
 }
 
 define <16 x i8> @fptos_v16f32_v16i8(<16 x float> %a) {
-; CHECK-SD-LABEL: fptos_v16f32_v16i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v16f32_v16i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v16f32_v16i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v16f32_v16i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v16f32_v16i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v16f32_v16i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <16 x float> %a to <16 x i8>
   ret <16 x i8> %c
 }
 
 define <16 x i8> @fptou_v16f32_v16i8(<16 x float> %a) {
-; CHECK-SD-LABEL: fptou_v16f32_v16i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v16f32_v16i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v16f32_v16i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v16f32_v16i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v16f32_v16i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v16f32_v16i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <16 x float> %a to <16 x i8>
   ret <16 x i8> %c
 }
 
 define <32 x i8> @fptos_v32f32_v32i8(<32 x float> %a) {
-; CHECK-SD-LABEL: fptos_v32f32_v32i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-SD-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-SD-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-SD-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v1.8h, v6.8h, v7.8h
-; CHECK-SD-NEXT:    uzp1 v3.8h, v4.8h, v5.8h
-; CHECK-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT:    uzp1 v1.16b, v3.16b, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v32f32_v32i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-GI-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-GI-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-GI-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v32f32_v32i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v6.8h, v7.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v3.8h, v4.8h, v5.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v32f32_v32i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v1.8h, v6.8h, v7.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v3.8h, v4.8h, v5.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    uzp1 v1.16b, v3.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v32f32_v32i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v32f32_v32i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <32 x float> %a to <32 x i8>
   ret <32 x i8> %c
 }
 
 define <32 x i8> @fptou_v32f32_v32i8(<32 x float> %a) {
-; CHECK-SD-LABEL: fptou_v32f32_v32i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-SD-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-SD-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-SD-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
-; CHECK-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT:    uzp1 v1.8h, v6.8h, v7.8h
-; CHECK-SD-NEXT:    uzp1 v3.8h, v4.8h, v5.8h
-; CHECK-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-SD-NEXT:    uzp1 v1.16b, v3.16b, v1.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v32f32_v32i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-NEXT:    fcvtzu v4.4s, v4.4s
-; CHECK-GI-NEXT:    fcvtzu v5.4s, v5.4s
-; CHECK-GI-NEXT:    fcvtzu v6.4s, v6.4s
-; CHECK-GI-NEXT:    fcvtzu v7.4s, v7.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
-; CHECK-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
-; CHECK-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
-; CHECK-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v32f32_v32i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v6.8h, v7.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v3.8h, v4.8h, v5.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.16b, v3.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v32f32_v32i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v1.8h, v6.8h, v7.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v3.8h, v4.8h, v5.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    uzp1 v1.16b, v3.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v32f32_v32i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v32f32_v32i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v2.8h, v4.8h, v5.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v3.8h, v6.8h, v7.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <32 x float> %a to <32 x i8>
   ret <32 x i8> %c
 }
 
 define <2 x i128> @fptos_v2f32_v2i128(<2 x float> %a) {
-; CHECK-SD-LABEL: fptos_v2f32_v2i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl __fixsfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    mov s0, v0.s[1]
-; CHECK-SD-NEXT:    bl __fixsfti
-; CHECK-SD-NEXT:    mov x2, x0
-; CHECK-SD-NEXT:    mov x3, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v2f32_v2i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -24
-; CHECK-GI-NEXT:    .cfi_offset b8, -32
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov s8, v0.s[1]
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-GI-NEXT:    bl __fixsfti
-; CHECK-GI-NEXT:    fmov s0, s8
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    mov x20, x1
-; CHECK-GI-NEXT:    bl __fixsfti
-; CHECK-GI-NEXT:    mov x2, x0
-; CHECK-GI-NEXT:    mov x3, x1
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f32_v2i128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    bl __fixsfti
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov x19, x0
+; CHECK-NOFP16-SD-NEXT:    mov x20, x1
+; CHECK-NOFP16-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-NOFP16-SD-NEXT:    bl __fixsfti
+; CHECK-NOFP16-SD-NEXT:    mov x2, x0
+; CHECK-NOFP16-SD-NEXT:    mov x3, x1
+; CHECK-NOFP16-SD-NEXT:    mov x0, x19
+; CHECK-NOFP16-SD-NEXT:    mov x1, x20
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f32_v2i128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixsfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-FP16-SD-NEXT:    bl __fixsfti
+; CHECK-FP16-SD-NEXT:    mov x2, x0
+; CHECK-FP16-SD-NEXT:    mov x3, x1
+; CHECK-FP16-SD-NEXT:    mov x0, x19
+; CHECK-FP16-SD-NEXT:    mov x1, x20
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f32_v2i128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    bl __fixsfti
+; CHECK-NOFP16-GI-NEXT:    fmov s0, s8
+; CHECK-NOFP16-GI-NEXT:    mov x19, x0
+; CHECK-NOFP16-GI-NEXT:    mov x20, x1
+; CHECK-NOFP16-GI-NEXT:    bl __fixsfti
+; CHECK-NOFP16-GI-NEXT:    mov x2, x0
+; CHECK-NOFP16-GI-NEXT:    mov x3, x1
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f32_v2i128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    fmov s0, s8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixsfti
+; CHECK-FP16-GI-NEXT:    mov x2, x0
+; CHECK-FP16-GI-NEXT:    mov x3, x1
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x float> %a to <2 x i128>
   ret <2 x i128> %c
 }
 
 define <2 x i128> @fptou_v2f32_v2i128(<2 x float> %a) {
-; CHECK-SD-LABEL: fptou_v2f32_v2i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl __fixunssfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    mov s0, v0.s[1]
-; CHECK-SD-NEXT:    bl __fixunssfti
-; CHECK-SD-NEXT:    mov x2, x0
-; CHECK-SD-NEXT:    mov x3, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f32_v2i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -24
-; CHECK-GI-NEXT:    .cfi_offset b8, -32
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    mov s8, v0.s[1]
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-GI-NEXT:    bl __fixunssfti
-; CHECK-GI-NEXT:    fmov s0, s8
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    mov x20, x1
-; CHECK-GI-NEXT:    bl __fixunssfti
-; CHECK-GI-NEXT:    mov x2, x0
-; CHECK-GI-NEXT:    mov x3, x1
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f32_v2i128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    bl __fixunssfti
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov x19, x0
+; CHECK-NOFP16-SD-NEXT:    mov x20, x1
+; CHECK-NOFP16-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-NOFP16-SD-NEXT:    bl __fixunssfti
+; CHECK-NOFP16-SD-NEXT:    mov x2, x0
+; CHECK-NOFP16-SD-NEXT:    mov x3, x1
+; CHECK-NOFP16-SD-NEXT:    mov x0, x19
+; CHECK-NOFP16-SD-NEXT:    mov x1, x20
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f32_v2i128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixunssfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov s0, v0.s[1]
+; CHECK-FP16-SD-NEXT:    bl __fixunssfti
+; CHECK-FP16-SD-NEXT:    mov x2, x0
+; CHECK-FP16-SD-NEXT:    mov x3, x1
+; CHECK-FP16-SD-NEXT:    mov x0, x19
+; CHECK-FP16-SD-NEXT:    mov x1, x20
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f32_v2i128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    bl __fixunssfti
+; CHECK-NOFP16-GI-NEXT:    fmov s0, s8
+; CHECK-NOFP16-GI-NEXT:    mov x19, x0
+; CHECK-NOFP16-GI-NEXT:    mov x20, x1
+; CHECK-NOFP16-GI-NEXT:    bl __fixunssfti
+; CHECK-NOFP16-GI-NEXT:    mov x2, x0
+; CHECK-NOFP16-GI-NEXT:    mov x3, x1
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f32_v2i128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #8] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -24
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -32
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-FP16-GI-NEXT:    bl __fixunssfti
+; CHECK-FP16-GI-NEXT:    fmov s0, s8
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    mov x20, x1
+; CHECK-FP16-GI-NEXT:    bl __fixunssfti
+; CHECK-FP16-GI-NEXT:    mov x2, x0
+; CHECK-FP16-GI-NEXT:    mov x3, x1
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #8] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x float> %a to <2 x i128>
   ret <2 x i128> %c
 }
 
-define <3 x i128> @fptos_v3f32_v3i128(<3 x float> %a) {
-; CHECK-SD-LABEL: fptos_v3f32_v3i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl __fixsfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl __fixsfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x21, x0
-; CHECK-SD-NEXT:    mov x22, x1
-; CHECK-SD-NEXT:    mov s0, v0.s[1]
-; CHECK-SD-NEXT:    bl __fixsfti
-; CHECK-SD-NEXT:    mov x2, x0
-; CHECK-SD-NEXT:    mov x3, x1
-; CHECK-SD-NEXT:    mov x0, x21
-; CHECK-SD-NEXT:    mov x1, x22
-; CHECK-SD-NEXT:    mov x4, x19
-; CHECK-SD-NEXT:    mov x5, x20
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f32_v3i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    .cfi_offset b8, -56
-; CHECK-GI-NEXT:    .cfi_offset b9, -64
-; CHECK-GI-NEXT:    mov s8, v0.s[1]
-; CHECK-GI-NEXT:    mov s9, v0.s[2]
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-GI-NEXT:    bl __fixsfti
-; CHECK-GI-NEXT:    fmov s0, s8
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    mov x20, x1
-; CHECK-GI-NEXT:    bl __fixsfti
-; CHECK-GI-NEXT:    fmov s0, s9
-; CHECK-GI-NEXT:    mov x21, x0
-; CHECK-GI-NEXT:    mov x22, x1
-; CHECK-GI-NEXT:    bl __fixsfti
-; CHECK-GI-NEXT:    mov x4, x0
-; CHECK-GI-NEXT:    mov x5, x1
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    mov x2, x21
-; CHECK-GI-NEXT:    mov x3, x22
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <3 x float> %a to <3 x i128>
-  ret <3 x i128> %c
-}
-
-define <3 x i128> @fptou_v3f32_v3i128(<3 x float> %a) {
-; CHECK-SD-LABEL: fptou_v3f32_v3i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl __fixunssfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-SD-NEXT:    bl __fixunssfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x21, x0
-; CHECK-SD-NEXT:    mov x22, x1
-; CHECK-SD-NEXT:    mov s0, v0.s[1]
-; CHECK-SD-NEXT:    bl __fixunssfti
-; CHECK-SD-NEXT:    mov x2, x0
-; CHECK-SD-NEXT:    mov x3, x1
-; CHECK-SD-NEXT:    mov x0, x21
-; CHECK-SD-NEXT:    mov x1, x22
-; CHECK-SD-NEXT:    mov x4, x19
-; CHECK-SD-NEXT:    mov x5, x20
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f32_v3i128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    .cfi_offset b8, -56
-; CHECK-GI-NEXT:    .cfi_offset b9, -64
-; CHECK-GI-NEXT:    mov s8, v0.s[1]
-; CHECK-GI-NEXT:    mov s9, v0.s[2]
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-GI-NEXT:    bl __fixunssfti
-; CHECK-GI-NEXT:    fmov s0, s8
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    mov x20, x1
-; CHECK-GI-NEXT:    bl __fixunssfti
-; CHECK-GI-NEXT:    fmov s0, s9
-; CHECK-GI-NEXT:    mov x21, x0
-; CHECK-GI-NEXT:    mov x22, x1
-; CHECK-GI-NEXT:    bl __fixunssfti
-; CHECK-GI-NEXT:    mov x4, x0
-; CHECK-GI-NEXT:    mov x5, x1
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    mov x2, x21
-; CHECK-GI-NEXT:    mov x3, x22
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <3 x float> %a to <3 x i128>
-  ret <3 x i128> %c
-}
-
 define <2 x i64> @fptos_v2f16_v2i64(<2 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v2f16_v2i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s1
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x9
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v2f16_v2i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzs x8, h0
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h1
-; CHECK-SD-FP16-NEXT:    fmov d0, x8
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x9
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v2f16_v2i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v2f16_v2i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d1, h1
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f16_v2i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x8, s0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x9, s1
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x8
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], x9
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f16_v2i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    fcvtzs x8, h0
+; CHECK-FP16-SD-NEXT:    fcvtzs x9, h1
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f16_v2i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f16_v2i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d1, h1
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x half> %a to <2 x i64>
   ret <2 x i64> %c
 }
 
 define <2 x i64> @fptou_v2f16_v2i64(<2 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v2f16_v2i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s0
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s1
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x9
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v2f16_v2i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzu x8, h0
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h1
-; CHECK-SD-FP16-NEXT:    fmov d0, x8
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x9
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v2f16_v2i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v2f16_v2i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d1, h1
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f16_v2i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x8, s0
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x9, s1
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x8
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], x9
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f16_v2i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    fcvtzu x8, h0
+; CHECK-FP16-SD-NEXT:    fcvtzu x9, h1
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x9
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f16_v2i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f16_v2i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d1, h1
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x half> %a to <2 x i64>
   ret <2 x i64> %c
 }
 
 define <3 x i64> @fptos_v3f16_v3i64(<3 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v3f16_v3i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s1
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s2
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x9
-; CHECK-SD-NOFP16-NEXT:    fmov d2, x10
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v3f16_v3i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzs x8, h0
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h1
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h2
-; CHECK-SD-FP16-NEXT:    fmov d0, x8
-; CHECK-SD-FP16-NEXT:    fmov d1, x9
-; CHECK-SD-FP16-NEXT:    fmov d2, x10
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v3f16_v3i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NOFP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v3f16_v3i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
-; CHECK-GI-FP16-NEXT:    fcvt d1, h0
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[2]
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d2, h2
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v2.d[0]
-; CHECK-GI-FP16-NEXT:    fcvt d2, h3
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v3f16_v3i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h2, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x8, s0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x9, s1
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x10, s2
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x8
+; CHECK-NOFP16-SD-NEXT:    fmov d1, x9
+; CHECK-NOFP16-SD-NEXT:    fmov d2, x10
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v3f16_v3i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-SD-NEXT:    fcvtzs x8, h0
+; CHECK-FP16-SD-NEXT:    fcvtzs x9, h1
+; CHECK-FP16-SD-NEXT:    fcvtzs x10, h2
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    fmov d2, x10
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v3f16_v3i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v3f16_v3i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvt d1, h0
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT:    fcvt d2, h3
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <3 x half> %a to <3 x i64>
   ret <3 x i64> %c
 }
 
 define <3 x i64> @fptou_v3f16_v3i64(<3 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v3f16_v3i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s0
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s1
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s2
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x9
-; CHECK-SD-NOFP16-NEXT:    fmov d2, x10
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v3f16_v3i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzu x8, h0
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h1
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h2
-; CHECK-SD-FP16-NEXT:    fmov d0, x8
-; CHECK-SD-FP16-NEXT:    fmov d1, x9
-; CHECK-SD-FP16-NEXT:    fmov d2, x10
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v3f16_v3i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NOFP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v3f16_v3i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
-; CHECK-GI-FP16-NEXT:    fcvt d1, h0
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[2]
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d2, h2
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v2.d[0]
-; CHECK-GI-FP16-NEXT:    fcvt d2, h3
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-FP16-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v3f16_v3i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h2, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x8, s0
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x9, s1
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x10, s2
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x8
+; CHECK-NOFP16-SD-NEXT:    fmov d1, x9
+; CHECK-NOFP16-SD-NEXT:    fmov d2, x10
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v3f16_v3i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-SD-NEXT:    fcvtzu x8, h0
+; CHECK-FP16-SD-NEXT:    fcvtzu x9, h1
+; CHECK-FP16-SD-NEXT:    fcvtzu x10, h2
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    fmov d2, x10
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v3f16_v3i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v3f16_v3i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvt d1, h0
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[2]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v2.d[0]
+; CHECK-FP16-GI-NEXT:    fcvt d2, h3
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <3 x half> %a to <3 x i64>
   ret <3 x i64> %c
 }
 
 define <4 x i64> @fptos_v4f16_v4i64(<4 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v4f16_v4i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s1
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s2
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s3
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x9
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x10
-; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v4f16_v4i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[2]
-; CHECK-SD-FP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x8, h0
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h1
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h2
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h3
-; CHECK-SD-FP16-NEXT:    fmov d0, x8
-; CHECK-SD-FP16-NEXT:    fmov d1, x9
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x10
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x11
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v4f16_v4i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v4f16_v4i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d1, h1
-; CHECK-GI-FP16-NEXT:    fcvt d2, h2
-; CHECK-GI-FP16-NEXT:    fcvt d3, h3
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v4f16_v4i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov h1, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h2, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x8, s0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x9, s1
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x10, s2
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x11, s3
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x8
+; CHECK-NOFP16-SD-NEXT:    fmov d1, x9
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], x10
+; CHECK-NOFP16-SD-NEXT:    mov v1.d[1], x11
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v4f16_v4i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov h1, v0.h[2]
+; CHECK-FP16-SD-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-SD-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzs x8, h0
+; CHECK-FP16-SD-NEXT:    fcvtzs x9, h1
+; CHECK-FP16-SD-NEXT:    fcvtzs x10, h2
+; CHECK-FP16-SD-NEXT:    fcvtzs x11, h3
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x10
+; CHECK-FP16-SD-NEXT:    mov v1.d[1], x11
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v4f16_v4i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v4f16_v4i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d1, h1
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <4 x half> %a to <4 x i64>
   ret <4 x i64> %c
 }
 
 define <4 x i64> @fptou_v4f16_v4i64(<4 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v4f16_v4i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov h1, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s0
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s1
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s2
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s3
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x9
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x10
-; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v4f16_v4i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[2]
-; CHECK-SD-FP16-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x8, h0
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h1
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h2
-; CHECK-SD-FP16-NEXT:    fcvtzu x11, h3
-; CHECK-SD-FP16-NEXT:    fmov d0, x8
-; CHECK-SD-FP16-NEXT:    fmov d1, x9
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x10
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x11
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v4f16_v4i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v4f16_v4i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d1, h1
-; CHECK-GI-FP16-NEXT:    fcvt d2, h2
-; CHECK-GI-FP16-NEXT:    fcvt d3, h3
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v4f16_v4i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov h1, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h2, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x8, s0
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x9, s1
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x10, s2
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x11, s3
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x8
+; CHECK-NOFP16-SD-NEXT:    fmov d1, x9
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], x10
+; CHECK-NOFP16-SD-NEXT:    mov v1.d[1], x11
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v4f16_v4i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov h1, v0.h[2]
+; CHECK-FP16-SD-NEXT:    mov h2, v0.h[1]
+; CHECK-FP16-SD-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzu x8, h0
+; CHECK-FP16-SD-NEXT:    fcvtzu x9, h1
+; CHECK-FP16-SD-NEXT:    fcvtzu x10, h2
+; CHECK-FP16-SD-NEXT:    fcvtzu x11, h3
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x9
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x10
+; CHECK-FP16-SD-NEXT:    mov v1.d[1], x11
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v4f16_v4i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v4f16_v4i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d1, h1
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <4 x half> %a to <4 x i64>
   ret <4 x i64> %c
 }
 
 define <8 x i64> @fptos_v8f16_v8i64(<8 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v8f16_v8i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s1
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s4
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s3
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s7
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x9
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s2
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s5
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s6
-; CHECK-SD-NOFP16-NEXT:    fmov d2, x8
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x12
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    fmov d3, x10
-; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x15
-; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v8f16_v8i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-FP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-FP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov h7, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h0
-; CHECK-SD-FP16-NEXT:    mov h2, v1.h[2]
-; CHECK-SD-FP16-NEXT:    mov h5, v1.h[1]
-; CHECK-SD-FP16-NEXT:    mov h6, v1.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x8, h1
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h4
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h3
-; CHECK-SD-FP16-NEXT:    fcvtzs x15, h7
-; CHECK-SD-FP16-NEXT:    fmov d0, x9
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h2
-; CHECK-SD-FP16-NEXT:    fcvtzs x13, h5
-; CHECK-SD-FP16-NEXT:    fcvtzs x14, h6
-; CHECK-SD-FP16-NEXT:    fmov d2, x8
-; CHECK-SD-FP16-NEXT:    fmov d1, x12
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x11
-; CHECK-SD-FP16-NEXT:    fmov d3, x10
-; CHECK-SD-FP16-NEXT:    mov v2.d[1], x13
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x15
-; CHECK-SD-FP16-NEXT:    mov v3.d[1], x14
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v8f16_v8i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.2d, v1.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.2d, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v8f16_v8i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    mov h4, v0.h[4]
-; CHECK-GI-FP16-NEXT:    mov h5, v0.h[5]
-; CHECK-GI-FP16-NEXT:    mov h6, v0.h[6]
-; CHECK-GI-FP16-NEXT:    mov h7, v0.h[7]
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d1, h1
-; CHECK-GI-FP16-NEXT:    fcvt d2, h2
-; CHECK-GI-FP16-NEXT:    fcvt d3, h3
-; CHECK-GI-FP16-NEXT:    fcvt d4, h4
-; CHECK-GI-FP16-NEXT:    fcvt d5, h5
-; CHECK-GI-FP16-NEXT:    fcvt d6, h6
-; CHECK-GI-FP16-NEXT:    fcvt d7, h7
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT:    mov v4.d[1], v5.d[0]
-; CHECK-GI-FP16-NEXT:    mov v6.d[1], v7.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v2.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v3.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f16_v8i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h4
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h7
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x9, s0
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x8, s1
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x12, s4
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x11, s3
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x15, s7
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x9
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x10, s2
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x13, s5
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x14, s6
+; CHECK-NOFP16-SD-NEXT:    fmov d2, x8
+; CHECK-NOFP16-SD-NEXT:    fmov d1, x12
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], x11
+; CHECK-NOFP16-SD-NEXT:    fmov d3, x10
+; CHECK-NOFP16-SD-NEXT:    mov v2.d[1], x13
+; CHECK-NOFP16-SD-NEXT:    mov v1.d[1], x15
+; CHECK-NOFP16-SD-NEXT:    mov v3.d[1], x14
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v8f16_v8i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-FP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-FP16-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzs x9, h0
+; CHECK-FP16-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-FP16-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-FP16-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzs x8, h1
+; CHECK-FP16-SD-NEXT:    fcvtzs x12, h4
+; CHECK-FP16-SD-NEXT:    fcvtzs x11, h3
+; CHECK-FP16-SD-NEXT:    fcvtzs x15, h7
+; CHECK-FP16-SD-NEXT:    fmov d0, x9
+; CHECK-FP16-SD-NEXT:    fcvtzs x10, h2
+; CHECK-FP16-SD-NEXT:    fcvtzs x13, h5
+; CHECK-FP16-SD-NEXT:    fcvtzs x14, h6
+; CHECK-FP16-SD-NEXT:    fmov d2, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x12
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x11
+; CHECK-FP16-SD-NEXT:    fmov d3, x10
+; CHECK-FP16-SD-NEXT:    mov v2.d[1], x13
+; CHECK-FP16-SD-NEXT:    mov v1.d[1], x15
+; CHECK-FP16-SD-NEXT:    mov v3.d[1], x14
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f16_v8i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.2d, v1.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.2d, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v4.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v8f16_v8i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[4]
+; CHECK-FP16-GI-NEXT:    mov h5, v0.h[5]
+; CHECK-FP16-GI-NEXT:    mov h6, v0.h[6]
+; CHECK-FP16-GI-NEXT:    mov h7, v0.h[7]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d1, h1
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    fcvt d4, h4
+; CHECK-FP16-GI-NEXT:    fcvt d5, h5
+; CHECK-FP16-GI-NEXT:    fcvt d6, h6
+; CHECK-FP16-GI-NEXT:    fcvt d7, h7
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT:    mov v4.d[1], v5.d[0]
+; CHECK-FP16-GI-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x half> %a to <8 x i64>
   ret <8 x i64> %c
 }
 
 define <8 x i64> @fptou_v8f16_v8i64(<8 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v8f16_v8i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h7, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    mov h2, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvt s3, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h7
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s1
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s4
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s3
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s7
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x9
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s2
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s5
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s6
-; CHECK-SD-NOFP16-NEXT:    fmov d2, x8
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x12
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    fmov d3, x10
-; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x15
-; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v8f16_v8i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-FP16-NEXT:    mov h4, v0.h[2]
-; CHECK-SD-FP16-NEXT:    mov h3, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov h7, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h0
-; CHECK-SD-FP16-NEXT:    mov h2, v1.h[2]
-; CHECK-SD-FP16-NEXT:    mov h5, v1.h[1]
-; CHECK-SD-FP16-NEXT:    mov h6, v1.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x8, h1
-; CHECK-SD-FP16-NEXT:    fcvtzu x12, h4
-; CHECK-SD-FP16-NEXT:    fcvtzu x11, h3
-; CHECK-SD-FP16-NEXT:    fcvtzu x15, h7
-; CHECK-SD-FP16-NEXT:    fmov d0, x9
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h2
-; CHECK-SD-FP16-NEXT:    fcvtzu x13, h5
-; CHECK-SD-FP16-NEXT:    fcvtzu x14, h6
-; CHECK-SD-FP16-NEXT:    fmov d2, x8
-; CHECK-SD-FP16-NEXT:    fmov d1, x12
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x11
-; CHECK-SD-FP16-NEXT:    fmov d3, x10
-; CHECK-SD-FP16-NEXT:    mov v2.d[1], x13
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x15
-; CHECK-SD-FP16-NEXT:    mov v3.d[1], x14
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v8f16_v8i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.2d, v1.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.2d, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.2d, v3.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v8f16_v8i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    mov h4, v0.h[4]
-; CHECK-GI-FP16-NEXT:    mov h5, v0.h[5]
-; CHECK-GI-FP16-NEXT:    mov h6, v0.h[6]
-; CHECK-GI-FP16-NEXT:    mov h7, v0.h[7]
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d1, h1
-; CHECK-GI-FP16-NEXT:    fcvt d2, h2
-; CHECK-GI-FP16-NEXT:    fcvt d3, h3
-; CHECK-GI-FP16-NEXT:    fcvt d4, h4
-; CHECK-GI-FP16-NEXT:    fcvt d5, h5
-; CHECK-GI-FP16-NEXT:    fcvt d6, h6
-; CHECK-GI-FP16-NEXT:    fcvt d7, h7
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT:    mov v4.d[1], v5.d[0]
-; CHECK-GI-FP16-NEXT:    mov v6.d[1], v7.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v2.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v3.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f16_v8i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h4
+; CHECK-NOFP16-SD-NEXT:    fcvt s3, h3
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h7
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x9, s0
+; CHECK-NOFP16-SD-NEXT:    fcvt s2, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h6
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x8, s1
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x12, s4
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x11, s3
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x15, s7
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x9
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x10, s2
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x13, s5
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x14, s6
+; CHECK-NOFP16-SD-NEXT:    fmov d2, x8
+; CHECK-NOFP16-SD-NEXT:    fmov d1, x12
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], x11
+; CHECK-NOFP16-SD-NEXT:    fmov d3, x10
+; CHECK-NOFP16-SD-NEXT:    mov v2.d[1], x13
+; CHECK-NOFP16-SD-NEXT:    mov v1.d[1], x15
+; CHECK-NOFP16-SD-NEXT:    mov v3.d[1], x14
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v8f16_v8i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-FP16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-FP16-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzu x9, h0
+; CHECK-FP16-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-FP16-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-FP16-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzu x8, h1
+; CHECK-FP16-SD-NEXT:    fcvtzu x12, h4
+; CHECK-FP16-SD-NEXT:    fcvtzu x11, h3
+; CHECK-FP16-SD-NEXT:    fcvtzu x15, h7
+; CHECK-FP16-SD-NEXT:    fmov d0, x9
+; CHECK-FP16-SD-NEXT:    fcvtzu x10, h2
+; CHECK-FP16-SD-NEXT:    fcvtzu x13, h5
+; CHECK-FP16-SD-NEXT:    fcvtzu x14, h6
+; CHECK-FP16-SD-NEXT:    fmov d2, x8
+; CHECK-FP16-SD-NEXT:    fmov d1, x12
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x11
+; CHECK-FP16-SD-NEXT:    fmov d3, x10
+; CHECK-FP16-SD-NEXT:    mov v2.d[1], x13
+; CHECK-FP16-SD-NEXT:    mov v1.d[1], x15
+; CHECK-FP16-SD-NEXT:    mov v3.d[1], x14
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f16_v8i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.2d, v1.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.2d, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v4.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v8f16_v8i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[4]
+; CHECK-FP16-GI-NEXT:    mov h5, v0.h[5]
+; CHECK-FP16-GI-NEXT:    mov h6, v0.h[6]
+; CHECK-FP16-GI-NEXT:    mov h7, v0.h[7]
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d1, h1
+; CHECK-FP16-GI-NEXT:    fcvt d2, h2
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    fcvt d4, h4
+; CHECK-FP16-GI-NEXT:    fcvt d5, h5
+; CHECK-FP16-GI-NEXT:    fcvt d6, h6
+; CHECK-FP16-GI-NEXT:    fcvt d7, h7
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT:    mov v4.d[1], v5.d[0]
+; CHECK-FP16-GI-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x half> %a to <8 x i64>
   ret <8 x i64> %c
 }
 
 define <16 x i64> @fptos_v16f16_v16i64(<16 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v16f16_v16i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NOFP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x8, s5
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s7
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s6
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmov d2, x9
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h3
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s1
-; CHECK-SD-NOFP16-NEXT:    fmov d6, x10
-; CHECK-SD-NOFP16-NEXT:    fmov d3, x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s0
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x16, s5
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s7
-; CHECK-SD-NOFP16-NEXT:    fmov d7, x14
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s16
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x0, s4
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x18, s19
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x15
-; CHECK-SD-NOFP16-NEXT:    fmov d4, x9
-; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x12
-; CHECK-SD-NOFP16-NEXT:    fmov d5, x10
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    mov v4.d[1], x16
-; CHECK-SD-NOFP16-NEXT:    mov v6.d[1], x17
-; CHECK-SD-NOFP16-NEXT:    mov v7.d[1], x18
-; CHECK-SD-NOFP16-NEXT:    mov v5.d[1], x0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v16f16_v16i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-SD-FP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-SD-FP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzs x8, h0
-; CHECK-SD-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h1
-; CHECK-SD-FP16-NEXT:    mov h7, v1.h[1]
-; CHECK-SD-FP16-NEXT:    mov h6, v2.h[2]
-; CHECK-SD-FP16-NEXT:    mov h16, v3.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h4
-; CHECK-SD-FP16-NEXT:    mov h4, v1.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h2
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h5
-; CHECK-SD-FP16-NEXT:    mov h5, v2.h[1]
-; CHECK-SD-FP16-NEXT:    mov h17, v2.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x13, h3
-; CHECK-SD-FP16-NEXT:    mov h18, v3.h[1]
-; CHECK-SD-FP16-NEXT:    mov h1, v1.h[3]
-; CHECK-SD-FP16-NEXT:    mov h19, v3.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x14, h6
-; CHECK-SD-FP16-NEXT:    fcvtzs x15, h16
-; CHECK-SD-FP16-NEXT:    fcvtzs x16, h0
-; CHECK-SD-FP16-NEXT:    fcvtzs x0, h4
-; CHECK-SD-FP16-NEXT:    fcvtzs x17, h7
-; CHECK-SD-FP16-NEXT:    fmov d2, x11
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h5
-; CHECK-SD-FP16-NEXT:    fcvtzs x18, h17
-; CHECK-SD-FP16-NEXT:    fmov d6, x13
-; CHECK-SD-FP16-NEXT:    fcvtzs x13, h18
-; CHECK-SD-FP16-NEXT:    fmov d0, x8
-; CHECK-SD-FP16-NEXT:    fmov d4, x9
-; CHECK-SD-FP16-NEXT:    fmov d3, x14
-; CHECK-SD-FP16-NEXT:    fmov d7, x15
-; CHECK-SD-FP16-NEXT:    fcvtzs x14, h19
-; CHECK-SD-FP16-NEXT:    fcvtzs x15, h1
-; CHECK-SD-FP16-NEXT:    fmov d1, x12
-; CHECK-SD-FP16-NEXT:    fmov d5, x0
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x10
-; CHECK-SD-FP16-NEXT:    mov v4.d[1], x17
-; CHECK-SD-FP16-NEXT:    mov v2.d[1], x11
-; CHECK-SD-FP16-NEXT:    mov v3.d[1], x18
-; CHECK-SD-FP16-NEXT:    mov v6.d[1], x13
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x16
-; CHECK-SD-FP16-NEXT:    mov v7.d[1], x14
-; CHECK-SD-FP16-NEXT:    mov v5.d[1], x15
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v16f16_v16i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.2d, v2.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.2d, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v5.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v6.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v7.2d, v3.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v16.2d, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v17.2d, v1.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v18.2d, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v4.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v5.2d, v16.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v6.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v7.2d, v18.2d
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v16f16_v16i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov h5, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvt d2, h0
-; CHECK-GI-FP16-NEXT:    mov h6, v0.h[4]
-; CHECK-GI-FP16-NEXT:    mov h7, v0.h[5]
-; CHECK-GI-FP16-NEXT:    mov h16, v0.h[6]
-; CHECK-GI-FP16-NEXT:    mov h0, v0.h[7]
-; CHECK-GI-FP16-NEXT:    mov h17, v1.h[1]
-; CHECK-GI-FP16-NEXT:    mov h18, v1.h[2]
-; CHECK-GI-FP16-NEXT:    mov h19, v1.h[3]
-; CHECK-GI-FP16-NEXT:    mov h20, v1.h[4]
-; CHECK-GI-FP16-NEXT:    mov h21, v1.h[5]
-; CHECK-GI-FP16-NEXT:    mov h22, v1.h[6]
-; CHECK-GI-FP16-NEXT:    mov h23, v1.h[7]
-; CHECK-GI-FP16-NEXT:    fcvt d3, h3
-; CHECK-GI-FP16-NEXT:    fcvt d4, h4
-; CHECK-GI-FP16-NEXT:    fcvt d5, h5
-; CHECK-GI-FP16-NEXT:    fcvt d6, h6
-; CHECK-GI-FP16-NEXT:    fcvt d7, h7
-; CHECK-GI-FP16-NEXT:    fcvt d16, h16
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d24, h1
-; CHECK-GI-FP16-NEXT:    fcvt d1, h17
-; CHECK-GI-FP16-NEXT:    fcvt d17, h18
-; CHECK-GI-FP16-NEXT:    fcvt d18, h19
-; CHECK-GI-FP16-NEXT:    fcvt d19, h20
-; CHECK-GI-FP16-NEXT:    fcvt d20, h21
-; CHECK-GI-FP16-NEXT:    fcvt d21, h22
-; CHECK-GI-FP16-NEXT:    fcvt d22, h23
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT:    mov v4.d[1], v5.d[0]
-; CHECK-GI-FP16-NEXT:    mov v6.d[1], v7.d[0]
-; CHECK-GI-FP16-NEXT:    mov v16.d[1], v0.d[0]
-; CHECK-GI-FP16-NEXT:    mov v24.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    mov v17.d[1], v18.d[0]
-; CHECK-GI-FP16-NEXT:    mov v19.d[1], v20.d[0]
-; CHECK-GI-FP16-NEXT:    mov v21.d[1], v22.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v2.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v3.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v4.2d, v24.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v5.2d, v17.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v6.2d, v19.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v7.2d, v21.2d
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v16f16_v16i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov h4, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h0
+; CHECK-NOFP16-SD-NEXT:    mov h18, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h2
+; CHECK-NOFP16-SD-NEXT:    mov h7, v2.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h16, v2.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v3.h[2]
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h3
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h4
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x8, s5
+; CHECK-NOFP16-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h2, v2.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h18
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x9, s6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    mov h16, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h17
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x10, s19
+; CHECK-NOFP16-SD-NEXT:    mov h19, v3.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x11, s4
+; CHECK-NOFP16-SD-NEXT:    mov h4, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v3.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x13, s7
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x12, s6
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x15, s18
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x14, s17
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h4
+; CHECK-NOFP16-SD-NEXT:    fmov d2, x9
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h3
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x9, s1
+; CHECK-NOFP16-SD-NEXT:    fmov d6, x10
+; CHECK-NOFP16-SD-NEXT:    fmov d3, x13
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x13, s0
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x16, s5
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x10, s7
+; CHECK-NOFP16-SD-NEXT:    fmov d7, x14
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x14, s16
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x17, s17
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x0, s4
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x8
+; CHECK-NOFP16-SD-NEXT:    fcvtzs x18, s19
+; CHECK-NOFP16-SD-NEXT:    fmov d1, x15
+; CHECK-NOFP16-SD-NEXT:    fmov d4, x9
+; CHECK-NOFP16-SD-NEXT:    mov v2.d[1], x12
+; CHECK-NOFP16-SD-NEXT:    fmov d5, x10
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], x11
+; CHECK-NOFP16-SD-NEXT:    mov v3.d[1], x14
+; CHECK-NOFP16-SD-NEXT:    mov v1.d[1], x13
+; CHECK-NOFP16-SD-NEXT:    mov v4.d[1], x16
+; CHECK-NOFP16-SD-NEXT:    mov v6.d[1], x17
+; CHECK-NOFP16-SD-NEXT:    mov v7.d[1], x18
+; CHECK-NOFP16-SD-NEXT:    mov v5.d[1], x0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v16f16_v16i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-FP16-SD-NEXT:    mov h4, v0.h[1]
+; CHECK-FP16-SD-NEXT:    mov h5, v0.h[2]
+; CHECK-FP16-SD-NEXT:    fcvtzs x8, h0
+; CHECK-FP16-SD-NEXT:    mov h0, v0.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzs x9, h1
+; CHECK-FP16-SD-NEXT:    mov h7, v1.h[1]
+; CHECK-FP16-SD-NEXT:    mov h6, v2.h[2]
+; CHECK-FP16-SD-NEXT:    mov h16, v3.h[2]
+; CHECK-FP16-SD-NEXT:    fcvtzs x10, h4
+; CHECK-FP16-SD-NEXT:    mov h4, v1.h[2]
+; CHECK-FP16-SD-NEXT:    fcvtzs x11, h2
+; CHECK-FP16-SD-NEXT:    fcvtzs x12, h5
+; CHECK-FP16-SD-NEXT:    mov h5, v2.h[1]
+; CHECK-FP16-SD-NEXT:    mov h17, v2.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzs x13, h3
+; CHECK-FP16-SD-NEXT:    mov h18, v3.h[1]
+; CHECK-FP16-SD-NEXT:    mov h1, v1.h[3]
+; CHECK-FP16-SD-NEXT:    mov h19, v3.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzs x14, h6
+; CHECK-FP16-SD-NEXT:    fcvtzs x15, h16
+; CHECK-FP16-SD-NEXT:    fcvtzs x16, h0
+; CHECK-FP16-SD-NEXT:    fcvtzs x0, h4
+; CHECK-FP16-SD-NEXT:    fcvtzs x17, h7
+; CHECK-FP16-SD-NEXT:    fmov d2, x11
+; CHECK-FP16-SD-NEXT:    fcvtzs x11, h5
+; CHECK-FP16-SD-NEXT:    fcvtzs x18, h17
+; CHECK-FP16-SD-NEXT:    fmov d6, x13
+; CHECK-FP16-SD-NEXT:    fcvtzs x13, h18
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d4, x9
+; CHECK-FP16-SD-NEXT:    fmov d3, x14
+; CHECK-FP16-SD-NEXT:    fmov d7, x15
+; CHECK-FP16-SD-NEXT:    fcvtzs x14, h19
+; CHECK-FP16-SD-NEXT:    fcvtzs x15, h1
+; CHECK-FP16-SD-NEXT:    fmov d1, x12
+; CHECK-FP16-SD-NEXT:    fmov d5, x0
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x10
+; CHECK-FP16-SD-NEXT:    mov v4.d[1], x17
+; CHECK-FP16-SD-NEXT:    mov v2.d[1], x11
+; CHECK-FP16-SD-NEXT:    mov v3.d[1], x18
+; CHECK-FP16-SD-NEXT:    mov v6.d[1], x13
+; CHECK-FP16-SD-NEXT:    mov v1.d[1], x16
+; CHECK-FP16-SD-NEXT:    mov v7.d[1], x14
+; CHECK-FP16-SD-NEXT:    mov v5.d[1], x15
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v16f16_v16i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.2d, v2.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.2d, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v5.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v6.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v7.2d, v3.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v16.2d, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v17.2d, v1.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v18.2d, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v4.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v5.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v6.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v7.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v16f16_v16i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h5, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d2, h0
+; CHECK-FP16-GI-NEXT:    mov h6, v0.h[4]
+; CHECK-FP16-GI-NEXT:    mov h7, v0.h[5]
+; CHECK-FP16-GI-NEXT:    mov h16, v0.h[6]
+; CHECK-FP16-GI-NEXT:    mov h0, v0.h[7]
+; CHECK-FP16-GI-NEXT:    mov h17, v1.h[1]
+; CHECK-FP16-GI-NEXT:    mov h18, v1.h[2]
+; CHECK-FP16-GI-NEXT:    mov h19, v1.h[3]
+; CHECK-FP16-GI-NEXT:    mov h20, v1.h[4]
+; CHECK-FP16-GI-NEXT:    mov h21, v1.h[5]
+; CHECK-FP16-GI-NEXT:    mov h22, v1.h[6]
+; CHECK-FP16-GI-NEXT:    mov h23, v1.h[7]
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    fcvt d4, h4
+; CHECK-FP16-GI-NEXT:    fcvt d5, h5
+; CHECK-FP16-GI-NEXT:    fcvt d6, h6
+; CHECK-FP16-GI-NEXT:    fcvt d7, h7
+; CHECK-FP16-GI-NEXT:    fcvt d16, h16
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d24, h1
+; CHECK-FP16-GI-NEXT:    fcvt d1, h17
+; CHECK-FP16-GI-NEXT:    fcvt d17, h18
+; CHECK-FP16-GI-NEXT:    fcvt d18, h19
+; CHECK-FP16-GI-NEXT:    fcvt d19, h20
+; CHECK-FP16-GI-NEXT:    fcvt d20, h21
+; CHECK-FP16-GI-NEXT:    fcvt d21, h22
+; CHECK-FP16-GI-NEXT:    fcvt d22, h23
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT:    mov v4.d[1], v5.d[0]
+; CHECK-FP16-GI-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-FP16-GI-NEXT:    mov v16.d[1], v0.d[0]
+; CHECK-FP16-GI-NEXT:    mov v24.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov v17.d[1], v18.d[0]
+; CHECK-FP16-GI-NEXT:    mov v19.d[1], v20.d[0]
+; CHECK-FP16-GI-NEXT:    mov v21.d[1], v22.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.2d, v16.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v4.2d, v24.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v5.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v6.2d, v19.2d
+; CHECK-FP16-GI-NEXT:    fcvtzs v7.2d, v21.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <16 x half> %a to <16 x i64>
   ret <16 x i64> %c
 }
 
 define <16 x i64> @fptou_v16f16_v16i64(<16 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v16f16_v16i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NOFP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NOFP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h0
-; CHECK-SD-NOFP16-NEXT:    mov h18, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h2
-; CHECK-SD-NOFP16-NEXT:    mov h7, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x8, s5
-; CHECK-SD-NOFP16-NEXT:    mov h5, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h2, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-SD-NOFP16-NEXT:    mov h16, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s19
-; CHECK-SD-NOFP16-NEXT:    mov h19, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s4
-; CHECK-SD-NOFP16-NEXT:    mov h4, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h3, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s7
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s6
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s18
-; CHECK-SD-NOFP16-NEXT:    fcvt s7, h16
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s17
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h2
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h19
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fmov d2, x9
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h3
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s1
-; CHECK-SD-NOFP16-NEXT:    fmov d6, x10
-; CHECK-SD-NOFP16-NEXT:    fmov d3, x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s0
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x16, s5
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s7
-; CHECK-SD-NOFP16-NEXT:    fmov d7, x14
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s16
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x17, s17
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x0, s4
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x8
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x18, s19
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x15
-; CHECK-SD-NOFP16-NEXT:    fmov d4, x9
-; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x12
-; CHECK-SD-NOFP16-NEXT:    fmov d5, x10
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    mov v4.d[1], x16
-; CHECK-SD-NOFP16-NEXT:    mov v6.d[1], x17
-; CHECK-SD-NOFP16-NEXT:    mov v7.d[1], x18
-; CHECK-SD-NOFP16-NEXT:    mov v5.d[1], x0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v16f16_v16i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-SD-FP16-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-SD-FP16-NEXT:    mov h4, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov h5, v0.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzu x8, h0
-; CHECK-SD-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h1
-; CHECK-SD-FP16-NEXT:    mov h7, v1.h[1]
-; CHECK-SD-FP16-NEXT:    mov h6, v2.h[2]
-; CHECK-SD-FP16-NEXT:    mov h16, v3.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h4
-; CHECK-SD-FP16-NEXT:    mov h4, v1.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzu x11, h2
-; CHECK-SD-FP16-NEXT:    fcvtzu x12, h5
-; CHECK-SD-FP16-NEXT:    mov h5, v2.h[1]
-; CHECK-SD-FP16-NEXT:    mov h17, v2.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x13, h3
-; CHECK-SD-FP16-NEXT:    mov h18, v3.h[1]
-; CHECK-SD-FP16-NEXT:    mov h1, v1.h[3]
-; CHECK-SD-FP16-NEXT:    mov h19, v3.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x14, h6
-; CHECK-SD-FP16-NEXT:    fcvtzu x15, h16
-; CHECK-SD-FP16-NEXT:    fcvtzu x16, h0
-; CHECK-SD-FP16-NEXT:    fcvtzu x0, h4
-; CHECK-SD-FP16-NEXT:    fcvtzu x17, h7
-; CHECK-SD-FP16-NEXT:    fmov d2, x11
-; CHECK-SD-FP16-NEXT:    fcvtzu x11, h5
-; CHECK-SD-FP16-NEXT:    fcvtzu x18, h17
-; CHECK-SD-FP16-NEXT:    fmov d6, x13
-; CHECK-SD-FP16-NEXT:    fcvtzu x13, h18
-; CHECK-SD-FP16-NEXT:    fmov d0, x8
-; CHECK-SD-FP16-NEXT:    fmov d4, x9
-; CHECK-SD-FP16-NEXT:    fmov d3, x14
-; CHECK-SD-FP16-NEXT:    fmov d7, x15
-; CHECK-SD-FP16-NEXT:    fcvtzu x14, h19
-; CHECK-SD-FP16-NEXT:    fcvtzu x15, h1
-; CHECK-SD-FP16-NEXT:    fmov d1, x12
-; CHECK-SD-FP16-NEXT:    fmov d5, x0
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x10
-; CHECK-SD-FP16-NEXT:    mov v4.d[1], x17
-; CHECK-SD-FP16-NEXT:    mov v2.d[1], x11
-; CHECK-SD-FP16-NEXT:    mov v3.d[1], x18
-; CHECK-SD-FP16-NEXT:    mov v6.d[1], x13
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x16
-; CHECK-SD-FP16-NEXT:    mov v7.d[1], x14
-; CHECK-SD-FP16-NEXT:    mov v5.d[1], x15
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v16f16_v16i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.2d, v2.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.2d, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v5.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v6.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v7.2d, v3.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v16.2d, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v17.2d, v1.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v18.2d, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v4.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v5.2d, v16.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v6.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v7.2d, v18.2d
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v16f16_v16i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov h5, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvt d2, h0
-; CHECK-GI-FP16-NEXT:    mov h6, v0.h[4]
-; CHECK-GI-FP16-NEXT:    mov h7, v0.h[5]
-; CHECK-GI-FP16-NEXT:    mov h16, v0.h[6]
-; CHECK-GI-FP16-NEXT:    mov h0, v0.h[7]
-; CHECK-GI-FP16-NEXT:    mov h17, v1.h[1]
-; CHECK-GI-FP16-NEXT:    mov h18, v1.h[2]
-; CHECK-GI-FP16-NEXT:    mov h19, v1.h[3]
-; CHECK-GI-FP16-NEXT:    mov h20, v1.h[4]
-; CHECK-GI-FP16-NEXT:    mov h21, v1.h[5]
-; CHECK-GI-FP16-NEXT:    mov h22, v1.h[6]
-; CHECK-GI-FP16-NEXT:    mov h23, v1.h[7]
-; CHECK-GI-FP16-NEXT:    fcvt d3, h3
-; CHECK-GI-FP16-NEXT:    fcvt d4, h4
-; CHECK-GI-FP16-NEXT:    fcvt d5, h5
-; CHECK-GI-FP16-NEXT:    fcvt d6, h6
-; CHECK-GI-FP16-NEXT:    fcvt d7, h7
-; CHECK-GI-FP16-NEXT:    fcvt d16, h16
-; CHECK-GI-FP16-NEXT:    fcvt d0, h0
-; CHECK-GI-FP16-NEXT:    fcvt d24, h1
-; CHECK-GI-FP16-NEXT:    fcvt d1, h17
-; CHECK-GI-FP16-NEXT:    fcvt d17, h18
-; CHECK-GI-FP16-NEXT:    fcvt d18, h19
-; CHECK-GI-FP16-NEXT:    fcvt d19, h20
-; CHECK-GI-FP16-NEXT:    fcvt d20, h21
-; CHECK-GI-FP16-NEXT:    fcvt d21, h22
-; CHECK-GI-FP16-NEXT:    fcvt d22, h23
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v3.d[0]
-; CHECK-GI-FP16-NEXT:    mov v4.d[1], v5.d[0]
-; CHECK-GI-FP16-NEXT:    mov v6.d[1], v7.d[0]
-; CHECK-GI-FP16-NEXT:    mov v16.d[1], v0.d[0]
-; CHECK-GI-FP16-NEXT:    mov v24.d[1], v1.d[0]
-; CHECK-GI-FP16-NEXT:    mov v17.d[1], v18.d[0]
-; CHECK-GI-FP16-NEXT:    mov v19.d[1], v20.d[0]
-; CHECK-GI-FP16-NEXT:    mov v21.d[1], v22.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v2.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v3.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v4.2d, v24.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v5.2d, v17.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v6.2d, v19.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v7.2d, v21.2d
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v16f16_v16i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov h4, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h0
+; CHECK-NOFP16-SD-NEXT:    mov h18, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h2
+; CHECK-NOFP16-SD-NEXT:    mov h7, v2.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h16, v2.h[2]
+; CHECK-NOFP16-SD-NEXT:    mov h17, v3.h[2]
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h3
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h4
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x8, s5
+; CHECK-NOFP16-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-NOFP16-SD-NEXT:    mov h2, v2.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s18, h18
+; CHECK-NOFP16-SD-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x9, s6
+; CHECK-NOFP16-SD-NEXT:    fcvt s6, h7
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    mov h16, v1.h[2]
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h17
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x10, s19
+; CHECK-NOFP16-SD-NEXT:    mov h19, v3.h[1]
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x11, s4
+; CHECK-NOFP16-SD-NEXT:    mov h4, v1.h[3]
+; CHECK-NOFP16-SD-NEXT:    mov h3, v3.h[3]
+; CHECK-NOFP16-SD-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-SD-NEXT:    fcvt s5, h5
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x13, s7
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x12, s6
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x15, s18
+; CHECK-NOFP16-SD-NEXT:    fcvt s7, h16
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x14, s17
+; CHECK-NOFP16-SD-NEXT:    fcvt s16, h2
+; CHECK-NOFP16-SD-NEXT:    fcvt s17, h19
+; CHECK-NOFP16-SD-NEXT:    fcvt s4, h4
+; CHECK-NOFP16-SD-NEXT:    fmov d2, x9
+; CHECK-NOFP16-SD-NEXT:    fcvt s19, h3
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x9, s1
+; CHECK-NOFP16-SD-NEXT:    fmov d6, x10
+; CHECK-NOFP16-SD-NEXT:    fmov d3, x13
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x13, s0
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x16, s5
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x10, s7
+; CHECK-NOFP16-SD-NEXT:    fmov d7, x14
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x14, s16
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x17, s17
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x0, s4
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x8
+; CHECK-NOFP16-SD-NEXT:    fcvtzu x18, s19
+; CHECK-NOFP16-SD-NEXT:    fmov d1, x15
+; CHECK-NOFP16-SD-NEXT:    fmov d4, x9
+; CHECK-NOFP16-SD-NEXT:    mov v2.d[1], x12
+; CHECK-NOFP16-SD-NEXT:    fmov d5, x10
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], x11
+; CHECK-NOFP16-SD-NEXT:    mov v3.d[1], x14
+; CHECK-NOFP16-SD-NEXT:    mov v1.d[1], x13
+; CHECK-NOFP16-SD-NEXT:    mov v4.d[1], x16
+; CHECK-NOFP16-SD-NEXT:    mov v6.d[1], x17
+; CHECK-NOFP16-SD-NEXT:    mov v7.d[1], x18
+; CHECK-NOFP16-SD-NEXT:    mov v5.d[1], x0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v16f16_v16i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-FP16-SD-NEXT:    mov h4, v0.h[1]
+; CHECK-FP16-SD-NEXT:    mov h5, v0.h[2]
+; CHECK-FP16-SD-NEXT:    fcvtzu x8, h0
+; CHECK-FP16-SD-NEXT:    mov h0, v0.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzu x9, h1
+; CHECK-FP16-SD-NEXT:    mov h7, v1.h[1]
+; CHECK-FP16-SD-NEXT:    mov h6, v2.h[2]
+; CHECK-FP16-SD-NEXT:    mov h16, v3.h[2]
+; CHECK-FP16-SD-NEXT:    fcvtzu x10, h4
+; CHECK-FP16-SD-NEXT:    mov h4, v1.h[2]
+; CHECK-FP16-SD-NEXT:    fcvtzu x11, h2
+; CHECK-FP16-SD-NEXT:    fcvtzu x12, h5
+; CHECK-FP16-SD-NEXT:    mov h5, v2.h[1]
+; CHECK-FP16-SD-NEXT:    mov h17, v2.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzu x13, h3
+; CHECK-FP16-SD-NEXT:    mov h18, v3.h[1]
+; CHECK-FP16-SD-NEXT:    mov h1, v1.h[3]
+; CHECK-FP16-SD-NEXT:    mov h19, v3.h[3]
+; CHECK-FP16-SD-NEXT:    fcvtzu x14, h6
+; CHECK-FP16-SD-NEXT:    fcvtzu x15, h16
+; CHECK-FP16-SD-NEXT:    fcvtzu x16, h0
+; CHECK-FP16-SD-NEXT:    fcvtzu x0, h4
+; CHECK-FP16-SD-NEXT:    fcvtzu x17, h7
+; CHECK-FP16-SD-NEXT:    fmov d2, x11
+; CHECK-FP16-SD-NEXT:    fcvtzu x11, h5
+; CHECK-FP16-SD-NEXT:    fcvtzu x18, h17
+; CHECK-FP16-SD-NEXT:    fmov d6, x13
+; CHECK-FP16-SD-NEXT:    fcvtzu x13, h18
+; CHECK-FP16-SD-NEXT:    fmov d0, x8
+; CHECK-FP16-SD-NEXT:    fmov d4, x9
+; CHECK-FP16-SD-NEXT:    fmov d3, x14
+; CHECK-FP16-SD-NEXT:    fmov d7, x15
+; CHECK-FP16-SD-NEXT:    fcvtzu x14, h19
+; CHECK-FP16-SD-NEXT:    fcvtzu x15, h1
+; CHECK-FP16-SD-NEXT:    fmov d1, x12
+; CHECK-FP16-SD-NEXT:    fmov d5, x0
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], x10
+; CHECK-FP16-SD-NEXT:    mov v4.d[1], x17
+; CHECK-FP16-SD-NEXT:    mov v2.d[1], x11
+; CHECK-FP16-SD-NEXT:    mov v3.d[1], x18
+; CHECK-FP16-SD-NEXT:    mov v6.d[1], x13
+; CHECK-FP16-SD-NEXT:    mov v1.d[1], x16
+; CHECK-FP16-SD-NEXT:    mov v7.d[1], x14
+; CHECK-FP16-SD-NEXT:    mov v5.d[1], x15
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v16f16_v16i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.2d, v2.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.2d, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v5.2d, v0.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v6.2d, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v7.2d, v3.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v16.2d, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtl v17.2d, v1.2s
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v18.2d, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v4.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v5.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v6.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v7.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v16f16_v16i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov h3, v0.h[1]
+; CHECK-FP16-GI-NEXT:    mov h4, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov h5, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fcvt d2, h0
+; CHECK-FP16-GI-NEXT:    mov h6, v0.h[4]
+; CHECK-FP16-GI-NEXT:    mov h7, v0.h[5]
+; CHECK-FP16-GI-NEXT:    mov h16, v0.h[6]
+; CHECK-FP16-GI-NEXT:    mov h0, v0.h[7]
+; CHECK-FP16-GI-NEXT:    mov h17, v1.h[1]
+; CHECK-FP16-GI-NEXT:    mov h18, v1.h[2]
+; CHECK-FP16-GI-NEXT:    mov h19, v1.h[3]
+; CHECK-FP16-GI-NEXT:    mov h20, v1.h[4]
+; CHECK-FP16-GI-NEXT:    mov h21, v1.h[5]
+; CHECK-FP16-GI-NEXT:    mov h22, v1.h[6]
+; CHECK-FP16-GI-NEXT:    mov h23, v1.h[7]
+; CHECK-FP16-GI-NEXT:    fcvt d3, h3
+; CHECK-FP16-GI-NEXT:    fcvt d4, h4
+; CHECK-FP16-GI-NEXT:    fcvt d5, h5
+; CHECK-FP16-GI-NEXT:    fcvt d6, h6
+; CHECK-FP16-GI-NEXT:    fcvt d7, h7
+; CHECK-FP16-GI-NEXT:    fcvt d16, h16
+; CHECK-FP16-GI-NEXT:    fcvt d0, h0
+; CHECK-FP16-GI-NEXT:    fcvt d24, h1
+; CHECK-FP16-GI-NEXT:    fcvt d1, h17
+; CHECK-FP16-GI-NEXT:    fcvt d17, h18
+; CHECK-FP16-GI-NEXT:    fcvt d18, h19
+; CHECK-FP16-GI-NEXT:    fcvt d19, h20
+; CHECK-FP16-GI-NEXT:    fcvt d20, h21
+; CHECK-FP16-GI-NEXT:    fcvt d21, h22
+; CHECK-FP16-GI-NEXT:    fcvt d22, h23
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], v3.d[0]
+; CHECK-FP16-GI-NEXT:    mov v4.d[1], v5.d[0]
+; CHECK-FP16-GI-NEXT:    mov v6.d[1], v7.d[0]
+; CHECK-FP16-GI-NEXT:    mov v16.d[1], v0.d[0]
+; CHECK-FP16-GI-NEXT:    mov v24.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    mov v17.d[1], v18.d[0]
+; CHECK-FP16-GI-NEXT:    mov v19.d[1], v20.d[0]
+; CHECK-FP16-GI-NEXT:    mov v21.d[1], v22.d[0]
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.2d, v16.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v4.2d, v24.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v5.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v6.2d, v19.2d
+; CHECK-FP16-GI-NEXT:    fcvtzu v7.2d, v21.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <16 x half> %a to <16 x i64>
   ret <16 x i64> %c
 }
 
-define <32 x i64> @fptos_v32f16_v32i64(<32 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v32f16_v32i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NOFP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
-; CHECK-SD-NOFP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
-; CHECK-SD-NOFP16-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h2
-; CHECK-SD-NOFP16-NEXT:    mov h26, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h0
-; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v4.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v5.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h6
-; CHECK-SD-NOFP16-NEXT:    mov h25, v6.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s21
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h7
-; CHECK-SD-NOFP16-NEXT:    mov h21, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s23
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    mov h26, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h24, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fmov d19, x9
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x9, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s16
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s17
-; CHECK-SD-NOFP16-NEXT:    fmov d2, x12
-; CHECK-SD-NOFP16-NEXT:    fmov d16, x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s23
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s25
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fmov d26, x11
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s21
-; CHECK-SD-NOFP16-NEXT:    fmov d3, x14
-; CHECK-SD-NOFP16-NEXT:    fmov d17, x15
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s22
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s27
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h23
-; CHECK-SD-NOFP16-NEXT:    fmov d23, x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s25
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fmov d25, x14
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s24
-; CHECK-SD-NOFP16-NEXT:    fmov d24, x15
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v7.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov v25.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s21
-; CHECK-SD-NOFP16-NEXT:    mov h21, v7.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov v24.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s20
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov v23.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s1
-; CHECK-SD-NOFP16-NEXT:    mov h1, v6.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v6.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov v19.d[1], x15
-; CHECK-SD-NOFP16-NEXT:    mov h7, v7.h[3]
-; CHECK-SD-NOFP16-NEXT:    stp q25, q24, [x8, #192]
-; CHECK-SD-NOFP16-NEXT:    fmov d24, x13
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    mov v26.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v5.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v24.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    mov h25, v4.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    stp q26, q23, [x8, #128]
-; CHECK-SD-NOFP16-NEXT:    fmov d23, x12
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s20
-; CHECK-SD-NOFP16-NEXT:    mov h20, v4.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s0
-; CHECK-SD-NOFP16-NEXT:    stp q19, q24, [x8, #64]
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x10
-; CHECK-SD-NOFP16-NEXT:    fmov d19, x11
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s5
-; CHECK-SD-NOFP16-NEXT:    mov v19.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h18
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s22
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x12
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x12, s4
-; CHECK-SD-NOFP16-NEXT:    mov v23.d[1], x10
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x10, s1
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x15, s24
-; CHECK-SD-NOFP16-NEXT:    mov v16.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x11, s20
-; CHECK-SD-NOFP16-NEXT:    mov v17.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x14, s6
-; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzs x13, s5
-; CHECK-SD-NOFP16-NEXT:    fmov d4, x9
-; CHECK-SD-NOFP16-NEXT:    stp q0, q19, [x8]
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x12
-; CHECK-SD-NOFP16-NEXT:    stp q16, q23, [x8, #224]
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x10
-; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x15
-; CHECK-SD-NOFP16-NEXT:    stp q2, q17, [x8, #160]
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    mov v4.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    stp q0, q3, [x8, #96]
-; CHECK-SD-NOFP16-NEXT:    stp q4, q1, [x8, #32]
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v32f16_v32i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-SD-FP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
-; CHECK-SD-FP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
-; CHECK-SD-FP16-NEXT:    mov h16, v3.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h0
-; CHECK-SD-FP16-NEXT:    mov h23, v3.h[3]
-; CHECK-SD-FP16-NEXT:    mov h25, v3.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzs x15, h3
-; CHECK-SD-FP16-NEXT:    mov h24, v2.h[2]
-; CHECK-SD-FP16-NEXT:    mov h19, v1.h[2]
-; CHECK-SD-FP16-NEXT:    mov h21, v2.h[1]
-; CHECK-SD-FP16-NEXT:    mov h26, v2.h[3]
-; CHECK-SD-FP16-NEXT:    mov h17, v4.h[2]
-; CHECK-SD-FP16-NEXT:    mov h18, v5.h[2]
-; CHECK-SD-FP16-NEXT:    mov h22, v6.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h5
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h16
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h6
-; CHECK-SD-FP16-NEXT:    mov h7, v1.h[1]
-; CHECK-SD-FP16-NEXT:    mov h20, v1.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x13, h17
-; CHECK-SD-FP16-NEXT:    fcvtzs x14, h18
-; CHECK-SD-FP16-NEXT:    fmov d18, x9
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h22
-; CHECK-SD-FP16-NEXT:    fmov d3, x10
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h23
-; CHECK-SD-FP16-NEXT:    fmov d22, x12
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h25
-; CHECK-SD-FP16-NEXT:    fmov d23, x15
-; CHECK-SD-FP16-NEXT:    fmov d16, x11
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h2
-; CHECK-SD-FP16-NEXT:    fcvtzs x15, h21
-; CHECK-SD-FP16-NEXT:    fmov d2, x13
-; CHECK-SD-FP16-NEXT:    fcvtzs x13, h24
-; CHECK-SD-FP16-NEXT:    fmov d17, x14
-; CHECK-SD-FP16-NEXT:    fcvtzs x14, h19
-; CHECK-SD-FP16-NEXT:    mov v22.d[1], x10
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h1
-; CHECK-SD-FP16-NEXT:    mov v23.d[1], x12
-; CHECK-SD-FP16-NEXT:    fmov d19, x9
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h26
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h20
-; CHECK-SD-FP16-NEXT:    mov h20, v0.h[2]
-; CHECK-SD-FP16-NEXT:    fmov d21, x11
-; CHECK-SD-FP16-NEXT:    fmov d1, x13
-; CHECK-SD-FP16-NEXT:    fcvtzs x13, h7
-; CHECK-SD-FP16-NEXT:    mov h24, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fmov d7, x14
-; CHECK-SD-FP16-NEXT:    stp q23, q22, [x8, #192]
-; CHECK-SD-FP16-NEXT:    fmov d22, x10
-; CHECK-SD-FP16-NEXT:    mov v21.d[1], x15
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x9
-; CHECK-SD-FP16-NEXT:    mov h23, v0.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h20
-; CHECK-SD-FP16-NEXT:    mov v7.d[1], x12
-; CHECK-SD-FP16-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-SD-FP16-NEXT:    mov h20, v6.h[3]
-; CHECK-SD-FP16-NEXT:    mov v22.d[1], x13
-; CHECK-SD-FP16-NEXT:    mov h6, v6.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h24
-; CHECK-SD-FP16-NEXT:    stp q21, q1, [x8, #128]
-; CHECK-SD-FP16-NEXT:    mov h1, v5.h[1]
-; CHECK-SD-FP16-NEXT:    mov h5, v5.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h20
-; CHECK-SD-FP16-NEXT:    mov h20, v0.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h0
-; CHECK-SD-FP16-NEXT:    stp q22, q7, [x8, #64]
-; CHECK-SD-FP16-NEXT:    fmov d7, x9
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h23
-; CHECK-SD-FP16-NEXT:    mov h21, v4.h[3]
-; CHECK-SD-FP16-NEXT:    mov h22, v4.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzs x13, h6
-; CHECK-SD-FP16-NEXT:    mov h6, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x14, h5
-; CHECK-SD-FP16-NEXT:    mov h0, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov v7.d[1], x10
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h1
-; CHECK-SD-FP16-NEXT:    mov v19.d[1], x12
-; CHECK-SD-FP16-NEXT:    mov v18.d[1], x9
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h4
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h20
-; CHECK-SD-FP16-NEXT:    fcvtzs x15, h21
-; CHECK-SD-FP16-NEXT:    mov v16.d[1], x13
-; CHECK-SD-FP16-NEXT:    fcvtzs x13, h22
-; CHECK-SD-FP16-NEXT:    mov v17.d[1], x14
-; CHECK-SD-FP16-NEXT:    fcvtzs x14, h6
-; CHECK-SD-FP16-NEXT:    fmov d4, x11
-; CHECK-SD-FP16-NEXT:    mov v3.d[1], x10
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h0
-; CHECK-SD-FP16-NEXT:    stp q18, q7, [x8]
-; CHECK-SD-FP16-NEXT:    fmov d0, x9
-; CHECK-SD-FP16-NEXT:    fmov d1, x12
-; CHECK-SD-FP16-NEXT:    stp q16, q19, [x8, #224]
-; CHECK-SD-FP16-NEXT:    mov v2.d[1], x15
-; CHECK-SD-FP16-NEXT:    stp q3, q17, [x8, #160]
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x13
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x14
-; CHECK-SD-FP16-NEXT:    mov v4.d[1], x10
-; CHECK-SD-FP16-NEXT:    stp q0, q2, [x8, #96]
-; CHECK-SD-FP16-NEXT:    stp q4, q1, [x8, #32]
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v32f16_v32i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v5.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v2.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v18.4s, v3.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v6.2d, v4.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.2d, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v7.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v17.2d, v5.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.2d, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v19.2d, v1.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.2d, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v20.2d, v16.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v17.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    stp q6, q4, [x8]
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v6.2d, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v16.2d, v19.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.2d, v2.2s
-; CHECK-GI-NOFP16-NEXT:    stp q7, q0, [x8, #32]
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.2d, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.2d, v18.2s
-; CHECK-GI-NOFP16-NEXT:    stp q17, q5, [x8, #64]
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.2d, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v17.2d, v3.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v7.2d, v20.2d
-; CHECK-GI-NOFP16-NEXT:    stp q16, q1, [x8, #96]
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.2d, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v5.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    stp q7, q6, [x8, #128]
-; CHECK-GI-NOFP16-NEXT:    stp q4, q2, [x8, #160]
-; CHECK-GI-NOFP16-NEXT:    stp q0, q3, [x8, #192]
-; CHECK-GI-NOFP16-NEXT:    stp q5, q1, [x8, #224]
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v32f16_v32i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    mov h4, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h5, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov h6, v0.h[3]
-; CHECK-GI-FP16-NEXT:    mov h17, v0.h[4]
-; CHECK-GI-FP16-NEXT:    mov h18, v0.h[5]
-; CHECK-GI-FP16-NEXT:    mov h19, v0.h[6]
-; CHECK-GI-FP16-NEXT:    mov h20, v0.h[7]
-; CHECK-GI-FP16-NEXT:    mov h21, v1.h[1]
-; CHECK-GI-FP16-NEXT:    fcvt d16, h0
-; CHECK-GI-FP16-NEXT:    fcvt d0, h1
-; CHECK-GI-FP16-NEXT:    mov h23, v2.h[2]
-; CHECK-GI-FP16-NEXT:    mov h24, v2.h[3]
-; CHECK-GI-FP16-NEXT:    fcvt d4, h4
-; CHECK-GI-FP16-NEXT:    fcvt d7, h5
-; CHECK-GI-FP16-NEXT:    fcvt d22, h6
-; CHECK-GI-FP16-NEXT:    fcvt d6, h17
-; CHECK-GI-FP16-NEXT:    fcvt d17, h18
-; CHECK-GI-FP16-NEXT:    fcvt d5, h19
-; CHECK-GI-FP16-NEXT:    fcvt d18, h20
-; CHECK-GI-FP16-NEXT:    fcvt d19, h21
-; CHECK-GI-FP16-NEXT:    mov h20, v1.h[2]
-; CHECK-GI-FP16-NEXT:    mov h21, v1.h[3]
-; CHECK-GI-FP16-NEXT:    mov h25, v2.h[4]
-; CHECK-GI-FP16-NEXT:    mov h26, v2.h[5]
-; CHECK-GI-FP16-NEXT:    mov v16.d[1], v4.d[0]
-; CHECK-GI-FP16-NEXT:    mov v7.d[1], v22.d[0]
-; CHECK-GI-FP16-NEXT:    mov h22, v2.h[1]
-; CHECK-GI-FP16-NEXT:    mov v6.d[1], v17.d[0]
-; CHECK-GI-FP16-NEXT:    mov h17, v1.h[4]
-; CHECK-GI-FP16-NEXT:    mov h27, v2.h[6]
-; CHECK-GI-FP16-NEXT:    mov v5.d[1], v18.d[0]
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v19.d[0]
-; CHECK-GI-FP16-NEXT:    fcvt d4, h20
-; CHECK-GI-FP16-NEXT:    mov h18, v1.h[5]
-; CHECK-GI-FP16-NEXT:    mov h19, v1.h[6]
-; CHECK-GI-FP16-NEXT:    mov h20, v1.h[7]
-; CHECK-GI-FP16-NEXT:    fcvt d21, h21
-; CHECK-GI-FP16-NEXT:    mov h28, v2.h[7]
-; CHECK-GI-FP16-NEXT:    fcvt d22, h22
-; CHECK-GI-FP16-NEXT:    fcvt d1, h17
-; CHECK-GI-FP16-NEXT:    fcvt d17, h23
-; CHECK-GI-FP16-NEXT:    fcvt d23, h24
-; CHECK-GI-FP16-NEXT:    fcvtzs v16.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v7.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v6.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    fcvt d29, h18
-; CHECK-GI-FP16-NEXT:    fcvt d19, h19
-; CHECK-GI-FP16-NEXT:    fcvt d30, h20
-; CHECK-GI-FP16-NEXT:    fcvt d20, h2
-; CHECK-GI-FP16-NEXT:    fcvtzs v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    fcvt d18, h25
-; CHECK-GI-FP16-NEXT:    fcvt d24, h26
-; CHECK-GI-FP16-NEXT:    fcvt d2, h27
-; CHECK-GI-FP16-NEXT:    fcvt d25, h28
-; CHECK-GI-FP16-NEXT:    stp q16, q7, [x8]
-; CHECK-GI-FP16-NEXT:    mov v4.d[1], v21.d[0]
-; CHECK-GI-FP16-NEXT:    mov v17.d[1], v23.d[0]
-; CHECK-GI-FP16-NEXT:    mov v1.d[1], v29.d[0]
-; CHECK-GI-FP16-NEXT:    mov v19.d[1], v30.d[0]
-; CHECK-GI-FP16-NEXT:    mov h16, v3.h[1]
-; CHECK-GI-FP16-NEXT:    stp q6, q5, [x8, #32]
-; CHECK-GI-FP16-NEXT:    mov v20.d[1], v22.d[0]
-; CHECK-GI-FP16-NEXT:    mov h21, v3.h[2]
-; CHECK-GI-FP16-NEXT:    mov h7, v3.h[3]
-; CHECK-GI-FP16-NEXT:    mov h22, v3.h[4]
-; CHECK-GI-FP16-NEXT:    mov h6, v3.h[5]
-; CHECK-GI-FP16-NEXT:    mov h23, v3.h[6]
-; CHECK-GI-FP16-NEXT:    mov h5, v3.h[7]
-; CHECK-GI-FP16-NEXT:    mov v18.d[1], v24.d[0]
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v25.d[0]
-; CHECK-GI-FP16-NEXT:    fcvt d3, h3
-; CHECK-GI-FP16-NEXT:    fcvt d16, h16
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvt d21, h21
-; CHECK-GI-FP16-NEXT:    fcvtzs v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    fcvt d7, h7
-; CHECK-GI-FP16-NEXT:    fcvt d22, h22
-; CHECK-GI-FP16-NEXT:    fcvt d6, h6
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    fcvt d23, h23
-; CHECK-GI-FP16-NEXT:    fcvt d5, h5
-; CHECK-GI-FP16-NEXT:    fcvtzs v19.2d, v19.2d
-; CHECK-GI-FP16-NEXT:    mov v3.d[1], v16.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzs v16.2d, v20.2d
-; CHECK-GI-FP16-NEXT:    stp q0, q4, [x8, #64]
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.2d, v17.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v4.2d, v18.2d
-; CHECK-GI-FP16-NEXT:    mov v21.d[1], v7.d[0]
-; CHECK-GI-FP16-NEXT:    mov v22.d[1], v6.d[0]
-; CHECK-GI-FP16-NEXT:    mov v23.d[1], v5.d[0]
-; CHECK-GI-FP16-NEXT:    stp q1, q19, [x8, #96]
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v2.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    stp q16, q0, [x8, #128]
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.2d, v21.2d
-; CHECK-GI-FP16-NEXT:    fcvtzs v3.2d, v22.2d
-; CHECK-GI-FP16-NEXT:    stp q4, q1, [x8, #160]
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.2d, v23.2d
-; CHECK-GI-FP16-NEXT:    stp q2, q0, [x8, #192]
-; CHECK-GI-FP16-NEXT:    stp q3, q1, [x8, #224]
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = fptosi <32 x half> %a to <32 x i64>
-  ret <32 x i64> %c
-}
-
-define <32 x i64> @fptou_v32f16_v32i64(<32 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v32f16_v32i64:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NOFP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
-; CHECK-SD-NOFP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
-; CHECK-SD-NOFP16-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h1
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h2
-; CHECK-SD-NOFP16-NEXT:    mov h26, v2.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s19, h0
-; CHECK-SD-NOFP16-NEXT:    mov h27, v3.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h20, v2.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h18, v1.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h16, v4.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov h17, v5.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h5
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h6
-; CHECK-SD-NOFP16-NEXT:    mov h25, v6.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s21
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h7
-; CHECK-SD-NOFP16-NEXT:    mov h21, v3.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s19
-; CHECK-SD-NOFP16-NEXT:    fcvt s27, h27
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s16, h16
-; CHECK-SD-NOFP16-NEXT:    fcvt s17, h17
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s23
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s24
-; CHECK-SD-NOFP16-NEXT:    fcvt s23, h25
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    mov h26, v3.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h24, v2.h[3]
-; CHECK-SD-NOFP16-NEXT:    fmov d19, x9
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x9, s22
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h3
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h21
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s16
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s17
-; CHECK-SD-NOFP16-NEXT:    fmov d2, x12
-; CHECK-SD-NOFP16-NEXT:    fmov d16, x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s23
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s25
-; CHECK-SD-NOFP16-NEXT:    mov h23, v1.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s25, h26
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h24
-; CHECK-SD-NOFP16-NEXT:    mov h1, v1.h[3]
-; CHECK-SD-NOFP16-NEXT:    fmov d26, x11
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s21
-; CHECK-SD-NOFP16-NEXT:    fmov d3, x14
-; CHECK-SD-NOFP16-NEXT:    fmov d17, x15
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s22
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s27
-; CHECK-SD-NOFP16-NEXT:    mov h22, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    fcvt s18, h18
-; CHECK-SD-NOFP16-NEXT:    fcvt s21, h23
-; CHECK-SD-NOFP16-NEXT:    fmov d23, x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s25
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    fmov d25, x14
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s24
-; CHECK-SD-NOFP16-NEXT:    fmov d24, x15
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s18
-; CHECK-SD-NOFP16-NEXT:    mov h18, v7.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov v25.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s21
-; CHECK-SD-NOFP16-NEXT:    mov h21, v7.h[2]
-; CHECK-SD-NOFP16-NEXT:    mov v24.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s20
-; CHECK-SD-NOFP16-NEXT:    mov h20, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov v23.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s1
-; CHECK-SD-NOFP16-NEXT:    mov h1, v6.h[3]
-; CHECK-SD-NOFP16-NEXT:    mov h6, v6.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov v19.d[1], x15
-; CHECK-SD-NOFP16-NEXT:    mov h7, v7.h[3]
-; CHECK-SD-NOFP16-NEXT:    stp q25, q24, [x8, #192]
-; CHECK-SD-NOFP16-NEXT:    fmov d24, x13
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    mov v26.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s22
-; CHECK-SD-NOFP16-NEXT:    mov h22, v5.h[1]
-; CHECK-SD-NOFP16-NEXT:    mov h5, v5.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-SD-NOFP16-NEXT:    mov v24.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    mov h25, v4.h[3]
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h6
-; CHECK-SD-NOFP16-NEXT:    stp q26, q23, [x8, #128]
-; CHECK-SD-NOFP16-NEXT:    fmov d23, x12
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s20
-; CHECK-SD-NOFP16-NEXT:    mov h20, v4.h[1]
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h5
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s0
-; CHECK-SD-NOFP16-NEXT:    stp q19, q24, [x8, #64]
-; CHECK-SD-NOFP16-NEXT:    fcvt s22, h22
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x10
-; CHECK-SD-NOFP16-NEXT:    fmov d19, x11
-; CHECK-SD-NOFP16-NEXT:    fcvt s4, h4
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s1
-; CHECK-SD-NOFP16-NEXT:    fcvt s1, h21
-; CHECK-SD-NOFP16-NEXT:    fcvt s24, h25
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s6
-; CHECK-SD-NOFP16-NEXT:    fcvt s20, h20
-; CHECK-SD-NOFP16-NEXT:    fcvt s6, h7
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s5
-; CHECK-SD-NOFP16-NEXT:    mov v19.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    fcvt s5, h18
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s22
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x12
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x12, s4
-; CHECK-SD-NOFP16-NEXT:    mov v23.d[1], x10
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x10, s1
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x15, s24
-; CHECK-SD-NOFP16-NEXT:    mov v16.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x11, s20
-; CHECK-SD-NOFP16-NEXT:    mov v17.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x14, s6
-; CHECK-SD-NOFP16-NEXT:    mov v2.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    fcvtzu x13, s5
-; CHECK-SD-NOFP16-NEXT:    fmov d4, x9
-; CHECK-SD-NOFP16-NEXT:    stp q0, q19, [x8]
-; CHECK-SD-NOFP16-NEXT:    fmov d0, x12
-; CHECK-SD-NOFP16-NEXT:    stp q16, q23, [x8, #224]
-; CHECK-SD-NOFP16-NEXT:    fmov d1, x10
-; CHECK-SD-NOFP16-NEXT:    mov v3.d[1], x15
-; CHECK-SD-NOFP16-NEXT:    stp q2, q17, [x8, #160]
-; CHECK-SD-NOFP16-NEXT:    mov v0.d[1], x11
-; CHECK-SD-NOFP16-NEXT:    mov v4.d[1], x13
-; CHECK-SD-NOFP16-NEXT:    mov v1.d[1], x14
-; CHECK-SD-NOFP16-NEXT:    stp q0, q3, [x8, #96]
-; CHECK-SD-NOFP16-NEXT:    stp q4, q1, [x8, #32]
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v32f16_v32i64:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
-; CHECK-SD-FP16-NEXT:    ext v5.16b, v2.16b, v2.16b, #8
-; CHECK-SD-FP16-NEXT:    ext v6.16b, v3.16b, v3.16b, #8
-; CHECK-SD-FP16-NEXT:    mov h16, v3.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h0
-; CHECK-SD-FP16-NEXT:    mov h23, v3.h[3]
-; CHECK-SD-FP16-NEXT:    mov h25, v3.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzu x15, h3
-; CHECK-SD-FP16-NEXT:    mov h24, v2.h[2]
-; CHECK-SD-FP16-NEXT:    mov h19, v1.h[2]
-; CHECK-SD-FP16-NEXT:    mov h21, v2.h[1]
-; CHECK-SD-FP16-NEXT:    mov h26, v2.h[3]
-; CHECK-SD-FP16-NEXT:    mov h17, v4.h[2]
-; CHECK-SD-FP16-NEXT:    mov h18, v5.h[2]
-; CHECK-SD-FP16-NEXT:    mov h22, v6.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h5
-; CHECK-SD-FP16-NEXT:    fcvtzu x12, h16
-; CHECK-SD-FP16-NEXT:    fcvtzu x11, h6
-; CHECK-SD-FP16-NEXT:    mov h7, v1.h[1]
-; CHECK-SD-FP16-NEXT:    mov h20, v1.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x13, h17
-; CHECK-SD-FP16-NEXT:    fcvtzu x14, h18
-; CHECK-SD-FP16-NEXT:    fmov d18, x9
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h22
-; CHECK-SD-FP16-NEXT:    fmov d3, x10
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h23
-; CHECK-SD-FP16-NEXT:    fmov d22, x12
-; CHECK-SD-FP16-NEXT:    fcvtzu x12, h25
-; CHECK-SD-FP16-NEXT:    fmov d23, x15
-; CHECK-SD-FP16-NEXT:    fmov d16, x11
-; CHECK-SD-FP16-NEXT:    fcvtzu x11, h2
-; CHECK-SD-FP16-NEXT:    fcvtzu x15, h21
-; CHECK-SD-FP16-NEXT:    fmov d2, x13
-; CHECK-SD-FP16-NEXT:    fcvtzu x13, h24
-; CHECK-SD-FP16-NEXT:    fmov d17, x14
-; CHECK-SD-FP16-NEXT:    fcvtzu x14, h19
-; CHECK-SD-FP16-NEXT:    mov v22.d[1], x10
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h1
-; CHECK-SD-FP16-NEXT:    mov v23.d[1], x12
-; CHECK-SD-FP16-NEXT:    fmov d19, x9
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h26
-; CHECK-SD-FP16-NEXT:    fcvtzu x12, h20
-; CHECK-SD-FP16-NEXT:    mov h20, v0.h[2]
-; CHECK-SD-FP16-NEXT:    fmov d21, x11
-; CHECK-SD-FP16-NEXT:    fmov d1, x13
-; CHECK-SD-FP16-NEXT:    fcvtzu x13, h7
-; CHECK-SD-FP16-NEXT:    mov h24, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fmov d7, x14
-; CHECK-SD-FP16-NEXT:    stp q23, q22, [x8, #192]
-; CHECK-SD-FP16-NEXT:    fmov d22, x10
-; CHECK-SD-FP16-NEXT:    mov v21.d[1], x15
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x9
-; CHECK-SD-FP16-NEXT:    mov h23, v0.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h20
-; CHECK-SD-FP16-NEXT:    mov v7.d[1], x12
-; CHECK-SD-FP16-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
-; CHECK-SD-FP16-NEXT:    mov h20, v6.h[3]
-; CHECK-SD-FP16-NEXT:    mov v22.d[1], x13
-; CHECK-SD-FP16-NEXT:    mov h6, v6.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h24
-; CHECK-SD-FP16-NEXT:    stp q21, q1, [x8, #128]
-; CHECK-SD-FP16-NEXT:    mov h1, v5.h[1]
-; CHECK-SD-FP16-NEXT:    mov h5, v5.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x12, h20
-; CHECK-SD-FP16-NEXT:    mov h20, v0.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzu x11, h0
-; CHECK-SD-FP16-NEXT:    stp q22, q7, [x8, #64]
-; CHECK-SD-FP16-NEXT:    fmov d7, x9
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h23
-; CHECK-SD-FP16-NEXT:    mov h21, v4.h[3]
-; CHECK-SD-FP16-NEXT:    mov h22, v4.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzu x13, h6
-; CHECK-SD-FP16-NEXT:    mov h6, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x14, h5
-; CHECK-SD-FP16-NEXT:    mov h0, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov v7.d[1], x10
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h1
-; CHECK-SD-FP16-NEXT:    mov v19.d[1], x12
-; CHECK-SD-FP16-NEXT:    mov v18.d[1], x9
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h4
-; CHECK-SD-FP16-NEXT:    fcvtzu x12, h20
-; CHECK-SD-FP16-NEXT:    fcvtzu x15, h21
-; CHECK-SD-FP16-NEXT:    mov v16.d[1], x13
-; CHECK-SD-FP16-NEXT:    fcvtzu x13, h22
-; CHECK-SD-FP16-NEXT:    mov v17.d[1], x14
-; CHECK-SD-FP16-NEXT:    fcvtzu x14, h6
-; CHECK-SD-FP16-NEXT:    fmov d4, x11
-; CHECK-SD-FP16-NEXT:    mov v3.d[1], x10
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h0
-; CHECK-SD-FP16-NEXT:    stp q18, q7, [x8]
-; CHECK-SD-FP16-NEXT:    fmov d0, x9
-; CHECK-SD-FP16-NEXT:    fmov d1, x12
-; CHECK-SD-FP16-NEXT:    stp q16, q19, [x8, #224]
-; CHECK-SD-FP16-NEXT:    mov v2.d[1], x15
-; CHECK-SD-FP16-NEXT:    stp q3, q17, [x8, #160]
-; CHECK-SD-FP16-NEXT:    mov v0.d[1], x13
-; CHECK-SD-FP16-NEXT:    mov v1.d[1], x14
-; CHECK-SD-FP16-NEXT:    mov v4.d[1], x10
-; CHECK-SD-FP16-NEXT:    stp q0, q2, [x8, #96]
-; CHECK-SD-FP16-NEXT:    stp q4, q1, [x8, #32]
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v32f16_v32i64:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v5.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v16.4s, v2.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v18.4s, v3.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v6.2d, v4.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v4.2d, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v7.2d, v0.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.2d, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v17.2d, v5.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.2d, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v19.2d, v1.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.2d, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v20.2d, v16.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v17.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    stp q6, q4, [x8]
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v6.2d, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v16.2d, v19.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.2d, v2.2s
-; CHECK-GI-NOFP16-NEXT:    stp q7, q0, [x8, #32]
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.2d, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.2d, v18.2s
-; CHECK-GI-NOFP16-NEXT:    stp q17, q5, [x8, #64]
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v5.2d, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtl v17.2d, v3.2s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v7.2d, v20.2d
-; CHECK-GI-NOFP16-NEXT:    stp q16, q1, [x8, #96]
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.2d, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.2d, v2.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.2d, v5.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v5.2d, v17.2d
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-NOFP16-NEXT:    stp q7, q6, [x8, #128]
-; CHECK-GI-NOFP16-NEXT:    stp q4, q2, [x8, #160]
-; CHECK-GI-NOFP16-NEXT:    stp q0, q3, [x8, #192]
-; CHECK-GI-NOFP16-NEXT:    stp q5, q1, [x8, #224]
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v32f16_v32i64:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    mov h4, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h5, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov h6, v0.h[3]
-; CHECK-GI-FP16-NEXT:    mov h17, v0.h[4]
-; CHECK-GI-FP16-NEXT:    mov h18, v0.h[5]
-; CHECK-GI-FP16-NEXT:    mov h19, v0.h[6]
-; CHECK-GI-FP16-NEXT:    mov h20, v0.h[7]
-; CHECK-GI-FP16-NEXT:    mov h21, v1.h[1]
-; CHECK-GI-FP16-NEXT:    fcvt d16, h0
-; CHECK-GI-FP16-NEXT:    fcvt d0, h1
-; CHECK-GI-FP16-NEXT:    mov h23, v2.h[2]
-; CHECK-GI-FP16-NEXT:    mov h24, v2.h[3]
-; CHECK-GI-FP16-NEXT:    fcvt d4, h4
-; CHECK-GI-FP16-NEXT:    fcvt d7, h5
-; CHECK-GI-FP16-NEXT:    fcvt d22, h6
-; CHECK-GI-FP16-NEXT:    fcvt d6, h17
-; CHECK-GI-FP16-NEXT:    fcvt d17, h18
-; CHECK-GI-FP16-NEXT:    fcvt d5, h19
-; CHECK-GI-FP16-NEXT:    fcvt d18, h20
-; CHECK-GI-FP16-NEXT:    fcvt d19, h21
-; CHECK-GI-FP16-NEXT:    mov h20, v1.h[2]
-; CHECK-GI-FP16-NEXT:    mov h21, v1.h[3]
-; CHECK-GI-FP16-NEXT:    mov h25, v2.h[4]
-; CHECK-GI-FP16-NEXT:    mov h26, v2.h[5]
-; CHECK-GI-FP16-NEXT:    mov v16.d[1], v4.d[0]
-; CHECK-GI-FP16-NEXT:    mov v7.d[1], v22.d[0]
-; CHECK-GI-FP16-NEXT:    mov h22, v2.h[1]
-; CHECK-GI-FP16-NEXT:    mov v6.d[1], v17.d[0]
-; CHECK-GI-FP16-NEXT:    mov h17, v1.h[4]
-; CHECK-GI-FP16-NEXT:    mov h27, v2.h[6]
-; CHECK-GI-FP16-NEXT:    mov v5.d[1], v18.d[0]
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], v19.d[0]
-; CHECK-GI-FP16-NEXT:    fcvt d4, h20
-; CHECK-GI-FP16-NEXT:    mov h18, v1.h[5]
-; CHECK-GI-FP16-NEXT:    mov h19, v1.h[6]
-; CHECK-GI-FP16-NEXT:    mov h20, v1.h[7]
-; CHECK-GI-FP16-NEXT:    fcvt d21, h21
-; CHECK-GI-FP16-NEXT:    mov h28, v2.h[7]
-; CHECK-GI-FP16-NEXT:    fcvt d22, h22
-; CHECK-GI-FP16-NEXT:    fcvt d1, h17
-; CHECK-GI-FP16-NEXT:    fcvt d17, h23
-; CHECK-GI-FP16-NEXT:    fcvt d23, h24
-; CHECK-GI-FP16-NEXT:    fcvtzu v16.2d, v16.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v7.2d, v7.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v6.2d, v6.2d
-; CHECK-GI-FP16-NEXT:    fcvt d29, h18
-; CHECK-GI-FP16-NEXT:    fcvt d19, h19
-; CHECK-GI-FP16-NEXT:    fcvt d30, h20
-; CHECK-GI-FP16-NEXT:    fcvt d20, h2
-; CHECK-GI-FP16-NEXT:    fcvtzu v5.2d, v5.2d
-; CHECK-GI-FP16-NEXT:    fcvt d18, h25
-; CHECK-GI-FP16-NEXT:    fcvt d24, h26
-; CHECK-GI-FP16-NEXT:    fcvt d2, h27
-; CHECK-GI-FP16-NEXT:    fcvt d25, h28
-; CHECK-GI-FP16-NEXT:    stp q16, q7, [x8]
-; CHECK-GI-FP16-NEXT:    mov v4.d[1], v21.d[0]
-; CHECK-GI-FP16-NEXT:    mov v17.d[1], v23.d[0]
-; CHECK-GI-FP16-NEXT:    mov v1.d[1], v29.d[0]
-; CHECK-GI-FP16-NEXT:    mov v19.d[1], v30.d[0]
-; CHECK-GI-FP16-NEXT:    mov h16, v3.h[1]
-; CHECK-GI-FP16-NEXT:    stp q6, q5, [x8, #32]
-; CHECK-GI-FP16-NEXT:    mov v20.d[1], v22.d[0]
-; CHECK-GI-FP16-NEXT:    mov h21, v3.h[2]
-; CHECK-GI-FP16-NEXT:    mov h7, v3.h[3]
-; CHECK-GI-FP16-NEXT:    mov h22, v3.h[4]
-; CHECK-GI-FP16-NEXT:    mov h6, v3.h[5]
-; CHECK-GI-FP16-NEXT:    mov h23, v3.h[6]
-; CHECK-GI-FP16-NEXT:    mov h5, v3.h[7]
-; CHECK-GI-FP16-NEXT:    mov v18.d[1], v24.d[0]
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], v25.d[0]
-; CHECK-GI-FP16-NEXT:    fcvt d3, h3
-; CHECK-GI-FP16-NEXT:    fcvt d16, h16
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.2d, v0.2d
-; CHECK-GI-FP16-NEXT:    fcvt d21, h21
-; CHECK-GI-FP16-NEXT:    fcvtzu v4.2d, v4.2d
-; CHECK-GI-FP16-NEXT:    fcvt d7, h7
-; CHECK-GI-FP16-NEXT:    fcvt d22, h22
-; CHECK-GI-FP16-NEXT:    fcvt d6, h6
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.2d, v1.2d
-; CHECK-GI-FP16-NEXT:    fcvt d23, h23
-; CHECK-GI-FP16-NEXT:    fcvt d5, h5
-; CHECK-GI-FP16-NEXT:    fcvtzu v19.2d, v19.2d
-; CHECK-GI-FP16-NEXT:    mov v3.d[1], v16.d[0]
-; CHECK-GI-FP16-NEXT:    fcvtzu v16.2d, v20.2d
-; CHECK-GI-FP16-NEXT:    stp q0, q4, [x8, #64]
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.2d, v17.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v4.2d, v18.2d
-; CHECK-GI-FP16-NEXT:    mov v21.d[1], v7.d[0]
-; CHECK-GI-FP16-NEXT:    mov v22.d[1], v6.d[0]
-; CHECK-GI-FP16-NEXT:    mov v23.d[1], v5.d[0]
-; CHECK-GI-FP16-NEXT:    stp q1, q19, [x8, #96]
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.2d, v2.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v2.2d, v3.2d
-; CHECK-GI-FP16-NEXT:    stp q16, q0, [x8, #128]
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.2d, v21.2d
-; CHECK-GI-FP16-NEXT:    fcvtzu v3.2d, v22.2d
-; CHECK-GI-FP16-NEXT:    stp q4, q1, [x8, #160]
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.2d, v23.2d
-; CHECK-GI-FP16-NEXT:    stp q2, q0, [x8, #192]
-; CHECK-GI-FP16-NEXT:    stp q3, q1, [x8, #224]
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = fptoui <32 x half> %a to <32 x i64>
-  ret <32 x i64> %c
-}
-
 define <2 x i32> @fptos_v2f16_v2i32(<2 x half> %a) {
-; CHECK-SD-LABEL: fptos_v2f16_v2i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v2f16_v2i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NEXT:    fcvtzs v0.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f16_v2i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f16_v2i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f16_v2i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f16_v2i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x half> %a to <2 x i32>
   ret <2 x i32> %c
 }
 
 define <2 x i32> @fptou_v2f16_v2i32(<2 x half> %a) {
-; CHECK-SD-LABEL: fptou_v2f16_v2i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f16_v2i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NEXT:    fcvtzu v0.2s, v0.2s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f16_v2i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f16_v2i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f16_v2i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f16_v2i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x half> %a to <2 x i32>
   ret <2 x i32> %c
@@ -5773,2221 +5639,1761 @@ entry:
 }
 
 define <8 x i32> @fptos_v8f16_v8i32(<8 x half> %a) {
-; CHECK-SD-LABEL: fptos_v8f16_v8i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v1.4s, v0.8h
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v8f16_v8i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NEXT:    fcvtl2 v2.4s, v0.8h
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v2.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f16_v8i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v8f16_v8i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f16_v8i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v8f16_v8i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x half> %a to <8 x i32>
   ret <8 x i32> %c
 }
 
 define <8 x i32> @fptou_v8f16_v8i32(<8 x half> %a) {
-; CHECK-SD-LABEL: fptou_v8f16_v8i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v1.4s, v0.8h
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v8f16_v8i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NEXT:    fcvtl2 v2.4s, v0.8h
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v1.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v2.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f16_v8i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v8f16_v8i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f16_v8i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v8f16_v8i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x half> %a to <8 x i32>
   ret <8 x i32> %c
 }
 
 define <16 x i32> @fptos_v16f16_v16i32(<16 x half> %a) {
-; CHECK-SD-LABEL: fptos_v16f16_v16i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtl2 v3.4s, v0.8h
-; CHECK-SD-NEXT:    fcvtl2 v4.4s, v1.8h
-; CHECK-SD-NEXT:    fcvtl v5.4s, v1.4h
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v2.4s
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzs v3.4s, v4.4s
-; CHECK-SD-NEXT:    fcvtzs v2.4s, v5.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v16f16_v16i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NEXT:    fcvtl2 v3.4s, v0.8h
-; CHECK-GI-NEXT:    fcvtl v4.4s, v1.4h
-; CHECK-GI-NEXT:    fcvtl2 v5.4s, v1.8h
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v3.4s
-; CHECK-GI-NEXT:    fcvtzs v2.4s, v4.4s
-; CHECK-GI-NEXT:    fcvtzs v3.4s, v5.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v16f16_v16i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v16f16_v16i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-FP16-SD-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-FP16-SD-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v16f16_v16i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v16f16_v16i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtl v4.4s, v1.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <16 x half> %a to <16 x i32>
   ret <16 x i32> %c
 }
 
 define <16 x i32> @fptou_v16f16_v16i32(<16 x half> %a) {
-; CHECK-SD-LABEL: fptou_v16f16_v16i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtl2 v3.4s, v0.8h
-; CHECK-SD-NEXT:    fcvtl2 v4.4s, v1.8h
-; CHECK-SD-NEXT:    fcvtl v5.4s, v1.4h
-; CHECK-SD-NEXT:    fcvtzu v0.4s, v2.4s
-; CHECK-SD-NEXT:    fcvtzu v1.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtzu v3.4s, v4.4s
-; CHECK-SD-NEXT:    fcvtzu v2.4s, v5.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v16f16_v16i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NEXT:    fcvtl2 v3.4s, v0.8h
-; CHECK-GI-NEXT:    fcvtl v4.4s, v1.4h
-; CHECK-GI-NEXT:    fcvtl2 v5.4s, v1.8h
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v2.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v3.4s
-; CHECK-GI-NEXT:    fcvtzu v2.4s, v4.4s
-; CHECK-GI-NEXT:    fcvtzu v3.4s, v5.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v16f16_v16i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v3.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v2.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v16f16_v16i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-FP16-SD-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-FP16-SD-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtzu v3.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    fcvtzu v2.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v16f16_v16i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v16f16_v16i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtl v4.4s, v1.4h
+; CHECK-FP16-GI-NEXT:    fcvtl2 v5.4s, v1.8h
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <16 x half> %a to <16 x i32>
   ret <16 x i32> %c
 }
 
-define <32 x i32> @fptos_v32f16_v32i32(<32 x half> %a) {
-; CHECK-SD-LABEL: fptos_v32f16_v32i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v4.4s, v0.8h
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtl2 v5.4s, v1.8h
-; CHECK-SD-NEXT:    fcvtl v6.4s, v1.4h
-; CHECK-SD-NEXT:    fcvtl v7.4s, v2.4h
-; CHECK-SD-NEXT:    fcvtl2 v16.4s, v2.8h
-; CHECK-SD-NEXT:    fcvtl2 v17.4s, v3.8h
-; CHECK-SD-NEXT:    fcvtl v18.4s, v3.4h
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v4.4s
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzs v3.4s, v5.4s
-; CHECK-SD-NEXT:    fcvtzs v2.4s, v6.4s
-; CHECK-SD-NEXT:    fcvtzs v4.4s, v7.4s
-; CHECK-SD-NEXT:    fcvtzs v5.4s, v16.4s
-; CHECK-SD-NEXT:    fcvtzs v7.4s, v17.4s
-; CHECK-SD-NEXT:    fcvtzs v6.4s, v18.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v32f16_v32i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v4.4s, v0.4h
-; CHECK-GI-NEXT:    fcvtl2 v5.4s, v0.8h
-; CHECK-GI-NEXT:    fcvtl v6.4s, v1.4h
-; CHECK-GI-NEXT:    fcvtl2 v7.4s, v1.8h
-; CHECK-GI-NEXT:    fcvtl v16.4s, v2.4h
-; CHECK-GI-NEXT:    fcvtl2 v17.4s, v2.8h
-; CHECK-GI-NEXT:    fcvtl v18.4s, v3.4h
-; CHECK-GI-NEXT:    fcvtl2 v19.4s, v3.8h
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v4.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v5.4s
-; CHECK-GI-NEXT:    fcvtzs v2.4s, v6.4s
-; CHECK-GI-NEXT:    fcvtzs v3.4s, v7.4s
-; CHECK-GI-NEXT:    fcvtzs v4.4s, v16.4s
-; CHECK-GI-NEXT:    fcvtzs v5.4s, v17.4s
-; CHECK-GI-NEXT:    fcvtzs v6.4s, v18.4s
-; CHECK-GI-NEXT:    fcvtzs v7.4s, v19.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <32 x half> %a to <32 x i32>
-  ret <32 x i32> %c
-}
-
-define <32 x i32> @fptou_v32f16_v32i32(<32 x half> %a) {
-; CHECK-SD-LABEL: fptou_v32f16_v32i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl2 v4.4s, v0.8h
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtl2 v5.4s, v1.8h
-; CHECK-SD-NEXT:    fcvtl v6.4s, v1.4h
-; CHECK-SD-NEXT:    fcvtl v7.4s, v2.4h
-; CHECK-SD-NEXT:    fcvtl2 v16.4s, v2.8h
-; CHECK-SD-NEXT:    fcvtl2 v17.4s, v3.8h
-; CHECK-SD-NEXT:    fcvtl v18.4s, v3.4h
-; CHECK-SD-NEXT:    fcvtzu v1.4s, v4.4s
-; CHECK-SD-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzu v3.4s, v5.4s
-; CHECK-SD-NEXT:    fcvtzu v2.4s, v6.4s
-; CHECK-SD-NEXT:    fcvtzu v4.4s, v7.4s
-; CHECK-SD-NEXT:    fcvtzu v5.4s, v16.4s
-; CHECK-SD-NEXT:    fcvtzu v7.4s, v17.4s
-; CHECK-SD-NEXT:    fcvtzu v6.4s, v18.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v32f16_v32i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fcvtl v4.4s, v0.4h
-; CHECK-GI-NEXT:    fcvtl2 v5.4s, v0.8h
-; CHECK-GI-NEXT:    fcvtl v6.4s, v1.4h
-; CHECK-GI-NEXT:    fcvtl2 v7.4s, v1.8h
-; CHECK-GI-NEXT:    fcvtl v16.4s, v2.4h
-; CHECK-GI-NEXT:    fcvtl2 v17.4s, v2.8h
-; CHECK-GI-NEXT:    fcvtl v18.4s, v3.4h
-; CHECK-GI-NEXT:    fcvtl2 v19.4s, v3.8h
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v4.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v5.4s
-; CHECK-GI-NEXT:    fcvtzu v2.4s, v6.4s
-; CHECK-GI-NEXT:    fcvtzu v3.4s, v7.4s
-; CHECK-GI-NEXT:    fcvtzu v4.4s, v16.4s
-; CHECK-GI-NEXT:    fcvtzu v5.4s, v17.4s
-; CHECK-GI-NEXT:    fcvtzu v6.4s, v18.4s
-; CHECK-GI-NEXT:    fcvtzu v7.4s, v19.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <32 x half> %a to <32 x i32>
-  ret <32 x i32> %c
-}
-
 define <2 x i16> @fptos_v2f16_v2i16(<2 x half> %a) {
-; CHECK-SD-LABEL: fptos_v2f16_v2i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v2f16_v2i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.2s, v0.2s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v2f16_v2i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f16_v2i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f16_v2i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f16_v2i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f16_v2i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x half> %a to <2 x i16>
   ret <2 x i16> %c
 }
 
 define <2 x i16> @fptou_v2f16_v2i16(<2 x half> %a) {
-; CHECK-SD-LABEL: fptou_v2f16_v2i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v2f16_v2i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.2s, v0.2s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v2f16_v2i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f16_v2i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f16_v2i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f16_v2i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f16_v2i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x half> %a to <2 x i16>
   ret <2 x i16> %c
 }
 
 define <3 x i16> @fptos_v3f16_v3i16(<3 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v3f16_v3i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v3f16_v3i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v3f16_v3i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v3f16_v3i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptos_v3f16_v3i16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_v3f16_v3i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptosi <3 x half> %a to <3 x i16>
   ret <3 x i16> %c
 }
 
 define <3 x i16> @fptou_v3f16_v3i16(<3 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v3f16_v3i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v3f16_v3i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v3f16_v3i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v3f16_v3i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptou_v3f16_v3i16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptou_v3f16_v3i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptoui <3 x half> %a to <3 x i16>
   ret <3 x i16> %c
 }
 
 define <4 x i16> @fptos_v4f16_v4i16(<4 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v4f16_v4i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v4f16_v4i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v4f16_v4i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v4f16_v4i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptos_v4f16_v4i16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_v4f16_v4i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptosi <4 x half> %a to <4 x i16>
   ret <4 x i16> %c
 }
 
 define <4 x i16> @fptou_v4f16_v4i16(<4 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v4f16_v4i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v4f16_v4i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v4f16_v4i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v4f16_v4i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptou_v4f16_v4i16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptou_v4f16_v4i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptoui <4 x half> %a to <4 x i16>
   ret <4 x i16> %c
 }
 
 define <8 x i16> @fptos_v8f16_v8i16(<8 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v8f16_v8i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v8f16_v8i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v8f16_v8i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v8f16_v8i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f16_v8i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_v8f16_v8i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f16_v8i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x half> %a to <8 x i16>
   ret <8 x i16> %c
 }
 
 define <8 x i16> @fptou_v8f16_v8i16(<8 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v8f16_v8i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v8f16_v8i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v8f16_v8i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v8f16_v8i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f16_v8i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptou_v8f16_v8i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f16_v8i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x half> %a to <8 x i16>
   ret <8 x i16> %c
 }
 
 define <16 x i16> @fptos_v16f16_v16i16(<16 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v16f16_v16i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v1.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v3.8h
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v16f16_v16i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v16f16_v16i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v16f16_v16i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v16f16_v16i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v1.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_v16f16_v16i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v16f16_v16i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = fptosi <16 x half> %a to <16 x i16>
   ret <16 x i16> %c
 }
 
 define <16 x i16> @fptou_v16f16_v16i16(<16 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v16f16_v16i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v1.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v3.8h
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v16f16_v16i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v16f16_v16i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v16f16_v16i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v16f16_v16i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v2.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v3.4s, v1.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v1.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptou_v16f16_v16i16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v16f16_v16i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = fptoui <16 x half> %a to <16 x i16>
   ret <16 x i16> %c
 }
 
-define <32 x i16> @fptos_v32f16_v32i16(<32 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v32f16_v32i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v6.4s, v2.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v7.4s, v3.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v5.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v6.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v7.8h
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v32f16_v32i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v2.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v3.8h, v3.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v32f16_v32i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v5.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v4.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v5.8h, v1.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v2.8h, v6.8h, v2.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v3.8h, v7.8h, v3.8h
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v32f16_v32i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v2.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v3.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = fptosi <32 x half> %a to <32 x i16>
-  ret <32 x i16> %c
-}
-
-define <32 x i16> @fptou_v32f16_v32i16(<32 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v32f16_v32i16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v1.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v6.4s, v2.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v7.4s, v3.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v4.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v5.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v6.4s, v6.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v7.4s, v7.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v4.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v5.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v6.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v7.8h
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v32f16_v32i16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v2.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v3.8h, v3.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v32f16_v32i16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v5.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v5.4s, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v6.4s, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v7.4s, v7.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v4.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v5.8h, v1.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v2.8h, v6.8h, v2.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v3.8h, v7.8h, v3.8h
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v32f16_v32i16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v2.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v3.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = fptoui <32 x half> %a to <32 x i16>
-  ret <32 x i16> %c
-}
-
 define <2 x i8> @fptos_v2f16_v2i8(<2 x half> %a) {
-; CHECK-SD-LABEL: fptos_v2f16_v2i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v2f16_v2i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.2s, v0.2s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v2f16_v2i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f16_v2i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f16_v2i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f16_v2i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f16_v2i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x half> %a to <2 x i8>
   ret <2 x i8> %c
 }
 
 define <2 x i8> @fptou_v2f16_v2i8(<2 x half> %a) {
-; CHECK-SD-LABEL: fptou_v2f16_v2i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v2f16_v2i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.2s, v0.2s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v2f16_v2i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f16_v2i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f16_v2i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f16_v2i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f16_v2i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x half> %a to <2 x i8>
   ret <2 x i8> %c
 }
 
 define <3 x i8> @fptos_v3f16_v3i8(<3 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v3f16_v3i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v3f16_v3i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-FP16-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v3f16_v3i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    mov s2, v0.s[2]
-; CHECK-GI-NOFP16-NEXT:    fmov w0, s0
-; CHECK-GI-NOFP16-NEXT:    fmov w1, s1
-; CHECK-GI-NOFP16-NEXT:    fmov w2, s2
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v3f16_v3i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    umov w0, v0.h[0]
-; CHECK-GI-FP16-NEXT:    umov w1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    umov w2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v3f16_v3i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_v3f16_v3i8:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-NEXT:    umov w0, v0.h[0]
+; CHECK-FP16-NEXT:    umov w1, v0.h[1]
+; CHECK-FP16-NEXT:    umov w2, v0.h[2]
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v3f16_v3i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-NOFP16-GI-NEXT:    fmov w0, s0
+; CHECK-NOFP16-GI-NEXT:    fmov w1, s1
+; CHECK-NOFP16-GI-NEXT:    fmov w2, s2
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = fptosi <3 x half> %a to <3 x i8>
   ret <3 x i8> %c
 }
 
 define <3 x i8> @fptou_v3f16_v3i8(<3 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v3f16_v3i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-NOFP16-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v3f16_v3i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-FP16-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v3f16_v3i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    mov s1, v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    mov s2, v0.s[2]
-; CHECK-GI-NOFP16-NEXT:    fmov w0, s0
-; CHECK-GI-NOFP16-NEXT:    fmov w1, s1
-; CHECK-GI-NOFP16-NEXT:    fmov w2, s2
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v3f16_v3i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    umov w0, v0.h[0]
-; CHECK-GI-FP16-NEXT:    umov w1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    umov w2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v3f16_v3i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-NOFP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v3f16_v3i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-SD-NEXT:    umov w0, v0.h[0]
+; CHECK-FP16-SD-NEXT:    umov w1, v0.h[1]
+; CHECK-FP16-SD-NEXT:    umov w2, v0.h[2]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v3f16_v3i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    mov s1, v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    mov s2, v0.s[2]
+; CHECK-NOFP16-GI-NEXT:    fmov w0, s0
+; CHECK-NOFP16-GI-NEXT:    fmov w1, s1
+; CHECK-NOFP16-GI-NEXT:    fmov w2, s2
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v3f16_v3i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    umov w0, v0.h[0]
+; CHECK-FP16-GI-NEXT:    umov w1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    umov w2, v0.h[2]
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <3 x half> %a to <3 x i8>
   ret <3 x i8> %c
 }
 
 define <4 x i8> @fptos_v4f16_v4i8(<4 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v4f16_v4i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v4f16_v4i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v4f16_v4i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v4f16_v4i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: fptos_v4f16_v4i8:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_v4f16_v4i8:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = fptosi <4 x half> %a to <4 x i8>
   ret <4 x i8> %c
 }
 
 define <4 x i8> @fptou_v4f16_v4i8(<4 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v4f16_v4i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v4f16_v4i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v4f16_v4i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v4f16_v4i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v4f16_v4i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v4f16_v4i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v4f16_v4i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v4f16_v4i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <4 x half> %a to <4 x i8>
   ret <4 x i8> %c
 }
 
 define <8 x i8> @fptos_v8f16_v8i8(<8 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v8f16_v8i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NOFP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v8f16_v8i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v8f16_v8i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v8f16_v8i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v8f16_v8i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptos_v8f16_v8i8:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v8f16_v8i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = fptosi <8 x half> %a to <8 x i8>
   ret <8 x i8> %c
 }
 
 define <8 x i8> @fptou_v8f16_v8i8(<8 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v8f16_v8i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v1.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-SD-NOFP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v8f16_v8i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v8f16_v8i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v1.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v8f16_v8i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v8f16_v8i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v1.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-NOFP16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: fptou_v8f16_v8i8:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v8f16_v8i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v1.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = fptoui <8 x half> %a to <8 x i8>
   ret <8 x i8> %c
 }
 
 define <16 x i8> @fptos_v16f16_v16i8(<16 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v16f16_v16i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v1.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v16f16_v16i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v16f16_v16i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v16f16_v16i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v16f16_v16i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v16f16_v16i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v16f16_v16i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v16f16_v16i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <16 x half> %a to <16 x i8>
   ret <16 x i8> %c
 }
 
 define <16 x i8> @fptou_v16f16_v16i8(<16 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v16f16_v16i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v2.4s, v1.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v3.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v16f16_v16i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v16f16_v16i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v2.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v3.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v16f16_v16i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v16f16_v16i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v2.4s, v1.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v3.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v1.8h, v2.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v3.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v16f16_v16i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v16f16_v16i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v3.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v16f16_v16i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <16 x half> %a to <16 x i8>
   ret <16 x i8> %c
 }
 
 define <32 x i8> @fptos_v32f16_v32i8(<32 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptos_v32f16_v32i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v1.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v6.4s, v3.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v7.4s, v2.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v4.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v5.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v6.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v7.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptos_v32f16_v32i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v3.8h, v3.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v2.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-FP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v32f16_v32i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v5.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v5.4s, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v6.4s, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v7.4s, v7.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v4.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v5.8h, v1.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v2.8h, v6.8h, v2.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v3.8h, v7.8h, v3.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v32f16_v32i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v2.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v3.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-FP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v32f16_v32i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v5.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v6.4s, v3.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v7.4s, v2.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v1.8h, v4.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v5.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v3.8h, v3.8h, v6.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v7.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v32f16_v32i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    fcvtzs v3.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    fcvtzs v2.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v32f16_v32i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v6.4s, v2.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v7.4s, v3.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v5.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v6.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzs v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v4.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v5.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.8h, v6.8h, v2.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.8h, v7.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v32f16_v32i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v2.8h, v2.8h
+; CHECK-FP16-GI-NEXT:    fcvtzs v3.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <32 x half> %a to <32 x i8>
   ret <32 x i8> %c
 }
 
 define <32 x i8> @fptou_v32f16_v32i8(<32 x half> %a) {
-; CHECK-SD-NOFP16-LABEL: fptou_v32f16_v32i8:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v4.4s, v1.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v1.4s, v1.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v5.4s, v0.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v6.4s, v3.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v3.4s, v3.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtl2 v7.4s, v2.8h
-; CHECK-SD-NOFP16-NEXT:    fcvtl v2.4s, v2.4h
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v4.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v5.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v6.4s, v6.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v7.4s, v7.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.8h, v1.8h, v4.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.8h, v0.8h, v5.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v3.8h, v3.8h, v6.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v2.8h, v2.8h, v7.8h
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-NOFP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: fptou_v32f16_v32i8:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v3.8h, v3.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v2.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-SD-FP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v32f16_v32i8:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fcvtl v4.4s, v0.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v5.4s, v1.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v6.4s, v2.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v2.4s, v2.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtl v7.4s, v3.4h
-; CHECK-GI-NOFP16-NEXT:    fcvtl2 v3.4s, v3.8h
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v5.4s, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v6.4s, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v7.4s, v7.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.8h, v4.8h, v0.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.8h, v5.8h, v1.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v2.8h, v6.8h, v2.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v3.8h, v7.8h, v3.8h
-; CHECK-GI-NOFP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-NOFP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v32f16_v32i8:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v2.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v3.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-FP16-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v32f16_v32i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v4.4s, v1.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v1.4s, v1.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v5.4s, v0.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v6.4s, v3.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v3.4s, v3.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtl2 v7.4s, v2.8h
+; CHECK-NOFP16-SD-NEXT:    fcvtl v2.4s, v2.4h
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.8h, v1.8h, v4.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.8h, v0.8h, v5.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v3.8h, v3.8h, v6.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v2.8h, v2.8h, v7.8h
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v32f16_v32i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    fcvtzu v3.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    fcvtzu v2.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v32f16_v32i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fcvtl v4.4s, v0.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v5.4s, v1.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v1.4s, v1.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v6.4s, v2.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v2.4s, v2.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtl v7.4s, v3.4h
+; CHECK-NOFP16-GI-NEXT:    fcvtl2 v3.4s, v3.8h
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v5.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v6.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtzu v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.8h, v4.8h, v0.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.8h, v5.8h, v1.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v2.8h, v6.8h, v2.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v3.8h, v7.8h, v3.8h
+; CHECK-NOFP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v32f16_v32i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-GI-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    fcvtzu v2.8h, v2.8h
+; CHECK-FP16-GI-NEXT:    fcvtzu v3.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    uzp1 v1.16b, v2.16b, v3.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <32 x half> %a to <32 x i8>
   ret <32 x i8> %c
 }
 
 define <2 x i128> @fptos_v2f16_v2i128(<2 x half> %a) {
-; CHECK-SD-LABEL: fptos_v2f16_v2i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
-; CHECK-SD-NEXT:    bl __fixhfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    mov h0, v0.h[1]
-; CHECK-SD-NEXT:    bl __fixhfti
-; CHECK-SD-NEXT:    mov x2, x0
-; CHECK-SD-NEXT:    mov x3, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v2f16_v2i128:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-GI-NOFP16-NEXT:    fcvtzs x0, s0
-; CHECK-GI-NOFP16-NEXT:    fcvtzs x2, s1
-; CHECK-GI-NOFP16-NEXT:    asr x1, x0, #63
-; CHECK-GI-NOFP16-NEXT:    asr x3, x2, #63
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v2f16_v2i128:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    fcvtzs x0, h0
-; CHECK-GI-FP16-NEXT:    fcvtzs x2, h1
-; CHECK-GI-FP16-NEXT:    asr x1, x0, #63
-; CHECK-GI-FP16-NEXT:    asr x3, x2, #63
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f16_v2i128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    bl __fixhfti
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov x19, x0
+; CHECK-NOFP16-SD-NEXT:    mov x20, x1
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    bl __fixhfti
+; CHECK-NOFP16-SD-NEXT:    mov x2, x0
+; CHECK-NOFP16-SD-NEXT:    mov x3, x1
+; CHECK-NOFP16-SD-NEXT:    mov x0, x19
+; CHECK-NOFP16-SD-NEXT:    mov x1, x20
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f16_v2i128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixhfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-FP16-SD-NEXT:    bl __fixhfti
+; CHECK-FP16-SD-NEXT:    mov x2, x0
+; CHECK-FP16-SD-NEXT:    mov x3, x1
+; CHECK-FP16-SD-NEXT:    mov x0, x19
+; CHECK-FP16-SD-NEXT:    mov x1, x20
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f16_v2i128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-GI-NEXT:    fcvtzs x0, s0
+; CHECK-NOFP16-GI-NEXT:    fcvtzs x2, s1
+; CHECK-NOFP16-GI-NEXT:    asr x1, x0, #63
+; CHECK-NOFP16-GI-NEXT:    asr x3, x2, #63
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f16_v2i128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvtzs x0, h0
+; CHECK-FP16-GI-NEXT:    fcvtzs x2, h1
+; CHECK-FP16-GI-NEXT:    asr x1, x0, #63
+; CHECK-FP16-GI-NEXT:    asr x3, x2, #63
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x half> %a to <2 x i128>
   ret <2 x i128> %c
 }
 
 define <2 x i128> @fptou_v2f16_v2i128(<2 x half> %a) {
-; CHECK-SD-LABEL: fptou_v2f16_v2i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
-; CHECK-SD-NEXT:    bl __fixunshfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    mov h0, v0.h[1]
-; CHECK-SD-NEXT:    bl __fixunshfti
-; CHECK-SD-NEXT:    mov x2, x0
-; CHECK-SD-NEXT:    mov x3, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v2f16_v2i128:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-NOFP16-NEXT:    mov x1, xzr
-; CHECK-GI-NOFP16-NEXT:    mov x3, xzr
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-GI-NOFP16-NEXT:    fcvtzu x0, s0
-; CHECK-GI-NOFP16-NEXT:    fcvtzu x2, s1
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v2f16_v2i128:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    fcvtzu x0, h0
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
-; CHECK-GI-FP16-NEXT:    mov x3, xzr
-; CHECK-GI-FP16-NEXT:    fcvtzu x2, h1
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f16_v2i128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    bl __fixunshfti
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov x19, x0
+; CHECK-NOFP16-SD-NEXT:    mov x20, x1
+; CHECK-NOFP16-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-NOFP16-SD-NEXT:    bl __fixunshfti
+; CHECK-NOFP16-SD-NEXT:    mov x2, x0
+; CHECK-NOFP16-SD-NEXT:    mov x3, x1
+; CHECK-NOFP16-SD-NEXT:    mov x0, x19
+; CHECK-NOFP16-SD-NEXT:    mov x1, x20
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f16_v2i128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
+; CHECK-FP16-SD-NEXT:    bl __fixunshfti
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x19, x0
+; CHECK-FP16-SD-NEXT:    mov x20, x1
+; CHECK-FP16-SD-NEXT:    mov h0, v0.h[1]
+; CHECK-FP16-SD-NEXT:    bl __fixunshfti
+; CHECK-FP16-SD-NEXT:    mov x2, x0
+; CHECK-FP16-SD-NEXT:    mov x3, x1
+; CHECK-FP16-SD-NEXT:    mov x0, x19
+; CHECK-FP16-SD-NEXT:    mov x1, x20
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f16_v2i128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-NOFP16-GI-NEXT:    mov x1, xzr
+; CHECK-NOFP16-GI-NEXT:    mov x3, xzr
+; CHECK-NOFP16-GI-NEXT:    fcvt s0, h0
+; CHECK-NOFP16-GI-NEXT:    fcvt s1, h1
+; CHECK-NOFP16-GI-NEXT:    fcvtzu x0, s0
+; CHECK-NOFP16-GI-NEXT:    fcvtzu x2, s1
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f16_v2i128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fcvtzu x0, h0
+; CHECK-FP16-GI-NEXT:    mov x1, xzr
+; CHECK-FP16-GI-NEXT:    mov x3, xzr
+; CHECK-FP16-GI-NEXT:    fcvtzu x2, h1
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x half> %a to <2 x i128>
   ret <2 x i128> %c
 }
 
-define <3 x i128> @fptos_v3f16_v3i128(<3 x half> %a) {
-; CHECK-SD-LABEL: fptos_v3f16_v3i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
-; CHECK-SD-NEXT:    bl __fixhfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    mov h0, v0.h[1]
-; CHECK-SD-NEXT:    bl __fixhfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x21, x0
-; CHECK-SD-NEXT:    mov x22, x1
-; CHECK-SD-NEXT:    mov h0, v0.h[2]
-; CHECK-SD-NEXT:    bl __fixhfti
-; CHECK-SD-NEXT:    mov x4, x0
-; CHECK-SD-NEXT:    mov x5, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    mov x2, x21
-; CHECK-SD-NEXT:    mov x3, x22
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptos_v3f16_v3i128:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-NOFP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-GI-NOFP16-NEXT:    fcvtzs x0, s0
-; CHECK-GI-NOFP16-NEXT:    fcvtzs x2, s1
-; CHECK-GI-NOFP16-NEXT:    fcvtzs x4, s2
-; CHECK-GI-NOFP16-NEXT:    asr x1, x0, #63
-; CHECK-GI-NOFP16-NEXT:    asr x3, x2, #63
-; CHECK-GI-NOFP16-NEXT:    asr x5, x4, #63
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptos_v3f16_v3i128:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    fcvtzs x0, h0
-; CHECK-GI-FP16-NEXT:    fcvtzs x2, h1
-; CHECK-GI-FP16-NEXT:    fcvtzs x4, h2
-; CHECK-GI-FP16-NEXT:    asr x1, x0, #63
-; CHECK-GI-FP16-NEXT:    asr x3, x2, #63
-; CHECK-GI-FP16-NEXT:    asr x5, x4, #63
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = fptosi <3 x half> %a to <3 x i128>
-  ret <3 x i128> %c
-}
-
-define <3 x i128> @fptou_v3f16_v3i128(<3 x half> %a) {
-; CHECK-SD-LABEL: fptou_v3f16_v3i128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $h0 killed $h0 killed $q0
-; CHECK-SD-NEXT:    bl __fixunshfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x19, x0
-; CHECK-SD-NEXT:    mov x20, x1
-; CHECK-SD-NEXT:    mov h0, v0.h[1]
-; CHECK-SD-NEXT:    bl __fixunshfti
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x21, x0
-; CHECK-SD-NEXT:    mov x22, x1
-; CHECK-SD-NEXT:    mov h0, v0.h[2]
-; CHECK-SD-NEXT:    bl __fixunshfti
-; CHECK-SD-NEXT:    mov x4, x0
-; CHECK-SD-NEXT:    mov x5, x1
-; CHECK-SD-NEXT:    mov x0, x19
-; CHECK-SD-NEXT:    mov x1, x20
-; CHECK-SD-NEXT:    mov x2, x21
-; CHECK-SD-NEXT:    mov x3, x22
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: fptou_v3f16_v3i128:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NOFP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-NOFP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-NOFP16-NEXT:    mov x1, xzr
-; CHECK-GI-NOFP16-NEXT:    fcvt s0, h0
-; CHECK-GI-NOFP16-NEXT:    mov x3, xzr
-; CHECK-GI-NOFP16-NEXT:    mov x5, xzr
-; CHECK-GI-NOFP16-NEXT:    fcvt s1, h1
-; CHECK-GI-NOFP16-NEXT:    fcvt s2, h2
-; CHECK-GI-NOFP16-NEXT:    fcvtzu x0, s0
-; CHECK-GI-NOFP16-NEXT:    fcvtzu x2, s1
-; CHECK-GI-NOFP16-NEXT:    fcvtzu x4, s2
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: fptou_v3f16_v3i128:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
-; CHECK-GI-FP16-NEXT:    fcvtzu x0, h0
-; CHECK-GI-FP16-NEXT:    mov x3, xzr
-; CHECK-GI-FP16-NEXT:    mov x5, xzr
-; CHECK-GI-FP16-NEXT:    fcvtzu x2, h1
-; CHECK-GI-FP16-NEXT:    fcvtzu x4, h2
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = fptoui <3 x half> %a to <3 x i128>
-  ret <3 x i128> %c
-}
-
 define <2 x i64> @fptos_v2f128_v2i64(<2 x fp128> %a) {
-; CHECK-SD-LABEL: fptos_v2f128_v2i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    bl __fixtfdi
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfdi
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v2f128_v2i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixtfdi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    bl __fixtfdi
-; CHECK-GI-NEXT:    fmov d0, x19
-; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.d[1], x0
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f128_v2i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    mov v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfdi
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfdi
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x0
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f128_v2i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov v0.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    bl __fixtfdi
+; CHECK-FP16-SD-NEXT:    fmov d0, x0
+; CHECK-FP16-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    bl __fixtfdi
+; CHECK-FP16-SD-NEXT:    fmov d0, x0
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f128_v2i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __fixtfdi
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov x19, x0
+; CHECK-NOFP16-GI-NEXT:    bl __fixtfdi
+; CHECK-NOFP16-GI-NEXT:    fmov d0, x19
+; CHECK-NOFP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], x0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f128_v2i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __fixtfdi
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    bl __fixtfdi
+; CHECK-FP16-GI-NEXT:    fmov d0, x19
+; CHECK-FP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x0
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x fp128> %a to <2 x i64>
   ret <2 x i64> %c
 }
 
 define <2 x i64> @fptou_v2f128_v2i64(<2 x fp128> %a) {
-; CHECK-SD-LABEL: fptou_v2f128_v2i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    bl __fixunstfdi
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixunstfdi
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f128_v2i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixunstfdi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    bl __fixunstfdi
-; CHECK-GI-NEXT:    fmov d0, x19
-; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.d[1], x0
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f128_v2i64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    mov v0.16b, v1.16b
+; CHECK-NOFP16-SD-NEXT:    bl __fixunstfdi
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    bl __fixunstfdi
+; CHECK-NOFP16-SD-NEXT:    fmov d0, x0
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f128_v2i64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov v0.16b, v1.16b
+; CHECK-FP16-SD-NEXT:    bl __fixunstfdi
+; CHECK-FP16-SD-NEXT:    fmov d0, x0
+; CHECK-FP16-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    bl __fixunstfdi
+; CHECK-FP16-SD-NEXT:    fmov d0, x0
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f128_v2i64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfdi
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov x19, x0
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfdi
+; CHECK-NOFP16-GI-NEXT:    fmov d0, x19
+; CHECK-NOFP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], x0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f128_v2i64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __fixunstfdi
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov x19, x0
+; CHECK-FP16-GI-NEXT:    bl __fixunstfdi
+; CHECK-FP16-GI-NEXT:    fmov d0, x19
+; CHECK-FP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], x0
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x fp128> %a to <2 x i64>
   ret <2 x i64> %c
 }
 
-define <3 x i64> @fptos_v3f128_v3i64(<3 x fp128> %a) {
-; CHECK-SD-LABEL: fptos_v3f128_v3i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str d8, [sp, #48] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #56] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w30, -8
-; CHECK-SD-NEXT:    .cfi_offset b8, -16
-; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NEXT:    bl __fixtfdi
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfdi
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d8, x0
-; CHECK-SD-NEXT:    bl __fixtfdi
-; CHECK-SD-NEXT:    fmov d0, d8
-; CHECK-SD-NEXT:    ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #56] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldr d8, [sp, #48] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d1, x0
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f128_v3i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixtfdi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    bl __fixtfdi
-; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov x20, x0
-; CHECK-GI-NEXT:    bl __fixtfdi
-; CHECK-GI-NEXT:    fmov d0, x19
-; CHECK-GI-NEXT:    fmov d1, x20
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    fmov d2, x0
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <3 x fp128> %a to <3 x i64>
-  ret <3 x i64> %c
-}
-
-define <3 x i64> @fptou_v3f128_v3i64(<3 x fp128> %a) {
-; CHECK-SD-LABEL: fptou_v3f128_v3i64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str d8, [sp, #48] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #56] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w30, -8
-; CHECK-SD-NEXT:    .cfi_offset b8, -16
-; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NEXT:    bl __fixunstfdi
-; CHECK-SD-NEXT:    fmov d0, x0
-; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixunstfdi
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d8, x0
-; CHECK-SD-NEXT:    bl __fixunstfdi
-; CHECK-SD-NEXT:    fmov d0, d8
-; CHECK-SD-NEXT:    ldr q2, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #56] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldr d8, [sp, #48] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d1, x0
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f128_v3i64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixunstfdi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov x19, x0
-; CHECK-GI-NEXT:    bl __fixunstfdi
-; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov x20, x0
-; CHECK-GI-NEXT:    bl __fixunstfdi
-; CHECK-GI-NEXT:    fmov d0, x19
-; CHECK-GI-NEXT:    fmov d1, x20
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    fmov d2, x0
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <3 x fp128> %a to <3 x i64>
-  ret <3 x i64> %c
-}
-
 define <2 x i32> @fptos_v2f128_v2i32(<2 x fp128> %a) {
-; CHECK-SD-LABEL: fptos_v2f128_v2i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v2f128_v2i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[1], w0
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f128_v2i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f128_v2i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f128_v2i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov w19, w0
+; CHECK-NOFP16-GI-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w19
+; CHECK-NOFP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f128_v2i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __fixtfsi
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov w19, w0
+; CHECK-FP16-GI-NEXT:    bl __fixtfsi
+; CHECK-FP16-GI-NEXT:    fmov s0, w19
+; CHECK-FP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x fp128> %a to <2 x i32>
   ret <2 x i32> %c
 }
 
 define <2 x i32> @fptou_v2f128_v2i32(<2 x fp128> %a) {
-; CHECK-SD-LABEL: fptou_v2f128_v2i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __fixunstfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixunstfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f128_v2i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[1], w0
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f128_v2i32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f128_v2i32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __fixunstfsi
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    bl __fixunstfsi
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f128_v2i32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov w19, w0
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w19
+; CHECK-NOFP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f128_v2i32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov w19, w0
+; CHECK-FP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-FP16-GI-NEXT:    fmov s0, w19
+; CHECK-FP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x fp128> %a to <2 x i32>
   ret <2 x i32> %c
 }
 
-define <3 x i32> @fptos_v3f128_v3i32(<3 x fp128> %a) {
-; CHECK-SD-LABEL: fptos_v3f128_v3i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[2], w0
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f128_v3i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w20, w0
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[1], w20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[2], w0
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <3 x fp128> %a to <3 x i32>
-  ret <3 x i32> %c
-}
-
-define <3 x i32> @fptou_v3f128_v3i32(<3 x fp128> %a) {
-; CHECK-SD-LABEL: fptou_v3f128_v3i32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    bl __fixunstfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixunstfsi
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixunstfsi
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[2], w0
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f128_v3i32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w20, w0
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[1], w20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[2], w0
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <3 x fp128> %a to <3 x i32>
-  ret <3 x i32> %c
-}
-
 define <2 x i16> @fptos_v2f128_v2i16(<2 x fp128> %a) {
-; CHECK-SD-LABEL: fptos_v2f128_v2i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v2f128_v2i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[1], w0
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f128_v2i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f128_v2i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f128_v2i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov w19, w0
+; CHECK-NOFP16-GI-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w19
+; CHECK-NOFP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f128_v2i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __fixtfsi
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov w19, w0
+; CHECK-FP16-GI-NEXT:    bl __fixtfsi
+; CHECK-FP16-GI-NEXT:    fmov s0, w19
+; CHECK-FP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x fp128> %a to <2 x i16>
   ret <2 x i16> %c
 }
 
 define <2 x i16> @fptou_v2f128_v2i16(<2 x fp128> %a) {
-; CHECK-SD-LABEL: fptou_v2f128_v2i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f128_v2i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[1], w0
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f128_v2i16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f128_v2i16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f128_v2i16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov w19, w0
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w19
+; CHECK-NOFP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f128_v2i16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov w19, w0
+; CHECK-FP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-FP16-GI-NEXT:    fmov s0, w19
+; CHECK-FP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x fp128> %a to <2 x i16>
   ret <2 x i16> %c
 }
 
-define <3 x i16> @fptos_v3f128_v3i16(<3 x fp128> %a) {
-; CHECK-SD-LABEL: fptos_v3f128_v3i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str d8, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -8
-; CHECK-SD-NEXT:    .cfi_offset b8, -16
-; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov s8, w0
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    uzp1 v0.4h, v0.4h, v8.4h
-; CHECK-SD-NEXT:    ldr d8, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f128_v3i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w20, w0
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.h[1], w20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.h[2], w0
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <3 x fp128> %a to <3 x i16>
-  ret <3 x i16> %c
-}
-
-define <3 x i16> @fptou_v3f128_v3i16(<3 x fp128> %a) {
-; CHECK-SD-LABEL: fptou_v3f128_v3i16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str d8, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -8
-; CHECK-SD-NEXT:    .cfi_offset b8, -16
-; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov s8, w0
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    uzp1 v0.4h, v0.4h, v8.4h
-; CHECK-SD-NEXT:    ldr d8, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f128_v3i16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w20, w0
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.h[1], w20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.h[2], w0
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <3 x fp128> %a to <3 x i16>
-  ret <3 x i16> %c
-}
-
 define <2 x i8> @fptos_v2f128_v2i8(<2 x fp128> %a) {
-; CHECK-SD-LABEL: fptos_v2f128_v2i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v2f128_v2i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[1], w0
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptos_v2f128_v2i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptos_v2f128_v2i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptos_v2f128_v2i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov w19, w0
+; CHECK-NOFP16-GI-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w19
+; CHECK-NOFP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptos_v2f128_v2i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __fixtfsi
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov w19, w0
+; CHECK-FP16-GI-NEXT:    bl __fixtfsi
+; CHECK-FP16-GI-NEXT:    fmov s0, w19
+; CHECK-FP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptosi <2 x fp128> %a to <2 x i8>
   ret <2 x i8> %c
 }
 
 define <2 x i8> @fptou_v2f128_v2i8(<2 x fp128> %a) {
-; CHECK-SD-LABEL: fptou_v2f128_v2i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v2f128_v2i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    fmov s0, w19
-; CHECK-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v0.s[1], w0
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: fptou_v2f128_v2i8:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    bl __fixtfsi
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: fptou_v2f128_v2i8:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    bl __fixtfsi
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: fptou_v2f128_v2i8:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov w19, w0
+; CHECK-NOFP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w19
+; CHECK-NOFP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: fptou_v2f128_v2i8:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov w19, w0
+; CHECK-FP16-GI-NEXT:    bl __fixunstfsi
+; CHECK-FP16-GI-NEXT:    fmov s0, w19
+; CHECK-FP16-GI-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], w0
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = fptoui <2 x fp128> %a to <2 x i8>
   ret <2 x i8> %c
 }
 
-define <3 x i8> @fptos_v3f128_v3i8(<3 x fp128> %a) {
-; CHECK-SD-LABEL: fptos_v3f128_v3i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str d8, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -8
-; CHECK-SD-NEXT:    .cfi_offset b8, -16
-; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov s8, w0
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    uzp1 v0.4h, v0.4h, v8.4h
-; CHECK-SD-NEXT:    ldr d8, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptos_v3f128_v3i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w20, w0
-; CHECK-GI-NEXT:    bl __fixtfsi
-; CHECK-GI-NEXT:    mov w2, w0
-; CHECK-GI-NEXT:    mov w0, w19
-; CHECK-GI-NEXT:    mov w1, w20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptosi <3 x fp128> %a to <3 x i8>
-  ret <3 x i8> %c
-}
-
-define <3 x i8> @fptou_v3f128_v3i8(<3 x fp128> %a) {
-; CHECK-SD-LABEL: fptou_v3f128_v3i8:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str d8, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #40] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -8
-; CHECK-SD-NEXT:    .cfi_offset b8, -16
-; CHECK-SD-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
-; CHECK-SD-NEXT:    mov v0.16b, v2.16b
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov s8, w0
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    bl __fixtfsi
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #40] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], w0
-; CHECK-SD-NEXT:    uzp1 v0.4h, v0.4h, v8.4h
-; CHECK-SD-NEXT:    ldr d8, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    umov w0, v0.h[0]
-; CHECK-SD-NEXT:    umov w1, v0.h[1]
-; CHECK-SD-NEXT:    umov w2, v0.h[2]
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: fptou_v3f128_v3i8:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w19, w0
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov w20, w0
-; CHECK-GI-NEXT:    bl __fixunstfsi
-; CHECK-GI-NEXT:    mov w2, w0
-; CHECK-GI-NEXT:    mov w0, w19
-; CHECK-GI-NEXT:    mov w1, w20
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = fptoui <3 x fp128> %a to <3 x i8>
-  ret <3 x i8> %c
-}
-
 define <2 x i128> @fptos_v2f128_v2i128(<2 x fp128> %a) {
 ; CHECK-LABEL: fptos_v2f128_v2i128:
 ; CHECK:       // %bb.0: // %entry
@@ -8045,81 +7451,3 @@ entry:
   %c = fptoui <2 x fp128> %a to <2 x i128>
   ret <2 x i128> %c
 }
-
-define <3 x i128> @fptos_v3f128_v3i128(<3 x fp128> %a) {
-; CHECK-LABEL: fptos_v3f128_v3i128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #80
-; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w21, -24
-; CHECK-NEXT:    .cfi_offset w22, -32
-; CHECK-NEXT:    .cfi_offset w30, -48
-; CHECK-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-NEXT:    bl __fixtfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    bl __fixtfti
-; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x21, x0
-; CHECK-NEXT:    mov x22, x1
-; CHECK-NEXT:    bl __fixtfti
-; CHECK-NEXT:    mov x4, x0
-; CHECK-NEXT:    mov x5, x1
-; CHECK-NEXT:    mov x0, x19
-; CHECK-NEXT:    mov x1, x20
-; CHECK-NEXT:    mov x2, x21
-; CHECK-NEXT:    mov x3, x22
-; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #80
-; CHECK-NEXT:    ret
-entry:
-  %c = fptosi <3 x fp128> %a to <3 x i128>
-  ret <3 x i128> %c
-}
-
-define <3 x i128> @fptou_v3f128_v3i128(<3 x fp128> %a) {
-; CHECK-LABEL: fptou_v3f128_v3i128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #80
-; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w21, -24
-; CHECK-NEXT:    .cfi_offset w22, -32
-; CHECK-NEXT:    .cfi_offset w30, -48
-; CHECK-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
-; CHECK-NEXT:    bl __fixunstfti
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x19, x0
-; CHECK-NEXT:    mov x20, x1
-; CHECK-NEXT:    bl __fixunstfti
-; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    mov x21, x0
-; CHECK-NEXT:    mov x22, x1
-; CHECK-NEXT:    bl __fixunstfti
-; CHECK-NEXT:    mov x4, x0
-; CHECK-NEXT:    mov x5, x1
-; CHECK-NEXT:    mov x0, x19
-; CHECK-NEXT:    mov x1, x20
-; CHECK-NEXT:    mov x2, x21
-; CHECK-NEXT:    mov x3, x22
-; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #80
-; CHECK-NEXT:    ret
-entry:
-  %c = fptoui <3 x fp128> %a to <3 x i128>
-  ret <3 x i128> %c
-}
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
index 39e2db3a52d2c..e3aef487890f9 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
@@ -24,7 +24,7 @@ define i1 @test_signed_i1_f32(float %f) nounwind {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    fcvtzs w8, s0
 ; CHECK-SD-NEXT:    ands w8, w8, w8, asr #31
-; CHECK-SD-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-SD-NEXT:    csinv w8, w8, wzr, pl
 ; CHECK-SD-NEXT:    and w0, w8, #0x1
 ; CHECK-SD-NEXT:    ret
 ;
@@ -32,9 +32,9 @@ define i1 @test_signed_i1_f32(float %f) nounwind {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    fcvtzs w8, s0
 ; CHECK-GI-NEXT:    cmp w8, #0
-; CHECK-GI-NEXT:    csel w8, w8, wzr, lt
+; CHECK-GI-NEXT:    csel w8, w8, wzr, mi
 ; CHECK-GI-NEXT:    cmp w8, #0
-; CHECK-GI-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-GI-NEXT:    csinv w8, w8, wzr, pl
 ; CHECK-GI-NEXT:    and w0, w8, #0x1
 ; CHECK-GI-NEXT:    ret
     %x = call i1 @llvm.fptosi.sat.i1.f32(float %f)
@@ -269,7 +269,7 @@ define i1 @test_signed_i1_f64(double %f) nounwind {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    fcvtzs w8, d0
 ; CHECK-SD-NEXT:    ands w8, w8, w8, asr #31
-; CHECK-SD-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-SD-NEXT:    csinv w8, w8, wzr, pl
 ; CHECK-SD-NEXT:    and w0, w8, #0x1
 ; CHECK-SD-NEXT:    ret
 ;
@@ -277,9 +277,9 @@ define i1 @test_signed_i1_f64(double %f) nounwind {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    fcvtzs w8, d0
 ; CHECK-GI-NEXT:    cmp w8, #0
-; CHECK-GI-NEXT:    csel w8, w8, wzr, lt
+; CHECK-GI-NEXT:    csel w8, w8, wzr, mi
 ; CHECK-GI-NEXT:    cmp w8, #0
-; CHECK-GI-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-GI-NEXT:    csinv w8, w8, wzr, pl
 ; CHECK-GI-NEXT:    and w0, w8, #0x1
 ; CHECK-GI-NEXT:    ret
     %x = call i1 @llvm.fptosi.sat.i1.f64(double %f)
@@ -519,7 +519,7 @@ define i1 @test_signed_i1_f16(half %f) nounwind {
 ; CHECK-SD-CVT-NEXT:    fcvt s0, h0
 ; CHECK-SD-CVT-NEXT:    fcvtzs w8, s0
 ; CHECK-SD-CVT-NEXT:    ands w8, w8, w8, asr #31
-; CHECK-SD-CVT-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-SD-CVT-NEXT:    csinv w8, w8, wzr, pl
 ; CHECK-SD-CVT-NEXT:    and w0, w8, #0x1
 ; CHECK-SD-CVT-NEXT:    ret
 ;
@@ -527,7 +527,7 @@ define i1 @test_signed_i1_f16(half %f) nounwind {
 ; CHECK-SD-FP16:       // %bb.0:
 ; CHECK-SD-FP16-NEXT:    fcvtzs w8, h0
 ; CHECK-SD-FP16-NEXT:    ands w8, w8, w8, asr #31
-; CHECK-SD-FP16-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-SD-FP16-NEXT:    csinv w8, w8, wzr, pl
 ; CHECK-SD-FP16-NEXT:    and w0, w8, #0x1
 ; CHECK-SD-FP16-NEXT:    ret
 ;
@@ -536,9 +536,9 @@ define i1 @test_signed_i1_f16(half %f) nounwind {
 ; CHECK-GI-CVT-NEXT:    fcvt s0, h0
 ; CHECK-GI-CVT-NEXT:    fcvtzs w8, s0
 ; CHECK-GI-CVT-NEXT:    cmp w8, #0
-; CHECK-GI-CVT-NEXT:    csel w8, w8, wzr, lt
+; CHECK-GI-CVT-NEXT:    csel w8, w8, wzr, mi
 ; CHECK-GI-CVT-NEXT:    cmp w8, #0
-; CHECK-GI-CVT-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-GI-CVT-NEXT:    csinv w8, w8, wzr, pl
 ; CHECK-GI-CVT-NEXT:    and w0, w8, #0x1
 ; CHECK-GI-CVT-NEXT:    ret
 ;
@@ -546,9 +546,9 @@ define i1 @test_signed_i1_f16(half %f) nounwind {
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    fcvtzs w8, h0
 ; CHECK-GI-FP16-NEXT:    cmp w8, #0
-; CHECK-GI-FP16-NEXT:    csel w8, w8, wzr, lt
+; CHECK-GI-FP16-NEXT:    csel w8, w8, wzr, mi
 ; CHECK-GI-FP16-NEXT:    cmp w8, #0
-; CHECK-GI-FP16-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-GI-FP16-NEXT:    csinv w8, w8, wzr, pl
 ; CHECK-GI-FP16-NEXT:    and w0, w8, #0x1
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call i1 @llvm.fptosi.sat.i1.f16(half %f)
@@ -959,7 +959,7 @@ define i32 @test_signed_f128_i32(fp128 %f) {
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    mov w8, #-2147483648 // =0x80000000
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    csel w19, w8, w0, lt
+; CHECK-SD-NEXT:    csel w19, w8, w0, mi
 ; CHECK-SD-NEXT:    adrp x8, .LCPI30_1
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI30_1]
 ; CHECK-SD-NEXT:    bl __gttf2
@@ -1001,11 +1001,11 @@ define i32 @test_signed_f128_i32(fp128 %f) {
 ; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI30_0]
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
 ; CHECK-GI-NEXT:    mov x8, #281474976448512 // =0xfffffffc0000
 ; CHECK-GI-NEXT:    movk x8, #16413, lsl #48
-; CHECK-GI-NEXT:    csel x8, x20, x8, lt
+; CHECK-GI-NEXT:    csel x8, x20, x8, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
index 9c21d2bf083a2..c74112937ba53 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-CVT
-; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
-; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-CVT
-; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-CVT,CHECK-SD-CVT
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-FP16,CHECK-SD-FP16
+; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-CVT,CHECK-GI-CVT
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-FP16,CHECK-GI-FP16
 
 ; CHECK-GI:       warning: Instruction selection used fallback path for test_signed_v4f32_v4i50
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_signed_v4f16_v4i50
@@ -499,7 +499,7 @@ define <1 x i32> @test_signed_v1f128_v1i32(<1 x fp128> %f) {
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    mov w8, #-2147483648 // =0x80000000
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    csel w19, w8, w0, lt
+; CHECK-SD-NEXT:    csel w19, w8, w0, mi
 ; CHECK-SD-NEXT:    adrp x8, .LCPI14_1
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI14_1]
 ; CHECK-SD-NEXT:    bl __gttf2
@@ -542,11 +542,11 @@ define <1 x i32> @test_signed_v1f128_v1i32(<1 x fp128> %f) {
 ; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI14_0]
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
 ; CHECK-GI-NEXT:    mov x8, #281474976448512 // =0xfffffffc0000
 ; CHECK-GI-NEXT:    movk x8, #16413, lsl #48
-; CHECK-GI-NEXT:    csel x8, x20, x8, lt
+; CHECK-GI-NEXT:    csel x8, x20, x8, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
@@ -592,7 +592,7 @@ define <2 x i32> @test_signed_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI15_1]
 ; CHECK-SD-NEXT:    mov w20, #-2147483648 // =0x80000000
-; CHECK-SD-NEXT:    csel w19, w20, w0, lt
+; CHECK-SD-NEXT:    csel w19, w20, w0, mi
 ; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
@@ -612,7 +612,7 @@ define <2 x i32> @test_signed_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, w20, w0, lt
+; CHECK-SD-NEXT:    csel w19, w20, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
@@ -664,10 +664,10 @@ define <2 x i32> @test_signed_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    mov x22, #281474976448512 // =0xfffffffc0000
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    movk x22, #16413, lsl #48
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x21, x22, lt
+; CHECK-GI-NEXT:    csel x8, x21, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
@@ -690,9 +690,9 @@ define <2 x i32> @test_signed_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x20
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x20, x22, lt
+; CHECK-GI-NEXT:    csel x8, x20, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
@@ -742,7 +742,7 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI16_1]
 ; CHECK-SD-NEXT:    mov w20, #-2147483648 // =0x80000000
-; CHECK-SD-NEXT:    csel w19, w20, w0, lt
+; CHECK-SD-NEXT:    csel w19, w20, w0, mi
 ; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
@@ -762,7 +762,7 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, w20, w0, lt
+; CHECK-SD-NEXT:    csel w19, w20, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
@@ -780,7 +780,7 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, w20, w0, lt
+; CHECK-SD-NEXT:    csel w19, w20, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
@@ -833,10 +833,10 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    mov x22, #281474976448512 // =0xfffffffc0000
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    movk x22, #16413, lsl #48
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x21, x22, lt
+; CHECK-GI-NEXT:    csel x8, x21, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
@@ -858,9 +858,9 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x23
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x23, x22, lt
+; CHECK-GI-NEXT:    csel x8, x23, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
@@ -883,9 +883,9 @@ define <3 x i32> @test_signed_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x20
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x20, x22, lt
+; CHECK-GI-NEXT:    csel x8, x20, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
@@ -936,7 +936,7 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI17_1]
 ; CHECK-SD-NEXT:    mov w20, #-2147483648 // =0x80000000
-; CHECK-SD-NEXT:    csel w19, w20, w0, lt
+; CHECK-SD-NEXT:    csel w19, w20, w0, mi
 ; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
@@ -955,7 +955,7 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, w20, w0, lt
+; CHECK-SD-NEXT:    csel w19, w20, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
@@ -976,7 +976,7 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, w20, w0, lt
+; CHECK-SD-NEXT:    csel w19, w20, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
@@ -996,7 +996,7 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, w20, w0, lt
+; CHECK-SD-NEXT:    csel w19, w20, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
@@ -1049,10 +1049,10 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    mov x22, #281474976448512 // =0xfffffffc0000
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    movk x22, #16413, lsl #48
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x21, x22, lt
+; CHECK-GI-NEXT:    csel x8, x21, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
@@ -1075,9 +1075,9 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x23
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x23, x22, lt
+; CHECK-GI-NEXT:    csel x8, x23, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
@@ -1099,9 +1099,9 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x24
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x24, x22, lt
+; CHECK-GI-NEXT:    csel x8, x24, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
@@ -1123,9 +1123,9 @@ define <4 x i32> @test_signed_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x20
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x20, x22, lt
+; CHECK-GI-NEXT:    csel x8, x20, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
@@ -1162,31 +1162,18 @@ declare <7 x i32> @llvm.fptosi.sat.v7f16.v7i32 (<7 x half>)
 declare <8 x i32> @llvm.fptosi.sat.v8f16.v8i32 (<8 x half>)
 
 define <1 x i32> @test_signed_v1f16_v1i32(<1 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_signed_v1f16_v1i32:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    fcvt s0, h0
-; CHECK-SD-CVT-NEXT:    fcvtzs w8, s0
-; CHECK-SD-CVT-NEXT:    fmov s0, w8
-; CHECK-SD-CVT-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: test_signed_v1f16_v1i32:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzs w8, h0
-; CHECK-SD-FP16-NEXT:    fmov s0, w8
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-CVT-LABEL: test_signed_v1f16_v1i32:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    fcvtzs w8, s0
-; CHECK-GI-CVT-NEXT:    fmov s0, w8
-; CHECK-GI-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_signed_v1f16_v1i32:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvtzs w8, s0
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-GI-FP16-LABEL: test_signed_v1f16_v1i32:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzs w8, h0
-; CHECK-GI-FP16-NEXT:    fmov s0, w8
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v1f16_v1i32:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs w8, h0
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    ret
     %x = call <1 x i32> @llvm.fptosi.sat.v1f16.v1i32(<1 x half> %f)
     ret <1 x i32> %x
 }
@@ -1850,21 +1837,11 @@ define <4 x i13> @test_signed_v4f32_v4i13(<4 x float> %f) {
 }
 
 define <4 x i16> @test_signed_v4f32_v4i16(<4 x float> %f) {
-; CHECK-SD-LABEL: test_signed_v4f32_v4i16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    sqxtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_signed_v4f32_v4i16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.4s, #127, msl #8
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    smin v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    mvni v1.4s, #127, msl #8
-; CHECK-GI-NEXT:    smax v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_signed_v4f32_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    sqxtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %x = call <4 x i16> @llvm.fptosi.sat.v4f32.v4i16(<4 x float> %f)
     ret <4 x i16> %x
 }
@@ -2359,9 +2336,9 @@ define <2 x i1> @test_signed_v2f64_v2i1(<2 x double> %f) {
 ; CHECK-SD-NEXT:    fcvtzs w9, d0
 ; CHECK-SD-NEXT:    fcvtzs w8, d1
 ; CHECK-SD-NEXT:    ands w8, w8, w8, asr #31
-; CHECK-SD-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-SD-NEXT:    csinv w8, w8, wzr, pl
 ; CHECK-SD-NEXT:    ands w9, w9, w9, asr #31
-; CHECK-SD-NEXT:    csinv w9, w9, wzr, ge
+; CHECK-SD-NEXT:    csinv w9, w9, wzr, pl
 ; CHECK-SD-NEXT:    fmov s0, w9
 ; CHECK-SD-NEXT:    mov v0.s[1], w8
 ; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
@@ -2905,16 +2882,16 @@ define <4 x i1> @test_signed_v4f16_v4i1(<4 x half> %f) {
 }
 
 define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_signed_v4f16_v4i8:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-CVT-NEXT:    movi v1.4s, #127
-; CHECK-SD-CVT-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-CVT-NEXT:    smin v0.4s, v0.4s, v1.4s
-; CHECK-SD-CVT-NEXT:    mvni v1.4s, #127
-; CHECK-SD-CVT-NEXT:    smax v0.4s, v0.4s, v1.4s
-; CHECK-SD-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_signed_v4f16_v4i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    movi v1.4s, #127
+; CHECK-CVT-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    mvni v1.4s, #127
+; CHECK-CVT-NEXT:    smax v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    xtn v0.4h, v0.4s
+; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-SD-FP16-LABEL: test_signed_v4f16_v4i8:
 ; CHECK-SD-FP16:       // %bb.0:
@@ -2925,17 +2902,6 @@ define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
 ; CHECK-SD-FP16-NEXT:    smax v0.4h, v0.4h, v1.4h
 ; CHECK-SD-FP16-NEXT:    ret
 ;
-; CHECK-GI-CVT-LABEL: test_signed_v4f16_v4i8:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-CVT-NEXT:    movi v1.4s, #127
-; CHECK-GI-CVT-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    smin v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    mvni v1.4s, #127
-; CHECK-GI-CVT-NEXT:    smax v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-CVT-NEXT:    ret
-;
 ; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i8:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    movi v1.4h, #127
@@ -2949,77 +2915,41 @@ define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
 }
 
 define <4 x i13> @test_signed_v4f16_v4i13(<4 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_signed_v4f16_v4i13:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-CVT-NEXT:    movi v1.4s, #15, msl #8
-; CHECK-SD-CVT-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-CVT-NEXT:    smin v0.4s, v0.4s, v1.4s
-; CHECK-SD-CVT-NEXT:    mvni v1.4s, #15, msl #8
-; CHECK-SD-CVT-NEXT:    smax v0.4s, v0.4s, v1.4s
-; CHECK-SD-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_signed_v4f16_v4i13:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    movi v1.4s, #15, msl #8
+; CHECK-CVT-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    mvni v1.4s, #15, msl #8
+; CHECK-CVT-NEXT:    smax v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    xtn v0.4h, v0.4s
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_signed_v4f16_v4i13:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    mvni v1.4h, #240, lsl #8
-; CHECK-SD-FP16-NEXT:    movi v2.4h, #240, lsl #8
-; CHECK-SD-FP16-NEXT:    smin v0.4h, v0.4h, v1.4h
-; CHECK-SD-FP16-NEXT:    smax v0.4h, v0.4h, v2.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-CVT-LABEL: test_signed_v4f16_v4i13:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-CVT-NEXT:    movi v1.4s, #15, msl #8
-; CHECK-GI-CVT-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    smin v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    mvni v1.4s, #15, msl #8
-; CHECK-GI-CVT-NEXT:    smax v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i13:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #240, lsl #8
-; CHECK-GI-FP16-NEXT:    movi v2.4h, #240, lsl #8
-; CHECK-GI-FP16-NEXT:    smin v0.4h, v0.4h, v1.4h
-; CHECK-GI-FP16-NEXT:    smax v0.4h, v0.4h, v2.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v4f16_v4i13:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-NEXT:    mvni v1.4h, #240, lsl #8
+; CHECK-FP16-NEXT:    movi v2.4h, #240, lsl #8
+; CHECK-FP16-NEXT:    smin v0.4h, v0.4h, v1.4h
+; CHECK-FP16-NEXT:    smax v0.4h, v0.4h, v2.4h
+; CHECK-FP16-NEXT:    ret
     %x = call <4 x i13> @llvm.fptosi.sat.v4f16.v4i13(<4 x half> %f)
     ret <4 x i13> %x
 }
 
 define <4 x i16> @test_signed_v4f16_v4i16(<4 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_signed_v4f16_v4i16:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-CVT-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-CVT-NEXT:    sqxtn v0.4h, v0.4s
-; CHECK-SD-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_signed_v4f16_v4i16:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-NEXT:    sqxtn v0.4h, v0.4s
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_signed_v4f16_v4i16:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-CVT-LABEL: test_signed_v4f16_v4i16:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-CVT-NEXT:    movi v1.4s, #127, msl #8
-; CHECK-GI-CVT-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    smin v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    mvni v1.4s, #127, msl #8
-; CHECK-GI-CVT-NEXT:    smax v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i16:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v4f16_v4i16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
     %x = call <4 x i16> @llvm.fptosi.sat.v4f16.v4i16(<4 x half> %f)
     ret <4 x i16> %x
 }
@@ -3049,133 +2979,69 @@ define <4 x i32> @test_signed_v4f16_v4i32_duplicate(<4 x half> %f) {
 }
 
 define <4 x i50> @test_signed_v4f16_v4i50(<4 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_signed_v4f16_v4i50:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-CVT-NEXT:    fcvt s2, h0
-; CHECK-SD-CVT-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
-; CHECK-SD-CVT-NEXT:    mov h3, v0.h[2]
-; CHECK-SD-CVT-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-CVT-NEXT:    mov x11, #-562949953421312 // =0xfffe000000000000
-; CHECK-SD-CVT-NEXT:    fcvt s1, h1
-; CHECK-SD-CVT-NEXT:    fcvtzs x9, s2
-; CHECK-SD-CVT-NEXT:    fcvt s2, h3
-; CHECK-SD-CVT-NEXT:    fcvt s0, h0
-; CHECK-SD-CVT-NEXT:    fcvtzs x10, s1
-; CHECK-SD-CVT-NEXT:    cmp x9, x8
-; CHECK-SD-CVT-NEXT:    csel x9, x9, x8, lt
-; CHECK-SD-CVT-NEXT:    fcvtzs x12, s2
-; CHECK-SD-CVT-NEXT:    cmp x9, x11
-; CHECK-SD-CVT-NEXT:    csel x0, x9, x11, gt
-; CHECK-SD-CVT-NEXT:    cmp x10, x8
-; CHECK-SD-CVT-NEXT:    csel x9, x10, x8, lt
-; CHECK-SD-CVT-NEXT:    fcvtzs x10, s0
-; CHECK-SD-CVT-NEXT:    cmp x9, x11
-; CHECK-SD-CVT-NEXT:    csel x1, x9, x11, gt
-; CHECK-SD-CVT-NEXT:    cmp x12, x8
-; CHECK-SD-CVT-NEXT:    csel x9, x12, x8, lt
-; CHECK-SD-CVT-NEXT:    cmp x9, x11
-; CHECK-SD-CVT-NEXT:    csel x2, x9, x11, gt
-; CHECK-SD-CVT-NEXT:    cmp x10, x8
-; CHECK-SD-CVT-NEXT:    csel x8, x10, x8, lt
-; CHECK-SD-CVT-NEXT:    cmp x8, x11
-; CHECK-SD-CVT-NEXT:    csel x3, x8, x11, gt
-; CHECK-SD-CVT-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: test_signed_v4f16_v4i50:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzs x9, h0
-; CHECK-SD-FP16-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
-; CHECK-SD-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-SD-FP16-NEXT:    mov x11, #-562949953421312 // =0xfffe000000000000
-; CHECK-SD-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h1
-; CHECK-SD-FP16-NEXT:    cmp x9, x8
-; CHECK-SD-FP16-NEXT:    csel x9, x9, x8, lt
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h2
-; CHECK-SD-FP16-NEXT:    cmp x9, x11
-; CHECK-SD-FP16-NEXT:    csel x0, x9, x11, gt
-; CHECK-SD-FP16-NEXT:    cmp x10, x8
-; CHECK-SD-FP16-NEXT:    csel x9, x10, x8, lt
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h0
-; CHECK-SD-FP16-NEXT:    cmp x9, x11
-; CHECK-SD-FP16-NEXT:    csel x1, x9, x11, gt
-; CHECK-SD-FP16-NEXT:    cmp x12, x8
-; CHECK-SD-FP16-NEXT:    csel x9, x12, x8, lt
-; CHECK-SD-FP16-NEXT:    cmp x9, x11
-; CHECK-SD-FP16-NEXT:    csel x2, x9, x11, gt
-; CHECK-SD-FP16-NEXT:    cmp x10, x8
-; CHECK-SD-FP16-NEXT:    csel x8, x10, x8, lt
-; CHECK-SD-FP16-NEXT:    cmp x8, x11
-; CHECK-SD-FP16-NEXT:    csel x3, x8, x11, gt
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-CVT-LABEL: test_signed_v4f16_v4i50:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-CVT-NEXT:    fcvt s2, h0
-; CHECK-GI-CVT-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
-; CHECK-GI-CVT-NEXT:    mov h3, v0.h[2]
-; CHECK-GI-CVT-NEXT:    mov h0, v0.h[3]
-; CHECK-GI-CVT-NEXT:    mov x11, #-562949953421312 // =0xfffe000000000000
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
-; CHECK-GI-CVT-NEXT:    fcvtzs x9, s2
-; CHECK-GI-CVT-NEXT:    fcvt s2, h3
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    fcvtzs x10, s1
-; CHECK-GI-CVT-NEXT:    cmp x9, x8
-; CHECK-GI-CVT-NEXT:    csel x9, x9, x8, lt
-; CHECK-GI-CVT-NEXT:    fcvtzs x12, s2
-; CHECK-GI-CVT-NEXT:    cmp x9, x11
-; CHECK-GI-CVT-NEXT:    csel x0, x9, x11, gt
-; CHECK-GI-CVT-NEXT:    cmp x10, x8
-; CHECK-GI-CVT-NEXT:    csel x9, x10, x8, lt
-; CHECK-GI-CVT-NEXT:    fcvtzs x10, s0
-; CHECK-GI-CVT-NEXT:    cmp x9, x11
-; CHECK-GI-CVT-NEXT:    csel x1, x9, x11, gt
-; CHECK-GI-CVT-NEXT:    cmp x12, x8
-; CHECK-GI-CVT-NEXT:    csel x9, x12, x8, lt
-; CHECK-GI-CVT-NEXT:    cmp x9, x11
-; CHECK-GI-CVT-NEXT:    csel x2, x9, x11, gt
-; CHECK-GI-CVT-NEXT:    cmp x10, x8
-; CHECK-GI-CVT-NEXT:    csel x8, x10, x8, lt
-; CHECK-GI-CVT-NEXT:    cmp x8, x11
-; CHECK-GI-CVT-NEXT:    csel x3, x8, x11, gt
-; CHECK-GI-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_signed_v4f16_v4i50:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-NEXT:    fcvt s2, h0
+; CHECK-CVT-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
+; CHECK-CVT-NEXT:    mov h3, v0.h[2]
+; CHECK-CVT-NEXT:    mov h0, v0.h[3]
+; CHECK-CVT-NEXT:    mov x11, #-562949953421312 // =0xfffe000000000000
+; CHECK-CVT-NEXT:    fcvt s1, h1
+; CHECK-CVT-NEXT:    fcvtzs x9, s2
+; CHECK-CVT-NEXT:    fcvt s2, h3
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvtzs x10, s1
+; CHECK-CVT-NEXT:    cmp x9, x8
+; CHECK-CVT-NEXT:    csel x9, x9, x8, lt
+; CHECK-CVT-NEXT:    fcvtzs x12, s2
+; CHECK-CVT-NEXT:    cmp x9, x11
+; CHECK-CVT-NEXT:    csel x0, x9, x11, gt
+; CHECK-CVT-NEXT:    cmp x10, x8
+; CHECK-CVT-NEXT:    csel x9, x10, x8, lt
+; CHECK-CVT-NEXT:    fcvtzs x10, s0
+; CHECK-CVT-NEXT:    cmp x9, x11
+; CHECK-CVT-NEXT:    csel x1, x9, x11, gt
+; CHECK-CVT-NEXT:    cmp x12, x8
+; CHECK-CVT-NEXT:    csel x9, x12, x8, lt
+; CHECK-CVT-NEXT:    cmp x9, x11
+; CHECK-CVT-NEXT:    csel x2, x9, x11, gt
+; CHECK-CVT-NEXT:    cmp x10, x8
+; CHECK-CVT-NEXT:    csel x8, x10, x8, lt
+; CHECK-CVT-NEXT:    cmp x8, x11
+; CHECK-CVT-NEXT:    csel x3, x8, x11, gt
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i50:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
-; CHECK-GI-FP16-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov x11, #-562949953421312 // =0xfffe000000000000
-; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzs x10, h1
-; CHECK-GI-FP16-NEXT:    cmp x9, x8
-; CHECK-GI-FP16-NEXT:    csel x9, x9, x8, lt
-; CHECK-GI-FP16-NEXT:    fcvtzs x12, h2
-; CHECK-GI-FP16-NEXT:    cmp x9, x11
-; CHECK-GI-FP16-NEXT:    csel x0, x9, x11, gt
-; CHECK-GI-FP16-NEXT:    cmp x10, x8
-; CHECK-GI-FP16-NEXT:    csel x9, x10, x8, lt
-; CHECK-GI-FP16-NEXT:    fcvtzs x10, h0
-; CHECK-GI-FP16-NEXT:    cmp x9, x11
-; CHECK-GI-FP16-NEXT:    csel x1, x9, x11, gt
-; CHECK-GI-FP16-NEXT:    cmp x12, x8
-; CHECK-GI-FP16-NEXT:    csel x9, x12, x8, lt
-; CHECK-GI-FP16-NEXT:    cmp x9, x11
-; CHECK-GI-FP16-NEXT:    csel x2, x9, x11, gt
-; CHECK-GI-FP16-NEXT:    cmp x10, x8
-; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
-; CHECK-GI-FP16-NEXT:    cmp x8, x11
-; CHECK-GI-FP16-NEXT:    csel x3, x8, x11, gt
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v4f16_v4i50:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-FP16-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
+; CHECK-FP16-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-NEXT:    mov x11, #-562949953421312 // =0xfffe000000000000
+; CHECK-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-FP16-NEXT:    fcvtzs x10, h1
+; CHECK-FP16-NEXT:    cmp x9, x8
+; CHECK-FP16-NEXT:    csel x9, x9, x8, lt
+; CHECK-FP16-NEXT:    fcvtzs x12, h2
+; CHECK-FP16-NEXT:    cmp x9, x11
+; CHECK-FP16-NEXT:    csel x0, x9, x11, gt
+; CHECK-FP16-NEXT:    cmp x10, x8
+; CHECK-FP16-NEXT:    csel x9, x10, x8, lt
+; CHECK-FP16-NEXT:    fcvtzs x10, h0
+; CHECK-FP16-NEXT:    cmp x9, x11
+; CHECK-FP16-NEXT:    csel x1, x9, x11, gt
+; CHECK-FP16-NEXT:    cmp x12, x8
+; CHECK-FP16-NEXT:    csel x9, x12, x8, lt
+; CHECK-FP16-NEXT:    cmp x9, x11
+; CHECK-FP16-NEXT:    csel x2, x9, x11, gt
+; CHECK-FP16-NEXT:    cmp x10, x8
+; CHECK-FP16-NEXT:    csel x8, x10, x8, lt
+; CHECK-FP16-NEXT:    cmp x8, x11
+; CHECK-FP16-NEXT:    csel x3, x8, x11, gt
+; CHECK-FP16-NEXT:    ret
     %x = call <4 x i50> @llvm.fptosi.sat.v4f16.v4i50(<4 x half> %f)
     ret <4 x i50> %x
 }
@@ -3556,15 +3422,15 @@ define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_signed_v8f16_v8i1:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    movi v2.2d, #0xffffffffffffffff
-; CHECK-SD-FP16-NEXT:    smin v0.8h, v0.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    smax v0.8h, v0.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v8f16_v8i1:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-NEXT:    movi v2.2d, #0xffffffffffffffff
+; CHECK-FP16-NEXT:    smin v0.8h, v0.8h, v1.8h
+; CHECK-FP16-NEXT:    smax v0.8h, v0.8h, v2.8h
+; CHECK-FP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_signed_v8f16_v8i1:
 ; CHECK-GI-CVT:       // %bb.0:
@@ -3581,16 +3447,6 @@ define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) {
 ; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
 ; CHECK-GI-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v8f16_v8i1:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    movi v2.2d, #0xffffffffffffffff
-; CHECK-GI-FP16-NEXT:    smin v0.8h, v0.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    smax v0.8h, v0.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f)
     ret <8 x i1> %x
 }
@@ -3612,11 +3468,11 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_signed_v8f16_v8i8:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    sqxtn v0.8b, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v8f16_v8i8:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-NEXT:    sqxtn v0.8b, v0.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_signed_v8f16_v8i8:
 ; CHECK-GI-CVT:       // %bb.0:
@@ -3633,16 +3489,6 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
 ; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
 ; CHECK-GI-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v8f16_v8i8:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    movi v1.8h, #127
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    mvni v2.8h, #127
-; CHECK-GI-FP16-NEXT:    smin v0.8h, v0.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    smax v0.8h, v0.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <8 x i8> @llvm.fptosi.sat.v8f16.v8i8(<8 x half> %f)
     ret <8 x i8> %x
 }
@@ -3663,14 +3509,14 @@ define <8 x i13> @test_signed_v8f16_v8i13(<8 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_signed_v8f16_v8i13:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    mvni v1.8h, #240, lsl #8
-; CHECK-SD-FP16-NEXT:    movi v2.8h, #240, lsl #8
-; CHECK-SD-FP16-NEXT:    smin v0.8h, v0.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    smax v0.8h, v0.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v8f16_v8i13:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-NEXT:    mvni v1.8h, #240, lsl #8
+; CHECK-FP16-NEXT:    movi v2.8h, #240, lsl #8
+; CHECK-FP16-NEXT:    smin v0.8h, v0.8h, v1.8h
+; CHECK-FP16-NEXT:    smax v0.8h, v0.8h, v2.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_signed_v8f16_v8i13:
 ; CHECK-GI-CVT:       // %bb.0:
@@ -3686,15 +3532,6 @@ define <8 x i13> @test_signed_v8f16_v8i13(<8 x half> %f) {
 ; CHECK-GI-CVT-NEXT:    smax v0.4s, v0.4s, v1.4s
 ; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v8f16_v8i13:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    mvni v1.8h, #240, lsl #8
-; CHECK-GI-FP16-NEXT:    movi v2.8h, #240, lsl #8
-; CHECK-GI-FP16-NEXT:    smin v0.8h, v0.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    smax v0.8h, v0.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <8 x i13> @llvm.fptosi.sat.v8f16.v8i13(<8 x half> %f)
     ret <8 x i13> %x
 }
@@ -3710,30 +3547,20 @@ define <8 x i16> @test_signed_v8f16_v8i16(<8 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    sqxtn2 v0.8h, v1.4s
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_signed_v8f16_v8i16:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v8f16_v8i16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_signed_v8f16_v8i16:
 ; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-GI-CVT-NEXT:    fcvtl v1.4s, v0.4h
 ; CHECK-GI-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-CVT-NEXT:    movi v1.4s, #127, msl #8
-; CHECK-GI-CVT-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    smin v2.4s, v2.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    smin v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    mvni v1.4s, #127, msl #8
-; CHECK-GI-CVT-NEXT:    smax v2.4s, v2.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    smax v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-GI-CVT-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-GI-CVT-NEXT:    fcvtzs v2.4s, v0.4s
+; CHECK-GI-CVT-NEXT:    sqxtn v0.4h, v1.4s
+; CHECK-GI-CVT-NEXT:    sqxtn2 v0.8h, v2.4s
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v8f16_v8i16:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <8 x i16> @llvm.fptosi.sat.v8f16.v8i16(<8 x half> %f)
     ret <8 x i16> %x
 }
@@ -3785,233 +3612,119 @@ define <8 x i32> @test_signed_v8f16_v8i32_duplicate(<8 x half> %f) {
 }
 
 define <8 x i50> @test_signed_v8f16_v8i50(<8 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_signed_v8f16_v8i50:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-CVT-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
-; CHECK-SD-CVT-NEXT:    mov x9, #-562949953421312 // =0xfffe000000000000
-; CHECK-SD-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-CVT-NEXT:    fcvt s3, h1
-; CHECK-SD-CVT-NEXT:    mov h4, v1.h[2]
-; CHECK-SD-CVT-NEXT:    mov h1, v1.h[3]
-; CHECK-SD-CVT-NEXT:    fcvt s2, h2
-; CHECK-SD-CVT-NEXT:    fcvtzs x10, s3
-; CHECK-SD-CVT-NEXT:    fcvt s3, h4
-; CHECK-SD-CVT-NEXT:    fcvt s1, h1
-; CHECK-SD-CVT-NEXT:    fcvtzs x11, s2
-; CHECK-SD-CVT-NEXT:    cmp x10, x8
-; CHECK-SD-CVT-NEXT:    fcvtzs x12, s3
-; CHECK-SD-CVT-NEXT:    csel x10, x10, x8, lt
-; CHECK-SD-CVT-NEXT:    mov h2, v0.h[1]
-; CHECK-SD-CVT-NEXT:    fcvt s3, h0
-; CHECK-SD-CVT-NEXT:    cmp x10, x9
-; CHECK-SD-CVT-NEXT:    csel x4, x10, x9, gt
-; CHECK-SD-CVT-NEXT:    cmp x11, x8
-; CHECK-SD-CVT-NEXT:    csel x10, x11, x8, lt
-; CHECK-SD-CVT-NEXT:    fcvtzs x11, s1
-; CHECK-SD-CVT-NEXT:    mov h1, v0.h[2]
-; CHECK-SD-CVT-NEXT:    cmp x10, x9
-; CHECK-SD-CVT-NEXT:    fcvt s2, h2
-; CHECK-SD-CVT-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-CVT-NEXT:    csel x5, x10, x9, gt
-; CHECK-SD-CVT-NEXT:    cmp x12, x8
-; CHECK-SD-CVT-NEXT:    csel x10, x12, x8, lt
-; CHECK-SD-CVT-NEXT:    fcvtzs x12, s3
-; CHECK-SD-CVT-NEXT:    cmp x10, x9
-; CHECK-SD-CVT-NEXT:    fcvt s1, h1
-; CHECK-SD-CVT-NEXT:    csel x6, x10, x9, gt
-; CHECK-SD-CVT-NEXT:    cmp x11, x8
-; CHECK-SD-CVT-NEXT:    fcvt s0, h0
-; CHECK-SD-CVT-NEXT:    csel x10, x11, x8, lt
-; CHECK-SD-CVT-NEXT:    fcvtzs x11, s2
-; CHECK-SD-CVT-NEXT:    cmp x10, x9
-; CHECK-SD-CVT-NEXT:    csel x7, x10, x9, gt
-; CHECK-SD-CVT-NEXT:    cmp x12, x8
-; CHECK-SD-CVT-NEXT:    csel x10, x12, x8, lt
-; CHECK-SD-CVT-NEXT:    fcvtzs x12, s1
-; CHECK-SD-CVT-NEXT:    cmp x10, x9
-; CHECK-SD-CVT-NEXT:    csel x0, x10, x9, gt
-; CHECK-SD-CVT-NEXT:    cmp x11, x8
-; CHECK-SD-CVT-NEXT:    csel x10, x11, x8, lt
-; CHECK-SD-CVT-NEXT:    fcvtzs x11, s0
-; CHECK-SD-CVT-NEXT:    cmp x10, x9
-; CHECK-SD-CVT-NEXT:    csel x1, x10, x9, gt
-; CHECK-SD-CVT-NEXT:    cmp x12, x8
-; CHECK-SD-CVT-NEXT:    csel x10, x12, x8, lt
-; CHECK-SD-CVT-NEXT:    cmp x10, x9
-; CHECK-SD-CVT-NEXT:    csel x2, x10, x9, gt
-; CHECK-SD-CVT-NEXT:    cmp x11, x8
-; CHECK-SD-CVT-NEXT:    csel x8, x11, x8, lt
-; CHECK-SD-CVT-NEXT:    cmp x8, x9
-; CHECK-SD-CVT-NEXT:    csel x3, x8, x9, gt
-; CHECK-SD-CVT-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: test_signed_v8f16_v8i50:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-FP16-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
-; CHECK-SD-FP16-NEXT:    mov x9, #-562949953421312 // =0xfffe000000000000
-; CHECK-SD-FP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzs x10, h1
-; CHECK-SD-FP16-NEXT:    mov h3, v1.h[2]
-; CHECK-SD-FP16-NEXT:    mov h1, v1.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h2
-; CHECK-SD-FP16-NEXT:    cmp x10, x8
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h3
-; CHECK-SD-FP16-NEXT:    csel x10, x10, x8, lt
-; CHECK-SD-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-SD-FP16-NEXT:    cmp x10, x9
-; CHECK-SD-FP16-NEXT:    csel x4, x10, x9, gt
-; CHECK-SD-FP16-NEXT:    cmp x11, x8
-; CHECK-SD-FP16-NEXT:    csel x10, x11, x8, lt
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h1
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    cmp x10, x9
-; CHECK-SD-FP16-NEXT:    csel x5, x10, x9, gt
-; CHECK-SD-FP16-NEXT:    cmp x12, x8
-; CHECK-SD-FP16-NEXT:    csel x10, x12, x8, lt
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h0
-; CHECK-SD-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-SD-FP16-NEXT:    cmp x10, x9
-; CHECK-SD-FP16-NEXT:    csel x6, x10, x9, gt
-; CHECK-SD-FP16-NEXT:    cmp x11, x8
-; CHECK-SD-FP16-NEXT:    csel x10, x11, x8, lt
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h1
-; CHECK-SD-FP16-NEXT:    cmp x10, x9
-; CHECK-SD-FP16-NEXT:    csel x7, x10, x9, gt
-; CHECK-SD-FP16-NEXT:    cmp x12, x8
-; CHECK-SD-FP16-NEXT:    csel x10, x12, x8, lt
-; CHECK-SD-FP16-NEXT:    fcvtzs x12, h2
-; CHECK-SD-FP16-NEXT:    cmp x10, x9
-; CHECK-SD-FP16-NEXT:    csel x0, x10, x9, gt
-; CHECK-SD-FP16-NEXT:    cmp x11, x8
-; CHECK-SD-FP16-NEXT:    csel x10, x11, x8, lt
-; CHECK-SD-FP16-NEXT:    fcvtzs x11, h0
-; CHECK-SD-FP16-NEXT:    cmp x10, x9
-; CHECK-SD-FP16-NEXT:    csel x1, x10, x9, gt
-; CHECK-SD-FP16-NEXT:    cmp x12, x8
-; CHECK-SD-FP16-NEXT:    csel x10, x12, x8, lt
-; CHECK-SD-FP16-NEXT:    cmp x10, x9
-; CHECK-SD-FP16-NEXT:    csel x2, x10, x9, gt
-; CHECK-SD-FP16-NEXT:    cmp x11, x8
-; CHECK-SD-FP16-NEXT:    csel x8, x11, x8, lt
-; CHECK-SD-FP16-NEXT:    cmp x8, x9
-; CHECK-SD-FP16-NEXT:    csel x3, x8, x9, gt
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-CVT-LABEL: test_signed_v8f16_v8i50:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-GI-CVT-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
-; CHECK-GI-CVT-NEXT:    mov x9, #-562949953421312 // =0xfffe000000000000
-; CHECK-GI-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-GI-CVT-NEXT:    fcvt s3, h1
-; CHECK-GI-CVT-NEXT:    mov h4, v1.h[2]
-; CHECK-GI-CVT-NEXT:    mov h1, v1.h[3]
-; CHECK-GI-CVT-NEXT:    fcvt s2, h2
-; CHECK-GI-CVT-NEXT:    fcvtzs x10, s3
-; CHECK-GI-CVT-NEXT:    fcvt s3, h4
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
-; CHECK-GI-CVT-NEXT:    fcvtzs x11, s2
-; CHECK-GI-CVT-NEXT:    cmp x10, x8
-; CHECK-GI-CVT-NEXT:    fcvtzs x12, s3
-; CHECK-GI-CVT-NEXT:    csel x10, x10, x8, lt
-; CHECK-GI-CVT-NEXT:    mov h2, v0.h[1]
-; CHECK-GI-CVT-NEXT:    fcvt s3, h0
-; CHECK-GI-CVT-NEXT:    cmp x10, x9
-; CHECK-GI-CVT-NEXT:    csel x4, x10, x9, gt
-; CHECK-GI-CVT-NEXT:    cmp x11, x8
-; CHECK-GI-CVT-NEXT:    csel x10, x11, x8, lt
-; CHECK-GI-CVT-NEXT:    fcvtzs x11, s1
-; CHECK-GI-CVT-NEXT:    mov h1, v0.h[2]
-; CHECK-GI-CVT-NEXT:    cmp x10, x9
-; CHECK-GI-CVT-NEXT:    fcvt s2, h2
-; CHECK-GI-CVT-NEXT:    mov h0, v0.h[3]
-; CHECK-GI-CVT-NEXT:    csel x5, x10, x9, gt
-; CHECK-GI-CVT-NEXT:    cmp x12, x8
-; CHECK-GI-CVT-NEXT:    csel x10, x12, x8, lt
-; CHECK-GI-CVT-NEXT:    fcvtzs x12, s3
-; CHECK-GI-CVT-NEXT:    cmp x10, x9
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
-; CHECK-GI-CVT-NEXT:    csel x6, x10, x9, gt
-; CHECK-GI-CVT-NEXT:    cmp x11, x8
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    csel x10, x11, x8, lt
-; CHECK-GI-CVT-NEXT:    fcvtzs x11, s2
-; CHECK-GI-CVT-NEXT:    cmp x10, x9
-; CHECK-GI-CVT-NEXT:    csel x7, x10, x9, gt
-; CHECK-GI-CVT-NEXT:    cmp x12, x8
-; CHECK-GI-CVT-NEXT:    csel x10, x12, x8, lt
-; CHECK-GI-CVT-NEXT:    fcvtzs x12, s1
-; CHECK-GI-CVT-NEXT:    cmp x10, x9
-; CHECK-GI-CVT-NEXT:    csel x0, x10, x9, gt
-; CHECK-GI-CVT-NEXT:    cmp x11, x8
-; CHECK-GI-CVT-NEXT:    csel x10, x11, x8, lt
-; CHECK-GI-CVT-NEXT:    fcvtzs x11, s0
-; CHECK-GI-CVT-NEXT:    cmp x10, x9
-; CHECK-GI-CVT-NEXT:    csel x1, x10, x9, gt
-; CHECK-GI-CVT-NEXT:    cmp x12, x8
-; CHECK-GI-CVT-NEXT:    csel x10, x12, x8, lt
-; CHECK-GI-CVT-NEXT:    cmp x10, x9
-; CHECK-GI-CVT-NEXT:    csel x2, x10, x9, gt
-; CHECK-GI-CVT-NEXT:    cmp x11, x8
-; CHECK-GI-CVT-NEXT:    csel x8, x11, x8, lt
-; CHECK-GI-CVT-NEXT:    cmp x8, x9
-; CHECK-GI-CVT-NEXT:    csel x3, x8, x9, gt
-; CHECK-GI-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_signed_v8f16_v8i50:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-CVT-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
+; CHECK-CVT-NEXT:    mov x9, #-562949953421312 // =0xfffe000000000000
+; CHECK-CVT-NEXT:    mov h2, v1.h[1]
+; CHECK-CVT-NEXT:    fcvt s3, h1
+; CHECK-CVT-NEXT:    mov h4, v1.h[2]
+; CHECK-CVT-NEXT:    mov h1, v1.h[3]
+; CHECK-CVT-NEXT:    fcvt s2, h2
+; CHECK-CVT-NEXT:    fcvtzs x10, s3
+; CHECK-CVT-NEXT:    fcvt s3, h4
+; CHECK-CVT-NEXT:    fcvt s1, h1
+; CHECK-CVT-NEXT:    fcvtzs x11, s2
+; CHECK-CVT-NEXT:    cmp x10, x8
+; CHECK-CVT-NEXT:    fcvtzs x12, s3
+; CHECK-CVT-NEXT:    csel x10, x10, x8, lt
+; CHECK-CVT-NEXT:    mov h2, v0.h[1]
+; CHECK-CVT-NEXT:    fcvt s3, h0
+; CHECK-CVT-NEXT:    cmp x10, x9
+; CHECK-CVT-NEXT:    csel x4, x10, x9, gt
+; CHECK-CVT-NEXT:    cmp x11, x8
+; CHECK-CVT-NEXT:    csel x10, x11, x8, lt
+; CHECK-CVT-NEXT:    fcvtzs x11, s1
+; CHECK-CVT-NEXT:    mov h1, v0.h[2]
+; CHECK-CVT-NEXT:    cmp x10, x9
+; CHECK-CVT-NEXT:    fcvt s2, h2
+; CHECK-CVT-NEXT:    mov h0, v0.h[3]
+; CHECK-CVT-NEXT:    csel x5, x10, x9, gt
+; CHECK-CVT-NEXT:    cmp x12, x8
+; CHECK-CVT-NEXT:    csel x10, x12, x8, lt
+; CHECK-CVT-NEXT:    fcvtzs x12, s3
+; CHECK-CVT-NEXT:    cmp x10, x9
+; CHECK-CVT-NEXT:    fcvt s1, h1
+; CHECK-CVT-NEXT:    csel x6, x10, x9, gt
+; CHECK-CVT-NEXT:    cmp x11, x8
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    csel x10, x11, x8, lt
+; CHECK-CVT-NEXT:    fcvtzs x11, s2
+; CHECK-CVT-NEXT:    cmp x10, x9
+; CHECK-CVT-NEXT:    csel x7, x10, x9, gt
+; CHECK-CVT-NEXT:    cmp x12, x8
+; CHECK-CVT-NEXT:    csel x10, x12, x8, lt
+; CHECK-CVT-NEXT:    fcvtzs x12, s1
+; CHECK-CVT-NEXT:    cmp x10, x9
+; CHECK-CVT-NEXT:    csel x0, x10, x9, gt
+; CHECK-CVT-NEXT:    cmp x11, x8
+; CHECK-CVT-NEXT:    csel x10, x11, x8, lt
+; CHECK-CVT-NEXT:    fcvtzs x11, s0
+; CHECK-CVT-NEXT:    cmp x10, x9
+; CHECK-CVT-NEXT:    csel x1, x10, x9, gt
+; CHECK-CVT-NEXT:    cmp x12, x8
+; CHECK-CVT-NEXT:    csel x10, x12, x8, lt
+; CHECK-CVT-NEXT:    cmp x10, x9
+; CHECK-CVT-NEXT:    csel x2, x10, x9, gt
+; CHECK-CVT-NEXT:    cmp x11, x8
+; CHECK-CVT-NEXT:    csel x8, x11, x8, lt
+; CHECK-CVT-NEXT:    cmp x8, x9
+; CHECK-CVT-NEXT:    csel x3, x8, x9, gt
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-GI-FP16-LABEL: test_signed_v8f16_v8i50:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-GI-FP16-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
-; CHECK-GI-FP16-NEXT:    mov x9, #-562949953421312 // =0xfffe000000000000
-; CHECK-GI-FP16-NEXT:    mov h2, v1.h[1]
-; CHECK-GI-FP16-NEXT:    fcvtzs x10, h1
-; CHECK-GI-FP16-NEXT:    mov h3, v1.h[2]
-; CHECK-GI-FP16-NEXT:    mov h1, v1.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzs x11, h2
-; CHECK-GI-FP16-NEXT:    cmp x10, x8
-; CHECK-GI-FP16-NEXT:    fcvtzs x12, h3
-; CHECK-GI-FP16-NEXT:    csel x10, x10, x8, lt
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    cmp x10, x9
-; CHECK-GI-FP16-NEXT:    csel x4, x10, x9, gt
-; CHECK-GI-FP16-NEXT:    cmp x11, x8
-; CHECK-GI-FP16-NEXT:    csel x10, x11, x8, lt
-; CHECK-GI-FP16-NEXT:    fcvtzs x11, h1
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    cmp x10, x9
-; CHECK-GI-FP16-NEXT:    csel x5, x10, x9, gt
-; CHECK-GI-FP16-NEXT:    cmp x12, x8
-; CHECK-GI-FP16-NEXT:    csel x10, x12, x8, lt
-; CHECK-GI-FP16-NEXT:    fcvtzs x12, h0
-; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-GI-FP16-NEXT:    cmp x10, x9
-; CHECK-GI-FP16-NEXT:    csel x6, x10, x9, gt
-; CHECK-GI-FP16-NEXT:    cmp x11, x8
-; CHECK-GI-FP16-NEXT:    csel x10, x11, x8, lt
-; CHECK-GI-FP16-NEXT:    fcvtzs x11, h1
-; CHECK-GI-FP16-NEXT:    cmp x10, x9
-; CHECK-GI-FP16-NEXT:    csel x7, x10, x9, gt
-; CHECK-GI-FP16-NEXT:    cmp x12, x8
-; CHECK-GI-FP16-NEXT:    csel x10, x12, x8, lt
-; CHECK-GI-FP16-NEXT:    fcvtzs x12, h2
-; CHECK-GI-FP16-NEXT:    cmp x10, x9
-; CHECK-GI-FP16-NEXT:    csel x0, x10, x9, gt
-; CHECK-GI-FP16-NEXT:    cmp x11, x8
-; CHECK-GI-FP16-NEXT:    csel x10, x11, x8, lt
-; CHECK-GI-FP16-NEXT:    fcvtzs x11, h0
-; CHECK-GI-FP16-NEXT:    cmp x10, x9
-; CHECK-GI-FP16-NEXT:    csel x1, x10, x9, gt
-; CHECK-GI-FP16-NEXT:    cmp x12, x8
-; CHECK-GI-FP16-NEXT:    csel x10, x12, x8, lt
-; CHECK-GI-FP16-NEXT:    cmp x10, x9
-; CHECK-GI-FP16-NEXT:    csel x2, x10, x9, gt
-; CHECK-GI-FP16-NEXT:    cmp x11, x8
-; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, lt
-; CHECK-GI-FP16-NEXT:    cmp x8, x9
-; CHECK-GI-FP16-NEXT:    csel x3, x8, x9, gt
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v8f16_v8i50:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-NEXT:    mov x8, #562949953421311 // =0x1ffffffffffff
+; CHECK-FP16-NEXT:    mov x9, #-562949953421312 // =0xfffe000000000000
+; CHECK-FP16-NEXT:    mov h2, v1.h[1]
+; CHECK-FP16-NEXT:    fcvtzs x10, h1
+; CHECK-FP16-NEXT:    mov h3, v1.h[2]
+; CHECK-FP16-NEXT:    mov h1, v1.h[3]
+; CHECK-FP16-NEXT:    fcvtzs x11, h2
+; CHECK-FP16-NEXT:    cmp x10, x8
+; CHECK-FP16-NEXT:    fcvtzs x12, h3
+; CHECK-FP16-NEXT:    csel x10, x10, x8, lt
+; CHECK-FP16-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-NEXT:    cmp x10, x9
+; CHECK-FP16-NEXT:    csel x4, x10, x9, gt
+; CHECK-FP16-NEXT:    cmp x11, x8
+; CHECK-FP16-NEXT:    csel x10, x11, x8, lt
+; CHECK-FP16-NEXT:    fcvtzs x11, h1
+; CHECK-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-NEXT:    cmp x10, x9
+; CHECK-FP16-NEXT:    csel x5, x10, x9, gt
+; CHECK-FP16-NEXT:    cmp x12, x8
+; CHECK-FP16-NEXT:    csel x10, x12, x8, lt
+; CHECK-FP16-NEXT:    fcvtzs x12, h0
+; CHECK-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-FP16-NEXT:    cmp x10, x9
+; CHECK-FP16-NEXT:    csel x6, x10, x9, gt
+; CHECK-FP16-NEXT:    cmp x11, x8
+; CHECK-FP16-NEXT:    csel x10, x11, x8, lt
+; CHECK-FP16-NEXT:    fcvtzs x11, h1
+; CHECK-FP16-NEXT:    cmp x10, x9
+; CHECK-FP16-NEXT:    csel x7, x10, x9, gt
+; CHECK-FP16-NEXT:    cmp x12, x8
+; CHECK-FP16-NEXT:    csel x10, x12, x8, lt
+; CHECK-FP16-NEXT:    fcvtzs x12, h2
+; CHECK-FP16-NEXT:    cmp x10, x9
+; CHECK-FP16-NEXT:    csel x0, x10, x9, gt
+; CHECK-FP16-NEXT:    cmp x11, x8
+; CHECK-FP16-NEXT:    csel x10, x11, x8, lt
+; CHECK-FP16-NEXT:    fcvtzs x11, h0
+; CHECK-FP16-NEXT:    cmp x10, x9
+; CHECK-FP16-NEXT:    csel x1, x10, x9, gt
+; CHECK-FP16-NEXT:    cmp x12, x8
+; CHECK-FP16-NEXT:    csel x10, x12, x8, lt
+; CHECK-FP16-NEXT:    cmp x10, x9
+; CHECK-FP16-NEXT:    csel x2, x10, x9, gt
+; CHECK-FP16-NEXT:    cmp x11, x8
+; CHECK-FP16-NEXT:    csel x8, x11, x8, lt
+; CHECK-FP16-NEXT:    cmp x8, x9
+; CHECK-FP16-NEXT:    csel x3, x8, x9, gt
+; CHECK-FP16-NEXT:    ret
     %x = call <8 x i50> @llvm.fptosi.sat.v8f16.v8i50(<8 x half> %f)
     ret <8 x i50> %x
 }
@@ -4681,26 +4394,13 @@ define <16 x i8> @test_signed_v16f32_v16i8(<16 x float> %f) {
 }
 
 define <8 x i16> @test_signed_v8f32_v8i16(<8 x float> %f) {
-; CHECK-SD-LABEL: test_signed_v8f32_v8i16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-SD-NEXT:    sqxtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    sqxtn2 v0.8h, v1.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_signed_v8f32_v8i16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.4s, #127, msl #8
-; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-NEXT:    smin v0.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT:    smin v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT:    mvni v2.4s, #127, msl #8
-; CHECK-GI-NEXT:    smax v0.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT:    smax v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_signed_v8f32_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-NEXT:    sqxtn v0.4h, v0.4s
+; CHECK-NEXT:    sqxtn2 v0.8h, v1.4s
+; CHECK-NEXT:    ret
     %x = call <8 x i16> @llvm.fptosi.sat.v8f32.v8i16(<8 x float> %f)
     ret <8 x i16> %x
 }
@@ -4720,22 +4420,14 @@ define <16 x i16> @test_signed_v16f32_v16i16(<16 x float> %f) {
 ;
 ; CHECK-GI-LABEL: test_signed_v16f32_v16i16:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v4.4s, #127, msl #8
 ; CHECK-GI-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzs v1.4s, v1.4s
 ; CHECK-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-GI-NEXT:    fcvtzs v4.4s, v1.4s
 ; CHECK-GI-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-NEXT:    mvni v5.4s, #127, msl #8
-; CHECK-GI-NEXT:    smin v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT:    smin v1.4s, v1.4s, v4.4s
-; CHECK-GI-NEXT:    smin v2.4s, v2.4s, v4.4s
-; CHECK-GI-NEXT:    smin v3.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT:    smax v0.4s, v0.4s, v5.4s
-; CHECK-GI-NEXT:    smax v1.4s, v1.4s, v5.4s
-; CHECK-GI-NEXT:    smax v2.4s, v2.4s, v5.4s
-; CHECK-GI-NEXT:    smax v3.4s, v3.4s, v5.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT:    sqxtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    sqxtn v1.4h, v2.4s
+; CHECK-GI-NEXT:    sqxtn2 v0.8h, v4.4s
+; CHECK-GI-NEXT:    sqxtn2 v1.8h, v3.4s
 ; CHECK-GI-NEXT:    ret
     %x = call <16 x i16> @llvm.fptosi.sat.v16f32.v16i16(<16 x float> %f)
     ret <16 x i16> %x
@@ -4769,13 +4461,13 @@ define <16 x i8> @test_signed_v16f16_v16i8(<16 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_signed_v16f16_v16i8:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    sqxtn v0.8b, v0.8h
-; CHECK-SD-FP16-NEXT:    sqxtn2 v0.16b, v1.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v16f16_v16i8:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-FP16-NEXT:    sqxtn v0.8b, v0.8h
+; CHECK-FP16-NEXT:    sqxtn2 v0.16b, v1.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_signed_v16f16_v16i8:
 ; CHECK-GI-CVT:       // %bb.0:
@@ -4801,19 +4493,6 @@ define <16 x i8> @test_signed_v16f16_v16i8(<16 x half> %f) {
 ; CHECK-GI-CVT-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
 ; CHECK-GI-CVT-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v16f16_v16i8:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    movi v2.8h, #127
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    mvni v3.8h, #127
-; CHECK-GI-FP16-NEXT:    smin v0.8h, v0.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    smin v1.8h, v1.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    smax v0.8h, v0.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    smax v1.8h, v1.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <16 x i8> @llvm.fptosi.sat.v16f16.v16i8(<16 x half> %f)
     ret <16 x i8> %x
 }
@@ -4835,41 +4514,27 @@ define <16 x i16> @test_signed_v16f16_v16i16(<16 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    sqxtn2 v1.8h, v3.4s
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_signed_v16f16_v16i16:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_signed_v16f16_v16i16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzs v0.8h, v0.8h
+; CHECK-FP16-NEXT:    fcvtzs v1.8h, v1.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_signed_v16f16_v16i16:
 ; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-GI-CVT-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-GI-CVT-NEXT:    fcvtl v3.4s, v1.4h
 ; CHECK-GI-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-CVT-NEXT:    fcvtl v4.4s, v1.4h
 ; CHECK-GI-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-CVT-NEXT:    movi v2.4s, #127, msl #8
-; CHECK-GI-CVT-NEXT:    mvni v5.4s, #127, msl #8
+; CHECK-GI-CVT-NEXT:    fcvtzs v2.4s, v2.4s
 ; CHECK-GI-CVT-NEXT:    fcvtzs v3.4s, v3.4s
-; CHECK-GI-CVT-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    fcvtzs v4.4s, v4.4s
-; CHECK-GI-CVT-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    smin v3.4s, v3.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    smin v0.4s, v0.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    smin v4.4s, v4.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    smin v1.4s, v1.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    smax v2.4s, v3.4s, v5.4s
-; CHECK-GI-CVT-NEXT:    smax v0.4s, v0.4s, v5.4s
-; CHECK-GI-CVT-NEXT:    smax v3.4s, v4.4s, v5.4s
-; CHECK-GI-CVT-NEXT:    smax v1.4s, v1.4s, v5.4s
-; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
-; CHECK-GI-CVT-NEXT:    uzp1 v1.8h, v3.8h, v1.8h
+; CHECK-GI-CVT-NEXT:    fcvtzs v4.4s, v0.4s
+; CHECK-GI-CVT-NEXT:    fcvtzs v5.4s, v1.4s
+; CHECK-GI-CVT-NEXT:    sqxtn v0.4h, v2.4s
+; CHECK-GI-CVT-NEXT:    sqxtn v1.4h, v3.4s
+; CHECK-GI-CVT-NEXT:    sqxtn2 v0.8h, v4.4s
+; CHECK-GI-CVT-NEXT:    sqxtn2 v1.8h, v5.4s
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_signed_v16f16_v16i16:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzs v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzs v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <16 x i16> @llvm.fptosi.sat.v16f16.v16i16(<16 x half> %f)
     ret <16 x i16> %x
 }
@@ -5425,7 +5090,7 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI86_1]
 ; CHECK-SD-NEXT:    mov x20, #-9223372036854775808 // =0x8000000000000000
-; CHECK-SD-NEXT:    csel x19, x20, x0, lt
+; CHECK-SD-NEXT:    csel x19, x20, x0, mi
 ; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
@@ -5447,7 +5112,7 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel x19, x20, x0, lt
+; CHECK-SD-NEXT:    csel x19, x20, x0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w0, #0
@@ -5500,9 +5165,9 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov x22, #-1125899906842624 // =0xfffc000000000000
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    mov x23, #4629137466983448575 // =0x403dffffffffffff
-; CHECK-GI-NEXT:    csel x8, x19, x22, lt
+; CHECK-GI-NEXT:    csel x8, x19, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x21, x23, lt
+; CHECK-GI-NEXT:    csel x8, x21, x23, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfdi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
@@ -5525,9 +5190,9 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x20
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, x22, lt
+; CHECK-GI-NEXT:    csel x8, x19, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x20, x23, lt
+; CHECK-GI-NEXT:    csel x8, x20, x23, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixtfdi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll
index 46950e7a60349..07e49e331415e 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll
@@ -777,7 +777,7 @@ define i32 @test_unsigned_f128_i32(fp128 %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI30_1]
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    csinv w0, w19, wzr, le
@@ -811,11 +811,11 @@ define i32 @test_unsigned_f128_i32(fp128 %f) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
 ; CHECK-GI-NEXT:    mov x8, #281474976579584 // =0xfffffffe0000
 ; CHECK-GI-NEXT:    movk x8, #16414, lsl #48
-; CHECK-GI-NEXT:    csel x8, x20, x8, lt
+; CHECK-GI-NEXT:    csel x8, x20, x8, mi
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    add sp, sp, #48
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
index 44847a41287d6..efe0a1bedbc9e 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-CVT
-; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
-; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-CVT
-; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-CVT,CHECK-SD-CVT
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-FP16,CHECK-SD-FP16
+; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-CVT,CHECK-GI-CVT
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-FP16,CHECK-GI-FP16
 
 ; CHECK-GI:       warning: Instruction selection used fallback path for test_unsigned_v4f32_v4i50
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for test_unsigned_v4f16_v4i50
@@ -463,7 +463,7 @@ define <1 x i32> @test_unsigned_v1f128_v1i32(<1 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI14_1]
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    csinv w8, w19, wzr, le
@@ -497,11 +497,11 @@ define <1 x i32> @test_unsigned_v1f128_v1i32(<1 x fp128> %f) {
 ; CHECK-GI-NEXT:    ldr q1, [x8, :lo12:.LCPI14_0]
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
 ; CHECK-GI-NEXT:    mov x8, #281474976579584 // =0xfffffffe0000
 ; CHECK-GI-NEXT:    movk x8, #16414, lsl #48
-; CHECK-GI-NEXT:    csel x8, x20, x8, lt
+; CHECK-GI-NEXT:    csel x8, x20, x8, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
@@ -537,7 +537,7 @@ define <2 x i32> @test_unsigned_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI15_1]
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
@@ -551,7 +551,7 @@ define <2 x i32> @test_unsigned_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
@@ -595,10 +595,10 @@ define <2 x i32> @test_unsigned_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    mov x21, #281474976579584 // =0xfffffffe0000
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    movk x21, #16414, lsl #48
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x20, x21, lt
+; CHECK-GI-NEXT:    csel x8, x20, x21, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    ldp q1, q0, [sp, #16] // 32-byte Folded Reload
@@ -615,9 +615,9 @@ define <2 x i32> @test_unsigned_v2f128_v2i32(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x22
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x20, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x20, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x22, x21, lt
+; CHECK-GI-NEXT:    csel x8, x22, x21, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    fmov s0, w19
@@ -657,7 +657,7 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI16_1]
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldp q1, q0, [sp, #32] // 32-byte Folded Reload
@@ -670,7 +670,7 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -683,7 +683,7 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    csinv w8, w19, wzr, le
@@ -727,10 +727,10 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    mov x21, #281474976579584 // =0xfffffffe0000
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    movk x21, #16414, lsl #48
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x20, x21, lt
+; CHECK-GI-NEXT:    csel x8, x20, x21, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
@@ -747,9 +747,9 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x22
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x20, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x20, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x22, x21, lt
+; CHECK-GI-NEXT:    csel x8, x22, x21, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    ldp q1, q0, [sp, #32] // 32-byte Folded Reload
@@ -766,9 +766,9 @@ define <3 x i32> @test_unsigned_v3f128_v3i32(<3 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x23
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x22, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x22, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x23, x21, lt
+; CHECK-GI-NEXT:    csel x8, x23, x21, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    fmov s0, w19
@@ -809,7 +809,7 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI17_1]
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    str q1, [sp, #48] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
@@ -823,7 +823,7 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr q1, [sp, #64] // 16-byte Folded Reload
@@ -838,7 +838,7 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-SD-NEXT:    bl __fixunstfsi
 ; CHECK-SD-NEXT:    ldp q0, q1, [sp, #32] // 32-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
@@ -853,7 +853,7 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel w19, wzr, w0, lt
+; CHECK-SD-NEXT:    csel w19, wzr, w0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
@@ -900,10 +900,10 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    mov x22, #281474976579584 // =0xfffffffe0000
-; CHECK-GI-NEXT:    csel x8, x19, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x19, xzr, mi
 ; CHECK-GI-NEXT:    movk x22, #16414, lsl #48
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x20, x22, lt
+; CHECK-GI-NEXT:    csel x8, x20, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
@@ -921,9 +921,9 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x21
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x20, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x20, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x21, x22, lt
+; CHECK-GI-NEXT:    csel x8, x21, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
@@ -941,9 +941,9 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x23
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x21, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x21, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x23, x22, lt
+; CHECK-GI-NEXT:    csel x8, x23, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
@@ -960,9 +960,9 @@ define <4 x i32> @test_unsigned_v4f128_v4i32(<4 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x24
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x23, xzr, lt
+; CHECK-GI-NEXT:    csel x8, x23, xzr, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x24, x22, lt
+; CHECK-GI-NEXT:    csel x8, x24, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfsi
 ; CHECK-GI-NEXT:    fmov s0, w19
@@ -993,31 +993,18 @@ declare <7 x i32> @llvm.fptoui.sat.v7f16.v7i32 (<7 x half>)
 declare <8 x i32> @llvm.fptoui.sat.v8f16.v8i32 (<8 x half>)
 
 define <1 x i32> @test_unsigned_v1f16_v1i32(<1 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_unsigned_v1f16_v1i32:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    fcvt s0, h0
-; CHECK-SD-CVT-NEXT:    fcvtzu w8, s0
-; CHECK-SD-CVT-NEXT:    fmov s0, w8
-; CHECK-SD-CVT-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: test_unsigned_v1f16_v1i32:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzu w8, h0
-; CHECK-SD-FP16-NEXT:    fmov s0, w8
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-CVT-LABEL: test_unsigned_v1f16_v1i32:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    fcvtzu w8, s0
-; CHECK-GI-CVT-NEXT:    fmov s0, w8
-; CHECK-GI-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_unsigned_v1f16_v1i32:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvtzu w8, s0
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-GI-FP16-LABEL: test_unsigned_v1f16_v1i32:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu w8, h0
-; CHECK-GI-FP16-NEXT:    fmov s0, w8
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v1f16_v1i32:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu w8, h0
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    ret
     %x = call <1 x i32> @llvm.fptoui.sat.v1f16.v1i32(<1 x half> %f)
     ret <1 x i32> %x
 }
@@ -1541,19 +1528,11 @@ define <4 x i13> @test_unsigned_v4f32_v4i13(<4 x float> %f) {
 }
 
 define <4 x i16> @test_unsigned_v4f32_v4i16(<4 x float> %f) {
-; CHECK-SD-LABEL: test_unsigned_v4f32_v4i16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NEXT:    uqxtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_unsigned_v4f32_v4i16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v1.2d, #0x00ffff0000ffff
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_unsigned_v4f32_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    uqxtn v0.4h, v0.4s
+; CHECK-NEXT:    ret
     %x = call <4 x i16> @llvm.fptoui.sat.v4f32.v4i16(<4 x float> %f)
     ret <4 x i16> %x
 }
@@ -2338,139 +2317,77 @@ declare <4 x i100> @llvm.fptoui.sat.v4f16.v4i100(<4 x half>)
 declare <4 x i128> @llvm.fptoui.sat.v4f16.v4i128(<4 x half>)
 
 define <4 x i1> @test_unsigned_v4f16_v4i1(<4 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_unsigned_v4f16_v4i1:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-CVT-NEXT:    movi v1.4s, #1
-; CHECK-SD-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-SD-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_unsigned_v4f16_v4i1:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    movi v1.4s, #1
+; CHECK-CVT-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    xtn v0.4h, v0.4s
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_unsigned_v4f16_v4i1:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    movi v1.4h, #1
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    umin v0.4h, v0.4h, v1.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-CVT-LABEL: test_unsigned_v4f16_v4i1:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-CVT-NEXT:    movi v1.4s, #1
-; CHECK-GI-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i1:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    movi v1.4h, #1
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    umin v0.4h, v0.4h, v1.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v4f16_v4i1:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    movi v1.4h, #1
+; CHECK-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-NEXT:    umin v0.4h, v0.4h, v1.4h
+; CHECK-FP16-NEXT:    ret
     %x = call <4 x i1> @llvm.fptoui.sat.v4f16.v4i1(<4 x half> %f)
     ret <4 x i1> %x
 }
 
 define <4 x i8> @test_unsigned_v4f16_v4i8(<4 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_unsigned_v4f16_v4i8:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-CVT-NEXT:    movi v1.2d, #0x0000ff000000ff
-; CHECK-SD-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-SD-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-CVT-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: test_unsigned_v4f16_v4i8:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    movi d1, #0xff00ff00ff00ff
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    umin v0.4h, v0.4h, v1.4h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-CVT-LABEL: test_unsigned_v4f16_v4i8:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-CVT-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    xtn v0.4h, v0.4s
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-GI-CVT-LABEL: test_unsigned_v4f16_v4i8:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-CVT-NEXT:    movi v1.2d, #0x0000ff000000ff
-; CHECK-GI-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i8:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    movi d1, #0xff00ff00ff00ff
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    umin v0.4h, v0.4h, v1.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v4f16_v4i8:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-NEXT:    umin v0.4h, v0.4h, v1.4h
+; CHECK-FP16-NEXT:    ret
     %x = call <4 x i8> @llvm.fptoui.sat.v4f16.v4i8(<4 x half> %f)
     ret <4 x i8> %x
 }
 
 define <4 x i13> @test_unsigned_v4f16_v4i13(<4 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_unsigned_v4f16_v4i13:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-CVT-NEXT:    movi v1.4s, #31, msl #8
-; CHECK-SD-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-SD-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-SD-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_unsigned_v4f16_v4i13:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    movi v1.4s, #31, msl #8
+; CHECK-CVT-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-NEXT:    xtn v0.4h, v0.4s
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_unsigned_v4f16_v4i13:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    mvni v1.4h, #224, lsl #8
-; CHECK-SD-FP16-NEXT:    umin v0.4h, v0.4h, v1.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-CVT-LABEL: test_unsigned_v4f16_v4i13:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-CVT-NEXT:    movi v1.4s, #31, msl #8
-; CHECK-GI-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i13:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #224, lsl #8
-; CHECK-GI-FP16-NEXT:    umin v0.4h, v0.4h, v1.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v4f16_v4i13:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-NEXT:    mvni v1.4h, #224, lsl #8
+; CHECK-FP16-NEXT:    umin v0.4h, v0.4h, v1.4h
+; CHECK-FP16-NEXT:    ret
     %x = call <4 x i13> @llvm.fptoui.sat.v4f16.v4i13(<4 x half> %f)
     ret <4 x i13> %x
 }
 
 define <4 x i16> @test_unsigned_v4f16_v4i16(<4 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_unsigned_v4f16_v4i16:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-SD-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-CVT-NEXT:    uqxtn v0.4h, v0.4s
-; CHECK-SD-CVT-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: test_unsigned_v4f16_v4i16:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-CVT-LABEL: test_unsigned_v4f16_v4i16:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    fcvtl v0.4s, v0.4h
+; CHECK-CVT-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-NEXT:    uqxtn v0.4h, v0.4s
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-GI-CVT-LABEL: test_unsigned_v4f16_v4i16:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v0.4s, v0.4h
-; CHECK-GI-CVT-NEXT:    movi v1.2d, #0x00ffff0000ffff
-; CHECK-GI-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    xtn v0.4h, v0.4s
-; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i16:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v4f16_v4i16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
     %x = call <4 x i16> @llvm.fptoui.sat.v4f16.v4i16(<4 x half> %f)
     ret <4 x i16> %x
 }
@@ -2498,97 +2415,51 @@ define <4 x i32> @test_unsigned_v4f16_v4i32_duplicate(<4 x half> %f) {
 }
 
 define <4 x i50> @test_unsigned_v4f16_v4i50(<4 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_unsigned_v4f16_v4i50:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-SD-CVT-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
-; CHECK-SD-CVT-NEXT:    mov h3, v0.h[3]
-; CHECK-SD-CVT-NEXT:    fcvt s0, h0
-; CHECK-SD-CVT-NEXT:    fcvt s1, h1
-; CHECK-SD-CVT-NEXT:    fcvt s2, h2
-; CHECK-SD-CVT-NEXT:    fcvt s3, h3
-; CHECK-SD-CVT-NEXT:    fcvtzu x9, s0
-; CHECK-SD-CVT-NEXT:    fcvtzu x10, s1
-; CHECK-SD-CVT-NEXT:    fcvtzu x11, s2
-; CHECK-SD-CVT-NEXT:    fcvtzu x12, s3
-; CHECK-SD-CVT-NEXT:    cmp x9, x8
-; CHECK-SD-CVT-NEXT:    csel x0, x9, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x10, x8
-; CHECK-SD-CVT-NEXT:    csel x1, x10, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x11, x8
-; CHECK-SD-CVT-NEXT:    csel x2, x11, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x12, x8
-; CHECK-SD-CVT-NEXT:    csel x3, x12, x8, lo
-; CHECK-SD-CVT-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: test_unsigned_v4f16_v4i50:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-SD-FP16-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
-; CHECK-SD-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h0
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h1
-; CHECK-SD-FP16-NEXT:    fcvtzu x11, h2
-; CHECK-SD-FP16-NEXT:    fcvtzu x12, h3
-; CHECK-SD-FP16-NEXT:    cmp x9, x8
-; CHECK-SD-FP16-NEXT:    csel x0, x9, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x10, x8
-; CHECK-SD-FP16-NEXT:    csel x1, x10, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x11, x8
-; CHECK-SD-FP16-NEXT:    csel x2, x11, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x12, x8
-; CHECK-SD-FP16-NEXT:    csel x3, x12, x8, lo
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-CVT-LABEL: test_unsigned_v4f16_v4i50:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-CVT-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
-; CHECK-GI-CVT-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
-; CHECK-GI-CVT-NEXT:    fcvt s2, h2
-; CHECK-GI-CVT-NEXT:    fcvt s3, h3
-; CHECK-GI-CVT-NEXT:    fcvtzu x9, s0
-; CHECK-GI-CVT-NEXT:    fcvtzu x10, s1
-; CHECK-GI-CVT-NEXT:    fcvtzu x11, s2
-; CHECK-GI-CVT-NEXT:    fcvtzu x12, s3
-; CHECK-GI-CVT-NEXT:    cmp x9, x8
-; CHECK-GI-CVT-NEXT:    csel x0, x9, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x10, x8
-; CHECK-GI-CVT-NEXT:    csel x1, x10, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x11, x8
-; CHECK-GI-CVT-NEXT:    csel x2, x11, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x12, x8
-; CHECK-GI-CVT-NEXT:    csel x3, x12, x8, lo
-; CHECK-GI-CVT-NEXT:    ret
+; CHECK-CVT-LABEL: test_unsigned_v4f16_v4i50:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-NEXT:    mov h1, v0.h[1]
+; CHECK-CVT-NEXT:    mov h2, v0.h[2]
+; CHECK-CVT-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
+; CHECK-CVT-NEXT:    mov h3, v0.h[3]
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    fcvt s1, h1
+; CHECK-CVT-NEXT:    fcvt s2, h2
+; CHECK-CVT-NEXT:    fcvt s3, h3
+; CHECK-CVT-NEXT:    fcvtzu x9, s0
+; CHECK-CVT-NEXT:    fcvtzu x10, s1
+; CHECK-CVT-NEXT:    fcvtzu x11, s2
+; CHECK-CVT-NEXT:    fcvtzu x12, s3
+; CHECK-CVT-NEXT:    cmp x9, x8
+; CHECK-CVT-NEXT:    csel x0, x9, x8, lo
+; CHECK-CVT-NEXT:    cmp x10, x8
+; CHECK-CVT-NEXT:    csel x1, x10, x8, lo
+; CHECK-CVT-NEXT:    cmp x11, x8
+; CHECK-CVT-NEXT:    csel x2, x11, x8, lo
+; CHECK-CVT-NEXT:    cmp x12, x8
+; CHECK-CVT-NEXT:    csel x3, x12, x8, lo
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i50:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
-; CHECK-GI-FP16-NEXT:    fcvtzu x10, h1
-; CHECK-GI-FP16-NEXT:    fcvtzu x11, h2
-; CHECK-GI-FP16-NEXT:    fcvtzu x12, h3
-; CHECK-GI-FP16-NEXT:    cmp x9, x8
-; CHECK-GI-FP16-NEXT:    csel x0, x9, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x10, x8
-; CHECK-GI-FP16-NEXT:    csel x1, x10, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x11, x8
-; CHECK-GI-FP16-NEXT:    csel x2, x11, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x12, x8
-; CHECK-GI-FP16-NEXT:    csel x3, x12, x8, lo
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v4f16_v4i50:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
+; CHECK-FP16-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-FP16-NEXT:    fcvtzu x10, h1
+; CHECK-FP16-NEXT:    fcvtzu x11, h2
+; CHECK-FP16-NEXT:    fcvtzu x12, h3
+; CHECK-FP16-NEXT:    cmp x9, x8
+; CHECK-FP16-NEXT:    csel x0, x9, x8, lo
+; CHECK-FP16-NEXT:    cmp x10, x8
+; CHECK-FP16-NEXT:    csel x1, x10, x8, lo
+; CHECK-FP16-NEXT:    cmp x11, x8
+; CHECK-FP16-NEXT:    csel x2, x11, x8, lo
+; CHECK-FP16-NEXT:    cmp x12, x8
+; CHECK-FP16-NEXT:    csel x3, x12, x8, lo
+; CHECK-FP16-NEXT:    ret
     %x = call <4 x i50> @llvm.fptoui.sat.v4f16.v4i50(<4 x half> %f)
     ret <4 x i50> %x
 }
@@ -2924,13 +2795,13 @@ define <8 x i1> @test_unsigned_v8f16_v8i1(<8 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_unsigned_v8f16_v8i1:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    movi v1.8h, #1
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    umin v0.8h, v0.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v8f16_v8i1:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    movi v1.8h, #1
+; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT:    umin v0.8h, v0.8h, v1.8h
+; CHECK-FP16-NEXT:    xtn v0.8b, v0.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_unsigned_v8f16_v8i1:
 ; CHECK-GI-CVT:       // %bb.0:
@@ -2944,14 +2815,6 @@ define <8 x i1> @test_unsigned_v8f16_v8i1(<8 x half> %f) {
 ; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
 ; CHECK-GI-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v8f16_v8i1:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    movi v1.8h, #1
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    umin v0.8h, v0.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <8 x i1> @llvm.fptoui.sat.v8f16.v8i1(<8 x half> %f)
     ret <8 x i1> %x
 }
@@ -2970,11 +2833,11 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_unsigned_v8f16_v8i8:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    uqxtn v0.8b, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v8f16_v8i8:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT:    uqxtn v0.8b, v0.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_unsigned_v8f16_v8i8:
 ; CHECK-GI-CVT:       // %bb.0:
@@ -2988,14 +2851,6 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
 ; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
 ; CHECK-GI-CVT-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v8f16_v8i8:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    movi v1.2d, #0xff00ff00ff00ff
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    umin v0.8h, v0.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    xtn v0.8b, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <8 x i8> @llvm.fptoui.sat.v8f16.v8i8(<8 x half> %f)
     ret <8 x i8> %x
 }
@@ -3013,12 +2868,12 @@ define <8 x i13> @test_unsigned_v8f16_v8i13(<8 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_unsigned_v8f16_v8i13:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    mvni v1.8h, #224, lsl #8
-; CHECK-SD-FP16-NEXT:    umin v0.8h, v0.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v8f16_v8i13:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT:    mvni v1.8h, #224, lsl #8
+; CHECK-FP16-NEXT:    umin v0.8h, v0.8h, v1.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_unsigned_v8f16_v8i13:
 ; CHECK-GI-CVT:       // %bb.0:
@@ -3031,13 +2886,6 @@ define <8 x i13> @test_unsigned_v8f16_v8i13(<8 x half> %f) {
 ; CHECK-GI-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
 ; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v8f16_v8i13:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    mvni v1.8h, #224, lsl #8
-; CHECK-GI-FP16-NEXT:    umin v0.8h, v0.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <8 x i13> @llvm.fptoui.sat.v8f16.v8i13(<8 x half> %f)
     ret <8 x i13> %x
 }
@@ -3053,27 +2901,20 @@ define <8 x i16> @test_unsigned_v8f16_v8i16(<8 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    uqxtn2 v0.8h, v1.4s
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_unsigned_v8f16_v8i16:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v8f16_v8i16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_unsigned_v8f16_v8i16:
 ; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-GI-CVT-NEXT:    fcvtl v1.4s, v0.4h
 ; CHECK-GI-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-CVT-NEXT:    movi v1.2d, #0x00ffff0000ffff
-; CHECK-GI-CVT-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    umin v2.4s, v2.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-GI-CVT-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-GI-CVT-NEXT:    fcvtzu v2.4s, v0.4s
+; CHECK-GI-CVT-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-GI-CVT-NEXT:    uqxtn2 v0.8h, v2.4s
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v8f16_v8i16:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <8 x i16> @llvm.fptoui.sat.v8f16.v8i16(<8 x half> %f)
     ret <8 x i16> %x
 }
@@ -3122,165 +2963,85 @@ define <8 x i32> @test_unsigned_v8f16_v8i32_duplicate(<8 x half> %f) {
 }
 
 define <8 x i50> @test_unsigned_v8f16_v8i50(<8 x half> %f) {
-; CHECK-SD-CVT-LABEL: test_unsigned_v8f16_v8i50:
-; CHECK-SD-CVT:       // %bb.0:
-; CHECK-SD-CVT-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-CVT-NEXT:    mov h5, v0.h[1]
-; CHECK-SD-CVT-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
-; CHECK-SD-CVT-NEXT:    mov h6, v0.h[2]
-; CHECK-SD-CVT-NEXT:    mov h7, v0.h[3]
-; CHECK-SD-CVT-NEXT:    fcvt s0, h0
-; CHECK-SD-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-CVT-NEXT:    mov h3, v1.h[2]
-; CHECK-SD-CVT-NEXT:    mov h4, v1.h[3]
-; CHECK-SD-CVT-NEXT:    fcvt s1, h1
-; CHECK-SD-CVT-NEXT:    fcvtzu x13, s0
-; CHECK-SD-CVT-NEXT:    fcvt s2, h2
-; CHECK-SD-CVT-NEXT:    fcvt s3, h3
-; CHECK-SD-CVT-NEXT:    fcvt s4, h4
-; CHECK-SD-CVT-NEXT:    fcvtzu x9, s1
-; CHECK-SD-CVT-NEXT:    fcvt s1, h5
-; CHECK-SD-CVT-NEXT:    fcvtzu x10, s2
-; CHECK-SD-CVT-NEXT:    fcvtzu x11, s3
-; CHECK-SD-CVT-NEXT:    fcvt s2, h6
-; CHECK-SD-CVT-NEXT:    fcvtzu x12, s4
-; CHECK-SD-CVT-NEXT:    fcvt s3, h7
-; CHECK-SD-CVT-NEXT:    cmp x9, x8
-; CHECK-SD-CVT-NEXT:    fcvtzu x14, s1
-; CHECK-SD-CVT-NEXT:    csel x4, x9, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x10, x8
-; CHECK-SD-CVT-NEXT:    fcvtzu x9, s2
-; CHECK-SD-CVT-NEXT:    csel x5, x10, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x11, x8
-; CHECK-SD-CVT-NEXT:    fcvtzu x10, s3
-; CHECK-SD-CVT-NEXT:    csel x6, x11, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x12, x8
-; CHECK-SD-CVT-NEXT:    csel x7, x12, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x13, x8
-; CHECK-SD-CVT-NEXT:    csel x0, x13, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x14, x8
-; CHECK-SD-CVT-NEXT:    csel x1, x14, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x9, x8
-; CHECK-SD-CVT-NEXT:    csel x2, x9, x8, lo
-; CHECK-SD-CVT-NEXT:    cmp x10, x8
-; CHECK-SD-CVT-NEXT:    csel x3, x10, x8, lo
-; CHECK-SD-CVT-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: test_unsigned_v8f16_v8i50:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-FP16-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
-; CHECK-SD-FP16-NEXT:    fcvtzu x13, h0
-; CHECK-SD-FP16-NEXT:    mov h2, v1.h[1]
-; CHECK-SD-FP16-NEXT:    mov h3, v1.h[2]
-; CHECK-SD-FP16-NEXT:    mov h4, v1.h[3]
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h1
-; CHECK-SD-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h2
-; CHECK-SD-FP16-NEXT:    fcvtzu x11, h3
-; CHECK-SD-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-SD-FP16-NEXT:    fcvtzu x12, h4
-; CHECK-SD-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-SD-FP16-NEXT:    cmp x9, x8
-; CHECK-SD-FP16-NEXT:    fcvtzu x14, h1
-; CHECK-SD-FP16-NEXT:    csel x4, x9, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x10, x8
-; CHECK-SD-FP16-NEXT:    fcvtzu x9, h2
-; CHECK-SD-FP16-NEXT:    csel x5, x10, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x11, x8
-; CHECK-SD-FP16-NEXT:    fcvtzu x10, h3
-; CHECK-SD-FP16-NEXT:    csel x6, x11, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x12, x8
-; CHECK-SD-FP16-NEXT:    csel x7, x12, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x13, x8
-; CHECK-SD-FP16-NEXT:    csel x0, x13, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x14, x8
-; CHECK-SD-FP16-NEXT:    csel x1, x14, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x9, x8
-; CHECK-SD-FP16-NEXT:    csel x2, x9, x8, lo
-; CHECK-SD-FP16-NEXT:    cmp x10, x8
-; CHECK-SD-FP16-NEXT:    csel x3, x10, x8, lo
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-CVT-LABEL: test_unsigned_v8f16_v8i50:
+; CHECK-CVT:       // %bb.0:
+; CHECK-CVT-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-CVT-NEXT:    mov h5, v0.h[1]
+; CHECK-CVT-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
+; CHECK-CVT-NEXT:    mov h6, v0.h[2]
+; CHECK-CVT-NEXT:    mov h7, v0.h[3]
+; CHECK-CVT-NEXT:    fcvt s0, h0
+; CHECK-CVT-NEXT:    mov h2, v1.h[1]
+; CHECK-CVT-NEXT:    mov h3, v1.h[2]
+; CHECK-CVT-NEXT:    mov h4, v1.h[3]
+; CHECK-CVT-NEXT:    fcvt s1, h1
+; CHECK-CVT-NEXT:    fcvtzu x13, s0
+; CHECK-CVT-NEXT:    fcvt s2, h2
+; CHECK-CVT-NEXT:    fcvt s3, h3
+; CHECK-CVT-NEXT:    fcvt s4, h4
+; CHECK-CVT-NEXT:    fcvtzu x9, s1
+; CHECK-CVT-NEXT:    fcvt s1, h5
+; CHECK-CVT-NEXT:    fcvtzu x10, s2
+; CHECK-CVT-NEXT:    fcvtzu x11, s3
+; CHECK-CVT-NEXT:    fcvt s2, h6
+; CHECK-CVT-NEXT:    fcvtzu x12, s4
+; CHECK-CVT-NEXT:    fcvt s3, h7
+; CHECK-CVT-NEXT:    cmp x9, x8
+; CHECK-CVT-NEXT:    fcvtzu x14, s1
+; CHECK-CVT-NEXT:    csel x4, x9, x8, lo
+; CHECK-CVT-NEXT:    cmp x10, x8
+; CHECK-CVT-NEXT:    fcvtzu x9, s2
+; CHECK-CVT-NEXT:    csel x5, x10, x8, lo
+; CHECK-CVT-NEXT:    cmp x11, x8
+; CHECK-CVT-NEXT:    fcvtzu x10, s3
+; CHECK-CVT-NEXT:    csel x6, x11, x8, lo
+; CHECK-CVT-NEXT:    cmp x12, x8
+; CHECK-CVT-NEXT:    csel x7, x12, x8, lo
+; CHECK-CVT-NEXT:    cmp x13, x8
+; CHECK-CVT-NEXT:    csel x0, x13, x8, lo
+; CHECK-CVT-NEXT:    cmp x14, x8
+; CHECK-CVT-NEXT:    csel x1, x14, x8, lo
+; CHECK-CVT-NEXT:    cmp x9, x8
+; CHECK-CVT-NEXT:    csel x2, x9, x8, lo
+; CHECK-CVT-NEXT:    cmp x10, x8
+; CHECK-CVT-NEXT:    csel x3, x10, x8, lo
+; CHECK-CVT-NEXT:    ret
 ;
-; CHECK-GI-CVT-LABEL: test_unsigned_v8f16_v8i50:
-; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-GI-CVT-NEXT:    mov h5, v0.h[1]
-; CHECK-GI-CVT-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
-; CHECK-GI-CVT-NEXT:    mov h6, v0.h[2]
-; CHECK-GI-CVT-NEXT:    mov h7, v0.h[3]
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    mov h2, v1.h[1]
-; CHECK-GI-CVT-NEXT:    mov h3, v1.h[2]
-; CHECK-GI-CVT-NEXT:    mov h4, v1.h[3]
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
-; CHECK-GI-CVT-NEXT:    fcvtzu x13, s0
-; CHECK-GI-CVT-NEXT:    fcvt s2, h2
-; CHECK-GI-CVT-NEXT:    fcvt s3, h3
-; CHECK-GI-CVT-NEXT:    fcvt s4, h4
-; CHECK-GI-CVT-NEXT:    fcvtzu x9, s1
-; CHECK-GI-CVT-NEXT:    fcvt s1, h5
-; CHECK-GI-CVT-NEXT:    fcvtzu x10, s2
-; CHECK-GI-CVT-NEXT:    fcvtzu x11, s3
-; CHECK-GI-CVT-NEXT:    fcvt s2, h6
-; CHECK-GI-CVT-NEXT:    fcvtzu x12, s4
-; CHECK-GI-CVT-NEXT:    fcvt s3, h7
-; CHECK-GI-CVT-NEXT:    cmp x9, x8
-; CHECK-GI-CVT-NEXT:    fcvtzu x14, s1
-; CHECK-GI-CVT-NEXT:    csel x4, x9, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x10, x8
-; CHECK-GI-CVT-NEXT:    fcvtzu x9, s2
-; CHECK-GI-CVT-NEXT:    csel x5, x10, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x11, x8
-; CHECK-GI-CVT-NEXT:    fcvtzu x10, s3
-; CHECK-GI-CVT-NEXT:    csel x6, x11, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x12, x8
-; CHECK-GI-CVT-NEXT:    csel x7, x12, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x13, x8
-; CHECK-GI-CVT-NEXT:    csel x0, x13, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x14, x8
-; CHECK-GI-CVT-NEXT:    csel x1, x14, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x9, x8
-; CHECK-GI-CVT-NEXT:    csel x2, x9, x8, lo
-; CHECK-GI-CVT-NEXT:    cmp x10, x8
-; CHECK-GI-CVT-NEXT:    csel x3, x10, x8, lo
-; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v8f16_v8i50:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-GI-FP16-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
-; CHECK-GI-FP16-NEXT:    fcvtzu x13, h0
-; CHECK-GI-FP16-NEXT:    mov h2, v1.h[1]
-; CHECK-GI-FP16-NEXT:    mov h3, v1.h[2]
-; CHECK-GI-FP16-NEXT:    mov h4, v1.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzu x9, h1
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    fcvtzu x10, h2
-; CHECK-GI-FP16-NEXT:    fcvtzu x11, h3
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    fcvtzu x12, h4
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    cmp x9, x8
-; CHECK-GI-FP16-NEXT:    fcvtzu x14, h1
-; CHECK-GI-FP16-NEXT:    csel x4, x9, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x10, x8
-; CHECK-GI-FP16-NEXT:    fcvtzu x9, h2
-; CHECK-GI-FP16-NEXT:    csel x5, x10, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x11, x8
-; CHECK-GI-FP16-NEXT:    fcvtzu x10, h3
-; CHECK-GI-FP16-NEXT:    csel x6, x11, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x12, x8
-; CHECK-GI-FP16-NEXT:    csel x7, x12, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x13, x8
-; CHECK-GI-FP16-NEXT:    csel x0, x13, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x14, x8
-; CHECK-GI-FP16-NEXT:    csel x1, x14, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x9, x8
-; CHECK-GI-FP16-NEXT:    csel x2, x9, x8, lo
-; CHECK-GI-FP16-NEXT:    cmp x10, x8
-; CHECK-GI-FP16-NEXT:    csel x3, x10, x8, lo
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v8f16_v8i50:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-NEXT:    mov x8, #1125899906842623 // =0x3ffffffffffff
+; CHECK-FP16-NEXT:    fcvtzu x13, h0
+; CHECK-FP16-NEXT:    mov h2, v1.h[1]
+; CHECK-FP16-NEXT:    mov h3, v1.h[2]
+; CHECK-FP16-NEXT:    mov h4, v1.h[3]
+; CHECK-FP16-NEXT:    fcvtzu x9, h1
+; CHECK-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-FP16-NEXT:    fcvtzu x10, h2
+; CHECK-FP16-NEXT:    fcvtzu x11, h3
+; CHECK-FP16-NEXT:    mov h2, v0.h[2]
+; CHECK-FP16-NEXT:    fcvtzu x12, h4
+; CHECK-FP16-NEXT:    mov h3, v0.h[3]
+; CHECK-FP16-NEXT:    cmp x9, x8
+; CHECK-FP16-NEXT:    fcvtzu x14, h1
+; CHECK-FP16-NEXT:    csel x4, x9, x8, lo
+; CHECK-FP16-NEXT:    cmp x10, x8
+; CHECK-FP16-NEXT:    fcvtzu x9, h2
+; CHECK-FP16-NEXT:    csel x5, x10, x8, lo
+; CHECK-FP16-NEXT:    cmp x11, x8
+; CHECK-FP16-NEXT:    fcvtzu x10, h3
+; CHECK-FP16-NEXT:    csel x6, x11, x8, lo
+; CHECK-FP16-NEXT:    cmp x12, x8
+; CHECK-FP16-NEXT:    csel x7, x12, x8, lo
+; CHECK-FP16-NEXT:    cmp x13, x8
+; CHECK-FP16-NEXT:    csel x0, x13, x8, lo
+; CHECK-FP16-NEXT:    cmp x14, x8
+; CHECK-FP16-NEXT:    csel x1, x14, x8, lo
+; CHECK-FP16-NEXT:    cmp x9, x8
+; CHECK-FP16-NEXT:    csel x2, x9, x8, lo
+; CHECK-FP16-NEXT:    cmp x10, x8
+; CHECK-FP16-NEXT:    csel x3, x10, x8, lo
+; CHECK-FP16-NEXT:    ret
     %x = call <8 x i50> @llvm.fptoui.sat.v8f16.v8i50(<8 x half> %f)
     ret <8 x i50> %x
 }
@@ -3865,23 +3626,13 @@ define <16 x i8> @test_unsigned_v16f32_v16i8(<16 x float> %f) {
 }
 
 define <8 x i16> @test_unsigned_v8f32_v8i16(<8 x float> %f) {
-; CHECK-SD-LABEL: test_unsigned_v8f32_v8i16:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-SD-NEXT:    uqxtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    uqxtn2 v0.8h, v1.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_unsigned_v8f32_v8i16:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v2.2d, #0x00ffff0000ffff
-; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-NEXT:    umin v0.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT:    umin v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: test_unsigned_v8f32_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-NEXT:    uqxtn v0.4h, v0.4s
+; CHECK-NEXT:    uqxtn2 v0.8h, v1.4s
+; CHECK-NEXT:    ret
     %x = call <8 x i16> @llvm.fptoui.sat.v8f32.v8i16(<8 x float> %f)
     ret <8 x i16> %x
 }
@@ -3901,17 +3652,14 @@ define <16 x i16> @test_unsigned_v16f32_v16i16(<16 x float> %f) {
 ;
 ; CHECK-GI-LABEL: test_unsigned_v16f32_v16i16:
 ; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    movi v4.2d, #0x00ffff0000ffff
 ; CHECK-GI-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-NEXT:    fcvtzu v1.4s, v1.4s
 ; CHECK-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-GI-NEXT:    fcvtzu v4.4s, v1.4s
 ; CHECK-GI-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-NEXT:    umin v0.4s, v0.4s, v4.4s
-; CHECK-GI-NEXT:    umin v1.4s, v1.4s, v4.4s
-; CHECK-GI-NEXT:    umin v2.4s, v2.4s, v4.4s
-; CHECK-GI-NEXT:    umin v3.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-GI-NEXT:    uzp1 v1.8h, v2.8h, v3.8h
+; CHECK-GI-NEXT:    uqxtn v0.4h, v0.4s
+; CHECK-GI-NEXT:    uqxtn v1.4h, v2.4s
+; CHECK-GI-NEXT:    uqxtn2 v0.8h, v4.4s
+; CHECK-GI-NEXT:    uqxtn2 v1.8h, v3.4s
 ; CHECK-GI-NEXT:    ret
     %x = call <16 x i16> @llvm.fptoui.sat.v16f32.v16i16(<16 x float> %f)
     ret <16 x i16> %x
@@ -3940,13 +3688,13 @@ define <16 x i8> @test_unsigned_v16f16_v16i8(<16 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_unsigned_v16f16_v16i8:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    uqxtn v0.8b, v0.8h
-; CHECK-SD-FP16-NEXT:    uqxtn2 v0.16b, v1.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v16f16_v16i8:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-FP16-NEXT:    uqxtn v0.8b, v0.8h
+; CHECK-FP16-NEXT:    uqxtn2 v0.16b, v1.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_unsigned_v16f16_v16i8:
 ; CHECK-GI-CVT:       // %bb.0:
@@ -3967,16 +3715,6 @@ define <16 x i8> @test_unsigned_v16f16_v16i8(<16 x half> %f) {
 ; CHECK-GI-CVT-NEXT:    uzp1 v1.8h, v4.8h, v1.8h
 ; CHECK-GI-CVT-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v16f16_v16i8:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    movi v2.2d, #0xff00ff00ff00ff
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    umin v0.8h, v0.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    umin v1.8h, v1.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <16 x i8> @llvm.fptoui.sat.v16f16.v16i8(<16 x half> %f)
     ret <16 x i8> %x
 }
@@ -3998,36 +3736,27 @@ define <16 x i16> @test_unsigned_v16f16_v16i16(<16 x half> %f) {
 ; CHECK-SD-CVT-NEXT:    uqxtn2 v1.8h, v3.4s
 ; CHECK-SD-CVT-NEXT:    ret
 ;
-; CHECK-SD-FP16-LABEL: test_unsigned_v16f16_v16i16:
-; CHECK-SD-FP16:       // %bb.0:
-; CHECK-SD-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    ret
+; CHECK-FP16-LABEL: test_unsigned_v16f16_v16i16:
+; CHECK-FP16:       // %bb.0:
+; CHECK-FP16-NEXT:    fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT:    fcvtzu v1.8h, v1.8h
+; CHECK-FP16-NEXT:    ret
 ;
 ; CHECK-GI-CVT-LABEL: test_unsigned_v16f16_v16i16:
 ; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    fcvtl v3.4s, v0.4h
+; CHECK-GI-CVT-NEXT:    fcvtl v2.4s, v0.4h
+; CHECK-GI-CVT-NEXT:    fcvtl v3.4s, v1.4h
 ; CHECK-GI-CVT-NEXT:    fcvtl2 v0.4s, v0.8h
-; CHECK-GI-CVT-NEXT:    fcvtl v4.4s, v1.4h
 ; CHECK-GI-CVT-NEXT:    fcvtl2 v1.4s, v1.8h
-; CHECK-GI-CVT-NEXT:    movi v2.2d, #0x00ffff0000ffff
+; CHECK-GI-CVT-NEXT:    fcvtzu v2.4s, v2.4s
 ; CHECK-GI-CVT-NEXT:    fcvtzu v3.4s, v3.4s
-; CHECK-GI-CVT-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-GI-CVT-NEXT:    fcvtzu v4.4s, v4.4s
-; CHECK-GI-CVT-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-GI-CVT-NEXT:    umin v3.4s, v3.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    umin v0.4s, v0.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    umin v4.4s, v4.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    umin v1.4s, v1.4s, v2.4s
-; CHECK-GI-CVT-NEXT:    uzp1 v0.8h, v3.8h, v0.8h
-; CHECK-GI-CVT-NEXT:    uzp1 v1.8h, v4.8h, v1.8h
+; CHECK-GI-CVT-NEXT:    fcvtzu v4.4s, v0.4s
+; CHECK-GI-CVT-NEXT:    fcvtzu v5.4s, v1.4s
+; CHECK-GI-CVT-NEXT:    uqxtn v0.4h, v2.4s
+; CHECK-GI-CVT-NEXT:    uqxtn v1.4h, v3.4s
+; CHECK-GI-CVT-NEXT:    uqxtn2 v0.8h, v4.4s
+; CHECK-GI-CVT-NEXT:    uqxtn2 v1.8h, v5.4s
 ; CHECK-GI-CVT-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: test_unsigned_v16f16_v16i16:
-; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    fcvtzu v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    ret
     %x = call <16 x i16> @llvm.fptoui.sat.v16f16.v16i16(<16 x half> %f)
     ret <16 x i16> %x
 }
@@ -4419,7 +4148,7 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
 ; CHECK-SD-NEXT:    ldr q1, [x8, :lo12:.LCPI86_1]
-; CHECK-SD-NEXT:    csel x19, xzr, x0, lt
+; CHECK-SD-NEXT:    csel x19, xzr, x0, mi
 ; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
@@ -4435,7 +4164,7 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-SD-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
 ; CHECK-SD-NEXT:    cmp w19, #0
-; CHECK-SD-NEXT:    csel x19, xzr, x0, lt
+; CHECK-SD-NEXT:    csel x19, xzr, x0, mi
 ; CHECK-SD-NEXT:    bl __gttf2
 ; CHECK-SD-NEXT:    cmp w0, #0
 ; CHECK-SD-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
@@ -4480,9 +4209,9 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov x21, #-562949953421312 // =0xfffe000000000000
 ; CHECK-GI-NEXT:    cmp w0, #0
 ; CHECK-GI-NEXT:    mov x22, #4629418941960159231 // =0x403effffffffffff
-; CHECK-GI-NEXT:    csel x8, x19, x21, lt
+; CHECK-GI-NEXT:    csel x8, x19, x21, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x20, x22, lt
+; CHECK-GI-NEXT:    csel x8, x20, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfdi
 ; CHECK-GI-NEXT:    ldp q1, q0, [sp, #16] // 32-byte Folded Reload
@@ -4499,9 +4228,9 @@ define <2 x i64> @test_signed_v2f128_v2i64(<2 x fp128> %f) {
 ; CHECK-GI-NEXT:    mov v0.d[1], x23
 ; CHECK-GI-NEXT:    bl __lttf2
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x8, x20, x21, lt
+; CHECK-GI-NEXT:    csel x8, x20, x21, mi
 ; CHECK-GI-NEXT:    mov v0.d[0], x8
-; CHECK-GI-NEXT:    csel x8, x23, x22, lt
+; CHECK-GI-NEXT:    csel x8, x23, x22, mi
 ; CHECK-GI-NEXT:    mov v0.d[1], x8
 ; CHECK-GI-NEXT:    bl __fixunstfdi
 ; CHECK-GI-NEXT:    fmov d0, x19
diff --git a/llvm/test/CodeGen/AArch64/freeze.ll b/llvm/test/CodeGen/AArch64/freeze.ll
index d428b6aa483a7..36d88c5524932 100644
--- a/llvm/test/CodeGen/AArch64/freeze.ll
+++ b/llvm/test/CodeGen/AArch64/freeze.ll
@@ -429,3 +429,37 @@ define <8 x i16> @freeze_abds(<8 x i16> %a, <8 x i16> %b) {
   %r = add <8 x i16> %a, %f
   ret <8 x i16> %r
 }
+
+define i32 @freeze_scmp(i32 %a0) nounwind {
+; CHECK-LABEL: freeze_scmp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2 // =0x2
+; CHECK-NEXT:    cmp w8, w0
+; CHECK-NEXT:    cset w8, gt
+; CHECK-NEXT:    csinv w8, w8, wzr, ge
+; CHECK-NEXT:    cmp wzr, w8
+; CHECK-NEXT:    cset w8, gt
+; CHECK-NEXT:    csinv w0, w8, wzr, ge
+; CHECK-NEXT:    ret
+  %x = call i32 @llvm.scmp.i32(i32 2, i32 %a0)
+  %y = freeze i32 %x
+  %z = call i32 @llvm.scmp.i32(i32 0, i32 %y)
+  ret i32 %z
+}
+
+define i32 @freeze_ucmp(i32 %a0) nounwind {
+; CHECK-LABEL: freeze_ucmp:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2 // =0x2
+; CHECK-NEXT:    cmp w8, w0
+; CHECK-NEXT:    cset w8, hi
+; CHECK-NEXT:    csinv w8, w8, wzr, hs
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    cset w8, hi
+; CHECK-NEXT:    csinv w0, w8, wzr, hs
+; CHECK-NEXT:    ret
+  %x = call i32 @llvm.ucmp.i32(i32 2, i32 %a0)
+  %y = freeze i32 %x
+  %z = call i32 @llvm.ucmp.i32(i32 %y, i32 1)
+  ret i32 %z
+}
diff --git a/llvm/test/CodeGen/AArch64/frexp-arm64ec.ll b/llvm/test/CodeGen/AArch64/frexp-arm64ec.ll
index ee326caa77c0a..c27d3c9588b9d 100644
--- a/llvm/test/CodeGen/AArch64/frexp-arm64ec.ll
+++ b/llvm/test/CodeGen/AArch64/frexp-arm64ec.ll
@@ -2,6 +2,15 @@
 
 ; Separate from llvm-frexp.ll test because this errors on half cases
 
+; ARM64EC-LABEL: test_frexp_f16_i32
+; ARM64EC: fcvt d0, h0
+; ARM64EC: bl "#frexp"
+; ARM64EC: fcvt h0, d0
+define { half, i32 } @test_frexp_f16_i32(half %a) {
+  %result = call { half, i32 } @llvm.frexp.f16.i32(half %a)
+  ret { half, i32 } %result
+}
+
 ; ARM64EC-LABEL: test_frexp_f32_i32
 ; ARM64EC: fcvt d0, s0
 ; ARM64EC: bl "#frexp"
diff --git a/llvm/test/CodeGen/AArch64/fsh.ll b/llvm/test/CodeGen/AArch64/fsh.ll
index 4c28c90824028..ae2ef2649102e 100644
--- a/llvm/test/CodeGen/AArch64/fsh.ll
+++ b/llvm/test/CodeGen/AArch64/fsh.ll
@@ -2509,87 +2509,88 @@ define <7 x i32> @fshl_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) {
 ;
 ; CHECK-GI-LABEL: fshl_v7i32:
 ; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldr s3, [sp, #48]
-; CHECK-GI-NEXT:    ldr s20, [sp, #56]
-; CHECK-GI-NEXT:    add x9, sp, #56
+; CHECK-GI-NEXT:    ldr s17, [sp, #48]
+; CHECK-GI-NEXT:    add x8, sp, #56
+; CHECK-GI-NEXT:    add x9, sp, #64
 ; CHECK-GI-NEXT:    ldr s4, [sp, #48]
-; CHECK-GI-NEXT:    ldr s7, [sp, #80]
-; CHECK-GI-NEXT:    mov w12, #-1 // =0xffffffff
-; CHECK-GI-NEXT:    ldr s21, [sp, #88]
-; CHECK-GI-NEXT:    mov v3.s[1], v20.s[0]
-; CHECK-GI-NEXT:    fmov s20, w12
-; CHECK-GI-NEXT:    ld1 { v4.s }[1], [x9]
-; CHECK-GI-NEXT:    ldr s17, [sp]
-; CHECK-GI-NEXT:    add x13, sp, #64
-; CHECK-GI-NEXT:    mov v7.s[1], v21.s[0]
+; CHECK-GI-NEXT:    ldr s21, [sp, #56]
+; CHECK-GI-NEXT:    mov w10, #-1 // =0xffffffff
+; CHECK-GI-NEXT:    ld1 { v17.s }[1], [x8]
+; CHECK-GI-NEXT:    ldr s20, [x9]
+; CHECK-GI-NEXT:    add x8, sp, #72
+; CHECK-GI-NEXT:    mov v4.s[1], v21.s[0]
 ; CHECK-GI-NEXT:    fmov s21, w7
+; CHECK-GI-NEXT:    ldr s6, [sp]
+; CHECK-GI-NEXT:    ld1 { v20.s }[1], [x8]
 ; CHECK-GI-NEXT:    ldr s19, [sp, #64]
-; CHECK-GI-NEXT:    mov w11, #31 // =0x1f
-; CHECK-GI-NEXT:    mov v20.s[1], w12
+; CHECK-GI-NEXT:    ldr s7, [sp, #80]
+; CHECK-GI-NEXT:    ldr s22, [sp, #88]
+; CHECK-GI-NEXT:    mov w9, #31 // =0x1f
+; CHECK-GI-NEXT:    mov w11, #1 // =0x1
+; CHECK-GI-NEXT:    mov v21.s[1], v6.s[0]
+; CHECK-GI-NEXT:    fmov s6, w9
 ; CHECK-GI-NEXT:    ldr s18, [sp, #96]
-; CHECK-GI-NEXT:    ld1 { v4.s }[2], [x13]
-; CHECK-GI-NEXT:    mov w13, #1 // =0x1
-; CHECK-GI-NEXT:    mov v3.s[2], v19.s[0]
-; CHECK-GI-NEXT:    mov v21.s[1], v17.s[0]
-; CHECK-GI-NEXT:    fmov s17, w11
-; CHECK-GI-NEXT:    fmov s19, w13
+; CHECK-GI-NEXT:    zip1 v17.2d, v17.2d, v20.2d
+; CHECK-GI-NEXT:    fmov s20, w10
+; CHECK-GI-NEXT:    mov v7.s[1], v22.s[0]
+; CHECK-GI-NEXT:    mov v4.s[2], v19.s[0]
+; CHECK-GI-NEXT:    fmov s19, w11
 ; CHECK-GI-NEXT:    fmov s23, w0
-; CHECK-GI-NEXT:    fmov s24, w11
-; CHECK-GI-NEXT:    ldr s6, [sp, #8]
+; CHECK-GI-NEXT:    mov v6.s[1], w9
+; CHECK-GI-NEXT:    fmov s24, w9
+; CHECK-GI-NEXT:    ldr s2, [sp, #8]
+; CHECK-GI-NEXT:    mov v20.s[1], w10
 ; CHECK-GI-NEXT:    ldr s0, [sp, #24]
 ; CHECK-GI-NEXT:    ldr s5, [sp, #32]
+; CHECK-GI-NEXT:    mov v19.s[1], w11
 ; CHECK-GI-NEXT:    mov v7.s[2], v18.s[0]
-; CHECK-GI-NEXT:    mov v17.s[1], w11
-; CHECK-GI-NEXT:    mov v19.s[1], w13
-; CHECK-GI-NEXT:    mov v20.s[2], w12
 ; CHECK-GI-NEXT:    ldr s16, [sp, #72]
 ; CHECK-GI-NEXT:    mov v23.s[1], w1
 ; CHECK-GI-NEXT:    ldr s18, [sp, #80]
-; CHECK-GI-NEXT:    mov v21.s[2], v6.s[0]
-; CHECK-GI-NEXT:    mov v24.s[1], w11
+; CHECK-GI-NEXT:    mov v21.s[2], v2.s[0]
+; CHECK-GI-NEXT:    mov v24.s[1], w9
 ; CHECK-GI-NEXT:    mov v0.s[1], v5.s[0]
-; CHECK-GI-NEXT:    fmov s6, w4
-; CHECK-GI-NEXT:    add x10, sp, #88
+; CHECK-GI-NEXT:    fmov s5, w4
+; CHECK-GI-NEXT:    mov v20.s[2], w10
+; CHECK-GI-NEXT:    add x8, sp, #88
 ; CHECK-GI-NEXT:    movi v22.4s, #31
-; CHECK-GI-NEXT:    mov v3.s[3], v16.s[0]
-; CHECK-GI-NEXT:    mov v17.s[2], w11
-; CHECK-GI-NEXT:    mov v19.s[2], w13
-; CHECK-GI-NEXT:    ldr s2, [sp, #16]
-; CHECK-GI-NEXT:    ldr s1, [sp, #40]
-; CHECK-GI-NEXT:    ld1 { v18.s }[1], [x10]
-; CHECK-GI-NEXT:    eor v5.16b, v7.16b, v20.16b
+; CHECK-GI-NEXT:    mov v4.s[3], v16.s[0]
+; CHECK-GI-NEXT:    mov v6.s[2], w9
+; CHECK-GI-NEXT:    mov v19.s[2], w11
+; CHECK-GI-NEXT:    ldr s1, [sp, #16]
+; CHECK-GI-NEXT:    ldr s3, [sp, #40]
+; CHECK-GI-NEXT:    ld1 { v18.s }[1], [x8]
 ; CHECK-GI-NEXT:    mov v23.s[2], w2
-; CHECK-GI-NEXT:    mov v6.s[1], w5
-; CHECK-GI-NEXT:    add x8, sp, #72
-; CHECK-GI-NEXT:    add x9, sp, #96
-; CHECK-GI-NEXT:    mov v21.s[3], v2.s[0]
-; CHECK-GI-NEXT:    mov v24.s[2], w11
-; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-GI-NEXT:    ld1 { v4.s }[3], [x8]
-; CHECK-GI-NEXT:    bic v2.16b, v22.16b, v3.16b
-; CHECK-GI-NEXT:    ld1 { v18.s }[2], [x9]
-; CHECK-GI-NEXT:    and v1.16b, v5.16b, v17.16b
+; CHECK-GI-NEXT:    mov v5.s[1], w5
+; CHECK-GI-NEXT:    add x8, sp, #96
+; CHECK-GI-NEXT:    eor v2.16b, v7.16b, v20.16b
+; CHECK-GI-NEXT:    mov v21.s[3], v1.s[0]
+; CHECK-GI-NEXT:    mov v24.s[2], w9
+; CHECK-GI-NEXT:    mov v0.s[2], v3.s[0]
+; CHECK-GI-NEXT:    bic v1.16b, v22.16b, v4.16b
+; CHECK-GI-NEXT:    ld1 { v18.s }[2], [x8]
 ; CHECK-GI-NEXT:    neg v3.4s, v19.4s
+; CHECK-GI-NEXT:    and v4.16b, v17.16b, v22.16b
+; CHECK-GI-NEXT:    and v2.16b, v2.16b, v6.16b
 ; CHECK-GI-NEXT:    mov v23.s[3], w3
-; CHECK-GI-NEXT:    mov v6.s[2], w6
-; CHECK-GI-NEXT:    and v4.16b, v4.16b, v22.16b
-; CHECK-GI-NEXT:    ushr v5.4s, v21.4s, #1
-; CHECK-GI-NEXT:    neg v2.4s, v2.4s
-; CHECK-GI-NEXT:    and v7.16b, v18.16b, v24.16b
+; CHECK-GI-NEXT:    mov v5.s[2], w6
+; CHECK-GI-NEXT:    ushr v6.4s, v21.4s, #1
 ; CHECK-GI-NEXT:    neg v1.4s, v1.4s
+; CHECK-GI-NEXT:    and v7.16b, v18.16b, v24.16b
 ; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v3.4s
+; CHECK-GI-NEXT:    neg v2.4s, v2.4s
 ; CHECK-GI-NEXT:    ushl v3.4s, v23.4s, v4.4s
-; CHECK-GI-NEXT:    ushl v2.4s, v5.4s, v2.4s
-; CHECK-GI-NEXT:    ushl v4.4s, v6.4s, v7.4s
-; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v2.16b
+; CHECK-GI-NEXT:    ushl v1.4s, v6.4s, v1.4s
+; CHECK-GI-NEXT:    ushl v4.4s, v5.4s, v7.4s
+; CHECK-GI-NEXT:    ushl v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT:    orr v1.16b, v3.16b, v1.16b
 ; CHECK-GI-NEXT:    orr v0.16b, v4.16b, v0.16b
 ; CHECK-GI-NEXT:    mov s2, v1.s[1]
 ; CHECK-GI-NEXT:    mov s3, v1.s[2]
 ; CHECK-GI-NEXT:    mov s4, v1.s[3]
+; CHECK-GI-NEXT:    fmov w0, s1
 ; CHECK-GI-NEXT:    mov s5, v0.s[1]
 ; CHECK-GI-NEXT:    mov s6, v0.s[2]
-; CHECK-GI-NEXT:    fmov w0, s1
 ; CHECK-GI-NEXT:    fmov w4, s0
 ; CHECK-GI-NEXT:    fmov w1, s2
 ; CHECK-GI-NEXT:    fmov w2, s3
diff --git a/llvm/test/CodeGen/AArch64/icmp-cst.ll b/llvm/test/CodeGen/AArch64/icmp-cst.ll
index b6f452bb42cec..b75e3535bf821 100644
--- a/llvm/test/CodeGen/AArch64/icmp-cst.ll
+++ b/llvm/test/CodeGen/AArch64/icmp-cst.ll
@@ -1,687 +1,415 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=aarch64-linux-gnu -global-isel=0 < %s | FileCheck %s --check-prefix=CHECK-SD
-; RUN: llc -mtriple=aarch64-linux-gnu -global-isel=1 < %s | FileCheck %s --check-prefix=CHECK-GI
+; RUN: llc -mtriple=aarch64-linux-gnu -global-isel=0 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-linux-gnu -global-isel=1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define i1 @ule_11111111(i32 noundef %in) {
-; CHECK-SD-LABEL: ule_11111111:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #4370 // =0x1112
-; CHECK-SD-NEXT:    movk w8, #4369, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: ule_11111111:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #286331153 // =0x11111111
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: ule_11111111:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #286331153 // =0x11111111
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, 286331154
   ret i1 %out
 }
 
 define i1 @ule_22222222(i32 noundef %in) {
-; CHECK-SD-LABEL: ule_22222222:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #8739 // =0x2223
-; CHECK-SD-NEXT:    movk w8, #8738, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: ule_22222222:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #572662306 // =0x22222222
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: ule_22222222:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #572662306 // =0x22222222
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, 572662307
   ret i1 %out
 }
 
 define i1 @ule_33333333(i32 noundef %in) {
-; CHECK-SD-LABEL: ule_33333333:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #13108 // =0x3334
-; CHECK-SD-NEXT:    movk w8, #13107, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: ule_33333333:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #858993459 // =0x33333333
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: ule_33333333:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #858993459 // =0x33333333
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, 858993460
   ret i1 %out
 }
 
 define i1 @ule_44444444(i32 noundef %in) {
-; CHECK-SD-LABEL: ule_44444444:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #17477 // =0x4445
-; CHECK-SD-NEXT:    movk w8, #17476, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: ule_44444444:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1145324612 // =0x44444444
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: ule_44444444:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1145324612 // =0x44444444
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, 1145324613
   ret i1 %out
 }
 
 define i1 @ule_55555555(i32 noundef %in) {
-; CHECK-SD-LABEL: ule_55555555:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #21846 // =0x5556
-; CHECK-SD-NEXT:    movk w8, #21845, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: ule_55555555:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1431655765 // =0x55555555
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: ule_55555555:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1431655765 // =0x55555555
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, 1431655766
   ret i1 %out
 }
 
 define i1 @ule_66666666(i32 noundef %in) {
-; CHECK-SD-LABEL: ule_66666666:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #26215 // =0x6667
-; CHECK-SD-NEXT:    movk w8, #26214, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: ule_66666666:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1717986918 // =0x66666666
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: ule_66666666:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1717986918 // =0x66666666
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, 1717986919
   ret i1 %out
 }
 
 define i1 @ule_77777777(i32 noundef %in) {
-; CHECK-SD-LABEL: ule_77777777:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #30584 // =0x7778
-; CHECK-SD-NEXT:    movk w8, #30583, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: ule_77777777:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #2004318071 // =0x77777777
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: ule_77777777:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2004318071 // =0x77777777
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, 2004318072
   ret i1 %out
 }
 
 define i1 @ule_88888888(i32 noundef %in) {
-; CHECK-SD-LABEL: ule_88888888:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #34953 // =0x8889
-; CHECK-SD-NEXT:    movk w8, #34952, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: ule_88888888:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #-2004318072 // =0x88888888
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: ule_88888888:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-2004318072 // =0x88888888
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, -2004318071
   ret i1 %out
 }
 
 define i1 @ule_99999999(i32 noundef %in) {
-; CHECK-SD-LABEL: ule_99999999:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #39322 // =0x999a
-; CHECK-SD-NEXT:    movk w8, #39321, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: ule_99999999:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #-1717986919 // =0x99999999
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: ule_99999999:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1717986919 // =0x99999999
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, -1717986918
   ret i1 %out
 }
 
 define i1 @uge_11111111(i32 noundef %in) {
-; CHECK-SD-LABEL: uge_11111111:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #4368 // =0x1110
-; CHECK-SD-NEXT:    movk w8, #4369, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: uge_11111111:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #286331153 // =0x11111111
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: uge_11111111:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #286331153 // =0x11111111
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, 286331152
   ret i1 %out
 }
 
 define i1 @uge_22222222(i32 noundef %in) {
-; CHECK-SD-LABEL: uge_22222222:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #8737 // =0x2221
-; CHECK-SD-NEXT:    movk w8, #8738, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: uge_22222222:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #572662306 // =0x22222222
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: uge_22222222:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #572662306 // =0x22222222
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, 572662305
   ret i1 %out
 }
 
 define i1 @uge_33333333(i32 noundef %in) {
-; CHECK-SD-LABEL: uge_33333333:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #13106 // =0x3332
-; CHECK-SD-NEXT:    movk w8, #13107, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: uge_33333333:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #858993459 // =0x33333333
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: uge_33333333:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #858993459 // =0x33333333
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, 858993458
   ret i1 %out
 }
 
 define i1 @uge_44444444(i32 noundef %in) {
-; CHECK-SD-LABEL: uge_44444444:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #17475 // =0x4443
-; CHECK-SD-NEXT:    movk w8, #17476, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: uge_44444444:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1145324612 // =0x44444444
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: uge_44444444:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1145324612 // =0x44444444
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, 1145324611
   ret i1 %out
 }
 
 define i1 @uge_55555555(i32 noundef %in) {
-; CHECK-SD-LABEL: uge_55555555:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #21844 // =0x5554
-; CHECK-SD-NEXT:    movk w8, #21845, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: uge_55555555:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1431655765 // =0x55555555
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: uge_55555555:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1431655765 // =0x55555555
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, 1431655764
   ret i1 %out
 }
 
 define i1 @uge_66666666(i32 noundef %in) {
-; CHECK-SD-LABEL: uge_66666666:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #26213 // =0x6665
-; CHECK-SD-NEXT:    movk w8, #26214, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: uge_66666666:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1717986918 // =0x66666666
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: uge_66666666:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1717986918 // =0x66666666
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, 1717986917
   ret i1 %out
 }
 
 define i1 @uge_77777777(i32 noundef %in) {
-; CHECK-SD-LABEL: uge_77777777:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #30582 // =0x7776
-; CHECK-SD-NEXT:    movk w8, #30583, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: uge_77777777:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #2004318071 // =0x77777777
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: uge_77777777:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2004318071 // =0x77777777
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, 2004318070
   ret i1 %out
 }
 
 define i1 @uge_88888888(i32 noundef %in) {
-; CHECK-SD-LABEL: uge_88888888:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #34951 // =0x8887
-; CHECK-SD-NEXT:    movk w8, #34952, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: uge_88888888:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #-2004318072 // =0x88888888
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: uge_88888888:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-2004318072 // =0x88888888
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, -2004318073
   ret i1 %out
 }
 
 define i1 @uge_99999999(i32 noundef %in) {
-; CHECK-SD-LABEL: uge_99999999:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #39320 // =0x9998
-; CHECK-SD-NEXT:    movk w8, #39321, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: uge_99999999:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #-1717986919 // =0x99999999
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: uge_99999999:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1717986919 // =0x99999999
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, -1717986920
   ret i1 %out
 }
 
 define i1 @sle_11111111(i32 noundef %in) {
-; CHECK-SD-LABEL: sle_11111111:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #4370 // =0x1112
-; CHECK-SD-NEXT:    movk w8, #4369, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sle_11111111:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #286331153 // =0x11111111
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, le
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sle_11111111:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #286331153 // =0x11111111
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
   %out = icmp slt i32 %in, 286331154
   ret i1 %out
 }
 
 define i1 @sle_22222222(i32 noundef %in) {
-; CHECK-SD-LABEL: sle_22222222:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #8739 // =0x2223
-; CHECK-SD-NEXT:    movk w8, #8738, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sle_22222222:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #572662306 // =0x22222222
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, le
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sle_22222222:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #572662306 // =0x22222222
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
   %out = icmp slt i32 %in, 572662307
   ret i1 %out
 }
 
 define i1 @sle_33333333(i32 noundef %in) {
-; CHECK-SD-LABEL: sle_33333333:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #13108 // =0x3334
-; CHECK-SD-NEXT:    movk w8, #13107, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sle_33333333:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #858993459 // =0x33333333
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, le
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sle_33333333:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #858993459 // =0x33333333
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
   %out = icmp slt i32 %in, 858993460
   ret i1 %out
 }
 
 define i1 @sle_44444444(i32 noundef %in) {
-; CHECK-SD-LABEL: sle_44444444:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #17477 // =0x4445
-; CHECK-SD-NEXT:    movk w8, #17476, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sle_44444444:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1145324612 // =0x44444444
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, le
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sle_44444444:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1145324612 // =0x44444444
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
   %out = icmp slt i32 %in, 1145324613
   ret i1 %out
 }
 
 define i1 @sle_55555555(i32 noundef %in) {
-; CHECK-SD-LABEL: sle_55555555:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #21846 // =0x5556
-; CHECK-SD-NEXT:    movk w8, #21845, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sle_55555555:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1431655765 // =0x55555555
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, le
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sle_55555555:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1431655765 // =0x55555555
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
   %out = icmp slt i32 %in, 1431655766
   ret i1 %out
 }
 
 define i1 @sle_66666666(i32 noundef %in) {
-; CHECK-SD-LABEL: sle_66666666:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #26215 // =0x6667
-; CHECK-SD-NEXT:    movk w8, #26214, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sle_66666666:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1717986918 // =0x66666666
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, le
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sle_66666666:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1717986918 // =0x66666666
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
   %out = icmp slt i32 %in, 1717986919
   ret i1 %out
 }
 
 define i1 @sle_77777777(i32 noundef %in) {
-; CHECK-SD-LABEL: sle_77777777:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #30584 // =0x7778
-; CHECK-SD-NEXT:    movk w8, #30583, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sle_77777777:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #2004318071 // =0x77777777
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, le
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sle_77777777:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2004318071 // =0x77777777
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, le
+; CHECK-NEXT:    ret
   %out = icmp slt i32 %in, 2004318072
   ret i1 %out
 }
 
 define i1 @sle_88888888(i32 noundef %in) {
-; CHECK-SD-LABEL: sle_88888888:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #34953 // =0x8889
-; CHECK-SD-NEXT:    movk w8, #34952, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sle_88888888:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #-2004318072 // =0x88888888
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sle_88888888:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-2004318072 // =0x88888888
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, -2004318071
   ret i1 %out
 }
 
 define i1 @sle_99999999(i32 noundef %in) {
-; CHECK-SD-LABEL: sle_99999999:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #39322 // =0x999a
-; CHECK-SD-NEXT:    movk w8, #39321, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, lo
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sle_99999999:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #-1717986919 // =0x99999999
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ls
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sle_99999999:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1717986919 // =0x99999999
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
   %out = icmp ult i32 %in, -1717986918
   ret i1 %out
 }
 
 define i1 @sge_11111111(i32 noundef %in) {
-; CHECK-SD-LABEL: sge_11111111:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #4368 // =0x1110
-; CHECK-SD-NEXT:    movk w8, #4369, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, gt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sge_11111111:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #286331153 // =0x11111111
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ge
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sge_11111111:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #286331153 // =0x11111111
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
   %out = icmp sgt i32 %in, 286331152
   ret i1 %out
 }
 
 define i1 @sge_22222222(i32 noundef %in) {
-; CHECK-SD-LABEL: sge_22222222:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #8737 // =0x2221
-; CHECK-SD-NEXT:    movk w8, #8738, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, gt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sge_22222222:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #572662306 // =0x22222222
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ge
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sge_22222222:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #572662306 // =0x22222222
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
   %out = icmp sgt i32 %in, 572662305
   ret i1 %out
 }
 
 define i1 @sge_33333333(i32 noundef %in) {
-; CHECK-SD-LABEL: sge_33333333:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #13106 // =0x3332
-; CHECK-SD-NEXT:    movk w8, #13107, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, gt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sge_33333333:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #858993459 // =0x33333333
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ge
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sge_33333333:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #858993459 // =0x33333333
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
   %out = icmp sgt i32 %in, 858993458
   ret i1 %out
 }
 
 define i1 @sge_44444444(i32 noundef %in) {
-; CHECK-SD-LABEL: sge_44444444:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #17475 // =0x4443
-; CHECK-SD-NEXT:    movk w8, #17476, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, gt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sge_44444444:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1145324612 // =0x44444444
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ge
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sge_44444444:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1145324612 // =0x44444444
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
   %out = icmp sgt i32 %in, 1145324611
   ret i1 %out
 }
 
 define i1 @sge_55555555(i32 noundef %in) {
-; CHECK-SD-LABEL: sge_55555555:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #21844 // =0x5554
-; CHECK-SD-NEXT:    movk w8, #21845, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, gt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sge_55555555:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1431655765 // =0x55555555
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ge
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sge_55555555:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1431655765 // =0x55555555
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
   %out = icmp sgt i32 %in, 1431655764
   ret i1 %out
 }
 
 define i1 @sge_66666666(i32 noundef %in) {
-; CHECK-SD-LABEL: sge_66666666:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #26213 // =0x6665
-; CHECK-SD-NEXT:    movk w8, #26214, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, gt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sge_66666666:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #1717986918 // =0x66666666
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ge
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sge_66666666:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #1717986918 // =0x66666666
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
   %out = icmp sgt i32 %in, 1717986917
   ret i1 %out
 }
 
 define i1 @sge_77777777(i32 noundef %in) {
-; CHECK-SD-LABEL: sge_77777777:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #30582 // =0x7776
-; CHECK-SD-NEXT:    movk w8, #30583, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, gt
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sge_77777777:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #2004318071 // =0x77777777
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, ge
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sge_77777777:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2004318071 // =0x77777777
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, ge
+; CHECK-NEXT:    ret
   %out = icmp sgt i32 %in, 2004318070
   ret i1 %out
 }
 
 define i1 @sge_88888888(i32 noundef %in) {
-; CHECK-SD-LABEL: sge_88888888:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #34951 // =0x8887
-; CHECK-SD-NEXT:    movk w8, #34952, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sge_88888888:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #-2004318072 // =0x88888888
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sge_88888888:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-2004318072 // =0x88888888
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, -2004318073
   ret i1 %out
 }
 
 define i1 @sge_99999999(i32 noundef %in) {
-; CHECK-SD-LABEL: sge_99999999:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    mov w8, #39320 // =0x9998
-; CHECK-SD-NEXT:    movk w8, #39321, lsl #16
-; CHECK-SD-NEXT:    cmp w0, w8
-; CHECK-SD-NEXT:    cset w0, hi
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: sge_99999999:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    mov w8, #-1717986919 // =0x99999999
-; CHECK-GI-NEXT:    cmp w0, w8
-; CHECK-GI-NEXT:    cset w0, hs
-; CHECK-GI-NEXT:    ret
+; CHECK-LABEL: sge_99999999:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #-1717986919 // =0x99999999
+; CHECK-NEXT:    cmp w0, w8
+; CHECK-NEXT:    cset w0, hs
+; CHECK-NEXT:    ret
   %out = icmp ugt i32 %in, -1717986920
   ret i1 %out
 }
+
+define i1 @ult_20014852997121(i64 noundef %in) {
+; CHECK-LABEL: ult_20014852997121:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x8, #305397760 // =0x12340000
+; CHECK-NEXT:    movk x8, #4660, lsl #32
+; CHECK-NEXT:    cmp x0, x8
+; CHECK-NEXT:    cset w0, ls
+; CHECK-NEXT:    ret
+  %out = icmp ult i64 %in, 20014852997121
+  ret i1 %out
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-GI: {{.*}}
+; CHECK-SD: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/insertextract.ll b/llvm/test/CodeGen/AArch64/insertextract.ll
index 5c89316e5f570..13a43d6d35239 100644
--- a/llvm/test/CodeGen/AArch64/insertextract.ll
+++ b/llvm/test/CodeGen/AArch64/insertextract.ll
@@ -1,6 +1,17 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:       warning: Instruction selection used fallback path for insert_v2i128_0
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for insert_v2i128_1
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for insert_v2i128_c
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for insert_v2fp128_0
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for insert_v2fp128_1
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for insert_v2fp128_c
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for extract_v2i128_0
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for extract_v2i128_1
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for extract_v2i128_c
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for extract_v2fp128_c
 
 define <2 x double> @insert_v2f64_0(<2 x double> %a, double %b, i32 %c) {
 ; CHECK-LABEL: insert_v2f64_0:
@@ -1312,6 +1323,137 @@ entry:
   ret <4 x i64> %d
 }
 
+define <2 x i128> @insert_v2i128_0(<2 x i128> %a, i128 %b, i32 %c) {
+; CHECK-LABEL: insert_v2i128_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adds x2, x2, x2
+; CHECK-NEXT:    mov x1, x5
+; CHECK-NEXT:    mov x0, x4
+; CHECK-NEXT:    adc x3, x3, x3
+; CHECK-NEXT:    ret
+entry:
+  %aa = add <2 x i128> %a, %a
+  %d = insertelement <2 x i128> %aa, i128 %b, i32 0
+  ret <2 x i128> %d
+}
+
+define <2 x i128> @insert_v2i128_1(<2 x i128> %a, i128 %b, i32 %c) {
+; CHECK-LABEL: insert_v2i128_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adds x0, x0, x0
+; CHECK-NEXT:    mov x3, x5
+; CHECK-NEXT:    mov x2, x4
+; CHECK-NEXT:    adc x1, x1, x1
+; CHECK-NEXT:    ret
+entry:
+  %aa = add <2 x i128> %a, %a
+  %d = insertelement <2 x i128> %aa, i128 %b, i32 1
+  ret <2 x i128> %d
+}
+
+define <2 x i128> @insert_v2i128_c(<2 x i128> %a, i128 %b, i32 %c) {
+; CHECK-LABEL: insert_v2i128_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    // kill: def $w6 killed $w6 def $x6
+; CHECK-NEXT:    adds x8, x0, x0
+; CHECK-NEXT:    and x11, x6, #0x1
+; CHECK-NEXT:    mov x12, sp
+; CHECK-NEXT:    adc x9, x1, x1
+; CHECK-NEXT:    adds x10, x2, x2
+; CHECK-NEXT:    add x11, x12, x11, lsl #4
+; CHECK-NEXT:    str x8, [sp]
+; CHECK-NEXT:    adc x8, x3, x3
+; CHECK-NEXT:    str x10, [sp, #16]
+; CHECK-NEXT:    str x4, [x11]
+; CHECK-NEXT:    str x8, [sp, #24]
+; CHECK-NEXT:    str x9, [sp, #8]
+; CHECK-NEXT:    str x5, [x11, #8]
+; CHECK-NEXT:    ldp x0, x1, [sp]
+; CHECK-NEXT:    ldp x2, x3, [sp, #16]
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+entry:
+  %aa = add <2 x i128> %a, %a
+  %d = insertelement <2 x i128> %aa, i128 %b, i32 %c
+  ret <2 x i128> %d
+}
+
+define <2 x fp128> @insert_v2fp128_0(<2 x fp128> %a, fp128 %b, i32 %c) {
+; CHECK-LABEL: insert_v2fp128_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    str q2, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+entry:
+  %aa = fadd <2 x fp128> %a, %a
+  %d = insertelement <2 x fp128> %aa, fp128 %b, i32 0
+  ret <2 x fp128> %d
+}
+
+define <2 x fp128> @insert_v2fp128_1(<2 x fp128> %a, fp128 %b, i32 %c) {
+; CHECK-LABEL: insert_v2fp128_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    str q2, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+entry:
+  %aa = fadd <2 x fp128> %a, %a
+  %d = insertelement <2 x fp128> %aa, fp128 %b, i32 1
+  ret <2 x fp128> %d
+}
+
+define <2 x fp128> @insert_v2fp128_c(<2 x fp128> %a, fp128 %b, i32 %c) {
+; CHECK-LABEL: insert_v2fp128_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #96
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov w19, w0
+; CHECK-NEXT:    str q2, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    bl __addtf3
+; CHECK-NEXT:    str q0, [sp, #64]
+; CHECK-NEXT:    ldp q3, q0, [sp, #16] // 32-byte Folded Reload
+; CHECK-NEXT:    and x8, x19, #0x1
+; CHECK-NEXT:    add x9, sp, #48
+; CHECK-NEXT:    str q3, [sp, #48]
+; CHECK-NEXT:    str q0, [x9, x8, lsl #4]
+; CHECK-NEXT:    ldp q0, q1, [sp, #48]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    ret
+entry:
+  %aa = fadd <2 x fp128> %a, %a
+  %d = insertelement <2 x fp128> %aa, fp128 %b, i32 %c
+  ret <2 x fp128> %d
+}
+
 define double @extract_v2f64_0(<2 x double> %a, i32 %c) {
 ; CHECK-LABEL: extract_v2f64_0:
 ; CHECK:       // %bb.0: // %entry
@@ -2573,3 +2715,94 @@ entry:
   %d = extractelement <4 x i64> %a, i32 %c
   ret i64 %d
 }
+
+define i128 @extract_v2i128_0(<2 x i128> %a, i32 %c) {
+; CHECK-LABEL: extract_v2i128_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adds x0, x0, x0
+; CHECK-NEXT:    adc x1, x1, x1
+; CHECK-NEXT:    ret
+entry:
+  %b = add <2 x i128> %a, %a
+  %d = extractelement <2 x i128> %b, i32 0
+  ret i128 %d
+}
+
+define i128 @extract_v2i128_1(<2 x i128> %a, i32 %c) {
+; CHECK-LABEL: extract_v2i128_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    adds x0, x2, x2
+; CHECK-NEXT:    adc x1, x3, x3
+; CHECK-NEXT:    ret
+entry:
+  %b = add <2 x i128> %a, %a
+  %d = extractelement <2 x i128> %b, i32 1
+  ret i128 %d
+}
+
+define i128 @extract_v2i128_c(<2 x i128> %a, i32 %c) {
+; CHECK-LABEL: extract_v2i128_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    adds x9, x0, x0
+; CHECK-NEXT:    mov w8, w4
+; CHECK-NEXT:    adc x10, x1, x1
+; CHECK-NEXT:    adds x11, x2, x2
+; CHECK-NEXT:    fmov d1, x9
+; CHECK-NEXT:    fmov d0, x11
+; CHECK-NEXT:    adc x12, x3, x3
+; CHECK-NEXT:    add x8, x8, x8
+; CHECK-NEXT:    and x9, x8, #0x3
+; CHECK-NEXT:    add w8, w8, #1
+; CHECK-NEXT:    mov x11, sp
+; CHECK-NEXT:    mov v1.d[1], x10
+; CHECK-NEXT:    add x10, sp, #32
+; CHECK-NEXT:    and x8, x8, #0x3
+; CHECK-NEXT:    mov v0.d[1], x12
+; CHECK-NEXT:    stp q1, q0, [sp]
+; CHECK-NEXT:    stp q1, q0, [sp, #32]
+; CHECK-NEXT:    ldr x0, [x10, x9, lsl #3]
+; CHECK-NEXT:    ldr x1, [x11, x8, lsl #3]
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ret
+entry:
+  %b = add <2 x i128> %a, %a
+  %d = extractelement <2 x i128> %b, i32 %c
+  ret i128 %d
+}
+
+define fp128 @extract_v2fp128_0(<2 x fp128> %a, i32 %c) {
+; CHECK-LABEL: extract_v2fp128_0:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ret
+entry:
+  %d = extractelement <2 x fp128> %a, i32 0
+  ret fp128 %d
+}
+
+define fp128 @extract_v2fp128_1(<2 x fp128> %a, i32 %c) {
+; CHECK-LABEL: extract_v2fp128_1:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %d = extractelement <2 x fp128> %a, i32 1
+  ret fp128 %d
+}
+
+define fp128 @extract_v2fp128_c(<2 x fp128> %a, i32 %c) {
+; CHECK-LABEL: extract_v2fp128_c:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp q0, q1, [sp, #-32]!
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    and x8, x0, #0x1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    ldr q0, [x9, x8, lsl #4]
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+entry:
+  %d = extractelement <2 x fp128> %a, i32 %c
+  ret fp128 %d
+}
diff --git a/llvm/test/CodeGen/AArch64/irg-nomem.mir b/llvm/test/CodeGen/AArch64/irg-nomem.mir
index 3b000fafbed46..078a82bd5242f 100644
--- a/llvm/test/CodeGen/AArch64/irg-nomem.mir
+++ b/llvm/test/CodeGen/AArch64/irg-nomem.mir
@@ -47,8 +47,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/itofp.ll b/llvm/test/CodeGen/AArch64/itofp.ll
index 217547c31f7f2..caf87a13f283b 100644
--- a/llvm/test/CodeGen/AArch64/itofp.ll
+++ b/llvm/test/CodeGen/AArch64/itofp.ll
@@ -1,232 +1,8 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16
-; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
-; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=1 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
-; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=1 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
-
-define fp128 @stofp_i128_f128(i128 %a) {
-; CHECK-SD-LABEL: stofp_i128_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floattitf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_i128_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floattitf
-entry:
-  %c = sitofp i128 %a to fp128
-  ret fp128 %c
-}
-
-define fp128 @utofp_i128_f128(i128 %a) {
-; CHECK-SD-LABEL: utofp_i128_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floatuntitf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_i128_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floatuntitf
-entry:
-  %c = uitofp i128 %a to fp128
-  ret fp128 %c
-}
-
-define fp128 @stofp_i64_f128(i64 %a) {
-; CHECK-SD-LABEL: stofp_i64_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floatditf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_i64_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floatditf
-entry:
-  %c = sitofp i64 %a to fp128
-  ret fp128 %c
-}
-
-define fp128 @utofp_i64_f128(i64 %a) {
-; CHECK-SD-LABEL: utofp_i64_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floatunditf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_i64_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floatunditf
-entry:
-  %c = uitofp i64 %a to fp128
-  ret fp128 %c
-}
-
-define fp128 @stofp_i32_f128(i32 %a) {
-; CHECK-SD-LABEL: stofp_i32_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_i32_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floatsitf
-entry:
-  %c = sitofp i32 %a to fp128
-  ret fp128 %c
-}
-
-define fp128 @utofp_i32_f128(i32 %a) {
-; CHECK-SD-LABEL: utofp_i32_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_i32_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floatunsitf
-entry:
-  %c = uitofp i32 %a to fp128
-  ret fp128 %c
-}
-
-define fp128 @stofp_i16_f128(i16 %a) {
-; CHECK-SD-LABEL: stofp_i16_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    sxth w0, w0
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_i16_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sxth w0, w0
-; CHECK-GI-NEXT:    b __floatsitf
-entry:
-  %c = sitofp i16 %a to fp128
-  ret fp128 %c
-}
-
-define fp128 @utofp_i16_f128(i16 %a) {
-; CHECK-SD-LABEL: utofp_i16_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    and w0, w0, #0xffff
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_i16_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    and w0, w0, #0xffff
-; CHECK-GI-NEXT:    b __floatunsitf
-entry:
-  %c = uitofp i16 %a to fp128
-  ret fp128 %c
-}
-
-define fp128 @stofp_i8_f128(i8 %a) {
-; CHECK-SD-LABEL: stofp_i8_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    sxtb w0, w0
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_i8_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sxtb w0, w0
-; CHECK-GI-NEXT:    b __floatsitf
-entry:
-  %c = sitofp i8 %a to fp128
-  ret fp128 %c
-}
-
-define fp128 @utofp_i8_f128(i8 %a) {
-; CHECK-SD-LABEL: utofp_i8_f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    and w0, w0, #0xff
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_i8_f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    and w0, w0, #0xff
-; CHECK-GI-NEXT:    b __floatunsitf
-entry:
-  %c = uitofp i8 %a to fp128
-  ret fp128 %c
-}
-
-define double @stofp_i128_f64(i128 %a) {
-; CHECK-SD-LABEL: stofp_i128_f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floattidf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_i128_f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floattidf
-entry:
-  %c = sitofp i128 %a to double
-  ret double %c
-}
-
-define double @utofp_i128_f64(i128 %a) {
-; CHECK-SD-LABEL: utofp_i128_f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floatuntidf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_i128_f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floatuntidf
-entry:
-  %c = uitofp i128 %a to double
-  ret double %c
-}
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16,CHECK-NOFP16-SD
+; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-SD
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-NOFP16,CHECK-NOFP16-GI
+; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16,CHECK-FP16-GI
 
 define double @stofp_i64_f64(i64 %a) {
 ; CHECK-LABEL: stofp_i64_f64:
@@ -312,40 +88,66 @@ entry:
   ret double %c
 }
 
-define float @stofp_i128_f32(i128 %a) {
-; CHECK-SD-LABEL: stofp_i128_f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floattisf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_i128_f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floattisf
+define double @stofp_i128_f64(i128 %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_i128_f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floattidf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_i128_f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floattidf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i128_f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floattidf
+;
+; CHECK-FP16-GI-LABEL: stofp_i128_f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floattidf
 entry:
-  %c = sitofp i128 %a to float
-  ret float %c
+  %c = sitofp i128 %a to double
+  ret double %c
 }
 
-define float @utofp_i128_f32(i128 %a) {
-; CHECK-SD-LABEL: utofp_i128_f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    bl __floatuntisf
-; CHECK-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_i128_f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    b __floatuntisf
+define double @utofp_i128_f64(i128 %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_i128_f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntidf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_i128_f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floatuntidf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i128_f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floatuntidf
+;
+; CHECK-FP16-GI-LABEL: utofp_i128_f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floatuntidf
 entry:
-  %c = uitofp i128 %a to float
-  ret float %c
+  %c = uitofp i128 %a to double
+  ret double %c
 }
 
 define float @stofp_i64_f32(i64 %a) {
@@ -432,1544 +234,1088 @@ entry:
   ret float %c
 }
 
-define half @stofp_i128_f16(i128 %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_i128_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NOFP16-NEXT:    bl __floattisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_i128_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-FP16-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-FP16-NEXT:    bl __floattihf
-; CHECK-SD-FP16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_i128_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NOFP16-NEXT:    bl __floattisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_i128_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    b __floattihf
+define float @stofp_i128_f32(i128 %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_i128_f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floattisf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_i128_f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floattisf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i128_f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floattisf
+;
+; CHECK-FP16-GI-LABEL: stofp_i128_f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floattisf
 entry:
-  %c = sitofp i128 %a to half
-  ret half %c
+  %c = sitofp i128 %a to float
+  ret float %c
 }
 
-define half @utofp_i128_f16(i128 %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_i128_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_i128_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-SD-FP16-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-FP16-NEXT:    bl __floatuntihf
-; CHECK-SD-FP16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_i128_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_i128_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    b __floatuntihf
+define float @utofp_i128_f32(i128 %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_i128_f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_i128_f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floatuntisf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i128_f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floatuntisf
+;
+; CHECK-FP16-GI-LABEL: utofp_i128_f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floatuntisf
 entry:
-  %c = uitofp i128 %a to half
-  ret half %c
+  %c = uitofp i128 %a to float
+  ret float %c
 }
 
 define half @stofp_i64_f16(i64 %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_i64_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    scvtf s0, x0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_i64_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    scvtf h0, x0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_i64_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf s0, x0
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_i64_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf h0, x0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: stofp_i64_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    scvtf s0, x0
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_i64_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    scvtf h0, x0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = sitofp i64 %a to half
   ret half %c
 }
 
 define half @utofp_i64_f16(i64 %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_i64_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ucvtf s0, x0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_i64_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ucvtf h0, x0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_i64_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf s0, x0
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_i64_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf h0, x0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: utofp_i64_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ucvtf s0, x0
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_i64_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ucvtf h0, x0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = uitofp i64 %a to half
   ret half %c
 }
 
 define half @stofp_i32_f16(i32 %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_i32_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    scvtf s0, w0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_i32_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    scvtf h0, w0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_i32_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    scvtf s0, w0
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_i32_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf h0, w0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: stofp_i32_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    scvtf s0, w0
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_i32_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    scvtf h0, w0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = sitofp i32 %a to half
   ret half %c
 }
 
 define half @utofp_i32_f16(i32 %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_i32_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ucvtf s0, w0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_i32_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ucvtf h0, w0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_i32_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ucvtf s0, w0
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_i32_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf h0, w0
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: utofp_i32_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ucvtf s0, w0
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_i32_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ucvtf h0, w0
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = uitofp i32 %a to half
   ret half %c
 }
 
 define half @stofp_i16_f16(i16 %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_i16_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sxth w8, w0
-; CHECK-SD-NOFP16-NEXT:    scvtf s0, w8
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_i16_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sxth w8, w0
-; CHECK-SD-FP16-NEXT:    scvtf h0, w8
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_i16_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sxth w8, w0
-; CHECK-GI-NOFP16-NEXT:    scvtf s0, w8
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_i16_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    sxth w8, w0
-; CHECK-GI-FP16-NEXT:    scvtf h0, w8
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: stofp_i16_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    sxth w8, w0
+; CHECK-NOFP16-NEXT:    scvtf s0, w8
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_i16_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    sxth w8, w0
+; CHECK-FP16-NEXT:    scvtf h0, w8
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = sitofp i16 %a to half
   ret half %c
 }
 
 define half @utofp_i16_f16(i16 %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_i16_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    and w8, w0, #0xffff
-; CHECK-SD-NOFP16-NEXT:    ucvtf s0, w8
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_i16_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    and w8, w0, #0xffff
-; CHECK-SD-FP16-NEXT:    ucvtf h0, w8
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_i16_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    and w8, w0, #0xffff
-; CHECK-GI-NOFP16-NEXT:    ucvtf s0, w8
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_i16_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    and w8, w0, #0xffff
-; CHECK-GI-FP16-NEXT:    ucvtf h0, w8
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: utofp_i16_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    and w8, w0, #0xffff
+; CHECK-NOFP16-NEXT:    ucvtf s0, w8
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_i16_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    and w8, w0, #0xffff
+; CHECK-FP16-NEXT:    ucvtf h0, w8
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = uitofp i16 %a to half
   ret half %c
 }
 
 define half @stofp_i8_f16(i8 %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_i8_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sxtb w8, w0
-; CHECK-SD-NOFP16-NEXT:    scvtf s0, w8
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_i8_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sxtb w8, w0
-; CHECK-SD-FP16-NEXT:    scvtf h0, w8
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_i8_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sxtb w8, w0
-; CHECK-GI-NOFP16-NEXT:    scvtf s0, w8
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_i8_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    sxtb w8, w0
-; CHECK-GI-FP16-NEXT:    scvtf h0, w8
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: stofp_i8_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    sxtb w8, w0
+; CHECK-NOFP16-NEXT:    scvtf s0, w8
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_i8_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    sxtb w8, w0
+; CHECK-FP16-NEXT:    scvtf h0, w8
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = sitofp i8 %a to half
   ret half %c
 }
 
 define half @utofp_i8_f16(i8 %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_i8_f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    and w8, w0, #0xff
-; CHECK-SD-NOFP16-NEXT:    ucvtf s0, w8
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_i8_f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    and w8, w0, #0xff
-; CHECK-SD-FP16-NEXT:    ucvtf h0, w8
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_i8_f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    and w8, w0, #0xff
-; CHECK-GI-NOFP16-NEXT:    ucvtf s0, w8
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_i8_f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    and w8, w0, #0xff
-; CHECK-GI-FP16-NEXT:    ucvtf h0, w8
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: utofp_i8_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    and w8, w0, #0xff
+; CHECK-NOFP16-NEXT:    ucvtf s0, w8
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_i8_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    and w8, w0, #0xff
+; CHECK-FP16-NEXT:    ucvtf h0, w8
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = uitofp i8 %a to half
   ret half %c
 }
 
-define <2 x fp128> @stofp_v2i128_v2f128(<2 x i128> %a) {
-; CHECK-SD-LABEL: stofp_v2i128_v2f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    mov x19, x3
-; CHECK-SD-NEXT:    mov x20, x2
-; CHECK-SD-NEXT:    bl __floattitf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floattitf
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov v1.16b, v0.16b
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i128_v2f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #48
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    bl __floattitf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floattitf
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v1.16b, v0.16b
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #48
-; CHECK-GI-NEXT:    ret
+define half @stofp_i128_f16(i128 %a) {
+; CHECK-NOFP16-LABEL: stofp_i128_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-NEXT:    bl __floattisf
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_i128_f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floattihf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_i128_f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floattihf
 entry:
-  %c = sitofp <2 x i128> %a to <2 x fp128>
-  ret <2 x fp128> %c
+  %c = sitofp i128 %a to half
+  ret half %c
 }
 
-define <2 x fp128> @utofp_v2i128_v2f128(<2 x i128> %a) {
-; CHECK-SD-LABEL: utofp_v2i128_v2f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    mov x19, x3
-; CHECK-SD-NEXT:    mov x20, x2
-; CHECK-SD-NEXT:    bl __floatuntitf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatuntitf
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov v1.16b, v0.16b
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i128_v2f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #48
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    bl __floatuntitf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatuntitf
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v1.16b, v0.16b
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #48
-; CHECK-GI-NEXT:    ret
+define half @utofp_i128_f16(i128 %a) {
+; CHECK-NOFP16-LABEL: utofp_i128_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_i128_f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floatuntihf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_i128_f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floatuntihf
 entry:
-  %c = uitofp <2 x i128> %a to <2 x fp128>
-  ret <2 x fp128> %c
+  %c = uitofp i128 %a to half
+  ret half %c
 }
 
-define <3 x fp128> @stofp_v3i128_v3f128(<3 x i128> %a) {
-; CHECK-SD-LABEL: stofp_v3i128_v3f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #80
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    mov x19, x5
-; CHECK-SD-NEXT:    mov x20, x4
-; CHECK-SD-NEXT:    mov x21, x3
-; CHECK-SD-NEXT:    mov x22, x2
-; CHECK-SD-NEXT:    bl __floattitf
-; CHECK-SD-NEXT:    mov x0, x22
-; CHECK-SD-NEXT:    mov x1, x21
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floattitf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floattitf
-; CHECK-SD-NEXT:    mov v2.16b, v0.16b
-; CHECK-SD-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #80
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i128_v3f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #80
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    mov x21, x4
-; CHECK-GI-NEXT:    mov x22, x5
-; CHECK-GI-NEXT:    bl __floattitf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floattitf
-; CHECK-GI-NEXT:    mov x0, x21
-; CHECK-GI-NEXT:    mov x1, x22
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floattitf
-; CHECK-GI-NEXT:    mov v2.16b, v0.16b
-; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #80
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <3 x i128> %a to <3 x fp128>
-  ret <3 x fp128> %c
-}
-
-define <3 x fp128> @utofp_v3i128_v3f128(<3 x i128> %a) {
-; CHECK-SD-LABEL: utofp_v3i128_v3f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #80
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    mov x19, x5
-; CHECK-SD-NEXT:    mov x20, x4
-; CHECK-SD-NEXT:    mov x21, x3
-; CHECK-SD-NEXT:    mov x22, x2
-; CHECK-SD-NEXT:    bl __floatuntitf
-; CHECK-SD-NEXT:    mov x0, x22
-; CHECK-SD-NEXT:    mov x1, x21
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatuntitf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatuntitf
-; CHECK-SD-NEXT:    mov v2.16b, v0.16b
-; CHECK-SD-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #80
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i128_v3f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #80
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    mov x21, x4
-; CHECK-GI-NEXT:    mov x22, x5
-; CHECK-GI-NEXT:    bl __floatuntitf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatuntitf
-; CHECK-GI-NEXT:    mov x0, x21
-; CHECK-GI-NEXT:    mov x1, x22
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatuntitf
-; CHECK-GI-NEXT:    mov v2.16b, v0.16b
-; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #80
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <3 x i128> %a to <3 x fp128>
-  ret <3 x fp128> %c
+define fp128 @stofp_i64_f128(i64 %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_i64_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floatditf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_i64_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floatditf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i64_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floatditf
+;
+; CHECK-FP16-GI-LABEL: stofp_i64_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floatditf
+entry:
+  %c = sitofp i64 %a to fp128
+  ret fp128 %c
 }
 
-define <2 x fp128> @stofp_v2i64_v2f128(<2 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v2i64_v2f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    fmov x0, d0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatditf
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x0, v0.d[1]
-; CHECK-SD-NEXT:    bl __floatditf
-; CHECK-SD-NEXT:    mov v1.16b, v0.16b
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i64_v2f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w30, -8
-; CHECK-GI-NEXT:    .cfi_offset b8, -16
-; CHECK-GI-NEXT:    fmov x0, d0
-; CHECK-GI-NEXT:    mov d8, v0.d[1]
-; CHECK-GI-NEXT:    bl __floatditf
-; CHECK-GI-NEXT:    fmov x0, d8
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatditf
-; CHECK-GI-NEXT:    mov v1.16b, v0.16b
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+define fp128 @utofp_i64_f128(i64 %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_i64_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floatunditf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_i64_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floatunditf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i64_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floatunditf
+;
+; CHECK-FP16-GI-LABEL: utofp_i64_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floatunditf
 entry:
-  %c = sitofp <2 x i64> %a to <2 x fp128>
-  ret <2 x fp128> %c
+  %c = uitofp i64 %a to fp128
+  ret fp128 %c
 }
 
-define <2 x fp128> @utofp_v2i64_v2f128(<2 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v2i64_v2f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    fmov x0, d0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatunditf
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov x0, v0.d[1]
-; CHECK-SD-NEXT:    bl __floatunditf
-; CHECK-SD-NEXT:    mov v1.16b, v0.16b
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i64_v2f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w30, -8
-; CHECK-GI-NEXT:    .cfi_offset b8, -16
-; CHECK-GI-NEXT:    fmov x0, d0
-; CHECK-GI-NEXT:    mov d8, v0.d[1]
-; CHECK-GI-NEXT:    bl __floatunditf
-; CHECK-GI-NEXT:    fmov x0, d8
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatunditf
-; CHECK-GI-NEXT:    mov v1.16b, v0.16b
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+define fp128 @stofp_i32_f128(i32 %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_i32_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floatsitf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_i32_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floatsitf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i32_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floatsitf
+;
+; CHECK-FP16-GI-LABEL: stofp_i32_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floatsitf
 entry:
-  %c = uitofp <2 x i64> %a to <2 x fp128>
-  ret <2 x fp128> %c
+  %c = sitofp i32 %a to fp128
+  ret fp128 %c
 }
 
-define <3 x fp128> @stofp_v3i64_v3f128(<3 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v3i64_v3f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov x0, d0
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    str q2, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatditf
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov x0, d0
-; CHECK-SD-NEXT:    bl __floatditf
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov x0, d0
-; CHECK-SD-NEXT:    bl __floatditf
-; CHECK-SD-NEXT:    mov v2.16b, v0.16b
-; CHECK-SD-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i64_v3f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    .cfi_offset b8, -24
-; CHECK-GI-NEXT:    .cfi_offset b9, -32
-; CHECK-GI-NEXT:    fmov x0, d0
-; CHECK-GI-NEXT:    fmov d8, d1
-; CHECK-GI-NEXT:    fmov d9, d2
-; CHECK-GI-NEXT:    bl __floatditf
-; CHECK-GI-NEXT:    fmov x0, d8
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatditf
-; CHECK-GI-NEXT:    fmov x0, d9
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatditf
-; CHECK-GI-NEXT:    mov v2.16b, v0.16b
-; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <3 x i64> %a to <3 x fp128>
-  ret <3 x fp128> %c
-}
-
-define <3 x fp128> @utofp_v3i64_v2f128(<3 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v3i64_v2f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov x0, d0
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    str q2, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatunditf
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov x0, d0
-; CHECK-SD-NEXT:    bl __floatunditf
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov x0, d0
-; CHECK-SD-NEXT:    bl __floatunditf
-; CHECK-SD-NEXT:    mov v2.16b, v0.16b
-; CHECK-SD-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i64_v2f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    .cfi_offset b8, -24
-; CHECK-GI-NEXT:    .cfi_offset b9, -32
-; CHECK-GI-NEXT:    fmov x0, d0
-; CHECK-GI-NEXT:    fmov d8, d1
-; CHECK-GI-NEXT:    fmov d9, d2
-; CHECK-GI-NEXT:    bl __floatunditf
-; CHECK-GI-NEXT:    fmov x0, d8
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatunditf
-; CHECK-GI-NEXT:    fmov x0, d9
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatunditf
-; CHECK-GI-NEXT:    mov v2.16b, v0.16b
-; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <3 x i64> %a to <3 x fp128>
-  ret <3 x fp128> %c
+define fp128 @utofp_i32_f128(i32 %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_i32_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floatunsitf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_i32_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floatunsitf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i32_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floatunsitf
+;
+; CHECK-FP16-GI-LABEL: utofp_i32_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floatunsitf
+entry:
+  %c = uitofp i32 %a to fp128
+  ret fp128 %c
 }
 
-define <2 x double> @stofp_v2i128_v2f64(<2 x i128> %a) {
-; CHECK-SD-LABEL: stofp_v2i128_v2f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    mov x19, x1
-; CHECK-SD-NEXT:    mov x20, x0
-; CHECK-SD-NEXT:    mov x0, x2
-; CHECK-SD-NEXT:    mov x1, x3
-; CHECK-SD-NEXT:    bl __floattidf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floattidf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i128_v2f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #48
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    bl __floattidf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floattidf
-; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v1.d[1], v0.d[0]
-; CHECK-GI-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NEXT:    add sp, sp, #48
-; CHECK-GI-NEXT:    ret
+define fp128 @stofp_i16_f128(i16 %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_i16_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    sxth w0, w0
+; CHECK-NOFP16-SD-NEXT:    bl __floatsitf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_i16_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    sxth w0, w0
+; CHECK-FP16-SD-NEXT:    bl __floatsitf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i16_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sxth w0, w0
+; CHECK-NOFP16-GI-NEXT:    b __floatsitf
+;
+; CHECK-FP16-GI-LABEL: stofp_i16_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sxth w0, w0
+; CHECK-FP16-GI-NEXT:    b __floatsitf
 entry:
-  %c = sitofp <2 x i128> %a to <2 x double>
-  ret <2 x double> %c
+  %c = sitofp i16 %a to fp128
+  ret fp128 %c
 }
 
-define <2 x double> @utofp_v2i128_v2f64(<2 x i128> %a) {
-; CHECK-SD-LABEL: utofp_v2i128_v2f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    mov x19, x1
-; CHECK-SD-NEXT:    mov x20, x0
-; CHECK-SD-NEXT:    mov x0, x2
-; CHECK-SD-NEXT:    mov x1, x3
-; CHECK-SD-NEXT:    bl __floatuntidf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatuntidf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i128_v2f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #48
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    bl __floatuntidf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatuntidf
-; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v1.d[1], v0.d[0]
-; CHECK-GI-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NEXT:    add sp, sp, #48
-; CHECK-GI-NEXT:    ret
+define fp128 @utofp_i16_f128(i16 %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_i16_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    and w0, w0, #0xffff
+; CHECK-NOFP16-SD-NEXT:    bl __floatunsitf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_i16_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    and w0, w0, #0xffff
+; CHECK-FP16-SD-NEXT:    bl __floatunsitf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i16_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    and w0, w0, #0xffff
+; CHECK-NOFP16-GI-NEXT:    b __floatunsitf
+;
+; CHECK-FP16-GI-LABEL: utofp_i16_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    and w0, w0, #0xffff
+; CHECK-FP16-GI-NEXT:    b __floatunsitf
 entry:
-  %c = uitofp <2 x i128> %a to <2 x double>
-  ret <2 x double> %c
+  %c = uitofp i16 %a to fp128
+  ret fp128 %c
 }
 
-define <3 x double> @stofp_v3i128_v3f64(<3 x i128> %a) {
-; CHECK-SD-LABEL: stofp_v3i128_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    .cfi_offset b8, -56
-; CHECK-SD-NEXT:    .cfi_offset b9, -64
-; CHECK-SD-NEXT:    mov x19, x5
-; CHECK-SD-NEXT:    mov x20, x4
-; CHECK-SD-NEXT:    mov x21, x3
-; CHECK-SD-NEXT:    mov x22, x2
-; CHECK-SD-NEXT:    bl __floattidf
-; CHECK-SD-NEXT:    mov x0, x22
-; CHECK-SD-NEXT:    mov x1, x21
-; CHECK-SD-NEXT:    fmov d8, d0
-; CHECK-SD-NEXT:    bl __floattidf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    fmov d9, d0
-; CHECK-SD-NEXT:    bl __floattidf
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d2, d0
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d0, d8
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d1, d9
-; CHECK-SD-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i128_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    .cfi_offset b8, -56
-; CHECK-GI-NEXT:    .cfi_offset b9, -64
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    mov x21, x4
-; CHECK-GI-NEXT:    mov x22, x5
-; CHECK-GI-NEXT:    bl __floattidf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    fmov d8, d0
-; CHECK-GI-NEXT:    bl __floattidf
-; CHECK-GI-NEXT:    mov x0, x21
-; CHECK-GI-NEXT:    mov x1, x22
-; CHECK-GI-NEXT:    fmov d9, d0
-; CHECK-GI-NEXT:    bl __floattidf
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    fmov d2, d0
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    fmov d0, d8
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    fmov d1, d9
-; CHECK-GI-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <3 x i128> %a to <3 x double>
-  ret <3 x double> %c
+define fp128 @stofp_i8_f128(i8 %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_i8_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    sxtb w0, w0
+; CHECK-NOFP16-SD-NEXT:    bl __floatsitf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_i8_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    sxtb w0, w0
+; CHECK-FP16-SD-NEXT:    bl __floatsitf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i8_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sxtb w0, w0
+; CHECK-NOFP16-GI-NEXT:    b __floatsitf
+;
+; CHECK-FP16-GI-LABEL: stofp_i8_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sxtb w0, w0
+; CHECK-FP16-GI-NEXT:    b __floatsitf
+entry:
+  %c = sitofp i8 %a to fp128
+  ret fp128 %c
 }
 
-define <3 x double> @utofp_v3i128_v3f64(<3 x i128> %a) {
-; CHECK-SD-LABEL: utofp_v3i128_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    .cfi_offset b8, -56
-; CHECK-SD-NEXT:    .cfi_offset b9, -64
-; CHECK-SD-NEXT:    mov x19, x5
-; CHECK-SD-NEXT:    mov x20, x4
-; CHECK-SD-NEXT:    mov x21, x3
-; CHECK-SD-NEXT:    mov x22, x2
-; CHECK-SD-NEXT:    bl __floatuntidf
-; CHECK-SD-NEXT:    mov x0, x22
-; CHECK-SD-NEXT:    mov x1, x21
-; CHECK-SD-NEXT:    fmov d8, d0
-; CHECK-SD-NEXT:    bl __floatuntidf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    fmov d9, d0
-; CHECK-SD-NEXT:    bl __floatuntidf
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d2, d0
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d0, d8
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    fmov d1, d9
-; CHECK-SD-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i128_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    .cfi_offset b8, -56
-; CHECK-GI-NEXT:    .cfi_offset b9, -64
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    mov x21, x4
-; CHECK-GI-NEXT:    mov x22, x5
-; CHECK-GI-NEXT:    bl __floatuntidf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    fmov d8, d0
-; CHECK-GI-NEXT:    bl __floatuntidf
-; CHECK-GI-NEXT:    mov x0, x21
-; CHECK-GI-NEXT:    mov x1, x22
-; CHECK-GI-NEXT:    fmov d9, d0
-; CHECK-GI-NEXT:    bl __floatuntidf
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    fmov d2, d0
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    fmov d0, d8
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    fmov d1, d9
-; CHECK-GI-NEXT:    ldp d9, d8, [sp], #64 // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <3 x i128> %a to <3 x double>
-  ret <3 x double> %c
+define fp128 @utofp_i8_f128(i8 %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_i8_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    and w0, w0, #0xff
+; CHECK-NOFP16-SD-NEXT:    bl __floatunsitf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_i8_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    and w0, w0, #0xff
+; CHECK-FP16-SD-NEXT:    bl __floatunsitf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i8_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    and w0, w0, #0xff
+; CHECK-NOFP16-GI-NEXT:    b __floatunsitf
+;
+; CHECK-FP16-GI-LABEL: utofp_i8_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    and w0, w0, #0xff
+; CHECK-FP16-GI-NEXT:    b __floatunsitf
+entry:
+  %c = uitofp i8 %a to fp128
+  ret fp128 %c
 }
 
-define <2 x double> @stofp_v2i64_v2f64(<2 x i64> %a) {
-; CHECK-LABEL: stofp_v2i64_v2f64:
+define fp128 @stofp_i128_f128(i128 %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_i128_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floattitf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_i128_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floattitf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_i128_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floattitf
+;
+; CHECK-FP16-GI-LABEL: stofp_i128_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floattitf
+entry:
+  %c = sitofp i128 %a to fp128
+  ret fp128 %c
+}
+
+define fp128 @utofp_i128_f128(i128 %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_i128_f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntitf
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_i128_f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    bl __floatuntitf
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_i128_f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    b __floatuntitf
+;
+; CHECK-FP16-GI-LABEL: utofp_i128_f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    b __floatuntitf
+entry:
+  %c = uitofp i128 %a to fp128
+  ret fp128 %c
+}
+
+define double @stofp_load_i64_f64(ptr %p) {
+; CHECK-LABEL: stofp_load_i64_f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    scvtf d0, d0
 ; CHECK-NEXT:    ret
 entry:
-  %c = sitofp <2 x i64> %a to <2 x double>
-  ret <2 x double> %c
+  %a = load i64, ptr %p
+  %c = sitofp i64 %a to double
+  ret double %c
 }
 
-define <2 x double> @utofp_v2i64_v2f64(<2 x i64> %a) {
-; CHECK-LABEL: utofp_v2i64_v2f64:
+define double @utofp_load_i64_f64(ptr %p) {
+; CHECK-LABEL: utofp_load_i64_f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    ucvtf d0, d0
 ; CHECK-NEXT:    ret
 entry:
-  %c = uitofp <2 x i64> %a to <2 x double>
-  ret <2 x double> %c
+  %a = load i64, ptr %p
+  %c = uitofp i64 %a to double
+  ret double %c
 }
 
-define <3 x double> @stofp_v3i64_v3f64(<3 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v3i64_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i64_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+define double @stofp_load_i32_f64(ptr %p) {
+; CHECK-LABEL: stofp_load_i32_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr w8, [x0]
+; CHECK-NEXT:    scvtf d0, w8
+; CHECK-NEXT:    ret
 entry:
-  %c = sitofp <3 x i64> %a to <3 x double>
-  ret <3 x double> %c
+  %a = load i32, ptr %p
+  %c = sitofp i32 %a to double
+  ret double %c
 }
 
-define <3 x double> @utofp_v3i64_v3f64(<3 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v3i64_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-SD-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i64_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+define double @utofp_load_i32_f64(ptr %p) {
+; CHECK-LABEL: utofp_load_i32_f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    ucvtf d0, d0
+; CHECK-NEXT:    ret
 entry:
-  %c = uitofp <3 x i64> %a to <3 x double>
-  ret <3 x double> %c
+  %a = load i32, ptr %p
+  %c = uitofp i32 %a to double
+  ret double %c
 }
 
-define <4 x double> @stofp_v4i64_v4f64(<4 x i64> %a) {
-; CHECK-LABEL: stofp_v4i64_v4f64:
+define double @stofp_load_i16_f64(ptr %p) {
+; CHECK-LABEL: stofp_load_i16_f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NEXT:    ldrsh w8, [x0]
+; CHECK-NEXT:    scvtf d0, w8
 ; CHECK-NEXT:    ret
 entry:
-  %c = sitofp <4 x i64> %a to <4 x double>
-  ret <4 x double> %c
+  %a = load i16, ptr %p
+  %c = sitofp i16 %a to double
+  ret double %c
 }
 
-define <4 x double> @utofp_v4i64_v4f64(<4 x i64> %a) {
-; CHECK-LABEL: utofp_v4i64_v4f64:
+define double @utofp_load_i16_f64(ptr %p) {
+; CHECK-LABEL: utofp_load_i16_f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NEXT:    ldr h0, [x0]
+; CHECK-NEXT:    ucvtf d0, d0
 ; CHECK-NEXT:    ret
 entry:
-  %c = uitofp <4 x i64> %a to <4 x double>
-  ret <4 x double> %c
+  %a = load i16, ptr %p
+  %c = uitofp i16 %a to double
+  ret double %c
 }
 
-define <8 x double> @stofp_v8i64_v8f64(<8 x i64> %a) {
-; CHECK-LABEL: stofp_v8i64_v8f64:
+define double @stofp_load_i8_f64(ptr %p) {
+; CHECK-LABEL: stofp_load_i8_f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NEXT:    ldrsb w8, [x0]
+; CHECK-NEXT:    scvtf d0, w8
 ; CHECK-NEXT:    ret
 entry:
-  %c = sitofp <8 x i64> %a to <8 x double>
-  ret <8 x double> %c
+  %a = load i8, ptr %p
+  %c = sitofp i8 %a to double
+  ret double %c
 }
 
-define <8 x double> @utofp_v8i64_v8f64(<8 x i64> %a) {
-; CHECK-LABEL: utofp_v8i64_v8f64:
+define double @utofp_load_i8_f64(ptr %p) {
+; CHECK-LABEL: utofp_load_i8_f64:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NEXT:    ldr b0, [x0]
+; CHECK-NEXT:    ucvtf d0, d0
 ; CHECK-NEXT:    ret
 entry:
-  %c = uitofp <8 x i64> %a to <8 x double>
-  ret <8 x double> %c
+  %a = load i8, ptr %p
+  %c = uitofp i8 %a to double
+  ret double %c
 }
 
-define <16 x double> @stofp_v16i64_v16f64(<16 x i64> %a) {
-; CHECK-LABEL: stofp_v16i64_v16f64:
+define float @stofp_load_i64_f32(ptr %p) {
+; CHECK-LABEL: stofp_load_i64_f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-NEXT:    ldr x8, [x0]
+; CHECK-NEXT:    scvtf s0, x8
 ; CHECK-NEXT:    ret
 entry:
-  %c = sitofp <16 x i64> %a to <16 x double>
-  ret <16 x double> %c
+  %a = load i64, ptr %p
+  %c = sitofp i64 %a to float
+  ret float %c
 }
 
-define <16 x double> @utofp_v16i64_v16f64(<16 x i64> %a) {
-; CHECK-LABEL: utofp_v16i64_v16f64:
+define float @utofp_load_i64_f32(ptr %p) {
+; CHECK-LABEL: utofp_load_i64_f32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-NEXT:    ldr x8, [x0]
+; CHECK-NEXT:    ucvtf s0, x8
 ; CHECK-NEXT:    ret
 entry:
-  %c = uitofp <16 x i64> %a to <16 x double>
-  ret <16 x double> %c
+  %a = load i64, ptr %p
+  %c = uitofp i64 %a to float
+  ret float %c
 }
 
-define <32 x double> @stofp_v32i64_v32f64(<32 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v32i64_v32f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q17, q16, [sp, #96]
-; CHECK-SD-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-SD-NEXT:    ldp q19, q18, [sp, #64]
-; CHECK-SD-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    ldp q21, q20, [sp, #32]
-; CHECK-SD-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-SD-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    scvtf v18.2d, v18.2d
-; CHECK-SD-NEXT:    scvtf v19.2d, v19.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    scvtf v20.2d, v20.2d
-; CHECK-SD-NEXT:    scvtf v21.2d, v21.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q5, q6, [x8, #80]
-; CHECK-SD-NEXT:    str q16, [x8, #240]
-; CHECK-SD-NEXT:    ldp q22, q16, [sp]
-; CHECK-SD-NEXT:    stp q3, q4, [x8, #48]
-; CHECK-SD-NEXT:    stp q20, q19, [x8, #176]
-; CHECK-SD-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    stp q1, q2, [x8, #16]
-; CHECK-SD-NEXT:    stp q18, q17, [x8, #208]
-; CHECK-SD-NEXT:    scvtf v17.2d, v22.2d
-; CHECK-SD-NEXT:    str q0, [x8]
-; CHECK-SD-NEXT:    stp q16, q21, [x8, #144]
-; CHECK-SD-NEXT:    stp q7, q17, [x8, #112]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v32i64_v32f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q0, q1, [x8]
-; CHECK-GI-NEXT:    scvtf v0.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v6.2d
-; CHECK-GI-NEXT:    str q2, [x8, #32]
-; CHECK-GI-NEXT:    ldp q2, q5, [sp]
-; CHECK-GI-NEXT:    scvtf v6.2d, v7.2d
-; CHECK-GI-NEXT:    stp q3, q4, [x8, #48]
-; CHECK-GI-NEXT:    ldp q3, q4, [sp, #32]
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    stp q0, q1, [x8, #80]
-; CHECK-GI-NEXT:    scvtf v0.2d, v3.2d
-; CHECK-GI-NEXT:    ldp q1, q3, [sp, #64]
-; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q6, q2, [x8, #112]
-; CHECK-GI-NEXT:    ldp q2, q6, [sp, #96]
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    stp q5, q0, [x8, #144]
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v0.2d, v6.2d
-; CHECK-GI-NEXT:    stp q4, q1, [x8, #176]
-; CHECK-GI-NEXT:    stp q3, q2, [x8, #208]
-; CHECK-GI-NEXT:    str q0, [x8, #240]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <32 x i64> %a to <32 x double>
-  ret <32 x double> %c
+define float @stofp_load_i32_f32(ptr %p) {
+; CHECK-LABEL: stofp_load_i32_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    scvtf s0, s0
+; CHECK-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %c = sitofp i32 %a to float
+  ret float %c
 }
 
-define <32 x double> @utofp_v32i64_v32f64(<32 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v32i64_v32f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q17, q16, [sp, #96]
-; CHECK-SD-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-SD-NEXT:    ldp q19, q18, [sp, #64]
-; CHECK-SD-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    ldp q21, q20, [sp, #32]
-; CHECK-SD-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    ucvtf v17.2d, v17.2d
-; CHECK-SD-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    ucvtf v18.2d, v18.2d
-; CHECK-SD-NEXT:    ucvtf v19.2d, v19.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    ucvtf v20.2d, v20.2d
-; CHECK-SD-NEXT:    ucvtf v21.2d, v21.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q5, q6, [x8, #80]
-; CHECK-SD-NEXT:    str q16, [x8, #240]
-; CHECK-SD-NEXT:    ldp q22, q16, [sp]
-; CHECK-SD-NEXT:    stp q3, q4, [x8, #48]
-; CHECK-SD-NEXT:    stp q20, q19, [x8, #176]
-; CHECK-SD-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    stp q1, q2, [x8, #16]
-; CHECK-SD-NEXT:    stp q18, q17, [x8, #208]
-; CHECK-SD-NEXT:    ucvtf v17.2d, v22.2d
-; CHECK-SD-NEXT:    str q0, [x8]
-; CHECK-SD-NEXT:    stp q16, q21, [x8, #144]
-; CHECK-SD-NEXT:    stp q7, q17, [x8, #112]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v32i64_v32f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q0, q1, [x8]
-; CHECK-GI-NEXT:    ucvtf v0.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v6.2d
-; CHECK-GI-NEXT:    str q2, [x8, #32]
-; CHECK-GI-NEXT:    ldp q2, q5, [sp]
-; CHECK-GI-NEXT:    ucvtf v6.2d, v7.2d
-; CHECK-GI-NEXT:    stp q3, q4, [x8, #48]
-; CHECK-GI-NEXT:    ldp q3, q4, [sp, #32]
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    stp q0, q1, [x8, #80]
-; CHECK-GI-NEXT:    ucvtf v0.2d, v3.2d
-; CHECK-GI-NEXT:    ldp q1, q3, [sp, #64]
-; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q6, q2, [x8, #112]
-; CHECK-GI-NEXT:    ldp q2, q6, [sp, #96]
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    stp q5, q0, [x8, #144]
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v0.2d, v6.2d
-; CHECK-GI-NEXT:    stp q4, q1, [x8, #176]
-; CHECK-GI-NEXT:    stp q3, q2, [x8, #208]
-; CHECK-GI-NEXT:    str q0, [x8, #240]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <32 x i64> %a to <32 x double>
-  ret <32 x double> %c
+define float @utofp_load_i32_f32(ptr %p) {
+; CHECK-LABEL: utofp_load_i32_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    ucvtf s0, s0
+; CHECK-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %c = uitofp i32 %a to float
+  ret float %c
 }
 
-define <2 x fp128> @stofp_v2i32_v2f128(<2 x i32> %a) {
-; CHECK-SD-LABEL: stofp_v2i32_v2f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #32
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov w0, s0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    mov w0, v1.s[1]
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    mov v1.16b, v0.16b
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #32
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i32_v2f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w30, -8
-; CHECK-GI-NEXT:    .cfi_offset b8, -16
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    mov s8, v0.s[1]
-; CHECK-GI-NEXT:    bl __floatsitf
-; CHECK-GI-NEXT:    fmov w0, s8
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatsitf
-; CHECK-GI-NEXT:    mov v1.16b, v0.16b
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+define float @stofp_load_i16_f32(ptr %p) {
+; CHECK-LABEL: stofp_load_i16_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrsh w8, [x0]
+; CHECK-NEXT:    scvtf s0, w8
+; CHECK-NEXT:    ret
 entry:
-  %c = sitofp <2 x i32> %a to <2 x fp128>
-  ret <2 x fp128> %c
+  %a = load i16, ptr %p
+  %c = sitofp i16 %a to float
+  ret float %c
 }
 
-define <2 x fp128> @utofp_v2i32_v2f128(<2 x i32> %a) {
-; CHECK-SD-LABEL: utofp_v2i32_v2f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #32
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov w0, s0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    mov w0, v1.s[1]
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    mov v1.16b, v0.16b
-; CHECK-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #32
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i32_v2f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #32
-; CHECK-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-GI-NEXT:    .cfi_offset w30, -8
-; CHECK-GI-NEXT:    .cfi_offset b8, -16
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    mov s8, v0.s[1]
-; CHECK-GI-NEXT:    bl __floatunsitf
-; CHECK-GI-NEXT:    fmov w0, s8
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatunsitf
-; CHECK-GI-NEXT:    mov v1.16b, v0.16b
-; CHECK-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #32
-; CHECK-GI-NEXT:    ret
+define float @utofp_load_i16_f32(ptr %p) {
+; CHECK-LABEL: utofp_load_i16_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr h0, [x0]
+; CHECK-NEXT:    ucvtf s0, s0
+; CHECK-NEXT:    ret
 entry:
-  %c = uitofp <2 x i32> %a to <2 x fp128>
-  ret <2 x fp128> %c
+  %a = load i16, ptr %p
+  %c = uitofp i16 %a to float
+  ret float %c
+}
+
+define float @stofp_load_i8_f32(ptr %p) {
+; CHECK-LABEL: stofp_load_i8_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrsb w8, [x0]
+; CHECK-NEXT:    scvtf s0, w8
+; CHECK-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %c = sitofp i8 %a to float
+  ret float %c
+}
+
+define float @utofp_load_i8_f32(ptr %p) {
+; CHECK-LABEL: utofp_load_i8_f32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr b0, [x0]
+; CHECK-NEXT:    ucvtf s0, s0
+; CHECK-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %c = uitofp i8 %a to float
+  ret float %c
+}
+
+define half @stofp_load_i64_f16(ptr %p) {
+; CHECK-NOFP16-LABEL: stofp_load_i64_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ldr x8, [x0]
+; CHECK-NOFP16-NEXT:    scvtf s0, x8
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_load_i64_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ldr x8, [x0]
+; CHECK-FP16-NEXT:    scvtf h0, x8
+; CHECK-FP16-NEXT:    ret
+entry:
+  %a = load i64, ptr %p
+  %c = sitofp i64 %a to half
+  ret half %c
+}
+
+define half @utofp_load_i64_f16(ptr %p) {
+; CHECK-NOFP16-LABEL: utofp_load_i64_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ldr x8, [x0]
+; CHECK-NOFP16-NEXT:    ucvtf s0, x8
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_load_i64_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ldr x8, [x0]
+; CHECK-FP16-NEXT:    ucvtf h0, x8
+; CHECK-FP16-NEXT:    ret
+entry:
+  %a = load i64, ptr %p
+  %c = uitofp i64 %a to half
+  ret half %c
+}
+
+define half @stofp_load_i32_f16(ptr %p) {
+; CHECK-NOFP16-LABEL: stofp_load_i32_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ldr s0, [x0]
+; CHECK-NOFP16-NEXT:    scvtf s0, s0
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_load_i32_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ldr w8, [x0]
+; CHECK-FP16-NEXT:    scvtf h0, w8
+; CHECK-FP16-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %c = sitofp i32 %a to half
+  ret half %c
+}
+
+define half @utofp_load_i32_f16(ptr %p) {
+; CHECK-NOFP16-LABEL: utofp_load_i32_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ldr s0, [x0]
+; CHECK-NOFP16-NEXT:    ucvtf s0, s0
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_load_i32_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ldr w8, [x0]
+; CHECK-FP16-NEXT:    ucvtf h0, w8
+; CHECK-FP16-NEXT:    ret
+entry:
+  %a = load i32, ptr %p
+  %c = uitofp i32 %a to half
+  ret half %c
+}
+
+define half @stofp_load_i16_f16(ptr %p) {
+; CHECK-NOFP16-LABEL: stofp_load_i16_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ldrsh w8, [x0]
+; CHECK-NOFP16-NEXT:    scvtf s0, w8
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_load_i16_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ldrsh w8, [x0]
+; CHECK-FP16-NEXT:    scvtf h0, w8
+; CHECK-FP16-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %c = sitofp i16 %a to half
+  ret half %c
+}
+
+define half @utofp_load_i16_f16(ptr %p) {
+; CHECK-NOFP16-LABEL: utofp_load_i16_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ldr h0, [x0]
+; CHECK-NOFP16-NEXT:    ucvtf s0, s0
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_load_i16_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ldrh w8, [x0]
+; CHECK-FP16-NEXT:    ucvtf h0, w8
+; CHECK-FP16-NEXT:    ret
+entry:
+  %a = load i16, ptr %p
+  %c = uitofp i16 %a to half
+  ret half %c
 }
 
-define <3 x fp128> @stofp_v3i32_v3f128(<3 x i32> %a) {
-; CHECK-SD-LABEL: stofp_v3i32_v3f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    fmov w0, s1
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov w0, s0
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov w0, v0.s[1]
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    mov v1.16b, v0.16b
-; CHECK-SD-NEXT:    ldp q0, q2, [sp] // 32-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i32_v3f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    .cfi_offset b8, -24
-; CHECK-GI-NEXT:    .cfi_offset b9, -32
-; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    mov s8, v0.s[1]
-; CHECK-GI-NEXT:    mov s9, v0.s[2]
-; CHECK-GI-NEXT:    bl __floatsitf
-; CHECK-GI-NEXT:    fmov w0, s8
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatsitf
-; CHECK-GI-NEXT:    fmov w0, s9
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatsitf
-; CHECK-GI-NEXT:    mov v2.16b, v0.16b
-; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <3 x i32> %a to <3 x fp128>
-  ret <3 x fp128> %c
-}
-
-define <3 x fp128> @utofp_v3i32_v3f128(<3 x i32> %a) {
-; CHECK-SD-LABEL: utofp_v3i32_v3f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w30, -16
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    fmov w0, s1
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    fmov w0, s0
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov w0, v0.s[1]
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    mov v1.16b, v0.16b
-; CHECK-SD-NEXT:    ldp q0, q2, [sp] // 32-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i32_v3f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    str x30, [sp, #48] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w30, -16
-; CHECK-GI-NEXT:    .cfi_offset b8, -24
-; CHECK-GI-NEXT:    .cfi_offset b9, -32
-; CHECK-GI-NEXT:    fmov w0, s0
-; CHECK-GI-NEXT:    mov s8, v0.s[1]
-; CHECK-GI-NEXT:    mov s9, v0.s[2]
-; CHECK-GI-NEXT:    bl __floatunsitf
-; CHECK-GI-NEXT:    fmov w0, s8
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatunsitf
-; CHECK-GI-NEXT:    fmov w0, s9
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatunsitf
-; CHECK-GI-NEXT:    mov v2.16b, v0.16b
-; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #48] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <3 x i32> %a to <3 x fp128>
-  ret <3 x fp128> %c
+define half @stofp_load_i8_f16(ptr %p) {
+; CHECK-NOFP16-LABEL: stofp_load_i8_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ldrsb w8, [x0]
+; CHECK-NOFP16-NEXT:    scvtf s0, w8
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_load_i8_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ldrsb w8, [x0]
+; CHECK-FP16-NEXT:    scvtf h0, w8
+; CHECK-FP16-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %c = sitofp i8 %a to half
+  ret half %c
+}
+
+define half @utofp_load_i8_f16(ptr %p) {
+; CHECK-NOFP16-LABEL: utofp_load_i8_f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ldr b0, [x0]
+; CHECK-NOFP16-NEXT:    ucvtf s0, s0
+; CHECK-NOFP16-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_load_i8_f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ldrb w8, [x0]
+; CHECK-FP16-NEXT:    ucvtf h0, w8
+; CHECK-FP16-NEXT:    ret
+entry:
+  %a = load i8, ptr %p
+  %c = uitofp i8 %a to half
+  ret half %c
+}
+
+define <2 x double> @stofp_v2i64_v2f64(<2 x i64> %a) {
+; CHECK-LABEL: stofp_v2i64_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = sitofp <2 x i64> %a to <2 x double>
+  ret <2 x double> %c
+}
+
+define <2 x double> @utofp_v2i64_v2f64(<2 x i64> %a) {
+; CHECK-LABEL: utofp_v2i64_v2f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = uitofp <2 x i64> %a to <2 x double>
+  ret <2 x double> %c
+}
+
+define <3 x double> @stofp_v3i64_v3f64(<3 x i64> %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_v3i64_v3f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v3i64_v3f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i64_v3f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v3i64_v3f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = sitofp <3 x i64> %a to <3 x double>
+  ret <3 x double> %c
+}
+
+define <3 x double> @utofp_v3i64_v3f64(<3 x i64> %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_v3i64_v3f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v3i64_v3f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i64_v3f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v3i64_v3f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = uitofp <3 x i64> %a to <3 x double>
+  ret <3 x double> %c
+}
+
+define <4 x double> @stofp_v4i64_v4f64(<4 x i64> %a) {
+; CHECK-LABEL: stofp_v4i64_v4f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = sitofp <4 x i64> %a to <4 x double>
+  ret <4 x double> %c
+}
+
+define <4 x double> @utofp_v4i64_v4f64(<4 x i64> %a) {
+; CHECK-LABEL: utofp_v4i64_v4f64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NEXT:    ret
+entry:
+  %c = uitofp <4 x i64> %a to <4 x double>
+  ret <4 x double> %c
 }
 
 define <2 x double> @stofp_v2i32_v2f64(<2 x i32> %a) {
@@ -1995,2317 +1341,2593 @@ entry:
 }
 
 define <3 x double> @stofp_v3i32_v3f64(<3 x i32> %a) {
-; CHECK-SD-LABEL: stofp_v3i32_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll v1.2d, v0.2s, #0
-; CHECK-SD-NEXT:    sshll2 v0.2d, v0.4s, #0
-; CHECK-SD-NEXT:    scvtf v3.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v0.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    fmov d0, d3
-; CHECK-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i32_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v1.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    scvtf v3.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v0.2d
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    mov d1, v3.d[1]
-; CHECK-GI-NEXT:    fmov d0, d3
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v3i32_v3f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    fmov d0, d3
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v3i32_v3f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    fmov d0, d3
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i32_v3f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    mov d1, v3.d[1]
+; CHECK-NOFP16-GI-NEXT:    fmov d0, d3
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v3i32_v3f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    mov d1, v3.d[1]
+; CHECK-FP16-GI-NEXT:    fmov d0, d3
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <3 x i32> %a to <3 x double>
   ret <3 x double> %c
 }
 
 define <3 x double> @utofp_v3i32_v3f64(<3 x i32> %a) {
-; CHECK-SD-LABEL: utofp_v3i32_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll v1.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ushll2 v0.2d, v0.4s, #0
-; CHECK-SD-NEXT:    ucvtf v3.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v0.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    fmov d0, d3
-; CHECK-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i32_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v1.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ucvtf v3.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v0.2d
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    mov d1, v3.d[1]
-; CHECK-GI-NEXT:    fmov d0, d3
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v3i32_v3f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    fmov d0, d3
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v3i32_v3f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    fmov d0, d3
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v3.16b, v3.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i32_v3f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    mov d1, v3.d[1]
+; CHECK-NOFP16-GI-NEXT:    fmov d0, d3
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v3i32_v3f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    mov d1, v3.d[1]
+; CHECK-FP16-GI-NEXT:    fmov d0, d3
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <3 x i32> %a to <3 x double>
   ret <3 x double> %c
 }
 
 define <4 x double> @stofp_v4i32_v4f64(<4 x i32> %a) {
-; CHECK-SD-LABEL: stofp_v4i32_v4f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v4i32_v4f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v1.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v2.2d, v0.4s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v4i32_v4f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v4i32_v4f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v4i32_v4f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v4i32_v4f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <4 x i32> %a to <4 x double>
   ret <4 x double> %c
 }
 
 define <4 x double> @utofp_v4i32_v4f64(<4 x i32> %a) {
-; CHECK-SD-LABEL: utofp_v4i32_v4f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v4i32_v4f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v1.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v2.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v4i32_v4f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v4i32_v4f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v4i32_v4f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v4i32_v4f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <4 x i32> %a to <4 x double>
   ret <4 x double> %c
 }
 
 define <8 x double> @stofp_v8i32_v8f64(<8 x i32> %a) {
-; CHECK-SD-LABEL: stofp_v8i32_v8f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll v2.2d, v0.2s, #0
-; CHECK-SD-NEXT:    sshll2 v3.2d, v0.4s, #0
-; CHECK-SD-NEXT:    sshll2 v4.2d, v1.4s, #0
-; CHECK-SD-NEXT:    sshll v5.2d, v1.2s, #0
-; CHECK-SD-NEXT:    scvtf v0.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v3.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v4.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v5.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v8i32_v8f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v2.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v3.2d, v0.4s, #0
-; CHECK-GI-NEXT:    sshll v4.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v5.2d, v1.4s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v5.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i32_v8f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v2.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v3.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v4.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v5.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v8i32_v8f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll v2.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v3.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v4.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll v5.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i32_v8f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v2.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v3.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v4.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v5.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i32_v8f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v2.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v3.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v4.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v5.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i32> %a to <8 x double>
   ret <8 x double> %c
 }
 
 define <8 x double> @utofp_v8i32_v8f64(<8 x i32> %a) {
-; CHECK-SD-LABEL: utofp_v8i32_v8f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll v2.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ushll2 v3.2d, v0.4s, #0
-; CHECK-SD-NEXT:    ushll2 v4.2d, v1.4s, #0
-; CHECK-SD-NEXT:    ushll v5.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v3.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v4.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v5.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v8i32_v8f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v2.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v3.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ushll v4.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v5.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v5.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i32_v8f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v2.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v3.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v4.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v5.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v8i32_v8f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v2.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v3.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v4.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll v5.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i32_v8f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v2.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v3.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v4.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v5.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i32_v8f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v2.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v3.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v4.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v5.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i32> %a to <8 x double>
   ret <8 x double> %c
 }
 
-define <16 x double> @stofp_v16i32_v16f64(<16 x i32> %a) {
-; CHECK-SD-LABEL: stofp_v16i32_v16f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll2 v4.2d, v0.4s, #0
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    sshll2 v5.2d, v1.4s, #0
-; CHECK-SD-NEXT:    sshll v6.2d, v1.2s, #0
-; CHECK-SD-NEXT:    sshll v7.2d, v2.2s, #0
-; CHECK-SD-NEXT:    sshll2 v16.2d, v2.4s, #0
-; CHECK-SD-NEXT:    sshll2 v17.2d, v3.4s, #0
-; CHECK-SD-NEXT:    sshll v18.2d, v3.2s, #0
-; CHECK-SD-NEXT:    scvtf v1.2d, v4.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v5.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v6.2d
-; CHECK-SD-NEXT:    scvtf v4.2d, v7.2d
-; CHECK-SD-NEXT:    scvtf v5.2d, v16.2d
-; CHECK-SD-NEXT:    scvtf v7.2d, v17.2d
-; CHECK-SD-NEXT:    scvtf v6.2d, v18.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v16i32_v16f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v4.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v5.2d, v0.4s, #0
-; CHECK-GI-NEXT:    sshll v6.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v7.2d, v1.4s, #0
-; CHECK-GI-NEXT:    sshll v16.2d, v2.2s, #0
-; CHECK-GI-NEXT:    sshll2 v17.2d, v2.4s, #0
-; CHECK-GI-NEXT:    sshll v18.2d, v3.2s, #0
-; CHECK-GI-NEXT:    sshll2 v19.2d, v3.4s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v16.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v17.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v18.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v19.2d
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <16 x i32> %a to <16 x double>
-  ret <16 x double> %c
-}
-
-define <16 x double> @utofp_v16i32_v16f64(<16 x i32> %a) {
-; CHECK-SD-LABEL: utofp_v16i32_v16f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll2 v4.2d, v0.4s, #0
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ushll2 v5.2d, v1.4s, #0
-; CHECK-SD-NEXT:    ushll v6.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ushll v7.2d, v2.2s, #0
-; CHECK-SD-NEXT:    ushll2 v16.2d, v2.4s, #0
-; CHECK-SD-NEXT:    ushll2 v17.2d, v3.4s, #0
-; CHECK-SD-NEXT:    ushll v18.2d, v3.2s, #0
-; CHECK-SD-NEXT:    ucvtf v1.2d, v4.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v5.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v6.2d
-; CHECK-SD-NEXT:    ucvtf v4.2d, v7.2d
-; CHECK-SD-NEXT:    ucvtf v5.2d, v16.2d
-; CHECK-SD-NEXT:    ucvtf v7.2d, v17.2d
-; CHECK-SD-NEXT:    ucvtf v6.2d, v18.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v16i32_v16f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v4.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v5.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ushll v6.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v7.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ushll v16.2d, v2.2s, #0
-; CHECK-GI-NEXT:    ushll2 v17.2d, v2.4s, #0
-; CHECK-GI-NEXT:    ushll v18.2d, v3.2s, #0
-; CHECK-GI-NEXT:    ushll2 v19.2d, v3.4s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v16.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v17.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v18.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v19.2d
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <16 x i32> %a to <16 x double>
-  ret <16 x double> %c
-}
-
-define <32 x double> @stofp_v32i32_v32f64(<32 x i32> %a) {
-; CHECK-SD-LABEL: stofp_v32i32_v32f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll2 v17.2d, v6.4s, #0
-; CHECK-SD-NEXT:    sshll v6.2d, v6.2s, #0
-; CHECK-SD-NEXT:    sshll2 v16.2d, v7.4s, #0
-; CHECK-SD-NEXT:    sshll v19.2d, v3.2s, #0
-; CHECK-SD-NEXT:    sshll v7.2d, v7.2s, #0
-; CHECK-SD-NEXT:    sshll2 v3.2d, v3.4s, #0
-; CHECK-SD-NEXT:    sshll2 v20.2d, v4.4s, #0
-; CHECK-SD-NEXT:    sshll v4.2d, v4.2s, #0
-; CHECK-SD-NEXT:    sshll2 v18.2d, v5.4s, #0
-; CHECK-SD-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-SD-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    sshll2 v21.2d, v2.4s, #0
-; CHECK-SD-NEXT:    scvtf v20.2d, v20.2d
-; CHECK-SD-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    sshll v5.2d, v5.2s, #0
-; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    scvtf v18.2d, v18.2d
-; CHECK-SD-NEXT:    stp q6, q17, [x8, #192]
-; CHECK-SD-NEXT:    scvtf v17.2d, v19.2d
-; CHECK-SD-NEXT:    stp q7, q16, [x8, #224]
-; CHECK-SD-NEXT:    sshll2 v7.2d, v0.4s, #0
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    stp q4, q20, [x8, #128]
-; CHECK-SD-NEXT:    scvtf v16.2d, v21.2d
-; CHECK-SD-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    stp q17, q3, [x8, #96]
-; CHECK-SD-NEXT:    sshll2 v3.2d, v1.4s, #0
-; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    scvtf v4.2d, v7.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q5, q18, [x8, #160]
-; CHECK-SD-NEXT:    stp q2, q16, [x8, #64]
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    stp q0, q4, [x8]
-; CHECK-SD-NEXT:    stp q1, q3, [x8, #32]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v32i32_v32f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v16.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    sshll v17.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    sshll v18.2d, v2.2s, #0
-; CHECK-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    sshll v19.2d, v3.2s, #0
-; CHECK-GI-NEXT:    sshll2 v3.2d, v3.4s, #0
-; CHECK-GI-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v18.2d, v18.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    stp q16, q0, [x8]
-; CHECK-GI-NEXT:    sshll v16.2d, v4.2s, #0
-; CHECK-GI-NEXT:    sshll2 v0.2d, v4.4s, #0
-; CHECK-GI-NEXT:    scvtf v4.2d, v19.2d
-; CHECK-GI-NEXT:    stp q17, q1, [x8, #32]
-; CHECK-GI-NEXT:    sshll v17.2d, v5.2s, #0
-; CHECK-GI-NEXT:    sshll2 v1.2d, v5.4s, #0
-; CHECK-GI-NEXT:    stp q18, q2, [x8, #64]
-; CHECK-GI-NEXT:    sshll2 v2.2d, v6.4s, #0
-; CHECK-GI-NEXT:    scvtf v5.2d, v16.2d
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    sshll v16.2d, v6.2s, #0
-; CHECK-GI-NEXT:    scvtf v6.2d, v17.2d
-; CHECK-GI-NEXT:    stp q4, q3, [x8, #96]
-; CHECK-GI-NEXT:    sshll v4.2d, v7.2s, #0
-; CHECK-GI-NEXT:    sshll2 v3.2d, v7.4s, #0
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v16.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    stp q5, q0, [x8, #128]
-; CHECK-GI-NEXT:    scvtf v0.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    stp q6, q1, [x8, #160]
-; CHECK-GI-NEXT:    stp q7, q2, [x8, #192]
-; CHECK-GI-NEXT:    stp q0, q3, [x8, #224]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <32 x i32> %a to <32 x double>
-  ret <32 x double> %c
-}
-
-define <32 x double> @utofp_v32i32_v32f64(<32 x i32> %a) {
-; CHECK-SD-LABEL: utofp_v32i32_v32f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll2 v17.2d, v6.4s, #0
-; CHECK-SD-NEXT:    ushll v6.2d, v6.2s, #0
-; CHECK-SD-NEXT:    ushll2 v16.2d, v7.4s, #0
-; CHECK-SD-NEXT:    ushll v19.2d, v3.2s, #0
-; CHECK-SD-NEXT:    ushll v7.2d, v7.2s, #0
-; CHECK-SD-NEXT:    ushll2 v3.2d, v3.4s, #0
-; CHECK-SD-NEXT:    ushll2 v20.2d, v4.4s, #0
-; CHECK-SD-NEXT:    ushll v4.2d, v4.2s, #0
-; CHECK-SD-NEXT:    ushll2 v18.2d, v5.4s, #0
-; CHECK-SD-NEXT:    ucvtf v17.2d, v17.2d
-; CHECK-SD-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    ushll2 v21.2d, v2.4s, #0
-; CHECK-SD-NEXT:    ucvtf v20.2d, v20.2d
-; CHECK-SD-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    ushll v5.2d, v5.2s, #0
-; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    ucvtf v18.2d, v18.2d
-; CHECK-SD-NEXT:    stp q6, q17, [x8, #192]
-; CHECK-SD-NEXT:    ucvtf v17.2d, v19.2d
-; CHECK-SD-NEXT:    stp q7, q16, [x8, #224]
-; CHECK-SD-NEXT:    ushll2 v7.2d, v0.4s, #0
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    stp q4, q20, [x8, #128]
-; CHECK-SD-NEXT:    ucvtf v16.2d, v21.2d
-; CHECK-SD-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    stp q17, q3, [x8, #96]
-; CHECK-SD-NEXT:    ushll2 v3.2d, v1.4s, #0
-; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ucvtf v4.2d, v7.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q5, q18, [x8, #160]
-; CHECK-SD-NEXT:    stp q2, q16, [x8, #64]
-; CHECK-SD-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    stp q0, q4, [x8]
-; CHECK-SD-NEXT:    stp q1, q3, [x8, #32]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v32i32_v32f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v16.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ushll v17.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ushll v18.2d, v2.2s, #0
-; CHECK-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    ushll v19.2d, v3.2s, #0
-; CHECK-GI-NEXT:    ushll2 v3.2d, v3.4s, #0
-; CHECK-GI-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v17.2d, v17.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v18.2d, v18.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    stp q16, q0, [x8]
-; CHECK-GI-NEXT:    ushll v16.2d, v4.2s, #0
-; CHECK-GI-NEXT:    ushll2 v0.2d, v4.4s, #0
-; CHECK-GI-NEXT:    ucvtf v4.2d, v19.2d
-; CHECK-GI-NEXT:    stp q17, q1, [x8, #32]
-; CHECK-GI-NEXT:    ushll v17.2d, v5.2s, #0
-; CHECK-GI-NEXT:    ushll2 v1.2d, v5.4s, #0
-; CHECK-GI-NEXT:    stp q18, q2, [x8, #64]
-; CHECK-GI-NEXT:    ushll2 v2.2d, v6.4s, #0
-; CHECK-GI-NEXT:    ucvtf v5.2d, v16.2d
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ushll v16.2d, v6.2s, #0
-; CHECK-GI-NEXT:    ucvtf v6.2d, v17.2d
-; CHECK-GI-NEXT:    stp q4, q3, [x8, #96]
-; CHECK-GI-NEXT:    ushll v4.2d, v7.2s, #0
-; CHECK-GI-NEXT:    ushll2 v3.2d, v7.4s, #0
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v16.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    stp q5, q0, [x8, #128]
-; CHECK-GI-NEXT:    ucvtf v0.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    stp q6, q1, [x8, #160]
-; CHECK-GI-NEXT:    stp q7, q2, [x8, #192]
-; CHECK-GI-NEXT:    stp q0, q3, [x8, #224]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <32 x i32> %a to <32 x double>
-  ret <32 x double> %c
-}
-
-define <2 x fp128> @stofp_v2i16_v2f128(<2 x i16> %a) {
-; CHECK-LABEL: stofp_v2i16_v2f128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #32
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    sxth w0, w8
-; CHECK-NEXT:    bl __floatsitf
-; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov w8, v1.s[1]
-; CHECK-NEXT:    sxth w0, w8
-; CHECK-NEXT:    bl __floatsitf
-; CHECK-NEXT:    mov v1.16b, v0.16b
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #32
-; CHECK-NEXT:    ret
-entry:
-  %c = sitofp <2 x i16> %a to <2 x fp128>
-  ret <2 x fp128> %c
-}
-
-define <2 x fp128> @utofp_v2i16_v2f128(<2 x i16> %a) {
-; CHECK-LABEL: utofp_v2i16_v2f128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #32
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    and w0, w8, #0xffff
-; CHECK-NEXT:    bl __floatunsitf
-; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov w8, v1.s[1]
-; CHECK-NEXT:    and w0, w8, #0xffff
-; CHECK-NEXT:    bl __floatunsitf
-; CHECK-NEXT:    mov v1.16b, v0.16b
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #32
-; CHECK-NEXT:    ret
-entry:
-  %c = uitofp <2 x i16> %a to <2 x fp128>
-  ret <2 x fp128> %c
-}
-
-define <3 x fp128> @stofp_v3i16_v3f128(<3 x i16> %a) {
-; CHECK-LABEL: stofp_v3i16_v3f128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    smov w0, v0.h[0]
-; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    bl __floatsitf
-; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    smov w0, v1.h[1]
-; CHECK-NEXT:    bl __floatsitf
-; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    smov w0, v1.h[2]
-; CHECK-NEXT:    bl __floatsitf
-; CHECK-NEXT:    mov v2.16b, v0.16b
-; CHECK-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
-entry:
-  %c = sitofp <3 x i16> %a to <3 x fp128>
-  ret <3 x fp128> %c
-}
-
-define <3 x fp128> @utofp_v3i16_v3f128(<3 x i16> %a) {
-; CHECK-LABEL: utofp_v3i16_v3f128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #48
-; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w0, v0.h[0]
-; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    bl __floatunsitf
-; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    umov w0, v1.h[1]
-; CHECK-NEXT:    bl __floatunsitf
-; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    umov w0, v1.h[2]
-; CHECK-NEXT:    bl __floatunsitf
-; CHECK-NEXT:    mov v2.16b, v0.16b
-; CHECK-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #48
-; CHECK-NEXT:    ret
-entry:
-  %c = uitofp <3 x i16> %a to <3 x fp128>
-  ret <3 x fp128> %c
-}
-
 define <2 x double> @stofp_v2i16_v2f64(<2 x i16> %a) {
-; CHECK-SD-LABEL: stofp_v2i16_v2f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #16
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i16_v2f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #48
-; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #48
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v2i16_v2f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-NOFP16-SD-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i16_v2f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-FP16-SD-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i16_v2f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    shl v0.2d, v0.2d, #48
+; CHECK-NOFP16-GI-NEXT:    sshr v0.2d, v0.2d, #48
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i16_v2f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    shl v0.2d, v0.2d, #48
+; CHECK-FP16-GI-NEXT:    sshr v0.2d, v0.2d, #48
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <2 x i16> %a to <2 x double>
   ret <2 x double> %c
 }
 
 define <2 x double> @utofp_v2i16_v2f64(<2 x i16> %a) {
-; CHECK-SD-LABEL: utofp_v2i16_v2f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi d1, #0x00ffff0000ffff
-; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i16_v2f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v1.2d, #0x0000000000ffff
-; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v2i16_v2f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-NOFP16-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i16_v2f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-FP16-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i16_v2f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    movi v1.2d, #0x0000000000ffff
+; CHECK-NOFP16-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i16_v2f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x0000000000ffff
+; CHECK-FP16-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <2 x i16> %a to <2 x double>
   ret <2 x double> %c
 }
 
 define <3 x double> @stofp_v3i16_v3f64(<3 x i16> %a) {
-; CHECK-SD-LABEL: stofp_v3i16_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-SD-NEXT:    sshll v0.2d, v1.2s, #0
-; CHECK-SD-NEXT:    sshll2 v1.2d, v1.4s, #0
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v1.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i16_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll v0.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v1.2d
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v3i16_v3f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v3i16_v3f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i16_v3f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v0.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v3i16_v3f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll v0.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <3 x i16> %a to <3 x double>
   ret <3 x double> %c
 }
 
 define <3 x double> @utofp_v3i16_v3f64(<3 x i16> %a) {
-; CHECK-SD-LABEL: utofp_v3i16_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ushll v0.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ushll2 v1.2d, v1.4s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v1.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i16_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll v0.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v1.2d
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v3i16_v3f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v3i16_v3f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i16_v3f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v3i16_v3f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <3 x i16> %a to <3 x double>
   ret <3 x double> %c
 }
 
 define <4 x double> @stofp_v4i16_v4f64(<4 x i16> %a) {
-; CHECK-SD-LABEL: stofp_v4i16_v4f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v4i16_v4f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll v1.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v2.2d, v0.4s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v4i16_v4f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v4i16_v4f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v1.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v4i16_v4f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v4i16_v4f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll v1.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <4 x i16> %a to <4 x double>
   ret <4 x double> %c
 }
 
 define <4 x double> @utofp_v4i16_v4f64(<4 x i16> %a) {
-; CHECK-SD-LABEL: utofp_v4i16_v4f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v4i16_v4f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll v1.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v2.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v4i16_v4f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v4i16_v4f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v1.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v4i16_v4f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v4i16_v4f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <4 x i16> %a to <4 x double>
   ret <4 x double> %c
 }
 
 define <8 x double> @stofp_v8i16_v8f64(<8 x i16> %a) {
-; CHECK-SD-LABEL: stofp_v8i16_v8f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-SD-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll v2.2d, v1.2s, #0
-; CHECK-SD-NEXT:    sshll2 v3.2d, v0.4s, #0
-; CHECK-SD-NEXT:    sshll2 v1.2d, v1.4s, #0
-; CHECK-SD-NEXT:    sshll v4.2d, v0.2s, #0
-; CHECK-SD-NEXT:    scvtf v0.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v4.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v8i16_v8f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    sshll v2.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    sshll v3.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v4.2d, v0.4s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v4.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i16_v8f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v3.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v4.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v8i16_v8f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v3.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll v4.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i16_v8f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v4.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i16_v8f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v3.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v4.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i16> %a to <8 x double>
   ret <8 x double> %c
 }
 
 define <8 x double> @utofp_v8i16_v8f64(<8 x i16> %a) {
-; CHECK-SD-LABEL: utofp_v8i16_v8f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-SD-NEXT:    ushll v2.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ushll2 v3.2d, v0.4s, #0
-; CHECK-SD-NEXT:    ushll2 v1.2d, v1.4s, #0
-; CHECK-SD-NEXT:    ushll v4.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v4.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v8i16_v8f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll v2.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ushll v3.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v4.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v4.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i16_v8f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v3.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v4.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v8i16_v8f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v3.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll v4.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i16_v8f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v4.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i16_v8f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v3.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v4.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i16> %a to <8 x double>
   ret <8 x double> %c
 }
 
 define <16 x double> @stofp_v16i16_v16f64(<16 x i16> %a) {
-; CHECK-SD-LABEL: stofp_v16i16_v16f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-SD-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll v3.4s, v1.4h, #0
-; CHECK-SD-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-SD-NEXT:    sshll2 v4.2d, v2.4s, #0
-; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    sshll2 v5.2d, v0.4s, #0
-; CHECK-SD-NEXT:    sshll v6.2d, v0.2s, #0
-; CHECK-SD-NEXT:    sshll v7.2d, v3.2s, #0
-; CHECK-SD-NEXT:    sshll2 v16.2d, v1.4s, #0
-; CHECK-SD-NEXT:    sshll2 v17.2d, v3.4s, #0
-; CHECK-SD-NEXT:    sshll v18.2d, v1.2s, #0
-; CHECK-SD-NEXT:    scvtf v1.2d, v4.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v5.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v6.2d
-; CHECK-SD-NEXT:    scvtf v4.2d, v7.2d
-; CHECK-SD-NEXT:    scvtf v7.2d, v16.2d
-; CHECK-SD-NEXT:    scvtf v5.2d, v17.2d
-; CHECK-SD-NEXT:    scvtf v6.2d, v18.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v16i16_v16f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    sshll v3.4s, v1.4h, #0
-; CHECK-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    sshll v4.2d, v2.2s, #0
-; CHECK-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    sshll v5.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v6.2d, v0.4s, #0
-; CHECK-GI-NEXT:    sshll v7.2d, v3.2s, #0
-; CHECK-GI-NEXT:    sshll2 v16.2d, v3.4s, #0
-; CHECK-GI-NEXT:    sshll v17.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v18.2d, v1.4s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v16.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v17.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v18.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i16_v16f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v4.2d, v2.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v5.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v6.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v7.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v16.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v17.2d, v3.4s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v18.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v4.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v7.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v5.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v6.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v16i16_v16f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v4.2d, v2.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v5.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll v6.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v7.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v16.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v17.2d, v3.4s, #0
+; CHECK-FP16-SD-NEXT:    sshll v18.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    scvtf v4.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    scvtf v7.2d, v16.2d
+; CHECK-FP16-SD-NEXT:    scvtf v5.2d, v17.2d
+; CHECK-FP16-SD-NEXT:    scvtf v6.2d, v18.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i16_v16f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v4.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v5.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v6.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v7.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v16.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v17.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v18.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v4.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v5.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v6.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v7.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i16_v16f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v4.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v5.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v6.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v7.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v16.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v17.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v18.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    scvtf v4.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    scvtf v5.2d, v16.2d
+; CHECK-FP16-GI-NEXT:    scvtf v6.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    scvtf v7.2d, v18.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i16> %a to <16 x double>
   ret <16 x double> %c
 }
 
 define <16 x double> @utofp_v16i16_v16f64(<16 x i16> %a) {
-; CHECK-SD-LABEL: utofp_v16i16_v16f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-SD-NEXT:    ushll v3.4s, v1.4h, #0
-; CHECK-SD-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-SD-NEXT:    ushll2 v4.2d, v2.4s, #0
-; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    ushll2 v5.2d, v0.4s, #0
-; CHECK-SD-NEXT:    ushll v6.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ushll v7.2d, v3.2s, #0
-; CHECK-SD-NEXT:    ushll2 v16.2d, v1.4s, #0
-; CHECK-SD-NEXT:    ushll2 v17.2d, v3.4s, #0
-; CHECK-SD-NEXT:    ushll v18.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ucvtf v1.2d, v4.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v5.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v6.2d
-; CHECK-SD-NEXT:    ucvtf v4.2d, v7.2d
-; CHECK-SD-NEXT:    ucvtf v7.2d, v16.2d
-; CHECK-SD-NEXT:    ucvtf v5.2d, v17.2d
-; CHECK-SD-NEXT:    ucvtf v6.2d, v18.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v16i16_v16f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll v3.4s, v1.4h, #0
-; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    ushll v4.2d, v2.2s, #0
-; CHECK-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    ushll v5.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v6.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ushll v7.2d, v3.2s, #0
-; CHECK-GI-NEXT:    ushll2 v16.2d, v3.4s, #0
-; CHECK-GI-NEXT:    ushll v17.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v18.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v16.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v17.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v18.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i16_v16f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v4.2d, v2.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v5.2d, v0.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v6.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v7.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v16.2d, v1.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v17.2d, v3.4s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v18.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v4.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v7.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v5.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v6.2d, v18.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v16i16_v16f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v4.2d, v2.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v5.2d, v0.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll v6.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v7.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v16.2d, v1.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v17.2d, v3.4s, #0
+; CHECK-FP16-SD-NEXT:    ushll v18.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v4.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v7.2d, v16.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v5.2d, v17.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v6.2d, v18.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i16_v16f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v4.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v5.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v6.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v7.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v16.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v17.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v18.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v4.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v5.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v6.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v7.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i16_v16f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v4.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v5.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v6.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v7.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v16.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v17.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v18.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v4.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v5.2d, v16.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v6.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v7.2d, v18.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i16> %a to <16 x double>
   ret <16 x double> %c
 }
 
-define <32 x double> @stofp_v32i16_v32f64(<32 x i16> %a) {
-; CHECK-SD-LABEL: stofp_v32i16_v32f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll2 v4.4s, v3.8h, #0
-; CHECK-SD-NEXT:    sshll2 v5.4s, v2.8h, #0
-; CHECK-SD-NEXT:    sshll2 v16.4s, v1.8h, #0
-; CHECK-SD-NEXT:    sshll2 v17.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT:    sshll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    sshll2 v6.2d, v4.4s, #0
-; CHECK-SD-NEXT:    sshll v4.2d, v4.2s, #0
-; CHECK-SD-NEXT:    sshll2 v7.2d, v5.4s, #0
-; CHECK-SD-NEXT:    sshll v5.2d, v5.2s, #0
-; CHECK-SD-NEXT:    sshll2 v18.2d, v16.4s, #0
-; CHECK-SD-NEXT:    sshll v16.2d, v16.2s, #0
-; CHECK-SD-NEXT:    sshll2 v19.2d, v17.4s, #0
-; CHECK-SD-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-SD-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    stp q4, q6, [x8, #224]
-; CHECK-SD-NEXT:    sshll v6.2d, v17.2s, #0
-; CHECK-SD-NEXT:    scvtf v17.2d, v18.2d
-; CHECK-SD-NEXT:    stp q5, q7, [x8, #160]
-; CHECK-SD-NEXT:    sshll2 v7.2d, v3.4s, #0
-; CHECK-SD-NEXT:    sshll v3.2d, v3.2s, #0
-; CHECK-SD-NEXT:    scvtf v4.2d, v19.2d
-; CHECK-SD-NEXT:    scvtf v5.2d, v6.2d
-; CHECK-SD-NEXT:    sshll2 v6.2d, v2.4s, #0
-; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    stp q16, q17, [x8, #96]
-; CHECK-SD-NEXT:    sshll2 v16.2d, v1.4s, #0
-; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    stp q5, q4, [x8, #32]
-; CHECK-SD-NEXT:    sshll2 v4.2d, v0.4s, #0
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    scvtf v5.2d, v16.2d
-; CHECK-SD-NEXT:    stp q3, q7, [x8, #192]
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v4.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q2, q6, [x8, #128]
-; CHECK-SD-NEXT:    stp q1, q5, [x8, #64]
-; CHECK-SD-NEXT:    stp q0, q3, [x8]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v32i16_v32f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v4.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    sshll v6.4s, v1.4h, #0
-; CHECK-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    sshll v16.4s, v2.4h, #0
-; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT:    sshll v18.4s, v3.4h, #0
-; CHECK-GI-NEXT:    sshll2 v3.4s, v3.8h, #0
-; CHECK-GI-NEXT:    sshll v5.2d, v4.2s, #0
-; CHECK-GI-NEXT:    sshll2 v4.2d, v4.4s, #0
-; CHECK-GI-NEXT:    sshll v7.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    sshll v17.2d, v6.2s, #0
-; CHECK-GI-NEXT:    sshll2 v6.2d, v6.4s, #0
-; CHECK-GI-NEXT:    sshll v19.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    stp q5, q4, [x8]
-; CHECK-GI-NEXT:    scvtf v5.2d, v17.2d
-; CHECK-GI-NEXT:    sshll v4.2d, v16.2s, #0
-; CHECK-GI-NEXT:    stp q7, q0, [x8, #32]
-; CHECK-GI-NEXT:    sshll2 v7.2d, v16.4s, #0
-; CHECK-GI-NEXT:    scvtf v16.2d, v19.2d
-; CHECK-GI-NEXT:    sshll v0.2d, v2.2s, #0
-; CHECK-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q5, q6, [x8, #64]
-; CHECK-GI-NEXT:    scvtf v6.2d, v7.2d
-; CHECK-GI-NEXT:    sshll v5.2d, v18.2s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    stp q16, q1, [x8, #96]
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    sshll2 v7.2d, v18.4s, #0
-; CHECK-GI-NEXT:    sshll v1.2d, v3.2s, #0
-; CHECK-GI-NEXT:    sshll2 v3.2d, v3.4s, #0
-; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    stp q4, q6, [x8, #128]
-; CHECK-GI-NEXT:    scvtf v4.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    stp q0, q2, [x8, #160]
-; CHECK-GI-NEXT:    scvtf v0.2d, v3.2d
-; CHECK-GI-NEXT:    stp q5, q4, [x8, #192]
-; CHECK-GI-NEXT:    stp q1, q0, [x8, #224]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <32 x i16> %a to <32 x double>
-  ret <32 x double> %c
-}
-
-define <32 x double> @utofp_v32i16_v32f64(<32 x i16> %a) {
-; CHECK-SD-LABEL: utofp_v32i16_v32f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll2 v4.4s, v3.8h, #0
-; CHECK-SD-NEXT:    ushll2 v5.4s, v2.8h, #0
-; CHECK-SD-NEXT:    ushll2 v16.4s, v1.8h, #0
-; CHECK-SD-NEXT:    ushll2 v17.4s, v0.8h, #0
-; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ushll2 v6.2d, v4.4s, #0
-; CHECK-SD-NEXT:    ushll v4.2d, v4.2s, #0
-; CHECK-SD-NEXT:    ushll2 v7.2d, v5.4s, #0
-; CHECK-SD-NEXT:    ushll v5.2d, v5.2s, #0
-; CHECK-SD-NEXT:    ushll2 v18.2d, v16.4s, #0
-; CHECK-SD-NEXT:    ushll v16.2d, v16.2s, #0
-; CHECK-SD-NEXT:    ushll2 v19.2d, v17.4s, #0
-; CHECK-SD-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-SD-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    stp q4, q6, [x8, #224]
-; CHECK-SD-NEXT:    ushll v6.2d, v17.2s, #0
-; CHECK-SD-NEXT:    ucvtf v17.2d, v18.2d
-; CHECK-SD-NEXT:    stp q5, q7, [x8, #160]
-; CHECK-SD-NEXT:    ushll2 v7.2d, v3.4s, #0
-; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
-; CHECK-SD-NEXT:    ucvtf v4.2d, v19.2d
-; CHECK-SD-NEXT:    ucvtf v5.2d, v6.2d
-; CHECK-SD-NEXT:    ushll2 v6.2d, v2.4s, #0
-; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    stp q16, q17, [x8, #96]
-; CHECK-SD-NEXT:    ushll2 v16.2d, v1.4s, #0
-; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    stp q5, q4, [x8, #32]
-; CHECK-SD-NEXT:    ushll2 v4.2d, v0.4s, #0
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ucvtf v5.2d, v16.2d
-; CHECK-SD-NEXT:    stp q3, q7, [x8, #192]
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v4.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q2, q6, [x8, #128]
-; CHECK-SD-NEXT:    stp q1, q5, [x8, #64]
-; CHECK-SD-NEXT:    stp q0, q3, [x8]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v32i16_v32f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v4.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll v6.4s, v1.4h, #0
-; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    ushll v16.4s, v2.4h, #0
-; CHECK-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT:    ushll v18.4s, v3.4h, #0
-; CHECK-GI-NEXT:    ushll2 v3.4s, v3.8h, #0
-; CHECK-GI-NEXT:    ushll v5.2d, v4.2s, #0
-; CHECK-GI-NEXT:    ushll2 v4.2d, v4.4s, #0
-; CHECK-GI-NEXT:    ushll v7.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ushll v17.2d, v6.2s, #0
-; CHECK-GI-NEXT:    ushll2 v6.2d, v6.4s, #0
-; CHECK-GI-NEXT:    ushll v19.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    stp q5, q4, [x8]
-; CHECK-GI-NEXT:    ucvtf v5.2d, v17.2d
-; CHECK-GI-NEXT:    ushll v4.2d, v16.2s, #0
-; CHECK-GI-NEXT:    stp q7, q0, [x8, #32]
-; CHECK-GI-NEXT:    ushll2 v7.2d, v16.4s, #0
-; CHECK-GI-NEXT:    ucvtf v16.2d, v19.2d
-; CHECK-GI-NEXT:    ushll v0.2d, v2.2s, #0
-; CHECK-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    stp q5, q6, [x8, #64]
-; CHECK-GI-NEXT:    ucvtf v6.2d, v7.2d
-; CHECK-GI-NEXT:    ushll v5.2d, v18.2s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    stp q16, q1, [x8, #96]
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ushll2 v7.2d, v18.4s, #0
-; CHECK-GI-NEXT:    ushll v1.2d, v3.2s, #0
-; CHECK-GI-NEXT:    ushll2 v3.2d, v3.4s, #0
-; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    stp q4, q6, [x8, #128]
-; CHECK-GI-NEXT:    ucvtf v4.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    stp q0, q2, [x8, #160]
-; CHECK-GI-NEXT:    ucvtf v0.2d, v3.2d
-; CHECK-GI-NEXT:    stp q5, q4, [x8, #192]
-; CHECK-GI-NEXT:    stp q1, q0, [x8, #224]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <32 x i16> %a to <32 x double>
-  ret <32 x double> %c
-}
-
-define <2 x fp128> @stofp_v2i8_v2f128(<2 x i8> %a) {
-; CHECK-LABEL: stofp_v2i8_v2f128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #32
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    sxtb w0, w8
-; CHECK-NEXT:    bl __floatsitf
-; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov w8, v1.s[1]
-; CHECK-NEXT:    sxtb w0, w8
-; CHECK-NEXT:    bl __floatsitf
-; CHECK-NEXT:    mov v1.16b, v0.16b
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #32
-; CHECK-NEXT:    ret
-entry:
-  %c = sitofp <2 x i8> %a to <2 x fp128>
-  ret <2 x fp128> %c
-}
-
-define <2 x fp128> @utofp_v2i8_v2f128(<2 x i8> %a) {
-; CHECK-LABEL: utofp_v2i8_v2f128:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #32
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fmov w8, s0
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    and w0, w8, #0xff
-; CHECK-NEXT:    bl __floatunsitf
-; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov w8, v1.s[1]
-; CHECK-NEXT:    and w0, w8, #0xff
-; CHECK-NEXT:    bl __floatunsitf
-; CHECK-NEXT:    mov v1.16b, v0.16b
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #32
-; CHECK-NEXT:    ret
-entry:
-  %c = uitofp <2 x i8> %a to <2 x fp128>
-  ret <2 x fp128> %c
-}
-
-define <3 x fp128> @stofp_v2i8_v3f128(<3 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v2i8_v3f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    sxtb w0, w0
-; CHECK-SD-NEXT:    mov w19, w2
-; CHECK-SD-NEXT:    mov w20, w1
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    sxtb w0, w20
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    sxtb w0, w19
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatsitf
-; CHECK-SD-NEXT:    mov v2.16b, v0.16b
-; CHECK-SD-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i8_v3f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    sxtb w0, w0
-; CHECK-GI-NEXT:    mov w19, w1
-; CHECK-GI-NEXT:    mov w20, w2
-; CHECK-GI-NEXT:    bl __floatsitf
-; CHECK-GI-NEXT:    sxtb w0, w19
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatsitf
-; CHECK-GI-NEXT:    sxtb w0, w20
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatsitf
-; CHECK-GI-NEXT:    mov v2.16b, v0.16b
-; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <3 x i8> %a to <3 x fp128>
-  ret <3 x fp128> %c
-}
-
-define <3 x fp128> @utofp_v2i8_v3f128(<3 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v2i8_v3f128:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #64
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    and w0, w0, #0xff
-; CHECK-SD-NEXT:    mov w19, w2
-; CHECK-SD-NEXT:    mov w20, w1
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    and w0, w20, #0xff
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    and w0, w19, #0xff
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatunsitf
-; CHECK-SD-NEXT:    mov v2.16b, v0.16b
-; CHECK-SD-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    add sp, sp, #64
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i8_v3f128:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #64
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    and w0, w0, #0xff
-; CHECK-GI-NEXT:    mov w19, w1
-; CHECK-GI-NEXT:    mov w20, w2
-; CHECK-GI-NEXT:    bl __floatunsitf
-; CHECK-GI-NEXT:    and w0, w19, #0xff
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatunsitf
-; CHECK-GI-NEXT:    and w0, w20, #0xff
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatunsitf
-; CHECK-GI-NEXT:    mov v2.16b, v0.16b
-; CHECK-GI-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    add sp, sp, #64
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <3 x i8> %a to <3 x fp128>
-  ret <3 x fp128> %c
-}
-
 define <2 x double> @stofp_v2i8_v2f64(<2 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v2i8_v2f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i8_v2f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #56
-; CHECK-GI-NEXT:    sshr v0.2d, v0.2d, #56
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v2i8_v2f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i8_v2f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i8_v2f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-NOFP16-GI-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i8_v2f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-FP16-GI-NEXT:    sshr v0.2d, v0.2d, #56
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <2 x i8> %a to <2 x double>
   ret <2 x double> %c
 }
 
 define <2 x double> @utofp_v2i8_v2f64(<2 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v2i8_v2f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
-; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i8_v2f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v1.2d, #0x000000000000ff
-; CHECK-GI-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v2i8_v2f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-NOFP16-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i8_v2f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-FP16-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i8_v2f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    movi v1.2d, #0x000000000000ff
+; CHECK-NOFP16-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i8_v2f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000000000ff
+; CHECK-FP16-GI-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <2 x i8> %a to <2 x double>
   ret <2 x double> %c
 }
 
 define <3 x double> @stofp_v3i8_v3f64(<3 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v3i8_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    fmov s1, w2
-; CHECK-SD-NEXT:    mov v0.s[1], w1
-; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #24
-; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #24
-; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    scvtf v2.2d, v1.2d
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i8_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov s0, w0
-; CHECK-GI-NEXT:    mov v0.h[1], w1
-; CHECK-GI-NEXT:    mov v0.h[2], w2
-; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-GI-NEXT:    smov x8, v0.h[0]
-; CHECK-GI-NEXT:    smov x9, v0.h[1]
-; CHECK-GI-NEXT:    fmov d1, x8
-; CHECK-GI-NEXT:    smov x8, v0.h[2]
-; CHECK-GI-NEXT:    mov v1.d[1], x9
-; CHECK-GI-NEXT:    smov x9, v0.h[3]
-; CHECK-GI-NEXT:    fmov d2, x8
-; CHECK-GI-NEXT:    scvtf v0.2d, v1.2d
-; CHECK-GI-NEXT:    mov v2.d[1], x9
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v3i8_v3f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    fmov s1, w2
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], w1
+; CHECK-NOFP16-SD-NEXT:    shl v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v3i8_v3f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    fmov s1, w2
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w1
+; CHECK-FP16-SD-NEXT:    shl v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i8_v3f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w0
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-GI-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-GI-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-GI-NEXT:    smov x8, v0.h[0]
+; CHECK-NOFP16-GI-NEXT:    smov x9, v0.h[1]
+; CHECK-NOFP16-GI-NEXT:    fmov d1, x8
+; CHECK-NOFP16-GI-NEXT:    smov x8, v0.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov v1.d[1], x9
+; CHECK-NOFP16-GI-NEXT:    smov x9, v0.h[3]
+; CHECK-NOFP16-GI-NEXT:    fmov d2, x8
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    mov v2.d[1], x9
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v3i8_v3f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov s0, w0
+; CHECK-FP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-FP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-FP16-GI-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-GI-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-GI-NEXT:    smov x8, v0.h[0]
+; CHECK-FP16-GI-NEXT:    smov x9, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fmov d1, x8
+; CHECK-FP16-GI-NEXT:    smov x8, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov v1.d[1], x9
+; CHECK-FP16-GI-NEXT:    smov x9, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fmov d2, x8
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], x9
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <3 x i8> %a to <3 x double>
   ret <3 x double> %c
 }
 
 define <3 x double> @utofp_v3i8_v3f64(<3 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v3i8_v3f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
-; CHECK-SD-NEXT:    fmov s2, w2
-; CHECK-SD-NEXT:    mov v0.s[1], w1
-; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    and v1.8b, v2.8b, v1.8b
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v1.2d
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i8_v3f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov s0, w0
-; CHECK-GI-NEXT:    movi d1, #0xff00ff00ff00ff
-; CHECK-GI-NEXT:    mov v0.h[1], w1
-; CHECK-GI-NEXT:    mov v0.h[2], w2
-; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    umov w8, v0.h[0]
-; CHECK-GI-NEXT:    umov w9, v0.h[1]
-; CHECK-GI-NEXT:    fmov d1, x8
-; CHECK-GI-NEXT:    umov w8, v0.h[2]
-; CHECK-GI-NEXT:    mov v1.d[1], x9
-; CHECK-GI-NEXT:    umov w9, v0.h[3]
-; CHECK-GI-NEXT:    fmov d2, x8
-; CHECK-GI-NEXT:    ucvtf v0.2d, v1.2d
-; CHECK-GI-NEXT:    mov v2.d[1], x9
-; CHECK-GI-NEXT:    mov d1, v0.d[1]
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v3i8_v3f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-NOFP16-SD-NEXT:    fmov s2, w2
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], w1
+; CHECK-NOFP16-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v1.8b, v2.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v3i8_v3f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-FP16-SD-NEXT:    fmov s2, w2
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], w1
+; CHECK-FP16-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v1.8b, v2.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i8_v3f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w0
+; CHECK-NOFP16-GI-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-GI-NEXT:    umov w8, v0.h[0]
+; CHECK-NOFP16-GI-NEXT:    umov w9, v0.h[1]
+; CHECK-NOFP16-GI-NEXT:    fmov d1, x8
+; CHECK-NOFP16-GI-NEXT:    umov w8, v0.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov v1.d[1], x9
+; CHECK-NOFP16-GI-NEXT:    umov w9, v0.h[3]
+; CHECK-NOFP16-GI-NEXT:    fmov d2, x8
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    mov v2.d[1], x9
+; CHECK-NOFP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v3i8_v3f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov s0, w0
+; CHECK-FP16-GI-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-FP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-FP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-FP16-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-GI-NEXT:    umov w8, v0.h[0]
+; CHECK-FP16-GI-NEXT:    umov w9, v0.h[1]
+; CHECK-FP16-GI-NEXT:    fmov d1, x8
+; CHECK-FP16-GI-NEXT:    umov w8, v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov v1.d[1], x9
+; CHECK-FP16-GI-NEXT:    umov w9, v0.h[3]
+; CHECK-FP16-GI-NEXT:    fmov d2, x8
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    mov v2.d[1], x9
+; CHECK-FP16-GI-NEXT:    mov d1, v0.d[1]
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-GI-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <3 x i8> %a to <3 x double>
   ret <3 x double> %c
 }
 
 define <4 x double> @stofp_v4i8_v4f64(<4 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v4i8_v4f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    shl v1.2s, v1.2s, #24
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #24
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v4i8_v4f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll v1.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    shl v1.2d, v1.2d, #56
-; CHECK-GI-NEXT:    shl v0.2d, v0.2d, #56
-; CHECK-GI-NEXT:    sshr v1.2d, v1.2d, #56
-; CHECK-GI-NEXT:    sshr v2.2d, v0.2d, #56
-; CHECK-GI-NEXT:    scvtf v0.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v4i8_v4f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v4i8_v4f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v4i8_v4f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    shl v1.2d, v1.2d, #56
+; CHECK-NOFP16-GI-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-NOFP16-GI-NEXT:    sshr v1.2d, v1.2d, #56
+; CHECK-NOFP16-GI-NEXT:    sshr v2.2d, v0.2d, #56
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v4i8_v4f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll v1.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    shl v1.2d, v1.2d, #56
+; CHECK-FP16-GI-NEXT:    shl v0.2d, v0.2d, #56
+; CHECK-FP16-GI-NEXT:    sshr v1.2d, v1.2d, #56
+; CHECK-FP16-GI-NEXT:    sshr v2.2d, v0.2d, #56
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <4 x i8> %a to <4 x double>
   ret <4 x double> %c
 }
 
 define <4 x double> @utofp_v4i8_v4f64(<4 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v4i8_v4f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
-; CHECK-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-SD-NEXT:    and v1.8b, v2.8b, v1.8b
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v4i8_v4f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT:    movi v1.2d, #0x000000000000ff
-; CHECK-GI-NEXT:    ushll v2.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    and v2.16b, v2.16b, v1.16b
-; CHECK-GI-NEXT:    and v1.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ucvtf v0.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v4i8_v4f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-NOFP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v1.8b, v2.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v4i8_v4f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-FP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v1.8b, v2.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v4i8_v4f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    movi v1.2d, #0x000000000000ff
+; CHECK-NOFP16-GI-NEXT:    ushll v2.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    and v2.16b, v2.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    and v1.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v4i8_v4f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x000000000000ff
+; CHECK-FP16-GI-NEXT:    ushll v2.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    and v2.16b, v2.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    and v1.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <4 x i8> %a to <4 x double>
   ret <4 x double> %c
 }
 
 define <8 x double> @stofp_v8i8_v8f64(<8 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v8i8_v8f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov b1, v0.b[0]
-; CHECK-SD-NEXT:    mov b2, v0.b[2]
-; CHECK-SD-NEXT:    mov b3, v0.b[4]
-; CHECK-SD-NEXT:    mov b4, v0.b[6]
-; CHECK-SD-NEXT:    mov v1.b[4], v0.b[1]
-; CHECK-SD-NEXT:    mov v2.b[4], v0.b[3]
-; CHECK-SD-NEXT:    mov v3.b[4], v0.b[5]
-; CHECK-SD-NEXT:    mov v4.b[4], v0.b[7]
-; CHECK-SD-NEXT:    shl v0.2s, v1.2s, #24
-; CHECK-SD-NEXT:    shl v1.2s, v2.2s, #24
-; CHECK-SD-NEXT:    shl v2.2s, v3.2s, #24
-; CHECK-SD-NEXT:    shl v3.2s, v4.2s, #24
-; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #24
-; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #24
-; CHECK-SD-NEXT:    sshr v3.2s, v3.2s, #24
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    sshll v3.2d, v3.2s, #0
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v8i8_v8f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    sshll v2.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    sshll v3.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v4.2d, v0.4s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v4.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i8_v8f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov b1, v0.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b2, v0.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b3, v0.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov b4, v0.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov v1.b[4], v0.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov v2.b[4], v0.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v3.b[4], v0.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v4.b[4], v0.b[7]
+; CHECK-NOFP16-SD-NEXT:    shl v0.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v1.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v2.2s, v3.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v3.2s, v4.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v3.2s, v3.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v3.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v8i8_v8f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov b1, v0.b[0]
+; CHECK-FP16-SD-NEXT:    mov b2, v0.b[2]
+; CHECK-FP16-SD-NEXT:    mov b3, v0.b[4]
+; CHECK-FP16-SD-NEXT:    mov b4, v0.b[6]
+; CHECK-FP16-SD-NEXT:    mov v1.b[4], v0.b[1]
+; CHECK-FP16-SD-NEXT:    mov v2.b[4], v0.b[3]
+; CHECK-FP16-SD-NEXT:    mov v3.b[4], v0.b[5]
+; CHECK-FP16-SD-NEXT:    mov v4.b[4], v0.b[7]
+; CHECK-FP16-SD-NEXT:    shl v0.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v1.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v2.2s, v3.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v3.2s, v4.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v3.2s, v3.2s, #24
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v3.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i8_v8f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v4.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i8_v8f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v2.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v3.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v4.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i8> %a to <8 x double>
   ret <8 x double> %c
 }
 
 define <8 x double> @utofp_v8i8_v8f64(<8 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v8i8_v8f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    mov b2, v0.b[0]
-; CHECK-SD-NEXT:    mov b3, v0.b[2]
-; CHECK-SD-NEXT:    mov b4, v0.b[4]
-; CHECK-SD-NEXT:    mov b5, v0.b[6]
-; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
-; CHECK-SD-NEXT:    mov v2.b[4], v0.b[1]
-; CHECK-SD-NEXT:    mov v3.b[4], v0.b[3]
-; CHECK-SD-NEXT:    mov v4.b[4], v0.b[5]
-; CHECK-SD-NEXT:    mov v5.b[4], v0.b[7]
-; CHECK-SD-NEXT:    and v0.8b, v2.8b, v1.8b
-; CHECK-SD-NEXT:    and v2.8b, v3.8b, v1.8b
-; CHECK-SD-NEXT:    and v3.8b, v4.8b, v1.8b
-; CHECK-SD-NEXT:    and v1.8b, v5.8b, v1.8b
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
-; CHECK-SD-NEXT:    ushll v4.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v3.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v4.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v8i8_v8f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-GI-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll v2.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ushll v3.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v4.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v4.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i8_v8f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov b2, v0.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b3, v0.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b4, v0.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov b5, v0.b[6]
+; CHECK-NOFP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-NOFP16-SD-NEXT:    mov v2.b[4], v0.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov v3.b[4], v0.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v4.b[4], v0.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v5.b[4], v0.b[7]
+; CHECK-NOFP16-SD-NEXT:    and v0.8b, v2.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v2.8b, v3.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v3.8b, v4.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v1.8b, v5.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v4.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v8i8_v8f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov b2, v0.b[0]
+; CHECK-FP16-SD-NEXT:    mov b3, v0.b[2]
+; CHECK-FP16-SD-NEXT:    mov b4, v0.b[4]
+; CHECK-FP16-SD-NEXT:    mov b5, v0.b[6]
+; CHECK-FP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-FP16-SD-NEXT:    mov v2.b[4], v0.b[1]
+; CHECK-FP16-SD-NEXT:    mov v3.b[4], v0.b[3]
+; CHECK-FP16-SD-NEXT:    mov v4.b[4], v0.b[5]
+; CHECK-FP16-SD-NEXT:    mov v5.b[4], v0.b[7]
+; CHECK-FP16-SD-NEXT:    and v0.8b, v2.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v2.8b, v3.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v3.8b, v4.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v1.8b, v5.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v4.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i8_v8f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v4.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i8_v8f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v2.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v3.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v4.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i8> %a to <8 x double>
   ret <8 x double> %c
 }
 
 define <16 x double> @stofp_v16i8_v16f64(<16 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v16i8_v16f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    mov b2, v0.b[0]
-; CHECK-SD-NEXT:    mov b3, v0.b[2]
-; CHECK-SD-NEXT:    mov b4, v0.b[4]
-; CHECK-SD-NEXT:    mov b5, v0.b[6]
-; CHECK-SD-NEXT:    mov b6, v1.b[0]
-; CHECK-SD-NEXT:    mov b7, v1.b[2]
-; CHECK-SD-NEXT:    mov b16, v1.b[4]
-; CHECK-SD-NEXT:    mov b17, v1.b[6]
-; CHECK-SD-NEXT:    mov v2.b[4], v0.b[1]
-; CHECK-SD-NEXT:    mov v3.b[4], v0.b[3]
-; CHECK-SD-NEXT:    mov v4.b[4], v0.b[5]
-; CHECK-SD-NEXT:    mov v5.b[4], v0.b[7]
-; CHECK-SD-NEXT:    mov v6.b[4], v1.b[1]
-; CHECK-SD-NEXT:    mov v7.b[4], v1.b[3]
-; CHECK-SD-NEXT:    mov v16.b[4], v1.b[5]
-; CHECK-SD-NEXT:    mov v17.b[4], v1.b[7]
-; CHECK-SD-NEXT:    shl v0.2s, v2.2s, #24
-; CHECK-SD-NEXT:    shl v1.2s, v3.2s, #24
-; CHECK-SD-NEXT:    shl v2.2s, v4.2s, #24
-; CHECK-SD-NEXT:    shl v3.2s, v5.2s, #24
-; CHECK-SD-NEXT:    shl v4.2s, v6.2s, #24
-; CHECK-SD-NEXT:    shl v5.2s, v7.2s, #24
-; CHECK-SD-NEXT:    shl v6.2s, v16.2s, #24
-; CHECK-SD-NEXT:    shl v7.2s, v17.2s, #24
-; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #24
-; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #24
-; CHECK-SD-NEXT:    sshr v3.2s, v3.2s, #24
-; CHECK-SD-NEXT:    sshr v4.2s, v4.2s, #24
-; CHECK-SD-NEXT:    sshr v5.2s, v5.2s, #24
-; CHECK-SD-NEXT:    sshr v6.2s, v6.2s, #24
-; CHECK-SD-NEXT:    sshr v7.2s, v7.2s, #24
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    sshll v3.2d, v3.2s, #0
-; CHECK-SD-NEXT:    sshll v4.2d, v4.2s, #0
-; CHECK-SD-NEXT:    sshll v5.2d, v5.2s, #0
-; CHECK-SD-NEXT:    sshll v6.2d, v6.2s, #0
-; CHECK-SD-NEXT:    sshll v7.2d, v7.2s, #0
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v16i8_v16f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v1.8h, v0.8b, #0
-; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT:    sshll v2.4s, v1.4h, #0
-; CHECK-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    sshll v3.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    sshll v4.2d, v2.2s, #0
-; CHECK-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    sshll v5.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v6.2d, v1.4s, #0
-; CHECK-GI-NEXT:    sshll v7.2d, v3.2s, #0
-; CHECK-GI-NEXT:    sshll2 v16.2d, v3.4s, #0
-; CHECK-GI-NEXT:    sshll v17.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v18.2d, v0.4s, #0
-; CHECK-GI-NEXT:    scvtf v0.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v16.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v17.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v18.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i8_v16f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov b2, v0.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b3, v0.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b4, v0.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov b5, v0.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov b6, v1.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b7, v1.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b16, v1.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov b17, v1.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov v2.b[4], v0.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov v3.b[4], v0.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v4.b[4], v0.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v5.b[4], v0.b[7]
+; CHECK-NOFP16-SD-NEXT:    mov v6.b[4], v1.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov v7.b[4], v1.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v16.b[4], v1.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v17.b[4], v1.b[7]
+; CHECK-NOFP16-SD-NEXT:    shl v0.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v1.2s, v3.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v2.2s, v4.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v3.2s, v5.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v4.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v5.2s, v7.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v6.2s, v16.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v7.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v3.2s, v3.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v4.2s, v4.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v5.2s, v5.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v6.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v7.2s, v7.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v3.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v5.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v7.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v16i8_v16f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    mov b2, v0.b[0]
+; CHECK-FP16-SD-NEXT:    mov b3, v0.b[2]
+; CHECK-FP16-SD-NEXT:    mov b4, v0.b[4]
+; CHECK-FP16-SD-NEXT:    mov b5, v0.b[6]
+; CHECK-FP16-SD-NEXT:    mov b6, v1.b[0]
+; CHECK-FP16-SD-NEXT:    mov b7, v1.b[2]
+; CHECK-FP16-SD-NEXT:    mov b16, v1.b[4]
+; CHECK-FP16-SD-NEXT:    mov b17, v1.b[6]
+; CHECK-FP16-SD-NEXT:    mov v2.b[4], v0.b[1]
+; CHECK-FP16-SD-NEXT:    mov v3.b[4], v0.b[3]
+; CHECK-FP16-SD-NEXT:    mov v4.b[4], v0.b[5]
+; CHECK-FP16-SD-NEXT:    mov v5.b[4], v0.b[7]
+; CHECK-FP16-SD-NEXT:    mov v6.b[4], v1.b[1]
+; CHECK-FP16-SD-NEXT:    mov v7.b[4], v1.b[3]
+; CHECK-FP16-SD-NEXT:    mov v16.b[4], v1.b[5]
+; CHECK-FP16-SD-NEXT:    mov v17.b[4], v1.b[7]
+; CHECK-FP16-SD-NEXT:    shl v0.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v1.2s, v3.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v2.2s, v4.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v3.2s, v5.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v4.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v5.2s, v7.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v6.2s, v16.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v7.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v3.2s, v3.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v4.2s, v4.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v5.2s, v5.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v6.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v7.2s, v7.2s, #24
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v3.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v5.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v7.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i8_v16f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v1.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v4.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v5.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v6.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v7.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v16.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v17.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v18.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v4.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v5.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v6.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v7.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i8_v16f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v1.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v3.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v4.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v5.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v6.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v7.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v16.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v17.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v18.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    scvtf v4.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    scvtf v5.2d, v16.2d
+; CHECK-FP16-GI-NEXT:    scvtf v6.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    scvtf v7.2d, v18.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i8> %a to <16 x double>
   ret <16 x double> %c
 }
 
 define <16 x double> @utofp_v16i8_v16f64(<16 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v16i8_v16f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    mov b3, v0.b[0]
-; CHECK-SD-NEXT:    mov b4, v0.b[2]
-; CHECK-SD-NEXT:    mov b5, v0.b[4]
-; CHECK-SD-NEXT:    mov b6, v0.b[6]
-; CHECK-SD-NEXT:    movi d1, #0x0000ff000000ff
-; CHECK-SD-NEXT:    mov b7, v2.b[0]
-; CHECK-SD-NEXT:    mov b16, v2.b[2]
-; CHECK-SD-NEXT:    mov b17, v2.b[4]
-; CHECK-SD-NEXT:    mov b18, v2.b[6]
-; CHECK-SD-NEXT:    mov v3.b[4], v0.b[1]
-; CHECK-SD-NEXT:    mov v4.b[4], v0.b[3]
-; CHECK-SD-NEXT:    mov v5.b[4], v0.b[5]
-; CHECK-SD-NEXT:    mov v6.b[4], v0.b[7]
-; CHECK-SD-NEXT:    mov v7.b[4], v2.b[1]
-; CHECK-SD-NEXT:    mov v16.b[4], v2.b[3]
-; CHECK-SD-NEXT:    mov v17.b[4], v2.b[5]
-; CHECK-SD-NEXT:    mov v18.b[4], v2.b[7]
-; CHECK-SD-NEXT:    and v0.8b, v3.8b, v1.8b
-; CHECK-SD-NEXT:    and v2.8b, v4.8b, v1.8b
-; CHECK-SD-NEXT:    and v3.8b, v5.8b, v1.8b
-; CHECK-SD-NEXT:    and v4.8b, v6.8b, v1.8b
-; CHECK-SD-NEXT:    and v5.8b, v7.8b, v1.8b
-; CHECK-SD-NEXT:    and v6.8b, v16.8b, v1.8b
-; CHECK-SD-NEXT:    and v7.8b, v17.8b, v1.8b
-; CHECK-SD-NEXT:    and v1.8b, v18.8b, v1.8b
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    ushll v3.2d, v3.2s, #0
-; CHECK-SD-NEXT:    ushll v4.2d, v4.2s, #0
-; CHECK-SD-NEXT:    ushll v5.2d, v5.2s, #0
-; CHECK-SD-NEXT:    ushll v6.2d, v6.2s, #0
-; CHECK-SD-NEXT:    ushll v7.2d, v7.2s, #0
-; CHECK-SD-NEXT:    ushll v16.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v3.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v4.2d
-; CHECK-SD-NEXT:    ucvtf v4.2d, v5.2d
-; CHECK-SD-NEXT:    ucvtf v5.2d, v6.2d
-; CHECK-SD-NEXT:    ucvtf v6.2d, v7.2d
-; CHECK-SD-NEXT:    ucvtf v7.2d, v16.2d
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v16i8_v16f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v1.8h, v0.8b, #0
-; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT:    ushll v2.4s, v1.4h, #0
-; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    ushll v3.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll v4.2d, v2.2s, #0
-; CHECK-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    ushll v5.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v6.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ushll v7.2d, v3.2s, #0
-; CHECK-GI-NEXT:    ushll2 v16.2d, v3.4s, #0
-; CHECK-GI-NEXT:    ushll v17.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v18.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ucvtf v0.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v16.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v17.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v18.2d
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i8_v16f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov b3, v0.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b4, v0.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b5, v0.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov b6, v0.b[6]
+; CHECK-NOFP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-NOFP16-SD-NEXT:    mov b7, v2.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b16, v2.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b17, v2.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov b18, v2.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov v3.b[4], v0.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov v4.b[4], v0.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v5.b[4], v0.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v6.b[4], v0.b[7]
+; CHECK-NOFP16-SD-NEXT:    mov v7.b[4], v2.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov v16.b[4], v2.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v17.b[4], v2.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v18.b[4], v2.b[7]
+; CHECK-NOFP16-SD-NEXT:    and v0.8b, v3.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v2.8b, v4.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v3.8b, v5.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v4.8b, v6.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v5.8b, v7.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v6.8b, v16.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v7.8b, v17.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    and v1.8b, v18.8b, v1.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v5.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v16.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v4.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v5.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v6.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v7.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v16i8_v16f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    mov b3, v0.b[0]
+; CHECK-FP16-SD-NEXT:    mov b4, v0.b[2]
+; CHECK-FP16-SD-NEXT:    mov b5, v0.b[4]
+; CHECK-FP16-SD-NEXT:    mov b6, v0.b[6]
+; CHECK-FP16-SD-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-FP16-SD-NEXT:    mov b7, v2.b[0]
+; CHECK-FP16-SD-NEXT:    mov b16, v2.b[2]
+; CHECK-FP16-SD-NEXT:    mov b17, v2.b[4]
+; CHECK-FP16-SD-NEXT:    mov b18, v2.b[6]
+; CHECK-FP16-SD-NEXT:    mov v3.b[4], v0.b[1]
+; CHECK-FP16-SD-NEXT:    mov v4.b[4], v0.b[3]
+; CHECK-FP16-SD-NEXT:    mov v5.b[4], v0.b[5]
+; CHECK-FP16-SD-NEXT:    mov v6.b[4], v0.b[7]
+; CHECK-FP16-SD-NEXT:    mov v7.b[4], v2.b[1]
+; CHECK-FP16-SD-NEXT:    mov v16.b[4], v2.b[3]
+; CHECK-FP16-SD-NEXT:    mov v17.b[4], v2.b[5]
+; CHECK-FP16-SD-NEXT:    mov v18.b[4], v2.b[7]
+; CHECK-FP16-SD-NEXT:    and v0.8b, v3.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v2.8b, v4.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v3.8b, v5.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v4.8b, v6.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v5.8b, v7.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v6.8b, v16.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v7.8b, v17.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    and v1.8b, v18.8b, v1.8b
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v3.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v5.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v16.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v4.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v5.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v6.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v7.2d, v16.2d
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i8_v16f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v4.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v5.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v6.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v7.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v16.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v17.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v18.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v4.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v5.2d, v16.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v6.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v7.2d, v18.2d
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i8_v16f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v3.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v4.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v5.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v6.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v7.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v16.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v17.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v18.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v4.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v5.2d, v16.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v6.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v7.2d, v18.2d
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i8> %a to <16 x double>
   ret <16 x double> %c
 }
 
 define <32 x double> @stofp_v32i8_v32f64(<32 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v32i8_v32f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    mov b5, v1.b[6]
-; CHECK-SD-NEXT:    mov b17, v1.b[4]
-; CHECK-SD-NEXT:    mov b20, v1.b[2]
-; CHECK-SD-NEXT:    mov b21, v1.b[0]
-; CHECK-SD-NEXT:    mov b18, v0.b[0]
-; CHECK-SD-NEXT:    mov b19, v0.b[6]
-; CHECK-SD-NEXT:    mov b22, v0.b[4]
-; CHECK-SD-NEXT:    ext v16.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    mov b2, v3.b[0]
-; CHECK-SD-NEXT:    mov b4, v3.b[2]
-; CHECK-SD-NEXT:    mov b6, v3.b[4]
-; CHECK-SD-NEXT:    mov b7, v3.b[6]
-; CHECK-SD-NEXT:    mov v5.b[4], v1.b[7]
-; CHECK-SD-NEXT:    mov v17.b[4], v1.b[5]
-; CHECK-SD-NEXT:    mov v20.b[4], v1.b[3]
-; CHECK-SD-NEXT:    mov v21.b[4], v1.b[1]
-; CHECK-SD-NEXT:    mov v19.b[4], v0.b[7]
-; CHECK-SD-NEXT:    mov v22.b[4], v0.b[5]
-; CHECK-SD-NEXT:    mov v18.b[4], v0.b[1]
-; CHECK-SD-NEXT:    mov b23, v16.b[0]
-; CHECK-SD-NEXT:    mov v2.b[4], v3.b[1]
-; CHECK-SD-NEXT:    mov v4.b[4], v3.b[3]
-; CHECK-SD-NEXT:    mov v6.b[4], v3.b[5]
-; CHECK-SD-NEXT:    mov v7.b[4], v3.b[7]
-; CHECK-SD-NEXT:    mov b3, v0.b[2]
-; CHECK-SD-NEXT:    shl v5.2s, v5.2s, #24
-; CHECK-SD-NEXT:    shl v17.2s, v17.2s, #24
-; CHECK-SD-NEXT:    shl v20.2s, v20.2s, #24
-; CHECK-SD-NEXT:    mov b24, v16.b[4]
-; CHECK-SD-NEXT:    mov v23.b[4], v16.b[1]
-; CHECK-SD-NEXT:    shl v18.2s, v18.2s, #24
-; CHECK-SD-NEXT:    shl v19.2s, v19.2s, #24
-; CHECK-SD-NEXT:    sshr v5.2s, v5.2s, #24
-; CHECK-SD-NEXT:    shl v1.2s, v2.2s, #24
-; CHECK-SD-NEXT:    shl v2.2s, v4.2s, #24
-; CHECK-SD-NEXT:    sshr v17.2s, v17.2s, #24
-; CHECK-SD-NEXT:    mov v3.b[4], v0.b[3]
-; CHECK-SD-NEXT:    shl v0.2s, v21.2s, #24
-; CHECK-SD-NEXT:    shl v4.2s, v6.2s, #24
-; CHECK-SD-NEXT:    shl v6.2s, v7.2s, #24
-; CHECK-SD-NEXT:    mov b7, v16.b[2]
-; CHECK-SD-NEXT:    sshll v5.2d, v5.2s, #0
-; CHECK-SD-NEXT:    sshr v20.2s, v20.2s, #24
-; CHECK-SD-NEXT:    mov b21, v16.b[6]
-; CHECK-SD-NEXT:    sshll v17.2d, v17.2s, #0
-; CHECK-SD-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-SD-NEXT:    shl v22.2s, v22.2s, #24
-; CHECK-SD-NEXT:    shl v3.2s, v3.2s, #24
-; CHECK-SD-NEXT:    mov v24.b[4], v16.b[5]
-; CHECK-SD-NEXT:    sshr v19.2s, v19.2s, #24
-; CHECK-SD-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    mov v7.b[4], v16.b[3]
-; CHECK-SD-NEXT:    sshll v20.2d, v20.2s, #0
-; CHECK-SD-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    mov v21.b[4], v16.b[7]
-; CHECK-SD-NEXT:    sshr v3.2s, v3.2s, #24
-; CHECK-SD-NEXT:    sshr v16.2s, v22.2s, #24
-; CHECK-SD-NEXT:    sshll v19.2d, v19.2s, #0
-; CHECK-SD-NEXT:    scvtf v20.2d, v20.2d
-; CHECK-SD-NEXT:    shl v22.2s, v24.2s, #24
-; CHECK-SD-NEXT:    sshr v6.2s, v6.2s, #24
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    shl v7.2s, v7.2s, #24
-; CHECK-SD-NEXT:    sshr v4.2s, v4.2s, #24
-; CHECK-SD-NEXT:    stp q17, q5, [x8, #160]
-; CHECK-SD-NEXT:    sshr v5.2s, v18.2s, #24
-; CHECK-SD-NEXT:    shl v17.2s, v23.2s, #24
-; CHECK-SD-NEXT:    sshll v3.2d, v3.2s, #0
-; CHECK-SD-NEXT:    sshll v16.2d, v16.2s, #0
-; CHECK-SD-NEXT:    shl v21.2s, v21.2s, #24
-; CHECK-SD-NEXT:    scvtf v19.2d, v19.2d
-; CHECK-SD-NEXT:    sshr v2.2s, v2.2s, #24
-; CHECK-SD-NEXT:    sshr v1.2s, v1.2s, #24
-; CHECK-SD-NEXT:    stp q0, q20, [x8, #128]
-; CHECK-SD-NEXT:    sshll v5.2d, v5.2s, #0
-; CHECK-SD-NEXT:    sshr v0.2s, v7.2s, #24
-; CHECK-SD-NEXT:    sshr v7.2s, v17.2s, #24
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    sshr v18.2s, v21.2s, #24
-; CHECK-SD-NEXT:    sshr v20.2s, v22.2s, #24
-; CHECK-SD-NEXT:    sshll v6.2d, v6.2s, #0
-; CHECK-SD-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    sshll v4.2d, v4.2s, #0
-; CHECK-SD-NEXT:    sshll v7.2d, v7.2s, #0
-; CHECK-SD-NEXT:    sshll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    stp q16, q19, [x8, #32]
-; CHECK-SD-NEXT:    sshll v17.2d, v18.2s, #0
-; CHECK-SD-NEXT:    sshll v16.2d, v20.2s, #0
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    stp q5, q3, [x8]
-; CHECK-SD-NEXT:    scvtf v3.2d, v7.2d
-; CHECK-SD-NEXT:    scvtf v5.2d, v6.2d
-; CHECK-SD-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-SD-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    stp q4, q5, [x8, #96]
-; CHECK-SD-NEXT:    stp q3, q0, [x8, #192]
-; CHECK-SD-NEXT:    scvtf v0.2d, v1.2d
-; CHECK-SD-NEXT:    stp q16, q17, [x8, #224]
-; CHECK-SD-NEXT:    stp q0, q2, [x8, #64]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v32i8_v32f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #0
-; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT:    sshll v4.8h, v1.8b, #0
-; CHECK-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-GI-NEXT:    sshll v3.4s, v2.4h, #0
-; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT:    sshll v6.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    sshll v16.4s, v4.4h, #0
-; CHECK-GI-NEXT:    sshll2 v4.4s, v4.8h, #0
-; CHECK-GI-NEXT:    sshll v18.4s, v1.4h, #0
-; CHECK-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    sshll v5.2d, v3.2s, #0
-; CHECK-GI-NEXT:    sshll2 v3.2d, v3.4s, #0
-; CHECK-GI-NEXT:    sshll v7.2d, v2.2s, #0
-; CHECK-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    sshll v17.2d, v6.2s, #0
-; CHECK-GI-NEXT:    sshll2 v6.2d, v6.4s, #0
-; CHECK-GI-NEXT:    sshll v19.2d, v0.2s, #0
-; CHECK-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    sshll v20.2d, v16.2s, #0
-; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    stp q5, q3, [x8]
-; CHECK-GI-NEXT:    sshll2 v5.2d, v16.4s, #0
-; CHECK-GI-NEXT:    scvtf v16.2d, v19.2d
-; CHECK-GI-NEXT:    stp q7, q2, [x8, #32]
-; CHECK-GI-NEXT:    scvtf v2.2d, v20.2d
-; CHECK-GI-NEXT:    sshll v3.2d, v4.2s, #0
-; CHECK-GI-NEXT:    stp q17, q6, [x8, #64]
-; CHECK-GI-NEXT:    sshll2 v4.2d, v4.4s, #0
-; CHECK-GI-NEXT:    sshll v6.2d, v18.2s, #0
-; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    sshll2 v7.2d, v18.4s, #0
-; CHECK-GI-NEXT:    stp q16, q0, [x8, #96]
-; CHECK-GI-NEXT:    sshll v0.2d, v1.2s, #0
-; CHECK-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    stp q2, q5, [x8, #128]
-; CHECK-GI-NEXT:    scvtf v2.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    stp q3, q4, [x8, #160]
-; CHECK-GI-NEXT:    stp q6, q2, [x8, #192]
-; CHECK-GI-NEXT:    stp q0, q1, [x8, #224]
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v32i8_v32f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov b5, v1.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov b17, v1.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov b20, v1.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b21, v1.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b18, v0.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b19, v0.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov b22, v0.b[4]
+; CHECK-NOFP16-SD-NEXT:    ext v16.16b, v1.16b, v1.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov b2, v3.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b4, v3.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b6, v3.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov b7, v3.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov v5.b[4], v1.b[7]
+; CHECK-NOFP16-SD-NEXT:    mov v17.b[4], v1.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v20.b[4], v1.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v21.b[4], v1.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov v19.b[4], v0.b[7]
+; CHECK-NOFP16-SD-NEXT:    mov v22.b[4], v0.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v18.b[4], v0.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov b23, v16.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov v2.b[4], v3.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov v4.b[4], v3.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v6.b[4], v3.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v7.b[4], v3.b[7]
+; CHECK-NOFP16-SD-NEXT:    mov b3, v0.b[2]
+; CHECK-NOFP16-SD-NEXT:    shl v5.2s, v5.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v17.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v20.2s, v20.2s, #24
+; CHECK-NOFP16-SD-NEXT:    mov b24, v16.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov v23.b[4], v16.b[1]
+; CHECK-NOFP16-SD-NEXT:    shl v18.2s, v18.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v19.2s, v19.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v5.2s, v5.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v1.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v2.2s, v4.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v17.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT:    mov v3.b[4], v0.b[3]
+; CHECK-NOFP16-SD-NEXT:    shl v0.2s, v21.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v4.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v6.2s, v7.2s, #24
+; CHECK-NOFP16-SD-NEXT:    mov b7, v16.b[2]
+; CHECK-NOFP16-SD-NEXT:    sshll v5.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshr v20.2s, v20.2s, #24
+; CHECK-NOFP16-SD-NEXT:    mov b21, v16.b[6]
+; CHECK-NOFP16-SD-NEXT:    sshll v17.2d, v17.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v22.2s, v22.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v3.2s, v3.2s, #24
+; CHECK-NOFP16-SD-NEXT:    mov v24.b[4], v16.b[5]
+; CHECK-NOFP16-SD-NEXT:    sshr v19.2s, v19.2s, #24
+; CHECK-NOFP16-SD-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    mov v7.b[4], v16.b[3]
+; CHECK-NOFP16-SD-NEXT:    sshll v20.2d, v20.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v17.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    mov v21.b[4], v16.b[7]
+; CHECK-NOFP16-SD-NEXT:    sshr v3.2s, v3.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v16.2s, v22.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshll v19.2d, v19.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v20.2d, v20.2d
+; CHECK-NOFP16-SD-NEXT:    shl v22.2s, v24.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v6.2s, v6.2s, #24
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    shl v7.2s, v7.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v4.2s, v4.2s, #24
+; CHECK-NOFP16-SD-NEXT:    stp q17, q5, [x8, #160]
+; CHECK-NOFP16-SD-NEXT:    sshr v5.2s, v18.2s, #24
+; CHECK-NOFP16-SD-NEXT:    shl v17.2s, v23.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshll v3.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v16.2d, v16.2s, #0
+; CHECK-NOFP16-SD-NEXT:    shl v21.2s, v21.2s, #24
+; CHECK-NOFP16-SD-NEXT:    scvtf v19.2d, v19.2d
+; CHECK-NOFP16-SD-NEXT:    sshr v2.2s, v2.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-NOFP16-SD-NEXT:    stp q0, q20, [x8, #128]
+; CHECK-NOFP16-SD-NEXT:    sshll v5.2d, v5.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshr v0.2s, v7.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v7.2s, v17.2s, #24
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT:    sshr v18.2s, v21.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshr v20.2s, v22.2s, #24
+; CHECK-NOFP16-SD-NEXT:    sshll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v7.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    stp q16, q19, [x8, #32]
+; CHECK-NOFP16-SD-NEXT:    sshll v17.2d, v18.2s, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v16.2d, v20.2s, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    stp q5, q3, [x8]
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v5.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v17.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    stp q4, q5, [x8, #96]
+; CHECK-NOFP16-SD-NEXT:    stp q3, q0, [x8, #192]
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    stp q16, q17, [x8, #224]
+; CHECK-NOFP16-SD-NEXT:    stp q0, q2, [x8, #64]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v32i8_v32f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    mov b5, v1.b[6]
+; CHECK-FP16-SD-NEXT:    mov b17, v1.b[4]
+; CHECK-FP16-SD-NEXT:    mov b20, v1.b[2]
+; CHECK-FP16-SD-NEXT:    mov b21, v1.b[0]
+; CHECK-FP16-SD-NEXT:    mov b18, v0.b[0]
+; CHECK-FP16-SD-NEXT:    mov b19, v0.b[6]
+; CHECK-FP16-SD-NEXT:    mov b22, v0.b[4]
+; CHECK-FP16-SD-NEXT:    ext v16.16b, v1.16b, v1.16b, #8
+; CHECK-FP16-SD-NEXT:    mov b2, v3.b[0]
+; CHECK-FP16-SD-NEXT:    mov b4, v3.b[2]
+; CHECK-FP16-SD-NEXT:    mov b6, v3.b[4]
+; CHECK-FP16-SD-NEXT:    mov b7, v3.b[6]
+; CHECK-FP16-SD-NEXT:    mov v5.b[4], v1.b[7]
+; CHECK-FP16-SD-NEXT:    mov v17.b[4], v1.b[5]
+; CHECK-FP16-SD-NEXT:    mov v20.b[4], v1.b[3]
+; CHECK-FP16-SD-NEXT:    mov v21.b[4], v1.b[1]
+; CHECK-FP16-SD-NEXT:    mov v19.b[4], v0.b[7]
+; CHECK-FP16-SD-NEXT:    mov v22.b[4], v0.b[5]
+; CHECK-FP16-SD-NEXT:    mov v18.b[4], v0.b[1]
+; CHECK-FP16-SD-NEXT:    mov b23, v16.b[0]
+; CHECK-FP16-SD-NEXT:    mov v2.b[4], v3.b[1]
+; CHECK-FP16-SD-NEXT:    mov v4.b[4], v3.b[3]
+; CHECK-FP16-SD-NEXT:    mov v6.b[4], v3.b[5]
+; CHECK-FP16-SD-NEXT:    mov v7.b[4], v3.b[7]
+; CHECK-FP16-SD-NEXT:    mov b3, v0.b[2]
+; CHECK-FP16-SD-NEXT:    shl v5.2s, v5.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v17.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v20.2s, v20.2s, #24
+; CHECK-FP16-SD-NEXT:    mov b24, v16.b[4]
+; CHECK-FP16-SD-NEXT:    mov v23.b[4], v16.b[1]
+; CHECK-FP16-SD-NEXT:    shl v18.2s, v18.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v19.2s, v19.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v5.2s, v5.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v1.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v2.2s, v4.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v17.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT:    mov v3.b[4], v0.b[3]
+; CHECK-FP16-SD-NEXT:    shl v0.2s, v21.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v4.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v6.2s, v7.2s, #24
+; CHECK-FP16-SD-NEXT:    mov b7, v16.b[2]
+; CHECK-FP16-SD-NEXT:    sshll v5.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT:    sshr v20.2s, v20.2s, #24
+; CHECK-FP16-SD-NEXT:    mov b21, v16.b[6]
+; CHECK-FP16-SD-NEXT:    sshll v17.2d, v17.2s, #0
+; CHECK-FP16-SD-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v22.2s, v22.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v3.2s, v3.2s, #24
+; CHECK-FP16-SD-NEXT:    mov v24.b[4], v16.b[5]
+; CHECK-FP16-SD-NEXT:    sshr v19.2s, v19.2s, #24
+; CHECK-FP16-SD-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    mov v7.b[4], v16.b[3]
+; CHECK-FP16-SD-NEXT:    sshll v20.2d, v20.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v17.2d, v17.2d
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    mov v21.b[4], v16.b[7]
+; CHECK-FP16-SD-NEXT:    sshr v3.2s, v3.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v16.2s, v22.2s, #24
+; CHECK-FP16-SD-NEXT:    sshll v19.2d, v19.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v20.2d, v20.2d
+; CHECK-FP16-SD-NEXT:    shl v22.2s, v24.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v6.2s, v6.2s, #24
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    shl v7.2s, v7.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v4.2s, v4.2s, #24
+; CHECK-FP16-SD-NEXT:    stp q17, q5, [x8, #160]
+; CHECK-FP16-SD-NEXT:    sshr v5.2s, v18.2s, #24
+; CHECK-FP16-SD-NEXT:    shl v17.2s, v23.2s, #24
+; CHECK-FP16-SD-NEXT:    sshll v3.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v16.2d, v16.2s, #0
+; CHECK-FP16-SD-NEXT:    shl v21.2s, v21.2s, #24
+; CHECK-FP16-SD-NEXT:    scvtf v19.2d, v19.2d
+; CHECK-FP16-SD-NEXT:    sshr v2.2s, v2.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v1.2s, v1.2s, #24
+; CHECK-FP16-SD-NEXT:    stp q0, q20, [x8, #128]
+; CHECK-FP16-SD-NEXT:    sshll v5.2d, v5.2s, #0
+; CHECK-FP16-SD-NEXT:    sshr v0.2s, v7.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v7.2s, v17.2s, #24
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    scvtf v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT:    sshr v18.2s, v21.2s, #24
+; CHECK-FP16-SD-NEXT:    sshr v20.2s, v22.2s, #24
+; CHECK-FP16-SD-NEXT:    sshll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    sshll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v7.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    stp q16, q19, [x8, #32]
+; CHECK-FP16-SD-NEXT:    sshll v17.2d, v18.2s, #0
+; CHECK-FP16-SD-NEXT:    sshll v16.2d, v20.2s, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    stp q5, q3, [x8]
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    scvtf v5.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    scvtf v17.2d, v17.2d
+; CHECK-FP16-SD-NEXT:    scvtf v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    stp q4, q5, [x8, #96]
+; CHECK-FP16-SD-NEXT:    stp q3, q0, [x8, #192]
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    stp q16, q17, [x8, #224]
+; CHECK-FP16-SD-NEXT:    stp q0, q2, [x8, #64]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v32i8_v32f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v4.8h, v1.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v6.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v16.4s, v4.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v4.4s, v4.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v18.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v5.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v7.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v17.2d, v6.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v6.2d, v6.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v19.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v20.2d, v16.2s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v17.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    stp q5, q3, [x8]
+; CHECK-NOFP16-GI-NEXT:    sshll2 v5.2d, v16.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v16.2d, v19.2d
+; CHECK-NOFP16-GI-NEXT:    stp q7, q2, [x8, #32]
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v20.2d
+; CHECK-NOFP16-GI-NEXT:    sshll v3.2d, v4.2s, #0
+; CHECK-NOFP16-GI-NEXT:    stp q17, q6, [x8, #64]
+; CHECK-NOFP16-GI-NEXT:    sshll2 v4.2d, v4.4s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v6.2d, v18.2s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    sshll2 v7.2d, v18.4s, #0
+; CHECK-NOFP16-GI-NEXT:    stp q16, q0, [x8, #96]
+; CHECK-NOFP16-GI-NEXT:    sshll v0.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    stp q2, q5, [x8, #128]
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    stp q3, q4, [x8, #160]
+; CHECK-NOFP16-GI-NEXT:    stp q6, q2, [x8, #192]
+; CHECK-NOFP16-GI-NEXT:    stp q0, q1, [x8, #224]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v32i8_v32f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    sshll v4.8h, v1.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
+; CHECK-FP16-GI-NEXT:    sshll v3.4s, v2.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v6.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v16.4s, v4.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v4.4s, v4.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v18.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v5.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v3.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v7.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v17.2d, v6.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v6.2d, v6.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v19.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v20.2d, v16.2s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    scvtf v17.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    stp q5, q3, [x8]
+; CHECK-FP16-GI-NEXT:    sshll2 v5.2d, v16.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v16.2d, v19.2d
+; CHECK-FP16-GI-NEXT:    stp q7, q2, [x8, #32]
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v20.2d
+; CHECK-FP16-GI-NEXT:    sshll v3.2d, v4.2s, #0
+; CHECK-FP16-GI-NEXT:    stp q17, q6, [x8, #64]
+; CHECK-FP16-GI-NEXT:    sshll2 v4.2d, v4.4s, #0
+; CHECK-FP16-GI-NEXT:    sshll v6.2d, v18.2s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    sshll2 v7.2d, v18.4s, #0
+; CHECK-FP16-GI-NEXT:    stp q16, q0, [x8, #96]
+; CHECK-FP16-GI-NEXT:    sshll v0.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    stp q2, q5, [x8, #128]
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    stp q3, q4, [x8, #160]
+; CHECK-FP16-GI-NEXT:    stp q6, q2, [x8, #192]
+; CHECK-FP16-GI-NEXT:    stp q0, q1, [x8, #224]
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <32 x i8> %a to <32 x double>
   ret <32 x double> %c
 }
 
 define <32 x double> @utofp_v32i8_v32f64(<32 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v32i8_v32f64:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov b6, v1.b[6]
-; CHECK-SD-NEXT:    mov b7, v1.b[4]
-; CHECK-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    mov b16, v1.b[2]
-; CHECK-SD-NEXT:    mov b17, v1.b[0]
-; CHECK-SD-NEXT:    mov b19, v0.b[6]
-; CHECK-SD-NEXT:    mov b20, v0.b[4]
-; CHECK-SD-NEXT:    movi d5, #0x0000ff000000ff
-; CHECK-SD-NEXT:    mov b24, v0.b[2]
-; CHECK-SD-NEXT:    mov b25, v0.b[0]
-; CHECK-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    mov v6.b[4], v1.b[7]
-; CHECK-SD-NEXT:    mov v7.b[4], v1.b[5]
-; CHECK-SD-NEXT:    mov b18, v3.b[0]
-; CHECK-SD-NEXT:    mov b21, v3.b[2]
-; CHECK-SD-NEXT:    mov b23, v3.b[4]
-; CHECK-SD-NEXT:    mov v16.b[4], v1.b[3]
-; CHECK-SD-NEXT:    mov v17.b[4], v1.b[1]
-; CHECK-SD-NEXT:    mov b1, v3.b[6]
-; CHECK-SD-NEXT:    mov v19.b[4], v0.b[7]
-; CHECK-SD-NEXT:    mov v20.b[4], v0.b[5]
-; CHECK-SD-NEXT:    mov v24.b[4], v0.b[3]
-; CHECK-SD-NEXT:    mov v25.b[4], v0.b[1]
-; CHECK-SD-NEXT:    and v6.8b, v6.8b, v5.8b
-; CHECK-SD-NEXT:    and v7.8b, v7.8b, v5.8b
-; CHECK-SD-NEXT:    mov v18.b[4], v3.b[1]
-; CHECK-SD-NEXT:    mov v21.b[4], v3.b[3]
-; CHECK-SD-NEXT:    mov v23.b[4], v3.b[5]
-; CHECK-SD-NEXT:    and v16.8b, v16.8b, v5.8b
-; CHECK-SD-NEXT:    and v17.8b, v17.8b, v5.8b
-; CHECK-SD-NEXT:    mov v1.b[4], v3.b[7]
-; CHECK-SD-NEXT:    and v3.8b, v19.8b, v5.8b
-; CHECK-SD-NEXT:    ushll v6.2d, v6.2s, #0
-; CHECK-SD-NEXT:    ushll v7.2d, v7.2s, #0
-; CHECK-SD-NEXT:    and v20.8b, v20.8b, v5.8b
-; CHECK-SD-NEXT:    ushll v16.2d, v16.2s, #0
-; CHECK-SD-NEXT:    mov b4, v2.b[0]
-; CHECK-SD-NEXT:    mov b22, v2.b[2]
-; CHECK-SD-NEXT:    ushll v17.2d, v17.2s, #0
-; CHECK-SD-NEXT:    ushll v0.2d, v3.2s, #0
-; CHECK-SD-NEXT:    mov b19, v2.b[4]
-; CHECK-SD-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v7.2d
-; CHECK-SD-NEXT:    ushll v20.2d, v20.2s, #0
-; CHECK-SD-NEXT:    mov b7, v2.b[6]
-; CHECK-SD-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    and v24.8b, v24.8b, v5.8b
-; CHECK-SD-NEXT:    ucvtf v17.2d, v17.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    mov v4.b[4], v2.b[1]
-; CHECK-SD-NEXT:    ucvtf v20.2d, v20.2d
-; CHECK-SD-NEXT:    mov v22.b[4], v2.b[3]
-; CHECK-SD-NEXT:    mov v19.b[4], v2.b[5]
-; CHECK-SD-NEXT:    stp q3, q6, [x8, #160]
-; CHECK-SD-NEXT:    and v6.8b, v25.8b, v5.8b
-; CHECK-SD-NEXT:    and v1.8b, v1.8b, v5.8b
-; CHECK-SD-NEXT:    mov v7.b[4], v2.b[7]
-; CHECK-SD-NEXT:    and v2.8b, v23.8b, v5.8b
-; CHECK-SD-NEXT:    ushll v3.2d, v24.2s, #0
-; CHECK-SD-NEXT:    stp q17, q16, [x8, #128]
-; CHECK-SD-NEXT:    and v16.8b, v21.8b, v5.8b
-; CHECK-SD-NEXT:    and v4.8b, v4.8b, v5.8b
-; CHECK-SD-NEXT:    stp q20, q0, [x8, #32]
-; CHECK-SD-NEXT:    ushll v6.2d, v6.2s, #0
-; CHECK-SD-NEXT:    and v0.8b, v18.8b, v5.8b
-; CHECK-SD-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-SD-NEXT:    ushll v2.2d, v2.2s, #0
-; CHECK-SD-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    ushll v16.2d, v16.2s, #0
-; CHECK-SD-NEXT:    and v7.8b, v7.8b, v5.8b
-; CHECK-SD-NEXT:    and v17.8b, v19.8b, v5.8b
-; CHECK-SD-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-SD-NEXT:    and v18.8b, v22.8b, v5.8b
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    ushll v4.2d, v4.2s, #0
-; CHECK-SD-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    ushll v5.2d, v7.2s, #0
-; CHECK-SD-NEXT:    ushll v7.2d, v17.2s, #0
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    stp q6, q3, [x8]
-; CHECK-SD-NEXT:    ushll v3.2d, v18.2s, #0
-; CHECK-SD-NEXT:    stp q2, q1, [x8, #224]
-; CHECK-SD-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v7.2d
-; CHECK-SD-NEXT:    stp q0, q16, [x8, #192]
-; CHECK-SD-NEXT:    ucvtf v2.2d, v3.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v4.2d
-; CHECK-SD-NEXT:    stp q1, q5, [x8, #96]
-; CHECK-SD-NEXT:    stp q0, q2, [x8, #64]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v32i8_v32f64:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v2.8h, v0.8b, #0
-; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT:    ushll v4.8h, v1.8b, #0
-; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-NEXT:    ushll v3.4s, v2.4h, #0
-; CHECK-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT:    ushll v6.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll v16.4s, v4.4h, #0
-; CHECK-GI-NEXT:    ushll2 v4.4s, v4.8h, #0
-; CHECK-GI-NEXT:    ushll v18.4s, v1.4h, #0
-; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    ushll v5.2d, v3.2s, #0
-; CHECK-GI-NEXT:    ushll2 v3.2d, v3.4s, #0
-; CHECK-GI-NEXT:    ushll v7.2d, v2.2s, #0
-; CHECK-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
-; CHECK-GI-NEXT:    ushll v17.2d, v6.2s, #0
-; CHECK-GI-NEXT:    ushll2 v6.2d, v6.4s, #0
-; CHECK-GI-NEXT:    ushll v19.2d, v0.2s, #0
-; CHECK-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
-; CHECK-GI-NEXT:    ushll v20.2d, v16.2s, #0
-; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v17.2d, v17.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    stp q5, q3, [x8]
-; CHECK-GI-NEXT:    ushll2 v5.2d, v16.4s, #0
-; CHECK-GI-NEXT:    ucvtf v16.2d, v19.2d
-; CHECK-GI-NEXT:    stp q7, q2, [x8, #32]
-; CHECK-GI-NEXT:    ucvtf v2.2d, v20.2d
-; CHECK-GI-NEXT:    ushll v3.2d, v4.2s, #0
-; CHECK-GI-NEXT:    stp q17, q6, [x8, #64]
-; CHECK-GI-NEXT:    ushll2 v4.2d, v4.4s, #0
-; CHECK-GI-NEXT:    ushll v6.2d, v18.2s, #0
-; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    ushll2 v7.2d, v18.4s, #0
-; CHECK-GI-NEXT:    stp q16, q0, [x8, #96]
-; CHECK-GI-NEXT:    ushll v0.2d, v1.2s, #0
-; CHECK-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    stp q2, q5, [x8, #128]
-; CHECK-GI-NEXT:    ucvtf v2.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    stp q3, q4, [x8, #160]
-; CHECK-GI-NEXT:    stp q6, q2, [x8, #192]
-; CHECK-GI-NEXT:    stp q0, q1, [x8, #224]
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v32i8_v32f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov b6, v1.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov b7, v1.b[4]
+; CHECK-NOFP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov b16, v1.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b17, v1.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b19, v0.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov b20, v0.b[4]
+; CHECK-NOFP16-SD-NEXT:    movi d5, #0x0000ff000000ff
+; CHECK-NOFP16-SD-NEXT:    mov b24, v0.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b25, v0.b[0]
+; CHECK-NOFP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    mov v6.b[4], v1.b[7]
+; CHECK-NOFP16-SD-NEXT:    mov v7.b[4], v1.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov b18, v3.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b21, v3.b[2]
+; CHECK-NOFP16-SD-NEXT:    mov b23, v3.b[4]
+; CHECK-NOFP16-SD-NEXT:    mov v16.b[4], v1.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v17.b[4], v1.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov b1, v3.b[6]
+; CHECK-NOFP16-SD-NEXT:    mov v19.b[4], v0.b[7]
+; CHECK-NOFP16-SD-NEXT:    mov v20.b[4], v0.b[5]
+; CHECK-NOFP16-SD-NEXT:    mov v24.b[4], v0.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v25.b[4], v0.b[1]
+; CHECK-NOFP16-SD-NEXT:    and v6.8b, v6.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    and v7.8b, v7.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    mov v18.b[4], v3.b[1]
+; CHECK-NOFP16-SD-NEXT:    mov v21.b[4], v3.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v23.b[4], v3.b[5]
+; CHECK-NOFP16-SD-NEXT:    and v16.8b, v16.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    and v17.8b, v17.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    mov v1.b[4], v3.b[7]
+; CHECK-NOFP16-SD-NEXT:    and v3.8b, v19.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT:    and v20.8b, v20.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v16.2d, v16.2s, #0
+; CHECK-NOFP16-SD-NEXT:    mov b4, v2.b[0]
+; CHECK-NOFP16-SD-NEXT:    mov b22, v2.b[2]
+; CHECK-NOFP16-SD-NEXT:    ushll v17.2d, v17.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v3.2s, #0
+; CHECK-NOFP16-SD-NEXT:    mov b19, v2.b[4]
+; CHECK-NOFP16-SD-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    ushll v20.2d, v20.2s, #0
+; CHECK-NOFP16-SD-NEXT:    mov b7, v2.b[6]
+; CHECK-NOFP16-SD-NEXT:    ucvtf v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT:    and v24.8b, v24.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    ucvtf v17.2d, v17.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    mov v4.b[4], v2.b[1]
+; CHECK-NOFP16-SD-NEXT:    ucvtf v20.2d, v20.2d
+; CHECK-NOFP16-SD-NEXT:    mov v22.b[4], v2.b[3]
+; CHECK-NOFP16-SD-NEXT:    mov v19.b[4], v2.b[5]
+; CHECK-NOFP16-SD-NEXT:    stp q3, q6, [x8, #160]
+; CHECK-NOFP16-SD-NEXT:    and v6.8b, v25.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    and v1.8b, v1.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    mov v7.b[4], v2.b[7]
+; CHECK-NOFP16-SD-NEXT:    and v2.8b, v23.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v3.2d, v24.2s, #0
+; CHECK-NOFP16-SD-NEXT:    stp q17, q16, [x8, #128]
+; CHECK-NOFP16-SD-NEXT:    and v16.8b, v21.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    and v4.8b, v4.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    stp q20, q0, [x8, #32]
+; CHECK-NOFP16-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-NOFP16-SD-NEXT:    and v0.8b, v18.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    ushll v16.2d, v16.2s, #0
+; CHECK-NOFP16-SD-NEXT:    and v7.8b, v7.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    and v17.8b, v19.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-NOFP16-SD-NEXT:    and v18.8b, v22.8b, v5.8b
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v16.2d, v16.2d
+; CHECK-NOFP16-SD-NEXT:    ushll v5.2d, v7.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v7.2d, v17.2s, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    stp q6, q3, [x8]
+; CHECK-NOFP16-SD-NEXT:    ushll v3.2d, v18.2s, #0
+; CHECK-NOFP16-SD-NEXT:    stp q2, q1, [x8, #224]
+; CHECK-NOFP16-SD-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    stp q0, q16, [x8, #192]
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    stp q1, q5, [x8, #96]
+; CHECK-NOFP16-SD-NEXT:    stp q0, q2, [x8, #64]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v32i8_v32f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    mov b6, v1.b[6]
+; CHECK-FP16-SD-NEXT:    mov b7, v1.b[4]
+; CHECK-FP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-FP16-SD-NEXT:    mov b16, v1.b[2]
+; CHECK-FP16-SD-NEXT:    mov b17, v1.b[0]
+; CHECK-FP16-SD-NEXT:    mov b19, v0.b[6]
+; CHECK-FP16-SD-NEXT:    mov b20, v0.b[4]
+; CHECK-FP16-SD-NEXT:    movi d5, #0x0000ff000000ff
+; CHECK-FP16-SD-NEXT:    mov b24, v0.b[2]
+; CHECK-FP16-SD-NEXT:    mov b25, v0.b[0]
+; CHECK-FP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    mov v6.b[4], v1.b[7]
+; CHECK-FP16-SD-NEXT:    mov v7.b[4], v1.b[5]
+; CHECK-FP16-SD-NEXT:    mov b18, v3.b[0]
+; CHECK-FP16-SD-NEXT:    mov b21, v3.b[2]
+; CHECK-FP16-SD-NEXT:    mov b23, v3.b[4]
+; CHECK-FP16-SD-NEXT:    mov v16.b[4], v1.b[3]
+; CHECK-FP16-SD-NEXT:    mov v17.b[4], v1.b[1]
+; CHECK-FP16-SD-NEXT:    mov b1, v3.b[6]
+; CHECK-FP16-SD-NEXT:    mov v19.b[4], v0.b[7]
+; CHECK-FP16-SD-NEXT:    mov v20.b[4], v0.b[5]
+; CHECK-FP16-SD-NEXT:    mov v24.b[4], v0.b[3]
+; CHECK-FP16-SD-NEXT:    mov v25.b[4], v0.b[1]
+; CHECK-FP16-SD-NEXT:    and v6.8b, v6.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    and v7.8b, v7.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    mov v18.b[4], v3.b[1]
+; CHECK-FP16-SD-NEXT:    mov v21.b[4], v3.b[3]
+; CHECK-FP16-SD-NEXT:    mov v23.b[4], v3.b[5]
+; CHECK-FP16-SD-NEXT:    and v16.8b, v16.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    and v17.8b, v17.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    mov v1.b[4], v3.b[7]
+; CHECK-FP16-SD-NEXT:    and v3.8b, v19.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v7.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT:    and v20.8b, v20.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    ushll v16.2d, v16.2s, #0
+; CHECK-FP16-SD-NEXT:    mov b4, v2.b[0]
+; CHECK-FP16-SD-NEXT:    mov b22, v2.b[2]
+; CHECK-FP16-SD-NEXT:    ushll v17.2d, v17.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v3.2s, #0
+; CHECK-FP16-SD-NEXT:    mov b19, v2.b[4]
+; CHECK-FP16-SD-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    ushll v20.2d, v20.2s, #0
+; CHECK-FP16-SD-NEXT:    mov b7, v2.b[6]
+; CHECK-FP16-SD-NEXT:    ucvtf v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT:    and v24.8b, v24.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    ucvtf v17.2d, v17.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    mov v4.b[4], v2.b[1]
+; CHECK-FP16-SD-NEXT:    ucvtf v20.2d, v20.2d
+; CHECK-FP16-SD-NEXT:    mov v22.b[4], v2.b[3]
+; CHECK-FP16-SD-NEXT:    mov v19.b[4], v2.b[5]
+; CHECK-FP16-SD-NEXT:    stp q3, q6, [x8, #160]
+; CHECK-FP16-SD-NEXT:    and v6.8b, v25.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    and v1.8b, v1.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    mov v7.b[4], v2.b[7]
+; CHECK-FP16-SD-NEXT:    and v2.8b, v23.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    ushll v3.2d, v24.2s, #0
+; CHECK-FP16-SD-NEXT:    stp q17, q16, [x8, #128]
+; CHECK-FP16-SD-NEXT:    and v16.8b, v21.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    and v4.8b, v4.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    stp q20, q0, [x8, #32]
+; CHECK-FP16-SD-NEXT:    ushll v6.2d, v6.2s, #0
+; CHECK-FP16-SD-NEXT:    and v0.8b, v18.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    ushll v1.2d, v1.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v2.2d, v2.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    ushll v16.2d, v16.2s, #0
+; CHECK-FP16-SD-NEXT:    and v7.8b, v7.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    and v17.8b, v19.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    ushll v0.2d, v0.2s, #0
+; CHECK-FP16-SD-NEXT:    and v18.8b, v22.8b, v5.8b
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    ushll v4.2d, v4.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v16.2d, v16.2d
+; CHECK-FP16-SD-NEXT:    ushll v5.2d, v7.2s, #0
+; CHECK-FP16-SD-NEXT:    ushll v7.2d, v17.2s, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    stp q6, q3, [x8]
+; CHECK-FP16-SD-NEXT:    ushll v3.2d, v18.2s, #0
+; CHECK-FP16-SD-NEXT:    stp q2, q1, [x8, #224]
+; CHECK-FP16-SD-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    stp q0, q16, [x8, #192]
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    stp q1, q5, [x8, #96]
+; CHECK-FP16-SD-NEXT:    stp q0, q2, [x8, #64]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v32i8_v32f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v4.8h, v1.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v6.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v16.4s, v4.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v4.4s, v4.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v18.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v5.2d, v3.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v3.2d, v3.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v7.2d, v2.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v17.2d, v6.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v6.2d, v6.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v19.2d, v0.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v20.2d, v16.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v17.2d, v17.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    stp q5, q3, [x8]
+; CHECK-NOFP16-GI-NEXT:    ushll2 v5.2d, v16.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v16.2d, v19.2d
+; CHECK-NOFP16-GI-NEXT:    stp q7, q2, [x8, #32]
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v20.2d
+; CHECK-NOFP16-GI-NEXT:    ushll v3.2d, v4.2s, #0
+; CHECK-NOFP16-GI-NEXT:    stp q17, q6, [x8, #64]
+; CHECK-NOFP16-GI-NEXT:    ushll2 v4.2d, v4.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v6.2d, v18.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    ushll2 v7.2d, v18.4s, #0
+; CHECK-NOFP16-GI-NEXT:    stp q16, q0, [x8, #96]
+; CHECK-NOFP16-GI-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    stp q2, q5, [x8, #128]
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    stp q3, q4, [x8, #160]
+; CHECK-NOFP16-GI-NEXT:    stp q6, q2, [x8, #192]
+; CHECK-NOFP16-GI-NEXT:    stp q0, q1, [x8, #224]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v32i8_v32f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    ushll v4.8h, v1.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-FP16-GI-NEXT:    ushll v3.4s, v2.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v6.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v16.4s, v4.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v4.4s, v4.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v18.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v5.2d, v3.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v3.2d, v3.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v7.2d, v2.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.2d, v2.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v17.2d, v6.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v6.2d, v6.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v19.2d, v0.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.2d, v0.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v20.2d, v16.2s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v17.2d, v17.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    stp q5, q3, [x8]
+; CHECK-FP16-GI-NEXT:    ushll2 v5.2d, v16.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v16.2d, v19.2d
+; CHECK-FP16-GI-NEXT:    stp q7, q2, [x8, #32]
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v20.2d
+; CHECK-FP16-GI-NEXT:    ushll v3.2d, v4.2s, #0
+; CHECK-FP16-GI-NEXT:    stp q17, q6, [x8, #64]
+; CHECK-FP16-GI-NEXT:    ushll2 v4.2d, v4.4s, #0
+; CHECK-FP16-GI-NEXT:    ushll v6.2d, v18.2s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    ushll2 v7.2d, v18.4s, #0
+; CHECK-FP16-GI-NEXT:    stp q16, q0, [x8, #96]
+; CHECK-FP16-GI-NEXT:    ushll v0.2d, v1.2s, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.2d, v1.4s, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    stp q2, q5, [x8, #128]
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    stp q3, q4, [x8, #160]
+; CHECK-FP16-GI-NEXT:    stp q6, q2, [x8, #192]
+; CHECK-FP16-GI-NEXT:    stp q0, q1, [x8, #224]
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <32 x i8> %a to <32 x double>
   ret <32 x double> %c
 }
 
-define <2 x float> @stofp_v2i128_v2f32(<2 x i128> %a) {
-; CHECK-SD-LABEL: stofp_v2i128_v2f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    mov x19, x1
-; CHECK-SD-NEXT:    mov x20, x0
-; CHECK-SD-NEXT:    mov x0, x2
-; CHECK-SD-NEXT:    mov x1, x3
-; CHECK-SD-NEXT:    bl __floattisf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floattisf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i128_v2f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #48
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    bl __floattisf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floattisf
-; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v1.s[1], v0.s[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    add sp, sp, #48
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <2 x i128> %a to <2 x float>
-  ret <2 x float> %c
-}
-
-define <2 x float> @utofp_v2i128_v2f32(<2 x i128> %a) {
-; CHECK-SD-LABEL: utofp_v2i128_v2f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #48
-; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NEXT:    mov x19, x1
-; CHECK-SD-NEXT:    mov x20, x0
-; CHECK-SD-NEXT:    mov x0, x2
-; CHECK-SD-NEXT:    mov x1, x3
-; CHECK-SD-NEXT:    bl __floatuntisf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatuntisf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NEXT:    add sp, sp, #48
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i128_v2f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #48
-; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    bl __floatuntisf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatuntisf
-; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    mov v1.s[1], v0.s[0]
-; CHECK-GI-NEXT:    fmov d0, d1
-; CHECK-GI-NEXT:    add sp, sp, #48
-; CHECK-GI-NEXT:    ret
+define <2 x double> @stofp_v2i128_v2f64(<2 x i128> %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_v2i128_v2f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    mov x19, x1
+; CHECK-NOFP16-SD-NEXT:    mov x20, x0
+; CHECK-NOFP16-SD-NEXT:    mov x0, x2
+; CHECK-NOFP16-SD-NEXT:    mov x1, x3
+; CHECK-NOFP16-SD-NEXT:    bl __floattidf
+; CHECK-NOFP16-SD-NEXT:    mov x0, x20
+; CHECK-NOFP16-SD-NEXT:    mov x1, x19
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floattidf
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i128_v2f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    mov x19, x1
+; CHECK-FP16-SD-NEXT:    mov x20, x0
+; CHECK-FP16-SD-NEXT:    mov x0, x2
+; CHECK-FP16-SD-NEXT:    mov x1, x3
+; CHECK-FP16-SD-NEXT:    bl __floattidf
+; CHECK-FP16-SD-NEXT:    mov x0, x20
+; CHECK-FP16-SD-NEXT:    mov x1, x19
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floattidf
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i128_v2f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-GI-NEXT:    mov x19, x2
+; CHECK-NOFP16-GI-NEXT:    mov x20, x3
+; CHECK-NOFP16-GI-NEXT:    bl __floattidf
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floattidf
+; CHECK-NOFP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i128_v2f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-GI-NEXT:    mov x19, x2
+; CHECK-FP16-GI-NEXT:    mov x20, x3
+; CHECK-FP16-GI-NEXT:    bl __floattidf
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floattidf
+; CHECK-FP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-FP16-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    add sp, sp, #48
+; CHECK-FP16-GI-NEXT:    ret
 entry:
-  %c = uitofp <2 x i128> %a to <2 x float>
-  ret <2 x float> %c
-}
-
-define <3 x float> @stofp_v3i128_v3f32(<3 x i128> %a) {
-; CHECK-SD-LABEL: stofp_v3i128_v3f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #80
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    mov x21, x1
-; CHECK-SD-NEXT:    mov x22, x0
-; CHECK-SD-NEXT:    mov x0, x4
-; CHECK-SD-NEXT:    mov x1, x5
-; CHECK-SD-NEXT:    mov x19, x3
-; CHECK-SD-NEXT:    mov x20, x2
-; CHECK-SD-NEXT:    bl __floattisf
-; CHECK-SD-NEXT:    mov x0, x22
-; CHECK-SD-NEXT:    mov x1, x21
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floattisf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floattisf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v1.s[1], v0.s[0]
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov v1.s[2], v0.s[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    add sp, sp, #80
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i128_v3f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #80
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    mov x21, x4
-; CHECK-GI-NEXT:    mov x22, x5
-; CHECK-GI-NEXT:    bl __floattisf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floattisf
-; CHECK-GI-NEXT:    mov x0, x21
-; CHECK-GI-NEXT:    mov x1, x22
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floattisf
-; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
-; CHECK-GI-NEXT:    mov v1.s[2], v0.s[0]
-; CHECK-GI-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NEXT:    add sp, sp, #80
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <3 x i128> %a to <3 x float>
-  ret <3 x float> %c
+  %c = sitofp <2 x i128> %a to <2 x double>
+  ret <2 x double> %c
 }
 
-define <3 x float> @utofp_v3i128_v3f32(<3 x i128> %a) {
-; CHECK-SD-LABEL: utofp_v3i128_v3f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sub sp, sp, #80
-; CHECK-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-SD-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NEXT:    mov x21, x1
-; CHECK-SD-NEXT:    mov x22, x0
-; CHECK-SD-NEXT:    mov x0, x4
-; CHECK-SD-NEXT:    mov x1, x5
-; CHECK-SD-NEXT:    mov x19, x3
-; CHECK-SD-NEXT:    mov x20, x2
-; CHECK-SD-NEXT:    bl __floatuntisf
-; CHECK-SD-NEXT:    mov x0, x22
-; CHECK-SD-NEXT:    mov x1, x21
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatuntisf
-; CHECK-SD-NEXT:    mov x0, x20
-; CHECK-SD-NEXT:    mov x1, x19
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NEXT:    bl __floatuntisf
-; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-SD-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NEXT:    mov v1.s[1], v0.s[0]
-; CHECK-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NEXT:    mov v1.s[2], v0.s[0]
-; CHECK-SD-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NEXT:    add sp, sp, #80
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i128_v3f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sub sp, sp, #80
-; CHECK-GI-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-GI-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NEXT:    mov x19, x2
-; CHECK-GI-NEXT:    mov x20, x3
-; CHECK-GI-NEXT:    mov x21, x4
-; CHECK-GI-NEXT:    mov x22, x5
-; CHECK-GI-NEXT:    bl __floatuntisf
-; CHECK-GI-NEXT:    mov x0, x19
-; CHECK-GI-NEXT:    mov x1, x20
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatuntisf
-; CHECK-GI-NEXT:    mov x0, x21
-; CHECK-GI-NEXT:    mov x1, x22
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    bl __floatuntisf
-; CHECK-GI-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
-; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-GI-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v1.s[1], v2.s[0]
-; CHECK-GI-NEXT:    mov v1.s[2], v0.s[0]
-; CHECK-GI-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NEXT:    add sp, sp, #80
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <3 x i128> %a to <3 x float>
-  ret <3 x float> %c
+define <2 x double> @utofp_v2i128_v2f64(<2 x i128> %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_v2i128_v2f64:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    mov x19, x1
+; CHECK-NOFP16-SD-NEXT:    mov x20, x0
+; CHECK-NOFP16-SD-NEXT:    mov x0, x2
+; CHECK-NOFP16-SD-NEXT:    mov x1, x3
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntidf
+; CHECK-NOFP16-SD-NEXT:    mov x0, x20
+; CHECK-NOFP16-SD-NEXT:    mov x1, x19
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntidf
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i128_v2f64:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    mov x19, x1
+; CHECK-FP16-SD-NEXT:    mov x20, x0
+; CHECK-FP16-SD-NEXT:    mov x0, x2
+; CHECK-FP16-SD-NEXT:    mov x1, x3
+; CHECK-FP16-SD-NEXT:    bl __floatuntidf
+; CHECK-FP16-SD-NEXT:    mov x0, x20
+; CHECK-FP16-SD-NEXT:    mov x1, x19
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floatuntidf
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i128_v2f64:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-GI-NEXT:    mov x19, x2
+; CHECK-NOFP16-GI-NEXT:    mov x20, x3
+; CHECK-NOFP16-GI-NEXT:    bl __floatuntidf
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floatuntidf
+; CHECK-NOFP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i128_v2f64:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-GI-NEXT:    mov x19, x2
+; CHECK-FP16-GI-NEXT:    mov x20, x3
+; CHECK-FP16-GI-NEXT:    bl __floatuntidf
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floatuntidf
+; CHECK-FP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v1.d[1], v0.d[0]
+; CHECK-FP16-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    add sp, sp, #48
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = uitofp <2 x i128> %a to <2 x double>
+  ret <2 x double> %c
 }
 
 define <2 x float> @stofp_v2i64_v2f32(<2 x i64> %a) {
@@ -4339,885 +3961,487 @@ entry:
 }
 
 define <3 x float> @stofp_v3i64_v3f32(<3 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v3i64_v3f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov x8, d0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT:    scvtf s3, x8
-; CHECK-SD-NEXT:    fmov x8, d1
-; CHECK-SD-NEXT:    scvtf s1, x8
-; CHECK-SD-NEXT:    fmov x8, d2
-; CHECK-SD-NEXT:    mov v0.s[0], v3.s[0]
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    scvtf s1, x8
-; CHECK-SD-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i64_v3f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    scvtf s0, x8
-; CHECK-GI-NEXT:    scvtf s1, x9
-; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    scvtf s2, x8
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v3i64_v3f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    fmov x8, d0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NOFP16-SD-NEXT:    scvtf s3, x8
+; CHECK-NOFP16-SD-NEXT:    fmov x8, d1
+; CHECK-NOFP16-SD-NEXT:    scvtf s1, x8
+; CHECK-NOFP16-SD-NEXT:    fmov x8, d2
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[0], v3.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    scvtf s1, x8
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v3i64_v3f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    fmov x8, d0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-FP16-SD-NEXT:    scvtf s3, x8
+; CHECK-FP16-SD-NEXT:    fmov x8, d1
+; CHECK-FP16-SD-NEXT:    scvtf s1, x8
+; CHECK-FP16-SD-NEXT:    fmov x8, d2
+; CHECK-FP16-SD-NEXT:    mov v0.s[0], v3.s[0]
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-FP16-SD-NEXT:    scvtf s1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i64_v3f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov x8, d0
+; CHECK-NOFP16-GI-NEXT:    fmov x9, d1
+; CHECK-NOFP16-GI-NEXT:    scvtf s0, x8
+; CHECK-NOFP16-GI-NEXT:    scvtf s1, x9
+; CHECK-NOFP16-GI-NEXT:    fmov x8, d2
+; CHECK-NOFP16-GI-NEXT:    scvtf s2, x8
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v3i64_v3f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov x8, d0
+; CHECK-FP16-GI-NEXT:    fmov x9, d1
+; CHECK-FP16-GI-NEXT:    scvtf s0, x8
+; CHECK-FP16-GI-NEXT:    scvtf s1, x9
+; CHECK-FP16-GI-NEXT:    fmov x8, d2
+; CHECK-FP16-GI-NEXT:    scvtf s2, x8
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-FP16-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <3 x i64> %a to <3 x float>
   ret <3 x float> %c
 }
 
 define <3 x float> @utofp_v3i64_v3f32(<3 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v3i64_v3f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    fmov x8, d0
-; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
-; CHECK-SD-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-SD-NEXT:    ucvtf s3, x8
-; CHECK-SD-NEXT:    fmov x8, d1
-; CHECK-SD-NEXT:    ucvtf s1, x8
-; CHECK-SD-NEXT:    fmov x8, d2
-; CHECK-SD-NEXT:    mov v0.s[0], v3.s[0]
-; CHECK-SD-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-SD-NEXT:    ucvtf s1, x8
-; CHECK-SD-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i64_v3f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    ucvtf s0, x8
-; CHECK-GI-NEXT:    ucvtf s1, x9
-; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    ucvtf s2, x8
-; CHECK-GI-NEXT:    mov v0.s[1], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[2], v2.s[0]
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v3i64_v3f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    fmov x8, d0
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NOFP16-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NOFP16-SD-NEXT:    ucvtf s3, x8
+; CHECK-NOFP16-SD-NEXT:    fmov x8, d1
+; CHECK-NOFP16-SD-NEXT:    ucvtf s1, x8
+; CHECK-NOFP16-SD-NEXT:    fmov x8, d2
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[0], v3.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    ucvtf s1, x8
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v3i64_v3f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    fmov x8, d0
+; CHECK-FP16-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-FP16-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-FP16-SD-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-FP16-SD-NEXT:    ucvtf s3, x8
+; CHECK-FP16-SD-NEXT:    fmov x8, d1
+; CHECK-FP16-SD-NEXT:    ucvtf s1, x8
+; CHECK-FP16-SD-NEXT:    fmov x8, d2
+; CHECK-FP16-SD-NEXT:    mov v0.s[0], v3.s[0]
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-FP16-SD-NEXT:    ucvtf s1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i64_v3f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov x8, d0
+; CHECK-NOFP16-GI-NEXT:    fmov x9, d1
+; CHECK-NOFP16-GI-NEXT:    ucvtf s0, x8
+; CHECK-NOFP16-GI-NEXT:    ucvtf s1, x9
+; CHECK-NOFP16-GI-NEXT:    fmov x8, d2
+; CHECK-NOFP16-GI-NEXT:    ucvtf s2, x8
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v3i64_v3f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov x8, d0
+; CHECK-FP16-GI-NEXT:    fmov x9, d1
+; CHECK-FP16-GI-NEXT:    ucvtf s0, x8
+; CHECK-FP16-GI-NEXT:    ucvtf s1, x9
+; CHECK-FP16-GI-NEXT:    fmov x8, d2
+; CHECK-FP16-GI-NEXT:    ucvtf s2, x8
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-FP16-GI-NEXT:    mov v0.s[2], v2.s[0]
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <3 x i64> %a to <3 x float>
   ret <3 x float> %c
 }
 
 define <4 x float> @stofp_v4i64_v4f32(<4 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v4i64_v4f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    fmov x9, d0
-; CHECK-SD-NEXT:    scvtf s0, x9
-; CHECK-SD-NEXT:    mov x9, v1.d[1]
-; CHECK-SD-NEXT:    scvtf s2, x8
-; CHECK-SD-NEXT:    fmov x8, d1
-; CHECK-SD-NEXT:    scvtf s1, x8
-; CHECK-SD-NEXT:    mov v0.s[1], v2.s[0]
-; CHECK-SD-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-SD-NEXT:    scvtf s1, x9
-; CHECK-SD-NEXT:    mov v0.s[3], v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v4i64_v4f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov x8, v0.d[1]
-; CHECK-GI-NEXT:    fmov x9, d0
-; CHECK-GI-NEXT:    scvtf s0, x9
-; CHECK-GI-NEXT:    mov x9, v1.d[1]
-; CHECK-GI-NEXT:    scvtf s2, x8
-; CHECK-GI-NEXT:    fmov x8, d1
-; CHECK-GI-NEXT:    scvtf s1, x8
-; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
-; CHECK-GI-NEXT:    scvtf s2, x9
-; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[3], v2.s[0]
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v4i64_v4f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    fmov x9, d0
+; CHECK-NOFP16-SD-NEXT:    scvtf s0, x9
+; CHECK-NOFP16-SD-NEXT:    mov x9, v1.d[1]
+; CHECK-NOFP16-SD-NEXT:    scvtf s2, x8
+; CHECK-NOFP16-SD-NEXT:    fmov x8, d1
+; CHECK-NOFP16-SD-NEXT:    scvtf s1, x8
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    scvtf s1, x9
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[3], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v4i64_v4f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-SD-NEXT:    fmov x9, d0
+; CHECK-FP16-SD-NEXT:    scvtf s0, x9
+; CHECK-FP16-SD-NEXT:    mov x9, v1.d[1]
+; CHECK-FP16-SD-NEXT:    scvtf s2, x8
+; CHECK-FP16-SD-NEXT:    fmov x8, d1
+; CHECK-FP16-SD-NEXT:    scvtf s1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-FP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-SD-NEXT:    scvtf s1, x9
+; CHECK-FP16-SD-NEXT:    mov v0.s[3], v1.s[0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v4i64_v4f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    fmov x9, d0
+; CHECK-NOFP16-GI-NEXT:    scvtf s0, x9
+; CHECK-NOFP16-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-NOFP16-GI-NEXT:    scvtf s2, x8
+; CHECK-NOFP16-GI-NEXT:    fmov x8, d1
+; CHECK-NOFP16-GI-NEXT:    scvtf s1, x8
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-NOFP16-GI-NEXT:    scvtf s2, x9
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[3], v2.s[0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v4i64_v4f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    fmov x9, d0
+; CHECK-FP16-GI-NEXT:    scvtf s0, x9
+; CHECK-FP16-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-FP16-GI-NEXT:    scvtf s2, x8
+; CHECK-FP16-GI-NEXT:    fmov x8, d1
+; CHECK-FP16-GI-NEXT:    scvtf s1, x8
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-FP16-GI-NEXT:    scvtf s2, x9
+; CHECK-FP16-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-GI-NEXT:    mov v0.s[3], v2.s[0]
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <4 x i64> %a to <4 x float>
   ret <4 x float> %c
 }
 
 define <4 x float> @utofp_v4i64_v4f32(<4 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v4i64_v4f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    fmov x9, d0
-; CHECK-SD-NEXT:    ucvtf s0, x9
-; CHECK-SD-NEXT:    mov x9, v1.d[1]
-; CHECK-SD-NEXT:    ucvtf s2, x8
-; CHECK-SD-NEXT:    fmov x8, d1
-; CHECK-SD-NEXT:    ucvtf s1, x8
-; CHECK-SD-NEXT:    mov v0.s[1], v2.s[0]
-; CHECK-SD-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-SD-NEXT:    ucvtf s1, x9
-; CHECK-SD-NEXT:    mov v0.s[3], v1.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v4i64_v4f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov x8, v0.d[1]
-; CHECK-GI-NEXT:    fmov x9, d0
-; CHECK-GI-NEXT:    ucvtf s0, x9
-; CHECK-GI-NEXT:    mov x9, v1.d[1]
-; CHECK-GI-NEXT:    ucvtf s2, x8
-; CHECK-GI-NEXT:    fmov x8, d1
-; CHECK-GI-NEXT:    ucvtf s1, x8
-; CHECK-GI-NEXT:    mov v0.s[1], v2.s[0]
-; CHECK-GI-NEXT:    ucvtf s2, x9
-; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-GI-NEXT:    mov v0.s[3], v2.s[0]
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v4i64_v4f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    fmov x9, d0
+; CHECK-NOFP16-SD-NEXT:    ucvtf s0, x9
+; CHECK-NOFP16-SD-NEXT:    mov x9, v1.d[1]
+; CHECK-NOFP16-SD-NEXT:    ucvtf s2, x8
+; CHECK-NOFP16-SD-NEXT:    fmov x8, d1
+; CHECK-NOFP16-SD-NEXT:    ucvtf s1, x8
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    ucvtf s1, x9
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[3], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v4i64_v4f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-SD-NEXT:    fmov x9, d0
+; CHECK-FP16-SD-NEXT:    ucvtf s0, x9
+; CHECK-FP16-SD-NEXT:    mov x9, v1.d[1]
+; CHECK-FP16-SD-NEXT:    ucvtf s2, x8
+; CHECK-FP16-SD-NEXT:    fmov x8, d1
+; CHECK-FP16-SD-NEXT:    ucvtf s1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-FP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-SD-NEXT:    ucvtf s1, x9
+; CHECK-FP16-SD-NEXT:    mov v0.s[3], v1.s[0]
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v4i64_v4f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    fmov x9, d0
+; CHECK-NOFP16-GI-NEXT:    ucvtf s0, x9
+; CHECK-NOFP16-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-NOFP16-GI-NEXT:    ucvtf s2, x8
+; CHECK-NOFP16-GI-NEXT:    fmov x8, d1
+; CHECK-NOFP16-GI-NEXT:    ucvtf s1, x8
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-NOFP16-GI-NEXT:    ucvtf s2, x9
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[3], v2.s[0]
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v4i64_v4f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    fmov x9, d0
+; CHECK-FP16-GI-NEXT:    ucvtf s0, x9
+; CHECK-FP16-GI-NEXT:    mov x9, v1.d[1]
+; CHECK-FP16-GI-NEXT:    ucvtf s2, x8
+; CHECK-FP16-GI-NEXT:    fmov x8, d1
+; CHECK-FP16-GI-NEXT:    ucvtf s1, x8
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-FP16-GI-NEXT:    ucvtf s2, x9
+; CHECK-FP16-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-GI-NEXT:    mov v0.s[3], v2.s[0]
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <4 x i64> %a to <4 x float>
   ret <4 x float> %c
 }
 
 define <8 x float> @stofp_v8i64_v8f32(<8 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v8i64_v8f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    mov x9, v2.d[1]
-; CHECK-SD-NEXT:    fmov x10, d0
-; CHECK-SD-NEXT:    fmov x11, d2
-; CHECK-SD-NEXT:    scvtf s0, x10
-; CHECK-SD-NEXT:    mov x10, v3.d[1]
-; CHECK-SD-NEXT:    scvtf s4, x8
-; CHECK-SD-NEXT:    scvtf s5, x9
-; CHECK-SD-NEXT:    scvtf s2, x11
-; CHECK-SD-NEXT:    fmov x9, d1
-; CHECK-SD-NEXT:    fmov x11, d3
-; CHECK-SD-NEXT:    mov x8, v1.d[1]
-; CHECK-SD-NEXT:    scvtf s1, x9
-; CHECK-SD-NEXT:    mov v0.s[1], v4.s[0]
-; CHECK-SD-NEXT:    scvtf s3, x11
-; CHECK-SD-NEXT:    mov v2.s[1], v5.s[0]
-; CHECK-SD-NEXT:    scvtf s4, x8
-; CHECK-SD-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-SD-NEXT:    scvtf s1, x10
-; CHECK-SD-NEXT:    mov v2.s[2], v3.s[0]
-; CHECK-SD-NEXT:    mov v0.s[3], v4.s[0]
-; CHECK-SD-NEXT:    mov v2.s[3], v1.s[0]
-; CHECK-SD-NEXT:    mov v1.16b, v2.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v8i64_v8f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov x10, d0
-; CHECK-GI-NEXT:    mov x9, v2.d[1]
-; CHECK-GI-NEXT:    mov x8, v0.d[1]
-; CHECK-GI-NEXT:    scvtf s0, x10
-; CHECK-GI-NEXT:    fmov x10, d2
-; CHECK-GI-NEXT:    scvtf s5, x9
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    scvtf s4, x8
-; CHECK-GI-NEXT:    mov x8, v1.d[1]
-; CHECK-GI-NEXT:    scvtf s2, x10
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    scvtf s1, x9
-; CHECK-GI-NEXT:    mov x9, v3.d[1]
-; CHECK-GI-NEXT:    mov v0.s[1], v4.s[0]
-; CHECK-GI-NEXT:    scvtf s3, x10
-; CHECK-GI-NEXT:    scvtf s4, x8
-; CHECK-GI-NEXT:    mov v2.s[1], v5.s[0]
-; CHECK-GI-NEXT:    scvtf s5, x9
-; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
-; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
-; CHECK-GI-NEXT:    mov v2.s[3], v5.s[0]
-; CHECK-GI-NEXT:    mov v1.16b, v2.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i64_v8f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    mov x9, v2.d[1]
+; CHECK-NOFP16-SD-NEXT:    fmov x10, d0
+; CHECK-NOFP16-SD-NEXT:    fmov x11, d2
+; CHECK-NOFP16-SD-NEXT:    scvtf s0, x10
+; CHECK-NOFP16-SD-NEXT:    mov x10, v3.d[1]
+; CHECK-NOFP16-SD-NEXT:    scvtf s4, x8
+; CHECK-NOFP16-SD-NEXT:    scvtf s5, x9
+; CHECK-NOFP16-SD-NEXT:    scvtf s2, x11
+; CHECK-NOFP16-SD-NEXT:    fmov x9, d1
+; CHECK-NOFP16-SD-NEXT:    fmov x11, d3
+; CHECK-NOFP16-SD-NEXT:    mov x8, v1.d[1]
+; CHECK-NOFP16-SD-NEXT:    scvtf s1, x9
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-NOFP16-SD-NEXT:    scvtf s3, x11
+; CHECK-NOFP16-SD-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-NOFP16-SD-NEXT:    scvtf s4, x8
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    scvtf s1, x10
+; CHECK-NOFP16-SD-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v2.s[3], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v8i64_v8f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-SD-NEXT:    mov x9, v2.d[1]
+; CHECK-FP16-SD-NEXT:    fmov x10, d0
+; CHECK-FP16-SD-NEXT:    fmov x11, d2
+; CHECK-FP16-SD-NEXT:    scvtf s0, x10
+; CHECK-FP16-SD-NEXT:    mov x10, v3.d[1]
+; CHECK-FP16-SD-NEXT:    scvtf s4, x8
+; CHECK-FP16-SD-NEXT:    scvtf s5, x9
+; CHECK-FP16-SD-NEXT:    scvtf s2, x11
+; CHECK-FP16-SD-NEXT:    fmov x9, d1
+; CHECK-FP16-SD-NEXT:    fmov x11, d3
+; CHECK-FP16-SD-NEXT:    mov x8, v1.d[1]
+; CHECK-FP16-SD-NEXT:    scvtf s1, x9
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-FP16-SD-NEXT:    scvtf s3, x11
+; CHECK-FP16-SD-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-FP16-SD-NEXT:    scvtf s4, x8
+; CHECK-FP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-SD-NEXT:    scvtf s1, x10
+; CHECK-FP16-SD-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-FP16-SD-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-FP16-SD-NEXT:    mov v2.s[3], v1.s[0]
+; CHECK-FP16-SD-NEXT:    mov v1.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i64_v8f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov x10, d0
+; CHECK-NOFP16-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-NOFP16-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    scvtf s0, x10
+; CHECK-NOFP16-GI-NEXT:    fmov x10, d2
+; CHECK-NOFP16-GI-NEXT:    scvtf s5, x9
+; CHECK-NOFP16-GI-NEXT:    fmov x9, d1
+; CHECK-NOFP16-GI-NEXT:    scvtf s4, x8
+; CHECK-NOFP16-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-NOFP16-GI-NEXT:    scvtf s2, x10
+; CHECK-NOFP16-GI-NEXT:    fmov x10, d3
+; CHECK-NOFP16-GI-NEXT:    scvtf s1, x9
+; CHECK-NOFP16-GI-NEXT:    mov x9, v3.d[1]
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-NOFP16-GI-NEXT:    scvtf s3, x10
+; CHECK-NOFP16-GI-NEXT:    scvtf s4, x8
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-NOFP16-GI-NEXT:    scvtf s5, x9
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[3], v5.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i64_v8f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov x10, d0
+; CHECK-FP16-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-FP16-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    scvtf s0, x10
+; CHECK-FP16-GI-NEXT:    fmov x10, d2
+; CHECK-FP16-GI-NEXT:    scvtf s5, x9
+; CHECK-FP16-GI-NEXT:    fmov x9, d1
+; CHECK-FP16-GI-NEXT:    scvtf s4, x8
+; CHECK-FP16-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-FP16-GI-NEXT:    scvtf s2, x10
+; CHECK-FP16-GI-NEXT:    fmov x10, d3
+; CHECK-FP16-GI-NEXT:    scvtf s1, x9
+; CHECK-FP16-GI-NEXT:    mov x9, v3.d[1]
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-FP16-GI-NEXT:    scvtf s3, x10
+; CHECK-FP16-GI-NEXT:    scvtf s4, x8
+; CHECK-FP16-GI-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-FP16-GI-NEXT:    scvtf s5, x9
+; CHECK-FP16-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-FP16-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-FP16-GI-NEXT:    mov v2.s[3], v5.s[0]
+; CHECK-FP16-GI-NEXT:    mov v1.16b, v2.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i64> %a to <8 x float>
   ret <8 x float> %c
 }
 
 define <8 x float> @utofp_v8i64_v8f32(<8 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v8i64_v8f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    mov x9, v2.d[1]
-; CHECK-SD-NEXT:    fmov x10, d0
-; CHECK-SD-NEXT:    fmov x11, d2
-; CHECK-SD-NEXT:    ucvtf s0, x10
-; CHECK-SD-NEXT:    mov x10, v3.d[1]
-; CHECK-SD-NEXT:    ucvtf s4, x8
-; CHECK-SD-NEXT:    ucvtf s5, x9
-; CHECK-SD-NEXT:    ucvtf s2, x11
-; CHECK-SD-NEXT:    fmov x9, d1
-; CHECK-SD-NEXT:    fmov x11, d3
-; CHECK-SD-NEXT:    mov x8, v1.d[1]
-; CHECK-SD-NEXT:    ucvtf s1, x9
-; CHECK-SD-NEXT:    mov v0.s[1], v4.s[0]
-; CHECK-SD-NEXT:    ucvtf s3, x11
-; CHECK-SD-NEXT:    mov v2.s[1], v5.s[0]
-; CHECK-SD-NEXT:    ucvtf s4, x8
-; CHECK-SD-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-SD-NEXT:    ucvtf s1, x10
-; CHECK-SD-NEXT:    mov v2.s[2], v3.s[0]
-; CHECK-SD-NEXT:    mov v0.s[3], v4.s[0]
-; CHECK-SD-NEXT:    mov v2.s[3], v1.s[0]
-; CHECK-SD-NEXT:    mov v1.16b, v2.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v8i64_v8f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov x10, d0
-; CHECK-GI-NEXT:    mov x9, v2.d[1]
-; CHECK-GI-NEXT:    mov x8, v0.d[1]
-; CHECK-GI-NEXT:    ucvtf s0, x10
-; CHECK-GI-NEXT:    fmov x10, d2
-; CHECK-GI-NEXT:    ucvtf s5, x9
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    ucvtf s4, x8
-; CHECK-GI-NEXT:    mov x8, v1.d[1]
-; CHECK-GI-NEXT:    ucvtf s2, x10
-; CHECK-GI-NEXT:    fmov x10, d3
-; CHECK-GI-NEXT:    ucvtf s1, x9
-; CHECK-GI-NEXT:    mov x9, v3.d[1]
-; CHECK-GI-NEXT:    mov v0.s[1], v4.s[0]
-; CHECK-GI-NEXT:    ucvtf s3, x10
-; CHECK-GI-NEXT:    ucvtf s4, x8
-; CHECK-GI-NEXT:    mov v2.s[1], v5.s[0]
-; CHECK-GI-NEXT:    ucvtf s5, x9
-; CHECK-GI-NEXT:    mov v0.s[2], v1.s[0]
-; CHECK-GI-NEXT:    mov v2.s[2], v3.s[0]
-; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
-; CHECK-GI-NEXT:    mov v2.s[3], v5.s[0]
-; CHECK-GI-NEXT:    mov v1.16b, v2.16b
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i64_v8f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    mov x9, v2.d[1]
+; CHECK-NOFP16-SD-NEXT:    fmov x10, d0
+; CHECK-NOFP16-SD-NEXT:    fmov x11, d2
+; CHECK-NOFP16-SD-NEXT:    ucvtf s0, x10
+; CHECK-NOFP16-SD-NEXT:    mov x10, v3.d[1]
+; CHECK-NOFP16-SD-NEXT:    ucvtf s4, x8
+; CHECK-NOFP16-SD-NEXT:    ucvtf s5, x9
+; CHECK-NOFP16-SD-NEXT:    ucvtf s2, x11
+; CHECK-NOFP16-SD-NEXT:    fmov x9, d1
+; CHECK-NOFP16-SD-NEXT:    fmov x11, d3
+; CHECK-NOFP16-SD-NEXT:    mov x8, v1.d[1]
+; CHECK-NOFP16-SD-NEXT:    ucvtf s1, x9
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-NOFP16-SD-NEXT:    ucvtf s3, x11
+; CHECK-NOFP16-SD-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-NOFP16-SD-NEXT:    ucvtf s4, x8
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    ucvtf s1, x10
+; CHECK-NOFP16-SD-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v2.s[3], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v2.16b
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v8i64_v8f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-SD-NEXT:    mov x9, v2.d[1]
+; CHECK-FP16-SD-NEXT:    fmov x10, d0
+; CHECK-FP16-SD-NEXT:    fmov x11, d2
+; CHECK-FP16-SD-NEXT:    ucvtf s0, x10
+; CHECK-FP16-SD-NEXT:    mov x10, v3.d[1]
+; CHECK-FP16-SD-NEXT:    ucvtf s4, x8
+; CHECK-FP16-SD-NEXT:    ucvtf s5, x9
+; CHECK-FP16-SD-NEXT:    ucvtf s2, x11
+; CHECK-FP16-SD-NEXT:    fmov x9, d1
+; CHECK-FP16-SD-NEXT:    fmov x11, d3
+; CHECK-FP16-SD-NEXT:    mov x8, v1.d[1]
+; CHECK-FP16-SD-NEXT:    ucvtf s1, x9
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-FP16-SD-NEXT:    ucvtf s3, x11
+; CHECK-FP16-SD-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-FP16-SD-NEXT:    ucvtf s4, x8
+; CHECK-FP16-SD-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-SD-NEXT:    ucvtf s1, x10
+; CHECK-FP16-SD-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-FP16-SD-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-FP16-SD-NEXT:    mov v2.s[3], v1.s[0]
+; CHECK-FP16-SD-NEXT:    mov v1.16b, v2.16b
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i64_v8f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov x10, d0
+; CHECK-NOFP16-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-NOFP16-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    ucvtf s0, x10
+; CHECK-NOFP16-GI-NEXT:    fmov x10, d2
+; CHECK-NOFP16-GI-NEXT:    ucvtf s5, x9
+; CHECK-NOFP16-GI-NEXT:    fmov x9, d1
+; CHECK-NOFP16-GI-NEXT:    ucvtf s4, x8
+; CHECK-NOFP16-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-NOFP16-GI-NEXT:    ucvtf s2, x10
+; CHECK-NOFP16-GI-NEXT:    fmov x10, d3
+; CHECK-NOFP16-GI-NEXT:    ucvtf s1, x9
+; CHECK-NOFP16-GI-NEXT:    mov x9, v3.d[1]
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-NOFP16-GI-NEXT:    ucvtf s3, x10
+; CHECK-NOFP16-GI-NEXT:    ucvtf s4, x8
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-NOFP16-GI-NEXT:    ucvtf s5, x9
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v2.s[3], v5.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.16b, v2.16b
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i64_v8f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov x10, d0
+; CHECK-FP16-GI-NEXT:    mov x9, v2.d[1]
+; CHECK-FP16-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    ucvtf s0, x10
+; CHECK-FP16-GI-NEXT:    fmov x10, d2
+; CHECK-FP16-GI-NEXT:    ucvtf s5, x9
+; CHECK-FP16-GI-NEXT:    fmov x9, d1
+; CHECK-FP16-GI-NEXT:    ucvtf s4, x8
+; CHECK-FP16-GI-NEXT:    mov x8, v1.d[1]
+; CHECK-FP16-GI-NEXT:    ucvtf s2, x10
+; CHECK-FP16-GI-NEXT:    fmov x10, d3
+; CHECK-FP16-GI-NEXT:    ucvtf s1, x9
+; CHECK-FP16-GI-NEXT:    mov x9, v3.d[1]
+; CHECK-FP16-GI-NEXT:    mov v0.s[1], v4.s[0]
+; CHECK-FP16-GI-NEXT:    ucvtf s3, x10
+; CHECK-FP16-GI-NEXT:    ucvtf s4, x8
+; CHECK-FP16-GI-NEXT:    mov v2.s[1], v5.s[0]
+; CHECK-FP16-GI-NEXT:    ucvtf s5, x9
+; CHECK-FP16-GI-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-FP16-GI-NEXT:    mov v2.s[2], v3.s[0]
+; CHECK-FP16-GI-NEXT:    mov v0.s[3], v4.s[0]
+; CHECK-FP16-GI-NEXT:    mov v2.s[3], v5.s[0]
+; CHECK-FP16-GI-NEXT:    mov v1.16b, v2.16b
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i64> %a to <8 x float>
   ret <8 x float> %c
 }
 
-define <16 x float> @stofp_v16i64_v16f32(<16 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v16i64_v16f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov x13, d2
-; CHECK-SD-NEXT:    mov x9, v0.d[1]
-; CHECK-SD-NEXT:    mov x10, v2.d[1]
-; CHECK-SD-NEXT:    fmov x11, d0
-; CHECK-SD-NEXT:    mov x12, v4.d[1]
-; CHECK-SD-NEXT:    mov x8, v1.d[1]
-; CHECK-SD-NEXT:    scvtf s16, x13
-; CHECK-SD-NEXT:    fmov x13, d4
-; CHECK-SD-NEXT:    scvtf s0, x11
-; CHECK-SD-NEXT:    mov x11, v6.d[1]
-; CHECK-SD-NEXT:    scvtf s17, x9
-; CHECK-SD-NEXT:    scvtf s18, x10
-; CHECK-SD-NEXT:    fmov x9, d1
-; CHECK-SD-NEXT:    scvtf s1, x12
-; CHECK-SD-NEXT:    fmov x12, d6
-; CHECK-SD-NEXT:    scvtf s2, x13
-; CHECK-SD-NEXT:    fmov x13, d3
-; CHECK-SD-NEXT:    mov x10, v3.d[1]
-; CHECK-SD-NEXT:    scvtf s4, x11
-; CHECK-SD-NEXT:    mov v0.s[1], v17.s[0]
-; CHECK-SD-NEXT:    scvtf s6, x9
-; CHECK-SD-NEXT:    scvtf s3, x12
-; CHECK-SD-NEXT:    mov v16.s[1], v18.s[0]
-; CHECK-SD-NEXT:    mov x9, v5.d[1]
-; CHECK-SD-NEXT:    fmov x11, d5
-; CHECK-SD-NEXT:    scvtf s5, x13
-; CHECK-SD-NEXT:    fmov x13, d7
-; CHECK-SD-NEXT:    mov x12, v7.d[1]
-; CHECK-SD-NEXT:    mov v2.s[1], v1.s[0]
-; CHECK-SD-NEXT:    mov v0.s[2], v6.s[0]
-; CHECK-SD-NEXT:    scvtf s6, x10
-; CHECK-SD-NEXT:    scvtf s7, x11
-; CHECK-SD-NEXT:    scvtf s1, x13
-; CHECK-SD-NEXT:    mov v3.s[1], v4.s[0]
-; CHECK-SD-NEXT:    mov v16.s[2], v5.s[0]
-; CHECK-SD-NEXT:    scvtf s4, x8
-; CHECK-SD-NEXT:    scvtf s5, x9
-; CHECK-SD-NEXT:    mov v2.s[2], v7.s[0]
-; CHECK-SD-NEXT:    mov v3.s[2], v1.s[0]
-; CHECK-SD-NEXT:    scvtf s1, x12
-; CHECK-SD-NEXT:    mov v16.s[3], v6.s[0]
-; CHECK-SD-NEXT:    mov v0.s[3], v4.s[0]
-; CHECK-SD-NEXT:    mov v2.s[3], v5.s[0]
-; CHECK-SD-NEXT:    mov v3.s[3], v1.s[0]
-; CHECK-SD-NEXT:    mov v1.16b, v16.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v16i64_v16f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov x9, v0.d[1]
-; CHECK-GI-NEXT:    fmov x13, d2
-; CHECK-GI-NEXT:    fmov x11, d0
-; CHECK-GI-NEXT:    mov x12, v4.d[1]
-; CHECK-GI-NEXT:    mov x8, v1.d[1]
-; CHECK-GI-NEXT:    mov x10, v2.d[1]
-; CHECK-GI-NEXT:    scvtf s0, x11
-; CHECK-GI-NEXT:    mov x11, v6.d[1]
-; CHECK-GI-NEXT:    scvtf s16, x9
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    scvtf s1, x13
-; CHECK-GI-NEXT:    fmov x13, d4
-; CHECK-GI-NEXT:    scvtf s4, x12
-; CHECK-GI-NEXT:    fmov x12, d6
-; CHECK-GI-NEXT:    scvtf s17, x10
-; CHECK-GI-NEXT:    mov x10, v3.d[1]
-; CHECK-GI-NEXT:    scvtf s6, x11
-; CHECK-GI-NEXT:    fmov x11, d5
-; CHECK-GI-NEXT:    scvtf s18, x9
-; CHECK-GI-NEXT:    scvtf s2, x13
-; CHECK-GI-NEXT:    fmov x13, d3
-; CHECK-GI-NEXT:    scvtf s3, x12
-; CHECK-GI-NEXT:    mov x9, v5.d[1]
-; CHECK-GI-NEXT:    mov x12, v7.d[1]
-; CHECK-GI-NEXT:    mov v0.s[1], v16.s[0]
-; CHECK-GI-NEXT:    scvtf s5, x11
-; CHECK-GI-NEXT:    mov v1.s[1], v17.s[0]
-; CHECK-GI-NEXT:    scvtf s19, x13
-; CHECK-GI-NEXT:    fmov x13, d7
-; CHECK-GI-NEXT:    mov v2.s[1], v4.s[0]
-; CHECK-GI-NEXT:    mov v3.s[1], v6.s[0]
-; CHECK-GI-NEXT:    scvtf s4, x8
-; CHECK-GI-NEXT:    scvtf s6, x10
-; CHECK-GI-NEXT:    scvtf s16, x9
-; CHECK-GI-NEXT:    scvtf s17, x12
-; CHECK-GI-NEXT:    scvtf s7, x13
-; CHECK-GI-NEXT:    mov v0.s[2], v18.s[0]
-; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
-; CHECK-GI-NEXT:    mov v2.s[2], v5.s[0]
-; CHECK-GI-NEXT:    mov v3.s[2], v7.s[0]
-; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
-; CHECK-GI-NEXT:    mov v1.s[3], v6.s[0]
-; CHECK-GI-NEXT:    mov v2.s[3], v16.s[0]
-; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <16 x i64> %a to <16 x float>
-  ret <16 x float> %c
-}
-
-define <16 x float> @utofp_v16i64_v16f32(<16 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v16i64_v16f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov x13, d2
-; CHECK-SD-NEXT:    mov x9, v0.d[1]
-; CHECK-SD-NEXT:    mov x10, v2.d[1]
-; CHECK-SD-NEXT:    fmov x11, d0
-; CHECK-SD-NEXT:    mov x12, v4.d[1]
-; CHECK-SD-NEXT:    mov x8, v1.d[1]
-; CHECK-SD-NEXT:    ucvtf s16, x13
-; CHECK-SD-NEXT:    fmov x13, d4
-; CHECK-SD-NEXT:    ucvtf s0, x11
-; CHECK-SD-NEXT:    mov x11, v6.d[1]
-; CHECK-SD-NEXT:    ucvtf s17, x9
-; CHECK-SD-NEXT:    ucvtf s18, x10
-; CHECK-SD-NEXT:    fmov x9, d1
-; CHECK-SD-NEXT:    ucvtf s1, x12
-; CHECK-SD-NEXT:    fmov x12, d6
-; CHECK-SD-NEXT:    ucvtf s2, x13
-; CHECK-SD-NEXT:    fmov x13, d3
-; CHECK-SD-NEXT:    mov x10, v3.d[1]
-; CHECK-SD-NEXT:    ucvtf s4, x11
-; CHECK-SD-NEXT:    mov v0.s[1], v17.s[0]
-; CHECK-SD-NEXT:    ucvtf s6, x9
-; CHECK-SD-NEXT:    ucvtf s3, x12
-; CHECK-SD-NEXT:    mov v16.s[1], v18.s[0]
-; CHECK-SD-NEXT:    mov x9, v5.d[1]
-; CHECK-SD-NEXT:    fmov x11, d5
-; CHECK-SD-NEXT:    ucvtf s5, x13
-; CHECK-SD-NEXT:    fmov x13, d7
-; CHECK-SD-NEXT:    mov x12, v7.d[1]
-; CHECK-SD-NEXT:    mov v2.s[1], v1.s[0]
-; CHECK-SD-NEXT:    mov v0.s[2], v6.s[0]
-; CHECK-SD-NEXT:    ucvtf s6, x10
-; CHECK-SD-NEXT:    ucvtf s7, x11
-; CHECK-SD-NEXT:    ucvtf s1, x13
-; CHECK-SD-NEXT:    mov v3.s[1], v4.s[0]
-; CHECK-SD-NEXT:    mov v16.s[2], v5.s[0]
-; CHECK-SD-NEXT:    ucvtf s4, x8
-; CHECK-SD-NEXT:    ucvtf s5, x9
-; CHECK-SD-NEXT:    mov v2.s[2], v7.s[0]
-; CHECK-SD-NEXT:    mov v3.s[2], v1.s[0]
-; CHECK-SD-NEXT:    ucvtf s1, x12
-; CHECK-SD-NEXT:    mov v16.s[3], v6.s[0]
-; CHECK-SD-NEXT:    mov v0.s[3], v4.s[0]
-; CHECK-SD-NEXT:    mov v2.s[3], v5.s[0]
-; CHECK-SD-NEXT:    mov v3.s[3], v1.s[0]
-; CHECK-SD-NEXT:    mov v1.16b, v16.16b
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v16i64_v16f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    mov x9, v0.d[1]
-; CHECK-GI-NEXT:    fmov x13, d2
-; CHECK-GI-NEXT:    fmov x11, d0
-; CHECK-GI-NEXT:    mov x12, v4.d[1]
-; CHECK-GI-NEXT:    mov x8, v1.d[1]
-; CHECK-GI-NEXT:    mov x10, v2.d[1]
-; CHECK-GI-NEXT:    ucvtf s0, x11
-; CHECK-GI-NEXT:    mov x11, v6.d[1]
-; CHECK-GI-NEXT:    ucvtf s16, x9
-; CHECK-GI-NEXT:    fmov x9, d1
-; CHECK-GI-NEXT:    ucvtf s1, x13
-; CHECK-GI-NEXT:    fmov x13, d4
-; CHECK-GI-NEXT:    ucvtf s4, x12
-; CHECK-GI-NEXT:    fmov x12, d6
-; CHECK-GI-NEXT:    ucvtf s17, x10
-; CHECK-GI-NEXT:    mov x10, v3.d[1]
-; CHECK-GI-NEXT:    ucvtf s6, x11
-; CHECK-GI-NEXT:    fmov x11, d5
-; CHECK-GI-NEXT:    ucvtf s18, x9
-; CHECK-GI-NEXT:    ucvtf s2, x13
-; CHECK-GI-NEXT:    fmov x13, d3
-; CHECK-GI-NEXT:    ucvtf s3, x12
-; CHECK-GI-NEXT:    mov x9, v5.d[1]
-; CHECK-GI-NEXT:    mov x12, v7.d[1]
-; CHECK-GI-NEXT:    mov v0.s[1], v16.s[0]
-; CHECK-GI-NEXT:    ucvtf s5, x11
-; CHECK-GI-NEXT:    mov v1.s[1], v17.s[0]
-; CHECK-GI-NEXT:    ucvtf s19, x13
-; CHECK-GI-NEXT:    fmov x13, d7
-; CHECK-GI-NEXT:    mov v2.s[1], v4.s[0]
-; CHECK-GI-NEXT:    mov v3.s[1], v6.s[0]
-; CHECK-GI-NEXT:    ucvtf s4, x8
-; CHECK-GI-NEXT:    ucvtf s6, x10
-; CHECK-GI-NEXT:    ucvtf s16, x9
-; CHECK-GI-NEXT:    ucvtf s17, x12
-; CHECK-GI-NEXT:    ucvtf s7, x13
-; CHECK-GI-NEXT:    mov v0.s[2], v18.s[0]
-; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
-; CHECK-GI-NEXT:    mov v2.s[2], v5.s[0]
-; CHECK-GI-NEXT:    mov v3.s[2], v7.s[0]
-; CHECK-GI-NEXT:    mov v0.s[3], v4.s[0]
-; CHECK-GI-NEXT:    mov v1.s[3], v6.s[0]
-; CHECK-GI-NEXT:    mov v2.s[3], v16.s[0]
-; CHECK-GI-NEXT:    mov v3.s[3], v17.s[0]
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <16 x i64> %a to <16 x float>
-  ret <16 x float> %c
-}
-
-define <32 x float> @stofp_v32i64_v32f32(<32 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v32i64_v32f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    fmov x10, d0
-; CHECK-SD-NEXT:    mov v16.16b, v1.16b
-; CHECK-SD-NEXT:    fmov x11, d2
-; CHECK-SD-NEXT:    ldp q24, q20, [sp]
-; CHECK-SD-NEXT:    mov x9, v2.d[1]
-; CHECK-SD-NEXT:    fmov x12, d3
-; CHECK-SD-NEXT:    fmov x13, d4
-; CHECK-SD-NEXT:    scvtf s0, x10
-; CHECK-SD-NEXT:    ldp q21, q18, [sp, #32]
-; CHECK-SD-NEXT:    scvtf s2, x8
-; CHECK-SD-NEXT:    scvtf s1, x11
-; CHECK-SD-NEXT:    mov x10, v4.d[1]
-; CHECK-SD-NEXT:    fmov x11, d16
-; CHECK-SD-NEXT:    ldp q19, q17, [sp, #96]
-; CHECK-SD-NEXT:    scvtf s22, x9
-; CHECK-SD-NEXT:    mov x8, v3.d[1]
-; CHECK-SD-NEXT:    scvtf s4, x12
-; CHECK-SD-NEXT:    mov x12, v24.d[1]
-; CHECK-SD-NEXT:    mov x9, v16.d[1]
-; CHECK-SD-NEXT:    scvtf s3, x11
-; CHECK-SD-NEXT:    ldp q23, q16, [sp, #64]
-; CHECK-SD-NEXT:    mov v0.s[1], v2.s[0]
-; CHECK-SD-NEXT:    scvtf s25, x10
-; CHECK-SD-NEXT:    fmov x10, d6
-; CHECK-SD-NEXT:    mov v1.s[1], v22.s[0]
-; CHECK-SD-NEXT:    mov x11, v6.d[1]
-; CHECK-SD-NEXT:    scvtf s2, x13
-; CHECK-SD-NEXT:    mov x13, v21.d[1]
-; CHECK-SD-NEXT:    fmov x14, d19
-; CHECK-SD-NEXT:    scvtf s22, x9
-; CHECK-SD-NEXT:    mov x9, v5.d[1]
-; CHECK-SD-NEXT:    fmov x15, d17
-; CHECK-SD-NEXT:    mov v0.s[2], v3.s[0]
-; CHECK-SD-NEXT:    scvtf s3, x10
-; CHECK-SD-NEXT:    fmov x10, d24
-; CHECK-SD-NEXT:    mov v1.s[2], v4.s[0]
-; CHECK-SD-NEXT:    scvtf s24, x12
-; CHECK-SD-NEXT:    scvtf s6, x11
-; CHECK-SD-NEXT:    fmov x11, d5
-; CHECK-SD-NEXT:    fmov x12, d7
-; CHECK-SD-NEXT:    mov v2.s[1], v25.s[0]
-; CHECK-SD-NEXT:    scvtf s4, x10
-; CHECK-SD-NEXT:    fmov x10, d21
-; CHECK-SD-NEXT:    scvtf s21, x8
-; CHECK-SD-NEXT:    mov x8, v23.d[1]
-; CHECK-SD-NEXT:    scvtf s25, x13
-; CHECK-SD-NEXT:    mov x13, v19.d[1]
-; CHECK-SD-NEXT:    scvtf s26, x11
-; CHECK-SD-NEXT:    mov x11, v20.d[1]
-; CHECK-SD-NEXT:    mov v3.s[1], v6.s[0]
-; CHECK-SD-NEXT:    scvtf s5, x10
-; CHECK-SD-NEXT:    mov x10, v7.d[1]
-; CHECK-SD-NEXT:    scvtf s7, x14
-; CHECK-SD-NEXT:    mov v4.s[1], v24.s[0]
-; CHECK-SD-NEXT:    scvtf s24, x12
-; CHECK-SD-NEXT:    fmov x12, d20
-; CHECK-SD-NEXT:    scvtf s20, x8
-; CHECK-SD-NEXT:    fmov x8, d23
-; CHECK-SD-NEXT:    scvtf s19, x13
-; CHECK-SD-NEXT:    fmov x13, d18
-; CHECK-SD-NEXT:    fmov x14, d16
-; CHECK-SD-NEXT:    mov v2.s[2], v26.s[0]
-; CHECK-SD-NEXT:    mov v5.s[1], v25.s[0]
-; CHECK-SD-NEXT:    scvtf s23, x10
-; CHECK-SD-NEXT:    mov v0.s[3], v22.s[0]
-; CHECK-SD-NEXT:    scvtf s6, x8
-; CHECK-SD-NEXT:    mov x8, v18.d[1]
-; CHECK-SD-NEXT:    scvtf s18, x12
-; CHECK-SD-NEXT:    mov x12, v16.d[1]
-; CHECK-SD-NEXT:    scvtf s16, x13
-; CHECK-SD-NEXT:    mov x13, v17.d[1]
-; CHECK-SD-NEXT:    scvtf s17, x14
-; CHECK-SD-NEXT:    mov v7.s[1], v19.s[0]
-; CHECK-SD-NEXT:    scvtf s19, x9
-; CHECK-SD-NEXT:    mov v3.s[2], v24.s[0]
-; CHECK-SD-NEXT:    scvtf s24, x11
-; CHECK-SD-NEXT:    mov v1.s[3], v21.s[0]
-; CHECK-SD-NEXT:    mov v6.s[1], v20.s[0]
-; CHECK-SD-NEXT:    scvtf s20, x15
-; CHECK-SD-NEXT:    mov v4.s[2], v18.s[0]
-; CHECK-SD-NEXT:    scvtf s18, x8
-; CHECK-SD-NEXT:    mov v5.s[2], v16.s[0]
-; CHECK-SD-NEXT:    scvtf s16, x12
-; CHECK-SD-NEXT:    mov v2.s[3], v19.s[0]
-; CHECK-SD-NEXT:    mov v3.s[3], v23.s[0]
-; CHECK-SD-NEXT:    mov v6.s[2], v17.s[0]
-; CHECK-SD-NEXT:    mov v7.s[2], v20.s[0]
-; CHECK-SD-NEXT:    scvtf s17, x13
-; CHECK-SD-NEXT:    mov v4.s[3], v24.s[0]
-; CHECK-SD-NEXT:    mov v5.s[3], v18.s[0]
-; CHECK-SD-NEXT:    mov v6.s[3], v16.s[0]
-; CHECK-SD-NEXT:    mov v7.s[3], v17.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v32i64_v32f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str d12, [sp, #-48]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp d11, d10, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NEXT:    .cfi_offset b8, -8
-; CHECK-GI-NEXT:    .cfi_offset b9, -16
-; CHECK-GI-NEXT:    .cfi_offset b10, -24
-; CHECK-GI-NEXT:    .cfi_offset b11, -32
-; CHECK-GI-NEXT:    .cfi_offset b12, -48
-; CHECK-GI-NEXT:    mov x8, v0.d[1]
-; CHECK-GI-NEXT:    mov x10, v1.d[1]
-; CHECK-GI-NEXT:    mov x12, v4.d[1]
-; CHECK-GI-NEXT:    mov x11, v3.d[1]
-; CHECK-GI-NEXT:    ldp q30, q25, [sp, #112]
-; CHECK-GI-NEXT:    fmov x9, d0
-; CHECK-GI-NEXT:    ldp q21, q24, [sp, #48]
-; CHECK-GI-NEXT:    ldp q11, q27, [sp, #80]
-; CHECK-GI-NEXT:    scvtf s28, x8
-; CHECK-GI-NEXT:    fmov x8, d1
-; CHECK-GI-NEXT:    scvtf s16, x10
-; CHECK-GI-NEXT:    mov x10, v6.d[1]
-; CHECK-GI-NEXT:    scvtf s9, x12
-; CHECK-GI-NEXT:    fmov x12, d6
-; CHECK-GI-NEXT:    scvtf s0, x9
-; CHECK-GI-NEXT:    mov x9, v2.d[1]
-; CHECK-GI-NEXT:    scvtf s18, x11
-; CHECK-GI-NEXT:    scvtf s17, x8
-; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    fmov x11, d5
-; CHECK-GI-NEXT:    ldp q31, q26, [sp, #144]
-; CHECK-GI-NEXT:    mov x13, v21.d[1]
-; CHECK-GI-NEXT:    scvtf s10, x10
-; CHECK-GI-NEXT:    fmov x10, d21
-; CHECK-GI-NEXT:    fmov x14, d30
-; CHECK-GI-NEXT:    scvtf s1, x8
-; CHECK-GI-NEXT:    fmov x8, d3
-; CHECK-GI-NEXT:    scvtf s3, x12
-; CHECK-GI-NEXT:    mov x12, v30.d[1]
-; CHECK-GI-NEXT:    scvtf s22, x11
-; CHECK-GI-NEXT:    fmov x11, d11
-; CHECK-GI-NEXT:    scvtf s29, x9
-; CHECK-GI-NEXT:    fmov x9, d4
-; CHECK-GI-NEXT:    scvtf s4, x10
-; CHECK-GI-NEXT:    scvtf s19, x8
-; CHECK-GI-NEXT:    mov x8, v5.d[1]
-; CHECK-GI-NEXT:    mov x10, v11.d[1]
-; CHECK-GI-NEXT:    scvtf s5, x11
-; CHECK-GI-NEXT:    mov x11, v31.d[1]
-; CHECK-GI-NEXT:    scvtf s8, x13
-; CHECK-GI-NEXT:    scvtf s11, x12
-; CHECK-GI-NEXT:    fmov x12, d31
-; CHECK-GI-NEXT:    scvtf s2, x9
-; CHECK-GI-NEXT:    mov x9, v7.d[1]
-; CHECK-GI-NEXT:    scvtf s6, x14
-; CHECK-GI-NEXT:    fmov x13, d27
-; CHECK-GI-NEXT:    scvtf s20, x8
-; CHECK-GI-NEXT:    fmov x8, d7
-; CHECK-GI-NEXT:    scvtf s12, x10
-; CHECK-GI-NEXT:    scvtf s7, x12
-; CHECK-GI-NEXT:    fmov x12, d25
-; CHECK-GI-NEXT:    scvtf s30, x11
-; CHECK-GI-NEXT:    mov x10, v25.d[1]
-; CHECK-GI-NEXT:    mov x11, v26.d[1]
-; CHECK-GI-NEXT:    mov v0.s[1], v28.s[0]
-; CHECK-GI-NEXT:    scvtf s21, x9
-; CHECK-GI-NEXT:    fmov x9, d24
-; CHECK-GI-NEXT:    scvtf s23, x8
-; CHECK-GI-NEXT:    scvtf s25, x12
-; CHECK-GI-NEXT:    fmov x12, d26
-; CHECK-GI-NEXT:    mov x8, v24.d[1]
-; CHECK-GI-NEXT:    mov v1.s[1], v29.s[0]
-; CHECK-GI-NEXT:    mov v2.s[1], v9.s[0]
-; CHECK-GI-NEXT:    mov v3.s[1], v10.s[0]
-; CHECK-GI-NEXT:    scvtf s24, x9
-; CHECK-GI-NEXT:    mov x9, v27.d[1]
-; CHECK-GI-NEXT:    scvtf s27, x13
-; CHECK-GI-NEXT:    scvtf s26, x12
-; CHECK-GI-NEXT:    mov v4.s[1], v8.s[0]
-; CHECK-GI-NEXT:    mov v5.s[1], v12.s[0]
-; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v6.s[1], v11.s[0]
-; CHECK-GI-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v7.s[1], v30.s[0]
-; CHECK-GI-NEXT:    scvtf s28, x8
-; CHECK-GI-NEXT:    scvtf s29, x9
-; CHECK-GI-NEXT:    scvtf s30, x10
-; CHECK-GI-NEXT:    scvtf s31, x11
-; CHECK-GI-NEXT:    mov v0.s[2], v17.s[0]
-; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
-; CHECK-GI-NEXT:    mov v2.s[2], v22.s[0]
-; CHECK-GI-NEXT:    mov v3.s[2], v23.s[0]
-; CHECK-GI-NEXT:    mov v4.s[2], v24.s[0]
-; CHECK-GI-NEXT:    mov v5.s[2], v27.s[0]
-; CHECK-GI-NEXT:    mov v6.s[2], v25.s[0]
-; CHECK-GI-NEXT:    mov v7.s[2], v26.s[0]
-; CHECK-GI-NEXT:    mov v0.s[3], v16.s[0]
-; CHECK-GI-NEXT:    mov v1.s[3], v18.s[0]
-; CHECK-GI-NEXT:    mov v2.s[3], v20.s[0]
-; CHECK-GI-NEXT:    mov v3.s[3], v21.s[0]
-; CHECK-GI-NEXT:    mov v4.s[3], v28.s[0]
-; CHECK-GI-NEXT:    mov v5.s[3], v29.s[0]
-; CHECK-GI-NEXT:    mov v6.s[3], v30.s[0]
-; CHECK-GI-NEXT:    mov v7.s[3], v31.s[0]
-; CHECK-GI-NEXT:    ldr d12, [sp], #48 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <32 x i64> %a to <32 x float>
-  ret <32 x float> %c
-}
-
-define <32 x float> @utofp_v32i64_v32f32(<32 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v32i64_v32f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NEXT:    fmov x10, d0
-; CHECK-SD-NEXT:    mov v16.16b, v1.16b
-; CHECK-SD-NEXT:    fmov x11, d2
-; CHECK-SD-NEXT:    ldp q24, q20, [sp]
-; CHECK-SD-NEXT:    mov x9, v2.d[1]
-; CHECK-SD-NEXT:    fmov x12, d3
-; CHECK-SD-NEXT:    fmov x13, d4
-; CHECK-SD-NEXT:    ucvtf s0, x10
-; CHECK-SD-NEXT:    ldp q21, q18, [sp, #32]
-; CHECK-SD-NEXT:    ucvtf s2, x8
-; CHECK-SD-NEXT:    ucvtf s1, x11
-; CHECK-SD-NEXT:    mov x10, v4.d[1]
-; CHECK-SD-NEXT:    fmov x11, d16
-; CHECK-SD-NEXT:    ldp q19, q17, [sp, #96]
-; CHECK-SD-NEXT:    ucvtf s22, x9
-; CHECK-SD-NEXT:    mov x8, v3.d[1]
-; CHECK-SD-NEXT:    ucvtf s4, x12
-; CHECK-SD-NEXT:    mov x12, v24.d[1]
-; CHECK-SD-NEXT:    mov x9, v16.d[1]
-; CHECK-SD-NEXT:    ucvtf s3, x11
-; CHECK-SD-NEXT:    ldp q23, q16, [sp, #64]
-; CHECK-SD-NEXT:    mov v0.s[1], v2.s[0]
-; CHECK-SD-NEXT:    ucvtf s25, x10
-; CHECK-SD-NEXT:    fmov x10, d6
-; CHECK-SD-NEXT:    mov v1.s[1], v22.s[0]
-; CHECK-SD-NEXT:    mov x11, v6.d[1]
-; CHECK-SD-NEXT:    ucvtf s2, x13
-; CHECK-SD-NEXT:    mov x13, v21.d[1]
-; CHECK-SD-NEXT:    fmov x14, d19
-; CHECK-SD-NEXT:    ucvtf s22, x9
-; CHECK-SD-NEXT:    mov x9, v5.d[1]
-; CHECK-SD-NEXT:    fmov x15, d17
-; CHECK-SD-NEXT:    mov v0.s[2], v3.s[0]
-; CHECK-SD-NEXT:    ucvtf s3, x10
-; CHECK-SD-NEXT:    fmov x10, d24
-; CHECK-SD-NEXT:    mov v1.s[2], v4.s[0]
-; CHECK-SD-NEXT:    ucvtf s24, x12
-; CHECK-SD-NEXT:    ucvtf s6, x11
-; CHECK-SD-NEXT:    fmov x11, d5
-; CHECK-SD-NEXT:    fmov x12, d7
-; CHECK-SD-NEXT:    mov v2.s[1], v25.s[0]
-; CHECK-SD-NEXT:    ucvtf s4, x10
-; CHECK-SD-NEXT:    fmov x10, d21
-; CHECK-SD-NEXT:    ucvtf s21, x8
-; CHECK-SD-NEXT:    mov x8, v23.d[1]
-; CHECK-SD-NEXT:    ucvtf s25, x13
-; CHECK-SD-NEXT:    mov x13, v19.d[1]
-; CHECK-SD-NEXT:    ucvtf s26, x11
-; CHECK-SD-NEXT:    mov x11, v20.d[1]
-; CHECK-SD-NEXT:    mov v3.s[1], v6.s[0]
-; CHECK-SD-NEXT:    ucvtf s5, x10
-; CHECK-SD-NEXT:    mov x10, v7.d[1]
-; CHECK-SD-NEXT:    ucvtf s7, x14
-; CHECK-SD-NEXT:    mov v4.s[1], v24.s[0]
-; CHECK-SD-NEXT:    ucvtf s24, x12
-; CHECK-SD-NEXT:    fmov x12, d20
-; CHECK-SD-NEXT:    ucvtf s20, x8
-; CHECK-SD-NEXT:    fmov x8, d23
-; CHECK-SD-NEXT:    ucvtf s19, x13
-; CHECK-SD-NEXT:    fmov x13, d18
-; CHECK-SD-NEXT:    fmov x14, d16
-; CHECK-SD-NEXT:    mov v2.s[2], v26.s[0]
-; CHECK-SD-NEXT:    mov v5.s[1], v25.s[0]
-; CHECK-SD-NEXT:    ucvtf s23, x10
-; CHECK-SD-NEXT:    mov v0.s[3], v22.s[0]
-; CHECK-SD-NEXT:    ucvtf s6, x8
-; CHECK-SD-NEXT:    mov x8, v18.d[1]
-; CHECK-SD-NEXT:    ucvtf s18, x12
-; CHECK-SD-NEXT:    mov x12, v16.d[1]
-; CHECK-SD-NEXT:    ucvtf s16, x13
-; CHECK-SD-NEXT:    mov x13, v17.d[1]
-; CHECK-SD-NEXT:    ucvtf s17, x14
-; CHECK-SD-NEXT:    mov v7.s[1], v19.s[0]
-; CHECK-SD-NEXT:    ucvtf s19, x9
-; CHECK-SD-NEXT:    mov v3.s[2], v24.s[0]
-; CHECK-SD-NEXT:    ucvtf s24, x11
-; CHECK-SD-NEXT:    mov v1.s[3], v21.s[0]
-; CHECK-SD-NEXT:    mov v6.s[1], v20.s[0]
-; CHECK-SD-NEXT:    ucvtf s20, x15
-; CHECK-SD-NEXT:    mov v4.s[2], v18.s[0]
-; CHECK-SD-NEXT:    ucvtf s18, x8
-; CHECK-SD-NEXT:    mov v5.s[2], v16.s[0]
-; CHECK-SD-NEXT:    ucvtf s16, x12
-; CHECK-SD-NEXT:    mov v2.s[3], v19.s[0]
-; CHECK-SD-NEXT:    mov v3.s[3], v23.s[0]
-; CHECK-SD-NEXT:    mov v6.s[2], v17.s[0]
-; CHECK-SD-NEXT:    mov v7.s[2], v20.s[0]
-; CHECK-SD-NEXT:    ucvtf s17, x13
-; CHECK-SD-NEXT:    mov v4.s[3], v24.s[0]
-; CHECK-SD-NEXT:    mov v5.s[3], v18.s[0]
-; CHECK-SD-NEXT:    mov v6.s[3], v16.s[0]
-; CHECK-SD-NEXT:    mov v7.s[3], v17.s[0]
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v32i64_v32f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    str d12, [sp, #-48]! // 8-byte Folded Spill
-; CHECK-GI-NEXT:    stp d11, d10, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    stp d9, d8, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NEXT:    .cfi_offset b8, -8
-; CHECK-GI-NEXT:    .cfi_offset b9, -16
-; CHECK-GI-NEXT:    .cfi_offset b10, -24
-; CHECK-GI-NEXT:    .cfi_offset b11, -32
-; CHECK-GI-NEXT:    .cfi_offset b12, -48
-; CHECK-GI-NEXT:    mov x8, v0.d[1]
-; CHECK-GI-NEXT:    mov x10, v1.d[1]
-; CHECK-GI-NEXT:    mov x12, v4.d[1]
-; CHECK-GI-NEXT:    mov x11, v3.d[1]
-; CHECK-GI-NEXT:    ldp q30, q25, [sp, #112]
-; CHECK-GI-NEXT:    fmov x9, d0
-; CHECK-GI-NEXT:    ldp q21, q24, [sp, #48]
-; CHECK-GI-NEXT:    ldp q11, q27, [sp, #80]
-; CHECK-GI-NEXT:    ucvtf s28, x8
-; CHECK-GI-NEXT:    fmov x8, d1
-; CHECK-GI-NEXT:    ucvtf s16, x10
-; CHECK-GI-NEXT:    mov x10, v6.d[1]
-; CHECK-GI-NEXT:    ucvtf s9, x12
-; CHECK-GI-NEXT:    fmov x12, d6
-; CHECK-GI-NEXT:    ucvtf s0, x9
-; CHECK-GI-NEXT:    mov x9, v2.d[1]
-; CHECK-GI-NEXT:    ucvtf s18, x11
-; CHECK-GI-NEXT:    ucvtf s17, x8
-; CHECK-GI-NEXT:    fmov x8, d2
-; CHECK-GI-NEXT:    fmov x11, d5
-; CHECK-GI-NEXT:    ldp q31, q26, [sp, #144]
-; CHECK-GI-NEXT:    mov x13, v21.d[1]
-; CHECK-GI-NEXT:    ucvtf s10, x10
-; CHECK-GI-NEXT:    fmov x10, d21
-; CHECK-GI-NEXT:    fmov x14, d30
-; CHECK-GI-NEXT:    ucvtf s1, x8
-; CHECK-GI-NEXT:    fmov x8, d3
-; CHECK-GI-NEXT:    ucvtf s3, x12
-; CHECK-GI-NEXT:    mov x12, v30.d[1]
-; CHECK-GI-NEXT:    ucvtf s22, x11
-; CHECK-GI-NEXT:    fmov x11, d11
-; CHECK-GI-NEXT:    ucvtf s29, x9
-; CHECK-GI-NEXT:    fmov x9, d4
-; CHECK-GI-NEXT:    ucvtf s4, x10
-; CHECK-GI-NEXT:    ucvtf s19, x8
-; CHECK-GI-NEXT:    mov x8, v5.d[1]
-; CHECK-GI-NEXT:    mov x10, v11.d[1]
-; CHECK-GI-NEXT:    ucvtf s5, x11
-; CHECK-GI-NEXT:    mov x11, v31.d[1]
-; CHECK-GI-NEXT:    ucvtf s8, x13
-; CHECK-GI-NEXT:    ucvtf s11, x12
-; CHECK-GI-NEXT:    fmov x12, d31
-; CHECK-GI-NEXT:    ucvtf s2, x9
-; CHECK-GI-NEXT:    mov x9, v7.d[1]
-; CHECK-GI-NEXT:    ucvtf s6, x14
-; CHECK-GI-NEXT:    fmov x13, d27
-; CHECK-GI-NEXT:    ucvtf s20, x8
-; CHECK-GI-NEXT:    fmov x8, d7
-; CHECK-GI-NEXT:    ucvtf s12, x10
-; CHECK-GI-NEXT:    ucvtf s7, x12
-; CHECK-GI-NEXT:    fmov x12, d25
-; CHECK-GI-NEXT:    ucvtf s30, x11
-; CHECK-GI-NEXT:    mov x10, v25.d[1]
-; CHECK-GI-NEXT:    mov x11, v26.d[1]
-; CHECK-GI-NEXT:    mov v0.s[1], v28.s[0]
-; CHECK-GI-NEXT:    ucvtf s21, x9
-; CHECK-GI-NEXT:    fmov x9, d24
-; CHECK-GI-NEXT:    ucvtf s23, x8
-; CHECK-GI-NEXT:    ucvtf s25, x12
-; CHECK-GI-NEXT:    fmov x12, d26
-; CHECK-GI-NEXT:    mov x8, v24.d[1]
-; CHECK-GI-NEXT:    mov v1.s[1], v29.s[0]
-; CHECK-GI-NEXT:    mov v2.s[1], v9.s[0]
-; CHECK-GI-NEXT:    mov v3.s[1], v10.s[0]
-; CHECK-GI-NEXT:    ucvtf s24, x9
-; CHECK-GI-NEXT:    mov x9, v27.d[1]
-; CHECK-GI-NEXT:    ucvtf s27, x13
-; CHECK-GI-NEXT:    ucvtf s26, x12
-; CHECK-GI-NEXT:    mov v4.s[1], v8.s[0]
-; CHECK-GI-NEXT:    mov v5.s[1], v12.s[0]
-; CHECK-GI-NEXT:    ldp d9, d8, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v6.s[1], v11.s[0]
-; CHECK-GI-NEXT:    ldp d11, d10, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT:    mov v7.s[1], v30.s[0]
-; CHECK-GI-NEXT:    ucvtf s28, x8
-; CHECK-GI-NEXT:    ucvtf s29, x9
-; CHECK-GI-NEXT:    ucvtf s30, x10
-; CHECK-GI-NEXT:    ucvtf s31, x11
-; CHECK-GI-NEXT:    mov v0.s[2], v17.s[0]
-; CHECK-GI-NEXT:    mov v1.s[2], v19.s[0]
-; CHECK-GI-NEXT:    mov v2.s[2], v22.s[0]
-; CHECK-GI-NEXT:    mov v3.s[2], v23.s[0]
-; CHECK-GI-NEXT:    mov v4.s[2], v24.s[0]
-; CHECK-GI-NEXT:    mov v5.s[2], v27.s[0]
-; CHECK-GI-NEXT:    mov v6.s[2], v25.s[0]
-; CHECK-GI-NEXT:    mov v7.s[2], v26.s[0]
-; CHECK-GI-NEXT:    mov v0.s[3], v16.s[0]
-; CHECK-GI-NEXT:    mov v1.s[3], v18.s[0]
-; CHECK-GI-NEXT:    mov v2.s[3], v20.s[0]
-; CHECK-GI-NEXT:    mov v3.s[3], v21.s[0]
-; CHECK-GI-NEXT:    mov v4.s[3], v28.s[0]
-; CHECK-GI-NEXT:    mov v5.s[3], v29.s[0]
-; CHECK-GI-NEXT:    mov v6.s[3], v30.s[0]
-; CHECK-GI-NEXT:    mov v7.s[3], v31.s[0]
-; CHECK-GI-NEXT:    ldr d12, [sp], #48 // 8-byte Folded Reload
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <32 x i64> %a to <32 x float>
-  ret <32 x float> %c
-}
-
 define <2 x float> @stofp_v2i32_v2f32(<2 x i32> %a) {
 ; CHECK-LABEL: stofp_v2i32_v2f32:
 ; CHECK:       // %bb.0: // %entry
@@ -5300,66 +4524,6 @@ entry:
   ret <8 x float> %c
 }
 
-define <16 x float> @stofp_v16i32_v16f32(<16 x i32> %a) {
-; CHECK-LABEL: stofp_v16i32_v16f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-NEXT:    scvtf v3.4s, v3.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = sitofp <16 x i32> %a to <16 x float>
-  ret <16 x float> %c
-}
-
-define <16 x float> @utofp_v16i32_v16f32(<16 x i32> %a) {
-; CHECK-LABEL: utofp_v16i32_v16f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-NEXT:    ucvtf v3.4s, v3.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = uitofp <16 x i32> %a to <16 x float>
-  ret <16 x float> %c
-}
-
-define <32 x float> @stofp_v32i32_v32f32(<32 x i32> %a) {
-; CHECK-LABEL: stofp_v32i32_v32f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-NEXT:    scvtf v3.4s, v3.4s
-; CHECK-NEXT:    scvtf v4.4s, v4.4s
-; CHECK-NEXT:    scvtf v5.4s, v5.4s
-; CHECK-NEXT:    scvtf v6.4s, v6.4s
-; CHECK-NEXT:    scvtf v7.4s, v7.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = sitofp <32 x i32> %a to <32 x float>
-  ret <32 x float> %c
-}
-
-define <32 x float> @utofp_v32i32_v32f32(<32 x i32> %a) {
-; CHECK-LABEL: utofp_v32i32_v32f32:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-NEXT:    ucvtf v3.4s, v3.4s
-; CHECK-NEXT:    ucvtf v4.4s, v4.4s
-; CHECK-NEXT:    ucvtf v5.4s, v5.4s
-; CHECK-NEXT:    ucvtf v6.4s, v6.4s
-; CHECK-NEXT:    ucvtf v7.4s, v7.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = uitofp <32 x i32> %a to <32 x float>
-  ret <32 x float> %c
-}
-
 define <2 x float> @stofp_v2i16_v2f32(<2 x i16> %a) {
 ; CHECK-LABEL: stofp_v2i16_v2f32:
 ; CHECK:       // %bb.0: // %entry
@@ -5429,195 +4593,185 @@ entry:
 }
 
 define <8 x float> @stofp_v8i16_v8f32(<8 x i16> %a) {
-; CHECK-SD-LABEL: stofp_v8i16_v8f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-SD-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v8i16_v8f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v2.4s, v0.8h, #0
-; CHECK-GI-NEXT:    scvtf v0.4s, v1.4s
-; CHECK-GI-NEXT:    scvtf v1.4s, v2.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i16_v8f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v8i16_v8f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll2 v1.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i16_v8f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i16_v8f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i16> %a to <8 x float>
   ret <8 x float> %c
 }
 
 define <8 x float> @utofp_v8i16_v8f32(<8 x i16> %a) {
-; CHECK-SD-LABEL: utofp_v8i16_v8f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
-; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-SD-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v8i16_v8f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ucvtf v0.4s, v1.4s
-; CHECK-GI-NEXT:    ucvtf v1.4s, v2.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i16_v8f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v8i16_v8f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll2 v1.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i16_v8f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i16_v8f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i16> %a to <8 x float>
   ret <8 x float> %c
 }
 
 define <16 x float> @stofp_v16i16_v16f32(<16 x i16> %a) {
-; CHECK-SD-LABEL: stofp_v16i16_v16f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-SD-NEXT:    sshll2 v3.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll2 v4.4s, v1.8h, #0
-; CHECK-SD-NEXT:    sshll v5.4s, v1.4h, #0
-; CHECK-SD-NEXT:    scvtf v0.4s, v2.4s
-; CHECK-SD-NEXT:    scvtf v1.4s, v3.4s
-; CHECK-SD-NEXT:    scvtf v3.4s, v4.4s
-; CHECK-SD-NEXT:    scvtf v2.4s, v5.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v16i16_v16f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v3.4s, v0.8h, #0
-; CHECK-GI-NEXT:    sshll v4.4s, v1.4h, #0
-; CHECK-GI-NEXT:    sshll2 v5.4s, v1.8h, #0
-; CHECK-GI-NEXT:    scvtf v0.4s, v2.4s
-; CHECK-GI-NEXT:    scvtf v1.4s, v3.4s
-; CHECK-GI-NEXT:    scvtf v2.4s, v4.4s
-; CHECK-GI-NEXT:    scvtf v3.4s, v5.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i16_v16f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v4.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v5.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v16i16_v16f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v3.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v4.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT:    sshll v5.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    scvtf v1.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    scvtf v3.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    scvtf v2.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i16_v16f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v4.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v5.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i16_v16f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v3.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v4.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v5.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    scvtf v1.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    scvtf v2.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    scvtf v3.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i16> %a to <16 x float>
   ret <16 x float> %c
 }
 
 define <16 x float> @utofp_v16i16_v16f32(<16 x i16> %a) {
-; CHECK-SD-LABEL: utofp_v16i16_v16f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ushll2 v3.4s, v0.8h, #0
-; CHECK-SD-NEXT:    ushll2 v4.4s, v1.8h, #0
-; CHECK-SD-NEXT:    ushll v5.4s, v1.4h, #0
-; CHECK-SD-NEXT:    ucvtf v0.4s, v2.4s
-; CHECK-SD-NEXT:    ucvtf v1.4s, v3.4s
-; CHECK-SD-NEXT:    ucvtf v3.4s, v4.4s
-; CHECK-SD-NEXT:    ucvtf v2.4s, v5.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v16i16_v16f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v3.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll v4.4s, v1.4h, #0
-; CHECK-GI-NEXT:    ushll2 v5.4s, v1.8h, #0
-; CHECK-GI-NEXT:    ucvtf v0.4s, v2.4s
-; CHECK-GI-NEXT:    ucvtf v1.4s, v3.4s
-; CHECK-GI-NEXT:    ucvtf v2.4s, v4.4s
-; CHECK-GI-NEXT:    ucvtf v3.4s, v5.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i16_v16f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v4.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v5.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v16i16_v16f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v3.4s, v0.8h, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v4.4s, v1.8h, #0
+; CHECK-FP16-SD-NEXT:    ushll v5.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v1.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v3.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v2.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i16_v16f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v3.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v4.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v5.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i16_v16f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v3.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v4.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v5.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v1.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v2.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v3.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i16> %a to <16 x float>
   ret <16 x float> %c
 }
 
-define <32 x float> @stofp_v32i16_v32f32(<32 x i16> %a) {
-; CHECK-SD-LABEL: stofp_v32i16_v32f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    sshll2 v5.4s, v1.8h, #0
-; CHECK-SD-NEXT:    sshll v6.4s, v1.4h, #0
-; CHECK-SD-NEXT:    sshll v7.4s, v2.4h, #0
-; CHECK-SD-NEXT:    sshll2 v16.4s, v2.8h, #0
-; CHECK-SD-NEXT:    sshll2 v17.4s, v3.8h, #0
-; CHECK-SD-NEXT:    sshll v18.4s, v3.4h, #0
-; CHECK-SD-NEXT:    scvtf v1.4s, v4.4s
-; CHECK-SD-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    scvtf v3.4s, v5.4s
-; CHECK-SD-NEXT:    scvtf v2.4s, v6.4s
-; CHECK-SD-NEXT:    scvtf v4.4s, v7.4s
-; CHECK-SD-NEXT:    scvtf v5.4s, v16.4s
-; CHECK-SD-NEXT:    scvtf v7.4s, v17.4s
-; CHECK-SD-NEXT:    scvtf v6.4s, v18.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v32i16_v32f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v4.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v5.4s, v0.8h, #0
-; CHECK-GI-NEXT:    sshll v6.4s, v1.4h, #0
-; CHECK-GI-NEXT:    sshll2 v7.4s, v1.8h, #0
-; CHECK-GI-NEXT:    sshll v16.4s, v2.4h, #0
-; CHECK-GI-NEXT:    sshll2 v17.4s, v2.8h, #0
-; CHECK-GI-NEXT:    sshll v18.4s, v3.4h, #0
-; CHECK-GI-NEXT:    sshll2 v19.4s, v3.8h, #0
-; CHECK-GI-NEXT:    scvtf v0.4s, v4.4s
-; CHECK-GI-NEXT:    scvtf v1.4s, v5.4s
-; CHECK-GI-NEXT:    scvtf v2.4s, v6.4s
-; CHECK-GI-NEXT:    scvtf v3.4s, v7.4s
-; CHECK-GI-NEXT:    scvtf v4.4s, v16.4s
-; CHECK-GI-NEXT:    scvtf v5.4s, v17.4s
-; CHECK-GI-NEXT:    scvtf v6.4s, v18.4s
-; CHECK-GI-NEXT:    scvtf v7.4s, v19.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <32 x i16> %a to <32 x float>
-  ret <32 x float> %c
-}
-
-define <32 x float> @utofp_v32i16_v32f32(<32 x i16> %a) {
-; CHECK-SD-LABEL: utofp_v32i16_v32f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ushll2 v4.4s, v0.8h, #0
-; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ushll2 v5.4s, v1.8h, #0
-; CHECK-SD-NEXT:    ushll v6.4s, v1.4h, #0
-; CHECK-SD-NEXT:    ushll v7.4s, v2.4h, #0
-; CHECK-SD-NEXT:    ushll2 v16.4s, v2.8h, #0
-; CHECK-SD-NEXT:    ushll2 v17.4s, v3.8h, #0
-; CHECK-SD-NEXT:    ushll v18.4s, v3.4h, #0
-; CHECK-SD-NEXT:    ucvtf v1.4s, v4.4s
-; CHECK-SD-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    ucvtf v3.4s, v5.4s
-; CHECK-SD-NEXT:    ucvtf v2.4s, v6.4s
-; CHECK-SD-NEXT:    ucvtf v4.4s, v7.4s
-; CHECK-SD-NEXT:    ucvtf v5.4s, v16.4s
-; CHECK-SD-NEXT:    ucvtf v7.4s, v17.4s
-; CHECK-SD-NEXT:    ucvtf v6.4s, v18.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v32i16_v32f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v4.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v5.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll v6.4s, v1.4h, #0
-; CHECK-GI-NEXT:    ushll2 v7.4s, v1.8h, #0
-; CHECK-GI-NEXT:    ushll v16.4s, v2.4h, #0
-; CHECK-GI-NEXT:    ushll2 v17.4s, v2.8h, #0
-; CHECK-GI-NEXT:    ushll v18.4s, v3.4h, #0
-; CHECK-GI-NEXT:    ushll2 v19.4s, v3.8h, #0
-; CHECK-GI-NEXT:    ucvtf v0.4s, v4.4s
-; CHECK-GI-NEXT:    ucvtf v1.4s, v5.4s
-; CHECK-GI-NEXT:    ucvtf v2.4s, v6.4s
-; CHECK-GI-NEXT:    ucvtf v3.4s, v7.4s
-; CHECK-GI-NEXT:    ucvtf v4.4s, v16.4s
-; CHECK-GI-NEXT:    ucvtf v5.4s, v17.4s
-; CHECK-GI-NEXT:    ucvtf v6.4s, v18.4s
-; CHECK-GI-NEXT:    ucvtf v7.4s, v19.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <32 x i16> %a to <32 x float>
-  ret <32 x float> %c
-}
-
 define <2 x float> @stofp_v2i8_v2f32(<2 x i8> %a) {
 ; CHECK-LABEL: stofp_v2i8_v2f32:
 ; CHECK:       // %bb.0: // %entry
@@ -5643,1014 +4797,1030 @@ entry:
 }
 
 define <3 x float> @stofp_v3i8_v3f32(<3 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v3i8_v3f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    mov v0.h[1], w1
-; CHECK-SD-NEXT:    mov v0.h[2], w2
-; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v3i8_v3f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov s0, w0
-; CHECK-GI-NEXT:    mov v0.h[1], w1
-; CHECK-GI-NEXT:    mov v0.h[2], w2
-; CHECK-GI-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-GI-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[2]
-; CHECK-GI-NEXT:    mov v1.h[1], v0.h[3]
-; CHECK-GI-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v3i8_v3f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v3i8_v3f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], w1
+; CHECK-FP16-SD-NEXT:    mov v0.h[2], w2
+; CHECK-FP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i8_v3f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w0
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-GI-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-GI-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-GI-NEXT:    mov v1.h[0], v0.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov v1.h[1], v0.h[3]
+; CHECK-NOFP16-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v3i8_v3f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov s0, w0
+; CHECK-FP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-FP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-FP16-GI-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-GI-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-GI-NEXT:    mov v1.h[0], v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov v1.h[1], v0.h[3]
+; CHECK-FP16-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <3 x i8> %a to <3 x float>
   ret <3 x float> %c
 }
 
 define <3 x float> @utofp_v3i8_v3f32(<3 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v3i8_v3f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    fmov s0, w0
-; CHECK-SD-NEXT:    mov v0.h[1], w1
-; CHECK-SD-NEXT:    mov v0.h[2], w2
-; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v3i8_v3f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    fmov s0, w0
-; CHECK-GI-NEXT:    movi d1, #0xff00ff00ff00ff
-; CHECK-GI-NEXT:    mov v0.h[1], w1
-; CHECK-GI-NEXT:    mov v0.h[2], w2
-; CHECK-GI-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NEXT:    mov v1.h[0], v0.h[2]
-; CHECK-GI-NEXT:    mov v1.h[1], v0.h[3]
-; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-GI-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v3i8_v3f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v3i8_v3f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], w1
+; CHECK-FP16-SD-NEXT:    mov v0.h[2], w2
+; CHECK-FP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i8_v3f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w0
+; CHECK-NOFP16-GI-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-GI-NEXT:    mov v1.h[0], v0.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov v1.h[1], v0.h[3]
+; CHECK-NOFP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v3i8_v3f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov s0, w0
+; CHECK-FP16-GI-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-FP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-FP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-FP16-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-GI-NEXT:    mov v1.h[0], v0.h[2]
+; CHECK-FP16-GI-NEXT:    mov v1.h[1], v0.h[3]
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <3 x i8> %a to <3 x float>
   ret <3 x float> %c
 }
 
 define <4 x float> @stofp_v4i8_v4f32(<4 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v4i8_v4f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v4i8_v4f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT:    shl v0.4s, v0.4s, #24
-; CHECK-GI-NEXT:    sshr v0.4s, v0.4s, #24
-; CHECK-GI-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v4i8_v4f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v4i8_v4f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v4i8_v4f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-NOFP16-GI-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v4i8_v4f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-FP16-GI-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-FP16-GI-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <4 x i8> %a to <4 x float>
   ret <4 x float> %c
 }
 
 define <4 x float> @utofp_v4i8_v4f32(<4 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v4i8_v4f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-SD-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v4i8_v4f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    movi v1.2d, #0x0000ff000000ff
-; CHECK-GI-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT:    and v0.16b, v0.16b, v1.16b
-; CHECK-GI-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v4i8_v4f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v4i8_v4f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v4i8_v4f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-NOFP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v4i8_v4f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-FP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <4 x i8> %a to <4 x float>
   ret <4 x float> %c
 }
 
 define <8 x float> @stofp_v8i8_v8f32(<8 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v8i8_v8f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    zip1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    shl v1.4h, v1.4h, #8
-; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshr v1.4h, v1.4h, #8
-; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-SD-NEXT:    scvtf v0.4s, v1.4s
-; CHECK-SD-NEXT:    scvtf v1.4s, v2.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v8i8_v8f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-GI-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v2.4s, v0.8h, #0
-; CHECK-GI-NEXT:    scvtf v0.4s, v1.4s
-; CHECK-GI-NEXT:    scvtf v1.4s, v2.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i8_v8f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    zip1 v1.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v8i8_v8f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    zip1 v1.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i8_v8f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i8_v8f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i8> %a to <8 x float>
   ret <8 x float> %c
 }
 
 define <8 x float> @utofp_v8i8_v8f32(<8 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v8i8_v8f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    zip1 v1.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
-; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-SD-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-SD-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ucvtf v0.4s, v1.4s
-; CHECK-SD-NEXT:    ucvtf v1.4s, v2.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v8i8_v8f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-GI-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ucvtf v0.4s, v1.4s
-; CHECK-GI-NEXT:    ucvtf v1.4s, v2.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i8_v8f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    zip1 v1.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v8i8_v8f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    zip1 v1.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i8_v8f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i8_v8f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i8> %a to <8 x float>
   ret <8 x float> %c
 }
 
 define <16 x float> @stofp_v16i8_v16f32(<16 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v16i8_v16f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    zip1 v2.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    zip1 v3.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT:    shl v2.4h, v2.4h, #8
-; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
-; CHECK-SD-NEXT:    shl v3.4h, v3.4h, #8
-; CHECK-SD-NEXT:    shl v1.4h, v1.4h, #8
-; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT:    sshr v3.4h, v3.4h, #8
-; CHECK-SD-NEXT:    sshr v1.4h, v1.4h, #8
-; CHECK-SD-NEXT:    sshll v4.4s, v0.4h, #0
-; CHECK-SD-NEXT:    scvtf v0.4s, v2.4s
-; CHECK-SD-NEXT:    sshll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT:    sshll v5.4s, v1.4h, #0
-; CHECK-SD-NEXT:    scvtf v1.4s, v4.4s
-; CHECK-SD-NEXT:    scvtf v2.4s, v3.4s
-; CHECK-SD-NEXT:    scvtf v3.4s, v5.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v16i8_v16f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v1.8h, v0.8b, #0
-; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT:    sshll v2.4s, v1.4h, #0
-; CHECK-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    sshll v3.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-GI-NEXT:    scvtf v0.4s, v2.4s
-; CHECK-GI-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-GI-NEXT:    scvtf v2.4s, v3.4s
-; CHECK-GI-NEXT:    scvtf v3.4s, v4.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i8_v16f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    zip1 v2.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip1 v3.8b, v1.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    shl v2.4h, v2.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v2.4h, v2.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v3.4h, v3.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshr v3.4h, v3.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshll v4.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    sshll v3.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v5.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v16i8_v16f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    zip1 v2.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip1 v3.8b, v1.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    shl v2.4h, v2.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v2.4h, v2.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v3.4h, v3.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT:    sshr v3.4h, v3.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-FP16-SD-NEXT:    sshll v4.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    sshll v3.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll v5.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    scvtf v1.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    scvtf v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    scvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i8_v16f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v1.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i8_v16f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v1.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v3.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    scvtf v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    scvtf v3.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i8> %a to <16 x float>
   ret <16 x float> %c
 }
 
 define <16 x float> @utofp_v16i8_v16f32(<16 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v16i8_v16f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    zip1 v2.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    zip1 v3.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
-; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-SD-NEXT:    ushll v2.4s, v2.4h, #0
-; CHECK-SD-NEXT:    bic v3.4h, #255, lsl #8
-; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
-; CHECK-SD-NEXT:    ushll v4.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ucvtf v0.4s, v2.4s
-; CHECK-SD-NEXT:    ushll v3.4s, v3.4h, #0
-; CHECK-SD-NEXT:    ushll v5.4s, v1.4h, #0
-; CHECK-SD-NEXT:    ucvtf v1.4s, v4.4s
-; CHECK-SD-NEXT:    ucvtf v2.4s, v3.4s
-; CHECK-SD-NEXT:    ucvtf v3.4s, v5.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v16i8_v16f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v1.8h, v0.8b, #0
-; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT:    ushll v2.4s, v1.4h, #0
-; CHECK-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT:    ushll v3.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v4.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ucvtf v0.4s, v2.4s
-; CHECK-GI-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-GI-NEXT:    ucvtf v2.4s, v3.4s
-; CHECK-GI-NEXT:    ucvtf v3.4s, v4.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i8_v16f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    zip1 v2.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip1 v3.8b, v1.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    bic v2.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT:    bic v3.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    ushll v4.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v5.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v16i8_v16f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    zip1 v2.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip1 v3.8b, v1.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    bic v2.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT:    bic v3.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    ushll v4.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll v5.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v1.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v3.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i8_v16f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v4.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i8_v16f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v3.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v4.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v2.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v3.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i8> %a to <16 x float>
   ret <16 x float> %c
 }
 
 define <32 x float> @stofp_v32i8_v32f32(<32 x i8> %a) {
-; CHECK-SD-LABEL: stofp_v32i8_v32f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    zip1 v4.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    shl v4.4h, v4.4h, #8
-; CHECK-SD-NEXT:    zip1 v5.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT:    zip1 v6.8b, v2.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v2.8b, v2.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT:    zip1 v7.8b, v3.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v3.8b, v3.8b, v0.8b
-; CHECK-SD-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshr v4.4h, v4.4h, #8
-; CHECK-SD-NEXT:    shl v5.4h, v5.4h, #8
-; CHECK-SD-NEXT:    shl v6.4h, v6.4h, #8
-; CHECK-SD-NEXT:    shl v2.4h, v2.4h, #8
-; CHECK-SD-NEXT:    shl v1.4h, v1.4h, #8
-; CHECK-SD-NEXT:    shl v7.4h, v7.4h, #8
-; CHECK-SD-NEXT:    shl v3.4h, v3.4h, #8
-; CHECK-SD-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-SD-NEXT:    sshll v4.4s, v4.4h, #0
-; CHECK-SD-NEXT:    sshr v5.4h, v5.4h, #8
-; CHECK-SD-NEXT:    sshr v6.4h, v6.4h, #8
-; CHECK-SD-NEXT:    sshr v2.4h, v2.4h, #8
-; CHECK-SD-NEXT:    sshr v1.4h, v1.4h, #8
-; CHECK-SD-NEXT:    sshr v7.4h, v7.4h, #8
-; CHECK-SD-NEXT:    sshr v3.4h, v3.4h, #8
-; CHECK-SD-NEXT:    sshll v16.4s, v0.4h, #0
-; CHECK-SD-NEXT:    scvtf v0.4s, v4.4s
-; CHECK-SD-NEXT:    sshll v5.4s, v5.4h, #0
-; CHECK-SD-NEXT:    sshll v6.4s, v6.4h, #0
-; CHECK-SD-NEXT:    sshll v17.4s, v2.4h, #0
-; CHECK-SD-NEXT:    sshll v18.4s, v1.4h, #0
-; CHECK-SD-NEXT:    sshll v7.4s, v7.4h, #0
-; CHECK-SD-NEXT:    sshll v19.4s, v3.4h, #0
-; CHECK-SD-NEXT:    scvtf v1.4s, v16.4s
-; CHECK-SD-NEXT:    scvtf v4.4s, v5.4s
-; CHECK-SD-NEXT:    scvtf v2.4s, v6.4s
-; CHECK-SD-NEXT:    scvtf v3.4s, v17.4s
-; CHECK-SD-NEXT:    scvtf v5.4s, v18.4s
-; CHECK-SD-NEXT:    scvtf v6.4s, v7.4s
-; CHECK-SD-NEXT:    scvtf v7.4s, v19.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v32i8_v32f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    sshll v2.8h, v0.8b, #0
-; CHECK-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT:    sshll v3.8h, v1.8b, #0
-; CHECK-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-GI-NEXT:    sshll v4.4s, v2.4h, #0
-; CHECK-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT:    sshll v5.4s, v0.4h, #0
-; CHECK-GI-NEXT:    sshll2 v6.4s, v0.8h, #0
-; CHECK-GI-NEXT:    sshll v7.4s, v3.4h, #0
-; CHECK-GI-NEXT:    sshll2 v16.4s, v3.8h, #0
-; CHECK-GI-NEXT:    sshll v17.4s, v1.4h, #0
-; CHECK-GI-NEXT:    sshll2 v18.4s, v1.8h, #0
-; CHECK-GI-NEXT:    scvtf v0.4s, v4.4s
-; CHECK-GI-NEXT:    scvtf v1.4s, v2.4s
-; CHECK-GI-NEXT:    scvtf v2.4s, v5.4s
-; CHECK-GI-NEXT:    scvtf v3.4s, v6.4s
-; CHECK-GI-NEXT:    scvtf v4.4s, v7.4s
-; CHECK-GI-NEXT:    scvtf v5.4s, v16.4s
-; CHECK-GI-NEXT:    scvtf v6.4s, v17.4s
-; CHECK-GI-NEXT:    scvtf v7.4s, v18.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v32i8_v32f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NOFP16-SD-NEXT:    zip1 v4.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    shl v4.4h, v4.4h, #8
+; CHECK-NOFP16-SD-NEXT:    zip1 v5.8b, v1.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip1 v6.8b, v2.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v2.8b, v2.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip1 v7.8b, v3.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v3.8b, v3.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v4.4h, v4.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v5.4h, v5.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v6.4h, v6.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v2.4h, v2.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v7.4h, v7.4h, #8
+; CHECK-NOFP16-SD-NEXT:    shl v3.4h, v3.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshll v4.4s, v4.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshr v5.4h, v5.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v6.4h, v6.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v2.4h, v2.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v7.4h, v7.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v3.4h, v3.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshll v16.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    sshll v5.4s, v5.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v6.4s, v6.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v17.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v18.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v7.4s, v7.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v19.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v4.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v5.4s, v18.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v6.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v7.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v32i8_v32f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-FP16-SD-NEXT:    zip1 v4.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    shl v4.4h, v4.4h, #8
+; CHECK-FP16-SD-NEXT:    zip1 v5.8b, v1.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip1 v6.8b, v2.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v2.8b, v2.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip1 v7.8b, v3.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v3.8b, v3.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v4.4h, v4.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v5.4h, v5.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v6.4h, v6.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v2.4h, v2.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v1.4h, v1.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v7.4h, v7.4h, #8
+; CHECK-FP16-SD-NEXT:    shl v3.4h, v3.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshll v4.4s, v4.4h, #0
+; CHECK-FP16-SD-NEXT:    sshr v5.4h, v5.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v6.4h, v6.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v2.4h, v2.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v1.4h, v1.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v7.4h, v7.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v3.4h, v3.4h, #8
+; CHECK-FP16-SD-NEXT:    sshll v16.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    sshll v5.4s, v5.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll v6.4s, v6.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll v17.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll v18.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll v7.4s, v7.4h, #0
+; CHECK-FP16-SD-NEXT:    sshll v19.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT:    scvtf v1.4s, v16.4s
+; CHECK-FP16-SD-NEXT:    scvtf v4.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    scvtf v2.4s, v6.4s
+; CHECK-FP16-SD-NEXT:    scvtf v3.4s, v17.4s
+; CHECK-FP16-SD-NEXT:    scvtf v5.4s, v18.4s
+; CHECK-FP16-SD-NEXT:    scvtf v6.4s, v7.4s
+; CHECK-FP16-SD-NEXT:    scvtf v7.4s, v19.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v32i8_v32f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.8h, v1.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v4.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v5.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v6.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v7.4s, v3.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v16.4s, v3.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v17.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v18.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v4.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v5.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v6.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v7.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v32i8_v32f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    sshll v3.8h, v1.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
+; CHECK-FP16-GI-NEXT:    sshll v4.4s, v2.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v5.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v6.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v7.4s, v3.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v16.4s, v3.8h, #0
+; CHECK-FP16-GI-NEXT:    sshll v17.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v18.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    scvtf v2.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    scvtf v3.4s, v6.4s
+; CHECK-FP16-GI-NEXT:    scvtf v4.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    scvtf v5.4s, v16.4s
+; CHECK-FP16-GI-NEXT:    scvtf v6.4s, v17.4s
+; CHECK-FP16-GI-NEXT:    scvtf v7.4s, v18.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <32 x i8> %a to <32 x float>
   ret <32 x float> %c
 }
 
 define <32 x float> @utofp_v32i8_v32f32(<32 x i8> %a) {
-; CHECK-SD-LABEL: utofp_v32i8_v32f32:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
-; CHECK-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
-; CHECK-SD-NEXT:    zip1 v4.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
-; CHECK-SD-NEXT:    bic v4.4h, #255, lsl #8
-; CHECK-SD-NEXT:    zip1 v5.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT:    zip1 v6.8b, v2.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v2.8b, v2.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
-; CHECK-SD-NEXT:    zip1 v7.8b, v3.8b, v0.8b
-; CHECK-SD-NEXT:    zip2 v3.8b, v3.8b, v0.8b
-; CHECK-SD-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-SD-NEXT:    ushll v4.4s, v4.4h, #0
-; CHECK-SD-NEXT:    bic v5.4h, #255, lsl #8
-; CHECK-SD-NEXT:    bic v6.4h, #255, lsl #8
-; CHECK-SD-NEXT:    bic v2.4h, #255, lsl #8
-; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
-; CHECK-SD-NEXT:    bic v7.4h, #255, lsl #8
-; CHECK-SD-NEXT:    bic v3.4h, #255, lsl #8
-; CHECK-SD-NEXT:    ushll v16.4s, v0.4h, #0
-; CHECK-SD-NEXT:    ucvtf v0.4s, v4.4s
-; CHECK-SD-NEXT:    ushll v5.4s, v5.4h, #0
-; CHECK-SD-NEXT:    ushll v6.4s, v6.4h, #0
-; CHECK-SD-NEXT:    ushll v17.4s, v2.4h, #0
-; CHECK-SD-NEXT:    ushll v18.4s, v1.4h, #0
-; CHECK-SD-NEXT:    ushll v7.4s, v7.4h, #0
-; CHECK-SD-NEXT:    ushll v19.4s, v3.4h, #0
-; CHECK-SD-NEXT:    ucvtf v1.4s, v16.4s
-; CHECK-SD-NEXT:    ucvtf v4.4s, v5.4s
-; CHECK-SD-NEXT:    ucvtf v2.4s, v6.4s
-; CHECK-SD-NEXT:    ucvtf v3.4s, v17.4s
-; CHECK-SD-NEXT:    ucvtf v5.4s, v18.4s
-; CHECK-SD-NEXT:    ucvtf v6.4s, v7.4s
-; CHECK-SD-NEXT:    ucvtf v7.4s, v19.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v32i8_v32f32:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ushll v2.8h, v0.8b, #0
-; CHECK-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-GI-NEXT:    ushll v3.8h, v1.8b, #0
-; CHECK-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-NEXT:    ushll v4.4s, v2.4h, #0
-; CHECK-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
-; CHECK-GI-NEXT:    ushll v5.4s, v0.4h, #0
-; CHECK-GI-NEXT:    ushll2 v6.4s, v0.8h, #0
-; CHECK-GI-NEXT:    ushll v7.4s, v3.4h, #0
-; CHECK-GI-NEXT:    ushll2 v16.4s, v3.8h, #0
-; CHECK-GI-NEXT:    ushll v17.4s, v1.4h, #0
-; CHECK-GI-NEXT:    ushll2 v18.4s, v1.8h, #0
-; CHECK-GI-NEXT:    ucvtf v0.4s, v4.4s
-; CHECK-GI-NEXT:    ucvtf v1.4s, v2.4s
-; CHECK-GI-NEXT:    ucvtf v2.4s, v5.4s
-; CHECK-GI-NEXT:    ucvtf v3.4s, v6.4s
-; CHECK-GI-NEXT:    ucvtf v4.4s, v7.4s
-; CHECK-GI-NEXT:    ucvtf v5.4s, v16.4s
-; CHECK-GI-NEXT:    ucvtf v6.4s, v17.4s
-; CHECK-GI-NEXT:    ucvtf v7.4s, v18.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v32i8_v32f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-NOFP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-NOFP16-SD-NEXT:    zip1 v4.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    bic v4.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    zip1 v5.8b, v1.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip1 v6.8b, v2.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v2.8b, v2.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip1 v7.8b, v3.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    zip2 v3.8b, v3.8b, v0.8b
+; CHECK-NOFP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    ushll v4.4s, v4.4h, #0
+; CHECK-NOFP16-SD-NEXT:    bic v5.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    bic v6.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    bic v2.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    bic v7.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    bic v3.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    ushll v16.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    ushll v5.4s, v5.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v6.4s, v6.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v17.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v18.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v7.4s, v7.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v19.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.4s, v16.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v4.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.4s, v17.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v5.4s, v18.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v6.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v7.4s, v19.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v32i8_v32f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
+; CHECK-FP16-SD-NEXT:    ext v3.16b, v1.16b, v1.16b, #8
+; CHECK-FP16-SD-NEXT:    zip1 v4.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    bic v4.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    zip1 v5.8b, v1.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip1 v6.8b, v2.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v2.8b, v2.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v1.8b, v1.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip1 v7.8b, v3.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    zip2 v3.8b, v3.8b, v0.8b
+; CHECK-FP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    ushll v4.4s, v4.4h, #0
+; CHECK-FP16-SD-NEXT:    bic v5.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    bic v6.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    bic v2.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    bic v7.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    bic v3.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    ushll v16.4s, v0.4h, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4s, v4.4s
+; CHECK-FP16-SD-NEXT:    ushll v5.4s, v5.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll v6.4s, v6.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll v17.4s, v2.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll v18.4s, v1.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll v7.4s, v7.4h, #0
+; CHECK-FP16-SD-NEXT:    ushll v19.4s, v3.4h, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v1.4s, v16.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v4.4s, v5.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v2.4s, v6.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v3.4s, v17.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v5.4s, v18.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v6.4s, v7.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v7.4s, v19.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v32i8_v32f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.8h, v1.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v4.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v5.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v6.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v7.4s, v3.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v16.4s, v3.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v17.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v18.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v4.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v5.4s, v16.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v6.4s, v17.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v7.4s, v18.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v32i8_v32f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    ushll v3.8h, v1.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-FP16-GI-NEXT:    ushll v4.4s, v2.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v5.4s, v0.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v6.4s, v0.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v7.4s, v3.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v16.4s, v3.8h, #0
+; CHECK-FP16-GI-NEXT:    ushll v17.4s, v1.4h, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v18.4s, v1.8h, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4s, v4.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v2.4s, v5.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v3.4s, v6.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v4.4s, v7.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v5.4s, v16.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v6.4s, v17.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v7.4s, v18.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <32 x i8> %a to <32 x float>
   ret <32 x float> %c
 }
 
-define <2 x half> @stofp_v2i128_v2f16(<2 x i128> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v2i128_v2f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sub sp, sp, #48
-; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NOFP16-NEXT:    mov x19, x1
-; CHECK-SD-NOFP16-NEXT:    mov x20, x0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x2
-; CHECK-SD-NOFP16-NEXT:    mov x1, x3
-; CHECK-SD-NOFP16-NEXT:    bl __floattisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x20
-; CHECK-SD-NOFP16-NEXT:    mov x1, x19
-; CHECK-SD-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    bl __floattisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    add sp, sp, #48
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v2i128_v2f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sub sp, sp, #48
-; CHECK-SD-FP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-FP16-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-FP16-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-FP16-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-FP16-NEXT:    mov x19, x1
-; CHECK-SD-FP16-NEXT:    mov x20, x0
-; CHECK-SD-FP16-NEXT:    mov x0, x2
-; CHECK-SD-FP16-NEXT:    mov x1, x3
-; CHECK-SD-FP16-NEXT:    bl __floattihf
-; CHECK-SD-FP16-NEXT:    mov x0, x20
-; CHECK-SD-FP16-NEXT:    mov x1, x19
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    bl __floattihf
-; CHECK-SD-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-FP16-NEXT:    add sp, sp, #48
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v2i128_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sub sp, sp, #48
-; CHECK-GI-NOFP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NOFP16-NEXT:    mov x19, x2
-; CHECK-GI-NOFP16-NEXT:    mov x20, x3
-; CHECK-GI-NOFP16-NEXT:    bl __floattisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    mov x0, x19
-; CHECK-GI-NOFP16-NEXT:    mov x1, x20
-; CHECK-GI-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    bl __floattisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h1, s0
-; CHECK-GI-NOFP16-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NOFP16-NEXT:    add sp, sp, #48
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v2i128_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    sub sp, sp, #48
-; CHECK-GI-FP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-FP16-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-FP16-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-FP16-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-FP16-NEXT:    mov x19, x2
-; CHECK-GI-FP16-NEXT:    mov x20, x3
-; CHECK-GI-FP16-NEXT:    bl __floattihf
-; CHECK-GI-FP16-NEXT:    mov x0, x19
-; CHECK-GI-FP16-NEXT:    mov x1, x20
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    bl __floattihf
-; CHECK-GI-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v0.h[0]
-; CHECK-GI-FP16-NEXT:    fmov d0, d1
-; CHECK-GI-FP16-NEXT:    add sp, sp, #48
-; CHECK-GI-FP16-NEXT:    ret
+define <2 x float> @stofp_v2i128_v2f32(<2 x i128> %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_v2i128_v2f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    mov x19, x1
+; CHECK-NOFP16-SD-NEXT:    mov x20, x0
+; CHECK-NOFP16-SD-NEXT:    mov x0, x2
+; CHECK-NOFP16-SD-NEXT:    mov x1, x3
+; CHECK-NOFP16-SD-NEXT:    bl __floattisf
+; CHECK-NOFP16-SD-NEXT:    mov x0, x20
+; CHECK-NOFP16-SD-NEXT:    mov x1, x19
+; CHECK-NOFP16-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floattisf
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i128_v2f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    mov x19, x1
+; CHECK-FP16-SD-NEXT:    mov x20, x0
+; CHECK-FP16-SD-NEXT:    mov x0, x2
+; CHECK-FP16-SD-NEXT:    mov x1, x3
+; CHECK-FP16-SD-NEXT:    bl __floattisf
+; CHECK-FP16-SD-NEXT:    mov x0, x20
+; CHECK-FP16-SD-NEXT:    mov x1, x19
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floattisf
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i128_v2f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-GI-NEXT:    mov x19, x2
+; CHECK-NOFP16-GI-NEXT:    mov x20, x3
+; CHECK-NOFP16-GI-NEXT:    bl __floattisf
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floattisf
+; CHECK-NOFP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    fmov d0, d1
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i128_v2f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-GI-NEXT:    mov x19, x2
+; CHECK-FP16-GI-NEXT:    mov x20, x3
+; CHECK-FP16-GI-NEXT:    bl __floattisf
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floattisf
+; CHECK-FP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v1.s[1], v0.s[0]
+; CHECK-FP16-GI-NEXT:    fmov d0, d1
+; CHECK-FP16-GI-NEXT:    add sp, sp, #48
+; CHECK-FP16-GI-NEXT:    ret
 entry:
-  %c = sitofp <2 x i128> %a to <2 x half>
-  ret <2 x half> %c
+  %c = sitofp <2 x i128> %a to <2 x float>
+  ret <2 x float> %c
 }
 
-define <2 x half> @utofp_v2i128_v2f16(<2 x i128> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v2i128_v2f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sub sp, sp, #48
-; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-NOFP16-NEXT:    mov x19, x1
-; CHECK-SD-NOFP16-NEXT:    mov x20, x0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x2
-; CHECK-SD-NOFP16-NEXT:    mov x1, x3
-; CHECK-SD-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x20
-; CHECK-SD-NOFP16-NEXT:    mov x1, x19
-; CHECK-SD-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    add sp, sp, #48
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v2i128_v2f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sub sp, sp, #48
-; CHECK-SD-FP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-SD-FP16-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-FP16-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-FP16-NEXT:    .cfi_offset w30, -32
-; CHECK-SD-FP16-NEXT:    mov x19, x1
-; CHECK-SD-FP16-NEXT:    mov x20, x0
-; CHECK-SD-FP16-NEXT:    mov x0, x2
-; CHECK-SD-FP16-NEXT:    mov x1, x3
-; CHECK-SD-FP16-NEXT:    bl __floatuntihf
-; CHECK-SD-FP16-NEXT:    mov x0, x20
-; CHECK-SD-FP16-NEXT:    mov x1, x19
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    bl __floatuntihf
-; CHECK-SD-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-FP16-NEXT:    add sp, sp, #48
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v2i128_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sub sp, sp, #48
-; CHECK-GI-NOFP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-NOFP16-NEXT:    mov x19, x2
-; CHECK-GI-NOFP16-NEXT:    mov x20, x3
-; CHECK-GI-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    mov x0, x19
-; CHECK-GI-NOFP16-NEXT:    mov x1, x20
-; CHECK-GI-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h1, s0
-; CHECK-GI-NOFP16-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NOFP16-NEXT:    add sp, sp, #48
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v2i128_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    sub sp, sp, #48
-; CHECK-GI-FP16-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-GI-FP16-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-FP16-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-FP16-NEXT:    .cfi_offset w30, -32
-; CHECK-GI-FP16-NEXT:    mov x19, x2
-; CHECK-GI-FP16-NEXT:    mov x20, x3
-; CHECK-GI-FP16-NEXT:    bl __floatuntihf
-; CHECK-GI-FP16-NEXT:    mov x0, x19
-; CHECK-GI-FP16-NEXT:    mov x1, x20
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    bl __floatuntihf
-; CHECK-GI-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v0.h[0]
-; CHECK-GI-FP16-NEXT:    fmov d0, d1
-; CHECK-GI-FP16-NEXT:    add sp, sp, #48
-; CHECK-GI-FP16-NEXT:    ret
+define <2 x float> @utofp_v2i128_v2f32(<2 x i128> %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_v2i128_v2f32:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    mov x19, x1
+; CHECK-NOFP16-SD-NEXT:    mov x20, x0
+; CHECK-NOFP16-SD-NEXT:    mov x0, x2
+; CHECK-NOFP16-SD-NEXT:    mov x1, x3
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-SD-NEXT:    mov x0, x20
+; CHECK-NOFP16-SD-NEXT:    mov x1, x19
+; CHECK-NOFP16-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i128_v2f32:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    mov x19, x1
+; CHECK-FP16-SD-NEXT:    mov x20, x0
+; CHECK-FP16-SD-NEXT:    mov x0, x2
+; CHECK-FP16-SD-NEXT:    mov x1, x3
+; CHECK-FP16-SD-NEXT:    bl __floatuntisf
+; CHECK-FP16-SD-NEXT:    mov x0, x20
+; CHECK-FP16-SD-NEXT:    mov x1, x19
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floatuntisf
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i128_v2f32:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-GI-NEXT:    mov x19, x2
+; CHECK-NOFP16-GI-NEXT:    mov x20, x3
+; CHECK-NOFP16-GI-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    fmov d0, d1
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i128_v2f32:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-GI-NEXT:    mov x19, x2
+; CHECK-FP16-GI-NEXT:    mov x20, x3
+; CHECK-FP16-GI-NEXT:    bl __floatuntisf
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floatuntisf
+; CHECK-FP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    // kill: def $s0 killed $s0 def $q0
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v1.s[1], v0.s[0]
+; CHECK-FP16-GI-NEXT:    fmov d0, d1
+; CHECK-FP16-GI-NEXT:    add sp, sp, #48
+; CHECK-FP16-GI-NEXT:    ret
 entry:
-  %c = uitofp <2 x i128> %a to <2 x half>
-  ret <2 x half> %c
-}
-
-define <3 x half> @stofp_v3i128_v3f16(<3 x i128> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v3i128_v3f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sub sp, sp, #80
-; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NOFP16-NEXT:    mov x21, x1
-; CHECK-SD-NOFP16-NEXT:    mov x22, x0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x4
-; CHECK-SD-NOFP16-NEXT:    mov x1, x5
-; CHECK-SD-NOFP16-NEXT:    mov x19, x3
-; CHECK-SD-NOFP16-NEXT:    mov x20, x2
-; CHECK-SD-NOFP16-NEXT:    bl __floattisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x22
-; CHECK-SD-NOFP16-NEXT:    mov x1, x21
-; CHECK-SD-NOFP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    bl __floattisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x20
-; CHECK-SD-NOFP16-NEXT:    mov x1, x19
-; CHECK-SD-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    bl __floattisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    add sp, sp, #80
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v3i128_v3f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sub sp, sp, #80
-; CHECK-SD-FP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-SD-FP16-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-FP16-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-FP16-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-FP16-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-FP16-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-FP16-NEXT:    mov x21, x1
-; CHECK-SD-FP16-NEXT:    mov x22, x0
-; CHECK-SD-FP16-NEXT:    mov x0, x4
-; CHECK-SD-FP16-NEXT:    mov x1, x5
-; CHECK-SD-FP16-NEXT:    mov x19, x3
-; CHECK-SD-FP16-NEXT:    mov x20, x2
-; CHECK-SD-FP16-NEXT:    bl __floattihf
-; CHECK-SD-FP16-NEXT:    mov x0, x22
-; CHECK-SD-FP16-NEXT:    mov x1, x21
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    bl __floattihf
-; CHECK-SD-FP16-NEXT:    mov x0, x20
-; CHECK-SD-FP16-NEXT:    mov x1, x19
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    bl __floattihf
-; CHECK-SD-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    mov v1.h[1], v0.h[0]
-; CHECK-SD-FP16-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-SD-FP16-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-FP16-NEXT:    add sp, sp, #80
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v3i128_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sub sp, sp, #80
-; CHECK-GI-NOFP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NOFP16-NEXT:    mov x19, x2
-; CHECK-GI-NOFP16-NEXT:    mov x20, x3
-; CHECK-GI-NOFP16-NEXT:    mov x21, x4
-; CHECK-GI-NOFP16-NEXT:    mov x22, x5
-; CHECK-GI-NOFP16-NEXT:    bl __floattisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    mov x0, x19
-; CHECK-GI-NOFP16-NEXT:    mov x1, x20
-; CHECK-GI-NOFP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    bl __floattisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    mov x0, x21
-; CHECK-GI-NOFP16-NEXT:    mov x1, x22
-; CHECK-GI-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    bl __floattisf
-; CHECK-GI-NOFP16-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    mov v1.h[1], v2.h[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-GI-NOFP16-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NOFP16-NEXT:    add sp, sp, #80
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v3i128_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    sub sp, sp, #80
-; CHECK-GI-FP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-GI-FP16-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-FP16-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-FP16-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-FP16-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-FP16-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-FP16-NEXT:    mov x19, x2
-; CHECK-GI-FP16-NEXT:    mov x20, x3
-; CHECK-GI-FP16-NEXT:    mov x21, x4
-; CHECK-GI-FP16-NEXT:    mov x22, x5
-; CHECK-GI-FP16-NEXT:    bl __floattihf
-; CHECK-GI-FP16-NEXT:    mov x0, x19
-; CHECK-GI-FP16-NEXT:    mov x1, x20
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    bl __floattihf
-; CHECK-GI-FP16-NEXT:    mov x0, x21
-; CHECK-GI-FP16-NEXT:    mov x1, x22
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    bl __floattihf
-; CHECK-GI-FP16-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    add sp, sp, #80
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = sitofp <3 x i128> %a to <3 x half>
-  ret <3 x half> %c
-}
-
-define <3 x half> @utofp_v3i128_v3f16(<3 x i128> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v3i128_v3f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sub sp, sp, #80
-; CHECK-SD-NOFP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-NOFP16-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-NOFP16-NEXT:    mov x21, x1
-; CHECK-SD-NOFP16-NEXT:    mov x22, x0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x4
-; CHECK-SD-NOFP16-NEXT:    mov x1, x5
-; CHECK-SD-NOFP16-NEXT:    mov x19, x3
-; CHECK-SD-NOFP16-NEXT:    mov x20, x2
-; CHECK-SD-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x22
-; CHECK-SD-NOFP16-NEXT:    mov x1, x21
-; CHECK-SD-NOFP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    mov x0, x20
-; CHECK-SD-NOFP16-NEXT:    mov x1, x19
-; CHECK-SD-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[1], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-NOFP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-SD-NOFP16-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    add sp, sp, #80
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v3i128_v3f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sub sp, sp, #80
-; CHECK-SD-FP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-SD-FP16-NEXT:    .cfi_offset w19, -8
-; CHECK-SD-FP16-NEXT:    .cfi_offset w20, -16
-; CHECK-SD-FP16-NEXT:    .cfi_offset w21, -24
-; CHECK-SD-FP16-NEXT:    .cfi_offset w22, -32
-; CHECK-SD-FP16-NEXT:    .cfi_offset w30, -48
-; CHECK-SD-FP16-NEXT:    mov x21, x1
-; CHECK-SD-FP16-NEXT:    mov x22, x0
-; CHECK-SD-FP16-NEXT:    mov x0, x4
-; CHECK-SD-FP16-NEXT:    mov x1, x5
-; CHECK-SD-FP16-NEXT:    mov x19, x3
-; CHECK-SD-FP16-NEXT:    mov x20, x2
-; CHECK-SD-FP16-NEXT:    bl __floatuntihf
-; CHECK-SD-FP16-NEXT:    mov x0, x22
-; CHECK-SD-FP16-NEXT:    mov x1, x21
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    bl __floatuntihf
-; CHECK-SD-FP16-NEXT:    mov x0, x20
-; CHECK-SD-FP16-NEXT:    mov x1, x19
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-SD-FP16-NEXT:    bl __floatuntihf
-; CHECK-SD-FP16-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-SD-FP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    mov v1.h[1], v0.h[0]
-; CHECK-SD-FP16-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
-; CHECK-SD-FP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-SD-FP16-NEXT:    mov v0.16b, v1.16b
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-FP16-NEXT:    add sp, sp, #80
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v3i128_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sub sp, sp, #80
-; CHECK-GI-NOFP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-NOFP16-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-NOFP16-NEXT:    mov x19, x2
-; CHECK-GI-NOFP16-NEXT:    mov x20, x3
-; CHECK-GI-NOFP16-NEXT:    mov x21, x4
-; CHECK-GI-NOFP16-NEXT:    mov x22, x5
-; CHECK-GI-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    mov x0, x19
-; CHECK-GI-NOFP16-NEXT:    mov x1, x20
-; CHECK-GI-NOFP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    mov x0, x21
-; CHECK-GI-NOFP16-NEXT:    mov x1, x22
-; CHECK-GI-NOFP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-NOFP16-NEXT:    bl __floatuntisf
-; CHECK-GI-NOFP16-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-GI-NOFP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NOFP16-NEXT:    mov v1.h[1], v2.h[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-GI-NOFP16-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-NOFP16-NEXT:    add sp, sp, #80
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v3i128_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    sub sp, sp, #80
-; CHECK-GI-FP16-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-GI-FP16-NEXT:    .cfi_offset w19, -8
-; CHECK-GI-FP16-NEXT:    .cfi_offset w20, -16
-; CHECK-GI-FP16-NEXT:    .cfi_offset w21, -24
-; CHECK-GI-FP16-NEXT:    .cfi_offset w22, -32
-; CHECK-GI-FP16-NEXT:    .cfi_offset w30, -48
-; CHECK-GI-FP16-NEXT:    mov x19, x2
-; CHECK-GI-FP16-NEXT:    mov x20, x3
-; CHECK-GI-FP16-NEXT:    mov x21, x4
-; CHECK-GI-FP16-NEXT:    mov x22, x5
-; CHECK-GI-FP16-NEXT:    bl __floatuntihf
-; CHECK-GI-FP16-NEXT:    mov x0, x19
-; CHECK-GI-FP16-NEXT:    mov x1, x20
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    bl __floatuntihf
-; CHECK-GI-FP16-NEXT:    mov x0, x21
-; CHECK-GI-FP16-NEXT:    mov x1, x22
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-GI-FP16-NEXT:    bl __floatuntihf
-; CHECK-GI-FP16-NEXT:    ldp q2, q1, [sp] // 32-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $q0
-; CHECK-GI-FP16-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    ldp x22, x21, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-FP16-NEXT:    mov v1.h[1], v2.h[0]
-; CHECK-GI-FP16-NEXT:    mov v1.h[2], v0.h[0]
-; CHECK-GI-FP16-NEXT:    mov v0.16b, v1.16b
-; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-GI-FP16-NEXT:    add sp, sp, #80
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = uitofp <3 x i128> %a to <3 x half>
-  ret <3 x half> %c
+  %c = uitofp <2 x i128> %a to <2 x float>
+  ret <2 x float> %c
 }
 
 define <2 x half> @stofp_v2i64_v2f16(<2 x i64> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v2i64_v2f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NOFP16-NEXT:    fmov x9, d0
-; CHECK-SD-NOFP16-NEXT:    scvtf s1, x9
-; CHECK-SD-NOFP16-NEXT:    scvtf s0, x8
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v2i64_v2f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-FP16-NEXT:    fmov x9, d0
-; CHECK-SD-FP16-NEXT:    scvtf h0, x9
-; CHECK-SD-FP16-NEXT:    scvtf h1, x8
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i64_v2f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v2i64_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    fmov x9, d0
+; CHECK-NOFP16-SD-NEXT:    scvtf s1, x9
+; CHECK-NOFP16-SD-NEXT:    scvtf s0, x8
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s0
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i64_v2f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-SD-NEXT:    fmov x9, d0
+; CHECK-FP16-SD-NEXT:    scvtf h0, x9
+; CHECK-FP16-SD-NEXT:    scvtf h1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i64_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i64_v2f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-FP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <2 x i64> %a to <2 x half>
   ret <2 x half> %c
 }
 
 define <2 x half> @utofp_v2i64_v2f16(<2 x i64> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v2i64_v2f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-NOFP16-NEXT:    fmov x9, d0
-; CHECK-SD-NOFP16-NEXT:    ucvtf s1, x9
-; CHECK-SD-NOFP16-NEXT:    ucvtf s0, x8
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v2i64_v2f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    mov x8, v0.d[1]
-; CHECK-SD-FP16-NEXT:    fmov x9, d0
-; CHECK-SD-FP16-NEXT:    ucvtf h0, x9
-; CHECK-SD-FP16-NEXT:    ucvtf h1, x8
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i64_v2f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v2i64_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    fmov x9, d0
+; CHECK-NOFP16-SD-NEXT:    ucvtf s1, x9
+; CHECK-NOFP16-SD-NEXT:    ucvtf s0, x8
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s0
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i64_v2f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    mov x8, v0.d[1]
+; CHECK-FP16-SD-NEXT:    fmov x9, d0
+; CHECK-FP16-SD-NEXT:    ucvtf h0, x9
+; CHECK-FP16-SD-NEXT:    ucvtf h1, x8
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i64_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i64_v2f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-FP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <2 x i64> %a to <2 x half>
   ret <2 x half> %c
@@ -6721,414 +5891,394 @@ entry:
 }
 
 define <8 x half> @stofp_v8i64_v8f16(<8 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v8i64_v8f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v8i64_v8f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i64_v8f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v8i64_v8f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i64_v8f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v8i64_v8f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %c
 }
 
 define <8 x half> @utofp_v8i64_v8f16(<8 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v8i64_v8f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v8i64_v8f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i64_v8f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v8i64_v8f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i64_v8f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v8i64_v8f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %c
 }
 
 define <16 x half> @stofp_v16i64_v16f16(<16 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v16i64_v16f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-SD-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-SD-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-SD-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    fcvtn2 v6.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-SD-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-SD-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v16i64_v16f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i64_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v6.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v16i64_v16f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-FP16-SD-NEXT:    scvtf v1.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v6.4s, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i64_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i64_v16f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    scvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    scvtf v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    scvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    scvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    scvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    scvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i64> %a to <16 x half>
   ret <16 x half> %c
 }
 
 define <16 x half> @utofp_v16i64_v16f16(<16 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v16i64_v16f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-SD-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-SD-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-SD-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-SD-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-SD-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    fcvtn2 v6.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-SD-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-SD-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v16i64_v16f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v2.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NEXT:    fcvtn2 v2.4s, v3.2d
-; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i64_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.2d, v7.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v6.4s, v1.2d
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v16i64_v16f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-FP16-SD-NEXT:    ucvtf v1.2d, v7.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-FP16-SD-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v6.4s, v1.2d
+; CHECK-FP16-SD-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i64_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NOFP16-GI-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i64_v16f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2d, v0.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v4.2d, v4.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v1.2d, v1.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v2.2d, v2.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v5.2d, v5.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v6.2d, v6.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-FP16-GI-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.2s, v0.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v4.2s, v4.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v6.2s, v6.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.4s, v1.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v4.4s, v5.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-FP16-GI-NEXT:    fcvtn2 v6.4s, v7.2d
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtn v1.4h, v4.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v1.8h, v6.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i64> %a to <16 x half>
   ret <16 x half> %c
 }
 
-define <32 x half> @stofp_v32i64_v32f16(<32 x i64> %a) {
-; CHECK-SD-LABEL: stofp_v32i64_v32f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q17, q16, [sp]
-; CHECK-SD-NEXT:    scvtf v18.2d, v0.2d
-; CHECK-SD-NEXT:    ldp q19, q0, [sp, #64]
-; CHECK-SD-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    ldp q21, q20, [sp, #96]
-; CHECK-SD-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-SD-NEXT:    ldp q23, q22, [sp, #32]
-; CHECK-SD-NEXT:    scvtf v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtn v18.2s, v18.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-SD-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    scvtf v23.2d, v23.2d
-; CHECK-SD-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    scvtf v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtn v17.2s, v17.2d
-; CHECK-SD-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtn v24.2s, v2.2d
-; CHECK-SD-NEXT:    fcvtn v19.2s, v19.2d
-; CHECK-SD-NEXT:    scvtf v2.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtn2 v18.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-SD-NEXT:    scvtf v1.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-SD-NEXT:    fcvtn v5.2s, v23.2d
-; CHECK-SD-NEXT:    scvtf v3.2d, v22.2d
-; CHECK-SD-NEXT:    fcvtn v7.2s, v21.2d
-; CHECK-SD-NEXT:    fcvtn2 v17.4s, v16.2d
-; CHECK-SD-NEXT:    scvtf v16.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtn2 v19.4s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v24.4s, v2.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v18.4s
-; CHECK-SD-NEXT:    fcvtn2 v6.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-SD-NEXT:    fcvtn2 v5.4s, v3.2d
-; CHECK-SD-NEXT:    fcvtn v2.4h, v17.4s
-; CHECK-SD-NEXT:    fcvtn2 v7.4s, v16.2d
-; CHECK-SD-NEXT:    fcvtn v3.4h, v19.4s
-; CHECK-SD-NEXT:    fcvtn2 v0.8h, v24.4s
-; CHECK-SD-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-SD-NEXT:    fcvtn2 v2.8h, v5.4s
-; CHECK-SD-NEXT:    fcvtn2 v3.8h, v7.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v32i64_v32f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    scvtf v16.2d, v16.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    scvtf v20.2d, v20.2d
-; CHECK-GI-NEXT:    scvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    scvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    scvtf v17.2d, v17.2d
-; CHECK-GI-NEXT:    scvtf v18.2d, v18.2d
-; CHECK-GI-NEXT:    scvtf v21.2d, v21.2d
-; CHECK-GI-NEXT:    scvtf v22.2d, v22.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v16.2s, v16.2d
-; CHECK-GI-NEXT:    scvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtn v20.2s, v20.2d
-; CHECK-GI-NEXT:    scvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    scvtf v19.2d, v19.2d
-; CHECK-GI-NEXT:    scvtf v23.2d, v23.2d
-; CHECK-GI-NEXT:    fcvtn v24.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn v18.2s, v18.2d
-; CHECK-GI-NEXT:    fcvtn v22.2s, v22.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NEXT:    fcvtn2 v16.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v20.4s, v21.2d
-; CHECK-GI-NEXT:    fcvtn2 v24.4s, v3.2d
-; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NEXT:    fcvtn2 v18.4s, v19.2d
-; CHECK-GI-NEXT:    fcvtn2 v22.4s, v23.2d
-; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NEXT:    fcvtn v2.4h, v16.4s
-; CHECK-GI-NEXT:    fcvtn v3.4h, v20.4s
-; CHECK-GI-NEXT:    fcvtn2 v0.8h, v24.4s
-; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NEXT:    fcvtn2 v3.8h, v22.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = sitofp <32 x i64> %a to <32 x half>
-  ret <32 x half> %c
-}
-
-define <32 x half> @utofp_v32i64_v32f16(<32 x i64> %a) {
-; CHECK-SD-LABEL: utofp_v32i64_v32f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ldp q17, q16, [sp]
-; CHECK-SD-NEXT:    ucvtf v18.2d, v0.2d
-; CHECK-SD-NEXT:    ldp q19, q0, [sp, #64]
-; CHECK-SD-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-SD-NEXT:    ldp q21, q20, [sp, #96]
-; CHECK-SD-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-SD-NEXT:    ucvtf v17.2d, v17.2d
-; CHECK-SD-NEXT:    ldp q23, q22, [sp, #32]
-; CHECK-SD-NEXT:    ucvtf v19.2d, v19.2d
-; CHECK-SD-NEXT:    fcvtn v18.2s, v18.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-SD-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-SD-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-SD-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-SD-NEXT:    ucvtf v23.2d, v23.2d
-; CHECK-SD-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-SD-NEXT:    ucvtf v21.2d, v21.2d
-; CHECK-SD-NEXT:    fcvtn v17.2s, v17.2d
-; CHECK-SD-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-SD-NEXT:    fcvtn v24.2s, v2.2d
-; CHECK-SD-NEXT:    fcvtn v19.2s, v19.2d
-; CHECK-SD-NEXT:    ucvtf v2.2d, v3.2d
-; CHECK-SD-NEXT:    fcvtn2 v18.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-SD-NEXT:    ucvtf v1.2d, v7.2d
-; CHECK-SD-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-SD-NEXT:    fcvtn v5.2s, v23.2d
-; CHECK-SD-NEXT:    ucvtf v3.2d, v22.2d
-; CHECK-SD-NEXT:    fcvtn v7.2s, v21.2d
-; CHECK-SD-NEXT:    fcvtn2 v17.4s, v16.2d
-; CHECK-SD-NEXT:    ucvtf v16.2d, v20.2d
-; CHECK-SD-NEXT:    fcvtn2 v19.4s, v0.2d
-; CHECK-SD-NEXT:    fcvtn2 v24.4s, v2.2d
-; CHECK-SD-NEXT:    fcvtn v0.4h, v18.4s
-; CHECK-SD-NEXT:    fcvtn2 v6.4s, v1.2d
-; CHECK-SD-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-SD-NEXT:    fcvtn2 v5.4s, v3.2d
-; CHECK-SD-NEXT:    fcvtn v2.4h, v17.4s
-; CHECK-SD-NEXT:    fcvtn2 v7.4s, v16.2d
-; CHECK-SD-NEXT:    fcvtn v3.4h, v19.4s
-; CHECK-SD-NEXT:    fcvtn2 v0.8h, v24.4s
-; CHECK-SD-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-SD-NEXT:    fcvtn2 v2.8h, v5.4s
-; CHECK-SD-NEXT:    fcvtn2 v3.8h, v7.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v32i64_v32f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ldp q16, q17, [sp]
-; CHECK-GI-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-GI-NEXT:    ldp q20, q21, [sp, #64]
-; CHECK-GI-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-GI-NEXT:    ldp q18, q19, [sp, #32]
-; CHECK-GI-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-GI-NEXT:    ucvtf v16.2d, v16.2d
-; CHECK-GI-NEXT:    ldp q22, q23, [sp, #96]
-; CHECK-GI-NEXT:    ucvtf v20.2d, v20.2d
-; CHECK-GI-NEXT:    ucvtf v2.2d, v2.2d
-; CHECK-GI-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-GI-NEXT:    ucvtf v6.2d, v6.2d
-; CHECK-GI-NEXT:    ucvtf v17.2d, v17.2d
-; CHECK-GI-NEXT:    ucvtf v18.2d, v18.2d
-; CHECK-GI-NEXT:    ucvtf v21.2d, v21.2d
-; CHECK-GI-NEXT:    ucvtf v22.2d, v22.2d
-; CHECK-GI-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-GI-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-GI-NEXT:    fcvtn v16.2s, v16.2d
-; CHECK-GI-NEXT:    ucvtf v3.2d, v3.2d
-; CHECK-GI-NEXT:    fcvtn v20.2s, v20.2d
-; CHECK-GI-NEXT:    ucvtf v7.2d, v7.2d
-; CHECK-GI-NEXT:    ucvtf v19.2d, v19.2d
-; CHECK-GI-NEXT:    ucvtf v23.2d, v23.2d
-; CHECK-GI-NEXT:    fcvtn v24.2s, v2.2d
-; CHECK-GI-NEXT:    fcvtn v6.2s, v6.2d
-; CHECK-GI-NEXT:    fcvtn v18.2s, v18.2d
-; CHECK-GI-NEXT:    fcvtn v22.2s, v22.2d
-; CHECK-GI-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-GI-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-GI-NEXT:    fcvtn2 v16.4s, v17.2d
-; CHECK-GI-NEXT:    fcvtn2 v20.4s, v21.2d
-; CHECK-GI-NEXT:    fcvtn2 v24.4s, v3.2d
-; CHECK-GI-NEXT:    fcvtn2 v6.4s, v7.2d
-; CHECK-GI-NEXT:    fcvtn2 v18.4s, v19.2d
-; CHECK-GI-NEXT:    fcvtn2 v22.4s, v23.2d
-; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-GI-NEXT:    fcvtn v2.4h, v16.4s
-; CHECK-GI-NEXT:    fcvtn v3.4h, v20.4s
-; CHECK-GI-NEXT:    fcvtn2 v0.8h, v24.4s
-; CHECK-GI-NEXT:    fcvtn2 v1.8h, v6.4s
-; CHECK-GI-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NEXT:    fcvtn2 v3.8h, v22.4s
-; CHECK-GI-NEXT:    ret
-entry:
-  %c = uitofp <32 x i64> %a to <32 x half>
-  ret <32 x half> %c
-}
-
 define <2 x half> @stofp_v2i32_v2f16(<2 x i32> %a) {
-; CHECK-SD-LABEL: stofp_v2i32_v2f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v2i32_v2f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.2s, v0.2s
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v2i32_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i32_v2f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i32_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i32_v2f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    scvtf v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-FP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <2 x i32> %a to <2 x half>
   ret <2 x half> %c
 }
 
 define <2 x half> @utofp_v2i32_v2f16(<2 x i32> %a) {
-; CHECK-SD-LABEL: utofp_v2i32_v2f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v2i32_v2f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.2s, v0.2s
-; CHECK-GI-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v2i32_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i32_v2f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i32_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i32_v2f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ucvtf v0.2s, v0.2s
+; CHECK-FP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-FP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <2 x i32> %a to <2 x half>
   ret <2 x half> %c
@@ -7205,1271 +6355,1890 @@ entry:
 }
 
 define <16 x half> @stofp_v16i32_v16f16(<16 x i32> %a) {
-; CHECK-SD-LABEL: stofp_v16i32_v16f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-SD-NEXT:    scvtf v4.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-SD-NEXT:    scvtf v2.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-SD-NEXT:    fcvtn2 v1.8h, v2.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: stofp_v16i32_v16f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-GI-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-GI-NEXT:    scvtf v4.4s, v1.4s
-; CHECK-GI-NEXT:    scvtf v3.4s, v3.4s
-; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-GI-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-GI-NEXT:    fcvtn2 v1.8h, v3.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i32_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v16i32_v16f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    scvtf v4.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-FP16-SD-NEXT:    scvtf v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v1.8h, v2.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i32_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v4.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i32_v16f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    scvtf v4.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    scvtf v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v1.8h, v3.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i32> %a to <16 x half>
   ret <16 x half> %c
 }
 
 define <16 x half> @utofp_v16i32_v16f16(<16 x i32> %a) {
-; CHECK-SD-LABEL: utofp_v16i32_v16f16:
-; CHECK-SD:       // %bb.0: // %entry
-; CHECK-SD-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-SD-NEXT:    ucvtf v4.4s, v1.4s
-; CHECK-SD-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-SD-NEXT:    ucvtf v2.4s, v3.4s
-; CHECK-SD-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-SD-NEXT:    fcvtn2 v1.8h, v2.4s
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: utofp_v16i32_v16f16:
-; CHECK-GI:       // %bb.0: // %entry
-; CHECK-GI-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-GI-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-GI-NEXT:    ucvtf v4.4s, v1.4s
-; CHECK-GI-NEXT:    ucvtf v3.4s, v3.4s
-; CHECK-GI-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-GI-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-GI-NEXT:    fcvtn2 v1.8h, v3.4s
-; CHECK-GI-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i32_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v4.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v16i32_v16f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v4.4s, v1.4s
+; CHECK-FP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-SD-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-FP16-SD-NEXT:    ucvtf v2.4s, v3.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-FP16-SD-NEXT:    fcvtn2 v1.8h, v2.4s
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i32_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v4.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i32_v16f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v4.4s, v1.4s
+; CHECK-FP16-GI-NEXT:    ucvtf v3.4s, v3.4s
+; CHECK-FP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-FP16-GI-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-FP16-GI-NEXT:    fcvtn2 v1.8h, v3.4s
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i32> %a to <16 x half>
   ret <16 x half> %c
 }
 
-define <32 x half> @stofp_v32i32_v32f16(<32 x i32> %a) {
-; CHECK-LABEL: stofp_v32i32_v32f16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-NEXT:    scvtf v4.4s, v4.4s
-; CHECK-NEXT:    scvtf v6.4s, v6.4s
-; CHECK-NEXT:    scvtf v16.4s, v1.4s
-; CHECK-NEXT:    scvtf v17.4s, v3.4s
-; CHECK-NEXT:    scvtf v5.4s, v5.4s
-; CHECK-NEXT:    scvtf v7.4s, v7.4s
-; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-NEXT:    fcvtn v2.4h, v4.4s
-; CHECK-NEXT:    fcvtn v3.4h, v6.4s
-; CHECK-NEXT:    fcvtn2 v0.8h, v16.4s
-; CHECK-NEXT:    fcvtn2 v1.8h, v17.4s
-; CHECK-NEXT:    fcvtn2 v2.8h, v5.4s
-; CHECK-NEXT:    fcvtn2 v3.8h, v7.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = sitofp <32 x i32> %a to <32 x half>
-  ret <32 x half> %c
-}
-
-define <32 x half> @utofp_v32i32_v32f16(<32 x i32> %a) {
-; CHECK-LABEL: utofp_v32i32_v32f16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-NEXT:    ucvtf v4.4s, v4.4s
-; CHECK-NEXT:    ucvtf v6.4s, v6.4s
-; CHECK-NEXT:    ucvtf v16.4s, v1.4s
-; CHECK-NEXT:    ucvtf v17.4s, v3.4s
-; CHECK-NEXT:    ucvtf v5.4s, v5.4s
-; CHECK-NEXT:    ucvtf v7.4s, v7.4s
-; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-NEXT:    fcvtn v2.4h, v4.4s
-; CHECK-NEXT:    fcvtn v3.4h, v6.4s
-; CHECK-NEXT:    fcvtn2 v0.8h, v16.4s
-; CHECK-NEXT:    fcvtn2 v1.8h, v17.4s
-; CHECK-NEXT:    fcvtn2 v2.8h, v5.4s
-; CHECK-NEXT:    fcvtn2 v3.8h, v7.4s
-; CHECK-NEXT:    ret
-entry:
-  %c = uitofp <32 x i32> %a to <32 x half>
-  ret <32 x half> %c
-}
-
 define <2 x half> @stofp_v2i16_v2f16(<2 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v2i16_v2f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-SD-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v2i16_v2f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v2i16_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    shl v0.2s, v0.2s, #16
-; CHECK-GI-NOFP16-NEXT:    sshr v0.2s, v0.2s, #16
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2s, v0.2s
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v2i16_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v2i16_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-NOFP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_v2i16_v2f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-FP16-NEXT:    scvtf v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i16_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    shl v0.2s, v0.2s, #16
+; CHECK-NOFP16-GI-NEXT:    sshr v0.2s, v0.2s, #16
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = sitofp <2 x i16> %a to <2 x half>
   ret <2 x half> %c
 }
 
 define <2 x half> @utofp_v2i16_v2f16(<2 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v2i16_v2f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-SD-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v2i16_v2f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v2i16_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    movi d1, #0x00ffff0000ffff
-; CHECK-GI-NOFP16-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2s, v0.2s
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v2i16_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v2i16_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-NOFP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_v2i16_v2f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ucvtf v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i16_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    movi d1, #0x00ffff0000ffff
+; CHECK-NOFP16-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = uitofp <2 x i16> %a to <2 x half>
   ret <2 x half> %c
 }
 
 define <3 x half> @stofp_v3i16_v3f16(<3 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v3i16_v3f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v3i16_v3f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v3i16_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v3i16_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: stofp_v3i16_v3f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_v3i16_v3f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    scvtf v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = sitofp <3 x i16> %a to <3 x half>
   ret <3 x half> %c
 }
 
 define <3 x half> @utofp_v3i16_v3f16(<3 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v3i16_v3f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v3i16_v3f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v3i16_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v3i16_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: utofp_v3i16_v3f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_v3i16_v3f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ucvtf v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = uitofp <3 x i16> %a to <3 x half>
   ret <3 x half> %c
 }
 
 define <4 x half> @stofp_v4i16_v4f16(<4 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v4i16_v4f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v4i16_v4f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v4i16_v4f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v4i16_v4f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: stofp_v4i16_v4f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_v4i16_v4f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    scvtf v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = sitofp <4 x i16> %a to <4 x half>
   ret <4 x half> %c
 }
 
 define <4 x half> @utofp_v4i16_v4f16(<4 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v4i16_v4f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v4i16_v4f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v4i16_v4f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v4i16_v4f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-LABEL: utofp_v4i16_v4f16:
+; CHECK-NOFP16:       // %bb.0: // %entry
+; CHECK-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_v4i16_v4f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ucvtf v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
 entry:
   %c = uitofp <4 x i16> %a to <4 x half>
   ret <4 x half> %c
 }
 
 define <8 x half> @stofp_v8i16_v8f16(<8 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v8i16_v8f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v2.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v1.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v8i16_v8f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    scvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v8i16_v8f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v8i16_v8f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i16_v8f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_v8i16_v8f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    scvtf v0.8h, v0.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i16_v8f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i16> %a to <8 x half>
   ret <8 x half> %c
 }
 
 define <8 x half> @utofp_v8i16_v8f16(<8 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v8i16_v8f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v1.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v8i16_v8f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ucvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v8i16_v8f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v8i16_v8f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i16_v8f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_v8i16_v8f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ucvtf v0.8h, v0.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i16_v8f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i16> %a to <8 x half>
   ret <8 x half> %c
 }
 
 define <16 x half> @stofp_v16i16_v16f16(<16 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v16i16_v16f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v3.4s, v1.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v4.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v5.4s, v1.8h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v2.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v3.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v16i16_v16f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    scvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    scvtf v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v16i16_v16f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sshll v2.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v3.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v4.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v5.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v16i16_v16f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    scvtf v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i16_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v5.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_v16i16_v16f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    scvtf v0.8h, v0.8h
+; CHECK-FP16-NEXT:    scvtf v1.8h, v1.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i16_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v2.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v4.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v5.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i16> %a to <16 x half>
   ret <16 x half> %c
 }
 
 define <16 x half> @utofp_v16i16_v16f16(<16 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v16i16_v16f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v3.4s, v1.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v4.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v5.4s, v1.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v2.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v3.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v3.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v16i16_v16f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ucvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ucvtf v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v16i16_v16f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ushll v2.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v3.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v4.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v5.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v16i16_v16f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ucvtf v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i16_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v4.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v5.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_v16i16_v16f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ucvtf v0.8h, v0.8h
+; CHECK-FP16-NEXT:    ucvtf v1.8h, v1.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i16_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v2.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v4.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v5.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i16> %a to <16 x half>
   ret <16 x half> %c
 }
 
-define <32 x half> @stofp_v32i16_v32f16(<32 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v32i16_v32f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sshll v4.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v5.4s, v1.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v6.4s, v2.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v7.4s, v3.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v2.4s, v2.8h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v3.4s, v3.8h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v4.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v5.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v6.4s, v6.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v7.4s, v7.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v16.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v17.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v18.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v19.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v16.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v17.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v19.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v32i16_v32f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    scvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    scvtf v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    scvtf v2.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    scvtf v3.8h, v3.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v32i16_v32f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sshll v4.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v5.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v6.4s, v2.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v7.4s, v3.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v2.4s, v2.8h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v3.4s, v3.8h, #0
-; CHECK-GI-NOFP16-NEXT:    scvtf v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v5.4s, v5.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v6.4s, v6.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v7.4s, v7.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v16.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v17.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v18.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v19.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v17.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v3.8h, v19.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v32i16_v32f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    scvtf v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    scvtf v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    scvtf v2.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    scvtf v3.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = sitofp <32 x i16> %a to <32 x half>
-  ret <32 x half> %c
-}
-
-define <32 x half> @utofp_v32i16_v32f16(<32 x i16> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v32i16_v32f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ushll v4.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v5.4s, v1.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v6.4s, v2.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v7.4s, v3.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v2.4s, v2.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v3.4s, v3.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v4.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v5.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v6.4s, v6.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v7.4s, v7.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v16.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v17.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v18.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v19.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v16.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v17.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v19.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v32i16_v32f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ucvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ucvtf v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    ucvtf v2.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    ucvtf v3.8h, v3.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v32i16_v32f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ushll v4.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v5.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v6.4s, v2.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v7.4s, v3.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v2.4s, v2.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v3.4s, v3.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ucvtf v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v5.4s, v5.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v6.4s, v6.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v7.4s, v7.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v16.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v17.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v18.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v19.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v17.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v3.8h, v19.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v32i16_v32f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ucvtf v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ucvtf v1.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    ucvtf v2.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    ucvtf v3.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    ret
-entry:
-  %c = uitofp <32 x i16> %a to <32 x half>
-  ret <32 x half> %c
-}
-
 define <2 x half> @stofp_v2i8_v2f16(<2 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v2i8_v2f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NOFP16-NEXT:    fmov w9, s0
-; CHECK-SD-NOFP16-NEXT:    sxtb w9, w9
-; CHECK-SD-NOFP16-NEXT:    sxtb w8, w8
-; CHECK-SD-NOFP16-NEXT:    scvtf s1, w9
-; CHECK-SD-NOFP16-NEXT:    scvtf s0, w8
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v2i8_v2f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-FP16-NEXT:    fmov w9, s0
-; CHECK-SD-FP16-NEXT:    sxtb w9, w9
-; CHECK-SD-FP16-NEXT:    sxtb w8, w8
-; CHECK-SD-FP16-NEXT:    scvtf h0, w9
-; CHECK-SD-FP16-NEXT:    scvtf h1, w8
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v2i8_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    shl v0.2s, v0.2s, #24
-; CHECK-GI-NOFP16-NEXT:    sshr v0.2s, v0.2s, #24
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.2s, v0.2s
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v2i8_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-GI-FP16-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-GI-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v2i8_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-NOFP16-SD-NEXT:    fmov w9, s0
+; CHECK-NOFP16-SD-NEXT:    sxtb w9, w9
+; CHECK-NOFP16-SD-NEXT:    sxtb w8, w8
+; CHECK-NOFP16-SD-NEXT:    scvtf s1, w9
+; CHECK-NOFP16-SD-NEXT:    scvtf s0, w8
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s0
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i8_v2f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-FP16-SD-NEXT:    fmov w9, s0
+; CHECK-FP16-SD-NEXT:    sxtb w9, w9
+; CHECK-FP16-SD-NEXT:    sxtb w8, w8
+; CHECK-FP16-SD-NEXT:    scvtf h0, w9
+; CHECK-FP16-SD-NEXT:    scvtf h1, w8
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i8_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    shl v0.2s, v0.2s, #24
+; CHECK-NOFP16-GI-NEXT:    sshr v0.2s, v0.2s, #24
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i8_v2f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-GI-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-GI-NEXT:    scvtf v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <2 x i8> %a to <2 x half>
   ret <2 x half> %c
 }
 
 define <2 x half> @utofp_v2i8_v2f16(<2 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v2i8_v2f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-NOFP16-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-NOFP16-NEXT:    fmov w9, s0
-; CHECK-SD-NOFP16-NEXT:    and w9, w9, #0xff
-; CHECK-SD-NOFP16-NEXT:    and w8, w8, #0xff
-; CHECK-SD-NOFP16-NEXT:    ucvtf s1, w9
-; CHECK-SD-NOFP16-NEXT:    ucvtf s0, w8
-; CHECK-SD-NOFP16-NEXT:    fcvt h2, s0
-; CHECK-SD-NOFP16-NEXT:    fcvt h0, s1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], v2.h[0]
-; CHECK-SD-NOFP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v2i8_v2f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-SD-FP16-NEXT:    mov w8, v0.s[1]
-; CHECK-SD-FP16-NEXT:    fmov w9, s0
-; CHECK-SD-FP16-NEXT:    and w9, w9, #0xff
-; CHECK-SD-FP16-NEXT:    and w8, w8, #0xff
-; CHECK-SD-FP16-NEXT:    ucvtf h0, w9
-; CHECK-SD-FP16-NEXT:    ucvtf h1, w8
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], v1.h[0]
-; CHECK-SD-FP16-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v2i8_v2f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    movi d1, #0x0000ff000000ff
-; CHECK-GI-NOFP16-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.2s, v0.2s
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[0], v0.s[0]
-; CHECK-GI-NOFP16-NEXT:    mov v1.s[1], v0.s[1]
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v2i8_v2f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    movi d1, #0x0000ff000000ff
-; CHECK-GI-FP16-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-FP16-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v2i8_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-NOFP16-SD-NEXT:    fmov w9, s0
+; CHECK-NOFP16-SD-NEXT:    and w9, w9, #0xff
+; CHECK-NOFP16-SD-NEXT:    and w8, w8, #0xff
+; CHECK-NOFP16-SD-NEXT:    ucvtf s1, w9
+; CHECK-NOFP16-SD-NEXT:    ucvtf s0, w8
+; CHECK-NOFP16-SD-NEXT:    fcvt h2, s0
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], v2.h[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i8_v2f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-FP16-SD-NEXT:    fmov w9, s0
+; CHECK-FP16-SD-NEXT:    and w9, w9, #0xff
+; CHECK-FP16-SD-NEXT:    and w8, w8, #0xff
+; CHECK-FP16-SD-NEXT:    ucvtf h0, w9
+; CHECK-FP16-SD-NEXT:    ucvtf h1, w8
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i8_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-NOFP16-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.2s, v0.2s
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[0], v0.s[0]
+; CHECK-NOFP16-GI-NEXT:    mov v1.s[1], v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i8_v2f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    movi d1, #0x0000ff000000ff
+; CHECK-FP16-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <2 x i8> %a to <2 x half>
   ret <2 x half> %c
 }
 
 define <3 x half> @stofp_v3i8_v3f16(<3 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v3i8_v3f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fmov s0, w0
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], w1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], w2
-; CHECK-SD-NOFP16-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-SD-NOFP16-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-SD-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v3i8_v3f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fmov s0, w0
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], w1
-; CHECK-SD-FP16-NEXT:    mov v0.h[2], w2
-; CHECK-SD-FP16-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-SD-FP16-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-SD-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v3i8_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fmov s0, w0
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], w1
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], w2
-; CHECK-GI-NOFP16-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-GI-NOFP16-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-GI-NOFP16-NEXT:    mov v1.h[0], v0.h[2]
-; CHECK-GI-NOFP16-NEXT:    mov v1.h[1], v0.h[3]
-; CHECK-GI-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v3i8_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fmov s0, w0
-; CHECK-GI-FP16-NEXT:    mov v0.b[1], w1
-; CHECK-GI-FP16-NEXT:    mov v0.b[2], w2
-; CHECK-GI-FP16-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-GI-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v3i8_v3f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v3i8_v3f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], w1
+; CHECK-FP16-SD-NEXT:    mov v0.h[2], w2
+; CHECK-FP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-SD-NEXT:    scvtf v0.4h, v0.4h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v3i8_v3f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w0
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-GI-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-GI-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-GI-NEXT:    mov v1.h[0], v0.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov v1.h[1], v0.h[3]
+; CHECK-NOFP16-GI-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v3i8_v3f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov s0, w0
+; CHECK-FP16-GI-NEXT:    mov v0.b[1], w1
+; CHECK-FP16-GI-NEXT:    mov v0.b[2], w2
+; CHECK-FP16-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <3 x i8> %a to <3 x half>
   ret <3 x half> %c
 }
 
 define <3 x half> @utofp_v3i8_v3f16(<3 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v3i8_v3f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    fmov s0, w0
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[1], w1
-; CHECK-SD-NOFP16-NEXT:    mov v0.h[2], w2
-; CHECK-SD-NOFP16-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-SD-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v3i8_v3f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    fmov s0, w0
-; CHECK-SD-FP16-NEXT:    mov v0.h[1], w1
-; CHECK-SD-FP16-NEXT:    mov v0.h[2], w2
-; CHECK-SD-FP16-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-SD-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v3i8_v3f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    fmov s0, w0
-; CHECK-GI-NOFP16-NEXT:    movi d1, #0xff00ff00ff00ff
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[1], w1
-; CHECK-GI-NOFP16-NEXT:    mov v0.h[2], w2
-; CHECK-GI-NOFP16-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-NOFP16-NEXT:    mov v1.h[0], v0.h[2]
-; CHECK-GI-NOFP16-NEXT:    mov v1.h[1], v0.h[3]
-; CHECK-GI-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v3i8_v3f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    fmov s0, w0
-; CHECK-GI-FP16-NEXT:    mov v0.b[1], w1
-; CHECK-GI-FP16-NEXT:    mov v0.b[2], w2
-; CHECK-GI-FP16-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-GI-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v3i8_v3f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    fmov s0, w0
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v3i8_v3f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    fmov s0, w0
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], w1
+; CHECK-FP16-SD-NEXT:    mov v0.h[2], w2
+; CHECK-FP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4h, v0.4h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v3i8_v3f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    fmov s0, w0
+; CHECK-NOFP16-GI-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], w1
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[2], w2
+; CHECK-NOFP16-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NOFP16-GI-NEXT:    mov v1.h[0], v0.h[2]
+; CHECK-NOFP16-GI-NEXT:    mov v1.h[1], v0.h[3]
+; CHECK-NOFP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v3i8_v3f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    fmov s0, w0
+; CHECK-FP16-GI-NEXT:    mov v0.b[1], w1
+; CHECK-FP16-GI-NEXT:    mov v0.b[2], w2
+; CHECK-FP16-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <3 x i8> %a to <3 x half>
   ret <3 x half> %c
 }
 
 define <4 x half> @stofp_v4i8_v4f16(<4 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v4i8_v4f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-SD-NOFP16-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-SD-NOFP16-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v4i8_v4f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-SD-FP16-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-SD-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v4i8_v4f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    shl v0.4s, v0.4s, #24
-; CHECK-GI-NOFP16-NEXT:    sshr v0.4s, v0.4s, #24
-; CHECK-GI-NOFP16-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v4i8_v4f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    shl v0.4h, v0.4h, #8
-; CHECK-GI-FP16-NEXT:    sshr v0.4h, v0.4h, #8
-; CHECK-GI-FP16-NEXT:    scvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v4i8_v4f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-NOFP16-SD-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_v4i8_v4f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    shl v0.4h, v0.4h, #8
+; CHECK-FP16-NEXT:    sshr v0.4h, v0.4h, #8
+; CHECK-FP16-NEXT:    scvtf v0.4h, v0.4h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v4i8_v4f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    shl v0.4s, v0.4s, #24
+; CHECK-NOFP16-GI-NEXT:    sshr v0.4s, v0.4s, #24
+; CHECK-NOFP16-GI-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = sitofp <4 x i8> %a to <4 x half>
   ret <4 x half> %c
 }
 
 define <4 x half> @utofp_v4i8_v4f16(<4 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v4i8_v4f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-SD-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v4i8_v4f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    bic v0.4h, #255, lsl #8
-; CHECK-SD-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v4i8_v4f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    movi v1.2d, #0x0000ff000000ff
-; CHECK-GI-NOFP16-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    and v0.16b, v0.16b, v1.16b
-; CHECK-GI-NOFP16-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v4i8_v4f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    movi d1, #0xff00ff00ff00ff
-; CHECK-GI-FP16-NEXT:    and v0.8b, v0.8b, v1.8b
-; CHECK-GI-FP16-NEXT:    ucvtf v0.4h, v0.4h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v4i8_v4f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-NOFP16-SD-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v4i8_v4f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-FP16-SD-NEXT:    ucvtf v0.4h, v0.4h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v4i8_v4f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-NOFP16-GI-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NOFP16-GI-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v4i8_v4f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    movi d1, #0xff00ff00ff00ff
+; CHECK-FP16-GI-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-FP16-GI-NEXT:    ucvtf v0.4h, v0.4h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <4 x i8> %a to <4 x half>
   ret <4 x half> %c
 }
 
 define <8 x half> @stofp_v8i8_v8f16(<8 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v8i8_v8f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v2.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v1.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v8i8_v8f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-FP16-NEXT:    scvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v8i8_v8f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v1.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v8i8_v8f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-GI-FP16-NEXT:    scvtf v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v8i8_v8f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: stofp_v8i8_v8f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-FP16-NEXT:    scvtf v0.8h, v0.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v8i8_v8f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = sitofp <8 x i8> %a to <8 x half>
   ret <8 x half> %c
 }
 
 define <8 x half> @utofp_v8i8_v8f16(<8 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v8i8_v8f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v2.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v1.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v1.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v8i8_v8f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-SD-FP16-NEXT:    ucvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v8i8_v8f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v1.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v8i8_v8f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-GI-FP16-NEXT:    ucvtf v0.8h, v0.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v8i8_v8f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v2.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v1.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-LABEL: utofp_v8i8_v8f16:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-FP16-NEXT:    ucvtf v0.8h, v0.8h
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v8i8_v8f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v1.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ret
 entry:
   %c = uitofp <8 x i8> %a to <8 x half>
   ret <8 x half> %c
 }
 
 define <16 x half> @stofp_v16i8_v16f16(<16 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v16i8_v16f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sshll2 v1.8h, v0.16b, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v2.4s, v1.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v3.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v4.4s, v1.8h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v5.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v2.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v3.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v3.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v3.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v16i8_v16f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sshll2 v1.8h, v0.16b, #0
-; CHECK-SD-FP16-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-SD-FP16-NEXT:    scvtf v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    scvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v16i8_v16f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sshll v1.8h, v0.8b, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v2.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v3.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v4.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v5.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v16i8_v16f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    sshll v1.8h, v0.8b, #0
-; CHECK-GI-FP16-NEXT:    sshll2 v2.8h, v0.16b, #0
-; CHECK-GI-FP16-NEXT:    scvtf v0.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    scvtf v1.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v16i8_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll2 v1.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v3.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v4.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v5.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v2.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v3.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v16i8_v16f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll2 v1.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT:    sshll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT:    scvtf v1.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    scvtf v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v16i8_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v1.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v4.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v5.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v16i8_v16f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v1.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v2.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    scvtf v1.8h, v2.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <16 x i8> %a to <16 x half>
   ret <16 x half> %c
 }
 
 define <16 x half> @utofp_v16i8_v16f16(<16 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v16i8_v16f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ushll2 v1.8h, v0.16b, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v2.4s, v1.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v3.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v4.4s, v1.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v5.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v3.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v2.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v3.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v3.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v2.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v3.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v16i8_v16f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ushll2 v1.8h, v0.16b, #0
-; CHECK-SD-FP16-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-SD-FP16-NEXT:    ucvtf v1.8h, v1.8h
-; CHECK-SD-FP16-NEXT:    ucvtf v0.8h, v0.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v16i8_v16f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ushll v1.8h, v0.8b, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v2.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v3.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v3.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v4.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v5.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v2.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v3.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v5.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v16i8_v16f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ushll v1.8h, v0.8b, #0
-; CHECK-GI-FP16-NEXT:    ushll2 v2.8h, v0.16b, #0
-; CHECK-GI-FP16-NEXT:    ucvtf v0.8h, v1.8h
-; CHECK-GI-FP16-NEXT:    ucvtf v1.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v16i8_v16f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll2 v1.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v3.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v4.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v5.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v2.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v3.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v2.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v16i8_v16f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll2 v1.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v1.8h, v1.8h
+; CHECK-FP16-SD-NEXT:    ucvtf v0.8h, v0.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v16i8_v16f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v2.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v3.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v4.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v5.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v2.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v3.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v16i8_v16f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v2.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.8h, v1.8h
+; CHECK-FP16-GI-NEXT:    ucvtf v1.8h, v2.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <16 x i8> %a to <16 x half>
   ret <16 x half> %c
 }
 
 define <32 x half> @stofp_v32i8_v32f16(<32 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: stofp_v32i8_v32f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    sshll v2.8h, v0.8b, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v3.8h, v1.16b, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v4.4s, v2.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v5.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v6.4s, v3.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll v7.4s, v1.4h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v3.4s, v3.8h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v2.4s, v2.8h, #0
-; CHECK-SD-NOFP16-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-SD-NOFP16-NEXT:    scvtf v4.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v5.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v6.4s, v6.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v7.4s, v7.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v16.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v17.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v18.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    scvtf v19.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v6.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v7.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v18.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v16.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v17.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v19.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: stofp_v32i8_v32f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    sshll v2.8h, v0.8b, #0
-; CHECK-SD-FP16-NEXT:    sshll2 v3.8h, v0.16b, #0
-; CHECK-SD-FP16-NEXT:    sshll2 v4.8h, v1.16b, #0
-; CHECK-SD-FP16-NEXT:    sshll v5.8h, v1.8b, #0
-; CHECK-SD-FP16-NEXT:    scvtf v0.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    scvtf v1.8h, v3.8h
-; CHECK-SD-FP16-NEXT:    scvtf v3.8h, v4.8h
-; CHECK-SD-FP16-NEXT:    scvtf v2.8h, v5.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: stofp_v32i8_v32f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    sshll v2.8h, v0.8b, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v0.8h, v0.16b, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v3.8h, v1.8b, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v1.8h, v1.16b, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v4.4s, v2.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v5.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v6.4s, v3.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll v7.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v2.4s, v2.8h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v3.4s, v3.8h, #0
-; CHECK-GI-NOFP16-NEXT:    sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NOFP16-NEXT:    scvtf v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v5.4s, v5.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v6.4s, v6.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v7.4s, v7.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v16.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v17.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v18.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    scvtf v19.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v17.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v3.8h, v19.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: stofp_v32i8_v32f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    sshll v2.8h, v0.8b, #0
-; CHECK-GI-FP16-NEXT:    sshll2 v3.8h, v0.16b, #0
-; CHECK-GI-FP16-NEXT:    sshll v4.8h, v1.8b, #0
-; CHECK-GI-FP16-NEXT:    sshll2 v5.8h, v1.16b, #0
-; CHECK-GI-FP16-NEXT:    scvtf v0.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    scvtf v1.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    scvtf v2.8h, v4.8h
-; CHECK-GI-FP16-NEXT:    scvtf v3.8h, v5.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: stofp_v32i8_v32f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v3.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v4.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v5.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v6.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll v7.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    scvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v16.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v17.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v18.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    scvtf v19.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v3.4h, v6.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v2.4h, v7.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v18.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v16.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v3.8h, v17.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v2.8h, v19.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v32i8_v32f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v3.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT:    sshll2 v4.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT:    sshll v5.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT:    scvtf v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    scvtf v1.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    scvtf v3.8h, v4.8h
+; CHECK-FP16-SD-NEXT:    scvtf v2.8h, v5.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v32i8_v32f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v3.8h, v1.8b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.8h, v1.16b, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v4.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v5.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v6.4s, v3.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll v7.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-GI-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    scvtf v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v5.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v6.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v16.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v17.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v18.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    scvtf v19.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v2.4h, v6.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v3.4h, v7.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v16.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v17.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v2.8h, v18.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v3.8h, v19.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v32i8_v32f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sshll v2.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v3.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    sshll v4.8h, v1.8b, #0
+; CHECK-FP16-GI-NEXT:    sshll2 v5.8h, v1.16b, #0
+; CHECK-FP16-GI-NEXT:    scvtf v0.8h, v2.8h
+; CHECK-FP16-GI-NEXT:    scvtf v1.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    scvtf v2.8h, v4.8h
+; CHECK-FP16-GI-NEXT:    scvtf v3.8h, v5.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = sitofp <32 x i8> %a to <32 x half>
   ret <32 x half> %c
 }
 
 define <32 x half> @utofp_v32i8_v32f16(<32 x i8> %a) {
-; CHECK-SD-NOFP16-LABEL: utofp_v32i8_v32f16:
-; CHECK-SD-NOFP16:       // %bb.0: // %entry
-; CHECK-SD-NOFP16-NEXT:    ushll v2.8h, v0.8b, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v3.8h, v1.16b, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v4.4s, v2.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v5.4s, v0.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v6.4s, v3.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll v7.4s, v1.4h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v3.4s, v3.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v2.4s, v2.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-SD-NOFP16-NEXT:    ucvtf v4.4s, v4.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v5.4s, v5.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v6.4s, v6.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v7.4s, v7.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v16.4s, v0.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v17.4s, v3.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v18.4s, v2.4s
-; CHECK-SD-NOFP16-NEXT:    ucvtf v19.4s, v1.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v3.4h, v6.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn v2.4h, v7.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v0.8h, v18.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v1.8h, v16.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v3.8h, v17.4s
-; CHECK-SD-NOFP16-NEXT:    fcvtn2 v2.8h, v19.4s
-; CHECK-SD-NOFP16-NEXT:    ret
-;
-; CHECK-SD-FP16-LABEL: utofp_v32i8_v32f16:
-; CHECK-SD-FP16:       // %bb.0: // %entry
-; CHECK-SD-FP16-NEXT:    ushll v2.8h, v0.8b, #0
-; CHECK-SD-FP16-NEXT:    ushll2 v3.8h, v0.16b, #0
-; CHECK-SD-FP16-NEXT:    ushll2 v4.8h, v1.16b, #0
-; CHECK-SD-FP16-NEXT:    ushll v5.8h, v1.8b, #0
-; CHECK-SD-FP16-NEXT:    ucvtf v0.8h, v2.8h
-; CHECK-SD-FP16-NEXT:    ucvtf v1.8h, v3.8h
-; CHECK-SD-FP16-NEXT:    ucvtf v3.8h, v4.8h
-; CHECK-SD-FP16-NEXT:    ucvtf v2.8h, v5.8h
-; CHECK-SD-FP16-NEXT:    ret
-;
-; CHECK-GI-NOFP16-LABEL: utofp_v32i8_v32f16:
-; CHECK-GI-NOFP16:       // %bb.0: // %entry
-; CHECK-GI-NOFP16-NEXT:    ushll v2.8h, v0.8b, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v0.8h, v0.16b, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v3.8h, v1.8b, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v1.8h, v1.16b, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v4.4s, v2.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v5.4s, v0.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v6.4s, v3.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll v7.4s, v1.4h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v2.4s, v2.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v3.4s, v3.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NOFP16-NEXT:    ucvtf v4.4s, v4.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v5.4s, v5.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v6.4s, v6.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v7.4s, v7.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v16.4s, v2.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v17.4s, v0.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v18.4s, v3.4s
-; CHECK-GI-NOFP16-NEXT:    ucvtf v19.4s, v1.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v0.4h, v4.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v1.4h, v5.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v2.4h, v6.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn v3.4h, v7.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v0.8h, v16.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v1.8h, v17.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v2.8h, v18.4s
-; CHECK-GI-NOFP16-NEXT:    fcvtn2 v3.8h, v19.4s
-; CHECK-GI-NOFP16-NEXT:    ret
-;
-; CHECK-GI-FP16-LABEL: utofp_v32i8_v32f16:
-; CHECK-GI-FP16:       // %bb.0: // %entry
-; CHECK-GI-FP16-NEXT:    ushll v2.8h, v0.8b, #0
-; CHECK-GI-FP16-NEXT:    ushll2 v3.8h, v0.16b, #0
-; CHECK-GI-FP16-NEXT:    ushll v4.8h, v1.8b, #0
-; CHECK-GI-FP16-NEXT:    ushll2 v5.8h, v1.16b, #0
-; CHECK-GI-FP16-NEXT:    ucvtf v0.8h, v2.8h
-; CHECK-GI-FP16-NEXT:    ucvtf v1.8h, v3.8h
-; CHECK-GI-FP16-NEXT:    ucvtf v2.8h, v4.8h
-; CHECK-GI-FP16-NEXT:    ucvtf v3.8h, v5.8h
-; CHECK-GI-FP16-NEXT:    ret
+; CHECK-NOFP16-SD-LABEL: utofp_v32i8_v32f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v3.8h, v1.16b, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v4.4s, v2.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v5.4s, v0.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v6.4s, v3.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll v7.4s, v1.4h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-SD-NEXT:    ucvtf v4.4s, v4.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v5.4s, v5.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v6.4s, v6.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v7.4s, v7.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v16.4s, v0.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v17.4s, v3.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v18.4s, v2.4s
+; CHECK-NOFP16-SD-NEXT:    ucvtf v19.4s, v1.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v3.4h, v6.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn v2.4h, v7.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v0.8h, v18.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v1.8h, v16.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v3.8h, v17.4s
+; CHECK-NOFP16-SD-NEXT:    fcvtn2 v2.8h, v19.4s
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v32i8_v32f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v3.8h, v0.16b, #0
+; CHECK-FP16-SD-NEXT:    ushll2 v4.8h, v1.16b, #0
+; CHECK-FP16-SD-NEXT:    ushll v5.8h, v1.8b, #0
+; CHECK-FP16-SD-NEXT:    ucvtf v0.8h, v2.8h
+; CHECK-FP16-SD-NEXT:    ucvtf v1.8h, v3.8h
+; CHECK-FP16-SD-NEXT:    ucvtf v3.8h, v4.8h
+; CHECK-FP16-SD-NEXT:    ucvtf v2.8h, v5.8h
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v32i8_v32f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.8h, v0.16b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v3.8h, v1.8b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.8h, v1.16b, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v4.4s, v2.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v5.4s, v0.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v6.4s, v3.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll v7.4s, v1.4h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v2.4s, v2.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v0.4s, v0.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v3.4s, v3.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-NOFP16-GI-NEXT:    ucvtf v4.4s, v4.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v5.4s, v5.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v6.4s, v6.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v7.4s, v7.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v16.4s, v2.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v17.4s, v0.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v18.4s, v3.4s
+; CHECK-NOFP16-GI-NEXT:    ucvtf v19.4s, v1.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v0.4h, v4.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v1.4h, v5.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v2.4h, v6.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn v3.4h, v7.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v0.8h, v16.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v1.8h, v17.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v2.8h, v18.4s
+; CHECK-NOFP16-GI-NEXT:    fcvtn2 v3.8h, v19.4s
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v32i8_v32f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    ushll v2.8h, v0.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v3.8h, v0.16b, #0
+; CHECK-FP16-GI-NEXT:    ushll v4.8h, v1.8b, #0
+; CHECK-FP16-GI-NEXT:    ushll2 v5.8h, v1.16b, #0
+; CHECK-FP16-GI-NEXT:    ucvtf v0.8h, v2.8h
+; CHECK-FP16-GI-NEXT:    ucvtf v1.8h, v3.8h
+; CHECK-FP16-GI-NEXT:    ucvtf v2.8h, v4.8h
+; CHECK-FP16-GI-NEXT:    ucvtf v3.8h, v5.8h
+; CHECK-FP16-GI-NEXT:    ret
 entry:
   %c = uitofp <32 x i8> %a to <32 x half>
   ret <32 x half> %c
 }
+
+define <2 x half> @stofp_v2i128_v2f16(<2 x i128> %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_v2i128_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    mov x19, x1
+; CHECK-NOFP16-SD-NEXT:    mov x20, x0
+; CHECK-NOFP16-SD-NEXT:    mov x0, x2
+; CHECK-NOFP16-SD-NEXT:    mov x1, x3
+; CHECK-NOFP16-SD-NEXT:    bl __floattisf
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    mov x0, x20
+; CHECK-NOFP16-SD-NEXT:    mov x1, x19
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floattisf
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i128_v2f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    mov x19, x1
+; CHECK-FP16-SD-NEXT:    mov x20, x0
+; CHECK-FP16-SD-NEXT:    mov x0, x2
+; CHECK-FP16-SD-NEXT:    mov x1, x3
+; CHECK-FP16-SD-NEXT:    bl __floattihf
+; CHECK-FP16-SD-NEXT:    mov x0, x20
+; CHECK-FP16-SD-NEXT:    mov x1, x19
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floattihf
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i128_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-GI-NEXT:    mov x19, x2
+; CHECK-NOFP16-GI-NEXT:    mov x20, x3
+; CHECK-NOFP16-GI-NEXT:    bl __floattisf
+; CHECK-NOFP16-GI-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floattisf
+; CHECK-NOFP16-GI-NEXT:    fcvt h1, s0
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i128_v2f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-GI-NEXT:    mov x19, x2
+; CHECK-FP16-GI-NEXT:    mov x20, x3
+; CHECK-FP16-GI-NEXT:    bl __floattihf
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floattihf
+; CHECK-FP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v1.h[1], v0.h[0]
+; CHECK-FP16-GI-NEXT:    fmov d0, d1
+; CHECK-FP16-GI-NEXT:    add sp, sp, #48
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = sitofp <2 x i128> %a to <2 x half>
+  ret <2 x half> %c
+}
+
+define <2 x half> @utofp_v2i128_v2f16(<2 x i128> %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_v2i128_v2f16:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    mov x19, x1
+; CHECK-NOFP16-SD-NEXT:    mov x20, x0
+; CHECK-NOFP16-SD-NEXT:    mov x0, x2
+; CHECK-NOFP16-SD-NEXT:    mov x1, x3
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    mov x0, x20
+; CHECK-NOFP16-SD-NEXT:    mov x1, x19
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-SD-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i128_v2f16:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    mov x19, x1
+; CHECK-FP16-SD-NEXT:    mov x20, x0
+; CHECK-FP16-SD-NEXT:    mov x0, x2
+; CHECK-FP16-SD-NEXT:    mov x1, x3
+; CHECK-FP16-SD-NEXT:    bl __floatuntihf
+; CHECK-FP16-SD-NEXT:    mov x0, x20
+; CHECK-FP16-SD-NEXT:    mov x1, x19
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floatuntihf
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i128_v2f16:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-GI-NEXT:    mov x19, x2
+; CHECK-NOFP16-GI-NEXT:    mov x20, x3
+; CHECK-NOFP16-GI-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-GI-NEXT:    fcvt h0, s0
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floatuntisf
+; CHECK-NOFP16-GI-NEXT:    fcvt h1, s0
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i128_v2f16:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-GI-NEXT:    mov x19, x2
+; CHECK-FP16-GI-NEXT:    mov x20, x3
+; CHECK-FP16-GI-NEXT:    bl __floatuntihf
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floatuntihf
+; CHECK-FP16-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v1.h[1], v0.h[0]
+; CHECK-FP16-GI-NEXT:    fmov d0, d1
+; CHECK-FP16-GI-NEXT:    add sp, sp, #48
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = uitofp <2 x i128> %a to <2 x half>
+  ret <2 x half> %c
+}
+
+define <2 x fp128> @stofp_v2i64_v2f128(<2 x i64> %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_v2i64_v2f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    fmov x0, d0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floatditf
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov x0, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    bl __floatditf
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i64_v2f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    fmov x0, d0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floatditf
+; CHECK-FP16-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x0, v0.d[1]
+; CHECK-FP16-SD-NEXT:    bl __floatditf
+; CHECK-FP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i64_v2f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-NOFP16-GI-NEXT:    fmov x0, d0
+; CHECK-NOFP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    bl __floatditf
+; CHECK-NOFP16-GI-NEXT:    fmov x0, d8
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floatditf
+; CHECK-NOFP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i64_v2f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-FP16-GI-NEXT:    fmov x0, d0
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    bl __floatditf
+; CHECK-FP16-GI-NEXT:    fmov x0, d8
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floatditf
+; CHECK-FP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = sitofp <2 x i64> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @utofp_v2i64_v2f128(<2 x i64> %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_v2i64_v2f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    fmov x0, d0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floatunditf
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov x0, v0.d[1]
+; CHECK-NOFP16-SD-NEXT:    bl __floatunditf
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i64_v2f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    fmov x0, d0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floatunditf
+; CHECK-FP16-SD-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov x0, v0.d[1]
+; CHECK-FP16-SD-NEXT:    bl __floatunditf
+; CHECK-FP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i64_v2f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-NOFP16-GI-NEXT:    fmov x0, d0
+; CHECK-NOFP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-NOFP16-GI-NEXT:    bl __floatunditf
+; CHECK-NOFP16-GI-NEXT:    fmov x0, d8
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floatunditf
+; CHECK-NOFP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i64_v2f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-FP16-GI-NEXT:    fmov x0, d0
+; CHECK-FP16-GI-NEXT:    mov d8, v0.d[1]
+; CHECK-FP16-GI-NEXT:    bl __floatunditf
+; CHECK-FP16-GI-NEXT:    fmov x0, d8
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floatunditf
+; CHECK-FP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = uitofp <2 x i64> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @stofp_v2i32_v2f128(<2 x i32> %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_v2i32_v2f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    fmov w0, s0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floatsitf
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    mov w0, v1.s[1]
+; CHECK-NOFP16-SD-NEXT:    bl __floatsitf
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i32_v2f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #32
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    fmov w0, s0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floatsitf
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov w0, v1.s[1]
+; CHECK-FP16-SD-NEXT:    bl __floatsitf
+; CHECK-FP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #32
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i32_v2f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    fmov w0, s0
+; CHECK-NOFP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    bl __floatsitf
+; CHECK-NOFP16-GI-NEXT:    fmov w0, s8
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floatsitf
+; CHECK-NOFP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i32_v2f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    fmov w0, s0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    bl __floatsitf
+; CHECK-FP16-GI-NEXT:    fmov w0, s8
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floatsitf
+; CHECK-FP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = sitofp <2 x i32> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @utofp_v2i32_v2f128(<2 x i32> %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_v2i32_v2f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-NOFP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-SD-NEXT:    fmov w0, s0
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floatunsitf
+; CHECK-NOFP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    mov w0, v1.s[1]
+; CHECK-NOFP16-SD-NEXT:    bl __floatunsitf
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i32_v2f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #32
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -16
+; CHECK-FP16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-SD-NEXT:    fmov w0, s0
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floatunsitf
+; CHECK-FP16-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    mov w0, v1.s[1]
+; CHECK-FP16-SD-NEXT:    bl __floatunsitf
+; CHECK-FP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #32
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i32_v2f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-NOFP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP16-GI-NEXT:    fmov w0, s0
+; CHECK-NOFP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-NOFP16-GI-NEXT:    bl __floatunsitf
+; CHECK-NOFP16-GI-NEXT:    fmov w0, s8
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floatunsitf
+; CHECK-NOFP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #32
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i32_v2f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #32
+; CHECK-FP16-GI-NEXT:    str d8, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #24] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset b8, -16
+; CHECK-FP16-GI-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP16-GI-NEXT:    fmov w0, s0
+; CHECK-FP16-GI-NEXT:    mov s8, v0.s[1]
+; CHECK-FP16-GI-NEXT:    bl __floatunsitf
+; CHECK-FP16-GI-NEXT:    fmov w0, s8
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floatunsitf
+; CHECK-FP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #24] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr d8, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    add sp, sp, #32
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = uitofp <2 x i32> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @stofp_v2i16_v2f128(<2 x i16> %a) {
+; CHECK-LABEL: stofp_v2i16_v2f128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    sxth w0, w8
+; CHECK-NEXT:    bl __floatsitf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    sxth w0, w8
+; CHECK-NEXT:    bl __floatsitf
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+entry:
+  %c = sitofp <2 x i16> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @utofp_v2i16_v2f128(<2 x i16> %a) {
+; CHECK-LABEL: utofp_v2i16_v2f128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    bl __floatunsitf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    and w0, w8, #0xffff
+; CHECK-NEXT:    bl __floatunsitf
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+entry:
+  %c = uitofp <2 x i16> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @stofp_v2i8_v2f128(<2 x i8> %a) {
+; CHECK-LABEL: stofp_v2i8_v2f128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    sxtb w0, w8
+; CHECK-NEXT:    bl __floatsitf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    sxtb w0, w8
+; CHECK-NEXT:    bl __floatsitf
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+entry:
+  %c = sitofp <2 x i8> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @utofp_v2i8_v2f128(<2 x i8> %a) {
+; CHECK-LABEL: utofp_v2i8_v2f128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    bl __floatunsitf
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov w8, v1.s[1]
+; CHECK-NEXT:    and w0, w8, #0xff
+; CHECK-NEXT:    bl __floatunsitf
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+entry:
+  %c = uitofp <2 x i8> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @stofp_v2i128_v2f128(<2 x i128> %a) {
+; CHECK-NOFP16-SD-LABEL: stofp_v2i128_v2f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    mov x19, x3
+; CHECK-NOFP16-SD-NEXT:    mov x20, x2
+; CHECK-NOFP16-SD-NEXT:    bl __floattitf
+; CHECK-NOFP16-SD-NEXT:    mov x0, x20
+; CHECK-NOFP16-SD-NEXT:    mov x1, x19
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floattitf
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: stofp_v2i128_v2f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    mov x19, x3
+; CHECK-FP16-SD-NEXT:    mov x20, x2
+; CHECK-FP16-SD-NEXT:    bl __floattitf
+; CHECK-FP16-SD-NEXT:    mov x0, x20
+; CHECK-FP16-SD-NEXT:    mov x1, x19
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floattitf
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: stofp_v2i128_v2f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-GI-NEXT:    mov x19, x2
+; CHECK-NOFP16-GI-NEXT:    mov x20, x3
+; CHECK-NOFP16-GI-NEXT:    bl __floattitf
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floattitf
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: stofp_v2i128_v2f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-GI-NEXT:    mov x19, x2
+; CHECK-FP16-GI-NEXT:    mov x20, x3
+; CHECK-FP16-GI-NEXT:    bl __floattitf
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floattitf
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    add sp, sp, #48
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = sitofp <2 x i128> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
+
+define <2 x fp128> @utofp_v2i128_v2f128(<2 x i128> %a) {
+; CHECK-NOFP16-SD-LABEL: utofp_v2i128_v2f128:
+; CHECK-NOFP16-SD:       // %bb.0: // %entry
+; CHECK-NOFP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-SD-NEXT:    mov x19, x3
+; CHECK-NOFP16-SD-NEXT:    mov x20, x2
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntitf
+; CHECK-NOFP16-SD-NEXT:    mov x0, x20
+; CHECK-NOFP16-SD-NEXT:    mov x1, x19
+; CHECK-NOFP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-SD-NEXT:    bl __floatuntitf
+; CHECK-NOFP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-SD-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-SD-NEXT:    ret
+;
+; CHECK-FP16-SD-LABEL: utofp_v2i128_v2f128:
+; CHECK-FP16-SD:       // %bb.0: // %entry
+; CHECK-FP16-SD-NEXT:    sub sp, sp, #48
+; CHECK-FP16-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-SD-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-SD-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-SD-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-SD-NEXT:    mov x19, x3
+; CHECK-FP16-SD-NEXT:    mov x20, x2
+; CHECK-FP16-SD-NEXT:    bl __floatuntitf
+; CHECK-FP16-SD-NEXT:    mov x0, x20
+; CHECK-FP16-SD-NEXT:    mov x1, x19
+; CHECK-FP16-SD-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-SD-NEXT:    bl __floatuntitf
+; CHECK-FP16-SD-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-SD-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-SD-NEXT:    add sp, sp, #48
+; CHECK-FP16-SD-NEXT:    ret
+;
+; CHECK-NOFP16-GI-LABEL: utofp_v2i128_v2f128:
+; CHECK-NOFP16-GI:       // %bb.0: // %entry
+; CHECK-NOFP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-NOFP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-NOFP16-GI-NEXT:    mov x19, x2
+; CHECK-NOFP16-GI-NEXT:    mov x20, x3
+; CHECK-NOFP16-GI-NEXT:    bl __floatuntitf
+; CHECK-NOFP16-GI-NEXT:    mov x0, x19
+; CHECK-NOFP16-GI-NEXT:    mov x1, x20
+; CHECK-NOFP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NOFP16-GI-NEXT:    bl __floatuntitf
+; CHECK-NOFP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-NOFP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NOFP16-GI-NEXT:    add sp, sp, #48
+; CHECK-NOFP16-GI-NEXT:    ret
+;
+; CHECK-FP16-GI-LABEL: utofp_v2i128_v2f128:
+; CHECK-FP16-GI:       // %bb.0: // %entry
+; CHECK-FP16-GI-NEXT:    sub sp, sp, #48
+; CHECK-FP16-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-FP16-GI-NEXT:    .cfi_offset w19, -8
+; CHECK-FP16-GI-NEXT:    .cfi_offset w20, -16
+; CHECK-FP16-GI-NEXT:    .cfi_offset w30, -32
+; CHECK-FP16-GI-NEXT:    mov x19, x2
+; CHECK-FP16-GI-NEXT:    mov x20, x3
+; CHECK-FP16-GI-NEXT:    bl __floatuntitf
+; CHECK-FP16-GI-NEXT:    mov x0, x19
+; CHECK-FP16-GI-NEXT:    mov x1, x20
+; CHECK-FP16-GI-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-FP16-GI-NEXT:    bl __floatuntitf
+; CHECK-FP16-GI-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    mov v1.16b, v0.16b
+; CHECK-FP16-GI-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-FP16-GI-NEXT:    add sp, sp, #48
+; CHECK-FP16-GI-NEXT:    ret
+entry:
+  %c = uitofp <2 x i128> %a to <2 x fp128>
+  ret <2 x fp128> %c
+}
diff --git a/llvm/test/CodeGen/AArch64/jump-table-duplicate.mir b/llvm/test/CodeGen/AArch64/jump-table-duplicate.mir
index a2532a854923f..3d28909c37ad2 100644
--- a/llvm/test/CodeGen/AArch64/jump-table-duplicate.mir
+++ b/llvm/test/CodeGen/AArch64/jump-table-duplicate.mir
@@ -92,8 +92,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8, 
diff --git a/llvm/test/CodeGen/AArch64/ldexp-arm64ec.ll b/llvm/test/CodeGen/AArch64/ldexp-arm64ec.ll
index 1f8eeccf9c338..0fde7b95f5462 100644
--- a/llvm/test/CodeGen/AArch64/ldexp-arm64ec.ll
+++ b/llvm/test/CodeGen/AArch64/ldexp-arm64ec.ll
@@ -3,6 +3,15 @@
 
 ; Separate from ldexp.ll test because this errors on half cases
 
+; ARM64EC-LABEL: ldexp_f16 =
+; ARM64EC: fcvt d0, h0
+; ARM64EC: bl "#ldexp"
+; ARM64EC: fcvt h0, d0
+define half @ldexp_f16(half %val, i32 %a) {
+  %call = call half @llvm.ldexp.f16(half %val, i32 %a)
+  ret half %call
+}
+
 ; ARM64EC-LABEL: ldexp_f32 =
 ; ARM64EC: fcvt d0, s0
 ; ARM64EC: bl "#ldexp"
diff --git a/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir b/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
index a10d7588cb442..8a5e0f6aa843a 100644
--- a/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
+++ b/llvm/test/CodeGen/AArch64/ldrpre-ldr-merge.mir
@@ -756,7 +756,7 @@ body:             |
     ; CHECK: liveins: $x0, $x1, $x2
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: early-clobber renamable $x1, renamable $x0 = LDRSWpre renamable $x1, 40, implicit $w1, implicit $w1_hi :: (load (s32))
-    ; CHECK-NEXT: renamable $w2 = LDRWui renamable $x1, 1, implicit-def $x2, implicit $w2_hi :: (load (s32))
+    ; CHECK-NEXT: renamable $w2 = LDRWui renamable $x1, 1, implicit-def $x2 :: (load (s32))
     ; CHECK-NEXT: STPXi renamable $x0, renamable $x2, renamable $x1, 0 :: (store (s64))
     ; CHECK-NEXT: RET undef $lr
     early-clobber renamable $x1, renamable $x0 = LDRSWpre killed renamable $x1, 40 :: (load (s32))
diff --git a/llvm/test/CodeGen/AArch64/ldst-nopreidx-sp-redzone.mir b/llvm/test/CodeGen/AArch64/ldst-nopreidx-sp-redzone.mir
index f1f9e5fbc9b08..7304472f57641 100644
--- a/llvm/test/CodeGen/AArch64/ldst-nopreidx-sp-redzone.mir
+++ b/llvm/test/CodeGen/AArch64/ldst-nopreidx-sp-redzone.mir
@@ -103,8 +103,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  480
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: StackGuardSlot, type: default, offset: -40, size: 8, 
@@ -216,8 +216,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  480
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: StackGuardSlot, type: default, offset: -40, size: 8, 
@@ -327,8 +327,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  480
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: StackGuardSlot, type: default, offset: -40, size: 8, 
diff --git a/llvm/test/CodeGen/AArch64/ldst_update_cfpath.mir b/llvm/test/CodeGen/AArch64/ldst_update_cfpath.mir
index 8f541af2eb1a3..07c5adfc7ef31 100644
--- a/llvm/test/CodeGen/AArch64/ldst_update_cfpath.mir
+++ b/llvm/test/CodeGen/AArch64/ldst_update_cfpath.mir
@@ -97,8 +97,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: true
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/AArch64/lifetime-poison.ll b/llvm/test/CodeGen/AArch64/lifetime-poison.ll
index e04530de528b5..dfb76d1628bdd 100644
--- a/llvm/test/CodeGen/AArch64/lifetime-poison.ll
+++ b/llvm/test/CodeGen/AArch64/lifetime-poison.ll
@@ -8,7 +8,7 @@ define void @test() {
 ; CHECK-LABEL: test:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ret
-  call void @llvm.lifetime.start.p0(i64 4, ptr poison)
-  call void @llvm.lifetime.end.p0(i64 4, ptr poison)
+  call void @llvm.lifetime.start.p0(ptr poison)
+  call void @llvm.lifetime.end.p0(ptr poison)
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/live-debugvalues-sve.mir b/llvm/test/CodeGen/AArch64/live-debugvalues-sve.mir
index 612453ab53f43..1e14e7149817a 100644
--- a/llvm/test/CodeGen/AArch64/live-debugvalues-sve.mir
+++ b/llvm/test/CodeGen/AArch64/live-debugvalues-sve.mir
@@ -120,8 +120,10 @@ frameInfo:
   adjustsStack:    true
   hasCalls:        true
   maxCallFrameSize: 0
-  savePoint:       '%bb.1'
-  restorePoint:    '%bb.1'
+  savePoint:
+  - point: '%bb.1'
+  restorePoint:
+  - point: '%bb.1'
 stack:
   - { id: 0, size: 16, alignment: 16, stack-id: scalable-vector }
 machineFunctionInfo: {}
diff --git a/llvm/test/CodeGen/AArch64/llvm.frexp.ll b/llvm/test/CodeGen/AArch64/llvm.frexp.ll
index 2213aa1429dbd..4e1876db772ed 100644
--- a/llvm/test/CodeGen/AArch64/llvm.frexp.ll
+++ b/llvm/test/CodeGen/AArch64/llvm.frexp.ll
@@ -700,13 +700,14 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) nounwi
 ; CHECK-NEXT:    ldr s1, [sp, #44]
 ; CHECK-NEXT:    ldr q2, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
 ; CHECK-NEXT:    mov v2.s[3], v0.s[0]
-; CHECK-NEXT:    ld1 { v1.s }[2], [x20]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x19]
+; CHECK-NEXT:    ldr s0, [x20]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x21]
 ; CHECK-NEXT:    ldp x20, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    mov v0.16b, v2.16b
-; CHECK-NEXT:    ld1 { v1.s }[3], [x21]
 ; CHECK-NEXT:    ldp x30, x21, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    zip1 v1.2d, v1.2d, v0.2d
+; CHECK-NEXT:    mov v0.16b, v2.16b
 ; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
 ;
@@ -872,10 +873,11 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) nounwind {
 ; CHECK-NEXT:    bl frexpf
 ; CHECK-NEXT:    ldr s0, [sp, #28]
 ; CHECK-NEXT:    ld1 { v0.s }[1], [x19]
-; CHECK-NEXT:    ld1 { v0.s }[2], [x20]
+; CHECK-NEXT:    ldr s1, [x20]
+; CHECK-NEXT:    ld1 { v1.s }[1], [x21]
 ; CHECK-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ld1 { v0.s }[3], [x21]
 ; CHECK-NEXT:    ldp x30, x21, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    zip1 v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    add sp, sp, #64
 ; CHECK-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/AArch64/logical_shifted_reg.ll b/llvm/test/CodeGen/AArch64/logical_shifted_reg.ll
index 81f13b8e7439a..3b43b4209b08f 100644
--- a/llvm/test/CodeGen/AArch64/logical_shifted_reg.ll
+++ b/llvm/test/CodeGen/AArch64/logical_shifted_reg.ll
@@ -256,7 +256,7 @@ define void @flag_setting() {
 ; CHECK-SD-NEXT:    b.gt .LBB2_4
 ; CHECK-SD-NEXT:  // %bb.1: // %test2
 ; CHECK-SD-NEXT:    tst x9, x10, lsl #63
-; CHECK-SD-NEXT:    b.lt .LBB2_4
+; CHECK-SD-NEXT:    b.mi .LBB2_4
 ; CHECK-SD-NEXT:  // %bb.2: // %test3
 ; CHECK-SD-NEXT:    tst x9, x10, asr #12
 ; CHECK-SD-NEXT:    b.gt .LBB2_4
@@ -277,7 +277,7 @@ define void @flag_setting() {
 ; CHECK-GI-NEXT:    b.gt .LBB2_4
 ; CHECK-GI-NEXT:  // %bb.1: // %test2
 ; CHECK-GI-NEXT:    tst x9, x10, lsl #63
-; CHECK-GI-NEXT:    b.lt .LBB2_4
+; CHECK-GI-NEXT:    b.mi .LBB2_4
 ; CHECK-GI-NEXT:  // %bb.2: // %test3
 ; CHECK-GI-NEXT:    asr x10, x10, #12
 ; CHECK-GI-NEXT:    tst x10, x9
diff --git a/llvm/test/CodeGen/AArch64/loop-sink-limit.mir b/llvm/test/CodeGen/AArch64/loop-sink-limit.mir
index 3e18f95cab848..4293006072c73 100644
--- a/llvm/test/CodeGen/AArch64/loop-sink-limit.mir
+++ b/llvm/test/CodeGen/AArch64/loop-sink-limit.mir
@@ -79,8 +79,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/loop-sink.mir b/llvm/test/CodeGen/AArch64/loop-sink.mir
index de81d99369394..7e081ecf4c6f5 100644
--- a/llvm/test/CodeGen/AArch64/loop-sink.mir
+++ b/llvm/test/CodeGen/AArch64/loop-sink.mir
@@ -300,8 +300,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -548,8 +548,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -675,8 +675,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -805,8 +805,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -945,8 +945,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -1071,8 +1071,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -1212,8 +1212,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -1359,8 +1359,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/machine-latecleanup-inlineasm.mir b/llvm/test/CodeGen/AArch64/machine-latecleanup-inlineasm.mir
index c5f871a37d5f6..91fc32721d3df 100644
--- a/llvm/test/CodeGen/AArch64/machine-latecleanup-inlineasm.mir
+++ b/llvm/test/CodeGen/AArch64/machine-latecleanup-inlineasm.mir
@@ -90,8 +90,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/AArch64/min-max-combine.ll b/llvm/test/CodeGen/AArch64/min-max-combine.ll
index 5111f838b73aa..558d4b8b499ba 100644
--- a/llvm/test/CodeGen/AArch64/min-max-combine.ll
+++ b/llvm/test/CodeGen/AArch64/min-max-combine.ll
@@ -123,7 +123,7 @@ define i8 @smini8_zero(i8 %a) {
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxtb w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, #0
-; CHECK-GLOBAL-NEXT:    csel w0, w0, wzr, lt
+; CHECK-GLOBAL-NEXT:    csel w0, w0, wzr, mi
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i8 @llvm.smin.i8(i8 %a, i8 0)
   ret i8 %c
@@ -148,7 +148,7 @@ define i16 @smini16_zero(i16 %a) {
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    sxth w8, w0
 ; CHECK-GLOBAL-NEXT:    cmp w8, #0
-; CHECK-GLOBAL-NEXT:    csel w0, w0, wzr, lt
+; CHECK-GLOBAL-NEXT:    csel w0, w0, wzr, mi
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i16 @llvm.smin.i16(i16 %a, i16 0)
   ret i16 %c
@@ -170,7 +170,7 @@ define i32 @smini32_zero(i32 %a) {
 ; CHECK-GLOBAL-LABEL: smini32_zero:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp w0, #0
-; CHECK-GLOBAL-NEXT:    csel w0, w0, wzr, lt
+; CHECK-GLOBAL-NEXT:    csel w0, w0, wzr, mi
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i32 @llvm.smin.i32(i32 %a, i32 0)
   ret i32 %c
@@ -192,7 +192,7 @@ define i64 @smini64_zero(i64 %a) {
 ; CHECK-GLOBAL-LABEL: smini64_zero:
 ; CHECK-GLOBAL:       // %bb.0:
 ; CHECK-GLOBAL-NEXT:    cmp x0, #0
-; CHECK-GLOBAL-NEXT:    csel x0, x0, xzr, lt
+; CHECK-GLOBAL-NEXT:    csel x0, x0, xzr, mi
 ; CHECK-GLOBAL-NEXT:    ret
   %c = call i64 @llvm.smin.i64(i64 %a, i64 0)
   ret i64 %c
diff --git a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
index 69e805d9ca2ee..72d839a21a29f 100644
--- a/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
+++ b/llvm/test/CodeGen/AArch64/named-vector-shuffles-sve.ll
@@ -33,6 +33,16 @@ define <vscale x 16 x i8> @splice_nxv16i8_last_idx(<vscale x 16 x i8> %a, <vscal
   ret <vscale x 16 x i8> %res
 }
 
+define <vscale x 16 x i8> @splice_nxv16i8_first_idx_unary(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
+; CHECK-LABEL: splice_nxv16i8_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #1
+; CHECK-NEXT:    ret
+  %res = call <vscale x 16 x i8> @llvm.vector.splice.nxv16i8(<vscale x 16 x i8> %b, <vscale x 16 x i8> %b, i32 1)
+  ret <vscale x 16 x i8> %res
+}
+
 define <vscale x 8 x i16> @splice_nxv8i16_first_idx(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
 ; CHECK-LABEL: splice_nxv8i16_first_idx:
 ; CHECK:       // %bb.0:
@@ -42,6 +52,16 @@ define <vscale x 8 x i16> @splice_nxv8i16_first_idx(<vscale x 8 x i16> %a, <vsca
   ret <vscale x 8 x i16> %res
 }
 
+define <vscale x 8 x i16> @splice_nxv8i16_first_idx_unary(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
+; CHECK-LABEL: splice_nxv8i16_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #2
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x i16> @llvm.vector.splice.nxv8i16(<vscale x 8 x i16> %b, <vscale x 8 x i16> %b, i32 1)
+  ret <vscale x 8 x i16> %res
+}
+
 define <vscale x 4 x i32> @splice_nxv4i32_first_idx(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
 ; CHECK-LABEL: splice_nxv4i32_first_idx:
 ; CHECK:       // %bb.0:
@@ -60,6 +80,16 @@ define <vscale x 4 x i32> @splice_nxv4i32_last_idx(<vscale x 4 x i32> %a, <vscal
   ret <vscale x 4 x i32> %res
 }
 
+define <vscale x 4 x i32> @splice_nxv4i32_first_idx_unary(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
+; CHECK-LABEL: splice_nxv4i32_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #4
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x i32> @llvm.vector.splice.nxv4i32(<vscale x 4 x i32> %b, <vscale x 4 x i32> %b, i32 1)
+  ret <vscale x 4 x i32> %res
+}
+
 define <vscale x 2 x i64> @splice_nxv2i64_first_idx(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
 ; CHECK-LABEL: splice_nxv2i64_first_idx:
 ; CHECK:       // %bb.0:
@@ -78,6 +108,16 @@ define <vscale x 2 x i64> @splice_nxv2i64_last_idx(<vscale x 2 x i64> %a, <vscal
   ret <vscale x 2 x i64> %res
 }
 
+define <vscale x 2 x i64> @splice_nxv2i64_first_idx_unary(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
+; CHECK-LABEL: splice_nxv2i64_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #8
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> %b, <vscale x 2 x i64> %b, i32 1)
+  ret <vscale x 2 x i64> %res
+}
+
 define <vscale x 2 x half> @splice_nxv2f16_neg_idx(<vscale x 2 x half> %a, <vscale x 2 x half> %b) #0 {
 ; CHECK-LABEL: splice_nxv2f16_neg_idx:
 ; CHECK:       // %bb.0:
@@ -100,6 +140,18 @@ define <vscale x 2 x half> @splice_nxv2f16_neg2_idx(<vscale x 2 x half> %a, <vsc
   ret <vscale x 2 x half> %res
 }
 
+define <vscale x 2 x half> @splice_nxv2f16_neg_idx_unary(<vscale x 2 x half> %a, <vscale x 2 x half> %b) #0 {
+; CHECK-LABEL: splice_nxv2f16_neg_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    rev p0.d, p0.d
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> %b, <vscale x 2 x half> %b, i32 -1)
+  ret <vscale x 2 x half> %res
+}
+
 define <vscale x 2 x half> @splice_nxv2f16_first_idx(<vscale x 2 x half> %a, <vscale x 2 x half> %b) #0 {
 ; CHECK-LABEL: splice_nxv2f16_first_idx:
 ; CHECK:       // %bb.0:
@@ -118,6 +170,16 @@ define <vscale x 2 x half> @splice_nxv2f16_last_idx(<vscale x 2 x half> %a, <vsc
   ret <vscale x 2 x half> %res
 }
 
+define <vscale x 2 x half> @splice_nxv2f16_first_idx_unary(<vscale x 2 x half> %a, <vscale x 2 x half> %b) #0 {
+; CHECK-LABEL: splice_nxv2f16_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #8
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x half> @llvm.vector.splice.nxv2f16(<vscale x 2 x half> %b, <vscale x 2 x half> %b, i32 1)
+  ret <vscale x 2 x half> %res
+}
+
 define <vscale x 4 x half> @splice_nxv4f16_neg_idx(<vscale x 4 x half> %a, <vscale x 4 x half> %b) #0 {
 ; CHECK-LABEL: splice_nxv4f16_neg_idx:
 ; CHECK:       // %bb.0:
@@ -140,6 +202,18 @@ define <vscale x 4 x half> @splice_nxv4f16_neg3_idx(<vscale x 4 x half> %a, <vsc
   ret <vscale x 4 x half> %res
 }
 
+define <vscale x 4 x half> @splice_nxv4f16_neg_idx_unary(<vscale x 4 x half> %a, <vscale x 4 x half> %b) #0 {
+; CHECK-LABEL: splice_nxv4f16_neg_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl1
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    rev p0.s, p0.s
+; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> %b, <vscale x 4 x half> %b, i32 -1)
+  ret <vscale x 4 x half> %res
+}
+
 define <vscale x 4 x half> @splice_nxv4f16_first_idx(<vscale x 4 x half> %a, <vscale x 4 x half> %b) #0 {
 ; CHECK-LABEL: splice_nxv4f16_first_idx:
 ; CHECK:       // %bb.0:
@@ -158,6 +232,16 @@ define <vscale x 4 x half> @splice_nxv4f16_last_idx(<vscale x 4 x half> %a, <vsc
   ret <vscale x 4 x half> %res
 }
 
+define <vscale x 4 x half> @splice_nxv4f16_first_idx_unary(<vscale x 4 x half> %a, <vscale x 4 x half> %b) #0 {
+; CHECK-LABEL: splice_nxv4f16_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #4
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x half> @llvm.vector.splice.nxv4f16(<vscale x 4 x half> %b, <vscale x 4 x half> %b, i32 1)
+  ret <vscale x 4 x half> %res
+}
+
 define <vscale x 8 x half> @splice_nxv8f16_first_idx(<vscale x 8 x half> %a, <vscale x 8 x half> %b) #0 {
 ; CHECK-LABEL: splice_nxv8f16_first_idx:
 ; CHECK:       // %bb.0:
@@ -176,6 +260,16 @@ define <vscale x 8 x half> @splice_nxv8f16_last_idx(<vscale x 8 x half> %a, <vsc
   ret <vscale x 8 x half> %res
 }
 
+define <vscale x 8 x half> @splice_nxv8f16_first_idx_unary(<vscale x 8 x half> %a, <vscale x 8 x half> %b) #0 {
+; CHECK-LABEL: splice_nxv8f16_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #2
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x half> @llvm.vector.splice.nxv8f16(<vscale x 8 x half> %b, <vscale x 8 x half> %b, i32 1)
+  ret <vscale x 8 x half> %res
+}
+
 define <vscale x 2 x float> @splice_nxv2f32_neg_idx(<vscale x 2 x float> %a, <vscale x 2 x float> %b) #0 {
 ; CHECK-LABEL: splice_nxv2f32_neg_idx:
 ; CHECK:       // %bb.0:
@@ -198,6 +292,18 @@ define <vscale x 2 x float> @splice_nxv2f32_neg2_idx(<vscale x 2 x float> %a, <v
   ret <vscale x 2 x float> %res
 }
 
+define <vscale x 2 x float> @splice_nxv2f32_neg_idx_unary(<vscale x 2 x float> %a, <vscale x 2 x float> %b) #0 {
+; CHECK-LABEL: splice_nxv2f32_neg_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    rev p0.d, p0.d
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> %b, <vscale x 2 x float> %b, i32 -1)
+  ret <vscale x 2 x float> %res
+}
+
 define <vscale x 2 x float> @splice_nxv2f32_first_idx(<vscale x 2 x float> %a, <vscale x 2 x float> %b) #0 {
 ; CHECK-LABEL: splice_nxv2f32_first_idx:
 ; CHECK:       // %bb.0:
@@ -216,6 +322,16 @@ define <vscale x 2 x float> @splice_nxv2f32_last_idx(<vscale x 2 x float> %a, <v
   ret <vscale x 2 x float> %res
 }
 
+define <vscale x 2 x float> @splice_nxv2f32_first_idx_unary(<vscale x 2 x float> %a, <vscale x 2 x float> %b) #0 {
+; CHECK-LABEL: splice_nxv2f32_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #8
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x float> @llvm.vector.splice.nxv2f32(<vscale x 2 x float> %b, <vscale x 2 x float> %b, i32 1)
+  ret <vscale x 2 x float> %res
+}
+
 define <vscale x 4 x float> @splice_nxv4f32_first_idx(<vscale x 4 x float> %a, <vscale x 4 x float> %b) #0 {
 ; CHECK-LABEL: splice_nxv4f32_first_idx:
 ; CHECK:       // %bb.0:
@@ -234,6 +350,16 @@ define <vscale x 4 x float> @splice_nxv4f32_last_idx(<vscale x 4 x float> %a, <v
   ret <vscale x 4 x float> %res
 }
 
+define <vscale x 4 x float> @splice_nxv4f32_first_idx_unary(<vscale x 4 x float> %a, <vscale x 4 x float> %b) #0 {
+; CHECK-LABEL: splice_nxv4f32_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #4
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x float> @llvm.vector.splice.nxv4f32(<vscale x 4 x float> %b, <vscale x 4 x float> %b, i32 1)
+  ret <vscale x 4 x float> %res
+}
+
 define <vscale x 2 x double> @splice_nxv2f64_first_idx(<vscale x 2 x double> %a, <vscale x 2 x double> %b) #0 {
 ; CHECK-LABEL: splice_nxv2f64_first_idx:
 ; CHECK:       // %bb.0:
@@ -252,6 +378,16 @@ define <vscale x 2 x double> @splice_nxv2f64_last_idx(<vscale x 2 x double> %a,
   ret <vscale x 2 x double> %res
 }
 
+define <vscale x 2 x double> @splice_nxv2f64_first_idx_unary(<vscale x 2 x double> %a, <vscale x 2 x double> %b) #0 {
+; CHECK-LABEL: splice_nxv2f64_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #8
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x double> @llvm.vector.splice.nxv2f64(<vscale x 2 x double> %b, <vscale x 2 x double> %b, i32 1)
+  ret <vscale x 2 x double> %res
+}
+
 ; Ensure predicate based splice is promoted to use ZPRs.
 define <vscale x 2 x i1> @splice_nxv2i1_idx(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv2i1_idx:
@@ -710,6 +846,18 @@ define <vscale x 2 x bfloat> @splice_nxv2bf16_neg2_idx(<vscale x 2 x bfloat> %a,
   ret <vscale x 2 x bfloat> %res
 }
 
+define <vscale x 2 x bfloat> @splice_nxv2bf16_neg_idx_unary(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv2bf16_neg_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl1
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    rev p0.d, p0.d
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> %b, <vscale x 2 x bfloat> %b, i32 -1)
+  ret <vscale x 2 x bfloat> %res
+}
+
 define <vscale x 2 x bfloat> @splice_nxv2bf16_first_idx(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) #0 {
 ; CHECK-LABEL: splice_nxv2bf16_first_idx:
 ; CHECK:       // %bb.0:
@@ -728,6 +876,16 @@ define <vscale x 2 x bfloat> @splice_nxv2bf16_last_idx(<vscale x 2 x bfloat> %a,
   ret <vscale x 2 x bfloat> %res
 }
 
+define <vscale x 2 x bfloat> @splice_nxv2bf16_first_idx_unary(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv2bf16_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #8
+; CHECK-NEXT:    ret
+  %res = call <vscale x 2 x bfloat> @llvm.vector.splice.nxv2bf16(<vscale x 2 x bfloat> %b, <vscale x 2 x bfloat> %b, i32 1)
+  ret <vscale x 2 x bfloat> %res
+}
+
 define <vscale x 4 x bfloat> @splice_nxv4bf16_neg_idx(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) #0 {
 ; CHECK-LABEL: splice_nxv4bf16_neg_idx:
 ; CHECK:       // %bb.0:
@@ -750,6 +908,18 @@ define <vscale x 4 x bfloat> @splice_nxv4bf16_neg3_idx(<vscale x 4 x bfloat> %a,
   ret <vscale x 4 x bfloat> %res
 }
 
+define <vscale x 4 x bfloat> @splice_nxv4bf16_neg_idx_unary(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv4bf16_neg_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s, vl1
+; CHECK-NEXT:    mov z0.d, z1.d
+; CHECK-NEXT:    rev p0.s, p0.s
+; CHECK-NEXT:    splice z0.s, p0, z0.s, z1.s
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %b, i32 -1)
+  ret <vscale x 4 x bfloat> %res
+}
+
 define <vscale x 4 x bfloat> @splice_nxv4bf16_first_idx(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) #0 {
 ; CHECK-LABEL: splice_nxv4bf16_first_idx:
 ; CHECK:       // %bb.0:
@@ -768,6 +938,16 @@ define <vscale x 4 x bfloat> @splice_nxv4bf16_last_idx(<vscale x 4 x bfloat> %a,
   ret <vscale x 4 x bfloat> %res
 }
 
+define <vscale x 4 x bfloat> @splice_nxv4bf16_first_idx_unary(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv4bf16_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #4
+; CHECK-NEXT:    ret
+  %res = call <vscale x 4 x bfloat> @llvm.vector.splice.nxv4bf16(<vscale x 4 x bfloat> %b, <vscale x 4 x bfloat> %b, i32 1)
+  ret <vscale x 4 x bfloat> %res
+}
+
 define <vscale x 8 x bfloat> @splice_nxv8bf16_first_idx(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
 ; CHECK-LABEL: splice_nxv8bf16_first_idx:
 ; CHECK:       // %bb.0:
@@ -786,6 +966,16 @@ define <vscale x 8 x bfloat> @splice_nxv8bf16_last_idx(<vscale x 8 x bfloat> %a,
   ret <vscale x 8 x bfloat> %res
 }
 
+define <vscale x 8 x bfloat> @splice_nxv8bf16_first_idx_unary(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: splice_nxv8bf16_first_idx_unary:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #2
+; CHECK-NEXT:    ret
+  %res = call <vscale x 8 x bfloat> @llvm.vector.splice.nxv8bf16(<vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %b, i32 1)
+  ret <vscale x 8 x bfloat> %res
+}
+
 ; Ensure predicate based splice is promoted to use ZPRs.
 define <vscale x 2 x i1> @splice_nxv2i1(<vscale x 2 x i1> %a, <vscale x 2 x i1> %b) #0 {
 ; CHECK-LABEL: splice_nxv2i1:
diff --git a/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll b/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
index 7e6f3548bdaf5..0c84468f3934b 100644
--- a/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/neon-bitwise-instructions.ll
@@ -884,8 +884,10 @@ define <2 x i64> @orn2xi64(<2 x i64> %a, <2 x i64> %b)  {
 define <2 x i32> @bsl2xi32_const(<2 x i32> %a, <2 x i32> %b)  {
 ; CHECK-SD-LABEL: bsl2xi32_const:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    movi d2, #0x000000ffffffff
-; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    mov v0.s[1], v1.s[1]
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: bsl2xi32_const:
@@ -923,8 +925,10 @@ define <4 x i16> @bsl4xi16_const(<4 x i16> %a, <4 x i16> %b)  {
 define <1 x i64> @bsl1xi64_const(<1 x i64> %a, <1 x i64> %b)  {
 ; CHECK-SD-LABEL: bsl1xi64_const:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    movi d2, #0xffffffffffffff00
-; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    mov v0.b[0], v1.b[0]
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: bsl1xi64_const:
@@ -981,12 +985,17 @@ define <8 x i16> @bsl8xi16_const(<8 x i16> %a, <8 x i16> %b)  {
 }
 
 define <2 x i64> @bsl2xi64_const(<2 x i64> %a, <2 x i64> %b)  {
-; CHECK-LABEL: bsl2xi64_const:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    adrp x8, .LCPI75_0
-; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI75_0]
-; CHECK-NEXT:    bif v0.16b, v1.16b, v2.16b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: bsl2xi64_const:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[1]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: bsl2xi64_const:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    adrp x8, .LCPI75_0
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI75_0]
+; CHECK-GI-NEXT:    bif v0.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    ret
 	%tmp1 = and <2 x i64> %a, < i64 -1, i64 0 >
 	%tmp2 = and <2 x i64> %b, < i64 0, i64 -1 >
 	%tmp3 = or <2 x i64> %tmp1, %tmp2
@@ -1158,11 +1167,8 @@ define <4 x i32> @vselect_constant_cond_zero_v4i32(<4 x i32> %a) {
 define <8 x i8> @vselect_constant_cond_v8i8(<8 x i8> %a, <8 x i8> %b) {
 ; CHECK-SD-LABEL: vselect_constant_cond_v8i8:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    movi d2, #0xffffffffff00ff00
-; CHECK-SD-NEXT:    movi d3, #0x00000000ff00ff
-; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
-; CHECK-SD-NEXT:    and v0.8b, v0.8b, v3.8b
-; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    movi d2, #0x00000000ff00ff
+; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vselect_constant_cond_v8i8:
@@ -1180,11 +1186,8 @@ define <8 x i8> @vselect_constant_cond_v8i8(<8 x i8> %a, <8 x i8> %b) {
 define <4 x i16> @vselect_constant_cond_v4i16(<4 x i16> %a, <4 x i16> %b) {
 ; CHECK-SD-LABEL: vselect_constant_cond_v4i16:
 ; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    movi d2, #0x00ffffffff0000
-; CHECK-SD-NEXT:    movi d3, #0xffff00000000ffff
-; CHECK-SD-NEXT:    and v1.8b, v1.8b, v2.8b
-; CHECK-SD-NEXT:    and v0.8b, v0.8b, v3.8b
-; CHECK-SD-NEXT:    orr v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    movi d2, #0xffff00000000ffff
+; CHECK-SD-NEXT:    bif v0.8b, v1.8b, v2.8b
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: vselect_constant_cond_v4i16:
@@ -2839,3 +2842,33 @@ define <2 x i64> @orr64imm8h_lsl8(<2 x i64> %a) {
 	ret <2 x i64> %tmp1
 }
 
+define <8 x i16> @pr149380(<4 x i16> %u1, <1 x i64> %u2, <8 x i16> %vqshlu_n169) {
+; CHECK-SD-LABEL: pr149380:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v0.8h, #1
+; CHECK-SD-NEXT:    orr v2.8h, #1
+; CHECK-SD-NEXT:    sqadd v0.8h, v2.8h, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: pr149380:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v0.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    neg v1.8h, v2.8h
+; CHECK-GI-NEXT:    movi v3.8h, #1
+; CHECK-GI-NEXT:    neg v1.8h, v1.8h
+; CHECK-GI-NEXT:    sub v0.8h, v0.8h, v2.8h
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    and v0.16b, v0.16b, v3.16b
+; CHECK-GI-NEXT:    orr v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    sqadd v0.8h, v3.8h, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %mul.i = mul <8 x i16> %vqshlu_n169, < i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1 >
+  %sub.i = sub <8 x i16> zeroinitializer, %mul.i
+  %vbsl3.i = and <8 x i16> %sub.i, %vqshlu_n169
+  %0 = add <8 x i16> %mul.i, < i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1 >
+  %vbsl4.i = and <8 x i16> %0, < i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1 >
+  %vbsl5.i = or <8 x i16> %vbsl3.i, %vbsl4.i
+  %vqaddq_v2.i26515 = tail call <8 x i16> @llvm.aarch64.neon.sqadd.v8i16(<8 x i16> < i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1 >, <8 x i16> %vbsl5.i)
+  ret <8 x i16> %vqaddq_v2.i26515
+}
diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
index 048e988b6c497..88b6f6c40baca 100644
--- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll
@@ -8062,195 +8062,200 @@ define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %
 ; CHECK-SD-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-SD-NEXT:    .cfi_offset w29, -16
-; CHECK-SD-NEXT:    ldr b5, [sp, #208]
+; CHECK-SD-NEXT:    ldr b0, [sp, #208]
 ; CHECK-SD-NEXT:    add x8, sp, #216
-; CHECK-SD-NEXT:    fmov s0, w0
+; CHECK-SD-NEXT:    add x9, sp, #272
+; CHECK-SD-NEXT:    ldr b2, [sp, #80]
 ; CHECK-SD-NEXT:    ldr b4, [sp, #976]
-; CHECK-SD-NEXT:    add x9, sp, #984
-; CHECK-SD-NEXT:    add x12, sp, #328
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #224
-; CHECK-SD-NEXT:    movi v1.16b, #1
-; CHECK-SD-NEXT:    mov v0.b[1], w1
-; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x9]
-; CHECK-SD-NEXT:    movi v3.2d, #0000000000000000
-; CHECK-SD-NEXT:    add x11, sp, #992
 ; CHECK-SD-NEXT:    ldr b6, [sp, #720]
-; CHECK-SD-NEXT:    ldr b7, [sp, #80]
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT:    ld1 { v0.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #224
+; CHECK-SD-NEXT:    fmov s16, w0
+; CHECK-SD-NEXT:    ldr b17, [sp, #848]
+; CHECK-SD-NEXT:    add x10, sp, #24
+; CHECK-SD-NEXT:    movi v19.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v0.b }[2], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #232
-; CHECK-SD-NEXT:    add x13, sp, #88
-; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x11]
-; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #856
-; CHECK-SD-NEXT:    mov v0.b[2], w2
-; CHECK-SD-NEXT:    add x14, sp, #1008
-; CHECK-SD-NEXT:    add x15, sp, #872
-; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x8]
+; CHECK-SD-NEXT:    mov v16.b[1], w1
+; CHECK-SD-NEXT:    ld1 { v0.b }[3], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #240
-; CHECK-SD-NEXT:    add x16, sp, #888
-; CHECK-SD-NEXT:    add x10, sp, #16
-; CHECK-SD-NEXT:    add x9, sp, #24
-; CHECK-SD-NEXT:    add x11, sp, #40
-; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT:    mov v16.b[2], w2
+; CHECK-SD-NEXT:    ld1 { v0.b }[4], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #248
-; CHECK-SD-NEXT:    mov v0.b[3], w3
-; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x8]
+; CHECK-SD-NEXT:    mov v16.b[3], w3
+; CHECK-SD-NEXT:    ld1 { v0.b }[5], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #256
-; CHECK-SD-NEXT:    mov v0.b[4], w4
-; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x8]
+; CHECK-SD-NEXT:    ld1 { v0.b }[6], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #264
-; CHECK-SD-NEXT:    mov v0.b[5], w5
-; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #272
-; CHECK-SD-NEXT:    ld1 { v5.b }[8], [x8]
+; CHECK-SD-NEXT:    mov v16.b[4], w4
+; CHECK-SD-NEXT:    ld1 { v0.b }[7], [x8]
+; CHECK-SD-NEXT:    ldr b1, [x9]
 ; CHECK-SD-NEXT:    add x8, sp, #280
-; CHECK-SD-NEXT:    mov v0.b[6], w6
-; CHECK-SD-NEXT:    ld1 { v5.b }[9], [x8]
+; CHECK-SD-NEXT:    add x9, sp, #88
+; CHECK-SD-NEXT:    mov v16.b[5], w5
+; CHECK-SD-NEXT:    ld1 { v1.b }[1], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #288
-; CHECK-SD-NEXT:    mov v0.b[7], w7
-; CHECK-SD-NEXT:    ld1 { v5.b }[10], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[2], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #296
-; CHECK-SD-NEXT:    ld1 { v0.b }[8], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #128
-; CHECK-SD-NEXT:    ld1 { v5.b }[11], [x8]
+; CHECK-SD-NEXT:    mov v16.b[6], w6
+; CHECK-SD-NEXT:    ld1 { v1.b }[3], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #304
-; CHECK-SD-NEXT:    ld1 { v0.b }[9], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #136
-; CHECK-SD-NEXT:    ld1 { v5.b }[12], [x8]
+; CHECK-SD-NEXT:    mov v16.b[7], w7
+; CHECK-SD-NEXT:    ld1 { v1.b }[4], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #312
-; CHECK-SD-NEXT:    ld1 { v5.b }[13], [x8]
+; CHECK-SD-NEXT:    ld1 { v1.b }[5], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #320
-; CHECK-SD-NEXT:    ld1 { v5.b }[14], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #32
-; CHECK-SD-NEXT:    ld1 { v0.b }[10], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #144
-; CHECK-SD-NEXT:    ld1 { v5.b }[15], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #728
-; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #1000
-; CHECK-SD-NEXT:    ld1 { v0.b }[11], [x11]
-; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #736
-; CHECK-SD-NEXT:    add x11, sp, #920
-; CHECK-SD-NEXT:    sdot v3.4s, v5.16b, v1.16b
-; CHECK-SD-NEXT:    ldr b5, [sp, #848]
-; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x12]
-; CHECK-SD-NEXT:    add x12, sp, #48
-; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #744
-; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x14]
-; CHECK-SD-NEXT:    add x14, sp, #96
-; CHECK-SD-NEXT:    ld1 { v0.b }[12], [x12]
-; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #864
-; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x14]
-; CHECK-SD-NEXT:    add x14, sp, #1016
-; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #752
-; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x14]
-; CHECK-SD-NEXT:    add x14, sp, #104
-; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #1024
-; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x14]
-; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x15]
-; CHECK-SD-NEXT:    add x15, sp, #760
-; CHECK-SD-NEXT:    add x14, sp, #112
-; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #880
-; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x15]
-; CHECK-SD-NEXT:    add x15, sp, #1032
-; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x14]
-; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x13]
-; CHECK-SD-NEXT:    add x14, sp, #768
-; CHECK-SD-NEXT:    add x13, sp, #120
-; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x15]
-; CHECK-SD-NEXT:    add x15, sp, #1040
-; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x14]
-; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #776
-; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x16]
-; CHECK-SD-NEXT:    add x14, sp, #1048
-; CHECK-SD-NEXT:    ld1 { v4.b }[8], [x15]
-; CHECK-SD-NEXT:    add x15, sp, #896
-; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x13]
-; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #784
-; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x15]
-; CHECK-SD-NEXT:    add x13, sp, #1056
-; CHECK-SD-NEXT:    ld1 { v4.b }[9], [x14]
-; CHECK-SD-NEXT:    add x14, sp, #904
-; CHECK-SD-NEXT:    ld1 { v6.b }[8], [x10]
-; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #792
-; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x14]
-; CHECK-SD-NEXT:    add x10, sp, #1064
-; CHECK-SD-NEXT:    ld1 { v4.b }[10], [x13]
-; CHECK-SD-NEXT:    add x13, sp, #912
-; CHECK-SD-NEXT:    ld1 { v6.b }[9], [x9]
-; CHECK-SD-NEXT:    ld1 { v7.b }[8], [x8]
-; CHECK-SD-NEXT:    add x9, sp, #800
-; CHECK-SD-NEXT:    ld1 { v5.b }[8], [x13]
+; CHECK-SD-NEXT:    ld1 { v1.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #328
+; CHECK-SD-NEXT:    ld1 { v1.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v2.b }[1], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #96
+; CHECK-SD-NEXT:    add x9, sp, #144
+; CHECK-SD-NEXT:    ld1 { v2.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #104
+; CHECK-SD-NEXT:    zip1 v0.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    movi v1.16b, #1
+; CHECK-SD-NEXT:    ld1 { v2.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #112
+; CHECK-SD-NEXT:    ld1 { v2.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #120
+; CHECK-SD-NEXT:    ld1 { v2.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #128
+; CHECK-SD-NEXT:    ld1 { v2.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #136
+; CHECK-SD-NEXT:    ld1 { v2.b }[7], [x8]
+; CHECK-SD-NEXT:    ldr b3, [x9]
 ; CHECK-SD-NEXT:    add x8, sp, #152
-; CHECK-SD-NEXT:    ld1 { v4.b }[11], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #1072
-; CHECK-SD-NEXT:    ld1 { v6.b }[10], [x9]
-; CHECK-SD-NEXT:    ld1 { v7.b }[9], [x8]
-; CHECK-SD-NEXT:    add x9, sp, #808
-; CHECK-SD-NEXT:    ld1 { v5.b }[9], [x11]
-; CHECK-SD-NEXT:    add x8, sp, #56
-; CHECK-SD-NEXT:    ld1 { v4.b }[12], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #160
-; CHECK-SD-NEXT:    ld1 { v0.b }[13], [x8]
-; CHECK-SD-NEXT:    ld1 { v6.b }[11], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #928
-; CHECK-SD-NEXT:    ld1 { v7.b }[10], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #1080
-; CHECK-SD-NEXT:    ld1 { v5.b }[10], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #984
+; CHECK-SD-NEXT:    ld1 { v3.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #160
+; CHECK-SD-NEXT:    ld1 { v3.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #168
+; CHECK-SD-NEXT:    ld1 { v3.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #176
+; CHECK-SD-NEXT:    ld1 { v3.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #184
+; CHECK-SD-NEXT:    ld1 { v3.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #192
+; CHECK-SD-NEXT:    ld1 { v3.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #200
+; CHECK-SD-NEXT:    ld1 { v3.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v4.b }[1], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #992
+; CHECK-SD-NEXT:    add x9, sp, #1040
+; CHECK-SD-NEXT:    ld1 { v4.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1000
+; CHECK-SD-NEXT:    zip1 v2.2d, v2.2d, v3.2d
+; CHECK-SD-NEXT:    ld1 { v4.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1008
+; CHECK-SD-NEXT:    ld1 { v4.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1016
+; CHECK-SD-NEXT:    ld1 { v4.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1024
+; CHECK-SD-NEXT:    ld1 { v4.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1032
+; CHECK-SD-NEXT:    ld1 { v4.b }[7], [x8]
+; CHECK-SD-NEXT:    ldr b5, [x9]
+; CHECK-SD-NEXT:    add x8, sp, #1048
+; CHECK-SD-NEXT:    add x9, sp, #728
+; CHECK-SD-NEXT:    ld1 { v5.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1056
+; CHECK-SD-NEXT:    ld1 { v5.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1064
+; CHECK-SD-NEXT:    ld1 { v5.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1072
+; CHECK-SD-NEXT:    ld1 { v5.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1080
+; CHECK-SD-NEXT:    ld1 { v5.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1088
+; CHECK-SD-NEXT:    ld1 { v5.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #1096
+; CHECK-SD-NEXT:    ld1 { v5.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v6.b }[1], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #736
+; CHECK-SD-NEXT:    add x9, sp, #784
+; CHECK-SD-NEXT:    ld1 { v6.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #744
+; CHECK-SD-NEXT:    zip1 v4.2d, v4.2d, v5.2d
+; CHECK-SD-NEXT:    movi v5.2d, #0000000000000000
+; CHECK-SD-NEXT:    ld1 { v6.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #752
+; CHECK-SD-NEXT:    sdot v19.4s, v4.16b, v1.16b
+; CHECK-SD-NEXT:    sdot v5.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ld1 { v6.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #760
+; CHECK-SD-NEXT:    ld1 { v6.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #768
+; CHECK-SD-NEXT:    ld1 { v6.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #776
+; CHECK-SD-NEXT:    ld1 { v6.b }[7], [x8]
+; CHECK-SD-NEXT:    ldr b7, [x9]
+; CHECK-SD-NEXT:    add x8, sp, #792
+; CHECK-SD-NEXT:    add x9, sp, #856
+; CHECK-SD-NEXT:    ld1 { v7.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #800
+; CHECK-SD-NEXT:    ld1 { v7.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #808
+; CHECK-SD-NEXT:    ld1 { v7.b }[3], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #816
-; CHECK-SD-NEXT:    ld1 { v4.b }[13], [x10]
-; CHECK-SD-NEXT:    add x9, sp, #168
-; CHECK-SD-NEXT:    add x10, sp, #176
-; CHECK-SD-NEXT:    ld1 { v6.b }[12], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #936
-; CHECK-SD-NEXT:    ld1 { v7.b }[11], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #1088
-; CHECK-SD-NEXT:    ld1 { v5.b }[11], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #64
-; CHECK-SD-NEXT:    ld1 { v4.b }[14], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #824
-; CHECK-SD-NEXT:    ld1 { v0.b }[14], [x8]
-; CHECK-SD-NEXT:    ld1 { v6.b }[13], [x9]
-; CHECK-SD-NEXT:    add x9, sp, #944
-; CHECK-SD-NEXT:    ld1 { v7.b }[12], [x10]
-; CHECK-SD-NEXT:    add x10, sp, #1096
-; CHECK-SD-NEXT:    ld1 { v5.b }[12], [x9]
+; CHECK-SD-NEXT:    ld1 { v7.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #824
+; CHECK-SD-NEXT:    ld1 { v7.b }[5], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #832
-; CHECK-SD-NEXT:    ld1 { v4.b }[15], [x10]
-; CHECK-SD-NEXT:    add x9, sp, #184
-; CHECK-SD-NEXT:    add x10, sp, #72
-; CHECK-SD-NEXT:    ld1 { v6.b }[14], [x8]
-; CHECK-SD-NEXT:    add x8, sp, #952
-; CHECK-SD-NEXT:    ld1 { v7.b }[13], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[13], [x8]
+; CHECK-SD-NEXT:    ld1 { v7.b }[6], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #840
-; CHECK-SD-NEXT:    ld1 { v0.b }[15], [x10]
-; CHECK-SD-NEXT:    sdot v2.4s, v4.16b, v1.16b
-; CHECK-SD-NEXT:    add x9, sp, #192
-; CHECK-SD-NEXT:    ld1 { v6.b }[15], [x8]
+; CHECK-SD-NEXT:    ld1 { v7.b }[7], [x8]
+; CHECK-SD-NEXT:    ld1 { v17.b }[1], [x9]
+; CHECK-SD-NEXT:    add x8, sp, #864
+; CHECK-SD-NEXT:    add x9, sp, #16
+; CHECK-SD-NEXT:    ld1 { v16.b }[8], [x9]
+; CHECK-SD-NEXT:    add x9, sp, #912
+; CHECK-SD-NEXT:    ld1 { v17.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #872
+; CHECK-SD-NEXT:    zip1 v0.2d, v6.2d, v7.2d
+; CHECK-SD-NEXT:    ld1 { v16.b }[9], [x10]
+; CHECK-SD-NEXT:    ld1 { v17.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #880
+; CHECK-SD-NEXT:    sdot v19.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ld1 { v17.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #888
+; CHECK-SD-NEXT:    ld1 { v17.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #896
+; CHECK-SD-NEXT:    ld1 { v17.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #904
+; CHECK-SD-NEXT:    ld1 { v17.b }[7], [x8]
+; CHECK-SD-NEXT:    ldr b18, [x9]
+; CHECK-SD-NEXT:    add x8, sp, #920
+; CHECK-SD-NEXT:    ld1 { v18.b }[1], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #32
+; CHECK-SD-NEXT:    ld1 { v16.b }[10], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #928
+; CHECK-SD-NEXT:    ld1 { v18.b }[2], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #40
+; CHECK-SD-NEXT:    ld1 { v16.b }[11], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #936
+; CHECK-SD-NEXT:    ld1 { v18.b }[3], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #48
+; CHECK-SD-NEXT:    ld1 { v16.b }[12], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #944
+; CHECK-SD-NEXT:    ld1 { v18.b }[4], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #56
+; CHECK-SD-NEXT:    ld1 { v16.b }[13], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #952
+; CHECK-SD-NEXT:    ld1 { v18.b }[5], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #64
+; CHECK-SD-NEXT:    ld1 { v16.b }[14], [x8]
 ; CHECK-SD-NEXT:    add x8, sp, #960
-; CHECK-SD-NEXT:    ld1 { v7.b }[14], [x9]
-; CHECK-SD-NEXT:    ld1 { v5.b }[14], [x8]
-; CHECK-SD-NEXT:    sdot v3.4s, v0.16b, v1.16b
-; CHECK-SD-NEXT:    add x8, sp, #200
-; CHECK-SD-NEXT:    add x9, sp, #968
-; CHECK-SD-NEXT:    sdot v2.4s, v6.16b, v1.16b
-; CHECK-SD-NEXT:    ld1 { v7.b }[15], [x8]
-; CHECK-SD-NEXT:    ld1 { v5.b }[15], [x9]
-; CHECK-SD-NEXT:    sdot v3.4s, v7.16b, v1.16b
-; CHECK-SD-NEXT:    sdot v2.4s, v5.16b, v1.16b
-; CHECK-SD-NEXT:    add v0.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT:    ld1 { v18.b }[6], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #72
+; CHECK-SD-NEXT:    ld1 { v16.b }[15], [x8]
+; CHECK-SD-NEXT:    add x8, sp, #968
+; CHECK-SD-NEXT:    ld1 { v18.b }[7], [x8]
+; CHECK-SD-NEXT:    sdot v5.4s, v16.16b, v1.16b
+; CHECK-SD-NEXT:    zip1 v0.2d, v17.2d, v18.2d
+; CHECK-SD-NEXT:    sdot v5.4s, v2.16b, v1.16b
+; CHECK-SD-NEXT:    sdot v19.4s, v0.16b, v1.16b
+; CHECK-SD-NEXT:    add v0.4s, v5.4s, v19.4s
 ; CHECK-SD-NEXT:    addv s0, v0.4s
 ; CHECK-SD-NEXT:    fmov w0, s0
 ; CHECK-SD-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir b/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir
index 4d67a242a236f..5eac3ec9a771c 100644
--- a/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir
+++ b/llvm/test/CodeGen/AArch64/nested-iv-regalloc.mir
@@ -189,8 +189,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/AArch64/nontemporal.ll b/llvm/test/CodeGen/AArch64/nontemporal.ll
index f8ba150a0405f..f7a87ae340a73 100644
--- a/llvm/test/CodeGen/AArch64/nontemporal.ll
+++ b/llvm/test/CodeGen/AArch64/nontemporal.ll
@@ -683,41 +683,43 @@ define void @test_stnp_v17f32(<17 x float> %v, ptr %ptr) {
 ;
 ; CHECK-BE-LABEL: test_stnp_v17f32:
 ; CHECK-BE:       // %bb.0: // %entry
-; CHECK-BE-NEXT:    // kill: def $s4 killed $s4 def $q4
+; CHECK-BE-NEXT:    // kill: def $s1 killed $s1 def $q1
 ; CHECK-BE-NEXT:    // kill: def $s0 killed $s0 def $q0
-; CHECK-BE-NEXT:    ldr s16, [sp, #36]
+; CHECK-BE-NEXT:    // kill: def $s4 killed $s4 def $q4
 ; CHECK-BE-NEXT:    // kill: def $s5 killed $s5 def $q5
-; CHECK-BE-NEXT:    // kill: def $s1 killed $s1 def $q1
-; CHECK-BE-NEXT:    ldr s17, [sp, #4]
-; CHECK-BE-NEXT:    add x8, sp, #44
-; CHECK-BE-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-BE-NEXT:    add x8, sp, #12
+; CHECK-BE-NEXT:    add x9, sp, #20
+; CHECK-BE-NEXT:    ldr s16, [sp, #36]
 ; CHECK-BE-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-BE-NEXT:    ldr s1, [sp, #4]
+; CHECK-BE-NEXT:    mov v4.s[1], v5.s[0]
+; CHECK-BE-NEXT:    add x10, sp, #52
 ; CHECK-BE-NEXT:    // kill: def $s6 killed $s6 def $q6
 ; CHECK-BE-NEXT:    // kill: def $s2 killed $s2 def $q2
 ; CHECK-BE-NEXT:    // kill: def $s7 killed $s7 def $q7
 ; CHECK-BE-NEXT:    // kill: def $s3 killed $s3 def $q3
-; CHECK-BE-NEXT:    ldr s1, [sp, #68]
-; CHECK-BE-NEXT:    ld1 { v16.s }[1], [x8]
-; CHECK-BE-NEXT:    add x8, sp, #12
-; CHECK-BE-NEXT:    ld1 { v17.s }[1], [x8]
-; CHECK-BE-NEXT:    add x8, sp, #52
-; CHECK-BE-NEXT:    str s1, [x0, #64]
-; CHECK-BE-NEXT:    ld1 { v16.s }[2], [x8]
-; CHECK-BE-NEXT:    add x8, sp, #20
+; CHECK-BE-NEXT:    ld1 { v1.s }[1], [x8]
+; CHECK-BE-NEXT:    ldr s5, [x9]
+; CHECK-BE-NEXT:    add x8, sp, #28
+; CHECK-BE-NEXT:    add x9, sp, #44
+; CHECK-BE-NEXT:    ld1 { v5.s }[1], [x8]
+; CHECK-BE-NEXT:    ld1 { v16.s }[1], [x9]
+; CHECK-BE-NEXT:    ldr s17, [x10]
+; CHECK-BE-NEXT:    add x8, sp, #60
 ; CHECK-BE-NEXT:    mov v4.s[2], v6.s[0]
 ; CHECK-BE-NEXT:    mov v0.s[2], v2.s[0]
-; CHECK-BE-NEXT:    ld1 { v17.s }[2], [x8]
-; CHECK-BE-NEXT:    add x8, sp, #60
-; CHECK-BE-NEXT:    ld1 { v16.s }[3], [x8]
-; CHECK-BE-NEXT:    add x8, sp, #28
-; CHECK-BE-NEXT:    ld1 { v17.s }[3], [x8]
+; CHECK-BE-NEXT:    ld1 { v17.s }[1], [x8]
+; CHECK-BE-NEXT:    ldr s2, [sp, #68]
+; CHECK-BE-NEXT:    add x8, x0, #32
+; CHECK-BE-NEXT:    zip1 v1.2d, v1.2d, v5.2d
+; CHECK-BE-NEXT:    add x9, x0, #48
+; CHECK-BE-NEXT:    str s2, [x0, #64]
+; CHECK-BE-NEXT:    zip1 v5.2d, v16.2d, v17.2d
 ; CHECK-BE-NEXT:    mov v4.s[3], v7.s[0]
-; CHECK-BE-NEXT:    add x8, x0, #48
 ; CHECK-BE-NEXT:    mov v0.s[3], v3.s[0]
-; CHECK-BE-NEXT:    st1 { v16.4s }, [x8]
-; CHECK-BE-NEXT:    add x8, x0, #32
-; CHECK-BE-NEXT:    st1 { v17.4s }, [x8]
+; CHECK-BE-NEXT:    st1 { v1.4s }, [x8]
 ; CHECK-BE-NEXT:    add x8, x0, #16
+; CHECK-BE-NEXT:    st1 { v5.4s }, [x9]
 ; CHECK-BE-NEXT:    st1 { v4.4s }, [x8]
 ; CHECK-BE-NEXT:    st1 { v0.4s }, [x0]
 ; CHECK-BE-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/outlining-with-streaming-mode-changes.ll b/llvm/test/CodeGen/AArch64/outlining-with-streaming-mode-changes.ll
index 94fe06733347a..22774ebf1a662 100644
--- a/llvm/test/CodeGen/AArch64/outlining-with-streaming-mode-changes.ll
+++ b/llvm/test/CodeGen/AArch64/outlining-with-streaming-mode-changes.ll
@@ -7,11 +7,10 @@ define void @streaming_mode_change1() #0 {
 ; CHECK-LABEL: streaming_mode_change1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
@@ -24,7 +23,6 @@ define void @streaming_mode_change1() #0 {
 ;
 ; OUTLINER-LABEL: streaming_mode_change1:
 ; OUTLINER-NOT: OUTLINED_FUNCTION_
-;
   call void @callee();
   ret void;
 }
@@ -33,11 +31,10 @@ define void @streaming_mode_change2() #0 {
 ; CHECK-LABEL: streaming_mode_change2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
@@ -50,7 +47,6 @@ define void @streaming_mode_change2() #0 {
 ;
 ; OUTLINER-LABEL: streaming_mode_change2:
 ; OUTLINER-NOT: OUTLINED_FUNCTION_
-;
   call void @callee();
   ret void;
 }
@@ -59,11 +55,10 @@ define void @streaming_mode_change3() #0 {
 ; CHECK-LABEL: streaming_mode_change3:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
@@ -76,7 +71,6 @@ define void @streaming_mode_change3() #0 {
 ;
 ; OUTLINER-LABEL: streaming_mode_change3:
 ; OUTLINER-NOT: OUTLINED_FUNCTION_
-;
   call void @callee();
   ret void;
 }
diff --git a/llvm/test/CodeGen/AArch64/powi-arm64ec.ll b/llvm/test/CodeGen/AArch64/powi-arm64ec.ll
index 707159eb432ec..2e38f3c5e9a54 100644
--- a/llvm/test/CodeGen/AArch64/powi-arm64ec.ll
+++ b/llvm/test/CodeGen/AArch64/powi-arm64ec.ll
@@ -1,8 +1,18 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=arm64ec-windows-msvc < %s | FileCheck -check-prefix=ARM64EC %s
 
-declare double @llvm.powi.f64.i32(double, i32)
+declare half @llvm.powi.f16.i32(half, i32)
 declare float @llvm.powi.f32.i32(float, i32)
+declare double @llvm.powi.f64.i32(double, i32)
+
+; ARM64EC-LABEL: powi_f16
+; ARM64EC: fcvt  s0, h0
+; ARM64EC: scvtf s1, w0
+; ARM64EC: bl "#powf"
+define half @powi_f16(half %x, i32 %n) nounwind {
+  %ret = tail call half @llvm.powi.f16.i32(half %x, i32 %n)
+  ret half %ret
+}
 
 ; ARM64EC-LABEL: powi_f32
 ; ARM64EC: scvtf s1, w0
diff --git a/llvm/test/CodeGen/AArch64/pr142314.ll b/llvm/test/CodeGen/AArch64/pr142314.ll
new file mode 100644
index 0000000000000..f707ed26026f2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/pr142314.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
+
+; Crash reproducer for: https://github.com/llvm/llvm-project/issues/142314
+
+define <2 x ptr addrspace(1)> @widget() nounwind gc "statepoint-example" {
+; CHECK-LABEL: widget:
+; CHECK:       // %bb.0: // %bb
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov x0, xzr
+; CHECK-NEXT:    mov x1, xzr
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    str q0, [sp]
+; CHECK-NEXT:    movi d0, #0000000000000000
+; CHECK-NEXT:    blr xzr
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    orr x8, x8, #0x8
+; CHECK-NEXT:    ld1 { v0.d }[1], [x8]
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+bb:
+  %call31 = call token (i64, i32, ptr, i32, i32, ...) @llvm.experimental.gc.statepoint.p0(i64 0, i32 0, ptr elementtype(i32 (ptr addrspace(1), ptr addrspace(1), double)) null, i32 3, i32 0, ptr addrspace(1) null, ptr addrspace(1) null, double 0.000000e+00, i32 0, i32 0) [ "gc-live"(ptr addrspace(1) null, <2 x ptr addrspace(1)> zeroinitializer, ptr addrspace(1) null) ]
+  %call4 = call coldcc <2 x ptr addrspace(1)> @llvm.experimental.gc.relocate.v2p1(token %call31, i32 0, i32 1) ; (null, zeroinitializer)
+  %shufflevector = shufflevector <2 x ptr addrspace(1)> zeroinitializer, <2 x ptr addrspace(1)> %call4, <2 x i32> <i32 0, i32 3>
+  ret <2 x ptr addrspace(1)> %shufflevector
+}
+
+declare token @llvm.experimental.gc.statepoint.p0(i64 immarg, i32 immarg, ptr, i32 immarg, i32 immarg, ...)
+declare <2 x ptr addrspace(1)> @llvm.experimental.gc.relocate.v2p1(token, i32 immarg, i32 immarg)
diff --git a/llvm/test/CodeGen/AArch64/pr72777.ll b/llvm/test/CodeGen/AArch64/pr72777.ll
index e9021d605f1fe..fa9f82f8c93c2 100644
--- a/llvm/test/CodeGen/AArch64/pr72777.ll
+++ b/llvm/test/CodeGen/AArch64/pr72777.ll
@@ -4,15 +4,14 @@
 define i64 @f(i64 %0, i64 %1) {
 ; CHECK-LABEL: f:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    orr x9, x1, #0x1
-; CHECK-NEXT:    add x10, x0, x0
-; CHECK-NEXT:    mov x8, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT:    add x9, x9, x10
-; CHECK-NEXT:    lsl x10, x9, #1
-; CHECK-NEXT:    cmp x9, #0
-; CHECK-NEXT:    cinv x8, x8, ge
-; CHECK-NEXT:    cmp x9, x10, asr #1
-; CHECK-NEXT:    csel x0, x8, x10, ne
+; CHECK-NEXT:    orr x8, x1, #0x1
+; CHECK-NEXT:    add x9, x0, x0
+; CHECK-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-NEXT:    adds x8, x8, x9
+; CHECK-NEXT:    lsl x9, x8, #1
+; CHECK-NEXT:    cinv x10, x10, pl
+; CHECK-NEXT:    cmp x8, x9, asr #1
+; CHECK-NEXT:    csel x0, x10, x9, ne
 ; CHECK-NEXT:    ret
   %3 = or i64 1, %1
   %4 = add i64 %3, %0
diff --git a/llvm/test/CodeGen/AArch64/qmovn.ll b/llvm/test/CodeGen/AArch64/qmovn.ll
index 2685ea9fb5d20..bd99c1ca81d3a 100644
--- a/llvm/test/CodeGen/AArch64/qmovn.ll
+++ b/llvm/test/CodeGen/AArch64/qmovn.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-none-elf -verify-machineinstrs %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-elf -verify-machineinstrs -global-isel=0 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-elf -verify-machineinstrs -global-isel=1 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 define <4 x i16> @vqmovni32_smaxmin(<4 x i32> %s0) {
 ; CHECK-LABEL: vqmovni32_smaxmin:
@@ -425,16 +426,31 @@ entry:
 ; Type support varification - not supported with saturated value
 ; i64 -> i16
 define <4 x i16> @sminsmax_range_unsigned_i64_to_i16(<2 x i16> %x, <2 x i64> %y) {
-; CHECK-LABEL: sminsmax_range_unsigned_i64_to_i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    cmgt v2.2d, v1.2d, #0
-; CHECK-NEXT:    movi v3.2d, #0x0000000000ffff
-; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
-; CHECK-NEXT:    cmgt v2.2d, v3.2d, v1.2d
-; CHECK-NEXT:    bif v1.16b, v3.16b, v2.16b
-; CHECK-NEXT:    xtn v1.2s, v1.2d
-; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sminsmax_range_unsigned_i64_to_i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    cmgt v2.2d, v1.2d, #0
+; CHECK-SD-NEXT:    movi v3.2d, #0x0000000000ffff
+; CHECK-SD-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    cmgt v2.2d, v3.2d, v1.2d
+; CHECK-SD-NEXT:    bif v1.16b, v3.16b, v2.16b
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sminsmax_range_unsigned_i64_to_i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    cmgt v2.2d, v1.2d, #0
+; CHECK-GI-NEXT:    movi v3.2d, #0x0000000000ffff
+; CHECK-GI-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-GI-NEXT:    cmgt v2.2d, v3.2d, v1.2d
+; CHECK-GI-NEXT:    bif v1.16b, v3.16b, v2.16b
+; CHECK-GI-NEXT:    xtn v1.2s, v1.2d
+; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %smax = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %y, <2 x i64> zeroinitializer)
   %smin = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %smax, <2 x i64> <i64 65535, i64 65535>)
@@ -444,19 +460,37 @@ entry:
 }
 
 define <4 x i16> @sminsmax_range_signed_i64_to_i16(<2 x i16> %x, <2 x i64> %y) {
-; CHECK-LABEL: sminsmax_range_signed_i64_to_i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov x8, #-32768 // =0xffffffffffff8000
-; CHECK-NEXT:    dup v2.2d, x8
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    cmgt v3.2d, v1.2d, v2.2d
-; CHECK-NEXT:    bif v1.16b, v2.16b, v3.16b
-; CHECK-NEXT:    dup v2.2d, x8
-; CHECK-NEXT:    cmgt v3.2d, v2.2d, v1.2d
-; CHECK-NEXT:    bif v1.16b, v2.16b, v3.16b
-; CHECK-NEXT:    xtn v1.2s, v1.2d
-; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sminsmax_range_signed_i64_to_i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov x8, #-32768 // =0xffffffffffff8000
+; CHECK-SD-NEXT:    dup v2.2d, x8
+; CHECK-SD-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-SD-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-SD-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-SD-NEXT:    dup v2.2d, x8
+; CHECK-SD-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-SD-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sminsmax_range_signed_i64_to_i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    adrp x8, .LCPI31_1
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI31_1]
+; CHECK-GI-NEXT:    adrp x8, .LCPI31_0
+; CHECK-GI-NEXT:    cmgt v3.2d, v1.2d, v2.2d
+; CHECK-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-GI-NEXT:    ldr q2, [x8, :lo12:.LCPI31_0]
+; CHECK-GI-NEXT:    cmgt v3.2d, v2.2d, v1.2d
+; CHECK-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-GI-NEXT:    xtn v1.2s, v1.2d
+; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %smax = call <2 x i64> @llvm.smax.v2i64(<2 x i64> %y, <2 x i64> <i64 -32768, i64 -32768>)
   %smin = call <2 x i64> @llvm.smin.v2i64(<2 x i64> %smax, <2 x i64> <i64 32767, i64 32767>)
@@ -466,14 +500,27 @@ entry:
 }
 
 define <4 x i16> @umin_range_unsigned_i64_to_i16(<2 x i16> %x, <2 x i64> %y) {
-; CHECK-LABEL: umin_range_unsigned_i64_to_i16:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v2.2d, #0x0000000000ffff
-; CHECK-NEXT:    cmhi v3.2d, v2.2d, v1.2d
-; CHECK-NEXT:    bif v1.16b, v2.16b, v3.16b
-; CHECK-NEXT:    xtn v1.2s, v1.2d
-; CHECK-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umin_range_unsigned_i64_to_i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v2.2d, #0x0000000000ffff
+; CHECK-SD-NEXT:    cmhi v3.2d, v2.2d, v1.2d
+; CHECK-SD-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-SD-NEXT:    xtn v1.2s, v1.2d
+; CHECK-SD-NEXT:    uzp1 v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umin_range_unsigned_i64_to_i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v2.2d, #0x0000000000ffff
+; CHECK-GI-NEXT:    cmhi v3.2d, v2.2d, v1.2d
+; CHECK-GI-NEXT:    bif v1.16b, v2.16b, v3.16b
+; CHECK-GI-NEXT:    xtn v1.2s, v1.2d
+; CHECK-GI-NEXT:    uzp1 v1.4h, v1.4h, v0.4h
+; CHECK-GI-NEXT:    uzp1 v0.4h, v0.4h, v0.4h
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %umin = call <2 x i64> @llvm.umin.v2i64(<2 x i64> %y, <2 x i64> <i64 65535, i64 65535>)
   %trunc = trunc <2 x i64> %umin to <2 x i16>
@@ -483,15 +530,29 @@ entry:
 
 ; i32 -> i8
 define <8 x i8> @sminsmax_range_unsigned_i64_to_i8(<4 x i8> %x, <4 x i32> %y) {
-; CHECK-LABEL: sminsmax_range_unsigned_i64_to_i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-NEXT:    smax v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    movi v2.2d, #0x0000ff000000ff
-; CHECK-NEXT:    smin v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    xtn v1.4h, v1.4s
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sminsmax_range_unsigned_i64_to_i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT:    smax v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    movi v2.2d, #0x0000ff000000ff
+; CHECK-SD-NEXT:    smin v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
+; CHECK-SD-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sminsmax_range_unsigned_i64_to_i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v2.2d, #0000000000000000
+; CHECK-GI-NEXT:    movi v3.2d, #0x0000ff000000ff
+; CHECK-GI-NEXT:    smax v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    smin v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    xtn v1.4h, v1.4s
+; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %smax = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %y, <4 x i32> zeroinitializer)
   %smin = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %smax, <4 x i32> <i32 255, i32 255, i32 255, i32 255>)
@@ -501,15 +562,29 @@ entry:
 }
 
 define <8 x i8> @sminsmax_range_signed_i32_to_i8(<4 x i8> %x, <4 x i32> %y) {
-; CHECK-LABEL: sminsmax_range_signed_i32_to_i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mvni v2.4s, #127
-; CHECK-NEXT:    smax v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    movi v2.4s, #127
-; CHECK-NEXT:    smin v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    xtn v1.4h, v1.4s
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: sminsmax_range_signed_i32_to_i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mvni v2.4s, #127
+; CHECK-SD-NEXT:    smax v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    movi v2.4s, #127
+; CHECK-SD-NEXT:    smin v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
+; CHECK-SD-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: sminsmax_range_signed_i32_to_i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mvni v2.4s, #127
+; CHECK-GI-NEXT:    movi v3.4s, #127
+; CHECK-GI-NEXT:    smax v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    smin v1.4s, v1.4s, v3.4s
+; CHECK-GI-NEXT:    xtn v1.4h, v1.4s
+; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %smax = call <4 x i32> @llvm.smax.v4i32(<4 x i32> %y, <4 x i32> <i32 -128, i32 -128, i32 -128, i32 -128>)
   %smin = call <4 x i32> @llvm.smin.v4i32(<4 x i32> %smax, <4 x i32> <i32 127, i32 127, i32 127, i32 127>)
@@ -519,13 +594,25 @@ entry:
 }
 
 define <8 x i8> @umin_range_unsigned_i32_to_i8(<4 x i8> %x, <4 x i32> %y) {
-; CHECK-LABEL: umin_range_unsigned_i32_to_i8:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v2.2d, #0x0000ff000000ff
-; CHECK-NEXT:    umin v1.4s, v1.4s, v2.4s
-; CHECK-NEXT:    xtn v1.4h, v1.4s
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: umin_range_unsigned_i32_to_i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v2.2d, #0x0000ff000000ff
+; CHECK-SD-NEXT:    umin v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT:    xtn v1.4h, v1.4s
+; CHECK-SD-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: umin_range_unsigned_i32_to_i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v2.2d, #0x0000ff000000ff
+; CHECK-GI-NEXT:    umin v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT:    xtn v1.4h, v1.4s
+; CHECK-GI-NEXT:    uzp1 v1.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    mov v0.s[1], w8
+; CHECK-GI-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT:    ret
 entry:
   %umin = call <4 x i32> @llvm.umin.v4i32(<4 x i32> %y, <4 x i32> <i32 255, i32 255, i32 255, i32 255>)
   %trunc = trunc <4 x i32> %umin to <4 x i8>
diff --git a/llvm/test/CodeGen/AArch64/regalloc-last-chance-recolor-with-split.mir b/llvm/test/CodeGen/AArch64/regalloc-last-chance-recolor-with-split.mir
index 8fd439f729a7e..73a22b71980cb 100644
--- a/llvm/test/CodeGen/AArch64/regalloc-last-chance-recolor-with-split.mir
+++ b/llvm/test/CodeGen/AArch64/regalloc-last-chance-recolor-with-split.mir
@@ -257,8 +257,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/sdivpow2.ll b/llvm/test/CodeGen/AArch64/sdivpow2.ll
index 2551be8555ce6..bb18ceb3fe69c 100644
--- a/llvm/test/CodeGen/AArch64/sdivpow2.ll
+++ b/llvm/test/CodeGen/AArch64/sdivpow2.ll
@@ -3,86 +3,143 @@
 ; RUN: llc -mtriple=aarch64-linux-gnu -fast-isel=1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,FAST
 
 define i32 @test1(i32 %x) {
-; CHECK-LABEL: test1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add w8, w0, #7
-; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    csel w8, w8, w0, lt
-; CHECK-NEXT:    asr w0, w8, #3
-; CHECK-NEXT:    ret
+; ISEL-LABEL: test1:
+; ISEL:       // %bb.0:
+; ISEL-NEXT:    add w8, w0, #7
+; ISEL-NEXT:    cmp w0, #0
+; ISEL-NEXT:    csel w8, w8, w0, mi
+; ISEL-NEXT:    asr w0, w8, #3
+; ISEL-NEXT:    ret
+;
+; FAST-LABEL: test1:
+; FAST:       // %bb.0:
+; FAST-NEXT:    add w8, w0, #7
+; FAST-NEXT:    cmp w0, #0
+; FAST-NEXT:    csel w8, w8, w0, lt
+; FAST-NEXT:    asr w0, w8, #3
+; FAST-NEXT:    ret
   %div = sdiv i32 %x, 8
   ret i32 %div
 }
 
 define i32 @test2(i32 %x) {
-; CHECK-LABEL: test2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add w8, w0, #7
-; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    csel w8, w8, w0, lt
-; CHECK-NEXT:    neg w0, w8, asr #3
-; CHECK-NEXT:    ret
+; ISEL-LABEL: test2:
+; ISEL:       // %bb.0:
+; ISEL-NEXT:    add w8, w0, #7
+; ISEL-NEXT:    cmp w0, #0
+; ISEL-NEXT:    csel w8, w8, w0, mi
+; ISEL-NEXT:    neg w0, w8, asr #3
+; ISEL-NEXT:    ret
+;
+; FAST-LABEL: test2:
+; FAST:       // %bb.0:
+; FAST-NEXT:    add w8, w0, #7
+; FAST-NEXT:    cmp w0, #0
+; FAST-NEXT:    csel w8, w8, w0, lt
+; FAST-NEXT:    neg w0, w8, asr #3
+; FAST-NEXT:    ret
   %div = sdiv i32 %x, -8
   ret i32 %div
 }
 
 define i32 @test3(i32 %x) {
-; CHECK-LABEL: test3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add w8, w0, #31
-; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    csel w8, w8, w0, lt
-; CHECK-NEXT:    asr w0, w8, #5
-; CHECK-NEXT:    ret
+; ISEL-LABEL: test3:
+; ISEL:       // %bb.0:
+; ISEL-NEXT:    add w8, w0, #31
+; ISEL-NEXT:    cmp w0, #0
+; ISEL-NEXT:    csel w8, w8, w0, mi
+; ISEL-NEXT:    asr w0, w8, #5
+; ISEL-NEXT:    ret
+;
+; FAST-LABEL: test3:
+; FAST:       // %bb.0:
+; FAST-NEXT:    add w8, w0, #31
+; FAST-NEXT:    cmp w0, #0
+; FAST-NEXT:    csel w8, w8, w0, lt
+; FAST-NEXT:    asr w0, w8, #5
+; FAST-NEXT:    ret
   %div = sdiv i32 %x, 32
   ret i32 %div
 }
 
 define i64 @test4(i64 %x) {
-; CHECK-LABEL: test4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #7
-; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    csel x8, x8, x0, lt
-; CHECK-NEXT:    asr x0, x8, #3
-; CHECK-NEXT:    ret
+; ISEL-LABEL: test4:
+; ISEL:       // %bb.0:
+; ISEL-NEXT:    add x8, x0, #7
+; ISEL-NEXT:    cmp x0, #0
+; ISEL-NEXT:    csel x8, x8, x0, mi
+; ISEL-NEXT:    asr x0, x8, #3
+; ISEL-NEXT:    ret
+;
+; FAST-LABEL: test4:
+; FAST:       // %bb.0:
+; FAST-NEXT:    add x8, x0, #7
+; FAST-NEXT:    cmp x0, #0
+; FAST-NEXT:    csel x8, x8, x0, lt
+; FAST-NEXT:    asr x0, x8, #3
+; FAST-NEXT:    ret
   %div = sdiv i64 %x, 8
   ret i64 %div
 }
 
 define i64 @test5(i64 %x) {
-; CHECK-LABEL: test5:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #7
-; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    csel x8, x8, x0, lt
-; CHECK-NEXT:    neg x0, x8, asr #3
-; CHECK-NEXT:    ret
+; ISEL-LABEL: test5:
+; ISEL:       // %bb.0:
+; ISEL-NEXT:    add x8, x0, #7
+; ISEL-NEXT:    cmp x0, #0
+; ISEL-NEXT:    csel x8, x8, x0, mi
+; ISEL-NEXT:    neg x0, x8, asr #3
+; ISEL-NEXT:    ret
+;
+; FAST-LABEL: test5:
+; FAST:       // %bb.0:
+; FAST-NEXT:    add x8, x0, #7
+; FAST-NEXT:    cmp x0, #0
+; FAST-NEXT:    csel x8, x8, x0, lt
+; FAST-NEXT:    neg x0, x8, asr #3
+; FAST-NEXT:    ret
   %div = sdiv i64 %x, -8
   ret i64 %div
 }
 
 define i64 @test6(i64 %x) {
-; CHECK-LABEL: test6:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    add x8, x0, #63
-; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    csel x8, x8, x0, lt
-; CHECK-NEXT:    asr x0, x8, #6
-; CHECK-NEXT:    ret
+; ISEL-LABEL: test6:
+; ISEL:       // %bb.0:
+; ISEL-NEXT:    add x8, x0, #63
+; ISEL-NEXT:    cmp x0, #0
+; ISEL-NEXT:    csel x8, x8, x0, mi
+; ISEL-NEXT:    asr x0, x8, #6
+; ISEL-NEXT:    ret
+;
+; FAST-LABEL: test6:
+; FAST:       // %bb.0:
+; FAST-NEXT:    add x8, x0, #63
+; FAST-NEXT:    cmp x0, #0
+; FAST-NEXT:    csel x8, x8, x0, lt
+; FAST-NEXT:    asr x0, x8, #6
+; FAST-NEXT:    ret
   %div = sdiv i64 %x, 64
   ret i64 %div
 }
 
 define i64 @test7(i64 %x) {
-; CHECK-LABEL: test7:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov x8, #281474976710655 // =0xffffffffffff
-; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    add x8, x0, x8
-; CHECK-NEXT:    csel x8, x8, x0, lt
-; CHECK-NEXT:    asr x0, x8, #48
-; CHECK-NEXT:    ret
+; ISEL-LABEL: test7:
+; ISEL:       // %bb.0:
+; ISEL-NEXT:    mov x8, #281474976710655 // =0xffffffffffff
+; ISEL-NEXT:    cmp x0, #0
+; ISEL-NEXT:    add x8, x0, x8
+; ISEL-NEXT:    csel x8, x8, x0, mi
+; ISEL-NEXT:    asr x0, x8, #48
+; ISEL-NEXT:    ret
+;
+; FAST-LABEL: test7:
+; FAST:       // %bb.0:
+; FAST-NEXT:    mov x8, #281474976710655 // =0xffffffffffff
+; FAST-NEXT:    cmp x0, #0
+; FAST-NEXT:    add x8, x0, x8
+; FAST-NEXT:    csel x8, x8, x0, lt
+; FAST-NEXT:    asr x0, x8, #48
+; FAST-NEXT:    ret
   %div = sdiv i64 %x, 281474976710656
   ret i64 %div
 }
@@ -132,3 +189,5 @@ define i32 @sdiv_int(i32 %begin, i32 %first) #0 {
 }
 
 attributes #0 = { "target-features"="+sve" vscale_range(2,2) }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/select-constant-xor.ll b/llvm/test/CodeGen/AArch64/select-constant-xor.ll
index fe9a2c0fad830..97ad579a39f78 100644
--- a/llvm/test/CodeGen/AArch64/select-constant-xor.ll
+++ b/llvm/test/CodeGen/AArch64/select-constant-xor.ll
@@ -26,7 +26,7 @@ define i64 @selecti64i64(i64 %a) {
 ; CHECK-GI-NEXT:    mov x8, #-2147483648 // =0xffffffff80000000
 ; CHECK-GI-NEXT:    mov w9, #2147483647 // =0x7fffffff
 ; CHECK-GI-NEXT:    cmp x0, #0
-; CHECK-GI-NEXT:    csel x0, x9, x8, ge
+; CHECK-GI-NEXT:    csel x0, x9, x8, pl
 ; CHECK-GI-NEXT:    ret
   %c = icmp sgt i64 %a, -1
   %s = select i1 %c, i64 2147483647, i64 -2147483648
@@ -44,7 +44,7 @@ define i32 @selecti64i32(i64 %a) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    cmp x0, #0
 ; CHECK-GI-NEXT:    mov w9, #-2147483648 // =0x80000000
-; CHECK-GI-NEXT:    cset w8, ge
+; CHECK-GI-NEXT:    cset w8, pl
 ; CHECK-GI-NEXT:    sbfx w8, w8, #0, #1
 ; CHECK-GI-NEXT:    add w0, w8, w9
 ; CHECK-GI-NEXT:    ret
@@ -66,7 +66,7 @@ define i64 @selecti32i64(i32 %a) {
 ; CHECK-GI-NEXT:    mov x8, #-2147483648 // =0xffffffff80000000
 ; CHECK-GI-NEXT:    mov w9, #2147483647 // =0x7fffffff
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel x0, x9, x8, ge
+; CHECK-GI-NEXT:    csel x0, x9, x8, pl
 ; CHECK-GI-NEXT:    ret
   %c = icmp sgt i32 %a, -1
   %s = select i1 %c, i64 2147483647, i64 -2147483648
@@ -99,7 +99,7 @@ define i32 @selecti32i32(i32 %a) {
 ; CHECK-GI-NEXT:    mov w8, #-85 // =0xffffffab
 ; CHECK-GI-NEXT:    mov w9, #84 // =0x54
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w9, w8, ge
+; CHECK-GI-NEXT:    csel w0, w9, w8, pl
 ; CHECK-GI-NEXT:    ret
   %c = icmp sgt i32 %a, -1
   %s = select i1 %c, i32 84, i32 -85
@@ -118,7 +118,7 @@ define i8 @selecti32i8(i32 %a) {
 ; CHECK-GI-NEXT:    mov w8, #84 // =0x54
 ; CHECK-GI-NEXT:    mov w9, #-85 // =0xffffffab
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w8, w9, ge
+; CHECK-GI-NEXT:    csel w0, w8, w9, pl
 ; CHECK-GI-NEXT:    ret
   %c = icmp sgt i32 %a, -1
   %s = select i1 %c, i8 84, i8 -85
@@ -139,7 +139,7 @@ define i32 @selecti8i32(i8 %a) {
 ; CHECK-GI-NEXT:    mov w9, #-85 // =0xffffffab
 ; CHECK-GI-NEXT:    mov w10, #84 // =0x54
 ; CHECK-GI-NEXT:    cmp w8, #0
-; CHECK-GI-NEXT:    csel w0, w10, w9, ge
+; CHECK-GI-NEXT:    csel w0, w10, w9, pl
 ; CHECK-GI-NEXT:    ret
   %c = icmp sgt i8 %a, -1
   %s = select i1 %c, i32 84, i32 -85
@@ -150,7 +150,7 @@ define i32 @icmpasreq(i32 %input, i32 %a, i32 %b) {
 ; CHECK-SD-LABEL: icmpasreq:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel w0, w1, w2, lt
+; CHECK-SD-NEXT:    csel w0, w1, w2, mi
 ; CHECK-SD-NEXT:    ret
 ;
 ; CHECK-GI-LABEL: icmpasreq:
@@ -193,7 +193,7 @@ define i32 @selecti32i32_0(i32 %a) {
 ; CHECK-GI-LABEL: selecti32i32_0:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    cset w8, lt
+; CHECK-GI-NEXT:    cset w8, mi
 ; CHECK-GI-NEXT:    sbfx w0, w8, #0, #1
 ; CHECK-GI-NEXT:    ret
   %c = icmp sgt i32 %a, -1
@@ -211,7 +211,7 @@ define i32 @selecti32i32_m1(i32 %a) {
 ; CHECK-GI-LABEL: selecti32i32_m1:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    cset w8, ge
+; CHECK-GI-NEXT:    cset w8, pl
 ; CHECK-GI-NEXT:    sbfx w0, w8, #0, #1
 ; CHECK-GI-NEXT:    ret
   %c = icmp sgt i32 %a, -1
@@ -230,7 +230,7 @@ define i32 @selecti32i32_1(i32 %a) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov w8, #-2 // =0xfffffffe
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csinc w0, w8, wzr, lt
+; CHECK-GI-NEXT:    csinc w0, w8, wzr, mi
 ; CHECK-GI-NEXT:    ret
   %c = icmp sgt i32 %a, -1
   %s = select i1 %c, i32 1, i32 -2
@@ -249,7 +249,7 @@ define i32 @selecti32i32_sge(i32 %a) {
 ; CHECK-GI-NEXT:    mov w8, #-13 // =0xfffffff3
 ; CHECK-GI-NEXT:    mov w9, #12 // =0xc
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w9, w8, ge
+; CHECK-GI-NEXT:    csel w0, w9, w8, pl
 ; CHECK-GI-NEXT:    ret
   %c = icmp sge i32 %a, 0
   %s = select i1 %c, i32 12, i32 -13
@@ -268,7 +268,7 @@ define i32 @selecti32i32_slt(i32 %a) {
 ; CHECK-GI-NEXT:    mov w8, #12 // =0xc
 ; CHECK-GI-NEXT:    mov w9, #-13 // =0xfffffff3
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w9, w8, lt
+; CHECK-GI-NEXT:    csel w0, w9, w8, mi
 ; CHECK-GI-NEXT:    ret
   %c = icmp slt i32 %a, 0
   %s = select i1 %c, i32 -13, i32 12
@@ -287,7 +287,7 @@ define i32 @selecti32i32_sle(i32 %a) {
 ; CHECK-GI-NEXT:    mov w8, #12 // =0xc
 ; CHECK-GI-NEXT:    mov w9, #-13 // =0xfffffff3
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w9, w8, lt
+; CHECK-GI-NEXT:    csel w0, w9, w8, mi
 ; CHECK-GI-NEXT:    ret
   %c = icmp sle i32 %a, -1
   %s = select i1 %c, i32 -13, i32 12
@@ -306,7 +306,7 @@ define i32 @selecti32i32_sgt(i32 %a) {
 ; CHECK-GI-NEXT:    mov w8, #12 // =0xc
 ; CHECK-GI-NEXT:    mov w9, #-13 // =0xfffffff3
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w9, w8, lt
+; CHECK-GI-NEXT:    csel w0, w9, w8, mi
 ; CHECK-GI-NEXT:    ret
   %c = icmp sle i32 %a, -1
   %s = select i1 %c, i32 -13, i32 12
@@ -318,7 +318,7 @@ define i32 @oneusecmp(i32 %a, i32 %b, i32 %d) {
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    asr w8, w0, #31
 ; CHECK-SD-NEXT:    cmp w0, #0
-; CHECK-SD-NEXT:    csel w9, w2, w1, lt
+; CHECK-SD-NEXT:    csel w9, w2, w1, mi
 ; CHECK-SD-NEXT:    eor w8, w8, #0x7f
 ; CHECK-SD-NEXT:    add w0, w8, w9
 ; CHECK-SD-NEXT:    ret
@@ -328,8 +328,8 @@ define i32 @oneusecmp(i32 %a, i32 %b, i32 %d) {
 ; CHECK-GI-NEXT:    mov w8, #127 // =0x7f
 ; CHECK-GI-NEXT:    mov w9, #-128 // =0xffffff80
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w8, w9, w8, lt
-; CHECK-GI-NEXT:    csel w9, w2, w1, lt
+; CHECK-GI-NEXT:    csel w8, w9, w8, mi
+; CHECK-GI-NEXT:    csel w9, w2, w1, mi
 ; CHECK-GI-NEXT:    add w0, w8, w9
 ; CHECK-GI-NEXT:    ret
   %c = icmp sle i32 %a, -1
diff --git a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
index 0d4a636446164..293b74ecd9d3a 100644
--- a/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
+++ b/llvm/test/CodeGen/AArch64/selectcc-to-shiftand.ll
@@ -15,7 +15,7 @@ define i32 @neg_sel_constants(i32 %a) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov w8, #5 // =0x5
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w8, wzr, lt
+; CHECK-GI-NEXT:    csel w0, w8, wzr, mi
 ; CHECK-GI-NEXT:    ret
   %tmp.1 = icmp slt i32 %a, 0
   %retval = select i1 %tmp.1, i32 5, i32 0
@@ -34,7 +34,7 @@ define i32 @neg_sel_special_constant(i32 %a) {
 ; CHECK-GI-LABEL: neg_sel_special_constant:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    cset w8, lt
+; CHECK-GI-NEXT:    cset w8, mi
 ; CHECK-GI-NEXT:    lsl w0, w8, #9
 ; CHECK-GI-NEXT:    ret
   %tmp.1 = icmp slt i32 %a, 0
@@ -53,7 +53,7 @@ define i32 @neg_sel_variable_and_zero(i32 %a, i32 %b) {
 ; CHECK-GI-LABEL: neg_sel_variable_and_zero:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w1, wzr, lt
+; CHECK-GI-NEXT:    csel w0, w1, wzr, mi
 ; CHECK-GI-NEXT:    ret
   %tmp.1 = icmp slt i32 %a, 0
   %retval = select i1 %tmp.1, i32 %b, i32 0
@@ -93,7 +93,7 @@ define i32 @pos_sel_constants(i32 %a) {
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    mov w8, #5 // =0x5
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w8, wzr, ge
+; CHECK-GI-NEXT:    csel w0, w8, wzr, pl
 ; CHECK-GI-NEXT:    ret
   %tmp.1 = icmp sgt i32 %a, -1
   %retval = select i1 %tmp.1, i32 5, i32 0
@@ -112,7 +112,7 @@ define i32 @pos_sel_special_constant(i32 %a) {
 ; CHECK-GI-LABEL: pos_sel_special_constant:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    cset w8, ge
+; CHECK-GI-NEXT:    cset w8, pl
 ; CHECK-GI-NEXT:    lsl w0, w8, #9
 ; CHECK-GI-NEXT:    ret
   %tmp.1 = icmp sgt i32 %a, -1
@@ -131,7 +131,7 @@ define i32 @pos_sel_variable_and_zero(i32 %a, i32 %b) {
 ; CHECK-GI-LABEL: pos_sel_variable_and_zero:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    cmp w0, #0
-; CHECK-GI-NEXT:    csel w0, w1, wzr, ge
+; CHECK-GI-NEXT:    csel w0, w1, wzr, pl
 ; CHECK-GI-NEXT:    ret
   %tmp.1 = icmp sgt i32 %a, -1
   %retval = select i1 %tmp.1, i32 %b, i32 0
diff --git a/llvm/test/CodeGen/AArch64/shrinkwrap-split-restore-point.mir b/llvm/test/CodeGen/AArch64/shrinkwrap-split-restore-point.mir
index 4049915391883..bcd517a5eaa40 100644
--- a/llvm/test/CodeGen/AArch64/shrinkwrap-split-restore-point.mir
+++ b/llvm/test/CodeGen/AArch64/shrinkwrap-split-restore-point.mir
@@ -709,8 +709,8 @@ registers:       []
 liveins:
   - { reg: '$w0', virtual-reg: '' }
 frameInfo:
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 body:             |
   ; CHECK-LABEL: name: noshrink_bb_as_inlineasmbr_target
   ; CHECK: bb.0.entry:
diff --git a/llvm/test/CodeGen/AArch64/signbit-shift.ll b/llvm/test/CodeGen/AArch64/signbit-shift.ll
index 0e6da326a31f4..ce8a96386d04c 100644
--- a/llvm/test/CodeGen/AArch64/signbit-shift.ll
+++ b/llvm/test/CodeGen/AArch64/signbit-shift.ll
@@ -128,7 +128,7 @@ define i32 @sel_ifneg_tval_bigger(i32 %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #41 // =0x29
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cinc w0, w8, lt
+; CHECK-NEXT:    cinc w0, w8, mi
 ; CHECK-NEXT:    ret
   %c = icmp slt i32 %x, 0
   %r = select i1 %c, i32 42, i32 41
@@ -162,7 +162,7 @@ define i32 @sel_ifneg_fval_bigger(i32 %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #41 // =0x29
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cinc w0, w8, ge
+; CHECK-NEXT:    cinc w0, w8, pl
 ; CHECK-NEXT:    ret
   %c = icmp slt i32 %x, 0
   %r = select i1 %c, i32 41, i32 42
diff --git a/llvm/test/CodeGen/AArch64/sink-and-fold-drop-dbg.mir b/llvm/test/CodeGen/AArch64/sink-and-fold-drop-dbg.mir
index b29737941b9d2..94fbea20f6d90 100644
--- a/llvm/test/CodeGen/AArch64/sink-and-fold-drop-dbg.mir
+++ b/llvm/test/CodeGen/AArch64/sink-and-fold-drop-dbg.mir
@@ -99,8 +99,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     true
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/AArch64/sink-and-fold-illegal-shift.mir b/llvm/test/CodeGen/AArch64/sink-and-fold-illegal-shift.mir
index 21194e812b897..f46db1a8efa5e 100644
--- a/llvm/test/CodeGen/AArch64/sink-and-fold-illegal-shift.mir
+++ b/llvm/test/CodeGen/AArch64/sink-and-fold-illegal-shift.mir
@@ -65,8 +65,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/AArch64/sink-and-fold-preserve-debugloc.mir b/llvm/test/CodeGen/AArch64/sink-and-fold-preserve-debugloc.mir
index e0c2a547217ee..a11d7125d15ca 100644
--- a/llvm/test/CodeGen/AArch64/sink-and-fold-preserve-debugloc.mir
+++ b/llvm/test/CodeGen/AArch64/sink-and-fold-preserve-debugloc.mir
@@ -117,8 +117,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -189,8 +189,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/AArch64/sme-abi-eh-liveins.mir b/llvm/test/CodeGen/AArch64/sme-abi-eh-liveins.mir
new file mode 100644
index 0000000000000..711745e528bbf
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-abi-eh-liveins.mir
@@ -0,0 +1,82 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=aarch64 -mattr=+sve -mattr=+sme -run-pass=aarch64-machine-sme-abi -verify-machineinstrs %s -o - | FileCheck %s
+
+# This test verifies that runtime defined live-ins are not included in the live
+# outs of predecessors in the MachineSMEABIPass, as including them would result
+# in copies of undefined registers.
+
+--- |
+  define void @sme_abi_eh_liveins() "aarch64_inout_za" personality ptr @__gxx_personality_v0 { entry: unreachable }
+
+  declare i32 @__gxx_personality_v0(...)
+...
+---
+name:            sme_abi_eh_liveins
+tracksRegLiveness: true
+isSSA:           true
+noVRegs:         false
+
+body:             |
+  ; CHECK-LABEL: name: sme_abi_eh_liveins
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x00000000), %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[RDSVLI_XI:%[0-9]+]]:gpr64 = RDSVLI_XI 1, implicit $vg
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $sp
+  ; CHECK-NEXT:   [[MSUBXrrr:%[0-9]+]]:gpr64 = MSUBXrrr [[RDSVLI_XI]], [[RDSVLI_XI]], [[COPY]]
+  ; CHECK-NEXT:   $sp = COPY [[MSUBXrrr]]
+  ; CHECK-NEXT:   STPXi [[MSUBXrrr]], [[RDSVLI_XI]], %stack.0, 0
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   InOutZAUsePseudo
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   [[ADDXri:%[0-9]+]]:gpr64sp = ADDXri %stack.0, 0, 0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64 = COPY [[ADDXri]]
+  ; CHECK-NEXT:   MSR 56965, [[COPY1]]
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   RequiresZASavePseudo
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   MSRpstatesvcrImm1 2, 1, implicit-def $nzcv
+  ; CHECK-NEXT:   [[MRS:%[0-9]+]]:gpr64 = MRS 56965, implicit-def $nzcv
+  ; CHECK-NEXT:   $x0 = ADDXri %stack.0, 0, 0
+  ; CHECK-NEXT:   RestoreZAPseudo [[MRS]], $x0, &__arm_tpidr2_restore, csr_aarch64_sme_abi_support_routines_preservemost_from_x0
+  ; CHECK-NEXT:   MSR 56965, $xzr
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1 (landing-pad):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT:   liveins: $x0, $x1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   MSRpstatesvcrImm1 2, 1, implicit-def $nzcv
+  ; CHECK-NEXT:   [[MRS1:%[0-9]+]]:gpr64 = MRS 56965, implicit-def $nzcv
+  ; CHECK-NEXT:   $x0 = ADDXri %stack.0, 0, 0
+  ; CHECK-NEXT:   RestoreZAPseudo [[MRS1]], $x0, &__arm_tpidr2_restore, csr_aarch64_sme_abi_support_routines_preservemost_from_x0
+  ; CHECK-NEXT:   MSR 56965, $xzr
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   InOutZAUsePseudo
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  bb.0:
+    successors: %bb.2(0x00000000), %bb.1(0x80000000)
+
+    ; Simulate shared ZA call
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    InOutZAUsePseudo
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+
+    ; Simulate private ZA call at the end of the block
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    RequiresZASavePseudo
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+
+    B %bb.2
+
+  bb.1 (landing-pad):
+    liveins: $x0, $x1
+
+    ; Simulate shared ZA call
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    InOutZAUsePseudo
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+
+  bb.2:
diff --git a/llvm/test/CodeGen/AArch64/sme-agnostic-za.ll b/llvm/test/CodeGen/AArch64/sme-agnostic-za.ll
index 1f68815411097..25a7b87d37d9e 100644
--- a/llvm/test/CodeGen/AArch64/sme-agnostic-za.ll
+++ b/llvm/test/CodeGen/AArch64/sme-agnostic-za.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mattr=+sme2 < %s | FileCheck %s
+; RUN: llc -mattr=+sme2 < %s -aarch64-new-sme-abi | FileCheck %s
 
 target triple = "aarch64"
 
@@ -87,18 +88,14 @@ define i64 @shared_caller_agnostic_callee(i64 %v) nounwind "aarch64_inout_za" "a
 define i64 @streaming_agnostic_caller_nonstreaming_private_za_callee(i64 %v) nounwind "aarch64_za_state_agnostic" "aarch64_pstate_sm_enabled" {
 ; CHECK-LABEL: streaming_agnostic_caller_nonstreaming_private_za_callee:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-112]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    mov x9, x0
+; CHECK-NEXT:    mov x8, x0
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    bl __arm_get_current_vg
-; CHECK-NEXT:    str x0, [sp, #80] // 8-byte Folded Spill
-; CHECK-NEXT:    mov x0, x9
 ; CHECK-NEXT:    add x29, sp, #64
-; CHECK-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
-; CHECK-NEXT:    mov x8, x0
+; CHECK-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state_size
 ; CHECK-NEXT:    sub sp, sp, x0
 ; CHECK-NEXT:    mov x20, sp
@@ -122,12 +119,12 @@ define i64 @streaming_agnostic_caller_nonstreaming_private_za_callee(i64 %v) nou
 ; CHECK-NEXT:    bl __arm_sme_restore
 ; CHECK-NEXT:    mov x0, x1
 ; CHECK-NEXT:    sub sp, x29, #64
-; CHECK-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #112 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %res = call i64 @private_za_decl(i64 %v)
   %res2 = call i64 @private_za_decl(i64 %res)
@@ -138,65 +135,97 @@ define i64 @streaming_agnostic_caller_nonstreaming_private_za_callee(i64 %v) nou
 define i64 @streaming_compatible_agnostic_caller_nonstreaming_private_za_callee(i64 %v) nounwind "aarch64_za_state_agnostic" "aarch64_pstate_sm_compatible" {
 ; CHECK-LABEL: streaming_compatible_agnostic_caller_nonstreaming_private_za_callee:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-112]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    mov x9, x0
+; CHECK-NEXT:    mov x8, x0
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    bl __arm_get_current_vg
-; CHECK-NEXT:    str x0, [sp, #80] // 8-byte Folded Spill
-; CHECK-NEXT:    mov x0, x9
 ; CHECK-NEXT:    add x29, sp, #64
-; CHECK-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
-; CHECK-NEXT:    mov x8, x0
+; CHECK-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    bl __arm_sme_state
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    bl __arm_sme_state_size
 ; CHECK-NEXT:    sub sp, sp, x0
-; CHECK-NEXT:    mov x19, sp
-; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    mov x20, sp
+; CHECK-NEXT:    mov x0, x20
 ; CHECK-NEXT:    bl __arm_sme_save
-; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x20, x0, #0x1
-; CHECK-NEXT:    tbz w20, #0, .LBB5_2
+; CHECK-NEXT:    tbz w19, #0, .LBB5_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB5_2:
 ; CHECK-NEXT:    mov x0, x8
 ; CHECK-NEXT:    bl private_za_decl
-; CHECK-NEXT:    mov x2, x0
-; CHECK-NEXT:    tbz w20, #0, .LBB5_4
+; CHECK-NEXT:    mov x1, x0
+; CHECK-NEXT:    tbz w19, #0, .LBB5_4
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB5_4:
-; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    mov x0, x20
 ; CHECK-NEXT:    bl __arm_sme_restore
-; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    mov x0, x20
 ; CHECK-NEXT:    bl __arm_sme_save
-; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x20, x0, #0x1
-; CHECK-NEXT:    tbz w20, #0, .LBB5_6
+; CHECK-NEXT:    tbz w19, #0, .LBB5_6
 ; CHECK-NEXT:  // %bb.5:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB5_6:
-; CHECK-NEXT:    mov x0, x2
+; CHECK-NEXT:    mov x0, x1
 ; CHECK-NEXT:    bl private_za_decl
 ; CHECK-NEXT:    mov x1, x0
-; CHECK-NEXT:    tbz w20, #0, .LBB5_8
+; CHECK-NEXT:    tbz w19, #0, .LBB5_8
 ; CHECK-NEXT:  // %bb.7:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB5_8:
-; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    mov x0, x20
 ; CHECK-NEXT:    bl __arm_sme_restore
 ; CHECK-NEXT:    mov x0, x1
 ; CHECK-NEXT:    sub sp, x29, #64
-; CHECK-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #112 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %res = call i64 @private_za_decl(i64 %v)
   %res2 = call i64 @private_za_decl(i64 %res)
   ret i64 %res2
 }
+
+declare i64 @many_args_private_za_callee(
+  i64, i64, i64, i64, i64, i64, i64, i64, i64, i64)
+
+; In this example some arguments are passed on the stack, which decrements the
+; stack pointer before the call -- in this test the call to __arm_sme_save
+; should occur _before_ the stack decrement.
+define i64  @test_many_callee_arguments(
+; CHECK-LABEL: test_many_callee_arguments:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    mov x8, x0
+; CHECK-NEXT:    bl __arm_sme_state_size
+; CHECK-NEXT:    sub sp, sp, x0
+; CHECK-NEXT:    ldp x9, x10, [x29, #32]
+; CHECK-NEXT:    mov x19, sp
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    bl __arm_sme_save
+; CHECK-NEXT:    stp x9, x10, [sp, #-16]!
+; CHECK-NEXT:    mov x0, x8
+; CHECK-NEXT:    bl many_args_private_za_callee
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    mov x1, x0
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    bl __arm_sme_restore
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+  i64 %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9
+) nounwind "aarch64_za_state_agnostic" {
+  %ret = call i64 @many_args_private_za_callee(
+    i64 %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9)
+  ret i64 %ret
+}
diff --git a/llvm/test/CodeGen/AArch64/sme-call-streaming-compatible-to-normal-fn-wihout-sme-attr.ll b/llvm/test/CodeGen/AArch64/sme-call-streaming-compatible-to-normal-fn-wihout-sme-attr.ll
index c4440e7bcc3ff..9bc5ee6988bcf 100644
--- a/llvm/test/CodeGen/AArch64/sme-call-streaming-compatible-to-normal-fn-wihout-sme-attr.ll
+++ b/llvm/test/CodeGen/AArch64/sme-call-streaming-compatible-to-normal-fn-wihout-sme-attr.ll
@@ -10,15 +10,13 @@ target triple = "aarch64"
 define void @streaming_compatible() #0 {
 ; CHECK-LABEL: streaming_compatible:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
-; CHECK-NEXT:    bl __arm_get_current_vg
-; CHECK-NEXT:    stp x0, x19, [sp, #72] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbz w19, #0, .LBB0_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
@@ -28,12 +26,11 @@ define void @streaming_compatible() #0 {
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   call void @non_streaming()
   ret void
@@ -47,17 +44,15 @@ declare void @non_streaming()
 define void @streaming_compatible_arg(float %f) #0 {
 ; CHECK-LABEL: streaming_compatible_arg:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
-; CHECK-NEXT:    bl __arm_get_current_vg
-; CHECK-NEXT:    stp x0, x19, [sp, #88] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbz w19, #0, .LBB1_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
@@ -68,13 +63,12 @@ define void @streaming_compatible_arg(float %f) #0 {
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   call void @non_streaming(float %f)
   ret void
diff --git a/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
index 980144d6ca584..8d6432ced8e1d 100644
--- a/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
+++ b/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
@@ -9,9 +9,8 @@ declare void @my_func2(<vscale x 16 x i8> %v)
 define void @fbyte(<vscale x 16 x i8> %v) #0{
 ; NOPAIR-LABEL: fbyte:
 ; NOPAIR:       // %bb.0:
-; NOPAIR-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; NOPAIR-NEXT:    cntd x9
-; NOPAIR-NEXT:    stp x9, x19, [sp, #16] // 16-byte Folded Spill
+; NOPAIR-NEXT:    str x29, [sp, #-32]! // 8-byte Folded Spill
+; NOPAIR-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
 ; NOPAIR-NEXT:    addvl sp, sp, #-18
 ; NOPAIR-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; NOPAIR-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
@@ -44,7 +43,7 @@ define void @fbyte(<vscale x 16 x i8> %v) #0{
 ; NOPAIR-NEXT:    addvl sp, sp, #-1
 ; NOPAIR-NEXT:    str z0, [sp] // 16-byte Folded Spill
 ; NOPAIR-NEXT:    bl __arm_sme_state
-; NOPAIR-NEXT:    and x19, x0, #0x1
+; NOPAIR-NEXT:    mov x19, x0
 ; NOPAIR-NEXT:    tbz w19, #0, .LBB0_2
 ; NOPAIR-NEXT:  // %bb.1:
 ; NOPAIR-NEXT:    smstop sm
@@ -85,15 +84,14 @@ define void @fbyte(<vscale x 16 x i8> %v) #0{
 ; NOPAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; NOPAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; NOPAIR-NEXT:    addvl sp, sp, #18
-; NOPAIR-NEXT:    ldr x19, [sp, #24] // 8-byte Folded Reload
-; NOPAIR-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldr x29, [sp], #32 // 8-byte Folded Reload
 ; NOPAIR-NEXT:    ret
 ;
 ; PAIR-LABEL: fbyte:
 ; PAIR:       // %bb.0:
-; PAIR-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; PAIR-NEXT:    cntd x9
-; PAIR-NEXT:    stp x9, x19, [sp, #16] // 16-byte Folded Spill
+; PAIR-NEXT:    str x29, [sp, #-32]! // 8-byte Folded Spill
+; PAIR-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
 ; PAIR-NEXT:    addvl sp, sp, #-18
 ; PAIR-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; PAIR-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
@@ -126,7 +124,7 @@ define void @fbyte(<vscale x 16 x i8> %v) #0{
 ; PAIR-NEXT:    addvl sp, sp, #-1
 ; PAIR-NEXT:    str z0, [sp] // 16-byte Folded Spill
 ; PAIR-NEXT:    bl __arm_sme_state
-; PAIR-NEXT:    and x19, x0, #0x1
+; PAIR-NEXT:    mov x19, x0
 ; PAIR-NEXT:    tbz w19, #0, .LBB0_2
 ; PAIR-NEXT:  // %bb.1:
 ; PAIR-NEXT:    smstop sm
@@ -167,8 +165,8 @@ define void @fbyte(<vscale x 16 x i8> %v) #0{
 ; PAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; PAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; PAIR-NEXT:    addvl sp, sp, #18
-; PAIR-NEXT:    ldr x19, [sp, #24] // 8-byte Folded Reload
-; PAIR-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; PAIR-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; PAIR-NEXT:    ldr x29, [sp], #32 // 8-byte Folded Reload
 ; PAIR-NEXT:    ret
   call void @my_func2(<vscale x 16 x i8> %v)
   ret void
@@ -177,9 +175,7 @@ define void @fbyte(<vscale x 16 x i8> %v) #0{
 define void @fhalf(<vscale x 8 x half> %v) #1{
 ; NOPAIR-LABEL: fhalf:
 ; NOPAIR:       // %bb.0:
-; NOPAIR-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; NOPAIR-NEXT:    cntd x9
-; NOPAIR-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
+; NOPAIR-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; NOPAIR-NEXT:    addvl sp, sp, #-18
 ; NOPAIR-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; NOPAIR-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
@@ -241,14 +237,12 @@ define void @fhalf(<vscale x 8 x half> %v) #1{
 ; NOPAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; NOPAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; NOPAIR-NEXT:    addvl sp, sp, #18
-; NOPAIR-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; NOPAIR-NEXT:    ret
 ;
 ; PAIR-LABEL: fhalf:
 ; PAIR:       // %bb.0:
-; PAIR-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; PAIR-NEXT:    cntd x9
-; PAIR-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
+; PAIR-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; PAIR-NEXT:    addvl sp, sp, #-18
 ; PAIR-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
 ; PAIR-NEXT:    ptrue pn8.b
@@ -298,7 +292,7 @@ define void @fhalf(<vscale x 8 x half> %v) #1{
 ; PAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; PAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; PAIR-NEXT:    addvl sp, sp, #18
-; PAIR-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; PAIR-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; PAIR-NEXT:    ret
   call void @my_func()
   ret void
@@ -307,12 +301,7 @@ define void @fhalf(<vscale x 8 x half> %v) #1{
 define void @ffloat(<vscale x 4 x i32> %v) #2 {
 ; NOPAIR-LABEL: ffloat:
 ; NOPAIR:       // %bb.0:
-; NOPAIR-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; NOPAIR-NEXT:    rdsvl x9, #1
-; NOPAIR-NEXT:    lsr x9, x9, #3
-; NOPAIR-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
-; NOPAIR-NEXT:    cntd x9
-; NOPAIR-NEXT:    str x9, [sp, #24] // 8-byte Folded Spill
+; NOPAIR-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; NOPAIR-NEXT:    addsvl sp, sp, #-18
 ; NOPAIR-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; NOPAIR-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
@@ -374,17 +363,12 @@ define void @ffloat(<vscale x 4 x i32> %v) #2 {
 ; NOPAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; NOPAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; NOPAIR-NEXT:    addsvl sp, sp, #18
-; NOPAIR-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; NOPAIR-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; NOPAIR-NEXT:    ret
 ;
 ; PAIR-LABEL: ffloat:
 ; PAIR:       // %bb.0:
-; PAIR-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; PAIR-NEXT:    rdsvl x9, #1
-; PAIR-NEXT:    lsr x9, x9, #3
-; PAIR-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
-; PAIR-NEXT:    cntd x9
-; PAIR-NEXT:    str x9, [sp, #24] // 8-byte Folded Spill
+; PAIR-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; PAIR-NEXT:    addsvl sp, sp, #-18
 ; PAIR-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; PAIR-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
@@ -446,7 +430,7 @@ define void @ffloat(<vscale x 4 x i32> %v) #2 {
 ; PAIR-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; PAIR-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; PAIR-NEXT:    addsvl sp, sp, #18
-; PAIR-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; PAIR-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; PAIR-NEXT:    ret
   call void @my_func()
   ret void
diff --git a/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll b/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll
index a08e4896f5ee9..48ac156a43875 100644
--- a/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll
+++ b/llvm/test/CodeGen/AArch64/sme-darwin-sve-vg.ll
@@ -5,18 +5,17 @@ declare void @normal_callee();
 define void @locally_streaming_fn() #0 {
 ; CHECK-LABEL: locally_streaming_fn:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! ; 16-byte Folded Spill
+; CHECK-NEXT:    stp     d15, d14, [sp, #-96]!           ; 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-NEXT:    rdsvl x9, #1
-; CHECK-NEXT:    stp d13, d12, [sp, #16] ; 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
-; CHECK-NEXT:    stp d11, d10, [sp, #32] ; 16-byte Folded Spill
-; CHECK-NEXT:    stp d9, d8, [sp, #48] ; 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] ; 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    str x9, [sp, #80] ; 8-byte Folded Spill
+; CHECK-NEXT:    cntd    x9
+; CHECK-NEXT:    stp     d13, d12, [sp, #16]             ; 16-byte Folded Spill
+; CHECK-NEXT:    stp     d11, d10, [sp, #32]             ; 16-byte Folded Spill
+; CHECK-NEXT:    stp     d9, d8, [sp, #48]               ; 16-byte Folded Spill
+; CHECK-NEXT:    stp     x29, x30, [sp, #64]             ; 16-byte Folded Spill
+; CHECK-NEXT:    str     x9, [sp, #80]                   ; 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset vg, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
 ; CHECK-NEXT:    .cfi_offset b9, -48
 ; CHECK-NEXT:    .cfi_offset b10, -56
@@ -26,19 +25,19 @@ define void @locally_streaming_fn() #0 {
 ; CHECK-NEXT:    .cfi_offset b14, -88
 ; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_offset vg, -24
-; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    bl _normal_callee
+; CHECK-NEXT:    smstop  sm
+; CHECK-NEXT:    bl      _normal_callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
-; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldp d9, d8, [sp, #48] ; 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] ; 8-byte Folded Reload
-; CHECK-NEXT:    ldp d11, d10, [sp, #32] ; 16-byte Folded Reload
-; CHECK-NEXT:    ldp d13, d12, [sp, #16] ; 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 ; 16-byte Folded Reload
+; CHECK-NEXT:    smstop  sm
+; CHECK-NEXT:    ldp     x29, x30, [sp, #64]             ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp     d9, d8, [sp, #48]               ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp     d11, d10, [sp, #32]             ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp     d13, d12, [sp, #16]             ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp     d15, d14, [sp], #96             ; 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
diff --git a/llvm/test/CodeGen/AArch64/sme-disable-gisel-fisel.ll b/llvm/test/CodeGen/AArch64/sme-disable-gisel-fisel.ll
index 4a52bf27a7591..e1bfdddaba923 100644
--- a/llvm/test/CodeGen/AArch64/sme-disable-gisel-fisel.ll
+++ b/llvm/test/CodeGen/AArch64/sme-disable-gisel-fisel.ll
@@ -17,8 +17,6 @@ define double @nonstreaming_caller_streaming_callee(double %x) nounwind noinline
 ; CHECK-FISEL-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-FISEL-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
 ; CHECK-FISEL-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
-; CHECK-FISEL-NEXT:    cntd x9
-; CHECK-FISEL-NEXT:    str x9, [sp, #88] // 8-byte Folded Spill
 ; CHECK-FISEL-NEXT:    str d0, [sp] // 8-byte Folded Spill
 ; CHECK-FISEL-NEXT:    smstart sm
 ; CHECK-FISEL-NEXT:    ldr d0, [sp] // 8-byte Folded Reload
@@ -45,8 +43,6 @@ define double @nonstreaming_caller_streaming_callee(double %x) nounwind noinline
 ; CHECK-GISEL-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-GISEL-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
 ; CHECK-GISEL-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
-; CHECK-GISEL-NEXT:    cntd x9
-; CHECK-GISEL-NEXT:    str x9, [sp, #88] // 8-byte Folded Spill
 ; CHECK-GISEL-NEXT:    str d0, [sp] // 8-byte Folded Spill
 ; CHECK-GISEL-NEXT:    smstart sm
 ; CHECK-GISEL-NEXT:    ldr d0, [sp] // 8-byte Folded Reload
@@ -80,8 +76,6 @@ define double @streaming_caller_nonstreaming_callee(double %x) nounwind noinline
 ; CHECK-COMMON-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
-; CHECK-COMMON-NEXT:    cntd x9
-; CHECK-COMMON-NEXT:    str x9, [sp, #88] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    str d0, [sp] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstop sm
 ; CHECK-COMMON-NEXT:    ldr d0, [sp] // 8-byte Folded Reload
@@ -108,17 +102,12 @@ entry:
 define double @locally_streaming_caller_normal_callee(double %x) nounwind noinline optnone "aarch64_pstate_sm_body" {
 ; CHECK-COMMON-LABEL: locally_streaming_caller_normal_callee:
 ; CHECK-COMMON:       // %bb.0:
-; CHECK-COMMON-NEXT:    sub sp, sp, #128
+; CHECK-COMMON-NEXT:    sub sp, sp, #112
 ; CHECK-COMMON-NEXT:    stp d15, d14, [sp, #32] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d13, d12, [sp, #48] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    str x30, [sp, #96] // 8-byte Folded Spill
-; CHECK-COMMON-NEXT:    rdsvl x9, #1
-; CHECK-COMMON-NEXT:    lsr x9, x9, #3
-; CHECK-COMMON-NEXT:    str x9, [sp, #104] // 8-byte Folded Spill
-; CHECK-COMMON-NEXT:    cntd x9
-; CHECK-COMMON-NEXT:    str x9, [sp, #112] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    str d0, [sp, #24] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstart sm
 ; CHECK-COMMON-NEXT:    ldr d0, [sp, #24] // 8-byte Folded Reload
@@ -140,7 +129,7 @@ define double @locally_streaming_caller_normal_callee(double %x) nounwind noinli
 ; CHECK-COMMON-NEXT:    ldp d11, d10, [sp, #64] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d13, d12, [sp, #48] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d15, d14, [sp, #32] // 16-byte Folded Reload
-; CHECK-COMMON-NEXT:    add sp, sp, #128
+; CHECK-COMMON-NEXT:    add sp, sp, #112
 ; CHECK-COMMON-NEXT:    ret
   %call = call double  @normal_callee(double %x);
   %add = fadd double %call, 4.200000e+01
@@ -177,16 +166,11 @@ define double @normal_caller_to_locally_streaming_callee(double %x) nounwind noi
 define void @locally_streaming_caller_streaming_callee_ptr(ptr %p) nounwind noinline optnone "aarch64_pstate_sm_body" {
 ; CHECK-COMMON-LABEL: locally_streaming_caller_streaming_callee_ptr:
 ; CHECK-COMMON:       // %bb.0:
-; CHECK-COMMON-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
-; CHECK-COMMON-NEXT:    rdsvl x9, #1
-; CHECK-COMMON-NEXT:    lsr x9, x9, #3
-; CHECK-COMMON-NEXT:    str x9, [sp, #72] // 8-byte Folded Spill
-; CHECK-COMMON-NEXT:    cntd x9
-; CHECK-COMMON-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstart sm
 ; CHECK-COMMON-NEXT:    blr x0
 ; CHECK-COMMON-NEXT:    smstop sm
@@ -194,7 +178,7 @@ define void @locally_streaming_caller_streaming_callee_ptr(ptr %p) nounwind noin
 ; CHECK-COMMON-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-COMMON-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ret
  call void %p() "aarch64_pstate_sm_enabled"
   ret void
@@ -208,8 +192,6 @@ define void @normal_call_to_streaming_callee_ptr(ptr %p) nounwind noinline optno
 ; CHECK-COMMON-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
-; CHECK-COMMON-NEXT:    cntd x9
-; CHECK-COMMON-NEXT:    str x9, [sp, #72] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstart sm
 ; CHECK-COMMON-NEXT:    blr x0
 ; CHECK-COMMON-NEXT:    smstop sm
@@ -339,22 +321,21 @@ define fp128 @f128_call_sm(fp128 %a, fp128 %b) "aarch64_pstate_sm_enabled" nounw
 ; CHECK-COMMON-LABEL: f128_call_sm:
 ; CHECK-COMMON:       // %bb.0:
 ; CHECK-COMMON-NEXT:    sub sp, sp, #112
-; CHECK-COMMON-NEXT:    cntd x9
 ; CHECK-COMMON-NEXT:    stp d15, d14, [sp, #32] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d13, d12, [sp, #48] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
-; CHECK-COMMON-NEXT:    stp x30, x9, [sp, #96] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str x30, [sp, #96] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstop sm
 ; CHECK-COMMON-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-COMMON-NEXT:    bl __addtf3
 ; CHECK-COMMON-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstart sm
-; CHECK-COMMON-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d9, d8, [sp, #80] // 16-byte Folded Reload
-; CHECK-COMMON-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d11, d10, [sp, #64] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d13, d12, [sp, #48] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d15, d14, [sp, #32] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    add sp, sp, #112
@@ -403,22 +384,21 @@ define float @frem_call_sm(float %a, float %b) "aarch64_pstate_sm_enabled" nounw
 ; CHECK-COMMON-LABEL: frem_call_sm:
 ; CHECK-COMMON:       // %bb.0:
 ; CHECK-COMMON-NEXT:    sub sp, sp, #96
-; CHECK-COMMON-NEXT:    cntd x9
 ; CHECK-COMMON-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-COMMON-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp s0, s1, [sp, #8] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstop sm
 ; CHECK-COMMON-NEXT:    ldp s0, s1, [sp, #8] // 8-byte Folded Reload
 ; CHECK-COMMON-NEXT:    bl fmodf
 ; CHECK-COMMON-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstart sm
-; CHECK-COMMON-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-COMMON-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    add sp, sp, #96
@@ -431,17 +411,15 @@ define float @frem_call_sm(float %a, float %b) "aarch64_pstate_sm_enabled" nounw
 define float @frem_call_sm_compat(float %a, float %b) "aarch64_pstate_sm_compatible" nounwind {
 ; CHECK-COMMON-LABEL: frem_call_sm_compat:
 ; CHECK-COMMON:       // %bb.0:
-; CHECK-COMMON-NEXT:    sub sp, sp, #112
-; CHECK-COMMON-NEXT:    cntd x9
+; CHECK-COMMON-NEXT:    sub sp, sp, #96
 ; CHECK-COMMON-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-COMMON-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-COMMON-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp s0, s1, [sp, #8] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    bl __arm_sme_state
-; CHECK-COMMON-NEXT:    and x19, x0, #0x1
+; CHECK-COMMON-NEXT:    mov x19, x0
 ; CHECK-COMMON-NEXT:    tbz w19, #0, .LBB12_2
 ; CHECK-COMMON-NEXT:  // %bb.1:
 ; CHECK-COMMON-NEXT:    smstop sm
@@ -453,14 +431,13 @@ define float @frem_call_sm_compat(float %a, float %b) "aarch64_pstate_sm_compati
 ; CHECK-COMMON-NEXT:  // %bb.3:
 ; CHECK-COMMON-NEXT:    smstart sm
 ; CHECK-COMMON-NEXT:  .LBB12_4:
+; CHECK-COMMON-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-COMMON-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-COMMON-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-COMMON-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-COMMON-NEXT:    add sp, sp, #112
+; CHECK-COMMON-NEXT:    add sp, sp, #96
 ; CHECK-COMMON-NEXT:    ret
   %res = frem float %a, %b
   ret float %res
diff --git a/llvm/test/CodeGen/AArch64/sme-lazy-save-call.ll b/llvm/test/CodeGen/AArch64/sme-lazy-save-call.ll
index e463e833bdbde..c57cb8e0873d0 100644
--- a/llvm/test/CodeGen/AArch64/sme-lazy-save-call.ll
+++ b/llvm/test/CodeGen/AArch64/sme-lazy-save-call.ll
@@ -1,7 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64 -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK
+; RUN: llc -mtriple=aarch64 -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme -aarch64-new-sme-abi < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-NEWLOWERING
 
 declare void @private_za_callee()
+declare void @shared_za_callee() "aarch64_inout_za"
+declare void @preserves_za_callee() "aarch64_preserves_za"
+
 declare float @llvm.cos.f32(float)
 
 ; Test lazy-save mechanism for a single callee.
@@ -35,6 +39,33 @@ define void @test_lazy_save_1_callee() nounwind "aarch64_inout_za" {
 ; CHECK-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: test_lazy_save_1_callee:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB0_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB0_2:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   call void @private_za_callee()
   ret void
 }
@@ -83,6 +114,34 @@ define void @test_lazy_save_2_callees() nounwind "aarch64_inout_za" {
 ; CHECK-NEXT:    ldr x21, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: test_lazy_save_2_callees:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB1_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB1_2:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   call void @private_za_callee()
   call void @private_za_callee()
   ret void
@@ -119,6 +178,33 @@ define float @test_lazy_save_expanded_intrinsic(float %a) nounwind "aarch64_inou
 ; CHECK-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: test_lazy_save_expanded_intrinsic:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    bl cosf
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB2_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB2_2:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   %res = call float @llvm.cos.f32(float %a)
   ret float %res
 }
@@ -127,18 +213,18 @@ define float @test_lazy_save_expanded_intrinsic(float %a) nounwind "aarch64_inou
 define void @test_lazy_save_and_conditional_smstart() nounwind "aarch64_inout_za" "aarch64_pstate_sm_compatible" {
 ; CHECK-LABEL: test_lazy_save_and_conditional_smstart:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-112]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x29, sp, #64
-; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    bl __arm_sme_state
 ; CHECK-NEXT:    rdsvl x8, #1
 ; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    mov x20, x0
 ; CHECK-NEXT:    msub x9, x8, x8, x9
 ; CHECK-NEXT:    mov sp, x9
 ; CHECK-NEXT:    stur x9, [x29, #-80]
@@ -147,8 +233,6 @@ define void @test_lazy_save_and_conditional_smstart() nounwind "aarch64_inout_za
 ; CHECK-NEXT:    stur wzr, [x29, #-68]
 ; CHECK-NEXT:    sturh w8, [x29, #-72]
 ; CHECK-NEXT:    msr TPIDR2_EL0, x9
-; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x20, x0, #0x1
 ; CHECK-NEXT:    tbz w20, #0, .LBB3_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
@@ -167,13 +251,552 @@ define void @test_lazy_save_and_conditional_smstart() nounwind "aarch64_inout_za
 ; CHECK-NEXT:  .LBB3_6:
 ; CHECK-NEXT:    msr TPIDR2_EL0, xzr
 ; CHECK-NEXT:    sub sp, x29, #64
-; CHECK-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #112 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: test_lazy_save_and_conditional_smstart:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    add x29, sp, #64
+; CHECK-NEWLOWERING-NEXT:    stp x20, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-80]
+; CHECK-NEWLOWERING-NEXT:    bl __arm_sme_state
+; CHECK-NEWLOWERING-NEXT:    mov x20, x0
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #80
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    tbz w20, #0, .LBB3_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    smstop sm
+; CHECK-NEWLOWERING-NEXT:  .LBB3_2:
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    tbz w20, #0, .LBB3_4
+; CHECK-NEWLOWERING-NEXT:  // %bb.3:
+; CHECK-NEWLOWERING-NEXT:    smstart sm
+; CHECK-NEWLOWERING-NEXT:  .LBB3_4:
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #80
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB3_6
+; CHECK-NEWLOWERING-NEXT:  // %bb.5:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB3_6:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    sub sp, x29, #64
+; CHECK-NEWLOWERING-NEXT:    ldp x20, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+  call void @private_za_callee()
+  ret void
+}
+
+; Note: For the final @private_za_callee() we setup a lazy save then don't
+; restore from it (since ZA is off on return). We could improve this case
+; by turning ZA off before the final private ZA call.
+define void @test_lazy_save_mixed_shared_and_private_callees() "aarch64_new_za"
+; CHECK-LABEL: test_lazy_save_mixed_shared_and_private_callees:
+; CHECK:       // %bb.0: // %prelude
+; CHECK-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x21, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa w29, 48
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w21, -32
+; CHECK-NEXT:    .cfi_offset w30, -40
+; CHECK-NEXT:    .cfi_offset w29, -48
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    cbz x8, .LBB4_2
+; CHECK-NEXT:  // %bb.1: // %save.za
+; CHECK-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:  .LBB4_2:
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    rdsvl x20, #1
+; CHECK-NEXT:    sub x21, x29, #16
+; CHECK-NEXT:    zero {za}
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_callee
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB4_4
+; CHECK-NEXT:  // %bb.3:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB4_4:
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    bl shared_za_callee
+; CHECK-NEXT:    bl preserves_za_callee
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_callee
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB4_6
+; CHECK-NEXT:  // %bb.5:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB4_6:
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    smstop za
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x21, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: test_lazy_save_mixed_shared_and_private_callees:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa w29, 32
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w19, -16
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w30, -24
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w29, -32
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    cbz x8, .LBB4_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    zero {za}
+; CHECK-NEWLOWERING-NEXT:  .LBB4_2:
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB4_4
+; CHECK-NEWLOWERING-NEXT:  // %bb.3:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB4_4:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_callee
+; CHECK-NEWLOWERING-NEXT:    bl preserves_za_callee
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    smstop za
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+{
+  call void @private_za_callee()
+  call void @shared_za_callee()
+  call void @preserves_za_callee()
+  call void @private_za_callee()
+  ret void
+}
+
+define void @test_many_back2back_private_za_calls() "aarch64_inout_za" {
+; CHECK-LABEL: test_many_back2back_private_za_calls:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x21, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa w29, 48
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset w21, -32
+; CHECK-NEXT:    .cfi_offset w30, -40
+; CHECK-NEXT:    .cfi_offset w29, -48
+; CHECK-NEXT:    rdsvl x20, #1
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    msub x8, x20, x20, x8
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    bl shared_za_callee
+; CHECK-NEXT:    sub x21, x29, #16
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_callee
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB5_2
+; CHECK-NEXT:  // %bb.1:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB5_2:
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_callee
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB5_4
+; CHECK-NEXT:  // %bb.3:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB5_4:
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_callee
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB5_6
+; CHECK-NEXT:  // %bb.5:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB5_6:
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_callee
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB5_8
+; CHECK-NEXT:  // %bb.7:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB5_8:
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_callee
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB5_10
+; CHECK-NEXT:  // %bb.9:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB5_10:
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_callee
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB5_12
+; CHECK-NEXT:  // %bb.11:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB5_12:
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    bl shared_za_callee
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x21, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: test_many_back2back_private_za_calls:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa w29, 32
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w19, -16
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w30, -24
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w29, -32
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_callee
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB5_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB5_2:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_callee
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+  call void @shared_za_callee()
+  call void @private_za_callee()
+  call void @private_za_callee()
+  call void @private_za_callee()
+  call void @private_za_callee()
+  call void @private_za_callee()
+  call void @private_za_callee()
+  call void @shared_za_callee()
+  ret void
+}
+
+define void @test_shared_private_shared() nounwind "aarch64_inout_za" {
+; CHECK-LABEL: test_shared_private_shared:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x20, #1
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    msub x8, x20, x20, x8
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    bl shared_za_callee
+; CHECK-NEXT:    sub x8, x29, #16
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    bl private_za_callee
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB6_2
+; CHECK-NEXT:  // %bb.1:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB6_2:
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    bl shared_za_callee
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: test_shared_private_shared:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_callee
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB6_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB6_2:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_callee
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+  call void @shared_za_callee()
   call void @private_za_callee()
+  call void @shared_za_callee()
   ret void
 }
+
+define void @test_only_shared_za() nounwind "aarch64_inout_za" {
+; CHECK-COMMON-LABEL: test_only_shared_za:
+; CHECK-COMMON:       // %bb.0:
+; CHECK-COMMON-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    bl shared_za_callee
+; CHECK-COMMON-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
+  call void @shared_za_callee()
+  ret void
+}
+
+declare i64 @shared_za_callee_i64(i64) "aarch64_inout_za"
+declare i64 @private_za_callee_i64(i64)
+
+define i64 @test_shared_private_shared_i64(i64 %x) nounwind "aarch64_inout_za" {
+; CHECK-LABEL: test_shared_private_shared_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x20, #1
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    msub x8, x20, x20, x8
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    bl shared_za_callee_i64
+; CHECK-NEXT:    sub x8, x29, #16
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    bl private_za_callee_i64
+; CHECK-NEXT:    mov x1, x0
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB8_2
+; CHECK-NEXT:  // %bb.1:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB8_2:
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    bl shared_za_callee_i64
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: test_shared_private_shared_i64:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_callee_i64
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee_i64
+; CHECK-NEWLOWERING-NEXT:    mov x1, x0
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB8_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB8_2:
+; CHECK-NEWLOWERING-NEXT:    mov x0, x1
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_callee_i64
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+  %a = call i64 @shared_za_callee_i64(i64 %x)
+  %b = call i64 @private_za_callee_i64(i64 %a)
+  %c = call i64 @shared_za_callee_i64(i64 %b)
+  ret i64 %c
+}
+
+declare i64 @many_args_private_za_callee(
+  i64, i64, i64, i64, i64, i64, i64, i64, i64, i64)
+
+; In this example some arguments are passed on the stack, which decrements the
+; stack pointer before the call -- in this test the lazy save should be setup
+; before the stack decrement.
+define i64  @test_many_callee_arguments(
+; CHECK-LABEL: test_many_callee_arguments:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    msub x8, x9, x9, x8
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    ldp x10, x11, [x29, #32]
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sub x8, x29, #16
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    sturh w9, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    stp x10, x11, [sp, #-16]!
+; CHECK-NEXT:    bl many_args_private_za_callee
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    mov x1, x0
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB9_2
+; CHECK-NEXT:  // %bb.1:
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB9_2:
+; CHECK-NEXT:    mov x0, x1
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: test_many_callee_arguments:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    ldp x10, x11, [x29, #32]
+; CHECK-NEWLOWERING-NEXT:    sub x12, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x12
+; CHECK-NEWLOWERING-NEXT:    stp x10, x11, [sp, #-16]!
+; CHECK-NEWLOWERING-NEXT:    bl many_args_private_za_callee
+; CHECK-NEWLOWERING-NEXT:    add sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    mov x8, x0
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x9, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x9, .LBB9_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB9_2:
+; CHECK-NEWLOWERING-NEXT:    mov x0, x8
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+  i64 %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9
+) nounwind "aarch64_inout_za" {
+  %ret = call i64 @many_args_private_za_callee(
+    i64 %0, i64 %1, i64 %2, i64 %3, i64 %4, i64 %5, i64 %6, i64 %7, i64 %8, i64 %9)
+  ret i64 %ret
+}
diff --git a/llvm/test/CodeGen/AArch64/sme-lazy-sve-nzcv-live.mir b/llvm/test/CodeGen/AArch64/sme-lazy-sve-nzcv-live.mir
new file mode 100644
index 0000000000000..18764d508d0fa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-lazy-sve-nzcv-live.mir
@@ -0,0 +1,132 @@
+# RUN: llc -mtriple=aarch64 -mattr=+sve -mattr=+sme -run-pass=aarch64-machine-sme-abi -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -mattr=+sve -mattr=+sme -aarch64-new-sme-abi %s -o - | FileCheck %s --check-prefix=CHECK-ASM
+
+# This tests the unfortunate case the status flags ($nzcv) are live at the point
+# we want to restore ZA. Currently, this is handled by saving them to a scratch
+# register.
+
+--- |
+  define void @cmp_branch(i32 %0) "aarch64_inout_za" {
+    tail call void @clobber()
+    %2 = icmp sgt i32 %0, 100
+    br i1 %2, label %3, label %4
+
+  3:                                                ; preds = %1
+    tail call void @inout_call() #3
+    br label %4
+
+  4:                                                ; preds = %3, %1
+    tail call void @inout_call() #3
+    ret void
+  }
+
+  declare void @clobber()
+  declare void @inout_call() "aarch64_inout_za"
+...
+---
+name:            cmp_branch
+alignment:       4
+tracksRegLiveness: true
+noPhis:          false
+isSSA:           true
+noVRegs:         false
+hasFakeUses:     false
+registers:
+  - { id: 0, class: gpr32common }
+  - { id: 1, class: gpr32 }
+liveins:
+  - { reg: '$w0' }
+frameInfo:
+  maxAlignment:    1
+  adjustsStack:    true
+  hasCalls:        true
+  maxCallFrameSize: 0
+  hasTailCall:     true
+machineFunctionInfo: {}
+body:             |
+  ; CHECK-LABEL: name: cmp_branch
+  ; CHECK: bb.0 (%ir-block.1):
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[RDSVLI_XI:%[0-9]+]]:gpr64 = RDSVLI_XI 1, implicit $vg
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $sp
+  ; CHECK-NEXT:   [[MSUBXrrr:%[0-9]+]]:gpr64 = MSUBXrrr [[RDSVLI_XI]], [[RDSVLI_XI]], [[COPY]]
+  ; CHECK-NEXT:   $sp = COPY [[MSUBXrrr]]
+  ; CHECK-NEXT:   STPXi [[MSUBXrrr]], [[RDSVLI_XI]], %stack.0, 0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32common = COPY $w0
+  ; CHECK-NEXT:   [[ADDXri:%[0-9]+]]:gpr64sp = ADDXri %stack.0, 0, 0
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr64 = COPY [[ADDXri]]
+  ; CHECK-NEXT:   MSR 56965, [[COPY2]]
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   RequiresZASavePseudo
+  ; CHECK-NEXT:   BL @clobber, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[COPY1]], 101, 0, implicit-def $nzcv
+  ; CHECK-NEXT:   [[MRS:%[0-9]+]]:gpr64 = MRS 55824, implicit-def $nzcv, implicit $nzcv
+  ; CHECK-NEXT:   MSRpstatesvcrImm1 2, 1, implicit-def $nzcv
+  ; CHECK-NEXT:   [[MRS1:%[0-9]+]]:gpr64 = MRS 56965, implicit-def $nzcv
+  ; CHECK-NEXT:   $x0 = ADDXri %stack.0, 0, 0
+  ; CHECK-NEXT:   RestoreZAPseudo [[MRS1]], $x0, &__arm_tpidr2_restore, csr_aarch64_sme_abi_support_routines_preservemost_from_x0
+  ; CHECK-NEXT:   MSR 56965, $xzr
+  ; CHECK-NEXT:   MSR 55824, [[MRS]], implicit-def $nzcv
+  ; CHECK-NEXT:   Bcc 11, %bb.2, implicit $nzcv
+  ; CHECK-NEXT:   B %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1 (%ir-block.3):
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   InOutZAUsePseudo
+  ; CHECK-NEXT:   BL @inout_call, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2 (%ir-block.4):
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   InOutZAUsePseudo
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; CHECK-NEXT:   TCRETURNdi @inout_call, 0, csr_aarch64_aapcs, implicit $sp
+
+  bb.0 (%ir-block.1):
+    successors: %bb.1, %bb.2
+    liveins: $w0
+
+    %0:gpr32common = COPY $w0
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    RequiresZASavePseudo
+    BL @clobber, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+    %1:gpr32 = SUBSWri %0, 101, 0, implicit-def $nzcv
+    Bcc 11, %bb.2, implicit $nzcv
+    B %bb.1
+
+  bb.1 (%ir-block.3):
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    InOutZAUsePseudo
+    BL @inout_call, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+
+  bb.2 (%ir-block.4):
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    InOutZAUsePseudo
+    ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    TCRETURNdi @inout_call, 0, csr_aarch64_aapcs, implicit $sp
+...
+
+# CHECK-ASM-LABEL: cmp_branch
+#       CHECK-ASM:   msr TPIDR2_EL0, x10
+#  CHECK-ASM-NEXT:   bl clobber
+#  CHECK-ASM-NEXT:   cmp w20, #101
+#  CHECK-ASM-NEXT:   mrs x8, NZCV
+#  CHECK-ASM-NEXT:   smstart za
+#  CHECK-ASM-NEXT:   mrs x9, TPIDR2_EL0
+#  CHECK-ASM-NEXT:   sub x0, x29, #16
+#  CHECK-ASM-NEXT:   cbnz x9, .LBB0_2
+#       CHECK-ASM:   bl __arm_tpidr2_restore
+#  CHECK-ASM-NEXT: .LBB0_2:
+#  CHECK-ASM-NEXT:   msr TPIDR2_EL0, xzr
+#  CHECK-ASM-NEXT:   msr NZCV, x8
+#  CHECK-ASM-NEXT:   b.lt .LBB0_4
+#       CHECK-ASM:   bl inout_call
+#  CHECK-ASM-NEXT: .LBB0_4:
+#       CHECK-ASM:   b inout_call
diff --git a/llvm/test/CodeGen/AArch64/sme-must-save-lr-for-vg.ll b/llvm/test/CodeGen/AArch64/sme-must-save-lr-for-vg.ll
index 69f603458670c..2e198ad8f0d05 100644
--- a/llvm/test/CodeGen/AArch64/sme-must-save-lr-for-vg.ll
+++ b/llvm/test/CodeGen/AArch64/sme-must-save-lr-for-vg.ll
@@ -11,14 +11,12 @@ define void @foo() "aarch64_pstate_sm_body" {
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
-; CHECK-NEXT:    rdsvl x9, #1
-; CHECK-NEXT:    lsr x9, x9, #3
-; CHECK-NEXT:    str x9, [sp, #72] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_get_current_vg
 ; CHECK-NEXT:    str x0, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset vg, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
 ; CHECK-NEXT:    .cfi_offset b9, -48
 ; CHECK-NEXT:    .cfi_offset b10, -56
@@ -29,13 +27,15 @@ define void @foo() "aarch64_pstate_sm_body" {
 ; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
diff --git a/llvm/test/CodeGen/AArch64/sme-new-za-function.ll b/llvm/test/CodeGen/AArch64/sme-new-za-function.ll
index 04d26902c536a..78f7e5c009288 100644
--- a/llvm/test/CodeGen/AArch64/sme-new-za-function.ll
+++ b/llvm/test/CodeGen/AArch64/sme-new-za-function.ll
@@ -1,53 +1,133 @@
-; RUN: opt -S -mtriple=aarch64-linux-gnu -aarch64-sme-abi %s | FileCheck %s
-; RUN: opt -S -mtriple=aarch64-linux-gnu -aarch64-sme-abi -aarch64-sme-abi %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -O0 -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -O0 -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs -aarch64-new-sme-abi < %s | FileCheck %s --check-prefix=CHECK-NEWLOWERING
 
 declare void @shared_za_callee() "aarch64_inout_za"
 
 define void @private_za() "aarch64_new_za" {
-; CHECK-LABEL: @private_za(
-; CHECK-NEXT:  prelude:
-; CHECK-NEXT:    [[TPIDR2:%.*]] = call i64 @llvm.aarch64.sme.get.tpidr2()
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[TPIDR2]], 0
-; CHECK-NEXT:    br i1 [[CMP]], label [[SAVE_ZA:%.*]], label [[TMP0:%.*]]
-; CHECK:       save.za:
-; CHECK-NEXT:    call aarch64_sme_preservemost_from_x0 void @__arm_tpidr2_save()
-; CHECK-NEXT:    call void @llvm.aarch64.sme.set.tpidr2(i64 0)
-; CHECK-NEXT:    br label [[TMP0]]
-; CHECK:       0:
-; CHECK-NEXT:    call void @llvm.aarch64.sme.za.enable()
-; CHECK-NEXT:    call void @llvm.aarch64.sme.zero(i32 255)
-; CHECK-NEXT:    call void @shared_za_callee()
-; CHECK-NEXT:    call void @llvm.aarch64.sme.za.disable()
-; CHECK-NEXT:    ret void
+; CHECK-LABEL: private_za:
+; CHECK:       // %bb.0: // %prelude
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    cbz x8, .LBB0_2
+; CHECK-NEXT:    b .LBB0_1
+; CHECK-NEXT:  .LBB0_1: // %save.za
+; CHECK-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    b .LBB0_2
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    zero {za}
+; CHECK-NEXT:    bl shared_za_callee
+; CHECK-NEXT:    smstop za
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
 ;
+; CHECK-NEWLOWERING-LABEL: private_za:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w30, -16
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB0_1
+; CHECK-NEWLOWERING-NEXT:    b .LBB0_2
+; CHECK-NEWLOWERING-NEXT:  .LBB0_1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    zero {za}
+; CHECK-NEWLOWERING-NEXT:    b .LBB0_2
+; CHECK-NEWLOWERING-NEXT:  .LBB0_2:
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_callee
+; CHECK-NEWLOWERING-NEXT:    smstop za
+; CHECK-NEWLOWERING-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   call void @shared_za_callee()
   ret void
 }
 
+; Note: This test must run at -O0 as otherwise the multiple exits are optimized out.
+; TODO: We should be able to omit the ZA save here (as this function does not use ZA).
 define i32 @private_za_multiple_exit(i32 %a, i32 %b, i64 %cond) "aarch64_new_za" {
-; CHECK-LABEL: @private_za_multiple_exit(
-; CHECK-NEXT:  prelude:
-; CHECK-NEXT:    [[TPIDR2:%.*]] = call i64 @llvm.aarch64.sme.get.tpidr2()
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[TPIDR2]], 0
-; CHECK-NEXT:    br i1 [[CMP]], label [[SAVE_ZA:%.*]], label [[ENTRY:%.*]]
-; CHECK:       save.za:
-; CHECK-NEXT:    call aarch64_sme_preservemost_from_x0 void @__arm_tpidr2_save()
-; CHECK-NEXT:    call void @llvm.aarch64.sme.set.tpidr2(i64 0)
-; CHECK-NEXT:    br label [[ENTRY]]
-; CHECK:       entry:
-; CHECK-NEXT:    call void @llvm.aarch64.sme.za.enable()
-; CHECK-NEXT:    call void @llvm.aarch64.sme.zero(i32 255)
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[COND:%.*]], 1
-; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.else:
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[A:%.*]], [[B:%.*]]
-; CHECK-NEXT:    call void @llvm.aarch64.sme.za.disable()
-; CHECK-NEXT:    ret i32 [[ADD]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[A]], [[B]]
-; CHECK-NEXT:    call void @llvm.aarch64.sme.za.disable()
-; CHECK-NEXT:    ret i32 [[SUB]]
+; CHECK-LABEL: private_za_multiple_exit:
+; CHECK:       // %bb.0: // %prelude
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str x2, [sp] // 8-byte Folded Spill
+; CHECK-NEXT:    str w1, [sp, #8] // 4-byte Folded Spill
+; CHECK-NEXT:    str w0, [sp, #12] // 4-byte Folded Spill
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    cbz x8, .LBB1_2
+; CHECK-NEXT:    b .LBB1_1
+; CHECK-NEXT:  .LBB1_1: // %save.za
+; CHECK-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    b .LBB1_2
+; CHECK-NEXT:  .LBB1_2: // %entry
+; CHECK-NEXT:    ldr x8, [sp] // 8-byte Folded Reload
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    zero {za}
+; CHECK-NEXT:    subs x8, x8, #1
+; CHECK-NEXT:    b.ne .LBB1_4
+; CHECK-NEXT:    b .LBB1_3
+; CHECK-NEXT:  .LBB1_3: // %if.else
+; CHECK-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
+; CHECK-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
+; CHECK-NEXT:    add w0, w8, w9
+; CHECK-NEXT:    smstop za
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB1_4: // %if.end
+; CHECK-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
+; CHECK-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
+; CHECK-NEXT:    subs w0, w8, w9
+; CHECK-NEXT:    smstop za
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
 ;
+; CHECK-NEWLOWERING-LABEL: private_za_multiple_exit:
+; CHECK-NEWLOWERING:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB1_1
+; CHECK-NEWLOWERING-NEXT:    b .LBB1_2
+; CHECK-NEWLOWERING-NEXT:  .LBB1_1: // %entry
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    zero {za}
+; CHECK-NEWLOWERING-NEXT:    b .LBB1_2
+; CHECK-NEWLOWERING-NEXT:  .LBB1_2: // %entry
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    str w1, [sp, #8] // 4-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str w0, [sp, #12] // 4-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    subs x8, x2, #1
+; CHECK-NEWLOWERING-NEXT:    b.ne .LBB1_4
+; CHECK-NEWLOWERING-NEXT:    b .LBB1_3
+; CHECK-NEWLOWERING-NEXT:  .LBB1_3: // %if.else
+; CHECK-NEWLOWERING-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    add w0, w8, w9
+; CHECK-NEWLOWERING-NEXT:    smstop za
+; CHECK-NEWLOWERING-NEXT:    add sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    ret
+; CHECK-NEWLOWERING-NEXT:  .LBB1_4: // %if.end
+; CHECK-NEWLOWERING-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldr w9, [sp, #8] // 4-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    subs w0, w8, w9
+; CHECK-NEWLOWERING-NEXT:    smstop za
+; CHECK-NEWLOWERING-NEXT:    add sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    ret
 entry:
   %tobool = icmp eq i64 %cond, 1
   br i1 %tobool, label %if.else, label %if.end
@@ -61,5 +141,38 @@ if.end:
   ret i32 %sub
 }
 
-; CHECK: declare void @__arm_tpidr2_save() #[[ATTR:[0-9]+]]
-; CHECK: attributes #[[ATTR]] = { "aarch64_pstate_sm_compatible" }
+; In simple cases like this we should omit all ZA setup.
+define i32 @private_za_trivially_does_not_use_za(i32 %x) "aarch64_new_za" {
+; CHECK-LABEL: private_za_trivially_does_not_use_za:
+; CHECK:       // %bb.0: // %prelude
+; CHECK-NEXT:    sub sp, sp, #32
+; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    str w0, [sp, #12] // 4-byte Folded Spill
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    cbz x8, .LBB2_2
+; CHECK-NEXT:    b .LBB2_1
+; CHECK-NEXT:  .LBB2_1: // %save.za
+; CHECK-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEXT:    mov x8, xzr
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    b .LBB2_2
+; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:    ldr w8, [sp, #12] // 4-byte Folded Reload
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    zero {za}
+; CHECK-NEXT:    add w0, w8, w8
+; CHECK-NEXT:    smstop za
+; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #32
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: private_za_trivially_does_not_use_za:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    add w0, w0, w0
+; CHECK-NEWLOWERING-NEXT:    ret
+    %ret = add i32 %x, %x
+    ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/AArch64/sme-peephole-opts.ll b/llvm/test/CodeGen/AArch64/sme-peephole-opts.ll
index 130a316bcc2ba..ab7c661d27187 100644
--- a/llvm/test/CodeGen/AArch64/sme-peephole-opts.ll
+++ b/llvm/test/CodeGen/AArch64/sme-peephole-opts.ll
@@ -11,11 +11,10 @@ define void @test0(ptr %callee) nounwind {
 ; CHECK-LABEL: test0:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    bl callee_sm
 ; CHECK-NEXT:    bl callee_sm
@@ -36,11 +35,10 @@ define void @test1() nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-LABEL: test1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    bl callee
@@ -57,45 +55,31 @@ define void @test1() nounwind "aarch64_pstate_sm_enabled" {
 }
 
 ; streaming-compatible caller -> normal callees
-; these conditional smstart/smstop are not yet optimized away.
 define void @test2() nounwind "aarch64_pstate_sm_compatible" {
 ; CHECK-LABEL: test2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbz w19, #0, .LBB2_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    tbz w19, #0, .LBB2_4
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB2_4:
-; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
-; CHECK-NEXT:    tbz w19, #0, .LBB2_6
-; CHECK-NEXT:  // %bb.5:
-; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:  .LBB2_6:
-; CHECK-NEXT:    bl callee
-; CHECK-NEXT:    tbz w19, #0, .LBB2_8
-; CHECK-NEXT:  // %bb.7:
-; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:  .LBB2_8:
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   call void @callee()
   call void @callee()
@@ -106,15 +90,13 @@ define void @test2() nounwind "aarch64_pstate_sm_compatible" {
 define void @test3() nounwind "aarch64_pstate_sm_compatible" {
 ; CHECK-LABEL: test3:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbnz w19, #0, .LBB3_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstart sm
@@ -124,8 +106,6 @@ define void @test3() nounwind "aarch64_pstate_sm_compatible" {
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB3_4:
-; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
 ; CHECK-NEXT:    tbz w19, #0, .LBB3_6
 ; CHECK-NEXT:  // %bb.5:
 ; CHECK-NEXT:    smstop sm
@@ -135,8 +115,6 @@ define void @test3() nounwind "aarch64_pstate_sm_compatible" {
 ; CHECK-NEXT:  // %bb.7:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB3_8:
-; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
 ; CHECK-NEXT:    tbnz w19, #0, .LBB3_10
 ; CHECK-NEXT:  // %bb.9:
 ; CHECK-NEXT:    smstart sm
@@ -146,12 +124,11 @@ define void @test3() nounwind "aarch64_pstate_sm_compatible" {
 ; CHECK-NEXT:  // %bb.11:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB3_12:
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   call void @callee_sm()
   call void @callee()
@@ -164,11 +141,10 @@ define void @test4() nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-LABEL: test4:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    fmov s0, wzr
 ; CHECK-NEXT:    bl callee_farg
@@ -191,12 +167,11 @@ define void @test5(float %f) nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-LABEL: test5:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #96
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
@@ -220,12 +195,11 @@ define float @test6(float %f) nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-LABEL: test6:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #96
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
@@ -233,10 +207,10 @@ define float @test6(float %f) nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-NEXT:    bl callee_farg_fret
 ; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #96
@@ -279,11 +253,10 @@ define void @test8() nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-LABEL: test8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
@@ -304,11 +277,98 @@ define void @test8() nounwind "aarch64_pstate_sm_enabled" {
 define void @test9() "aarch64_pstate_sm_body" {
 ; CHECK-LABEL: test9:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+  call void @callee()
+  ret void
+}
+
+; test that if the 'smstart' and 'smtop' are entirely removed in a locally
+; streaming function, we use the FP, not an expression to describe the CFA.
+define aarch64_sve_vector_pcs void @test9_1() "aarch64_pstate_sm_body" {
+; CHECK-LABEL: test9_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x28, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    addsvl sp, sp, #-18
+; CHECK-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa w29, 32
+; CHECK-NEXT:    .cfi_offset w28, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d8 @ cfa - 8 * VG - 32
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d9 @ cfa - 16 * VG - 32
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d10 @ cfa - 24 * VG - 32
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d11 @ cfa - 32 * VG - 32
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d12 @ cfa - 40 * VG - 32
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d13 @ cfa - 48 * VG - 32
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d14 @ cfa - 56 * VG - 32
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d15 @ cfa - 64 * VG - 32
 ; CHECK-NEXT:    bl callee
-; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addsvl sp, sp, #18
+; CHECK-NEXT:    ldr x28, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   call void @callee()
   ret void
@@ -322,16 +382,15 @@ define void @test10() "aarch64_pstate_sm_body" {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset vg, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
 ; CHECK-NEXT:    .cfi_offset b9, -48
 ; CHECK-NEXT:    .cfi_offset b10, -56
@@ -342,18 +401,18 @@ define void @test10() "aarch64_pstate_sm_body" {
 ; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    bl callee_sm
-; CHECK-NEXT:    .cfi_offset vg, -24
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
@@ -374,13 +433,11 @@ define void @test10() "aarch64_pstate_sm_body" {
 define void @test11(ptr %p) nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-LABEL: test11:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
@@ -390,12 +447,11 @@ define void @test11(ptr %p) nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   call void @callee()
   store <vscale x 16 x i8> zeroinitializer, ptr %p
@@ -411,16 +467,15 @@ define void @test12() "aarch64_pstate_sm_body" {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset vg, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
 ; CHECK-NEXT:    .cfi_offset b9, -48
 ; CHECK-NEXT:    .cfi_offset b10, -56
@@ -431,20 +486,20 @@ define void @test12() "aarch64_pstate_sm_body" {
 ; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    smstop za
-; CHECK-NEXT:    .cfi_offset vg, -24
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    smstart za
 ; CHECK-NEXT:    smstop sm
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
@@ -467,12 +522,11 @@ define void @test13(ptr %ptr) nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-LABEL: test13:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    mov z0.s, #0 // =0x0
 ; CHECK-NEXT:    mov x19, x0
@@ -490,8 +544,8 @@ define void @test13(ptr %ptr) nounwind "aarch64_pstate_sm_enabled" {
 ; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll
index 5ea5e3e7766e8..b947c943ba448 100644
--- a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll
+++ b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll
@@ -16,12 +16,11 @@ define void @dont_coalesce_arg_i8(i8 %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    fmov s0, w0
 ; CHECK-NEXT:    mov x19, x1
@@ -32,8 +31,8 @@ define void @dont_coalesce_arg_i8(i8 %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -49,12 +48,11 @@ define void @dont_coalesce_arg_i16(i16 %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    fmov s0, w0
 ; CHECK-NEXT:    mov x19, x1
@@ -65,8 +63,8 @@ define void @dont_coalesce_arg_i16(i16 %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -82,12 +80,11 @@ define void @dont_coalesce_arg_i32(i32 %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    fmov s0, w0
 ; CHECK-NEXT:    mov x19, x1
@@ -98,8 +95,8 @@ define void @dont_coalesce_arg_i32(i32 %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -115,12 +112,11 @@ define void @dont_coalesce_arg_i64(i64 %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    fmov d0, x0
 ; CHECK-NEXT:    mov x19, x1
@@ -131,8 +127,8 @@ define void @dont_coalesce_arg_i64(i64 %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -148,12 +144,11 @@ define void @dont_coalesce_arg_f16(half %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 def $z0
@@ -171,8 +166,8 @@ define void @dont_coalesce_arg_f16(half %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -188,12 +183,11 @@ define void @dont_coalesce_arg_f32(float %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $s0 killed $s0 def $z0
@@ -211,8 +205,8 @@ define void @dont_coalesce_arg_f32(float %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -228,12 +222,11 @@ define void @dont_coalesce_arg_f64(double %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
@@ -251,8 +244,8 @@ define void @dont_coalesce_arg_f64(double %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -273,12 +266,11 @@ define void @dont_coalesce_arg_v1i8(<1 x i8> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v1i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
@@ -296,8 +288,8 @@ define void @dont_coalesce_arg_v1i8(<1 x i8> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -314,12 +306,11 @@ define void @dont_coalesce_arg_v1i16(<1 x i16> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v1i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
@@ -337,8 +328,8 @@ define void @dont_coalesce_arg_v1i16(<1 x i16> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -355,12 +346,11 @@ define void @dont_coalesce_arg_v1i32(<1 x i32> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v1i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
@@ -378,8 +368,8 @@ define void @dont_coalesce_arg_v1i32(<1 x i32> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -396,12 +386,11 @@ define void @dont_coalesce_arg_v1i64(<1 x i64> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v1i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
@@ -419,8 +408,8 @@ define void @dont_coalesce_arg_v1i64(<1 x i64> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -437,12 +426,11 @@ define void @dont_coalesce_arg_v1f16(<1 x half> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v1f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $h0 killed $h0 def $z0
@@ -460,8 +448,8 @@ define void @dont_coalesce_arg_v1f16(<1 x half> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -478,12 +466,11 @@ define void @dont_coalesce_arg_v1f32(<1 x float> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v1f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
@@ -501,8 +488,8 @@ define void @dont_coalesce_arg_v1f32(<1 x float> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -519,12 +506,11 @@ define void @dont_coalesce_arg_v1f64(<1 x double> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
@@ -542,8 +528,8 @@ define void @dont_coalesce_arg_v1f64(<1 x double> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -564,12 +550,11 @@ define void @dont_coalesce_arg_v16i8(<16 x i8> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
@@ -587,8 +572,8 @@ define void @dont_coalesce_arg_v16i8(<16 x i8> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -604,12 +589,11 @@ define void @dont_coalesce_arg_v8i16(<8 x i16> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
@@ -627,8 +611,8 @@ define void @dont_coalesce_arg_v8i16(<8 x i16> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -644,12 +628,11 @@ define void @dont_coalesce_arg_v4i32(<4 x i32> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
@@ -667,8 +650,8 @@ define void @dont_coalesce_arg_v4i32(<4 x i32> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -684,12 +667,11 @@ define void @dont_coalesce_arg_v2i64(<2 x i64> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v2i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
@@ -707,8 +689,8 @@ define void @dont_coalesce_arg_v2i64(<2 x i64> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -724,12 +706,11 @@ define void @dont_coalesce_arg_v8f16(<8 x half> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v8f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
@@ -747,8 +728,8 @@ define void @dont_coalesce_arg_v8f16(<8 x half> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -764,12 +745,11 @@ define void @dont_coalesce_arg_v8bf16(<8 x bfloat> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v8bf16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
@@ -787,8 +767,8 @@ define void @dont_coalesce_arg_v8bf16(<8 x bfloat> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -804,12 +784,11 @@ define void @dont_coalesce_arg_v4f32(<4 x float> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v4f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
@@ -827,8 +806,8 @@ define void @dont_coalesce_arg_v4f32(<4 x float> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -844,12 +823,11 @@ define void @dont_coalesce_arg_v2f64(<2 x double> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v2f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
@@ -867,8 +845,8 @@ define void @dont_coalesce_arg_v2f64(<2 x double> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -887,12 +865,11 @@ define void @dont_coalesce_arg_v8i1(<8 x i1> %arg, ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_arg_v8i1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
@@ -901,10 +878,10 @@ define void @dont_coalesce_arg_v8i1(<8 x i1> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    add x8, sp, #16
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    str d0, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    and z1.b, z1.b, #0x1
 ; CHECK-NEXT:    cmpne p0.b, p0/z, z1.b, #0
 ; CHECK-NEXT:    str p0, [x8, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str d0, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    bl use_v8i1
@@ -914,8 +891,8 @@ define void @dont_coalesce_arg_v8i1(<8 x i1> %arg, ptr %ptr) #0 {
 ; CHECK-NEXT:    str p0, [x19]
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -934,25 +911,22 @@ define void @dont_coalesce_arg_v8i1(<8 x i1> %arg, ptr %ptr) #0 {
 define void @dont_coalesce_res_i8(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_i8
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    fmov s0, w0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %res = call i8 @get_i8()
   %vec = insertelement <vscale x 16 x i8> poison, i8 %res, i32 0
@@ -963,25 +937,22 @@ define void @dont_coalesce_res_i8(ptr %ptr) #0 {
 define void @dont_coalesce_res_i16(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_i16
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    fmov s0, w0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %res = call i16 @get_i16()
   %vec = insertelement <vscale x 8 x i16> poison, i16 %res, i32 0
@@ -992,25 +963,22 @@ define void @dont_coalesce_res_i16(ptr %ptr) #0 {
 define void @dont_coalesce_res_i32(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_i32
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    fmov s0, w0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %res = call i32 @get_i32()
   %vec = insertelement <vscale x 4 x i32> poison, i32 %res, i32 0
@@ -1021,25 +989,22 @@ define void @dont_coalesce_res_i32(ptr %ptr) #0 {
 define void @dont_coalesce_res_i64(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_i64
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    fmov d0, x0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %res = call i64 @get_i64()
   %vec = insertelement <vscale x 2 x i64> poison, i64 %res, i32 0
@@ -1050,29 +1015,26 @@ define void @dont_coalesce_res_i64(ptr %ptr) #0 {
 define void @dont_coalesce_res_f16(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_f16
 ; CHECK-NEXT:    str h0, [sp, #14] // 2-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr h0, [sp, #14] // 2-byte Folded Reload
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $z0
 ; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call half @get_f16()
   %vec = insertelement <vscale x 8 x half> poison, half %res, i32 0
@@ -1083,14 +1045,12 @@ define void @dont_coalesce_res_f16(ptr %ptr) #0 {
 define void @dont_coalesce_res_f32(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_f32
@@ -1098,13 +1058,12 @@ define void @dont_coalesce_res_f32(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call float @get_f32()
   %vec = insertelement <vscale x 4 x float> poison, float %res, i32 0
@@ -1115,14 +1074,12 @@ define void @dont_coalesce_res_f32(ptr %ptr) #0 {
 define void @dont_coalesce_res_f64(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_f64
@@ -1130,13 +1087,12 @@ define void @dont_coalesce_res_f64(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call double @get_f64()
   %vec = insertelement <vscale x 2 x double> poison, double %res, i32 0
@@ -1151,14 +1107,12 @@ define void @dont_coalesce_res_f64(ptr %ptr) #0 {
 define void @dont_coalesce_res_v1i8(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v1i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v1i8
@@ -1166,13 +1120,12 @@ define void @dont_coalesce_res_v1i8(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <1 x i8> @get_v1i8()
   %elt = extractelement <1 x i8> %res, i32 0
@@ -1184,14 +1137,12 @@ define void @dont_coalesce_res_v1i8(ptr %ptr) #0 {
 define void @dont_coalesce_res_v1i16(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v1i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v1i16
@@ -1199,13 +1150,12 @@ define void @dont_coalesce_res_v1i16(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <1 x i16> @get_v1i16()
   %elt = extractelement <1 x i16> %res, i32 0
@@ -1217,14 +1167,12 @@ define void @dont_coalesce_res_v1i16(ptr %ptr) #0 {
 define void @dont_coalesce_res_v1i32(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v1i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v1i32
@@ -1232,13 +1180,12 @@ define void @dont_coalesce_res_v1i32(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <1 x i32> @get_v1i32()
   %elt = extractelement <1 x i32> %res, i32 0
@@ -1250,14 +1197,12 @@ define void @dont_coalesce_res_v1i32(ptr %ptr) #0 {
 define void @dont_coalesce_res_v1i64(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v1i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v1i64
@@ -1265,13 +1210,12 @@ define void @dont_coalesce_res_v1i64(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <1 x i64> @get_v1i64()
   %elt = extractelement <1 x i64> %res, i32 0
@@ -1283,29 +1227,26 @@ define void @dont_coalesce_res_v1i64(ptr %ptr) #0 {
 define void @dont_coalesce_res_v1f16(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v1f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v1f16
 ; CHECK-NEXT:    str h0, [sp, #14] // 2-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr h0, [sp, #14] // 2-byte Folded Reload
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $h0 killed $h0 def $z0
 ; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <1 x half> @get_v1f16()
   %elt = extractelement <1 x half> %res, i32 0
@@ -1317,14 +1258,12 @@ define void @dont_coalesce_res_v1f16(ptr %ptr) #0 {
 define void @dont_coalesce_res_v1f32(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v1f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v1f32
@@ -1332,13 +1271,12 @@ define void @dont_coalesce_res_v1f32(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <1 x float> @get_v1f32()
   %elt = extractelement <1 x float> %res, i32 0
@@ -1350,14 +1288,12 @@ define void @dont_coalesce_res_v1f32(ptr %ptr) #0 {
 define void @dont_coalesce_res_v1f64(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v1f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v1f64
@@ -1365,13 +1301,12 @@ define void @dont_coalesce_res_v1f64(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    str z0, [x19]
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <1 x double> @get_v1f64()
   %elt = extractelement <1 x double> %res, i32 0
@@ -1387,29 +1322,26 @@ define void @dont_coalesce_res_v1f64(ptr %ptr) #0 {
 define void @dont_coalesce_res_v16i8(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v16i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v16i8
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <16 x i8> @get_v16i8()
   %vec = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> poison, <16 x i8> %res, i64 0)
@@ -1420,29 +1352,26 @@ define void @dont_coalesce_res_v16i8(ptr %ptr) #0 {
 define void @dont_coalesce_res_v8i16(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v8i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v8i16
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <8 x i16> @get_v8i16()
   %vec = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16> poison, <8 x i16> %res, i64 0)
@@ -1453,29 +1382,26 @@ define void @dont_coalesce_res_v8i16(ptr %ptr) #0 {
 define void @dont_coalesce_res_v4i32(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v4i32
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <4 x i32> @get_v4i32()
   %vec = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> poison, <4 x i32> %res, i64 0)
@@ -1486,29 +1412,26 @@ define void @dont_coalesce_res_v4i32(ptr %ptr) #0 {
 define void @dont_coalesce_res_v2i64(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v2i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v2i64
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <2 x i64> @get_v2i64()
   %vec = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> %res, i64 0)
@@ -1519,29 +1442,26 @@ define void @dont_coalesce_res_v2i64(ptr %ptr) #0 {
 define void @dont_coalesce_res_v8f16(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v8f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v8f16
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <8 x half> @get_v8f16()
   %vec = call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half> poison, <8 x half> %res, i64 0)
@@ -1552,29 +1472,26 @@ define void @dont_coalesce_res_v8f16(ptr %ptr) #0 {
 define void @dont_coalesce_res_v4f32(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v4f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v4f32
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <4 x float> @get_v4f32()
   %vec = call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %res, i64 0)
@@ -1585,29 +1502,26 @@ define void @dont_coalesce_res_v4f32(ptr %ptr) #0 {
 define void @dont_coalesce_res_v2f64(ptr %ptr) #0 {
 ; CHECK-LABEL: dont_coalesce_res_v2f64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl get_v2f64
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    str z0, [x19]
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #96] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <2 x double> @get_v2f64()
   %vec = call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> poison, <2 x double> %res, i64 0)
diff --git a/llvm/test/CodeGen/AArch64/sme-shared-za-interface.ll b/llvm/test/CodeGen/AArch64/sme-shared-za-interface.ll
index 393ff3b79aedf..b4ff8d085ff40 100644
--- a/llvm/test/CodeGen/AArch64/sme-shared-za-interface.ll
+++ b/llvm/test/CodeGen/AArch64/sme-shared-za-interface.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme  -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs -aarch64-new-sme-abi  < %s | FileCheck %s --check-prefix=CHECK-NEWLOWERING
 
 declare void @private_za_callee()
 
@@ -32,6 +33,31 @@ define void @disable_tailcallopt() "aarch64_inout_za" nounwind {
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: disable_tailcallopt:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    bl private_za_callee
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB0_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB0_2:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   tail call void @private_za_callee()
   ret void
 }
@@ -65,6 +91,31 @@ define fp128 @f128_call_za(fp128 %a, fp128 %b) "aarch64_inout_za" nounwind {
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: f128_call_za:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    bl __addtf3
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB1_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB1_2:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   %res = fadd fp128 %a, %b
   ret fp128 %res
 }
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll b/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll
index 1a49da84c00ce..39ea180e7ed81 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll
@@ -8,22 +8,17 @@ declare void @streaming_compatible_callee() "aarch64_pstate_sm_compatible";
 define float @sm_body_sm_compatible_simple() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: sm_body_sm_compatible_simple:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x8, x0, #0x1
-; CHECK-NEXT:    tbnz w8, #0, .LBB0_2
+; CHECK-NEXT:    tbnz w0, #0, .LBB0_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB0_2:
-; CHECK-NEXT:    tbnz w8, #0, .LBB0_4
+; CHECK-NEXT:    tbnz w0, #0, .LBB0_4
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB0_4:
@@ -32,7 +27,7 @@ define float @sm_body_sm_compatible_simple() "aarch64_pstate_sm_compatible" "aar
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   ret float zeroinitializer
 }
@@ -40,17 +35,13 @@ define float @sm_body_sm_compatible_simple() "aarch64_pstate_sm_compatible" "aar
 define void @sm_body_caller_sm_compatible_caller_normal_callee() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: sm_body_caller_sm_compatible_caller_normal_callee:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbnz w19, #0, .LBB1_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstart sm
@@ -62,12 +53,11 @@ define void @sm_body_caller_sm_compatible_caller_normal_callee() "aarch64_pstate
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   call void @normal_callee()
   ret void
@@ -77,18 +67,14 @@ define void @sm_body_caller_sm_compatible_caller_normal_callee() "aarch64_pstate
 define void @streaming_body_and_streaming_compatible_interface_multi_basic_block(i32 noundef %x) "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: streaming_body_and_streaming_compatible_interface_multi_basic_block:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov w8, w0
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbnz w19, #0, .LBB2_2
 ; CHECK-NEXT:  // %bb.1: // %entry
 ; CHECK-NEXT:    smstart sm
@@ -100,12 +86,11 @@ define void @streaming_body_and_streaming_compatible_interface_multi_basic_block
 ; CHECK-NEXT:  // %bb.4: // %if.else
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB2_5: // %if.else
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB2_6: // %if.then
 ; CHECK-NEXT:    smstop sm
@@ -115,12 +100,11 @@ define void @streaming_body_and_streaming_compatible_interface_multi_basic_block
 ; CHECK-NEXT:  // %bb.7: // %if.then
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB2_8: // %if.then
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
   %cmp = icmp eq i32 %x, 0
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-body.ll b/llvm/test/CodeGen/AArch64/sme-streaming-body.ll
index dd336e0f2e686..a3ec2ddb2b872 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-body.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-body.ll
@@ -8,15 +8,11 @@ declare void @streaming_compatible_callee() "aarch64_pstate_sm_compatible";
 define void @locally_streaming_caller_streaming_callee() "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: locally_streaming_caller_streaming_callee:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    bl streaming_compatible_callee
 ; CHECK-NEXT:    bl streaming_compatible_callee
@@ -25,7 +21,7 @@ define void @locally_streaming_caller_streaming_callee() "aarch64_pstate_sm_body
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 
   call void @streaming_compatible_callee();
@@ -51,33 +47,26 @@ define void @streaming_and_locally_streaming_caller_streaming_callee() "aarch64_
 define void @locally_streaming_multiple_exit(i64 %cond) "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: locally_streaming_multiple_exit:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    rdsvl x9, #1
-; CHECK-NEXT:    lsr x9, x9, #3
-; CHECK-NEXT:    str x9, [sp, #-80]! // 8-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    str x9, [sp, #8] // 8-byte Folded Spill
-; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    cmp x0, #1
 ; CHECK-NEXT:    b.ne .LBB2_2
 ; CHECK-NEXT:  // %bb.1: // %if.else
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB2_2: // %if.end
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 
 entry:
@@ -98,16 +87,11 @@ if.end:
 define <2 x i64> @locally_streaming_caller_no_callee(<2 x i64> %a) "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: locally_streaming_caller_no_callee:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #96
-; CHECK-NEXT:    rdsvl x9, #1
-; CHECK-NEXT:    stp d15, d14, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
-; CHECK-NEXT:    stp d13, d12, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    str x9, [sp, #24] // 8-byte Folded Spill
-; CHECK-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    index z0.d, #0, #1
@@ -118,12 +102,12 @@ define <2 x i64> @locally_streaming_caller_no_callee(<2 x i64> %a) "aarch64_psta
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldp d9, d8, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d11, d10, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d13, d12, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
 
   %add = add <2 x i64> %a, <i64 41, i64 42>;
@@ -155,16 +139,12 @@ define void @locally_streaming_caller_locally_streaming_callee() "aarch64_pstate
 define <2 x i64> @locally_streaming_caller_compatible_callee_vec_args_ret(<2 x i64> %a) "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: locally_streaming_caller_compatible_callee_vec_args_ret:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    sub sp, sp, #96
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x9, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
@@ -177,7 +157,7 @@ define <2 x i64> @locally_streaming_caller_compatible_callee_vec_args_ret(<2 x i
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #112
+; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
   %res = call <2 x i64> @streaming_compatible_callee_vec_args_ret(<2 x i64> %a) "aarch64_pstate_sm_compatible"
   ret <2 x i64> %res;
@@ -188,16 +168,12 @@ declare <2 x i64> @streaming_compatible_callee_vec_args_ret(<2 x i64>) "aarch64_
 define {<2 x i64>, <2 x i64>} @locally_streaming_caller_compatible_callee_struct_arg_ret({<2 x i64>, <2 x i64>} %arg) "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: locally_streaming_caller_compatible_callee_struct_arg_ret:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #128
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    sub sp, sp, #112
 ; CHECK-NEXT:    stp d15, d14, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d13, d12, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #96] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    str x9, [sp, #112] // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #96] // 8-byte Folded Spill
 ; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
@@ -210,7 +186,7 @@ define {<2 x i64>, <2 x i64>} @locally_streaming_caller_compatible_callee_struct
 ; CHECK-NEXT:    ldp d11, d10, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    add sp, sp, #112
 ; CHECK-NEXT:    ret
   %v1.arg = extractvalue {<2 x i64>, <2 x i64>} %arg, 1
   %res = call {<2 x i64>, <2 x i64>} @streaming_compatible_callee_vec_arg_struct_ret(<2 x i64> %v1.arg) "aarch64_pstate_sm_compatible"
@@ -224,16 +200,11 @@ declare {<2 x i64>, <2 x i64>} @streaming_compatible_callee_vec_arg_struct_ret(<
 define void @locally_streaming_caller_alloca() nounwind "aarch64_pstate_sm_body" {
 ; CHECK-LABEL: locally_streaming_caller_alloca:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x9, [sp, #88] // 8-byte Folded Spill
 ; CHECK-NEXT:    addsvl sp, sp, #-1
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    mov x0, sp
@@ -244,7 +215,7 @@ define void @locally_streaming_caller_alloca() nounwind "aarch64_pstate_sm_body"
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %alloca = alloca <vscale x 4 x i32>
   call void @use_ptr(ptr %alloca) "aarch64_pstate_sm_compatible"
@@ -271,16 +242,11 @@ declare double @llvm.cos.f64(double)
 define float @test_arg_survives_loop(float %arg, i32 %N) nounwind "aarch64_pstate_sm_body" {
 ; CHECK-LABEL: test_arg_survives_loop:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sub sp, sp, #96
-; CHECK-NEXT:    rdsvl x9, #1
-; CHECK-NEXT:    stp d15, d14, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
-; CHECK-NEXT:    stp d13, d12, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    str x9, [sp, #24] // 8-byte Folded Spill
-; CHECK-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB9_1: // %for.body
@@ -293,12 +259,12 @@ define float @test_arg_survives_loop(float %arg, i32 %N) nounwind "aarch64_pstat
 ; CHECK-NEXT:    fadd s0, s1, s0
 ; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldp d9, d8, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
-; CHECK-NEXT:    ldp d11, d10, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d13, d12, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
 entry:
   br label %for.body
@@ -318,15 +284,11 @@ for.cond.cleanup:
 define void @disable_tailcallopt() "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: disable_tailcallopt:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    bl streaming_compatible_callee
 ; CHECK-NEXT:    smstop sm
@@ -334,7 +296,7 @@ define void @disable_tailcallopt() "aarch64_pstate_sm_body" nounwind {
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   tail call void @streaming_compatible_callee();
   ret void;
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll b/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll
index e967f3b7be5e8..ff4f36363edcf 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll
@@ -36,15 +36,13 @@ define void @normal_caller_streaming_compatible_callee() nounwind {
 define void @streaming_compatible_caller_normal_callee() "aarch64_pstate_sm_compatible" nounwind {
 ; CHECK-LABEL: streaming_compatible_caller_normal_callee:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbz w19, #0, .LBB1_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
@@ -54,12 +52,11 @@ define void @streaming_compatible_caller_normal_callee() "aarch64_pstate_sm_comp
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 
   call void @normal_callee();
@@ -75,15 +72,13 @@ define void @streaming_compatible_caller_normal_callee() "aarch64_pstate_sm_comp
 define void @streaming_compatible_caller_streaming_callee() "aarch64_pstate_sm_compatible" nounwind {
 ; CHECK-LABEL: streaming_compatible_caller_streaming_callee:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbnz w19, #0, .LBB2_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstart sm
@@ -93,12 +88,11 @@ define void @streaming_compatible_caller_streaming_callee() "aarch64_pstate_sm_c
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB2_4:
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 
   call void @streaming_callee();
@@ -130,12 +124,11 @@ define <2 x double> @streaming_compatible_with_neon_vectors(<2 x double> %arg) "
 ; CHECK-LABEL: streaming_compatible_with_neon_vectors:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    add x8, sp, #16
@@ -143,10 +136,10 @@ define <2 x double> @streaming_compatible_with_neon_vectors(<2 x double> %arg) "
 ; CHECK-NEXT:    str z0, [x8] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
 ; CHECK-NEXT:    add x8, sp, #16
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    ldr z0, [x8] // 16-byte Folded Reload
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    and x19, x0, #0x1
 ; CHECK-NEXT:    tbz w19, #0, .LBB4_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
@@ -167,8 +160,8 @@ define <2 x double> @streaming_compatible_with_neon_vectors(<2 x double> %arg) "
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -183,9 +176,8 @@ declare <2 x double> @normal_callee_vec_arg(<2 x double>)
 define <vscale x 2 x double> @streaming_compatible_with_scalable_vectors(<vscale x 2 x double> %arg) "aarch64_pstate_sm_compatible" nounwind {
 ; CHECK-LABEL: streaming_compatible_with_scalable_vectors:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    stp x9, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-18
 ; CHECK-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
@@ -218,7 +210,7 @@ define <vscale x 2 x double> @streaming_compatible_with_scalable_vectors(<vscale
 ; CHECK-NEXT:    addvl sp, sp, #-2
 ; CHECK-NEXT:    str z0, [sp, #1, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbz w19, #0, .LBB5_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
@@ -263,8 +255,8 @@ define <vscale x 2 x double> @streaming_compatible_with_scalable_vectors(<vscale
 ; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #18
-; CHECK-NEXT:    ldr x19, [sp, #24] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x double> @normal_callee_scalable_vec_arg(<vscale x 2 x double> %arg)
   %fadd = fadd <vscale x 2 x double> %res, %arg
@@ -276,9 +268,8 @@ declare <vscale x 2 x double> @normal_callee_scalable_vec_arg(<vscale x 2 x doub
 define <vscale x 2 x i1> @streaming_compatible_with_predicate_vectors(<vscale x 2 x i1> %arg) "aarch64_pstate_sm_compatible" nounwind {
 ; CHECK-LABEL: streaming_compatible_with_predicate_vectors:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    stp x9, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-18
 ; CHECK-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
@@ -311,7 +302,7 @@ define <vscale x 2 x i1> @streaming_compatible_with_predicate_vectors(<vscale x
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    str p0, [sp, #7, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbz w19, #0, .LBB6_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
@@ -356,8 +347,8 @@ define <vscale x 2 x i1> @streaming_compatible_with_predicate_vectors(<vscale x
 ; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #18
-; CHECK-NEXT:    ldr x19, [sp, #24] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %res = call <vscale x 2 x i1> @normal_callee_predicate_vec_arg(<vscale x 2 x i1> %arg)
   %and = and <vscale x 2 x i1> %res, %arg
@@ -369,24 +360,18 @@ declare <vscale x 2 x i1> @normal_callee_predicate_vec_arg(<vscale x 2 x i1>)
 define i32 @conditional_smstart_unreachable_block() "aarch64_pstate_sm_compatible" nounwind {
 ; CHECK-LABEL: conditional_smstart_unreachable_block:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbnz w19, #0, .LBB7_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB7_2:
 ; CHECK-NEXT:    bl streaming_callee
-; CHECK-NEXT:    tbnz w19, #0, .LBB7_4
-; CHECK-NEXT:  // %bb.3:
-; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:  .LBB7_4:
   call void @streaming_callee()
   unreachable
 }
@@ -394,33 +379,30 @@ define i32 @conditional_smstart_unreachable_block() "aarch64_pstate_sm_compatibl
 define void @conditional_smstart_no_successor_block(i1 %p) "aarch64_pstate_sm_compatible" nounwind {
 ; CHECK-LABEL: conditional_smstart_no_successor_block:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    tbz w0, #0, .LBB8_6
-; CHECK-NEXT:  // %bb.1: // %if.then
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov w8, w0
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
-; CHECK-NEXT:    tbnz w19, #0, .LBB8_3
+; CHECK-NEXT:    tbz w8, #0, .LBB8_5
+; CHECK-NEXT:  // %bb.1: // %if.then
+; CHECK-NEXT:    tbnz w0, #0, .LBB8_3
 ; CHECK-NEXT:  // %bb.2: // %if.then
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB8_3: // %if.then
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    bl streaming_callee
 ; CHECK-NEXT:    tbnz w19, #0, .LBB8_5
 ; CHECK-NEXT:  // %bb.4: // %if.then
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:  .LBB8_5: // %if.then
+; CHECK-NEXT:  .LBB8_5: // %exit
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
-; CHECK-NEXT:  .LBB8_6: // %exit
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   br i1 %p, label %if.then, label %exit
 
@@ -435,15 +417,13 @@ exit:
 define void @disable_tailcallopt() "aarch64_pstate_sm_compatible" nounwind {
 ; CHECK-LABEL: disable_tailcallopt:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbz w19, #0, .LBB9_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
@@ -453,12 +433,11 @@ define void @disable_tailcallopt() "aarch64_pstate_sm_compatible" nounwind {
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB9_4:
+; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 
   tail call void @normal_callee();
@@ -475,10 +454,12 @@ define void @call_to_non_streaming_pass_args(ptr nocapture noundef readnone %ptr
 ; CHECK-NEXT:    stp d13, d12, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #96] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #112] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w19, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    stp x29, x30, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x9, x19, [sp, #112] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
 ; CHECK-NEXT:    .cfi_offset b9, -48
 ; CHECK-NEXT:    .cfi_offset b10, -56
@@ -492,8 +473,7 @@ define void @call_to_non_streaming_pass_args(ptr nocapture noundef readnone %ptr
 ; CHECK-NEXT:    mov x9, x0
 ; CHECK-NEXT:    stp s0, s1, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
-; CHECK-NEXT:    .cfi_offset vg, -24
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbz w19, #0, .LBB10_2
 ; CHECK-NEXT:  // %bb.1: // %entry
 ; CHECK-NEXT:    smstop sm
@@ -507,17 +487,18 @@ define void @call_to_non_streaming_pass_args(ptr nocapture noundef readnone %ptr
 ; CHECK-NEXT:  // %bb.3: // %entry
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB10_4: // %entry
-; CHECK-NEXT:    .cfi_restore vg
+; CHECK-NEXT:    ldp x29, x30, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x19, [sp, #120] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #80] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #112] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #96] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #128
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-interface.ll b/llvm/test/CodeGen/AArch64/sme-streaming-interface.ll
index 438b941198449..8c4d57e244e03 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-interface.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-interface.ll
@@ -22,11 +22,10 @@ define void @normal_caller_streaming_callee() nounwind {
 ; CHECK-LABEL: normal_caller_streaming_callee:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    bl streaming_callee
 ; CHECK-NEXT:    smstop sm
@@ -48,11 +47,10 @@ define void @streaming_caller_normal_callee() nounwind "aarch64_pstate_sm_enable
 ; CHECK-LABEL: streaming_caller_normal_callee:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl normal_callee
 ; CHECK-NEXT:    smstart sm
@@ -105,11 +103,10 @@ define void @call_to_function_pointer_streaming_enabled(ptr %p) nounwind {
 ; CHECK-LABEL: call_to_function_pointer_streaming_enabled:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    blr x0
 ; CHECK-NEXT:    smstop sm
@@ -128,20 +125,19 @@ define <4 x i32> @smstart_clobber_simdfp(<4 x i32> %x) nounwind {
 ; CHECK-LABEL: smstart_clobber_simdfp:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #96
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    bl streaming_callee
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #96
@@ -154,9 +150,7 @@ define <4 x i32> @smstart_clobber_simdfp(<4 x i32> %x) nounwind {
 define <vscale x 4 x i32> @smstart_clobber_sve(<vscale x 4 x i32> %x) nounwind {
 ; CHECK-LABEL: smstart_clobber_sve:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-18
 ; CHECK-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
@@ -222,7 +216,7 @@ define <vscale x 4 x i32> @smstart_clobber_sve(<vscale x 4 x i32> %x) nounwind {
 ; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #18
-; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   call void @streaming_callee()
   ret <vscale x 4 x i32> %x;
@@ -233,9 +227,7 @@ define <vscale x 4 x i32> @smstart_clobber_sve(<vscale x 4 x i32> %x) nounwind {
 define <vscale x 4 x i32> @smstart_clobber_sve_duplicate(<vscale x 4 x i32> %x) nounwind {
 ; CHECK-LABEL: smstart_clobber_sve_duplicate:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-18
 ; CHECK-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
@@ -302,7 +294,7 @@ define <vscale x 4 x i32> @smstart_clobber_sve_duplicate(<vscale x 4 x i32> %x)
 ; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #18
-; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   call void @streaming_callee()
   call void @streaming_callee()
@@ -314,12 +306,11 @@ define double @call_to_intrinsic_without_chain(double %x) nounwind "aarch64_psta
 ; CHECK-LABEL: call_to_intrinsic_without_chain:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #96
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    stp d0, d0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    ldr d0, [sp] // 8-byte Folded Reload
@@ -327,11 +318,11 @@ define double @call_to_intrinsic_without_chain(double %x) nounwind "aarch64_psta
 ; CHECK-NEXT:    str d0, [sp] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldp d1, d0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    fadd d0, d1, d0
-; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    fadd d0, d1, d0
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #96
 ; CHECK-NEXT:    ret
@@ -349,11 +340,10 @@ define void @disable_tailcallopt() nounwind {
 ; CHECK-LABEL: disable_tailcallopt:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    bl streaming_callee
 ; CHECK-NEXT:    smstop sm
@@ -370,13 +360,11 @@ define void @disable_tailcallopt() nounwind {
 define i8 @call_to_non_streaming_pass_sve_objects(ptr nocapture noundef readnone %ptr) #0 {
 ; CHECK-LABEL: call_to_non_streaming_pass_sve_objects:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-3
 ; CHECK-NEXT:    rdsvl x3, #1
 ; CHECK-NEXT:    addvl x0, sp, #2
@@ -392,7 +380,7 @@ define i8 @call_to_non_streaming_pass_sve_objects(ptr nocapture noundef readnone
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 entry:
   %Data1 = alloca <vscale x 16 x i8>, align 16
@@ -409,12 +397,11 @@ define void @call_to_non_streaming_pass_args(ptr nocapture noundef readnone %ptr
 ; CHECK-LABEL: call_to_non_streaming_pass_args:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub sp, sp, #112
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d15, d14, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #96] // 8-byte Folded Spill
 ; CHECK-NEXT:    stp d2, d3, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp s0, s1, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-mode-changes-unwindinfo.ll b/llvm/test/CodeGen/AArch64/sme-streaming-mode-changes-unwindinfo.ll
new file mode 100644
index 0000000000000..991776f11ae40
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-mode-changes-unwindinfo.ll
@@ -0,0 +1,308 @@
+; DEFINE: %{compile} = llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sme -mattr=+sve -verify-machineinstrs -enable-aarch64-sme-peephole-opt=false < %s
+; RUN: %{compile} | FileCheck %s
+; RUN: %{compile} -filetype=obj -o %t
+; RUN: llvm-objdump --dwarf=frames %t | FileCheck %s --check-prefix=UNWINDINFO
+
+; This tests that functions with streaming mode changes use explicitly use the
+; "IncomingVG" (the value of VG on entry to the function) in SVE unwind information.
+;
+; [ ] N  ->  S    (Normal -> Streaming, mode change)
+; [ ] S  ->  N    (Streaming -> Normal, mode change)
+; [ ] N  ->  N    (Normal -> Normal, no mode change)
+; [ ] S  ->  S    (Streaming -> Streaming, no mode change)
+; [ ] LS ->  S    (Locally-streaming -> Streaming, mode change)
+; [ ] SC ->  S    (Streaming-compatible -> Streaming, mode change)
+
+declare void @normal_callee()
+declare void @streaming_callee() "aarch64_pstate_sm_enabled"
+
+; [x] N  ->  S
+; [ ] S  ->  N
+; [ ] N  ->  N
+; [ ] S  ->  S
+; [ ] LS ->  S
+; [ ] SC ->  S
+define aarch64_sve_vector_pcs void @normal_caller_streaming_callee() {
+; CHECK-LABEL: normal_caller_streaming_callee:
+; CHECK:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK:    .cfi_def_cfa_offset 32
+; CHECK:    cntd x9
+; CHECK:    stp x9, x28, [sp, #16] // 16-byte Folded Spill
+; CHECK:    mov x29, sp
+; CHECK:    .cfi_def_cfa w29, 32
+; CHECK:    .cfi_offset vg, -16
+; CHECK:    addvl sp, sp, #-18
+; CHECK:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK:    .cfi_escape 0x10, 0x48, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d8 @ cfa - 8 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x49, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d9 @ cfa - 16 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4a, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d10 @ cfa - 24 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4b, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d11 @ cfa - 32 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4c, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d12 @ cfa - 40 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4d, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d13 @ cfa - 48 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4e, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d14 @ cfa - 56 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4f, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d15 @ cfa - 64 * IncomingVG - 32
+; CHECK:    smstart sm
+; CHECK:    bl streaming_callee
+; CHECK:    smstop sm
+;
+; UNWINDINFO:      DW_CFA_def_cfa: reg29 +32
+; UNWINDINFO:      DW_CFA_offset: reg46 -16
+; UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -8, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -16, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -24, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -32, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -40, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -48, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -56, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -64, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+  call void @streaming_callee()
+  ret void
+}
+
+; [ ] N  ->  S
+; [x] S  ->  N
+; [ ] N  ->  N
+; [ ] S  ->  S
+; [ ] LS ->  S
+; [ ] SC ->  S
+define aarch64_sve_vector_pcs void @streaming_caller_normal_callee() "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: streaming_caller_normal_callee:
+; CHECK:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK:    .cfi_def_cfa_offset 32
+; CHECK:    cntd x9
+; CHECK:    stp x9, x28, [sp, #16] // 16-byte Folded Spill
+; CHECK:    mov x29, sp
+; CHECK:    .cfi_def_cfa w29, 32
+; CHECK:    .cfi_offset vg, -16
+; CHECK:    addvl sp, sp, #-18
+; CHECK:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK:    .cfi_escape 0x10, 0x48, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d8 @ cfa - 8 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x49, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d9 @ cfa - 16 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4a, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d10 @ cfa - 24 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4b, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d11 @ cfa - 32 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4c, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d12 @ cfa - 40 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4d, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d13 @ cfa - 48 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4e, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d14 @ cfa - 56 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4f, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d15 @ cfa - 64 * IncomingVG - 32
+; CHECK:    smstop sm
+; CHECK:    bl normal_callee
+; CHECK:    smstart sm
+;
+; UNWINDINFO:      DW_CFA_def_cfa: reg29 +32
+; UNWINDINFO:      DW_CFA_offset: reg46 -16
+; UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -8, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -16, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -24, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -32, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -40, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -48, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -56, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -64, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+  call void @normal_callee()
+  ret void
+}
+
+; [ ] N  ->  S
+; [ ] S  ->  N
+; [x] N  ->  N
+; [ ] S  ->  S
+; [ ] LS ->  S
+; [ ] SC ->  S
+define aarch64_sve_vector_pcs void @normal_caller_normal_callee() {
+; CHECK-LABEL: normal_caller_normal_callee:
+; CHECK:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK:    addvl sp, sp, #-18
+; CHECK:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 16 + 144 * VG
+; CHECK:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4b, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x40, 0x1c // $d11 @ cfa - 32 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4c, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x40, 0x1c // $d12 @ cfa - 40 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4d, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x40, 0x1c // $d13 @ cfa - 48 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4e, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x40, 0x1c // $d14 @ cfa - 56 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16
+; CHECK:    bl normal_callee
+;
+; UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +16, DW_OP_bregx 0x2e +0, DW_OP_consts +144, DW_OP_mul, DW_OP_plus
+; UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_bregx 0x2e +0, DW_OP_consts -8, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_bregx 0x2e +0, DW_OP_consts -16, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_bregx 0x2e +0, DW_OP_consts -24, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_bregx 0x2e +0, DW_OP_consts -32, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_bregx 0x2e +0, DW_OP_consts -40, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_bregx 0x2e +0, DW_OP_consts -48, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_bregx 0x2e +0, DW_OP_consts -56, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_bregx 0x2e +0, DW_OP_consts -64, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+  call void @normal_callee()
+  ret void
+}
+
+; [ ] N  ->  S
+; [ ] S  ->  N
+; [ ] N  ->  N
+; [x] S  ->  S
+; [ ] LS ->  S
+; [ ] SC ->  S
+define aarch64_sve_vector_pcs void @streaming_caller_streaming_callee() "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: streaming_caller_streaming_callee:
+; CHECK:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK:    addvl sp, sp, #-18
+; CHECK:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 16 + 144 * VG
+; CHECK:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4b, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x40, 0x1c // $d11 @ cfa - 32 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4c, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x40, 0x1c // $d12 @ cfa - 40 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4d, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x40, 0x1c // $d13 @ cfa - 48 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4e, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x40, 0x1c // $d14 @ cfa - 56 * VG - 16
+; CHECK:    .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16
+; CHECK:    bl streaming_callee
+;
+; UNWINDINFO:      DW_CFA_def_cfa_expression: DW_OP_breg31 +16, DW_OP_bregx 0x2e +0, DW_OP_consts +144, DW_OP_mul, DW_OP_plus
+; UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_bregx 0x2e +0, DW_OP_consts -8, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_bregx 0x2e +0, DW_OP_consts -16, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_bregx 0x2e +0, DW_OP_consts -24, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_bregx 0x2e +0, DW_OP_consts -32, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_bregx 0x2e +0, DW_OP_consts -40, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_bregx 0x2e +0, DW_OP_consts -48, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_bregx 0x2e +0, DW_OP_consts -56, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_bregx 0x2e +0, DW_OP_consts -64, DW_OP_mul, DW_OP_plus, DW_OP_lit16, DW_OP_minus
+  call void @streaming_callee()
+  ret void
+}
+
+; [ ] N  ->  S
+; [ ] S  ->  N
+; [ ] N  ->  N
+; [ ] S  ->  S
+; [x] LS ->  S
+; [ ] SC ->  S
+define aarch64_sve_vector_pcs void @locally_streaming() "aarch64_pstate_sm_body" {
+; CHECK-LABEL: locally_streaming:
+; CHECK:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK:    .cfi_def_cfa_offset 32
+; CHECK:    cntd x9
+; CHECK:    stp x9, x28, [sp, #16] // 16-byte Folded Spill
+; CHECK:    mov x29, sp
+; CHECK:    .cfi_def_cfa w29, 32
+; CHECK:    .cfi_offset vg, -16
+; CHECK:    addsvl sp, sp, #-18
+; CHECK:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK:    .cfi_escape 0x10, 0x48, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d8 @ cfa - 8 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x49, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d9 @ cfa - 16 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4a, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d10 @ cfa - 24 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4b, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d11 @ cfa - 32 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4c, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d12 @ cfa - 40 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4d, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d13 @ cfa - 48 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4e, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d14 @ cfa - 56 * IncomingVG - 32
+; CHECK:    .cfi_escape 0x10, 0x4f, 0x0b, 0x12, 0x40, 0x1c, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d15 @ cfa - 64 * IncomingVG - 32
+; CHECK:    smstart sm
+; CHECK:    bl streaming_callee
+; CHECK:    smstop sm
+;
+; UNWINDINFO:      DW_CFA_def_cfa: reg29 +32
+; UNWINDINFO:      DW_CFA_offset: reg46 -16
+; UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -8, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -16, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -24, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -32, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -40, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -48, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -56, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_dup, DW_OP_lit16, DW_OP_minus, DW_OP_deref, DW_OP_consts -64, DW_OP_mul, DW_OP_plus, DW_OP_consts -32, DW_OP_plus
+  call void @streaming_callee()
+  ret void
+}
+
+; [ ] N  ->  S
+; [ ] S  ->  N
+; [ ] N  ->  N
+; [ ] S  ->  S
+; [ ] LS ->  S
+; [x] SC ->  S
+define aarch64_sve_vector_pcs void @streaming_compatible_caller_conditional_mode_switch() "aarch64_pstate_sm_compatible" {
+; CHECK-LABEL: streaming_compatible_caller_conditional_mode_switch:
+; CHECK:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
+; CHECK:    .cfi_def_cfa_offset 48
+; CHECK:    cntd x9
+; CHECK:    stp x28, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK:    str x9, [sp, #16] // 8-byte Folded Spill
+; CHECK:    mov x29, sp
+; CHECK:    .cfi_def_cfa w29, 48
+; CHECK:    .cfi_offset vg, -32
+; CHECK:    addvl sp, sp, #-18
+; CHECK:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK:    .cfi_escape 0x10, 0x48, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d8 @ cfa - 8 * IncomingVG - 48
+; CHECK:    .cfi_escape 0x10, 0x49, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d9 @ cfa - 16 * IncomingVG - 48
+; CHECK:    .cfi_escape 0x10, 0x4a, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d10 @ cfa - 24 * IncomingVG - 48
+; CHECK:    .cfi_escape 0x10, 0x4b, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d11 @ cfa - 32 * IncomingVG - 48
+; CHECK:    .cfi_escape 0x10, 0x4c, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d12 @ cfa - 40 * IncomingVG - 48
+; CHECK:    .cfi_escape 0x10, 0x4d, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d13 @ cfa - 48 * IncomingVG - 48
+; CHECK:    .cfi_escape 0x10, 0x4e, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d14 @ cfa - 56 * IncomingVG - 48
+; CHECK:    .cfi_escape 0x10, 0x4f, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d15 @ cfa - 64 * IncomingVG - 48
+; CHECK:    bl __arm_sme_state
+; CHECK:    mov x19, x0
+; CHECK:    tbnz w19, #0, .LBB5_2
+; CHECK:    smstart sm
+; CHECK:  .LBB5_2:
+; CHECK:    bl streaming_callee
+; CHECK:    tbnz w19, #0, .LBB5_4
+; CHECK:    smstop sm
+; CHECK:  .LBB5_4:
+;
+; UNWINDINFO:      DW_CFA_def_cfa: reg29 +48
+; UNWINDINFO:      DW_CFA_offset: reg46 -32
+; UNWINDINFO:      DW_CFA_expression: reg72 DW_OP_dup, DW_OP_consts -32, DW_OP_plus, DW_OP_deref, DW_OP_consts -8, DW_OP_mul, DW_OP_plus, DW_OP_consts -48, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg73 DW_OP_dup, DW_OP_consts -32, DW_OP_plus, DW_OP_deref, DW_OP_consts -16, DW_OP_mul, DW_OP_plus, DW_OP_consts -48, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg74 DW_OP_dup, DW_OP_consts -32, DW_OP_plus, DW_OP_deref, DW_OP_consts -24, DW_OP_mul, DW_OP_plus, DW_OP_consts -48, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg75 DW_OP_dup, DW_OP_consts -32, DW_OP_plus, DW_OP_deref, DW_OP_consts -32, DW_OP_mul, DW_OP_plus, DW_OP_consts -48, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg76 DW_OP_dup, DW_OP_consts -32, DW_OP_plus, DW_OP_deref, DW_OP_consts -40, DW_OP_mul, DW_OP_plus, DW_OP_consts -48, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg77 DW_OP_dup, DW_OP_consts -32, DW_OP_plus, DW_OP_deref, DW_OP_consts -48, DW_OP_mul, DW_OP_plus, DW_OP_consts -48, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg78 DW_OP_dup, DW_OP_consts -32, DW_OP_plus, DW_OP_deref, DW_OP_consts -56, DW_OP_mul, DW_OP_plus, DW_OP_consts -48, DW_OP_plus
+; UNWINDINFO-NEXT: DW_CFA_expression: reg79 DW_OP_dup, DW_OP_consts -32, DW_OP_plus, DW_OP_deref, DW_OP_consts -64, DW_OP_mul, DW_OP_plus, DW_OP_consts -48, DW_OP_plus
+  call void @streaming_callee()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-mode-changing-call-disable-stackslot-scavenging.ll b/llvm/test/CodeGen/AArch64/sme-streaming-mode-changing-call-disable-stackslot-scavenging.ll
index fe3f493353b50..7efa1d8f7a6a7 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-mode-changing-call-disable-stackslot-scavenging.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-mode-changing-call-disable-stackslot-scavenging.ll
@@ -15,12 +15,11 @@ define void @test_no_stackslot_scavenging(float %f) #0 {
 ; CHECK-LABEL: test_no_stackslot_scavenging:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x9, x24, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x29, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x30, x24, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
@@ -32,8 +31,8 @@ define void @test_no_stackslot_scavenging(float %f) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x24, [sp, #88] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x30, x24, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -48,21 +47,20 @@ define void @test_no_stackslot_scavenging(float %f) #0 {
 define void @test_no_stackslot_scavenging_with_fp(float %f, i64 %n) #0 "frame-pointer"="all" {
 ; CHECK-LABEL: test_no_stackslot_scavenging_with_fp:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-128]! // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp d15, d14, [sp, #-112]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    add x29, sp, #64
-; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
-; CHECK-NEXT:    stp x28, x25, [sp, #96] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x24, x19, [sp, #112] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x28, x25, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x24, x19, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    lsl x9, x0, #3
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    mov x19, sp
-; CHECK-NEXT:    str s0, [x29, #28] // 4-byte Folded Spill
+; CHECK-NEXT:    str s0, [x19, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    add x9, x9, #15
 ; CHECK-NEXT:    and x9, x9, #0xfffffffffffffff0
 ; CHECK-NEXT:    sub x8, x8, x9
@@ -70,17 +68,17 @@ define void @test_no_stackslot_scavenging_with_fp(float %f, i64 %n) #0 "frame-po
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    //NO_APP
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldr s0, [x29, #28] // 4-byte Folded Reload
+; CHECK-NEXT:    ldr s0, [x19, #12] // 4-byte Folded Reload
 ; CHECK-NEXT:    bl use_f
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    sub sp, x29, #64
-; CHECK-NEXT:    ldp x24, x19, [sp, #112] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp x28, x25, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x24, x19, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x28, x25, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #128 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #112 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %ptr2 = alloca i64, i64 %n, align 8
   %ptr = alloca <vscale x 16 x i8>
diff --git a/llvm/test/CodeGen/AArch64/sme-tileslice-addrmodes.ll b/llvm/test/CodeGen/AArch64/sme-tileslice-addrmodes.ll
new file mode 100644
index 0000000000000..4af3fabb2b4d5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-tileslice-addrmodes.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sme2 -enable-subreg-liveness < %s| FileCheck %s
+
+target triple = "aarch64"
+
+; The tile-slice addressing mode supports an immediate of 0-7.
+; This is testing an immediate of 0, 1, 7 (folded) and 8 (not folded).
+define void @sme_tileslice_addrmode_zero_base_plus_constant_offset(i32 %slice, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: sme_tileslice_addrmode_zero_base_plus_constant_offset:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, wzr
+; CHECK-NEXT:    mov w9, #8 // =0x8
+; CHECK-NEXT:    fdot za.s[w8, 0, vgx4], { z0.h - z3.h }, z4.h[0]
+; CHECK-NEXT:    fdot za.s[w8, 1, vgx4], { z0.h - z3.h }, z4.h[0]
+; CHECK-NEXT:    fdot za.s[w8, 7, vgx4], { z0.h - z3.h }, z4.h[0]
+; CHECK-NEXT:    fdot za.s[w9, 0, vgx4], { z0.h - z3.h }, z4.h[0]
+; CHECK-NEXT:    ret
+  tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 0, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4, i32 0)
+  tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 1, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4, i32 0)
+  tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 7, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4, i32 0)
+  tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 8, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4, i32 0)
+  ret void
+}
+
+; The tile-slice addressing mode supports an immediate of 0-7.
+; This is testing an immediate of 0, 1, 7 (folded) and 8 (not folded).
+define void @sme_tileslice_addrmode_base_plus_constant_offset(i32 %slice, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: sme_tileslice_addrmode_base_plus_constant_offset:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    add w9, w0, #8
+; CHECK-NEXT:    fdot za.s[w8, 0, vgx4], { z0.h - z3.h }, z4.h[0]
+; CHECK-NEXT:    fdot za.s[w8, 1, vgx4], { z0.h - z3.h }, z4.h[0]
+; CHECK-NEXT:    fdot za.s[w8, 7, vgx4], { z0.h - z3.h }, z4.h[0]
+; CHECK-NEXT:    fdot za.s[w9, 0, vgx4], { z0.h - z3.h }, z4.h[0]
+; CHECK-NEXT:    ret
+  %slice0 = add i32 %slice, 0
+  tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 %slice0, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4, i32 0)
+  %slice1 = add i32 %slice, 1
+  tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 %slice1, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4, i32 0)
+  %slice7 = add i32 %slice, 7
+  tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 %slice7, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4, i32 0)
+  %slice8 = add i32 %slice, 8
+  tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 %slice8, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4, i32 0)
+  ret void
+}
+
+define void @sme_tileslice_addrmode_base_plus_zero_offset(i32 %slice, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4) "aarch64_pstate_sm_enabled" {
+; CHECK-LABEL: sme_tileslice_addrmode_base_plus_zero_offset:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    fdot za.s[w8, 0, vgx4], { z0.h - z3.h }, z4.h[0]
+; CHECK-NEXT:    ret
+  tail call void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32 %slice, <vscale x 8 x half> %0, <vscale x 8 x half> %1, <vscale x 8 x half> %2, <vscale x 8 x half> %3, <vscale x 8 x half> %4, i32 0)
+  ret void
+}
+
+declare void @llvm.aarch64.sme.fdot.lane.za32.vg1x4.nxv8f16(i32, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, i32 immarg)
diff --git a/llvm/test/CodeGen/AArch64/sme-vg-to-stack.ll b/llvm/test/CodeGen/AArch64/sme-vg-to-stack.ll
index 6fcfc5b242c11..dec8eb0d8a936 100644
--- a/llvm/test/CodeGen/AArch64/sme-vg-to-stack.ll
+++ b/llvm/test/CodeGen/AArch64/sme-vg-to-stack.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -verify-machineinstrs < %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-streaming-hazard-size=0 -mattr=+sve -mattr=+sme2 -frame-pointer=non-leaf -verify-machineinstrs < %s | FileCheck %s --check-prefix=FP-CHECK
 ; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -frame-pointer=non-leaf -verify-machineinstrs < %s | FileCheck %s --check-prefix=NO-SVE-CHECK
@@ -15,34 +16,37 @@ declare void @streaming_callee_with_arg(i32) #0;
 define void @vg_unwind_simple() #0 {
 ; CHECK-LABEL: vg_unwind_simple:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
 ; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    .cfi_offset b8, -24
-; CHECK-NEXT:    .cfi_offset b9, -32
-; CHECK-NEXT:    .cfi_offset b10, -40
-; CHECK-NEXT:    .cfi_offset b11, -48
-; CHECK-NEXT:    .cfi_offset b12, -56
-; CHECK-NEXT:    .cfi_offset b13, -64
-; CHECK-NEXT:    .cfi_offset b14, -72
-; CHECK-NEXT:    .cfi_offset b15, -80
-; CHECK-NEXT:    .cfi_offset vg, -8
+; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
+; CHECK-NEXT:    .cfi_offset b8, -40
+; CHECK-NEXT:    .cfi_offset b9, -48
+; CHECK-NEXT:    .cfi_offset b10, -56
+; CHECK-NEXT:    .cfi_offset b11, -64
+; CHECK-NEXT:    .cfi_offset b12, -72
+; CHECK-NEXT:    .cfi_offset b13, -80
+; CHECK-NEXT:    .cfi_offset b14, -88
+; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
@@ -65,6 +69,7 @@ define void @vg_unwind_simple() #0 {
 ; FP-CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
 ; FP-CHECK-NEXT:    add x29, sp, #64
 ; FP-CHECK-NEXT:    .cfi_def_cfa w29, 32
+; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    .cfi_offset w30, -24
 ; FP-CHECK-NEXT:    .cfi_offset w29, -32
 ; FP-CHECK-NEXT:    .cfi_offset b8, -40
@@ -75,11 +80,9 @@ define void @vg_unwind_simple() #0 {
 ; FP-CHECK-NEXT:    .cfi_offset b13, -80
 ; FP-CHECK-NEXT:    .cfi_offset b14, -88
 ; FP-CHECK-NEXT:    .cfi_offset b15, -96
-; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    smstop sm
 ; FP-CHECK-NEXT:    bl callee
 ; FP-CHECK-NEXT:    smstart sm
-; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_def_cfa wsp, 96
 ; FP-CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
@@ -87,6 +90,7 @@ define void @vg_unwind_simple() #0 {
 ; FP-CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 0
+; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_restore w30
 ; FP-CHECK-NEXT:    .cfi_restore w29
 ; FP-CHECK-NEXT:    .cfi_restore b8
@@ -101,7 +105,6 @@ define void @vg_unwind_simple() #0 {
 ;
 ; OUTLINER-CHECK-LABEL: vg_unwind_simple:
 ; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
-;
   call void @callee();
   ret void;
 }
@@ -118,10 +121,12 @@ define void @vg_unwind_needs_gap() #0 {
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x20, [sp, #80] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w20, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x9, x20, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w20, -8
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
 ; CHECK-NEXT:    .cfi_offset b9, -48
 ; CHECK-NEXT:    .cfi_offset b10, -56
@@ -132,20 +137,20 @@ define void @vg_unwind_needs_gap() #0 {
 ; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    .cfi_offset vg, -24
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x20, [sp, #88] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x20, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
@@ -169,6 +174,7 @@ define void @vg_unwind_needs_gap() #0 {
 ; FP-CHECK-NEXT:    add x29, sp, #64
 ; FP-CHECK-NEXT:    .cfi_def_cfa w29, 32
 ; FP-CHECK-NEXT:    .cfi_offset w20, -8
+; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    .cfi_offset w30, -24
 ; FP-CHECK-NEXT:    .cfi_offset w29, -32
 ; FP-CHECK-NEXT:    .cfi_offset b8, -40
@@ -181,11 +187,9 @@ define void @vg_unwind_needs_gap() #0 {
 ; FP-CHECK-NEXT:    .cfi_offset b15, -96
 ; FP-CHECK-NEXT:    //APP
 ; FP-CHECK-NEXT:    //NO_APP
-; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    smstop sm
 ; FP-CHECK-NEXT:    bl callee
 ; FP-CHECK-NEXT:    smstart sm
-; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_def_cfa wsp, 96
 ; FP-CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    ldr x20, [sp, #88] // 8-byte Folded Reload
@@ -195,6 +199,7 @@ define void @vg_unwind_needs_gap() #0 {
 ; FP-CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; FP-CHECK-NEXT:    .cfi_restore w20
+; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_restore w30
 ; FP-CHECK-NEXT:    .cfi_restore w29
 ; FP-CHECK-NEXT:    .cfi_restore b8
@@ -218,38 +223,41 @@ define void @vg_unwind_needs_gap() #0 {
 define void @vg_unwind_with_fixed_args(<4 x i32> %x) #0 {
 ; CHECK-LABEL: vg_unwind_with_fixed_args:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #96
-; CHECK-NEXT:    .cfi_def_cfa_offset 96
+; CHECK-NEXT:    sub sp, sp, #112
+; CHECK-NEXT:    .cfi_def_cfa_offset 112
 ; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #80] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    .cfi_offset b8, -24
-; CHECK-NEXT:    .cfi_offset b9, -32
-; CHECK-NEXT:    .cfi_offset b10, -40
-; CHECK-NEXT:    .cfi_offset b11, -48
-; CHECK-NEXT:    .cfi_offset b12, -56
-; CHECK-NEXT:    .cfi_offset b13, -64
-; CHECK-NEXT:    .cfi_offset b14, -72
-; CHECK-NEXT:    .cfi_offset b15, -80
+; CHECK-NEXT:    stp x29, x30, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    str x9, [sp, #96] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
+; CHECK-NEXT:    .cfi_offset b8, -40
+; CHECK-NEXT:    .cfi_offset b9, -48
+; CHECK-NEXT:    .cfi_offset b10, -56
+; CHECK-NEXT:    .cfi_offset b11, -64
+; CHECK-NEXT:    .cfi_offset b12, -72
+; CHECK-NEXT:    .cfi_offset b13, -80
+; CHECK-NEXT:    .cfi_offset b14, -88
+; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset vg, -8
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl fixed_callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
+; CHECK-NEXT:    ldp x29, x30, [sp, #80] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    add sp, sp, #112
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
@@ -273,6 +281,7 @@ define void @vg_unwind_with_fixed_args(<4 x i32> %x) #0 {
 ; FP-CHECK-NEXT:    str x9, [sp, #96] // 8-byte Folded Spill
 ; FP-CHECK-NEXT:    add x29, sp, #80
 ; FP-CHECK-NEXT:    .cfi_def_cfa w29, 32
+; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    .cfi_offset w30, -24
 ; FP-CHECK-NEXT:    .cfi_offset w29, -32
 ; FP-CHECK-NEXT:    .cfi_offset b8, -40
@@ -284,12 +293,10 @@ define void @vg_unwind_with_fixed_args(<4 x i32> %x) #0 {
 ; FP-CHECK-NEXT:    .cfi_offset b14, -88
 ; FP-CHECK-NEXT:    .cfi_offset b15, -96
 ; FP-CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
-; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    smstop sm
 ; FP-CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    bl fixed_callee
 ; FP-CHECK-NEXT:    smstart sm
-; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_def_cfa wsp, 112
 ; FP-CHECK-NEXT:    ldp x29, x30, [sp, #80] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
@@ -298,6 +305,7 @@ define void @vg_unwind_with_fixed_args(<4 x i32> %x) #0 {
 ; FP-CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    add sp, sp, #112
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 0
+; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_restore w30
 ; FP-CHECK-NEXT:    .cfi_restore w29
 ; FP-CHECK-NEXT:    .cfi_restore b8
@@ -320,15 +328,19 @@ define void @vg_unwind_with_fixed_args(<4 x i32> %x) #0 {
 define void @vg_unwind_with_sve_args(<vscale x 2 x i64> %x) #0 {
 ; CHECK-LABEL: vg_unwind_with_sve_args:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    stp x9, x28, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w28, -8
-; CHECK-NEXT:    .cfi_offset w30, -24
-; CHECK-NEXT:    .cfi_offset w29, -32
+; CHECK-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    .cfi_def_cfa w29, 48
+; CHECK-NEXT:    .cfi_offset w27, -8
+; CHECK-NEXT:    .cfi_offset w28, -16
+; CHECK-NEXT:    .cfi_offset vg, -32
+; CHECK-NEXT:    .cfi_offset w30, -40
+; CHECK-NEXT:    .cfi_offset w29, -48
 ; CHECK-NEXT:    addvl sp, sp, #-18
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 32 + 144 * VG
 ; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    ptrue pn8.b
 ; CHECK-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
@@ -351,27 +363,23 @@ define void @vg_unwind_with_sve_args(<vscale x 2 x i64> %x) #0 {
 ; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d8 @ cfa - 8 * VG - 32
-; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d9 @ cfa - 16 * VG - 32
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d10 @ cfa - 24 * VG - 32
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d11 @ cfa - 32 * VG - 32
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d12 @ cfa - 40 * VG - 32
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d13 @ cfa - 48 * VG - 32
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d14 @ cfa - 56 * VG - 32
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x60, 0x22 // $d15 @ cfa - 64 * VG - 32
+; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d8 @ cfa - 8 * IncomingVG - 48
+; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d9 @ cfa - 16 * IncomingVG - 48
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d10 @ cfa - 24 * IncomingVG - 48
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d11 @ cfa - 32 * IncomingVG - 48
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d12 @ cfa - 40 * IncomingVG - 48
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d13 @ cfa - 48 * IncomingVG - 48
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d14 @ cfa - 56 * IncomingVG - 48
+; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d15 @ cfa - 64 * IncomingVG - 48
 ; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x01, 0x1e, 0x22 // sp + 32 + 152 * VG
-; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    str z0, [x29, #-19, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    .cfi_offset vg, -16
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z0, [x29, #-19, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl scalable_callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 32 + 144 * VG
 ; CHECK-NEXT:    ptrue pn8.b
 ; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
@@ -395,7 +403,6 @@ define void @vg_unwind_with_sve_args(<vscale x 2 x i64> %x) #0 {
 ; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    addvl sp, sp, #18
-; CHECK-NEXT:    .cfi_def_cfa wsp, 32
 ; CHECK-NEXT:    .cfi_restore z8
 ; CHECK-NEXT:    .cfi_restore z9
 ; CHECK-NEXT:    .cfi_restore z10
@@ -404,10 +411,13 @@ define void @vg_unwind_with_sve_args(<vscale x 2 x i64> %x) #0 {
 ; CHECK-NEXT:    .cfi_restore z13
 ; CHECK-NEXT:    .cfi_restore z14
 ; CHECK-NEXT:    .cfi_restore z15
-; CHECK-NEXT:    ldr x28, [sp, #24] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa wsp, 48
+; CHECK-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w27
 ; CHECK-NEXT:    .cfi_restore w28
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    ret
@@ -423,6 +433,7 @@ define void @vg_unwind_with_sve_args(<vscale x 2 x i64> %x) #0 {
 ; FP-CHECK-NEXT:    .cfi_def_cfa w29, 48
 ; FP-CHECK-NEXT:    .cfi_offset w27, -8
 ; FP-CHECK-NEXT:    .cfi_offset w28, -16
+; FP-CHECK-NEXT:    .cfi_offset vg, -32
 ; FP-CHECK-NEXT:    .cfi_offset w30, -40
 ; FP-CHECK-NEXT:    .cfi_offset w29, -48
 ; FP-CHECK-NEXT:    addvl sp, sp, #-18
@@ -448,24 +459,22 @@ define void @vg_unwind_with_sve_args(<vscale x 2 x i64> %x) #0 {
 ; FP-CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
 ; FP-CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; FP-CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d8 @ cfa - 8 * VG - 48
-; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d9 @ cfa - 16 * VG - 48
-; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d10 @ cfa - 24 * VG - 48
-; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d11 @ cfa - 32 * VG - 48
-; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d12 @ cfa - 40 * VG - 48
-; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d13 @ cfa - 48 * VG - 48
-; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d14 @ cfa - 56 * VG - 48
-; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d15 @ cfa - 64 * VG - 48
+; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d8 @ cfa - 8 * IncomingVG - 48
+; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d9 @ cfa - 16 * IncomingVG - 48
+; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d10 @ cfa - 24 * IncomingVG - 48
+; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d11 @ cfa - 32 * IncomingVG - 48
+; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d12 @ cfa - 40 * IncomingVG - 48
+; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d13 @ cfa - 48 * IncomingVG - 48
+; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d14 @ cfa - 56 * IncomingVG - 48
+; FP-CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0c, 0x12, 0x11, 0x60, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d15 @ cfa - 64 * IncomingVG - 48
 ; FP-CHECK-NEXT:    addvl sp, sp, #-1
 ; FP-CHECK-NEXT:    str z0, [x29, #-19, mul vl] // 16-byte Folded Spill
 ; FP-CHECK-NEXT:    //APP
 ; FP-CHECK-NEXT:    //NO_APP
-; FP-CHECK-NEXT:    .cfi_offset vg, -32
 ; FP-CHECK-NEXT:    smstop sm
 ; FP-CHECK-NEXT:    ldr z0, [x29, #-19, mul vl] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    bl scalable_callee
 ; FP-CHECK-NEXT:    smstart sm
-; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    addvl sp, sp, #1
 ; FP-CHECK-NEXT:    ptrue pn8.b
 ; FP-CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
@@ -504,6 +513,7 @@ define void @vg_unwind_with_sve_args(<vscale x 2 x i64> %x) #0 {
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; FP-CHECK-NEXT:    .cfi_restore w27
 ; FP-CHECK-NEXT:    .cfi_restore w28
+; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_restore w30
 ; FP-CHECK-NEXT:    .cfi_restore w29
 ; FP-CHECK-NEXT:    ret
@@ -529,7 +539,9 @@ define void @vg_unwind_multiple_scratch_regs(ptr %out) #1 {
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp x9, x28, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w28, -8
+; CHECK-NEXT:    .cfi_offset vg, -16
 ; CHECK-NEXT:    .cfi_offset w30, -24
 ; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
@@ -552,19 +564,20 @@ define void @vg_unwind_multiple_scratch_regs(ptr %out) #1 {
 ; CHECK-NEXT:    .cfi_def_cfa_register wsp
 ; CHECK-NEXT:    mov x8, sp
 ; CHECK-NEXT:    str x8, [x0]
-; CHECK-NEXT:    .cfi_offset vg, -16
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    add sp, sp, #80, lsl #12 // =327680
 ; CHECK-NEXT:    .cfi_def_cfa_offset 96
 ; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x28, [sp, #88] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w28
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
@@ -590,6 +603,7 @@ define void @vg_unwind_multiple_scratch_regs(ptr %out) #1 {
 ; FP-CHECK-NEXT:    add x29, sp, #64
 ; FP-CHECK-NEXT:    .cfi_def_cfa w29, 32
 ; FP-CHECK-NEXT:    .cfi_offset w28, -8
+; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    .cfi_offset w30, -24
 ; FP-CHECK-NEXT:    .cfi_offset w29, -32
 ; FP-CHECK-NEXT:    .cfi_offset b8, -40
@@ -610,11 +624,9 @@ define void @vg_unwind_multiple_scratch_regs(ptr %out) #1 {
 ; FP-CHECK-NEXT:  // %bb.2: // %entry
 ; FP-CHECK-NEXT:    mov x8, sp
 ; FP-CHECK-NEXT:    str x8, [x0]
-; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    smstop sm
 ; FP-CHECK-NEXT:    bl callee
 ; FP-CHECK-NEXT:    smstart sm
-; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    add sp, sp, #80, lsl #12 // =327680
 ; FP-CHECK-NEXT:    .cfi_def_cfa wsp, 96
 ; FP-CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
@@ -625,6 +637,7 @@ define void @vg_unwind_multiple_scratch_regs(ptr %out) #1 {
 ; FP-CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; FP-CHECK-NEXT:    .cfi_restore w28
+; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_restore w30
 ; FP-CHECK-NEXT:    .cfi_restore w29
 ; FP-CHECK-NEXT:    .cfi_restore b8
@@ -647,24 +660,20 @@ entry:
   ret void
 }
 
-; Locally streaming functions require storing both the streaming and
-; non-streaming values of VG.
-;
 define void @vg_locally_streaming_fn() #3 {
 ; CHECK-LABEL: vg_locally_streaming_fn:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
+; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_offset vg, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
 ; CHECK-NEXT:    .cfi_offset b9, -48
 ; CHECK-NEXT:    .cfi_offset b10, -56
@@ -675,18 +684,18 @@ define void @vg_locally_streaming_fn() #3 {
 ; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    bl streaming_callee
-; CHECK-NEXT:    .cfi_offset vg, -24
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
@@ -701,18 +710,15 @@ define void @vg_locally_streaming_fn() #3 {
 ; FP-CHECK:       // %bb.0:
 ; FP-CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 96
-; FP-CHECK-NEXT:    rdsvl x9, #1
+; FP-CHECK-NEXT:    cntd x9
 ; FP-CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
-; FP-CHECK-NEXT:    lsr x9, x9, #3
 ; FP-CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; FP-CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; FP-CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
-; FP-CHECK-NEXT:    cntd x9
 ; FP-CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
-; FP-CHECK-NEXT:    str x9, [sp, #88] // 8-byte Folded Spill
+; FP-CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
 ; FP-CHECK-NEXT:    add x29, sp, #64
 ; FP-CHECK-NEXT:    .cfi_def_cfa w29, 32
-; FP-CHECK-NEXT:    .cfi_offset vg, -8
+; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    .cfi_offset w30, -24
 ; FP-CHECK-NEXT:    .cfi_offset w29, -32
 ; FP-CHECK-NEXT:    .cfi_offset b8, -40
@@ -725,9 +731,7 @@ define void @vg_locally_streaming_fn() #3 {
 ; FP-CHECK-NEXT:    .cfi_offset b15, -96
 ; FP-CHECK-NEXT:    bl callee
 ; FP-CHECK-NEXT:    smstart sm
-; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    bl streaming_callee
-; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    smstop sm
 ; FP-CHECK-NEXT:    bl callee
 ; FP-CHECK-NEXT:    .cfi_def_cfa wsp, 96
@@ -737,6 +741,7 @@ define void @vg_locally_streaming_fn() #3 {
 ; FP-CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 0
+; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_restore w30
 ; FP-CHECK-NEXT:    .cfi_restore w29
 ; FP-CHECK-NEXT:    .cfi_restore b8
@@ -767,10 +772,12 @@ define void @streaming_compatible_to_streaming() #4 {
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w19, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
 ; CHECK-NEXT:    .cfi_offset b9, -48
 ; CHECK-NEXT:    .cfi_offset b10, -56
@@ -780,8 +787,7 @@ define void @streaming_compatible_to_streaming() #4 {
 ; CHECK-NEXT:    .cfi_offset b14, -88
 ; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
-; CHECK-NEXT:    .cfi_offset vg, -24
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbnz w19, #0, .LBB6_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstart sm
@@ -791,16 +797,17 @@ define void @streaming_compatible_to_streaming() #4 {
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB6_4:
-; CHECK-NEXT:    .cfi_restore vg
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
@@ -824,6 +831,7 @@ define void @streaming_compatible_to_streaming() #4 {
 ; FP-CHECK-NEXT:    add x29, sp, #64
 ; FP-CHECK-NEXT:    .cfi_def_cfa w29, 32
 ; FP-CHECK-NEXT:    .cfi_offset w19, -8
+; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    .cfi_offset w30, -24
 ; FP-CHECK-NEXT:    .cfi_offset w29, -32
 ; FP-CHECK-NEXT:    .cfi_offset b8, -40
@@ -835,8 +843,7 @@ define void @streaming_compatible_to_streaming() #4 {
 ; FP-CHECK-NEXT:    .cfi_offset b14, -88
 ; FP-CHECK-NEXT:    .cfi_offset b15, -96
 ; FP-CHECK-NEXT:    bl __arm_sme_state
-; FP-CHECK-NEXT:    and x19, x0, #0x1
-; FP-CHECK-NEXT:    .cfi_offset vg, -16
+; FP-CHECK-NEXT:    mov x19, x0
 ; FP-CHECK-NEXT:    tbnz w19, #0, .LBB6_2
 ; FP-CHECK-NEXT:  // %bb.1:
 ; FP-CHECK-NEXT:    smstart sm
@@ -846,7 +853,6 @@ define void @streaming_compatible_to_streaming() #4 {
 ; FP-CHECK-NEXT:  // %bb.3:
 ; FP-CHECK-NEXT:    smstop sm
 ; FP-CHECK-NEXT:  .LBB6_4:
-; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_def_cfa wsp, 96
 ; FP-CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
@@ -856,6 +862,7 @@ define void @streaming_compatible_to_streaming() #4 {
 ; FP-CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; FP-CHECK-NEXT:    .cfi_restore w19
+; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_restore w30
 ; FP-CHECK-NEXT:    .cfi_restore w29
 ; FP-CHECK-NEXT:    .cfi_restore b8
@@ -884,10 +891,12 @@ define void @streaming_compatible_to_non_streaming() #4 {
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w19, -16
-; CHECK-NEXT:    .cfi_offset w30, -32
+; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x9, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
 ; CHECK-NEXT:    .cfi_offset b8, -40
 ; CHECK-NEXT:    .cfi_offset b9, -48
 ; CHECK-NEXT:    .cfi_offset b10, -56
@@ -897,8 +906,7 @@ define void @streaming_compatible_to_non_streaming() #4 {
 ; CHECK-NEXT:    .cfi_offset b14, -88
 ; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
-; CHECK-NEXT:    .cfi_offset vg, -24
+; CHECK-NEXT:    mov x19, x0
 ; CHECK-NEXT:    tbz w19, #0, .LBB7_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
@@ -908,16 +916,17 @@ define void @streaming_compatible_to_non_streaming() #4 {
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB7_4:
-; CHECK-NEXT:    .cfi_restore vg
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
@@ -941,6 +950,7 @@ define void @streaming_compatible_to_non_streaming() #4 {
 ; FP-CHECK-NEXT:    add x29, sp, #64
 ; FP-CHECK-NEXT:    .cfi_def_cfa w29, 32
 ; FP-CHECK-NEXT:    .cfi_offset w19, -8
+; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    .cfi_offset w30, -24
 ; FP-CHECK-NEXT:    .cfi_offset w29, -32
 ; FP-CHECK-NEXT:    .cfi_offset b8, -40
@@ -952,8 +962,7 @@ define void @streaming_compatible_to_non_streaming() #4 {
 ; FP-CHECK-NEXT:    .cfi_offset b14, -88
 ; FP-CHECK-NEXT:    .cfi_offset b15, -96
 ; FP-CHECK-NEXT:    bl __arm_sme_state
-; FP-CHECK-NEXT:    and x19, x0, #0x1
-; FP-CHECK-NEXT:    .cfi_offset vg, -16
+; FP-CHECK-NEXT:    mov x19, x0
 ; FP-CHECK-NEXT:    tbz w19, #0, .LBB7_2
 ; FP-CHECK-NEXT:  // %bb.1:
 ; FP-CHECK-NEXT:    smstop sm
@@ -963,7 +972,6 @@ define void @streaming_compatible_to_non_streaming() #4 {
 ; FP-CHECK-NEXT:  // %bb.3:
 ; FP-CHECK-NEXT:    smstart sm
 ; FP-CHECK-NEXT:  .LBB7_4:
-; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_def_cfa wsp, 96
 ; FP-CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
@@ -973,6 +981,7 @@ define void @streaming_compatible_to_non_streaming() #4 {
 ; FP-CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; FP-CHECK-NEXT:    .cfi_restore w19
+; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_restore w30
 ; FP-CHECK-NEXT:    .cfi_restore w29
 ; FP-CHECK-NEXT:    .cfi_restore b8
@@ -1013,6 +1022,7 @@ define void @streaming_compatible_no_sve(i32 noundef %x) #4 {
 ; NO-SVE-CHECK-NEXT:    add x29, sp, #64
 ; NO-SVE-CHECK-NEXT:    .cfi_def_cfa w29, 32
 ; NO-SVE-CHECK-NEXT:    .cfi_offset w19, -8
+; NO-SVE-CHECK-NEXT:    .cfi_offset vg, -16
 ; NO-SVE-CHECK-NEXT:    .cfi_offset w30, -24
 ; NO-SVE-CHECK-NEXT:    .cfi_offset w29, -32
 ; NO-SVE-CHECK-NEXT:    .cfi_offset b8, -40
@@ -1025,8 +1035,7 @@ define void @streaming_compatible_no_sve(i32 noundef %x) #4 {
 ; NO-SVE-CHECK-NEXT:    .cfi_offset b15, -96
 ; NO-SVE-CHECK-NEXT:    mov w8, w0
 ; NO-SVE-CHECK-NEXT:    bl __arm_sme_state
-; NO-SVE-CHECK-NEXT:    and x19, x0, #0x1
-; NO-SVE-CHECK-NEXT:    .cfi_offset vg, -16
+; NO-SVE-CHECK-NEXT:    mov x19, x0
 ; NO-SVE-CHECK-NEXT:    tbnz w19, #0, .LBB8_2
 ; NO-SVE-CHECK-NEXT:  // %bb.1:
 ; NO-SVE-CHECK-NEXT:    smstart sm
@@ -1037,7 +1046,6 @@ define void @streaming_compatible_no_sve(i32 noundef %x) #4 {
 ; NO-SVE-CHECK-NEXT:  // %bb.3:
 ; NO-SVE-CHECK-NEXT:    smstop sm
 ; NO-SVE-CHECK-NEXT:  .LBB8_4:
-; NO-SVE-CHECK-NEXT:    .cfi_restore vg
 ; NO-SVE-CHECK-NEXT:    .cfi_def_cfa wsp, 96
 ; NO-SVE-CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; NO-SVE-CHECK-NEXT:    ldr x19, [sp, #88] // 8-byte Folded Reload
@@ -1047,6 +1055,7 @@ define void @streaming_compatible_no_sve(i32 noundef %x) #4 {
 ; NO-SVE-CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; NO-SVE-CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; NO-SVE-CHECK-NEXT:    .cfi_restore w19
+; NO-SVE-CHECK-NEXT:    .cfi_restore vg
 ; NO-SVE-CHECK-NEXT:    .cfi_restore w30
 ; NO-SVE-CHECK-NEXT:    .cfi_restore w29
 ; NO-SVE-CHECK-NEXT:    .cfi_restore b8
@@ -1072,30 +1081,29 @@ define void @streaming_compatible_no_sve(i32 noundef %x) #4 {
 ; user-code as if it is part of the frame-setup when doing so.
 define void @test_rdsvl_right_after_prologue(i64 %x0) nounwind {
 ; NO-SVE-CHECK-LABEL: test_rdsvl_right_after_prologue:
-; NO-SVE-CHECK:     // %bb.0:
-; NO-SVE-CHECK-NEXT: stp     d15, d14, [sp, #-96]!           // 16-byte Folded Spill
-; NO-SVE-CHECK-NEXT: stp     d13, d12, [sp, #16]             // 16-byte Folded Spill
-; NO-SVE-CHECK-NEXT: mov     x9, x0
-; NO-SVE-CHECK-NEXT: stp     d11, d10, [sp, #32]             // 16-byte Folded Spill
-; NO-SVE-CHECK-NEXT: stp     d9, d8, [sp, #48]               // 16-byte Folded Spill
-; NO-SVE-CHECK-NEXT: stp     x29, x30, [sp, #64]             // 16-byte Folded Spill
-; NO-SVE-CHECK-NEXT: bl      __arm_get_current_vg
-; NO-SVE-CHECK-NEXT: str     x0, [sp, #80]                   // 8-byte Folded Spill
-; NO-SVE-CHECK-NEXT: mov     x0, x9
-; NO-SVE-CHECK-NEXT: rdsvl   x8, #1
-; NO-SVE-CHECK-NEXT: add     x29, sp, #64
-; NO-SVE-CHECK-NEXT: lsr     x8, x8, #3
-; NO-SVE-CHECK-NEXT: mov     x1, x0
-; NO-SVE-CHECK-NEXT: smstart sm
-; NO-SVE-CHECK-NEXT: mov     x0, x8
-; NO-SVE-CHECK-NEXT: bl      bar
-; NO-SVE-CHECK-NEXT: smstop  sm
-; NO-SVE-CHECK-NEXT: ldp     x29, x30, [sp, #64]             // 16-byte Folded Reload
-; NO-SVE-CHECK-NEXT: ldp     d9, d8, [sp, #48]               // 16-byte Folded Reload
-; NO-SVE-CHECK-NEXT: ldp     d11, d10, [sp, #32]             // 16-byte Folded Reload
-; NO-SVE-CHECK-NEXT: ldp     d13, d12, [sp, #16]             // 16-byte Folded Reload
-; NO-SVE-CHECK-NEXT: ldp     d15, d14, [sp], #96             // 16-byte Folded Reload
-; NO-SVE-CHECK-NEXT: ret
+; NO-SVE-CHECK:       // %bb.0:
+; NO-SVE-CHECK-NEXT:    sub sp, sp, #96
+; NO-SVE-CHECK-NEXT:    rdsvl x8, #1
+; NO-SVE-CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT:    mov x1, x0
+; NO-SVE-CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT:    lsr x8, x8, #3
+; NO-SVE-CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT:    stp x29, x30, [sp, #80] // 16-byte Folded Spill
+; NO-SVE-CHECK-NEXT:    add x29, sp, #80
+; NO-SVE-CHECK-NEXT:    smstart sm
+; NO-SVE-CHECK-NEXT:    mov x0, x8
+; NO-SVE-CHECK-NEXT:    bl bar
+; NO-SVE-CHECK-NEXT:    smstop sm
+; NO-SVE-CHECK-NEXT:    ldp x29, x30, [sp, #80] // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; NO-SVE-CHECK-NEXT:    add sp, sp, #96
+; NO-SVE-CHECK-NEXT:    ret
+;
   %some_alloc = alloca i64, align 8
   %rdsvl = tail call i64 @llvm.aarch64.sme.cntsd()
   call void @bar(i64 %rdsvl, i64 %x0)
@@ -1110,34 +1118,37 @@ declare void @bar(i64, i64) "aarch64_pstate_sm_enabled"
 define void @vg_unwind_noasync() #5 {
 ; CHECK-LABEL: vg_unwind_noasync:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 96
 ; CHECK-NEXT:    cntd x9
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    .cfi_offset b8, -24
-; CHECK-NEXT:    .cfi_offset b9, -32
-; CHECK-NEXT:    .cfi_offset b10, -40
-; CHECK-NEXT:    .cfi_offset b11, -48
-; CHECK-NEXT:    .cfi_offset b12, -56
-; CHECK-NEXT:    .cfi_offset b13, -64
-; CHECK-NEXT:    .cfi_offset b14, -72
-; CHECK-NEXT:    .cfi_offset b15, -80
-; CHECK-NEXT:    .cfi_offset vg, -8
+; CHECK-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset vg, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
+; CHECK-NEXT:    .cfi_offset b8, -40
+; CHECK-NEXT:    .cfi_offset b9, -48
+; CHECK-NEXT:    .cfi_offset b10, -56
+; CHECK-NEXT:    .cfi_offset b11, -64
+; CHECK-NEXT:    .cfi_offset b12, -72
+; CHECK-NEXT:    .cfi_offset b13, -80
+; CHECK-NEXT:    .cfi_offset b14, -88
+; CHECK-NEXT:    .cfi_offset b15, -96
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl callee
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
+; CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
 ; CHECK-NEXT:    .cfi_restore b9
 ; CHECK-NEXT:    .cfi_restore b10
@@ -1160,6 +1171,7 @@ define void @vg_unwind_noasync() #5 {
 ; FP-CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
 ; FP-CHECK-NEXT:    add x29, sp, #64
 ; FP-CHECK-NEXT:    .cfi_def_cfa w29, 32
+; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    .cfi_offset w30, -24
 ; FP-CHECK-NEXT:    .cfi_offset w29, -32
 ; FP-CHECK-NEXT:    .cfi_offset b8, -40
@@ -1170,11 +1182,9 @@ define void @vg_unwind_noasync() #5 {
 ; FP-CHECK-NEXT:    .cfi_offset b13, -80
 ; FP-CHECK-NEXT:    .cfi_offset b14, -88
 ; FP-CHECK-NEXT:    .cfi_offset b15, -96
-; FP-CHECK-NEXT:    .cfi_offset vg, -16
 ; FP-CHECK-NEXT:    smstop sm
 ; FP-CHECK-NEXT:    bl callee
 ; FP-CHECK-NEXT:    smstart sm
-; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_def_cfa wsp, 96
 ; FP-CHECK-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
@@ -1182,6 +1192,7 @@ define void @vg_unwind_noasync() #5 {
 ; FP-CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
 ; FP-CHECK-NEXT:    .cfi_def_cfa_offset 0
+; FP-CHECK-NEXT:    .cfi_restore vg
 ; FP-CHECK-NEXT:    .cfi_restore w30
 ; FP-CHECK-NEXT:    .cfi_restore w29
 ; FP-CHECK-NEXT:    .cfi_restore b8
@@ -1193,6 +1204,7 @@ define void @vg_unwind_noasync() #5 {
 ; FP-CHECK-NEXT:    .cfi_restore b14
 ; FP-CHECK-NEXT:    .cfi_restore b15
 ; FP-CHECK-NEXT:    ret
+;
 ; OUTLINER-CHECK-LABEL: vg_unwind_noasync:
 ; OUTLINER-CHECK-NOT: OUTLINED_FUNCTION_
 ;
diff --git a/llvm/test/CodeGen/AArch64/sme-za-control-flow.ll b/llvm/test/CodeGen/AArch64/sme-za-control-flow.ll
new file mode 100644
index 0000000000000..d3d7e953bedfa
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-za-control-flow.ll
@@ -0,0 +1,1131 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -aarch64-new-sme-abi < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-NEWLOWERING
+
+declare void @private_za_call()
+declare void @shared_za_call() "aarch64_inout_za"
+
+define void @private_za_loop(i32 %n) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: private_za_loop:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x21, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    cmp w0, #1
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    b.lt .LBB0_5
+; CHECK-NEXT:  // %bb.1: // %loop.preheader
+; CHECK-NEXT:    mov w19, w0
+; CHECK-NEXT:    rdsvl x20, #1
+; CHECK-NEXT:    sub x21, x29, #16
+; CHECK-NEXT:    b .LBB0_3
+; CHECK-NEXT:  .LBB0_2: // %loop
+; CHECK-NEXT:    // in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    subs w19, w19, #1
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    b.eq .LBB0_5
+; CHECK-NEXT:  .LBB0_3: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB0_2
+; CHECK-NEXT:  // %bb.4: // %loop
+; CHECK-NEXT:    // in Loop: Header=BB0_3 Depth=1
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:    b .LBB0_2
+; CHECK-NEXT:  .LBB0_5: // %exit
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x21, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: private_za_loop:
+; CHECK-NEWLOWERING:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cmp w0, #1
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    b.lt .LBB0_3
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %loop.preheader
+; CHECK-NEWLOWERING-NEXT:    mov w19, w0
+; CHECK-NEWLOWERING-NEXT:  .LBB0_2: // %loop
+; CHECK-NEWLOWERING-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    subs w19, w19, #1
+; CHECK-NEWLOWERING-NEXT:    b.ne .LBB0_2
+; CHECK-NEWLOWERING-NEXT:  .LBB0_3: // %exit
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB0_5
+; CHECK-NEWLOWERING-NEXT:  // %bb.4: // %exit
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB0_5: // %exit
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+entry:
+  %cmpgt = icmp sgt i32 %n, 0
+  br i1 %cmpgt, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ %next_iv, %loop ], [ 0, %entry ]
+  tail call void @private_za_call()
+  %next_iv = add nuw nsw i32 %iv, 1
+  %cmpeq = icmp eq i32 %next_iv, %n
+  br i1 %cmpeq, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; FIXME: In the new lowering we could weight edges to avoid doing the lazy save in the loop.
+define void @private_za_loop_active_entry_and_exit(i32 %n) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: private_za_loop_active_entry_and_exit:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x21, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    mov w19, w0
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    cmp w19, #1
+; CHECK-NEXT:    b.lt .LBB1_5
+; CHECK-NEXT:  // %bb.1: // %loop.preheader
+; CHECK-NEXT:    rdsvl x20, #1
+; CHECK-NEXT:    sub x21, x29, #16
+; CHECK-NEXT:    b .LBB1_3
+; CHECK-NEXT:  .LBB1_2: // %loop
+; CHECK-NEXT:    // in Loop: Header=BB1_3 Depth=1
+; CHECK-NEXT:    subs w19, w19, #1
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    b.eq .LBB1_5
+; CHECK-NEXT:  .LBB1_3: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB1_2
+; CHECK-NEXT:  // %bb.4: // %loop
+; CHECK-NEXT:    // in Loop: Header=BB1_3 Depth=1
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:    b .LBB1_2
+; CHECK-NEXT:  .LBB1_5: // %exit
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x21, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
+; CHECK-NEXT:    b shared_za_call
+;
+; CHECK-NEWLOWERING-LABEL: private_za_loop_active_entry_and_exit:
+; CHECK-NEWLOWERING:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    mov w19, w0
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:    cmp w19, #1
+; CHECK-NEWLOWERING-NEXT:    b.lt .LBB1_5
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %loop.preheader
+; CHECK-NEWLOWERING-NEXT:    sub x20, x29, #16
+; CHECK-NEWLOWERING-NEXT:    b .LBB1_3
+; CHECK-NEWLOWERING-NEXT:  .LBB1_2: // %loop
+; CHECK-NEWLOWERING-NEXT:    // in Loop: Header=BB1_3 Depth=1
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    cbz w19, .LBB1_5
+; CHECK-NEWLOWERING-NEXT:  .LBB1_3: // %loop
+; CHECK-NEWLOWERING-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x20
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    sub w19, w19, #1
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB1_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.4: // %loop
+; CHECK-NEWLOWERING-NEXT:    // in Loop: Header=BB1_3 Depth=1
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:    b .LBB1_2
+; CHECK-NEWLOWERING-NEXT:  .LBB1_5: // %exit
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    b shared_za_call
+entry:
+  %cmpgt = icmp sgt i32 %n, 0
+  tail call void @shared_za_call()
+  br i1 %cmpgt, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ %next_iv, %loop ], [ 0, %entry ]
+  tail call void @private_za_call()
+  %next_iv = add nuw nsw i32 %iv, 1
+  %cmpeq = icmp eq i32 %next_iv, %n
+  br i1 %cmpeq, label %exit, label %loop
+
+exit:
+  tail call void @shared_za_call()
+  ret void
+}
+
+define void @shared_za_loop(i32 %n) "aarch64_inout_za" nounwind {
+; CHECK-COMMON-LABEL: shared_za_loop:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    cmp w0, #1
+; CHECK-COMMON-NEXT:    b.lt .LBB2_4
+; CHECK-COMMON-NEXT:  // %bb.1: // %loop.preheader
+; CHECK-COMMON-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    mov w19, w0
+; CHECK-COMMON-NEXT:  .LBB2_2: // %loop
+; CHECK-COMMON-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-COMMON-NEXT:    bl shared_za_call
+; CHECK-COMMON-NEXT:    subs w19, w19, #1
+; CHECK-COMMON-NEXT:    b.ne .LBB2_2
+; CHECK-COMMON-NEXT:  // %bb.3:
+; CHECK-COMMON-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:  .LBB2_4: // %exit
+; CHECK-COMMON-NEXT:    ret
+entry:
+  %cmpgt = icmp sgt i32 %n, 0
+  br i1 %cmpgt, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ %next_iv, %loop ], [ 0, %entry ]
+  tail call void @shared_za_call()
+  %next_iv = add nuw nsw i32 %iv, 1
+  %cmpeq = icmp eq i32 %next_iv, %n
+  br i1 %cmpeq, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; FIXME: The codegen for this case could be improved (by tuning weights).
+; Here the ZA save has been hoisted out of the conditional, but would be better
+; to sink it.
+define void @cond_private_za_call(i1 %cond) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: cond_private_za_call:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    tbz w0, #0, .LBB3_4
+; CHECK-NEXT:  // %bb.1: // %private_za_call
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    sub x9, x29, #16
+; CHECK-NEXT:    sturh w8, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x9
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB3_3
+; CHECK-NEXT:  // %bb.2: // %private_za_call
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB3_3: // %private_za_call
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:  .LBB3_4: // %exit
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    b shared_za_call
+;
+; CHECK-NEWLOWERING-LABEL: cond_private_za_call:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    tbz w0, #0, .LBB3_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %private_za_call
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:  .LBB3_2: // %exit
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB3_4
+; CHECK-NEWLOWERING-NEXT:  // %bb.3: // %exit
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB3_4: // %exit
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    b shared_za_call
+  br i1 %cond, label %private_za_call, label %exit
+
+private_za_call:
+  tail call void @private_za_call()
+  br label %exit
+
+exit:
+  tail call void @shared_za_call()
+  ret void
+}
+
+define void @mixed_shared_private_za_loop(ptr %cond) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: mixed_shared_private_za_loop:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x21, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x20, #1
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:    msub x8, x20, x20, x8
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    sub x21, x29, #16
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    b .LBB4_2
+; CHECK-NEXT:  .LBB4_1: // %loop
+; CHECK-NEXT:    // in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    ldrb w8, [x19]
+; CHECK-NEXT:    tbz w8, #0, .LBB4_4
+; CHECK-NEXT:  .LBB4_2: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB4_1
+; CHECK-NEXT:  // %bb.3: // %loop
+; CHECK-NEXT:    // in Loop: Header=BB4_2 Depth=1
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:    b .LBB4_1
+; CHECK-NEXT:  .LBB4_4: // %exit
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x21, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: mixed_shared_private_za_loop:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    mov x19, x0
+; CHECK-NEWLOWERING-NEXT:    sub x20, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    b .LBB4_2
+; CHECK-NEWLOWERING-NEXT:  .LBB4_1: // %loop
+; CHECK-NEWLOWERING-NEXT:    // in Loop: Header=BB4_2 Depth=1
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    tbz w8, #0, .LBB4_4
+; CHECK-NEWLOWERING-NEXT:  .LBB4_2: // %loop
+; CHECK-NEWLOWERING-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x20
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    ldrb w8, [x19]
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x9, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x9, .LBB4_1
+; CHECK-NEWLOWERING-NEXT:  // %bb.3: // %loop
+; CHECK-NEWLOWERING-NEXT:    // in Loop: Header=BB4_2 Depth=1
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:    b .LBB4_1
+; CHECK-NEWLOWERING-NEXT:  .LBB4_4: // %exit
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+  br label %loop
+
+loop:
+  call void @shared_za_call()
+  call void @private_za_call()
+  br label %latch
+
+latch:
+  %bool = load volatile i8, ptr %cond, align 1
+  %trunc = trunc i8 %bool to i1
+  br i1 %trunc, label %loop, label %exit
+
+exit:
+  call void @shared_za_call()
+  ret void
+}
+
+
+define void @cond_clobber_followed_by_clobber(i1 %cond) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: cond_clobber_followed_by_clobber:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    mov w19, w0
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    tbz w19, #0, .LBB5_4
+; CHECK-NEXT:  // %bb.1: // %cond_clobber
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    sub x9, x29, #16
+; CHECK-NEXT:    sturh w8, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x9
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB5_3
+; CHECK-NEXT:  // %bb.2: // %cond_clobber
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB5_3: // %cond_clobber
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:  .LBB5_4: // %exit
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    sub x9, x29, #16
+; CHECK-NEXT:    sturh w8, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x9
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB5_6
+; CHECK-NEXT:  // %bb.5: // %exit
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB5_6: // %exit
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    b shared_za_call
+;
+; CHECK-NEWLOWERING-LABEL: cond_clobber_followed_by_clobber:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    mov w19, w0
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    tbz w19, #0, .LBB5_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %cond_clobber
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:  .LBB5_2: // %exit
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB5_4
+; CHECK-NEWLOWERING-NEXT:  // %bb.3: // %exit
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB5_4: // %exit
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    b shared_za_call
+  tail call void @shared_za_call()
+  br i1 %cond, label %cond_clobber, label %exit
+
+cond_clobber:
+  tail call void @private_za_call()
+  br label %exit
+
+exit:
+  tail call void @private_za_call()
+  tail call void @shared_za_call()
+  ret void
+}
+
+define void @conditionally_use_za(i1 %cond) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: conditionally_use_za:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    tbz w0, #0, .LBB6_4
+; CHECK-NEXT:  // %bb.1: // %use_za
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    sub x9, x29, #16
+; CHECK-NEXT:    sturh w8, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x9
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB6_3
+; CHECK-NEXT:  // %bb.2: // %use_za
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB6_3: // %use_za
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:  .LBB6_4: // %exit
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: conditionally_use_za:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    tbz w0, #0, .LBB6_4
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %use_za
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB6_3
+; CHECK-NEWLOWERING-NEXT:  // %bb.2: // %use_za
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB6_3: // %use_za
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:  .LBB6_4: // %exit
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+  br i1 %cond, label %use_za, label %exit
+
+use_za:
+  tail call void @shared_za_call()
+  tail call void @private_za_call()
+  br label %exit
+
+exit:
+  ret void
+}
+
+
+define void @diamond_mixed_za_merge_shared(i1 %cond) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: diamond_mixed_za_merge_shared:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    tbz w0, #0, .LBB7_2
+; CHECK-NEXT:  // %bb.1: // %then
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    b .LBB7_5
+; CHECK-NEXT:  .LBB7_2: // %else
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    sub x9, x29, #16
+; CHECK-NEXT:    sturh w8, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x9
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB7_4
+; CHECK-NEXT:  // %bb.3: // %else
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB7_4: // %else
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:  .LBB7_5: // %merge_shared
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: diamond_mixed_za_merge_shared:
+; CHECK-NEWLOWERING:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    tbz w0, #0, .LBB7_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %then
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:    b .LBB7_5
+; CHECK-NEWLOWERING-NEXT:  .LBB7_2: // %else
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB7_4
+; CHECK-NEWLOWERING-NEXT:  // %bb.3: // %else
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB7_4: // %else
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:  .LBB7_5: // %merge_shared
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  call void @shared_za_call()
+  br label %merge_shared
+
+else:
+  call void @private_za_call()
+  br label %merge_shared
+
+merge_shared:
+  call void @shared_za_call()
+  ret void
+}
+
+
+define void @diamond_mixed_za_merge_private(i1 %cond) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: diamond_mixed_za_merge_private:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    tbz w0, #0, .LBB8_2
+; CHECK-NEXT:  // %bb.1: // %then
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    b .LBB8_5
+; CHECK-NEXT:  .LBB8_2: // %else
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    sub x9, x29, #16
+; CHECK-NEXT:    sturh w8, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x9
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB8_4
+; CHECK-NEXT:  // %bb.3: // %else
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB8_4: // %else
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:  .LBB8_5: // %merge_private_za
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    sub x9, x29, #16
+; CHECK-NEXT:    sturh w8, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x9
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB8_7
+; CHECK-NEXT:  // %bb.6: // %merge_private_za
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB8_7: // %merge_private_za
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: diamond_mixed_za_merge_private:
+; CHECK-NEWLOWERING:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    tbz w0, #0, .LBB8_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %then
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    b .LBB8_3
+; CHECK-NEWLOWERING-NEXT:  .LBB8_2: // %else
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:  .LBB8_3: // %merge_private_za
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB8_5
+; CHECK-NEWLOWERING-NEXT:  // %bb.4: // %merge_private_za
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB8_5: // %merge_private_za
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  call void @shared_za_call()
+  br label %merge_private_za
+
+else:
+  call void @private_za_call()
+  br label %merge_private_za
+
+merge_private_za:
+  call void @private_za_call()
+  ret void
+}
+
+define void @critical_edge_mixed_za(i1 %c1, i1 %c2) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: critical_edge_mixed_za:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    mov w19, w1
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    tbz w0, #0, .LBB9_5
+; CHECK-NEXT:  // %bb.1: // %shared_path
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    tbz w19, #0, .LBB9_8
+; CHECK-NEXT:  .LBB9_2: // %exit_private
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    sub x9, x29, #16
+; CHECK-NEXT:    sturh w8, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x9
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB9_4
+; CHECK-NEXT:  // %bb.3: // %exit_private
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB9_4: // %exit_private
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    b .LBB9_9
+; CHECK-NEXT:  .LBB9_5: // %private_path
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    sub x9, x29, #16
+; CHECK-NEXT:    sturh w8, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x9
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB9_7
+; CHECK-NEXT:  // %bb.6: // %private_path
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB9_7: // %private_path
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    tbnz w19, #0, .LBB9_2
+; CHECK-NEXT:  .LBB9_8: // %exit_shared
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:  .LBB9_9: // %common.ret
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: critical_edge_mixed_za:
+; CHECK-NEWLOWERING:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    mov w19, w1
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    tbz w0, #0, .LBB9_5
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %shared_path
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:    tbz w19, #0, .LBB9_8
+; CHECK-NEWLOWERING-NEXT:  .LBB9_2: // %exit_private
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB9_4
+; CHECK-NEWLOWERING-NEXT:  // %bb.3: // %exit_private
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB9_4: // %exit_private
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    b .LBB9_9
+; CHECK-NEWLOWERING-NEXT:  .LBB9_5: // %private_path
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB9_7
+; CHECK-NEWLOWERING-NEXT:  // %bb.6: // %private_path
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB9_7: // %private_path
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    tbnz w19, #0, .LBB9_2
+; CHECK-NEWLOWERING-NEXT:  .LBB9_8: // %exit_shared
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:  .LBB9_9: // %common.ret
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+entry:
+  br i1 %c1, label %shared_path, label %private_path
+
+shared_path:
+  call void @shared_za_call()
+  br label %merge
+
+private_path:
+  call void @private_za_call()
+  br label %merge
+
+merge:
+  br i1 %c2, label %exit_private, label %exit_shared
+
+exit_private:
+  call void @private_za_call()
+  ret void
+
+exit_shared:
+  call void @shared_za_call()
+  ret void
+}
+
+define void @nested_cond_in_loop(i32 %n, i1 %cond) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: nested_cond_in_loop:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x29, x30, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x23, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    cmp w0, #1
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    b.lt .LBB10_8
+; CHECK-NEXT:  // %bb.1: // %loop.preheader
+; CHECK-NEXT:    mov w19, w1
+; CHECK-NEXT:    mov w20, w0
+; CHECK-NEXT:    mov w21, wzr
+; CHECK-NEXT:    rdsvl x22, #1
+; CHECK-NEXT:    sub x23, x29, #16
+; CHECK-NEXT:    b .LBB10_4
+; CHECK-NEXT:  .LBB10_2: // %use_shared
+; CHECK-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:  .LBB10_3: // %latch
+; CHECK-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEXT:    add w21, w21, #1
+; CHECK-NEXT:    cmp w21, w20
+; CHECK-NEXT:    b.ge .LBB10_8
+; CHECK-NEXT:  .LBB10_4: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    tbnz w19, #0, .LBB10_2
+; CHECK-NEXT:  // %bb.5: // %use_private
+; CHECK-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEXT:    sturh w22, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x23
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB10_7
+; CHECK-NEXT:  // %bb.6: // %use_private
+; CHECK-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB10_7: // %use_private
+; CHECK-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    b .LBB10_3
+; CHECK-NEXT:  .LBB10_8: // %exit
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x23, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x22, x21, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #64 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: nested_cond_in_loop:
+; CHECK-NEWLOWERING:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp x22, x21, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    cmp w0, #1
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    b.lt .LBB10_8
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %loop.preheader
+; CHECK-NEWLOWERING-NEXT:    mov w19, w1
+; CHECK-NEWLOWERING-NEXT:    mov w20, w0
+; CHECK-NEWLOWERING-NEXT:    mov w21, wzr
+; CHECK-NEWLOWERING-NEXT:    sub x22, x29, #16
+; CHECK-NEWLOWERING-NEXT:    b .LBB10_4
+; CHECK-NEWLOWERING-NEXT:  .LBB10_2: // %use_shared
+; CHECK-NEWLOWERING-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:  .LBB10_3: // %latch
+; CHECK-NEWLOWERING-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEWLOWERING-NEXT:    add w21, w21, #1
+; CHECK-NEWLOWERING-NEXT:    cmp w21, w20
+; CHECK-NEWLOWERING-NEXT:    b.ge .LBB10_8
+; CHECK-NEWLOWERING-NEXT:  .LBB10_4: // %loop
+; CHECK-NEWLOWERING-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-NEXT:    tbnz w19, #0, .LBB10_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.5: // %use_private
+; CHECK-NEWLOWERING-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x22
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB10_7
+; CHECK-NEWLOWERING-NEXT:  // %bb.6: // %use_private
+; CHECK-NEWLOWERING-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB10_7: // %use_private
+; CHECK-NEWLOWERING-NEXT:    // in Loop: Header=BB10_4 Depth=1
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    b .LBB10_3
+; CHECK-NEWLOWERING-NEXT:  .LBB10_8: // %exit
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+entry:
+  %cmp = icmp sgt i32 %n, 0
+  br i1 %cmp, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %inc, %latch ]
+  br i1 %cond, label %use_shared, label %use_private
+
+use_shared:
+  call void @shared_za_call()
+  br label %latch
+
+use_private:
+  call void @private_za_call()
+  br label %latch
+
+latch:
+  %inc = add i32 %iv, 1
+  %cmp2 = icmp slt i32 %inc, %n
+  br i1 %cmp2, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @loop_with_external_entry(i1 %c1, i1 %c2) "aarch64_inout_za" nounwind {
+; CHECK-LABEL: loop_with_external_entry:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x21, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    mov w19, w1
+; CHECK-NEXT:    msub x8, x8, x8, x9
+; CHECK-NEXT:    mov sp, x8
+; CHECK-NEXT:    stur x8, [x29, #-16]
+; CHECK-NEXT:    sturh wzr, [x29, #-6]
+; CHECK-NEXT:    stur wzr, [x29, #-4]
+; CHECK-NEXT:    tbz w0, #0, .LBB11_2
+; CHECK-NEXT:  // %bb.1: // %init
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:  .LBB11_2: // %loop.preheader
+; CHECK-NEXT:    rdsvl x20, #1
+; CHECK-NEXT:    sub x21, x29, #16
+; CHECK-NEXT:    b .LBB11_4
+; CHECK-NEXT:  .LBB11_3: // %loop
+; CHECK-NEXT:    // in Loop: Header=BB11_4 Depth=1
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    tbz w19, #0, .LBB11_6
+; CHECK-NEXT:  .LBB11_4: // %loop
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    sturh w20, [x29, #-8]
+; CHECK-NEXT:    msr TPIDR2_EL0, x21
+; CHECK-NEXT:    bl private_za_call
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB11_3
+; CHECK-NEXT:  // %bb.5: // %loop
+; CHECK-NEXT:    // in Loop: Header=BB11_4 Depth=1
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:    b .LBB11_3
+; CHECK-NEXT:  .LBB11_6: // %exit
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x21, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: loop_with_external_entry:
+; CHECK-NEWLOWERING:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    mov w19, w1
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    tbz w0, #0, .LBB11_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %init
+; CHECK-NEWLOWERING-NEXT:    bl shared_za_call
+; CHECK-NEWLOWERING-NEXT:  .LBB11_2: // %loop.preheader
+; CHECK-NEWLOWERING-NEXT:    sub x8, x29, #16
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEWLOWERING-NEXT:  .LBB11_3: // %loop
+; CHECK-NEWLOWERING-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEWLOWERING-NEXT:    bl private_za_call
+; CHECK-NEWLOWERING-NEXT:    tbnz w19, #0, .LBB11_3
+; CHECK-NEWLOWERING-NEXT:  // %bb.4: // %exit
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB11_6
+; CHECK-NEWLOWERING-NEXT:  // %bb.5: // %exit
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB11_6: // %exit
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
+entry:
+  br i1 %c1, label %init, label %loop
+
+init:
+  call void @shared_za_call()
+  br label %loop
+
+loop:
+  call void @private_za_call()
+  br i1 %c2, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll b/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll
new file mode 100644
index 0000000000000..bb88142efa592
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-za-exceptions.ll
@@ -0,0 +1,286 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -aarch64-new-sme-abi -verify-machineinstrs < %s | FileCheck %s
+
+; A simple EH test case that corresponds to the following C++ source:
+;
+; struct ZAResource {
+;     ~ZAResource() __arm_inout("za") {
+;         shared_za_call(); // simulate cleanup in destructor
+;     }
+; };
+;
+; void za_with_raii(bool fail) __arm_inout("za") {
+;     ZAResource r;
+;     if (fail)
+;         throw "Unwinding needs ZA state reload";
+; }
+;
+; Here if an exception is thrown we must call the ~ZAResource destructor while
+; unwinding the stack. That requires us to restore ZA state before the
+; shared_za_call in the cleanup block.
+
+@.str = private unnamed_addr constant [32 x i8] c"Unwinding needs ZA state reload\00", align 1
+@typeinfo_for_char_const_ptr = external constant ptr
+
+define void @za_with_raii(i1 %fail) "aarch64_inout_za" personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: za_with_raii:
+; CHECK:       .Lfunc_begin0:
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:    .cfi_personality 156, DW.ref.__gxx_personality_v0
+; CHECK-NEXT:    .cfi_lsda 28, .Lexception0
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa w29, 32
+; CHECK-NEXT:    .cfi_offset w19, -16
+; CHECK-NEXT:    .cfi_offset w30, -24
+; CHECK-NEXT:    .cfi_offset w29, -32
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEXT:    tbnz w0, #0, .LBB0_2
+; CHECK-NEXT:  // %bb.1: // %return_normally
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEXT:    b shared_za_call
+; CHECK-NEXT:  .LBB0_2: // %throw_exception
+; CHECK-NEXT:    sub x8, x29, #16
+; CHECK-NEXT:    mov w0, #8 // =0x8
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    bl __cxa_allocate_exception
+; CHECK-NEXT:    adrp x8, .L.str
+; CHECK-NEXT:    add x8, x8, :lo12:.L.str
+; CHECK-NEXT:    str x8, [x0]
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    adrp x1, :got:typeinfo_for_char_const_ptr
+; CHECK-NEXT:    mov x2, xzr
+; CHECK-NEXT:    ldr x1, [x1, :got_lo12:typeinfo_for_char_const_ptr]
+; CHECK-NEXT:    bl __cxa_throw
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB0_4
+; CHECK-NEXT:  // %bb.3: // %throw_exception
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB0_4: // %throw_exception
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:  // %bb.5: // %throw_fail
+; CHECK-NEXT:  .LBB0_6: // %unwind_dtors
+; CHECK-NEXT:  .Ltmp2:
+; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB0_8
+; CHECK-NEXT:  // %bb.7: // %unwind_dtors
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB0_8: // %unwind_dtors
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    sub x8, x29, #16
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    bl _Unwind_Resume
+  br i1 %fail, label %throw_exception, label %return_normally
+
+throw_exception:
+  %exception_ptr = tail call ptr @__cxa_allocate_exception(i64 8) #3
+  store ptr @.str, ptr %exception_ptr, align 16
+  invoke void @__cxa_throw(ptr nonnull %exception_ptr, ptr nonnull @typeinfo_for_char_const_ptr, ptr null)
+          to label %throw_fail unwind label %unwind_dtors
+
+unwind_dtors:
+  %5 = landingpad { ptr, i32 }
+          cleanup
+  tail call void @shared_za_call()
+  resume { ptr, i32 } %5
+
+return_normally:
+  tail call void @shared_za_call()
+  ret void
+
+throw_fail:
+  unreachable
+}
+
+
+; Another simple exception handling example. Here we need to restore ZA in two
+; places. After the may_throw() call to handle the case it does not throw, and
+; within the catch block for the shared_za_call(). We also need to setup the
+; lazy save around C++ exception ABI routines (to handle the _very_ unlikely
+; case they use ZA state).
+;
+; void za_try_catch() __arm_inout("za") {
+;     try {
+;         may_throw();
+;     } catch (...) {
+;        shared_za_call();
+;     }
+;     shared_za_call();
+; }
+define dso_local void @try_catch() "aarch64_inout_za" personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: try_catch:
+; CHECK:       .Lfunc_begin1:
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:    .cfi_personality 156, DW.ref.__gxx_personality_v0
+; CHECK-NEXT:    .cfi_lsda 28, .Lexception1
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEXT:  .Ltmp3:
+; CHECK-NEXT:    sub x8, x29, #16
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    bl may_throw
+; CHECK-NEXT:  .Ltmp4:
+; CHECK-NEXT:  .LBB1_1: // %after_catch
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB1_3
+; CHECK-NEXT:  // %bb.2: // %after_catch
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB1_3: // %after_catch
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    b shared_za_call
+; CHECK-NEXT:  .LBB1_4: // %catch
+; CHECK-NEXT:  .Ltmp5:
+; CHECK-NEXT:    bl __cxa_begin_catch
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB1_6
+; CHECK-NEXT:  // %bb.5: // %catch
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB1_6: // %catch
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:    sub x8, x29, #16
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    bl __cxa_end_catch
+; CHECK-NEXT:    b .LBB1_1
+  invoke void @may_throw()
+          to label %after_catch unwind label %catch
+
+catch:                                                ; preds = %0
+  %eh_info = landingpad { ptr, i32 }
+          catch ptr null
+  %exception_ptr = extractvalue { ptr, i32 } %eh_info, 0
+  tail call ptr @__cxa_begin_catch(ptr %exception_ptr)
+  tail call void @shared_za_call()
+  tail call void @__cxa_end_catch()
+  br label %after_catch
+
+after_catch:
+  tail call void @shared_za_call()
+  ret void
+}
+
+; This example corresponds to:
+;
+; __arm_new("za") void try_catch_shared_za_callee()
+; {
+;    try {
+;        shared_za_call();
+;    } catch(...) {
+;        noexcept_shared_za_call();
+;    }
+; }
+;
+; In this example we don't setup a lazy save before shared_za_call(), however,
+; we still enter the catch block in a ZA off state. This leads to us emitting a
+; restore of a uninitialized save buffer in the catch block. This is not ideal
+; but is valid in the SME ABI. Ideally, we would omit the save buffer and
+; restore and simply set ZA to "on" in the catch block.
+
+define void @try_catch_shared_za_callee() "aarch64_new_za" personality ptr @__gxx_personality_v0 {
+; CHECK-LABEL: try_catch_shared_za_callee:
+; CHECK:       .Lfunc_begin2:
+; CHECK-NEXT:    .cfi_startproc
+; CHECK-NEXT:    .cfi_personality 156, DW.ref.__gxx_personality_v0
+; CHECK-NEXT:    .cfi_lsda 28, .Lexception2
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    mov x29, sp
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    rdsvl x8, #1
+; CHECK-NEXT:    mov x9, sp
+; CHECK-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEXT:    mov sp, x9
+; CHECK-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    cbz x8, .LBB2_2
+; CHECK-NEXT:  // %bb.1:
+; CHECK-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    zero {za}
+; CHECK-NEXT:  .LBB2_2:
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:  .Ltmp6:
+; CHECK-NEXT:    bl shared_za_call
+; CHECK-NEXT:  .Ltmp7:
+; CHECK-NEXT:  .LBB2_3: // %exit
+; CHECK-NEXT:    smstop za
+; CHECK-NEXT:    mov sp, x29
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB2_4: // %catch
+; CHECK-NEXT:  .Ltmp8:
+; CHECK-NEXT:    bl __cxa_begin_catch
+; CHECK-NEXT:    smstart za
+; CHECK-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEXT:    sub x0, x29, #16
+; CHECK-NEXT:    cbnz x8, .LBB2_6
+; CHECK-NEXT:  // %bb.5: // %catch
+; CHECK-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEXT:  .LBB2_6: // %catch
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    bl noexcept_shared_za_call
+; CHECK-NEXT:    sub x8, x29, #16
+; CHECK-NEXT:    msr TPIDR2_EL0, x8
+; CHECK-NEXT:    bl __cxa_end_catch
+; CHECK-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEXT:    b .LBB2_3
+  invoke void @shared_za_call() #4
+          to label %exit unwind label %catch
+catch:
+  %eh_info = landingpad { ptr, i32 }
+          catch ptr null
+  %exception_ptr = extractvalue { ptr, i32 } %eh_info, 0
+  tail call ptr @__cxa_begin_catch(ptr %exception_ptr)
+  tail call void @noexcept_shared_za_call()
+  tail call void @__cxa_end_catch()
+  br label %exit
+
+exit:
+  ret void
+}
+
+declare ptr @__cxa_allocate_exception(i64)
+declare void @__cxa_throw(ptr, ptr, ptr)
+declare ptr @__cxa_begin_catch(ptr)
+declare void @__cxa_end_catch()
+declare i32 @__gxx_personality_v0(...)
+
+declare void @may_throw()
+declare void @shared_za_call() "aarch64_inout_za"
+declare void @noexcept_shared_za_call() "aarch64_inout_za"
diff --git a/llvm/test/CodeGen/AArch64/sme-za-lazy-save-buffer.ll b/llvm/test/CodeGen/AArch64/sme-za-lazy-save-buffer.ll
index ad3f7f5514d0e..a9ad6f695cf8f 100644
--- a/llvm/test/CodeGen/AArch64/sme-za-lazy-save-buffer.ll
+++ b/llvm/test/CodeGen/AArch64/sme-za-lazy-save-buffer.ll
@@ -1,11 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -aarch64-new-sme-abi < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-NEWLOWERING
 
 define i32 @no_tpidr2_save_required() "aarch64_inout_za" {
-; CHECK-LABEL: no_tpidr2_save_required:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w0, #42 // =0x2a
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: no_tpidr2_save_required:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    mov w0, #42 // =0x2a
+; CHECK-COMMON-NEXT:    ret
 entry:
   ret i32 42
 }
@@ -50,6 +51,42 @@ define float @multi_bb_stpidr2_save_required(i32 %a, float %b, float %c) "aarch6
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: multi_bb_stpidr2_save_required:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w30, -8
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w29, -16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    cbz w0, .LBB1_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %use_b
+; CHECK-NEWLOWERING-NEXT:    fmov s1, #4.00000000
+; CHECK-NEWLOWERING-NEXT:    fadd s0, s0, s1
+; CHECK-NEWLOWERING-NEXT:    b .LBB1_3
+; CHECK-NEWLOWERING-NEXT:  .LBB1_2: // %use_c
+; CHECK-NEWLOWERING-NEXT:    fmov s0, s1
+; CHECK-NEWLOWERING-NEXT:    bl cosf
+; CHECK-NEWLOWERING-NEXT:  .LBB1_3: // %exit
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB1_5
+; CHECK-NEWLOWERING-NEXT:  // %bb.4: // %exit
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB1_5: // %exit
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   %cmp = icmp ne i32 %a, 0
   br i1 %cmp, label %use_b, label %use_c
 
@@ -66,6 +103,7 @@ exit:
   ret float %ret
 }
 
+; FIXME: This is missing stack probes with -aarch64-new-sme-abi.
 define float @multi_bb_stpidr2_save_required_stackprobe(i32 %a, float %b, float %c) "aarch64_inout_za" "probe-stack"="inline-asm" "stack-probe-size"="65536" {
 ; CHECK-LABEL: multi_bb_stpidr2_save_required_stackprobe:
 ; CHECK:       // %bb.0:
@@ -115,6 +153,42 @@ define float @multi_bb_stpidr2_save_required_stackprobe(i32 %a, float %b, float
 ; CHECK-NEXT:    mov sp, x29
 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: multi_bb_stpidr2_save_required_stackprobe:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    str xzr, [sp, #-16]!
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w30, -8
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w29, -16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #16
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-16]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    cbz w0, .LBB2_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %use_b
+; CHECK-NEWLOWERING-NEXT:    fmov s1, #4.00000000
+; CHECK-NEWLOWERING-NEXT:    fadd s0, s0, s1
+; CHECK-NEWLOWERING-NEXT:    b .LBB2_3
+; CHECK-NEWLOWERING-NEXT:  .LBB2_2: // %use_c
+; CHECK-NEWLOWERING-NEXT:    fmov s0, s1
+; CHECK-NEWLOWERING-NEXT:    bl cosf
+; CHECK-NEWLOWERING-NEXT:  .LBB2_3: // %exit
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #16
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB2_5
+; CHECK-NEWLOWERING-NEXT:  // %bb.4: // %exit
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB2_5: // %exit
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   %cmp = icmp ne i32 %a, 0
   br i1 %cmp, label %use_b, label %use_c
 
diff --git a/llvm/test/CodeGen/AArch64/sme-zt0-state.ll b/llvm/test/CodeGen/AArch64/sme-zt0-state.ll
index 63577e4d217a8..57c1ced8ab125 100644
--- a/llvm/test/CodeGen/AArch64/sme-zt0-state.ll
+++ b/llvm/test/CodeGen/AArch64/sme-zt0-state.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -start-after=simplifycfg -enable-tail-merge=false -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -start-after=simplifycfg -enable-tail-merge=false -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -start-after=simplifycfg -enable-tail-merge=false -aarch64-new-sme-abi -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-NEWLOWERING
 
 ;
 ; Private-ZA Callee
@@ -8,19 +9,19 @@
 ; Expect spill & fill of ZT0 around call
 ; Expect smstop/smstart za around call
 define void @zt0_in_caller_no_state_callee(ptr %callee) "aarch64_in_zt0" nounwind {
-; CHECK-LABEL: zt0_in_caller_no_state_callee:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #80
-; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    mov x19, sp
-; CHECK-NEXT:    str zt0, [x19]
-; CHECK-NEXT:    smstop za
-; CHECK-NEXT:    blr x0
-; CHECK-NEXT:    smstart za
-; CHECK-NEXT:    ldr zt0, [x19]
-; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #80
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: zt0_in_caller_no_state_callee:
+; CHECK-COMMON:       // %bb.0:
+; CHECK-COMMON-NEXT:    sub sp, sp, #80
+; CHECK-COMMON-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    mov x19, sp
+; CHECK-COMMON-NEXT:    str zt0, [x19]
+; CHECK-COMMON-NEXT:    smstop za
+; CHECK-COMMON-NEXT:    blr x0
+; CHECK-COMMON-NEXT:    smstart za
+; CHECK-COMMON-NEXT:    ldr zt0, [x19]
+; CHECK-COMMON-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    add sp, sp, #80
+; CHECK-COMMON-NEXT:    ret
   call void %callee();
   ret void;
 }
@@ -61,6 +62,36 @@ define void @za_zt0_shared_caller_no_state_callee(ptr %callee) "aarch64_inout_za
 ; CHECK-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: za_zt0_shared_caller_no_state_callee:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x19, [sp, #16] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mov x29, sp
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #80
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x19, x29, #64
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #80
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-80]
+; CHECK-NEWLOWERING-NEXT:    str zt0, [x19]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    blr x0
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #80
+; CHECK-NEWLOWERING-NEXT:    cbnz x8, .LBB1_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB1_2:
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    ldr zt0, [x19]
+; CHECK-NEWLOWERING-NEXT:    mov sp, x29
+; CHECK-NEWLOWERING-NEXT:    ldr x19, [sp, #16] // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   call void %callee();
   ret void;
 }
@@ -71,41 +102,41 @@ define void @za_zt0_shared_caller_no_state_callee(ptr %callee) "aarch64_inout_za
 
 ; Caller and callee have shared ZT0 state, no spill/fill of ZT0 required
 define void @zt0_shared_caller_zt0_shared_callee(ptr %callee) "aarch64_in_zt0" nounwind {
-; CHECK-LABEL: zt0_shared_caller_zt0_shared_callee:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    blr x0
-; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: zt0_shared_caller_zt0_shared_callee:
+; CHECK-COMMON:       // %bb.0:
+; CHECK-COMMON-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    blr x0
+; CHECK-COMMON-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
   call void %callee() "aarch64_in_zt0";
   ret void;
 }
 
 ; Expect spill & fill of ZT0 around call
 define void @za_zt0_shared_caller_za_shared_callee(ptr %callee) "aarch64_inout_za" "aarch64_in_zt0" nounwind {
-; CHECK-LABEL: za_zt0_shared_caller_za_shared_callee:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #80
-; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    mov x19, sp
-; CHECK-NEXT:    str zt0, [x19]
-; CHECK-NEXT:    blr x0
-; CHECK-NEXT:    ldr zt0, [x19]
-; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #80
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: za_zt0_shared_caller_za_shared_callee:
+; CHECK-COMMON:       // %bb.0:
+; CHECK-COMMON-NEXT:    sub sp, sp, #80
+; CHECK-COMMON-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    mov x19, sp
+; CHECK-COMMON-NEXT:    str zt0, [x19]
+; CHECK-COMMON-NEXT:    blr x0
+; CHECK-COMMON-NEXT:    ldr zt0, [x19]
+; CHECK-COMMON-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    add sp, sp, #80
+; CHECK-COMMON-NEXT:    ret
   call void %callee() "aarch64_inout_za";
   ret void;
 }
 
 ; Caller and callee have shared ZA & ZT0
 define void @za_zt0_shared_caller_za_zt0_shared_callee(ptr %callee) "aarch64_inout_za" "aarch64_in_zt0" nounwind {
-; CHECK-LABEL: za_zt0_shared_caller_za_zt0_shared_callee:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    blr x0
-; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: za_zt0_shared_caller_za_zt0_shared_callee:
+; CHECK-COMMON:       // %bb.0:
+; CHECK-COMMON-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    blr x0
+; CHECK-COMMON-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
   call void %callee() "aarch64_inout_za" "aarch64_in_zt0";
   ret void;
 }
@@ -115,19 +146,19 @@ define void @za_zt0_shared_caller_za_zt0_shared_callee(ptr %callee) "aarch64_ino
 ; Expect spill & fill of ZT0 around call
 ; Expect smstop/smstart za around call
 define void @zt0_in_caller_zt0_new_callee(ptr %callee) "aarch64_in_zt0" nounwind {
-; CHECK-LABEL: zt0_in_caller_zt0_new_callee:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #80
-; CHECK-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    mov x19, sp
-; CHECK-NEXT:    str zt0, [x19]
-; CHECK-NEXT:    smstop za
-; CHECK-NEXT:    blr x0
-; CHECK-NEXT:    smstart za
-; CHECK-NEXT:    ldr zt0, [x19]
-; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #80
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: zt0_in_caller_zt0_new_callee:
+; CHECK-COMMON:       // %bb.0:
+; CHECK-COMMON-NEXT:    sub sp, sp, #80
+; CHECK-COMMON-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    mov x19, sp
+; CHECK-COMMON-NEXT:    str zt0, [x19]
+; CHECK-COMMON-NEXT:    smstop za
+; CHECK-COMMON-NEXT:    blr x0
+; CHECK-COMMON-NEXT:    smstart za
+; CHECK-COMMON-NEXT:    ldr zt0, [x19]
+; CHECK-COMMON-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    add sp, sp, #80
+; CHECK-COMMON-NEXT:    ret
   call void %callee() "aarch64_new_zt0";
   ret void;
 }
@@ -161,6 +192,29 @@ define void @zt0_new_caller_zt0_new_callee(ptr %callee) "aarch64_new_zt0" nounwi
 ; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: zt0_new_caller_zt0_new_callee:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #80
+; CHECK-NEWLOWERING-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    cbz x8, .LBB6_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:  .LBB6_2:
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    zero { zt0 }
+; CHECK-NEWLOWERING-NEXT:    mov x19, sp
+; CHECK-NEWLOWERING-NEXT:    str zt0, [x19]
+; CHECK-NEWLOWERING-NEXT:    smstop za
+; CHECK-NEWLOWERING-NEXT:    blr x0
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    ldr zt0, [x19]
+; CHECK-NEWLOWERING-NEXT:    smstop za
+; CHECK-NEWLOWERING-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    add sp, sp, #80
+; CHECK-NEWLOWERING-NEXT:    ret
   call void %callee() "aarch64_new_zt0";
   ret void;
 }
@@ -191,6 +245,27 @@ define i64 @zt0_new_caller_abi_routine_callee() "aarch64_new_zt0" nounwind {
 ; CHECK-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    add sp, sp, #80
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: zt0_new_caller_abi_routine_callee:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #80
+; CHECK-NEWLOWERING-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    cbz x8, .LBB7_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:  .LBB7_2:
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    zero { zt0 }
+; CHECK-NEWLOWERING-NEXT:    mov x19, sp
+; CHECK-NEWLOWERING-NEXT:    str zt0, [x19]
+; CHECK-NEWLOWERING-NEXT:    bl __arm_sme_state
+; CHECK-NEWLOWERING-NEXT:    ldr zt0, [x19]
+; CHECK-NEWLOWERING-NEXT:    smstop za
+; CHECK-NEWLOWERING-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    add sp, sp, #80
+; CHECK-NEWLOWERING-NEXT:    ret
   %res = call {i64, i64} @__arm_sme_state()
   %res.0 = extractvalue {i64, i64} %res, 0
   ret i64 %res.0
@@ -221,6 +296,22 @@ define void @zt0_new_caller(ptr %callee) "aarch64_new_zt0" nounwind {
 ; CHECK-NEXT:    smstop za
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: zt0_new_caller:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    cbz x8, .LBB8_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:  .LBB8_2:
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    zero { zt0 }
+; CHECK-NEWLOWERING-NEXT:    blr x0
+; CHECK-NEWLOWERING-NEXT:    smstop za
+; CHECK-NEWLOWERING-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   call void %callee() "aarch64_in_zt0";
   ret void;
 }
@@ -245,6 +336,23 @@ define void @new_za_zt0_caller(ptr %callee) "aarch64_new_za" "aarch64_new_zt0" n
 ; CHECK-NEXT:    smstop za
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: new_za_zt0_caller:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    mrs x8, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    cbz x8, .LBB9_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1:
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_save
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    zero {za}
+; CHECK-NEWLOWERING-NEXT:  .LBB9_2:
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    zero { zt0 }
+; CHECK-NEWLOWERING-NEXT:    blr x0
+; CHECK-NEWLOWERING-NEXT:    smstop za
+; CHECK-NEWLOWERING-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   call void %callee() "aarch64_inout_za" "aarch64_in_zt0";
   ret void;
 }
@@ -258,19 +366,26 @@ define void @new_za_shared_zt0_caller(ptr %callee) "aarch64_new_za" "aarch64_in_
 ; CHECK-NEXT:    blr x0
 ; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: new_za_shared_zt0_caller:
+; CHECK-NEWLOWERING:       // %bb.0:
+; CHECK-NEWLOWERING-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    blr x0
+; CHECK-NEWLOWERING-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ret
   call void %callee() "aarch64_inout_za" "aarch64_in_zt0";
   ret void;
 }
 
 ; Expect clear ZT0 on entry
 define void @shared_za_new_zt0(ptr %callee) "aarch64_inout_za" "aarch64_new_zt0" nounwind {
-; CHECK-LABEL: shared_za_new_zt0:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    zero { zt0 }
-; CHECK-NEXT:    blr x0
-; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: shared_za_new_zt0:
+; CHECK-COMMON:       // %bb.0:
+; CHECK-COMMON-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    zero { zt0 }
+; CHECK-COMMON-NEXT:    blr x0
+; CHECK-COMMON-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
   call void %callee() "aarch64_inout_za" "aarch64_in_zt0";
   ret void;
 }
diff --git a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
index c2d8d34b9305a..7cb680b8449cf 100644
--- a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
@@ -63,7 +63,7 @@ define i32 @func4(i32 %x, i32 %y) nounwind {
 ; CHECK-NEXT:    eor w10, w0, w1
 ; CHECK-NEXT:    mov w8, #-2147483648 // =0x80000000
 ; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    cinv w8, w8, ge
+; CHECK-NEXT:    cinv w8, w8, pl
 ; CHECK-NEXT:    cmp x9, w9, sxtw
 ; CHECK-NEXT:    csel w0, w8, w9, ne
 ; CHECK-NEXT:    ret
@@ -79,7 +79,7 @@ define i64 @func5(i64 %x, i64 %y) {
 ; CHECK-NEXT:    mov x8, #-9223372036854775808 // =0x8000000000000000
 ; CHECK-NEXT:    cmp x11, #0
 ; CHECK-NEXT:    smulh x10, x0, x1
-; CHECK-NEXT:    cinv x8, x8, ge
+; CHECK-NEXT:    cinv x8, x8, pl
 ; CHECK-NEXT:    cmp x10, x9, asr #63
 ; CHECK-NEXT:    csel x0, x8, x9, ne
 ; CHECK-NEXT:    ret
@@ -96,7 +96,7 @@ define i4 @func6(i4 %x, i4 %y) nounwind {
 ; CHECK-NEXT:    smull x11, w10, w9
 ; CHECK-NEXT:    eor w9, w10, w9
 ; CHECK-NEXT:    cmp w9, #0
-; CHECK-NEXT:    cinv w8, w8, ge
+; CHECK-NEXT:    cinv w8, w8, pl
 ; CHECK-NEXT:    cmp x11, w11, sxtw
 ; CHECK-NEXT:    csel w8, w8, w11, ne
 ; CHECK-NEXT:    asr w0, w8, #28
@@ -158,11 +158,11 @@ define <2 x i32> @vec(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-NEXT:    cmp w9, #0
 ; CHECK-NEXT:    smull x9, w12, w10
 ; CHECK-NEXT:    eor w10, w12, w10
-; CHECK-NEXT:    cinv w12, w8, ge
+; CHECK-NEXT:    cinv w12, w8, pl
 ; CHECK-NEXT:    cmp x11, w11, sxtw
 ; CHECK-NEXT:    csel w11, w12, w11, ne
 ; CHECK-NEXT:    cmp w10, #0
-; CHECK-NEXT:    cinv w8, w8, ge
+; CHECK-NEXT:    cinv w8, w8, pl
 ; CHECK-NEXT:    cmp x9, w9, sxtw
 ; CHECK-NEXT:    csel w8, w8, w9, ne
 ; CHECK-NEXT:    fmov s0, w8
@@ -188,12 +188,12 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-NEXT:    cmp w11, #0
 ; CHECK-NEXT:    smull x11, w13, w12
 ; CHECK-NEXT:    eor w12, w13, w12
-; CHECK-NEXT:    cinv w13, w8, ge
+; CHECK-NEXT:    cinv w13, w8, pl
 ; CHECK-NEXT:    cmp x9, w9, sxtw
 ; CHECK-NEXT:    csel w9, w13, w9, ne
 ; CHECK-NEXT:    cmp w12, #0
 ; CHECK-NEXT:    mov w13, v1.s[3]
-; CHECK-NEXT:    cinv w12, w8, ge
+; CHECK-NEXT:    cinv w12, w8, pl
 ; CHECK-NEXT:    cmp x11, w11, sxtw
 ; CHECK-NEXT:    csel w11, w12, w11, ne
 ; CHECK-NEXT:    mov w12, v0.s[3]
@@ -203,13 +203,13 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-NEXT:    eor w9, w14, w10
 ; CHECK-NEXT:    smull x10, w12, w13
 ; CHECK-NEXT:    cmp w9, #0
-; CHECK-NEXT:    cinv w9, w8, ge
+; CHECK-NEXT:    cinv w9, w8, pl
 ; CHECK-NEXT:    cmp x11, w11, sxtw
 ; CHECK-NEXT:    csel w9, w9, w11, ne
 ; CHECK-NEXT:    mov v0.s[2], w9
 ; CHECK-NEXT:    eor w9, w12, w13
 ; CHECK-NEXT:    cmp w9, #0
-; CHECK-NEXT:    cinv w8, w8, ge
+; CHECK-NEXT:    cinv w8, w8, pl
 ; CHECK-NEXT:    cmp x10, w10, sxtw
 ; CHECK-NEXT:    csel w8, w8, w10, ne
 ; CHECK-NEXT:    mov v0.s[3], w8
diff --git a/llvm/test/CodeGen/AArch64/spill-reload-remarks.ll b/llvm/test/CodeGen/AArch64/spill-reload-remarks.ll
index 47906252382f4..6c248048e682f 100644
--- a/llvm/test/CodeGen/AArch64/spill-reload-remarks.ll
+++ b/llvm/test/CodeGen/AArch64/spill-reload-remarks.ll
@@ -2,8 +2,7 @@
 
 ; We should have both spill and reload for %arg.
 
-; CHECK: remark: <unknown>:0:0: 2 spills 2.000000e+00 total spills cost 3 reloads 3.000000e+00 total reloads cost generated in function
-
+; CHECK: remark: <unknown>:0:0: 2 spills 2.000000e+00 total spills cost 3 reloads 3.000000e+00 total reloads cost 1 virtual registers copies 1.000000e+00 total copies cost generated in function
 define <vscale x 2 x i1> @streaming_compatible_with_predicate_vectors(<vscale x 2 x i1> %arg) "aarch64_pstate_sm_compatible" nounwind #0 {
   %res = call <vscale x 2 x i1> @normal_callee_predicate_vec_arg(<vscale x 2 x i1> %arg)
   %and = and <vscale x 2 x i1> %res, %arg
diff --git a/llvm/test/CodeGen/AArch64/split-deadloop.mir b/llvm/test/CodeGen/AArch64/split-deadloop.mir
index 000aea02e3425..fc21d3ec564a2 100644
--- a/llvm/test/CodeGen/AArch64/split-deadloop.mir
+++ b/llvm/test/CodeGen/AArch64/split-deadloop.mir
@@ -66,8 +66,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/AArch64/srem-pow2.ll b/llvm/test/CodeGen/AArch64/srem-pow2.ll
index 4c114d185997e..a0124b9ab4a5e 100644
--- a/llvm/test/CodeGen/AArch64/srem-pow2.ll
+++ b/llvm/test/CodeGen/AArch64/srem-pow2.ll
@@ -45,7 +45,7 @@ define i32 @fold_srem_2_i64(i32 %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and w8, w0, #0x1
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    cneg w0, w8, lt
+; CHECK-NEXT:    cneg w0, w8, mi
 ; CHECK-NEXT:    ret
   %1 = srem i32 %x, 2
   ret i32 %1
@@ -56,7 +56,7 @@ define i64 @fold_srem_2_i32(i64 %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    and x8, x0, #0x1
 ; CHECK-NEXT:    cmp x0, #0
-; CHECK-NEXT:    cneg x0, x8, lt
+; CHECK-NEXT:    cneg x0, x8, mi
 ; CHECK-NEXT:    ret
   %1 = srem i64 %x, 2
   ret i64 %1
diff --git a/llvm/test/CodeGen/AArch64/srem-seteq.ll b/llvm/test/CodeGen/AArch64/srem-seteq.ll
index 4b8cbc46a6102..3b344feebb58e 100644
--- a/llvm/test/CodeGen/AArch64/srem-seteq.ll
+++ b/llvm/test/CodeGen/AArch64/srem-seteq.ll
@@ -166,10 +166,9 @@ define i32 @test_srem_odd_setne(i32 %X) nounwind {
 ; CHECK-NEXT:    movk w8, #52428, lsl #16
 ; CHECK-NEXT:    movk w9, #6553, lsl #16
 ; CHECK-NEXT:    madd w8, w0, w8, w9
-; CHECK-NEXT:    mov w9, #13106 // =0x3332
-; CHECK-NEXT:    movk w9, #13107, lsl #16
+; CHECK-NEXT:    mov w9, #858993459 // =0x33333333
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    cset w0, hs
 ; CHECK-NEXT:    ret
   %srem = srem i32 %X, 5
   %cmp = icmp ne i32 %srem, 0
@@ -186,10 +185,9 @@ define i32 @test_srem_negative_odd(i32 %X) nounwind {
 ; CHECK-NEXT:    movk w8, #52428, lsl #16
 ; CHECK-NEXT:    movk w9, #6553, lsl #16
 ; CHECK-NEXT:    madd w8, w0, w8, w9
-; CHECK-NEXT:    mov w9, #13106 // =0x3332
-; CHECK-NEXT:    movk w9, #13107, lsl #16
+; CHECK-NEXT:    mov w9, #858993459 // =0x33333333
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    cset w0, hs
 ; CHECK-NEXT:    ret
   %srem = srem i32 %X, -5
   %cmp = icmp ne i32 %srem, 0
diff --git a/llvm/test/CodeGen/AArch64/sshl_sat.ll b/llvm/test/CodeGen/AArch64/sshl_sat.ll
index fbcd2db1298f0..be2b3e763733b 100644
--- a/llvm/test/CodeGen/AArch64/sshl_sat.ll
+++ b/llvm/test/CodeGen/AArch64/sshl_sat.ll
@@ -146,7 +146,7 @@ define i16 @combine_shlsat_to_shl_no_fold(i16 %x) nounwind {
 ; CHECK-NEXT:    mov w9, #-65536 // =0xffff0000
 ; CHECK-NEXT:    mov w10, #-2147483648 // =0x80000000
 ; CHECK-NEXT:    ands w8, w9, w8, lsl #14
-; CHECK-NEXT:    cinv w10, w10, ge
+; CHECK-NEXT:    cinv w10, w10, pl
 ; CHECK-NEXT:    lsl w9, w8, #3
 ; CHECK-NEXT:    cmp w8, w9, asr #3
 ; CHECK-NEXT:    csel w8, w10, w9, ne
diff --git a/llvm/test/CodeGen/AArch64/ssve-stack-hazard-remarks.ll b/llvm/test/CodeGen/AArch64/ssve-stack-hazard-remarks.ll
index c67d91952c618..1de8d0a080b70 100644
--- a/llvm/test/CodeGen/AArch64/ssve-stack-hazard-remarks.ll
+++ b/llvm/test/CodeGen/AArch64/ssve-stack-hazard-remarks.ll
@@ -72,12 +72,12 @@ entry:
 ; mitigated with the -aarch64-enable-zpr-predicate-spills option.
 
 define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3, i16 %P4) #2 {
-; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at [SP-48-258 * vscale] is too close to FPR stack object at [SP-48-256 * vscale]
-; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_call': FPR stack object at [SP-48-16 * vscale] is too close to GPR stack object at [SP-48]
-; CHECK-PADDING: remark: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at [SP-1072-258 * vscale] is too close to FPR stack object at [SP-1072-256 * vscale]
+; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at [SP-64-258 * vscale] is too close to FPR stack object at [SP-64-256 * vscale]
+; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_call': FPR stack object at [SP-64-16 * vscale] is too close to GPR stack object at [SP-64]
+; CHECK-PADDING: remark: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at [SP-1088-258 * vscale] is too close to FPR stack object at [SP-1088-256 * vscale]
 ; CHECK-PADDING-NOT: remark: <unknown>:0:0: stack hazard in 'svecc_call':
 ; CHECK-ZPR-PRED-SPILLS-NOT: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at {{.*}} is too close to FPR stack object
-; CHECK-ZPR-PRED-SPILLS: <unknown>:0:0: stack hazard in 'svecc_call': FPR stack object at [SP-48-16 * vscale] is too close to GPR stack object at [SP-48]
+; CHECK-ZPR-PRED-SPILLS: <unknown>:0:0: stack hazard in 'svecc_call': FPR stack object at [SP-64-16 * vscale] is too close to GPR stack object at [SP-64]
 ; CHECK-ZPR-PRED-SPILLS-WITH-PADDING-NOT: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at {{.*}} is too close to FPR stack object
 ; CHECK-ZPR-PRED-SPILLS-WITH-PADDING-NOT: <unknown>:0:0: stack hazard in 'svecc_call': FPR stack object at {{.*}} is too close to GPR stack object
 entry:
@@ -87,12 +87,12 @@ entry:
 }
 
 define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3, i16 %P4) #2 {
-; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call': PPR stack object at [SP-48-258 * vscale] is too close to FPR stack object at [SP-48-256 * vscale]
-; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call': FPR stack object at [SP-48-16 * vscale] is too close to GPR stack object at [SP-48]
-; CHECK-PADDING: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call': PPR stack object at [SP-1072-258 * vscale] is too close to FPR stack object at [SP-1072-256 * vscale]
+; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call': PPR stack object at [SP-64-258 * vscale] is too close to FPR stack object at [SP-64-256 * vscale]
+; CHECK: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call': FPR stack object at [SP-64-16 * vscale] is too close to GPR stack object at [SP-64]
+; CHECK-PADDING: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call': PPR stack object at [SP-1088-258 * vscale] is too close to FPR stack object at [SP-1088-256 * vscale]
 ; CHECK-PADDING-NOT: remark: <unknown>:0:0: stack hazard in 'svecc_alloca_call':
 ; CHECK-ZPR-PRED-SPILLS-NOT: <unknown>:0:0: stack hazard in 'svecc_call': PPR stack object at {{.*}} is too close to FPR stack object
-; CHECK-ZPR-PRED-SPILLS: <unknown>:0:0: stack hazard in 'svecc_alloca_call': FPR stack object at [SP-48-16 * vscale] is too close to GPR stack object at [SP-48]
+; CHECK-ZPR-PRED-SPILLS: <unknown>:0:0: stack hazard in 'svecc_alloca_call': FPR stack object at [SP-64-16 * vscale] is too close to GPR stack object at [SP-64]
 ; CHECK-ZPR-PRED-SPILLS-WITH-PADDING-NOT: <unknown>:0:0: stack hazard in 'svecc_alloca_call': PPR stack object at {{.*}} is too close to FPR stack object
 ; CHECK-ZPR-PRED-SPILLS-WITH-PADDING-NOT: <unknown>:0:0: stack hazard in 'svecc_alloca_call': FPR stack object at {{.*}} is too close to GPR stack object
 entry:
diff --git a/llvm/test/CodeGen/AArch64/stack-hazard.ll b/llvm/test/CodeGen/AArch64/stack-hazard.ll
index 4615b1a6a9b2e..c878d888b5f03 100644
--- a/llvm/test/CodeGen/AArch64/stack-hazard.ll
+++ b/llvm/test/CodeGen/AArch64/stack-hazard.ll
@@ -616,16 +616,13 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK0:       // %bb.0: // %entry
 ; CHECK0-NEXT:    sub sp, sp, #176
 ; CHECK0-NEXT:    .cfi_def_cfa_offset 176
-; CHECK0-NEXT:    rdsvl x9, #1
-; CHECK0-NEXT:    stp d15, d14, [sp, #48] // 16-byte Folded Spill
-; CHECK0-NEXT:    lsr x9, x9, #3
-; CHECK0-NEXT:    stp d13, d12, [sp, #64] // 16-byte Folded Spill
-; CHECK0-NEXT:    stp d11, d10, [sp, #80] // 16-byte Folded Spill
-; CHECK0-NEXT:    str x9, [sp, #32] // 8-byte Folded Spill
 ; CHECK0-NEXT:    cntd x9
-; CHECK0-NEXT:    str x9, [sp, #40] // 8-byte Folded Spill
-; CHECK0-NEXT:    stp d9, d8, [sp, #96] // 16-byte Folded Spill
-; CHECK0-NEXT:    str x25, [sp, #112] // 8-byte Folded Spill
+; CHECK0-NEXT:    stp d15, d14, [sp, #32] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp d13, d12, [sp, #48] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x29, x30, [sp, #96] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x9, x25, [sp, #112] // 16-byte Folded Spill
 ; CHECK0-NEXT:    stp x24, x23, [sp, #128] // 16-byte Folded Spill
 ; CHECK0-NEXT:    stp x22, x21, [sp, #144] // 16-byte Folded Spill
 ; CHECK0-NEXT:    stp x20, x19, [sp, #160] // 16-byte Folded Spill
@@ -635,16 +632,18 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK0-NEXT:    .cfi_offset w22, -32
 ; CHECK0-NEXT:    .cfi_offset w23, -40
 ; CHECK0-NEXT:    .cfi_offset w24, -48
-; CHECK0-NEXT:    .cfi_offset w25, -64
-; CHECK0-NEXT:    .cfi_offset b8, -72
-; CHECK0-NEXT:    .cfi_offset b9, -80
-; CHECK0-NEXT:    .cfi_offset b10, -88
-; CHECK0-NEXT:    .cfi_offset b11, -96
-; CHECK0-NEXT:    .cfi_offset b12, -104
-; CHECK0-NEXT:    .cfi_offset b13, -112
-; CHECK0-NEXT:    .cfi_offset b14, -120
-; CHECK0-NEXT:    .cfi_offset b15, -128
-; CHECK0-NEXT:    .cfi_offset vg, -136
+; CHECK0-NEXT:    .cfi_offset w25, -56
+; CHECK0-NEXT:    .cfi_offset vg, -64
+; CHECK0-NEXT:    .cfi_offset w30, -72
+; CHECK0-NEXT:    .cfi_offset w29, -80
+; CHECK0-NEXT:    .cfi_offset b8, -88
+; CHECK0-NEXT:    .cfi_offset b9, -96
+; CHECK0-NEXT:    .cfi_offset b10, -104
+; CHECK0-NEXT:    .cfi_offset b11, -112
+; CHECK0-NEXT:    .cfi_offset b12, -120
+; CHECK0-NEXT:    .cfi_offset b13, -128
+; CHECK0-NEXT:    .cfi_offset b14, -136
+; CHECK0-NEXT:    .cfi_offset b15, -144
 ; CHECK0-NEXT:    str d0, [sp, #8] // 8-byte Folded Spill
 ; CHECK0-NEXT:    smstart sm
 ; CHECK0-NEXT:    //APP
@@ -658,12 +657,13 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK0-NEXT:    ldp x20, x19, [sp, #160] // 16-byte Folded Reload
 ; CHECK0-NEXT:    mov w0, wzr
 ; CHECK0-NEXT:    ldp x22, x21, [sp, #144] // 16-byte Folded Reload
-; CHECK0-NEXT:    ldr x25, [sp, #112] // 8-byte Folded Reload
+; CHECK0-NEXT:    ldr x25, [sp, #120] // 8-byte Folded Reload
 ; CHECK0-NEXT:    ldp x24, x23, [sp, #128] // 16-byte Folded Reload
-; CHECK0-NEXT:    ldp d9, d8, [sp, #96] // 16-byte Folded Reload
-; CHECK0-NEXT:    ldp d11, d10, [sp, #80] // 16-byte Folded Reload
-; CHECK0-NEXT:    ldp d13, d12, [sp, #64] // 16-byte Folded Reload
-; CHECK0-NEXT:    ldp d15, d14, [sp, #48] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x29, x30, [sp, #96] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp d9, d8, [sp, #80] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp d11, d10, [sp, #64] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp d13, d12, [sp, #48] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp d15, d14, [sp, #32] // 16-byte Folded Reload
 ; CHECK0-NEXT:    add sp, sp, #176
 ; CHECK0-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK0-NEXT:    .cfi_restore w19
@@ -673,6 +673,9 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK0-NEXT:    .cfi_restore w23
 ; CHECK0-NEXT:    .cfi_restore w24
 ; CHECK0-NEXT:    .cfi_restore w25
+; CHECK0-NEXT:    .cfi_restore vg
+; CHECK0-NEXT:    .cfi_restore w30
+; CHECK0-NEXT:    .cfi_restore w29
 ; CHECK0-NEXT:    .cfi_restore b8
 ; CHECK0-NEXT:    .cfi_restore b9
 ; CHECK0-NEXT:    .cfi_restore b10
@@ -687,16 +690,13 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK64:       // %bb.0: // %entry
 ; CHECK64-NEXT:    sub sp, sp, #304
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 304
-; CHECK64-NEXT:    rdsvl x9, #1
-; CHECK64-NEXT:    stp d15, d14, [sp, #112] // 16-byte Folded Spill
-; CHECK64-NEXT:    lsr x9, x9, #3
-; CHECK64-NEXT:    stp d13, d12, [sp, #128] // 16-byte Folded Spill
-; CHECK64-NEXT:    stp d11, d10, [sp, #144] // 16-byte Folded Spill
-; CHECK64-NEXT:    str x9, [sp, #96] // 8-byte Folded Spill
 ; CHECK64-NEXT:    cntd x9
-; CHECK64-NEXT:    str x9, [sp, #104] // 8-byte Folded Spill
-; CHECK64-NEXT:    stp d9, d8, [sp, #160] // 16-byte Folded Spill
-; CHECK64-NEXT:    stp x29, x25, [sp, #240] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp d15, d14, [sp, #96] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp d13, d12, [sp, #112] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp d11, d10, [sp, #128] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp d9, d8, [sp, #144] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x29, x30, [sp, #224] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x9, x25, [sp, #240] // 16-byte Folded Spill
 ; CHECK64-NEXT:    stp x24, x23, [sp, #256] // 16-byte Folded Spill
 ; CHECK64-NEXT:    stp x22, x21, [sp, #272] // 16-byte Folded Spill
 ; CHECK64-NEXT:    stp x20, x19, [sp, #288] // 16-byte Folded Spill
@@ -707,16 +707,17 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK64-NEXT:    .cfi_offset w23, -40
 ; CHECK64-NEXT:    .cfi_offset w24, -48
 ; CHECK64-NEXT:    .cfi_offset w25, -56
-; CHECK64-NEXT:    .cfi_offset w29, -64
-; CHECK64-NEXT:    .cfi_offset b8, -136
-; CHECK64-NEXT:    .cfi_offset b9, -144
-; CHECK64-NEXT:    .cfi_offset b10, -152
-; CHECK64-NEXT:    .cfi_offset b11, -160
-; CHECK64-NEXT:    .cfi_offset b12, -168
-; CHECK64-NEXT:    .cfi_offset b13, -176
-; CHECK64-NEXT:    .cfi_offset b14, -184
-; CHECK64-NEXT:    .cfi_offset b15, -192
-; CHECK64-NEXT:    .cfi_offset vg, -200
+; CHECK64-NEXT:    .cfi_offset vg, -64
+; CHECK64-NEXT:    .cfi_offset w30, -72
+; CHECK64-NEXT:    .cfi_offset w29, -80
+; CHECK64-NEXT:    .cfi_offset b8, -152
+; CHECK64-NEXT:    .cfi_offset b9, -160
+; CHECK64-NEXT:    .cfi_offset b10, -168
+; CHECK64-NEXT:    .cfi_offset b11, -176
+; CHECK64-NEXT:    .cfi_offset b12, -184
+; CHECK64-NEXT:    .cfi_offset b13, -192
+; CHECK64-NEXT:    .cfi_offset b14, -200
+; CHECK64-NEXT:    .cfi_offset b15, -208
 ; CHECK64-NEXT:    str d0, [sp, #80] // 8-byte Folded Spill
 ; CHECK64-NEXT:    smstart sm
 ; CHECK64-NEXT:    //APP
@@ -730,12 +731,13 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK64-NEXT:    ldp x20, x19, [sp, #288] // 16-byte Folded Reload
 ; CHECK64-NEXT:    mov w0, wzr
 ; CHECK64-NEXT:    ldp x22, x21, [sp, #272] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr x25, [sp, #248] // 8-byte Folded Reload
 ; CHECK64-NEXT:    ldp x24, x23, [sp, #256] // 16-byte Folded Reload
-; CHECK64-NEXT:    ldp x29, x25, [sp, #240] // 16-byte Folded Reload
-; CHECK64-NEXT:    ldp d9, d8, [sp, #160] // 16-byte Folded Reload
-; CHECK64-NEXT:    ldp d11, d10, [sp, #144] // 16-byte Folded Reload
-; CHECK64-NEXT:    ldp d13, d12, [sp, #128] // 16-byte Folded Reload
-; CHECK64-NEXT:    ldp d15, d14, [sp, #112] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp x29, x30, [sp, #224] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp d9, d8, [sp, #144] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp d11, d10, [sp, #128] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp d13, d12, [sp, #112] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp d15, d14, [sp, #96] // 16-byte Folded Reload
 ; CHECK64-NEXT:    add sp, sp, #304
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK64-NEXT:    .cfi_restore w19
@@ -745,6 +747,8 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK64-NEXT:    .cfi_restore w23
 ; CHECK64-NEXT:    .cfi_restore w24
 ; CHECK64-NEXT:    .cfi_restore w25
+; CHECK64-NEXT:    .cfi_restore vg
+; CHECK64-NEXT:    .cfi_restore w30
 ; CHECK64-NEXT:    .cfi_restore w29
 ; CHECK64-NEXT:    .cfi_restore b8
 ; CHECK64-NEXT:    .cfi_restore b9
@@ -758,18 +762,16 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ;
 ; CHECK1024-LABEL: csr_x18_25_d8_15_allocdi64_locallystreaming:
 ; CHECK1024:       // %bb.0: // %entry
-; CHECK1024-NEXT:    rdsvl x9, #1
-; CHECK1024-NEXT:    lsr x9, x9, #3
 ; CHECK1024-NEXT:    sub sp, sp, #1168
 ; CHECK1024-NEXT:    .cfi_def_cfa_offset 1168
-; CHECK1024-NEXT:    str x9, [sp] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    cntd x9
-; CHECK1024-NEXT:    str x9, [sp, #8] // 8-byte Folded Spill
-; CHECK1024-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
-; CHECK1024-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
-; CHECK1024-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
-; CHECK1024-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK1024-NEXT:    str x29, [sp, #1104] // 8-byte Folded Spill
+; CHECK1024-NEXT:    stp d15, d14, [sp] // 16-byte Folded Spill
+; CHECK1024-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK1024-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK1024-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str x29, [sp, #1088] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x30, [sp, #1096] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x9, [sp, #1104] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x25, [sp, #1112] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x24, [sp, #1120] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x23, [sp, #1128] // 8-byte Folded Spill
@@ -784,16 +786,17 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK1024-NEXT:    .cfi_offset w23, -40
 ; CHECK1024-NEXT:    .cfi_offset w24, -48
 ; CHECK1024-NEXT:    .cfi_offset w25, -56
-; CHECK1024-NEXT:    .cfi_offset w29, -64
-; CHECK1024-NEXT:    .cfi_offset b8, -1096
-; CHECK1024-NEXT:    .cfi_offset b9, -1104
-; CHECK1024-NEXT:    .cfi_offset b10, -1112
-; CHECK1024-NEXT:    .cfi_offset b11, -1120
-; CHECK1024-NEXT:    .cfi_offset b12, -1128
-; CHECK1024-NEXT:    .cfi_offset b13, -1136
-; CHECK1024-NEXT:    .cfi_offset b14, -1144
-; CHECK1024-NEXT:    .cfi_offset b15, -1152
-; CHECK1024-NEXT:    .cfi_offset vg, -1160
+; CHECK1024-NEXT:    .cfi_offset vg, -64
+; CHECK1024-NEXT:    .cfi_offset w30, -72
+; CHECK1024-NEXT:    .cfi_offset w29, -80
+; CHECK1024-NEXT:    .cfi_offset b8, -1112
+; CHECK1024-NEXT:    .cfi_offset b9, -1120
+; CHECK1024-NEXT:    .cfi_offset b10, -1128
+; CHECK1024-NEXT:    .cfi_offset b11, -1136
+; CHECK1024-NEXT:    .cfi_offset b12, -1144
+; CHECK1024-NEXT:    .cfi_offset b13, -1152
+; CHECK1024-NEXT:    .cfi_offset b14, -1160
+; CHECK1024-NEXT:    .cfi_offset b15, -1168
 ; CHECK1024-NEXT:    sub sp, sp, #1056
 ; CHECK1024-NEXT:    .cfi_def_cfa_offset 2224
 ; CHECK1024-NEXT:    str d0, [sp, #1040] // 8-byte Folded Spill
@@ -809,18 +812,19 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK1024-NEXT:    mov w0, wzr
 ; CHECK1024-NEXT:    add sp, sp, #1056
 ; CHECK1024-NEXT:    .cfi_def_cfa_offset 1168
-; CHECK1024-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x19, [sp, #1160] // 8-byte Folded Reload
-; CHECK1024-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x20, [sp, #1152] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x21, [sp, #1144] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x22, [sp, #1136] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x23, [sp, #1128] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x24, [sp, #1120] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x25, [sp, #1112] // 8-byte Folded Reload
-; CHECK1024-NEXT:    ldr x29, [sp, #1104] // 8-byte Folded Reload
-; CHECK1024-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
-; CHECK1024-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr x30, [sp, #1096] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x29, [sp, #1088] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldp d15, d14, [sp] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    add sp, sp, #1168
 ; CHECK1024-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK1024-NEXT:    .cfi_restore w19
@@ -830,6 +834,8 @@ define i32 @csr_x18_25_d8_15_allocdi64_locallystreaming(i64 %d, double %e) "aarc
 ; CHECK1024-NEXT:    .cfi_restore w23
 ; CHECK1024-NEXT:    .cfi_restore w24
 ; CHECK1024-NEXT:    .cfi_restore w25
+; CHECK1024-NEXT:    .cfi_restore vg
+; CHECK1024-NEXT:    .cfi_restore w30
 ; CHECK1024-NEXT:    .cfi_restore w29
 ; CHECK1024-NEXT:    .cfi_restore b8
 ; CHECK1024-NEXT:    .cfi_restore b9
@@ -1570,36 +1576,38 @@ define [2 x <vscale x 4 x i1>] @sve_signature_pred_2xv4i1_caller([2 x <vscale x
 define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32 %b) "aarch64_pstate_sm_compatible" {
 ; CHECK0-LABEL: f128_libcall:
 ; CHECK0:       // %bb.0:
-; CHECK0-NEXT:    sub sp, sp, #176
-; CHECK0-NEXT:    .cfi_def_cfa_offset 176
+; CHECK0-NEXT:    sub sp, sp, #192
+; CHECK0-NEXT:    .cfi_def_cfa_offset 192
 ; CHECK0-NEXT:    cntd x9
 ; CHECK0-NEXT:    stp d15, d14, [sp, #64] // 16-byte Folded Spill
 ; CHECK0-NEXT:    stp d13, d12, [sp, #80] // 16-byte Folded Spill
 ; CHECK0-NEXT:    stp d11, d10, [sp, #96] // 16-byte Folded Spill
 ; CHECK0-NEXT:    stp d9, d8, [sp, #112] // 16-byte Folded Spill
-; CHECK0-NEXT:    stp x30, x9, [sp, #128] // 16-byte Folded Spill
-; CHECK0-NEXT:    stp x22, x21, [sp, #144] // 16-byte Folded Spill
-; CHECK0-NEXT:    stp x20, x19, [sp, #160] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x29, x30, [sp, #128] // 16-byte Folded Spill
+; CHECK0-NEXT:    str x9, [sp, #144] // 8-byte Folded Spill
+; CHECK0-NEXT:    stp x22, x21, [sp, #160] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x20, x19, [sp, #176] // 16-byte Folded Spill
 ; CHECK0-NEXT:    .cfi_offset w19, -8
 ; CHECK0-NEXT:    .cfi_offset w20, -16
 ; CHECK0-NEXT:    .cfi_offset w21, -24
 ; CHECK0-NEXT:    .cfi_offset w22, -32
-; CHECK0-NEXT:    .cfi_offset w30, -48
-; CHECK0-NEXT:    .cfi_offset b8, -56
-; CHECK0-NEXT:    .cfi_offset b9, -64
-; CHECK0-NEXT:    .cfi_offset b10, -72
-; CHECK0-NEXT:    .cfi_offset b11, -80
-; CHECK0-NEXT:    .cfi_offset b12, -88
-; CHECK0-NEXT:    .cfi_offset b13, -96
-; CHECK0-NEXT:    .cfi_offset b14, -104
-; CHECK0-NEXT:    .cfi_offset b15, -112
+; CHECK0-NEXT:    .cfi_offset vg, -48
+; CHECK0-NEXT:    .cfi_offset w30, -56
+; CHECK0-NEXT:    .cfi_offset w29, -64
+; CHECK0-NEXT:    .cfi_offset b8, -72
+; CHECK0-NEXT:    .cfi_offset b9, -80
+; CHECK0-NEXT:    .cfi_offset b10, -88
+; CHECK0-NEXT:    .cfi_offset b11, -96
+; CHECK0-NEXT:    .cfi_offset b12, -104
+; CHECK0-NEXT:    .cfi_offset b13, -112
+; CHECK0-NEXT:    .cfi_offset b14, -120
+; CHECK0-NEXT:    .cfi_offset b15, -128
 ; CHECK0-NEXT:    mov w19, w1
 ; CHECK0-NEXT:    mov w20, w0
 ; CHECK0-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK0-NEXT:    stp q2, q3, [sp, #32] // 32-byte Folded Spill
 ; CHECK0-NEXT:    bl __arm_sme_state
-; CHECK0-NEXT:    and x21, x0, #0x1
-; CHECK0-NEXT:    .cfi_offset vg, -40
+; CHECK0-NEXT:    mov x21, x0
 ; CHECK0-NEXT:    tbz w21, #0, .LBB27_2
 ; CHECK0-NEXT:  // %bb.1:
 ; CHECK0-NEXT:    smstop sm
@@ -1611,40 +1619,37 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK0-NEXT:    smstart sm
 ; CHECK0-NEXT:  .LBB27_4:
 ; CHECK0-NEXT:    cmp w0, #0
-; CHECK0-NEXT:    .cfi_restore vg
-; CHECK0-NEXT:    cset w21, lt
-; CHECK0-NEXT:    bl __arm_sme_state
-; CHECK0-NEXT:    and x22, x0, #0x1
-; CHECK0-NEXT:    .cfi_offset vg, -40
-; CHECK0-NEXT:    tbz w22, #0, .LBB27_6
+; CHECK0-NEXT:    cset w22, mi
+; CHECK0-NEXT:    tbz w21, #0, .LBB27_6
 ; CHECK0-NEXT:  // %bb.5:
 ; CHECK0-NEXT:    smstop sm
 ; CHECK0-NEXT:  .LBB27_6:
 ; CHECK0-NEXT:    ldp q0, q1, [sp, #32] // 32-byte Folded Reload
 ; CHECK0-NEXT:    bl __getf2
-; CHECK0-NEXT:    tbz w22, #0, .LBB27_8
+; CHECK0-NEXT:    tbz w21, #0, .LBB27_8
 ; CHECK0-NEXT:  // %bb.7:
 ; CHECK0-NEXT:    smstart sm
 ; CHECK0-NEXT:  .LBB27_8:
 ; CHECK0-NEXT:    cmp w0, #0
-; CHECK0-NEXT:    cset w8, ge
-; CHECK0-NEXT:    tst w8, w21
-; CHECK0-NEXT:    csel w0, w20, w19, ne
-; CHECK0-NEXT:    .cfi_restore vg
-; CHECK0-NEXT:    ldp x20, x19, [sp, #160] // 16-byte Folded Reload
-; CHECK0-NEXT:    ldr x30, [sp, #128] // 8-byte Folded Reload
-; CHECK0-NEXT:    ldp x22, x21, [sp, #144] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x29, x30, [sp, #128] // 16-byte Folded Reload
+; CHECK0-NEXT:    cset w8, pl
 ; CHECK0-NEXT:    ldp d9, d8, [sp, #112] // 16-byte Folded Reload
+; CHECK0-NEXT:    tst w8, w22
+; CHECK0-NEXT:    ldp x22, x21, [sp, #160] // 16-byte Folded Reload
+; CHECK0-NEXT:    csel w0, w20, w19, ne
+; CHECK0-NEXT:    ldp x20, x19, [sp, #176] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldp d11, d10, [sp, #96] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldp d13, d12, [sp, #80] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldp d15, d14, [sp, #64] // 16-byte Folded Reload
-; CHECK0-NEXT:    add sp, sp, #176
+; CHECK0-NEXT:    add sp, sp, #192
 ; CHECK0-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK0-NEXT:    .cfi_restore w19
 ; CHECK0-NEXT:    .cfi_restore w20
 ; CHECK0-NEXT:    .cfi_restore w21
 ; CHECK0-NEXT:    .cfi_restore w22
+; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    .cfi_restore w30
+; CHECK0-NEXT:    .cfi_restore w29
 ; CHECK0-NEXT:    .cfi_restore b8
 ; CHECK0-NEXT:    .cfi_restore b9
 ; CHECK0-NEXT:    .cfi_restore b10
@@ -1665,13 +1670,15 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK64-NEXT:    stp d11, d10, [sp, #160] // 16-byte Folded Spill
 ; CHECK64-NEXT:    stp d9, d8, [sp, #176] // 16-byte Folded Spill
 ; CHECK64-NEXT:    stp x29, x30, [sp, #256] // 16-byte Folded Spill
-; CHECK64-NEXT:    stp x9, x22, [sp, #272] // 16-byte Folded Spill
-; CHECK64-NEXT:    stp x21, x20, [sp, #288] // 16-byte Folded Spill
-; CHECK64-NEXT:    str x19, [sp, #304] // 8-byte Folded Spill
-; CHECK64-NEXT:    .cfi_offset w19, -16
-; CHECK64-NEXT:    .cfi_offset w20, -24
-; CHECK64-NEXT:    .cfi_offset w21, -32
-; CHECK64-NEXT:    .cfi_offset w22, -40
+; CHECK64-NEXT:    stp x9, x28, [sp, #272] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x22, x21, [sp, #288] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x20, x19, [sp, #304] // 16-byte Folded Spill
+; CHECK64-NEXT:    .cfi_offset w19, -8
+; CHECK64-NEXT:    .cfi_offset w20, -16
+; CHECK64-NEXT:    .cfi_offset w21, -24
+; CHECK64-NEXT:    .cfi_offset w22, -32
+; CHECK64-NEXT:    .cfi_offset w28, -40
+; CHECK64-NEXT:    .cfi_offset vg, -48
 ; CHECK64-NEXT:    .cfi_offset w30, -56
 ; CHECK64-NEXT:    .cfi_offset w29, -64
 ; CHECK64-NEXT:    .cfi_offset b8, -136
@@ -1687,8 +1694,7 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK64-NEXT:    stp q0, q1, [sp, #64] // 32-byte Folded Spill
 ; CHECK64-NEXT:    stp q2, q3, [sp, #96] // 32-byte Folded Spill
 ; CHECK64-NEXT:    bl __arm_sme_state
-; CHECK64-NEXT:    and x21, x0, #0x1
-; CHECK64-NEXT:    .cfi_offset vg, -48
+; CHECK64-NEXT:    mov x21, x0
 ; CHECK64-NEXT:    tbz w21, #0, .LBB27_2
 ; CHECK64-NEXT:  // %bb.1:
 ; CHECK64-NEXT:    smstop sm
@@ -1700,31 +1706,27 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK64-NEXT:    smstart sm
 ; CHECK64-NEXT:  .LBB27_4:
 ; CHECK64-NEXT:    cmp w0, #0
-; CHECK64-NEXT:    .cfi_restore vg
-; CHECK64-NEXT:    cset w21, lt
-; CHECK64-NEXT:    bl __arm_sme_state
-; CHECK64-NEXT:    and x22, x0, #0x1
-; CHECK64-NEXT:    .cfi_offset vg, -48
-; CHECK64-NEXT:    tbz w22, #0, .LBB27_6
+; CHECK64-NEXT:    cset w22, mi
+; CHECK64-NEXT:    tbz w21, #0, .LBB27_6
 ; CHECK64-NEXT:  // %bb.5:
 ; CHECK64-NEXT:    smstop sm
 ; CHECK64-NEXT:  .LBB27_6:
 ; CHECK64-NEXT:    ldp q0, q1, [sp, #96] // 32-byte Folded Reload
 ; CHECK64-NEXT:    bl __getf2
-; CHECK64-NEXT:    tbz w22, #0, .LBB27_8
+; CHECK64-NEXT:    tbz w21, #0, .LBB27_8
 ; CHECK64-NEXT:  // %bb.7:
 ; CHECK64-NEXT:    smstart sm
 ; CHECK64-NEXT:  .LBB27_8:
 ; CHECK64-NEXT:    cmp w0, #0
-; CHECK64-NEXT:    cset w8, ge
-; CHECK64-NEXT:    tst w8, w21
-; CHECK64-NEXT:    csel w0, w20, w19, ne
-; CHECK64-NEXT:    .cfi_restore vg
-; CHECK64-NEXT:    ldp x20, x19, [sp, #296] // 16-byte Folded Reload
-; CHECK64-NEXT:    ldp x22, x21, [sp, #280] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldp x29, x30, [sp, #256] // 16-byte Folded Reload
+; CHECK64-NEXT:    cset w8, pl
 ; CHECK64-NEXT:    ldp d9, d8, [sp, #176] // 16-byte Folded Reload
+; CHECK64-NEXT:    tst w8, w22
+; CHECK64-NEXT:    ldp x22, x21, [sp, #288] // 16-byte Folded Reload
+; CHECK64-NEXT:    csel w0, w20, w19, ne
+; CHECK64-NEXT:    ldp x20, x19, [sp, #304] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldp d11, d10, [sp, #160] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr x28, [sp, #280] // 8-byte Folded Reload
 ; CHECK64-NEXT:    ldp d13, d12, [sp, #144] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldp d15, d14, [sp, #128] // 16-byte Folded Reload
 ; CHECK64-NEXT:    add sp, sp, #320
@@ -1733,6 +1735,8 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK64-NEXT:    .cfi_restore w20
 ; CHECK64-NEXT:    .cfi_restore w21
 ; CHECK64-NEXT:    .cfi_restore w22
+; CHECK64-NEXT:    .cfi_restore w28
+; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    .cfi_restore w30
 ; CHECK64-NEXT:    .cfi_restore w29
 ; CHECK64-NEXT:    .cfi_restore b8
@@ -1757,14 +1761,17 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK1024-NEXT:    str x29, [sp, #1088] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x30, [sp, #1096] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x9, [sp, #1104] // 8-byte Folded Spill
-; CHECK1024-NEXT:    str x22, [sp, #1112] // 8-byte Folded Spill
-; CHECK1024-NEXT:    str x21, [sp, #1120] // 8-byte Folded Spill
-; CHECK1024-NEXT:    str x20, [sp, #1128] // 8-byte Folded Spill
-; CHECK1024-NEXT:    str x19, [sp, #1136] // 8-byte Folded Spill
-; CHECK1024-NEXT:    .cfi_offset w19, -16
-; CHECK1024-NEXT:    .cfi_offset w20, -24
-; CHECK1024-NEXT:    .cfi_offset w21, -32
-; CHECK1024-NEXT:    .cfi_offset w22, -40
+; CHECK1024-NEXT:    str x28, [sp, #1112] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x22, [sp, #1120] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x21, [sp, #1128] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x20, [sp, #1136] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x19, [sp, #1144] // 8-byte Folded Spill
+; CHECK1024-NEXT:    .cfi_offset w19, -8
+; CHECK1024-NEXT:    .cfi_offset w20, -16
+; CHECK1024-NEXT:    .cfi_offset w21, -24
+; CHECK1024-NEXT:    .cfi_offset w22, -32
+; CHECK1024-NEXT:    .cfi_offset w28, -40
+; CHECK1024-NEXT:    .cfi_offset vg, -48
 ; CHECK1024-NEXT:    .cfi_offset w30, -56
 ; CHECK1024-NEXT:    .cfi_offset w29, -64
 ; CHECK1024-NEXT:    .cfi_offset b8, -1096
@@ -1784,8 +1791,7 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK1024-NEXT:    str q1, [sp, #1040] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    str q0, [sp, #1024] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    bl __arm_sme_state
-; CHECK1024-NEXT:    and x21, x0, #0x1
-; CHECK1024-NEXT:    .cfi_offset vg, -48
+; CHECK1024-NEXT:    mov x21, x0
 ; CHECK1024-NEXT:    tbz w21, #0, .LBB27_2
 ; CHECK1024-NEXT:  // %bb.1:
 ; CHECK1024-NEXT:    smstop sm
@@ -1798,35 +1804,31 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK1024-NEXT:    smstart sm
 ; CHECK1024-NEXT:  .LBB27_4:
 ; CHECK1024-NEXT:    cmp w0, #0
-; CHECK1024-NEXT:    .cfi_restore vg
-; CHECK1024-NEXT:    cset w21, lt
-; CHECK1024-NEXT:    bl __arm_sme_state
-; CHECK1024-NEXT:    and x22, x0, #0x1
-; CHECK1024-NEXT:    .cfi_offset vg, -48
-; CHECK1024-NEXT:    tbz w22, #0, .LBB27_6
+; CHECK1024-NEXT:    cset w22, mi
+; CHECK1024-NEXT:    tbz w21, #0, .LBB27_6
 ; CHECK1024-NEXT:  // %bb.5:
 ; CHECK1024-NEXT:    smstop sm
 ; CHECK1024-NEXT:  .LBB27_6:
 ; CHECK1024-NEXT:    ldr q0, [sp, #1056] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr q1, [sp, #1072] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    bl __getf2
-; CHECK1024-NEXT:    tbz w22, #0, .LBB27_8
+; CHECK1024-NEXT:    tbz w21, #0, .LBB27_8
 ; CHECK1024-NEXT:  // %bb.7:
 ; CHECK1024-NEXT:    smstart sm
 ; CHECK1024-NEXT:  .LBB27_8:
 ; CHECK1024-NEXT:    cmp w0, #0
-; CHECK1024-NEXT:    cset w8, ge
-; CHECK1024-NEXT:    tst w8, w21
+; CHECK1024-NEXT:    cset w8, pl
+; CHECK1024-NEXT:    tst w8, w22
 ; CHECK1024-NEXT:    csel w0, w20, w19, ne
-; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    add sp, sp, #1088
 ; CHECK1024-NEXT:    .cfi_def_cfa_offset 1152
 ; CHECK1024-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK1024-NEXT:    ldr x19, [sp, #1136] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x19, [sp, #1144] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK1024-NEXT:    ldr x20, [sp, #1128] // 8-byte Folded Reload
-; CHECK1024-NEXT:    ldr x21, [sp, #1120] // 8-byte Folded Reload
-; CHECK1024-NEXT:    ldr x22, [sp, #1112] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x20, [sp, #1136] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x21, [sp, #1128] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x22, [sp, #1120] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x28, [sp, #1112] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x30, [sp, #1096] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x29, [sp, #1088] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
@@ -1837,6 +1839,8 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK1024-NEXT:    .cfi_restore w20
 ; CHECK1024-NEXT:    .cfi_restore w21
 ; CHECK1024-NEXT:    .cfi_restore w22
+; CHECK1024-NEXT:    .cfi_restore w28
+; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    .cfi_restore w30
 ; CHECK1024-NEXT:    .cfi_restore w29
 ; CHECK1024-NEXT:    .cfi_restore b8
@@ -1848,6 +1852,7 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 ; CHECK1024-NEXT:    .cfi_restore b14
 ; CHECK1024-NEXT:    .cfi_restore b15
 ; CHECK1024-NEXT:    ret
+;el w0, w20, w19, ne
   %c0 = fcmp olt fp128 %v0, %v1
   %c1 = fcmp oge fp128 %v2, %v3
   %cr = and i1 %c1, %c0
@@ -1858,18 +1863,22 @@ define i32 @f128_libcall(fp128 %v0, fp128 %v1, fp128 %v2, fp128 %v3, i32 %a, i32
 define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3, i16 %P4) "aarch64_pstate_sm_compatible" {
 ; CHECK0-LABEL: svecc_call:
 ; CHECK0:       // %bb.0: // %entry
-; CHECK0-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
-; CHECK0-NEXT:    .cfi_def_cfa_offset 48
+; CHECK0-NEXT:    stp x29, x30, [sp, #-64]! // 16-byte Folded Spill
+; CHECK0-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK0-NEXT:    cntd x9
-; CHECK0-NEXT:    stp x9, x28, [sp, #16] // 16-byte Folded Spill
-; CHECK0-NEXT:    stp x27, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; CHECK0-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
+; CHECK0-NEXT:    stp x26, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK0-NEXT:    mov x29, sp
+; CHECK0-NEXT:    .cfi_def_cfa w29, 64
 ; CHECK0-NEXT:    .cfi_offset w19, -8
-; CHECK0-NEXT:    .cfi_offset w27, -16
-; CHECK0-NEXT:    .cfi_offset w28, -24
-; CHECK0-NEXT:    .cfi_offset w30, -40
-; CHECK0-NEXT:    .cfi_offset w29, -48
+; CHECK0-NEXT:    .cfi_offset w26, -16
+; CHECK0-NEXT:    .cfi_offset w27, -24
+; CHECK0-NEXT:    .cfi_offset w28, -32
+; CHECK0-NEXT:    .cfi_offset vg, -48
+; CHECK0-NEXT:    .cfi_offset w30, -56
+; CHECK0-NEXT:    .cfi_offset w29, -64
 ; CHECK0-NEXT:    addvl sp, sp, #-18
-; CHECK0-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 48 + 144 * VG
 ; CHECK0-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK0-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK0-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
@@ -1898,20 +1907,19 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK0-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK0-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK0-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d8 @ cfa - 8 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d9 @ cfa - 16 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d10 @ cfa - 24 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d11 @ cfa - 32 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d12 @ cfa - 40 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d13 @ cfa - 48 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d14 @ cfa - 56 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d15 @ cfa - 64 * VG - 48
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d8 @ cfa - 8 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d9 @ cfa - 16 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d10 @ cfa - 24 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d11 @ cfa - 32 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d12 @ cfa - 40 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d13 @ cfa - 48 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d14 @ cfa - 56 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d15 @ cfa - 64 * IncomingVG - 64
 ; CHECK0-NEXT:    mov x8, x0
+; CHECK0-NEXT:    bl __arm_sme_state
+; CHECK0-NEXT:    mov x19, x0
 ; CHECK0-NEXT:    //APP
 ; CHECK0-NEXT:    //NO_APP
-; CHECK0-NEXT:    bl __arm_sme_state
-; CHECK0-NEXT:    and x19, x0, #0x1
-; CHECK0-NEXT:    .cfi_offset vg, -32
 ; CHECK0-NEXT:    tbz w19, #0, .LBB28_2
 ; CHECK0-NEXT:  // %bb.1: // %entry
 ; CHECK0-NEXT:    smstop sm
@@ -1924,13 +1932,12 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK0-NEXT:  // %bb.3: // %entry
 ; CHECK0-NEXT:    smstart sm
 ; CHECK0-NEXT:  .LBB28_4: // %entry
-; CHECK0-NEXT:    mov w0, #22647 // =0x5877
-; CHECK0-NEXT:    movk w0, #59491, lsl #16
-; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    mov w0, #22647 // =0x5877
 ; CHECK0-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    movk w0, #59491, lsl #16
 ; CHECK0-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
@@ -1956,7 +1963,6 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK0-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK0-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK0-NEXT:    addvl sp, sp, #18
-; CHECK0-NEXT:    .cfi_def_cfa wsp, 48
 ; CHECK0-NEXT:    .cfi_restore z8
 ; CHECK0-NEXT:    .cfi_restore z9
 ; CHECK0-NEXT:    .cfi_restore z10
@@ -1965,32 +1971,39 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK0-NEXT:    .cfi_restore z13
 ; CHECK0-NEXT:    .cfi_restore z14
 ; CHECK0-NEXT:    .cfi_restore z15
-; CHECK0-NEXT:    ldp x27, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK0-NEXT:    ldr x28, [sp, #24] // 8-byte Folded Reload
-; CHECK0-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
+; CHECK0-NEXT:    .cfi_def_cfa wsp, 64
+; CHECK0-NEXT:    ldp x26, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x29, x30, [sp], #64 // 16-byte Folded Reload
 ; CHECK0-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK0-NEXT:    .cfi_restore w19
+; CHECK0-NEXT:    .cfi_restore w26
 ; CHECK0-NEXT:    .cfi_restore w27
 ; CHECK0-NEXT:    .cfi_restore w28
+; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    .cfi_restore w30
 ; CHECK0-NEXT:    .cfi_restore w29
 ; CHECK0-NEXT:    ret
 ;
 ; CHECK64-LABEL: svecc_call:
 ; CHECK64:       // %bb.0: // %entry
-; CHECK64-NEXT:    sub sp, sp, #112
-; CHECK64-NEXT:    .cfi_def_cfa_offset 112
+; CHECK64-NEXT:    sub sp, sp, #128
+; CHECK64-NEXT:    .cfi_def_cfa_offset 128
 ; CHECK64-NEXT:    cntd x9
 ; CHECK64-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
 ; CHECK64-NEXT:    stp x9, x28, [sp, #80] // 16-byte Folded Spill
-; CHECK64-NEXT:    stp x27, x19, [sp, #96] // 16-byte Folded Spill
-; CHECK64-NEXT:    .cfi_offset w19, -8
-; CHECK64-NEXT:    .cfi_offset w27, -16
-; CHECK64-NEXT:    .cfi_offset w28, -24
-; CHECK64-NEXT:    .cfi_offset w30, -40
-; CHECK64-NEXT:    .cfi_offset w29, -48
+; CHECK64-NEXT:    stp x27, x26, [sp, #96] // 16-byte Folded Spill
+; CHECK64-NEXT:    str x19, [sp, #112] // 8-byte Folded Spill
+; CHECK64-NEXT:    add x29, sp, #64
+; CHECK64-NEXT:    .cfi_def_cfa w29, 64
+; CHECK64-NEXT:    .cfi_offset w19, -16
+; CHECK64-NEXT:    .cfi_offset w26, -24
+; CHECK64-NEXT:    .cfi_offset w27, -32
+; CHECK64-NEXT:    .cfi_offset w28, -40
+; CHECK64-NEXT:    .cfi_offset vg, -48
+; CHECK64-NEXT:    .cfi_offset w30, -56
+; CHECK64-NEXT:    .cfi_offset w29, -64
 ; CHECK64-NEXT:    addvl sp, sp, #-18
-; CHECK64-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xf0, 0x00, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 112 + 144 * VG
 ; CHECK64-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK64-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK64-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
@@ -2019,22 +2032,20 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK64-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK64-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK64-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d8 @ cfa - 8 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d9 @ cfa - 16 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d10 @ cfa - 24 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d11 @ cfa - 32 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d12 @ cfa - 40 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d13 @ cfa - 48 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d14 @ cfa - 56 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d15 @ cfa - 64 * VG - 112
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d8 @ cfa - 8 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d9 @ cfa - 16 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d10 @ cfa - 24 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d11 @ cfa - 32 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d12 @ cfa - 40 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d13 @ cfa - 48 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d14 @ cfa - 56 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d15 @ cfa - 64 * IncomingVG - 128
 ; CHECK64-NEXT:    sub sp, sp, #64
-; CHECK64-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xb0, 0x01, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 176 + 144 * VG
 ; CHECK64-NEXT:    mov x8, x0
+; CHECK64-NEXT:    bl __arm_sme_state
+; CHECK64-NEXT:    mov x19, x0
 ; CHECK64-NEXT:    //APP
 ; CHECK64-NEXT:    //NO_APP
-; CHECK64-NEXT:    bl __arm_sme_state
-; CHECK64-NEXT:    and x19, x0, #0x1
-; CHECK64-NEXT:    .cfi_offset vg, -32
 ; CHECK64-NEXT:    tbz w19, #0, .LBB28_2
 ; CHECK64-NEXT:  // %bb.1: // %entry
 ; CHECK64-NEXT:    smstop sm
@@ -2049,9 +2060,7 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK64-NEXT:  .LBB28_4: // %entry
 ; CHECK64-NEXT:    mov w0, #22647 // =0x5877
 ; CHECK64-NEXT:    movk w0, #59491, lsl #16
-; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    add sp, sp, #64
-; CHECK64-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xf0, 0x00, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 112 + 144 * VG
 ; CHECK64-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
@@ -2081,7 +2090,6 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK64-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK64-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK64-NEXT:    addvl sp, sp, #18
-; CHECK64-NEXT:    .cfi_def_cfa wsp, 112
 ; CHECK64-NEXT:    .cfi_restore z8
 ; CHECK64-NEXT:    .cfi_restore z9
 ; CHECK64-NEXT:    .cfi_restore z10
@@ -2090,36 +2098,43 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK64-NEXT:    .cfi_restore z13
 ; CHECK64-NEXT:    .cfi_restore z14
 ; CHECK64-NEXT:    .cfi_restore z15
-; CHECK64-NEXT:    ldp x27, x19, [sp, #96] // 16-byte Folded Reload
-; CHECK64-NEXT:    ldr x28, [sp, #88] // 8-byte Folded Reload
+; CHECK64-NEXT:    .cfi_def_cfa wsp, 128
+; CHECK64-NEXT:    ldp x26, x19, [sp, #104] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp x28, x27, [sp, #88] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK64-NEXT:    add sp, sp, #112
+; CHECK64-NEXT:    add sp, sp, #128
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK64-NEXT:    .cfi_restore w19
+; CHECK64-NEXT:    .cfi_restore w26
 ; CHECK64-NEXT:    .cfi_restore w27
 ; CHECK64-NEXT:    .cfi_restore w28
+; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    .cfi_restore w30
 ; CHECK64-NEXT:    .cfi_restore w29
 ; CHECK64-NEXT:    ret
 ;
 ; CHECK1024-LABEL: svecc_call:
 ; CHECK1024:       // %bb.0: // %entry
-; CHECK1024-NEXT:    sub sp, sp, #1072
-; CHECK1024-NEXT:    .cfi_def_cfa_offset 1072
+; CHECK1024-NEXT:    sub sp, sp, #1088
+; CHECK1024-NEXT:    .cfi_def_cfa_offset 1088
 ; CHECK1024-NEXT:    cntd x9
 ; CHECK1024-NEXT:    str x29, [sp, #1024] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x30, [sp, #1032] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x9, [sp, #1040] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x28, [sp, #1048] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x27, [sp, #1056] // 8-byte Folded Spill
-; CHECK1024-NEXT:    str x19, [sp, #1064] // 8-byte Folded Spill
-; CHECK1024-NEXT:    .cfi_offset w19, -8
-; CHECK1024-NEXT:    .cfi_offset w27, -16
-; CHECK1024-NEXT:    .cfi_offset w28, -24
-; CHECK1024-NEXT:    .cfi_offset w30, -40
-; CHECK1024-NEXT:    .cfi_offset w29, -48
+; CHECK1024-NEXT:    str x26, [sp, #1064] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x19, [sp, #1072] // 8-byte Folded Spill
+; CHECK1024-NEXT:    add x29, sp, #1024
+; CHECK1024-NEXT:    .cfi_def_cfa w29, 64
+; CHECK1024-NEXT:    .cfi_offset w19, -16
+; CHECK1024-NEXT:    .cfi_offset w26, -24
+; CHECK1024-NEXT:    .cfi_offset w27, -32
+; CHECK1024-NEXT:    .cfi_offset w28, -40
+; CHECK1024-NEXT:    .cfi_offset vg, -48
+; CHECK1024-NEXT:    .cfi_offset w30, -56
+; CHECK1024-NEXT:    .cfi_offset w29, -64
 ; CHECK1024-NEXT:    addvl sp, sp, #-18
-; CHECK1024-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xb0, 0x08, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 1072 + 144 * VG
 ; CHECK1024-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK1024-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK1024-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
@@ -2148,22 +2163,20 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK1024-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d8 @ cfa - 8 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d9 @ cfa - 16 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d10 @ cfa - 24 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d11 @ cfa - 32 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d12 @ cfa - 40 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d13 @ cfa - 48 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d14 @ cfa - 56 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d15 @ cfa - 64 * VG - 1072
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d8 @ cfa - 8 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d9 @ cfa - 16 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d10 @ cfa - 24 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d11 @ cfa - 32 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d12 @ cfa - 40 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d13 @ cfa - 48 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d14 @ cfa - 56 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d15 @ cfa - 64 * IncomingVG - 1088
 ; CHECK1024-NEXT:    sub sp, sp, #1024
-; CHECK1024-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xb0, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 2096 + 144 * VG
 ; CHECK1024-NEXT:    mov x8, x0
+; CHECK1024-NEXT:    bl __arm_sme_state
+; CHECK1024-NEXT:    mov x19, x0
 ; CHECK1024-NEXT:    //APP
 ; CHECK1024-NEXT:    //NO_APP
-; CHECK1024-NEXT:    bl __arm_sme_state
-; CHECK1024-NEXT:    and x19, x0, #0x1
-; CHECK1024-NEXT:    .cfi_offset vg, -32
 ; CHECK1024-NEXT:    tbz w19, #0, .LBB28_2
 ; CHECK1024-NEXT:  // %bb.1: // %entry
 ; CHECK1024-NEXT:    smstop sm
@@ -2178,9 +2191,7 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK1024-NEXT:  .LBB28_4: // %entry
 ; CHECK1024-NEXT:    mov w0, #22647 // =0x5877
 ; CHECK1024-NEXT:    movk w0, #59491, lsl #16
-; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    add sp, sp, #1024
-; CHECK1024-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xb0, 0x08, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 1072 + 144 * VG
 ; CHECK1024-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
@@ -2210,7 +2221,6 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK1024-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK1024-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK1024-NEXT:    addvl sp, sp, #18
-; CHECK1024-NEXT:    .cfi_def_cfa wsp, 1072
 ; CHECK1024-NEXT:    .cfi_restore z8
 ; CHECK1024-NEXT:    .cfi_restore z9
 ; CHECK1024-NEXT:    .cfi_restore z10
@@ -2219,16 +2229,20 @@ define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3,
 ; CHECK1024-NEXT:    .cfi_restore z13
 ; CHECK1024-NEXT:    .cfi_restore z14
 ; CHECK1024-NEXT:    .cfi_restore z15
-; CHECK1024-NEXT:    ldr x19, [sp, #1064] // 8-byte Folded Reload
+; CHECK1024-NEXT:    .cfi_def_cfa wsp, 1088
+; CHECK1024-NEXT:    ldr x19, [sp, #1072] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x26, [sp, #1064] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x27, [sp, #1056] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x28, [sp, #1048] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x30, [sp, #1032] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x29, [sp, #1024] // 8-byte Folded Reload
-; CHECK1024-NEXT:    add sp, sp, #1072
+; CHECK1024-NEXT:    add sp, sp, #1088
 ; CHECK1024-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK1024-NEXT:    .cfi_restore w19
+; CHECK1024-NEXT:    .cfi_restore w26
 ; CHECK1024-NEXT:    .cfi_restore w27
 ; CHECK1024-NEXT:    .cfi_restore w28
+; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    .cfi_restore w30
 ; CHECK1024-NEXT:    .cfi_restore w29
 ; CHECK1024-NEXT:    ret
@@ -2241,18 +2255,22 @@ entry:
 define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3, i16 %P4) "aarch64_pstate_sm_compatible" {
 ; CHECK0-LABEL: svecc_alloca_call:
 ; CHECK0:       // %bb.0: // %entry
-; CHECK0-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
-; CHECK0-NEXT:    .cfi_def_cfa_offset 48
+; CHECK0-NEXT:    stp x29, x30, [sp, #-64]! // 16-byte Folded Spill
+; CHECK0-NEXT:    .cfi_def_cfa_offset 64
 ; CHECK0-NEXT:    cntd x9
-; CHECK0-NEXT:    stp x9, x28, [sp, #16] // 16-byte Folded Spill
-; CHECK0-NEXT:    stp x27, x19, [sp, #32] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; CHECK0-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
+; CHECK0-NEXT:    stp x26, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK0-NEXT:    mov x29, sp
+; CHECK0-NEXT:    .cfi_def_cfa w29, 64
 ; CHECK0-NEXT:    .cfi_offset w19, -8
-; CHECK0-NEXT:    .cfi_offset w27, -16
-; CHECK0-NEXT:    .cfi_offset w28, -24
-; CHECK0-NEXT:    .cfi_offset w30, -40
-; CHECK0-NEXT:    .cfi_offset w29, -48
+; CHECK0-NEXT:    .cfi_offset w26, -16
+; CHECK0-NEXT:    .cfi_offset w27, -24
+; CHECK0-NEXT:    .cfi_offset w28, -32
+; CHECK0-NEXT:    .cfi_offset vg, -48
+; CHECK0-NEXT:    .cfi_offset w30, -56
+; CHECK0-NEXT:    .cfi_offset w29, -64
 ; CHECK0-NEXT:    addvl sp, sp, #-18
-; CHECK0-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 48 + 144 * VG
 ; CHECK0-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK0-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK0-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
@@ -2281,21 +2299,19 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK0-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK0-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK0-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d8 @ cfa - 8 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d9 @ cfa - 16 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d10 @ cfa - 24 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d11 @ cfa - 32 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d12 @ cfa - 40 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d13 @ cfa - 48 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d14 @ cfa - 56 * VG - 48
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d15 @ cfa - 64 * VG - 48
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d8 @ cfa - 8 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d9 @ cfa - 16 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d10 @ cfa - 24 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d11 @ cfa - 32 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d12 @ cfa - 40 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d13 @ cfa - 48 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d14 @ cfa - 56 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d15 @ cfa - 64 * IncomingVG - 64
 ; CHECK0-NEXT:    sub sp, sp, #48
-; CHECK0-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xe0, 0x00, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 96 + 144 * VG
+; CHECK0-NEXT:    bl __arm_sme_state
+; CHECK0-NEXT:    mov x19, x0
 ; CHECK0-NEXT:    //APP
 ; CHECK0-NEXT:    //NO_APP
-; CHECK0-NEXT:    bl __arm_sme_state
-; CHECK0-NEXT:    and x19, x0, #0x1
-; CHECK0-NEXT:    .cfi_offset vg, -32
 ; CHECK0-NEXT:    tbz w19, #0, .LBB29_2
 ; CHECK0-NEXT:  // %bb.1: // %entry
 ; CHECK0-NEXT:    smstop sm
@@ -2310,9 +2326,7 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK0-NEXT:  .LBB29_4: // %entry
 ; CHECK0-NEXT:    mov w0, #22647 // =0x5877
 ; CHECK0-NEXT:    movk w0, #59491, lsl #16
-; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    add sp, sp, #48
-; CHECK0-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 48 + 144 * VG
 ; CHECK0-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
@@ -2342,7 +2356,6 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK0-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK0-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK0-NEXT:    addvl sp, sp, #18
-; CHECK0-NEXT:    .cfi_def_cfa wsp, 48
 ; CHECK0-NEXT:    .cfi_restore z8
 ; CHECK0-NEXT:    .cfi_restore z9
 ; CHECK0-NEXT:    .cfi_restore z10
@@ -2351,32 +2364,39 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK0-NEXT:    .cfi_restore z13
 ; CHECK0-NEXT:    .cfi_restore z14
 ; CHECK0-NEXT:    .cfi_restore z15
-; CHECK0-NEXT:    ldp x27, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK0-NEXT:    ldr x28, [sp, #24] // 8-byte Folded Reload
-; CHECK0-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
+; CHECK0-NEXT:    .cfi_def_cfa wsp, 64
+; CHECK0-NEXT:    ldp x26, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x29, x30, [sp], #64 // 16-byte Folded Reload
 ; CHECK0-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK0-NEXT:    .cfi_restore w19
+; CHECK0-NEXT:    .cfi_restore w26
 ; CHECK0-NEXT:    .cfi_restore w27
 ; CHECK0-NEXT:    .cfi_restore w28
+; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    .cfi_restore w30
 ; CHECK0-NEXT:    .cfi_restore w29
 ; CHECK0-NEXT:    ret
 ;
 ; CHECK64-LABEL: svecc_alloca_call:
 ; CHECK64:       // %bb.0: // %entry
-; CHECK64-NEXT:    sub sp, sp, #112
-; CHECK64-NEXT:    .cfi_def_cfa_offset 112
+; CHECK64-NEXT:    sub sp, sp, #128
+; CHECK64-NEXT:    .cfi_def_cfa_offset 128
 ; CHECK64-NEXT:    cntd x9
 ; CHECK64-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
 ; CHECK64-NEXT:    stp x9, x28, [sp, #80] // 16-byte Folded Spill
-; CHECK64-NEXT:    stp x27, x19, [sp, #96] // 16-byte Folded Spill
-; CHECK64-NEXT:    .cfi_offset w19, -8
-; CHECK64-NEXT:    .cfi_offset w27, -16
-; CHECK64-NEXT:    .cfi_offset w28, -24
-; CHECK64-NEXT:    .cfi_offset w30, -40
-; CHECK64-NEXT:    .cfi_offset w29, -48
+; CHECK64-NEXT:    stp x27, x26, [sp, #96] // 16-byte Folded Spill
+; CHECK64-NEXT:    str x19, [sp, #112] // 8-byte Folded Spill
+; CHECK64-NEXT:    add x29, sp, #64
+; CHECK64-NEXT:    .cfi_def_cfa w29, 64
+; CHECK64-NEXT:    .cfi_offset w19, -16
+; CHECK64-NEXT:    .cfi_offset w26, -24
+; CHECK64-NEXT:    .cfi_offset w27, -32
+; CHECK64-NEXT:    .cfi_offset w28, -40
+; CHECK64-NEXT:    .cfi_offset vg, -48
+; CHECK64-NEXT:    .cfi_offset w30, -56
+; CHECK64-NEXT:    .cfi_offset w29, -64
 ; CHECK64-NEXT:    addvl sp, sp, #-18
-; CHECK64-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xf0, 0x00, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 112 + 144 * VG
 ; CHECK64-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK64-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK64-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
@@ -2405,21 +2425,19 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK64-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK64-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK64-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d8 @ cfa - 8 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d9 @ cfa - 16 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d10 @ cfa - 24 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d11 @ cfa - 32 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d12 @ cfa - 40 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d13 @ cfa - 48 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d14 @ cfa - 56 * VG - 112
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x90, 0x7f, 0x22 // $d15 @ cfa - 64 * VG - 112
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d8 @ cfa - 8 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d9 @ cfa - 16 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d10 @ cfa - 24 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d11 @ cfa - 32 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d12 @ cfa - 40 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d13 @ cfa - 48 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d14 @ cfa - 56 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d15 @ cfa - 64 * IncomingVG - 128
 ; CHECK64-NEXT:    sub sp, sp, #112
-; CHECK64-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xe0, 0x01, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 224 + 144 * VG
+; CHECK64-NEXT:    bl __arm_sme_state
+; CHECK64-NEXT:    mov x19, x0
 ; CHECK64-NEXT:    //APP
 ; CHECK64-NEXT:    //NO_APP
-; CHECK64-NEXT:    bl __arm_sme_state
-; CHECK64-NEXT:    and x19, x0, #0x1
-; CHECK64-NEXT:    .cfi_offset vg, -32
 ; CHECK64-NEXT:    tbz w19, #0, .LBB29_2
 ; CHECK64-NEXT:  // %bb.1: // %entry
 ; CHECK64-NEXT:    smstop sm
@@ -2434,9 +2452,7 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK64-NEXT:  .LBB29_4: // %entry
 ; CHECK64-NEXT:    mov w0, #22647 // =0x5877
 ; CHECK64-NEXT:    movk w0, #59491, lsl #16
-; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    add sp, sp, #112
-; CHECK64-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xf0, 0x00, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 112 + 144 * VG
 ; CHECK64-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
@@ -2466,7 +2482,6 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK64-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK64-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK64-NEXT:    addvl sp, sp, #18
-; CHECK64-NEXT:    .cfi_def_cfa wsp, 112
 ; CHECK64-NEXT:    .cfi_restore z8
 ; CHECK64-NEXT:    .cfi_restore z9
 ; CHECK64-NEXT:    .cfi_restore z10
@@ -2475,36 +2490,43 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK64-NEXT:    .cfi_restore z13
 ; CHECK64-NEXT:    .cfi_restore z14
 ; CHECK64-NEXT:    .cfi_restore z15
-; CHECK64-NEXT:    ldp x27, x19, [sp, #96] // 16-byte Folded Reload
-; CHECK64-NEXT:    ldr x28, [sp, #88] // 8-byte Folded Reload
+; CHECK64-NEXT:    .cfi_def_cfa wsp, 128
+; CHECK64-NEXT:    ldp x26, x19, [sp, #104] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp x28, x27, [sp, #88] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
-; CHECK64-NEXT:    add sp, sp, #112
+; CHECK64-NEXT:    add sp, sp, #128
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK64-NEXT:    .cfi_restore w19
+; CHECK64-NEXT:    .cfi_restore w26
 ; CHECK64-NEXT:    .cfi_restore w27
 ; CHECK64-NEXT:    .cfi_restore w28
+; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    .cfi_restore w30
 ; CHECK64-NEXT:    .cfi_restore w29
 ; CHECK64-NEXT:    ret
 ;
 ; CHECK1024-LABEL: svecc_alloca_call:
 ; CHECK1024:       // %bb.0: // %entry
-; CHECK1024-NEXT:    sub sp, sp, #1072
-; CHECK1024-NEXT:    .cfi_def_cfa_offset 1072
+; CHECK1024-NEXT:    sub sp, sp, #1088
+; CHECK1024-NEXT:    .cfi_def_cfa_offset 1088
 ; CHECK1024-NEXT:    cntd x9
 ; CHECK1024-NEXT:    str x29, [sp, #1024] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x30, [sp, #1032] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x9, [sp, #1040] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x28, [sp, #1048] // 8-byte Folded Spill
 ; CHECK1024-NEXT:    str x27, [sp, #1056] // 8-byte Folded Spill
-; CHECK1024-NEXT:    str x19, [sp, #1064] // 8-byte Folded Spill
-; CHECK1024-NEXT:    .cfi_offset w19, -8
-; CHECK1024-NEXT:    .cfi_offset w27, -16
-; CHECK1024-NEXT:    .cfi_offset w28, -24
-; CHECK1024-NEXT:    .cfi_offset w30, -40
-; CHECK1024-NEXT:    .cfi_offset w29, -48
+; CHECK1024-NEXT:    str x26, [sp, #1064] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x19, [sp, #1072] // 8-byte Folded Spill
+; CHECK1024-NEXT:    add x29, sp, #1024
+; CHECK1024-NEXT:    .cfi_def_cfa w29, 64
+; CHECK1024-NEXT:    .cfi_offset w19, -16
+; CHECK1024-NEXT:    .cfi_offset w26, -24
+; CHECK1024-NEXT:    .cfi_offset w27, -32
+; CHECK1024-NEXT:    .cfi_offset w28, -40
+; CHECK1024-NEXT:    .cfi_offset vg, -48
+; CHECK1024-NEXT:    .cfi_offset w30, -56
+; CHECK1024-NEXT:    .cfi_offset w29, -64
 ; CHECK1024-NEXT:    addvl sp, sp, #-18
-; CHECK1024-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xb0, 0x08, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 1072 + 144 * VG
 ; CHECK1024-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
 ; CHECK1024-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK1024-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
@@ -2533,21 +2555,19 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK1024-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d8 @ cfa - 8 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d9 @ cfa - 16 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d10 @ cfa - 24 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d11 @ cfa - 32 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d12 @ cfa - 40 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d13 @ cfa - 48 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d14 @ cfa - 56 * VG - 1072
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0xd0, 0x77, 0x22 // $d15 @ cfa - 64 * VG - 1072
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d8 @ cfa - 8 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d9 @ cfa - 16 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d10 @ cfa - 24 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d11 @ cfa - 32 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d12 @ cfa - 40 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d13 @ cfa - 48 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d14 @ cfa - 56 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d15 @ cfa - 64 * IncomingVG - 1088
 ; CHECK1024-NEXT:    sub sp, sp, #1072
-; CHECK1024-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xe0, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 2144 + 144 * VG
+; CHECK1024-NEXT:    bl __arm_sme_state
+; CHECK1024-NEXT:    mov x19, x0
 ; CHECK1024-NEXT:    //APP
 ; CHECK1024-NEXT:    //NO_APP
-; CHECK1024-NEXT:    bl __arm_sme_state
-; CHECK1024-NEXT:    and x19, x0, #0x1
-; CHECK1024-NEXT:    .cfi_offset vg, -32
 ; CHECK1024-NEXT:    tbz w19, #0, .LBB29_2
 ; CHECK1024-NEXT:  // %bb.1: // %entry
 ; CHECK1024-NEXT:    smstop sm
@@ -2562,9 +2582,7 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK1024-NEXT:  .LBB29_4: // %entry
 ; CHECK1024-NEXT:    mov w0, #22647 // =0x5877
 ; CHECK1024-NEXT:    movk w0, #59491, lsl #16
-; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    add sp, sp, #1072
-; CHECK1024-NEXT:    .cfi_escape 0x0f, 0x0b, 0x8f, 0xb0, 0x08, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 1072 + 144 * VG
 ; CHECK1024-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
@@ -2594,7 +2612,6 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK1024-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
 ; CHECK1024-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
 ; CHECK1024-NEXT:    addvl sp, sp, #18
-; CHECK1024-NEXT:    .cfi_def_cfa wsp, 1072
 ; CHECK1024-NEXT:    .cfi_restore z8
 ; CHECK1024-NEXT:    .cfi_restore z9
 ; CHECK1024-NEXT:    .cfi_restore z10
@@ -2603,19 +2620,24 @@ define i32 @svecc_alloca_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8
 ; CHECK1024-NEXT:    .cfi_restore z13
 ; CHECK1024-NEXT:    .cfi_restore z14
 ; CHECK1024-NEXT:    .cfi_restore z15
-; CHECK1024-NEXT:    ldr x19, [sp, #1064] // 8-byte Folded Reload
+; CHECK1024-NEXT:    .cfi_def_cfa wsp, 1088
+; CHECK1024-NEXT:    ldr x19, [sp, #1072] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x26, [sp, #1064] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x27, [sp, #1056] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x28, [sp, #1048] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x30, [sp, #1032] // 8-byte Folded Reload
 ; CHECK1024-NEXT:    ldr x29, [sp, #1024] // 8-byte Folded Reload
-; CHECK1024-NEXT:    add sp, sp, #1072
+; CHECK1024-NEXT:    add sp, sp, #1088
 ; CHECK1024-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK1024-NEXT:    .cfi_restore w19
+; CHECK1024-NEXT:    .cfi_restore w26
 ; CHECK1024-NEXT:    .cfi_restore w27
 ; CHECK1024-NEXT:    .cfi_restore w28
+; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    .cfi_restore w30
 ; CHECK1024-NEXT:    .cfi_restore w29
 ; CHECK1024-NEXT:    ret
+
 entry:
   tail call void asm sideeffect "", "~{x0},~{x28},~{x27},~{x3}"() #2
   %0 = alloca [37 x i8], align 16
@@ -2816,6 +2838,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK0-NEXT:    .cfi_def_cfa w29, 48
 ; CHECK0-NEXT:    .cfi_offset w19, -8
 ; CHECK0-NEXT:    .cfi_offset w20, -16
+; CHECK0-NEXT:    .cfi_offset vg, -32
 ; CHECK0-NEXT:    .cfi_offset w30, -40
 ; CHECK0-NEXT:    .cfi_offset w29, -48
 ; CHECK0-NEXT:    .cfi_offset b8, -56
@@ -2838,11 +2861,9 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK0-NEXT:    stur wzr, [x29, #-68]
 ; CHECK0-NEXT:    sturh w8, [x29, #-72]
 ; CHECK0-NEXT:    msr TPIDR2_EL0, x9
-; CHECK0-NEXT:    .cfi_offset vg, -32
 ; CHECK0-NEXT:    smstop sm
 ; CHECK0-NEXT:    bl other
 ; CHECK0-NEXT:    smstart sm
-; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    smstart za
 ; CHECK0-NEXT:    mrs x8, TPIDR2_EL0
 ; CHECK0-NEXT:    sub x0, x29, #80
@@ -2863,6 +2884,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK0-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK0-NEXT:    .cfi_restore w19
 ; CHECK0-NEXT:    .cfi_restore w20
+; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    .cfi_restore w30
 ; CHECK0-NEXT:    .cfi_restore w29
 ; CHECK0-NEXT:    .cfi_restore b8
@@ -2890,6 +2912,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK64-NEXT:    .cfi_def_cfa w29, 48
 ; CHECK64-NEXT:    .cfi_offset w19, -16
 ; CHECK64-NEXT:    .cfi_offset w20, -24
+; CHECK64-NEXT:    .cfi_offset vg, -32
 ; CHECK64-NEXT:    .cfi_offset w30, -40
 ; CHECK64-NEXT:    .cfi_offset w29, -48
 ; CHECK64-NEXT:    .cfi_offset b8, -120
@@ -2913,11 +2936,9 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK64-NEXT:    str wzr, [x19, #12]
 ; CHECK64-NEXT:    strh w8, [x19, #8]
 ; CHECK64-NEXT:    msr TPIDR2_EL0, x9
-; CHECK64-NEXT:    .cfi_offset vg, -32
 ; CHECK64-NEXT:    smstop sm
 ; CHECK64-NEXT:    bl other
 ; CHECK64-NEXT:    smstart sm
-; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    smstart za
 ; CHECK64-NEXT:    mrs x8, TPIDR2_EL0
 ; CHECK64-NEXT:    add x0, x19, #0
@@ -2939,6 +2960,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK64-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK64-NEXT:    .cfi_restore w19
 ; CHECK64-NEXT:    .cfi_restore w20
+; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    .cfi_restore w30
 ; CHECK64-NEXT:    .cfi_restore w29
 ; CHECK64-NEXT:    .cfi_restore b8
@@ -2971,6 +2993,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK1024-NEXT:    .cfi_offset w19, -8
 ; CHECK1024-NEXT:    .cfi_offset w20, -16
 ; CHECK1024-NEXT:    .cfi_offset w28, -24
+; CHECK1024-NEXT:    .cfi_offset vg, -32
 ; CHECK1024-NEXT:    .cfi_offset w30, -40
 ; CHECK1024-NEXT:    .cfi_offset w29, -48
 ; CHECK1024-NEXT:    .cfi_offset b8, -1080
@@ -2994,11 +3017,9 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK1024-NEXT:    str wzr, [x19, #12]
 ; CHECK1024-NEXT:    strh w8, [x19, #8]
 ; CHECK1024-NEXT:    msr TPIDR2_EL0, x9
-; CHECK1024-NEXT:    .cfi_offset vg, -32
 ; CHECK1024-NEXT:    smstop sm
 ; CHECK1024-NEXT:    bl other
 ; CHECK1024-NEXT:    smstart sm
-; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    smstart za
 ; CHECK1024-NEXT:    mrs x8, TPIDR2_EL0
 ; CHECK1024-NEXT:    add x0, x19, #0
@@ -3024,6 +3045,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK1024-NEXT:    .cfi_restore w19
 ; CHECK1024-NEXT:    .cfi_restore w20
 ; CHECK1024-NEXT:    .cfi_restore w28
+; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    .cfi_restore w30
 ; CHECK1024-NEXT:    .cfi_restore w29
 ; CHECK1024-NEXT:    .cfi_restore b8
@@ -3161,6 +3183,7 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK0-NEXT:    .cfi_offset w26, -24
 ; CHECK0-NEXT:    .cfi_offset w27, -32
 ; CHECK0-NEXT:    .cfi_offset w28, -40
+; CHECK0-NEXT:    .cfi_offset vg, -48
 ; CHECK0-NEXT:    .cfi_offset w30, -56
 ; CHECK0-NEXT:    .cfi_offset w29, -64
 ; CHECK0-NEXT:    addvl sp, sp, #-18
@@ -3192,27 +3215,27 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK0-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK0-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK0-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d8 @ cfa - 8 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d9 @ cfa - 16 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d10 @ cfa - 24 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d11 @ cfa - 32 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d12 @ cfa - 40 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d13 @ cfa - 48 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d14 @ cfa - 56 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d15 @ cfa - 64 * VG - 64
-; CHECK0-NEXT:    mov w9, w0
-; CHECK0-NEXT:    mov x8, sp
-; CHECK0-NEXT:    mov w2, w1
-; CHECK0-NEXT:    add x9, x9, #15
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d8 @ cfa - 8 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d9 @ cfa - 16 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d10 @ cfa - 24 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d11 @ cfa - 32 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d12 @ cfa - 40 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d13 @ cfa - 48 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d14 @ cfa - 56 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d15 @ cfa - 64 * IncomingVG - 64
 ; CHECK0-NEXT:    mov x19, sp
-; CHECK0-NEXT:    and x9, x9, #0x1fffffff0
-; CHECK0-NEXT:    sub x8, x8, x9
+; CHECK0-NEXT:    mov w2, w1
+; CHECK0-NEXT:    mov w8, w0
+; CHECK0-NEXT:    bl __arm_sme_state
+; CHECK0-NEXT:    mov w8, w8
+; CHECK0-NEXT:    mov x9, sp
+; CHECK0-NEXT:    mov x20, x0
+; CHECK0-NEXT:    add x8, x8, #15
+; CHECK0-NEXT:    and x8, x8, #0x1fffffff0
+; CHECK0-NEXT:    sub x8, x9, x8
 ; CHECK0-NEXT:    mov sp, x8
 ; CHECK0-NEXT:    //APP
 ; CHECK0-NEXT:    //NO_APP
-; CHECK0-NEXT:    bl __arm_sme_state
-; CHECK0-NEXT:    and x20, x0, #0x1
-; CHECK0-NEXT:    .cfi_offset vg, -48
 ; CHECK0-NEXT:    tbz w20, #0, .LBB35_2
 ; CHECK0-NEXT:  // %bb.1: // %entry
 ; CHECK0-NEXT:    smstop sm
@@ -3226,7 +3249,6 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK0-NEXT:  .LBB35_4: // %entry
 ; CHECK0-NEXT:    mov w0, #22647 // =0x5877
 ; CHECK0-NEXT:    movk w0, #59491, lsl #16
-; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    addvl sp, x29, #-18
 ; CHECK0-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
@@ -3276,6 +3298,7 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK0-NEXT:    .cfi_restore w26
 ; CHECK0-NEXT:    .cfi_restore w27
 ; CHECK0-NEXT:    .cfi_restore w28
+; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    .cfi_restore w30
 ; CHECK0-NEXT:    .cfi_restore w29
 ; CHECK0-NEXT:    ret
@@ -3296,6 +3319,7 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK64-NEXT:    .cfi_offset w26, -24
 ; CHECK64-NEXT:    .cfi_offset w27, -32
 ; CHECK64-NEXT:    .cfi_offset w28, -40
+; CHECK64-NEXT:    .cfi_offset vg, -48
 ; CHECK64-NEXT:    .cfi_offset w30, -56
 ; CHECK64-NEXT:    .cfi_offset w29, -64
 ; CHECK64-NEXT:    addvl sp, sp, #-18
@@ -3327,28 +3351,28 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK64-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK64-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK64-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d8 @ cfa - 8 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d9 @ cfa - 16 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d10 @ cfa - 24 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d11 @ cfa - 32 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d12 @ cfa - 40 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d13 @ cfa - 48 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d14 @ cfa - 56 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d15 @ cfa - 64 * VG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d8 @ cfa - 8 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d9 @ cfa - 16 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d10 @ cfa - 24 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d11 @ cfa - 32 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d12 @ cfa - 40 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d13 @ cfa - 48 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d14 @ cfa - 56 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d15 @ cfa - 64 * IncomingVG - 128
 ; CHECK64-NEXT:    sub sp, sp, #64
-; CHECK64-NEXT:    mov w9, w0
-; CHECK64-NEXT:    mov x8, sp
-; CHECK64-NEXT:    mov w2, w1
-; CHECK64-NEXT:    add x9, x9, #15
 ; CHECK64-NEXT:    mov x19, sp
-; CHECK64-NEXT:    and x9, x9, #0x1fffffff0
-; CHECK64-NEXT:    sub x8, x8, x9
+; CHECK64-NEXT:    mov w2, w1
+; CHECK64-NEXT:    mov w8, w0
+; CHECK64-NEXT:    bl __arm_sme_state
+; CHECK64-NEXT:    mov w8, w8
+; CHECK64-NEXT:    mov x9, sp
+; CHECK64-NEXT:    mov x20, x0
+; CHECK64-NEXT:    add x8, x8, #15
+; CHECK64-NEXT:    and x8, x8, #0x1fffffff0
+; CHECK64-NEXT:    sub x8, x9, x8
 ; CHECK64-NEXT:    mov sp, x8
 ; CHECK64-NEXT:    //APP
 ; CHECK64-NEXT:    //NO_APP
-; CHECK64-NEXT:    bl __arm_sme_state
-; CHECK64-NEXT:    and x20, x0, #0x1
-; CHECK64-NEXT:    .cfi_offset vg, -48
 ; CHECK64-NEXT:    tbz w20, #0, .LBB35_2
 ; CHECK64-NEXT:  // %bb.1: // %entry
 ; CHECK64-NEXT:    smstop sm
@@ -3361,9 +3385,8 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK64-NEXT:    smstart sm
 ; CHECK64-NEXT:  .LBB35_4: // %entry
 ; CHECK64-NEXT:    mov w0, #22647 // =0x5877
-; CHECK64-NEXT:    movk w0, #59491, lsl #16
-; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    sub x8, x29, #64
+; CHECK64-NEXT:    movk w0, #59491, lsl #16
 ; CHECK64-NEXT:    addvl sp, x8, #-18
 ; CHECK64-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
@@ -3414,6 +3437,7 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK64-NEXT:    .cfi_restore w26
 ; CHECK64-NEXT:    .cfi_restore w27
 ; CHECK64-NEXT:    .cfi_restore w28
+; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    .cfi_restore w30
 ; CHECK64-NEXT:    .cfi_restore w29
 ; CHECK64-NEXT:    ret
@@ -3438,6 +3462,7 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK1024-NEXT:    .cfi_offset w26, -24
 ; CHECK1024-NEXT:    .cfi_offset w27, -32
 ; CHECK1024-NEXT:    .cfi_offset w28, -40
+; CHECK1024-NEXT:    .cfi_offset vg, -48
 ; CHECK1024-NEXT:    .cfi_offset w30, -56
 ; CHECK1024-NEXT:    .cfi_offset w29, -64
 ; CHECK1024-NEXT:    addvl sp, sp, #-18
@@ -3469,28 +3494,28 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK1024-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d8 @ cfa - 8 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d9 @ cfa - 16 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d10 @ cfa - 24 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d11 @ cfa - 32 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d12 @ cfa - 40 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d13 @ cfa - 48 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d14 @ cfa - 56 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d15 @ cfa - 64 * VG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d8 @ cfa - 8 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d9 @ cfa - 16 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d10 @ cfa - 24 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d11 @ cfa - 32 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d12 @ cfa - 40 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d13 @ cfa - 48 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d14 @ cfa - 56 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d15 @ cfa - 64 * IncomingVG - 1088
 ; CHECK1024-NEXT:    sub sp, sp, #1024
-; CHECK1024-NEXT:    mov w9, w0
-; CHECK1024-NEXT:    mov x8, sp
-; CHECK1024-NEXT:    mov w2, w1
-; CHECK1024-NEXT:    add x9, x9, #15
 ; CHECK1024-NEXT:    mov x19, sp
-; CHECK1024-NEXT:    and x9, x9, #0x1fffffff0
-; CHECK1024-NEXT:    sub x8, x8, x9
+; CHECK1024-NEXT:    mov w2, w1
+; CHECK1024-NEXT:    mov w8, w0
+; CHECK1024-NEXT:    bl __arm_sme_state
+; CHECK1024-NEXT:    mov w8, w8
+; CHECK1024-NEXT:    mov x9, sp
+; CHECK1024-NEXT:    mov x20, x0
+; CHECK1024-NEXT:    add x8, x8, #15
+; CHECK1024-NEXT:    and x8, x8, #0x1fffffff0
+; CHECK1024-NEXT:    sub x8, x9, x8
 ; CHECK1024-NEXT:    mov sp, x8
 ; CHECK1024-NEXT:    //APP
 ; CHECK1024-NEXT:    //NO_APP
-; CHECK1024-NEXT:    bl __arm_sme_state
-; CHECK1024-NEXT:    and x20, x0, #0x1
-; CHECK1024-NEXT:    .cfi_offset vg, -48
 ; CHECK1024-NEXT:    tbz w20, #0, .LBB35_2
 ; CHECK1024-NEXT:  // %bb.1: // %entry
 ; CHECK1024-NEXT:    smstop sm
@@ -3503,9 +3528,8 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK1024-NEXT:    smstart sm
 ; CHECK1024-NEXT:  .LBB35_4: // %entry
 ; CHECK1024-NEXT:    mov w0, #22647 // =0x5877
-; CHECK1024-NEXT:    movk w0, #59491, lsl #16
-; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    sub x8, x29, #1024
+; CHECK1024-NEXT:    movk w0, #59491, lsl #16
 ; CHECK1024-NEXT:    addvl sp, x8, #-18
 ; CHECK1024-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
@@ -3559,6 +3583,7 @@ define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x
 ; CHECK1024-NEXT:    .cfi_restore w26
 ; CHECK1024-NEXT:    .cfi_restore w27
 ; CHECK1024-NEXT:    .cfi_restore w28
+; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    .cfi_restore w30
 ; CHECK1024-NEXT:    .cfi_restore w29
 ; CHECK1024-NEXT:    ret
@@ -3585,6 +3610,7 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK0-NEXT:    .cfi_offset w26, -16
 ; CHECK0-NEXT:    .cfi_offset w27, -24
 ; CHECK0-NEXT:    .cfi_offset w28, -32
+; CHECK0-NEXT:    .cfi_offset vg, -48
 ; CHECK0-NEXT:    .cfi_offset w30, -56
 ; CHECK0-NEXT:    .cfi_offset w29, -64
 ; CHECK0-NEXT:    addvl sp, sp, #-18
@@ -3616,22 +3642,21 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK0-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK0-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK0-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d8 @ cfa - 8 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d9 @ cfa - 16 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d10 @ cfa - 24 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d11 @ cfa - 32 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d12 @ cfa - 40 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d13 @ cfa - 48 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d14 @ cfa - 56 * VG - 64
-; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d15 @ cfa - 64 * VG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d8 @ cfa - 8 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d9 @ cfa - 16 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d10 @ cfa - 24 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d11 @ cfa - 32 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d12 @ cfa - 40 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d13 @ cfa - 48 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d14 @ cfa - 56 * IncomingVG - 64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d15 @ cfa - 64 * IncomingVG - 64
 ; CHECK0-NEXT:    sub x9, sp, #1024
 ; CHECK0-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; CHECK0-NEXT:    mov w2, w1
+; CHECK0-NEXT:    bl __arm_sme_state
+; CHECK0-NEXT:    mov x19, x0
 ; CHECK0-NEXT:    //APP
 ; CHECK0-NEXT:    //NO_APP
-; CHECK0-NEXT:    bl __arm_sme_state
-; CHECK0-NEXT:    and x19, x0, #0x1
-; CHECK0-NEXT:    .cfi_offset vg, -48
 ; CHECK0-NEXT:    tbz w19, #0, .LBB36_2
 ; CHECK0-NEXT:  // %bb.1: // %entry
 ; CHECK0-NEXT:    smstop sm
@@ -3645,7 +3670,6 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK0-NEXT:  .LBB36_4: // %entry
 ; CHECK0-NEXT:    mov w0, #22647 // =0x5877
 ; CHECK0-NEXT:    movk w0, #59491, lsl #16
-; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    addvl sp, x29, #-18
 ; CHECK0-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK0-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
@@ -3693,6 +3717,7 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK0-NEXT:    .cfi_restore w26
 ; CHECK0-NEXT:    .cfi_restore w27
 ; CHECK0-NEXT:    .cfi_restore w28
+; CHECK0-NEXT:    .cfi_restore vg
 ; CHECK0-NEXT:    .cfi_restore w30
 ; CHECK0-NEXT:    .cfi_restore w29
 ; CHECK0-NEXT:    ret
@@ -3712,6 +3737,7 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK64-NEXT:    .cfi_offset w26, -24
 ; CHECK64-NEXT:    .cfi_offset w27, -32
 ; CHECK64-NEXT:    .cfi_offset w28, -40
+; CHECK64-NEXT:    .cfi_offset vg, -48
 ; CHECK64-NEXT:    .cfi_offset w30, -56
 ; CHECK64-NEXT:    .cfi_offset w29, -64
 ; CHECK64-NEXT:    addvl sp, sp, #-18
@@ -3743,22 +3769,21 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK64-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK64-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK64-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d8 @ cfa - 8 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d9 @ cfa - 16 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d10 @ cfa - 24 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d11 @ cfa - 32 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d12 @ cfa - 40 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d13 @ cfa - 48 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d14 @ cfa - 56 * VG - 128
-; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d15 @ cfa - 64 * VG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d8 @ cfa - 8 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d9 @ cfa - 16 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d10 @ cfa - 24 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d11 @ cfa - 32 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d12 @ cfa - 40 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d13 @ cfa - 48 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d14 @ cfa - 56 * IncomingVG - 128
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x80, 0x7f, 0x22 // $d15 @ cfa - 64 * IncomingVG - 128
 ; CHECK64-NEXT:    sub x9, sp, #1088
 ; CHECK64-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; CHECK64-NEXT:    mov w2, w1
+; CHECK64-NEXT:    bl __arm_sme_state
+; CHECK64-NEXT:    mov x19, x0
 ; CHECK64-NEXT:    //APP
 ; CHECK64-NEXT:    //NO_APP
-; CHECK64-NEXT:    bl __arm_sme_state
-; CHECK64-NEXT:    and x19, x0, #0x1
-; CHECK64-NEXT:    .cfi_offset vg, -48
 ; CHECK64-NEXT:    tbz w19, #0, .LBB36_2
 ; CHECK64-NEXT:  // %bb.1: // %entry
 ; CHECK64-NEXT:    smstop sm
@@ -3771,9 +3796,8 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK64-NEXT:    smstart sm
 ; CHECK64-NEXT:  .LBB36_4: // %entry
 ; CHECK64-NEXT:    mov w0, #22647 // =0x5877
-; CHECK64-NEXT:    movk w0, #59491, lsl #16
-; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    sub x8, x29, #64
+; CHECK64-NEXT:    movk w0, #59491, lsl #16
 ; CHECK64-NEXT:    addvl sp, x8, #-18
 ; CHECK64-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK64-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
@@ -3822,6 +3846,7 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK64-NEXT:    .cfi_restore w26
 ; CHECK64-NEXT:    .cfi_restore w27
 ; CHECK64-NEXT:    .cfi_restore w28
+; CHECK64-NEXT:    .cfi_restore vg
 ; CHECK64-NEXT:    .cfi_restore w30
 ; CHECK64-NEXT:    .cfi_restore w29
 ; CHECK64-NEXT:    ret
@@ -3844,6 +3869,7 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK1024-NEXT:    .cfi_offset w26, -24
 ; CHECK1024-NEXT:    .cfi_offset w27, -32
 ; CHECK1024-NEXT:    .cfi_offset w28, -40
+; CHECK1024-NEXT:    .cfi_offset vg, -48
 ; CHECK1024-NEXT:    .cfi_offset w30, -56
 ; CHECK1024-NEXT:    .cfi_offset w29, -64
 ; CHECK1024-NEXT:    addvl sp, sp, #-18
@@ -3875,22 +3901,21 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK1024-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
 ; CHECK1024-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d8 @ cfa - 8 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d9 @ cfa - 16 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d10 @ cfa - 24 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d11 @ cfa - 32 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d12 @ cfa - 40 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d13 @ cfa - 48 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d14 @ cfa - 56 * VG - 1088
-; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d15 @ cfa - 64 * VG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d8 @ cfa - 8 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d9 @ cfa - 16 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d10 @ cfa - 24 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d11 @ cfa - 32 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d12 @ cfa - 40 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d13 @ cfa - 48 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d14 @ cfa - 56 * IncomingVG - 1088
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0d, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0xc0, 0x77, 0x22 // $d15 @ cfa - 64 * IncomingVG - 1088
 ; CHECK1024-NEXT:    sub x9, sp, #2048
 ; CHECK1024-NEXT:    and sp, x9, #0xffffffffffffffe0
 ; CHECK1024-NEXT:    mov w2, w1
+; CHECK1024-NEXT:    bl __arm_sme_state
+; CHECK1024-NEXT:    mov x19, x0
 ; CHECK1024-NEXT:    //APP
 ; CHECK1024-NEXT:    //NO_APP
-; CHECK1024-NEXT:    bl __arm_sme_state
-; CHECK1024-NEXT:    and x19, x0, #0x1
-; CHECK1024-NEXT:    .cfi_offset vg, -48
 ; CHECK1024-NEXT:    tbz w19, #0, .LBB36_2
 ; CHECK1024-NEXT:  // %bb.1: // %entry
 ; CHECK1024-NEXT:    smstop sm
@@ -3903,9 +3928,8 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK1024-NEXT:    smstart sm
 ; CHECK1024-NEXT:  .LBB36_4: // %entry
 ; CHECK1024-NEXT:    mov w0, #22647 // =0x5877
-; CHECK1024-NEXT:    movk w0, #59491, lsl #16
-; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    sub x8, x29, #1024
+; CHECK1024-NEXT:    movk w0, #59491, lsl #16
 ; CHECK1024-NEXT:    addvl sp, x8, #-18
 ; CHECK1024-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK1024-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
@@ -3957,6 +3981,7 @@ define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i
 ; CHECK1024-NEXT:    .cfi_restore w26
 ; CHECK1024-NEXT:    .cfi_restore w27
 ; CHECK1024-NEXT:    .cfi_restore w28
+; CHECK1024-NEXT:    .cfi_restore vg
 ; CHECK1024-NEXT:    .cfi_restore w30
 ; CHECK1024-NEXT:    .cfi_restore w29
 ; CHECK1024-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/stack-probing-last-in-block.mir b/llvm/test/CodeGen/AArch64/stack-probing-last-in-block.mir
index 39dba30de469f..25184b0c0b56a 100644
--- a/llvm/test/CodeGen/AArch64/stack-probing-last-in-block.mir
+++ b/llvm/test/CodeGen/AArch64/stack-probing-last-in-block.mir
@@ -65,8 +65,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  200000
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: p, type: default, offset: 0, size: 200000, alignment: 4,
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll b/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll
index 0711f69e7ca72..df8376280318c 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-initializer-merge.ll
@@ -5,8 +5,8 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
 
 declare void @use(ptr)
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
 
 define void @OneVarNoInit() sanitize_memtag {
@@ -16,18 +16,18 @@ define void @OneVarNoInit() sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
 ; CHECK-NEXT:    [[TX:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[TX]], i64 16)
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TX]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 16)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -39,19 +39,19 @@ define void @OneVarInitConst() sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
 ; CHECK-NEXT:    [[TX:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    call void @llvm.aarch64.stgp(ptr [[TX]], i64 42, i64 0)
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TX]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 16)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   store i32 42, ptr %x, align 4
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -64,21 +64,21 @@ define void @ArrayInitConst() sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca i32, i32 16, align 16
 ; CHECK-NEXT:    [[TX:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    call void @llvm.aarch64.stgp(ptr [[TX]], i64 42, i64 0)
 ; CHECK-NEXT:    [[TX8_16:%.*]] = getelementptr i8, ptr [[TX]], i32 16
 ; CHECK-NEXT:    call void @llvm.aarch64.settag.zero(ptr [[TX8_16]], i64 48)
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TX]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 64)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, i32 16, align 4
-  call void @llvm.lifetime.start.p0(i64 64, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   store i32 42, ptr %x, align 4
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 64, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -90,7 +90,7 @@ define void @ArrayInitConst2() sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca i32, i32 16, align 16
 ; CHECK-NEXT:    [[TX:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[TX]], i32 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[TX]], i32 2
 ; CHECK-NEXT:    call void @llvm.aarch64.stgp(ptr [[TX]], i64 184683593770, i64 -1)
@@ -98,19 +98,19 @@ define void @ArrayInitConst2() sanitize_memtag {
 ; CHECK-NEXT:    call void @llvm.aarch64.settag.zero(ptr [[TX8_16]], i64 48)
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TX]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 64)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, i32 16, align 4
-  call void @llvm.lifetime.start.p0(i64 64, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   store i32 42, ptr %x, align 4
   %0 = getelementptr i32, ptr %x, i32 1
   store i32 43, ptr %0, align 4
   %1 = getelementptr i32, ptr %x, i32 2
   store i64 -1, ptr %1, align 4
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 64, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -122,23 +122,23 @@ define void @ArrayInitConstSplit() sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca i32, i32 16, align 16
 ; CHECK-NEXT:    [[TX:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[TX]], i32 1
 ; CHECK-NEXT:    call void @llvm.aarch64.stgp(ptr [[TX]], i64 -4294967296, i64 4294967295)
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[TX]], i32 16
 ; CHECK-NEXT:    call void @llvm.aarch64.settag.zero(ptr [[TMP1]], i64 48)
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TX]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 64)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, i32 16, align 4
-  call void @llvm.lifetime.start.p0(i64 64, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   %0 = getelementptr i32, ptr %x, i32 1
   store i64 -1, ptr %0, align 4
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 64, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -150,7 +150,7 @@ define void @ArrayInitConstWithHoles() sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca i32, i32 32, align 16
 ; CHECK-NEXT:    [[TX:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 128, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[TX]], i32 5
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[TX]], i32 14
 ; CHECK-NEXT:    call void @llvm.aarch64.settag.zero(ptr [[TX]], i64 16)
@@ -164,18 +164,18 @@ define void @ArrayInitConstWithHoles() sanitize_memtag {
 ; CHECK-NEXT:    call void @llvm.aarch64.settag.zero(ptr [[TX8_64]], i64 64)
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TX]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 128)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 128, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, i32 32, align 4
-  call void @llvm.lifetime.start.p0(i64 128, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   %0 = getelementptr i32, ptr %x, i32 5
   store i32 42, ptr %0, align 4
   %1 = getelementptr i32, ptr %x, i32 14
   store i32 43, ptr %1, align 4
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 128, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -187,20 +187,20 @@ define void @InitNonConst(i32 %v) sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca { i32, [12 x i8] }, align 16
 ; CHECK-NEXT:    [[X_TAG:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[V]] to i64
 ; CHECK-NEXT:    call void @llvm.aarch64.stgp(ptr [[X_TAG]], i64 [[TMP0]], i64 0)
 ; CHECK-NEXT:    call void @use(ptr nonnull [[X_TAG]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 16)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   store i32 %v, ptr %x, align 4
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -212,7 +212,7 @@ define void @InitNonConst2(i32 %v, i32 %w) sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca i32, i32 4, align 16
 ; CHECK-NEXT:    [[TX:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[V]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[TX]], i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[W]] to i64
@@ -221,17 +221,17 @@ define void @InitNonConst2(i32 %v, i32 %w) sanitize_memtag {
 ; CHECK-NEXT:    call void @llvm.aarch64.stgp(ptr [[TX]], i64 [[VW]], i64 0)
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TX]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 16)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, i32 4, align 4
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   store i32 %v, ptr %x, align 4
   %0 = getelementptr i32, ptr %x, i32 1
   store i32 %w, ptr %0, align 4
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -243,19 +243,19 @@ define void @InitVector() sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca i32, i32 4, align 16
 ; CHECK-NEXT:    [[TX:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    call void @llvm.aarch64.stgp(ptr [[TX]], i64 bitcast (<2 x i32> <i32 1, i32 2> to i64), i64 0)
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TX]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 16)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, i32 4, align 4
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   store <2 x i32> <i32 1, i32 2>, ptr %x, align 4
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -302,23 +302,23 @@ define void @InitVectorSplit() sanitize_memtag {
 ; CHECK-NEXT:    [[BASETAG:%.*]] = call ptr @llvm.aarch64.irg.sp(i64 0)
 ; CHECK-NEXT:    [[X:%.*]] = alloca i32, i32 4, align 16
 ; CHECK-NEXT:    [[TX:%.*]] = call ptr @llvm.aarch64.tagp.p0(ptr [[X]], ptr [[BASETAG]], i64 0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[TX]], i32 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 bitcast (<2 x i32> <i32 1, i32 2> to i64), 32
 ; CHECK-NEXT:    [[LSHR:%.*]] = lshr i64 bitcast (<2 x i32> <i32 1, i32 2> to i64), 32
 ; CHECK-NEXT:    call void @llvm.aarch64.stgp(ptr [[TX]], i64 [[TMP1]], i64 [[LSHR]])
 ; CHECK-NEXT:    call void @use(ptr nonnull [[TX]])
 ; CHECK-NEXT:    call void @llvm.aarch64.settag(ptr [[X]], i64 16)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %x = alloca i32, i32 4, align 4
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   %0 = getelementptr i32, ptr %x, i32 1
   store <2 x i32> <i32 1, i32 2>, ptr %0, align 4
   call void @use(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-merge-past-memcpy.mir b/llvm/test/CodeGen/AArch64/stack-tagging-merge-past-memcpy.mir
index 45f6bfe80ac2b..0fa5103232a89 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-merge-past-memcpy.mir
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-merge-past-memcpy.mir
@@ -18,15 +18,15 @@
     %C.tag = call ptr @llvm.aarch64.tagp.p0(ptr %C, ptr %basetag, i64 1)
     call void @llvm.aarch64.settag(ptr %C.tag, i64 32)
     call void @F56(ptr %C.tag)
-    call void @llvm.lifetime.start.p0(i64 32, ptr %A)
+    call void @llvm.lifetime.start.p0(ptr %A)
     call void @llvm.aarch64.settag(ptr %A.tag, i64 32)
     call void @F56(ptr %A.tag)
     call void @llvm.aarch64.settag(ptr %A, i64 32)
-    call void @llvm.lifetime.end.p0(i64 32, ptr %A)
-    call void @llvm.lifetime.start.p0(i64 32, ptr %A)
+    call void @llvm.lifetime.end.p0(ptr %A)
+    call void @llvm.lifetime.start.p0(ptr %A)
     call void @llvm.memcpy.p0.p0.i64(ptr align 4 %A, ptr align 4 @glob, i64 32, i1 false)
     call void @F78(ptr %A)
-    call void @llvm.lifetime.end.p0(i64 32, ptr %A)
+    call void @llvm.lifetime.end.p0(ptr %A)
     call void @llvm.aarch64.settag(ptr %C, i64 32)
     ret void
   }
diff --git a/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll b/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll
index aa9cccc58712d..91adf8257c992 100644
--- a/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll
+++ b/llvm/test/CodeGen/AArch64/stack-tagging-untag-placement.ll
@@ -17,17 +17,17 @@ S0:
 
 S1:
 ; CHECK-LABEL: S1:
-  call void @llvm.lifetime.start.p0(i64 48, ptr nonnull %v) #1
+  call void @llvm.lifetime.start.p0(ptr nonnull %v) #1
 ; CHECK: call void @llvm.aarch64.settag(ptr %v.tag, i64 48)
-  call void @llvm.lifetime.start.p0(i64 48, ptr nonnull %w) #1
+  call void @llvm.lifetime.start.p0(ptr nonnull %w) #1
 ; CHECK: call void @llvm.aarch64.settag(ptr %w.tag, i64 48)
   %t1 = call i32 @g1(ptr nonnull %v, ptr nonnull %w) #1
 ; CHECK: call i32 @g1
 ; CHECK-NOT: settag{{.*}}%v
 ; CHECK: call void @llvm.aarch64.settag(ptr %w, i64 48)
 ; CHECK-NOT: settag{{.*}}%v
-  call void @llvm.lifetime.end.p0(i64 48, ptr nonnull %w) #1
-; CHECK: call void @llvm.lifetime.end.p0(i64 48, ptr nonnull %w)
+  call void @llvm.lifetime.end.p0(ptr nonnull %w) #1
+; CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %w)
   %b1 = icmp eq i32 %t1, 0
   br i1 %b1, label %S2, label %S3
 ; CHECK-NOT: settag
@@ -40,7 +40,7 @@ S2:
 
 S3:
 ; CHECK-LABEL: S3:
-  call void @llvm.lifetime.end.p0(i64 48, ptr nonnull %v) #1
+  call void @llvm.lifetime.end.p0(ptr nonnull %v) #1
   tail call void @z1() #1
   br label %exit2
 ; CHECK-NOT: settag
@@ -73,9 +73,9 @@ declare void @z1() #0
 
 declare void @z2() #0
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { sanitize_memtag "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+mte,+neon,+v8.5a" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AArch64/streaming-compatible-memory-ops.ll b/llvm/test/CodeGen/AArch64/streaming-compatible-memory-ops.ll
index f1e684c86e896..7ad95429949a0 100644
--- a/llvm/test/CodeGen/AArch64/streaming-compatible-memory-ops.ll
+++ b/llvm/test/CodeGen/AArch64/streaming-compatible-memory-ops.ll
@@ -22,14 +22,13 @@ define void @se_memcpy(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {
 ; CHECK-NO-SME-ROUTINES-LABEL: se_memcpy:
 ; CHECK-NO-SME-ROUTINES:       // %bb.0: // %entry
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    cntd x9
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    mov x2, x0
-; CHECK-NO-SME-ROUTINES-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    adrp x0, :got:dst
+; CHECK-NO-SME-ROUTINES-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    adrp x1, :got:src
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NO-SME-ROUTINES-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldr x0, [x0, :got_lo12:dst]
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldr x1, [x1, :got_lo12:src]
 ; CHECK-NO-SME-ROUTINES-NEXT:    smstop sm
@@ -72,13 +71,12 @@ define void @se_memset(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {
 ; CHECK-NO-SME-ROUTINES-LABEL: se_memset:
 ; CHECK-NO-SME-ROUTINES:       // %bb.0: // %entry
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    cntd x9
+; CHECK-NO-SME-ROUTINES-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    mov x2, x0
 ; CHECK-NO-SME-ROUTINES-NEXT:    adrp x0, :got:dst
-; CHECK-NO-SME-ROUTINES-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NO-SME-ROUTINES-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldr x0, [x0, :got_lo12:dst]
 ; CHECK-NO-SME-ROUTINES-NEXT:    smstop sm
 ; CHECK-NO-SME-ROUTINES-NEXT:    mov w1, #2 // =0x2
@@ -121,14 +119,13 @@ define void @se_memmove(i64 noundef %n) "aarch64_pstate_sm_enabled" nounwind {
 ; CHECK-NO-SME-ROUTINES-LABEL: se_memmove:
 ; CHECK-NO-SME-ROUTINES:       // %bb.0: // %entry
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    cntd x9
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    mov x2, x0
-; CHECK-NO-SME-ROUTINES-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    adrp x0, :got:dst
+; CHECK-NO-SME-ROUTINES-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    adrp x1, :got:src
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
+; CHECK-NO-SME-ROUTINES-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldr x0, [x0, :got_lo12:dst]
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldr x1, [x1, :got_lo12:src]
 ; CHECK-NO-SME-ROUTINES-NEXT:    smstop sm
@@ -171,36 +168,32 @@ define void @sc_memcpy(i64 noundef %n) "aarch64_pstate_sm_compatible" nounwind {
 ;
 ; CHECK-NO-SME-ROUTINES-LABEL: sc_memcpy:
 ; CHECK-NO-SME-ROUTINES:       // %bb.0: // %entry
-; CHECK-NO-SME-ROUTINES-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    cntd x9
+; CHECK-NO-SME-ROUTINES-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    mov x2, x0
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    str x19, [sp, #80] // 8-byte Folded Spill
+; CHECK-NO-SME-ROUTINES-NEXT:    stp x30, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    bl __arm_sme_state
-; CHECK-NO-SME-ROUTINES-NEXT:    adrp x8, :got:dst
-; CHECK-NO-SME-ROUTINES-NEXT:    and x19, x0, #0x1
+; CHECK-NO-SME-ROUTINES-NEXT:    mov x19, x0
+; CHECK-NO-SME-ROUTINES-NEXT:    adrp x0, :got:dst
 ; CHECK-NO-SME-ROUTINES-NEXT:    adrp x1, :got:src
-; CHECK-NO-SME-ROUTINES-NEXT:    ldr x8, [x8, :got_lo12:dst]
+; CHECK-NO-SME-ROUTINES-NEXT:    ldr x0, [x0, :got_lo12:dst]
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldr x1, [x1, :got_lo12:src]
 ; CHECK-NO-SME-ROUTINES-NEXT:    tbz w19, #0, .LBB3_2
 ; CHECK-NO-SME-ROUTINES-NEXT:  // %bb.1: // %entry
 ; CHECK-NO-SME-ROUTINES-NEXT:    smstop sm
 ; CHECK-NO-SME-ROUTINES-NEXT:  .LBB3_2: // %entry
-; CHECK-NO-SME-ROUTINES-NEXT:    mov x0, x8
 ; CHECK-NO-SME-ROUTINES-NEXT:    bl memcpy
 ; CHECK-NO-SME-ROUTINES-NEXT:    tbz w19, #0, .LBB3_4
 ; CHECK-NO-SME-ROUTINES-NEXT:  // %bb.3: // %entry
 ; CHECK-NO-SME-ROUTINES-NEXT:    smstart sm
 ; CHECK-NO-SME-ROUTINES-NEXT:  .LBB3_4: // %entry
+; CHECK-NO-SME-ROUTINES-NEXT:    ldp x30, x19, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
-; CHECK-NO-SME-ROUTINES-NEXT:    ldr x19, [sp, #80] // 8-byte Folded Reload
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
-; CHECK-NO-SME-ROUTINES-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NO-SME-ROUTINES-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NO-SME-ROUTINES-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NO-SME-ROUTINES-NEXT:    ret
 ;
 ; CHECK-MOPS-LABEL: sc_memcpy:
@@ -221,16 +214,12 @@ entry:
 define void @sb_memcpy(i64 noundef %n) "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: sb_memcpy:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NEXT:    rdsvl x9, #1
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov x2, x0
-; CHECK-NEXT:    lsr x9, x9, #3
 ; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    adrp x0, :got:dst
 ; CHECK-NEXT:    adrp x1, :got:src
@@ -242,21 +231,17 @@ define void @sb_memcpy(i64 noundef %n) "aarch64_pstate_sm_body" nounwind {
 ; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
 ;
 ; CHECK-NO-SME-ROUTINES-LABEL: sb_memcpy:
 ; CHECK-NO-SME-ROUTINES:       // %bb.0: // %entry
-; CHECK-NO-SME-ROUTINES-NEXT:    stp d15, d14, [sp, #-96]! // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    rdsvl x9, #1
+; CHECK-NO-SME-ROUTINES-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    mov x2, x0
-; CHECK-NO-SME-ROUTINES-NEXT:    lsr x9, x9, #3
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    stp x30, x9, [sp, #64] // 16-byte Folded Spill
-; CHECK-NO-SME-ROUTINES-NEXT:    cntd x9
-; CHECK-NO-SME-ROUTINES-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
+; CHECK-NO-SME-ROUTINES-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NO-SME-ROUTINES-NEXT:    smstart sm
 ; CHECK-NO-SME-ROUTINES-NEXT:    adrp x0, :got:dst
 ; CHECK-NO-SME-ROUTINES-NEXT:    adrp x1, :got:src
@@ -268,20 +253,15 @@ define void @sb_memcpy(i64 noundef %n) "aarch64_pstate_sm_body" nounwind {
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
 ; CHECK-NO-SME-ROUTINES-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
-; CHECK-NO-SME-ROUTINES-NEXT:    ldp d15, d14, [sp], #96 // 16-byte Folded Reload
+; CHECK-NO-SME-ROUTINES-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NO-SME-ROUTINES-NEXT:    ret
 ;
 ; CHECK-MOPS-LABEL: sb_memcpy:
 ; CHECK-MOPS:       // %bb.0: // %entry
-; CHECK-MOPS-NEXT:    rdsvl x9, #1
-; CHECK-MOPS-NEXT:    lsr x9, x9, #3
-; CHECK-MOPS-NEXT:    str x9, [sp, #-80]! // 8-byte Folded Spill
-; CHECK-MOPS-NEXT:    cntd x9
-; CHECK-MOPS-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
-; CHECK-MOPS-NEXT:    str x9, [sp, #8] // 8-byte Folded Spill
-; CHECK-MOPS-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
-; CHECK-MOPS-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
-; CHECK-MOPS-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
+; CHECK-MOPS-NEXT:    stp d15, d14, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-MOPS-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-MOPS-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-MOPS-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
 ; CHECK-MOPS-NEXT:    smstart sm
 ; CHECK-MOPS-NEXT:    adrp x8, :got:src
 ; CHECK-MOPS-NEXT:    adrp x9, :got:dst
@@ -291,11 +271,10 @@ define void @sb_memcpy(i64 noundef %n) "aarch64_pstate_sm_body" nounwind {
 ; CHECK-MOPS-NEXT:    cpyfm [x9]!, [x8]!, x0!
 ; CHECK-MOPS-NEXT:    cpyfe [x9]!, [x8]!, x0!
 ; CHECK-MOPS-NEXT:    smstop sm
-; CHECK-MOPS-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-MOPS-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-MOPS-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
-; CHECK-MOPS-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-MOPS-NEXT:    add sp, sp, #80
+; CHECK-MOPS-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-MOPS-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-MOPS-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-MOPS-NEXT:    ldp d15, d14, [sp], #64 // 16-byte Folded Reload
 ; CHECK-MOPS-NEXT:    ret
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll
index 83f4f8fc57aae..0580f5e0b019a 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mattr=+sve,+bf16                        < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16
-; RUN: llc -mattr=+sve,+bf16,+sve-b16b16            < %s | FileCheck %s --check-prefixes=CHECK,B16B16
-; RUN: llc -mattr=+sme -force-streaming             < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16
-; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,B16B16
+; RUN: llc -mattr=+sve,+bf16                         < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16
+; RUN: llc -mattr=+sve,+bf16,+sve-b16b16             < %s | FileCheck %s --check-prefixes=CHECK,B16B16
+; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming  < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16
+; RUN: llc -mattr=+sme2,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,B16B16
 
 target triple = "aarch64-unknown-linux-gnu"
 
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll b/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll
index d63f7e6f3242e..120ab7cc4552e 100644
--- a/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll
@@ -171,11 +171,11 @@ define <vscale x 8 x bfloat> @fptrunc_nxv8f32_to_nxv8bf16(<vscale x 8 x float> %
 ; NOBF16-NEXT:    ptrue p0.s
 ; NOBF16-NEXT:    and z3.s, z3.s, #0x1
 ; NOBF16-NEXT:    and z4.s, z4.s, #0x1
-; NOBF16-NEXT:    fcmuo p1.s, p0/z, z1.s, z1.s
 ; NOBF16-NEXT:    add z5.s, z1.s, z2.s
 ; NOBF16-NEXT:    add z2.s, z0.s, z2.s
-; NOBF16-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; NOBF16-NEXT:    fcmuo p1.s, p0/z, z1.s, z1.s
 ; NOBF16-NEXT:    orr z1.s, z1.s, #0x400000
+; NOBF16-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
 ; NOBF16-NEXT:    orr z0.s, z0.s, #0x400000
 ; NOBF16-NEXT:    add z3.s, z3.s, z5.s
 ; NOBF16-NEXT:    add z2.s, z4.s, z2.s
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll
index 00002dd3269a2..7b438743487e1 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-extract-subvector.ll
@@ -5,6 +5,12 @@
 
 target triple = "aarch64-unknown-linux-gnu"
 
+; Note that both the vector.extract intrinsics and SK_ExtractSubvector
+; shufflevector instructions get detected as a extract_subvector ISD node in
+; SelectionDAG. We'll test both cases for the sake of completeness, even though
+; vector.extract intrinsics should get lowered into shufflevector by the time we
+; reach the backend.
+
 ; i8
 
 ; Don't use SVE for 64-bit vectors.
@@ -40,6 +46,68 @@ define void @extract_subvector_v32i8(ptr %a, ptr %b) vscale_range(2,0) #0 {
   ret void
 }
 
+define void @extract_v32i8_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v32i8_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    str q1, [x1]
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <32 x i8>, ptr %in
+  %hi = shufflevector <32 x i8> %b, <32 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  store <16 x i8> %hi, ptr %out
+  %lo = shufflevector <32 x i8> %b, <32 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <16 x i8> %lo, ptr %out2
+  ret void
+}
+
+define void @extract_v32i8_half_unaligned(ptr %in, ptr %out) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v32i8_half_unaligned:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #4
+; CHECK-NEXT:    str q0, [x1]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <32 x i8>, ptr %in
+  %d = shufflevector <32 x i8> %b, <32 x i8> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+  store <16 x i8> %d, ptr %out
+  ret void
+}
+
+define void @extract_v32i8_quarters(ptr %in, ptr %out, ptr %out2, ptr %out3, ptr %out4) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v32i8_quarters:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #24
+; CHECK-NEXT:    movprfx z3, z0
+; CHECK-NEXT:    ext z3.b, z3.b, z0.b, #8
+; CHECK-NEXT:    str d1, [x1]
+; CHECK-NEXT:    str d2, [x2]
+; CHECK-NEXT:    str d0, [x3]
+; CHECK-NEXT:    str d3, [x4]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <32 x i8>, ptr %in
+  %hilo = shufflevector <32 x i8> %b, <32 x i8> poison, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+  store <8 x i8> %hilo, ptr %out
+  %hihi = shufflevector <32 x i8> %b, <32 x i8> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  store <8 x i8> %hihi, ptr %out2
+  %lolo = shufflevector <32 x i8> %b, <32 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i8> %lolo, ptr %out3
+  %lohi = shufflevector <32 x i8> %b, <32 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <8 x i8> %lohi, ptr %out4
+  ret void
+}
+
 define void @extract_subvector_v64i8(ptr %a, ptr %b) #0 {
 ; CHECK-LABEL: extract_subvector_v64i8:
 ; CHECK:       // %bb.0:
@@ -54,6 +122,25 @@ define void @extract_subvector_v64i8(ptr %a, ptr %b) #0 {
   ret void
 }
 
+define void @extract_v64i8_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(4,4) {
+; CHECK-LABEL: extract_v64i8_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ptrue p0.b, vl32
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #32
+; CHECK-NEXT:    st1b { z1.b }, p0, [x1]
+; CHECK-NEXT:    st1b { z0.b }, p0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <64 x i8>, ptr %in
+  %hi = shufflevector <64 x i8> %b, <64 x i8> poison, <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+  store <32 x i8> %hi, ptr %out
+  %lo = shufflevector <64 x i8> %b, <64 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  store <32 x i8> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v128i8(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: extract_subvector_v128i8:
 ; CHECK:       // %bb.0:
@@ -117,6 +204,24 @@ define void @extract_subvector_v16i16(ptr %a, ptr %b) vscale_range(2,0) #0 {
   ret void
 }
 
+define void @extract_v16i16_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v16i16_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    str q1, [x1]
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <16 x i16>, ptr %in
+  %hi = shufflevector <16 x i16> %b, <16 x i16> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <8 x i16> %hi, ptr %out
+  %lo = shufflevector <16 x i16> %b, <16 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i16> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v32i16(ptr %a, ptr %b) #0 {
 ; CHECK-LABEL: extract_subvector_v32i16:
 ; CHECK:       // %bb.0:
@@ -131,6 +236,25 @@ define void @extract_subvector_v32i16(ptr %a, ptr %b) #0 {
   ret void
 }
 
+define void @extract_v32i16_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(4,4) {
+; CHECK-LABEL: extract_v32i16_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ptrue p0.h, vl16
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #32
+; CHECK-NEXT:    st1h { z1.h }, p0, [x1]
+; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <32 x i16>, ptr %in
+  %hi = shufflevector <32 x i16> %b, <32 x i16> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  store <16 x i16> %hi, ptr %out
+  %lo = shufflevector <32 x i16> %b, <32 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <16 x i16> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v64i16(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: extract_subvector_v64i16:
 ; CHECK:       // %bb.0:
@@ -195,6 +319,24 @@ define void @extract_subvector_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {
   ret void
 }
 
+define void @extract_v8i32_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v8i32_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    str q1, [x1]
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <8 x i32>, ptr %in
+  %hi = shufflevector <8 x i32> %b, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  store <4 x i32> %hi, ptr %out
+  %lo = shufflevector <8 x i32> %b, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x i32> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v16i32(ptr %a, ptr %b) #0 {
 ; CHECK-LABEL: extract_subvector_v16i32:
 ; CHECK:       // %bb.0:
@@ -209,6 +351,25 @@ define void @extract_subvector_v16i32(ptr %a, ptr %b) #0 {
   ret void
 }
 
+define void @extract_v16i32_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(4,4) {
+; CHECK-LABEL: extract_v16i32_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ptrue p0.s, vl8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #32
+; CHECK-NEXT:    st1w { z1.s }, p0, [x1]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <16 x i32>, ptr %in
+  %hi = shufflevector <16 x i32> %b, <16 x i32> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <8 x i32> %hi, ptr %out
+  %lo = shufflevector <16 x i32> %b, <16 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x i32> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v32i32(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: extract_subvector_v32i32:
 ; CHECK:       // %bb.0:
@@ -262,6 +423,24 @@ define void @extract_subvector_v4i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
   ret void
 }
 
+define void @extract_v4i64_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v4i64_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    str q1, [x1]
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <4 x i64>, ptr %in
+  %hi = shufflevector <4 x i64> %b, <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+  store <2 x i64> %hi, ptr %out
+  %lo = shufflevector <4 x i64> %b, <4 x i64> poison, <2 x i32> <i32 0, i32 1>
+  store <2 x i64> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v8i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
 ; CHECK-LABEL: extract_subvector_v8i64:
 ; CHECK:       // %bb.0:
@@ -276,6 +455,25 @@ define void @extract_subvector_v8i64(ptr %a, ptr %b) vscale_range(2,0) #0 {
   ret void
 }
 
+define void @extract_v8i64_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(4,4) {
+; CHECK-LABEL: extract_v8i64_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #32
+; CHECK-NEXT:    st1d { z1.d }, p0, [x1]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <8 x i64>, ptr %in
+  %hi = shufflevector <8 x i64> %b, <8 x i64> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  store <4 x i64> %hi, ptr %out
+  %lo = shufflevector <8 x i64> %b, <8 x i64> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x i64> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v16i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-LABEL: extract_subvector_v16i64:
 ; VBITS_GE_256:       // %bb.0:
@@ -352,6 +550,24 @@ define void @extract_subvector_v16f16(ptr %a, ptr %b) vscale_range(2,0) #0 {
   ret void
 }
 
+define void @extract_v16half_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v16half_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    str q1, [x1]
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <16 x half>, ptr %in
+  %hi = shufflevector <16 x half> %b, <16 x half> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <8 x half> %hi, ptr %out
+  %lo = shufflevector <16 x half> %b, <16 x half> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x half> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v32f16(ptr %a, ptr %b) #0 {
 ; CHECK-LABEL: extract_subvector_v32f16:
 ; CHECK:       // %bb.0:
@@ -366,6 +582,25 @@ define void @extract_subvector_v32f16(ptr %a, ptr %b) #0 {
   ret void
 }
 
+define void @extract_v32half_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(4,4) {
+; CHECK-LABEL: extract_v32half_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ptrue p0.h, vl16
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #32
+; CHECK-NEXT:    st1h { z1.h }, p0, [x1]
+; CHECK-NEXT:    st1h { z0.h }, p0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <32 x half>, ptr %in
+  %hi = shufflevector <32 x half> %b, <32 x half> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  store <16 x half> %hi, ptr %out
+  %lo = shufflevector <32 x half> %b, <32 x half> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <16 x half> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v64f16(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: extract_subvector_v64f16:
 ; CHECK:       // %bb.0:
@@ -430,6 +665,24 @@ define void @extract_subvector_v8f32(ptr %a, ptr %b) vscale_range(2,0) #0 {
   ret void
 }
 
+define void @extract_v8float_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v8float_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    str q1, [x1]
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <8 x float>, ptr %in
+  %hi = shufflevector <8 x float> %b, <8 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  store <4 x float> %hi, ptr %out
+  %lo = shufflevector <8 x float> %b, <8 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x float> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v16f32(ptr %a, ptr %b) #0 {
 ; CHECK-LABEL: extract_subvector_v16f32:
 ; CHECK:       // %bb.0:
@@ -444,6 +697,25 @@ define void @extract_subvector_v16f32(ptr %a, ptr %b) #0 {
   ret void
 }
 
+define void @extract_v16float_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(4,4) {
+; CHECK-LABEL: extract_v16float_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ptrue p0.s, vl8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #32
+; CHECK-NEXT:    st1w { z1.s }, p0, [x1]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <16 x float>, ptr %in
+  %hi = shufflevector <16 x float> %b, <16 x float> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <8 x float> %hi, ptr %out
+  %lo = shufflevector <16 x float> %b, <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x float> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v32f32(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: extract_subvector_v32f32:
 ; CHECK:       // %bb.0:
@@ -497,6 +769,24 @@ define void @extract_subvector_v4f64(ptr %a, ptr %b) vscale_range(2,0) #0 {
   ret void
 }
 
+define void @extract_v4double_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v4double_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    str q1, [x1]
+; CHECK-NEXT:    str q0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <4 x double>, ptr %in
+  %hi = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 2, i32 3>
+  store <2 x double> %hi, ptr %out
+  %lo = shufflevector <4 x double> %b, <4 x double> poison, <2 x i32> <i32 0, i32 1>
+  store <2 x double> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v8f64(ptr %a, ptr %b) #0 {
 ; CHECK-LABEL: extract_subvector_v8f64:
 ; CHECK:       // %bb.0:
@@ -511,6 +801,25 @@ define void @extract_subvector_v8f64(ptr %a, ptr %b) #0 {
   ret void
 }
 
+define void @extract_v8double_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(4,4) {
+; CHECK-LABEL: extract_v8double_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #32
+; CHECK-NEXT:    st1d { z1.d }, p0, [x1]
+; CHECK-NEXT:    st1d { z0.d }, p0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <8 x double>, ptr %in
+  %hi = shufflevector <8 x double> %b, <8 x double> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  store <4 x double> %hi, ptr %out
+  %lo = shufflevector <8 x double> %b, <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x double> %lo, ptr %out2
+  ret void
+}
+
 define void @extract_subvector_v16f64(ptr %a, ptr %b) vscale_range(8,0) #0 {
 ; CHECK-LABEL: extract_subvector_v16f64:
 ; CHECK:       // %bb.0:
@@ -539,16 +848,68 @@ define void @extract_subvector_v32f64(ptr %a, ptr %b) vscale_range(16,0) #0 {
   ret void
 }
 
+; bf16
+
+define void @extract_v8bfloat_halves(ptr %in, ptr %out, ptr %out2) #0 {
+; CHECK-LABEL: extract_v8bfloat_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    str d1, [x1]
+; CHECK-NEXT:    str d0, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <8 x bfloat>, ptr %in
+  %hi = shufflevector <8 x bfloat> %b, <8 x bfloat> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  store <4 x bfloat> %hi, ptr %out
+  %lo = shufflevector <8 x bfloat> %b, <8 x bfloat> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x bfloat> %lo, ptr %out2
+  ret void
+}
+
+define void @extract_v16bfloat_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(2,2) {
+; CHECK-LABEL: extract_v16bfloat_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q1, q0, [x0]
+; CHECK-NEXT:    str q0, [x1]
+; CHECK-NEXT:    str q1, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <16 x bfloat>, ptr %in
+  %hi = shufflevector <16 x bfloat> %b, <16 x bfloat> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <8 x bfloat> %hi, ptr %out
+  %lo = shufflevector <16 x bfloat> %b, <16 x bfloat> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <8 x bfloat> %lo, ptr %out2
+  ret void
+}
+
+define void @extract_v32bfloat_halves(ptr %in, ptr %out, ptr %out2) #0 vscale_range(4,4) {
+; CHECK-LABEL: extract_v32bfloat_halves:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldp q0, q1, [x0, #32]
+; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    stp q0, q1, [x1]
+; CHECK-NEXT:    stp q3, q2, [x2]
+; CHECK-NEXT:    ret
+entry:
+  %b = load <32 x bfloat>, ptr %in
+  %hi = shufflevector <32 x bfloat> %b, <32 x bfloat> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  store <16 x bfloat> %hi, ptr %out
+  %lo = shufflevector <32 x bfloat> %b, <32 x bfloat> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <16 x bfloat> %lo, ptr %out2
+  ret void
+}
+
 ; Test for infinite loop due to fold:
 ; extract_subvector(insert_subvector(x,y,c1),c2)--> extract_subvector(y,c2-c1)
 define void @extract_subvector_legalization_v8i32() vscale_range(2,2) #0 {
 ; CHECK-LABEL: extract_subvector_legalization_v8i32:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    adrp x8, .LCPI40_0
-; CHECK-NEXT:    add x8, x8, :lo12:.LCPI40_0
+; CHECK-NEXT:    adrp x8, .LCPI59_0
+; CHECK-NEXT:    add x8, x8, :lo12:.LCPI59_0
 ; CHECK-NEXT:    ptrue p1.d
 ; CHECK-NEXT:    ldr z0, [x8]
-; CHECK-NEXT:    mov z1.d, z0.d
+; CHECK-NEXT:    movprfx z1, z0
 ; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; CHECK-NEXT:    cmeq v0.4s, v0.4s, #0
 ; CHECK-NEXT:    cmeq v1.4s, v1.4s, #0
@@ -556,11 +917,11 @@ define void @extract_subvector_legalization_v8i32() vscale_range(2,2) #0 {
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
 ; CHECK-NEXT:    cmpne p0.d, p1/z, z1.d, #0
 ; CHECK-NEXT:    cmpne p1.d, p1/z, z0.d, #0
-; CHECK-NEXT:  .LBB40_1: // %body
+; CHECK-NEXT:  .LBB59_1: // %body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    st1d { z0.d }, p1, [x8]
 ; CHECK-NEXT:    st1d { z0.d }, p0, [x8]
-; CHECK-NEXT:    b .LBB40_1
+; CHECK-NEXT:    b .LBB59_1
 entry:
   %splat = shufflevector <8 x i32> poison, <8 x i32> poison, <8 x i32> zeroinitializer
   br label %body
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll
index af54b146c5b66..c8f6d98f5a63f 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-to-int.ll
@@ -150,13 +150,14 @@ define void @fcvtzu_v16f16_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
-; VBITS_GE_256-NEXT:    uunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    fcvtzu z1.s, p0/m, z1.h
+; VBITS_GE_256-NEXT:    uunpklo z1.s, z1.h
 ; VBITS_GE_256-NEXT:    fcvtzu z0.s, p0/m, z0.h
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    fcvtzu z1.s, p0/m, z1.h
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fcvtzu_v16f16_v16i32:
@@ -551,13 +552,14 @@ define void @fcvtzu_v8f32_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
-; VBITS_GE_256-NEXT:    uunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    fcvtzu z1.d, p0/m, z1.s
+; VBITS_GE_256-NEXT:    uunpklo z1.d, z1.s
 ; VBITS_GE_256-NEXT:    fcvtzu z0.d, p0/m, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT:    fcvtzu z1.d, p0/m, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fcvtzu_v8f32_v8i64:
@@ -1043,13 +1045,14 @@ define void @fcvtzs_v16f16_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
-; VBITS_GE_256-NEXT:    uunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    fcvtzs z1.s, p0/m, z1.h
+; VBITS_GE_256-NEXT:    uunpklo z1.s, z1.h
 ; VBITS_GE_256-NEXT:    fcvtzs z0.s, p0/m, z0.h
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    fcvtzs z1.s, p0/m, z1.h
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fcvtzs_v16f16_v16i32:
@@ -1444,13 +1447,14 @@ define void @fcvtzs_v8f32_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
-; VBITS_GE_256-NEXT:    uunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    fcvtzs z1.d, p0/m, z1.s
+; VBITS_GE_256-NEXT:    uunpklo z1.d, z1.s
 ; VBITS_GE_256-NEXT:    fcvtzs z0.d, p0/m, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT:    fcvtzs z1.d, p0/m, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fcvtzs_v8f32_v8i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll
index 4feb86305f8f6..d2fa65599b973 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-extends.ll
@@ -77,11 +77,12 @@ define void @sext_v32i8_v32i16(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
 ; VBITS_GE_256-NEXT:    add z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT:    sunpklo z1.h, z0.b
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.h, z0.b
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT:    sunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x1, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: sext_v32i8_v32i16:
@@ -326,11 +327,12 @@ define void @sext_v16i16_v16i32(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    add z0.h, z0.h, z0.h
-; VBITS_GE_256-NEXT:    sunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: sext_v16i16_v16i32:
@@ -490,11 +492,12 @@ define void @sext_v8i32_v8i64(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    add z0.s, z0.s, z0.s
-; VBITS_GE_256-NEXT:    sunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT:    sunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: sext_v8i32_v8i64:
@@ -573,11 +576,12 @@ define void @zext_v32i8_v32i16(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
 ; VBITS_GE_256-NEXT:    add z0.b, z0.b, z0.b
-; VBITS_GE_256-NEXT:    uunpklo z1.h, z0.b
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.h, z0.b
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
+; VBITS_GE_256-NEXT:    uunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x1, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: zext_v32i8_v32i16:
@@ -822,11 +826,12 @@ define void @zext_v16i16_v16i32(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
 ; VBITS_GE_256-NEXT:    add z0.h, z0.h, z0.h
-; VBITS_GE_256-NEXT:    uunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    uunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: zext_v16i16_v16i32:
@@ -986,11 +991,12 @@ define void @zext_v8i32_v8i64(ptr %in, ptr %out) #0 {
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
 ; VBITS_GE_256-NEXT:    add z0.s, z0.s, z0.s
-; VBITS_GE_256-NEXT:    uunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT:    uunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: zext_v8i32_v8i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll
index 2d78945399176..27be84419d59e 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll
@@ -259,17 +259,17 @@ define void @srem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-NEXT:    sunpklo z2.s, z2.h
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    sdivr z4.s, p1/m, z4.s, z5.s
-; CHECK-NEXT:    mov z5.d, z0.d
+; CHECK-NEXT:    movprfx z5, z0
 ; CHECK-NEXT:    ext z5.b, z5.b, z0.b, #128
 ; CHECK-NEXT:    sunpklo z5.h, z5.b
 ; CHECK-NEXT:    sunpklo z7.s, z5.h
 ; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #128
-; CHECK-NEXT:    sdivr z2.s, p1/m, z2.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
 ; CHECK-NEXT:    sunpklo z5.s, z5.h
+; CHECK-NEXT:    sdivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #128
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
 ; CHECK-NEXT:    sunpklo z3.h, z3.b
+; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
 ; CHECK-NEXT:    sunpklo z6.s, z3.h
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #128
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
@@ -420,11 +420,11 @@ define void @srem_v16i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    sunpklo z2.s, z1.h
 ; VBITS_GE_256-NEXT:    sunpklo z3.s, z0.h
-; VBITS_GE_256-NEXT:    mov z4.d, z0.d
+; VBITS_GE_256-NEXT:    movprfx z4, z0
 ; VBITS_GE_256-NEXT:    ext z4.b, z4.b, z0.b, #16
-; VBITS_GE_256-NEXT:    sdivr z2.s, p1/m, z2.s, z3.s
-; VBITS_GE_256-NEXT:    mov z3.d, z1.d
 ; VBITS_GE_256-NEXT:    sunpklo z4.s, z4.h
+; VBITS_GE_256-NEXT:    sdivr z2.s, p1/m, z2.s, z3.s
+; VBITS_GE_256-NEXT:    movprfx z3, z1
 ; VBITS_GE_256-NEXT:    ext z3.b, z3.b, z1.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z3.s, z3.h
 ; VBITS_GE_256-NEXT:    sdivr z3.s, p1/m, z3.s, z4.s
@@ -507,11 +507,11 @@ define void @srem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; CHECK-NEXT:    sunpklo z2.s, z1.h
 ; CHECK-NEXT:    sunpklo z3.s, z0.h
-; CHECK-NEXT:    mov z4.d, z0.d
+; CHECK-NEXT:    movprfx z4, z0
 ; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #128
-; CHECK-NEXT:    sdivr z2.s, p1/m, z2.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
 ; CHECK-NEXT:    sunpklo z4.s, z4.h
+; CHECK-NEXT:    sdivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #128
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    sdivr z3.s, p1/m, z3.s, z4.s
@@ -1077,17 +1077,17 @@ define void @urem_v256i8(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-NEXT:    uunpklo z2.s, z2.h
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    udivr z4.s, p1/m, z4.s, z5.s
-; CHECK-NEXT:    mov z5.d, z0.d
+; CHECK-NEXT:    movprfx z5, z0
 ; CHECK-NEXT:    ext z5.b, z5.b, z0.b, #128
 ; CHECK-NEXT:    uunpklo z5.h, z5.b
 ; CHECK-NEXT:    uunpklo z7.s, z5.h
 ; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #128
-; CHECK-NEXT:    udivr z2.s, p1/m, z2.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
 ; CHECK-NEXT:    uunpklo z5.s, z5.h
+; CHECK-NEXT:    udivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #128
-; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
 ; CHECK-NEXT:    uunpklo z3.h, z3.b
+; CHECK-NEXT:    uzp1 z4.h, z4.h, z4.h
 ; CHECK-NEXT:    uunpklo z6.s, z3.h
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #128
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
@@ -1238,11 +1238,11 @@ define void @urem_v16i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; VBITS_GE_256-NEXT:    uunpklo z2.s, z1.h
 ; VBITS_GE_256-NEXT:    uunpklo z3.s, z0.h
-; VBITS_GE_256-NEXT:    mov z4.d, z0.d
+; VBITS_GE_256-NEXT:    movprfx z4, z0
 ; VBITS_GE_256-NEXT:    ext z4.b, z4.b, z0.b, #16
-; VBITS_GE_256-NEXT:    udivr z2.s, p1/m, z2.s, z3.s
-; VBITS_GE_256-NEXT:    mov z3.d, z1.d
 ; VBITS_GE_256-NEXT:    uunpklo z4.s, z4.h
+; VBITS_GE_256-NEXT:    udivr z2.s, p1/m, z2.s, z3.s
+; VBITS_GE_256-NEXT:    movprfx z3, z1
 ; VBITS_GE_256-NEXT:    ext z3.b, z3.b, z1.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z3.s, z3.h
 ; VBITS_GE_256-NEXT:    udivr z3.s, p1/m, z3.s, z4.s
@@ -1325,11 +1325,11 @@ define void @urem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x1]
 ; CHECK-NEXT:    uunpklo z2.s, z1.h
 ; CHECK-NEXT:    uunpklo z3.s, z0.h
-; CHECK-NEXT:    mov z4.d, z0.d
+; CHECK-NEXT:    movprfx z4, z0
 ; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #128
-; CHECK-NEXT:    udivr z2.s, p1/m, z2.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
 ; CHECK-NEXT:    uunpklo z4.s, z4.h
+; CHECK-NEXT:    udivr z2.s, p1/m, z2.s, z3.s
+; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #128
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    udivr z3.s, p1/m, z3.s, z4.s
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll
index 1d9e01f4ecfdf..5753e5972f9c8 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-to-fp.ll
@@ -150,13 +150,14 @@ define void @ucvtf_v16i16_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
-; VBITS_GE_256-NEXT:    uunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    ucvtf z1.s, p0/m, z1.s
+; VBITS_GE_256-NEXT:    uunpklo z1.s, z1.h
 ; VBITS_GE_256-NEXT:    ucvtf z0.s, p0/m, z0.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    ucvtf z1.s, p0/m, z1.s
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: ucvtf_v16i16_v16f32:
@@ -554,13 +555,14 @@ define void @ucvtf_v8i32_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
-; VBITS_GE_256-NEXT:    uunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    ucvtf z1.d, p0/m, z1.d
+; VBITS_GE_256-NEXT:    uunpklo z1.d, z1.s
 ; VBITS_GE_256-NEXT:    ucvtf z0.d, p0/m, z0.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT:    ucvtf z1.d, p0/m, z1.d
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: ucvtf_v8i32_v8f64:
@@ -1063,13 +1065,14 @@ define void @scvtf_v16i16_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
-; VBITS_GE_256-NEXT:    sunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    scvtf z1.s, p0/m, z1.s
+; VBITS_GE_256-NEXT:    sunpklo z1.s, z1.h
 ; VBITS_GE_256-NEXT:    scvtf z0.s, p0/m, z0.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; VBITS_GE_256-NEXT:    scvtf z1.s, p0/m, z1.s
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x1, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: scvtf_v16i16_v16f32:
@@ -1479,13 +1482,14 @@ define void @scvtf_v8i32_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
-; VBITS_GE_256-NEXT:    sunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    scvtf z1.d, p0/m, z1.d
+; VBITS_GE_256-NEXT:    sunpklo z1.d, z1.s
 ; VBITS_GE_256-NEXT:    scvtf z0.d, p0/m, z0.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; VBITS_GE_256-NEXT:    scvtf z1.d, p0/m, z1.d
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x1]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x1, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: scvtf_v8i32_v8f64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-limit-duplane.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-limit-duplane.ll
index 78b41f71f0ea2..8b584c448bbae 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-limit-duplane.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-limit-duplane.ll
@@ -8,10 +8,11 @@ define <4 x i32> @test(ptr %arg1, ptr %arg2) {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ptrue p0.s, vl8
 ; CHECK-NEXT:    mov x8, #8 // =0x8
-; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0]
-; CHECK-NEXT:    add z1.s, z0.s, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #16
+; CHECK-NEXT:    add z1.s, z1.s, z1.s
 ; CHECK-NEXT:    add z2.s, z2.s, z2.s
 ; CHECK-NEXT:    dup v0.4s, v0.s[2]
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0, x8, lsl #2]
@@ -31,10 +32,11 @@ define <2 x i32> @test2(ptr %arg1, ptr %arg2) {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ptrue p0.s, vl8
 ; CHECK-NEXT:    mov x8, #8 // =0x8
-; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0, x8, lsl #2]
 ; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0]
-; CHECK-NEXT:    add z1.s, z0.s, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #24
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #24
+; CHECK-NEXT:    add z1.s, z1.s, z1.s
 ; CHECK-NEXT:    add z2.s, z2.s, z2.s
 ; CHECK-NEXT:    dup v0.2s, v0.s[0]
 ; CHECK-NEXT:    st1w { z1.s }, p0, [x0, x8, lsl #2]
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll
index 34dc0bb5ef2d2..864a9f7987bdb 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-masked-loads.ll
@@ -375,11 +375,12 @@ define void @masked_load_sext_v32i8i16(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    cmpeq p0.b, p0/z, z0.b, #0
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
-; VBITS_GE_256-NEXT:    sunpklo z1.h, z0.b
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.h, z0.b
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2, x8, lsl #1]
+; VBITS_GE_256-NEXT:    sunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_sext_v32i8i16:
@@ -476,11 +477,12 @@ define void @masked_load_sext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    cmpeq p0.h, p0/z, z0.h, #0
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
-; VBITS_GE_256-NEXT:    sunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2, x8, lsl #2]
+; VBITS_GE_256-NEXT:    sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_sext_v16i16i32:
@@ -542,11 +544,12 @@ define void @masked_load_sext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    cmpeq p0.s, p0/z, z0.s, #0
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
-; VBITS_GE_256-NEXT:    sunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2, x8, lsl #3]
+; VBITS_GE_256-NEXT:    sunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_sext_v8i32i64:
@@ -574,11 +577,12 @@ define void @masked_load_zext_v32i8i16(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    cmpeq p0.b, p0/z, z0.b, #0
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.h, vl16
-; VBITS_GE_256-NEXT:    uunpklo z1.h, z0.b
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.h, z0.b
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2, x8, lsl #1]
+; VBITS_GE_256-NEXT:    uunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_zext_v32i8i16:
@@ -675,11 +679,12 @@ define void @masked_load_zext_v16i16i32(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    cmpeq p0.h, p0/z, z0.h, #0
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.s, vl8
-; VBITS_GE_256-NEXT:    uunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2, x8, lsl #2]
+; VBITS_GE_256-NEXT:    uunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_zext_v16i16i32:
@@ -741,11 +746,12 @@ define void @masked_load_zext_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    cmpeq p0.s, p0/z, z0.s, #0
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
-; VBITS_GE_256-NEXT:    uunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2, x8, lsl #3]
+; VBITS_GE_256-NEXT:    uunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_zext_v8i32i64:
@@ -782,11 +788,12 @@ define void @masked_load_sext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    ptrue p1.b, vl32
 ; VBITS_GE_256-NEXT:    cmpne p1.b, p1/z, z1.b, #0
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p1/z, [x0]
-; VBITS_GE_256-NEXT:    sunpklo z1.h, z0.b
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.h, z0.b
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2, x8, lsl #1]
+; VBITS_GE_256-NEXT:    sunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_sext_v32i8i16_m16:
@@ -915,11 +922,12 @@ define void @masked_load_sext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    sunpklo z0.h, z1.b
 ; VBITS_GE_256-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p1/z, [x0]
-; VBITS_GE_256-NEXT:    sunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2, x8, lsl #2]
+; VBITS_GE_256-NEXT:    sunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_sext_v16i16i32_m32:
@@ -1000,11 +1008,12 @@ define void @masked_load_sext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    ptrue p1.s, vl8
 ; VBITS_GE_256-NEXT:    cmpne p1.s, p1/z, z1.s, #0
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p1/z, [x0]
-; VBITS_GE_256-NEXT:    sunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2, x8, lsl #3]
+; VBITS_GE_256-NEXT:    sunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_sext_v8i32i64_m64:
@@ -1041,11 +1050,12 @@ define void @masked_load_zext_v32i8i16_m16(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    ptrue p1.b, vl32
 ; VBITS_GE_256-NEXT:    cmpne p1.b, p1/z, z1.b, #0
 ; VBITS_GE_256-NEXT:    ld1b { z0.b }, p1/z, [x0]
-; VBITS_GE_256-NEXT:    uunpklo z1.h, z0.b
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.h, z0.b
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2, x8, lsl #1]
+; VBITS_GE_256-NEXT:    uunpklo z1.h, z1.b
+; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x2, x8, lsl #1]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_zext_v32i8i16_m16:
@@ -1174,11 +1184,12 @@ define void @masked_load_zext_v16i16i32_m32(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    sunpklo z0.h, z1.b
 ; VBITS_GE_256-NEXT:    cmpne p1.h, p1/z, z0.h, #0
 ; VBITS_GE_256-NEXT:    ld1h { z0.h }, p1/z, [x0]
-; VBITS_GE_256-NEXT:    uunpklo z1.s, z0.h
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.s, z0.h
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2, x8, lsl #2]
+; VBITS_GE_256-NEXT:    uunpklo z1.s, z1.h
+; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x2, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_zext_v16i16i32_m32:
@@ -1259,11 +1270,12 @@ define void @masked_load_zext_v8i32i64_m64(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    ptrue p1.s, vl8
 ; VBITS_GE_256-NEXT:    cmpne p1.s, p1/z, z1.s, #0
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p1/z, [x0]
-; VBITS_GE_256-NEXT:    uunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2, x8, lsl #3]
+; VBITS_GE_256-NEXT:    uunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_zext_v8i32i64_m64:
@@ -1495,11 +1507,12 @@ define void @masked_load_sext_ugt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    cmpne p0.s, p0/z, z0.s, #0
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
-; VBITS_GE_256-NEXT:    sunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    sunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2, x8, lsl #3]
+; VBITS_GE_256-NEXT:    sunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_sext_ugt_v8i32i64:
@@ -1527,11 +1540,12 @@ define void @masked_load_zext_sgt_v8i32i64(ptr %ap, ptr %bp, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    cmpgt p0.s, p0/z, z0.s, #0
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
-; VBITS_GE_256-NEXT:    uunpklo z1.d, z0.s
-; VBITS_GE_256-NEXT:    ext z0.b, z0.b, z0.b, #16
+; VBITS_GE_256-NEXT:    movprfx z1, z0
+; VBITS_GE_256-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; VBITS_GE_256-NEXT:    uunpklo z0.d, z0.s
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2]
-; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2, x8, lsl #3]
+; VBITS_GE_256-NEXT:    uunpklo z1.d, z1.s
+; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x2]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x2, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: masked_load_zext_sgt_v8i32i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
index 33d5ac4cd299e..cc19f6c2cbbc8 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-partial-reduce.ll
@@ -100,7 +100,7 @@ define <16 x i16> @two_way_i8_i16_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SVE-NEXT:    uunpkhi z1.h, z1.b
 ; SVE-NEXT:    mad z2.h, p0/m, z3.h, z4.h
 ; SVE-NEXT:    mad z0.h, p0/m, z1.h, z2.h
-; SVE-NEXT:    mov z1.d, z0.d
+; SVE-NEXT:    movprfx z1, z0
 ; SVE-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -113,7 +113,7 @@ define <16 x i16> @two_way_i8_i16_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SME-NEXT:    ldr z2, [x2]
 ; SME-NEXT:    umlalb z0.h, z2.b, z1.b
 ; SME-NEXT:    umlalt z0.h, z2.b, z1.b
-; SME-NEXT:    mov z1.d, z0.d
+; SME-NEXT:    movprfx z1, z0
 ; SME-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SME-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -223,7 +223,7 @@ define <8 x i32> @two_way_i16_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SVE-NEXT:    uunpkhi z1.s, z1.h
 ; SVE-NEXT:    mad z2.s, p0/m, z3.s, z4.s
 ; SVE-NEXT:    mad z0.s, p0/m, z1.s, z2.s
-; SVE-NEXT:    mov z1.d, z0.d
+; SVE-NEXT:    movprfx z1, z0
 ; SVE-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -236,7 +236,7 @@ define <8 x i32> @two_way_i16_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SME-NEXT:    ldr z2, [x2]
 ; SME-NEXT:    umlalb z0.s, z2.h, z1.h
 ; SME-NEXT:    umlalt z0.s, z2.h, z1.h
-; SME-NEXT:    mov z1.d, z0.d
+; SME-NEXT:    movprfx z1, z0
 ; SME-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SME-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -346,7 +346,7 @@ define <4 x i64> @two_way_i32_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SVE-NEXT:    uunpkhi z1.d, z1.s
 ; SVE-NEXT:    mad z2.d, p0/m, z3.d, z4.d
 ; SVE-NEXT:    mad z0.d, p0/m, z1.d, z2.d
-; SVE-NEXT:    mov z1.d, z0.d
+; SVE-NEXT:    movprfx z1, z0
 ; SVE-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -359,7 +359,7 @@ define <4 x i64> @two_way_i32_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SME-NEXT:    ldr z2, [x2]
 ; SME-NEXT:    umlalb z0.d, z2.s, z1.s
 ; SME-NEXT:    umlalt z0.d, z2.s, z1.s
-; SME-NEXT:    mov z1.d, z0.d
+; SME-NEXT:    movprfx z1, z0
 ; SME-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SME-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -635,7 +635,7 @@ define <8 x i32> @four_way_i8_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SVE-NEXT:    ldr z1, [x1]
 ; SVE-NEXT:    ldr z2, [x2]
 ; SVE-NEXT:    udot z0.s, z2.b, z1.b
-; SVE-NEXT:    mov z1.d, z0.d
+; SVE-NEXT:    movprfx z1, z0
 ; SVE-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -647,7 +647,7 @@ define <8 x i32> @four_way_i8_i32_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SME-NEXT:    ldr z1, [x1]
 ; SME-NEXT:    ldr z2, [x2]
 ; SME-NEXT:    udot z0.s, z2.b, z1.b
-; SME-NEXT:    mov z1.d, z0.d
+; SME-NEXT:    movprfx z1, z0
 ; SME-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SME-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -680,7 +680,7 @@ define <8 x i32> @four_way_i8_i32_vl256_usdot(ptr %accptr, ptr %uptr, ptr %sptr)
 ; SVE-NEXT:    ldr z1, [x1]
 ; SVE-NEXT:    ldr z2, [x2]
 ; SVE-NEXT:    usdot z0.s, z1.b, z2.b
-; SVE-NEXT:    mov z1.d, z0.d
+; SVE-NEXT:    movprfx z1, z0
 ; SVE-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -692,7 +692,7 @@ define <8 x i32> @four_way_i8_i32_vl256_usdot(ptr %accptr, ptr %uptr, ptr %sptr)
 ; SME-NEXT:    ldr z1, [x1]
 ; SME-NEXT:    ldr z2, [x2]
 ; SME-NEXT:    usdot z0.s, z1.b, z2.b
-; SME-NEXT:    mov z1.d, z0.d
+; SME-NEXT:    movprfx z1, z0
 ; SME-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SME-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -813,7 +813,7 @@ define <4 x i64> @four_way_i16_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
 ; SVE-NEXT:    ldr z1, [x1]
 ; SVE-NEXT:    ldr z2, [x2]
 ; SVE-NEXT:    udot z0.d, z2.h, z1.h
-; SVE-NEXT:    mov z1.d, z0.d
+; SVE-NEXT:    movprfx z1, z0
 ; SVE-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -825,7 +825,7 @@ define <4 x i64> @four_way_i16_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vsca
 ; SME-NEXT:    ldr z1, [x1]
 ; SME-NEXT:    ldr z2, [x2]
 ; SME-NEXT:    udot z0.d, z2.h, z1.h
-; SME-NEXT:    mov z1.d, z0.d
+; SME-NEXT:    movprfx z1, z0
 ; SME-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SME-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -984,7 +984,7 @@ define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SVE-NEXT:    uunpkhi z0.d, z0.s
 ; SVE-NEXT:    add z1.d, z2.d, z1.d
 ; SVE-NEXT:    add z0.d, z1.d, z0.d
-; SVE-NEXT:    mov z1.d, z0.d
+; SVE-NEXT:    movprfx z1, z0
 ; SVE-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SVE-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SVE-NEXT:    // kill: def $q1 killed $q1 killed $z1
@@ -999,7 +999,7 @@ define <4 x i64> @four_way_i8_i64_vl256(ptr %accptr, ptr %uptr, ptr %sptr) vscal
 ; SME-NEXT:    ldr z0, [x0]
 ; SME-NEXT:    uaddwb z0.d, z0.d, z2.s
 ; SME-NEXT:    uaddwt z0.d, z0.d, z2.s
-; SME-NEXT:    mov z1.d, z0.d
+; SME-NEXT:    movprfx z1, z0
 ; SME-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; SME-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; SME-NEXT:    // kill: def $q1 killed $q1 killed $z1
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
index 2eff6da0866f8..ba4a3a2042305 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll
@@ -12,8 +12,8 @@ define void @hang_when_merging_stores_after_legalisation(ptr %a, <2 x i32> %b) v
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    mov z0.s, s0
-; CHECK-NEXT:    mov z1.d, z0.d
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #16
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
 ; CHECK-NEXT:    st2 { v0.4s, v1.4s }, [x0]
 ; CHECK-NEXT:    ret
   %splat = shufflevector <2 x i32> %b, <2 x i32> poison, <8 x i32> zeroinitializer
@@ -33,6 +33,9 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    umov w8, v0.b[8]
 ; CHECK-NEXT:    mov v1.b[1], v0.b[1]
+; CHECK-NEXT:    movprfx z3, z0
+; CHECK-NEXT:    ext z3.b, z3.b, z0.b, #16
+; CHECK-NEXT:    ext v4.16b, v3.16b, v3.16b, #8
 ; CHECK-NEXT:    fmov s2, w8
 ; CHECK-NEXT:    mov v1.b[2], v0.b[2]
 ; CHECK-NEXT:    mov v2.b[1], v0.b[9]
@@ -48,32 +51,30 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang
 ; CHECK-NEXT:    mov v2.b[6], v0.b[14]
 ; CHECK-NEXT:    uunpklo z1.h, z1.b
 ; CHECK-NEXT:    mov v2.b[7], v0.b[15]
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
+; CHECK-NEXT:    uunpklo z0.h, z3.b
+; CHECK-NEXT:    uunpklo z3.h, z4.b
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    uunpklo z0.h, z0.b
 ; CHECK-NEXT:    uunpklo z2.h, z2.b
-; CHECK-NEXT:    lsl z1.s, z1.s, #31
-; CHECK-NEXT:    uunpklo z3.h, z3.b
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z2.s, z2.h
-; CHECK-NEXT:    asr z1.s, z1.s, #31
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
+; CHECK-NEXT:    lsl z1.s, z1.s, #31
+; CHECK-NEXT:    uunpklo z2.s, z2.h
 ; CHECK-NEXT:    lsl z0.s, z0.s, #31
-; CHECK-NEXT:    cmpne p1.s, p0/z, z1.s, #0
-; CHECK-NEXT:    lsl z2.s, z2.s, #31
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    asr z0.s, z0.s, #31
 ; CHECK-NEXT:    lsl z3.s, z3.s, #31
-; CHECK-NEXT:    asr z2.s, z2.s, #31
-; CHECK-NEXT:    st1w { z1.s }, p1, [x0]
-; CHECK-NEXT:    cmpne p2.s, p0/z, z0.s, #0
+; CHECK-NEXT:    asr z1.s, z1.s, #31
+; CHECK-NEXT:    asr z0.s, z0.s, #31
 ; CHECK-NEXT:    asr z3.s, z3.s, #31
-; CHECK-NEXT:    cmpne p3.s, p0/z, z3.s, #0
+; CHECK-NEXT:    lsl z2.s, z2.s, #31
+; CHECK-NEXT:    cmpne p3.s, p0/z, z1.s, #0
+; CHECK-NEXT:    cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    cmpne p2.s, p0/z, z3.s, #0
+; CHECK-NEXT:    asr z2.s, z2.s, #31
 ; CHECK-NEXT:    cmpne p0.s, p0/z, z2.s, #0
-; CHECK-NEXT:    st1w { z1.s }, p2, [x0, #2, mul vl]
-; CHECK-NEXT:    st1w { z1.s }, p3, [x0, #3, mul vl]
-; CHECK-NEXT:    st1w { z1.s }, p0, [x0, #1, mul vl]
+; CHECK-NEXT:    st1w { z0.s }, p1, [x0, #2, mul vl]
+; CHECK-NEXT:    st1w { z0.s }, p2, [x0, #3, mul vl]
+; CHECK-NEXT:    st1w { z0.s }, p3, [x0]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, #1, mul vl]
 ; CHECK-NEXT:  .LBB1_2: // %exit
 ; CHECK-NEXT:    ret
   %broadcast.splat = shufflevector <32 x i1> zeroinitializer, <32 x i1> zeroinitializer, <32 x i32> zeroinitializer
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll b/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll
index 7f144df499be0..f1d5813433489 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-vector-llrint.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64 -mattr=+sve -aarch64-sve-vector-bits-min=256 | FileCheck %s
 
-define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
+define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v1i64_v1f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx h0, h0
@@ -13,7 +13,7 @@ define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half>)
 
-define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
+define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v1i64_v2f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -30,7 +30,7 @@ define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half>)
 
-define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
+define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v4i64_v4f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.4h, v0.4h
@@ -51,7 +51,7 @@ define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half>)
 
-define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
+define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v8i64_v8f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
@@ -85,7 +85,7 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half>)
 
-define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
+define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v16i64_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v2.16b, v1.16b, v1.16b, #8
@@ -144,16 +144,13 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half>)
 
-define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
+define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v32i64_v32f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    sub x9, sp, #272
 ; CHECK-NEXT:    mov x29, sp
 ; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
-; CHECK-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    frintx v5.4h, v0.4h
 ; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
@@ -278,7 +275,7 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 }
 declare <32 x i64> @llvm.llrint.v32i64.v32f16(<32 x half>)
 
-define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
+define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v1i64_v1f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -291,7 +288,7 @@ define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
 
-define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
+define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v2i64_v2f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.2s, v0.2s
@@ -303,7 +300,7 @@ define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
 
-define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
+define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v4i64_v4f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.4s, v0.4s
@@ -324,7 +321,7 @@ define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
 
-define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
+define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v8i64_v8f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.4s, v0.4s
@@ -357,7 +354,7 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
 
-define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
+define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v16i64_v16f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v3.4s, v3.4s
@@ -414,16 +411,13 @@ define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
 
-define <32 x i64> @llrint_v32i64_v32f32(<32 x float> %x) {
+define <32 x i64> @llrint_v32i64_v32f32(<32 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v32i64_v32f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    sub x9, sp, #272
 ; CHECK-NEXT:    mov x29, sp
 ; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
-; CHECK-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    frintx v0.4s, v0.4s
 ; CHECK-NEXT:    frintx v1.4s, v1.4s
 ; CHECK-NEXT:    frintx v2.4s, v2.4s
@@ -544,7 +538,7 @@ define <32 x i64> @llrint_v32i64_v32f32(<32 x float> %x) {
 }
 declare <32 x i64> @llvm.llrint.v32i64.v32f32(<32 x float>)
 
-define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
+define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v1i64_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx d0, d0
@@ -556,7 +550,7 @@ define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
 
-define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
+define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v2i64_v2f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.2d, v0.2d
@@ -567,7 +561,7 @@ define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
 
-define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
+define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v4i64_v4f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl2
@@ -593,7 +587,7 @@ define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
 
-define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
+define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v8i64_v8f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl2
@@ -635,7 +629,7 @@ define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
 
-define <16 x i64> @llrint_v16f64(<16 x double> %x) {
+define <16 x i64> @llrint_v16f64(<16 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v16f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p1.d, vl2
@@ -708,16 +702,13 @@ define <16 x i64> @llrint_v16f64(<16 x double> %x) {
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double>)
 
-define <32 x i64> @llrint_v32f64(<32 x double> %x) {
+define <32 x i64> @llrint_v32f64(<32 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v32f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
 ; CHECK-NEXT:    sub x9, sp, #272
 ; CHECK-NEXT:    mov x29, sp
 ; CHECK-NEXT:    and sp, x9, #0xffffffffffffffe0
-; CHECK-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p1.d, vl2
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
@@ -861,3 +852,322 @@ define <32 x i64> @llrint_v32f64(<32 x double> %x) {
   ret <32 x i64> %a
 }
 declare <32 x i64> @llvm.llrint.v32i64.v32f64(<32 x double>)
+
+define <1 x i64> @llrint_v1i64_v1fp128(<1 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v1i64_v1fp128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1fp128(<1 x fp128> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1fp128(<1 x fp128>)
+
+define <2 x i64> @llrint_v2i64_v2fp128(<2 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v2i64_v2fp128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2fp128(<2 x fp128> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2fp128(<2 x fp128>)
+
+define <4 x i64> @llrint_v4i64_v4fp128(<4 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v4i64_v4fp128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #64
+; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    mov v0.16b, v3.16b
+; CHECK-NEXT:    stp q2, q1, [sp, #16] // 32-byte Folded Spill
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #64
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str z0, [x8] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #64
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ldr z1, [x8] // 16-byte Folded Reload
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
+; CHECK-NEXT:    addvl sp, sp, #1
+; CHECK-NEXT:    add sp, sp, #64
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4fp128(<4 x fp128> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4fp128(<4 x fp128>)
+
+define <8 x i64> @llrint_v8i64_v8fp128(<8 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v8i64_v8fp128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #128
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
+; CHECK-NEXT:    mov v0.16b, v7.16b
+; CHECK-NEXT:    stp q6, q5, [sp, #16] // 32-byte Folded Spill
+; CHECK-NEXT:    stp q4, q3, [sp, #48] // 32-byte Folded Spill
+; CHECK-NEXT:    stp q2, q1, [sp, #80] // 32-byte Folded Spill
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #128
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #128
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ldr z1, [x8, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #128
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str z0, [x8] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #128
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    ldr z2, [x8, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ldr z1, [x8] // 16-byte Folded Reload
+; CHECK-NEXT:    movprfx z3, z2
+; CHECK-NEXT:    ext z3.b, z3.b, z2.b, #16
+; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $z2
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    // kill: def $q3 killed $q3 killed $z3
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    add sp, sp, #128
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8fp128(<8 x fp128> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8fp128(<8 x fp128>)
+
+define <16 x i64> @llrint_v16fp128(<16 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v16fp128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    sub sp, sp, #256
+; CHECK-NEXT:    addvl sp, sp, #-4
+; CHECK-NEXT:    addvl x8, sp, #4
+; CHECK-NEXT:    str q1, [sp, #240] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, #272]
+; CHECK-NEXT:    addvl x8, sp, #4
+; CHECK-NEXT:    str q0, [sp, #224] // 16-byte Folded Spill
+; CHECK-NEXT:    stp q7, q6, [sp, #128] // 32-byte Folded Spill
+; CHECK-NEXT:    str q1, [sp, #112] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, #288]
+; CHECK-NEXT:    addvl x8, sp, #4
+; CHECK-NEXT:    stp q5, q4, [sp, #160] // 32-byte Folded Spill
+; CHECK-NEXT:    str q1, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, #304]
+; CHECK-NEXT:    addvl x8, sp, #4
+; CHECK-NEXT:    stp q3, q2, [sp, #192] // 32-byte Folded Spill
+; CHECK-NEXT:    str q1, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, #320]
+; CHECK-NEXT:    addvl x8, sp, #4
+; CHECK-NEXT:    str q1, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, #336]
+; CHECK-NEXT:    addvl x8, sp, #4
+; CHECK-NEXT:    str q1, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, #352]
+; CHECK-NEXT:    addvl x8, sp, #4
+; CHECK-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, #368]
+; CHECK-NEXT:    addvl x8, sp, #4
+; CHECK-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q1, [x8, #384]
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str z0, [x8, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ldr z1, [x8, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    str z0, [x8, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str z0, [x8, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ldr z1, [x8, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    str z0, [x8, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #128] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #128] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #144] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #128] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #160] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #160] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #176] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #160] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ldr z1, [x8, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #192] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #192] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #208] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #192] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str z0, [x8] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #240] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #240] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #224] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #240] // 16-byte Folded Reload
+; CHECK-NEXT:    add x8, sp, #256
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    ldr z2, [x8, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z4, [x8, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z6, [x8, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    ldr z1, [x8] // 16-byte Folded Reload
+; CHECK-NEXT:    movprfx z3, z2
+; CHECK-NEXT:    ext z3.b, z3.b, z2.b, #16
+; CHECK-NEXT:    movprfx z5, z4
+; CHECK-NEXT:    ext z5.b, z5.b, z4.b, #16
+; CHECK-NEXT:    // kill: def $q2 killed $q2 killed $z2
+; CHECK-NEXT:    // kill: def $q4 killed $q4 killed $z4
+; CHECK-NEXT:    movprfx z7, z6
+; CHECK-NEXT:    ext z7.b, z7.b, z6.b, #16
+; CHECK-NEXT:    // kill: def $q6 killed $q6 killed $z6
+; CHECK-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-NEXT:    // kill: def $q3 killed $q3 killed $z3
+; CHECK-NEXT:    // kill: def $q5 killed $q5 killed $z5
+; CHECK-NEXT:    // kill: def $q7 killed $q7 killed $z7
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
+; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    add sp, sp, #256
+; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16fp128(<16 x fp128> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16fp128(<16 x fp128>)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll b/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll
index 9fe8d92a182ac..62a3fa7f29bb5 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-vector-lrint.ll
@@ -4,7 +4,7 @@
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=aarch64 -mattr=+sve \
 ; RUN:   -aarch64-sve-vector-bits-min=256 | FileCheck --check-prefixes=CHECK-i64 %s
 
-define <1 x iXLen> @lrint_v1f16(<1 x half> %x) {
+define <1 x iXLen> @lrint_v1f16(<1 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v1f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx h0, h0
@@ -23,7 +23,7 @@ define <1 x iXLen> @lrint_v1f16(<1 x half> %x) {
 }
 declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f16(<1 x half>)
 
-define <2 x iXLen> @lrint_v2f16(<2 x half> %x) {
+define <2 x iXLen> @lrint_v2f16(<2 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v2f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -53,7 +53,7 @@ define <2 x iXLen> @lrint_v2f16(<2 x half> %x) {
 }
 declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f16(<2 x half>)
 
-define <4 x iXLen> @lrint_v4f16(<4 x half> %x) {
+define <4 x iXLen> @lrint_v4f16(<4 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v4f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.4h, v0.4h
@@ -81,7 +81,7 @@ define <4 x iXLen> @lrint_v4f16(<4 x half> %x) {
 }
 declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f16(<4 x half>)
 
-define <8 x iXLen> @lrint_v8f16(<8 x half> %x) {
+define <8 x iXLen> @lrint_v8f16(<8 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v8f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v2.8h, v0.8h
@@ -143,7 +143,7 @@ define <8 x iXLen> @lrint_v8f16(<8 x half> %x) {
 }
 declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f16(<8 x half>)
 
-define <16 x iXLen> @lrint_v16f16(<16 x half> %x) {
+define <16 x iXLen> @lrint_v16f16(<16 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v16f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v1.8h, v1.8h
@@ -254,26 +254,17 @@ define <16 x iXLen> @lrint_v16f16(<16 x half> %x) {
 }
 declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f16(<16 x half>)
 
-define <32 x iXLen> @lrint_v32f16(<32 x half> %x) {
+define <32 x iXLen> @lrint_v32f16(<32 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v32f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    stp x26, x25, [sp, #-64]! // 16-byte Folded Spill
-; CHECK-i32-NEXT:    stp x24, x23, [sp, #16] // 16-byte Folded Spill
-; CHECK-i32-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
-; CHECK-i32-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
-; CHECK-i32-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-i32-NEXT:    .cfi_offset w19, -8
-; CHECK-i32-NEXT:    .cfi_offset w20, -16
-; CHECK-i32-NEXT:    .cfi_offset w21, -24
-; CHECK-i32-NEXT:    .cfi_offset w22, -32
-; CHECK-i32-NEXT:    .cfi_offset w23, -40
-; CHECK-i32-NEXT:    .cfi_offset w24, -48
-; CHECK-i32-NEXT:    .cfi_offset w25, -56
-; CHECK-i32-NEXT:    .cfi_offset w26, -64
 ; CHECK-i32-NEXT:    frintx v3.8h, v3.8h
 ; CHECK-i32-NEXT:    frintx v2.8h, v2.8h
+; CHECK-i32-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
 ; CHECK-i32-NEXT:    frintx v1.8h, v1.8h
 ; CHECK-i32-NEXT:    frintx v0.8h, v0.8h
+; CHECK-i32-NEXT:    stp x24, x23, [sp, #16] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x22, x21, [sp, #32] // 16-byte Folded Spill
 ; CHECK-i32-NEXT:    mov h4, v3.h[7]
 ; CHECK-i32-NEXT:    mov h5, v3.h[6]
 ; CHECK-i32-NEXT:    mov h6, v3.h[5]
@@ -378,9 +369,6 @@ define <32 x iXLen> @lrint_v32f16(<32 x half> %x) {
 ; CHECK-i64-NEXT:    sub x9, sp, #272
 ; CHECK-i64-NEXT:    mov x29, sp
 ; CHECK-i64-NEXT:    and sp, x9, #0xffffffffffffffe0
-; CHECK-i64-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-i64-NEXT:    .cfi_offset w30, -8
-; CHECK-i64-NEXT:    .cfi_offset w29, -16
 ; CHECK-i64-NEXT:    frintx v5.4h, v0.4h
 ; CHECK-i64-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-i64-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
@@ -505,7 +493,7 @@ define <32 x iXLen> @lrint_v32f16(<32 x half> %x) {
 }
 declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f16(<32 x half>)
 
-define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
+define <1 x iXLen> @lrint_v1f32(<1 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v1f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.2s, v0.2s
@@ -524,7 +512,7 @@ define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
 }
 declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float>)
 
-define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
+define <2 x iXLen> @lrint_v2f32(<2 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v2f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.2s, v0.2s
@@ -542,7 +530,7 @@ define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
 }
 declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float>)
 
-define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
+define <4 x iXLen> @lrint_v4f32(<4 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v4f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
@@ -569,7 +557,7 @@ define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
 }
 declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float>)
 
-define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
+define <8 x iXLen> @lrint_v8f32(<8 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v8f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    ptrue p0.d, vl2
@@ -636,7 +624,7 @@ define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
 }
 declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float>)
 
-define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
+define <16 x iXLen> @lrint_v16f32(<16 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v16f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    ptrue p0.d, vl2
@@ -754,24 +742,10 @@ define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
 }
 declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float>)
 
-define <32 x iXLen> @lrint_v32f32(<32 x float> %x) {
+define <32 x iXLen> @lrint_v32f32(<32 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v32f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    str x27, [sp, #-80]! // 8-byte Folded Spill
-; CHECK-i32-NEXT:    stp x26, x25, [sp, #16] // 16-byte Folded Spill
-; CHECK-i32-NEXT:    stp x24, x23, [sp, #32] // 16-byte Folded Spill
-; CHECK-i32-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
-; CHECK-i32-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
-; CHECK-i32-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-i32-NEXT:    .cfi_offset w19, -8
-; CHECK-i32-NEXT:    .cfi_offset w20, -16
-; CHECK-i32-NEXT:    .cfi_offset w21, -24
-; CHECK-i32-NEXT:    .cfi_offset w22, -32
-; CHECK-i32-NEXT:    .cfi_offset w23, -40
-; CHECK-i32-NEXT:    .cfi_offset w24, -48
-; CHECK-i32-NEXT:    .cfi_offset w25, -56
-; CHECK-i32-NEXT:    .cfi_offset w26, -64
-; CHECK-i32-NEXT:    .cfi_offset w27, -80
 ; CHECK-i32-NEXT:    ptrue p1.d, vl2
 ; CHECK-i32-NEXT:    // kill: def $q6 killed $q6 def $z6
 ; CHECK-i32-NEXT:    // kill: def $q7 killed $q7 def $z7
@@ -781,11 +755,15 @@ define <32 x iXLen> @lrint_v32f32(<32 x float> %x) {
 ; CHECK-i32-NEXT:    // kill: def $q5 killed $q5 def $z5
 ; CHECK-i32-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-i32-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-i32-NEXT:    stp x24, x23, [sp, #32] // 16-byte Folded Spill
 ; CHECK-i32-NEXT:    ptrue p0.s, vl8
+; CHECK-i32-NEXT:    stp x22, x21, [sp, #48] // 16-byte Folded Spill
 ; CHECK-i32-NEXT:    splice z6.d, p1, z6.d, z7.d
 ; CHECK-i32-NEXT:    splice z2.d, p1, z2.d, z3.d
 ; CHECK-i32-NEXT:    splice z4.d, p1, z4.d, z5.d
 ; CHECK-i32-NEXT:    splice z0.d, p1, z0.d, z1.d
+; CHECK-i32-NEXT:    stp x26, x25, [sp, #16] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x20, x19, [sp, #64] // 16-byte Folded Spill
 ; CHECK-i32-NEXT:    movprfx z3, z6
 ; CHECK-i32-NEXT:    frintx z3.s, p0/m, z6.s
 ; CHECK-i32-NEXT:    frintx z2.s, p0/m, z2.s
@@ -897,9 +875,6 @@ define <32 x iXLen> @lrint_v32f32(<32 x float> %x) {
 ; CHECK-i64-NEXT:    sub x9, sp, #272
 ; CHECK-i64-NEXT:    mov x29, sp
 ; CHECK-i64-NEXT:    and sp, x9, #0xffffffffffffffe0
-; CHECK-i64-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-i64-NEXT:    .cfi_offset w30, -8
-; CHECK-i64-NEXT:    .cfi_offset w29, -16
 ; CHECK-i64-NEXT:    frintx v0.4s, v0.4s
 ; CHECK-i64-NEXT:    frintx v1.4s, v1.4s
 ; CHECK-i64-NEXT:    frintx v2.4s, v2.4s
@@ -1020,7 +995,7 @@ define <32 x iXLen> @lrint_v32f32(<32 x float> %x) {
 }
 declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f32(<32 x float>)
 
-define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
+define <1 x iXLen> @lrint_v1f64(<1 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v1f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx d0, d0
@@ -1039,7 +1014,7 @@ define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
 }
 declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double>)
 
-define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
+define <2 x iXLen> @lrint_v2f64(<2 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v2f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.2d, v0.2d
@@ -1061,7 +1036,7 @@ define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
 }
 declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double>)
 
-define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
+define <4 x iXLen> @lrint_v4f64(<4 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v4f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    ptrue p0.d, vl2
@@ -1109,7 +1084,7 @@ define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
 }
 declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double>)
 
-define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
+define <8 x iXLen> @lrint_v8f64(<8 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v8f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    ptrue p0.d, vl2
@@ -1188,7 +1163,7 @@ define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
 }
 declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double>)
 
-define <16 x iXLen> @lrint_v16f64(<16 x double> %x) {
+define <16 x iXLen> @lrint_v16f64(<16 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v16f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    ptrue p1.d, vl2
@@ -1329,7 +1304,7 @@ define <16 x iXLen> @lrint_v16f64(<16 x double> %x) {
 }
 declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f64(<16 x double>)
 
-define <32 x iXLen> @lrint_v32f64(<32 x double> %x) {
+define <32 x iXLen> @lrint_v32f64(<32 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v32f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    ptrue p1.d, vl2
@@ -1465,9 +1440,6 @@ define <32 x iXLen> @lrint_v32f64(<32 x double> %x) {
 ; CHECK-i64-NEXT:    sub x9, sp, #272
 ; CHECK-i64-NEXT:    mov x29, sp
 ; CHECK-i64-NEXT:    and sp, x9, #0xffffffffffffffe0
-; CHECK-i64-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-i64-NEXT:    .cfi_offset w30, -8
-; CHECK-i64-NEXT:    .cfi_offset w29, -16
 ; CHECK-i64-NEXT:    ptrue p1.d, vl2
 ; CHECK-i64-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-i64-NEXT:    // kill: def $q1 killed $q1 def $z1
@@ -1611,3 +1583,527 @@ define <32 x iXLen> @lrint_v32f64(<32 x double> %x) {
   ret <32 x iXLen> %a
 }
 declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f64(<32 x double>)
+
+define <1 x iXLen> @lrint_v1fp128(<1 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v1fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v1fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1fp128(<1 x fp128> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1fp128(<1 x fp128>)
+
+define <2 x iXLen> @lrint_v2fp128(<2 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v2fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    sub sp, sp, #48
+; CHECK-i32-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-i32-NEXT:    add sp, sp, #48
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v2fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    sub sp, sp, #48
+; CHECK-i64-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    mov v0.16b, v1.16b
+; CHECK-i64-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    add sp, sp, #48
+; CHECK-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2fp128(<2 x fp128> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2fp128(<2 x fp128>)
+
+define <4 x iXLen> @lrint_v4fp128(<4 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v4fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    sub sp, sp, #80
+; CHECK-i32-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
+; CHECK-i32-NEXT:    stp q2, q3, [sp, #16] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w0
+; CHECK-i32-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[3], w0
+; CHECK-i32-NEXT:    add sp, sp, #80
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v4fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-i64-NEXT:    sub sp, sp, #64
+; CHECK-i64-NEXT:    addvl sp, sp, #-1
+; CHECK-i64-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    mov v0.16b, v3.16b
+; CHECK-i64-NEXT:    stp q2, q1, [sp, #16] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #64
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str z0, [x8] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #64
+; CHECK-i64-NEXT:    ptrue p0.d, vl2
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    ldr z1, [x8] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i64-NEXT:    movprfx z1, z0
+; CHECK-i64-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-i64-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-i64-NEXT:    // kill: def $q1 killed $q1 killed $z1
+; CHECK-i64-NEXT:    addvl sp, sp, #1
+; CHECK-i64-NEXT:    add sp, sp, #64
+; CHECK-i64-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4fp128(<4 x fp128> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4fp128(<4 x fp128>)
+
+define <8 x iXLen> @lrint_v8fp128(<8 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v8fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    sub sp, sp, #176
+; CHECK-i32-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    mov v0.16b, v7.16b
+; CHECK-i32-NEXT:    stp x30, x25, [sp, #112] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x24, x23, [sp, #128] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x22, x21, [sp, #144] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp x20, x19, [sp, #160] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp q6, q5, [sp] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    stp q4, q3, [sp, #32] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    stp q2, q1, [sp, #64] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w19, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w20, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w21, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w22, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w23, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w24, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w25, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s1, w22
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    mov v0.s[1], w25
+; CHECK-i32-NEXT:    mov v1.s[1], w21
+; CHECK-i32-NEXT:    ldp x22, x21, [sp, #144] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    ldp x30, x25, [sp, #112] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w24
+; CHECK-i32-NEXT:    mov v1.s[2], w20
+; CHECK-i32-NEXT:    mov v0.s[3], w23
+; CHECK-i32-NEXT:    mov v1.s[3], w19
+; CHECK-i32-NEXT:    ldp x20, x19, [sp, #160] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    ldp x24, x23, [sp, #128] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    add sp, sp, #176
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v8fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-i64-NEXT:    sub sp, sp, #128
+; CHECK-i64-NEXT:    addvl sp, sp, #-2
+; CHECK-i64-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    mov v0.16b, v7.16b
+; CHECK-i64-NEXT:    stp q6, q5, [sp, #16] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    stp q4, q3, [sp, #48] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    stp q2, q1, [sp, #80] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #128
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #128
+; CHECK-i64-NEXT:    ptrue p0.d, vl2
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    ldr z1, [x8, #1, mul vl] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i64-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #64] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #128
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str z0, [x8] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #96] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #128
+; CHECK-i64-NEXT:    ptrue p0.d, vl2
+; CHECK-i64-NEXT:    ldr z2, [x8, #1, mul vl] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    ldr z1, [x8] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    movprfx z3, z2
+; CHECK-i64-NEXT:    ext z3.b, z3.b, z2.b, #16
+; CHECK-i64-NEXT:    // kill: def $q2 killed $q2 killed $z2
+; CHECK-i64-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i64-NEXT:    // kill: def $q3 killed $q3 killed $z3
+; CHECK-i64-NEXT:    movprfx z1, z0
+; CHECK-i64-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-i64-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-i64-NEXT:    // kill: def $q1 killed $q1 killed $z1
+; CHECK-i64-NEXT:    addvl sp, sp, #2
+; CHECK-i64-NEXT:    add sp, sp, #128
+; CHECK-i64-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8fp128(<8 x fp128> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8fp128(<8 x fp128>)
+
+define <16 x iXLen> @lrint_v16fp128(<16 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v16fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    sub sp, sp, #368
+; CHECK-i32-NEXT:    stp q3, q0, [sp, #144] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    stp q2, q1, [sp, #176] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #368]
+; CHECK-i32-NEXT:    stp x29, x30, [sp, #272] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #64] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #384]
+; CHECK-i32-NEXT:    stp x28, x27, [sp, #288] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #48] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #400]
+; CHECK-i32-NEXT:    stp x26, x25, [sp, #304] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #416]
+; CHECK-i32-NEXT:    stp x24, x23, [sp, #320] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #208] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #432]
+; CHECK-i32-NEXT:    stp x22, x21, [sp, #336] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #448]
+; CHECK-i32-NEXT:    stp x20, x19, [sp, #352] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #224] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #464]
+; CHECK-i32-NEXT:    stp q7, q6, [sp, #80] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #240] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #480]
+; CHECK-i32-NEXT:    stp q5, q4, [sp, #112] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    mov v0.16b, v1.16b
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #240] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    str w0, [sp, #268] // 4-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #224] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    str w0, [sp, #240] // 4-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    str w0, [sp, #224] // 4-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #208] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w23, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    str w0, [sp, #208] // 4-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w24, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w25, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w27, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w26, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w28, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #128] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w29, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #144] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w19, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #176] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w20, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #192] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w21, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #160] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov w22, w0
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s1, w19
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    ldr w8, [sp, #224] // 4-byte Folded Reload
+; CHECK-i32-NEXT:    fmov s2, w27
+; CHECK-i32-NEXT:    fmov s3, w23
+; CHECK-i32-NEXT:    mov v0.s[1], w22
+; CHECK-i32-NEXT:    mov v1.s[1], w29
+; CHECK-i32-NEXT:    mov v2.s[1], w25
+; CHECK-i32-NEXT:    mov v3.s[1], w8
+; CHECK-i32-NEXT:    ldr w8, [sp, #240] // 4-byte Folded Reload
+; CHECK-i32-NEXT:    ldp x29, x30, [sp, #272] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w21
+; CHECK-i32-NEXT:    mov v1.s[2], w28
+; CHECK-i32-NEXT:    mov v2.s[2], w24
+; CHECK-i32-NEXT:    mov v3.s[2], w8
+; CHECK-i32-NEXT:    ldr w8, [sp, #208] // 4-byte Folded Reload
+; CHECK-i32-NEXT:    ldp x22, x21, [sp, #336] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    ldp x24, x23, [sp, #320] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[3], w20
+; CHECK-i32-NEXT:    mov v1.s[3], w26
+; CHECK-i32-NEXT:    mov v2.s[3], w8
+; CHECK-i32-NEXT:    ldr w8, [sp, #268] // 4-byte Folded Reload
+; CHECK-i32-NEXT:    ldp x20, x19, [sp, #352] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    ldp x26, x25, [sp, #304] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v3.s[3], w8
+; CHECK-i32-NEXT:    ldp x28, x27, [sp, #288] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    add sp, sp, #368
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v16fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-i64-NEXT:    sub sp, sp, #256
+; CHECK-i64-NEXT:    addvl sp, sp, #-4
+; CHECK-i64-NEXT:    addvl x8, sp, #4
+; CHECK-i64-NEXT:    str q1, [sp, #240] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q1, [x8, #272]
+; CHECK-i64-NEXT:    addvl x8, sp, #4
+; CHECK-i64-NEXT:    str q0, [sp, #224] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    stp q7, q6, [sp, #128] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    str q1, [sp, #112] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q1, [x8, #288]
+; CHECK-i64-NEXT:    addvl x8, sp, #4
+; CHECK-i64-NEXT:    stp q5, q4, [sp, #160] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    str q1, [sp, #96] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q1, [x8, #304]
+; CHECK-i64-NEXT:    addvl x8, sp, #4
+; CHECK-i64-NEXT:    stp q3, q2, [sp, #192] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    str q1, [sp, #80] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q1, [x8, #320]
+; CHECK-i64-NEXT:    addvl x8, sp, #4
+; CHECK-i64-NEXT:    str q1, [sp, #64] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q1, [x8, #336]
+; CHECK-i64-NEXT:    addvl x8, sp, #4
+; CHECK-i64-NEXT:    str q1, [sp, #48] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q1, [x8, #352]
+; CHECK-i64-NEXT:    addvl x8, sp, #4
+; CHECK-i64-NEXT:    str q1, [sp, #32] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q1, [x8, #368]
+; CHECK-i64-NEXT:    addvl x8, sp, #4
+; CHECK-i64-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q1, [x8, #384]
+; CHECK-i64-NEXT:    mov v0.16b, v1.16b
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #256
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str z0, [x8, #3, mul vl] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #256
+; CHECK-i64-NEXT:    ptrue p0.d, vl2
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    ldr z1, [x8, #3, mul vl] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i64-NEXT:    str z0, [x8, #3, mul vl] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #64] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #256
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str z0, [x8, #2, mul vl] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #96] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #256
+; CHECK-i64-NEXT:    ptrue p0.d, vl2
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    ldr z1, [x8, #2, mul vl] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i64-NEXT:    str z0, [x8, #2, mul vl] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #128] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #128] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #144] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #128] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #256
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #160] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #160] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #176] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #160] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #256
+; CHECK-i64-NEXT:    ptrue p0.d, vl2
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    ldr z1, [x8, #1, mul vl] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i64-NEXT:    str z0, [x8, #1, mul vl] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #192] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #192] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #208] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #192] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #256
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str z0, [x8] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #240] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #240] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #224] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #240] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add x8, sp, #256
+; CHECK-i64-NEXT:    ptrue p0.d, vl2
+; CHECK-i64-NEXT:    ldr z2, [x8, #1, mul vl] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ldr z4, [x8, #2, mul vl] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ldr z6, [x8, #3, mul vl] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    ldr z1, [x8] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    movprfx z3, z2
+; CHECK-i64-NEXT:    ext z3.b, z3.b, z2.b, #16
+; CHECK-i64-NEXT:    movprfx z5, z4
+; CHECK-i64-NEXT:    ext z5.b, z5.b, z4.b, #16
+; CHECK-i64-NEXT:    // kill: def $q2 killed $q2 killed $z2
+; CHECK-i64-NEXT:    // kill: def $q4 killed $q4 killed $z4
+; CHECK-i64-NEXT:    movprfx z7, z6
+; CHECK-i64-NEXT:    ext z7.b, z7.b, z6.b, #16
+; CHECK-i64-NEXT:    // kill: def $q6 killed $q6 killed $z6
+; CHECK-i64-NEXT:    splice z0.d, p0, z0.d, z1.d
+; CHECK-i64-NEXT:    // kill: def $q3 killed $q3 killed $z3
+; CHECK-i64-NEXT:    // kill: def $q5 killed $q5 killed $z5
+; CHECK-i64-NEXT:    // kill: def $q7 killed $q7 killed $z7
+; CHECK-i64-NEXT:    movprfx z1, z0
+; CHECK-i64-NEXT:    ext z1.b, z1.b, z0.b, #16
+; CHECK-i64-NEXT:    // kill: def $q0 killed $q0 killed $z0
+; CHECK-i64-NEXT:    // kill: def $q1 killed $q1 killed $z1
+; CHECK-i64-NEXT:    addvl sp, sp, #4
+; CHECK-i64-NEXT:    add sp, sp, #256
+; CHECK-i64-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16fp128(<16 x fp128> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16fp128(<16 x fp128>)
diff --git a/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll b/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll
index 43744092a1348..71108f00a0054 100644
--- a/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll
@@ -94,8 +94,8 @@ define <vscale x 4 x i16> @test_signed_v4f32_v4i16(<vscale x 4 x float> %f) {
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z1.s
 ; CHECK-NEXT:    mov z1.s, #32767 // =0x7fff
-; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
 ; CHECK-NEXT:    fcvtzs z2.s, p1/m, z0.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
 ; CHECK-NEXT:    sel z0.s, p2, z1.s, z2.s
 ; CHECK-NEXT:    mov z0.s, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
@@ -264,37 +264,37 @@ define <vscale x 8 x i32> @test_signed_v8f64_v8i32(<vscale x 8 x double> %f) {
 ; CHECK-NEXT:    mov z6.d, #0xffffffff80000000
 ; CHECK-NEXT:    movk x8, #16863, lsl #48
 ; CHECK-NEXT:    mov z7.d, #0xffffffff80000000
-; CHECK-NEXT:    mov z24.d, #0xffffffff80000000
-; CHECK-NEXT:    mov z25.d, x8
-; CHECK-NEXT:    fcmuo p6.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    mov z25.d, #0x7fffffff
+; CHECK-NEXT:    mov z24.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, z4.d
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    fcmge p3.d, p0/z, z3.d, z4.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z2.d, z4.d
-; CHECK-NEXT:    mov z4.d, #0x7fffffff
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z2.d, z25.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z0.d, z24.d
 ; CHECK-NEXT:    fcvtzs z5.d, p1/m, z1.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z1.d, z25.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z1.d, z24.d
 ; CHECK-NEXT:    fcvtzs z6.d, p2/m, z0.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z2.d, z4.d
+; CHECK-NEXT:    mov z4.d, #0xffffffff80000000
 ; CHECK-NEXT:    fcvtzs z7.d, p3/m, z3.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z0.d, z25.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z3.d, z25.d
-; CHECK-NEXT:    fcvtzs z24.d, p4/m, z2.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z1.d, z1.d
-; CHECK-NEXT:    sel z0.d, p1, z4.d, z5.d
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z2.d, z24.d
+; CHECK-NEXT:    sel z1.d, p1, z25.d, z5.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z24.d
+; CHECK-NEXT:    fcvtzs z4.d, p2/m, z2.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    sel z0.d, p4, z25.d, z6.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z1.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z3.d, z3.d
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z2.d, z2.d
-; CHECK-NEXT:    sel z1.d, p2, z4.d, z6.d
-; CHECK-NEXT:    sel z2.d, p3, z4.d, z7.d
-; CHECK-NEXT:    sel z3.d, p5, z4.d, z24.d
+; CHECK-NEXT:    sel z2.d, p1, z25.d, z7.d
+; CHECK-NEXT:    sel z3.d, p5, z25.d, z4.d
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    mov z2.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p6/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    uzp1 z0.s, z1.s, z0.s
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z1.s
 ; CHECK-NEXT:    uzp1 z1.s, z3.s, z2.s
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -348,41 +348,41 @@ define <vscale x 8 x i16> @test_signed_v8f64_v8i16(<vscale x 8 x double> %f) {
 ; CHECK-NEXT:    mov z5.d, #-32768 // =0xffffffffffff8000
 ; CHECK-NEXT:    mov z4.d, x8
 ; CHECK-NEXT:    mov x8, #281200098803712 // =0xffc000000000
-; CHECK-NEXT:    mov z6.d, #-32768 // =0xffffffffffff8000
-; CHECK-NEXT:    movk x8, #16607, lsl #48
 ; CHECK-NEXT:    mov z7.d, #-32768 // =0xffffffffffff8000
+; CHECK-NEXT:    movk x8, #16607, lsl #48
+; CHECK-NEXT:    mov z24.d, #-32768 // =0xffffffffffff8000
 ; CHECK-NEXT:    mov z25.d, #32767 // =0x7fff
-; CHECK-NEXT:    mov z24.d, x8
-; CHECK-NEXT:    fcmuo p6.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    mov z6.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z3.d, z4.d
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z2.d, z4.d
 ; CHECK-NEXT:    fcmge p3.d, p0/z, z1.d, z4.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z0.d, z4.d
-; CHECK-NEXT:    mov z4.d, #-32768 // =0xffffffffffff8000
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z0.d, z24.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z2.d, z6.d
 ; CHECK-NEXT:    fcvtzs z5.d, p1/m, z3.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z24.d
-; CHECK-NEXT:    fcvtzs z6.d, p2/m, z2.d
-; CHECK-NEXT:    fcvtzs z7.d, p3/m, z1.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z2.d, z24.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z1.d, z24.d
-; CHECK-NEXT:    fcvtzs z4.d, p4/m, z0.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z3.d, z3.d
-; CHECK-NEXT:    sel z2.d, p1, z25.d, z5.d
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z6.d
+; CHECK-NEXT:    fcvtzs z7.d, p2/m, z2.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, z4.d
+; CHECK-NEXT:    fcvtzs z24.d, p3/m, z1.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    mov z3.d, #-32768 // =0xffffffffffff8000
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z0.d, z6.d
+; CHECK-NEXT:    sel z4.d, p1, z25.d, z5.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z1.d, z6.d
+; CHECK-NEXT:    fcvtzs z3.d, p2/m, z0.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    mov z4.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    sel z0.d, p2, z25.d, z6.d
-; CHECK-NEXT:    sel z1.d, p3, z25.d, z7.d
-; CHECK-NEXT:    sel z3.d, p5, z25.d, z4.d
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    sel z0.d, p4, z25.d, z7.d
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    mov z1.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    sel z1.d, p1, z25.d, z24.d
+; CHECK-NEXT:    sel z2.d, p5, z25.d, z3.d
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    uzp1 z0.s, z0.s, z2.s
-; CHECK-NEXT:    uzp1 z1.s, z3.s, z1.s
+; CHECK-NEXT:    uzp1 z0.s, z0.s, z4.s
+; CHECK-NEXT:    uzp1 z1.s, z2.s, z1.s
 ; CHECK-NEXT:    uzp1 z0.h, z1.h, z0.h
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -535,8 +535,8 @@ define <vscale x 4 x i16> @test_signed_v4f16_v4i16(<vscale x 4 x half> %f) {
 ; CHECK-NEXT:    mov z1.h, w8
 ; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z1.h
 ; CHECK-NEXT:    mov z1.s, #32767 // =0x7fff
-; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
 ; CHECK-NEXT:    fcvtzs z2.s, p1/m, z0.h
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
 ; CHECK-NEXT:    sel z0.s, p2, z1.s, z2.s
 ; CHECK-NEXT:    mov z0.s, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
@@ -556,8 +556,8 @@ define <vscale x 8 x i16> @test_signed_v8f16_v8i16(<vscale x 8 x half> %f) {
 ; CHECK-NEXT:    mov z1.h, w8
 ; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z1.h
 ; CHECK-NEXT:    mov z1.h, #32767 // =0x7fff
-; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
 ; CHECK-NEXT:    fcvtzs z2.h, p1/m, z0.h
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
 ; CHECK-NEXT:    sel z0.h, p2, z1.h, z2.h
 ; CHECK-NEXT:    mov z0.h, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll b/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll
index 1df28198711e1..123f6c55c20ab 100644
--- a/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll
@@ -49,16 +49,16 @@ define <vscale x 8 x i32> @test_signed_v8f32_v8i32(<vscale x 8 x float> %f) {
 ; CHECK-LABEL: test_signed_v8f32_v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-NEXT:    mov w8, #1333788671 // =0x4f7fffff
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-NEXT:    mov z4.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    fcmge p2.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    fcvtzu z2.s, p1/m, z0.s
 ; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z4.s
-; CHECK-NEXT:    fcmgt p0.s, p0/z, z1.s, z4.s
 ; CHECK-NEXT:    fcvtzu z3.s, p2/m, z1.s
+; CHECK-NEXT:    fcmgt p0.s, p0/z, z1.s, z4.s
 ; CHECK-NEXT:    mov z2.s, p1/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z3.s, p0/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z0.d, z2.d
@@ -95,13 +95,13 @@ define <vscale x 8 x i16> @test_signed_v8f32_v8i16(<vscale x 8 x float> %f) {
 ; CHECK-NEXT:    movk w8, #18303, lsl #16
 ; CHECK-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    mov z4.s, w8
+; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    fcvtzu z2.s, p1/m, z1.s
 ; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z4.s
 ; CHECK-NEXT:    mov z1.s, #65535 // =0xffff
-; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, z4.s
 ; CHECK-NEXT:    fcvtzu z3.s, p2/m, z0.s
+; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, z4.s
 ; CHECK-NEXT:    sel z0.s, p1, z1.s, z2.s
 ; CHECK-NEXT:    sel z1.s, p0, z1.s, z3.s
 ; CHECK-NEXT:    uzp1 z0.h, z1.h, z0.h
@@ -141,8 +141,8 @@ define <vscale x 4 x i64> @test_signed_v4f32_v4i64(<vscale x 4 x float> %f) {
 ; CHECK-NEXT:    fcmge p2.s, p0/z, z3.s, #0.0
 ; CHECK-NEXT:    fcvtzu z0.d, p1/m, z2.s
 ; CHECK-NEXT:    fcmgt p1.s, p0/z, z2.s, z4.s
-; CHECK-NEXT:    fcmgt p0.s, p0/z, z3.s, z4.s
 ; CHECK-NEXT:    fcvtzu z1.d, p2/m, z3.s
+; CHECK-NEXT:    fcmgt p0.s, p0/z, z3.s, z4.s
 ; CHECK-NEXT:    mov z0.d, p1/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z1.d, p0/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ret
@@ -187,13 +187,13 @@ define <vscale x 4 x i32> @test_signed_v4f64_v4i32(<vscale x 4 x double> %f) {
 ; CHECK-NEXT:    movk x8, #16879, lsl #48
 ; CHECK-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, #0.0
-; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, #0.0
 ; CHECK-NEXT:    mov z4.d, x8
+; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, #0.0
 ; CHECK-NEXT:    fcvtzu z2.d, p1/m, z1.d
 ; CHECK-NEXT:    fcmgt p1.d, p0/z, z1.d, z4.d
 ; CHECK-NEXT:    mov z1.d, #0xffffffff
-; CHECK-NEXT:    fcmgt p0.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    fcvtzu z3.d, p2/m, z0.d
+; CHECK-NEXT:    fcmgt p0.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    sel z1.d, p0, z1.d, z3.d
 ; CHECK-NEXT:    uzp1 z0.s, z1.s, z0.s
@@ -213,29 +213,29 @@ define <vscale x 8 x i32> @test_signed_v8f64_v8i32(<vscale x 8 x double> %f) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #281474974613504 // =0xffffffe00000
 ; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    movk x8, #16879, lsl #48
 ; CHECK-NEXT:    movi v5.2d, #0000000000000000
 ; CHECK-NEXT:    movi v6.2d, #0000000000000000
-; CHECK-NEXT:    movk x8, #16879, lsl #48
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, #0.0
+; CHECK-NEXT:    movi v24.2d, #0000000000000000
+; CHECK-NEXT:    mov z7.d, x8
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, #0.0
 ; CHECK-NEXT:    fcmge p3.d, p0/z, z3.d, #0.0
-; CHECK-NEXT:    movi v7.2d, #0000000000000000
 ; CHECK-NEXT:    fcmge p4.d, p0/z, z2.d, #0.0
-; CHECK-NEXT:    mov z24.d, x8
 ; CHECK-NEXT:    fcvtzu z4.d, p1/m, z1.d
 ; CHECK-NEXT:    fcvtzu z5.d, p2/m, z0.d
 ; CHECK-NEXT:    fcvtzu z6.d, p3/m, z3.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z1.d, z24.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z0.d, z24.d
-; CHECK-NEXT:    mov z0.d, #0xffffffff
-; CHECK-NEXT:    fcvtzu z7.d, p4/m, z2.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z3.d, z24.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z1.d, z7.d
+; CHECK-NEXT:    fcvtzu z24.d, p4/m, z2.d
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    fcmgt p0.d, p0/z, z2.d, z24.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z0.d, z7.d
+; CHECK-NEXT:    mov z0.d, #0xffffffff
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z3.d, z7.d
+; CHECK-NEXT:    fcmgt p0.d, p0/z, z2.d, z7.d
 ; CHECK-NEXT:    sel z1.d, p1, z0.d, z4.d
 ; CHECK-NEXT:    sel z2.d, p2, z0.d, z5.d
 ; CHECK-NEXT:    sel z3.d, p3, z0.d, z6.d
-; CHECK-NEXT:    sel z4.d, p0, z0.d, z7.d
+; CHECK-NEXT:    sel z4.d, p0, z0.d, z24.d
 ; CHECK-NEXT:    uzp1 z0.s, z2.s, z1.s
 ; CHECK-NEXT:    uzp1 z1.s, z4.s, z3.s
 ; CHECK-NEXT:    addvl sp, sp, #1
@@ -254,13 +254,13 @@ define <vscale x 4 x i16> @test_signed_v4f64_v4i16(<vscale x 4 x double> %f) {
 ; CHECK-NEXT:    movk x8, #16623, lsl #48
 ; CHECK-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, #0.0
-; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, #0.0
 ; CHECK-NEXT:    mov z4.d, x8
+; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, #0.0
 ; CHECK-NEXT:    fcvtzu z2.d, p1/m, z1.d
 ; CHECK-NEXT:    fcmgt p1.d, p0/z, z1.d, z4.d
 ; CHECK-NEXT:    mov z1.d, #65535 // =0xffff
-; CHECK-NEXT:    fcmgt p0.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    fcvtzu z3.d, p2/m, z0.d
+; CHECK-NEXT:    fcmgt p0.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    sel z1.d, p0, z1.d, z3.d
 ; CHECK-NEXT:    uzp1 z0.s, z1.s, z0.s
@@ -280,29 +280,29 @@ define <vscale x 8 x i16> @test_signed_v8f64_v8i16(<vscale x 8 x double> %f) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #281337537757184 // =0xffe000000000
 ; CHECK-NEXT:    movi v4.2d, #0000000000000000
+; CHECK-NEXT:    movk x8, #16623, lsl #48
 ; CHECK-NEXT:    movi v5.2d, #0000000000000000
 ; CHECK-NEXT:    movi v6.2d, #0000000000000000
-; CHECK-NEXT:    movk x8, #16623, lsl #48
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z3.d, #0.0
+; CHECK-NEXT:    movi v24.2d, #0000000000000000
+; CHECK-NEXT:    mov z7.d, x8
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z2.d, #0.0
 ; CHECK-NEXT:    fcmge p3.d, p0/z, z1.d, #0.0
-; CHECK-NEXT:    movi v7.2d, #0000000000000000
 ; CHECK-NEXT:    fcmge p4.d, p0/z, z0.d, #0.0
-; CHECK-NEXT:    mov z24.d, x8
 ; CHECK-NEXT:    fcvtzu z4.d, p1/m, z3.d
 ; CHECK-NEXT:    fcvtzu z5.d, p2/m, z2.d
 ; CHECK-NEXT:    fcvtzu z6.d, p3/m, z1.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z24.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z2.d, z24.d
-; CHECK-NEXT:    mov z2.d, #65535 // =0xffff
-; CHECK-NEXT:    fcvtzu z7.d, p4/m, z0.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z1.d, z24.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z7.d
+; CHECK-NEXT:    fcvtzu z24.d, p4/m, z0.d
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    fcmgt p0.d, p0/z, z0.d, z24.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z2.d, z7.d
+; CHECK-NEXT:    mov z2.d, #65535 // =0xffff
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z1.d, z7.d
+; CHECK-NEXT:    fcmgt p0.d, p0/z, z0.d, z7.d
 ; CHECK-NEXT:    sel z0.d, p1, z2.d, z4.d
 ; CHECK-NEXT:    sel z1.d, p2, z2.d, z5.d
 ; CHECK-NEXT:    sel z3.d, p3, z2.d, z6.d
-; CHECK-NEXT:    sel z2.d, p0, z2.d, z7.d
+; CHECK-NEXT:    sel z2.d, p0, z2.d, z24.d
 ; CHECK-NEXT:    uzp1 z0.s, z1.s, z0.s
 ; CHECK-NEXT:    uzp1 z1.s, z2.s, z3.s
 ; CHECK-NEXT:    uzp1 z0.h, z1.h, z0.h
@@ -334,16 +334,16 @@ define <vscale x 4 x i64> @test_signed_v4f64_v4i64(<vscale x 4 x double> %f) {
 ; CHECK-LABEL: test_signed_v4f64_v4i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-NEXT:    mov x8, #4895412794951729151 // =0x43efffffffffffff
+; CHECK-NEXT:    movi v2.2d, #0000000000000000
 ; CHECK-NEXT:    movi v3.2d, #0000000000000000
 ; CHECK-NEXT:    mov z4.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, #0.0
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, #0.0
 ; CHECK-NEXT:    fcvtzu z2.d, p1/m, z0.d
 ; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z4.d
-; CHECK-NEXT:    fcmgt p0.d, p0/z, z1.d, z4.d
 ; CHECK-NEXT:    fcvtzu z3.d, p2/m, z1.d
+; CHECK-NEXT:    fcmgt p0.d, p0/z, z1.d, z4.d
 ; CHECK-NEXT:    mov z2.d, p1/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z3.d, p0/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z0.d, z2.d
@@ -412,8 +412,8 @@ define <vscale x 8 x i32> @test_signed_v8f16_v8i32(<vscale x 8 x half> %f) {
 ; CHECK-NEXT:    fcmge p2.h, p0/z, z3.h, #0.0
 ; CHECK-NEXT:    fcvtzu z0.s, p1/m, z2.h
 ; CHECK-NEXT:    fcmgt p1.h, p0/z, z2.h, z4.h
-; CHECK-NEXT:    fcmgt p0.h, p0/z, z3.h, z4.h
 ; CHECK-NEXT:    fcvtzu z1.s, p2/m, z3.h
+; CHECK-NEXT:    fcmgt p0.h, p0/z, z3.h, z4.h
 ; CHECK-NEXT:    mov z0.s, p1/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z1.s, p0/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ret
@@ -486,8 +486,8 @@ define <vscale x 4 x i64> @test_signed_v4f16_v4i64(<vscale x 4 x half> %f) {
 ; CHECK-NEXT:    fcmge p2.h, p0/z, z3.h, #0.0
 ; CHECK-NEXT:    fcvtzu z0.d, p1/m, z2.h
 ; CHECK-NEXT:    fcmgt p1.h, p0/z, z2.h, z4.h
-; CHECK-NEXT:    fcmgt p0.h, p0/z, z3.h, z4.h
 ; CHECK-NEXT:    fcvtzu z1.d, p2/m, z3.h
+; CHECK-NEXT:    fcmgt p0.h, p0/z, z3.h, z4.h
 ; CHECK-NEXT:    mov z0.d, p1/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z1.d, p0/m, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.mir b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.mir
index f85658e9a596a..a342ee8e6405e 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.mir
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-merging.mir
@@ -28,7 +28,7 @@ body:             |
     ; CHECK-LABEL: name: bic_i16_zero
     ; CHECK: liveins: $p0, $z0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $h0, implicit-def $b0, implicit-def $b0_hi, implicit-def $h0_hi, implicit-def $s0_hi, implicit-def $d0_hi, implicit-def $q0_hi, implicit killed $p0, implicit $z0 {
+    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit killed $p0, implicit $z0 {
     ; CHECK-NEXT:   $z0 = MOVPRFX_ZPzZ_H $p0, $z0
     ; CHECK-NEXT:   $z0 = LSL_ZPmI_H killed renamable $p0, internal $z0, 0
     ; CHECK-NEXT:   $z0 = BIC_ZPmZ_H killed renamable $p0, internal killed $z0, internal killed renamable $z0
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryComm-merging.mir b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryComm-merging.mir
index 0e2f1c3ff4f69..f01ca7f8b73c6 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryComm-merging.mir
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryComm-merging.mir
@@ -28,7 +28,7 @@ body:             |
     ; CHECK-LABEL: name: fmul_float_zero
     ; CHECK: liveins: $p0, $z0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $h0, implicit-def $b0, implicit-def $b0_hi, implicit-def $h0_hi, implicit-def $s0_hi, implicit-def $d0_hi, implicit-def $q0_hi, implicit $p0, implicit $z0 {
+    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit $p0, implicit $z0 {
     ; CHECK-NEXT:   $z0 = MOVPRFX_ZPzZ_S $p0, $z0
     ; CHECK-NEXT:   $z0 = LSL_ZPmI_S renamable $p0, internal $z0, 0
     ; CHECK-NEXT:   $z0 = FMUL_ZPmZ_S renamable $p0, internal killed $z0, internal killed renamable $z0
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryCommWithRev-merging.mir b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryCommWithRev-merging.mir
index 548ebc26b83e8..301d657006da4 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryCommWithRev-merging.mir
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-binaryCommWithRev-merging.mir
@@ -30,7 +30,7 @@ body:             |
     ; CHECK-LABEL: name: fsub_s_zero
     ; CHECK: liveins: $p0, $z0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $h0, implicit-def $b0, implicit-def $b0_hi, implicit-def $h0_hi, implicit-def $s0_hi, implicit-def $d0_hi, implicit-def $q0_hi, implicit $p0, implicit $z0 {
+    ; CHECK-NEXT: BUNDLE implicit-def $z0, implicit $p0, implicit $z0 {
     ; CHECK-NEXT:   $z0 = MOVPRFX_ZPzZ_S $p0, $z0
     ; CHECK-NEXT:   $z0 = LSL_ZPmI_S renamable $p0, internal $z0, 0
     ; CHECK-NEXT:   $z0 = FSUBR_ZPmZ_S renamable $p0, internal killed $z0, internal killed renamable $z0
diff --git a/llvm/test/CodeGen/AArch64/sve-llrint.ll b/llvm/test/CodeGen/AArch64/sve-llrint.ll
index 12d49183edea4..d5a4838ff9687 100644
--- a/llvm/test/CodeGen/AArch64/sve-llrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-llrint.ll
@@ -6,17 +6,17 @@ define <vscale x 1 x i64> @llrint_v1i64_v1f16(<vscale x 1 x half> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.h, w8
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
 ; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
-; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z2.h
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f16(<vscale x 1 x half> %x)
@@ -29,17 +29,17 @@ define <vscale x 2 x i64> @llrint_v1i64_v2f16(<vscale x 2 x half> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.h, w8
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
 ; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
-; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z2.h
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f16(<vscale x 2 x half> %x)
@@ -58,20 +58,20 @@ define <vscale x 4 x i64> @llrint_v4i64_v4f16(<vscale x 4 x half> %x) {
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z4.d, #0x8000000000000000
-; CHECK-NEXT:    mov z5.h, w8
+; CHECK-NEXT:    mov z5.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    frintx z1.h, p0/m, z1.h
 ; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z1.h, z2.h
 ; CHECK-NEXT:    fcmge p2.h, p0/z, z0.h, z2.h
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    fcmuo p3.h, p0/z, z1.h, z1.h
-; CHECK-NEXT:    fcvtzs z3.d, p1/m, z1.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z1.h, z5.h
-; CHECK-NEXT:    fcvtzs z4.d, p2/m, z0.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z5.h
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z1.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z1.h, z2.h
+; CHECK-NEXT:    fcvtzs z3.d, p2/m, z0.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z2.h
 ; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z3.d
-; CHECK-NEXT:    sel z1.d, p2, z2.d, z4.d
+; CHECK-NEXT:    sel z0.d, p1, z5.d, z4.d
+; CHECK-NEXT:    sel z1.d, p2, z5.d, z3.d
 ; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
@@ -96,10 +96,10 @@ define <vscale x 8 x i64> @llrint_v8i64_v8f16(<vscale x 8 x half> %x) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov z4.h, w8
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z6.d, #0x8000000000000000
-; CHECK-NEXT:    mov z25.h, w8
 ; CHECK-NEXT:    mov z7.d, #0x8000000000000000
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
+; CHECK-NEXT:    mov z25.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    uunpklo z2.d, z1.s
 ; CHECK-NEXT:    uunpkhi z1.d, z1.s
 ; CHECK-NEXT:    uunpklo z3.d, z0.s
@@ -107,36 +107,36 @@ define <vscale x 8 x i64> @llrint_v8i64_v8f16(<vscale x 8 x half> %x) {
 ; CHECK-NEXT:    frintx z2.h, p0/m, z2.h
 ; CHECK-NEXT:    frintx z1.h, p0/m, z1.h
 ; CHECK-NEXT:    frintx z3.h, p0/m, z3.h
-; CHECK-NEXT:    movprfx z5, z0
-; CHECK-NEXT:    frintx z5.h, p0/m, z0.h
-; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z24, z0
+; CHECK-NEXT:    frintx z24.h, p0/m, z0.h
+; CHECK-NEXT:    mov z0.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z2.h, z4.h
 ; CHECK-NEXT:    fcmge p2.h, p0/z, z1.h, z4.h
 ; CHECK-NEXT:    fcmge p3.h, p0/z, z3.h, z4.h
-; CHECK-NEXT:    fcmge p4.h, p0/z, z5.h, z4.h
-; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z5.h, z25.h
-; CHECK-NEXT:    fcmuo p6.h, p0/z, z1.h, z1.h
-; CHECK-NEXT:    fcvtzs z0.d, p1/m, z2.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z1.h, z0.h
+; CHECK-NEXT:    fcvtzs z5.d, p1/m, z2.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z24.h, z4.h
+; CHECK-NEXT:    mov z4.d, #0x8000000000000000
 ; CHECK-NEXT:    fcvtzs z6.d, p2/m, z1.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z2.h, z25.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z1.h, z25.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z2.h, z0.h
 ; CHECK-NEXT:    fcvtzs z7.d, p3/m, z3.h
-; CHECK-NEXT:    fcmgt p3.h, p0/z, z3.h, z25.h
-; CHECK-NEXT:    fcvtzs z24.d, p4/m, z5.h
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z2.h, z2.h
-; CHECK-NEXT:    mov z0.d, p1/m, z4.d
-; CHECK-NEXT:    fcmuo p1.h, p0/z, z3.h, z3.h
-; CHECK-NEXT:    fcmuo p0.h, p0/z, z5.h, z5.h
-; CHECK-NEXT:    sel z1.d, p2, z4.d, z6.d
-; CHECK-NEXT:    sel z2.d, p3, z4.d, z7.d
-; CHECK-NEXT:    sel z3.d, p5, z4.d, z24.d
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z3.h, z0.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z24.h, z0.h
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z24.h
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z2.h, z2.h
+; CHECK-NEXT:    sel z0.d, p2, z25.d, z5.d
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    sel z1.d, p4, z25.d, z6.d
+; CHECK-NEXT:    sel z2.d, p5, z25.d, z7.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z3.h, z3.h
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z24.h, z24.h
+; CHECK-NEXT:    sel z3.d, p6, z25.d, z4.d
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -150,7 +150,7 @@ define <vscale x 16 x i64> @llrint_v16i64_v16f16(<vscale x 16 x half> %x) {
 ; CHECK-LABEL: llrint_v16i64_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-4
+; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
@@ -158,110 +158,102 @@ define <vscale x 16 x i64> @llrint_v16i64_v16f16(<vscale x 16 x half> %x) {
 ; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16
 ; CHECK-NEXT:    uunpklo z2.s, z0.h
-; CHECK-NEXT:    uunpkhi z0.s, z0.h
+; CHECK-NEXT:    uunpkhi z3.s, z0.h
 ; CHECK-NEXT:    mov w8, #64511 // =0xfbff
-; CHECK-NEXT:    uunpklo z3.s, z1.h
-; CHECK-NEXT:    uunpkhi z1.s, z1.h
+; CHECK-NEXT:    uunpklo z7.s, z1.h
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov z24.h, w8
-; CHECK-NEXT:    mov w8, #31743 // =0x7bff
-; CHECK-NEXT:    mov z7.d, #0x8000000000000000
-; CHECK-NEXT:    mov z27.d, #0x8000000000000000
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    mov z30.d, #0x8000000000000000
-; CHECK-NEXT:    uunpklo z4.d, z2.s
-; CHECK-NEXT:    uunpkhi z2.d, z2.s
-; CHECK-NEXT:    uunpklo z5.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    uunpklo z6.d, z3.s
-; CHECK-NEXT:    uunpklo z25.d, z1.s
-; CHECK-NEXT:    uunpkhi z3.d, z3.s
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    uunpkhi z1.s, z1.h
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    mov z10.d, #0x7fffffffffffffff
-; CHECK-NEXT:    frintx z4.h, p0/m, z4.h
-; CHECK-NEXT:    frintx z2.h, p0/m, z2.h
-; CHECK-NEXT:    frintx z5.h, p0/m, z5.h
-; CHECK-NEXT:    movprfx z26, z0
-; CHECK-NEXT:    frintx z26.h, p0/m, z0.h
-; CHECK-NEXT:    uunpkhi z0.d, z1.s
-; CHECK-NEXT:    frintx z6.h, p0/m, z6.h
-; CHECK-NEXT:    movprfx z29, z3
-; CHECK-NEXT:    frintx z29.h, p0/m, z3.h
-; CHECK-NEXT:    frintx z25.h, p0/m, z25.h
-; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    uunpklo z4.d, z2.s
+; CHECK-NEXT:    uunpklo z24.d, z3.s
+; CHECK-NEXT:    uunpkhi z25.d, z3.s
+; CHECK-NEXT:    uunpkhi z6.d, z2.s
+; CHECK-NEXT:    uunpklo z26.d, z7.s
+; CHECK-NEXT:    uunpkhi z7.d, z7.s
+; CHECK-NEXT:    mov z2.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    uunpklo z30.d, z1.s
+; CHECK-NEXT:    mov z29.h, w8
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.h, p0/z, z4.h, z24.h
-; CHECK-NEXT:    fcmge p2.h, p0/z, z2.h, z24.h
-; CHECK-NEXT:    fcmge p3.h, p0/z, z5.h, z24.h
-; CHECK-NEXT:    movprfx z9, z0
-; CHECK-NEXT:    frintx z9.h, p0/m, z0.h
-; CHECK-NEXT:    fcmge p4.h, p0/z, z26.h, z24.h
-; CHECK-NEXT:    fcmge p5.h, p0/z, z6.h, z24.h
-; CHECK-NEXT:    fcmge p7.h, p0/z, z25.h, z24.h
-; CHECK-NEXT:    fcmge p6.h, p0/z, z29.h, z24.h
-; CHECK-NEXT:    fcmgt p8.h, p0/z, z6.h, z1.h
-; CHECK-NEXT:    fcmgt p10.h, p0/z, z25.h, z1.h
-; CHECK-NEXT:    fcmuo p9.h, p0/z, z5.h, z5.h
-; CHECK-NEXT:    fcvtzs z7.d, p1/m, z4.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z4.h, z1.h
-; CHECK-NEXT:    fcvtzs z27.d, p2/m, z2.h
-; CHECK-NEXT:    fcmge p2.h, p0/z, z9.h, z24.h
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z28.d, p3/m, z5.h
-; CHECK-NEXT:    fcvtzs z3.d, p4/m, z26.h
-; CHECK-NEXT:    fcvtzs z30.d, p5/m, z6.h
-; CHECK-NEXT:    fcvtzs z8.d, p7/m, z25.h
-; CHECK-NEXT:    fcmgt p4.h, p0/z, z2.h, z1.h
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z5.h, z1.h
-; CHECK-NEXT:    fcmgt p7.h, p0/z, z26.h, z1.h
-; CHECK-NEXT:    fcvtzs z31.d, p6/m, z29.h
-; CHECK-NEXT:    sel z0.d, p1, z10.d, z7.d
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z29.h, z1.h
-; CHECK-NEXT:    fcvtzs z24.d, p2/m, z9.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z9.h, z1.h
-; CHECK-NEXT:    fcmuo p3.h, p0/z, z4.h, z4.h
-; CHECK-NEXT:    fcmuo p6.h, p0/z, z2.h, z2.h
-; CHECK-NEXT:    sel z4.d, p8, z10.d, z30.d
-; CHECK-NEXT:    fcmuo p8.h, p0/z, z25.h, z25.h
-; CHECK-NEXT:    sel z1.d, p4, z10.d, z27.d
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z26.h, z26.h
-; CHECK-NEXT:    sel z2.d, p5, z10.d, z28.d
-; CHECK-NEXT:    mov z3.d, p7/m, z10.d
-; CHECK-NEXT:    fcmuo p5.h, p0/z, z6.h, z6.h
-; CHECK-NEXT:    fcmuo p7.h, p0/z, z29.h, z29.h
-; CHECK-NEXT:    sel z5.d, p1, z10.d, z31.d
-; CHECK-NEXT:    sel z6.d, p10, z10.d, z8.d
-; CHECK-NEXT:    ldr z8, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    movprfx z27, z4
+; CHECK-NEXT:    frintx z27.h, p0/m, z4.h
+; CHECK-NEXT:    frintx z24.h, p0/m, z24.h
+; CHECK-NEXT:    frintx z25.h, p0/m, z25.h
+; CHECK-NEXT:    movprfx z28, z6
+; CHECK-NEXT:    frintx z28.h, p0/m, z6.h
+; CHECK-NEXT:    mov z4.d, #0x8000000000000000
+; CHECK-NEXT:    frintx z26.h, p0/m, z26.h
+; CHECK-NEXT:    frintx z7.h, p0/m, z7.h
+; CHECK-NEXT:    mov z6.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p1.h, p0/z, z27.h, z2.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z24.h, z2.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z25.h, z2.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z28.h, z2.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z26.h, z2.h
+; CHECK-NEXT:    fcvtzs z0.d, p1/m, z27.h
+; CHECK-NEXT:    fcvtzs z4.d, p3/m, z24.h
+; CHECK-NEXT:    fcvtzs z5.d, p4/m, z25.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z27.h, z29.h
+; CHECK-NEXT:    fcvtzs z3.d, p2/m, z28.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z7.h, z2.h
+; CHECK-NEXT:    fcvtzs z6.d, p5/m, z26.h
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z27.h, z27.h
+; CHECK-NEXT:    movprfx z27, z30
+; CHECK-NEXT:    frintx z27.h, p0/m, z30.h
+; CHECK-NEXT:    movprfx z30, z1
+; CHECK-NEXT:    frintx z30.h, p0/m, z1.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z28.h, z29.h
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z28.h, z28.h
+; CHECK-NEXT:    mov z28.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z31.d, p4/m, z7.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z27.h, z2.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z24.h, z29.h
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z24.h, z24.h
+; CHECK-NEXT:    mov z24.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcmgt p8.h, p0/z, z25.h, z29.h
+; CHECK-NEXT:    fcvtzs z28.d, p4/m, z27.h
+; CHECK-NEXT:    fcmuo p10.h, p0/z, z25.h, z25.h
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
+; CHECK-NEXT:    sel z1.d, p5, z24.d, z3.d
+; CHECK-NEXT:    mov z0.d, p3/m, z24.d
+; CHECK-NEXT:    sel z3.d, p8, z24.d, z5.d
+; CHECK-NEXT:    fcmge p4.h, p0/z, z30.h, z2.h
+; CHECK-NEXT:    sel z2.d, p6, z24.d, z4.d
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z3.d, p10/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    fcmuo p0.h, p0/z, z9.h, z9.h
-; CHECK-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    sel z7.d, p2, z10.d, z24.d
-; CHECK-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
-; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    mov z5.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p7/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z6.d, p8/m, #0 // =0x0
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p9.h, p0/z, z26.h, z29.h
+; CHECK-NEXT:    fcvtzs z25.d, p4/m, z30.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z7.h, z29.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z27.h, z29.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z30.h, z29.h
+; CHECK-NEXT:    sel z4.d, p9, z24.d, z6.d
+; CHECK-NEXT:    fcmuo p8.h, p0/z, z7.h, z7.h
+; CHECK-NEXT:    sel z5.d, p5, z24.d, z31.d
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z6.d, p6, z24.d, z28.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    fcmuo p9.h, p0/z, z27.h, z27.h
+; CHECK-NEXT:    sel z7.d, p4, z24.d, z25.d
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z26.h, z26.h
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z30.h, z30.h
+; CHECK-NEXT:    mov z6.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f16(<vscale x 16 x half> %x)
@@ -273,32 +265,32 @@ define <vscale x 32 x i64> @llrint_v32i64_v32f16(<vscale x 32 x half> %x) {
 ; CHECK-LABEL: llrint_v32i64_v32f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-17
-; CHECK-NEXT:    str p11, [sp] // 2-byte Folded Spill
-; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
@@ -310,216 +302,215 @@ define <vscale x 32 x i64> @llrint_v32i64_v32f16(<vscale x 32 x half> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x40, 0x1c // $d14 @ cfa - 56 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16
 ; CHECK-NEXT:    uunpklo z4.s, z0.h
-; CHECK-NEXT:    uunpkhi z0.s, z0.h
+; CHECK-NEXT:    uunpkhi z5.s, z0.h
 ; CHECK-NEXT:    mov w9, #64511 // =0xfbff
-; CHECK-NEXT:    uunpklo z25.s, z1.h
-; CHECK-NEXT:    uunpkhi z10.s, z1.h
-; CHECK-NEXT:    uunpklo z9.s, z2.h
+; CHECK-NEXT:    uunpklo z6.s, z1.h
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uunpkhi z12.s, z3.h
-; CHECK-NEXT:    mov z27.h, w9
+; CHECK-NEXT:    uunpkhi z28.s, z1.h
+; CHECK-NEXT:    mov z30.h, w9
 ; CHECK-NEXT:    mov w9, #31743 // =0x7bff
+; CHECK-NEXT:    uunpklo z13.s, z2.h
+; CHECK-NEXT:    mov z9.d, #0x8000000000000000
 ; CHECK-NEXT:    uunpkhi z14.s, z2.h
+; CHECK-NEXT:    uunpkhi z17.s, z3.h
+; CHECK-NEXT:    uunpklo z7.d, z4.s
+; CHECK-NEXT:    uunpkhi z4.d, z4.s
+; CHECK-NEXT:    uunpklo z27.d, z5.s
+; CHECK-NEXT:    uunpklo z31.d, z6.s
+; CHECK-NEXT:    uunpkhi z8.d, z6.s
+; CHECK-NEXT:    uunpkhi z29.d, z5.s
+; CHECK-NEXT:    uunpkhi z11.d, z28.s
+; CHECK-NEXT:    uunpklo z10.d, z28.s
 ; CHECK-NEXT:    uunpklo z15.s, z3.h
-; CHECK-NEXT:    uunpklo z7.d, z0.s
-; CHECK-NEXT:    uunpklo z5.d, z4.s
-; CHECK-NEXT:    uunpkhi z6.d, z4.s
-; CHECK-NEXT:    uunpklo z29.d, z25.s
-; CHECK-NEXT:    uunpkhi z26.d, z0.s
-; CHECK-NEXT:    uunpklo z8.d, z10.s
-; CHECK-NEXT:    uunpkhi z11.d, z10.s
-; CHECK-NEXT:    uunpklo z10.d, z9.s
-; CHECK-NEXT:    uunpkhi z13.d, z9.s
-; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    uunpklo z16.d, z12.s
-; CHECK-NEXT:    uunpklo z18.d, z14.s
+; CHECK-NEXT:    uunpklo z16.d, z14.s
+; CHECK-NEXT:    uunpkhi z14.d, z14.s
+; CHECK-NEXT:    mov z24.d, #0x8000000000000000
 ; CHECK-NEXT:    movprfx z1, z7
 ; CHECK-NEXT:    frintx z1.h, p0/m, z7.h
-; CHECK-NEXT:    movprfx z4, z5
-; CHECK-NEXT:    frintx z4.h, p0/m, z5.h
-; CHECK-NEXT:    movprfx z5, z6
-; CHECK-NEXT:    frintx z5.h, p0/m, z6.h
-; CHECK-NEXT:    movprfx z7, z29
-; CHECK-NEXT:    frintx z7.h, p0/m, z29.h
-; CHECK-NEXT:    movprfx z6, z26
-; CHECK-NEXT:    frintx z6.h, p0/m, z26.h
-; CHECK-NEXT:    mov z29.d, #0x8000000000000000
-; CHECK-NEXT:    movprfx z9, z11
-; CHECK-NEXT:    frintx z9.h, p0/m, z11.h
-; CHECK-NEXT:    movprfx z3, z10
-; CHECK-NEXT:    frintx z3.h, p0/m, z10.h
-; CHECK-NEXT:    movprfx z10, z13
-; CHECK-NEXT:    frintx z10.h, p0/m, z13.h
-; CHECK-NEXT:    uunpkhi z26.d, z25.s
-; CHECK-NEXT:    uunpkhi z13.d, z12.s
-; CHECK-NEXT:    frintx z8.h, p0/m, z8.h
-; CHECK-NEXT:    fcmge p3.h, p0/z, z1.h, z27.h
-; CHECK-NEXT:    uunpkhi z14.d, z14.s
+; CHECK-NEXT:    movprfx z5, z27
+; CHECK-NEXT:    frintx z5.h, p0/m, z27.h
+; CHECK-NEXT:    frintx z4.h, p0/m, z4.h
+; CHECK-NEXT:    movprfx z12, z31
+; CHECK-NEXT:    frintx z12.h, p0/m, z31.h
+; CHECK-NEXT:    movprfx z27, z8
+; CHECK-NEXT:    frintx z27.h, p0/m, z8.h
+; CHECK-NEXT:    movprfx z6, z29
+; CHECK-NEXT:    frintx z6.h, p0/m, z29.h
+; CHECK-NEXT:    movprfx z31, z10
+; CHECK-NEXT:    frintx z31.h, p0/m, z10.h
+; CHECK-NEXT:    mov z7.d, #0x8000000000000000
+; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z3, z16
+; CHECK-NEXT:    frintx z3.h, p0/m, z16.h
+; CHECK-NEXT:    frintx z11.h, p0/m, z11.h
+; CHECK-NEXT:    mov z29.h, w9
+; CHECK-NEXT:    uunpklo z10.d, z13.s
+; CHECK-NEXT:    uunpkhi z13.d, z13.s
+; CHECK-NEXT:    uunpkhi z20.d, z15.s
+; CHECK-NEXT:    uunpklo z16.d, z17.s
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z0.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p2.h, p0/z, z7.h, z27.h
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p4.h, p0/z, z4.h, z27.h
-; CHECK-NEXT:    fcmge p5.h, p0/z, z5.h, z27.h
-; CHECK-NEXT:    uunpklo z19.d, z15.s
-; CHECK-NEXT:    uunpkhi z15.d, z15.s
-; CHECK-NEXT:    movprfx z20, z13
-; CHECK-NEXT:    frintx z20.h, p0/m, z13.h
-; CHECK-NEXT:    mov z30.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.h, p0/z, z6.h, z27.h
-; CHECK-NEXT:    frintx z26.h, p0/m, z26.h
-; CHECK-NEXT:    fcvtzs z29.d, p3/m, z1.h
-; CHECK-NEXT:    fcmge p3.h, p0/z, z9.h, z27.h
-; CHECK-NEXT:    mov z11.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z31.d, p2/m, z7.h
-; CHECK-NEXT:    fcmge p2.h, p0/z, z8.h, z27.h
-; CHECK-NEXT:    mov z17.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z16.h, p0/m, z16.h
-; CHECK-NEXT:    frintx z14.h, p0/m, z14.h
-; CHECK-NEXT:    fcvtzs z0.d, p4/m, z4.h
-; CHECK-NEXT:    fcvtzs z28.d, p5/m, z5.h
-; CHECK-NEXT:    fcmge p4.h, p0/z, z3.h, z27.h
-; CHECK-NEXT:    mov z12.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.h, p0/z, z10.h, z27.h
-; CHECK-NEXT:    mov z13.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z19.h, p0/m, z19.h
-; CHECK-NEXT:    frintx z15.h, p0/m, z15.h
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z30.d, p1/m, z6.h
-; CHECK-NEXT:    fcmge p1.h, p0/z, z26.h, z27.h
+; CHECK-NEXT:    mov z18.d, #0x8000000000000000
+; CHECK-NEXT:    uunpklo z15.d, z15.s
 ; CHECK-NEXT:    mov z2.d, #0x8000000000000000
-; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    frintx z18.h, p0/m, z18.h
-; CHECK-NEXT:    fcvtzs z11.d, p3/m, z9.h
-; CHECK-NEXT:    fcmge p3.h, p0/z, z20.h, z27.h
-; CHECK-NEXT:    mov z25.h, w9
-; CHECK-NEXT:    fcvtzs z17.d, p2/m, z8.h
-; CHECK-NEXT:    fcmge p6.h, p0/z, z16.h, z27.h
 ; CHECK-NEXT:    mov z21.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p2.h, p0/z, z14.h, z27.h
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
+; CHECK-NEXT:    mov z28.d, #0x7fffffffffffffff
+; CHECK-NEXT:    movprfx z19, z13
+; CHECK-NEXT:    frintx z19.h, p0/m, z13.h
+; CHECK-NEXT:    movprfx z13, z14
+; CHECK-NEXT:    frintx z13.h, p0/m, z14.h
+; CHECK-NEXT:    frintx z10.h, p0/m, z10.h
+; CHECK-NEXT:    frintx z16.h, p0/m, z16.h
 ; CHECK-NEXT:    mov z22.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z12.d, p4/m, z3.h
-; CHECK-NEXT:    fcvtzs z13.d, p5/m, z10.h
-; CHECK-NEXT:    fcmge p4.h, p0/z, z19.h, z27.h
 ; CHECK-NEXT:    mov z23.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.h, p0/z, z15.h, z27.h
-; CHECK-NEXT:    mov z0.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z2.d, p1/m, z26.h
-; CHECK-NEXT:    fcmge p1.h, p0/z, z18.h, z27.h
-; CHECK-NEXT:    fcvtzs z24.d, p3/m, z20.h
-; CHECK-NEXT:    mov z27.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p11.h, p0/z, z20.h, z25.h
-; CHECK-NEXT:    fcvtzs z21.d, p6/m, z16.h
-; CHECK-NEXT:    fcmgt p3.h, p0/z, z16.h, z25.h
-; CHECK-NEXT:    fcmuo p6.h, p0/z, z16.h, z16.h
-; CHECK-NEXT:    fcvtzs z22.d, p2/m, z14.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z8.h, z25.h
-; CHECK-NEXT:    mov z16.d, #0x8000000000000000
-; CHECK-NEXT:    fcmgt p7.h, p0/z, z5.h, z25.h
-; CHECK-NEXT:    fcvtzs z23.d, p4/m, z19.h
-; CHECK-NEXT:    fcvtzs z0.d, p5/m, z15.h
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z20.h, z20.h
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z15.h, z25.h
-; CHECK-NEXT:    mov z24.d, p11/m, z27.d
-; CHECK-NEXT:    sel z20.d, p3, z27.d, z21.d
-; CHECK-NEXT:    fcmgt p3.h, p0/z, z19.h, z25.h
-; CHECK-NEXT:    fcmgt p8.h, p0/z, z1.h, z25.h
-; CHECK-NEXT:    mov z17.d, p2/m, z27.d
-; CHECK-NEXT:    fcvtzs z16.d, p1/m, z18.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z18.h, z25.h
-; CHECK-NEXT:    mov z28.d, p7/m, z27.d
-; CHECK-NEXT:    fcmgt p7.h, p0/z, z14.h, z25.h
-; CHECK-NEXT:    fcmuo p1.h, p0/z, z15.h, z15.h
-; CHECK-NEXT:    mov z0.d, p5/m, z27.d
-; CHECK-NEXT:    mov z24.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z10.h, z25.h
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z19.h, z19.h
-; CHECK-NEXT:    sel z19.d, p3, z27.d, z23.d
-; CHECK-NEXT:    fcmuo p3.h, p0/z, z14.h, z14.h
-; CHECK-NEXT:    mov z20.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p9.h, p0/z, z6.h, z25.h
-; CHECK-NEXT:    fcmgt p10.h, p0/z, z7.h, z25.h
-; CHECK-NEXT:    str z24, [x8, #15, mul vl]
-; CHECK-NEXT:    sel z24.d, p2, z27.d, z16.d
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z3.h, z25.h
-; CHECK-NEXT:    sel z15.d, p7, z27.d, z22.d
-; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z13.d, p5/m, z27.d
-; CHECK-NEXT:    str z20, [x8, #14, mul vl]
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z9.h, z25.h
-; CHECK-NEXT:    fcmuo p1.h, p0/z, z18.h, z18.h
-; CHECK-NEXT:    mov z19.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z10.h, z10.h
-; CHECK-NEXT:    mov z29.d, p8/m, z27.d
-; CHECK-NEXT:    str z0, [x8, #13, mul vl]
-; CHECK-NEXT:    mov z15.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    frintx z15.h, p0/m, z15.h
+; CHECK-NEXT:    mov z14.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p4.h, p0/z, z4.h, z30.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z12.h, z30.h
+; CHECK-NEXT:    fcmgt p9.h, p0/z, z12.h, z29.h
+; CHECK-NEXT:    fcmuo p8.h, p0/z, z12.h, z12.h
+; CHECK-NEXT:    fcvtzs z7.d, p4/m, z4.h
+; CHECK-NEXT:    fcvtzs z8.d, p2/m, z12.h
+; CHECK-NEXT:    mov z12.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p4.h, p0/z, z27.h, z30.h
+; CHECK-NEXT:    fcmuo p10.h, p0/z, z11.h, z11.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z5.h, z30.h
+; CHECK-NEXT:    mov z8.d, p9/m, z28.d
+; CHECK-NEXT:    fcvtzs z9.d, p4/m, z27.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z11.h, z30.h
+; CHECK-NEXT:    fcvtzs z24.d, p3/m, z5.h
+; CHECK-NEXT:    mov z8.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p1.h, p0/z, z6.h, z30.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z1.h, z30.h
+; CHECK-NEXT:    str z8, [x8, #4, mul vl]
+; CHECK-NEXT:    fcvtzs z12.d, p4/m, z11.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z11.h, z29.h
+; CHECK-NEXT:    uunpkhi z11.d, z17.s
+; CHECK-NEXT:    movprfx z17, z20
+; CHECK-NEXT:    frintx z17.h, p0/m, z20.h
+; CHECK-NEXT:    fcvtzs z25.d, p1/m, z6.h
+; CHECK-NEXT:    mov z20.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z0.d, p5/m, z1.h
+; CHECK-NEXT:    fcmge p6.h, p0/z, z10.h, z30.h
+; CHECK-NEXT:    frintx z11.h, p0/m, z11.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z31.h, z30.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z13.h, z30.h
+; CHECK-NEXT:    fcvtzs z18.d, p6/m, z10.h
+; CHECK-NEXT:    fcmgt p11.h, p0/z, z10.h, z29.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z11.h, z30.h
+; CHECK-NEXT:    fcvtzs z2.d, p3/m, z31.h
+; CHECK-NEXT:    fcvtzs z21.d, p1/m, z13.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z17.h, z30.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z16.h, z30.h
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z10.h, z10.h
+; CHECK-NEXT:    sel z10.d, p4, z28.d, z12.d
+; CHECK-NEXT:    sel z12.d, p11, z28.d, z18.d
+; CHECK-NEXT:    fcvtzs z26.d, p5/m, z11.h
+; CHECK-NEXT:    fcvtzs z22.d, p2/m, z17.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z11.h, z29.h
+; CHECK-NEXT:    fcvtzs z23.d, p3/m, z16.h
+; CHECK-NEXT:    mov z10.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    mov z12.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p6.h, p0/z, z19.h, z30.h
+; CHECK-NEXT:    str z10, [x8, #7, mul vl]
+; CHECK-NEXT:    fcmge p7.h, p0/z, z3.h, z30.h
+; CHECK-NEXT:    str z12, [x8, #8, mul vl]
+; CHECK-NEXT:    mov z26.d, p4/m, z28.d
+; CHECK-NEXT:    fcmge p2.h, p0/z, z15.h, z30.h
+; CHECK-NEXT:    mov z30.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z14.d, p6/m, z19.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z16.h, z29.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z17.h, z29.h
+; CHECK-NEXT:    fcvtzs z20.d, p7/m, z3.h
+; CHECK-NEXT:    fcvtzs z30.d, p2/m, z15.h
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z11.h, z11.h
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z16.h, z16.h
+; CHECK-NEXT:    sel z11.d, p5, z28.d, z23.d
+; CHECK-NEXT:    sel z16.d, p3, z28.d, z22.d
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z19.h, z29.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z15.h, z29.h
+; CHECK-NEXT:    mov z26.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z11.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z13.h, z29.h
+; CHECK-NEXT:    fcmuo p6.h, p0/z, z17.h, z17.h
+; CHECK-NEXT:    str z26, [x8, #15, mul vl]
+; CHECK-NEXT:    sel z26.d, p4, z28.d, z14.d
+; CHECK-NEXT:    str z11, [x8, #14, mul vl]
+; CHECK-NEXT:    mov z30.d, p3/m, z28.d
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z3.h, z29.h
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z13.h, z13.h
 ; CHECK-NEXT:    fcmuo p3.h, p0/z, z3.h, z3.h
-; CHECK-NEXT:    sel z0.d, p2, z27.d, z12.d
-; CHECK-NEXT:    fcmuo p2.h, p0/z, z9.h, z9.h
-; CHECK-NEXT:    mov z30.d, p9/m, z27.d
-; CHECK-NEXT:    str z19, [x8, #12, mul vl]
-; CHECK-NEXT:    sel z3.d, p5, z27.d, z11.d
-; CHECK-NEXT:    mov z24.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    str z15, [x8, #11, mul vl]
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z26.h, z25.h
-; CHECK-NEXT:    mov z13.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p5.h, p0/z, z8.h, z8.h
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z26.h, z26.h
-; CHECK-NEXT:    str z24, [x8, #10, mul vl]
-; CHECK-NEXT:    mov z3.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p3.h, p0/z, z4.h, z25.h
-; CHECK-NEXT:    str z13, [x8, #9, mul vl]
+; CHECK-NEXT:    sel z3.d, p1, z28.d, z21.d
+; CHECK-NEXT:    mov z16.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p12.h, p0/z, z27.h, z29.h
+; CHECK-NEXT:    sel z11.d, p2, z28.d, z20.d
+; CHECK-NEXT:    str z16, [x8, #13, mul vl]
+; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p6.h, p0/z, z15.h, z15.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z4.h, z29.h
+; CHECK-NEXT:    mov z11.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z9.d, p12/m, z28.d
+; CHECK-NEXT:    str z3, [x8, #11, mul vl]
+; CHECK-NEXT:    fcmuo p5.h, p0/z, z19.h, z19.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z5.h, z29.h
+; CHECK-NEXT:    str z11, [x8, #10, mul vl]
+; CHECK-NEXT:    mov z30.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    sel z3.d, p1, z28.d, z7.d
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z6.h, z29.h
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z27.h, z27.h
+; CHECK-NEXT:    str z30, [x8, #12, mul vl]
+; CHECK-NEXT:    mov z26.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    sel z7.d, p2, z28.d, z24.d
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z31.h, z29.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z1.h, z29.h
+; CHECK-NEXT:    str z26, [x8, #9, mul vl]
+; CHECK-NEXT:    sel z24.d, p4, z28.d, z25.d
+; CHECK-NEXT:    mov z9.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p5.h, p0/z, z31.h, z31.h
 ; CHECK-NEXT:    fcmuo p2.h, p0/z, z6.h, z6.h
-; CHECK-NEXT:    mov z31.d, p10/m, z27.d
-; CHECK-NEXT:    str z0, [x8, #8, mul vl]
-; CHECK-NEXT:    mov z2.d, p1/m, z27.d
-; CHECK-NEXT:    fcmuo p1.h, p0/z, z7.h, z7.h
-; CHECK-NEXT:    str z3, [x8, #7, mul vl]
-; CHECK-NEXT:    mov z17.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p5.h, p0/z, z1.h, z1.h
-; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z5.h, z5.h
-; CHECK-NEXT:    fcmuo p0.h, p0/z, z4.h, z4.h
-; CHECK-NEXT:    str z17, [x8, #6, mul vl]
-; CHECK-NEXT:    mov z31.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z30.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    mov z0.d, p3/m, z27.d
-; CHECK-NEXT:    mov z29.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    str z2, [x8, #5, mul vl]
-; CHECK-NEXT:    str z31, [x8, #4, mul vl]
-; CHECK-NEXT:    mov z28.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    str z30, [x8, #3, mul vl]
+; CHECK-NEXT:    mov z2.d, p6/m, z28.d
+; CHECK-NEXT:    str z9, [x8, #5, mul vl]
+; CHECK-NEXT:    mov z0.d, p1/m, z28.d
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z5.h, z5.h
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z4.h, z4.h
+; CHECK-NEXT:    mov z2.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z24.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    mov z7.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    str z2, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    str z24, [x8, #3, mul vl]
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    str z29, [x8, #2, mul vl]
-; CHECK-NEXT:    str z28, [x8, #1, mul vl]
+; CHECK-NEXT:    str z7, [x8, #2, mul vl]
+; CHECK-NEXT:    str z3, [x8, #1, mul vl]
 ; CHECK-NEXT:    str z0, [x8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr p11, [sp] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #18
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f16(<vscale x 32 x half> %x)
@@ -532,17 +523,17 @@ define <vscale x 1 x i64> @llrint_v1i64_v1f32(<vscale x 1 x float> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
 ; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
-; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z2.s
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.s
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f32(<vscale x 1 x float> %x)
@@ -555,17 +546,17 @@ define <vscale x 2 x i64> @llrint_v2i64_v2f32(<vscale x 2 x float> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
 ; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
-; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z2.s
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.s
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f32(<vscale x 2 x float> %x)
@@ -584,20 +575,20 @@ define <vscale x 4 x i64> @llrint_v4i64_v4f32(<vscale x 4 x float> %x) {
 ; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z4.d, #0x8000000000000000
-; CHECK-NEXT:    mov z5.s, w8
+; CHECK-NEXT:    mov z5.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
 ; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, z2.s
 ; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z2.s
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    fcmuo p3.s, p0/z, z1.s, z1.s
-; CHECK-NEXT:    fcvtzs z3.d, p1/m, z1.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z5.s
-; CHECK-NEXT:    fcvtzs z4.d, p2/m, z0.s
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z5.s
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z1.s
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z2.s
+; CHECK-NEXT:    fcvtzs z3.d, p2/m, z0.s
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z2.s
 ; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z3.d
-; CHECK-NEXT:    sel z1.d, p2, z2.d, z4.d
+; CHECK-NEXT:    sel z0.d, p1, z5.d, z4.d
+; CHECK-NEXT:    sel z1.d, p2, z5.d, z3.d
 ; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
@@ -617,49 +608,50 @@ define <vscale x 8 x i64> @llrint_v8i64_v8f32(<vscale x 8 x float> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    uunpklo z2.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
 ; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
 ; CHECK-NEXT:    uunpklo z3.d, z1.s
 ; CHECK-NEXT:    uunpkhi z1.d, z1.s
 ; CHECK-NEXT:    mov z4.s, w8
 ; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
 ; CHECK-NEXT:    mov z5.d, #0x8000000000000000
-; CHECK-NEXT:    mov z6.d, #0x8000000000000000
-; CHECK-NEXT:    mov z25.s, w8
 ; CHECK-NEXT:    mov z7.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z2.s, p0/m, z2.s
-; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
 ; CHECK-NEXT:    mov z24.d, #0x8000000000000000
+; CHECK-NEXT:    mov z25.s, w8
+; CHECK-NEXT:    frintx z2.s, p0/m, z2.s
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z6, z0
+; CHECK-NEXT:    frintx z6.s, p0/m, z0.s
 ; CHECK-NEXT:    frintx z3.s, p0/m, z3.s
 ; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z2.s, z4.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z4.s
-; CHECK-NEXT:    fcmuo p6.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z6.s, z4.s
 ; CHECK-NEXT:    fcmge p3.s, p0/z, z3.s, z4.s
-; CHECK-NEXT:    fcmge p4.s, p0/z, z1.s, z4.s
-; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z1.s, z25.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z2.s, z25.s
 ; CHECK-NEXT:    fcvtzs z5.d, p1/m, z2.s
-; CHECK-NEXT:    fcvtzs z6.d, p2/m, z0.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z2.s, z25.s
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z25.s
-; CHECK-NEXT:    fcvtzs z7.d, p3/m, z3.s
-; CHECK-NEXT:    fcmgt p3.s, p0/z, z3.s, z25.s
-; CHECK-NEXT:    fcvtzs z24.d, p4/m, z1.s
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z2.s, z2.s
-; CHECK-NEXT:    sel z0.d, p1, z4.d, z5.d
-; CHECK-NEXT:    fcmuo p1.s, p0/z, z3.s, z3.s
-; CHECK-NEXT:    fcmuo p0.s, p0/z, z1.s, z1.s
-; CHECK-NEXT:    sel z1.d, p2, z4.d, z6.d
-; CHECK-NEXT:    sel z2.d, p3, z4.d, z7.d
-; CHECK-NEXT:    sel z3.d, p5, z4.d, z24.d
+; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, z4.s
+; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcvtzs z7.d, p2/m, z6.s
+; CHECK-NEXT:    fcvtzs z24.d, p3/m, z3.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z6.s, z25.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z3.s, z25.s
+; CHECK-NEXT:    fcvtzs z26.d, p1/m, z1.s
+; CHECK-NEXT:    sel z0.d, p4, z4.d, z5.d
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z25.s
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z6.s, z6.s
+; CHECK-NEXT:    fcmuo p6.s, p0/z, z3.s, z3.s
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z2.s, z2.s
+; CHECK-NEXT:    sel z2.d, p5, z4.d, z24.d
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    sel z1.d, p3, z4.d, z7.d
+; CHECK-NEXT:    sel z3.d, p1, z4.d, z26.d
+; CHECK-NEXT:    mov z2.d, p6/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -673,7 +665,7 @@ define <vscale x 16 x i64> @llrint_v16i64_v16f32(<vscale x 16 x float> %x) {
 ; CHECK-LABEL: llrint_v16i64_v16f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-4
+; CHECK-NEXT:    addvl sp, sp, #-3
 ; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
@@ -681,106 +673,104 @@ define <vscale x 16 x i64> @llrint_v16i64_v16f32(<vscale x 16 x float> %x) {
 ; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
+; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16
 ; CHECK-NEXT:    uunpklo z4.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    uunpkhi z5.d, z0.s
 ; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpkhi z7.d, z1.s
+; CHECK-NEXT:    uunpklo z24.d, z2.s
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uunpklo z6.d, z2.s
+; CHECK-NEXT:    uunpklo z6.d, z1.s
 ; CHECK-NEXT:    uunpkhi z2.d, z2.s
-; CHECK-NEXT:    uunpklo z5.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
-; CHECK-NEXT:    uunpklo z7.d, z3.s
-; CHECK-NEXT:    mov z24.s, w8
-; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
-; CHECK-NEXT:    mov z26.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z4.s, p0/m, z4.s
-; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
-; CHECK-NEXT:    mov z30.s, w8
-; CHECK-NEXT:    movprfx z27, z2
-; CHECK-NEXT:    frintx z27.s, p0/m, z2.s
-; CHECK-NEXT:    uunpkhi z2.d, z3.s
-; CHECK-NEXT:    frintx z6.s, p0/m, z6.s
-; CHECK-NEXT:    movprfx z25, z1
-; CHECK-NEXT:    frintx z25.s, p0/m, z1.s
-; CHECK-NEXT:    frintx z5.s, p0/m, z5.s
-; CHECK-NEXT:    frintx z7.s, p0/m, z7.s
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    mov z3.d, #0x8000000000000000
+; CHECK-NEXT:    mov z27.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.s, p0/z, z4.s, z24.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z24.s
-; CHECK-NEXT:    mov z29.d, #0x8000000000000000
-; CHECK-NEXT:    movprfx z9, z2
-; CHECK-NEXT:    frintx z9.s, p0/m, z2.s
-; CHECK-NEXT:    fcmge p5.s, p0/z, z6.s, z24.s
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p3.s, p0/z, z5.s, z24.s
-; CHECK-NEXT:    fcmge p4.s, p0/z, z25.s, z24.s
-; CHECK-NEXT:    fcmge p7.s, p0/z, z7.s, z24.s
-; CHECK-NEXT:    fcmge p6.s, p0/z, z27.s, z24.s
+; CHECK-NEXT:    movprfx z25, z4
+; CHECK-NEXT:    frintx z25.s, p0/m, z4.s
+; CHECK-NEXT:    frintx z5.s, p0/m, z5.s
+; CHECK-NEXT:    mov z4.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    frintx z7.s, p0/m, z7.s
+; CHECK-NEXT:    frintx z24.s, p0/m, z24.s
+; CHECK-NEXT:    movprfx z30, z2
+; CHECK-NEXT:    frintx z30.s, p0/m, z2.s
+; CHECK-NEXT:    frintx z6.s, p0/m, z6.s
+; CHECK-NEXT:    uunpklo z2.d, z3.s
+; CHECK-NEXT:    mov z29.s, w8
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    mov z10.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z4.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z4.s, z30.s
-; CHECK-NEXT:    fcvtzs z26.d, p2/m, z0.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z9.s, z24.s
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z29.d, p5/m, z6.s
-; CHECK-NEXT:    fcvtzs z3.d, p3/m, z5.s
-; CHECK-NEXT:    fcvtzs z28.d, p4/m, z25.s
-; CHECK-NEXT:    fcvtzs z8.d, p7/m, z7.s
-; CHECK-NEXT:    fcmgt p4.s, p0/z, z0.s, z30.s
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z5.s, z30.s
-; CHECK-NEXT:    fcmgt p7.s, p0/z, z25.s, z30.s
-; CHECK-NEXT:    fcmgt p8.s, p0/z, z6.s, z30.s
-; CHECK-NEXT:    fcvtzs z31.d, p6/m, z27.s
-; CHECK-NEXT:    fcmuo p6.s, p0/z, z0.s, z0.s
-; CHECK-NEXT:    sel z0.d, p1, z10.d, z1.d
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z27.s, z30.s
-; CHECK-NEXT:    fcmgt p10.s, p0/z, z7.s, z30.s
-; CHECK-NEXT:    fcvtzs z24.d, p2/m, z9.s
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z9.s, z30.s
-; CHECK-NEXT:    fcmuo p3.s, p0/z, z4.s, z4.s
-; CHECK-NEXT:    fcmuo p9.s, p0/z, z5.s, z5.s
-; CHECK-NEXT:    sel z1.d, p4, z10.d, z26.d
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z25.s, z25.s
-; CHECK-NEXT:    sel z2.d, p5, z10.d, z3.d
-; CHECK-NEXT:    sel z3.d, p7, z10.d, z28.d
-; CHECK-NEXT:    sel z4.d, p8, z10.d, z29.d
-; CHECK-NEXT:    fcmuo p5.s, p0/z, z6.s, z6.s
-; CHECK-NEXT:    fcmuo p7.s, p0/z, z27.s, z27.s
-; CHECK-NEXT:    fcmuo p8.s, p0/z, z7.s, z7.s
-; CHECK-NEXT:    sel z5.d, p1, z10.d, z31.d
-; CHECK-NEXT:    sel z6.d, p10, z10.d, z8.d
-; CHECK-NEXT:    ldr z8, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    mov z9.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcmge p5.s, p0/z, z25.s, z4.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z5.s, z4.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z7.s, z4.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z24.s, z4.s
+; CHECK-NEXT:    fcvtzs z0.d, p5/m, z25.s
+; CHECK-NEXT:    fcvtzs z1.d, p1/m, z5.s
+; CHECK-NEXT:    fcvtzs z27.d, p3/m, z7.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z6.s, z4.s
+; CHECK-NEXT:    fcvtzs z28.d, p4/m, z24.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z25.s, z29.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z30.s, z4.s
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    movprfx z25, z2
+; CHECK-NEXT:    frintx z25.s, p0/m, z2.s
+; CHECK-NEXT:    uunpkhi z2.d, z3.s
+; CHECK-NEXT:    fcvtzs z26.d, p2/m, z6.s
+; CHECK-NEXT:    mov z0.d, p3/m, z9.d
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z5.s, z29.s
+; CHECK-NEXT:    fcvtzs z31.d, p4/m, z30.s
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z5.s, z5.s
+; CHECK-NEXT:    movprfx z5, z2
+; CHECK-NEXT:    frintx z5.s, p0/m, z2.s
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p4.s, p0/z, z25.s, z4.s
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z6.s, z29.s
+; CHECK-NEXT:    mov z1.d, p5/m, z9.d
+; CHECK-NEXT:    fcmgt p8.s, p0/z, z7.s, z29.s
+; CHECK-NEXT:    fcmuo p10.s, p0/z, z7.s, z7.s
+; CHECK-NEXT:    mov z7.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z8.d, p4/m, z25.s
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    sel z2.d, p6, z9.d, z26.d
+; CHECK-NEXT:    sel z3.d, p8, z9.d, z27.d
+; CHECK-NEXT:    fcmge p4.s, p0/z, z5.s, z4.s
+; CHECK-NEXT:    fcmgt p9.s, p0/z, z24.s, z29.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z30.s, z29.s
+; CHECK-NEXT:    mov z3.d, p10/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    fcmuo p0.s, p0/z, z9.s, z9.s
-; CHECK-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    sel z7.d, p2, z10.d, z24.d
-; CHECK-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z25.s, z29.s
+; CHECK-NEXT:    fcvtzs z7.d, p4/m, z5.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z5.s, z29.s
+; CHECK-NEXT:    sel z4.d, p9, z9.d, z28.d
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z6.s, z6.s
+; CHECK-NEXT:    sel z6.d, p6, z9.d, z8.d
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
-; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    mov z5.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p8.s, p0/z, z30.s, z30.s
+; CHECK-NEXT:    fcmuo p9.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    mov z7.d, p4/m, z9.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z2.d, p7/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z6.d, p8/m, #0 // =0x0
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z24.s, z24.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z5.s, z5.s
+; CHECK-NEXT:    sel z5.d, p5, z9.d, z31.d
+; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z6.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 16 x i64> @llvm.llrint.nxv16i64.nxv16f32(<vscale x 16 x float> %x)
@@ -792,33 +782,34 @@ define <vscale x 32 x i64> @llrint_v32i64_v32f32(<vscale x 32 x float> %x) {
 ; CHECK-LABEL: llrint_v32i64_v32f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-17
-; CHECK-NEXT:    str p11, [sp] // 2-byte Folded Spill
-; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 16 + 144 * VG
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x01, 0x1e, 0x22 // sp + 16 + 152 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
@@ -829,210 +820,212 @@ define <vscale x 32 x i64> @llrint_v32i64_v32f32(<vscale x 32 x float> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x40, 0x1c // $d14 @ cfa - 56 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16
 ; CHECK-NEXT:    uunpklo z24.d, z0.s
-; CHECK-NEXT:    uunpkhi z25.d, z0.s
-; CHECK-NEXT:    mov w9, #-553648128 // =0xdf000000
 ; CHECK-NEXT:    uunpklo z26.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    mov w9, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpklo z28.d, z2.s
+; CHECK-NEXT:    uunpkhi z30.d, z2.s
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uunpklo z27.d, z2.s
-; CHECK-NEXT:    uunpkhi z9.d, z2.s
-; CHECK-NEXT:    uunpklo z11.d, z3.s
-; CHECK-NEXT:    uunpkhi z12.d, z3.s
-; CHECK-NEXT:    mov z10.s, w9
-; CHECK-NEXT:    mov w9, #1593835519 // =0x5effffff
+; CHECK-NEXT:    uunpkhi z25.d, z0.s
+; CHECK-NEXT:    uunpkhi z13.d, z3.s
+; CHECK-NEXT:    uunpklo z14.d, z4.s
+; CHECK-NEXT:    uunpkhi z27.d, z1.s
+; CHECK-NEXT:    uunpklo z9.d, z3.s
+; CHECK-NEXT:    mov z29.s, w9
 ; CHECK-NEXT:    movprfx z0, z24
 ; CHECK-NEXT:    frintx z0.s, p0/m, z24.s
-; CHECK-NEXT:    movprfx z24, z25
-; CHECK-NEXT:    frintx z24.s, p0/m, z25.s
-; CHECK-NEXT:    uunpklo z13.d, z4.s
-; CHECK-NEXT:    movprfx z25, z26
-; CHECK-NEXT:    frintx z25.s, p0/m, z26.s
-; CHECK-NEXT:    movprfx z26, z1
-; CHECK-NEXT:    frintx z26.s, p0/m, z1.s
-; CHECK-NEXT:    uunpkhi z14.d, z4.s
-; CHECK-NEXT:    movprfx z2, z27
-; CHECK-NEXT:    frintx z2.s, p0/m, z27.s
-; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    movprfx z27, z9
-; CHECK-NEXT:    frintx z27.s, p0/m, z9.s
-; CHECK-NEXT:    movprfx z9, z11
-; CHECK-NEXT:    frintx z9.s, p0/m, z11.s
-; CHECK-NEXT:    movprfx z11, z12
-; CHECK-NEXT:    frintx z11.s, p0/m, z12.s
-; CHECK-NEXT:    uunpklo z15.d, z7.s
-; CHECK-NEXT:    uunpkhi z7.d, z7.s
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    mov z29.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p3.s, p0/z, z26.s, z10.s
-; CHECK-NEXT:    mov z30.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.s, p0/z, z0.s, z10.s
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.s, p0/z, z24.s, z10.s
-; CHECK-NEXT:    movprfx z12, z13
-; CHECK-NEXT:    frintx z12.s, p0/m, z13.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z25.s, z10.s
-; CHECK-NEXT:    fcmge p4.s, p0/z, z2.s, z10.s
+; CHECK-NEXT:    movprfx z24, z26
+; CHECK-NEXT:    frintx z24.s, p0/m, z26.s
+; CHECK-NEXT:    mov w9, #1593835519 // =0x5effffff
+; CHECK-NEXT:    movprfx z10, z28
+; CHECK-NEXT:    frintx z10.s, p0/m, z28.s
+; CHECK-NEXT:    frintx z30.s, p0/m, z30.s
+; CHECK-NEXT:    uunpklo z17.d, z5.s
+; CHECK-NEXT:    movprfx z1, z25
+; CHECK-NEXT:    frintx z1.s, p0/m, z25.s
+; CHECK-NEXT:    movprfx z15, z13
+; CHECK-NEXT:    frintx z15.s, p0/m, z13.s
 ; CHECK-NEXT:    movprfx z13, z14
 ; CHECK-NEXT:    frintx z13.s, p0/m, z14.s
-; CHECK-NEXT:    uunpklo z17.d, z5.s
+; CHECK-NEXT:    uunpkhi z14.d, z4.s
 ; CHECK-NEXT:    uunpkhi z18.d, z5.s
-; CHECK-NEXT:    movprfx z21, z7
-; CHECK-NEXT:    frintx z21.s, p0/m, z7.s
-; CHECK-NEXT:    uunpklo z19.d, z6.s
-; CHECK-NEXT:    uunpkhi z20.d, z6.s
+; CHECK-NEXT:    uunpkhi z19.d, z6.s
+; CHECK-NEXT:    movprfx z25, z27
+; CHECK-NEXT:    frintx z25.s, p0/m, z27.s
+; CHECK-NEXT:    mov z27.d, #0x8000000000000000
+; CHECK-NEXT:    mov z11.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p3.s, p0/z, z24.s, z29.s
 ; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    fcvtzs z31.d, p3/m, z26.s
-; CHECK-NEXT:    fcmge p3.s, p0/z, z11.s, z10.s
-; CHECK-NEXT:    mov z5.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z15.s, p0/m, z15.s
-; CHECK-NEXT:    fcvtzs z1.d, p5/m, z0.s
-; CHECK-NEXT:    fcvtzs z29.d, p1/m, z24.s
-; CHECK-NEXT:    fcvtzs z30.d, p2/m, z25.s
-; CHECK-NEXT:    fcvtzs z8.d, p4/m, z2.s
-; CHECK-NEXT:    fcmge p1.s, p0/z, z27.s, z10.s
+; CHECK-NEXT:    mov z12.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p5.s, p0/z, z10.s, z29.s
+; CHECK-NEXT:    frintx z9.s, p0/m, z9.s
+; CHECK-NEXT:    uunpklo z20.d, z7.s
+; CHECK-NEXT:    movprfx z5, z14
+; CHECK-NEXT:    frintx z5.s, p0/m, z14.s
+; CHECK-NEXT:    movprfx z14, z17
+; CHECK-NEXT:    frintx z14.s, p0/m, z17.s
+; CHECK-NEXT:    movprfx z17, z18
+; CHECK-NEXT:    frintx z17.s, p0/m, z18.s
+; CHECK-NEXT:    fcmge p6.s, p0/z, z30.s, z29.s
+; CHECK-NEXT:    movprfx z18, z19
+; CHECK-NEXT:    frintx z18.s, p0/m, z19.s
+; CHECK-NEXT:    uunpkhi z7.d, z7.s
+; CHECK-NEXT:    mov z31.s, w9
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
+; CHECK-NEXT:    uunpklo z6.d, z6.s
+; CHECK-NEXT:    mov z28.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z27.d, p3/m, z24.s
+; CHECK-NEXT:    fcvtzs z11.d, p5/m, z10.s
 ; CHECK-NEXT:    mov z4.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p2.s, p0/z, z9.s, z10.s
 ; CHECK-NEXT:    mov z16.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p4.s, p0/z, z12.s, z10.s
-; CHECK-NEXT:    mov z6.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.s, p0/z, z13.s, z10.s
-; CHECK-NEXT:    mov z14.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z17.s, p0/m, z17.s
-; CHECK-NEXT:    frintx z18.s, p0/m, z18.s
-; CHECK-NEXT:    frintx z19.s, p0/m, z19.s
+; CHECK-NEXT:    movprfx z22, z7
+; CHECK-NEXT:    frintx z22.s, p0/m, z7.s
+; CHECK-NEXT:    fcvtzs z12.d, p6/m, z30.s
 ; CHECK-NEXT:    frintx z20.s, p0/m, z20.s
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z5.d, p3/m, z11.s
-; CHECK-NEXT:    fcmge p3.s, p0/z, z21.s, z10.s
-; CHECK-NEXT:    mov z3.s, w9
-; CHECK-NEXT:    fcmge p6.s, p0/z, z15.s, z10.s
-; CHECK-NEXT:    mov z22.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z4.d, p1/m, z27.s
-; CHECK-NEXT:    fcvtzs z16.d, p2/m, z9.s
-; CHECK-NEXT:    fcvtzs z6.d, p4/m, z12.s
-; CHECK-NEXT:    fcvtzs z14.d, p5/m, z13.s
-; CHECK-NEXT:    fcmge p1.s, p0/z, z17.s, z10.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z18.s, z10.s
+; CHECK-NEXT:    mov z21.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z23.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p4.s, p0/z, z19.s, z10.s
-; CHECK-NEXT:    fcmge p5.s, p0/z, z20.s, z10.s
-; CHECK-NEXT:    mov z10.d, #0x8000000000000000
+; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    frintx z6.s, p0/m, z6.s
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z19.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z29.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z1.s, z29.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z25.s, z29.s
+; CHECK-NEXT:    fcmgt p9.s, p0/z, z10.s, z31.s
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.s
 ; CHECK-NEXT:    mov z0.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z28.d, p3/m, z21.s
-; CHECK-NEXT:    mov z7.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p11.s, p0/z, z21.s, z3.s
-; CHECK-NEXT:    fcvtzs z22.d, p6/m, z15.s
-; CHECK-NEXT:    fcmgt p3.s, p0/z, z15.s, z3.s
-; CHECK-NEXT:    fcmuo p6.s, p0/z, z15.s, z15.s
-; CHECK-NEXT:    mov z15.d, #0x8000000000000000
-; CHECK-NEXT:    fcmgt p7.s, p0/z, z24.s, z3.s
+; CHECK-NEXT:    fcvtzs z26.d, p2/m, z1.s
+; CHECK-NEXT:    fcvtzs z28.d, p4/m, z25.s
+; CHECK-NEXT:    mov z11.d, p9/m, z3.d
+; CHECK-NEXT:    fcmuo p8.s, p0/z, z10.s, z10.s
+; CHECK-NEXT:    mov z10.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p5.s, p0/z, z9.s, z29.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z15.s, z29.s
+; CHECK-NEXT:    fcmge p6.s, p0/z, z13.s, z29.s
+; CHECK-NEXT:    mov z11.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z10.d, p5/m, z9.s
+; CHECK-NEXT:    fcvtzs z4.d, p3/m, z15.s
+; CHECK-NEXT:    fcvtzs z16.d, p6/m, z13.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z17.s, z29.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z18.s, z29.s
+; CHECK-NEXT:    fcmgt p12.s, p0/z, z30.s, z31.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z15.s, z31.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z20.s, z29.s
+; CHECK-NEXT:    fcvtzs z21.d, p1/m, z17.s
 ; CHECK-NEXT:    fcvtzs z23.d, p2/m, z18.s
-; CHECK-NEXT:    fcvtzs z10.d, p5/m, z20.s
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z9.s, z3.s
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z20.s, z3.s
-; CHECK-NEXT:    fcvtzs z0.d, p4/m, z19.s
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z21.s, z21.s
-; CHECK-NEXT:    mov z28.d, p11/m, z7.d
-; CHECK-NEXT:    sel z21.d, p3, z7.d, z22.d
-; CHECK-NEXT:    fcmgt p3.s, p0/z, z19.s, z3.s
-; CHECK-NEXT:    fcvtzs z15.d, p1/m, z17.s
-; CHECK-NEXT:    fcmuo p1.s, p0/z, z20.s, z20.s
-; CHECK-NEXT:    mov z29.d, p7/m, z7.d
-; CHECK-NEXT:    fcmgt p7.s, p0/z, z18.s, z3.s
-; CHECK-NEXT:    mov z16.d, p2/m, z7.d
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z17.s, z3.s
-; CHECK-NEXT:    mov z10.d, p5/m, z7.d
-; CHECK-NEXT:    mov z28.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z19.s, z19.s
-; CHECK-NEXT:    mov z0.d, p3/m, z7.d
-; CHECK-NEXT:    fcmuo p3.s, p0/z, z18.s, z18.s
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z13.s, z3.s
-; CHECK-NEXT:    mov z21.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p10.s, p0/z, z2.s, z3.s
-; CHECK-NEXT:    fcmgt p8.s, p0/z, z25.s, z3.s
-; CHECK-NEXT:    str z28, [x8, #15, mul vl]
-; CHECK-NEXT:    mov z10.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p1.s, p0/z, z17.s, z17.s
-; CHECK-NEXT:    sel z19.d, p7, z7.d, z23.d
-; CHECK-NEXT:    sel z28.d, p2, z7.d, z15.d
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z12.s, z3.s
-; CHECK-NEXT:    str z21, [x8, #14, mul vl]
-; CHECK-NEXT:    mov z0.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    mov z14.d, p5/m, z7.d
-; CHECK-NEXT:    str z10, [x8, #13, mul vl]
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z11.s, z3.s
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z13.s, z13.s
-; CHECK-NEXT:    mov z19.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z28.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z27.s, z3.s
+; CHECK-NEXT:    fcmgt p11.s, p0/z, z13.s, z31.s
+; CHECK-NEXT:    sel z7.d, p12, z3.d, z12.d
+; CHECK-NEXT:    mov z4.d, p5/m, z3.d
+; CHECK-NEXT:    fcmge p4.s, p0/z, z22.s, z29.s
+; CHECK-NEXT:    fcvtzs z0.d, p3/m, z20.s
+; CHECK-NEXT:    fcmge p6.s, p0/z, z5.s, z29.s
+; CHECK-NEXT:    sel z12.d, p11, z3.d, z16.d
+; CHECK-NEXT:    fcmge p7.s, p0/z, z14.s, z29.s
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z13.s, z13.s
+; CHECK-NEXT:    fcvtzs z8.d, p4/m, z22.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z6.s, z29.s
+; CHECK-NEXT:    mov z29.d, #0x8000000000000000
+; CHECK-NEXT:    fcmuo p10.s, p0/z, z15.s, z15.s
+; CHECK-NEXT:    mov z15.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z19.d, p7/m, z14.s
+; CHECK-NEXT:    mov z12.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z29.d, p2/m, z6.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z22.s, z31.s
+; CHECK-NEXT:    fcvtzs z15.d, p6/m, z5.s
+; CHECK-NEXT:    mov z4.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    str z12, [x8, #8, mul vl]
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z20.s, z31.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z18.s, z31.s
+; CHECK-NEXT:    str z4, [x8, #7, mul vl]
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z22.s, z22.s
+; CHECK-NEXT:    mov z8.d, p4/m, z3.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z20.s, z20.s
+; CHECK-NEXT:    mov z0.d, p5/m, z3.d
+; CHECK-NEXT:    fcmuo p6.s, p0/z, z18.s, z18.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z5.s, z31.s
+; CHECK-NEXT:    mov z8.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p5.s, p0/z, z5.s, z5.s
+; CHECK-NEXT:    sel z5.d, p3, z3.d, z23.d
+; CHECK-NEXT:    str z8, [x8, #15, mul vl]
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z6.s, z31.s
+; CHECK-NEXT:    str z0, [x8, #14, mul vl]
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z17.s, z31.s
+; CHECK-NEXT:    mov z5.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z14.s, z31.s
+; CHECK-NEXT:    fcmuo p6.s, p0/z, z6.s, z6.s
+; CHECK-NEXT:    sel z6.d, p4, z3.d, z15.d
+; CHECK-NEXT:    str z5, [x8, #13, mul vl]
+; CHECK-NEXT:    sel z0.d, p3, z3.d, z29.d
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z17.s, z17.s
+; CHECK-NEXT:    sel z5.d, p1, z3.d, z21.d
+; CHECK-NEXT:    sel z29.d, p2, z3.d, z19.d
+; CHECK-NEXT:    mov z6.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z14.s, z14.s
+; CHECK-NEXT:    mov z5.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    str z6, [x8, #9, mul vl]
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z31.s
 ; CHECK-NEXT:    str z0, [x8, #12, mul vl]
-; CHECK-NEXT:    fcmuo p3.s, p0/z, z12.s, z12.s
-; CHECK-NEXT:    sel z0.d, p2, z7.d, z6.d
-; CHECK-NEXT:    fcmuo p2.s, p0/z, z11.s, z11.s
-; CHECK-NEXT:    fcmgt p9.s, p0/z, z26.s, z3.s
-; CHECK-NEXT:    mov z30.d, p8/m, z7.d
-; CHECK-NEXT:    str z19, [x8, #11, mul vl]
-; CHECK-NEXT:    mov z5.d, p5/m, z7.d
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z24.s, z31.s
+; CHECK-NEXT:    mov z29.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    str z5, [x8, #11, mul vl]
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z25.s, z31.s
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z30.s, z30.s
+; CHECK-NEXT:    sel z5.d, p1, z3.d, z26.d
+; CHECK-NEXT:    str z29, [x8, #10, mul vl]
+; CHECK-NEXT:    sel z26.d, p2, z3.d, z27.d
+; CHECK-NEXT:    ldr z4, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    str z11, [x8, #4, mul vl]
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z9.s, z31.s
+; CHECK-NEXT:    sel z6.d, p4, z3.d, z28.d
 ; CHECK-NEXT:    fcmuo p5.s, p0/z, z9.s, z9.s
-; CHECK-NEXT:    str z28, [x8, #10, mul vl]
-; CHECK-NEXT:    mov z4.d, p1/m, z7.d
-; CHECK-NEXT:    fcmuo p1.s, p0/z, z2.s, z2.s
-; CHECK-NEXT:    ldr z2, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z14.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z27.s, z27.s
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z5.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p2.s, p0/z, z26.s, z26.s
-; CHECK-NEXT:    mov z16.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p5.s, p0/z, z25.s, z25.s
-; CHECK-NEXT:    mov z31.d, p9/m, z7.d
-; CHECK-NEXT:    str z14, [x8, #9, mul vl]
-; CHECK-NEXT:    fcmgt p3.s, p0/z, z2.s, z3.s
-; CHECK-NEXT:    mov z8.d, p10/m, z7.d
-; CHECK-NEXT:    str z0, [x8, #8, mul vl]
-; CHECK-NEXT:    mov z4.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z24.s, z24.s
-; CHECK-NEXT:    str z5, [x8, #7, mul vl]
-; CHECK-NEXT:    fcmuo p0.s, p0/z, z2.s, z2.s
-; CHECK-NEXT:    mov z31.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    str z16, [x8, #6, mul vl]
-; CHECK-NEXT:    mov z8.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z30.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    str z4, [x8, #5, mul vl]
-; CHECK-NEXT:    sel z0.d, p3, z7.d, z1.d
-; CHECK-NEXT:    str z31, [x8, #3, mul vl]
-; CHECK-NEXT:    mov z29.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    str z8, [x8, #4, mul vl]
-; CHECK-NEXT:    str z30, [x8, #2, mul vl]
-; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    str z29, [x8, #1, mul vl]
-; CHECK-NEXT:    str z0, [x8]
+; CHECK-NEXT:    mov z7.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z4.s, z31.s
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z24.s, z24.s
+; CHECK-NEXT:    sel z0.d, p6, z3.d, z10.d
+; CHECK-NEXT:    str z7, [x8, #5, mul vl]
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z4.s, z4.s
+; CHECK-NEXT:    sel z1.d, p1, z3.d, z2.d
+; CHECK-NEXT:    mov z0.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z6.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z26.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z5.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    str z0, [x8, #6, mul vl]
+; CHECK-NEXT:    str z6, [x8, #3, mul vl]
+; CHECK-NEXT:    str z26, [x8, #2, mul vl]
+; CHECK-NEXT:    str z5, [x8, #1, mul vl]
+; CHECK-NEXT:    str z1, [x8]
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr p11, [sp] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #18
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv32f32(<vscale x 32 x float> %x)
@@ -1045,17 +1038,17 @@ define <vscale x 1 x i64> @llrint_v1i64_v1f64(<vscale x 1 x double> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.d, x8
 ; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
-; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z1.d
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z2.d
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 1 x i64> @llvm.llrint.nxv1i64.nxv1f64(<vscale x 1 x double> %x)
@@ -1068,17 +1061,17 @@ define <vscale x 2 x i64> @llrint_v2i64_v2f64(<vscale x 2 x double> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.d, x8
 ; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
-; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z1.d
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z2.d
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x i64> @llvm.llrint.nxv2i64.nxv2f64(<vscale x 2 x double> %x)
@@ -1093,22 +1086,22 @@ define <vscale x 4 x i64> @llrint_v4i64_v4f64(<vscale x 4 x double> %x) {
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z2.d, x8
-; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    mov z4.d, #0x8000000000000000
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
 ; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
-; CHECK-NEXT:    mov z5.d, x8
+; CHECK-NEXT:    mov z5.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z2.d
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z2.d
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    fcmuo p3.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    fcvtzs z3.d, p1/m, z0.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z5.d
-; CHECK-NEXT:    fcvtzs z4.d, p2/m, z1.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z1.d, z5.d
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z0.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z2.d
+; CHECK-NEXT:    fcvtzs z3.d, p2/m, z1.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z1.d, z2.d
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z1.d, z1.d
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z3.d
-; CHECK-NEXT:    sel z1.d, p2, z2.d, z4.d
+; CHECK-NEXT:    sel z0.d, p1, z5.d, z4.d
+; CHECK-NEXT:    sel z1.d, p2, z5.d, z3.d
 ; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
@@ -1129,44 +1122,44 @@ define <vscale x 8 x i64> @llrint_v8i64_v8f64(<vscale x 8 x double> %x) {
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
-; CHECK-NEXT:    mov z5.d, #0x8000000000000000
-; CHECK-NEXT:    mov z4.d, x8
-; CHECK-NEXT:    mov z6.d, #0x8000000000000000
+; CHECK-NEXT:    mov z4.d, #0x8000000000000000
+; CHECK-NEXT:    mov z5.d, x8
 ; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    mov z6.d, #0x8000000000000000
 ; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
 ; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
 ; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-NEXT:    mov z24.d, x8
 ; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
-; CHECK-NEXT:    mov z25.d, x8
 ; CHECK-NEXT:    mov z7.d, #0x8000000000000000
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z4.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z4.d
-; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z4.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z4.d
-; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z3.d, z25.d
-; CHECK-NEXT:    fcmuo p6.d, p0/z, z1.d, z1.d
-; CHECK-NEXT:    fcvtzs z5.d, p1/m, z0.d
+; CHECK-NEXT:    mov z25.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z5.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z5.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z5.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z0.d, z24.d
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z0.d
+; CHECK-NEXT:    fcmge p1.d, p0/z, z3.d, z5.d
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
 ; CHECK-NEXT:    fcvtzs z6.d, p2/m, z1.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z25.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z1.d, z25.d
 ; CHECK-NEXT:    fcvtzs z7.d, p3/m, z2.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z2.d, z25.d
-; CHECK-NEXT:    fcvtzs z24.d, p4/m, z3.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    sel z0.d, p1, z4.d, z5.d
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z1.d, z24.d
+; CHECK-NEXT:    sel z0.d, p4, z25.d, z4.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z2.d, z24.d
+; CHECK-NEXT:    fcvtzs z5.d, p1/m, z3.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z24.d
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    sel z1.d, p3, z25.d, z6.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    sel z2.d, p4, z25.d, z7.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z3.d, z3.d
-; CHECK-NEXT:    sel z1.d, p2, z4.d, z6.d
-; CHECK-NEXT:    sel z2.d, p3, z4.d, z7.d
-; CHECK-NEXT:    sel z3.d, p5, z4.d, z24.d
+; CHECK-NEXT:    sel z3.d, p1, z25.d, z5.d
+; CHECK-NEXT:    mov z1.d, p5/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p6/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p1/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1196,84 +1189,84 @@ define <vscale x 16 x i64> @llrint_v16f64(<vscale x 16 x double> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
+; CHECK-NEXT:    mov z28.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z26.d, #0x8000000000000000
-; CHECK-NEXT:    mov z24.d, x8
-; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    mov z27.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z24, z0
+; CHECK-NEXT:    frintx z24.d, p0/m, z0.d
+; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
+; CHECK-NEXT:    mov z0.d, x8
 ; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
-; CHECK-NEXT:    movprfx z25, z4
-; CHECK-NEXT:    frintx z25.d, p0/m, z4.d
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
-; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
-; CHECK-NEXT:    frintx z5.d, p0/m, z5.d
+; CHECK-NEXT:    frintx z4.d, p0/m, z4.d
 ; CHECK-NEXT:    frintx z6.d, p0/m, z6.d
 ; CHECK-NEXT:    mov z30.d, x8
-; CHECK-NEXT:    mov z4.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z29.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z24.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z24.d
-; CHECK-NEXT:    fcmge p5.d, p0/z, z25.d, z24.d
-; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z24.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z24.d
-; CHECK-NEXT:    fcmge p7.d, p0/z, z5.d, z24.d
+; CHECK-NEXT:    frintx z5.d, p0/m, z5.d
+; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
 ; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p6.d, p0/z, z6.d, z24.d
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
-; CHECK-NEXT:    mov z9.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p8.d, p0/z, z25.d, z30.d
-; CHECK-NEXT:    fcmgt p10.d, p0/z, z6.d, z30.d
-; CHECK-NEXT:    fcvtzs z26.d, p1/m, z0.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z30.d
-; CHECK-NEXT:    fcvtzs z4.d, p2/m, z1.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z7.d, z24.d
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z27.d, p3/m, z2.d
+; CHECK-NEXT:    mov z8.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z9.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p1.d, p0/z, z24.d, z0.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z0.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z0.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z0.d
+; CHECK-NEXT:    fcvtzs z25.d, p1/m, z24.d
 ; CHECK-NEXT:    fcvtzs z28.d, p4/m, z3.d
-; CHECK-NEXT:    fcvtzs z29.d, p5/m, z25.d
-; CHECK-NEXT:    fcvtzs z31.d, p7/m, z5.d
-; CHECK-NEXT:    fcmgt p4.d, p0/z, z1.d, z30.d
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z2.d, z30.d
-; CHECK-NEXT:    fcmgt p7.d, p0/z, z3.d, z30.d
-; CHECK-NEXT:    fcvtzs z8.d, p6/m, z6.d
-; CHECK-NEXT:    fcmuo p3.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    sel z0.d, p1, z9.d, z26.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z5.d, z30.d
-; CHECK-NEXT:    fcvtzs z24.d, p2/m, z7.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z7.d, z30.d
-; CHECK-NEXT:    fcmuo p6.d, p0/z, z1.d, z1.d
-; CHECK-NEXT:    fcmuo p9.d, p0/z, z2.d, z2.d
-; CHECK-NEXT:    sel z1.d, p4, z9.d, z4.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z3.d, z3.d
-; CHECK-NEXT:    sel z2.d, p5, z9.d, z27.d
-; CHECK-NEXT:    sel z3.d, p7, z9.d, z28.d
-; CHECK-NEXT:    sel z4.d, p8, z9.d, z29.d
-; CHECK-NEXT:    fcmuo p5.d, p0/z, z25.d, z25.d
-; CHECK-NEXT:    fcmuo p7.d, p0/z, z5.d, z5.d
-; CHECK-NEXT:    fcmuo p8.d, p0/z, z6.d, z6.d
-; CHECK-NEXT:    sel z5.d, p1, z9.d, z31.d
-; CHECK-NEXT:    sel z6.d, p10, z9.d, z8.d
-; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvtzs z26.d, p2/m, z1.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z4.d, z0.d
+; CHECK-NEXT:    fcvtzs z27.d, p3/m, z2.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z24.d, z30.d
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z24.d, z24.d
+; CHECK-NEXT:    mov z24.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p7.d, p0/z, z6.d, z0.d
+; CHECK-NEXT:    fcvtzs z29.d, p5/m, z4.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z5.d, z0.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z1.d, z30.d
+; CHECK-NEXT:    fcvtzs z24.d, p7/m, z6.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z2.d, z30.d
+; CHECK-NEXT:    fcmge p7.d, p0/z, z7.d, z0.d
+; CHECK-NEXT:    fcvtzs z31.d, p3/m, z5.d
+; CHECK-NEXT:    sel z0.d, p4, z8.d, z25.d
+; CHECK-NEXT:    fcmgt p8.d, p0/z, z3.d, z30.d
+; CHECK-NEXT:    fcmgt p9.d, p0/z, z4.d, z30.d
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z9.d, p7/m, z7.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    sel z1.d, p5, z8.d, z26.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    sel z2.d, p6, z8.d, z27.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z5.d, z30.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z6.d, z30.d
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p7.d, p0/z, z7.d, z30.d
+; CHECK-NEXT:    fcmuo p10.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    sel z3.d, p8, z8.d, z28.d
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z4.d, z4.d
+; CHECK-NEXT:    sel z4.d, p9, z8.d, z29.d
+; CHECK-NEXT:    fcmuo p8.d, p0/z, z5.d, z5.d
+; CHECK-NEXT:    sel z5.d, p5, z8.d, z31.d
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    fcmuo p9.d, p0/z, z6.d, z6.d
+; CHECK-NEXT:    sel z6.d, p6, z8.d, z24.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p10/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z6.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z7.d, z7.d
-; CHECK-NEXT:    sel z7.d, p2, z9.d, z24.d
+; CHECK-NEXT:    sel z7.d, p7, z8.d, z9.d
 ; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z5.d, p7/m, #0 // =0x0
-; CHECK-NEXT:    mov z6.d, p8/m, #0 // =0x0
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1286,33 +1279,34 @@ define <vscale x 32 x i64> @llrint_v32f64(<vscale x 32 x double> %x) {
 ; CHECK-LABEL: llrint_v32f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-17
-; CHECK-NEXT:    str p11, [sp] // 2-byte Folded Spill
-; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 16 + 144 * VG
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x01, 0x1e, 0x22 // sp + 16 + 152 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
@@ -1324,202 +1318,207 @@ define <vscale x 32 x i64> @llrint_v32f64(<vscale x 32 x double> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16
 ; CHECK-NEXT:    ldr z0, [x0]
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    ldr z2, [x0, #2, mul vl]
-; CHECK-NEXT:    ldr z1, [x0, #1, mul vl]
-; CHECK-NEXT:    ldr z6, [x0, #4, mul vl]
 ; CHECK-NEXT:    mov x9, #-4332462841530417152 // =0xc3e0000000000000
-; CHECK-NEXT:    ldr z5, [x0, #3, mul vl]
 ; CHECK-NEXT:    mov z25.d, x9
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
-; CHECK-NEXT:    movprfx z4, z2
-; CHECK-NEXT:    frintx z4.d, p0/m, z2.d
-; CHECK-NEXT:    mov z27.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
-; CHECK-NEXT:    frintx z6.d, p0/m, z6.d
+; CHECK-NEXT:    ldr z7, [x0, #3, mul vl]
+; CHECK-NEXT:    ldr z27, [x0, #4, mul vl]
+; CHECK-NEXT:    ldr z4, [x0, #2, mul vl]
+; CHECK-NEXT:    ldr z2, [x0, #1, mul vl]
+; CHECK-NEXT:    ldr z9, [x0, #15, mul vl]
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    frintx z5.d, p0/m, z0.d
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    ldr z10, [x0, #14, mul vl]
+; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
+; CHECK-NEXT:    movprfx z14, z27
+; CHECK-NEXT:    frintx z14.d, p0/m, z27.d
+; CHECK-NEXT:    ldr z11, [x0, #13, mul vl]
+; CHECK-NEXT:    frintx z4.d, p0/m, z4.d
+; CHECK-NEXT:    ldr z8, [x0, #12, mul vl]
+; CHECK-NEXT:    ldr z27, [x0, #5, mul vl]
+; CHECK-NEXT:    ldr z18, [x0, #11, mul vl]
+; CHECK-NEXT:    ldr z13, [x0, #10, mul vl]
+; CHECK-NEXT:    ldr z29, [x0, #6, mul vl]
+; CHECK-NEXT:    fcmge p1.d, p0/z, z5.d, z25.d
+; CHECK-NEXT:    mov x9, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    mov z24.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z30.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z5.d, p0/m, z5.d
-; CHECK-NEXT:    mov z26.d, #0x8000000000000000
-; CHECK-NEXT:    mov z2.d, #0x8000000000000000
-; CHECK-NEXT:    mov z13.d, #0x8000000000000000
-; CHECK-NEXT:    mov z12.d, #0x8000000000000000
-; CHECK-NEXT:    mov x10, #4890909195324358655 // =0x43dfffffffffffff
-; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    fcmge p3.d, p0/z, z4.d, z25.d
-; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z25.d
-; CHECK-NEXT:    ldr z29, [x0, #7, mul vl]
-; CHECK-NEXT:    ldr z24, [x0, #6, mul vl]
-; CHECK-NEXT:    ldr z10, [x0, #9, mul vl]
-; CHECK-NEXT:    ldr z8, [x0, #8, mul vl]
-; CHECK-NEXT:    ldr z7, [x0, #5, mul vl]
-; CHECK-NEXT:    ldr z14, [x0, #15, mul vl]
-; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z25.d
-; CHECK-NEXT:    fcmge p5.d, p0/z, z6.d, z25.d
-; CHECK-NEXT:    ldr z15, [x0, #14, mul vl]
+; CHECK-NEXT:    movprfx z28, z27
+; CHECK-NEXT:    frintx z28.d, p0/m, z27.d
+; CHECK-NEXT:    mov z1.d, x9
+; CHECK-NEXT:    fcmge p2.d, p0/z, z7.d, z25.d
 ; CHECK-NEXT:    frintx z29.d, p0/m, z29.d
-; CHECK-NEXT:    frintx z24.d, p0/m, z24.d
-; CHECK-NEXT:    movprfx z11, z10
-; CHECK-NEXT:    frintx z11.d, p0/m, z10.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z5.d, z25.d
-; CHECK-NEXT:    movprfx z9, z8
-; CHECK-NEXT:    frintx z9.d, p0/m, z8.d
-; CHECK-NEXT:    ldr z16, [x0, #11, mul vl]
-; CHECK-NEXT:    ldr z20, [x0, #13, mul vl]
-; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
-; CHECK-NEXT:    fcvtzs z28.d, p3/m, z4.d
-; CHECK-NEXT:    mov z10.d, #0x8000000000000000
-; CHECK-NEXT:    ldr z18, [x0, #12, mul vl]
-; CHECK-NEXT:    movprfx z19, z14
-; CHECK-NEXT:    frintx z19.d, p0/m, z14.d
-; CHECK-NEXT:    fcmge p3.d, p0/z, z29.d, z25.d
-; CHECK-NEXT:    ldr z17, [x0, #10, mul vl]
-; CHECK-NEXT:    frintx z15.d, p0/m, z15.d
-; CHECK-NEXT:    fcvtzs z27.d, p2/m, z1.d
-; CHECK-NEXT:    fcvtzs z30.d, p5/m, z6.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z24.d, z25.d
-; CHECK-NEXT:    fcmge p5.d, p0/z, z11.d, z25.d
-; CHECK-NEXT:    mov z14.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z16.d, p0/m, z16.d
-; CHECK-NEXT:    frintx z20.d, p0/m, z20.d
-; CHECK-NEXT:    fcvtzs z26.d, p4/m, z5.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z9.d, z25.d
+; CHECK-NEXT:    movprfx z3, z2
+; CHECK-NEXT:    frintx z3.d, p0/m, z2.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z14.d, z25.d
+; CHECK-NEXT:    mov z6.d, #0x8000000000000000
+; CHECK-NEXT:    mov z12.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z0.d, p1/m, z5.d
 ; CHECK-NEXT:    frintx z18.d, p0/m, z18.d
-; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.d
-; CHECK-NEXT:    fcmge p1.d, p0/z, z7.d, z25.d
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z17.d, p0/m, z17.d
-; CHECK-NEXT:    fcvtzs z10.d, p3/m, z29.d
-; CHECK-NEXT:    fcmge p3.d, p0/z, z19.d, z25.d
-; CHECK-NEXT:    mov z3.d, x10
-; CHECK-NEXT:    fcmge p6.d, p0/z, z15.d, z25.d
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p5.d, p0/z, z4.d, z25.d
+; CHECK-NEXT:    mov z16.d, #0x8000000000000000
+; CHECK-NEXT:    mov z17.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z24.d, p2/m, z7.d
+; CHECK-NEXT:    frintx z10.d, p0/m, z10.d
+; CHECK-NEXT:    frintx z9.d, p0/m, z9.d
+; CHECK-NEXT:    fcvtzs z30.d, p3/m, z14.d
+; CHECK-NEXT:    frintx z13.d, p0/m, z13.d
 ; CHECK-NEXT:    mov z21.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z13.d, p2/m, z24.d
-; CHECK-NEXT:    fcvtzs z14.d, p5/m, z11.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z16.d, z25.d
+; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov z22.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.d, p0/z, z20.d, z25.d
-; CHECK-NEXT:    mov z0.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z12.d, p4/m, z9.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z18.d, z25.d
+; CHECK-NEXT:    movprfx z0, z8
+; CHECK-NEXT:    frintx z0.d, p0/m, z8.d
+; CHECK-NEXT:    ldr z31, [x0, #7, mul vl]
+; CHECK-NEXT:    ldr z15, [x0, #8, mul vl]
+; CHECK-NEXT:    ldr z19, [x0, #9, mul vl]
+; CHECK-NEXT:    fcmge p1.d, p0/z, z28.d, z25.d
+; CHECK-NEXT:    fcvtzs z6.d, p5/m, z4.d
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
+; CHECK-NEXT:    fcmgt p10.d, p0/z, z14.d, z1.d
+; CHECK-NEXT:    mov z27.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    mov z23.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z8.d, p1/m, z7.d
-; CHECK-NEXT:    fcmge p1.d, p0/z, z17.d, z25.d
-; CHECK-NEXT:    fcvtzs z31.d, p3/m, z19.d
-; CHECK-NEXT:    mov z25.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p11.d, p0/z, z19.d, z3.d
-; CHECK-NEXT:    fcvtzs z21.d, p6/m, z15.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z15.d, z3.d
-; CHECK-NEXT:    fcmuo p6.d, p0/z, z15.d, z15.d
-; CHECK-NEXT:    mov z15.d, #0x8000000000000000
-; CHECK-NEXT:    fcmgt p7.d, p0/z, z1.d, z3.d
-; CHECK-NEXT:    fcvtzs z22.d, p2/m, z16.d
-; CHECK-NEXT:    fcvtzs z0.d, p5/m, z20.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z24.d, z3.d
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z20.d, z3.d
-; CHECK-NEXT:    fcvtzs z23.d, p4/m, z18.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z19.d, z19.d
-; CHECK-NEXT:    mov z31.d, p11/m, z25.d
-; CHECK-NEXT:    sel z19.d, p3, z25.d, z21.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z18.d, z3.d
-; CHECK-NEXT:    fcvtzs z15.d, p1/m, z17.d
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z20.d, z20.d
-; CHECK-NEXT:    mov z27.d, p7/m, z25.d
-; CHECK-NEXT:    fcmgt p7.d, p0/z, z16.d, z3.d
-; CHECK-NEXT:    mov z13.d, p2/m, z25.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z17.d, z3.d
-; CHECK-NEXT:    mov z0.d, p5/m, z25.d
-; CHECK-NEXT:    mov z31.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    movprfx z20, z31
+; CHECK-NEXT:    frintx z20.d, p0/m, z31.d
+; CHECK-NEXT:    frintx z15.d, p0/m, z15.d
+; CHECK-NEXT:    mov z31.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p2.d, p0/z, z29.d, z25.d
+; CHECK-NEXT:    fcmuo p8.d, p0/z, z14.d, z14.d
+; CHECK-NEXT:    movprfx z14, z19
+; CHECK-NEXT:    frintx z14.d, p0/m, z19.d
+; CHECK-NEXT:    movprfx z19, z11
+; CHECK-NEXT:    frintx z19.d, p0/m, z11.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z25.d
+; CHECK-NEXT:    fcvtzs z12.d, p1/m, z28.d
+; CHECK-NEXT:    mov z11.d, #0x8000000000000000
+; CHECK-NEXT:    mov z30.d, p10/m, z27.d
+; CHECK-NEXT:    fcvtzs z31.d, p2/m, z29.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z20.d, z25.d
+; CHECK-NEXT:    fcmge p6.d, p0/z, z15.d, z25.d
+; CHECK-NEXT:    fcvtzs z2.d, p4/m, z3.d
+; CHECK-NEXT:    mov z30.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p1.d, p0/z, z18.d, z25.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z19.d, z25.d
+; CHECK-NEXT:    fcvtzs z16.d, p5/m, z20.d
+; CHECK-NEXT:    fcvtzs z17.d, p6/m, z15.d
+; CHECK-NEXT:    fcmgt p12.d, p0/z, z28.d, z1.d
+; CHECK-NEXT:    fcvtzs z21.d, p1/m, z18.d
+; CHECK-NEXT:    fcvtzs z22.d, p2/m, z19.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z20.d, z1.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z10.d, z25.d
+; CHECK-NEXT:    fcmgt p11.d, p0/z, z15.d, z1.d
+; CHECK-NEXT:    sel z8.d, p12, z27.d, z12.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z9.d, z25.d
+; CHECK-NEXT:    sel z12.d, p5, z27.d, z16.d
+; CHECK-NEXT:    fcmge p6.d, p0/z, z14.d, z25.d
+; CHECK-NEXT:    fcvtzs z23.d, p3/m, z10.d
+; CHECK-NEXT:    fcmge p7.d, p0/z, z13.d, z25.d
+; CHECK-NEXT:    fcvtzs z26.d, p4/m, z9.d
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z15.d, z15.d
+; CHECK-NEXT:    sel z15.d, p11, z27.d, z17.d
+; CHECK-NEXT:    fcvtzs z11.d, p6/m, z14.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, z25.d
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z9.d, z1.d
+; CHECK-NEXT:    mov z15.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z10.d, z1.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z19.d, z1.d
+; CHECK-NEXT:    fcvtzs z25.d, p2/m, z0.d
+; CHECK-NEXT:    str z15, [x8, #8, mul vl]
+; CHECK-NEXT:    mov z26.d, p4/m, z27.d
+; CHECK-NEXT:    fcmuo p9.d, p0/z, z20.d, z20.d
+; CHECK-NEXT:    mov z20.d, #0x8000000000000000
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z9.d, z9.d
+; CHECK-NEXT:    sel z9.d, p5, z27.d, z23.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z10.d, z10.d
+; CHECK-NEXT:    sel z10.d, p3, z27.d, z22.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z19.d, z19.d
+; CHECK-NEXT:    fcvtzs z20.d, p7/m, z13.d
+; CHECK-NEXT:    mov z12.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    mov z26.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z9.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    str z12, [x8, #7, mul vl]
+; CHECK-NEXT:    mov z10.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z14.d, z1.d
+; CHECK-NEXT:    str z26, [x8, #15, mul vl]
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    str z9, [x8, #14, mul vl]
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z18.d, z1.d
+; CHECK-NEXT:    str z10, [x8, #13, mul vl]
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z13.d, z1.d
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z14.d, z14.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    sel z0.d, p4, z27.d, z11.d
+; CHECK-NEXT:    mov z25.d, p3/m, z27.d
+; CHECK-NEXT:    sel z26.d, p1, z27.d, z21.d
+; CHECK-NEXT:    sel z9.d, p2, z27.d, z20.d
 ; CHECK-NEXT:    fcmuo p4.d, p0/z, z18.d, z18.d
-; CHECK-NEXT:    sel z20.d, p3, z25.d, z23.d
-; CHECK-NEXT:    fcmuo p3.d, p0/z, z16.d, z16.d
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z11.d, z3.d
-; CHECK-NEXT:    mov z19.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p10.d, p0/z, z6.d, z3.d
-; CHECK-NEXT:    fcmgt p8.d, p0/z, z4.d, z3.d
-; CHECK-NEXT:    str z31, [x8, #15, mul vl]
-; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z17.d, z17.d
-; CHECK-NEXT:    sel z18.d, p7, z25.d, z22.d
-; CHECK-NEXT:    sel z31.d, p2, z25.d, z15.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z9.d, z3.d
-; CHECK-NEXT:    str z19, [x8, #14, mul vl]
-; CHECK-NEXT:    mov z20.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z11.d, z11.d
-; CHECK-NEXT:    str z0, [x8, #13, mul vl]
-; CHECK-NEXT:    mov z14.d, p5/m, z25.d
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z29.d, z3.d
-; CHECK-NEXT:    mov z18.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z31.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z7.d, z3.d
-; CHECK-NEXT:    str z20, [x8, #12, mul vl]
-; CHECK-NEXT:    fcmuo p3.d, p0/z, z9.d, z9.d
-; CHECK-NEXT:    sel z0.d, p2, z25.d, z12.d
-; CHECK-NEXT:    mov z14.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z7.d, z7.d
-; CHECK-NEXT:    fcmuo p2.d, p0/z, z29.d, z29.d
-; CHECK-NEXT:    str z18, [x8, #11, mul vl]
-; CHECK-NEXT:    sel z29.d, p5, z25.d, z10.d
-; CHECK-NEXT:    fcmuo p5.d, p0/z, z24.d, z24.d
-; CHECK-NEXT:    str z31, [x8, #10, mul vl]
-; CHECK-NEXT:    sel z7.d, p1, z25.d, z8.d
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z6.d, z6.d
-; CHECK-NEXT:    ldr z6, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    str z14, [x8, #9, mul vl]
-; CHECK-NEXT:    fcmgt p9.d, p0/z, z5.d, z3.d
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z29.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p2.d, p0/z, z5.d, z5.d
-; CHECK-NEXT:    mov z13.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p5.d, p0/z, z4.d, z4.d
-; CHECK-NEXT:    mov z7.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z6.d, z3.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z1.d, z1.d
-; CHECK-NEXT:    fcmuo p0.d, p0/z, z6.d, z6.d
-; CHECK-NEXT:    str z0, [x8, #8, mul vl]
-; CHECK-NEXT:    mov z28.d, p8/m, z25.d
-; CHECK-NEXT:    mov z26.d, p9/m, z25.d
-; CHECK-NEXT:    str z29, [x8, #7, mul vl]
-; CHECK-NEXT:    mov z30.d, p10/m, z25.d
-; CHECK-NEXT:    str z13, [x8, #6, mul vl]
-; CHECK-NEXT:    str z7, [x8, #5, mul vl]
-; CHECK-NEXT:    sel z0.d, p3, z25.d, z2.d
-; CHECK-NEXT:    mov z26.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    mov z30.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z28.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    mov z27.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    str z26, [x8, #3, mul vl]
-; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z25.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z13.d, z13.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z1.d
+; CHECK-NEXT:    str z0, [x8, #9, mul vl]
+; CHECK-NEXT:    mov z26.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    str z25, [x8, #12, mul vl]
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z4.d, z1.d
+; CHECK-NEXT:    mov z9.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z7.d, z1.d
+; CHECK-NEXT:    str z26, [x8, #11, mul vl]
+; CHECK-NEXT:    mov z2.d, p1/m, z27.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z28.d, z28.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z29.d, z1.d
+; CHECK-NEXT:    str z9, [x8, #10, mul vl]
+; CHECK-NEXT:    mov z6.d, p2/m, z27.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z5.d, z1.d
+; CHECK-NEXT:    ldr z1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    str z30, [x8, #4, mul vl]
-; CHECK-NEXT:    str z28, [x8, #2, mul vl]
-; CHECK-NEXT:    str z27, [x8, #1, mul vl]
-; CHECK-NEXT:    str z0, [x8]
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z29.d, z29.d
+; CHECK-NEXT:    sel z0.d, p4, z27.d, z24.d
+; CHECK-NEXT:    mov z8.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z7.d, z7.d
+; CHECK-NEXT:    sel z25.d, p6, z27.d, z31.d
+; CHECK-NEXT:    mov z1.d, p1/m, z27.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z4.d, z4.d
+; CHECK-NEXT:    str z8, [x8, #5, mul vl]
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    mov z25.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z5.d, z5.d
+; CHECK-NEXT:    mov z6.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    str z25, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    str z0, [x8, #3, mul vl]
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    str z6, [x8, #2, mul vl]
+; CHECK-NEXT:    str z2, [x8, #1, mul vl]
+; CHECK-NEXT:    str z1, [x8]
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr p11, [sp] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #18
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 32 x i64> @llvm.llrint.nxv32i64.nxv16f64(<vscale x 32 x double> %x)
diff --git a/llvm/test/CodeGen/AArch64/sve-lrint.ll b/llvm/test/CodeGen/AArch64/sve-lrint.ll
index 58ac53d36f9ae..c072005c793d7 100644
--- a/llvm/test/CodeGen/AArch64/sve-lrint.ll
+++ b/llvm/test/CodeGen/AArch64/sve-lrint.ll
@@ -7,17 +7,17 @@ define <vscale x 1 x iXLen> @lrint_v1f16(<vscale x 1 x half> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.h, w8
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
 ; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
-; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z2.h
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f16(<vscale x 1 x half> %x)
@@ -30,17 +30,17 @@ define <vscale x 2 x iXLen> @lrint_v2f16(<vscale x 2 x half> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #64511 // =0xfbff
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.h, w8
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
 ; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
-; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z2.h
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z1.h
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f16(<vscale x 2 x half> %x)
@@ -59,20 +59,20 @@ define <vscale x 4 x iXLen> @lrint_v4f16(<vscale x 4 x half> %x) {
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z4.d, #0x8000000000000000
-; CHECK-NEXT:    mov z5.h, w8
+; CHECK-NEXT:    mov z5.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    frintx z1.h, p0/m, z1.h
 ; CHECK-NEXT:    frintx z0.h, p0/m, z0.h
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z1.h, z2.h
 ; CHECK-NEXT:    fcmge p2.h, p0/z, z0.h, z2.h
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    fcmuo p3.h, p0/z, z1.h, z1.h
-; CHECK-NEXT:    fcvtzs z3.d, p1/m, z1.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z1.h, z5.h
-; CHECK-NEXT:    fcvtzs z4.d, p2/m, z0.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z5.h
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z1.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z1.h, z2.h
+; CHECK-NEXT:    fcvtzs z3.d, p2/m, z0.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z0.h, z2.h
 ; CHECK-NEXT:    fcmuo p0.h, p0/z, z0.h, z0.h
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z3.d
-; CHECK-NEXT:    sel z1.d, p2, z2.d, z4.d
+; CHECK-NEXT:    sel z0.d, p1, z5.d, z4.d
+; CHECK-NEXT:    sel z1.d, p2, z5.d, z3.d
 ; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
@@ -97,10 +97,10 @@ define <vscale x 8 x iXLen> @lrint_v8f16(<vscale x 8 x half> %x) {
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov z4.h, w8
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z6.d, #0x8000000000000000
-; CHECK-NEXT:    mov z25.h, w8
 ; CHECK-NEXT:    mov z7.d, #0x8000000000000000
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
+; CHECK-NEXT:    mov z25.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    uunpklo z2.d, z1.s
 ; CHECK-NEXT:    uunpkhi z1.d, z1.s
 ; CHECK-NEXT:    uunpklo z3.d, z0.s
@@ -108,36 +108,36 @@ define <vscale x 8 x iXLen> @lrint_v8f16(<vscale x 8 x half> %x) {
 ; CHECK-NEXT:    frintx z2.h, p0/m, z2.h
 ; CHECK-NEXT:    frintx z1.h, p0/m, z1.h
 ; CHECK-NEXT:    frintx z3.h, p0/m, z3.h
-; CHECK-NEXT:    movprfx z5, z0
-; CHECK-NEXT:    frintx z5.h, p0/m, z0.h
-; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z24, z0
+; CHECK-NEXT:    frintx z24.h, p0/m, z0.h
+; CHECK-NEXT:    mov z0.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z2.h, z4.h
 ; CHECK-NEXT:    fcmge p2.h, p0/z, z1.h, z4.h
 ; CHECK-NEXT:    fcmge p3.h, p0/z, z3.h, z4.h
-; CHECK-NEXT:    fcmge p4.h, p0/z, z5.h, z4.h
-; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z5.h, z25.h
-; CHECK-NEXT:    fcmuo p6.h, p0/z, z1.h, z1.h
-; CHECK-NEXT:    fcvtzs z0.d, p1/m, z2.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z1.h, z0.h
+; CHECK-NEXT:    fcvtzs z5.d, p1/m, z2.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z24.h, z4.h
+; CHECK-NEXT:    mov z4.d, #0x8000000000000000
 ; CHECK-NEXT:    fcvtzs z6.d, p2/m, z1.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z2.h, z25.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z1.h, z25.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z2.h, z0.h
 ; CHECK-NEXT:    fcvtzs z7.d, p3/m, z3.h
-; CHECK-NEXT:    fcmgt p3.h, p0/z, z3.h, z25.h
-; CHECK-NEXT:    fcvtzs z24.d, p4/m, z5.h
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z2.h, z2.h
-; CHECK-NEXT:    mov z0.d, p1/m, z4.d
-; CHECK-NEXT:    fcmuo p1.h, p0/z, z3.h, z3.h
-; CHECK-NEXT:    fcmuo p0.h, p0/z, z5.h, z5.h
-; CHECK-NEXT:    sel z1.d, p2, z4.d, z6.d
-; CHECK-NEXT:    sel z2.d, p3, z4.d, z7.d
-; CHECK-NEXT:    sel z3.d, p5, z4.d, z24.d
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z3.h, z0.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z24.h, z0.h
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z24.h
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z2.h, z2.h
+; CHECK-NEXT:    sel z0.d, p2, z25.d, z5.d
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    sel z1.d, p4, z25.d, z6.d
+; CHECK-NEXT:    sel z2.d, p5, z25.d, z7.d
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z3.h, z3.h
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z24.h, z24.h
+; CHECK-NEXT:    sel z3.d, p6, z25.d, z4.d
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -151,7 +151,7 @@ define <vscale x 16 x iXLen> @lrint_v16f16(<vscale x 16 x half> %x) {
 ; CHECK-LABEL: lrint_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-4
+; CHECK-NEXT:    addvl sp, sp, #-1
 ; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
@@ -159,110 +159,102 @@ define <vscale x 16 x iXLen> @lrint_v16f16(<vscale x 16 x half> %x) {
 ; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16
 ; CHECK-NEXT:    uunpklo z2.s, z0.h
-; CHECK-NEXT:    uunpkhi z0.s, z0.h
+; CHECK-NEXT:    uunpkhi z3.s, z0.h
 ; CHECK-NEXT:    mov w8, #64511 // =0xfbff
-; CHECK-NEXT:    uunpklo z3.s, z1.h
-; CHECK-NEXT:    uunpkhi z1.s, z1.h
+; CHECK-NEXT:    uunpklo z7.s, z1.h
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov z24.h, w8
-; CHECK-NEXT:    mov w8, #31743 // =0x7bff
-; CHECK-NEXT:    mov z7.d, #0x8000000000000000
-; CHECK-NEXT:    mov z27.d, #0x8000000000000000
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    mov z30.d, #0x8000000000000000
-; CHECK-NEXT:    uunpklo z4.d, z2.s
-; CHECK-NEXT:    uunpkhi z2.d, z2.s
-; CHECK-NEXT:    uunpklo z5.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    uunpklo z6.d, z3.s
-; CHECK-NEXT:    uunpklo z25.d, z1.s
-; CHECK-NEXT:    uunpkhi z3.d, z3.s
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    uunpkhi z1.s, z1.h
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    mov z10.d, #0x7fffffffffffffff
-; CHECK-NEXT:    frintx z4.h, p0/m, z4.h
-; CHECK-NEXT:    frintx z2.h, p0/m, z2.h
-; CHECK-NEXT:    frintx z5.h, p0/m, z5.h
-; CHECK-NEXT:    movprfx z26, z0
-; CHECK-NEXT:    frintx z26.h, p0/m, z0.h
-; CHECK-NEXT:    uunpkhi z0.d, z1.s
-; CHECK-NEXT:    frintx z6.h, p0/m, z6.h
-; CHECK-NEXT:    movprfx z29, z3
-; CHECK-NEXT:    frintx z29.h, p0/m, z3.h
-; CHECK-NEXT:    frintx z25.h, p0/m, z25.h
-; CHECK-NEXT:    mov z1.h, w8
+; CHECK-NEXT:    uunpklo z4.d, z2.s
+; CHECK-NEXT:    uunpklo z24.d, z3.s
+; CHECK-NEXT:    uunpkhi z25.d, z3.s
+; CHECK-NEXT:    uunpkhi z6.d, z2.s
+; CHECK-NEXT:    uunpklo z26.d, z7.s
+; CHECK-NEXT:    uunpkhi z7.d, z7.s
+; CHECK-NEXT:    mov z2.h, w8
+; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    uunpklo z30.d, z1.s
+; CHECK-NEXT:    mov z29.h, w8
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.h, p0/z, z4.h, z24.h
-; CHECK-NEXT:    fcmge p2.h, p0/z, z2.h, z24.h
-; CHECK-NEXT:    fcmge p3.h, p0/z, z5.h, z24.h
-; CHECK-NEXT:    movprfx z9, z0
-; CHECK-NEXT:    frintx z9.h, p0/m, z0.h
-; CHECK-NEXT:    fcmge p4.h, p0/z, z26.h, z24.h
-; CHECK-NEXT:    fcmge p5.h, p0/z, z6.h, z24.h
-; CHECK-NEXT:    fcmge p7.h, p0/z, z25.h, z24.h
-; CHECK-NEXT:    fcmge p6.h, p0/z, z29.h, z24.h
-; CHECK-NEXT:    fcmgt p8.h, p0/z, z6.h, z1.h
-; CHECK-NEXT:    fcmgt p10.h, p0/z, z25.h, z1.h
-; CHECK-NEXT:    fcmuo p9.h, p0/z, z5.h, z5.h
-; CHECK-NEXT:    fcvtzs z7.d, p1/m, z4.h
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z4.h, z1.h
-; CHECK-NEXT:    fcvtzs z27.d, p2/m, z2.h
-; CHECK-NEXT:    fcmge p2.h, p0/z, z9.h, z24.h
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z28.d, p3/m, z5.h
-; CHECK-NEXT:    fcvtzs z3.d, p4/m, z26.h
-; CHECK-NEXT:    fcvtzs z30.d, p5/m, z6.h
-; CHECK-NEXT:    fcvtzs z8.d, p7/m, z25.h
-; CHECK-NEXT:    fcmgt p4.h, p0/z, z2.h, z1.h
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z5.h, z1.h
-; CHECK-NEXT:    fcmgt p7.h, p0/z, z26.h, z1.h
-; CHECK-NEXT:    fcvtzs z31.d, p6/m, z29.h
-; CHECK-NEXT:    sel z0.d, p1, z10.d, z7.d
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z29.h, z1.h
-; CHECK-NEXT:    fcvtzs z24.d, p2/m, z9.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z9.h, z1.h
-; CHECK-NEXT:    fcmuo p3.h, p0/z, z4.h, z4.h
-; CHECK-NEXT:    fcmuo p6.h, p0/z, z2.h, z2.h
-; CHECK-NEXT:    sel z4.d, p8, z10.d, z30.d
-; CHECK-NEXT:    fcmuo p8.h, p0/z, z25.h, z25.h
-; CHECK-NEXT:    sel z1.d, p4, z10.d, z27.d
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z26.h, z26.h
-; CHECK-NEXT:    sel z2.d, p5, z10.d, z28.d
-; CHECK-NEXT:    mov z3.d, p7/m, z10.d
-; CHECK-NEXT:    fcmuo p5.h, p0/z, z6.h, z6.h
-; CHECK-NEXT:    fcmuo p7.h, p0/z, z29.h, z29.h
-; CHECK-NEXT:    sel z5.d, p1, z10.d, z31.d
-; CHECK-NEXT:    sel z6.d, p10, z10.d, z8.d
-; CHECK-NEXT:    ldr z8, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    movprfx z27, z4
+; CHECK-NEXT:    frintx z27.h, p0/m, z4.h
+; CHECK-NEXT:    frintx z24.h, p0/m, z24.h
+; CHECK-NEXT:    frintx z25.h, p0/m, z25.h
+; CHECK-NEXT:    movprfx z28, z6
+; CHECK-NEXT:    frintx z28.h, p0/m, z6.h
+; CHECK-NEXT:    mov z4.d, #0x8000000000000000
+; CHECK-NEXT:    frintx z26.h, p0/m, z26.h
+; CHECK-NEXT:    frintx z7.h, p0/m, z7.h
+; CHECK-NEXT:    mov z6.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p1.h, p0/z, z27.h, z2.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z24.h, z2.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z25.h, z2.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z28.h, z2.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z26.h, z2.h
+; CHECK-NEXT:    fcvtzs z0.d, p1/m, z27.h
+; CHECK-NEXT:    fcvtzs z4.d, p3/m, z24.h
+; CHECK-NEXT:    fcvtzs z5.d, p4/m, z25.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z27.h, z29.h
+; CHECK-NEXT:    fcvtzs z3.d, p2/m, z28.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z7.h, z2.h
+; CHECK-NEXT:    fcvtzs z6.d, p5/m, z26.h
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z27.h, z27.h
+; CHECK-NEXT:    movprfx z27, z30
+; CHECK-NEXT:    frintx z27.h, p0/m, z30.h
+; CHECK-NEXT:    movprfx z30, z1
+; CHECK-NEXT:    frintx z30.h, p0/m, z1.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z28.h, z29.h
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z28.h, z28.h
+; CHECK-NEXT:    mov z28.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z31.d, p4/m, z7.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z27.h, z2.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z24.h, z29.h
+; CHECK-NEXT:    fcmuo p7.h, p0/z, z24.h, z24.h
+; CHECK-NEXT:    mov z24.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcmgt p8.h, p0/z, z25.h, z29.h
+; CHECK-NEXT:    fcvtzs z28.d, p4/m, z27.h
+; CHECK-NEXT:    fcmuo p10.h, p0/z, z25.h, z25.h
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
+; CHECK-NEXT:    sel z1.d, p5, z24.d, z3.d
+; CHECK-NEXT:    mov z0.d, p3/m, z24.d
+; CHECK-NEXT:    sel z3.d, p8, z24.d, z5.d
+; CHECK-NEXT:    fcmge p4.h, p0/z, z30.h, z2.h
+; CHECK-NEXT:    sel z2.d, p6, z24.d, z4.d
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z3.d, p10/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    fcmuo p0.h, p0/z, z9.h, z9.h
-; CHECK-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    sel z7.d, p2, z10.d, z24.d
-; CHECK-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
-; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    mov z5.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p7/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z6.d, p8/m, #0 // =0x0
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p9.h, p0/z, z26.h, z29.h
+; CHECK-NEXT:    fcvtzs z25.d, p4/m, z30.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z7.h, z29.h
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z27.h, z29.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z30.h, z29.h
+; CHECK-NEXT:    sel z4.d, p9, z24.d, z6.d
+; CHECK-NEXT:    fcmuo p8.h, p0/z, z7.h, z7.h
+; CHECK-NEXT:    sel z5.d, p5, z24.d, z31.d
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    sel z6.d, p6, z24.d, z28.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    fcmuo p9.h, p0/z, z27.h, z27.h
+; CHECK-NEXT:    sel z7.d, p4, z24.d, z25.d
 ; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z26.h, z26.h
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z30.h, z30.h
+; CHECK-NEXT:    mov z6.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f16(<vscale x 16 x half> %x)
@@ -274,32 +266,32 @@ define <vscale x 32 x iXLen> @lrint_v32f16(<vscale x 32 x half> %x) {
 ; CHECK-LABEL: lrint_v32f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-17
-; CHECK-NEXT:    str p11, [sp] // 2-byte Folded Spill
-; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 16 + 144 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
@@ -311,216 +303,215 @@ define <vscale x 32 x iXLen> @lrint_v32f16(<vscale x 32 x half> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x40, 0x1c // $d14 @ cfa - 56 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16
 ; CHECK-NEXT:    uunpklo z4.s, z0.h
-; CHECK-NEXT:    uunpkhi z0.s, z0.h
+; CHECK-NEXT:    uunpkhi z5.s, z0.h
 ; CHECK-NEXT:    mov w9, #64511 // =0xfbff
-; CHECK-NEXT:    uunpklo z25.s, z1.h
-; CHECK-NEXT:    uunpkhi z10.s, z1.h
-; CHECK-NEXT:    uunpklo z9.s, z2.h
+; CHECK-NEXT:    uunpklo z6.s, z1.h
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uunpkhi z12.s, z3.h
-; CHECK-NEXT:    mov z27.h, w9
+; CHECK-NEXT:    uunpkhi z28.s, z1.h
+; CHECK-NEXT:    mov z30.h, w9
 ; CHECK-NEXT:    mov w9, #31743 // =0x7bff
+; CHECK-NEXT:    uunpklo z13.s, z2.h
+; CHECK-NEXT:    mov z9.d, #0x8000000000000000
 ; CHECK-NEXT:    uunpkhi z14.s, z2.h
+; CHECK-NEXT:    uunpkhi z17.s, z3.h
+; CHECK-NEXT:    uunpklo z7.d, z4.s
+; CHECK-NEXT:    uunpkhi z4.d, z4.s
+; CHECK-NEXT:    uunpklo z27.d, z5.s
+; CHECK-NEXT:    uunpklo z31.d, z6.s
+; CHECK-NEXT:    uunpkhi z8.d, z6.s
+; CHECK-NEXT:    uunpkhi z29.d, z5.s
+; CHECK-NEXT:    uunpkhi z11.d, z28.s
+; CHECK-NEXT:    uunpklo z10.d, z28.s
 ; CHECK-NEXT:    uunpklo z15.s, z3.h
-; CHECK-NEXT:    uunpklo z7.d, z0.s
-; CHECK-NEXT:    uunpklo z5.d, z4.s
-; CHECK-NEXT:    uunpkhi z6.d, z4.s
-; CHECK-NEXT:    uunpklo z29.d, z25.s
-; CHECK-NEXT:    uunpkhi z26.d, z0.s
-; CHECK-NEXT:    uunpklo z8.d, z10.s
-; CHECK-NEXT:    uunpkhi z11.d, z10.s
-; CHECK-NEXT:    uunpklo z10.d, z9.s
-; CHECK-NEXT:    uunpkhi z13.d, z9.s
-; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    uunpklo z16.d, z12.s
-; CHECK-NEXT:    uunpklo z18.d, z14.s
+; CHECK-NEXT:    uunpklo z16.d, z14.s
+; CHECK-NEXT:    uunpkhi z14.d, z14.s
+; CHECK-NEXT:    mov z24.d, #0x8000000000000000
 ; CHECK-NEXT:    movprfx z1, z7
 ; CHECK-NEXT:    frintx z1.h, p0/m, z7.h
-; CHECK-NEXT:    movprfx z4, z5
-; CHECK-NEXT:    frintx z4.h, p0/m, z5.h
-; CHECK-NEXT:    movprfx z5, z6
-; CHECK-NEXT:    frintx z5.h, p0/m, z6.h
-; CHECK-NEXT:    movprfx z7, z29
-; CHECK-NEXT:    frintx z7.h, p0/m, z29.h
-; CHECK-NEXT:    movprfx z6, z26
-; CHECK-NEXT:    frintx z6.h, p0/m, z26.h
-; CHECK-NEXT:    mov z29.d, #0x8000000000000000
-; CHECK-NEXT:    movprfx z9, z11
-; CHECK-NEXT:    frintx z9.h, p0/m, z11.h
-; CHECK-NEXT:    movprfx z3, z10
-; CHECK-NEXT:    frintx z3.h, p0/m, z10.h
-; CHECK-NEXT:    movprfx z10, z13
-; CHECK-NEXT:    frintx z10.h, p0/m, z13.h
-; CHECK-NEXT:    uunpkhi z26.d, z25.s
-; CHECK-NEXT:    uunpkhi z13.d, z12.s
-; CHECK-NEXT:    frintx z8.h, p0/m, z8.h
-; CHECK-NEXT:    fcmge p3.h, p0/z, z1.h, z27.h
-; CHECK-NEXT:    uunpkhi z14.d, z14.s
+; CHECK-NEXT:    movprfx z5, z27
+; CHECK-NEXT:    frintx z5.h, p0/m, z27.h
+; CHECK-NEXT:    frintx z4.h, p0/m, z4.h
+; CHECK-NEXT:    movprfx z12, z31
+; CHECK-NEXT:    frintx z12.h, p0/m, z31.h
+; CHECK-NEXT:    movprfx z27, z8
+; CHECK-NEXT:    frintx z27.h, p0/m, z8.h
+; CHECK-NEXT:    movprfx z6, z29
+; CHECK-NEXT:    frintx z6.h, p0/m, z29.h
+; CHECK-NEXT:    movprfx z31, z10
+; CHECK-NEXT:    frintx z31.h, p0/m, z10.h
+; CHECK-NEXT:    mov z7.d, #0x8000000000000000
+; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z3, z16
+; CHECK-NEXT:    frintx z3.h, p0/m, z16.h
+; CHECK-NEXT:    frintx z11.h, p0/m, z11.h
+; CHECK-NEXT:    mov z29.h, w9
+; CHECK-NEXT:    uunpklo z10.d, z13.s
+; CHECK-NEXT:    uunpkhi z13.d, z13.s
+; CHECK-NEXT:    uunpkhi z20.d, z15.s
+; CHECK-NEXT:    uunpklo z16.d, z17.s
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z0.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p2.h, p0/z, z7.h, z27.h
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p4.h, p0/z, z4.h, z27.h
-; CHECK-NEXT:    fcmge p5.h, p0/z, z5.h, z27.h
-; CHECK-NEXT:    uunpklo z19.d, z15.s
-; CHECK-NEXT:    uunpkhi z15.d, z15.s
-; CHECK-NEXT:    movprfx z20, z13
-; CHECK-NEXT:    frintx z20.h, p0/m, z13.h
-; CHECK-NEXT:    mov z30.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.h, p0/z, z6.h, z27.h
-; CHECK-NEXT:    frintx z26.h, p0/m, z26.h
-; CHECK-NEXT:    fcvtzs z29.d, p3/m, z1.h
-; CHECK-NEXT:    fcmge p3.h, p0/z, z9.h, z27.h
-; CHECK-NEXT:    mov z11.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z31.d, p2/m, z7.h
-; CHECK-NEXT:    fcmge p2.h, p0/z, z8.h, z27.h
-; CHECK-NEXT:    mov z17.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z16.h, p0/m, z16.h
-; CHECK-NEXT:    frintx z14.h, p0/m, z14.h
-; CHECK-NEXT:    fcvtzs z0.d, p4/m, z4.h
-; CHECK-NEXT:    fcvtzs z28.d, p5/m, z5.h
-; CHECK-NEXT:    fcmge p4.h, p0/z, z3.h, z27.h
-; CHECK-NEXT:    mov z12.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.h, p0/z, z10.h, z27.h
-; CHECK-NEXT:    mov z13.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z19.h, p0/m, z19.h
-; CHECK-NEXT:    frintx z15.h, p0/m, z15.h
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z30.d, p1/m, z6.h
-; CHECK-NEXT:    fcmge p1.h, p0/z, z26.h, z27.h
+; CHECK-NEXT:    mov z18.d, #0x8000000000000000
+; CHECK-NEXT:    uunpklo z15.d, z15.s
 ; CHECK-NEXT:    mov z2.d, #0x8000000000000000
-; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    frintx z18.h, p0/m, z18.h
-; CHECK-NEXT:    fcvtzs z11.d, p3/m, z9.h
-; CHECK-NEXT:    fcmge p3.h, p0/z, z20.h, z27.h
-; CHECK-NEXT:    mov z25.h, w9
-; CHECK-NEXT:    fcvtzs z17.d, p2/m, z8.h
-; CHECK-NEXT:    fcmge p6.h, p0/z, z16.h, z27.h
 ; CHECK-NEXT:    mov z21.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p2.h, p0/z, z14.h, z27.h
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
+; CHECK-NEXT:    mov z28.d, #0x7fffffffffffffff
+; CHECK-NEXT:    movprfx z19, z13
+; CHECK-NEXT:    frintx z19.h, p0/m, z13.h
+; CHECK-NEXT:    movprfx z13, z14
+; CHECK-NEXT:    frintx z13.h, p0/m, z14.h
+; CHECK-NEXT:    frintx z10.h, p0/m, z10.h
+; CHECK-NEXT:    frintx z16.h, p0/m, z16.h
 ; CHECK-NEXT:    mov z22.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z12.d, p4/m, z3.h
-; CHECK-NEXT:    fcvtzs z13.d, p5/m, z10.h
-; CHECK-NEXT:    fcmge p4.h, p0/z, z19.h, z27.h
 ; CHECK-NEXT:    mov z23.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.h, p0/z, z15.h, z27.h
-; CHECK-NEXT:    mov z0.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z2.d, p1/m, z26.h
-; CHECK-NEXT:    fcmge p1.h, p0/z, z18.h, z27.h
-; CHECK-NEXT:    fcvtzs z24.d, p3/m, z20.h
-; CHECK-NEXT:    mov z27.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p11.h, p0/z, z20.h, z25.h
-; CHECK-NEXT:    fcvtzs z21.d, p6/m, z16.h
-; CHECK-NEXT:    fcmgt p3.h, p0/z, z16.h, z25.h
-; CHECK-NEXT:    fcmuo p6.h, p0/z, z16.h, z16.h
-; CHECK-NEXT:    fcvtzs z22.d, p2/m, z14.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z8.h, z25.h
-; CHECK-NEXT:    mov z16.d, #0x8000000000000000
-; CHECK-NEXT:    fcmgt p7.h, p0/z, z5.h, z25.h
-; CHECK-NEXT:    fcvtzs z23.d, p4/m, z19.h
-; CHECK-NEXT:    fcvtzs z0.d, p5/m, z15.h
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z20.h, z20.h
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z15.h, z25.h
-; CHECK-NEXT:    mov z24.d, p11/m, z27.d
-; CHECK-NEXT:    sel z20.d, p3, z27.d, z21.d
-; CHECK-NEXT:    fcmgt p3.h, p0/z, z19.h, z25.h
-; CHECK-NEXT:    fcmgt p8.h, p0/z, z1.h, z25.h
-; CHECK-NEXT:    mov z17.d, p2/m, z27.d
-; CHECK-NEXT:    fcvtzs z16.d, p1/m, z18.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z18.h, z25.h
-; CHECK-NEXT:    mov z28.d, p7/m, z27.d
-; CHECK-NEXT:    fcmgt p7.h, p0/z, z14.h, z25.h
-; CHECK-NEXT:    fcmuo p1.h, p0/z, z15.h, z15.h
-; CHECK-NEXT:    mov z0.d, p5/m, z27.d
-; CHECK-NEXT:    mov z24.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z10.h, z25.h
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z19.h, z19.h
-; CHECK-NEXT:    sel z19.d, p3, z27.d, z23.d
-; CHECK-NEXT:    fcmuo p3.h, p0/z, z14.h, z14.h
-; CHECK-NEXT:    mov z20.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p9.h, p0/z, z6.h, z25.h
-; CHECK-NEXT:    fcmgt p10.h, p0/z, z7.h, z25.h
-; CHECK-NEXT:    str z24, [x8, #15, mul vl]
-; CHECK-NEXT:    sel z24.d, p2, z27.d, z16.d
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z3.h, z25.h
-; CHECK-NEXT:    sel z15.d, p7, z27.d, z22.d
-; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z13.d, p5/m, z27.d
-; CHECK-NEXT:    str z20, [x8, #14, mul vl]
-; CHECK-NEXT:    fcmgt p5.h, p0/z, z9.h, z25.h
-; CHECK-NEXT:    fcmuo p1.h, p0/z, z18.h, z18.h
-; CHECK-NEXT:    mov z19.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z10.h, z10.h
-; CHECK-NEXT:    mov z29.d, p8/m, z27.d
-; CHECK-NEXT:    str z0, [x8, #13, mul vl]
-; CHECK-NEXT:    mov z15.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    frintx z15.h, p0/m, z15.h
+; CHECK-NEXT:    mov z14.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p4.h, p0/z, z4.h, z30.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z12.h, z30.h
+; CHECK-NEXT:    fcmgt p9.h, p0/z, z12.h, z29.h
+; CHECK-NEXT:    fcmuo p8.h, p0/z, z12.h, z12.h
+; CHECK-NEXT:    fcvtzs z7.d, p4/m, z4.h
+; CHECK-NEXT:    fcvtzs z8.d, p2/m, z12.h
+; CHECK-NEXT:    mov z12.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p4.h, p0/z, z27.h, z30.h
+; CHECK-NEXT:    fcmuo p10.h, p0/z, z11.h, z11.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z5.h, z30.h
+; CHECK-NEXT:    mov z8.d, p9/m, z28.d
+; CHECK-NEXT:    fcvtzs z9.d, p4/m, z27.h
+; CHECK-NEXT:    fcmge p4.h, p0/z, z11.h, z30.h
+; CHECK-NEXT:    fcvtzs z24.d, p3/m, z5.h
+; CHECK-NEXT:    mov z8.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p1.h, p0/z, z6.h, z30.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z1.h, z30.h
+; CHECK-NEXT:    str z8, [x8, #4, mul vl]
+; CHECK-NEXT:    fcvtzs z12.d, p4/m, z11.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z11.h, z29.h
+; CHECK-NEXT:    uunpkhi z11.d, z17.s
+; CHECK-NEXT:    movprfx z17, z20
+; CHECK-NEXT:    frintx z17.h, p0/m, z20.h
+; CHECK-NEXT:    fcvtzs z25.d, p1/m, z6.h
+; CHECK-NEXT:    mov z20.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z0.d, p5/m, z1.h
+; CHECK-NEXT:    fcmge p6.h, p0/z, z10.h, z30.h
+; CHECK-NEXT:    frintx z11.h, p0/m, z11.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z31.h, z30.h
+; CHECK-NEXT:    fcmge p1.h, p0/z, z13.h, z30.h
+; CHECK-NEXT:    fcvtzs z18.d, p6/m, z10.h
+; CHECK-NEXT:    fcmgt p11.h, p0/z, z10.h, z29.h
+; CHECK-NEXT:    fcmge p5.h, p0/z, z11.h, z30.h
+; CHECK-NEXT:    fcvtzs z2.d, p3/m, z31.h
+; CHECK-NEXT:    fcvtzs z21.d, p1/m, z13.h
+; CHECK-NEXT:    fcmge p2.h, p0/z, z17.h, z30.h
+; CHECK-NEXT:    fcmge p3.h, p0/z, z16.h, z30.h
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z10.h, z10.h
+; CHECK-NEXT:    sel z10.d, p4, z28.d, z12.d
+; CHECK-NEXT:    sel z12.d, p11, z28.d, z18.d
+; CHECK-NEXT:    fcvtzs z26.d, p5/m, z11.h
+; CHECK-NEXT:    fcvtzs z22.d, p2/m, z17.h
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z11.h, z29.h
+; CHECK-NEXT:    fcvtzs z23.d, p3/m, z16.h
+; CHECK-NEXT:    mov z10.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    mov z12.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p6.h, p0/z, z19.h, z30.h
+; CHECK-NEXT:    str z10, [x8, #7, mul vl]
+; CHECK-NEXT:    fcmge p7.h, p0/z, z3.h, z30.h
+; CHECK-NEXT:    str z12, [x8, #8, mul vl]
+; CHECK-NEXT:    mov z26.d, p4/m, z28.d
+; CHECK-NEXT:    fcmge p2.h, p0/z, z15.h, z30.h
+; CHECK-NEXT:    mov z30.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z14.d, p6/m, z19.h
+; CHECK-NEXT:    fcmgt p5.h, p0/z, z16.h, z29.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z17.h, z29.h
+; CHECK-NEXT:    fcvtzs z20.d, p7/m, z3.h
+; CHECK-NEXT:    fcvtzs z30.d, p2/m, z15.h
+; CHECK-NEXT:    fcmuo p1.h, p0/z, z11.h, z11.h
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z16.h, z16.h
+; CHECK-NEXT:    sel z11.d, p5, z28.d, z23.d
+; CHECK-NEXT:    sel z16.d, p3, z28.d, z22.d
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z19.h, z29.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z15.h, z29.h
+; CHECK-NEXT:    mov z26.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z11.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z13.h, z29.h
+; CHECK-NEXT:    fcmuo p6.h, p0/z, z17.h, z17.h
+; CHECK-NEXT:    str z26, [x8, #15, mul vl]
+; CHECK-NEXT:    sel z26.d, p4, z28.d, z14.d
+; CHECK-NEXT:    str z11, [x8, #14, mul vl]
+; CHECK-NEXT:    mov z30.d, p3/m, z28.d
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z3.h, z29.h
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z13.h, z13.h
 ; CHECK-NEXT:    fcmuo p3.h, p0/z, z3.h, z3.h
-; CHECK-NEXT:    sel z0.d, p2, z27.d, z12.d
-; CHECK-NEXT:    fcmuo p2.h, p0/z, z9.h, z9.h
-; CHECK-NEXT:    mov z30.d, p9/m, z27.d
-; CHECK-NEXT:    str z19, [x8, #12, mul vl]
-; CHECK-NEXT:    sel z3.d, p5, z27.d, z11.d
-; CHECK-NEXT:    mov z24.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    str z15, [x8, #11, mul vl]
-; CHECK-NEXT:    fcmgt p1.h, p0/z, z26.h, z25.h
-; CHECK-NEXT:    mov z13.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p5.h, p0/z, z8.h, z8.h
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z26.h, z26.h
-; CHECK-NEXT:    str z24, [x8, #10, mul vl]
-; CHECK-NEXT:    mov z3.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p3.h, p0/z, z4.h, z25.h
-; CHECK-NEXT:    str z13, [x8, #9, mul vl]
+; CHECK-NEXT:    sel z3.d, p1, z28.d, z21.d
+; CHECK-NEXT:    mov z16.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p12.h, p0/z, z27.h, z29.h
+; CHECK-NEXT:    sel z11.d, p2, z28.d, z20.d
+; CHECK-NEXT:    str z16, [x8, #13, mul vl]
+; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p6.h, p0/z, z15.h, z15.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z4.h, z29.h
+; CHECK-NEXT:    mov z11.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z9.d, p12/m, z28.d
+; CHECK-NEXT:    str z3, [x8, #11, mul vl]
+; CHECK-NEXT:    fcmuo p5.h, p0/z, z19.h, z19.h
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z5.h, z29.h
+; CHECK-NEXT:    str z11, [x8, #10, mul vl]
+; CHECK-NEXT:    mov z30.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    sel z3.d, p1, z28.d, z7.d
+; CHECK-NEXT:    fcmgt p4.h, p0/z, z6.h, z29.h
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z27.h, z27.h
+; CHECK-NEXT:    str z30, [x8, #12, mul vl]
+; CHECK-NEXT:    mov z26.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    sel z7.d, p2, z28.d, z24.d
+; CHECK-NEXT:    fcmgt p6.h, p0/z, z31.h, z29.h
+; CHECK-NEXT:    fcmgt p1.h, p0/z, z1.h, z29.h
+; CHECK-NEXT:    str z26, [x8, #9, mul vl]
+; CHECK-NEXT:    sel z24.d, p4, z28.d, z25.d
+; CHECK-NEXT:    mov z9.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p5.h, p0/z, z31.h, z31.h
 ; CHECK-NEXT:    fcmuo p2.h, p0/z, z6.h, z6.h
-; CHECK-NEXT:    mov z31.d, p10/m, z27.d
-; CHECK-NEXT:    str z0, [x8, #8, mul vl]
-; CHECK-NEXT:    mov z2.d, p1/m, z27.d
-; CHECK-NEXT:    fcmuo p1.h, p0/z, z7.h, z7.h
-; CHECK-NEXT:    str z3, [x8, #7, mul vl]
-; CHECK-NEXT:    mov z17.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p5.h, p0/z, z1.h, z1.h
-; CHECK-NEXT:    ldr z0, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.h, p0/z, z5.h, z5.h
-; CHECK-NEXT:    fcmuo p0.h, p0/z, z4.h, z4.h
-; CHECK-NEXT:    str z17, [x8, #6, mul vl]
-; CHECK-NEXT:    mov z31.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z30.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    mov z0.d, p3/m, z27.d
-; CHECK-NEXT:    mov z29.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    str z2, [x8, #5, mul vl]
-; CHECK-NEXT:    str z31, [x8, #4, mul vl]
-; CHECK-NEXT:    mov z28.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    str z30, [x8, #3, mul vl]
+; CHECK-NEXT:    mov z2.d, p6/m, z28.d
+; CHECK-NEXT:    str z9, [x8, #5, mul vl]
+; CHECK-NEXT:    mov z0.d, p1/m, z28.d
+; CHECK-NEXT:    fcmuo p3.h, p0/z, z5.h, z5.h
+; CHECK-NEXT:    fcmuo p4.h, p0/z, z4.h, z4.h
+; CHECK-NEXT:    mov z2.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z24.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z1.h, z1.h
+; CHECK-NEXT:    mov z7.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    str z2, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    str z24, [x8, #3, mul vl]
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    str z29, [x8, #2, mul vl]
-; CHECK-NEXT:    str z28, [x8, #1, mul vl]
+; CHECK-NEXT:    str z7, [x8, #2, mul vl]
+; CHECK-NEXT:    str z3, [x8, #1, mul vl]
 ; CHECK-NEXT:    str z0, [x8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr p11, [sp] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #18
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv32f16(<vscale x 32 x half> %x)
@@ -533,17 +524,17 @@ define <vscale x 1 x iXLen> @lrint_v1f32(<vscale x 1 x float> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
 ; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
-; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z2.s
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.s
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f32(<vscale x 1 x float> %x)
@@ -556,17 +547,17 @@ define <vscale x 2 x iXLen> @lrint_v2f32(<vscale x 2 x float> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
 ; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
-; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z2.s
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.s, w8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.s
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f32(<vscale x 2 x float> %x)
@@ -585,20 +576,20 @@ define <vscale x 4 x iXLen> @lrint_v4f32(<vscale x 4 x float> %x) {
 ; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z4.d, #0x8000000000000000
-; CHECK-NEXT:    mov z5.s, w8
+; CHECK-NEXT:    mov z5.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
 ; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, z2.s
 ; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z2.s
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    fcmuo p3.s, p0/z, z1.s, z1.s
-; CHECK-NEXT:    fcvtzs z3.d, p1/m, z1.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z5.s
-; CHECK-NEXT:    fcvtzs z4.d, p2/m, z0.s
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z5.s
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z1.s
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z2.s
+; CHECK-NEXT:    fcvtzs z3.d, p2/m, z0.s
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z2.s
 ; CHECK-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z3.d
-; CHECK-NEXT:    sel z1.d, p2, z2.d, z4.d
+; CHECK-NEXT:    sel z0.d, p1, z5.d, z4.d
+; CHECK-NEXT:    sel z1.d, p2, z5.d, z3.d
 ; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
@@ -618,49 +609,50 @@ define <vscale x 8 x iXLen> @lrint_v8f32(<vscale x 8 x float> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 16 + 8 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    uunpklo z2.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
-; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
 ; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpkhi z0.d, z0.s
 ; CHECK-NEXT:    uunpklo z3.d, z1.s
 ; CHECK-NEXT:    uunpkhi z1.d, z1.s
 ; CHECK-NEXT:    mov z4.s, w8
 ; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
 ; CHECK-NEXT:    mov z5.d, #0x8000000000000000
-; CHECK-NEXT:    mov z6.d, #0x8000000000000000
-; CHECK-NEXT:    mov z25.s, w8
 ; CHECK-NEXT:    mov z7.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z2.s, p0/m, z2.s
-; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
 ; CHECK-NEXT:    mov z24.d, #0x8000000000000000
+; CHECK-NEXT:    mov z25.s, w8
+; CHECK-NEXT:    frintx z2.s, p0/m, z2.s
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
+; CHECK-NEXT:    movprfx z6, z0
+; CHECK-NEXT:    frintx z6.s, p0/m, z0.s
 ; CHECK-NEXT:    frintx z3.s, p0/m, z3.s
 ; CHECK-NEXT:    frintx z1.s, p0/m, z1.s
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z2.s, z4.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z4.s
-; CHECK-NEXT:    fcmuo p6.s, p0/z, z0.s, z0.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z6.s, z4.s
 ; CHECK-NEXT:    fcmge p3.s, p0/z, z3.s, z4.s
-; CHECK-NEXT:    fcmge p4.s, p0/z, z1.s, z4.s
-; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z1.s, z25.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z2.s, z25.s
 ; CHECK-NEXT:    fcvtzs z5.d, p1/m, z2.s
-; CHECK-NEXT:    fcvtzs z6.d, p2/m, z0.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z2.s, z25.s
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z0.s, z25.s
-; CHECK-NEXT:    fcvtzs z7.d, p3/m, z3.s
-; CHECK-NEXT:    fcmgt p3.s, p0/z, z3.s, z25.s
-; CHECK-NEXT:    fcvtzs z24.d, p4/m, z1.s
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z2.s, z2.s
-; CHECK-NEXT:    sel z0.d, p1, z4.d, z5.d
-; CHECK-NEXT:    fcmuo p1.s, p0/z, z3.s, z3.s
-; CHECK-NEXT:    fcmuo p0.s, p0/z, z1.s, z1.s
-; CHECK-NEXT:    sel z1.d, p2, z4.d, z6.d
-; CHECK-NEXT:    sel z2.d, p3, z4.d, z7.d
-; CHECK-NEXT:    sel z3.d, p5, z4.d, z24.d
+; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, z4.s
+; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcvtzs z7.d, p2/m, z6.s
+; CHECK-NEXT:    fcvtzs z24.d, p3/m, z3.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z6.s, z25.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z3.s, z25.s
+; CHECK-NEXT:    fcvtzs z26.d, p1/m, z1.s
+; CHECK-NEXT:    sel z0.d, p4, z4.d, z5.d
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z25.s
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z6.s, z6.s
+; CHECK-NEXT:    fcmuo p6.s, p0/z, z3.s, z3.s
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z2.s, z2.s
+; CHECK-NEXT:    sel z2.d, p5, z4.d, z24.d
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    sel z1.d, p3, z4.d, z7.d
+; CHECK-NEXT:    sel z3.d, p1, z4.d, z26.d
+; CHECK-NEXT:    mov z2.d, p6/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -674,7 +666,7 @@ define <vscale x 16 x iXLen> @lrint_v16f32(<vscale x 16 x float> %x) {
 ; CHECK-LABEL: lrint_v16f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-4
+; CHECK-NEXT:    addvl sp, sp, #-3
 ; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
@@ -682,106 +674,104 @@ define <vscale x 16 x iXLen> @lrint_v16f32(<vscale x 16 x float> %x) {
 ; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
 ; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x20, 0x1e, 0x22 // sp + 16 + 32 * VG
+; CHECK-NEXT:    str z9, [sp, #1, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x40, 0x1c // $d10 @ cfa - 24 * VG - 16
 ; CHECK-NEXT:    uunpklo z4.d, z0.s
-; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    uunpkhi z5.d, z0.s
 ; CHECK-NEXT:    mov w8, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpkhi z7.d, z1.s
+; CHECK-NEXT:    uunpklo z24.d, z2.s
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uunpklo z6.d, z2.s
+; CHECK-NEXT:    uunpklo z6.d, z1.s
 ; CHECK-NEXT:    uunpkhi z2.d, z2.s
-; CHECK-NEXT:    uunpklo z5.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
-; CHECK-NEXT:    uunpklo z7.d, z3.s
-; CHECK-NEXT:    mov z24.s, w8
-; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
-; CHECK-NEXT:    mov z26.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z4.s, p0/m, z4.s
-; CHECK-NEXT:    frintx z0.s, p0/m, z0.s
-; CHECK-NEXT:    mov z30.s, w8
-; CHECK-NEXT:    movprfx z27, z2
-; CHECK-NEXT:    frintx z27.s, p0/m, z2.s
-; CHECK-NEXT:    uunpkhi z2.d, z3.s
-; CHECK-NEXT:    frintx z6.s, p0/m, z6.s
-; CHECK-NEXT:    movprfx z25, z1
-; CHECK-NEXT:    frintx z25.s, p0/m, z1.s
-; CHECK-NEXT:    frintx z5.s, p0/m, z5.s
-; CHECK-NEXT:    frintx z7.s, p0/m, z7.s
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    mov z3.d, #0x8000000000000000
+; CHECK-NEXT:    mov z27.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.s, p0/z, z4.s, z24.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z24.s
-; CHECK-NEXT:    mov z29.d, #0x8000000000000000
-; CHECK-NEXT:    movprfx z9, z2
-; CHECK-NEXT:    frintx z9.s, p0/m, z2.s
-; CHECK-NEXT:    fcmge p5.s, p0/z, z6.s, z24.s
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p3.s, p0/z, z5.s, z24.s
-; CHECK-NEXT:    fcmge p4.s, p0/z, z25.s, z24.s
-; CHECK-NEXT:    fcmge p7.s, p0/z, z7.s, z24.s
-; CHECK-NEXT:    fcmge p6.s, p0/z, z27.s, z24.s
+; CHECK-NEXT:    movprfx z25, z4
+; CHECK-NEXT:    frintx z25.s, p0/m, z4.s
+; CHECK-NEXT:    frintx z5.s, p0/m, z5.s
+; CHECK-NEXT:    mov z4.s, w8
+; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
+; CHECK-NEXT:    frintx z7.s, p0/m, z7.s
+; CHECK-NEXT:    frintx z24.s, p0/m, z24.s
+; CHECK-NEXT:    movprfx z30, z2
+; CHECK-NEXT:    frintx z30.s, p0/m, z2.s
+; CHECK-NEXT:    frintx z6.s, p0/m, z6.s
+; CHECK-NEXT:    uunpklo z2.d, z3.s
+; CHECK-NEXT:    mov z29.s, w8
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    mov z10.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z4.s
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z4.s, z30.s
-; CHECK-NEXT:    fcvtzs z26.d, p2/m, z0.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z9.s, z24.s
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z29.d, p5/m, z6.s
-; CHECK-NEXT:    fcvtzs z3.d, p3/m, z5.s
-; CHECK-NEXT:    fcvtzs z28.d, p4/m, z25.s
-; CHECK-NEXT:    fcvtzs z8.d, p7/m, z7.s
-; CHECK-NEXT:    fcmgt p4.s, p0/z, z0.s, z30.s
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z5.s, z30.s
-; CHECK-NEXT:    fcmgt p7.s, p0/z, z25.s, z30.s
-; CHECK-NEXT:    fcmgt p8.s, p0/z, z6.s, z30.s
-; CHECK-NEXT:    fcvtzs z31.d, p6/m, z27.s
-; CHECK-NEXT:    fcmuo p6.s, p0/z, z0.s, z0.s
-; CHECK-NEXT:    sel z0.d, p1, z10.d, z1.d
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z27.s, z30.s
-; CHECK-NEXT:    fcmgt p10.s, p0/z, z7.s, z30.s
-; CHECK-NEXT:    fcvtzs z24.d, p2/m, z9.s
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z9.s, z30.s
-; CHECK-NEXT:    fcmuo p3.s, p0/z, z4.s, z4.s
-; CHECK-NEXT:    fcmuo p9.s, p0/z, z5.s, z5.s
-; CHECK-NEXT:    sel z1.d, p4, z10.d, z26.d
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z25.s, z25.s
-; CHECK-NEXT:    sel z2.d, p5, z10.d, z3.d
-; CHECK-NEXT:    sel z3.d, p7, z10.d, z28.d
-; CHECK-NEXT:    sel z4.d, p8, z10.d, z29.d
-; CHECK-NEXT:    fcmuo p5.s, p0/z, z6.s, z6.s
-; CHECK-NEXT:    fcmuo p7.s, p0/z, z27.s, z27.s
-; CHECK-NEXT:    fcmuo p8.s, p0/z, z7.s, z7.s
-; CHECK-NEXT:    sel z5.d, p1, z10.d, z31.d
-; CHECK-NEXT:    sel z6.d, p10, z10.d, z8.d
-; CHECK-NEXT:    ldr z8, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    mov z9.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcmge p5.s, p0/z, z25.s, z4.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z5.s, z4.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z7.s, z4.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z24.s, z4.s
+; CHECK-NEXT:    fcvtzs z0.d, p5/m, z25.s
+; CHECK-NEXT:    fcvtzs z1.d, p1/m, z5.s
+; CHECK-NEXT:    fcvtzs z27.d, p3/m, z7.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z6.s, z4.s
+; CHECK-NEXT:    fcvtzs z28.d, p4/m, z24.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z25.s, z29.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z30.s, z4.s
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    movprfx z25, z2
+; CHECK-NEXT:    frintx z25.s, p0/m, z2.s
+; CHECK-NEXT:    uunpkhi z2.d, z3.s
+; CHECK-NEXT:    fcvtzs z26.d, p2/m, z6.s
+; CHECK-NEXT:    mov z0.d, p3/m, z9.d
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z5.s, z29.s
+; CHECK-NEXT:    fcvtzs z31.d, p4/m, z30.s
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z5.s, z5.s
+; CHECK-NEXT:    movprfx z5, z2
+; CHECK-NEXT:    frintx z5.s, p0/m, z2.s
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p4.s, p0/z, z25.s, z4.s
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z6.s, z29.s
+; CHECK-NEXT:    mov z1.d, p5/m, z9.d
+; CHECK-NEXT:    fcmgt p8.s, p0/z, z7.s, z29.s
+; CHECK-NEXT:    fcmuo p10.s, p0/z, z7.s, z7.s
+; CHECK-NEXT:    mov z7.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z8.d, p4/m, z25.s
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    sel z2.d, p6, z9.d, z26.d
+; CHECK-NEXT:    sel z3.d, p8, z9.d, z27.d
+; CHECK-NEXT:    fcmge p4.s, p0/z, z5.s, z4.s
+; CHECK-NEXT:    fcmgt p9.s, p0/z, z24.s, z29.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z30.s, z29.s
+; CHECK-NEXT:    mov z3.d, p10/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    fcmuo p0.s, p0/z, z9.s, z9.s
-; CHECK-NEXT:    ldr z9, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    sel z7.d, p2, z10.d, z24.d
-; CHECK-NEXT:    ldr z10, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z25.s, z29.s
+; CHECK-NEXT:    fcvtzs z7.d, p4/m, z5.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z5.s, z29.s
+; CHECK-NEXT:    sel z4.d, p9, z9.d, z28.d
+; CHECK-NEXT:    fcmuo p7.s, p0/z, z6.s, z6.s
+; CHECK-NEXT:    sel z6.d, p6, z9.d, z8.d
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
-; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    mov z5.d, p7/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p8.s, p0/z, z30.s, z30.s
+; CHECK-NEXT:    fcmuo p9.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    mov z7.d, p4/m, z9.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z2.d, p7/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z6.d, p8/m, #0 // =0x0
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z24.s, z24.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z5.s, z5.s
+; CHECK-NEXT:    sel z5.d, p5, z9.d, z31.d
+; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z6.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 16 x iXLen> @llvm.lrint.nxv16iXLen.nxv16f32(<vscale x 16 x float> %x)
@@ -793,33 +783,34 @@ define <vscale x 32 x iXLen> @lrint_v32f32(<vscale x 32 x float> %x) {
 ; CHECK-LABEL: lrint_v32f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-17
-; CHECK-NEXT:    str p11, [sp] // 2-byte Folded Spill
-; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 16 + 144 * VG
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x01, 0x1e, 0x22 // sp + 16 + 152 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
@@ -830,210 +821,212 @@ define <vscale x 32 x iXLen> @lrint_v32f32(<vscale x 32 x float> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x40, 0x1c // $d14 @ cfa - 56 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16
 ; CHECK-NEXT:    uunpklo z24.d, z0.s
-; CHECK-NEXT:    uunpkhi z25.d, z0.s
-; CHECK-NEXT:    mov w9, #-553648128 // =0xdf000000
 ; CHECK-NEXT:    uunpklo z26.d, z1.s
-; CHECK-NEXT:    uunpkhi z1.d, z1.s
+; CHECK-NEXT:    mov w9, #-553648128 // =0xdf000000
+; CHECK-NEXT:    uunpklo z28.d, z2.s
+; CHECK-NEXT:    uunpkhi z30.d, z2.s
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    uunpklo z27.d, z2.s
-; CHECK-NEXT:    uunpkhi z9.d, z2.s
-; CHECK-NEXT:    uunpklo z11.d, z3.s
-; CHECK-NEXT:    uunpkhi z12.d, z3.s
-; CHECK-NEXT:    mov z10.s, w9
-; CHECK-NEXT:    mov w9, #1593835519 // =0x5effffff
+; CHECK-NEXT:    uunpkhi z25.d, z0.s
+; CHECK-NEXT:    uunpkhi z13.d, z3.s
+; CHECK-NEXT:    uunpklo z14.d, z4.s
+; CHECK-NEXT:    uunpkhi z27.d, z1.s
+; CHECK-NEXT:    uunpklo z9.d, z3.s
+; CHECK-NEXT:    mov z29.s, w9
 ; CHECK-NEXT:    movprfx z0, z24
 ; CHECK-NEXT:    frintx z0.s, p0/m, z24.s
-; CHECK-NEXT:    movprfx z24, z25
-; CHECK-NEXT:    frintx z24.s, p0/m, z25.s
-; CHECK-NEXT:    uunpklo z13.d, z4.s
-; CHECK-NEXT:    movprfx z25, z26
-; CHECK-NEXT:    frintx z25.s, p0/m, z26.s
-; CHECK-NEXT:    movprfx z26, z1
-; CHECK-NEXT:    frintx z26.s, p0/m, z1.s
-; CHECK-NEXT:    uunpkhi z14.d, z4.s
-; CHECK-NEXT:    movprfx z2, z27
-; CHECK-NEXT:    frintx z2.s, p0/m, z27.s
-; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    movprfx z27, z9
-; CHECK-NEXT:    frintx z27.s, p0/m, z9.s
-; CHECK-NEXT:    movprfx z9, z11
-; CHECK-NEXT:    frintx z9.s, p0/m, z11.s
-; CHECK-NEXT:    movprfx z11, z12
-; CHECK-NEXT:    frintx z11.s, p0/m, z12.s
-; CHECK-NEXT:    uunpklo z15.d, z7.s
-; CHECK-NEXT:    uunpkhi z7.d, z7.s
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    mov z29.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p3.s, p0/z, z26.s, z10.s
-; CHECK-NEXT:    mov z30.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.s, p0/z, z0.s, z10.s
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.s, p0/z, z24.s, z10.s
-; CHECK-NEXT:    movprfx z12, z13
-; CHECK-NEXT:    frintx z12.s, p0/m, z13.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z25.s, z10.s
-; CHECK-NEXT:    fcmge p4.s, p0/z, z2.s, z10.s
+; CHECK-NEXT:    movprfx z24, z26
+; CHECK-NEXT:    frintx z24.s, p0/m, z26.s
+; CHECK-NEXT:    mov w9, #1593835519 // =0x5effffff
+; CHECK-NEXT:    movprfx z10, z28
+; CHECK-NEXT:    frintx z10.s, p0/m, z28.s
+; CHECK-NEXT:    frintx z30.s, p0/m, z30.s
+; CHECK-NEXT:    uunpklo z17.d, z5.s
+; CHECK-NEXT:    movprfx z1, z25
+; CHECK-NEXT:    frintx z1.s, p0/m, z25.s
+; CHECK-NEXT:    movprfx z15, z13
+; CHECK-NEXT:    frintx z15.s, p0/m, z13.s
 ; CHECK-NEXT:    movprfx z13, z14
 ; CHECK-NEXT:    frintx z13.s, p0/m, z14.s
-; CHECK-NEXT:    uunpklo z17.d, z5.s
+; CHECK-NEXT:    uunpkhi z14.d, z4.s
 ; CHECK-NEXT:    uunpkhi z18.d, z5.s
-; CHECK-NEXT:    movprfx z21, z7
-; CHECK-NEXT:    frintx z21.s, p0/m, z7.s
-; CHECK-NEXT:    uunpklo z19.d, z6.s
-; CHECK-NEXT:    uunpkhi z20.d, z6.s
+; CHECK-NEXT:    uunpkhi z19.d, z6.s
+; CHECK-NEXT:    movprfx z25, z27
+; CHECK-NEXT:    frintx z25.s, p0/m, z27.s
+; CHECK-NEXT:    mov z27.d, #0x8000000000000000
+; CHECK-NEXT:    mov z11.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p3.s, p0/z, z24.s, z29.s
 ; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    fcvtzs z31.d, p3/m, z26.s
-; CHECK-NEXT:    fcmge p3.s, p0/z, z11.s, z10.s
-; CHECK-NEXT:    mov z5.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z15.s, p0/m, z15.s
-; CHECK-NEXT:    fcvtzs z1.d, p5/m, z0.s
-; CHECK-NEXT:    fcvtzs z29.d, p1/m, z24.s
-; CHECK-NEXT:    fcvtzs z30.d, p2/m, z25.s
-; CHECK-NEXT:    fcvtzs z8.d, p4/m, z2.s
-; CHECK-NEXT:    fcmge p1.s, p0/z, z27.s, z10.s
+; CHECK-NEXT:    mov z12.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p5.s, p0/z, z10.s, z29.s
+; CHECK-NEXT:    frintx z9.s, p0/m, z9.s
+; CHECK-NEXT:    uunpklo z20.d, z7.s
+; CHECK-NEXT:    movprfx z5, z14
+; CHECK-NEXT:    frintx z5.s, p0/m, z14.s
+; CHECK-NEXT:    movprfx z14, z17
+; CHECK-NEXT:    frintx z14.s, p0/m, z17.s
+; CHECK-NEXT:    movprfx z17, z18
+; CHECK-NEXT:    frintx z17.s, p0/m, z18.s
+; CHECK-NEXT:    fcmge p6.s, p0/z, z30.s, z29.s
+; CHECK-NEXT:    movprfx z18, z19
+; CHECK-NEXT:    frintx z18.s, p0/m, z19.s
+; CHECK-NEXT:    uunpkhi z7.d, z7.s
+; CHECK-NEXT:    mov z31.s, w9
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
+; CHECK-NEXT:    uunpklo z6.d, z6.s
+; CHECK-NEXT:    mov z28.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z27.d, p3/m, z24.s
+; CHECK-NEXT:    fcvtzs z11.d, p5/m, z10.s
 ; CHECK-NEXT:    mov z4.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p2.s, p0/z, z9.s, z10.s
 ; CHECK-NEXT:    mov z16.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p4.s, p0/z, z12.s, z10.s
-; CHECK-NEXT:    mov z6.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.s, p0/z, z13.s, z10.s
-; CHECK-NEXT:    mov z14.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z17.s, p0/m, z17.s
-; CHECK-NEXT:    frintx z18.s, p0/m, z18.s
-; CHECK-NEXT:    frintx z19.s, p0/m, z19.s
+; CHECK-NEXT:    movprfx z22, z7
+; CHECK-NEXT:    frintx z22.s, p0/m, z7.s
+; CHECK-NEXT:    fcvtzs z12.d, p6/m, z30.s
 ; CHECK-NEXT:    frintx z20.s, p0/m, z20.s
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z5.d, p3/m, z11.s
-; CHECK-NEXT:    fcmge p3.s, p0/z, z21.s, z10.s
-; CHECK-NEXT:    mov z3.s, w9
-; CHECK-NEXT:    fcmge p6.s, p0/z, z15.s, z10.s
-; CHECK-NEXT:    mov z22.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z4.d, p1/m, z27.s
-; CHECK-NEXT:    fcvtzs z16.d, p2/m, z9.s
-; CHECK-NEXT:    fcvtzs z6.d, p4/m, z12.s
-; CHECK-NEXT:    fcvtzs z14.d, p5/m, z13.s
-; CHECK-NEXT:    fcmge p1.s, p0/z, z17.s, z10.s
-; CHECK-NEXT:    fcmge p2.s, p0/z, z18.s, z10.s
+; CHECK-NEXT:    mov z21.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z23.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p4.s, p0/z, z19.s, z10.s
-; CHECK-NEXT:    fcmge p5.s, p0/z, z20.s, z10.s
-; CHECK-NEXT:    mov z10.d, #0x8000000000000000
+; CHECK-NEXT:    mov z8.d, #0x8000000000000000
+; CHECK-NEXT:    frintx z6.s, p0/m, z6.s
+; CHECK-NEXT:    mov z3.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z19.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z29.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z1.s, z29.s
+; CHECK-NEXT:    fcmge p4.s, p0/z, z25.s, z29.s
+; CHECK-NEXT:    fcmgt p9.s, p0/z, z10.s, z31.s
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.s
 ; CHECK-NEXT:    mov z0.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z28.d, p3/m, z21.s
-; CHECK-NEXT:    mov z7.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p11.s, p0/z, z21.s, z3.s
-; CHECK-NEXT:    fcvtzs z22.d, p6/m, z15.s
-; CHECK-NEXT:    fcmgt p3.s, p0/z, z15.s, z3.s
-; CHECK-NEXT:    fcmuo p6.s, p0/z, z15.s, z15.s
-; CHECK-NEXT:    mov z15.d, #0x8000000000000000
-; CHECK-NEXT:    fcmgt p7.s, p0/z, z24.s, z3.s
+; CHECK-NEXT:    fcvtzs z26.d, p2/m, z1.s
+; CHECK-NEXT:    fcvtzs z28.d, p4/m, z25.s
+; CHECK-NEXT:    mov z11.d, p9/m, z3.d
+; CHECK-NEXT:    fcmuo p8.s, p0/z, z10.s, z10.s
+; CHECK-NEXT:    mov z10.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p5.s, p0/z, z9.s, z29.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z15.s, z29.s
+; CHECK-NEXT:    fcmge p6.s, p0/z, z13.s, z29.s
+; CHECK-NEXT:    mov z11.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z10.d, p5/m, z9.s
+; CHECK-NEXT:    fcvtzs z4.d, p3/m, z15.s
+; CHECK-NEXT:    fcvtzs z16.d, p6/m, z13.s
+; CHECK-NEXT:    fcmge p1.s, p0/z, z17.s, z29.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z18.s, z29.s
+; CHECK-NEXT:    fcmgt p12.s, p0/z, z30.s, z31.s
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z15.s, z31.s
+; CHECK-NEXT:    fcmge p3.s, p0/z, z20.s, z29.s
+; CHECK-NEXT:    fcvtzs z21.d, p1/m, z17.s
 ; CHECK-NEXT:    fcvtzs z23.d, p2/m, z18.s
-; CHECK-NEXT:    fcvtzs z10.d, p5/m, z20.s
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z9.s, z3.s
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z20.s, z3.s
-; CHECK-NEXT:    fcvtzs z0.d, p4/m, z19.s
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z21.s, z21.s
-; CHECK-NEXT:    mov z28.d, p11/m, z7.d
-; CHECK-NEXT:    sel z21.d, p3, z7.d, z22.d
-; CHECK-NEXT:    fcmgt p3.s, p0/z, z19.s, z3.s
-; CHECK-NEXT:    fcvtzs z15.d, p1/m, z17.s
-; CHECK-NEXT:    fcmuo p1.s, p0/z, z20.s, z20.s
-; CHECK-NEXT:    mov z29.d, p7/m, z7.d
-; CHECK-NEXT:    fcmgt p7.s, p0/z, z18.s, z3.s
-; CHECK-NEXT:    mov z16.d, p2/m, z7.d
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z17.s, z3.s
-; CHECK-NEXT:    mov z10.d, p5/m, z7.d
-; CHECK-NEXT:    mov z28.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z19.s, z19.s
-; CHECK-NEXT:    mov z0.d, p3/m, z7.d
-; CHECK-NEXT:    fcmuo p3.s, p0/z, z18.s, z18.s
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z13.s, z3.s
-; CHECK-NEXT:    mov z21.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p10.s, p0/z, z2.s, z3.s
-; CHECK-NEXT:    fcmgt p8.s, p0/z, z25.s, z3.s
-; CHECK-NEXT:    str z28, [x8, #15, mul vl]
-; CHECK-NEXT:    mov z10.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p1.s, p0/z, z17.s, z17.s
-; CHECK-NEXT:    sel z19.d, p7, z7.d, z23.d
-; CHECK-NEXT:    sel z28.d, p2, z7.d, z15.d
-; CHECK-NEXT:    fcmgt p2.s, p0/z, z12.s, z3.s
-; CHECK-NEXT:    str z21, [x8, #14, mul vl]
-; CHECK-NEXT:    mov z0.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    mov z14.d, p5/m, z7.d
-; CHECK-NEXT:    str z10, [x8, #13, mul vl]
-; CHECK-NEXT:    fcmgt p5.s, p0/z, z11.s, z3.s
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z13.s, z13.s
-; CHECK-NEXT:    mov z19.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z28.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p1.s, p0/z, z27.s, z3.s
+; CHECK-NEXT:    fcmgt p11.s, p0/z, z13.s, z31.s
+; CHECK-NEXT:    sel z7.d, p12, z3.d, z12.d
+; CHECK-NEXT:    mov z4.d, p5/m, z3.d
+; CHECK-NEXT:    fcmge p4.s, p0/z, z22.s, z29.s
+; CHECK-NEXT:    fcvtzs z0.d, p3/m, z20.s
+; CHECK-NEXT:    fcmge p6.s, p0/z, z5.s, z29.s
+; CHECK-NEXT:    sel z12.d, p11, z3.d, z16.d
+; CHECK-NEXT:    fcmge p7.s, p0/z, z14.s, z29.s
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z13.s, z13.s
+; CHECK-NEXT:    fcvtzs z8.d, p4/m, z22.s
+; CHECK-NEXT:    fcmge p2.s, p0/z, z6.s, z29.s
+; CHECK-NEXT:    mov z29.d, #0x8000000000000000
+; CHECK-NEXT:    fcmuo p10.s, p0/z, z15.s, z15.s
+; CHECK-NEXT:    mov z15.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z19.d, p7/m, z14.s
+; CHECK-NEXT:    mov z12.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z29.d, p2/m, z6.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z22.s, z31.s
+; CHECK-NEXT:    fcvtzs z15.d, p6/m, z5.s
+; CHECK-NEXT:    mov z4.d, p10/m, #0 // =0x0
+; CHECK-NEXT:    str z12, [x8, #8, mul vl]
+; CHECK-NEXT:    fcmgt p5.s, p0/z, z20.s, z31.s
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z18.s, z31.s
+; CHECK-NEXT:    str z4, [x8, #7, mul vl]
+; CHECK-NEXT:    fcmuo p1.s, p0/z, z22.s, z22.s
+; CHECK-NEXT:    mov z8.d, p4/m, z3.d
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z20.s, z20.s
+; CHECK-NEXT:    mov z0.d, p5/m, z3.d
+; CHECK-NEXT:    fcmuo p6.s, p0/z, z18.s, z18.s
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z5.s, z31.s
+; CHECK-NEXT:    mov z8.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p5.s, p0/z, z5.s, z5.s
+; CHECK-NEXT:    sel z5.d, p3, z3.d, z23.d
+; CHECK-NEXT:    str z8, [x8, #15, mul vl]
+; CHECK-NEXT:    fcmgt p3.s, p0/z, z6.s, z31.s
+; CHECK-NEXT:    str z0, [x8, #14, mul vl]
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z17.s, z31.s
+; CHECK-NEXT:    mov z5.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z14.s, z31.s
+; CHECK-NEXT:    fcmuo p6.s, p0/z, z6.s, z6.s
+; CHECK-NEXT:    sel z6.d, p4, z3.d, z15.d
+; CHECK-NEXT:    str z5, [x8, #13, mul vl]
+; CHECK-NEXT:    sel z0.d, p3, z3.d, z29.d
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z17.s, z17.s
+; CHECK-NEXT:    sel z5.d, p1, z3.d, z21.d
+; CHECK-NEXT:    sel z29.d, p2, z3.d, z19.d
+; CHECK-NEXT:    mov z6.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z14.s, z14.s
+; CHECK-NEXT:    mov z5.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    str z6, [x8, #9, mul vl]
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z1.s, z31.s
 ; CHECK-NEXT:    str z0, [x8, #12, mul vl]
-; CHECK-NEXT:    fcmuo p3.s, p0/z, z12.s, z12.s
-; CHECK-NEXT:    sel z0.d, p2, z7.d, z6.d
-; CHECK-NEXT:    fcmuo p2.s, p0/z, z11.s, z11.s
-; CHECK-NEXT:    fcmgt p9.s, p0/z, z26.s, z3.s
-; CHECK-NEXT:    mov z30.d, p8/m, z7.d
-; CHECK-NEXT:    str z19, [x8, #11, mul vl]
-; CHECK-NEXT:    mov z5.d, p5/m, z7.d
+; CHECK-NEXT:    fcmgt p2.s, p0/z, z24.s, z31.s
+; CHECK-NEXT:    mov z29.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    str z5, [x8, #11, mul vl]
+; CHECK-NEXT:    fcmgt p4.s, p0/z, z25.s, z31.s
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z30.s, z30.s
+; CHECK-NEXT:    sel z5.d, p1, z3.d, z26.d
+; CHECK-NEXT:    str z29, [x8, #10, mul vl]
+; CHECK-NEXT:    sel z26.d, p2, z3.d, z27.d
+; CHECK-NEXT:    ldr z4, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    str z11, [x8, #4, mul vl]
+; CHECK-NEXT:    fcmgt p6.s, p0/z, z9.s, z31.s
+; CHECK-NEXT:    sel z6.d, p4, z3.d, z28.d
 ; CHECK-NEXT:    fcmuo p5.s, p0/z, z9.s, z9.s
-; CHECK-NEXT:    str z28, [x8, #10, mul vl]
-; CHECK-NEXT:    mov z4.d, p1/m, z7.d
-; CHECK-NEXT:    fcmuo p1.s, p0/z, z2.s, z2.s
-; CHECK-NEXT:    ldr z2, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z14.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z27.s, z27.s
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z5.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p2.s, p0/z, z26.s, z26.s
-; CHECK-NEXT:    mov z16.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p5.s, p0/z, z25.s, z25.s
-; CHECK-NEXT:    mov z31.d, p9/m, z7.d
-; CHECK-NEXT:    str z14, [x8, #9, mul vl]
-; CHECK-NEXT:    fcmgt p3.s, p0/z, z2.s, z3.s
-; CHECK-NEXT:    mov z8.d, p10/m, z7.d
-; CHECK-NEXT:    str z0, [x8, #8, mul vl]
-; CHECK-NEXT:    mov z4.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.s, p0/z, z24.s, z24.s
-; CHECK-NEXT:    str z5, [x8, #7, mul vl]
-; CHECK-NEXT:    fcmuo p0.s, p0/z, z2.s, z2.s
-; CHECK-NEXT:    mov z31.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    str z16, [x8, #6, mul vl]
-; CHECK-NEXT:    mov z8.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z30.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    str z4, [x8, #5, mul vl]
-; CHECK-NEXT:    sel z0.d, p3, z7.d, z1.d
-; CHECK-NEXT:    str z31, [x8, #3, mul vl]
-; CHECK-NEXT:    mov z29.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    str z8, [x8, #4, mul vl]
-; CHECK-NEXT:    str z30, [x8, #2, mul vl]
-; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    str z29, [x8, #1, mul vl]
-; CHECK-NEXT:    str z0, [x8]
+; CHECK-NEXT:    mov z7.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p1.s, p0/z, z4.s, z31.s
+; CHECK-NEXT:    fcmuo p2.s, p0/z, z25.s, z25.s
+; CHECK-NEXT:    fcmuo p3.s, p0/z, z24.s, z24.s
+; CHECK-NEXT:    sel z0.d, p6, z3.d, z10.d
+; CHECK-NEXT:    str z7, [x8, #5, mul vl]
+; CHECK-NEXT:    fcmuo p4.s, p0/z, z1.s, z1.s
+; CHECK-NEXT:    fcmuo p0.s, p0/z, z4.s, z4.s
+; CHECK-NEXT:    sel z1.d, p1, z3.d, z2.d
+; CHECK-NEXT:    mov z0.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z6.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z26.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    mov z5.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    str z0, [x8, #6, mul vl]
+; CHECK-NEXT:    str z6, [x8, #3, mul vl]
+; CHECK-NEXT:    str z26, [x8, #2, mul vl]
+; CHECK-NEXT:    str z5, [x8, #1, mul vl]
+; CHECK-NEXT:    str z1, [x8]
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr p11, [sp] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #18
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv32f32(<vscale x 32 x float> %x)
@@ -1046,17 +1039,17 @@ define <vscale x 1 x iXLen> @lrint_v1f64(<vscale x 1 x double> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.d, x8
 ; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
-; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z1.d
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z2.d
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 1 x iXLen> @llvm.lrint.nxv1iXLen.nxv1f64(<vscale x 1 x double> %x)
@@ -1069,17 +1062,17 @@ define <vscale x 2 x iXLen> @lrint_v2f64(<vscale x 2 x double> %x) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z1.d, x8
 ; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
-; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z1.d
-; CHECK-NEXT:    mov z1.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z1.d, p1/m, z0.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z2.d
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z1.d, x8
+; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    mov z1.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z1.d
+; CHECK-NEXT:    sel z0.d, p1, z1.d, z2.d
 ; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call <vscale x 2 x iXLen> @llvm.lrint.nxv2iXLen.nxv2f64(<vscale x 2 x double> %x)
@@ -1094,22 +1087,22 @@ define <vscale x 4 x iXLen> @lrint_v4f64(<vscale x 4 x double> %x) {
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z2.d, x8
-; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    mov z4.d, #0x8000000000000000
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
 ; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
-; CHECK-NEXT:    mov z5.d, x8
+; CHECK-NEXT:    mov z5.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z2.d
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z2.d
-; CHECK-NEXT:    mov z2.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    fcmuo p3.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    fcvtzs z3.d, p1/m, z0.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z5.d
-; CHECK-NEXT:    fcvtzs z4.d, p2/m, z1.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z1.d, z5.d
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z0.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z2.d
+; CHECK-NEXT:    fcvtzs z3.d, p2/m, z1.d
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z1.d, z2.d
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z1.d, z1.d
-; CHECK-NEXT:    sel z0.d, p1, z2.d, z3.d
-; CHECK-NEXT:    sel z1.d, p2, z2.d, z4.d
+; CHECK-NEXT:    sel z0.d, p1, z5.d, z4.d
+; CHECK-NEXT:    sel z1.d, p2, z5.d, z3.d
 ; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    ret
@@ -1130,44 +1123,44 @@ define <vscale x 8 x iXLen> @lrint_v8f64(<vscale x 8 x double> %x) {
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
-; CHECK-NEXT:    mov z5.d, #0x8000000000000000
-; CHECK-NEXT:    mov z4.d, x8
-; CHECK-NEXT:    mov z6.d, #0x8000000000000000
+; CHECK-NEXT:    mov z4.d, #0x8000000000000000
+; CHECK-NEXT:    mov z5.d, x8
 ; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    mov z6.d, #0x8000000000000000
 ; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
 ; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
 ; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
+; CHECK-NEXT:    mov z24.d, x8
 ; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
-; CHECK-NEXT:    mov z25.d, x8
 ; CHECK-NEXT:    mov z7.d, #0x8000000000000000
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z4.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z4.d
-; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z4.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z4.d
-; CHECK-NEXT:    mov z4.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z3.d, z25.d
-; CHECK-NEXT:    fcmuo p6.d, p0/z, z1.d, z1.d
-; CHECK-NEXT:    fcvtzs z5.d, p1/m, z0.d
+; CHECK-NEXT:    mov z25.d, #0x7fffffffffffffff
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z5.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z5.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z5.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z0.d, z24.d
+; CHECK-NEXT:    fcvtzs z4.d, p1/m, z0.d
+; CHECK-NEXT:    fcmge p1.d, p0/z, z3.d, z5.d
+; CHECK-NEXT:    mov z5.d, #0x8000000000000000
 ; CHECK-NEXT:    fcvtzs z6.d, p2/m, z1.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z25.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z1.d, z25.d
 ; CHECK-NEXT:    fcvtzs z7.d, p3/m, z2.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z2.d, z25.d
-; CHECK-NEXT:    fcvtzs z24.d, p4/m, z3.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    sel z0.d, p1, z4.d, z5.d
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z1.d, z24.d
+; CHECK-NEXT:    sel z0.d, p4, z25.d, z4.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z2.d, z24.d
+; CHECK-NEXT:    fcvtzs z5.d, p1/m, z3.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z24.d
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    sel z1.d, p3, z25.d, z6.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    sel z2.d, p4, z25.d, z7.d
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z3.d, z3.d
-; CHECK-NEXT:    sel z1.d, p2, z4.d, z6.d
-; CHECK-NEXT:    sel z2.d, p3, z4.d, z7.d
-; CHECK-NEXT:    sel z3.d, p5, z4.d, z24.d
+; CHECK-NEXT:    sel z3.d, p1, z25.d, z5.d
+; CHECK-NEXT:    mov z1.d, p5/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p6/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p1/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    addvl sp, sp, #1
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
@@ -1197,84 +1190,84 @@ define <vscale x 16 x iXLen> @lrint_v16f64(<vscale x 16 x double> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4332462841530417152 // =0xc3e0000000000000
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
+; CHECK-NEXT:    mov z28.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z26.d, #0x8000000000000000
-; CHECK-NEXT:    mov z24.d, x8
-; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    mov z27.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z24, z0
+; CHECK-NEXT:    frintx z24.d, p0/m, z0.d
+; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
+; CHECK-NEXT:    mov z0.d, x8
 ; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
-; CHECK-NEXT:    movprfx z25, z4
-; CHECK-NEXT:    frintx z25.d, p0/m, z4.d
+; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    frintx z2.d, p0/m, z2.d
-; CHECK-NEXT:    frintx z3.d, p0/m, z3.d
-; CHECK-NEXT:    frintx z5.d, p0/m, z5.d
+; CHECK-NEXT:    frintx z4.d, p0/m, z4.d
 ; CHECK-NEXT:    frintx z6.d, p0/m, z6.d
 ; CHECK-NEXT:    mov z30.d, x8
-; CHECK-NEXT:    mov z4.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z29.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z24.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z24.d
-; CHECK-NEXT:    fcmge p5.d, p0/z, z25.d, z24.d
-; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z24.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z24.d
-; CHECK-NEXT:    fcmge p7.d, p0/z, z5.d, z24.d
+; CHECK-NEXT:    frintx z5.d, p0/m, z5.d
+; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
 ; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p6.d, p0/z, z6.d, z24.d
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
-; CHECK-NEXT:    mov z9.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p8.d, p0/z, z25.d, z30.d
-; CHECK-NEXT:    fcmgt p10.d, p0/z, z6.d, z30.d
-; CHECK-NEXT:    fcvtzs z26.d, p1/m, z0.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z0.d, z30.d
-; CHECK-NEXT:    fcvtzs z4.d, p2/m, z1.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z7.d, z24.d
-; CHECK-NEXT:    mov z24.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z27.d, p3/m, z2.d
+; CHECK-NEXT:    mov z8.d, #0x7fffffffffffffff
+; CHECK-NEXT:    mov z9.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p1.d, p0/z, z24.d, z0.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z0.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z0.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z2.d, z0.d
+; CHECK-NEXT:    fcvtzs z25.d, p1/m, z24.d
 ; CHECK-NEXT:    fcvtzs z28.d, p4/m, z3.d
-; CHECK-NEXT:    fcvtzs z29.d, p5/m, z25.d
-; CHECK-NEXT:    fcvtzs z31.d, p7/m, z5.d
-; CHECK-NEXT:    fcmgt p4.d, p0/z, z1.d, z30.d
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z2.d, z30.d
-; CHECK-NEXT:    fcmgt p7.d, p0/z, z3.d, z30.d
-; CHECK-NEXT:    fcvtzs z8.d, p6/m, z6.d
-; CHECK-NEXT:    fcmuo p3.d, p0/z, z0.d, z0.d
-; CHECK-NEXT:    sel z0.d, p1, z9.d, z26.d
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z5.d, z30.d
-; CHECK-NEXT:    fcvtzs z24.d, p2/m, z7.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z7.d, z30.d
-; CHECK-NEXT:    fcmuo p6.d, p0/z, z1.d, z1.d
-; CHECK-NEXT:    fcmuo p9.d, p0/z, z2.d, z2.d
-; CHECK-NEXT:    sel z1.d, p4, z9.d, z4.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z3.d, z3.d
-; CHECK-NEXT:    sel z2.d, p5, z9.d, z27.d
-; CHECK-NEXT:    sel z3.d, p7, z9.d, z28.d
-; CHECK-NEXT:    sel z4.d, p8, z9.d, z29.d
-; CHECK-NEXT:    fcmuo p5.d, p0/z, z25.d, z25.d
-; CHECK-NEXT:    fcmuo p7.d, p0/z, z5.d, z5.d
-; CHECK-NEXT:    fcmuo p8.d, p0/z, z6.d, z6.d
-; CHECK-NEXT:    sel z5.d, p1, z9.d, z31.d
-; CHECK-NEXT:    sel z6.d, p10, z9.d, z8.d
-; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    fcvtzs z26.d, p2/m, z1.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z4.d, z0.d
+; CHECK-NEXT:    fcvtzs z27.d, p3/m, z2.d
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z24.d, z30.d
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z24.d, z24.d
+; CHECK-NEXT:    mov z24.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p7.d, p0/z, z6.d, z0.d
+; CHECK-NEXT:    fcvtzs z29.d, p5/m, z4.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z5.d, z0.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z1.d, z30.d
+; CHECK-NEXT:    fcvtzs z24.d, p7/m, z6.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z2.d, z30.d
+; CHECK-NEXT:    fcmge p7.d, p0/z, z7.d, z0.d
+; CHECK-NEXT:    fcvtzs z31.d, p3/m, z5.d
+; CHECK-NEXT:    sel z0.d, p4, z8.d, z25.d
+; CHECK-NEXT:    fcmgt p8.d, p0/z, z3.d, z30.d
+; CHECK-NEXT:    fcmgt p9.d, p0/z, z4.d, z30.d
+; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcvtzs z9.d, p7/m, z7.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z1.d, z1.d
+; CHECK-NEXT:    sel z1.d, p5, z8.d, z26.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z2.d, z2.d
+; CHECK-NEXT:    sel z2.d, p6, z8.d, z27.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z5.d, z30.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z6.d, z30.d
+; CHECK-NEXT:    mov z1.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p7.d, p0/z, z7.d, z30.d
+; CHECK-NEXT:    fcmuo p10.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    sel z3.d, p8, z8.d, z28.d
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z4.d, z4.d
+; CHECK-NEXT:    sel z4.d, p9, z8.d, z29.d
+; CHECK-NEXT:    fcmuo p8.d, p0/z, z5.d, z5.d
+; CHECK-NEXT:    sel z5.d, p5, z8.d, z31.d
+; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    fcmuo p9.d, p0/z, z6.d, z6.d
+; CHECK-NEXT:    sel z6.d, p6, z8.d, z24.d
+; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z3.d, p10/m, #0 // =0x0
 ; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z4.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z5.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z6.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
 ; CHECK-NEXT:    fcmuo p0.d, p0/z, z7.d, z7.d
-; CHECK-NEXT:    sel z7.d, p2, z9.d, z24.d
+; CHECK-NEXT:    sel z7.d, p7, z8.d, z9.d
 ; CHECK-NEXT:    ldr z9, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    mov z1.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z2.d, p9/m, #0 // =0x0
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z3.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    mov z4.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z5.d, p7/m, #0 // =0x0
-; CHECK-NEXT:    mov z6.d, p8/m, #0 // =0x0
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    ldr z8, [sp, #2, mul vl] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    mov z7.d, p0/m, #0 // =0x0
 ; CHECK-NEXT:    addvl sp, sp, #3
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
@@ -1287,33 +1280,34 @@ define <vscale x 32 x iXLen> @lrint_v32f64(<vscale x 32 x double> %x) {
 ; CHECK-LABEL: lrint_v32f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    addvl sp, sp, #-17
-; CHECK-NEXT:    str p11, [sp] // 2-byte Folded Spill
-; CHECK-NEXT:    str p10, [sp, #1, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p9, [sp, #2, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p8, [sp, #3, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p7, [sp, #4, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p6, [sp, #5, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p5, [sp, #6, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p4, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z23, [sp, #1, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z22, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z21, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z20, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z19, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z18, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z17, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z16, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z15, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #11, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #12, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #13, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #14, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #15, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-18
+; CHECK-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
 ; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 16 + 144 * VG
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x98, 0x01, 0x1e, 0x22 // sp + 16 + 152 * VG
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x40, 0x1c // $d9 @ cfa - 16 * VG - 16
@@ -1325,202 +1319,207 @@ define <vscale x 32 x iXLen> @lrint_v32f64(<vscale x 32 x double> %x) {
 ; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x40, 0x1c // $d15 @ cfa - 64 * VG - 16
 ; CHECK-NEXT:    ldr z0, [x0]
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    ldr z2, [x0, #2, mul vl]
-; CHECK-NEXT:    ldr z1, [x0, #1, mul vl]
-; CHECK-NEXT:    ldr z6, [x0, #4, mul vl]
 ; CHECK-NEXT:    mov x9, #-4332462841530417152 // =0xc3e0000000000000
-; CHECK-NEXT:    ldr z5, [x0, #3, mul vl]
 ; CHECK-NEXT:    mov z25.d, x9
-; CHECK-NEXT:    mov z28.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z0.d, p0/m, z0.d
-; CHECK-NEXT:    movprfx z4, z2
-; CHECK-NEXT:    frintx z4.d, p0/m, z2.d
-; CHECK-NEXT:    mov z27.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z1.d, p0/m, z1.d
-; CHECK-NEXT:    frintx z6.d, p0/m, z6.d
+; CHECK-NEXT:    ldr z7, [x0, #3, mul vl]
+; CHECK-NEXT:    ldr z27, [x0, #4, mul vl]
+; CHECK-NEXT:    ldr z4, [x0, #2, mul vl]
+; CHECK-NEXT:    ldr z2, [x0, #1, mul vl]
+; CHECK-NEXT:    ldr z9, [x0, #15, mul vl]
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    frintx z5.d, p0/m, z0.d
+; CHECK-NEXT:    mov z0.d, #0x8000000000000000
+; CHECK-NEXT:    ldr z10, [x0, #14, mul vl]
+; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
+; CHECK-NEXT:    movprfx z14, z27
+; CHECK-NEXT:    frintx z14.d, p0/m, z27.d
+; CHECK-NEXT:    ldr z11, [x0, #13, mul vl]
+; CHECK-NEXT:    frintx z4.d, p0/m, z4.d
+; CHECK-NEXT:    ldr z8, [x0, #12, mul vl]
+; CHECK-NEXT:    ldr z27, [x0, #5, mul vl]
+; CHECK-NEXT:    ldr z18, [x0, #11, mul vl]
+; CHECK-NEXT:    ldr z13, [x0, #10, mul vl]
+; CHECK-NEXT:    ldr z29, [x0, #6, mul vl]
+; CHECK-NEXT:    fcmge p1.d, p0/z, z5.d, z25.d
+; CHECK-NEXT:    mov x9, #4890909195324358655 // =0x43dfffffffffffff
+; CHECK-NEXT:    mov z24.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z30.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z5.d, p0/m, z5.d
-; CHECK-NEXT:    mov z26.d, #0x8000000000000000
-; CHECK-NEXT:    mov z2.d, #0x8000000000000000
-; CHECK-NEXT:    mov z13.d, #0x8000000000000000
-; CHECK-NEXT:    mov z12.d, #0x8000000000000000
-; CHECK-NEXT:    mov x10, #4890909195324358655 // =0x43dfffffffffffff
-; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    fcmge p3.d, p0/z, z4.d, z25.d
-; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z25.d
-; CHECK-NEXT:    ldr z29, [x0, #7, mul vl]
-; CHECK-NEXT:    ldr z24, [x0, #6, mul vl]
-; CHECK-NEXT:    ldr z10, [x0, #9, mul vl]
-; CHECK-NEXT:    ldr z8, [x0, #8, mul vl]
-; CHECK-NEXT:    ldr z7, [x0, #5, mul vl]
-; CHECK-NEXT:    ldr z14, [x0, #15, mul vl]
-; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z25.d
-; CHECK-NEXT:    fcmge p5.d, p0/z, z6.d, z25.d
-; CHECK-NEXT:    ldr z15, [x0, #14, mul vl]
+; CHECK-NEXT:    movprfx z28, z27
+; CHECK-NEXT:    frintx z28.d, p0/m, z27.d
+; CHECK-NEXT:    mov z1.d, x9
+; CHECK-NEXT:    fcmge p2.d, p0/z, z7.d, z25.d
 ; CHECK-NEXT:    frintx z29.d, p0/m, z29.d
-; CHECK-NEXT:    frintx z24.d, p0/m, z24.d
-; CHECK-NEXT:    movprfx z11, z10
-; CHECK-NEXT:    frintx z11.d, p0/m, z10.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z5.d, z25.d
-; CHECK-NEXT:    movprfx z9, z8
-; CHECK-NEXT:    frintx z9.d, p0/m, z8.d
-; CHECK-NEXT:    ldr z16, [x0, #11, mul vl]
-; CHECK-NEXT:    ldr z20, [x0, #13, mul vl]
-; CHECK-NEXT:    frintx z7.d, p0/m, z7.d
-; CHECK-NEXT:    fcvtzs z28.d, p3/m, z4.d
-; CHECK-NEXT:    mov z10.d, #0x8000000000000000
-; CHECK-NEXT:    ldr z18, [x0, #12, mul vl]
-; CHECK-NEXT:    movprfx z19, z14
-; CHECK-NEXT:    frintx z19.d, p0/m, z14.d
-; CHECK-NEXT:    fcmge p3.d, p0/z, z29.d, z25.d
-; CHECK-NEXT:    ldr z17, [x0, #10, mul vl]
-; CHECK-NEXT:    frintx z15.d, p0/m, z15.d
-; CHECK-NEXT:    fcvtzs z27.d, p2/m, z1.d
-; CHECK-NEXT:    fcvtzs z30.d, p5/m, z6.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z24.d, z25.d
-; CHECK-NEXT:    fcmge p5.d, p0/z, z11.d, z25.d
-; CHECK-NEXT:    mov z14.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z16.d, p0/m, z16.d
-; CHECK-NEXT:    frintx z20.d, p0/m, z20.d
-; CHECK-NEXT:    fcvtzs z26.d, p4/m, z5.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z9.d, z25.d
+; CHECK-NEXT:    movprfx z3, z2
+; CHECK-NEXT:    frintx z3.d, p0/m, z2.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z14.d, z25.d
+; CHECK-NEXT:    mov z6.d, #0x8000000000000000
+; CHECK-NEXT:    mov z12.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z0.d, p1/m, z5.d
 ; CHECK-NEXT:    frintx z18.d, p0/m, z18.d
-; CHECK-NEXT:    mov z31.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z2.d, p1/m, z0.d
-; CHECK-NEXT:    fcmge p1.d, p0/z, z7.d, z25.d
-; CHECK-NEXT:    mov z8.d, #0x8000000000000000
-; CHECK-NEXT:    frintx z17.d, p0/m, z17.d
-; CHECK-NEXT:    fcvtzs z10.d, p3/m, z29.d
-; CHECK-NEXT:    fcmge p3.d, p0/z, z19.d, z25.d
-; CHECK-NEXT:    mov z3.d, x10
-; CHECK-NEXT:    fcmge p6.d, p0/z, z15.d, z25.d
+; CHECK-NEXT:    mov z2.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p5.d, p0/z, z4.d, z25.d
+; CHECK-NEXT:    mov z16.d, #0x8000000000000000
+; CHECK-NEXT:    mov z17.d, #0x8000000000000000
+; CHECK-NEXT:    fcvtzs z24.d, p2/m, z7.d
+; CHECK-NEXT:    frintx z10.d, p0/m, z10.d
+; CHECK-NEXT:    frintx z9.d, p0/m, z9.d
+; CHECK-NEXT:    fcvtzs z30.d, p3/m, z14.d
+; CHECK-NEXT:    frintx z13.d, p0/m, z13.d
 ; CHECK-NEXT:    mov z21.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z13.d, p2/m, z24.d
-; CHECK-NEXT:    fcvtzs z14.d, p5/m, z11.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z16.d, z25.d
+; CHECK-NEXT:    str z0, [sp] // 16-byte Folded Spill
 ; CHECK-NEXT:    mov z22.d, #0x8000000000000000
-; CHECK-NEXT:    fcmge p5.d, p0/z, z20.d, z25.d
-; CHECK-NEXT:    mov z0.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z12.d, p4/m, z9.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z18.d, z25.d
+; CHECK-NEXT:    movprfx z0, z8
+; CHECK-NEXT:    frintx z0.d, p0/m, z8.d
+; CHECK-NEXT:    ldr z31, [x0, #7, mul vl]
+; CHECK-NEXT:    ldr z15, [x0, #8, mul vl]
+; CHECK-NEXT:    ldr z19, [x0, #9, mul vl]
+; CHECK-NEXT:    fcmge p1.d, p0/z, z28.d, z25.d
+; CHECK-NEXT:    fcvtzs z6.d, p5/m, z4.d
+; CHECK-NEXT:    mov z26.d, #0x8000000000000000
+; CHECK-NEXT:    fcmgt p10.d, p0/z, z14.d, z1.d
+; CHECK-NEXT:    mov z27.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    mov z23.d, #0x8000000000000000
-; CHECK-NEXT:    fcvtzs z8.d, p1/m, z7.d
-; CHECK-NEXT:    fcmge p1.d, p0/z, z17.d, z25.d
-; CHECK-NEXT:    fcvtzs z31.d, p3/m, z19.d
-; CHECK-NEXT:    mov z25.d, #0x7fffffffffffffff
-; CHECK-NEXT:    fcmgt p11.d, p0/z, z19.d, z3.d
-; CHECK-NEXT:    fcvtzs z21.d, p6/m, z15.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z15.d, z3.d
-; CHECK-NEXT:    fcmuo p6.d, p0/z, z15.d, z15.d
-; CHECK-NEXT:    mov z15.d, #0x8000000000000000
-; CHECK-NEXT:    fcmgt p7.d, p0/z, z1.d, z3.d
-; CHECK-NEXT:    fcvtzs z22.d, p2/m, z16.d
-; CHECK-NEXT:    fcvtzs z0.d, p5/m, z20.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z24.d, z3.d
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z20.d, z3.d
-; CHECK-NEXT:    fcvtzs z23.d, p4/m, z18.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z19.d, z19.d
-; CHECK-NEXT:    mov z31.d, p11/m, z25.d
-; CHECK-NEXT:    sel z19.d, p3, z25.d, z21.d
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z18.d, z3.d
-; CHECK-NEXT:    fcvtzs z15.d, p1/m, z17.d
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z20.d, z20.d
-; CHECK-NEXT:    mov z27.d, p7/m, z25.d
-; CHECK-NEXT:    fcmgt p7.d, p0/z, z16.d, z3.d
-; CHECK-NEXT:    mov z13.d, p2/m, z25.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z17.d, z3.d
-; CHECK-NEXT:    mov z0.d, p5/m, z25.d
-; CHECK-NEXT:    mov z31.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    movprfx z20, z31
+; CHECK-NEXT:    frintx z20.d, p0/m, z31.d
+; CHECK-NEXT:    frintx z15.d, p0/m, z15.d
+; CHECK-NEXT:    mov z31.d, #0x8000000000000000
+; CHECK-NEXT:    fcmge p2.d, p0/z, z29.d, z25.d
+; CHECK-NEXT:    fcmuo p8.d, p0/z, z14.d, z14.d
+; CHECK-NEXT:    movprfx z14, z19
+; CHECK-NEXT:    frintx z14.d, p0/m, z19.d
+; CHECK-NEXT:    movprfx z19, z11
+; CHECK-NEXT:    frintx z19.d, p0/m, z11.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z3.d, z25.d
+; CHECK-NEXT:    fcvtzs z12.d, p1/m, z28.d
+; CHECK-NEXT:    mov z11.d, #0x8000000000000000
+; CHECK-NEXT:    mov z30.d, p10/m, z27.d
+; CHECK-NEXT:    fcvtzs z31.d, p2/m, z29.d
+; CHECK-NEXT:    fcmge p5.d, p0/z, z20.d, z25.d
+; CHECK-NEXT:    fcmge p6.d, p0/z, z15.d, z25.d
+; CHECK-NEXT:    fcvtzs z2.d, p4/m, z3.d
+; CHECK-NEXT:    mov z30.d, p8/m, #0 // =0x0
+; CHECK-NEXT:    fcmge p1.d, p0/z, z18.d, z25.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z19.d, z25.d
+; CHECK-NEXT:    fcvtzs z16.d, p5/m, z20.d
+; CHECK-NEXT:    fcvtzs z17.d, p6/m, z15.d
+; CHECK-NEXT:    fcmgt p12.d, p0/z, z28.d, z1.d
+; CHECK-NEXT:    fcvtzs z21.d, p1/m, z18.d
+; CHECK-NEXT:    fcvtzs z22.d, p2/m, z19.d
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z20.d, z1.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z10.d, z25.d
+; CHECK-NEXT:    fcmgt p11.d, p0/z, z15.d, z1.d
+; CHECK-NEXT:    sel z8.d, p12, z27.d, z12.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z9.d, z25.d
+; CHECK-NEXT:    sel z12.d, p5, z27.d, z16.d
+; CHECK-NEXT:    fcmge p6.d, p0/z, z14.d, z25.d
+; CHECK-NEXT:    fcvtzs z23.d, p3/m, z10.d
+; CHECK-NEXT:    fcmge p7.d, p0/z, z13.d, z25.d
+; CHECK-NEXT:    fcvtzs z26.d, p4/m, z9.d
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z15.d, z15.d
+; CHECK-NEXT:    sel z15.d, p11, z27.d, z17.d
+; CHECK-NEXT:    fcvtzs z11.d, p6/m, z14.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, z25.d
+; CHECK-NEXT:    mov z25.d, #0x8000000000000000
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z9.d, z1.d
+; CHECK-NEXT:    mov z15.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p5.d, p0/z, z10.d, z1.d
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z19.d, z1.d
+; CHECK-NEXT:    fcvtzs z25.d, p2/m, z0.d
+; CHECK-NEXT:    str z15, [x8, #8, mul vl]
+; CHECK-NEXT:    mov z26.d, p4/m, z27.d
+; CHECK-NEXT:    fcmuo p9.d, p0/z, z20.d, z20.d
+; CHECK-NEXT:    mov z20.d, #0x8000000000000000
+; CHECK-NEXT:    fcmuo p1.d, p0/z, z9.d, z9.d
+; CHECK-NEXT:    sel z9.d, p5, z27.d, z23.d
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z10.d, z10.d
+; CHECK-NEXT:    sel z10.d, p3, z27.d, z22.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z19.d, z19.d
+; CHECK-NEXT:    fcvtzs z20.d, p7/m, z13.d
+; CHECK-NEXT:    mov z12.d, p9/m, #0 // =0x0
+; CHECK-NEXT:    mov z26.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z9.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    str z12, [x8, #7, mul vl]
+; CHECK-NEXT:    mov z10.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z14.d, z1.d
+; CHECK-NEXT:    str z26, [x8, #15, mul vl]
+; CHECK-NEXT:    fcmgt p3.d, p0/z, z0.d, z1.d
+; CHECK-NEXT:    str z9, [x8, #14, mul vl]
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z18.d, z1.d
+; CHECK-NEXT:    str z10, [x8, #13, mul vl]
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z13.d, z1.d
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z14.d, z14.d
+; CHECK-NEXT:    fcmuo p6.d, p0/z, z0.d, z0.d
+; CHECK-NEXT:    sel z0.d, p4, z27.d, z11.d
+; CHECK-NEXT:    mov z25.d, p3/m, z27.d
+; CHECK-NEXT:    sel z26.d, p1, z27.d, z21.d
+; CHECK-NEXT:    sel z9.d, p2, z27.d, z20.d
 ; CHECK-NEXT:    fcmuo p4.d, p0/z, z18.d, z18.d
-; CHECK-NEXT:    sel z20.d, p3, z25.d, z23.d
-; CHECK-NEXT:    fcmuo p3.d, p0/z, z16.d, z16.d
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z11.d, z3.d
-; CHECK-NEXT:    mov z19.d, p6/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p10.d, p0/z, z6.d, z3.d
-; CHECK-NEXT:    fcmgt p8.d, p0/z, z4.d, z3.d
-; CHECK-NEXT:    str z31, [x8, #15, mul vl]
-; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z17.d, z17.d
-; CHECK-NEXT:    sel z18.d, p7, z25.d, z22.d
-; CHECK-NEXT:    sel z31.d, p2, z25.d, z15.d
-; CHECK-NEXT:    fcmgt p2.d, p0/z, z9.d, z3.d
-; CHECK-NEXT:    str z19, [x8, #14, mul vl]
-; CHECK-NEXT:    mov z20.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z11.d, z11.d
-; CHECK-NEXT:    str z0, [x8, #13, mul vl]
-; CHECK-NEXT:    mov z14.d, p5/m, z25.d
-; CHECK-NEXT:    fcmgt p5.d, p0/z, z29.d, z3.d
-; CHECK-NEXT:    mov z18.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z31.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p1.d, p0/z, z7.d, z3.d
-; CHECK-NEXT:    str z20, [x8, #12, mul vl]
-; CHECK-NEXT:    fcmuo p3.d, p0/z, z9.d, z9.d
-; CHECK-NEXT:    sel z0.d, p2, z25.d, z12.d
-; CHECK-NEXT:    mov z14.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z7.d, z7.d
-; CHECK-NEXT:    fcmuo p2.d, p0/z, z29.d, z29.d
-; CHECK-NEXT:    str z18, [x8, #11, mul vl]
-; CHECK-NEXT:    sel z29.d, p5, z25.d, z10.d
-; CHECK-NEXT:    fcmuo p5.d, p0/z, z24.d, z24.d
-; CHECK-NEXT:    str z31, [x8, #10, mul vl]
-; CHECK-NEXT:    sel z7.d, p1, z25.d, z8.d
-; CHECK-NEXT:    fcmuo p1.d, p0/z, z6.d, z6.d
-; CHECK-NEXT:    ldr z6, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    str z14, [x8, #9, mul vl]
-; CHECK-NEXT:    fcmgt p9.d, p0/z, z5.d, z3.d
-; CHECK-NEXT:    mov z0.d, p3/m, #0 // =0x0
-; CHECK-NEXT:    mov z29.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p2.d, p0/z, z5.d, z5.d
-; CHECK-NEXT:    mov z13.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    fcmuo p5.d, p0/z, z4.d, z4.d
-; CHECK-NEXT:    mov z7.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    fcmgt p3.d, p0/z, z6.d, z3.d
-; CHECK-NEXT:    fcmuo p4.d, p0/z, z1.d, z1.d
-; CHECK-NEXT:    fcmuo p0.d, p0/z, z6.d, z6.d
-; CHECK-NEXT:    str z0, [x8, #8, mul vl]
-; CHECK-NEXT:    mov z28.d, p8/m, z25.d
-; CHECK-NEXT:    mov z26.d, p9/m, z25.d
-; CHECK-NEXT:    str z29, [x8, #7, mul vl]
-; CHECK-NEXT:    mov z30.d, p10/m, z25.d
-; CHECK-NEXT:    str z13, [x8, #6, mul vl]
-; CHECK-NEXT:    str z7, [x8, #5, mul vl]
-; CHECK-NEXT:    sel z0.d, p3, z25.d, z2.d
-; CHECK-NEXT:    mov z26.d, p2/m, #0 // =0x0
-; CHECK-NEXT:    mov z30.d, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z28.d, p5/m, #0 // =0x0
-; CHECK-NEXT:    mov z27.d, p4/m, #0 // =0x0
-; CHECK-NEXT:    str z26, [x8, #3, mul vl]
-; CHECK-NEXT:    mov z0.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z25.d, p6/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z13.d, z13.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z3.d, z1.d
+; CHECK-NEXT:    str z0, [x8, #9, mul vl]
+; CHECK-NEXT:    mov z26.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    str z25, [x8, #12, mul vl]
+; CHECK-NEXT:    fcmgt p2.d, p0/z, z4.d, z1.d
+; CHECK-NEXT:    mov z9.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmgt p4.d, p0/z, z7.d, z1.d
+; CHECK-NEXT:    str z26, [x8, #11, mul vl]
+; CHECK-NEXT:    mov z2.d, p1/m, z27.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z28.d, z28.d
+; CHECK-NEXT:    fcmgt p6.d, p0/z, z29.d, z1.d
+; CHECK-NEXT:    str z9, [x8, #10, mul vl]
+; CHECK-NEXT:    mov z6.d, p2/m, z27.d
+; CHECK-NEXT:    fcmgt p1.d, p0/z, z5.d, z1.d
+; CHECK-NEXT:    ldr z1, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:    str z30, [x8, #4, mul vl]
-; CHECK-NEXT:    str z28, [x8, #2, mul vl]
-; CHECK-NEXT:    str z27, [x8, #1, mul vl]
-; CHECK-NEXT:    str z0, [x8]
+; CHECK-NEXT:    fcmuo p5.d, p0/z, z29.d, z29.d
+; CHECK-NEXT:    sel z0.d, p4, z27.d, z24.d
+; CHECK-NEXT:    mov z8.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p2.d, p0/z, z7.d, z7.d
+; CHECK-NEXT:    sel z25.d, p6, z27.d, z31.d
+; CHECK-NEXT:    mov z1.d, p1/m, z27.d
+; CHECK-NEXT:    fcmuo p3.d, p0/z, z4.d, z4.d
+; CHECK-NEXT:    str z8, [x8, #5, mul vl]
+; CHECK-NEXT:    fcmuo p4.d, p0/z, z3.d, z3.d
+; CHECK-NEXT:    mov z25.d, p5/m, #0 // =0x0
+; CHECK-NEXT:    mov z0.d, p2/m, #0 // =0x0
+; CHECK-NEXT:    fcmuo p0.d, p0/z, z5.d, z5.d
+; CHECK-NEXT:    mov z6.d, p3/m, #0 // =0x0
+; CHECK-NEXT:    str z25, [x8, #6, mul vl]
+; CHECK-NEXT:    mov z2.d, p4/m, #0 // =0x0
+; CHECK-NEXT:    str z0, [x8, #3, mul vl]
+; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
+; CHECK-NEXT:    str z6, [x8, #2, mul vl]
+; CHECK-NEXT:    str z2, [x8, #1, mul vl]
+; CHECK-NEXT:    str z1, [x8]
 ; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr z23, [sp, #1, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z22, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z21, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z20, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z19, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z18, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z17, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #10, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #11, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #12, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #13, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #14, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #15, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #16, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr p11, [sp] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p10, [sp, #1, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #17
+; CHECK-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK-NEXT:    addvl sp, sp, #18
 ; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
 ; CHECK-NEXT:    ret
   %a = call <vscale x 32 x iXLen> @llvm.lrint.nxv32iXLen.nxv16f64(<vscale x 32 x double> %x)
diff --git a/llvm/test/CodeGen/AArch64/sve-pr92779.ll b/llvm/test/CodeGen/AArch64/sve-pr92779.ll
index 3f34d79b3bb49..bce2d4bede385 100644
--- a/llvm/test/CodeGen/AArch64/sve-pr92779.ll
+++ b/llvm/test/CodeGen/AArch64/sve-pr92779.ll
@@ -5,16 +5,16 @@ define void @main(ptr %0) {
 ; CHECK-LABEL: main:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi v0.2d, #0000000000000000
-; CHECK-NEXT:    movi v1.2d, #0000000000000000
 ; CHECK-NEXT:    ptrue p0.d, vl1
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uzp1 v0.2s, v1.2s, v0.2s
-; CHECK-NEXT:    neg v0.2s, v0.2s
-; CHECK-NEXT:    smov x8, v0.s[0]
-; CHECK-NEXT:    smov x9, v0.s[1]
-; CHECK-NEXT:    mov z1.d, p0/m, x8
-; CHECK-NEXT:    mov z1.d, p0/m, x9
-; CHECK-NEXT:    str z1, [x0]
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
+; CHECK-NEXT:    uzp1 v1.2s, v0.2s, v1.2s
+; CHECK-NEXT:    neg v1.2s, v1.2s
+; CHECK-NEXT:    smov x8, v1.s[0]
+; CHECK-NEXT:    smov x9, v1.s[1]
+; CHECK-NEXT:    mov z0.d, p0/m, x8
+; CHECK-NEXT:    mov z0.d, p0/m, x9
+; CHECK-NEXT:    str z0, [x0]
 ; CHECK-NEXT:    ret
 "entry":
   %1 = bitcast <vscale x 2 x i64> zeroinitializer to <vscale x 4 x i32>
diff --git a/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll b/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll
index e0da9b57c6556..1f1ca7e3b9ee9 100644
--- a/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll
+++ b/llvm/test/CodeGen/AArch64/sve-stack-frame-layout.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -aarch64-streaming-hazard-size=0 | FileCheck %s --check-prefixes=CHECK
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -aarch64-streaming-hazard-size=0 | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK
+; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -aarch64-streaming-hazard-size=0 -aarch64-new-sme-abi | FileCheck %s --check-prefixes=CHECK-COMMON,CHECK-NEWLOWERING
 ; RUN: llc < %s -mtriple=aarch64 -mattr=+sve2 -aarch64-streaming-hazard-size=0 -pass-remarks-analysis=stack-frame-layout 2>&1 >/dev/null | FileCheck %s --check-prefixes=CHECK-FRAMELAYOUT
 
 ; CHECK-FRAMELAYOUT-LABEL: Function: csr_d8_allocnxv4i32i32f64
@@ -10,28 +11,29 @@
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32-16 x vscale], Type: Variable, Align: 8, Size: 8
 
 define i32 @csr_d8_allocnxv4i32i32f64(double %d) "aarch64_pstate_sm_compatible" {
-; CHECK-LABEL: csr_d8_allocnxv4i32i32f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str d8, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    str x29, [sp, #8] // 8-byte Folded Spill
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 32 + 8 * VG
-; CHECK-NEXT:    .cfi_offset w29, -8
-; CHECK-NEXT:    .cfi_offset b8, -16
-; CHECK-NEXT:    mov z1.s, #0 // =0x0
-; CHECK-NEXT:    add x8, sp, #16
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    //APP
-; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    str wzr, [sp, #12]
-; CHECK-NEXT:    str d0, [sp]
-; CHECK-NEXT:    str z1, [x8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldr x29, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr d8, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: csr_d8_allocnxv4i32i32f64:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    str d8, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    str x29, [sp, #8] // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    sub sp, sp, #16
+; CHECK-COMMON-NEXT:    addvl sp, sp, #-1
+; CHECK-COMMON-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 32 + 8 * VG
+; CHECK-COMMON-NEXT:    .cfi_offset w29, -8
+; CHECK-COMMON-NEXT:    .cfi_offset b8, -16
+; CHECK-COMMON-NEXT:    mov z1.s, #0 // =0x0
+; CHECK-COMMON-NEXT:    add x8, sp, #16
+; CHECK-COMMON-NEXT:    mov w0, wzr
+; CHECK-COMMON-NEXT:    //APP
+; CHECK-COMMON-NEXT:    //NO_APP
+; CHECK-COMMON-NEXT:    str wzr, [sp, #12]
+; CHECK-COMMON-NEXT:    str d0, [sp]
+; CHECK-COMMON-NEXT:    str z1, [x8]
+; CHECK-COMMON-NEXT:    addvl sp, sp, #1
+; CHECK-COMMON-NEXT:    add sp, sp, #16
+; CHECK-COMMON-NEXT:    ldr x29, [sp, #8] // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr d8, [sp], #16 // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
+; CHECK-COMMON-NE
 entry:
   %a = alloca <vscale x 4 x i32>
   %b = alloca i32
@@ -52,31 +54,31 @@ entry:
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-40-16 x vscale], Type: Variable, Align: 8, Size: 8
 
 define i32 @csr_d8_allocnxv4i32i32f64_fp(double %d) "aarch64_pstate_sm_compatible" "frame-pointer"="all" {
-; CHECK-LABEL: csr_d8_allocnxv4i32i32f64_fp:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    add x29, sp, #16
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    .cfi_offset b8, -32
-; CHECK-NEXT:    mov z1.s, #0 // =0x0
-; CHECK-NEXT:    addvl x8, sp, #1
-; CHECK-NEXT:    //APP
-; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    str wzr, [x8, #28]
-; CHECK-NEXT:    sub x8, x29, #16
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    str d0, [sp, #8]
-; CHECK-NEXT:    str z1, [x8, #-1, mul vl]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: csr_d8_allocnxv4i32i32f64_fp:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    add x29, sp, #16
+; CHECK-COMMON-NEXT:    sub sp, sp, #16
+; CHECK-COMMON-NEXT:    addvl sp, sp, #-1
+; CHECK-COMMON-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-COMMON-NEXT:    .cfi_offset w30, -8
+; CHECK-COMMON-NEXT:    .cfi_offset w29, -16
+; CHECK-COMMON-NEXT:    .cfi_offset b8, -32
+; CHECK-COMMON-NEXT:    mov z1.s, #0 // =0x0
+; CHECK-COMMON-NEXT:    addvl x8, sp, #1
+; CHECK-COMMON-NEXT:    //APP
+; CHECK-COMMON-NEXT:    //NO_APP
+; CHECK-COMMON-NEXT:    str wzr, [x8, #28]
+; CHECK-COMMON-NEXT:    sub x8, x29, #16
+; CHECK-COMMON-NEXT:    mov w0, wzr
+; CHECK-COMMON-NEXT:    str d0, [sp, #8]
+; CHECK-COMMON-NEXT:    str z1, [x8, #-1, mul vl]
+; CHECK-COMMON-NEXT:    addvl sp, sp, #1
+; CHECK-COMMON-NEXT:    add sp, sp, #16
+; CHECK-COMMON-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
 entry:
   %a = alloca <vscale x 4 x i32>
   %b = alloca i32
@@ -102,30 +104,30 @@ entry:
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-128-16 x vscale], Type: Variable, Align: 128, Size: 4
 
 define i32 @csr_d8_allocnxv4i32i32f64_dynamicrealign(double %d) "aarch64_pstate_sm_compatible" {
-; CHECK-LABEL: csr_d8_allocnxv4i32i32f64_dynamicrealign:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-NEXT:    sub x9, sp, #96
-; CHECK-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    add x29, sp, #16
-; CHECK-NEXT:    addvl x9, x9, #-1
-; CHECK-NEXT:    and sp, x9, #0xffffffffffffff80
-; CHECK-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    .cfi_offset b8, -32
-; CHECK-NEXT:    mov z1.s, #0 // =0x0
-; CHECK-NEXT:    sub x8, x29, #16
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    //APP
-; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    str wzr, [sp]
-; CHECK-NEXT:    stur d0, [x29, #-8]
-; CHECK-NEXT:    str z1, [x8, #-1, mul vl]
-; CHECK-NEXT:    sub sp, x29, #16
-; CHECK-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: csr_d8_allocnxv4i32i32f64_dynamicrealign:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    sub x9, sp, #96
+; CHECK-COMMON-NEXT:    stp x29, x30, [sp, #16] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    add x29, sp, #16
+; CHECK-COMMON-NEXT:    addvl x9, x9, #-1
+; CHECK-COMMON-NEXT:    and sp, x9, #0xffffffffffffff80
+; CHECK-COMMON-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-COMMON-NEXT:    .cfi_offset w30, -8
+; CHECK-COMMON-NEXT:    .cfi_offset w29, -16
+; CHECK-COMMON-NEXT:    .cfi_offset b8, -32
+; CHECK-COMMON-NEXT:    mov z1.s, #0 // =0x0
+; CHECK-COMMON-NEXT:    sub x8, x29, #16
+; CHECK-COMMON-NEXT:    mov w0, wzr
+; CHECK-COMMON-NEXT:    //APP
+; CHECK-COMMON-NEXT:    //NO_APP
+; CHECK-COMMON-NEXT:    str wzr, [sp]
+; CHECK-COMMON-NEXT:    stur d0, [x29, #-8]
+; CHECK-COMMON-NEXT:    str z1, [x8, #-1, mul vl]
+; CHECK-COMMON-NEXT:    sub sp, x29, #16
+; CHECK-COMMON-NEXT:    ldp x29, x30, [sp, #16] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
 entry:
   %a = alloca <vscale x 4 x i32>
   %b = alloca i32, align 128
@@ -151,44 +153,44 @@ entry:
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-16 x vscale], Type: VariableSized, Align: 1, Size: 0
 
 define i32 @csr_d8_allocnxv4i32i32f64_vla(double %d, i32 %i) "aarch64_pstate_sm_compatible" {
-; CHECK-LABEL: csr_d8_allocnxv4i32i32f64_vla:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
-; CHECK-NEXT:    stp x29, x30, [sp, #8] // 16-byte Folded Spill
-; CHECK-NEXT:    add x29, sp, #8
-; CHECK-NEXT:    str x19, [sp, #24] // 8-byte Folded Spill
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    mov x19, sp
-; CHECK-NEXT:    .cfi_def_cfa w29, 24
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    .cfi_offset w29, -24
-; CHECK-NEXT:    .cfi_offset b8, -32
-; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
-; CHECK-NEXT:    ubfiz x8, x0, #2, #32
-; CHECK-NEXT:    mov x9, sp
-; CHECK-NEXT:    add x8, x8, #15
-; CHECK-NEXT:    and x8, x8, #0x7fffffff0
-; CHECK-NEXT:    sub x9, x9, x8
-; CHECK-NEXT:    mov sp, x9
-; CHECK-NEXT:    mov x10, sp
-; CHECK-NEXT:    sub x8, x10, x8
-; CHECK-NEXT:    mov sp, x8
-; CHECK-NEXT:    mov z1.s, #0 // =0x0
-; CHECK-NEXT:    //APP
-; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    str wzr, [x8]
-; CHECK-NEXT:    sub x8, x29, #8
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    str wzr, [x9]
-; CHECK-NEXT:    str d0, [x19, #8]
-; CHECK-NEXT:    str z1, [x8, #-1, mul vl]
-; CHECK-NEXT:    sub sp, x29, #8
-; CHECK-NEXT:    ldp x29, x30, [sp, #8] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x19, [sp, #24] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: csr_d8_allocnxv4i32i32f64_vla:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    stp x29, x30, [sp, #8] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    add x29, sp, #8
+; CHECK-COMMON-NEXT:    str x19, [sp, #24] // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    sub sp, sp, #16
+; CHECK-COMMON-NEXT:    addvl sp, sp, #-1
+; CHECK-COMMON-NEXT:    mov x19, sp
+; CHECK-COMMON-NEXT:    .cfi_def_cfa w29, 24
+; CHECK-COMMON-NEXT:    .cfi_offset w19, -8
+; CHECK-COMMON-NEXT:    .cfi_offset w30, -16
+; CHECK-COMMON-NEXT:    .cfi_offset w29, -24
+; CHECK-COMMON-NEXT:    .cfi_offset b8, -32
+; CHECK-COMMON-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-COMMON-NEXT:    ubfiz x8, x0, #2, #32
+; CHECK-COMMON-NEXT:    mov x9, sp
+; CHECK-COMMON-NEXT:    add x8, x8, #15
+; CHECK-COMMON-NEXT:    and x8, x8, #0x7fffffff0
+; CHECK-COMMON-NEXT:    sub x9, x9, x8
+; CHECK-COMMON-NEXT:    mov sp, x9
+; CHECK-COMMON-NEXT:    mov x10, sp
+; CHECK-COMMON-NEXT:    sub x8, x10, x8
+; CHECK-COMMON-NEXT:    mov sp, x8
+; CHECK-COMMON-NEXT:    mov z1.s, #0 // =0x0
+; CHECK-COMMON-NEXT:    //APP
+; CHECK-COMMON-NEXT:    //NO_APP
+; CHECK-COMMON-NEXT:    str wzr, [x8]
+; CHECK-COMMON-NEXT:    sub x8, x29, #8
+; CHECK-COMMON-NEXT:    mov w0, wzr
+; CHECK-COMMON-NEXT:    str wzr, [x9]
+; CHECK-COMMON-NEXT:    str d0, [x19, #8]
+; CHECK-COMMON-NEXT:    str z1, [x8, #-1, mul vl]
+; CHECK-COMMON-NEXT:    sub sp, x29, #8
+; CHECK-COMMON-NEXT:    ldp x29, x30, [sp, #8] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr x19, [sp, #24] // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
 entry:
   %a = alloca <vscale x 4 x i32>
   %0 = zext i32 %i to i64
@@ -213,28 +215,28 @@ entry:
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32-16 x vscale], Type: Variable, Align: 8, Size: 8
 
 define i32 @csr_d8_allocnxv4i32i32f64_stackargsi32f64(double %d0, double %d1, double %d2, double %d3, double %d4, double %d5, double %d6, double %d7, double %d8, i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6, i32 %i7, i32 %i8) "aarch64_pstate_sm_compatible" {
-; CHECK-LABEL: csr_d8_allocnxv4i32i32f64_stackargsi32f64:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    str d8, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT:    str x29, [sp, #8] // 8-byte Folded Spill
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 32 + 8 * VG
-; CHECK-NEXT:    .cfi_offset w29, -8
-; CHECK-NEXT:    .cfi_offset b8, -16
-; CHECK-NEXT:    mov z1.s, #0 // =0x0
-; CHECK-NEXT:    add x8, sp, #16
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    //APP
-; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    str wzr, [sp, #12]
-; CHECK-NEXT:    str d0, [sp]
-; CHECK-NEXT:    str z1, [x8]
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    add sp, sp, #16
-; CHECK-NEXT:    ldr x29, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    ldr d8, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: csr_d8_allocnxv4i32i32f64_stackargsi32f64:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    str d8, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    str x29, [sp, #8] // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    sub sp, sp, #16
+; CHECK-COMMON-NEXT:    addvl sp, sp, #-1
+; CHECK-COMMON-NEXT:    .cfi_escape 0x0f, 0x08, 0x8f, 0x20, 0x92, 0x2e, 0x00, 0x38, 0x1e, 0x22 // sp + 32 + 8 * VG
+; CHECK-COMMON-NEXT:    .cfi_offset w29, -8
+; CHECK-COMMON-NEXT:    .cfi_offset b8, -16
+; CHECK-COMMON-NEXT:    mov z1.s, #0 // =0x0
+; CHECK-COMMON-NEXT:    add x8, sp, #16
+; CHECK-COMMON-NEXT:    mov w0, wzr
+; CHECK-COMMON-NEXT:    //APP
+; CHECK-COMMON-NEXT:    //NO_APP
+; CHECK-COMMON-NEXT:    str wzr, [sp, #12]
+; CHECK-COMMON-NEXT:    str d0, [sp]
+; CHECK-COMMON-NEXT:    str z1, [x8]
+; CHECK-COMMON-NEXT:    addvl sp, sp, #1
+; CHECK-COMMON-NEXT:    add sp, sp, #16
+; CHECK-COMMON-NEXT:    ldr x29, [sp, #8] // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr d8, [sp], #16 // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
 entry:
   %a = alloca <vscale x 4 x i32>
   %b = alloca i32
@@ -255,29 +257,29 @@ entry:
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32-32 x vscale], Type: Variable, Align: 8, Size: 8
 
 define i32 @svecc_z8_allocnxv4i32i32f64_fp(double %d, <vscale x 4 x i32> %v) "aarch64_pstate_sm_compatible" "frame-pointer"="all" {
-; CHECK-LABEL: svecc_z8_allocnxv4i32i32f64_fp:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
-; CHECK-NEXT:    mov x29, sp
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    //APP
-; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    str wzr, [sp, #12]
-; CHECK-NEXT:    str z1, [x29, #-2, mul vl]
-; CHECK-NEXT:    str d0, [sp], #16
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: svecc_z8_allocnxv4i32i32f64_fp:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    mov x29, sp
+; CHECK-COMMON-NEXT:    addvl sp, sp, #-1
+; CHECK-COMMON-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    sub sp, sp, #16
+; CHECK-COMMON-NEXT:    addvl sp, sp, #-1
+; CHECK-COMMON-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-COMMON-NEXT:    .cfi_offset w30, -8
+; CHECK-COMMON-NEXT:    .cfi_offset w29, -16
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
+; CHECK-COMMON-NEXT:    mov w0, wzr
+; CHECK-COMMON-NEXT:    //APP
+; CHECK-COMMON-NEXT:    //NO_APP
+; CHECK-COMMON-NEXT:    str wzr, [sp, #12]
+; CHECK-COMMON-NEXT:    str z1, [x29, #-2, mul vl]
+; CHECK-COMMON-NEXT:    str d0, [sp], #16
+; CHECK-COMMON-NEXT:    addvl sp, sp, #1
+; CHECK-COMMON-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    addvl sp, sp, #1
+; CHECK-COMMON-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
 entry:
   %a = alloca <vscale x 4 x i32>
   %b = alloca i32
@@ -299,29 +301,29 @@ entry:
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32-32 x vscale], Type: Variable, Align: 8, Size: 8
 
 define i32 @svecc_z8_allocnxv4i32i32f64_stackargsi32_fp(double %d, i32 %i0, i32 %i1, i32 %i2, i32 %i3, i32 %i4, i32 %i5, i32 %i6, i32 %i7, i32 %i8, <vscale x 4 x i32> %v) "aarch64_pstate_sm_compatible" "frame-pointer"="all"{
-; CHECK-LABEL: svecc_z8_allocnxv4i32i32f64_stackargsi32_fp:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
-; CHECK-NEXT:    mov x29, sp
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    str z8, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    sub sp, sp, #16
-; CHECK-NEXT:    addvl sp, sp, #-1
-; CHECK-NEXT:    .cfi_def_cfa w29, 16
-; CHECK-NEXT:    .cfi_offset w30, -8
-; CHECK-NEXT:    .cfi_offset w29, -16
-; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    //APP
-; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    str wzr, [sp, #12]
-; CHECK-NEXT:    str z1, [x29, #-2, mul vl]
-; CHECK-NEXT:    str d0, [sp], #16
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #1
-; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: svecc_z8_allocnxv4i32i32f64_stackargsi32_fp:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    mov x29, sp
+; CHECK-COMMON-NEXT:    addvl sp, sp, #-1
+; CHECK-COMMON-NEXT:    str z8, [sp] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    sub sp, sp, #16
+; CHECK-COMMON-NEXT:    addvl sp, sp, #-1
+; CHECK-COMMON-NEXT:    .cfi_def_cfa w29, 16
+; CHECK-COMMON-NEXT:    .cfi_offset w30, -8
+; CHECK-COMMON-NEXT:    .cfi_offset w29, -16
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x48, 0x09, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x40, 0x1c // $d8 @ cfa - 8 * VG - 16
+; CHECK-COMMON-NEXT:    mov w0, wzr
+; CHECK-COMMON-NEXT:    //APP
+; CHECK-COMMON-NEXT:    //NO_APP
+; CHECK-COMMON-NEXT:    str wzr, [sp, #12]
+; CHECK-COMMON-NEXT:    str z1, [x29, #-2, mul vl]
+; CHECK-COMMON-NEXT:    str d0, [sp], #16
+; CHECK-COMMON-NEXT:    addvl sp, sp, #1
+; CHECK-COMMON-NEXT:    ldr z8, [sp] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    addvl sp, sp, #1
+; CHECK-COMMON-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ret
 entry:
   %a = alloca <vscale x 4 x i32>
   %b = alloca i32
@@ -338,157 +340,162 @@ entry:
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-16], Type: Spill, Align: 8, Size: 8
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-24], Type: Spill, Align: 8, Size: 8
 ; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-32], Type: Spill, Align: 8, Size: 8
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-40], Type: Spill, Align: 8, Size: 8
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48], Type: Spill, Align: 8, Size: 8
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-16 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-32 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-48 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-64 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-80 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-96 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-112 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-128 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-144 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-160 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-176 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-192 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-208 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-224 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-240 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-256 x vscale], Type: Spill, Align: 16, Size: vscale x 16
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-258 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-260 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-262 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-264 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-266 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-268 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-270 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-272 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-274 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-276 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-278 x vscale], Type: Spill, Align: 2, Size: vscale x 2
-; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48-280 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-48], Type: Spill, Align: 16, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-56], Type: Spill, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64], Type: Spill, Align: 8, Size: 8
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-16 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-32 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-48 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-64 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-80 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-96 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-112 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-128 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-144 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-160 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-176 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-192 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-208 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-224 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-240 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-256 x vscale], Type: Spill, Align: 16, Size: vscale x 16
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-258 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-260 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-262 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-264 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-266 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-268 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-270 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-272 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-274 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-276 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-278 x vscale], Type: Spill, Align: 2, Size: vscale x 2
+; CHECK-FRAMELAYOUT-NEXT: Offset: [SP-64-280 x vscale], Type: Spill, Align: 2, Size: vscale x 2
 
 define i32 @svecc_call(<4 x i16> %P0, ptr %P1, i32 %P2, <vscale x 16 x i8> %P3, i16 %P4) "aarch64_pstate_sm_compatible" {
-; CHECK-LABEL: svecc_call:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    stp x29, x30, [sp, #-48]! // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 48
-; CHECK-NEXT:    cntd x9
-; CHECK-NEXT:    stp x9, x28, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    stp x27, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w27, -16
-; CHECK-NEXT:    .cfi_offset w28, -24
-; CHECK-NEXT:    .cfi_offset w30, -40
-; CHECK-NEXT:    .cfi_offset w29, -48
-; CHECK-NEXT:    addvl sp, sp, #-18
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0a, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x11, 0x90, 0x01, 0x1e, 0x22 // sp + 48 + 144 * VG
-; CHECK-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
-; CHECK-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d8 @ cfa - 8 * VG - 48
-; CHECK-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d9 @ cfa - 16 * VG - 48
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d10 @ cfa - 24 * VG - 48
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d11 @ cfa - 32 * VG - 48
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d12 @ cfa - 40 * VG - 48
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d13 @ cfa - 48 * VG - 48
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d14 @ cfa - 56 * VG - 48
-; CHECK-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x92, 0x2e, 0x00, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x50, 0x22 // $d15 @ cfa - 64 * VG - 48
-; CHECK-NEXT:    mov x8, x0
-; CHECK-NEXT:    //APP
-; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    and x19, x0, #0x1
-; CHECK-NEXT:    .cfi_offset vg, -32
-; CHECK-NEXT:    tbz w19, #0, .LBB7_2
-; CHECK-NEXT:  // %bb.1: // %entry
-; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:  .LBB7_2: // %entry
-; CHECK-NEXT:    mov x0, x8
-; CHECK-NEXT:    mov w1, #45 // =0x2d
-; CHECK-NEXT:    mov w2, #37 // =0x25
-; CHECK-NEXT:    bl memset
-; CHECK-NEXT:    tbz w19, #0, .LBB7_4
-; CHECK-NEXT:  // %bb.3: // %entry
-; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:  .LBB7_4: // %entry
-; CHECK-NEXT:    mov w0, #22647 // =0x5877
-; CHECK-NEXT:    movk w0, #59491, lsl #16
-; CHECK-NEXT:    .cfi_restore vg
-; CHECK-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
-; CHECK-NEXT:    addvl sp, sp, #18
-; CHECK-NEXT:    .cfi_def_cfa wsp, 48
-; CHECK-NEXT:    .cfi_restore z8
-; CHECK-NEXT:    .cfi_restore z9
-; CHECK-NEXT:    .cfi_restore z10
-; CHECK-NEXT:    .cfi_restore z11
-; CHECK-NEXT:    .cfi_restore z12
-; CHECK-NEXT:    .cfi_restore z13
-; CHECK-NEXT:    .cfi_restore z14
-; CHECK-NEXT:    .cfi_restore z15
-; CHECK-NEXT:    ldp x27, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x28, [sp, #24] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp x29, x30, [sp], #48 // 16-byte Folded Reload
-; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:    .cfi_restore w19
-; CHECK-NEXT:    .cfi_restore w27
-; CHECK-NEXT:    .cfi_restore w28
-; CHECK-NEXT:    .cfi_restore w30
-; CHECK-NEXT:    .cfi_restore w29
-; CHECK-NEXT:    ret
+; CHECK-COMMON-LABEL: svecc_call:
+; CHECK-COMMON:       // %bb.0: // %entry
+; CHECK-COMMON-NEXT:    stp x29, x30, [sp, #-64]! // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-COMMON-NEXT:    cntd x9
+; CHECK-COMMON-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    stp x26, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    mov x29, sp
+; CHECK-COMMON-NEXT:    .cfi_def_cfa w29, 64
+; CHECK-COMMON-NEXT:    .cfi_offset w19, -8
+; CHECK-COMMON-NEXT:    .cfi_offset w26, -16
+; CHECK-COMMON-NEXT:    .cfi_offset w27, -24
+; CHECK-COMMON-NEXT:    .cfi_offset w28, -32
+; CHECK-COMMON-NEXT:    .cfi_offset vg, -48
+; CHECK-COMMON-NEXT:    .cfi_offset w30, -56
+; CHECK-COMMON-NEXT:    .cfi_offset w29, -64
+; CHECK-COMMON-NEXT:    addvl sp, sp, #-18
+; CHECK-COMMON-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x48, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x78, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d8 @ cfa - 8 * IncomingVG - 64
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x49, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x70, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d9 @ cfa - 16 * IncomingVG - 64
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x4a, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x68, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d10 @ cfa - 24 * IncomingVG - 64
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x4b, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x60, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d11 @ cfa - 32 * IncomingVG - 64
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x4c, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x58, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d12 @ cfa - 40 * IncomingVG - 64
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x4d, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x50, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d13 @ cfa - 48 * IncomingVG - 64
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x4e, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x48, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d14 @ cfa - 56 * IncomingVG - 64
+; CHECK-COMMON-NEXT:    .cfi_escape 0x10, 0x4f, 0x0c, 0x12, 0x11, 0x50, 0x22, 0x06, 0x11, 0x40, 0x1e, 0x22, 0x11, 0x40, 0x22 // $d15 @ cfa - 64 * IncomingVG - 64
+; CHECK-COMMON-NEXT:    mov x8, x0
+; CHECK-COMMON-NEXT:    bl __arm_sme_state
+; CHECK-COMMON-NEXT:    mov x19, x0
+; CHECK-COMMON-NEXT:    //APP
+; CHECK-COMMON-NEXT:    //NO_APP
+; CHECK-COMMON-NEXT:    tbz w19, #0, .LBB7_2
+; CHECK-COMMON-NEXT:  // %bb.1: // %entry
+; CHECK-COMMON-NEXT:    smstop sm
+; CHECK-COMMON-NEXT:  .LBB7_2: // %entry
+; CHECK-COMMON-NEXT:    mov x0, x8
+; CHECK-COMMON-NEXT:    mov w1, #45 // =0x2d
+; CHECK-COMMON-NEXT:    mov w2, #37 // =0x25
+; CHECK-COMMON-NEXT:    bl memset
+; CHECK-COMMON-NEXT:    tbz w19, #0, .LBB7_4
+; CHECK-COMMON-NEXT:  // %bb.3: // %entry
+; CHECK-COMMON-NEXT:    smstart sm
+; CHECK-COMMON-NEXT:  .LBB7_4: // %entry
+; CHECK-COMMON-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    mov w0, #22647 // =0x5877
+; CHECK-COMMON-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    movk w0, #59491, lsl #16
+; CHECK-COMMON-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK-COMMON-NEXT:    addvl sp, sp, #18
+; CHECK-COMMON-NEXT:    .cfi_restore z8
+; CHECK-COMMON-NEXT:    .cfi_restore z9
+; CHECK-COMMON-NEXT:    .cfi_restore z10
+; CHECK-COMMON-NEXT:    .cfi_restore z11
+; CHECK-COMMON-NEXT:    .cfi_restore z12
+; CHECK-COMMON-NEXT:    .cfi_restore z13
+; CHECK-COMMON-NEXT:    .cfi_restore z14
+; CHECK-COMMON-NEXT:    .cfi_restore z15
+; CHECK-COMMON-NEXT:    .cfi_def_cfa wsp, 64
+; CHECK-COMMON-NEXT:    ldp x26, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldp x29, x30, [sp], #64 // 16-byte Folded Reload
+; CHECK-COMMON-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-COMMON-NEXT:    .cfi_restore w19
+; CHECK-COMMON-NEXT:    .cfi_restore w26
+; CHECK-COMMON-NEXT:    .cfi_restore w27
+; CHECK-COMMON-NEXT:    .cfi_restore w28
+; CHECK-COMMON-NEXT:    .cfi_restore vg
+; CHECK-COMMON-NEXT:    .cfi_restore w30
+; CHECK-COMMON-NEXT:    .cfi_restore w29
+; CHECK-COMMON-NEXT:    ret
 entry:
   tail call void asm sideeffect "", "~{x0},~{x28},~{x27},~{x3}"() #2
   %call = call ptr @memset(ptr noundef nonnull %P1, i32 noundef 45, i32 noundef 37)
@@ -532,6 +539,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK-NEXT:    .cfi_def_cfa w29, 48
 ; CHECK-NEXT:    .cfi_offset w19, -8
 ; CHECK-NEXT:    .cfi_offset w20, -16
+; CHECK-NEXT:    .cfi_offset vg, -32
 ; CHECK-NEXT:    .cfi_offset w30, -40
 ; CHECK-NEXT:    .cfi_offset w29, -48
 ; CHECK-NEXT:    .cfi_offset b8, -56
@@ -554,11 +562,9 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK-NEXT:    stur wzr, [x29, #-68]
 ; CHECK-NEXT:    sturh w8, [x29, #-72]
 ; CHECK-NEXT:    msr TPIDR2_EL0, x9
-; CHECK-NEXT:    .cfi_offset vg, -32
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    bl other
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    smstart za
 ; CHECK-NEXT:    mrs x8, TPIDR2_EL0
 ; CHECK-NEXT:    sub x0, x29, #80
@@ -579,6 +585,7 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    .cfi_restore w19
 ; CHECK-NEXT:    .cfi_restore w20
+; CHECK-NEXT:    .cfi_restore vg
 ; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    .cfi_restore w29
 ; CHECK-NEXT:    .cfi_restore b8
@@ -590,6 +597,79 @@ define i32 @vastate(i32 %x) "aarch64_inout_za" "aarch64_pstate_sm_enabled" "targ
 ; CHECK-NEXT:    .cfi_restore b14
 ; CHECK-NEXT:    .cfi_restore b15
 ; CHECK-NEXT:    ret
+;
+; CHECK-NEWLOWERING-LABEL: vastate:
+; CHECK-NEWLOWERING:       // %bb.0: // %entry
+; CHECK-NEWLOWERING-NEXT:    stp d15, d14, [sp, #-112]! // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa_offset 112
+; CHECK-NEWLOWERING-NEXT:    cntd x9
+; CHECK-NEWLOWERING-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    str x9, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    stp x20, x19, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEWLOWERING-NEXT:    add x29, sp, #64
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa w29, 48
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w19, -8
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w20, -16
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset vg, -32
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w30, -40
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset w29, -48
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset b8, -56
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset b9, -64
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset b10, -72
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset b11, -80
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset b12, -88
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset b13, -96
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset b14, -104
+; CHECK-NEWLOWERING-NEXT:    .cfi_offset b15, -112
+; CHECK-NEWLOWERING-NEXT:    sub sp, sp, #16
+; CHECK-NEWLOWERING-NEXT:    rdsvl x8, #1
+; CHECK-NEWLOWERING-NEXT:    mov x9, sp
+; CHECK-NEWLOWERING-NEXT:    msub x9, x8, x8, x9
+; CHECK-NEWLOWERING-NEXT:    mov sp, x9
+; CHECK-NEWLOWERING-NEXT:    sub x10, x29, #80
+; CHECK-NEWLOWERING-NEXT:    mov w20, w0
+; CHECK-NEWLOWERING-NEXT:    stp x9, x8, [x29, #-80]
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, x10
+; CHECK-NEWLOWERING-NEXT:    smstop sm
+; CHECK-NEWLOWERING-NEXT:    bl other
+; CHECK-NEWLOWERING-NEXT:    smstart sm
+; CHECK-NEWLOWERING-NEXT:    mov w0, w20
+; CHECK-NEWLOWERING-NEXT:    mov w8, w0
+; CHECK-NEWLOWERING-NEXT:    smstart za
+; CHECK-NEWLOWERING-NEXT:    mrs x9, TPIDR2_EL0
+; CHECK-NEWLOWERING-NEXT:    sub x0, x29, #80
+; CHECK-NEWLOWERING-NEXT:    cbnz x9, .LBB8_2
+; CHECK-NEWLOWERING-NEXT:  // %bb.1: // %entry
+; CHECK-NEWLOWERING-NEXT:    bl __arm_tpidr2_restore
+; CHECK-NEWLOWERING-NEXT:  .LBB8_2: // %entry
+; CHECK-NEWLOWERING-NEXT:    mov w0, w8
+; CHECK-NEWLOWERING-NEXT:    msr TPIDR2_EL0, xzr
+; CHECK-NEWLOWERING-NEXT:    sub sp, x29, #64
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa wsp, 112
+; CHECK-NEWLOWERING-NEXT:    ldp x20, x19, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    ldp d15, d14, [sp], #112 // 16-byte Folded Reload
+; CHECK-NEWLOWERING-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore w19
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore w20
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore vg
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore w30
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore w29
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore b8
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore b9
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore b10
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore b11
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore b12
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore b13
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore b14
+; CHECK-NEWLOWERING-NEXT:    .cfi_restore b15
+; CHECK-NEWLOWERING-NEXT:    ret
 entry:
   tail call void @other()
   ret i32 %x
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll
index 200ffb60a7928..a8220d9993227 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-compares.ll
@@ -560,16 +560,16 @@ define void @fcmp_oeq_v4f64(ptr %a, ptr %b, ptr %c) {
 define void @fcmp_ueq_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-LABEL: fcmp_ueq_v16f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q3, [x1]
+; CHECK-NEXT:    ldp q3, q0, [x1]
 ; CHECK-NEXT:    ptrue p0.h, vl8
-; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    ldp q2, q1, [x0]
 ; CHECK-NEXT:    fcmuo p1.h, p0/z, z1.h, z0.h
-; CHECK-NEXT:    fcmeq p2.h, p0/z, z1.h, z0.h
-; CHECK-NEXT:    fcmuo p3.h, p0/z, z2.h, z3.h
-; CHECK-NEXT:    fcmeq p0.h, p0/z, z2.h, z3.h
-; CHECK-NEXT:    mov p1.b, p2/m, p2.b
-; CHECK-NEXT:    sel p0.b, p0, p0.b, p3.b
-; CHECK-NEXT:    mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmuo p2.h, p0/z, z2.h, z3.h
+; CHECK-NEXT:    fcmeq p3.h, p0/z, z2.h, z3.h
+; CHECK-NEXT:    fcmeq p0.h, p0/z, z1.h, z0.h
+; CHECK-NEXT:    mov p2.b, p3/m, p3.b
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
+; CHECK-NEXT:    mov z0.h, p2/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
@@ -729,16 +729,16 @@ define void @fcmp_ueq_v16f16(ptr %a, ptr %b, ptr %c) {
 define void @fcmp_one_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-LABEL: fcmp_one_v16f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q3, [x1]
+; CHECK-NEXT:    ldp q3, q0, [x1]
 ; CHECK-NEXT:    ptrue p0.h, vl8
-; CHECK-NEXT:    ldp q1, q2, [x0]
+; CHECK-NEXT:    ldp q2, q1, [x0]
 ; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    fcmgt p2.h, p0/z, z1.h, z0.h
-; CHECK-NEXT:    fcmgt p3.h, p0/z, z3.h, z2.h
-; CHECK-NEXT:    fcmgt p0.h, p0/z, z2.h, z3.h
-; CHECK-NEXT:    mov p1.b, p2/m, p2.b
-; CHECK-NEXT:    sel p0.b, p0, p0.b, p3.b
-; CHECK-NEXT:    mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmgt p2.h, p0/z, z3.h, z2.h
+; CHECK-NEXT:    fcmgt p3.h, p0/z, z2.h, z3.h
+; CHECK-NEXT:    fcmgt p0.h, p0/z, z1.h, z0.h
+; CHECK-NEXT:    mov p2.b, p3/m, p3.b
+; CHECK-NEXT:    sel p0.b, p0, p0.b, p1.b
+; CHECK-NEXT:    mov z0.h, p2/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z1.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    stp q0, q1, [x2]
 ; CHECK-NEXT:    ret
@@ -1200,8 +1200,8 @@ define void @fcmp_ugt_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    ldp q1, q2, [x0]
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    fcmge p0.h, p0/z, z3.h, z2.h
 ; CHECK-NEXT:    mov z0.h, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmge p0.h, p0/z, z3.h, z2.h
 ; CHECK-NEXT:    mov z1.h, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    eor z1.d, z1.d, z0.d
@@ -1501,8 +1501,8 @@ define void @fcmp_ult_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    ldp q1, q2, [x0]
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z1.h, z0.h
-; CHECK-NEXT:    fcmge p0.h, p0/z, z2.h, z3.h
 ; CHECK-NEXT:    mov z0.h, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmge p0.h, p0/z, z2.h, z3.h
 ; CHECK-NEXT:    mov z1.h, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    eor z1.d, z1.d, z0.d
@@ -1802,8 +1802,8 @@ define void @fcmp_uge_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    ldp q1, q2, [x0]
 ; CHECK-NEXT:    fcmgt p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT:    fcmgt p0.h, p0/z, z3.h, z2.h
 ; CHECK-NEXT:    mov z0.h, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmgt p0.h, p0/z, z3.h, z2.h
 ; CHECK-NEXT:    mov z1.h, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    eor z1.d, z1.d, z0.d
@@ -2103,8 +2103,8 @@ define void @fcmp_ule_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    ldp q1, q2, [x0]
 ; CHECK-NEXT:    fcmgt p1.h, p0/z, z1.h, z0.h
-; CHECK-NEXT:    fcmgt p0.h, p0/z, z2.h, z3.h
 ; CHECK-NEXT:    mov z0.h, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmgt p0.h, p0/z, z2.h, z3.h
 ; CHECK-NEXT:    mov z1.h, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    eor z1.d, z1.d, z0.d
@@ -2404,8 +2404,8 @@ define void @fcmp_ord_v16f16(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    ldp q1, q2, [x0]
 ; CHECK-NEXT:    fcmuo p1.h, p0/z, z1.h, z0.h
-; CHECK-NEXT:    fcmuo p0.h, p0/z, z2.h, z3.h
 ; CHECK-NEXT:    mov z0.h, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmuo p0.h, p0/z, z2.h, z3.h
 ; CHECK-NEXT:    mov z1.h, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.h, p0/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    eor z1.d, z1.d, z0.d
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
index 56149e99b15f8..6951a6cd50ef6 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
@@ -75,13 +75,14 @@ define void @fcvt_v8f16_to_v8f32(<8 x half> %a, ptr %b) {
 ; CHECK-LABEL: fcvt_v8f16_to_v8f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z1.s, z0.h
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    fcvt z1.s, p0/m, z1.h
+; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.h
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    fcvt z1.s, p0/m, z1.h
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvt_v8f16_to_v8f32:
@@ -124,19 +125,21 @@ define void @fcvt_v16f16_to_v16f32(<16 x half> %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z2.s, z1.h
-; CHECK-NEXT:    uunpklo z3.s, z0.h
+; CHECK-NEXT:    movprfx z2, z1
+; CHECK-NEXT:    ext z2.b, z2.b, z1.b, #8
+; CHECK-NEXT:    movprfx z3, z0
+; CHECK-NEXT:    ext z3.b, z3.b, z0.b, #8
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    fcvt z2.s, p0/m, z2.h
-; CHECK-NEXT:    fcvt z3.s, p0/m, z3.h
+; CHECK-NEXT:    uunpklo z2.s, z2.h
+; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    fcvt z1.s, p0/m, z1.h
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z0.h
-; CHECK-NEXT:    stp q3, q0, [x0]
-; CHECK-NEXT:    stp q2, q1, [x0, #32]
+; CHECK-NEXT:    fcvt z2.s, p0/m, z2.h
+; CHECK-NEXT:    fcvt z3.s, p0/m, z3.h
+; CHECK-NEXT:    stp q0, q3, [x0]
+; CHECK-NEXT:    stp q1, q2, [x0, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvt_v16f16_to_v16f32:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
index 94d756a36ab92..66e157c779abf 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-to-int.ll
@@ -262,12 +262,13 @@ define void @fcvtzu_v8f16_v8i32(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    uunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    fcvtzu z1.s, p0/m, z1.h
+; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.h
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    fcvtzu z1.s, p0/m, z1.h
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzu_v8f16_v8i32:
@@ -320,18 +321,20 @@ define void @fcvtzu_v16f16_v16i32(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    uunpklo z2.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    fcvtzu z2.s, p0/m, z2.h
-; CHECK-NEXT:    fcvtzu z3.s, p0/m, z3.h
+; CHECK-NEXT:    uunpklo z2.s, z2.h
+; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    fcvtzu z0.s, p0/m, z0.h
 ; CHECK-NEXT:    fcvtzu z1.s, p0/m, z1.h
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    fcvtzu z2.s, p0/m, z2.h
+; CHECK-NEXT:    fcvtzu z3.s, p0/m, z3.h
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzu_v16f16_v16i32:
@@ -521,27 +524,27 @@ define void @fcvtzu_v8f16_v8i64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov z1.d, z0.d
 ; CHECK-NEXT:    mov z2.h, z0.h[3]
 ; CHECK-NEXT:    mov z3.h, z0.h[2]
 ; CHECK-NEXT:    mov z4.h, z0.h[1]
+; CHECK-NEXT:    movprfx z1, z0
 ; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.h
 ; CHECK-NEXT:    fcvtzu z2.d, p0/m, z2.h
 ; CHECK-NEXT:    fcvtzu z3.d, p0/m, z3.h
-; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.h
 ; CHECK-NEXT:    fcvtzu z4.d, p0/m, z4.h
 ; CHECK-NEXT:    mov z5.h, z1.h[3]
 ; CHECK-NEXT:    mov z6.h, z1.h[2]
 ; CHECK-NEXT:    mov z7.h, z1.h[1]
 ; CHECK-NEXT:    fcvtzu z1.d, p0/m, z1.h
-; CHECK-NEXT:    zip1 z2.d, z3.d, z2.d
-; CHECK-NEXT:    zip1 z0.d, z0.d, z4.d
 ; CHECK-NEXT:    fcvtzu z5.d, p0/m, z5.h
 ; CHECK-NEXT:    fcvtzu z6.d, p0/m, z6.h
 ; CHECK-NEXT:    fcvtzu z7.d, p0/m, z7.h
-; CHECK-NEXT:    stp q0, q2, [x1]
+; CHECK-NEXT:    zip1 z2.d, z3.d, z2.d
+; CHECK-NEXT:    zip1 z0.d, z0.d, z4.d
 ; CHECK-NEXT:    zip1 z3.d, z6.d, z5.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z7.d
+; CHECK-NEXT:    stp q0, q2, [x1]
 ; CHECK-NEXT:    stp q1, q3, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
@@ -597,52 +600,53 @@ define void @fcvtzu_v16f16_v16i64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov z3.h, z1.h[1]
-; CHECK-NEXT:    mov z5.h, z0.h[3]
-; CHECK-NEXT:    mov z6.h, z0.h[2]
-; CHECK-NEXT:    mov z16.d, z0.d
-; CHECK-NEXT:    movprfx z2, z1
-; CHECK-NEXT:    fcvtzu z2.d, p0/m, z1.h
-; CHECK-NEXT:    mov z4.h, z1.h[3]
-; CHECK-NEXT:    mov z7.h, z1.h[2]
+; CHECK-NEXT:    mov z2.h, z1.h[1]
+; CHECK-NEXT:    mov z3.h, z0.h[3]
+; CHECK-NEXT:    mov z4.h, z0.h[2]
+; CHECK-NEXT:    movprfx z5, z1
+; CHECK-NEXT:    ext z5.b, z5.b, z1.b, #8
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    fcvtzu z7.d, p0/m, z1.h
+; CHECK-NEXT:    mov z16.h, z1.h[3]
+; CHECK-NEXT:    mov z1.h, z1.h[2]
 ; CHECK-NEXT:    mov z17.h, z0.h[1]
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z6, z0
+; CHECK-NEXT:    ext z6.b, z6.b, z0.b, #8
+; CHECK-NEXT:    fcvtzu z2.d, p0/m, z2.h
 ; CHECK-NEXT:    fcvtzu z3.d, p0/m, z3.h
-; CHECK-NEXT:    fcvtzu z5.d, p0/m, z5.h
-; CHECK-NEXT:    fcvtzu z6.d, p0/m, z6.h
-; CHECK-NEXT:    ext z16.b, z16.b, z0.b, #8
-; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.h
 ; CHECK-NEXT:    fcvtzu z4.d, p0/m, z4.h
+; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.h
+; CHECK-NEXT:    fcvtzu z16.d, p0/m, z16.h
+; CHECK-NEXT:    mov z18.h, z5.h[3]
 ; CHECK-NEXT:    fcvtzu z17.d, p0/m, z17.h
-; CHECK-NEXT:    fcvtzu z7.d, p0/m, z7.h
-; CHECK-NEXT:    mov z20.h, z1.h[3]
-; CHECK-NEXT:    mov z18.h, z16.h[3]
-; CHECK-NEXT:    mov z19.h, z16.h[2]
-; CHECK-NEXT:    mov z21.h, z16.h[1]
-; CHECK-NEXT:    zip1 z2.d, z2.d, z3.d
-; CHECK-NEXT:    mov z3.h, z1.h[2]
-; CHECK-NEXT:    zip1 z5.d, z6.d, z5.d
-; CHECK-NEXT:    mov z6.h, z1.h[1]
+; CHECK-NEXT:    fcvtzu z1.d, p0/m, z1.h
+; CHECK-NEXT:    mov z19.h, z6.h[3]
+; CHECK-NEXT:    mov z20.h, z6.h[2]
+; CHECK-NEXT:    mov z21.h, z6.h[1]
+; CHECK-NEXT:    fcvtzu z6.d, p0/m, z6.h
+; CHECK-NEXT:    zip1 z2.d, z7.d, z2.d
+; CHECK-NEXT:    mov z7.h, z5.h[2]
+; CHECK-NEXT:    zip1 z3.d, z4.d, z3.d
+; CHECK-NEXT:    mov z4.h, z5.h[1]
+; CHECK-NEXT:    fcvtzu z19.d, p0/m, z19.h
+; CHECK-NEXT:    fcvtzu z5.d, p0/m, z5.h
+; CHECK-NEXT:    fcvtzu z20.d, p0/m, z20.h
 ; CHECK-NEXT:    zip1 z0.d, z0.d, z17.d
-; CHECK-NEXT:    fcvtzu z16.d, p0/m, z16.h
-; CHECK-NEXT:    fcvtzu z18.d, p0/m, z18.h
 ; CHECK-NEXT:    movprfx z17, z21
 ; CHECK-NEXT:    fcvtzu z17.d, p0/m, z21.h
-; CHECK-NEXT:    fcvtzu z19.d, p0/m, z19.h
-; CHECK-NEXT:    zip1 z4.d, z7.d, z4.d
-; CHECK-NEXT:    movprfx z7, z20
-; CHECK-NEXT:    fcvtzu z7.d, p0/m, z20.h
-; CHECK-NEXT:    fcvtzu z3.d, p0/m, z3.h
-; CHECK-NEXT:    fcvtzu z1.d, p0/m, z1.h
-; CHECK-NEXT:    stp q0, q5, [x1, #64]
-; CHECK-NEXT:    fcvtzu z6.d, p0/m, z6.h
-; CHECK-NEXT:    zip1 z0.d, z19.d, z18.d
-; CHECK-NEXT:    zip1 z5.d, z16.d, z17.d
-; CHECK-NEXT:    stp q2, q4, [x1]
-; CHECK-NEXT:    zip1 z2.d, z3.d, z7.d
-; CHECK-NEXT:    zip1 z1.d, z1.d, z6.d
-; CHECK-NEXT:    stp q5, q0, [x1, #96]
-; CHECK-NEXT:    stp q1, q2, [x1, #32]
+; CHECK-NEXT:    zip1 z1.d, z1.d, z16.d
+; CHECK-NEXT:    movprfx z16, z18
+; CHECK-NEXT:    fcvtzu z16.d, p0/m, z18.h
+; CHECK-NEXT:    fcvtzu z7.d, p0/m, z7.h
+; CHECK-NEXT:    fcvtzu z4.d, p0/m, z4.h
+; CHECK-NEXT:    stp q0, q3, [x1, #64]
+; CHECK-NEXT:    zip1 z0.d, z20.d, z19.d
+; CHECK-NEXT:    zip1 z3.d, z6.d, z17.d
+; CHECK-NEXT:    stp q2, q1, [x1]
+; CHECK-NEXT:    zip1 z1.d, z7.d, z16.d
+; CHECK-NEXT:    zip1 z2.d, z5.d, z4.d
+; CHECK-NEXT:    stp q3, q0, [x1, #96]
+; CHECK-NEXT:    stp q2, q1, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzu_v16f16_v16i64:
@@ -1068,12 +1072,13 @@ define void @fcvtzu_v4f32_v4i64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    fcvtzu z1.d, p0/m, z1.s
+; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.s
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    fcvtzu z1.d, p0/m, z1.s
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzu_v4f32_v4i64:
@@ -1106,18 +1111,20 @@ define void @fcvtzu_v8f32_v8i64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    uunpklo z2.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    fcvtzu z2.d, p0/m, z2.s
-; CHECK-NEXT:    fcvtzu z3.d, p0/m, z3.s
+; CHECK-NEXT:    uunpklo z2.d, z2.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z0.s
 ; CHECK-NEXT:    fcvtzu z1.d, p0/m, z1.s
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    fcvtzu z2.d, p0/m, z2.s
+; CHECK-NEXT:    fcvtzu z3.d, p0/m, z3.s
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzu_v8f32_v8i64:
@@ -1957,12 +1964,13 @@ define void @fcvtzs_v8f16_v8i32(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    uunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    fcvtzs z1.s, p0/m, z1.h
+; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.h
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    fcvtzs z1.s, p0/m, z1.h
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzs_v8f16_v8i32:
@@ -2015,18 +2023,20 @@ define void @fcvtzs_v16f16_v16i32(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    uunpklo z2.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    fcvtzs z2.s, p0/m, z2.h
-; CHECK-NEXT:    fcvtzs z3.s, p0/m, z3.h
+; CHECK-NEXT:    uunpklo z2.s, z2.h
+; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z0.h
 ; CHECK-NEXT:    fcvtzs z1.s, p0/m, z1.h
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    fcvtzs z2.s, p0/m, z2.h
+; CHECK-NEXT:    fcvtzs z3.s, p0/m, z3.h
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzs_v16f16_v16i32:
@@ -2217,27 +2227,27 @@ define void @fcvtzs_v8f16_v8i64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov z1.d, z0.d
 ; CHECK-NEXT:    mov z2.h, z0.h[3]
 ; CHECK-NEXT:    mov z3.h, z0.h[2]
 ; CHECK-NEXT:    mov z4.h, z0.h[1]
+; CHECK-NEXT:    movprfx z1, z0
 ; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.h
 ; CHECK-NEXT:    fcvtzs z2.d, p0/m, z2.h
 ; CHECK-NEXT:    fcvtzs z3.d, p0/m, z3.h
-; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.h
 ; CHECK-NEXT:    fcvtzs z4.d, p0/m, z4.h
 ; CHECK-NEXT:    mov z5.h, z1.h[3]
 ; CHECK-NEXT:    mov z6.h, z1.h[2]
 ; CHECK-NEXT:    mov z7.h, z1.h[1]
 ; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.h
-; CHECK-NEXT:    zip1 z2.d, z3.d, z2.d
-; CHECK-NEXT:    zip1 z0.d, z0.d, z4.d
 ; CHECK-NEXT:    fcvtzs z5.d, p0/m, z5.h
 ; CHECK-NEXT:    fcvtzs z6.d, p0/m, z6.h
 ; CHECK-NEXT:    fcvtzs z7.d, p0/m, z7.h
-; CHECK-NEXT:    stp q0, q2, [x1]
+; CHECK-NEXT:    zip1 z2.d, z3.d, z2.d
+; CHECK-NEXT:    zip1 z0.d, z0.d, z4.d
 ; CHECK-NEXT:    zip1 z3.d, z6.d, z5.d
 ; CHECK-NEXT:    zip1 z1.d, z1.d, z7.d
+; CHECK-NEXT:    stp q0, q2, [x1]
 ; CHECK-NEXT:    stp q1, q3, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
@@ -2293,52 +2303,53 @@ define void @fcvtzs_v16f16_v16i64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d
-; CHECK-NEXT:    mov z3.h, z1.h[1]
-; CHECK-NEXT:    mov z5.h, z0.h[3]
-; CHECK-NEXT:    mov z6.h, z0.h[2]
-; CHECK-NEXT:    mov z16.d, z0.d
-; CHECK-NEXT:    movprfx z2, z1
-; CHECK-NEXT:    fcvtzs z2.d, p0/m, z1.h
-; CHECK-NEXT:    mov z4.h, z1.h[3]
-; CHECK-NEXT:    mov z7.h, z1.h[2]
+; CHECK-NEXT:    mov z2.h, z1.h[1]
+; CHECK-NEXT:    mov z3.h, z0.h[3]
+; CHECK-NEXT:    mov z4.h, z0.h[2]
+; CHECK-NEXT:    movprfx z5, z1
+; CHECK-NEXT:    ext z5.b, z5.b, z1.b, #8
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z1.h
+; CHECK-NEXT:    mov z16.h, z1.h[3]
+; CHECK-NEXT:    mov z1.h, z1.h[2]
 ; CHECK-NEXT:    mov z17.h, z0.h[1]
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z6, z0
+; CHECK-NEXT:    ext z6.b, z6.b, z0.b, #8
+; CHECK-NEXT:    fcvtzs z2.d, p0/m, z2.h
 ; CHECK-NEXT:    fcvtzs z3.d, p0/m, z3.h
-; CHECK-NEXT:    fcvtzs z5.d, p0/m, z5.h
-; CHECK-NEXT:    fcvtzs z6.d, p0/m, z6.h
-; CHECK-NEXT:    ext z16.b, z16.b, z0.b, #8
-; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.h
 ; CHECK-NEXT:    fcvtzs z4.d, p0/m, z4.h
+; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.h
+; CHECK-NEXT:    fcvtzs z16.d, p0/m, z16.h
+; CHECK-NEXT:    mov z18.h, z5.h[3]
 ; CHECK-NEXT:    fcvtzs z17.d, p0/m, z17.h
-; CHECK-NEXT:    fcvtzs z7.d, p0/m, z7.h
-; CHECK-NEXT:    mov z20.h, z1.h[3]
-; CHECK-NEXT:    mov z18.h, z16.h[3]
-; CHECK-NEXT:    mov z19.h, z16.h[2]
-; CHECK-NEXT:    mov z21.h, z16.h[1]
-; CHECK-NEXT:    zip1 z2.d, z2.d, z3.d
-; CHECK-NEXT:    mov z3.h, z1.h[2]
-; CHECK-NEXT:    zip1 z5.d, z6.d, z5.d
-; CHECK-NEXT:    mov z6.h, z1.h[1]
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.h
+; CHECK-NEXT:    mov z19.h, z6.h[3]
+; CHECK-NEXT:    mov z20.h, z6.h[2]
+; CHECK-NEXT:    mov z21.h, z6.h[1]
+; CHECK-NEXT:    fcvtzs z6.d, p0/m, z6.h
+; CHECK-NEXT:    zip1 z2.d, z7.d, z2.d
+; CHECK-NEXT:    mov z7.h, z5.h[2]
+; CHECK-NEXT:    zip1 z3.d, z4.d, z3.d
+; CHECK-NEXT:    mov z4.h, z5.h[1]
+; CHECK-NEXT:    fcvtzs z19.d, p0/m, z19.h
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z5.h
+; CHECK-NEXT:    fcvtzs z20.d, p0/m, z20.h
 ; CHECK-NEXT:    zip1 z0.d, z0.d, z17.d
-; CHECK-NEXT:    fcvtzs z16.d, p0/m, z16.h
-; CHECK-NEXT:    fcvtzs z18.d, p0/m, z18.h
 ; CHECK-NEXT:    movprfx z17, z21
 ; CHECK-NEXT:    fcvtzs z17.d, p0/m, z21.h
-; CHECK-NEXT:    fcvtzs z19.d, p0/m, z19.h
-; CHECK-NEXT:    zip1 z4.d, z7.d, z4.d
-; CHECK-NEXT:    movprfx z7, z20
-; CHECK-NEXT:    fcvtzs z7.d, p0/m, z20.h
-; CHECK-NEXT:    fcvtzs z3.d, p0/m, z3.h
-; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.h
-; CHECK-NEXT:    stp q0, q5, [x1, #64]
-; CHECK-NEXT:    fcvtzs z6.d, p0/m, z6.h
-; CHECK-NEXT:    zip1 z0.d, z19.d, z18.d
-; CHECK-NEXT:    zip1 z5.d, z16.d, z17.d
-; CHECK-NEXT:    stp q2, q4, [x1]
-; CHECK-NEXT:    zip1 z2.d, z3.d, z7.d
-; CHECK-NEXT:    zip1 z1.d, z1.d, z6.d
-; CHECK-NEXT:    stp q5, q0, [x1, #96]
-; CHECK-NEXT:    stp q1, q2, [x1, #32]
+; CHECK-NEXT:    zip1 z1.d, z1.d, z16.d
+; CHECK-NEXT:    movprfx z16, z18
+; CHECK-NEXT:    fcvtzs z16.d, p0/m, z18.h
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z7.h
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z4.h
+; CHECK-NEXT:    stp q0, q3, [x1, #64]
+; CHECK-NEXT:    zip1 z0.d, z20.d, z19.d
+; CHECK-NEXT:    zip1 z3.d, z6.d, z17.d
+; CHECK-NEXT:    stp q2, q1, [x1]
+; CHECK-NEXT:    zip1 z1.d, z7.d, z16.d
+; CHECK-NEXT:    zip1 z2.d, z5.d, z4.d
+; CHECK-NEXT:    stp q3, q0, [x1, #96]
+; CHECK-NEXT:    stp q2, q1, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzs_v16f16_v16i64:
@@ -2764,12 +2775,13 @@ define void @fcvtzs_v4f32_v4i64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.s
+; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.s
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.s
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzs_v4f32_v4i64:
@@ -2802,18 +2814,20 @@ define void @fcvtzs_v8f32_v8i64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    uunpklo z2.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    fcvtzs z2.d, p0/m, z2.s
-; CHECK-NEXT:    fcvtzs z3.d, p0/m, z3.s
+; CHECK-NEXT:    uunpklo z2.d, z2.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z0.s
 ; CHECK-NEXT:    fcvtzs z1.d, p0/m, z1.s
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    fcvtzs z2.d, p0/m, z2.s
+; CHECK-NEXT:    fcvtzs z3.d, p0/m, z3.s
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: fcvtzs_v8f32_v8i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
index 75911e5ff1569..c3dcb0f6d7f1f 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-extends.ll
@@ -18,14 +18,15 @@ define void @sext_v8i1_v8i32(<8 x i1> %a, ptr %out) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
-; CHECK-NEXT:    uunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    lsl z1.s, z1.s, #31
+; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    lsl z0.s, z0.s, #31
-; CHECK-NEXT:    asr z1.s, z1.s, #31
+; CHECK-NEXT:    lsl z1.s, z1.s, #31
 ; CHECK-NEXT:    asr z0.s, z0.s, #31
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    asr z1.s, z1.s, #31
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i1_v8i32:
@@ -74,14 +75,15 @@ define void @sext_v4i3_v4i64(<4 x i3> %a, ptr %out) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    lsl z1.d, z1.d, #61
+; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    lsl z0.d, z0.d, #61
-; CHECK-NEXT:    asr z1.d, z1.d, #61
+; CHECK-NEXT:    lsl z1.d, z1.d, #61
 ; CHECK-NEXT:    asr z0.d, z0.d, #61
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    asr z1.d, z1.d, #61
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v4i3_v4i64:
@@ -116,10 +118,11 @@ define void @sext_v16i8_v16i16(<16 x i8> %a, ptr %out) {
 ; CHECK-LABEL: sext_v16i8_v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    sunpklo z1.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.h, z0.b
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    sunpklo z1.h, z1.b
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v16i8_v16i16:
@@ -176,14 +179,16 @@ define void @sext_v32i8_v32i16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
 ; CHECK-NEXT:    add z1.b, z1.b, z1.b
-; CHECK-NEXT:    sunpklo z2.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z3.h, z1.b
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z0.h, z0.b
 ; CHECK-NEXT:    sunpklo z1.h, z1.b
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    sunpklo z2.h, z2.b
+; CHECK-NEXT:    sunpklo z3.h, z3.b
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v32i8_v32i16:
@@ -369,10 +374,11 @@ define void @sext_v8i8_v8i32(<8 x i8> %a, ptr %out) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    sunpklo z0.h, z0.b
-; CHECK-NEXT:    sunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i8_v8i32:
@@ -405,17 +411,20 @@ define void @sext_v16i8_v16i32(<16 x i8> %a, ptr %out) {
 ; CHECK-LABEL: sext_v16i8_v16i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    sunpklo z1.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.h, z0.b
-; CHECK-NEXT:    sunpklo z2.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z1.s, z1.h
-; CHECK-NEXT:    sunpklo z3.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z1.h, z1.b
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    stp q2, q1, [x0]
-; CHECK-NEXT:    stp q3, q0, [x0, #32]
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
+; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    sunpklo z2.s, z2.h
+; CHECK-NEXT:    sunpklo z3.s, z3.h
+; CHECK-NEXT:    stp q0, q2, [x0]
+; CHECK-NEXT:    stp q1, q3, [x0, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v16i8_v16i32:
@@ -465,28 +474,33 @@ define void @sext_v32i8_v32i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
 ; CHECK-NEXT:    add z1.b, z1.b, z1.b
-; CHECK-NEXT:    sunpklo z2.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z0.h, z0.b
 ; CHECK-NEXT:    sunpklo z3.h, z1.b
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z0.h, z0.b
+; CHECK-NEXT:    sunpklo z2.h, z2.b
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z1.h, z1.b
-; CHECK-NEXT:    sunpklo z4.s, z2.h
-; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
-; CHECK-NEXT:    sunpklo z5.s, z3.h
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    sunpklo z6.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z2.s, z2.h
-; CHECK-NEXT:    sunpklo z7.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z3.s, z3.h
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    ext z5.b, z5.b, z3.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
+; CHECK-NEXT:    sunpklo z3.s, z3.h
+; CHECK-NEXT:    sunpklo z4.s, z4.h
+; CHECK-NEXT:    sunpklo z5.s, z5.h
+; CHECK-NEXT:    movprfx z6, z2
+; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
+; CHECK-NEXT:    sunpklo z2.s, z2.h
 ; CHECK-NEXT:    sunpklo z1.s, z1.h
-; CHECK-NEXT:    stp q4, q2, [x1, #64]
-; CHECK-NEXT:    stp q5, q3, [x1]
-; CHECK-NEXT:    stp q6, q0, [x1, #96]
-; CHECK-NEXT:    stp q7, q1, [x1, #32]
+; CHECK-NEXT:    stp q3, q5, [x1]
+; CHECK-NEXT:    sunpklo z3.s, z7.h
+; CHECK-NEXT:    stp q0, q4, [x1, #64]
+; CHECK-NEXT:    sunpklo z0.s, z6.h
+; CHECK-NEXT:    stp q1, q3, [x1, #32]
+; CHECK-NEXT:    stp q2, q0, [x1, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v32i8_v32i32:
@@ -664,12 +678,13 @@ define void @sext_v4i8_v4i64(<4 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    sxtb z1.d, p0/m, z1.d
+; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    sxtb z0.d, p0/m, z0.d
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    sxtb z1.d, p0/m, z1.d
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v4i8_v4i64:
@@ -699,17 +714,20 @@ define void @sext_v8i8_v8i64(<8 x i8> %a, ptr %out) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    sunpklo z0.h, z0.b
-; CHECK-NEXT:    sunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    sunpklo z2.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z3.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z1.d, z1.s
+; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q2, q1, [x0]
-; CHECK-NEXT:    stp q3, q0, [x0, #32]
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
+; CHECK-NEXT:    sunpklo z2.d, z2.s
+; CHECK-NEXT:    sunpklo z1.d, z1.s
+; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    stp q0, q2, [x0]
+; CHECK-NEXT:    stp q1, q3, [x0, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i8_v8i64:
@@ -744,31 +762,37 @@ define void @sext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
 ; CHECK-LABEL: sext_v16i8_v16i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    sunpklo z1.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.h, z0.b
-; CHECK-NEXT:    sunpklo z2.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z1.s, z1.h
-; CHECK-NEXT:    sunpklo z3.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z4.d, z2.s
-; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
+; CHECK-NEXT:    sunpklo z1.h, z1.b
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    sunpklo z6.d, z1.s
+; CHECK-NEXT:    sunpklo z3.s, z1.h
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z5.d, z3.s
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
+; CHECK-NEXT:    sunpklo z2.s, z2.h
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z0.d, z0.s
+; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    ext z5.b, z5.b, z3.b, #8
+; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    sunpklo z4.d, z4.s
+; CHECK-NEXT:    movprfx z6, z2
+; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
 ; CHECK-NEXT:    sunpklo z2.d, z2.s
+; CHECK-NEXT:    sunpklo z5.d, z5.s
+; CHECK-NEXT:    sunpklo z6.d, z6.s
+; CHECK-NEXT:    stp q0, q4, [x0]
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
-; CHECK-NEXT:    sunpklo z7.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z3.d, z3.s
-; CHECK-NEXT:    stp q4, q2, [x0]
+; CHECK-NEXT:    stp q3, q5, [x0, #64]
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q6, q1, [x0, #32]
-; CHECK-NEXT:    stp q5, q3, [x0, #64]
-; CHECK-NEXT:    stp q7, q0, [x0, #96]
+; CHECK-NEXT:    stp q2, q6, [x0, #32]
+; CHECK-NEXT:    stp q1, q0, [x0, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v16i8_v16i64:
@@ -819,64 +843,69 @@ define void @sext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
 define void @sext_v32i8_v32i64(ptr %in, ptr %out) {
 ; CHECK-LABEL: sext_v32i8_v32i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
 ; CHECK-NEXT:    add z1.b, z1.b, z1.b
-; CHECK-NEXT:    mov z2.d, z0.d
-; CHECK-NEXT:    sunpklo z3.h, z1.b
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.h, z0.b
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z1.h, z1.b
-; CHECK-NEXT:    sunpklo z4.s, z3.h
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
+; CHECK-NEXT:    sunpklo z3.h, z3.b
 ; CHECK-NEXT:    sunpklo z2.h, z2.b
-; CHECK-NEXT:    sunpklo z5.s, z0.h
-; CHECK-NEXT:    mov z7.d, z1.d
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z0.s, z0.h
+; CHECK-NEXT:    sunpklo z5.s, z1.h
+; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z7, z3
+; CHECK-NEXT:    ext z7.b, z7.b, z3.b, #8
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
-; CHECK-NEXT:    sunpklo z16.d, z4.s
-; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z4.s, z4.h
+; CHECK-NEXT:    movprfx z16, z0
+; CHECK-NEXT:    ext z16.b, z16.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z0.d, z0.s
+; CHECK-NEXT:    sunpklo z1.s, z1.h
 ; CHECK-NEXT:    sunpklo z6.s, z2.h
 ; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
-; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
-; CHECK-NEXT:    mov z17.d, z5.d
-; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z17, z5
+; CHECK-NEXT:    ext z17.b, z17.b, z5.b, #8
+; CHECK-NEXT:    sunpklo z7.s, z7.h
 ; CHECK-NEXT:    sunpklo z5.d, z5.s
-; CHECK-NEXT:    sunpklo z4.d, z4.s
-; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    sunpklo z19.d, z3.s
+; CHECK-NEXT:    sunpklo z16.d, z16.s
+; CHECK-NEXT:    movprfx z20, z3
+; CHECK-NEXT:    ext z20.b, z20.b, z3.b, #8
+; CHECK-NEXT:    sunpklo z19.d, z4.s
 ; CHECK-NEXT:    sunpklo z2.s, z2.h
-; CHECK-NEXT:    sunpklo z7.s, z7.h
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    ext z17.b, z17.b, z17.b, #8
+; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
+; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    sunpklo z17.d, z17.s
 ; CHECK-NEXT:    sunpklo z18.d, z6.s
 ; CHECK-NEXT:    ext z6.b, z6.b, z6.b, #8
-; CHECK-NEXT:    sunpklo z20.d, z1.s
+; CHECK-NEXT:    str q5, [x1]
+; CHECK-NEXT:    stp q0, q16, [x1, #128]
+; CHECK-NEXT:    sunpklo z0.d, z1.s
+; CHECK-NEXT:    sunpklo z16.d, z20.s
+; CHECK-NEXT:    sunpklo z4.d, z4.s
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    stp q16, q4, [x1, #128]
-; CHECK-NEXT:    sunpklo z3.d, z3.s
-; CHECK-NEXT:    sunpklo z16.d, z0.s
-; CHECK-NEXT:    sunpklo z17.d, z17.s
-; CHECK-NEXT:    mov z4.d, z7.d
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z6.d, z6.s
+; CHECK-NEXT:    stp q17, q0, [x1, #16]
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    ext z0.b, z0.b, z2.b, #8
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
-; CHECK-NEXT:    ext z4.b, z4.b, z7.b, #8
-; CHECK-NEXT:    stp q19, q3, [x1, #160]
-; CHECK-NEXT:    sunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q5, q17, [x1]
-; CHECK-NEXT:    sunpklo z5.d, z6.s
-; CHECK-NEXT:    mov z6.d, z2.d
-; CHECK-NEXT:    stp q20, q1, [x1, #192]
-; CHECK-NEXT:    sunpklo z7.d, z7.s
-; CHECK-NEXT:    sunpklo z1.d, z4.s
-; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
+; CHECK-NEXT:    stp q3, q16, [x1, #64]
+; CHECK-NEXT:    movprfx z3, z7
+; CHECK-NEXT:    ext z3.b, z3.b, z7.b, #8
 ; CHECK-NEXT:    sunpklo z2.d, z2.s
-; CHECK-NEXT:    stp q16, q0, [x1, #32]
-; CHECK-NEXT:    stp q18, q5, [x1, #64]
-; CHECK-NEXT:    sunpklo z3.d, z6.s
-; CHECK-NEXT:    stp q7, q1, [x1, #224]
-; CHECK-NEXT:    stp q2, q3, [x1, #96]
+; CHECK-NEXT:    stp q19, q4, [x1, #160]
+; CHECK-NEXT:    sunpklo z4.d, z7.s
+; CHECK-NEXT:    sunpklo z0.d, z0.s
+; CHECK-NEXT:    stp q18, q6, [x1, #192]
+; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    str q1, [x1, #48]
+; CHECK-NEXT:    stp q2, q0, [x1, #224]
+; CHECK-NEXT:    stp q4, q3, [x1, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v32i8_v32i64:
@@ -1057,10 +1086,11 @@ define void @sext_v8i16_v8i32(<8 x i16> %a, ptr %out) {
 ; CHECK-LABEL: sext_v8i16_v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    sunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i16_v8i32:
@@ -1096,14 +1126,16 @@ define void @sext_v16i16_v16i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.h, z0.h, z0.h
 ; CHECK-NEXT:    add z1.h, z1.h, z1.h
-; CHECK-NEXT:    sunpklo z2.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z3.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
 ; CHECK-NEXT:    sunpklo z1.s, z1.h
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    sunpklo z2.s, z2.h
+; CHECK-NEXT:    sunpklo z3.s, z3.h
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v16i16_v16i32:
@@ -1189,10 +1221,11 @@ define void @sext_v4i16_v4i64(<4 x i16> %a, ptr %out) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    sunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    sunpklo z1.d, z1.s
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v4i16_v4i64:
@@ -1219,17 +1252,20 @@ define void @sext_v8i16_v8i64(<8 x i16> %a, ptr %out) {
 ; CHECK-LABEL: sext_v8i16_v8i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    sunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    sunpklo z2.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z1.d, z1.s
-; CHECK-NEXT:    sunpklo z3.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q2, q1, [x0]
-; CHECK-NEXT:    stp q3, q0, [x0, #32]
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
+; CHECK-NEXT:    sunpklo z1.d, z1.s
+; CHECK-NEXT:    sunpklo z2.d, z2.s
+; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    stp q0, q2, [x0]
+; CHECK-NEXT:    stp q1, q3, [x0, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i16_v8i64:
@@ -1267,28 +1303,33 @@ define void @sext_v16i16_v16i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.h, z0.h, z0.h
 ; CHECK-NEXT:    add z1.h, z1.h, z1.h
-; CHECK-NEXT:    sunpklo z2.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z0.s, z0.h
 ; CHECK-NEXT:    sunpklo z3.s, z1.h
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z0.s, z0.h
+; CHECK-NEXT:    sunpklo z2.s, z2.h
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z1.s, z1.h
-; CHECK-NEXT:    sunpklo z4.d, z2.s
-; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
-; CHECK-NEXT:    sunpklo z5.d, z3.s
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    sunpklo z6.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z2.d, z2.s
-; CHECK-NEXT:    sunpklo z7.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    ext z5.b, z5.b, z3.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
+; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    sunpklo z4.d, z4.s
+; CHECK-NEXT:    sunpklo z5.d, z5.s
+; CHECK-NEXT:    movprfx z6, z2
+; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
+; CHECK-NEXT:    sunpklo z2.d, z2.s
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
-; CHECK-NEXT:    stp q4, q2, [x1, #64]
-; CHECK-NEXT:    stp q5, q3, [x1]
-; CHECK-NEXT:    stp q6, q0, [x1, #96]
-; CHECK-NEXT:    stp q7, q1, [x1, #32]
+; CHECK-NEXT:    stp q3, q5, [x1]
+; CHECK-NEXT:    sunpklo z3.d, z7.s
+; CHECK-NEXT:    stp q0, q4, [x1, #64]
+; CHECK-NEXT:    sunpklo z0.d, z6.s
+; CHECK-NEXT:    stp q1, q3, [x1, #32]
+; CHECK-NEXT:    stp q2, q0, [x1, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v16i16_v16i64:
@@ -1378,10 +1419,11 @@ define void @sext_v4i32_v4i64(<4 x i32> %a, ptr %out) {
 ; CHECK-LABEL: sext_v4i32_v4i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    sunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    sunpklo z1.d, z1.s
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v4i32_v4i64:
@@ -1409,14 +1451,16 @@ define void @sext_v8i32_v8i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.s, z0.s, z0.s
 ; CHECK-NEXT:    add z1.s, z1.s, z1.s
-; CHECK-NEXT:    sunpklo z2.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z3.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    sunpklo z2.d, z2.s
+; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: sext_v8i32_v8i64:
@@ -1469,10 +1513,11 @@ define void @zext_v16i8_v16i16(<16 x i8> %a, ptr %out) {
 ; CHECK-LABEL: zext_v16i8_v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z1.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    uunpklo z1.h, z1.b
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v16i8_v16i16:
@@ -1529,14 +1574,16 @@ define void @zext_v32i8_v32i16(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
 ; CHECK-NEXT:    add z1.b, z1.b, z1.b
-; CHECK-NEXT:    uunpklo z2.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.h, z1.b
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
 ; CHECK-NEXT:    uunpklo z1.h, z1.b
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    uunpklo z2.h, z2.b
+; CHECK-NEXT:    uunpklo z3.h, z3.b
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v32i8_v32i16:
@@ -1722,10 +1769,11 @@ define void @zext_v8i8_v8i32(<8 x i8> %a, ptr %out) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
-; CHECK-NEXT:    uunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v8i8_v8i32:
@@ -1758,17 +1806,20 @@ define void @zext_v16i8_v16i32(<16 x i8> %a, ptr %out) {
 ; CHECK-LABEL: zext_v16i8_v16i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z1.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
-; CHECK-NEXT:    uunpklo z2.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    uunpklo z3.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z1.h, z1.b
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    stp q2, q1, [x0]
-; CHECK-NEXT:    stp q3, q0, [x0, #32]
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
+; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    uunpklo z2.s, z2.h
+; CHECK-NEXT:    uunpklo z3.s, z3.h
+; CHECK-NEXT:    stp q0, q2, [x0]
+; CHECK-NEXT:    stp q1, q3, [x0, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v16i8_v16i32:
@@ -1818,28 +1869,33 @@ define void @zext_v32i8_v32i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
 ; CHECK-NEXT:    add z1.b, z1.b, z1.b
-; CHECK-NEXT:    uunpklo z2.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z0.h, z0.b
 ; CHECK-NEXT:    uunpklo z3.h, z1.b
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z0.h, z0.b
+; CHECK-NEXT:    uunpklo z2.h, z2.b
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z1.h, z1.b
-; CHECK-NEXT:    uunpklo z4.s, z2.h
-; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
-; CHECK-NEXT:    uunpklo z5.s, z3.h
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    uunpklo z6.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z2.s, z2.h
-; CHECK-NEXT:    uunpklo z7.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z3.s, z3.h
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    ext z5.b, z5.b, z3.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    uunpklo z3.s, z3.h
+; CHECK-NEXT:    uunpklo z4.s, z4.h
+; CHECK-NEXT:    uunpklo z5.s, z5.h
+; CHECK-NEXT:    movprfx z6, z2
+; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
+; CHECK-NEXT:    uunpklo z2.s, z2.h
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    stp q4, q2, [x1, #64]
-; CHECK-NEXT:    stp q5, q3, [x1]
-; CHECK-NEXT:    stp q6, q0, [x1, #96]
-; CHECK-NEXT:    stp q7, q1, [x1, #32]
+; CHECK-NEXT:    stp q3, q5, [x1]
+; CHECK-NEXT:    uunpklo z3.s, z7.h
+; CHECK-NEXT:    stp q0, q4, [x1, #64]
+; CHECK-NEXT:    uunpklo z0.s, z6.h
+; CHECK-NEXT:    stp q1, q3, [x1, #32]
+; CHECK-NEXT:    stp q2, q0, [x1, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v32i8_v32i32:
@@ -2017,10 +2073,11 @@ define void @zext_v4i8_v4i64(<4 x i8> %a, ptr %out) {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    and z0.h, z0.h, #0xff
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    uunpklo z1.d, z1.s
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v4i8_v4i64:
@@ -2050,17 +2107,20 @@ define void @zext_v8i8_v8i64(<8 x i8> %a, ptr %out) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
-; CHECK-NEXT:    uunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z2.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z1.d, z1.s
+; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q2, q1, [x0]
-; CHECK-NEXT:    stp q3, q0, [x0, #32]
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
+; CHECK-NEXT:    uunpklo z2.d, z2.s
+; CHECK-NEXT:    uunpklo z1.d, z1.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    stp q0, q2, [x0]
+; CHECK-NEXT:    stp q1, q3, [x0, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v8i8_v8i64:
@@ -2099,31 +2159,37 @@ define void @zext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
 ; CHECK-LABEL: zext_v16i8_v16i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z1.h, z0.b
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
-; CHECK-NEXT:    uunpklo z2.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    uunpklo z3.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z4.d, z2.s
-; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
+; CHECK-NEXT:    uunpklo z1.h, z1.b
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z6.d, z1.s
+; CHECK-NEXT:    uunpklo z3.s, z1.h
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z5.d, z3.s
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
+; CHECK-NEXT:    uunpklo z2.s, z2.h
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    ext z5.b, z5.b, z3.b, #8
+; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    uunpklo z4.d, z4.s
+; CHECK-NEXT:    movprfx z6, z2
+; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
 ; CHECK-NEXT:    uunpklo z2.d, z2.s
+; CHECK-NEXT:    uunpklo z5.d, z5.s
+; CHECK-NEXT:    uunpklo z6.d, z6.s
+; CHECK-NEXT:    stp q0, q4, [x0]
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    ext z0.b, z0.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z7.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.d, z3.s
-; CHECK-NEXT:    stp q4, q2, [x0]
+; CHECK-NEXT:    stp q3, q5, [x0, #64]
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q6, q1, [x0, #32]
-; CHECK-NEXT:    stp q5, q3, [x0, #64]
-; CHECK-NEXT:    stp q7, q0, [x0, #96]
+; CHECK-NEXT:    stp q2, q6, [x0, #32]
+; CHECK-NEXT:    stp q1, q0, [x0, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v16i8_v16i64:
@@ -2182,64 +2248,69 @@ define void @zext_v16i8_v16i64(<16 x i8> %a, ptr %out) {
 define void @zext_v32i8_v32i64(ptr %in, ptr %out) {
 ; CHECK-LABEL: zext_v32i8_v32i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.b, z0.b, z0.b
 ; CHECK-NEXT:    add z1.b, z1.b, z1.b
-; CHECK-NEXT:    mov z2.d, z0.d
-; CHECK-NEXT:    uunpklo z3.h, z1.b
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z1.h, z1.b
-; CHECK-NEXT:    uunpklo z4.s, z3.h
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
+; CHECK-NEXT:    uunpklo z3.h, z3.b
 ; CHECK-NEXT:    uunpklo z2.h, z2.b
-; CHECK-NEXT:    uunpklo z5.s, z0.h
-; CHECK-NEXT:    mov z7.d, z1.d
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    uunpklo z5.s, z1.h
+; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z7, z3
+; CHECK-NEXT:    ext z7.b, z7.b, z3.b, #8
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
-; CHECK-NEXT:    uunpklo z16.d, z4.s
-; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z4.s, z4.h
+; CHECK-NEXT:    movprfx z16, z0
+; CHECK-NEXT:    ext z16.b, z16.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    uunpklo z6.s, z2.h
 ; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
-; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
-; CHECK-NEXT:    mov z17.d, z5.d
-; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z17, z5
+; CHECK-NEXT:    ext z17.b, z17.b, z5.b, #8
+; CHECK-NEXT:    uunpklo z7.s, z7.h
 ; CHECK-NEXT:    uunpklo z5.d, z5.s
-; CHECK-NEXT:    uunpklo z4.d, z4.s
-; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z19.d, z3.s
+; CHECK-NEXT:    uunpklo z16.d, z16.s
+; CHECK-NEXT:    movprfx z20, z3
+; CHECK-NEXT:    ext z20.b, z20.b, z3.b, #8
+; CHECK-NEXT:    uunpklo z19.d, z4.s
 ; CHECK-NEXT:    uunpklo z2.s, z2.h
-; CHECK-NEXT:    uunpklo z7.s, z7.h
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    ext z17.b, z17.b, z17.b, #8
+; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
+; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    uunpklo z17.d, z17.s
 ; CHECK-NEXT:    uunpklo z18.d, z6.s
 ; CHECK-NEXT:    ext z6.b, z6.b, z6.b, #8
-; CHECK-NEXT:    uunpklo z20.d, z1.s
+; CHECK-NEXT:    str q5, [x1]
+; CHECK-NEXT:    stp q0, q16, [x1, #128]
+; CHECK-NEXT:    uunpklo z0.d, z1.s
+; CHECK-NEXT:    uunpklo z16.d, z20.s
+; CHECK-NEXT:    uunpklo z4.d, z4.s
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    stp q16, q4, [x1, #128]
-; CHECK-NEXT:    uunpklo z3.d, z3.s
-; CHECK-NEXT:    uunpklo z16.d, z0.s
-; CHECK-NEXT:    uunpklo z17.d, z17.s
-; CHECK-NEXT:    mov z4.d, z7.d
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z6.d, z6.s
+; CHECK-NEXT:    stp q17, q0, [x1, #16]
+; CHECK-NEXT:    movprfx z0, z2
+; CHECK-NEXT:    ext z0.b, z0.b, z2.b, #8
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    ext z4.b, z4.b, z7.b, #8
-; CHECK-NEXT:    stp q19, q3, [x1, #160]
-; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q5, q17, [x1]
-; CHECK-NEXT:    uunpklo z5.d, z6.s
-; CHECK-NEXT:    mov z6.d, z2.d
-; CHECK-NEXT:    stp q20, q1, [x1, #192]
-; CHECK-NEXT:    uunpklo z7.d, z7.s
-; CHECK-NEXT:    uunpklo z1.d, z4.s
-; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
+; CHECK-NEXT:    stp q3, q16, [x1, #64]
+; CHECK-NEXT:    movprfx z3, z7
+; CHECK-NEXT:    ext z3.b, z3.b, z7.b, #8
 ; CHECK-NEXT:    uunpklo z2.d, z2.s
-; CHECK-NEXT:    stp q16, q0, [x1, #32]
-; CHECK-NEXT:    stp q18, q5, [x1, #64]
-; CHECK-NEXT:    uunpklo z3.d, z6.s
-; CHECK-NEXT:    stp q7, q1, [x1, #224]
-; CHECK-NEXT:    stp q2, q3, [x1, #96]
+; CHECK-NEXT:    stp q19, q4, [x1, #160]
+; CHECK-NEXT:    uunpklo z4.d, z7.s
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    stp q18, q6, [x1, #192]
+; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    str q1, [x1, #48]
+; CHECK-NEXT:    stp q2, q0, [x1, #224]
+; CHECK-NEXT:    stp q4, q3, [x1, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v32i8_v32i64:
@@ -2443,10 +2514,11 @@ define void @zext_v8i16_v8i32(<8 x i16> %a, ptr %out) {
 ; CHECK-LABEL: zext_v8i16_v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v8i16_v8i32:
@@ -2482,14 +2554,16 @@ define void @zext_v16i16_v16i32(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.h, z0.h, z0.h
 ; CHECK-NEXT:    add z1.h, z1.h, z1.h
-; CHECK-NEXT:    uunpklo z2.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    uunpklo z2.s, z2.h
+; CHECK-NEXT:    uunpklo z3.s, z3.h
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v16i16_v16i32:
@@ -2575,10 +2649,11 @@ define void @zext_v4i16_v4i64(<4 x i16> %a, ptr %out) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    uunpklo z1.d, z1.s
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v4i16_v4i64:
@@ -2607,17 +2682,20 @@ define void @zext_v8i16_v8i64(<8 x i16> %a, ptr %out) {
 ; CHECK-LABEL: zext_v8i16_v8i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z2.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q2, q1, [x0]
-; CHECK-NEXT:    stp q3, q0, [x0, #32]
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
+; CHECK-NEXT:    uunpklo z1.d, z1.s
+; CHECK-NEXT:    uunpklo z2.d, z2.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    stp q0, q2, [x0]
+; CHECK-NEXT:    stp q1, q3, [x0, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v8i16_v8i64:
@@ -2659,28 +2737,33 @@ define void @zext_v16i16_v16i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.h, z0.h, z0.h
 ; CHECK-NEXT:    add z1.h, z1.h, z1.h
-; CHECK-NEXT:    uunpklo z2.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z0.s, z0.h
 ; CHECK-NEXT:    uunpklo z3.s, z1.h
 ; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    uunpklo z2.s, z2.h
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    uunpklo z4.d, z2.s
-; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
-; CHECK-NEXT:    uunpklo z5.d, z3.s
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    uunpklo z6.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z2.d, z2.s
-; CHECK-NEXT:    uunpklo z7.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    ext z5.b, z5.b, z3.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    uunpklo z4.d, z4.s
+; CHECK-NEXT:    uunpklo z5.d, z5.s
+; CHECK-NEXT:    movprfx z6, z2
+; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
+; CHECK-NEXT:    uunpklo z2.d, z2.s
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    stp q4, q2, [x1, #64]
-; CHECK-NEXT:    stp q5, q3, [x1]
-; CHECK-NEXT:    stp q6, q0, [x1, #96]
-; CHECK-NEXT:    stp q7, q1, [x1, #32]
+; CHECK-NEXT:    stp q3, q5, [x1]
+; CHECK-NEXT:    uunpklo z3.d, z7.s
+; CHECK-NEXT:    stp q0, q4, [x1, #64]
+; CHECK-NEXT:    uunpklo z0.d, z6.s
+; CHECK-NEXT:    stp q1, q3, [x1, #32]
+; CHECK-NEXT:    stp q2, q0, [x1, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v16i16_v16i64:
@@ -2780,10 +2863,11 @@ define void @zext_v4i32_v4i64(<4 x i32> %a, ptr %out) {
 ; CHECK-LABEL: zext_v4i32_v4i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    stp q1, q0, [x0]
+; CHECK-NEXT:    uunpklo z1.d, z1.s
+; CHECK-NEXT:    stp q0, q1, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v4i32_v4i64:
@@ -2813,14 +2897,16 @@ define void @zext_v8i32_v8i64(ptr %in, ptr %out) {
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    add z0.s, z0.s, z0.s
 ; CHECK-NEXT:    add z1.s, z1.s, z1.s
-; CHECK-NEXT:    uunpklo z2.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    uunpklo z2.d, z2.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: zext_v8i32_v8i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
index 9497ec88e57b4..372f6a06bf64b 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
@@ -151,14 +151,14 @@ define <16 x i8> @srem_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    sunpklo z2.s, z2.h
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    sdivr z4.s, p0/m, z4.s, z5.s
-; CHECK-NEXT:    mov z5.d, z0.d
+; CHECK-NEXT:    movprfx z5, z0
 ; CHECK-NEXT:    ext z5.b, z5.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z5.h, z5.b
 ; CHECK-NEXT:    sunpklo z7.s, z5.h
 ; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #8
-; CHECK-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
 ; CHECK-NEXT:    sunpklo z5.s, z5.h
+; CHECK-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
+; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z3.h, z3.b
 ; CHECK-NEXT:    sunpklo z6.s, z3.h
@@ -279,35 +279,35 @@ define void @srem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    ldr q1, [x1, #16]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    sunpklo z3.h, z1.b
-; CHECK-NEXT:    sunpklo z4.h, z0.b
-; CHECK-NEXT:    sunpklo z2.s, z3.h
-; CHECK-NEXT:    sunpklo z5.s, z4.h
+; CHECK-NEXT:    sunpklo z2.h, z1.b
+; CHECK-NEXT:    sunpklo z3.h, z0.b
+; CHECK-NEXT:    sunpklo z4.s, z2.h
+; CHECK-NEXT:    sunpklo z5.s, z3.h
+; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
+; CHECK-NEXT:    sunpklo z2.s, z2.h
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
-; CHECK-NEXT:    sunpklo z4.s, z4.h
-; CHECK-NEXT:    sdivr z2.s, p0/m, z2.s, z5.s
-; CHECK-NEXT:    movprfx z5, z4
-; CHECK-NEXT:    sdiv z5.s, p0/m, z5.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
-; CHECK-NEXT:    mov z4.d, z0.d
-; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
-; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z7.h, z3.b
-; CHECK-NEXT:    sunpklo z16.h, z4.b
-; CHECK-NEXT:    sunpklo z3.s, z7.h
-; CHECK-NEXT:    sunpklo z4.s, z16.h
+; CHECK-NEXT:    sdivr z4.s, p0/m, z4.s, z5.s
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    sdiv z5.s, p0/m, z5.s, z2.s
+; CHECK-NEXT:    movprfx z2, z1
+; CHECK-NEXT:    ext z2.b, z2.b, z1.b, #8
+; CHECK-NEXT:    movprfx z3, z0
+; CHECK-NEXT:    ext z3.b, z3.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z7.h, z2.b
+; CHECK-NEXT:    sunpklo z16.h, z3.b
+; CHECK-NEXT:    sunpklo z2.s, z7.h
+; CHECK-NEXT:    sunpklo z3.s, z16.h
 ; CHECK-NEXT:    ext z7.b, z7.b, z7.b, #8
 ; CHECK-NEXT:    ext z16.b, z16.b, z16.b, #8
 ; CHECK-NEXT:    sunpklo z7.s, z7.h
-; CHECK-NEXT:    movprfx z6, z4
-; CHECK-NEXT:    sdiv z6.s, p0/m, z6.s, z3.s
-; CHECK-NEXT:    ldr q3, [x0]
-; CHECK-NEXT:    ldr q4, [x1]
+; CHECK-NEXT:    movprfx z6, z3
+; CHECK-NEXT:    sdiv z6.s, p0/m, z6.s, z2.s
+; CHECK-NEXT:    ldr q2, [x0]
+; CHECK-NEXT:    ldr q3, [x1]
 ; CHECK-NEXT:    sunpklo z16.s, z16.h
-; CHECK-NEXT:    sunpklo z17.h, z4.b
-; CHECK-NEXT:    sunpklo z18.h, z3.b
+; CHECK-NEXT:    sunpklo z17.h, z3.b
+; CHECK-NEXT:    sunpklo z18.h, z2.b
 ; CHECK-NEXT:    sdivr z7.s, p0/m, z7.s, z16.s
 ; CHECK-NEXT:    sunpklo z19.s, z17.h
 ; CHECK-NEXT:    sunpklo z20.s, z18.h
@@ -316,44 +316,44 @@ define void @srem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sunpklo z17.s, z17.h
 ; CHECK-NEXT:    sunpklo z18.s, z18.h
 ; CHECK-NEXT:    sdivr z19.s, p0/m, z19.s, z20.s
-; CHECK-NEXT:    mov z20.d, z3.d
-; CHECK-NEXT:    ext z20.b, z20.b, z3.b, #8
+; CHECK-NEXT:    movprfx z20, z2
+; CHECK-NEXT:    ext z20.b, z20.b, z2.b, #8
 ; CHECK-NEXT:    sunpklo z20.h, z20.b
 ; CHECK-NEXT:    sunpklo z22.s, z20.h
 ; CHECK-NEXT:    ext z20.b, z20.b, z20.b, #8
-; CHECK-NEXT:    sdivr z17.s, p0/m, z17.s, z18.s
-; CHECK-NEXT:    mov z18.d, z4.d
 ; CHECK-NEXT:    sunpklo z20.s, z20.h
-; CHECK-NEXT:    ext z18.b, z18.b, z4.b, #8
+; CHECK-NEXT:    sdivr z17.s, p0/m, z17.s, z18.s
+; CHECK-NEXT:    movprfx z18, z3
+; CHECK-NEXT:    ext z18.b, z18.b, z3.b, #8
 ; CHECK-NEXT:    sunpklo z18.h, z18.b
 ; CHECK-NEXT:    sunpklo z21.s, z18.h
 ; CHECK-NEXT:    ext z18.b, z18.b, z18.b, #8
 ; CHECK-NEXT:    sunpklo z18.s, z18.h
 ; CHECK-NEXT:    sdivr z21.s, p0/m, z21.s, z22.s
-; CHECK-NEXT:    uzp1 z22.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z22.h, z4.h, z4.h
 ; CHECK-NEXT:    uzp1 z23.h, z5.h, z5.h
-; CHECK-NEXT:    uzp1 z5.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z6.h, z7.h, z7.h
 ; CHECK-NEXT:    sdivr z18.s, p0/m, z18.s, z20.s
 ; CHECK-NEXT:    uzp1 z19.h, z19.h, z19.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z20.h, z17.h, z17.h
+; CHECK-NEXT:    uzp1 z16.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z17.h, z7.h, z7.h
 ; CHECK-NEXT:    splice z7.h, p0, { z22.h, z23.h }
-; CHECK-NEXT:    splice z5.h, p0, { z5.h, z6.h }
-; CHECK-NEXT:    uzp1 z16.h, z21.h, z21.h
-; CHECK-NEXT:    splice z2.h, p0, { z19.h, z20.h }
+; CHECK-NEXT:    uzp1 z4.h, z21.h, z21.h
+; CHECK-NEXT:    splice z6.h, p0, { z19.h, z20.h }
+; CHECK-NEXT:    uzp1 z5.h, z18.h, z18.h
+; CHECK-NEXT:    splice z4.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    splice z5.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z16.b, z6.b, z6.b
 ; CHECK-NEXT:    uzp1 z6.b, z7.b, z7.b
-; CHECK-NEXT:    uzp1 z7.b, z5.b, z5.b
-; CHECK-NEXT:    uzp1 z17.h, z18.h, z18.h
-; CHECK-NEXT:    splice z16.h, p0, { z16.h, z17.h }
-; CHECK-NEXT:    uzp1 z17.b, z2.b, z2.b
 ; CHECK-NEXT:    ptrue p0.b, vl8
+; CHECK-NEXT:    uzp1 z17.b, z4.b, z4.b
+; CHECK-NEXT:    uzp1 z7.b, z5.b, z5.b
+; CHECK-NEXT:    splice z4.b, p0, { z16.b, z17.b }
 ; CHECK-NEXT:    splice z5.b, p0, { z6.b, z7.b }
-; CHECK-NEXT:    uzp1 z18.b, z16.b, z16.b
-; CHECK-NEXT:    splice z2.b, p0, { z17.b, z18.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
+; CHECK-NEXT:    mls z2.b, p0/m, z4.b, z3.b
 ; CHECK-NEXT:    mls z0.b, p0/m, z5.b, z1.b
-; CHECK-NEXT:    msb z2.b, p0/m, z4.b, z3.b
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
 ;
@@ -590,13 +590,13 @@ define <8 x i16> @srem_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    sunpklo z2.s, z1.h
 ; CHECK-NEXT:    sunpklo z3.s, z0.h
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    mov z4.d, z0.d
+; CHECK-NEXT:    sunpklo z4.s, z4.h
 ; CHECK-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
-; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z4.s, z4.h
 ; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    sdivr z3.s, p0/m, z3.s, z4.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
@@ -668,21 +668,21 @@ define void @srem_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sunpklo z2.s, z1.h
 ; CHECK-NEXT:    sunpklo z3.s, z0.h
 ; CHECK-NEXT:    sunpklo z5.s, z4.h
-; CHECK-NEXT:    mov z16.d, z0.d
+; CHECK-NEXT:    movprfx z16, z0
+; CHECK-NEXT:    ext z16.b, z16.b, z0.b, #8
 ; CHECK-NEXT:    sdivr z2.s, p0/m, z2.s, z3.s
 ; CHECK-NEXT:    ldr q3, [x0]
-; CHECK-NEXT:    ext z16.b, z16.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z6.s, z3.h
-; CHECK-NEXT:    mov z7.d, z3.d
 ; CHECK-NEXT:    sunpklo z16.s, z16.h
+; CHECK-NEXT:    sunpklo z6.s, z3.h
+; CHECK-NEXT:    movprfx z7, z3
 ; CHECK-NEXT:    ext z7.b, z7.b, z3.b, #8
 ; CHECK-NEXT:    sunpklo z7.s, z7.h
 ; CHECK-NEXT:    sdivr z5.s, p0/m, z5.s, z6.s
-; CHECK-NEXT:    mov z6.d, z4.d
+; CHECK-NEXT:    movprfx z6, z4
 ; CHECK-NEXT:    ext z6.b, z6.b, z4.b, #8
 ; CHECK-NEXT:    sunpklo z6.s, z6.h
 ; CHECK-NEXT:    sdivr z6.s, p0/m, z6.s, z7.s
-; CHECK-NEXT:    mov z7.d, z1.d
+; CHECK-NEXT:    movprfx z7, z1
 ; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z7.s, z7.h
 ; CHECK-NEXT:    sdivr z7.s, p0/m, z7.s, z16.s
@@ -1201,14 +1201,14 @@ define <16 x i8> @urem_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
 ; CHECK-NEXT:    uunpklo z2.s, z2.h
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    udivr z4.s, p0/m, z4.s, z5.s
-; CHECK-NEXT:    mov z5.d, z0.d
+; CHECK-NEXT:    movprfx z5, z0
 ; CHECK-NEXT:    ext z5.b, z5.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z5.h, z5.b
 ; CHECK-NEXT:    uunpklo z7.s, z5.h
 ; CHECK-NEXT:    ext z5.b, z5.b, z5.b, #8
-; CHECK-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
 ; CHECK-NEXT:    uunpklo z5.s, z5.h
+; CHECK-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
+; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z3.h, z3.b
 ; CHECK-NEXT:    uunpklo z6.s, z3.h
@@ -1329,35 +1329,35 @@ define void @urem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr q0, [x0, #16]
 ; CHECK-NEXT:    ldr q1, [x1, #16]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    uunpklo z3.h, z1.b
-; CHECK-NEXT:    uunpklo z4.h, z0.b
-; CHECK-NEXT:    uunpklo z2.s, z3.h
-; CHECK-NEXT:    uunpklo z5.s, z4.h
+; CHECK-NEXT:    uunpklo z2.h, z1.b
+; CHECK-NEXT:    uunpklo z3.h, z0.b
+; CHECK-NEXT:    uunpklo z4.s, z2.h
+; CHECK-NEXT:    uunpklo z5.s, z3.h
+; CHECK-NEXT:    ext z2.b, z2.b, z2.b, #8
 ; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
-; CHECK-NEXT:    ext z4.b, z4.b, z4.b, #8
+; CHECK-NEXT:    uunpklo z2.s, z2.h
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
-; CHECK-NEXT:    uunpklo z4.s, z4.h
-; CHECK-NEXT:    udivr z2.s, p0/m, z2.s, z5.s
-; CHECK-NEXT:    movprfx z5, z4
-; CHECK-NEXT:    udiv z5.s, p0/m, z5.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
-; CHECK-NEXT:    mov z4.d, z0.d
-; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
-; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z7.h, z3.b
-; CHECK-NEXT:    uunpklo z16.h, z4.b
-; CHECK-NEXT:    uunpklo z3.s, z7.h
-; CHECK-NEXT:    uunpklo z4.s, z16.h
+; CHECK-NEXT:    udivr z4.s, p0/m, z4.s, z5.s
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    udiv z5.s, p0/m, z5.s, z2.s
+; CHECK-NEXT:    movprfx z2, z1
+; CHECK-NEXT:    ext z2.b, z2.b, z1.b, #8
+; CHECK-NEXT:    movprfx z3, z0
+; CHECK-NEXT:    ext z3.b, z3.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z7.h, z2.b
+; CHECK-NEXT:    uunpklo z16.h, z3.b
+; CHECK-NEXT:    uunpklo z2.s, z7.h
+; CHECK-NEXT:    uunpklo z3.s, z16.h
 ; CHECK-NEXT:    ext z7.b, z7.b, z7.b, #8
 ; CHECK-NEXT:    ext z16.b, z16.b, z16.b, #8
 ; CHECK-NEXT:    uunpklo z7.s, z7.h
-; CHECK-NEXT:    movprfx z6, z4
-; CHECK-NEXT:    udiv z6.s, p0/m, z6.s, z3.s
-; CHECK-NEXT:    ldr q3, [x0]
-; CHECK-NEXT:    ldr q4, [x1]
+; CHECK-NEXT:    movprfx z6, z3
+; CHECK-NEXT:    udiv z6.s, p0/m, z6.s, z2.s
+; CHECK-NEXT:    ldr q2, [x0]
+; CHECK-NEXT:    ldr q3, [x1]
 ; CHECK-NEXT:    uunpklo z16.s, z16.h
-; CHECK-NEXT:    uunpklo z17.h, z4.b
-; CHECK-NEXT:    uunpklo z18.h, z3.b
+; CHECK-NEXT:    uunpklo z17.h, z3.b
+; CHECK-NEXT:    uunpklo z18.h, z2.b
 ; CHECK-NEXT:    udivr z7.s, p0/m, z7.s, z16.s
 ; CHECK-NEXT:    uunpklo z19.s, z17.h
 ; CHECK-NEXT:    uunpklo z20.s, z18.h
@@ -1366,44 +1366,44 @@ define void @urem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    uunpklo z17.s, z17.h
 ; CHECK-NEXT:    uunpklo z18.s, z18.h
 ; CHECK-NEXT:    udivr z19.s, p0/m, z19.s, z20.s
-; CHECK-NEXT:    mov z20.d, z3.d
-; CHECK-NEXT:    ext z20.b, z20.b, z3.b, #8
+; CHECK-NEXT:    movprfx z20, z2
+; CHECK-NEXT:    ext z20.b, z20.b, z2.b, #8
 ; CHECK-NEXT:    uunpklo z20.h, z20.b
 ; CHECK-NEXT:    uunpklo z22.s, z20.h
 ; CHECK-NEXT:    ext z20.b, z20.b, z20.b, #8
-; CHECK-NEXT:    udivr z17.s, p0/m, z17.s, z18.s
-; CHECK-NEXT:    mov z18.d, z4.d
 ; CHECK-NEXT:    uunpklo z20.s, z20.h
-; CHECK-NEXT:    ext z18.b, z18.b, z4.b, #8
+; CHECK-NEXT:    udivr z17.s, p0/m, z17.s, z18.s
+; CHECK-NEXT:    movprfx z18, z3
+; CHECK-NEXT:    ext z18.b, z18.b, z3.b, #8
 ; CHECK-NEXT:    uunpklo z18.h, z18.b
 ; CHECK-NEXT:    uunpklo z21.s, z18.h
 ; CHECK-NEXT:    ext z18.b, z18.b, z18.b, #8
 ; CHECK-NEXT:    uunpklo z18.s, z18.h
 ; CHECK-NEXT:    udivr z21.s, p0/m, z21.s, z22.s
-; CHECK-NEXT:    uzp1 z22.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z22.h, z4.h, z4.h
 ; CHECK-NEXT:    uzp1 z23.h, z5.h, z5.h
-; CHECK-NEXT:    uzp1 z5.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z6.h, z7.h, z7.h
 ; CHECK-NEXT:    udivr z18.s, p0/m, z18.s, z20.s
 ; CHECK-NEXT:    uzp1 z19.h, z19.h, z19.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z20.h, z17.h, z17.h
+; CHECK-NEXT:    uzp1 z16.h, z6.h, z6.h
+; CHECK-NEXT:    uzp1 z17.h, z7.h, z7.h
 ; CHECK-NEXT:    splice z7.h, p0, { z22.h, z23.h }
-; CHECK-NEXT:    splice z5.h, p0, { z5.h, z6.h }
-; CHECK-NEXT:    uzp1 z16.h, z21.h, z21.h
-; CHECK-NEXT:    splice z2.h, p0, { z19.h, z20.h }
+; CHECK-NEXT:    uzp1 z4.h, z21.h, z21.h
+; CHECK-NEXT:    splice z6.h, p0, { z19.h, z20.h }
+; CHECK-NEXT:    uzp1 z5.h, z18.h, z18.h
+; CHECK-NEXT:    splice z4.h, p0, { z4.h, z5.h }
+; CHECK-NEXT:    splice z5.h, p0, { z16.h, z17.h }
+; CHECK-NEXT:    uzp1 z16.b, z6.b, z6.b
 ; CHECK-NEXT:    uzp1 z6.b, z7.b, z7.b
-; CHECK-NEXT:    uzp1 z7.b, z5.b, z5.b
-; CHECK-NEXT:    uzp1 z17.h, z18.h, z18.h
-; CHECK-NEXT:    splice z16.h, p0, { z16.h, z17.h }
-; CHECK-NEXT:    uzp1 z17.b, z2.b, z2.b
 ; CHECK-NEXT:    ptrue p0.b, vl8
+; CHECK-NEXT:    uzp1 z17.b, z4.b, z4.b
+; CHECK-NEXT:    uzp1 z7.b, z5.b, z5.b
+; CHECK-NEXT:    splice z4.b, p0, { z16.b, z17.b }
 ; CHECK-NEXT:    splice z5.b, p0, { z6.b, z7.b }
-; CHECK-NEXT:    uzp1 z18.b, z16.b, z16.b
-; CHECK-NEXT:    splice z2.b, p0, { z17.b, z18.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
+; CHECK-NEXT:    mls z2.b, p0/m, z4.b, z3.b
 ; CHECK-NEXT:    mls z0.b, p0/m, z5.b, z1.b
-; CHECK-NEXT:    msb z2.b, p0/m, z4.b, z3.b
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
 ;
@@ -1640,13 +1640,13 @@ define <8 x i16> @urem_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    uunpklo z2.s, z1.h
 ; CHECK-NEXT:    uunpklo z3.s, z0.h
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    mov z4.d, z0.d
+; CHECK-NEXT:    uunpklo z4.s, z4.h
 ; CHECK-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
-; CHECK-NEXT:    mov z3.d, z1.d
-; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z4.s, z4.h
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    udivr z3.s, p0/m, z3.s, z4.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
@@ -1718,21 +1718,21 @@ define void @urem_v16i16(ptr %a, ptr %b) {
 ; CHECK-NEXT:    uunpklo z2.s, z1.h
 ; CHECK-NEXT:    uunpklo z3.s, z0.h
 ; CHECK-NEXT:    uunpklo z5.s, z4.h
-; CHECK-NEXT:    mov z16.d, z0.d
+; CHECK-NEXT:    movprfx z16, z0
+; CHECK-NEXT:    ext z16.b, z16.b, z0.b, #8
 ; CHECK-NEXT:    udivr z2.s, p0/m, z2.s, z3.s
 ; CHECK-NEXT:    ldr q3, [x0]
-; CHECK-NEXT:    ext z16.b, z16.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z6.s, z3.h
-; CHECK-NEXT:    mov z7.d, z3.d
 ; CHECK-NEXT:    uunpklo z16.s, z16.h
+; CHECK-NEXT:    uunpklo z6.s, z3.h
+; CHECK-NEXT:    movprfx z7, z3
 ; CHECK-NEXT:    ext z7.b, z7.b, z3.b, #8
 ; CHECK-NEXT:    uunpklo z7.s, z7.h
 ; CHECK-NEXT:    udivr z5.s, p0/m, z5.s, z6.s
-; CHECK-NEXT:    mov z6.d, z4.d
+; CHECK-NEXT:    movprfx z6, z4
 ; CHECK-NEXT:    ext z6.b, z6.b, z4.b, #8
 ; CHECK-NEXT:    uunpklo z6.s, z6.h
 ; CHECK-NEXT:    udivr z6.s, p0/m, z6.s, z7.s
-; CHECK-NEXT:    mov z7.d, z1.d
+; CHECK-NEXT:    movprfx z7, z1
 ; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z7.s, z7.h
 ; CHECK-NEXT:    udivr z7.s, p0/m, z7.s, z16.s
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
index 7df362826d052..f7fadaa1217bc 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
@@ -256,12 +256,13 @@ define void @ucvtf_v8i16_v8f32(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    uunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    ucvtf z1.s, p0/m, z1.s
+; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    ucvtf z0.s, p0/m, z0.s
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    ucvtf z1.s, p0/m, z1.s
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v8i16_v8f32:
@@ -306,18 +307,20 @@ define void @ucvtf_v16i16_v16f32(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    uunpklo z2.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
 ; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    ucvtf z2.s, p0/m, z2.s
-; CHECK-NEXT:    ucvtf z3.s, p0/m, z3.s
+; CHECK-NEXT:    uunpklo z2.s, z2.h
+; CHECK-NEXT:    uunpklo z3.s, z3.h
 ; CHECK-NEXT:    ucvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    ucvtf z1.s, p0/m, z1.s
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    ucvtf z2.s, p0/m, z2.s
+; CHECK-NEXT:    ucvtf z3.s, p0/m, z3.s
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v16i16_v16f32:
@@ -442,12 +445,13 @@ define void @ucvtf_v4i16_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v4i16_v4f64:
@@ -481,21 +485,24 @@ define void @ucvtf_v8i16_v8f64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    uunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z2.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
+; CHECK-NEXT:    uunpklo z2.d, z2.s
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
+; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
+; CHECK-NEXT:    uunpklo z3.d, z3.s
 ; CHECK-NEXT:    ucvtf z2.d, p0/m, z2.d
-; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
 ; CHECK-NEXT:    ucvtf z3.d, p0/m, z3.d
-; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    stp q2, q1, [x1]
-; CHECK-NEXT:    stp q3, q0, [x1, #32]
+; CHECK-NEXT:    stp q0, q2, [x1]
+; CHECK-NEXT:    stp q1, q3, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v8i16_v8f64:
@@ -542,43 +549,43 @@ define void @ucvtf_v16i16_v16f64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    mov z2.d, z0.d
-; CHECK-NEXT:    mov z3.d, z1.d
-; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    mov z4.d, z1.d
+; CHECK-NEXT:    uunpklo z3.s, z1.h
+; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z2.s, z2.h
-; CHECK-NEXT:    uunpklo z3.s, z3.h
-; CHECK-NEXT:    uunpklo z5.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    ext z4.b, z4.b, z1.b, #8
-; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    mov z6.d, z2.d
-; CHECK-NEXT:    mov z7.d, z3.d
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
+; CHECK-NEXT:    uunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    ext z5.b, z5.b, z3.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
 ; CHECK-NEXT:    uunpklo z4.d, z4.s
-; CHECK-NEXT:    ucvtf z5.d, p0/m, z5.d
-; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    uunpklo z5.d, z5.s
+; CHECK-NEXT:    movprfx z6, z2
 ; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
-; CHECK-NEXT:    ext z7.b, z7.b, z3.b, #8
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z2.d, z2.s
-; CHECK-NEXT:    uunpklo z3.d, z3.s
+; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    ucvtf z4.d, p0/m, z4.d
+; CHECK-NEXT:    ucvtf z3.d, p0/m, z3.d
 ; CHECK-NEXT:    uunpklo z6.d, z6.s
+; CHECK-NEXT:    ucvtf z4.d, p0/m, z4.d
 ; CHECK-NEXT:    uunpklo z7.d, z7.s
+; CHECK-NEXT:    ucvtf z5.d, p0/m, z5.d
 ; CHECK-NEXT:    ucvtf z2.d, p0/m, z2.d
-; CHECK-NEXT:    stp q5, q0, [x1, #64]
-; CHECK-NEXT:    ucvtf z3.d, p0/m, z3.d
-; CHECK-NEXT:    stp q1, q4, [x1]
-; CHECK-NEXT:    movprfx z1, z6
-; CHECK-NEXT:    ucvtf z1.d, p0/m, z6.d
-; CHECK-NEXT:    movprfx z0, z7
-; CHECK-NEXT:    ucvtf z0.d, p0/m, z7.d
-; CHECK-NEXT:    stp q3, q0, [x1, #32]
-; CHECK-NEXT:    stp q2, q1, [x1, #96]
+; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    stp q3, q5, [x1]
+; CHECK-NEXT:    movprfx z3, z7
+; CHECK-NEXT:    ucvtf z3.d, p0/m, z7.d
+; CHECK-NEXT:    stp q0, q4, [x1, #64]
+; CHECK-NEXT:    movprfx z0, z6
+; CHECK-NEXT:    ucvtf z0.d, p0/m, z6.d
+; CHECK-NEXT:    stp q1, q3, [x1, #32]
+; CHECK-NEXT:    stp q2, q0, [x1, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v16i16_v16f64:
@@ -997,12 +1004,13 @@ define void @ucvtf_v4i32_v4f64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v4i32_v4f64:
@@ -1035,18 +1043,20 @@ define void @ucvtf_v8i32_v8f64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    uunpklo z2.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    uunpklo z3.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
-; CHECK-NEXT:    ucvtf z2.d, p0/m, z2.d
-; CHECK-NEXT:    ucvtf z3.d, p0/m, z3.d
+; CHECK-NEXT:    uunpklo z2.d, z2.s
+; CHECK-NEXT:    uunpklo z3.d, z3.s
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    ucvtf z2.d, p0/m, z2.d
+; CHECK-NEXT:    ucvtf z3.d, p0/m, z3.d
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: ucvtf_v8i32_v8f64:
@@ -1669,12 +1679,13 @@ define void @scvtf_v8i16_v8f32(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    sunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    scvtf z1.s, p0/m, z1.s
+; CHECK-NEXT:    sunpklo z1.s, z1.h
 ; CHECK-NEXT:    scvtf z0.s, p0/m, z0.s
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    scvtf z1.s, p0/m, z1.s
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v8i16_v8f32:
@@ -1719,18 +1730,20 @@ define void @scvtf_v16i16_v16f32(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    sunpklo z2.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z3.s, z1.h
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
 ; CHECK-NEXT:    sunpklo z1.s, z1.h
-; CHECK-NEXT:    scvtf z2.s, p0/m, z2.s
-; CHECK-NEXT:    scvtf z3.s, p0/m, z3.s
+; CHECK-NEXT:    sunpklo z2.s, z2.h
+; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    scvtf z0.s, p0/m, z0.s
 ; CHECK-NEXT:    scvtf z1.s, p0/m, z1.s
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    scvtf z2.s, p0/m, z2.s
+; CHECK-NEXT:    scvtf z3.s, p0/m, z3.s
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v16i16_v16f32:
@@ -1833,12 +1846,13 @@ define void @scvtf_v4i16_v4f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldr d0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    sunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
-; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    sunpklo z1.d, z1.s
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v4i16_v4f64:
@@ -1872,21 +1886,24 @@ define void @scvtf_v8i16_v8f64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    sunpklo z1.s, z0.h
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    sunpklo z2.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z3.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z0.d, z0.s
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
+; CHECK-NEXT:    sunpklo z2.d, z2.s
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
+; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
+; CHECK-NEXT:    sunpklo z3.d, z3.s
 ; CHECK-NEXT:    scvtf z2.d, p0/m, z2.d
-; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
 ; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
-; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    stp q2, q1, [x1]
-; CHECK-NEXT:    stp q3, q0, [x1, #32]
+; CHECK-NEXT:    stp q0, q2, [x1]
+; CHECK-NEXT:    stp q1, q3, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v8i16_v8f64:
@@ -1933,43 +1950,43 @@ define void @scvtf_v16i16_v16f64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    mov z2.d, z0.d
-; CHECK-NEXT:    mov z3.d, z1.d
-; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
-; CHECK-NEXT:    ext z3.b, z3.b, z3.b, #8
 ; CHECK-NEXT:    sunpklo z0.s, z0.h
-; CHECK-NEXT:    mov z4.d, z1.d
+; CHECK-NEXT:    sunpklo z3.s, z1.h
+; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z2.s, z2.h
-; CHECK-NEXT:    sunpklo z3.s, z3.h
-; CHECK-NEXT:    sunpklo z5.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    ext z4.b, z4.b, z1.b, #8
-; CHECK-NEXT:    sunpklo z1.d, z1.s
-; CHECK-NEXT:    mov z6.d, z2.d
-; CHECK-NEXT:    mov z7.d, z3.d
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    ext z4.b, z4.b, z0.b, #8
+; CHECK-NEXT:    sunpklo z1.s, z1.h
+; CHECK-NEXT:    movprfx z5, z3
+; CHECK-NEXT:    ext z5.b, z5.b, z3.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
+; CHECK-NEXT:    sunpklo z3.d, z3.s
 ; CHECK-NEXT:    sunpklo z4.d, z4.s
-; CHECK-NEXT:    scvtf z5.d, p0/m, z5.d
-; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    sunpklo z5.d, z5.s
+; CHECK-NEXT:    movprfx z6, z2
 ; CHECK-NEXT:    ext z6.b, z6.b, z2.b, #8
-; CHECK-NEXT:    ext z7.b, z7.b, z3.b, #8
+; CHECK-NEXT:    movprfx z7, z1
+; CHECK-NEXT:    ext z7.b, z7.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z2.d, z2.s
-; CHECK-NEXT:    sunpklo z3.d, z3.s
+; CHECK-NEXT:    sunpklo z1.d, z1.s
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    scvtf z4.d, p0/m, z4.d
+; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
 ; CHECK-NEXT:    sunpklo z6.d, z6.s
+; CHECK-NEXT:    scvtf z4.d, p0/m, z4.d
 ; CHECK-NEXT:    sunpklo z7.d, z7.s
+; CHECK-NEXT:    scvtf z5.d, p0/m, z5.d
 ; CHECK-NEXT:    scvtf z2.d, p0/m, z2.d
-; CHECK-NEXT:    stp q5, q0, [x1, #64]
-; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
-; CHECK-NEXT:    stp q1, q4, [x1]
-; CHECK-NEXT:    movprfx z1, z6
-; CHECK-NEXT:    scvtf z1.d, p0/m, z6.d
-; CHECK-NEXT:    movprfx z0, z7
-; CHECK-NEXT:    scvtf z0.d, p0/m, z7.d
-; CHECK-NEXT:    stp q3, q0, [x1, #32]
-; CHECK-NEXT:    stp q2, q1, [x1, #96]
+; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    stp q3, q5, [x1]
+; CHECK-NEXT:    movprfx z3, z7
+; CHECK-NEXT:    scvtf z3.d, p0/m, z7.d
+; CHECK-NEXT:    stp q0, q4, [x1, #64]
+; CHECK-NEXT:    movprfx z0, z6
+; CHECK-NEXT:    scvtf z0.d, p0/m, z6.d
+; CHECK-NEXT:    stp q1, q3, [x1, #32]
+; CHECK-NEXT:    stp q2, q0, [x1, #96]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v16i16_v16f64:
@@ -2293,12 +2310,13 @@ define void @scvtf_v4i32_v4f64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    sunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
-; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    sunpklo z1.d, z1.s
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    stp q1, q0, [x1]
+; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
+; CHECK-NEXT:    stp q0, q1, [x1]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v4i32_v4f64:
@@ -2331,18 +2349,20 @@ define void @scvtf_v8i32_v8f64(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0]
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    sunpklo z2.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
-; CHECK-NEXT:    sunpklo z3.d, z1.s
-; CHECK-NEXT:    ext z1.b, z1.b, z1.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    ext z3.b, z3.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
-; CHECK-NEXT:    scvtf z2.d, p0/m, z2.d
-; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
+; CHECK-NEXT:    sunpklo z2.d, z2.s
+; CHECK-NEXT:    sunpklo z3.d, z3.s
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
-; CHECK-NEXT:    stp q2, q0, [x1, #32]
-; CHECK-NEXT:    stp q3, q1, [x1]
+; CHECK-NEXT:    scvtf z2.d, p0/m, z2.d
+; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
+; CHECK-NEXT:    stp q1, q3, [x1]
+; CHECK-NEXT:    stp q0, q2, [x1, #32]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: scvtf_v8i32_v8f64:
@@ -2388,15 +2408,15 @@ define void @scvtf_v16i32_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    ldp q5, q3, [x0]
-; CHECK-NEXT:    mov z2.d, z0.d
-; CHECK-NEXT:    mov z4.d, z1.d
-; CHECK-NEXT:    mov z6.d, z3.d
-; CHECK-NEXT:    mov z7.d, z5.d
+; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
+; CHECK-NEXT:    movprfx z4, z1
 ; CHECK-NEXT:    ext z4.b, z4.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
+; CHECK-NEXT:    movprfx z6, z3
 ; CHECK-NEXT:    ext z6.b, z6.b, z3.b, #8
+; CHECK-NEXT:    movprfx z7, z5
 ; CHECK-NEXT:    ext z7.b, z7.b, z5.b, #8
 ; CHECK-NEXT:    sunpklo z3.d, z3.s
 ; CHECK-NEXT:    sunpklo z5.d, z5.s
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-limit-duplane.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-limit-duplane.ll
index 3627390b5edfa..daf2734ce7d55 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-limit-duplane.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-limit-duplane.ll
@@ -99,15 +99,16 @@ define <2 x i32> @test2(ptr %arg1, ptr %arg2) {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ldp q3, q4, [x0]
-; CHECK-NEXT:    add z2.s, z0.s, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    ext z2.b, z2.b, z0.b, #8
 ; CHECK-NEXT:    add z1.s, z1.s, z1.s
-; CHECK-NEXT:    add z3.s, z3.s, z3.s
-; CHECK-NEXT:    add z4.s, z4.s, z4.s
-; CHECK-NEXT:    mov z0.s, s0
-; CHECK-NEXT:    stp q1, q2, [x0, #32]
-; CHECK-NEXT:    stp q3, q4, [x0]
+; CHECK-NEXT:    add z5.s, z0.s, z0.s
+; CHECK-NEXT:    mov z0.s, s2
+; CHECK-NEXT:    add z2.s, z3.s, z3.s
+; CHECK-NEXT:    add z3.s, z4.s, z4.s
+; CHECK-NEXT:    stp q1, q5, [x0, #32]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
+; CHECK-NEXT:    stp q2, q3, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: test2:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
index 48a642c908bfe..42b947604b860 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll
@@ -1815,17 +1815,18 @@ define <16 x half> @masked_load_v16f16(ptr %src, <16 x i1> %mask) {
 ; CHECK-LABEL: masked_load_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z1.h, z0.b
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    mov x8, #8 // =0x8
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
-; CHECK-NEXT:    lsl z1.h, z1.h, #15
-; CHECK-NEXT:    asr z1.h, z1.h, #15
+; CHECK-NEXT:    uunpklo z1.h, z1.b
 ; CHECK-NEXT:    lsl z0.h, z0.h, #15
-; CHECK-NEXT:    cmpne p1.h, p0/z, z1.h, #0
+; CHECK-NEXT:    lsl z1.h, z1.h, #15
 ; CHECK-NEXT:    asr z0.h, z0.h, #15
-; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT:    asr z1.h, z1.h, #15
+; CHECK-NEXT:    cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT:    cmpne p0.h, p0/z, z1.h, #0
 ; CHECK-NEXT:    ld1h { z0.h }, p1/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0, x8, lsl #1]
@@ -2572,15 +2573,16 @@ define <4 x double> @masked_load_v4f64(ptr %src, <4 x i1> %mask) {
 ; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    mov x8, #2 // =0x2
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    lsl z1.d, z1.d, #63
+; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    lsl z0.d, z0.d, #63
-; CHECK-NEXT:    asr z1.d, z1.d, #63
+; CHECK-NEXT:    lsl z1.d, z1.d, #63
 ; CHECK-NEXT:    asr z0.d, z0.d, #63
-; CHECK-NEXT:    cmpne p1.d, p0/z, z1.d, #0
-; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT:    asr z1.d, z1.d, #63
+; CHECK-NEXT:    cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT:    cmpne p0.d, p0/z, z1.d, #0
 ; CHECK-NEXT:    ld1d { z0.d }, p1/z, [x0]
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, x8, lsl #3]
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
index 265480b571970..9b3da75be47ec 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll
@@ -797,20 +797,21 @@ define void @masked_store_v16f16(ptr %dst, <16 x i1> %mask) {
 ; CHECK-LABEL: masked_store_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    uunpklo z1.h, z0.b
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    ptrue p0.h, vl8
 ; CHECK-NEXT:    mov x8, #8 // =0x8
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.h, z0.b
-; CHECK-NEXT:    lsl z1.h, z1.h, #15
-; CHECK-NEXT:    asr z1.h, z1.h, #15
+; CHECK-NEXT:    uunpklo z1.h, z1.b
 ; CHECK-NEXT:    lsl z0.h, z0.h, #15
+; CHECK-NEXT:    lsl z1.h, z1.h, #15
 ; CHECK-NEXT:    asr z0.h, z0.h, #15
-; CHECK-NEXT:    cmpne p1.h, p0/z, z0.h, #0
-; CHECK-NEXT:    cmpne p0.h, p0/z, z1.h, #0
-; CHECK-NEXT:    mov z0.h, #0 // =0x0
-; CHECK-NEXT:    st1h { z0.h }, p1, [x0, x8, lsl #1]
-; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
+; CHECK-NEXT:    asr z1.h, z1.h, #15
+; CHECK-NEXT:    cmpne p1.h, p0/z, z1.h, #0
+; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
+; CHECK-NEXT:    mov z1.h, #0 // =0x0
+; CHECK-NEXT:    st1h { z1.h }, p1, [x0, x8, lsl #1]
+; CHECK-NEXT:    st1h { z1.h }, p0, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: masked_store_v16f16:
@@ -1165,18 +1166,19 @@ define void @masked_store_v4f64(ptr %dst, <4 x i1> %mask) {
 ; CHECK-NEXT:    ptrue p0.d, vl2
 ; CHECK-NEXT:    mov x8, #2 // =0x2
 ; CHECK-NEXT:    uunpklo z0.s, z0.h
-; CHECK-NEXT:    uunpklo z1.d, z0.s
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    ext z1.b, z1.b, z0.b, #8
 ; CHECK-NEXT:    uunpklo z0.d, z0.s
-; CHECK-NEXT:    lsl z1.d, z1.d, #63
+; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    lsl z0.d, z0.d, #63
-; CHECK-NEXT:    asr z1.d, z1.d, #63
+; CHECK-NEXT:    lsl z1.d, z1.d, #63
 ; CHECK-NEXT:    asr z0.d, z0.d, #63
-; CHECK-NEXT:    cmpne p1.d, p0/z, z0.d, #0
-; CHECK-NEXT:    cmpne p0.d, p0/z, z1.d, #0
-; CHECK-NEXT:    mov z0.d, #0 // =0x0
-; CHECK-NEXT:    st1d { z0.d }, p1, [x0, x8, lsl #3]
-; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    asr z1.d, z1.d, #63
+; CHECK-NEXT:    cmpne p1.d, p0/z, z1.d, #0
+; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
+; CHECK-NEXT:    mov z1.d, #0 // =0x0
+; CHECK-NEXT:    st1d { z1.d }, p1, [x0, x8, lsl #3]
+; CHECK-NEXT:    st1d { z1.d }, p0, [x0]
 ; CHECK-NEXT:    ret
 ;
 ; NONEON-NOSVE-LABEL: masked_store_v4f64:
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
index 90466e3cebd5e..5586de39fb1bf 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-ptest.ll
@@ -11,17 +11,17 @@ define i1 @ptest_v16i1(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q1, q0, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ldp q3, q2, [x0]
+; CHECK-NEXT:    ldp q2, q3, [x0]
 ; CHECK-NEXT:    fcmne p1.s, p0/z, z0.s, #0.0
 ; CHECK-NEXT:    fcmne p2.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    fcmne p3.s, p0/z, z2.s, #0.0
-; CHECK-NEXT:    fcmne p0.s, p0/z, z3.s, #0.0
+; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
+; CHECK-NEXT:    fcmne p0.s, p0/z, z2.s, #0.0
 ; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z2.s, p3/z, #-1 // =0xffffffffffffffff
 ; CHECK-NEXT:    mov z3.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z5.h, z0.h, z0.h
+; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z4.h, z1.h, z1.h
 ; CHECK-NEXT:    uzp1 z1.h, z2.h, z2.h
 ; CHECK-NEXT:    uzp1 z0.h, z3.h, z3.h
@@ -122,47 +122,47 @@ define i1 @ptest_or_v16i1(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ldp q2, q3, [x1, #32]
-; CHECK-NEXT:    ldp q4, q5, [x0]
+; CHECK-NEXT:    ldp q4, q5, [x1, #32]
+; CHECK-NEXT:    ldp q2, q3, [x0]
 ; CHECK-NEXT:    fcmne p1.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    ldp q6, q1, [x1]
-; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
 ; CHECK-NEXT:    fcmne p2.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    fcmne p5.s, p0/z, z2.s, #0.0
-; CHECK-NEXT:    fcmne p4.s, p0/z, z5.s, #0.0
-; CHECK-NEXT:    fcmne p7.s, p0/z, z4.s, #0.0
-; CHECK-NEXT:    fcmne p6.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    fcmne p0.s, p0/z, z6.s, #0.0
-; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z2.s, p3/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    ldp q0, q1, [x1]
+; CHECK-NEXT:    fcmne p5.s, p0/z, z5.s, #0.0
+; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
+; CHECK-NEXT:    fcmne p6.s, p0/z, z4.s, #0.0
+; CHECK-NEXT:    fcmne p7.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z4.s, p5/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z3.s, p4/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z6.s, p7/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z5.s, p6/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z7.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    uzp1 z17.h, z0.h, z0.h
-; CHECK-NEXT:    uzp1 z19.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z16.h, z1.h, z1.h
+; CHECK-NEXT:    fcmne p4.s, p0/z, z2.s, #0.0
+; CHECK-NEXT:    mov z2.s, p5/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z3.s, p3/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z4.s, p6/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z5.s, p7/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    uzp1 z7.h, z0.h, z0.h
+; CHECK-NEXT:    mov z0.s, p4/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    uzp1 z18.h, z3.h, z3.h
+; CHECK-NEXT:    mov z16.s, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    uzp1 z6.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z1.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z4.h, z5.h, z5.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z1.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z18.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z3.h, z5.h, z5.h
-; CHECK-NEXT:    uzp1 z0.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z2.h, z7.h, z7.h
-; CHECK-NEXT:    splice z4.h, p0, { z16.h, z17.h }
-; CHECK-NEXT:    splice z5.h, p0, { z18.h, z19.h }
-; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
-; CHECK-NEXT:    splice z1.h, p0, { z2.h, z3.h }
+; CHECK-NEXT:    uzp1 z17.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z3.h, z16.h, z16.h
+; CHECK-NEXT:    splice z0.h, p0, { z6.h, z7.h }
+; CHECK-NEXT:    splice z1.h, p0, { z1.h, z2.h }
+; CHECK-NEXT:    splice z2.h, p0, { z17.h, z18.h }
+; CHECK-NEXT:    splice z3.h, p0, { z3.h, z4.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z3.b, z4.b, z4.b
-; CHECK-NEXT:    uzp1 z5.b, z5.b, z5.b
-; CHECK-NEXT:    uzp1 z2.b, z0.b, z0.b
-; CHECK-NEXT:    uzp1 z4.b, z1.b, z1.b
-; CHECK-NEXT:    splice z0.b, p0, { z2.b, z3.b }
-; CHECK-NEXT:    splice z1.b, p0, { z4.b, z5.b }
+; CHECK-NEXT:    uzp1 z5.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z1.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z4.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z0.b, z3.b, z3.b
+; CHECK-NEXT:    splice z2.b, p0, { z4.b, z5.b }
+; CHECK-NEXT:    splice z0.b, p0, { z0.b, z1.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    orr z0.d, z0.d, z1.d
+; CHECK-NEXT:    orr z0.d, z2.d, z0.d
 ; CHECK-NEXT:    umaxv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
@@ -331,47 +331,47 @@ define i1 @ptest_and_v16i1(ptr %a, ptr %b) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q0, q1, [x0, #32]
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    ldp q2, q3, [x1, #32]
-; CHECK-NEXT:    ldp q4, q5, [x0]
+; CHECK-NEXT:    ldp q4, q5, [x1, #32]
+; CHECK-NEXT:    ldp q2, q3, [x0]
 ; CHECK-NEXT:    fcmne p1.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    ldp q6, q1, [x1]
-; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
 ; CHECK-NEXT:    fcmne p2.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    fcmne p5.s, p0/z, z2.s, #0.0
-; CHECK-NEXT:    fcmne p4.s, p0/z, z5.s, #0.0
-; CHECK-NEXT:    fcmne p7.s, p0/z, z4.s, #0.0
-; CHECK-NEXT:    fcmne p6.s, p0/z, z1.s, #0.0
-; CHECK-NEXT:    fcmne p0.s, p0/z, z6.s, #0.0
-; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z2.s, p3/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    ldp q0, q1, [x1]
+; CHECK-NEXT:    fcmne p5.s, p0/z, z5.s, #0.0
+; CHECK-NEXT:    fcmne p3.s, p0/z, z3.s, #0.0
+; CHECK-NEXT:    fcmne p6.s, p0/z, z4.s, #0.0
+; CHECK-NEXT:    fcmne p7.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    mov z1.s, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z4.s, p5/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z3.s, p4/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z6.s, p7/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z5.s, p6/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    mov z7.s, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT:    uzp1 z17.h, z0.h, z0.h
-; CHECK-NEXT:    uzp1 z19.h, z2.h, z2.h
-; CHECK-NEXT:    uzp1 z16.h, z1.h, z1.h
+; CHECK-NEXT:    fcmne p4.s, p0/z, z2.s, #0.0
+; CHECK-NEXT:    mov z2.s, p5/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    fcmne p0.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z3.s, p3/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z4.s, p6/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    mov z5.s, p7/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    uzp1 z7.h, z0.h, z0.h
+; CHECK-NEXT:    mov z0.s, p4/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    uzp1 z18.h, z3.h, z3.h
+; CHECK-NEXT:    mov z16.s, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT:    uzp1 z6.h, z1.h, z1.h
+; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z1.h, z4.h, z4.h
+; CHECK-NEXT:    uzp1 z4.h, z5.h, z5.h
 ; CHECK-NEXT:    ptrue p0.h, vl4
-; CHECK-NEXT:    uzp1 z1.h, z3.h, z3.h
-; CHECK-NEXT:    uzp1 z18.h, z4.h, z4.h
-; CHECK-NEXT:    uzp1 z3.h, z5.h, z5.h
-; CHECK-NEXT:    uzp1 z0.h, z6.h, z6.h
-; CHECK-NEXT:    uzp1 z2.h, z7.h, z7.h
-; CHECK-NEXT:    splice z4.h, p0, { z16.h, z17.h }
-; CHECK-NEXT:    splice z5.h, p0, { z18.h, z19.h }
-; CHECK-NEXT:    splice z0.h, p0, { z0.h, z1.h }
-; CHECK-NEXT:    splice z1.h, p0, { z2.h, z3.h }
+; CHECK-NEXT:    uzp1 z17.h, z0.h, z0.h
+; CHECK-NEXT:    uzp1 z3.h, z16.h, z16.h
+; CHECK-NEXT:    splice z0.h, p0, { z6.h, z7.h }
+; CHECK-NEXT:    splice z1.h, p0, { z1.h, z2.h }
+; CHECK-NEXT:    splice z2.h, p0, { z17.h, z18.h }
+; CHECK-NEXT:    splice z3.h, p0, { z3.h, z4.h }
 ; CHECK-NEXT:    ptrue p0.b, vl8
-; CHECK-NEXT:    uzp1 z3.b, z4.b, z4.b
-; CHECK-NEXT:    uzp1 z5.b, z5.b, z5.b
-; CHECK-NEXT:    uzp1 z2.b, z0.b, z0.b
-; CHECK-NEXT:    uzp1 z4.b, z1.b, z1.b
-; CHECK-NEXT:    splice z0.b, p0, { z2.b, z3.b }
-; CHECK-NEXT:    splice z1.b, p0, { z4.b, z5.b }
+; CHECK-NEXT:    uzp1 z5.b, z0.b, z0.b
+; CHECK-NEXT:    uzp1 z1.b, z1.b, z1.b
+; CHECK-NEXT:    uzp1 z4.b, z2.b, z2.b
+; CHECK-NEXT:    uzp1 z0.b, z3.b, z3.b
+; CHECK-NEXT:    splice z2.b, p0, { z4.b, z5.b }
+; CHECK-NEXT:    splice z0.b, p0, { z0.b, z1.b }
 ; CHECK-NEXT:    ptrue p0.b, vl16
-; CHECK-NEXT:    and z0.d, z0.d, z1.d
+; CHECK-NEXT:    and z0.d, z2.d, z0.d
 ; CHECK-NEXT:    uminv b0, p0, z0.b
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    and w0, w8, #0x1
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll
index 688537704a6f7..7363c306033a1 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-reductions.ll
@@ -37,34 +37,39 @@ define i32 @reduce_uaddv_v16i8(<32 x i8> %a) {
 ; STREAMING-SVE-LABEL: reduce_uaddv_v16i8:
 ; STREAMING-SVE:       // %bb.0:
 ; STREAMING-SVE-NEXT:    // kill: def $q1 killed $q1 def $z1
-; STREAMING-SVE-NEXT:    uunpklo z2.h, z1.b
 ; STREAMING-SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
-; STREAMING-SVE-NEXT:    uunpklo z3.h, z0.b
+; STREAMING-SVE-NEXT:    movprfx z2, z1
+; STREAMING-SVE-NEXT:    ext z2.b, z2.b, z1.b, #8
+; STREAMING-SVE-NEXT:    movprfx z3, z0
+; STREAMING-SVE-NEXT:    ext z3.b, z3.b, z0.b, #8
 ; STREAMING-SVE-NEXT:    ptrue p0.s, vl4
-; STREAMING-SVE-NEXT:    ext z1.b, z1.b, z1.b, #8
-; STREAMING-SVE-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; STREAMING-SVE-NEXT:    uunpklo z1.h, z1.b
 ; STREAMING-SVE-NEXT:    uunpklo z0.h, z0.b
-; STREAMING-SVE-NEXT:    uunpklo z4.s, z2.h
-; STREAMING-SVE-NEXT:    ext z2.b, z2.b, z2.b, #8
-; STREAMING-SVE-NEXT:    uunpklo z6.s, z3.h
-; STREAMING-SVE-NEXT:    ext z3.b, z3.b, z3.b, #8
-; STREAMING-SVE-NEXT:    mov z5.d, z1.d
-; STREAMING-SVE-NEXT:    uunpklo z7.s, z0.h
-; STREAMING-SVE-NEXT:    ext z0.b, z0.b, z0.b, #8
-; STREAMING-SVE-NEXT:    uunpklo z2.s, z2.h
-; STREAMING-SVE-NEXT:    uunpklo z3.s, z3.h
-; STREAMING-SVE-NEXT:    add z4.s, z6.s, z4.s
-; STREAMING-SVE-NEXT:    ext z5.b, z5.b, z1.b, #8
+; STREAMING-SVE-NEXT:    uunpklo z2.h, z2.b
+; STREAMING-SVE-NEXT:    uunpklo z3.h, z3.b
+; STREAMING-SVE-NEXT:    movprfx z4, z1
+; STREAMING-SVE-NEXT:    ext z4.b, z4.b, z1.b, #8
+; STREAMING-SVE-NEXT:    movprfx z7, z0
+; STREAMING-SVE-NEXT:    ext z7.b, z7.b, z0.b, #8
 ; STREAMING-SVE-NEXT:    uunpklo z1.s, z1.h
 ; STREAMING-SVE-NEXT:    uunpklo z0.s, z0.h
-; STREAMING-SVE-NEXT:    add z2.s, z3.s, z2.s
+; STREAMING-SVE-NEXT:    movprfx z5, z2
+; STREAMING-SVE-NEXT:    ext z5.b, z5.b, z2.b, #8
+; STREAMING-SVE-NEXT:    movprfx z6, z3
+; STREAMING-SVE-NEXT:    ext z6.b, z6.b, z3.b, #8
+; STREAMING-SVE-NEXT:    uunpklo z2.s, z2.h
+; STREAMING-SVE-NEXT:    uunpklo z4.s, z4.h
+; STREAMING-SVE-NEXT:    uunpklo z7.s, z7.h
+; STREAMING-SVE-NEXT:    uunpklo z3.s, z3.h
 ; STREAMING-SVE-NEXT:    uunpklo z5.s, z5.h
-; STREAMING-SVE-NEXT:    add z1.s, z7.s, z1.s
-; STREAMING-SVE-NEXT:    add z0.s, z0.s, z5.s
-; STREAMING-SVE-NEXT:    add z1.s, z4.s, z1.s
-; STREAMING-SVE-NEXT:    add z0.s, z2.s, z0.s
-; STREAMING-SVE-NEXT:    add z0.s, z1.s, z0.s
+; STREAMING-SVE-NEXT:    uunpklo z6.s, z6.h
+; STREAMING-SVE-NEXT:    add z0.s, z0.s, z1.s
+; STREAMING-SVE-NEXT:    add z1.s, z3.s, z2.s
+; STREAMING-SVE-NEXT:    add z2.s, z7.s, z4.s
+; STREAMING-SVE-NEXT:    add z3.s, z6.s, z5.s
+; STREAMING-SVE-NEXT:    add z0.s, z0.s, z1.s
+; STREAMING-SVE-NEXT:    add z1.s, z2.s, z3.s
+; STREAMING-SVE-NEXT:    add z0.s, z0.s, z1.s
 ; STREAMING-SVE-NEXT:    uaddv d0, p0, z0.s
 ; STREAMING-SVE-NEXT:    fmov w0, s0
 ; STREAMING-SVE-NEXT:    ret
@@ -104,34 +109,39 @@ define i32 @reduce_saddv_v16i8(<32 x i8> %a) {
 ; STREAMING-SVE-LABEL: reduce_saddv_v16i8:
 ; STREAMING-SVE:       // %bb.0:
 ; STREAMING-SVE-NEXT:    // kill: def $q1 killed $q1 def $z1
-; STREAMING-SVE-NEXT:    sunpklo z2.h, z1.b
 ; STREAMING-SVE-NEXT:    // kill: def $q0 killed $q0 def $z0
-; STREAMING-SVE-NEXT:    sunpklo z3.h, z0.b
+; STREAMING-SVE-NEXT:    movprfx z2, z1
+; STREAMING-SVE-NEXT:    ext z2.b, z2.b, z1.b, #8
+; STREAMING-SVE-NEXT:    movprfx z3, z0
+; STREAMING-SVE-NEXT:    ext z3.b, z3.b, z0.b, #8
 ; STREAMING-SVE-NEXT:    ptrue p0.s, vl4
-; STREAMING-SVE-NEXT:    ext z1.b, z1.b, z1.b, #8
-; STREAMING-SVE-NEXT:    ext z0.b, z0.b, z0.b, #8
 ; STREAMING-SVE-NEXT:    sunpklo z1.h, z1.b
 ; STREAMING-SVE-NEXT:    sunpklo z0.h, z0.b
-; STREAMING-SVE-NEXT:    sunpklo z4.s, z2.h
-; STREAMING-SVE-NEXT:    ext z2.b, z2.b, z2.b, #8
-; STREAMING-SVE-NEXT:    sunpklo z6.s, z3.h
-; STREAMING-SVE-NEXT:    ext z3.b, z3.b, z3.b, #8
-; STREAMING-SVE-NEXT:    mov z5.d, z1.d
-; STREAMING-SVE-NEXT:    sunpklo z7.s, z0.h
-; STREAMING-SVE-NEXT:    ext z0.b, z0.b, z0.b, #8
-; STREAMING-SVE-NEXT:    sunpklo z2.s, z2.h
-; STREAMING-SVE-NEXT:    sunpklo z3.s, z3.h
-; STREAMING-SVE-NEXT:    add z4.s, z6.s, z4.s
-; STREAMING-SVE-NEXT:    ext z5.b, z5.b, z1.b, #8
+; STREAMING-SVE-NEXT:    sunpklo z2.h, z2.b
+; STREAMING-SVE-NEXT:    sunpklo z3.h, z3.b
+; STREAMING-SVE-NEXT:    movprfx z4, z1
+; STREAMING-SVE-NEXT:    ext z4.b, z4.b, z1.b, #8
+; STREAMING-SVE-NEXT:    movprfx z7, z0
+; STREAMING-SVE-NEXT:    ext z7.b, z7.b, z0.b, #8
 ; STREAMING-SVE-NEXT:    sunpklo z1.s, z1.h
 ; STREAMING-SVE-NEXT:    sunpklo z0.s, z0.h
-; STREAMING-SVE-NEXT:    add z2.s, z3.s, z2.s
+; STREAMING-SVE-NEXT:    movprfx z5, z2
+; STREAMING-SVE-NEXT:    ext z5.b, z5.b, z2.b, #8
+; STREAMING-SVE-NEXT:    movprfx z6, z3
+; STREAMING-SVE-NEXT:    ext z6.b, z6.b, z3.b, #8
+; STREAMING-SVE-NEXT:    sunpklo z2.s, z2.h
+; STREAMING-SVE-NEXT:    sunpklo z4.s, z4.h
+; STREAMING-SVE-NEXT:    sunpklo z7.s, z7.h
+; STREAMING-SVE-NEXT:    sunpklo z3.s, z3.h
 ; STREAMING-SVE-NEXT:    sunpklo z5.s, z5.h
-; STREAMING-SVE-NEXT:    add z1.s, z7.s, z1.s
-; STREAMING-SVE-NEXT:    add z0.s, z0.s, z5.s
-; STREAMING-SVE-NEXT:    add z1.s, z4.s, z1.s
-; STREAMING-SVE-NEXT:    add z0.s, z2.s, z0.s
-; STREAMING-SVE-NEXT:    add z0.s, z1.s, z0.s
+; STREAMING-SVE-NEXT:    sunpklo z6.s, z6.h
+; STREAMING-SVE-NEXT:    add z0.s, z0.s, z1.s
+; STREAMING-SVE-NEXT:    add z1.s, z3.s, z2.s
+; STREAMING-SVE-NEXT:    add z2.s, z7.s, z4.s
+; STREAMING-SVE-NEXT:    add z3.s, z6.s, z5.s
+; STREAMING-SVE-NEXT:    add z0.s, z0.s, z1.s
+; STREAMING-SVE-NEXT:    add z1.s, z2.s, z3.s
+; STREAMING-SVE-NEXT:    add z0.s, z0.s, z1.s
 ; STREAMING-SVE-NEXT:    uaddv d0, p0, z0.s
 ; STREAMING-SVE-NEXT:    fmov w0, s0
 ; STREAMING-SVE-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sve-vls-ldst-opt.mir b/llvm/test/CodeGen/AArch64/sve-vls-ldst-opt.mir
index 49453bc178914..a3e6cc10c1d0b 100644
--- a/llvm/test/CodeGen/AArch64/sve-vls-ldst-opt.mir
+++ b/llvm/test/CodeGen/AArch64/sve-vls-ldst-opt.mir
@@ -72,3 +72,21 @@ body: |
 # CHECK: STURQi killed renamable $q1, renamable $x1, 16 :: (store (s128))
 # CHECK: STURQi killed renamable $q2, renamable $x1, 48 :: (store (s128))
 # CHECK: STR_ZXI killed renamable $z3, renamable $x1, 4 :: (store (<vscale x 1 x s128>))
+---
+name: clear-kill-in-bundle-forward
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0, $z0
+    STR_ZXI $z0, $x0, 0 :: (store (<vscale x 1 x s128>))
+    BUNDLE implicit-def $z1, implicit killed $z0 {
+      $z1 = ADD_ZZZ_D killed $z0, $z0
+    }
+    STR_ZXI renamable $z1, $x0, 1 :: (store (<vscale x 1 x s128>))
+    RET_ReallyLR
+...
+# CHECK-LABEL: name: clear-kill-in-bundle-forward
+# CHECK: BUNDLE implicit-def $z1, implicit $z0 {
+# CHECK:   $z1 = ADD_ZZZ_D $z0, $z0
+# CHECK: }
+# CHECK: STPQi $q0, $q1, $x0, 0 :: (store (<vscale x 1 x s128>))
diff --git a/llvm/test/CodeGen/AArch64/sve2-bf16-converts.ll b/llvm/test/CodeGen/AArch64/sve2-bf16-converts.ll
index e5d4e1e9bc7da..ca0a2bf0a4915 100644
--- a/llvm/test/CodeGen/AArch64/sve2-bf16-converts.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-bf16-converts.ll
@@ -52,17 +52,17 @@ define <vscale x 4 x bfloat> @fptrunc_nxv4f64_to_nxv4bf16(<vscale x 4 x double>
 ; NOBF16-NEXT:    fcvtx z1.s, p0/m, z1.d
 ; NOBF16-NEXT:    fcvtx z0.s, p0/m, z0.d
 ; NOBF16-NEXT:    lsr z3.s, z1.s, #16
-; NOBF16-NEXT:    lsr z4.s, z0.s, #16
-; NOBF16-NEXT:    add z5.s, z1.s, z2.s
+; NOBF16-NEXT:    lsr z5.s, z0.s, #16
+; NOBF16-NEXT:    add z4.s, z1.s, z2.s
 ; NOBF16-NEXT:    add z2.s, z0.s, z2.s
 ; NOBF16-NEXT:    fcmuo p1.s, p0/z, z1.s, z1.s
-; NOBF16-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
 ; NOBF16-NEXT:    orr z1.s, z1.s, #0x400000
-; NOBF16-NEXT:    orr z0.s, z0.s, #0x400000
 ; NOBF16-NEXT:    and z3.s, z3.s, #0x1
-; NOBF16-NEXT:    and z4.s, z4.s, #0x1
-; NOBF16-NEXT:    add z3.s, z3.s, z5.s
-; NOBF16-NEXT:    add z2.s, z4.s, z2.s
+; NOBF16-NEXT:    and z5.s, z5.s, #0x1
+; NOBF16-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; NOBF16-NEXT:    orr z0.s, z0.s, #0x400000
+; NOBF16-NEXT:    add z3.s, z3.s, z4.s
+; NOBF16-NEXT:    add z2.s, z5.s, z2.s
 ; NOBF16-NEXT:    sel z1.s, p1, z1.s, z3.s
 ; NOBF16-NEXT:    sel z0.s, p0, z0.s, z2.s
 ; NOBF16-NEXT:    lsr z1.s, z1.s, #16
@@ -119,20 +119,20 @@ define <vscale x 8 x bfloat> @fptrunc_nxv8f64_to_nxv8bf16(<vscale x 8 x double>
 ; NOBF16-NEXT:    add z26.s, z2.s, z4.s
 ; NOBF16-NEXT:    add z27.s, z1.s, z4.s
 ; NOBF16-NEXT:    add z4.s, z0.s, z4.s
-; NOBF16-NEXT:    fcmuo p1.s, p0/z, z3.s, z3.s
 ; NOBF16-NEXT:    and z5.s, z5.s, #0x1
 ; NOBF16-NEXT:    and z6.s, z6.s, #0x1
 ; NOBF16-NEXT:    and z7.s, z7.s, #0x1
 ; NOBF16-NEXT:    and z24.s, z24.s, #0x1
-; NOBF16-NEXT:    fcmuo p2.s, p0/z, z2.s, z2.s
-; NOBF16-NEXT:    fcmuo p3.s, p0/z, z1.s, z1.s
-; NOBF16-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; NOBF16-NEXT:    fcmuo p1.s, p0/z, z3.s, z3.s
 ; NOBF16-NEXT:    orr z3.s, z3.s, #0x400000
-; NOBF16-NEXT:    orr z2.s, z2.s, #0x400000
+; NOBF16-NEXT:    fcmuo p2.s, p0/z, z2.s, z2.s
 ; NOBF16-NEXT:    add z5.s, z5.s, z25.s
 ; NOBF16-NEXT:    add z6.s, z6.s, z26.s
+; NOBF16-NEXT:    fcmuo p3.s, p0/z, z1.s, z1.s
 ; NOBF16-NEXT:    add z7.s, z7.s, z27.s
 ; NOBF16-NEXT:    add z4.s, z24.s, z4.s
+; NOBF16-NEXT:    fcmuo p0.s, p0/z, z0.s, z0.s
+; NOBF16-NEXT:    orr z2.s, z2.s, #0x400000
 ; NOBF16-NEXT:    orr z1.s, z1.s, #0x400000
 ; NOBF16-NEXT:    orr z0.s, z0.s, #0x400000
 ; NOBF16-NEXT:    sel z3.s, p1, z3.s, z5.s
diff --git a/llvm/test/CodeGen/AArch64/tail-dup-redundant-phi.mir b/llvm/test/CodeGen/AArch64/tail-dup-redundant-phi.mir
index b728eedfa5a92..034d226f4890b 100644
--- a/llvm/test/CodeGen/AArch64/tail-dup-redundant-phi.mir
+++ b/llvm/test/CodeGen/AArch64/tail-dup-redundant-phi.mir
@@ -172,8 +172,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/taildup-addrtaken.mir b/llvm/test/CodeGen/AArch64/taildup-addrtaken.mir
index 5acebceff291d..dfc0410a0adb1 100644
--- a/llvm/test/CodeGen/AArch64/taildup-addrtaken.mir
+++ b/llvm/test/CodeGen/AArch64/taildup-addrtaken.mir
@@ -75,8 +75,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/tbz-tbnz.ll b/llvm/test/CodeGen/AArch64/tbz-tbnz.ll
index 3fe7346b3db28..4a04934971711 100644
--- a/llvm/test/CodeGen/AArch64/tbz-tbnz.ll
+++ b/llvm/test/CodeGen/AArch64/tbz-tbnz.ll
@@ -200,18 +200,18 @@ define void @test8(i64 %val1, i64 %val2, i64 %val3) {
 ; CHECK-SD-LABEL: test8:
 ; CHECK-SD:       // %bb.0:
 ; CHECK-SD-NEXT:    tst x0, x1
-; CHECK-SD-NEXT:    b.ge .LBB7_3
+; CHECK-SD-NEXT:    b.pl .LBB7_3
 ; CHECK-SD-NEXT:  // %bb.1:
 ; CHECK-SD-NEXT:    and x8, x1, x2
 ; CHECK-SD-NEXT:    tbnz x8, #63, .LBB7_3
 ; CHECK-SD-NEXT:  // %bb.2: // %if.then2
 ; CHECK-SD-NEXT:    tst x0, x1, lsl #63
-; CHECK-SD-NEXT:    b.lt .LBB7_4
+; CHECK-SD-NEXT:    b.mi .LBB7_4
 ; CHECK-SD-NEXT:  .LBB7_3: // %if.end
 ; CHECK-SD-NEXT:    ret
 ; CHECK-SD-NEXT:  .LBB7_4: // %if.then3
 ; CHECK-SD-NEXT:    tst x0, x1, lsl #62
-; CHECK-SD-NEXT:    b.lt .LBB7_3
+; CHECK-SD-NEXT:    b.mi .LBB7_3
 ; CHECK-SD-NEXT:  // %bb.5: // %if.then4
 ; CHECK-SD-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
@@ -223,18 +223,18 @@ define void @test8(i64 %val1, i64 %val2, i64 %val3) {
 ; CHECK-GI-LABEL: test8:
 ; CHECK-GI:       // %bb.0:
 ; CHECK-GI-NEXT:    tst x0, x1
-; CHECK-GI-NEXT:    b.ge .LBB7_3
+; CHECK-GI-NEXT:    b.pl .LBB7_3
 ; CHECK-GI-NEXT:  // %bb.1:
 ; CHECK-GI-NEXT:    tst x1, x2
-; CHECK-GI-NEXT:    b.lt .LBB7_3
+; CHECK-GI-NEXT:    b.mi .LBB7_3
 ; CHECK-GI-NEXT:  // %bb.2: // %if.then2
 ; CHECK-GI-NEXT:    tst x0, x1, lsl #63
-; CHECK-GI-NEXT:    b.lt .LBB7_4
+; CHECK-GI-NEXT:    b.mi .LBB7_4
 ; CHECK-GI-NEXT:  .LBB7_3: // %if.end
 ; CHECK-GI-NEXT:    ret
 ; CHECK-GI-NEXT:  .LBB7_4: // %if.then3
 ; CHECK-GI-NEXT:    tst x0, x1, lsl #62
-; CHECK-GI-NEXT:    b.lt .LBB7_3
+; CHECK-GI-NEXT:    b.mi .LBB7_3
 ; CHECK-GI-NEXT:  // %bb.5: // %if.then4
 ; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-GI-NEXT:    .cfi_def_cfa_offset 16
diff --git a/llvm/test/CodeGen/AArch64/trunc-avg-fold.ll b/llvm/test/CodeGen/AArch64/trunc-avg-fold.ll
new file mode 100644
index 0000000000000..54fcae4ba28b7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/trunc-avg-fold.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-- -O2 -mattr=+neon < %s | FileCheck %s
+
+define <8 x i8> @avgceil_u_i8_to_i16(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: avgceil_u_i8_to_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    urhadd v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %a16 = zext <8 x i8> %a to <8 x i16>
+  %b16 = zext <8 x i8> %b to <8 x i16>
+  %avg16 = call <8 x i16> @llvm.aarch64.neon.urhadd.v8i16(<8 x i16> %a16, <8 x i16> %b16)
+  %r = trunc <8 x i16> %avg16 to <8 x i8>
+  ret <8 x i8> %r
+}
+
+
+define <8 x i8> @test_avgceil_s(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: test_avgceil_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    srhadd v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %a16 = sext <8 x i8> %a to <8 x i16>
+  %b16 = sext <8 x i8> %b to <8 x i16>
+  %avg16 = call <8 x i16> @llvm.aarch64.neon.srhadd.v8i16(<8 x i16> %a16, <8 x i16> %b16)
+  %res  = trunc <8 x i16> %avg16 to <8 x i8>
+  ret <8 x i8> %res
+}
+
+define <8 x i8> @avgfloor_u_i8_to_i16(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: avgfloor_u_i8_to_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uhadd v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %a16 = zext  <8 x i8>  %a to <8 x i16>
+  %b16 = zext  <8 x i8>  %b to <8 x i16>
+  %avg16 = call <8 x i16> @llvm.aarch64.neon.uhadd.v8i16(<8 x i16> %a16, <8 x i16> %b16)
+  %res = trunc <8 x i16> %avg16 to <8 x i8>
+  ret <8 x i8> %res
+}
+
+define <8 x i8> @test_avgfloor_s(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: test_avgfloor_s:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    shadd v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ret
+  %a16 = sext  <8 x i8>  %a to <8 x i16>
+  %b16 = sext  <8 x i8>  %b to <8 x i16>
+  %avg16 = call <8 x i16> @llvm.aarch64.neon.shadd.v8i16(<8 x i16> %a16, <8 x i16> %b16)
+  %res  = trunc <8 x i16> %avg16 to <8 x i8>
+  ret <8 x i8> %res
+}
+
+
diff --git a/llvm/test/CodeGen/AArch64/urem-seteq-optsize.ll b/llvm/test/CodeGen/AArch64/urem-seteq-optsize.ll
index 45726e92463b9..bb5aa1fd0684d 100644
--- a/llvm/test/CodeGen/AArch64/urem-seteq-optsize.ll
+++ b/llvm/test/CodeGen/AArch64/urem-seteq-optsize.ll
@@ -22,14 +22,13 @@ define i32 @test_optsize(i32 %X) optsize nounwind readnone {
 ; CHECK-LABEL: test_optsize:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #52429 // =0xcccd
-; CHECK-NEXT:    mov w9, #13108 // =0x3334
+; CHECK-NEXT:    mov w9, #858993459 // =0x33333333
 ; CHECK-NEXT:    movk w8, #52428, lsl #16
-; CHECK-NEXT:    movk w9, #13107, lsl #16
 ; CHECK-NEXT:    mul w8, w0, w8
 ; CHECK-NEXT:    cmp w8, w9
 ; CHECK-NEXT:    mov w8, #-10 // =0xfffffff6
 ; CHECK-NEXT:    mov w9, #42 // =0x2a
-; CHECK-NEXT:    csel w0, w9, w8, lo
+; CHECK-NEXT:    csel w0, w9, w8, ls
 ; CHECK-NEXT:    ret
   %rem = urem i32 %X, 5
   %cmp = icmp eq i32 %rem, 0
diff --git a/llvm/test/CodeGen/AArch64/urem-seteq.ll b/llvm/test/CodeGen/AArch64/urem-seteq.ll
index df87e60c4f8d5..5473991e77c3e 100644
--- a/llvm/test/CodeGen/AArch64/urem-seteq.ll
+++ b/llvm/test/CodeGen/AArch64/urem-seteq.ll
@@ -9,12 +9,11 @@ define i32 @test_urem_odd(i32 %X) nounwind {
 ; CHECK-LABEL: test_urem_odd:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov w8, #52429 // =0xcccd
-; CHECK-NEXT:    mov w9, #13108 // =0x3334
+; CHECK-NEXT:    mov w9, #858993459 // =0x33333333
 ; CHECK-NEXT:    movk w8, #52428, lsl #16
-; CHECK-NEXT:    movk w9, #13107, lsl #16
 ; CHECK-NEXT:    mul w8, w0, w8
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    cset w0, ls
 ; CHECK-NEXT:    ret
   %urem = urem i32 %X, 5
   %cmp = icmp eq i32 %urem, 0
diff --git a/llvm/test/CodeGen/AArch64/urem-vector-lkk.ll b/llvm/test/CodeGen/AArch64/urem-vector-lkk.ll
index 468a33ce5bfcf..4be8c3775c704 100644
--- a/llvm/test/CodeGen/AArch64/urem-vector-lkk.ll
+++ b/llvm/test/CodeGen/AArch64/urem-vector-lkk.ll
@@ -88,7 +88,6 @@ define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) {
 ; CHECK-LABEL: dont_fold_urem_one:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    adrp x8, .LCPI4_0
-; CHECK-NEXT:    movi d4, #0x0000000000ffff
 ; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI4_0]
 ; CHECK-NEXT:    adrp x8, .LCPI4_1
 ; CHECK-NEXT:    ldr d3, [x8, :lo12:.LCPI4_1]
@@ -97,16 +96,14 @@ define <4 x i16> @dont_fold_urem_one(<4 x i16> %x) {
 ; CHECK-NEXT:    shrn v1.4h, v1.4s, #16
 ; CHECK-NEXT:    sub v2.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    umull v2.4s, v2.4h, v3.4h
-; CHECK-NEXT:    movi d3, #0xffffffffffff0000
+; CHECK-NEXT:    movi d3, #0x0000000000ffff
 ; CHECK-NEXT:    shrn v2.4h, v2.4s, #16
 ; CHECK-NEXT:    add v1.4h, v2.4h, v1.4h
 ; CHECK-NEXT:    ldr d2, [x8, :lo12:.LCPI4_2]
 ; CHECK-NEXT:    adrp x8, .LCPI4_3
 ; CHECK-NEXT:    ushl v1.4h, v1.4h, v2.4h
-; CHECK-NEXT:    and v2.8b, v0.8b, v4.8b
-; CHECK-NEXT:    and v1.8b, v1.8b, v3.8b
-; CHECK-NEXT:    orr v1.8b, v2.8b, v1.8b
 ; CHECK-NEXT:    ldr d2, [x8, :lo12:.LCPI4_3]
+; CHECK-NEXT:    bit v1.8b, v0.8b, v3.8b
 ; CHECK-NEXT:    mls v0.4h, v1.4h, v2.4h
 ; CHECK-NEXT:    ret
   %1 = urem <4 x i16> %x, <i16 1, i16 654, i16 23, i16 5423>
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-bool.ll b/llvm/test/CodeGen/AArch64/vecreduce-bool.ll
index 1bdf7bbb7f813..62d41fca10db3 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-bool.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-bool.ll
@@ -28,7 +28,7 @@ define i32 @reduce_and_v1i8(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    smov w8, v0.b[0]
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i8> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
@@ -122,7 +122,7 @@ define i32 @reduce_and_v1i16(<1 x i16> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    smov w8, v0.h[0]
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i16> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
@@ -200,7 +200,7 @@ define i32 @reduce_and_v1i32(<1 x i32> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i32> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
@@ -261,7 +261,7 @@ define i32 @reduce_and_v1i64(<1 x i64> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    fmov x8, d0
 ; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i64> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.and.v1i1(<1 x i1> %x)
@@ -307,7 +307,7 @@ define i32 @reduce_or_v1i8(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    smov w8, v0.b[0]
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i8> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
@@ -401,7 +401,7 @@ define i32 @reduce_or_v1i16(<1 x i16> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    smov w8, v0.h[0]
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i16> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
@@ -479,7 +479,7 @@ define i32 @reduce_or_v1i32(<1 x i32> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i32> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
@@ -540,7 +540,7 @@ define i32 @reduce_or_v1i64(<1 x i64> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    fmov x8, d0
 ; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i64> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.or.v1i1(<1 x i1> %x)
@@ -586,7 +586,7 @@ define i32 @reduce_xor_v1i8(<1 x i8> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    smov w8, v0.b[0]
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i8> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
@@ -681,7 +681,7 @@ define i32 @reduce_xor_v1i16(<1 x i16> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    smov w8, v0.h[0]
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i16> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
@@ -759,7 +759,7 @@ define i32 @reduce_xor_v1i32(<1 x i32> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    fmov w8, s0
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i32> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
@@ -820,7 +820,7 @@ define i32 @reduce_xor_v1i64(<1 x i64> %a0, i32 %a1, i32 %a2) nounwind {
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
 ; CHECK-NEXT:    fmov x8, d0
 ; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    csel w0, w0, w1, lt
+; CHECK-NEXT:    csel w0, w0, w1, mi
 ; CHECK-NEXT:    ret
   %x = icmp slt <1 x i64> %a0, zeroinitializer
   %y = call i1 @llvm.vector.reduce.xor.v1i1(<1 x i1> %x)
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization-strict.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization-strict.ll
index 215b6e086591d..3fc8d6f78296c 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization-strict.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization-strict.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -global-isel -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
 ; Same as vecreduce-fadd-legalization.ll, but without fmf.
 
@@ -14,13 +15,21 @@ declare fp128 @llvm.vector.reduce.fadd.f128.v2f128(fp128, <2 x fp128>)
 declare float @llvm.vector.reduce.fadd.f32.v16f32(float, <16 x float>)
 
 define half @test_v1f16(<1 x half> %a, half %s) nounwind {
-; CHECK-LABEL: test_v1f16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fcvt s0, h0
-; CHECK-NEXT:    fcvt s1, h1
-; CHECK-NEXT:    fadd s0, s1, s0
-; CHECK-NEXT:    fcvt h0, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_v1f16:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fcvt s0, h0
+; CHECK-SD-NEXT:    fcvt s1, h1
+; CHECK-SD-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NEXT:    fcvt h0, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v1f16:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fcvt s1, h1
+; CHECK-GI-NEXT:    fcvt s0, h0
+; CHECK-GI-NEXT:    fadd s0, s1, s0
+; CHECK-GI-NEXT:    fcvt h0, s0
+; CHECK-GI-NEXT:    ret
   %b = call half @llvm.vector.reduce.fadd.f16.v1f16(half %s, <1 x half> %a)
   ret half %b
 }
@@ -34,21 +43,31 @@ define half @test_v1f16_neutral(<1 x half> %a) nounwind {
 }
 
 define float @test_v1f32(<1 x float> %a, float %s) nounwind {
-; CHECK-LABEL: test_v1f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    fadd s0, s1, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_v1f32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v1f32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fadd s0, s1, s0
+; CHECK-GI-NEXT:    ret
   %b = call float @llvm.vector.reduce.fadd.f32.v1f32(float %s, <1 x float> %a)
   ret float %b
 }
 
 define float @test_v1f32_neutral(<1 x float> %a) nounwind {
-; CHECK-LABEL: test_v1f32_neutral:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_v1f32_neutral:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v1f32_neutral:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $d0
+; CHECK-GI-NEXT:    ret
   %b = call float @llvm.vector.reduce.fadd.f32.v1f32(float -0.0, <1 x float> %a)
   ret float %b
 }
@@ -90,14 +109,23 @@ define fp128 @test_v1f128_neutral(<1 x fp128> %a) nounwind {
 }
 
 define float @test_v3f32(<3 x float> %a, float %s) nounwind {
-; CHECK-LABEL: test_v3f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd s1, s1, s0
-; CHECK-NEXT:    mov s2, v0.s[1]
-; CHECK-NEXT:    mov s0, v0.s[2]
-; CHECK-NEXT:    fadd s1, s1, s2
-; CHECK-NEXT:    fadd s0, s1, s0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_v3f32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd s1, s1, s0
+; CHECK-SD-NEXT:    mov s2, v0.s[1]
+; CHECK-SD-NEXT:    mov s0, v0.s[2]
+; CHECK-SD-NEXT:    fadd s1, s1, s2
+; CHECK-SD-NEXT:    fadd s0, s1, s0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v3f32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov s2, v0.s[1]
+; CHECK-GI-NEXT:    fadd s1, s1, s0
+; CHECK-GI-NEXT:    mov s0, v0.s[2]
+; CHECK-GI-NEXT:    fadd s1, s1, s2
+; CHECK-GI-NEXT:    fadd s0, s1, s0
+; CHECK-GI-NEXT:    ret
   %b = call float @llvm.vector.reduce.fadd.f32.v3f32(float %s, <3 x float> %a)
   ret float %b
 }
@@ -139,96 +167,189 @@ define float @test_v5f32_neutral(<5 x float> %a) nounwind {
 }
 
 define fp128 @test_v2f128(<2 x fp128> %a, fp128 %s) nounwind {
-; CHECK-LABEL: test_v2f128:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #32
-; CHECK-NEXT:    str q1, [sp] // 16-byte Folded Spill
-; CHECK-NEXT:    mov v1.16b, v0.16b
-; CHECK-NEXT:    mov v0.16b, v2.16b
-; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
-; CHECK-NEXT:    bl __addtf3
-; CHECK-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #32
-; CHECK-NEXT:    b __addtf3
+; CHECK-SD-LABEL: test_v2f128:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    sub sp, sp, #32
+; CHECK-SD-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-SD-NEXT:    mov v1.16b, v0.16b
+; CHECK-SD-NEXT:    mov v0.16b, v2.16b
+; CHECK-SD-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-SD-NEXT:    bl __addtf3
+; CHECK-SD-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-SD-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-SD-NEXT:    add sp, sp, #32
+; CHECK-SD-NEXT:    b __addtf3
+;
+; CHECK-GI-LABEL: test_v2f128:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    sub sp, sp, #32
+; CHECK-GI-NEXT:    mov v3.16b, v0.16b
+; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    mov v1.16b, v3.16b
+; CHECK-GI-NEXT:    bl __addtf3
+; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    bl __addtf3
+; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    add sp, sp, #32
+; CHECK-GI-NEXT:    ret
   %b = call fp128 @llvm.vector.reduce.fadd.f128.v2f128(fp128 %s, <2 x fp128> %a)
   ret fp128 %b
 }
 
 define fp128 @test_v2f128_neutral(<2 x fp128> %a) nounwind {
-; CHECK-LABEL: test_v2f128_neutral:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    b __addtf3
+; CHECK-SD-LABEL: test_v2f128_neutral:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    b __addtf3
+;
+; CHECK-GI-LABEL: test_v2f128_neutral:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    sub sp, sp, #32
+; CHECK-GI-NEXT:    mov v2.16b, v0.16b
+; CHECK-GI-NEXT:    adrp x8, .LCPI13_0
+; CHECK-GI-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-GI-NEXT:    ldr q0, [x8, :lo12:.LCPI13_0]
+; CHECK-GI-NEXT:    str x30, [sp, #16] // 8-byte Folded Spill
+; CHECK-GI-NEXT:    mov v1.16b, v2.16b
+; CHECK-GI-NEXT:    bl __addtf3
+; CHECK-GI-NEXT:    ldr q1, [sp] // 16-byte Folded Reload
+; CHECK-GI-NEXT:    bl __addtf3
+; CHECK-GI-NEXT:    ldr x30, [sp, #16] // 8-byte Folded Reload
+; CHECK-GI-NEXT:    add sp, sp, #32
+; CHECK-GI-NEXT:    ret
   %b = call fp128 @llvm.vector.reduce.fadd.f128.v2f128(fp128 0xL00000000000000008000000000000000, <2 x fp128> %a)
   ret fp128 %b
 }
 
 define float @test_v16f32(<16 x float> %a, float %s) nounwind {
-; CHECK-LABEL: test_v16f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov s6, v0.s[1]
-; CHECK-NEXT:    fadd s4, s4, s0
-; CHECK-NEXT:    mov s7, v0.s[2]
-; CHECK-NEXT:    mov s0, v0.s[3]
-; CHECK-NEXT:    mov s5, v2.s[1]
-; CHECK-NEXT:    fadd s4, s4, s6
-; CHECK-NEXT:    mov s6, v1.s[2]
-; CHECK-NEXT:    fadd s4, s4, s7
-; CHECK-NEXT:    fadd s0, s4, s0
-; CHECK-NEXT:    mov s4, v1.s[1]
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s1, v1.s[3]
-; CHECK-NEXT:    fadd s0, s0, s4
-; CHECK-NEXT:    fadd s0, s0, s6
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s1, v2.s[2]
-; CHECK-NEXT:    fadd s0, s0, s2
-; CHECK-NEXT:    mov s2, v2.s[3]
-; CHECK-NEXT:    fadd s0, s0, s5
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s1, v3.s[1]
-; CHECK-NEXT:    fadd s0, s0, s2
-; CHECK-NEXT:    mov s2, v3.s[2]
-; CHECK-NEXT:    fadd s0, s0, s3
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s1, v3.s[3]
-; CHECK-NEXT:    fadd s0, s0, s2
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_v16f32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    mov s6, v0.s[1]
+; CHECK-SD-NEXT:    fadd s4, s4, s0
+; CHECK-SD-NEXT:    mov s7, v0.s[2]
+; CHECK-SD-NEXT:    mov s0, v0.s[3]
+; CHECK-SD-NEXT:    mov s5, v2.s[1]
+; CHECK-SD-NEXT:    fadd s4, s4, s6
+; CHECK-SD-NEXT:    mov s6, v1.s[2]
+; CHECK-SD-NEXT:    fadd s4, s4, s7
+; CHECK-SD-NEXT:    fadd s0, s4, s0
+; CHECK-SD-NEXT:    mov s4, v1.s[1]
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    mov s1, v1.s[3]
+; CHECK-SD-NEXT:    fadd s0, s0, s4
+; CHECK-SD-NEXT:    fadd s0, s0, s6
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    mov s1, v2.s[2]
+; CHECK-SD-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NEXT:    mov s2, v2.s[3]
+; CHECK-SD-NEXT:    fadd s0, s0, s5
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    mov s1, v3.s[1]
+; CHECK-SD-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NEXT:    mov s2, v3.s[2]
+; CHECK-SD-NEXT:    fadd s0, s0, s3
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    mov s1, v3.s[3]
+; CHECK-SD-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v16f32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov s5, v0.s[1]
+; CHECK-GI-NEXT:    fadd s4, s4, s0
+; CHECK-GI-NEXT:    mov s6, v0.s[2]
+; CHECK-GI-NEXT:    mov s0, v0.s[3]
+; CHECK-GI-NEXT:    fadd s4, s4, s5
+; CHECK-GI-NEXT:    mov s5, v1.s[2]
+; CHECK-GI-NEXT:    fadd s4, s4, s6
+; CHECK-GI-NEXT:    fadd s0, s4, s0
+; CHECK-GI-NEXT:    mov s4, v1.s[1]
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v1.s[3]
+; CHECK-GI-NEXT:    fadd s0, s0, s4
+; CHECK-GI-NEXT:    mov s4, v2.s[2]
+; CHECK-GI-NEXT:    fadd s0, s0, s5
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v2.s[1]
+; CHECK-GI-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v2.s[3]
+; CHECK-GI-NEXT:    mov s2, v3.s[2]
+; CHECK-GI-NEXT:    fadd s0, s0, s4
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v3.s[1]
+; CHECK-GI-NEXT:    fadd s0, s0, s3
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v3.s[3]
+; CHECK-GI-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %b = call float @llvm.vector.reduce.fadd.f32.v16f32(float %s, <16 x float> %a)
   ret float %b
 }
 
 define float @test_v16f32_neutral(<16 x float> %a) nounwind {
-; CHECK-LABEL: test_v16f32_neutral:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov s5, v0.s[2]
-; CHECK-NEXT:    faddp s6, v0.2s
-; CHECK-NEXT:    mov s0, v0.s[3]
-; CHECK-NEXT:    mov s4, v1.s[1]
-; CHECK-NEXT:    fadd s5, s6, s5
-; CHECK-NEXT:    fadd s0, s5, s0
-; CHECK-NEXT:    mov s5, v1.s[2]
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s1, v1.s[3]
-; CHECK-NEXT:    fadd s0, s0, s4
-; CHECK-NEXT:    mov s4, v2.s[2]
-; CHECK-NEXT:    fadd s0, s0, s5
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s1, v2.s[1]
-; CHECK-NEXT:    fadd s0, s0, s2
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s1, v2.s[3]
-; CHECK-NEXT:    mov s2, v3.s[2]
-; CHECK-NEXT:    fadd s0, s0, s4
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s1, v3.s[1]
-; CHECK-NEXT:    fadd s0, s0, s3
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    mov s1, v3.s[3]
-; CHECK-NEXT:    fadd s0, s0, s2
-; CHECK-NEXT:    fadd s0, s0, s1
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_v16f32_neutral:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    mov s5, v0.s[2]
+; CHECK-SD-NEXT:    faddp s6, v0.2s
+; CHECK-SD-NEXT:    mov s0, v0.s[3]
+; CHECK-SD-NEXT:    mov s4, v1.s[1]
+; CHECK-SD-NEXT:    fadd s5, s6, s5
+; CHECK-SD-NEXT:    fadd s0, s5, s0
+; CHECK-SD-NEXT:    mov s5, v1.s[2]
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    mov s1, v1.s[3]
+; CHECK-SD-NEXT:    fadd s0, s0, s4
+; CHECK-SD-NEXT:    mov s4, v2.s[2]
+; CHECK-SD-NEXT:    fadd s0, s0, s5
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    mov s1, v2.s[1]
+; CHECK-SD-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    mov s1, v2.s[3]
+; CHECK-SD-NEXT:    mov s2, v3.s[2]
+; CHECK-SD-NEXT:    fadd s0, s0, s4
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    mov s1, v3.s[1]
+; CHECK-SD-NEXT:    fadd s0, s0, s3
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    mov s1, v3.s[3]
+; CHECK-SD-NEXT:    fadd s0, s0, s2
+; CHECK-SD-NEXT:    fadd s0, s0, s1
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v16f32_neutral:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    mov s4, v0.s[2]
+; CHECK-GI-NEXT:    faddp s5, v0.2s
+; CHECK-GI-NEXT:    mov s0, v0.s[3]
+; CHECK-GI-NEXT:    fadd s4, s5, s4
+; CHECK-GI-NEXT:    mov s5, v1.s[2]
+; CHECK-GI-NEXT:    fadd s0, s4, s0
+; CHECK-GI-NEXT:    mov s4, v1.s[1]
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v1.s[3]
+; CHECK-GI-NEXT:    fadd s0, s0, s4
+; CHECK-GI-NEXT:    mov s4, v2.s[2]
+; CHECK-GI-NEXT:    fadd s0, s0, s5
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v2.s[1]
+; CHECK-GI-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v2.s[3]
+; CHECK-GI-NEXT:    mov s2, v3.s[2]
+; CHECK-GI-NEXT:    fadd s0, s0, s4
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v3.s[1]
+; CHECK-GI-NEXT:    fadd s0, s0, s3
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    mov s1, v3.s[3]
+; CHECK-GI-NEXT:    fadd s0, s0, s2
+; CHECK-GI-NEXT:    fadd s0, s0, s1
+; CHECK-GI-NEXT:    ret
   %b = call float @llvm.vector.reduce.fadd.f32.v16f32(float -0.0, <16 x float> %a)
   ret float %b
 }
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization.ll
index a2e5a8a1b4c46..13005c58a4a95 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd-legalization.ll
@@ -1,5 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:       warning: Instruction selection used fallback path for test_v5f32
 
 declare half @llvm.vector.reduce.fadd.f16.v1f16(half, <1 x half>)
 declare float @llvm.vector.reduce.fadd.f32.v1f32(float, <1 x float>)
@@ -20,11 +23,16 @@ define half @test_v1f16(<1 x half> %a) nounwind {
 }
 
 define float @test_v1f32(<1 x float> %a) nounwind {
-; CHECK-LABEL: test_v1f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_v1f32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v1f32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    // kill: def $s0 killed $s0 killed $d0
+; CHECK-GI-NEXT:    ret
   %b = call reassoc float @llvm.vector.reduce.fadd.f32.v1f32(float -0.0, <1 x float> %a)
   ret float %b
 }
@@ -79,22 +87,38 @@ define float @test_v5f32(<5 x float> %a) nounwind {
 }
 
 define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
-; CHECK-LABEL: test_v2f128:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    b __addtf3
+; CHECK-SD-LABEL: test_v2f128:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    b __addtf3
+;
+; CHECK-GI-LABEL: test_v2f128:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-GI-NEXT:    bl __addtf3
+; CHECK-GI-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-GI-NEXT:    ret
   %b = call reassoc fp128 @llvm.vector.reduce.fadd.f128.v2f128(fp128 0xL00000000000000008000000000000000, <2 x fp128> %a)
   ret fp128 %b
 }
 
 define float @test_v16f32(<16 x float> %a) nounwind {
-; CHECK-LABEL: test_v16f32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    fadd v1.4s, v1.4s, v3.4s
-; CHECK-NEXT:    fadd v0.4s, v0.4s, v2.4s
-; CHECK-NEXT:    fadd v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    faddp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    faddp s0, v0.2s
-; CHECK-NEXT:    ret
+; CHECK-SD-LABEL: test_v16f32:
+; CHECK-SD:       // %bb.0:
+; CHECK-SD-NEXT:    fadd v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT:    fadd v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-SD-NEXT:    faddp s0, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: test_v16f32:
+; CHECK-GI:       // %bb.0:
+; CHECK-GI-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    fadd v1.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT:    fadd v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    faddp v0.4s, v0.4s, v0.4s
+; CHECK-GI-NEXT:    faddp s0, v0.2s
+; CHECK-GI-NEXT:    ret
   %b = call reassoc float @llvm.vector.reduce.fadd.f32.v16f32(float -0.0, <16 x float> %a)
   ret float %b
 }
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
index 59dfcf9850a49..18f463cfcf7c9 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
@@ -208,6 +208,15 @@ define double @add_D(<2 x double> %bin.rdx)  {
   ret double %r
 }
 
+define double @add_D_pos0(<2 x double> %bin.rdx)  {
+; CHECK-LABEL: add_D_pos0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    faddp d0, v0.2d
+; CHECK-NEXT:    ret
+  %r = call fast double @llvm.vector.reduce.fadd.f64.v2f64(double 0.0, <2 x double> %bin.rdx)
+  ret double %r
+}
+
 define half @add_2H(<16 x half> %bin.rdx)  {
 ; CHECK-SD-NOFP16-LABEL: add_2H:
 ; CHECK-SD-NOFP16:       // %bb.0:
@@ -319,7 +328,7 @@ define float @fadd_reduction_v4f32_in_loop(ptr %ptr.start) {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi d0, #0000000000000000
 ; CHECK-NEXT:    mov x8, xzr
-; CHECK-NEXT:  .LBB11_1: // %loop
+; CHECK-NEXT:  .LBB12_1: // %loop
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    ldr q1, [x0, x8]
 ; CHECK-NEXT:    add x8, x8, #16
@@ -327,7 +336,7 @@ define float @fadd_reduction_v4f32_in_loop(ptr %ptr.start) {
 ; CHECK-NEXT:    faddp v1.4s, v1.4s, v1.4s
 ; CHECK-NEXT:    faddp s1, v1.2s
 ; CHECK-NEXT:    fadd s0, s1, s0
-; CHECK-NEXT:    b.ne .LBB11_1
+; CHECK-NEXT:    b.ne .LBB12_1
 ; CHECK-NEXT:  // %bb.2: // %exit
 ; CHECK-NEXT:    ret
 entry:
@@ -356,7 +365,7 @@ define half @fadd_reduction_v4f16_in_loop(ptr %ptr.start) {
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
 ; CHECK-SD-NOFP16-NEXT:    movi d0, #0000000000000000
 ; CHECK-SD-NOFP16-NEXT:    mov x8, xzr
-; CHECK-SD-NOFP16-NEXT:  .LBB12_1: // %loop
+; CHECK-SD-NOFP16-NEXT:  .LBB13_1: // %loop
 ; CHECK-SD-NOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-SD-NOFP16-NEXT:    ldr d1, [x0, x8]
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
@@ -374,7 +383,7 @@ define half @fadd_reduction_v4f16_in_loop(ptr %ptr.start) {
 ; CHECK-SD-NOFP16-NEXT:    fadd s1, s1, s2
 ; CHECK-SD-NOFP16-NEXT:    fadd s0, s1, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    b.ne .LBB12_1
+; CHECK-SD-NOFP16-NEXT:    b.ne .LBB13_1
 ; CHECK-SD-NOFP16-NEXT:  // %bb.2: // %exit
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -382,7 +391,7 @@ define half @fadd_reduction_v4f16_in_loop(ptr %ptr.start) {
 ; CHECK-SD-FP16:       // %bb.0: // %entry
 ; CHECK-SD-FP16-NEXT:    movi d0, #0000000000000000
 ; CHECK-SD-FP16-NEXT:    mov x8, xzr
-; CHECK-SD-FP16-NEXT:  .LBB12_1: // %loop
+; CHECK-SD-FP16-NEXT:  .LBB13_1: // %loop
 ; CHECK-SD-FP16-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-SD-FP16-NEXT:    ldr d1, [x0, x8]
 ; CHECK-SD-FP16-NEXT:    add x8, x8, #8
@@ -390,7 +399,7 @@ define half @fadd_reduction_v4f16_in_loop(ptr %ptr.start) {
 ; CHECK-SD-FP16-NEXT:    faddp v1.4h, v1.4h, v1.4h
 ; CHECK-SD-FP16-NEXT:    faddp h1, v1.2h
 ; CHECK-SD-FP16-NEXT:    fadd h0, h1, h0
-; CHECK-SD-FP16-NEXT:    b.ne .LBB12_1
+; CHECK-SD-FP16-NEXT:    b.ne .LBB13_1
 ; CHECK-SD-FP16-NEXT:  // %bb.2: // %exit
 ; CHECK-SD-FP16-NEXT:    ret
 ;
@@ -398,7 +407,7 @@ define half @fadd_reduction_v4f16_in_loop(ptr %ptr.start) {
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
 ; CHECK-GI-NOFP16-NEXT:    mov x8, xzr
 ; CHECK-GI-NOFP16-NEXT:    mov w9, #0 // =0x0
-; CHECK-GI-NOFP16-NEXT:  .LBB12_1: // %loop
+; CHECK-GI-NOFP16-NEXT:  .LBB13_1: // %loop
 ; CHECK-GI-NOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-GI-NOFP16-NEXT:    ldr d0, [x0, x8]
 ; CHECK-GI-NOFP16-NEXT:    fmov s1, w9
@@ -413,7 +422,7 @@ define half @fadd_reduction_v4f16_in_loop(ptr %ptr.start) {
 ; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
 ; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-GI-NOFP16-NEXT:    fmov w9, s0
-; CHECK-GI-NOFP16-NEXT:    b.ne .LBB12_1
+; CHECK-GI-NOFP16-NEXT:    b.ne .LBB13_1
 ; CHECK-GI-NOFP16-NEXT:  // %bb.2: // %exit
 ; CHECK-GI-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-GI-NOFP16-NEXT:    ret
@@ -422,7 +431,7 @@ define half @fadd_reduction_v4f16_in_loop(ptr %ptr.start) {
 ; CHECK-GI-FP16:       // %bb.0: // %entry
 ; CHECK-GI-FP16-NEXT:    movi d0, #0000000000000000
 ; CHECK-GI-FP16-NEXT:    mov x8, xzr
-; CHECK-GI-FP16-NEXT:  .LBB12_1: // %loop
+; CHECK-GI-FP16-NEXT:  .LBB13_1: // %loop
 ; CHECK-GI-FP16-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-GI-FP16-NEXT:    ldr d1, [x0, x8]
 ; CHECK-GI-FP16-NEXT:    add x8, x8, #8
@@ -430,7 +439,7 @@ define half @fadd_reduction_v4f16_in_loop(ptr %ptr.start) {
 ; CHECK-GI-FP16-NEXT:    faddp v1.4h, v1.4h, v1.4h
 ; CHECK-GI-FP16-NEXT:    faddp h1, v1.2h
 ; CHECK-GI-FP16-NEXT:    fadd h0, h1, h0
-; CHECK-GI-FP16-NEXT:    b.ne .LBB12_1
+; CHECK-GI-FP16-NEXT:    b.ne .LBB13_1
 ; CHECK-GI-FP16-NEXT:  // %bb.2: // %exit
 ; CHECK-GI-FP16-NEXT:    ret
 entry:
@@ -459,7 +468,7 @@ define half @fadd_reduction_v8f16_in_loop(ptr %ptr.start) {
 ; CHECK-SD-NOFP16:       // %bb.0: // %entry
 ; CHECK-SD-NOFP16-NEXT:    movi d0, #0000000000000000
 ; CHECK-SD-NOFP16-NEXT:    mov x8, xzr
-; CHECK-SD-NOFP16-NEXT:  .LBB13_1: // %loop
+; CHECK-SD-NOFP16-NEXT:  .LBB14_1: // %loop
 ; CHECK-SD-NOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-SD-NOFP16-NEXT:    ldr q1, [x0, x8]
 ; CHECK-SD-NOFP16-NEXT:    fcvt s0, h0
@@ -489,7 +498,7 @@ define half @fadd_reduction_v8f16_in_loop(ptr %ptr.start) {
 ; CHECK-SD-NOFP16-NEXT:    fadd s2, s2, s3
 ; CHECK-SD-NOFP16-NEXT:    fadd s0, s2, s0
 ; CHECK-SD-NOFP16-NEXT:    fcvt h0, s0
-; CHECK-SD-NOFP16-NEXT:    b.ne .LBB13_1
+; CHECK-SD-NOFP16-NEXT:    b.ne .LBB14_1
 ; CHECK-SD-NOFP16-NEXT:  // %bb.2: // %exit
 ; CHECK-SD-NOFP16-NEXT:    ret
 ;
@@ -497,7 +506,7 @@ define half @fadd_reduction_v8f16_in_loop(ptr %ptr.start) {
 ; CHECK-SD-FP16:       // %bb.0: // %entry
 ; CHECK-SD-FP16-NEXT:    movi d0, #0000000000000000
 ; CHECK-SD-FP16-NEXT:    mov x8, xzr
-; CHECK-SD-FP16-NEXT:  .LBB13_1: // %loop
+; CHECK-SD-FP16-NEXT:  .LBB14_1: // %loop
 ; CHECK-SD-FP16-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-SD-FP16-NEXT:    ldr q1, [x0, x8]
 ; CHECK-SD-FP16-NEXT:    add x8, x8, #8
@@ -506,7 +515,7 @@ define half @fadd_reduction_v8f16_in_loop(ptr %ptr.start) {
 ; CHECK-SD-FP16-NEXT:    faddp v1.8h, v2.8h, v1.8h
 ; CHECK-SD-FP16-NEXT:    faddp h1, v1.2h
 ; CHECK-SD-FP16-NEXT:    fadd h0, h1, h0
-; CHECK-SD-FP16-NEXT:    b.ne .LBB13_1
+; CHECK-SD-FP16-NEXT:    b.ne .LBB14_1
 ; CHECK-SD-FP16-NEXT:  // %bb.2: // %exit
 ; CHECK-SD-FP16-NEXT:    ret
 ;
@@ -514,7 +523,7 @@ define half @fadd_reduction_v8f16_in_loop(ptr %ptr.start) {
 ; CHECK-GI-NOFP16:       // %bb.0: // %entry
 ; CHECK-GI-NOFP16-NEXT:    mov x8, xzr
 ; CHECK-GI-NOFP16-NEXT:    mov w9, #0 // =0x0
-; CHECK-GI-NOFP16-NEXT:  .LBB13_1: // %loop
+; CHECK-GI-NOFP16-NEXT:  .LBB14_1: // %loop
 ; CHECK-GI-NOFP16-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-GI-NOFP16-NEXT:    ldr q0, [x0, x8]
 ; CHECK-GI-NOFP16-NEXT:    add x8, x8, #8
@@ -531,7 +540,7 @@ define half @fadd_reduction_v8f16_in_loop(ptr %ptr.start) {
 ; CHECK-GI-NOFP16-NEXT:    fadd s0, s0, s1
 ; CHECK-GI-NOFP16-NEXT:    fcvt h0, s0
 ; CHECK-GI-NOFP16-NEXT:    fmov w9, s0
-; CHECK-GI-NOFP16-NEXT:    b.ne .LBB13_1
+; CHECK-GI-NOFP16-NEXT:    b.ne .LBB14_1
 ; CHECK-GI-NOFP16-NEXT:  // %bb.2: // %exit
 ; CHECK-GI-NOFP16-NEXT:    // kill: def $h0 killed $h0 killed $s0
 ; CHECK-GI-NOFP16-NEXT:    ret
@@ -540,7 +549,7 @@ define half @fadd_reduction_v8f16_in_loop(ptr %ptr.start) {
 ; CHECK-GI-FP16:       // %bb.0: // %entry
 ; CHECK-GI-FP16-NEXT:    movi d0, #0000000000000000
 ; CHECK-GI-FP16-NEXT:    mov x8, xzr
-; CHECK-GI-FP16-NEXT:  .LBB13_1: // %loop
+; CHECK-GI-FP16-NEXT:  .LBB14_1: // %loop
 ; CHECK-GI-FP16-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-GI-FP16-NEXT:    ldr q1, [x0, x8]
 ; CHECK-GI-FP16-NEXT:    add x8, x8, #8
@@ -549,7 +558,7 @@ define half @fadd_reduction_v8f16_in_loop(ptr %ptr.start) {
 ; CHECK-GI-FP16-NEXT:    faddp v1.8h, v2.8h, v1.8h
 ; CHECK-GI-FP16-NEXT:    faddp h1, v1.2h
 ; CHECK-GI-FP16-NEXT:    fadd h0, h1, h0
-; CHECK-GI-FP16-NEXT:    b.ne .LBB13_1
+; CHECK-GI-FP16-NEXT:    b.ne .LBB14_1
 ; CHECK-GI-FP16-NEXT:  // %bb.2: // %exit
 ; CHECK-GI-FP16-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll
index 300081dc3ec40..79a8fc35e833d 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll
@@ -589,7 +589,7 @@ define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
 ; CHECK-NEXT:    bl __lttf2
 ; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    cmp w0, #0
-; CHECK-NEXT:    b.ge .LBB18_2
+; CHECK-NEXT:    b.pl .LBB18_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
 ; CHECK-NEXT:  .LBB18_2:
diff --git a/llvm/test/CodeGen/AArch64/vector-llrint.ll b/llvm/test/CodeGen/AArch64/vector-llrint.ll
index 5503de2b4c5db..3547c6f092b41 100644
--- a/llvm/test/CodeGen/AArch64/vector-llrint.ll
+++ b/llvm/test/CodeGen/AArch64/vector-llrint.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64 -mattr=+neon | FileCheck %s
 
-define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
+define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v1i64_v1f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    fcvt s0, h0
@@ -14,7 +14,7 @@ define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half>)
 
-define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
+define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v1i64_v2f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -33,7 +33,7 @@ define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half>)
 
-define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
+define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v4i64_v4f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -62,7 +62,7 @@ define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half>)
 
-define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
+define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v8i64_v8f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
@@ -110,7 +110,7 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half>)
 
-define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
+define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v16i64_v16f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
@@ -197,7 +197,7 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half>)
 
-define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
+define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) nounwind {
 ; CHECK-LABEL: llrint_v32i64_v32f16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
@@ -370,7 +370,7 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 }
 declare <32 x i64> @llvm.llrint.v32i64.v32f16(<32 x half>)
 
-define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
+define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v1i64_v1f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -383,7 +383,7 @@ define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
 
-define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
+define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v2i64_v2f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.2s, v0.2s
@@ -395,7 +395,7 @@ define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
 
-define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
+define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v4i64_v4f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
@@ -411,7 +411,7 @@ define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
 
-define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
+define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v8i64_v8f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
@@ -434,7 +434,7 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
 
-define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
+define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v16i64_v16f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v4.16b, v1.16b, v1.16b, #8
@@ -471,7 +471,7 @@ define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
 
-define <32 x i64> @llrint_v32i64_v32f32(<32 x float> %x) {
+define <32 x i64> @llrint_v32i64_v32f32(<32 x float> %x) nounwind {
 ; CHECK-LABEL: llrint_v32i64_v32f32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ext v16.16b, v7.16b, v7.16b, #8
@@ -544,7 +544,7 @@ define <32 x i64> @llrint_v32i64_v32f32(<32 x float> %x) {
 }
 declare <32 x i64> @llvm.llrint.v32i64.v32f32(<32 x float>)
 
-define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
+define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v1i64_v1f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx d0, d0
@@ -556,7 +556,7 @@ define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
 
-define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
+define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v2i64_v2f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.2d, v0.2d
@@ -567,7 +567,7 @@ define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
 
-define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
+define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v4i64_v4f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.2d, v0.2d
@@ -580,7 +580,7 @@ define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
 
-define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
+define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v8i64_v8f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.2d, v0.2d
@@ -597,7 +597,7 @@ define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
 
-define <16 x i64> @llrint_v16f64(<16 x double> %x) {
+define <16 x i64> @llrint_v16f64(<16 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v16f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    frintx v0.2d, v0.2d
@@ -622,7 +622,7 @@ define <16 x i64> @llrint_v16f64(<16 x double> %x) {
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double>)
 
-define <32 x i64> @llrint_v32f64(<32 x double> %x) {
+define <32 x i64> @llrint_v32f64(<32 x double> %x) nounwind {
 ; CHECK-LABEL: llrint_v32f64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp q17, q16, [sp, #96]
@@ -674,3 +674,246 @@ define <32 x i64> @llrint_v32f64(<32 x double> %x) {
   ret <32 x i64> %a
 }
 declare <32 x i64> @llvm.llrint.v32i64.v32f64(<32 x double>)
+
+define <1 x i64> @llrint_v1i64_v1f128(<1 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v1i64_v1f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f128(<1 x fp128> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f128(<1 x fp128>)
+
+define <2 x i64> @llrint_v2i64_v2f128(<2 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v2i64_v2f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #48
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    add sp, sp, #48
+; CHECK-NEXT:    ret
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f128(<2 x fp128> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f128(<2 x fp128>)
+
+define <4 x i64> @llrint_v4i64_v4f128(<4 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v4i64_v4f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #80
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
+; CHECK-NEXT:    stp q3, q2, [sp, #32] // 32-byte Folded Spill
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d1, x0
+; CHECK-NEXT:    ldp q0, q4, [sp, #16] // 32-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v1.d[1], v4.d[0]
+; CHECK-NEXT:    add sp, sp, #80
+; CHECK-NEXT:    ret
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f128(<4 x fp128> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f128(<4 x fp128>)
+
+define <8 x i64> @llrint_v8i64_v8f128(<8 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v8i64_v8f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #144
+; CHECK-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    str x30, [sp, #128] // 8-byte Folded Spill
+; CHECK-NEXT:    stp q3, q2, [sp, #16] // 32-byte Folded Spill
+; CHECK-NEXT:    stp q5, q4, [sp, #48] // 32-byte Folded Spill
+; CHECK-NEXT:    stp q7, q6, [sp, #96] // 32-byte Folded Spill
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d3, x0
+; CHECK-NEXT:    ldp q0, q1, [sp, #80] // 32-byte Folded Reload
+; CHECK-NEXT:    ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #128] // 8-byte Folded Reload
+; CHECK-NEXT:    mov v3.d[1], v1.d[0]
+; CHECK-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #144
+; CHECK-NEXT:    ret
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f128(<8 x fp128> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f128(<8 x fp128>)
+
+define <16 x i64> @llrint_v16f128(<16 x fp128> %x) nounwind {
+; CHECK-LABEL: llrint_v16f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub sp, sp, #272
+; CHECK-NEXT:    str q2, [sp, #160] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q2, [sp, #368]
+; CHECK-NEXT:    stp q0, q3, [sp] // 32-byte Folded Spill
+; CHECK-NEXT:    mov v0.16b, v1.16b
+; CHECK-NEXT:    str q2, [sp, #240] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q2, [sp, #384]
+; CHECK-NEXT:    stp x29, x30, [sp, #256] // 16-byte Folded Spill
+; CHECK-NEXT:    str q2, [sp, #224] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q2, [sp, #336]
+; CHECK-NEXT:    stp q5, q7, [sp, #32] // 32-byte Folded Spill
+; CHECK-NEXT:    str q2, [sp, #192] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q2, [sp, #352]
+; CHECK-NEXT:    str q2, [sp, #176] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q2, [sp, #304]
+; CHECK-NEXT:    str q2, [sp, #144] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q2, [sp, #320]
+; CHECK-NEXT:    stp q4, q2, [sp, #112] // 32-byte Folded Spill
+; CHECK-NEXT:    ldr q2, [sp, #272]
+; CHECK-NEXT:    stp q6, q2, [sp, #80] // 32-byte Folded Spill
+; CHECK-NEXT:    ldr q2, [sp, #288]
+; CHECK-NEXT:    str q2, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #208] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #208] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #208] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #160] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #160] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #64] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #128] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #128] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #144] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #128] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #144] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #176] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #176] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #192] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    ldr q1, [sp, #176] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    str q0, [sp, #192] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #224] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d0, x0
+; CHECK-NEXT:    str q0, [sp, #224] // 16-byte Folded Spill
+; CHECK-NEXT:    ldr q0, [sp, #240] // 16-byte Folded Reload
+; CHECK-NEXT:    bl llrintl
+; CHECK-NEXT:    fmov d7, x0
+; CHECK-NEXT:    ldp q0, q1, [sp, #208] // 32-byte Folded Reload
+; CHECK-NEXT:    ldp q4, q2, [sp, #96] // 32-byte Folded Reload
+; CHECK-NEXT:    ldr q3, [sp, #80] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x29, x30, [sp, #256] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr q6, [sp, #192] // 16-byte Folded Reload
+; CHECK-NEXT:    mov v7.d[1], v1.d[0]
+; CHECK-NEXT:    ldp q5, q1, [sp, #144] // 32-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #272
+; CHECK-NEXT:    ret
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f128(<16 x fp128> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f128(<16 x fp128>)
diff --git a/llvm/test/CodeGen/AArch64/vector-lrint.ll b/llvm/test/CodeGen/AArch64/vector-lrint.ll
index 602643264e7be..927c6142138b3 100644
--- a/llvm/test/CodeGen/AArch64/vector-lrint.ll
+++ b/llvm/test/CodeGen/AArch64/vector-lrint.ll
@@ -45,7 +45,7 @@
 ; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16f64
 ; CHECK-i64-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v32f64
 
-define <1 x iXLen> @lrint_v1f16(<1 x half> %x) {
+define <1 x iXLen> @lrint_v1f16(<1 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v1f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    fcvt s0, h0
@@ -66,7 +66,7 @@ define <1 x iXLen> @lrint_v1f16(<1 x half> %x) {
 }
 declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f16(<1 x half>)
 
-define <2 x iXLen> @lrint_v2f16(<2 x half> %x) {
+define <2 x iXLen> @lrint_v2f16(<2 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v2f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -100,7 +100,7 @@ define <2 x iXLen> @lrint_v2f16(<2 x half> %x) {
 }
 declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f16(<2 x half>)
 
-define <4 x iXLen> @lrint_v4f16(<4 x half> %x) {
+define <4 x iXLen> @lrint_v4f16(<4 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v4f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 def $q0
@@ -153,7 +153,7 @@ define <4 x iXLen> @lrint_v4f16(<4 x half> %x) {
 }
 declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f16(<4 x half>)
 
-define <8 x iXLen> @lrint_v8f16(<8 x half> %x) {
+define <8 x iXLen> @lrint_v8f16(<8 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v8f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
@@ -244,7 +244,7 @@ define <8 x iXLen> @lrint_v8f16(<8 x half> %x) {
 }
 declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f16(<8 x half>)
 
-define <16 x iXLen> @lrint_v16f16(<16 x half> %x) {
+define <16 x iXLen> @lrint_v16f16(<16 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v16f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    ext v2.16b, v0.16b, v0.16b, #8
@@ -413,7 +413,7 @@ define <16 x iXLen> @lrint_v16f16(<16 x half> %x) {
 }
 declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f16(<16 x half>)
 
-define <32 x iXLen> @lrint_v32f16(<32 x half> %x) {
+define <32 x iXLen> @lrint_v32f16(<32 x half> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v32f16:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    ext v5.16b, v0.16b, v0.16b, #8
@@ -748,7 +748,7 @@ define <32 x iXLen> @lrint_v32f16(<32 x half> %x) {
 }
 declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f16(<32 x half>)
 
-define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
+define <1 x iXLen> @lrint_v1f32(<1 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v1f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.2s, v0.2s
@@ -774,7 +774,7 @@ define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
 }
 declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float>)
 
-define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
+define <2 x iXLen> @lrint_v2f32(<2 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v2f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.2s, v0.2s
@@ -792,7 +792,7 @@ define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
 }
 declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float>)
 
-define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
+define <4 x iXLen> @lrint_v4f32(<4 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v4f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
@@ -814,7 +814,7 @@ define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
 }
 declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float>)
 
-define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
+define <8 x iXLen> @lrint_v8f32(<8 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v8f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
@@ -845,7 +845,7 @@ define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
 }
 declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float>)
 
-define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
+define <16 x iXLen> @lrint_v16f32(<16 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v16f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
@@ -894,7 +894,7 @@ define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
 }
 declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float>)
 
-define <32 x iXLen> @lrint_v32f32(<32 x float> %x) {
+define <32 x iXLen> @lrint_v32f32(<32 x float> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v32f32:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.4s, v0.4s
@@ -987,7 +987,7 @@ define <32 x iXLen> @lrint_v32f32(<32 x float> %x) {
 }
 declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f32(<32 x float>)
 
-define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
+define <1 x iXLen> @lrint_v1f64(<1 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v1f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx d0, d0
@@ -1006,7 +1006,7 @@ define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
 }
 declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double>)
 
-define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
+define <2 x iXLen> @lrint_v2f64(<2 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v2f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.2d, v0.2d
@@ -1028,7 +1028,7 @@ define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
 }
 declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double>)
 
-define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
+define <4 x iXLen> @lrint_v4f64(<4 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v4f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.2d, v0.2d
@@ -1057,7 +1057,7 @@ define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
 }
 declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double>)
 
-define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
+define <8 x iXLen> @lrint_v8f64(<8 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v8f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v2.2d, v2.2d
@@ -1102,7 +1102,7 @@ define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
 }
 declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double>)
 
-define <16 x iXLen> @lrint_v16f64(<16 x double> %x) {
+define <16 x iXLen> @lrint_v16f64(<16 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v16f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v0.2d, v0.2d
@@ -1179,7 +1179,7 @@ define <16 x iXLen> @lrint_v16f64(<16 x double> %x) {
 }
 declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f64(<16 x double>)
 
-define <32 x iXLen> @lrint_v32f64(<32 x double> %x) {
+define <32 x iXLen> @lrint_v32f64(<32 x double> %x) nounwind {
 ; CHECK-i32-LABEL: lrint_v32f64:
 ; CHECK-i32:       // %bb.0:
 ; CHECK-i32-NEXT:    frintx v17.2d, v0.2d
@@ -1335,3 +1335,451 @@ define <32 x iXLen> @lrint_v32f64(<32 x double> %x) {
   ret <32 x iXLen> %a
 }
 declare <32 x iXLen> @llvm.lrint.v32iXLen.v32f64(<32 x double>)
+
+define <1 x iXLen> @lrint_v1fp128(<1 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v1fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v1fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-i64-NEXT:    ret
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1fp128(<1 x fp128> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1fp128(<1 x fp128>)
+
+define <2 x iXLen> @lrint_v2fp128(<2 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v2fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    sub sp, sp, #48
+; CHECK-i32-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #16] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-i32-NEXT:    add sp, sp, #48
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v2fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    sub sp, sp, #48
+; CHECK-i64-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    mov v0.16b, v1.16b
+; CHECK-i64-NEXT:    str x30, [sp, #32] // 8-byte Folded Spill
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ldr x30, [sp, #32] // 8-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    add sp, sp, #48
+; CHECK-i64-NEXT:    ret
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2fp128(<2 x fp128> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2fp128(<2 x fp128>)
+
+define <4 x iXLen> @lrint_v4fp128(<4 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v4fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    sub sp, sp, #80
+; CHECK-i32-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
+; CHECK-i32-NEXT:    stp q2, q3, [sp, #16] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w0
+; CHECK-i32-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[3], w0
+; CHECK-i32-NEXT:    add sp, sp, #80
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v4fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    sub sp, sp, #80
+; CHECK-i64-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    mov v0.16b, v1.16b
+; CHECK-i64-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
+; CHECK-i64-NEXT:    stp q3, q2, [sp, #32] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d1, x0
+; CHECK-i64-NEXT:    ldp q0, q4, [sp, #16] // 32-byte Folded Reload
+; CHECK-i64-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
+; CHECK-i64-NEXT:    mov v1.d[1], v4.d[0]
+; CHECK-i64-NEXT:    add sp, sp, #80
+; CHECK-i64-NEXT:    ret
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4fp128(<4 x fp128> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4fp128(<4 x fp128>)
+
+define <8 x iXLen> @lrint_v8fp128(<8 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v8fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    sub sp, sp, #144
+; CHECK-i32-NEXT:    str x30, [sp, #128] // 8-byte Folded Spill
+; CHECK-i32-NEXT:    str q4, [sp, #96] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    stp q3, q5, [sp, #32] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    stp q6, q7, [sp, #64] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w0
+; CHECK-i32-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[3], w0
+; CHECK-i32-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w0
+; CHECK-i32-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldp q1, q0, [sp, #96] // 32-byte Folded Reload
+; CHECK-i32-NEXT:    ldr x30, [sp, #128] // 8-byte Folded Reload
+; CHECK-i32-NEXT:    mov v1.s[3], w0
+; CHECK-i32-NEXT:    add sp, sp, #144
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v8fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    sub sp, sp, #144
+; CHECK-i64-NEXT:    str q0, [sp] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    mov v0.16b, v1.16b
+; CHECK-i64-NEXT:    str x30, [sp, #128] // 8-byte Folded Spill
+; CHECK-i64-NEXT:    stp q3, q2, [sp, #16] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    stp q5, q4, [sp, #48] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    stp q7, q6, [sp, #96] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #80] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #48] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d3, x0
+; CHECK-i64-NEXT:    ldp q0, q1, [sp, #80] // 32-byte Folded Reload
+; CHECK-i64-NEXT:    ldr q2, [sp, #64] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ldr x30, [sp, #128] // 8-byte Folded Reload
+; CHECK-i64-NEXT:    mov v3.d[1], v1.d[0]
+; CHECK-i64-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    add sp, sp, #144
+; CHECK-i64-NEXT:    ret
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8fp128(<8 x fp128> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8fp128(<8 x fp128>)
+
+define <16 x iXLen> @lrint_v16fp128(<16 x fp128> %x) nounwind {
+; CHECK-i32-LABEL: lrint_v16fp128:
+; CHECK-i32:       // %bb.0:
+; CHECK-i32-NEXT:    sub sp, sp, #272
+; CHECK-i32-NEXT:    stp q1, q2, [sp] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #384]
+; CHECK-i32-NEXT:    stp x29, x30, [sp, #256] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #176] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #368]
+; CHECK-i32-NEXT:    stp q3, q5, [sp, #32] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #160] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #352]
+; CHECK-i32-NEXT:    stp q7, q4, [sp, #208] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    str q1, [sp, #144] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #336]
+; CHECK-i32-NEXT:    str q1, [sp, #192] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #320]
+; CHECK-i32-NEXT:    str q1, [sp, #128] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #304]
+; CHECK-i32-NEXT:    str q1, [sp, #112] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #288]
+; CHECK-i32-NEXT:    stp q6, q1, [sp, #80] // 32-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q1, [sp, #272]
+; CHECK-i32-NEXT:    str q1, [sp, #64] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp, #240] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #240] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    str q0, [sp, #240] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #240] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w0
+; CHECK-i32-NEXT:    str q0, [sp, #240] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #240] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[3], w0
+; CHECK-i32-NEXT:    str q0, [sp, #240] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #224] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp, #224] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #224] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    str q0, [sp, #224] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #224] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w0
+; CHECK-i32-NEXT:    str q0, [sp, #224] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #208] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #224] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[3], w0
+; CHECK-i32-NEXT:    str q0, [sp, #224] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp, #208] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #208] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    str q0, [sp, #208] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #208] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w0
+; CHECK-i32-NEXT:    str q0, [sp, #208] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #128] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #208] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[3], w0
+; CHECK-i32-NEXT:    str q0, [sp, #208] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #192] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    fmov s0, w0
+; CHECK-i32-NEXT:    str q0, [sp, #192] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #144] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #192] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[1], w0
+; CHECK-i32-NEXT:    str q0, [sp, #192] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #160] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldr q0, [sp, #192] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v0.s[2], w0
+; CHECK-i32-NEXT:    str q0, [sp, #192] // 16-byte Folded Spill
+; CHECK-i32-NEXT:    ldr q0, [sp, #176] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    bl lrintl
+; CHECK-i32-NEXT:    ldp q3, q2, [sp, #192] // 32-byte Folded Reload
+; CHECK-i32-NEXT:    ldp q1, q0, [sp, #224] // 32-byte Folded Reload
+; CHECK-i32-NEXT:    ldp x29, x30, [sp, #256] // 16-byte Folded Reload
+; CHECK-i32-NEXT:    mov v3.s[3], w0
+; CHECK-i32-NEXT:    add sp, sp, #272
+; CHECK-i32-NEXT:    ret
+;
+; CHECK-i64-LABEL: lrint_v16fp128:
+; CHECK-i64:       // %bb.0:
+; CHECK-i64-NEXT:    sub sp, sp, #272
+; CHECK-i64-NEXT:    str q2, [sp, #160] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q2, [sp, #368]
+; CHECK-i64-NEXT:    stp q0, q3, [sp] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    mov v0.16b, v1.16b
+; CHECK-i64-NEXT:    str q2, [sp, #240] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q2, [sp, #384]
+; CHECK-i64-NEXT:    stp x29, x30, [sp, #256] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    str q2, [sp, #224] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q2, [sp, #336]
+; CHECK-i64-NEXT:    stp q5, q7, [sp, #32] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    str q2, [sp, #192] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q2, [sp, #352]
+; CHECK-i64-NEXT:    str q2, [sp, #176] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q2, [sp, #304]
+; CHECK-i64-NEXT:    str q2, [sp, #144] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q2, [sp, #320]
+; CHECK-i64-NEXT:    stp q4, q2, [sp, #112] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q2, [sp, #272]
+; CHECK-i64-NEXT:    stp q6, q2, [sp, #80] // 32-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q2, [sp, #288]
+; CHECK-i64-NEXT:    str q2, [sp, #64] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #208] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #208] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #208] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #160] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #16] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #160] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #32] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #112] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #32] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #112] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #48] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #48] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #80] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #48] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #80] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #64] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #64] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #96] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #64] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #96] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #128] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #128] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #144] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #128] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #144] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #176] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #176] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #192] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    ldr q1, [sp, #176] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-i64-NEXT:    str q0, [sp, #192] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #224] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d0, x0
+; CHECK-i64-NEXT:    str q0, [sp, #224] // 16-byte Folded Spill
+; CHECK-i64-NEXT:    ldr q0, [sp, #240] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    bl lrintl
+; CHECK-i64-NEXT:    fmov d7, x0
+; CHECK-i64-NEXT:    ldp q0, q1, [sp, #208] // 32-byte Folded Reload
+; CHECK-i64-NEXT:    ldp q4, q2, [sp, #96] // 32-byte Folded Reload
+; CHECK-i64-NEXT:    ldr q3, [sp, #80] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ldp x29, x30, [sp, #256] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    ldr q6, [sp, #192] // 16-byte Folded Reload
+; CHECK-i64-NEXT:    mov v7.d[1], v1.d[0]
+; CHECK-i64-NEXT:    ldp q5, q1, [sp, #144] // 32-byte Folded Reload
+; CHECK-i64-NEXT:    add sp, sp, #272
+; CHECK-i64-NEXT:    ret
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16fp128(<16 x fp128> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16fp128(<16 x fp128>)
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-i32-GI: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame-predecrement.mir b/llvm/test/CodeGen/AArch64/wineh-frame-predecrement.mir
index 1bed8f6b547a2..ebe3c41b68c1e 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame-predecrement.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame-predecrement.mir
@@ -45,8 +45,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame-scavenge.mir b/llvm/test/CodeGen/AArch64/wineh-frame-scavenge.mir
index 09799ee266491..e0e367668213e 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame-scavenge.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame-scavenge.mir
@@ -61,8 +61,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame1.mir b/llvm/test/CodeGen/AArch64/wineh-frame1.mir
index 37a0394415a31..531ca14ee67ec 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame1.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame1.mir
@@ -64,8 +64,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame2.mir b/llvm/test/CodeGen/AArch64/wineh-frame2.mir
index ca785b93b9714..1c8cb1e79a998 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame2.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame2.mir
@@ -52,8 +52,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame3.mir b/llvm/test/CodeGen/AArch64/wineh-frame3.mir
index 30d804eeb8e54..c9a393144bc67 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame3.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame3.mir
@@ -44,8 +44,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame4.mir b/llvm/test/CodeGen/AArch64/wineh-frame4.mir
index 73d30cab947ae..75dc37d4bd734 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame4.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame4.mir
@@ -44,8 +44,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame5.mir b/llvm/test/CodeGen/AArch64/wineh-frame5.mir
index 0589d97ca64a2..97c5c85c1279d 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame5.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame5.mir
@@ -100,8 +100,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  492
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: B, type: default, offset: 0, size: 492, alignment: 4,
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame6.mir b/llvm/test/CodeGen/AArch64/wineh-frame6.mir
index 188a905a0ab75..5ba7842642d47 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame6.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame6.mir
@@ -85,8 +85,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  24
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: c.addr, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame7.mir b/llvm/test/CodeGen/AArch64/wineh-frame7.mir
index b30afd2b57153..159909814ff33 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame7.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame7.mir
@@ -105,8 +105,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  2993276
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: retval, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/AArch64/wineh-frame8.mir b/llvm/test/CodeGen/AArch64/wineh-frame8.mir
index be14dcd29c66d..9de99ac25bb6c 100644
--- a/llvm/test/CodeGen/AArch64/wineh-frame8.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-frame8.mir
@@ -60,8 +60,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  8
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: a.addr, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/AArch64/wineh-save-lrpair1.mir b/llvm/test/CodeGen/AArch64/wineh-save-lrpair1.mir
index 7959ad611ed40..e70242638a0f0 100644
--- a/llvm/test/CodeGen/AArch64/wineh-save-lrpair1.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-save-lrpair1.mir
@@ -52,8 +52,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/wineh-save-lrpair2.mir b/llvm/test/CodeGen/AArch64/wineh-save-lrpair2.mir
index 2ec296f7b9e7e..a78ed0694674a 100644
--- a/llvm/test/CodeGen/AArch64/wineh-save-lrpair2.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-save-lrpair2.mir
@@ -45,8 +45,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/wineh-save-lrpair3.mir b/llvm/test/CodeGen/AArch64/wineh-save-lrpair3.mir
index e45cf9ff3e4c8..a576380c09738 100644
--- a/llvm/test/CodeGen/AArch64/wineh-save-lrpair3.mir
+++ b/llvm/test/CodeGen/AArch64/wineh-save-lrpair3.mir
@@ -46,8 +46,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/AArch64/wineh2.mir b/llvm/test/CodeGen/AArch64/wineh2.mir
index 572df37bf2b51..529a9823a1ad4 100644
--- a/llvm/test/CodeGen/AArch64/wineh2.mir
+++ b/llvm/test/CodeGen/AArch64/wineh2.mir
@@ -64,8 +64,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/AArch64/wineh3.mir b/llvm/test/CodeGen/AArch64/wineh3.mir
index 395180ee540cd..7cc15badb6567 100644
--- a/llvm/test/CodeGen/AArch64/wineh3.mir
+++ b/llvm/test/CodeGen/AArch64/wineh3.mir
@@ -52,8 +52,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/AArch64/wineh4.mir b/llvm/test/CodeGen/AArch64/wineh4.mir
index baeb31167f096..bdac1c04fb0b8 100644
--- a/llvm/test/CodeGen/AArch64/wineh4.mir
+++ b/llvm/test/CodeGen/AArch64/wineh4.mir
@@ -81,8 +81,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/AArch64/wineh5.mir b/llvm/test/CodeGen/AArch64/wineh5.mir
index 053db98e6d39b..efdd4b00c90b6 100644
--- a/llvm/test/CodeGen/AArch64/wineh5.mir
+++ b/llvm/test/CodeGen/AArch64/wineh5.mir
@@ -110,8 +110,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  2993276
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: retval, type: default, offset: -36, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/AArch64/wineh6.mir b/llvm/test/CodeGen/AArch64/wineh6.mir
index f3d5fc8921a57..098efe8da4535 100644
--- a/llvm/test/CodeGen/AArch64/wineh6.mir
+++ b/llvm/test/CodeGen/AArch64/wineh6.mir
@@ -54,8 +54,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  24
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: default, offset: -20, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/AArch64/wineh7.mir b/llvm/test/CodeGen/AArch64/wineh7.mir
index ada4ebcf5057e..32d0ab805e4c7 100644
--- a/llvm/test/CodeGen/AArch64/wineh7.mir
+++ b/llvm/test/CodeGen/AArch64/wineh7.mir
@@ -52,8 +52,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: variable-sized, offset: -48,
diff --git a/llvm/test/CodeGen/AArch64/wineh8.mir b/llvm/test/CodeGen/AArch64/wineh8.mir
index f1fad8b70ff27..4209b5bdf4c28 100644
--- a/llvm/test/CodeGen/AArch64/wineh8.mir
+++ b/llvm/test/CodeGen/AArch64/wineh8.mir
@@ -80,8 +80,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/AArch64/wineh9.mir b/llvm/test/CodeGen/AArch64/wineh9.mir
index ef26970dda04c..39bc411db0569 100644
--- a/llvm/test/CodeGen/AArch64/wineh9.mir
+++ b/llvm/test/CodeGen/AArch64/wineh9.mir
@@ -57,8 +57,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  8
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: x.addr, type: default, offset: -24, size: 8, alignment: 8, 
diff --git a/llvm/test/CodeGen/AArch64/wineh_shrinkwrap.mir b/llvm/test/CodeGen/AArch64/wineh_shrinkwrap.mir
index aad271313c850..2f631c217e8eb 100644
--- a/llvm/test/CodeGen/AArch64/wineh_shrinkwrap.mir
+++ b/llvm/test/CodeGen/AArch64/wineh_shrinkwrap.mir
@@ -102,8 +102,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  4000
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: A, type: default, offset: 0, size: 4000, alignment: 4,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll
index b68df4fbbbb9e..59036338eaf15 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add_shl.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=fiji < %s | FileCheck -check-prefix=VI %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-mesa3d -mcpu=fiji < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX10 %s
 
 ; ===================================================================================
 ; V_ADD_LSHL_U32
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
index 523d51ddcd2bc..fc236147f1238 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/amdgpu-irtranslator.ll
@@ -1,12 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=fiji -O0 -stop-after=irtranslator -global-isel %s -o - 2>&1 | FileCheck %s
 
 ; This file checks that the translation from llvm IR to generic
 ; MachineInstr is correct.
 
 ; Tests for add.
-; CHECK: name: addi32
-; CHECK: {{%[0-9]+}}:_(s32) = G_ADD
-define amdgpu_kernel void @addi32(i32 %arg1, i32 %arg2) {
+define void @addi32(i32 %arg1, i32 %arg2) {
+  ; CHECK-LABEL: name: addi32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+  ; CHECK-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
+  ; CHECK-NEXT:   G_STORE [[ADD]](s32), [[DEF]](p1) :: (store (s32) into `ptr addrspace(1) poison`, addrspace 1)
+  ; CHECK-NEXT:   SI_RETURN
   %res = add i32 %arg1, %arg2
   store i32 %res, ptr addrspace(1) poison
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.i1.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.i1.ll
index 74422a1962344..25d70002a7a8e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.i1.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefix=WAVE64 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=WAVE32 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefix=WAVE64 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=WAVE32 %s
 
 define i32 @s_andn2_i1_vcc(i32 %arg0, i32 %arg1) {
 ; WAVE64-LABEL: s_andn2_i1_vcc:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-asserts.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-asserts.ll
index cdcc3a4f27071..fae3a75101ee5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-asserts.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-asserts.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1031 < %s | FileCheck %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1031 < %s | FileCheck %s
 
 define hidden <2 x i64> @icmp_v2i32_sext_to_v2i64(<2 x i32> %arg) {
 ; CHECK-LABEL: icmp_v2i32_sext_to_v2i64:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
index a86939fc2ce8e..dac726df5decb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
@@ -91,7 +91,7 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
 ; GFX11-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -175,7 +175,7 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out,
 ; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -233,7 +233,7 @@ define amdgpu_kernel void @lds_atomic_dec_noret_i32(ptr addrspace(3) %ptr) #1 {
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -291,7 +291,7 @@ define amdgpu_kernel void @lds_atomic_dec_noret_i32_offset(ptr addrspace(3) %ptr
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -368,7 +368,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -451,7 +451,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset(ptr addrspace(1) %ou
 ; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -534,7 +534,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_system(ptr addrspace
 ; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -603,7 +603,7 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32(ptr addrspace(1) %ptr) #1
 ; GFX11-NEXT:    buffer_gl1_inv
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -676,7 +676,7 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset(ptr addrspace(1) %
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -749,7 +749,7 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_system(ptr addrspa
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -846,7 +846,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i32_offset_addr64(ptr addrspace
   %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id
   %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id
   %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out.gep, align 4
   ret void
 }
@@ -930,7 +930,7 @@ define amdgpu_kernel void @global_atomic_dec_noret_i32_offset_addr64(ptr addrspa
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id
   %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -1021,7 +1021,7 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32(ptr %out, ptr %ptr) #1 {
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr %out, align 4
   ret void
 }
@@ -1119,8 +1119,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset(ptr %out, ptr %ptr) #1
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
-  %gep = getelementptr i32, ptr %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
+  %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr %out, align 4
   ret void
 }
@@ -1218,8 +1218,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_system(ptr %out, ptr %
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
-  %gep = getelementptr i32, ptr %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr %gep, i32 42 seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
+  %result = atomicrmw udec_wrap ptr %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr %out, align 4
   ret void
 }
@@ -1299,7 +1299,7 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32(ptr %ptr) #1 {
 ; GFX11-NEXT:    buffer_gl1_inv
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -1384,8 +1384,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset(ptr %ptr) #1 {
 ; GFX11-NEXT:    buffer_gl1_inv
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %gep = getelementptr i32, ptr %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
+  %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -1470,8 +1470,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_system(ptr %ptr) #1
 ; GFX11-NEXT:    buffer_gl1_inv
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %gep = getelementptr i32, ptr %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr %gep, i32 42 seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
+  %result = atomicrmw udec_wrap ptr %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -1599,8 +1599,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i32_offset_addr64(ptr %out, ptr %
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, ptr %ptr, i32 %id
   %out.gep = getelementptr i32, ptr %out, i32 %id
-  %gep = getelementptr i32, ptr %gep.tid, i32 5
-  %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %gep.tid, i32 5
+  %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr %out.gep, align 4
   ret void
 }
@@ -1706,8 +1706,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i32_offset_addr64(ptr %ptr) #1
 ; GFX11-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, ptr %ptr, i32 %id
-  %gep = getelementptr i32, ptr %gep.tid, i32 5
-  %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %gep.tid, i32 5
+  %result = atomicrmw udec_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -1813,7 +1813,7 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64(ptr %out, ptr %ptr) #1 {
 ; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX11-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %result = atomicrmw udec_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   store i64 %result, ptr %out, align 4
   ret void
 }
@@ -1926,8 +1926,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset(ptr %out, ptr %ptr) #1
 ; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX11-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %gep = getelementptr i64, ptr %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %ptr, i32 4
+  %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   store i64 %result, ptr %out, align 4
   ret void
 }
@@ -2012,7 +2012,7 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64(ptr %ptr) #1 {
 ; GFX11-NEXT:    buffer_gl1_inv
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %result = atomicrmw udec_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2102,8 +2102,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset(ptr %ptr) #1 {
 ; GFX11-NEXT:    buffer_gl1_inv
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %gep = getelementptr i64, ptr %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %ptr, i32 4
+  %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2193,8 +2193,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_system(ptr %ptr) #1
 ; GFX11-NEXT:    buffer_gl1_inv
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %gep = getelementptr i64, ptr %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr %gep, i64 42 seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %ptr, i32 4
+  %result = atomicrmw udec_wrap ptr %gep, i64 42 seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2333,8 +2333,8 @@ define amdgpu_kernel void @flat_atomic_dec_ret_i64_offset_addr64(ptr %out, ptr %
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, ptr %ptr, i32 %id
   %out.gep = getelementptr i64, ptr %out, i32 %id
-  %gep = getelementptr i64, ptr %gep.tid, i32 5
-  %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %gep.tid, i32 5
+  %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   store i64 %result, ptr %out.gep, align 4
   ret void
 }
@@ -2444,8 +2444,8 @@ define amdgpu_kernel void @flat_atomic_dec_noret_i64_offset_addr64(ptr %ptr) #1
 ; GFX11-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, ptr %ptr, i32 %id
-  %gep = getelementptr i64, ptr %gep.tid, i32 5
-  %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %gep.tid, i32 5
+  %result = atomicrmw udec_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2540,7 +2540,7 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0(ptr addrspace(1) %out, ptr
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #2
   %idx.0 = add nsw i32 %tid.x, 2
   %arrayidx0 = getelementptr inbounds [512 x i32], ptr addrspace(3) @lds0, i32 0, i32 %idx.0
-  %result = atomicrmw udec_wrap ptr addrspace(3) %arrayidx0, i32 9 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw udec_wrap ptr addrspace(3) %arrayidx0, i32 9 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %idx.0, ptr addrspace(1) %add_use, align 4
   store i32 %result, ptr addrspace(1) %out, align 4
   ret void
@@ -2629,7 +2629,7 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   store i64 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -2781,7 +2781,7 @@ define amdgpu_kernel void @lds_atomic_dec_noret_i64(ptr addrspace(3) %ptr) #1 {
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(3) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2844,7 +2844,7 @@ define amdgpu_kernel void @lds_atomic_dec_noret_i64_offset(ptr addrspace(3) %ptr
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(3) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2926,7 +2926,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   store i64 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -3014,7 +3014,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset(ptr addrspace(1) %ou
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   store i64 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -3102,7 +3102,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_system(ptr addrspace
 ; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8, !amdgpu.no.remote.memory !1
   store i64 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -3176,7 +3176,7 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64(ptr addrspace(1) %ptr) #1
 ; GFX11-NEXT:    buffer_gl1_inv
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
-  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -3254,7 +3254,7 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset(ptr addrspace(1) %
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -3332,7 +3332,7 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_system(ptr addrspa
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -3434,7 +3434,7 @@ define amdgpu_kernel void @global_atomic_dec_ret_i64_offset_addr64(ptr addrspace
   %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id
   %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %id
   %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   store i64 %result, ptr addrspace(1) %out.gep, align 4
   ret void
 }
@@ -3523,7 +3523,7 @@ define amdgpu_kernel void @global_atomic_dec_noret_i64_offset_addr64(ptr addrspa
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id
   %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5
-  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -3624,7 +3624,7 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(ptr addrspace(1) %out,
   %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #2
   %idx.0 = add nsw i32 %tid.x, 2
   %arrayidx0 = getelementptr inbounds [512 x i64], ptr addrspace(3) @lds1, i32 0, i32 %idx.0
-  %result = atomicrmw udec_wrap ptr addrspace(3) %arrayidx0, i64 9 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %result = atomicrmw udec_wrap ptr addrspace(3) %arrayidx0, i64 9 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   store i32 %idx.0, ptr addrspace(1) %add_use, align 4
   store i64 %result, ptr addrspace(1) %out, align 4
   ret void
@@ -3635,6 +3635,7 @@ attributes #1 = { nounwind }
 attributes #2 = { nounwind memory(none) }
 
 !0 = !{i32 5, i32 6}
+!1 = !{}
 
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
index 7958e40ea0e68..77d212afa0594 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
@@ -424,7 +424,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-NEXT:    s_endpgm
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -518,7 +518,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset(ptr addrspace(1) %ou
 ; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -614,7 +614,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_sistem(ptr addrspace
 ; GFX12-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX12-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -693,7 +693,7 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32(ptr addrspace(1) %ptr) #1
 ; GFX12-NEXT:    s_wait_storecnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -776,7 +776,7 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset(ptr addrspace(1) %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -861,7 +861,7 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_system(ptr addrspa
 ; GFX12-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -971,7 +971,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i32_offset_addr64(ptr addrspace
   %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id
   %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %id
   %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr addrspace(1) %out.gep, align 4
   ret void
 }
@@ -1067,7 +1067,7 @@ define amdgpu_kernel void @global_atomic_inc_noret_i32_offset_addr64(ptr addrspa
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, ptr addrspace(1) %ptr, i32 %id
   %gep = getelementptr i32, ptr addrspace(1) %gep.tid, i32 5
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -1625,7 +1625,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX12-NEXT:    s_endpgm
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   store i64 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -1725,7 +1725,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset(ptr addrspace(1) %ou
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX12-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   store i64 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -1827,7 +1827,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_system(ptr addrspace
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX12-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8, !amdgpu.no.remote.memory !1
   store i64 %result, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -1912,7 +1912,7 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64(ptr addrspace(1) %ptr) #1
 ; GFX12-NEXT:    s_wait_storecnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2001,7 +2001,7 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset(ptr addrspace(1) %
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2092,7 +2092,7 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_system(ptr addrspa
 ; GFX12-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2208,7 +2208,7 @@ define amdgpu_kernel void @global_atomic_inc_ret_i64_offset_addr64(ptr addrspace
   %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id
   %out.gep = getelementptr i64, ptr addrspace(1) %out, i32 %id
   %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   store i64 %result, ptr addrspace(1) %out.gep, align 4
   ret void
 }
@@ -2310,7 +2310,7 @@ define amdgpu_kernel void @global_atomic_inc_noret_i64_offset_addr64(ptr addrspa
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, ptr addrspace(1) %ptr, i32 %id
   %gep = getelementptr i64, ptr addrspace(1) %gep.tid, i32 5
-  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8
+  %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 42 syncscope("agent") seq_cst, align 8, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2414,7 +2414,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #1 {
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX12-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-NEXT:    s_endpgm
-  %result = atomicrmw uinc_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr %out, align 4
   ret void
 }
@@ -2525,8 +2525,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #1
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX12-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-NEXT:    s_endpgm
-  %gep = getelementptr i32, ptr %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
+  %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr %out, align 4
   ret void
 }
@@ -2639,8 +2639,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_system(ptr %out, ptr %
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX12-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-NEXT:    s_endpgm
-  %gep = getelementptr i32, ptr %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
+  %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr %out, align 4
   ret void
 }
@@ -2731,7 +2731,7 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) #1 {
 ; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
-  %result = atomicrmw uinc_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4
+  %result = atomicrmw uinc_wrap ptr %ptr, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2827,8 +2827,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) #1 {
 ; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
-  %gep = getelementptr i32, ptr %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
+  %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -2926,8 +2926,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_system(ptr %ptr) #1
 ; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-NEXT:    s_endpgm
-  %gep = getelementptr i32, ptr %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %ptr, i32 4
+  %result = atomicrmw uinc_wrap ptr %gep, i32 42 seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -3077,8 +3077,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, ptr %ptr, i32 %id
   %out.gep = getelementptr i32, ptr %out, i32 %id
-  %gep = getelementptr i32, ptr %gep.tid, i32 5
-  %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %gep.tid, i32 5
+  %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   store i32 %result, ptr %out.gep, align 4
   ret void
 }
@@ -3201,8 +3201,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #1
 ; GFX12-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, ptr %ptr, i32 %id
-  %gep = getelementptr i32, ptr %gep.tid, i32 5
-  %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4
+  %gep = getelementptr inbounds i32, ptr %gep.tid, i32 5
+  %result = atomicrmw uinc_wrap ptr %gep, i32 42 syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -3444,7 +3444,7 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #1 {
 ; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
-  %result = atomicrmw uinc_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %result = atomicrmw uinc_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   store i64 %result, ptr %out, align 4
   ret void
 }
@@ -3571,8 +3571,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #1
 ; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
-  %gep = getelementptr i64, ptr %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %ptr, i32 4
+  %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   store i64 %result, ptr %out, align 4
   ret void
 }
@@ -3701,8 +3701,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_system(ptr %out, ptr %
 ; GFX12-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
-  %gep = getelementptr i64, ptr %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %ptr, i32 4
+  %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   store i64 %result, ptr %out, align 4
   ret void
 }
@@ -3799,7 +3799,7 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) #1 {
 ; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
-  %result = atomicrmw uinc_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %result = atomicrmw uinc_wrap ptr %ptr, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -3901,8 +3901,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) #1 {
 ; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
-  %gep = getelementptr i64, ptr %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %ptr, i32 4
+  %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -4006,8 +4006,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_system(ptr %ptr) #1
 ; GFX12-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-NEXT:    s_endpgm
-  %gep = getelementptr i64, ptr %ptr, i32 4
-  %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %ptr, i32 4
+  %result = atomicrmw uinc_wrap ptr %gep, i64 42 seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -4169,8 +4169,8 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, ptr %ptr, i32 %id
   %out.gep = getelementptr i64, ptr %out, i32 %id
-  %gep = getelementptr i64, ptr %gep.tid, i32 5
-  %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %gep.tid, i32 5
+  %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   store i64 %result, ptr %out.gep, align 4
   ret void
 }
@@ -4297,8 +4297,8 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #1
 ; GFX12-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, ptr %ptr, i32 %id
-  %gep = getelementptr i64, ptr %gep.tid, i32 5
-  %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0
+  %gep = getelementptr inbounds i64, ptr %gep.tid, i32 5
+  %result = atomicrmw uinc_wrap ptr %gep, i64 42 syncscope("agent") seq_cst, align 8, !noalias.addrspace !0, !amdgpu.no.remote.memory !1
   ret void
 }
 
@@ -4434,6 +4434,7 @@ attributes #1 = { nounwind }
 attributes #2 = { nounwind memory(none) }
 
 !0 = !{i32 5, i32 6}
+!1 = !{}
 
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; GCN: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/br-constant-invalid-sgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/br-constant-invalid-sgpr-copy.ll
index 439ffbac960b8..22324e62c2ab5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/br-constant-invalid-sgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/br-constant-invalid-sgpr-copy.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=WAVE64 %s
-; RUN: llc -global-isel -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1031 < %s | FileCheck -check-prefix=WAVE32 %s
+; RUN: llc -global-isel -new-reg-bank-select -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=WAVE64 %s
+; RUN: llc -global-isel -new-reg-bank-select -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1031 < %s | FileCheck -check-prefix=WAVE32 %s
 
 ; This was mishandling the constant true and false values used as a
 ; scalar branch condition.
@@ -76,7 +76,8 @@ define void @br_undef() {
 ; WAVE64-NEXT:  .LBB2_1: ; %bb0
 ; WAVE64-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; WAVE64-NEXT:    ; implicit-def: $sgpr4
-; WAVE64-NEXT:    s_and_b32 s4, s4, 1
+; WAVE64-NEXT:    s_mov_b32 s5, 1
+; WAVE64-NEXT:    s_and_b32 s4, s4, s5
 ; WAVE64-NEXT:    s_cmp_lg_u32 s4, 0
 ; WAVE64-NEXT:    s_cbranch_scc1 .LBB2_1
 ; WAVE64-NEXT:  ; %bb.2: ; %.exit5
@@ -88,7 +89,8 @@ define void @br_undef() {
 ; WAVE32-NEXT:  .LBB2_1: ; %bb0
 ; WAVE32-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; WAVE32-NEXT:    ; implicit-def: $sgpr4
-; WAVE32-NEXT:    s_and_b32 s4, s4, 1
+; WAVE32-NEXT:    s_mov_b32 s5, 1
+; WAVE32-NEXT:    s_and_b32 s4, s4, s5
 ; WAVE32-NEXT:    s_cmp_lg_u32 s4, 0
 ; WAVE32-NEXT:    s_cbranch_scc1 .LBB2_1
 ; WAVE32-NEXT:  ; %bb.2: ; %.exit5
@@ -110,7 +112,8 @@ define void @br_poison() {
 ; WAVE64-NEXT:  .LBB3_1: ; %bb0
 ; WAVE64-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; WAVE64-NEXT:    ; implicit-def: $sgpr4
-; WAVE64-NEXT:    s_and_b32 s4, s4, 1
+; WAVE64-NEXT:    s_mov_b32 s5, 1
+; WAVE64-NEXT:    s_and_b32 s4, s4, s5
 ; WAVE64-NEXT:    s_cmp_lg_u32 s4, 0
 ; WAVE64-NEXT:    s_cbranch_scc1 .LBB3_1
 ; WAVE64-NEXT:  ; %bb.2: ; %.exit5
@@ -122,7 +125,8 @@ define void @br_poison() {
 ; WAVE32-NEXT:  .LBB3_1: ; %bb0
 ; WAVE32-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; WAVE32-NEXT:    ; implicit-def: $sgpr4
-; WAVE32-NEXT:    s_and_b32 s4, s4, 1
+; WAVE32-NEXT:    s_mov_b32 s5, 1
+; WAVE32-NEXT:    s_and_b32 s4, s4, s5
 ; WAVE32-NEXT:    s_cmp_lg_u32 s4, 0
 ; WAVE32-NEXT:    s_cbranch_scc1 .LBB3_1
 ; WAVE32-NEXT:  ; %bb.2: ; %.exit5
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir
new file mode 100644
index 0000000000000..9259678072646
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-binop-s64-with-s32-mask.mir
@@ -0,0 +1,321 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=tahiti -run-pass=amdgpu-prelegalizer-combiner %s -o - | FileCheck %s
+
+---
+name: test_and_mask_hi_rhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_and_mask_hi_rhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 -4294967296
+    %2:_(s64) = G_AND %0, %1
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_and_mask_hi_lhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_and_mask_hi_lhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 -4294967296
+    %2:_(s64) = G_AND %1, %0
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_and_mask_hi_48bit_mask_rhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_and_mask_hi_48bit_mask_rhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -65536
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[AND]](s32), [[UV1]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 -65536
+    %2:_(s64) = G_AND %0, %1
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_and_mask_hi_16bit_mask_rhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_and_mask_hi_16bit_mask_rhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -281474976710656
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[AND]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 -281474976710656
+    %2:_(s64) = G_AND %0, %1
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_and_mask_lo_rhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_and_mask_lo_rhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[ZEXT]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 4294967295
+    %2:_(s64) = G_AND %0, %1
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_and_mask_lo_lhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_and_mask_lo_lhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[ZEXT]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 4294967295
+    %2:_(s64) = G_AND %1, %0
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_and_mask_lo_36bit_mask_rhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_and_mask_lo_36bit_mask_rhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UV]](s32), [[AND]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 68719476735
+    %2:_(s64) = G_AND %0, %1
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_and_mask_hi_with_merge_unmerge
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_and_mask_hi_with_merge_unmerge
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: $sgpr0 = COPY [[C]](s32)
+    ; CHECK-NEXT: $sgpr1 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s64) = G_CONSTANT i64 -4294967296
+    %4:_(s64) = G_AND %2, %3
+    %5:_(s32), %6:_(s32) = G_UNMERGE_VALUES %4(s64)
+    $sgpr0 = COPY %5(s32)
+    $sgpr1 = COPY %6(s32)
+    SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+...
+---
+name: negative_and_test_incorrect_types
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: negative_and_test_incorrect_types
+    ; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 -4294967296
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s128) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[AND]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = COPY $vgpr4_vgpr5
+    %2:_(s128) = G_CONSTANT i128 -4294967296
+    %3:_(s128) = G_AND %0, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+---
+name: test_or_mask_hi_rhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_or_mask_hi_rhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](s32), [[C]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 -4294967296
+    %2:_(s64) = G_OR %0, %1
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_or_mask_hi_lhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_or_mask_hi_lhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](s32), [[C]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 -4294967296
+    %2:_(s64) = G_OR %1, %0
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_or_mask_lo_rhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_or_mask_lo_rhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 4294967295
+    %2:_(s64) = G_OR %0, %1
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_or_mask_lo_lhs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_or_mask_lo_lhs
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[UV1]](s32)
+    ; CHECK-NEXT: $sgpr0_sgpr1 = COPY [[MV]](s64)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 4294967295
+    %2:_(s64) = G_OR %1, %0
+    $sgpr0_sgpr1 = COPY %2(s64)
+    SI_RETURN_TO_EPILOG implicit $sgpr0_sgpr1
+...
+---
+name: test_or_mask_hi_with_merge_unmerge
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; CHECK-LABEL: name: test_or_mask_hi_with_merge_unmerge
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; CHECK-NEXT: $sgpr0 = COPY [[COPY]](s32)
+    ; CHECK-NEXT: $sgpr1 = COPY [[C]](s32)
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s64) = G_CONSTANT i64 -4294967296
+    %4:_(s64) = G_OR %2, %3
+    %5:_(s32), %6:_(s32) = G_UNMERGE_VALUES %4(s64)
+    $sgpr0 = COPY %5(s32)
+    $sgpr1 = COPY %6(s32)
+    SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+...
+---
+name: negative_or_test_incorrect_types
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: negative_or_test_incorrect_types
+    ; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 -4294967296
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s128) = G_OR [[COPY]], [[C]]
+    ; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[OR]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = COPY $vgpr4_vgpr5
+    %2:_(s128) = G_CONSTANT i128 -4294967296
+    %3:_(s128) = G_OR %0, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shift-imm-chain.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shift-imm-chain.ll
index 2d3088f3edb72..917cdb3f49a26 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shift-imm-chain.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shift-imm-chain.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn < %s | FileCheck %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn < %s | FileCheck %s
 
 define amdgpu_cs i32 @test_shl_1(i32 inreg %arg1) {
 ; CHECK-LABEL: test_shl_1:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shift-of-shifted-logic.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shift-of-shifted-logic.ll
index 5532443c0dfc8..914a26b2fb525 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shift-of-shifted-logic.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-shift-of-shifted-logic.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn < %s | FileCheck %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn < %s | FileCheck %s
 
 define amdgpu_cs i32 @test_shl_and_1(i32 inreg %arg1) {
 ; CHECK-LABEL: test_shl_and_1:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
index ff26ea21390e2..5dff8c16f7c89 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
@@ -227,54 +227,52 @@ exit:
 define amdgpu_cs void @single_lane_execution_attribute(i32 inreg %.userdata0, <3 x i32> inreg %.WorkgroupId, <3 x i32> %.LocalInvocationId) #0 {
 ; GFX10-LABEL: single_lane_execution_attribute:
 ; GFX10:       ; %bb.0: ; %.entry
-; GFX10-NEXT:    s_mov_b32 s6, 0
-; GFX10-NEXT:    s_getpc_b64 s[4:5]
-; GFX10-NEXT:    s_mov_b32 s7, -1
-; GFX10-NEXT:    s_mov_b32 s2, s1
-; GFX10-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
-; GFX10-NEXT:    s_mov_b32 s1, 0
+; GFX10-NEXT:    s_getpc_b64 s[12:13]
+; GFX10-NEXT:    s_mov_b32 s12, 0
+; GFX10-NEXT:    s_mov_b32 s2, s0
+; GFX10-NEXT:    s_mov_b32 s3, s12
 ; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v1, -1, 0
-; GFX10-NEXT:    s_or_b64 s[12:13], s[4:5], s[0:1]
-; GFX10-NEXT:    s_load_dwordx8 s[4:11], s[12:13], 0x0
+; GFX10-NEXT:    s_or_b64 s[2:3], s[12:13], s[2:3]
+; GFX10-NEXT:    s_load_dwordx8 s[4:11], s[2:3], 0x0
 ; GFX10-NEXT:    v_mbcnt_hi_u32_b32 v1, -1, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 2, v1
 ; GFX10-NEXT:    v_and_b32_e32 v3, 1, v1
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
-; GFX10-NEXT:    s_xor_b32 s3, vcc_lo, exec_lo
+; GFX10-NEXT:    s_xor_b32 s2, vcc_lo, exec_lo
+; GFX10-NEXT:    s_and_b32 vcc_lo, s2, exec_lo
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    buffer_load_dword v2, v2, s[4:7], 0 offen
-; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, s3
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 0, v2
 ; GFX10-NEXT:    s_cbranch_vccnz .LBB4_4
 ; GFX10-NEXT:  ; %bb.1: ; %.preheader.preheader
-; GFX10-NEXT:    s_mov_b32 s3, 0
+; GFX10-NEXT:    s_mov_b32 s2, 0
 ; GFX10-NEXT:  .LBB4_2: ; %.preheader
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_mov_b32_e32 v3, s1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s12
 ; GFX10-NEXT:    v_add_nc_u32_e32 v1, -1, v1
-; GFX10-NEXT:    s_add_i32 s1, s1, 4
+; GFX10-NEXT:    s_add_i32 s12, s12, 4
 ; GFX10-NEXT:    buffer_load_dword v3, v3, s[4:7], 0 offen
 ; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v1
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_readfirstlane_b32 s12, v3
-; GFX10-NEXT:    s_add_i32 s3, s12, s3
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    s_add_i32 s2, s3, s2
 ; GFX10-NEXT:    s_cbranch_vccnz .LBB4_2
 ; GFX10-NEXT:  ; %bb.3: ; %.preheader._crit_edge
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, s3, v2
-; GFX10-NEXT:    s_or_b32 s1, s0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s1
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, s2, v2
+; GFX10-NEXT:    s_or_b32 s2, s0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s2
 ; GFX10-NEXT:    s_branch .LBB4_6
 ; GFX10-NEXT:  .LBB4_4:
-; GFX10-NEXT:    s_mov_b32 s1, exec_lo
+; GFX10-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX10-NEXT:    ; implicit-def: $vgpr1
-; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 vcc_lo, exec_lo, s2
 ; GFX10-NEXT:    s_cbranch_vccz .LBB4_6
 ; GFX10-NEXT:  ; %bb.5: ; %.19
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
 ; GFX10-NEXT:    v_or_b32_e32 v1, 2, v1
 ; GFX10-NEXT:  .LBB4_6: ; %.22
-; GFX10-NEXT:    v_add_lshl_u32 v0, v0, s2, 2
+; GFX10-NEXT:    v_add_lshl_u32 v0, v0, s1, 2
 ; GFX10-NEXT:    buffer_store_dword v1, v0, s[8:11], 0 offen
 ; GFX10-NEXT:    s_endpgm
 .entry:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
index a8a75cd2ffaa8..dd01112d97a18 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
 
 ; This file contains various tests that have divergent i1s used outside of
 ; the loop. These are lane masks is sgpr and need to have correct value in
@@ -13,30 +13,27 @@ define void @divergent_i1_phi_used_outside_loop(float %val, float %pre.cond.val,
 ; GFX10-LABEL: divergent_i1_phi_used_outside_loop:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:    v_cmp_lt_f32_e64 s5, 1.0, v1
-; GFX10-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    s_mov_b32 s6, 0
 ; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:  .LBB0_1: ; %loop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v4, v1
-; GFX10-NEXT:    s_xor_b32 s8, s5, -1
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v0
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v1, s6
+; GFX10-NEXT:    s_mov_b32 s8, exec_lo
+; GFX10-NEXT:    s_add_i32 s6, s6, 1
+; GFX10-NEXT:    s_xor_b32 s8, s5, s8
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v0
 ; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, s5
-; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    s_or_b32 s7, s7, s5
+; GFX10-NEXT:    s_and_b32 s9, exec_lo, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s8
-; GFX10-NEXT:    s_and_b32 s9, exec_lo, s7
-; GFX10-NEXT:    s_or_b32 s6, s6, s9
+; GFX10-NEXT:    s_or_b32 s7, s7, s9
 ; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execnz .LBB0_1
 ; GFX10-NEXT:  ; %bb.2: ; %exit
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s7
 ; GFX10-NEXT:    flat_store_dword v[2:3], v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -63,43 +60,44 @@ define void @divergent_i1_phi_used_outside_loop_larger_loop_body(float %val, ptr
 ; GFX10-LABEL: divergent_i1_phi_used_outside_loop_larger_loop_body:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s4, -1
-; GFX10-NEXT:    ; implicit-def: $sgpr6
-; GFX10-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX10-NEXT:    s_andn2_b32 s5, s4, exec_lo
-; GFX10-NEXT:    s_and_b32 s4, exec_lo, -1
-; GFX10-NEXT:    s_or_b32 s4, s5, s4
+; GFX10-NEXT:    s_and_b32 s6, exec_lo, exec_lo
+; GFX10-NEXT:    s_mov_b32 s4, -1
+; GFX10-NEXT:    s_or_b32 s7, s5, s6
+; GFX10-NEXT:    ; implicit-def: $sgpr5
 ; GFX10-NEXT:    s_branch .LBB1_2
 ; GFX10-NEXT:  .LBB1_1: ; %loop.cond
 ; GFX10-NEXT:    ; in Loop: Header=BB1_2 Depth=1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    v_add_co_u32 v1, s4, v1, 4
-; GFX10-NEXT:    v_add_nc_u32_e32 v0, 1, v0
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v2, s4, 0, v2, s4
-; GFX10-NEXT:    s_andn2_b32 s7, s5, exec_lo
-; GFX10-NEXT:    s_and_b32 s8, exec_lo, s6
-; GFX10-NEXT:    v_cmp_le_i32_e32 vcc_lo, 10, v0
-; GFX10-NEXT:    s_or_b32 s4, s7, s8
-; GFX10-NEXT:    s_cbranch_vccz .LBB1_4
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT:    s_add_i32 s4, s4, 1
+; GFX10-NEXT:    v_add_co_u32 v1, vcc_lo, v1, 4
+; GFX10-NEXT:    s_cmp_ge_i32 s4, 10
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, 0, v2, vcc_lo
+; GFX10-NEXT:    s_cselect_b32 s8, 1, 0
+; GFX10-NEXT:    s_andn2_b32 s7, s6, exec_lo
+; GFX10-NEXT:    s_and_b32 s9, exec_lo, s5
+; GFX10-NEXT:    s_or_b32 s7, s7, s9
+; GFX10-NEXT:    s_cmp_lg_u32 s8, 0
+; GFX10-NEXT:    s_cbranch_scc0 .LBB1_4
 ; GFX10-NEXT:  .LBB1_2: ; %loop.start
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_mov_b32 s5, s4
-; GFX10-NEXT:    s_andn2_b32 s4, s6, exec_lo
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, s5
-; GFX10-NEXT:    s_or_b32 s6, s4, s6
-; GFX10-NEXT:    s_and_saveexec_b32 s4, s5
+; GFX10-NEXT:    s_mov_b32 s6, s7
+; GFX10-NEXT:    s_andn2_b32 s5, s5, exec_lo
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, s7
+; GFX10-NEXT:    s_or_b32 s5, s5, s7
+; GFX10-NEXT:    s_and_saveexec_b32 s7, s6
 ; GFX10-NEXT:    s_cbranch_execz .LBB1_1
 ; GFX10-NEXT:  ; %bb.3: ; %is.eq.zero
 ; GFX10-NEXT:    ; in Loop: Header=BB1_2 Depth=1
-; GFX10-NEXT:    global_load_dword v5, v[1:2], off
-; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT:    global_load_dword v0, v[1:2], off
+; GFX10-NEXT:    s_andn2_b32 s5, s5, exec_lo
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v5
-; GFX10-NEXT:    s_and_b32 s7, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s6, s6, s7
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, vcc_lo
+; GFX10-NEXT:    s_or_b32 s5, s5, s8
 ; GFX10-NEXT:    s_branch .LBB1_1
 ; GFX10-NEXT:  .LBB1_4: ; %exit
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s6
 ; GFX10-NEXT:    flat_store_dword v[3:4], v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -135,29 +133,26 @@ define void @divergent_i1_xor_used_outside_loop(float %val, float %pre.cond.val,
 ; GFX10-LABEL: divergent_i1_xor_used_outside_loop:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:    v_cmp_lt_f32_e64 s5, 1.0, v1
-; GFX10-NEXT:    v_mov_b32_e32 v1, s4
-; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    s_mov_b32 s6, 0
 ; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:  .LBB2_1: ; %loop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v4, v1
-; GFX10-NEXT:    s_xor_b32 s5, s5, -1
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v0
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v1, s6
+; GFX10-NEXT:    s_mov_b32 s8, exec_lo
+; GFX10-NEXT:    s_add_i32 s6, s6, 1
+; GFX10-NEXT:    s_xor_b32 s5, s5, s8
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v0
 ; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
 ; GFX10-NEXT:    s_and_b32 s8, exec_lo, s5
-; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
 ; GFX10-NEXT:    s_or_b32 s7, s7, s8
-; GFX10-NEXT:    s_and_b32 s8, exec_lo, s7
-; GFX10-NEXT:    s_or_b32 s6, s6, s8
 ; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execnz .LBB2_1
 ; GFX10-NEXT:  ; %bb.2: ; %exit
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s6
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s7
 ; GFX10-NEXT:    flat_store_dword v[2:3], v0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
@@ -184,23 +179,20 @@ define void @divergent_i1_xor_used_outside_loop_twice(float %val, float %pre.con
 ; GFX10-LABEL: divergent_i1_xor_used_outside_loop_twice:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:    v_cmp_lt_f32_e64 s5, 1.0, v1
-; GFX10-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    s_mov_b32 s7, 0
 ; GFX10-NEXT:    ; implicit-def: $sgpr6
-; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:  .LBB3_1: ; %loop
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v6, v1
-; GFX10-NEXT:    s_xor_b32 s5, s5, -1
-; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v0
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v1, s7
+; GFX10-NEXT:    s_mov_b32 s8, exec_lo
+; GFX10-NEXT:    s_add_i32 s7, s7, 1
+; GFX10-NEXT:    s_xor_b32 s5, s5, s8
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v1, v0
 ; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
-; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT:    s_and_b32 s8, exec_lo, s5
 ; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    s_or_b32 s7, s7, s8
-; GFX10-NEXT:    s_and_b32 s8, exec_lo, s7
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, s5
 ; GFX10-NEXT:    s_or_b32 s6, s6, s8
 ; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execnz .LBB3_1
@@ -247,66 +239,64 @@ define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts,
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT:    s_mov_b32 s5, 0
-; GFX10-NEXT:    s_mov_b32 s6, -1
-; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX10-NEXT:    s_mov_b32 s6, exec_lo
+; GFX10-NEXT:    s_mov_b32 s8, 0
+; GFX10-NEXT:    s_and_saveexec_b32 s7, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB4_6
 ; GFX10-NEXT:  ; %bb.1: ; %loop.start.preheader
-; GFX10-NEXT:    v_mov_b32_e32 v5, s5
-; GFX10-NEXT:    ; implicit-def: $sgpr6
-; GFX10-NEXT:    ; implicit-def: $sgpr8
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    ; implicit-def: $sgpr10
+; GFX10-NEXT:    ; implicit-def: $sgpr11
 ; GFX10-NEXT:    ; implicit-def: $sgpr9
-; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:    s_branch .LBB4_3
 ; GFX10-NEXT:  .LBB4_2: ; %Flow
 ; GFX10-NEXT:    ; in Loop: Header=BB4_3 Depth=1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s10
-; GFX10-NEXT:    s_xor_b32 s10, s9, -1
-; GFX10-NEXT:    s_and_b32 s11, exec_lo, s8
-; GFX10-NEXT:    s_or_b32 s5, s11, s5
-; GFX10-NEXT:    s_andn2_b32 s7, s7, exec_lo
-; GFX10-NEXT:    s_and_b32 s10, exec_lo, s10
-; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    s_or_b32 s7, s7, s10
-; GFX10-NEXT:    s_and_b32 s10, exec_lo, s7
-; GFX10-NEXT:    s_or_b32 s6, s6, s10
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_xor_b32 s5, s11, exec_lo
+; GFX10-NEXT:    s_and_b32 s12, exec_lo, s10
+; GFX10-NEXT:    s_or_b32 s8, s12, s8
+; GFX10-NEXT:    s_andn2_b32 s9, s9, exec_lo
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s5
+; GFX10-NEXT:    s_or_b32 s9, s9, s5
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s8
 ; GFX10-NEXT:    s_cbranch_execz .LBB4_5
 ; GFX10-NEXT:  .LBB4_3: ; %loop.start
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
-; GFX10-NEXT:    s_andn2_b32 s9, s9, exec_lo
-; GFX10-NEXT:    s_and_b32 s10, exec_lo, -1
-; GFX10-NEXT:    s_andn2_b32 s8, s8, exec_lo
-; GFX10-NEXT:    s_or_b32 s9, s9, s10
-; GFX10-NEXT:    v_lshlrev_b64 v[6:7], 2, v[5:6]
-; GFX10-NEXT:    s_or_b32 s8, s8, s10
-; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v1, v6
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v2, v7, vcc_lo
-; GFX10-NEXT:    global_load_dword v6, v[6:7], off
+; GFX10-NEXT:    s_ashr_i32 s5, s4, 31
+; GFX10-NEXT:    s_andn2_b32 s10, s10, exec_lo
+; GFX10-NEXT:    s_lshl_b64 s[12:13], s[4:5], 2
+; GFX10-NEXT:    s_andn2_b32 s5, s11, exec_lo
+; GFX10-NEXT:    v_mov_b32_e32 v5, s12
+; GFX10-NEXT:    v_mov_b32_e32 v6, s13
+; GFX10-NEXT:    s_and_b32 s11, exec_lo, exec_lo
+; GFX10-NEXT:    s_and_b32 s12, exec_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s11, s5, s11
+; GFX10-NEXT:    v_add_co_u32 v5, vcc_lo, v1, v5
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v2, v6, vcc_lo
+; GFX10-NEXT:    s_or_b32 s10, s10, s12
+; GFX10-NEXT:    global_load_dword v5, v[5:6], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v6
-; GFX10-NEXT:    s_and_saveexec_b32 s10, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v5
+; GFX10-NEXT:    s_and_saveexec_b32 s5, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB4_2
 ; GFX10-NEXT:  ; %bb.4: ; %loop.cond
 ; GFX10-NEXT:    ; in Loop: Header=BB4_3 Depth=1
-; GFX10-NEXT:    v_add_nc_u32_e32 v6, 1, v5
-; GFX10-NEXT:    v_cmp_lt_i32_e32 vcc_lo, v5, v0
-; GFX10-NEXT:    s_andn2_b32 s9, s9, exec_lo
-; GFX10-NEXT:    s_and_b32 s11, exec_lo, 0
-; GFX10-NEXT:    s_andn2_b32 s8, s8, exec_lo
-; GFX10-NEXT:    v_mov_b32_e32 v5, v6
-; GFX10-NEXT:    s_and_b32 s12, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s9, s9, s11
-; GFX10-NEXT:    s_or_b32 s8, s8, s12
+; GFX10-NEXT:    v_cmp_lt_i32_e32 vcc_lo, s4, v0
+; GFX10-NEXT:    s_andn2_b32 s11, s11, exec_lo
+; GFX10-NEXT:    s_and_b32 s12, exec_lo, 0
+; GFX10-NEXT:    s_andn2_b32 s10, s10, exec_lo
+; GFX10-NEXT:    s_add_i32 s4, s4, 1
+; GFX10-NEXT:    s_and_b32 s13, exec_lo, vcc_lo
+; GFX10-NEXT:    s_or_b32 s11, s11, s12
+; GFX10-NEXT:    s_or_b32 s10, s10, s13
 ; GFX10-NEXT:    s_branch .LBB4_2
 ; GFX10-NEXT:  .LBB4_5: ; %loop.exit.guard
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    s_andn2_b32 s5, -1, exec_lo
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, s6
-; GFX10-NEXT:    s_or_b32 s6, s5, s6
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT:    s_andn2_b32 s4, s6, exec_lo
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s9
+; GFX10-NEXT:    s_or_b32 s6, s4, s5
 ; GFX10-NEXT:  .LBB4_6: ; %Flow1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s7
 ; GFX10-NEXT:    s_and_saveexec_b32 s4, s6
 ; GFX10-NEXT:    s_cbranch_execz .LBB4_8
 ; GFX10-NEXT:  ; %bb.7: ; %block.after.loop
@@ -355,56 +345,54 @@ define void @divergent_i1_icmp_used_outside_loop(i32 %v0, i32 %v1, ptr addrspace
 ; GFX10-LABEL: divergent_i1_icmp_used_outside_loop:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s6, 0
 ; GFX10-NEXT:    s_mov_b32 s4, 0
-; GFX10-NEXT:    ; implicit-def: $sgpr5
-; GFX10-NEXT:    ; implicit-def: $sgpr6
-; GFX10-NEXT:    v_mov_b32_e32 v5, s4
+; GFX10-NEXT:    ; implicit-def: $sgpr7
 ; GFX10-NEXT:    s_branch .LBB5_2
 ; GFX10-NEXT:  .LBB5_1: ; %Flow
 ; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT:    s_and_b32 s7, exec_lo, s7
-; GFX10-NEXT:    s_or_b32 s4, s7, s4
-; GFX10-NEXT:    s_andn2_b32 s5, s5, exec_lo
-; GFX10-NEXT:    s_and_b32 s7, exec_lo, s6
-; GFX10-NEXT:    s_or_b32 s5, s5, s7
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s5
+; GFX10-NEXT:    s_or_b32 s6, s5, s6
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s6
 ; GFX10-NEXT:    s_cbranch_execz .LBB5_6
 ; GFX10-NEXT:  .LBB5_2: ; %cond.block.0
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_mov_b32_e32 v4, v5
-; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, s4, v0
+; GFX10-NEXT:    v_mov_b32_e32 v4, s4
+; GFX10-NEXT:    s_andn2_b32 s5, s7, exec_lo
 ; GFX10-NEXT:    s_and_b32 s7, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s6, s6, s7
-; GFX10-NEXT:    s_and_saveexec_b32 s7, vcc_lo
+; GFX10-NEXT:    s_or_b32 s7, s5, s7
+; GFX10-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB5_4
 ; GFX10-NEXT:  ; %bb.3: ; %if.block.0
 ; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
-; GFX10-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
-; GFX10-NEXT:    v_lshlrev_b64 v[8:9], 2, v[4:5]
+; GFX10-NEXT:    s_ashr_i32 s5, s4, 31
+; GFX10-NEXT:    v_mov_b32_e32 v5, s4
+; GFX10-NEXT:    s_lshl_b64 s[10:11], s[4:5], 2
+; GFX10-NEXT:    v_mov_b32_e32 v8, s10
+; GFX10-NEXT:    v_mov_b32_e32 v9, s11
 ; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v2, v8
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v3, v9, vcc_lo
-; GFX10-NEXT:    global_store_dword v[8:9], v4, off
+; GFX10-NEXT:    global_store_dword v[8:9], v5, off
 ; GFX10-NEXT:  .LBB5_4: ; %loop.break.block
 ; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s7
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v4
-; GFX10-NEXT:    s_mov_b32 s7, -1
-; GFX10-NEXT:    ; implicit-def: $vgpr5
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s4, v1
+; GFX10-NEXT:    s_mov_b32 s5, exec_lo
 ; GFX10-NEXT:    s_and_saveexec_b32 s8, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB5_1
 ; GFX10-NEXT:  ; %bb.5: ; %loop.cond
 ; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
-; GFX10-NEXT:    v_add_nc_u32_e32 v5, 1, v4
-; GFX10-NEXT:    s_andn2_b32 s7, -1, exec_lo
+; GFX10-NEXT:    s_andn2_b32 s5, s5, exec_lo
 ; GFX10-NEXT:    s_and_b32 s9, exec_lo, 0
-; GFX10-NEXT:    s_or_b32 s7, s7, s9
+; GFX10-NEXT:    s_add_i32 s4, s4, 1
+; GFX10-NEXT:    s_or_b32 s5, s5, s9
 ; GFX10-NEXT:    s_branch .LBB5_1
 ; GFX10-NEXT:  .LBB5_6: ; %cond.block.1
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT:    s_and_saveexec_b32 s4, s5
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT:    s_and_saveexec_b32 s4, s7
 ; GFX10-NEXT:    s_cbranch_execz .LBB5_8
 ; GFX10-NEXT:  ; %bb.7: ; %if.block.1
 ; GFX10-NEXT:    global_store_dword v[6:7], v4, off
@@ -468,52 +456,50 @@ exit:
 define amdgpu_ps void @divergent_i1_freeze_used_outside_loop(i32 %n, ptr addrspace(1) %a, ptr %addr) {
 ; GFX10-LABEL: divergent_i1_freeze_used_outside_loop:
 ; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s1, exec_lo
+; GFX10-NEXT:    s_mov_b32 s2, 0
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    s_mov_b32 s4, -1
-; GFX10-NEXT:    v_mov_b32_e32 v5, s0
+; GFX10-NEXT:    ; implicit-def: $sgpr4
 ; GFX10-NEXT:    ; implicit-def: $sgpr3
-; GFX10-NEXT:    ; implicit-def: $sgpr1
-; GFX10-NEXT:    ; implicit-def: $sgpr2
 ; GFX10-NEXT:    s_branch .LBB6_2
 ; GFX10-NEXT:  .LBB6_1: ; %loop.cond
 ; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
 ; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    v_cmp_lt_i32_e32 vcc_lo, v5, v0
-; GFX10-NEXT:    v_add_nc_u32_e32 v5, 1, v5
-; GFX10-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, s1
-; GFX10-NEXT:    s_andn2_b32 s4, s4, exec_lo
-; GFX10-NEXT:    s_or_b32 s2, s2, s5
+; GFX10-NEXT:    v_cmp_lt_i32_e32 vcc_lo, s0, v0
+; GFX10-NEXT:    s_add_i32 s0, s0, 1
+; GFX10-NEXT:    s_or_b32 s2, vcc_lo, s2
 ; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, s2
-; GFX10-NEXT:    s_or_b32 s4, s4, s5
-; GFX10-NEXT:    s_or_b32 s3, s3, s6
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s4
+; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
+; GFX10-NEXT:    s_or_b32 s3, s3, s5
+; GFX10-NEXT:    s_or_b32 s1, s1, s5
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s2
 ; GFX10-NEXT:    s_cbranch_execz .LBB6_4
 ; GFX10-NEXT:  .LBB6_2: ; %loop.start
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, s4
-; GFX10-NEXT:    s_or_b32 s1, s1, s5
-; GFX10-NEXT:    s_and_saveexec_b32 s5, s4
+; GFX10-NEXT:    s_andn2_b32 s4, s4, exec_lo
+; GFX10-NEXT:    s_and_b32 s5, exec_lo, s1
+; GFX10-NEXT:    s_or_b32 s4, s4, s5
+; GFX10-NEXT:    s_and_saveexec_b32 s5, s1
 ; GFX10-NEXT:    s_cbranch_execz .LBB6_1
 ; GFX10-NEXT:  ; %bb.3: ; %is.eq.zero
 ; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
-; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    v_lshlrev_b64 v[6:7], 2, v[5:6]
-; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v1, v6
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v2, v7, vcc_lo
-; GFX10-NEXT:    global_load_dword v6, v[6:7], off
+; GFX10-NEXT:    s_ashr_i32 s1, s0, 31
+; GFX10-NEXT:    s_lshl_b64 s[6:7], s[0:1], 2
+; GFX10-NEXT:    s_andn2_b32 s1, s4, exec_lo
+; GFX10-NEXT:    v_mov_b32_e32 v5, s6
+; GFX10-NEXT:    v_mov_b32_e32 v6, s7
+; GFX10-NEXT:    v_add_co_u32 v5, vcc_lo, v1, v5
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v2, v6, vcc_lo
+; GFX10-NEXT:    global_load_dword v5, v[5:6], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v5
 ; GFX10-NEXT:    s_and_b32 s4, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s1, s1, s4
-; GFX10-NEXT:    ; implicit-def: $sgpr4
+; GFX10-NEXT:    s_or_b32 s4, s1, s4
+; GFX10-NEXT:    ; implicit-def: $sgpr1
 ; GFX10-NEXT:    s_branch .LBB6_1
 ; GFX10-NEXT:  .LBB6_4: ; %exit
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s3
 ; GFX10-NEXT:    flat_store_dword v[3:4], v0
 ; GFX10-NEXT:    s_endpgm
@@ -548,64 +534,67 @@ exit:
 define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a, ptr addrspace(1) %a.break) {
 ; GFX10-LABEL: loop_with_1break:
 ; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s4, 0
 ; GFX10-NEXT:    s_mov_b32 s0, 0
-; GFX10-NEXT:    ; implicit-def: $sgpr1
-; GFX10-NEXT:    ; implicit-def: $sgpr3
-; GFX10-NEXT:    ; implicit-def: $sgpr4
-; GFX10-NEXT:    ; implicit-def: $sgpr2
-; GFX10-NEXT:    v_mov_b32_e32 v6, s0
+; GFX10-NEXT:    ; implicit-def: $sgpr6
+; GFX10-NEXT:    ; implicit-def: $sgpr7
+; GFX10-NEXT:    ; implicit-def: $sgpr5
 ; GFX10-NEXT:    s_branch .LBB7_2
 ; GFX10-NEXT:  .LBB7_1: ; %Flow
 ; GFX10-NEXT:    ; in Loop: Header=BB7_2 Depth=1
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, s3
-; GFX10-NEXT:    s_or_b32 s0, s5, s0
-; GFX10-NEXT:    s_andn2_b32 s2, s2, exec_lo
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, s4
-; GFX10-NEXT:    s_andn2_b32 s1, s1, exec_lo
-; GFX10-NEXT:    s_or_b32 s2, s2, s5
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, s2
-; GFX10-NEXT:    s_or_b32 s1, s1, s5
-; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s6
+; GFX10-NEXT:    s_or_b32 s4, s1, s4
+; GFX10-NEXT:    s_andn2_b32 s1, s5, exec_lo
+; GFX10-NEXT:    s_and_b32 s2, exec_lo, s7
+; GFX10-NEXT:    s_or_b32 s5, s1, s2
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
 ; GFX10-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX10-NEXT:  .LBB7_2: ; %A
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
-; GFX10-NEXT:    s_andn2_b32 s4, s4, exec_lo
-; GFX10-NEXT:    s_and_b32 s5, exec_lo, -1
-; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT:    s_or_b32 s4, s4, s5
-; GFX10-NEXT:    v_lshlrev_b64 v[7:8], 2, v[6:7]
-; GFX10-NEXT:    s_or_b32 s3, s3, s5
-; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v2, v7
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v3, v8, vcc_lo
-; GFX10-NEXT:    global_load_dword v9, v[9:10], off
+; GFX10-NEXT:    s_ashr_i32 s1, s0, 31
+; GFX10-NEXT:    s_mov_b32 s8, exec_lo
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[0:1], 2
+; GFX10-NEXT:    s_andn2_b32 s1, s7, exec_lo
+; GFX10-NEXT:    v_mov_b32_e32 v7, s3
+; GFX10-NEXT:    v_mov_b32_e32 v6, s2
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, s8
+; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT:    s_and_b32 s8, exec_lo, exec_lo
+; GFX10-NEXT:    s_or_b32 s7, s1, s7
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v2, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v3, v7, vcc_lo
+; GFX10-NEXT:    s_or_b32 s6, s6, s8
+; GFX10-NEXT:    global_load_dword v6, v[6:7], off
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
-; GFX10-NEXT:    s_and_saveexec_b32 s5, vcc_lo
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
 ; GFX10-NEXT:    s_cbranch_execz .LBB7_1
 ; GFX10-NEXT:  ; %bb.3: ; %loop.body
 ; GFX10-NEXT:    ; in Loop: Header=BB7_2 Depth=1
-; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v7
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v8, vcc_lo
-; GFX10-NEXT:    v_add_nc_u32_e32 v10, 1, v6
-; GFX10-NEXT:    v_cmp_gt_u32_e32 vcc_lo, 0x64, v6
-; GFX10-NEXT:    s_andn2_b32 s4, s4, exec_lo
-; GFX10-NEXT:    global_load_dword v9, v[7:8], off
-; GFX10-NEXT:    s_and_b32 s6, exec_lo, 0
-; GFX10-NEXT:    v_mov_b32_e32 v6, v10
-; GFX10-NEXT:    s_andn2_b32 s3, s3, exec_lo
-; GFX10-NEXT:    s_and_b32 s7, exec_lo, vcc_lo
-; GFX10-NEXT:    s_or_b32 s4, s4, s6
-; GFX10-NEXT:    s_or_b32 s3, s3, s7
+; GFX10-NEXT:    v_mov_b32_e32 v7, s3
+; GFX10-NEXT:    v_mov_b32_e32 v6, s2
+; GFX10-NEXT:    s_add_i32 s2, s0, 1
+; GFX10-NEXT:    s_cmpk_lt_u32 s0, 0x64
+; GFX10-NEXT:    s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v0, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
+; GFX10-NEXT:    s_andn2_b32 s3, s7, exec_lo
+; GFX10-NEXT:    s_and_b32 s7, exec_lo, 0
+; GFX10-NEXT:    s_andn2_b32 s6, s6, exec_lo
+; GFX10-NEXT:    global_load_dword v8, v[6:7], off
+; GFX10-NEXT:    s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 s7, s3, s7
+; GFX10-NEXT:    s_or_b32 s6, s6, s0
+; GFX10-NEXT:    s_mov_b32 s0, s2
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v9
-; GFX10-NEXT:    global_store_dword v[7:8], v9, off
+; GFX10-NEXT:    v_add_nc_u32_e32 v8, 1, v8
+; GFX10-NEXT:    global_store_dword v[6:7], v8, off
 ; GFX10-NEXT:    s_branch .LBB7_1
 ; GFX10-NEXT:  .LBB7_4: ; %loop.exit.guard
-; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT:    s_and_saveexec_b32 s0, s1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_and_saveexec_b32 s0, s5
 ; GFX10-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX10-NEXT:    s_cbranch_execz .LBB7_6
 ; GFX10-NEXT:  ; %bb.5: ; %break.body
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch-init.gfx.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch-init.gfx.ll
index 4fdb4082346af..b520ce1826ec9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch-init.gfx.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch-init.gfx.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mattr=+enable-flat-scratch -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefix=MESA %s
-; RUN: llc -global-isel -mattr=+enable-flat-scratch -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=PAL %s
+; RUN: llc -global-isel -new-reg-bank-select -mattr=+enable-flat-scratch -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 < %s | FileCheck -check-prefix=MESA %s
+; RUN: llc -global-isel -new-reg-bank-select -mattr=+enable-flat-scratch -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=PAL %s
 
 ; Test that the initialization for flat_scratch doesn't crash.  PAL
 ; doesn't add a user SGPR for initializing flat_scratch, mesa does
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
index a066b15f84d6b..e6a8baceee020 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
@@ -1917,8 +1917,9 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; GFX9-NEXT:    s_mov_b32 s0, 0
 ; GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s0, 0x3e80
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 15
-; GFX9-NEXT:    s_movk_i32 s0, 0x3e84
+; GFX9-NEXT:    s_add_i32 s0, s0, 4
 ; GFX9-NEXT:    scratch_store_dword off, v0, s0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
@@ -1933,7 +1934,8 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 13
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
-; GFX10-NEXT:    s_movk_i32 s0, 0x3e84
+; GFX10-NEXT:    s_movk_i32 s0, 0x3e80
+; GFX10-NEXT:    s_add_i32 s0, s0, 4
 ; GFX10-NEXT:    scratch_store_dword off, v0, off offset:4
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_store_dword off, v1, s0
@@ -1945,10 +1947,11 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; GFX942-LABEL: store_load_large_imm_offset_kernel:
 ; GFX942:       ; %bb.0: ; %bb
 ; GFX942-NEXT:    v_mov_b32_e32 v0, 13
+; GFX942-NEXT:    s_movk_i32 s0, 0x3e80
 ; GFX942-NEXT:    scratch_store_dword off, v0, off offset:4 sc0 sc1
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_mov_b32_e32 v0, 15
-; GFX942-NEXT:    s_movk_i32 s0, 0x3e84
+; GFX942-NEXT:    s_add_i32 s0, s0, 4
 ; GFX942-NEXT:    scratch_store_dword off, v0, s0 sc0 sc1
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
@@ -1958,7 +1961,9 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; GFX11-LABEL: store_load_large_imm_offset_kernel:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 13 :: v_dual_mov_b32 v1, 15
-; GFX11-NEXT:    s_movk_i32 s0, 0x3e84
+; GFX11-NEXT:    s_movk_i32 s0, 0x3e80
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_i32 s0, s0, 4
 ; GFX11-NEXT:    scratch_store_b32 off, v0, off offset:4 dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    scratch_store_b32 off, v1, s0 dlc
@@ -1986,8 +1991,9 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; UNALIGNED_GFX9-NEXT:    s_mov_b32 s0, 0
 ; UNALIGNED_GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:4
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
+; UNALIGNED_GFX9-NEXT:    s_movk_i32 s0, 0x3e80
 ; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v0, 15
-; UNALIGNED_GFX9-NEXT:    s_movk_i32 s0, 0x3e84
+; UNALIGNED_GFX9-NEXT:    s_add_i32 s0, s0, 4
 ; UNALIGNED_GFX9-NEXT:    scratch_store_dword off, v0, s0
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
@@ -2002,7 +2008,8 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; UNALIGNED_GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v0, 13
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v1, 15
-; UNALIGNED_GFX10-NEXT:    s_movk_i32 s0, 0x3e84
+; UNALIGNED_GFX10-NEXT:    s_movk_i32 s0, 0x3e80
+; UNALIGNED_GFX10-NEXT:    s_add_i32 s0, s0, 4
 ; UNALIGNED_GFX10-NEXT:    scratch_store_dword off, v0, off offset:4
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX10-NEXT:    scratch_store_dword off, v1, s0
@@ -2014,10 +2021,11 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; UNALIGNED_GFX942-LABEL: store_load_large_imm_offset_kernel:
 ; UNALIGNED_GFX942:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX942-NEXT:    v_mov_b32_e32 v0, 13
+; UNALIGNED_GFX942-NEXT:    s_movk_i32 s0, 0x3e80
 ; UNALIGNED_GFX942-NEXT:    scratch_store_dword off, v0, off offset:4 sc0 sc1
 ; UNALIGNED_GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX942-NEXT:    v_mov_b32_e32 v0, 15
-; UNALIGNED_GFX942-NEXT:    s_movk_i32 s0, 0x3e84
+; UNALIGNED_GFX942-NEXT:    s_add_i32 s0, s0, 4
 ; UNALIGNED_GFX942-NEXT:    scratch_store_dword off, v0, s0 sc0 sc1
 ; UNALIGNED_GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX942-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
@@ -2027,7 +2035,9 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; UNALIGNED_GFX11-LABEL: store_load_large_imm_offset_kernel:
 ; UNALIGNED_GFX11:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v0, 13 :: v_dual_mov_b32 v1, 15
-; UNALIGNED_GFX11-NEXT:    s_movk_i32 s0, 0x3e84
+; UNALIGNED_GFX11-NEXT:    s_movk_i32 s0, 0x3e80
+; UNALIGNED_GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; UNALIGNED_GFX11-NEXT:    s_add_i32 s0, s0, 4
 ; UNALIGNED_GFX11-NEXT:    scratch_store_b32 off, v0, off offset:4 dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX11-NEXT:    scratch_store_b32 off, v1, s0 dlc
@@ -2061,11 +2071,13 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX9-LABEL: store_load_large_imm_offset_foo:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s0, 0x3e80
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 13
+; GFX9-NEXT:    s_add_i32 s1, s32, s0
 ; GFX9-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 15
-; GFX9-NEXT:    s_add_i32 s0, s32, 0x3e84
+; GFX9-NEXT:    s_add_i32 s0, s1, 4
 ; GFX9-NEXT:    scratch_store_dword off, v0, s0
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
@@ -2076,8 +2088,10 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 13
+; GFX10-NEXT:    s_movk_i32 s0, 0x3e80
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
-; GFX10-NEXT:    s_add_i32 s0, s32, 0x3e84
+; GFX10-NEXT:    s_add_i32 s1, s32, s0
+; GFX10-NEXT:    s_add_i32 s0, s1, 4
 ; GFX10-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_store_dword off, v1, s0
@@ -2089,11 +2103,13 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX942-LABEL: store_load_large_imm_offset_foo:
 ; GFX942:       ; %bb.0: ; %bb
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    s_movk_i32 s0, 0x3e80
 ; GFX942-NEXT:    v_mov_b32_e32 v0, 13
+; GFX942-NEXT:    s_add_i32 s1, s32, s0
 ; GFX942-NEXT:    scratch_store_dword off, v0, s32 offset:4 sc0 sc1
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    v_mov_b32_e32 v0, 15
-; GFX942-NEXT:    s_add_i32 s0, s32, 0x3e84
+; GFX942-NEXT:    s_add_i32 s0, s1, 4
 ; GFX942-NEXT:    scratch_store_dword off, v0, s0 sc0 sc1
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
@@ -2104,7 +2120,10 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, 13 :: v_dual_mov_b32 v1, 15
-; GFX11-NEXT:    s_add_i32 s0, s32, 0x3e84
+; GFX11-NEXT:    s_movk_i32 s0, 0x3e80
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_i32 s1, s32, s0
+; GFX11-NEXT:    s_add_i32 s0, s1, 4
 ; GFX11-NEXT:    scratch_store_b32 off, v0, s32 offset:4 dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    scratch_store_b32 off, v1, s0 dlc
@@ -2133,11 +2152,13 @@ define void @store_load_large_imm_offset_foo() {
 ; UNALIGNED_GFX9-LABEL: store_load_large_imm_offset_foo:
 ; UNALIGNED_GFX9:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNALIGNED_GFX9-NEXT:    s_movk_i32 s0, 0x3e80
 ; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v0, 13
+; UNALIGNED_GFX9-NEXT:    s_add_i32 s1, s32, s0
 ; UNALIGNED_GFX9-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    v_mov_b32_e32 v0, 15
-; UNALIGNED_GFX9-NEXT:    s_add_i32 s0, s32, 0x3e84
+; UNALIGNED_GFX9-NEXT:    s_add_i32 s0, s1, 4
 ; UNALIGNED_GFX9-NEXT:    scratch_store_dword off, v0, s0
 ; UNALIGNED_GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX9-NEXT:    scratch_load_dword v0, off, s0 glc
@@ -2148,8 +2169,10 @@ define void @store_load_large_imm_offset_foo() {
 ; UNALIGNED_GFX10:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v0, 13
+; UNALIGNED_GFX10-NEXT:    s_movk_i32 s0, 0x3e80
 ; UNALIGNED_GFX10-NEXT:    v_mov_b32_e32 v1, 15
-; UNALIGNED_GFX10-NEXT:    s_add_i32 s0, s32, 0x3e84
+; UNALIGNED_GFX10-NEXT:    s_add_i32 s1, s32, s0
+; UNALIGNED_GFX10-NEXT:    s_add_i32 s0, s1, 4
 ; UNALIGNED_GFX10-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; UNALIGNED_GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX10-NEXT:    scratch_store_dword off, v1, s0
@@ -2161,11 +2184,13 @@ define void @store_load_large_imm_offset_foo() {
 ; UNALIGNED_GFX942-LABEL: store_load_large_imm_offset_foo:
 ; UNALIGNED_GFX942:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; UNALIGNED_GFX942-NEXT:    s_movk_i32 s0, 0x3e80
 ; UNALIGNED_GFX942-NEXT:    v_mov_b32_e32 v0, 13
+; UNALIGNED_GFX942-NEXT:    s_add_i32 s1, s32, s0
 ; UNALIGNED_GFX942-NEXT:    scratch_store_dword off, v0, s32 offset:4 sc0 sc1
 ; UNALIGNED_GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX942-NEXT:    v_mov_b32_e32 v0, 15
-; UNALIGNED_GFX942-NEXT:    s_add_i32 s0, s32, 0x3e84
+; UNALIGNED_GFX942-NEXT:    s_add_i32 s0, s1, 4
 ; UNALIGNED_GFX942-NEXT:    scratch_store_dword off, v0, s0 sc0 sc1
 ; UNALIGNED_GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; UNALIGNED_GFX942-NEXT:    scratch_load_dword v0, off, s0 sc0 sc1
@@ -2176,7 +2201,10 @@ define void @store_load_large_imm_offset_foo() {
 ; UNALIGNED_GFX11:       ; %bb.0: ; %bb
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; UNALIGNED_GFX11-NEXT:    v_dual_mov_b32 v0, 13 :: v_dual_mov_b32 v1, 15
-; UNALIGNED_GFX11-NEXT:    s_add_i32 s0, s32, 0x3e84
+; UNALIGNED_GFX11-NEXT:    s_movk_i32 s0, 0x3e80
+; UNALIGNED_GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; UNALIGNED_GFX11-NEXT:    s_add_i32 s1, s32, s0
+; UNALIGNED_GFX11-NEXT:    s_add_i32 s0, s1, 4
 ; UNALIGNED_GFX11-NEXT:    scratch_store_b32 off, v0, s32 offset:4 dlc
 ; UNALIGNED_GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; UNALIGNED_GFX11-NEXT:    scratch_store_b32 off, v1, s0 dlc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-atomics-gfx942.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-atomics-gfx942.ll
index 6792612ded368..7766b3ad45962 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-atomics-gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-atomics-gfx942.ll
@@ -108,7 +108,7 @@ define <2 x half> @flat_atomic_fadd_ret_v2f16_agent_offset(ptr %ptr, <2 x half>
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    buffer_inv sc1
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr <2 x half>, ptr %ptr, i32 256
+  %gep = getelementptr inbounds <2 x half>, ptr %ptr, i32 256
   %result = atomicrmw fadd ptr %gep, <2 x half> %val syncscope("agent") seq_cst
   ret <2 x half> %result
 }
@@ -122,7 +122,7 @@ define void @flat_atomic_fadd_noret_v2f16_agent_offset(ptr %ptr, <2 x half> %val
 ; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX942-NEXT:    buffer_inv sc1
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr <2 x half>, ptr %ptr, i32 256
+  %gep = getelementptr inbounds <2 x half>, ptr %ptr, i32 256
   %unused = atomicrmw fadd ptr %gep, <2 x half> %val syncscope("agent") seq_cst
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
index 2785b78da99e2..481a2540eacb7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
@@ -2243,36 +2243,22 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
 ;
 ; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat:
 ; GFX1250:       ; %bb.0: ; %main_body
+; GFX1250-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1250-NEXT:    s_mov_b32 s1, exec_lo
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0
-; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v0
-; GFX1250-NEXT:    s_cbranch_execz .LBB51_3
+; GFX1250-NEXT:    s_cbranch_execz .LBB51_2
 ; GFX1250-NEXT:  ; %bb.1:
-; GFX1250-NEXT:    s_bcnt1_i32_b32 s1, s1
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s1
-; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x24
+; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_mov_b32_e32 v4, s1
-; GFX1250-NEXT:    ds_load_b64 v[2:3], v4
-; GFX1250-NEXT:    v_mul_f64_e32 v[0:1], 4.0, v[0:1]
-; GFX1250-NEXT:  .LBB51_2: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[6:7], v[2:3], v[0:1]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[6:7], v4, v[6:7], v[2:3]
+; GFX1250-NEXT:    v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[2:3]
-; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], v[6:7]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB51_2
-; GFX1250-NEXT:  .LBB51_3:
+; GFX1250-NEXT:  .LBB51_2:
 ; GFX1250-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -2322,36 +2308,22 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
 ;
 ; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush:
 ; GFX1250:       ; %bb.0: ; %main_body
+; GFX1250-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1250-NEXT:    s_mov_b32 s1, exec_lo
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0
-; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v0
-; GFX1250-NEXT:    s_cbranch_execz .LBB52_3
+; GFX1250-NEXT:    s_cbranch_execz .LBB52_2
 ; GFX1250-NEXT:  ; %bb.1:
-; GFX1250-NEXT:    s_bcnt1_i32_b32 s1, s1
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s1
-; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x24
+; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_mov_b32_e32 v4, s1
-; GFX1250-NEXT:    ds_load_b64 v[2:3], v4
-; GFX1250-NEXT:    v_mul_f64_e32 v[0:1], 4.0, v[0:1]
-; GFX1250-NEXT:  .LBB52_2: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[6:7], v[2:3], v[0:1]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[6:7], v4, v[6:7], v[2:3]
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[2:3]
-; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], v[6:7]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB52_2
-; GFX1250-NEXT:  .LBB52_3:
+; GFX1250-NEXT:  .LBB52_2:
 ; GFX1250-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -2401,36 +2373,22 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
 ;
 ; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
 ; GFX1250:       ; %bb.0: ; %main_body
+; GFX1250-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX1250-NEXT:    s_mov_b32 s1, exec_lo
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s1, 0
-; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:    v_mbcnt_lo_u32_b32 v0, s0, 0
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-NEXT:    v_cmpx_eq_u32_e32 0, v0
-; GFX1250-NEXT:    s_cbranch_execz .LBB53_3
+; GFX1250-NEXT:    s_cbranch_execz .LBB53_2
 ; GFX1250-NEXT:  ; %bb.1:
-; GFX1250-NEXT:    s_bcnt1_i32_b32 s1, s1
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s1
-; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x24
+; GFX1250-NEXT:    s_bcnt1_i32_b32 s0, s0
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x24
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_mov_b32_e32 v4, s1
-; GFX1250-NEXT:    ds_load_b64 v[2:3], v4
-; GFX1250-NEXT:    v_mul_f64_e32 v[0:1], 4.0, v[0:1]
-; GFX1250-NEXT:  .LBB53_2: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[6:7], v[2:3], v[0:1]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[6:7], v4, v[6:7], v[2:3]
+; GFX1250-NEXT:    v_dual_mul_f64 v[0:1], 4.0, v[0:1] :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[2:3]
-; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], v[6:7]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB53_2
-; GFX1250-NEXT:  .LBB53_3:
+; GFX1250-NEXT:  .LBB53_2:
 ; GFX1250-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -2459,23 +2417,9 @@ define double @local_atomic_fadd_f64_rtn_pat(ptr addrspace(3) %ptr, double %data
 ; GFX1250:       ; %bb.0: ; %main_body
 ; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:  .LBB54_1: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_mov_b64_e32 v[4:5], v[0:1]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[0:1], 4.0, v[4:5]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[4:5]
+; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX1250-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB54_1
-; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-flat.mir
index 85d852fc779b2..be9de72a4ea9f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgpu-atomic-cmpxchg-flat.mir
@@ -153,7 +153,7 @@ body:             |
     %2:vgpr(s32) = COPY $vgpr3
     %3:vgpr(<2 x s32>) = G_BUILD_VECTOR %1, %2
     %4:vgpr(s64) = G_CONSTANT i64 4
-    %5:vgpr(p0) = G_PTR_ADD %0, %4
+    %5:vgpr(p0) = inbounds G_PTR_ADD %0, %4
     %6:vgpr(s32) = G_AMDGPU_ATOMIC_CMPXCHG %5, %3 :: (load store seq_cst (s32), addrspace 0)
     $vgpr0 = COPY %6
 
@@ -305,7 +305,7 @@ body:             |
     %2:vgpr(s64) = COPY $vgpr4_vgpr5
     %3:vgpr(<2 x s64>) = G_BUILD_VECTOR %1, %2
     %4:vgpr(s64) = G_CONSTANT i64 4
-    %5:vgpr(p0) = G_PTR_ADD %0, %4
+    %5:vgpr(p0) = inbounds G_PTR_ADD %0, %4
     %6:vgpr(s64) = G_AMDGPU_ATOMIC_CMPXCHG %5, %3 :: (load store seq_cst (s64), addrspace 0)
     $vgpr0_vgpr1 = COPY %6
 
@@ -406,7 +406,7 @@ body:             |
     %2:vgpr(s32) = COPY $vgpr3
     %3:vgpr(<2 x s32>) = G_BUILD_VECTOR %1, %2
     %4:vgpr(s64) = G_CONSTANT i64 -4
-    %5:vgpr(p0) = G_PTR_ADD %0, %4
+    %5:vgpr(p0) = inbounds G_PTR_ADD %0, %4
     %6:vgpr(s32) = G_AMDGPU_ATOMIC_CMPXCHG %5, %3 :: (load store seq_cst (s32), addrspace 0)
     $vgpr0 = COPY %6
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-flat.mir
index dc317a8413cd5..3389ed72fe7d9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-add-flat.mir
@@ -101,7 +101,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 2047
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s32) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s32), addrspace 0)
     $vgpr0 = COPY %4
 
@@ -155,7 +155,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 2047
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s32) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s32), addrspace 0)
 
 ...
@@ -211,7 +211,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 2048
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s32) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s32), addrspace 0)
     $vgpr0 = COPY %4
 
@@ -265,7 +265,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 2048
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s32) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s32), addrspace 0)
 
 ...
@@ -321,7 +321,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 4095
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s32) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s32), addrspace 0)
     $vgpr0 = COPY %4
 
@@ -375,7 +375,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 4095
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s32) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s32), addrspace 0)
 
 ...
@@ -463,7 +463,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 4097
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s32) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s32), addrspace 0)
     $vgpr0 = COPY %4
 
@@ -547,7 +547,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 4097
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s32) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s32), addrspace 0)
 
 ...
@@ -647,7 +647,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = COPY $vgpr2_vgpr3
     %2:vgpr(s64) = G_CONSTANT i64 4095
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s64) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s64), addrspace 0)
     $vgpr0_vgpr1 = COPY %4
 
@@ -701,7 +701,7 @@ body:             |
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = COPY $vgpr2_vgpr3
     %2:vgpr(s64) = G_CONSTANT i64 4095
-    %3:vgpr(p0) = G_PTR_ADD %0, %2
+    %3:vgpr(p0) = inbounds G_PTR_ADD %0, %2
     %4:vgpr(s64) = G_ATOMICRMW_ADD %3, %1 :: (load store seq_cst (s64), addrspace 0)
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
index eba64b853ac05..5bfb2b2e4d578 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-atomic-flat.mir
@@ -492,7 +492,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 -2048
-    %2:vgpr(p0) = G_PTR_ADD %0, %1
+    %2:vgpr(p0) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load seq_cst (s32), align 4, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -561,7 +561,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_DWORD]]
     %0:vgpr(p0) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 4095
-    %2:vgpr(p0) = G_PTR_ADD %0, %1
+    %2:vgpr(p0) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load seq_cst (s32), align 4, addrspace 0)
     $vgpr0 = COPY %3
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir
index e1325a0a0bc50..532b4bfee3320 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-flat.mir
@@ -1191,7 +1191,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 2047
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -1275,7 +1275,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 2048
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -1375,7 +1375,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 -2047
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -1475,7 +1475,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 -2048
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -1559,7 +1559,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 4095
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -1659,7 +1659,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 4096
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -1759,7 +1759,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 -4095
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -1859,7 +1859,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 -4096
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -1959,7 +1959,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 8191
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -2059,7 +2059,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 8192
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -2159,7 +2159,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 -8191
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -2259,7 +2259,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 -8192
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -2359,7 +2359,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 8388607
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
@@ -2567,7 +2567,7 @@ body: |
     ; GFX12-NEXT: $vgpr0 = COPY [[FLAT_LOAD_UBYTE]]
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s64) = G_CONSTANT i64 -8388608
-    %2:vgpr(p1) = G_PTR_ADD %0, %1
+    %2:vgpr(p1) = inbounds G_PTR_ADD %0, %1
     %3:vgpr(s32) = G_LOAD %2 :: (load (s8), align 1, addrspace 0)
     $vgpr0 = COPY %3
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir
index 6e92d851dee2e..5b65c0e3734e7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-flat.mir
@@ -1237,7 +1237,7 @@ body: |
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 2047
-    %3:vgpr(p1) = G_PTR_ADD %0, %2
+    %3:vgpr(p1) = inbounds G_PTR_ADD %0, %2
     G_STORE %1, %3 :: (store (s32), align 4, addrspace 0)
 
 ...
@@ -1337,7 +1337,7 @@ body: |
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 8388607
-    %3:vgpr(p1) = G_PTR_ADD %0, %2
+    %3:vgpr(p1) = inbounds G_PTR_ADD %0, %2
     G_STORE %1, %3 :: (store (s32), align 4, addrspace 0)
 
 ...
@@ -1545,7 +1545,7 @@ body: |
     %0:vgpr(p1) = COPY $vgpr0_vgpr1
     %1:vgpr(s32) = COPY $vgpr2
     %2:vgpr(s64) = G_CONSTANT i64 -8388608
-    %3:vgpr(p1) = G_PTR_ADD %0, %2
+    %3:vgpr(p1) = inbounds G_PTR_ADD %0, %2
     G_STORE %1, %3 :: (store (s32), align 4, addrspace 0)
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
index 2b84c6bcba7b5..acbcb098e8367 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
@@ -886,33 +886,34 @@ body: |
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
     ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C]](s32)
+    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY2]], [[C]](s32)
     ; SI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
     ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
+    ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C2]](s32)
     ; SI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](s64)
     ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
     ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
-    ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C5]]
+    ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C5]]
     ; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[C4]](s32)
     ; SI-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C6]](s64)
-    ; SI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; SI-NEXT: G_STORE [[COPY3]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; SI-NEXT: G_STORE [[LSHR2]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
-    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
-    ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR1]], [[COPY3]](s32)
+    ; SI-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR1]], [[COPY4]](s32)
     ; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C6]](s64)
     ; SI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
     ; SI-NEXT: G_STORE [[LSHR3]](s32), [[PTR_ADD3]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
     ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
-    ; SI-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
+    ; SI-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C4]](s32)
     ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C5]]
-    ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[COPY4]](s32)
+    ; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[COPY5]](s32)
     ; SI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD]], [[C6]](s64)
     ; SI-NEXT: G_STORE [[TRUNC1]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 4, addrspace 1)
     ; SI-NEXT: G_STORE [[LSHR4]](s32), [[PTR_ADD4]](p1) :: (store (s8) into unknown-address + 5, addrspace 1)
@@ -922,11 +923,12 @@ body: |
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; CI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CI-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
     ; CI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C]](s32)
+    ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY2]], [[C]](s32)
     ; CI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; CI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
     ; CI-NEXT: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store (s32), align 1, addrspace 1)
     ; CI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
     ; CI-NEXT: G_STORE [[TRUNC1]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 4, align 1, addrspace 1)
@@ -936,22 +938,23 @@ body: |
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C]](s32)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY2]], [[C]](s32)
     ; VI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+    ; VI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
     ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
+    ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C2]](s32)
     ; VI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s64)
+    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s64)
     ; VI-NEXT: [[C4:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
     ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C4]](s16)
     ; VI-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](s64)
-    ; VI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; VI-NEXT: G_STORE [[COPY3]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR2]](s16)
     ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
     ; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
@@ -960,11 +963,11 @@ body: |
     ; VI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
     ; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR3]](s16)
     ; VI-NEXT: G_STORE [[ANYEXT1]](s32), [[PTR_ADD3]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
-    ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
-    ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s64)
-    ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC4]], [[C4]](s16)
+    ; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s64)
+    ; VI-NEXT: [[TRUNC4:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
+    ; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C4]](s16)
     ; VI-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD]], [[C5]](s64)
-    ; VI-NEXT: G_STORE [[TRUNC3]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 4, addrspace 1)
+    ; VI-NEXT: G_STORE [[TRUNC4]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 4, addrspace 1)
     ; VI-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR4]](s16)
     ; VI-NEXT: G_STORE [[ANYEXT2]](s32), [[PTR_ADD4]](p1) :: (store (s8) into unknown-address + 5, addrspace 1)
     ;
@@ -973,11 +976,12 @@ body: |
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
     ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY2]], [[C]](s32)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
     ; GFX9-NEXT: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store (s32), align 1, addrspace 1)
     ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
     ; GFX9-NEXT: G_STORE [[TRUNC1]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 4, align 1, addrspace 1)
@@ -998,17 +1002,18 @@ body: |
     ; SI-NEXT: {{  $}}
     ; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; SI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
     ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C]](s32)
+    ; SI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY2]], [[C]](s32)
     ; SI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; SI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; SI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+    ; SI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
     ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
+    ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C2]](s32)
     ; SI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; SI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+    ; SI-NEXT: G_STORE [[COPY3]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
     ; SI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
     ; SI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
     ; SI-NEXT: G_STORE [[TRUNC1]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
@@ -1018,11 +1023,12 @@ body: |
     ; CI-NEXT: {{  $}}
     ; CI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; CI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CI-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
     ; CI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C]](s32)
+    ; CI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY2]], [[C]](s32)
     ; CI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; CI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
     ; CI-NEXT: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store (s32), align 2, addrspace 1)
     ; CI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
     ; CI-NEXT: G_STORE [[TRUNC1]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
@@ -1032,17 +1038,18 @@ body: |
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C]](s32)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY2]], [[C]](s32)
     ; VI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; VI-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
+    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
+    ; VI-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[TRUNC]](s32)
     ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
-    ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C2]](s32)
+    ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C2]](s32)
     ; VI-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
     ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](s64)
-    ; VI-NEXT: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
+    ; VI-NEXT: G_STORE [[COPY3]](s32), [[COPY]](p1) :: (store (s16), addrspace 1)
     ; VI-NEXT: G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
     ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
     ; VI-NEXT: G_STORE [[TRUNC1]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
@@ -1052,11 +1059,12 @@ body: |
     ; GFX9-NEXT: {{  $}}
     ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[COPY1]](s64)
     ; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
-    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY1]], [[C]](s32)
+    ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY2]], [[C]](s32)
     ; GFX9-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
     ; GFX9-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
+    ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY2]](s64)
     ; GFX9-NEXT: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store (s32), align 2, addrspace 1)
     ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
     ; GFX9-NEXT: G_STORE [[TRUNC1]](s32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 4, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
index a931c6366c403..7fd23197a5dd6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
@@ -285,13 +285,13 @@ body: |
     ; VI-NEXT: {{  $}}
     ; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; VI-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s64)
+    ; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s64)
+    ; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; VI-NEXT: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
-    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[C]](s16)
+    ; VI-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[C]](s16)
     ; VI-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
     ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](s64)
-    ; VI-NEXT: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
+    ; VI-NEXT: G_STORE [[TRUNC1]](s32), [[COPY]](p1) :: (store (s8), addrspace 1)
     ; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
     ; VI-NEXT: G_STORE [[ANYEXT]](s32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
index 85c1d3a8c3ee4..390f62d8193f7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.end.cf.i32.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn--amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
 
 define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
 ; GFX10-LABEL: test_wave32:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
index ce8cba266aeec..67a388e29087e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.if.break.i32.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn--amdhsa -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn--amdhsa -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
 
 define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
 ; GFX10-LABEL: test_wave32:
@@ -10,9 +10,8 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
 ; GFX10-NEXT:    s_load_dword s1, s[8:9], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_cmp_eq_u32 s0, 0
-; GFX10-NEXT:    s_cselect_b32 s0, 1, 0
-; GFX10-NEXT:    s_and_b32 s0, 1, s0
-; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s0
+; GFX10-NEXT:    s_cselect_b32 s0, exec_lo, 0
+; GFX10-NEXT:    s_and_b32 s0, exec_lo, s0
 ; GFX10-NEXT:    s_or_b32 s0, s0, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-NEXT:    global_store_dword v[0:1], v0, off
@@ -26,9 +25,8 @@ define amdgpu_kernel void @test_wave32(i32 %arg0, [8 x i32], i32 %saved) {
 ; GFX11-NEXT:    s_load_b32 s1, s[4:5], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_cmp_eq_u32 s0, 0
-; GFX11-NEXT:    s_cselect_b32 s0, 1, 0
-; GFX11-NEXT:    s_and_b32 s0, 1, s0
-; GFX11-NEXT:    v_cmp_ne_u32_e64 s0, 0, s0
+; GFX11-NEXT:    s_cselect_b32 s0, exec_lo, 0
+; GFX11-NEXT:    s_and_b32 s0, exec_lo, s0
 ; GFX11-NEXT:    s_or_b32 s0, s0, s1
 ; GFX11-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX11-NEXT:    global_store_b32 v[0:1], v0, off dlc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.implicit.ptr.buffer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.implicit.ptr.buffer.ll
index 8d9f9d107b925..4687b83c744cc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.implicit.ptr.buffer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.implicit.ptr.buffer.ll
@@ -1,4 +1,4 @@
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-mesa-mesa3d < %s | FileCheck -check-prefix=GCN %s
 
 ; FIXME: Dropped parts from original test
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.add.ll
index 62f8f8959eba9..79a92918bfe85 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.add.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck --check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck --check-prefixes=GFX12,GFX1200 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -stop-after=instruction-select -o - %s | FileCheck --check-prefixes=GFX12,GFX1250 %s
 
 ; Natural mapping
 define amdgpu_ps float @raw_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(i32 %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
@@ -99,26 +100,47 @@ define amdgpu_ps <2 x float> @raw_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc__vg
   ; GFX8-NEXT:   $vgpr1 = COPY [[COPY9]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN]].sub1
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY8]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY9]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ; GFX1200-LABEL: name: raw_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN]].sub1
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN_RTN]].sub1
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   %ret = call i64 @llvm.amdgcn.raw.buffer.atomic.add.i64(i64 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i64 %ret to <2 x float>
   ret <2 x float> %cast
@@ -142,22 +164,39 @@ define amdgpu_ps void @raw_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__vgp
   ; GFX8-NEXT:   BUFFER_ATOMIC_ADD_X2_OFFEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_ADD_X2_VBUFFER_OFFEN [[REG_SEQUENCE]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i64 @llvm.amdgcn.raw.buffer.atomic.add.i64(i64 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -217,58 +256,111 @@ define amdgpu_ps float @raw_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_vof
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_OFFEN_RTN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
   ret float %cast
@@ -328,57 +420,109 @@ define amdgpu_ps void @raw_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__sgp
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   BUFFER_ATOMIC_ADD_VBUFFER_OFFEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_ADD_VBUFFER_OFFEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_ADD_VBUFFER_OFFEN [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -400,21 +544,40 @@ define amdgpu_ps float @raw_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_vof
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_OFFEN_RTN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN [[COPY]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_OFFEN_RTN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.cmpswap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.cmpswap.ll
index 364ed62d9838c..9f1b7a670a40e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.cmpswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.atomic.cmpswap.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck  --check-prefixes=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck  --check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck  --check-prefix=GFX1200 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -stop-after=instruction-select -o - %s | FileCheck  --check-prefix=GFX1250 %s
 
 
 ; Natural mapping
@@ -24,24 +25,43 @@ define amdgpu_ps float @raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_
   ; GFX8-NEXT:   $vgpr0 = COPY [[COPY8]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN]].sub0
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY8]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN]].sub0
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN]].sub0
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
   ret float %cast
@@ -66,22 +86,39 @@ define amdgpu_ps void @raw_buffer_atomic_cmpswap_i32_noret__vgpr_val__vgpr_cmp__
   ; GFX8-NEXT:   BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i32_noret__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i32_noret__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i32_noret__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -145,62 +182,119 @@ define amdgpu_ps float @raw_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vgpr_
   ; GFX8-NEXT:   $vgpr0 = COPY [[COPY15]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN]].sub0
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY15]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN]].sub0
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY15]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN]].sub0
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY15]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
   ret float %cast
@@ -263,60 +357,115 @@ define amdgpu_ps void @raw_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cmp__
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN [[REG_SEQUENCE2]], [[COPY10]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -341,24 +490,46 @@ define amdgpu_ps float @raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_
   ; GFX8-NEXT:   $vgpr0 = COPY [[COPY8]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN]].sub0
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY8]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN]].sub0
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY6]], [[COPY8]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN [[REG_SEQUENCE1]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_OFFEN_RTN]].sub0
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
   %ret = call i32 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
@@ -395,33 +566,61 @@ define amdgpu_ps double @raw_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr
   ; GFX8-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE3]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub0
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub1
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
-  ; GFX12-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
-  ; GFX12-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE3]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub0
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub1
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_128_align2 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE3]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vreg_64_align2 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub0
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub1
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   %ret = call i64 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i64 %ret to double
   ret double %cast
@@ -450,26 +649,47 @@ define amdgpu_ps void @raw_buffer_atomic_cmpswap_i64_noret__vgpr_val__vgpr_cmp__
   ; GFX8-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_OFFEN [[REG_SEQUENCE3]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i64_noret__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN [[REG_SEQUENCE3]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i64_noret__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN [[REG_SEQUENCE3]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i64_noret__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN [[REG_SEQUENCE3]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i64 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -542,71 +762,137 @@ define amdgpu_ps double @raw_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cmp__vgpr
   ; GFX8-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_6]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY9]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY10]], %subreg.sub0_sub1, [[COPY11]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE4]], [[COPY12]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY17:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN]].sub0_sub1
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY17]].sub0
-  ; GFX12-NEXT:   [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY17]].sub1
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY18]], implicit $exec
-  ; GFX12-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_5]]
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY19]], implicit $exec
-  ; GFX12-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_6]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY9]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY10]], %subreg.sub0_sub1, [[COPY11]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE4]], [[COPY12]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY17:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN]].sub0_sub1
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY17]].sub0
+  ; GFX1200-NEXT:   [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY17]].sub1
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY18]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_5]]
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY19]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_6]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY9]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY10]], %subreg.sub0_sub1, [[COPY11]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_128_align2 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE4]], [[COPY12]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY17:%[0-9]+]]:vreg_64_align2 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN]].sub0_sub1
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[COPY17]].sub0
+  ; GFX1250-NEXT:   [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[COPY17]].sub1
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY18]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_5]]
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY19]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_6]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   %ret = call i64 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i64 %ret to double
   ret double %cast
@@ -673,64 +959,123 @@ define amdgpu_ps void @raw_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgpr_cmp__
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY9]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY10]], %subreg.sub0_sub1, [[COPY11]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN [[REG_SEQUENCE4]], [[COPY12]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY9]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY10]], %subreg.sub0_sub1, [[COPY11]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN [[REG_SEQUENCE4]], [[COPY12]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY9]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY10]], %subreg.sub0_sub1, [[COPY11]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN [[REG_SEQUENCE4]], [[COPY12]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i64 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -764,33 +1109,64 @@ define amdgpu_ps double @raw_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr
   ; GFX8-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   ;
-  ; GFX12-LABEL: name: raw_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE3]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub0
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub1
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
-  ; GFX12-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
-  ; GFX12-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ; GFX1200-LABEL: name: raw_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE3]], [[COPY8]], [[REG_SEQUENCE2]], [[COPY9]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub0
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY10]].sub1
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY8]], [[COPY10]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN:%[0-9]+]]:vreg_128_align2 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN [[REG_SEQUENCE3]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE2]], [[COPY9]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_OFFEN_RTN]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub0
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub1
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   %voffset = add i32 %voffset.base, 4095
   %ret = call i64 @llvm.amdgcn.raw.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i64 %ret to double
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
index 46ca43b7af023..7003bb1a09eae 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12,GFX1200 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12,GFX1250 %s
 ; FIXME: Test with SI when argument lowering not broken for f16
 
 ; Natural mapping
@@ -124,52 +125,99 @@ define amdgpu_ps float @raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
 }
@@ -226,55 +274,105 @@ define amdgpu_ps float @raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffs
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f32__vgpr_rsrc__vgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
 }
@@ -509,23 +607,41 @@ define amdgpu_ps <2 x float> @raw_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sg
   ; GFX8-NEXT:   $vgpr1 = COPY [[COPY7]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY6]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY7]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ; GFX1200-LABEL: name: raw_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v2f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   %val = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret <2 x float> %val
 }
@@ -551,25 +667,45 @@ define amdgpu_ps <3 x float> @raw_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sg
   ; GFX8-NEXT:   $vgpr2 = COPY [[COPY8]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub0
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub1
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub2
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY6]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY7]]
-  ; GFX12-NEXT:   $vgpr2 = COPY [[COPY8]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  ; GFX1200-LABEL: name: raw_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub0
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub1
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub2
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v3f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub0
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub1
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_OFFEN]].sub2
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
   %val = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret <3 x float> %val
 }
@@ -597,27 +733,49 @@ define amdgpu_ps <4 x float> @raw_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sg
   ; GFX8-NEXT:   $vgpr3 = COPY [[COPY9]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub0
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub1
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub2
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub3
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY6]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY7]]
-  ; GFX12-NEXT:   $vgpr2 = COPY [[COPY8]]
-  ; GFX12-NEXT:   $vgpr3 = COPY [[COPY9]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ; GFX1200-LABEL: name: raw_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub0
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub1
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub2
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub3
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   $vgpr3 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v4f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub0
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub1
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub2
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_OFFEN]].sub3
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   $vgpr2 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   $vgpr3 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
   %val = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret <4 x float> %val
 }
@@ -715,23 +873,41 @@ define amdgpu_ps <4 x half> @raw_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgp
   ; GFX8-NEXT:   $vgpr1 = COPY [[COPY7]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY6]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY7]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ; GFX1200-LABEL: name: raw_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub0
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_OFFEN]].sub1
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   %val = call <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret <4 x half> %val
 }
@@ -929,52 +1105,99 @@ define amdgpu_ps half @raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffse
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret half %val
 }
@@ -1028,52 +1251,99 @@ define amdgpu_ps float @raw_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffse
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_OFFEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   %zext = zext i8 %val to i32
   %cast = bitcast i32 %zext to float
@@ -1194,20 +1464,38 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 16, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 16, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add16
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY6]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 16
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
@@ -1229,20 +1517,38 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4095
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY6]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
@@ -1267,20 +1573,38 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4096, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4096, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset__voffset_add4096
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY6]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4096
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
@@ -1522,54 +1846,103 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
-  ; GFX12-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_1]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+  ; GFX1200-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_1]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_soffset_add5000
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+  ; GFX1250-NEXT:   [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[COPY5]], [[S_MOV_B32_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[S_ADD_I32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_1]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %soffset = add i32 %soffset.base, 5000
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
@@ -1627,52 +2000,102 @@ define amdgpu_ps float @raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffs
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 5000, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 5000, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f32__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add5000
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY4]], [[COPY6]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_OFFEN [[V_ADD_U32_e64_]], [[REG_SEQUENCE1]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_1]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_OFFEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 5000
   %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
index 3fbfb630ce08b..4784ac5de17b0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.tfe.ll
@@ -5,7 +5,8 @@
 ; RUN: llc -global-isel -mcpu=gfx900 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
 ; RUN: llc -global-isel -mcpu=gfx1010 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
 ; RUN: llc -global-isel -mcpu=gfx1100 -mattr=-real-true16 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
-; RUN: llc -global-isel -mcpu=gfx1200 -mattr=-real-true16 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX12
+; RUN: llc -global-isel -mcpu=gfx1200 -mattr=-real-true16 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
+; RUN: llc -global-isel -mcpu=gfx1250 -mattr=-real-true16 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
 
 define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
   ; GFX67-LABEL: name: raw_buffer_load_i8_tfe
@@ -110,27 +111,49 @@ define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspa
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_i8_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s8) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_i8_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s8) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_i8_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s8) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { i8, i32 } @llvm.amdgcn.raw.buffer.load.sl_i8i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { i8, i32 } %res, 0
   store i8 %data, ptr addrspace(1) %data_addr
@@ -242,27 +265,49 @@ define amdgpu_ps void @raw_buffer_load_i16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_i16_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_i16_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_i16_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { i16, i32 } @llvm.amdgcn.raw.buffer.load.sl_i16i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { i16, i32 } %res, 0
   store i16 %data, ptr addrspace(1) %data_addr
@@ -374,27 +419,49 @@ define amdgpu_ps void @raw_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f16_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_f16_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f16_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { half, i32 } @llvm.amdgcn.raw.buffer.load.sl_f16i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { half, i32 } %res, 0
   store half %data, ptr addrspace(1) %data_addr
@@ -506,27 +573,49 @@ define amdgpu_ps void @raw_buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrsp
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_i32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s32) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_i32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s32) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_i32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (store (s32) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { i32, i32 } @llvm.amdgcn.raw.buffer.load.sl_i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { i32, i32 } %res, 0
   store i32 %data, ptr addrspace(1) %data_addr
@@ -646,29 +735,53 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v2i32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v2i32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v2i32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <2 x i32>, i32 } @llvm.amdgcn.raw.buffer.load.sl_v2i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { <2 x i32>, i32 } %res, 0
   store <2 x i32> %data, ptr addrspace(1) %data_addr
@@ -788,29 +901,53 @@ define amdgpu_ps void @raw_buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v2f32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v2f32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v2f32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <2 x float>, i32 } @llvm.amdgcn.raw.buffer.load.sl_v2f32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { <2 x float>, i32 } %res, 0
   store <2 x float> %data, ptr addrspace(1) %data_addr
@@ -977,30 +1114,55 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v3i32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v3i32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v3i32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <3 x i32>, i32 } @llvm.amdgcn.raw.buffer.load.sl_v3i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { <3 x i32>, i32 } %res, 0
   store <3 x i32> %data, ptr addrspace(1) %data_addr
@@ -1167,30 +1329,55 @@ define amdgpu_ps void @raw_buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v3f32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v3f32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v3f32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_OFFSET]].sub3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <3 x float>, i32 } @llvm.amdgcn.raw.buffer.load.sl_v3f32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { <3 x float>, i32 } %res, 0
   store <3 x float> %data, ptr addrspace(1) %data_addr
@@ -1318,31 +1505,57 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v4i32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v4i32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v4i32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160_align2 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <4 x i32>, i32 } @llvm.amdgcn.raw.buffer.load.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { <4 x i32>, i32 } %res, 0
   store <4 x i32> %data, ptr addrspace(1) %data_addr
@@ -1470,31 +1683,57 @@ define amdgpu_ps void @raw_buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v4f32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v4f32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v4f32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET:%[0-9]+]]:vreg_160_align2 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub0
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub2
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub3
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_OFFSET]].sub4
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY11]], %subreg.sub3
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <4 x float>, i32 } @llvm.amdgcn.raw.buffer.load.sl_v4f32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0)
   %data = extractvalue { <4 x float>, i32 } %res, 0
   store <4 x float> %data, ptr addrspace(1) %data_addr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
index 63ca7be08d060..c365d5711f6ce 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12,GFX1200 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12,GFX1250 %s
 ; FIXME: Test with SI when argument lowering not broken for f16
 
 ; Natural mapping
@@ -126,52 +127,99 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -309,55 +357,105 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -618,22 +716,39 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -657,23 +772,41 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORDX3_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX3_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX3_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX3_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -698,24 +831,43 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORDX4_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX4_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -876,22 +1028,39 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -946,54 +1115,103 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE2]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -1080,20 +1298,38 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_16
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_16
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_16
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 16
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
@@ -1115,20 +1351,38 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4095
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4095
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4095
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4095
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
@@ -1153,20 +1407,38 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4096
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4096, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4096
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4096, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4096
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4096
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
@@ -1256,20 +1528,38 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_16
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_16
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 16, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_16
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 16
   call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
@@ -1291,20 +1581,38 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4095
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4095
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4095
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4095
   call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
@@ -1329,20 +1637,38 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_ADD_CO_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4096
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4096, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4096
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 4096, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4096
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 4096
   call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
@@ -1400,52 +1726,102 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_offset_add_5000
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 5000, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_offset_add_5000
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 5000, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_offset_add_5000
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 5000
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact [[COPY4]], [[V_ADD_U32_e64_]], [[REG_SEQUENCE1]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_1]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %voffset.add = add i32 %voffset, 5000
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
@@ -1501,51 +1877,97 @@ define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr_soffset_offset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 5000, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr_soffset_offset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 5000, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr_soffset_offset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY8]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact [[COPY4]], [[REG_SEQUENCE1]], [[COPY5]], 5000, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 5000, i32 %soffset, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.add.ll
index 75d6c59560884..484639a23d284 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.add.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX1250 %s
 
 ; Natural mapping
 define amdgpu_ps float @struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(i32 %val, <4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
@@ -22,23 +23,41 @@ define amdgpu_ps float @struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_BOTHEN_RTN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
   ret float %cast
@@ -63,23 +82,41 @@ define amdgpu_ps float @struct_buffer_atomic_add_i32_noret__vgpr_val__sgpr_rsrc_
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_BOTHEN_RTN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_add_i32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_add_i32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_add_i32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
   ret float %cast
@@ -109,28 +146,51 @@ define amdgpu_ps <2 x float> @struct_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc_
   ; GFX8-NEXT:   $vgpr1 = COPY [[COPY10]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN]].sub1
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY9]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY10]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ; GFX1200-LABEL: name: struct_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN]].sub1
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY10]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_add_i64__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN_RTN]].sub1
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY10]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   %ret = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i64 %ret to <2 x float>
   ret <2 x float> %cast
@@ -156,24 +216,43 @@ define amdgpu_ps void @struct_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__
   ; GFX8-NEXT:   BUFFER_ATOMIC_ADD_X2_BOTHEN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_add_i64_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_ADD_X2_VBUFFER_BOTHEN [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i64 @llvm.amdgcn.struct.buffer.atomic.add.i64(i64 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -236,61 +315,117 @@ define amdgpu_ps float @struct_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_BOTHEN_RTN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_add_i32__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
   ret float %cast
@@ -353,60 +488,115 @@ define amdgpu_ps void @struct_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_add_i32_noret__sgpr_val__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY14]], [[COPY12]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN [[COPY8]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -431,23 +621,41 @@ define amdgpu_ps float @struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_BOTHEN_RTN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 3, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 3, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_add_i32__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vgpr_32 = BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 3, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_ATOMIC_ADD_VBUFFER_BOTHEN_RTN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.add.i32(i32 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   %cast = bitcast i32 %ret to float
   ret float %cast
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.cmpswap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.cmpswap.ll
index c9d1227cf27ab..7dab257aaaaf0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.cmpswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.atomic.cmpswap.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX1250 %s
 
 ; Natural mapping
 define amdgpu_ps float @struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset(i32 %val, i32 %cmp, <4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
@@ -25,26 +26,47 @@ define amdgpu_ps float @struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sg
   ; GFX8-NEXT:   $vgpr0 = COPY [[COPY9]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN]].sub0
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY9]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN]].sub0
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN]].sub0
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
   ret float %cast
@@ -71,24 +93,43 @@ define amdgpu_ps void @struct_buffer_atomic_cmpswap_noret_i32__vgpr_val__vgpr_cm
   ; GFX8-NEXT:   BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_noret_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_noret_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_noret_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -155,65 +196,125 @@ define amdgpu_ps float @struct_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vg
   ; GFX8-NEXT:   $vgpr0 = COPY [[COPY17]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY8]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY12]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN]].sub0
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY17]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY8]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY12]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN]].sub0
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY17]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_i32__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY8]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY12]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN]].sub0
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY17]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
   ret float %cast
@@ -279,63 +380,121 @@ define amdgpu_ps void @struct_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cm
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY8]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY12]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY8]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY12]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_i32_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY15:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY15]], [[COPY13]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY8]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY11]], %subreg.sub0, [[COPY12]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -362,26 +521,50 @@ define amdgpu_ps float @struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sg
   ; GFX8-NEXT:   $vgpr0 = COPY [[COPY9]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN]].sub0
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY9]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN]].sub0
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_i32__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY7]], [[COPY9]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[V_ADD_U32_e64_]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_64_align2 = BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_ATOMIC_CMPSWAP_VBUFFER_BOTHEN_RTN]].sub0
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY10]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
   %ret = call i32 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i32(i32 %val, i32 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i32 %ret to float
@@ -420,35 +603,65 @@ define amdgpu_ps double @struct_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__s
   ; GFX8-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub0
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub1
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
-  ; GFX12-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
-  ; GFX12-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub0
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub1
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_128_align2 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub0
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub1
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   %ret = call i64 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i64 %ret to double
   ret double %cast
@@ -479,28 +692,51 @@ define amdgpu_ps void @struct_buffer_atomic_cmpswap_noret_i64__vgpr_val__vgpr_cm
   ; GFX8-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_BOTHEN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_noret_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_noret_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_noret_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i64 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -576,74 +812,143 @@ define amdgpu_ps double @struct_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cmp__v
   ; GFX8-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_6]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr7
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY15:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY17:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY17]], [[COPY15]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY18]], [[COPY16]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY10]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY13]], %subreg.sub0, [[COPY14]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE5:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY11]], %subreg.sub0_sub1, [[COPY12]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE5]], [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY19:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN]].sub0_sub1
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[COPY19]].sub0
-  ; GFX12-NEXT:   [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[COPY19]].sub1
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY20]], implicit $exec
-  ; GFX12-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_5]]
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY21]], implicit $exec
-  ; GFX12-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_6]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY15:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY17:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY17]], [[COPY15]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY18]], [[COPY16]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY10]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY13]], %subreg.sub0, [[COPY14]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE5:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY11]], %subreg.sub0_sub1, [[COPY12]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE5]], [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY19:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN]].sub0_sub1
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[COPY19]].sub0
+  ; GFX1200-NEXT:   [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[COPY19]].sub1
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY20]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_5]]
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY21]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_6]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_i64__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY15:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY16:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY17:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY17]], [[COPY15]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY18]], [[COPY16]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY10]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY13]], %subreg.sub0, [[COPY14]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE5:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY11]], %subreg.sub0_sub1, [[COPY12]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_128_align2 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE5]], [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY19:%[0-9]+]]:vreg_64_align2 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN]].sub0_sub1
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[COPY19]].sub0
+  ; GFX1250-NEXT:   [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[COPY19]].sub1
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY20]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_5]]
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY21]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_6]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   %ret = call i64 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i64 %ret to double
   ret double %cast
@@ -713,67 +1018,129 @@ define amdgpu_ps void @struct_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgpr_cm
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr7
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY15:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY17:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY17]], [[COPY15]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY18]], [[COPY16]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY10]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY13]], %subreg.sub0, [[COPY14]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE5:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY11]], %subreg.sub0_sub1, [[COPY12]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN [[REG_SEQUENCE5]], [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]]
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY15:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY16:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY17:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY17]], [[COPY15]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY18]], [[COPY16]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY10]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY13]], %subreg.sub0, [[COPY14]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE5:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY11]], %subreg.sub0_sub1, [[COPY12]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN [[REG_SEQUENCE5]], [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_i64_noret__sgpr_val__sgpr_cmp__vgpr_rsrc__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]]
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]]
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY15:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY16:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY17:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY17]], [[COPY15]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY18]], [[COPY16]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY10]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY13]], %subreg.sub0, [[COPY14]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE5:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY11]], %subreg.sub0_sub1, [[COPY12]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN [[REG_SEQUENCE5]], [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %ret = call i64 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -809,35 +1176,68 @@ define amdgpu_ps double @struct_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__s
   ; GFX8-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   ;
-  ; GFX12-LABEL: name: struct_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; GFX12-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
-  ; GFX12-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub0
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub1
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
-  ; GFX12-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
-  ; GFX12-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ; GFX1200-LABEL: name: struct_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1200-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_128 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 4095, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub0
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY11]].sub1
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; GFX1200-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_atomic_cmpswap_i64__vgpr_val__vgpr_cmp__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add4095
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY9]], [[COPY11]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[V_ADD_U32_e64_]], %subreg.sub1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[REG_SEQUENCE]], %subreg.sub0_sub1, [[REG_SEQUENCE1]], %subreg.sub2_sub3
+  ; GFX1250-NEXT:   [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN:%[0-9]+]]:vreg_128_align2 = BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN [[REG_SEQUENCE4]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[COPY10]], 0, 1, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vreg_64_align2 = COPY [[BUFFER_ATOMIC_CMPSWAP_X2_VBUFFER_BOTHEN_RTN]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY12]].sub0
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[COPY12]].sub1
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr0 = COPY [[V_READFIRSTLANE_B32_]]
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY14]], implicit $exec
+  ; GFX1250-NEXT:   $sgpr1 = COPY [[V_READFIRSTLANE_B32_1]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0, implicit $sgpr1
   %voffset = add i32 %voffset.base, 4095
   %ret = call i64 @llvm.amdgcn.struct.buffer.atomic.cmpswap.i64(i64 %val, i64 %cmp, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %cast = bitcast i64 %ret to double
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
index 9b5e46b382d7e..dbef90f6c9ff9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX1250 %s
 
 ; Natural mapping
 define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
@@ -21,22 +22,39 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
 }
@@ -63,25 +81,45 @@ define amdgpu_ps <2 x float> @struct_buffer_load_v2f32__sgpr_rsrc__vgpr_vindex__
   ; GFX8-NEXT:   $vgpr1 = COPY [[COPY8]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_v2f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub1
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY7]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY8]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ; GFX1200-LABEL: name: struct_buffer_load_v2f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub1
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_v2f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub1
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   %val = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret <2 x float> %val
 }
@@ -110,27 +148,49 @@ define amdgpu_ps <3 x float> @struct_buffer_load_v3f32__sgpr_rsrc__vgpr_vindex__
   ; GFX8-NEXT:   $vgpr2 = COPY [[COPY9]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_v3f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub2
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY7]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY8]]
-  ; GFX12-NEXT:   $vgpr2 = COPY [[COPY9]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  ; GFX1200-LABEL: name: struct_buffer_load_v3f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub2
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   $vgpr2 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_v3f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_VBUFFER_BOTHEN]].sub2
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   $vgpr2 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
   %val = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret <3 x float> %val
 }
@@ -161,29 +221,53 @@ define amdgpu_ps <4 x float> @struct_buffer_load_v4f32__sgpr_rsrc__vgpr_vindex__
   ; GFX8-NEXT:   $vgpr3 = COPY [[COPY10]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_v4f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub1
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub2
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub3
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY7]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY8]]
-  ; GFX12-NEXT:   $vgpr2 = COPY [[COPY9]]
-  ; GFX12-NEXT:   $vgpr3 = COPY [[COPY10]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ; GFX1200-LABEL: name: struct_buffer_load_v4f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub1
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub2
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub3
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   $vgpr2 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   $vgpr3 = COPY [[COPY10]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_v4f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub1
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub2
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_VBUFFER_BOTHEN]].sub3
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   $vgpr2 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   $vgpr3 = COPY [[COPY10]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3
   %val = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret <4 x float> %val
 }
@@ -208,23 +292,41 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 0, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
 }
@@ -248,22 +350,42 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_ADD_U32_e64_]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %voffset = add i32 %voffset.base, 4095
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
@@ -287,22 +409,39 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_soffset_64
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 64
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_soffset_64
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 64
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_soffset_64
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 64
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 64, i32 0)
   ret float %val
 }
@@ -363,59 +502,113 @@ define amdgpu_ps float @struct_buffer_load_f32__vgpr_rsrc__sgpr_vindex__sgpr_vof
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_f32__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_f32__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_f32__vgpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret float %val
 }
@@ -438,22 +631,39 @@ define amdgpu_ps float @struct_buffer_load_i8_zext__sgpr_rsrc__vgpr_vindex__vgpr
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_i8_zext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_i8_zext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_i8_zext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %ext = zext i8 %val to i32
   %cast = bitcast i32 %ext to float
@@ -478,22 +688,39 @@ define amdgpu_ps float @struct_buffer_load_i8_sext__sgpr_rsrc__vgpr_vindex__vgpr
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_i8_sext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_SBYTE_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_i8_sext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_SBYTE_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_i8_sext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_SBYTE_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SBYTE_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SBYTE_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %ext = sext i8 %val to i32
   %cast = bitcast i32 %ext to float
@@ -519,23 +746,41 @@ define amdgpu_ps float @struct_buffer_load_i8_sext_wrong_width(<4 x i32> inreg %
   ; GFX8-NEXT:   $vgpr0 = COPY [[V_BFE_I32_e64_]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_i8_sext_wrong_width
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
-  ; GFX12-NEXT:   [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN]], 0, 4, implicit $exec
-  ; GFX12-NEXT:   $vgpr0 = COPY [[V_BFE_I32_e64_]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_i8_sext_wrong_width
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1200-NEXT:   [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN]], 0, 4, implicit $exec
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[V_BFE_I32_e64_]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_i8_sext_wrong_width
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1250-NEXT:   [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[BUFFER_LOAD_UBYTE_VBUFFER_BOTHEN]], 0, 4, implicit $exec
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[V_BFE_I32_e64_]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %trunc = trunc i8 %val to i4
   %ext = sext i4 %trunc to i32
@@ -561,22 +806,39 @@ define amdgpu_ps float @struct_buffer_load_i16_zext__sgpr_rsrc__vgpr_vindex__vgp
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_i16_zext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_i16_zext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_i16_zext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i16 @llvm.amdgcn.struct.buffer.load.i16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %ext = zext i16 %val to i32
   %cast = bitcast i32 %ext to float
@@ -601,22 +863,39 @@ define amdgpu_ps float @struct_buffer_load_i16_sext__sgpr_rsrc__vgpr_vindex__vgp
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_i16_sext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_SSHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_i16_sext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_SSHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_i16_sext__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_SSHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_SSHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_SSHORT_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i16 @llvm.amdgcn.struct.buffer.load.i16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %ext = sext i16 %val to i32
   %cast = bitcast i32 %ext to float
@@ -642,23 +921,41 @@ define amdgpu_ps float @struct_buffer_load_i16_sext_wrong_width(<4 x i32> inreg
   ; GFX8-NEXT:   $vgpr0 = COPY [[V_BFE_I32_e64_]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_i16_sext_wrong_width
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]], 0, 8, implicit $exec
-  ; GFX12-NEXT:   $vgpr0 = COPY [[V_BFE_I32_e64_]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_i16_sext_wrong_width
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]], 0, 8, implicit $exec
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[V_BFE_I32_e64_]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_i16_sext_wrong_width
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[V_BFE_I32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_I32_e64 [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]], 0, 8, implicit $exec
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[V_BFE_I32_e64_]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call i16 @llvm.amdgcn.struct.buffer.load.i16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   %trunc = trunc i16 %val to i8
   %ext = sext i8 %trunc to i32
@@ -685,22 +982,39 @@ define amdgpu_ps half @struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voff
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret half %val
 }
@@ -724,22 +1038,39 @@ define amdgpu_ps <2 x half> @struct_buffer_load_v2f16__sgpr_rsrc__vgpr_vindex__v
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_v2f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_v2f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_v2f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call <2 x half> @llvm.amdgcn.struct.buffer.load.v2f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret <2 x half> %val
 }
@@ -772,25 +1103,45 @@ define amdgpu_ps <4 x half> @struct_buffer_load_v4f16__sgpr_rsrc__vgpr_vindex__v
   ; GFX8-NEXT:   $vgpr1 = COPY [[COPY8]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_v4f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub1
-  ; GFX12-NEXT:   $vgpr0 = COPY [[COPY7]]
-  ; GFX12-NEXT:   $vgpr1 = COPY [[COPY8]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ; GFX1200-LABEL: name: struct_buffer_load_v4f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub1
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[COPY7]]
+  ; GFX1200-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_v4f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_VBUFFER_BOTHEN]].sub1
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[COPY7]]
+  ; GFX1250-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
   %val = call <4 x half> @llvm.amdgcn.struct.buffer.load.v4f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret <4 x half> %val
 }
@@ -814,22 +1165,39 @@ define amdgpu_ps float @struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_vof
   ; GFX8-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_BOTHEN]]
   ; GFX8-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   ;
-  ; GFX12-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_glc
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
-  ; GFX12-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ; GFX1200-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_glc
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1200-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_load_f32__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_glc
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $vgpr0 = COPY [[BUFFER_LOAD_DWORD_VBUFFER_BOTHEN]]
+  ; GFX1250-NEXT:   SI_RETURN_TO_EPILOG implicit $vgpr0
   %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 1)
   ret float %val
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
index 674fe1c194e88..39cce20cc63f7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.tfe.ll
@@ -5,7 +5,8 @@
 ; RUN: llc -global-isel -mcpu=gfx900 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
 ; RUN: llc -global-isel -mcpu=gfx1010 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefix=GFX910
 ; RUN: llc -global-isel -mcpu=gfx1100 -mattr=-real-true16 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX11
-; RUN: llc -global-isel -mcpu=gfx1200 -mattr=-real-true16 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX12
+; RUN: llc -global-isel -mcpu=gfx1200 -mattr=-real-true16 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1200
+; RUN: llc -global-isel -mcpu=gfx1250 -mattr=-real-true16 -mtriple=amdgcn-- -stop-after=instruction-select < %s | FileCheck %s -check-prefixes=GFX1250
 
 define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
   ; GFX67-LABEL: name: raw_buffer_load_i8_tfe
@@ -114,29 +115,53 @@ define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspa
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_i8_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s8) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_i8_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s8) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_i8_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_UBYTE_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_BYTE [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s8) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { i8, i32 } @llvm.amdgcn.struct.buffer.load.sl_i8i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { i8, i32 } %res, 0
   store i8 %data, ptr addrspace(1) %data_addr
@@ -252,29 +277,53 @@ define amdgpu_ps void @raw_buffer_load_i16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_i16_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_i16_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_i16_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { i16, i32 } @llvm.amdgcn.struct.buffer.load.sl_i16i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { i16, i32 } %res, 0
   store i16 %data, ptr addrspace(1) %data_addr
@@ -390,29 +439,53 @@ define amdgpu_ps void @raw_buffer_load_f16_tfe(<4 x i32> inreg %rsrc, ptr addrsp
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_f16_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_f16_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_f16_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_USHORT_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_SHORT [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s16) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { half, i32 } @llvm.amdgcn.struct.buffer.load.sl_f16i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { half, i32 } %res, 0
   store half %data, ptr addrspace(1) %data_addr
@@ -528,29 +601,53 @@ define amdgpu_ps void @raw_buffer_load_i32_tfe(<4 x i32> inreg %rsrc, ptr addrsp
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_i32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_i32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_i32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_64_align2 = BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORD_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE1]], [[COPY9]], 0, 0, implicit $exec :: (store (s32) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY10]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { i32, i32 } @llvm.amdgcn.struct.buffer.load.sl_i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { i32, i32 } %res, 0
   store i32 %data, ptr addrspace(1) %data_addr
@@ -674,31 +771,57 @@ define amdgpu_ps void @raw_buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v2i32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v2i32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v2i32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <2 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.sl_v2i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <2 x i32>, i32 } %res, 0
   store <2 x i32> %data, ptr addrspace(1) %data_addr
@@ -822,31 +945,57 @@ define amdgpu_ps void @raw_buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v2f32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v2f32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v2f32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_96_align2 = BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX2 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<2 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY11]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <2 x float>, i32 } @llvm.amdgcn.struct.buffer.load.sl_v2f32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <2 x float>, i32 } %res, 0
   store <2 x float> %data, ptr addrspace(1) %data_addr
@@ -1018,32 +1167,59 @@ define amdgpu_ps void @raw_buffer_load_v3i32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v3i32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v3i32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v3i32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <3 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.sl_v3i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <3 x i32>, i32 } %res, 0
   store <3 x i32> %data, ptr addrspace(1) %data_addr
@@ -1215,32 +1391,59 @@ define amdgpu_ps void @raw_buffer_load_v3f32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v3f32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v3f32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v3f32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<3 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX3_TFE_VBUFFER_IDXEN]].sub3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX3 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<3 x s32>) into %ir.data_addr, align 16, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY12]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <3 x float>, i32 } @llvm.amdgcn.struct.buffer.load.sl_v3f32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <3 x float>, i32 } %res, 0
   store <3 x float> %data, ptr addrspace(1) %data_addr
@@ -1372,33 +1575,61 @@ define amdgpu_ps void @raw_buffer_load_v4i32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v4i32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v4i32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v4i32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160_align2 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <4 x i32>, i32 } @llvm.amdgcn.struct.buffer.load.sl_v4i32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <4 x i32>, i32 } %res, 0
   store <4 x i32> %data, ptr addrspace(1) %data_addr
@@ -1530,33 +1761,61 @@ define amdgpu_ps void @raw_buffer_load_v4f32_tfe(<4 x i32> inreg %rsrc, ptr addr
   ; GFX11-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
   ; GFX11-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: raw_buffer_load_v4f32_tfe
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
-  ; GFX12-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
-  ; GFX12-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: raw_buffer_load_v4f32_tfe
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1200-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1200-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: raw_buffer_load_v4f32_tfe
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; GFX1250-NEXT:   [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN:%[0-9]+]]:vreg_160_align2 = BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN [[COPY8]], [[REG_SEQUENCE]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub0
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub1
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub2
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub3
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_TFE_VBUFFER_IDXEN]].sub4
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY9]], %subreg.sub0, [[COPY10]], %subreg.sub1, [[COPY11]], %subreg.sub2, [[COPY12]], %subreg.sub3
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORDX4 [[REG_SEQUENCE1]], [[REG_SEQUENCE3]], 0, 0, implicit $exec :: (store (<4 x s32>) into %ir.data_addr, addrspace 1)
+  ; GFX1250-NEXT:   GLOBAL_STORE_DWORD [[REG_SEQUENCE2]], [[COPY13]], 0, 0, implicit $exec :: (store (s32) into %ir.tfe_addr, addrspace 1)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %res = call { <4 x float>, i32 } @llvm.amdgcn.struct.buffer.load.sl_v4f32i32s(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 0)
   %data = extractvalue { <4 x float>, i32 } %res, 0
   store <4 x float> %data, ptr addrspace(1) %data_addr
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.ll
index 8183d8532cdfb..c9771b5aca0db 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX1250 %s
 
 ; Natural mapping
 define amdgpu_ps void @struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(float %val, <4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
@@ -21,22 +22,39 @@ define amdgpu_ps void @struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex_
   ; GFX8-NEXT:   BUFFER_STORE_DWORD_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -61,24 +79,43 @@ define amdgpu_ps void @struct_buffer_store_v2f32_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; GFX8-NEXT:   BUFFER_STORE_DWORDX2_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_v2f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_v2f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_v2f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -104,25 +141,45 @@ define amdgpu_ps void @struct_buffer_store_v3f32_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; GFX8-NEXT:   BUFFER_STORE_DWORDX3_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_v3f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX3_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_v3f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX3_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_v3f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[COPY6]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX3_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY9]], 0, 0, 0, implicit $exec :: (dereferenceable store (<3 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -149,26 +206,47 @@ define amdgpu_ps void @struct_buffer_store_v4f32_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; GFX8-NEXT:   BUFFER_STORE_DWORDX4_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY10]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_v4f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX4_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY10]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_v4f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX4_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY10]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_v4f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX4_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY10]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -233,64 +311,123 @@ define amdgpu_ps void @struct_buffer_store_v4f32_vgpr_rsrc__sgpr_val__sgpr_vinde
   ; GFX8-NEXT: bb.5:
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_v4f32_vgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__vgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   successors: %bb.2(0x80000000)
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr7
-  ; GFX12-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr4
-  ; GFX12-NEXT:   [[COPY11:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
-  ; GFX12-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
-  ; GFX12-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
-  ; GFX12-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.2:
-  ; GFX12-NEXT:   successors: %bb.3(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY15:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
-  ; GFX12-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
-  ; GFX12-NEXT:   [[COPY17:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY17]], [[COPY15]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
-  ; GFX12-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY10]], implicit $exec
-  ; GFX12-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
-  ; GFX12-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.3:
-  ; GFX12-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX4_VBUFFER_BOTHEN_exact [[COPY11]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
-  ; GFX12-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
-  ; GFX12-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.4:
-  ; GFX12-NEXT:   successors: %bb.5(0x80000000)
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT: bb.5:
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_v4f32_vgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__vgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX1200-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1200-NEXT:   [[COPY11:%[0-9]+]]:vreg_128 = COPY [[REG_SEQUENCE]]
+  ; GFX1200-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1200-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; GFX1200-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.2:
+  ; GFX1200-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY14:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY15:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1200-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1200-NEXT:   [[COPY17:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY17]], [[COPY15]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GFX1200-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY10]], implicit $exec
+  ; GFX1200-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1200-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.3:
+  ; GFX1200-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX4_VBUFFER_BOTHEN_exact [[COPY11]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1200-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.4:
+  ; GFX1200-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT: bb.5:
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_v4f32_vgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__vgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr7
+  ; GFX1250-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GFX1250-NEXT:   [[COPY11:%[0-9]+]]:vreg_128_align2 = COPY [[REG_SEQUENCE]]
+  ; GFX1250-NEXT:   [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[COPY8]]
+  ; GFX1250-NEXT:   [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[COPY9]]
+  ; GFX1250-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.2:
+  ; GFX1250-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY14:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY15:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE1]].sub2_sub3
+  ; GFX1250-NEXT:   [[COPY16:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+  ; GFX1250-NEXT:   [[COPY17:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY16]], [[COPY14]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY17]], [[COPY15]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+  ; GFX1250-NEXT:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY10]], implicit $exec
+  ; GFX1250-NEXT:   [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+  ; GFX1250-NEXT:   [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.3:
+  ; GFX1250-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY13]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX4_VBUFFER_BOTHEN_exact [[COPY11]], [[REG_SEQUENCE3]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s32>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
+  ; GFX1250-NEXT:   SI_WATERFALL_LOOP %bb.2, implicit $exec
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.4:
+  ; GFX1250-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT: bb.5:
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -313,22 +450,39 @@ define amdgpu_ps void @struct_buffer_store_i8_sgpr_rsrc__vgpr_val__vgpr_vindex__
   ; GFX8-NEXT:   BUFFER_STORE_BYTE_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_i8_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_BYTE_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_i8_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_BYTE_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_i8_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_BYTE_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s8), addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %val.trunc = trunc i32 %val to i8
   call void @llvm.amdgcn.struct.buffer.store.i8(i8 %val.trunc, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -352,22 +506,39 @@ define amdgpu_ps void @struct_buffer_store_i16_sgpr_rsrc__vgpr_val__vgpr_vindex_
   ; GFX8-NEXT:   BUFFER_STORE_SHORT_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_i16_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_SHORT_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_i16_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_SHORT_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_i16_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_SHORT_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (s16), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   %val.trunc = trunc i32 %val to i16
   call void @llvm.amdgcn.struct.buffer.store.i16(i16 %val.trunc, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
@@ -391,22 +562,39 @@ define amdgpu_ps void @struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex_
   ; GFX8-NEXT:   BUFFER_STORE_DWORD_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset_glc
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset_glc
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_f32_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset_glc
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 1, 0, implicit $exec :: (dereferenceable store (s32), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 1)
   ret void
 }
@@ -429,22 +617,39 @@ define amdgpu_ps void @struct_buffer_store_v2f16_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; GFX8-NEXT:   BUFFER_STORE_DWORD_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_v2f16_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_v2f16_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_v2f16_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -475,24 +680,43 @@ define amdgpu_ps void @struct_buffer_store_v4f16_sgpr_rsrc__vgpr_val__vgpr_vinde
   ; GFX8-NEXT:   BUFFER_STORE_DWORDX2_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
   ; GFX8-NEXT:   S_ENDPGM 0
   ;
-  ; GFX12-LABEL: name: struct_buffer_store_v4f16_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
-  ; GFX12: bb.1 (%ir-block.0):
-  ; GFX12-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
-  ; GFX12-NEXT: {{  $}}
-  ; GFX12-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-  ; GFX12-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-  ; GFX12-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
-  ; GFX12-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
-  ; GFX12-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
-  ; GFX12-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
-  ; GFX12-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
-  ; GFX12-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
-  ; GFX12-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
-  ; GFX12-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
-  ; GFX12-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
-  ; GFX12-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
-  ; GFX12-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
-  ; GFX12-NEXT:   S_ENDPGM 0
+  ; GFX1200-LABEL: name: struct_buffer_store_v4f16_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1200: bb.1 (%ir-block.0):
+  ; GFX1200-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1200-NEXT: {{  $}}
+  ; GFX1200-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1200-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1200-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1200-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1200-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1200-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1200-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1200-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1200-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1200-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1200-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1200-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1200-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
+  ; GFX1200-NEXT:   S_ENDPGM 0
+  ;
+  ; GFX1250-LABEL: name: struct_buffer_store_v4f16_sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; GFX1250: bb.1 (%ir-block.0):
+  ; GFX1250-NEXT:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX1250-NEXT: {{  $}}
+  ; GFX1250-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX1250-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GFX1250-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GFX1250-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; GFX1250-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
+  ; GFX1250-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
+  ; GFX1250-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
+  ; GFX1250-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
+  ; GFX1250-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GFX1250-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GFX1250-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; GFX1250-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
+  ; GFX1250-NEXT:   BUFFER_STORE_DWORDX2_VBUFFER_BOTHEN_exact [[REG_SEQUENCE]], [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (<4 x s16>), align 1, addrspace 8)
+  ; GFX1250-NEXT:   S_ENDPGM 0
   call void @llvm.amdgcn.struct.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.inline.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.inline.ll
index 7c0484bf3f317..1ba2558a49ad4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.inline.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memcpy.inline.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn -mem-intrinsic-expand-size=3 %s -o - | FileCheck -check-prefix=GCN %s
-; RUN: llc -global-isel -mtriple=amdgcn -mem-intrinsic-expand-size=5 %s -o - | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mem-intrinsic-expand-size=3 %s -o - | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mem-intrinsic-expand-size=5 %s -o - | FileCheck -check-prefix=GCN %s
 
 declare void @llvm.memcpy.inline.p1.p1.i32(ptr addrspace(1), ptr addrspace(1), i32, i1 immarg)
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 3daae98961bff..637aaf7529364 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -164,7 +164,7 @@ define zeroext i16 @v_mul_i16_zeroext(i16 zeroext %num, i16 zeroext %den) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_mul_i16_zeroext:
@@ -2854,89 +2854,90 @@ define i256 @v_mul_i256(i256 %num, i256 %den) {
 ; GFX1250:       ; %bb.0:
 ; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[16:17], v0, v14, 0
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[18:19], v0, v12, 0
+; GFX1250-NEXT:    v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v1
 ; GFX1250-NEXT:    v_mul_lo_u32 v27, v5, v10
 ; GFX1250-NEXT:    v_mul_lo_u32 v29, v3, v12
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[16:17], v1, v13, v[16:17]
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], s0, v1, v11, v[18:19]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v16, v14, 0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[18:19], v16, v12, 0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v17, v13, v[0:1]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], s0, v17, v11, v[18:19]
 ; GFX1250-NEXT:    v_cndmask_b32_e64 v20, 0, 1, s0
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[16:17], v2, v12, v[16:17]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v2, v12, v[0:1]
 ; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], vcc_lo, v2, v10, v[18:19]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX1250-NEXT:    v_add_co_ci_u32_e64 v22, null, 0, v20, vcc_lo
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[20:21], v0, v10, 0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[16:17], v3, v11, v[16:17]
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[20:21], v16, v10, 0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v3, v11, v[0:1]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], vcc_lo, v3, v9, v[18:19]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1250-NEXT:    v_add_co_ci_u32_e64 v24, null, 0, v22, vcc_lo
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[16:17], v4, v10, v[16:17]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v4, v10, v[0:1]
 ; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], vcc_lo, v4, v8, v[18:19]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX1250-NEXT:    v_add_co_ci_u32_e64 v26, null, 0, v24, vcc_lo
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[16:17], v5, v9, v[16:17]
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[22:23], v6, v8, v[16:17]
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[16:17], s0, v1, v9, v[20:21]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v5, v9, v[0:1]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[22:23], v6, v8, v[0:1]
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[0:1], s0, v17, v9, v[20:21]
 ; GFX1250-NEXT:    v_dual_mov_b32 v20, v19 :: v_dual_mov_b32 v21, v22
 ; GFX1250-NEXT:    v_mul_lo_u32 v22, v6, v9
 ; GFX1250-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s0
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[24:25], s0, v2, v8, v[16:17]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[20:21], vcc_lo, v0, v13, v[20:21]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[24:25], s0, v2, v8, v[0:1]
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[20:21], vcc_lo, v16, v13, v[20:21]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX1250-NEXT:    v_add_co_ci_u32_e64 v6, null, 0, v6, s0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[16:17], s0, v1, v12, v[20:21]
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[0:1], s0, v17, v12, v[20:21]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1250-NEXT:    v_dual_mov_b32 v20, v25 :: v_dual_mov_b32 v21, v18
 ; GFX1250-NEXT:    v_mul_lo_u32 v25, v4, v11
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], s2, v0, v11, v[20:21]
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], s2, v16, v11, v[20:21]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX1250-NEXT:    v_cndmask_b32_e64 v28, 0, 1, s2
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[20:21], s1, v2, v11, v[16:17]
-; GFX1250-NEXT:    v_mad_nc_u64_u32 v[16:17], v0, v8, 0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], s2, v1, v10, v[18:19]
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[20:21], s1, v2, v11, v[0:1]
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v16, v8, 0
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], s2, v17, v10, v[18:19]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX1250-NEXT:    v_mad_co_u64_u32 v[10:11], s3, v3, v10, v[20:21]
 ; GFX1250-NEXT:    v_mul_lo_u32 v20, v2, v13
 ; GFX1250-NEXT:    v_add_co_ci_u32_e64 v21, null, 0, v28, s2
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX1250-NEXT:    v_mad_co_u64_u32 v[12:13], s2, v2, v9, v[18:19]
-; GFX1250-NEXT:    v_dual_mov_b32 v18, v17 :: v_dual_mov_b32 v19, v24
+; GFX1250-NEXT:    v_dual_mov_b32 v18, v1 :: v_dual_mov_b32 v19, v24
 ; GFX1250-NEXT:    v_mad_co_u64_u32 v[10:11], s4, v4, v9, v[10:11]
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v2, null, 0, v21, s2
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], s6, v0, v9, v[18:19]
-; GFX1250-NEXT:    v_mul_lo_u32 v0, v0, v15
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v21, s2
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[18:19], s6, v16, v9, v[18:19]
+; GFX1250-NEXT:    v_mul_lo_u32 v2, v16, v15
 ; GFX1250-NEXT:    v_mad_co_u64_u32 v[12:13], s2, v3, v8, v[12:13]
 ; GFX1250-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s6
-; GFX1250-NEXT:    v_mul_lo_u32 v9, v1, v14
+; GFX1250-NEXT:    v_mul_lo_u32 v9, v17, v14
 ; GFX1250-NEXT:    v_mad_co_u64_u32 v[10:11], s5, v5, v8, v[10:11]
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v2, null, 0, v2, s2
-; GFX1250-NEXT:    v_mad_co_u64_u32 v[14:15], s2, v1, v8, v[18:19]
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, s2
+; GFX1250-NEXT:    v_mad_co_u64_u32 v[14:15], s2, v17, v8, v[18:19]
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1250-NEXT:    v_add_co_ci_u32_e64 v3, s2, v3, v12, s2
 ; GFX1250-NEXT:    v_add_co_ci_u32_e64 v4, s2, v6, v13, s2
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v5, s2, v2, v10, s2
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v5, s2, v1, v10, s2
 ; GFX1250-NEXT:    v_add_co_ci_u32_e64 v6, s2, v26, v11, s2
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v0, null, v23, v0, s2
-; GFX1250-NEXT:    v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v1, v14
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v0, null, v0, v9, s5
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v1, null, v23, v2, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, v15
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v9, s5
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v0, null, v0, v20, s4
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v0, null, v0, v29, s3
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v20, s4
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v29, s3
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v0, null, v0, v25, s1
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v0, null, v0, v27, s0
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v25, s1
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v27, s0
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_co_ci_u32_e64 v0, null, v0, v22, vcc_lo
-; GFX1250-NEXT:    v_mad_u32 v7, v7, v8, v0
-; GFX1250-NEXT:    v_mov_b32_e32 v0, v16
+; GFX1250-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v22, vcc_lo
+; GFX1250-NEXT:    v_mad_u32 v7, v7, v8, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v14
 ; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %result = mul i256 %num, %den
   ret i256 %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-zextload-from-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-zextload-from-and.mir
index 73e06de1923df..8d72d11fe71d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-zextload-from-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizer-combiner-zextload-from-and.mir
@@ -14,9 +14,9 @@ body: |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
     ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p1) :: (load (s64), addrspace 1)
-    ; CHECK-NEXT: %k:_(s64) = G_CONSTANT i64 4294967295
-    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[LOAD]], %k
-    ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[AND]](s64)
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[LOAD]](s64)
+    ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[TRUNC]](s32)
+    ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s64) = G_LOAD %0 :: (load (s64), align 8, addrspace 1)
     %k:_(s64) = G_CONSTANT i64 4294967295
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s32.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s32.mir
index 45332c2870c02..4fdc8e435c23d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s32.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-add.s32.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck %s
 
 ---
 name: add_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readfirstlane.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readfirstlane.mir
index 04cdf2e9fce73..17fc2d0c23771 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readfirstlane.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.readfirstlane.mir
@@ -1,6 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" -o - %s | FileCheck %s
 
 ---
 name: readfirstlane_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getpc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getpc.mir
index 9650da855ba5a..cba0db7163eeb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getpc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.s.getpc.mir
@@ -1,6 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" -o -  %s | FileCheck %s
 
 ---
 name: getpc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir
index 550f042618abf..66bdf4193010b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-bitcast.mir
@@ -1,6 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck %s
 
 ---
 name: bitcast_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
index 3b2b141539fc5..cd957c8b9c464 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
@@ -1,6 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck %s
 
 ---
 name: brcond_vcc_cond
@@ -40,9 +39,9 @@ body:             |
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
   ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
-  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[ICMP]](s32)
-  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
-  ; CHECK-NEXT:   G_BRCOND [[ZEXT]](s32), %bb.1
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
+  ; CHECK-NEXT:   G_BRCOND [[AND]](s32), %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   bb.0.entry:
@@ -66,9 +65,9 @@ body:             |
   ; CHECK-NEXT:   liveins: $sgpr0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s32)
-  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:sgpr(s32) = G_ZEXT [[TRUNC]](s1)
-  ; CHECK-NEXT:   G_BRCOND [[ZEXT]](s32), %bb.1
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[C]]
+  ; CHECK-NEXT:   G_BRCOND [[AND]](s32), %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   bb.0.entry:
@@ -91,9 +90,11 @@ body:             |
   ; CHECK-NEXT:   liveins: $vgpr0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
-  ; CHECK-NEXT:   G_BRCOND [[COPY1]](s1), %bb.1
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY]], [[C]]
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](s32), [[C1]]
+  ; CHECK-NEXT:   G_BRCOND [[ICMP]](s1), %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   bb.0.entry:
@@ -120,13 +121,15 @@ body:             |
   ; CHECK-NEXT:   liveins: $vgpr0
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY]], [[C]]
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](s32), [[C1]]
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
-  ; CHECK-NEXT:   G_BRCOND [[COPY1]](s1), %bb.1
+  ; CHECK-NEXT:   G_BRCOND [[ICMP]](s1), %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   bb.0.entry:
@@ -157,9 +160,11 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
-  ; CHECK-NEXT:   G_BRCOND [[COPY1]](s1), %bb.1
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY]], [[C]]
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](s32), [[C1]]
+  ; CHECK-NEXT:   G_BRCOND [[ICMP]](s1), %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   bb.0.entry:
@@ -189,10 +194,12 @@ body:             |
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK-NEXT:   [[C:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 1
+  ; CHECK-NEXT:   [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY]], [[C]]
+  ; CHECK-NEXT:   [[C1:%[0-9]+]]:vgpr(s32) = G_CONSTANT i32 0
+  ; CHECK-NEXT:   [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[AND]](s32), [[C1]]
   ; CHECK-NEXT:   S_NOP 0
-  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
-  ; CHECK-NEXT:   G_BRCOND [[COPY1]](s1), %bb.1
+  ; CHECK-NEXT:   G_BRCOND [[ICMP]](s1), %bb.1
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   bb.0.entry:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-build-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-build-vector.mir
index 0dc1165843e8f..ef2477cde4ddc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-build-vector.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-build-vector.mir
@@ -1,6 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" -o - %s | FileCheck %s
 
 ---
 name: build_vector_v2s32_ss
@@ -88,9 +87,9 @@ body: |
     ; CHECK-LABEL: name: build_vector_v2s32_aa
     ; CHECK: liveins: $agpr0, $agpr1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr(s32) = COPY $agpr0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr(s32) = COPY $agpr1
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:agpr(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $agpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $agpr1
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<2 x s32>)
     %0:_(s32) = COPY $agpr0
     %1:_(s32) = COPY $agpr1
@@ -111,9 +110,8 @@ body: |
     ; CHECK: liveins: $vgpr0, $agpr0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr(s32) = COPY $agpr0
-    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY2]](s32)
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $agpr0
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<2 x s32>)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $agpr0
@@ -133,10 +131,9 @@ body: |
     ; CHECK-LABEL: name: build_vector_v2s32_av
     ; CHECK: liveins: $vgpr0, $agpr0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr(s32) = COPY $agpr0
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $agpr0
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY1]](s32)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<2 x s32>)
     %0:_(s32) = COPY $agpr0
     %1:_(s32) = COPY $vgpr0
@@ -157,10 +154,9 @@ body: |
     ; CHECK: liveins: $sgpr0, $agpr0
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr(s32) = COPY $agpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $agpr0
     ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
-    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY1]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<2 x s32>)
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $agpr0
@@ -180,11 +176,10 @@ body: |
     ; CHECK-LABEL: name: build_vector_v2s32_as
     ; CHECK: liveins: $sgpr0, $agpr0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr(s32) = COPY $agpr0
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $agpr0
     ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
-    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY2]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<2 x s32>)
     %0:_(s32) = COPY $agpr0
     %1:_(s32) = COPY $sgpr0
@@ -204,10 +199,10 @@ body: |
     ; CHECK-LABEL: name: build_vector_v3s32_aaa
     ; CHECK: liveins: $agpr0, $agpr1, $agpr2
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr(s32) = COPY $agpr0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr(s32) = COPY $agpr1
-    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(s32) = COPY $agpr2
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:agpr(<3 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32)
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $agpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $agpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $agpr2
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<3 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<3 x s32>)
     %0:_(s32) = COPY $agpr0
     %1:_(s32) = COPY $agpr1
@@ -228,11 +223,11 @@ body: |
     ; CHECK-LABEL: name: build_vector_v4s32_aaaa
     ; CHECK: liveins: $agpr0, $agpr1, $agpr2
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr(s32) = COPY $agpr0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr(s32) = COPY $agpr1
-    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(s32) = COPY $agpr2
-    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(s32) = COPY $agpr2
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:agpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $agpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $agpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $agpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $agpr2
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<4 x s32>)
     %0:_(s32) = COPY $agpr0
     %1:_(s32) = COPY $agpr1
@@ -254,15 +249,15 @@ body: |
     ; CHECK-LABEL: name: build_vector_v8s32_aaaaaaaa
     ; CHECK: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5, $agpr6, $agpr7
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr(s32) = COPY $agpr0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr(s32) = COPY $agpr1
-    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(s32) = COPY $agpr2
-    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(s32) = COPY $agpr3
-    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(s32) = COPY $agpr4
-    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(s32) = COPY $agpr5
-    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:agpr(s32) = COPY $agpr6
-    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:agpr(s32) = COPY $agpr7
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:agpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $agpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $agpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $agpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $agpr3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $agpr4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $agpr5
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $agpr6
+    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $agpr7
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<8 x s32>)
     %0:_(s32) = COPY $agpr0
     %1:_(s32) = COPY $agpr1
@@ -288,23 +283,23 @@ body: |
     ; CHECK-LABEL: name: build_vector_v16s32_aaaaaaaaaaaaaaaa
     ; CHECK: liveins: $agpr0, $agpr1, $agpr2, $agpr3, $agpr4, $agpr5, $agpr6, $agpr7, $agpr8, $agpr9, $agpr10, $agpr11, $agpr12, $agpr13, $agpr14, $agpr15
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr(s32) = COPY $agpr0
-    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr(s32) = COPY $agpr1
-    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:agpr(s32) = COPY $agpr2
-    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr(s32) = COPY $agpr3
-    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr(s32) = COPY $agpr4
-    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:agpr(s32) = COPY $agpr5
-    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:agpr(s32) = COPY $agpr6
-    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:agpr(s32) = COPY $agpr7
-    ; CHECK-NEXT: [[COPY8:%[0-9]+]]:agpr(s32) = COPY $agpr8
-    ; CHECK-NEXT: [[COPY9:%[0-9]+]]:agpr(s32) = COPY $agpr9
-    ; CHECK-NEXT: [[COPY10:%[0-9]+]]:agpr(s32) = COPY $agpr10
-    ; CHECK-NEXT: [[COPY11:%[0-9]+]]:agpr(s32) = COPY $agpr11
-    ; CHECK-NEXT: [[COPY12:%[0-9]+]]:agpr(s32) = COPY $agpr12
-    ; CHECK-NEXT: [[COPY13:%[0-9]+]]:agpr(s32) = COPY $agpr13
-    ; CHECK-NEXT: [[COPY14:%[0-9]+]]:agpr(s32) = COPY $agpr14
-    ; CHECK-NEXT: [[COPY15:%[0-9]+]]:agpr(s32) = COPY $agpr15
-    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:agpr(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $agpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $agpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $agpr2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $agpr3
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $agpr4
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $agpr5
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $agpr6
+    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $agpr7
+    ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $agpr8
+    ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY $agpr9
+    ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY $agpr10
+    ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY $agpr11
+    ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $agpr12
+    ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vgpr(s32) = COPY $agpr13
+    ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vgpr(s32) = COPY $agpr14
+    ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vgpr(s32) = COPY $agpr15
+    ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
     ; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<16 x s32>)
     %0:_(s32) = COPY $agpr0
     %1:_(s32) = COPY $agpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir
index bd699956500ca..456e0c135a874 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -march amdgcn -mcpu=fiji -run-pass=regbankselect %s -o - | FileCheck %s
+# RUN: llc -O0 -march amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck %s
 
 # Check the default mappings for various instructions.
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fadd.mir
index 4fba30325f98b..8944fd78ca9ee 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fadd.mir
@@ -1,6 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck %s
 
 ---
 name: fadd_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir
index 2b60dcd9dcabe..62bf14b091340 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir
@@ -1,6 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck %s
 
 ---
 name: test_frame_index_p5
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptr-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptr-add.mir
index 55048d51918fa..57b7a822ed7aa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptr-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptr-add.mir
@@ -1,6 +1,102 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck %s
+
+---
+name: gep_p0_s_k
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+
+    ; CHECK-LABEL: name: gep_p0_s_k
+    ; CHECK: liveins: $sgpr0_sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p0) = G_PTR_ADD [[COPY]], [[C]](s64)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(p0) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p0_s_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
+
+    ; CHECK-LABEL: name: gep_p0_s_s
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p0) = G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s64) = COPY $sgpr2_sgpr3
+    %2:_(p0) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p0_v_k
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: gep_p0_v_k
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY [[C]](s64)
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p0) = G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(p0) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p0_v_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
+
+    ; CHECK-LABEL: name: gep_p0_v_s
+    ; CHECK: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s64) = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p0) = G_PTR_ADD [[COPY]], [[COPY2]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $sgpr0_sgpr1
+    %2:_(p0) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p0_v_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: gep_p0_v_v
+    ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p0) = G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(p0) = G_PTR_ADD %0, %1
+...
 
 ---
 name: gep_p1_s_k
@@ -98,3 +194,294 @@ body: |
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(p1) = G_PTR_ADD %0, %1
 ...
+
+---
+name: gep_p3_s_k
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; CHECK-LABEL: name: gep_p3_s_k
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p3) = G_PTR_ADD [[COPY]], [[C]](s32)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(p3) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p3_s_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+
+    ; CHECK-LABEL: name: gep_p3_s_s
+    ; CHECK: liveins: $sgpr0, $sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p3) = G_PTR_ADD [[COPY]], [[COPY1]](s32)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(p3) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p3_v_k
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: gep_p3_v_k
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p3) = G_PTR_ADD [[COPY]], [[COPY1]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(p3) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p3_v_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $sgpr0
+
+    ; CHECK-LABEL: name: gep_p3_v_s
+    ; CHECK: liveins: $vgpr0, $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p3) = G_PTR_ADD [[COPY]], [[COPY2]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(p3) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p3_v_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: gep_p3_v_v
+    ; CHECK: liveins: $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p3) = G_PTR_ADD [[COPY]], [[COPY1]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p3) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p4_s_k
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+
+    ; CHECK-LABEL: name: gep_p4_s_k
+    ; CHECK: liveins: $sgpr0_sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = G_PTR_ADD [[COPY]], [[C]](s64)
+    %0:_(p4) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(p4) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p4_s_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
+
+    ; CHECK-LABEL: name: gep_p4_s_s
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p4) = G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    %0:_(p4) = COPY $sgpr0_sgpr1
+    %1:_(s64) = COPY $sgpr2_sgpr3
+    %2:_(p4) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p4_v_k
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: gep_p4_v_k
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p4) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY [[C]](s64)
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p4) = G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    %0:_(p4) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(p4) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p4_v_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
+
+    ; CHECK-LABEL: name: gep_p4_v_s
+    ; CHECK: liveins: $vgpr0_vgpr1, $sgpr0_sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p4) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s64) = COPY [[COPY1]](s64)
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p4) = G_PTR_ADD [[COPY]], [[COPY2]](s64)
+    %0:_(p4) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $sgpr0_sgpr1
+    %2:_(p4) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p4_v_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: gep_p4_v_v
+    ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p4) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p4) = G_PTR_ADD [[COPY]], [[COPY1]](s64)
+    %0:_(p4) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(p4) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p5_s_k
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; CHECK-LABEL: name: gep_p5_s_k
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sgpr0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[C]](s32)
+    %0:_(p5) = COPY $sgpr0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(p5) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p5_s_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+
+    ; CHECK-LABEL: name: gep_p5_s_s
+    ; CHECK: liveins: $sgpr0, $sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:sgpr(p5) = G_PTR_ADD [[COPY]], [[COPY1]](s32)
+    %0:_(p5) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(p5) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p5_v_k
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: gep_p5_v_k
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p5) = G_PTR_ADD [[COPY]], [[COPY1]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(p5) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p5_v_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $sgpr0
+
+    ; CHECK-LABEL: name: gep_p5_v_s
+    ; CHECK: liveins: $vgpr0, $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p5) = G_PTR_ADD [[COPY]], [[COPY2]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(p5) = G_PTR_ADD %0, %1
+...
+
+---
+name: gep_p5_v_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: gep_p5_v_v
+    ; CHECK: liveins: $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:vgpr(p5) = G_PTR_ADD [[COPY]], [[COPY1]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p5) = G_PTR_ADD %0, %1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
index 2177cd7493c45..997ac804f710d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-split-scalar-load-metadata.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -run-pass=regbankselect %s -o - | FileCheck -check-prefix=GFX7 %s
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -run-pass=regbankselect %s -o - | FileCheck -check-prefix=GFX12 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=hawaii -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck -check-prefix=GFX12 %s
 
 --- |
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir
index 4fcd0fdf105d9..b0199d3ad5cd1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sub.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck %s
 
 ---
 name: sub_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
index 554c88a68972b..e95be13c47d3b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
@@ -1,6 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -o - | FileCheck %s
 
 ---
 name: uitofp_s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
index 4031fe0be2823..f57fc005b994b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdiv.i64.ll
@@ -194,10 +194,8 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
 define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-LABEL: s_sdiv_i64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_mov_b32 s6, 0
 ; CHECK-NEXT:    s_or_b64 s[0:1], s[2:3], s[4:5]
-; CHECK-NEXT:    s_mov_b32 s7, -1
-; CHECK-NEXT:    s_and_b64 s[0:1], s[0:1], s[6:7]
+; CHECK-NEXT:    s_mov_b32 s0, 0
 ; CHECK-NEXT:    v_cmp_ne_u64_e64 vcc, s[0:1], 0
 ; CHECK-NEXT:    s_mov_b32 s0, 1
 ; CHECK-NEXT:    s_cbranch_vccz .LBB1_2
@@ -218,7 +216,6 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; CHECK-NEXT:    s_subb_u32 s5, 0, s11
-; CHECK-NEXT:    s_xor_b64 s[6:7], s[6:7], s[8:9]
 ; CHECK-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; CHECK-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; CHECK-NEXT:    v_trunc_f32_e32 v2, v1
@@ -327,9 +324,10 @@ define amdgpu_ps i64 @s_sdiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v1, vcc
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; CHECK-NEXT:    v_xor_b32_e32 v0, s6, v0
+; CHECK-NEXT:    s_xor_b64 s[0:1], s[6:7], s[8:9]
+; CHECK-NEXT:    v_xor_b32_e32 v0, s0, v0
+; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s0, v0
 ; CHECK-NEXT:    s_mov_b32 s0, 0
-; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
 ; CHECK-NEXT:    s_branch .LBB1_3
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shlN_add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shlN_add.ll
index ad60a617dcd1c..9f4a6f2f63f15 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shlN_add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shlN_add.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GCN,GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GCN,GFX10 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GCN,GFX10 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GCN,GFX10 %s
 
 ; Test gfx9+ s_shl[1-4]_add_u32 pattern matching
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
index 1a10f5fb7a5ce..8d8eca162257a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/srem.i64.ll
@@ -188,12 +188,10 @@ declare i32 @llvm.amdgcn.readfirstlane(i32)
 define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-LABEL: s_srem_i64:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_mov_b32 s6, 0
 ; CHECK-NEXT:    s_or_b64 s[0:1], s[2:3], s[4:5]
-; CHECK-NEXT:    s_mov_b32 s7, -1
-; CHECK-NEXT:    s_and_b64 s[0:1], s[0:1], s[6:7]
+; CHECK-NEXT:    s_mov_b32 s0, 0
 ; CHECK-NEXT:    v_cmp_ne_u64_e64 vcc, s[0:1], 0
-; CHECK-NEXT:    s_mov_b32 s7, 1
+; CHECK-NEXT:    s_mov_b32 s0, 1
 ; CHECK-NEXT:    s_cbranch_vccz .LBB1_2
 ; CHECK-NEXT:  ; %bb.1:
 ; CHECK-NEXT:    s_ashr_i32 s6, s3, 31
@@ -212,7 +210,6 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_mac_f32_e32 v0, 0x4f800000, v1
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; CHECK-NEXT:    s_subb_u32 s5, 0, s9
-; CHECK-NEXT:    s_mov_b32 s7, 0
 ; CHECK-NEXT:    v_mul_f32_e32 v0, 0x5f7ffffc, v0
 ; CHECK-NEXT:    v_mul_f32_e32 v1, 0x2f800000, v0
 ; CHECK-NEXT:    v_trunc_f32_e32 v2, v1
@@ -273,43 +270,43 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_addc_u32_e32 v1, vcc, v4, v1, vcc
 ; CHECK-NEXT:    v_mul_lo_u32 v2, s11, v0
 ; CHECK-NEXT:    v_mul_lo_u32 v3, s10, v1
-; CHECK-NEXT:    v_mul_hi_u32 v5, s10, v0
+; CHECK-NEXT:    v_mul_hi_u32 v4, s10, v0
 ; CHECK-NEXT:    v_mul_hi_u32 v0, s11, v0
-; CHECK-NEXT:    v_mul_hi_u32 v6, s11, v1
+; CHECK-NEXT:    v_mul_hi_u32 v5, s11, v1
 ; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
 ; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v5
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v2, v4
 ; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; CHECK-NEXT:    v_mul_lo_u32 v5, s11, v1
+; CHECK-NEXT:    v_mul_lo_u32 v4, s11, v1
 ; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; CHECK-NEXT:    v_mul_hi_u32 v3, s10, v1
-; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
-; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
+; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
 ; CHECK-NEXT:    v_add_i32_e32 v0, vcc, v0, v3
 ; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v5, v3
-; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v0, v2
-; CHECK-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s8, v5, 0
+; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
+; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
+; CHECK-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], s8, v4, 0
 ; CHECK-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
 ; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
-; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v6, v2
+; CHECK-NEXT:    v_add_i32_e32 v2, vcc, v5, v2
 ; CHECK-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s8, v2, v[1:2]
-; CHECK-NEXT:    v_mov_b32_e32 v3, s11
+; CHECK-NEXT:    v_mov_b32_e32 v5, s11
 ; CHECK-NEXT:    v_sub_i32_e32 v0, vcc, s10, v0
-; CHECK-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s9, v5, v[1:2]
-; CHECK-NEXT:    v_mov_b32_e32 v4, s9
-; CHECK-NEXT:    v_subb_u32_e64 v2, s[0:1], v3, v1, vcc
+; CHECK-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s9, v4, v[1:2]
+; CHECK-NEXT:    v_mov_b32_e32 v3, s9
+; CHECK-NEXT:    v_subb_u32_e64 v2, s[0:1], v5, v1, vcc
 ; CHECK-NEXT:    v_sub_i32_e64 v1, s[0:1], s11, v1
+; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
 ; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v2
-; CHECK-NEXT:    v_cndmask_b32_e64 v3, 0, -1, s[0:1]
+; CHECK-NEXT:    v_subrev_i32_e32 v3, vcc, s8, v0
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
 ; CHECK-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v0
+; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
 ; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
 ; CHECK-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v2
-; CHECK-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
-; CHECK-NEXT:    v_cndmask_b32_e64 v2, v3, v5, s[0:1]
-; CHECK-NEXT:    v_subrev_i32_e32 v3, vcc, s8, v0
-; CHECK-NEXT:    v_subbrev_u32_e32 v1, vcc, 0, v1, vcc
 ; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s9, v1
+; CHECK-NEXT:    v_cndmask_b32_e64 v2, v4, v5, s[0:1]
 ; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
 ; CHECK-NEXT:    v_cmp_le_u32_e32 vcc, s8, v3
 ; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, -1, vcc
@@ -322,11 +319,12 @@ define amdgpu_ps i64 @s_srem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc
 ; CHECK-NEXT:    v_xor_b32_e32 v0, s6, v0
 ; CHECK-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v0
+; CHECK-NEXT:    s_mov_b32 s0, 0
 ; CHECK-NEXT:    s_branch .LBB1_3
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; CHECK-NEXT:  .LBB1_3: ; %Flow
-; CHECK-NEXT:    s_xor_b32 s0, s7, 1
+; CHECK-NEXT:    s_xor_b32 s0, s0, 1
 ; CHECK-NEXT:    s_cmp_lg_u32 s0, 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB1_5
 ; CHECK-NEXT:  ; %bb.4:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-weird-size.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-weird-size.ll
new file mode 100644
index 0000000000000..0aa08cc2b1d6f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-weird-size.ll
@@ -0,0 +1,224 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -O0 -global-isel=true -stop-after=legalizer -o - %s | FileCheck -check-prefix=UNPACKED %s
+
+define void @store_i48(ptr addrspace(1) %ptr, i48 %arg) #0 {
+  ; UNPACKED-LABEL: name: store_i48
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; UNPACKED-NEXT: {{  $}}
+  ; UNPACKED-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; UNPACKED-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; UNPACKED-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; UNPACKED-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; UNPACKED-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; UNPACKED-NEXT:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:_(s64) = COPY [[MV1]](s64)
+  ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+  ; UNPACKED-NEXT:   [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY4]], [[C]](s32)
+  ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; UNPACKED-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[MV]], [[C1]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[COPY2]](s32), [[MV]](p1) :: (store (s32) into %ir.ptr, addrspace 1)
+  ; UNPACKED-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[TRUNC]](s32), [[PTR_ADD]](p1) :: (store (s16) into %ir.ptr + 4, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   SI_RETURN
+  store i48 %arg, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
+define void @store_i55(ptr addrspace(1) %ptr, i55 %arg) #0 {
+  ; UNPACKED-LABEL: name: store_i55
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; UNPACKED-NEXT: {{  $}}
+  ; UNPACKED-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; UNPACKED-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; UNPACKED-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; UNPACKED-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; UNPACKED-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; UNPACKED-NEXT:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36028797018963967
+  ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[MV1]], [[C]]
+  ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:_(s64) = COPY [[AND]](s64)
+  ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+  ; UNPACKED-NEXT:   [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY4]], [[C1]](s32)
+  ; UNPACKED-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; UNPACKED-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[MV]], [[C2]](s64)
+  ; UNPACKED-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY4]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[TRUNC]](s32), [[MV]](p1) :: (store (s32) into %ir.ptr, addrspace 1)
+  ; UNPACKED-NEXT:   [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
+  ; UNPACKED-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; UNPACKED-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC1]], [[C3]](s32)
+  ; UNPACKED-NEXT:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; UNPACKED-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD]], [[C4]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[TRUNC1]](s32), [[PTR_ADD]](p1) :: (store (s16) into %ir.ptr + 4, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p1) :: (store (s8) into %ir.ptr + 6, align 2, basealign 4, addrspace 1)
+  ; UNPACKED-NEXT:   SI_RETURN
+  store i55 %arg, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
+define void @store_i56(ptr addrspace(1) %ptr, i56 %arg) #0 {
+  ; UNPACKED-LABEL: name: store_i56
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; UNPACKED-NEXT: {{  $}}
+  ; UNPACKED-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; UNPACKED-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; UNPACKED-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; UNPACKED-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; UNPACKED-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; UNPACKED-NEXT:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:_(s64) = COPY [[MV1]](s64)
+  ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+  ; UNPACKED-NEXT:   [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY4]], [[C]](s32)
+  ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; UNPACKED-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[MV]], [[C1]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[COPY2]](s32), [[MV]](p1) :: (store (s32) into %ir.ptr, addrspace 1)
+  ; UNPACKED-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
+  ; UNPACKED-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+  ; UNPACKED-NEXT:   [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[TRUNC]], [[C2]](s32)
+  ; UNPACKED-NEXT:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+  ; UNPACKED-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD]], [[C3]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[TRUNC]](s32), [[PTR_ADD]](p1) :: (store (s16) into %ir.ptr + 4, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   G_STORE [[LSHR1]](s32), [[PTR_ADD1]](p1) :: (store (s8) into %ir.ptr + 6, align 2, basealign 4, addrspace 1)
+  ; UNPACKED-NEXT:   SI_RETURN
+  store i56 %arg, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
+define void @store_i65(ptr addrspace(1) %ptr, i65 %arg) #0 {
+  ; UNPACKED-LABEL: name: store_i65
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; UNPACKED-NEXT: {{  $}}
+  ; UNPACKED-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; UNPACKED-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; UNPACKED-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; UNPACKED-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; UNPACKED-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; UNPACKED-NEXT:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; UNPACKED-NEXT:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[DEF]](s32)
+  ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+  ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+  ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[MV1]], [[C]]
+  ; UNPACKED-NEXT:   [[AND1:%[0-9]+]]:_(s64) = G_AND [[MV2]], [[C1]]
+  ; UNPACKED-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; UNPACKED-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[MV]], [[C2]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[AND]](s64), [[MV]](p1) :: (store (s64) into %ir.ptr, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[TRUNC]](s32), [[PTR_ADD]](p1) :: (store (s8) into %ir.ptr + 8, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   SI_RETURN
+  store i65 %arg, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
+define void @store_i95(ptr addrspace(1) %ptr, i95 %arg) #0 {
+  ; UNPACKED-LABEL: name: store_i95
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; UNPACKED-NEXT: {{  $}}
+  ; UNPACKED-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; UNPACKED-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; UNPACKED-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; UNPACKED-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; UNPACKED-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; UNPACKED-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; UNPACKED-NEXT:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; UNPACKED-NEXT:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[DEF]](s32)
+  ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+  ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 2147483647
+  ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[MV1]], [[C]]
+  ; UNPACKED-NEXT:   [[AND1:%[0-9]+]]:_(s64) = G_AND [[MV2]], [[C1]]
+  ; UNPACKED-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; UNPACKED-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[MV]], [[C2]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[AND]](s64), [[MV]](p1) :: (store (s64) into %ir.ptr, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[TRUNC]](s32), [[PTR_ADD]](p1) :: (store (s32) into %ir.ptr + 8, addrspace 1)
+  ; UNPACKED-NEXT:   SI_RETURN
+  store i95 %arg, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
+define void @store_i96(ptr addrspace(1) %ptr, i96 %arg) #0 {
+  ; UNPACKED-LABEL: name: store_i96
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; UNPACKED-NEXT: {{  $}}
+  ; UNPACKED-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; UNPACKED-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; UNPACKED-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; UNPACKED-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; UNPACKED-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; UNPACKED-NEXT:   [[MV1:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+  ; UNPACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<3 x s32>) = G_BITCAST [[MV1]](s96)
+  ; UNPACKED-NEXT:   G_STORE [[BITCAST]](<3 x s32>), [[MV]](p1) :: (store (<3 x s32>) into %ir.ptr, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   SI_RETURN
+  store i96 %arg, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
+define void @store_i97(ptr addrspace(1) %ptr, i97 %arg) #0 {
+  ; UNPACKED-LABEL: name: store_i97
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; UNPACKED-NEXT: {{  $}}
+  ; UNPACKED-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; UNPACKED-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; UNPACKED-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; UNPACKED-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; UNPACKED-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+  ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8589934591
+  ; UNPACKED-NEXT:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; UNPACKED-NEXT:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[MV1]], [[C]]
+  ; UNPACKED-NEXT:   [[AND1:%[0-9]+]]:_(s64) = G_AND [[MV2]], [[C1]]
+  ; UNPACKED-NEXT:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; UNPACKED-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[MV]], [[C2]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[AND]](s64), [[MV]](p1) :: (store (s64) into %ir.ptr, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+  ; UNPACKED-NEXT:   [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[AND1]], [[C3]](s32)
+  ; UNPACKED-NEXT:   [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; UNPACKED-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD]], [[C4]](s64)
+  ; UNPACKED-NEXT:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[AND1]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[TRUNC]](s32), [[PTR_ADD]](p1) :: (store (s32) into %ir.ptr + 8, addrspace 1)
+  ; UNPACKED-NEXT:   [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[LSHR]](s64)
+  ; UNPACKED-NEXT:   G_STORE [[TRUNC1]](s32), [[PTR_ADD1]](p1) :: (store (s8) into %ir.ptr + 12, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   SI_RETURN
+  store i97 %arg, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
+define void @store_i127(ptr addrspace(1) %ptr, i127 %arg) #0 {
+  ; UNPACKED-LABEL: name: store_i127
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; UNPACKED-NEXT: {{  $}}
+  ; UNPACKED-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; UNPACKED-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; UNPACKED-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; UNPACKED-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; UNPACKED-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; UNPACKED-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; UNPACKED-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; UNPACKED-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -1
+  ; UNPACKED-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 9223372036854775807
+  ; UNPACKED-NEXT:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; UNPACKED-NEXT:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; UNPACKED-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[MV1]], [[C]]
+  ; UNPACKED-NEXT:   [[AND1:%[0-9]+]]:_(s64) = G_AND [[MV2]], [[C1]]
+  ; UNPACKED-NEXT:   [[MV3:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[AND]](s64), [[AND1]](s64)
+  ; UNPACKED-NEXT:   [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[MV3]](s128)
+  ; UNPACKED-NEXT:   G_STORE [[BITCAST]](<4 x s32>), [[MV]](p1) :: (store (<4 x s32>) into %ir.ptr, align 4, addrspace 1)
+  ; UNPACKED-NEXT:   SI_RETURN
+  store i127 %arg, ptr addrspace(1) %ptr, align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/trunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/trunc.ll
index c19992325ea67..b6c8f21143ba1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/trunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/trunc.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=bonaire < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=bonaire < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GFX8 %s
 
 define i16 @v_trunc_i32_to_i16(i32 %src) {
 ; GFX7-LABEL: v_trunc_i32_to_i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
index 1a6d26142208f..4de10788a6bd7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udiv.i64.ll
@@ -187,11 +187,9 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-LABEL: s_udiv_i64:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_or_b64 s[4:5], s[0:1], s[2:3]
-; CHECK-NEXT:    s_mov_b32 s6, 0
-; CHECK-NEXT:    s_mov_b32 s7, -1
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:    v_cmp_ne_u64_e64 vcc, s[4:5], 0
-; CHECK-NEXT:    s_mov_b32 s6, 1
+; CHECK-NEXT:    s_mov_b32 s4, 1
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s2
 ; CHECK-NEXT:    s_cbranch_vccz .LBB1_2
 ; CHECK-NEXT:  ; %bb.1:
@@ -199,7 +197,6 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, s3
 ; CHECK-NEXT:    s_sub_u32 s4, 0, s2
 ; CHECK-NEXT:    v_mov_b32_e32 v3, s1
-; CHECK-NEXT:    s_mov_b32 s6, 0
 ; CHECK-NEXT:    v_madmk_f32 v1, v1, 0x4f800000, v2
 ; CHECK-NEXT:    s_subb_u32 s5, 0, s3
 ; CHECK-NEXT:    v_rcp_iflag_f32_e32 v1, v1
@@ -318,11 +315,12 @@ define amdgpu_ps i64 @s_udiv_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v9, v5, vcc
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v3
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:    s_branch .LBB1_3
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; CHECK-NEXT:  .LBB1_3: ; %Flow
-; CHECK-NEXT:    s_xor_b32 s1, s6, 1
+; CHECK-NEXT:    s_xor_b32 s1, s4, 1
 ; CHECK-NEXT:    s_cmp_lg_u32 s1, 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB1_5
 ; CHECK-NEXT:  ; %bb.4:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index 2a1bf4bf068f0..a41ec8e7ce3ea 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -184,18 +184,15 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-LABEL: s_urem_i64:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_or_b64 s[4:5], s[0:1], s[2:3]
-; CHECK-NEXT:    s_mov_b32 s6, 0
-; CHECK-NEXT:    s_mov_b32 s7, -1
-; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
+; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:    v_cmp_ne_u64_e64 vcc, s[4:5], 0
-; CHECK-NEXT:    s_mov_b32 s6, 1
+; CHECK-NEXT:    s_mov_b32 s4, 1
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v2, s2
 ; CHECK-NEXT:    s_cbranch_vccz .LBB1_2
 ; CHECK-NEXT:  ; %bb.1:
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s3
 ; CHECK-NEXT:    v_cvt_f32_u32_e32 v1, s3
 ; CHECK-NEXT:    s_sub_u32 s4, 0, s2
-; CHECK-NEXT:    s_mov_b32 s6, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v3, s1
 ; CHECK-NEXT:    v_madmk_f32 v1, v1, 0x4f800000, v2
 ; CHECK-NEXT:    s_subb_u32 s5, 0, s3
@@ -314,11 +311,12 @@ define amdgpu_ps i64 @s_urem_i64(i64 inreg %num, i64 inreg %den) {
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v3, v6, vcc
 ; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v1
 ; CHECK-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:    s_branch .LBB1_3
 ; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; CHECK-NEXT:  .LBB1_3: ; %Flow
-; CHECK-NEXT:    s_xor_b32 s1, s6, 1
+; CHECK-NEXT:    s_xor_b32 s1, s4, 1
 ; CHECK-NEXT:    s_cmp_lg_u32 s1, 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB1_5
 ; CHECK-NEXT:  ; %bb.4:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/v_bfe_i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/v_bfe_i32.ll
index 5408ad0747b74..abfb4fea69923 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/v_bfe_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/v_bfe_i32.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel < %s | FileCheck --check-prefix=PREGFX9 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -mcpu=hawaii < %s | FileCheck --check-prefix=PREGFX9 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -mcpu=fiji < %s | FileCheck --check-prefix=PREGFX9 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -mcpu=gfx90a < %s | FileCheck --check-prefix=PREGFX9 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -mcpu=gfx1030 < %s | FileCheck --check-prefix=GFX10PLUS %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -mcpu=gfx1100 < %s | FileCheck --check-prefix=GFX10PLUS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -new-reg-bank-select < %s | FileCheck --check-prefix=PREGFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -new-reg-bank-select -mcpu=hawaii < %s | FileCheck --check-prefix=PREGFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -new-reg-bank-select -mcpu=fiji < %s | FileCheck --check-prefix=PREGFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -new-reg-bank-select -mcpu=gfx90a < %s | FileCheck --check-prefix=PREGFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -new-reg-bank-select -mcpu=gfx1030 < %s | FileCheck --check-prefix=GFX10PLUS %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa --global-isel -new-reg-bank-select -mcpu=gfx1100 < %s | FileCheck --check-prefix=GFX10PLUS %s
 
 define i32 @check_v_bfe(i16 %a) {
 ; PREGFX9-LABEL: check_v_bfe:
diff --git a/llvm/test/CodeGen/AMDGPU/acc-ldst.ll b/llvm/test/CodeGen/AMDGPU/acc-ldst.ll
index be4e369545aa0..002ccd6060681 100644
--- a/llvm/test/CodeGen/AMDGPU/acc-ldst.ll
+++ b/llvm/test/CodeGen/AMDGPU/acc-ldst.ll
@@ -203,7 +203,7 @@ define amdgpu_kernel void @test_atomic_mfma_4xi32_atomic_store(ptr addrspace(1)
 bb:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %tid
-  %in.1 = atomicrmw volatile sub ptr addrspace(1) %gep, i32 1 syncscope("agent") seq_cst
+  %in.1 = atomicrmw volatile sub ptr addrspace(1) %gep, i32 1 syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   %tmp0 = insertelement <4 x i32> poison, i32 %in.1, i32 0
   %tmp1 = insertelement <4 x i32> %tmp0, i32 0, i32 1
   %tmp2 = insertelement <4 x i32> %tmp1, i32 0, i32 2
@@ -229,7 +229,7 @@ define amdgpu_kernel void @test_atomic_mfma_4xi32_atomic64_store(ptr addrspace(1
 bb:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %tid
-  %in.1 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 1 syncscope("agent") seq_cst
+  %in.1 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 1 syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   %tmp0 = insertelement <2 x i64> poison, i64 %in.1, i32 0
   %tmp1 = insertelement <2 x i64> %tmp0, i64 0, i32 1
   %tmp2 = bitcast <2 x i64> %tmp1 to <4 x i32>
@@ -319,3 +319,5 @@ exit:
 }
 
 attributes #0 = { "amdgpu-flat-work-group-size"="1,256" }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/add_i64.ll b/llvm/test/CodeGen/AMDGPU/add_i64.ll
index eedd56de8c7c2..a8560e8498c4d 100644
--- a/llvm/test/CodeGen/AMDGPU/add_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_i64.ll
@@ -1,12 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI %s
 
-
-declare i32 @llvm.amdgcn.workitem.id.x() readnone
-
-; SI-LABEL: {{^}}test_i64_vreg:
-; SI: v_add_i32
-; SI: v_addc_u32
 define amdgpu_kernel void @test_i64_vreg(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %inA, ptr addrspace(1) noalias %inB) {
+; SI-LABEL: test_i64_vreg:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s14, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_mov_b32 s15, s11
+; SI-NEXT:    s_mov_b64 s[6:7], s[14:15]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[12:13], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[12:15], 0 addr64
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_mov_b32 s8, s0
+; SI-NEXT:    s_mov_b32 s9, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, v2, v0
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, v3, v1, vcc
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() readnone
   %a_ptr = getelementptr i64, ptr addrspace(1) %inA, i32 %tid
   %b_ptr = getelementptr i64, ptr addrspace(1) %inB, i32 %tid
@@ -18,10 +35,22 @@ define amdgpu_kernel void @test_i64_vreg(ptr addrspace(1) noalias %out, ptr addr
 }
 
 ; Check that the SGPR add operand is correctly moved to a VGPR.
-; SI-LABEL: {{^}}sgpr_operand:
-; SI: s_add_u32
-; SI: s_addc_u32
 define amdgpu_kernel void @sgpr_operand(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %in_bar, i64 %a) {
+; SI-LABEL: sgpr_operand:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xf
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dwordx2 s[6:7], s[2:3], 0x0
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_add_u32 s4, s6, s4
+; SI-NEXT:    s_addc_u32 s5, s7, s5
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
   %foo = load i64, ptr addrspace(1) %in, align 8
   %result = add i64 %foo, %a
   store i64 %result, ptr addrspace(1) %out
@@ -30,35 +59,76 @@ define amdgpu_kernel void @sgpr_operand(ptr addrspace(1) noalias %out, ptr addrs
 
 ; Swap the arguments. Check that the SGPR -> VGPR copy works with the
 ; SGPR as other operand.
-;
-; SI-LABEL: {{^}}sgpr_operand_reversed:
-; SI: s_add_u32
-; SI: s_addc_u32
 define amdgpu_kernel void @sgpr_operand_reversed(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in, i64 %a) {
+; SI-LABEL: sgpr_operand_reversed:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dwordx2 s[6:7], s[2:3], 0x0
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_add_u32 s4, s4, s6
+; SI-NEXT:    s_addc_u32 s5, s5, s7
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
   %foo = load i64, ptr addrspace(1) %in, align 8
   %result = add i64 %a, %foo
   store i64 %result, ptr addrspace(1) %out
   ret void
 }
 
-
-; SI-LABEL: {{^}}test_v2i64_sreg:
-; SI: s_add_u32
-; SI: s_addc_u32
-; SI: s_add_u32
-; SI: s_addc_u32
 define amdgpu_kernel void @test_v2i64_sreg(ptr addrspace(1) noalias %out, <2 x i64> %a, <2 x i64> %b) {
+; SI-LABEL: test_v2i64_sreg:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0xd
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_add_u32 s4, s10, s14
+; SI-NEXT:    s_addc_u32 s5, s11, s15
+; SI-NEXT:    s_add_u32 s6, s8, s12
+; SI-NEXT:    s_addc_u32 s7, s9, s13
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v2, s4
+; SI-NEXT:    v_mov_b32_e32 v3, s5
+; SI-NEXT:    v_mov_b32_e32 v0, s6
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    s_endpgm
   %result = add <2 x i64> %a, %b
   store <2 x i64> %result, ptr addrspace(1) %out
   ret void
 }
 
-; SI-LABEL: {{^}}test_v2i64_vreg:
-; SI: v_add_i32
-; SI: v_addc_u32
-; SI: v_add_i32
-; SI: v_addc_u32
 define amdgpu_kernel void @test_v2i64_vreg(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %inA, ptr addrspace(1) noalias %inB) {
+; SI-LABEL: test_v2i64_vreg:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s14, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 4, v0
+; SI-NEXT:    v_mov_b32_e32 v5, 0
+; SI-NEXT:    s_mov_b32 s15, s11
+; SI-NEXT:    s_mov_b64 s[6:7], s[14:15]
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[12:13], s[2:3]
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[4:5], s[12:15], 0 addr64
+; SI-NEXT:    buffer_load_dwordx4 v[4:7], v[4:5], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_mov_b32 s8, s0
+; SI-NEXT:    s_mov_b32 s9, s1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v2, vcc, v2, v6
+; SI-NEXT:    v_addc_u32_e32 v3, vcc, v3, v7, vcc
+; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v4
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v5, vcc
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
+; SI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() readnone
   %a_ptr = getelementptr <2 x i64>, ptr addrspace(1) %inA, i32 %tid
   %b_ptr = getelementptr <2 x i64>, ptr addrspace(1) %inB, i32 %tid
@@ -69,14 +139,19 @@ define amdgpu_kernel void @test_v2i64_vreg(ptr addrspace(1) noalias %out, ptr ad
   ret void
 }
 
-; SI-LABEL: {{^}}trunc_i64_add_to_i32:
-; SI: s_load_dword s[[SREG0:[0-9]+]]
-; SI: s_load_dword s[[SREG1:[0-9]+]]
-; SI: s_add_i32 [[SRESULT:s[0-9]+]], s[[SREG1]], s[[SREG0]]
-; SI-NOT: addc
-; SI: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[SRESULT]]
-; SI: buffer_store_dword [[VRESULT]],
 define amdgpu_kernel void @trunc_i64_add_to_i32(ptr addrspace(1) %out, i32, i64 %a, i32, i64 %b) {
+; SI-LABEL: trunc_i64_add_to_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s2, s[4:5], 0xd
+; SI-NEXT:    s_load_dword s6, s[4:5], 0x11
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_add_i32 s4, s6, s2
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
   %add = add i64 %b, %a
   %trunc = trunc i64 %add to i32
   store i32 %trunc, ptr addrspace(1) %out, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/allow-check.ll b/llvm/test/CodeGen/AMDGPU/allow-check.ll
index d4f5621ce26a4..162a8bcfc9ff4 100644
--- a/llvm/test/CodeGen/AMDGPU/allow-check.ll
+++ b/llvm/test/CodeGen/AMDGPU/allow-check.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -global-isel=0 -fast-isel=0 | FileCheck %s
-; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -global-isel=1 -fast-isel=0 | FileCheck %s
+; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -global-isel=1 -new-reg-bank-select -fast-isel=0 | FileCheck %s
 ; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -global-isel=0 -fast-isel=1 | FileCheck %s
 
 define i1 @test_runtime() local_unnamed_addr {
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-call-whole-wave.ll b/llvm/test/CodeGen/AMDGPU/amdgcn-call-whole-wave.ll
new file mode 100644
index 0000000000000..eac0767c88d80
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-call-whole-wave.ll
@@ -0,0 +1,174 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn--amdpal -mcpu=gfx1200 < %s | FileCheck %s --check-prefix=DAGISEL
+; RUN: llc -global-isel=1 -mtriple=amdgcn--amdpal -mcpu=gfx1200 < %s | FileCheck %s --check-prefix=GISEL
+
+declare amdgpu_gfx_whole_wave i32 @good_callee(i1 %active, i32 %x, i32 %y, i32 inreg %c)
+
+define amdgpu_gfx void @basic_test(i32 %x, i32 inreg %c, ptr addrspace(1) %ptr) {
+; DAGISEL-LABEL: basic_test:
+; DAGISEL:       ; %bb.0:
+; DAGISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; DAGISEL-NEXT:    s_wait_expcnt 0x0
+; DAGISEL-NEXT:    s_wait_samplecnt 0x0
+; DAGISEL-NEXT:    s_wait_bvhcnt 0x0
+; DAGISEL-NEXT:    s_wait_kmcnt 0x0
+; DAGISEL-NEXT:    s_mov_b32 s0, s33
+; DAGISEL-NEXT:    s_mov_b32 s33, s32
+; DAGISEL-NEXT:    s_or_saveexec_b32 s1, -1
+; DAGISEL-NEXT:    scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_mov_b32 exec_lo, s1
+; DAGISEL-NEXT:    v_writelane_b32 v42, s0, 2
+; DAGISEL-NEXT:    s_clause 0x1
+; DAGISEL-NEXT:    scratch_store_b32 off, v40, s33 offset:4
+; DAGISEL-NEXT:    scratch_store_b32 off, v41, s33
+; DAGISEL-NEXT:    v_dual_mov_b32 v41, v2 :: v_dual_mov_b32 v40, v1
+; DAGISEL-NEXT:    v_add_nc_u32_e32 v1, 13, v0
+; DAGISEL-NEXT:    v_writelane_b32 v42, s30, 0
+; DAGISEL-NEXT:    s_mov_b32 s1, good_callee@abs32@hi
+; DAGISEL-NEXT:    s_mov_b32 s0, good_callee@abs32@lo
+; DAGISEL-NEXT:    s_add_co_i32 s32, s32, 16
+; DAGISEL-NEXT:    v_writelane_b32 v42, s31, 1
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; DAGISEL-NEXT:    global_store_b32 v[40:41], v0, off
+; DAGISEL-NEXT:    s_clause 0x1
+; DAGISEL-NEXT:    scratch_load_b32 v41, off, s33
+; DAGISEL-NEXT:    scratch_load_b32 v40, off, s33 offset:4
+; DAGISEL-NEXT:    v_readlane_b32 s31, v42, 1
+; DAGISEL-NEXT:    v_readlane_b32 s30, v42, 0
+; DAGISEL-NEXT:    s_mov_b32 s32, s33
+; DAGISEL-NEXT:    v_readlane_b32 s0, v42, 2
+; DAGISEL-NEXT:    s_or_saveexec_b32 s1, -1
+; DAGISEL-NEXT:    scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_mov_b32 exec_lo, s1
+; DAGISEL-NEXT:    s_mov_b32 s33, s0
+; DAGISEL-NEXT:    s_wait_loadcnt 0x0
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: basic_test:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT:    s_wait_expcnt 0x0
+; GISEL-NEXT:    s_wait_samplecnt 0x0
+; GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GISEL-NEXT:    s_wait_kmcnt 0x0
+; GISEL-NEXT:    s_mov_b32 s0, s33
+; GISEL-NEXT:    s_mov_b32 s33, s32
+; GISEL-NEXT:    s_or_saveexec_b32 s1, -1
+; GISEL-NEXT:    scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_mov_b32 exec_lo, s1
+; GISEL-NEXT:    v_writelane_b32 v42, s0, 2
+; GISEL-NEXT:    s_clause 0x1
+; GISEL-NEXT:    scratch_store_b32 off, v40, s33 offset:4
+; GISEL-NEXT:    scratch_store_b32 off, v41, s33
+; GISEL-NEXT:    v_dual_mov_b32 v40, v1 :: v_dual_mov_b32 v41, v2
+; GISEL-NEXT:    v_add_nc_u32_e32 v1, 13, v0
+; GISEL-NEXT:    v_writelane_b32 v42, s30, 0
+; GISEL-NEXT:    s_mov_b32 s0, good_callee@abs32@lo
+; GISEL-NEXT:    s_mov_b32 s1, good_callee@abs32@hi
+; GISEL-NEXT:    s_add_co_i32 s32, s32, 16
+; GISEL-NEXT:    v_writelane_b32 v42, s31, 1
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GISEL-NEXT:    global_store_b32 v[40:41], v0, off
+; GISEL-NEXT:    s_clause 0x1
+; GISEL-NEXT:    scratch_load_b32 v41, off, s33
+; GISEL-NEXT:    scratch_load_b32 v40, off, s33 offset:4
+; GISEL-NEXT:    v_readlane_b32 s31, v42, 1
+; GISEL-NEXT:    v_readlane_b32 s30, v42, 0
+; GISEL-NEXT:    s_mov_b32 s32, s33
+; GISEL-NEXT:    v_readlane_b32 s0, v42, 2
+; GISEL-NEXT:    s_or_saveexec_b32 s1, -1
+; GISEL-NEXT:    scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_mov_b32 exec_lo, s1
+; GISEL-NEXT:    s_mov_b32 s33, s0
+; GISEL-NEXT:    s_wait_loadcnt 0x0
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %y = add i32 %x, 13
+  %ret = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @good_callee, i32 %x, i32 %y, i32 inreg %c)
+  store i32 %ret, ptr addrspace(1) %ptr
+  ret void
+}
+
+declare amdgpu_gfx_whole_wave void @void_callee(i1 %active, i32 %x)
+
+define amdgpu_gfx void @ret_void(i32 %x) {
+; DAGISEL-LABEL: ret_void:
+; DAGISEL:       ; %bb.0:
+; DAGISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; DAGISEL-NEXT:    s_wait_expcnt 0x0
+; DAGISEL-NEXT:    s_wait_samplecnt 0x0
+; DAGISEL-NEXT:    s_wait_bvhcnt 0x0
+; DAGISEL-NEXT:    s_wait_kmcnt 0x0
+; DAGISEL-NEXT:    s_mov_b32 s0, s33
+; DAGISEL-NEXT:    s_mov_b32 s33, s32
+; DAGISEL-NEXT:    s_or_saveexec_b32 s1, -1
+; DAGISEL-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_mov_b32 exec_lo, s1
+; DAGISEL-NEXT:    v_writelane_b32 v40, s0, 2
+; DAGISEL-NEXT:    s_mov_b32 s1, void_callee@abs32@hi
+; DAGISEL-NEXT:    s_mov_b32 s0, void_callee@abs32@lo
+; DAGISEL-NEXT:    s_add_co_i32 s32, s32, 16
+; DAGISEL-NEXT:    v_writelane_b32 v40, s30, 0
+; DAGISEL-NEXT:    v_writelane_b32 v40, s31, 1
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; DAGISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; DAGISEL-NEXT:    v_readlane_b32 s31, v40, 1
+; DAGISEL-NEXT:    v_readlane_b32 s30, v40, 0
+; DAGISEL-NEXT:    s_mov_b32 s32, s33
+; DAGISEL-NEXT:    v_readlane_b32 s0, v40, 2
+; DAGISEL-NEXT:    s_or_saveexec_b32 s1, -1
+; DAGISEL-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_mov_b32 exec_lo, s1
+; DAGISEL-NEXT:    s_mov_b32 s33, s0
+; DAGISEL-NEXT:    s_wait_loadcnt 0x0
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: ret_void:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT:    s_wait_expcnt 0x0
+; GISEL-NEXT:    s_wait_samplecnt 0x0
+; GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GISEL-NEXT:    s_wait_kmcnt 0x0
+; GISEL-NEXT:    s_mov_b32 s0, s33
+; GISEL-NEXT:    s_mov_b32 s33, s32
+; GISEL-NEXT:    s_or_saveexec_b32 s1, -1
+; GISEL-NEXT:    scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_mov_b32 exec_lo, s1
+; GISEL-NEXT:    v_writelane_b32 v40, s0, 2
+; GISEL-NEXT:    s_mov_b32 s0, void_callee@abs32@lo
+; GISEL-NEXT:    s_mov_b32 s1, void_callee@abs32@hi
+; GISEL-NEXT:    s_add_co_i32 s32, s32, 16
+; GISEL-NEXT:    v_writelane_b32 v40, s30, 0
+; GISEL-NEXT:    v_writelane_b32 v40, s31, 1
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-NEXT:    v_readlane_b32 s31, v40, 1
+; GISEL-NEXT:    v_readlane_b32 s30, v40, 0
+; GISEL-NEXT:    s_mov_b32 s32, s33
+; GISEL-NEXT:    v_readlane_b32 s0, v40, 2
+; GISEL-NEXT:    s_or_saveexec_b32 s1, -1
+; GISEL-NEXT:    scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_mov_b32 exec_lo, s1
+; GISEL-NEXT:    s_mov_b32 s33, s0
+; GISEL-NEXT:    s_wait_loadcnt 0x0
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+  call void(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @void_callee, i32 %x)
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn-cs-chain-intrinsic-dyn-vgpr-w32.ll b/llvm/test/CodeGen/AMDGPU/amdgcn-cs-chain-intrinsic-dyn-vgpr-w32.ll
index 77c9b9813571a..3e80a58bda4a0 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn-cs-chain-intrinsic-dyn-vgpr-w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn-cs-chain-intrinsic-dyn-vgpr-w32.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
-; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GISEL-GFX12 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GISEL-GFX12 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=DAGISEL-GFX12 %s
 
 declare amdgpu_cs_chain void @callee(<3 x i32> inreg, { i32, ptr addrspace(5), i32, i32 })
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 0d5f538215f18..d03d6a8940b2f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -6309,64 +6309,64 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr161_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
@@ -6394,50 +6394,50 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 24, v32
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v27
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v26
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v24
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v21
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v20
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v9
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[1:2]
@@ -6498,50 +6498,50 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 24, v32
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v27
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v26
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v24
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v21
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v20
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v9
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB12_4: ; %end
@@ -6549,307 +6549,266 @@ define <128 x i8> @bitcast_v32i32_to_v128i8(<32 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v162.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v34.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v1.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v39, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v2.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v65.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v161.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v1.h, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v160.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v39, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v3.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff, v66
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v55, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v65.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v150.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v150.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v66, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v55, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v65
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v4.l, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v148.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v55, v39
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v39, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v149.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v39, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v5.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v148.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v147.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v147.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.h, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v65
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v39, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v6.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v54.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v39, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v7.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v145.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v55, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v8.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v55, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v135.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v54, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v144.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v39, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v135.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v39, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v9.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v134.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v8.h, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v133.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v39, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v10.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v132.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v39, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v11.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v131.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v130.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v54, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v133.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v132.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v39, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v12.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v129.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v51.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v53, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v10.l, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v53, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v130.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v11.h, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v39, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v13.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v119.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v39, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v14.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v118.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v50.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v39, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v15.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v116.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v129.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v39, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v16.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v115.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v39, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v17.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v114.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v113.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v53
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff, v53
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v119.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v118.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v51, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v14.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v39, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v18.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v112.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v48.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v39, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v19.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v39, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v20.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v51, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v116.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v115.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v39, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v21.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v99.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v50, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v50, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v113.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v17.h, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v39, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v22.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v39, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v23.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v96.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v39, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v24.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v36.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v49, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v19.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v112.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v39, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v25.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v39, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v26.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v84.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v35.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v50
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v18.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v50
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v49, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v102.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v19.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v101.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v48, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v49
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v20.h, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v39, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v27.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v82.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v39, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v28.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v81.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v39, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v29.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v71.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v48, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v22.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v99.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v49
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v21.l, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v98.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v39, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v30.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v33.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v38, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v22.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v97.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v38, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v23.h, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v39, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v31.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v33.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v68.l
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v37, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v25.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v87.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v24.l, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v37, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v85.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v25.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v36, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v26.h, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v36, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v28.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v82.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v27.l, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v34.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v35, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v28.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.l, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v71.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v29.h, v30.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.l, v31.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v69.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v31.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v30.l, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v31.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v32.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v39
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v31.l, v31.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v32.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v32.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v34
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v32.l, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v33, v39
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v39, v31
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v32.l, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v32.h, v33.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v39.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v39, v32
 ; GFX11-TRUE16-NEXT:    s_clause 0x5
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[9:12], off offset:32
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[13:16], off offset:48
@@ -15413,63 +15372,63 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:384
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:380
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:376
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:372
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v49, off, s32 offset:368
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:364
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:360
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:356
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v50, off, s32 offset:352
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:348
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:344
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:372
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v50, off, s32 offset:368
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:364
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:360
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:356
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v51, off, s32 offset:352
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:348
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:344
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:340
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v52, off, s32 offset:336
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:332
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:328
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:324
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:324
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v53, off, s32 offset:320
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:316
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:312
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:308
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v54, off, s32 offset:304
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:300
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:296
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:292
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v55, off, s32 offset:288
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:284
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:280
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:276
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:272
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:268
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:264
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:292
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:288
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:284
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:280
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:276
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:272
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:268
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:264
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v39, off, s32 offset:260
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:256
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:256
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:252
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:248
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v48, off, s32 offset:244
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:240
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:240
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:236
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:232
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v51, off, s32 offset:228
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:224
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:220
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:216
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v103, off, s32 offset:388
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:232
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v49, off, s32 offset:228
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:224
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:220
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:216
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v114, off, s32 offset:388
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:16
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v83, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32 offset:64
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v97, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v98, off, s32 offset:80
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v99, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v100, off, s32 offset:96
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v101, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v102, off, s32 offset:112
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:120
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v160, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:136
@@ -15483,144 +15442,143 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v164, off, s32 offset:192
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:200
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v165, off, s32 offset:208
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:212
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:204
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:196
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:188
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:180
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:172
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:164
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:156
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v55, off, s32 offset:212
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:204
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:196
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:188
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:180
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:172
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:100
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v97, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v98, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v100, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v102, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v112, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v99, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v101, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v113, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v116, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v115, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v117.l, v30.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.h, v28.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.h, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.h, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.h, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.h, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.h, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.h, v12.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v147.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v148.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v150.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v151.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v149.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.l, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v149.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v146.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.l, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.l, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v134.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.l, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v145.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.h, v0.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v144.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.l, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v131.h, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(54)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v103
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v114
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(53)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v130.l, 8, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.l, 8, v81.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(52)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v130.h, 8, v82.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.l, 8, v82.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(51)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.l, 8, v82.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.h, 8, v83.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(50)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.l, 8, v83.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.l, 8, v83.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(49)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v118.l, 8, v84.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.h, 8, v85.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(48)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v119.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v117.h, 8, v85.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(47)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.l, 8, v86.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v118.l, 8, v87.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(46)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v117.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.l, 8, v87.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(45)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.h, 8, v96.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.h, 8, v97.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(44)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v115.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.l, 8, v97.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(43)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.l, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.h, 8, v98.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(42)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.h, 8, v99.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v112.h, 8, v99.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(41)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v103.l, 8, v99.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.l, 8, v100.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(40)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v103.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v102.l, 8, v101.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(39)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.l, 8, v101.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v102.h, 8, v102.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(38)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v100.h, 8, v160.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(37)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.l, 8, v160.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.l, 8, v160.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(36)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v98.h, 8, v161.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(35)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.l, 8, v161.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.l, 8, v161.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(34)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v98.l, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.l, 8, v162.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.l, 8, v162.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.h, 8, v162.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(32)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v96.h, 8, v163.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.l, 8, v163.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.l, 8, v163.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.h, 8, v163.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(30)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v86.h, 8, v164.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.l, 8, v164.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.h, 8, v164.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.h, 8, v164.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(28)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v84.h, 8, v165.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.l, 8, v165.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.l, 8, v165.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v80.l, 8, v70.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v80.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.l, 8, v68.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.l, 8, v66.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.l, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.h, 8, v165.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v81.h, 8, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.l, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v71.l, 8, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v71.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.l, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.h, 8, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v67.h, 8, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.l, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.l, 8, v64.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.l, 8, v55.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v65.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v54.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.h, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v52.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.l, 8, v49.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.l, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.h, 8, v54.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v53.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v52.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v51.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.l, 8, v50.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v31.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v31.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GFX11-TRUE16-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -15634,746 +15592,660 @@ define <32 x i32> @bitcast_v128i8_to_v32i32(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB14_3: ; %cmp.false
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v148.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v150.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v146.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v151.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v150.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v147.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v0.h, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v149.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v144.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v133.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v148.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v145.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v1.h, v147.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v149.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v146.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v146.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v0.l, v151.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v151.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v0.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v150.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v3.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v145.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v144.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v149, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v1.h, v150.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v132.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v131.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v148.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v130.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v149, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v2.l, v148.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v2.l, v149.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v134.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v2.l, v146.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v131.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v135.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v3.l, v145.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v128.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v134.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v145.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v130.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v149, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v3.l, v147.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v147.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v3.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v135.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v119.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v119.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v149, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v4.l, v144.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v4.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v133.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v117.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v116.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v4.l, v135.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v129.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v134.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v5.l, v133.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v119.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v132.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v6.l, v132.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v130.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v112.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v7.l, v130.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v115.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v129.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v102.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v8.l, v128.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v112.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.h, v119.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v100.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v9.l, v118.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v102.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v9.h, v117.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v97.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v10.l, v116.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v100.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v10.h, v115.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v87.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v11.l, v114.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v98.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v11.h, v113.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v85.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v12.l, v113.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v12.h, v103.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v83.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v13.l, v103.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v86.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v101.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v16, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v14.l, v101.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v14.h, v99.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v71.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v15.l, v99.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v81.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v15.h, v98.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v69.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v18, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v16.l, v97.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v71.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v16.h, v96.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v67.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v19, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v17.l, v87.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v115.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v115.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v149, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v5.l, v135.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v5.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v132.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v129.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v113.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v112.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v149, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v6.l, v133.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v6.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v103.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v128.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v103.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v101.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v149, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v7.l, v131.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v7.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v118.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v100.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v99.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v98.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v149, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v8.l, v129.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v8.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v116.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v114.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v96.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v96.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v149, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v9.l, v128.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v9.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v86.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v113.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v86.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v84.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v149, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v10.l, v117.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v10.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v102.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v84.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v82.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v81.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v149, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v11.l, v116.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v11.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v101.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v99.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v80.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v80.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v149, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v12.l, v114.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v12.l, v149.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v69.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v17.h, v86.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v64.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v20, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v18.l, v85.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v67.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v18.h, v84.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v51.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v21, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v19.l, v83.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v97.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v69.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v68.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v149, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v13.l, v112.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v13.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v87.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v67.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v65.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v19.h, v82.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v48.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v22, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v20.l, v82.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v20.h, v80.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v39.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v23, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v21.l, v80.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v21.h, v70.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v24, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v22.l, v70.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v22.h, v68.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v25, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v23.l, v68.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v38.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v23.h, v66.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v26, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v24.l, v66.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v24.h, v65.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v27, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v28
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v25.l, v64.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v25.h, v55.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v28, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v26.l, v55.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v26.h, v54.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v34.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v29, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v27.l, v54.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_hi16
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v65.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v149, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v14.l, v102.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v14.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v85.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v83.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v50.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v149, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v15.l, v100.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v15.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v49.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v82.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v149, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v16.l, v98.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v16.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v71.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v149, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v17.l, v97.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v17.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v70.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v68.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v38.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v149, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v18.l, v87.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v18.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v66.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v149, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v19.l, v85.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v19.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v35.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v149, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v20.l, v83.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v20.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v55.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v54.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v149, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v21.l, v81.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v21.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v33.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v149, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v22.l, v71.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v22.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v32.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v29, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v28.h, v53.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v34.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v28.l, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v33.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v28.h, v52.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v33.l, v29.h, v51.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v34
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v29.l, v50.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v33
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v149, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v23.l, v70.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v23.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v51.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v149, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v24.l, v67.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v24.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v149, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v25.l, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v25.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v149, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v26.l, v64.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v26.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v149, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v27.l, v54.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v27.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v149, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v28.l, v53.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v28.l, v149.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v149, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v29.l, v52.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v29.l, v149.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v30.l, v50.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v32.l, v49.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v149, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v30.l, v51.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v30.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v149, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v31.l, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v31.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v31
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v149, v31
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB14_2
 ; GFX11-TRUE16-NEXT:  .LBB14_4: ; %cmp.true
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v151.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v148.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v150.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v146.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v147.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v149.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v149.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v146.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v144.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v151.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v150.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v144.h, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v149.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v151.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v151.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v150.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v150.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v145.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v144.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v134.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v31, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v134.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v148.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v148.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v149.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v147.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v134.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v133.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v146.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v148.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v145.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v31, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v31.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v147.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v147.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v132.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v131.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v145.h, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v131.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v31, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v130.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v130.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v144.h, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v145.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v128.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v135.h, v4.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v135.l, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v129.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v32.l, v32.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v134.l, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v118.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v133.l, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v119.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v31, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v135.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v135.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v119.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v119.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v31, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v118.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v117.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v133.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v133.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v132.h, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v31, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v131.h, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v132.l, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v116.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v132.l, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v117.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v130.h, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v114.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v130.l, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v115.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v115.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v115.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v31, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v113.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v112.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v129.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v129.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v129.l, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, v112.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v128.l, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v112.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v119.h, v9.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v102.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v118.l, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v102.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v31, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v128.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v128.h, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v103.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v103.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v31, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v101.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v100.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v117.h, v9.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v118.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v117.h, v10.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, v100.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v116.l, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, v100.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, 0x300, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v115.h, v11.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v97.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v114.h, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v98.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v31, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v116.l, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v116.h, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v99.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v98.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v31, v13
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v96.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v96.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v114.l, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v114.h, v11.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v113.h, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, v87.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v113.l, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, v96.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, 0x300, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v103.h, v13.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v85.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v103.l, v13.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v86.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v13.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v31, v14
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v112.h, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v113.l, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v86.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v86.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v31, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v84.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v84.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v102.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v102.h, v13.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v101.h, v14.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, v83.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v16, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v101.l, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, v84.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, 0x300, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v99.h, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v31, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v100.h, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v101.l, v14.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v82.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, v81.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v99.l, v15.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v81.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v15.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v31, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v80.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, v80.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v98.h, v15.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v99.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v98.l, v16.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, v71.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v18, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v97.l, v16.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, v71.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v16.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, 0x300, v16.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v96.h, v17.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v69.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v19, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v87.l, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v31, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v17.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.h, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v97.l, v16.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v97.h, v16.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, v69.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v17.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.l, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v69.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v31, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v17.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, v68.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v67.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v87.l, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v87.h, v17.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v86.h, v18.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.h, v67.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v20, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v85.l, v18.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.l, v67.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v18.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, 0x300, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v84.h, v19.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v64.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v21, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v83.h, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v31, v20
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v19.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v85.l, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v85.h, v18.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, v65.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v19.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v65.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v31, v21
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v19.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, v55.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v50.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v83.l, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v83.h, v19.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v82.h, v20.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, v51.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v22, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v82.l, v20.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, v52.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v20.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, 0x300, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v80.h, v21.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v48.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v23, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v80.l, v21.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v49.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v21.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.l, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v31, v22
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v21.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.h, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v81.h, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v82.l, v20.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v49.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v49.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v31, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v21.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v48.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v48.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v71.l, v21.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v71.h, v21.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v70.h, v22.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.h, v39.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v24, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v70.l, v22.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.l, v48.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v22.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, 0x300, v22.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v68.h, v23.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v38.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v25, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v68.l, v23.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v39.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v23.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v31, v24
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v23.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.h, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v70.l, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v70.h, v22.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v39.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v39.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v31, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v23.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v38.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v38.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v67.h, v23.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v68.l, v23.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v66.l, v24.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.h, v37.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v26, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v66.h, v24.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, v38.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, 0x300, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v64.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v26, v31
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v65.l, v25.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v37.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v36.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v25.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v31, v26
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v25.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v66.l, v24.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v66.h, v24.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v37.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v37.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v31, v27
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, 0x300, v25.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v36.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v36.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v64.l, v25.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v64.h, v25.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v27, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v55.l, v26.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.l, v36.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v55.h, v26.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v28
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, 0x300, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v34.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v29, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v54.h, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v54.l, v27.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.l, 0x300, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v27.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v53.h, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v31, v28
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v27.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.h, 0x300, v27.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v54.h, v26.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v55.l, v26.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v35.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v35.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v31, v29
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v27.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, 0x300, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v34.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v34.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v53.h, v27.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v54.l, v27.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v29, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.l, v34.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.h, v33.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, 0x300, v28.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v31, v30
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v29.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v34.h, 0x300, v29.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v52.h, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v53.l, v28.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, v33.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, v33.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v53.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v31, v34
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v29.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.h, 0x300, v29.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v52.h, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v50.h, v29.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, v32.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v51.h, v29.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v52.l, v29.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.l, 0x300, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v51.l, v30.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v33
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v50.l, v30.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.l, 0x300, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v33
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v49.h, v30.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v32.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v31
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v31, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v32.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.h, 0x300, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v50.h, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v51.l, v30.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v31, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v32.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v32.h, 0x300, v32.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v31, v32
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -42156,64 +42028,64 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr161_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
@@ -42241,50 +42113,50 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 24, v32
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v27
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v26
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v24
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v21
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v20
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v9
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[1:2]
@@ -42328,50 +42200,50 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 24, v32
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v27
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v26
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v24
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v21
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v20
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v9
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB36_4: ; %end
@@ -42379,307 +42251,266 @@ define <128 x i8> @bitcast_v32f32_to_v128i8(<32 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v162.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v34.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v1.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v39, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v2.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v65.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v161.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v1.h, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v160.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v39, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v3.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff, v66
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v55, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v65.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v150.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v150.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v66, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v55, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v65
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v4.l, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v148.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v55, v39
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v39, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v149.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v39, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v5.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v148.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v147.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v147.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.h, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v65
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v39, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v6.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v54.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v39, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v7.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v145.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v55, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v8.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v55, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v135.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v54, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v144.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v39, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v135.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v39, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v9.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v134.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v8.h, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v133.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v39, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v10.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v132.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v39, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v11.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v131.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v130.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v54, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v133.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v132.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v39, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v12.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v129.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v51.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v53, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v10.l, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v53, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v130.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v11.h, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v39, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v13.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v119.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v39, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v14.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v118.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v50.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v39, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v15.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v116.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v129.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v39, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v16.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v115.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v39, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v17.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v114.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v113.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v53
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff, v53
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v119.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v118.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v51, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v14.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v39, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v18.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v112.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v48.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v39, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v19.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v39, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v20.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v51, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v116.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v115.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v39, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v21.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v99.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v50, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v50, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v113.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v17.h, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v39, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v22.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v39, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v23.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v96.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v39, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v24.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v36.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v49, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v19.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v112.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v39, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v25.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v39, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v26.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v84.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v35.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v50
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v18.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v50
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v49, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v102.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v19.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v101.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v48, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v49
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v20.h, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v39, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v27.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v82.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v39, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v28.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v81.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v39, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v29.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v71.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v48, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v22.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v99.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v49
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v21.l, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v98.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v39, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v30.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v33.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v38, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v22.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v97.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v38, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v23.h, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v39, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v31.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v33.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v68.l
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v37, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v25.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v87.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v24.l, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v37, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v85.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v25.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v36, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v26.h, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v36, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v28.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v82.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v27.l, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v34.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v35, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v28.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.l, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v71.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v29.h, v30.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.l, v31.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v69.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v31.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v30.l, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v31.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v32.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v39
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v31.l, v31.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v32.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v32.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v34
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v32.l, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v33, v39
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v39, v31
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v32.l, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v32.h, v33.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v39.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v39, v32
 ; GFX11-TRUE16-NEXT:    s_clause 0x5
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[9:12], off offset:32
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[13:16], off offset:48
@@ -52210,63 +52041,63 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:384
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:380
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:376
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:372
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v49, off, s32 offset:368
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:364
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:360
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:356
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v50, off, s32 offset:352
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:348
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:344
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:372
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v50, off, s32 offset:368
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:364
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:360
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:356
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v51, off, s32 offset:352
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:348
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:344
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:340
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v52, off, s32 offset:336
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:332
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:328
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:324
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:324
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v53, off, s32 offset:320
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:316
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:312
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:308
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v54, off, s32 offset:304
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:300
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:296
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:292
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v55, off, s32 offset:288
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:284
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:280
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:276
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:272
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:268
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:264
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:292
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:288
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:284
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:280
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:276
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:272
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:268
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:264
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v39, off, s32 offset:260
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:256
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:256
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:252
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:248
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v48, off, s32 offset:244
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:240
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:240
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:236
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:232
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v51, off, s32 offset:228
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:224
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:220
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:216
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v103, off, s32 offset:388
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:232
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v49, off, s32 offset:228
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:224
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:220
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:216
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v114, off, s32 offset:388
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:16
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v83, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32 offset:64
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v97, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v98, off, s32 offset:80
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v99, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v100, off, s32 offset:96
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v101, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v102, off, s32 offset:112
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:120
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v160, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:136
@@ -52280,144 +52111,143 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v164, off, s32 offset:192
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:200
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v165, off, s32 offset:208
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:212
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:204
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:196
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:188
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:180
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:172
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:164
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:156
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v55, off, s32 offset:212
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:204
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:196
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:188
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:180
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:172
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:100
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v97, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v98, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v100, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v102, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v112, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v99, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v101, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v113, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v116, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v115, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v117.l, v30.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.h, v28.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.h, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.h, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.h, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.h, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.h, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.h, v12.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v147.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v148.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v150.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v151.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v149.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.l, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v149.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v146.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.l, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.l, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v134.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.l, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v145.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.h, v0.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v144.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.l, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v131.h, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(54)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v103
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v114
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(53)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v130.l, 8, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.l, 8, v81.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(52)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v130.h, 8, v82.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.l, 8, v82.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(51)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.l, 8, v82.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.h, 8, v83.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(50)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.l, 8, v83.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.l, 8, v83.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(49)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v118.l, 8, v84.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.h, 8, v85.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(48)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v119.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v117.h, 8, v85.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(47)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.l, 8, v86.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v118.l, 8, v87.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(46)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v117.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.l, 8, v87.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(45)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.h, 8, v96.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.h, 8, v97.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(44)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v115.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.l, 8, v97.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(43)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.l, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.h, 8, v98.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(42)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.h, 8, v99.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v112.h, 8, v99.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(41)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v103.l, 8, v99.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.l, 8, v100.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(40)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v103.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v102.l, 8, v101.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(39)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.l, 8, v101.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v102.h, 8, v102.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(38)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v100.h, 8, v160.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(37)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.l, 8, v160.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.l, 8, v160.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(36)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v98.h, 8, v161.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(35)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.l, 8, v161.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.l, 8, v161.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(34)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v98.l, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.l, 8, v162.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.l, 8, v162.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.h, 8, v162.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(32)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v96.h, 8, v163.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.l, 8, v163.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.l, 8, v163.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.h, 8, v163.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(30)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v86.h, 8, v164.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.l, 8, v164.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.h, 8, v164.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.h, 8, v164.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(28)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v84.h, 8, v165.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.l, 8, v165.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.l, 8, v165.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v80.l, 8, v70.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v80.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.l, 8, v68.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.l, 8, v66.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.l, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.h, 8, v165.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v81.h, 8, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.l, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v71.l, 8, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v71.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.l, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.h, 8, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v67.h, 8, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.l, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.l, 8, v64.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.l, 8, v55.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v65.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v54.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.h, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v52.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.l, 8, v49.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.l, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.h, 8, v54.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v53.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v52.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v51.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.l, 8, v50.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v31.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v31.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GFX11-TRUE16-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -52431,746 +52261,660 @@ define <32 x float> @bitcast_v128i8_to_v32f32(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB38_3: ; %cmp.false
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v148.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v150.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v146.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v151.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v150.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v147.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v0.h, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v149.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v144.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v133.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v148.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v145.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v1.h, v147.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v149.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v146.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v146.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v0.l, v151.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v151.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v0.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v150.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v3.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v145.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v144.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v149, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v1.h, v150.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v132.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v131.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v148.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v130.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v149, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v2.l, v148.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v2.l, v149.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v134.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v2.l, v146.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v131.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v135.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v3.l, v145.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v128.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v134.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v145.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v130.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v149, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v3.l, v147.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v147.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v3.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v135.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v119.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v119.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v149, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v4.l, v144.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v4.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v133.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v117.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v116.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v4.l, v135.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v129.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v134.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v5.l, v133.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v119.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v132.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v6.l, v132.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v130.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v112.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v7.l, v130.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v115.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v129.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v102.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v8.l, v128.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v112.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.h, v119.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v100.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v9.l, v118.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v102.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v9.h, v117.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v97.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v10.l, v116.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v100.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v10.h, v115.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v87.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v11.l, v114.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v98.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v11.h, v113.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v85.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v12.l, v113.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v12.h, v103.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v83.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v13.l, v103.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v86.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v101.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v16, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v14.l, v101.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v14.h, v99.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v71.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v15.l, v99.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v81.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v15.h, v98.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v69.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v18, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v16.l, v97.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v71.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v16.h, v96.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v67.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v19, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v17.l, v87.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v115.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v115.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v149, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v5.l, v135.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v5.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v132.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v129.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v113.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v112.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v149, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v6.l, v133.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v6.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v103.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v128.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v103.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v101.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v149, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v7.l, v131.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v7.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v118.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v100.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v99.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v98.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v149, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v8.l, v129.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v8.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v116.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v114.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v96.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v96.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v149, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v9.l, v128.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v9.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v86.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v113.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v86.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v84.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v149, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v10.l, v117.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v10.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v102.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v84.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v82.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v81.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v149, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v11.l, v116.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v11.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v101.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v99.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v80.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v80.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v149, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v12.l, v114.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v12.l, v149.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v69.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v17.h, v86.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v64.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v20, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v18.l, v85.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v67.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v18.h, v84.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v51.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v21, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v19.l, v83.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v97.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v69.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v68.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v149, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v13.l, v112.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v13.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v87.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v67.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v65.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v19.h, v82.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v48.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v22, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v20.l, v82.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v20.h, v80.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v39.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v23, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v21.l, v80.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v21.h, v70.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v24, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v22.l, v70.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v22.h, v68.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v25, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v23.l, v68.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v38.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v23.h, v66.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v26, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v24.l, v66.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v24.h, v65.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v27, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v28
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v25.l, v64.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v25.h, v55.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v28, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v26.l, v55.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v26.h, v54.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v34.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v29, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v27.l, v54.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_hi16
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v65.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v149, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v14.l, v102.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v14.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v85.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v83.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v50.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v149, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v15.l, v100.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v15.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v49.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v82.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v149, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v16.l, v98.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v16.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v71.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v149, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v17.l, v97.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v17.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v70.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v68.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v38.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v149, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v18.l, v87.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v18.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v66.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v149, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v19.l, v85.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v19.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v35.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v149, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v20.l, v83.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v20.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v55.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v54.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v149, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v21.l, v81.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v21.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v33.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v149, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v22.l, v71.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v22.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v32.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v29, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v28.h, v53.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v34.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v28.l, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v33.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v28.h, v52.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v33.l, v29.h, v51.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v34
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v29.l, v50.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v33
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v149, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v23.l, v70.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v23.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v51.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v149, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v24.l, v67.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v24.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v149, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v25.l, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v25.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v149, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v26.l, v64.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v26.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v149, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v27.l, v54.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v27.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v149, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v28.l, v53.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v28.l, v149.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v149, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v29.l, v52.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v29.l, v149.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v30.l, v50.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v32.l, v49.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v149, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v30.l, v51.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v30.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v149, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v31.l, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v31.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v31
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v149, v31
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB38_2
 ; GFX11-TRUE16-NEXT:  .LBB38_4: ; %cmp.true
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v151.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v148.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v150.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v146.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v147.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v149.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v149.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v146.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v144.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v151.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v150.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v144.h, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v149.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v151.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v151.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v150.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v150.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v145.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v144.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v134.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v31, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v134.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v148.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v148.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v149.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v147.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v134.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v133.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v146.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v148.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v145.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v31, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v31.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v147.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v147.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v132.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v131.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v145.h, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v131.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v31, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v130.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v130.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v144.h, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v145.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v128.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v135.h, v4.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v135.l, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v129.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v32.l, v32.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v134.l, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v118.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v133.l, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v119.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v31, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v135.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v135.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v119.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v119.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v31, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v118.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v117.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v133.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v133.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v132.h, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v31, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v131.h, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v132.l, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v116.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v132.l, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v117.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v130.h, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v114.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v130.l, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v115.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v115.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v115.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v31, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v113.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v112.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v129.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v129.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v129.l, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, v112.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v128.l, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v112.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v119.h, v9.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v102.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v118.l, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v102.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v31, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v128.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v128.h, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v103.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v103.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v31, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v101.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v100.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v117.h, v9.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v118.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v117.h, v10.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, v100.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v116.l, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, v100.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, 0x300, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v115.h, v11.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v97.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v114.h, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v98.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v31, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v116.l, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v116.h, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v99.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v98.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v31, v13
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v96.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v96.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v114.l, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v114.h, v11.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v113.h, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, v87.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v113.l, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, v96.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, 0x300, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v103.h, v13.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v85.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v103.l, v13.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v86.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v13.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v31, v14
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v112.h, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v113.l, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v86.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v86.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v31, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v84.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v84.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v102.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v102.h, v13.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v101.h, v14.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, v83.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v16, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v101.l, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, v84.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, 0x300, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v99.h, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v31, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v100.h, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v101.l, v14.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v82.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, v81.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v99.l, v15.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v81.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v15.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v31, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v80.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, v80.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v98.h, v15.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v99.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v98.l, v16.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, v71.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v18, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v97.l, v16.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, v71.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v16.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, 0x300, v16.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v96.h, v17.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v69.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v19, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v87.l, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v31, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v17.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.h, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v97.l, v16.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v97.h, v16.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, v69.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v17.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.l, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v69.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v31, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v17.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, v68.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v67.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v87.l, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v87.h, v17.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v86.h, v18.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.h, v67.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v20, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v85.l, v18.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.l, v67.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v18.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, 0x300, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v84.h, v19.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v64.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v21, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v83.h, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v31, v20
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v19.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v85.l, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v85.h, v18.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, v65.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v19.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v65.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v31, v21
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v19.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, v55.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v50.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v83.l, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v83.h, v19.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v82.h, v20.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, v51.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v22, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v82.l, v20.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, v52.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v20.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, 0x300, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v80.h, v21.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v48.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v23, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v80.l, v21.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v49.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v21.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.l, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v31, v22
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v21.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.h, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v81.h, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v82.l, v20.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v49.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v49.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v31, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v21.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v48.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v48.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v71.l, v21.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v71.h, v21.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v70.h, v22.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.h, v39.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v24, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v70.l, v22.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.l, v48.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v22.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, 0x300, v22.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v68.h, v23.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v38.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v25, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v68.l, v23.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v39.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v23.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v31, v24
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v23.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.h, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v70.l, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v70.h, v22.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v39.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v39.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v31, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v23.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v38.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v38.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v67.h, v23.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v68.l, v23.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v66.l, v24.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.h, v37.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v26, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v66.h, v24.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, v38.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, 0x300, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v64.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v26, v31
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v65.l, v25.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v37.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v36.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v25.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v31, v26
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v25.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v66.l, v24.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v66.h, v24.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v37.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v37.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v31, v27
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, 0x300, v25.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v36.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v36.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v64.l, v25.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v64.h, v25.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v27, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v55.l, v26.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.l, v36.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v55.h, v26.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v28
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, 0x300, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v34.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v29, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v54.h, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v54.l, v27.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.l, 0x300, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v27.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v53.h, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v31, v28
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v27.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.h, 0x300, v27.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v54.h, v26.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v55.l, v26.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v35.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v35.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v31, v29
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v27.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, 0x300, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v34.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v34.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v53.h, v27.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v54.l, v27.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v29, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.l, v34.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.h, v33.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, 0x300, v28.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v31, v30
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v29.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v34.h, 0x300, v29.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v52.h, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v53.l, v28.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, v33.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, v33.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v53.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v31, v34
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v29.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.h, 0x300, v29.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v52.h, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v50.h, v29.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, v32.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v51.h, v29.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v52.l, v29.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.l, 0x300, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v51.l, v30.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v33
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v50.l, v30.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.l, 0x300, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v33
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v49.h, v30.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v32.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v31
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v31, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v32.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.h, 0x300, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v50.h, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v51.l, v30.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v31, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v32.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v32.h, 0x300, v32.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v31, v32
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -77938,64 +77682,64 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr161_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
@@ -78023,50 +77767,50 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 24, v32
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v27
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v26
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v24
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v21
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v20
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v9
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[1:2]
@@ -78135,50 +77879,50 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 24, v32
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v27
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v26
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v24
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v21
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v20
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v9
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB56_4: ; %end
@@ -78186,307 +77930,266 @@ define <128 x i8> @bitcast_v16i64_to_v128i8(<16 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v162.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v34.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v1.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v39, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v2.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v65.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v161.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v1.h, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v160.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v39, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v3.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff, v66
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v55, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v65.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v150.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v150.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v66, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v55, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v65
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v4.l, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v148.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v55, v39
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v39, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v149.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v39, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v5.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v148.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v147.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v147.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.h, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v65
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v39, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v6.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v54.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v39, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v7.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v145.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v55, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v8.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v55, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v135.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v54, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v144.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v39, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v135.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v39, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v9.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v134.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v8.h, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v133.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v39, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v10.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v132.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v39, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v11.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v131.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v130.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v54, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v133.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v132.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v39, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v12.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v129.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v51.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v53, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v10.l, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v53, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v130.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v11.h, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v39, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v13.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v119.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v39, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v14.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v118.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v50.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v39, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v15.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v116.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v129.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v39, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v16.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v115.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v39, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v17.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v114.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v113.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v53
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff, v53
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v119.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v118.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v51, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v14.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v39, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v18.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v112.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v48.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v39, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v19.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v39, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v20.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v51, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v116.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v115.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v39, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v21.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v99.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v50, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v50, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v113.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v17.h, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v39, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v22.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v39, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v23.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v96.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v39, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v24.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v36.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v49, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v19.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v112.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v39, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v25.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v39, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v26.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v84.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v35.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v50
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v18.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v50
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v49, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v102.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v19.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v101.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v48, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v49
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v20.h, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v39, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v27.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v82.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v39, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v28.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v81.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v39, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v29.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v71.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v48, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v22.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v99.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v49
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v21.l, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v98.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v39, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v30.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v33.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v38, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v22.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v97.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v38, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v23.h, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v39, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v31.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v33.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v68.l
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v37, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v25.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v87.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v24.l, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v37, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v85.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v25.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v36, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v26.h, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v36, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v28.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v82.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v27.l, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v34.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v35, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v28.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.l, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v71.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v29.h, v30.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.l, v31.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v69.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v31.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v30.l, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v31.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v32.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v39
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v31.l, v31.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v32.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v32.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v34
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v32.l, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v33, v39
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v39, v31
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v32.l, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v32.h, v33.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v39.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v39, v32
 ; GFX11-TRUE16-NEXT:    s_clause 0x5
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[9:12], off offset:32
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[13:16], off offset:48
@@ -87060,63 +86763,63 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:384
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:380
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:376
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:372
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v49, off, s32 offset:368
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:364
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:360
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:356
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v50, off, s32 offset:352
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:348
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:344
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:372
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v50, off, s32 offset:368
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:364
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:360
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:356
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v51, off, s32 offset:352
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:348
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:344
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:340
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v52, off, s32 offset:336
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:332
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:328
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:324
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:324
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v53, off, s32 offset:320
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:316
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:312
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:308
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v54, off, s32 offset:304
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:300
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:296
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:292
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v55, off, s32 offset:288
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:284
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:280
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:276
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:272
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:268
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:264
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:292
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:288
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:284
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:280
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:276
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:272
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:268
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:264
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v39, off, s32 offset:260
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:256
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:256
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:252
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:248
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v48, off, s32 offset:244
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:240
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:240
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:236
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:232
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v51, off, s32 offset:228
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:224
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:220
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:216
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v103, off, s32 offset:388
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:232
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v49, off, s32 offset:228
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:224
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:220
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:216
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v114, off, s32 offset:388
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:16
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v83, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32 offset:64
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v97, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v98, off, s32 offset:80
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v99, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v100, off, s32 offset:96
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v101, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v102, off, s32 offset:112
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:120
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v160, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:136
@@ -87130,144 +86833,143 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v164, off, s32 offset:192
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:200
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v165, off, s32 offset:208
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:212
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:204
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:196
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:188
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:180
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:172
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:164
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:156
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v55, off, s32 offset:212
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:204
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:196
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:188
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:180
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:172
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:100
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v97, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v98, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v100, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v102, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v112, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v99, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v101, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v113, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v116, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v115, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v117.l, v30.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.h, v28.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.h, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.h, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.h, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.h, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.h, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.h, v12.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v147.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v148.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v150.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v151.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v149.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.l, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v149.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v146.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.l, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.l, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v134.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.l, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v145.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.h, v0.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v144.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.l, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v131.h, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(54)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v103
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v114
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(53)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v130.l, 8, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.l, 8, v81.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(52)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v130.h, 8, v82.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.l, 8, v82.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(51)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.l, 8, v82.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.h, 8, v83.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(50)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.l, 8, v83.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.l, 8, v83.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(49)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v118.l, 8, v84.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.h, 8, v85.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(48)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v119.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v117.h, 8, v85.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(47)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.l, 8, v86.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v118.l, 8, v87.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(46)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v117.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.l, 8, v87.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(45)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.h, 8, v96.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.h, 8, v97.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(44)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v115.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.l, 8, v97.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(43)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.l, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.h, 8, v98.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(42)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.h, 8, v99.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v112.h, 8, v99.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(41)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v103.l, 8, v99.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.l, 8, v100.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(40)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v103.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v102.l, 8, v101.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(39)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.l, 8, v101.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v102.h, 8, v102.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(38)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v100.h, 8, v160.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(37)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.l, 8, v160.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.l, 8, v160.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(36)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v98.h, 8, v161.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(35)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.l, 8, v161.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.l, 8, v161.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(34)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v98.l, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.l, 8, v162.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.l, 8, v162.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.h, 8, v162.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(32)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v96.h, 8, v163.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.l, 8, v163.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.l, 8, v163.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.h, 8, v163.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(30)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v86.h, 8, v164.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.l, 8, v164.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.h, 8, v164.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.h, 8, v164.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(28)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v84.h, 8, v165.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.l, 8, v165.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.l, 8, v165.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v80.l, 8, v70.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v80.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.l, 8, v68.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.l, 8, v66.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.l, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.h, 8, v165.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v81.h, 8, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.l, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v71.l, 8, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v71.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.l, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.h, 8, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v67.h, 8, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.l, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.l, 8, v64.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.l, 8, v55.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v65.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v54.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.h, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v52.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.l, 8, v49.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.l, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.h, 8, v54.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v53.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v52.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v51.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.l, 8, v50.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v31.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v31.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GFX11-TRUE16-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -87281,746 +86983,660 @@ define <16 x i64> @bitcast_v128i8_to_v16i64(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB58_3: ; %cmp.false
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v148.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v150.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v146.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v151.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v150.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v147.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v0.h, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v149.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v144.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v133.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v148.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v145.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v1.h, v147.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v149.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v146.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v146.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v0.l, v151.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v151.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v0.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v150.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v3.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v145.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v144.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v149, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v1.h, v150.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v132.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v131.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v148.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v130.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v149, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v2.l, v148.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v2.l, v149.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v134.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v2.l, v146.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v131.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v135.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v3.l, v145.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v128.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v134.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v145.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v130.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v149, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v3.l, v147.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v147.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v3.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v135.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v119.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v119.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v149, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v4.l, v144.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v4.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v133.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v117.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v116.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v4.l, v135.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v129.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v134.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v5.l, v133.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v119.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v132.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v6.l, v132.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v130.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v112.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v7.l, v130.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v115.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v129.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v102.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v8.l, v128.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v112.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.h, v119.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v100.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v9.l, v118.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v102.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v9.h, v117.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v97.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v10.l, v116.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v100.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v10.h, v115.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v87.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v11.l, v114.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v98.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v11.h, v113.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v85.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v12.l, v113.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v12.h, v103.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v83.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v13.l, v103.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v86.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v101.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v16, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v14.l, v101.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v14.h, v99.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v71.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v15.l, v99.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v81.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v15.h, v98.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v69.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v18, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v16.l, v97.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v71.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v16.h, v96.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v67.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v19, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v17.l, v87.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v115.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v115.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v149, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v5.l, v135.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v5.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v132.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v129.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v113.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v112.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v149, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v6.l, v133.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v6.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v103.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v128.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v103.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v101.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v149, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v7.l, v131.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v7.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v118.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v100.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v99.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v98.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v149, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v8.l, v129.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v8.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v116.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v114.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v96.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v96.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v149, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v9.l, v128.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v9.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v86.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v113.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v86.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v84.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v149, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v10.l, v117.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v10.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v102.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v84.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v82.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v81.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v149, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v11.l, v116.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v11.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v101.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v99.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v80.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v80.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v149, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v12.l, v114.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v12.l, v149.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v69.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v17.h, v86.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v64.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v20, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v18.l, v85.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v67.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v18.h, v84.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v51.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v21, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v19.l, v83.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v97.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v69.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v68.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v149, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v13.l, v112.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v13.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v87.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v67.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v65.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v19.h, v82.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v48.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v22, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v20.l, v82.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v20.h, v80.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v39.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v23, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v21.l, v80.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v21.h, v70.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v24, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v22.l, v70.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v22.h, v68.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v25, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v23.l, v68.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v38.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v23.h, v66.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v26, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v24.l, v66.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v24.h, v65.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v27, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v28
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v25.l, v64.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v25.h, v55.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v28, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v26.l, v55.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v26.h, v54.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v34.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v29, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v27.l, v54.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_hi16
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v65.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v149, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v14.l, v102.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v14.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v85.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v83.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v50.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v149, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v15.l, v100.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v15.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v49.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v82.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v149, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v16.l, v98.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v16.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v71.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v149, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v17.l, v97.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v17.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v70.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v68.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v38.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v149, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v18.l, v87.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v18.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v66.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v149, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v19.l, v85.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v19.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v35.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v149, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v20.l, v83.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v20.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v55.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v54.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v149, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v21.l, v81.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v21.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v33.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v149, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v22.l, v71.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v22.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v32.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v29, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v28.h, v53.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v34.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v28.l, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v33.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v28.h, v52.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v33.l, v29.h, v51.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v34
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v29.l, v50.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v33
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v149, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v23.l, v70.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v23.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v51.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v149, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v24.l, v67.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v24.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v149, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v25.l, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v25.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v149, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v26.l, v64.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v26.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v149, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v27.l, v54.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v27.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v149, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v28.l, v53.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v28.l, v149.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v149, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v29.l, v52.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v29.l, v149.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v30.l, v50.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v32.l, v49.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v149, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v30.l, v51.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v30.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v149, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v31.l, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v31.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v31
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v149, v31
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB58_2
 ; GFX11-TRUE16-NEXT:  .LBB58_4: ; %cmp.true
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v151.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v148.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v150.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v146.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v147.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v149.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v149.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v146.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v144.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v151.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v150.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v144.h, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v149.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v151.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v151.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v150.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v150.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v145.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v144.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v134.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v31, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v134.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v148.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v148.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v149.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v147.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v134.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v133.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v146.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v148.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v145.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v31, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v31.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v147.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v147.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v132.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v131.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v145.h, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v131.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v31, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v130.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v130.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v144.h, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v145.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v128.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v135.h, v4.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v135.l, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v129.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v32.l, v32.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v134.l, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v118.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v133.l, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v119.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v31, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v135.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v135.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v119.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v119.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v31, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v118.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v117.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v133.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v133.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v132.h, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v31, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v131.h, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v132.l, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v116.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v132.l, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v117.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v130.h, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v114.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v130.l, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v115.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v115.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v115.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v31, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v113.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v112.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v129.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v129.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v129.l, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, v112.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v128.l, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v112.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v119.h, v9.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v102.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v118.l, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v102.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v31, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v128.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v128.h, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v103.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v103.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v31, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v101.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v100.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v117.h, v9.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v118.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v117.h, v10.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, v100.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v116.l, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, v100.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, 0x300, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v115.h, v11.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v97.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v114.h, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v98.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v31, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v116.l, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v116.h, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v99.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v98.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v31, v13
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v96.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v96.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v114.l, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v114.h, v11.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v113.h, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, v87.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v113.l, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, v96.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, 0x300, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v103.h, v13.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v85.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v103.l, v13.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v86.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v13.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v31, v14
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v112.h, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v113.l, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v86.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v86.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v31, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v84.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v84.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v102.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v102.h, v13.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v101.h, v14.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, v83.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v16, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v101.l, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, v84.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, 0x300, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v99.h, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v31, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v100.h, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v101.l, v14.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v82.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, v81.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v99.l, v15.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v81.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v15.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v31, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v80.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, v80.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v98.h, v15.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v99.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v98.l, v16.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, v71.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v18, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v97.l, v16.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, v71.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v16.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, 0x300, v16.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v96.h, v17.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v69.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v19, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v87.l, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v31, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v17.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.h, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v97.l, v16.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v97.h, v16.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, v69.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v17.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.l, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v69.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v31, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v17.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, v68.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v67.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v87.l, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v87.h, v17.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v86.h, v18.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.h, v67.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v20, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v85.l, v18.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.l, v67.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v18.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, 0x300, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v84.h, v19.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v64.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v21, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v83.h, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v31, v20
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v19.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v85.l, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v85.h, v18.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, v65.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v19.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v65.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v31, v21
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v19.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, v55.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v50.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v83.l, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v83.h, v19.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v82.h, v20.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, v51.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v22, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v82.l, v20.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, v52.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v20.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, 0x300, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v80.h, v21.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v48.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v23, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v80.l, v21.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v49.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v21.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.l, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v31, v22
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v21.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.h, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v81.h, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v82.l, v20.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v49.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v49.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v31, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v21.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v48.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v48.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v71.l, v21.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v71.h, v21.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v70.h, v22.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.h, v39.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v24, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v70.l, v22.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.l, v48.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v22.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, 0x300, v22.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v68.h, v23.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v38.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v25, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v68.l, v23.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v39.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v23.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v31, v24
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v23.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.h, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v70.l, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v70.h, v22.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v39.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v39.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v31, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v23.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v38.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v38.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v67.h, v23.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v68.l, v23.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v66.l, v24.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.h, v37.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v26, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v66.h, v24.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, v38.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, 0x300, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v64.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v26, v31
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v65.l, v25.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v37.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v36.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v25.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v31, v26
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v25.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v66.l, v24.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v66.h, v24.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v37.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v37.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v31, v27
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, 0x300, v25.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v36.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v36.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v64.l, v25.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v64.h, v25.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v27, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v55.l, v26.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.l, v36.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v55.h, v26.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v28
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, 0x300, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v34.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v29, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v54.h, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v54.l, v27.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.l, 0x300, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v27.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v53.h, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v31, v28
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v27.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.h, 0x300, v27.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v54.h, v26.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v55.l, v26.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v35.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v35.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v31, v29
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v27.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, 0x300, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v34.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v34.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v53.h, v27.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v54.l, v27.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v29, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.l, v34.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.h, v33.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, 0x300, v28.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v31, v30
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v29.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v34.h, 0x300, v29.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v52.h, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v53.l, v28.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, v33.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, v33.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v53.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v31, v34
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v29.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.h, 0x300, v29.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v52.h, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v50.h, v29.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, v32.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v51.h, v29.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v52.l, v29.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.l, 0x300, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v51.l, v30.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v33
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v50.l, v30.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.l, 0x300, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v33
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v49.h, v30.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v32.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v31
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v31, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v32.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.h, 0x300, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v50.h, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v51.l, v30.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v31, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v32.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v32.h, 0x300, v32.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v31, v32
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -111800,64 +111416,64 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr161_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
@@ -111885,50 +111501,50 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 24, v32
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v27
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v26
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v24
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v21
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v20
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v9
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[1:2]
@@ -111972,50 +111588,50 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 24, v32
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v28
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v27
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v26
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v24
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v22
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v21
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v20
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v18
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v9
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB72_4: ; %end
@@ -112023,307 +111639,266 @@ define <128 x i8> @bitcast_v16f64_to_v128i8(<16 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v162.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v34.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v1.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v39, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v2.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v65.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v161.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v1.h, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v160.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v39, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v3.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v66, 0xffff, v66
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v55, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v65.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v150.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v150.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v66, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v55, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v65
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v4.l, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v148.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v55, v39
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v39, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v149.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v39, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v5.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v148.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v147.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v147.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.h, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v65
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v39, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v6.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v54.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v39, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v7.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v145.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v55, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v55, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v8.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v55, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v135.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v54, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v144.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v39, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v135.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v39, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v9.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v134.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v8.h, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v133.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v39, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v10.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v132.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v39, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v11.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v131.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v130.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v54, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v133.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v132.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v39, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v12.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v129.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v51.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v53, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v53, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v10.l, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v53, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v130.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v11.h, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v39, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v13.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v119.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v39, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v14.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v118.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v50.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v39, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v15.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v116.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v129.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v39, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v16.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v115.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v39, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v17.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v114.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v113.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v53
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff, v53
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v119.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v118.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v51, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v14.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v39, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v18.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v112.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v48.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v39, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v19.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v39, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v20.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v51, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v116.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v115.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v39, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v21.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v99.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v50, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v50, 0xffff, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v50, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v113.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v17.h, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v39, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v22.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v39, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v23.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v96.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v39, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v24.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v36.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v49, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v19.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v112.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v39, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v25.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v39, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v26.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v84.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v35.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v50
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v18.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v50
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v49, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v102.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v19.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v101.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v48, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v49
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v20.h, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v39, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v27.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v82.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v39, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v28.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v81.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v39, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v29.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v71.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v48, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v22.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v99.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v49
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v21.l, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v98.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v39, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v30.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v33.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v38, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v22.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v97.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v38, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v23.h, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v39, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v31.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v33.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v68.l
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v37, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v25.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v87.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v24.l, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v37, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v85.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v25.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v36, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v26.h, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v36, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v28.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v82.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v27.l, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v34.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v35, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v28.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.l, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v71.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v29.h, v30.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.l, v31.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v69.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v31.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v30.l, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v31.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v32.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v39
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v31.l, v31.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v32.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v32.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v39
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v34
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v39.h, v32.l, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v33, v39
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v39, v31
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v32.l, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v32.h, v33.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v39.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v39, v32
 ; GFX11-TRUE16-NEXT:    s_clause 0x5
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[9:12], off offset:32
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[13:16], off offset:48
@@ -121839,63 +121414,63 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:384
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:380
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:376
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:372
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v49, off, s32 offset:368
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:364
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:360
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:356
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v50, off, s32 offset:352
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:348
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:344
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:372
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v50, off, s32 offset:368
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:364
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v51, off, s32 offset:360
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:356
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v51, off, s32 offset:352
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:348
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:344
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:340
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v52, off, s32 offset:336
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:332
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v53, off, s32 offset:328
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:324
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:324
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v53, off, s32 offset:320
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:316
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:316
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v54, off, s32 offset:312
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:308
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v54, off, s32 offset:304
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:300
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v55, off, s32 offset:296
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:292
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v55, off, s32 offset:288
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:284
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:280
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:276
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:272
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:268
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:264
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:292
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:288
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:284
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:280
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:276
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:272
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v39, off, s32 offset:268
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:264
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v39, off, s32 offset:260
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:256
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:256
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v48, off, s32 offset:252
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:248
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v48, off, s32 offset:244
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:240
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:240
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v49, off, s32 offset:236
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v70, off, s32 offset:232
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v51, off, s32 offset:228
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:224
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v52, off, s32 offset:220
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:216
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v103, off, s32 offset:388
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v70, off, s32 offset:232
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v49, off, s32 offset:228
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:224
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v50, off, s32 offset:220
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:216
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v114, off, s32 offset:388
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v82, off, s32 offset:8
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:16
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v83, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32 offset:64
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v85, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v87, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v97, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v97, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v98, off, s32 offset:80
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v99, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v99, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v100, off, s32 offset:96
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v101, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v101, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v102, off, s32 offset:112
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v160, off, s32 offset:120
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v160, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v161, off, s32 offset:136
@@ -121909,144 +121484,143 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v164, off, s32 offset:192
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v165, off, s32 offset:200
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v165, off, s32 offset:208
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:212
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:204
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:196
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:188
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:180
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:172
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v71, off, s32 offset:164
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v71, off, s32 offset:156
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v81, off, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v83, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v55, off, s32 offset:212
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:204
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:196
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32 offset:188
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v68, off, s32 offset:180
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v69, off, s32 offset:172
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v69, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v80, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v81, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v82, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v84, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v85, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v84, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v86, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v87, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v86, off, s32 offset:100
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v96, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v97, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v98, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v100, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v102, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v102, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v112, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v114, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v96, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v98, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v99, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v100, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v101, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v103, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v103, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v112, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v113, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v115, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v116, off, s32 offset:4
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v115, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v117.l, v30.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.h, v28.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.h, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.h, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.h, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v130.h, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.h, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v134.h, v12.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v147.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v148.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v150.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v151.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v149.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.l, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v149.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v146.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.l, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.l, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v134.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.l, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(62)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v145.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.h, v0.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v151.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v150.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v148.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v147.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v144.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v145.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.l, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v135.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v133.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v131.h, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(54)
-; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v103
+; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v114
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(53)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v130.l, 8, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v132.l, 8, v81.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(52)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v130.h, 8, v82.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.l, 8, v82.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(51)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.l, 8, v82.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.h, 8, v83.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(50)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v129.l, 8, v83.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.l, 8, v83.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(49)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v118.l, 8, v84.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v128.h, 8, v85.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(48)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v119.h, 8, v85.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v117.h, 8, v85.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(47)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.l, 8, v86.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v118.l, 8, v87.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(46)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v117.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.l, 8, v87.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(45)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.h, 8, v96.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v116.h, 8, v97.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(44)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v115.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.l, 8, v97.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(43)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.l, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v114.h, 8, v98.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(42)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.h, 8, v99.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v112.h, 8, v99.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(41)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v103.l, 8, v99.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v113.l, 8, v100.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(40)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v103.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v102.l, 8, v101.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(39)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.l, 8, v101.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v102.h, 8, v102.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(38)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v100.h, 8, v160.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(37)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.l, 8, v160.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v101.l, 8, v160.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(36)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v98.h, 8, v161.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(35)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.l, 8, v161.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v99.l, 8, v161.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(34)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v98.l, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.l, 8, v162.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.l, 8, v162.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v97.h, 8, v162.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(32)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v96.h, 8, v163.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.l, 8, v163.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.l, 8, v163.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v87.h, 8, v163.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(30)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v86.h, 8, v164.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.l, 8, v164.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.h, 8, v164.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v85.h, 8, v164.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(28)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v84.h, 8, v165.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.l, 8, v165.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.l, 8, v165.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v80.l, 8, v70.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v80.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.l, 8, v68.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.l, 8, v66.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.l, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v83.h, 8, v165.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v81.h, 8, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v82.l, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v71.l, 8, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v71.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.l, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v70.h, 8, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v67.h, 8, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v68.l, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.l, 8, v64.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v66.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.l, 8, v55.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v65.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v54.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.h, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v53.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v52.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.l, 8, v49.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.l, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v64.h, 8, v54.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v53.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v52.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v51.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.l, 8, v50.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v31.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v31.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
 ; GFX11-TRUE16-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
@@ -122060,746 +121634,660 @@ define <16 x double> @bitcast_v128i8_to_v16f64(<128 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB74_3: ; %cmp.false
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v148.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v150.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v146.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v151.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v150.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v147.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v0.h, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v149.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v144.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v133.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v148.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v145.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v1.h, v147.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v149.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v149.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v146.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v146.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v0.l, v151.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v151.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v0.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v150.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v3.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v145.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v144.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v149, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v1.h, v150.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v132.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v131.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v148.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v130.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v149, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v2.l, v148.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v2.l, v149.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v134.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v2.l, v146.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v131.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v135.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v3.l, v145.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v128.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v134.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v145.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v130.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v149, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v3.l, v147.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v147.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v3.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v135.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v119.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v119.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v149, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v4.l, v144.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v4.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v133.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v117.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v116.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v4.l, v135.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v129.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v134.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v5.l, v133.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v119.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v132.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v6.l, v132.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v130.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v112.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v7.l, v130.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v115.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v129.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v102.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v8.l, v128.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v112.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.h, v119.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v100.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v9.l, v118.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v102.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v9.h, v117.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v97.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v10.l, v116.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v100.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v10.h, v115.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v87.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v11.l, v114.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v98.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v11.h, v113.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v85.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v12.l, v113.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v12.h, v103.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v83.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v13.l, v103.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v86.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v101.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v16, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v14.l, v101.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v14.h, v99.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v71.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v15.l, v99.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v81.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v15.h, v98.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v69.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v18, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v16.l, v97.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v71.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v16.h, v96.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v67.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v19, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v17.l, v87.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v115.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v115.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v149, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v5.l, v135.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v5.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v132.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v129.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v113.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v112.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v149, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v6.l, v133.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v6.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v103.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v128.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v103.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v101.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v149, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v7.l, v131.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v7.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v118.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v100.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v99.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v98.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v149, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v8.l, v129.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v8.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v116.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v114.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v96.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v96.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v149, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v9.l, v128.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v9.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v86.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v113.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v86.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v84.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v149, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v10.l, v117.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v10.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v102.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v84.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v82.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v81.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v149, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v11.l, v116.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v11.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v101.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v99.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v80.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v80.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v149, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v12.l, v114.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v12.l, v149.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v69.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v17.h, v86.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v64.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v20, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v18.l, v85.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v67.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v18.h, v84.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v51.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v21, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v19.l, v83.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v97.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v69.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v68.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v149, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v13.l, v112.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v13.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v87.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v67.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v65.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v19.h, v82.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v48.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v22, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v20.l, v82.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v20.h, v80.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v39.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v23, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v21.l, v80.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v21.h, v70.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v24, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v22.l, v70.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v22.h, v68.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v25, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v23.l, v68.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v38.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v23.h, v66.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v26, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v24.l, v66.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v24.h, v65.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v27, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v28
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v25.l, v64.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v25.h, v55.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v28, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v26.l, v55.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v26.h, v54.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v34.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v29, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v27.l, v54.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_hi16
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v65.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v149, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v14.l, v102.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v14.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v85.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v83.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v50.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v149, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v15.l, v100.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v15.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v49.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v82.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v49.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v149, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v16.l, v98.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v16.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v71.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v149, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v17.l, v97.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v17.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v70.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v68.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v38.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v149, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v18.l, v87.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v18.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v66.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v149, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v19.l, v85.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v19.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v35.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v149, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v20.l, v83.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v20.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v55.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v54.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v149, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v21.l, v81.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v21.l, v149.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v33.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v149, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v22.l, v71.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v22.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v32.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v29, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v28.h, v53.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v34.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v28.l, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v33.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v28.h, v52.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v33.l, v29.h, v51.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v34
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v29.l, v50.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v33
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v149, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v23.l, v70.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v23.l, v149.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v51.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v149, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v24.l, v67.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v24.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v149, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v25.l, v66.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v25.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v149, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v26.l, v64.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v26.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v149, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v27.l, v54.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v27.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v149, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v28.l, v53.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v28.l, v149.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v149, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v29.l, v52.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v29.l, v149.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v30.l, v50.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v32.l, v49.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v149, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v30.l, v51.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v30.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v149, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v149.l, v31.l, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v31.l, v149.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v31
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v149, v31
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB74_2
 ; GFX11-TRUE16-NEXT:  .LBB74_4: ; %cmp.true
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v151.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v148.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v150.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v146.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v147.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v149.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v149.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v146.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v146.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v144.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v151.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v150.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v144.h, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v149.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v151.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v151.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v150.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v150.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v145.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v144.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v134.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v31, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v134.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v148.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v148.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v149.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v147.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v134.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v133.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v146.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v148.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v145.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v31, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v31.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v147.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v147.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v132.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v131.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v145.h, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v131.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v31, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v130.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v130.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v144.h, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v145.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v128.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v135.h, v4.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v135.l, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v129.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v32.l, v32.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v134.l, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v118.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v133.l, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v119.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v31, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v135.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v135.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v119.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v119.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v31, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v118.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v117.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v133.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v133.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v132.h, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v31, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v131.h, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v132.l, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v116.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v132.l, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v117.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v130.h, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v114.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v130.l, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v115.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v115.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v115.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v31, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v113.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v112.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v129.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v129.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v129.l, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, v112.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v128.l, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v112.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v119.h, v9.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v102.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v118.l, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v102.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v31, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v128.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v128.h, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v103.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v103.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v31, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v101.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v100.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v117.h, v9.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v118.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v117.h, v10.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, v100.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v116.l, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, v100.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, 0x300, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v115.h, v11.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v97.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v114.h, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v98.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v31, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v116.l, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v116.h, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v99.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v98.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v31, v13
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v96.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v96.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v114.l, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v114.h, v11.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v113.h, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, v87.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v14, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v113.l, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, v96.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, 0x300, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v103.h, v13.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v85.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v103.l, v13.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v86.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v13.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v31, v14
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v112.h, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v113.l, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v86.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v86.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v31, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v84.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v84.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v102.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v102.h, v13.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v101.h, v14.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, v83.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v16, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v101.l, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, v84.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, 0x300, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v99.h, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v31, v16
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v100.h, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v101.l, v14.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v82.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, v81.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v99.l, v15.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v81.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v15.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v31, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.l, v80.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v16.h, v80.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v98.h, v15.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v99.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v98.l, v16.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, v71.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v18, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v97.l, v16.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, v71.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v16.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, 0x300, v16.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v96.h, v17.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v69.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v19, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v87.l, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v31, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v17.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.h, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v97.l, v16.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v97.h, v16.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, v69.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v17.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.l, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v69.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v31, v19
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v17.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, 0x300, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.l, v68.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, v67.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v87.l, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v87.h, v17.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v86.h, v18.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.h, v67.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v20, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v85.l, v18.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v19.l, v67.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v18.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, 0x300, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v84.h, v19.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v64.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v21, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v83.h, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v31, v20
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v19.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v85.l, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v85.h, v18.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, v65.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v19.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v65.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v31, v21
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v19.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, 0x300, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.l, v55.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v20.h, v50.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v83.l, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v83.h, v19.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v82.h, v20.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, v51.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v22, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v82.l, v20.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, v52.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v20.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, 0x300, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v80.h, v21.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v48.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v23, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v80.l, v21.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v49.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v24, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v21.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.l, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v31, v22
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v21.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.h, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v81.h, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v82.l, v20.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v49.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v49.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v31, v23
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v21.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, 0x300, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.l, v48.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v22.h, v48.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v71.l, v21.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v71.h, v21.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v70.h, v22.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.h, v39.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v24, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v70.l, v22.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v23.l, v48.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v22.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, 0x300, v22.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v68.h, v23.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v38.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v25, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v68.l, v23.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v39.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v23.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v31, v24
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v23.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.h, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v70.l, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v70.h, v22.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v39.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v39.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v31, v25
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v23.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, 0x300, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, v38.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.h, v38.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v67.h, v23.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v68.l, v23.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v66.l, v24.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.h, v37.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v26, v31
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v66.h, v24.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, v38.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v24.l, 0x300, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v64.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v26, v31
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v65.l, v25.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v37.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v36.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v27, 0xffff, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v25.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v31, v26
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v25.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v66.l, v24.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v66.h, v24.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v37.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v37.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v31, v27
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v25.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, 0x300, v25.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.l, v36.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v26.h, v36.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v64.l, v25.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v64.h, v25.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v27, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v55.l, v26.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.l, v36.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v55.h, v26.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v28
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, 0x300, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v34.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v29, v31
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v54.h, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v54.l, v27.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.l, 0x300, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v29, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v27.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v53.h, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v31, v28
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v27.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.h, 0x300, v27.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v54.h, v26.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v55.l, v26.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v35.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v35.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v31, v29
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v27.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, 0x300, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.l, v34.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v28.h, v34.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v53.h, v27.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v54.l, v27.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v29, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.l, v34.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.h, v33.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, 0x300, v28.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v31, v30
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v29.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v34.h, 0x300, v29.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v52.h, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v53.l, v28.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, v33.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, v33.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v53.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v31, v34
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v29.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.h, 0x300, v29.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.l, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v52.h, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v50.h, v29.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v30.h, v32.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v51.h, v29.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v52.l, v29.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v29.l, 0x300, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v51.l, v30.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v33
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v50.l, v30.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.l, 0x300, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v31
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v33
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v49.h, v30.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v31
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.h, 0x300, v32.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v31
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v31, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v32.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v33.h, 0x300, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v50.h, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v51.l, v30.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v31, v33
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v31.l, 0x300, v32.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v32.h, 0x300, v32.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v31, v32
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -160089,159 +159577,162 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v40, s32 offset:156
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v41, s32 offset:152
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v42, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v43, s32 offset:144
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v44, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v45, s32 offset:136
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v46, s32 offset:132
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v47, s32 offset:128
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v56, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v57, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v58, s32 offset:116
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v59, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v60, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v61, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v62, s32 offset:100
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v63, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v72, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v73, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v74, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v75, s32 offset:80
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v76, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v77, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v78, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v79, s32 offset:64
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v88, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v89, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v90, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v91, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v92, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v93, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v94, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v95, s32 offset:32
-; GFX11-TRUE16-NEXT:    s_clause 0x4
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v104, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v105, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v106, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v107, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v108, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v40, s32 offset:168
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v41, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v42, s32 offset:160
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v43, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v44, s32 offset:152
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v45, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v46, s32 offset:144
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v47, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v56, s32 offset:136
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v57, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v58, s32 offset:128
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v59, s32 offset:124
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v60, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v61, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v62, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v63, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v72, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v73, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v74, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v75, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v76, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v77, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v78, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v79, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v88, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v89, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v90, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v91, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v92, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v93, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v94, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v95, s32 offset:44
+; GFX11-TRUE16-NEXT:    s_clause 0x7
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v104, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v105, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v106, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v107, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v108, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v109, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v110, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_store_b32 off, v111, s32 offset:12
 ; GFX11-TRUE16-NEXT:    s_clause 0x2
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v33, off, s32 offset:8
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v32, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr111_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr106_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr105_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr108_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr104_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr107_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr105_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr164_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr106_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr94_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr90_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr180_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr91_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr165_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr95_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr93_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr166_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr88_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr161_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr75_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr47_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr78_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr76_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr179_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr72_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr43_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr74_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr177_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr63_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr178_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr59_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr60_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr73_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr58_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr57_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr44_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr56_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr41_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr42_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr89_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr43_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr61_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr183_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr57_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr167_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr104_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr176_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr78_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr166_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr77_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr95_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr93_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr47_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr44_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr45_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr92_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr79_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr40_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr74_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr62_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr63_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr59_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr182_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr62_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr180_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr108_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr176_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr60_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr46_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr45_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr91_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr165_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr89_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr163_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr110_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr161_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr107_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr109_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr181_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr182_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr177_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr94_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr90_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr79_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr77_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr75_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr72_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr61_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr58_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr56_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr42_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr183_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr181_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr179_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr167_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr164_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr162_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v33
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
@@ -160250,143 +159741,142 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB90_2
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[33:34], 24, v[31:32]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[5:6]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[34:35], 24, v[29:30]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[7:8]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[13:14]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[11:12]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[69:70], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[35:36], 24, v[27:28]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[15:16]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[5:6]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[36:37], 24, v[25:26]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[49:50], 24, v[13:14]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[52:53], 24, v[11:12]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[3:4]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[37:38], 24, v[23:24]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v166, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v165, 8, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v176, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v167, 24, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v183, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v43, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v42, 24, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v56, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v58, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v59, 24, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v72, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v76, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v75, 24, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v88, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v91, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v90, 24, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v94, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v106, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v180, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v182, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v40, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v45, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v47, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v57, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v60, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v63, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v74, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v76, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v78, 8, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v88, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v93, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v95, 8, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v104, 8, v3
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v105, 24, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v107, 8, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v108, 8, v1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 24, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v27
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 24, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v177, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v182, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v181, 24, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v40, 8, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v45, 8, v21
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v46, 24, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v60, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v63, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v62, 24, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v74, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v79, 8, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[1:2]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[21:22]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[19:20]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[17:18]
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.h, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v132.h, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v164.h, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v148.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v144.h, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v180.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v165.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v161.h, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v47.h, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v179.h, v8.l
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v106, 8, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v111, 8, v1
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v164, 8, v27
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v167, 24, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v179, 8, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v181, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v183, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v42, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v46, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v56, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v58, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v61, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v72, 24, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v75, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v77, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v79, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v90, 8, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v94, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[15:16]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[70:71], 24, v[1:2]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[36:37], 24, v[25:26]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[23:24]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[51:52], 24, v[21:22]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[19:20]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[17:18]
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v131.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v128.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v129.h, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v146.h, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v133.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v135.h, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v166.h, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v150.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v151.h, v6.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v43.h, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v177.h, v8.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v178.h, v8.h
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v73.h, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v44.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v41.h, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v89.h, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v61.h, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v57.h, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v104.h, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v78.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v77.h, v14.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v95.h, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v93.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v92.h, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v71.h, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v70.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.h, v18.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.h, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.h, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.h, v20.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.h, v21.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.h, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.h, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v97.h, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v87.h, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.h, v24.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v101.h, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v98.h, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v96.h, v26.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v112.h, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v100.h, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v99.h, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v113.h, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v103.h, v30.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v102.h, v30.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v116.h, v31.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v115.h, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v114.h, v32.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v41.h, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v44.h, v10.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v92.h, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v59.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v62.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v108.h, v13.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v91.h, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v89.h, v14.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v110.h, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v107.h, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v109.h, v16.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.h, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.h, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.h, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v84.h, v20.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v96.h, v21.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.h, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v87.h, v22.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v99.h, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v97.h, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v98.h, v24.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v102.h, v25.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v100.h, v26.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v101.h, v26.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v113.h, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v103.h, v28.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v112.h, v28.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v116.h, v29.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v114.h, v30.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v115.h, v30.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v119.h, v31.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v117.h, v32.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v118.h, v32.h
 ; GFX11-TRUE16-NEXT:  .LBB90_2: ; %Flow
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB90_4
 ; GFX11-TRUE16-NEXT:  ; %bb.3: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff0000, v18
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v20
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v18, 0x40c00000, v18
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v37, v18, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v18
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v37, v37, v18, 0x7fff
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v70, v37, v39 :: v_dual_add_f32 v33, 0x40c00000, v33
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v80, v37, v39 :: v_dual_add_f32 v33, 0x40c00000, v33
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v36, v33, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v33
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v36, v36, v33, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff0000, v17
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v70.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v80.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v55, v36, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v81, v36, v38, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_lshlrev_b32 v17, 16, v17
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v17, 0x40c00000, v17
@@ -160399,498 +159889,500 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v18, v48, v34, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v37, v50, v17, 0x7fff
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v71, v37, v51 :: v_dual_lshlrev_b32 v20, 16, v20
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_add_f32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v82, v37, v51 :: v_dual_and_b32 v35, 0xffff0000, v20
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_lshlrev_b32 v20, 16, v20
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v51, 0xffff0000, v11
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v71.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v82.h
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v20, 0x40c00000, v20
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v20
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v18, v49, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v18, 0xffff, v33, v55
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v18, 0xffff, v33, v81
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v35
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v20, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v20
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v35
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v17, 0xffff, v34, v17
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v19, 16, v19
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v20, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v62, 24, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v74, 8, v18
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v19, 0x40c00000, v19
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v81, v33, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v79, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v90, 8, v18
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v83, v33, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_lshlrev_b32 v19, 16, v19
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v22
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v19, 16, 1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v19, 0x40c00000, v19 :: v_dual_lshlrev_b32 v22, 16, v22
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v20, v36, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v80, v34, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v84, v34, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v36
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v83.h
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v19, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v19
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v22, 0x40c00000, v22
-; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v19, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v20, v36, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v81.h
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v79, 8, v17
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v84, v33, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v94, 8, v17
+; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v19, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v22, 0x40c00000, v22 :: v_dual_cndmask_b32 v85, v33, v37
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v22, 16, 1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v35, 0x40c00000, v35
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v84.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v85.h
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v36, v20, v38, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v35, 16, 1
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v33, v22, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v33, 0x400000, v22
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v86, v20, v33 :: v_dual_add_f32 v35, 0x40c00000, v35
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v20, 0xffff, v34, v84
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v86.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v35, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v35
-; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v35, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v83, v20, v33, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v20, 0xffff, v34, v80
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v82, v19, v39, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v19, 0xffff, v37, v36
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v83.h
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v46, 24, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v60, 8, v20
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v24, 0x40c00000, v24
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v22, 0xffff, v22, v82
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v63, 8, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v36
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v72, 24, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v75, 8, v20
+; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v35, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v21
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v181, 24, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v40, 8, v22
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v33, 0x40c00000, v38
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v87, v19, v39, vcc_lo
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v19, 0xffff, v37, v36
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v24
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v21, 0x40c00000, v21
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v33, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v33, 0x40c00000, v38 :: v_dual_lshlrev_b32 v24, 16, v24
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v22, 0xffff, v22, v87
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v36, 0x40c00000, v36
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v21, 16, 1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v33, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v21
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v24, 0x40c00000, v24
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v33
-; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v33, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v86, v34, v37 :: v_dual_and_b32 v37, 0xffff0000, v23
+; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v33, 0x7fff
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v33
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v36
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v56, 24, v22
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v96, v34, v37 :: v_dual_and_b32 v37, 0xffff0000, v23
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v24, 16, 1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v24, 16, 1
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v36, 16, 1
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v58, 8, v22
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v21, v35, v38, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v24, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v24
+; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v24, 0x7fff
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v23, 0x40c00000, v23
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v24, v24
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v36, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v86.h
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v87, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v26
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v23, 0x40c00000, v23 :: v_dual_lshlrev_b32 v26, 16, v26
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v96.h
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v77, 8, v19
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v97, v34, v38, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v21, 0xffff, v35, v21
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v26, 0x40c00000, v26
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v37, 0x40c00000, v37
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v26, 16, v26
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v23, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v85, v33, v39, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v98, v33, v39, vcc_lo
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v37, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v36, 0x400000, v23
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v45, 8, v21
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v23, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v97, v34, v36, vcc_lo
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v37, 0x40c00000, v37 :: v_dual_add_f32 v34, 0x40c00000, v38
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v97.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v37, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v37
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v34
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v24, v24, v37, 0x7fff
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v37, v34, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v26, 0x40c00000, v26
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v21, 0xffff, v35, v21
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v99, v34, v36, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v34, 0x40c00000, v38
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v61, 8, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v99.h
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v23, v24, v39, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v37, v34, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v26
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v37, v34, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v34
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v23, 0xffff, v36, v23
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v87.h
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v182, 8, v23
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v97.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v37, v34, 0x7fff
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v26
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v24, 0xffff, v33, v85
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v24, 0xffff, v33, v98
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v26, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 24, v24
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v25, 0x40c00000, v25
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v46, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v183, 24, v24
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v26, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v26, v36, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v177, 8, v24
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v98, v33, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v42, 8, v24
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v100, v33, v37, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT:    v_add3_u32 v26, v26, v36, 0x7fff
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v96, v35, v38 :: v_dual_add_f32 v25, 0x40c00000, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v98.h
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v25, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v25
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_add_f32 v28, 0x40c00000, v28
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_add3_u32 v26, v26, v36, 0x7fff
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v101, v35, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v25, 0x7fff
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v28
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v36
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v100.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v102, v33, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_lshlrev_b32 v28, 16, v28
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v102.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v28, 0x40c00000, v28
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v35, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v35
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v101, v33, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v36, v26, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v28, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v25, v25, v35, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v36, v26, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v35
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v27, 0x40c00000, v27
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v26, v33, v28, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v33, 0x400000, v28
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v27
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v101.h
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v100, v26, v33, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v103, v26, v33, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v27, 0x40c00000, v27
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v26, 0xffff, v34, v96
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v100.h
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v99, v25, v39, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v25, 0xffff, v37, v36
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v33, 0x40c00000, v38
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v26, 0xffff, v34, v101
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v27, 16, 1
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v33, 0x40c00000, v38
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v103.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v112, v25, v39, vcc_lo
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v25, 0xffff, v37, v36
+; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v27, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v27
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v33, 16, 1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v27, 0x7fff
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v30
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v33, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v33
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v30, 0x40c00000, v30
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v28, 0xffff, v28, v112
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v113, v34, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v29
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v30, 16, v30
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v33, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v112, v34, v37, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v36, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v28, 0xffff, v28, v99
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v26
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v37, 0x40c00000, v37
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v30, 0x40c00000, v30 :: v_dual_lshlrev_b32 v29, 16, v29
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v27, v35, v38, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v36, 0x7fff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v29
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v29, 16, v29
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v36, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v36
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v30, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v30
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v112.h
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v28
-; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v30, 0x7fff
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v25
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v27, 0xffff, v35, v27
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v103, v34, v38 :: v_dual_and_b32 v38, 0xffff0000, v32
+; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v36, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v29, 0x40c00000, v29
+; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v30, 0x7fff
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v30, v37, 16, 1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v113.h
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v28
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v114, v34, v38, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v32
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v32, 16, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v27
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v29, 16, 1
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v102, v33, v39 :: v_dual_add_f32 v37, 0x40c00000, v37
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v36, 0x400000, v29
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v115, v33, v39, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-TRUE16-NEXT:    v_add3_u32 v30, v30, v37, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v29, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v103.h
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v30, v37, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v37
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v32, 0x40c00000, v32
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v113, v34, v36, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.l, v114.h
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v27, 0xffff, v35, v27
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v167, 24, v26
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v116, v34, v36, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT:    v_add3_u32 v30, v30, v37, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v34, 0x40c00000, v38
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v113.h
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v164, 8, v27
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v179, 8, v26
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v116.h
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v29, v30, v39, vcc_lo
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v37, v34, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v30, 0xffff, v33, v102
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v32, 16, 1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v30, 0xffff, v33, v115
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v34
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v181, 8, v25
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v29, 0xffff, v36, v29
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v31
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v32, 0x40c00000, v32 :: v_dual_lshlrev_b32 v31, 16, v31
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v37, v34, 0x7fff
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v30
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v31, 0x40c00000, v31
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v32, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v32
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v29
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v32, 0x7fff
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v31
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v30
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v115, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_lshlrev_b32 v31, 16, v31
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v117, v33, v37, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v29
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v115.h
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v31, 0x40c00000, v31
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v32, v36, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v114, v35, v38, vcc_lo
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v31, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v31
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX11-TRUE16-NEXT:    v_add3_u32 v32, v32, v36, 0x7fff
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v117.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v31, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v116, v33, v37 :: v_dual_and_b32 v35, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v118, v35, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v2
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v36, 0x40c00000, v36
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v31, v31
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v35, 0x40c00000, v35
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v32, v36, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v36
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v119, v33, v37, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v116.h
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v31, v35, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v35
+; GFX11-TRUE16-NEXT:    v_add3_u32 v32, v32, v36, 0x7fff
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v119.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_add3_u32 v31, v31, v35, 0x7fff
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v36, v32, v38, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v2, 16, 1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v31, v31, v35, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v35
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v1, 0x40c00000, v1
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v32, v33, v2, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v33, 0x400000, v2
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v133, v32, v33, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v33, 0x40c00000, v38
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v128, v32, v33, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v32, 0xffff, v34, v114
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v32, 0xffff, v34, v118
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v1, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v132, v31, v39, vcc_lo
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v33, 0x40c00000, v38
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v2.l, v128.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v129, v31, v39, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v31, 0xffff, v37, v36
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v4
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v33, 16, 1
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v1, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v33, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v33
-; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v33, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v2.l, v133.h
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 24, v32
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v146, v34, v37 :: v_dual_and_b32 v37, 0xffff0000, v3
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v131, v34, v37, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v2, 0xffff, v2, v132
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v37, 0x40c00000, v37
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v33, 0x7fff
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v3
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v4, 16, 1
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v1, v35, v38 :: v_dual_add_f32 v36, 0x40c00000, v36
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v1, v35, v38, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v4
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v4, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v3, 0x40c00000, v3
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v36, 16, 1
+; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v4, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v4, v37, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v148, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v3, 16, 1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v35.l, v131.h
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v2, 0xffff, v2, v129
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v36, 0x7fff
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v133, v34, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v6
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v3, 0x40c00000, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v1, 0xffff, v35, v1
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v105, 24, v2
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v3, 16, 1
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v135, v33, v39, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v36, 0x400000, v3
-; GFX11-TRUE16-NEXT:    v_add3_u32 v4, v4, v37, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v3, 0x7fff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v6
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v144, v33, v39, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v3, 0x7fff
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v106, 8, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v111, 8, v1
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v32
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v146, v34, v36 :: v_dual_add_f32 v37, 0x40c00000, v37
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v31
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v36.l, v146.h
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v4, v37, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v37
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v33.l, v148.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v35.l, v146.h
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v105, 24, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v164, v34, v36, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v34, 0x40c00000, v38
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v1, 0xffff, v35, v1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v107, 8, v2
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v36.l, v164.h
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v3, v4, v39, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v37, v34, 16, 1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add3_u32 v4, v4, v37, 0x7fff
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v3, v4, v39 :: v_dual_add_f32 v34, 0x40c00000, v38
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v7
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v34
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v3, 0xffff, v36, v3
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff0000, v5
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v33.l, v133.h
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v37, v34, 16, 1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v4, 0xffff, v33, v144
-; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v37, v34, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v7, 0x40c00000, v7
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v90, 24, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v94, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v106, 8, v3
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v36, 0x40c00000, v36
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v4, 0xffff, v33, v135
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v6, 16, 1
+; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v37, v34, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v6
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v108, 8, v1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v32
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v34
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v6, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v6, v36, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v31
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v165, v33, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v93, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v95, 8, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v104, 8, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v150, v33, v37, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v34, v34
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v5, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v5
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v6, v6, v36, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v34.l, v165.h
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v161, v35, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v151, v35, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v33, v33, v5, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v180, v33, v37 :: v_dual_add_f32 v35, 0x40c00000, v35
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v34.l, v150.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v166, v33, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_lshlrev_b32 v8, 16, v8
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v37.l, v166.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v37.l, v180.h
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v5, v35, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v8, 16, 1
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v36, v6, v38, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v35
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v33, v8, 16, 1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v5, v5, v35, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v6, v33, v8, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v33, 0x400000, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v179, v6, v33, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v177, v6, v33, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v6, 0xffff, v34, v161
-; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v8.l, v179.h
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v178, v5, v38 :: v_dual_add_f32 v33, 0x40c00000, v39
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v33, 0x40c00000, v39
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v6, 0xffff, v34, v151
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v178, v5, v38, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v5, 0xffff, v37, v36
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v36, 16, v10
-; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v7, 0x7fff
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v36, 16, v10
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v34, v33, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v33
+; GFX11-TRUE16-NEXT:    v_mov_b16_e64 v8.l, v177.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v36, 0x40c00000, v36
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v7, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v7
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v33
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v34, v34, v33, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v8, 0xffff, v8, v178
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v47, v35, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v7, 0x7fff
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v7, v36, 16, 1
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v76, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v60, 24, v8
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v43, v35, v37, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v36
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v59, 24, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v72, 8, v8
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v7, v36, 0x7fff
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v36
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v63, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v78, 8, v6
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v33, v34, v38, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v47.h
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v75, 24, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v88, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v91, 8, v5
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v44, v7, v37, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v9, 0x40c00000, v39
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v7, 0x40c00000, v7
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v36, v9, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v9
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v49, 0x400000, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
-; GFX11-TRUE16-NEXT:    v_add3_u32 v36, v36, v9, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v10, 0x40c00000, v10
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v43.h
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v88, 8, v5
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v41, v7, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v10, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v10
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v41, v35, v38 :: v_dual_lshlrev_b32 v10, 16, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v12
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v44, v35, v38 :: v_dual_and_b32 v39, 0xffff0000, v9
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v37, 0x40c00000, v10
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v44.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v41.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v38, v37, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v50, 0x400000, v37
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v10, 0xffff, v35, v41
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v7, 16, 1
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v10, 0xffff, v35, v44
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v38, v38, v37, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v37, 0x40c00000, v51
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v42, 24, v10
-; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v61, v38, v50 :: v_dual_add_f32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v45, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v47, 8, v10
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v59, v38, v50, vcc_lo
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v7, 16, v9
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v61.h
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v37, 0x40c00000, v51 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v7, 0x40c00000, v7
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v48, v12, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v52, 0x400000, v12
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v14, 0x40c00000, v14
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v7, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v49, 0x400000, v7
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT:    v_add3_u32 v48, v48, v12, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v35, v7, 0x7fff
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v59.h
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v73, v35, v49, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v9, 0x40c00000, v39
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v35, v37, 16, 1
-; GFX11-TRUE16-NEXT:    v_add3_u32 v48, v48, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v56, 8, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v57, v48, v52, vcc_lo
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v49, v14, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v12, 0xffff, v7, v57
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v9, v36, v39, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v62, v48, v52, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v36, v9, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v39, 0x400000, v9
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v12, 0xffff, v7, v62
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_add3_u32 v36, v36, v9, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v7, 0x40c00000, v11
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v35, v37, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v35, 0x400000, v37
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v180, 24, v12
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v9, v36, v39, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v73.h
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v39, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v167, 24, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v183, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v182, 8, v12
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v11, v35, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v35, 0x40c00000, v38
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v37, v39, v7, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v7
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff0000, v13
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v48, v35, 16, 1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v9, 0xffff, v36, v9
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v7, 0x40c00000, v39
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v89, v37, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v7, 0x40c00000, v39 :: v_dual_cndmask_b32 v92, v37, v38
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v37, v48, v35, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v35
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v35, v35
@@ -160898,18 +160390,18 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v48, 0x400000, v14
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v49, v7, 16, 1
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v77, v37, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v89, v37, v38, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v35, 0x400000, v7
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v14, v49, v7, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff0000, v16
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v78, v39, v48, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v91, v39, v48, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v39, v13, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v78.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v73.h
 ; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v7, v14, v35 :: v_dual_add_f32 v14, 0x40c00000, v37
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v37, 16, v15
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v39, v13, 0x7fff
@@ -160919,7 +160411,7 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v37, 0x40c00000, v37
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v49, 0x400000, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v104, v35, v39, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v108, v35, v39, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v13, v16, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v39, v37, 16, 1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
@@ -160927,405 +160419,366 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v15, 0x40c00000, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v51, 0x400000, v37
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v39, v39, v37, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v93, v13, v49, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v107, v13, v49, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v37, v37
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v35, v48, v14, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v48, 0x400000, v14
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v50, v15, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v95, v39, v51, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v110, v39, v51, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v104.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v108.h
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v50, v15, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v89.h
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v14, 0xffff, v38, v77
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v92, v35, v48, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v91.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v92.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v109, v35, v48, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v95.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v93.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v110.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v107.h
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v14, 0xffff, v38, v89
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v11, 0xffff, v39, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v14
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v13, v13, v16, vcc_lo
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v166, 8, v14
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v16, 0xffff, v35, v92
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[52:53], 24, v[11:12]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[17:18]
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v9, 0xffff, v36, v9
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v16, 0xffff, v35, v109
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[23:24]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[11:12]
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v15, 0xffff, v15, v13
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v13, 0xffff, v37, v7
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v7, 0xffff, v34, v33
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[33:34], 24, v[31:32]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[34:35], 24, v[29:30]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[13:14]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[7:8]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[5:6]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[69:70], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[35:36], 24, v[27:28]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[15:16]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[7:8]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[5:6]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[15:16]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[70:71], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[36:37], 24, v[25:26]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[49:50], 24, v[13:14]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[3:4]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[37:38], 24, v[23:24]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[1:2]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[21:22]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[19:20]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[51:52], 24, v[21:22]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[19:20]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[17:18]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v165, 8, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v176, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v43, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v58, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v76, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v40, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v57, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v74, 8, v7
 ; GFX11-TRUE16-NEXT:  .LBB90_4: ; %end
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v146.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v108.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v131.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v111.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v133.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v107.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v68.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v132.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v2.h, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v129.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v1.l, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v128.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v6.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v106.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v164.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v1.h, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v105.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v2.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v94.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v91.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v148.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v67.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v8, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v180.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v90.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v144.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v4.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v165.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v88.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v8, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v5.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v47.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.h, v6.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v76.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v58.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v75.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v161.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v6.h, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v179.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v72.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v6.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v105.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v2.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v146.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v104.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v4.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v95.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v2
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v135.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v133.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v93.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v5.l, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v166.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v88.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v5.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v6, v4
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v78.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v150.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v151.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v76.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v66, v6, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v43.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v74.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v65.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v8.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v73.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v178.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v59.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v8.h, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v56.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v8.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v44.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v43.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v89.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v41.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v42.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v10.h, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v16, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v10.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v61.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v183.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v16, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v11.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v104.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v11.h, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v176.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v166.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v16, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v167.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v57.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v12.h, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v78.h
-; GFX11-TRUE16-NEXT:    s_clause 0x1
-; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
-; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v12.l, v14.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v12.h, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v16, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v160.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v77.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v95.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v150.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v64, v18, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v93.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v149.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v65, v18, v14
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v71.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v79.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v92.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v134.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v13.h, v15.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v66, v18, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v70.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v74.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v67, v18, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v46.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v15.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v84.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v63.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v16, v14
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v55.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v62.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v81.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v60.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v13.h, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v20, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v86.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v80.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v45.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v19, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v20
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v18.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v13.h, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v83.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v40.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v19, v14
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v97.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v182.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v82.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.l, 8, v181.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v13.h, v19.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v22, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v87.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v177.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v22, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v147.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v13.h, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v21.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v101.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v163.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v22, v14
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v85.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 8, v162.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v98.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v26, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v13.h, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v26, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v28
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v112.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v96.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v145.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v25, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v26
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v24.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v13.h, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v100.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v135.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v25, v14
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v113.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v131.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v26
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v99.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v26.l, 8, v130.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v13.h, v25.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v28, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v30
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v103.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v26.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v129.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v67.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v62.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v67, v6, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v177.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v7.l, v7.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v63.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v178.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v60.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v180.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v68, v6, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v73.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v7.l, v7.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v57.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v69, v6, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v41.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.l, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v47.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v44.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v45.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v6, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v92.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.l, v8.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v40.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.l, v10.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v6, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v59.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v182.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v50.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v6, v9
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v89.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v108.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v176.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.l, v11.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v6, v10
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v163.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v91.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v165.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.l, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v6, v11
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v110.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v161.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v38.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v109.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v6, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v147.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.l, v13.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v107.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v149.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.l, v14.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v6, v13
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v82.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v94.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v81.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v6, v14
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v79.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.l, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v90.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.l, v16.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v6, v15
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v19.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v85.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v77.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v84.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v6, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v72.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.l, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v83.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v75.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.l, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v6, v17
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v96.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v61.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v87.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v6, v18
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v56.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.l, v19.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v86.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v58.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.l, v20.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v6, v19
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v23.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v99.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v46.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v98.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v6, v20
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v183.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.l, v21.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v97.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v42.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.l, v22.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v6, v21
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v102.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v181.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v36.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v101.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v6, v22
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v167.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.l, v23.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v100.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v179.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.l, v24.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v6, v23
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v113.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v164.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v112.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v6, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v26.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.l, v25.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v103.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.l, v26.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v6, v25
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v29.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v116.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v148.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v34.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v28, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v30
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v13.h, v27.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v27.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v116.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v128.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v27.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v28, v14
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v102.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.l, 8, v119.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v32, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v115.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v118.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v28.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v115.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v6, v26
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v144.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.l, v27.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v114.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v145.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.l, v28.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v6, v27
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v13.h, v28.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v32, v14
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v31, 0xffff, v34
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v114.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v30, 0xffff, v30
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v31, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.l, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v119.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v134.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v33.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v118.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v6, v28
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v130.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.l, v29.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v117.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v132.l
+; GFX11-TRUE16-NEXT:    s_clause 0x1
+; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
+; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[66:69], off offset:16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v6, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v30.l, v30.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v30, v14
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v6, v5
 ; GFX11-TRUE16-NEXT:    s_clause 0x5
-; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[9:12], off offset:32
-; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[64:67], off offset:48
+; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[7:10], off offset:32
+; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[11:14], off offset:48
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[15:18], off offset:64
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[19:22], off offset:80
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[23:26], off offset:96
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[27:30], off offset:112
 ; GFX11-TRUE16-NEXT:    s_clause 0x1f
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v108, off, s32 offset:12
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v107, off, s32 offset:16
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v106, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v105, off, s32 offset:24
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v104, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v95, off, s32 offset:32
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v94, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v93, off, s32 offset:40
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v92, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v91, off, s32 offset:48
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v90, off, s32 offset:52
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v89, off, s32 offset:56
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v88, off, s32 offset:60
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v79, off, s32 offset:64
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v78, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v77, off, s32 offset:72
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v76, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v75, off, s32 offset:80
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v74, off, s32 offset:84
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v73, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v72, off, s32 offset:92
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v63, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v62, off, s32 offset:100
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v61, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v60, off, s32 offset:108
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v59, off, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v58, off, s32 offset:116
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v57, off, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v56, off, s32 offset:124
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v47, off, s32 offset:128
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v46, off, s32 offset:132
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v45, off, s32 offset:136
-; GFX11-TRUE16-NEXT:    s_clause 0x4
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v44, off, s32 offset:140
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v43, off, s32 offset:144
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v42, off, s32 offset:148
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v41, off, s32 offset:152
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v40, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v111, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v110, off, s32 offset:16
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v109, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v108, off, s32 offset:24
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v107, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v106, off, s32 offset:32
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v105, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v104, off, s32 offset:40
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v95, off, s32 offset:44
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v94, off, s32 offset:48
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v93, off, s32 offset:52
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v92, off, s32 offset:56
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v91, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v90, off, s32 offset:64
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v89, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v88, off, s32 offset:72
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v79, off, s32 offset:76
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v78, off, s32 offset:80
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v77, off, s32 offset:84
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v76, off, s32 offset:88
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v75, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v74, off, s32 offset:96
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v73, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v72, off, s32 offset:104
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v63, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v62, off, s32 offset:112
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v61, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v60, off, s32 offset:120
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v59, off, s32 offset:124
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v58, off, s32 offset:128
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v57, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v56, off, s32 offset:136
+; GFX11-TRUE16-NEXT:    s_clause 0x7
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v47, off, s32 offset:140
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v46, off, s32 offset:144
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v45, off, s32 offset:148
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v44, off, s32 offset:152
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v43, off, s32 offset:156
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v42, off, s32 offset:160
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v41, off, s32 offset:164
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v40, off, s32 offset:168
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -185302,69 +184755,69 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v33, off, s32 offset:8
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v32, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr165_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr164_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr161_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v33
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
@@ -185375,69 +184828,69 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[33:34], 24, v[31:32]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[34:35], 24, v[29:30]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[5:6]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[35:36], 24, v[27:28]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[7:8]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[13:14]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[11:12]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[69:70], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[36:37], 24, v[25:26]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[15:16]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[5:6]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[37:38], 24, v[23:24]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[49:50], 24, v[13:14]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[52:53], 24, v[11:12]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[3:4]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[21:22]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 24, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 24, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 24, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 8, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v27
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 24, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 24, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v21
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 24, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[1:2]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[19:20]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[17:18]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v164, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v165, 8, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v166, 8, v1
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 24, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v27
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[15:16]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[70:71], 24, v[1:2]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[23:24]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[51:52], 24, v[21:22]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[19:20]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[17:18]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v17
 ; GFX11-TRUE16-NEXT:  .LBB94_2: ; %Flow
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB94_4
@@ -185446,405 +184899,364 @@ define <128 x i8> @bitcast_v64f16_to_v128i8(<64 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v32, 0x200, v32 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v31, 0x200, v31 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v30, 0x200, v30 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v29, 0x200, v29 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v28, 0x200, v28 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v7, 0x200, v7 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v30, 0x200, v30 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v29, 0x200, v29 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v6, 0x200, v6 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v5, 0x200, v5 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v28, 0x200, v28 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v4, 0x200, v4 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v3, 0x200, v3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v14, 0x200, v14 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v20, 0x200, v20 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v22, 0x200, v22 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v21, 0x200, v21 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v24, 0x200, v24 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v23, 0x200, v23 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v25, 0x200, v25 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
 ; GFX11-TRUE16-NEXT:    v_pk_add_f16 v1, 0x200, v1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[33:34], 24, v[31:32]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[5:6]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[34:35], 24, v[29:30]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[7:8]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[13:14]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[11:12]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[69:70], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[35:36], 24, v[27:28]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[15:16]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[5:6]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[15:16]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[70:71], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[36:37], 24, v[25:26]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[49:50], 24, v[13:14]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[52:53], 24, v[11:12]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[3:4]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[37:38], 24, v[23:24]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[1:2]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[21:22]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[19:20]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[17:18]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 24, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[23:24]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[51:52], 24, v[21:22]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[19:20]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[17:18]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 24, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 24, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 8, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v27
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 24, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 24, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v21
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 24, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v164, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v165, 8, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v166, 8, v1
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 24, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v27
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v17
 ; GFX11-TRUE16-NEXT:  .LBB94_4: ; %end
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v163.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v166.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v34.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v1.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v165.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v164.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v39, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v2.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v163.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v69.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v161.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v1.h, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v160.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff, v68
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v54, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v67.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v39, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v3.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v161.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v150.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v68, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v67
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v4.l, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v148.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v54, v51
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v39, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v151.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v39, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v5.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v149.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v147.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v147.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.h, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v67
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v39, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v6.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v145.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v67.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v39, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v7.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v135.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v133.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v39, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v131.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v146.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v66
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v65.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v8.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v132.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v66
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v65
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v8.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v130.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v8.h, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v39, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v9.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v129.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v119.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v39, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v10.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v39, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v11.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v115.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v113.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v54, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v119.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v65
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v118.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v39, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v12.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v50.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v10.l, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v116.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v112.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v11.h, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v39, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v13.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v99.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v39, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v14.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v39, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v15.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v85.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v103.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v39, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v16.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v39, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v17.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v150.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v99.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v97.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v49, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v14.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v39, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v18.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v148.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v54.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v39, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v19.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v144.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v39, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v20.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v134.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v51.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v49, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v48, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v145.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v39, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v135.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v17.h, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v39, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v21.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v132.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v130.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v39, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v22.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v39, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v23.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v118.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v116.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v39, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v19.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v134.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v133.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v39, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v24.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v114.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v36.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v18.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v129.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v39, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v19.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v128.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v39, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v20.h, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v39, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v25.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v112.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v39, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v26.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v35.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v39, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v27.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v96.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v39, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v22.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v115.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v21.l, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v39, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v28.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v39, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v29.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v84.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v83.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v38, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v22.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v113.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v38, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v101.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v23.h, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v39, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v30.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v82.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v33.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v39, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v31.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v33.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v81.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v80.l
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v37, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v25.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v100.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v98.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v24.l, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v37, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v87.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v25.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v86.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v36, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v85.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v26.h, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v36, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v28.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v27.l, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v82.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v34.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v35, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v28.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.l, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v29.h, v30.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.l, v31.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v71.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v31.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v30.l, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v31.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v32.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v51
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v31.l, v31.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v32.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v32.h, 8, v55.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v34
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v32.l, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v33, v51
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v39, v31
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v32.l, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v32.h, v33.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v39.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v39, v32
 ; GFX11-TRUE16-NEXT:    s_clause 0x5
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[9:12], off offset:32
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[13:16], off offset:48
@@ -208055,69 +207467,69 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v33, off, s32 offset:8
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v32, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr166_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr165_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr164_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr161_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr151_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr149_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr147_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr131_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr119_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr145_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr160_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr150_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr148_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr144_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr135_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr133_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr132_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr130_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr128_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr116_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr114_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr115_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(2)
 ; GFX11-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v33
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
@@ -208128,69 +207540,69 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[33:34], 24, v[31:32]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[34:35], 24, v[29:30]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[5:6]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[35:36], 24, v[27:28]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[7:8]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[13:14]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[11:12]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[69:70], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[36:37], 24, v[25:26]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[15:16]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[5:6]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[37:38], 24, v[23:24]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[49:50], 24, v[13:14]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[52:53], 24, v[11:12]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[3:4]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[21:22]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 24, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 24, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 24, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 8, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v27
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 24, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 24, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v21
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 24, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[1:2]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[19:20]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[17:18]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v164, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v165, 8, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v166, 8, v1
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 24, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v27
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[15:16]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[70:71], 24, v[1:2]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[23:24]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[51:52], 24, v[21:22]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[19:20]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[17:18]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v17
 ; GFX11-TRUE16-NEXT:  .LBB98_2: ; %Flow
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB98_4
@@ -208199,405 +207611,364 @@ define <128 x i8> @bitcast_v64i16_to_v128i8(<64 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v32, v32, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v31, v31, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v30, v30, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v7, v7, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v30, v30, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v29, v29, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v6, v6, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v5, v5, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v28, v28, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v4, v4, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v3, v3, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v20, v20, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v22, v22, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v21, v21, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v24, v24, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v23, v23, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v25, v25, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
 ; GFX11-TRUE16-NEXT:    v_pk_add_u16 v1, v1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_pk_add_u16 v15, v15, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[33:34], 24, v[31:32]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[5:6]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[34:35], 24, v[29:30]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[7:8]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[13:14]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[11:12]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[64:65], 24, v[9:10]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[69:70], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[35:36], 24, v[27:28]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[15:16]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[66:67], 24, v[5:6]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[15:16]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[70:71], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[36:37], 24, v[25:26]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[49:50], 24, v[13:14]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[52:53], 24, v[11:12]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[67:68], 24, v[3:4]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[37:38], 24, v[23:24]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[68:69], 24, v[1:2]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[38:39], 24, v[21:22]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[50:51], 24, v[19:20]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[53:54], 24, v[17:18]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 24, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[48:49], 24, v[23:24]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[51:52], 24, v[21:22]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[54:55], 24, v[19:20]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[65:66], 24, v[17:18]
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 24, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 24, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 24, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 8, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v119, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 8, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v131, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 24, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 24, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 8, v5
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 24, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 24, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v2
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 8, v32
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v31
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v30
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 8, v29
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v85, 8, v28
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v27
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 24, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v26
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v25
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 24, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v24
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v115, 8, v23
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 24, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v117, 8, v22
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v21
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v129, 24, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 8, v20
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 24, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v18
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v17
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v133, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v135, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v145, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v147, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v149, 8, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v151, 8, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v161, 24, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v162, 8, v4
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v163, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v164, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v165, 8, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v166, 8, v1
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 24, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v32
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 8, v31
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 24, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v30
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v86, 8, v29
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v28
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 8, v27
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v26
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v114, 8, v25
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v116, 24, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v118, 8, v24
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v128, 8, v23
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v130, 24, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v132, 8, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v134, 8, v21
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v144, 24, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v146, 8, v20
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v148, 8, v19
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v150, 24, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v160, 8, v18
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v17
 ; GFX11-TRUE16-NEXT:  .LBB98_4: ; %end
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v163.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v166.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v68.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v34.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v1.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v165.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v39.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v164.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v39, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v2.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v163.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v69.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v161.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v1.h, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v160.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v151.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v68, 0xffff, v68
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v54, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v67.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v39, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v3.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v162.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v161.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v150.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v68, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v149.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v67
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v66.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v4.l, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v148.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v54, v51
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v39, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v4.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v151.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v39, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v5.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v149.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v147.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v5.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v147.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v5.h, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v67
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v39, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v6.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v145.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v67.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v39, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v7.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v135.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v133.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v39, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v8.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v131.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v146.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v144.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v66
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v65.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v8.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v132.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v66
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v131.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v65
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v8.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v130.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v8.h, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v39, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v9.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v129.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v119.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v39, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v10.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v39, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v11.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v115.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v113.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v54, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v119.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v65
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v118.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v39, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v12.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v103.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v50.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v10.l, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v116.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v54, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v64
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v11.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v112.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v11.h, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v39, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v13.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v99.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v39, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v14.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v39, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v15.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v87.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v85.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v103.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v39, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v16.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v18.h, 0xff, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v39, v16
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v17.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v17.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v160.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v150.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v52, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v54
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v99.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v13.l, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v97.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v49, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v14.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v14.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v19.h, 0xff, v19.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v39, v17
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v18.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v18.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v148.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v54.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v39, v18
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v19.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v19.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v146.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v144.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, 0xff, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v39, v19
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v20.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v20.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v134.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v51.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v49, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v48, 0xffff, v52
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v84.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v17.l, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v39
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v48, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v145.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v39, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v17.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v135.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v17.h, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v22.h, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v39, v20
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v21.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.h, v21.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v132.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v130.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v23.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v39, v21
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v22.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v128.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v39, v22
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v23.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v23.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v118.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v116.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, v39, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v19.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v134.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v133.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, 0xff, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v39, v23
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v24.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v114.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v36.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v18.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v19.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.l, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v129.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v18, v39, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v19.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v128.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, v39, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v39, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v20.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v117.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v20.h, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v39, v24
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v25.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v112.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v27.h, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v39, v25
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v26.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v26.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v100.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v35.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v39, v26
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v27.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.h, v27.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v98.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v96.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v20, v39, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.l, v22.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v21.l, 0xff, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v115.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v48
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v21.l, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v114.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.l, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v39, v27
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v28.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v28.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v86.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.h, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v39, v28
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v29.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v29.h, v29.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v84.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v83.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, v38, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v22.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.l, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v23.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v113.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, v38, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v39
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v23.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v101.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v23.h, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v31.h, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v39, v29
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v30.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v30.h, v30.h, v34.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v82.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v33.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v32.h, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v39, v30
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v31.l, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v31.h, v31.h, v33.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.l, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v81.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v80.l
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, v37, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.l, v25.l, v33.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v100.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v98.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v37, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v24.l, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.l, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v26.l, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v38
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, v37, v51
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v87.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v25.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v26.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v26.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v86.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, v36, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v36, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v26.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v85.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v26.h, v27.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v26, v36, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v28.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.l, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v27.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v83.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v37
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v27.l, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v82.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v29.l, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v34.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v27, v35, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v35, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v28.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.l, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v29.h, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.l, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v28, v35, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v36
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v29.l, v33.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v30.l, v29.h, v30.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v29, v34, v51
-; GFX11-TRUE16-NEXT:    v_or_b16 v31.l, v31.l, v33.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v30.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v71.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v34, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v31.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v30.l, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v31.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v31.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.l, v32.l, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v35
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v30, v34, v51
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v31.l, v31.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.l, v32.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v32.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v32.h, 8, v55.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v33, v51
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v33, 0xffff, v34
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v51.h, v32.l, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v33, v51
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v31, v39, v31
+; GFX11-TRUE16-NEXT:    v_or_b16 v39.l, v32.l, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v32.h, v33.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, v39.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v32, v39, v32
 ; GFX11-TRUE16-NEXT:    s_clause 0x5
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[9:12], off offset:32
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[13:16], off offset:48
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 3e96ab1d597d6..21ec3ee1996a6 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -4118,19 +4118,19 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.h, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v7.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v7.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v15.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v16
@@ -4144,103 +4144,95 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB26_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v8.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.h, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v0.l, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v1.h, v6.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v2.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v3.l, v4.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v9
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v5.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v11, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v2.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v11, v3
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX11-TRUE16-NEXT:  .LBB26_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v10.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v9.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v12.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v8.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v9.l, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v7.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v6.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v6.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v9, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v5.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.h, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v9, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v9.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v4.h, v2.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v11
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8592,19 +8584,19 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.h, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v7.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v7.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v15.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v16
@@ -8618,103 +8610,95 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB50_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v8.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.h, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v0.l, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v1.h, v6.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v2.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v3.l, v4.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v9
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v5.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v11, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v2.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v11, v3
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB50_2
 ; GFX11-TRUE16-NEXT:  .LBB50_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v10.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v9.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v12.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v8.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v9.l, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v7.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v6.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v6.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v9, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v5.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.h, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v9, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v9.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v4.h, v2.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v11
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -12682,19 +12666,19 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.h, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v7.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v7.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v15.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v16
@@ -12708,103 +12692,95 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB70_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v8.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.h, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v0.l, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v1.h, v6.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v2.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v3.l, v4.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v9
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v5.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v11, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v2.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v11, v3
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB70_2
 ; GFX11-TRUE16-NEXT:  .LBB70_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v10.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v9.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v12.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v8.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v9.l, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v7.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v6.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v6.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v9, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v5.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.h, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v9, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v9.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v4.h, v2.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v11
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -16382,19 +16358,19 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.h, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v7.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v7.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v15.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v16
@@ -16408,103 +16384,95 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB86_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v8.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.h, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v0.l, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v1.h, v6.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v2.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v3.l, v4.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v9
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v5.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v11, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v2.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v11, v3
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB86_2
 ; GFX11-TRUE16-NEXT:  .LBB86_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v10.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v9.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v12.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v8.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v9.l, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v7.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v6.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v6.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v9, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v5.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.h, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v9, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v9.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v4.h, v2.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v11
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -19811,19 +19779,19 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.h, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v7.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v7.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v15.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v16
@@ -19837,103 +19805,95 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB98_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v8.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.h, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v0.l, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v1.h, v6.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v2.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v3.l, v4.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v9
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v5.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v11, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v2.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v11, v3
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB98_2
 ; GFX11-TRUE16-NEXT:  .LBB98_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v10.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v9.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v12.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v8.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v9.l, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v7.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v6.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v6.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v9, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v5.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.h, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v9, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v9.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v4.h, v2.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v11
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -22725,19 +22685,19 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.h, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v7.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v7.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v15.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v16
@@ -22751,103 +22711,95 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB106_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v8.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.h, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v0.l, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v1.h, v6.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v2.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v3.l, v4.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v9
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v5.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v11, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v2.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v11, v3
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB106_2
 ; GFX11-TRUE16-NEXT:  .LBB106_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v10.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v9.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v12.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v8.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v9.l, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v7.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v6.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v6.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v9, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v5.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.h, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v9, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v9.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v4.h, v2.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v11
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -24944,19 +24896,19 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.h, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v7.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v12.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.h, 8, v7.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v15.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v16
@@ -24970,103 +24922,95 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB110_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v8.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.h, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v0.l, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v11, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v1.h, v6.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v1.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v2.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v3.l, v4.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v9
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v5.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v11, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v2.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v11, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v11.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v11, v3
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB110_2
 ; GFX11-TRUE16-NEXT:  .LBB110_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v10.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v9.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v12.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v8.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v9.l, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v7.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v6.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v6.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v9, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v5.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.h, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v9, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v9.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v4.h, v2.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v4, v11
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v9, v3
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index f8ffaa456c2b3..38302a75fe26d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -6296,32 +6296,31 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.h, v19.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.h, v17.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v12.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.h, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v22.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v23.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v21.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v31.l
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v32
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB26_3
@@ -6333,194 +6332,175 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB26_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v19.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v0.h, v15.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v1.h, v14.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v2.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v3.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v12, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v4.l, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v0.l, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v19.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v21, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v1.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v21, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v2.l, v14.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v21, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v3.l, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v13.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v10.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v21, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v4.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v5.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v9.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v6.l, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v21, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v5.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v8.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v6.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v21.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v9, v19
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v7
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX11-TRUE16-NEXT:  .LBB26_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v21.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v19.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v20.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v18.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v16.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v15.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v19.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v17.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v19.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v19.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v18.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v17.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v15.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v13.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v15.h, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v16.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v21, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v14.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v15.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v14.h, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v16.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v13.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v12.l, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v12.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v18.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v11.h, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v21, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v12.h, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v13.l, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v16.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v18.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v21, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v22.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v11.l, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v11.h, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v10.h, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v21
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v11.l, v4.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v24.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v22.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v9.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v21, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v10.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v10.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v21, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v9.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v9.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v9.h, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v5.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v8.h, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v21
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v8.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v21
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v8, v21
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v8.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v8.h, v6.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v7
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -13335,32 +13315,31 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.h, v19.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.h, v17.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v12.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.h, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v22.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v23.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v21.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v31.l
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v32
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB50_3
@@ -13372,194 +13351,175 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB50_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v19.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v0.h, v15.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v1.h, v14.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v2.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v3.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v12, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v4.l, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v0.l, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v19.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v21, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v1.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v21, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v2.l, v14.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v21, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v3.l, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v13.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v10.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v21, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v4.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v5.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v9.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v6.l, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v21, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v5.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v8.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v6.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v21.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v9, v19
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v7
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB50_2
 ; GFX11-TRUE16-NEXT:  .LBB50_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v21.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v19.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v20.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v18.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v16.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v15.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v19.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v17.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v19.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v19.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v18.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v17.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v15.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v13.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v15.h, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v16.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v21, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v14.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v15.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v14.h, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v16.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v13.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v12.l, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v12.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v18.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v11.h, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v21, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v12.h, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v13.l, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v16.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v18.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v21, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v22.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v11.l, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v11.h, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v10.h, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v21
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v11.l, v4.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v24.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v22.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v9.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v21, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v10.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v10.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v21, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v9.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v9.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v9.h, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v5.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v8.h, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v21
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v8.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v21
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v8, v21
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v8.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v8.h, v6.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v7
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -19892,32 +19852,31 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.h, v19.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.h, v17.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v12.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.h, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v22.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v23.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v21.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v31.l
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v32
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB70_3
@@ -19929,194 +19888,175 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB70_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v19.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v0.h, v15.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v1.h, v14.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v2.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v3.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v12, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v4.l, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v0.l, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v19.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v21, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v1.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v21, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v2.l, v14.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v21, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v3.l, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v13.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v10.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v21, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v4.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v5.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v9.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v6.l, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v21, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v5.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v8.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v6.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v21.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v9, v19
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v7
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB70_2
 ; GFX11-TRUE16-NEXT:  .LBB70_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v21.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v19.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v20.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v18.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v16.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v15.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v19.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v17.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v19.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v19.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v18.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v17.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v15.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v13.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v15.h, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v16.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v21, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v14.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v15.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v14.h, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v16.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v13.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v12.l, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v12.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v18.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v11.h, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v21, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v12.h, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v13.l, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v16.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v18.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v21, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v22.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v11.l, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v11.h, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v10.h, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v21
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v11.l, v4.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v24.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v22.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v9.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v21, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v10.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v10.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v21, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v9.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v9.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v9.h, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v5.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v8.h, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v21
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v8.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v21
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v8, v21
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v8.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v8.h, v6.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v7
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -25939,32 +25879,31 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.h, v19.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.h, v17.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v12.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.h, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v22.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v21.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v23.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v21.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v31.l
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v32
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB86_3
@@ -25976,194 +25915,175 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB86_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v19.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v0.h, v15.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v1.h, v14.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v2.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v3.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v12, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v4.l, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v0.l, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v19.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v21, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v1.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v21, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v2.l, v14.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v21, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v3.l, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v13.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v10.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v21, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v4.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v5.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v9.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v6.l, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v19.h, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v21, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v5.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v8.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v6.l, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v21.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v21.l, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v9, v19
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v7
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB86_2
 ; GFX11-TRUE16-NEXT:  .LBB86_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v21.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v19.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v20.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v18.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v16.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v15.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v19.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v17.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v19.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v19.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v18.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v17.h, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v15.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v13.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v15.h, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v16.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v12.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v21, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v14.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v14.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v15.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v14.h, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v16.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v13.l, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v14.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v12.l, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v12.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v18.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v11.h, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v21
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v21, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v12.h, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v13.l, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v16.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v18.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v21, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v22.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v11.l, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v11.h, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v10.h, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v21
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v11.l, v4.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v24.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v22.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v9.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v21, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v10.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v10.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v21, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v9.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v9.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v9.h, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v5.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v8.h, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v21
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v8.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v21
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v8, v21
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v21, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v8.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v8.h, v6.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v21, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v21.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v7
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index 0cefbc1c2dee5..436b1a038b274 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -2966,20 +2966,20 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
@@ -2995,17 +2995,17 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB12_2: ; %Flow
@@ -3029,17 +3029,17 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB12_4: ; %end
@@ -3047,105 +3047,93 @@ define <40 x i8> @bitcast_v10i32_to_v40i8(<10 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v30.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v1.l, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v15, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v14.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v1.h, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v15, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v16, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v28, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v15, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v15, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v5.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v15, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v6.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v15, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v7.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v11, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v15
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v15, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v8.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v15, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v11.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v9.h, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v11
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v10.l, v10.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v9
 ; GFX11-TRUE16-NEXT:    s_clause 0x2
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[10:11], off offset:32
+; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[11:12], off offset:32
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: bitcast_v10i32_to_v40i8:
@@ -5038,48 +5026,49 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:24
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v25.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.h, v23.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.h, v21.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.h, v17.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.h, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.h, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 8, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v30.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v29.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v28.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v35.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v30.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v29.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v28.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v27.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(8)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v33.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v33.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v33.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v34.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v34.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v34.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v35.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v35.l
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v36
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB14_3
@@ -5092,245 +5081,217 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB14_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v25.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v24.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v0.h, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v21.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v19.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v1.h, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v2.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v3.l, v15.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v0.l, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v25.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v23.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v27, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v1.h, v23.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v27, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v2.l, v20.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v27, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v3.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v14.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v27, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v4.l, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v27, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v5.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v31.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v27, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v6.l, v13.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v27.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v4.l, v14.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v14.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v25
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v5.l, v13.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v6.h, v13.l
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v6.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v7.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v8.l, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v25
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v9.l, v10.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v10.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v27, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v7.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v27.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v27, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v8.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v27.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v27, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v9.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v27.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v11, v25
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v27, v9
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB14_2
 ; GFX11-TRUE16-NEXT:  .LBB14_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v26.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v25.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v25.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v22.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v23.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v22.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v21.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v18.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v24.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v23.l, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v20.h, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v21.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v24.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v25.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v23.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v23.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v19.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v19.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v25, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v16.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v20.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v21.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v21.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v19.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v16.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v17.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v19.h, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v16.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v15.h, v3.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v27
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v17.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v18.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v25, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v25.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v17.h, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v18.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v16.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v18.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v25, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v22.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v15.l, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v15.h, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v27
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v14.h, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v24.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v15.l, v4.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v22.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v27
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v14.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v13.h, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v25, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v14.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v14.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v25, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v13.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v13.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v5.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v13.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v12.h, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v27
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v25, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v12.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v12.h, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v32.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v32.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v27
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v12.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v25, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v31.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v11.l, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v31.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v11.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v11.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v11.h, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v31.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v10.h, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v10.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v27
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v10, v27
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v25, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v10.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v10.h, v8.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v25, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v25, v9
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -9951,20 +9912,20 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
@@ -9980,17 +9941,17 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB32_2: ; %Flow
@@ -10010,17 +9971,17 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[13:14], 24, v[5:6]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[14:15], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB32_4: ; %end
@@ -10028,105 +9989,93 @@ define <40 x i8> @bitcast_v10f32_to_v40i8(<10 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v30.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v1.l, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v15, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v14.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v1.h, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v15, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v16, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v28, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v15, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v15, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v5.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v15, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v6.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v15, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v7.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v11, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v15
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v15, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v8.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v15, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v11.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v9.h, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v11
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v10.l, v10.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v9
 ; GFX11-TRUE16-NEXT:    s_clause 0x2
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[10:11], off offset:32
+; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[11:12], off offset:32
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: bitcast_v10f32_to_v40i8:
@@ -12037,48 +11986,49 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:24
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v25.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.h, v23.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.h, v21.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.h, v17.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.h, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.h, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 8, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v30.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v29.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v28.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v35.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v30.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v29.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v28.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v27.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.h, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(8)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v33.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v33.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v33.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v34.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v34.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v34.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v35.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v35.l
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v36
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB34_3
@@ -12091,245 +12041,217 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB34_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v25.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v25.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v24.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v23.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v0.h, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v21.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v19.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v18.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v1.h, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v2.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v3.l, v15.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v22.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v21.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v0.l, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v25.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v23.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v27, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v1.h, v23.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v27, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v2.l, v20.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v27, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v3.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v14.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v27, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v4.l, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v27, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v5.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v31.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v27, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v6.l, v13.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v27.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v4.l, v14.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v14.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v25
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v5.l, v13.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v6.h, v13.l
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v6.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v25
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v7.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v25
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v8.l, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v25
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v9.l, v10.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v10.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v27, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v7.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v27.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v27, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v8.l, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v27.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v27, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v27.l, v9.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v27.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v11, v25
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v27, v9
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB34_2
 ; GFX11-TRUE16-NEXT:  .LBB34_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v26.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v25.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v25.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v22.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v23.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v22.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v21.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v18.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v24.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v23.l, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v20.h, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v21.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v24.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v25.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v23.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v23.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v19.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v19.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v17.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v25, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v16.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v20.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v21.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v21.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v19.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v16.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v17.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v27
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v19.h, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v16.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v15.h, v3.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v27
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v17.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v18.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v25, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v25.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v17.h, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v18.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v16.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v18.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v25, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v22.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v15.l, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v15.h, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v27
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v14.h, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v24.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v15.l, v4.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v22.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v27
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v14.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v13.h, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v25, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v14.l, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v14.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v25, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v13.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v13.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v5.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v13.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v12.h, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v27
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v25, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v12.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v12.h, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v32.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v32.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v27
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v12.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v25, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v31.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v11.l, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v31.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v11.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v11.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v11.h, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v31.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v10.h, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v27
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v10.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v27
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v27.h, 0x300, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v10, v27
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v25, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v10.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v10.h, v8.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v25, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v25.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v25, v9
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -16358,20 +16280,20 @@ define <40 x i8> @bitcast_v20i16_to_v40i8(<20 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
@@ -16387,17 +16309,17 @@ define <40 x i8> @bitcast_v20i16_to_v40i8(<20 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB48_2: ; %Flow
@@ -16421,17 +16343,17 @@ define <40 x i8> @bitcast_v20i16_to_v40i8(<20 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB48_4: ; %end
@@ -16439,105 +16361,93 @@ define <40 x i8> @bitcast_v20i16_to_v40i8(<20 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v30.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v1.l, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v15, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v14.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v1.h, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v15, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v16, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v28, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v15, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v15, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v5.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v15, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v6.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v15, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v7.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v11, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v15
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v15, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v8.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v15, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v11.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v9.h, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v11
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v10.l, v10.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v9
 ; GFX11-TRUE16-NEXT:    s_clause 0x2
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[10:11], off offset:32
+; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[11:12], off offset:32
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: bitcast_v20i16_to_v40i8:
@@ -22479,20 +22389,20 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
@@ -22508,17 +22418,17 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB60_2: ; %Flow
@@ -22542,17 +22452,17 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB60_4: ; %end
@@ -22560,105 +22470,93 @@ define <40 x i8> @bitcast_v20f16_to_v40i8(<20 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v30.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v1.l, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v15, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v14.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v1.h, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v15, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v16, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v28, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v15, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v15, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v5.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v15, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v6.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v15, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v7.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v11, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v15
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v15, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v8.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v15, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v11.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v9.h, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v11
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v10.l, v10.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v9
 ; GFX11-TRUE16-NEXT:    s_clause 0x2
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[10:11], off offset:32
+; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[11:12], off offset:32
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: bitcast_v20f16_to_v40i8:
@@ -28859,50 +28757,51 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:24
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.h, v29.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v27.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, v25.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, v21.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v16.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.h, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.h, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v38.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v26.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v21.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v38.h
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(8)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v36.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v36.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v36.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v36.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v37.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v38.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v49
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB72_3
@@ -28915,245 +28814,216 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB72_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v34.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v34.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v33.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v0.h, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v27.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v1.h, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v2.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v3.l, v23.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v0.l, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v33.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v10, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v1.h, v33.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v10, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v2.l, v28.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v10.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v10, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v3.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v26.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v22.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v10, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v4.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v10, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v31.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v6.l, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v10.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v4.l, v21.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v21.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v5.l, v19.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v6.h, v19.l
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v6.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v7.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v11, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v8.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v16.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v9.l, v16.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v7.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v10.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.l, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v10.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v9.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v10.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v11, v10
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v10, v9
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB72_2
 ; GFX11-TRUE16-NEXT:  .LBB72_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v34.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v30.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v33.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v30.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v29.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v26.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v34.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v33.l, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v28.h, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v29.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v34.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v34.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v33.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v33.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v27.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v27.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v25.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v10, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v24.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v28.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v29.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v29.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v27.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v24.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v25.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v27.h, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v23.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v23.h, v3.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v25.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v10, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v25.h, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v26.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v21.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v10, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v22.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v23.l, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v23.h, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v21.h, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v24.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v22.h, v4.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v22.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v21.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v19.h, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v10, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v21.h, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v22.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v10, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v19.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v19.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v5.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v19.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v18.h, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v11
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v18.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v18.h, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v32.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v32.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v18.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v31.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v17.l, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v31.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v17.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v17.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v17.h, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v31.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v16.h, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v16.l, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v10, v11
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v16.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v16.h, v8.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v10, v9
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -30908,20 +30778,20 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
@@ -30937,17 +30807,17 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB74_2: ; %Flow
@@ -30966,17 +30836,17 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB74_4: ; %end
@@ -30984,105 +30854,93 @@ define <40 x i8> @bitcast_v5f64_to_v40i8(<5 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v30.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v1.l, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v15, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v14.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v1.h, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v15, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v16, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v28, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v15, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v15, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v5.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v15, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v6.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v15, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v7.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v11, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v15
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v15, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v8.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v15, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v11.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v9.h, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v11
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v10.l, v10.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v9
 ; GFX11-TRUE16-NEXT:    s_clause 0x2
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[10:11], off offset:32
+; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[11:12], off offset:32
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: bitcast_v5f64_to_v40i8:
@@ -33010,50 +32868,51 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:24
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:20
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.h, v29.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v27.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, v25.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.h, v21.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v16.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.h, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.h, v4.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.l, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v34.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.l, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v39.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v38.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v33.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v26.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v21.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v39.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v38.h
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(8)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v36.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v36.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v36.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v36.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v37.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(4)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v38.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v49
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB76_3
@@ -33066,245 +32925,216 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB76_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v34.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v34.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v33.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v0.h, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v23.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v27.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v1.h, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v2.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v20.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v3.l, v23.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v29.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v0.l, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v34.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v33.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v27.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v10, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v1.h, v33.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v10, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v2.l, v28.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v10.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v10, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v3.l, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v26.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v22.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v10, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v4.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v10, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v5.l, v21.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v18.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v31.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v6.l, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v10.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v4.l, v21.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v21.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v5.l, v19.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v6.h, v19.l
-; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v6.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v7.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v11, v10
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v8.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v31.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v16.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v9.l, v16.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v7.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v10.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.l, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v10.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v9.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v10.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v11, v10
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v10, v9
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB76_2
 ; GFX11-TRUE16-NEXT:  .LBB76_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v34.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v30.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v33.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v30.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v29.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v26.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v34.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v33.l, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v28.h, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v29.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v34.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v34.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v33.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v33.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v27.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v27.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v25.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v10, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v24.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v28.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v29.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v29.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v27.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v24.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v25.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v27.h, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v23.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v23.h, v3.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v11
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v25.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v10, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v25.h, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v26.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v21.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v20.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v10, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v20.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v22.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v23.l, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v23.h, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v5, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v21.h, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v24.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v22.h, v4.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v22.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v21.l, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v19.h, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v10, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v21.h, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v22.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v10, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v19.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v19.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v5.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v19.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v18.h, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v7, v11
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v18.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v18.h, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v32.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v32.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v18.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v31.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v17.l, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v31.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v17.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v17.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v17.h, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v31.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v7.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v16.h, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v11
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v16.l, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v10, v11
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v16.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v16.h, v8.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v10, v9
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -35074,20 +34904,20 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr15_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr14_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr13_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr12_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr11_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
@@ -35103,17 +34933,17 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB78_2: ; %Flow
@@ -35140,17 +34970,17 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[15:16], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v17, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v18, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v19, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v20, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v21, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v22, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v23, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v24, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v16, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB78_4: ; %end
@@ -35158,105 +34988,93 @@ define <40 x i8> @bitcast_v5i64_to_v40i8(<5 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v30.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v1.l, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v15, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v2.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v14.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v1.h, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v15, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v3.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v28, 0xffff, v29
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v16, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v28, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v14, v15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v15, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v4.l, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v15, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v5.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v15.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v15, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v6.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v15, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v7.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v14, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v13, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v7.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v12, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v11.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v13
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v15
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v11, v15
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v11, v15
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v15, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v8.l, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v15, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v11.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v9.h, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v15.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v15, v11
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v9.l, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v10.l, v10.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v9
 ; GFX11-TRUE16-NEXT:    s_clause 0x2
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
-; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[10:11], off offset:32
+; GFX11-TRUE16-NEXT:    scratch_store_b64 v0, v[11:12], off offset:32
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: bitcast_v5i64_to_v40i8:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 48c9b8775a474..8e30ee659a260 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -2257,8 +2257,8 @@ define i32 @bitcast_v4i8_to_i32(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v4
@@ -2273,19 +2273,17 @@ define i32 @bitcast_v4i8_to_i32(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB22_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB22_2
 ; GFX11-TRUE16-NEXT:  .LBB22_4: ; %cmp.true
@@ -2295,16 +2293,14 @@ define i32 @bitcast_v4i8_to_i32(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v0.l
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
@@ -4506,8 +4502,8 @@ define float @bitcast_v4i8_to_f32(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v4
@@ -4522,19 +4518,17 @@ define float @bitcast_v4i8_to_f32(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB42_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB42_2
 ; GFX11-TRUE16-NEXT:  .LBB42_4: ; %cmp.true
@@ -4544,16 +4538,14 @@ define float @bitcast_v4i8_to_f32(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v0.l
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
@@ -6467,8 +6459,8 @@ define <2 x i16> @bitcast_v4i8_to_v2i16(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v4
@@ -6483,19 +6475,17 @@ define <2 x i16> @bitcast_v4i8_to_v2i16(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB58_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB58_2
 ; GFX11-TRUE16-NEXT:  .LBB58_4: ; %cmp.true
@@ -6505,16 +6495,14 @@ define <2 x i16> @bitcast_v4i8_to_v2i16(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v0.l
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
@@ -8116,8 +8104,8 @@ define <2 x half> @bitcast_v4i8_to_v2f16(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v4
@@ -8132,19 +8120,17 @@ define <2 x half> @bitcast_v4i8_to_v2f16(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB70_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB70_2
 ; GFX11-TRUE16-NEXT:  .LBB70_4: ; %cmp.true
@@ -8154,16 +8140,14 @@ define <2 x half> @bitcast_v4i8_to_v2f16(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v0.l
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
@@ -9479,8 +9463,8 @@ define <2 x bfloat> @bitcast_v4i8_to_v2bf16(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v4
@@ -9495,19 +9479,17 @@ define <2 x bfloat> @bitcast_v4i8_to_v2bf16(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB78_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB78_2
 ; GFX11-TRUE16-NEXT:  .LBB78_4: ; %cmp.true
@@ -9517,16 +9499,14 @@ define <2 x bfloat> @bitcast_v4i8_to_v2bf16(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v0.l
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
@@ -10193,8 +10173,8 @@ define <1 x i32> @bitcast_v4i8_to_v1i32(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v4
@@ -10209,19 +10189,17 @@ define <1 x i32> @bitcast_v4i8_to_v1i32(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB82_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v2, v0
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr1_lo16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB82_2
 ; GFX11-TRUE16-NEXT:  .LBB82_4: ; %cmp.true
@@ -10231,16 +10209,14 @@ define <1 x i32> @bitcast_v4i8_to_v1i32(<4 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v0.l
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.h
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 5aac06a7f3a2b..35d135b123969 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -8768,32 +8768,32 @@ define <64 x i8> @bitcast_v16i32_to_v64i8(<16 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
@@ -8812,26 +8812,26 @@ define <64 x i8> @bitcast_v16i32_to_v64i8(<16 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB24_2: ; %Flow
@@ -8864,26 +8864,26 @@ define <64 x i8> @bitcast_v16i32_to_v64i8(<16 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB24_4: ; %end
@@ -8891,156 +8891,135 @@ define <64 x i8> @bitcast_v16i32_to_v64i8(<16 x i32> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v1.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v18.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v24, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v2.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v1.h, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v3.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v25, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v17.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v54, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v23, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v22.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v24, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v5.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v49.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v24, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v6.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v7.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v23, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v24, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v8.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v24, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v9.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v21, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v34.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v20, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v24, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v24, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v11.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v10.h, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v24, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v12.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v24, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v13.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v24, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v14.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v17.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v20, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v12.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v11.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v13.h, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v24, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v15.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v26.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v15.l, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v14.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v24
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v17, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v24, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v16.l, v17.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v24, v16
 ; GFX11-TRUE16-NEXT:    s_clause 0x3
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
@@ -12470,15 +12449,15 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:124
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:108
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:100
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:92
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v82, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v81, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:8
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:16
@@ -12492,84 +12471,82 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:80
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:84
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:60
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:52
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.h, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.h, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v48.l, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v39.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 8, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v26.h, 8, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v39.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v48.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v27.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 8, v80.h
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v64.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v65.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v65.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v65.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(25)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v66.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v66.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(21)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v67.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v67.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v67.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v68.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(18)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v68.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 8, v68.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(17)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v69.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(16)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 8, v69.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v69.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v70.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(14)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.l, 8, v70.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v71.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.l, 8, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v71.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v82
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v81
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB26_3
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %Flow
@@ -12581,384 +12558,338 @@ define <16 x i32> @bitcast_v64i8_to_v16i32(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB26_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v55.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v54.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v55.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v51.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v54.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v53.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v53.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v0.h, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v52.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v50.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v51.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v48.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v1.h, v50.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v2.l, v49.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v3.l, v48.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v53.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v52.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v52.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v0.l, v54.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v55.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v54.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v49.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v49.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v64, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v1.h, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v51.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v64, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v2.l, v51.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v64, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v3.l, v50.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v30.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v64, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v38.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v4.l, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v38.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v64, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v5.l, v29.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v64, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v6.l, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v22.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v64, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v7.l, v25.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v64, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v8.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v64, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v9.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v64, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v10.l, v21.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v31.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v5.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v6.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v7.l, v23.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v8.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v36.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.h, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v9.l, v21.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v9.h, v21.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v10.l, v20.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v34.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v10.h, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v11.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v34.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v13, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v12.h, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v33.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v12.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v12.h, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v13.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v14.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v31.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v54
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v15.l, v16.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v64, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v11.l, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v16.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v64, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v12.l, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v64.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v64, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v13.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v64.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v64, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v14.l, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v64.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v64, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v15.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v64.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v54
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v64, v15
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX11-TRUE16-NEXT:  .LBB26_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v55.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v54.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v55.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v51.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v53.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v53.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v52.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v52.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v48.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v54.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v53.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v50.l, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v52.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v54.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v55.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v54.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v53.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v49.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v48.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v52, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v39.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v51.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v51.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v52.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v50.h, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v39.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v30.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v49.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v51.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v48.l, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v27.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v49.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v27.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v52, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v52.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v50.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v50.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v29.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v28.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v52, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v26.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v24.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v39.h, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v48.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v24.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v39.l, v4.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v29.h, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v25.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v29.l, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v28.h, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v52, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v29.h, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v30.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v52, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v27.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v27.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v26.h, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v52, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v25.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v25.h, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v38.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v25.h, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v24.l, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v37.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v23.h, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v38.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v38.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v38.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v52, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v37.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v37.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v23.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v23.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v22.h, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, v36.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v23.l, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v37.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v21.h, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v55
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v36.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v9.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v52, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v22.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v22.h, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v36.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v36.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v52, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v35.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v35.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v21.l, v9.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v21.h, v9.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v11, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v20.h, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v21.l, v10.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, v34.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, 0x300, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v33.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v20.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v34.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v19.h, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v52, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v20.l, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v20.h, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v34.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v34.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v52, v13
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v33.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v33.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v19.l, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v19.h, v11.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v19.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v18.h, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v13, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, v33.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, 0x300, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v14
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v18.l, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v18.h, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v32.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v32.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v18.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v31.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v17.l, v13.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v31.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v17.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v17.h, v13.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v17.h, v14.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, v31.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v16.h, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, 0x300, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v14.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v16.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v55
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v16, v55
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v52, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v16.l, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v16.h, v14.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v52, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v52, v15
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -23588,32 +23519,32 @@ define <64 x i8> @bitcast_v16f32_to_v64i8(<16 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
@@ -23632,26 +23563,26 @@ define <64 x i8> @bitcast_v16f32_to_v64i8(<16 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB48_2: ; %Flow
@@ -23676,26 +23607,26 @@ define <64 x i8> @bitcast_v16f32_to_v64i8(<16 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB48_4: ; %end
@@ -23703,156 +23634,135 @@ define <64 x i8> @bitcast_v16f32_to_v64i8(<16 x float> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v1.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v18.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v24, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v2.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v1.h, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v3.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v25, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v17.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v54, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v23, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v22.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v24, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v5.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v49.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v24, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v6.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v7.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v23, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v24, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v8.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v24, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v9.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v21, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v34.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v20, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v24, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v24, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v11.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v10.h, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v24, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v12.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v24, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v13.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v24, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v14.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v17.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v20, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v12.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v11.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v13.h, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v24, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v15.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v26.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v15.l, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v14.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v24
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v17, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v24, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v16.l, v17.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v24, v16
 ; GFX11-TRUE16-NEXT:    s_clause 0x3
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
@@ -27413,15 +27323,15 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:124
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:108
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:100
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:92
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v82, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v81, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:8
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:16
@@ -27435,84 +27345,82 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:80
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:84
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:60
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:52
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.h, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.h, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v48.l, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v39.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 8, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v26.h, 8, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v39.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v48.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v27.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 8, v80.h
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v64.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v65.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v65.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v65.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(25)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v66.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v66.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(21)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v67.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v67.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v67.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v68.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(18)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v68.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 8, v68.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(17)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v69.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(16)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 8, v69.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v69.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v70.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(14)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.l, 8, v70.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v71.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.l, 8, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v71.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v82
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v81
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB50_3
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %Flow
@@ -27524,384 +27432,338 @@ define <16 x float> @bitcast_v64i8_to_v16f32(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB50_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v55.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v54.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v55.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v51.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v54.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v53.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v53.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v0.h, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v52.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v50.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v51.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v48.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v1.h, v50.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v2.l, v49.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v3.l, v48.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v53.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v52.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v52.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v0.l, v54.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v55.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v54.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v49.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v49.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v64, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v1.h, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v51.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v64, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v2.l, v51.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v64, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v3.l, v50.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v30.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v64, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v38.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v4.l, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v38.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v64, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v5.l, v29.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v64, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v6.l, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v22.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v64, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v7.l, v25.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v64, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v8.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v64, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v9.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v64, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v10.l, v21.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v31.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v5.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v6.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v7.l, v23.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v8.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v36.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.h, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v9.l, v21.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v9.h, v21.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v10.l, v20.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v34.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v10.h, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v11.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v34.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v13, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v12.h, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v33.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v12.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v12.h, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v13.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v14.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v31.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v54
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v15.l, v16.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v64, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v11.l, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v16.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v64, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v12.l, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v64.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v64, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v13.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v64.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v64, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v14.l, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v64.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v64, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v15.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v64.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v54
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v64, v15
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB50_2
 ; GFX11-TRUE16-NEXT:  .LBB50_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v55.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v54.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v55.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v51.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v53.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v53.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v52.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v52.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v48.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v54.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v53.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v50.l, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v52.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v54.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v55.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v54.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v53.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v49.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v48.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v52, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v39.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v51.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v51.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v52.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v50.h, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v39.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v30.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v49.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v51.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v48.l, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v27.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v49.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v27.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v52, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v52.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v50.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v50.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v29.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v28.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v52, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v26.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v24.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v39.h, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v48.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v24.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v39.l, v4.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v29.h, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v25.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v29.l, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v28.h, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v52, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v29.h, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v30.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v52, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v27.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v27.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v26.h, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v52, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v25.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v25.h, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v38.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v25.h, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v24.l, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v37.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v23.h, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v38.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v38.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v38.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v52, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v37.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v37.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v23.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v23.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v22.h, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, v36.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v23.l, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v37.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v21.h, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v55
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v36.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v9.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v52, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v22.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v22.h, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v36.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v36.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v52, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v35.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v35.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v21.l, v9.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v21.h, v9.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v11, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v20.h, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v21.l, v10.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, v34.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, 0x300, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v33.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v20.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v34.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v19.h, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v52, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v20.l, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v20.h, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v34.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v34.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v52, v13
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v33.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v33.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v19.l, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v19.h, v11.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v19.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v18.h, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v13, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, v33.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, 0x300, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v14
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v18.l, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v18.h, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v32.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v32.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v18.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v31.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v17.l, v13.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v31.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v17.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v17.h, v13.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v17.h, v14.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, v31.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v16.h, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, 0x300, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v14.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v16.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v55
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v16, v55
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v52, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v16.l, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v16.h, v14.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v52, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v52, v15
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -37916,32 +37778,32 @@ define <64 x i8> @bitcast_v8i64_to_v64i8(<8 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
@@ -37960,26 +37822,26 @@ define <64 x i8> @bitcast_v8i64_to_v64i8(<8 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB68_2: ; %Flow
@@ -38017,26 +37879,26 @@ define <64 x i8> @bitcast_v8i64_to_v64i8(<8 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB68_4: ; %end
@@ -38044,156 +37906,135 @@ define <64 x i8> @bitcast_v8i64_to_v64i8(<8 x i64> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v1.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v18.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v24, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v2.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v1.h, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v3.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v25, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v17.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v54, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v23, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v22.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v24, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v5.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v49.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v24, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v6.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v7.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v23, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v24, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v8.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v24, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v9.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v21, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v34.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v20, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v24, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v24, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v11.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v10.h, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v24, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v12.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v24, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v13.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v24, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v14.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v17.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v20, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v12.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v11.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v13.h, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v24, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v15.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v26.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v15.l, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v14.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v24
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v17, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v24, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v16.l, v17.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v24, v16
 ; GFX11-TRUE16-NEXT:    s_clause 0x3
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
@@ -41628,15 +41469,15 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:124
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:108
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:100
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:92
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v82, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v81, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:8
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:16
@@ -41650,84 +41491,82 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:80
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:84
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:60
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:52
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.h, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.h, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v48.l, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v39.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 8, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v26.h, 8, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v39.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v48.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v27.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 8, v80.h
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v64.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v65.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v65.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v65.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(25)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v66.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v66.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(21)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v67.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v67.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v67.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v68.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(18)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v68.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 8, v68.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(17)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v69.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(16)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 8, v69.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v69.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v70.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(14)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.l, 8, v70.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v71.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.l, 8, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v71.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v82
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v81
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB70_3
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %Flow
@@ -41739,384 +41578,338 @@ define <8 x i64> @bitcast_v64i8_to_v8i64(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB70_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v55.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v54.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v55.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v51.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v54.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v53.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v53.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v0.h, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v52.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v50.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v51.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v48.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v1.h, v50.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v2.l, v49.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v3.l, v48.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v53.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v52.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v52.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v0.l, v54.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v55.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v54.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v49.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v49.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v64, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v1.h, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v51.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v64, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v2.l, v51.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v64, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v3.l, v50.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v30.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v64, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v38.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v4.l, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v38.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v64, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v5.l, v29.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v64, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v6.l, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v22.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v64, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v7.l, v25.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v64, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v8.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v64, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v9.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v64, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v10.l, v21.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v31.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v5.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v6.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v7.l, v23.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v8.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v36.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.h, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v9.l, v21.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v9.h, v21.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v10.l, v20.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v34.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v10.h, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v11.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v34.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v13, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v12.h, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v33.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v12.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v12.h, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v13.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v14.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v31.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v54
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v15.l, v16.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v64, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v11.l, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v16.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v64, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v12.l, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v64.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v64, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v13.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v64.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v64, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v14.l, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v64.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v64, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v15.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v64.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v54
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v64, v15
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB70_2
 ; GFX11-TRUE16-NEXT:  .LBB70_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v55.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v54.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v55.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v51.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v53.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v53.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v52.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v52.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v48.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v54.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v53.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v50.l, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v52.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v54.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v55.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v54.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v53.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v49.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v48.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v52, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v39.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v51.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v51.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v52.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v50.h, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v39.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v30.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v49.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v51.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v48.l, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v27.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v49.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v27.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v52, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v52.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v50.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v50.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v29.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v28.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v52, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v26.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v24.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v39.h, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v48.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v24.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v39.l, v4.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v29.h, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v25.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v29.l, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v28.h, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v52, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v29.h, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v30.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v52, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v27.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v27.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v26.h, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v52, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v25.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v25.h, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v38.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v25.h, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v24.l, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v37.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v23.h, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v38.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v38.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v38.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v52, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v37.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v37.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v23.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v23.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v22.h, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, v36.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v23.l, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v37.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v21.h, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v55
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v36.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v9.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v52, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v22.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v22.h, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v36.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v36.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v52, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v35.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v35.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v21.l, v9.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v21.h, v9.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v11, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v20.h, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v21.l, v10.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, v34.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, 0x300, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v33.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v20.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v34.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v19.h, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v52, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v20.l, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v20.h, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v34.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v34.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v52, v13
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v33.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v33.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v19.l, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v19.h, v11.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v19.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v18.h, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v13, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, v33.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, 0x300, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v14
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v18.l, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v18.h, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v32.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v32.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v18.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v31.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v17.l, v13.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v31.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v17.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v17.h, v13.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v17.h, v14.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, v31.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v16.h, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, 0x300, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v14.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v16.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v55
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v16, v55
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v52, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v16.l, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v16.h, v14.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v52, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v52, v15
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -51295,32 +51088,32 @@ define <64 x i8> @bitcast_v8f64_to_v64i8(<8 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
@@ -51339,26 +51132,26 @@ define <64 x i8> @bitcast_v8f64_to_v64i8(<8 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB84_2: ; %Flow
@@ -51383,26 +51176,26 @@ define <64 x i8> @bitcast_v8f64_to_v64i8(<8 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB84_4: ; %end
@@ -51410,156 +51203,135 @@ define <64 x i8> @bitcast_v8f64_to_v64i8(<8 x double> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v1.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v18.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v24, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v2.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v1.h, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v3.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v25, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v17.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v54, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v23, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v22.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v24, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v5.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v49.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v24, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v6.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v7.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v23, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v24, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v8.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v24, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v9.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v21, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v34.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v20, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v24, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v24, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v11.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v10.h, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v24, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v12.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v24, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v13.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v24, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v14.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v17.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v20, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v12.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v11.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v13.h, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v24, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v15.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v26.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v15.l, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v14.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v24
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v17, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v24, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v16.l, v17.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v24, v16
 ; GFX11-TRUE16-NEXT:    s_clause 0x3
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
@@ -54989,15 +54761,15 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v64, off, s32 offset:128
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v31, off, s32 offset:124
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v64, off, s32 offset:120
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:116
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:116
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v65, off, s32 offset:112
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v31, off, s32 offset:108
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v32, off, s32 offset:108
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v65, off, s32 offset:104
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:100
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:100
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v66, off, s32 offset:96
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v32, off, s32 offset:92
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v33, off, s32 offset:92
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v66, off, s32 offset:88
-; GFX11-TRUE16-NEXT:    scratch_load_b32 v82, off, s32 offset:132
+; GFX11-TRUE16-NEXT:    scratch_load_b32 v81, off, s32 offset:132
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v67, off, s32
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v67, off, s32 offset:8
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v68, off, s32 offset:16
@@ -55011,84 +54783,82 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v80, off, s32 offset:80
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v33, off, s32 offset:84
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v34, off, s32 offset:76
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:68
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:60
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v34, off, s32 offset:68
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v35, off, s32 offset:60
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v35, off, s32 offset:52
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v36, off, s32 offset:44
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:36
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:28
-; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:20
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v36, off, s32 offset:36
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v37, off, s32 offset:28
+; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:20
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
-; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v37, off, s32 offset:12
+; GFX11-TRUE16-NEXT:    scratch_load_d16_b16 v38, off, s32 offset:12
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_hi_b16 v38, off, s32 offset:4
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v80.h, v29.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.l, v27.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v24.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.l, v20.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v18.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v50.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v51.h, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.l, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v52.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v48.l, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v49.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v39.l, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 8, v25.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v81.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v26.h, 8, v80.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v55.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v53.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v54.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v51.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.l, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v50.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v39.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v48.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v27.h, 8, v27.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 8, v80.h
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(33)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(31)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v64.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(29)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v65.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(27)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v65.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v65.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(25)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v66.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(23)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v66.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v66.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(21)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v67.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 8, v67.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(20)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v24.l, 8, v67.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v67.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(19)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.h, 8, v68.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(18)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 8, v68.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 8, v68.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(17)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v69.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 8, v69.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(16)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 8, v69.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v69.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(15)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v70.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.h, 8, v70.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(14)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 8, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.l, 8, v70.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(13)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 8, v71.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(12)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.l, 8, v71.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.l, 8, v71.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(11)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v82
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v19.h, 8, v80.l
+; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v81
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execnz .LBB86_3
 ; GFX11-TRUE16-NEXT:  ; %bb.1: ; %Flow
@@ -55100,384 +54870,338 @@ define <8 x double> @bitcast_v64i8_to_v8f64(<64 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB86_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v55.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v54.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v55.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v51.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v54.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v53.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v53.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v0.h, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v52.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v50.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v51.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v48.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.l, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v1.h, v50.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v27.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v2.l, v49.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v6, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v3.l, v48.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v24.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v53.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v64.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v52.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v52.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v0.l, v54.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v55.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v54.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v49.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v49.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v64, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v1.h, v53.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v28.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v51.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v64, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v2.l, v51.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v39.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v48.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v64, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v3.l, v50.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v50.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v30.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v64, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v4.l, v39.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v27.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v30.l
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v38.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v4.l, v29.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v4.h, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v38.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v64, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v5.l, v29.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v23.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v37.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v64, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v6.l, v27.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v22.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v36.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v64, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v7.l, v25.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v21.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v34.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v64, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v8.l, v23.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v19.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v33.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v64, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v9.l, v22.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v64.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v64, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v10.l, v21.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v31.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v5.l, v28.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v5.h, v26.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v6.l, v25.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v6.h, v24.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v7.l, v23.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v8.l, v22.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v36.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.h, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v11, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v9.l, v21.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v36.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v9.h, v21.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v12, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v10.l, v20.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v34.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v10.h, v20.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v11.l, v19.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v34.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v13, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v12.h, v19.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v33.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v12.l, v18.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v12.h, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v54
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v13.h, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v13.l, v17.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v54
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v14.l, v16.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v31.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v17, v54
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v54.h, v15.l, v16.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v64, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v11.l, v20.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v64.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v16.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v64, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v12.l, v19.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v64.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v64, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v13.l, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v64.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v64, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v14.l, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v64.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v64, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v64.l, v15.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v64.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr16_lo16
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v54
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v64, v15
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB86_2
 ; GFX11-TRUE16-NEXT:  .LBB86_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v55.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v54.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v55.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v51.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v53.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v53.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v52.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v52.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v48.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v54.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v53.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v50.l, 3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v52.l, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v54.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v55.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v54.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v53.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v49.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v49.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v48.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v52, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v39.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v51.l, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v51.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v52.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v50.h, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, v39.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, v30.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v49.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v51.l, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v48.l, v3.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v27.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v49.l, v3.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v27.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v3.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v52, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v52.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v50.l, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v50.h, v2.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v29.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v28.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v52, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v3.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v3.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, v26.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.h, v24.h, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v39.h, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v48.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, v24.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v39.l, v4.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v29.h, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, v25.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v4.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, 0x300, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v29.l, v5.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v28.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v28.h, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v26.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v5.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v52, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v29.h, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v30.h, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v24.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v26.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v52, v7
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v5.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, 0x300, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.l, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, v30.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v27.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v27.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v26.h, v6.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v52, v8
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v25.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v25.h, v6.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, v38.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v8, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v25.h, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, v30.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v6.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v24.l, v7.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v37.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v23.h, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v38.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v7.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v38.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v38.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v52, v9
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v7.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, 0x300, v7.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, v37.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.h, v37.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v23.l, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v23.h, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v22.h, v8.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, v36.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v23.l, v8.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, v37.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v8.l, 0x300, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v21.h, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v10, v55
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v36.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v35.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v9.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v52, v10
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v22.l, v8.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v22.h, v8.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v36.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v36.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v52, v11
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v9.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, 0x300, v9.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.l, v35.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v10.h, v35.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v21.l, v9.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v21.h, v9.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v11, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v20.h, v10.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, v35.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v21.l, v10.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.h, v34.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, 0x300, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v33.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v13, v55
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v20.l, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v34.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v19.h, v11.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v11.l, 0x300, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v52, v12
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v20.l, v10.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v20.h, v10.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v34.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v34.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v52, v13
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v11.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, 0x300, v11.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.l, v33.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v12.h, v33.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v19.l, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v19.h, v11.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v11.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v19.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v18.h, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v13, v55
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, v33.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, 0x300, v12.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v12.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.h, v32.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v52, v14
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v18.l, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v18.h, v12.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v32.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v32.l, 3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v15, v55
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v18.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v52, v15
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v18.h, 0x300, v13.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.h
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.l, v31.h, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v15
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v17.l, v13.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v13.l, 0x300, v13.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v14.h, v31.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v17.l, v13.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v17.h, v13.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v17.h, v14.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.l, v31.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v13.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v16.h, v14.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.l, 0x300, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v55
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v14.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v16.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v17
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v55
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v55.h, 0x300, v15.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v16, v55
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v52, v18
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v17.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v16.l, v14.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v16.h, v14.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v52, v17
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v52.l, 0x300, v15.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v15.h, 0x300, v15.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v52.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v52, v15
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -64573,32 +64297,32 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
@@ -64617,26 +64341,26 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB96_2: ; %Flow
@@ -64669,26 +64393,26 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB96_4: ; %end
@@ -64696,156 +64420,135 @@ define <64 x i8> @bitcast_v32i16_to_v64i8(<32 x i16> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v1.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v18.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v24, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v2.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v1.h, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v3.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v25, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v17.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v54, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v23, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v22.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v24, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v5.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v49.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v24, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v6.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v7.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v23, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v24, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v8.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v24, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v9.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v21, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v34.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v20, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v24, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v24, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v11.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v10.h, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v24, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v12.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v24, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v13.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v24, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v14.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v17.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v20, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v12.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v11.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v13.h, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v24, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v15.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v26.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v15.l, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v14.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v24
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v17, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v24, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v16.l, v17.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v24, v16
 ; GFX11-TRUE16-NEXT:    s_clause 0x3
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
@@ -76701,32 +76404,32 @@ define <64 x i8> @bitcast_v32f16_to_v64i8(<32 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr25_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_lo16
@@ -76745,26 +76448,26 @@ define <64 x i8> @bitcast_v32f16_to_v64i8(<32 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB104_2: ; %Flow
@@ -76797,26 +76500,26 @@ define <64 x i8> @bitcast_v32f16_to_v64i8(<32 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v26, 24, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v27, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v28, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v29, 24, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v30, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v31, 8, v13
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v32, 24, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v33, 8, v12
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v34, 8, v11
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v35, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v36, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 24, v8
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v39, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v52, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v25, 8, v1
 ; GFX11-TRUE16-NEXT:  .LBB104_4: ; %end
@@ -76824,156 +76527,135 @@ define <64 x i8> @bitcast_v32f16_to_v64i8(<32 x half> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v64.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v1.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v2.l, v18.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v55.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v24, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v2.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v23.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v1.h, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v25, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v54.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.l, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v2
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v3.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v54, 0xffff, v55
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v25, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v17.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v54, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v52.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v51.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v23, v24
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v52.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v4.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v22.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v4.l, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v24, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v5.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff, v25
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v49.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v24, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v6.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v6.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v48.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v7.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v39.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v38.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v25.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v49.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v23, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v25
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v48.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v39.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v22, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v23
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v7.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v24, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v8.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v8.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v20.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v21, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v9.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v37.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v24, v8
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v9.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v36.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v9.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v10.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v22
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v21, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v35.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v11.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v34.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v20, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v35.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v24, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v10.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v10.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v34.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v24, v10
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v11.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v33.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v10.h, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 8, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v24, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v12.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v12.h, v18.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v18.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v24, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v13.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v30.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v24, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v14.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v14.h, v18.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v28.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v17.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v20, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v12.l, v17.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v32.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v21
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v11.l, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v31.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v19, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v20
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v13.l, v17.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v13.h, v14.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, 0xff, v16.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v24, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v15.l, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v17.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v26.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v18, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v15.l, v17.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v14.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v28.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v14.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v17.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v24
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v16.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v26.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v17, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v24, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v16.l, v17.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v24, v16
 ; GFX11-TRUE16-NEXT:    s_clause 0x3
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
@@ -85692,59 +85374,59 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr28_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr113_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr27_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr103_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr102_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr29_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr30_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr99_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr32_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr33_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr97_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr96_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr21_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr35_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr84_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr36_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr70_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr52_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr64_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr68_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr66_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr69_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr67_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr87_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr50_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr82_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr81_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr83_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr48_lo16
 ; GFX11-TRUE16-NEXT:    s_and_saveexec_b32 s0, vcc_lo
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-TRUE16-NEXT:    s_xor_b32 s0, exec_lo, s0
@@ -85757,307 +85439,302 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[21:22], 24, v[7:8]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[22:23], 24, v[5:6]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[23:24], 24, v[3:4]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v15
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 24, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v14
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v67, 8, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v10
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v65, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v66, 8, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 8, v11
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v10
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v10
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v9
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 24, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v5
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v4
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 8, v3
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 24, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v2
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v1
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v28.h, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.h, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.h, v4.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.h, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.h, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v49.h, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v26.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v27.h, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v31.h, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v29.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v30.h, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.h, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v33.h, v6.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v38.h, v7.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v35.h, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v34.h, v8.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v65.h, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v48.h, v10.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v71.h, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v54.h, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, v12.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.h, v13.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v68.h, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v66.h, v14.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.h, v15.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v82.h, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v81.h, v16.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v36.h, v8.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v55.h, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v37.h, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v39.h, v10.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v70.h, v11.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v52.h, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v53.h, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v85.h, v13.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v69.h, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v67.h, v14.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v87.h, v15.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v83.h, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v86.h, v16.h
 ; GFX11-TRUE16-NEXT:  .LBB108_2: ; %Flow
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB108_4
 ; GFX11-TRUE16-NEXT:  ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff0000, v1
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v4
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v17, 16, v2
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v17, 0x40c00000, v17
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v49, 0xffff0000, v11
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v17, 0x40c00000, v17
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v21, v2, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v2
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v20, v17, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v17
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v21, v2, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v2
-; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v20, v17, 0x7fff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff0000, v1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v21, v21, v2, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v27, v20, v22, vcc_lo
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v27.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v20, v17, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v26, v20, v22, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v20, v1, 16, 1
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v26, v21, v23, vcc_lo
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v1
-; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v18
+; GFX11-TRUE16-NEXT:    v_add3_u32 v17, v17, v18, 0x7fff
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v26.h
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v20, v1, 0x7fff
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v27, v21, v23, vcc_lo
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v1
+; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v19, 0x40c00000, v19
-; GFX11-TRUE16-NEXT:    v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v2, 0xffff, v2, v26
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v2, 0xffff, v2, v27
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v28, v20, v21, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v18, v4, 16, 1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v3
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v19
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 24, v2
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v28.h
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v1, v17, v22, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v18, v18, v4, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v4
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v3
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v17, v19, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v2
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v30, v18, v22, vcc_lo
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v1, 0xffff, v20, v1
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v20, 0xffff0000, v5
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v29, v18, v22, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v6
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_lshlrev_b32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v17, v17, v19, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_lshlrev_b32 v6, 16, v6
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v30, v17, v23, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v4, v21, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v29, v17, v23
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v21
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v18, v3, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, 0x400000, v3
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v4, v4, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v29.h
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v18, v18, v3, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v30.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v32, v18, v19, vcc_lo
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 24, v2
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v112, 8, v2
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v31, v18, v19, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v1, 0xffff, v20, v1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v32.h
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v3, v4, v23 :: v_dual_add_f32 v18, 0x40c00000, v22
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v4, 0xffff, v17, v29
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v18, 0x40c00000, v22
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v1
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v31.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v3, v4, v23, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v21, v18, 16, 1
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v4, 0xffff, v17, v30
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v17, v6, 16, 1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v3, 0xffff, v19, v3
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v21, v18, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v18
+; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v21, v18, 0x7fff
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v6
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v17, v17, v6, 0x7fff
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v18
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v6, v20, 16, 1
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v100, 24, v4
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v101, 8, v4
-; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v21, v18, 0x7fff
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v6
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v103, 8, v3
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v113, 8, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v33, v17, v21, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v32, v17, v21, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v31, v19, v22 :: v_dual_and_b32 v20, 0xffff0000, v5
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v20, 0x40c00000, v20 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT:    v_add3_u32 v6, v6, v20, 0x7fff
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v102, 8, v3
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v33, v19, v22, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v19, 0xffff0000, v8
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v8, 16, v8
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v6, v20, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v20
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v33.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v8
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v17, v5, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v5
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT:    v_add3_u32 v6, v6, v20, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v20
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v8, 0x40c00000, v8
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v17, v17, v5, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v36, v17, v21, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v32.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v34, v17, v21 :: v_dual_add_f32 v19, 0x40c00000, v19
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v17, v8, 16, 1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v19, 0x40c00000, v19
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v36.h
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v20, v6, v22, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v34.h
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v5, v19, 16, 1
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v20, v6, v22, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v6, v17, v8, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v17, 0x400000, v8
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v19
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v5, v5, v19, 0x7fff
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v19
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v35, v6, v17, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v7
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v6, 0xffff, v18, v31
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v34, v5, v22, vcc_lo
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v6, 0xffff, v18, v33
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v36, v5, v22, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v5, 0xffff, v21, v20
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v20, 16, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v35.h
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 24, v6
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 24, v6
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v98, 8, v6
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_dual_add_f32 v20, 0x40c00000, v20 :: v_dual_add_f32 v7, 0x40c00000, v7
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v8, 0xffff, v8, v34
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v8, 0xffff, v8, v36
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v99, 8, v5
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v7, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v7
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v83, 24, v8
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v87, 8, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v82, 24, v8
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v8
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v7, v20, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v49, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v17, 0x40c00000, v23 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v38, v19, v21, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v7, v7, v20, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v21, 0x400000, v20
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff0000, v10
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v17, 0x40c00000, v23 :: v_dual_add_f32 v10, 0x40c00000, v10
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v18, v17, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v17
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v10, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v10, 0x7fff
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v17, v18, v22, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v20, v20
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v10
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v49.h
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v39, v7, v21, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v38.h
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v37, v7, v21, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v10, v10
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v12
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff0000, v12
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v48, v19, v22 :: v_dual_lshlrev_b32 v7, 16, v9
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v39, v19, v22, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v21, 0x40c00000, v10
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v39.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v37.h
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v22, v21, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v37, 0x400000, v21
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v48, 0x400000, v21
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v10, 0xffff, v19, v48
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v10, 0xffff, v19, v39
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v12, 16, 1
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v22, v22, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v9
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v50, 0x400000, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v70, 24, v10
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v21, 0x40c00000, v49
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v24, v24, v12, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v54, v22, v37, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v52, v22, v48 :: v_dual_add_f32 v9, 0x40c00000, v23
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v14
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v71, 24, v10
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v20, v9, 16, 1
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v9
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v14, 0x40c00000, v14
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v7, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v7
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v22, 0xffff0000, v14
-; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v20, v9, 0x7fff
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v10
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v19, v7, 0x7fff
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v38, 0xffff0000, v11
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v54.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v65, v19, v25, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v52.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v55, v19, v25, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v9
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v21, 0x40c00000, v38
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v14, 16, 1
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v80, 8, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v52, v24, v50 :: v_dual_add_f32 v9, 0x40c00000, v23
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v19, v21, 16, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v12, 0xffff, v7, v52
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v20, v9, 16, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v23, 0x400000, v9
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v14, 16, 1
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v53, v24, v50, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v12, 0xffff, v7, v53
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v9, v20, v23, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v7, 0x40c00000, v11
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v11, v19, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_add3_u32 v20, v20, v9, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, 0x400000, v21
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v55, 24, v12
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v67, 8, v12
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v9, v20, v23, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v55.h
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v20.l, v65.h
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v65, 24, v12
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v66, 8, v12
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v11, v11, v19, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v21, v23, v7, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v7
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v23, 0xffff0000, v13
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v19, 16, 1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_add_f32_e32 v7, 0x40c00000, v23
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v71, v21, v22, vcc_lo
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v9, 0xffff, v20, v9
+; GFX11-TRUE16-NEXT:    v_dual_add_f32 v7, 0x40c00000, v23 :: v_dual_cndmask_b32 v70, v21, v22
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v21, v24, v19, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v22, 0x400000, v19
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v19, v19
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v23, v25, v14, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, 0x400000, v14
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v25, v7, 16, 1
-; GFX11-TRUE16-NEXT:    v_dual_add_f32 v13, 0x40c00000, v13 :: v_dual_cndmask_b32 v66, v21, v22
+; GFX11-TRUE16-NEXT:    v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v67, v21, v22, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v19, 0x400000, v7
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v14, v25, v7, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v21, 0xffff0000, v16
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v68, v23, v24, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v69, v23, v24, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v13, 16, 1
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v9, 0xffff, v20, v9
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v81, 8, v9
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v68.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v22.l, v69.h
 ; GFX11-TRUE16-NEXT:    v_dual_cndmask_b32 v7, v14, v19 :: v_dual_add_f32 v14, 0x40c00000, v21
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v21, 16, v15
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v23, v13, 0x7fff
@@ -86067,42 +85744,42 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v21, 0x40c00000, v21
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v15, 0xffff0000, v15
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v25, 0x400000, v16
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v86, v19, v23, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v85, v19, v23, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v13, v16, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v23, v21, 16, 1
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v16, v16
 ; GFX11-TRUE16-NEXT:    v_bfe_u32 v24, v14, 16, 1
 ; GFX11-TRUE16-NEXT:    v_add_f32_e32 v15, 0x40c00000, v15
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v38, 0x400000, v21
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v49, 0x400000, v21
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v23, v23, v21, 0x7fff
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v82, v13, v25, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v83, v13, v25, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v21, v21
 ; GFX11-TRUE16-NEXT:    v_add3_u32 v19, v24, v14, 0x7fff
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v24, 0x400000, v14
-; GFX11-TRUE16-NEXT:    v_bfe_u32 v37, v15, 16, 1
+; GFX11-TRUE16-NEXT:    v_bfe_u32 v48, v15, 16, 1
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, 0x400000, v15
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v85, v23, v38, vcc_lo
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v87, v23, v49, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v86.h
-; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v37, v15, 0x7fff
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v71.h
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v14, 0xffff, v22, v66
-; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v81, v19, v24, vcc_lo
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v21.l, v85.h
+; GFX11-TRUE16-NEXT:    v_add3_u32 v13, v48, v15, 0x7fff
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v23.l, v70.h
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v14, 0xffff, v22, v67
+; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v86, v19, v24, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v82.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v85.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v83.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v87.h
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v11, 0xffff, v23, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 24, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 24, v14
 ; GFX11-TRUE16-NEXT:    v_cndmask_b32_e32 v13, v13, v16, vcc_lo
-; GFX11-TRUE16-NEXT:    v_bfi_b32 v16, 0xffff, v19, v81
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v53, 8, v14
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v69, 8, v11
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v84, 8, v9
+; GFX11-TRUE16-NEXT:    v_bfi_b32 v16, 0xffff, v19, v86
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v54, 8, v14
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v68, 8, v11
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v15, 0xffff, v15, v13
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v13, 0xffff, v21, v7
 ; GFX11-TRUE16-NEXT:    v_bfi_b32 v7, 0xffff, v18, v17
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v37, 24, v16
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v38, 8, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v48, 24, v16
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v49, 8, v16
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[17:18], 24, v[15:16]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[18:19], 24, v[13:14]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[19:20], 24, v[11:12]
@@ -86111,159 +85788,142 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[22:23], 24, v[5:6]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[23:24], 24, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b64 v[24:25], 24, v[1:2]
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v51, 8, v15
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v50, 8, v15
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v64, 8, v13
-; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v97, 8, v7
+; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v96, 8, v7
 ; GFX11-TRUE16-NEXT:  .LBB108_4: ; %end
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v28.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v113.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v27.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v112.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v2.h, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v103.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.l, 0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v1.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v102.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v1.l, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v101.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v24
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v24.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v24.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v27.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v1.l, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v26.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v112.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v103.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v3.l, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v8, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v6
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v36.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v3.h, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v24, v1
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v2.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v102.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v3.h, v4.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v101.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v24, v2
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v30.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v3.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v99.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v100.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v8, v24
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v24, v3
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v5.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v99.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v5.h, v6.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v98.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v4.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v33.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v97.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v6.l, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v49.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v8, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v5.l, v5.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v96.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v6.h, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v10, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v12
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v24, v4
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v5.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v97.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v7.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v6.l, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v35.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v87.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v10, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v10, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v7.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v65.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v7.h, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v84.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v69.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v10, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v83.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v34.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v8.h, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v39.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v24, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v96.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v7.h, v8.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v84.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v24, v6
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v7.l, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v82.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v9.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v24, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v8.l, v8.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v9.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v81.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.h, v9.h, v10.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 8, v80.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v8.l, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v8.h, v10.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v71.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v10.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v48.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v70.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v10.h, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v14, v24
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v67.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v14, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v10.l, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v54.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v11.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v64.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v12.l, v12.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v12.h, 0xff, v86.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v14, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v16
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v11.l, v11.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v24, v8
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v9.l, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v71.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v11.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v24, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v10.l, v10.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.h, v11.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v11.l, 0xff, v70.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.l, 8, v68.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.h, v11.h, v12.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v66.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v10, v24, v10
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v53.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v11.l, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v24.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v52.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v55.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v12.h, v13.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v16, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v19
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v65.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v13.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v12.l, v14.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v68.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v53.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v16, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v16, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v13.l, v14.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v85.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v13.h, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.l, 8, v51.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v16, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v11, v24, v11
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v12.l, v12.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v13.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.l, 0xff, v85.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.l, 8, v64.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v13.h, v14.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v14.h, 8, v54.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v12, v24, v12
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v67.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v13.l, v14.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v69.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v51.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.h, 0xff, v15.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v13, v24, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v14.l, v14.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v15.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v87.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.l, 8, v50.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v14.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, 0xff, v66.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v14.h, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, 0xff, v82.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v38.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v18, 0xffff, v18
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v14.l, v16.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v19.l, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v15.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v15.h, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v14.h, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v18, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v19
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v37.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v18.l, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v81.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v15.h, v16.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 8, v49.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v14, v24, v14
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.l, 0xff, v86.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v15.l, v16.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.l, v24.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, 0xff, v83.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 8, v48.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v17, v24
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v17, 0xffff, v18
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v17, v24
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v15, v24, v15
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v16.l, v16.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v17.l, v17.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v16.l, v24.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v16, v24, v16
 ; GFX11-TRUE16-NEXT:    s_clause 0x3
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[1:4], off
 ; GFX11-TRUE16-NEXT:    scratch_store_b128 v0, v[5:8], off offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 6fe66655de3d6..4c485768bcbbf 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -3065,13 +3065,12 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v7.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v8
@@ -3085,61 +3084,53 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB26_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v0.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v1.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v2
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v4
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB26_2
 ; GFX11-TRUE16-NEXT:  .LBB26_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v5.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v3.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v4.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v3.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v4.h, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v2.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v3.l, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v3.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v2.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v4, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6214,13 +6205,12 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v7.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v8
@@ -6234,61 +6224,53 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB50_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v0.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v1.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v2
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v4
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB50_2
 ; GFX11-TRUE16-NEXT:  .LBB50_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v5.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v3.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v4.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v3.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v4.h, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v2.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v3.l, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v3.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v2.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v4, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -9063,13 +9045,12 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v7.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v8
@@ -9083,61 +9064,53 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB70_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v0.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v1.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v2
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v4
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB70_2
 ; GFX11-TRUE16-NEXT:  .LBB70_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v5.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v3.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v4.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v3.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v4.h, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v2.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v3.l, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v3.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v2.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v4, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -11603,13 +11576,12 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v7.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v8
@@ -11623,61 +11595,53 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB86_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v0.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v1.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v2
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v4
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB86_2
 ; GFX11-TRUE16-NEXT:  .LBB86_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v5.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v3.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v4.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v3.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v4.h, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v2.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v3.l, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v3.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v2.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v4, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -13829,13 +13793,12 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v7.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v8
@@ -13849,61 +13812,53 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB98_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v0.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v1.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v2
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v4
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB98_2
 ; GFX11-TRUE16-NEXT:  .LBB98_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v5.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v3.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v4.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v3.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v4.h, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v2.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v3.l, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v3.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v2.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v4, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -15655,13 +15610,12 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v7.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v8
@@ -15675,61 +15629,53 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB106_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v0.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v1.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v2
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v4
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB106_2
 ; GFX11-TRUE16-NEXT:  .LBB106_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v5.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v3.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v4.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v3.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v4.h, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v2.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v3.l, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v3.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v2.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v4, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -16966,13 +16912,12 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v5.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v7.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v8
@@ -16986,61 +16931,53 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ; GFX11-TRUE16-NEXT:  .LBB110_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v3.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.h, v4.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v0.l, v2.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v1.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v5, v2
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v2.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v4
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB110_2
 ; GFX11-TRUE16-NEXT:  .LBB110_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v5.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v3.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, 0
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.h, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v4.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v3.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v4.h, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.h, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v2.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v3.l, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v3.h, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v2.l, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.h, 0x300, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v4, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index e5245f7bd71d3..879e8520d8e18 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -1102,15 +1102,16 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.h, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v9.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v8.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v11.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v12
@@ -1125,80 +1126,74 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB6_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v5.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v0.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v4.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v3.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v1.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v3.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v2.l, v3.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v2.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v2
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB6_2
 ; GFX11-TRUE16-NEXT:  .LBB6_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v7.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v7.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v6.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v5.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v5.h, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v5.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v5.l, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v7.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v4.l, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v4.h, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v3.h, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v4, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v2.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4241,15 +4236,16 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.h, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v8.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v9.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v8.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v3.h, 8, v11.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v12
@@ -4264,80 +4260,74 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB22_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v7.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v5.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v5.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v0.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v5.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v4.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v4, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v3.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v1.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v3.h
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v2.l, v3.l
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v7.l, v2.l, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v2
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB22_2
 ; GFX11-TRUE16-NEXT:  .LBB22_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v7.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v7.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v6.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v5.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v5.h, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v5.h, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v5.l, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v7.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v4.l, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v4.h, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v4.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v3.h, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v7, v5
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v4, v7
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v7
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.l, 0x300, v2.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v7, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6885,16 +6875,16 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v9.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v10.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v10.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v9.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v12
@@ -6909,80 +6899,74 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB36_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v9.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v6.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v1.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v2.l, v4.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.h, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v4.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v2.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v7
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB36_2
 ; GFX11-TRUE16-NEXT:  .LBB36_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v8.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v7.l, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v6.l, v0.l
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.h, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v4.h, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v4.h, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v2.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8651,16 +8635,16 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v9.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v10.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v10.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v9.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v12
@@ -8675,80 +8659,74 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB40_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v9.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v6.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v1.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v2.l, v4.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.h, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v4.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v2.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v7
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB40_2
 ; GFX11-TRUE16-NEXT:  .LBB40_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v8.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v7.l, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v6.l, v0.l
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.h, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v4.h, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v4.h, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v2.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -10065,16 +10043,16 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v9.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v7.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v5.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.h, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v5.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v10.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v9.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v10.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v9.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, exec_lo
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GFX11-TRUE16-NEXT:    v_cmpx_ne_u32_e32 0, v12
@@ -10089,80 +10067,74 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) {
 ; GFX11-TRUE16-NEXT:  .LBB44_3: ; %cmp.false
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v9.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.h, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v5.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v10.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v7.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v6.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v1.l, v5.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v10.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr9_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr8_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v1.l, v4.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v5
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v6, v7
-; GFX11-TRUE16-NEXT:    v_or_b16 v7.h, v2.l, v4.l
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_lo16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr6_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v0
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v1.h, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v2.h, v4.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v5
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v2.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr5_hi16
 ; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v7
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_hi16
-; GFX11-TRUE16-NEXT:    ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    s_and_not1_saveexec_b32 s0, s0
 ; GFX11-TRUE16-NEXT:    s_cbranch_execz .LBB44_2
 ; GFX11-TRUE16-NEXT:  .LBB44_4: ; %cmp.true
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v9.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v8.h, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, v7.h, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v8.l, 3
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v6.l, 3
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, v7.l, 3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.l, v8.l, 3
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, v10.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v7.l, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v6.l, v0.l
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v6.h, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v5.l, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x300, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v5.h, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.l, 0
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v4.h, v0.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v2.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v6
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v1.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v1.h, 0x300, v1.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v5, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v1.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v9
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v9.h, 0x300, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v9
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v4.h, v2.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v3, v1
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v3.l, 0x300, v2.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v2.h, 0x300, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-llvm-debuginfo-analyzer.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-llvm-debuginfo-analyzer.ll
new file mode 100644
index 0000000000000..d6922bc09ff0a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-llvm-debuginfo-analyzer.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc %s -o %t.o -mcpu=gfx1030 -filetype=obj -O0
+; RUN: llvm-debuginfo-analyzer %t.o --print=all --attribute=all | FileCheck %s
+
+; This test compiles this module with AMDGPU backend under -O0,
+; and makes sure llvm-debuginfo-analyzer works for it.
+
+; Simple checks to make sure llvm-debuginfo-analzyer didn't fail early.
+; CHECK: Logical View:
+; CHECK: {CompileUnit}
+; CHECK-DAG: {Parameter} 'dtid' -> [0x{{[a-f0-9]+}}]'uint3'
+; CHECK-DAG: {Variable} 'my_var2' -> [0x{{[a-f0-9]+}}]'float'
+; CHECK-DAG: {Line} {{.+}}basic_var.hlsl
+; CHECK: {Code} 's_endpgm'
+
+source_filename = "module"
+target triple = "amdgcn-amd-amdpal"
+
+%dx.types.ResRet.f32 = type { float, float, float, float, i32 }
+
+define dllexport amdgpu_cs void @_amdgpu_cs_main(i32 inreg noundef %globalTable, i32 inreg noundef %userdata4, <3 x i32> inreg noundef %WorkgroupId, i32 inreg noundef %MultiDispatchInfo, <3 x i32> noundef %LocalInvocationId) #0 !dbg !14 {
+  %LocalInvocationId.i0 = extractelement <3 x i32> %LocalInvocationId, i64 0, !dbg !28
+  %WorkgroupId.i0 = extractelement <3 x i32> %WorkgroupId, i64 0, !dbg !28
+  %pc = call i64 @llvm.amdgcn.s.getpc(), !dbg !28
+  %offset = shl i32 %WorkgroupId.i0, 6, !dbg !28
+  %dtid = add i32 %LocalInvocationId.i0, %offset, !dbg !28
+    #dbg_value(i32 %dtid, !29, !DIExpression(DW_OP_LLVM_fragment, 0, 32), !28)
+  %pc_hi = and i64 %pc, -4294967296, !dbg !30
+  %zext = zext i32 %userdata4 to i64, !dbg !30
+  %ptr_val = or disjoint i64 %pc_hi, %zext, !dbg !30
+  %ptr = inttoptr i64 %ptr_val to ptr addrspace(4), !dbg !30
+  call void @llvm.assume(i1 true) [ "align"(ptr addrspace(4) %ptr, i32 4), "dereferenceable"(ptr addrspace(4) %ptr, i32 -1) ], !dbg !30
+  %uav_0 = load <4 x i32>, ptr addrspace(4) %ptr, align 4, !dbg !30, !invariant.load !2
+  %uav_load_1 = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %uav_0, i32 %dtid, i32 0, i32 0, i32 0), !dbg !30
+    #dbg_value(%dx.types.ResRet.f32 poison, !31, !DIExpression(), !32)
+  %mul = fmul reassoc arcp contract afn float %uav_load_1, 2.000000e+00, !dbg !33
+    #dbg_value(float %mul, !34, !DIExpression(), !35)
+  call void @llvm.assume(i1 true) [ "align"(ptr addrspace(4) %ptr, i32 4), "dereferenceable"(ptr addrspace(4) %ptr, i32 -1) ], !dbg !36
+  %uav_1_ptr = getelementptr i8, ptr addrspace(4) %ptr, i64 32, !dbg !36
+  %.upto01 = insertelement <4 x float> poison, float %mul, i64 0, !dbg !36
+  %filled_vector = shufflevector <4 x float> %.upto01, <4 x float> poison, <4 x i32> zeroinitializer, !dbg !36
+  %uav_1 = load <4 x i32>, ptr addrspace(4) %uav_1_ptr, align 4, !dbg !36, !invariant.load !2
+  call void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float> %filled_vector, <4 x i32> %uav_1, i32 %dtid, i32 0, i32 0, i32 0), !dbg !36
+  ret void, !dbg !37
+}
+
+declare noundef i64 @llvm.amdgcn.s.getpc() #1
+
+declare void @llvm.assume(i1 noundef) #2
+
+declare void @llvm.amdgcn.struct.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32 immarg) #3
+
+declare float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32>, i32, i32, i32, i32 immarg) #4
+
+attributes #0 = { memory(readwrite) }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #2 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+attributes #3 = { nocallback nofree nosync nounwind willreturn memory(write) }
+attributes #4 = { nocallback nofree nosync nounwind willreturn memory(read) }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!12, !13}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "dxcoob 1.7.2308.16 (52da17e29)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !3)
+!1 = !DIFile(filename: "tests\\basic_var.hlsl", directory: "")
+!2 = !{}
+!3 = !{!4, !10}
+!4 = distinct !DIGlobalVariableExpression(var: !5, expr: !DIExpression())
+!5 = !DIGlobalVariable(name: "u0", linkageName: "\01?u0@@3V?$RWBuffer@M@@A", scope: !0, file: !1, line: 2, type: !6, isLocal: false, isDefinition: true)
+!6 = !DICompositeType(tag: DW_TAG_class_type, name: "RWBuffer<float>", file: !1, line: 2, size: 32, align: 32, elements: !2, templateParams: !7)
+!7 = !{!8}
+!8 = !DITemplateTypeParameter(name: "element", type: !9)
+!9 = !DIBasicType(name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+!10 = distinct !DIGlobalVariableExpression(var: !11, expr: !DIExpression())
+!11 = !DIGlobalVariable(name: "u1", linkageName: "\01?u1@@3V?$RWBuffer@M@@A", scope: !0, file: !1, line: 3, type: !6, isLocal: false, isDefinition: true)
+!12 = !{i32 2, !"Dwarf Version", i32 5}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !15, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!15 = !DISubroutineType(types: !16)
+!16 = !{null, !17}
+!17 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint3", file: !1, baseType: !18)
+!18 = !DICompositeType(tag: DW_TAG_class_type, name: "vector<unsigned int, 3>", file: !1, size: 96, align: 32, elements: !19, templateParams: !24)
+!19 = !{!20, !22, !23}
+!20 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !18, file: !1, baseType: !21, size: 32, align: 32, flags: DIFlagPublic)
+!21 = !DIBasicType(name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+!22 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !18, file: !1, baseType: !21, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
+!23 = !DIDerivedType(tag: DW_TAG_member, name: "z", scope: !18, file: !1, baseType: !21, size: 32, align: 32, offset: 64, flags: DIFlagPublic)
+!24 = !{!25, !26}
+!25 = !DITemplateTypeParameter(name: "element", type: !21)
+!26 = !DITemplateValueParameter(name: "element_count", type: !27, value: i32 3)
+!27 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+!28 = !DILocation(line: 7, column: 17, scope: !14)
+!29 = !DILocalVariable(name: "dtid", arg: 1, scope: !14, file: !1, line: 7, type: !17)
+!30 = !DILocation(line: 11, column: 18, scope: !14)
+!31 = !DILocalVariable(name: "my_var", scope: !14, file: !1, line: 11, type: !9)
+!32 = !DILocation(line: 11, column: 9, scope: !14)
+!33 = !DILocation(line: 14, column: 26, scope: !14)
+!34 = !DILocalVariable(name: "my_var2", scope: !14, file: !1, line: 14, type: !9)
+!35 = !DILocation(line: 14, column: 9, scope: !14)
+!36 = !DILocation(line: 17, column: 14, scope: !14)
+!37 = !DILocation(line: 19, column: 1, scope: !14)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
index 69bdb1f5066f0..aaacf1d6f793b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-prepare-agpr-alloc.mir
@@ -13,6 +13,14 @@
     ret void
   }
 
+  define void @func64() {
+    ret void
+  }
+
+  define void @func64_no_agprs() "amdgpu-agpr-alloc"="0,0" {
+    ret void
+  }
+
 ...
 ---
 name: func
@@ -93,3 +101,116 @@ body:             |
     %1:agpr_32 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec
 
 ...
+
+---
+name: func64
+tracksRegLiveness: true
+stack:
+  - { id: 0, size: 4 }
+body:             |
+  ; HAS-AGPR-LABEL: name: func64
+  ; HAS-AGPR: bb.0:
+  ; HAS-AGPR-NEXT:   successors: %bb.1(0x80000000)
+  ; HAS-AGPR-NEXT:   liveins: $vgpr0_vgpr1
+  ; HAS-AGPR-NEXT: {{  $}}
+  ; HAS-AGPR-NEXT:   [[V_MOV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_e64 $vgpr0_vgpr1, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_:%[0-9]+]]:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 54, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_1:%[0-9]+]]:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 1, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_2:%[0-9]+]]:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 64, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B64_e64_1:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_e64 %stack.0, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 65, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B1:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 279172874240, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B2:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 279172874305, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B3:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 4290672329938, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B4:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO -9223372036854775808, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_3:%[0-9]+]]:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 1042479491, implicit $exec
+  ; HAS-AGPR-NEXT:   [[AV_MOV_4:%[0-9]+]]:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 4477415320595726336, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B5:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO $vgpr0_vgpr1, implicit $exec
+  ; HAS-AGPR-NEXT:   [[V_MOV_B6:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO %stack.0, implicit $exec
+  ; HAS-AGPR-NEXT: {{  $}}
+  ; HAS-AGPR-NEXT: bb.1:
+  ; HAS-AGPR-NEXT:   [[AV_MOV_5:%[0-9]+]]:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 3, implicit $exec
+  ;
+  ; NO-AGPR-LABEL: name: func64
+  ; NO-AGPR: bb.0:
+  ; NO-AGPR-NEXT:   successors: %bb.1(0x80000000)
+  ; NO-AGPR-NEXT:   liveins: $vgpr0_vgpr1
+  ; NO-AGPR-NEXT: {{  $}}
+  ; NO-AGPR-NEXT:   [[V_MOV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_e64 $vgpr0_vgpr1, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 54, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B64_e64_1:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_e64 1, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B64_e64_2:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_e64 64, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B64_e64_3:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_e64 %stack.0, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B1:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 65, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B2:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 279172874240, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B3:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 279172874305, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B4:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 4290672329938, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B5:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO -9223372036854775808, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B6:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1042479491, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B7:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 4477415320595726336, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B8:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO $vgpr0_vgpr1, implicit $exec
+  ; NO-AGPR-NEXT:   [[V_MOV_B9:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO %stack.0, implicit $exec
+  ; NO-AGPR-NEXT: {{  $}}
+  ; NO-AGPR-NEXT: bb.1:
+  ; NO-AGPR-NEXT:   [[V_MOV_B64_e64_4:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_e64 3, implicit $exec
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    %0:vreg_64_align2 = V_MOV_B64_e64 $vgpr0_vgpr1, implicit $exec
+    %1:vreg_64_align2 = V_MOV_B64_PSEUDO 54, implicit $exec
+    %2:vreg_64_align2 = V_MOV_B64_e64 1, implicit $exec
+    %3:vreg_64_align2 = V_MOV_B64_e64 64, implicit $exec
+    %4:vreg_64_align2 = V_MOV_B64_e64 %stack.0, implicit $exec
+    %5:vreg_64_align2 = V_MOV_B64_PSEUDO 65, implicit $exec
+    %6:vreg_64_align2 = V_MOV_B64_PSEUDO 279172874240, implicit $exec
+    %7:vreg_64_align2 = V_MOV_B64_PSEUDO 279172874305, implicit $exec
+    %8:vreg_64_align2 = V_MOV_B64_PSEUDO 4290672329938, implicit $exec
+    %9:vreg_64_align2 = V_MOV_B64_PSEUDO 9223372036854775808, implicit $exec
+    %10:vreg_64_align2 = V_MOV_B64_PSEUDO 1042479491, implicit $exec
+    %11:vreg_64_align2 = V_MOV_B64_PSEUDO 4477415320595726336, implicit $exec
+    %12:vreg_64_align2 = V_MOV_B64_PSEUDO $vgpr0_vgpr1, implicit $exec
+    %13:vreg_64_align2 = V_MOV_B64_PSEUDO %stack.0, implicit $exec
+
+  bb.1:
+    %14:vreg_64_align2 = V_MOV_B64_e64 3, implicit $exec
+
+...
+
+---
+name: func64_no_agprs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0
+    ; GFX90A-LABEL: name: func64_no_agprs
+    ; GFX90A: liveins: $vgpr0
+    ; GFX90A-NEXT: {{  $}}
+    ; GFX90A-NEXT: [[V_MOV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_e64 1, implicit $exec
+    ; GFX90A-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 4290672329938, implicit $exec
+    ; GFX90A-NEXT: [[V_MOV_B1:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO -9223372036854775808, implicit $exec
+    ; GFX90A-NEXT: [[V_MOV_B2:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1042479491, implicit $exec
+    ; GFX90A-NEXT: [[V_MOV_B3:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 4477415320595726336, implicit $exec
+    ;
+    ; GFX908-LABEL: name: func64_no_agprs
+    ; GFX908: liveins: $vgpr0
+    ; GFX908-NEXT: {{  $}}
+    ; GFX908-NEXT: [[AV_MOV_:%[0-9]+]]:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 1, implicit $exec
+    ; GFX908-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 4290672329938, implicit $exec
+    ; GFX908-NEXT: [[V_MOV_B1:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO -9223372036854775808, implicit $exec
+    ; GFX908-NEXT: [[AV_MOV_1:%[0-9]+]]:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 1042479491, implicit $exec
+    ; GFX908-NEXT: [[AV_MOV_2:%[0-9]+]]:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 4477415320595726336, implicit $exec
+    ;
+    ; NO-AGPR-LABEL: name: func64_no_agprs
+    ; NO-AGPR: liveins: $vgpr0
+    ; NO-AGPR-NEXT: {{  $}}
+    ; NO-AGPR-NEXT: [[V_MOV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_e64 1, implicit $exec
+    ; NO-AGPR-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 4290672329938, implicit $exec
+    ; NO-AGPR-NEXT: [[V_MOV_B1:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO -9223372036854775808, implicit $exec
+    ; NO-AGPR-NEXT: [[V_MOV_B2:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1042479491, implicit $exec
+    ; NO-AGPR-NEXT: [[V_MOV_B3:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 4477415320595726336, implicit $exec
+    %0:vreg_64_align2 = V_MOV_B64_e64 1, implicit $exec
+    %1:vreg_64_align2 = V_MOV_B64_PSEUDO 4290672329938, implicit $exec
+    %2:vreg_64_align2 = V_MOV_B64_PSEUDO 9223372036854775808, implicit $exec
+    %3:vreg_64_align2 = V_MOV_B64_PSEUDO 1042479491, implicit $exec
+    %4:vreg_64_align2 = V_MOV_B64_PSEUDO 4477415320595726336, implicit $exec
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll b/llvm/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
index 45192be37c242..9a4040a25419a 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
@@ -1,24 +1,94 @@
-; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefixes=SI,SICI,SICIVI,PREGFX11,GCN %s
-; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefixes=SICI,CIVI,SICIVI,PREGFX11,GCN %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=CIVI,SICIVI,GFX8PLUS,PREGFX11,GCN %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9PLUS,GFX8PLUS,PREGFX11,GCN %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11,GFX9PLUS,GFX8PLUS,GCN %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn -mcpu=bonaire < %s | FileCheck -check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=GFX8 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GFX11 %s
 
-; GCN-LABEL: {{^}}lds_atomic_cmpxchg_ret_i32_offset:
-; GFX9PLUS-NOT: m0
-; SICIVI-DAG: s_mov_b32 m0
-
-; SICI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x13
-; SICI-DAG: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x1c
-; GFX8PLUS-DAG: s_load_{{dword|b32}} [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x4c
-; GFX8PLUS-DAG: s_load_{{dword|b32}} [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x70
-; GCN-DAG: v_mov_b32_e32 [[VCMP:v[0-9]+]], 7
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
-; GCN-DAG: v_mov_b32_e32 [[VSWAP:v[0-9]+]], [[SWAP]]
-; PREGFX11: ds_cmpst_rtn_b32 [[RESULT:v[0-9]+]], [[VPTR]], [[VCMP]], [[VSWAP]] offset:16
-; GFX11: ds_cmpstore_rtn_b32 [[RESULT:v[0-9]+]], [[VPTR]], [[VSWAP]], [[VCMP]] offset:16
-; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i32_offset(ptr addrspace(1) %out, [8 x i32], ptr addrspace(3) %ptr, [8 x i32], i32 %swap) nounwind {
+; CHECK-LABEL: lds_atomic_cmpxchg_ret_i32_offset:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[4:5], 0x13
+; CHECK-NEXT:    s_load_dword s3, s[4:5], 0x1c
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CHECK-NEXT:    v_mov_b32_e32 v0, 7
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    v_mov_b32_e32 v2, s3
+; CHECK-NEXT:    s_mov_b32 m0, -1
+; CHECK-NEXT:    ds_cmpst_rtn_b32 v0, v1, v0, v2 offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s3, 0xf000
+; CHECK-NEXT:    s_mov_b32 s2, -1
+; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; CHECK-NEXT:    s_endpgm
+;
+; GFX7-LABEL: lds_atomic_cmpxchg_ret_i32_offset:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x13
+; GFX7-NEXT:    s_load_dword s3, s[4:5], 0x1c
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7-NEXT:    v_mov_b32_e32 v0, 7
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v2, s3
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v0, v1, v0, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: lds_atomic_cmpxchg_ret_i32_offset:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x4c
+; GFX8-NEXT:    s_load_dword s3, s[4:5], 0x70
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v0, 7
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, s2
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    ds_cmpst_rtn_b32 v2, v1, v0, v2 offset:16
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: lds_atomic_cmpxchg_ret_i32_offset:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x4c
+; GFX9-NEXT:    s_load_dword s3, s[4:5], 0x70
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 7
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    ds_cmpst_rtn_b32 v0, v1, v0, v2 offset:16
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: lds_atomic_cmpxchg_ret_i32_offset:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s0, s[4:5], 0x4c
+; GFX11-NEXT:    s_load_b32 s1, s[4:5], 0x70
+; GFX11-NEXT:    v_mov_b32_e32 v0, 7
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s0
+; GFX11-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    ds_cmpstore_rtn_b32 v0, v1, v2, v0 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4
   %pair = cmpxchg ptr addrspace(3) %gep, i32 7, i32 %swap seq_cst monotonic
   %result = extractvalue { i32, i1 } %pair, 0
@@ -26,24 +96,100 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i32_offset(ptr addrspace(1) %o
   ret void
 }
 
-; GCN-LABEL: {{^}}lds_atomic_cmpxchg_ret_i64_offset:
-; GFX9PLUS-NOT: m0
-; SICIVI-DAG: s_mov_b32 m0
-
-; SICI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
-; SICI-DAG: s_load_dwordx2 s[[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, 0xd
-; GFX8PLUS-DAG: s_load_{{dword|b32}} [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
-; GFX8PLUS-DAG: s_load_{{dwordx2|b64}} s[[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, 0x34
-; GCN-DAG: v_mov_b32_e32 v[[LOVCMP:[0-9]+]], 7
-; GCN-DAG: v_mov_b32_e32 v[[HIVCMP:[0-9]+]], 0
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
-; GCN-DAG: v_mov_b32_e32 v[[LOSWAPV:[0-9]+]], s[[LOSWAP]]
-; GCN-DAG: v_mov_b32_e32 v[[HISWAPV:[0-9]+]], s[[HISWAP]]
-; PREGFX11: ds_cmpst_rtn_b64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v[[[LOVCMP]]:[[HIVCMP]]], v[[[LOSWAPV]]:[[HISWAPV]]] offset:32
-; GFX11: ds_cmpstore_rtn_b64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v[[[LOSWAPV]]:[[HISWAPV]]], v[[[LOVCMP]]:[[HIVCMP]]] offset:32
-; GCN: [[RESULT]]
-; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr, i64 %swap) nounwind {
+; CHECK-LABEL: lds_atomic_cmpxchg_ret_i64_offset:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s6, s[4:5], 0xb
+; CHECK-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0xd
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CHECK-NEXT:    v_mov_b32_e32 v0, 7
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v4, s6
+; CHECK-NEXT:    v_mov_b32_e32 v2, s2
+; CHECK-NEXT:    v_mov_b32_e32 v3, s3
+; CHECK-NEXT:    s_mov_b32 m0, -1
+; CHECK-NEXT:    ds_cmpst_rtn_b64 v[0:1], v4, v[0:1], v[2:3] offset:32
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s3, 0xf000
+; CHECK-NEXT:    s_mov_b32 s2, -1
+; CHECK-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; CHECK-NEXT:    s_endpgm
+;
+; GFX7-LABEL: lds_atomic_cmpxchg_ret_i64_offset:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dword s6, s[4:5], 0xb
+; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0xd
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7-NEXT:    v_mov_b32_e32 v0, 7
+; GFX7-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v4, s6
+; GFX7-NEXT:    v_mov_b32_e32 v2, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s3
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    ds_cmpst_rtn_b64 v[0:1], v4, v[0:1], v[2:3] offset:32
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: lds_atomic_cmpxchg_ret_i64_offset:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX8-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v0, 7
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v4, s6
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    ds_cmpst_rtn_b64 v[0:1], v4, v[0:1], v[2:3] offset:32
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v2, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: lds_atomic_cmpxchg_ret_i64_offset:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 7
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    ds_cmpst_rtn_b64 v[0:1], v4, v[0:1], v[2:3] offset:32
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: lds_atomic_cmpxchg_ret_i64_offset:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT:    v_mov_b32_e32 v0, 7
+; GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v4, s2
+; GFX11-NEXT:    v_mov_b32_e32 v3, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s0
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    ds_cmpstore_rtn_b64 v[0:1], v4, v[2:3], v[0:1] offset:32
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4
   %pair = cmpxchg ptr addrspace(3) %gep, i64 7, i64 %swap seq_cst monotonic
   %result = extractvalue { i64, i1 } %pair, 0
@@ -51,13 +197,103 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i64_offset(ptr addrspace(1) %o
   ret void
 }
 
-; GCN-LABEL: {{^}}lds_atomic_cmpxchg_ret_i32_bad_si_offset
-; GFX9PLUS-NOT: m0
-; SI: ds_cmpst_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; CIVI: ds_cmpst_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
-; GFX9PLUS: ds_{{cmpst|cmpstore}}_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
-; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i32_bad_si_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr, i32 %swap, i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: lds_atomic_cmpxchg_ret_i32_bad_si_offset:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xc
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_load_dword s3, s[4:5], 0xb
+; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; CHECK-NEXT:    s_sub_i32 s1, s1, s2
+; CHECK-NEXT:    s_lshl_b32 s1, s1, 2
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_add_i32 s1, s3, s1
+; CHECK-NEXT:    s_add_i32 s1, s1, 16
+; CHECK-NEXT:    v_mov_b32_e32 v0, 7
+; CHECK-NEXT:    v_mov_b32_e32 v1, s0
+; CHECK-NEXT:    v_mov_b32_e32 v2, s1
+; CHECK-NEXT:    s_mov_b32 m0, -1
+; CHECK-NEXT:    ds_cmpst_rtn_b32 v0, v2, v0, v1
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_mov_b32 s7, 0xf000
+; CHECK-NEXT:    s_mov_b32 s6, -1
+; CHECK-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; CHECK-NEXT:    s_endpgm
+;
+; GFX7-LABEL: lds_atomic_cmpxchg_ret_i32_bad_si_offset:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX7-NEXT:    v_mov_b32_e32 v0, 7
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_sub_i32 s2, s2, s3
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    s_lshl_b32 s1, s2, 2
+; GFX7-NEXT:    s_add_i32 s0, s0, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:    ds_cmpst_rtn_b32 v0, v2, v0, v1 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_mov_b32 s6, -1
+; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: lds_atomic_cmpxchg_ret_i32_bad_si_offset:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v0, 7
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_sub_i32 s2, s2, s3
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_lshl_b32 s1, s2, 2
+; GFX8-NEXT:    s_add_i32 s0, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    ds_cmpst_rtn_b32 v2, v2, v0, v1 offset:16
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: lds_atomic_cmpxchg_ret_i32_bad_si_offset:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v0, 7
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_sub_i32 s2, s2, s3
+; GFX9-NEXT:    s_lshl_b32 s2, s2, 2
+; GFX9-NEXT:    s_add_i32 s0, s0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    ds_cmpst_rtn_b32 v0, v1, v0, v2 offset:16
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_store_dword v1, v0, s[6:7]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: lds_atomic_cmpxchg_ret_i32_bad_si_offset:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT:    v_mov_b32_e32 v0, 7
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_sub_i32 s2, s2, s3
+; GFX11-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-NEXT:    s_lshl_b32 s2, s2, 2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_add_i32 s0, s0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s0
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    ds_cmpstore_rtn_b32 v0, v1, v2, v0 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-NEXT:    global_store_b32 v1, v0, s[0:1]
+; GFX11-NEXT:    s_endpgm
   %sub = sub i32 %a, %b
   %add = add i32 %sub, 4
   %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 %add
@@ -67,45 +303,152 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i32_bad_si_offset(ptr addrspac
   ret void
 }
 
-; GCN-LABEL: {{^}}lds_atomic_cmpxchg_noret_i32_offset:
-; GFX9PLUS-NOT: m0
-; SICIVI-DAG: s_mov_b32 m0
-
-
-; SICI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x9
-; SICI-DAG: s_load_dword [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x12
-; GFX8PLUS-DAG: s_load_{{dword|b32}} [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x24
-; GFX8PLUS-DAG: s_load_{{dword|b32}} [[SWAP:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x48
-; GCN-DAG: v_mov_b32_e32 [[VCMP:v[0-9]+]], 7
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
-; GCN-DAG: v_mov_b32_e32 [[VSWAP:v[0-9]+]], [[SWAP]]
-; PREGFX11: ds_cmpst_b32 [[VPTR]], [[VCMP]], [[VSWAP]] offset:16
-; GFX11: ds_cmpstore_b32 [[VPTR]], [[VSWAP]], [[VCMP]] offset:16
-; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_cmpxchg_noret_i32_offset(ptr addrspace(3) %ptr, [8 x i32], i32 %swap) nounwind {
+; CHECK-LABEL: lds_atomic_cmpxchg_noret_i32_offset:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x9
+; CHECK-NEXT:    s_load_dword s1, s[4:5], 0x12
+; CHECK-NEXT:    v_mov_b32_e32 v0, 7
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s0
+; CHECK-NEXT:    v_mov_b32_e32 v2, s1
+; CHECK-NEXT:    s_mov_b32 m0, -1
+; CHECK-NEXT:    ds_cmpst_b32 v1, v0, v2 offset:16
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_endpgm
+;
+; GFX7-LABEL: lds_atomic_cmpxchg_noret_i32_offset:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dword s0, s[4:5], 0x9
+; GFX7-NEXT:    s_load_dword s1, s[4:5], 0x12
+; GFX7-NEXT:    v_mov_b32_e32 v0, 7
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v1, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    ds_cmpst_b32 v1, v0, v2 offset:16
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: lds_atomic_cmpxchg_noret_i32_offset:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x24
+; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x48
+; GFX8-NEXT:    v_mov_b32_e32 v0, 7
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    ds_cmpst_b32 v1, v0, v2 offset:16
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: lds_atomic_cmpxchg_noret_i32_offset:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x24
+; GFX9-NEXT:    s_load_dword s1, s[4:5], 0x48
+; GFX9-NEXT:    v_mov_b32_e32 v0, 7
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    ds_cmpst_b32 v1, v0, v2 offset:16
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: lds_atomic_cmpxchg_noret_i32_offset:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s0, s[4:5], 0x24
+; GFX11-NEXT:    s_load_b32 s1, s[4:5], 0x48
+; GFX11-NEXT:    v_mov_b32_e32 v0, 7
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s0
+; GFX11-NEXT:    v_mov_b32_e32 v2, s1
+; GFX11-NEXT:    ds_cmpstore_b32 v1, v2, v0 offset:16
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(3) %ptr, i32 4
   %pair = cmpxchg ptr addrspace(3) %gep, i32 7, i32 %swap seq_cst monotonic
   %result = extractvalue { i32, i1 } %pair, 0
   ret void
 }
 
-; GCN-LABEL: {{^}}lds_atomic_cmpxchg_noret_i64_offset:
-; GFX9PLUS-NOT: m0
-; SICIVI-DAG: s_mov_b32 m0
-
-; SICI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x9
-; SICI-DAG: s_load_dwordx2 s[[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
-; GFX8PLUS-DAG: s_load_{{dword|b32}} [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x24
-; GFX8PLUS-DAG: s_load_{{dwordx2|b64}} s[[[LOSWAP:[0-9]+]]:[[HISWAP:[0-9]+]]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
-; GCN-DAG: v_mov_b32_e32 v[[LOVCMP:[0-9]+]], 7
-; GCN-DAG: v_mov_b32_e32 v[[HIVCMP:[0-9]+]], 0
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
-; GCN-DAG: v_mov_b32_e32 v[[LOSWAPV:[0-9]+]], s[[LOSWAP]]
-; GCN-DAG: v_mov_b32_e32 v[[HISWAPV:[0-9]+]], s[[HISWAP]]
-; PREGFX11: ds_cmpst_b64 [[VPTR]], v[[[LOVCMP]]:[[HIVCMP]]], v[[[LOSWAPV]]:[[HISWAPV]]] offset:32
-; GFX11: ds_cmpstore_b64 [[VPTR]], v[[[LOSWAPV]]:[[HISWAPV]]], v[[[LOVCMP]]:[[HIVCMP]]] offset:32
-; GCN: s_endpgm
 define amdgpu_kernel void @lds_atomic_cmpxchg_noret_i64_offset(ptr addrspace(3) %ptr, i64 %swap) nounwind {
+; CHECK-LABEL: lds_atomic_cmpxchg_noret_i64_offset:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[4:5], 0x9
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; CHECK-NEXT:    v_mov_b32_e32 v0, 7
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v4, s2
+; CHECK-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    s_mov_b32 m0, -1
+; CHECK-NEXT:    ds_cmpst_b64 v4, v[0:1], v[2:3] offset:32
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_endpgm
+;
+; GFX7-LABEL: lds_atomic_cmpxchg_noret_i64_offset:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xb
+; GFX7-NEXT:    v_mov_b32_e32 v0, 7
+; GFX7-NEXT:    v_mov_b32_e32 v1, 0
+; GFX7-NEXT:    s_mov_b32 m0, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v4, s2
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:    ds_cmpst_b64 v4, v[0:1], v[2:3] offset:32
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: lds_atomic_cmpxchg_noret_i64_offset:
+; GFX8:       ; %bb.0:
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX8-NEXT:    v_mov_b32_e32 v0, 7
+; GFX8-NEXT:    v_mov_b32_e32 v1, 0
+; GFX8-NEXT:    s_mov_b32 m0, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v4, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:    ds_cmpst_b64 v4, v[0:1], v[2:3] offset:32
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: lds_atomic_cmpxchg_noret_i64_offset:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX9-NEXT:    v_mov_b32_e32 v0, 7
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, s2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    ds_cmpst_b64 v4, v[0:1], v[2:3] offset:32
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: lds_atomic_cmpxchg_noret_i64_offset:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s2, s[4:5], 0x24
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX11-NEXT:    v_mov_b32_e32 v0, 7
+; GFX11-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v4, s2
+; GFX11-NEXT:    v_mov_b32_e32 v3, s1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s0
+; GFX11-NEXT:    ds_cmpstore_b64 v4, v[2:3], v[0:1] offset:32
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_gl0_inv
+; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr addrspace(3) %ptr, i32 4
   %pair = cmpxchg ptr addrspace(3) %gep, i64 7, i64 %swap seq_cst monotonic
   %result = extractvalue { i64, i1 } %pair, 0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 4cc39d93854a0..0fccdba729132 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -328,7 +328,7 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-NEXT:    buffer_store_b32 v0, off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
 entry:
-  %old = atomicrmw add ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel
+  %old = atomicrmw add ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i32 %old, ptr addrspace(1) %out
   ret void
 }
@@ -655,7 +655,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232-NEXT:    buffer_store_b32 v0, off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
 entry:
-  %old = atomicrmw add ptr addrspace(1) %inout, i32 %additive syncscope("agent") acq_rel
+  %old = atomicrmw add ptr addrspace(1) %inout, i32 %additive syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i32 %old, ptr addrspace(1) %out
   ret void
 }
@@ -1565,7 +1565,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
-  %old = atomicrmw add ptr addrspace(1) %inout, i32 %lane  syncscope("agent") acq_rel
+  %old = atomicrmw add ptr addrspace(1) %inout, i32 %lane  syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i32 %old, ptr addrspace(1) %out
   ret void
 }
@@ -1899,7 +1899,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
 entry:
-  %old = atomicrmw add ptr addrspace(1) %inout, i64 5  syncscope("agent") acq_rel
+  %old = atomicrmw add ptr addrspace(1) %inout, i64 5  syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i64 %old, ptr addrspace(1) %out
   ret void
 }
@@ -2284,7 +2284,7 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
 entry:
-  %old = atomicrmw add ptr addrspace(1) %inout, i64 %additive  syncscope("agent") acq_rel
+  %old = atomicrmw add ptr addrspace(1) %inout, i64 %additive  syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i64 %old, ptr addrspace(1) %out
   ret void
 }
@@ -3545,7 +3545,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %zext = zext i32 %lane to i64
-  %old = atomicrmw add ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel
+  %old = atomicrmw add ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i64 %old, ptr addrspace(1) %out
   ret void
 }
@@ -3861,7 +3861,7 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-NEXT:    buffer_store_b32 v0, off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
 entry:
-  %old = atomicrmw sub ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel
+  %old = atomicrmw sub ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i32 %old, ptr addrspace(1) %out
   ret void
 }
@@ -4190,7 +4190,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232-NEXT:    buffer_store_b32 v0, off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
 entry:
-  %old = atomicrmw sub ptr addrspace(1) %inout, i32 %subitive syncscope("agent") acq_rel
+  %old = atomicrmw sub ptr addrspace(1) %inout, i32 %subitive syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i32 %old, ptr addrspace(1) %out
   ret void
 }
@@ -5100,7 +5100,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232_DPP-NEXT:    s_endpgm
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
-  %old = atomicrmw sub ptr addrspace(1) %inout, i32 %lane syncscope("agent") acq_rel
+  %old = atomicrmw sub ptr addrspace(1) %inout, i32 %lane syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i32 %old, ptr addrspace(1) %out
   ret void
 }
@@ -5454,7 +5454,7 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
 ; GFX1232-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
 entry:
-  %old = atomicrmw sub ptr addrspace(1) %inout, i64 5 syncscope("agent") acq_rel
+  %old = atomicrmw sub ptr addrspace(1) %inout, i64 5 syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i64 %old, ptr addrspace(1) %out
   ret void
 }
@@ -5850,7 +5850,7 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
 ; GFX1232-NEXT:    buffer_store_b64 v[0:1], off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
 entry:
-  %old = atomicrmw sub ptr addrspace(1) %inout, i64 %subitive syncscope("agent") acq_rel
+  %old = atomicrmw sub ptr addrspace(1) %inout, i64 %subitive syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i64 %old, ptr addrspace(1) %out
   ret void
 }
@@ -7111,7 +7111,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
 entry:
   %lane = call i32 @llvm.amdgcn.workitem.id.x()
   %zext = zext i32 %lane to i64
-  %old = atomicrmw sub ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel
+  %old = atomicrmw sub ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
   store i64 %old, ptr addrspace(1) %out
   ret void
 }
@@ -7616,7 +7616,7 @@ define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-FAKE16-NEXT:    buffer_store_b8 v0, off, s[0:3], null
 ; GFX1232-FAKE16-NEXT:    s_endpgm
-  %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
+  %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1, !amdgpu.no.fine.grained.memory !0
   store i8 %rmw, ptr addrspace(1) %result
   ret void
 }
@@ -8432,7 +8432,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-FAKE16-NEXT:    buffer_store_b8 v0, off, s[0:3], null
 ; GFX1232-FAKE16-NEXT:    s_endpgm
-  %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
+  %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1, !amdgpu.no.fine.grained.memory !0
   store i8 %rmw, ptr addrspace(1) %result
   ret void
 }
@@ -8804,7 +8804,7 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
 ; GFX1232-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-NEXT:    buffer_store_b8 v0, off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
-  %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
+  %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1, !amdgpu.no.fine.grained.memory !0
   store i8 %rmw, ptr addrspace(1) %result
   ret void
 }
@@ -9022,13 +9022,12 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
 ; GFX1164-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s7, v0
 ; GFX1164-TRUE16-NEXT:  .LBB15_2:
 ; GFX1164-TRUE16-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1164-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1164-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1164-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1164-TRUE16-NEXT:    v_cndmask_b16 v0.l, s6, 0, vcc
-; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-TRUE16-NEXT:    v_or_b16 v0.l, s2, v0.l
 ; GFX1164-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1164-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], 0
@@ -9101,13 +9100,12 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
 ; GFX1132-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s6, v0
 ; GFX1132-TRUE16-NEXT:  .LBB15_2:
 ; GFX1132-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1132-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1132-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1132-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1132-TRUE16-NEXT:    v_cndmask_b16 v0.l, s4, 0, vcc_lo
-; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-TRUE16-NEXT:    v_or_b16 v0.l, s2, v0.l
 ; GFX1132-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1132-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], 0
@@ -9180,13 +9178,12 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
 ; GFX1264-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s7, v0
 ; GFX1264-TRUE16-NEXT:  .LBB15_2:
 ; GFX1264-TRUE16-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1264-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1264-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX1264-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1264-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1264-TRUE16-NEXT:    v_cndmask_b16 v0.l, s6, 0, vcc
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1264-TRUE16-NEXT:    v_or_b16 v0.l, s2, v0.l
 ; GFX1264-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1264-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
@@ -9259,13 +9256,12 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
 ; GFX1232-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s6, v0
 ; GFX1232-TRUE16-NEXT:  .LBB15_2:
 ; GFX1232-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s5
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1232-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1232-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX1232-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1232-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1232-TRUE16-NEXT:    v_cndmask_b16 v0.l, s4, 0, vcc_lo
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1232-TRUE16-NEXT:    v_or_b16 v0.l, s2, v0.l
 ; GFX1232-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
@@ -9309,7 +9305,7 @@ define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspac
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-FAKE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
 ; GFX1232-FAKE16-NEXT:    s_endpgm
-  %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
+  %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
   store i16 %rmw, ptr addrspace(1) %result
   ret void
 }
@@ -9662,12 +9658,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX1164-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s11, v2
 ; GFX1164-TRUE16-NEXT:  .LBB16_4: ; %Flow
 ; GFX1164-TRUE16-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1164-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1164-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1164-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1164-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1164-TRUE16-NEXT:    v_mad_u16 v0.l, s10, v4.l, s2
 ; GFX1164-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1164-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], 0
@@ -9789,12 +9784,11 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX1132-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s2, v2
 ; GFX1132-TRUE16-NEXT:  .LBB16_4: ; %Flow
 ; GFX1132-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s9
-; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1132-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1132-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX1132-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
-; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-TRUE16-NEXT:    v_mad_u16 v0.l, s8, v4.l, s2
 ; GFX1132-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1132-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], 0
@@ -9916,13 +9910,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX1264-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s11, v2
 ; GFX1264-TRUE16-NEXT:  .LBB16_4: ; %Flow
 ; GFX1264-TRUE16-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1264-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1264-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX1264-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1264-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1264-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1264-TRUE16-NEXT:    v_mad_u16 v0.l, s10, v4.l, s2
 ; GFX1264-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1264-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
@@ -10048,13 +10041,12 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX1232-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s2, v2
 ; GFX1232-TRUE16-NEXT:  .LBB16_4: ; %Flow
 ; GFX1232-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s9
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1232-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1232-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX1232-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX1232-TRUE16-NEXT:    v_readfirstlane_b32 s2, v0
 ; GFX1232-TRUE16-NEXT:    s_wait_alu 0xf1ff
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1232-TRUE16-NEXT:    v_mad_u16 v0.l, s8, v4.l, s2
 ; GFX1232-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-TRUE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
@@ -10125,7 +10117,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-FAKE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
 ; GFX1232-FAKE16-NEXT:    s_endpgm
-  %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
+  %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
   store i16 %rmw, ptr addrspace(1) %result
   ret void
 }
@@ -10497,7 +10489,7 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
 ; GFX1232-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-NEXT:    buffer_store_b16 v0, off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
-  %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
+  %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
   store i16 %rmw, ptr addrspace(1) %result
   ret void
 }
@@ -10734,15 +10726,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
 ; GFX1164-TRUE16-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1164-TRUE16-NEXT:  .LBB18_1: ; %atomicrmw.start
 ; GFX1164-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1164-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s9, v1
+; GFX1164-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1164-TRUE16-NEXT:    v_add_f16_e32 v0.l, s8, v0.l
 ; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX1164-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, s9, v0
-; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1164-TRUE16-NEXT:    v_and_or_b32 v0, v1, s10, v0
 ; GFX1164-TRUE16-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1164-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1164-TRUE16-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX1164-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
 ; GFX1164-TRUE16-NEXT:    s_waitcnt vmcnt(0)
@@ -10828,14 +10820,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
 ; GFX1132-TRUE16-NEXT:    s_mov_b32 s6, -1
 ; GFX1132-TRUE16-NEXT:  .LBB18_1: ; %atomicrmw.start
 ; GFX1132-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1132-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s2, v1
+; GFX1132-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1132-TRUE16-NEXT:    v_add_f16_e32 v0.l, s8, v0.l
 ; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX1132-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, s2, v0
-; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1132-TRUE16-NEXT:    v_and_or_b32 v0, v1, s3, v0
+; GFX1132-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1132-TRUE16-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
 ; GFX1132-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
 ; GFX1132-TRUE16-NEXT:    s_waitcnt vmcnt(0)
@@ -10920,15 +10912,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
 ; GFX1264-TRUE16-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX1264-TRUE16-NEXT:  .LBB18_1: ; %atomicrmw.start
 ; GFX1264-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1264-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s9, v1
+; GFX1264-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1264-TRUE16-NEXT:    v_add_f16_e32 v0.l, s8, v0.l
 ; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX1264-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, s9, v0
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1264-TRUE16-NEXT:    v_and_or_b32 v0, v1, s10, v0
 ; GFX1264-TRUE16-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX1264-TRUE16-NEXT:    v_mov_b32_e32 v2, v0
 ; GFX1264-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
 ; GFX1264-TRUE16-NEXT:    s_wait_loadcnt 0x0
@@ -11014,14 +11006,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
 ; GFX1232-TRUE16-NEXT:    s_mov_b32 s6, -1
 ; GFX1232-TRUE16-NEXT:  .LBB18_1: ; %atomicrmw.start
 ; GFX1232-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX1232-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, s2, v1
+; GFX1232-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1232-TRUE16-NEXT:    v_add_f16_e32 v0.l, s8, v0.l
 ; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX1232-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, s2, v0
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1232-TRUE16-NEXT:    v_and_or_b32 v0, v1, s3, v0
+; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1232-TRUE16-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
 ; GFX1232-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
 ; GFX1232-TRUE16-NEXT:    s_wait_loadcnt 0x0
@@ -11084,7 +11076,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-FAKE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
 ; GFX1232-FAKE16-NEXT:    s_endpgm
-  %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, half %val monotonic, align 2
+  %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, half %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
   store half %rmw, ptr addrspace(1) %result
   ret void
 }
@@ -11774,7 +11766,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
 ; GFX1232-FAKE16-NEXT:    s_mov_b32 s2, -1
 ; GFX1232-FAKE16-NEXT:    buffer_store_b16 v0, off, s[0:3], null
 ; GFX1232-FAKE16-NEXT:    s_endpgm
-  %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 2
+  %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
   store bfloat %rmw, ptr addrspace(1) %result
   ret void
 }
@@ -12045,72 +12037,32 @@ define amdgpu_kernel void @uniform_fadd_v2f16(ptr addrspace(1) %result, ptr addr
 ; GFX1264-LABEL: uniform_fadd_v2f16:
 ; GFX1264:       ; %bb.0:
 ; GFX1264-NEXT:    s_clause 0x1
+; GFX1264-NEXT:    s_load_b32 s6, s[4:5], 0x34
 ; GFX1264-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT:    s_load_b32 s10, s[4:5], 0x34
-; GFX1264-NEXT:    s_mov_b64 s[8:9], 0
-; GFX1264-NEXT:    s_mov_b32 s7, 0x31016000
-; GFX1264-NEXT:    s_mov_b32 s6, -1
+; GFX1264-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1264-NEXT:    s_wait_kmcnt 0x0
-; GFX1264-NEXT:    s_load_b32 s4, s[2:3], 0x0
-; GFX1264-NEXT:    s_mov_b32 s5, s3
-; GFX1264-NEXT:    s_wait_kmcnt 0x0
-; GFX1264-NEXT:    v_mov_b32_e32 v1, s4
-; GFX1264-NEXT:    s_mov_b32 s4, s2
-; GFX1264-NEXT:  .LBB20_1: ; %atomicrmw.start
-; GFX1264-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1264-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-NEXT:    v_pk_add_f16 v0, v1, s10
-; GFX1264-NEXT:    v_mov_b32_e32 v3, v1
-; GFX1264-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1264-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1264-NEXT:    s_wait_loadcnt 0x0
-; GFX1264-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1264-NEXT:    v_mov_b32_e32 v1, v2
-; GFX1264-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; GFX1264-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-NEXT:    s_and_not1_b64 exec, exec, s[8:9]
-; GFX1264-NEXT:    s_cbranch_execnz .LBB20_1
-; GFX1264-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1264-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX1264-NEXT:    v_mov_b32_e32 v1, s6
+; GFX1264-NEXT:    global_atomic_pk_add_f16 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
 ; GFX1264-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1264-NEXT:    s_mov_b32 s2, -1
-; GFX1264-NEXT:    buffer_store_b32 v2, off, s[0:3], null
+; GFX1264-NEXT:    s_wait_loadcnt 0x0
+; GFX1264-NEXT:    buffer_store_b32 v0, off, s[0:3], null
 ; GFX1264-NEXT:    s_endpgm
 ;
 ; GFX1232-LABEL: uniform_fadd_v2f16:
 ; GFX1232:       ; %bb.0:
 ; GFX1232-NEXT:    s_clause 0x1
+; GFX1232-NEXT:    s_load_b32 s6, s[4:5], 0x34
 ; GFX1232-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT:    s_load_b32 s8, s[4:5], 0x34
-; GFX1232-NEXT:    s_mov_b32 s9, 0
-; GFX1232-NEXT:    s_mov_b32 s7, 0x31016000
-; GFX1232-NEXT:    s_mov_b32 s6, -1
 ; GFX1232-NEXT:    s_wait_kmcnt 0x0
-; GFX1232-NEXT:    s_load_b32 s4, s[2:3], 0x0
-; GFX1232-NEXT:    s_mov_b32 s5, s3
-; GFX1232-NEXT:    s_wait_kmcnt 0x0
-; GFX1232-NEXT:    v_mov_b32_e32 v1, s4
-; GFX1232-NEXT:    s_mov_b32 s4, s2
-; GFX1232-NEXT:  .LBB20_1: ; %atomicrmw.start
-; GFX1232-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1232-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-NEXT:    v_pk_add_f16 v0, v1, s8
-; GFX1232-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1232-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1232-NEXT:    s_wait_loadcnt 0x0
-; GFX1232-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX1232-NEXT:    v_mov_b32_e32 v1, v2
-; GFX1232-NEXT:    s_or_b32 s9, vcc_lo, s9
-; GFX1232-NEXT:    s_wait_alu 0xfffe
-; GFX1232-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s9
-; GFX1232-NEXT:    s_cbranch_execnz .LBB20_1
-; GFX1232-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1232-NEXT:    s_or_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX1232-NEXT:    global_atomic_pk_add_f16 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
 ; GFX1232-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX1232-NEXT:    s_mov_b32 s2, -1
-; GFX1232-NEXT:    buffer_store_b32 v2, off, s[0:3], null
+; GFX1232-NEXT:    s_wait_loadcnt 0x0
+; GFX1232-NEXT:    buffer_store_b32 v0, off, s[0:3], null
 ; GFX1232-NEXT:    s_endpgm
-  %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x half> %val monotonic, align 4
+  %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x half> %val monotonic, align 4, !amdgpu.no.fine.grained.memory !0
   store <2 x half> %rmw, ptr addrspace(1) %result
   ret void
 }
@@ -12605,237 +12557,41 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
 ; GFX1132-FAKE16-NEXT:    buffer_store_b32 v2, off, s[8:11], 0
 ; GFX1132-FAKE16-NEXT:    s_endpgm
 ;
-; GFX1264-TRUE16-LABEL: uniform_fadd_v2bf16:
-; GFX1264-TRUE16:       ; %bb.0:
-; GFX1264-TRUE16-NEXT:    s_clause 0x1
-; GFX1264-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
-; GFX1264-TRUE16-NEXT:    s_mov_b64 s[8:9], 0
-; GFX1264-TRUE16-NEXT:    s_mov_b32 s7, 0x31016000
-; GFX1264-TRUE16-NEXT:    s_mov_b32 s6, -1
-; GFX1264-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT:    s_load_b32 s5, s[2:3], 0x0
-; GFX1264-TRUE16-NEXT:    s_and_b32 s10, s4, 0xffff0000
-; GFX1264-TRUE16-NEXT:    s_lshl_b32 s11, s4, 16
-; GFX1264-TRUE16-NEXT:    s_mov_b32 s4, s2
-; GFX1264-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT:    v_mov_b32_e32 v1, s5
-; GFX1264-TRUE16-NEXT:    s_mov_b32 s5, s3
-; GFX1264-TRUE16-NEXT:  .LBB21_1: ; %atomicrmw.start
-; GFX1264-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v1
-; GFX1264-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
-; GFX1264-TRUE16-NEXT:    v_add_f32_e32 v0, s11, v0
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT:    v_add_f32_e32 v2, s10, v2
-; GFX1264-TRUE16-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT:    v_bfe_u32 v4, v2, 16, 1
-; GFX1264-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX1264-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX1264-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v2
-; GFX1264-TRUE16-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX1264-TRUE16-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
-; GFX1264-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc
-; GFX1264-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX1264-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
-; GFX1264-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX1264-TRUE16-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v0, v2
-; GFX1264-TRUE16-NEXT:    v_mov_b32_e32 v3, v1
-; GFX1264-TRUE16-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1264-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1264-TRUE16-NEXT:    s_wait_loadcnt 0x0
-; GFX1264-TRUE16-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1264-TRUE16-NEXT:    v_mov_b32_e32 v1, v2
-; GFX1264-TRUE16-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; GFX1264-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-TRUE16-NEXT:    s_and_not1_b64 exec, exec, s[8:9]
-; GFX1264-TRUE16-NEXT:    s_cbranch_execnz .LBB21_1
-; GFX1264-TRUE16-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1264-TRUE16-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GFX1264-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1264-TRUE16-NEXT:    s_mov_b32 s2, -1
-; GFX1264-TRUE16-NEXT:    buffer_store_b32 v2, off, s[0:3], null
-; GFX1264-TRUE16-NEXT:    s_endpgm
-;
-; GFX1264-FAKE16-LABEL: uniform_fadd_v2bf16:
-; GFX1264-FAKE16:       ; %bb.0:
-; GFX1264-FAKE16-NEXT:    s_clause 0x1
-; GFX1264-FAKE16-NEXT:    s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT:    s_load_b32 s0, s[4:5], 0x34
-; GFX1264-FAKE16-NEXT:    s_mov_b64 s[2:3], 0
-; GFX1264-FAKE16-NEXT:    s_mov_b32 s7, 0x31016000
-; GFX1264-FAKE16-NEXT:    s_mov_b32 s6, -1
-; GFX1264-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT:    s_load_b32 s1, s[10:11], 0x0
-; GFX1264-FAKE16-NEXT:    s_lshl_b32 s12, s0, 16
-; GFX1264-FAKE16-NEXT:    s_and_b32 s13, s0, 0xffff0000
-; GFX1264-FAKE16-NEXT:    s_mov_b32 s4, s10
-; GFX1264-FAKE16-NEXT:    s_mov_b32 s5, s11
-; GFX1264-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT:    v_mov_b32_e32 v1, s1
-; GFX1264-FAKE16-NEXT:  .LBB21_1: ; %atomicrmw.start
-; GFX1264-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1264-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v1
-; GFX1264-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
-; GFX1264-FAKE16-NEXT:    v_add_f32_e32 v0, s12, v0
-; GFX1264-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT:    v_add_f32_e32 v2, s13, v2
-; GFX1264-FAKE16-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX1264-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT:    v_bfe_u32 v4, v2, 16, 1
-; GFX1264-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX1264-FAKE16-NEXT:    v_or_b32_e32 v6, 0x400000, v2
-; GFX1264-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc, v2, v2
-; GFX1264-FAKE16-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX1264-FAKE16-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
-; GFX1264-FAKE16-NEXT:    v_cmp_u_f32_e64 s[0:1], v0, v0
-; GFX1264-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX1264-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc
-; GFX1264-FAKE16-NEXT:    s_wait_alu 0xf1ff
-; GFX1264-FAKE16-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s[0:1]
-; GFX1264-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1264-FAKE16-NEXT:    v_perm_b32 v0, v2, v0, 0x7060302
-; GFX1264-FAKE16-NEXT:    v_mov_b32_e32 v3, v1
-; GFX1264-FAKE16-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1264-FAKE16-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1264-FAKE16-NEXT:    s_wait_loadcnt 0x0
-; GFX1264-FAKE16-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX1264-FAKE16-NEXT:    v_mov_b32_e32 v1, v2
-; GFX1264-FAKE16-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
-; GFX1264-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1264-FAKE16-NEXT:    s_and_not1_b64 exec, exec, s[2:3]
-; GFX1264-FAKE16-NEXT:    s_cbranch_execnz .LBB21_1
-; GFX1264-FAKE16-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1264-FAKE16-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX1264-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX1264-FAKE16-NEXT:    s_mov_b32 s10, -1
-; GFX1264-FAKE16-NEXT:    buffer_store_b32 v2, off, s[8:11], null
-; GFX1264-FAKE16-NEXT:    s_endpgm
-;
-; GFX1232-TRUE16-LABEL: uniform_fadd_v2bf16:
-; GFX1232-TRUE16:       ; %bb.0:
-; GFX1232-TRUE16-NEXT:    s_clause 0x1
-; GFX1232-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-TRUE16-NEXT:    s_load_b32 s4, s[4:5], 0x34
-; GFX1232-TRUE16-NEXT:    s_mov_b32 s8, 0
-; GFX1232-TRUE16-NEXT:    s_mov_b32 s7, 0x31016000
-; GFX1232-TRUE16-NEXT:    s_mov_b32 s6, -1
-; GFX1232-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT:    s_load_b32 s5, s[2:3], 0x0
-; GFX1232-TRUE16-NEXT:    s_and_b32 s9, s4, 0xffff0000
-; GFX1232-TRUE16-NEXT:    s_lshl_b32 s10, s4, 16
-; GFX1232-TRUE16-NEXT:    s_mov_b32 s4, s2
-; GFX1232-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT:    v_mov_b32_e32 v1, s5
-; GFX1232-TRUE16-NEXT:    s_mov_b32 s5, s3
-; GFX1232-TRUE16-NEXT:  .LBB21_1: ; %atomicrmw.start
-; GFX1232-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1232-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v1
-; GFX1232-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
-; GFX1232-TRUE16-NEXT:    v_add_f32_e32 v0, s10, v0
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1232-TRUE16-NEXT:    v_add_f32_e32 v2, s9, v2
-; GFX1232-TRUE16-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1232-TRUE16-NEXT:    v_bfe_u32 v4, v2, 16, 1
-; GFX1232-TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX1232-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX1232-TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v2
-; GFX1232-TRUE16-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX1232-TRUE16-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
-; GFX1232-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1232-TRUE16-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1232-TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX1232-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
-; GFX1232-TRUE16-NEXT:    s_wait_alu 0xfffd
-; GFX1232-TRUE16-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX1232-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v0, v2
-; GFX1232-TRUE16-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1232-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1232-TRUE16-NEXT:    s_wait_loadcnt 0x0
-; GFX1232-TRUE16-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX1232-TRUE16-NEXT:    v_mov_b32_e32 v1, v2
-; GFX1232-TRUE16-NEXT:    s_or_b32 s8, vcc_lo, s8
-; GFX1232-TRUE16-NEXT:    s_wait_alu 0xfffe
-; GFX1232-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s8
-; GFX1232-TRUE16-NEXT:    s_cbranch_execnz .LBB21_1
-; GFX1232-TRUE16-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1232-TRUE16-NEXT:    s_or_b32 exec_lo, exec_lo, s8
-; GFX1232-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX1232-TRUE16-NEXT:    s_mov_b32 s2, -1
-; GFX1232-TRUE16-NEXT:    buffer_store_b32 v2, off, s[0:3], null
-; GFX1232-TRUE16-NEXT:    s_endpgm
+; GFX1264-LABEL: uniform_fadd_v2bf16:
+; GFX1264:       ; %bb.0:
+; GFX1264-NEXT:    s_clause 0x1
+; GFX1264-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX1264-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1264-NEXT:    s_wait_kmcnt 0x0
+; GFX1264-NEXT:    v_mov_b32_e32 v1, s6
+; GFX1264-NEXT:    global_atomic_pk_add_bf16 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1264-NEXT:    s_mov_b32 s2, -1
+; GFX1264-NEXT:    s_wait_loadcnt 0x0
+; GFX1264-NEXT:    buffer_store_b32 v0, off, s[0:3], null
+; GFX1264-NEXT:    s_endpgm
 ;
-; GFX1232-FAKE16-LABEL: uniform_fadd_v2bf16:
-; GFX1232-FAKE16:       ; %bb.0:
-; GFX1232-FAKE16-NEXT:    s_clause 0x1
-; GFX1232-FAKE16-NEXT:    s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1232-FAKE16-NEXT:    s_load_b32 s0, s[4:5], 0x34
-; GFX1232-FAKE16-NEXT:    s_mov_b32 s1, 0
-; GFX1232-FAKE16-NEXT:    s_mov_b32 s7, 0x31016000
-; GFX1232-FAKE16-NEXT:    s_mov_b32 s6, -1
-; GFX1232-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT:    s_load_b32 s4, s[10:11], 0x0
-; GFX1232-FAKE16-NEXT:    s_lshl_b32 s2, s0, 16
-; GFX1232-FAKE16-NEXT:    s_and_b32 s3, s0, 0xffff0000
-; GFX1232-FAKE16-NEXT:    s_mov_b32 s5, s11
-; GFX1232-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT:    v_mov_b32_e32 v1, s4
-; GFX1232-FAKE16-NEXT:    s_mov_b32 s4, s10
-; GFX1232-FAKE16-NEXT:  .LBB21_1: ; %atomicrmw.start
-; GFX1232-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1232-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1232-FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v1
-; GFX1232-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v1
-; GFX1232-FAKE16-NEXT:    v_add_f32_e32 v0, s2, v0
-; GFX1232-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1232-FAKE16-NEXT:    v_add_f32_e32 v2, s3, v2
-; GFX1232-FAKE16-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX1232-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX1232-FAKE16-NEXT:    v_bfe_u32 v4, v2, 16, 1
-; GFX1232-FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX1232-FAKE16-NEXT:    v_or_b32_e32 v6, 0x400000, v2
-; GFX1232-FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX1232-FAKE16-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX1232-FAKE16-NEXT:    v_add3_u32 v4, v4, v2, 0x7fff
-; GFX1232-FAKE16-NEXT:    v_cmp_u_f32_e64 s0, v0, v0
-; GFX1232-FAKE16-NEXT:    s_wait_alu 0xfffd
-; GFX1232-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1232-FAKE16-NEXT:    v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX1232-FAKE16-NEXT:    s_wait_alu 0xf1ff
-; GFX1232-FAKE16-NEXT:    v_cndmask_b32_e64 v0, v3, v5, s0
-; GFX1232-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT:    v_perm_b32 v0, v2, v0, 0x7060302
-; GFX1232-FAKE16-NEXT:    v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX1232-FAKE16-NEXT:    buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1232-FAKE16-NEXT:    s_wait_loadcnt 0x0
-; GFX1232-FAKE16-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX1232-FAKE16-NEXT:    v_mov_b32_e32 v1, v2
-; GFX1232-FAKE16-NEXT:    s_or_b32 s1, vcc_lo, s1
-; GFX1232-FAKE16-NEXT:    s_wait_alu 0xfffe
-; GFX1232-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s1
-; GFX1232-FAKE16-NEXT:    s_cbranch_execnz .LBB21_1
-; GFX1232-FAKE16-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1232-FAKE16-NEXT:    s_or_b32 exec_lo, exec_lo, s1
-; GFX1232-FAKE16-NEXT:    s_mov_b32 s11, 0x31016000
-; GFX1232-FAKE16-NEXT:    s_mov_b32 s10, -1
-; GFX1232-FAKE16-NEXT:    buffer_store_b32 v2, off, s[8:11], null
-; GFX1232-FAKE16-NEXT:    s_endpgm
-  %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x bfloat> %val monotonic, align 4
+; GFX1232-LABEL: uniform_fadd_v2bf16:
+; GFX1232:       ; %bb.0:
+; GFX1232-NEXT:    s_clause 0x1
+; GFX1232-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX1232-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT:    s_wait_kmcnt 0x0
+; GFX1232-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX1232-NEXT:    global_atomic_pk_add_bf16 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX1232-NEXT:    s_mov_b32 s2, -1
+; GFX1232-NEXT:    s_wait_loadcnt 0x0
+; GFX1232-NEXT:    buffer_store_b32 v0, off, s[0:3], null
+; GFX1232-NEXT:    s_endpgm
+  %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x bfloat> %val monotonic, align 4, !amdgpu.no.fine.grained.memory !0
   store <2 x bfloat> %rmw, ptr addrspace(1) %result
   ret void
 }
+
+!0 = !{}
+
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; GFX1132_DPP-FAKE16: {{.*}}
 ; GFX1132_DPP-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll b/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll
new file mode 100644
index 0000000000000..5fc9f4a0f8038
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll
@@ -0,0 +1,1486 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN:llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck --check-prefix=GFX1250 %s
+
+define float @global_system_atomic_fadd_f32(ptr addrspace(1) %ptr, float %val) {
+; GFX1250-LABEL: global_system_atomic_fadd_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_add_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val monotonic
+  ret float %result
+}
+
+define float @global_one_as_atomic_fadd_f32(ptr addrspace(1) %ptr, float %val) {
+; GFX1250-LABEL: global_one_as_atomic_fadd_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_add_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic
+  ret float %result
+}
+
+define double @global_system_atomic_fadd_f64(ptr addrspace(1) %ptr, double %val) {
+; GFX1250-LABEL: global_system_atomic_fadd_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val monotonic
+  ret double %result
+}
+
+define double @global_one_as_atomic_fadd_f64(ptr addrspace(1) %ptr, double %val) {
+; GFX1250-LABEL: global_one_as_atomic_fadd_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_add_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fadd ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic
+  ret double %result
+}
+
+define float @global_system_atomic_fmin_f32(ptr addrspace(1) %ptr, float %val) {
+; GFX1250-LABEL: global_system_atomic_fmin_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val monotonic
+  ret float %result
+}
+
+define float @global_one_as_atomic_fmin_f32(ptr addrspace(1) %ptr, float %val) {
+; GFX1250-LABEL: global_one_as_atomic_fmin_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic
+  ret float %result
+}
+
+define double @global_system_atomic_fmin_f64(ptr addrspace(1) %ptr, double %val) {
+; GFX1250-LABEL: global_system_atomic_fmin_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_num_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val monotonic
+  ret double %result
+}
+
+define double @global_one_as_atomic_fmin_f64(ptr addrspace(1) %ptr, double %val) {
+; GFX1250-LABEL: global_one_as_atomic_fmin_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_num_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmin ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic
+  ret double %result
+}
+
+define float @global_system_atomic_fmax_f32(ptr addrspace(1) %ptr, float %val) {
+; GFX1250-LABEL: global_system_atomic_fmax_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val monotonic
+  ret float %result
+}
+
+define float @global_one_as_atomic_fmax_f32(ptr addrspace(1) %ptr, float %val) {
+; GFX1250-LABEL: global_one_as_atomic_fmax_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_num_f32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, float %val syncscope("one-as") monotonic
+  ret float %result
+}
+
+define double @global_system_atomic_fmax_f64(ptr addrspace(1) %ptr, double %val) {
+; GFX1250-LABEL: global_system_atomic_fmax_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_num_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val monotonic
+  ret double %result
+}
+
+define double @global_one_as_atomic_fmax_f64(ptr addrspace(1) %ptr, double %val) {
+; GFX1250-LABEL: global_one_as_atomic_fmax_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_num_f64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmax ptr addrspace(1) %ptr, double %val syncscope("one-as") monotonic
+  ret double %result
+}
+
+define i32 @global_one_as_atomic_min_i32(ptr addrspace(1) %ptr, i32 %val) {
+; GFX1250-LABEL: global_one_as_atomic_min_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_i32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr addrspace(1) %ptr, i32 %val syncscope("one-as") monotonic
+  ret i32 %result
+}
+
+define i32 @global_system_atomic_min_i32(ptr addrspace(1) %ptr, i32 %val) {
+; GFX1250-LABEL: global_system_atomic_min_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_i32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr addrspace(1) %ptr, i32 %val monotonic
+  ret i32 %result
+}
+
+define i32 @global_one_as_atomic_max_i32(ptr addrspace(1) %ptr, i32 %val) {
+; GFX1250-LABEL: global_one_as_atomic_max_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_i32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr addrspace(1) %ptr, i32 %val syncscope("one-as") monotonic
+  ret i32 %result
+}
+
+define i32 @global_system_atomic_max_i32(ptr addrspace(1) %ptr, i32 %val) {
+; GFX1250-LABEL: global_system_atomic_max_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_i32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr addrspace(1) %ptr, i32 %val monotonic
+  ret i32 %result
+}
+
+define i32 @global_one_as_atomic_umin_i32(ptr addrspace(1) %ptr, i32 %val) {
+; GFX1250-LABEL: global_one_as_atomic_umin_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr addrspace(1) %ptr, i32 %val syncscope("one-as") monotonic
+  ret i32 %result
+}
+
+define i32 @global_system_atomic_umin_i32(ptr addrspace(1) %ptr, i32 %val) {
+; GFX1250-LABEL: global_system_atomic_umin_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr addrspace(1) %ptr, i32 %val monotonic
+  ret i32 %result
+}
+
+define i32 @global_one_as_atomic_umax_i32(ptr addrspace(1) %ptr, i32 %val) {
+; GFX1250-LABEL: global_one_as_atomic_umax_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr addrspace(1) %ptr, i32 %val syncscope("one-as") monotonic
+  ret i32 %result
+}
+
+define i32 @global_system_atomic_umax_i32(ptr addrspace(1) %ptr, i32 %val) {
+; GFX1250-LABEL: global_system_atomic_umax_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_u32 v0, v[0:1], v2, off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr addrspace(1) %ptr, i32 %val monotonic
+  ret i32 %result
+}
+
+define i64 @global_one_as_atomic_min_i64(ptr addrspace(1) %ptr, i64 %val) {
+; GFX1250-LABEL: global_one_as_atomic_min_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_i64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr addrspace(1) %ptr, i64 %val syncscope("one-as") monotonic
+  ret i64 %result
+}
+
+define i64 @global_system_atomic_min_i64(ptr addrspace(1) %ptr, i64 %val) {
+; GFX1250-LABEL: global_system_atomic_min_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_i64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr addrspace(1) %ptr, i64 %val monotonic
+  ret i64 %result
+}
+
+define i64 @global_one_as_atomic_max_i64(ptr addrspace(1) %ptr, i64 %val) {
+; GFX1250-LABEL: global_one_as_atomic_max_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_i64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr addrspace(1) %ptr, i64 %val syncscope("one-as") monotonic
+  ret i64 %result
+}
+
+define i64 @global_system_atomic_max_i64(ptr addrspace(1) %ptr, i64 %val) {
+; GFX1250-LABEL: global_system_atomic_max_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_i64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr addrspace(1) %ptr, i64 %val monotonic
+  ret i64 %result
+}
+
+define i64 @global_one_as_atomic_umin_i64(ptr addrspace(1) %ptr, i64 %val) {
+; GFX1250-LABEL: global_one_as_atomic_umin_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_u64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr addrspace(1) %ptr, i64 %val syncscope("one-as") monotonic
+  ret i64 %result
+}
+
+define i64 @global_system_atomic_umin_i64(ptr addrspace(1) %ptr, i64 %val) {
+; GFX1250-LABEL: global_system_atomic_umin_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_min_u64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr addrspace(1) %ptr, i64 %val monotonic
+  ret i64 %result
+}
+
+define i64 @global_one_as_atomic_umax_i64(ptr addrspace(1) %ptr, i64 %val) {
+; GFX1250-LABEL: global_one_as_atomic_umax_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_u64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr addrspace(1) %ptr, i64 %val syncscope("one-as") monotonic
+  ret i64 %result
+}
+
+define i64 @global_system_atomic_umax_i64(ptr addrspace(1) %ptr, i64 %val) {
+; GFX1250-LABEL: global_system_atomic_umax_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_atomic_max_u64 v[0:1], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr addrspace(1) %ptr, i64 %val monotonic
+  ret i64 %result
+}
+
+define i16 @global_one_as_atomic_min_i16(ptr addrspace(1) %ptr, i16 %val) {
+; GFX1250-LABEL: global_one_as_atomic_min_i16:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v0
+; GFX1250-NEXT:    s_mov_b32 s0, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT:    v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_not_b32_e32 v4, v4
+; GFX1250-NEXT:  .LBB28_1: ; %atomicrmw.start
+; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT:    v_min_i16 v5, v5, v2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB28_1
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr addrspace(1) %ptr, i16 %val syncscope("one-as") monotonic
+  ret i16 %result
+}
+
+define i16 @global_one_as_atomic_umin_i16(ptr addrspace(1) %ptr, i16 %val) {
+; GFX1250-LABEL: global_one_as_atomic_umin_i16:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v0
+; GFX1250-NEXT:    s_mov_b32 s0, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT:    v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_not_b32_e32 v4, v4
+; GFX1250-NEXT:  .LBB29_1: ; %atomicrmw.start
+; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT:    v_min_u16 v5, v5, v2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB29_1
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr addrspace(1) %ptr, i16 %val syncscope("one-as") monotonic
+  ret i16 %result
+}
+
+define i16 @global_one_as_atomic_max_i16(ptr addrspace(1) %ptr, i16 %val) {
+; GFX1250-LABEL: global_one_as_atomic_max_i16:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v0
+; GFX1250-NEXT:    s_mov_b32 s0, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT:    v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_not_b32_e32 v4, v4
+; GFX1250-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT:    v_max_i16 v5, v5, v2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB30_1
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr addrspace(1) %ptr, i16 %val syncscope("one-as") monotonic
+  ret i16 %result
+}
+
+define i16 @global_one_as_atomic_umax_i16(ptr addrspace(1) %ptr, i16 %val) {
+; GFX1250-LABEL: global_one_as_atomic_umax_i16:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v0
+; GFX1250-NEXT:    s_mov_b32 s0, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT:    v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_not_b32_e32 v4, v4
+; GFX1250-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT:    v_max_u16 v5, v5, v2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB31_1
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr addrspace(1) %ptr, i16 %val syncscope("one-as") monotonic
+  ret i16 %result
+}
+
+define float @flat_system_atomic_fadd_f32(ptr %ptr, float %val) {
+; GFX1250-LABEL: flat_system_atomic_fadd_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fadd ptr %ptr, float %val monotonic
+  ret float %result
+}
+
+define float @flat_one_as_atomic_fadd_f32(ptr %ptr, float %val) {
+; GFX1250-LABEL: flat_one_as_atomic_fadd_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_add_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fadd ptr %ptr, float %val syncscope("one-as") monotonic
+  ret float %result
+}
+
+define double @flat_system_atomic_fadd_f64(ptr %ptr, double %val) {
+; GFX1250-LABEL: flat_system_atomic_fadd_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b64 s[0:1], src_shared_base
+; GFX1250-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    v_cmpx_ne_u32_e64 s1, v1
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB34_6
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.check.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s1, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX1250-NEXT:    s_cbranch_execz .LBB34_3
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.global
+; GFX1250-NEXT:    global_atomic_add_f64 v[4:5], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB34_3: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s1, s1
+; GFX1250-NEXT:    s_cbranch_execz .LBB34_5
+; GFX1250-NEXT:  ; %bb.4: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s2, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s2, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_add_f64_e32 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB34_5: ; %Flow1
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB34_6: ; %Flow2
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB34_8
+; GFX1250-NEXT:  ; %bb.7: ; %atomicrmw.shared
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc_lo
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    ds_add_rtn_f64 v[4:5], v0, v[2:3]
+; GFX1250-NEXT:  .LBB34_8: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fadd ptr %ptr, double %val monotonic
+  ret double %result
+}
+
+define double @flat_one_as_atomic_fadd_f64(ptr %ptr, double %val) {
+; GFX1250-LABEL: flat_one_as_atomic_fadd_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b64 s[0:1], src_shared_base
+; GFX1250-NEXT:    s_mov_b32 s0, exec_lo
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    v_cmpx_ne_u32_e64 s1, v1
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB35_6
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.check.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s1, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX1250-NEXT:    s_cbranch_execz .LBB35_3
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.global
+; GFX1250-NEXT:    global_atomic_add_f64 v[4:5], v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB35_3: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s1, s1
+; GFX1250-NEXT:    s_cbranch_execz .LBB35_5
+; GFX1250-NEXT:  ; %bb.4: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s2, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s2, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_add_f64_e32 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB35_5: ; %Flow1
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB35_6: ; %Flow2
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB35_8
+; GFX1250-NEXT:  ; %bb.7: ; %atomicrmw.shared
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc_lo
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    ds_add_rtn_f64 v[4:5], v0, v[2:3]
+; GFX1250-NEXT:  .LBB35_8: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fadd ptr %ptr, double %val syncscope("one-as") monotonic
+  ret double %result
+}
+
+define float @flat_system_atomic_fmin_f32(ptr %ptr, float %val) {
+; GFX1250-LABEL: flat_system_atomic_fmin_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_min_num_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmin ptr %ptr, float %val monotonic
+  ret float %result
+}
+
+define float @flat_one_as_atomic_fmin_f32(ptr %ptr, float %val) {
+; GFX1250-LABEL: flat_one_as_atomic_fmin_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_min_num_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmin ptr %ptr, float %val syncscope("one-as") monotonic
+  ret float %result
+}
+
+define double @flat_system_atomic_fmin_f64(ptr %ptr, double %val) {
+; GFX1250-LABEL: flat_system_atomic_fmin_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB38_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_min_num_f64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB38_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB38_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_dual_max_num_f64 v[2:3], v[2:3], v[2:3] :: v_dual_cndmask_b32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX1250-NEXT:    v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB38_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmin ptr %ptr, double %val monotonic
+  ret double %result
+}
+
+define double @flat_one_as_atomic_fmin_f64(ptr %ptr, double %val) {
+; GFX1250-LABEL: flat_one_as_atomic_fmin_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB39_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_min_num_f64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB39_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB39_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_dual_max_num_f64 v[2:3], v[2:3], v[2:3] :: v_dual_cndmask_b32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX1250-NEXT:    v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB39_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmin ptr %ptr, double %val syncscope("one-as") monotonic
+  ret double %result
+}
+
+define float @flat_system_atomic_fmax_f32(ptr %ptr, float %val) {
+; GFX1250-LABEL: flat_system_atomic_fmax_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_max_num_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmax ptr %ptr, float %val monotonic
+  ret float %result
+}
+
+define float @flat_one_as_atomic_fmax_f32(ptr %ptr, float %val) {
+; GFX1250-LABEL: flat_one_as_atomic_fmax_f32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_max_num_f32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmax ptr %ptr, float %val syncscope("one-as") monotonic
+  ret float %result
+}
+
+define double @flat_system_atomic_fmax_f64(ptr %ptr, double %val) {
+; GFX1250-LABEL: flat_system_atomic_fmax_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB42_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_max_num_f64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB42_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB42_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_dual_max_num_f64 v[2:3], v[2:3], v[2:3] :: v_dual_cndmask_b32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX1250-NEXT:    v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB42_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmax ptr %ptr, double %val monotonic
+  ret double %result
+}
+
+define double @flat_one_as_atomic_fmax_f64(ptr %ptr, double %val) {
+; GFX1250-LABEL: flat_one_as_atomic_fmax_f64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB43_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_max_num_f64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB43_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB43_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_dual_max_num_f64 v[2:3], v[2:3], v[2:3] :: v_dual_cndmask_b32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX1250-NEXT:    v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB43_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw fmax ptr %ptr, double %val syncscope("one-as") monotonic
+  ret double %result
+}
+
+define i32 @flat_one_as_atomic_min_i32(ptr %ptr, i32 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_min_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_min_i32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr %ptr, i32 %val syncscope("one-as") monotonic
+  ret i32 %result
+}
+
+define i32 @flat_system_atomic_min_i32(ptr %ptr, i32 %val) {
+; GFX1250-LABEL: flat_system_atomic_min_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_min_i32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr %ptr, i32 %val monotonic
+  ret i32 %result
+}
+
+define i32 @flat_one_as_atomic_max_i32(ptr %ptr, i32 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_max_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_max_i32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr %ptr, i32 %val syncscope("one-as") monotonic
+  ret i32 %result
+}
+
+define i32 @flat_system_atomic_max_i32(ptr %ptr, i32 %val) {
+; GFX1250-LABEL: flat_system_atomic_max_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_max_i32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr %ptr, i32 %val monotonic
+  ret i32 %result
+}
+
+define i32 @flat_one_as_atomic_umin_i32(ptr %ptr, i32 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_umin_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_min_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr %ptr, i32 %val syncscope("one-as") monotonic
+  ret i32 %result
+}
+
+define i32 @flat_system_atomic_umin_i32(ptr %ptr, i32 %val) {
+; GFX1250-LABEL: flat_system_atomic_umin_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_min_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr %ptr, i32 %val monotonic
+  ret i32 %result
+}
+
+define i32 @flat_one_as_atomic_umax_i32(ptr %ptr, i32 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_umax_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_max_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr %ptr, i32 %val syncscope("one-as") monotonic
+  ret i32 %result
+}
+
+define i32 @flat_system_atomic_umax_i32(ptr %ptr, i32 %val) {
+; GFX1250-LABEL: flat_system_atomic_umax_i32:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_atomic_max_u32 v0, v[0:1], v2 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr %ptr, i32 %val monotonic
+  ret i32 %result
+}
+
+define i64 @flat_one_as_atomic_min_i64(ptr %ptr, i64 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_min_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB52_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_min_i64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB52_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB52_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_min_i64 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB52_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr %ptr, i64 %val syncscope("one-as") monotonic
+  ret i64 %result
+}
+
+define i64 @flat_system_atomic_min_i64(ptr %ptr, i64 %val) {
+; GFX1250-LABEL: flat_system_atomic_min_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB53_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_min_i64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB53_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB53_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_min_i64 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB53_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr %ptr, i64 %val monotonic
+  ret i64 %result
+}
+
+define i64 @flat_one_as_atomic_max_i64(ptr %ptr, i64 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_max_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB54_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_max_i64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB54_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB54_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_max_i64 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB54_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr %ptr, i64 %val syncscope("one-as") monotonic
+  ret i64 %result
+}
+
+define i64 @flat_system_atomic_max_i64(ptr %ptr, i64 %val) {
+; GFX1250-LABEL: flat_system_atomic_max_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB55_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_max_i64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB55_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB55_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_max_i64 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB55_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr %ptr, i64 %val monotonic
+  ret i64 %result
+}
+
+define i64 @flat_one_as_atomic_umin_i64(ptr %ptr, i64 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_umin_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB56_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_min_u64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB56_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB56_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_min_u64 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB56_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr %ptr, i64 %val syncscope("one-as") monotonic
+  ret i64 %result
+}
+
+define i64 @flat_system_atomic_umin_i64(ptr %ptr, i64 %val) {
+; GFX1250-LABEL: flat_system_atomic_umin_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB57_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_min_u64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB57_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB57_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_min_u64 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB57_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr %ptr, i64 %val monotonic
+  ret i64 %result
+}
+
+define i64 @flat_one_as_atomic_umax_i64(ptr %ptr, i64 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_umax_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB58_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_max_u64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB58_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB58_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_max_u64 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB58_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr %ptr, i64 %val syncscope("one-as") monotonic
+  ret i64 %result
+}
+
+define i64 @flat_system_atomic_umax_i64(ptr %ptr, i64 %val) {
+; GFX1250-LABEL: flat_system_atomic_umax_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s0, src_flat_scratch_base_hi
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_xor_b32_e32 v4, s0, v1
+; GFX1250-NEXT:    v_cmp_lt_u32_e32 vcc_lo, 0x3ffffff, v4
+; GFX1250-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB59_2
+; GFX1250-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GFX1250-NEXT:    flat_atomic_max_u64 v[4:5], v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX1250-NEXT:  .LBB59_2: ; %Flow
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_cbranch_execz .LBB59_4
+; GFX1250-NEXT:  ; %bb.3: ; %atomicrmw.private
+; GFX1250-NEXT:    s_mov_b32 s1, src_flat_scratch_base_lo
+; GFX1250-NEXT:    v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_subrev_nc_u32_e32 v4, s1, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX1250-NEXT:    scratch_load_b64 v[4:5], v6, off
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    v_max_u64 v[0:1], v[4:5], v[2:3]
+; GFX1250-NEXT:    scratch_store_b64 v6, v[0:1], off scope:SCOPE_SE
+; GFX1250-NEXT:  .LBB59_4: ; %atomicrmw.phi
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr %ptr, i64 %val monotonic
+  ret i64 %result
+}
+
+define i16 @flat_one_as_atomic_min_i16(ptr %ptr, i16 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_min_i16:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v0
+; GFX1250-NEXT:    s_mov_b32 s0, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT:    v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    flat_load_b32 v5, v[0:1]
+; GFX1250-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_not_b32_e32 v4, v4
+; GFX1250-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT:    v_min_i16 v5, v5, v2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB60_1
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw min ptr %ptr, i16 %val syncscope("one-as") monotonic
+  ret i16 %result
+}
+
+define i16 @flat_one_as_atomic_umin_i16(ptr %ptr, i16 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_umin_i16:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v0
+; GFX1250-NEXT:    s_mov_b32 s0, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT:    v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    flat_load_b32 v5, v[0:1]
+; GFX1250-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_not_b32_e32 v4, v4
+; GFX1250-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT:    v_min_u16 v5, v5, v2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB61_1
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umin ptr %ptr, i16 %val syncscope("one-as") monotonic
+  ret i16 %result
+}
+
+define i16 @flat_one_as_atomic_max_i16(ptr %ptr, i16 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_max_i16:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v0
+; GFX1250-NEXT:    s_mov_b32 s0, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT:    v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    flat_load_b32 v5, v[0:1]
+; GFX1250-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_not_b32_e32 v4, v4
+; GFX1250-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT:    v_max_i16 v5, v5, v2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB62_1
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw max ptr %ptr, i16 %val syncscope("one-as") monotonic
+  ret i16 %result
+}
+
+define i16 @flat_one_as_atomic_umax_i16(ptr %ptr, i16 %val) {
+; GFX1250-LABEL: flat_one_as_atomic_umax_i16:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v0
+; GFX1250-NEXT:    s_mov_b32 s0, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v0, -4, v3
+; GFX1250-NEXT:    v_and_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT:    flat_load_b32 v5, v[0:1]
+; GFX1250-NEXT:    v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_not_b32_e32 v4, v4
+; GFX1250-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v5, v3, v7
+; GFX1250-NEXT:    v_max_u16 v5, v5, v2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_or_b32 v6, v7, v4, v5
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB63_1
+; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v0, v3, v5
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
+  %result = atomicrmw umax ptr %ptr, i16 %val syncscope("one-as") monotonic
+  ret i16 %result
+}
diff --git a/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll b/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll
index 887f489d504f2..3266fde10f9fb 100644
--- a/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll
@@ -25,7 +25,7 @@ define amdgpu_kernel void @flat_atomic_cond_sub_no_rtn_u32(ptr %addr, i32 %in) {
 ; GFX12-GISEL-NEXT:    flat_atomic_cond_sub_u32 v0, v[0:1], v2 offset:-16 th:TH_ATOMIC_RETURN
 ; GFX12-GISEL-NEXT:    s_endpgm
 entry:
-  %gep = getelementptr i32, ptr %addr, i32 -4
+  %gep = getelementptr inbounds i32, ptr %addr, i32 -4
   %unused = call i32 @llvm.amdgcn.atomic.cond.sub.u32.p0(ptr %gep, i32 %in)
   ret void
 }
@@ -49,7 +49,7 @@ define amdgpu_kernel void @flat_atomic_cond_sub_no_rtn_u32_forced(ptr %addr, i32
 ; GFX12-GISEL-NEXT:    flat_atomic_cond_sub_u32 v[0:1], v2 offset:-16
 ; GFX12-GISEL-NEXT:    s_endpgm
 entry:
-  %gep = getelementptr i32, ptr %addr, i32 -4
+  %gep = getelementptr inbounds i32, ptr %addr, i32 -4
   %unused = call i32 @llvm.amdgcn.atomic.cond.sub.u32.p0(ptr %gep, i32 %in)
   ret void
 }
@@ -83,7 +83,7 @@ define amdgpu_kernel void @flat_atomic_cond_sub_rtn_u32(ptr %addr, i32 %in, ptr
 ; GFX12-GISEL-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-GISEL-NEXT:    s_endpgm
 entry:
-  %gep = getelementptr i32, ptr %addr, i32 4
+  %gep = getelementptr inbounds i32, ptr %addr, i32 4
   %val = call i32 @llvm.amdgcn.atomic.cond.sub.u32.p0(ptr %gep, i32 %in)
   store i32 %val, ptr %use
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
new file mode 100644
index 0000000000000..614b1e38a530f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -stress-regalloc=18 < %s | FileCheck %s
+
+define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2, i1 %cond.i.i.i2295, ptr addrspace(1) %ptr, ptr %ptr1) #0 {
+; CHECK-LABEL: vgpr_mfma_pass_av_split_crash:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[10:11], s[4:5], 0x0
+; CHECK-NEXT:    s_load_dwordx4 s[12:15], s[4:5], 0x10
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0x3e21eeb6
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0xa17f65f6
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_bitcmp1_b32 s0, 0
+; CHECK-NEXT:    s_cselect_b64 s[16:17], -1, 0
+; CHECK-NEXT:    s_xor_b64 s[18:19], s[16:17], -1
+; CHECK-NEXT:    s_bitcmp1_b32 s0, 8
+; CHECK-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; CHECK-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[2:3]
+; CHECK-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0x9037ab78
+; CHECK-NEXT:    v_accvgpr_write_b32 a3, v1
+; CHECK-NEXT:    s_xor_b64 s[20:21], s[2:3], -1
+; CHECK-NEXT:    s_and_b64 s[2:3], exec, s[2:3]
+; CHECK-NEXT:    v_accvgpr_write_b32 a2, v0
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0xbe927e4f
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0x19f4ec90
+; CHECK-NEXT:    v_mov_b32_e32 v5, 0x3efa01a0
+; CHECK-NEXT:    v_mov_b32_e32 v6, 0x16c16967
+; CHECK-NEXT:    v_mov_b32_e32 v7, 0xbf56c16c
+; CHECK-NEXT:    v_mov_b32_e32 v8, 0x69efb384
+; CHECK-NEXT:    v_mov_b32_e32 v9, 0x3f4b2bb0
+; CHECK-NEXT:    v_mov_b32_e32 v10, 0xa57d9582
+; CHECK-NEXT:    v_mov_b32_e32 v11, 0xbf8c6ea4
+; CHECK-NEXT:    v_mov_b32_e32 v12, 0xe82d3ff0
+; CHECK-NEXT:    v_mov_b32_e32 v13, 0xbfa59976
+; CHECK-NEXT:    v_mov_b32_e32 v14, 0x8427b883
+; CHECK-NEXT:    v_mov_b32_e32 v15, 0x3fae1bb4
+; CHECK-NEXT:    s_mov_b64 s[22:23], 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0x57b87036
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0x3fb3b136
+; CHECK-NEXT:    s_and_b64 s[4:5], exec, s[16:17]
+; CHECK-NEXT:    v_mov_b32_e32 v18, 0x55555523
+; CHECK-NEXT:    v_mov_b32_e32 v19, 0xbfd55555
+; CHECK-NEXT:    s_and_b64 s[6:7], exec, s[18:19]
+; CHECK-NEXT:    v_mov_b32_e32 v20, 0
+; CHECK-NEXT:    ; implicit-def: $vgpr30_vgpr31
+; CHECK-NEXT:    ; implicit-def: $vgpr22_vgpr23
+; CHECK-NEXT:    s_branch .LBB0_2
+; CHECK-NEXT:  .LBB0_1: ; %Flow9
+; CHECK-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    s_andn2_b64 vcc, exec, s[24:25]
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_17
+; CHECK-NEXT:  .LBB0_2: ; %._crit_edge1942.i.i.i3548
+; CHECK-NEXT:    ; =>This Loop Header: Depth=1
+; CHECK-NEXT:    ; Child Loop BB0_6 Depth 2
+; CHECK-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; CHECK-NEXT:    s_cbranch_vccnz .LBB0_9
+; CHECK-NEXT:  ; %bb.3: ; %.preheader1868.i.i.i3244
+; CHECK-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    s_mov_b64 vcc, s[4:5]
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_10
+; CHECK-NEXT:  ; %bb.4: ; %.preheader1855.i.i.i3329.preheader
+; CHECK-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    v_mov_b64_e32 v[24:25], s[14:15]
+; CHECK-NEXT:    flat_load_dwordx2 v[24:25], v[24:25]
+; CHECK-NEXT:    v_accvgpr_read_b32 v27, a3
+; CHECK-NEXT:    v_accvgpr_read_b32 v26, a2
+; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[2:3]
+; CHECK-NEXT:    v_mov_b64_e32 v[16:17], v[0:1]
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, 0
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, 0
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_fmac_f64_e32 v[26:27], 0, v[24:25]
+; CHECK-NEXT:    v_fmac_f64_e32 v[28:29], 0, v[26:27]
+; CHECK-NEXT:    v_mov_b64_e32 v[26:27], v[4:5]
+; CHECK-NEXT:    v_fmac_f64_e32 v[26:27], 0, v[28:29]
+; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[6:7]
+; CHECK-NEXT:    v_fmac_f64_e32 v[28:29], 0, v[26:27]
+; CHECK-NEXT:    v_mov_b64_e32 v[26:27], v[8:9]
+; CHECK-NEXT:    v_fmac_f64_e32 v[26:27], 0, v[28:29]
+; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[10:11]
+; CHECK-NEXT:    v_fmac_f64_e32 v[28:29], 0, v[26:27]
+; CHECK-NEXT:    v_mov_b64_e32 v[26:27], v[12:13]
+; CHECK-NEXT:    v_fmac_f64_e32 v[26:27], 0, v[28:29]
+; CHECK-NEXT:    v_mov_b64_e32 v[28:29], v[14:15]
+; CHECK-NEXT:    v_fmac_f64_e32 v[28:29], 0, v[26:27]
+; CHECK-NEXT:    v_fmac_f64_e32 v[16:17], 0, v[28:29]
+; CHECK-NEXT:    v_mov_b64_e32 v[26:27], v[18:19]
+; CHECK-NEXT:    v_fmac_f64_e32 v[26:27], 0, v[16:17]
+; CHECK-NEXT:    s_branch .LBB0_6
+; CHECK-NEXT:  .LBB0_5: ; %Flow
+; CHECK-NEXT:    ; in Loop: Header=BB0_6 Depth=2
+; CHECK-NEXT:    s_and_b64 vcc, exec, s[8:9]
+; CHECK-NEXT:    s_cbranch_vccnz .LBB0_11
+; CHECK-NEXT:  .LBB0_6: ; %.preheader1855.i.i.i3329
+; CHECK-NEXT:    ; Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:    ; => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    v_accvgpr_read_b32 v29, a1
+; CHECK-NEXT:    v_accvgpr_read_b32 v28, a0
+; CHECK-NEXT:    s_mov_b64 s[24:25], -1
+; CHECK-NEXT:    s_mov_b64 s[8:9], -1
+; CHECK-NEXT:    s_mov_b64 vcc, s[2:3]
+; CHECK-NEXT:    ; implicit-def: $agpr0_agpr1
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_5
+; CHECK-NEXT:  ; %bb.7: ; %.lr.ph2070.i.i.i3291
+; CHECK-NEXT:    ; in Loop: Header=BB0_6 Depth=2
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, v30
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, v31
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[18:19]
+; CHECK-NEXT:    s_mov_b64 vcc, s[6:7]
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_5
+; CHECK-NEXT:  ; %bb.8: ; %.preheader1856.preheader.i.i.i3325
+; CHECK-NEXT:    ; in Loop: Header=BB0_6 Depth=2
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, v26
+; CHECK-NEXT:    s_mov_b64 s[24:25], 0
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, v27
+; CHECK-NEXT:    s_mov_b64 s[8:9], 0
+; CHECK-NEXT:    s_branch .LBB0_5
+; CHECK-NEXT:  .LBB0_9: ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    s_mov_b64 s[22:23], 0
+; CHECK-NEXT:    v_mov_b64_e32 v[30:31], s[10:11]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[20:21]
+; CHECK-NEXT:    s_branch .LBB0_15
+; CHECK-NEXT:  .LBB0_10: ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    s_mov_b64 s[8:9], -1
+; CHECK-NEXT:    v_mov_b64_e32 v[22:23], 0
+; CHECK-NEXT:    s_branch .LBB0_15
+; CHECK-NEXT:  .LBB0_11: ; %loop.exit.guard
+; CHECK-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    s_and_b64 vcc, exec, s[24:25]
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_13
+; CHECK-NEXT:  ; %bb.12: ; %._crit_edge2105.i.i.i2330.loopexit
+; CHECK-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    v_cmp_nlg_f64_e64 s[8:9], 0, v[28:29]
+; CHECK-NEXT:    v_cndmask_b32_e64 v23, v23, 0, s[16:17]
+; CHECK-NEXT:    v_cndmask_b32_e64 v22, v22, 0, s[16:17]
+; CHECK-NEXT:    v_cndmask_b32_e64 v16, 0, 1, s[8:9]
+; CHECK-NEXT:    v_mov_b32_e32 v17, v16
+; CHECK-NEXT:    s_and_b64 s[8:9], exec, s[16:17]
+; CHECK-NEXT:    global_store_dwordx2 v20, v[16:17], s[12:13]
+; CHECK-NEXT:    s_cselect_b32 s23, s23, 0
+; CHECK-NEXT:    s_cselect_b32 s22, s22, 0
+; CHECK-NEXT:    s_mov_b64 s[8:9], -1
+; CHECK-NEXT:    s_branch .LBB0_14
+; CHECK-NEXT:  .LBB0_13: ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    s_mov_b64 s[8:9], 0
+; CHECK-NEXT:    v_mov_b64_e32 v[22:23], 0
+; CHECK-NEXT:  .LBB0_14: ; %Flow6
+; CHECK-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    v_mov_b64_e32 v[30:31], v[24:25]
+; CHECK-NEXT:  .LBB0_15: ; %Flow6
+; CHECK-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    s_mov_b64 s[24:25], -1
+; CHECK-NEXT:    s_and_b64 vcc, exec, s[8:9]
+; CHECK-NEXT:    s_cbranch_vccz .LBB0_1
+; CHECK-NEXT:  ; %bb.16: ; %._crit_edge2105.i.i.i2330
+; CHECK-NEXT:    ; in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    v_mov_b32_e32 v21, v20
+; CHECK-NEXT:    s_mov_b64 s[24:25], 0
+; CHECK-NEXT:    global_store_dwordx2 v20, v[20:21], s[12:13]
+; CHECK-NEXT:    s_branch .LBB0_1
+; CHECK-NEXT:  .LBB0_17: ; %DummyReturnBlock
+; CHECK-NEXT:    s_endpgm
+entry:
+  br label %._crit_edge1942.i.i.i3548
+
+._crit_edge1942.i.i.i3548:                        ; preds = %._crit_edge2105.i.i.i2330, %entry
+  %.sroa.02591.0.i.i.i226323 = phi double [ poison, %entry ], [ %.sroa.02591.3.i.i.i2301, %._crit_edge2105.i.i.i2330 ]
+  %.sroa.3.0.i.i.i2270 = phi double [ poison, %entry ], [ %.sroa.3.3.i.i.i2308, %._crit_edge2105.i.i.i2330 ]
+  %.014942244.i.i.i2280 = phi double [ 0.000000e+00, %entry ], [ %.31497.i.i.i2317, %._crit_edge2105.i.i.i2330 ]
+  br i1 %cond.i.i.i2295, label %.preheader1868.i.i.i3244, label %._crit_edge2105.i.i.i2330
+
+.preheader1868.i.i.i3244:                         ; preds = %._crit_edge1942.i.i.i3548
+  %i = load double, ptr %ptr1, align 8
+  %i3 = call double @llvm.fma.f64(double %i, double 0.000000e+00, double 0x3E21EEB69037AB78)
+  %i4 = call double @llvm.fma.f64(double 0.000000e+00, double %i3, double 0xBE927E4FA17F65F6)
+  %i5 = call double @llvm.fma.f64(double 0.000000e+00, double %i4, double 0x3EFA01A019F4EC90)
+  %i6 = call double @llvm.fma.f64(double 0.000000e+00, double %i5, double 0xBF56C16C16C16967)
+  %spec.select.i.i.i3288 = select i1 %arg2, double 0.000000e+00, double %.sroa.3.0.i.i.i2270
+  br i1 %arg2, label %.preheader1855.i.i.i3329, label %._crit_edge2105.i.i.i2330
+
+.lr.ph2070.i.i.i3291:                             ; preds = %.preheader1855.i.i.i3329
+  br i1 %arg2, label %.preheader1855.i.i.i3329, label %.preheader1856.preheader.i.i.i3325
+
+.preheader1856.preheader.i.i.i3325:               ; preds = %.lr.ph2070.i.i.i3291
+  %i11 = call double @llvm.fma.f64(double 0.000000e+00, double %i6, double 0x3F4B2BB069EFB384)
+  %i14 = call double @llvm.fma.f64(double 0.000000e+00, double %i11, double 0xBF8C6EA4A57D9582)
+  %i18 = call double @llvm.fma.f64(double 0.000000e+00, double %i14, double 0xBFA59976E82D3FF0)
+  %i21 = call double @llvm.fma.f64(double 0.000000e+00, double %i18, double 0x3FAE1BB48427B883)
+  %i23 = call double @llvm.fma.f64(double 0.000000e+00, double %i21, double 0x3FB3B13657B87036)
+  %i28 = call double @llvm.fma.f64(double 0.000000e+00, double %i23, double 0xBFD5555555555523)
+  br label %.preheader1855.i.i.i3329
+
+.preheader1855.i.i.i3329:                         ; preds = %.preheader1856.preheader.i.i.i3325, %.lr.ph2070.i.i.i3291, %.preheader1868.i.i.i3244
+  %.sroa.02591.4.i.i.i3335 = phi double [ %i28, %.preheader1856.preheader.i.i.i3325 ], [ %.sroa.02591.0.i.i.i226323, %.lr.ph2070.i.i.i3291 ], [ 0.000000e+00, %.preheader1868.i.i.i3244 ]
+  %.21496.ph.i.i.i3348 = select i1 %arg2, double %.014942244.i.i.i2280, double 0.000000e+00
+  %i31 = fcmp one double %.sroa.02591.4.i.i.i3335, 0.000000e+00
+  %i32 = select i1 %i31, <2 x i32> zeroinitializer, <2 x i32> splat (i32 1)
+  store <2 x i32> %i32, ptr addrspace(1) %ptr, align 8
+  br i1 %cond.i.i.i2295, label %.lr.ph2070.i.i.i3291, label %._crit_edge2105.i.i.i2330
+
+._crit_edge2105.i.i.i2330:                        ; preds = %.preheader1855.i.i.i3329, %.preheader1868.i.i.i3244, %._crit_edge1942.i.i.i3548
+  %.sroa.02591.3.i.i.i2301 = phi double [ %.sroa.02591.0.i.i.i226323, %.preheader1868.i.i.i3244 ], [ %arg1, %._crit_edge1942.i.i.i3548 ], [ %i, %.preheader1855.i.i.i3329 ]
+  %.sroa.3.3.i.i.i2308 = phi double [ 0.000000e+00, %.preheader1868.i.i.i3244 ], [ %.sroa.3.0.i.i.i2270, %._crit_edge1942.i.i.i3548 ], [ %spec.select.i.i.i3288, %.preheader1855.i.i.i3329 ]
+  %.31497.i.i.i2317 = phi double [ %.014942244.i.i.i2280, %.preheader1868.i.i.i3244 ], [ 0.000000e+00, %._crit_edge1942.i.i.i3548 ], [ %.21496.ph.i.i.i3348, %.preheader1855.i.i.i3329 ]
+  store double 0.000000e+00, ptr addrspace(1) %ptr, align 8
+  br label %._crit_edge1942.i.i.i3548
+}
+
+declare double @llvm.fma.f64(double, double, double) #1
+
+attributes #0 = { "amdgpu-waves-per-eu"="8,8" "target-cpu"="gfx942" }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/CodeGen/AMDGPU/av_movimm_pseudo_expansion.mir b/llvm/test/CodeGen/AMDGPU/av_movimm_pseudo_expansion.mir
index 1f9d49073da2c..272997cf1a347 100644
--- a/llvm/test/CodeGen/AMDGPU/av_movimm_pseudo_expansion.mir
+++ b/llvm/test/CodeGen/AMDGPU/av_movimm_pseudo_expansion.mir
@@ -1,6 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
-# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=postrapseudos %s -o - | FileCheck %s
-# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=postrapseudos %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx908 -run-pass=postrapseudos %s -o - | FileCheck -check-prefixes=CHECK,GFX908 %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx90a -run-pass=postrapseudos %s -o - | FileCheck -check-prefixes=CHECK,GFX90A %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx942 -run-pass=postrapseudos %s -o - | FileCheck -check-prefixes=CHECK,GFX942 %s
 
 ---
 name: av_mov_b32_imm_pseudo_agpr_0
@@ -54,3 +55,134 @@ body: |
     ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 $vgpr0, implicit $exec
     $agpr1 = AV_MOV_B32_IMM_PSEUDO $vgpr0, implicit $exec
 ...
+
+---
+name: av_mov_b64_imm_pseudo_agpr_0
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b64_imm_pseudo_agpr_0
+    ; CHECK: $agpr0 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec, implicit-def $agpr0_agpr1
+    ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec, implicit-def $agpr0_agpr1
+    $agpr0_agpr1 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+...
+
+---
+name: av_mov_b64_imm_pseudo_agpr_neg1
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b64_imm_pseudo_agpr_neg1
+    ; CHECK: $agpr0 = V_ACCVGPR_WRITE_B32_e64 -1, implicit $exec, implicit-def $agpr0_agpr1
+    ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 -1, implicit $exec, implicit-def $agpr0_agpr1
+    $agpr0_agpr1 = AV_MOV_B64_IMM_PSEUDO -1, implicit $exec
+...
+
+---
+name: av_mov_b64_imm_pseudo_agpr_64
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b64_imm_pseudo_agpr_64
+    ; CHECK: $agpr0 = V_ACCVGPR_WRITE_B32_e64 64, implicit $exec, implicit-def $agpr0_agpr1
+    ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec, implicit-def $agpr0_agpr1
+    $agpr0_agpr1 = AV_MOV_B64_IMM_PSEUDO 64, implicit $exec
+...
+
+---
+name: av_mov_b64_imm_pseudo_vgpr_0
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GFX908-LABEL: name: av_mov_b64_imm_pseudo_vgpr_0
+    ; GFX908: $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; GFX908-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ;
+    ; GFX90A-LABEL: name: av_mov_b64_imm_pseudo_vgpr_0
+    ; GFX90A: $vgpr0_vgpr1 = V_PK_MOV_B32 8, 0, 8, 0, 0, 0, 0, 0, 0, implicit $exec
+    ;
+    ; GFX942-LABEL: name: av_mov_b64_imm_pseudo_vgpr_0
+    ; GFX942: $vgpr0_vgpr1 = V_MOV_B64_e32 0, implicit $exec
+    $vgpr0_vgpr1 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+...
+
+---
+name: av_mov_b64_imm_pseudo_vgpr_64
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GFX908-LABEL: name: av_mov_b64_imm_pseudo_vgpr_64
+    ; GFX908: $vgpr0 = V_MOV_B32_e32 64, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; GFX908-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ;
+    ; GFX90A-LABEL: name: av_mov_b64_imm_pseudo_vgpr_64
+    ; GFX90A: $vgpr0 = V_MOV_B32_e32 64, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ;
+    ; GFX942-LABEL: name: av_mov_b64_imm_pseudo_vgpr_64
+    ; GFX942: $vgpr0_vgpr1 = V_MOV_B64_e32 64, implicit $exec
+    $vgpr0_vgpr1 = AV_MOV_B64_IMM_PSEUDO 64, implicit $exec
+...
+
+---
+name: av_mov_b64_imm_pseudo_agpr_64_hi_0_lo
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b64_imm_pseudo_agpr_64_hi_0_lo
+    ; CHECK: $agpr0 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec, implicit-def $agpr0_agpr1
+    ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 64, implicit $exec, implicit-def $agpr0_agpr1
+    $agpr0_agpr1 = AV_MOV_B64_IMM_PSEUDO 274877906944, implicit $exec
+...
+
+---
+name: av_mov_b64_imm_pseudo_agpr_64_hi_2_lo
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b64_imm_pseudo_agpr_64_hi_2_lo
+    ; CHECK: $agpr0 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec, implicit-def $agpr0_agpr1
+    ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 64, implicit $exec, implicit-def $agpr0_agpr1
+    $agpr0_agpr1 = AV_MOV_B64_IMM_PSEUDO 274877906946, implicit $exec
+...
+
+---
+name: av_mov_b64_imm_pseudo_agpr_neg16_hi_9_lo
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: av_mov_b64_imm_pseudo_agpr_neg16_hi_9_lo
+    ; CHECK: $agpr0 = V_ACCVGPR_WRITE_B32_e64 9, implicit $exec, implicit-def $agpr0_agpr1
+    ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 -16, implicit $exec, implicit-def $agpr0_agpr1
+    $agpr0_agpr1 = AV_MOV_B64_IMM_PSEUDO 18446744004990074889, implicit $exec
+...
+
+---
+name: av_mov_b64_imm_pseudo_vgpr_inv2pi
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GFX908-LABEL: name: av_mov_b64_imm_pseudo_vgpr_inv2pi
+    ; GFX908: $vgpr0 = V_MOV_B32_e32 1042479491, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; GFX908-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; GFX908-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr2_vgpr3
+    ; GFX908-NEXT: $vgpr3 = V_MOV_B32_e32 1042479491, implicit $exec, implicit-def $vgpr2_vgpr3
+    ; GFX908-NEXT: $vgpr4 = V_MOV_B32_e32 1042479491, implicit $exec, implicit-def $vgpr4_vgpr5
+    ; GFX908-NEXT: $vgpr5 = V_MOV_B32_e32 1042479491, implicit $exec, implicit-def $vgpr4_vgpr5
+    ;
+    ; GFX90A-LABEL: name: av_mov_b64_imm_pseudo_vgpr_inv2pi
+    ; GFX90A: $vgpr0 = V_MOV_B32_e32 1042479491, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; GFX90A-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; GFX90A-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr2_vgpr3
+    ; GFX90A-NEXT: $vgpr3 = V_MOV_B32_e32 1042479491, implicit $exec, implicit-def $vgpr2_vgpr3
+    ; GFX90A-NEXT: $vgpr4_vgpr5 = V_PK_MOV_B32 8, 1042479491, 8, 1042479491, 0, 0, 0, 0, 0, implicit $exec
+    ;
+    ; GFX942-LABEL: name: av_mov_b64_imm_pseudo_vgpr_inv2pi
+    ; GFX942: $vgpr0_vgpr1 = V_MOV_B64_e32 1042479491, implicit $exec
+    ; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr2_vgpr3
+    ; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 1042479491, implicit $exec, implicit-def $vgpr2_vgpr3
+    ; GFX942-NEXT: $vgpr4_vgpr5 = V_PK_MOV_B32 8, 1042479491, 8, 1042479491, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0_vgpr1 = AV_MOV_B64_IMM_PSEUDO 1042479491, implicit $exec
+    $vgpr2_vgpr3 = AV_MOV_B64_IMM_PSEUDO 4477415320595726336, implicit $exec
+    $vgpr4_vgpr5 = AV_MOV_B64_IMM_PSEUDO 4477415321638205827, implicit $exec
+...
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index 752a87ac3cb73..02ead572145f9 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -344,7 +344,7 @@ define amdgpu_ps void @fptrunc_f32_to_bf16(float %a, ptr %out) {
 ; GFX1250:       ; %bb.0: ; %entry
 ; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
 ; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT:    flat_store_b16 v[2:3], v0
+; GFX1250-NEXT:    flat_store_b16 v[2:3], v0 scope:SCOPE_SE
 ; GFX1250-NEXT:    s_endpgm
 entry:
   %a.cvt = fptrunc float %a to bfloat
@@ -380,7 +380,7 @@ define amdgpu_ps void @fptrunc_f32_to_bf16_abs(float %a, ptr %out) {
 ; GFX1250:       ; %bb.0: ; %entry
 ; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
 ; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, |v0|, s0
-; GFX1250-NEXT:    flat_store_b16 v[2:3], v0
+; GFX1250-NEXT:    flat_store_b16 v[2:3], v0 scope:SCOPE_SE
 ; GFX1250-NEXT:    s_endpgm
 entry:
   %a.abs = call float @llvm.fabs.f32(float %a)
@@ -417,7 +417,7 @@ define amdgpu_ps void @fptrunc_f32_to_bf16_neg(float %a, ptr %out) {
 ; GFX1250:       ; %bb.0: ; %entry
 ; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
 ; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, -v0, s0
-; GFX1250-NEXT:    flat_store_b16 v[2:3], v0
+; GFX1250-NEXT:    flat_store_b16 v[2:3], v0 scope:SCOPE_SE
 ; GFX1250-NEXT:    s_endpgm
 entry:
   %a.neg = fneg float %a
@@ -480,7 +480,7 @@ define amdgpu_ps void @fptrunc_f64_to_bf16(double %a, ptr %out) {
 ; GFX1250-NEXT:    s_or_b32 vcc_lo, vcc_lo, s0
 ; GFX1250-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc_lo
 ; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT:    flat_store_b16 v[2:3], v0
+; GFX1250-NEXT:    flat_store_b16 v[2:3], v0 scope:SCOPE_SE
 ; GFX1250-NEXT:    s_endpgm
 entry:
   %a.cvt = fptrunc double %a to bfloat
@@ -543,7 +543,7 @@ define amdgpu_ps void @fptrunc_f64_to_bf16_neg(double %a, ptr %out) {
 ; GFX1250-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc_lo
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT:    flat_store_b16 v[2:3], v0
+; GFX1250-NEXT:    flat_store_b16 v[2:3], v0 scope:SCOPE_SE
 ; GFX1250-NEXT:    s_endpgm
 entry:
   %a.neg = fneg double %a
@@ -607,7 +607,7 @@ define amdgpu_ps void @fptrunc_f64_to_bf16_abs(double %a, ptr %out) {
 ; GFX1250-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc_lo
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT:    flat_store_b16 v[2:3], v0
+; GFX1250-NEXT:    flat_store_b16 v[2:3], v0 scope:SCOPE_SE
 ; GFX1250-NEXT:    s_endpgm
 entry:
   %a.abs = call double @llvm.fabs.f64(double %a)
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-math.ll b/llvm/test/CodeGen/AMDGPU/bf16-math.ll
index 9979e832b799a..30a78648c186a 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-math.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-math.ll
@@ -368,10 +368,7 @@ define amdgpu_ps float @test_clamp_v2bf16_s(<2 x bfloat> inreg %src) {
 define amdgpu_ps bfloat @test_clamp_bf16_folding(bfloat %src) {
 ; GCN-LABEL: test_clamp_bf16_folding:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_exp_bf16_e32 v0, v0
-; GCN-NEXT:    v_nop
-; GCN-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
-; GCN-NEXT:    v_pk_max_num_bf16 v0, v0, v0 clamp
+; GCN-NEXT:    v_exp_bf16_e64 v0, v0 clamp
 ; GCN-NEXT:    ; return to shader part epilog
   %exp = call bfloat @llvm.exp2.bf16(bfloat %src)
   %max = call bfloat @llvm.maxnum.bf16(bfloat %exp, bfloat 0.0)
@@ -382,9 +379,7 @@ define amdgpu_ps bfloat @test_clamp_bf16_folding(bfloat %src) {
 define amdgpu_ps float @test_clamp_v2bf16_folding(<2 x bfloat> %src0, <2 x bfloat> %src1) {
 ; GCN-LABEL: test_clamp_v2bf16_folding:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_mul_bf16 v0, v0, v1
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT:    v_pk_max_num_bf16 v0, v0, v0 clamp
+; GCN-NEXT:    v_pk_mul_bf16 v0, v0, v1 clamp
 ; GCN-NEXT:    ; return to shader part epilog
   %mul = fmul <2 x bfloat> %src0, %src1
   %max = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %mul, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
@@ -396,9 +391,7 @@ define amdgpu_ps float @test_clamp_v2bf16_folding(<2 x bfloat> %src0, <2 x bfloa
 define amdgpu_ps void @v_test_mul_add_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
 ; GCN-LABEL: v_test_mul_add_v2bf16_vvv:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_mul_bf16 v2, v2, v3
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT:    v_pk_add_bf16 v2, v2, v4
+; GCN-NEXT:    v_pk_fma_bf16 v2, v2, v3, v4
 ; GCN-NEXT:    global_store_b32 v[0:1], v2, off
 ; GCN-NEXT:    s_endpgm
   %mul = fmul contract <2 x bfloat> %a, %b
@@ -410,9 +403,7 @@ define amdgpu_ps void @v_test_mul_add_v2bf16_vvv(ptr addrspace(1) %out, <2 x bfl
 define amdgpu_ps void @v_test_mul_add_v2bf16_vss(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
 ; GCN-LABEL: v_test_mul_add_v2bf16_vss:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_mul_bf16 v2, v2, s0
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT:    v_pk_add_bf16 v2, v2, s1
+; GCN-NEXT:    v_pk_fma_bf16 v2, v2, s0, s1
 ; GCN-NEXT:    global_store_b32 v[0:1], v2, off
 ; GCN-NEXT:    s_endpgm
   %mul = fmul contract <2 x bfloat> %a, %b
@@ -424,9 +415,9 @@ define amdgpu_ps void @v_test_mul_add_v2bf16_vss(ptr addrspace(1) %out, <2 x bfl
 define amdgpu_ps void @v_test_mul_add_v2bf16_sss(ptr addrspace(1) %out, <2 x bfloat> inreg %a, <2 x bfloat> inreg %b, <2 x bfloat> inreg %c) {
 ; GCN-LABEL: v_test_mul_add_v2bf16_sss:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_mul_bf16 v2, s0, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
 ; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT:    v_pk_add_bf16 v2, v2, s2
+; GCN-NEXT:    v_pk_fma_bf16 v2, s0, s1, v2
 ; GCN-NEXT:    global_store_b32 v[0:1], v2, off
 ; GCN-NEXT:    s_endpgm
   %mul = fmul contract <2 x bfloat> %a, %b
@@ -438,9 +429,7 @@ define amdgpu_ps void @v_test_mul_add_v2bf16_sss(ptr addrspace(1) %out, <2 x bfl
 define amdgpu_ps void @v_test_mul_add_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfloat> %a, <2 x bfloat> inreg %b) {
 ; GCN-LABEL: v_test_mul_add_v2bf16_vsc:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_mul_bf16 v2, v2, s0
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT:    v_pk_add_bf16 v2, v2, 0.5 op_sel_hi:[1,0]
+; GCN-NEXT:    v_pk_fma_bf16 v2, v2, s0, 0.5 op_sel_hi:[1,1,0]
 ; GCN-NEXT:    global_store_b32 v[0:1], v2, off
 ; GCN-NEXT:    s_endpgm
   %mul = fmul contract <2 x bfloat> %a, %b
@@ -452,9 +441,9 @@ define amdgpu_ps void @v_test_mul_add_v2bf16_vsc(ptr addrspace(1) %out, <2 x bfl
 define amdgpu_ps void @v_test_mul_add_v2bf16_vll(ptr addrspace(1) %out, <2 x bfloat> %a) {
 ; GCN-LABEL: v_test_mul_add_v2bf16_vll:
 ; GCN:       ; %bb.0:
-; GCN-NEXT:    v_pk_mul_bf16 v2, 0x42c83f80, v2
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT:    v_pk_add_bf16 v2, 0x43484000, v2
+; GCN-NEXT:    s_mov_b32 s0, 0x43484000
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GCN-NEXT:    v_pk_fma_bf16 v2, 0x42c83f80, v2, s0
 ; GCN-NEXT:    global_store_b32 v[0:1], v2, off
 ; GCN-NEXT:    s_endpgm
   %mul = fmul contract <2 x bfloat> %a, <bfloat 1.0, bfloat 100.0>
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 52e697cae9fe5..10e523d1a0cf1 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -24671,7 +24671,6 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
   ret <32 x bfloat> %op
 }
 
-
 declare bfloat @llvm.maxnum.bf16(bfloat, bfloat)
 declare <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat>, <2 x bfloat>)
 declare <3 x bfloat> @llvm.maxnum.v3bf16(<3 x bfloat>, <3 x bfloat>)
@@ -29673,7 +29672,6 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) {
   ret { bfloat, i16 } %op
 }
 
-
 declare bfloat @llvm.log.bf16(bfloat)
 declare bfloat @llvm.log2.bf16(bfloat)
 declare bfloat @llvm.log10.bf16(bfloat)
@@ -37776,9 +37774,10 @@ define bfloat @v_uitofp_i16_to_bf16(i16 %x) {
 ; GFX11TRUE16-LABEL: v_uitofp_i16_to_bf16:
 ; GFX11TRUE16:       ; %bb.0:
 ; GFX11TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
 ; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_cvt_f32_u32_e32 v0, v0
+; GFX11TRUE16-NEXT:    v_cvt_f32_u32_e32 v0, v1
 ; GFX11TRUE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX11TRUE16-NEXT:    v_or_b32_e32 v2, 0x400000, v0
 ; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -40752,12 +40751,11 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) {
 ;
 ; GFX11TRUE16-LABEL: s_select_bf16:
 ; GFX11TRUE16:       ; %bb.0:
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v1.l, s0
 ; GFX11TRUE16-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11TRUE16-NEXT:    v_mov_b16_e32 v0.l, s0
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_cndmask_b16 v0.l, s1, v0.l, vcc_lo
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT:    v_cndmask_b16 v0.l, s1, v1.l, vcc_lo
 ; GFX11TRUE16-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11TRUE16-NEXT:    ; return to shader part epilog
 ;
@@ -47043,18 +47041,10 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
 ; GFX8-LABEL: v_fmuladd_bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX8-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 0x7fff, v1
-; GFX8-NEXT:    v_or_b32_e32 v3, 0x400000, v0
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v3, vcc
-; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT:    v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT:    v_fma_f32 v0, v0, v1, v2
 ; GFX8-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 0x7fff, v1
@@ -47067,20 +47057,13 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
 ; GFX900-LABEL: v_fmuladd_bf16:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX900-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-NEXT:    v_fma_f32 v0, v0, v1, v2
 ; GFX900-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX900-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX900-NEXT:    v_or_b32_e32 v3, 0x400000, v0
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v1, v3, vcc
-; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
-; GFX900-NEXT:    v_add_f32_e32 v0, v0, v1
-; GFX900-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX900-NEXT:    v_add3_u32 v1, v1, v0, s4
 ; GFX900-NEXT:    v_or_b32_e32 v2, 0x400000, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
@@ -47090,35 +47073,25 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
 ; GFX950-LABEL: v_fmuladd_bf16:
 ; GFX950:       ; %bb.0:
 ; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
-; GFX950-NEXT:    v_add_f32_e32 v0, v0, v1
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX950-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v2, s0
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fmuladd_bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v3, 0x400000, v0
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v3, vcc_lo
-; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT:    v_add_f32_e32 v0, v0, v1
-; GFX10-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX10-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v0, v2, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v1, 0x400000, v2
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT:    v_add3_u32 v0, v0, v2, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -47126,55 +47099,38 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
 ; GFX11TRUE16:       ; %bb.0:
 ; GFX11TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11TRUE16-NEXT:    v_mov_b16_e32 v3.l, 0
-; GFX11TRUE16-NEXT:    v_mov_b16_e32 v3.h, v1.l
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v3.h, v2.l
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v2.h, v1.l
 ; GFX11TRUE16-NEXT:    v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
 ; GFX11TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
-; GFX11TRUE16-NEXT:    v_mul_f32_e32 v0, v1, v3
-; GFX11TRUE16-NEXT:    v_mov_b16_e32 v3.h, v2.l
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v0
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v1, v4, vcc_lo
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT:    v_fmac_f32_e32 v3, v1, v2
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT:    v_bfe_u32 v0, v3, 16, 1
+; GFX11TRUE16-NEXT:    v_or_b32_e32 v1, 0x400000, v3
+; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT:    v_add3_u32 v0, v0, v3, 0x7fff
 ; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_add_f32_e32 v0, v0, v3
-; GFX11TRUE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
 ; GFX11TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
 ; GFX11TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11FAKE16-LABEL: v_fmuladd_bf16:
 ; GFX11FAKE16:       ; %bb.0:
 ; GFX11FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11FAKE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v3, 0x400000, v0
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11FAKE16-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX11FAKE16-NEXT:    v_bfe_u32 v0, v2, 16, 1
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v1, 0x400000, v2
+; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
 ; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT:    v_dual_cndmask_b32 v0, v1, v3 :: v_dual_lshlrev_b32 v1, 16, v2
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11FAKE16-NEXT:    v_add_f32_e32 v0, v0, v1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT:    v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11FAKE16-NEXT:    v_add3_u32 v0, v0, v2, 0x7fff
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
 ; GFX11FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %op = call bfloat @llvm.fmuladd.bf16(bfloat %a, bfloat %b, bfloat %c)
@@ -47235,39 +47191,22 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
 ; GFX8-LABEL: v_fmuladd_v2bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX8-NEXT:    v_mul_f32_e32 v3, v4, v3
-; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 0x7fff, v4
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT:    v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
+; GFX8-NEXT:    v_fma_f32 v3, v5, v4, v3
 ; GFX8-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v4, v3
+; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, s4, v4
-; GFX8-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 0x7fff, v4
+; GFX8-NEXT:    v_fma_f32 v0, v0, v1, v2
 ; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v3
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, s4, v1
-; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v0
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v4, vcc
-; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
-; GFX8-NEXT:    v_add_f32_e32 v0, v0, v1
-; GFX8-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v1, vcc, v1, v0
 ; GFX8-NEXT:    v_add_u32_e32 v1, vcc, 0x7fff, v1
 ; GFX8-NEXT:    v_or_b32_e32 v2, 0x400000, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
@@ -47279,36 +47218,22 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
 ; GFX900-LABEL: v_fmuladd_v2bf16:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX900-NEXT:    v_mul_f32_e32 v3, v4, v3
-; GFX900-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v4, v4, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
-; GFX900-NEXT:    v_add_f32_e32 v3, v3, v4
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX900-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
+; GFX900-NEXT:    v_fma_f32 v3, v5, v4, v3
+; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX900-NEXT:    v_bfe_u32 v4, v3, 16, 1
-; GFX900-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX900-NEXT:    v_fma_f32 v0, v0, v1, v2
 ; GFX900-NEXT:    v_add3_u32 v4, v4, v3, s4
 ; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v3
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_bfe_u32 v1, v0, 16, 1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
 ; GFX900-NEXT:    v_add3_u32 v1, v1, v0, s4
-; GFX900-NEXT:    v_or_b32_e32 v4, 0x400000, v0
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v1, v4, vcc
-; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
-; GFX900-NEXT:    v_add_f32_e32 v0, v0, v1
-; GFX900-NEXT:    v_bfe_u32 v1, v0, 16, 1
-; GFX900-NEXT:    v_add3_u32 v1, v1, v0, s4
 ; GFX900-NEXT:    v_or_b32_e32 v2, 0x400000, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
@@ -47319,150 +47244,94 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
 ; GFX950-LABEL: v_fmuladd_v2bf16:
 ; GFX950:       ; %bb.0:
 ; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT:    v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT:    v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
+; GFX950-NEXT:    v_and_b32_e32 v4, 0xffff0000, v1
+; GFX950-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
+; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_mul_f32_e32 v3, v4, v3
-; GFX950-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_and_b32_e32 v4, 0xffff0000, v2
-; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
-; GFX950-NEXT:    v_add_f32_e32 v3, v3, v4
-; GFX950-NEXT:    v_add_f32_e32 v0, v0, v1
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v3
+; GFX950-NEXT:    v_fmac_f32_e32 v3, v5, v4
+; GFX950-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v2, v3
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fmuladd_v2bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT:    v_mul_f32_e32 v3, v4, v3
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT:    v_bfe_u32 v1, v3, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX10-NEXT:    v_bfe_u32 v4, v0, 16, 1
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT:    v_or_b32_e32 v6, 0x400000, v0
-; GFX10-NEXT:    v_add3_u32 v1, v1, v3, 0x7fff
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT:    v_add3_u32 v4, v4, v0, 0x7fff
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc_lo
-; GFX10-NEXT:    v_add_f32_e32 v1, v1, v3
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX10-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX10-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX10-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
+; GFX10-NEXT:    v_fmac_f32_e32 v3, v5, v4
+; GFX10-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX10-NEXT:    v_bfe_u32 v0, v3, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v4, 0x400000, v3
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX10-NEXT:    v_bfe_u32 v1, v2, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v5, 0x400000, v2
+; GFX10-NEXT:    v_add3_u32 v0, v0, v3, 0x7fff
+; GFX10-NEXT:    v_add3_u32 v1, v1, v2, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11TRUE16-LABEL: v_fmuladd_v2bf16:
 ; GFX11TRUE16:       ; %bb.0:
 ; GFX11TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
+; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX11TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_dual_mul_f32 v3, v4, v3 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_bfe_u32 v1, v3, 16, 1
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT:    v_add3_u32 v1, v1, v3, 0x7fff
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX11TRUE16-NEXT:    v_bfe_u32 v4, v0, 16, 1
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v0
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT:    v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX11TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT:    v_dual_cndmask_b32 v0, v4, v6 :: v_dual_add_f32 v1, v1, v3
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX11TRUE16-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX11TRUE16-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v1, v0
-; GFX11TRUE16-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fmuladd_v2bf16:
-; GFX11FAKE16:       ; %bb.0:
-; GFX11FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v1
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v0
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_dual_mul_f32 v3, v4, v3 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11FAKE16-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_bfe_u32 v1, v3, 16, 1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11FAKE16-NEXT:    v_add3_u32 v1, v1, v3, 0x7fff
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX11FAKE16-NEXT:    v_bfe_u32 v4, v0, 16, 1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v6, 0x400000, v0
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT:    v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT:    v_dual_fmac_f32 v2, v0, v1 :: v_dual_fmac_f32 v3, v5, v4
+; GFX11TRUE16-NEXT:    v_bfe_u32 v1, v2, 16, 1
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT:    v_bfe_u32 v0, v3, 16, 1
+; GFX11TRUE16-NEXT:    v_or_b32_e32 v4, 0x400000, v3
+; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v2
+; GFX11TRUE16-NEXT:    v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11TRUE16-NEXT:    v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
+; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11FAKE16-LABEL: v_fmuladd_v2bf16:
+; GFX11FAKE16:       ; %bb.0:
+; GFX11FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v1
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
 ; GFX11FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11FAKE16-NEXT:    v_dual_cndmask_b32 v0, v4, v6 :: v_dual_add_f32 v1, v1, v3
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v1
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX11FAKE16-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX11FAKE16-NEXT:    v_bfe_u32 v3, v0, 16, 1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX11FAKE16-NEXT:    v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v1, v2, v4, vcc_lo
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
+; GFX11FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT:    v_dual_fmac_f32 v2, v0, v1 :: v_dual_fmac_f32 v3, v5, v4
+; GFX11FAKE16-NEXT:    v_bfe_u32 v1, v2, 16, 1
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11FAKE16-NEXT:    v_bfe_u32 v0, v3, 16, 1
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v4, 0x400000, v3
+; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v2
+; GFX11FAKE16-NEXT:    v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11FAKE16-NEXT:    v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc_lo
 ; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_perm_b32 v0, v0, v1, 0x7060302
+; GFX11FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x7060302
 ; GFX11FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %op = call <2 x bfloat> @llvm.fmuladd.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
   ret <2 x bfloat> %op
@@ -47542,57 +47411,33 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
 ; GFX8-LABEL: v_fmuladd_v3bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GFX8-NEXT:    v_fma_f32 v1, v1, v3, v5
 ; GFX8-NEXT:    v_bfe_u32 v3, v1, 16, 1
 ; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v1
 ; GFX8-NEXT:    v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v1
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v6, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX8-NEXT:    v_add_f32_e32 v1, v1, v3
-; GFX8-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
 ; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v1
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX8-NEXT:    v_mul_f32_e32 v3, v5, v3
-; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT:    v_add_f32_e32 v3, v3, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX8-NEXT:    v_fma_f32 v3, v6, v5, v3
 ; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
+; GFX8-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
 ; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX8-NEXT:    v_fma_f32 v0, v0, v2, v4
 ; GFX8-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_bfe_u32 v2, v0, 16, 1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v0
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s4, v2
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v5, vcc
-; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX8-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX8-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v0
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 0x7fff, v2
 ; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
@@ -47605,52 +47450,31 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
 ; GFX900-LABEL: v_fmuladd_v3bf16:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX900-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GFX900-NEXT:    v_fma_f32 v1, v1, v3, v5
 ; GFX900-NEXT:    v_bfe_u32 v3, v1, 16, 1
 ; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX900-NEXT:    v_add3_u32 v3, v3, v1, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v1
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v3, v6, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX900-NEXT:    v_add_f32_e32 v1, v1, v3
-; GFX900-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX900-NEXT:    v_add3_u32 v3, v3, v1, s4
 ; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v1
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX900-NEXT:    v_mul_f32_e32 v3, v5, v3
-; GFX900-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v5, v5, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
-; GFX900-NEXT:    v_add_f32_e32 v3, v3, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
+; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v0
+; GFX900-NEXT:    v_fma_f32 v3, v6, v5, v3
+; GFX900-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
 ; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX900-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-NEXT:    v_fma_f32 v0, v0, v2, v4
 ; GFX900-NEXT:    v_add3_u32 v5, v5, v3, s4
 ; GFX900-NEXT:    v_or_b32_e32 v6, 0x400000, v3
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_bfe_u32 v2, v0, 16, 1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v6, vcc
 ; GFX900-NEXT:    v_add3_u32 v2, v2, v0, s4
-; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v5, vcc
-; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX900-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX900-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX900-NEXT:    v_add3_u32 v2, v2, v0, s4
 ; GFX900-NEXT:    v_or_b32_e32 v4, 0x400000, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
@@ -47662,211 +47486,132 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
 ; GFX950-LABEL: v_fmuladd_v3bf16:
 ; GFX950:       ; %bb.0:
 ; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v1, s0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX950-NEXT:    v_add_f32_e32 v1, v1, v3
-; GFX950-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX950-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
+; GFX950-NEXT:    v_fmac_f32_e32 v5, v1, v3
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v5, s0
+; GFX950-NEXT:    v_and_b32_e32 v3, 0xffff0000, v4
+; GFX950-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
+; GFX950-NEXT:    v_and_b32_e32 v6, 0xffff0000, v0
+; GFX950-NEXT:    v_lshlrev_b32_e32 v4, 16, v4
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_mul_f32_e32 v3, v5, v3
-; GFX950-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_and_b32_e32 v5, 0xffff0000, v4
-; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
-; GFX950-NEXT:    v_add_f32_e32 v3, v3, v5
-; GFX950-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v1, s0
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v3
+; GFX950-NEXT:    v_fmac_f32_e32 v3, v6, v5
+; GFX950-NEXT:    v_fmac_f32_e32 v4, v0, v2
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v4, v3
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fmuladd_v3bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GFX10-NEXT:    v_mul_f32_e32 v3, v7, v6
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX10-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v6, 0x400000, v1
-; GFX10-NEXT:    v_bfe_u32 v7, v3, 16, 1
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT:    v_bfe_u32 v8, v0, 16, 1
-; GFX10-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v3
-; GFX10-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v10, 0x400000, v0
-; GFX10-NEXT:    v_add3_u32 v8, v8, v0, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v2, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX10-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
 ; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v7, v9, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT:    v_add_f32_e32 v1, v1, v3
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v8, v10, vcc_lo
-; GFX10-NEXT:    v_or_b32_e32 v6, 0x400000, v1
-; GFX10-NEXT:    v_add_f32_e32 v2, v2, v5
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX10-NEXT:    v_add_f32_e32 v0, v0, v4
-; GFX10-NEXT:    v_or_b32_e32 v7, 0x400000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX10-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX10-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v5, v0, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT:    v_add3_u32 v4, v4, v1, 0x7fff
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v3, v7, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v5, v5, v0, 0x7fff
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT:    v_perm_b32 v0, v0, v2, 0x7060302
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX10-NEXT:    v_alignbit_b32 v1, s4, v1, 16
+; GFX10-NEXT:    v_fmac_f32_e32 v6, v8, v7
+; GFX10-NEXT:    v_fmac_f32_e32 v5, v1, v3
+; GFX10-NEXT:    v_fmac_f32_e32 v4, v0, v2
+; GFX10-NEXT:    v_bfe_u32 v1, v6, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v3, 0x400000, v6
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT:    v_bfe_u32 v0, v5, 16, 1
+; GFX10-NEXT:    v_bfe_u32 v2, v4, 16, 1
+; GFX10-NEXT:    v_add3_u32 v1, v1, v6, 0x7fff
+; GFX10-NEXT:    v_or_b32_e32 v7, 0x400000, v4
+; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v5
+; GFX10-NEXT:    v_add3_u32 v0, v0, v5, 0x7fff
+; GFX10-NEXT:    v_add3_u32 v2, v2, v4, 0x7fff
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v0, v8, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v0, v2, v1, 0x7060302
+; GFX10-NEXT:    v_alignbit_b32 v1, s4, v3, 16
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11TRUE16-LABEL: v_fmuladd_v3bf16:
 ; GFX11TRUE16:       ; %bb.0:
 ; GFX11TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
-; GFX11TRUE16-NEXT:    v_dual_mul_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
 ; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT:    v_mul_f32_e32 v6, v7, v6
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_bfe_u32 v9, v1, 16, 1
-; GFX11TRUE16-NEXT:    v_bfe_u32 v7, v0, 16, 1
+; GFX11TRUE16-NEXT:    v_dual_fmac_f32 v4, v0, v2 :: v_dual_fmac_f32 v5, v1, v3
+; GFX11TRUE16-NEXT:    v_fmac_f32_e32 v6, v8, v7
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT:    v_bfe_u32 v1, v4, 16, 1
+; GFX11TRUE16-NEXT:    v_bfe_u32 v3, v5, 16, 1
 ; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_bfe_u32 v2, v6, 16, 1
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v6
+; GFX11TRUE16-NEXT:    v_bfe_u32 v0, v6, 16, 1
+; GFX11TRUE16-NEXT:    v_or_b32_e32 v2, 0x400000, v6
 ; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v0
-; GFX11TRUE16-NEXT:    v_add3_u32 v7, v7, v0, 0x7fff
-; GFX11TRUE16-NEXT:    v_add3_u32 v2, v2, v6, 0x7fff
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc_lo
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
-; GFX11TRUE16-NEXT:    v_add3_u32 v8, v9, v1, 0x7fff
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v7, v10, vcc_lo
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_add_f32_e32 v2, v2, v3
-; GFX11TRUE16-NEXT:    v_dual_cndmask_b32 v1, v8, v6 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11TRUE16-NEXT:    v_dual_add_f32 v0, v0, v4 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11TRUE16-NEXT:    v_bfe_u32 v4, v0, 16, 1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_add_f32_e32 v1, v1, v5
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v5, 0x400000, v2
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT:    v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v2, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v3, 0x400000, v1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT:    v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v4, v7, vcc_lo
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT:    v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11TRUE16-NEXT:    v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT:    v_add3_u32 v0, v0, v6, 0x7fff
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT:    v_add3_u32 v2, v3, v5, 0x7fff
+; GFX11TRUE16-NEXT:    v_or_b32_e32 v3, 0x400000, v5
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
+; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc_lo
+; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
 ; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v2, v0
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v1, v5, v3, vcc_lo
-; GFX11TRUE16-NEXT:    v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.h
 ; GFX11TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11FAKE16-LABEL: v_fmuladd_v3bf16:
 ; GFX11FAKE16:       ; %bb.0:
 ; GFX11FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v2
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11FAKE16-NEXT:    v_bfe_u32 v8, v0, 16, 1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v0
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GFX11FAKE16-NEXT:    v_mul_f32_e32 v3, v7, v6
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v6, 0x400000, v1
-; GFX11FAKE16-NEXT:    v_bfe_u32 v7, v3, 16, 1
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v3
-; GFX11FAKE16-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v1, v2, v6, vcc_lo
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v0
 ; GFX11FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v2, v7, v9, vcc_lo
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v0, v8, v10, vcc_lo
-; GFX11FAKE16-NEXT:    v_add_f32_e32 v2, v2, v5
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v7, 0x400000, v2
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v3
-; GFX11FAKE16-NEXT:    v_bfe_u32 v3, v2, 16, 1
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11FAKE16-NEXT:    v_bfe_u32 v5, v0, 16, 1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11FAKE16-NEXT:    v_bfe_u32 v4, v1, 16, 1
-; GFX11FAKE16-NEXT:    v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v0
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v6, 0x400000, v1
-; GFX11FAKE16-NEXT:    v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11FAKE16-NEXT:    v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v2, v3, v7, vcc_lo
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11FAKE16-NEXT:    v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_alignbit_b32 v1, s0, v1, 16
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11FAKE16-NEXT:    v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11FAKE16-NEXT:    v_fmac_f32_e32 v4, v0, v2
+; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11FAKE16-NEXT:    v_bfe_u32 v2, v4, 16, 1
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v7, 0x400000, v4
+; GFX11FAKE16-NEXT:    v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11FAKE16-NEXT:    v_fmac_f32_e32 v5, v1, v3
+; GFX11FAKE16-NEXT:    v_bfe_u32 v1, v6, 16, 1
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v3, 0x400000, v6
+; GFX11FAKE16-NEXT:    v_bfe_u32 v0, v5, 16, 1
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11FAKE16-NEXT:    v_add3_u32 v1, v1, v6, 0x7fff
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v5
+; GFX11FAKE16-NEXT:    v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v3, v0, v8, vcc_lo
+; GFX11FAKE16-NEXT:    v_perm_b32 v0, v2, v1, 0x7060302
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11FAKE16-NEXT:    v_alignbit_b32 v1, s0, v3, 16
 ; GFX11FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %op = call <3 x bfloat> @llvm.fmuladd.v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat> %c)
   ret <3 x bfloat> %op
@@ -47966,75 +47711,43 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
 ; GFX8-LABEL: v_fmuladd_v4bf16:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
-; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX8-NEXT:    v_mul_f32_e32 v6, v7, v6
-; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX8-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
-; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
-; GFX8-NEXT:    v_add_f32_e32 v6, v6, v7
+; GFX8-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX8-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX8-NEXT:    v_fma_f32 v6, v8, v7, v6
 ; GFX8-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX8-NEXT:    v_add_u32_e32 v7, vcc, v7, v6
+; GFX8-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
 ; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
 ; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT:    v_add_u32_e32 v7, vcc, s4, v7
-; GFX8-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GFX8-NEXT:    v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT:    v_fma_f32 v1, v1, v3, v5
 ; GFX8-NEXT:    v_or_b32_e32 v8, 0x400000, v6
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX8-NEXT:    v_bfe_u32 v3, v1, 16, 1
+; GFX8-NEXT:    s_movk_i32 s4, 0x7fff
 ; GFX8-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
 ; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v1
 ; GFX8-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v1
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v7, vcc
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
-; GFX8-NEXT:    v_add_f32_e32 v1, v1, v3
-; GFX8-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, v3, v1
-; GFX8-NEXT:    v_add_u32_e32 v3, vcc, s4, v3
 ; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v1
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc
-; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX8-NEXT:    v_mul_f32_e32 v3, v5, v3
-; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v7, vcc
-; GFX8-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT:    v_add_f32_e32 v3, v3, v5
+; GFX8-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; GFX8-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
+; GFX8-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
+; GFX8-NEXT:    v_fma_f32 v3, v7, v5, v3
 ; GFX8-NEXT:    v_bfe_u32 v5, v3, 16, 1
 ; GFX8-NEXT:    v_add_u32_e32 v5, vcc, v5, v3
+; GFX8-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
 ; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX8-NEXT:    v_add_u32_e32 v5, vcc, s4, v5
-; GFX8-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX8-NEXT:    v_fma_f32 v0, v0, v2, v4
 ; GFX8-NEXT:    v_or_b32_e32 v7, 0x400000, v3
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX8-NEXT:    v_bfe_u32 v2, v0, 16, 1
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v5, v7, vcc
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v0
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s4, v2
-; GFX8-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v5, vcc
-; GFX8-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX8-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX8-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX8-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v0
 ; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 0x7fff, v2
 ; GFX8-NEXT:    v_or_b32_e32 v4, 0x400000, v0
 ; GFX8-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
@@ -48048,68 +47761,40 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
 ; GFX900-LABEL: v_fmuladd_v4bf16:
 ; GFX900:       ; %bb.0:
 ; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
-; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX900-NEXT:    v_mul_f32_e32 v6, v7, v6
-; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
-; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
-; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
-; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX900-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
-; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v5
-; GFX900-NEXT:    v_add_f32_e32 v6, v6, v7
+; GFX900-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX900-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX900-NEXT:    v_fma_f32 v6, v8, v7, v6
+; GFX900-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
 ; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
 ; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX900-NEXT:    v_bfe_u32 v7, v6, 16, 1
-; GFX900-NEXT:    v_mul_f32_e32 v1, v1, v3
+; GFX900-NEXT:    s_movk_i32 s4, 0x7fff
+; GFX900-NEXT:    v_fma_f32 v1, v1, v3, v5
 ; GFX900-NEXT:    v_add3_u32 v7, v7, v6, s4
 ; GFX900-NEXT:    v_or_b32_e32 v8, 0x400000, v6
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v6, v6
 ; GFX900-NEXT:    v_bfe_u32 v3, v1, 16, 1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v6, v7, v8, vcc
 ; GFX900-NEXT:    v_add3_u32 v3, v3, v1, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v1
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
-; GFX900-NEXT:    v_cndmask_b32_e32 v1, v3, v7, vcc
-; GFX900-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v5
-; GFX900-NEXT:    v_add_f32_e32 v1, v1, v3
-; GFX900-NEXT:    v_bfe_u32 v3, v1, 16, 1
-; GFX900-NEXT:    v_add3_u32 v3, v3, v1, s4
 ; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v1
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v1, v1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v1, v3, v5, vcc
-; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v0
-; GFX900-NEXT:    v_mul_f32_e32 v3, v5, v3
-; GFX900-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-NEXT:    v_add3_u32 v5, v5, v3, s4
-; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v3
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
-; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v7, vcc
-; GFX900-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v4
-; GFX900-NEXT:    v_add_f32_e32 v3, v3, v5
+; GFX900-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
+; GFX900-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
+; GFX900-NEXT:    v_lshlrev_b32_e32 v7, 16, v0
+; GFX900-NEXT:    v_fma_f32 v3, v7, v5, v3
+; GFX900-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
 ; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
 ; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX900-NEXT:    v_bfe_u32 v5, v3, 16, 1
-; GFX900-NEXT:    v_mul_f32_e32 v0, v0, v2
+; GFX900-NEXT:    v_fma_f32 v0, v0, v2, v4
 ; GFX900-NEXT:    v_add3_u32 v5, v5, v3, s4
 ; GFX900-NEXT:    v_or_b32_e32 v7, 0x400000, v3
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v3, v3
 ; GFX900-NEXT:    v_bfe_u32 v2, v0, 16, 1
 ; GFX900-NEXT:    v_cndmask_b32_e32 v3, v5, v7, vcc
 ; GFX900-NEXT:    v_add3_u32 v2, v2, v0, s4
-; GFX900-NEXT:    v_or_b32_e32 v5, 0x400000, v0
-; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
-; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v5, vcc
-; GFX900-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX900-NEXT:    v_and_b32_e32 v2, 0xffff0000, v4
-; GFX900-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX900-NEXT:    v_bfe_u32 v2, v0, 16, 1
-; GFX900-NEXT:    v_add3_u32 v2, v2, v0, s4
 ; GFX900-NEXT:    v_or_b32_e32 v4, 0x400000, v0
 ; GFX900-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
 ; GFX900-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
@@ -48121,264 +47806,162 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
 ; GFX950-LABEL: v_fmuladd_v4bf16:
 ; GFX950:       ; %bb.0:
 ; GFX950-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT:    v_and_b32_e32 v6, 0xffff0000, v3
-; GFX950-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
+; GFX950-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
+; GFX950-NEXT:    v_and_b32_e32 v7, 0xffff0000, v3
+; GFX950-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
+; GFX950-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v1, s0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v5
-; GFX950-NEXT:    v_mul_f32_e32 v6, v7, v6
-; GFX950-NEXT:    v_and_b32_e32 v7, 0xffff0000, v5
-; GFX950-NEXT:    v_add_f32_e32 v1, v1, v3
+; GFX950-NEXT:    v_fmac_f32_e32 v6, v8, v7
+; GFX950-NEXT:    v_fmac_f32_e32 v5, v1, v3
+; GFX950-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
 ; GFX950-NEXT:    v_and_b32_e32 v3, 0xffff0000, v2
-; GFX950-NEXT:    v_and_b32_e32 v5, 0xffff0000, v0
+; GFX950-NEXT:    v_and_b32_e32 v7, 0xffff0000, v0
+; GFX950-NEXT:    v_fmac_f32_e32 v1, v7, v3
+; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v4
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
 ; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_mul_f32_e32 v3, v5, v3
-; GFX950-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v6, v6, s0
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v3, v3, s0
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v6, 16, v6
-; GFX950-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX950-NEXT:    v_and_b32_e32 v5, 0xffff0000, v4
-; GFX950-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX950-NEXT:    v_lshlrev_b32_e32 v2, 16, v4
-; GFX950-NEXT:    v_add_f32_e32 v6, v6, v7
-; GFX950-NEXT:    v_add_f32_e32 v3, v3, v5
-; GFX950-NEXT:    v_add_f32_e32 v0, v0, v2
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v3
-; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v1, v6
+; GFX950-NEXT:    v_fmac_f32_e32 v3, v0, v2
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v0, v3, v1
+; GFX950-NEXT:    v_cvt_pk_bf16_f32 v1, v5, v6
 ; GFX950-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_fmuladd_v4bf16:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
 ; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
 ; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT:    v_mul_f32_e32 v6, v7, v6
-; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
+; GFX10-NEXT:    v_fmac_f32_e32 v6, v8, v7
+; GFX10-NEXT:    v_lshlrev_b32_e32 v7, 16, v4
+; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
 ; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GFX10-NEXT:    v_lshlrev_b32_e32 v8, 16, v5
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
 ; GFX10-NEXT:    v_bfe_u32 v10, v6, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v3, 0x400000, v6
-; GFX10-NEXT:    v_mul_f32_e32 v7, v9, v7
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX10-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX10-NEXT:    v_add3_u32 v10, v10, v6, 0x7fff
+; GFX10-NEXT:    v_fmac_f32_e32 v5, v1, v3
+; GFX10-NEXT:    v_fmac_f32_e32 v7, v9, v8
+; GFX10-NEXT:    v_or_b32_e32 v1, 0x400000, v6
+; GFX10-NEXT:    v_fmac_f32_e32 v4, v0, v2
+; GFX10-NEXT:    v_add3_u32 v0, v10, v6, 0x7fff
+; GFX10-NEXT:    v_bfe_u32 v2, v5, 16, 1
+; GFX10-NEXT:    v_bfe_u32 v3, v7, 16, 1
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT:    v_or_b32_e32 v6, 0x400000, v1
-; GFX10-NEXT:    v_bfe_u32 v9, v7, 16, 1
-; GFX10-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v11, v0, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v10, v3, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT:    v_or_b32_e32 v10, 0x400000, v7
-; GFX10-NEXT:    v_add3_u32 v9, v9, v7, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v12, 0x400000, v0
-; GFX10-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX10-NEXT:    v_bfe_u32 v8, v4, 16, 1
+; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc_lo
+; GFX10-NEXT:    v_add3_u32 v0, v2, v5, 0x7fff
+; GFX10-NEXT:    v_add3_u32 v2, v3, v7, 0x7fff
+; GFX10-NEXT:    v_or_b32_e32 v3, 0x400000, v7
 ; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT:    v_add3_u32 v11, v11, v0, 0x7fff
-; GFX10-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
-; GFX10-NEXT:    v_add_f32_e32 v3, v3, v8
-; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v9, v10, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT:    v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX10-NEXT:    v_bfe_u32 v7, v3, 16, 1
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v11, v12, vcc_lo
-; GFX10-NEXT:    v_add_f32_e32 v1, v1, v5
-; GFX10-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT:    v_add_f32_e32 v2, v2, v6
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX10-NEXT:    v_bfe_u32 v6, v1, 16, 1
-; GFX10-NEXT:    v_or_b32_e32 v9, 0x400000, v1
-; GFX10-NEXT:    v_add_f32_e32 v0, v0, v4
-; GFX10-NEXT:    v_add3_u32 v4, v7, v3, 0x7fff
-; GFX10-NEXT:    v_bfe_u32 v7, v2, 16, 1
-; GFX10-NEXT:    v_bfe_u32 v8, v0, 16, 1
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc_lo
-; GFX10-NEXT:    v_add3_u32 v4, v6, v1, 0x7fff
-; GFX10-NEXT:    v_add3_u32 v5, v7, v2, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v6, 0x400000, v2
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX10-NEXT:    v_add3_u32 v7, v8, v0, 0x7fff
-; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v7, v8, vcc_lo
-; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT:    v_perm_b32 v0, v0, v2, 0x7060302
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v4, v9, vcc_lo
-; GFX10-NEXT:    v_perm_b32 v1, v1, v3, 0x7060302
+; GFX10-NEXT:    v_add3_u32 v6, v8, v4, 0x7fff
+; GFX10-NEXT:    v_or_b32_e32 v8, 0x400000, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v0, v9, vcc_lo
+; GFX10-NEXT:    v_perm_b32 v0, v3, v2, 0x7060302
+; GFX10-NEXT:    v_perm_b32 v1, v4, v1, 0x7060302
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11TRUE16-LABEL: v_fmuladd_v4bf16:
 ; GFX11TRUE16:       ; %bb.0:
 ; GFX11TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff0000, v0
-; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_dual_mul_f32 v6, v7, v6 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v3
+; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v10, 16, v0
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v1
 ; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff0000, v5
-; GFX11TRUE16-NEXT:    v_bfe_u32 v10, v6, 16, 1
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v5
 ; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_mul_f32_e32 v1, v1, v3
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11TRUE16-NEXT:    v_dual_mul_f32 v3, v9, v7 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v7, 0x400000, v6
-; GFX11TRUE16-NEXT:    v_add3_u32 v9, v10, v6, 0x7fff
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_mul_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v1
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v6, v9, v7, vcc_lo
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v11, 0x400000, v0
-; GFX11TRUE16-NEXT:    v_bfe_u32 v7, v3, 16, 1
-; GFX11TRUE16-NEXT:    v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v1, v2, v10, vcc_lo
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11TRUE16-NEXT:    v_dual_cndmask_b32 v0, v9, v11 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff0000, v6
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_dual_add_f32 v1, v1, v5 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT:    v_add_f32_e32 v2, v6, v8
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v6, 0x400000, v3
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT:    v_bfe_u32 v5, v1, 16, 1
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT:    v_fmac_f32_e32 v5, v1, v3
+; GFX11TRUE16-NEXT:    v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v7, 16, v4
+; GFX11TRUE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v4
+; GFX11TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11TRUE16-NEXT:    v_bfe_u32 v3, v5, 16, 1
+; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT:    v_fmac_f32_e32 v7, v10, v8
+; GFX11TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v5
+; GFX11TRUE16-NEXT:    v_fmac_f32_e32 v1, v0, v2
+; GFX11TRUE16-NEXT:    v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11TRUE16-NEXT:    v_bfe_u32 v9, v6, 16, 1
+; GFX11TRUE16-NEXT:    v_bfe_u32 v0, v7, 16, 1
+; GFX11TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v7
+; GFX11TRUE16-NEXT:    v_or_b32_e32 v2, 0x400000, v6
+; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v3, v3, v8, vcc_lo
+; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT:    v_add3_u32 v0, v0, v7, 0x7fff
+; GFX11TRUE16-NEXT:    v_add3_u32 v4, v9, v6, 0x7fff
+; GFX11TRUE16-NEXT:    v_bfe_u32 v9, v1, 16, 1
 ; GFX11TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT:    v_dual_cndmask_b32 v3, v7, v6 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11TRUE16-NEXT:    v_bfe_u32 v7, v2, 16, 1
-; GFX11TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT:    v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_dual_add_f32 v0, v0, v6 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11TRUE16-NEXT:    v_add3_u32 v6, v7, v2, 0x7fff
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v7, 0x400000, v2
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v3.l, v3.h
+; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
+; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT:    v_add3_u32 v5, v9, v1, 0x7fff
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
+; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc_lo
 ; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_add_f32_e32 v3, v3, v4
-; GFX11TRUE16-NEXT:    v_bfe_u32 v4, v0, 16, 1
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v10, 0x400000, v0
 ; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT:    v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT:    v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v0, v4, v10, vcc_lo
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT:    v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v2, v6, v7, vcc_lo
-; GFX11TRUE16-NEXT:    v_bfe_u32 v9, v3, 16, 1
-; GFX11TRUE16-NEXT:    v_or_b32_e32 v8, 0x400000, v3
-; GFX11TRUE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT:    v_bfi_b32 v1, 0xffff, v1, v2
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_add3_u32 v5, v9, v3, 0x7fff
-; GFX11TRUE16-NEXT:    v_cndmask_b32_e32 v3, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v0, v3
+; GFX11TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT:    v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT:    v_bfi_b32 v1, 0xffff, v3, v2
 ; GFX11TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11FAKE16-LABEL: v_fmuladd_v4bf16:
 ; GFX11FAKE16:       ; %bb.0:
 ; GFX11FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v1
+; GFX11FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v9, 16, v0
 ; GFX11FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v1
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v5
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v3
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v3
 ; GFX11FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_dual_mul_f32 v6, v7, v6 :: v_dual_and_b32 v5, 0xffff0000, v5
-; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v7, 16, v2
-; GFX11FAKE16-NEXT:    v_dual_mul_f32 v1, v1, v3 :: v_dual_and_b32 v2, 0xffff0000, v2
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v6, 16, v5
+; GFX11FAKE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v5
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11FAKE16-NEXT:    v_fmac_f32_e32 v5, v1, v3
+; GFX11FAKE16-NEXT:    v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v7, 16, v4
+; GFX11FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
 ; GFX11FAKE16-NEXT:    v_bfe_u32 v10, v6, 16, 1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v3, 0x400000, v6
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v1, 0x400000, v6
 ; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11FAKE16-NEXT:    v_mul_f32_e32 v7, v9, v7
-; GFX11FAKE16-NEXT:    v_add3_u32 v10, v10, v6, 0x7fff
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v6, 0x400000, v1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_bfe_u32 v9, v7, 16, 1
-; GFX11FAKE16-NEXT:    v_dual_cndmask_b32 v3, v10, v3 :: v_dual_mul_f32 v0, v0, v2
-; GFX11FAKE16-NEXT:    v_bfe_u32 v2, v1, 16, 1
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v10, 0x400000, v7
-; GFX11FAKE16-NEXT:    v_add3_u32 v9, v9, v7, 0x7fff
-; GFX11FAKE16-NEXT:    v_bfe_u32 v11, v0, 16, 1
-; GFX11FAKE16-NEXT:    v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v12, 0x400000, v0
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_add3_u32 v11, v11, v0, 0x7fff
-; GFX11FAKE16-NEXT:    v_dual_cndmask_b32 v1, v2, v6 :: v_dual_lshlrev_b32 v6, 16, v4
+; GFX11FAKE16-NEXT:    v_lshlrev_b32_e32 v8, 16, v2
+; GFX11FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11FAKE16-NEXT:    v_fmac_f32_e32 v4, v0, v2
+; GFX11FAKE16-NEXT:    v_add3_u32 v0, v10, v6, 0x7fff
+; GFX11FAKE16-NEXT:    v_bfe_u32 v2, v5, 16, 1
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v1, v0, v1, vcc_lo
+; GFX11FAKE16-NEXT:    v_fmac_f32_e32 v7, v9, v8
+; GFX11FAKE16-NEXT:    v_bfe_u32 v8, v4, 16, 1
+; GFX11FAKE16-NEXT:    v_add3_u32 v0, v2, v5, 0x7fff
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v5
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11FAKE16-NEXT:    v_bfe_u32 v3, v7, 16, 1
 ; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT:    v_dual_cndmask_b32 v2, v9, v10 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT:    v_dual_add_f32 v1, v1, v5 :: v_dual_and_b32 v2, 0xffff0000, v2
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v0, v11, v12, vcc_lo
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v9, 0x400000, v1
-; GFX11FAKE16-NEXT:    v_add_f32_e32 v2, v2, v6
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11FAKE16-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11FAKE16-NEXT:    v_bfe_u32 v6, v1, 16, 1
-; GFX11FAKE16-NEXT:    v_add_f32_e32 v0, v0, v4
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT:    v_add_f32_e32 v3, v3, v8
-; GFX11FAKE16-NEXT:    v_bfe_u32 v8, v0, 16, 1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_bfe_u32 v7, v3, 16, 1
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v5, 0x400000, v3
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11FAKE16-NEXT:    v_add3_u32 v4, v7, v3, 0x7fff
-; GFX11FAKE16-NEXT:    v_bfe_u32 v7, v2, 16, 1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc_lo
-; GFX11FAKE16-NEXT:    v_add3_u32 v4, v6, v1, 0x7fff
-; GFX11FAKE16-NEXT:    v_add3_u32 v5, v7, v2, 0x7fff
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v6, 0x400000, v2
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11FAKE16-NEXT:    v_add3_u32 v7, v8, v0, 0x7fff
-; GFX11FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v0
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v2, v5, v6, vcc_lo
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v0, v7, v8, vcc_lo
-; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT:    v_perm_b32 v0, v0, v2, 0x7060302
-; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v1, v4, v9, vcc_lo
-; GFX11FAKE16-NEXT:    v_perm_b32 v1, v1, v3, 0x7060302
+; GFX11FAKE16-NEXT:    v_add3_u32 v6, v8, v4, 0x7fff
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v8, 0x400000, v4
+; GFX11FAKE16-NEXT:    v_add3_u32 v2, v3, v7, 0x7fff
+; GFX11FAKE16-NEXT:    v_or_b32_e32 v3, 0x400000, v7
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v2, v2, v3, vcc_lo
+; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11FAKE16-NEXT:    v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11FAKE16-NEXT:    v_cndmask_b32_e32 v4, v0, v9, vcc_lo
+; GFX11FAKE16-NEXT:    v_perm_b32 v0, v3, v2, 0x7060302
+; GFX11FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11FAKE16-NEXT:    v_perm_b32 v1, v4, v1, 0x7060302
 ; GFX11FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %op = call <4 x bfloat> @llvm.fmuladd.v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat> %c)
   ret <4 x bfloat> %op
diff --git a/llvm/test/CodeGen/AMDGPU/bitop3.ll b/llvm/test/CodeGen/AMDGPU/bitop3.ll
index ba818f6ecc069..187f19f653858 100644
--- a/llvm/test/CodeGen/AMDGPU/bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitop3.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-- -mcpu=gfx950 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-SDAG %s
-; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=gfx950 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-- -mcpu=gfx950 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX950,GFX950-GISEL %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-- -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-SDAG,GFX1250-SDAG-FAKE16,GFX1250-FAKE16 %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-- -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-SDAG,GFX1250-SDAG-TRUE16,GFX1250-TRUE16 %s
-; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16,GFX1250-FAKE16 %s
-; RUN: llc -global-isel -mtriple=amdgcn-- -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-TRUE16,GFX1250-TRUE16 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-- -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16,GFX1250-FAKE16 %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-- -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX1250,GFX1250-GISEL,GFX1250-GISEL-TRUE16,GFX1250-TRUE16 %s
 
 ; ========= Single bit functions =========
 
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index f4b432dce8c8a..0ceb9019eb990 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -3443,15 +3443,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v1.l, v1.l, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffe
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
 ; GFX12-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
@@ -3569,14 +3568,13 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v1.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
 ; GFX11-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
@@ -3884,15 +3882,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v1.l, v1.l, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffe
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
 ; GFX12-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
@@ -4007,14 +4004,13 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v1.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
 ; GFX11-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
@@ -4328,15 +4324,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
 ; GFX12-TRUE16-NEXT:    ; Child Loop BB15_4 Depth 2
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, v4, v7
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v6.h, 0
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v6.l, v6.l, v5.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, v4, v6
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v6, v7, v11, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
 ; GFX12-TRUE16-NEXT:  .LBB15_4: ; Parent Loop BB15_3 Depth=1
 ; GFX12-TRUE16-NEXT:    ; => This Inner Loop Header: Depth=2
@@ -4556,15 +4551,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
 ; GFX11-TRUE16-NEXT:    ; Child Loop BB15_4 Depth 2
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, v4, v7
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.h, 0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v6.l, v6.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v6, v4, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v6, v7, v11, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
 ; GFX11-TRUE16-NEXT:  .LBB15_4: ; Parent Loop BB15_3 Depth=1
 ; GFX11-TRUE16-NEXT:    ; => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 6f1675edbe58a..cad4c39eaf39f 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -2512,16 +2512,16 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v1.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v1.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffe
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
 ; GFX12-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
@@ -2640,20 +2640,19 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX11-TRUE16-NEXT:    buffer_load_b32 v2, v5, s[0:3], 0 offen
 ; GFX11-TRUE16-NEXT:    s_not_b32 s6, s5
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v1.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v1.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
 ; GFX11-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
@@ -2973,16 +2972,16 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v1.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v1.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffe
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
 ; GFX12-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
@@ -3098,20 +3097,19 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
 ; GFX11-TRUE16-NEXT:    buffer_load_b32 v2, v3, s[0:3], 0 offen
 ; GFX11-TRUE16-NEXT:    s_not_b32 s6, s5
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB11_1: ; %atomicrmw.start
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v1.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v1.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
 ; GFX11-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
@@ -3437,16 +3435,16 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
 ; GFX12-TRUE16-NEXT:    ; Child Loop BB12_4 Depth 2
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v9, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v4.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v4.h, v4.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v9, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v11, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
 ; GFX12-TRUE16-NEXT:  .LBB12_4: ; Parent Loop BB12_3 Depth=1
 ; GFX12-TRUE16-NEXT:    ; => This Inner Loop Header: Depth=2
@@ -3672,16 +3670,16 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
 ; GFX11-TRUE16-NEXT:    ; Child Loop BB12_4 Depth 2
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v9, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v4.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v4.h, v4.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v9, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v11, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
 ; GFX11-TRUE16-NEXT:  .LBB12_4: ; Parent Loop BB12_3 Depth=1
 ; GFX11-TRUE16-NEXT:    ; => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index acb27be1846b9..6275afd2c6994 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -2512,16 +2512,16 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v1.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v1.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffe
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
 ; GFX12-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
@@ -2640,20 +2640,19 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
 ; GFX11-TRUE16-NEXT:    buffer_load_b32 v2, v5, s[0:3], 0 offen
 ; GFX11-TRUE16-NEXT:    s_not_b32 s6, s5
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v1.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v1.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
 ; GFX11-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
@@ -2973,16 +2972,16 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v1.l, v1.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v1.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
 ; GFX12-TRUE16-NEXT:    s_wait_alu 0xfffe
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
 ; GFX12-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
@@ -3098,20 +3097,19 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
 ; GFX11-TRUE16-NEXT:    buffer_load_b32 v2, v3, s[0:3], 0 offen
 ; GFX11-TRUE16-NEXT:    s_not_b32 s6, s5
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB11_1: ; %atomicrmw.start
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, s4, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v1.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v1.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, s4, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
 ; GFX11-TRUE16-NEXT:    buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
@@ -3437,16 +3435,16 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
 ; GFX12-TRUE16-NEXT:    ; Child Loop BB12_4 Depth 2
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v9, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v4.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v4.h, v4.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v9, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v11, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
 ; GFX12-TRUE16-NEXT:  .LBB12_4: ; Parent Loop BB12_3 Depth=1
 ; GFX12-TRUE16-NEXT:    ; => This Inner Loop Header: Depth=2
@@ -3672,16 +3670,16 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
 ; GFX11-TRUE16-NEXT:    ; Child Loop BB12_4 Depth 2
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v9, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, exec_lo
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v4.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v4.h, v4.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v9, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v11, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
 ; GFX11-TRUE16-NEXT:  .LBB12_4: ; Parent Loop BB12_3 Depth=1
 ; GFX11-TRUE16-NEXT:    ; => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/AMDGPU/bundle-breaks-phy-liveness.mir b/llvm/test/CodeGen/AMDGPU/bundle-breaks-phy-liveness.mir
new file mode 100644
index 0000000000000..5c7dc8a079345
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/bundle-breaks-phy-liveness.mir
@@ -0,0 +1,36 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=si-insert-hard-clauses -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN-CLAUSE %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -run-pass=si-post-ra-bundler -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN-BUNDLE %s
+
+---
+name: clause_implicit_def
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $sgpr32
+
+    ; GCN-CLAUSE-LABEL: name: clause_implicit_def
+    ; GCN-CLAUSE: liveins: $vgpr0, $sgpr32
+    ; GCN-CLAUSE-NEXT: {{  $}}
+    ; GCN-CLAUSE-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit $sgpr32, implicit $exec, implicit $flat_scr {
+    ; GCN-CLAUSE-NEXT:   S_CLAUSE 1
+    ; GCN-CLAUSE-NEXT:   SCRATCH_STORE_DWORD_SADDR $vgpr0, $sgpr32, 8, 0, implicit $exec, implicit $flat_scr
+    ; GCN-CLAUSE-NEXT:   $vgpr2 = IMPLICIT_DEF
+    ; GCN-CLAUSE-NEXT:   $vgpr3 = IMPLICIT_DEF
+    ; GCN-CLAUSE-NEXT:   SCRATCH_STORE_DWORDX2_SADDR internal $vgpr2_vgpr3, $sgpr32, 16, 0, implicit $exec, implicit $flat_scr
+    ; GCN-CLAUSE-NEXT: }
+    ;
+    ; GCN-BUNDLE-LABEL: name: clause_implicit_def
+    ; GCN-BUNDLE: liveins: $vgpr0, $sgpr32
+    ; GCN-BUNDLE-NEXT: {{  $}}
+    ; GCN-BUNDLE-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit $sgpr32, implicit $exec, implicit $flat_scr {
+    ; GCN-BUNDLE-NEXT:   SCRATCH_STORE_DWORD_SADDR $vgpr0, $sgpr32, 8, 0, implicit $exec, implicit $flat_scr
+    ; GCN-BUNDLE-NEXT:   $vgpr2 = IMPLICIT_DEF
+    ; GCN-BUNDLE-NEXT:   $vgpr3 = IMPLICIT_DEF
+    ; GCN-BUNDLE-NEXT:   SCRATCH_STORE_DWORDX2_SADDR internal $vgpr2_vgpr3, $sgpr32, 16, 0, implicit $exec, implicit $flat_scr
+    ; GCN-BUNDLE-NEXT: }
+    SCRATCH_STORE_DWORD_SADDR $vgpr0, $sgpr32, 8, 0, implicit $exec, implicit $flat_scr
+    $vgpr2 = IMPLICIT_DEF
+    $vgpr3 = IMPLICIT_DEF
+    SCRATCH_STORE_DWORDX2_SADDR $vgpr2_vgpr3, $sgpr32, 16, 0, implicit $exec, implicit $flat_scr
+...
diff --git a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
index ff80250bfc880..2db7b28c7de97 100644
--- a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
+++ b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
@@ -2745,6 +2745,15 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
 ;
 ; GFX11-TRUE16-LABEL: amdgpu_cs_v32i1:
 ; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    v_and_b16 v26.l, v26.l, 1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 1, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, v24.l, 1
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.h, v22.l, 1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 1, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, v20.l, 1
+; GFX11-TRUE16-NEXT:    v_and_b16 v17.h, v18.l, 1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 1, v17.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, v16.l, 1
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, v10.l, 1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 1, v9.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, v8.l, 1
@@ -2754,6 +2763,18 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, v2.l, 1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 1, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, v0.l, 1
+; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, v30.l, 1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 1, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, v28.l, 1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 3, v27.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v26.l, 2, v26.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v24.l, v25.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 3, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v20.h, 2, v20.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v20.l, v21.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.h, 3, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 2, v17.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.l, v16.l, v17.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v14.l, v14.l, 1
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 1, v13.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, v12.l, 1
@@ -2766,15 +2787,15 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 3, v3.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 2, v1.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v25.h, v26.l, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.l, 1, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, v24.l, 1
-; GFX11-TRUE16-NEXT:    v_and_b16 v22.l, v22.l, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v21.l, 1, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v20.l, v20.l, 1
-; GFX11-TRUE16-NEXT:    v_and_b16 v18.l, v18.l, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 1, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, v16.l, 1
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 3, v31.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.h, 2, v30.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v28.l, v28.l, v29.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v25.h, v25.h, v26.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v21.h, v22.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b16 v18.l, v22.l, v20.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.h, v16.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b16 v17.h, v18.h, v17.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v16.l, v16.l, 3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 3, v15.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.h, 2, v14.l
 ; GFX11-TRUE16-NEXT:    v_or_b16 v8.h, v12.l, v13.l
@@ -2784,65 +2805,42 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, v0.h, 3
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v2.h, v1.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, v0.l, 3
-; GFX11-TRUE16-NEXT:    v_and_b16 v30.l, v30.l, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v29.l, 1, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v28.l, v28.l, 1
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.h, 3, v27.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v25.h, 2, v25.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.l, v24.l, v25.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v23.l, 3, v23.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v22.l, 2, v22.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v20.l, v21.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.h, 3, v19.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v18.l, 2, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v16.l, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v26.h, v28.h, v29.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, v28.l, 3
+; GFX11-TRUE16-NEXT:    v_or_b16 v19.l, v21.h, v25.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v16.h, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.h, v16.l, v17.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v12.h, v10.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v3.l, v8.h, 3
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.l, v6.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v0.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v30.h, 3, v31.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v28.h, 2, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v28.l, v29.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v22.h, v22.h, v25.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.l, v24.l, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v18.h, v23.l, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, v16.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v16.h, v17.h, v18.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, v15.h, 3
+; GFX11-TRUE16-NEXT:    v_or_b16 v24.h, v24.h, v26.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v14.h, v19.l, 15
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v16.h, 4, v16.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, v15.h, 15
 ; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v3.l, v2.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, v1.l, 15
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 4, v0.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, v0.l, 15
-; GFX11-TRUE16-NEXT:    v_or_b16 v28.h, v30.h, v28.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v24.h, v24.h, 3
-; GFX11-TRUE16-NEXT:    v_or_b16 v20.h, v24.l, v22.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v14.h, v18.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v17.l, 12, v24.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v14.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v16.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 12, v2.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v23.h, v24.h, v28.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, v20.h, 15
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 4, v2.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, v1.h, 15
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v17.l, v2.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v2.l, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 12, v23.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.h, v1.h, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, 0
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.h, v0.h
 ; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v2.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX11-TRUE16-NEXT:    global_store_b32 v[0:1], v0, off
 ; GFX11-TRUE16-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
index b9caf8e80bcdf..ccdc0b1bf43c4 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
+++ b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
@@ -1561,10 +1561,10 @@ define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f16_src(ptr addrspace(1) %ou
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v0.l, 1.0, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_pk_max_f16 v0, v0, v0 clamp
 ; GFX11-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-TRUE16-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index b5bc09a1684ee..26f204f29f5a4 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -946,9 +946,9 @@ define double @v_uitofp_i8_to_f64(i8 %arg0) nounwind {
 ; GFX11-TRUE16-LABEL: v_uitofp_i8_to_f64:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1770,40 +1770,38 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o
 ; GFX11-TRUE16-LABEL: load_v4i8_to_v4f32_2_uses:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.h
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    global_load_b32 v4, v0, s[0:1]
 ; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v4.l, 9
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff00, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff00, v4.h
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.h, 9
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff00, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff00, v4.h
 ; GFX11-TRUE16-NEXT:    v_cvt_f32_ubyte3_e32 v3, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.h, v4.h, 9
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0x900, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.l, v0.h
+; GFX11-TRUE16-NEXT:    v_cvt_f32_ubyte2_e32 v2, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.h, v0.h
 ; GFX11-TRUE16-NEXT:    v_cvt_f32_ubyte1_e32 v1, v4
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_add_nc_u16 v6.h, 0x900, v0.l
-; GFX11-TRUE16-NEXT:    v_cvt_f32_ubyte2_e32 v2, v4
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v5.l, 0x900, v0.l
+; GFX11-TRUE16-NEXT:    v_add_nc_u16 v7.h, 0x900, v0.h
 ; GFX11-TRUE16-NEXT:    v_cvt_f32_ubyte0_e32 v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v7, v6
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v5, v7
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
-; GFX11-TRUE16-NEXT:    global_store_b128 v5, v[0:3], s[0:1]
-; GFX11-TRUE16-NEXT:    global_store_b32 v5, v4, s[2:3]
+; GFX11-TRUE16-NEXT:    global_store_b128 v6, v[0:3], s[0:1]
+; GFX11-TRUE16-NEXT:    global_store_b32 v6, v4, s[2:3]
 ; GFX11-TRUE16-NEXT:    s_endpgm
 ;
 ; GFX11-FAKE16-LABEL: load_v4i8_to_v4f32_2_uses:
diff --git a/llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll b/llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll
index 9a98a7cd01ed4..12de3750640db 100644
--- a/llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll
+++ b/llvm/test/CodeGen/AMDGPU/dag-divergence-atomic.ll
@@ -42,7 +42,7 @@ define protected amdgpu_kernel void @sub(ptr addrspace(1) %p, ptr addrspace(1) %
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw sub ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw sub ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -64,7 +64,7 @@ define protected amdgpu_kernel void @and(ptr addrspace(1) %p, ptr addrspace(1) %
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw and ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw and ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -86,7 +86,7 @@ define protected amdgpu_kernel void @or(ptr addrspace(1) %p, ptr addrspace(1) %q
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw or ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw or ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -108,7 +108,7 @@ define protected amdgpu_kernel void @xor(ptr addrspace(1) %p, ptr addrspace(1) %
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw xor ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw xor ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -144,7 +144,7 @@ define protected amdgpu_kernel void @nand(ptr addrspace(1) %p, ptr addrspace(1)
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw nand ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw nand ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -166,7 +166,7 @@ define protected amdgpu_kernel void @max_workgroup(ptr addrspace(1) %p, ptr addr
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw max ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic
+  %n32 = atomicrmw max ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -188,7 +188,7 @@ define protected amdgpu_kernel void @max(ptr addrspace(1) %p, ptr addrspace(1) %
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw max ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw max ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -210,7 +210,7 @@ define protected amdgpu_kernel void @min_workgroup(ptr addrspace(1) %p, ptr addr
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw min ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic
+  %n32 = atomicrmw min ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -232,7 +232,7 @@ define protected amdgpu_kernel void @min(ptr addrspace(1) %p, ptr addrspace(1) %
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw min ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw min ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -254,7 +254,7 @@ define protected amdgpu_kernel void @umax_workgroup(ptr addrspace(1) %p, ptr add
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw umax ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic
+  %n32 = atomicrmw umax ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -276,7 +276,7 @@ define protected amdgpu_kernel void @umax(ptr addrspace(1) %p, ptr addrspace(1)
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw umax ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw umax ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -298,7 +298,7 @@ define protected amdgpu_kernel void @umin_workgroup(ptr addrspace(1) %p, ptr add
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw umin ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic
+  %n32 = atomicrmw umin ptr addrspace(1) %p, i32 1 syncscope("workgroup") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -320,7 +320,7 @@ define protected amdgpu_kernel void @umin(ptr addrspace(1) %p, ptr addrspace(1)
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw umin ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw umin ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -388,7 +388,7 @@ define protected amdgpu_kernel void @inc(ptr addrspace(1) %p, ptr addrspace(1) %
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw uinc_wrap ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw uinc_wrap ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -410,7 +410,7 @@ define protected amdgpu_kernel void @dec(ptr addrspace(1) %p, ptr addrspace(1) %
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %n32 = atomicrmw udec_wrap ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic
+  %n32 = atomicrmw udec_wrap ptr addrspace(1) %p, i32 1 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
   store float 1.0, ptr addrspace(1) %p1
@@ -446,7 +446,7 @@ define protected amdgpu_kernel void @fadd(ptr addrspace(1) %p, ptr addrspace(1)
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %f32 = atomicrmw fadd ptr addrspace(1) %p, float 1.0 syncscope("agent") monotonic
+  %f32 = atomicrmw fadd ptr addrspace(1) %p, float 1.0 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n32 = fptoui float %f32 to i32
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
@@ -483,7 +483,7 @@ define protected amdgpu_kernel void @fsub(ptr addrspace(1) %p, ptr addrspace(1)
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 1.0
 ; CHECK-NEXT:    global_store_dword v[0:1], v2, off
 ; CHECK-NEXT:    s_endpgm
-  %f32 = atomicrmw fsub ptr addrspace(1) %p, float 1.0 syncscope("agent") monotonic
+  %f32 = atomicrmw fsub ptr addrspace(1) %p, float 1.0 syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %n32 = fptoui float %f32 to i32
   %n64 = zext i32 %n32 to i64
   %p1 = getelementptr inbounds %S, ptr addrspace(1) %q, i64 %n64, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
index 85180a2dc6348..c429b1a32bde6 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX942 %s
 
 define amdgpu_kernel void @select_and1(ptr addrspace(1) %p, i32 %x, i32 %y) {
 ; GCN-LABEL: select_and1:
@@ -56,24 +57,43 @@ define amdgpu_kernel void @select_and3(ptr addrspace(1) %p, i32 %x, i32 %y) {
 }
 
 define amdgpu_kernel void @select_and_v4(ptr addrspace(1) %p, i32 %x, <4 x i32> %y) {
-; GCN-LABEL: select_and_v4:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dword s8, s[4:5], 0x2c
-; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GCN-NEXT:    v_mov_b32_e32 v4, 0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_cmp_gt_i32 s8, 10
-; GCN-NEXT:    s_cselect_b32 s3, s3, 0
-; GCN-NEXT:    s_cselect_b32 s2, s2, 0
-; GCN-NEXT:    s_cselect_b32 s1, s1, 0
-; GCN-NEXT:    s_cselect_b32 s0, s0, 0
-; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    v_mov_b32_e32 v1, s1
-; GCN-NEXT:    v_mov_b32_e32 v2, s2
-; GCN-NEXT:    v_mov_b32_e32 v3, s3
-; GCN-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
-; GCN-NEXT:    s_endpgm
+; GFX9-LABEL: select_and_v4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_cmp_gt_i32 s8, 10
+; GFX9-NEXT:    s_cselect_b32 s3, s3, 0
+; GFX9-NEXT:    s_cselect_b32 s2, s2, 0
+; GFX9-NEXT:    s_cselect_b32 s1, s1, 0
+; GFX9-NEXT:    s_cselect_b32 s0, s0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX942-LABEL: select_and_v4:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_gt_i32 s8, 10
+; GFX942-NEXT:    s_cselect_b32 s3, s3, 0
+; GFX942-NEXT:    s_cselect_b32 s2, s2, 0
+; GFX942-NEXT:    s_cselect_b32 s1, s1, 0
+; GFX942-NEXT:    s_cselect_b32 s0, s0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v2, s0
+; GFX942-NEXT:    v_mov_b32_e32 v3, s1
+; GFX942-NEXT:    v_mov_b32_e32 v4, s2
+; GFX942-NEXT:    v_mov_b32_e32 v5, s3
+; GFX942-NEXT:    global_store_dwordx4 v0, v[2:5], s[6:7]
+; GFX942-NEXT:    s_endpgm
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, <4 x i32> zeroinitializer, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = and <4 x i32> %s, %y
@@ -136,24 +156,43 @@ define amdgpu_kernel void @select_or3(ptr addrspace(1) %p, i32 %x, i32 %y) {
 }
 
 define amdgpu_kernel void @select_or_v4(ptr addrspace(1) %p, i32 %x, <4 x i32> %y) {
-; GCN-LABEL: select_or_v4:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dword s8, s[4:5], 0x2c
-; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GCN-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GCN-NEXT:    v_mov_b32_e32 v4, 0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_cmp_lt_i32 s8, 11
-; GCN-NEXT:    s_cselect_b32 s3, s3, -1
-; GCN-NEXT:    s_cselect_b32 s2, s2, -1
-; GCN-NEXT:    s_cselect_b32 s1, s1, -1
-; GCN-NEXT:    s_cselect_b32 s0, s0, -1
-; GCN-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-NEXT:    v_mov_b32_e32 v1, s1
-; GCN-NEXT:    v_mov_b32_e32 v2, s2
-; GCN-NEXT:    v_mov_b32_e32 v3, s3
-; GCN-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
-; GCN-NEXT:    s_endpgm
+; GFX9-LABEL: select_or_v4:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s8, s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GFX9-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_cmp_lt_i32 s8, 11
+; GFX9-NEXT:    s_cselect_b32 s3, s3, -1
+; GFX9-NEXT:    s_cselect_b32 s2, s2, -1
+; GFX9-NEXT:    s_cselect_b32 s1, s1, -1
+; GFX9-NEXT:    s_cselect_b32 s0, s0, -1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX942-LABEL: select_or_v4:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s8, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x34
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_lt_i32 s8, 11
+; GFX942-NEXT:    s_cselect_b32 s3, s3, -1
+; GFX942-NEXT:    s_cselect_b32 s2, s2, -1
+; GFX942-NEXT:    s_cselect_b32 s1, s1, -1
+; GFX942-NEXT:    s_cselect_b32 s0, s0, -1
+; GFX942-NEXT:    v_mov_b32_e32 v2, s0
+; GFX942-NEXT:    v_mov_b32_e32 v3, s1
+; GFX942-NEXT:    v_mov_b32_e32 v4, s2
+; GFX942-NEXT:    v_mov_b32_e32 v5, s3
+; GFX942-NEXT:    global_store_dwordx4 v0, v[2:5], s[6:7]
+; GFX942-NEXT:    s_endpgm
   %c = icmp slt i32 %x, 11
   %s = select i1 %c, <4 x i32> zeroinitializer, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
   %a = or <4 x i32> %s, %y
@@ -236,23 +275,41 @@ define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v2i16(ptr ad
 }
 
 define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v4i32(ptr addrspace(1) %p, i1 %cond) {
-; GCN-LABEL: sel_constants_sub_constant_sel_constants_v4i32:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT:    v_mov_b32_e32 v4, 0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_bitcmp1_b32 s2, 0
-; GCN-NEXT:    s_cselect_b32 s2, 7, 14
-; GCN-NEXT:    s_cselect_b32 s3, 6, 10
-; GCN-NEXT:    s_cselect_b32 s4, 5, 6
-; GCN-NEXT:    s_cselect_b32 s5, 9, 2
-; GCN-NEXT:    v_mov_b32_e32 v0, s5
-; GCN-NEXT:    v_mov_b32_e32 v1, s4
-; GCN-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-NEXT:    v_mov_b32_e32 v3, s2
-; GCN-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GCN-NEXT:    s_endpgm
+; GFX9-LABEL: sel_constants_sub_constant_sel_constants_v4i32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s2, 7, 14
+; GFX9-NEXT:    s_cselect_b32 s3, 6, 10
+; GFX9-NEXT:    s_cselect_b32 s4, 5, 6
+; GFX9-NEXT:    s_cselect_b32 s5, 9, 2
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX942-LABEL: sel_constants_sub_constant_sel_constants_v4i32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, 7, 14
+; GFX942-NEXT:    s_cselect_b32 s3, 6, 10
+; GFX942-NEXT:    s_cselect_b32 s4, 5, 6
+; GFX942-NEXT:    s_cselect_b32 s5, 9, 2
+; GFX942-NEXT:    v_mov_b32_e32 v2, s5
+; GFX942-NEXT:    v_mov_b32_e32 v3, s4
+; GFX942-NEXT:    v_mov_b32_e32 v4, s3
+; GFX942-NEXT:    v_mov_b32_e32 v5, s2
+; GFX942-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %sel = select i1 %cond, <4 x i32> <i32 -4, i32 2, i32 3, i32 4>, <4 x i32> <i32 3, i32 1, i32 -1, i32 -3>
   %bo = sub <4 x i32> <i32 5, i32 7, i32 9, i32 11>, %sel
   store <4 x i32> %bo, ptr addrspace(1) %p, align 32
@@ -461,24 +518,43 @@ define amdgpu_kernel void @fsub_constant_sel_constants_v2f16(ptr addrspace(1) %p
 }
 
 define amdgpu_kernel void @fsub_constant_sel_constants_v4f32(ptr addrspace(1) %p, i1 %cond) {
-; GCN-LABEL: fsub_constant_sel_constants_v4f32:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT:    s_mov_b32 s3, 0x41500000
-; GCN-NEXT:    v_mov_b32_e32 v4, 0
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_bitcmp1_b32 s2, 0
-; GCN-NEXT:    s_cselect_b32 s2, s3, 0x40c00000
-; GCN-NEXT:    s_cselect_b32 s3, 0x41100000, 4.0
-; GCN-NEXT:    s_cselect_b32 s4, 0x40a00000, 2.0
-; GCN-NEXT:    s_cselect_b32 s5, 1.0, 0
-; GCN-NEXT:    v_mov_b32_e32 v0, s5
-; GCN-NEXT:    v_mov_b32_e32 v1, s4
-; GCN-NEXT:    v_mov_b32_e32 v2, s3
-; GCN-NEXT:    v_mov_b32_e32 v3, s2
-; GCN-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
-; GCN-NEXT:    s_endpgm
+; GFX9-LABEL: fsub_constant_sel_constants_v4f32:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_mov_b32 s3, 0x41500000
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX9-NEXT:    s_cselect_b32 s2, s3, 0x40c00000
+; GFX9-NEXT:    s_cselect_b32 s3, 0x41100000, 4.0
+; GFX9-NEXT:    s_cselect_b32 s4, 0x40a00000, 2.0
+; GFX9-NEXT:    s_cselect_b32 s5, 1.0, 0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX942-LABEL: fsub_constant_sel_constants_v4f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0x41500000
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, s3, 0x40c00000
+; GFX942-NEXT:    s_cselect_b32 s3, 0x41100000, 4.0
+; GFX942-NEXT:    s_cselect_b32 s4, 0x40a00000, 2.0
+; GFX942-NEXT:    s_cselect_b32 s5, 1.0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v2, s5
+; GFX942-NEXT:    v_mov_b32_e32 v3, s4
+; GFX942-NEXT:    v_mov_b32_e32 v4, s3
+; GFX942-NEXT:    v_mov_b32_e32 v5, s2
+; GFX942-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %sel = select i1 %cond, <4 x float> <float -2.0, float -3.0, float -4.0, float -5.0>, <4 x float> <float -1.0, float 0.0, float 1.0, float 2.0>
   %bo = fsub <4 x float> <float -1.0, float 2.0, float 5.0, float 8.0>, %sel
   store <4 x float> %bo, ptr addrspace(1) %p, align 32
diff --git a/llvm/test/CodeGen/AMDGPU/disable-preload-kernargs.ll b/llvm/test/CodeGen/AMDGPU/disable-preload-kernargs.ll
new file mode 100644
index 0000000000000..75aaec6f1fa70
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/disable-preload-kernargs.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -passes=amdgpu-preload-kernel-arguments -amdgpu-kernarg-preload=0 %s -o - | FileCheck -check-prefix=NO-PRELOAD %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -passes=amdgpu-preload-kernel-arguments %s -o - | FileCheck -check-prefix=DEFAULT-PRELOAD %s
+
+@g1 = protected addrspace(1) externally_initialized global i16 0, align 2
+
+define amdgpu_kernel void @test_kernel_with_zero_kernel_arg() {
+; NO-PRELOAD-LABEL: define amdgpu_kernel void @test_kernel_with_zero_kernel_arg(
+; NO-PRELOAD-SAME: ) #[[ATTR0:[0-9]+]] {
+; NO-PRELOAD-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; NO-PRELOAD-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 12
+; NO-PRELOAD-NEXT:    [[GROUP_SIZE_X:%.*]] = load i16, ptr addrspace(4) [[GEP]], align 2
+; NO-PRELOAD-NEXT:    store i16 [[GROUP_SIZE_X]], ptr addrspace(1) @g1, align 2
+; NO-PRELOAD-NEXT:    ret void
+;
+; DEFAULT-PRELOAD-LABEL: define amdgpu_kernel void @test_kernel_with_zero_kernel_arg(
+; DEFAULT-PRELOAD-SAME: i32 inreg "amdgpu-hidden-argument" [[_HIDDEN_BLOCK_COUNT_X:%.*]], i32 inreg "amdgpu-hidden-argument" [[_HIDDEN_BLOCK_COUNT_Y:%.*]], i32 inreg "amdgpu-hidden-argument" [[_HIDDEN_BLOCK_COUNT_Z:%.*]], i16 inreg "amdgpu-hidden-argument" [[_HIDDEN_GROUP_SIZE_X:%.*]]) #[[ATTR0:[0-9]+]] {
+; DEFAULT-PRELOAD-NEXT:    [[IMPLICITARG_PTR:%.*]] = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+; DEFAULT-PRELOAD-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(4) [[IMPLICITARG_PTR]], i64 12
+; DEFAULT-PRELOAD-NEXT:    [[GROUP_SIZE_X:%.*]] = load i16, ptr addrspace(4) [[GEP]], align 2
+; DEFAULT-PRELOAD-NEXT:    store i16 [[_HIDDEN_GROUP_SIZE_X]], ptr addrspace(1) @g1, align 2
+; DEFAULT-PRELOAD-NEXT:    ret void
+;
+  %implicitarg.ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
+  %gep = getelementptr inbounds i8, ptr addrspace(4) %implicitarg.ptr, i64 12
+  %group_size_x = load i16, ptr addrspace(4) %gep
+  store i16 %group_size_x, ptr addrspace(1) @g1
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/dpp64_combine.ll b/llvm/test/CodeGen/AMDGPU/dpp64_combine.ll
index bf37ccf3ac89f..43f6def22d981 100644
--- a/llvm/test/CodeGen/AMDGPU/dpp64_combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/dpp64_combine.ll
@@ -1,12 +1,13 @@
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck %s -check-prefixes=GCN,DPP64,GFX90A
-; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck %s -check-prefixes=GCN,DPP64,DPPMOV64,GFX942
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS,GFX10
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS,GFX11
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck %s -check-prefixes=GCN,DPP64,GFX90A,DPP64-GFX9 -DCTL=row_newbcast
+; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck %s -check-prefixes=GCN,DPP64,DPPMOV64,DPP64-GFX9,GFX942 -DCTL=row_newbcast
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS,GFX10 -DCTL=row_share
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX10PLUS,GFX11 -DCTL=row_share
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck %s -check-prefixes=GCN,DPP32,GFX1250 -DCTL=row_share
 
 ; GCN-LABEL: {{^}}dpp64_ceil:
 ; GCN:           global_load_{{dwordx2|b64}} [[V:v\[[0-9:]+\]]],
-; DPP64:         v_ceil_f64_dpp [[V]], [[V]] row_newbcast:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
-; DPP32-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_share:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
+; DPP64:         v_ceil_f64_dpp [[V]], [[V]] [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
+; DPP32-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
 define amdgpu_kernel void @dpp64_ceil(ptr addrspace(1) %arg, i64 %in1) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id
@@ -21,8 +22,8 @@ define amdgpu_kernel void @dpp64_ceil(ptr addrspace(1) %arg, i64 %in1) {
 
 ; GCN-LABEL: {{^}}dpp64_rcp:
 ; GCN:           global_load_{{dwordx2|b64}} [[V:v\[[0-9:]+\]]],
-; DPP64:         v_rcp_f64_dpp [[V]], [[V]] row_newbcast:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
-; DPP32-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_share:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
+; DPP64-GFX9:    v_rcp_f64_dpp [[V]], [[V]] [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
+; DPP32-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
 define amdgpu_kernel void @dpp64_rcp(ptr addrspace(1) %arg, i64 %in1) {
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds i64, ptr addrspace(1) %arg, i32 %id
@@ -52,9 +53,9 @@ define amdgpu_kernel void @dpp64_rcp_unsupported_ctl(ptr addrspace(1) %arg, i64
 
 ; GCN-LABEL: {{^}}dpp64_div:
 ; GCN:               global_load_{{dwordx2|b64}} [[V:v\[[0-9:]+\]]],
-; DPPMOV64:          v_mov_b64_dpp v[{{[0-9:]+}}], [[V]] row_newbcast:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
-; GFX90A-COUNT-2:    v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_newbcast:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
-; GFX10PLUS-COUNT-2: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_share:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
+; DPPMOV64:          v_mov_b64_dpp v[{{[0-9:]+}}], [[V]] [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
+; GFX90A-COUNT-2:    v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
+; DPP32-COUNT-2:     v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} [[CTL]]:1 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
 ; GCN:               v_div_scale_f64
 ; GCN:               v_rcp_f64_e32
 define amdgpu_kernel void @dpp64_div(ptr addrspace(1) %arg, i64 %in1) {
@@ -69,6 +70,25 @@ define amdgpu_kernel void @dpp64_div(ptr addrspace(1) %arg, i64 %in1) {
   ret void
 }
 
+; On GFX9 it fails to combine because v_mul_lo_u32 has no e32 or dpp form.
+; GCN-LABEL: {{^}}dpp_mul_row_share:
+; GCN: global_load_{{dword|b32}} [[V:v[0-9]+]],
+; DPP64-GFX9: v_mov_b32_e32 [[V2:v[0-9]+]], [[V]]
+; DPP64-GFX9: v_mov_b32_dpp [[V2]], [[V2]] {{row_share|row_newbcast}}:0 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
+; DPP64-GFX9: v_mul_lo_u32 [[V]], [[V2]], [[V]]{{$}}
+; GFX1250: v_mov_b32_e32 [[V2:v[0-9]+]], [[V]]
+; GFX1250: v_mov_b32_dpp [[V2]], [[V2]] {{row_share|row_newbcast}}:0 row_mask:0xf bank_mask:0xf bound_ctrl:1{{$}}
+; GFX1250: v_mul_lo_u32 [[V]], [[V2]], [[V]]{{$}}
+define amdgpu_kernel void @dpp_mul_row_share(ptr addrspace(1) %arg) {
+  %id = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i32, ptr addrspace(1) %arg, i32 %id
+  %load = load i32, ptr addrspace(1) %gep
+  %tmp0 = call i32 @llvm.amdgcn.update.dpp.i32(i32 %load, i32 %load, i32 336, i32 15, i32 15, i1 1)
+  %mul = mul i32 %tmp0, %load
+  store i32 %mul, ptr addrspace(1) %gep
+  ret void
+}
+
 ; GCN-LABEL: {{^}}dpp64_loop:
 ; GCN: v_mov_b32_dpp
 ; DPP64: v_mov_b32_dpp
diff --git a/llvm/test/CodeGen/AMDGPU/dpp_combine_gfx1250.mir b/llvm/test/CodeGen/AMDGPU/dpp_combine_gfx1250.mir
new file mode 100644
index 0000000000000..9972ec82f63cd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dpp_combine_gfx1250.mir
@@ -0,0 +1,18 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=gcn-dpp-combine -o - %s | FileCheck %s -check-prefix=GFX1250
+
+---
+name: v_bitop3_dpp
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GFX1250-LABEL: name: v_bitop3_dpp
+    ; GFX1250: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX1250-NEXT: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GFX1250-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_]], 0, 0, 0, 0, implicit $exec
+    ; GFX1250-NEXT: [[V_BITOP3_B32_e64_dpp:%[0-9]+]]:vgpr_32 = V_BITOP3_B32_e64_dpp [[DEF]], [[V_MOV_B32_e32_]], 1, [[V_MOV_B32_dpp]], 128, 0, 15, 15, 1, implicit $exec
+    %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %1:vgpr_32 = V_MOV_B32_dpp %0, %0, 0, 15, 15, 0, implicit $exec
+    %2:vgpr_32 = V_MOV_B32_dpp %0, %0, 0, 0, 0, 0, implicit $exec
+    %3:vgpr_32 = V_BITOP3_B32_e64 %1, 1, %2, 128, implicit $exec
+...
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr_sym.ll b/llvm/test/CodeGen/AMDGPU/dvgpr_sym.ll
new file mode 100644
index 0000000000000..a7c1c223546f5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr_sym.ll
@@ -0,0 +1,70 @@
+; Test generation of _dvgpr$ symbol for an amdgpu_cs_chain function with dynamic vgprs.
+
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefixes=DVGPR %s
+
+; Function with 0 VGPRs, which counts as 1 block.
+;
+; DVGPR-LABEL: func0:
+; DVGPR: .set _dvgpr$func0, func0+0
+;
+define amdgpu_cs_chain void @func0() #0 {
+  ret void
+}
+
+; Function with 21 VGPRs, which is 2 blocks.
+;
+; DVGPR-LABEL: func21:
+; DVGPR: .set func21.num_vgpr, 21
+; DVGPR: .set _dvgpr$func21, func21+8
+;
+define amdgpu_cs_chain void @func21(<13 x float> %arg) #0 {
+  tail call void @func21(<13 x float> %arg)
+  ret void
+}
+
+; Anonymous function with 87 VGPRs, which is 6 blocks.
+;
+; DVGPR: [[FUNC87:__unnamed[^:]*]]:
+; DVGPR: .set [[FUNC87]].num_vgpr, 87
+; DVGPR: .set _dvgpr$[[FUNC87]], [[FUNC87]]+40
+;
+define amdgpu_cs_chain void @0(<79 x float> %arg) #0 {
+  tail call void @0(<79 x float> %arg)
+  ret void
+}
+
+; Function with 128 VGPRs, which is 8 blocks.
+;
+; DVGPR-LABEL: func128:
+; DVGPR: .set func128.num_vgpr, 128
+; DVGPR: .set _dvgpr$func128, func128+56
+;
+define amdgpu_cs_chain void @func128(<120 x float> %arg) #0 {
+  tail call void @func128(<120 x float> %arg)
+  ret void
+}
+
+; Function with 79 VGPRs, which is 3 blocks with a block size of 32.
+;
+; DVGPR-LABEL: func79:
+; DVGPR: .set func79.num_vgpr, 79
+; DVGPR: .set _dvgpr$func79, func79+16
+;
+define amdgpu_cs_chain void @func79(<71 x float> %arg) #1 {
+  tail call void @func79(<71 x float> %arg)
+  ret void
+}
+
+; Function with 225 VGPRs, which is 8 blocks with a block size of 32.
+;
+; DVGPR-LABEL: func225:
+; DVGPR: .set func225.num_vgpr, 225
+; DVGPR: .set _dvgpr$func225, func225+56
+;
+define amdgpu_cs_chain void @func225(<217 x float> %arg) #1 {
+  tail call void @func225(<217 x float> %arg)
+  ret void
+}
+
+attributes #0 = { "amdgpu-dynamic-vgpr-block-size"="16" }
+attributes #1 = { "amdgpu-dynamic-vgpr-block-size"="32" }
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr_sym_fail_too_many_block_size_16.ll b/llvm/test/CodeGen/AMDGPU/dvgpr_sym_fail_too_many_block_size_16.ll
new file mode 100644
index 0000000000000..362a5e7286285
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr_sym_fail_too_many_block_size_16.ll
@@ -0,0 +1,15 @@
+; Test failure to generate the _dvgpr$ symbol for an amdgpu_cs_chain function with dynamic vgprs.
+
+; RUN: not llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s 2>&1 | FileCheck -check-prefixes=ERR %s
+
+; Function with 129 VGPRs, which is too many with a block size of 16.
+;
+; ERR-DAG: .set func129.num_vgpr, 129
+; ERR-DAG: too many DVGPR blocks for _dvgpr$ symbol for 'func129'
+;
+define amdgpu_cs_chain void @func129(<121 x float> %arg) #0 {
+  tail call void @func129(<121 x float> %arg)
+  ret void
+}
+
+attributes #0 = { "amdgpu-dynamic-vgpr-block-size"="16" }
diff --git a/llvm/test/CodeGen/AMDGPU/dvgpr_sym_fail_too_many_block_size_16_anon.ll b/llvm/test/CodeGen/AMDGPU/dvgpr_sym_fail_too_many_block_size_16_anon.ll
new file mode 100644
index 0000000000000..218c009809a50
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/dvgpr_sym_fail_too_many_block_size_16_anon.ll
@@ -0,0 +1,24 @@
+; Test failure to generate the _dvgpr$ symbol for an anonymous amdgpu_cs_chain function with dynamic vgprs.
+
+; RUN: not llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s 2>&1 | FileCheck -check-prefixes=ERR %s
+
+; Anonymous function with 129 VGPRs, which is too many with a block size of 16.
+;
+; ERR-DAG: .set __unnamed_1.num_vgpr, 129
+; ERR-DAG: too many DVGPR blocks for _dvgpr$ symbol for '__unnamed_1'
+;
+define amdgpu_cs_chain void @0(<121 x float> %arg) #0 {
+  tail call void @0(<121 x float> %arg)
+  ret void
+}
+
+; Function that is OK, that chains to @1.
+;
+define amdgpu_cs_chain void @funcOk(<16 x float> %arg) {
+  %vec87 = shufflevector <16 x float> %arg, <16 x float> %arg, <121 x i32> splat(i32 0)
+  tail call void @0(<121 x float> %vec87)
+  ret void
+}
+
+attributes #0 = { "amdgpu-dynamic-vgpr-block-size"="16" }
+
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index b0439b1f7968f..c5db7a33f70e0 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -2536,12 +2536,13 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
 ; GFX11-SDAG-LABEL: test_dynamic_stackalloc_device_divergent_non_standard_size_i16:
 ; GFX11-SDAG:       ; %bb.0:
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-SDAG-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-SDAG-NEXT:    v_mov_b16_e32 v1.l, v0.l
 ; GFX11-SDAG-NEXT:    s_mov_b32 s4, s33
 ; GFX11-SDAG-NEXT:    s_mov_b32 s1, exec_lo
 ; GFX11-SDAG-NEXT:    s_mov_b32 s0, 0
 ; GFX11-SDAG-NEXT:    s_mov_b32 s33, s32
-; GFX11-SDAG-NEXT:    v_lshl_add_u32 v0, v0, 2, 15
+; GFX11-SDAG-NEXT:    v_lshl_add_u32 v0, v1, 2, 15
 ; GFX11-SDAG-NEXT:    s_add_i32 s32, s32, 16
 ; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-NEXT:    v_and_b32_e32 v0, 0x7fff0, v0
diff --git a/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll b/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll
index f58cb8472bc64..839d0ba35cf11 100644
--- a/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/expand-variadic-call.ll
@@ -38,11 +38,11 @@ define hidden void @copy(ptr noundef %va) {
 ; CHECK-NEXT:    %va.addr.ascast = addrspacecast ptr addrspace(5) %va.addr to ptr
 ; CHECK-NEXT:    %cp.ascast = addrspacecast ptr addrspace(5) %cp to ptr
 ; CHECK-NEXT:    store ptr %va, ptr addrspace(5) %va.addr, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %cp)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %cp)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr %cp.ascast, ptr %va.addr.ascast, i32 8, i1 false)
 ; CHECK-NEXT:    %0 = load ptr, ptr addrspace(5) %cp, align 8
 ; CHECK-NEXT:    call void @valist(ptr noundef %0)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %cp)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %cp)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -51,43 +51,43 @@ entry:
   %va.addr.ascast = addrspacecast ptr addrspace(5) %va.addr to ptr
   %cp.ascast = addrspacecast ptr addrspace(5) %cp to ptr
   store ptr %va, ptr addrspace(5) %va.addr, align 8
-  call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %cp)
+  call void @llvm.lifetime.start.p5(ptr addrspace(5) %cp)
   call void @llvm.va_copy.p0(ptr %cp.ascast, ptr nonnull %va.addr.ascast)
   %0 = load ptr, ptr addrspace(5) %cp, align 8
   call void @valist(ptr noundef %0)
-  call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %cp)
+  call void @llvm.lifetime.end.p5(ptr addrspace(5) %cp)
   ret void
 }
 
-declare void @llvm.lifetime.start.p5(i64 immarg, ptr addrspace(5) nocapture)
+declare void @llvm.lifetime.start.p5(ptr addrspace(5) nocapture)
 
 declare void @llvm.va_copy.p0(ptr, ptr)
 
 declare hidden void @valist(ptr noundef)
 
-declare void @llvm.lifetime.end.p5(i64 immarg, ptr addrspace(5) nocapture)
+declare void @llvm.lifetime.end.p5(ptr addrspace(5) nocapture)
 
 define hidden void @start_once(...) {
 ; CHECK-LABEL: define {{[^@]+}}@start_once(ptr %varargs) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %s = alloca ptr, align 8, addrspace(5)
 ; CHECK-NEXT:    %s.ascast = addrspacecast ptr addrspace(5) %s to ptr
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %s)
 ; CHECK-NEXT:    store ptr %varargs, ptr %s.ascast, align 8
 ; CHECK-NEXT:    %0 = load ptr, ptr addrspace(5) %s, align 8
 ; CHECK-NEXT:    call void @valist(ptr noundef %0)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %s)
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %s = alloca ptr, align 8, addrspace(5)
   %s.ascast = addrspacecast ptr addrspace(5) %s to ptr
-  call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s)
+  call void @llvm.lifetime.start.p5(ptr addrspace(5) %s)
   call void @llvm.va_start.p0(ptr %s.ascast)
   %0 = load ptr, ptr addrspace(5) %s, align 8
   call void @valist(ptr noundef %0)
   call void @llvm.va_end.p0(ptr %s.ascast)
-  call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s)
+  call void @llvm.lifetime.end.p5(ptr addrspace(5) %s)
   ret void
 }
 
@@ -102,16 +102,16 @@ define hidden void @start_twice(...) {
 ; CHECK-NEXT:    %s1 = alloca ptr, align 8, addrspace(5)
 ; CHECK-NEXT:    %s0.ascast = addrspacecast ptr addrspace(5) %s0 to ptr
 ; CHECK-NEXT:    %s1.ascast = addrspacecast ptr addrspace(5) %s1 to ptr
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s1)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %s0)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %s1)
 ; CHECK-NEXT:    store ptr %varargs, ptr %s0.ascast, align 8
 ; CHECK-NEXT:    %0 = load ptr, ptr addrspace(5) %s0, align 8
 ; CHECK-NEXT:    call void @valist(ptr noundef %0)
 ; CHECK-NEXT:    store ptr %varargs, ptr %s1.ascast, align 8
 ; CHECK-NEXT:    %1 = load ptr, ptr addrspace(5) %s1, align 8
 ; CHECK-NEXT:    call void @valist(ptr noundef %1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s0)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %s1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %s0)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -119,8 +119,8 @@ entry:
   %s1 = alloca ptr, align 8, addrspace(5)
   %s0.ascast = addrspacecast ptr addrspace(5) %s0 to ptr
   %s1.ascast = addrspacecast ptr addrspace(5) %s1 to ptr
-  call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s0)
-  call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %s1)
+  call void @llvm.lifetime.start.p5(ptr addrspace(5) %s0)
+  call void @llvm.lifetime.start.p5(ptr addrspace(5) %s1)
   call void @llvm.va_start.p0(ptr %s0.ascast)
   %0 = load ptr, ptr addrspace(5) %s0, align 8
   call void @valist(ptr noundef %0)
@@ -129,8 +129,8 @@ entry:
   %1 = load ptr, ptr addrspace(5) %s1, align 8
   call void @valist(ptr noundef %1)
   call void @llvm.va_end.p0(ptr %s1.ascast)
-  call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s1)
-  call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %s0)
+  call void @llvm.lifetime.end.p5(ptr addrspace(5) %s1)
+  call void @llvm.lifetime.end.p5(ptr addrspace(5) %s0)
   ret void
 }
 
@@ -138,12 +138,12 @@ define hidden void @single_i32(i32 noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_i32(i32 noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_i32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
 ; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -157,12 +157,12 @@ define hidden void @single_double(double noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_double(double noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_double.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 8, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_double.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store double %x, ptr addrspace(5) %0, align 8
 ; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 8, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -174,12 +174,12 @@ define hidden void @single_v4f32(<4 x float> noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_v4f32(<4 x float> noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_v4f32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 16, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_v4f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <4 x float> %x, ptr addrspace(5) %0, align 16
 ; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 16, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -191,12 +191,12 @@ define hidden void @single_v8f32(<8 x float> noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_v8f32(<8 x float> noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_v8f32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 32, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_v8f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <8 x float> %x, ptr addrspace(5) %0, align 32
 ; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 32, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -208,12 +208,12 @@ define hidden void @single_v16f32(<16 x float> noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_v16f32(<16 x float> noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_v16f32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 64, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_v16f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <16 x float> %x, ptr addrspace(5) %0, align 64
 ; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 64, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -225,12 +225,12 @@ define hidden void @single_v32f32(<32 x float> noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_v32f32(<32 x float> noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_v32f32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 128, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_v32f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <32 x float> %x, ptr addrspace(5) %0, align 128
 ; CHECK-NEXT:    %1 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 128, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -242,14 +242,14 @@ define hidden void @i32_double(i32 noundef %x, double noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_double(i32 noundef %x, double noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_double.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 12, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_double.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_double.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store double %y, ptr addrspace(5) %1, align 8
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 12, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -261,14 +261,14 @@ define hidden void @double_i32(double noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@double_i32(double noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %double_i32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 12, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %double_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store double %x, ptr addrspace(5) %0, align 8
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %double_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 12, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -286,14 +286,14 @@ define hidden void @i32_libcS(i32 noundef %x, i8 %y.coerce0, i16 %y.coerce1, i32
 ; CHECK-NEXT:    %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %y.coerce3, 3
 ; CHECK-NEXT:    %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %y.coerce4, 4
 ; CHECK-NEXT:    %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %y.coerce5, 5
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_libcS.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_libcS.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store %struct.libcS %.fca.5.insert, ptr addrspace(5) %1, align 8
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -317,14 +317,14 @@ define hidden void @libcS_i32(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i64
 ; CHECK-NEXT:    %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3
 ; CHECK-NEXT:    %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4
 ; CHECK-NEXT:    %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %libcS_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store %struct.libcS %.fca.5.insert, ptr addrspace(5) %0, align 8
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %libcS_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -342,14 +342,14 @@ define hidden void @i32_v4f32(i32 noundef %x, <4 x float> noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_v4f32(i32 noundef %x, <4 x float> noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_v4f32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 20, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_v4f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_v4f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store <4 x float> %y, ptr addrspace(5) %1, align 16
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 20, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -361,14 +361,14 @@ define hidden void @v4f32_i32(<4 x float> noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@v4f32_i32(<4 x float> noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %v4f32_i32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 20, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %v4f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <4 x float> %x, ptr addrspace(5) %0, align 16
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %v4f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 20, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -380,14 +380,14 @@ define hidden void @i32_v8f32(i32 noundef %x, <8 x float> noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_v8f32(i32 noundef %x, <8 x float> noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_v8f32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_v8f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_v8f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store <8 x float> %y, ptr addrspace(5) %1, align 32
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -399,14 +399,14 @@ define hidden void @v8f32_i32(<8 x float> noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@v8f32_i32(<8 x float> noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %v8f32_i32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %v8f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <8 x float> %x, ptr addrspace(5) %0, align 32
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %v8f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 36, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -418,14 +418,14 @@ define hidden void @i32_v16f32(i32 noundef %x, <16 x float> noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_v16f32(i32 noundef %x, <16 x float> noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_v16f32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 68, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_v16f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_v16f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store <16 x float> %y, ptr addrspace(5) %1, align 64
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 68, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -437,14 +437,14 @@ define hidden void @v16f32_i32(<16 x float> noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@v16f32_i32(<16 x float> noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %v16f32_i32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 68, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %v16f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <16 x float> %x, ptr addrspace(5) %0, align 64
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %v16f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 68, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -456,14 +456,14 @@ define hidden void @i32_v32f32(i32 noundef %x, <32 x float> noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_v32f32(i32 noundef %x, <32 x float> noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_v32f32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 132, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_v32f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_v32f32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store <32 x float> %y, ptr addrspace(5) %1, align 128
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 132, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -475,14 +475,14 @@ define hidden void @v32f32_i32(<32 x float> noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@v32f32_i32(<32 x float> noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %v32f32_i32.vararg, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 132, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %v32f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <32 x float> %x, ptr addrspace(5) %0, align 128
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %v32f32_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr addrspace(5) %1, align 4
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void @vararg(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 132, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -495,12 +495,12 @@ define hidden void @fptr_single_i32(i32 noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %fptr_single_i32.vararg, align 4, addrspace(5)
 ; CHECK-NEXT:    %0 = load volatile ptr, ptr addrspacecast (ptr addrspace(1) @vararg_ptr to ptr), align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %fptr_single_i32.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr addrspace(5) %1, align 4
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void %0(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -520,12 +520,12 @@ define hidden void @fptr_libcS(i8 %x.coerce0, i16 %x.coerce1, i32 %x.coerce2, i6
 ; CHECK-NEXT:    %.fca.3.insert = insertvalue %struct.libcS %.fca.2.insert, i64 %x.coerce3, 3
 ; CHECK-NEXT:    %.fca.4.insert = insertvalue %struct.libcS %.fca.3.insert, float %x.coerce4, 4
 ; CHECK-NEXT:    %.fca.5.insert = insertvalue %struct.libcS %.fca.4.insert, double %x.coerce5, 5
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 32, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %fptr_libcS.vararg, ptr addrspace(5) %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store %struct.libcS %.fca.5.insert, ptr addrspace(5) %1, align 8
 ; CHECK-NEXT:    %2 = addrspacecast ptr addrspace(5) %vararg_buffer to ptr
 ; CHECK-NEXT:    call void %0(ptr %2)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 32, ptr addrspace(5) %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/extra-lds-size.ll b/llvm/test/CodeGen/AMDGPU/extra-lds-size.ll
index e1ce5341efdd1..4349b18fd394c 100644
--- a/llvm/test/CodeGen/AMDGPU/extra-lds-size.ll
+++ b/llvm/test/CodeGen/AMDGPU/extra-lds-size.ll
@@ -6,6 +6,8 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx950 < %s | FileCheck -check-prefix=GFX950-MESA %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200-PAL %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200-MESA %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250-PAL %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250-MESA %s
 
 ; Check EXTRA_LDS_SIZE in SPI_SHADER_PGM_RSRC2_PS.
 
@@ -29,6 +31,11 @@
 ; GFX1200-MESA: .long 45100
 ; GFX1200-MESA-NEXT: .long 1024
 
+; GFX1250-PAL: '0x2c0b (SPI_SHADER_PGM_RSRC2_PS)': 0x200
+
+; GFX1250-MESA: .long 45100
+; GFX1250-MESA-NEXT: .long 512
+
 @lds = internal addrspace(3) global [4096 x i8] poison
 
 define amdgpu_ps void @global_store_saddr_uniform_ptr_in_vgprs(i32 %voffset) {
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
index 01ebe7d71428b..00cde422a2297 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
@@ -2,12 +2,68 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GFX1250-TRUE16 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -denormal-fp-math-f32=preserve-sign < %s | FileCheck -check-prefixes=GFX1250-FAKE16 %s
 
-/* TODO: Support safe bf16 fdiv lowering.
 define bfloat @v_fdiv_bf16(bfloat %x, bfloat %y) {
+; GFX1250-TRUE16-LABEL: v_fdiv_bf16:
+; GFX1250-TRUE16:       ; %bb.0:
+; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT:    v_mov_b16_e32 v2.l, 0
+; GFX1250-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v1.l
+; GFX1250-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v0.l
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX1250-TRUE16-NEXT:    v_div_scale_f32 v0, null, v2, v2, v1
+; GFX1250-TRUE16-NEXT:    v_div_scale_f32 v4, vcc_lo, v1, v2, v1
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(TRANS32_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_rcp_f32_e32 v3, v0
+; GFX1250-TRUE16-NEXT:    s_denorm_mode 15
+; GFX1250-TRUE16-NEXT:    v_nop
+; GFX1250-TRUE16-NEXT:    v_fma_f32 v5, -v0, v3, 1.0
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_fmac_f32_e32 v3, v5, v3
+; GFX1250-TRUE16-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_fma_f32 v6, -v0, v5, v4
+; GFX1250-TRUE16-NEXT:    v_fmac_f32_e32 v5, v6, v3
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_fma_f32 v0, -v0, v5, v4
+; GFX1250-TRUE16-NEXT:    s_denorm_mode 12
+; GFX1250-TRUE16-NEXT:    v_div_fmas_f32 v0, v0, v3, v5
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_div_fixup_f32 v0, v0, v2, v1
+; GFX1250-TRUE16-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
+;
+; GFX1250-FAKE16-LABEL: v_fdiv_bf16:
+; GFX1250-FAKE16:       ; %bb.0:
+; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT:    v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT:    v_div_scale_f32 v2, null, v1, v1, v0
+; GFX1250-FAKE16-NEXT:    v_div_scale_f32 v4, vcc_lo, v0, v1, v0
+; GFX1250-FAKE16-NEXT:    v_rcp_f32_e32 v3, v2
+; GFX1250-FAKE16-NEXT:    s_denorm_mode 15
+; GFX1250-FAKE16-NEXT:    v_nop
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_fma_f32 v5, -v2, v3, 1.0
+; GFX1250-FAKE16-NEXT:    v_fmac_f32_e32 v3, v5, v3
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_mul_f32_e32 v5, v4, v3
+; GFX1250-FAKE16-NEXT:    v_fma_f32 v6, -v2, v5, v4
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_fmac_f32_e32 v5, v6, v3
+; GFX1250-FAKE16-NEXT:    v_fma_f32 v2, -v2, v5, v4
+; GFX1250-FAKE16-NEXT:    s_denorm_mode 12
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_div_fmas_f32 v2, v2, v3, v5
+; GFX1250-FAKE16-NEXT:    v_div_fixup_f32 v0, v2, v1, v0
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
   %fdiv = fdiv bfloat %x, %y
   ret bfloat %fdiv
 }
-*/
 
 define bfloat @v_rcp_bf16(bfloat %x) {
 ; GFX1250-TRUE16-LABEL: v_rcp_bf16:
@@ -82,67 +138,59 @@ define bfloat @v_rcp_bf16_neg(bfloat %x) {
   ret bfloat %fdiv
 }
 
-; TODO: Support lowering to v_rsq_bf16.
 define bfloat @v_rsq_bf16(bfloat %x) {
 ; GFX1250-TRUE16-LABEL: v_rsq_bf16:
 ; GFX1250-TRUE16:       ; %bb.0:
 ; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT:    v_sqrt_bf16_e32 v0.l, v0.l
-; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
-; GFX1250-TRUE16-NEXT:    v_rcp_bf16_e32 v0.l, v0.l
+; GFX1250-TRUE16-NEXT:    v_rsq_bf16_e32 v0.l, v0.l
 ; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
 ;
 ; GFX1250-FAKE16-LABEL: v_rsq_bf16:
 ; GFX1250-FAKE16:       ; %bb.0:
 ; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT:    v_sqrt_bf16_e32 v0, v0
-; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
-; GFX1250-FAKE16-NEXT:    v_rcp_bf16_e32 v0, v0
+; GFX1250-FAKE16-NEXT:    v_rsq_bf16_e32 v0, v0
 ; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
   %sqrt = call contract bfloat @llvm.sqrt.bf16(bfloat %x)
   %fdiv = fdiv contract bfloat 1.0, %sqrt
   ret bfloat %fdiv
 }
 
-; TODO: Support lowering to v_rsq_bf16.
 define bfloat @v_rsq_bf16_neg(bfloat %x) {
 ; GFX1250-TRUE16-LABEL: v_rsq_bf16_neg:
 ; GFX1250-TRUE16:       ; %bb.0:
 ; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT:    v_sqrt_bf16_e32 v0.l, v0.l
+; GFX1250-TRUE16-NEXT:    v_rsq_bf16_e32 v0.l, v0.l
+; GFX1250-TRUE16-NEXT:    v_nop
 ; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
-; GFX1250-TRUE16-NEXT:    v_rcp_bf16_e64 v0.l, -v0.l
+; GFX1250-TRUE16-NEXT:    v_xor_b16 v0.l, 0x8000, v0.l
 ; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
 ;
 ; GFX1250-FAKE16-LABEL: v_rsq_bf16_neg:
 ; GFX1250-FAKE16:       ; %bb.0:
 ; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT:    v_sqrt_bf16_e32 v0, v0
+; GFX1250-FAKE16-NEXT:    v_rsq_bf16_e32 v0, v0
+; GFX1250-FAKE16-NEXT:    v_nop
 ; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
-; GFX1250-FAKE16-NEXT:    v_rcp_bf16_e64 v0, -v0
+; GFX1250-FAKE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
 ; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
   %sqrt = call contract bfloat @llvm.sqrt.bf16(bfloat %x)
   %fdiv = fdiv contract bfloat -1.0, %sqrt
   ret bfloat %fdiv
 }
 
-; TODO: Support lowering to v_rsq_bf16.
 define <2 x bfloat> @v_rsq_bf16_multi_use(bfloat %x) {
 ; GFX1250-TRUE16-LABEL: v_rsq_bf16_multi_use:
 ; GFX1250-TRUE16:       ; %bb.0:
 ; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX1250-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX1250-TRUE16-NEXT:    v_sqrt_bf16_e32 v1.l, v1.l
-; GFX1250-TRUE16-NEXT:    v_rcp_bf16_e32 v1.h, v1.l
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_rsq_bf16_e32 v1.h, v1.l
 ; GFX1250-TRUE16-NEXT:    v_nop
-; GFX1250-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_1)
 ; GFX1250-TRUE16-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
 ;
@@ -150,10 +198,9 @@ define <2 x bfloat> @v_rsq_bf16_multi_use(bfloat %x) {
 ; GFX1250-FAKE16:       ; %bb.0:
 ; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT:    v_sqrt_bf16_e32 v1, v0
-; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250-FAKE16-NEXT:    v_rcp_bf16_e32 v1, v1
+; GFX1250-FAKE16-NEXT:    v_rsq_bf16_e32 v1, v0
 ; GFX1250-FAKE16-NEXT:    v_nop
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
 ; GFX1250-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
   %sqrt = call contract bfloat @llvm.sqrt.bf16(bfloat %x)
@@ -163,7 +210,6 @@ define <2 x bfloat> @v_rsq_bf16_multi_use(bfloat %x) {
   ret <2 x bfloat> %r2
 }
 
-; TODO: Support lowering to v_rsq_bf16.
 define bfloat @v_rsq_bf16_missing_contract0(bfloat %x) {
 ; GFX1250-TRUE16-LABEL: v_rsq_bf16_missing_contract0:
 ; GFX1250-TRUE16:       ; %bb.0:
@@ -187,7 +233,6 @@ define bfloat @v_rsq_bf16_missing_contract0(bfloat %x) {
   ret bfloat %fdiv
 }
 
-; TODO: Support lowering to v_rsq_bf16.
 define bfloat @v_rsq_bf16_missing_contract1(bfloat %x) {
 ; GFX1250-TRUE16-LABEL: v_rsq_bf16_missing_contract1:
 ; GFX1250-TRUE16:       ; %bb.0:
@@ -211,7 +256,6 @@ define bfloat @v_rsq_bf16_missing_contract1(bfloat %x) {
   ret bfloat %fdiv
 }
 
-; TODO: Support lowering to v_rsq_bf16.
 define bfloat @v_neg_rsq_bf16_missing_contract1(bfloat %x) {
 ; GFX1250-TRUE16-LABEL: v_neg_rsq_bf16_missing_contract1:
 ; GFX1250-TRUE16:       ; %bb.0:
@@ -240,11 +284,8 @@ define <2 x bfloat> @v_rsq_v2bf16(<2 x bfloat> %a) {
 ; GFX1250-TRUE16:       ; %bb.0:
 ; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT:    v_sqrt_bf16_e32 v0.h, v0.h
-; GFX1250-TRUE16-NEXT:    v_sqrt_bf16_e32 v0.l, v0.l
-; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_2)
-; GFX1250-TRUE16-NEXT:    v_rcp_bf16_e32 v0.h, v0.h
-; GFX1250-TRUE16-NEXT:    v_rcp_bf16_e32 v0.l, v0.l
+; GFX1250-TRUE16-NEXT:    v_rsq_bf16_e32 v0.h, v0.h
+; GFX1250-TRUE16-NEXT:    v_rsq_bf16_e32 v0.l, v0.l
 ; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
 ;
 ; GFX1250-FAKE16-LABEL: v_rsq_v2bf16:
@@ -252,12 +293,9 @@ define <2 x bfloat> @v_rsq_v2bf16(<2 x bfloat> %a) {
 ; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX1250-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
-; GFX1250-FAKE16-NEXT:    v_sqrt_bf16_e32 v0, v0
-; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_2)
-; GFX1250-FAKE16-NEXT:    v_sqrt_bf16_e32 v1, v1
-; GFX1250-FAKE16-NEXT:    v_rcp_bf16_e32 v0, v0
-; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250-FAKE16-NEXT:    v_rcp_bf16_e32 v1, v1
+; GFX1250-FAKE16-NEXT:    v_rsq_bf16_e32 v0, v0
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_rsq_bf16_e32 v1, v1
 ; GFX1250-FAKE16-NEXT:    v_nop
 ; GFX1250-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
@@ -271,11 +309,11 @@ define <2 x bfloat> @v_neg_rsq_v2bf16(<2 x bfloat> %a) {
 ; GFX1250-TRUE16:       ; %bb.0:
 ; GFX1250-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT:    v_sqrt_bf16_e32 v0.h, v0.h
-; GFX1250-TRUE16-NEXT:    v_sqrt_bf16_e32 v0.l, v0.l
-; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_2)
-; GFX1250-TRUE16-NEXT:    v_rcp_bf16_e64 v0.h, -v0.h
-; GFX1250-TRUE16-NEXT:    v_rcp_bf16_e64 v0.l, -v0.l
+; GFX1250-TRUE16-NEXT:    v_rsq_bf16_e32 v0.h, v0.h
+; GFX1250-TRUE16-NEXT:    v_rsq_bf16_e32 v0.l, v0.l
+; GFX1250-TRUE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1250-TRUE16-NEXT:    v_xor_b16 v0.h, 0x8000, v0.h
+; GFX1250-TRUE16-NEXT:    v_xor_b16 v0.l, 0x8000, v0.l
 ; GFX1250-TRUE16-NEXT:    s_set_pc_i64 s[30:31]
 ;
 ; GFX1250-FAKE16-LABEL: v_neg_rsq_v2bf16:
@@ -283,13 +321,12 @@ define <2 x bfloat> @v_neg_rsq_v2bf16(<2 x bfloat> %a) {
 ; GFX1250-FAKE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX1250-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
-; GFX1250-FAKE16-NEXT:    v_sqrt_bf16_e32 v0, v0
+; GFX1250-FAKE16-NEXT:    v_rsq_bf16_e32 v0, v0
 ; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_2)
-; GFX1250-FAKE16-NEXT:    v_sqrt_bf16_e32 v1, v1
-; GFX1250-FAKE16-NEXT:    v_rcp_bf16_e64 v0, -v0
-; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250-FAKE16-NEXT:    v_rcp_bf16_e64 v1, -v1
-; GFX1250-FAKE16-NEXT:    v_nop
+; GFX1250-FAKE16-NEXT:    v_rsq_bf16_e32 v1, v1
+; GFX1250-FAKE16-NEXT:    v_xor_b32_e32 v0, 0x8000, v0
+; GFX1250-FAKE16-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT:    v_xor_b32_e32 v1, 0x8000, v1
 ; GFX1250-FAKE16-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
 ; GFX1250-FAKE16-NEXT:    s_set_pc_i64 s[30:31]
   %sqrt = call contract <2 x bfloat> @llvm.sqrt.v2bf16(<2 x bfloat> %a)
diff --git a/llvm/test/CodeGen/AMDGPU/finalizebundle.mir b/llvm/test/CodeGen/AMDGPU/finalizebundle.mir
index ea1ae04a63259..46ca6d347781e 100644
--- a/llvm/test/CodeGen/AMDGPU/finalizebundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/finalizebundle.mir
@@ -9,7 +9,7 @@ body: |
     ; CHECK-LABEL: name: test_overlap
     ; CHECK: liveins: $vgpr0_vgpr1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr2_vgpr3, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr3_vgpr4, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit $vgpr0_vgpr1, implicit $exec, implicit $vgpr1_vgpr2 {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr2_vgpr3, implicit-def $vgpr3_vgpr4, implicit $vgpr0_vgpr1, implicit $exec, implicit $vgpr1_vgpr2 {
     ; CHECK-NEXT:   $vgpr2_vgpr3 = V_LSHLREV_B64_pseudo_e32 1, $vgpr0_vgpr1, implicit $exec
     ; CHECK-NEXT:   $vgpr3_vgpr4 = V_LSHLREV_B64_pseudo_e32 1, $vgpr1_vgpr2, implicit $exec
     ; CHECK-NEXT: }
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 8c7d5cffe39d9..ae5da3ad094c7 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -503,9 +503,10 @@ define float @flat_agent_atomic_fadd_ret_f32__offset12b_neg__amdgpu_no_fine_grai
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB2_1: ; %atomicrmw.start
@@ -2827,9 +2828,10 @@ define float @flat_agent_atomic_fadd_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB14_1: ; %atomicrmw.start
@@ -8410,13 +8412,12 @@ define half @flat_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr %
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8529,13 +8530,12 @@ define half @flat_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr %
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -8785,13 +8785,12 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8908,13 +8907,12 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -9171,13 +9169,12 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -9295,13 +9292,12 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -9557,11 +9553,11 @@ define void @flat_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -9671,11 +9667,11 @@ define void @flat_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -9917,11 +9913,11 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10035,11 +10031,11 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -10288,11 +10284,11 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10407,11 +10403,11 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -10651,8 +10647,8 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_no_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10735,8 +10731,8 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_no_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -10925,10 +10921,9 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_fi
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -11014,10 +11009,9 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_fi
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -11220,13 +11214,12 @@ define half @flat_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -11345,13 +11338,12 @@ define half @flat_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -11610,11 +11602,11 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -11730,11 +11722,11 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -16769,14 +16761,11 @@ define <2 x half> @flat_agent_atomic_fadd_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX11-LABEL: flat_agent_atomic_fadd_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-NEXT:  .LBB58_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16826,9 +16815,10 @@ define <2 x half> @flat_agent_atomic_fadd_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:  .LBB58_1: ; %atomicrmw.start
@@ -16851,9 +16841,10 @@ define <2 x half> @flat_agent_atomic_fadd_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB58_1: ; %atomicrmw.start
@@ -17341,13 +17332,11 @@ define void @flat_agent_atomic_fadd_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX11-LABEL: flat_agent_atomic_fadd_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT:    flat_load_b32 v4, v[3:4]
 ; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v4, v[0:1]
 ; GFX11-NEXT:  .LBB61_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17396,9 +17385,10 @@ define void @flat_agent_atomic_fadd_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:  .LBB61_1: ; %atomicrmw.start
@@ -17421,9 +17411,10 @@ define void @flat_agent_atomic_fadd_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB61_1: ; %atomicrmw.start
@@ -19323,16 +19314,13 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-TRUE16-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB70_1: ; %atomicrmw.start
@@ -19377,16 +19365,13 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-FAKE16-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB70_1: ; %atomicrmw.start
@@ -19473,9 +19458,10 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
@@ -19516,9 +19502,10 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
@@ -20309,15 +20296,13 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB73_1: ; %atomicrmw.start
@@ -20362,15 +20347,13 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB73_1: ; %atomicrmw.start
@@ -20456,9 +20439,10 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
@@ -20499,9 +20483,10 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index 56ad91dd59ffb..6218a5c82afcd 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -382,9 +382,10 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grai
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_max_f32_e32 v1, v2, v2
@@ -409,9 +410,10 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grai
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_max_f32_e32 v1, v2, v2
@@ -836,9 +838,10 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_gra
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_max_f32_e32 v2, v2, v2
@@ -863,9 +866,10 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_gra
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_max_f32_e32 v2, v2, v2
@@ -1936,9 +1940,10 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_max_f32_e32 v1, v2, v2
@@ -1963,9 +1968,10 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_max_f32_e32 v1, v2, v2
@@ -2390,9 +2396,10 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_max_f32_e32 v2, v2, v2
@@ -2417,9 +2424,10 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_max_f32_e32 v2, v2, v2
@@ -6043,14 +6051,14 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6168,14 +6176,14 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -6438,14 +6446,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6570,14 +6578,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -6847,14 +6855,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6980,14 +6988,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -7254,13 +7262,12 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v3.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v2.h, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7376,13 +7383,12 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v3.l, v3.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v2.h, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -7638,13 +7644,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7767,13 +7772,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -8036,13 +8040,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8166,13 +8169,12 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -8424,11 +8426,11 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v4.l, v4.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8519,11 +8521,11 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v4.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -8728,10 +8730,9 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8820,10 +8821,9 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -9035,14 +9035,14 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -9169,14 +9169,14 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -9448,13 +9448,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -9579,13 +9578,12 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -14777,15 +14775,12 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-NEXT:    v_pk_max_f16 v1, v2, v2
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-NEXT:  .LBB48_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14838,9 +14833,10 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_pk_max_f16 v1, v2, v2
@@ -14865,9 +14861,10 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_pk_max_f16 v1, v2, v2
@@ -15492,14 +15489,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-NEXT:    v_pk_max_f16 v4, v2, v2
 ; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-NEXT:  .LBB51_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15552,9 +15547,10 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_pk_max_f16 v2, v2, v2
@@ -15579,9 +15575,10 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_pk_max_f16 v2, v2, v2
@@ -17225,16 +17222,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-TRUE16-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB56_1: ; %atomicrmw.start
@@ -17279,16 +17273,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-FAKE16-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB56_1: ; %atomicrmw.start
@@ -17375,9 +17366,10 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
@@ -17418,9 +17410,10 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
@@ -18577,15 +18570,13 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB59_1: ; %atomicrmw.start
@@ -18630,15 +18621,13 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB59_1: ; %atomicrmw.start
@@ -18724,9 +18713,10 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
@@ -18767,9 +18757,10 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index f0083bd23660a..6eafbb50e4bb9 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -382,9 +382,10 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grai
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_max_f32_e32 v1, v2, v2
@@ -409,9 +410,10 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grai
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_max_f32_e32 v1, v2, v2
@@ -836,9 +838,10 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_gra
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_max_f32_e32 v2, v2, v2
@@ -863,9 +866,10 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_gra
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_max_f32_e32 v2, v2, v2
@@ -1936,9 +1940,10 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_max_f32_e32 v1, v2, v2
@@ -1963,9 +1968,10 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_max_f32_e32 v1, v2, v2
@@ -2390,9 +2396,10 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_max_f32_e32 v2, v2, v2
@@ -2417,9 +2424,10 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_max_f32_e32 v2, v2, v2
@@ -6043,14 +6051,14 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6168,14 +6176,14 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -6438,14 +6446,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6570,14 +6578,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -6847,14 +6855,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6980,14 +6988,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -7254,13 +7262,12 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v3.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v3.l, v2.h, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7376,13 +7383,12 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v3.l, v3.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v3.l, v2.h, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -7638,13 +7644,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7767,13 +7772,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -8036,13 +8040,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8166,13 +8169,12 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -8424,11 +8426,11 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v4.l, v4.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8519,11 +8521,11 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v4.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -8728,10 +8730,9 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8820,10 +8821,9 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -9035,14 +9035,14 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -9169,14 +9169,14 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -9448,13 +9448,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -9579,13 +9578,12 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -14777,15 +14775,12 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-NEXT:    v_pk_max_f16 v1, v2, v2
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-NEXT:  .LBB48_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14838,9 +14833,10 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_pk_max_f16 v1, v2, v2
@@ -14865,9 +14861,10 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_pk_max_f16 v1, v2, v2
@@ -15492,14 +15489,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-NEXT:    v_pk_max_f16 v4, v2, v2
 ; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-NEXT:  .LBB51_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15552,9 +15547,10 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:    v_pk_max_f16 v2, v2, v2
@@ -15579,9 +15575,10 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:    v_pk_max_f16 v2, v2, v2
@@ -17225,16 +17222,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-TRUE16-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB56_1: ; %atomicrmw.start
@@ -17279,16 +17273,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-FAKE16-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB56_1: ; %atomicrmw.start
@@ -17375,9 +17366,10 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
@@ -17418,9 +17410,10 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
@@ -18577,15 +18570,13 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB59_1: ; %atomicrmw.start
@@ -18630,15 +18621,13 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB59_1: ; %atomicrmw.start
@@ -18724,9 +18713,10 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
@@ -18767,9 +18757,10 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 3ee0bb2122abe..25f29c8c87c96 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -475,14 +475,11 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_neg(ptr %ptr, float %val
 ; GFX11-LABEL: flat_agent_atomic_fsub_ret_f32__offset12b_neg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-NEXT:  .LBB2_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -532,9 +529,10 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_neg(ptr %ptr, float %val
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:  .LBB2_1: ; %atomicrmw.start
@@ -557,9 +555,10 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_neg(ptr %ptr, float %val
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB2_1: ; %atomicrmw.start
@@ -1068,13 +1067,11 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_neg(ptr %ptr, float %va
 ; GFX11-LABEL: flat_agent_atomic_fsub_noret_f32__offset12b_neg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT:    flat_load_b32 v4, v[3:4]
 ; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v4, v[0:1]
 ; GFX11-NEXT:  .LBB5_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1123,9 +1120,10 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_neg(ptr %ptr, float %va
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:  .LBB5_1: ; %atomicrmw.start
@@ -1148,9 +1146,10 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_neg(ptr %ptr, float %va
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB5_1: ; %atomicrmw.start
@@ -2083,14 +2082,11 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_neg__ftz(ptr %ptr, float
 ; GFX11-LABEL: flat_agent_atomic_fsub_ret_f32__offset12b_neg__ftz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2140,9 +2136,10 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_neg__ftz(ptr %ptr, float
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:  .LBB10_1: ; %atomicrmw.start
@@ -2165,9 +2162,10 @@ define float @flat_agent_atomic_fsub_ret_f32__offset12b_neg__ftz(ptr %ptr, float
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB10_1: ; %atomicrmw.start
@@ -2676,13 +2674,11 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
 ; GFX11-LABEL: flat_agent_atomic_fsub_noret_f32__offset12b_neg__ftz:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT:    flat_load_b32 v4, v[3:4]
 ; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v4, v[0:1]
 ; GFX11-NEXT:  .LBB13_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2731,9 +2727,10 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:  .LBB13_1: ; %atomicrmw.start
@@ -2756,9 +2753,10 @@ define void @flat_agent_atomic_fsub_noret_f32__offset12b_neg__ftz(ptr %ptr, floa
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB13_1: ; %atomicrmw.start
@@ -5855,13 +5853,12 @@ define half @flat_agent_atomic_fsub_ret_f16(ptr %ptr, half %val) #0 {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5974,13 +5971,12 @@ define half @flat_agent_atomic_fsub_ret_f16(ptr %ptr, half %val) #0 {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -6230,13 +6226,12 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6353,13 +6348,12 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -6616,13 +6610,12 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_neg(ptr %ptr, half %val)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6740,13 +6733,12 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_neg(ptr %ptr, half %val)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -7002,11 +6994,11 @@ define void @flat_agent_atomic_fsub_noret_f16(ptr %ptr, half %val) #0 {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7116,11 +7108,11 @@ define void @flat_agent_atomic_fsub_noret_f16(ptr %ptr, half %val) #0 {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -7362,11 +7354,11 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %val
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7480,11 +7472,11 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %val
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -7733,11 +7725,11 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_neg(ptr %ptr, half %val
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7852,11 +7844,11 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_neg(ptr %ptr, half %val
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -8096,10 +8088,9 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr %ptr, hal
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8185,10 +8176,9 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr %ptr, hal
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -8382,8 +8372,8 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr %ptr, h
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8466,8 +8456,8 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr %ptr, h
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -8665,13 +8655,12 @@ define half @flat_system_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -8790,13 +8779,12 @@ define half @flat_system_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -9055,11 +9043,11 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -9175,11 +9163,11 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -14311,14 +14299,11 @@ define <2 x half> @flat_agent_atomic_fsub_ret_v2f16__offset12b_neg(ptr %ptr, <2
 ; GFX11-LABEL: flat_agent_atomic_fsub_ret_v2f16__offset12b_neg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-NEXT:    s_mov_b32 s0, 0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14368,9 +14353,10 @@ define <2 x half> @flat_agent_atomic_fsub_ret_v2f16__offset12b_neg(ptr %ptr, <2
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:  .LBB44_1: ; %atomicrmw.start
@@ -14393,9 +14379,10 @@ define <2 x half> @flat_agent_atomic_fsub_ret_v2f16__offset12b_neg(ptr %ptr, <2
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB44_1: ; %atomicrmw.start
@@ -14970,13 +14957,11 @@ define void @flat_agent_atomic_fsub_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
 ; GFX11-LABEL: flat_agent_atomic_fsub_noret_v2f16__offset12b_neg:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT:    flat_load_b32 v4, v[3:4]
 ; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    flat_load_b32 v4, v[0:1]
 ; GFX11-NEXT:  .LBB47_1: ; %atomicrmw.start
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15025,9 +15010,10 @@ define void @flat_agent_atomic_fsub_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX90A-NEXT:  .LBB47_1: ; %atomicrmw.start
@@ -15050,9 +15036,10 @@ define void @flat_agent_atomic_fsub_noret_v2f16__offset12b_neg(ptr %ptr, <2 x ha
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX908-NEXT:  .LBB47_1: ; %atomicrmw.start
@@ -16654,16 +16641,13 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
 ; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-TRUE16-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB52_1: ; %atomicrmw.start
@@ -16708,16 +16692,13 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
 ; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v3, v0
-; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v4, vcc_lo, 0xfffff800, v3
-; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v3
-; GFX11-FAKE16-NEXT:    flat_load_b32 v0, v[4:5]
+; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT:    flat_load_b32 v0, v[3:4]
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB52_1: ; %atomicrmw.start
@@ -16804,9 +16785,10 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
@@ -16847,9 +16829,10 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v0, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v1, 16, v2
@@ -18006,15 +17989,13 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
 ; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-TRUE16-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff0000, v2
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-TRUE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-TRUE16-NEXT:    .p2align 6
 ; GFX11-TRUE16-NEXT:  .LBB55_1: ; %atomicrmw.start
@@ -18059,15 +18040,13 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
 ; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
 ; GFX11-FAKE16:       ; %bb.0:
 ; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT:    v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
 ; GFX11-FAKE16-NEXT:    v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-FAKE16-NEXT:    v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT:    flat_load_b32 v3, v[3:4]
 ; GFX11-FAKE16-NEXT:    v_lshlrev_b32_e32 v4, 16, v2
 ; GFX11-FAKE16-NEXT:    v_and_b32_e32 v5, 0xffff0000, v2
 ; GFX11-FAKE16-NEXT:    s_mov_b32 s1, 0
+; GFX11-FAKE16-NEXT:    flat_load_b32 v3, v[0:1]
 ; GFX11-FAKE16-NEXT:    s_set_inst_prefetch_distance 0x1
 ; GFX11-FAKE16-NEXT:    .p2align 6
 ; GFX11-FAKE16-NEXT:  .LBB55_1: ; %atomicrmw.start
@@ -18153,9 +18132,10 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
 ; GFX90A:       ; %bb.0:
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX90A-NEXT:    v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
-; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX90A-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX90A-NEXT:    v_addc_co_u32_e64 v5, s[4:5], -1, v1, s[4:5]
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX90A-NEXT:    v_mov_b32_e32 v0, v4
 ; GFX90A-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX90A-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX90A-NEXT:    v_lshlrev_b32_e32 v3, 16, v2
@@ -18196,9 +18176,10 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
 ; GFX908:       ; %bb.0:
 ; GFX908-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX908-NEXT:    v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
-; GFX908-NEXT:    v_addc_co_u32_e32 v4, vcc, -1, v1, vcc
-; GFX908-NEXT:    v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT:    s_mov_b64 s[4:5], vcc
+; GFX908-NEXT:    v_addc_co_u32_e64 v4, s[4:5], -1, v1, s[4:5]
 ; GFX908-NEXT:    v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
+; GFX908-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX908-NEXT:    flat_load_dword v1, v[0:1]
 ; GFX908-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX908-NEXT:    v_lshlrev_b32_e32 v5, 16, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index 7d36c9f07ea73..004d3c0c1cf53 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -284,6 +284,7 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
 ; GFX1250-SDAG-NEXT:    v_subrev_nc_u32_e32 v0, s1, v4
 ; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-SDAG-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-SDAG-NEXT:    s_clause 0x1
 ; GFX1250-SDAG-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX1250-SDAG-NEXT:    scratch_store_b64 v4, v[2:3], off scope:SCOPE_SE
 ; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
@@ -329,6 +330,7 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn(ptr inreg %sbase, i32 %vof
 ; GFX1250-GISEL-NEXT:    v_subrev_nc_u32_e32 v0, s1, v6
 ; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-GISEL-NEXT:    v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT:    s_clause 0x1
 ; GFX1250-GISEL-NEXT:    scratch_load_b64 v[0:1], v2, off
 ; GFX1250-GISEL-NEXT:    scratch_store_b64 v2, v[4:5], off scope:SCOPE_SE
 ; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
@@ -382,6 +384,7 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
 ; GFX1250-SDAG-NEXT:    v_subrev_nc_u32_e32 v0, s1, v4
 ; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-SDAG-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc_lo
+; GFX1250-SDAG-NEXT:    s_clause 0x1
 ; GFX1250-SDAG-NEXT:    scratch_load_b64 v[0:1], v4, off
 ; GFX1250-SDAG-NEXT:    scratch_store_b64 v4, v[2:3], off scope:SCOPE_SE
 ; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
@@ -430,6 +433,7 @@ define amdgpu_ps <2 x float> @flat_xchg_saddr_i64_rtn_neg128(ptr inreg %sbase, i
 ; GFX1250-GISEL-NEXT:    v_subrev_nc_u32_e32 v0, s1, v6
 ; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-GISEL-NEXT:    v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX1250-GISEL-NEXT:    s_clause 0x1
 ; GFX1250-GISEL-NEXT:    scratch_load_b64 v[0:1], v2, off
 ; GFX1250-GISEL-NEXT:    scratch_store_b64 v2, v[4:5], off scope:SCOPE_SE
 ; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
index b25d9b245f5f6..fc8883924dfbc 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
@@ -3621,7 +3621,8 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; GFX9-NEXT:    s_mov_b32 s0, 0
 ; GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    s_movk_i32 s0, 0x3004
+; GFX9-NEXT:    s_movk_i32 s0, 0x3000
+; GFX9-NEXT:    s_add_i32 s0, s0, 4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:3712
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -3637,7 +3638,8 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 13
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
-; GFX10-NEXT:    s_movk_i32 s0, 0x3804
+; GFX10-NEXT:    s_movk_i32 s0, 0x3800
+; GFX10-NEXT:    s_add_i32 s0, s0, 4
 ; GFX10-NEXT:    scratch_store_dword off, v0, off offset:4
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_store_dword off, v1, s0 offset:1664
@@ -3682,7 +3684,8 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; GFX9-PAL-NEXT:    s_addc_u32 flat_scratch_hi, s13, 0
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s0 offset:4
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-PAL-NEXT:    s_movk_i32 s0, 0x3004
+; GFX9-PAL-NEXT:    s_movk_i32 s0, 0x3000
+; GFX9-PAL-NEXT:    s_add_i32 s0, s0, 4
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s0 offset:3712
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
@@ -3716,8 +3719,9 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; GFX1010-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v0, 13
 ; GFX1010-PAL-NEXT:    v_mov_b32_e32 v1, 15
+; GFX1010-PAL-NEXT:    s_movk_i32 s0, 0x3800
 ; GFX1010-PAL-NEXT:    s_mov_b32 s1, 0
-; GFX1010-PAL-NEXT:    s_movk_i32 s0, 0x3804
+; GFX1010-PAL-NEXT:    s_add_i32 s0, s0, 4
 ; GFX1010-PAL-NEXT:    scratch_store_dword off, v0, s1 offset:4
 ; GFX1010-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1010-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
@@ -3739,7 +3743,8 @@ define amdgpu_kernel void @store_load_large_imm_offset_kernel() {
 ; GFX1030-PAL-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v0, 13
 ; GFX1030-PAL-NEXT:    v_mov_b32_e32 v1, 15
-; GFX1030-PAL-NEXT:    s_movk_i32 s0, 0x3804
+; GFX1030-PAL-NEXT:    s_movk_i32 s0, 0x3800
+; GFX1030-PAL-NEXT:    s_add_i32 s0, s0, 4
 ; GFX1030-PAL-NEXT:    scratch_store_dword off, v0, off offset:4
 ; GFX1030-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX1030-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
@@ -3785,10 +3790,12 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX9-LABEL: store_load_large_imm_offset_foo:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_movk_i32 s0, 0x3000
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 13
+; GFX9-NEXT:    s_add_i32 s1, s32, s0
 ; GFX9-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    s_add_i32 s0, s32, 0x3004
+; GFX9-NEXT:    s_add_i32 s0, s1, 4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX9-NEXT:    scratch_store_dword off, v0, s0 offset:3712
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
@@ -3800,8 +3807,10 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX10:       ; %bb.0: ; %bb
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 13
+; GFX10-NEXT:    s_movk_i32 s0, 0x3800
 ; GFX10-NEXT:    v_mov_b32_e32 v1, 15
-; GFX10-NEXT:    s_add_i32 s0, s32, 0x3804
+; GFX10-NEXT:    s_add_i32 s1, s32, s0
+; GFX10-NEXT:    s_add_i32 s0, s1, 4
 ; GFX10-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    scratch_store_dword off, v1, s0 offset:1664
@@ -3843,10 +3852,12 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX9-PAL-LABEL: store_load_large_imm_offset_foo:
 ; GFX9-PAL:       ; %bb.0: ; %bb
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-PAL-NEXT:    s_movk_i32 s0, 0x3000
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 13
+; GFX9-PAL-NEXT:    s_add_i32 s1, s32, s0
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-PAL-NEXT:    s_add_i32 s0, s32, 0x3004
+; GFX9-PAL-NEXT:    s_add_i32 s0, s1, 4
 ; GFX9-PAL-NEXT:    v_mov_b32_e32 v0, 15
 ; GFX9-PAL-NEXT:    scratch_store_dword off, v0, s0 offset:3712
 ; GFX9-PAL-NEXT:    s_waitcnt vmcnt(0)
@@ -3872,8 +3883,10 @@ define void @store_load_large_imm_offset_foo() {
 ; GFX10-PAL:       ; %bb.0: ; %bb
 ; GFX10-PAL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v0, 13
+; GFX10-PAL-NEXT:    s_movk_i32 s0, 0x3800
 ; GFX10-PAL-NEXT:    v_mov_b32_e32 v1, 15
-; GFX10-PAL-NEXT:    s_add_i32 s0, s32, 0x3804
+; GFX10-PAL-NEXT:    s_add_i32 s1, s32, s0
+; GFX10-PAL-NEXT:    s_add_i32 s0, s1, 4
 ; GFX10-PAL-NEXT:    scratch_store_dword off, v0, s32 offset:4
 ; GFX10-PAL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-PAL-NEXT:    scratch_store_dword off, v1, s0 offset:1664
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
index 3304dbf3eaa3d..57be2907da4a0 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
@@ -805,7 +805,7 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -879,7 +879,7 @@ define amdgpu_kernel void @atomic_and_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -962,7 +962,7 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr %out, i32 %in, i64 %
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1054,7 +1054,7 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -1114,7 +1114,7 @@ define amdgpu_kernel void @atomic_and_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile and ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1183,7 +1183,7 @@ define amdgpu_kernel void @atomic_and_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile and ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -1261,7 +1261,7 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile and ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1348,7 +1348,7 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile and ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -1413,7 +1413,7 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1487,7 +1487,7 @@ define amdgpu_kernel void @atomic_sub_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -1570,7 +1570,7 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr %out, i32 %in, i64 %
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1662,7 +1662,7 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -1722,7 +1722,7 @@ define amdgpu_kernel void @atomic_sub_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile sub ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1791,7 +1791,7 @@ define amdgpu_kernel void @atomic_sub_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile sub ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -1869,7 +1869,7 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile sub ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1956,7 +1956,7 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile sub ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -2017,7 +2017,7 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2090,7 +2090,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -2169,7 +2169,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i64 %
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2260,7 +2260,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -2316,7 +2316,7 @@ define amdgpu_kernel void @atomic_max_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile max ptr %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2384,7 +2384,7 @@ define amdgpu_kernel void @atomic_max_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile max ptr %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -2458,7 +2458,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile max ptr %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2544,7 +2544,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile max ptr %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -2605,7 +2605,7 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2678,7 +2678,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr %out, ptr %out2, i32 %
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -2757,7 +2757,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i64
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2848,7 +2848,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -2904,7 +2904,7 @@ define amdgpu_kernel void @atomic_umax_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile umax ptr %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2972,7 +2972,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile umax ptr %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -3046,7 +3046,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile umax ptr %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3132,7 +3132,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile umax ptr %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -3193,7 +3193,7 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3266,7 +3266,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -3345,7 +3345,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i64 %
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3436,7 +3436,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -3492,7 +3492,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile min ptr %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3560,7 +3560,7 @@ define amdgpu_kernel void @atomic_min_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile min ptr %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -3634,7 +3634,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile min ptr %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3720,7 +3720,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile min ptr %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -3781,7 +3781,7 @@ define amdgpu_kernel void @atomic_umin_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3854,7 +3854,7 @@ define amdgpu_kernel void @atomic_umin_i32_ret_offset(ptr %out, ptr %out2, i32 %
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -3933,7 +3933,7 @@ define amdgpu_kernel void @atomic_umin_i32_addr64_offset(ptr %out, i32 %in, i64
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4024,7 +4024,7 @@ define amdgpu_kernel void @atomic_umin_i32_ret_addr64_offset(ptr %out, ptr %out2
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -4080,7 +4080,7 @@ define amdgpu_kernel void @atomic_umin_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile umin ptr %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4148,7 +4148,7 @@ define amdgpu_kernel void @atomic_umin_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile umin ptr %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -4222,7 +4222,7 @@ define amdgpu_kernel void @atomic_umin_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile umin ptr %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4308,7 +4308,7 @@ define amdgpu_kernel void @atomic_umin_i32_ret_addr64(ptr %out, ptr %out2, i32 %
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile umin ptr %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -4373,7 +4373,7 @@ define amdgpu_kernel void @atomic_or_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4447,7 +4447,7 @@ define amdgpu_kernel void @atomic_or_i32_ret_offset(ptr %out, ptr %out2, i32 %in
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -4530,7 +4530,7 @@ define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr %out, i32 %in, i64 %i
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4622,7 +4622,7 @@ define amdgpu_kernel void @atomic_or_i32_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -4682,7 +4682,7 @@ define amdgpu_kernel void @atomic_or_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile or ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4751,7 +4751,7 @@ define amdgpu_kernel void @atomic_or_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile or ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -4829,7 +4829,7 @@ define amdgpu_kernel void @atomic_or_i32_addr64(ptr %out, i32 %in, i64 %index) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile or ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4916,7 +4916,7 @@ define amdgpu_kernel void @atomic_or_i32_ret_addr64(ptr %out, ptr %out2, i32 %in
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile or ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -5653,7 +5653,7 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr %out, i32 %in, i32 %old
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst
+  %val = cmpxchg volatile ptr %gep, i32 %old, i32 %in syncscope("agent") seq_cst seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -6297,7 +6297,7 @@ define amdgpu_kernel void @atomic_xor_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -6371,7 +6371,7 @@ define amdgpu_kernel void @atomic_xor_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -6454,7 +6454,7 @@ define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr %out, i32 %in, i64 %
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -6546,7 +6546,7 @@ define amdgpu_kernel void @atomic_xor_i32_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -6606,7 +6606,7 @@ define amdgpu_kernel void @atomic_xor_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile xor ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -6675,7 +6675,7 @@ define amdgpu_kernel void @atomic_xor_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile xor ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -6753,7 +6753,7 @@ define amdgpu_kernel void @atomic_xor_i32_addr64(ptr %out, i32 %in, i64 %index)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile xor ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -6840,7 +6840,7 @@ define amdgpu_kernel void @atomic_xor_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile xor ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -9126,7 +9126,7 @@ define amdgpu_kernel void @atomic_inc_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -9190,7 +9190,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1023
-  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -9258,7 +9258,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset_p1(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1024
-  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -9332,7 +9332,7 @@ define amdgpu_kernel void @atomic_inc_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -9415,7 +9415,7 @@ define amdgpu_kernel void @atomic_inc_i32_incr64_offset(ptr %out, i32 %in, i64 %
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -9507,7 +9507,7 @@ define amdgpu_kernel void @atomic_inc_i32_ret_incr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -9567,7 +9567,7 @@ define amdgpu_kernel void @atomic_inc_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile uinc_wrap ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -9636,7 +9636,7 @@ define amdgpu_kernel void @atomic_inc_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile uinc_wrap ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -9714,7 +9714,7 @@ define amdgpu_kernel void @atomic_inc_i32_incr64(ptr %out, i32 %in, i64 %index)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile uinc_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -9801,7 +9801,7 @@ define amdgpu_kernel void @atomic_inc_i32_ret_incr64(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile uinc_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -9866,7 +9866,7 @@ define amdgpu_kernel void @atomic_dec_i32_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -9930,7 +9930,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1023
-  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -9998,7 +9998,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset_p1(ptr %out, i32 %in) {
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 1024
-  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -10072,7 +10072,7 @@ define amdgpu_kernel void @atomic_dec_i32_ret_offset(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
-  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -10155,7 +10155,7 @@ define amdgpu_kernel void @atomic_dec_i32_decr64_offset(ptr %out, i32 %in, i64 %
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -10247,7 +10247,7 @@ define amdgpu_kernel void @atomic_dec_i32_ret_decr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
   %gep = getelementptr i32, ptr %ptr, i32 4
-  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -10307,7 +10307,7 @@ define amdgpu_kernel void @atomic_dec_i32(ptr %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile udec_wrap ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -10376,7 +10376,7 @@ define amdgpu_kernel void @atomic_dec_i32_ret(ptr %out, ptr %out2, i32 %in) {
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile udec_wrap ptr %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -10454,7 +10454,7 @@ define amdgpu_kernel void @atomic_dec_i32_decr64(ptr %out, i32 %in, i64 %index)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile udec_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -10541,7 +10541,7 @@ define amdgpu_kernel void @atomic_dec_i32_ret_decr64(ptr %out, ptr %out2, i32 %i
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr %out, i64 %index
-  %val = atomicrmw volatile udec_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %val, ptr %out2
   ret void
 }
@@ -10851,3 +10851,5 @@ define amdgpu_kernel void @atomic_load_bf16(ptr %in, ptr %out) {
   store bfloat %val, ptr %out
   ret void
 }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
index 1311560715ddd..e74ad3d62bea4 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
@@ -1061,25 +1061,64 @@ define void @flat_atomic_sub_i32_noret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_sub_i32_noret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_sub_i32_e32 v3, vcc, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB30_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_sub_i32_noret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_sub_u32_e32 v3, vcc, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB30_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_sub_i32_noret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v4, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB30_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw sub ptr %ptr, i32 %in seq_cst
   ret void
@@ -1091,9 +1130,22 @@ define void @flat_atomic_sub_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_sub_i32_e32 v3, vcc, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB31_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_sub_i32_noret_offset:
@@ -1101,17 +1153,43 @@ define void @flat_atomic_sub_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_sub_u32_e32 v3, vcc, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB31_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_sub_i32_noret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_sub v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB31_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw sub ptr %gep, i32 %in seq_cst
@@ -1122,25 +1200,67 @@ define i32 @flat_atomic_sub_i32_ret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_sub_i32_ret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    v_sub_i32_e32 v3, vcc, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB32_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_sub_i32_ret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    v_sub_u32_e32 v3, vcc, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB32_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_sub_i32_ret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB32_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw sub ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -1150,29 +1270,69 @@ define i32 @flat_atomic_sub_i32_ret_offset(ptr %out, i32 %in) {
 ; GCN1-LABEL: flat_atomic_sub_i32_ret_offset:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v0, v[3:4]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v1, v0
+; GCN1-NEXT:    v_sub_i32_e32 v0, vcc, v1, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB33_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_sub_i32_ret_offset:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v0, v[3:4]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v1, v0
+; GCN2-NEXT:    v_sub_u32_e32 v0, vcc, v1, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB33_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_sub_i32_ret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_sub v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB33_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw sub ptr %gep, i32 %in seq_cst
@@ -1185,10 +1345,22 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_scalar(ptr inreg %ptr, i32 inr
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v3
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB34_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_sub_i32_noret_scalar:
@@ -1196,10 +1368,22 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_scalar(ptr inreg %ptr, i32 inr
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v3
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB34_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_sub_i32_noret_scalar:
@@ -1207,10 +1391,22 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_scalar(ptr inreg %ptr, i32 inr
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_subrev_u32_e32 v2, s6, v3
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB34_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw sub ptr %ptr, i32 %in seq_cst
   ret void
@@ -1224,10 +1420,22 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_offset_scalar(ptr inreg %out,
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_subrev_i32_e32 v2, vcc, s6, v3
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB35_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_sub_i32_noret_offset_scalar:
@@ -1237,10 +1445,22 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_offset_scalar(ptr inreg %out,
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_sub v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v3
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB35_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_sub_i32_noret_offset_scalar:
@@ -1248,10 +1468,22 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_offset_scalar(ptr inreg %out,
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_sub v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_subrev_u32_e32 v2, s6, v3
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB35_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw sub ptr %gep, i32 %in seq_cst
@@ -1264,10 +1496,24 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_scalar(ptr inreg %ptr, i32 inreg
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v0, v[0:1]
+; GCN1-NEXT:    v_mov_b32_e32 v1, s4
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:    v_mov_b32_e32 v2, s5
+; GCN1-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_subrev_i32_e32 v3, vcc, s6, v4
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB36_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_sub_i32_ret_scalar:
@@ -1275,10 +1521,24 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_scalar(ptr inreg %ptr, i32 inreg
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v0, v[0:1]
+; GCN2-NEXT:    v_mov_b32_e32 v1, s4
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:    v_mov_b32_e32 v2, s5
+; GCN2-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v4
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB36_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_sub_i32_ret_scalar:
@@ -1286,10 +1546,24 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_scalar(ptr inreg %ptr, i32 inreg
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1]
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_subrev_u32_e32 v3, s6, v4
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB36_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw sub ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -1301,12 +1575,24 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_add_u32 s34, s4, 16
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_mov_b32_e32 v1, s34
+; GCN1-NEXT:    v_mov_b32_e32 v2, s35
+; GCN1-NEXT:    flat_load_dword v0, v[1:2]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_subrev_i32_e32 v3, vcc, s6, v4
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB37_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_sub_i32_ret_offset_scalar:
@@ -1314,12 +1600,24 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_add_u32 s34, s4, 16
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_mov_b32_e32 v1, s34
+; GCN2-NEXT:    v_mov_b32_e32 v2, s35
+; GCN2-NEXT:    flat_load_dword v0, v[1:2]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v4
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB37_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_sub_i32_ret_offset_scalar:
@@ -1327,10 +1625,24 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_sub v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_subrev_u32_e32 v3, s6, v4
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB37_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw sub ptr %gep, i32 %in seq_cst
@@ -1411,25 +1723,64 @@ define void @flat_atomic_and_i32_noret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_and_i32_noret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_and v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB40_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_and_i32_noret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_and v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB40_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_and_i32_noret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_and v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v4, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB40_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw and ptr %ptr, i32 %in seq_cst
   ret void
@@ -1441,9 +1792,22 @@ define void @flat_atomic_and_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_and v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB41_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_and_i32_noret_offset:
@@ -1451,17 +1815,43 @@ define void @flat_atomic_and_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_and v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB41_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_and_i32_noret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_and v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB41_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw and ptr %gep, i32 %in seq_cst
@@ -1472,25 +1862,67 @@ define i32 @flat_atomic_and_i32_ret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_and_i32_ret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB42_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_and_i32_ret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB42_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_and_i32_ret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB42_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw and ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -1500,29 +1932,69 @@ define i32 @flat_atomic_and_i32_ret_offset(ptr %out, i32 %in) {
 ; GCN1-LABEL: flat_atomic_and_i32_ret_offset:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v0, v[3:4]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v1, v0
+; GCN1-NEXT:    v_and_b32_e32 v0, v1, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB43_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_and_i32_ret_offset:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v0, v[3:4]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v1, v0
+; GCN2-NEXT:    v_and_b32_e32 v0, v1, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB43_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_and_i32_ret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_and v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_and_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB43_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw and ptr %gep, i32 %in seq_cst
@@ -1535,10 +2007,22 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_scalar(ptr inreg %ptr, i32 inr
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_and v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_and_b32_e32 v2, s6, v3
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB44_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_and_i32_noret_scalar:
@@ -1546,10 +2030,22 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_scalar(ptr inreg %ptr, i32 inr
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_and v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_and_b32_e32 v2, s6, v3
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB44_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_and_i32_noret_scalar:
@@ -1557,10 +2053,22 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_scalar(ptr inreg %ptr, i32 inr
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_and v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_and_b32_e32 v2, s6, v3
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB44_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw and ptr %ptr, i32 %in seq_cst
   ret void
@@ -1574,10 +2082,22 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_offset_scalar(ptr inreg %out,
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_and v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_and_b32_e32 v2, s6, v3
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB45_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_and_i32_noret_offset_scalar:
@@ -1587,10 +2107,22 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_offset_scalar(ptr inreg %out,
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_and v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_and_b32_e32 v2, s6, v3
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB45_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_and_i32_noret_offset_scalar:
@@ -1598,10 +2130,22 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_offset_scalar(ptr inreg %out,
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_and v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_and_b32_e32 v2, s6, v3
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB45_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw and ptr %gep, i32 %in seq_cst
@@ -1614,10 +2158,24 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_scalar(ptr inreg %ptr, i32 inreg
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v0, v[0:1]
+; GCN1-NEXT:    v_mov_b32_e32 v1, s4
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:    v_mov_b32_e32 v2, s5
+; GCN1-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_and_b32_e32 v3, s6, v4
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB46_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_and_i32_ret_scalar:
@@ -1625,10 +2183,24 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_scalar(ptr inreg %ptr, i32 inreg
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v0, v[0:1]
+; GCN2-NEXT:    v_mov_b32_e32 v1, s4
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:    v_mov_b32_e32 v2, s5
+; GCN2-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_and_b32_e32 v3, s6, v4
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB46_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_and_i32_ret_scalar:
@@ -1636,10 +2208,24 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_scalar(ptr inreg %ptr, i32 inreg
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1]
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_and_b32_e32 v3, s6, v4
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB46_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw and ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -1651,12 +2237,24 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_add_u32 s34, s4, 16
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_mov_b32_e32 v1, s34
+; GCN1-NEXT:    v_mov_b32_e32 v2, s35
+; GCN1-NEXT:    flat_load_dword v0, v[1:2]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_and_b32_e32 v3, s6, v4
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB47_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_and_i32_ret_offset_scalar:
@@ -1664,12 +2262,24 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_add_u32 s34, s4, 16
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_mov_b32_e32 v1, s34
+; GCN2-NEXT:    v_mov_b32_e32 v2, s35
+; GCN2-NEXT:    flat_load_dword v0, v[1:2]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_and_b32_e32 v3, s6, v4
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB47_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_and_i32_ret_offset_scalar:
@@ -1677,10 +2287,24 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_and v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_and_b32_e32 v3, s6, v4
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB47_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw and ptr %gep, i32 %in seq_cst
@@ -2532,25 +3156,64 @@ define void @flat_atomic_or_i32_noret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_or_i32_noret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_or v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB60_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_or_i32_noret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_or v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB60_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_or_i32_noret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_or v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v4, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB60_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw or ptr %ptr, i32 %in seq_cst
   ret void
@@ -2562,9 +3225,22 @@ define void @flat_atomic_or_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_or v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB61_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_or_i32_noret_offset:
@@ -2572,17 +3248,43 @@ define void @flat_atomic_or_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_or v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB61_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_or_i32_noret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_or v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB61_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw or ptr %gep, i32 %in seq_cst
@@ -2593,25 +3295,67 @@ define i32 @flat_atomic_or_i32_ret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_or_i32_ret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB62_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_or_i32_ret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB62_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_or_i32_ret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB62_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw or ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -2621,29 +3365,69 @@ define i32 @flat_atomic_or_i32_ret_offset(ptr %out, i32 %in) {
 ; GCN1-LABEL: flat_atomic_or_i32_ret_offset:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v0, v[3:4]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v1, v0
+; GCN1-NEXT:    v_or_b32_e32 v0, v1, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB63_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_or_i32_ret_offset:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v0, v[3:4]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v1, v0
+; GCN2-NEXT:    v_or_b32_e32 v0, v1, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB63_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_or_i32_ret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_or v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_or_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB63_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw or ptr %gep, i32 %in seq_cst
@@ -2656,10 +3440,22 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_scalar(ptr inreg %ptr, i32 inre
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_or v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB64_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_or_b32_e32 v2, s6, v3
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB64_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_or_i32_noret_scalar:
@@ -2667,10 +3463,22 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_scalar(ptr inreg %ptr, i32 inre
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_or v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB64_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_or_b32_e32 v2, s6, v3
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB64_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_or_i32_noret_scalar:
@@ -2678,10 +3486,22 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_scalar(ptr inreg %ptr, i32 inre
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_or v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB64_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_or_b32_e32 v2, s6, v3
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB64_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw or ptr %ptr, i32 %in seq_cst
   ret void
@@ -2695,10 +3515,22 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_offset_scalar(ptr inreg %out, i
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_or v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB65_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_or_b32_e32 v2, s6, v3
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB65_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_or_i32_noret_offset_scalar:
@@ -2708,10 +3540,22 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_offset_scalar(ptr inreg %out, i
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_or v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB65_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_or_b32_e32 v2, s6, v3
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB65_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_or_i32_noret_offset_scalar:
@@ -2719,10 +3563,22 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_offset_scalar(ptr inreg %out, i
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_or v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB65_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_or_b32_e32 v2, s6, v3
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB65_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw or ptr %gep, i32 %in seq_cst
@@ -2735,10 +3591,24 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_scalar(ptr inreg %ptr, i32 inreg %
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v0, v[0:1]
+; GCN1-NEXT:    v_mov_b32_e32 v1, s4
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:    v_mov_b32_e32 v2, s5
+; GCN1-NEXT:  .LBB66_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_or_b32_e32 v3, s6, v4
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB66_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_or_i32_ret_scalar:
@@ -2746,10 +3616,24 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_scalar(ptr inreg %ptr, i32 inreg %
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v0, v[0:1]
+; GCN2-NEXT:    v_mov_b32_e32 v1, s4
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:    v_mov_b32_e32 v2, s5
+; GCN2-NEXT:  .LBB66_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_or_b32_e32 v3, s6, v4
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB66_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_or_i32_ret_scalar:
@@ -2757,10 +3641,24 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_scalar(ptr inreg %ptr, i32 inreg %
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1]
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB66_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_or_b32_e32 v3, s6, v4
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB66_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw or ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -2772,12 +3670,24 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_add_u32 s34, s4, 16
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_mov_b32_e32 v1, s34
+; GCN1-NEXT:    v_mov_b32_e32 v2, s35
+; GCN1-NEXT:    flat_load_dword v0, v[1:2]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB67_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_or_b32_e32 v3, s6, v4
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB67_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_or_i32_ret_offset_scalar:
@@ -2785,12 +3695,24 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_add_u32 s34, s4, 16
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_mov_b32_e32 v1, s34
+; GCN2-NEXT:    v_mov_b32_e32 v2, s35
+; GCN2-NEXT:    flat_load_dword v0, v[1:2]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB67_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_or_b32_e32 v3, s6, v4
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB67_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_or_i32_ret_offset_scalar:
@@ -2798,10 +3720,24 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_or v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB67_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_or_b32_e32 v3, s6, v4
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB67_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw or ptr %gep, i32 %in seq_cst
@@ -2882,25 +3818,64 @@ define void @flat_atomic_xor_i32_noret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_xor_i32_noret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB70_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB70_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_xor_i32_noret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB70_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB70_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_xor_i32_noret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v4, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB70_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB70_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw xor ptr %ptr, i32 %in seq_cst
   ret void
@@ -2912,9 +3887,22 @@ define void @flat_atomic_xor_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB71_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB71_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_xor_i32_noret_offset:
@@ -2922,17 +3910,43 @@ define void @flat_atomic_xor_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB71_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB71_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_xor_i32_noret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_xor v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB71_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB71_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw xor ptr %gep, i32 %in seq_cst
@@ -2943,25 +3957,67 @@ define i32 @flat_atomic_xor_i32_ret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_xor_i32_ret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB72_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB72_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_xor_i32_ret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB72_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB72_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_xor_i32_ret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB72_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB72_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw xor ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -2971,29 +4027,69 @@ define i32 @flat_atomic_xor_i32_ret_offset(ptr %out, i32 %in) {
 ; GCN1-LABEL: flat_atomic_xor_i32_ret_offset:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v0, v[3:4]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB73_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v1, v0
+; GCN1-NEXT:    v_xor_b32_e32 v0, v1, v2
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB73_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_xor_i32_ret_offset:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v0, v[3:4]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB73_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v1, v0
+; GCN2-NEXT:    v_xor_b32_e32 v0, v1, v2
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB73_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_xor_i32_ret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_xor v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB73_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB73_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw xor ptr %gep, i32 %in seq_cst
@@ -3006,10 +4102,22 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_scalar(ptr inreg %ptr, i32 inr
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_xor_b32_e32 v2, s6, v3
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB74_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_xor_i32_noret_scalar:
@@ -3017,10 +4125,22 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_scalar(ptr inreg %ptr, i32 inr
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_xor_b32_e32 v2, s6, v3
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB74_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_xor_i32_noret_scalar:
@@ -3028,10 +4148,22 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_scalar(ptr inreg %ptr, i32 inr
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_xor_b32_e32 v2, s6, v3
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB74_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw xor ptr %ptr, i32 %in seq_cst
   ret void
@@ -3045,10 +4177,22 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_offset_scalar(ptr inreg %out,
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_xor_b32_e32 v2, s6, v3
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB75_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_xor_i32_noret_offset_scalar:
@@ -3058,10 +4202,22 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_offset_scalar(ptr inreg %out,
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_xor v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_xor_b32_e32 v2, s6, v3
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB75_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_xor_i32_noret_offset_scalar:
@@ -3069,10 +4225,22 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_offset_scalar(ptr inreg %out,
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_xor v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_xor_b32_e32 v2, s6, v3
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB75_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw xor ptr %gep, i32 %in seq_cst
@@ -3085,10 +4253,24 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_scalar(ptr inreg %ptr, i32 inreg
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v0, v[0:1]
+; GCN1-NEXT:    v_mov_b32_e32 v1, s4
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:    v_mov_b32_e32 v2, s5
+; GCN1-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_xor_b32_e32 v3, s6, v4
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB76_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_xor_i32_ret_scalar:
@@ -3096,10 +4278,24 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_scalar(ptr inreg %ptr, i32 inreg
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v0, v[0:1]
+; GCN2-NEXT:    v_mov_b32_e32 v1, s4
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:    v_mov_b32_e32 v2, s5
+; GCN2-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_xor_b32_e32 v3, s6, v4
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB76_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_xor_i32_ret_scalar:
@@ -3107,10 +4303,24 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_scalar(ptr inreg %ptr, i32 inreg
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1]
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_xor_b32_e32 v3, s6, v4
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB76_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw xor ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -3122,12 +4332,24 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_add_u32 s34, s4, 16
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_mov_b32_e32 v1, s34
+; GCN1-NEXT:    v_mov_b32_e32 v2, s35
+; GCN1-NEXT:    flat_load_dword v0, v[1:2]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_xor_b32_e32 v3, s6, v4
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB77_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_xor_i32_ret_offset_scalar:
@@ -3135,12 +4357,24 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_add_u32 s34, s4, 16
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_mov_b32_e32 v1, s34
+; GCN2-NEXT:    v_mov_b32_e32 v2, s35
+; GCN2-NEXT:    flat_load_dword v0, v[1:2]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_xor_b32_e32 v3, s6, v4
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB77_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_xor_i32_ret_offset_scalar:
@@ -3148,10 +4382,24 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_offset_scalar(ptr inreg %out, i32
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_xor v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_xor_b32_e32 v3, s6, v4
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB77_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw xor ptr %gep, i32 %in seq_cst
@@ -4228,22 +5476,9 @@ define void @flat_max_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %i
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v4, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB92_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_max_i32_e32 v3, v4, v2
-; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GCN1-NEXT:    flat_atomic_smax v[0:1], v2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v4, v3
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB92_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_max_i32_noret_offset__amdgpu_no_remote_memory:
@@ -4251,43 +5486,17 @@ define void @flat_max_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %i
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v4, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB92_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_max_i32_e32 v3, v4, v2
-; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GCN2-NEXT:    flat_atomic_smax v[0:1], v2
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v4, v3
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB92_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_max_i32_noret_offset__amdgpu_no_remote_memory:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB92_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_max_i32_e32 v3, v4, v2
-; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
+; GCN3-NEXT:    flat_atomic_smax v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v4, v3
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB92_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i64 4
   %tmp0 = atomicrmw max ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -4298,69 +5507,29 @@ define i32 @flat_atomic_max_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i3
 ; GCN1-LABEL: flat_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v0, v[3:4]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB93_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v1, v0
-; GCN1-NEXT:    v_max_i32_e32 v0, v1, v2
-; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_atomic_smax v0, v[0:1], v2 glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB93_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v0, v[3:4]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB93_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v1, v0
-; GCN2-NEXT:    v_max_i32_e32 v0, v1, v2
-; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_atomic_smax v0, v[0:1], v2 glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB93_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB93_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v4, v3
-; GCN3-NEXT:    v_max_i32_e32 v3, v4, v2
-; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
+; GCN3-NEXT:    flat_atomic_smax v0, v[0:1], v2 offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB93_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i64 4
   %result = atomicrmw max ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -5281,22 +6450,9 @@ define void @flat_umax_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v4, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB105_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_max_u32_e32 v3, v4, v2
-; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GCN1-NEXT:    flat_atomic_umax v[0:1], v2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v4, v3
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB105_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    buffer_wbinvl1_vol
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_umax_i32_noret_offset__amdgpu_no_remote_memory:
@@ -5304,43 +6460,17 @@ define void @flat_umax_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v4, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB105_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_max_u32_e32 v3, v4, v2
-; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GCN2-NEXT:    flat_atomic_umax v[0:1], v2
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v4, v3
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB105_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_umax_i32_noret_offset__amdgpu_no_remote_memory:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB105_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_max_u32_e32 v3, v4, v2
-; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
+; GCN3-NEXT:    flat_atomic_umax v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v4, v3
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB105_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i64 4
   %tmp0 = atomicrmw umax ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -5351,69 +6481,29 @@ define i32 @flat_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GCN1-LABEL: flat_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v0, v[3:4]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB106_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v1, v0
-; GCN1-NEXT:    v_max_u32_e32 v0, v1, v2
-; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_atomic_umax v0, v[0:1], v2 glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB106_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v0, v[3:4]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB106_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v1, v0
-; GCN2-NEXT:    v_max_u32_e32 v0, v1, v2
-; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_atomic_umax v0, v[0:1], v2 glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB106_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB106_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v4, v3
-; GCN3-NEXT:    v_max_u32_e32 v3, v4, v2
-; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
+; GCN3-NEXT:    flat_atomic_umax v0, v[0:1], v2 offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB106_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i64 4
   %result = atomicrmw umax ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -6022,22 +7112,9 @@ define void @flat_umin_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v4, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB115_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_min_u32_e32 v3, v4, v2
-; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GCN1-NEXT:    flat_atomic_umin v[0:1], v2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v4, v3
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB115_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_umin_i32_noret_offset__amdgpu_no_remote_memory:
@@ -6045,43 +7122,17 @@ define void @flat_umin_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v4, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB115_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_min_u32_e32 v3, v4, v2
-; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GCN2-NEXT:    flat_atomic_umin v[0:1], v2
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v4, v3
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB115_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_umin_i32_noret_offset__amdgpu_no_remote_memory:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB115_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_min_u32_e32 v3, v4, v2
-; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
+; GCN3-NEXT:    flat_atomic_umin v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v4, v3
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB115_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i64 4
   %tmp0 = atomicrmw umin ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -6092,69 +7143,29 @@ define i32 @flat_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GCN1-LABEL: flat_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v0, v[3:4]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB116_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v1, v0
-; GCN1-NEXT:    v_min_u32_e32 v0, v1, v2
-; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_atomic_umin v0, v[0:1], v2 glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB116_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v0, v[3:4]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB116_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v1, v0
-; GCN2-NEXT:    v_min_u32_e32 v0, v1, v2
-; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_atomic_umin v0, v[0:1], v2 glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB116_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB116_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v4, v3
-; GCN3-NEXT:    v_min_u32_e32 v3, v4, v2
-; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
+; GCN3-NEXT:    flat_atomic_umin v0, v[0:1], v2 offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB116_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i64 4
   %result = atomicrmw umin ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -7152,22 +8163,9 @@ define void @flat_min_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %i
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v4, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB129_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_min_i32_e32 v3, v4, v2
-; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GCN1-NEXT:    flat_atomic_smin v[0:1], v2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    v_mov_b32_e32 v4, v3
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB129_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_min_i32_noret_offset__amdgpu_no_remote_memory:
@@ -7175,43 +8173,17 @@ define void @flat_min_i32_noret_offset__amdgpu_no_remote_memory(ptr %out, i32 %i
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v4, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB129_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_min_i32_e32 v3, v4, v2
-; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GCN2-NEXT:    flat_atomic_smin v[0:1], v2
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    v_mov_b32_e32 v4, v3
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB129_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_min_i32_noret_offset__amdgpu_no_remote_memory:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB129_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_min_i32_e32 v3, v4, v2
-; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
+; GCN3-NEXT:    flat_atomic_smin v[0:1], v2 offset:16
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v4, v3
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB129_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i64 4
   %tmp0 = atomicrmw min ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -7222,69 +8194,29 @@ define i32 @flat_atomic_min_i32_ret_offset__amdgpu_no_remote_memory(ptr %out, i3
 ; GCN1-LABEL: flat_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v0, v[3:4]
-; GCN1-NEXT:    s_mov_b64 s[4:5], 0
-; GCN1-NEXT:  .LBB130_1: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v1, v0
-; GCN1-NEXT:    v_min_i32_e32 v0, v1, v2
-; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_atomic_smin v0, v[0:1], v2 glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB130_1
-; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v0, v[3:4]
-; GCN2-NEXT:    s_mov_b64 s[4:5], 0
-; GCN2-NEXT:  .LBB130_1: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v1, v0
-; GCN2-NEXT:    v_min_i32_e32 v0, v1, v2
-; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_atomic_smin v0, v[0:1], v2 glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB130_1
-; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
-; GCN3-NEXT:    s_mov_b64 s[4:5], 0
-; GCN3-NEXT:  .LBB130_1: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v4, v3
-; GCN3-NEXT:    v_min_i32_e32 v3, v4, v2
-; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
+; GCN3-NEXT:    flat_atomic_smin v0, v[0:1], v2 offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB130_1
-; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i64 4
   %result = atomicrmw min ptr %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -7299,25 +8231,70 @@ define void @flat_atomic_uinc_wrap_i32_noret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_uinc_wrap_i32_noret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB131_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
+; GCN1-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB131_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_uinc_wrap_i32_noret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB131_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 1, v4
+; GCN2-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB131_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_uinc_wrap_i32_noret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v4, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB131_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_add_u32_e32 v3, 1, v4
+; GCN3-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB131_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw uinc_wrap ptr %ptr, i32 %in seq_cst
   ret void
@@ -7329,9 +8306,24 @@ define void @flat_atomic_uinc_wrap_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB132_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
+; GCN1-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB132_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_uinc_wrap_i32_noret_offset:
@@ -7339,17 +8331,47 @@ define void @flat_atomic_uinc_wrap_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB132_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 1, v4
+; GCN2-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB132_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_uinc_wrap_i32_noret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_inc v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB132_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_add_u32_e32 v3, 1, v4
+; GCN3-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB132_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst
@@ -7360,25 +8382,73 @@ define i32 @flat_atomic_uinc_wrap_i32_ret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_uinc_wrap_i32_ret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB133_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
+; GCN1-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB133_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB133_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 1, v4
+; GCN2-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB133_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB133_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_add_u32_e32 v3, 1, v4
+; GCN3-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB133_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw uinc_wrap ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -7388,29 +8458,75 @@ define i32 @flat_atomic_uinc_wrap_i32_ret_offset(ptr %out, i32 %in) {
 ; GCN1-LABEL: flat_atomic_uinc_wrap_i32_ret_offset:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v0, v[3:4]
+; GCN1-NEXT:    s_mov_b64 s[4:5], 0
+; GCN1-NEXT:  .LBB134_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v1, v0
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v1
+; GCN1-NEXT:    v_cmp_lt_u32_e32 vcc, v1, v2
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execnz .LBB134_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret_offset:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v0, v[3:4]
+; GCN2-NEXT:    s_mov_b64 s[4:5], 0
+; GCN2-NEXT:  .LBB134_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v1, v0
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v1
+; GCN2-NEXT:    v_cmp_lt_u32_e32 vcc, v1, v2
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execnz .LBB134_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_inc v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[4:5], 0
+; GCN3-NEXT:  .LBB134_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_add_u32_e32 v3, 1, v4
+; GCN3-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    s_cbranch_execnz .LBB134_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst
@@ -7423,10 +8539,24 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_scalar(ptr inreg %ptr, i
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB135_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v2, vcc, 1, v3
+; GCN1-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN1-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB135_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_uinc_wrap_i32_noret_scalar:
@@ -7434,10 +8564,24 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_scalar(ptr inreg %ptr, i
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB135_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
+; GCN2-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN2-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB135_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_uinc_wrap_i32_noret_scalar:
@@ -7445,10 +8589,24 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_scalar(ptr inreg %ptr, i
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB135_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_add_u32_e32 v2, 1, v3
+; GCN3-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN3-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB135_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw uinc_wrap ptr %ptr, i32 %in seq_cst
   ret void
@@ -7462,10 +8620,24 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_offset_scalar(ptr inreg
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB136_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v2, vcc, 1, v3
+; GCN1-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN1-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB136_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_uinc_wrap_i32_noret_offset_scalar:
@@ -7475,10 +8647,24 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_offset_scalar(ptr inreg
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_inc v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB136_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
+; GCN2-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN2-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB136_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_uinc_wrap_i32_noret_offset_scalar:
@@ -7486,10 +8672,24 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_offset_scalar(ptr inreg
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_inc v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB136_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_add_u32_e32 v2, 1, v3
+; GCN3-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN3-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB136_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst
@@ -7502,10 +8702,26 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_scalar(ptr inreg %ptr, i32
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v0, v[0:1]
+; GCN1-NEXT:    v_mov_b32_e32 v1, s4
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:    v_mov_b32_e32 v2, s5
+; GCN1-NEXT:  .LBB137_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v4
+; GCN1-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB137_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret_scalar:
@@ -7513,10 +8729,26 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_scalar(ptr inreg %ptr, i32
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v0, v[0:1]
+; GCN2-NEXT:    v_mov_b32_e32 v1, s4
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:    v_mov_b32_e32 v2, s5
+; GCN2-NEXT:  .LBB137_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v4
+; GCN2-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB137_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret_scalar:
@@ -7524,10 +8756,26 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_scalar(ptr inreg %ptr, i32
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1]
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB137_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_add_u32_e32 v0, 1, v4
+; GCN3-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB137_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw uinc_wrap ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -7538,13 +8786,27 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_offset_scalar(ptr inreg %ou
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_add_u32 s34, s4, 16
-; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v1, s34
+; GCN1-NEXT:    v_mov_b32_e32 v2, s35
+; GCN1-NEXT:    flat_load_dword v0, v[1:2]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB138_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v0
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v4
+; GCN1-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB138_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret_offset_scalar:
@@ -7552,12 +8814,26 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_offset_scalar(ptr inreg %ou
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_add_u32 s34, s4, 16
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_mov_b32_e32 v1, s34
+; GCN2-NEXT:    v_mov_b32_e32 v2, s35
+; GCN2-NEXT:    flat_load_dword v0, v[1:2]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB138_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v0
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v4
+; GCN2-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB138_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret_offset_scalar:
@@ -7565,10 +8841,26 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_offset_scalar(ptr inreg %ou
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_inc v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB138_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v0
+; GCN3-NEXT:    v_add_u32_e32 v0, 1, v4
+; GCN3-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB138_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw uinc_wrap ptr %gep, i32 %in seq_cst
@@ -7649,25 +8941,76 @@ define void @flat_atomic_udec_wrap_i32_noret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_udec_wrap_i32_noret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB141_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, -1, v4
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN1-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN1-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB141_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_udec_wrap_i32_noret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB141_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, -1, v4
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN2-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN2-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB141_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_udec_wrap_i32_noret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v4, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB141_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN3-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN3-NEXT:    v_add_u32_e32 v3, -1, v4
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB141_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw udec_wrap ptr %ptr, i32 %in seq_cst
   ret void
@@ -7679,9 +9022,26 @@ define void @flat_atomic_udec_wrap_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
 ; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB142_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, -1, v4
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN1-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN1-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB142_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_udec_wrap_i32_noret_offset:
@@ -7689,17 +9049,51 @@ define void @flat_atomic_udec_wrap_i32_noret_offset(ptr %out, i32 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB142_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, -1, v4
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN2-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN2-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB142_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_udec_wrap_i32_noret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_dec v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v4, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB142_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN3-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN3-NEXT:    v_add_u32_e32 v3, -1, v4
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB142_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst
@@ -7710,25 +9104,79 @@ define i32 @flat_atomic_udec_wrap_i32_ret(ptr %ptr, i32 %in) {
 ; GCN1-LABEL: flat_atomic_udec_wrap_i32_ret:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB143_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v4, v3
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, -1, v4
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN1-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN1-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB143_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB143_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v4, v3
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, -1, v4
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN2-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN2-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB143_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB143_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN3-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN3-NEXT:    v_add_u32_e32 v3, -1, v4
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB143_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw udec_wrap ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -7738,29 +9186,81 @@ define i32 @flat_atomic_udec_wrap_i32_ret_offset(ptr %out, i32 %in) {
 ; GCN1-LABEL: flat_atomic_udec_wrap_i32_ret_offset:
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 16, v0
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_add_i32_e32 v3, vcc, 16, v0
+; GCN1-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v0, v[3:4]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB144_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v1, v0
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, -1, v1
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN1-NEXT:    v_cmp_gt_u32_e64 s[4:5], v1, v2
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB144_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret_offset:
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; GCN2-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v0, v[3:4]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB144_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v1, v0
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, -1, v1
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN2-NEXT:    v_cmp_gt_u32_e64 s[4:5], v1, v2
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB144_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret_offset:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    flat_atomic_dec v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB144_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v4, v3
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GCN3-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GCN3-NEXT:    v_add_u32_e32 v3, -1, v4
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN3-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB144_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v0, v3
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst
@@ -7773,10 +9273,27 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_scalar(ptr inreg %ptr, i
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:    v_mov_b32_e32 v4, s6
+; GCN1-NEXT:  .LBB145_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v2, vcc, -1, v3
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GCN1-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN1-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB145_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_udec_wrap_i32_noret_scalar:
@@ -7784,10 +9301,27 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_scalar(ptr inreg %ptr, i
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:    v_mov_b32_e32 v4, s6
+; GCN2-NEXT:  .LBB145_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v2, vcc, -1, v3
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GCN2-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN2-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB145_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_udec_wrap_i32_noret_scalar:
@@ -7795,10 +9329,27 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_scalar(ptr inreg %ptr, i
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN3-NEXT:    flat_load_dword v3, v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:    v_mov_b32_e32 v4, s6
+; GCN3-NEXT:  .LBB145_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GCN3-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
+; GCN3-NEXT:    v_add_u32_e32 v2, -1, v3
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN3-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB145_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw udec_wrap ptr %ptr, i32 %in seq_cst
   ret void
@@ -7812,10 +9363,27 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_offset_scalar(ptr inreg
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:    v_mov_b32_e32 v4, s6
+; GCN1-NEXT:  .LBB146_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v2, vcc, -1, v3
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GCN1-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN1-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v2
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB146_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_udec_wrap_i32_noret_offset_scalar:
@@ -7825,10 +9393,27 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_offset_scalar(ptr inreg
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_dec v[0:1], v2
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:    v_mov_b32_e32 v4, s6
+; GCN2-NEXT:  .LBB146_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v2, vcc, -1, v3
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GCN2-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN2-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v2
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB146_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_udec_wrap_i32_noret_offset_scalar:
@@ -7836,10 +9421,27 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_offset_scalar(ptr inreg
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_dec v[0:1], v2 offset:16
+; GCN3-NEXT:    flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:    v_mov_b32_e32 v4, s6
+; GCN3-NEXT:  .LBB146_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; GCN3-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
+; GCN3-NEXT:    v_add_u32_e32 v2, -1, v3
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN3-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v2
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB146_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %tmp0 = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst
@@ -7852,10 +9454,29 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_scalar(ptr inreg %ptr, i32
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN1-NEXT:    flat_load_dword v0, v[0:1]
+; GCN1-NEXT:    v_mov_b32_e32 v1, s4
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:    v_mov_b32_e32 v3, s6
+; GCN1-NEXT:    v_mov_b32_e32 v2, s5
+; GCN1-NEXT:  .LBB147_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v5, v0
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, -1, v5
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GCN1-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB147_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret_scalar:
@@ -7863,10 +9484,29 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_scalar(ptr inreg %ptr, i32
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN2-NEXT:    flat_load_dword v0, v[0:1]
+; GCN2-NEXT:    v_mov_b32_e32 v1, s4
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:    v_mov_b32_e32 v3, s6
+; GCN2-NEXT:    v_mov_b32_e32 v2, s5
+; GCN2-NEXT:  .LBB147_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v5, v0
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, -1, v5
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GCN2-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB147_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret_scalar:
@@ -7874,10 +9514,29 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_scalar(ptr inreg %ptr, i32
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1]
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:    v_mov_b32_e32 v3, s6
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB147_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v5, v0
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GCN3-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
+; GCN3-NEXT:    v_add_u32_e32 v0, -1, v5
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN3-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB147_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw udec_wrap ptr %ptr, i32 %in seq_cst
   ret i32 %result
@@ -7889,12 +9548,29 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_offset_scalar(ptr inreg %ou
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_add_u32 s34, s4, 16
 ; GCN1-NEXT:    s_addc_u32 s35, s5, 0
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN1-NEXT:    v_mov_b32_e32 v1, s34
+; GCN1-NEXT:    v_mov_b32_e32 v2, s35
+; GCN1-NEXT:    flat_load_dword v0, v[1:2]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:    v_mov_b32_e32 v3, s6
+; GCN1-NEXT:  .LBB148_1: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v5, v0
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, -1, v5
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GCN1-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB148_1
+; GCN1-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret_offset_scalar:
@@ -7902,12 +9578,29 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_offset_scalar(ptr inreg %ou
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_add_u32 s34, s4, 16
 ; GCN2-NEXT:    s_addc_u32 s35, s5, 0
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; GCN2-NEXT:    v_mov_b32_e32 v1, s34
+; GCN2-NEXT:    v_mov_b32_e32 v2, s35
+; GCN2-NEXT:    flat_load_dword v0, v[1:2]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:    v_mov_b32_e32 v3, s6
+; GCN2-NEXT:  .LBB148_1: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v5, v0
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, -1, v5
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GCN2-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB148_1
+; GCN2-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret_offset_scalar:
@@ -7915,10 +9608,29 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_offset_scalar(ptr inreg %ou
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    flat_atomic_dec v0, v[0:1], v2 offset:16 glc
+; GCN3-NEXT:    flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT:    v_mov_b32_e32 v1, s4
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:    v_mov_b32_e32 v3, s6
+; GCN3-NEXT:    v_mov_b32_e32 v2, s5
+; GCN3-NEXT:  .LBB148_1: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v5, v0
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GCN3-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
+; GCN3-NEXT:    v_add_u32_e32 v0, -1, v5
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN3-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] offset:16 glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB148_1
+; GCN3-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr %out, i32 4
   %result = atomicrmw udec_wrap ptr %gep, i32 %in seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
index ffe0596a95e33..d9a596283db1e 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64.ll
@@ -1299,7 +1299,7 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1442,7 +1442,7 @@ define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -1594,7 +1594,7 @@ define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1743,7 +1743,7 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -1875,7 +1875,7 @@ define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2012,7 +2012,7 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -2157,7 +2157,7 @@ define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2300,7 +2300,7 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -2441,7 +2441,7 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2587,7 +2587,7 @@ define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -2742,7 +2742,7 @@ define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2894,7 +2894,7 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -3029,7 +3029,7 @@ define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3169,7 +3169,7 @@ define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -3317,7 +3317,7 @@ define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3463,7 +3463,7 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -3604,7 +3604,7 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3752,7 +3752,7 @@ define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -3907,7 +3907,7 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4061,7 +4061,7 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -4196,7 +4196,7 @@ define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4338,7 +4338,7 @@ define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -4486,7 +4486,7 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4634,7 +4634,7 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -4775,7 +4775,7 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4923,7 +4923,7 @@ define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -5078,7 +5078,7 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -5232,7 +5232,7 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -5367,7 +5367,7 @@ define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -5509,7 +5509,7 @@ define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -5657,7 +5657,7 @@ define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -5805,7 +5805,7 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -5946,7 +5946,7 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -6094,7 +6094,7 @@ define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -6249,7 +6249,7 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -6403,7 +6403,7 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -6538,7 +6538,7 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -6680,7 +6680,7 @@ define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -6828,7 +6828,7 @@ define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -6976,7 +6976,7 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -7117,7 +7117,7 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -7265,7 +7265,7 @@ define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -7420,7 +7420,7 @@ define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -7574,7 +7574,7 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -7709,7 +7709,7 @@ define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -7851,7 +7851,7 @@ define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -7999,7 +7999,7 @@ define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -8147,7 +8147,7 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -8285,7 +8285,7 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -8428,7 +8428,7 @@ define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -8580,7 +8580,7 @@ define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %i
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -8729,7 +8729,7 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -8861,7 +8861,7 @@ define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -8998,7 +8998,7 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -9143,7 +9143,7 @@ define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -9286,7 +9286,7 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -10759,7 +10759,7 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -10902,7 +10902,7 @@ define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -11054,7 +11054,7 @@ define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -11203,7 +11203,7 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -11335,7 +11335,7 @@ define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -11472,7 +11472,7 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -11617,7 +11617,7 @@ define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -11760,7 +11760,7 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -14107,7 +14107,7 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -14260,7 +14260,7 @@ define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -14422,7 +14422,7 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -14581,7 +14581,7 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -14723,7 +14723,7 @@ define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s0
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -14870,7 +14870,7 @@ define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -15025,7 +15025,7 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -15178,7 +15178,7 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -15335,7 +15335,7 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -15499,7 +15499,7 @@ define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr %out, i64 4
-  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -15670,7 +15670,7 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -15840,7 +15840,7 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2,
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
   %gep = getelementptr i64, ptr %ptr, i64 4
-  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -15991,7 +15991,7 @@ define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
 ; GFX12-NEXT:    scratch_store_b64 off, v[0:1], s4
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -16149,7 +16149,7 @@ define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX12-NEXT:    flat_store_b64 v[2:3], v[0:1]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
@@ -16313,7 +16313,7 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -16477,7 +16477,9 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %i
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr %out, i64 %index
-  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i64 %tmp0, ptr %out2
   ret void
 }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
index 3c1bc95cc38f6..757649ca592b3 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
@@ -458,13 +458,25 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB8_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_and_i64_offset:
@@ -473,13 +485,25 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB8_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_and_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_and_i64_offset:
@@ -501,40 +525,66 @@ entry:
 define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_and_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB9_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_and_b32_e32 v3, s5, v5
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_and_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB9_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_and_b32_e32 v3, s5, v5
+; GFX8-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_and_i64_ret_offset:
@@ -561,40 +611,64 @@ entry:
 define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_and_i64_addr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB10_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB10_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_and_i64_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB10_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_and_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB10_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_and_i64_addr64_offset:
@@ -624,42 +698,68 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB11_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_and_b32_e32 v3, s5, v5
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB11_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_and_i64_ret_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB11_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_and_b32_e32 v3, s5, v5
+; GFX8-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB11_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_and_i64_ret_addr64_offset:
@@ -689,27 +789,55 @@ define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_and_i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:  .LBB12_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB12_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_and_i64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:  .LBB12_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_and_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB12_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_and_i64:
@@ -732,14 +860,29 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:  .LBB13_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_and_b32_e32 v5, s5, v7
+; GFX7-NEXT:    v_and_b32_e32 v4, s4, v6
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB13_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -749,14 +892,29 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:  .LBB13_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_and_b32_e32 v5, s5, v7
+; GFX8-NEXT:    v_and_b32_e32 v4, s4, v6
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB13_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -785,36 +943,60 @@ entry:
 define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_and_i64_addr64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB14_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB14_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_and_i64_addr64:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB14_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_and_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB14_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_and_i64_addr64:
@@ -843,38 +1025,64 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB15_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_and_b32_e32 v3, s5, v5
+; GFX7-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB15_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_and_i64_ret_addr64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB15_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_and_b32_e32 v3, s5, v5
+; GFX8-NEXT:    v_and_b32_e32 v2, s4, v4
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB15_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_and_i64_ret_addr64:
@@ -906,13 +1114,26 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:  .LBB16_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v2
+; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB16_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_sub_i64_offset:
@@ -921,13 +1142,26 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:  .LBB16_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s2, v2
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB16_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_sub_i64_offset:
@@ -949,40 +1183,68 @@ entry:
 define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_sub_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:  .LBB17_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v8, v3
+; GFX7-NEXT:    v_mov_b32_e32 v7, v2
+; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v7
+; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB17_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_sub_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:  .LBB17_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v8, v3
+; GFX8-NEXT:    v_mov_b32_e32 v7, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v7
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB17_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_sub_i64_ret_offset:
@@ -1009,40 +1271,66 @@ entry:
 define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_sub_i64_addr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:  .LBB18_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v2
+; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB18_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_sub_i64_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:  .LBB18_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s2, v2
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB18_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_sub_i64_addr64_offset:
@@ -1072,42 +1360,70 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:  .LBB19_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v8, v3
+; GFX7-NEXT:    v_mov_b32_e32 v7, v2
+; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v7
+; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB19_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_sub_i64_ret_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:  .LBB19_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v8, v3
+; GFX8-NEXT:    v_mov_b32_e32 v7, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v7
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB19_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_sub_i64_ret_addr64_offset:
@@ -1137,27 +1453,57 @@ define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_sub_i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:  .LBB20_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v2
+; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB20_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_sub_i64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:  .LBB20_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s2, v2
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB20_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_sub_i64:
@@ -1180,14 +1526,30 @@ define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:  .LBB21_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v8, v1
+; GFX7-NEXT:    v_mov_b32_e32 v7, v0
+; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v7
+; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB21_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -1197,14 +1559,30 @@ define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:  .LBB21_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v8, v1
+; GFX8-NEXT:    v_mov_b32_e32 v7, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v7
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB21_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -1233,36 +1611,62 @@ entry:
 define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_sub_i64_addr64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:  .LBB22_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s2, v2
+; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB22_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_sub_i64_addr64:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:  .LBB22_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s2, v2
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB22_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_sub_i64_addr64:
@@ -1291,38 +1695,66 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:  .LBB23_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v8, v3
+; GFX7-NEXT:    v_mov_b32_e32 v7, v2
+; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s4, v7
+; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB23_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_sub_i64_ret_addr64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:  .LBB23_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v8, v3
+; GFX8-NEXT:    v_mov_b32_e32 v7, v2
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s4, v7
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB23_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_sub_i64_ret_addr64:
@@ -1354,12 +1786,27 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smax_x2 v[2:3], v[0:1]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB24_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB24_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_max_i64_offset:
@@ -1368,12 +1815,27 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smax_x2 v[2:3], v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB24_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB24_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_max_i64_offset:
@@ -1395,40 +1857,70 @@ entry:
 define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_max_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB25_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB25_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_max_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB25_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB25_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_max_i64_ret_offset:
@@ -1455,38 +1947,68 @@ entry:
 define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_max_i64_addr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smax_x2 v[2:3], v[0:1]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB26_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB26_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_max_i64_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smax_x2 v[2:3], v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB26_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB26_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_max_i64_addr64_offset:
@@ -1516,42 +2038,72 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB27_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB27_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_max_i64_ret_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB27_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB27_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_max_i64_ret_addr64_offset:
@@ -1581,25 +2133,59 @@ define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_max_i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:  .LBB28_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB28_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_max_i64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:  .LBB28_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB28_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_max_i64:
@@ -1622,16 +2208,33 @@ define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:  .LBB29_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB29_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -1639,16 +2242,33 @@ define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:  .LBB29_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB29_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -1675,34 +2295,64 @@ entry:
 define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_max_i64_addr64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smax_x2 v[2:3], v[0:1]
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB30_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_max_i64_addr64:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smax_x2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB30_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_max_i64_addr64:
@@ -1731,38 +2381,68 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB31_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_max_i64_ret_addr64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB31_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_max_i64_ret_addr64:
@@ -1794,12 +2474,27 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umax_x2 v[2:3], v[0:1]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB32_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umax_i64_offset:
@@ -1808,12 +2503,27 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umax_x2 v[2:3], v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB32_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umax_i64_offset:
@@ -1835,40 +2545,70 @@ entry:
 define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_umax_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB33_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umax_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB33_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umax_i64_ret_offset:
@@ -1895,38 +2635,68 @@ entry:
 define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_umax_i64_addr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umax_x2 v[2:3], v[0:1]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB34_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umax_i64_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umax_x2 v[2:3], v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB34_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umax_i64_addr64_offset:
@@ -1956,42 +2726,72 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB35_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umax_i64_ret_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB35_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umax_i64_ret_addr64_offset:
@@ -2021,25 +2821,59 @@ define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_umax_i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB36_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umax_i64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB36_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umax_i64:
@@ -2062,16 +2896,33 @@ define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB37_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -2079,16 +2930,33 @@ define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB37_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2115,34 +2983,64 @@ entry:
 define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_umax_i64_addr64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umax_x2 v[2:3], v[0:1]
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: atomic_umax_i64_addr64:
-; GFX8:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB38_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB38_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: atomic_umax_i64_addr64:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umax_x2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB38_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB38_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umax_i64_addr64:
@@ -2171,38 +3069,68 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB39_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB39_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umax_i64_ret_addr64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB39_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB39_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umax_i64_ret_addr64:
@@ -2234,12 +3162,27 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smin_x2 v[2:3], v[0:1]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB40_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_min_i64_offset:
@@ -2248,12 +3191,27 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smin_x2 v[2:3], v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB40_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_min_i64_offset:
@@ -2275,40 +3233,70 @@ entry:
 define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_min_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB41_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_min_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB41_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_min_i64_ret_offset:
@@ -2335,38 +3323,68 @@ entry:
 define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_min_i64_addr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smin_x2 v[2:3], v[0:1]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB42_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_min_i64_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smin_x2 v[2:3], v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB42_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_min_i64_addr64_offset:
@@ -2396,42 +3414,72 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB43_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_min_i64_ret_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB43_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_min_i64_ret_addr64_offset:
@@ -2461,25 +3509,59 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_min_i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB44_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_min_i64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB44_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_min_i64:
@@ -2502,16 +3584,33 @@ define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB45_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -2519,16 +3618,33 @@ define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB45_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2555,34 +3671,64 @@ entry:
 define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_min_i64_addr64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smin_x2 v[2:3], v[0:1]
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB46_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_min_i64_addr64:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smin_x2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB46_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_min_i64_addr64:
@@ -2611,38 +3757,68 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB47_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_min_i64_ret_addr64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB47_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_min_i64_ret_addr64:
@@ -2674,12 +3850,27 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umin_x2 v[2:3], v[0:1]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB48_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB48_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umin_i64_offset:
@@ -2688,12 +3879,27 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umin_x2 v[2:3], v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB48_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB48_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umin_i64_offset:
@@ -2715,40 +3921,70 @@ entry:
 define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_umin_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB49_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB49_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umin_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB49_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB49_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umin_i64_ret_offset:
@@ -2775,38 +4011,68 @@ entry:
 define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_umin_i64_addr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umin_x2 v[2:3], v[0:1]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB50_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB50_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umin_i64_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umin_x2 v[2:3], v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB50_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB50_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umin_i64_addr64_offset:
@@ -2836,42 +4102,72 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB51_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB51_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umin_i64_ret_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB51_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB51_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umin_i64_ret_addr64_offset:
@@ -2901,25 +4197,59 @@ define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_umin_i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:  .LBB52_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB52_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umin_i64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:  .LBB52_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB52_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umin_i64:
@@ -2942,16 +4272,33 @@ define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:  .LBB53_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB53_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX7-NEXT:    s_endpgm
 ;
@@ -2959,16 +4306,33 @@ define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:  .LBB53_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB53_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_endpgm
 ;
@@ -2995,34 +4359,64 @@ entry:
 define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_umin_i64_addr64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umin_x2 v[2:3], v[0:1]
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s3
+; GFX7-NEXT:    v_mov_b32_e32 v7, s2
+; GFX7-NEXT:  .LBB54_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB54_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umin_i64_addr64:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umin_x2 v[2:3], v[0:1]
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NEXT:  .LBB54_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v7, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB54_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umin_i64_addr64:
@@ -3051,38 +4445,68 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB55_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB55_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_umin_i64_ret_addr64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3], v[0:1] glc
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB55_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB55_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_umin_i64_ret_addr64:
@@ -3114,13 +4538,25 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB56_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_or_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB56_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_or_i64_offset:
@@ -3129,13 +4565,25 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB56_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_or_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB56_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_or_i64_offset:
@@ -3157,40 +4605,66 @@ entry:
 define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_or_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB57_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_or_b32_e32 v3, s5, v5
+; GFX7-NEXT:    v_or_b32_e32 v2, s4, v4
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB57_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_or_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB57_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, s5, v5
+; GFX8-NEXT:    v_or_b32_e32 v2, s4, v4
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB57_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_or_i64_ret_offset:
@@ -3217,40 +4691,64 @@ entry:
 define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_or_i64_addr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB58_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_or_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB58_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_or_i64_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB58_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_or_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB58_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_or_i64_addr64_offset:
@@ -3280,42 +4778,68 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB59_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_or_b32_e32 v3, s5, v5
+; GFX7-NEXT:    v_or_b32_e32 v2, s4, v4
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB59_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_or_i64_ret_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB59_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, s5, v5
+; GFX8-NEXT:    v_or_b32_e32 v2, s4, v4
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB59_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_or_i64_ret_addr64_offset:
@@ -3345,27 +4869,55 @@ define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_or_i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_or_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB60_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_or_i64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_or_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB60_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_or_i64:
@@ -3388,14 +4940,29 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_or_b32_e32 v5, s5, v7
+; GFX7-NEXT:    v_or_b32_e32 v4, s4, v6
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB61_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -3405,14 +4972,29 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_or_b32_e32 v5, s5, v7
+; GFX8-NEXT:    v_or_b32_e32 v4, s4, v6
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB61_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -3441,36 +5023,60 @@ entry:
 define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_or_i64_addr64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_or_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB62_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_or_i64_addr64:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_or_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB62_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_or_i64_addr64:
@@ -3499,38 +5105,64 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_or_b32_e32 v3, s5, v5
+; GFX7-NEXT:    v_or_b32_e32 v2, s4, v4
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB63_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_or_i64_ret_addr64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_or_b32_e32 v3, s5, v5
+; GFX8-NEXT:    v_or_b32_e32 v2, s4, v4
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB63_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_or_i64_ret_addr64:
@@ -4104,13 +5736,25 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_xor_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_xor_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB74_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_xor_i64_offset:
@@ -4119,13 +5763,25 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB74_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_xor_i64_offset:
@@ -4147,40 +5803,66 @@ entry:
 define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_xor_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_xor_b32_e32 v3, s5, v5
+; GFX7-NEXT:    v_xor_b32_e32 v2, s4, v4
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB75_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_xor_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_xor_b32_e32 v3, s5, v5
+; GFX8-NEXT:    v_xor_b32_e32 v2, s4, v4
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB75_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_xor_i64_ret_offset:
@@ -4207,40 +5889,64 @@ entry:
 define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_xor_i64_addr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_xor_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_xor_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB76_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_xor_i64_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB76_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_xor_i64_addr64_offset:
@@ -4269,43 +5975,69 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2,
 ; GFX7-LABEL: atomic_xor_i64_ret_addr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_xor_b32_e32 v3, s5, v5
+; GFX7-NEXT:    v_xor_b32_e32 v2, s4, v4
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB77_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_xor_i64_ret_addr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_xor_b32_e32 v3, s5, v5
+; GFX8-NEXT:    v_xor_b32_e32 v2, s4, v4
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB77_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_xor_i64_ret_addr64_offset:
@@ -4335,27 +6067,55 @@ define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_xor_i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:  .LBB78_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_xor_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_xor_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB78_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_xor_i64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:  .LBB78_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB78_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_xor_i64:
@@ -4378,14 +6138,29 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s0
+; GFX7-NEXT:  .LBB79_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_xor_b32_e32 v5, s5, v7
+; GFX7-NEXT:    v_xor_b32_e32 v4, s4, v6
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB79_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -4395,14 +6170,29 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s0
+; GFX8-NEXT:  .LBB79_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_xor_b32_e32 v5, s5, v7
+; GFX8-NEXT:    v_xor_b32_e32 v4, s4, v6
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB79_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s3
 ; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
@@ -4431,36 +6221,60 @@ entry:
 define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_xor_i64_addr64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB80_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_xor_b32_e32 v1, s3, v3
+; GFX7-NEXT:    v_xor_b32_e32 v0, s2, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB80_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_xor_i64_addr64:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB80_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v1, s3, v3
+; GFX8-NEXT:    v_xor_b32_e32 v0, s2, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB80_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_xor_i64_addr64:
@@ -4489,38 +6303,64 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB81_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_xor_b32_e32 v3, s5, v5
+; GFX7-NEXT:    v_xor_b32_e32 v2, s4, v4
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB81_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_xor_i64_ret_addr64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB81_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_xor_b32_e32 v3, s5, v5
+; GFX8-NEXT:    v_xor_b32_e32 v2, s4, v4
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB81_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_xor_i64_ret_addr64:
@@ -5920,13 +7760,28 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB107_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB107_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_inc_i64_offset:
@@ -5935,13 +7790,28 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB107_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB107_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_inc_i64_offset:
@@ -5963,40 +7833,72 @@ entry:
 define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_inc_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB108_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v4
+; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB108_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_inc_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB108_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v4
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB108_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_inc_i64_ret_offset:
@@ -6023,40 +7925,70 @@ entry:
 define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_inc_i64_incr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB109_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB109_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_inc_i64_incr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB109_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB109_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_inc_i64_incr64_offset:
@@ -6086,42 +8018,74 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB110_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v4
+; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB110_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_inc_i64_ret_incr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB110_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v4
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB110_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_inc_i64_ret_incr64_offset:
@@ -6151,27 +8115,61 @@ define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_inc_i64:
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:  .LBB111_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB111_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_inc_i64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:  .LBB111_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB111_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_inc_i64:
@@ -6194,34 +8192,66 @@ define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:  .LBB112_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v4
+; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB112_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_inc_i64_ret:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:  .LBB112_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v4
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB112_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_inc_i64_ret:
@@ -6247,36 +8277,66 @@ entry:
 define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_inc_i64_incr64:
 ; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s4
+; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB113_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB113_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_inc_i64_incr64:
 ; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s4
+; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB113_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB113_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_inc_i64_incr64:
@@ -6305,38 +8365,70 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %i
 ; GFX7:       ; %bb.0: ; %entry
 ; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX7-NEXT:    s_add_u32 s0, s0, s6
+; GFX7-NEXT:    s_addc_u32 s1, s1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[0:1], 0
+; GFX7-NEXT:  .LBB114_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v5, v3
+; GFX7-NEXT:    v_mov_b32_e32 v4, v2
+; GFX7-NEXT:    v_add_i32_e32 v2, vcc, 1, v4
+; GFX7-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    s_cbranch_execnz .LBB114_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s2
+; GFX7-NEXT:    v_mov_b32_e32 v1, s3
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_inc_i64_ret_incr64:
 ; GFX8:       ; %bb.0: ; %entry
 ; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    s_lshl_b64 s[6:7], s[6:7], 3
+; GFX8-NEXT:    s_add_u32 s0, s0, s6
+; GFX8-NEXT:    s_addc_u32 s1, s1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[0:1], 0
+; GFX8-NEXT:  .LBB114_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v5, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, v2
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, 1, v4
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    s_cbranch_execnz .LBB114_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v1, s3
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_inc_i64_ret_incr64:
@@ -6364,32 +8456,70 @@ entry:
 define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_dec_i64_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_add_u32 s0, s0, 32
-; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_add_u32 s0, s4, 32
+; GFX7-NEXT:    s_addc_u32 s1, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s7
+; GFX7-NEXT:    v_mov_b32_e32 v7, s6
+; GFX7-NEXT:  .LBB115_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v2
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB115_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_dec_i64_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_add_u32 s0, s0, 32
-; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_add_u32 s0, s4, 32
+; GFX8-NEXT:    s_addc_u32 s1, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:  .LBB115_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v2
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB115_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_dec_i64_offset:
@@ -6411,40 +8541,80 @@ entry:
 define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_dec_i64_ret_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0xd
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    s_add_u32 s0, s0, 32
-; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    s_add_u32 s0, s8, 32
+; GFX7-NEXT:    s_addc_u32 s1, s9, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB116_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], v[8:9]
+; GFX7-NEXT:    v_add_i32_e64 v2, s[2:3], -1, v8
+; GFX7-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB116_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s10
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_dec_i64_ret_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    s_add_u32 s0, s0, 32
-; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    s_add_u32 s0, s8, 32
+; GFX8-NEXT:    s_addc_u32 s1, s9, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB116_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], v[8:9]
+; GFX8-NEXT:    v_add_u32_e64 v2, s[2:3], -1, v8
+; GFX8-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB116_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s10
+; GFX8-NEXT:    v_mov_b32_e32 v1, s11
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_dec_i64_ret_offset:
@@ -6471,40 +8641,78 @@ entry:
 define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_dec_i64_decr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 3
+; GFX7-NEXT:    s_add_u32 s0, s4, s0
+; GFX7-NEXT:    s_addc_u32 s1, s5, s1
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s7
+; GFX7-NEXT:    v_mov_b32_e32 v7, s6
+; GFX7-NEXT:  .LBB117_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v2
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB117_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_dec_i64_decr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], 3
+; GFX8-NEXT:    s_add_u32 s0, s4, s0
+; GFX8-NEXT:    s_addc_u32 s1, s5, s1
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:  .LBB117_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v2
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB117_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_dec_i64_decr64_offset:
@@ -6532,44 +8740,84 @@ entry:
 define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_dec_i64_ret_decr64_offset:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx8 s[4:11], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[10:11], 3
+; GFX7-NEXT:    s_add_u32 s0, s4, s0
+; GFX7-NEXT:    s_addc_u32 s1, s5, s1
 ; GFX7-NEXT:    s_add_u32 s0, s0, 32
 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s9
+; GFX7-NEXT:    v_mov_b32_e32 v5, s8
+; GFX7-NEXT:  .LBB118_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], v[8:9]
+; GFX7-NEXT:    v_add_i32_e64 v2, s[2:3], -1, v8
+; GFX7-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB118_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_dec_i64_ret_decr64_offset:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[10:11], 3
+; GFX8-NEXT:    s_add_u32 s0, s4, s0
+; GFX8-NEXT:    s_addc_u32 s1, s5, s1
 ; GFX8-NEXT:    s_add_u32 s0, s0, 32
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s9
+; GFX8-NEXT:    v_mov_b32_e32 v5, s8
+; GFX8-NEXT:  .LBB118_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], v[8:9]
+; GFX8-NEXT:    v_add_u32_e64 v2, s[2:3], -1, v8
+; GFX8-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB118_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_dec_i64_ret_decr64_offset:
@@ -6598,28 +8846,70 @@ entry:
 define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
 ; GFX7-LABEL: atomic_dec_i64:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v4, s4
+; GFX7-NEXT:    v_mov_b32_e32 v6, s7
+; GFX7-NEXT:    v_mov_b32_e32 v7, s6
+; GFX7-NEXT:    v_mov_b32_e32 v5, s5
+; GFX7-NEXT:  .LBB119_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v2
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    s_cbranch_execnz .LBB119_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_dec_i64:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GFX8-NEXT:    s_mov_b64 s[8:9], 0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:  .LBB119_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v2
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT:    s_cbranch_execnz .LBB119_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_dec_i64:
@@ -6640,36 +8930,76 @@ entry:
 define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {
 ; GFX7-LABEL: atomic_dec_i64_ret:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
 ; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    s_mov_b64 s[6:7], 0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v0, s8
+; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v4, s5
+; GFX7-NEXT:    v_mov_b32_e32 v5, s4
+; GFX7-NEXT:  .LBB120_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], v[8:9]
+; GFX7-NEXT:    v_add_i32_e64 v2, s[2:3], -1, v8
+; GFX7-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    s_cbranch_execnz .LBB120_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s10
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_dec_i64_ret:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
 ; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    s_mov_b64 s[6:7], 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s8
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:  .LBB120_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[4:5], v[8:9]
+; GFX8-NEXT:    v_add_u32_e64 v2, s[2:3], -1, v8
+; GFX8-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    s_cbranch_execnz .LBB120_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s10
+; GFX8-NEXT:    v_mov_b32_e32 v1, s11
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_dec_i64_ret:
@@ -6695,36 +9025,74 @@ entry:
 define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_dec_i64_decr64:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s2
-; GFX7-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s2
-; GFX7-NEXT:    s_addc_u32 s1, s1, s3
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 3
+; GFX7-NEXT:    s_add_u32 s0, s4, s0
+; GFX7-NEXT:    s_addc_u32 s1, s5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s7
+; GFX7-NEXT:    v_mov_b32_e32 v7, s6
+; GFX7-NEXT:  .LBB121_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[2:3], -1, v2
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB121_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_dec_i64_decr64:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    s_lshl_b64 s[2:3], s[4:5], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s2
-; GFX8-NEXT:    s_addc_u32 s1, s1, s3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[0:1], 3
+; GFX8-NEXT:    s_add_u32 s0, s4, s0
+; GFX8-NEXT:    s_addc_u32 s1, s5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s0
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:  .LBB121_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[2:3], -1, v2
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[2:3], -1, v3, s[2:3]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB121_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_dec_i64_decr64:
@@ -6751,40 +9119,80 @@ entry:
 define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %in, i64 %index) {
 ; GFX7-LABEL: atomic_dec_i64_ret_decr64:
 ; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GFX7-NEXT:    s_load_dwordx8 s[4:11], s[4:5], 0x9
 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT:    s_add_u32 s0, s0, s4
-; GFX7-NEXT:    s_addc_u32 s1, s1, s5
-; GFX7-NEXT:    v_mov_b32_e32 v3, s1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[10:11], 3
+; GFX7-NEXT:    s_add_u32 s0, s4, s0
+; GFX7-NEXT:    s_addc_u32 s1, s5, s1
+; GFX7-NEXT:    v_mov_b32_e32 v0, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s9
+; GFX7-NEXT:    v_mov_b32_e32 v5, s8
+; GFX7-NEXT:  .LBB122_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v3
+; GFX7-NEXT:    v_mov_b32_e32 v8, v2
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], v[8:9]
+; GFX7-NEXT:    v_add_i32_e64 v2, s[2:3], -1, v8
+; GFX7-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB122_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s6
+; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: atomic_dec_i64_ret_decr64:
 ; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX8-NEXT:    s_load_dwordx8 s[4:11], s[4:5], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NEXT:    s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT:    s_add_u32 s0, s0, s4
-; GFX8-NEXT:    s_addc_u32 s1, s1, s5
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    s_lshl_b64 s[0:1], s[10:11], 3
+; GFX8-NEXT:    s_add_u32 s0, s4, s0
+; GFX8-NEXT:    s_addc_u32 s1, s5, s1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s9
+; GFX8-NEXT:    v_mov_b32_e32 v5, s8
+; GFX8-NEXT:  .LBB122_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v3
+; GFX8-NEXT:    v_mov_b32_e32 v8, v2
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[0:1], s[8:9], v[8:9]
+; GFX8-NEXT:    v_add_u32_e64 v2, s[2:3], -1, v8
+; GFX8-NEXT:    v_addc_u32_e64 v3, s[2:3], -1, v9, s[2:3]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_mov_b32_e32 v2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB122_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: atomic_dec_i64_ret_decr64:
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
index 23dfe2f70fa7e..524100c5b7a25 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
@@ -3633,21 +3633,40 @@ define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB30_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB30_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB30_6
 ; GCN1-NEXT:  .LBB30_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB30_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB30_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_sub_i32_e32 v4, vcc, v6, v2
+; GCN1-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB30_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2
+; GCN1-NEXT:    ; implicit-def: $vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB30_2
-; GCN1-NEXT:  .LBB30_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB30_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN1-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen
@@ -3673,21 +3692,40 @@ define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB30_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB30_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB30_6
 ; GCN2-NEXT:  .LBB30_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB30_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB30_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_sub_u32_e32 v4, vcc, v6, v2
+; GCN2-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB30_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2
+; GCN2-NEXT:    ; implicit-def: $vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB30_2
-; GCN2-NEXT:  .LBB30_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB30_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN2-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen
@@ -3711,21 +3749,37 @@ define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB30_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB30_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB30_6
 ; GCN3-NEXT:  .LBB30_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB30_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB30_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GCN3-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB30_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2
+; GCN3-NEXT:    ; implicit-def: $vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB30_2
-; GCN3-NEXT:  .LBB30_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB30_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen
@@ -3756,21 +3810,40 @@ define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB31_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB31_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB31_6
 ; GCN1-NEXT:  .LBB31_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB31_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB31_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_sub_i32_e32 v4, vcc, v6, v2
+; GCN1-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB31_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2
+; GCN1-NEXT:    ; implicit-def: $vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB31_2
-; GCN1-NEXT:  .LBB31_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB31_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN1-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen
@@ -3798,21 +3871,40 @@ define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB31_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB31_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB31_6
 ; GCN2-NEXT:  .LBB31_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB31_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB31_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_sub_u32_e32 v4, vcc, v6, v2
+; GCN2-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB31_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2
+; GCN2-NEXT:    ; implicit-def: $vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB31_2
-; GCN2-NEXT:  .LBB31_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB31_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN2-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen
@@ -3838,21 +3930,37 @@ define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB31_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB31_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB31_6
 ; GCN3-NEXT:  .LBB31_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB31_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB31_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GCN3-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB31_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2
+; GCN3-NEXT:    ; implicit-def: $vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB31_2
-; GCN3-NEXT:  .LBB31_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB31_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen
@@ -3877,41 +3985,56 @@ define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN1-NEXT:    v_mov_b32_e32 v5, v1
-; GCN1-NEXT:    v_mov_b32_e32 v4, v0
-; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB32_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB32_4
-; GCN1-NEXT:  .LBB32_2: ; %atomicrmw.phi
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-; GCN1-NEXT:  .LBB32_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    s_cbranch_execz .LBB32_4
+; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v5, v[4:5]
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB32_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    v_sub_i32_e32 v4, vcc, v6, v2
+; GCN1-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB32_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2
+; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:  .LBB32_4: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execz .LBB32_2
-; GCN1-NEXT:  .LBB32_4: ; %atomicrmw.private
-; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
-; GCN1-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN1-NEXT:    s_cbranch_execz .LBB32_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN1-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN1-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN1-NEXT:    s_waitcnt vmcnt(1)
-; GCN1-NEXT:    v_sub_i32_e32 v2, vcc, v0, v2
+; GCN1-NEXT:    v_sub_i32_e32 v2, vcc, v4, v2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
-; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v1, v3, vcc
-; GCN1-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN1-NEXT:    v_subb_u32_e32 v3, vcc, v5, v3, vcc
+; GCN1-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN1-NEXT:  .LBB32_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v4
+; GCN1-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3920,41 +4043,56 @@ define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN2-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN2-NEXT:    v_mov_b32_e32 v5, v1
-; GCN2-NEXT:    v_mov_b32_e32 v4, v0
-; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB32_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB32_4
-; GCN2-NEXT:  .LBB32_2: ; %atomicrmw.phi
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-; GCN2-NEXT:  .LBB32_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    s_cbranch_execz .LBB32_4
+; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v5, v[4:5]
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB32_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    v_sub_u32_e32 v4, vcc, v6, v2
+; GCN2-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB32_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2
+; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:  .LBB32_4: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execz .LBB32_2
-; GCN2-NEXT:  .LBB32_4: ; %atomicrmw.private
-; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
-; GCN2-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN2-NEXT:    s_cbranch_execz .LBB32_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN2-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
+; GCN2-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN2-NEXT:    s_waitcnt vmcnt(1)
-; GCN2-NEXT:    v_sub_u32_e32 v2, vcc, v0, v2
+; GCN2-NEXT:    v_sub_u32_e32 v2, vcc, v4, v2
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
-; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v1, v3, vcc
-; GCN2-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN2-NEXT:    v_subb_u32_e32 v3, vcc, v5, v3, vcc
+; GCN2-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN2-NEXT:  .LBB32_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v4
+; GCN2-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3969,21 +4107,37 @@ define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB32_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB32_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB32_6
 ; GCN3-NEXT:  .LBB32_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB32_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB32_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_sub_co_u32_e32 v6, vcc, v8, v2
+; GCN3-NEXT:    v_subb_co_u32_e32 v7, vcc, v9, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB32_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2
+; GCN3-NEXT:    ; implicit-def: $vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB32_2
-; GCN3-NEXT:  .LBB32_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB32_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -4015,21 +4169,40 @@ define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB33_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB33_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB33_6
 ; GCN1-NEXT:  .LBB33_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB33_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB33_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v9, v1
+; GCN1-NEXT:    v_mov_b32_e32 v8, v0
+; GCN1-NEXT:    v_sub_i32_e32 v6, vcc, v8, v2
+; GCN1-NEXT:    v_subb_u32_e32 v7, vcc, v9, v3, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB33_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr2
+; GCN1-NEXT:    ; implicit-def: $vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB33_2
-; GCN1-NEXT:  .LBB33_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB33_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -4058,21 +4231,40 @@ define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB33_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB33_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB33_6
 ; GCN2-NEXT:  .LBB33_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB33_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB33_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v9, v1
+; GCN2-NEXT:    v_mov_b32_e32 v8, v0
+; GCN2-NEXT:    v_sub_u32_e32 v6, vcc, v8, v2
+; GCN2-NEXT:    v_subb_u32_e32 v7, vcc, v9, v3, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB33_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr2
+; GCN2-NEXT:    ; implicit-def: $vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB33_2
-; GCN2-NEXT:  .LBB33_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB33_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -4099,21 +4291,37 @@ define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB33_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB33_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB33_6
 ; GCN3-NEXT:  .LBB33_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB33_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_sub_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB33_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_sub_co_u32_e32 v6, vcc, v8, v2
+; GCN3-NEXT:    v_subb_co_u32_e32 v7, vcc, v9, v3, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB33_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2
+; GCN3-NEXT:    ; implicit-def: $vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB33_2
-; GCN3-NEXT:  .LBB33_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB33_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -4144,21 +4352,40 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN1-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB34_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB34_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB34_6
 ; GCN1-NEXT:  .LBB34_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB34_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v4, s4
+; GCN1-NEXT:    v_mov_b32_e32 v5, s5
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:    v_mov_b32_e32 v6, s7
+; GCN1-NEXT:  .LBB34_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v2
+; GCN1-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB34_2
-; GCN1-NEXT:  .LBB34_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB34_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB34_2
+; GCN1-NEXT:  .LBB34_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    v_mov_b32_e32 v4, s7
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
@@ -4188,21 +4415,40 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN2-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB34_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB34_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB34_6
 ; GCN2-NEXT:  .LBB34_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB34_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v4, s4
+; GCN2-NEXT:    v_mov_b32_e32 v5, s5
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:    v_mov_b32_e32 v6, s7
+; GCN2-NEXT:  .LBB34_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_subrev_u32_e32 v0, vcc, s6, v2
+; GCN2-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB34_2
-; GCN2-NEXT:  .LBB34_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB34_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB34_2
+; GCN2-NEXT:  .LBB34_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
@@ -4229,21 +4475,35 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN3-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB34_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB34_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB34_6
 ; GCN3-NEXT:  .LBB34_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB34_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s4
+; GCN3-NEXT:    v_mov_b32_e32 v5, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v6, s7
+; GCN3-NEXT:  .LBB34_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_subrev_co_u32_e32 v0, vcc, s6, v2
+; GCN3-NEXT:    v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB34_2
-; GCN3-NEXT:  .LBB34_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB34_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB34_2
+; GCN3-NEXT:  .LBB34_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s34
@@ -4276,21 +4536,40 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN1-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB35_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB35_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB35_6
 ; GCN1-NEXT:  .LBB35_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB35_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v4, s34
+; GCN1-NEXT:    v_mov_b32_e32 v5, s35
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:    v_mov_b32_e32 v6, s7
+; GCN1-NEXT:  .LBB35_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v2
+; GCN1-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB35_2
-; GCN1-NEXT:  .LBB35_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB35_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB35_2
+; GCN1-NEXT:  .LBB35_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    v_mov_b32_e32 v4, s7
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
@@ -4322,21 +4601,40 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN2-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB35_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB35_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB35_6
 ; GCN2-NEXT:  .LBB35_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB35_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v4, s34
+; GCN2-NEXT:    v_mov_b32_e32 v5, s35
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:    v_mov_b32_e32 v6, s7
+; GCN2-NEXT:  .LBB35_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_subrev_u32_e32 v0, vcc, s6, v2
+; GCN2-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB35_2
-; GCN2-NEXT:  .LBB35_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB35_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB35_2
+; GCN2-NEXT:  .LBB35_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
@@ -4365,21 +4663,35 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN3-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB35_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB35_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB35_6
 ; GCN3-NEXT:  .LBB35_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB35_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s34
+; GCN3-NEXT:    v_mov_b32_e32 v5, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:    v_mov_b32_e32 v6, s7
+; GCN3-NEXT:  .LBB35_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_subrev_co_u32_e32 v0, vcc, s6, v2
+; GCN3-NEXT:    v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB35_2
-; GCN3-NEXT:  .LBB35_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB35_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB35_2
+; GCN3-NEXT:  .LBB35_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s34
@@ -4409,20 +4721,39 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN1-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN1-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB36_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB36_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v2, s4
+; GCN1-NEXT:    v_mov_b32_e32 v3, s5
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:    v_mov_b32_e32 v4, s7
+; GCN1-NEXT:  .LBB36_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v8, v1
+; GCN1-NEXT:    v_mov_b32_e32 v7, v0
+; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s6, v7
+; GCN1-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB36_3
-; GCN1-NEXT:    s_branch .LBB36_4
-; GCN1-NEXT:  .LBB36_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB36_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB36_6
+; GCN1-NEXT:  .LBB36_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB36_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB36_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    v_mov_b32_e32 v4, s7
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
@@ -4438,7 +4769,7 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
 ; GCN1-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB36_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB36_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4451,20 +4782,39 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN2-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN2-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB36_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB36_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v2, s4
+; GCN2-NEXT:    v_mov_b32_e32 v3, s5
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:    v_mov_b32_e32 v4, s7
+; GCN2-NEXT:  .LBB36_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v8, v1
+; GCN2-NEXT:    v_mov_b32_e32 v7, v0
+; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s6, v7
+; GCN2-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB36_3
-; GCN2-NEXT:    s_branch .LBB36_4
-; GCN2-NEXT:  .LBB36_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB36_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB36_6
+; GCN2-NEXT:  .LBB36_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB36_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB36_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -4479,7 +4829,7 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
 ; GCN2-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB36_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB36_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4490,20 +4840,34 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN3-NEXT:    s_cmp_eq_u32 s5, s35
 ; GCN3-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB36_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB36_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s4
+; GCN3-NEXT:    v_mov_b32_e32 v3, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:    v_mov_b32_e32 v4, s7
+; GCN3-NEXT:  .LBB36_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v8, v1
+; GCN3-NEXT:    v_mov_b32_e32 v7, v0
+; GCN3-NEXT:    v_subrev_co_u32_e32 v5, vcc, s6, v7
+; GCN3-NEXT:    v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB36_3
-; GCN3-NEXT:    s_branch .LBB36_4
-; GCN3-NEXT:  .LBB36_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB36_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB36_6
+; GCN3-NEXT:  .LBB36_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB36_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB36_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -4516,7 +4880,7 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN3-NEXT:    v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB36_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB36_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw sub ptr %ptr, i64 %in seq_cst
@@ -4535,20 +4899,39 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN1-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN1-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB37_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB37_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v2, s34
+; GCN1-NEXT:    v_mov_b32_e32 v3, s35
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:    v_mov_b32_e32 v4, s7
+; GCN1-NEXT:  .LBB37_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v8, v1
+; GCN1-NEXT:    v_mov_b32_e32 v7, v0
+; GCN1-NEXT:    v_subrev_i32_e32 v5, vcc, s6, v7
+; GCN1-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB37_3
-; GCN1-NEXT:    s_branch .LBB37_4
-; GCN1-NEXT:  .LBB37_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB37_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB37_6
+; GCN1-NEXT:  .LBB37_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB37_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB37_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    v_mov_b32_e32 v4, s7
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
@@ -4564,7 +4947,7 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN1-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
 ; GCN1-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB37_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB37_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4579,20 +4962,39 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN2-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN2-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB37_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB37_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v2, s34
+; GCN2-NEXT:    v_mov_b32_e32 v3, s35
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:    v_mov_b32_e32 v4, s7
+; GCN2-NEXT:  .LBB37_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v8, v1
+; GCN2-NEXT:    v_mov_b32_e32 v7, v0
+; GCN2-NEXT:    v_subrev_u32_e32 v5, vcc, s6, v7
+; GCN2-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB37_3
-; GCN2-NEXT:    s_branch .LBB37_4
-; GCN2-NEXT:  .LBB37_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB37_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB37_6
+; GCN2-NEXT:  .LBB37_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB37_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB37_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -4607,7 +5009,7 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN2-NEXT:    v_subb_u32_e32 v4, vcc, v1, v4, vcc
 ; GCN2-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB37_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB37_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -4620,20 +5022,34 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN3-NEXT:    s_cmp_eq_u32 s35, s37
 ; GCN3-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB37_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB37_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s34
+; GCN3-NEXT:    v_mov_b32_e32 v3, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:    v_mov_b32_e32 v4, s7
+; GCN3-NEXT:  .LBB37_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v8, v1
+; GCN3-NEXT:    v_mov_b32_e32 v7, v0
+; GCN3-NEXT:    v_subrev_co_u32_e32 v5, vcc, s6, v7
+; GCN3-NEXT:    v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB37_3
-; GCN3-NEXT:    s_branch .LBB37_4
-; GCN3-NEXT:  .LBB37_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB37_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB37_6
+; GCN3-NEXT:  .LBB37_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB37_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB37_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -4646,7 +5062,7 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN3-NEXT:    v_subb_co_u32_e32 v3, vcc, v1, v3, vcc
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB37_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB37_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
@@ -4928,21 +5344,40 @@ define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB40_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB40_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB40_6
 ; GCN1-NEXT:  .LBB40_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB40_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB40_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_and_b32_e32 v5, v7, v3
+; GCN1-NEXT:    v_and_b32_e32 v4, v6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB40_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB40_2
-; GCN1-NEXT:  .LBB40_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB40_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
@@ -4968,21 +5403,40 @@ define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB40_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB40_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB40_6
 ; GCN2-NEXT:  .LBB40_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB40_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB40_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_and_b32_e32 v5, v7, v3
+; GCN2-NEXT:    v_and_b32_e32 v4, v6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB40_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB40_2
-; GCN2-NEXT:  .LBB40_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB40_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
@@ -5006,21 +5460,37 @@ define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB40_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB40_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB40_6
 ; GCN3-NEXT:  .LBB40_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB40_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB40_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_and_b32_e32 v5, v7, v3
+; GCN3-NEXT:    v_and_b32_e32 v4, v6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB40_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB40_2
-; GCN3-NEXT:  .LBB40_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB40_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
@@ -5051,21 +5521,40 @@ define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB41_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB41_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB41_6
 ; GCN1-NEXT:  .LBB41_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB41_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB41_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_and_b32_e32 v5, v7, v3
+; GCN1-NEXT:    v_and_b32_e32 v4, v6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB41_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB41_2
-; GCN1-NEXT:  .LBB41_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB41_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
@@ -5093,21 +5582,40 @@ define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB41_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB41_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB41_6
 ; GCN2-NEXT:  .LBB41_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB41_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB41_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_and_b32_e32 v5, v7, v3
+; GCN2-NEXT:    v_and_b32_e32 v4, v6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB41_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB41_2
-; GCN2-NEXT:  .LBB41_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB41_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
@@ -5133,21 +5641,37 @@ define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB41_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB41_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB41_6
 ; GCN3-NEXT:  .LBB41_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB41_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB41_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_and_b32_e32 v5, v7, v3
+; GCN3-NEXT:    v_and_b32_e32 v4, v6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB41_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB41_2
-; GCN3-NEXT:  .LBB41_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB41_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
@@ -5172,41 +5696,56 @@ define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN1-NEXT:    v_mov_b32_e32 v5, v1
-; GCN1-NEXT:    v_mov_b32_e32 v4, v0
-; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB42_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB42_4
-; GCN1-NEXT:  .LBB42_2: ; %atomicrmw.phi
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-; GCN1-NEXT:  .LBB42_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    s_cbranch_execz .LBB42_4
+; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v5, v[4:5]
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB42_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    v_and_b32_e32 v5, v7, v3
+; GCN1-NEXT:    v_and_b32_e32 v4, v6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB42_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
+; GCN1-NEXT:  .LBB42_4: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execz .LBB42_2
-; GCN1-NEXT:  .LBB42_4: ; %atomicrmw.private
-; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
-; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN1-NEXT:    s_cbranch_execz .LBB42_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN1-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN1-NEXT:    s_waitcnt vmcnt(1)
-; GCN1-NEXT:    v_and_b32_e32 v2, v0, v2
+; GCN1-NEXT:    v_and_b32_e32 v2, v4, v2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
-; GCN1-NEXT:    v_and_b32_e32 v3, v1, v3
-; GCN1-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN1-NEXT:    v_and_b32_e32 v3, v5, v3
+; GCN1-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN1-NEXT:  .LBB42_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v4
+; GCN1-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5215,41 +5754,56 @@ define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN2-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN2-NEXT:    v_mov_b32_e32 v5, v1
-; GCN2-NEXT:    v_mov_b32_e32 v4, v0
-; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB42_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB42_4
-; GCN2-NEXT:  .LBB42_2: ; %atomicrmw.phi
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-; GCN2-NEXT:  .LBB42_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    s_cbranch_execz .LBB42_4
+; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v5, v[4:5]
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB42_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
-; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    v_and_b32_e32 v5, v7, v3
+; GCN2-NEXT:    v_and_b32_e32 v4, v6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB42_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
+; GCN2-NEXT:  .LBB42_4: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execz .LBB42_2
-; GCN2-NEXT:  .LBB42_4: ; %atomicrmw.private
-; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
-; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN2-NEXT:    s_cbranch_execz .LBB42_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
+; GCN2-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN2-NEXT:    s_waitcnt vmcnt(1)
-; GCN2-NEXT:    v_and_b32_e32 v2, v0, v2
+; GCN2-NEXT:    v_and_b32_e32 v2, v4, v2
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
-; GCN2-NEXT:    v_and_b32_e32 v3, v1, v3
-; GCN2-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN2-NEXT:    v_and_b32_e32 v3, v5, v3
+; GCN2-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN2-NEXT:  .LBB42_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v4
+; GCN2-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5264,21 +5818,37 @@ define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB42_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB42_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB42_6
 ; GCN3-NEXT:  .LBB42_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB42_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB42_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_and_b32_e32 v7, v9, v3
+; GCN3-NEXT:    v_and_b32_e32 v6, v8, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB42_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB42_2
-; GCN3-NEXT:  .LBB42_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB42_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
@@ -5310,21 +5880,40 @@ define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB43_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB43_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB43_6
 ; GCN1-NEXT:  .LBB43_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB43_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB43_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v9, v1
+; GCN1-NEXT:    v_mov_b32_e32 v8, v0
+; GCN1-NEXT:    v_and_b32_e32 v7, v9, v3
+; GCN1-NEXT:    v_and_b32_e32 v6, v8, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB43_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB43_2
-; GCN1-NEXT:  .LBB43_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB43_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -5353,21 +5942,40 @@ define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB43_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB43_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB43_6
 ; GCN2-NEXT:  .LBB43_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB43_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB43_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v9, v1
+; GCN2-NEXT:    v_mov_b32_e32 v8, v0
+; GCN2-NEXT:    v_and_b32_e32 v7, v9, v3
+; GCN2-NEXT:    v_and_b32_e32 v6, v8, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB43_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB43_2
-; GCN2-NEXT:  .LBB43_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB43_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -5394,21 +6002,37 @@ define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB43_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB43_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB43_6
 ; GCN3-NEXT:  .LBB43_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB43_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_and_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB43_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_and_b32_e32 v7, v9, v3
+; GCN3-NEXT:    v_and_b32_e32 v6, v8, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB43_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB43_2
-; GCN3-NEXT:  .LBB43_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB43_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
@@ -5439,21 +6063,39 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN1-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB44_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB44_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB44_6
 ; GCN1-NEXT:  .LBB44_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB44_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v4, s4
+; GCN1-NEXT:    v_mov_b32_e32 v5, s5
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB44_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_and_b32_e32 v1, s7, v3
+; GCN1-NEXT:    v_and_b32_e32 v0, s6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB44_2
-; GCN1-NEXT:  .LBB44_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB44_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB44_2
+; GCN1-NEXT:  .LBB44_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s4, -1
@@ -5482,21 +6124,39 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN2-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB44_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB44_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB44_6
 ; GCN2-NEXT:  .LBB44_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB44_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v4, s4
+; GCN2-NEXT:    v_mov_b32_e32 v5, s5
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB44_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_and_b32_e32 v1, s7, v3
+; GCN2-NEXT:    v_and_b32_e32 v0, s6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB44_2
-; GCN2-NEXT:  .LBB44_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB44_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB44_2
+; GCN2-NEXT:  .LBB44_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
@@ -5522,21 +6182,34 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN3-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB44_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB44_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB44_6
 ; GCN3-NEXT:  .LBB44_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB44_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s4
+; GCN3-NEXT:    v_mov_b32_e32 v5, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB44_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_and_b32_e32 v1, s7, v3
+; GCN3-NEXT:    v_and_b32_e32 v0, s6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB44_2
-; GCN3-NEXT:  .LBB44_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB44_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB44_2
+; GCN3-NEXT:  .LBB44_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s34
@@ -5568,21 +6241,39 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN1-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB45_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB45_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB45_6
 ; GCN1-NEXT:  .LBB45_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB45_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v4, s34
+; GCN1-NEXT:    v_mov_b32_e32 v5, s35
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:  .LBB45_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_and_b32_e32 v1, s7, v3
+; GCN1-NEXT:    v_and_b32_e32 v0, s6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB45_2
-; GCN1-NEXT:  .LBB45_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB45_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB45_2
+; GCN1-NEXT:  .LBB45_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
@@ -5613,21 +6304,39 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN2-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB45_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB45_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB45_6
 ; GCN2-NEXT:  .LBB45_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB45_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v4, s34
+; GCN2-NEXT:    v_mov_b32_e32 v5, s35
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:  .LBB45_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_and_b32_e32 v1, s7, v3
+; GCN2-NEXT:    v_and_b32_e32 v0, s6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB45_2
-; GCN2-NEXT:  .LBB45_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB45_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB45_2
+; GCN2-NEXT:  .LBB45_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
@@ -5655,21 +6364,34 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN3-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB45_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB45_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB45_6
 ; GCN3-NEXT:  .LBB45_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB45_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s34
+; GCN3-NEXT:    v_mov_b32_e32 v5, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:  .LBB45_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_and_b32_e32 v1, s7, v3
+; GCN3-NEXT:    v_and_b32_e32 v0, s6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB45_2
-; GCN3-NEXT:  .LBB45_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB45_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB45_2
+; GCN3-NEXT:  .LBB45_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s34
@@ -5698,20 +6420,38 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN1-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN1-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB46_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB46_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v2, s4
+; GCN1-NEXT:    v_mov_b32_e32 v3, s5
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB46_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v1
+; GCN1-NEXT:    v_mov_b32_e32 v6, v0
+; GCN1-NEXT:    v_and_b32_e32 v5, s7, v7
+; GCN1-NEXT:    v_and_b32_e32 v4, s6, v6
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB46_3
-; GCN1-NEXT:    s_branch .LBB46_4
-; GCN1-NEXT:  .LBB46_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB46_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB46_6
+; GCN1-NEXT:  .LBB46_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB46_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB46_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s4, -1
@@ -5726,7 +6466,7 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN1-NEXT:    v_and_b32_e32 v5, s7, v1
 ; GCN1-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB46_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB46_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5739,20 +6479,38 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN2-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN2-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB46_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB46_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v2, s4
+; GCN2-NEXT:    v_mov_b32_e32 v3, s5
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB46_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v1
+; GCN2-NEXT:    v_mov_b32_e32 v6, v0
+; GCN2-NEXT:    v_and_b32_e32 v5, s7, v7
+; GCN2-NEXT:    v_and_b32_e32 v4, s6, v6
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB46_3
-; GCN2-NEXT:    s_branch .LBB46_4
-; GCN2-NEXT:  .LBB46_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB46_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB46_6
+; GCN2-NEXT:  .LBB46_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB46_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB46_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -5766,7 +6524,7 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN2-NEXT:    v_and_b32_e32 v5, s7, v1
 ; GCN2-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB46_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB46_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5777,20 +6535,33 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN3-NEXT:    s_cmp_eq_u32 s5, s35
 ; GCN3-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB46_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB46_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s4
+; GCN3-NEXT:    v_mov_b32_e32 v3, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB46_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v7, v1
+; GCN3-NEXT:    v_mov_b32_e32 v6, v0
+; GCN3-NEXT:    v_and_b32_e32 v5, s7, v7
+; GCN3-NEXT:    v_and_b32_e32 v4, s6, v6
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB46_3
-; GCN3-NEXT:    s_branch .LBB46_4
-; GCN3-NEXT:  .LBB46_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB46_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB46_6
+; GCN3-NEXT:  .LBB46_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB46_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB46_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -5802,7 +6573,7 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN3-NEXT:    v_and_b32_e32 v4, s6, v0
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB46_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB46_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw and ptr %ptr, i64 %in seq_cst
@@ -5821,20 +6592,38 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN1-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN1-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB47_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB47_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v2, s34
+; GCN1-NEXT:    v_mov_b32_e32 v3, s35
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:  .LBB47_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v1
+; GCN1-NEXT:    v_mov_b32_e32 v6, v0
+; GCN1-NEXT:    v_and_b32_e32 v5, s7, v7
+; GCN1-NEXT:    v_and_b32_e32 v4, s6, v6
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB47_3
-; GCN1-NEXT:    s_branch .LBB47_4
-; GCN1-NEXT:  .LBB47_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB47_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB47_6
+; GCN1-NEXT:  .LBB47_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB47_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB47_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
@@ -5849,7 +6638,7 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN1-NEXT:    v_and_b32_e32 v5, s7, v1
 ; GCN1-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB47_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB47_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5864,20 +6653,38 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN2-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN2-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB47_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB47_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v2, s34
+; GCN2-NEXT:    v_mov_b32_e32 v3, s35
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:  .LBB47_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v1
+; GCN2-NEXT:    v_mov_b32_e32 v6, v0
+; GCN2-NEXT:    v_and_b32_e32 v5, s7, v7
+; GCN2-NEXT:    v_and_b32_e32 v4, s6, v6
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB47_3
-; GCN2-NEXT:    s_branch .LBB47_4
-; GCN2-NEXT:  .LBB47_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB47_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB47_6
+; GCN2-NEXT:  .LBB47_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB47_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB47_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -5891,7 +6698,7 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN2-NEXT:    v_and_b32_e32 v5, s7, v1
 ; GCN2-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB47_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB47_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5904,20 +6711,33 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN3-NEXT:    s_cmp_eq_u32 s35, s37
 ; GCN3-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB47_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB47_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s34
+; GCN3-NEXT:    v_mov_b32_e32 v3, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:  .LBB47_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v7, v1
+; GCN3-NEXT:    v_mov_b32_e32 v6, v0
+; GCN3-NEXT:    v_and_b32_e32 v5, s7, v7
+; GCN3-NEXT:    v_and_b32_e32 v4, s6, v6
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB47_3
-; GCN3-NEXT:    s_branch .LBB47_4
-; GCN3-NEXT:  .LBB47_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB47_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB47_6
+; GCN3-NEXT:  .LBB47_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB47_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB47_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -5929,7 +6749,7 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN3-NEXT:    v_and_b32_e32 v4, s6, v0
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB47_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB47_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
@@ -8126,21 +8946,40 @@ define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB60_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB60_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB60_6
 ; GCN1-NEXT:  .LBB60_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB60_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB60_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_or_b32_e32 v5, v7, v3
+; GCN1-NEXT:    v_or_b32_e32 v4, v6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB60_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB60_2
-; GCN1-NEXT:  .LBB60_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB60_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
@@ -8166,21 +9005,40 @@ define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB60_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB60_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB60_6
 ; GCN2-NEXT:  .LBB60_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB60_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB60_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_or_b32_e32 v5, v7, v3
+; GCN2-NEXT:    v_or_b32_e32 v4, v6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB60_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB60_2
-; GCN2-NEXT:  .LBB60_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB60_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
@@ -8204,21 +9062,37 @@ define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB60_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB60_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB60_6
 ; GCN3-NEXT:  .LBB60_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB60_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB60_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_or_b32_e32 v5, v7, v3
+; GCN3-NEXT:    v_or_b32_e32 v4, v6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB60_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB60_2
-; GCN3-NEXT:  .LBB60_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB60_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
@@ -8249,21 +9123,40 @@ define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB61_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB61_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB61_6
 ; GCN1-NEXT:  .LBB61_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB61_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB61_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_or_b32_e32 v5, v7, v3
+; GCN1-NEXT:    v_or_b32_e32 v4, v6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB61_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB61_2
-; GCN1-NEXT:  .LBB61_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB61_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
@@ -8291,21 +9184,40 @@ define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB61_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB61_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB61_6
 ; GCN2-NEXT:  .LBB61_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB61_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB61_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_or_b32_e32 v5, v7, v3
+; GCN2-NEXT:    v_or_b32_e32 v4, v6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB61_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB61_2
-; GCN2-NEXT:  .LBB61_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB61_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
@@ -8331,21 +9243,37 @@ define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB61_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB61_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB61_6
 ; GCN3-NEXT:  .LBB61_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB61_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB61_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_or_b32_e32 v5, v7, v3
+; GCN3-NEXT:    v_or_b32_e32 v4, v6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB61_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB61_2
-; GCN3-NEXT:  .LBB61_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB61_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
@@ -8370,41 +9298,56 @@ define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN1-NEXT:    v_mov_b32_e32 v5, v1
-; GCN1-NEXT:    v_mov_b32_e32 v4, v0
-; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB62_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB62_4
-; GCN1-NEXT:  .LBB62_2: ; %atomicrmw.phi
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-; GCN1-NEXT:  .LBB62_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    s_cbranch_execz .LBB62_4
+; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v5, v[4:5]
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB62_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    v_or_b32_e32 v5, v7, v3
+; GCN1-NEXT:    v_or_b32_e32 v4, v6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB62_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
+; GCN1-NEXT:  .LBB62_4: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execz .LBB62_2
-; GCN1-NEXT:  .LBB62_4: ; %atomicrmw.private
-; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
-; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN1-NEXT:    s_cbranch_execz .LBB62_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN1-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN1-NEXT:    s_waitcnt vmcnt(1)
-; GCN1-NEXT:    v_or_b32_e32 v2, v0, v2
+; GCN1-NEXT:    v_or_b32_e32 v2, v4, v2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
-; GCN1-NEXT:    v_or_b32_e32 v3, v1, v3
-; GCN1-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN1-NEXT:    v_or_b32_e32 v3, v5, v3
+; GCN1-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN1-NEXT:  .LBB62_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v4
+; GCN1-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8413,41 +9356,56 @@ define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN2-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN2-NEXT:    v_mov_b32_e32 v5, v1
-; GCN2-NEXT:    v_mov_b32_e32 v4, v0
-; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB62_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB62_4
-; GCN2-NEXT:  .LBB62_2: ; %atomicrmw.phi
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-; GCN2-NEXT:  .LBB62_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    s_cbranch_execz .LBB62_4
+; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v5, v[4:5]
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB62_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    v_or_b32_e32 v5, v7, v3
+; GCN2-NEXT:    v_or_b32_e32 v4, v6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB62_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
+; GCN2-NEXT:  .LBB62_4: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execz .LBB62_2
-; GCN2-NEXT:  .LBB62_4: ; %atomicrmw.private
-; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
-; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN2-NEXT:    s_cbranch_execz .LBB62_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
+; GCN2-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN2-NEXT:    s_waitcnt vmcnt(1)
-; GCN2-NEXT:    v_or_b32_e32 v2, v0, v2
+; GCN2-NEXT:    v_or_b32_e32 v2, v4, v2
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
-; GCN2-NEXT:    v_or_b32_e32 v3, v1, v3
-; GCN2-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN2-NEXT:    v_or_b32_e32 v3, v5, v3
+; GCN2-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN2-NEXT:  .LBB62_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v4
+; GCN2-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8462,21 +9420,37 @@ define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB62_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB62_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB62_6
 ; GCN3-NEXT:  .LBB62_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB62_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB62_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_or_b32_e32 v7, v9, v3
+; GCN3-NEXT:    v_or_b32_e32 v6, v8, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB62_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB62_2
-; GCN3-NEXT:  .LBB62_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB62_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
@@ -8508,21 +9482,40 @@ define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB63_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB63_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB63_6
 ; GCN1-NEXT:  .LBB63_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB63_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB63_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v9, v1
+; GCN1-NEXT:    v_mov_b32_e32 v8, v0
+; GCN1-NEXT:    v_or_b32_e32 v7, v9, v3
+; GCN1-NEXT:    v_or_b32_e32 v6, v8, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB63_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB63_2
-; GCN1-NEXT:  .LBB63_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB63_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -8551,21 +9544,40 @@ define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB63_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB63_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB63_6
 ; GCN2-NEXT:  .LBB63_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB63_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB63_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v9, v1
+; GCN2-NEXT:    v_mov_b32_e32 v8, v0
+; GCN2-NEXT:    v_or_b32_e32 v7, v9, v3
+; GCN2-NEXT:    v_or_b32_e32 v6, v8, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB63_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB63_2
-; GCN2-NEXT:  .LBB63_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB63_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -8592,21 +9604,37 @@ define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB63_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB63_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB63_6
 ; GCN3-NEXT:  .LBB63_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB63_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_or_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB63_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_or_b32_e32 v7, v9, v3
+; GCN3-NEXT:    v_or_b32_e32 v6, v8, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB63_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB63_2
-; GCN3-NEXT:  .LBB63_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB63_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
@@ -8637,21 +9665,39 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN1-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB64_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB64_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB64_6
 ; GCN1-NEXT:  .LBB64_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB64_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v4, s4
+; GCN1-NEXT:    v_mov_b32_e32 v5, s5
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB64_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_or_b32_e32 v1, s7, v3
+; GCN1-NEXT:    v_or_b32_e32 v0, s6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB64_2
-; GCN1-NEXT:  .LBB64_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB64_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB64_2
+; GCN1-NEXT:  .LBB64_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s4, -1
@@ -8680,21 +9726,39 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN2-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB64_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB64_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB64_6
 ; GCN2-NEXT:  .LBB64_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB64_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v4, s4
+; GCN2-NEXT:    v_mov_b32_e32 v5, s5
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB64_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_or_b32_e32 v1, s7, v3
+; GCN2-NEXT:    v_or_b32_e32 v0, s6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB64_2
-; GCN2-NEXT:  .LBB64_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB64_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB64_2
+; GCN2-NEXT:  .LBB64_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
@@ -8720,21 +9784,34 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN3-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB64_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB64_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB64_6
 ; GCN3-NEXT:  .LBB64_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB64_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s4
+; GCN3-NEXT:    v_mov_b32_e32 v5, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB64_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_or_b32_e32 v1, s7, v3
+; GCN3-NEXT:    v_or_b32_e32 v0, s6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB64_2
-; GCN3-NEXT:  .LBB64_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB64_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB64_2
+; GCN3-NEXT:  .LBB64_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s34
@@ -8766,21 +9843,39 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN1-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB65_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB65_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB65_6
 ; GCN1-NEXT:  .LBB65_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB65_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v4, s34
+; GCN1-NEXT:    v_mov_b32_e32 v5, s35
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:  .LBB65_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_or_b32_e32 v1, s7, v3
+; GCN1-NEXT:    v_or_b32_e32 v0, s6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB65_2
-; GCN1-NEXT:  .LBB65_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB65_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB65_2
+; GCN1-NEXT:  .LBB65_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
@@ -8811,21 +9906,39 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN2-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB65_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB65_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB65_6
 ; GCN2-NEXT:  .LBB65_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB65_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v4, s34
+; GCN2-NEXT:    v_mov_b32_e32 v5, s35
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:  .LBB65_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_or_b32_e32 v1, s7, v3
+; GCN2-NEXT:    v_or_b32_e32 v0, s6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB65_2
-; GCN2-NEXT:  .LBB65_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB65_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB65_2
+; GCN2-NEXT:  .LBB65_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
@@ -8853,21 +9966,34 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN3-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB65_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB65_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB65_6
 ; GCN3-NEXT:  .LBB65_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB65_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s34
+; GCN3-NEXT:    v_mov_b32_e32 v5, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:  .LBB65_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_or_b32_e32 v1, s7, v3
+; GCN3-NEXT:    v_or_b32_e32 v0, s6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB65_2
-; GCN3-NEXT:  .LBB65_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB65_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB65_2
+; GCN3-NEXT:  .LBB65_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s34
@@ -8896,20 +10022,38 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
 ; GCN1-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN1-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB66_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB66_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v2, s4
+; GCN1-NEXT:    v_mov_b32_e32 v3, s5
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB66_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v1
+; GCN1-NEXT:    v_mov_b32_e32 v6, v0
+; GCN1-NEXT:    v_or_b32_e32 v5, s7, v7
+; GCN1-NEXT:    v_or_b32_e32 v4, s6, v6
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB66_3
-; GCN1-NEXT:    s_branch .LBB66_4
-; GCN1-NEXT:  .LBB66_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB66_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB66_6
+; GCN1-NEXT:  .LBB66_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB66_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB66_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s4, -1
@@ -8924,7 +10068,7 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
 ; GCN1-NEXT:    v_or_b32_e32 v5, s7, v1
 ; GCN1-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB66_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB66_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8937,20 +10081,38 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
 ; GCN2-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN2-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB66_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB66_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v2, s4
+; GCN2-NEXT:    v_mov_b32_e32 v3, s5
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB66_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v1
+; GCN2-NEXT:    v_mov_b32_e32 v6, v0
+; GCN2-NEXT:    v_or_b32_e32 v5, s7, v7
+; GCN2-NEXT:    v_or_b32_e32 v4, s6, v6
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB66_3
-; GCN2-NEXT:    s_branch .LBB66_4
-; GCN2-NEXT:  .LBB66_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB66_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB66_6
+; GCN2-NEXT:  .LBB66_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB66_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB66_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -8964,7 +10126,7 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
 ; GCN2-NEXT:    v_or_b32_e32 v5, s7, v1
 ; GCN2-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB66_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB66_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8975,20 +10137,33 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
 ; GCN3-NEXT:    s_cmp_eq_u32 s5, s35
 ; GCN3-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB66_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB66_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s4
+; GCN3-NEXT:    v_mov_b32_e32 v3, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB66_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v7, v1
+; GCN3-NEXT:    v_mov_b32_e32 v6, v0
+; GCN3-NEXT:    v_or_b32_e32 v5, s7, v7
+; GCN3-NEXT:    v_or_b32_e32 v4, s6, v6
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB66_3
-; GCN3-NEXT:    s_branch .LBB66_4
-; GCN3-NEXT:  .LBB66_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB66_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB66_6
+; GCN3-NEXT:  .LBB66_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB66_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB66_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -9000,7 +10175,7 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
 ; GCN3-NEXT:    v_or_b32_e32 v4, s6, v0
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB66_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB66_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw or ptr %ptr, i64 %in seq_cst
@@ -9019,20 +10194,38 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN1-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN1-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB67_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB67_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v2, s34
+; GCN1-NEXT:    v_mov_b32_e32 v3, s35
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:  .LBB67_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v1
+; GCN1-NEXT:    v_mov_b32_e32 v6, v0
+; GCN1-NEXT:    v_or_b32_e32 v5, s7, v7
+; GCN1-NEXT:    v_or_b32_e32 v4, s6, v6
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB67_3
-; GCN1-NEXT:    s_branch .LBB67_4
-; GCN1-NEXT:  .LBB67_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB67_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB67_6
+; GCN1-NEXT:  .LBB67_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB67_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB67_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
@@ -9047,7 +10240,7 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN1-NEXT:    v_or_b32_e32 v5, s7, v1
 ; GCN1-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB67_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB67_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -9062,20 +10255,38 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN2-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN2-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB67_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB67_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v2, s34
+; GCN2-NEXT:    v_mov_b32_e32 v3, s35
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:  .LBB67_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v1
+; GCN2-NEXT:    v_mov_b32_e32 v6, v0
+; GCN2-NEXT:    v_or_b32_e32 v5, s7, v7
+; GCN2-NEXT:    v_or_b32_e32 v4, s6, v6
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB67_3
-; GCN2-NEXT:    s_branch .LBB67_4
-; GCN2-NEXT:  .LBB67_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB67_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB67_6
+; GCN2-NEXT:  .LBB67_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB67_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB67_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -9089,7 +10300,7 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN2-NEXT:    v_or_b32_e32 v5, s7, v1
 ; GCN2-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB67_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB67_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -9102,20 +10313,33 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN3-NEXT:    s_cmp_eq_u32 s35, s37
 ; GCN3-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB67_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB67_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s34
+; GCN3-NEXT:    v_mov_b32_e32 v3, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:  .LBB67_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v7, v1
+; GCN3-NEXT:    v_mov_b32_e32 v6, v0
+; GCN3-NEXT:    v_or_b32_e32 v5, s7, v7
+; GCN3-NEXT:    v_or_b32_e32 v4, s6, v6
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB67_3
-; GCN3-NEXT:    s_branch .LBB67_4
-; GCN3-NEXT:  .LBB67_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB67_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB67_6
+; GCN3-NEXT:  .LBB67_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB67_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB67_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -9127,7 +10351,7 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN3-NEXT:    v_or_b32_e32 v4, s6, v0
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB67_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB67_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
@@ -9409,21 +10633,40 @@ define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB70_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB70_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB70_6
 ; GCN1-NEXT:  .LBB70_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB70_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB70_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GCN1-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB70_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB70_2
-; GCN1-NEXT:  .LBB70_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB70_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
@@ -9449,21 +10692,40 @@ define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB70_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB70_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB70_6
 ; GCN2-NEXT:  .LBB70_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB70_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB70_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GCN2-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB70_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB70_2
-; GCN2-NEXT:  .LBB70_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB70_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
@@ -9487,21 +10749,37 @@ define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB70_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB70_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB70_6
 ; GCN3-NEXT:  .LBB70_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB70_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB70_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GCN3-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB70_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB70_2
-; GCN3-NEXT:  .LBB70_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB70_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
@@ -9532,21 +10810,40 @@ define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB71_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB71_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB71_6
 ; GCN1-NEXT:  .LBB71_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB71_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB71_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GCN1-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB71_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB71_2
-; GCN1-NEXT:  .LBB71_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB71_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
@@ -9574,21 +10871,40 @@ define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB71_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB71_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB71_6
 ; GCN2-NEXT:  .LBB71_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB71_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB71_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GCN2-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB71_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB71_2
-; GCN2-NEXT:  .LBB71_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB71_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
@@ -9614,21 +10930,37 @@ define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB71_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB71_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB71_6
 ; GCN3-NEXT:  .LBB71_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB71_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB71_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GCN3-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB71_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB71_2
-; GCN3-NEXT:  .LBB71_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB71_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen offset:4
@@ -9653,41 +10985,56 @@ define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN1-NEXT:    v_mov_b32_e32 v5, v1
-; GCN1-NEXT:    v_mov_b32_e32 v4, v0
-; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB72_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB72_4
-; GCN1-NEXT:  .LBB72_2: ; %atomicrmw.phi
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-; GCN1-NEXT:  .LBB72_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    s_cbranch_execz .LBB72_4
+; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v5, v[4:5]
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB72_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GCN1-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB72_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
+; GCN1-NEXT:  .LBB72_4: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execz .LBB72_2
-; GCN1-NEXT:  .LBB72_4: ; %atomicrmw.private
-; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
-; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN1-NEXT:    s_cbranch_execz .LBB72_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN1-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN1-NEXT:    s_waitcnt vmcnt(1)
-; GCN1-NEXT:    v_xor_b32_e32 v2, v0, v2
+; GCN1-NEXT:    v_xor_b32_e32 v2, v4, v2
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
-; GCN1-NEXT:    v_xor_b32_e32 v3, v1, v3
-; GCN1-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN1-NEXT:    v_xor_b32_e32 v3, v5, v3
+; GCN1-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN1-NEXT:  .LBB72_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v4
+; GCN1-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -9696,41 +11043,56 @@ define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN2-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN2-NEXT:    v_mov_b32_e32 v5, v1
-; GCN2-NEXT:    v_mov_b32_e32 v4, v0
-; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB72_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB72_4
-; GCN2-NEXT:  .LBB72_2: ; %atomicrmw.phi
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-; GCN2-NEXT:  .LBB72_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    s_cbranch_execz .LBB72_4
+; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v5, v[4:5]
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB72_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GCN2-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB72_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
+; GCN2-NEXT:  .LBB72_4: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execz .LBB72_2
-; GCN2-NEXT:  .LBB72_4: ; %atomicrmw.private
-; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
-; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN2-NEXT:    s_cbranch_execz .LBB72_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
+; GCN2-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN2-NEXT:    s_waitcnt vmcnt(1)
-; GCN2-NEXT:    v_xor_b32_e32 v2, v0, v2
+; GCN2-NEXT:    v_xor_b32_e32 v2, v4, v2
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
-; GCN2-NEXT:    v_xor_b32_e32 v3, v1, v3
-; GCN2-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN2-NEXT:    v_xor_b32_e32 v3, v5, v3
+; GCN2-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN2-NEXT:  .LBB72_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v4
+; GCN2-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -9745,21 +11107,37 @@ define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB72_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB72_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB72_6
 ; GCN3-NEXT:  .LBB72_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB72_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB72_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_xor_b32_e32 v7, v9, v3
+; GCN3-NEXT:    v_xor_b32_e32 v6, v8, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB72_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB72_2
-; GCN3-NEXT:  .LBB72_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB72_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
@@ -9791,21 +11169,40 @@ define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB73_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB73_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB73_6
 ; GCN1-NEXT:  .LBB73_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB73_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB73_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v9, v1
+; GCN1-NEXT:    v_mov_b32_e32 v8, v0
+; GCN1-NEXT:    v_xor_b32_e32 v7, v9, v3
+; GCN1-NEXT:    v_xor_b32_e32 v6, v8, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB73_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr3
+; GCN1-NEXT:    ; implicit-def: $vgpr2
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB73_2
-; GCN1-NEXT:  .LBB73_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB73_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -9834,21 +11231,40 @@ define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB73_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB73_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB73_6
 ; GCN2-NEXT:  .LBB73_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB73_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB73_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v9, v1
+; GCN2-NEXT:    v_mov_b32_e32 v8, v0
+; GCN2-NEXT:    v_xor_b32_e32 v7, v9, v3
+; GCN2-NEXT:    v_xor_b32_e32 v6, v8, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB73_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr3
+; GCN2-NEXT:    ; implicit-def: $vgpr2
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB73_2
-; GCN2-NEXT:  .LBB73_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB73_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -9875,21 +11291,37 @@ define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB73_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB73_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB73_6
 ; GCN3-NEXT:  .LBB73_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB73_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_xor_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB73_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_xor_b32_e32 v7, v9, v3
+; GCN3-NEXT:    v_xor_b32_e32 v6, v8, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB73_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr3
+; GCN3-NEXT:    ; implicit-def: $vgpr2
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB73_2
-; GCN3-NEXT:  .LBB73_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB73_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
@@ -9920,21 +11352,39 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN1-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB74_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB74_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB74_6
 ; GCN1-NEXT:  .LBB74_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB74_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v4, s4
+; GCN1-NEXT:    v_mov_b32_e32 v5, s5
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB74_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GCN1-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB74_2
-; GCN1-NEXT:  .LBB74_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB74_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB74_2
+; GCN1-NEXT:  .LBB74_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s4, -1
@@ -9963,21 +11413,39 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN2-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB74_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB74_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB74_6
 ; GCN2-NEXT:  .LBB74_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB74_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v4, s4
+; GCN2-NEXT:    v_mov_b32_e32 v5, s5
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB74_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GCN2-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB74_2
-; GCN2-NEXT:  .LBB74_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB74_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB74_2
+; GCN2-NEXT:  .LBB74_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
@@ -10003,21 +11471,34 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN3-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB74_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB74_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB74_6
 ; GCN3-NEXT:  .LBB74_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB74_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s4
+; GCN3-NEXT:    v_mov_b32_e32 v5, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB74_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GCN3-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB74_2
-; GCN3-NEXT:  .LBB74_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB74_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB74_2
+; GCN3-NEXT:  .LBB74_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s34
@@ -10049,21 +11530,39 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN1-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB75_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB75_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB75_6
 ; GCN1-NEXT:  .LBB75_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB75_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v4, s34
+; GCN1-NEXT:    v_mov_b32_e32 v5, s35
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:  .LBB75_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GCN1-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB75_2
-; GCN1-NEXT:  .LBB75_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB75_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB75_2
+; GCN1-NEXT:  .LBB75_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
@@ -10094,21 +11593,39 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN2-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB75_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB75_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB75_6
 ; GCN2-NEXT:  .LBB75_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB75_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v4, s34
+; GCN2-NEXT:    v_mov_b32_e32 v5, s35
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:  .LBB75_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GCN2-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB75_2
-; GCN2-NEXT:  .LBB75_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB75_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB75_2
+; GCN2-NEXT:  .LBB75_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
@@ -10136,21 +11653,34 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN3-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB75_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB75_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB75_6
 ; GCN3-NEXT:  .LBB75_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB75_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s34
+; GCN3-NEXT:    v_mov_b32_e32 v5, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:  .LBB75_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GCN3-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB75_2
-; GCN3-NEXT:  .LBB75_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB75_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB75_2
+; GCN3-NEXT:  .LBB75_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v0, s34
@@ -10179,20 +11709,38 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN1-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN1-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB76_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB76_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v2, s4
+; GCN1-NEXT:    v_mov_b32_e32 v3, s5
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB76_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v1
+; GCN1-NEXT:    v_mov_b32_e32 v6, v0
+; GCN1-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GCN1-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB76_3
-; GCN1-NEXT:    s_branch .LBB76_4
-; GCN1-NEXT:  .LBB76_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB76_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB76_6
+; GCN1-NEXT:  .LBB76_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB76_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB76_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s4, -1
@@ -10207,7 +11755,7 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN1-NEXT:    v_xor_b32_e32 v5, s7, v1
 ; GCN1-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB76_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB76_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -10220,20 +11768,38 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN2-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN2-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB76_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB76_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v2, s4
+; GCN2-NEXT:    v_mov_b32_e32 v3, s5
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB76_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v1
+; GCN2-NEXT:    v_mov_b32_e32 v6, v0
+; GCN2-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GCN2-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB76_3
-; GCN2-NEXT:    s_branch .LBB76_4
-; GCN2-NEXT:  .LBB76_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB76_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB76_6
+; GCN2-NEXT:  .LBB76_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB76_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB76_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -10247,7 +11813,7 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN2-NEXT:    v_xor_b32_e32 v5, s7, v1
 ; GCN2-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB76_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB76_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -10257,21 +11823,34 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN3-NEXT:    s_mov_b64 s[34:35], src_private_base
 ; GCN3-NEXT:    s_cmp_eq_u32 s5, s35
 ; GCN3-NEXT:    s_cselect_b64 s[34:35], -1, 0
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB76_2
-; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_vccz .LBB76_4
+; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN3-NEXT:    v_mov_b32_e32 v2, s4
+; GCN3-NEXT:    v_mov_b32_e32 v3, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB76_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v7, v1
+; GCN3-NEXT:    v_mov_b32_e32 v6, v0
+; GCN3-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GCN3-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB76_3
-; GCN3-NEXT:    s_branch .LBB76_4
-; GCN3-NEXT:  .LBB76_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB76_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB76_6
+; GCN3-NEXT:  .LBB76_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB76_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB76_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -10283,7 +11862,7 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GCN3-NEXT:    v_xor_b32_e32 v4, s6, v0
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB76_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB76_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw xor ptr %ptr, i64 %in seq_cst
@@ -10302,20 +11881,38 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN1-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN1-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB77_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB77_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v2, s34
+; GCN1-NEXT:    v_mov_b32_e32 v3, s35
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:  .LBB77_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v1
+; GCN1-NEXT:    v_mov_b32_e32 v6, v0
+; GCN1-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GCN1-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB77_3
-; GCN1-NEXT:    s_branch .LBB77_4
-; GCN1-NEXT:  .LBB77_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB77_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB77_6
+; GCN1-NEXT:  .LBB77_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB77_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB77_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
@@ -10330,7 +11927,7 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN1-NEXT:    v_xor_b32_e32 v5, s7, v1
 ; GCN1-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB77_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB77_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -10345,20 +11942,38 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN2-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN2-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB77_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB77_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v2, s34
+; GCN2-NEXT:    v_mov_b32_e32 v3, s35
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:  .LBB77_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v1
+; GCN2-NEXT:    v_mov_b32_e32 v6, v0
+; GCN2-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GCN2-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB77_3
-; GCN2-NEXT:    s_branch .LBB77_4
-; GCN2-NEXT:  .LBB77_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB77_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB77_6
+; GCN2-NEXT:  .LBB77_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB77_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB77_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -10372,7 +11987,7 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN2-NEXT:    v_xor_b32_e32 v5, s7, v1
 ; GCN2-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB77_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB77_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -10385,20 +12000,33 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN3-NEXT:    s_cmp_eq_u32 s35, s37
 ; GCN3-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB77_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB77_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s34
+; GCN3-NEXT:    v_mov_b32_e32 v3, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:  .LBB77_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v7, v1
+; GCN3-NEXT:    v_mov_b32_e32 v6, v0
+; GCN3-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GCN3-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB77_3
-; GCN3-NEXT:    s_branch .LBB77_4
-; GCN3-NEXT:  .LBB77_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB77_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB77_6
+; GCN3-NEXT:  .LBB77_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB77_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB77_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -10410,7 +12038,7 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GCN3-NEXT:    v_xor_b32_e32 v4, s6, v0
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB77_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB77_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
@@ -13030,40 +14658,21 @@ define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out,
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB92_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:  ; %bb.1: ; %Flow
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB92_6
+; GCN1-NEXT:    s_cbranch_execnz .LBB92_4
 ; GCN1-NEXT:  .LBB92_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB92_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
-; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v7, v[4:5]
-; GCN1-NEXT:    flat_load_dword v6, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[6:7], 0
-; GCN1-NEXT:  .LBB92_4: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; GCN1-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN1-NEXT:    v_mov_b32_e32 v7, v5
-; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT:    v_mov_b32_e32 v6, v4
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN1-NEXT:    s_cbranch_execnz .LBB92_4
-; GCN1-NEXT:  ; %bb.5: ; %Flow
-; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB92_2
-; GCN1-NEXT:  .LBB92_6: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB92_4: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -13091,40 +14700,21 @@ define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out,
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB92_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:  ; %bb.1: ; %Flow
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB92_6
+; GCN2-NEXT:    s_cbranch_execnz .LBB92_4
 ; GCN2-NEXT:  .LBB92_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB92_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
-; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v7, v[4:5]
-; GCN2-NEXT:    flat_load_dword v6, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[6:7], 0
-; GCN2-NEXT:  .LBB92_4: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; GCN2-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN2-NEXT:    v_mov_b32_e32 v7, v5
-; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT:    v_mov_b32_e32 v6, v4
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN2-NEXT:    s_cbranch_execnz .LBB92_4
-; GCN2-NEXT:  ; %bb.5: ; %Flow
-; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB92_2
-; GCN2-NEXT:  .LBB92_6: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB92_4: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -13150,37 +14740,21 @@ define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out,
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB92_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:  ; %bb.1: ; %Flow
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB92_6
+; GCN3-NEXT:    s_cbranch_execnz .LBB92_4
 ; GCN3-NEXT:  .LBB92_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB92_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[6:7], 0
-; GCN3-NEXT:  .LBB92_4: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; GCN3-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN3-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN3-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN3-NEXT:    v_mov_b32_e32 v7, v5
-; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT:    v_mov_b32_e32 v6, v4
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT:    s_cbranch_execnz .LBB92_4
-; GCN3-NEXT:  ; %bb.5: ; %Flow
-; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB92_2
-; GCN3-NEXT:  .LBB92_6: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB92_4: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -13213,40 +14787,21 @@ define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i6
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB93_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:  ; %bb.1: ; %Flow
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB93_6
+; GCN1-NEXT:    s_cbranch_execnz .LBB93_4
 ; GCN1-NEXT:  .LBB93_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB93_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; GCN1-NEXT:    flat_load_dword v1, v[0:1]
-; GCN1-NEXT:    flat_load_dword v0, v[4:5]
-; GCN1-NEXT:    s_mov_b64 s[6:7], 0
-; GCN1-NEXT:  .LBB93_4: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v9, v1
-; GCN1-NEXT:    v_mov_b32_e32 v8, v0
-; GCN1-NEXT:    v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
-; GCN1-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN1-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN1-NEXT:    flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN1-NEXT:    s_cbranch_execnz .LBB93_4
-; GCN1-NEXT:  ; %bb.5: ; %Flow
-; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB93_2
-; GCN1-NEXT:  .LBB93_6: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB93_4: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -13275,40 +14830,21 @@ define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i6
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB93_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:  ; %bb.1: ; %Flow
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB93_6
+; GCN2-NEXT:    s_cbranch_execnz .LBB93_4
 ; GCN2-NEXT:  .LBB93_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB93_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; GCN2-NEXT:    flat_load_dword v1, v[0:1]
-; GCN2-NEXT:    flat_load_dword v0, v[4:5]
-; GCN2-NEXT:    s_mov_b64 s[6:7], 0
-; GCN2-NEXT:  .LBB93_4: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v9, v1
-; GCN2-NEXT:    v_mov_b32_e32 v8, v0
-; GCN2-NEXT:    v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
-; GCN2-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN2-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN2-NEXT:    flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN2-NEXT:    s_cbranch_execnz .LBB93_4
-; GCN2-NEXT:  ; %bb.5: ; %Flow
-; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB93_2
-; GCN2-NEXT:  .LBB93_6: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB93_4: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -13335,37 +14871,21 @@ define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i6
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB93_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:  ; %bb.1: ; %Flow
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB93_6
+; GCN3-NEXT:    s_cbranch_execnz .LBB93_4
 ; GCN3-NEXT:  .LBB93_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB93_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
-; GCN3-NEXT:    s_mov_b64 s[6:7], 0
-; GCN3-NEXT:  .LBB93_4: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v9, v1
-; GCN3-NEXT:    v_mov_b32_e32 v8, v0
-; GCN3-NEXT:    v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
-; GCN3-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN3-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT:    flat_atomic_smax_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT:    s_cbranch_execnz .LBB93_4
-; GCN3-NEXT:  ; %bb.5: ; %Flow
-; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB93_2
-; GCN3-NEXT:  .LBB93_6: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB93_4: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -15528,40 +17048,21 @@ define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB105_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:  ; %bb.1: ; %Flow
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB105_6
+; GCN1-NEXT:    s_cbranch_execnz .LBB105_4
 ; GCN1-NEXT:  .LBB105_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB105_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
-; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v7, v[4:5]
-; GCN1-NEXT:    flat_load_dword v6, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[6:7], 0
-; GCN1-NEXT:  .LBB105_4: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; GCN1-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN1-NEXT:    v_mov_b32_e32 v7, v5
-; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT:    v_mov_b32_e32 v6, v4
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN1-NEXT:    s_cbranch_execnz .LBB105_4
-; GCN1-NEXT:  ; %bb.5: ; %Flow
-; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB105_2
-; GCN1-NEXT:  .LBB105_6: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB105_4: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -15589,40 +17090,21 @@ define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB105_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:  ; %bb.1: ; %Flow
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB105_6
+; GCN2-NEXT:    s_cbranch_execnz .LBB105_4
 ; GCN2-NEXT:  .LBB105_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB105_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
-; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v7, v[4:5]
-; GCN2-NEXT:    flat_load_dword v6, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[6:7], 0
-; GCN2-NEXT:  .LBB105_4: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; GCN2-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN2-NEXT:    v_mov_b32_e32 v7, v5
-; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT:    v_mov_b32_e32 v6, v4
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN2-NEXT:    s_cbranch_execnz .LBB105_4
-; GCN2-NEXT:  ; %bb.5: ; %Flow
-; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    buffer_wbinvl1_vol
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB105_2
-; GCN2-NEXT:  .LBB105_6: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB105_4: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -15648,37 +17130,21 @@ define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB105_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:  ; %bb.1: ; %Flow
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB105_6
+; GCN3-NEXT:    s_cbranch_execnz .LBB105_4
 ; GCN3-NEXT:  .LBB105_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB105_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[6:7], 0
-; GCN3-NEXT:  .LBB105_4: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; GCN3-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN3-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN3-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN3-NEXT:    v_mov_b32_e32 v7, v5
-; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT:    v_mov_b32_e32 v6, v4
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT:    s_cbranch_execnz .LBB105_4
-; GCN3-NEXT:  ; %bb.5: ; %Flow
-; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB105_2
-; GCN3-NEXT:  .LBB105_6: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB105_4: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -15711,40 +17177,21 @@ define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB106_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:  ; %bb.1: ; %Flow
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB106_6
+; GCN1-NEXT:    s_cbranch_execnz .LBB106_4
 ; GCN1-NEXT:  .LBB106_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB106_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; GCN1-NEXT:    flat_load_dword v1, v[0:1]
-; GCN1-NEXT:    flat_load_dword v0, v[4:5]
-; GCN1-NEXT:    s_mov_b64 s[6:7], 0
-; GCN1-NEXT:  .LBB106_4: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v9, v1
-; GCN1-NEXT:    v_mov_b32_e32 v8, v0
-; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
-; GCN1-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN1-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN1-NEXT:    flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN1-NEXT:    s_cbranch_execnz .LBB106_4
-; GCN1-NEXT:  ; %bb.5: ; %Flow
-; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB106_2
-; GCN1-NEXT:  .LBB106_6: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB106_4: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -15773,40 +17220,21 @@ define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB106_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:  ; %bb.1: ; %Flow
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB106_6
+; GCN2-NEXT:    s_cbranch_execnz .LBB106_4
 ; GCN2-NEXT:  .LBB106_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB106_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; GCN2-NEXT:    flat_load_dword v1, v[0:1]
-; GCN2-NEXT:    flat_load_dword v0, v[4:5]
-; GCN2-NEXT:    s_mov_b64 s[6:7], 0
-; GCN2-NEXT:  .LBB106_4: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v9, v1
-; GCN2-NEXT:    v_mov_b32_e32 v8, v0
-; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
-; GCN2-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN2-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN2-NEXT:    flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN2-NEXT:    s_cbranch_execnz .LBB106_4
-; GCN2-NEXT:  ; %bb.5: ; %Flow
-; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB106_2
-; GCN2-NEXT:  .LBB106_6: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB106_4: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -15833,37 +17261,21 @@ define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB106_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:  ; %bb.1: ; %Flow
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB106_6
+; GCN3-NEXT:    s_cbranch_execnz .LBB106_4
 ; GCN3-NEXT:  .LBB106_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB106_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
-; GCN3-NEXT:    s_mov_b64 s[6:7], 0
-; GCN3-NEXT:  .LBB106_4: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v9, v1
-; GCN3-NEXT:    v_mov_b32_e32 v8, v0
-; GCN3-NEXT:    v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
-; GCN3-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN3-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT:    flat_atomic_umax_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT:    s_cbranch_execnz .LBB106_4
-; GCN3-NEXT:  ; %bb.5: ; %Flow
-; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB106_2
-; GCN3-NEXT:  .LBB106_6: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB106_4: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -17383,40 +18795,21 @@ define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB115_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:  ; %bb.1: ; %Flow
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB115_6
+; GCN1-NEXT:    s_cbranch_execnz .LBB115_4
 ; GCN1-NEXT:  .LBB115_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB115_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
-; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v7, v[4:5]
-; GCN1-NEXT:    flat_load_dword v6, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[6:7], 0
-; GCN1-NEXT:  .LBB115_4: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; GCN1-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN1-NEXT:    v_mov_b32_e32 v7, v5
-; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT:    v_mov_b32_e32 v6, v4
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN1-NEXT:    s_cbranch_execnz .LBB115_4
-; GCN1-NEXT:  ; %bb.5: ; %Flow
-; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB115_2
-; GCN1-NEXT:  .LBB115_6: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB115_4: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -17444,40 +18837,21 @@ define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB115_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:  ; %bb.1: ; %Flow
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB115_6
+; GCN2-NEXT:    s_cbranch_execnz .LBB115_4
 ; GCN2-NEXT:  .LBB115_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB115_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
-; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v7, v[4:5]
-; GCN2-NEXT:    flat_load_dword v6, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[6:7], 0
-; GCN2-NEXT:  .LBB115_4: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; GCN2-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN2-NEXT:    v_mov_b32_e32 v7, v5
-; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT:    v_mov_b32_e32 v6, v4
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN2-NEXT:    s_cbranch_execnz .LBB115_4
-; GCN2-NEXT:  ; %bb.5: ; %Flow
-; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB115_2
-; GCN2-NEXT:  .LBB115_6: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB115_4: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -17503,37 +18877,21 @@ define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB115_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:  ; %bb.1: ; %Flow
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB115_6
+; GCN3-NEXT:    s_cbranch_execnz .LBB115_4
 ; GCN3-NEXT:  .LBB115_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB115_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[6:7], 0
-; GCN3-NEXT:  .LBB115_4: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; GCN3-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN3-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN3-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN3-NEXT:    v_mov_b32_e32 v7, v5
-; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT:    v_mov_b32_e32 v6, v4
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT:    s_cbranch_execnz .LBB115_4
-; GCN3-NEXT:  ; %bb.5: ; %Flow
-; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB115_2
-; GCN3-NEXT:  .LBB115_6: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB115_4: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -17566,40 +18924,21 @@ define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB116_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:  ; %bb.1: ; %Flow
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB116_6
+; GCN1-NEXT:    s_cbranch_execnz .LBB116_4
 ; GCN1-NEXT:  .LBB116_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB116_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; GCN1-NEXT:    flat_load_dword v1, v[0:1]
-; GCN1-NEXT:    flat_load_dword v0, v[4:5]
-; GCN1-NEXT:    s_mov_b64 s[6:7], 0
-; GCN1-NEXT:  .LBB116_4: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v9, v1
-; GCN1-NEXT:    v_mov_b32_e32 v8, v0
-; GCN1-NEXT:    v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
-; GCN1-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN1-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN1-NEXT:    flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN1-NEXT:    s_cbranch_execnz .LBB116_4
-; GCN1-NEXT:  ; %bb.5: ; %Flow
-; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB116_2
-; GCN1-NEXT:  .LBB116_6: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB116_4: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -17628,40 +18967,21 @@ define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB116_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:  ; %bb.1: ; %Flow
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB116_6
+; GCN2-NEXT:    s_cbranch_execnz .LBB116_4
 ; GCN2-NEXT:  .LBB116_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB116_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; GCN2-NEXT:    flat_load_dword v1, v[0:1]
-; GCN2-NEXT:    flat_load_dword v0, v[4:5]
-; GCN2-NEXT:    s_mov_b64 s[6:7], 0
-; GCN2-NEXT:  .LBB116_4: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v9, v1
-; GCN2-NEXT:    v_mov_b32_e32 v8, v0
-; GCN2-NEXT:    v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
-; GCN2-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN2-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN2-NEXT:    flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN2-NEXT:    s_cbranch_execnz .LBB116_4
-; GCN2-NEXT:  ; %bb.5: ; %Flow
-; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB116_2
-; GCN2-NEXT:  .LBB116_6: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB116_4: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -17688,37 +19008,21 @@ define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB116_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:  ; %bb.1: ; %Flow
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB116_6
+; GCN3-NEXT:    s_cbranch_execnz .LBB116_4
 ; GCN3-NEXT:  .LBB116_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB116_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
-; GCN3-NEXT:    s_mov_b64 s[6:7], 0
-; GCN3-NEXT:  .LBB116_4: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v9, v1
-; GCN3-NEXT:    v_mov_b32_e32 v8, v0
-; GCN3-NEXT:    v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
-; GCN3-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN3-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT:    flat_atomic_umin_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT:    s_cbranch_execnz .LBB116_4
-; GCN3-NEXT:  ; %bb.5: ; %Flow
-; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB116_2
-; GCN3-NEXT:  .LBB116_6: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB116_4: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20077,40 +21381,21 @@ define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out,
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB129_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:  ; %bb.1: ; %Flow
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB129_6
+; GCN1-NEXT:    s_cbranch_execnz .LBB129_4
 ; GCN1-NEXT:  .LBB129_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB129_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
-; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN1-NEXT:    flat_load_dword v7, v[4:5]
-; GCN1-NEXT:    flat_load_dword v6, v[0:1]
-; GCN1-NEXT:    s_mov_b64 s[6:7], 0
-; GCN1-NEXT:  .LBB129_4: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; GCN1-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN1-NEXT:    v_mov_b32_e32 v7, v5
-; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT:    v_mov_b32_e32 v6, v4
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN1-NEXT:    s_cbranch_execnz .LBB129_4
-; GCN1-NEXT:  ; %bb.5: ; %Flow
-; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB129_2
-; GCN1-NEXT:  .LBB129_6: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB129_4: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -20138,40 +21423,21 @@ define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out,
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB129_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:  ; %bb.1: ; %Flow
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB129_6
+; GCN2-NEXT:    s_cbranch_execnz .LBB129_4
 ; GCN2-NEXT:  .LBB129_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB129_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
-; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GCN2-NEXT:    flat_load_dword v7, v[4:5]
-; GCN2-NEXT:    flat_load_dword v6, v[0:1]
-; GCN2-NEXT:    s_mov_b64 s[6:7], 0
-; GCN2-NEXT:  .LBB129_4: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; GCN2-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN2-NEXT:    v_mov_b32_e32 v7, v5
-; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT:    v_mov_b32_e32 v6, v4
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN2-NEXT:    s_cbranch_execnz .LBB129_4
-; GCN2-NEXT:  ; %bb.5: ; %Flow
-; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    buffer_wbinvl1_vol
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB129_2
-; GCN2-NEXT:  .LBB129_6: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB129_4: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -20197,37 +21463,21 @@ define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out,
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB129_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:  ; %bb.1: ; %Flow
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB129_6
+; GCN3-NEXT:    s_cbranch_execnz .LBB129_4
 ; GCN3-NEXT:  .LBB129_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB129_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
-; GCN3-NEXT:    s_mov_b64 s[6:7], 0
-; GCN3-NEXT:  .LBB129_4: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; GCN3-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GCN3-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GCN3-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GCN3-NEXT:    v_mov_b32_e32 v7, v5
-; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT:    v_mov_b32_e32 v6, v4
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT:    s_cbranch_execnz .LBB129_4
-; GCN3-NEXT:  ; %bb.5: ; %Flow
-; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB129_2
-; GCN3-NEXT:  .LBB129_6: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB129_4: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20260,40 +21510,21 @@ define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i6
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB130_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:  ; %bb.1: ; %Flow
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB130_6
+; GCN1-NEXT:    s_cbranch_execnz .LBB130_4
 ; GCN1-NEXT:  .LBB130_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB130_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
-; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; GCN1-NEXT:    flat_load_dword v1, v[0:1]
-; GCN1-NEXT:    flat_load_dword v0, v[4:5]
-; GCN1-NEXT:    s_mov_b64 s[6:7], 0
-; GCN1-NEXT:  .LBB130_4: ; %atomicrmw.start
-; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT:    v_mov_b32_e32 v9, v1
-; GCN1-NEXT:    v_mov_b32_e32 v8, v0
-; GCN1-NEXT:    v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
-; GCN1-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN1-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN1-NEXT:    flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN1-NEXT:    s_cbranch_execnz .LBB130_4
-; GCN1-NEXT:  ; %bb.5: ; %Flow
-; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB130_2
-; GCN1-NEXT:  .LBB130_6: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB130_4: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -20322,40 +21553,21 @@ define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i6
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB130_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:  ; %bb.1: ; %Flow
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB130_6
+; GCN2-NEXT:    s_cbranch_execnz .LBB130_4
 ; GCN2-NEXT:  .LBB130_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB130_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
-; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
-; GCN2-NEXT:    flat_load_dword v1, v[0:1]
-; GCN2-NEXT:    flat_load_dword v0, v[4:5]
-; GCN2-NEXT:    s_mov_b64 s[6:7], 0
-; GCN2-NEXT:  .LBB130_4: ; %atomicrmw.start
-; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT:    v_mov_b32_e32 v9, v1
-; GCN2-NEXT:    v_mov_b32_e32 v8, v0
-; GCN2-NEXT:    v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
-; GCN2-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN2-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN2-NEXT:    flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN2-NEXT:    s_cbranch_execnz .LBB130_4
-; GCN2-NEXT:  ; %bb.5: ; %Flow
-; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB130_2
-; GCN2-NEXT:  .LBB130_6: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB130_4: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -20382,37 +21594,21 @@ define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i6
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB130_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:  ; %bb.1: ; %Flow
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB130_6
+; GCN3-NEXT:    s_cbranch_execnz .LBB130_4
 ; GCN3-NEXT:  .LBB130_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB130_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
-; GCN3-NEXT:    s_mov_b64 s[6:7], 0
-; GCN3-NEXT:  .LBB130_4: ; %atomicrmw.start
-; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    v_mov_b32_e32 v9, v1
-; GCN3-NEXT:    v_mov_b32_e32 v8, v0
-; GCN3-NEXT:    v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
-; GCN3-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GCN3-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GCN3-NEXT:    flat_atomic_smin_x2 v[0:1], v[4:5], v[2:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
-; GCN3-NEXT:    s_cbranch_execnz .LBB130_4
-; GCN3-NEXT:  ; %bb.5: ; %Flow
-; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB130_2
-; GCN3-NEXT:  .LBB130_6: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB130_4: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20446,21 +21642,42 @@ define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB131_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB131_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB131_6
 ; GCN1-NEXT:  .LBB131_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB131_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB131_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v6
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB131_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB131_2
-; GCN1-NEXT:  .LBB131_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB131_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20489,21 +21706,42 @@ define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB131_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB131_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB131_6
 ; GCN2-NEXT:  .LBB131_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB131_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB131_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v6
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB131_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB131_2
-; GCN2-NEXT:  .LBB131_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB131_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20530,21 +21768,39 @@ define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB131_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB131_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB131_6
 ; GCN3-NEXT:  .LBB131_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB131_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB131_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GCN3-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GCN3-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN3-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB131_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB131_2
-; GCN3-NEXT:  .LBB131_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB131_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20578,21 +21834,42 @@ define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB132_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB132_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB132_6
 ; GCN1-NEXT:  .LBB132_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB132_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB132_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v6
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB132_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB132_2
-; GCN1-NEXT:  .LBB132_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB132_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20623,21 +21900,42 @@ define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB132_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB132_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB132_6
 ; GCN2-NEXT:  .LBB132_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB132_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB132_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v6
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB132_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB132_2
-; GCN2-NEXT:  .LBB132_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB132_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20666,21 +21964,39 @@ define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB132_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB132_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB132_6
 ; GCN3-NEXT:  .LBB132_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB132_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB132_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GCN3-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GCN3-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN3-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB132_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB132_2
-; GCN3-NEXT:  .LBB132_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB132_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20708,44 +22024,61 @@ define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN1-NEXT:    v_mov_b32_e32 v5, v1
-; GCN1-NEXT:    v_mov_b32_e32 v4, v0
-; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB133_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB133_4
-; GCN1-NEXT:  .LBB133_2: ; %atomicrmw.phi
-; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-; GCN1-NEXT:  .LBB133_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    s_cbranch_execz .LBB133_4
+; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v5, v[4:5]
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB133_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 1, v6
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB133_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GCN1-NEXT:  .LBB133_4: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execz .LBB133_2
-; GCN1-NEXT:  .LBB133_4: ; %atomicrmw.private
-; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
-; GCN1-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN1-NEXT:    s_cbranch_execz .LBB133_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN1-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN1-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN1-NEXT:    s_waitcnt vmcnt(1)
-; GCN1-NEXT:    v_add_i32_e32 v6, vcc, 1, v0
+; GCN1-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
-; GCN1-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_addc_u32_e32 v7, vcc, 0, v5, vcc
+; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[2:3]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
 ; GCN1-NEXT:    v_cndmask_b32_e32 v2, 0, v7, vcc
-; GCN1-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_store_dword v2, v5, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_store_dword v2, v1, s[0:3], 0 offen
+; GCN1-NEXT:  .LBB133_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v4
+; GCN1-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -20754,44 +22087,61 @@ define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN2-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN2-NEXT:    v_mov_b32_e32 v5, v1
-; GCN2-NEXT:    v_mov_b32_e32 v4, v0
-; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB133_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB133_4
-; GCN2-NEXT:  .LBB133_2: ; %atomicrmw.phi
-; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-; GCN2-NEXT:  .LBB133_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    s_cbranch_execz .LBB133_4
+; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v5, v[4:5]
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB133_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 1, v6
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB133_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GCN2-NEXT:  .LBB133_4: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execz .LBB133_2
-; GCN2-NEXT:  .LBB133_4: ; %atomicrmw.private
-; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
-; GCN2-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN2-NEXT:    s_cbranch_execz .LBB133_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN2-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
+; GCN2-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN2-NEXT:    s_waitcnt vmcnt(1)
-; GCN2-NEXT:    v_add_u32_e32 v6, vcc, 1, v0
+; GCN2-NEXT:    v_add_u32_e32 v6, vcc, 1, v4
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
-; GCN2-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_addc_u32_e32 v7, vcc, 0, v5, vcc
+; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, v[4:5], v[2:3]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v3, 0, v6, vcc
 ; GCN2-NEXT:    v_cndmask_b32_e32 v2, 0, v7, vcc
-; GCN2-NEXT:    buffer_store_dword v3, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_store_dword v2, v5, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_store_dword v3, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_store_dword v2, v1, s[0:3], 0 offen
+; GCN2-NEXT:  .LBB133_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v4
+; GCN2-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -20806,21 +22156,39 @@ define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB133_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB133_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB133_6
 ; GCN3-NEXT:  .LBB133_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB133_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB133_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v8
+; GCN3-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; GCN3-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
+; GCN3-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v6, 0, v0, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB133_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB133_2
-; GCN3-NEXT:  .LBB133_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB133_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20855,21 +22223,42 @@ define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB134_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB134_4
+; GCN1-NEXT:    s_cbranch_execnz .LBB134_6
 ; GCN1-NEXT:  .LBB134_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB134_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[6:7], 0
+; GCN1-NEXT:  .LBB134_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v9, v1
+; GCN1-NEXT:    v_mov_b32_e32 v8, v0
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v8
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v9, vcc
+; GCN1-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
+; GCN1-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v6, 0, v0, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT:    s_cbranch_execnz .LBB134_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN1-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN1-NEXT:    s_cbranch_execz .LBB134_2
-; GCN1-NEXT:  .LBB134_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB134_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20901,21 +22290,42 @@ define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB134_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB134_4
+; GCN2-NEXT:    s_cbranch_execnz .LBB134_6
 ; GCN2-NEXT:  .LBB134_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB134_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[6:7], 0
+; GCN2-NEXT:  .LBB134_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v9, v1
+; GCN2-NEXT:    v_mov_b32_e32 v8, v0
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v8
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v9, vcc
+; GCN2-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
+; GCN2-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v6, 0, v0, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT:    s_cbranch_execnz .LBB134_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN2-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN2-NEXT:    s_cbranch_execz .LBB134_2
-; GCN2-NEXT:  .LBB134_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB134_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20945,21 +22355,39 @@ define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB134_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB134_4
+; GCN3-NEXT:    s_cbranch_execnz .LBB134_6
 ; GCN3-NEXT:  .LBB134_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB134_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_inc_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[6:7], 0
+; GCN3-NEXT:  .LBB134_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v8
+; GCN3-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v9, vcc
+; GCN3-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
+; GCN3-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v6, 0, v0, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT:    s_cbranch_execnz .LBB134_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
 ; GCN3-NEXT:    s_andn2_saveexec_b64 s[4:5], s[4:5]
 ; GCN3-NEXT:    s_cbranch_execz .LBB134_2
-; GCN3-NEXT:  .LBB134_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB134_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -20993,21 +22421,42 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN1-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB135_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB135_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB135_6
 ; GCN1-NEXT:  .LBB135_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB135_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v4, s4
+; GCN1-NEXT:    v_mov_b32_e32 v5, s5
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB135_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB135_2
-; GCN1-NEXT:  .LBB135_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB135_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB135_2
+; GCN1-NEXT:  .LBB135_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s4, -1
@@ -21039,21 +22488,42 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN2-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB135_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB135_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB135_6
 ; GCN2-NEXT:  .LBB135_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB135_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v4, s4
+; GCN2-NEXT:    v_mov_b32_e32 v5, s5
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB135_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB135_2
-; GCN2-NEXT:  .LBB135_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB135_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB135_2
+; GCN2-NEXT:  .LBB135_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -21082,21 +22552,37 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN3-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB135_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB135_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB135_6
 ; GCN3-NEXT:  .LBB135_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB135_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s4
+; GCN3-NEXT:    v_mov_b32_e32 v5, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB135_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v2
+; GCN3-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GCN3-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN3-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB135_2
-; GCN3-NEXT:  .LBB135_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB135_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB135_2
+; GCN3-NEXT:  .LBB135_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -21131,21 +22617,42 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN1-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB136_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB136_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB136_6
 ; GCN1-NEXT:  .LBB136_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB136_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v4, s34
+; GCN1-NEXT:    v_mov_b32_e32 v5, s35
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:  .LBB136_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN1-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB136_2
-; GCN1-NEXT:  .LBB136_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB136_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB136_2
+; GCN1-NEXT:  .LBB136_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
@@ -21179,21 +22686,42 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN2-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB136_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB136_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB136_6
 ; GCN2-NEXT:  .LBB136_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB136_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v4, s34
+; GCN2-NEXT:    v_mov_b32_e32 v5, s35
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:  .LBB136_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN2-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB136_2
-; GCN2-NEXT:  .LBB136_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB136_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB136_2
+; GCN2-NEXT:  .LBB136_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -21224,21 +22752,37 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
 ; GCN3-NEXT:    s_mov_b64 s[36:37], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB136_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB136_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB136_6
 ; GCN3-NEXT:  .LBB136_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB136_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s34
+; GCN3-NEXT:    v_mov_b32_e32 v5, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:  .LBB136_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v2
+; GCN3-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GCN3-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN3-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB136_2
-; GCN3-NEXT:  .LBB136_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB136_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB136_2
+; GCN3-NEXT:  .LBB136_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -21270,20 +22814,41 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN1-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN1-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB137_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB137_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v2, s4
+; GCN1-NEXT:    v_mov_b32_e32 v3, s5
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[34:35], 0
+; GCN1-NEXT:  .LBB137_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v1
+; GCN1-NEXT:    v_mov_b32_e32 v6, v0
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v6
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB137_3
-; GCN1-NEXT:    s_branch .LBB137_4
-; GCN1-NEXT:  .LBB137_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_execnz .LBB137_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN1-NEXT:    s_branch .LBB137_6
+; GCN1-NEXT:  .LBB137_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB137_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB137_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s4, -1
@@ -21301,7 +22866,7 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
 ; GCN1-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB137_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB137_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -21314,20 +22879,41 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN2-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN2-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB137_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB137_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v2, s4
+; GCN2-NEXT:    v_mov_b32_e32 v3, s5
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[34:35], 0
+; GCN2-NEXT:  .LBB137_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v1
+; GCN2-NEXT:    v_mov_b32_e32 v6, v0
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v6
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB137_3
-; GCN2-NEXT:    s_branch .LBB137_4
-; GCN2-NEXT:  .LBB137_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_execnz .LBB137_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN2-NEXT:    s_branch .LBB137_6
+; GCN2-NEXT:  .LBB137_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB137_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB137_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -21344,7 +22930,7 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
 ; GCN2-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB137_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB137_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -21355,20 +22941,36 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN3-NEXT:    s_cmp_eq_u32 s5, s35
 ; GCN3-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB137_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB137_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s4
+; GCN3-NEXT:    v_mov_b32_e32 v3, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[34:35], 0
+; GCN3-NEXT:  .LBB137_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v7, v1
+; GCN3-NEXT:    v_mov_b32_e32 v6, v0
+; GCN3-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v6
+; GCN3-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GCN3-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN3-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB137_3
-; GCN3-NEXT:    s_branch .LBB137_4
-; GCN3-NEXT:  .LBB137_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN3-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_execnz .LBB137_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GCN3-NEXT:    s_branch .LBB137_6
+; GCN3-NEXT:  .LBB137_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB137_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB137_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -21383,7 +22985,7 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB137_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB137_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst
@@ -21402,20 +23004,41 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN1-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN1-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB138_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB138_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s36, s34, 4
+; GCN1-NEXT:    s_addc_u32 s37, s35, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s36
+; GCN1-NEXT:    v_mov_b32_e32 v1, s37
+; GCN1-NEXT:    v_mov_b32_e32 v2, s34
+; GCN1-NEXT:    v_mov_b32_e32 v3, s35
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[36:37], 0
+; GCN1-NEXT:  .LBB138_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v1
+; GCN1-NEXT:    v_mov_b32_e32 v6, v0
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 1, v6
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GCN1-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB138_3
-; GCN1-NEXT:    s_branch .LBB138_4
-; GCN1-NEXT:  .LBB138_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_cbranch_execnz .LBB138_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT:    s_branch .LBB138_6
+; GCN1-NEXT:  .LBB138_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB138_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB138_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
 ; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
 ; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
@@ -21433,7 +23056,7 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN1-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
 ; GCN1-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB138_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB138_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -21448,20 +23071,41 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN2-NEXT:    s_cmp_eq_u32 s35, s36
 ; GCN2-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB138_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB138_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s36, s34, 4
+; GCN2-NEXT:    s_addc_u32 s37, s35, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s36
+; GCN2-NEXT:    v_mov_b32_e32 v1, s37
+; GCN2-NEXT:    v_mov_b32_e32 v2, s34
+; GCN2-NEXT:    v_mov_b32_e32 v3, s35
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[36:37], 0
+; GCN2-NEXT:  .LBB138_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v1
+; GCN2-NEXT:    v_mov_b32_e32 v6, v0
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 1, v6
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GCN2-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB138_3
-; GCN2-NEXT:    s_branch .LBB138_4
-; GCN2-NEXT:  .LBB138_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_cbranch_execnz .LBB138_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT:    s_branch .LBB138_6
+; GCN2-NEXT:  .LBB138_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB138_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB138_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -21478,7 +23122,7 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN2-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
 ; GCN2-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v5, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB138_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB138_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -21491,20 +23135,36 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN3-NEXT:    s_cmp_eq_u32 s35, s37
 ; GCN3-NEXT:    s_cselect_b64 s[36:37], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB138_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB138_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s34
+; GCN3-NEXT:    v_mov_b32_e32 v3, s35
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[36:37], 0
+; GCN3-NEXT:  .LBB138_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v7, v1
+; GCN3-NEXT:    v_mov_b32_e32 v6, v0
+; GCN3-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v6
+; GCN3-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GCN3-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN3-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB138_3
-; GCN3-NEXT:    s_branch .LBB138_4
-; GCN3-NEXT:  .LBB138_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN3-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_cbranch_execnz .LBB138_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[36:37]
+; GCN3-NEXT:    s_branch .LBB138_6
+; GCN3-NEXT:  .LBB138_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB138_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB138_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -21519,7 +23179,7 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB138_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB138_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
@@ -21817,23 +23477,46 @@ define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN1-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB141_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB141_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN1-NEXT:    s_cbranch_execnz .LBB141_6
 ; GCN1-NEXT:  .LBB141_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB141_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[10:11], 0
+; GCN1-NEXT:  .LBB141_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN1-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GCN1-NEXT:    v_add_i32_e64 v4, s[6:7], -1, v6
+; GCN1-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN1-NEXT:    s_cbranch_execnz .LBB141_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN1-NEXT:    s_cbranch_execz .LBB141_2
-; GCN1-NEXT:  .LBB141_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB141_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -21861,23 +23544,46 @@ define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN2-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB141_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB141_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN2-NEXT:    s_cbranch_execnz .LBB141_6
 ; GCN2-NEXT:  .LBB141_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB141_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[10:11], 0
+; GCN2-NEXT:  .LBB141_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN2-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GCN2-NEXT:    v_add_u32_e64 v4, s[6:7], -1, v6
+; GCN2-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN2-NEXT:    s_cbranch_execnz .LBB141_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN2-NEXT:    s_cbranch_execz .LBB141_2
-; GCN2-NEXT:  .LBB141_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB141_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -21903,23 +23609,43 @@ define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    s_mov_b64 s[4:5], src_private_base
 ; GCN3-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v1
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN3-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB141_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB141_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN3-NEXT:    s_cbranch_execnz .LBB141_6
 ; GCN3-NEXT:  .LBB141_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB141_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[10:11], 0
+; GCN3-NEXT:  .LBB141_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN3-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GCN3-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GCN3-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN3-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN3-NEXT:    s_cbranch_execnz .LBB141_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN3-NEXT:    s_cbranch_execz .LBB141_2
-; GCN3-NEXT:  .LBB141_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB141_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -21952,23 +23678,46 @@ define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN1-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB142_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB142_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN1-NEXT:    s_cbranch_execnz .LBB142_6
 ; GCN1-NEXT:  .LBB142_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB142_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v7, v[4:5]
+; GCN1-NEXT:    flat_load_dword v6, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[10:11], 0
+; GCN1-NEXT:  .LBB142_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN1-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GCN1-NEXT:    v_add_i32_e64 v4, s[6:7], -1, v6
+; GCN1-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN1-NEXT:    s_cbranch_execnz .LBB142_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN1-NEXT:    s_cbranch_execz .LBB142_2
-; GCN1-NEXT:  .LBB142_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB142_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -21998,23 +23747,46 @@ define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN2-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB142_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB142_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN2-NEXT:    s_cbranch_execnz .LBB142_6
 ; GCN2-NEXT:  .LBB142_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB142_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v7, v[4:5]
+; GCN2-NEXT:    flat_load_dword v6, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[10:11], 0
+; GCN2-NEXT:  .LBB142_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN2-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GCN2-NEXT:    v_add_u32_e64 v4, s[6:7], -1, v6
+; GCN2-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN2-NEXT:    s_cbranch_execnz .LBB142_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN2-NEXT:    s_cbranch_execz .LBB142_2
-; GCN2-NEXT:  .LBB142_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB142_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -22042,23 +23814,43 @@ define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    s_mov_b64 s[4:5], src_private_base
 ; GCN3-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v1
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN3-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB142_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB142_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN3-NEXT:    s_cbranch_execnz .LBB142_6
 ; GCN3-NEXT:  .LBB142_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB142_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GCN3-NEXT:    s_mov_b64 s[10:11], 0
+; GCN3-NEXT:  .LBB142_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN3-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GCN3-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GCN3-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN3-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN3-NEXT:    v_mov_b32_e32 v7, v5
+; GCN3-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN3-NEXT:    v_mov_b32_e32 v6, v4
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN3-NEXT:    s_cbranch_execnz .LBB142_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN3-NEXT:    s_cbranch_execz .LBB142_2
-; GCN3-NEXT:  .LBB142_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB142_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v0, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -22087,46 +23879,65 @@ define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN1-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN1-NEXT:    v_mov_b32_e32 v5, v1
-; GCN1-NEXT:    v_mov_b32_e32 v4, v0
-; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB143_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB143_4
-; GCN1-NEXT:  .LBB143_2: ; %atomicrmw.phi
-; GCN1-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN1-NEXT:    s_setpc_b64 s[30:31]
-; GCN1-NEXT:  .LBB143_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
+; GCN1-NEXT:    s_cbranch_execz .LBB143_4
+; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN1-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GCN1-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN1-NEXT:    flat_load_dword v5, v[4:5]
+; GCN1-NEXT:    flat_load_dword v4, v[0:1]
+; GCN1-NEXT:    s_mov_b64 s[10:11], 0
+; GCN1-NEXT:  .LBB143_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v7, v5
+; GCN1-NEXT:    v_mov_b32_e32 v6, v4
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN1-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GCN1-NEXT:    v_add_i32_e64 v4, s[6:7], -1, v6
+; GCN1-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN1-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN1-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN1-NEXT:    s_cbranch_execnz .LBB143_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[10:11]
+; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN1-NEXT:    s_cbranch_execz .LBB143_2
-; GCN1-NEXT:  .LBB143_4: ; %atomicrmw.private
-; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
-; GCN1-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN1-NEXT:  .LBB143_4: ; %Flow3
+; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN1-NEXT:    s_cbranch_execz .LBB143_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN1-NEXT:    v_add_i32_e32 v1, vcc, 4, v0
+; GCN1-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN1-NEXT:    s_waitcnt vmcnt(1)
-; GCN1-NEXT:    v_add_i32_e64 v6, s[6:7], -1, v0
+; GCN1-NEXT:    v_add_i32_e64 v6, s[6:7], -1, v4
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
-; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GCN1-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
-; GCN1-NEXT:    v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7]
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GCN1-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[4:5], v[2:3]
+; GCN1-NEXT:    v_addc_u32_e64 v7, s[6:7], -1, v5, s[6:7]
 ; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
 ; GCN1-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GCN1-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN1-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN1-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN1-NEXT:  .LBB143_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GCN1-NEXT:    v_mov_b32_e32 v0, v4
+; GCN1-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -22135,46 +23946,65 @@ define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_mov_b64 s[4:5], 0xe4
 ; GCN2-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GCN2-NEXT:    v_mov_b32_e32 v5, v1
-; GCN2-NEXT:    v_mov_b32_e32 v4, v0
-; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
+; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v1
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB143_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB143_4
-; GCN2-NEXT:  .LBB143_2: ; %atomicrmw.phi
-; GCN2-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN2-NEXT:    s_setpc_b64 s[30:31]
-; GCN2-NEXT:  .LBB143_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
+; GCN2-NEXT:    s_cbranch_execz .LBB143_4
+; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN2-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GCN2-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GCN2-NEXT:    flat_load_dword v5, v[4:5]
+; GCN2-NEXT:    flat_load_dword v4, v[0:1]
+; GCN2-NEXT:    s_mov_b64 s[10:11], 0
+; GCN2-NEXT:  .LBB143_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v7, v5
+; GCN2-NEXT:    v_mov_b32_e32 v6, v4
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN2-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GCN2-NEXT:    v_add_u32_e64 v4, s[6:7], -1, v6
+; GCN2-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN2-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GCN2-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN2-NEXT:    s_cbranch_execnz .LBB143_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[10:11]
+; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN2-NEXT:    s_cbranch_execz .LBB143_2
-; GCN2-NEXT:  .LBB143_4: ; %atomicrmw.private
-; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
-; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
-; GCN2-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_load_dword v1, v5, s[0:3], 0 offen
+; GCN2-NEXT:  .LBB143_4: ; %Flow3
+; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN2-NEXT:    s_cbranch_execz .LBB143_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, -1, v0, vcc
+; GCN2-NEXT:    v_add_u32_e32 v1, vcc, 4, v0
+; GCN2-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_load_dword v5, v1, s[0:3], 0 offen
 ; GCN2-NEXT:    s_waitcnt vmcnt(1)
-; GCN2-NEXT:    v_add_u32_e64 v6, s[6:7], -1, v0
+; GCN2-NEXT:    v_add_u32_e64 v6, s[6:7], -1, v4
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
-; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GCN2-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[0:1], v[2:3]
-; GCN2-NEXT:    v_addc_u32_e64 v7, s[6:7], -1, v1, s[6:7]
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GCN2-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[4:5], v[2:3]
+; GCN2-NEXT:    v_addc_u32_e64 v7, s[6:7], -1, v5, s[6:7]
 ; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v2, v6, v2, vcc
 ; GCN2-NEXT:    v_cndmask_b32_e32 v3, v7, v3, vcc
-; GCN2-NEXT:    buffer_store_dword v2, v4, s[0:3], 0 offen
-; GCN2-NEXT:    buffer_store_dword v3, v5, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_store_dword v2, v0, s[0:3], 0 offen
+; GCN2-NEXT:    buffer_store_dword v3, v1, s[0:3], 0 offen
+; GCN2-NEXT:  .LBB143_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GCN2-NEXT:    v_mov_b32_e32 v0, v4
+; GCN2-NEXT:    v_mov_b32_e32 v1, v5
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -22187,23 +24017,43 @@ define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {
 ; GCN3-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v5
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN3-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB143_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB143_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN3-NEXT:    s_cbranch_execnz .LBB143_6
 ; GCN3-NEXT:  .LBB143_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB143_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[10:11], 0
+; GCN3-NEXT:  .LBB143_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN3-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
+; GCN3-NEXT:    v_add_co_u32_e64 v0, s[6:7], -1, v8
+; GCN3-NEXT:    v_addc_co_u32_e64 v1, s[6:7], -1, v9, s[6:7]
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN3-NEXT:    v_cndmask_b32_e32 v7, v1, v3, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN3-NEXT:    s_cbranch_execnz .LBB143_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN3-NEXT:    s_cbranch_execz .LBB143_2
-; GCN3-NEXT:  .LBB143_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB143_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -22238,23 +24088,46 @@ define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN1-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN1-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN1-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN1-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN1-NEXT:    s_cbranch_execnz .LBB144_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN1-NEXT:    s_cbranch_execnz .LBB144_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN1-NEXT:    s_cbranch_execnz .LBB144_6
 ; GCN1-NEXT:  .LBB144_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB144_3: ; %atomicrmw.global
-; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN1-NEXT:    v_add_i32_e32 v0, vcc, 4, v4
+; GCN1-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[10:11], 0
+; GCN1-NEXT:  .LBB144_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v9, v1
+; GCN1-NEXT:    v_mov_b32_e32 v8, v0
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN1-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
+; GCN1-NEXT:    v_add_i32_e64 v0, s[6:7], -1, v8
+; GCN1-NEXT:    v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN1-NEXT:    v_cndmask_b32_e32 v7, v1, v3, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN1-NEXT:    s_cbranch_execnz .LBB144_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN1-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN1-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN1-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN1-NEXT:    s_cbranch_execz .LBB144_2
-; GCN1-NEXT:  .LBB144_4: ; %atomicrmw.private
+; GCN1-NEXT:  .LBB144_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN1-NEXT:    v_add_i32_e32 v5, vcc, 4, v4
@@ -22286,23 +24159,46 @@ define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN2-NEXT:    v_cmp_ne_u32_e32 vcc, s4, v5
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN2-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN2-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN2-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN2-NEXT:    s_cbranch_execnz .LBB144_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN2-NEXT:    s_cbranch_execnz .LBB144_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN2-NEXT:    s_cbranch_execnz .LBB144_6
 ; GCN2-NEXT:  .LBB144_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB144_3: ; %atomicrmw.global
-; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN2-NEXT:    v_add_u32_e32 v0, vcc, 4, v4
+; GCN2-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[10:11], 0
+; GCN2-NEXT:  .LBB144_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v9, v1
+; GCN2-NEXT:    v_mov_b32_e32 v8, v0
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN2-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
+; GCN2-NEXT:    v_add_u32_e64 v0, s[6:7], -1, v8
+; GCN2-NEXT:    v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN2-NEXT:    v_cndmask_b32_e32 v7, v1, v3, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN2-NEXT:    s_cbranch_execnz .LBB144_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN2-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN2-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN2-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN2-NEXT:    s_cbranch_execz .LBB144_2
-; GCN2-NEXT:  .LBB144_4: ; %atomicrmw.private
+; GCN2-NEXT:  .LBB144_6: ; %atomicrmw.private
 ; GCN2-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN2-NEXT:    v_add_u32_e32 v5, vcc, 4, v4
@@ -22332,23 +24228,43 @@ define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) {
 ; GCN3-NEXT:    v_cmp_ne_u32_e32 vcc, s5, v5
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
 ; GCN3-NEXT:    s_and_saveexec_b64 s[4:5], vcc
-; GCN3-NEXT:    s_xor_b64 s[4:5], exec, s[4:5]
+; GCN3-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN3-NEXT:    s_cbranch_execnz .LBB144_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
-; GCN3-NEXT:    s_cbranch_execnz .LBB144_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
+; GCN3-NEXT:    s_cbranch_execnz .LBB144_6
 ; GCN3-NEXT:  .LBB144_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB144_3: ; %atomicrmw.global
-; GCN3-NEXT:    flat_atomic_dec_x2 v[0:1], v[4:5], v[2:3] glc
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[10:11], 0
+; GCN3-NEXT:  .LBB144_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN3-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
+; GCN3-NEXT:    v_add_co_u32_e64 v0, s[6:7], -1, v8
+; GCN3-NEXT:    v_addc_co_u32_e64 v1, s[6:7], -1, v9, s[6:7]
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GCN3-NEXT:    v_cndmask_b32_e32 v7, v1, v3, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[10:11]
+; GCN3-NEXT:    s_cbranch_execnz .LBB144_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN3-NEXT:    ; implicit-def: $vgpr4_vgpr5
 ; GCN3-NEXT:    ; implicit-def: $vgpr2_vgpr3
-; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[4:5]
+; GCN3-NEXT:    s_andn2_saveexec_b64 s[8:9], s[8:9]
 ; GCN3-NEXT:    s_cbranch_execz .LBB144_2
-; GCN3-NEXT:  .LBB144_4: ; %atomicrmw.private
+; GCN3-NEXT:  .LBB144_6: ; %atomicrmw.private
 ; GCN3-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
 ; GCN3-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
 ; GCN3-NEXT:    buffer_load_dword v0, v4, s[0:3], 0 offen
@@ -22384,21 +24300,46 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN1-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB145_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB145_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB145_6
 ; GCN1-NEXT:  .LBB145_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB145_3: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v4, s4
+; GCN1-NEXT:    v_mov_b32_e32 v5, s5
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[38:39], 0
+; GCN1-NEXT:    v_mov_b32_e32 v6, s7
+; GCN1-NEXT:    v_mov_b32_e32 v7, s6
+; GCN1-NEXT:  .LBB145_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GCN1-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GCN1-NEXT:    v_add_i32_e64 v0, s[36:37], -1, v2
+; GCN1-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB145_2
-; GCN1-NEXT:  .LBB145_4: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GCN1-NEXT:    s_cbranch_execnz .LBB145_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[38:39]
+; GCN1-NEXT:    s_branch .LBB145_2
+; GCN1-NEXT:  .LBB145_6: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    v_mov_b32_e32 v5, s6
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
@@ -22433,21 +24374,46 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN2-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB145_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB145_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB145_6
 ; GCN2-NEXT:  .LBB145_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB145_3: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v4, s4
+; GCN2-NEXT:    v_mov_b32_e32 v5, s5
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[38:39], 0
+; GCN2-NEXT:    v_mov_b32_e32 v6, s7
+; GCN2-NEXT:    v_mov_b32_e32 v7, s6
+; GCN2-NEXT:  .LBB145_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GCN2-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GCN2-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v2
+; GCN2-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB145_2
-; GCN2-NEXT:  .LBB145_4: ; %atomicrmw.private
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GCN2-NEXT:    s_cbranch_execnz .LBB145_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[38:39]
+; GCN2-NEXT:    s_branch .LBB145_2
+; GCN2-NEXT:  .LBB145_6: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -22479,21 +24445,41 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
 ; GCN3-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB145_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB145_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB145_6
 ; GCN3-NEXT:  .LBB145_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB145_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s4
+; GCN3-NEXT:    v_mov_b32_e32 v5, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[38:39], 0
+; GCN3-NEXT:    v_mov_b32_e32 v6, s7
+; GCN3-NEXT:    v_mov_b32_e32 v7, s6
+; GCN3-NEXT:  .LBB145_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GCN3-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GCN3-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v2
+; GCN3-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN3-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB145_2
-; GCN3-NEXT:  .LBB145_4: ; %atomicrmw.private
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GCN3-NEXT:    s_cbranch_execnz .LBB145_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[38:39]
+; GCN3-NEXT:    s_branch .LBB145_2
+; GCN3-NEXT:  .LBB145_6: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -22522,34 +24508,59 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_mov_b64 s[34:35], 0xe4
-; GCN1-NEXT:    s_load_dword s36, s[34:35], 0x0
-; GCN1-NEXT:    s_add_u32 s34, s4, 32
-; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    s_load_dword s34, s[34:35], 0x0
+; GCN1-NEXT:    s_add_u32 s38, s4, 32
+; GCN1-NEXT:    s_addc_u32 s39, s5, 0
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN1-NEXT:    s_cmp_eq_u32 s35, s36
-; GCN1-NEXT:    s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_mov_b64 s[36:37], -1
+; GCN1-NEXT:    s_cmp_eq_u32 s39, s34
+; GCN1-NEXT:    s_cselect_b64 s[34:35], -1, 0
+; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN1-NEXT:    s_cbranch_vccnz .LBB146_3
-; GCN1-NEXT:  ; %bb.1: ; %Flow
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB146_4
+; GCN1-NEXT:  ; %bb.1: ; %Flow3
+; GCN1-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_vccnz .LBB146_6
 ; GCN1-NEXT:  .LBB146_2: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ; GCN1-NEXT:  .LBB146_3: ; %atomicrmw.global
+; GCN1-NEXT:    s_add_u32 s34, s38, 4
+; GCN1-NEXT:    s_addc_u32 s35, s39, 0
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v4, s38
+; GCN1-NEXT:    v_mov_b32_e32 v5, s39
+; GCN1-NEXT:    flat_load_dword v3, v[0:1]
+; GCN1-NEXT:    flat_load_dword v2, v[4:5]
+; GCN1-NEXT:    s_mov_b64 s[40:41], 0
+; GCN1-NEXT:    v_mov_b32_e32 v6, s7
+; GCN1-NEXT:    v_mov_b32_e32 v7, s6
+; GCN1-NEXT:  .LBB146_4: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GCN1-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GCN1-NEXT:    v_add_i32_e64 v0, s[36:37], -1, v2
+; GCN1-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN1-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execnz .LBB146_2
-; GCN1-NEXT:  .LBB146_4: ; %atomicrmw.private
-; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT:    v_mov_b32_e32 v3, v1
+; GCN1-NEXT:    s_or_b64 s[40:41], vcc, s[40:41]
+; GCN1-NEXT:    v_mov_b32_e32 v2, v0
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[40:41]
+; GCN1-NEXT:    s_cbranch_execnz .LBB146_4
+; GCN1-NEXT:  ; %bb.5: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[40:41]
+; GCN1-NEXT:    s_branch .LBB146_2
+; GCN1-NEXT:  .LBB146_6: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[38:39], 0
 ; GCN1-NEXT:    v_mov_b32_e32 v5, s6
-; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
-; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
+; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT:    s_cselect_b32 s34, s38, -1
 ; GCN1-NEXT:    v_mov_b32_e32 v2, s34
 ; GCN1-NEXT:    s_add_i32 s34, s34, 4
 ; GCN1-NEXT:    v_mov_b32_e32 v3, s34
@@ -22573,32 +24584,57 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_mov_b64 s[34:35], 0xe4
-; GCN2-NEXT:    s_load_dword s36, s[34:35], 0x0
-; GCN2-NEXT:    s_add_u32 s34, s4, 32
-; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    s_load_dword s34, s[34:35], 0x0
+; GCN2-NEXT:    s_add_u32 s38, s4, 32
+; GCN2-NEXT:    s_addc_u32 s39, s5, 0
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN2-NEXT:    s_cmp_eq_u32 s35, s36
-; GCN2-NEXT:    s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_mov_b64 s[36:37], -1
+; GCN2-NEXT:    s_cmp_eq_u32 s39, s34
+; GCN2-NEXT:    s_cselect_b64 s[34:35], -1, 0
+; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN2-NEXT:    s_cbranch_vccnz .LBB146_3
-; GCN2-NEXT:  ; %bb.1: ; %Flow
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB146_4
+; GCN2-NEXT:  ; %bb.1: ; %Flow3
+; GCN2-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_vccnz .LBB146_6
 ; GCN2-NEXT:  .LBB146_2: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ; GCN2-NEXT:  .LBB146_3: ; %atomicrmw.global
+; GCN2-NEXT:    s_add_u32 s34, s38, 4
+; GCN2-NEXT:    s_addc_u32 s35, s39, 0
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v4, s38
+; GCN2-NEXT:    v_mov_b32_e32 v5, s39
+; GCN2-NEXT:    flat_load_dword v3, v[0:1]
+; GCN2-NEXT:    flat_load_dword v2, v[4:5]
+; GCN2-NEXT:    s_mov_b64 s[40:41], 0
+; GCN2-NEXT:    v_mov_b32_e32 v6, s7
+; GCN2-NEXT:    v_mov_b32_e32 v7, s6
+; GCN2-NEXT:  .LBB146_4: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GCN2-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GCN2-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v2
+; GCN2-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN2-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execnz .LBB146_2
-; GCN2-NEXT:  .LBB146_4: ; %atomicrmw.private
-; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
-; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT:    v_mov_b32_e32 v3, v1
+; GCN2-NEXT:    s_or_b64 s[40:41], vcc, s[40:41]
+; GCN2-NEXT:    v_mov_b32_e32 v2, v0
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[40:41]
+; GCN2-NEXT:    s_cbranch_execnz .LBB146_4
+; GCN2-NEXT:  ; %bb.5: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[40:41]
+; GCN2-NEXT:    s_branch .LBB146_2
+; GCN2-NEXT:  .LBB146_6: ; %atomicrmw.private
+; GCN2-NEXT:    s_cmp_lg_u64 s[38:39], 0
+; GCN2-NEXT:    s_cselect_b32 s34, s38, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
 ; GCN2-NEXT:    s_add_i32 s34, s34, 4
 ; GCN2-NEXT:    v_mov_b32_e32 v3, s34
@@ -22622,31 +24658,51 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
 ; GCN3-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    s_add_u32 s34, s4, 32
-; GCN3-NEXT:    s_addc_u32 s35, s5, 0
-; GCN3-NEXT:    s_mov_b64 s[36:37], src_private_base
-; GCN3-NEXT:    s_cmp_eq_u32 s35, s37
-; GCN3-NEXT:    s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_mov_b64 s[36:37], -1
+; GCN3-NEXT:    s_add_u32 s38, s4, 32
+; GCN3-NEXT:    s_addc_u32 s39, s5, 0
+; GCN3-NEXT:    s_mov_b64 s[34:35], src_private_base
+; GCN3-NEXT:    s_cmp_eq_u32 s39, s35
+; GCN3-NEXT:    s_cselect_b64 s[34:35], -1, 0
+; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_mov_b64 s[34:35], -1
 ; GCN3-NEXT:    s_cbranch_vccnz .LBB146_3
-; GCN3-NEXT:  ; %bb.1: ; %Flow
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB146_4
+; GCN3-NEXT:  ; %bb.1: ; %Flow3
+; GCN3-NEXT:    s_and_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_vccnz .LBB146_6
 ; GCN3-NEXT:  .LBB146_2: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
 ; GCN3-NEXT:  .LBB146_3: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v4, s38
+; GCN3-NEXT:    v_mov_b32_e32 v5, s39
+; GCN3-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT:    s_mov_b64 s[40:41], 0
+; GCN3-NEXT:    v_mov_b32_e32 v6, s7
+; GCN3-NEXT:    v_mov_b32_e32 v7, s6
+; GCN3-NEXT:  .LBB146_4: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GCN3-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GCN3-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v2
+; GCN3-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN3-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execnz .LBB146_2
-; GCN3-NEXT:  .LBB146_4: ; %atomicrmw.private
-; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
-; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT:    v_mov_b32_e32 v3, v1
+; GCN3-NEXT:    s_or_b64 s[40:41], vcc, s[40:41]
+; GCN3-NEXT:    v_mov_b32_e32 v2, v0
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[40:41]
+; GCN3-NEXT:    s_cbranch_execnz .LBB146_4
+; GCN3-NEXT:  ; %bb.5: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[40:41]
+; GCN3-NEXT:    s_branch .LBB146_2
+; GCN3-NEXT:  .LBB146_6: ; %atomicrmw.private
+; GCN3-NEXT:    s_cmp_lg_u64 s[38:39], 0
+; GCN3-NEXT:    s_cselect_b32 s34, s38, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
 ; GCN3-NEXT:    buffer_load_dword v0, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
@@ -22679,20 +24735,45 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN1-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN1-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN1-NEXT:    s_cbranch_vccz .LBB147_2
+; GCN1-NEXT:    s_cbranch_vccz .LBB147_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT:    v_mov_b32_e32 v0, s4
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
-; GCN1-NEXT:    v_mov_b32_e32 v1, s5
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    s_add_u32 s34, s4, 4
+; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    v_mov_b32_e32 v0, s34
+; GCN1-NEXT:    v_mov_b32_e32 v1, s35
+; GCN1-NEXT:    v_mov_b32_e32 v2, s4
+; GCN1-NEXT:    v_mov_b32_e32 v3, s5
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[38:39], 0
+; GCN1-NEXT:    v_mov_b32_e32 v4, s7
+; GCN1-NEXT:    v_mov_b32_e32 v5, s6
+; GCN1-NEXT:  .LBB147_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v9, v1
+; GCN1-NEXT:    v_mov_b32_e32 v8, v0
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN1-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GCN1-NEXT:    v_add_i32_e64 v0, s[36:37], -1, v8
+; GCN1-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN1-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB147_3
-; GCN1-NEXT:    s_branch .LBB147_4
-; GCN1-NEXT:  .LBB147_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GCN1-NEXT:    s_cbranch_execnz .LBB147_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[38:39]
+; GCN1-NEXT:    s_branch .LBB147_6
+; GCN1-NEXT:  .LBB147_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB147_3: ; %atomicrmw.private
+; GCN1-NEXT:    s_cbranch_execz .LBB147_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[4:5], 0
 ; GCN1-NEXT:    v_mov_b32_e32 v5, s6
 ; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
@@ -22714,7 +24795,7 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
 ; GCN1-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB147_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB147_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -22727,20 +24808,45 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN2-NEXT:    s_cmp_eq_u32 s5, s34
 ; GCN2-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN2-NEXT:    s_cbranch_vccz .LBB147_2
+; GCN2-NEXT:    s_cbranch_vccz .LBB147_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT:    v_mov_b32_e32 v0, s4
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
-; GCN2-NEXT:    v_mov_b32_e32 v1, s5
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    s_add_u32 s34, s4, 4
+; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    v_mov_b32_e32 v0, s34
+; GCN2-NEXT:    v_mov_b32_e32 v1, s35
+; GCN2-NEXT:    v_mov_b32_e32 v2, s4
+; GCN2-NEXT:    v_mov_b32_e32 v3, s5
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[38:39], 0
+; GCN2-NEXT:    v_mov_b32_e32 v4, s7
+; GCN2-NEXT:    v_mov_b32_e32 v5, s6
+; GCN2-NEXT:  .LBB147_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v9, v1
+; GCN2-NEXT:    v_mov_b32_e32 v8, v0
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN2-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GCN2-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v8
+; GCN2-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN2-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB147_3
-; GCN2-NEXT:    s_branch .LBB147_4
-; GCN2-NEXT:  .LBB147_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GCN2-NEXT:    s_cbranch_execnz .LBB147_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[38:39]
+; GCN2-NEXT:    s_branch .LBB147_6
+; GCN2-NEXT:  .LBB147_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB147_3: ; %atomicrmw.private
+; GCN2-NEXT:    s_cbranch_execz .LBB147_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN2-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN2-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
@@ -22761,7 +24867,7 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
 ; GCN2-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB147_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB147_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -22772,20 +24878,40 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN3-NEXT:    s_cmp_eq_u32 s5, s35
 ; GCN3-NEXT:    s_cselect_b64 s[34:35], -1, 0
 ; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
-; GCN3-NEXT:    s_cbranch_vccz .LBB147_2
+; GCN3-NEXT:    s_cbranch_vccz .LBB147_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s4
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s5
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s4
+; GCN3-NEXT:    v_mov_b32_e32 v3, s5
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[38:39], 0
+; GCN3-NEXT:    v_mov_b32_e32 v4, s7
+; GCN3-NEXT:    v_mov_b32_e32 v5, s6
+; GCN3-NEXT:  .LBB147_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN3-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GCN3-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v8
+; GCN3-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN3-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB147_3
-; GCN3-NEXT:    s_branch .LBB147_4
-; GCN3-NEXT:  .LBB147_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GCN3-NEXT:    s_cbranch_execnz .LBB147_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[38:39]
+; GCN3-NEXT:    s_branch .LBB147_6
+; GCN3-NEXT:  .LBB147_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB147_3: ; %atomicrmw.private
+; GCN3-NEXT:    s_cbranch_execz .LBB147_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
 ; GCN3-NEXT:    s_cmp_lg_u64 s[4:5], 0
 ; GCN3-NEXT:    s_cselect_b32 s34, s4, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
@@ -22804,7 +24930,7 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GCN3-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB147_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB147_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst
@@ -22816,31 +24942,56 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN1:       ; %bb.0:
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    s_mov_b64 s[34:35], 0xe4
-; GCN1-NEXT:    s_load_dword s36, s[34:35], 0x0
-; GCN1-NEXT:    s_add_u32 s34, s4, 32
-; GCN1-NEXT:    s_addc_u32 s35, s5, 0
+; GCN1-NEXT:    s_load_dword s34, s[34:35], 0x0
+; GCN1-NEXT:    s_add_u32 s38, s4, 32
+; GCN1-NEXT:    s_addc_u32 s39, s5, 0
 ; GCN1-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN1-NEXT:    s_cmp_eq_u32 s35, s36
-; GCN1-NEXT:    s_cselect_b64 s[36:37], -1, 0
-; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN1-NEXT:    s_cbranch_vccz .LBB148_2
+; GCN1-NEXT:    s_cmp_eq_u32 s39, s34
+; GCN1-NEXT:    s_cselect_b64 s[34:35], -1, 0
+; GCN1-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
+; GCN1-NEXT:    s_cbranch_vccz .LBB148_4
 ; GCN1-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN1-NEXT:    s_add_u32 s34, s38, 4
+; GCN1-NEXT:    s_addc_u32 s35, s39, 0
 ; GCN1-NEXT:    v_mov_b32_e32 v0, s34
-; GCN1-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN1-NEXT:    v_mov_b32_e32 v1, s35
-; GCN1-NEXT:    v_mov_b32_e32 v3, s7
-; GCN1-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN1-NEXT:    v_mov_b32_e32 v2, s38
+; GCN1-NEXT:    v_mov_b32_e32 v3, s39
+; GCN1-NEXT:    flat_load_dword v1, v[0:1]
+; GCN1-NEXT:    flat_load_dword v0, v[2:3]
+; GCN1-NEXT:    s_mov_b64 s[40:41], 0
+; GCN1-NEXT:    v_mov_b32_e32 v4, s7
+; GCN1-NEXT:    v_mov_b32_e32 v5, s6
+; GCN1-NEXT:  .LBB148_2: ; %atomicrmw.start
+; GCN1-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN1-NEXT:    v_mov_b32_e32 v9, v1
+; GCN1-NEXT:    v_mov_b32_e32 v8, v0
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN1-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GCN1-NEXT:    v_add_i32_e64 v0, s[36:37], -1, v8
+; GCN1-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GCN1-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN1-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN1-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN1-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GCN1-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN1-NEXT:    buffer_wbinvl1_vol
-; GCN1-NEXT:    s_cbranch_execz .LBB148_3
-; GCN1-NEXT:    s_branch .LBB148_4
-; GCN1-NEXT:  .LBB148_2:
+; GCN1-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT:    s_or_b64 s[40:41], vcc, s[40:41]
+; GCN1-NEXT:    s_andn2_b64 exec, exec, s[40:41]
+; GCN1-NEXT:    s_cbranch_execnz .LBB148_2
+; GCN1-NEXT:  ; %bb.3: ; %Flow
+; GCN1-NEXT:    s_or_b64 exec, exec, s[40:41]
+; GCN1-NEXT:    s_branch .LBB148_6
+; GCN1-NEXT:  .LBB148_4:
 ; GCN1-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN1-NEXT:  .LBB148_3: ; %atomicrmw.private
-; GCN1-NEXT:    v_cmp_ne_u64_e64 s[36:37], s[34:35], 0
+; GCN1-NEXT:    s_cbranch_execz .LBB148_6
+; GCN1-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN1-NEXT:    v_cmp_ne_u64_e64 s[34:35], s[38:39], 0
 ; GCN1-NEXT:    v_mov_b32_e32 v5, s6
-; GCN1-NEXT:    s_and_b64 s[36:37], s[36:37], exec
-; GCN1-NEXT:    s_cselect_b32 s34, s34, -1
+; GCN1-NEXT:    s_and_b64 s[34:35], s[34:35], exec
+; GCN1-NEXT:    s_cselect_b32 s34, s38, -1
 ; GCN1-NEXT:    v_mov_b32_e32 v2, s34
 ; GCN1-NEXT:    s_add_i32 s34, s34, 4
 ; GCN1-NEXT:    v_mov_b32_e32 v3, s34
@@ -22858,7 +25009,7 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN1-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
 ; GCN1-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
 ; GCN1-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen
-; GCN1-NEXT:  .LBB148_4: ; %atomicrmw.end
+; GCN1-NEXT:  .LBB148_6: ; %atomicrmw.phi
 ; GCN1-NEXT:    s_waitcnt vmcnt(0)
 ; GCN1-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -22866,29 +25017,54 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN2:       ; %bb.0:
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    s_mov_b64 s[34:35], 0xe4
-; GCN2-NEXT:    s_load_dword s36, s[34:35], 0x0
-; GCN2-NEXT:    s_add_u32 s34, s4, 32
-; GCN2-NEXT:    s_addc_u32 s35, s5, 0
+; GCN2-NEXT:    s_load_dword s34, s[34:35], 0x0
+; GCN2-NEXT:    s_add_u32 s38, s4, 32
+; GCN2-NEXT:    s_addc_u32 s39, s5, 0
 ; GCN2-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN2-NEXT:    s_cmp_eq_u32 s35, s36
-; GCN2-NEXT:    s_cselect_b64 s[36:37], -1, 0
-; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN2-NEXT:    s_cbranch_vccz .LBB148_2
+; GCN2-NEXT:    s_cmp_eq_u32 s39, s34
+; GCN2-NEXT:    s_cselect_b64 s[34:35], -1, 0
+; GCN2-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
+; GCN2-NEXT:    s_cbranch_vccz .LBB148_4
 ; GCN2-NEXT:  ; %bb.1: ; %atomicrmw.global
+; GCN2-NEXT:    s_add_u32 s34, s38, 4
+; GCN2-NEXT:    s_addc_u32 s35, s39, 0
 ; GCN2-NEXT:    v_mov_b32_e32 v0, s34
-; GCN2-NEXT:    v_mov_b32_e32 v2, s6
 ; GCN2-NEXT:    v_mov_b32_e32 v1, s35
-; GCN2-NEXT:    v_mov_b32_e32 v3, s7
-; GCN2-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN2-NEXT:    v_mov_b32_e32 v2, s38
+; GCN2-NEXT:    v_mov_b32_e32 v3, s39
+; GCN2-NEXT:    flat_load_dword v1, v[0:1]
+; GCN2-NEXT:    flat_load_dword v0, v[2:3]
+; GCN2-NEXT:    s_mov_b64 s[40:41], 0
+; GCN2-NEXT:    v_mov_b32_e32 v4, s7
+; GCN2-NEXT:    v_mov_b32_e32 v5, s6
+; GCN2-NEXT:  .LBB148_2: ; %atomicrmw.start
+; GCN2-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN2-NEXT:    v_mov_b32_e32 v9, v1
+; GCN2-NEXT:    v_mov_b32_e32 v8, v0
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN2-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GCN2-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v8
+; GCN2-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GCN2-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN2-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN2-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN2-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GCN2-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN2-NEXT:    buffer_wbinvl1_vol
-; GCN2-NEXT:    s_cbranch_execz .LBB148_3
-; GCN2-NEXT:    s_branch .LBB148_4
-; GCN2-NEXT:  .LBB148_2:
+; GCN2-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT:    s_or_b64 s[40:41], vcc, s[40:41]
+; GCN2-NEXT:    s_andn2_b64 exec, exec, s[40:41]
+; GCN2-NEXT:    s_cbranch_execnz .LBB148_2
+; GCN2-NEXT:  ; %bb.3: ; %Flow
+; GCN2-NEXT:    s_or_b64 exec, exec, s[40:41]
+; GCN2-NEXT:    s_branch .LBB148_6
+; GCN2-NEXT:  .LBB148_4:
 ; GCN2-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN2-NEXT:  .LBB148_3: ; %atomicrmw.private
-; GCN2-NEXT:    s_cmp_lg_u64 s[34:35], 0
-; GCN2-NEXT:    s_cselect_b32 s34, s34, -1
+; GCN2-NEXT:    s_cbranch_execz .LBB148_6
+; GCN2-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN2-NEXT:    s_cmp_lg_u64 s[38:39], 0
+; GCN2-NEXT:    s_cselect_b32 s34, s38, -1
 ; GCN2-NEXT:    v_mov_b32_e32 v2, s34
 ; GCN2-NEXT:    s_add_i32 s34, s34, 4
 ; GCN2-NEXT:    v_mov_b32_e32 v3, s34
@@ -22907,35 +25083,55 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN2-NEXT:    v_cndmask_b32_e32 v4, v7, v4, vcc
 ; GCN2-NEXT:    buffer_store_dword v5, v2, s[0:3], 0 offen
 ; GCN2-NEXT:    buffer_store_dword v4, v3, s[0:3], 0 offen
-; GCN2-NEXT:  .LBB148_4: ; %atomicrmw.end
+; GCN2-NEXT:  .LBB148_6: ; %atomicrmw.phi
 ; GCN2-NEXT:    s_waitcnt vmcnt(0)
 ; GCN2-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GCN3-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
 ; GCN3:       ; %bb.0:
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN3-NEXT:    s_add_u32 s34, s4, 32
-; GCN3-NEXT:    s_addc_u32 s35, s5, 0
-; GCN3-NEXT:    s_mov_b64 s[36:37], src_private_base
-; GCN3-NEXT:    s_cmp_eq_u32 s35, s37
-; GCN3-NEXT:    s_cselect_b64 s[36:37], -1, 0
-; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[36:37]
-; GCN3-NEXT:    s_cbranch_vccz .LBB148_2
+; GCN3-NEXT:    s_add_u32 s38, s4, 32
+; GCN3-NEXT:    s_addc_u32 s39, s5, 0
+; GCN3-NEXT:    s_mov_b64 s[34:35], src_private_base
+; GCN3-NEXT:    s_cmp_eq_u32 s39, s35
+; GCN3-NEXT:    s_cselect_b64 s[34:35], -1, 0
+; GCN3-NEXT:    s_andn2_b64 vcc, exec, s[34:35]
+; GCN3-NEXT:    s_cbranch_vccz .LBB148_4
 ; GCN3-NEXT:  ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT:    v_mov_b32_e32 v0, s34
-; GCN3-NEXT:    v_mov_b32_e32 v2, s6
-; GCN3-NEXT:    v_mov_b32_e32 v1, s35
-; GCN3-NEXT:    v_mov_b32_e32 v3, s7
-; GCN3-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GCN3-NEXT:    v_mov_b32_e32 v2, s38
+; GCN3-NEXT:    v_mov_b32_e32 v3, s39
+; GCN3-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; GCN3-NEXT:    s_mov_b64 s[40:41], 0
+; GCN3-NEXT:    v_mov_b32_e32 v4, s7
+; GCN3-NEXT:    v_mov_b32_e32 v5, s6
+; GCN3-NEXT:  .LBB148_2: ; %atomicrmw.start
+; GCN3-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN3-NEXT:    v_mov_b32_e32 v9, v1
+; GCN3-NEXT:    v_mov_b32_e32 v8, v0
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN3-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GCN3-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v8
+; GCN3-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GCN3-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GCN3-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN3-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN3-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GCN3-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GCN3-NEXT:    buffer_wbinvl1_vol
-; GCN3-NEXT:    s_cbranch_execz .LBB148_3
-; GCN3-NEXT:    s_branch .LBB148_4
-; GCN3-NEXT:  .LBB148_2:
+; GCN3-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN3-NEXT:    s_or_b64 s[40:41], vcc, s[40:41]
+; GCN3-NEXT:    s_andn2_b64 exec, exec, s[40:41]
+; GCN3-NEXT:    s_cbranch_execnz .LBB148_2
+; GCN3-NEXT:  ; %bb.3: ; %Flow
+; GCN3-NEXT:    s_or_b64 exec, exec, s[40:41]
+; GCN3-NEXT:    s_branch .LBB148_6
+; GCN3-NEXT:  .LBB148_4:
 ; GCN3-NEXT:    ; implicit-def: $vgpr0_vgpr1
-; GCN3-NEXT:  .LBB148_3: ; %atomicrmw.private
-; GCN3-NEXT:    s_cmp_lg_u64 s[34:35], 0
-; GCN3-NEXT:    s_cselect_b32 s34, s34, -1
+; GCN3-NEXT:    s_cbranch_execz .LBB148_6
+; GCN3-NEXT:  ; %bb.5: ; %atomicrmw.private
+; GCN3-NEXT:    s_cmp_lg_u64 s[38:39], 0
+; GCN3-NEXT:    s_cselect_b32 s34, s38, -1
 ; GCN3-NEXT:    v_mov_b32_e32 v2, s34
 ; GCN3-NEXT:    buffer_load_dword v0, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
@@ -22952,7 +25148,7 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GCN3-NEXT:    v_cndmask_b32_e32 v3, v6, v3, vcc
 ; GCN3-NEXT:    buffer_store_dword v4, v2, s[0:3], 0 offen
 ; GCN3-NEXT:    buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; GCN3-NEXT:  .LBB148_4: ; %atomicrmw.end
+; GCN3-NEXT:  .LBB148_6: ; %atomicrmw.phi
 ; GCN3-NEXT:    s_waitcnt vmcnt(0)
 ; GCN3-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system_noprivate.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system_noprivate.ll
index fe47461ebf956..4dea4495b36fb 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system_noprivate.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system_noprivate.ll
@@ -1097,25 +1097,76 @@ define void @flat_atomic_sub_i64_noret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_sub_i64_noret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v6, v[0:1]
+; GFX7-NEXT:    flat_load_dword v7, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_sub_i32_e32 v4, vcc, v6, v2
+; GFX7-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB30_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_sub_i64_noret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v6, v[0:1]
+; GFX8-NEXT:    flat_load_dword v7, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v6, v2
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB30_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_sub_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB30_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -1125,29 +1176,80 @@ define void @flat_atomic_sub_i64_noret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_sub_i64_noret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dword v7, v[0:1]
+; GFX7-NEXT:    flat_load_dword v6, v[8:9]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_sub_i32_e32 v4, vcc, v6, v2
+; GFX7-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB31_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_sub_i64_noret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dword v7, v[0:1]
+; GFX8-NEXT:    flat_load_dword v6, v[8:9]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v6, v2
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB31_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_sub_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3] offset:32
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB31_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -1158,25 +1260,82 @@ define i64 @flat_atomic_sub_i64_ret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_sub_i64_ret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_add_i32_e32 v5, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v4, v[0:1]
+; GFX7-NEXT:    flat_load_dword v5, v[5:6]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    v_sub_i32_e32 v4, vcc, v6, v2
+; GFX7-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB32_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
+; GFX7-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_sub_i64_ret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v4, v[0:1]
+; GFX8-NEXT:    flat_load_dword v5, v[5:6]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    v_sub_u32_e32 v4, vcc, v6, v2
+; GFX8-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB32_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
+; GFX8-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_sub_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB32_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -1186,29 +1345,82 @@ define i64 @flat_atomic_sub_i64_ret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_sub_i64_ret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_sub_i32_e32 v6, vcc, v8, v2
+; GFX7-NEXT:    v_subb_u32_e32 v7, vcc, v9, v3, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB33_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_sub_i64_ret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_sub_u32_e32 v6, vcc, v8, v2
+; GFX8-NEXT:    v_subb_u32_e32 v7, vcc, v9, v3, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB33_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_sub_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB33_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -1219,41 +1431,99 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GFX7-LABEL: flat_atomic_sub_i64_noret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v3, s34
+; GFX7-NEXT:    v_mov_b32_e32 v4, s35
+; GFX7-NEXT:    flat_load_dword v2, v[0:1]
+; GFX7-NEXT:    flat_load_dword v3, v[3:4]
+; GFX7-NEXT:    v_mov_b32_e32 v4, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s7
+; GFX7-NEXT:    v_mov_b32_e32 v5, s5
+; GFX7-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v2
+; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB34_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_sub_i64_noret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s34
+; GFX8-NEXT:    v_mov_b32_e32 v4, s35
+; GFX8-NEXT:    flat_load_dword v2, v[0:1]
+; GFX8-NEXT:    flat_load_dword v3, v[3:4]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB34_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_sub_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
-; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    s_setpc_b64 s[30:31]
-  %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
-  ret void
-}
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, s7
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, s6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB34_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tmp0 = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
+  ret void
+}
 
 define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out, i64 inreg %in) {
 ; GFX7-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
@@ -1261,13 +1531,32 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
+; GFX7-NEXT:    v_mov_b32_e32 v4, s34
+; GFX7-NEXT:    v_mov_b32_e32 v5, s35
+; GFX7-NEXT:    flat_load_dword v3, v[0:1]
+; GFX7-NEXT:    flat_load_dword v2, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s7
+; GFX7-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_subrev_i32_e32 v0, vcc, s6, v2
+; GFX7-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB35_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
@@ -1275,25 +1564,60 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
+; GFX8-NEXT:    v_mov_b32_e32 v4, s34
+; GFX8-NEXT:    v_mov_b32_e32 v5, s35
+; GFX8-NEXT:    flat_load_dword v3, v[0:1]
+; GFX8-NEXT:    flat_load_dword v2, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_subrev_u32_e32 v0, vcc, s6, v2
+; GFX8-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB35_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, s7
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, s6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB35_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -1304,37 +1628,95 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GFX7-LABEL: flat_atomic_sub_i64_ret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s34
+; GFX7-NEXT:    v_mov_b32_e32 v3, s35
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v8, v1
+; GFX7-NEXT:    v_mov_b32_e32 v7, v0
+; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s6, v7
+; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB36_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_sub_i64_ret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s34
+; GFX8-NEXT:    v_mov_b32_e32 v3, s35
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s7
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v8, v1
+; GFX8-NEXT:    v_mov_b32_e32 v7, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s6, v7
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB36_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_sub_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX9-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v8, v1
+; GFX9-NEXT:    v_mov_b32_e32 v7, v0
+; GFX9-NEXT:    v_subrev_co_u32_e32 v5, vcc, s6, v7
+; GFX9-NEXT:    v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB36_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw sub ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -1346,13 +1728,32 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[2:3]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s7
+; GFX7-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v8, v1
+; GFX7-NEXT:    v_mov_b32_e32 v7, v0
+; GFX7-NEXT:    v_subrev_i32_e32 v5, vcc, s6, v7
+; GFX7-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB37_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
@@ -1360,25 +1761,60 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[2:3]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s7
+; GFX8-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v8, v1
+; GFX8-NEXT:    v_mov_b32_e32 v7, v0
+; GFX8-NEXT:    v_subrev_u32_e32 v5, vcc, s6, v7
+; GFX8-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB37_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] offset:32
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
+; GFX9-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v8, v1
+; GFX9-NEXT:    v_mov_b32_e32 v7, v0
+; GFX9-NEXT:    v_subrev_co_u32_e32 v5, vcc, s6, v7
+; GFX9-NEXT:    v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB37_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw sub ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -1459,25 +1895,76 @@ define void @flat_atomic_and_i64_noret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_and_i64_noret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v6, v[0:1]
+; GFX7-NEXT:    flat_load_dword v7, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX7-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB40_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_and_i64_noret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v6, v[0:1]
+; GFX8-NEXT:    flat_load_dword v7, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX8-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB40_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_and_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB40_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -1487,29 +1974,80 @@ define void @flat_atomic_and_i64_noret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_and_i64_noret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dword v7, v[0:1]
+; GFX7-NEXT:    flat_load_dword v6, v[8:9]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX7-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB41_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_and_i64_noret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    s_setpc_b64 s[30:31]
+; GFX8-NEXT:    flat_load_dword v7, v[0:1]
+; GFX8-NEXT:    flat_load_dword v6, v[8:9]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX8-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB41_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_and_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3] offset:32
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB41_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -1520,25 +2058,82 @@ define i64 @flat_atomic_and_i64_ret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_and_i64_ret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_add_i32_e32 v5, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v4, v[0:1]
+; GFX7-NEXT:    flat_load_dword v5, v[5:6]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX7-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB42_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
+; GFX7-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_and_i64_ret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v4, v[0:1]
+; GFX8-NEXT:    flat_load_dword v5, v[5:6]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX8-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB42_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
+; GFX8-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_and_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB42_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -1548,29 +2143,82 @@ define i64 @flat_atomic_and_i64_ret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_and_i64_ret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_and_b32_e32 v7, v9, v3
+; GFX7-NEXT:    v_and_b32_e32 v6, v8, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB43_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_and_i64_ret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_and_b32_e32 v7, v9, v3
+; GFX8-NEXT:    v_and_b32_e32 v6, v8, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB43_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_and_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB43_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -1581,37 +2229,92 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GFX7-LABEL: flat_atomic_and_i64_noret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v3, s34
+; GFX7-NEXT:    v_mov_b32_e32 v4, s35
+; GFX7-NEXT:    flat_load_dword v2, v[0:1]
+; GFX7-NEXT:    flat_load_dword v3, v[3:4]
+; GFX7-NEXT:    v_mov_b32_e32 v4, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:    v_mov_b32_e32 v5, s5
+; GFX7-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v1, s7, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB44_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_and_i64_noret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s34
+; GFX8-NEXT:    v_mov_b32_e32 v4, s35
+; GFX8-NEXT:    flat_load_dword v2, v[0:1]
+; GFX8-NEXT:    flat_load_dword v3, v[3:4]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v1, s7, v3
+; GFX8-NEXT:    v_and_b32_e32 v0, s6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB44_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_and_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_and_b32_e32 v0, s6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB44_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -1623,13 +2326,31 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
+; GFX7-NEXT:    v_mov_b32_e32 v4, s34
+; GFX7-NEXT:    v_mov_b32_e32 v5, s35
+; GFX7-NEXT:    flat_load_dword v3, v[0:1]
+; GFX7-NEXT:    flat_load_dword v2, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v1, s7, v3
+; GFX7-NEXT:    v_and_b32_e32 v0, s6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB45_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_and_i64_noret_offset_scalar:
@@ -1637,25 +2358,58 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
+; GFX8-NEXT:    v_mov_b32_e32 v4, s34
+; GFX8-NEXT:    v_mov_b32_e32 v5, s35
+; GFX8-NEXT:    flat_load_dword v3, v[0:1]
+; GFX8-NEXT:    flat_load_dword v2, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_and_b32_e32 v1, s7, v3
+; GFX8-NEXT:    v_and_b32_e32 v0, s6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB45_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_and_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_and_b32_e32 v0, s6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB45_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -1666,37 +2420,92 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GFX7-LABEL: flat_atomic_and_i64_ret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s34
+; GFX7-NEXT:    v_mov_b32_e32 v3, s35
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_and_b32_e32 v5, s7, v7
+; GFX7-NEXT:    v_and_b32_e32 v4, s6, v6
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB46_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_and_i64_ret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s34
+; GFX8-NEXT:    v_mov_b32_e32 v3, s35
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_and_b32_e32 v5, s7, v7
+; GFX8-NEXT:    v_and_b32_e32 v4, s6, v6
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB46_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_and_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX9-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_and_b32_e32 v5, s7, v7
+; GFX9-NEXT:    v_and_b32_e32 v4, s6, v6
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB46_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw and ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -1708,13 +2517,31 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[2:3]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_and_b32_e32 v5, s7, v7
+; GFX7-NEXT:    v_and_b32_e32 v4, s6, v6
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB47_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_and_i64_ret_offset_scalar:
@@ -1722,25 +2549,58 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[2:3]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_and_b32_e32 v5, s7, v7
+; GFX8-NEXT:    v_and_b32_e32 v4, s6, v6
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB47_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_and_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] offset:32
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
+; GFX9-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_and_b32_e32 v5, s7, v7
+; GFX9-NEXT:    v_and_b32_e32 v4, s6, v6
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB47_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw and ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -2771,25 +3631,76 @@ define void @flat_atomic_or_i64_noret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_or_i64_noret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v6, v[0:1]
+; GFX7-NEXT:    flat_load_dword v7, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX7-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB60_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_or_i64_noret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v6, v[0:1]
+; GFX8-NEXT:    flat_load_dword v7, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX8-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB60_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_or_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB60_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -2799,29 +3710,80 @@ define void @flat_atomic_or_i64_noret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_or_i64_noret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dword v7, v[0:1]
+; GFX7-NEXT:    flat_load_dword v6, v[8:9]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX7-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB61_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_or_i64_noret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dword v7, v[0:1]
+; GFX8-NEXT:    flat_load_dword v6, v[8:9]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX8-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB61_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_or_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3] offset:32
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB61_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -2832,25 +3794,82 @@ define i64 @flat_atomic_or_i64_ret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_or_i64_ret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_add_i32_e32 v5, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v4, v[0:1]
+; GFX7-NEXT:    flat_load_dword v5, v[5:6]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX7-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB62_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
+; GFX7-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_or_i64_ret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v4, v[0:1]
+; GFX8-NEXT:    flat_load_dword v5, v[5:6]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX8-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB62_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
+; GFX8-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_or_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB62_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -2860,29 +3879,82 @@ define i64 @flat_atomic_or_i64_ret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_or_i64_ret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_or_b32_e32 v7, v9, v3
+; GFX7-NEXT:    v_or_b32_e32 v6, v8, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB63_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_or_i64_ret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_or_b32_e32 v7, v9, v3
+; GFX8-NEXT:    v_or_b32_e32 v6, v8, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB63_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_or_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB63_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -2893,37 +3965,92 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
 ; GFX7-LABEL: flat_atomic_or_i64_noret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v3, s34
+; GFX7-NEXT:    v_mov_b32_e32 v4, s35
+; GFX7-NEXT:    flat_load_dword v2, v[0:1]
+; GFX7-NEXT:    flat_load_dword v3, v[3:4]
+; GFX7-NEXT:    v_mov_b32_e32 v4, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:    v_mov_b32_e32 v5, s5
+; GFX7-NEXT:  .LBB64_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_or_b32_e32 v1, s7, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, s6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB64_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_or_i64_noret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s34
+; GFX8-NEXT:    v_mov_b32_e32 v4, s35
+; GFX8-NEXT:    flat_load_dword v2, v[0:1]
+; GFX8-NEXT:    flat_load_dword v3, v[3:4]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:  .LBB64_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_or_b32_e32 v1, s7, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, s6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB64_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_or_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB64_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_or_b32_e32 v0, s6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB64_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -2935,13 +4062,31 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
+; GFX7-NEXT:    v_mov_b32_e32 v4, s34
+; GFX7-NEXT:    v_mov_b32_e32 v5, s35
+; GFX7-NEXT:    flat_load_dword v3, v[0:1]
+; GFX7-NEXT:    flat_load_dword v2, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:  .LBB65_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_or_b32_e32 v1, s7, v3
+; GFX7-NEXT:    v_or_b32_e32 v0, s6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB65_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_or_i64_noret_offset_scalar:
@@ -2949,25 +4094,58 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
+; GFX8-NEXT:    v_mov_b32_e32 v4, s34
+; GFX8-NEXT:    v_mov_b32_e32 v5, s35
+; GFX8-NEXT:    flat_load_dword v3, v[0:1]
+; GFX8-NEXT:    flat_load_dword v2, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:  .LBB65_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_or_b32_e32 v1, s7, v3
+; GFX8-NEXT:    v_or_b32_e32 v0, s6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB65_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_or_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB65_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_or_b32_e32 v0, s6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB65_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -2978,37 +4156,92 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
 ; GFX7-LABEL: flat_atomic_or_i64_ret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s34
+; GFX7-NEXT:    v_mov_b32_e32 v3, s35
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:  .LBB66_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_or_b32_e32 v5, s7, v7
+; GFX7-NEXT:    v_or_b32_e32 v4, s6, v6
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB66_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_or_i64_ret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s34
+; GFX8-NEXT:    v_mov_b32_e32 v3, s35
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:  .LBB66_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_or_b32_e32 v5, s7, v7
+; GFX8-NEXT:    v_or_b32_e32 v4, s6, v6
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB66_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_or_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX9-NEXT:  .LBB66_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_or_b32_e32 v5, s7, v7
+; GFX9-NEXT:    v_or_b32_e32 v4, s6, v6
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB66_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw or ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -3020,13 +4253,31 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[2:3]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:  .LBB67_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_or_b32_e32 v5, s7, v7
+; GFX7-NEXT:    v_or_b32_e32 v4, s6, v6
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB67_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_or_i64_ret_offset_scalar:
@@ -3034,25 +4285,58 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[2:3]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:  .LBB67_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_or_b32_e32 v5, s7, v7
+; GFX8-NEXT:    v_or_b32_e32 v4, s6, v6
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB67_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_or_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] offset:32
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
+; GFX9-NEXT:  .LBB67_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_or_b32_e32 v5, s7, v7
+; GFX9-NEXT:    v_or_b32_e32 v4, s6, v6
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB67_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw or ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -3133,25 +4417,76 @@ define void @flat_atomic_xor_i64_noret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_xor_i64_noret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v6, v[0:1]
+; GFX7-NEXT:    flat_load_dword v7, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB70_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX7-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB70_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_xor_i64_noret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v6, v[0:1]
+; GFX8-NEXT:    flat_load_dword v7, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB70_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX8-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB70_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_xor_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB70_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB70_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -3161,29 +4496,80 @@ define void @flat_atomic_xor_i64_noret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_xor_i64_noret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dword v7, v[0:1]
+; GFX7-NEXT:    flat_load_dword v6, v[8:9]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB71_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX7-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB71_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_xor_i64_noret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dword v7, v[0:1]
+; GFX8-NEXT:    flat_load_dword v6, v[8:9]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB71_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX8-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB71_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_xor_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3] offset:32
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB71_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB71_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -3194,25 +4580,82 @@ define i64 @flat_atomic_xor_i64_ret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_xor_i64_ret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_add_i32_e32 v5, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v4, v[0:1]
+; GFX7-NEXT:    flat_load_dword v5, v[5:6]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB72_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX7-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB72_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
+; GFX7-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_xor_i64_ret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v4, v[0:1]
+; GFX8-NEXT:    flat_load_dword v5, v[5:6]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB72_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX8-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB72_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
+; GFX8-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_xor_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB72_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB72_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -3222,29 +4665,82 @@ define i64 @flat_atomic_xor_i64_ret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_xor_i64_ret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB73_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_xor_b32_e32 v7, v9, v3
+; GFX7-NEXT:    v_xor_b32_e32 v6, v8, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB73_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_xor_i64_ret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB73_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_xor_b32_e32 v7, v9, v3
+; GFX8-NEXT:    v_xor_b32_e32 v6, v8, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB73_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_xor_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB73_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB73_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -3255,37 +4751,92 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
 ; GFX7-LABEL: flat_atomic_xor_i64_noret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v3, s34
+; GFX7-NEXT:    v_mov_b32_e32 v4, s35
+; GFX7-NEXT:    flat_load_dword v2, v[0:1]
+; GFX7-NEXT:    flat_load_dword v3, v[3:4]
+; GFX7-NEXT:    v_mov_b32_e32 v4, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:    v_mov_b32_e32 v5, s5
+; GFX7-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GFX7-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB74_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_xor_i64_noret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s34
+; GFX8-NEXT:    v_mov_b32_e32 v4, s35
+; GFX8-NEXT:    flat_load_dword v2, v[0:1]
+; GFX8-NEXT:    flat_load_dword v3, v[3:4]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GFX8-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB74_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_xor_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB74_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -3297,13 +4848,31 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
+; GFX7-NEXT:    v_mov_b32_e32 v4, s34
+; GFX7-NEXT:    v_mov_b32_e32 v5, s35
+; GFX7-NEXT:    flat_load_dword v3, v[0:1]
+; GFX7-NEXT:    flat_load_dword v2, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GFX7-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB75_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
@@ -3311,25 +4880,58 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
+; GFX8-NEXT:    v_mov_b32_e32 v4, s34
+; GFX8-NEXT:    v_mov_b32_e32 v5, s35
+; GFX8-NEXT:    flat_load_dword v3, v[0:1]
+; GFX8-NEXT:    flat_load_dword v2, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GFX8-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB75_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB75_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -3340,37 +4942,92 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
 ; GFX7-LABEL: flat_atomic_xor_i64_ret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s34
+; GFX7-NEXT:    v_mov_b32_e32 v3, s35
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GFX7-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB76_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_xor_i64_ret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s34
+; GFX8-NEXT:    v_mov_b32_e32 v3, s35
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GFX8-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB76_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_xor_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX9-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GFX9-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB76_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw xor ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -3382,13 +5039,31 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[2:3]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GFX7-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB77_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
@@ -3396,25 +5071,58 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[2:3]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GFX8-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB77_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] offset:32
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
+; GFX9-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_xor_b32_e32 v5, s7, v7
+; GFX9-NEXT:    v_xor_b32_e32 v4, s6, v6
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB77_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw xor ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -4697,83 +6405,29 @@ define void @flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr %out,
 ; GFX7-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
-; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_load_dword v7, v[0:1]
-; GFX7-NEXT:    flat_load_dword v6, v[8:9]
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:  .LBB92_1: ; %atomicrmw.start
-; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
+; GFX7-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_mov_b32_e32 v7, v1
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    v_mov_b32_e32 v6, v0
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB92_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
-; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dword v7, v[0:1]
-; GFX8-NEXT:    flat_load_dword v6, v[8:9]
-; GFX8-NEXT:    s_mov_b64 s[4:5], 0
-; GFX8-NEXT:  .LBB92_1: ; %atomicrmw.start
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
+; GFX8-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_mov_b32_e32 v7, v1
-; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT:    v_mov_b32_e32 v6, v0
-; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execnz .LBB92_1
-; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB92_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
+; GFX9-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3] offset:32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB92_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
@@ -4784,85 +6438,29 @@ define i64 @flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i6
 ; GFX7-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
-; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_load_dword v1, v[0:1]
-; GFX7-NEXT:    flat_load_dword v0, v[4:5]
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:  .LBB93_1: ; %atomicrmw.start
-; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v9, v1
-; GFX7-NEXT:    v_mov_b32_e32 v8, v0
-; GFX7-NEXT:    v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX7-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB93_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
-; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dword v1, v[0:1]
-; GFX8-NEXT:    flat_load_dword v0, v[4:5]
-; GFX8-NEXT:    s_mov_b64 s[4:5], 0
-; GFX8-NEXT:  .LBB93_1: ; %atomicrmw.start
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v9, v1
-; GFX8-NEXT:    v_mov_b32_e32 v8, v0
-; GFX8-NEXT:    v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execnz .LBB93_1
-; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB93_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
+; GFX9-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB93_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw max ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
@@ -5977,83 +7575,29 @@ define void @flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr %out
 ; GFX7-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
-; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_load_dword v7, v[0:1]
-; GFX7-NEXT:    flat_load_dword v6, v[8:9]
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:  .LBB105_1: ; %atomicrmw.start
-; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
+; GFX7-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_mov_b32_e32 v7, v1
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    v_mov_b32_e32 v6, v0
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB105_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
-; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dword v7, v[0:1]
-; GFX8-NEXT:    flat_load_dword v6, v[8:9]
-; GFX8-NEXT:    s_mov_b64 s[4:5], 0
-; GFX8-NEXT:  .LBB105_1: ; %atomicrmw.start
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
+; GFX8-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_mov_b32_e32 v7, v1
-; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT:    v_mov_b32_e32 v6, v0
-; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execnz .LBB105_1
-; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB105_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
+; GFX9-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3] offset:32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB105_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
@@ -6064,85 +7608,29 @@ define i64 @flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GFX7-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
-; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_load_dword v1, v[0:1]
-; GFX7-NEXT:    flat_load_dword v0, v[4:5]
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:  .LBB106_1: ; %atomicrmw.start
-; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v9, v1
-; GFX7-NEXT:    v_mov_b32_e32 v8, v0
-; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX7-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB106_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
-; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dword v1, v[0:1]
-; GFX8-NEXT:    flat_load_dword v0, v[4:5]
-; GFX8-NEXT:    s_mov_b64 s[4:5], 0
-; GFX8-NEXT:  .LBB106_1: ; %atomicrmw.start
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v9, v1
-; GFX8-NEXT:    v_mov_b32_e32 v8, v0
-; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execnz .LBB106_1
-; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB106_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
+; GFX9-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB106_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw umax ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
@@ -6921,83 +8409,29 @@ define void @flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr %out
 ; GFX7-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
-; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
-; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_load_dword v7, v[0:1]
-; GFX7-NEXT:    flat_load_dword v6, v[8:9]
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:  .LBB115_1: ; %atomicrmw.start
-; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_mov_b32_e32 v7, v1
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    v_mov_b32_e32 v6, v0
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB115_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
-; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dword v7, v[0:1]
-; GFX8-NEXT:    flat_load_dword v6, v[8:9]
-; GFX8-NEXT:    s_mov_b64 s[4:5], 0
-; GFX8-NEXT:  .LBB115_1: ; %atomicrmw.start
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
+; GFX8-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_mov_b32_e32 v7, v1
-; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT:    v_mov_b32_e32 v6, v0
-; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execnz .LBB115_1
-; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB115_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
+; GFX9-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3] offset:32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB115_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
@@ -7008,85 +8442,29 @@ define i64 @flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i
 ; GFX7-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
-; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_load_dword v1, v[0:1]
-; GFX7-NEXT:    flat_load_dword v0, v[4:5]
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:  .LBB116_1: ; %atomicrmw.start
-; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v9, v1
-; GFX7-NEXT:    v_mov_b32_e32 v8, v0
-; GFX7-NEXT:    v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX7-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB116_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
-; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dword v1, v[0:1]
-; GFX8-NEXT:    flat_load_dword v0, v[4:5]
-; GFX8-NEXT:    s_mov_b64 s[4:5], 0
-; GFX8-NEXT:  .LBB116_1: ; %atomicrmw.start
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v9, v1
-; GFX8-NEXT:    v_mov_b32_e32 v8, v0
-; GFX8-NEXT:    v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execnz .LBB116_1
-; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB116_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
+; GFX9-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB116_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw umin ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
@@ -8296,83 +9674,29 @@ define void @flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr %out,
 ; GFX7-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
-; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_load_dword v7, v[0:1]
-; GFX7-NEXT:    flat_load_dword v6, v[8:9]
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:  .LBB129_1: ; %atomicrmw.start
-; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
+; GFX7-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
-; GFX7-NEXT:    v_mov_b32_e32 v7, v1
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    v_mov_b32_e32 v6, v0
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB129_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
-; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dword v7, v[0:1]
-; GFX8-NEXT:    flat_load_dword v6, v[8:9]
-; GFX8-NEXT:    s_mov_b64 s[4:5], 0
-; GFX8-NEXT:  .LBB129_1: ; %atomicrmw.start
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
+; GFX8-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
-; GFX8-NEXT:    v_mov_b32_e32 v7, v1
-; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT:    v_mov_b32_e32 v6, v0
-; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execnz .LBB129_1
-; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB129_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
+; GFX9-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3] offset:32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB129_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
@@ -8383,85 +9707,29 @@ define i64 @flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr %out, i6
 ; GFX7-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
-; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_load_dword v1, v[0:1]
-; GFX7-NEXT:    flat_load_dword v0, v[4:5]
-; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:  .LBB130_1: ; %atomicrmw.start
-; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v9, v1
-; GFX7-NEXT:    v_mov_b32_e32 v8, v0
-; GFX7-NEXT:    v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
-; GFX7-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX7-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX7-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
-; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT:    s_cbranch_execnz .LBB130_1
-; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
-; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dword v1, v[0:1]
-; GFX8-NEXT:    flat_load_dword v0, v[4:5]
-; GFX8-NEXT:    s_mov_b64 s[4:5], 0
-; GFX8-NEXT:  .LBB130_1: ; %atomicrmw.start
-; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v9, v1
-; GFX8-NEXT:    v_mov_b32_e32 v8, v0
-; GFX8-NEXT:    v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX8-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
-; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX8-NEXT:    s_cbranch_execnz .LBB130_1
-; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB130_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
+; GFX9-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB130_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw min ptr %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0, !noalias.addrspace !1
@@ -8476,25 +9744,85 @@ define void @flat_atomic_uinc_wrap_i64_noret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v6, v[0:1]
+; GFX7-NEXT:    flat_load_dword v7, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB131_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 1, v6
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB131_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v6, v[0:1]
+; GFX8-NEXT:    flat_load_dword v7, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB131_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 1, v6
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB131_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB131_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB131_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -8504,29 +9832,89 @@ define void @flat_atomic_uinc_wrap_i64_noret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dword v7, v[0:1]
+; GFX7-NEXT:    flat_load_dword v6, v[8:9]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB132_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v6
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB132_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dword v7, v[0:1]
+; GFX8-NEXT:    flat_load_dword v6, v[8:9]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB132_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v6
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB132_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3] offset:32
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB132_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB132_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -8537,25 +9925,91 @@ define i64 @flat_atomic_uinc_wrap_i64_ret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_add_i32_e32 v5, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v4, v[0:1]
+; GFX7-NEXT:    flat_load_dword v5, v[5:6]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB133_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 1, v6
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB133_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
+; GFX7-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v4, v[0:1]
+; GFX8-NEXT:    flat_load_dword v5, v[5:6]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB133_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 1, v6
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB133_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
+; GFX8-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB133_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB133_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -8565,29 +10019,91 @@ define i64 @flat_atomic_uinc_wrap_i64_ret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[4:5], 0
+; GFX7-NEXT:  .LBB134_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v8
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v9, vcc
+; GFX7-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT:    s_cbranch_execnz .LBB134_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[4:5], 0
+; GFX8-NEXT:  .LBB134_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v8
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v9, vcc
+; GFX8-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT:    s_cbranch_execnz .LBB134_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB134_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB134_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -8598,37 +10114,101 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v3, s34
+; GFX7-NEXT:    v_mov_b32_e32 v4, s35
+; GFX7-NEXT:    flat_load_dword v2, v[0:1]
+; GFX7-NEXT:    flat_load_dword v3, v[3:4]
+; GFX7-NEXT:    v_mov_b32_e32 v4, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:    v_mov_b32_e32 v5, s5
+; GFX7-NEXT:  .LBB135_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB135_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s34
+; GFX8-NEXT:    v_mov_b32_e32 v4, s35
+; GFX8-NEXT:    flat_load_dword v2, v[0:1]
+; GFX8-NEXT:    flat_load_dword v3, v[3:4]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:  .LBB135_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB135_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB135_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB135_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -8640,13 +10220,34 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
+; GFX7-NEXT:    v_mov_b32_e32 v4, s34
+; GFX7-NEXT:    v_mov_b32_e32 v5, s35
+; GFX7-NEXT:    flat_load_dword v3, v[0:1]
+; GFX7-NEXT:    flat_load_dword v2, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:  .LBB136_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB136_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
@@ -8654,25 +10255,64 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
+; GFX8-NEXT:    v_mov_b32_e32 v4, s34
+; GFX8-NEXT:    v_mov_b32_e32 v5, s35
+; GFX8-NEXT:    flat_load_dword v3, v[0:1]
+; GFX8-NEXT:    flat_load_dword v2, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:  .LBB136_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB136_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB136_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB136_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -8683,37 +10323,101 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GFX7-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s34
+; GFX7-NEXT:    v_mov_b32_e32 v3, s35
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:  .LBB137_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v6
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB137_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s34
+; GFX8-NEXT:    v_mov_b32_e32 v3, s35
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:  .LBB137_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v6
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB137_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX9-NEXT:  .LBB137_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB137_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw uinc_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -8725,13 +10429,34 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[2:3]
+; GFX7-NEXT:    s_mov_b64 s[34:35], 0
+; GFX7-NEXT:  .LBB138_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 1, v6
+; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX7-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX7-NEXT:    s_cbranch_execnz .LBB138_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
@@ -8739,25 +10464,64 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[2:3]
+; GFX8-NEXT:    s_mov_b64 s[34:35], 0
+; GFX8-NEXT:  .LBB138_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 1, v6
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX8-NEXT:    s_cbranch_execnz .LBB138_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] offset:32
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
+; GFX9-NEXT:  .LBB138_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB138_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw uinc_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -8838,25 +10602,91 @@ define void @flat_atomic_udec_wrap_i64_noret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v6, v[0:1]
+; GFX7-NEXT:    flat_load_dword v7, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
+; GFX7-NEXT:  .LBB141_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX7-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v4, s[6:7], -1, v6
+; GFX7-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    s_cbranch_execnz .LBB141_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v6, v[0:1]
+; GFX8-NEXT:    flat_load_dword v7, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[8:9], 0
+; GFX8-NEXT:  .LBB141_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX8-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v4, s[6:7], -1, v6
+; GFX8-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT:    s_cbranch_execnz .LBB141_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[8:9], 0
+; GFX9-NEXT:  .LBB141_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_cbranch_execnz .LBB141_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -8866,29 +10696,95 @@ define void @flat_atomic_udec_wrap_i64_noret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v8, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GFX7-NEXT:    flat_load_dword v7, v[0:1]
+; GFX7-NEXT:    flat_load_dword v6, v[8:9]
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
+; GFX7-NEXT:  .LBB142_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX7-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[6:7], -1, v6
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[6:7], -1, v7, s[6:7]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, v1, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT:    v_mov_b32_e32 v7, v1
+; GFX7-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7-NEXT:    v_mov_b32_e32 v6, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    s_cbranch_execnz .LBB142_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; GFX8-NEXT:    flat_load_dword v7, v[0:1]
+; GFX8-NEXT:    flat_load_dword v6, v[8:9]
+; GFX8-NEXT:    s_mov_b64 s[8:9], 0
+; GFX8-NEXT:  .LBB142_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX8-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[6:7], -1, v6
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[6:7], -1, v7, s[6:7]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v1, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT:    v_mov_b32_e32 v7, v1
+; GFX8-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT:    v_mov_b32_e32 v6, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT:    s_cbranch_execnz .LBB142_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3] offset:32
+; GFX9-NEXT:    flat_load_dwordx2 v[6:7], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[8:9], 0
+; GFX9-NEXT:  .LBB142_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_cbranch_execnz .LBB142_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -8899,25 +10795,97 @@ define i64 @flat_atomic_udec_wrap_i64_ret(ptr %ptr, i64 %in) {
 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    v_add_i32_e32 v5, vcc, 4, v0
+; GFX7-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX7-NEXT:    flat_load_dword v4, v[0:1]
+; GFX7-NEXT:    flat_load_dword v5, v[5:6]
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
+; GFX7-NEXT:  .LBB143_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v7, v5
+; GFX7-NEXT:    v_mov_b32_e32 v6, v4
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX7-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v4, s[6:7], -1, v6
+; GFX7-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    s_cbranch_execnz .LBB143_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    v_mov_b32_e32 v0, v4
+; GFX7-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT:    v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_load_dword v4, v[0:1]
+; GFX8-NEXT:    flat_load_dword v5, v[5:6]
+; GFX8-NEXT:    s_mov_b64 s[8:9], 0
+; GFX8-NEXT:  .LBB143_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v7, v5
+; GFX8-NEXT:    v_mov_b32_e32 v6, v4
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX8-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v4, s[6:7], -1, v6
+; GFX8-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT:    s_cbranch_execnz .LBB143_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT:    v_mov_b32_e32 v0, v4
+; GFX8-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; GFX9-NEXT:    s_mov_b64 s[8:9], 0
+; GFX9-NEXT:  .LBB143_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_cbranch_execnz .LBB143_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -8927,29 +10895,97 @@ define i64 @flat_atomic_udec_wrap_i64_ret_offset(ptr %out, i64 %in) {
 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 32, v0
+; GFX7-NEXT:    v_add_i32_e32 v4, vcc, 32, v0
+; GFX7-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, 36, v0
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[8:9], 0
+; GFX7-NEXT:  .LBB144_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX7-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[6:7], -1, v8
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v1, v3, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT:    s_cbranch_execnz .LBB144_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 36, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[8:9], 0
+; GFX8-NEXT:  .LBB144_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[6:7], -1, v8
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v1, v3, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT:    s_cbranch_execnz .LBB144_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] offset:32 glc
+; GFX9-NEXT:    flat_load_dwordx2 v[4:5], v[0:1] offset:32
+; GFX9-NEXT:    s_mov_b64 s[8:9], 0
+; GFX9-NEXT:  .LBB144_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_cbranch_execnz .LBB144_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -8960,37 +10996,113 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v3, s34
+; GFX7-NEXT:    v_mov_b32_e32 v4, s35
+; GFX7-NEXT:    flat_load_dword v2, v[0:1]
+; GFX7-NEXT:    flat_load_dword v3, v[3:4]
+; GFX7-NEXT:    v_mov_b32_e32 v4, s4
+; GFX7-NEXT:    s_mov_b64 s[38:39], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s7
+; GFX7-NEXT:    v_mov_b32_e32 v7, s6
+; GFX7-NEXT:    v_mov_b32_e32 v5, s5
+; GFX7-NEXT:  .LBB145_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[36:37], -1, v2
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX7-NEXT:    s_cbranch_execnz .LBB145_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v2, s4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v3, s34
+; GFX8-NEXT:    v_mov_b32_e32 v4, s35
+; GFX8-NEXT:    flat_load_dword v2, v[0:1]
+; GFX8-NEXT:    flat_load_dword v3, v[3:4]
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    s_mov_b64 s[38:39], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:  .LBB145_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v2
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX8-NEXT:    s_cbranch_execnz .LBB145_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[38:39], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, s7
+; GFX9-NEXT:    v_mov_b32_e32 v7, s6
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB145_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v2
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX9-NEXT:    s_cbranch_execnz .LBB145_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret void
@@ -9002,13 +11114,38 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
+; GFX7-NEXT:    v_mov_b32_e32 v4, s34
+; GFX7-NEXT:    v_mov_b32_e32 v5, s35
+; GFX7-NEXT:    flat_load_dword v3, v[0:1]
+; GFX7-NEXT:    flat_load_dword v2, v[4:5]
+; GFX7-NEXT:    s_mov_b64 s[38:39], 0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s7
+; GFX7-NEXT:    v_mov_b32_e32 v7, s6
+; GFX7-NEXT:  .LBB146_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[36:37], -1, v2
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX7-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT:    v_mov_b32_e32 v3, v1
+; GFX7-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX7-NEXT:    v_mov_b32_e32 v2, v0
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX7-NEXT:    s_cbranch_execnz .LBB146_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
@@ -9016,25 +11153,72 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
+; GFX8-NEXT:    v_mov_b32_e32 v4, s34
+; GFX8-NEXT:    v_mov_b32_e32 v5, s35
+; GFX8-NEXT:    flat_load_dword v3, v[0:1]
+; GFX8-NEXT:    flat_load_dword v2, v[4:5]
+; GFX8-NEXT:    s_mov_b64 s[38:39], 0
+; GFX8-NEXT:    v_mov_b32_e32 v6, s7
+; GFX8-NEXT:    v_mov_b32_e32 v7, s6
+; GFX8-NEXT:  .LBB146_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v2
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT:    v_mov_b32_e32 v3, v1
+; GFX8-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX8-NEXT:    v_mov_b32_e32 v2, v0
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX8-NEXT:    s_cbranch_execnz .LBB146_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_mov_b64 s[38:39], 0
+; GFX9-NEXT:    v_mov_b32_e32 v6, s7
+; GFX9-NEXT:    v_mov_b32_e32 v7, s6
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:  .LBB146_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v2
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX9-NEXT:    s_cbranch_execnz .LBB146_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %tmp0 = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
@@ -9045,37 +11229,113 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
 ; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    s_add_u32 s34, s4, 4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s34
+; GFX7-NEXT:    v_mov_b32_e32 v3, s35
+; GFX7-NEXT:    flat_load_dword v0, v[0:1]
+; GFX7-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
+; GFX7-NEXT:    s_mov_b64 s[38:39], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s7
+; GFX7-NEXT:    v_mov_b32_e32 v5, s6
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:  .LBB147_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[36:37], -1, v8
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX7-NEXT:    s_cbranch_execnz .LBB147_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    s_add_u32 s34, s4, 4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v2, s34
+; GFX8-NEXT:    v_mov_b32_e32 v3, s35
+; GFX8-NEXT:    flat_load_dword v0, v[0:1]
+; GFX8-NEXT:    flat_load_dword v1, v[2:3]
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s4
+; GFX8-NEXT:    s_mov_b64 s[38:39], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s7
+; GFX8-NEXT:    v_mov_b32_e32 v5, s6
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:  .LBB147_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v8
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX8-NEXT:    s_cbranch_execnz .LBB147_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[38:39], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, s7
+; GFX9-NEXT:    v_mov_b32_e32 v5, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX9-NEXT:  .LBB147_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v9, v1
+; GFX9-NEXT:    v_mov_b32_e32 v8, v0
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GFX9-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v8
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX9-NEXT:    s_cbranch_execnz .LBB147_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw udec_wrap ptr %ptr, i64 %in seq_cst, !noalias.addrspace !1
   ret i64 %result
@@ -9087,13 +11347,38 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_add_u32 s34, s4, 32
 ; GFX7-NEXT:    s_addc_u32 s35, s5, 0
+; GFX7-NEXT:    s_add_u32 s36, s4, 36
+; GFX7-NEXT:    s_addc_u32 s37, s5, 0
+; GFX7-NEXT:    v_mov_b32_e32 v0, s36
+; GFX7-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s34
-; GFX7-NEXT:    v_mov_b32_e32 v0, s6
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s35
-; GFX7-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX7-NEXT:    flat_load_dword v1, v[0:1]
+; GFX7-NEXT:    flat_load_dword v0, v[2:3]
+; GFX7-NEXT:    s_mov_b64 s[38:39], 0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s7
+; GFX7-NEXT:    v_mov_b32_e32 v5, s6
+; GFX7-NEXT:  .LBB148_1: ; %atomicrmw.start
+; GFX7-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_mov_b32_e32 v9, v1
+; GFX7-NEXT:    v_mov_b32_e32 v8, v0
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX7-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GFX7-NEXT:    v_add_i32_e64 v0, s[36:37], -1, v8
+; GFX7-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GFX7-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX7-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX7-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX7-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    buffer_wbinvl1_vol
+; GFX7-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX7-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX7-NEXT:    s_cbranch_execnz .LBB148_1
+; GFX7-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
@@ -9101,25 +11386,72 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    s_add_u32 s34, s4, 32
 ; GFX8-NEXT:    s_addc_u32 s35, s5, 0
+; GFX8-NEXT:    s_add_u32 s36, s4, 36
+; GFX8-NEXT:    s_addc_u32 s37, s5, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s36
+; GFX8-NEXT:    v_mov_b32_e32 v1, s37
 ; GFX8-NEXT:    v_mov_b32_e32 v2, s34
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX8-NEXT:    v_mov_b32_e32 v3, s35
-; GFX8-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; GFX8-NEXT:    flat_load_dword v1, v[0:1]
+; GFX8-NEXT:    flat_load_dword v0, v[2:3]
+; GFX8-NEXT:    s_mov_b64 s[38:39], 0
+; GFX8-NEXT:    v_mov_b32_e32 v4, s7
+; GFX8-NEXT:    v_mov_b32_e32 v5, s6
+; GFX8-NEXT:  .LBB148_1: ; %atomicrmw.start
+; GFX8-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v9, v1
+; GFX8-NEXT:    v_mov_b32_e32 v8, v0
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX8-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GFX8-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v8
+; GFX8-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GFX8-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX8-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX8-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX8-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    buffer_wbinvl1_vol
+; GFX8-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX8-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX8-NEXT:    s_cbranch_execnz .LBB148_1
+; GFX8-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX8-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    flat_load_dwordx2 v[0:1], v[0:1] offset:32
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-NEXT:    s_mov_b64 s[38:39], 0
+; GFX9-NEXT:    v_mov_b32_e32 v4, s7
+; GFX9-NEXT:    v_mov_b32_e32 v5, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] offset:32 glc
+; GFX9-NEXT:  .LBB148_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v9, v1
+; GFX9-NEXT:    v_mov_b32_e32 v8, v0
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; GFX9-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v8
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX9-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX9-NEXT:    s_cbranch_execnz .LBB148_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr %out, i64 4
   %result = atomicrmw udec_wrap ptr %gep, i64 %in seq_cst, !noalias.addrspace !1
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.gfx11plus.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.gfx11plus.ll
new file mode 100644
index 0000000000000..5f86f2e48137d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.gfx11plus.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -stop-after=amdgpu-isel -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX11
+
+@const_half = internal constant half 1.0
+
+define amdgpu_kernel void @fma_v2f16_divergent(
+  ; GFX11-LABEL: name: fma_v2f16_divergent
+  ; GFX11: bb.0 (%ir-block.0):
+  ; GFX11-NEXT:   liveins: $vgpr0, $sgpr4_sgpr5
+  ; GFX11-NEXT: {{  $}}
+  ; GFX11-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; GFX11-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+  ; GFX11-NEXT:   [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s256) from %ir.r.kernarg.offset, align 4, addrspace 4)
+  ; GFX11-NEXT:   [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 68, 0 :: (dereferenceable invariant load (s32) from %ir.d.kernarg.offset, addrspace 4)
+  ; GFX11-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub1
+  ; GFX11-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub0
+  ; GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1
+  ; GFX11-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX11-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub3
+  ; GFX11-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub2
+  ; GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
+  ; GFX11-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub5
+  ; GFX11-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub4
+  ; GFX11-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1
+  ; GFX11-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub7
+  ; GFX11-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub6
+  ; GFX11-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1
+  ; GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1023
+  ; GFX11-NEXT:   [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]](s32), killed [[S_MOV_B32_]], implicit $exec
+  ; GFX11-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+  ; GFX11-NEXT:   [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = nuw nsw V_LSHLREV_B32_e64 killed [[S_MOV_B32_1]], killed [[V_AND_B32_e64_]], implicit $exec
+  ; GFX11-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 killed [[REG_SEQUENCE1]], killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $exec :: (load (s16) from %ir.f.gep, addrspace 1)
+  ; GFX11-NEXT:   [[V_AND_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_AND_B16_t16_e64 0, 32767, 0, [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], 0, implicit $exec
+  ; GFX11-NEXT:   [[V_XOR_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_XOR_B16_t16_e64 0, -32768, 0, [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], 0, implicit $exec
+  ; GFX11-NEXT:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; GFX11-NEXT:   S_CMP_LG_U32 killed [[S_LOAD_DWORD_IMM]], killed [[S_MOV_B32_2]], implicit-def $scc
+  ; GFX11-NEXT:   [[COPY10:%[0-9]+]]:sreg_32_xm0_xexec = COPY $scc
+  ; GFX11-NEXT:   [[V_CNDMASK_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_CNDMASK_B16_t16_e64 0, killed [[V_XOR_B16_t16_e64_]], 0, killed [[V_AND_B16_t16_e64_]], killed [[COPY10]], 0, implicit $exec
+  ; GFX11-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX11-NEXT:   [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+  ; GFX11-NEXT:   [[REG_SEQUENCE4:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[V_CNDMASK_B16_t16_e64_]], %subreg.lo16, killed [[DEF]], %subreg.hi16
+  ; GFX11-NEXT:   [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM killed [[REG_SEQUENCE2]], 0, 0 :: ("amdgpu-noclobber" load (s32) from %ir.4, addrspace 1)
+  ; GFX11-NEXT:   [[S_LOAD_DWORD_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM killed [[REG_SEQUENCE3]], 0, 0 :: ("amdgpu-noclobber" load (s32) from %ir.5, addrspace 1)
+  ; GFX11-NEXT:   [[V_PK_FMA_F16_:%[0-9]+]]:vgpr_32 = nofpexcept V_PK_FMA_F16 0, killed [[REG_SEQUENCE4]], 8, killed [[S_LOAD_DWORD_IMM1]], 8, killed [[S_LOAD_DWORD_IMM2]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX11-NEXT:   GLOBAL_STORE_DWORD_SADDR killed [[V_MOV_B32_e32_]], killed [[V_PK_FMA_F16_]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s32) into %ir.2, addrspace 1)
+  ; GFX11-NEXT:   S_ENDPGM 0
+      ptr addrspace(1) %r,
+    ptr addrspace(1) %fptr,
+    ptr addrspace(1) %b,
+    ptr addrspace(1) %c,
+	i32 %d) {
+
+  %idx = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %f.gep = getelementptr half, ptr addrspace(1) %fptr, i32 %idx
+  %f = load half, ptr addrspace(1) %f.gep
+  %f.abs = call half @llvm.fabs.f16(half %f)
+  %f.neg = fneg half %f
+  %setcc = icmp ne i32 %d, 0
+  %select = select i1 %setcc, half %f.abs, half %f.neg
+  %vec = insertelement <2 x half> poison, half %select, i32 0
+  %a.val = insertelement <2 x half> %vec, half %select, i32 1
+  %b.v = load i32, ptr addrspace(1) %b
+  %b.val = bitcast i32 %b.v to <2 x half>
+  %c.v = load i32, ptr addrspace(1) %c
+  %c.val = bitcast i32 %c.v to <2 x half>
+  %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> %a.val, <2 x half> %b.val, <2 x half> %c.val)
+  store <2 x half> %r.val, ptr addrspace(1) %r
+  ret void
+}
+
+define amdgpu_kernel void @fma_v2f16_uniform(
+  ; GFX11-LABEL: name: fma_v2f16_uniform
+  ; GFX11: bb.0 (%ir-block.0):
+  ; GFX11-NEXT:   liveins: $sgpr4_sgpr5
+  ; GFX11-NEXT: {{  $}}
+  ; GFX11-NEXT:   [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+  ; GFX11-NEXT:   [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s256) from %ir.r.kernarg.offset, align 4, addrspace 4)
+  ; GFX11-NEXT:   [[COPY1:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub1
+  ; GFX11-NEXT:   [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub0
+  ; GFX11-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, killed [[COPY1]], %subreg.sub1
+  ; GFX11-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX11-NEXT:   [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub3
+  ; GFX11-NEXT:   [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub2
+  ; GFX11-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY4]], %subreg.sub0, killed [[COPY3]], %subreg.sub1
+  ; GFX11-NEXT:   [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub5
+  ; GFX11-NEXT:   [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub4
+  ; GFX11-NEXT:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY6]], %subreg.sub0, killed [[COPY5]], %subreg.sub1
+  ; GFX11-NEXT:   [[COPY7:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub7
+  ; GFX11-NEXT:   [[COPY8:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub6
+  ; GFX11-NEXT:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1
+  ; GFX11-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 killed [[REG_SEQUENCE1]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: ("amdgpu-noclobber" load (s16) from %ir.3, addrspace 1)
+  ; GFX11-NEXT:   [[COPY9:%[0-9]+]]:sreg_32 = COPY [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]]
+  ; GFX11-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 killed [[COPY9]]
+  ; GFX11-NEXT:   [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM killed [[REG_SEQUENCE2]], 0, 0 :: ("amdgpu-noclobber" load (s32) from %ir.4, addrspace 1)
+  ; GFX11-NEXT:   [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM killed [[REG_SEQUENCE3]], 0, 0 :: ("amdgpu-noclobber" load (s32) from %ir.5, addrspace 1)
+  ; GFX11-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY killed [[S_LOAD_DWORD_IMM1]]
+  ; GFX11-NEXT:   [[V_PK_FMA_F16_:%[0-9]+]]:vgpr_32 = nofpexcept V_PK_FMA_F16 0, killed [[S_MOV_B32_]], 8, killed [[S_LOAD_DWORD_IMM]], 8, [[COPY10]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; GFX11-NEXT:   GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_]], killed [[V_PK_FMA_F16_]], killed [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s32) into %ir.2, addrspace 1)
+  ; GFX11-NEXT:   S_ENDPGM 0
+      ptr addrspace(1) %r,
+    ptr addrspace(1) %a,
+    ptr addrspace(1) %b,
+    ptr addrspace(1) %c) {
+  %a.half = load half, ptr addrspace(1) %a
+  %vec = insertelement <2 x half> poison, half %a.half, i32 0
+  %a.val = insertelement <2 x half> %vec, half %a.half, i32 1
+  %b.v = load i32, ptr addrspace(1) %b
+  %b.val = bitcast i32 %b.v to <2 x half>
+  %c.v = load i32, ptr addrspace(1) %c
+  %c.val = bitcast i32 %c.v to <2 x half>
+  %r.val = call <2 x half> @llvm.fma.v2f16(<2 x half> %a.val, <2 x half> %b.val, <2 x half> %c.val)
+  store <2 x half> %r.val, ptr addrspace(1) %r
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index 9c4901eb19f37..899cc89405440 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -4238,7 +4238,7 @@ define amdgpu_ps i32 @s_mul_32_f16(half inreg %x, half inreg %y) {
 ; GFX11-GISEL-TRUE16-LABEL: s_mul_32_f16:
 ; GFX11-GISEL-TRUE16:       ; %bb.0:
 ; GFX11-GISEL-TRUE16-NEXT:    v_mul_f16_e64 v0.l, 0x5000, s0
-; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-GISEL-TRUE16-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11-GISEL-TRUE16-NEXT:    ; return to shader part epilog
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/fncall-implicitdef.ll b/llvm/test/CodeGen/AMDGPU/fncall-implicitdef.ll
new file mode 100644
index 0000000000000..66a8b424b5763
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fncall-implicitdef.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -O1 %s -o - | FileCheck %s
+
+define amdgpu_ps <4 x float> @caller(ptr %ptr) {
+; CHECK-LABEL: caller:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    flat_load_dword v1, v[0:1]
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_mov_b32 s1, 0
+; CHECK-NEXT:    s_mov_b32 s2, 0
+; CHECK-NEXT:    s_mov_b32 s5, fn@abs32@hi
+; CHECK-NEXT:    s_mov_b32 s4, fn@abs32@lo
+; CHECK-NEXT:    s_mov_b64 s[8:9], 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_mov_b32 s3, 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    s_mov_b32 s32, 0
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; CHECK-NEXT:    ; return to shader part epilog
+  %L = load i32, ptr %ptr, align 4
+  %R = call <4 x float> @fn(<4 x i32> zeroinitializer, i32 0, i32 %L, i32 0)
+  ret <4 x float> %R
+}
+
+declare hidden <4 x float> @fn(<4 x i32> inreg, i32, i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
index 1b092b283290a..5674ae328406d 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
@@ -349,29 +349,24 @@ define i32 @select_fneg_xor_select_i32(i1 %cond0, i1 %cond1, i32 %arg0, i32 %arg
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
-; GCN-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GCN-NEXT:    v_and_b32_e32 v1, 1, v1
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc
-; GCN-NEXT:    v_xor_b32_e32 v2, 0x80000000, v0
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, -v2, v3, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
-; GCN-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v0, -v0, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: select_fneg_xor_select_i32:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX11-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
 ; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, -v2, v3, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
-; GFX11-NEXT:    v_xor_b32_e32 v2, 0x80000000, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, -v0, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %fneg0 = xor i32 %arg0, -2147483648
   %select0 = select i1 %cond0, i32 %arg1, i32 %fneg0
@@ -550,31 +545,25 @@ define i64 @select_fneg_xor_select_i64(i1 %cond0, i1 %cond1, i64 %arg0, i64 %arg
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
-; GCN-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
-; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GCN-NEXT:    v_and_b32_e32 v1, 1, v1
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
-; GCN-NEXT:    v_cndmask_b32_e32 v2, v3, v5, vcc
-; GCN-NEXT:    v_xor_b32_e32 v3, 0x80000000, v2
+; GCN-NEXT:    v_cndmask_b32_e64 v2, -v3, v5, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
-; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v2, -v2, vcc
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: select_fneg_xor_select_i64:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    v_and_b32_e32 v0, 1, v0
-; GFX11-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
-; GFX11-NEXT:    v_and_b32_e32 v1, 1, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, v3, v5, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_dual_cndmask_b32 v0, v2, v4 :: v_dual_and_b32 v1, 1, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, -v3, v5, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v1
-; GFX11-NEXT:    v_xor_b32_e32 v3, 0x80000000, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, -v2, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %fneg0 = xor i64 %arg0, 9223372036854775808
   %select0 = select i1 %cond0, i64 %arg1, i64 %fneg0
diff --git a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
index 3e2680f55832d..6bb68e1e26a14 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-gep-offset.ll
@@ -1,12 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-MUBUF %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-FLATSCR %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-MUBUF %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-FLATSCR %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-SDAG,GFX90A-MUBUF,GFX90A-SDAG-MUBUF %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-SDAG,GFX90A-FLATSCR,GFX90A-SDAG-FLATSCR %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-SDAG,GFX10-MUBUF,GFX10-SDAG-MUBUF %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1030 -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-SDAG,GFX10-FLATSCR,GFX10-SDAG-FLATSCR %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11,GFX11-SDAG %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-SDAG %s
+
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-GISEL,GFX90A-MUBUF,GFX90A-GISEL-MUBUF %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx90a -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX90A,GFX90A-GISEL,GFX90A-FLATSCR,GFX90A-GISEL-FLATSCR %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -mattr=-enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-GISEL,GFX10-MUBUF,GFX10-GISEL-MUBUF %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1030 -mattr=+enable-flat-scratch < %s | FileCheck --check-prefixes=GFX10,GFX10-GISEL,GFX10-FLATSCR,GFX10-GISEL-FLATSCR %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck --check-prefixes=GFX942,GFX942-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck --check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12,GFX12-GISEL %s
 
 ; This test checks memory addresses with constant offset components that should
 ; not be folded into memory accesses with immediate offsets.
@@ -19,67 +27,146 @@
 ; FIXME the offset here should not be folded: if %p points to the beginning of
 ; scratch or LDS and %i is -1, a folded offset crashes the program.
 define i32 @flat_offset_maybe_oob(ptr %p, i32 %i) {
-; GFX90A-LABEL: flat_offset_maybe_oob:
-; GFX90A:       ; %bb.0:
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX90A-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
-; GFX90A-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
-; GFX90A-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
-; GFX90A-NEXT:    flat_load_dword v0, v[0:1] offset:12
-; GFX90A-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: flat_offset_maybe_oob:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX10-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
-; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX10-NEXT:    flat_load_dword v0, v[0:1] offset:12
-; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: flat_offset_maybe_oob:
-; GFX942:       ; %bb.0:
-; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX942-NEXT:    v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
-; GFX942-NEXT:    flat_load_dword v0, v[0:1] offset:12
-; GFX942-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT:    s_setpc_b64 s[30:31]
+; GFX90A-SDAG-LABEL: flat_offset_maybe_oob:
+; GFX90A-SDAG:       ; %bb.0:
+; GFX90A-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX90A-SDAG-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX90A-SDAG-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX90A-SDAG-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX90A-SDAG-NEXT:    flat_load_dword v0, v[0:1] offset:12
+; GFX90A-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: flat_offset_maybe_oob:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
-; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-NEXT:    flat_load_b32 v0, v[0:1] offset:12
-; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-SDAG-LABEL: flat_offset_maybe_oob:
+; GFX10-SDAG:       ; %bb.0:
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX10-SDAG-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX10-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX10-SDAG-NEXT:    flat_load_dword v0, v[0:1] offset:12
+; GFX10-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-SDAG-LABEL: flat_offset_maybe_oob:
+; GFX942-SDAG:       ; %bb.0:
+; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX942-SDAG-NEXT:    v_lshl_add_u64 v[0:1], v[2:3], 2, v[0:1]
+; GFX942-SDAG-NEXT:    flat_load_dword v0, v[0:1] offset:12
+; GFX942-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: flat_offset_maybe_oob:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT:    v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
-; GFX12-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
-; GFX12-NEXT:    s_wait_alu 0xfffd
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX12-NEXT:    flat_load_b32 v0, v[0:1] offset:12
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-SDAG-LABEL: flat_offset_maybe_oob:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX11-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-SDAG-NEXT:    flat_load_b32 v0, v[0:1] offset:12
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-SDAG-LABEL: flat_offset_maybe_oob:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT:    v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
+; GFX12-SDAG-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffd
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-SDAG-NEXT:    flat_load_b32 v0, v[0:1] offset:12
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-GISEL-LABEL: flat_offset_maybe_oob:
+; GFX90A-GISEL:       ; %bb.0:
+; GFX90A-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX90A-GISEL-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX90A-GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX90A-GISEL-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX90A-GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 12, v0
+; GFX90A-GISEL-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX90A-GISEL-NEXT:    flat_load_dword v0, v[0:1]
+; GFX90A-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: flat_offset_maybe_oob:
+; GFX10-GISEL:       ; %bb.0:
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX10-GISEL-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX10-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX10-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 12
+; GFX10-GISEL-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX10-GISEL-NEXT:    flat_load_dword v0, v[0:1]
+; GFX10-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: flat_offset_maybe_oob:
+; GFX942-GISEL:       ; %bb.0:
+; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX942-GISEL-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v2
+; GFX942-GISEL-NEXT:    s_nop 1
+; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v1, vcc, v1, v3, vcc
+; GFX942-GISEL-NEXT:    v_add_co_u32_e32 v0, vcc, 12, v0
+; GFX942-GISEL-NEXT:    s_nop 1
+; GFX942-GISEL-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT:    flat_load_dword v0, v[0:1]
+; GFX942-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: flat_offset_maybe_oob:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_lshlrev_b64 v[2:3], 2, v[2:3]
+; GFX11-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 12
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-GISEL-NEXT:    flat_load_b32 v0, v[0:1]
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: flat_offset_maybe_oob:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_lshlrev_b64_e32 v[2:3], 2, v[2:3]
+; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX12-GISEL-NEXT:    v_add_co_u32 v0, vcc_lo, v0, 12
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffd
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-GISEL-NEXT:    flat_load_b32 v0, v[0:1]
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %idx = add nsw i32 %i, 3
   %arrayidx = getelementptr inbounds i32, ptr %p, i32 %idx
   %l = load i32, ptr %arrayidx
@@ -88,13 +175,13 @@ define i32 @flat_offset_maybe_oob(ptr %p, i32 %i) {
 
 ; For MUBUF and for GFX12, folding the offset is okay.
 define i32 @private_offset_maybe_oob(ptr addrspace(5) %p, i32 %i) {
-; GFX90A-MUBUF-LABEL: private_offset_maybe_oob:
-; GFX90A-MUBUF:       ; %bb.0:
-; GFX90A-MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-MUBUF-NEXT:    v_lshl_add_u32 v0, v1, 2, v0
-; GFX90A-MUBUF-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen offset:12
-; GFX90A-MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-MUBUF-NEXT:    s_setpc_b64 s[30:31]
+; GFX90A-SDAG-MUBUF-LABEL: private_offset_maybe_oob:
+; GFX90A-SDAG-MUBUF:       ; %bb.0:
+; GFX90A-SDAG-MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-SDAG-MUBUF-NEXT:    v_lshl_add_u32 v0, v1, 2, v0
+; GFX90A-SDAG-MUBUF-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen offset:12
+; GFX90A-SDAG-MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-SDAG-MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX90A-FLATSCR-LABEL: private_offset_maybe_oob:
 ; GFX90A-FLATSCR:       ; %bb.0:
@@ -105,13 +192,13 @@ define i32 @private_offset_maybe_oob(ptr addrspace(5) %p, i32 %i) {
 ; GFX90A-FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-FLATSCR-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX10-MUBUF-LABEL: private_offset_maybe_oob:
-; GFX10-MUBUF:       ; %bb.0:
-; GFX10-MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-MUBUF-NEXT:    v_lshl_add_u32 v0, v1, 2, v0
-; GFX10-MUBUF-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen offset:12
-; GFX10-MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-MUBUF-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-SDAG-MUBUF-LABEL: private_offset_maybe_oob:
+; GFX10-SDAG-MUBUF:       ; %bb.0:
+; GFX10-SDAG-MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-MUBUF-NEXT:    v_lshl_add_u32 v0, v1, 2, v0
+; GFX10-SDAG-MUBUF-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen offset:12
+; GFX10-SDAG-MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-SDAG-MUBUF-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-FLATSCR-LABEL: private_offset_maybe_oob:
 ; GFX10-FLATSCR:       ; %bb.0:
@@ -141,19 +228,61 @@ define i32 @private_offset_maybe_oob(ptr addrspace(5) %p, i32 %i) {
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: private_offset_maybe_oob:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_lshl_add_u32 v0, v1, 2, v0
-; GFX12-NEXT:    scratch_load_b32 v0, v0, off offset:12
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX12-SDAG-LABEL: private_offset_maybe_oob:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    v_lshl_add_u32 v0, v1, 2, v0
+; GFX12-SDAG-NEXT:    scratch_load_b32 v0, v0, off offset:12
+; GFX12-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX90A-GISEL-MUBUF-LABEL: private_offset_maybe_oob:
+; GFX90A-GISEL-MUBUF:       ; %bb.0:
+; GFX90A-GISEL-MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-GISEL-MUBUF-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX90A-GISEL-MUBUF-NEXT:    v_add_u32_e32 v0, v0, v1
+; GFX90A-GISEL-MUBUF-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen offset:12
+; GFX90A-GISEL-MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; GFX90A-GISEL-MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-MUBUF-LABEL: private_offset_maybe_oob:
+; GFX10-GISEL-MUBUF:       ; %bb.0:
+; GFX10-GISEL-MUBUF-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-MUBUF-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX10-GISEL-MUBUF-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX10-GISEL-MUBUF-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen offset:12
+; GFX10-GISEL-MUBUF-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-GISEL-MUBUF-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: private_offset_maybe_oob:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT:    v_add_nc_u32_e32 v0, v0, v1
+; GFX12-GISEL-NEXT:    scratch_load_b32 v0, v0, off offset:12
+; GFX12-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %idx = add nsw i32 %i, 3
   %arrayidx = getelementptr inbounds i32, ptr addrspace(5) %p, i32 %idx
   %l = load i32, ptr addrspace(5) %arrayidx
   ret i32 %l
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
+; GFX10-GISEL-FLATSCR: {{.*}}
+; GFX10-MUBUF: {{.*}}
+; GFX10-SDAG-FLATSCR: {{.*}}
+; GFX12: {{.*}}
+; GFX90A: {{.*}}
+; GFX90A-GISEL-FLATSCR: {{.*}}
+; GFX90A-MUBUF: {{.*}}
+; GFX90A-SDAG-FLATSCR: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
index 74c4a2da50221..9a51f457a567a 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
@@ -63,8 +63,8 @@ tracksRegLiveness: true
 body:             |
   bb.0:
     ; GCN-LABEL: name: clear_subreg_imm_fold
-    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 4294967288
-    ; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 4294967295
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sgpr_32 = S_MOV_B32 -8
+    ; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 -1
     ; GCN-NEXT: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]]
     %0:sreg_64 = S_MOV_B64 -8
     %1:sgpr_32 = COPY %0.sub0
@@ -781,3 +781,110 @@ body:             |
     S_ENDPGM 0
 
 ...
+
+---
+name: av_mov_b64_imm_pseudo_copy_av_64_to_physreg_agpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b64_imm_pseudo_copy_av_64_to_physreg_agpr
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    ; GCN-NEXT: $agpr0_agpr1 = COPY [[AV_MOV_]]
+    ; GCN-NEXT: S_ENDPGM 0, implicit $agpr0_agpr1
+    %0:av_64 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    $agpr0_agpr1 = COPY %0
+    S_ENDPGM 0, implicit $agpr0_agpr1
+
+...
+
+---
+name: av_mov_b64_imm_pseudo_copy_av_64_to_physreg_vgpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b64_imm_pseudo_copy_av_64_to_physreg_vgpr
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    ; GCN-NEXT: $vgpr0_vgpr1 = COPY [[AV_MOV_]]
+    ; GCN-NEXT: S_ENDPGM 0, implicit $vgpr0_vgpr1
+    %0:av_64 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    $vgpr0_vgpr1 = COPY %0
+    S_ENDPGM 0, implicit $vgpr0_vgpr1
+
+...
+
+---
+name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_agpr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_agpr
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:areg_64 = COPY [[AV_MOV_]]
+    ; GCN-NEXT: S_ENDPGM 0, implicit [[COPY]]
+    %0:av_64 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    %1:areg_64 = COPY %0
+    S_ENDPGM 0, implicit %1
+
+...
+
+# Splat value across 2 halves of register
+---
+name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_vgpr_0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_vgpr_0
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY [[AV_MOV_]]
+    ; GCN-NEXT: S_ENDPGM 0, implicit [[COPY]]
+    %0:av_64 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    %1:vreg_64 = COPY %0
+    S_ENDPGM 0, implicit %1
+
+...
+
+# Low and hi are different inline constants
+---
+name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_vgpr_nonsplat_value
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_vgpr_nonsplat_value
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64 = AV_MOV_B64_IMM_PSEUDO 274877906961, implicit $exec
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64 = COPY [[AV_MOV_]]
+    ; GCN-NEXT: S_ENDPGM 0, implicit [[COPY]]
+    %0:av_64 = AV_MOV_B64_IMM_PSEUDO 274877906961, implicit $exec
+    %1:vreg_64 = COPY %0
+    S_ENDPGM 0, implicit %1
+
+...
+
+---
+name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_vgpr_nonsplat_value_copy_sub0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_vgpr_nonsplat_value_copy_sub0
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64 = AV_MOV_B64_IMM_PSEUDO 274877906961, implicit $exec
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub0
+    ; GCN-NEXT: S_ENDPGM 0, implicit [[COPY]]
+    %0:av_64 = AV_MOV_B64_IMM_PSEUDO 274877906961, implicit $exec
+    %1:vgpr_32 = COPY %0.sub0
+    S_ENDPGM 0, implicit %1
+
+...
+
+---
+name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_vgpr_nonsplat_value_copy_sub1
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: av_mov_b64_imm_pseudo_copy_av_64_to_virtreg_vgpr_nonsplat_value_copy_sub1
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64 = AV_MOV_B64_IMM_PSEUDO 274877906961, implicit $exec
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[AV_MOV_]].sub1
+    ; GCN-NEXT: S_ENDPGM 0, implicit [[COPY]]
+    %0:av_64 = AV_MOV_B64_IMM_PSEUDO 274877906961, implicit $exec
+    %1:vgpr_32 = COPY %0.sub1
+    S_ENDPGM 0, implicit %1
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
index f09c25767648f..a859cc91b7fde 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -644,11 +644,10 @@ define double @fmul_pow_mul_max_pow2(i16 %cnt) nounwind {
 ; GFX11-TRUE16-LABEL: fmul_pow_mul_max_pow2:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, v0.l, 2
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_mul_f64 v[0:1], 0x40080000, v[0:1]
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1194,13 +1193,12 @@ define double @fmul_pow_shl_cnt_safe(i16 %cnt) nounwind {
 ; GFX11-TRUE16-LABEL: fmul_pow_shl_cnt_safe:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, v0.l, 1
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s0, 0xff5f3992
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s1, 0x7befffff
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-TRUE16-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_mul_f64 v[0:1], v[0:1], s[0:1]
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.mir b/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.mir
index 7fad2f466bc9f..a88b1ecc40cc9 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-operands-frame-index.mir
@@ -75,7 +75,8 @@ stack:
 body:             |
   bb.0:
     ; CHECK-LABEL: name: fold_frame_index__s_add_i32__fi_materializedconst_0
-    ; CHECK: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 %stack.0, 256, implicit-def $scc
+    ; CHECK: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 256
+    ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 %stack.0, [[S_MOV_B32_]], implicit-def $scc
     ; CHECK-NEXT: $sgpr4 = COPY [[S_ADD_I32_]]
     ; CHECK-NEXT: SI_RETURN implicit $sgpr4
     %0:sreg_32 = S_MOV_B32 %stack.0
diff --git a/llvm/test/CodeGen/AMDGPU/fold-sgpr-multi-imm.mir b/llvm/test/CodeGen/AMDGPU/fold-sgpr-multi-imm.mir
index cc4314263bcba..2f2d727ee2c59 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-sgpr-multi-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-sgpr-multi-imm.mir
@@ -46,7 +46,8 @@ body: |
     %2:sreg_32 = S_LSHL2_ADD_U32 %0, %1, implicit-def $scc
 ...
 # GCN-LABEL: name: test_frameindex{{$}}
-# GCN: %1:sreg_32 = S_ADD_I32 %stack.0, 70
+# GCN:      [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 70
+# GCN-NEXT: %1:sreg_32 = S_ADD_I32 %stack.0, [[S_MOV_B32_]]
 ---
 name: test_frameindex
 tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
index f9a24fee59692..0cb2b0b7df3d2 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
@@ -2102,23 +2102,10 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret(ptr addrspace(3) %ptr, do
 ; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x24
 ; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x2c
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s2
-; GFX1250-NEXT:    s_mov_b32 s2, 0
-; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
-; GFX1250-NEXT:  .LBB51_1: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s2
+; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[4:5], v2, v[4:5], v[0:1]
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], v[4:5]
-; GFX1250-NEXT:    s_or_b32 s2, vcc_lo, s2
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT:    s_cbranch_execnz .LBB51_1
-; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX1250-NEXT:    s_endpgm
 main_body:
   %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)
@@ -2148,24 +2135,9 @@ define double @local_atomic_fadd_f64_rtn(ptr addrspace(3) %ptr, double %data) {
 ; GFX1250:       ; %bb.0: ; %main_body
 ; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v2, v0
-; GFX1250-NEXT:    v_mov_b32_e32 v4, v1
-; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:  .LBB52_1: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_mov_b64_e32 v[6:7], v[0:1]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[0:1], v[6:7], v[4:5]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[6:7]
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB52_1
-; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)
@@ -2197,24 +2169,11 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat(ptr addrspace(3) %ptr
 ; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat:
 ; GFX1250:       ; %bb.0: ; %main_body
 ; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x24
+; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], 4.0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v2, s0
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
-; GFX1250-NEXT:  .LBB53_1: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[4:5], 4.0, v[0:1]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[4:5], v2, v[4:5], v[0:1]
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], v[4:5]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB53_1
-; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX1250-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -2246,24 +2205,11 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush(ptr addrspace(3
 ; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush:
 ; GFX1250:       ; %bb.0: ; %main_body
 ; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x24
+; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], 4.0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v2, s0
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
-; GFX1250-NEXT:  .LBB54_1: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[4:5], 4.0, v[0:1]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[4:5], v2, v[4:5], v[0:1]
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], v[4:5]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB54_1
-; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX1250-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -2295,24 +2241,11 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
 ; GFX1250-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
 ; GFX1250:       ; %bb.0: ; %main_body
 ; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x24
+; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], 4.0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v2, s0
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
-; GFX1250-NEXT:  .LBB55_1: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[4:5], 4.0, v[0:1]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[4:5], v2, v[4:5], v[0:1]
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_add_f64 v2, v[0:1]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], v[4:5]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB55_1
-; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX1250-NEXT:    s_endpgm
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst
@@ -2341,23 +2274,9 @@ define double @local_atomic_fadd_f64_rtn_pat(ptr addrspace(3) %ptr, double %data
 ; GFX1250:       ; %bb.0: ; %main_body
 ; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_mov_b32_e32 v2, v0
-; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:  .LBB56_1: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_mov_b64_e32 v[4:5], v[0:1]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[0:1], 4.0, v[4:5]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[4:5]
+; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], 4.0
+; GFX1250-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB56_1
-; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
 main_body:
   %ret = atomicrmw fadd ptr addrspace(3) %ptr, double 4.0 seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -2387,24 +2306,9 @@ define double @local_atomic_fadd_f64_rtn_ieee_unsafe(ptr addrspace(3) %ptr, doub
 ; GFX1250:       ; %bb.0: ; %main_body
 ; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v2, v0
-; GFX1250-NEXT:    v_mov_b32_e32 v4, v1
-; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:  .LBB57_1: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_mov_b64_e32 v[6:7], v[0:1]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[0:1], v[6:7], v[4:5]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[6:7]
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB57_1
-; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)
@@ -2434,24 +2338,9 @@ define double @local_atomic_fadd_f64_rtn_ieee_safe(ptr addrspace(3) %ptr, double
 ; GFX1250:       ; %bb.0: ; %main_body
 ; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v2, v0
-; GFX1250-NEXT:    v_mov_b32_e32 v4, v1
-; GFX1250-NEXT:    ds_load_b64 v[0:1], v0
-; GFX1250-NEXT:    s_mov_b32 s0, 0
-; GFX1250-NEXT:  .LBB58_1: ; %atomicrmw.start
-; GFX1250-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_mov_b64_e32 v[6:7], v[0:1]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT:    v_add_f64_e32 v[0:1], v[6:7], v[4:5]
-; GFX1250-NEXT:    ds_cmpstore_rtn_b64 v[0:1], v2, v[0:1], v[6:7]
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    ds_add_rtn_f64 v[0:1], v0, v[2:3]
 ; GFX1250-NEXT:    s_wait_dscnt 0x0
-; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[6:7]
-; GFX1250-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX1250-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-NEXT:    s_cbranch_execnz .LBB58_1
-; GFX1250-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX1250-NEXT:    s_or_b32 exec_lo, exec_lo, s0
 ; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
 main_body:
   %ret = call double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) %ptr, double %data, i32 0, i32 0, i1 0)
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index 189b897793381..cdd34cbde6ddd 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -223,9 +223,10 @@ define i128 @fptosi_f64_to_i128(double %x) {
 ; GISEL-NEXT:    v_or3_b32 v8, v1, v2, 1
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0x433
 ; GISEL-NEXT:    v_mov_b32_e32 v1, 0
-; GISEL-NEXT:    v_and_b32_e32 v2, 0xfffff, v5
+; GISEL-NEXT:    v_mov_b32_e32 v2, 0xfffff
+; GISEL-NEXT:    s_mov_b32 s6, 0x100000
 ; GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, v[6:7], v[0:1]
-; GISEL-NEXT:    v_or_b32_e32 v5, 0x100000, v2
+; GISEL-NEXT:    v_and_or_b32 v5, v5, v2, s6
 ; GISEL-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GISEL-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; GISEL-NEXT:    s_xor_b64 s[16:17], exec, s[6:7]
@@ -587,9 +588,10 @@ define i128 @fptoui_f64_to_i128(double %x) {
 ; GISEL-NEXT:    v_or3_b32 v8, v1, v2, 1
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0x433
 ; GISEL-NEXT:    v_mov_b32_e32 v1, 0
-; GISEL-NEXT:    v_and_b32_e32 v2, 0xfffff, v5
+; GISEL-NEXT:    v_mov_b32_e32 v2, 0xfffff
+; GISEL-NEXT:    s_mov_b32 s6, 0x100000
 ; GISEL-NEXT:    v_cmp_ge_u64_e32 vcc, v[6:7], v[0:1]
-; GISEL-NEXT:    v_or_b32_e32 v5, 0x100000, v2
+; GISEL-NEXT:    v_and_or_b32 v5, v5, v2, s6
 ; GISEL-NEXT:    ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
 ; GISEL-NEXT:    s_and_saveexec_b64 s[6:7], vcc
 ; GISEL-NEXT:    s_xor_b64 s[16:17], exec, s[6:7]
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index c52fb6197e3e3..40d2765395543 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -4372,14 +4372,13 @@ define amdgpu_kernel void @fptrunc_f32_to_f16_zext_i32(
 ; GFX11-GISEL-TRUE16-LABEL: fptrunc_f32_to_f16_zext_i32:
 ; GFX11-GISEL-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-TRUE16-NEXT:    s_load_b32 s2, s[2:3], 0x0
 ; GFX11-GISEL-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, s2
 ; GFX11-GISEL-TRUE16-NEXT:    s_mov_b32 s2, -1
-; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-GISEL-TRUE16-NEXT:    buffer_store_b32 v0, off, s[0:3], 0
 ; GFX11-GISEL-TRUE16-NEXT:    s_endpgm
 ;
@@ -4607,14 +4606,13 @@ define amdgpu_kernel void @fptrunc_fabs_f32_to_f16_zext_i32(
 ; GFX11-GISEL-TRUE16-LABEL: fptrunc_fabs_f32_to_f16_zext_i32:
 ; GFX11-GISEL-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-TRUE16-NEXT:    s_load_b32 s2, s[2:3], 0x0
 ; GFX11-GISEL-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-TRUE16-NEXT:    v_cvt_f16_f32_e64 v0.l, |s2|
 ; GFX11-GISEL-TRUE16-NEXT:    s_mov_b32 s2, -1
-; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-GISEL-TRUE16-NEXT:    buffer_store_b32 v0, off, s[0:3], 0
 ; GFX11-GISEL-TRUE16-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll b/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll
index 15cda622b902d..f2fe61f5376e4 100644
--- a/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll
+++ b/llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll
@@ -360,7 +360,8 @@ entry:
 ; s_add_i32.
 
 ; GCN-LABEL: {{^}}fi_sop2_s_add_u32_literal_error:
-; GCN: s_add_u32 [[ADD_LO:s[0-9]+]], 0, 0x2010
+; GCN: s_movk_i32 [[S_MOVK_I32_:s[0-9]+]], 0x1000
+; GCN: s_add_u32 [[ADD_LO:s[0-9]+]], 0x1010, [[S_MOVK_I32_]]
 ; GCN: s_addc_u32 [[ADD_HI:s[0-9]+]], s{{[0-9]+}}, 0
 define amdgpu_kernel void @fi_sop2_s_add_u32_literal_error() #0 {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 0df1a0fcc76ef..35913b9a21d30 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -1582,28 +1582,22 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
 ; SI-NEXT:    s_nop 1
 ; SI-NEXT:    v_div_fmas_f64 v[4:5], v[12:13], v[6:7], v[10:11]
 ; SI-NEXT:    v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
-; SI-NEXT:    v_readfirstlane_b32 s2, v5
-; SI-NEXT:    s_bfe_u32 s0, s2, 0xb0014
-; SI-NEXT:    s_add_i32 s3, s0, 0xfffffc01
-; SI-NEXT:    s_mov_b32 s1, 0xfffff
-; SI-NEXT:    s_mov_b32 s0, s6
-; SI-NEXT:    s_lshr_b64 s[0:1], s[0:1], s3
-; SI-NEXT:    v_not_b32_e32 v6, s0
-; SI-NEXT:    v_and_b32_e32 v6, v4, v6
-; SI-NEXT:    v_not_b32_e32 v7, s1
-; SI-NEXT:    v_and_b32_e32 v5, v5, v7
-; SI-NEXT:    s_and_b32 s0, s2, 0x80000000
-; SI-NEXT:    s_cmp_lt_i32 s3, 0
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_cndmask_b32_e64 v6, v6, 0, vcc
-; SI-NEXT:    v_mov_b32_e32 v7, s0
-; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT:    s_cmp_gt_i32 s3, 51
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_mov_b32_e32 v7, s2
-; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
+; SI-NEXT:    v_readfirstlane_b32 s0, v4
+; SI-NEXT:    v_readfirstlane_b32 s1, v5
+; SI-NEXT:    s_bfe_u32 s2, s1, 0xb0014
+; SI-NEXT:    s_add_i32 s8, s2, 0xfffffc01
+; SI-NEXT:    s_mov_b32 s3, 0xfffff
+; SI-NEXT:    s_mov_b32 s2, s6
+; SI-NEXT:    s_lshr_b64 s[2:3], s[2:3], s8
+; SI-NEXT:    s_andn2_b64 s[2:3], s[0:1], s[2:3]
+; SI-NEXT:    s_and_b32 s9, s1, 0x80000000
+; SI-NEXT:    s_cmp_lt_i32 s8, 0
+; SI-NEXT:    s_cselect_b32 s2, 0, s2
+; SI-NEXT:    s_cselect_b32 s3, s9, s3
+; SI-NEXT:    s_cmp_gt_i32 s8, 51
+; SI-NEXT:    s_cselect_b32 s1, s1, s3
+; SI-NEXT:    s_cselect_b32 s0, s0, s2
+; SI-NEXT:    v_fma_f64 v[0:1], -s[0:1], v[2:3], v[0:1]
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -1859,28 +1853,22 @@ define amdgpu_kernel void @fast_frem_f64(ptr addrspace(1) %out, ptr addrspace(1)
 ; SI-NEXT:    v_mul_f64 v[6:7], v[0:1], v[4:5]
 ; SI-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
 ; SI-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
-; SI-NEXT:    v_readfirstlane_b32 s6, v5
-; SI-NEXT:    s_bfe_u32 s4, s6, 0xb0014
-; SI-NEXT:    s_add_i32 s7, s4, 0xfffffc01
-; SI-NEXT:    s_mov_b32 s5, 0xfffff
-; SI-NEXT:    s_mov_b32 s4, s2
-; SI-NEXT:    s_lshr_b64 s[4:5], s[4:5], s7
-; SI-NEXT:    v_not_b32_e32 v6, s4
-; SI-NEXT:    v_and_b32_e32 v6, v4, v6
-; SI-NEXT:    v_not_b32_e32 v7, s5
-; SI-NEXT:    v_and_b32_e32 v5, v5, v7
-; SI-NEXT:    s_and_b32 s4, s6, 0x80000000
-; SI-NEXT:    s_cmp_lt_i32 s7, 0
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_cndmask_b32_e64 v6, v6, 0, vcc
-; SI-NEXT:    v_mov_b32_e32 v7, s4
-; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT:    s_cmp_gt_i32 s7, 51
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_mov_b32_e32 v7, s6
-; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
+; SI-NEXT:    v_readfirstlane_b32 s4, v4
+; SI-NEXT:    v_readfirstlane_b32 s5, v5
+; SI-NEXT:    s_bfe_u32 s6, s5, 0xb0014
+; SI-NEXT:    s_add_i32 s8, s6, 0xfffffc01
+; SI-NEXT:    s_mov_b32 s7, 0xfffff
+; SI-NEXT:    s_mov_b32 s6, s2
+; SI-NEXT:    s_lshr_b64 s[6:7], s[6:7], s8
+; SI-NEXT:    s_andn2_b64 s[6:7], s[4:5], s[6:7]
+; SI-NEXT:    s_and_b32 s9, s5, 0x80000000
+; SI-NEXT:    s_cmp_lt_i32 s8, 0
+; SI-NEXT:    s_cselect_b32 s6, 0, s6
+; SI-NEXT:    s_cselect_b32 s7, s9, s7
+; SI-NEXT:    s_cmp_gt_i32 s8, 51
+; SI-NEXT:    s_cselect_b32 s5, s5, s7
+; SI-NEXT:    s_cselect_b32 s4, s4, s6
+; SI-NEXT:    v_fma_f64 v[0:1], -s[4:5], v[2:3], v[0:1]
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -2109,28 +2097,22 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
 ; SI-NEXT:    v_mul_f64 v[6:7], v[0:1], v[4:5]
 ; SI-NEXT:    v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
 ; SI-NEXT:    v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
-; SI-NEXT:    v_readfirstlane_b32 s6, v5
-; SI-NEXT:    s_bfe_u32 s4, s6, 0xb0014
-; SI-NEXT:    s_add_i32 s7, s4, 0xfffffc01
-; SI-NEXT:    s_mov_b32 s5, 0xfffff
-; SI-NEXT:    s_mov_b32 s4, s2
-; SI-NEXT:    s_lshr_b64 s[4:5], s[4:5], s7
-; SI-NEXT:    v_not_b32_e32 v6, s4
-; SI-NEXT:    v_and_b32_e32 v6, v4, v6
-; SI-NEXT:    v_not_b32_e32 v7, s5
-; SI-NEXT:    v_and_b32_e32 v5, v5, v7
-; SI-NEXT:    s_and_b32 s4, s6, 0x80000000
-; SI-NEXT:    s_cmp_lt_i32 s7, 0
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_cndmask_b32_e64 v6, v6, 0, vcc
-; SI-NEXT:    v_mov_b32_e32 v7, s4
-; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT:    s_cmp_gt_i32 s7, 51
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_mov_b32_e32 v7, s6
-; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT:    v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
+; SI-NEXT:    v_readfirstlane_b32 s4, v4
+; SI-NEXT:    v_readfirstlane_b32 s5, v5
+; SI-NEXT:    s_bfe_u32 s6, s5, 0xb0014
+; SI-NEXT:    s_add_i32 s8, s6, 0xfffffc01
+; SI-NEXT:    s_mov_b32 s7, 0xfffff
+; SI-NEXT:    s_mov_b32 s6, s2
+; SI-NEXT:    s_lshr_b64 s[6:7], s[6:7], s8
+; SI-NEXT:    s_andn2_b64 s[6:7], s[4:5], s[6:7]
+; SI-NEXT:    s_and_b32 s9, s5, 0x80000000
+; SI-NEXT:    s_cmp_lt_i32 s8, 0
+; SI-NEXT:    s_cselect_b32 s6, 0, s6
+; SI-NEXT:    s_cselect_b32 s7, s9, s7
+; SI-NEXT:    s_cmp_gt_i32 s8, 51
+; SI-NEXT:    s_cselect_b32 s5, s5, s7
+; SI-NEXT:    s_cselect_b32 s4, s4, s6
+; SI-NEXT:    v_fma_f64 v[0:1], -s[4:5], v[2:3], v[0:1]
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -5251,27 +5233,22 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; SI-NEXT:    s_nop 1
 ; SI-NEXT:    v_div_fmas_f64 v[8:9], v[16:17], v[10:11], v[14:15]
 ; SI-NEXT:    v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
-; SI-NEXT:    v_readfirstlane_b32 s8, v9
-; SI-NEXT:    s_bfe_u32 s0, s8, 0xb0014
-; SI-NEXT:    s_add_i32 s9, s0, 0xfffffc01
+; SI-NEXT:    v_readfirstlane_b32 s0, v8
+; SI-NEXT:    v_readfirstlane_b32 s1, v9
+; SI-NEXT:    s_bfe_u32 s2, s1, 0xb0014
+; SI-NEXT:    s_add_i32 s10, s2, 0xfffffc01
 ; SI-NEXT:    s_mov_b32 s3, 0xfffff
-; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s9
-; SI-NEXT:    v_not_b32_e32 v10, s0
-; SI-NEXT:    v_and_b32_e32 v10, v8, v10
-; SI-NEXT:    v_not_b32_e32 v11, s1
-; SI-NEXT:    v_and_b32_e32 v9, v9, v11
-; SI-NEXT:    s_and_b32 s0, s8, 0x80000000
-; SI-NEXT:    s_cmp_lt_i32 s9, 0
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_cndmask_b32_e64 v10, v10, 0, vcc
-; SI-NEXT:    v_mov_b32_e32 v11, s0
-; SI-NEXT:    v_cndmask_b32_e32 v9, v9, v11, vcc
-; SI-NEXT:    s_cmp_gt_i32 s9, 51
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_mov_b32_e32 v11, s8
-; SI-NEXT:    v_cndmask_b32_e32 v9, v9, v11, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v8, v10, v8, vcc
-; SI-NEXT:    v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
+; SI-NEXT:    s_mov_b32 s2, s6
+; SI-NEXT:    s_lshr_b64 s[8:9], s[2:3], s10
+; SI-NEXT:    s_andn2_b64 s[8:9], s[0:1], s[8:9]
+; SI-NEXT:    s_and_b32 s11, s1, 0x80000000
+; SI-NEXT:    s_cmp_lt_i32 s10, 0
+; SI-NEXT:    s_cselect_b32 s8, 0, s8
+; SI-NEXT:    s_cselect_b32 s9, s11, s9
+; SI-NEXT:    s_cmp_gt_i32 s10, 51
+; SI-NEXT:    s_cselect_b32 s1, s1, s9
+; SI-NEXT:    s_cselect_b32 s0, s0, s8
+; SI-NEXT:    v_fma_f64 v[2:3], -s[0:1], v[6:7], v[2:3]
 ; SI-NEXT:    v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1]
 ; SI-NEXT:    v_rcp_f64_e32 v[8:9], v[6:7]
 ; SI-NEXT:    v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
@@ -5287,26 +5264,20 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
 ; SI-NEXT:    s_nop 1
 ; SI-NEXT:    v_div_fmas_f64 v[6:7], v[14:15], v[8:9], v[12:13]
 ; SI-NEXT:    v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
-; SI-NEXT:    v_readfirstlane_b32 s8, v7
-; SI-NEXT:    s_bfe_u32 s0, s8, 0xb0014
-; SI-NEXT:    s_add_i32 s9, s0, 0xfffffc01
-; SI-NEXT:    s_lshr_b64 s[0:1], s[2:3], s9
-; SI-NEXT:    v_not_b32_e32 v8, s0
-; SI-NEXT:    v_and_b32_e32 v8, v6, v8
-; SI-NEXT:    v_not_b32_e32 v9, s1
-; SI-NEXT:    v_and_b32_e32 v7, v7, v9
-; SI-NEXT:    s_and_b32 s0, s8, 0x80000000
-; SI-NEXT:    s_cmp_lt_i32 s9, 0
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_cndmask_b32_e64 v8, v8, 0, vcc
-; SI-NEXT:    v_mov_b32_e32 v9, s0
-; SI-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
-; SI-NEXT:    s_cmp_gt_i32 s9, 51
-; SI-NEXT:    s_cselect_b64 vcc, -1, 0
-; SI-NEXT:    v_mov_b32_e32 v9, s8
-; SI-NEXT:    v_cndmask_b32_e32 v7, v7, v9, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
-; SI-NEXT:    v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
+; SI-NEXT:    v_readfirstlane_b32 s0, v6
+; SI-NEXT:    v_readfirstlane_b32 s1, v7
+; SI-NEXT:    s_bfe_u32 s8, s1, 0xb0014
+; SI-NEXT:    s_addk_i32 s8, 0xfc01
+; SI-NEXT:    s_lshr_b64 s[2:3], s[2:3], s8
+; SI-NEXT:    s_andn2_b64 s[2:3], s[0:1], s[2:3]
+; SI-NEXT:    s_and_b32 s9, s1, 0x80000000
+; SI-NEXT:    s_cmp_lt_i32 s8, 0
+; SI-NEXT:    s_cselect_b32 s2, 0, s2
+; SI-NEXT:    s_cselect_b32 s3, s9, s3
+; SI-NEXT:    s_cmp_gt_i32 s8, 51
+; SI-NEXT:    s_cselect_b32 s1, s1, s3
+; SI-NEXT:    s_cselect_b32 s0, s0, s2
+; SI-NEXT:    v_fma_f64 v[0:1], -s[0:1], v[4:5], v[0:1]
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll
index 8f3b9a5006fc4..cffa287dd91f5 100644
--- a/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsqrt.f64.ll
@@ -1,136 +1,472 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=pitcairn < %s | FileCheck -check-prefixes=SDAG %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=SDAG %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=pitcairn < %s | FileCheck -check-prefixes=GFX6,GFX6-SDAG %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
 
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=pitcairn < %s | FileCheck -check-prefixes=GISEL %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=pitcairn < %s | FileCheck -check-prefixes=GFX6,GFX6-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
 
 define double @v_sqrt_f64(double %x) {
-; GISEL-LABEL: v_sqrt_f64:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64_fneg(double %x) {
-; GISEL-LABEL: v_sqrt_f64_fneg:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, -v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_fneg:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 9
+; GFX6-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_fneg:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 9
+; GFX8-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_fneg:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, -v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_fneg:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, -v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], -v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %x.neg = fneg double %x
   %result = call double @llvm.sqrt.f64(double %x.neg)
   ret double %result
 }
 
 define double @v_sqrt_f64_fabs(double %x) {
-; GISEL-LABEL: v_sqrt_f64_fabs:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_fabs:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_fabs:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_fabs:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_fabs:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %x.fabs = call double @llvm.fabs.f64(double %x)
   %result = call double @llvm.sqrt.f64(double %x.fabs)
   ret double %result
 }
 
 define double @v_sqrt_f64_fneg_fabs(double %x) {
-; GISEL-LABEL: v_sqrt_f64_fneg_fabs:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, -|v[0:1]|, v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], -|v[0:1]|, v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_fneg_fabs:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 9
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], -|v[0:1]|, v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_fneg_fabs:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 9
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], -|v[0:1]|, v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_fneg_fabs:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, -|v[0:1]|, v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], -|v[0:1]|, v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_fneg_fabs:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, -|v[0:1]|, v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], -|v[0:1]|, v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %x.fabs = call double @llvm.fabs.f64(double %x)
   %x.fabs.neg = fneg double %x.fabs
   %result = call double @llvm.sqrt.f64(double %x.fabs.neg)
@@ -138,159 +474,469 @@ define double @v_sqrt_f64_fneg_fabs(double %x) {
 }
 
 define double @v_sqrt_f64_ninf(double %x) {
-; GISEL-LABEL: v_sqrt_f64_ninf:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_ninf:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_ninf:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_ninf:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_ninf:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call ninf double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64_no_infs_attribute(double %x) "no-infs-fp-math"="true" {
-; GISEL-LABEL: v_sqrt_f64_no_infs_attribute:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_no_infs_attribute:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_no_infs_attribute:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_no_infs_attribute:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_no_infs_attribute:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call ninf double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64_nnan(double %x) {
-; GISEL-LABEL: v_sqrt_f64_nnan:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_nnan:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_nnan:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_nnan:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_nnan:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call nnan double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define amdgpu_ps <2 x i32> @s_sqrt_f64(double inreg %x) {
-; SDAG-LABEL: s_sqrt_f64:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
-; SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
-; SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
-; SDAG-NEXT:    v_mov_b32_e32 v0, s2
-; SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
-; SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
-; SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
-; SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
-; SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
-; SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
-; SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SDAG-NEXT:    v_readfirstlane_b32 s0, v0
-; SDAG-NEXT:    v_readfirstlane_b32 s1, v1
-; SDAG-NEXT:    ; return to shader part epilog
-;
-; GISEL-LABEL: s_sqrt_f64:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    v_readfirstlane_b32 s0, v0
-; GISEL-NEXT:    v_readfirstlane_b32 s1, v1
-; GISEL-NEXT:    ; return to shader part epilog
+; GFX6-SDAG-LABEL: s_sqrt_f64:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX6-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX6-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX6-SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX6-SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-SDAG-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX8-SDAG-LABEL: s_sqrt_f64:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX8-SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX8-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX8-SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX8-SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-SDAG-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX6-GISEL-LABEL: s_sqrt_f64:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-GISEL-NEXT:    ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: s_sqrt_f64:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-GISEL-NEXT:    ; return to shader part epilog
   %result = call double @llvm.sqrt.f64(double %x)
   %cast = bitcast double %result to <2 x i32>
   %cast.0 = extractelement <2 x i32> %cast, i32 0
@@ -303,63 +949,121 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64(double inreg %x) {
 }
 
 define amdgpu_ps <2 x i32> @s_sqrt_f64_ninf(double inreg %x) {
-; SDAG-LABEL: s_sqrt_f64_ninf:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
-; SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
-; SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
-; SDAG-NEXT:    v_mov_b32_e32 v0, s2
-; SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
-; SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
-; SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
-; SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
-; SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
-; SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
-; SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SDAG-NEXT:    v_readfirstlane_b32 s0, v0
-; SDAG-NEXT:    v_readfirstlane_b32 s1, v1
-; SDAG-NEXT:    ; return to shader part epilog
-;
-; GISEL-LABEL: s_sqrt_f64_ninf:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    v_readfirstlane_b32 s0, v0
-; GISEL-NEXT:    v_readfirstlane_b32 s1, v1
-; GISEL-NEXT:    ; return to shader part epilog
+; GFX6-SDAG-LABEL: s_sqrt_f64_ninf:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX6-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX6-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX6-SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX6-SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-SDAG-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX8-SDAG-LABEL: s_sqrt_f64_ninf:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX8-SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX8-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX8-SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX8-SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-SDAG-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX6-GISEL-LABEL: s_sqrt_f64_ninf:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-GISEL-NEXT:    ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: s_sqrt_f64_ninf:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-GISEL-NEXT:    ; return to shader part epilog
   %result = call ninf double @llvm.sqrt.f64(double %x)
   %cast = bitcast double %result to <2 x i32>
   %cast.0 = extractelement <2 x i32> %cast, i32 0
@@ -372,63 +1076,121 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64_ninf(double inreg %x) {
 }
 
 define amdgpu_ps <2 x i32> @s_sqrt_f64_afn(double inreg %x) {
-; SDAG-LABEL: s_sqrt_f64_afn:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
-; SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
-; SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
-; SDAG-NEXT:    v_mov_b32_e32 v0, s2
-; SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
-; SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
-; SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
-; SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
-; SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
-; SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
-; SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SDAG-NEXT:    v_readfirstlane_b32 s0, v0
-; SDAG-NEXT:    v_readfirstlane_b32 s1, v1
-; SDAG-NEXT:    ; return to shader part epilog
-;
-; GISEL-LABEL: s_sqrt_f64_afn:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    v_readfirstlane_b32 s0, v0
-; GISEL-NEXT:    v_readfirstlane_b32 s1, v1
-; GISEL-NEXT:    ; return to shader part epilog
+; GFX6-SDAG-LABEL: s_sqrt_f64_afn:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX6-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX6-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX6-SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX6-SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-SDAG-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX8-SDAG-LABEL: s_sqrt_f64_afn:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX8-SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX8-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX8-SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX8-SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-SDAG-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX6-GISEL-LABEL: s_sqrt_f64_afn:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-GISEL-NEXT:    ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: s_sqrt_f64_afn:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-GISEL-NEXT:    ; return to shader part epilog
   %result = call afn double @llvm.sqrt.f64(double %x)
   %cast = bitcast double %result to <2 x i32>
   %cast.0 = extractelement <2 x i32> %cast, i32 0
@@ -441,63 +1203,121 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64_afn(double inreg %x) {
 }
 
 define amdgpu_ps <2 x i32> @s_sqrt_f64_afn_nnan_ninf(double inreg %x) {
-; SDAG-LABEL: s_sqrt_f64_afn_nnan_ninf:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
-; SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
-; SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
-; SDAG-NEXT:    v_mov_b32_e32 v0, s2
-; SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
-; SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
-; SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
-; SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
-; SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
-; SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
-; SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; SDAG-NEXT:    v_readfirstlane_b32 s0, v0
-; SDAG-NEXT:    v_readfirstlane_b32 s1, v1
-; SDAG-NEXT:    ; return to shader part epilog
-;
-; GISEL-LABEL: s_sqrt_f64_afn_nnan_ninf:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
-; GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    v_readfirstlane_b32 s0, v0
-; GISEL-NEXT:    v_readfirstlane_b32 s1, v1
-; GISEL-NEXT:    ; return to shader part epilog
+; GFX6-SDAG-LABEL: s_sqrt_f64_afn_nnan_ninf:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX6-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX6-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX6-SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX6-SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-SDAG-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX8-SDAG-LABEL: s_sqrt_f64_afn_nnan_ninf:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX8-SDAG-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX8-SDAG-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-SDAG-NEXT:    s_and_b64 s[2:3], vcc, exec
+; GFX8-SDAG-NEXT:    s_cselect_b32 s2, 0x100, 0
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v0, s2
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX8-SDAG-NEXT:    s_cselect_b32 s0, 0xffffff80, 0
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0x260
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v4
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], s0
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-SDAG-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX6-GISEL-LABEL: s_sqrt_f64_afn_nnan_ninf:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX6-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX6-GISEL-NEXT:    ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: s_sqrt_f64_afn_nnan_ninf:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v1, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, s[0:1], v[0:1]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], s[0:1], v0
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX8-GISEL-NEXT:    ; return to shader part epilog
   %result = call afn nnan ninf double @llvm.sqrt.f64(double %x)
   %cast = bitcast double %result to <2 x i32>
   %cast.0 = extractelement <2 x i32> %cast, i32 0
@@ -510,819 +1330,2244 @@ define amdgpu_ps <2 x i32> @s_sqrt_f64_afn_nnan_ninf(double inreg %x) {
 }
 
 define double @v_sqrt_f64_nsz(double %x) {
-; GISEL-LABEL: v_sqrt_f64_nsz:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_nsz:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_nsz:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_nsz:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_nsz:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call nsz double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64_nnan_ninf(double %x) {
-; GISEL-LABEL: v_sqrt_f64_nnan_ninf:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_nnan_ninf:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_nnan_ninf:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_nnan_ninf:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_nnan_ninf:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call nnan ninf double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64_nnan_ninf_nsz(double %x) {
-; GISEL-LABEL: v_sqrt_f64_nnan_ninf_nsz:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_nnan_ninf_nsz:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_nnan_ninf_nsz:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_nnan_ninf_nsz:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_nnan_ninf_nsz:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call nnan ninf nsz double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64_afn(double %x) {
-; GISEL-LABEL: v_sqrt_f64_afn:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_afn:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_afn:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_afn:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_afn:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call afn double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64_afn_nsz(double %x) {
-; GISEL-LABEL: v_sqrt_f64_afn_nsz:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_afn_nsz:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_afn_nsz:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_afn_nsz:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_afn_nsz:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call afn nsz double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define <2 x double> @v_sqrt_v2f64_afn(<2 x double> %x) {
-; SDAG-LABEL: v_sqrt_v2f64_afn:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    s_mov_b32 s4, 0
-; SDAG-NEXT:    s_brev_b32 s5, 8
-; SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[4:5], v[2:3]
-; SDAG-NEXT:    v_mov_b32_e32 v4, 0x100
-; SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[4:5]
-; SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v5
-; SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; SDAG-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
-; SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
-; SDAG-NEXT:    v_mul_f64 v[8:9], v[0:1], v[4:5]
-; SDAG-NEXT:    v_mul_f64 v[4:5], v[4:5], 0.5
-; SDAG-NEXT:    v_mul_f64 v[10:11], v[2:3], v[6:7]
-; SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
-; SDAG-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 0.5
-; SDAG-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 0.5
-; SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
-; SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
-; SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[8:9], v[12:13], v[4:5], v[8:9]
-; SDAG-NEXT:    v_fma_f64 v[10:11], v[14:15], v[6:7], v[10:11]
-; SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[8:9]
-; SDAG-NEXT:    v_mov_b32_e32 v8, 0xffffff80
-; SDAG-NEXT:    v_fma_f64 v[6:7], v[14:15], v[6:7], v[10:11]
-; SDAG-NEXT:    v_mov_b32_e32 v9, 0x260
-; SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
-; SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
-; SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
-; SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: v_sqrt_v2f64_afn:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    s_mov_b32 s4, 0
-; GISEL-NEXT:    s_brev_b32 s5, 8
-; GISEL-NEXT:    v_mov_b32_e32 v4, s4
-; GISEL-NEXT:    v_mov_b32_e32 v5, s5
-; GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
-; GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
-; GISEL-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
-; GISEL-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
-; GISEL-NEXT:    v_mul_f64 v[8:9], v[4:5], 0.5
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
-; GISEL-NEXT:    v_mul_f64 v[10:11], v[6:7], 0.5
-; GISEL-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
-; GISEL-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[4:5], 0.5
-; GISEL-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[6:7], 0.5
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
-; GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
-; GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
-; GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
-; GISEL-NEXT:    v_mov_b32_e32 v8, 0xffffff80
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
-; GISEL-NEXT:    v_mov_b32_e32 v9, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
-; GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
-; GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
-; GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_v2f64_afn:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[4:5], v[2:3]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[4:5]
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[8:9], v[0:1], v[4:5]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_mul_f64 v[10:11], v[2:3], v[6:7]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[8:9], v[12:13], v[4:5], v[8:9]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], v[14:15], v[6:7], v[10:11]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[8:9]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], v[14:15], v[6:7], v[10:11]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_v2f64_afn:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[4:5], v[2:3]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[4:5]
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[8:9], v[0:1], v[4:5]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_mul_f64 v[10:11], v[2:3], v[6:7]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[8:9], v[12:13], v[4:5], v[8:9]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], v[14:15], v[6:7], v[10:11]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[8:9]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], v[14:15], v[6:7], v[10:11]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_v2f64_afn:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX6-GISEL-NEXT:    s_brev_b32 s5, 8
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX6-GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[4:5]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[8:9], v[4:5], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[10:11], v[6:7], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[4:5], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[6:7], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_v2f64_afn:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX8-GISEL-NEXT:    s_brev_b32 s5, 8
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[4:5]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[8:9], v[4:5], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[10:11], v[6:7], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[4:5], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[6:7], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call afn <2 x double> @llvm.sqrt.v2f64(<2 x double> %x)
   ret <2 x double> %result
 }
 
 define double @v_sqrt_f64_afn_nnan(double %x) {
-; GISEL-LABEL: v_sqrt_f64_afn_nnan:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_afn_nnan:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_afn_nnan:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_afn_nnan:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_afn_nnan:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call afn nnan double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64_fabs_afn_ninf(double %x) {
-; GISEL-LABEL: v_sqrt_f64_fabs_afn_ninf:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_fabs_afn_ninf:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_fabs_afn_ninf:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, s[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_fabs_afn_ninf:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_fabs_afn_ninf:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e64 vcc, |v[0:1]|, v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], |v[0:1]|, v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %fabs = call double @llvm.fabs.f64(double %x)
   %result = call afn ninf double @llvm.sqrt.f64(double %fabs)
   ret double %result
 }
 
 define double @v_sqrt_f64_afn_nnan_ninf(double %x) {
-; GISEL-LABEL: v_sqrt_f64_afn_nnan_ninf:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_afn_nnan_ninf:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_afn_nnan_ninf:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_afn_nnan_ninf:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_afn_nnan_ninf:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call afn nnan ninf double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define <2 x double> @v_sqrt_v2f64_afn_nnan_ninf(<2 x double> %x) {
-; SDAG-LABEL: v_sqrt_v2f64_afn_nnan_ninf:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    s_mov_b32 s4, 0
-; SDAG-NEXT:    s_brev_b32 s5, 8
-; SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[4:5], v[2:3]
-; SDAG-NEXT:    v_mov_b32_e32 v4, 0x100
-; SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[4:5]
-; SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v5
-; SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; SDAG-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
-; SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
-; SDAG-NEXT:    v_mul_f64 v[8:9], v[0:1], v[4:5]
-; SDAG-NEXT:    v_mul_f64 v[4:5], v[4:5], 0.5
-; SDAG-NEXT:    v_mul_f64 v[10:11], v[2:3], v[6:7]
-; SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
-; SDAG-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 0.5
-; SDAG-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 0.5
-; SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
-; SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
-; SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[8:9], v[12:13], v[4:5], v[8:9]
-; SDAG-NEXT:    v_fma_f64 v[10:11], v[14:15], v[6:7], v[10:11]
-; SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[8:9]
-; SDAG-NEXT:    v_mov_b32_e32 v8, 0xffffff80
-; SDAG-NEXT:    v_fma_f64 v[6:7], v[14:15], v[6:7], v[10:11]
-; SDAG-NEXT:    v_mov_b32_e32 v9, 0x260
-; SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
-; SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
-; SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
-; SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: v_sqrt_v2f64_afn_nnan_ninf:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    s_mov_b32 s4, 0
-; GISEL-NEXT:    s_brev_b32 s5, 8
-; GISEL-NEXT:    v_mov_b32_e32 v4, s4
-; GISEL-NEXT:    v_mov_b32_e32 v5, s5
-; GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
-; GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
-; GISEL-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
-; GISEL-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
-; GISEL-NEXT:    v_mul_f64 v[8:9], v[4:5], 0.5
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
-; GISEL-NEXT:    v_mul_f64 v[10:11], v[6:7], 0.5
-; GISEL-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
-; GISEL-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[4:5], 0.5
-; GISEL-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[6:7], 0.5
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
-; GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
-; GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
-; GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
-; GISEL-NEXT:    v_mov_b32_e32 v8, 0xffffff80
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
-; GISEL-NEXT:    v_mov_b32_e32 v9, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
-; GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
-; GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
-; GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_v2f64_afn_nnan_ninf:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[4:5], v[2:3]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[4:5]
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[8:9], v[0:1], v[4:5]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_mul_f64 v[10:11], v[2:3], v[6:7]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[8:9], v[12:13], v[4:5], v[8:9]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], v[14:15], v[6:7], v[10:11]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[8:9]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], v[14:15], v[6:7], v[10:11]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_v2f64_afn_nnan_ninf:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[4:5], v[2:3]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[4:5]
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[8:9], v[0:1], v[4:5]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_mul_f64 v[10:11], v[2:3], v[6:7]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[8:9], v[12:13], v[4:5], v[8:9]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], v[14:15], v[6:7], v[10:11]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[8:9]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], v[14:15], v[6:7], v[10:11]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_v2f64_afn_nnan_ninf:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX6-GISEL-NEXT:    s_brev_b32 s5, 8
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX6-GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[4:5]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[8:9], v[4:5], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[10:11], v[6:7], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[4:5], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[6:7], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_v2f64_afn_nnan_ninf:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX8-GISEL-NEXT:    s_brev_b32 s5, 8
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[4:5]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[8:9], v[4:5], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[10:11], v[6:7], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[4:5], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[6:7], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call afn nnan ninf <2 x double> @llvm.sqrt.v2f64(<2 x double> %x)
   ret <2 x double> %result
 }
 
 define double @v_sqrt_f64_afn_nnan_ninf_nsz(double %x) {
-; GISEL-LABEL: v_sqrt_f64_afn_nnan_ninf_nsz:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64_afn_nnan_ninf_nsz:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64_afn_nnan_ninf_nsz:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64_afn_nnan_ninf_nsz:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64_afn_nnan_ninf_nsz:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call afn nnan ninf nsz double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64__approx_func_fp_math(double %x) #2 {
-; GISEL-LABEL: v_sqrt_f64__approx_func_fp_math:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64__approx_func_fp_math:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64__approx_func_fp_math:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64__approx_func_fp_math:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64__approx_func_fp_math:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call nsz double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64__enough_unsafe_attrs(double %x) #3 {
-; GISEL-LABEL: v_sqrt_f64__enough_unsafe_attrs:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64__enough_unsafe_attrs:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64__enough_unsafe_attrs:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64__enough_unsafe_attrs:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64__enough_unsafe_attrs:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call nsz double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define double @v_sqrt_f64__unsafe_attr(double %x) #4 {
-; GISEL-LABEL: v_sqrt_f64__unsafe_attr:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
-; GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
-; GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
-; GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_f64__unsafe_attr:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_f64__unsafe_attr:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v2, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[0:1], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[2:3], v[2:3], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[2:3], v[6:7], v[2:3], v[4:5]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_f64__unsafe_attr:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX6-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_f64__unsafe_attr:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-GISEL-NEXT:    v_bfrev_b32_e32 v3, 8
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e32 vcc, v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v2
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[2:3], v[0:1], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[4:5], v[2:3], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[2:3], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[2:3], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[2:3], v[6:7], v[4:5], v[2:3]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v5
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call nsz double @llvm.sqrt.f64(double %x)
   ret double %result
 }
 
 define <2 x double> @v_sqrt_v2f64(<2 x double> %x) {
-; SDAG-LABEL: v_sqrt_v2f64:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    s_mov_b32 s4, 0
-; SDAG-NEXT:    s_brev_b32 s5, 8
-; SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[4:5], v[2:3]
-; SDAG-NEXT:    v_mov_b32_e32 v4, 0x100
-; SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[4:5]
-; SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v5
-; SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; SDAG-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
-; SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
-; SDAG-NEXT:    v_mul_f64 v[8:9], v[0:1], v[4:5]
-; SDAG-NEXT:    v_mul_f64 v[4:5], v[4:5], 0.5
-; SDAG-NEXT:    v_mul_f64 v[10:11], v[2:3], v[6:7]
-; SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
-; SDAG-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 0.5
-; SDAG-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 0.5
-; SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
-; SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
-; SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[8:9], v[12:13], v[4:5], v[8:9]
-; SDAG-NEXT:    v_fma_f64 v[10:11], v[14:15], v[6:7], v[10:11]
-; SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[8:9]
-; SDAG-NEXT:    v_mov_b32_e32 v8, 0xffffff80
-; SDAG-NEXT:    v_fma_f64 v[6:7], v[14:15], v[6:7], v[10:11]
-; SDAG-NEXT:    v_mov_b32_e32 v9, 0x260
-; SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
-; SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
-; SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
-; SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: v_sqrt_v2f64:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    s_mov_b32 s4, 0
-; GISEL-NEXT:    s_brev_b32 s5, 8
-; GISEL-NEXT:    v_mov_b32_e32 v4, s4
-; GISEL-NEXT:    v_mov_b32_e32 v5, s5
-; GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
-; GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
-; GISEL-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
-; GISEL-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
-; GISEL-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
-; GISEL-NEXT:    v_mul_f64 v[8:9], v[4:5], 0.5
-; GISEL-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
-; GISEL-NEXT:    v_mul_f64 v[10:11], v[6:7], 0.5
-; GISEL-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
-; GISEL-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[4:5], 0.5
-; GISEL-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[6:7], 0.5
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
-; GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
-; GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
-; GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
-; GISEL-NEXT:    v_mov_b32_e32 v8, 0xffffff80
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
-; GISEL-NEXT:    v_mov_b32_e32 v9, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
-; GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
-; GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
-; GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_v2f64:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[4:5], v[2:3]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v4, 0x100
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[4:5]
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v5
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[8:9], v[0:1], v[4:5]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[4:5], v[4:5], 0.5
+; GFX6-SDAG-NEXT:    v_mul_f64 v[10:11], v[2:3], v[6:7]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[8:9], v[12:13], v[4:5], v[8:9]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], v[14:15], v[6:7], v[10:11]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[8:9]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], v[14:15], v[6:7], v[10:11]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_v2f64:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s4, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s5, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[4:5], v[2:3]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v4, 0x100
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v5, 0, v4, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v4, 0, v4, s[4:5]
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v5
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[8:9], v[0:1], v[4:5]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[4:5], v[4:5], 0.5
+; GFX8-SDAG-NEXT:    v_mul_f64 v[10:11], v[2:3], v[6:7]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[8:9], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[10:11], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[8:9], v[12:13], v[4:5], v[8:9]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], v[14:15], v[6:7], v[10:11]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[8:9], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[10:11], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[4:5], v[12:13], v[4:5], v[8:9]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], v[14:15], v[6:7], v[10:11]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_v2f64:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX6-GISEL-NEXT:    s_brev_b32 s5, 8
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX6-GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[4:5]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[8:9], v[4:5], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[10:11], v[6:7], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[4:5], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[6:7], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_v2f64:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX8-GISEL-NEXT:    s_brev_b32 s5, 8
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[4:5]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[4:5]
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v4, 8, v4
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v4
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[8:9], v[4:5], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[4:5], v[0:1], v[4:5]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[10:11], v[6:7], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[6:7], v[2:3], v[6:7]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], -v[8:9], v[4:5], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], -v[10:11], v[6:7], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[12:13], v[8:9]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[6:7]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[14:15], v[10:11]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], -v[4:5], v[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], -v[6:7], v[6:7], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[4:5], v[12:13], v[8:9], v[4:5]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v8, 0xffffff80
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[14:15], v[10:11], v[6:7]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v9, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v10, 0, v8, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, v8, s[4:5]
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v9
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v9
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v8
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, v6, v2, s[4:5]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v3, v7, v3, s[4:5]
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call <2 x double> @llvm.sqrt.v2f64(<2 x double> %x)
   ret <2 x double> %result
 }
 
 define <3 x double> @v_sqrt_v3f64(<3 x double> %x) {
-; SDAG-LABEL: v_sqrt_v3f64:
-; SDAG:       ; %bb.0:
-; SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT:    s_mov_b32 s6, 0
-; SDAG-NEXT:    s_brev_b32 s7, 8
-; SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[6:7], v[0:1]
-; SDAG-NEXT:    v_mov_b32_e32 v10, 0x100
-; SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[6:7], v[2:3]
-; SDAG-NEXT:    v_cmp_gt_f64_e64 s[6:7], s[6:7], v[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v10, vcc
-; SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
-; SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, v10, s[4:5]
-; SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v6
-; SDAG-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s[6:7]
-; SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
-; SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[0:1]
-; SDAG-NEXT:    v_rsq_f64_e32 v[8:9], v[2:3]
-; SDAG-NEXT:    v_rsq_f64_e32 v[12:13], v[4:5]
-; SDAG-NEXT:    v_mul_f64 v[10:11], v[0:1], v[6:7]
-; SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
-; SDAG-NEXT:    v_mul_f64 v[14:15], v[2:3], v[8:9]
-; SDAG-NEXT:    v_mul_f64 v[8:9], v[8:9], 0.5
-; SDAG-NEXT:    v_fma_f64 v[16:17], -v[6:7], v[10:11], 0.5
-; SDAG-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[14:15], 0.5
-; SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[16:17], v[10:11]
-; SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[16:17], v[6:7]
-; SDAG-NEXT:    v_mul_f64 v[16:17], v[4:5], v[12:13]
-; SDAG-NEXT:    v_mul_f64 v[12:13], v[12:13], 0.5
-; SDAG-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
-; SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[18:19], v[8:9]
-; SDAG-NEXT:    v_fma_f64 v[18:19], -v[12:13], v[16:17], 0.5
-; SDAG-NEXT:    v_fma_f64 v[16:17], v[16:17], v[18:19], v[16:17]
-; SDAG-NEXT:    v_fma_f64 v[12:13], v[12:13], v[18:19], v[12:13]
-; SDAG-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[10:11], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[10:11], v[18:19], v[6:7], v[10:11]
-; SDAG-NEXT:    v_fma_f64 v[18:19], -v[14:15], v[14:15], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[14:15], v[18:19], v[8:9], v[14:15]
-; SDAG-NEXT:    v_fma_f64 v[18:19], -v[16:17], v[16:17], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[16:17], v[18:19], v[12:13], v[16:17]
-; SDAG-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[10:11], v[0:1]
-; SDAG-NEXT:    v_fma_f64 v[6:7], v[18:19], v[6:7], v[10:11]
-; SDAG-NEXT:    v_fma_f64 v[10:11], -v[14:15], v[14:15], v[2:3]
-; SDAG-NEXT:    v_fma_f64 v[18:19], -v[16:17], v[16:17], v[4:5]
-; SDAG-NEXT:    v_fma_f64 v[8:9], v[10:11], v[8:9], v[14:15]
-; SDAG-NEXT:    v_fma_f64 v[10:11], v[18:19], v[12:13], v[16:17]
-; SDAG-NEXT:    v_mov_b32_e32 v14, 0xffffff80
-; SDAG-NEXT:    v_mov_b32_e32 v15, 0x260
-; SDAG-NEXT:    v_cndmask_b32_e32 v12, 0, v14, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v13, 0, v14, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v14, 0, v14, s[6:7]
-; SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v12
-; SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v15
-; SDAG-NEXT:    v_ldexp_f64 v[8:9], v[8:9], v13
-; SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v15
-; SDAG-NEXT:    v_ldexp_f64 v[10:11], v[10:11], v14
-; SDAG-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], v15
-; SDAG-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; SDAG-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; SDAG-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
-; SDAG-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
-; SDAG-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
-; SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GISEL-LABEL: v_sqrt_v3f64:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    s_mov_b32 s4, 0
-; GISEL-NEXT:    s_brev_b32 s5, 8
-; GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
-; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
-; GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
-; GISEL-NEXT:    v_mov_b32_e32 v6, s4
-; GISEL-NEXT:    v_mov_b32_e32 v7, s5
-; GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7]
-; GISEL-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
-; GISEL-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
-; GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v8
-; GISEL-NEXT:    v_rsq_f64_e32 v[8:9], v[0:1]
-; GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[6:7]
-; GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
-; GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v6
-; GISEL-NEXT:    v_rsq_f64_e32 v[10:11], v[2:3]
-; GISEL-NEXT:    v_rsq_f64_e32 v[12:13], v[4:5]
-; GISEL-NEXT:    v_mul_f64 v[6:7], v[8:9], 0.5
-; GISEL-NEXT:    v_mul_f64 v[8:9], v[0:1], v[8:9]
-; GISEL-NEXT:    v_mul_f64 v[14:15], v[10:11], 0.5
-; GISEL-NEXT:    v_mul_f64 v[10:11], v[2:3], v[10:11]
-; GISEL-NEXT:    v_fma_f64 v[16:17], -v[6:7], v[8:9], 0.5
-; GISEL-NEXT:    v_fma_f64 v[18:19], -v[14:15], v[10:11], 0.5
-; GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[16:17], v[8:9]
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[16:17], v[6:7]
-; GISEL-NEXT:    v_mul_f64 v[16:17], v[12:13], 0.5
-; GISEL-NEXT:    v_mul_f64 v[12:13], v[4:5], v[12:13]
-; GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[18:19], v[10:11]
-; GISEL-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
-; GISEL-NEXT:    v_fma_f64 v[18:19], -v[16:17], v[12:13], 0.5
-; GISEL-NEXT:    v_fma_f64 v[12:13], v[12:13], v[18:19], v[12:13]
-; GISEL-NEXT:    v_fma_f64 v[16:17], v[16:17], v[18:19], v[16:17]
-; GISEL-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[8:9], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[8:9], v[18:19], v[6:7], v[8:9]
-; GISEL-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[10:11], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[10:11], v[18:19], v[14:15], v[10:11]
-; GISEL-NEXT:    v_fma_f64 v[18:19], -v[12:13], v[12:13], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[12:13], v[18:19], v[16:17], v[12:13]
-; GISEL-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[8:9], v[0:1]
-; GISEL-NEXT:    v_fma_f64 v[6:7], v[18:19], v[6:7], v[8:9]
-; GISEL-NEXT:    v_fma_f64 v[8:9], -v[10:11], v[10:11], v[2:3]
-; GISEL-NEXT:    v_fma_f64 v[18:19], -v[12:13], v[12:13], v[4:5]
-; GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[14:15], v[10:11]
-; GISEL-NEXT:    v_fma_f64 v[10:11], v[18:19], v[16:17], v[12:13]
-; GISEL-NEXT:    v_mov_b32_e32 v14, 0xffffff80
-; GISEL-NEXT:    v_mov_b32_e32 v15, 0x260
-; GISEL-NEXT:    v_cndmask_b32_e32 v12, 0, v14, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, v14, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, v14, s[6:7]
-; GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v12
-; GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v15
-; GISEL-NEXT:    v_ldexp_f64 v[8:9], v[8:9], v13
-; GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v15
-; GISEL-NEXT:    v_ldexp_f64 v[10:11], v[10:11], v14
-; GISEL-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], v15
-; GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
-; GISEL-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
-; GISEL-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
-; GISEL-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
-; GISEL-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
-; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: v_sqrt_v3f64:
+; GFX6-SDAG:       ; %bb.0:
+; GFX6-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT:    s_mov_b32 s6, 0
+; GFX6-SDAG-NEXT:    s_brev_b32 s7, 8
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[6:7], v[0:1]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v10, 0x100
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[6:7], v[2:3]
+; GFX6-SDAG-NEXT:    v_cmp_gt_f64_e64 s[6:7], s[6:7], v[4:5]
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v10, vcc
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, v10, s[4:5]
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v6
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s[6:7]
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[0:1]
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[8:9], v[2:3]
+; GFX6-SDAG-NEXT:    v_rsq_f64_e32 v[12:13], v[4:5]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[10:11], v[0:1], v[6:7]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
+; GFX6-SDAG-NEXT:    v_mul_f64 v[14:15], v[2:3], v[8:9]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[8:9], v[8:9], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[16:17], -v[6:7], v[10:11], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[14:15], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[16:17], v[10:11]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[16:17], v[6:7]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[16:17], v[4:5], v[12:13]
+; GFX6-SDAG-NEXT:    v_mul_f64 v[12:13], v[12:13], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[18:19], v[8:9]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[18:19], -v[12:13], v[16:17], 0.5
+; GFX6-SDAG-NEXT:    v_fma_f64 v[16:17], v[16:17], v[18:19], v[16:17]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[12:13], v[12:13], v[18:19], v[12:13]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[10:11], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], v[18:19], v[6:7], v[10:11]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[18:19], -v[14:15], v[14:15], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[14:15], v[18:19], v[8:9], v[14:15]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[18:19], -v[16:17], v[16:17], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[16:17], v[18:19], v[12:13], v[16:17]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[10:11], v[0:1]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[6:7], v[18:19], v[6:7], v[10:11]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], -v[14:15], v[14:15], v[2:3]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[18:19], -v[16:17], v[16:17], v[4:5]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[8:9], v[10:11], v[8:9], v[14:15]
+; GFX6-SDAG-NEXT:    v_fma_f64 v[10:11], v[18:19], v[12:13], v[16:17]
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v14, 0xffffff80
+; GFX6-SDAG-NEXT:    v_mov_b32_e32 v15, 0x260
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v12, 0, v14, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v13, 0, v14, s[4:5]
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v14, 0, v14, s[6:7]
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v12
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v15
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[8:9], v[8:9], v13
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v15
+; GFX6-SDAG-NEXT:    v_ldexp_f64 v[10:11], v[10:11], v14
+; GFX6-SDAG-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], v15
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
+; GFX6-SDAG-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX6-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: v_sqrt_v3f64:
+; GFX8-SDAG:       ; %bb.0:
+; GFX8-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT:    s_mov_b32 s6, 0
+; GFX8-SDAG-NEXT:    s_brev_b32 s7, 8
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e32 vcc, s[6:7], v[0:1]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v10, 0x100
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e64 s[4:5], s[6:7], v[2:3]
+; GFX8-SDAG-NEXT:    v_cmp_gt_f64_e64 s[6:7], s[6:7], v[4:5]
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v6, 0, v10, vcc
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v6, 0, v10, s[4:5]
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v6
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v10, 0, v10, s[6:7]
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v10
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[6:7], v[0:1]
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[8:9], v[2:3]
+; GFX8-SDAG-NEXT:    v_rsq_f64_e32 v[12:13], v[4:5]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[10:11], v[0:1], v[6:7]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[6:7], v[6:7], 0.5
+; GFX8-SDAG-NEXT:    v_mul_f64 v[14:15], v[2:3], v[8:9]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[8:9], v[8:9], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[16:17], -v[6:7], v[10:11], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[14:15], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], v[10:11], v[16:17], v[10:11]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], v[6:7], v[16:17], v[6:7]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[16:17], v[4:5], v[12:13]
+; GFX8-SDAG-NEXT:    v_mul_f64 v[12:13], v[12:13], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[8:9], v[8:9], v[18:19], v[8:9]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[18:19], -v[12:13], v[16:17], 0.5
+; GFX8-SDAG-NEXT:    v_fma_f64 v[16:17], v[16:17], v[18:19], v[16:17]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[12:13], v[12:13], v[18:19], v[12:13]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[10:11], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], v[18:19], v[6:7], v[10:11]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[18:19], -v[14:15], v[14:15], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[14:15], v[18:19], v[8:9], v[14:15]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[18:19], -v[16:17], v[16:17], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[16:17], v[18:19], v[12:13], v[16:17]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[10:11], v[0:1]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[6:7], v[18:19], v[6:7], v[10:11]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], -v[14:15], v[14:15], v[2:3]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[18:19], -v[16:17], v[16:17], v[4:5]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[8:9], v[10:11], v[8:9], v[14:15]
+; GFX8-SDAG-NEXT:    v_fma_f64 v[10:11], v[18:19], v[12:13], v[16:17]
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v14, 0xffffff80
+; GFX8-SDAG-NEXT:    v_mov_b32_e32 v15, 0x260
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v12, 0, v14, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v13, 0, v14, s[4:5]
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v14, 0, v14, s[6:7]
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v12
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v15
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[8:9], v[8:9], v13
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v15
+; GFX8-SDAG-NEXT:    v_ldexp_f64 v[10:11], v[10:11], v14
+; GFX8-SDAG-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], v15
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
+; GFX8-SDAG-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX8-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX6-GISEL-LABEL: v_sqrt_v3f64:
+; GFX6-GISEL:       ; %bb.0:
+; GFX6-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX6-GISEL-NEXT:    s_brev_b32 s5, 8
+; GFX6-GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v6, s4
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v7, s5
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX6-GISEL-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[6:7]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v8
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[8:9], v[0:1]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[6:7]
+; GFX6-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v6
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[10:11], v[2:3]
+; GFX6-GISEL-NEXT:    v_rsq_f64_e32 v[12:13], v[4:5]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[6:7], v[8:9], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[8:9], v[0:1], v[8:9]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[14:15], v[10:11], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[10:11], v[2:3], v[10:11]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[16:17], -v[6:7], v[8:9], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[18:19], -v[14:15], v[10:11], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[16:17], v[8:9]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[16:17], v[6:7]
+; GFX6-GISEL-NEXT:    v_mul_f64 v[16:17], v[12:13], 0.5
+; GFX6-GISEL-NEXT:    v_mul_f64 v[12:13], v[4:5], v[12:13]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[18:19], v[10:11]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[18:19], -v[16:17], v[12:13], 0.5
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], v[12:13], v[18:19], v[12:13]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[16:17], v[16:17], v[18:19], v[16:17]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[8:9], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[8:9], v[18:19], v[6:7], v[8:9]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[10:11], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[10:11], v[18:19], v[14:15], v[10:11]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[18:19], -v[12:13], v[12:13], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[12:13], v[18:19], v[16:17], v[12:13]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[8:9], v[0:1]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[6:7], v[18:19], v[6:7], v[8:9]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[8:9], -v[10:11], v[10:11], v[2:3]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[18:19], -v[12:13], v[12:13], v[4:5]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[14:15], v[10:11]
+; GFX6-GISEL-NEXT:    v_fma_f64 v[10:11], v[18:19], v[16:17], v[12:13]
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v14, 0xffffff80
+; GFX6-GISEL-NEXT:    v_mov_b32_e32 v15, 0x260
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v12, 0, v14, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, v14, s[4:5]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, v14, s[6:7]
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v12
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v15
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[8:9], v[8:9], v13
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v15
+; GFX6-GISEL-NEXT:    v_ldexp_f64 v[10:11], v[10:11], v14
+; GFX6-GISEL-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], v15
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
+; GFX6-GISEL-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX6-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_sqrt_v3f64:
+; GFX8-GISEL:       ; %bb.0:
+; GFX8-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX8-GISEL-NEXT:    s_brev_b32 s5, 8
+; GFX8-GISEL-NEXT:    v_cmp_gt_f64_e32 vcc, s[4:5], v[0:1]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[0:1], v[0:1], v6
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v6, s4
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v7, s5
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e64 s[4:5], v[2:3], v[6:7]
+; GFX8-GISEL-NEXT:    v_cmp_lt_f64_e64 s[6:7], v[4:5], v[6:7]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, s[4:5]
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v8, 8, v8
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[2:3], v[2:3], v8
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[8:9], v[0:1]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[6:7]
+; GFX8-GISEL-NEXT:    v_lshlrev_b32_e32 v6, 8, v6
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[4:5], v[4:5], v6
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[10:11], v[2:3]
+; GFX8-GISEL-NEXT:    v_rsq_f64_e32 v[12:13], v[4:5]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[6:7], v[8:9], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[8:9], v[0:1], v[8:9]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[14:15], v[10:11], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[10:11], v[2:3], v[10:11]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[16:17], -v[6:7], v[8:9], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[18:19], -v[14:15], v[10:11], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[16:17], v[8:9]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[6:7], v[16:17], v[6:7]
+; GFX8-GISEL-NEXT:    v_mul_f64 v[16:17], v[12:13], 0.5
+; GFX8-GISEL-NEXT:    v_mul_f64 v[12:13], v[4:5], v[12:13]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[10:11], v[10:11], v[18:19], v[10:11]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[14:15], v[14:15], v[18:19], v[14:15]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[18:19], -v[16:17], v[12:13], 0.5
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], v[12:13], v[18:19], v[12:13]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[16:17], v[16:17], v[18:19], v[16:17]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[8:9], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[8:9], v[18:19], v[6:7], v[8:9]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[18:19], -v[10:11], v[10:11], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[10:11], v[18:19], v[14:15], v[10:11]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[18:19], -v[12:13], v[12:13], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[12:13], v[18:19], v[16:17], v[12:13]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[18:19], -v[8:9], v[8:9], v[0:1]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[6:7], v[18:19], v[6:7], v[8:9]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[8:9], -v[10:11], v[10:11], v[2:3]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[18:19], -v[12:13], v[12:13], v[4:5]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[8:9], v[8:9], v[14:15], v[10:11]
+; GFX8-GISEL-NEXT:    v_fma_f64 v[10:11], v[18:19], v[16:17], v[12:13]
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v14, 0xffffff80
+; GFX8-GISEL-NEXT:    v_mov_b32_e32 v15, 0x260
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v12, 0, v14, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v13, 0, v14, s[4:5]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v14, 0, v14, s[6:7]
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[6:7], v[6:7], v12
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e32 vcc, v[0:1], v15
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[8:9], v[8:9], v13
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e64 s[4:5], v[2:3], v15
+; GFX8-GISEL-NEXT:    v_ldexp_f64 v[10:11], v[10:11], v14
+; GFX8-GISEL-NEXT:    v_cmp_class_f64_e64 s[6:7], v[4:5], v15
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v0, v6, v0, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e32 v1, v7, v1, vcc
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v2, v8, v2, s[4:5]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v3, v9, v3, s[4:5]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v4, v10, v4, s[6:7]
+; GFX8-GISEL-NEXT:    v_cndmask_b32_e64 v5, v11, v5, s[6:7]
+; GFX8-GISEL-NEXT:    s_setpc_b64 s[30:31]
   %result = call <3 x double> @llvm.sqrt.v3f64(<3 x double> %x)
   ret <3 x double> %result
 }
@@ -1338,3 +3583,6 @@ attributes #1 = { convergent nounwind willreturn memory(none) }
 attributes #2 = { "approx-func-fp-math"="true" }
 attributes #3 = { "approx-func-fp-math"="true" "no-nans-fp-math"="true" "no-infs-fp-math"="true" }
 attributes #4 = { "unsafe-fp-math"="true" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX6: {{.*}}
+; GFX8: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index 95e28a37f5ee1..3c41cc43a089e 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -1107,21 +1107,19 @@ define void @void_func_v4i8(<4 x i8> %arg0) #0 {
 ; GFX11-TRUE16-LABEL: void_func_v4i8:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, 0
 ; GFX11-TRUE16-NEXT:    s_mov_b64 s[0:1], 0
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.h, v0.h
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[0:3], 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1190,22 +1188,20 @@ define void @void_func_v5i8(<5 x i8> %arg0) #0 {
 ; GFX11-TRUE16-LABEL: void_func_v5i8:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
 ; GFX11-TRUE16-NEXT:    s_mov_b64 s[0:1], 4
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, v2.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v1.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
 ; GFX11-TRUE16-NEXT:    buffer_store_b8 v4, off, s[0:3], 0
 ; GFX11-TRUE16-NEXT:    s_mov_b64 s[0:1], 0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v3, v2
 ; GFX11-TRUE16-NEXT:    buffer_store_b32 v0, off, s[0:3], 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1285,29 +1281,27 @@ define void @void_func_v8i8(<8 x i8> %arg0) #0 {
 ; GFX11-TRUE16-LABEL: void_func_v8i8:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v5.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v5.h, v4.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v6.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v4.h
 ; GFX11-TRUE16-NEXT:    s_mov_b64 s[0:1], 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v0.h, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v4, v6
+; GFX11-TRUE16-NEXT:    v_or_b16 v6.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v6.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v4
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v3
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v0, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v0, v6
 ; GFX11-TRUE16-NEXT:    buffer_store_b64 v[1:2], off, s[0:3], 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1422,47 +1416,44 @@ define void @void_func_v16i8(<16 x i8> %arg0) #0 {
 ; GFX11-TRUE16-LABEL: void_func_v16i8:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v12.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v13.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v13.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v12.l, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v13.l, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.l, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v12.l, v12.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v13.h, v12.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v10.l, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v8.h, v13.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.h, 8, v11.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v8.l, v9.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v12.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, 0
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v12.l, v13.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.h, v14.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v6.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v12, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v10.l, v9.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v10.h, v9.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v14.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v5.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v13
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v9, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v5.h, v4.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v14.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.l, v4.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v7.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v9, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v10.l, v8.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v11
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v8, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v1.h, v0.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
 ; GFX11-TRUE16-NEXT:    s_mov_b64 s[0:1], 0
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v4, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v4, v14
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v14.l
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v0, v12
-; GFX11-TRUE16-NEXT:    buffer_store_b128 v[6:9], off, s[0:3], 0
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v0, v14
+; GFX11-TRUE16-NEXT:    buffer_store_b128 v[5:8], off, s[0:3], 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: void_func_v16i8:
@@ -1658,83 +1649,77 @@ define void @void_func_v32i8(<32 x i8> %arg0) #0 {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    scratch_load_d16_u8 v31, off, s32
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, 0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v15.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v3.h, v2.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v32.l, 0
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v3.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.h, v32.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.h, 8, v11.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v5.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v7.h, v6.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v8.l
 ; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v1.h, v0.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v4.l, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v13
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v3.h, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v32.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v7.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v29.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.h, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v12, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v13, v32
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v7.h, v6.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v12, v32
 ; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v5.h, v4.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v11.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v11.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v9.h, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v13, 0xffff, v14
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v12, v32
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.h, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v10.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v9, v32
 ; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v6.l, v7.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v9.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.h, 8, v21.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v11.h, v11.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v5
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v13, v32
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v8.h, v8.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v10.l, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v10.h, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v6.l, v0.h, v6.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v11.h, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v28.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v11.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v10, v32
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v0.h, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.h, v32.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v5.l, 0xff, v26.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v25.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v7.h, 0xff, v24.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v6.h, v5.h
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v11, v32
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v4.h, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v11, 0xffff, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v14.h, v32.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.h, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v6.h, 0xff, v20.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v14.l, v7.h, v6.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v15.h, v32.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v8.h, 0xff, v16.l
 ; GFX11-TRUE16-NEXT:    s_mov_b64 s[0:1], 16
+; GFX11-TRUE16-NEXT:    v_or_b16 v15.l, v6.h, v5.h
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s3, 0x31016000
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s2, -1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v31.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v9.l, v5.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v8.l, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v7, v32
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v10.h, v10.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v4.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v9, v32
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v4.h, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v10
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v11, v32
-; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v4.l, v8.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v31.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v4.h, v7.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v13, v32
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v5.l, v4.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v5.l, 8, v17.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v14, v32
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v4.h, v8.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v8.h, v5.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v15, v32
+; GFX11-TRUE16-NEXT:    v_or_b16 v32.h, v4.h, v4.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v9, v32
 ; GFX11-TRUE16-NEXT:    buffer_store_b128 v[4:7], off, s[0:3], 0
 ; GFX11-TRUE16-NEXT:    s_mov_b64 s[0:1], 0
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index 2fdc1a8854863..919464a936740 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -4896,23 +4896,22 @@ define amdgpu_gfx void @test_call_external_void_func_v4i8_ret() #0 {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
 ; GFX11-TRUE16-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_readlane_b32 s31, v42, 1
 ; GFX11-TRUE16-NEXT:    v_readlane_b32 s30, v42, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, 0
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v2
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s32, s33
 ; GFX11-TRUE16-NEXT:    v_readlane_b32 s0, v42, 2
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX11-TRUE16-NEXT:    global_store_b32 v[40:41], v0, off
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v41, off, s33
@@ -5156,30 +5155,29 @@ define amdgpu_gfx void @test_call_external_void_func_v5i8_ret() #0 {
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, v5
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v6
 ; GFX11-TRUE16-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v3.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b16 v3.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, v2.l
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v0, 4
-; GFX11-TRUE16-NEXT:    v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_readlane_b32 s30, v42, 0
-; GFX11-TRUE16-NEXT:    s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT:    v_readlane_b32 s0, v42, 2
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v2
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    global_store_b8 v[0:1], v4, off
 ; GFX11-TRUE16-NEXT:    global_store_b32 v[40:41], v2, off
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v41, off, s33
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v40, off, s33 offset:4
+; GFX11-TRUE16-NEXT:    v_readlane_b32 s31, v42, 1
+; GFX11-TRUE16-NEXT:    v_readlane_b32 s30, v42, 0
+; GFX11-TRUE16-NEXT:    s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT:    v_readlane_b32 s0, v42, 2
 ; GFX11-TRUE16-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
 ; GFX11-TRUE16-NEXT:    s_mov_b32 exec_lo, s1
@@ -5441,36 +5439,34 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8_ret() #0 {
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v7, 24, v1
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v8
 ; GFX11-TRUE16-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.l, 0
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v1.h, v0.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v7.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v7.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.h, v0.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v1.h, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v4
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v5.l, v3.h, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, v4.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v3, v5
-; GFX11-TRUE16-NEXT:    v_or_b16 v5.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v4
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v5, v4
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v0, v4
 ; GFX11-TRUE16-NEXT:    v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v0, v5
 ; GFX11-TRUE16-NEXT:    v_readlane_b32 s30, v42, 0
-; GFX11-TRUE16-NEXT:    s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT:    v_readlane_b32 s0, v42, 2
 ; GFX11-TRUE16-NEXT:    global_store_b64 v[40:41], v[1:2], off
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v41, off, s33
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v40, off, s33 offset:4
+; GFX11-TRUE16-NEXT:    s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT:    v_readlane_b32 s0, v42, 2
 ; GFX11-TRUE16-NEXT:    s_or_saveexec_b32 s1, -1
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
 ; GFX11-TRUE16-NEXT:    s_mov_b32 exec_lo, s1
@@ -5910,85 +5906,77 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8_ret() #0 {
 ; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v17, v32 :: v_dual_mov_b32 v18, v33
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v19, v34
 ; GFX11-TRUE16-NEXT:    s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v13.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v12.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v15.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v8.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.l, 0
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v15.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v14.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v13.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v12.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v12.l, 0
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v12.l, v1.h, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v14.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.h, 8, v9.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v9, 0xffff, v12
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v0.h, v2.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b16 v8.l, v3.h, v1.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v5.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v1.h, v0.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v11.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v13.l, v3.h, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v13.h, v12.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v10.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v9, v13
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v8, 0xffff, v8
-; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v3.h, v2.h
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v9.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v8.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v9.h, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v13, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v1.h, v0.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v7.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v9.l, v3.h, v2.h
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v4, v8, v13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v1.h, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v6
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v2.h, 8, v5.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v9, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v1.h, v0.h
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v3.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, 8, v29.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v4.l, v3.h, v2.h
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v12.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v4, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, 8, v31.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v30.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v29.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v28.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v6, v13
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v5, v2, v12
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v4.l, 8, v17.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, 8, v27.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v1.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v26.l
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v25.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v0.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v24.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v31.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v13
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v26.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v6.l, v0.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v27.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, 8, v21.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v4.h, 0xff, v16.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v3, v2, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, 8, v23.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v2.l, v1.h, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v22.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v21.l
 ; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v20.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v9, v6, v13
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v17.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v16.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v23.l
-; GFX11-TRUE16-NEXT:    v_and_b16 v6.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v0.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v8, v7, v13
-; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v18.l
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v6.l, v0.h
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v6, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v19.l
-; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v10.l, v0.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v7, v6, v13
-; GFX11-TRUE16-NEXT:    v_or_b16 v13.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v10
-; GFX11-TRUE16-NEXT:    v_or_b32_e32 v6, v0, v13
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v12
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, 8, v19.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_or_b16 v1.l, v1.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v12.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v1, v1, v12
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_or_b16 v12.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT:    v_or_b16 v0.l, v4.h, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v12.l
+; GFX11-TRUE16-NEXT:    v_or_b32_e32 v0, v0, v12
 ; GFX11-TRUE16-NEXT:    s_clause 0x1
-; GFX11-TRUE16-NEXT:    global_store_b128 v[42:43], v[6:9], off
-; GFX11-TRUE16-NEXT:    global_store_b128 v[40:41], v[2:5], off
+; GFX11-TRUE16-NEXT:    global_store_b128 v[42:43], v[0:3], off
+; GFX11-TRUE16-NEXT:    global_store_b128 v[40:41], v[5:8], off
 ; GFX11-TRUE16-NEXT:    s_clause 0x3
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v43, off, s33
 ; GFX11-TRUE16-NEXT:    scratch_load_b32 v42, off, s33 offset:4
diff --git a/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll b/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll
index d1e82a06077f5..99025f0a983c0 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll
@@ -39,20 +39,19 @@ define void @test_flat_store_no_scratch_alloc(ptr %ptr, i32 %val) #0 {
     ret void
 }
 
-; TODO: handle
 define void @test_flat_store_noalias_addrspace(ptr %ptr, i32 %val) {
 ; GCN-LABEL: test_flat_store_noalias_addrspace:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_SE
+; GCN-NEXT:    flat_store_b32 v[0:1], v2
 ; GCN-NEXT:    s_wait_dscnt 0x0
 ; GCN-NEXT:    s_set_pc_i64 s[30:31]
     store i32 %val, ptr %ptr, !noalias.addrspace !{i32 5, i32 6}
     ret void
 }
 
-; TODO: would be nice to handle too
+; TODO: would be nice to handle
 define void @test_flat_store_select(ptr addrspace(1) %a, ptr addrspace(3) %b, i1 %cond, i32 %val) {
 ; GCN-SDAG-LABEL: test_flat_store_select:
 ; GCN-SDAG:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 1f74fbdc46e98..9c1f9d21b9da3 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -8275,13 +8275,12 @@ define half @global_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8394,13 +8393,12 @@ define half @global_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -8700,13 +8698,12 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8823,13 +8820,12 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -9138,13 +9134,12 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -9262,13 +9257,12 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -9576,11 +9570,11 @@ define void @global_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -9690,11 +9684,11 @@ define void @global_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -9985,11 +9979,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10103,11 +10097,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -10406,11 +10400,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10525,11 +10519,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -10819,10 +10813,9 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10908,10 +10901,9 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -11144,8 +11136,8 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -11228,8 +11220,8 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -11464,13 +11456,12 @@ define half @global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -11589,13 +11580,12 @@ define half @global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -11906,11 +11896,11 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -12026,11 +12016,11 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index faa74fef2be2f..f7cc0709109f9 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -4467,14 +4467,14 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4592,14 +4592,14 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -4912,14 +4912,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5044,14 +5044,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -5373,14 +5373,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5506,14 +5506,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -5832,13 +5832,12 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v3.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v2.h, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5954,13 +5953,12 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v3.l, v3.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v2.h, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -6265,13 +6263,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6394,13 +6391,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -6713,13 +6709,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6843,13 +6838,12 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -7151,11 +7145,11 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v4.l, v4.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7246,11 +7240,11 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v4.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -7494,10 +7488,9 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7586,10 +7579,9 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -7838,14 +7830,14 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -7972,14 +7964,14 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -8303,13 +8295,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -8434,13 +8425,12 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index a46b0129b79e6..b81af1fc9233d 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -4467,14 +4467,14 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4592,14 +4592,14 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -4912,14 +4912,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5044,14 +5044,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -5373,14 +5373,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5506,14 +5506,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -5832,13 +5832,12 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v3.l, v3.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v3.l, v2.h, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5954,13 +5953,12 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v3.l, v3.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v3.l, v2.h, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -6265,13 +6263,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6394,13 +6391,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -6713,13 +6709,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6843,13 +6838,12 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -7151,11 +7145,11 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v4.l, v4.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7246,11 +7240,11 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v4.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -7494,10 +7488,9 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7586,10 +7579,9 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -7838,14 +7830,14 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -7972,14 +7964,14 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -8303,13 +8295,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v0.h, v5.l, v5.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v5.l, v0.h, v0.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -8434,13 +8425,12 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v0.h, v5.l, v5.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v5.l, v0.h, v0.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v2, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index 053efdcb76261..b8762d13e1327 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -5221,13 +5221,12 @@ define half @global_agent_atomic_fsub_ret_f16(ptr addrspace(1) %ptr, half %val)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5340,13 +5339,12 @@ define half @global_agent_atomic_fsub_ret_f16(ptr addrspace(1) %ptr, half %val)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -5646,13 +5644,12 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %p
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5769,13 +5766,12 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %p
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -6084,13 +6080,12 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_neg(ptr addrspace(1) %p
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6208,13 +6203,12 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_neg(ptr addrspace(1) %p
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -6522,11 +6516,11 @@ define void @global_agent_atomic_fsub_noret_f16(ptr addrspace(1) %ptr, half %val
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6636,11 +6630,11 @@ define void @global_agent_atomic_fsub_noret_f16(ptr addrspace(1) %ptr, half %val
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -6931,11 +6925,11 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7049,11 +7043,11 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -7352,11 +7346,11 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_neg(ptr addrspace(1)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7471,11 +7465,11 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_neg(ptr addrspace(1)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -7765,10 +7759,9 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr addrspa
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7854,10 +7847,9 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr addrspa
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -8090,8 +8082,8 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr addrs
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8174,8 +8166,8 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr addrs
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, 0xffff0000, v4, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -8410,13 +8402,12 @@ define half @global_system_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -8535,13 +8526,12 @@ define half @global_system_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v5.h, 0
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v5.l, v5.l, v2.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v5, 0xffff, v5
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v5, v6, v4, v5
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -8852,11 +8842,11 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX12-TRUE16-NEXT:    global_wb scope:SCOPE_SYS
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
@@ -8972,11 +8962,11 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_sub_f16_e32 v3.l, v3.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v6, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
diff --git a/llvm/test/CodeGen/AMDGPU/global-extload-gfx11plus.ll b/llvm/test/CodeGen/AMDGPU/global-extload-gfx11plus.ll
new file mode 100644
index 0000000000000..f92ba7a8978b9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/global-extload-gfx11plus.ll
@@ -0,0 +1,207 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-REAL16 %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+
+define amdgpu_kernel void @zextload_global_i8_to_i16(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; GFX11-REAL16-LABEL: zextload_global_i8_to_i16:
+; GFX11-REAL16:       ; %bb.0:
+; GFX11-REAL16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-REAL16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-REAL16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-REAL16-NEXT:    global_load_d16_u8 v0, v1, s[2:3]
+; GFX11-REAL16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-REAL16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-REAL16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: zextload_global_i8_to_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u8 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %a = load i8, ptr addrspace(1) %in
+  %ext = zext i8 %a to i16
+  store i16 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @sextload_global_i8_to_i16(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; GFX11-REAL16-LABEL: sextload_global_i8_to_i16:
+; GFX11-REAL16:       ; %bb.0:
+; GFX11-REAL16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-REAL16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-REAL16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-REAL16-NEXT:    global_load_d16_i8 v0, v1, s[2:3]
+; GFX11-REAL16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-REAL16-NEXT:    global_store_b16 v1, v0, s[0:1]
+; GFX11-REAL16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: sextload_global_i8_to_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_i8 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %a = load i8, ptr addrspace(1) %in
+  %ext = sext i8 %a to i16
+  store i16 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @zextload_global_i8_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; GFX11-REAL16-LABEL: zextload_global_i8_to_i64:
+; GFX11-REAL16:       ; %bb.0:
+; GFX11-REAL16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-REAL16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-REAL16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-REAL16-NEXT:    global_load_d16_u8 v0, v1, s[2:3]
+; GFX11-REAL16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-REAL16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-REAL16-NEXT:    global_store_b64 v1, v[0:1], s[0:1]
+; GFX11-REAL16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: zextload_global_i8_to_i64:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u8 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    global_store_b64 v1, v[0:1], s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %a = load i8, ptr addrspace(1) %in
+  %ext = zext i8 %a to i64
+  store i64 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @sextload_global_i8_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; GFX11-REAL16-LABEL: sextload_global_i8_to_i64:
+; GFX11-REAL16:       ; %bb.0:
+; GFX11-REAL16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-REAL16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-REAL16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-REAL16-NEXT:    global_load_d16_i8 v0, v2, s[2:3]
+; GFX11-REAL16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-REAL16-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX11-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-REAL16-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-REAL16-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-REAL16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: sextload_global_i8_to_i64:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_i8 v0, v2, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-FAKE16-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %a = load i8, ptr addrspace(1) %in
+  %ext = sext i8 %a to i64
+  store i64 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @zextload_global_i16_to_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; GFX11-LABEL: zextload_global_i16_to_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %a = load i16, ptr addrspace(1) %in
+  %ext = zext i16 %a to i32
+  store i32 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @sextload_global_i16_to_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; GFX11-LABEL: sextload_global_i16_to_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_i16 v1, v0, s[2:3]
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-NEXT:    s_endpgm
+  %a = load i16, ptr addrspace(1) %in
+  %ext = sext i16 %a to i32
+  store i32 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @zextload_global_i16_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; GFX11-REAL16-LABEL: zextload_global_i16_to_i64:
+; GFX11-REAL16:       ; %bb.0:
+; GFX11-REAL16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-REAL16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-REAL16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-REAL16-NEXT:    global_load_d16_b16 v0, v1, s[2:3]
+; GFX11-REAL16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-REAL16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-REAL16-NEXT:    global_store_b64 v1, v[0:1], s[0:1]
+; GFX11-REAL16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: zextload_global_i16_to_i64:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v0, v1, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    global_store_b64 v1, v[0:1], s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %a = load i16, ptr addrspace(1) %in
+  %ext = zext i16 %a to i64
+  store i64 %ext, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_kernel void @sextload_global_i16_to_i64(ptr addrspace(1) %out, ptr addrspace(1) %in) nounwind {
+; GFX11-REAL16-LABEL: sextload_global_i16_to_i64:
+; GFX11-REAL16:       ; %bb.0:
+; GFX11-REAL16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-REAL16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-REAL16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-REAL16-NEXT:    global_load_d16_b16 v0, v2, s[2:3]
+; GFX11-REAL16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-REAL16-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX11-REAL16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-REAL16-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-REAL16-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-REAL16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: sextload_global_i16_to_i64:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_u16 v0, v2, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-FAKE16-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %a = load i16, ptr addrspace(1) %in
+  %ext = sext i16 %a to i64
+  store i64 %ext, ptr addrspace(1) %out
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
index 3a898a9214461..e532deaca98a8 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GCN-SDAG %s
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GCN-GISEL %s
 
-; Test S_WAIT_XCNT insertion for global_load/store instructions.
+; Test S_WAIT_XCNT insertion for global_load/store clauses.
 ; Introduced additional operations in between the clauses to have the register dependency
 ; between the operands of VMEM operations and the def ops of VALU instructions that followed.
 
@@ -123,29 +123,10 @@ define i32 @test_v64i32_load_store(ptr addrspace(1) %ptr, i32 %idx, ptr addrspac
 ; GCN-SDAG:       ; %bb.0:
 ; GCN-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GCN-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GCN-SDAG-NEXT:    s_clause 0xd
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v40, s32 offset:52 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v41, s32 offset:48 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v42, s32 offset:44 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v43, s32 offset:40 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v44, s32 offset:36 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v45, s32 offset:32 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v56, s32 offset:28 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v57, s32 offset:24 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v58, s32 offset:20 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v59, s32 offset:16 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v60, s32 offset:12 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v61, s32 offset:8 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v62, s32 offset:4 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v63, s32 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:224
-; GCN-SDAG-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
-; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GCN-SDAG-NEXT:    scratch_store_b128 off, v[6:9], s32 offset:56 scope:SCOPE_SE ; 16-byte Folded Spill
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v39, v4 :: v_dual_mov_b32 v38, v3
+; GCN-SDAG-NEXT:    s_clause 0xf
+; GCN-SDAG-NEXT:    global_load_b128 v[2:5], v[0:1], off offset:224
 ; GCN-SDAG-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:240
-; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GCN-SDAG-NEXT:    scratch_store_b128 off, v[6:9], s32 offset:72 scope:SCOPE_SE ; 16-byte Folded Spill
-; GCN-SDAG-NEXT:    s_clause 0xd
 ; GCN-SDAG-NEXT:    global_load_b128 v[10:13], v[0:1], off offset:192
 ; GCN-SDAG-NEXT:    global_load_b128 v[14:17], v[0:1], off offset:208
 ; GCN-SDAG-NEXT:    global_load_b128 v[18:21], v[0:1], off offset:160
@@ -155,137 +136,103 @@ define i32 @test_v64i32_load_store(ptr addrspace(1) %ptr, i32 %idx, ptr addrspac
 ; GCN-SDAG-NEXT:    global_load_b128 v[34:37], v[0:1], off offset:96
 ; GCN-SDAG-NEXT:    global_load_b128 v[48:51], v[0:1], off offset:112
 ; GCN-SDAG-NEXT:    global_load_b128 v[52:55], v[0:1], off offset:64
-; GCN-SDAG-NEXT:    global_load_b128 v[38:41], v[0:1], off offset:80
-; GCN-SDAG-NEXT:    global_load_b128 v[42:45], v[0:1], off offset:32
-; GCN-SDAG-NEXT:    global_load_b128 v[56:59], v[0:1], off offset:48
-; GCN-SDAG-NEXT:    global_load_b128 v[60:63], v[0:1], off
-; GCN-SDAG-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:16
-; GCN-SDAG-NEXT:    scratch_load_b128 v[6:9], off, s32 offset:56 th:TH_LOAD_LU ; 16-byte Folded Reload
-; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[6:9], off offset:224
-; GCN-SDAG-NEXT:    scratch_load_b128 v[6:9], off, s32 offset:72 th:TH_LOAD_LU ; 16-byte Folded Reload
-; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GCN-SDAG-NEXT:    s_clause 0xe
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[6:9], off offset:240
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[10:13], off offset:192
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[14:17], off offset:208
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[18:21], off offset:160
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[22:25], off offset:176
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[26:29], off offset:128
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[30:33], off offset:144
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[34:37], off offset:96
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[48:51], off offset:112
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[52:55], off offset:64
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[38:41], off offset:80
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[42:45], off offset:32
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[56:59], off offset:48
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[60:63], off
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[0:3], off offset:16
-; GCN-SDAG-NEXT:    s_clause 0xd
-; GCN-SDAG-NEXT:    scratch_load_b32 v63, off, s32
-; GCN-SDAG-NEXT:    scratch_load_b32 v62, off, s32 offset:4
-; GCN-SDAG-NEXT:    scratch_load_b32 v61, off, s32 offset:8
-; GCN-SDAG-NEXT:    scratch_load_b32 v60, off, s32 offset:12
-; GCN-SDAG-NEXT:    scratch_load_b32 v59, off, s32 offset:16
-; GCN-SDAG-NEXT:    scratch_load_b32 v58, off, s32 offset:20
-; GCN-SDAG-NEXT:    scratch_load_b32 v57, off, s32 offset:24
-; GCN-SDAG-NEXT:    scratch_load_b32 v56, off, s32 offset:28
-; GCN-SDAG-NEXT:    scratch_load_b32 v45, off, s32 offset:32
-; GCN-SDAG-NEXT:    scratch_load_b32 v44, off, s32 offset:36
-; GCN-SDAG-NEXT:    scratch_load_b32 v43, off, s32 offset:40
-; GCN-SDAG-NEXT:    scratch_load_b32 v42, off, s32 offset:44
-; GCN-SDAG-NEXT:    scratch_load_b32 v41, off, s32 offset:48
-; GCN-SDAG-NEXT:    scratch_load_b32 v40, off, s32 offset:52
-; GCN-SDAG-NEXT:    s_wait_xcnt 0xe
-; GCN-SDAG-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-SDAG-NEXT:    global_load_b128 v[64:67], v[0:1], off offset:80
+; GCN-SDAG-NEXT:    global_load_b128 v[68:71], v[0:1], off offset:32
+; GCN-SDAG-NEXT:    global_load_b128 v[80:83], v[0:1], off offset:48
+; GCN-SDAG-NEXT:    global_load_b128 v[84:87], v[0:1], off
+; GCN-SDAG-NEXT:    global_load_b128 v[96:99], v[0:1], off offset:16
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0xf
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[2:5], off offset:224
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0xe
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[6:9], off offset:240
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0xd
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[10:13], off offset:192
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0xc
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[14:17], off offset:208
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0xb
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[18:21], off offset:160
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0xa
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[22:25], off offset:176
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x9
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[26:29], off offset:128
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x8
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[30:33], off offset:144
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x7
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[34:37], off offset:96
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x6
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[48:51], off offset:112
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x5
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[52:55], off offset:64
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x4
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[64:67], off offset:80
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x3
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[68:71], off offset:32
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x2
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[80:83], off offset:48
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x1
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[84:87], off
 ; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[96:99], off offset:16
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x10
+; GCN-SDAG-NEXT:    v_mov_b32_e32 v0, v98
 ; GCN-SDAG-NEXT:    s_set_pc_i64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: test_v64i32_load_store:
 ; GCN-GISEL:       ; %bb.0:
 ; GCN-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GCN-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GCN-GISEL-NEXT:    v_dual_mov_b32 v38, v3 :: v_dual_mov_b32 v39, v4
 ; GCN-GISEL-NEXT:    s_clause 0xf
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v40, s32 offset:60 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v41, s32 offset:56 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v42, s32 offset:52 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v43, s32 offset:48 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v44, s32 offset:44 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v45, s32 offset:40 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v46, s32 offset:36 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v47, s32 offset:32 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v56, s32 offset:28 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v57, s32 offset:24 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v58, s32 offset:20 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v59, s32 offset:16 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v60, s32 offset:12 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v61, s32 offset:8 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v62, s32 offset:4 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v63, s32 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    s_wait_xcnt 0x8
-; GCN-GISEL-NEXT:    v_dual_mov_b32 v46, v3 :: v_dual_mov_b32 v47, v4
 ; GCN-GISEL-NEXT:    global_load_b128 v[2:5], v[0:1], off offset:32
-; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GCN-GISEL-NEXT:    scratch_store_b128 off, v[2:5], s32 offset:80 scope:SCOPE_SE ; 16-byte Folded Spill
-; GCN-GISEL-NEXT:    s_clause 0xe
 ; GCN-GISEL-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:48
-; GCN-GISEL-NEXT:    global_load_b128 v[10:13], v[0:1], off offset:64
-; GCN-GISEL-NEXT:    global_load_b128 v[14:17], v[0:1], off offset:80
-; GCN-GISEL-NEXT:    global_load_b128 v[18:21], v[0:1], off offset:96
-; GCN-GISEL-NEXT:    global_load_b128 v[22:25], v[0:1], off offset:112
-; GCN-GISEL-NEXT:    global_load_b128 v[26:29], v[0:1], off offset:128
-; GCN-GISEL-NEXT:    global_load_b128 v[30:33], v[0:1], off offset:144
-; GCN-GISEL-NEXT:    global_load_b128 v[34:37], v[0:1], off offset:160
-; GCN-GISEL-NEXT:    global_load_b128 v[48:51], v[0:1], off offset:176
-; GCN-GISEL-NEXT:    global_load_b128 v[52:55], v[0:1], off offset:192
-; GCN-GISEL-NEXT:    global_load_b128 v[38:41], v[0:1], off offset:208
-; GCN-GISEL-NEXT:    global_load_b128 v[42:45], v[0:1], off offset:224
-; GCN-GISEL-NEXT:    global_load_b128 v[56:59], v[0:1], off
-; GCN-GISEL-NEXT:    global_load_b128 v[60:63], v[0:1], off offset:16
-; GCN-GISEL-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:240
-; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GCN-GISEL-NEXT:    scratch_store_b128 off, v[0:3], s32 offset:64 scope:SCOPE_SE ; 16-byte Folded Spill
-; GCN-GISEL-NEXT:    scratch_load_b128 v[0:3], off, s32 offset:80 th:TH_LOAD_LU ; 16-byte Folded Reload
-; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GCN-GISEL-NEXT:    s_clause 0xe
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[0:3], off offset:32
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[6:9], off offset:48
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[10:13], off offset:64
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[14:17], off offset:80
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[18:21], off offset:96
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[22:25], off offset:112
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[26:29], off offset:128
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[30:33], off offset:144
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[34:37], off offset:160
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[48:51], off offset:176
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[52:55], off offset:192
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[38:41], off offset:208
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[42:45], off offset:224
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[56:59], off
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[60:63], off offset:16
-; GCN-GISEL-NEXT:    scratch_load_b128 v[0:3], off, s32 offset:64 th:TH_LOAD_LU ; 16-byte Folded Reload
-; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GCN-GISEL-NEXT:    global_store_b128 v[46:47], v[0:3], off offset:240
-; GCN-GISEL-NEXT:    s_wait_xcnt 0x0
-; GCN-GISEL-NEXT:    v_mov_b32_e32 v0, v62
-; GCN-GISEL-NEXT:    s_clause 0xf
-; GCN-GISEL-NEXT:    scratch_load_b32 v63, off, s32
-; GCN-GISEL-NEXT:    scratch_load_b32 v62, off, s32 offset:4
-; GCN-GISEL-NEXT:    scratch_load_b32 v61, off, s32 offset:8
-; GCN-GISEL-NEXT:    scratch_load_b32 v60, off, s32 offset:12
-; GCN-GISEL-NEXT:    scratch_load_b32 v59, off, s32 offset:16
-; GCN-GISEL-NEXT:    scratch_load_b32 v58, off, s32 offset:20
-; GCN-GISEL-NEXT:    scratch_load_b32 v57, off, s32 offset:24
-; GCN-GISEL-NEXT:    scratch_load_b32 v56, off, s32 offset:28
-; GCN-GISEL-NEXT:    scratch_load_b32 v47, off, s32 offset:32
-; GCN-GISEL-NEXT:    scratch_load_b32 v46, off, s32 offset:36
-; GCN-GISEL-NEXT:    scratch_load_b32 v45, off, s32 offset:40
-; GCN-GISEL-NEXT:    scratch_load_b32 v44, off, s32 offset:44
-; GCN-GISEL-NEXT:    scratch_load_b32 v43, off, s32 offset:48
-; GCN-GISEL-NEXT:    scratch_load_b32 v42, off, s32 offset:52
-; GCN-GISEL-NEXT:    scratch_load_b32 v41, off, s32 offset:56
-; GCN-GISEL-NEXT:    scratch_load_b32 v40, off, s32 offset:60
+; GCN-GISEL-NEXT:    global_load_b128 v[10:13], v[0:1], off
+; GCN-GISEL-NEXT:    global_load_b128 v[14:17], v[0:1], off offset:16
+; GCN-GISEL-NEXT:    global_load_b128 v[18:21], v[0:1], off offset:64
+; GCN-GISEL-NEXT:    global_load_b128 v[22:25], v[0:1], off offset:80
+; GCN-GISEL-NEXT:    global_load_b128 v[26:29], v[0:1], off offset:96
+; GCN-GISEL-NEXT:    global_load_b128 v[30:33], v[0:1], off offset:112
+; GCN-GISEL-NEXT:    global_load_b128 v[34:37], v[0:1], off offset:128
+; GCN-GISEL-NEXT:    global_load_b128 v[48:51], v[0:1], off offset:144
+; GCN-GISEL-NEXT:    global_load_b128 v[52:55], v[0:1], off offset:160
+; GCN-GISEL-NEXT:    global_load_b128 v[64:67], v[0:1], off offset:176
+; GCN-GISEL-NEXT:    global_load_b128 v[68:71], v[0:1], off offset:192
+; GCN-GISEL-NEXT:    global_load_b128 v[80:83], v[0:1], off offset:208
+; GCN-GISEL-NEXT:    global_load_b128 v[84:87], v[0:1], off offset:224
+; GCN-GISEL-NEXT:    global_load_b128 v[96:99], v[0:1], off offset:240
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0xf
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[2:5], off offset:32
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0xe
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[6:9], off offset:48
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0xd
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[10:13], off
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0xc
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[14:17], off offset:16
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0xb
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[18:21], off offset:64
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0xa
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[22:25], off offset:80
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x9
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[26:29], off offset:96
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x8
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[30:33], off offset:112
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x7
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[34:37], off offset:128
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x6
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[48:51], off offset:144
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x5
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[52:55], off offset:160
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x4
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[64:67], off offset:176
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x3
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[68:71], off offset:192
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x2
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[80:83], off offset:208
+; GCN-GISEL-NEXT:    s_wait_loadcnt 0x1
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[84:87], off offset:224
 ; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[96:99], off offset:240
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x10
+; GCN-GISEL-NEXT:    v_mov_b32_e32 v0, v16
 ; GCN-GISEL-NEXT:    s_set_pc_i64 s[30:31]
   %vec = load <64 x i32>, ptr addrspace(1) %ptr
   store <64 x i32> %vec, ptr addrspace(1) %out, align 4
@@ -298,99 +245,78 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
 ; GCN-SDAG:       ; %bb.0:
 ; GCN-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GCN-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GCN-SDAG-NEXT:    s_clause 0x3
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v40, s32 offset:12 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v41, s32 offset:8 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v42, s32 offset:4 scope:SCOPE_SE
-; GCN-SDAG-NEXT:    scratch_store_b32 off, v43, s32 scope:SCOPE_SE
 ; GCN-SDAG-NEXT:    s_clause 0x7
-; GCN-SDAG-NEXT:    global_load_b128 v[10:13], v[0:1], off offset:112
-; GCN-SDAG-NEXT:    global_load_b128 v[18:21], v[0:1], off offset:96
-; GCN-SDAG-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:80
-; GCN-SDAG-NEXT:    global_load_b128 v[34:37], v[0:1], off offset:48
-; GCN-SDAG-NEXT:    global_load_b128 v[14:17], v[0:1], off offset:32
+; GCN-SDAG-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:112
+; GCN-SDAG-NEXT:    global_load_b128 v[10:13], v[0:1], off offset:96
+; GCN-SDAG-NEXT:    global_load_b128 v[14:17], v[0:1], off offset:80
+; GCN-SDAG-NEXT:    global_load_b128 v[18:21], v[0:1], off offset:48
+; GCN-SDAG-NEXT:    global_load_b128 v[22:25], v[0:1], off offset:32
 ; GCN-SDAG-NEXT:    global_load_b128 v[26:29], v[0:1], off offset:16
 ; GCN-SDAG-NEXT:    global_load_b128 v[30:33], v[0:1], off
 ; GCN-SDAG-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:64
-; GCN-SDAG-NEXT:    v_mov_b64_e32 v[24:25], 0x70
-; GCN-SDAG-NEXT:    v_mov_b64_e32 v[50:51], 0x60
-; GCN-SDAG-NEXT:    v_mov_b64_e32 v[52:53], 48
-; GCN-SDAG-NEXT:    v_mov_b64_e32 v[54:55], 32
-; GCN-SDAG-NEXT:    v_mov_b64_e32 v[40:41], 16
-; GCN-SDAG-NEXT:    v_mov_b64_e32 v[38:39], 0x50
-; GCN-SDAG-NEXT:    v_mov_b64_e32 v[42:43], 0
-; GCN-SDAG-NEXT:    v_mov_b64_e32 v[48:49], 64
-; GCN-SDAG-NEXT:    v_dual_mov_b32 v22, 0xc8 :: v_dual_mov_b32 v23, 0
+; GCN-SDAG-NEXT:    v_mov_b64_e32 v[36:37], 0x70
+; GCN-SDAG-NEXT:    v_mov_b64_e32 v[48:49], 48
+; GCN-SDAG-NEXT:    v_mov_b64_e32 v[38:39], 0x60
+; GCN-SDAG-NEXT:    v_mov_b64_e32 v[50:51], 32
+; GCN-SDAG-NEXT:    v_mov_b64_e32 v[64:65], 16
+; GCN-SDAG-NEXT:    v_mov_b64_e32 v[66:67], 0
+; GCN-SDAG-NEXT:    v_mov_b64_e32 v[52:53], 0x50
+; GCN-SDAG-NEXT:    v_mov_b64_e32 v[54:55], 64
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v34, 0xc8 :: v_dual_mov_b32 v35, 0
 ; GCN-SDAG-NEXT:    s_wait_loadcnt 0x7
-; GCN-SDAG-NEXT:    global_store_b128 v[24:25], v[10:13], off
+; GCN-SDAG-NEXT:    global_store_b128 v[36:37], v[6:9], off
 ; GCN-SDAG-NEXT:    s_wait_loadcnt 0x6
-; GCN-SDAG-NEXT:    global_store_b128 v[50:51], v[18:21], off
+; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[10:13], off
 ; GCN-SDAG-NEXT:    s_wait_loadcnt 0x5
 ; GCN-SDAG-NEXT:    s_wait_xcnt 0x1
-; GCN-SDAG-NEXT:    v_dual_mov_b32 v24, v8 :: v_dual_mov_b32 v25, v9
+; GCN-SDAG-NEXT:    v_dual_mov_b32 v36, v16 :: v_dual_mov_b32 v37, v17
 ; GCN-SDAG-NEXT:    s_wait_xcnt 0x0
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[20:21], v[20:21], v[20:21]
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[18:19], v[18:19], v[18:19]
 ; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[12:13], v[12:13], v[12:13]
 ; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[10:11], v[10:11], v[10:11]
 ; GCN-SDAG-NEXT:    s_wait_loadcnt 0x4
-; GCN-SDAG-NEXT:    global_store_b128 v[52:53], v[34:37], off
+; GCN-SDAG-NEXT:    global_store_b128 v[48:49], v[18:21], off
 ; GCN-SDAG-NEXT:    s_wait_loadcnt 0x3
-; GCN-SDAG-NEXT:    global_store_b128 v[54:55], v[14:17], off
+; GCN-SDAG-NEXT:    global_store_b128 v[50:51], v[22:25], off
 ; GCN-SDAG-NEXT:    s_wait_loadcnt 0x2
-; GCN-SDAG-NEXT:    global_store_b128 v[40:41], v[26:29], off
+; GCN-SDAG-NEXT:    global_store_b128 v[64:65], v[26:29], off
 ; GCN-SDAG-NEXT:    s_wait_loadcnt 0x1
-; GCN-SDAG-NEXT:    global_store_b128 v[42:43], v[30:33], off
-; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GCN-SDAG-NEXT:    s_wait_xcnt 0x3
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[52:53], v[2:3], v[2:3]
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[50:51], v[0:1], v[0:1]
+; GCN-SDAG-NEXT:    global_store_b128 v[66:67], v[30:33], off
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x0
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[32:33], v[32:33], v[32:33]
 ; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[8:9], v[8:9], v[8:9]
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[6:7], 0xc8, v[6:7]
-; GCN-SDAG-NEXT:    s_wait_xcnt 0x2
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[16:17], 0x64, v[16:17]
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[14:15], v[14:15], v[14:15]
-; GCN-SDAG-NEXT:    s_wait_xcnt 0x1
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[6:7], v[6:7], v[6:7]
+; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[50:51], v[2:3], v[2:3]
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[48:49], v[0:1], v[0:1]
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[16:17], v[16:17], v[16:17]
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[14:15], 0xc8, v[14:15]
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[24:25], 0x64, v[24:25]
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[22:23], v[22:23], v[22:23]
 ; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[28:29], v[28:29], v[28:29]
 ; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[26:27], v[26:27], v[26:27]
-; GCN-SDAG-NEXT:    s_wait_xcnt 0x0
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[32:33], v[32:33], v[32:33]
 ; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[30:31], v[30:31], v[30:31]
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[36:37], v[36:37], v[36:37]
-; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[34:35], v[34:35], v[34:35]
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[20:21], v[20:21], v[20:21]
+; GCN-SDAG-NEXT:    v_add_nc_u64_e32 v[18:19], v[18:19], v[18:19]
 ; GCN-SDAG-NEXT:    s_clause 0x1
-; GCN-SDAG-NEXT:    global_store_b128 v[38:39], v[22:25], off
-; GCN-SDAG-NEXT:    global_store_b128 v[48:49], v[0:3], off
+; GCN-SDAG-NEXT:    global_store_b128 v[52:53], v[34:37], off
+; GCN-SDAG-NEXT:    global_store_b128 v[54:55], v[0:3], off
 ; GCN-SDAG-NEXT:    s_clause 0x7
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[18:21], off offset:96
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[10:13], off offset:112
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[50:53], off offset:64
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[6:9], off offset:80
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[14:17], off offset:32
-; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[34:37], off offset:48
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[10:13], off offset:96
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[6:9], off offset:112
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[48:51], off offset:64
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[14:17], off offset:80
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[22:25], off offset:32
+; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[18:21], off offset:48
 ; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[30:33], off
 ; GCN-SDAG-NEXT:    global_store_b128 v[4:5], v[26:29], off offset:16
-; GCN-SDAG-NEXT:    s_clause 0x3
-; GCN-SDAG-NEXT:    scratch_load_b32 v43, off, s32
-; GCN-SDAG-NEXT:    scratch_load_b32 v42, off, s32 offset:4
-; GCN-SDAG-NEXT:    scratch_load_b32 v41, off, s32 offset:8
-; GCN-SDAG-NEXT:    scratch_load_b32 v40, off, s32 offset:12
-; GCN-SDAG-NEXT:    s_wait_xcnt 0xc
+; GCN-SDAG-NEXT:    s_wait_xcnt 0x8
 ; GCN-SDAG-NEXT:    v_dual_mov_b32 v0, v32 :: v_dual_mov_b32 v1, v33
-; GCN-SDAG-NEXT:    s_wait_loadcnt 0x0
 ; GCN-SDAG-NEXT:    s_set_pc_i64 s[30:31]
 ;
 ; GCN-GISEL-LABEL: test_v16i64_load_store:
 ; GCN-GISEL:       ; %bb.0:
 ; GCN-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GCN-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GCN-GISEL-NEXT:    s_clause 0x5
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v40, s32 offset:20 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v41, s32 offset:16 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v42, s32 offset:12 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v43, s32 offset:8 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v44, s32 offset:4 scope:SCOPE_SE
-; GCN-GISEL-NEXT:    scratch_store_b32 off, v45, s32 scope:SCOPE_SE
 ; GCN-GISEL-NEXT:    s_clause 0x7
 ; GCN-GISEL-NEXT:    global_load_b128 v[6:9], v[0:1], off offset:80
 ; GCN-GISEL-NEXT:    global_load_b128 v[10:13], v[0:1], off
@@ -404,11 +330,11 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
 ; GCN-GISEL-NEXT:    v_mov_b64_e32 v[48:49], 16
 ; GCN-GISEL-NEXT:    v_mov_b64_e32 v[50:51], 32
 ; GCN-GISEL-NEXT:    v_mov_b64_e32 v[52:53], 48
-; GCN-GISEL-NEXT:    v_mov_b64_e32 v[42:43], 0x60
+; GCN-GISEL-NEXT:    v_mov_b64_e32 v[66:67], 0x60
+; GCN-GISEL-NEXT:    v_mov_b64_e32 v[68:69], 0x70
 ; GCN-GISEL-NEXT:    v_mov_b64_e32 v[54:55], 64
-; GCN-GISEL-NEXT:    v_mov_b64_e32 v[44:45], 0x70
 ; GCN-GISEL-NEXT:    v_mov_b64_e32 v[34:35], 0xc8
-; GCN-GISEL-NEXT:    v_mov_b64_e32 v[40:41], 0x50
+; GCN-GISEL-NEXT:    v_mov_b64_e32 v[64:65], 0x50
 ; GCN-GISEL-NEXT:    s_wait_loadcnt 0x6
 ; GCN-GISEL-NEXT:    global_store_b128 v[38:39], v[10:13], off
 ; GCN-GISEL-NEXT:    s_wait_loadcnt 0x5
@@ -418,13 +344,13 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
 ; GCN-GISEL-NEXT:    s_wait_loadcnt 0x3
 ; GCN-GISEL-NEXT:    global_store_b128 v[52:53], v[22:25], off
 ; GCN-GISEL-NEXT:    s_wait_loadcnt 0x2
-; GCN-GISEL-NEXT:    global_store_b128 v[42:43], v[26:29], off
+; GCN-GISEL-NEXT:    global_store_b128 v[66:67], v[26:29], off
 ; GCN-GISEL-NEXT:    s_wait_loadcnt 0x1
-; GCN-GISEL-NEXT:    global_store_b128 v[44:45], v[30:33], off
-; GCN-GISEL-NEXT:    v_mov_b64_e32 v[36:37], v[8:9]
+; GCN-GISEL-NEXT:    global_store_b128 v[68:69], v[30:33], off
 ; GCN-GISEL-NEXT:    s_wait_xcnt 0x5
-; GCN-GISEL-NEXT:    v_add_nc_u64_e32 v[10:11], v[10:11], v[10:11]
 ; GCN-GISEL-NEXT:    v_add_nc_u64_e32 v[12:13], v[12:13], v[12:13]
+; GCN-GISEL-NEXT:    v_mov_b64_e32 v[36:37], v[8:9]
+; GCN-GISEL-NEXT:    v_add_nc_u64_e32 v[10:11], v[10:11], v[10:11]
 ; GCN-GISEL-NEXT:    s_wait_xcnt 0x4
 ; GCN-GISEL-NEXT:    v_add_nc_u64_e32 v[14:15], v[14:15], v[14:15]
 ; GCN-GISEL-NEXT:    v_add_nc_u64_e32 v[16:17], v[16:17], v[16:17]
@@ -447,7 +373,7 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
 ; GCN-GISEL-NEXT:    v_add_nc_u64_e32 v[32:33], v[32:33], v[32:33]
 ; GCN-GISEL-NEXT:    s_clause 0x1
 ; GCN-GISEL-NEXT:    global_store_b128 v[54:55], v[0:3], off
-; GCN-GISEL-NEXT:    global_store_b128 v[40:41], v[34:37], off
+; GCN-GISEL-NEXT:    global_store_b128 v[64:65], v[34:37], off
 ; GCN-GISEL-NEXT:    s_clause 0x7
 ; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[10:13], off
 ; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[14:17], off offset:16
@@ -457,15 +383,8 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
 ; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[6:9], off offset:80
 ; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[26:29], off offset:96
 ; GCN-GISEL-NEXT:    global_store_b128 v[4:5], v[30:33], off offset:112
-; GCN-GISEL-NEXT:    s_clause 0x5
-; GCN-GISEL-NEXT:    scratch_load_b32 v45, off, s32
-; GCN-GISEL-NEXT:    scratch_load_b32 v44, off, s32 offset:4
-; GCN-GISEL-NEXT:    scratch_load_b32 v43, off, s32 offset:8
-; GCN-GISEL-NEXT:    scratch_load_b32 v42, off, s32 offset:12
-; GCN-GISEL-NEXT:    scratch_load_b32 v41, off, s32 offset:16
-; GCN-GISEL-NEXT:    scratch_load_b32 v40, off, s32 offset:20
+; GCN-GISEL-NEXT:    s_wait_xcnt 0x9
 ; GCN-GISEL-NEXT:    v_dual_mov_b32 v0, v12 :: v_dual_mov_b32 v1, v13
-; GCN-GISEL-NEXT:    s_wait_loadcnt 0x0
 ; GCN-GISEL-NEXT:    s_set_pc_i64 s[30:31]
   %a = load <16 x i64>, ptr addrspace(1) %ptr_a, align 4
   %in_a = insertelement <16 x i64> %a, i64 100, i32 5
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll
index 1def479be283a..d297955f109ab 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-atomics.ll
@@ -985,7 +985,7 @@ define amdgpu_ps float @global_sub_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw sub ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw sub ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -1024,7 +1024,7 @@ define amdgpu_ps float @global_sub_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw sub ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw sub ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -1061,7 +1061,7 @@ define amdgpu_ps void @global_sub_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw sub ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst
+  %unused = atomicrmw sub ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1098,7 +1098,7 @@ define amdgpu_ps void @global_sub_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw sub ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst
+  %unused = atomicrmw sub ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1135,7 +1135,7 @@ define amdgpu_ps <2 x float> @global_sub_saddr_i64_rtn(ptr addrspace(1) inreg %s
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw sub ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw sub ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -1174,7 +1174,7 @@ define amdgpu_ps <2 x float> @global_sub_saddr_i64_rtn_neg128(ptr addrspace(1) i
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw sub ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw sub ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -1211,7 +1211,7 @@ define amdgpu_ps void @global_sub_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw sub ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst
+  %unused = atomicrmw sub ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1248,7 +1248,7 @@ define amdgpu_ps void @global_sub_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw sub ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst
+  %unused = atomicrmw sub ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1289,7 +1289,7 @@ define amdgpu_ps float @global_and_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw and ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw and ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -1328,7 +1328,7 @@ define amdgpu_ps float @global_and_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw and ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw and ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -1365,7 +1365,7 @@ define amdgpu_ps void @global_and_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw and ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst
+  %unused = atomicrmw and ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1402,7 +1402,7 @@ define amdgpu_ps void @global_and_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw and ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst
+  %unused = atomicrmw and ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1439,7 +1439,7 @@ define amdgpu_ps <2 x float> @global_and_saddr_i64_rtn(ptr addrspace(1) inreg %s
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw and ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw and ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -1478,7 +1478,7 @@ define amdgpu_ps <2 x float> @global_and_saddr_i64_rtn_neg128(ptr addrspace(1) i
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw and ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw and ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -1515,7 +1515,7 @@ define amdgpu_ps void @global_and_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw and ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst
+  %unused = atomicrmw and ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1552,7 +1552,7 @@ define amdgpu_ps void @global_and_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw and ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst
+  %unused = atomicrmw and ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1593,7 +1593,7 @@ define amdgpu_ps float @global_or_saddr_i32_rtn(ptr addrspace(1) inreg %sbase, i
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw or ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw or ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -1632,7 +1632,7 @@ define amdgpu_ps float @global_or_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %s
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw or ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw or ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -1669,7 +1669,7 @@ define amdgpu_ps void @global_or_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw or ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst
+  %unused = atomicrmw or ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1706,7 +1706,7 @@ define amdgpu_ps void @global_or_saddr_i32_nortn_neg128(ptr addrspace(1) inreg %
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw or ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst
+  %unused = atomicrmw or ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1743,7 +1743,7 @@ define amdgpu_ps <2 x float> @global_or_saddr_i64_rtn(ptr addrspace(1) inreg %sb
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw or ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw or ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -1782,7 +1782,7 @@ define amdgpu_ps <2 x float> @global_or_saddr_i64_rtn_neg128(ptr addrspace(1) in
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw or ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw or ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -1819,7 +1819,7 @@ define amdgpu_ps void @global_or_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw or ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst
+  %unused = atomicrmw or ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1856,7 +1856,7 @@ define amdgpu_ps void @global_or_saddr_i64_nortn_neg128(ptr addrspace(1) inreg %
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw or ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst
+  %unused = atomicrmw or ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -1897,7 +1897,7 @@ define amdgpu_ps float @global_xor_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw xor ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw xor ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -1936,7 +1936,7 @@ define amdgpu_ps float @global_xor_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw xor ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw xor ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -1973,7 +1973,7 @@ define amdgpu_ps void @global_xor_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw xor ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst
+  %unused = atomicrmw xor ptr addrspace(1) %gep0, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2010,7 +2010,7 @@ define amdgpu_ps void @global_xor_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw xor ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst
+  %unused = atomicrmw xor ptr addrspace(1) %gep1, i32 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2047,7 +2047,7 @@ define amdgpu_ps <2 x float> @global_xor_saddr_i64_rtn(ptr addrspace(1) inreg %s
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw xor ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw xor ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -2086,7 +2086,7 @@ define amdgpu_ps <2 x float> @global_xor_saddr_i64_rtn_neg128(ptr addrspace(1) i
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw xor ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst
+  %rtn = atomicrmw xor ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -2123,7 +2123,7 @@ define amdgpu_ps void @global_xor_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw xor ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst
+  %unused = atomicrmw xor ptr addrspace(1) %gep0, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2160,7 +2160,7 @@ define amdgpu_ps void @global_xor_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw xor ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst
+  %unused = atomicrmw xor ptr addrspace(1) %gep1, i64 %data syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2198,7 +2198,7 @@ define amdgpu_ps float @global_max_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw max ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw max ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -2234,7 +2234,7 @@ define amdgpu_ps float @global_max_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw max ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw max ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -2267,7 +2267,7 @@ define amdgpu_ps void @global_max_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw max ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw max ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2300,7 +2300,7 @@ define amdgpu_ps void @global_max_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw max ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw max ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2334,7 +2334,7 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn(ptr addrspace(1) inreg %s
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw max ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw max ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -2370,7 +2370,7 @@ define amdgpu_ps <2 x float> @global_max_saddr_i64_rtn_neg128(ptr addrspace(1) i
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw max ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw max ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -2403,7 +2403,7 @@ define amdgpu_ps void @global_max_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw max ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw max ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2436,7 +2436,7 @@ define amdgpu_ps void @global_max_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw max ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw max ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2474,7 +2474,7 @@ define amdgpu_ps float @global_min_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw min ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw min ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -2510,7 +2510,7 @@ define amdgpu_ps float @global_min_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw min ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw min ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -2543,7 +2543,7 @@ define amdgpu_ps void @global_min_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw min ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw min ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2576,7 +2576,7 @@ define amdgpu_ps void @global_min_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw min ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw min ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2610,7 +2610,7 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn(ptr addrspace(1) inreg %s
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw min ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw min ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -2646,7 +2646,7 @@ define amdgpu_ps <2 x float> @global_min_saddr_i64_rtn_neg128(ptr addrspace(1) i
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw min ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw min ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -2679,7 +2679,7 @@ define amdgpu_ps void @global_min_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw min ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw min ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2712,7 +2712,7 @@ define amdgpu_ps void @global_min_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw min ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw min ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2750,7 +2750,7 @@ define amdgpu_ps float @global_umax_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw umax ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw umax ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -2786,7 +2786,7 @@ define amdgpu_ps float @global_umax_saddr_i32_rtn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw umax ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw umax ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -2819,7 +2819,7 @@ define amdgpu_ps void @global_umax_saddr_i32_nortn(ptr addrspace(1) inreg %sbase
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw umax ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw umax ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2852,7 +2852,7 @@ define amdgpu_ps void @global_umax_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw umax ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw umax ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2886,7 +2886,7 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn(ptr addrspace(1) inreg %
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw umax ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw umax ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -2922,7 +2922,7 @@ define amdgpu_ps <2 x float> @global_umax_saddr_i64_rtn_neg128(ptr addrspace(1)
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw umax ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw umax ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -2955,7 +2955,7 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw umax ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw umax ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -2988,7 +2988,7 @@ define amdgpu_ps void @global_umax_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw umax ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw umax ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3026,7 +3026,7 @@ define amdgpu_ps float @global_umin_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw umin ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw umin ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -3062,7 +3062,7 @@ define amdgpu_ps float @global_umin_saddr_i32_rtn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw umin ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw umin ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -3095,7 +3095,7 @@ define amdgpu_ps void @global_umin_saddr_i32_nortn(ptr addrspace(1) inreg %sbase
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw umin ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw umin ptr addrspace(1) %gep0, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3128,7 +3128,7 @@ define amdgpu_ps void @global_umin_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw umin ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw umin ptr addrspace(1) %gep1, i32 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3162,7 +3162,7 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn(ptr addrspace(1) inreg %
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw umin ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw umin ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -3198,7 +3198,7 @@ define amdgpu_ps <2 x float> @global_umin_saddr_i64_rtn_neg128(ptr addrspace(1)
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw umin ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst
+  %rtn = atomicrmw umin ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -3231,7 +3231,7 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn(ptr addrspace(1) inreg %sbase
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw umin ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw umin ptr addrspace(1) %gep0, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3264,7 +3264,7 @@ define amdgpu_ps void @global_umin_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw umin ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst
+  %unused = atomicrmw umin ptr addrspace(1) %gep1, i64 %data syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3664,7 +3664,7 @@ define amdgpu_ps float @global_inc_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i32 %data syncscope("agent") monotonic
+  %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i32 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -3690,7 +3690,7 @@ define amdgpu_ps float @global_inc_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i32 %data syncscope("agent") monotonic
+  %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i32 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -3712,7 +3712,7 @@ define amdgpu_ps void @global_inc_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i32 %data syncscope("agent") monotonic
+  %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i32 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3734,7 +3734,7 @@ define amdgpu_ps void @global_inc_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i32 %data syncscope("agent") monotonic
+  %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i32 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3758,7 +3758,7 @@ define amdgpu_ps <2 x float> @global_inc_saddr_i64_rtn(ptr addrspace(1) inreg %s
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i64 %data syncscope("agent") monotonic
+  %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i64 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -3784,7 +3784,7 @@ define amdgpu_ps <2 x float> @global_inc_saddr_i64_rtn_neg128(ptr addrspace(1) i
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i64 %data syncscope("agent") monotonic
+  %rtn = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i64 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -3806,7 +3806,7 @@ define amdgpu_ps void @global_inc_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i64 %data syncscope("agent") monotonic
+  %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep0, i64 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3828,7 +3828,7 @@ define amdgpu_ps void @global_inc_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i64 %data syncscope("agent") monotonic
+  %unused = atomicrmw uinc_wrap ptr addrspace(1) %gep1, i64 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3857,7 +3857,7 @@ define amdgpu_ps float @global_dec_saddr_i32_rtn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep0, i32 %data syncscope("agent") monotonic
+  %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep0, i32 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -3883,7 +3883,7 @@ define amdgpu_ps float @global_dec_saddr_i32_rtn_neg128(ptr addrspace(1) inreg %
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep1, i32 %data syncscope("agent") monotonic
+  %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep1, i32 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i32 %rtn to float
   ret float %cast.rtn
 }
@@ -3905,7 +3905,7 @@ define amdgpu_ps void @global_dec_saddr_i32_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw udec_wrap ptr addrspace(1) %gep0, i32 %data syncscope("agent") monotonic
+  %unused = atomicrmw udec_wrap ptr addrspace(1) %gep0, i32 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3927,7 +3927,7 @@ define amdgpu_ps void @global_dec_saddr_i32_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw udec_wrap ptr addrspace(1) %gep1, i32 %data syncscope("agent") monotonic
+  %unused = atomicrmw udec_wrap ptr addrspace(1) %gep1, i32 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -3951,7 +3951,7 @@ define amdgpu_ps <2 x float> @global_dec_saddr_i64_rtn(ptr addrspace(1) inreg %s
 ; GFX12-NEXT:    ; return to shader part epilog
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep0, i64 %data syncscope("agent") monotonic
+  %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep0, i64 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -3977,7 +3977,7 @@ define amdgpu_ps <2 x float> @global_dec_saddr_i64_rtn_neg128(ptr addrspace(1) i
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep1, i64 %data syncscope("agent") monotonic
+  %rtn = atomicrmw udec_wrap ptr addrspace(1) %gep1, i64 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   %cast.rtn = bitcast i64 %rtn to <2 x float>
   ret <2 x float> %cast.rtn
 }
@@ -3999,7 +3999,7 @@ define amdgpu_ps void @global_dec_saddr_i64_nortn(ptr addrspace(1) inreg %sbase,
 ; GFX12-NEXT:    s_endpgm
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
-  %unused = atomicrmw udec_wrap ptr addrspace(1) %gep0, i64 %data syncscope("agent") monotonic
+  %unused = atomicrmw udec_wrap ptr addrspace(1) %gep0, i64 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
@@ -4021,8 +4021,10 @@ define amdgpu_ps void @global_dec_saddr_i64_nortn_neg128(ptr addrspace(1) inreg
   %zext.offset = zext i32 %voffset to i64
   %gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
   %gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
-  %unused = atomicrmw udec_wrap ptr addrspace(1) %gep1, i64 %data syncscope("agent") monotonic
+  %unused = atomicrmw udec_wrap ptr addrspace(1) %gep1, i64 %data syncscope("agent") monotonic, !amdgpu.no.fine.grained.memory !0
   ret void
 }
 
 attributes #0 = { argmemonly nounwind willreturn }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics.ll b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
index 2aa198fd897b6..da132d0269e6b 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
@@ -824,7 +824,7 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -893,7 +893,7 @@ define amdgpu_kernel void @atomic_and_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -971,7 +971,7 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1060,7 +1060,7 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -1117,7 +1117,7 @@ define amdgpu_kernel void @atomic_and_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1185,7 +1185,7 @@ define amdgpu_kernel void @atomic_and_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -1260,7 +1260,7 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1346,7 +1346,7 @@ define amdgpu_kernel void @atomic_and_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile and ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -1404,7 +1404,7 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1473,7 +1473,7 @@ define amdgpu_kernel void @atomic_sub_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -1551,7 +1551,7 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1640,7 +1640,7 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -1697,7 +1697,7 @@ define amdgpu_kernel void @atomic_sub_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1765,7 +1765,7 @@ define amdgpu_kernel void @atomic_sub_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -1840,7 +1840,7 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1926,7 +1926,7 @@ define amdgpu_kernel void @atomic_sub_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile sub ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -1984,7 +1984,7 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2049,7 +2049,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -2120,7 +2120,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2205,7 +2205,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -2255,7 +2255,7 @@ define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2319,7 +2319,7 @@ define amdgpu_kernel void @atomic_max_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -2387,7 +2387,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2469,7 +2469,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile max ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -2520,7 +2520,7 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr addrspace(1) %out, i32 %in
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2585,7 +2585,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -2656,7 +2656,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2741,7 +2741,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -2791,7 +2791,7 @@ define amdgpu_kernel void @atomic_umax_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2855,7 +2855,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret(ptr addrspace(1) %out, ptr addrsp
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -2923,7 +2923,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64(ptr addrspace(1) %out, i32 %in
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3005,7 +3005,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umax ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -3056,7 +3056,7 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3121,7 +3121,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -3192,7 +3192,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3277,7 +3277,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -3327,7 +3327,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3391,7 +3391,7 @@ define amdgpu_kernel void @atomic_min_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -3459,7 +3459,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3541,7 +3541,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile min ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -3592,7 +3592,7 @@ define amdgpu_kernel void @atomic_umin_i32_offset(ptr addrspace(1) %out, i32 %in
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3657,7 +3657,7 @@ define amdgpu_kernel void @atomic_umin_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -3728,7 +3728,7 @@ define amdgpu_kernel void @atomic_umin_i32_addr64_offset(ptr addrspace(1) %out,
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3813,7 +3813,7 @@ define amdgpu_kernel void @atomic_umin_i32_ret_addr64_offset(ptr addrspace(1) %o
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr addrspace(1) %gep, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -3863,7 +3863,7 @@ define amdgpu_kernel void @atomic_umin_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile umin ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3927,7 +3927,7 @@ define amdgpu_kernel void @atomic_umin_i32_ret(ptr addrspace(1) %out, ptr addrsp
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile umin ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -3995,7 +3995,7 @@ define amdgpu_kernel void @atomic_umin_i32_addr64(ptr addrspace(1) %out, i32 %in
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile umin ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4077,7 +4077,7 @@ define amdgpu_kernel void @atomic_umin_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile umin ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst
+  %val = atomicrmw volatile umin ptr addrspace(1) %ptr, i32 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -4135,7 +4135,7 @@ define amdgpu_kernel void @atomic_or_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4204,7 +4204,7 @@ define amdgpu_kernel void @atomic_or_i32_ret_offset(ptr addrspace(1) %out, ptr a
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -4282,7 +4282,7 @@ define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr addrspace(1) %out, i3
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4371,7 +4371,7 @@ define amdgpu_kernel void @atomic_or_i32_ret_addr64_offset(ptr addrspace(1) %out
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -4428,7 +4428,7 @@ define amdgpu_kernel void @atomic_or_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile or ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4496,7 +4496,7 @@ define amdgpu_kernel void @atomic_or_i32_ret(ptr addrspace(1) %out, ptr addrspac
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile or ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -4571,7 +4571,7 @@ define amdgpu_kernel void @atomic_or_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile or ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4657,7 +4657,7 @@ define amdgpu_kernel void @atomic_or_i32_ret_addr64(ptr addrspace(1) %out, ptr a
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile or ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile or ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -4715,7 +4715,7 @@ define amdgpu_kernel void @atomic_xchg_i32_offset(ptr addrspace(1) %out, i32 %in
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile xchg ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xchg ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -5978,7 +5978,7 @@ define amdgpu_kernel void @atomic_xor_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -6047,7 +6047,7 @@ define amdgpu_kernel void @atomic_xor_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -6125,7 +6125,7 @@ define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -6214,7 +6214,7 @@ define amdgpu_kernel void @atomic_xor_i32_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -6271,7 +6271,7 @@ define amdgpu_kernel void @atomic_xor_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX11-NEXT:    buffer_gl0_inv
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile xor ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -6339,7 +6339,7 @@ define amdgpu_kernel void @atomic_xor_i32_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[2:3]
 ; GFX11-NEXT:    s_endpgm
 entry:
-  %val = atomicrmw volatile xor ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -6414,7 +6414,7 @@ define amdgpu_kernel void @atomic_xor_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile xor ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -6500,7 +6500,7 @@ define amdgpu_kernel void @atomic_xor_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
-  %val = atomicrmw volatile xor ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile xor ptr addrspace(1) %ptr, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -8233,7 +8233,7 @@ define amdgpu_kernel void @atomic_inc_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -8294,7 +8294,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_neg_offset(ptr addrspace(1) %out,
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 -1024
-  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -8353,7 +8353,7 @@ define amdgpu_kernel void @atomic_inc_i32_soffset(ptr addrspace(1) %out, i32 %in
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 9000
-  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -8418,7 +8418,7 @@ define amdgpu_kernel void @atomic_inc_i32_huge_offset(ptr addrspace(1) %out, i32
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 47224239175595
-  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -8487,7 +8487,7 @@ define amdgpu_kernel void @atomic_inc_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -8565,7 +8565,7 @@ define amdgpu_kernel void @atomic_inc_i32_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -8654,7 +8654,7 @@ define amdgpu_kernel void @atomic_inc_i32_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -8712,7 +8712,7 @@ define amdgpu_kernel void @atomic_dec_i32_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -8773,7 +8773,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_neg_offset(ptr addrspace(1) %out,
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 -1024
-  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -8832,7 +8832,7 @@ define amdgpu_kernel void @atomic_dec_i32_soffset(ptr addrspace(1) %out, i32 %in
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 9000
-  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -8897,7 +8897,7 @@ define amdgpu_kernel void @atomic_dec_i32_huge_offset(ptr addrspace(1) %out, i32
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 47224239175595
-  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -8966,7 +8966,7 @@ define amdgpu_kernel void @atomic_dec_i32_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
-  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -9044,7 +9044,7 @@ define amdgpu_kernel void @atomic_dec_i32_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -9133,7 +9133,7 @@ define amdgpu_kernel void @atomic_dec_i32_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 4
-  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst
+  %val = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i32 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i32 %val, ptr addrspace(1) %out2
   ret void
 }
@@ -9437,3 +9437,243 @@ define amdgpu_kernel void @atomic_load_bf16_negoffset(ptr addrspace(1) %in, ptr
   store bfloat %val, ptr addrspace(1) %out
   ret void
 }
+
+define amdgpu_kernel void @atomic_sub_i16_soffset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i16 %in) {
+; SI-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s2, s[4:5], 0xb
+; SI-NEXT:    s_mov_b64 s[4:5], 0
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_add_u32 s6, s0, 0x4650
+; SI-NEXT:    s_addc_u32 s1, s1, 0
+; SI-NEXT:    s_and_b32 s0, s6, -4
+; SI-NEXT:    s_and_b32 s6, s6, 3
+; SI-NEXT:    s_and_b32 s2, s2, 0xffff
+; SI-NEXT:    s_load_dword s9, s[0:1], 0x0
+; SI-NEXT:    s_lshl_b32 s7, s6, 3
+; SI-NEXT:    s_lshl_b32 s6, 0xffff, s7
+; SI-NEXT:    s_lshl_b32 s7, s2, s7
+; SI-NEXT:    s_not_b32 s8, s6
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:  .LBB136_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s7, v1
+; SI-NEXT:    v_and_b32_e32 v0, s6, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, s8, v1
+; SI-NEXT:    v_or_b32_e32 v0, v2, v0
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; SI-NEXT:    s_cbranch_execnz .LBB136_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_u32 s3, s0, 0x4650
+; VI-NEXT:    s_addc_u32 s1, s1, 0
+; VI-NEXT:    s_and_b32 s0, s3, -4
+; VI-NEXT:    s_load_dword s9, s[0:1], 0x0
+; VI-NEXT:    s_and_b32 s3, s3, 3
+; VI-NEXT:    s_lshl_b32 s3, s3, 3
+; VI-NEXT:    s_lshl_b32 s6, 0xffff, s3
+; VI-NEXT:    s_and_b32 s2, s2, 0xffff
+; VI-NEXT:    s_not_b32 s7, s6
+; VI-NEXT:    s_lshl_b32 s8, s2, s3
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:  .LBB136_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    v_subrev_u32_e32 v0, vcc, s8, v1
+; VI-NEXT:    v_and_b32_e32 v2, s7, v1
+; VI-NEXT:    v_and_b32_e32 v0, s6, v0
+; VI-NEXT:    v_or_b32_e32 v0, v2, v0
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v1, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB136_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_add_u32 s3, s0, 0x4650
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0
+; GFX9-NEXT:    s_and_b32 s0, s3, -4
+; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x0
+; GFX9-NEXT:    s_and_b32 s3, s3, 3
+; GFX9-NEXT:    s_lshl_b32 s3, s3, 3
+; GFX9-NEXT:    s_lshl_b32 s4, 0xffff, s3
+; GFX9-NEXT:    s_and_b32 s2, s2, 0xffff
+; GFX9-NEXT:    s_not_b32 s5, s4
+; GFX9-NEXT:    s_lshl_b32 s6, s2, s3
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:  .LBB136_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_subrev_u32_e32 v0, s6, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    s_cbranch_execnz .LBB136_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_endpgm
+  %gep = getelementptr i16, ptr addrspace(1) %out, i64 9000
+  %val = atomicrmw sub ptr addrspace(1) %gep, i16 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+  ret void
+}
+
+define amdgpu_kernel void @atomic_sub_i8_soffset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i8 %in) {
+; SI-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT:    s_load_dword s2, s[4:5], 0xb
+; SI-NEXT:    s_mov_b64 s[4:5], 0
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_add_u32 s6, s0, 0x2328
+; SI-NEXT:    s_addc_u32 s1, s1, 0
+; SI-NEXT:    s_and_b32 s0, s6, -4
+; SI-NEXT:    s_and_b32 s6, s6, 3
+; SI-NEXT:    s_and_b32 s2, s2, 0xff
+; SI-NEXT:    s_load_dword s9, s[0:1], 0x0
+; SI-NEXT:    s_lshl_b32 s7, s6, 3
+; SI-NEXT:    s_lshl_b32 s6, 0xff, s7
+; SI-NEXT:    s_lshl_b32 s7, s2, s7
+; SI-NEXT:    s_not_b32 s8, s6
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v1, s9
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:  .LBB137_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s7, v1
+; SI-NEXT:    v_and_b32_e32 v0, s6, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v2, s8, v1
+; SI-NEXT:    v_or_b32_e32 v0, v2, v0
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; SI-NEXT:    s_cbranch_execnz .LBB137_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_u32 s3, s0, 0x2328
+; VI-NEXT:    s_addc_u32 s1, s1, 0
+; VI-NEXT:    s_and_b32 s0, s3, -4
+; VI-NEXT:    s_load_dword s9, s[0:1], 0x0
+; VI-NEXT:    s_and_b32 s3, s3, 3
+; VI-NEXT:    s_lshl_b32 s3, s3, 3
+; VI-NEXT:    s_lshl_b32 s6, 0xff, s3
+; VI-NEXT:    s_and_b32 s2, s2, 0xff
+; VI-NEXT:    s_not_b32 s7, s6
+; VI-NEXT:    s_lshl_b32 s8, s2, s3
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s9
+; VI-NEXT:    s_mov_b32 s3, 0xf000
+; VI-NEXT:    s_mov_b32 s2, -1
+; VI-NEXT:  .LBB137_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    v_subrev_u32_e32 v0, vcc, s8, v1
+; VI-NEXT:    v_and_b32_e32 v2, s7, v1
+; VI-NEXT:    v_and_b32_e32 v0, s6, v0
+; VI-NEXT:    v_or_b32_e32 v0, v2, v0
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v1, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB137_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_endpgm
+;
+; GFX9-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_add_u32 s3, s0, 0x2328
+; GFX9-NEXT:    s_addc_u32 s1, s1, 0
+; GFX9-NEXT:    s_and_b32 s0, s3, -4
+; GFX9-NEXT:    s_load_dword s7, s[0:1], 0x0
+; GFX9-NEXT:    s_and_b32 s3, s3, 3
+; GFX9-NEXT:    s_lshl_b32 s3, s3, 3
+; GFX9-NEXT:    s_lshl_b32 s4, 0xff, s3
+; GFX9-NEXT:    s_and_b32 s2, s2, 0xff
+; GFX9-NEXT:    s_not_b32 s5, s4
+; GFX9-NEXT:    s_lshl_b32 s6, s2, s3
+; GFX9-NEXT:    s_mov_b64 s[2:3], 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s7
+; GFX9-NEXT:  .LBB137_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    v_subrev_u32_e32 v0, s6, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, s4, v0
+; GFX9-NEXT:    v_and_or_b32 v0, v1, s5, v0
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    s_cbranch_execnz .LBB137_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_endpgm
+  %gep = getelementptr i8, ptr addrspace(1) %out, i64 9000
+  %val = atomicrmw sub ptr addrspace(1) %gep, i8 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+  ret void
+}
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
index a867c6c1affb8..ffab56847edca 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
@@ -1284,26 +1284,68 @@ define void @global_atomic_sub_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB30_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_sub_i32_e32 v3, vcc, v4, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB30_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_sub_i32_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_sub v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB30_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_sub_u32_e32 v3, vcc, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB30_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i32_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_sub v[0:1], v2, off
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB30_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw sub ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -1317,9 +1359,25 @@ define void @global_atomic_sub_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB31_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_sub_i32_e32 v3, vcc, v4, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB31_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1328,17 +1386,43 @@ define void @global_atomic_sub_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_sub v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB31_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_sub_u32_e32 v3, vcc, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB31_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i32_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_sub v[0:1], v2, off offset:16
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB31_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst
@@ -1353,27 +1437,71 @@ define i32 @global_atomic_sub_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB32_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_sub_i32_e32 v4, vcc, v5, v2
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB32_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_sub_i32_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB32_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    v_sub_u32_e32 v3, vcc, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB32_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v3
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i32_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_sub v0, v[0:1], v2, off glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB32_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw sub ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -1387,29 +1515,72 @@ define i32 @global_atomic_sub_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_sub v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB33_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_sub_i32_e32 v4, vcc, v5, v2
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB33_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_sub_i32_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[3:4]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB33_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, v0
+; VI-NEXT:    v_sub_u32_e32 v0, vcc, v1, v2
+; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB33_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i32_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_sub v0, v[0:1], v2, off offset:16 glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_sub_u32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB33_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst
@@ -1421,23 +1592,37 @@ define amdgpu_gfx void @global_atomic_sub_i32_noret_scalar(ptr addrspace(1) inre
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB34_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_sub v0, off, s[4:7], 0
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s34, v1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB34_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1447,20 +1632,44 @@ define amdgpu_gfx void @global_atomic_sub_i32_noret_scalar(ptr addrspace(1) inre
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_sub v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB34_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v3
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB34_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i32_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_sub v0, v1, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_subrev_u32_e32 v0, s6, v1
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB34_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw sub ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -1471,23 +1680,37 @@ define amdgpu_gfx void @global_atomic_sub_i32_noret_offset_scalar(ptr addrspace(
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB35_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_sub v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s34, v1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB35_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1499,20 +1722,44 @@ define amdgpu_gfx void @global_atomic_sub_i32_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s34
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_sub v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB35_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_subrev_u32_e32 v2, vcc, s6, v3
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB35_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i32_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_sub v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_subrev_u32_e32 v0, s6, v1
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB35_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst
@@ -1524,23 +1771,37 @@ define amdgpu_gfx i32 @global_atomic_sub_i32_ret_scalar(ptr addrspace(1) inreg %
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB36_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_sub v0, off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_subrev_i32_e32 v1, vcc, s34, v2
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB36_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1550,20 +1811,46 @@ define amdgpu_gfx i32 @global_atomic_sub_i32_ret_scalar(ptr addrspace(1) inreg %
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v2, s5
+; VI-NEXT:  .LBB36_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v4
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB36_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i32_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_sub v0, v0, v1, s[4:5] glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v2, s6, v3
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB36_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw sub ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -1574,23 +1861,37 @@ define amdgpu_gfx i32 @global_atomic_sub_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB37_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_sub v0, off, s[4:7], 0 offset:16 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_subrev_i32_e32 v1, vcc, s34, v2
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB37_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1600,22 +1901,46 @@ define amdgpu_gfx i32 @global_atomic_sub_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 16
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v0, s34
-; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_sub v0, v[0:1], v2 glc
+; VI-NEXT:    v_mov_b32_e32 v1, s34
+; VI-NEXT:    v_mov_b32_e32 v2, s35
+; VI-NEXT:    flat_load_dword v0, v[1:2]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB37_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_subrev_u32_e32 v3, vcc, s6, v4
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB37_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i32_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_sub v0, v0, v1, s[4:5] offset:16 glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_subrev_u32_e32 v2, s6, v3
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB37_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw sub ptr addrspace(1) %gep, i32 %in seq_cst
@@ -1744,26 +2069,68 @@ define void @global_atomic_and_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB41_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v3, v4, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB41_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_and_i32_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_and v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB41_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v3, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB41_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i32_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_and v[0:1], v2, off
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB41_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw and ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -1777,9 +2144,25 @@ define void @global_atomic_and_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB42_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v3, v4, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB42_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1788,17 +2171,43 @@ define void @global_atomic_and_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_and v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB42_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v3, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB42_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i32_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_and v[0:1], v2, off offset:16
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB42_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst
@@ -1813,27 +2222,71 @@ define i32 @global_atomic_and_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB43_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v4, v5, v2
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB43_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_and_i32_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB43_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    v_and_b32_e32 v3, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB43_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v3
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i32_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_and v0, v[0:1], v2, off glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_and_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB43_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw and ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -1847,29 +2300,72 @@ define i32 @global_atomic_and_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_and v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB44_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v4, v5, v2
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB44_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_and_i32_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[3:4]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB44_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, v0
+; VI-NEXT:    v_and_b32_e32 v0, v1, v2
+; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB44_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i32_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_and v0, v[0:1], v2, off offset:16 glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_and_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB44_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst
@@ -1881,23 +2377,37 @@ define amdgpu_gfx void @global_atomic_and_i32_noret_scalar(ptr addrspace(1) inre
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB45_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_and v0, off, s[4:7], 0
+; SI-NEXT:    v_and_b32_e32 v0, s34, v1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB45_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1907,20 +2417,44 @@ define amdgpu_gfx void @global_atomic_and_i32_noret_scalar(ptr addrspace(1) inre
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_and v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB45_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, s6, v3
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB45_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i32_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_and v0, v1, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, s6, v1
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB45_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw and ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -1931,23 +2465,37 @@ define amdgpu_gfx void @global_atomic_and_i32_noret_offset_scalar(ptr addrspace(
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB46_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_and v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    v_and_b32_e32 v0, s34, v1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB46_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1959,20 +2507,44 @@ define amdgpu_gfx void @global_atomic_and_i32_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s34
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_and v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB46_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v2, s6, v3
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB46_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i32_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_and v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, s6, v1
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB46_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst
@@ -1984,23 +2556,37 @@ define amdgpu_gfx i32 @global_atomic_and_i32_ret_scalar(ptr addrspace(1) inreg %
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB47_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_and v0, off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, s34, v2
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB47_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -2010,20 +2596,46 @@ define amdgpu_gfx i32 @global_atomic_and_i32_ret_scalar(ptr addrspace(1) inreg %
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v2, s5
+; VI-NEXT:  .LBB47_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_and_b32_e32 v3, s6, v4
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB47_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i32_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_and v0, v0, v1, s[4:5] glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_and_b32_e32 v2, s6, v3
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB47_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw and ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -2034,23 +2646,37 @@ define amdgpu_gfx i32 @global_atomic_and_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB48_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_and v0, off, s[4:7], 0 offset:16 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v1, s34, v2
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB48_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -2060,22 +2686,46 @@ define amdgpu_gfx i32 @global_atomic_and_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 16
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v0, s34
-; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_and v0, v[0:1], v2 glc
+; VI-NEXT:    v_mov_b32_e32 v1, s34
+; VI-NEXT:    v_mov_b32_e32 v2, s35
+; VI-NEXT:    flat_load_dword v0, v[1:2]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB48_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_and_b32_e32 v3, s6, v4
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB48_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i32_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_and v0, v0, v1, s[4:5] offset:16 glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB48_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_and_b32_e32 v2, s6, v3
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB48_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw and ptr addrspace(1) %gep, i32 %in seq_cst
@@ -3030,26 +3680,68 @@ define void @global_atomic_or_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB61_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v3, v4, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB61_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_or_i32_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_or v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB61_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v3, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB61_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i32_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_or v[0:1], v2, off
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB61_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw or ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -3063,9 +3755,25 @@ define void @global_atomic_or_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB62_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v3, v4, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB62_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3074,17 +3782,43 @@ define void @global_atomic_or_i32_noret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_or v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB62_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v3, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB62_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i32_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_or v[0:1], v2, off offset:16
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB62_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst
@@ -3099,27 +3833,71 @@ define i32 @global_atomic_or_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB63_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_or_b32_e32 v4, v5, v2
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB63_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_or_i32_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB63_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    v_or_b32_e32 v3, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB63_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v3
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i32_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_or v0, v[0:1], v2, off glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_or_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB63_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw or ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -3133,29 +3911,72 @@ define i32 @global_atomic_or_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_or v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB64_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_or_b32_e32 v4, v5, v2
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB64_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_or_i32_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[3:4]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB64_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, v0
+; VI-NEXT:    v_or_b32_e32 v0, v1, v2
+; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB64_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i32_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_or v0, v[0:1], v2, off offset:16 glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB64_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_or_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB64_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst
@@ -3167,23 +3988,37 @@ define amdgpu_gfx void @global_atomic_or_i32_noret_scalar(ptr addrspace(1) inreg
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB65_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_or v0, off, s[4:7], 0
+; SI-NEXT:    v_or_b32_e32 v0, s34, v1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB65_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3193,20 +4028,44 @@ define amdgpu_gfx void @global_atomic_or_i32_noret_scalar(ptr addrspace(1) inreg
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_or v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB65_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v2, s6, v3
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB65_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i32_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_or v0, v1, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB65_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v0, s6, v1
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB65_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw or ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -3217,23 +4076,37 @@ define amdgpu_gfx void @global_atomic_or_i32_noret_offset_scalar(ptr addrspace(1
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB66_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_or v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    v_or_b32_e32 v0, s34, v1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB66_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3245,20 +4118,44 @@ define amdgpu_gfx void @global_atomic_or_i32_noret_offset_scalar(ptr addrspace(1
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s34
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_or v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB66_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v2, s6, v3
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB66_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i32_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_or v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB66_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v0, s6, v1
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB66_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst
@@ -3270,23 +4167,37 @@ define amdgpu_gfx i32 @global_atomic_or_i32_ret_scalar(ptr addrspace(1) inreg %p
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB67_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_or v0, off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_or_b32_e32 v1, s34, v2
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB67_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3296,20 +4207,46 @@ define amdgpu_gfx i32 @global_atomic_or_i32_ret_scalar(ptr addrspace(1) inreg %p
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v2, s5
+; VI-NEXT:  .LBB67_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_or_b32_e32 v3, s6, v4
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB67_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i32_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_or v0, v0, v1, s[4:5] glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB67_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_or_b32_e32 v2, s6, v3
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB67_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw or ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -3320,23 +4257,37 @@ define amdgpu_gfx i32 @global_atomic_or_i32_ret_offset_scalar(ptr addrspace(1) i
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB68_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_or v0, off, s[4:7], 0 offset:16 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_or_b32_e32 v1, s34, v2
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB68_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3346,22 +4297,46 @@ define amdgpu_gfx i32 @global_atomic_or_i32_ret_offset_scalar(ptr addrspace(1) i
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 16
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v0, s34
-; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_or v0, v[0:1], v2 glc
+; VI-NEXT:    v_mov_b32_e32 v1, s34
+; VI-NEXT:    v_mov_b32_e32 v2, s35
+; VI-NEXT:    flat_load_dword v0, v[1:2]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB68_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_or_b32_e32 v3, s6, v4
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB68_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i32_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_or v0, v0, v1, s[4:5] offset:16 glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB68_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_or_b32_e32 v2, s6, v3
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB68_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw or ptr addrspace(1) %gep, i32 %in seq_cst
@@ -3490,26 +4465,68 @@ define void @global_atomic_xor_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB72_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v3, v4, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB72_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_xor_i32_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_xor v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB72_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v3, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB72_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i32_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_xor v[0:1], v2, off
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB72_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB72_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw xor ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -3523,9 +4540,25 @@ define void @global_atomic_xor_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB73_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v3, v4, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB73_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3534,17 +4567,43 @@ define void @global_atomic_xor_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_xor v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB73_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v3, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB73_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i32_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_xor v[0:1], v2, off offset:16
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB73_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB73_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst
@@ -3559,27 +4618,71 @@ define i32 @global_atomic_xor_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB74_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v4, v5, v2
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB74_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_xor_i32_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB74_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    v_xor_b32_e32 v3, v4, v2
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB74_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v3
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i32_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_xor v0, v[0:1], v2, off glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB74_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw xor ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -3593,29 +4696,72 @@ define i32 @global_atomic_xor_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_xor v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB75_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v4, v5, v2
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB75_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_xor_i32_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[3:4]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB75_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, v0
+; VI-NEXT:    v_xor_b32_e32 v0, v1, v2
+; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB75_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i32_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_xor v0, v[0:1], v2, off offset:16 glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_xor_b32_e32 v3, v4, v2
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB75_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst
@@ -3627,23 +4773,37 @@ define amdgpu_gfx void @global_atomic_xor_i32_noret_scalar(ptr addrspace(1) inre
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB76_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_xor v0, off, s[4:7], 0
+; SI-NEXT:    v_xor_b32_e32 v0, s34, v1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB76_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3653,20 +4813,44 @@ define amdgpu_gfx void @global_atomic_xor_i32_noret_scalar(ptr addrspace(1) inre
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_xor v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB76_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v2, s6, v3
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB76_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i32_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_xor v0, v1, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v0, s6, v1
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB76_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw xor ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -3677,23 +4861,37 @@ define amdgpu_gfx void @global_atomic_xor_i32_noret_offset_scalar(ptr addrspace(
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB77_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_xor v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    v_xor_b32_e32 v0, s34, v1
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB77_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3705,20 +4903,44 @@ define amdgpu_gfx void @global_atomic_xor_i32_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s34
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_xor v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB77_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v2, s6, v3
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB77_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i32_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_xor v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v0, s6, v1
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB77_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst
@@ -3730,23 +4952,37 @@ define amdgpu_gfx i32 @global_atomic_xor_i32_ret_scalar(ptr addrspace(1) inreg %
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB78_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_xor v0, off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v1, s34, v2
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB78_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3756,20 +4992,46 @@ define amdgpu_gfx i32 @global_atomic_xor_i32_ret_scalar(ptr addrspace(1) inreg %
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v2, s5
+; VI-NEXT:  .LBB78_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_xor_b32_e32 v3, s6, v4
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB78_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i32_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_xor v0, v0, v1, s[4:5] glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB78_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_xor_b32_e32 v2, s6, v3
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB78_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw xor ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -3780,23 +5042,37 @@ define amdgpu_gfx i32 @global_atomic_xor_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB79_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_xor v0, off, s[4:7], 0 offset:16 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v1, s34, v2
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB79_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3806,22 +5082,46 @@ define amdgpu_gfx i32 @global_atomic_xor_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 16
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v0, s34
-; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_xor v0, v[0:1], v2 glc
+; VI-NEXT:    v_mov_b32_e32 v1, s34
+; VI-NEXT:    v_mov_b32_e32 v2, s35
+; VI-NEXT:    flat_load_dword v0, v[1:2]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB79_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_xor_b32_e32 v3, s6, v4
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB79_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i32_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_xor v0, v0, v1, s[4:5] offset:16 glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB79_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_xor_b32_e32 v2, s6, v3
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB79_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw xor ptr addrspace(1) %gep, i32 %in seq_cst
@@ -5041,25 +6341,9 @@ define void @global_atomic_max_i32_noret_offset__amdgpu_no_remote_memory(ptr add
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB95_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_max_i32_e32 v3, v4, v2
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v6, v4
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_atomic_smax v2, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v4, v5
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB95_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5068,43 +6352,17 @@ define void @global_atomic_max_i32_noret_offset__amdgpu_no_remote_memory(ptr add
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dword v4, v[0:1]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB95_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_max_i32_e32 v3, v4, v2
-; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; VI-NEXT:    flat_atomic_smax v[0:1], v2
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    v_mov_b32_e32 v4, v3
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB95_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_max_i32_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB95_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_i32_e32 v3, v4, v2
-; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
+; GFX9-NEXT:    global_atomic_smax v[0:1], v2, off offset:16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v4, v3
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB95_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -5119,72 +6377,29 @@ define i32 @global_atomic_max_i32_ret_offset__amdgpu_no_remote_memory(ptr addrsp
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB96_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_max_i32_e32 v4, v5, v2
-; SI-NEXT:    v_mov_b32_e32 v3, v4
-; SI-NEXT:    v_mov_b32_e32 v4, v5
-; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_atomic_smax v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB96_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v0, v3
+; SI-NEXT:    v_mov_b32_e32 v0, v2
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dword v0, v[3:4]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB96_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v1, v0
-; VI-NEXT:    v_max_i32_e32 v0, v1, v2
-; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_atomic_smax v0, v[0:1], v2 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB96_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_max_i32_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB96_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, v3
-; GFX9-NEXT:    v_max_i32_e32 v3, v4, v2
-; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
+; GFX9-NEXT:    global_atomic_smax v0, v[0:1], v2, off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB96_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %result = atomicrmw max ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -6199,25 +7414,9 @@ define void @global_atomic_umax_i32_noret_offset__amdgpu_no_remote_memory(ptr ad
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB108_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_max_u32_e32 v3, v4, v2
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v6, v4
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_atomic_umax v2, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v4, v5
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB108_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6226,43 +7425,17 @@ define void @global_atomic_umax_i32_noret_offset__amdgpu_no_remote_memory(ptr ad
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dword v4, v[0:1]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB108_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_max_u32_e32 v3, v4, v2
-; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; VI-NEXT:    flat_atomic_umax v[0:1], v2
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    v_mov_b32_e32 v4, v3
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB108_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_umax_i32_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB108_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_max_u32_e32 v3, v4, v2
-; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
+; GFX9-NEXT:    global_atomic_umax v[0:1], v2, off offset:16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v4, v3
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB108_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -6277,72 +7450,29 @@ define i32 @global_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory(ptr addrs
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB109_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_max_u32_e32 v4, v5, v2
-; SI-NEXT:    v_mov_b32_e32 v3, v4
-; SI-NEXT:    v_mov_b32_e32 v4, v5
-; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_atomic_umax v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB109_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v0, v3
+; SI-NEXT:    v_mov_b32_e32 v0, v2
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dword v0, v[3:4]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB109_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v1, v0
-; VI-NEXT:    v_max_u32_e32 v0, v1, v2
-; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_atomic_umax v0, v[0:1], v2 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB109_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_umax_i32_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB109_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, v3
-; GFX9-NEXT:    v_max_u32_e32 v3, v4, v2
-; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
+; GFX9-NEXT:    global_atomic_umax v0, v[0:1], v2, off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB109_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %result = atomicrmw umax ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -7032,25 +8162,9 @@ define void @global_atomic_umin_i32_noret_offset__amdgpu_no_remote_memory(ptr ad
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB118_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_min_u32_e32 v3, v4, v2
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v6, v4
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_atomic_umin v2, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v4, v5
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB118_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -7058,44 +8172,18 @@ define void @global_atomic_umin_i32_noret_offset__amdgpu_no_remote_memory(ptr ad
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dword v4, v[0:1]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB118_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_min_u32_e32 v3, v4, v2
-; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    v_mov_b32_e32 v4, v3
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB118_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_atomic_umin v[0:1], v2
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_wbinvl1_vol
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_umin_i32_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB118_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_min_u32_e32 v3, v4, v2
-; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
+; GFX9-NEXT:    global_atomic_umin v[0:1], v2, off offset:16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v4, v3
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB118_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw umin ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -7110,72 +8198,29 @@ define i32 @global_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory(ptr addrs
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB119_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_min_u32_e32 v4, v5, v2
-; SI-NEXT:    v_mov_b32_e32 v3, v4
-; SI-NEXT:    v_mov_b32_e32 v4, v5
-; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_atomic_umin v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB119_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v0, v3
+; SI-NEXT:    v_mov_b32_e32 v0, v2
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dword v0, v[3:4]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB119_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v1, v0
-; VI-NEXT:    v_min_u32_e32 v0, v1, v2
-; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_atomic_umin v0, v[0:1], v2 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB119_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_umin_i32_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB119_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, v3
-; GFX9-NEXT:    v_min_u32_e32 v3, v4, v2
-; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
+; GFX9-NEXT:    global_atomic_umin v0, v[0:1], v2, off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB119_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %result = atomicrmw umin ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -8272,25 +9317,9 @@ define void @global_atomic_min_i32_noret_offset__amdgpu_no_remote_memory(ptr add
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB132_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_min_i32_e32 v3, v4, v2
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v6, v4
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_atomic_smin v2, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v4, v5
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB132_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8299,43 +9328,17 @@ define void @global_atomic_min_i32_noret_offset__amdgpu_no_remote_memory(ptr add
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dword v4, v[0:1]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB132_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_min_i32_e32 v3, v4, v2
-; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; VI-NEXT:    flat_atomic_smin v[0:1], v2
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    v_mov_b32_e32 v4, v3
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB132_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_min_i32_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB132_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_min_i32_e32 v3, v4, v2
-; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
+; GFX9-NEXT:    global_atomic_smin v[0:1], v2, off offset:16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v4, v3
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB132_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -8350,72 +9353,29 @@ define i32 @global_atomic_min_i32_ret_offset__amdgpu_no_remote_memory(ptr addrsp
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB133_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_min_i32_e32 v4, v5, v2
-; SI-NEXT:    v_mov_b32_e32 v3, v4
-; SI-NEXT:    v_mov_b32_e32 v4, v5
-; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_atomic_smin v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB133_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v0, v3
+; SI-NEXT:    v_mov_b32_e32 v0, v2
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dword v0, v[3:4]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB133_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v1, v0
-; VI-NEXT:    v_min_i32_e32 v0, v1, v2
-; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_atomic_smin v0, v[0:1], v2 glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB133_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_min_i32_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB133_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v4, v3
-; GFX9-NEXT:    v_min_i32_e32 v3, v4, v2
-; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
+; GFX9-NEXT:    global_atomic_smin v0, v[0:1], v2, off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB133_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i64 4
   %result = atomicrmw min ptr addrspace(1) %gep, i32 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -8434,26 +9394,74 @@ define void @global_atomic_uinc_wrap_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB134_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
+; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; SI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB134_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_uinc_wrap_i32_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_inc v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB134_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 1, v4
+; VI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB134_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_inc v[0:1], v2, off
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB134_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v4
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB134_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -8467,9 +9475,27 @@ define void @global_atomic_uinc_wrap_i32_noret_offset(ptr addrspace(1) %out, i32
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB135_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 1, v4
+; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; SI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB135_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8478,17 +9504,47 @@ define void @global_atomic_uinc_wrap_i32_noret_offset(ptr addrspace(1) %out, i32
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_inc v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB135_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 1, v4
+; VI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB135_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_inc v[0:1], v2, off offset:16
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB135_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v4
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB135_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst
@@ -8503,27 +9559,77 @@ define i32 @global_atomic_uinc_wrap_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB136_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 1, v5
+; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v5, v2
+; SI-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB136_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_uinc_wrap_i32_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB136_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 1, v4
+; VI-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB136_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v3
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_inc v0, v[0:1], v2, off glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB136_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v4
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB136_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -8537,29 +9643,78 @@ define i32 @global_atomic_uinc_wrap_i32_ret_offset(ptr addrspace(1) %out, i32 %i
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_inc v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB137_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, 1, v5
+; SI-NEXT:    v_cmp_lt_u32_e32 vcc, v5, v2
+; SI-NEXT:    v_cndmask_b32_e32 v4, 0, v3, vcc
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[4:7], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB137_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_uinc_wrap_i32_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[3:4]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB137_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v1
+; VI-NEXT:    v_cmp_lt_u32_e32 vcc, v1, v2
+; VI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB137_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_inc v0, v[0:1], v2, off offset:16 glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB137_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_add_u32_e32 v3, 1, v4
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, v4, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB137_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst
@@ -8571,23 +9726,39 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i32_noret_scalar(ptr addrspace(1
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB138_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_inc v0, off, s[4:7], 0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v1
+; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s34, v1
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB138_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -8597,20 +9768,48 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i32_noret_scalar(ptr addrspace(1
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_inc v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB138_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
+; VI-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; VI-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB138_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_inc v0, v1, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB138_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v0, 1, v1
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB138_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -8621,23 +9820,39 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i32_noret_offset_scalar(ptr addr
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v4, s6, 0
+; SI-NEXT:    v_writelane_b32 v4, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB139_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_inc v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v1
+; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s34, v1
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB139_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v4, 1
+; SI-NEXT:    v_readlane_b32 s6, v4, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -8649,20 +9864,48 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i32_noret_offset_scalar(ptr addr
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s34
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_inc v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB139_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 1, v3
+; VI-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; VI-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB139_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_inc v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB139_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v0, 1, v1
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB139_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst
@@ -8674,23 +9917,39 @@ define amdgpu_gfx i32 @global_atomic_uinc_wrap_i32_ret_scalar(ptr addrspace(1) i
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB140_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_inc v0, off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s34, v2
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB140_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -8700,20 +9959,50 @@ define amdgpu_gfx i32 @global_atomic_uinc_wrap_i32_ret_scalar(ptr addrspace(1) i
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v2, s5
+; VI-NEXT:  .LBB140_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v4
+; VI-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
+; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB140_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_inc v0, v0, v1, s[4:5] glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB140_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_add_u32_e32 v0, 1, v3
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v0, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB140_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -8724,23 +10013,39 @@ define amdgpu_gfx i32 @global_atomic_uinc_wrap_i32_ret_offset_scalar(ptr addrspa
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v3, s6, 0
+; SI-NEXT:    v_writelane_b32 v3, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB141_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_inc v0, off, s[4:7], 0 offset:16 glc
+; SI-NEXT:    v_mov_b32_e32 v2, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; SI-NEXT:    v_cmp_gt_u32_e32 vcc, s34, v2
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v1
+; SI-NEXT:    v_mov_b32_e32 v1, v2
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB141_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v3, 1
+; SI-NEXT:    v_readlane_b32 s6, v3, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -8750,22 +10055,50 @@ define amdgpu_gfx i32 @global_atomic_uinc_wrap_i32_ret_offset_scalar(ptr addrspa
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 16
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v0, s34
-; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_inc v0, v[0:1], v2 glc
+; VI-NEXT:    v_mov_b32_e32 v1, s34
+; VI-NEXT:    v_mov_b32_e32 v2, s35
+; VI-NEXT:    flat_load_dword v0, v[1:2]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB141_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v4
+; VI-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v4
+; VI-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB141_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_inc v0, v0, v1, s[4:5] offset:16 glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB141_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v3, v0
+; GFX9-NEXT:    v_add_u32_e32 v0, 1, v3
+; GFX9-NEXT:    v_cmp_gt_u32_e32 vcc, s6, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v0, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB141_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i32 %in seq_cst
@@ -8853,30 +10186,84 @@ define void @global_atomic_udec_wrap_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-LABEL: global_atomic_udec_wrap_i32_noret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s4, s6
-; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB144_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, -1, v4
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; SI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[8:11], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB144_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_udec_wrap_i32_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_dec v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[6:7], 0
+; VI-NEXT:  .LBB144_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v3, vcc, -1, v4
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; VI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT:    s_cbranch_execnz .LBB144_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i32_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_dec v[0:1], v2, off
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:  .LBB144_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GFX9-NEXT:    v_add_u32_e32 v3, -1, v4
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_cbranch_execnz .LBB144_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -8886,13 +10273,33 @@ define void @global_atomic_udec_wrap_i32_noret_offset(ptr addrspace(1) %out, i32
 ; SI-LABEL: global_atomic_udec_wrap_i32_noret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s4, s6
-; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    buffer_load_dword v4, v[0:1], s[8:11], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB145_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, -1, v4
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; SI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v4
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    buffer_atomic_cmpswap v[5:6], v[0:1], s[8:11], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
+; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB145_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -8901,17 +10308,51 @@ define void @global_atomic_udec_wrap_i32_noret_offset(ptr addrspace(1) %out, i32
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_dec v[0:1], v2
+; VI-NEXT:    flat_load_dword v4, v[0:1]
+; VI-NEXT:    s_mov_b64 s[6:7], 0
+; VI-NEXT:  .LBB145_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v3, vcc, -1, v4
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; VI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT:    s_cbranch_execnz .LBB145_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i32_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_dec v[0:1], v2, off offset:16
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:  .LBB145_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GFX9-NEXT:    v_add_u32_e32 v3, -1, v4
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_cbranch_execnz .LBB145_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst
@@ -8922,31 +10363,87 @@ define i32 @global_atomic_udec_wrap_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-LABEL: global_atomic_udec_wrap_i32_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s4, s6
-; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB146_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, -1, v5
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; SI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v5, v2
+; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[8:11], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB146_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_udec_wrap_i32_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[6:7], 0
+; VI-NEXT:  .LBB146_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v4, v3
+; VI-NEXT:    v_add_u32_e32 v3, vcc, -1, v4
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; VI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; VI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT:    s_cbranch_execnz .LBB146_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[6:7]
+; VI-NEXT:    v_mov_b32_e32 v0, v3
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i32_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_dec v0, v[0:1], v2, off glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:  .LBB146_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GFX9-NEXT:    v_add_u32_e32 v3, -1, v4
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_cbranch_execnz .LBB146_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -8956,33 +10453,88 @@ define i32 @global_atomic_udec_wrap_i32_ret_offset(ptr addrspace(1) %out, i32 %i
 ; SI-LABEL: global_atomic_udec_wrap_i32_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s4, s6
-; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_dec v2, v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[8:11], 0 addr64 offset:16
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB147_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v3, vcc, -1, v5
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; SI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v5, v2
+; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v4, v3, v2, vcc
+; SI-NEXT:    v_mov_b32_e32 v3, v4
+; SI-NEXT:    v_mov_b32_e32 v4, v5
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], v[0:1], s[8:11], 0 addr64 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
+; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB147_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
+; SI-NEXT:    v_mov_b32_e32 v0, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_udec_wrap_i32_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; VI-NEXT:    v_add_u32_e32 v3, vcc, 16, v0
+; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dword v0, v[3:4]
+; VI-NEXT:    s_mov_b64 s[6:7], 0
+; VI-NEXT:  .LBB147_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -1, v1
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; VI-NEXT:    v_cmp_gt_u32_e64 s[4:5], v1, v2
+; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT:    s_cbranch_execnz .LBB147_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i32_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_dec v0, v[0:1], v2, off offset:16 glc
+; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
+; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:  .LBB147_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v3
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_gt_u32_e64 s[4:5], v4, v2
+; GFX9-NEXT:    v_add_u32_e32 v3, -1, v4
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX9-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_cbranch_execnz .LBB147_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst
@@ -8994,23 +10546,42 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_scalar(ptr addrspace(1
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v5, s6, 0
+; SI-NEXT:    v_writelane_b32 v5, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[38:39], 0
+; SI-NEXT:    v_mov_b32_e32 v2, s34
+; SI-NEXT:  .LBB148_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_dec v0, off, s[4:7], 0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; SI-NEXT:    v_cmp_lt_u32_e64 s[36:37], s34, v1
+; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v4, v1
+; SI-NEXT:    v_mov_b32_e32 v3, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v1
+; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; SI-NEXT:    s_cbranch_execnz .LBB148_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[38:39]
+; SI-NEXT:    v_readlane_b32 s7, v5, 1
+; SI-NEXT:    v_readlane_b32 s6, v5, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9020,20 +10591,54 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_scalar(ptr addrspace(1
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_dec v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[36:37], 0
+; VI-NEXT:    v_mov_b32_e32 v4, s6
+; VI-NEXT:  .LBB148_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v2, vcc, -1, v3
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; VI-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
+; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; VI-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT:    s_cbranch_execnz .LBB148_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i32_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_dec v0, v1, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[36:37], 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-NEXT:  .LBB148_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, -1, v1
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GFX9-NEXT:    s_cbranch_execnz .LBB148_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %in seq_cst
   ret void
@@ -9044,23 +10649,42 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_offset_scalar(ptr addr
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v5, s6, 0
+; SI-NEXT:    v_writelane_b32 v5, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[38:39], 0
+; SI-NEXT:    v_mov_b32_e32 v2, s34
+; SI-NEXT:  .LBB149_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_dec v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v1
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; SI-NEXT:    v_cmp_lt_u32_e64 s[36:37], s34, v1
+; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v4, v1
+; SI-NEXT:    v_mov_b32_e32 v3, v0
+; SI-NEXT:    buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v1
+; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; SI-NEXT:    s_cbranch_execnz .LBB149_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[38:39]
+; SI-NEXT:    v_readlane_b32 s7, v5, 1
+; SI-NEXT:    v_readlane_b32 s6, v5, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9072,20 +10696,54 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_offset_scalar(ptr addr
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v0, s34
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_dec v[0:1], v2
+; VI-NEXT:    flat_load_dword v3, v[0:1]
+; VI-NEXT:    s_mov_b64 s[36:37], 0
+; VI-NEXT:    v_mov_b32_e32 v4, s6
+; VI-NEXT:  .LBB149_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v2, vcc, -1, v3
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v3
+; VI-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v3
+; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; VI-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT:    v_mov_b32_e32 v3, v2
+; VI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT:    s_cbranch_execnz .LBB149_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i32_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_dec v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[36:37], 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-NEXT:  .LBB149_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GFX9-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v1
+; GFX9-NEXT:    v_add_u32_e32 v0, -1, v1
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GFX9-NEXT:    s_cbranch_execnz .LBB149_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst
@@ -9097,23 +10755,42 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_scalar(ptr addrspace(1) i
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v5, s6, 0
+; SI-NEXT:    v_writelane_b32 v5, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[38:39], 0
+; SI-NEXT:    v_mov_b32_e32 v2, s34
+; SI-NEXT:  .LBB150_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_dec v0, off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v4
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; SI-NEXT:    v_cmp_lt_u32_e64 s[36:37], s34, v4
+; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v2, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v3
+; SI-NEXT:    v_mov_b32_e32 v1, v4
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; SI-NEXT:    s_cbranch_execnz .LBB150_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[38:39]
+; SI-NEXT:    v_readlane_b32 s7, v5, 1
+; SI-NEXT:    v_readlane_b32 s6, v5, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9123,20 +10800,56 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_scalar(ptr addrspace(1) i
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; VI-NEXT:    flat_load_dword v0, v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v1, s4
+; VI-NEXT:    s_mov_b64 s[36:37], 0
+; VI-NEXT:    v_mov_b32_e32 v3, s6
+; VI-NEXT:    v_mov_b32_e32 v2, s5
+; VI-NEXT:  .LBB150_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v5, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -1, v5
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; VI-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
+; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; VI-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
+; VI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT:    s_cbranch_execnz .LBB150_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i32_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_dec v0, v0, v1, s[4:5] glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[36:37], 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:  .LBB150_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v4
+; GFX9-NEXT:    v_add_u32_e32 v0, -1, v4
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[3:4], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX9-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GFX9-NEXT:    s_cbranch_execnz .LBB150_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %in seq_cst
   ret i32 %result
@@ -9147,23 +10860,42 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_offset_scalar(ptr addrspa
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v1, s6, 0
-; SI-NEXT:    v_writelane_b32 v1, s7, 1
+; SI-NEXT:    v_writelane_b32 v5, s6, 0
+; SI-NEXT:    v_writelane_b32 v5, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s34
+; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
+; SI-NEXT:    s_mov_b64 s[38:39], 0
+; SI-NEXT:    v_mov_b32_e32 v2, s34
+; SI-NEXT:  .LBB151_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_dec v0, off, s[4:7], 0 offset:16 glc
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v4
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; SI-NEXT:    v_cmp_lt_u32_e64 s[36:37], s34, v4
+; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT:    v_cndmask_b32_e32 v3, v0, v2, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v3
+; SI-NEXT:    v_mov_b32_e32 v1, v4
+; SI-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v1, 1
-; SI-NEXT:    v_readlane_b32 s6, v1, 0
+; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; SI-NEXT:    s_cbranch_execnz .LBB151_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[38:39]
+; SI-NEXT:    v_readlane_b32 s7, v5, 1
+; SI-NEXT:    v_readlane_b32 s6, v5, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9173,22 +10905,56 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_offset_scalar(ptr addrspa
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 16
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v0, s34
-; VI-NEXT:    v_mov_b32_e32 v1, s35
-; VI-NEXT:    v_mov_b32_e32 v2, s6
-; VI-NEXT:    flat_atomic_dec v0, v[0:1], v2 glc
+; VI-NEXT:    v_mov_b32_e32 v1, s34
+; VI-NEXT:    v_mov_b32_e32 v2, s35
+; VI-NEXT:    flat_load_dword v0, v[1:2]
+; VI-NEXT:    s_mov_b64 s[36:37], 0
+; VI-NEXT:    v_mov_b32_e32 v3, s6
+; VI-NEXT:  .LBB151_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v5, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, -1, v5
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; VI-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v5
+; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; VI-NEXT:    v_cndmask_b32_e32 v4, v0, v3, vcc
+; VI-NEXT:    flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v5
+; VI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT:    s_cbranch_execnz .LBB151_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i32_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s6
-; GFX9-NEXT:    global_atomic_dec v0, v0, v1, s[4:5] offset:16 glc
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
+; GFX9-NEXT:    s_mov_b64 s[36:37], 0
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:  .LBB151_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v4, v0
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_lt_u32_e64 s[34:35], s6, v4
+; GFX9-NEXT:    v_add_u32_e32 v0, -1, v4
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, v0, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap v0, v1, v[3:4], s[4:5] offset:16 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX9-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; GFX9-NEXT:    s_cbranch_execnz .LBB151_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
   %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i32 %in seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
index 778fc2e00c604..483880bd9f4b0 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i64.ll
@@ -623,7 +623,7 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr addrspace(1) %out, i64 %in)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -695,7 +695,7 @@ define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -771,7 +771,7 @@ define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -853,7 +853,7 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -912,7 +912,7 @@ define amdgpu_kernel void @atomic_and_i64(ptr addrspace(1) %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile and ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -983,7 +983,7 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile and ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -1056,7 +1056,7 @@ define amdgpu_kernel void @atomic_and_i64_addr64(ptr addrspace(1) %out, i64 %in,
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile and ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1135,7 +1135,7 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile and ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile and ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -1191,7 +1191,7 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr addrspace(1) %out, i64 %in)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1263,7 +1263,7 @@ define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -1339,7 +1339,7 @@ define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1421,7 +1421,7 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -1480,7 +1480,7 @@ define amdgpu_kernel void @atomic_sub_i64(ptr addrspace(1) %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1551,7 +1551,7 @@ define amdgpu_kernel void @atomic_sub_i64_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -1624,7 +1624,7 @@ define amdgpu_kernel void @atomic_sub_i64_addr64(ptr addrspace(1) %out, i64 %in,
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1703,7 +1703,7 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile sub ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -1753,7 +1753,7 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr addrspace(1) %out, i64 %in)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1822,7 +1822,7 @@ define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -1892,7 +1892,7 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -1971,7 +1971,7 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -2024,7 +2024,7 @@ define amdgpu_kernel void @atomic_max_i64(ptr addrspace(1) %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile max ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2092,7 +2092,7 @@ define amdgpu_kernel void @atomic_max_i64_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile max ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -2159,7 +2159,7 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr addrspace(1) %out, i64 %in,
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile max ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2235,7 +2235,7 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile max ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile max ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -2285,7 +2285,7 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr addrspace(1) %out, i64 %in
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2354,7 +2354,7 @@ define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -2424,7 +2424,7 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr addrspace(1) %out,
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2503,7 +2503,7 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr addrspace(1) %o
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -2556,7 +2556,7 @@ define amdgpu_kernel void @atomic_umax_i64(ptr addrspace(1) %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2624,7 +2624,7 @@ define amdgpu_kernel void @atomic_umax_i64_ret(ptr addrspace(1) %out, ptr addrsp
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -2691,7 +2691,7 @@ define amdgpu_kernel void @atomic_umax_i64_addr64(ptr addrspace(1) %out, i64 %in
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2767,7 +2767,7 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umax ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -2817,7 +2817,7 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr addrspace(1) %out, i64 %in)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -2886,7 +2886,7 @@ define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -2956,7 +2956,7 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3035,7 +3035,7 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -3088,7 +3088,7 @@ define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile min ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3156,7 +3156,7 @@ define amdgpu_kernel void @atomic_min_i64_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile min ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -3223,7 +3223,7 @@ define amdgpu_kernel void @atomic_min_i64_addr64(ptr addrspace(1) %out, i64 %in,
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile min ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3299,7 +3299,7 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile min ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile min ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -3349,7 +3349,7 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr addrspace(1) %out, i64 %in
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3418,7 +3418,7 @@ define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -3488,7 +3488,7 @@ define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr addrspace(1) %out,
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3567,7 +3567,7 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr addrspace(1) %o
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %gep, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -3620,7 +3620,7 @@ define amdgpu_kernel void @atomic_umin_i64(ptr addrspace(1) %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_SE
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3688,7 +3688,7 @@ define amdgpu_kernel void @atomic_umin_i64_ret(ptr addrspace(1) %out, ptr addrsp
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %out, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -3755,7 +3755,7 @@ define amdgpu_kernel void @atomic_umin_i64_addr64(ptr addrspace(1) %out, i64 %in
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3831,7 +3831,7 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst
+  %tmp0 = atomicrmw volatile umin ptr addrspace(1) %ptr, i64 %in syncscope("workgroup") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -3887,7 +3887,7 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr addrspace(1) %out, i64 %in)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -3959,7 +3959,7 @@ define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr addrspace(1) %out, ptr a
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -4035,7 +4035,7 @@ define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr addrspace(1) %out, i6
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4117,7 +4117,7 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr addrspace(1) %out
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -4176,7 +4176,7 @@ define amdgpu_kernel void @atomic_or_i64(ptr addrspace(1) %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile or ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4247,7 +4247,7 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr addrspace(1) %out, ptr addrspac
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile or ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -4320,7 +4320,7 @@ define amdgpu_kernel void @atomic_or_i64_addr64(ptr addrspace(1) %out, i64 %in,
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile or ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4399,7 +4399,7 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr addrspace(1) %out, ptr a
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile or ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile or ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -4455,7 +4455,7 @@ define amdgpu_kernel void @atomic_xchg_i64_offset(ptr addrspace(1) %out, i64 %in
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4510,7 +4510,7 @@ define amdgpu_kernel void @atomic_xchg_f64_offset(ptr addrspace(1) %out, double
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr double, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, double %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, double %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4565,7 +4565,7 @@ define amdgpu_kernel void @atomic_xchg_pointer_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr ptr, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, ptr %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, ptr %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4637,7 +4637,7 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -4713,7 +4713,7 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64_offset(ptr addrspace(1) %out,
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4795,7 +4795,7 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64_offset(ptr addrspace(1) %o
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -4854,7 +4854,7 @@ define amdgpu_kernel void @atomic_xchg_i64(ptr addrspace(1) %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -4925,7 +4925,7 @@ define amdgpu_kernel void @atomic_xchg_i64_ret(ptr addrspace(1) %out, ptr addrsp
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -4998,7 +4998,7 @@ define amdgpu_kernel void @atomic_xchg_i64_addr64(ptr addrspace(1) %out, i64 %in
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -5077,7 +5077,7 @@ define amdgpu_kernel void @atomic_xchg_i64_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xchg ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -5133,7 +5133,7 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr addrspace(1) %out, i64 %in)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -5205,7 +5205,7 @@ define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -5281,7 +5281,7 @@ define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -5363,7 +5363,7 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr addrspace(1) %ou
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -5422,7 +5422,7 @@ define amdgpu_kernel void @atomic_xor_i64(ptr addrspace(1) %out, i64 %in) {
 ; GFX12-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -5493,7 +5493,7 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr addrspace(1) %out, ptr addrspa
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
 ; GFX12-NEXT:    s_endpgm
 entry:
-  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %out, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -5566,7 +5566,7 @@ define amdgpu_kernel void @atomic_xor_i64_addr64(ptr addrspace(1) %out, i64 %in,
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -5645,7 +5645,7 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
-  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile xor ptr addrspace(1) %ptr, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -7146,7 +7146,7 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr addrspace(1) %out, i64 %in)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -7218,7 +7218,7 @@ define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -7294,7 +7294,7 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile uinc_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -7349,7 +7349,7 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr addrspace(1) %out, i64 %in)
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
 
@@ -7421,7 +7421,7 @@ define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr addrspace(1) %out, ptr
 ; GFX12-NEXT:    s_endpgm
 entry:
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
-  %tmp0 = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   store i64 %tmp0, ptr addrspace(1) %out2
   ret void
 }
@@ -7497,6 +7497,8 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr addrspace(1) %out, i
 entry:
   %ptr = getelementptr i64, ptr addrspace(1) %out, i64 %index
   %gep = getelementptr i64, ptr addrspace(1) %ptr, i64 4
-  %tmp0 = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst
+  %tmp0 = atomicrmw volatile udec_wrap ptr addrspace(1) %gep, i64 %in syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret void
 }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
index a7f16449f058e..74f0f64c935b4 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
@@ -1329,26 +1329,76 @@ define void @global_atomic_sub_i64_noret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_sub_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB30_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_sub_i32_e32 v4, vcc, v6, v2
+; SI-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB30_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_sub_i64_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB30_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_sub_u32_e32 v4, vcc, v6, v2
+; VI-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB30_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_sub_x2 v[0:1], v[2:3], off
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB30_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB30_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw sub ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -1362,9 +1412,29 @@ define void @global_atomic_sub_i64_noret_offset(ptr addrspace(1) %out, i64 %in)
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_sub_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB31_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_sub_i32_e32 v4, vcc, v6, v2
+; SI-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB31_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1373,17 +1443,47 @@ define void @global_atomic_sub_i64_noret_offset(ptr addrspace(1) %out, i64 %in)
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB31_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_sub_u32_e32 v4, vcc, v6, v2
+; VI-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB31_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_sub_x2 v[0:1], v[2:3], off offset:32
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB31_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB31_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst
@@ -1394,32 +1494,88 @@ define i64 @global_atomic_sub_i64_ret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-LABEL: global_atomic_sub_i64_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_sub_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB32_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    v_sub_i32_e32 v8, vcc, v10, v7
+; SI-NEXT:    v_subb_u32_e32 v9, vcc, v11, v6, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB32_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_sub_i64_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB32_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    v_sub_u32_e32 v4, vcc, v6, v2
+; VI-NEXT:    v_subb_u32_e32 v5, vcc, v7, v3, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB32_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v4
+; VI-NEXT:    v_mov_b32_e32 v1, v5
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_sub_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB32_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB32_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw sub ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -1429,34 +1585,88 @@ define i64 @global_atomic_sub_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {
 ; SI-LABEL: global_atomic_sub_i64_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_sub_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB33_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    v_sub_i32_e32 v8, vcc, v10, v7
+; SI-NEXT:    v_subb_u32_e32 v9, vcc, v11, v6, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB33_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_sub_i64_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_sub_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB33_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v9, v1
+; VI-NEXT:    v_mov_b32_e32 v8, v0
+; VI-NEXT:    v_sub_u32_e32 v6, vcc, v8, v2
+; VI-NEXT:    v_subb_u32_e32 v7, vcc, v9, v3, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB33_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_sub_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB33_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_sub_co_u32_e32 v4, vcc, v6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB33_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst
@@ -1468,25 +1678,43 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_scalar(ptr addrspace(1) inre
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    s_mov_b32 s34, s7
-; SI-NEXT:    s_mov_b32 s35, s6
+; SI-NEXT:    v_writelane_b32 v9, s6, 0
+; SI-NEXT:    v_writelane_b32 v9, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:    v_mov_b32_e32 v4, s35
+; SI-NEXT:  .LBB34_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s34, v2
+; SI-NEXT:    v_subb_u32_e32 v1, vcc, v3, v4, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v8, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v6, v1
+; SI-NEXT:    v_mov_b32_e32 v5, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[5:6], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v5
+; SI-NEXT:    v_mov_b32_e32 v3, v6
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB34_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v9, 1
+; SI-NEXT:    v_readlane_b32 s6, v9, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1494,24 +1722,54 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_scalar(ptr addrspace(1) inre
 ; VI-LABEL: global_atomic_sub_i64_noret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v2, s4
-; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v6, s7
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:  .LBB34_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_subrev_u32_e32 v0, vcc, s6, v2
+; VI-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB34_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_sub_x2 v2, v[0:1], s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:  .LBB34_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, s6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB34_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw sub ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -1522,23 +1780,43 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_offset_scalar(ptr addrspace(
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v9, s6, 0
+; SI-NEXT:    v_writelane_b32 v9, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:    v_mov_b32_e32 v4, s35
+; SI-NEXT:  .LBB35_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    v_subrev_i32_e32 v0, vcc, s34, v2
+; SI-NEXT:    v_subb_u32_e32 v1, vcc, v3, v4, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v8, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v6, v1
+; SI-NEXT:    v_mov_b32_e32 v5, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[5:6], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v5
+; SI-NEXT:    v_mov_b32_e32 v3, v6
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB35_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v9, 1
+; SI-NEXT:    v_readlane_b32 s6, v9, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1548,24 +1826,52 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_sub_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s34
+; VI-NEXT:    v_mov_b32_e32 v5, s35
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v6, s7
+; VI-NEXT:  .LBB35_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_subrev_u32_e32 v0, vcc, s6, v2
+; VI-NEXT:    v_subb_u32_e32 v1, vcc, v3, v6, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB35_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_sub_x2 v2, v[0:1], s[4:5] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:  .LBB35_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_subrev_co_u32_e32 v0, vcc, s6, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB35_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst
@@ -1577,25 +1883,43 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_scalar(ptr addrspace(1) inreg %
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    s_mov_b32 s34, s7
-; SI-NEXT:    s_mov_b32 s35, s6
+; SI-NEXT:    v_writelane_b32 v9, s6, 0
+; SI-NEXT:    v_writelane_b32 v9, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:    v_mov_b32_e32 v4, s35
+; SI-NEXT:  .LBB36_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v8, v1
+; SI-NEXT:    v_mov_b32_e32 v7, v0
+; SI-NEXT:    v_subrev_i32_e32 v5, vcc, s34, v7
+; SI-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v5
+; SI-NEXT:    v_mov_b32_e32 v1, v6
+; SI-NEXT:    v_mov_b32_e32 v2, v7
+; SI-NEXT:    v_mov_b32_e32 v3, v8
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB36_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v9, 1
+; SI-NEXT:    v_readlane_b32 s6, v9, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1603,24 +1927,54 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_scalar(ptr addrspace(1) inreg %
 ; VI-LABEL: global_atomic_sub_i64_ret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v4, s7
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:  .LBB36_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v8, v1
+; VI-NEXT:    v_mov_b32_e32 v7, v0
+; VI-NEXT:    v_subrev_u32_e32 v5, vcc, s6, v7
+; VI-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB36_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_sub_x2 v[0:1], v2, v[0:1], s[4:5] glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:  .LBB36_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_subrev_co_u32_e32 v4, vcc, s6, v6
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[4:7], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB36_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw sub ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -1631,23 +1985,43 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_offset_scalar(ptr addrspace(1)
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v9, s6, 0
+; SI-NEXT:    v_writelane_b32 v9, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:    v_mov_b32_e32 v4, s35
+; SI-NEXT:  .LBB37_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_sub_x2 v[0:1], off, s[4:7], 0 offset:32 glc
+; SI-NEXT:    v_mov_b32_e32 v8, v1
+; SI-NEXT:    v_mov_b32_e32 v7, v0
+; SI-NEXT:    v_subrev_i32_e32 v5, vcc, s34, v7
+; SI-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v5
+; SI-NEXT:    v_mov_b32_e32 v1, v6
+; SI-NEXT:    v_mov_b32_e32 v2, v7
+; SI-NEXT:    v_mov_b32_e32 v3, v8
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB37_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v9, 1
+; SI-NEXT:    v_readlane_b32 s6, v9, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1658,23 +2032,51 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_sub_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v4, s7
+; VI-NEXT:  .LBB37_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v8, v1
+; VI-NEXT:    v_mov_b32_e32 v7, v0
+; VI-NEXT:    v_subrev_u32_e32 v5, vcc, s6, v7
+; VI-NEXT:    v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB37_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_sub_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_sub_x2 v[0:1], v2, v[0:1], s[4:5] offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:  .LBB37_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v0
+; GFX9-NEXT:    v_subrev_co_u32_e32 v4, vcc, s6, v6
+; GFX9-NEXT:    v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[4:7], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB37_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw sub ptr addrspace(1) %gep, i64 %in seq_cst
@@ -1767,26 +2169,76 @@ define void @global_atomic_and_i64_noret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_and_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB40_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v5, v7, v3
+; SI-NEXT:    v_and_b32_e32 v4, v6, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB40_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_and_i64_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB40_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v5, v7, v3
+; VI-NEXT:    v_and_b32_e32 v4, v6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB40_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_and_x2 v[0:1], v[2:3], off
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB40_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw and ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -1800,9 +2252,29 @@ define void @global_atomic_and_i64_noret_offset(ptr addrspace(1) %out, i64 %in)
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_and_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB41_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_and_b32_e32 v5, v7, v3
+; SI-NEXT:    v_and_b32_e32 v4, v6, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB41_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1811,17 +2283,47 @@ define void @global_atomic_and_i64_noret_offset(ptr addrspace(1) %out, i64 %in)
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB41_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v5, v7, v3
+; VI-NEXT:    v_and_b32_e32 v4, v6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB41_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_and_x2 v[0:1], v[2:3], off offset:32
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB41_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst
@@ -1832,32 +2334,88 @@ define i64 @global_atomic_and_i64_ret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-LABEL: global_atomic_and_i64_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_and_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB42_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    v_and_b32_e32 v9, v11, v6
+; SI-NEXT:    v_and_b32_e32 v8, v10, v7
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB42_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_and_i64_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB42_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    v_and_b32_e32 v5, v7, v3
+; VI-NEXT:    v_and_b32_e32 v4, v6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB42_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v4
+; VI-NEXT:    v_mov_b32_e32 v1, v5
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_and_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB42_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw and ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -1867,34 +2425,88 @@ define i64 @global_atomic_and_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {
 ; SI-LABEL: global_atomic_and_i64_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_and_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB43_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    v_and_b32_e32 v9, v11, v6
+; SI-NEXT:    v_and_b32_e32 v8, v10, v7
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB43_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_and_i64_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_and_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB43_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v9, v1
+; VI-NEXT:    v_mov_b32_e32 v8, v0
+; VI-NEXT:    v_and_b32_e32 v7, v9, v3
+; VI-NEXT:    v_and_b32_e32 v6, v8, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB43_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_and_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_and_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_and_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB43_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst
@@ -1906,25 +2518,42 @@ define amdgpu_gfx void @global_atomic_and_i64_noret_scalar(ptr addrspace(1) inre
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
+; SI-NEXT:    v_writelane_b32 v8, s6, 0
+; SI-NEXT:    v_writelane_b32 v8, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s7
 ; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB44_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_and_x2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    v_and_b32_e32 v1, s34, v3
+; SI-NEXT:    v_and_b32_e32 v0, s35, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v7, v3
+; SI-NEXT:    v_mov_b32_e32 v6, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB44_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v8, 1
+; SI-NEXT:    v_readlane_b32 s6, v8, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1932,24 +2561,52 @@ define amdgpu_gfx void @global_atomic_and_i64_noret_scalar(ptr addrspace(1) inre
 ; VI-LABEL: global_atomic_and_i64_noret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v2, s4
-; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:  .LBB44_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v1, s7, v3
+; VI-NEXT:    v_and_b32_e32 v0, s6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB44_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_and_x2 v2, v[0:1], s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_and_b32_e32 v0, s6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB44_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw and ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -1960,23 +2617,42 @@ define amdgpu_gfx void @global_atomic_and_i64_noret_offset_scalar(ptr addrspace(
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v8, s6, 0
+; SI-NEXT:    v_writelane_b32 v8, s7, 1
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB45_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_and_x2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    v_and_b32_e32 v1, s34, v3
+; SI-NEXT:    v_and_b32_e32 v0, s35, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v7, v3
+; SI-NEXT:    v_mov_b32_e32 v6, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB45_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v8, 1
+; SI-NEXT:    v_readlane_b32 s6, v8, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -1986,24 +2662,50 @@ define amdgpu_gfx void @global_atomic_and_i64_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_and_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s34
+; VI-NEXT:    v_mov_b32_e32 v5, s35
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB45_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_and_b32_e32 v1, s7, v3
+; VI-NEXT:    v_and_b32_e32 v0, s6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB45_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_and_x2 v2, v[0:1], s[4:5] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_and_b32_e32 v0, s6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB45_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst
@@ -2015,25 +2717,42 @@ define amdgpu_gfx i64 @global_atomic_and_i64_ret_scalar(ptr addrspace(1) inreg %
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
+; SI-NEXT:    v_writelane_b32 v6, s6, 0
+; SI-NEXT:    v_writelane_b32 v6, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s7
 ; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB46_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_and_x2 v[0:1], off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v3, s34, v5
+; SI-NEXT:    v_and_b32_e32 v2, s35, v4
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB46_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v6, 1
+; SI-NEXT:    v_readlane_b32 s6, v6, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -2041,24 +2760,52 @@ define amdgpu_gfx i64 @global_atomic_and_i64_ret_scalar(ptr addrspace(1) inreg %
 ; VI-LABEL: global_atomic_and_i64_ret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:  .LBB46_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v1
+; VI-NEXT:    v_mov_b32_e32 v6, v0
+; VI-NEXT:    v_and_b32_e32 v5, s7, v7
+; VI-NEXT:    v_and_b32_e32 v4, s6, v6
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    s_setpc_b64 s[30:31]
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB46_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_and_x2 v[0:1], v2, v[0:1], s[4:5] glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v6, v1
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_and_b32_e32 v4, s7, v6
+; GFX9-NEXT:    v_and_b32_e32 v3, s6, v5
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB46_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw and ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -2069,23 +2816,42 @@ define amdgpu_gfx i64 @global_atomic_and_i64_ret_offset_scalar(ptr addrspace(1)
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v6, s6, 0
+; SI-NEXT:    v_writelane_b32 v6, s7, 1
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB47_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_and_x2 v[0:1], off, s[4:7], 0 offset:32 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_and_b32_e32 v3, s34, v5
+; SI-NEXT:    v_and_b32_e32 v2, s35, v4
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB47_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v6, 1
+; SI-NEXT:    v_readlane_b32 s6, v6, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -2096,23 +2862,49 @@ define amdgpu_gfx i64 @global_atomic_and_i64_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_and_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB47_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v1
+; VI-NEXT:    v_mov_b32_e32 v6, v0
+; VI-NEXT:    v_and_b32_e32 v5, s7, v7
+; VI-NEXT:    v_and_b32_e32 v4, s6, v6
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB47_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_and_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_and_x2 v[0:1], v2, v[0:1], s[4:5] offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v6, v1
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_and_b32_e32 v4, s7, v6
+; GFX9-NEXT:    v_and_b32_e32 v3, s6, v5
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB47_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw and ptr addrspace(1) %gep, i64 %in seq_cst
@@ -3197,26 +3989,76 @@ define void @global_atomic_or_i64_noret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_or_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB60_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v5, v7, v3
+; SI-NEXT:    v_or_b32_e32 v4, v6, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB60_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_or_i64_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB60_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v5, v7, v3
+; VI-NEXT:    v_or_b32_e32 v4, v6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB60_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_or_x2 v[0:1], v[2:3], off
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB60_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB60_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw or ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -3230,9 +4072,29 @@ define void @global_atomic_or_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_or_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB61_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v5, v7, v3
+; SI-NEXT:    v_or_b32_e32 v4, v6, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB61_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3241,17 +4103,47 @@ define void @global_atomic_or_i64_noret_offset(ptr addrspace(1) %out, i64 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB61_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v5, v7, v3
+; VI-NEXT:    v_or_b32_e32 v4, v6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB61_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_or_x2 v[0:1], v[2:3], off offset:32
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB61_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB61_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst
@@ -3262,32 +4154,88 @@ define i64 @global_atomic_or_i64_ret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-LABEL: global_atomic_or_i64_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_or_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB62_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    v_or_b32_e32 v9, v11, v6
+; SI-NEXT:    v_or_b32_e32 v8, v10, v7
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB62_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_or_i64_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB62_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    v_or_b32_e32 v5, v7, v3
+; VI-NEXT:    v_or_b32_e32 v4, v6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB62_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v4
+; VI-NEXT:    v_mov_b32_e32 v1, v5
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_or_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB62_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB62_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw or ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -3297,34 +4245,88 @@ define i64 @global_atomic_or_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {
 ; SI-LABEL: global_atomic_or_i64_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_or_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB63_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    v_or_b32_e32 v9, v11, v6
+; SI-NEXT:    v_or_b32_e32 v8, v10, v7
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB63_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_or_i64_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_or_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB63_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v9, v1
+; VI-NEXT:    v_mov_b32_e32 v8, v0
+; VI-NEXT:    v_or_b32_e32 v7, v9, v3
+; VI-NEXT:    v_or_b32_e32 v6, v8, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    s_setpc_b64 s[30:31]
-;
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB63_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GFX9-LABEL: global_atomic_or_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_or_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB63_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_or_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_or_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB63_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst
@@ -3336,25 +4338,42 @@ define amdgpu_gfx void @global_atomic_or_i64_noret_scalar(ptr addrspace(1) inreg
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
+; SI-NEXT:    v_writelane_b32 v8, s6, 0
+; SI-NEXT:    v_writelane_b32 v8, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s7
 ; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB64_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_or_x2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    v_or_b32_e32 v1, s34, v3
+; SI-NEXT:    v_or_b32_e32 v0, s35, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v7, v3
+; SI-NEXT:    v_mov_b32_e32 v6, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB64_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v8, 1
+; SI-NEXT:    v_readlane_b32 s6, v8, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3362,24 +4381,52 @@ define amdgpu_gfx void @global_atomic_or_i64_noret_scalar(ptr addrspace(1) inreg
 ; VI-LABEL: global_atomic_or_i64_noret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v2, s4
-; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:  .LBB64_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v1, s7, v3
+; VI-NEXT:    v_or_b32_e32 v0, s6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB64_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_or_x2 v2, v[0:1], s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB64_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_or_b32_e32 v0, s6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB64_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw or ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -3390,23 +4437,42 @@ define amdgpu_gfx void @global_atomic_or_i64_noret_offset_scalar(ptr addrspace(1
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v8, s6, 0
+; SI-NEXT:    v_writelane_b32 v8, s7, 1
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB65_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_or_x2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    v_or_b32_e32 v1, s34, v3
+; SI-NEXT:    v_or_b32_e32 v0, s35, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v7, v3
+; SI-NEXT:    v_mov_b32_e32 v6, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB65_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v8, 1
+; SI-NEXT:    v_readlane_b32 s6, v8, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3416,24 +4482,50 @@ define amdgpu_gfx void @global_atomic_or_i64_noret_offset_scalar(ptr addrspace(1
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_or_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s34
+; VI-NEXT:    v_mov_b32_e32 v5, s35
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB65_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_or_b32_e32 v1, s7, v3
+; VI-NEXT:    v_or_b32_e32 v0, s6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB65_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_or_x2 v2, v[0:1], s[4:5] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB65_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_or_b32_e32 v0, s6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB65_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst
@@ -3445,25 +4537,42 @@ define amdgpu_gfx i64 @global_atomic_or_i64_ret_scalar(ptr addrspace(1) inreg %p
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
+; SI-NEXT:    v_writelane_b32 v6, s6, 0
+; SI-NEXT:    v_writelane_b32 v6, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s7
 ; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB66_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_or_x2 v[0:1], off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_or_b32_e32 v3, s34, v5
+; SI-NEXT:    v_or_b32_e32 v2, s35, v4
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB66_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v6, 1
+; SI-NEXT:    v_readlane_b32 s6, v6, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3471,24 +4580,52 @@ define amdgpu_gfx i64 @global_atomic_or_i64_ret_scalar(ptr addrspace(1) inreg %p
 ; VI-LABEL: global_atomic_or_i64_ret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:  .LBB66_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v1
+; VI-NEXT:    v_mov_b32_e32 v6, v0
+; VI-NEXT:    v_or_b32_e32 v5, s7, v7
+; VI-NEXT:    v_or_b32_e32 v4, s6, v6
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB66_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_or_x2 v[0:1], v2, v[0:1], s[4:5] glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB66_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v6, v1
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_or_b32_e32 v4, s7, v6
+; GFX9-NEXT:    v_or_b32_e32 v3, s6, v5
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB66_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw or ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -3499,23 +4636,42 @@ define amdgpu_gfx i64 @global_atomic_or_i64_ret_offset_scalar(ptr addrspace(1) i
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v6, s6, 0
+; SI-NEXT:    v_writelane_b32 v6, s7, 1
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB67_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_or_x2 v[0:1], off, s[4:7], 0 offset:32 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_or_b32_e32 v3, s34, v5
+; SI-NEXT:    v_or_b32_e32 v2, s35, v4
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB67_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v6, 1
+; SI-NEXT:    v_readlane_b32 s6, v6, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3526,23 +4682,49 @@ define amdgpu_gfx i64 @global_atomic_or_i64_ret_offset_scalar(ptr addrspace(1) i
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_or_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB67_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v1
+; VI-NEXT:    v_mov_b32_e32 v6, v0
+; VI-NEXT:    v_or_b32_e32 v5, s7, v7
+; VI-NEXT:    v_or_b32_e32 v4, s6, v6
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    s_setpc_b64 s[30:31]
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB67_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_or_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_or_x2 v[0:1], v2, v[0:1], s[4:5] offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB67_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v6, v1
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_or_b32_e32 v4, s7, v6
+; GFX9-NEXT:    v_or_b32_e32 v3, s6, v5
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB67_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw or ptr addrspace(1) %gep, i64 %in seq_cst
@@ -3635,26 +4817,76 @@ define void @global_atomic_xor_i64_noret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_xor_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB70_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v5, v7, v3
+; SI-NEXT:    v_xor_b32_e32 v4, v6, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB70_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_xor_i64_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB70_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v5, v7, v3
+; VI-NEXT:    v_xor_b32_e32 v4, v6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB70_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_xor_x2 v[0:1], v[2:3], off
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB70_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB70_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw xor ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -3668,9 +4900,29 @@ define void @global_atomic_xor_i64_noret_offset(ptr addrspace(1) %out, i64 %in)
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_xor_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB71_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v5, v7, v3
+; SI-NEXT:    v_xor_b32_e32 v4, v6, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB71_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -3679,17 +4931,47 @@ define void @global_atomic_xor_i64_noret_offset(ptr addrspace(1) %out, i64 %in)
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB71_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v5, v7, v3
+; VI-NEXT:    v_xor_b32_e32 v4, v6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB71_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_xor_x2 v[0:1], v[2:3], off offset:32
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB71_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB71_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst
@@ -3700,32 +4982,88 @@ define i64 @global_atomic_xor_i64_ret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-LABEL: global_atomic_xor_i64_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_xor_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB72_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    v_xor_b32_e32 v9, v11, v6
+; SI-NEXT:    v_xor_b32_e32 v8, v10, v7
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB72_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_xor_i64_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB72_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    v_xor_b32_e32 v5, v7, v3
+; VI-NEXT:    v_xor_b32_e32 v4, v6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB72_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v4
+; VI-NEXT:    v_mov_b32_e32 v1, v5
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_xor_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB72_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB72_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw xor ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -3735,34 +5073,88 @@ define i64 @global_atomic_xor_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {
 ; SI-LABEL: global_atomic_xor_i64_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_xor_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB73_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    v_xor_b32_e32 v9, v11, v6
+; SI-NEXT:    v_xor_b32_e32 v8, v10, v7
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB73_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_xor_i64_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_xor_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB73_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v9, v1
+; VI-NEXT:    v_mov_b32_e32 v8, v0
+; VI-NEXT:    v_xor_b32_e32 v7, v9, v3
+; VI-NEXT:    v_xor_b32_e32 v6, v8, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB73_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_xor_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB73_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_xor_b32_e32 v5, v7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v4, v6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB73_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst
@@ -3774,25 +5166,42 @@ define amdgpu_gfx void @global_atomic_xor_i64_noret_scalar(ptr addrspace(1) inre
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
+; SI-NEXT:    v_writelane_b32 v8, s6, 0
+; SI-NEXT:    v_writelane_b32 v8, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s7
 ; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB74_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_xor_x2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    v_xor_b32_e32 v1, s34, v3
+; SI-NEXT:    v_xor_b32_e32 v0, s35, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v7, v3
+; SI-NEXT:    v_mov_b32_e32 v6, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB74_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v8, 1
+; SI-NEXT:    v_readlane_b32 s6, v8, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3800,24 +5209,52 @@ define amdgpu_gfx void @global_atomic_xor_i64_noret_scalar(ptr addrspace(1) inre
 ; VI-LABEL: global_atomic_xor_i64_noret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v2, s4
-; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:  .LBB74_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_xor_b32_e32 v1, s7, v3
+; VI-NEXT:    v_xor_b32_e32 v0, s6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB74_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_xor_x2 v2, v[0:1], s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB74_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw xor ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -3828,23 +5265,42 @@ define amdgpu_gfx void @global_atomic_xor_i64_noret_offset_scalar(ptr addrspace(
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v8, s6, 0
+; SI-NEXT:    v_writelane_b32 v8, s7, 1
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB75_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_xor_x2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    v_xor_b32_e32 v1, s34, v3
+; SI-NEXT:    v_xor_b32_e32 v0, s35, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v7, v3
+; SI-NEXT:    v_mov_b32_e32 v6, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB75_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v8, 1
+; SI-NEXT:    v_readlane_b32 s6, v8, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3854,24 +5310,50 @@ define amdgpu_gfx void @global_atomic_xor_i64_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_xor_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s34
+; VI-NEXT:    v_mov_b32_e32 v5, s35
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB75_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_xor_b32_e32 v1, s7, v3
+; VI-NEXT:    v_xor_b32_e32 v0, s6, v2
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB75_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_xor_x2 v2, v[0:1], s[4:5] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, s7, v3
+; GFX9-NEXT:    v_xor_b32_e32 v0, s6, v2
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB75_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst
@@ -3883,25 +5365,42 @@ define amdgpu_gfx i64 @global_atomic_xor_i64_ret_scalar(ptr addrspace(1) inreg %
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
+; SI-NEXT:    v_writelane_b32 v6, s6, 0
+; SI-NEXT:    v_writelane_b32 v6, s7, 1
 ; SI-NEXT:    s_mov_b32 s34, s7
 ; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB76_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_xor_x2 v[0:1], off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v3, s34, v5
+; SI-NEXT:    v_xor_b32_e32 v2, s35, v4
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB76_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v6, 1
+; SI-NEXT:    v_readlane_b32 s6, v6, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3909,24 +5408,52 @@ define amdgpu_gfx i64 @global_atomic_xor_i64_ret_scalar(ptr addrspace(1) inreg %
 ; VI-LABEL: global_atomic_xor_i64_ret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:  .LBB76_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v1
+; VI-NEXT:    v_mov_b32_e32 v6, v0
+; VI-NEXT:    v_xor_b32_e32 v5, s7, v7
+; VI-NEXT:    v_xor_b32_e32 v4, s6, v6
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB76_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_xor_x2 v[0:1], v2, v[0:1], s[4:5] glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v6, v1
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_xor_b32_e32 v4, s7, v6
+; GFX9-NEXT:    v_xor_b32_e32 v3, s6, v5
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB76_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw xor ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -3937,23 +5464,42 @@ define amdgpu_gfx i64 @global_atomic_xor_i64_ret_offset_scalar(ptr addrspace(1)
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v6, s6, 0
+; SI-NEXT:    v_writelane_b32 v6, s7, 1
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB77_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_xor_x2 v[0:1], off, s[4:7], 0 offset:32 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_xor_b32_e32 v3, s34, v5
+; SI-NEXT:    v_xor_b32_e32 v2, s35, v4
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB77_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v6, 1
+; SI-NEXT:    v_readlane_b32 s6, v6, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -3964,23 +5510,49 @@ define amdgpu_gfx i64 @global_atomic_xor_i64_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_xor_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB77_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v1
+; VI-NEXT:    v_mov_b32_e32 v6, v0
+; VI-NEXT:    v_xor_b32_e32 v5, s7, v7
+; VI-NEXT:    v_xor_b32_e32 v4, s6, v6
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB77_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_xor_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_xor_x2 v[0:1], v2, v[0:1], s[4:5] offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v6, v1
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_xor_b32_e32 v4, s7, v6
+; GFX9-NEXT:    v_xor_b32_e32 v3, s6, v5
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB77_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw xor ptr addrspace(1) %gep, i64 %in seq_cst
@@ -5347,30 +6919,9 @@ define void @global_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr add
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:32
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB92_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; SI-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v11, v7
-; SI-NEXT:    v_mov_b32_e32 v10, v6
-; SI-NEXT:    v_mov_b32_e32 v9, v5
-; SI-NEXT:    v_mov_b32_e32 v8, v4
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_atomic_smax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v6, v8
-; SI-NEXT:    v_mov_b32_e32 v7, v9
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB92_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -5379,49 +6930,17 @@ define void @global_atomic_max_i64_noret_offset__amdgpu_no_remote_memory(ptr add
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB92_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; VI-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; VI-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; VI-NEXT:    flat_atomic_smax_x2 v[0:1], v[2:3]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; VI-NEXT:    v_mov_b32_e32 v7, v5
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    v_mov_b32_e32 v6, v4
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB92_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_max_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB92_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
+; GFX9-NEXT:    global_atomic_smax_x2 v[0:1], v[2:3], off offset:32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB92_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -5432,91 +6951,34 @@ define i64 @global_atomic_max_i64_ret_offset__amdgpu_no_remote_memory(ptr addrsp
 ; SI-LABEL: global_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    v_mov_b32_e32 v4, v2
-; SI-NEXT:    v_mov_b32_e32 v7, v1
-; SI-NEXT:    v_mov_b32_e32 v6, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:32
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB93_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v11, v1
-; SI-NEXT:    v_mov_b32_e32 v10, v0
-; SI-NEXT:    v_cmp_gt_i64_e32 vcc, v[10:11], v[4:5]
-; SI-NEXT:    v_cndmask_b32_e32 v9, v5, v11, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v8, v4, v10, vcc
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, v8
-; SI-NEXT:    v_mov_b32_e32 v1, v9
-; SI-NEXT:    v_mov_b32_e32 v2, v10
-; SI-NEXT:    v_mov_b32_e32 v3, v11
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_atomic_smax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB93_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB93_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v9, v1
-; VI-NEXT:    v_mov_b32_e32 v8, v0
-; VI-NEXT:    v_cmp_gt_i64_e32 vcc, v[8:9], v[2:3]
-; VI-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; VI-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_atomic_smax_x2 v[0:1], v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB93_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_max_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB93_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
+; GFX9-NEXT:    global_atomic_smax_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB93_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw max ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -6697,30 +8159,9 @@ define void @global_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr ad
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:32
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB105_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; SI-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v11, v7
-; SI-NEXT:    v_mov_b32_e32 v10, v6
-; SI-NEXT:    v_mov_b32_e32 v9, v5
-; SI-NEXT:    v_mov_b32_e32 v8, v4
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_atomic_umax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v6, v8
-; SI-NEXT:    v_mov_b32_e32 v7, v9
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB105_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -6729,49 +8170,17 @@ define void @global_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory(ptr ad
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB105_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; VI-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; VI-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; VI-NEXT:    flat_atomic_umax_x2 v[0:1], v[2:3]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; VI-NEXT:    v_mov_b32_e32 v7, v5
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    v_mov_b32_e32 v6, v4
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB105_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_umax_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB105_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
+; GFX9-NEXT:    global_atomic_umax_x2 v[0:1], v[2:3], off offset:32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB105_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -6782,91 +8191,34 @@ define i64 @global_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory(ptr addrs
 ; SI-LABEL: global_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    v_mov_b32_e32 v4, v2
-; SI-NEXT:    v_mov_b32_e32 v7, v1
-; SI-NEXT:    v_mov_b32_e32 v6, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:32
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB106_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v11, v1
-; SI-NEXT:    v_mov_b32_e32 v10, v0
-; SI-NEXT:    v_cmp_gt_u64_e32 vcc, v[10:11], v[4:5]
-; SI-NEXT:    v_cndmask_b32_e32 v9, v5, v11, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v8, v4, v10, vcc
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, v8
-; SI-NEXT:    v_mov_b32_e32 v1, v9
-; SI-NEXT:    v_mov_b32_e32 v2, v10
-; SI-NEXT:    v_mov_b32_e32 v3, v11
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_atomic_umax_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB106_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB106_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v9, v1
-; VI-NEXT:    v_mov_b32_e32 v8, v0
-; VI-NEXT:    v_cmp_gt_u64_e32 vcc, v[8:9], v[2:3]
-; VI-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; VI-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_atomic_umax_x2 v[0:1], v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB106_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_umax_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB106_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
+; GFX9-NEXT:    global_atomic_umax_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB106_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw umax ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -7683,30 +9035,9 @@ define void @global_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr ad
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:32
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB115_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; SI-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v11, v7
-; SI-NEXT:    v_mov_b32_e32 v10, v6
-; SI-NEXT:    v_mov_b32_e32 v9, v5
-; SI-NEXT:    v_mov_b32_e32 v8, v4
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_atomic_umin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v6, v8
-; SI-NEXT:    v_mov_b32_e32 v7, v9
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB115_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -7715,49 +9046,17 @@ define void @global_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory(ptr ad
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB115_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; VI-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; VI-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; VI-NEXT:    flat_atomic_umin_x2 v[0:1], v[2:3]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; VI-NEXT:    v_mov_b32_e32 v7, v5
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    v_mov_b32_e32 v6, v4
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB115_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_umin_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB115_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
+; GFX9-NEXT:    global_atomic_umin_x2 v[0:1], v[2:3], off offset:32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB115_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw umin ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -7768,91 +9067,34 @@ define i64 @global_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory(ptr addrs
 ; SI-LABEL: global_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    v_mov_b32_e32 v4, v2
-; SI-NEXT:    v_mov_b32_e32 v7, v1
-; SI-NEXT:    v_mov_b32_e32 v6, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:32
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB116_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v11, v1
-; SI-NEXT:    v_mov_b32_e32 v10, v0
-; SI-NEXT:    v_cmp_le_u64_e32 vcc, v[10:11], v[4:5]
-; SI-NEXT:    v_cndmask_b32_e32 v9, v5, v11, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v8, v4, v10, vcc
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, v8
-; SI-NEXT:    v_mov_b32_e32 v1, v9
-; SI-NEXT:    v_mov_b32_e32 v2, v10
-; SI-NEXT:    v_mov_b32_e32 v3, v11
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_atomic_umin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB116_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB116_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v9, v1
-; VI-NEXT:    v_mov_b32_e32 v8, v0
-; VI-NEXT:    v_cmp_le_u64_e32 vcc, v[8:9], v[2:3]
-; VI-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; VI-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_atomic_umin_x2 v[0:1], v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB116_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_umin_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB116_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    v_cmp_le_u64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
+; GFX9-NEXT:    global_atomic_umin_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB116_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw umin ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -9134,30 +10376,9 @@ define void @global_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr add
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:32
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB129_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; SI-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v11, v7
-; SI-NEXT:    v_mov_b32_e32 v10, v6
-; SI-NEXT:    v_mov_b32_e32 v9, v5
-; SI-NEXT:    v_mov_b32_e32 v8, v4
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_atomic_smin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    v_mov_b32_e32 v6, v8
-; SI-NEXT:    v_mov_b32_e32 v7, v9
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB129_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -9166,49 +10387,17 @@ define void @global_atomic_min_i64_noret_offset__amdgpu_no_remote_memory(ptr add
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB129_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; VI-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; VI-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; VI-NEXT:    flat_atomic_smin_x2 v[0:1], v[2:3]
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; VI-NEXT:    v_mov_b32_e32 v7, v5
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    v_mov_b32_e32 v6, v4
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB129_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_min_i64_noret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB129_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
+; GFX9-NEXT:    global_atomic_smin_x2 v[0:1], v[2:3], off offset:32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB129_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -9219,91 +10408,34 @@ define i64 @global_atomic_min_i64_ret_offset__amdgpu_no_remote_memory(ptr addrsp
 ; SI-LABEL: global_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v5, v3
-; SI-NEXT:    v_mov_b32_e32 v4, v2
-; SI-NEXT:    v_mov_b32_e32 v7, v1
-; SI-NEXT:    v_mov_b32_e32 v6, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:32
-; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB130_1: ; %atomicrmw.start
-; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v11, v1
-; SI-NEXT:    v_mov_b32_e32 v10, v0
-; SI-NEXT:    v_cmp_le_i64_e32 vcc, v[10:11], v[4:5]
-; SI-NEXT:    v_cndmask_b32_e32 v9, v5, v11, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v8, v4, v10, vcc
-; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, v8
-; SI-NEXT:    v_mov_b32_e32 v1, v9
-; SI-NEXT:    v_mov_b32_e32 v2, v10
-; SI-NEXT:    v_mov_b32_e32 v3, v11
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_atomic_smin_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
-; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB130_1
-; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
-; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB130_1: ; %atomicrmw.start
-; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v9, v1
-; VI-NEXT:    v_mov_b32_e32 v8, v0
-; VI-NEXT:    v_cmp_le_i64_e32 vcc, v[8:9], v[2:3]
-; VI-NEXT:    v_cndmask_b32_e32 v7, v3, v9, vcc
-; VI-NEXT:    v_cndmask_b32_e32 v6, v2, v8, vcc
-; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-NEXT:    flat_atomic_smin_x2 v[0:1], v[0:1], v[2:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
-; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB130_1
-; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
-; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_min_i64_ret_offset__amdgpu_no_remote_memory:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
-; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB130_1: ; %atomicrmw.start
-; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v7, v5
-; GFX9-NEXT:    v_mov_b32_e32 v6, v4
-; GFX9-NEXT:    v_cmp_le_i64_e32 vcc, v[6:7], v[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e32 v5, v3, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e32 v4, v2, v6, vcc
-; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
+; GFX9-NEXT:    global_atomic_smin_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
-; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB130_1
-; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw min ptr addrspace(1) %gep, i64 %in seq_cst, !amdgpu.no.remote.memory !0
@@ -9322,26 +10454,85 @@ define void @global_atomic_uinc_wrap_i64_noret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_inc_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB131_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 1, v6
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; SI-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB131_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_uinc_wrap_i64_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB131_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 1, v6
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; VI-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; VI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB131_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_inc_x2 v[0:1], v[2:3], off
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB131_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB131_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -9355,9 +10546,32 @@ define void @global_atomic_uinc_wrap_i64_noret_offset(ptr addrspace(1) %out, i64
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_inc_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB132_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, 1, v6
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; SI-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB132_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -9366,17 +10580,53 @@ define void @global_atomic_uinc_wrap_i64_noret_offset(ptr addrspace(1) %out, i64
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB132_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 1, v6
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; VI-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; VI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB132_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_inc_x2 v[0:1], v[2:3], off offset:32
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB132_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB132_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst
@@ -9387,32 +10637,97 @@ define i64 @global_atomic_uinc_wrap_i64_ret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-LABEL: global_atomic_uinc_wrap_i64_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    v_mov_b32_e32 v4, v2
+; SI-NEXT:    v_mov_b32_e32 v7, v1
+; SI-NEXT:    v_mov_b32_e32 v6, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_inc_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB133_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v10
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v11, vcc
+; SI-NEXT:    v_cmp_lt_u64_e32 vcc, v[10:11], v[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v9, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v8, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB133_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_uinc_wrap_i64_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB133_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 1, v6
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v7, vcc
+; VI-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; VI-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB133_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; VI-NEXT:    v_mov_b32_e32 v0, v4
+; VI-NEXT:    v_mov_b32_e32 v1, v5
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_inc_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB133_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB133_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -9422,34 +10737,97 @@ define i64 @global_atomic_uinc_wrap_i64_ret_offset(ptr addrspace(1) %out, i64 %i
 ; SI-LABEL: global_atomic_uinc_wrap_i64_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    v_mov_b32_e32 v4, v2
+; SI-NEXT:    v_mov_b32_e32 v7, v1
+; SI-NEXT:    v_mov_b32_e32 v6, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_inc_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[8:9], 0
+; SI-NEXT:  .LBB134_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v10
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v11, vcc
+; SI-NEXT:    v_cmp_lt_u64_e32 vcc, v[10:11], v[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v9, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v8, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; SI-NEXT:    s_cbranch_execnz .LBB134_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_uinc_wrap_i64_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; VI-NEXT:    s_mov_b64 s[4:5], 0
+; VI-NEXT:  .LBB134_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v9, v1
+; VI-NEXT:    v_mov_b32_e32 v8, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v8
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v9, vcc
+; VI-NEXT:    v_cmp_lt_u64_e32 vcc, v[8:9], v[2:3]
+; VI-NEXT:    v_cndmask_b32_e32 v7, 0, v1, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v6, 0, v0, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT:    s_cbranch_execnz .LBB134_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_inc_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
+; GFX9-NEXT:  .LBB134_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v7, vcc
+; GFX9-NEXT:    v_cmp_lt_u64_e32 vcc, v[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v4, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execnz .LBB134_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst
@@ -9461,25 +10839,45 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i64_noret_scalar(ptr addrspace(1
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    s_mov_b32 s34, s7
-; SI-NEXT:    s_mov_b32 s35, s6
+; SI-NEXT:    v_writelane_b32 v8, s6, 0
+; SI-NEXT:    v_writelane_b32 v8, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB135_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_inc_x2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; SI-NEXT:    v_cmp_gt_u64_e32 vcc, s[34:35], v[2:3]
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v7, v3
+; SI-NEXT:    v_mov_b32_e32 v6, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB135_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v8, 1
+; SI-NEXT:    v_readlane_b32 s6, v8, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9487,24 +10885,58 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i64_noret_scalar(ptr addrspace(1
 ; VI-LABEL: global_atomic_uinc_wrap_i64_noret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v2, s4
-; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:  .LBB135_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; VI-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; VI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB135_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_inc_x2 v2, v[0:1], s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB135_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB135_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -9515,23 +10947,45 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i64_noret_offset_scalar(ptr addr
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v8, s6, 0
+; SI-NEXT:    v_writelane_b32 v8, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB136_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_inc_x2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v2
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; SI-NEXT:    v_cmp_gt_u64_e32 vcc, s[34:35], v[2:3]
+; SI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v7, v3
+; SI-NEXT:    v_mov_b32_e32 v6, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB136_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v8, 1
+; SI-NEXT:    v_readlane_b32 s6, v8, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9541,24 +10995,56 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i64_noret_offset_scalar(ptr addr
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_inc_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s34
+; VI-NEXT:    v_mov_b32_e32 v5, s35
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB136_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; VI-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; VI-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB136_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_inc_x2 v2, v[0:1], s[4:5] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB136_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB136_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst
@@ -9570,25 +11056,45 @@ define amdgpu_gfx i64 @global_atomic_uinc_wrap_i64_ret_scalar(ptr addrspace(1) i
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    s_mov_b32 s34, s7
-; SI-NEXT:    s_mov_b32 s35, s6
+; SI-NEXT:    v_writelane_b32 v6, s6, 0
+; SI-NEXT:    v_writelane_b32 v6, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB137_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_inc_x2 v[0:1], off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v4
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; SI-NEXT:    v_cmp_gt_u64_e32 vcc, s[34:35], v[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v3, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v2, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB137_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v6, 1
+; SI-NEXT:    v_readlane_b32 s6, v6, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9596,24 +11102,58 @@ define amdgpu_gfx i64 @global_atomic_uinc_wrap_i64_ret_scalar(ptr addrspace(1) i
 ; VI-LABEL: global_atomic_uinc_wrap_i64_ret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    s_mov_b64 s[34:35], 0
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:  .LBB137_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v1
+; VI-NEXT:    v_mov_b32_e32 v6, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v6
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; VI-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; VI-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB137_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_inc_x2 v[0:1], v2, v[0:1], s[4:5] glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB137_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v6, v1
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v5
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[5:6]
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB137_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -9624,23 +11164,45 @@ define amdgpu_gfx i64 @global_atomic_uinc_wrap_i64_ret_offset_scalar(ptr addrspa
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v6, s6, 0
+; SI-NEXT:    v_writelane_b32 v6, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[36:37], 0
+; SI-NEXT:  .LBB138_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_inc_x2 v[0:1], off, s[4:7], 0 offset:32 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 1, v4
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; SI-NEXT:    v_cmp_gt_u64_e32 vcc, s[34:35], v[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v3, 0, v1, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v2, 0, v0, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_mov_b32_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v5
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
+; SI-NEXT:    s_cbranch_execnz .LBB138_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
+; SI-NEXT:    v_readlane_b32 s7, v6, 1
+; SI-NEXT:    v_readlane_b32 s6, v6, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9651,23 +11213,55 @@ define amdgpu_gfx i64 @global_atomic_uinc_wrap_i64_ret_offset_scalar(ptr addrspa
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_inc_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_mov_b64 s[34:35], 0
+; VI-NEXT:  .LBB138_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v1
+; VI-NEXT:    v_mov_b32_e32 v6, v0
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 1, v6
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; VI-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; VI-NEXT:    v_cndmask_b32_e32 v5, 0, v1, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v4, 0, v0, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; VI-NEXT:    s_cbranch_execnz .LBB138_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_inc_x2 v[0:1], v2, v[0:1], s[4:5] offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[34:35], 0
+; GFX9-NEXT:  .LBB138_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v6, v1
+; GFX9-NEXT:    v_mov_b32_e32 v5, v0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, 1, v5
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
+; GFX9-NEXT:    v_cmp_gt_u64_e32 vcc, s[6:7], v[5:6]
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, 0, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v0, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
+; GFX9-NEXT:    s_cbranch_execnz .LBB138_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw uinc_wrap ptr addrspace(1) %gep, i64 %in seq_cst
@@ -9756,30 +11350,95 @@ define void @global_atomic_udec_wrap_i64_noret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-LABEL: global_atomic_udec_wrap_i64_noret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s4, s6
-; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_dec_x2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[8:11], 0 addr64
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB141_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, -1, v6
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, -1, v7, vcc
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; SI-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[8:11], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB141_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_udec_wrap_i64_noret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[8:9], 0
+; VI-NEXT:  .LBB141_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; VI-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; VI-NEXT:    v_add_u32_e64 v4, s[6:7], -1, v6
+; VI-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; VI-NEXT:    s_cbranch_execnz .LBB141_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i64_noret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_dec_x2 v[0:1], v[2:3], off
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[8:9], 0
+; GFX9-NEXT:  .LBB141_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_cbranch_execnz .LBB141_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -9789,13 +11448,38 @@ define void @global_atomic_udec_wrap_i64_noret_offset(ptr addrspace(1) %out, i64
 ; SI-LABEL: global_atomic_udec_wrap_i64_noret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s4, s6
-; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_dec_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[8:11], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB142_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, -1, v6
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, -1, v7, vcc
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; SI-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v7
+; SI-NEXT:    v_mov_b32_e32 v10, v6
+; SI-NEXT:    v_mov_b32_e32 v9, v5
+; SI-NEXT:    v_mov_b32_e32 v8, v4
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[8:11], v[0:1], s[8:11], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[8:9], v[6:7]
+; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SI-NEXT:    v_mov_b32_e32 v6, v8
+; SI-NEXT:    v_mov_b32_e32 v7, v9
+; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB142_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -9804,17 +11488,57 @@ define void @global_atomic_udec_wrap_i64_noret_offset(ptr addrspace(1) %out, i64
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3]
+; VI-NEXT:    flat_load_dwordx2 v[6:7], v[0:1]
+; VI-NEXT:    s_mov_b64 s[8:9], 0
+; VI-NEXT:  .LBB142_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; VI-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; VI-NEXT:    v_add_u32_e64 v4, s[6:7], -1, v6
+; VI-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; VI-NEXT:    s_cbranch_execnz .LBB142_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i64_noret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_dec_x2 v[0:1], v[2:3], off offset:32
+; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[8:9], 0
+; GFX9-NEXT:  .LBB142_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_cbranch_execnz .LBB142_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst
@@ -9825,32 +11549,103 @@ define i64 @global_atomic_udec_wrap_i64_ret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-LABEL: global_atomic_udec_wrap_i64_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s4, s6
-; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_dec_x2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    v_mov_b32_e32 v4, v2
+; SI-NEXT:    v_mov_b32_e32 v7, v1
+; SI-NEXT:    v_mov_b32_e32 v6, v0
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[8:11], 0 addr64
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB143_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v10
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, -1, v11, vcc
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; SI-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[10:11], v[4:5]
+; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v9, v1, v5, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v8, v0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[8:11], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB143_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_udec_wrap_i64_ret:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    flat_load_dwordx2 v[4:5], v[0:1]
+; VI-NEXT:    s_mov_b64 s[8:9], 0
+; VI-NEXT:  .LBB143_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v7, v5
+; VI-NEXT:    v_mov_b32_e32 v6, v4
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; VI-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; VI-NEXT:    v_add_u32_e64 v4, s[6:7], -1, v6
+; VI-NEXT:    v_addc_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; VI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; VI-NEXT:    s_cbranch_execnz .LBB143_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[8:9]
+; VI-NEXT:    v_mov_b32_e32 v0, v4
+; VI-NEXT:    v_mov_b32_e32 v1, v5
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i64_ret:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_dec_x2 v[0:1], v[0:1], v[2:3], off glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off
+; GFX9-NEXT:    s_mov_b64 s[8:9], 0
+; GFX9-NEXT:  .LBB143_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_cbranch_execnz .LBB143_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -9860,34 +11655,103 @@ define i64 @global_atomic_udec_wrap_i64_ret_offset(ptr addrspace(1) %out, i64 %i
 ; SI-LABEL: global_atomic_udec_wrap_i64_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s6, 0
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s4, s6
-; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_atomic_dec_x2 v[2:3], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    v_mov_b32_e32 v5, v3
+; SI-NEXT:    v_mov_b32_e32 v4, v2
+; SI-NEXT:    v_mov_b32_e32 v7, v1
+; SI-NEXT:    v_mov_b32_e32 v6, v0
+; SI-NEXT:    s_mov_b32 s10, 0
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s8, s10
+; SI-NEXT:    s_mov_b32 s9, s10
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[8:11], 0 addr64 offset:32
+; SI-NEXT:    s_mov_b64 s[6:7], 0
+; SI-NEXT:  .LBB144_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v11, v1
+; SI-NEXT:    v_mov_b32_e32 v10, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v10
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, -1, v11, vcc
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
+; SI-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[10:11], v[4:5]
+; SI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; SI-NEXT:    v_cndmask_b32_e32 v9, v1, v5, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v8, v0, v4, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v8
+; SI-NEXT:    v_mov_b32_e32 v1, v9
+; SI-NEXT:    v_mov_b32_e32 v2, v10
+; SI-NEXT:    v_mov_b32_e32 v3, v11
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[8:11], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_mov_b32_e32 v0, v2
-; SI-NEXT:    v_mov_b32_e32 v1, v3
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
+; SI-NEXT:    s_or_b64 s[6:7], vcc, s[6:7]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_cbranch_execnz .LBB144_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; VI-LABEL: global_atomic_udec_wrap_i64_ret_offset:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
-; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 32, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[4:5]
+; VI-NEXT:    s_mov_b64 s[8:9], 0
+; VI-NEXT:  .LBB144_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v9, v1
+; VI-NEXT:    v_mov_b32_e32 v8, v0
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; VI-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[8:9], v[2:3]
+; VI-NEXT:    v_add_u32_e64 v0, s[6:7], -1, v8
+; VI-NEXT:    v_addc_u32_e64 v1, s[6:7], -1, v9, s[6:7]
+; VI-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; VI-NEXT:    v_cndmask_b32_e32 v7, v1, v3, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v6, v0, v2, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; VI-NEXT:    s_cbranch_execnz .LBB144_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i64_ret_offset:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_atomic_dec_x2 v[0:1], v[0:1], v[2:3], off offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[0:1], off offset:32
+; GFX9-NEXT:    s_mov_b64 s[8:9], 0
+; GFX9-NEXT:  .LBB144_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v7, v5
+; GFX9-NEXT:    v_mov_b32_e32 v6, v4
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GFX9-NEXT:    v_cmp_gt_u64_e64 s[4:5], v[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v4, s[6:7], -1, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v5, s[6:7], -1, v7, s[6:7]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX9-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    s_cbranch_execnz .LBB144_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst
@@ -9899,25 +11763,49 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_scalar(ptr addrspace(1
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    s_mov_b32 s34, s7
-; SI-NEXT:    s_mov_b32 s35, s6
+; SI-NEXT:    v_writelane_b32 v10, s6, 0
+; SI-NEXT:    v_writelane_b32 v10, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[38:39], 0
+; SI-NEXT:    v_mov_b32_e32 v4, s35
+; SI-NEXT:    v_mov_b32_e32 v5, s34
+; SI-NEXT:  .LBB145_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_dec_x2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v2
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, -1, v3, vcc
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; SI-NEXT:    v_cmp_lt_u64_e64 s[36:37], s[34:35], v[2:3]
+; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v9, v3
+; SI-NEXT:    v_mov_b32_e32 v8, v2
+; SI-NEXT:    v_mov_b32_e32 v7, v1
+; SI-NEXT:    v_mov_b32_e32 v6, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; SI-NEXT:    v_mov_b32_e32 v2, v6
+; SI-NEXT:    v_mov_b32_e32 v3, v7
+; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; SI-NEXT:    s_cbranch_execnz .LBB145_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[38:39]
+; SI-NEXT:    v_readlane_b32 s7, v10, 1
+; SI-NEXT:    v_readlane_b32 s6, v10, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9925,24 +11813,66 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_scalar(ptr addrspace(1
 ; VI-LABEL: global_atomic_udec_wrap_i64_noret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v2, s4
-; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    s_mov_b64 s[38:39], 0
+; VI-NEXT:    v_mov_b32_e32 v6, s7
+; VI-NEXT:    v_mov_b32_e32 v7, s6
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:  .LBB145_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; VI-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; VI-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v2
+; VI-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; VI-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; VI-NEXT:    s_cbranch_execnz .LBB145_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i64_noret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_dec_x2 v2, v[0:1], s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[38:39], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:    v_mov_b32_e32 v6, s6
+; GFX9-NEXT:  .LBB145_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v2
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX9-NEXT:    s_cbranch_execnz .LBB145_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %in seq_cst
   ret void
@@ -9953,23 +11883,49 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_offset_scalar(ptr addr
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v10, s6, 0
+; SI-NEXT:    v_writelane_b32 v10, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[38:39], 0
+; SI-NEXT:    v_mov_b32_e32 v4, s35
+; SI-NEXT:    v_mov_b32_e32 v5, s34
+; SI-NEXT:  .LBB146_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_dec_x2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v2
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, -1, v3, vcc
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; SI-NEXT:    v_cmp_lt_u64_e64 s[36:37], s[34:35], v[2:3]
+; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v4, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v5, vcc
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v9, v3
+; SI-NEXT:    v_mov_b32_e32 v8, v2
+; SI-NEXT:    v_mov_b32_e32 v7, v1
+; SI-NEXT:    v_mov_b32_e32 v6, v0
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; SI-NEXT:    v_mov_b32_e32 v2, v6
+; SI-NEXT:    v_mov_b32_e32 v3, v7
+; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; SI-NEXT:    s_cbranch_execnz .LBB146_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[38:39]
+; SI-NEXT:    v_readlane_b32 s7, v10, 1
+; SI-NEXT:    v_readlane_b32 s6, v10, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -9979,24 +11935,64 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_offset_scalar(ptr addr
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
-; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_dec_x2 v[2:3], v[0:1]
+; VI-NEXT:    v_mov_b32_e32 v4, s34
+; VI-NEXT:    v_mov_b32_e32 v5, s35
+; VI-NEXT:    flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT:    s_mov_b64 s[38:39], 0
+; VI-NEXT:    v_mov_b32_e32 v6, s7
+; VI-NEXT:    v_mov_b32_e32 v7, s6
+; VI-NEXT:  .LBB146_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; VI-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; VI-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v2
+; VI-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; VI-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; VI-NEXT:    v_mov_b32_e32 v2, v0
+; VI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; VI-NEXT:    s_cbranch_execnz .LBB146_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i64_noret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
-; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_dec_x2 v2, v[0:1], s[4:5] offset:32
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[38:39], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s7
+; GFX9-NEXT:    v_mov_b32_e32 v6, s6
+; GFX9-NEXT:  .LBB146_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
+; GFX9-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v2
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v3, s[36:37]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v6, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX9-NEXT:    v_mov_b32_e32 v2, v0
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX9-NEXT:    s_cbranch_execnz .LBB146_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %tmp0 = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst
@@ -10008,25 +12004,49 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_scalar(ptr addrspace(1) i
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    s_mov_b32 s34, s7
-; SI-NEXT:    s_mov_b32 s35, s6
+; SI-NEXT:    v_writelane_b32 v10, s6, 0
+; SI-NEXT:    v_writelane_b32 v10, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:    v_mov_b32_e32 v0, s35
-; SI-NEXT:    v_mov_b32_e32 v1, s34
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_mov_b64 s[38:39], 0
+; SI-NEXT:    v_mov_b32_e32 v4, s35
+; SI-NEXT:    v_mov_b32_e32 v5, s34
+; SI-NEXT:  .LBB147_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_dec_x2 v[0:1], off, s[4:7], 0 glc
+; SI-NEXT:    v_mov_b32_e32 v9, v1
+; SI-NEXT:    v_mov_b32_e32 v8, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v8
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, -1, v9, vcc
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; SI-NEXT:    v_cmp_lt_u64_e64 s[36:37], s[34:35], v[8:9]
+; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v6
+; SI-NEXT:    v_mov_b32_e32 v1, v7
+; SI-NEXT:    v_mov_b32_e32 v2, v8
+; SI-NEXT:    v_mov_b32_e32 v3, v9
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; SI-NEXT:    s_cbranch_execnz .LBB147_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[38:39]
+; SI-NEXT:    v_readlane_b32 s7, v10, 1
+; SI-NEXT:    v_readlane_b32 s6, v10, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -10034,24 +12054,66 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_scalar(ptr addrspace(1) i
 ; VI-LABEL: global_atomic_udec_wrap_i64_ret_scalar:
 ; VI:       ; %bb.0:
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_mov_b32_e32 v0, s4
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v2, s4
+; VI-NEXT:    s_mov_b64 s[38:39], 0
+; VI-NEXT:    v_mov_b32_e32 v4, s7
+; VI-NEXT:    v_mov_b32_e32 v5, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s5
-; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:  .LBB147_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v9, v1
+; VI-NEXT:    v_mov_b32_e32 v8, v0
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; VI-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; VI-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v8
+; VI-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; VI-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; VI-NEXT:    s_cbranch_execnz .LBB147_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i64_ret_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_dec_x2 v[0:1], v2, v[0:1], s[4:5] glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5]
+; GFX9-NEXT:    s_mov_b64 s[38:39], 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_mov_b32_e32 v4, s6
+; GFX9-NEXT:  .LBB147_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v8, v1
+; GFX9-NEXT:    v_mov_b32_e32 v7, v0
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[7:8]
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[7:8]
+; GFX9-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v7
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v8, s[36:37]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v1, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v0, v4, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX9-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX9-NEXT:    s_cbranch_execnz .LBB147_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %result = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %in seq_cst
   ret i64 %result
@@ -10062,23 +12124,49 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_offset_scalar(ptr addrspa
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT:    buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_writelane_b32 v2, s6, 0
-; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    v_mov_b32_e32 v0, s6
-; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_writelane_b32 v10, s6, 0
+; SI-NEXT:    v_writelane_b32 v10, s7, 1
+; SI-NEXT:    s_mov_b32 s35, s7
+; SI-NEXT:    s_mov_b32 s34, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[4:7], 0 offset:32
+; SI-NEXT:    s_mov_b64 s[38:39], 0
+; SI-NEXT:    v_mov_b32_e32 v4, s35
+; SI-NEXT:    v_mov_b32_e32 v5, s34
+; SI-NEXT:  .LBB148_1: ; %atomicrmw.start
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    buffer_atomic_dec_x2 v[0:1], off, s[4:7], 0 offset:32 glc
+; SI-NEXT:    v_mov_b32_e32 v9, v1
+; SI-NEXT:    v_mov_b32_e32 v8, v0
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, -1, v8
+; SI-NEXT:    v_addc_u32_e32 v1, vcc, -1, v9, vcc
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; SI-NEXT:    v_cmp_lt_u64_e64 s[36:37], s[34:35], v[8:9]
+; SI-NEXT:    s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; SI-NEXT:    v_mov_b32_e32 v0, v6
+; SI-NEXT:    v_mov_b32_e32 v1, v7
+; SI-NEXT:    v_mov_b32_e32 v2, v8
+; SI-NEXT:    v_mov_b32_e32 v3, v9
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
-; SI-NEXT:    v_readlane_b32 s7, v2, 1
-; SI-NEXT:    v_readlane_b32 s6, v2, 0
+; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; SI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; SI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; SI-NEXT:    s_cbranch_execnz .LBB148_1
+; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; SI-NEXT:    s_or_b64 exec, exec, s[38:39]
+; SI-NEXT:    v_readlane_b32 s7, v10, 1
+; SI-NEXT:    v_readlane_b32 s6, v10, 0
 ; SI-NEXT:    s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT:    buffer_load_dword v2, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT:    buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
 ; SI-NEXT:    s_mov_b64 exec, s[34:35]
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0)
 ; SI-NEXT:    s_setpc_b64 s[30:31]
@@ -10089,23 +12177,63 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_offset_scalar(ptr addrspa
 ; VI-NEXT:    s_add_u32 s34, s4, 32
 ; VI-NEXT:    s_addc_u32 s35, s5, 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s34
-; VI-NEXT:    v_mov_b32_e32 v0, s6
-; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    v_mov_b32_e32 v3, s35
-; VI-NEXT:    flat_atomic_dec_x2 v[0:1], v[2:3], v[0:1] glc
+; VI-NEXT:    flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_mov_b64 s[38:39], 0
+; VI-NEXT:    v_mov_b32_e32 v4, s7
+; VI-NEXT:    v_mov_b32_e32 v5, s6
+; VI-NEXT:  .LBB148_1: ; %atomicrmw.start
+; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v9, v1
+; VI-NEXT:    v_mov_b32_e32 v8, v0
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; VI-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[8:9]
+; VI-NEXT:    v_add_u32_e64 v0, s[36:37], -1, v8
+; VI-NEXT:    v_addc_u32_e64 v1, s[36:37], -1, v9, s[36:37]
+; VI-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; VI-NEXT:    v_cndmask_b32_e32 v7, v1, v4, vcc
+; VI-NEXT:    v_cndmask_b32_e32 v6, v0, v5, vcc
+; VI-NEXT:    flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; VI-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; VI-NEXT:    s_cbranch_execnz .LBB148_1
+; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
+; VI-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: global_atomic_udec_wrap_i64_ret_offset_scalar:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s6
-; GFX9-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GFX9-NEXT:    global_atomic_dec_x2 v[0:1], v2, v[0:1], s[4:5] offset:32 glc
+; GFX9-NEXT:    global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
+; GFX9-NEXT:    s_mov_b64 s[38:39], 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    v_mov_b32_e32 v4, s6
+; GFX9-NEXT:  .LBB148_1: ; %atomicrmw.start
+; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v8, v1
+; GFX9-NEXT:    v_mov_b32_e32 v7, v0
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[7:8]
+; GFX9-NEXT:    v_cmp_lt_u64_e64 s[34:35], s[6:7], v[7:8]
+; GFX9-NEXT:    v_add_co_u32_e64 v0, s[36:37], -1, v7
+; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[36:37], -1, v8, s[36:37]
+; GFX9-NEXT:    s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT:    v_cndmask_b32_e32 v6, v1, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v0, v4, vcc
+; GFX9-NEXT:    global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX9-NEXT:    s_or_b64 s[38:39], vcc, s[38:39]
+; GFX9-NEXT:    s_andn2_b64 exec, exec, s[38:39]
+; GFX9-NEXT:    s_cbranch_execnz .LBB148_1
+; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
+; GFX9-NEXT:    s_or_b64 exec, exec, s[38:39]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %gep = getelementptr i64, ptr addrspace(1) %out, i64 4
   %result = atomicrmw udec_wrap ptr addrspace(1) %gep, i64 %in seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/hard-clause-limit-attr.mir b/llvm/test/CodeGen/AMDGPU/hard-clause-limit-attr.mir
index cfd4ea155316a..fb3f328f9d58c 100644
--- a/llvm/test/CodeGen/AMDGPU/hard-clause-limit-attr.mir
+++ b/llvm/test/CodeGen/AMDGPU/hard-clause-limit-attr.mir
@@ -18,7 +18,7 @@ body: |
     ; GFX11-LABEL: name: long_clause_32
     ; GFX11: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-NEXT:   S_CLAUSE 31
     ; GFX11-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
@@ -96,7 +96,7 @@ body: |
     ; GFX11-LABEL: name: long_clause_7
     ; GFX11: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-NEXT:   S_CLAUSE 6
     ; GFX11-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
@@ -106,7 +106,7 @@ body: |
     ; GFX11-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
     ; GFX11-NEXT: }
-    ; GFX11-NEXT: BUNDLE implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-NEXT:   S_CLAUSE 6
     ; GFX11-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
@@ -116,7 +116,7 @@ body: |
     ; GFX11-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
     ; GFX11-NEXT: }
-    ; GFX11-NEXT: BUNDLE implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-NEXT:   S_CLAUSE 6
     ; GFX11-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
@@ -126,7 +126,7 @@ body: |
     ; GFX11-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
     ; GFX11-NEXT: }
-    ; GFX11-NEXT: BUNDLE implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-NEXT:   S_CLAUSE 6
     ; GFX11-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
@@ -136,7 +136,7 @@ body: |
     ; GFX11-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
     ; GFX11-NEXT: }
-    ; GFX11-NEXT: BUNDLE implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-NEXT:   S_CLAUSE 3
     ; GFX11-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/hard-clause-limit.mir b/llvm/test/CodeGen/AMDGPU/hard-clause-limit.mir
index 98221c21eee2e..c00e25cfbcbff 100644
--- a/llvm/test/CodeGen/AMDGPU/hard-clause-limit.mir
+++ b/llvm/test/CodeGen/AMDGPU/hard-clause-limit.mir
@@ -14,7 +14,7 @@ body: |
     ; GFX11-32-LABEL: name: long_clause
     ; GFX11-32: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
     ; GFX11-32-NEXT: {{  $}}
-    ; GFX11-32-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-32-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-32-NEXT:   S_CLAUSE 31
     ; GFX11-32-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
     ; GFX11-32-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
@@ -53,7 +53,7 @@ body: |
     ; GFX11-16-LABEL: name: long_clause
     ; GFX11-16: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
     ; GFX11-16-NEXT: {{  $}}
-    ; GFX11-16-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-16-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-16-NEXT:   S_CLAUSE 15
     ; GFX11-16-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
     ; GFX11-16-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
@@ -72,7 +72,7 @@ body: |
     ; GFX11-16-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
     ; GFX11-16-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
     ; GFX11-16-NEXT: }
-    ; GFX11-16-NEXT: BUNDLE implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-16-NEXT: BUNDLE implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-16-NEXT:   S_CLAUSE 15
     ; GFX11-16-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
     ; GFX11-16-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
@@ -95,7 +95,7 @@ body: |
     ; GFX11-10-LABEL: name: long_clause
     ; GFX11-10: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
     ; GFX11-10-NEXT: {{  $}}
-    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-10-NEXT:   S_CLAUSE 9
     ; GFX11-10-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
     ; GFX11-10-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
@@ -108,7 +108,7 @@ body: |
     ; GFX11-10-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
     ; GFX11-10-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
     ; GFX11-10-NEXT: }
-    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-10-NEXT:   S_CLAUSE 9
     ; GFX11-10-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
     ; GFX11-10-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
@@ -121,7 +121,7 @@ body: |
     ; GFX11-10-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
     ; GFX11-10-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
     ; GFX11-10-NEXT: }
-    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-10-NEXT:   S_CLAUSE 9
     ; GFX11-10-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
     ; GFX11-10-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
@@ -134,7 +134,7 @@ body: |
     ; GFX11-10-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
     ; GFX11-10-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
     ; GFX11-10-NEXT: }
-    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-10-NEXT: BUNDLE implicit-def $vgpr31, implicit-def $vgpr32, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-10-NEXT:   S_CLAUSE 1
     ; GFX11-10-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
     ; GFX11-10-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
@@ -143,56 +143,56 @@ body: |
     ; GFX11-4-LABEL: name: long_clause
     ; GFX11-4: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
     ; GFX11-4-NEXT: {{  $}}
-    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-4-NEXT:   S_CLAUSE 3
     ; GFX11-4-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
     ; GFX11-4-NEXT: }
-    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-4-NEXT:   S_CLAUSE 3
     ; GFX11-4-NEXT:   $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
     ; GFX11-4-NEXT: }
-    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-4-NEXT:   S_CLAUSE 3
     ; GFX11-4-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
     ; GFX11-4-NEXT: }
-    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-4-NEXT:   S_CLAUSE 3
     ; GFX11-4-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
     ; GFX11-4-NEXT: }
-    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-4-NEXT:   S_CLAUSE 3
     ; GFX11-4-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
     ; GFX11-4-NEXT: }
-    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-4-NEXT:   S_CLAUSE 3
     ; GFX11-4-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
     ; GFX11-4-NEXT: }
-    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-4-NEXT:   S_CLAUSE 3
     ; GFX11-4-NEXT:   $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
     ; GFX11-4-NEXT: }
-    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-4-NEXT: BUNDLE implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-4-NEXT:   S_CLAUSE 3
     ; GFX11-4-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
     ; GFX11-4-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/hard-clauses-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/hard-clauses-gfx1250.mir
index 8007597a32fbe..79480bc8017f7 100644
--- a/llvm/test/CodeGen/AMDGPU/hard-clauses-gfx1250.mir
+++ b/llvm/test/CodeGen/AMDGPU/hard-clauses-gfx1250.mir
@@ -1,6 +1,507 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass si-insert-hard-clauses %s -o - | FileCheck %s -check-prefixes=GFX12
-# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass si-insert-hard-clauses %s -o - | FileCheck %s -check-prefixes=GFX12
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -verify-machineinstrs -run-pass si-insert-hard-clauses %s -o - | FileCheck %s -check-prefixes=GFX12,GFX1200
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass si-insert-hard-clauses %s -o - | FileCheck %s -check-prefixes=GFX12,GFX1250
+
+---
+name: long_clause
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX1200-LABEL: name: long_clause
+    ; GFX1200: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX1200-NEXT: {{  $}}
+    ; GFX1200-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX1200-NEXT:   S_CLAUSE 31
+    ; GFX1200-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    ; GFX1200-NEXT: }
+    ; GFX1200-NEXT: BUNDLE implicit-def $vgpr33, implicit-def $vgpr34, implicit-def $vgpr35, implicit-def $vgpr36, implicit-def $vgpr37, implicit-def $vgpr38, implicit-def $vgpr39, implicit-def $vgpr40, implicit-def $vgpr41, implicit-def $vgpr42, implicit-def $vgpr43, implicit-def $vgpr44, implicit-def $vgpr45, implicit-def $vgpr46, implicit-def $vgpr47, implicit-def $vgpr48, implicit-def $vgpr49, implicit-def $vgpr50, implicit-def $vgpr51, implicit-def $vgpr52, implicit-def $vgpr53, implicit-def $vgpr54, implicit-def $vgpr55, implicit-def $vgpr56, implicit-def $vgpr57, implicit-def $vgpr58, implicit-def $vgpr59, implicit-def $vgpr60, implicit-def $vgpr61, implicit-def $vgpr62, implicit-def $vgpr63, implicit-def $vgpr64, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX1200-NEXT:   S_CLAUSE 31
+    ; GFX1200-NEXT:   $vgpr33 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 132, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr34 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 136, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr35 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 140, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr36 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 144, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr37 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 148, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr38 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 152, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr39 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 156, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr40 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 160, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr41 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 164, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr42 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 168, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr43 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 172, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr44 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 176, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr45 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 180, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr46 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 184, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr47 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 188, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr48 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 192, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr49 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 196, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr50 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 200, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr51 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 204, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr52 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 208, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr53 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 212, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr54 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 216, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr55 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 220, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr56 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 224, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr57 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 228, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr58 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 232, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr59 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 236, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr60 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 240, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr61 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 244, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr62 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 248, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr63 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 252, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr64 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 256, 0, 0, implicit $exec
+    ; GFX1200-NEXT: }
+    ; GFX1200-NEXT: BUNDLE implicit-def $vgpr65, implicit-def $vgpr66, implicit-def $vgpr67, implicit-def $vgpr68, implicit-def $vgpr69, implicit-def $vgpr70, implicit-def $vgpr71, implicit-def $vgpr72, implicit-def $vgpr73, implicit-def $vgpr74, implicit-def $vgpr75, implicit-def $vgpr76, implicit-def $vgpr77, implicit-def $vgpr78, implicit-def $vgpr79, implicit-def $vgpr80, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX1200-NEXT:   S_CLAUSE 15
+    ; GFX1200-NEXT:   $vgpr65 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 260, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr66 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 264, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr67 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 268, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr68 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 272, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr69 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 276, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr70 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 280, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr71 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 284, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr72 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 288, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr73 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 292, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr74 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 296, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr75 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 300, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr76 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 304, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr77 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 308, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr78 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 312, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr79 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 316, 0, 0, implicit $exec
+    ; GFX1200-NEXT:   $vgpr80 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 320, 0, 0, implicit $exec
+    ; GFX1200-NEXT: }
+    ;
+    ; GFX1250-LABEL: name: long_clause
+    ; GFX1250: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; GFX1250-NEXT: {{  $}}
+    ; GFX1250-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit-def $vgpr33, implicit-def $vgpr34, implicit-def $vgpr35, implicit-def $vgpr36, implicit-def $vgpr37, implicit-def $vgpr38, implicit-def $vgpr39, implicit-def $vgpr40, implicit-def $vgpr41, implicit-def $vgpr42, implicit-def $vgpr43, implicit-def $vgpr44, implicit-def $vgpr45, implicit-def $vgpr46, implicit-def $vgpr47, implicit-def $vgpr48, implicit-def $vgpr49, implicit-def $vgpr50, implicit-def $vgpr51, implicit-def $vgpr52, implicit-def $vgpr53, implicit-def $vgpr54, implicit-def $vgpr55, implicit-def $vgpr56, implicit-def $vgpr57, implicit-def $vgpr58, implicit-def $vgpr59, implicit-def $vgpr60, implicit-def $vgpr61, implicit-def $vgpr62, implicit-def $vgpr63, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX1250-NEXT:   S_CLAUSE 62
+    ; GFX1250-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr33 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 132, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr34 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 136, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr35 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 140, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr36 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 144, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr37 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 148, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr38 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 152, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr39 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 156, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr40 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 160, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr41 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 164, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr42 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 168, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr43 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 172, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr44 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 176, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr45 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 180, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr46 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 184, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr47 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 188, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr48 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 192, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr49 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 196, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr50 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 200, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr51 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 204, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr52 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 208, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr53 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 212, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr54 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 216, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr55 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 220, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr56 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 224, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr57 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 228, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr58 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 232, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr59 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 236, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr60 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 240, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr61 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 244, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr62 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 248, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr63 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 252, 0, 0, implicit $exec
+    ; GFX1250-NEXT: }
+    ; GFX1250-NEXT: BUNDLE implicit-def $vgpr64, implicit-def $vgpr65, implicit-def $vgpr66, implicit-def $vgpr67, implicit-def $vgpr68, implicit-def $vgpr69, implicit-def $vgpr70, implicit-def $vgpr71, implicit-def $vgpr72, implicit-def $vgpr73, implicit-def $vgpr74, implicit-def $vgpr75, implicit-def $vgpr76, implicit-def $vgpr77, implicit-def $vgpr78, implicit-def $vgpr79, implicit-def $vgpr80, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX1250-NEXT:   S_CLAUSE 16
+    ; GFX1250-NEXT:   $vgpr64 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 256, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr65 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 260, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr66 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 264, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr67 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 268, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr68 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 272, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr69 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 276, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr70 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 280, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr71 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 284, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr72 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 288, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr73 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 292, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr74 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 296, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr75 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 300, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr76 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 304, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr77 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 308, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr78 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 312, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr79 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 316, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr80 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 320, 0, 0, implicit $exec
+    ; GFX1250-NEXT: }
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
+    $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 12, 0, 0, implicit $exec
+    $vgpr4 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 16, 0, 0, implicit $exec
+    $vgpr5 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 20, 0, 0, implicit $exec
+    $vgpr6 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 24, 0, 0, implicit $exec
+    $vgpr7 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 28, 0, 0, implicit $exec
+    $vgpr8 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 32, 0, 0, implicit $exec
+    $vgpr9 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 36, 0, 0, implicit $exec
+    $vgpr10 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 40, 0, 0, implicit $exec
+    $vgpr11 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 44, 0, 0, implicit $exec
+    $vgpr12 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 48, 0, 0, implicit $exec
+    $vgpr13 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 52, 0, 0, implicit $exec
+    $vgpr14 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 56, 0, 0, implicit $exec
+    $vgpr15 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 60, 0, 0, implicit $exec
+    $vgpr16 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 64, 0, 0, implicit $exec
+    $vgpr17 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 68, 0, 0, implicit $exec
+    $vgpr18 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 72, 0, 0, implicit $exec
+    $vgpr19 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 76, 0, 0, implicit $exec
+    $vgpr20 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 80, 0, 0, implicit $exec
+    $vgpr21 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 84, 0, 0, implicit $exec
+    $vgpr22 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 88, 0, 0, implicit $exec
+    $vgpr23 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 92, 0, 0, implicit $exec
+    $vgpr24 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 96, 0, 0, implicit $exec
+    $vgpr25 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 100, 0, 0, implicit $exec
+    $vgpr26 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 104, 0, 0, implicit $exec
+    $vgpr27 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 108, 0, 0, implicit $exec
+    $vgpr28 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 112, 0, 0, implicit $exec
+    $vgpr29 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 116, 0, 0, implicit $exec
+    $vgpr30 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 120, 0, 0, implicit $exec
+    $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
+    $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
+    $vgpr33 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 132, 0, 0, implicit $exec
+    $vgpr34 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 136, 0, 0, implicit $exec
+    $vgpr35 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 140, 0, 0, implicit $exec
+    $vgpr36 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 144, 0, 0, implicit $exec
+    $vgpr37 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 148, 0, 0, implicit $exec
+    $vgpr38 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 152, 0, 0, implicit $exec
+    $vgpr39 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 156, 0, 0, implicit $exec
+    $vgpr40 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 160, 0, 0, implicit $exec
+    $vgpr41 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 164, 0, 0, implicit $exec
+    $vgpr42 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 168, 0, 0, implicit $exec
+    $vgpr43 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 172, 0, 0, implicit $exec
+    $vgpr44 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 176, 0, 0, implicit $exec
+    $vgpr45 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 180, 0, 0, implicit $exec
+    $vgpr46 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 184, 0, 0, implicit $exec
+    $vgpr47 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 188, 0, 0, implicit $exec
+    $vgpr48 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 192, 0, 0, implicit $exec
+    $vgpr49 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 196, 0, 0, implicit $exec
+    $vgpr50 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 200, 0, 0, implicit $exec
+    $vgpr51 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 204, 0, 0, implicit $exec
+    $vgpr52 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 208, 0, 0, implicit $exec
+    $vgpr53 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 212, 0, 0, implicit $exec
+    $vgpr54 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 216, 0, 0, implicit $exec
+    $vgpr55 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 220, 0, 0, implicit $exec
+    $vgpr56 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 224, 0, 0, implicit $exec
+    $vgpr57 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 228, 0, 0, implicit $exec
+    $vgpr58 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 232, 0, 0, implicit $exec
+    $vgpr59 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 236, 0, 0, implicit $exec
+    $vgpr60 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 240, 0, 0, implicit $exec
+    $vgpr61 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 244, 0, 0, implicit $exec
+    $vgpr62 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 248, 0, 0, implicit $exec
+    $vgpr63 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 252, 0, 0, implicit $exec
+    $vgpr64 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 256, 0, 0, implicit $exec
+    $vgpr65 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 260, 0, 0, implicit $exec
+    $vgpr66 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 264, 0, 0, implicit $exec
+    $vgpr67 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 268, 0, 0, implicit $exec
+    $vgpr68 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 272, 0, 0, implicit $exec
+    $vgpr69 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 276, 0, 0, implicit $exec
+    $vgpr70 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 280, 0, 0, implicit $exec
+    $vgpr71 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 284, 0, 0, implicit $exec
+    $vgpr72 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 288, 0, 0, implicit $exec
+    $vgpr73 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 292, 0, 0, implicit $exec
+    $vgpr74 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 296, 0, 0, implicit $exec
+    $vgpr75 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 300, 0, 0, implicit $exec
+    $vgpr76 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 304, 0, 0, implicit $exec
+    $vgpr77 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 308, 0, 0, implicit $exec
+    $vgpr78 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 312, 0, 0, implicit $exec
+    $vgpr79 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 316, 0, 0, implicit $exec
+    $vgpr80 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 320, 0, 0, implicit $exec
+...
+
+---
+name: kill
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr4
+    ; GFX12-LABEL: name: kill
+    ; GFX12: liveins: $sgpr0_sgpr1, $sgpr4
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
+    ; GFX12-NEXT:   S_CLAUSE 1
+    ; GFX12-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
+    ; GFX12-NEXT:   KILL undef renamable $sgpr4
+    ; GFX12-NEXT:   $sgpr3 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 4, 0
+    ; GFX12-NEXT: }
+    $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
+    KILL undef renamable $sgpr4
+    $sgpr3 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 4, 0
+...
+
+---
+name: kill2
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr4, $sgpr5
+    ; GFX12-LABEL: name: kill2
+    ; GFX12: liveins: $sgpr0_sgpr1, $sgpr4, $sgpr5
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
+    ; GFX12-NEXT:   S_CLAUSE 1
+    ; GFX12-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
+    ; GFX12-NEXT:   KILL undef renamable $sgpr4
+    ; GFX12-NEXT:   $sgpr3 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 4, 0
+    ; GFX12-NEXT: }
+    ; GFX12-NEXT: KILL undef renamable $sgpr5
+    $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
+    KILL undef renamable $sgpr4
+    $sgpr3 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 4, 0
+    KILL undef renamable $sgpr5
+...
+
+---
+name: flat_load_atomic
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1200-LABEL: name: flat_load_atomic
+    ; GFX1200: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1200-NEXT: {{  $}}
+    ; GFX1200-NEXT: $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX1200-NEXT: $vgpr4 = FLAT_ATOMIC_ADD_RTN $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec, implicit $flat_scr
+    ;
+    ; GFX1250-LABEL: name: flat_load_atomic
+    ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1250-NEXT: {{  $}}
+    ; GFX1250-NEXT: BUNDLE implicit-def $vgpr3, implicit-def $vgpr4, implicit $vgpr0_vgpr1, implicit $exec, implicit $flat_scr, implicit $vgpr2 {
+    ; GFX1250-NEXT:   S_CLAUSE 1
+    ; GFX1250-NEXT:   $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX1250-NEXT:   $vgpr4 = FLAT_ATOMIC_ADD_RTN $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec, implicit $flat_scr
+    ; GFX1250-NEXT: }
+    $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr4 = FLAT_ATOMIC_ADD_RTN $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec, implicit $flat_scr
+...
+
+---
+name: global_load_atomic
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1200-LABEL: name: global_load_atomic
+    ; GFX1200: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1200-NEXT: {{  $}}
+    ; GFX1200-NEXT: $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GFX1200-NEXT: $vgpr4 = GLOBAL_ATOMIC_ADD_RTN $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec
+    ;
+    ; GFX1250-LABEL: name: global_load_atomic
+    ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1250-NEXT: {{  $}}
+    ; GFX1250-NEXT: BUNDLE implicit-def $vgpr3, implicit-def $vgpr4, implicit $vgpr0_vgpr1, implicit $exec, implicit $vgpr2 {
+    ; GFX1250-NEXT:   S_CLAUSE 1
+    ; GFX1250-NEXT:   $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr4 = GLOBAL_ATOMIC_ADD_RTN $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec
+    ; GFX1250-NEXT: }
+    $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    $vgpr4 = GLOBAL_ATOMIC_ADD_RTN $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec
+...
+
+---
+name: flat_global_load
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; GFX12-LABEL: name: flat_global_load
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 0, implicit $exec, implicit $flat_scr
+    $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 0, implicit $exec, implicit $flat_scr
+...
+
+---
+name: buffer_load_atomic
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0
+    ; GFX1200-LABEL: name: buffer_load_atomic
+    ; GFX1200: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0
+    ; GFX1200-NEXT: {{  $}}
+    ; GFX1200-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+    ; GFX1200-NEXT: $vgpr0 = BUFFER_ATOMIC_ADD_OFFSET_RTN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, implicit $exec
+    ;
+    ; GFX1250-LABEL: name: buffer_load_atomic
+    ; GFX1250: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0
+    ; GFX1250-NEXT: {{  $}}
+    ; GFX1250-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4, implicit $exec, implicit $vgpr0 {
+    ; GFX1250-NEXT:   S_CLAUSE 1
+    ; GFX1250-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   $vgpr0 = BUFFER_ATOMIC_ADD_OFFSET_RTN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, implicit $exec
+    ; GFX1250-NEXT: }
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_ATOMIC_ADD_OFFSET_RTN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, implicit $exec
+...
+
+---
+name: flat_load_store
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1200-LABEL: name: flat_load_store
+    ; GFX1200: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1200-NEXT: {{  $}}
+    ; GFX1200-NEXT: $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX1200-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec, implicit $flat_scr
+    ;
+    ; GFX1250-LABEL: name: flat_load_store
+    ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1250-NEXT: {{  $}}
+    ; GFX1250-NEXT: BUNDLE implicit-def $vgpr3, implicit $vgpr0_vgpr1, implicit $exec, implicit $flat_scr, implicit $vgpr2 {
+    ; GFX1250-NEXT:   S_CLAUSE 1
+    ; GFX1250-NEXT:   $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX1250-NEXT:   FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec, implicit $flat_scr
+    ; GFX1250-NEXT: }
+    $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec, implicit $flat_scr
+...
+
+---
+name: global_load_store
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1200-LABEL: name: global_load_store
+    ; GFX1200: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1200-NEXT: {{  $}}
+    ; GFX1200-NEXT: $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GFX1200-NEXT: GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec
+    ;
+    ; GFX1250-LABEL: name: global_load_store
+    ; GFX1250: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX1250-NEXT: {{  $}}
+    ; GFX1250-NEXT: BUNDLE implicit-def $vgpr3, implicit $vgpr0_vgpr1, implicit $exec, implicit $vgpr2 {
+    ; GFX1250-NEXT:   S_CLAUSE 1
+    ; GFX1250-NEXT:   $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec
+    ; GFX1250-NEXT: }
+    $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec
+...
+
+---
+name: buffer_load_store
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0
+    ; GFX1200-LABEL: name: buffer_load_store
+    ; GFX1200: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0
+    ; GFX1200-NEXT: {{  $}}
+    ; GFX1200-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+    ; GFX1200-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4, 0, 0, implicit $exec
+    ;
+    ; GFX1250-LABEL: name: buffer_load_store
+    ; GFX1250: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0
+    ; GFX1250-NEXT: {{  $}}
+    ; GFX1250-NEXT: BUNDLE implicit-def $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4, implicit $exec, implicit $vgpr0 {
+    ; GFX1250-NEXT:   S_CLAUSE 1
+    ; GFX1250-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+    ; GFX1250-NEXT:   BUFFER_STORE_DWORD_OFFSET $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4, 0, 0, implicit $exec
+    ; GFX1250-NEXT: }
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4, 0, 0, implicit $exec
+...
+
+---
+name: flat_load_global_load
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX12-LABEL: name: flat_load_global_load
+    ; GFX12: liveins: $vgpr0_vgpr1, $vgpr2
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: $vgpr4 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr4 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+...
+
+---
+name: global_load_buffer_store
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4
+    ; GFX12-LABEL: name: global_load_buffer_store
+    ; GFX12: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: $vgpr4 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    ; GFX12-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4, 0, 0, implicit $exec
+    $vgpr4 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4, 0, 0, implicit $exec
+...
 
 ---
 name: flat_prefetch_flat_load
@@ -31,3 +532,106 @@ body: |
     GLOBAL_PREFETCH_B8 $vgpr0_vgpr1, 0, 0, implicit $exec
     $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
 ...
+
+---
+name: async_load_async_store
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX12-LABEL: name: async_load_async_store
+    ; GFX12: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: BUNDLE implicit-def $asynccnt, implicit $vgpr2, implicit $vgpr0_vgpr1, implicit $exec, implicit $asynccnt {
+    ; GFX12-NEXT:   S_CLAUSE 1
+    ; GFX12-NEXT:   GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; GFX12-NEXT:   GLOBAL_STORE_ASYNC_FROM_LDS_B32 $vgpr0_vgpr1, $vgpr2, 32, 0, implicit-def $asynccnt, implicit $exec, implicit internal $asynccnt
+    ; GFX12-NEXT: }
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    GLOBAL_STORE_ASYNC_FROM_LDS_B32 $vgpr0_vgpr1, $vgpr2, 32, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+...
+
+---
+name: async_load_ds_load_tr
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX12-LABEL: name: async_load_ds_load_tr
+    ; GFX12: liveins: $vgpr0, $vgpr1, $vgpr2
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    ; GFX12-NEXT: $vgpr0_vgpr1 = DS_LOAD_TR8_B64 $vgpr2, 8, 0, implicit $exec
+    GLOBAL_LOAD_ASYNC_TO_LDS_B32 $vgpr2, $vgpr0_vgpr1, 0, 0, implicit-def $asynccnt, implicit $exec, implicit $asynccnt
+    $vgpr0_vgpr1 = DS_LOAD_TR8_B64 $vgpr2, 8, 0, implicit $exec
+...
+
+---
+name: ds_load_trs_ds_load
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GFX12-LABEL: name: ds_load_trs_ds_load
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: $vgpr4_vgpr5 = DS_LOAD_TR8_B64 $vgpr0, 0, 0, implicit $exec
+    ; GFX12-NEXT: $vgpr0_vgpr1 = DS_LOAD_TR8_B64 $vgpr0, 8, 0, implicit $exec
+    ; GFX12-NEXT: $vgpr2_vgpr3 = DS_READ_B64_gfx9 $vgpr0, 16, 0, implicit $exec
+    $vgpr4_vgpr5 = DS_LOAD_TR8_B64 $vgpr0, 0, 0, implicit $exec
+    $vgpr0_vgpr1 = DS_LOAD_TR8_B64 $vgpr0, 8, 0, implicit $exec
+    $vgpr2_vgpr3 = DS_READ_B64_gfx9 $vgpr0, 16, 0, implicit $exec
+...
+
+# Make sure we do not clause DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 with anything
+---
+name: ds_atomic_async_barrier_arrive_b64_ds_read
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX12-LABEL: name: ds_atomic_async_barrier_arrive_b64_ds_read
+    ; GFX12: liveins: $vgpr0, $vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
+    ; GFX12-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr1, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; GFX12-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr0, 16, 0, implicit $exec
+    $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr1, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    $vgpr3 = DS_READ_B32_gfx9 $vgpr0, 16, 0, implicit $exec
+...
+
+---
+name: ds_atomic_async_barrier_arrive_b64_flat_load
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GFX12-LABEL: name: ds_atomic_async_barrier_arrive_b64_flat_load
+    ; GFX12: liveins: $vgpr0, $vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr1, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; GFX12-NEXT: $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 16, 0, implicit $exec, implicit $flat_scr
+    $vgpr2 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr1, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 16, 0, implicit $exec, implicit $flat_scr
+...
+
+---
+name: global_load_switching_scope
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; GFX12-LABEL: name: global_load_switching_scope
+    ; GFX12: liveins: $vgpr0_vgpr1
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0_vgpr1, implicit $exec, implicit $flat_scr {
+    ; GFX12-NEXT:   S_CLAUSE 1
+    ; GFX12-NEXT:   $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT:   $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 24, implicit $exec, implicit $flat_scr
+    ; GFX12-NEXT: }
+    $vgpr2 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
+    $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 4, 24, implicit $exec, implicit $flat_scr
+...
diff --git a/llvm/test/CodeGen/AMDGPU/hard-clauses-img-gfx11.mir b/llvm/test/CodeGen/AMDGPU/hard-clauses-img-gfx11.mir
index 0829cabe56fe1..7e1055b2a28a4 100644
--- a/llvm/test/CodeGen/AMDGPU/hard-clauses-img-gfx11.mir
+++ b/llvm/test/CodeGen/AMDGPU/hard-clauses-img-gfx11.mir
@@ -11,7 +11,7 @@ body: |
     ; CHECK-LABEL: name: mimg_nsa
     ; CHECK: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr10_vgpr11_vgpr12, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr10_vgpr11, implicit-def $vgpr11_vgpr12, implicit-def $vgpr20_vgpr21_vgpr22, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr20_vgpr21, implicit-def $vgpr21_vgpr22, implicit $vgpr3, implicit $vgpr8, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr10_vgpr11_vgpr12, implicit-def $vgpr20_vgpr21_vgpr22, implicit $vgpr3, implicit $vgpr8, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec {
     ; CHECK-NEXT:   S_CLAUSE 1
     ; CHECK-NEXT:   $vgpr10_vgpr11_vgpr12 = IMAGE_SAMPLE_LZ_V3_V2_nsa_gfx11 $vgpr3, $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
     ; CHECK-NEXT:   $vgpr20_vgpr21_vgpr22 = IMAGE_SAMPLE_LZ_V3_V2_nsa_gfx11 $vgpr3, $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 14, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
@@ -29,7 +29,7 @@ body: |
     ; CHECK-LABEL: name: mimg_nsa_mixed
     ; CHECK: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr20_vgpr21_vgpr22, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr20_vgpr21, implicit-def $vgpr21_vgpr22, implicit $vgpr3, implicit $vgpr8, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr10, implicit-def $vgpr14, implicit-def $vgpr20_vgpr21_vgpr22, implicit $vgpr3, implicit $vgpr8, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec, implicit $vgpr5_vgpr6 {
     ; CHECK-NEXT:   S_CLAUSE 2
     ; CHECK-NEXT:   $vgpr10 = IMAGE_SAMPLE_LZ_V1_V2_nsa_gfx11 $vgpr3, $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
     ; CHECK-NEXT:   $vgpr14 = IMAGE_SAMPLE_LZ_V1_V2_gfx11 $vgpr5_vgpr6, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 7)
diff --git a/llvm/test/CodeGen/AMDGPU/hard-clauses-img-gfx12.mir b/llvm/test/CodeGen/AMDGPU/hard-clauses-img-gfx12.mir
index 243a84562ab30..9689dda9932ed 100644
--- a/llvm/test/CodeGen/AMDGPU/hard-clauses-img-gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/hard-clauses-img-gfx12.mir
@@ -10,7 +10,7 @@ body: |
     ; CHECK-LABEL: name: mimg
     ; CHECK: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr10_vgpr11_vgpr12, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr10_vgpr11, implicit-def $vgpr11_vgpr12, implicit-def $vgpr20_vgpr21_vgpr22, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr20_vgpr21, implicit-def $vgpr21_vgpr22, implicit $vgpr3, implicit $vgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr10_vgpr11_vgpr12, implicit-def $vgpr20_vgpr21_vgpr22, implicit $vgpr3, implicit $vgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec {
     ; CHECK-NEXT:   S_CLAUSE 1
     ; CHECK-NEXT:   $vgpr10_vgpr11_vgpr12 = IMAGE_SAMPLE_LZ_V3_V2_gfx12 $vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
     ; CHECK-NEXT:   $vgpr20_vgpr21_vgpr22 = IMAGE_SAMPLE_LZ_V3_V2_gfx12 $vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 14, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
@@ -28,7 +28,7 @@ body: |
     ; CHECK-LABEL: name: mimg_mixed
     ; CHECK: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr20_vgpr21_vgpr22, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr20_vgpr21, implicit-def $vgpr21_vgpr22, implicit $vgpr3, implicit $vgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec, implicit $vgpr5, implicit $vgpr6 {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr10, implicit-def $vgpr14, implicit-def $vgpr20_vgpr21_vgpr22, implicit $vgpr3, implicit $vgpr4, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec, implicit $vgpr5, implicit $vgpr6 {
     ; CHECK-NEXT:   S_CLAUSE 2
     ; CHECK-NEXT:   $vgpr10 = IMAGE_SAMPLE_LZ_V1_V2_gfx12 $vgpr3, $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s128))
     ; CHECK-NEXT:   $vgpr14 = IMAGE_SAMPLE_LZ_V1_V2_gfx12 $vgpr5, $vgpr6, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 1, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 7)
diff --git a/llvm/test/CodeGen/AMDGPU/hard-clauses.mir b/llvm/test/CodeGen/AMDGPU/hard-clauses.mir
index 44b988a7121c7..1341a5916df4b 100644
--- a/llvm/test/CodeGen/AMDGPU/hard-clauses.mir
+++ b/llvm/test/CodeGen/AMDGPU/hard-clauses.mir
@@ -39,7 +39,7 @@ body: |
     ; CHECK-LABEL: name: nop2
     ; CHECK: liveins: $sgpr0_sgpr1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1 {
+    ; CHECK-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1 {
     ; CHECK-NEXT:   S_CLAUSE 2
     ; CHECK-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; CHECK-NEXT:   S_NOP 2
@@ -49,7 +49,7 @@ body: |
     ; GFX11-LABEL: name: nop2
     ; GFX11: liveins: $sgpr0_sgpr1
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1 {
+    ; GFX11-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1 {
     ; GFX11-NEXT:   S_CLAUSE 2
     ; GFX11-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; GFX11-NEXT:   S_NOP 2
@@ -59,7 +59,7 @@ body: |
     ; GFX12-LABEL: name: nop2
     ; GFX12: liveins: $sgpr0_sgpr1
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1 {
+    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1 {
     ; GFX12-NEXT:   S_CLAUSE 2
     ; GFX12-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; GFX12-NEXT:   S_NOP 2
@@ -79,7 +79,7 @@ body: |
     ; CHECK-LABEL: name: nop3
     ; CHECK: liveins: $sgpr0_sgpr1
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1 {
+    ; CHECK-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1 {
     ; CHECK-NEXT:   S_CLAUSE 2
     ; CHECK-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; CHECK-NEXT:   S_NOP 2
@@ -90,7 +90,7 @@ body: |
     ; GFX11-LABEL: name: nop3
     ; GFX11: liveins: $sgpr0_sgpr1
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1 {
+    ; GFX11-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1 {
     ; GFX11-NEXT:   S_CLAUSE 2
     ; GFX11-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; GFX11-NEXT:   S_NOP 2
@@ -101,7 +101,7 @@ body: |
     ; GFX12-LABEL: name: nop3
     ; GFX12: liveins: $sgpr0_sgpr1
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1 {
+    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1 {
     ; GFX12-NEXT:   S_CLAUSE 2
     ; GFX12-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; GFX12-NEXT:   S_NOP 2
@@ -123,7 +123,7 @@ body: |
     ; CHECK-LABEL: name: long_clause
     ; CHECK: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit-def $vgpr33, implicit-def $vgpr33_lo16, implicit-def $vgpr33_hi16, implicit-def $vgpr34, implicit-def $vgpr34_lo16, implicit-def $vgpr34_hi16, implicit-def $vgpr35, implicit-def $vgpr35_lo16, implicit-def $vgpr35_hi16, implicit-def $vgpr36, implicit-def $vgpr36_lo16, implicit-def $vgpr36_hi16, implicit-def $vgpr37, implicit-def $vgpr37_lo16, implicit-def $vgpr37_hi16, implicit-def $vgpr38, implicit-def $vgpr38_lo16, implicit-def $vgpr38_hi16, implicit-def $vgpr39, implicit-def $vgpr39_lo16, implicit-def $vgpr39_hi16, implicit-def $vgpr40, implicit-def $vgpr40_lo16, implicit-def $vgpr40_hi16, implicit-def $vgpr41, implicit-def $vgpr41_lo16, implicit-def $vgpr41_hi16, implicit-def $vgpr42, implicit-def $vgpr42_lo16, implicit-def $vgpr42_hi16, implicit-def $vgpr43, implicit-def $vgpr43_lo16, implicit-def $vgpr43_hi16, implicit-def $vgpr44, implicit-def $vgpr44_lo16, implicit-def $vgpr44_hi16, implicit-def $vgpr45, implicit-def $vgpr45_lo16, implicit-def $vgpr45_hi16, implicit-def $vgpr46, implicit-def $vgpr46_lo16, implicit-def $vgpr46_hi16, implicit-def $vgpr47, implicit-def $vgpr47_lo16, implicit-def $vgpr47_hi16, implicit-def $vgpr48, implicit-def $vgpr48_lo16, implicit-def $vgpr48_hi16, implicit-def $vgpr49, implicit-def $vgpr49_lo16, implicit-def $vgpr49_hi16, implicit-def $vgpr50, implicit-def $vgpr50_lo16, implicit-def $vgpr50_hi16, implicit-def $vgpr51, implicit-def $vgpr51_lo16, implicit-def $vgpr51_hi16, implicit-def $vgpr52, implicit-def $vgpr52_lo16, implicit-def $vgpr52_hi16, implicit-def $vgpr53, implicit-def $vgpr53_lo16, implicit-def $vgpr53_hi16, implicit-def $vgpr54, implicit-def $vgpr54_lo16, implicit-def $vgpr54_hi16, implicit-def $vgpr55, implicit-def $vgpr55_lo16, implicit-def $vgpr55_hi16, implicit-def $vgpr56, implicit-def $vgpr56_lo16, implicit-def $vgpr56_hi16, implicit-def $vgpr57, implicit-def $vgpr57_lo16, implicit-def $vgpr57_hi16, implicit-def $vgpr58, implicit-def $vgpr58_lo16, implicit-def $vgpr58_hi16, implicit-def $vgpr59, implicit-def $vgpr59_lo16, implicit-def $vgpr59_hi16, implicit-def $vgpr60, implicit-def $vgpr60_lo16, implicit-def $vgpr60_hi16, implicit-def $vgpr61, implicit-def $vgpr61_lo16, implicit-def $vgpr61_hi16, implicit-def $vgpr62, implicit-def $vgpr62_lo16, implicit-def $vgpr62_hi16, implicit-def $vgpr63, implicit-def $vgpr63_lo16, implicit-def $vgpr63_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit-def $vgpr33, implicit-def $vgpr34, implicit-def $vgpr35, implicit-def $vgpr36, implicit-def $vgpr37, implicit-def $vgpr38, implicit-def $vgpr39, implicit-def $vgpr40, implicit-def $vgpr41, implicit-def $vgpr42, implicit-def $vgpr43, implicit-def $vgpr44, implicit-def $vgpr45, implicit-def $vgpr46, implicit-def $vgpr47, implicit-def $vgpr48, implicit-def $vgpr49, implicit-def $vgpr50, implicit-def $vgpr51, implicit-def $vgpr52, implicit-def $vgpr53, implicit-def $vgpr54, implicit-def $vgpr55, implicit-def $vgpr56, implicit-def $vgpr57, implicit-def $vgpr58, implicit-def $vgpr59, implicit-def $vgpr60, implicit-def $vgpr61, implicit-def $vgpr62, implicit-def $vgpr63, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; CHECK-NEXT:   S_CLAUSE 62
     ; CHECK-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
     ; CHECK-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
@@ -189,7 +189,7 @@ body: |
     ; CHECK-NEXT:   $vgpr62 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 248, 0, 0, implicit $exec
     ; CHECK-NEXT:   $vgpr63 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 252, 0, 0, implicit $exec
     ; CHECK-NEXT: }
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr64, implicit-def $vgpr64_lo16, implicit-def $vgpr64_hi16, implicit-def $vgpr65, implicit-def $vgpr65_lo16, implicit-def $vgpr65_hi16, implicit-def $vgpr66, implicit-def $vgpr66_lo16, implicit-def $vgpr66_hi16, implicit-def $vgpr67, implicit-def $vgpr67_lo16, implicit-def $vgpr67_hi16, implicit-def $vgpr68, implicit-def $vgpr68_lo16, implicit-def $vgpr68_hi16, implicit-def $vgpr69, implicit-def $vgpr69_lo16, implicit-def $vgpr69_hi16, implicit-def $vgpr70, implicit-def $vgpr70_lo16, implicit-def $vgpr70_hi16, implicit-def $vgpr71, implicit-def $vgpr71_lo16, implicit-def $vgpr71_hi16, implicit-def $vgpr72, implicit-def $vgpr72_lo16, implicit-def $vgpr72_hi16, implicit-def $vgpr73, implicit-def $vgpr73_lo16, implicit-def $vgpr73_hi16, implicit-def $vgpr74, implicit-def $vgpr74_lo16, implicit-def $vgpr74_hi16, implicit-def $vgpr75, implicit-def $vgpr75_lo16, implicit-def $vgpr75_hi16, implicit-def $vgpr76, implicit-def $vgpr76_lo16, implicit-def $vgpr76_hi16, implicit-def $vgpr77, implicit-def $vgpr77_lo16, implicit-def $vgpr77_hi16, implicit-def $vgpr78, implicit-def $vgpr78_lo16, implicit-def $vgpr78_hi16, implicit-def $vgpr79, implicit-def $vgpr79_lo16, implicit-def $vgpr79_hi16, implicit-def $vgpr80, implicit-def $vgpr80_lo16, implicit-def $vgpr80_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr64, implicit-def $vgpr65, implicit-def $vgpr66, implicit-def $vgpr67, implicit-def $vgpr68, implicit-def $vgpr69, implicit-def $vgpr70, implicit-def $vgpr71, implicit-def $vgpr72, implicit-def $vgpr73, implicit-def $vgpr74, implicit-def $vgpr75, implicit-def $vgpr76, implicit-def $vgpr77, implicit-def $vgpr78, implicit-def $vgpr79, implicit-def $vgpr80, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; CHECK-NEXT:   S_CLAUSE 16
     ; CHECK-NEXT:   $vgpr64 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 256, 0, 0, implicit $exec
     ; CHECK-NEXT:   $vgpr65 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 260, 0, 0, implicit $exec
@@ -213,7 +213,7 @@ body: |
     ; GFX11-LABEL: name: long_clause
     ; GFX11: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-NEXT:   S_CLAUSE 31
     ; GFX11-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
@@ -248,7 +248,7 @@ body: |
     ; GFX11-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
     ; GFX11-NEXT: }
-    ; GFX11-NEXT: BUNDLE implicit-def $vgpr33, implicit-def $vgpr33_lo16, implicit-def $vgpr33_hi16, implicit-def $vgpr34, implicit-def $vgpr34_lo16, implicit-def $vgpr34_hi16, implicit-def $vgpr35, implicit-def $vgpr35_lo16, implicit-def $vgpr35_hi16, implicit-def $vgpr36, implicit-def $vgpr36_lo16, implicit-def $vgpr36_hi16, implicit-def $vgpr37, implicit-def $vgpr37_lo16, implicit-def $vgpr37_hi16, implicit-def $vgpr38, implicit-def $vgpr38_lo16, implicit-def $vgpr38_hi16, implicit-def $vgpr39, implicit-def $vgpr39_lo16, implicit-def $vgpr39_hi16, implicit-def $vgpr40, implicit-def $vgpr40_lo16, implicit-def $vgpr40_hi16, implicit-def $vgpr41, implicit-def $vgpr41_lo16, implicit-def $vgpr41_hi16, implicit-def $vgpr42, implicit-def $vgpr42_lo16, implicit-def $vgpr42_hi16, implicit-def $vgpr43, implicit-def $vgpr43_lo16, implicit-def $vgpr43_hi16, implicit-def $vgpr44, implicit-def $vgpr44_lo16, implicit-def $vgpr44_hi16, implicit-def $vgpr45, implicit-def $vgpr45_lo16, implicit-def $vgpr45_hi16, implicit-def $vgpr46, implicit-def $vgpr46_lo16, implicit-def $vgpr46_hi16, implicit-def $vgpr47, implicit-def $vgpr47_lo16, implicit-def $vgpr47_hi16, implicit-def $vgpr48, implicit-def $vgpr48_lo16, implicit-def $vgpr48_hi16, implicit-def $vgpr49, implicit-def $vgpr49_lo16, implicit-def $vgpr49_hi16, implicit-def $vgpr50, implicit-def $vgpr50_lo16, implicit-def $vgpr50_hi16, implicit-def $vgpr51, implicit-def $vgpr51_lo16, implicit-def $vgpr51_hi16, implicit-def $vgpr52, implicit-def $vgpr52_lo16, implicit-def $vgpr52_hi16, implicit-def $vgpr53, implicit-def $vgpr53_lo16, implicit-def $vgpr53_hi16, implicit-def $vgpr54, implicit-def $vgpr54_lo16, implicit-def $vgpr54_hi16, implicit-def $vgpr55, implicit-def $vgpr55_lo16, implicit-def $vgpr55_hi16, implicit-def $vgpr56, implicit-def $vgpr56_lo16, implicit-def $vgpr56_hi16, implicit-def $vgpr57, implicit-def $vgpr57_lo16, implicit-def $vgpr57_hi16, implicit-def $vgpr58, implicit-def $vgpr58_lo16, implicit-def $vgpr58_hi16, implicit-def $vgpr59, implicit-def $vgpr59_lo16, implicit-def $vgpr59_hi16, implicit-def $vgpr60, implicit-def $vgpr60_lo16, implicit-def $vgpr60_hi16, implicit-def $vgpr61, implicit-def $vgpr61_lo16, implicit-def $vgpr61_hi16, implicit-def $vgpr62, implicit-def $vgpr62_lo16, implicit-def $vgpr62_hi16, implicit-def $vgpr63, implicit-def $vgpr63_lo16, implicit-def $vgpr63_hi16, implicit-def $vgpr64, implicit-def $vgpr64_lo16, implicit-def $vgpr64_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr33, implicit-def $vgpr34, implicit-def $vgpr35, implicit-def $vgpr36, implicit-def $vgpr37, implicit-def $vgpr38, implicit-def $vgpr39, implicit-def $vgpr40, implicit-def $vgpr41, implicit-def $vgpr42, implicit-def $vgpr43, implicit-def $vgpr44, implicit-def $vgpr45, implicit-def $vgpr46, implicit-def $vgpr47, implicit-def $vgpr48, implicit-def $vgpr49, implicit-def $vgpr50, implicit-def $vgpr51, implicit-def $vgpr52, implicit-def $vgpr53, implicit-def $vgpr54, implicit-def $vgpr55, implicit-def $vgpr56, implicit-def $vgpr57, implicit-def $vgpr58, implicit-def $vgpr59, implicit-def $vgpr60, implicit-def $vgpr61, implicit-def $vgpr62, implicit-def $vgpr63, implicit-def $vgpr64, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-NEXT:   S_CLAUSE 31
     ; GFX11-NEXT:   $vgpr33 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 132, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr34 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 136, 0, 0, implicit $exec
@@ -283,7 +283,7 @@ body: |
     ; GFX11-NEXT:   $vgpr63 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 252, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr64 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 256, 0, 0, implicit $exec
     ; GFX11-NEXT: }
-    ; GFX11-NEXT: BUNDLE implicit-def $vgpr65, implicit-def $vgpr65_lo16, implicit-def $vgpr65_hi16, implicit-def $vgpr66, implicit-def $vgpr66_lo16, implicit-def $vgpr66_hi16, implicit-def $vgpr67, implicit-def $vgpr67_lo16, implicit-def $vgpr67_hi16, implicit-def $vgpr68, implicit-def $vgpr68_lo16, implicit-def $vgpr68_hi16, implicit-def $vgpr69, implicit-def $vgpr69_lo16, implicit-def $vgpr69_hi16, implicit-def $vgpr70, implicit-def $vgpr70_lo16, implicit-def $vgpr70_hi16, implicit-def $vgpr71, implicit-def $vgpr71_lo16, implicit-def $vgpr71_hi16, implicit-def $vgpr72, implicit-def $vgpr72_lo16, implicit-def $vgpr72_hi16, implicit-def $vgpr73, implicit-def $vgpr73_lo16, implicit-def $vgpr73_hi16, implicit-def $vgpr74, implicit-def $vgpr74_lo16, implicit-def $vgpr74_hi16, implicit-def $vgpr75, implicit-def $vgpr75_lo16, implicit-def $vgpr75_hi16, implicit-def $vgpr76, implicit-def $vgpr76_lo16, implicit-def $vgpr76_hi16, implicit-def $vgpr77, implicit-def $vgpr77_lo16, implicit-def $vgpr77_hi16, implicit-def $vgpr78, implicit-def $vgpr78_lo16, implicit-def $vgpr78_hi16, implicit-def $vgpr79, implicit-def $vgpr79_lo16, implicit-def $vgpr79_hi16, implicit-def $vgpr80, implicit-def $vgpr80_lo16, implicit-def $vgpr80_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX11-NEXT: BUNDLE implicit-def $vgpr65, implicit-def $vgpr66, implicit-def $vgpr67, implicit-def $vgpr68, implicit-def $vgpr69, implicit-def $vgpr70, implicit-def $vgpr71, implicit-def $vgpr72, implicit-def $vgpr73, implicit-def $vgpr74, implicit-def $vgpr75, implicit-def $vgpr76, implicit-def $vgpr77, implicit-def $vgpr78, implicit-def $vgpr79, implicit-def $vgpr80, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX11-NEXT:   S_CLAUSE 15
     ; GFX11-NEXT:   $vgpr65 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 260, 0, 0, implicit $exec
     ; GFX11-NEXT:   $vgpr66 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 264, 0, 0, implicit $exec
@@ -306,7 +306,7 @@ body: |
     ; GFX12-LABEL: name: long_clause
     ; GFX12: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr14, implicit-def $vgpr14_lo16, implicit-def $vgpr14_hi16, implicit-def $vgpr15, implicit-def $vgpr15_lo16, implicit-def $vgpr15_hi16, implicit-def $vgpr16, implicit-def $vgpr16_lo16, implicit-def $vgpr16_hi16, implicit-def $vgpr17, implicit-def $vgpr17_lo16, implicit-def $vgpr17_hi16, implicit-def $vgpr18, implicit-def $vgpr18_lo16, implicit-def $vgpr18_hi16, implicit-def $vgpr19, implicit-def $vgpr19_lo16, implicit-def $vgpr19_hi16, implicit-def $vgpr20, implicit-def $vgpr20_lo16, implicit-def $vgpr20_hi16, implicit-def $vgpr21, implicit-def $vgpr21_lo16, implicit-def $vgpr21_hi16, implicit-def $vgpr22, implicit-def $vgpr22_lo16, implicit-def $vgpr22_hi16, implicit-def $vgpr23, implicit-def $vgpr23_lo16, implicit-def $vgpr23_hi16, implicit-def $vgpr24, implicit-def $vgpr24_lo16, implicit-def $vgpr24_hi16, implicit-def $vgpr25, implicit-def $vgpr25_lo16, implicit-def $vgpr25_hi16, implicit-def $vgpr26, implicit-def $vgpr26_lo16, implicit-def $vgpr26_hi16, implicit-def $vgpr27, implicit-def $vgpr27_lo16, implicit-def $vgpr27_hi16, implicit-def $vgpr28, implicit-def $vgpr28_lo16, implicit-def $vgpr28_hi16, implicit-def $vgpr29, implicit-def $vgpr29_lo16, implicit-def $vgpr29_hi16, implicit-def $vgpr30, implicit-def $vgpr30_lo16, implicit-def $vgpr30_hi16, implicit-def $vgpr31, implicit-def $vgpr31_lo16, implicit-def $vgpr31_hi16, implicit-def $vgpr32, implicit-def $vgpr32_lo16, implicit-def $vgpr32_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX12-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr3, implicit-def $vgpr4, implicit-def $vgpr5, implicit-def $vgpr6, implicit-def $vgpr7, implicit-def $vgpr8, implicit-def $vgpr9, implicit-def $vgpr10, implicit-def $vgpr11, implicit-def $vgpr12, implicit-def $vgpr13, implicit-def $vgpr14, implicit-def $vgpr15, implicit-def $vgpr16, implicit-def $vgpr17, implicit-def $vgpr18, implicit-def $vgpr19, implicit-def $vgpr20, implicit-def $vgpr21, implicit-def $vgpr22, implicit-def $vgpr23, implicit-def $vgpr24, implicit-def $vgpr25, implicit-def $vgpr26, implicit-def $vgpr27, implicit-def $vgpr28, implicit-def $vgpr29, implicit-def $vgpr30, implicit-def $vgpr31, implicit-def $vgpr32, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX12-NEXT:   S_CLAUSE 31
     ; GFX12-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec
     ; GFX12-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 8, 0, 0, implicit $exec
@@ -341,7 +341,7 @@ body: |
     ; GFX12-NEXT:   $vgpr31 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 124, 0, 0, implicit $exec
     ; GFX12-NEXT:   $vgpr32 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 128, 0, 0, implicit $exec
     ; GFX12-NEXT: }
-    ; GFX12-NEXT: BUNDLE implicit-def $vgpr33, implicit-def $vgpr33_lo16, implicit-def $vgpr33_hi16, implicit-def $vgpr34, implicit-def $vgpr34_lo16, implicit-def $vgpr34_hi16, implicit-def $vgpr35, implicit-def $vgpr35_lo16, implicit-def $vgpr35_hi16, implicit-def $vgpr36, implicit-def $vgpr36_lo16, implicit-def $vgpr36_hi16, implicit-def $vgpr37, implicit-def $vgpr37_lo16, implicit-def $vgpr37_hi16, implicit-def $vgpr38, implicit-def $vgpr38_lo16, implicit-def $vgpr38_hi16, implicit-def $vgpr39, implicit-def $vgpr39_lo16, implicit-def $vgpr39_hi16, implicit-def $vgpr40, implicit-def $vgpr40_lo16, implicit-def $vgpr40_hi16, implicit-def $vgpr41, implicit-def $vgpr41_lo16, implicit-def $vgpr41_hi16, implicit-def $vgpr42, implicit-def $vgpr42_lo16, implicit-def $vgpr42_hi16, implicit-def $vgpr43, implicit-def $vgpr43_lo16, implicit-def $vgpr43_hi16, implicit-def $vgpr44, implicit-def $vgpr44_lo16, implicit-def $vgpr44_hi16, implicit-def $vgpr45, implicit-def $vgpr45_lo16, implicit-def $vgpr45_hi16, implicit-def $vgpr46, implicit-def $vgpr46_lo16, implicit-def $vgpr46_hi16, implicit-def $vgpr47, implicit-def $vgpr47_lo16, implicit-def $vgpr47_hi16, implicit-def $vgpr48, implicit-def $vgpr48_lo16, implicit-def $vgpr48_hi16, implicit-def $vgpr49, implicit-def $vgpr49_lo16, implicit-def $vgpr49_hi16, implicit-def $vgpr50, implicit-def $vgpr50_lo16, implicit-def $vgpr50_hi16, implicit-def $vgpr51, implicit-def $vgpr51_lo16, implicit-def $vgpr51_hi16, implicit-def $vgpr52, implicit-def $vgpr52_lo16, implicit-def $vgpr52_hi16, implicit-def $vgpr53, implicit-def $vgpr53_lo16, implicit-def $vgpr53_hi16, implicit-def $vgpr54, implicit-def $vgpr54_lo16, implicit-def $vgpr54_hi16, implicit-def $vgpr55, implicit-def $vgpr55_lo16, implicit-def $vgpr55_hi16, implicit-def $vgpr56, implicit-def $vgpr56_lo16, implicit-def $vgpr56_hi16, implicit-def $vgpr57, implicit-def $vgpr57_lo16, implicit-def $vgpr57_hi16, implicit-def $vgpr58, implicit-def $vgpr58_lo16, implicit-def $vgpr58_hi16, implicit-def $vgpr59, implicit-def $vgpr59_lo16, implicit-def $vgpr59_hi16, implicit-def $vgpr60, implicit-def $vgpr60_lo16, implicit-def $vgpr60_hi16, implicit-def $vgpr61, implicit-def $vgpr61_lo16, implicit-def $vgpr61_hi16, implicit-def $vgpr62, implicit-def $vgpr62_lo16, implicit-def $vgpr62_hi16, implicit-def $vgpr63, implicit-def $vgpr63_lo16, implicit-def $vgpr63_hi16, implicit-def $vgpr64, implicit-def $vgpr64_lo16, implicit-def $vgpr64_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX12-NEXT: BUNDLE implicit-def $vgpr33, implicit-def $vgpr34, implicit-def $vgpr35, implicit-def $vgpr36, implicit-def $vgpr37, implicit-def $vgpr38, implicit-def $vgpr39, implicit-def $vgpr40, implicit-def $vgpr41, implicit-def $vgpr42, implicit-def $vgpr43, implicit-def $vgpr44, implicit-def $vgpr45, implicit-def $vgpr46, implicit-def $vgpr47, implicit-def $vgpr48, implicit-def $vgpr49, implicit-def $vgpr50, implicit-def $vgpr51, implicit-def $vgpr52, implicit-def $vgpr53, implicit-def $vgpr54, implicit-def $vgpr55, implicit-def $vgpr56, implicit-def $vgpr57, implicit-def $vgpr58, implicit-def $vgpr59, implicit-def $vgpr60, implicit-def $vgpr61, implicit-def $vgpr62, implicit-def $vgpr63, implicit-def $vgpr64, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX12-NEXT:   S_CLAUSE 31
     ; GFX12-NEXT:   $vgpr33 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 132, 0, 0, implicit $exec
     ; GFX12-NEXT:   $vgpr34 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 136, 0, 0, implicit $exec
@@ -376,7 +376,7 @@ body: |
     ; GFX12-NEXT:   $vgpr63 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 252, 0, 0, implicit $exec
     ; GFX12-NEXT:   $vgpr64 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 256, 0, 0, implicit $exec
     ; GFX12-NEXT: }
-    ; GFX12-NEXT: BUNDLE implicit-def $vgpr65, implicit-def $vgpr65_lo16, implicit-def $vgpr65_hi16, implicit-def $vgpr66, implicit-def $vgpr66_lo16, implicit-def $vgpr66_hi16, implicit-def $vgpr67, implicit-def $vgpr67_lo16, implicit-def $vgpr67_hi16, implicit-def $vgpr68, implicit-def $vgpr68_lo16, implicit-def $vgpr68_hi16, implicit-def $vgpr69, implicit-def $vgpr69_lo16, implicit-def $vgpr69_hi16, implicit-def $vgpr70, implicit-def $vgpr70_lo16, implicit-def $vgpr70_hi16, implicit-def $vgpr71, implicit-def $vgpr71_lo16, implicit-def $vgpr71_hi16, implicit-def $vgpr72, implicit-def $vgpr72_lo16, implicit-def $vgpr72_hi16, implicit-def $vgpr73, implicit-def $vgpr73_lo16, implicit-def $vgpr73_hi16, implicit-def $vgpr74, implicit-def $vgpr74_lo16, implicit-def $vgpr74_hi16, implicit-def $vgpr75, implicit-def $vgpr75_lo16, implicit-def $vgpr75_hi16, implicit-def $vgpr76, implicit-def $vgpr76_lo16, implicit-def $vgpr76_hi16, implicit-def $vgpr77, implicit-def $vgpr77_lo16, implicit-def $vgpr77_hi16, implicit-def $vgpr78, implicit-def $vgpr78_lo16, implicit-def $vgpr78_hi16, implicit-def $vgpr79, implicit-def $vgpr79_lo16, implicit-def $vgpr79_hi16, implicit-def $vgpr80, implicit-def $vgpr80_lo16, implicit-def $vgpr80_hi16, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
+    ; GFX12-NEXT: BUNDLE implicit-def $vgpr65, implicit-def $vgpr66, implicit-def $vgpr67, implicit-def $vgpr68, implicit-def $vgpr69, implicit-def $vgpr70, implicit-def $vgpr71, implicit-def $vgpr72, implicit-def $vgpr73, implicit-def $vgpr74, implicit-def $vgpr75, implicit-def $vgpr76, implicit-def $vgpr77, implicit-def $vgpr78, implicit-def $vgpr79, implicit-def $vgpr80, implicit $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $exec {
     ; GFX12-NEXT:   S_CLAUSE 15
     ; GFX12-NEXT:   $vgpr65 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 260, 0, 0, implicit $exec
     ; GFX12-NEXT:   $vgpr66 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 264, 0, 0, implicit $exec
@@ -486,7 +486,7 @@ body: |
     ; CHECK-LABEL: name: kill
     ; CHECK: liveins: $sgpr0_sgpr1, $sgpr4
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
+    ; CHECK-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
     ; CHECK-NEXT:   S_CLAUSE 1
     ; CHECK-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; CHECK-NEXT:   KILL undef renamable $sgpr4
@@ -496,7 +496,7 @@ body: |
     ; GFX11-LABEL: name: kill
     ; GFX11: liveins: $sgpr0_sgpr1, $sgpr4
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
+    ; GFX11-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
     ; GFX11-NEXT:   S_CLAUSE 1
     ; GFX11-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; GFX11-NEXT:   KILL undef renamable $sgpr4
@@ -506,7 +506,7 @@ body: |
     ; GFX12-LABEL: name: kill
     ; GFX12: liveins: $sgpr0_sgpr1, $sgpr4
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
+    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
     ; GFX12-NEXT:   S_CLAUSE 1
     ; GFX12-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; GFX12-NEXT:   KILL undef renamable $sgpr4
@@ -526,7 +526,7 @@ body: |
     ; CHECK-LABEL: name: kill2
     ; CHECK: liveins: $sgpr0_sgpr1, $sgpr4, $sgpr5
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
+    ; CHECK-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
     ; CHECK-NEXT:   S_CLAUSE 1
     ; CHECK-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; CHECK-NEXT:   KILL undef renamable $sgpr4
@@ -537,7 +537,7 @@ body: |
     ; GFX11-LABEL: name: kill2
     ; GFX11: liveins: $sgpr0_sgpr1, $sgpr4, $sgpr5
     ; GFX11-NEXT: {{  $}}
-    ; GFX11-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
+    ; GFX11-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
     ; GFX11-NEXT:   S_CLAUSE 1
     ; GFX11-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; GFX11-NEXT:   KILL undef renamable $sgpr4
@@ -548,7 +548,7 @@ body: |
     ; GFX12-LABEL: name: kill2
     ; GFX12: liveins: $sgpr0_sgpr1, $sgpr4, $sgpr5
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
+    ; GFX12-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit $sgpr0_sgpr1, implicit undef $sgpr4 {
     ; GFX12-NEXT:   S_CLAUSE 1
     ; GFX12-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0
     ; GFX12-NEXT:   KILL undef renamable $sgpr4
@@ -570,7 +570,7 @@ body: |
     ; CHECK-LABEL: name: flat_load_atomic
     ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit $vgpr0_vgpr1, implicit $exec, implicit $flat_scr, implicit $vgpr2 {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr3, implicit-def $vgpr4, implicit $vgpr0_vgpr1, implicit $exec, implicit $flat_scr, implicit $vgpr2 {
     ; CHECK-NEXT:   S_CLAUSE 1
     ; CHECK-NEXT:   $vgpr3 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; CHECK-NEXT:   $vgpr4 = FLAT_ATOMIC_ADD_RTN $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec, implicit $flat_scr
@@ -600,7 +600,7 @@ body: |
     ; CHECK-LABEL: name: global_load_atomic
     ; CHECK: liveins: $vgpr0_vgpr1, $vgpr2
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit $vgpr0_vgpr1, implicit $exec, implicit $vgpr2 {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr3, implicit-def $vgpr4, implicit $vgpr0_vgpr1, implicit $exec, implicit $vgpr2 {
     ; CHECK-NEXT:   S_CLAUSE 1
     ; CHECK-NEXT:   $vgpr3 = GLOBAL_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec
     ; CHECK-NEXT:   $vgpr4 = GLOBAL_ATOMIC_ADD_RTN $vgpr0_vgpr1, $vgpr2, 4, 0, implicit $exec
@@ -657,7 +657,7 @@ body: |
     ; CHECK-LABEL: name: buffer_load_atomic
     ; CHECK: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, $vgpr0
     ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4, implicit $exec, implicit $vgpr0 {
+    ; CHECK-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr0, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4, implicit $exec, implicit $vgpr0 {
     ; CHECK-NEXT:   S_CLAUSE 1
     ; CHECK-NEXT:   $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, implicit $exec
     ; CHECK-NEXT:   $vgpr0 = BUFFER_ATOMIC_ADD_OFFSET_RTN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 4, 0, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/hazard-getreg-waitalu.mir b/llvm/test/CodeGen/AMDGPU/hazard-getreg-waitalu.mir
new file mode 100644
index 0000000000000..213fba9eb1156
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/hazard-getreg-waitalu.mir
@@ -0,0 +1,91 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass post-RA-hazard-rec -o - %s | FileCheck --check-prefix=GCN %s
+
+---
+name: s_getreg_mode
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_getreg_mode
+    ; GCN: $sgpr0 = S_GETREG_B32 1, implicit $mode
+    $sgpr0 = S_GETREG_B32 1, implicit $mode
+...
+
+---
+name: s_getreg_status
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_getreg_status
+    ; GCN: S_WAITCNT_DEPCTR 0
+    ; GCN-NEXT: $sgpr0 = S_GETREG_B32 2, implicit $mode
+    $sgpr0 = S_GETREG_B32 2, implicit $mode
+...
+
+---
+name: s_getreg_status_masked
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_getreg_status_masked
+    ; GCN: S_WAITCNT_DEPCTR 0
+    ; GCN-NEXT: $sgpr0 = S_GETREG_B32 66, implicit $mode
+    $sgpr0 = S_GETREG_B32 66, implicit $mode
+...
+
+---
+name: s_getreg_state_priv
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_getreg_state_priv
+    ; GCN: S_WAITCNT_DEPCTR 0
+    ; GCN-NEXT: $sgpr0 = S_GETREG_B32 4, implicit $mode
+    $sgpr0 = S_GETREG_B32 4, implicit $mode
+...
+
+---
+name: s_getreg_excp_flag_priv
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_getreg_excp_flag_priv
+    ; GCN: S_WAITCNT_DEPCTR 0
+    ; GCN-NEXT: $sgpr0 = S_GETREG_B32 17, implicit $mode
+    $sgpr0 = S_GETREG_B32 17, implicit $mode
+...
+
+---
+name: s_getreg_excp_flag_user
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_getreg_excp_flag_user
+    ; GCN: S_WAITCNT_DEPCTR 0
+    ; GCN-NEXT: $sgpr0 = S_GETREG_B32 18, implicit $mode
+    $sgpr0 = S_GETREG_B32 18, implicit $mode
+...
+
+---
+name: s_getreg_status_in_bundle
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_getreg_status_in_bundle
+    ; GCN: BUNDLE {
+    ; GCN-NEXT:   S_NOP 0
+    ; GCN-NEXT:   S_WAITCNT_DEPCTR 0
+    ; GCN-NEXT:   $sgpr0 = S_GETREG_B32 2, implicit $mode
+    ; GCN-NEXT: }
+    BUNDLE {
+      S_NOP 0
+      $sgpr0 = S_GETREG_B32 2, implicit $mode
+    }
+...
+
+---
+name: s_getreg_status_top_of_bundle
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_getreg_status_top_of_bundle
+    ; GCN: BUNDLE {
+    ; GCN-NEXT:   S_WAITCNT_DEPCTR 0
+    ; GCN-NEXT:   $sgpr0 = S_GETREG_B32 2, implicit $mode
+    ; GCN-NEXT: }
+    BUNDLE {
+      $sgpr0 = S_GETREG_B32 2, implicit $mode
+    }
+...
diff --git a/llvm/test/CodeGen/AMDGPU/hazards-gfx1250.mir b/llvm/test/CodeGen/AMDGPU/hazards-gfx1250.mir
new file mode 100644
index 0000000000000..170478539d8a9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/hazards-gfx1250.mir
@@ -0,0 +1,549 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -run-pass=post-RA-hazard-rec %s -o - | FileCheck -check-prefix=GCN %s
+
+---
+name: ds_atomic_async_barrier_arrive_b64
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; GCN-LABEL: name: ds_atomic_async_barrier_arrive_b64
+    ; GCN: liveins: $vgpr0, $vgpr1
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65507
+    ; GCN-NEXT: DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr1, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65507
+    DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 $vgpr1, 0, 0, implicit-def $asynccnt, implicit $asynccnt, implicit $exec
+...
+
+---
+name: write_s102_read_flat_scr_base_lo
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_read_flat_scr_base_lo
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+    $sgpr102 = S_MOV_B32 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+...
+
+---
+name: write_s103_read_flat_scr_base_hi
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s103_read_flat_scr_base_hi
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr103 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+    $sgpr103 = S_MOV_B32 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+...
+
+---
+name: write_s102_read_flat_scr_base
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: write_s102_read_flat_scr_base
+    ; GCN: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $src_flat_scratch_base
+    $sgpr102 = S_MOV_B32 0
+    $sgpr0_sgpr1 = S_MOV_B64 $src_flat_scratch_base
+...
+
+---
+name: write_s103_read_flat_scr_base
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: write_s103_read_flat_scr_base
+    ; GCN: $sgpr103 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $src_flat_scratch_base
+    $sgpr103 = S_MOV_B32 0
+    $sgpr0_sgpr1 = S_MOV_B64 $src_flat_scratch_base
+...
+
+---
+name: write_s102_s103_read_flat_scr_base
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: write_s102_s103_read_flat_scr_base
+    ; GCN: $sgpr102_sgpr103 = S_MOV_B64 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $sgpr0_sgpr1 = S_MOV_B64 $src_flat_scratch_base
+    $sgpr102_sgpr103 = S_MOV_B64 0
+    $sgpr0_sgpr1 = S_MOV_B64 $src_flat_scratch_base
+...
+
+---
+name: write_s102_getreg_flat_scr_base_lo
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_getreg_flat_scr_base_lo
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $sgpr1 = S_GETREG_B32 20, implicit $mode
+    $sgpr102 = S_MOV_B32 0
+    $sgpr1 = S_GETREG_B32 20, implicit $mode
+...
+
+---
+name: write_s103_getreg_flat_scr_base_hi
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s103_getreg_flat_scr_base_hi
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr103 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $sgpr1 = S_GETREG_B32 21, implicit $mode
+    $sgpr103 = S_MOV_B32 0
+    $sgpr1 = S_GETREG_B32 21, implicit $mode
+...
+
+---
+name: write_s102_s103_getreg_flat_scr_base_hi
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: write_s102_s103_getreg_flat_scr_base_hi
+    ; GCN: $sgpr102_sgpr103 = S_MOV_B64 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $sgpr1 = S_GETREG_B32 21, implicit $mode
+    $sgpr102_sgpr103 = S_MOV_B64 0
+    $sgpr1 = S_GETREG_B32 21, implicit $mode
+...
+
+---
+name: write_s102_read_flat_scr_base_lo_9_salu_valu
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_read_flat_scr_base_lo_9_salu_valu
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr1 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr2_sgpr3 = S_MOV_B64 0
+    ; GCN-NEXT: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr0, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr2 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc_lo, implicit $vcc_lo, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: $vgpr3 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
+    ; GCN-NEXT: $sgpr4 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr5 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr6 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr7 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+    $sgpr102 = S_MOV_B32 0
+    $sgpr0 = S_MOV_B32 0
+    $sgpr1 = S_MOV_B32 0
+    $sgpr2_sgpr3 = S_MOV_B64 0
+    $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr0, $sgpr0, 0, implicit $exec
+    $vgpr2 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    ; NOP does not count because it does not write SGPRs
+    S_NOP 0
+    ; DS_READ_B32 does not count because it is not SALU or VALU
+    $vgpr3 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
+    $sgpr4 = S_MOV_B32 0
+    $sgpr5 = S_MOV_B32 0
+    $sgpr6 = S_MOV_B32 0
+    $sgpr7 = S_MOV_B32 0
+    ; S_LOAD_DWORDX2_IMM does not count because it is not SALU
+    $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+...
+
+---
+name: write_s102_read_flat_scr_base_lo_10_salu_valu_expired
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_read_flat_scr_base_lo_10_salu_valu_expired
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr1 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr2_sgpr3 = S_MOV_B64 0
+    ; GCN-NEXT: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr0, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr2 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc_lo, implicit $vcc_lo, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: $vgpr3 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
+    ; GCN-NEXT: $sgpr4 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr5 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr6 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr7 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0
+    ; GCN-NEXT: $sgpr10 = S_MOV_B32 0
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+    $sgpr102 = S_MOV_B32 0
+    $sgpr0 = S_MOV_B32 0
+    $sgpr1 = S_MOV_B32 0
+    $sgpr2_sgpr3 = S_MOV_B64 0
+    $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr0, $sgpr0, 0, implicit $exec
+    $vgpr2 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    ; NOP does not count because it does not write SGPRs
+    S_NOP 0
+    ; DS_READ_B32 does not count because it is not SALU or VALU
+    $vgpr3 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
+    $sgpr4 = S_MOV_B32 0
+    $sgpr5 = S_MOV_B32 0
+    $sgpr6 = S_MOV_B32 0
+    $sgpr7 = S_MOV_B32 0
+    ; S_LOAD_DWORDX2_IMM does not count because it is not SALU
+    $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0
+    $sgpr10 = S_MOV_B32 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+...
+
+---
+name: write_s103_read_flat_scr_base_hi_9_salu_valu
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s103_read_flat_scr_base_hi_9_salu_valu
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr103 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr1 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr2_sgpr3 = S_MOV_B64 0
+    ; GCN-NEXT: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr0, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr2 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc_lo, implicit $vcc_lo, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: $vgpr3 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
+    ; GCN-NEXT: $sgpr4 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr5 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr6 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr7 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+    $sgpr103 = S_MOV_B32 0
+    $sgpr0 = S_MOV_B32 0
+    $sgpr1 = S_MOV_B32 0
+    $sgpr2_sgpr3 = S_MOV_B64 0
+    $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr0, $sgpr0, 0, implicit $exec
+    $vgpr2 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    ; NOP does not count because it does not write SGPRs
+    S_NOP 0
+    ; DS_READ_B32 does not count because it is not SALU or VALU
+    $vgpr3 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
+    $sgpr4 = S_MOV_B32 0
+    $sgpr5 = S_MOV_B32 0
+    $sgpr6 = S_MOV_B32 0
+    $sgpr7 = S_MOV_B32 0
+    ; S_LOAD_DWORDX2_IMM does not count because it is not SALU
+    $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+...
+
+---
+name: write_s103_read_flat_scr_base_hi_10_salu_valu_expired
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s103_read_flat_scr_base_hi_10_salu_valu_expired
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr103 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr1 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr2_sgpr3 = S_MOV_B64 0
+    ; GCN-NEXT: $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr0, $sgpr0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr2 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc_lo, implicit $vcc_lo, implicit $exec
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: $vgpr3 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
+    ; GCN-NEXT: $sgpr4 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr5 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr6 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr7 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0
+    ; GCN-NEXT: $sgpr10 = S_MOV_B32 0
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+    $sgpr103 = S_MOV_B32 0
+    $sgpr0 = S_MOV_B32 0
+    $sgpr1 = S_MOV_B32 0
+    $sgpr2_sgpr3 = S_MOV_B64 0
+    $vgpr1, $vcc_lo = V_ADDC_U32_e64 0, $vgpr0, $sgpr0, 0, implicit $exec
+    $vgpr2 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    ; NOP does not count because it does not write SGPRs
+    S_NOP 0
+    ; DS_READ_B32 does not count because it is not SALU or VALU
+    $vgpr3 = DS_READ_B32 $vgpr0, 0, 0, implicit $m0, implicit $exec
+    $sgpr4 = S_MOV_B32 0
+    $sgpr5 = S_MOV_B32 0
+    $sgpr6 = S_MOV_B32 0
+    $sgpr7 = S_MOV_B32 0
+    ; S_LOAD_DWORDX2_IMM does not count because it is not SALU
+    $sgpr8_sgpr9 = S_LOAD_DWORDX2_IMM renamable $sgpr4_sgpr5, 0, 0
+    $sgpr10 = S_MOV_B32 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+...
+
+---
+name: write_s102_read_flat_scr_base_hi_no_hazard
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_read_flat_scr_base_hi_no_hazard
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+    $sgpr102 = S_MOV_B32 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+...
+
+---
+name: write_s102_read_flat_scr_base_lo_expired_by_wait0
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_read_flat_scr_base_lo_expired_by_wait0
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 0
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+    $sgpr102 = S_MOV_B32 0
+    S_WAITCNT_DEPCTR 0
+    S_NOP 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+...
+
+---
+name: write_s102_read_flat_scr_base_lo_expired_by_wait_vs_sdst_sa_sdst
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_read_flat_scr_base_lo_expired_by_wait_vs_sdst_sa_sdst
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+    $sgpr102 = S_MOV_B32 0
+    S_WAITCNT_DEPCTR 61950
+    S_NOP 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+...
+
+---
+name: write_s102_read_flat_scr_base_lo_not_expired_by_wait_va_sdst_only
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_read_flat_scr_base_lo_not_expired_by_wait_va_sdst_only
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61951
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+    $sgpr102 = S_MOV_B32 0
+    S_WAITCNT_DEPCTR 61951
+    S_NOP 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+...
+
+---
+name: write_s102_read_flat_scr_base_lo_not_expired_by_wait_sa_sdst_only
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_read_flat_scr_base_lo_not_expired_by_wait_sa_sdst_only
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 65534
+    ; GCN-NEXT: S_NOP 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+    $sgpr102 = S_MOV_B32 0
+    S_WAITCNT_DEPCTR 65534
+    S_NOP 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+...
+
+---
+name: write_s102_write_s103_read_flat_scr_base_lo_read_flat_scr_base_hi
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; GCN-LABEL: name: write_s102_write_s103_read_flat_scr_base_lo_read_flat_scr_base_hi
+    ; GCN: liveins: $vgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: $sgpr102 = S_MOV_B32 0
+    ; GCN-NEXT: $sgpr103 = S_MOV_B32 0
+    ; GCN-NEXT: S_WAITCNT_DEPCTR 61950
+    ; GCN-NEXT: $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+    ; GCN-NEXT: $vgpr1 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+    $sgpr102 = S_MOV_B32 0
+    $sgpr103 = S_MOV_B32 0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+    $vgpr1 = V_ADD_U32_e32 $src_flat_scratch_base_hi, $vgpr0, implicit $exec
+...
+
+---
+name: write_s102_read_flat_scr_base_lo_cross_blocks
+tracksRegLiveness: true
+body: |
+  ; GCN-LABEL: name: write_s102_read_flat_scr_base_lo_cross_blocks
+  ; GCN: bb.0:
+  ; GCN-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; GCN-NEXT:   liveins: $vgpr0, $sgpr0
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $sgpr102 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+  ; GCN-NEXT:   $sgpr1 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr2 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr3 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr4 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr5 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr6 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr7 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr8 = S_MOV_B32 0
+  ; GCN-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit $scc
+  ; GCN-NEXT:   S_BRANCH %bb.1
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.1:
+  ; GCN-NEXT:   successors: %bb.2(0x80000000)
+  ; GCN-NEXT:   liveins: $vgpr0
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   $sgpr102 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr1 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr2 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr3 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr4 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr5 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr6 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr7 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr8 = S_MOV_B32 0
+  ; GCN-NEXT:   $sgpr9 = S_MOV_B32 0
+  ; GCN-NEXT:   S_BRANCH %bb.2
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT: bb.2:
+  ; GCN-NEXT:   liveins: $vgpr0
+  ; GCN-NEXT: {{  $}}
+  ; GCN-NEXT:   S_WAITCNT_DEPCTR 61950
+  ; GCN-NEXT:   $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+  bb.0:
+    liveins: $vgpr0, $sgpr0
+    $sgpr102 = S_ADD_U32 $sgpr0, 0, implicit-def $scc
+    $sgpr1 = S_MOV_B32 0
+    $sgpr2 = S_MOV_B32 0
+    $sgpr3 = S_MOV_B32 0
+    $sgpr4 = S_MOV_B32 0
+    $sgpr5 = S_MOV_B32 0
+    $sgpr6 = S_MOV_B32 0
+    $sgpr7 = S_MOV_B32 0
+    $sgpr8 = S_MOV_B32 0
+    S_CBRANCH_SCC0 %bb.2, implicit $scc
+    S_BRANCH %bb.1
+
+  bb.1:
+    liveins: $vgpr0
+    $sgpr102 = S_MOV_B32 0
+    $sgpr1 = S_MOV_B32 0
+    $sgpr2 = S_MOV_B32 0
+    $sgpr3 = S_MOV_B32 0
+    $sgpr4 = S_MOV_B32 0
+    $sgpr5 = S_MOV_B32 0
+    $sgpr6 = S_MOV_B32 0
+    $sgpr7 = S_MOV_B32 0
+    $sgpr8 = S_MOV_B32 0
+    $sgpr9 = S_MOV_B32 0
+    S_BRANCH %bb.2
+
+  bb.2:
+    liveins: $vgpr0
+    $vgpr0 = V_ADD_U32_e32 $src_flat_scratch_base_lo, $vgpr0, implicit $exec
+...
+
+---
+name: s_setreg_b32_hwreg_mode
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GCN-LABEL: name: s_setreg_b32_hwreg_mode
+    ; GCN: liveins: $sgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: V_NOP_e32 implicit $exec
+    ; GCN-NEXT: V_NOP_e32 implicit $exec
+    ; GCN-NEXT: S_SETREG_B32 $sgpr0, 1, implicit-def $mode, implicit $mode
+    S_SETREG_B32 $sgpr0, 1, implicit-def $mode, implicit $mode
+...
+
+---
+name: s_setreg_b32_mode
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; GCN-LABEL: name: s_setreg_b32_mode
+    ; GCN: liveins: $sgpr0
+    ; GCN-NEXT: {{  $}}
+    ; GCN-NEXT: V_NOP_e32 implicit $exec
+    ; GCN-NEXT: V_NOP_e32 implicit $exec
+    ; GCN-NEXT: S_SETREG_B32_mode $sgpr0, 1, implicit-def $mode, implicit $mode
+    S_SETREG_B32_mode $sgpr0, 1, implicit-def $mode, implicit $mode
+...
+
+---
+name: s_setreg_imm32_b32_hwreg_mode
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_setreg_imm32_b32_hwreg_mode
+    ; GCN: V_NOP_e32 implicit $exec
+    ; GCN-NEXT: V_NOP_e32 implicit $exec
+    ; GCN-NEXT: S_SETREG_IMM32_B32 1, 1, implicit-def $mode, implicit $mode
+    S_SETREG_IMM32_B32 1, 1, implicit-def $mode, implicit $mode
+...
+
+---
+name: s_setreg_imm32_b32_mode
+tracksRegLiveness: true
+body: |
+  bb.0:
+    ; GCN-LABEL: name: s_setreg_imm32_b32_mode
+    ; GCN: V_NOP_e32 implicit $exec
+    ; GCN-NEXT: V_NOP_e32 implicit $exec
+    ; GCN-NEXT: S_SETREG_IMM32_B32_mode 1, 1, implicit-def $mode, implicit $mode
+    S_SETREG_IMM32_B32_mode 1, 1, implicit-def $mode, implicit $mode
+...
diff --git a/llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll b/llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll
index b68d74b927eca..704b325210b50 100644
--- a/llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/idemponent-atomics.ll
@@ -48,7 +48,7 @@ define i32 @global_agent_release_idempotent_or(ptr addrspace(1) %in) {
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ; OPT-LABEL: @global_agent_release_idempotent_or(
 ; OPT-NEXT:  entry:
-; OPT-NEXT:    [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") release, align 4
+; OPT-NEXT:    [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") release, align 4
 ; OPT-NEXT:    ret i32 [[VAL]]
 ;
 entry:
@@ -56,6 +56,42 @@ entry:
   ret i32 %val
 }
 
+define i32 @global_agent_release_idempotent_or_no_remote(ptr addrspace(1) %in) {
+; GFX942-LABEL: global_agent_release_idempotent_or_no_remote:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    buffer_wbl2 sc1
+; GFX942-NEXT:    global_atomic_or v0, v[0:1], v2, off sc0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+; OPT-LABEL: @global_agent_release_idempotent_or_no_remote(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.remote.memory [[META0:![0-9]+]]
+; OPT-NEXT:    ret i32 [[VAL]]
+entry:
+  %val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.remote.memory !0
+  ret i32 %val
+}
+
+define i32 @global_agent_release_idempotent_or_no_fine_grained(ptr addrspace(1) %in) {
+; GFX942-LABEL: global_agent_release_idempotent_or_no_fine_grained:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    buffer_wbl2 sc1
+; GFX942-NEXT:    global_atomic_or v0, v[0:1], v2, off sc0
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+; OPT-LABEL: @global_agent_release_idempotent_or_no_fine_grained(
+; OPT-NEXT:  entry:
+; OPT-NEXT:    [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; OPT-NEXT:    ret i32 [[VAL]]
+entry:
+  %val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") release, align 4, !amdgpu.no.fine.grained.memory !0
+  ret i32 %val
+}
+
 define i32 @global_agent_acquire_release_idempotent_or(ptr addrspace(1) %in) {
 ; GFX942-LABEL: global_agent_acquire_release_idempotent_or:
 ; GFX942:       ; %bb.0: ; %entry
@@ -68,7 +104,7 @@ define i32 @global_agent_acquire_release_idempotent_or(ptr addrspace(1) %in) {
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ; OPT-LABEL: @global_agent_acquire_release_idempotent_or(
 ; OPT-NEXT:  entry:
-; OPT-NEXT:    [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") acq_rel, align 4
+; OPT-NEXT:    [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") acq_rel, align 4
 ; OPT-NEXT:    ret i32 [[VAL]]
 ;
 entry:
@@ -88,9 +124,8 @@ define i32 @global_agent_acquire_release_idempotent_or__no_fine_grained(ptr addr
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ; OPT-LABEL: @global_agent_acquire_release_idempotent_or__no_fine_grained(
 ; OPT-NEXT:  entry:
-; OPT-NEXT:    [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") acq_rel, align 4, !amdgpu.no.fine.grained.memory [[META0:![0-9]+]]
+; OPT-NEXT:    [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") acq_rel, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; OPT-NEXT:    ret i32 [[VAL]]
-;
 entry:
   %val = atomicrmw or ptr addrspace(1) %in, i32 0 syncscope("agent-one-as") acq_rel, align 4, !amdgpu.no.fine.grained.memory !0
   ret i32 %val
@@ -108,7 +143,7 @@ define i32 @global_agent_seq_cst_idempotent_or(ptr addrspace(1) %in) {
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ; OPT-LABEL: @global_agent_seq_cst_idempotent_or(
 ; OPT-NEXT:  entry:
-; OPT-NEXT:    [[VAL:%.*]] = atomicrmw or ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") seq_cst, align 4
+; OPT-NEXT:    [[VAL:%.*]] = atomicrmw add ptr addrspace(1) [[IN:%.*]], i32 0 syncscope("agent-one-as") seq_cst, align 4
 ; OPT-NEXT:    ret i32 [[VAL]]
 ;
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index 7ebd69204d87f..305461ed6b208 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -1693,12 +1693,11 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v0.h, v1.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-DL-TRUE16-NEXT:    v_perm_b32 v1, v5, v5, 0xc0c0302
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v2.l, v3.l, v0.l
 ; GFX11-DL-TRUE16-NEXT:    v_perm_b32 v2, v4, v4, 0xc0c0302
-; GFX11-DL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-DL-TRUE16-NEXT:    v_dot4_u32_u8 v0, v2, v1, v0
 ; GFX11-DL-TRUE16-NEXT:    global_store_b16 v6, v0, s[4:5]
 ; GFX11-DL-TRUE16-NEXT:    s_endpgm
@@ -2724,32 +2723,32 @@ define amdgpu_kernel void @udot4_acc8_vecMul(ptr addrspace(1) %src1,
 ; GFX11-DL-TRUE16-NEXT:    global_load_b32 v4, v0, s[2:3]
 ; GFX11-DL-TRUE16-NEXT:    global_load_d16_u8 v0, v5, s[4:5]
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(2)
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b16 v0.h, 8, v3.l
-; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b16 v1.l, 8, v4.l
 ; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 24, v3
+; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(1)
 ; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 24, v4
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b16 v0.h, 8, v3.l
+; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v1.l, v3.h, v4.h
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b16 v1.h, 8, v4.l
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v3.l, v4.l, v0.l
-; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v0.h, v0.h, v1.l
-; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v1.l, v3.h, v4.h
-; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v1.h, v2.l, v6.l
+; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v2.l, v2.l, v6.l
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v6.l, 0
+; GFX11-DL-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
+; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v0.h, v0.h, v1.h
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-DL-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v0.h
-; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v1.l
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-DL-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.h
-; GFX11-DL-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-DL-TRUE16-NEXT:    v_or_b16 v6.h, v0.h, v1.l
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-DL-TRUE16-NEXT:    v_or_b32_e32 v2, v2, v6
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 8, v2
-; GFX11-DL-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-DL-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v2.l
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v7.h, v6.l
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-DL-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v0.h
+; GFX11-DL-TRUE16-NEXT:    v_or_b16 v6.h, v1.l, v2.l
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-TRUE16-NEXT:    v_or_b32_e32 v1, v7, v6
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v3.h, v4.h, v0.l
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-DL-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
 ; GFX11-DL-TRUE16-NEXT:    global_store_b8 v5, v0, s[4:5]
 ; GFX11-DL-TRUE16-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/imm.ll b/llvm/test/CodeGen/AMDGPU/imm.ll
index 58cfd40113be2..21390003ee565 100644
--- a/llvm/test/CodeGen/AMDGPU/imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/imm.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn -mcpu=verde < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
 ; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; RUN:  llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn -mcpu=gfx942 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX942 %s
 
 ; Use a 64-bit value with lo bits that can be represented as an inline constant
 define amdgpu_kernel void @i64_imm_inline_lo(ptr addrspace(1) %out) {
@@ -25,6 +26,17 @@ define amdgpu_kernel void @i64_imm_inline_lo(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: i64_imm_inline_lo:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 5
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0x12345678
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
 entry:
   store i64 1311768464867721221, ptr addrspace(1) %out ; 0x1234567800000005
   ret void
@@ -53,6 +65,17 @@ define amdgpu_kernel void @i64_imm_inline_hi(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: i64_imm_inline_hi:
+; GFX942:       ; %bb.0: ; %entry
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0x12345678
+; GFX942-NEXT:    v_mov_b32_e32 v1, 5
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
 entry:
   store i64 21780256376, ptr addrspace(1) %out ; 0x0000000512345678
   ret void
@@ -80,6 +103,17 @@ define amdgpu_kernel void @store_imm_neg_0.0_i64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_imm_neg_0.0_i64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store i64 -9223372036854775808, ptr addrspace(1) %out
   ret void
 }
@@ -104,6 +138,16 @@ define amdgpu_kernel void @store_inline_imm_neg_0.0_i32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_neg_0.0_i32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_bfrev_b32_e32 v0, 1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store i32 -2147483648, ptr addrspace(1) %out
   ret void
 }
@@ -128,6 +172,16 @@ define amdgpu_kernel void @store_inline_imm_0.0_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_0.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float 0.0, ptr addrspace(1) %out
   ret void
 }
@@ -152,6 +206,16 @@ define amdgpu_kernel void @store_imm_neg_0.0_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_imm_neg_0.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_bfrev_b32_e32 v0, 1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float -0.0, ptr addrspace(1) %out
   ret void
 }
@@ -176,6 +240,16 @@ define amdgpu_kernel void @store_inline_imm_0.5_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_0.5_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0.5
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float 0.5, ptr addrspace(1) %out
   ret void
 }
@@ -200,6 +274,16 @@ define amdgpu_kernel void @store_inline_imm_m_0.5_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_0.5_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, -0.5
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float -0.5, ptr addrspace(1) %out
   ret void
 }
@@ -224,6 +308,16 @@ define amdgpu_kernel void @store_inline_imm_1.0_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_1.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 1.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float 1.0, ptr addrspace(1) %out
   ret void
 }
@@ -248,6 +342,16 @@ define amdgpu_kernel void @store_inline_imm_m_1.0_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_1.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, -1.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float -1.0, ptr addrspace(1) %out
   ret void
 }
@@ -272,6 +376,16 @@ define amdgpu_kernel void @store_inline_imm_2.0_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_2.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 2.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float 2.0, ptr addrspace(1) %out
   ret void
 }
@@ -296,6 +410,16 @@ define amdgpu_kernel void @store_inline_imm_m_2.0_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_2.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, -2.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float -2.0, ptr addrspace(1) %out
   ret void
 }
@@ -320,6 +444,16 @@ define amdgpu_kernel void @store_inline_imm_4.0_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_4.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 4.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float 4.0, ptr addrspace(1) %out
   ret void
 }
@@ -344,6 +478,16 @@ define amdgpu_kernel void @store_inline_imm_m_4.0_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_4.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, -4.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float -4.0, ptr addrspace(1) %out
   ret void
 }
@@ -368,6 +512,16 @@ define amdgpu_kernel void @store_inline_imm_inv_2pi_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_inv_2pi_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0.15915494
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float 0x3FC45F3060000000, ptr addrspace(1) %out
   ret void
 }
@@ -392,6 +546,16 @@ define amdgpu_kernel void @store_inline_imm_m_inv_2pi_f32(ptr addrspace(1) %out)
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_inv_2pi_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0xbe22f983
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float 0xBFC45F3060000000, ptr addrspace(1) %out
   ret void
 }
@@ -416,6 +580,16 @@ define amdgpu_kernel void @store_literal_imm_f32(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_literal_imm_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0x45800000
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store float 4096.0, ptr addrspace(1) %out
   ret void
 }
@@ -442,6 +616,17 @@ define amdgpu_kernel void @add_inline_imm_0.0_f32(ptr addrspace(1) %out, float %
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_0.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 0
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 0.0
   store float %y, ptr addrspace(1) %out
   ret void
@@ -469,6 +654,17 @@ define amdgpu_kernel void @add_inline_imm_0.5_f32(ptr addrspace(1) %out, float %
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 0.5
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_0.5_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 0.5
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 0.5
   store float %y, ptr addrspace(1) %out
   ret void
@@ -496,6 +692,17 @@ define amdgpu_kernel void @add_inline_imm_neg_0.5_f32(ptr addrspace(1) %out, flo
 ; VI-NEXT:    v_add_f32_e64 v0, s6, -0.5
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_0.5_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, -0.5
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, -0.5
   store float %y, ptr addrspace(1) %out
   ret void
@@ -523,6 +730,17 @@ define amdgpu_kernel void @add_inline_imm_1.0_f32(ptr addrspace(1) %out, float %
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 1.0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_1.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 1.0
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 1.0
   store float %y, ptr addrspace(1) %out
   ret void
@@ -550,6 +768,17 @@ define amdgpu_kernel void @add_inline_imm_neg_1.0_f32(ptr addrspace(1) %out, flo
 ; VI-NEXT:    v_add_f32_e64 v0, s6, -1.0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_1.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, -1.0
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, -1.0
   store float %y, ptr addrspace(1) %out
   ret void
@@ -577,6 +806,17 @@ define amdgpu_kernel void @add_inline_imm_2.0_f32(ptr addrspace(1) %out, float %
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 2.0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_2.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 2.0
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 2.0
   store float %y, ptr addrspace(1) %out
   ret void
@@ -604,6 +844,17 @@ define amdgpu_kernel void @add_inline_imm_neg_2.0_f32(ptr addrspace(1) %out, flo
 ; VI-NEXT:    v_add_f32_e64 v0, s6, -2.0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_2.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, -2.0
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, -2.0
   store float %y, ptr addrspace(1) %out
   ret void
@@ -631,6 +882,17 @@ define amdgpu_kernel void @add_inline_imm_4.0_f32(ptr addrspace(1) %out, float %
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 4.0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_4.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 4.0
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 4.0
   store float %y, ptr addrspace(1) %out
   ret void
@@ -658,6 +920,17 @@ define amdgpu_kernel void @add_inline_imm_neg_4.0_f32(ptr addrspace(1) %out, flo
 ; VI-NEXT:    v_add_f32_e64 v0, s6, -4.0
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_4.0_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, -4.0
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, -4.0
   store float %y, ptr addrspace(1) %out
   ret void
@@ -699,6 +972,24 @@ define amdgpu_kernel void @commute_add_inline_imm_0.5_f32(ptr addrspace(1) %out,
 ; VI-NEXT:    v_add_f32_e32 v0, 0.5, v0
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: commute_add_inline_imm_0.5_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s7, 0xf000
+; GFX942-NEXT:    s_mov_b32 s6, -1
+; GFX942-NEXT:    s_mov_b32 s10, s6
+; GFX942-NEXT:    s_mov_b32 s11, s7
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_mov_b32 s8, s2
+; GFX942-NEXT:    s_mov_b32 s9, s3
+; GFX942-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GFX942-NEXT:    s_mov_b32 s4, s0
+; GFX942-NEXT:    s_mov_b32 s5, s1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_add_f32_e32 v0, 0.5, v0
+; GFX942-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX942-NEXT:    s_endpgm
   %x = load float, ptr addrspace(1) %in
   %y = fadd float %x, 0.5
   store float %y, ptr addrspace(1) %out
@@ -741,6 +1032,24 @@ define amdgpu_kernel void @commute_add_literal_f32(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    v_add_f32_e32 v0, 0x44800000, v0
 ; VI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: commute_add_literal_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s7, 0xf000
+; GFX942-NEXT:    s_mov_b32 s6, -1
+; GFX942-NEXT:    s_mov_b32 s10, s6
+; GFX942-NEXT:    s_mov_b32 s11, s7
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_mov_b32 s8, s2
+; GFX942-NEXT:    s_mov_b32 s9, s3
+; GFX942-NEXT:    buffer_load_dword v0, off, s[8:11], 0
+; GFX942-NEXT:    s_mov_b32 s4, s0
+; GFX942-NEXT:    s_mov_b32 s5, s1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_add_f32_e32 v0, 0x44800000, v0
+; GFX942-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX942-NEXT:    s_endpgm
   %x = load float, ptr addrspace(1) %in
   %y = fadd float %x, 1024.0
   store float %y, ptr addrspace(1) %out
@@ -769,6 +1078,17 @@ define amdgpu_kernel void @add_inline_imm_1_f32(ptr addrspace(1) %out, float %x)
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 1
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_1_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 1
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 0x36a0000000000000
   store float %y, ptr addrspace(1) %out
   ret void
@@ -796,6 +1116,17 @@ define amdgpu_kernel void @add_inline_imm_2_f32(ptr addrspace(1) %out, float %x)
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 2
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_2_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 2
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 0x36b0000000000000
   store float %y, ptr addrspace(1) %out
   ret void
@@ -823,6 +1154,17 @@ define amdgpu_kernel void @add_inline_imm_16_f32(ptr addrspace(1) %out, float %x
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 16
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_16_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 16
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 0x36e0000000000000
   store float %y, ptr addrspace(1) %out
   ret void
@@ -852,6 +1194,18 @@ define amdgpu_kernel void @add_inline_imm_neg_1_f32(ptr addrspace(1) %out, float
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_1_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_add_i32 s4, s6, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, s4
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %xbc = bitcast float %x to i32
   %y = add i32 %xbc, -1
   %ybc = bitcast i32 %y to float
@@ -883,6 +1237,18 @@ define amdgpu_kernel void @add_inline_imm_neg_2_f32(ptr addrspace(1) %out, float
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_2_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_add_i32 s4, s6, -2
+; GFX942-NEXT:    v_mov_b32_e32 v0, s4
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %xbc = bitcast float %x to i32
   %y = add i32 %xbc, -2
   %ybc = bitcast i32 %y to float
@@ -914,6 +1280,18 @@ define amdgpu_kernel void @add_inline_imm_neg_16_f32(ptr addrspace(1) %out, floa
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_16_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_add_i32 s4, s6, -16
+; GFX942-NEXT:    v_mov_b32_e32 v0, s4
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %xbc = bitcast float %x to i32
   %y = add i32 %xbc, -16
   %ybc = bitcast i32 %y to float
@@ -943,6 +1321,17 @@ define amdgpu_kernel void @add_inline_imm_63_f32(ptr addrspace(1) %out, float %x
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 63
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_63_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 63
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 0x36ff800000000000
   store float %y, ptr addrspace(1) %out
   ret void
@@ -970,6 +1359,17 @@ define amdgpu_kernel void @add_inline_imm_64_f32(ptr addrspace(1) %out, float %x
 ; VI-NEXT:    v_add_f32_e64 v0, s6, 64
 ; VI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_64_f32:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s6, s[4:5], 0x2c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f32_e64 v0, s6, 64
+; GFX942-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd float %x, 0x3700000000000000
   store float %y, ptr addrspace(1) %out
   ret void
@@ -999,6 +1399,17 @@ define amdgpu_kernel void @add_inline_imm_0.0_f64(ptr addrspace(1) %out, [8 x i3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_0.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 0
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0.0
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1028,6 +1439,17 @@ define amdgpu_kernel void @add_inline_imm_0.5_f64(ptr addrspace(1) %out, [8 x i3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_0.5_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 0.5
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0.5
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1057,6 +1479,17 @@ define amdgpu_kernel void @add_inline_imm_neg_0.5_f64(ptr addrspace(1) %out, [8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_0.5_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], -0.5
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, -0.5
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1086,6 +1519,17 @@ define amdgpu_kernel void @add_inline_imm_1.0_f64(ptr addrspace(1) %out, [8 x i3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_1.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 1.0
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 1.0
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1115,6 +1559,17 @@ define amdgpu_kernel void @add_inline_imm_neg_1.0_f64(ptr addrspace(1) %out, [8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_1.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], -1.0
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, -1.0
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1144,6 +1599,17 @@ define amdgpu_kernel void @add_inline_imm_2.0_f64(ptr addrspace(1) %out, [8 x i3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_2.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 2.0
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 2.0
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1173,6 +1639,17 @@ define amdgpu_kernel void @add_inline_imm_neg_2.0_f64(ptr addrspace(1) %out, [8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_2.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], -2.0
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, -2.0
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1202,6 +1679,17 @@ define amdgpu_kernel void @add_inline_imm_4.0_f64(ptr addrspace(1) %out, [8 x i3
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_4.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 4.0
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 4.0
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1231,6 +1719,17 @@ define amdgpu_kernel void @add_inline_imm_neg_4.0_f64(ptr addrspace(1) %out, [8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_4.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], -4.0
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, -4.0
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1262,6 +1761,17 @@ define amdgpu_kernel void @add_inline_imm_inv_2pi_f64(ptr addrspace(1) %out, [8
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_inv_2pi_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 0.15915494309189532
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0x3fc45f306dc9c882
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1295,6 +1805,19 @@ define amdgpu_kernel void @add_m_inv_2pi_f64(ptr addrspace(1) %out, [8 x i32], d
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_m_inv_2pi_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0x6dc9c882
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0xbfc45f30
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], v[0:1]
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0xbfc45f306dc9c882
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1324,6 +1847,17 @@ define amdgpu_kernel void @add_inline_imm_1_f64(ptr addrspace(1) %out, [8 x i32]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_1_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 1
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0x0000000000000001
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1353,6 +1887,17 @@ define amdgpu_kernel void @add_inline_imm_2_f64(ptr addrspace(1) %out, [8 x i32]
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_2_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 2
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0x0000000000000002
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1382,6 +1927,17 @@ define amdgpu_kernel void @add_inline_imm_16_f64(ptr addrspace(1) %out, [8 x i32
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_16_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 16
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0x0000000000000010
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1409,6 +1965,17 @@ define amdgpu_kernel void @add_inline_imm_neg_1_f64(ptr addrspace(1) %out, [8 x
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_1_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    v_mov_b32_e32 v0, -1
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v1, v0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0xffffffffffffffff
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1436,6 +2003,17 @@ define amdgpu_kernel void @add_inline_imm_neg_2_f64(ptr addrspace(1) %out, [8 x
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_2_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, -2
+; GFX942-NEXT:    v_mov_b32_e32 v1, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0xfffffffffffffffe
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1463,6 +2041,17 @@ define amdgpu_kernel void @add_inline_imm_neg_16_f64(ptr addrspace(1) %out, [8 x
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_neg_16_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, -16
+; GFX942-NEXT:    v_mov_b32_e32 v1, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0xfffffffffffffff0
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1492,6 +2081,17 @@ define amdgpu_kernel void @add_inline_imm_63_f64(ptr addrspace(1) %out, [8 x i32
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_63_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 63
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0x000000000000003F
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1521,6 +2121,17 @@ define amdgpu_kernel void @add_inline_imm_64_f64(ptr addrspace(1) %out, [8 x i32
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: add_inline_imm_64_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x4c
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_add_f64 v[0:1], s[6:7], 64
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   %y = fadd double %x, 0x0000000000000040
   store double %y, ptr addrspace(1) %out
   ret void
@@ -1548,6 +2159,17 @@ define amdgpu_kernel void @store_inline_imm_0.0_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_0.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v1, v0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double 0.0, ptr addrspace(1) %out
   ret void
 }
@@ -1574,6 +2196,17 @@ define amdgpu_kernel void @store_literal_imm_neg_0.0_f64(ptr addrspace(1) %out)
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_literal_imm_neg_0.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_bfrev_b32_e32 v1, 1
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double -0.0, ptr addrspace(1) %out
   ret void
 }
@@ -1600,6 +2233,17 @@ define amdgpu_kernel void @store_inline_imm_0.5_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_0.5_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0x3fe00000
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double 0.5, ptr addrspace(1) %out
   ret void
 }
@@ -1626,6 +2270,17 @@ define amdgpu_kernel void @store_inline_imm_m_0.5_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_0.5_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0xbfe00000
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double -0.5, ptr addrspace(1) %out
   ret void
 }
@@ -1652,6 +2307,17 @@ define amdgpu_kernel void @store_inline_imm_1.0_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_1.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0x3ff00000
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double 1.0, ptr addrspace(1) %out
   ret void
 }
@@ -1678,6 +2344,17 @@ define amdgpu_kernel void @store_inline_imm_m_1.0_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_1.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0xbff00000
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double -1.0, ptr addrspace(1) %out
   ret void
 }
@@ -1704,6 +2381,17 @@ define amdgpu_kernel void @store_inline_imm_2.0_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_2.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, 2.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double 2.0, ptr addrspace(1) %out
   ret void
 }
@@ -1730,6 +2418,17 @@ define amdgpu_kernel void @store_inline_imm_m_2.0_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_2.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, -2.0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double -2.0, ptr addrspace(1) %out
   ret void
 }
@@ -1756,6 +2455,17 @@ define amdgpu_kernel void @store_inline_imm_4.0_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_4.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0x40100000
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double 4.0, ptr addrspace(1) %out
   ret void
 }
@@ -1782,6 +2492,17 @@ define amdgpu_kernel void @store_inline_imm_m_4.0_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_4.0_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0xc0100000
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double -4.0, ptr addrspace(1) %out
   ret void
 }
@@ -1808,6 +2529,17 @@ define amdgpu_kernel void @store_inv_2pi_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inv_2pi_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0x6dc9c882
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0x3fc45f30
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double 0x3fc45f306dc9c882, ptr addrspace(1) %out
   ret void
 }
@@ -1834,6 +2566,17 @@ define amdgpu_kernel void @store_inline_imm_m_inv_2pi_f64(ptr addrspace(1) %out)
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_inline_imm_m_inv_2pi_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0x6dc9c882
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0xbfc45f30
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double 0xbfc45f306dc9c882, ptr addrspace(1) %out
   ret void
 }
@@ -1860,6 +2603,17 @@ define amdgpu_kernel void @store_literal_imm_f64(ptr addrspace(1) %out) {
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: store_literal_imm_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT:    s_mov_b32 s3, 0xf000
+; GFX942-NEXT:    s_mov_b32 s2, -1
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, 0x40b00000
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX942-NEXT:    s_endpgm
   store double 4096.0, ptr addrspace(1) %out
   ret void
 }
@@ -1871,6 +2625,13 @@ define amdgpu_vs void @literal_folding(float %arg) {
 ; GCN-NEXT:    v_mul_f32_e32 v0, 0xbf4353f8, v0
 ; GCN-NEXT:    exp pos0 v1, v1, v0, v0 done
 ; GCN-NEXT:    s_endpgm
+;
+; GFX942-LABEL: literal_folding:
+; GFX942:       ; %bb.0: ; %main_body
+; GFX942-NEXT:    v_mul_f32_e32 v1, 0x3f4353f8, v0
+; GFX942-NEXT:    v_mul_f32_e32 v0, 0xbf4353f8, v0
+; GFX942-NEXT:    exp pos0 v1, v1, v0, v0 done
+; GFX942-NEXT:    s_endpgm
 main_body:
   %tmp = fmul float %arg, 0x3FE86A7F00000000
   %tmp1 = fmul float %arg, 0xBFE86A7F00000000
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
index c34c9749d553a..4d8fb8db624f8 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-av-remat-imm.mir
@@ -105,3 +105,60 @@ body:             |
 
 ...
 
+---
+name: av_mov_b64_split
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr72_sgpr73_sgpr74_sgpr75'
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       7
+body:             |
+  bb.0:
+    liveins: $vgpr0, $sgpr4_sgpr5
+
+    ; CHECK-LABEL: name: av_mov_b64_split
+    ; CHECK: liveins: $agpr6, $agpr7, $agpr8, $agpr9, $vgpr0, $sgpr4_sgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $agpr0 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec, implicit-def $agpr0_agpr1
+    ; CHECK-NEXT: $agpr1 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec, implicit-def $agpr0_agpr1
+    ; CHECK-NEXT: $agpr2 = V_ACCVGPR_WRITE_B32_e64 1, implicit $exec, implicit-def $agpr2_agpr3
+    ; CHECK-NEXT: $agpr3 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec, implicit-def $agpr2_agpr3
+    ; CHECK-NEXT: $agpr4 = V_ACCVGPR_WRITE_B32_e64 2, implicit $exec, implicit-def $agpr4_agpr5
+    ; CHECK-NEXT: $agpr5 = V_ACCVGPR_WRITE_B32_e64 0, implicit $exec, implicit-def $agpr4_agpr5
+    ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 3, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; CHECK-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; CHECK-NEXT: $agpr7 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1
+    ; CHECK-NEXT: $agpr6 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr1, implicit $exec, implicit killed $vgpr0_vgpr1
+    ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 4, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; CHECK-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; CHECK-NEXT: $agpr9 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $vgpr0_vgpr1
+    ; CHECK-NEXT: $agpr8 = V_ACCVGPR_WRITE_B32_e64 killed $vgpr1, implicit $exec, implicit killed $vgpr0_vgpr1
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr0_agpr1
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr2_agpr3
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $agpr4_agpr5
+    ; CHECK-NEXT: $vgpr0 = V_ACCVGPR_READ_B32_e64 $agpr7, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; CHECK-NEXT: $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr6, implicit $exec, implicit $vgpr0_vgpr1
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr0_vgpr1
+    ; CHECK-NEXT: $vgpr0 = V_ACCVGPR_READ_B32_e64 $agpr9, implicit $exec, implicit-def $vgpr0_vgpr1
+    ; CHECK-NEXT: $vgpr1 = V_ACCVGPR_READ_B32_e64 $agpr8, implicit $exec, implicit $vgpr0_vgpr1
+    ; CHECK-NEXT: S_NOP 0, implicit killed renamable $vgpr0_vgpr1
+    %0:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    %1:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 1, implicit $exec
+    %2:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 2, implicit $exec
+    %3:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 3, implicit $exec
+    %4:vreg_64_align2 = AV_MOV_B64_IMM_PSEUDO 4, implicit $exec
+
+    %5:areg_64_align2 = COPY %0
+    %6:areg_64_align2 = COPY %1
+    %7:areg_64_align2 = COPY %2
+    %8:areg_64_align2 = COPY %3
+    %9:areg_64_align2 = COPY %4
+
+    S_NOP 0, implicit %5
+    S_NOP 0, implicit %6
+    S_NOP 0, implicit %7
+    S_NOP 0, implicit %8
+    S_NOP 0, implicit %9
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
index c7767cb832078..45c185b6154ea 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-agpr-negative-tests.mir
@@ -16,15 +16,28 @@
     ret void
   }
 
-  define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_imm_src2() #0 {
+  define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_physreg_src2() #0 {
     ret void
   }
 
-  define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_src2_different_subreg() #0 {
+  define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first() #1 {
+    ret void
+  }
+
+  define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second() #1 {
+    ret void
+  }
+
+  define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first_physreg() #1 {
+    ret void
+  }
+
+  define amdgpu_kernel void @inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second_physreg() #1 {
     ret void
   }
 
   attributes #0 = { "amdgpu-wave-limiter"="true" "amdgpu-waves-per-eu"="8,8" }
+  attributes #1 = { "amdgpu-wave-limiter"="true" "amdgpu-waves-per-eu"="10,10" }
 ...
 
 # Inflate pattern, except the defining instruction isn't an MFMA.
@@ -320,9 +333,9 @@ body:             |
 
 ...
 
-# Non-mac variant, src2 is an immediate.
+# Non-mac variant, src2 is a physical register
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_imm_src2
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_physreg_src2
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -330,7 +343,7 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_imm_src2
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_physreg_src2
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
@@ -346,7 +359,7 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   early-clobber renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, 0, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, undef $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
@@ -380,7 +393,7 @@ body:             |
   bb.1:
     liveins: $vcc
 
-    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, 0, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, undef $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
@@ -403,9 +416,9 @@ body:             |
 
 ...
 
-# Non-mac variant, src2 is the same VGPR, but a different subregister.
+# There isn't an assignable AGPR around the first MFMA.
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_src2_different_subreg
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -413,7 +426,7 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_src2_different_subreg
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
@@ -429,18 +442,20 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
   ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit-def renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit-def renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit-def renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
   ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit killed renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit killed renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+  ; CHECK-NEXT:   early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -456,23 +471,25 @@ body:             |
   bb.0:
     S_NOP 0, implicit-def $agpr0
     renamable $sgpr0 = S_MOV_B32 0
-    undef %0.sub8:vreg_1024_align2 = V_MOV_B32_e32 0, implicit $exec
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
     renamable $sgpr1 = COPY renamable $sgpr0
     %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
     renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
-    %0.sub9:vreg_1024_align2 = COPY %0.sub8
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
 
   bb.1:
     liveins: $vcc
 
-    undef %0.sub0_sub1:vreg_1024_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %0.sub16_sub17:vreg_1024_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:vreg_1024_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0.sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23_sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31, 0, 0, 0, implicit $mode, implicit $exec
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    S_NOP 0, implicit-def %6:areg_512_align2, implicit-def %7:areg_512_align2, implicit-def %8:areg_512_align2, implicit-def %9:areg_512_align2
+    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_NOP 0, implicit %6, implicit %7, implicit %8, implicit %9
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
   bb.2:
-    ; No VGPRs available for %0
+    ; No VGPRs available for %0 or %4
     S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -481,11 +498,340 @@ body:             |
     S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
     S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
     S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
-    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# There isn't an assignable AGPR around the second MFMA.
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit-def renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit-def renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit-def renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+  ; CHECK-NEXT:   early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit killed renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit killed renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_NOP 0, implicit-def %6:areg_512_align2, implicit-def %7:areg_512_align2, implicit-def %8:areg_512_align2, implicit-def %9:areg_512_align2
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec
+    S_NOP 0, implicit %6, implicit %7, implicit %8, implicit %9
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0 or %4
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# There isn't an assignable AGPR around the first MFMA, with physreg interference
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first_physreg
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first_physreg
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+  ; CHECK-NEXT:   early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    S_NOP 0, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    S_NOP 0, implicit-def $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    S_NOP 0, implicit-def $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23
+    S_NOP 0, implicit-def $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    S_NOP 0, implicit-def $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39
+    S_NOP 0, implicit-def $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47
+    S_NOP 0, implicit-def $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55
+    S_NOP 0, implicit-def $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_NOP 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    S_NOP 0, implicit $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    S_NOP 0, implicit $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23
+    S_NOP 0, implicit $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    S_NOP 0, implicit $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39
+    S_NOP 0, implicit $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47
+    S_NOP 0, implicit $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55
+    S_NOP 0, implicit $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0 or %4
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# There isn't an assignable AGPR around the second MFMA, physreg interference
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second_physreg
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second_physreg
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+  ; CHECK-NEXT:   early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_NOP 0, implicit-def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    S_NOP 0, implicit-def $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    S_NOP 0, implicit-def $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23
+    S_NOP 0, implicit-def $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    S_NOP 0, implicit-def $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39
+    S_NOP 0, implicit-def $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47
+    S_NOP 0, implicit-def $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55
+    S_NOP 0, implicit-def $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec
+    S_NOP 0, implicit $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7
+    S_NOP 0, implicit $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    S_NOP 0, implicit $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23
+    S_NOP 0, implicit $agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+    S_NOP 0, implicit $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39
+    S_NOP 0, implicit $agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47
+    S_NOP 0, implicit $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55
+    S_NOP 0, implicit $agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0 or %4
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
index b907c134d06b1..735c7e081b21a 100644
--- a/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
+++ b/llvm/test/CodeGen/AMDGPU/inflate-reg-class-vgpr-mfma-to-av-with-load-source.mir
@@ -209,15 +209,525 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT:   liveins: $vcc, $vgpr2_vgpr3
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-  ; CHECK-NEXT:   early-clobber renamable $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr2_vgpr3, $vgpr2_vgpr3, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr16_agpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr2_vgpr3, $vgpr2_vgpr3, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %4.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %4, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# TODO: The two MFMAs are both rewritable as a set, but if we look at
+# one at a time the register class constraints of the other block the
+# transform.
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# The second MFMA has additional references to the rewritten register
+# %0 in src0/src1, not just src2.
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses_repeated_operand
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses_repeated_operand
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 killed $agpr2_agpr3, $agpr0_agpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %0.sub2_sub3, %0.sub0_sub1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses_cannot_rewrite_final_use
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses_cannot_rewrite_final_use
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+  ; CHECK-NEXT:   INLINEASM &"; use $0 ", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    INLINEASM &"; use $0 ", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, %0:vreg_512_align2
+    S_ENDPGM 0
+
+...
+
+# There is a rewrite candidate, but it is used by another MFMA which
+# does not have a tied result.
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_2nd_use_not_tied
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_2nd_use_not_tied
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr16_vgpr17
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr16_agpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0 or %3
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %4:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %4, %3.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %4, %3.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %4, %3.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %4, %3.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# Chain of 2 untied cases
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X8F16_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0 or %4
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+# Chain of 2 untied cases, but the use isn't in src2.
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_non_src2
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_non_src2
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 killed $vgpr4_vgpr5, $vgpr8_vgpr9, undef $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -240,16 +750,406 @@ body:             |
     %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
     renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
     %0.sub9:vreg_512_align2 = COPY %0.sub8
-
-  bb.1:
-    liveins: $vcc
-
-    undef %4.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %4, 0, 0, 0, implicit $mode, implicit $exec
-    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
-    S_BRANCH %bb.2
-
-  bb.2:
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %3.sub4_sub5, %3.sub8_sub9, undef %6:vreg_512_align2, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0 or %4
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# Chain of 2 untied cases, but the second mfma is a different size and
+# uses a subregister.
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_subreg
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_subreg
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
+  ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
+  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $agpr16_agpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_16X16X16F16_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $agpr2_agpr3_agpr4_agpr5, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+  ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+  ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+
+  bb.1:
+    liveins: $vcc
+
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %4:vreg_128_align2 = V_MFMA_F32_16X16X16F16_vgprcd_e64 %1, %1, %3.sub2_sub3_sub4_sub5, 0, 0, 0, implicit $mode, implicit $exec
+    S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    ; No VGPRs available for %0 or %4
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    S_ENDPGM 0
+
+...
+
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_local_split
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_local_split
+    ; CHECK: S_NOP 0, implicit-def $agpr0
+    ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0
+    ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+    ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8
+    ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 killed $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# Performs a split and inflate around the single instruction
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_instruction_split
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_instruction_split
+    ; CHECK: S_NOP 0, implicit-def $agpr0
+    ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0
+    ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+    ; CHECK-NEXT: SI_SPILL_AV64_SAVE killed $vgpr0_vgpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
+    ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8
+    ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    ; CHECK-NEXT: renamable $vgpr2_vgpr3 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 killed $vgpr2_vgpr3, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# Performs a split and inflate around the single instruction, non-tied case
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_instruction_split
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_instruction_split
+    ; CHECK: S_NOP 0, implicit-def $agpr0
+    ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0
+    ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+    ; CHECK-NEXT: SI_SPILL_AV64_SAVE killed $vgpr0_vgpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
+    ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8
+    ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
+    ; CHECK-NEXT: renamable $agpr16_agpr17 = COPY killed renamable $agpr0_agpr1
+    ; CHECK-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 killed $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr6_vgpr7_vgpr8_vgpr9, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    ; No VGPRs available for %0
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    S_ENDPGM 0
+
+...
+
+# This case does not fully use %0 after the MFMA. As a result,
+# SplitKits insert a copy bundle for the subset of used lanes instead
+# of a simple copy.
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_instruction_split_partial_uses_only
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_instruction_split_partial_uses_only
+    ; CHECK: S_NOP 0, implicit-def $agpr0
+    ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0
+    ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+    ; CHECK-NEXT: SI_SPILL_AV64_SAVE killed $vgpr0_vgpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
+    ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8
+    ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    ; CHECK-NEXT: renamable $vgpr2_vgpr3 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
+    ; CHECK-NEXT: renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 killed $vgpr2_vgpr3, $vgpr2_vgpr3, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = COPY renamable $agpr0_agpr1_agpr2_agpr3
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
     ; No VGPRs available for %0
     S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
@@ -263,16 +1163,15 @@ body:             |
     GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
     GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
     GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
     S_ENDPGM 0
 
 ...
 
-# TODO: The two MFMAs are both rewritable as a set, but if we look at
-# one at a time the register class constraints of the other block the
-# transform.
+# Untied version of previous. This case does not fully use %4 after
+# the MFMA. As a result, SplitKits insert a copy bundle for the subset
+# of used lanes instead of a simple copy,
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_instruction_split_partial_uses_only
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -280,30 +1179,112 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses
+  bb.0:
+    ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_instruction_split_partial_uses_only
+    ; CHECK: S_NOP 0, implicit-def $agpr0
+    ; CHECK-NEXT: renamable $sgpr0 = S_MOV_B32 0
+    ; CHECK-NEXT: renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $sgpr1 = COPY renamable $sgpr0
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+    ; CHECK-NEXT: SI_SPILL_AV64_SAVE killed $vgpr0_vgpr1, %stack.0, $sgpr32, 0, implicit $exec :: (store (s64) into %stack.0, align 4, addrspace 5)
+    ; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    ; CHECK-NEXT: dead renamable $vgpr9 = COPY renamable $vgpr8
+    ; CHECK-NEXT: renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    ; CHECK-NEXT: renamable $vgpr0_vgpr1 = SI_SPILL_AV64_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s64) from %stack.0, align 4, addrspace 5)
+    ; CHECK-NEXT: renamable $agpr16_agpr17 = COPY killed renamable $agpr0_agpr1
+    ; CHECK-NEXT: early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 killed $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    ; CHECK-NEXT: S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: renamable $vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = COPY killed renamable $agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15
+    ; CHECK-NEXT: renamable $vgpr2_vgpr3_vgpr4_vgpr5 = COPY renamable $agpr0_agpr1_agpr2_agpr3
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr10_vgpr11_vgpr12_vgpr13, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $vgpr14_vgpr15_vgpr16_vgpr17, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3_vgpr4_vgpr5, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    ; CHECK-NEXT: S_ENDPGM 0
+    S_NOP 0, implicit-def $agpr0
+    renamable $sgpr0 = S_MOV_B32 0
+    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    renamable $sgpr1 = COPY renamable $sgpr0
+    %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
+    renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
+    %0.sub9:vreg_512_align2 = COPY %0.sub8
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    ; No VGPRs available for %4
+    S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
+    S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
+    S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
+    S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
+    S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    S_ENDPGM 0
+
+...
+
+---
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_same_subreg
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       10
+  sgprForEXECCopy: '$sgpr100_sgpr101'
+body:             |
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_same_subreg
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
   ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
   ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
   ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
-  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
+  ; CHECK-NEXT:   renamable $vgpr11 = COPY renamable $vgpr10
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+  ; CHECK-NEXT:   renamable $vgpr6_vgpr7_vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+  ; CHECK-NEXT:   renamable $vgpr10_vgpr11_vgpr12_vgpr13 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+  ; CHECK-NEXT:   renamable $vgpr14_vgpr15_vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1)
   ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, killed $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
@@ -323,18 +1304,20 @@ body:             |
   bb.0:
     S_NOP 0, implicit-def $agpr0
     renamable $sgpr0 = S_MOV_B32 0
-    undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
+    undef %0.sub8:vreg_1024_align2 = V_MOV_B32_e32 0, implicit $exec
     renamable $sgpr1 = COPY renamable $sgpr0
     %1:vreg_64_align2 = COPY killed renamable $sgpr0_sgpr1
     renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
-    %0.sub9:vreg_512_align2 = COPY %0.sub8
+    %0.sub9:vreg_1024_align2 = COPY %0.sub8
 
   bb.1:
     liveins: $vcc
 
-    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
-    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %0.sub0_sub1_sub2_sub3:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %0.sub4_sub5_sub6_sub7:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %0.sub8_sub9_sub10_sub11:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %0.sub12_sub13_sub14_sub15:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:vreg_1024_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 0, 0, 0, implicit $mode, implicit $exec
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
@@ -357,10 +1340,8 @@ body:             |
 
 ...
 
-# The second MFMA has additional references to the rewritten register
-# %0 in src0/src1, not just src2.
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses_repeated_operand
+name:            chained_mfma_dst_user_is_vgpr
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -368,7 +1349,7 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_two_chained_uses_repeated_operand
+  ; CHECK-LABEL: name: chained_mfma_dst_user_is_vgpr
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
@@ -376,26 +1357,27 @@ body:             |
   ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
   ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
-  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = COPY killed renamable $sgpr0_sgpr1
   ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
   ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr16_vgpr17
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-  ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 killed $vgpr4_vgpr5, $vgpr2_vgpr3, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, killed renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -409,6 +1391,8 @@ body:             |
   ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
+    successors: %bb.1(0x80000000)
+
     S_NOP 0, implicit-def $agpr0
     renamable $sgpr0 = S_MOV_B32 0
     undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
@@ -418,16 +1402,17 @@ body:             |
     %0.sub9:vreg_512_align2 = COPY %0.sub8
 
   bb.1:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
     liveins: $vcc
 
-    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
-    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %0.sub2_sub3, %0.sub0_sub1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    undef %2.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    early-clobber %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %2, 0, 0, 0, implicit $mode, implicit $exec
+    early-clobber %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, %4
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
   bb.2:
-    ; No VGPRs available for %0
     S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -436,19 +1421,19 @@ body:             |
     S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
     S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
     S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
-    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
     S_ENDPGM 0
 
 ...
 
-# There is a rewrite candidate, but it is used by another MFMA which
-# does not have a tied result.
+# TODO: In this trivial case, the single copy required is cheaper than
+# the tuple copy.
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_2nd_use_not_tied
+name:            chained_mfma_dst_user_is_vgpr_small_subreg
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -456,7 +1441,7 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_2nd_use_not_tied
+  ; CHECK-LABEL: name: chained_mfma_dst_user_is_vgpr_small_subreg
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
@@ -473,8 +1458,9 @@ body:             |
   ; CHECK-NEXT:   liveins: $vcc, $vgpr16_vgpr17
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-  ; CHECK-NEXT:   renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   dead renamable $vgpr0 = nofpexcept V_CVT_F16_F32_e32 killed $vgpr0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
@@ -497,6 +1483,8 @@ body:             |
   ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
+    successors: %bb.1(0x80000000)
+
     S_NOP 0, implicit-def $agpr0
     renamable $sgpr0 = S_MOV_B32 0
     undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
@@ -506,16 +1494,17 @@ body:             |
     %0.sub9:vreg_512_align2 = COPY %0.sub8
 
   bb.1:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
     liveins: $vcc
 
-    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
-    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    undef %2.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    early-clobber %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %2, 0, 0, 0, implicit $mode, implicit $exec
+    early-clobber %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %5:vgpr_32 = nofpexcept V_CVT_F16_F32_e32 %4.sub0, implicit $mode, implicit $exec
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
   bb.2:
-    ; No VGPRs available for %0 or %3
     S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -524,18 +1513,18 @@ body:             |
     S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
     S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
     S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
-    %4:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    GLOBAL_STORE_DWORDX4_SADDR %4, %3.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %4, %3.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %4, %3.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %4, %3.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
     S_ENDPGM 0
 
 ...
 
-# Chain of 2 untied cases
+# Transitive user of the register is an MFMA with non-register src2
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain
+name:            chained_mfma_dst_user_has_imm_src2
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -543,7 +1532,7 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain
+  ; CHECK-LABEL: name: chained_mfma_dst_user_has_imm_src2
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
@@ -559,18 +1548,18 @@ body:             |
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X8F16_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X4F16_e64 $agpr0_agpr1, $vgpr18_vgpr19, 0, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -584,6 +1573,8 @@ body:             |
   ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
+    successors: %bb.1(0x80000000)
+
     S_NOP 0, implicit-def $agpr0
     renamable $sgpr0 = S_MOV_B32 0
     undef %0.sub8:vreg_512_align2 = V_MOV_B32_e32 0, implicit $exec
@@ -593,16 +1584,18 @@ body:             |
     %0.sub9:vreg_512_align2 = COPY %0.sub8
 
   bb.1:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
     liveins: $vcc
 
-    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
-    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec
+    undef %2.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    early-clobber %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %2, 0, 0, 0, implicit $mode, implicit $exec
+    early-clobber %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %4.sub0_sub1_sub2_sub3:vreg_512_align2 = V_MFMA_F32_4X4X4F16_vgprcd_e64 %4.sub0_sub1, %1, 0, 0, 0, 0, implicit $mode, implicit $exec
+
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
   bb.2:
-    ; No VGPRs available for %0 or %4
     S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -611,18 +1604,17 @@ body:             |
     S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
     S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
     S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
-    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
-    GLOBAL_STORE_DWORDX4_SADDR %5, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
+    %6:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %6, %4.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %4.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %4.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %6, %4.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
     S_ENDPGM 0
 
 ...
 
-# There isn't an assignable AGPR around the first MFMA.
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_has_untied_user
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -630,7 +1622,7 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_first
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_has_untied_user
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
@@ -638,28 +1630,25 @@ body:             |
   ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
   ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
-  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = COPY killed renamable $sgpr0_sgpr1
   ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
   ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr16_vgpr17
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-  ; CHECK-NEXT:   S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit-def renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit-def renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit-def renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
-  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit killed renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit killed renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
-  ; CHECK-NEXT:   early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr16_agpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, killed $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -684,16 +1673,14 @@ body:             |
   bb.1:
     liveins: $vcc
 
-    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    S_NOP 0, implicit-def %6:areg_512_align2, implicit-def %7:areg_512_align2, implicit-def %8:areg_512_align2, implicit-def %9:areg_512_align2
-    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
-    S_NOP 0, implicit %6, implicit %7, implicit %8, implicit %9
-    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
   bb.2:
-    ; No VGPRs available for %0 or %4
+    ; No VGPRs available for %0
     S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -711,9 +1698,8 @@ body:             |
 
 ...
 
-# There isn't an assignable AGPR around the second MFMA.
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_has_untied_user_with_vgpr_use
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -721,7 +1707,7 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_no_agprs_second
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_has_untied_user_with_vgpr_use
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
@@ -729,28 +1715,27 @@ body:             |
   ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
   ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
-  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = COPY killed renamable $sgpr0_sgpr1
   ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
   ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr16_vgpr17
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   S_NOP 0, implicit-def renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit-def renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit-def renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit-def renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
-  ; CHECK-NEXT:   early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   S_NOP 0, implicit killed renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, implicit killed renamable $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, implicit killed renamable $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47, implicit killed renamable $agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr16_vgpr17, $vgpr16_vgpr17, killed $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -775,16 +1760,15 @@ body:             |
   bb.1:
     liveins: $vcc
 
-    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
-    S_NOP 0, implicit-def %6:areg_512_align2, implicit-def %7:areg_512_align2, implicit-def %8:areg_512_align2, implicit-def %9:areg_512_align2
-    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %3, 0, 0, 0, implicit $mode, implicit $exec
-    S_NOP 0, implicit %6, implicit %7, implicit %8, implicit %9
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 27983881 /* reguse:VReg_512_Align2 */, %4
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
   bb.2:
-    ; No VGPRs available for %0 or %4
+    ; No VGPRs available for %0
     S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -802,9 +1786,8 @@ body:             |
 
 ...
 
-# Chain of 2 untied cases, but the use isn't in src2.
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_non_src2
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_has_tied_user
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -812,7 +1795,7 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_non_src2
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_has_tied_user
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
@@ -820,26 +1803,25 @@ body:             |
   ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
   ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
-  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vgpr2_vgpr3 = COPY killed renamable $sgpr0_sgpr1
   ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
   ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr2_vgpr3
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   early-clobber renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35 = V_MFMA_F32_32X32X8F16_vgprcd_e64 killed $vgpr4_vgpr5, $vgpr8_vgpr9, undef $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr16_agpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr2_vgpr3, $vgpr2_vgpr3, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_mac_e64 $vgpr2_vgpr3, $vgpr2_vgpr3, killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33_vgpr34_vgpr35
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39
@@ -864,14 +1846,14 @@ body:             |
   bb.1:
     liveins: $vcc
 
-    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
-    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %3.sub4_sub5, %3.sub8_sub9, undef %6:vreg_512_align2, 0, 0, 0, implicit $mode, implicit $exec
+    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
+    %4:vreg_512_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %4, 0, 0, 0, implicit $mode, implicit $exec
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
   bb.2:
-    ; No VGPRs available for %0 or %4
+    ; No VGPRs available for %0
     S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -889,10 +1871,9 @@ body:             |
 
 ...
 
-# Chain of 2 untied cases, but the second mfma is a different size and
-# uses a subregister.
+# Non-mac variant, src2 is an immediate.
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_subreg
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_imm_src2
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -900,7 +1881,7 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_chain_subreg
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_imm_src2
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
@@ -908,24 +1889,21 @@ body:             |
   ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
   ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
-  ; CHECK-NEXT:   renamable $vgpr18_vgpr19 = COPY killed renamable $sgpr0_sgpr1
+  ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
   ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
   ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT:   liveins: $vcc, $vgpr18_vgpr19
+  ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-  ; CHECK-NEXT:   early-clobber renamable $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15 = V_MFMA_F32_32X32X8F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31, 0, 0, 0, implicit $mode, implicit $exec
-  ; CHECK-NEXT:   renamable $vgpr0_vgpr1_vgpr2_vgpr3 = V_MFMA_F32_16X16X16F16_vgprcd_e64 $vgpr18_vgpr19, $vgpr18_vgpr19, killed $vgpr2_vgpr3_vgpr4_vgpr5, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, 0, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3 = COPY killed renamable $vgpr0_vgpr1_vgpr2_vgpr3
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -935,7 +1913,10 @@ body:             |
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
   ; CHECK-NEXT:   renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr8_agpr9_agpr10_agpr11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr12_agpr13_agpr14_agpr15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR renamable $vgpr0, renamable $agpr0_agpr1_agpr2_agpr3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+  ; CHECK-NEXT:   GLOBAL_STORE_DWORDX4_SADDR killed renamable $vgpr0, killed renamable $agpr4_agpr5_agpr6_agpr7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
   ; CHECK-NEXT:   S_ENDPGM 0
   bb.0:
     S_NOP 0, implicit-def $agpr0
@@ -949,14 +1930,12 @@ body:             |
   bb.1:
     liveins: $vcc
 
-    undef %0.sub0_sub1:vreg_512_align2 = GLOBAL_LOAD_DWORDX2 undef %2:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
-    %3:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0, 0, 0, 0, implicit $mode, implicit $exec
-    %4:vreg_128_align2 = V_MFMA_F32_16X16X16F16_vgprcd_e64 %1, %1, %3.sub2_sub3_sub4_sub5, 0, 0, 0, implicit $mode, implicit $exec
+    %0:vreg_512_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, 0, 0, 0, 0, implicit $mode, implicit $exec
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
   bb.2:
-    ; No VGPRs available for %0 or %4
+    ; No VGPRs available for %0
     S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
     S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
     S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -965,14 +1944,18 @@ body:             |
     S_NOP 0, implicit-def $vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
     S_NOP 0, implicit-def $vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55
     S_NOP 0, implicit-def $vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
-    %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    GLOBAL_STORE_DWORDX4_SADDR %5, %4, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub8_sub9_sub10_sub11, undef $sgpr0_sgpr1, 32, 0, implicit $exec :: (store (s128), align 32, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub12_sub13_sub14_sub15, undef $sgpr0_sgpr1, 48, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub0_sub1_sub2_sub3, undef $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s128), align 128, addrspace 1)
+    GLOBAL_STORE_DWORDX4_SADDR %2, %0.sub4_sub5_sub6_sub7, killed undef $sgpr0_sgpr1, 16, 0, implicit $exec :: (store (s128), addrspace 1)
     S_ENDPGM 0
 
 ...
 
+# Non-mac variant, src2 is the same VGPR, but a different subregister.
 ---
-name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_same_subreg
+name:            inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_src2_different_subreg
 tracksRegLiveness: true
 machineFunctionInfo:
   isEntryFunction: true
@@ -980,34 +1963,31 @@ machineFunctionInfo:
   occupancy:       10
   sgprForEXECCopy: '$sgpr100_sgpr101'
 body:             |
-  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_mac_vgprcd_e64_same_subreg
+  ; CHECK-LABEL: name: inflate_result_to_agpr__V_MFMA_F32_32X32X8F16_vgprcd_e64_src2_different_subreg
   ; CHECK: bb.0:
   ; CHECK-NEXT:   successors: %bb.1(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   S_NOP 0, implicit-def $agpr0
   ; CHECK-NEXT:   renamable $sgpr0 = S_MOV_B32 0
-  ; CHECK-NEXT:   renamable $vgpr10 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   renamable $vgpr8 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   renamable $sgpr1 = COPY renamable $sgpr0
   ; CHECK-NEXT:   renamable $vgpr0_vgpr1 = COPY killed renamable $sgpr0_sgpr1
   ; CHECK-NEXT:   renamable $vcc = S_AND_B64 $exec, -1, implicit-def dead $scc
-  ; CHECK-NEXT:   renamable $vgpr11 = COPY renamable $vgpr10
+  ; CHECK-NEXT:   dead renamable $vgpr9 = COPY renamable $vgpr8
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.1:
   ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK-NEXT:   liveins: $vcc, $vgpr0_vgpr1
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1)
-  ; CHECK-NEXT:   renamable $vgpr6_vgpr7_vgpr8_vgpr9 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1)
-  ; CHECK-NEXT:   renamable $vgpr10_vgpr11_vgpr12_vgpr13 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1)
-  ; CHECK-NEXT:   renamable $vgpr14_vgpr15_vgpr16_vgpr17 = GLOBAL_LOAD_DWORDX4 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s128), addrspace 1)
-  ; CHECK-NEXT:   renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   renamable $agpr0_agpr1 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   renamable $agpr16_agpr17 = GLOBAL_LOAD_DWORDX2 undef renamable $vgpr0_vgpr1, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+  ; CHECK-NEXT:   early-clobber renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_32X32X8F16_e64 $vgpr0_vgpr1, $vgpr0_vgpr1, $agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.1, implicit $vcc
   ; CHECK-NEXT:   S_BRANCH %bb.2
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   liveins: $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32_vgpr33:0x00000000FFFFFFFF
+  ; CHECK-NEXT:   liveins: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31:0x00000000FFFFFFFF
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15 = COPY killed renamable $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
   ; CHECK-NEXT:   S_NOP 0, implicit-def $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23
@@ -1034,11 +2014,9 @@ body:             |
   bb.1:
     liveins: $vcc
 
-    %0.sub0_sub1_sub2_sub3:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1)
-    %0.sub4_sub5_sub6_sub7:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1)
-    %0.sub8_sub9_sub10_sub11:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1)
-    %0.sub12_sub13_sub14_sub15:vreg_1024_align2 = GLOBAL_LOAD_DWORDX4 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s128), addrspace 1)
-    %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:vreg_1024_align2 = V_MFMA_F32_32X32X8F16_mac_vgprcd_e64 %1, %1, %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, 0, 0, 0, implicit $mode, implicit $exec
+    undef %0.sub0_sub1:vreg_1024_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0.sub16_sub17:vreg_1024_align2 = GLOBAL_LOAD_DWORDX2 undef %3:vreg_64_align2, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %0.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7_sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15:vreg_1024_align2 = V_MFMA_F32_32X32X8F16_vgprcd_e64 %1, %1, %0.sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23_sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31, 0, 0, 0, implicit $mode, implicit $exec
     S_CBRANCH_VCCNZ %bb.1, implicit $vcc
     S_BRANCH %bb.2
 
diff --git a/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count-large.ll b/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count-large.ll
new file mode 100644
index 0000000000000..63cde0d391a1b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count-large.ll
@@ -0,0 +1,72 @@
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1200 < %s | FileCheck %s
+
+; CHECK-LABEL: .shader_functions:
+
+; Use VGPRs above the input arguments.
+; CHECK-LABEL: _miss_1:
+; CHECK: .vgpr_count: 0x1d{{$}}
+
+define amdgpu_cs_chain void @_miss_1(ptr inreg %next.callee, i32 inreg %global.table, i32 inreg %max.outgoing.vgpr.count,
+                                    i32 %vcr, { i32 } %system.data,
+                                    i32 %inactive.vgpr, i32 %inactive.vgpr1, i32 %inactive.vgpr2, i32 %inactive.vgpr3,
+                                    i32 %inactive.vgpr4, i32 %inactive.vgpr5, i32 %inactive.vgpr6, i32 %inactive.vgpr7,
+                                    i32 %inactive.vgpr8, i32 %inactive.vgpr9)
+                                    local_unnamed_addr {
+entry:
+  %system.data.value = extractvalue { i32 } %system.data, 0
+  %dead.val = call i32 @llvm.amdgcn.dead.i32()
+  %is.whole.wave = call i1 @llvm.amdgcn.init.whole.wave()
+  br i1 %is.whole.wave, label %shader, label %tail
+
+shader:
+  %system.data.extract = extractvalue { i32 } %system.data, 0
+  %data.mul = mul i32 %system.data.extract, 2
+  %data.add = add i32 %data.mul, 1
+  call void asm sideeffect "; clobber v28", "~{v28}"()
+  br label %tail
+
+tail:
+  %final.vcr = phi i32 [ %vcr, %entry ], [ %data.mul, %shader ]
+  %final.sys.data = phi i32 [ %system.data.value, %entry ], [ %data.add, %shader ]
+  %final.inactive0 = phi i32 [ %inactive.vgpr, %entry ], [ %dead.val, %shader ]
+  %final.inactive1 = phi i32 [ %inactive.vgpr1, %entry ], [ %dead.val, %shader ]
+  %final.inactive2 = phi i32 [ %inactive.vgpr2, %entry ], [ %dead.val, %shader ]
+  %final.inactive3 = phi i32 [ %inactive.vgpr3, %entry ], [ %dead.val, %shader ]
+  %final.inactive4 = phi i32 [ %inactive.vgpr4, %entry ], [ %dead.val, %shader ]
+  %final.inactive5 = phi i32 [ %inactive.vgpr5, %entry ], [ %dead.val, %shader ]
+  %final.inactive6 = phi i32 [ %inactive.vgpr6, %entry ], [ %dead.val, %shader ]
+  %final.inactive7 = phi i32 [ %inactive.vgpr7, %entry ], [ %dead.val, %shader ]
+  %final.inactive8 = phi i32 [ %inactive.vgpr8, %entry ], [ %dead.val, %shader ]
+  %final.inactive9 = phi i32 [ %inactive.vgpr9, %entry ], [ %dead.val, %shader ]
+
+  %struct.init = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } poison, i32 %final.vcr, 0
+  %struct.with.data = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.init, i32 %final.sys.data, 1
+  %struct.with.inactive0 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.data, i32 %final.inactive0, 2
+  %struct.with.inactive1 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive0, i32 %final.inactive1, 3
+  %struct.with.inactive2 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive1, i32 %final.inactive2, 4
+  %struct.with.inactive3 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive2, i32 %final.inactive3, 5
+  %struct.with.inactive4 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive3, i32 %final.inactive4, 6
+  %struct.with.inactive5 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive4, i32 %final.inactive5, 7
+  %struct.with.inactive6 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive5, i32 %final.inactive6, 8
+  %struct.with.inactive7 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive6, i32 %final.inactive7, 9
+  %struct.with.inactive8 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive7, i32 %final.inactive8, 10
+  %final.struct = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive8, i32 %final.inactive9, 11
+
+  %vec.global = insertelement <4 x i32> poison, i32 %global.table, i64 0
+  %vec.max.vgpr = insertelement <4 x i32> %vec.global, i32 %max.outgoing.vgpr.count, i64 1
+  %vec.sys.data = insertelement <4 x i32> %vec.max.vgpr, i32 %final.sys.data, i64 2
+  %final.vec = insertelement <4 x i32> %vec.sys.data, i32 0, i64 3
+
+  call void (ptr, i32, <4 x i32>, { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }, i32, ...)
+        @llvm.amdgcn.cs.chain.p0.i32.v4i32.sl_i32i32i32i32i32i32i32i32i32i32i32i32s(
+        ptr %next.callee, i32 0, <4 x i32> inreg %final.vec,
+        { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %final.struct,
+        i32 1, i32 %max.outgoing.vgpr.count, i32 -1, ptr @retry_vgpr_alloc.v4i32)
+  unreachable
+}
+
+declare i32 @llvm.amdgcn.dead.i32()
+declare i1 @llvm.amdgcn.init.whole.wave()
+declare void @llvm.amdgcn.cs.chain.p0.i32.v4i32.sl_i32i32i32i32i32i32i32i32i32i32i32i32s(ptr, i32, <4 x i32>, { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }, i32 immarg, ...)
+
+declare amdgpu_cs_chain void @retry_vgpr_alloc.v4i32(<4 x i32> inreg)
diff --git a/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count-leaf.ll b/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count-leaf.ll
new file mode 100644
index 0000000000000..236f2c1c24fe3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count-leaf.ll
@@ -0,0 +1,46 @@
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1200 < %s | FileCheck %s
+
+; CHECK-LABEL: .shader_functions:
+
+; Make sure that .vgpr_count doesn't include the %inactive.vgpr registers.
+; CHECK-LABEL: leaf_shader:
+; CHECK: .vgpr_count: 0xc{{$}}
+
+; Function without calls.
+define amdgpu_cs_chain void @_leaf_shader(ptr %output.ptr, i32 inreg %input.value,
+                              i32 %active.vgpr1, i32 %active.vgpr2,
+                              i32 %inactive.vgpr1, i32 %inactive.vgpr2, i32 %inactive.vgpr3,
+                              i32 %inactive.vgpr4, i32 %inactive.vgpr5, i32 %inactive.vgpr6)
+                              local_unnamed_addr {
+entry:
+  %dead.val = call i32 @llvm.amdgcn.dead.i32()
+  %is.whole.wave = call i1 @llvm.amdgcn.init.whole.wave()
+  br i1 %is.whole.wave, label %compute, label %merge
+
+compute:
+  ; Perform a more complex computation using active VGPRs
+  %square = mul i32 %active.vgpr1, %active.vgpr1
+  %product = mul i32 %square, %active.vgpr2
+  %sum = add i32 %product, %input.value
+  %result = add i32 %sum, 42
+  br label %merge
+
+merge:
+  %final.result = phi i32 [ 0, %entry ], [ %result, %compute ]
+  %final.inactive1 = phi i32 [ %inactive.vgpr1, %entry ], [ %dead.val, %compute ]
+  %final.inactive2 = phi i32 [ %inactive.vgpr2, %entry ], [ %dead.val, %compute ]
+  %final.inactive3 = phi i32 [ %inactive.vgpr3, %entry ], [ %dead.val, %compute ]
+  %final.inactive4 = phi i32 [ %inactive.vgpr4, %entry ], [ %dead.val, %compute ]
+  %final.inactive5 = phi i32 [ %inactive.vgpr5, %entry ], [ %dead.val, %compute ]
+  %final.inactive6 = phi i32 [ %inactive.vgpr6, %entry ], [ %dead.val, %compute ]
+
+  store i32 %final.result, ptr %output.ptr, align 4
+
+  ret void
+}
+
+declare i32 @llvm.amdgcn.dead.i32()
+declare i1 @llvm.amdgcn.init.whole.wave()
+declare void @llvm.amdgcn.cs.chain.p0.i32.v4i32.sl_i32i32i32i32i32i32i32i32i32i32i32i32s(ptr, i32, <4 x i32>, { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }, i32 immarg, ...)
+
+declare amdgpu_cs_chain void @retry_vgpr_alloc.v4i32(<4 x i32> inreg)
diff --git a/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count-use-inactive.ll b/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count-use-inactive.ll
new file mode 100644
index 0000000000000..515e0db5af55b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count-use-inactive.ll
@@ -0,0 +1,74 @@
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1200 < %s | FileCheck %s
+
+; CHECK-LABEL: .shader_functions:
+
+; Make sure that .vgpr_count doesn't include the %inactive.vgpr registers.
+; The shader is free to use any of the VGPRs mapped to a %inactive.vgpr as long as it only touches its active lanes.
+; In that case, the VGPR should be included in the .vgpr_count
+; CHECK-LABEL: _miss_1:
+; CHECK: .vgpr_count: 0xd{{$}}
+
+define amdgpu_cs_chain void @_miss_1(ptr inreg %next.callee, i32 inreg %global.table, i32 inreg %max.outgoing.vgpr.count,
+                                    i32 %vcr, { i32 } %system.data,
+                                    i32 %inactive.vgpr, i32 %inactive.vgpr1, i32 %inactive.vgpr2, i32 %inactive.vgpr3,
+                                    i32 %inactive.vgpr4, i32 %inactive.vgpr5, i32 %inactive.vgpr6, i32 %inactive.vgpr7,
+                                    i32 %inactive.vgpr8, i32 %inactive.vgpr9)
+                                    local_unnamed_addr {
+entry:
+  %system.data.value = extractvalue { i32 } %system.data, 0
+  %dead.val = call i32 @llvm.amdgcn.dead.i32()
+  %is.whole.wave = call i1 @llvm.amdgcn.init.whole.wave()
+  br i1 %is.whole.wave, label %shader, label %tail
+
+shader:
+  %system.data.extract = extractvalue { i32 } %system.data, 0
+  %data.mul = mul i32 %system.data.extract, 2
+  %data.add = add i32 %data.mul, 1
+  call void asm sideeffect "; clobber VGPR for %inactive.vgpr2", "~{v12}"()
+  br label %tail
+
+tail:
+  %final.vcr = phi i32 [ %vcr, %entry ], [ %data.mul, %shader ]
+  %final.sys.data = phi i32 [ %system.data.value, %entry ], [ %data.add, %shader ]
+  %final.inactive0 = phi i32 [ %inactive.vgpr, %entry ], [ %dead.val, %shader ]
+  %final.inactive1 = phi i32 [ %inactive.vgpr1, %entry ], [ %dead.val, %shader ]
+  %final.inactive2 = phi i32 [ %inactive.vgpr2, %entry ], [ %dead.val, %shader ]
+  %final.inactive3 = phi i32 [ %inactive.vgpr3, %entry ], [ %dead.val, %shader ]
+  %final.inactive4 = phi i32 [ %inactive.vgpr4, %entry ], [ %dead.val, %shader ]
+  %final.inactive5 = phi i32 [ %inactive.vgpr5, %entry ], [ %dead.val, %shader ]
+  %final.inactive6 = phi i32 [ %inactive.vgpr6, %entry ], [ %dead.val, %shader ]
+  %final.inactive7 = phi i32 [ %inactive.vgpr7, %entry ], [ %dead.val, %shader ]
+  %final.inactive8 = phi i32 [ %inactive.vgpr8, %entry ], [ %dead.val, %shader ]
+  %final.inactive9 = phi i32 [ %inactive.vgpr9, %entry ], [ %dead.val, %shader ]
+
+  %struct.init = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } poison, i32 %final.vcr, 0
+  %struct.with.data = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.init, i32 %final.sys.data, 1
+  %struct.with.inactive0 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.data, i32 %final.inactive0, 2
+  %struct.with.inactive1 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive0, i32 %final.inactive1, 3
+  %struct.with.inactive2 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive1, i32 %final.inactive2, 4
+  %struct.with.inactive3 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive2, i32 %final.inactive3, 5
+  %struct.with.inactive4 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive3, i32 %final.inactive4, 6
+  %struct.with.inactive5 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive4, i32 %final.inactive5, 7
+  %struct.with.inactive6 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive5, i32 %final.inactive6, 8
+  %struct.with.inactive7 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive6, i32 %final.inactive7, 9
+  %struct.with.inactive8 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive7, i32 %final.inactive8, 10
+  %final.struct = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive8, i32 %final.inactive9, 11
+
+  %vec.global = insertelement <4 x i32> poison, i32 %global.table, i64 0
+  %vec.max.vgpr = insertelement <4 x i32> %vec.global, i32 %max.outgoing.vgpr.count, i64 1
+  %vec.sys.data = insertelement <4 x i32> %vec.max.vgpr, i32 %final.sys.data, i64 2
+  %final.vec = insertelement <4 x i32> %vec.sys.data, i32 0, i64 3
+
+  call void (ptr, i32, <4 x i32>, { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }, i32, ...)
+        @llvm.amdgcn.cs.chain.p0.i32.v4i32.sl_i32i32i32i32i32i32i32i32i32i32i32i32s(
+        ptr %next.callee, i32 0, <4 x i32> inreg %final.vec,
+        { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %final.struct,
+        i32 1, i32 %max.outgoing.vgpr.count, i32 -1, ptr @retry_vgpr_alloc.v4i32)
+  unreachable
+}
+
+declare i32 @llvm.amdgcn.dead.i32()
+declare i1 @llvm.amdgcn.init.whole.wave()
+declare void @llvm.amdgcn.cs.chain.p0.i32.v4i32.sl_i32i32i32i32i32i32i32i32i32i32i32i32s(ptr, i32, <4 x i32>, { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }, i32 immarg, ...)
+
+declare amdgpu_cs_chain void @retry_vgpr_alloc.v4i32(<4 x i32> inreg)
diff --git a/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count.ll b/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count.ll
new file mode 100644
index 0000000000000..2428f708951ca
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/init-whole-wave-vgpr-count.ll
@@ -0,0 +1,71 @@
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1200 < %s | FileCheck %s
+
+; CHECK-LABEL: .shader_functions:
+
+; Make sure that .vgpr_count doesn't include the %inactive.vgpr registers.
+; CHECK-LABEL: _miss_1:
+; CHECK: .vgpr_count: 0xa{{$}}
+
+define amdgpu_cs_chain void @_miss_1(ptr inreg %next.callee, i32 inreg %global.table, i32 inreg %max.outgoing.vgpr.count,
+                                    i32 %vcr, { i32 } %system.data,
+                                    i32 %inactive.vgpr, i32 %inactive.vgpr1, i32 %inactive.vgpr2, i32 %inactive.vgpr3,
+                                    i32 %inactive.vgpr4, i32 %inactive.vgpr5, i32 %inactive.vgpr6, i32 %inactive.vgpr7,
+                                    i32 %inactive.vgpr8, i32 %inactive.vgpr9)
+                                    local_unnamed_addr {
+entry:
+  %system.data.value = extractvalue { i32 } %system.data, 0
+  %dead.val = call i32 @llvm.amdgcn.dead.i32()
+  %is.whole.wave = call i1 @llvm.amdgcn.init.whole.wave()
+  br i1 %is.whole.wave, label %shader, label %tail
+
+shader:
+  %system.data.extract = extractvalue { i32 } %system.data, 0
+  %data.mul = mul i32 %system.data.extract, 2
+  %data.add = add i32 %data.mul, 1
+  br label %tail
+
+tail:
+  %final.vcr = phi i32 [ %vcr, %entry ], [ %data.mul, %shader ]
+  %final.sys.data = phi i32 [ %system.data.value, %entry ], [ %data.add, %shader ]
+  %final.inactive0 = phi i32 [ %inactive.vgpr, %entry ], [ %dead.val, %shader ]
+  %final.inactive1 = phi i32 [ %inactive.vgpr1, %entry ], [ %dead.val, %shader ]
+  %final.inactive2 = phi i32 [ %inactive.vgpr2, %entry ], [ %dead.val, %shader ]
+  %final.inactive3 = phi i32 [ %inactive.vgpr3, %entry ], [ %dead.val, %shader ]
+  %final.inactive4 = phi i32 [ %inactive.vgpr4, %entry ], [ %dead.val, %shader ]
+  %final.inactive5 = phi i32 [ %inactive.vgpr5, %entry ], [ %dead.val, %shader ]
+  %final.inactive6 = phi i32 [ %inactive.vgpr6, %entry ], [ %dead.val, %shader ]
+  %final.inactive7 = phi i32 [ %inactive.vgpr7, %entry ], [ %dead.val, %shader ]
+  %final.inactive8 = phi i32 [ %inactive.vgpr8, %entry ], [ %dead.val, %shader ]
+  %final.inactive9 = phi i32 [ %inactive.vgpr9, %entry ], [ %dead.val, %shader ]
+
+  %struct.init = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } poison, i32 %final.vcr, 0
+  %struct.with.data = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.init, i32 %final.sys.data, 1
+  %struct.with.inactive0 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.data, i32 %final.inactive0, 2
+  %struct.with.inactive1 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive0, i32 %final.inactive1, 3
+  %struct.with.inactive2 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive1, i32 %final.inactive2, 4
+  %struct.with.inactive3 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive2, i32 %final.inactive3, 5
+  %struct.with.inactive4 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive3, i32 %final.inactive4, 6
+  %struct.with.inactive5 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive4, i32 %final.inactive5, 7
+  %struct.with.inactive6 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive5, i32 %final.inactive6, 8
+  %struct.with.inactive7 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive6, i32 %final.inactive7, 9
+  %struct.with.inactive8 = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive7, i32 %final.inactive8, 10
+  %final.struct = insertvalue { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %struct.with.inactive8, i32 %final.inactive9, 11
+
+  %vec.global = insertelement <4 x i32> poison, i32 %global.table, i64 0
+  %vec.max.vgpr = insertelement <4 x i32> %vec.global, i32 %max.outgoing.vgpr.count, i64 1
+  %vec.sys.data = insertelement <4 x i32> %vec.max.vgpr, i32 %final.sys.data, i64 2
+  %final.vec = insertelement <4 x i32> %vec.sys.data, i32 0, i64 3
+
+  call void (ptr, i32, <4 x i32>, { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }, i32, ...)
+        @llvm.amdgcn.cs.chain.p0.i32.v4i32.sl_i32i32i32i32i32i32i32i32i32i32i32i32s(
+        ptr %next.callee, i32 0, <4 x i32> inreg %final.vec,
+        { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } %final.struct,
+        i32 1, i32 %max.outgoing.vgpr.count, i32 -1, ptr @retry_vgpr_alloc.v4i32)
+  unreachable
+}
+
+declare i32 @llvm.amdgcn.dead.i32()
+declare i1 @llvm.amdgcn.init.whole.wave()
+declare void @llvm.amdgcn.cs.chain.p0.i32.v4i32.sl_i32i32i32i32i32i32i32i32i32i32i32i32s(ptr, i32, <4 x i32>, { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }, i32 immarg, ...)
+
+declare amdgpu_cs_chain void @retry_vgpr_alloc.v4i32(<4 x i32> inreg)
diff --git a/llvm/test/CodeGen/AMDGPU/inline-asm-av-constraint-err.ll b/llvm/test/CodeGen/AMDGPU/inline-asm-av-constraint-err.ll
new file mode 100644
index 0000000000000..8b45f0de2300e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/inline-asm-av-constraint-err.ll
@@ -0,0 +1,27 @@
+; RUN: not llc -mtriple=amdgcn -mcpu=gfx90a -filetype=null %s 2>&1 | FileCheck %s
+
+; Make sure illegal type uses are correctly diagnosed
+
+; CHECK: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_i8(i8 %x) {
+  call void asm sideeffect "; use $0", "^VA"(i8 %x)
+  ret void
+}
+
+; CHECK: error: couldn't allocate output register for constraint 'VA'
+define i8 @def_A_i8() {
+  %ret = call i8 asm sideeffect "; def $0", "=^VA"()
+  ret i8 %ret
+}
+
+; CHECK: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_i1(i1 %x) {
+  call void asm sideeffect "; use $0", "^VA"(i1 %x)
+  ret void
+}
+
+; CHECK: error: couldn't allocate output register for constraint 'VA'
+define i1 @def_A_i1() {
+  %ret = call i1 asm sideeffect "; def $0", "=^VA"()
+  ret i1 %ret
+}
diff --git a/llvm/test/CodeGen/AMDGPU/inline-asm-av-constraint.ll b/llvm/test/CodeGen/AMDGPU/inline-asm-av-constraint.ll
new file mode 100644
index 0000000000000..407a802545906
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/inline-asm-av-constraint.ll
@@ -0,0 +1,217 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: not llc -mtriple=amdgcn -mcpu=gfx908 -filetype=null %s 2>&1 | FileCheck -check-prefix=ERR %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx90a < %s | FileCheck %s
+
+; FIXME: Shouldn't emit and
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_i16(i16 %x) {
+; CHECK-LABEL: use_A_i16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(i16 %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_f16(half %x) {
+; CHECK-LABEL: use_A_f16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(half %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_bf16(bfloat %x) {
+; CHECK-LABEL: use_A_bf16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(bfloat %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_v2i16(<2 x i16> %x) {
+; CHECK-LABEL: use_A_v2i16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(<2 x i16> %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_v2f16(<2 x half> %x) {
+; CHECK-LABEL: use_A_v2f16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(<2 x half> %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_v2bf16(<2 x bfloat> %x) {
+; CHECK-LABEL: use_A_v2bf16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(<2 x bfloat> %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_i32(i32 %x) {
+; CHECK-LABEL: use_A_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(i32 %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_f32(float %x) {
+; CHECK-LABEL: use_A_f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(float %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_i64(i64 %x) {
+; CHECK-LABEL: use_A_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v[0:1]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(i64 %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_f64(double %x) {
+; CHECK-LABEL: use_A_f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v[0:1]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(double %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_p1(ptr addrspace(1) %x) {
+; CHECK-LABEL: use_A_p1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v[0:1]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(ptr addrspace(1) %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_v32i32(<32 x i32> %x) {
+; CHECK-LABEL: use_A_v32i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(<32 x i32> %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate input reg for constraint 'VA'
+define void @use_A_v32f32(<32 x float> %x) {
+; CHECK-LABEL: use_A_v32f32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    buffer_load_dword v31, off, s[0:3], s32
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use v[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  call void asm sideeffect "; use $0", "^VA"(<32 x float> %x)
+  ret void
+}
+
+; ERR: error: couldn't allocate output register for constraint 'VA'
+define i16 @def_A_i16() {
+; CHECK-LABEL: def_A_i16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %ret = call i16 asm sideeffect "; def $0", "=^VA"()
+  ret i16 %ret
+}
+
+; ERR: error: couldn't allocate output register for constraint 'VA'
+define i32 @def_A_i32() {
+; CHECK-LABEL: def_A_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def v0
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %ret = call i32 asm sideeffect "; def $0", "=^VA"()
+  ret i32 %ret
+}
+
+; ERR: error: couldn't allocate output register for constraint 'VA'
+define ptr addrspace(1) @def_A_p1() {
+; CHECK-LABEL: def_A_p1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def v[0:1]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %ret = call ptr addrspace(1) asm sideeffect "; def $0", "=^VA"()
+  ret ptr addrspace(1) %ret
+}
diff --git a/llvm/test/CodeGen/AMDGPU/inlineasm-illegal-type.ll b/llvm/test/CodeGen/AMDGPU/inlineasm-illegal-type.ll
index 9f7f228297d47..535e02cf80c2c 100644
--- a/llvm/test/CodeGen/AMDGPU/inlineasm-illegal-type.ll
+++ b/llvm/test/CodeGen/AMDGPU/inlineasm-illegal-type.ll
@@ -18,6 +18,12 @@ define amdgpu_kernel void @v_input_output_i8() {
   ret void
 }
 
+; GCN: error: couldn't allocate input reg for constraint 'v'
+define amdgpu_kernel void @v_input_empty_struct() {
+  call void asm "", "v"({} poison)
+  ret void
+}
+
 ; SICI: error: couldn't allocate output register for constraint 's'
 ; SICI: error: couldn't allocate input reg for constraint 's'
 ; VI-NOT: error
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir
index f437dee253d00..d33a809a2ee88 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir
@@ -489,3 +489,126 @@ body: |
     S_ENDPGM 0
 ...
 
+---
+name: skip_barrier_init_imm
+body: |
+  ; CHECK-LABEL: name: skip_barrier_init_imm
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   V_NOP_e32 implicit $exec
+  ; CHECK-NEXT:   $m0 = S_MOV_B32 -1
+  ; CHECK-NEXT:   S_BARRIER_INIT_IMM -1, implicit $m0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.2
+    S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+  bb.1:
+    successors: %bb.2
+    V_NOP_e32 implicit $exec
+    $m0 = S_MOV_B32 -1
+    S_BARRIER_INIT_IMM -1, implicit $m0
+
+  bb.2:
+    S_ENDPGM 0
+...
+
+---
+name: skip_barrier_init_m0
+body: |
+  ; CHECK-LABEL: name: skip_barrier_init_m0
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   V_NOP_e32 implicit $exec
+  ; CHECK-NEXT:   $m0 = S_MOV_B32 -1
+  ; CHECK-NEXT:   S_BARRIER_INIT_M0 implicit $m0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.2
+    S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+  bb.1:
+    successors: %bb.2
+    V_NOP_e32 implicit $exec
+    $m0 = S_MOV_B32 -1
+    S_BARRIER_INIT_M0 implicit $m0
+
+  bb.2:
+    S_ENDPGM 0
+...
+
+---
+name: skip_barrier_join_imm
+body: |
+  ; CHECK-LABEL: name: skip_barrier_join_imm
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   V_NOP_e32 implicit $exec
+  ; CHECK-NEXT:   S_BARRIER_JOIN_IMM -1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.2
+    S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+  bb.1:
+    successors: %bb.2
+    V_NOP_e32 implicit $exec
+    S_BARRIER_JOIN_IMM -1
+
+  bb.2:
+    S_ENDPGM 0
+...
+
+---
+name: skip_barrier_leave
+body: |
+  ; CHECK-LABEL: name: skip_barrier_leave
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   V_NOP_e32 implicit $exec
+  ; CHECK-NEXT:   S_BARRIER_LEAVE implicit-def $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1, %bb.2
+    S_CBRANCH_EXECZ %bb.2, implicit $exec
+
+  bb.1:
+    successors: %bb.2
+    V_NOP_e32 implicit $exec
+    S_BARRIER_LEAVE implicit-def $scc
+
+  bb.2:
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-crash.ll b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-crash.ll
index fb075221706dd..58cd2f5bc11af 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-crash.ll
@@ -17,7 +17,7 @@ define fastcc i32 @foo() {
   ; CHECK-NEXT:   $exec_lo = S_MOV_B32 killed $sgpr17
   ; CHECK-NEXT:   $sgpr32 = frame-setup S_ADDK_I32 $sgpr32, 512, implicit-def dead $scc
   ; CHECK-NEXT:   $vgpr40 = V_WRITELANE_B32 killed $sgpr16, 2, undef $vgpr40
-  ; CHECK-NEXT:   BUNDLE implicit-def $sgpr16_sgpr17, implicit-def $sgpr16, implicit-def $sgpr16_lo16, implicit-def $sgpr16_hi16, implicit-def $sgpr17, implicit-def $sgpr17_lo16, implicit-def $sgpr17_hi16, implicit-def $scc {
+  ; CHECK-NEXT:   BUNDLE implicit-def $sgpr16_sgpr17, implicit-def $sgpr16, implicit-def $scc, implicit-def $sgpr17 {
   ; CHECK-NEXT:     $sgpr16_sgpr17 = S_GETPC_B64
   ; CHECK-NEXT:     $sgpr16 = S_ADD_U32 internal $sgpr16, target-flags(amdgpu-gotprel32-lo) @bar + 4, implicit-def $scc
   ; CHECK-NEXT:     $sgpr17 = S_ADDC_U32 internal $sgpr17, target-flags(amdgpu-gotprel32-hi) @bar + 12, implicit-def $scc, implicit internal $scc
diff --git a/llvm/test/CodeGen/AMDGPU/integer-canonicalizing-src-modifiers.ll b/llvm/test/CodeGen/AMDGPU/integer-canonicalizing-src-modifiers.ll
new file mode 100644
index 0000000000000..7b356d26d608a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/integer-canonicalizing-src-modifiers.ll
@@ -0,0 +1,237 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+
+; Demonstrate that the conversion of bitmasks affecting the sign bit on integers to srcmods
+; does not apply to canonicalizing instructions.
+
+define double @v_uitofp_i32_to_f64_abs(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_i32_to_f64_abs:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v0
+; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uitofp_i32_to_f64_abs:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x7fffffff, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %arg0.abs = and i32 %arg0, u0x7fffffff
+  %cvt = uitofp i32 %arg0.abs to double
+  ret double %cvt
+}
+
+define double @v_uitofp_i32_to_f64_neg(i32 %arg0) nounwind {
+; GCN-LABEL: v_uitofp_i32_to_f64_neg:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_and_b32_e32 v0, 0x80000000, v0
+; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_uitofp_i32_to_f64_neg:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x80000000, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %arg0.neg = and i32 %arg0, u0x80000000
+  %cvt = uitofp i32 %arg0.neg to double
+  ret double %cvt
+}
+
+define double @s_uitofp_i32_to_f64_abs(i32 inreg %arg0) nounwind {
+; GCN-LABEL: s_uitofp_i32_to_f64_abs:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_bitset0_b32 s16, 31
+; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], s16
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_uitofp_i32_to_f64_abs:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_bitset0_b32 s0, 31
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %arg0.abs = and i32 %arg0, u0x7fffffff
+  %cvt = uitofp i32 %arg0.abs to double
+  ret double %cvt
+}
+
+define double @s_uitofp_i32_to_f64_neg(i32 inreg %arg0) nounwind {
+; GCN-LABEL: s_uitofp_i32_to_f64_neg:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s4, s16, 0x80000000
+; GCN-NEXT:    v_cvt_f64_u32_e32 v[0:1], s4
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_uitofp_i32_to_f64_neg:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_and_b32 s0, s0, 0x80000000
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %arg0.neg = and i32 %arg0, u0x80000000
+  %cvt = uitofp i32 %arg0.neg to double
+  ret double %cvt
+}
+
+define half @v_uitofp_i16_to_f16_abs(i16 %arg0) nounwind {
+; GFX7-LABEL: v_uitofp_i16_to_f16_abs:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
+; GFX7-NEXT:    v_cvt_f32_u32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uitofp_i16_to_f16_abs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
+; GFX9-NEXT:    v_cvt_f16_u16_e32 v0, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_uitofp_i16_to_f16_abs:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0x7fff, v0.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_u16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_uitofp_i16_to_f16_abs:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x7fff, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_u16_e32 v0, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+  %arg0.abs = and i16 %arg0, u0x7fff
+  %cvt = uitofp i16 %arg0.abs to half
+  ret half %cvt
+}
+
+define half @v_uitofp_i16_to_f16_neg(i16 %arg0) nounwind {
+; GFX7-LABEL: v_uitofp_i16_to_f16_neg:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v0, 0x8000, v0
+; GFX7-NEXT:    v_cvt_f32_u32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_uitofp_i16_to_f16_neg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff8000, v0
+; GFX9-NEXT:    v_cvt_f16_u16_e32 v0, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_uitofp_i16_to_f16_neg:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_and_b16 v0.l, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_u16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_uitofp_i16_to_f16_neg:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff8000, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_u16_e32 v0, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+  %arg0.neg = and i16 %arg0, u0x8000
+  %cvt = uitofp i16 %arg0.neg to half
+  ret half %cvt
+}
+
+define half @s_uitofp_i16_to_f16_abs(i16 inreg %arg0) nounwind {
+; GFX7-LABEL: s_uitofp_i16_to_f16_abs:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_and_b32 s4, s16, 0x7fff
+; GFX7-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: s_uitofp_i16_to_f16_abs:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s4, s16, 0x7fff
+; GFX9-NEXT:    v_cvt_f16_u16_e32 v0, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: s_uitofp_i16_to_f16_abs:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_and_b32 s0, s0, 0x7fff
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_u16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: s_uitofp_i16_to_f16_abs:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_and_b32 s0, s0, 0x7fff
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_u16_e32 v0, s0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+  %arg0.abs = and i16 %arg0, u0x7fff
+  %cvt = uitofp i16 %arg0.abs to half
+  ret half %cvt
+}
+
+define half @s_uitofp_i16_to_f16_neg(i16 inreg %arg0) nounwind {
+; GFX7-LABEL: s_uitofp_i16_to_f16_neg:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    s_and_b32 s4, s16, 0x8000
+; GFX7-NEXT:    v_cvt_f32_u32_e32 v0, s4
+; GFX7-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT:    v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: s_uitofp_i16_to_f16_neg:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s4, s16, 0x8000
+; GFX9-NEXT:    v_cvt_f16_u16_e32 v0, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: s_uitofp_i16_to_f16_neg:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    s_and_b32 s0, s0, 0x8000
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT:    v_cvt_f16_u16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: s_uitofp_i16_to_f16_neg:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    s_and_b32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    v_cvt_f16_u16_e32 v0, s0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+  %arg0.neg = and i16 %arg0, u0x8000
+  %cvt = uitofp i16 %arg0.neg to half
+  ret half %cvt
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 742d87f099ce4..31b6b533866d4 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -1715,9 +1715,9 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mad_u16 v0.h, v0.l, v1.l, v0.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-FAKE16-LABEL: clpeak_umad_pat_i16:
@@ -1745,8 +1745,7 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
 ; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-GISEL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
 ; GFX11-GISEL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-GISEL-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-FAKE16-LABEL: clpeak_umad_pat_i16:
@@ -1777,9 +1776,9 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
 ; GFX1200-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1200-SDAG-TRUE16-NEXT:    v_mad_u16 v0.h, v0.l, v1.l, v0.l
 ; GFX1200-SDAG-TRUE16-NEXT:    v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX1200-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-SDAG-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX1200-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1200-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1200-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1200-SDAG-FAKE16-LABEL: clpeak_umad_pat_i16:
@@ -1815,8 +1814,7 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
 ; GFX1200-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1200-GISEL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
 ; GFX1200-GISEL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX1200-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1200-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1200-GISEL-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1200-GISEL-FAKE16-LABEL: clpeak_umad_pat_i16:
@@ -9363,9 +9361,9 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mad_u16 v0.h, v0.l, v0.h, v0.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-FAKE16-LABEL: clpeak_umad_pat_i16_x2:
@@ -9409,8 +9407,7 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
 ; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-GISEL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
 ; GFX11-GISEL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-GISEL-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-FAKE16-LABEL: clpeak_umad_pat_i16_x2:
@@ -9457,9 +9454,9 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
 ; GFX1200-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1200-SDAG-TRUE16-NEXT:    v_mad_u16 v0.h, v0.l, v0.h, v0.l
 ; GFX1200-SDAG-TRUE16-NEXT:    v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX1200-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1200-SDAG-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX1200-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1200-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1200-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1200-SDAG-FAKE16-LABEL: clpeak_umad_pat_i16_x2:
@@ -9511,8 +9508,7 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
 ; GFX1200-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1200-GISEL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
 ; GFX1200-GISEL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX1200-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX1200-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX1200-GISEL-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX1200-GISEL-FAKE16-LABEL: clpeak_umad_pat_i16_x2:
diff --git a/llvm/test/CodeGen/AMDGPU/integer-select-src-modifiers.ll b/llvm/test/CodeGen/AMDGPU/integer-select-src-modifiers.ll
new file mode 100644
index 0000000000000..b3c7ac80dd014
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/integer-select-src-modifiers.ll
@@ -0,0 +1,1011 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+
+define i32 @fneg_select_i32_1(i32 %cond, i32 %a, i32 %b) {
+; GCN-LABEL: fneg_select_i32_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, -v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_select_i32_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v2, -v1, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i32 %a, u0x80000000
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %neg.a, i32 %b
+  ret i32 %select
+}
+
+define i32 @fneg_select_i32_2(i32 %cond, i32 %a, i32 %b) {
+; GCN-LABEL: fneg_select_i32_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, -v1, v2, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_select_i32_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, -v1, v2, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i32 %a, u0x80000000
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %b, i32 %neg.a
+  ret i32 %select
+}
+
+define i32 @fneg_select_i32_both(i32 %cond, i32 %a, i32 %b) {
+; GCN-LABEL: fneg_select_i32_both:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, -v2, -v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_select_i32_both:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, -v2, -v1, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i32 %a, u0x80000000
+  %neg.b = xor i32 %b, u0x80000000
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %neg.a, i32 %neg.b
+  ret i32 %select
+}
+
+define i32 @fneg_1_fabs_2_select_i32(i32 %cond, i32 %a, i32 %b) {
+; GCN-LABEL: fneg_1_fabs_2_select_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, |v1|, -v1, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_1_fabs_2_select_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, |v1|, -v1, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i32 %a, u0x80000000
+  %abs.b = and i32 %a, u0x7fffffff
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %neg.a, i32 %abs.b
+  ret i32 %select
+}
+
+define i32 @s_fneg_select_i32_1(i32 inreg %cond, i32 inreg %a, i32 inreg %b) {
+; GCN-LABEL: s_fneg_select_i32_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_xor_b32 s4, s17, 0x80000000
+; GCN-NEXT:    s_cmp_eq_u32 s16, 0
+; GCN-NEXT:    s_cselect_b32 s4, s4, s18
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fneg_select_i32_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s1, s1, 0x80000000
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, s1, s2
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i32 %a, u0x80000000
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %neg.a, i32 %b
+  ret i32 %select
+}
+
+define i32 @s_fneg_1_fabs_2_select_i32(i32 inreg %cond, i32 %a, i32 %b) {
+; GCN-LABEL: s_fneg_1_fabs_2_select_i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_cmp_eq_u32 s16, 0
+; GCN-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, |v0|, -v0, s[4:5]
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fneg_1_fabs_2_select_i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, |v0|, -v0, s0
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i32 %a, u0x80000000
+  %abs.b = and i32 %a, u0x7fffffff
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %neg.a, i32 %abs.b
+  ret i32 %select
+}
+
+define <2 x i32> @fneg_select_v2i32_1(<2 x i32> %cond, <2 x i32> %a, <2 x i32> %b) {
+; GCN-LABEL: fneg_select_v2i32_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, -v2, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, -v3, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_select_v2i32_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, -v2, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, -v3, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor <2 x i32> %a, splat (i32 u0x80000000)
+  %cmp = icmp eq <2 x i32> %cond, zeroinitializer
+  %select = select <2 x i1> %cmp, <2 x i32> %neg.a, <2 x i32> %b
+  ret <2 x i32> %select
+}
+
+define <2 x i32> @fneg_select_v2i32_2(<2 x i32> %cond, <2 x i32> %a, <2 x i32> %b) {
+; GCN-LABEL: fneg_select_v2i32_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, -v2, v4, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, -v3, v5, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_select_v2i32_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, -v2, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, -v3, v5, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor <2 x i32> %a, splat (i32 u0x80000000)
+  %cmp = icmp eq <2 x i32> %cond, zeroinitializer
+  %select = select <2 x i1> %cmp, <2 x i32> %b, <2 x i32> %neg.a
+  ret <2 x i32> %select
+}
+
+define i32 @fabs_select_i32_1(i32 %cond, i32 %a, i32 %b) {
+; GCN-LABEL: fabs_select_i32_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, |v1|, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fabs_select_i32_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v2, |v1|, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = and i32 %a, u0x7fffffff
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %neg.a, i32 %b
+  ret i32 %select
+}
+
+define i32 @fabs_select_i32_2(i32 %cond, i32 %a, i32 %b) {
+; GCN-LABEL: fabs_select_i32_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, |v1|, v2, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fabs_select_i32_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, |v1|, v2, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = and i32 %a, u0x7fffffff
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %b, i32 %neg.a
+  ret i32 %select
+}
+
+define <2 x i32> @fneg_1_fabs_2_select_v2i32(<2 x i32> %cond, <2 x i32> %a, <2 x i32> %b) {
+; GCN-LABEL: fneg_1_fabs_2_select_v2i32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, -v2, |v2|, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, -v3, |v3|, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_1_fabs_2_select_v2i32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, -v2, |v2|, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, -v3, |v3|, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor <2 x i32> %a, splat (i32 u0x80000000)
+  %abs.b = and <2 x i32> %a, splat (i32 u0x7fffffff)
+  %cmp = icmp eq <2 x i32> %cond, zeroinitializer
+  %select = select <2 x i1> %cmp, <2 x i32> %abs.b, <2 x i32> %neg.a
+  ret <2 x i32> %select
+}
+
+define i32 @fneg_fabs_select_i32_1(i32 %cond, i32 %a, i32 %b) {
+; GCN-LABEL: fneg_fabs_select_i32_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v2, -|v1|, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_fabs_select_i32_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v2, -|v1|, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = or i32 %a, u0x80000000
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %neg.a, i32 %b
+  ret i32 %select
+}
+
+define i32 @fneg_fabs_select_i32_2(i32 %cond, i32 %a, i32 %b) {
+; GCN-LABEL: fneg_fabs_select_i32_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, -|v1|, v2, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_fabs_select_i32_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, -|v1|, v2, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = or i32 %a, u0x80000000
+  %cmp = icmp eq i32 %cond, zeroinitializer
+  %select = select i1 %cmp, i32 %b, i32 %neg.a
+  ret i32 %select
+}
+
+define <2 x i32> @fneg_fabs_select_v2i32_1(<2 x i32> %cond, <2 x i32> %a, <2 x i32> %b) {
+; GCN-LABEL: fneg_fabs_select_v2i32_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, v4, -|v2|, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, -|v3|, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_fabs_select_v2i32_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v4, -|v2|, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, -|v3|, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = or <2 x i32> %a, splat (i32 u0x80000000)
+  %cmp = icmp eq <2 x i32> %cond, zeroinitializer
+  %select = select <2 x i1> %cmp, <2 x i32> %neg.a, <2 x i32> %b
+  ret <2 x i32> %select
+}
+
+define <2 x i32> @fneg_fabs_select_v2i32_2(<2 x i32> %cond, <2 x i32> %a, <2 x i32> %b) {
+; GCN-LABEL: fneg_fabs_select_v2i32_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    v_cndmask_b32_e64 v0, -|v2|, v4, vcc
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v1
+; GCN-NEXT:    v_cndmask_b32_e64 v1, -|v3|, v5, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_fabs_select_v2i32_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, -|v2|, v4, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, -|v3|, v5, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = or <2 x i32> %a, splat (i32 u0x80000000)
+  %cmp = icmp eq <2 x i32> %cond, zeroinitializer
+  %select = select <2 x i1> %cmp, <2 x i32> %b, <2 x i32> %neg.a
+  ret <2 x i32> %select
+}
+
+
+define <2 x i32> @s_fneg_select_v2i32_1(<2 x i32> inreg %cond, <2 x i32> inreg %a, <2 x i32> inreg %b) {
+; GCN-LABEL: s_fneg_select_v2i32_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_xor_b32 s4, s19, 0x80000000
+; GCN-NEXT:    s_xor_b32 s5, s18, 0x80000000
+; GCN-NEXT:    s_cmp_eq_u32 s16, 0
+; GCN-NEXT:    s_cselect_b32 s5, s5, s20
+; GCN-NEXT:    s_cmp_eq_u32 s17, 0
+; GCN-NEXT:    s_cselect_b32 s4, s4, s21
+; GCN-NEXT:    v_mov_b32_e32 v0, s5
+; GCN-NEXT:    v_mov_b32_e32 v1, s4
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fneg_select_v2i32_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s3, s3, 0x80000000
+; GFX11-NEXT:    s_xor_b32 s2, s2, 0x80000000
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, s2, s16
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 0
+; GFX11-NEXT:    s_cselect_b32 s1, s3, s17
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor <2 x i32> %a, splat (i32 u0x80000000)
+  %cmp = icmp eq <2 x i32> %cond, zeroinitializer
+  %select = select <2 x i1> %cmp, <2 x i32> %neg.a, <2 x i32> %b
+  ret <2 x i32> %select
+}
+
+define <2 x i32> @s_fneg_fabs_select_v2i32_2(<2 x i32> inreg %cond, <2 x i32> inreg %a, <2 x i32> inreg %b) {
+; GCN-LABEL: s_fneg_fabs_select_v2i32_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_bitset1_b32 s19, 31
+; GCN-NEXT:    s_bitset1_b32 s18, 31
+; GCN-NEXT:    s_cmp_eq_u32 s16, 0
+; GCN-NEXT:    s_cselect_b32 s4, s20, s18
+; GCN-NEXT:    s_cmp_eq_u32 s17, 0
+; GCN-NEXT:    s_cselect_b32 s5, s21, s19
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fneg_fabs_select_v2i32_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_bitset1_b32 s3, 31
+; GFX11-NEXT:    s_bitset1_b32 s2, 31
+; GFX11-NEXT:    s_cmp_eq_u32 s0, 0
+; GFX11-NEXT:    s_cselect_b32 s0, s16, s2
+; GFX11-NEXT:    s_cmp_eq_u32 s1, 0
+; GFX11-NEXT:    s_cselect_b32 s1, s17, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = or <2 x i32> %a, splat (i32 u0x80000000)
+  %cmp = icmp eq <2 x i32> %cond, zeroinitializer
+  %select = select <2 x i1> %cmp, <2 x i32> %b, <2 x i32> %neg.a
+  ret <2 x i32> %select
+}
+
+define i64 @fneg_select_i64_1(i64 %cond, i64 %a, i64 %b) {
+; GCN-LABEL: fneg_select_i64_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, -v3, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_select_i64_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, -v3, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i64 %a, u0x8000000000000000
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %neg.a, i64 %b
+  ret i64 %select
+}
+
+define i64 @fneg_select_i64_2(i64 %cond, i64 %a, i64 %b) {
+; GCN-LABEL: fneg_select_i64_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, -v3, v5, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_select_i64_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, -v3, v5, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i64 %a, u0x8000000000000000
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %b, i64 %neg.a
+  ret i64 %select
+}
+
+define i64 @fneg_1_fabs_2_select_i64(i64 %cond, i64 %a, i64 %b) {
+; GCN-LABEL: fneg_1_fabs_2_select_i64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, |v5|, -v3, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_1_fabs_2_select_i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, |v5|, -v3, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i64 %a, u0x8000000000000000
+  %abs.b = and i64 %b, u0x7fffffffffffffff
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %neg.a, i64 %abs.b
+  ret i64 %select
+}
+
+define i64 @fabs_select_i64_1(i64 %cond, i64 %a, i64 %b) {
+; GCN-LABEL: fabs_select_i64_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, |v3|, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fabs_select_i64_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, |v3|, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = and i64 %a, u0x7fffffffffffffff
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %neg.a, i64 %b
+  ret i64 %select
+}
+
+define i64 @fabs_select_i64_2(i64 %cond, i64 %a, i64 %b) {
+; GCN-LABEL: fabs_select_i64_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, |v3|, v5, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fabs_select_i64_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, |v3|, v5, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = and i64 %a, u0x7fffffffffffffff
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %b, i64 %neg.a
+  ret i64 %select
+}
+
+define i64 @fneg_fabs_select_i64_1(i64 %cond, i64 %a, i64 %b) {
+; GCN-LABEL: fneg_fabs_select_i64_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, v5, -|v3|, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_fabs_select_i64_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, -|v3|, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = or i64 %a, u0x8000000000000000
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %neg.a, i64 %b
+  ret i64 %select
+}
+
+define i64 @fneg_fabs_select_i64_2(i64 %cond, i64 %a, i64 %b) {
+; GCN-LABEL: fneg_fabs_select_i64_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
+; GCN-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc
+; GCN-NEXT:    v_cndmask_b32_e64 v1, -|v3|, v5, vcc
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fneg_fabs_select_i64_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_eq_u64_e32 vcc_lo, 0, v[0:1]
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, -|v3|, v5, vcc_lo
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = or i64 %a, u0x8000000000000000
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %b, i64 %neg.a
+  ret i64 %select
+}
+
+define i64 @s_fneg_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
+; GFX7-LABEL: s_fneg_select_i64_1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
+; GFX7-NEXT:    s_xor_b32 s6, s19, 0x80000000
+; GFX7-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT:    s_cselect_b32 s4, s18, s20
+; GFX7-NEXT:    s_cselect_b32 s5, s6, s21
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: s_fneg_select_i64_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b32 s4, s19, 0x80000000
+; GFX9-NEXT:    s_cmp_eq_u64 s[16:17], 0
+; GFX9-NEXT:    s_cselect_b32 s5, s18, s20
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s21
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fneg_select_i64_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s3, s3, 0x80000000
+; GFX11-NEXT:    s_cmp_eq_u64 s[0:1], 0
+; GFX11-NEXT:    s_cselect_b32 s0, s2, s16
+; GFX11-NEXT:    s_cselect_b32 s1, s3, s17
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i64 %a, u0x8000000000000000
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %neg.a, i64 %b
+  ret i64 %select
+}
+
+define i64 @s_fneg_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
+; GFX7-LABEL: s_fneg_select_i64_2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
+; GFX7-NEXT:    s_xor_b32 s6, s19, 0x80000000
+; GFX7-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT:    s_cselect_b32 s4, s20, s18
+; GFX7-NEXT:    s_cselect_b32 s5, s21, s6
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: s_fneg_select_i64_2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b32 s4, s19, 0x80000000
+; GFX9-NEXT:    s_cmp_eq_u64 s[16:17], 0
+; GFX9-NEXT:    s_cselect_b32 s5, s20, s18
+; GFX9-NEXT:    s_cselect_b32 s4, s21, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fneg_select_i64_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s3, s3, 0x80000000
+; GFX11-NEXT:    s_cmp_eq_u64 s[0:1], 0
+; GFX11-NEXT:    s_cselect_b32 s0, s16, s2
+; GFX11-NEXT:    s_cselect_b32 s1, s17, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i64 %a, u0x8000000000000000
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %b, i64 %neg.a
+  ret i64 %select
+}
+
+define i64 @s_fneg_1_fabs_2_select_i64(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
+; GFX7-LABEL: s_fneg_1_fabs_2_select_i64:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
+; GFX7-NEXT:    s_xor_b32 s6, s19, 0x80000000
+; GFX7-NEXT:    s_bitset0_b32 s21, 31
+; GFX7-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT:    s_cselect_b32 s4, s18, s20
+; GFX7-NEXT:    s_cselect_b32 s5, s6, s21
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: s_fneg_1_fabs_2_select_i64:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_xor_b32 s4, s19, 0x80000000
+; GFX9-NEXT:    s_bitset0_b32 s21, 31
+; GFX9-NEXT:    s_cmp_eq_u64 s[16:17], 0
+; GFX9-NEXT:    s_cselect_b32 s5, s18, s20
+; GFX9-NEXT:    s_cselect_b32 s4, s4, s21
+; GFX9-NEXT:    v_mov_b32_e32 v0, s5
+; GFX9-NEXT:    v_mov_b32_e32 v1, s4
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fneg_1_fabs_2_select_i64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_xor_b32 s3, s3, 0x80000000
+; GFX11-NEXT:    s_bitset0_b32 s17, 31
+; GFX11-NEXT:    s_cmp_eq_u64 s[0:1], 0
+; GFX11-NEXT:    s_cselect_b32 s0, s2, s16
+; GFX11-NEXT:    s_cselect_b32 s1, s3, s17
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i64 %a, u0x8000000000000000
+  %abs.b = and i64 %b, u0x7fffffffffffffff
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %neg.a, i64 %abs.b
+  ret i64 %select
+}
+
+define i64 @s_fabs_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
+; GFX7-LABEL: s_fabs_select_i64_1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
+; GFX7-NEXT:    s_bitset0_b32 s19, 31
+; GFX7-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT:    s_cselect_b32 s4, s18, s20
+; GFX7-NEXT:    s_cselect_b32 s5, s19, s21
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: s_fabs_select_i64_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_bitset0_b32 s19, 31
+; GFX9-NEXT:    s_cmp_eq_u64 s[16:17], 0
+; GFX9-NEXT:    s_cselect_b32 s4, s18, s20
+; GFX9-NEXT:    s_cselect_b32 s5, s19, s21
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fabs_select_i64_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_bitset0_b32 s3, 31
+; GFX11-NEXT:    s_cmp_eq_u64 s[0:1], 0
+; GFX11-NEXT:    s_cselect_b32 s0, s2, s16
+; GFX11-NEXT:    s_cselect_b32 s1, s3, s17
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = and i64 %a, u0x7fffffffffffffff
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %neg.a, i64 %b
+  ret i64 %select
+}
+
+define i64 @s_fabs_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
+; GFX7-LABEL: s_fabs_select_i64_2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
+; GFX7-NEXT:    s_bitset0_b32 s19, 31
+; GFX7-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT:    s_cselect_b32 s4, s20, s18
+; GFX7-NEXT:    s_cselect_b32 s5, s21, s19
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: s_fabs_select_i64_2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_bitset0_b32 s19, 31
+; GFX9-NEXT:    s_cmp_eq_u64 s[16:17], 0
+; GFX9-NEXT:    s_cselect_b32 s4, s20, s18
+; GFX9-NEXT:    s_cselect_b32 s5, s21, s19
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fabs_select_i64_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_bitset0_b32 s3, 31
+; GFX11-NEXT:    s_cmp_eq_u64 s[0:1], 0
+; GFX11-NEXT:    s_cselect_b32 s0, s16, s2
+; GFX11-NEXT:    s_cselect_b32 s1, s17, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = and i64 %a, u0x7fffffffffffffff
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %b, i64 %neg.a
+  ret i64 %select
+}
+
+define i64 @s_fneg_fabs_select_i64_1(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
+; GFX7-LABEL: s_fneg_fabs_select_i64_1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
+; GFX7-NEXT:    s_bitset1_b32 s19, 31
+; GFX7-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT:    s_cselect_b32 s4, s18, s20
+; GFX7-NEXT:    s_cselect_b32 s5, s19, s21
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: s_fneg_fabs_select_i64_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_bitset1_b32 s19, 31
+; GFX9-NEXT:    s_cmp_eq_u64 s[16:17], 0
+; GFX9-NEXT:    s_cselect_b32 s4, s18, s20
+; GFX9-NEXT:    s_cselect_b32 s5, s19, s21
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fneg_fabs_select_i64_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_bitset1_b32 s3, 31
+; GFX11-NEXT:    s_cmp_eq_u64 s[0:1], 0
+; GFX11-NEXT:    s_cselect_b32 s0, s2, s16
+; GFX11-NEXT:    s_cselect_b32 s1, s3, s17
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = or i64 %a, u0x8000000000000000
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %neg.a, i64 %b
+  ret i64 %select
+}
+
+define i64 @s_fneg_fabs_select_i64_2(i64 inreg %cond, i64 inreg %a, i64 inreg %b) {
+; GFX7-LABEL: s_fneg_fabs_select_i64_2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[16:17], 0
+; GFX7-NEXT:    s_bitset1_b32 s19, 31
+; GFX7-NEXT:    s_and_b64 s[4:5], s[4:5], exec
+; GFX7-NEXT:    s_cselect_b32 s4, s20, s18
+; GFX7-NEXT:    s_cselect_b32 s5, s21, s19
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: s_fneg_fabs_select_i64_2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_bitset1_b32 s19, 31
+; GFX9-NEXT:    s_cmp_eq_u64 s[16:17], 0
+; GFX9-NEXT:    s_cselect_b32 s4, s20, s18
+; GFX9-NEXT:    s_cselect_b32 s5, s21, s19
+; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: s_fneg_fabs_select_i64_2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_bitset1_b32 s3, 31
+; GFX11-NEXT:    s_cmp_eq_u64 s[0:1], 0
+; GFX11-NEXT:    s_cselect_b32 s0, s16, s2
+; GFX11-NEXT:    s_cselect_b32 s1, s17, s3
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = or i64 %a, u0x8000000000000000
+  %cmp = icmp eq i64 %cond, zeroinitializer
+  %select = select i1 %cmp, i64 %b, i64 %neg.a
+  ret i64 %select
+}
+
+define i16 @fneg_select_i16_1(i16 %cond, i16 %a, i16 %b) {
+; GFX7-LABEL: fneg_select_i16_1:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fneg_select_i16_1:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: fneg_select_i16_1:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_xor_b16 v0.h, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: fneg_select_i16_1:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i16 %a, u0x8000
+  %cmp = icmp eq i16 %cond, zeroinitializer
+  %select = select i1 %cmp, i16 %neg.a, i16 %b
+  ret i16 %select
+}
+
+define i16 @fneg_select_i16_2(i16 %cond, i16 %a, i16 %b) {
+; GFX7-LABEL: fneg_select_i16_2:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fneg_select_i16_2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: fneg_select_i16_2:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_xor_b16 v0.h, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v0.h, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: fneg_select_i16_2:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_xor_b32_e32 v1, 0xffff8000, v1
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i16 %a, u0x8000
+  %cmp = icmp eq i16 %cond, zeroinitializer
+  %select = select i1 %cmp, i16 %b, i16 %neg.a
+  ret i16 %select
+}
+
+define i16 @fneg_select_i16_both(i16 %cond, i16 %a, i16 %b) {
+; GFX7-LABEL: fneg_select_i16_both:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX7-NEXT:    v_xor_b32_e32 v0, 0xffff8000, v0
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fneg_select_i16_both:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc
+; GFX9-NEXT:    v_xor_b32_e32 v0, 0xffff8000, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: fneg_select_i16_both:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v2.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_xor_b16 v0.l, 0x8000, v0.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: fneg_select_i16_both:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v2, v1, vcc_lo
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_xor_b32_e32 v0, 0xffff8000, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i16 %a, u0x8000
+  %neg.b = xor i16 %b, u0x8000
+  %cmp = icmp eq i16 %cond, zeroinitializer
+  %select = select i1 %cmp, i16 %neg.a, i16 %neg.b
+  ret i16 %select
+}
+
+define i16 @fneg_1_fabs_2_select_i16(i16 %cond, i16 %a, i16 %b) {
+; GFX7-LABEL: fneg_1_fabs_2_select_i16:
+; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT:    v_xor_b32_e32 v2, 0xffff8000, v1
+; GFX7-NEXT:    v_and_b32_e32 v1, 0x7fff, v1
+; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX7-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX7-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: fneg_1_fabs_2_select_i16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_xor_b32_e32 v2, 0xffff8000, v1
+; GFX9-NEXT:    v_and_b32_e32 v1, 0x7fff, v1
+; GFX9-NEXT:    v_cmp_eq_u16_e32 vcc, 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: fneg_1_fabs_2_select_i16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_xor_b16 v0.h, 0x8000, v1.l
+; GFX11-TRUE16-NEXT:    v_and_b16 v1.l, 0x7fff, v1.l
+; GFX11-TRUE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT:    v_cndmask_b16 v0.l, v1.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: fneg_1_fabs_2_select_i16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_xor_b32_e32 v2, 0xffff8000, v1
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT:    v_cmp_eq_u16_e32 vcc_lo, 0, v0
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT:    v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
+  %neg.a = xor i16 %a, u0x8000
+  %abs.b = and i16 %a, u0x7fff
+  %cmp = icmp eq i16 %cond, zeroinitializer
+  %select = select i1 %cmp, i16 %neg.a, i16 %abs.b
+  ret i16 %select
+}
diff --git a/llvm/test/CodeGen/AMDGPU/invalid-addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/invalid-addrspacecast.ll
index 5dff7372ab561..294c9045ebc14 100644
--- a/llvm/test/CodeGen/AMDGPU/invalid-addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/invalid-addrspacecast.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdhsad < %s | FileCheck %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s
 
 ; Check illegal casts are codegened as poison, and not an error.
 
diff --git a/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll b/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll
index 8fc5afb155573..17c8010bcbe05 100644
--- a/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll
+++ b/llvm/test/CodeGen/AMDGPU/irtranslator-whole-wave-functions.ll
@@ -101,3 +101,29 @@ define amdgpu_gfx_whole_wave i64 @ret_64(i1 %active, i64 %a, i64 %b) {
   %ret = call i64 @llvm.amdgcn.update.dpp.i64(i64 %x, i64 %y, i32 1, i32 1, i32 1, i1 false)
   ret i64 %ret
 }
+
+declare amdgpu_gfx_whole_wave i32 @callee(i1 %active, i32 %x)
+
+; Make sure we don't pass the first argument (i1).
+define amdgpu_cs void @call(i32 %x, ptr %p) {
+  ; CHECK-LABEL: name: call
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; CHECK-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; CHECK-NEXT:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @callee
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $scc
+  ; CHECK-NEXT:   [[GV1:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @callee
+  ; CHECK-NEXT:   $vgpr0 = COPY [[COPY]](s32)
+  ; CHECK-NEXT:   $sgpr30_sgpr31 = G_SI_CALL [[GV1]](p0), @callee, csr_amdgpu_si_gfx, implicit $vgpr0, implicit-def $vgpr0
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc
+  ; CHECK-NEXT:   G_STORE [[COPY3]](s32), [[MV]](p0) :: (store (s32) into %ir.p)
+  ; CHECK-NEXT:   S_ENDPGM 0
+  %ret = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @callee, i32 %x) convergent
+  store i32 %ret, ptr %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/isel-whole-wave-functions.ll b/llvm/test/CodeGen/AMDGPU/isel-whole-wave-functions.ll
index 3450d63ff7b4a..69809b115e037 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-whole-wave-functions.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-whole-wave-functions.ll
@@ -189,3 +189,79 @@ define amdgpu_gfx_whole_wave i64 @ret_64(i1 %active, i64 %a, i64 %b) {
   ret i64 %ret
 }
 
+declare amdgpu_gfx_whole_wave i32 @callee(i1 %active, <8 x i32> %x)
+
+; Make sure we don't pass the first argument (i1).
+define amdgpu_cs void @call(<8 x i32> %x, ptr %p) {
+  ; DAGISEL-LABEL: name: call
+  ; DAGISEL: bb.0 (%ir-block.0):
+  ; DAGISEL-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+  ; DAGISEL-NEXT: {{  $}}
+  ; DAGISEL-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; DAGISEL-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; DAGISEL-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; DAGISEL-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; DAGISEL-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; DAGISEL-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; DAGISEL-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; DAGISEL-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; DAGISEL-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; DAGISEL-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; DAGISEL-NEXT:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-NEXT:   [[DEF1:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
+  ; DAGISEL-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY]], %subreg.sub1
+  ; DAGISEL-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
+  ; DAGISEL-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
+  ; DAGISEL-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[S_MOV_B32_1]], %subreg.sub0, killed [[S_MOV_B32_]], %subreg.sub1
+  ; DAGISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; DAGISEL-NEXT:   $vgpr0 = COPY [[COPY9]]
+  ; DAGISEL-NEXT:   $vgpr1 = COPY [[COPY8]]
+  ; DAGISEL-NEXT:   $vgpr2 = COPY [[COPY7]]
+  ; DAGISEL-NEXT:   $vgpr3 = COPY [[COPY6]]
+  ; DAGISEL-NEXT:   $vgpr4 = COPY [[COPY5]]
+  ; DAGISEL-NEXT:   $vgpr5 = COPY [[COPY4]]
+  ; DAGISEL-NEXT:   $vgpr6 = COPY [[COPY3]]
+  ; DAGISEL-NEXT:   $vgpr7 = COPY [[COPY2]]
+  ; DAGISEL-NEXT:   $sgpr30_sgpr31 = SI_CALL killed [[REG_SEQUENCE1]], @callee, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit-def $vgpr0
+  ; DAGISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; DAGISEL-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; DAGISEL-NEXT:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]]
+  ; DAGISEL-NEXT:   FLAT_STORE_DWORD killed [[COPY11]], [[COPY10]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %ir.p)
+  ; DAGISEL-NEXT:   S_ENDPGM 0
+  ;
+  ; GISEL-LABEL: name: call
+  ; GISEL: bb.1 (%ir-block.0):
+  ; GISEL-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+  ; GISEL-NEXT: {{  $}}
+  ; GISEL-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GISEL-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GISEL-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GISEL-NEXT:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; GISEL-NEXT:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; GISEL-NEXT:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; GISEL-NEXT:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; GISEL-NEXT:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; GISEL-NEXT:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; GISEL-NEXT:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY $vgpr9
+  ; GISEL-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY9]], %subreg.sub1
+  ; GISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; GISEL-NEXT:   $vgpr0 = COPY [[COPY]]
+  ; GISEL-NEXT:   $vgpr1 = COPY [[COPY1]]
+  ; GISEL-NEXT:   $vgpr2 = COPY [[COPY2]]
+  ; GISEL-NEXT:   $vgpr3 = COPY [[COPY3]]
+  ; GISEL-NEXT:   $vgpr4 = COPY [[COPY4]]
+  ; GISEL-NEXT:   $vgpr5 = COPY [[COPY5]]
+  ; GISEL-NEXT:   $vgpr6 = COPY [[COPY6]]
+  ; GISEL-NEXT:   $vgpr7 = COPY [[COPY7]]
+  ; GISEL-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-lo) @callee
+  ; GISEL-NEXT:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 target-flags(amdgpu-abs32-hi) @callee
+  ; GISEL-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_MOV_B32_1]], %subreg.sub1
+  ; GISEL-NEXT:   $sgpr30_sgpr31 = SI_CALL [[REG_SEQUENCE1]], @callee, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $vgpr4, implicit $vgpr5, implicit $vgpr6, implicit $vgpr7, implicit-def $vgpr0
+  ; GISEL-NEXT:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
+  ; GISEL-NEXT:   FLAT_STORE_DWORD [[REG_SEQUENCE]], [[COPY10]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %ir.p)
+  ; GISEL-NEXT:   S_ENDPGM 0
+  %ret = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @callee, <8 x i32> %x) convergent
+  store i32 %ret, ptr %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/issue130120-eliminate-frame-index.ll b/llvm/test/CodeGen/AMDGPU/issue130120-eliminate-frame-index.ll
index 1c298014e33e7..300124848c1aa 100644
--- a/llvm/test/CodeGen/AMDGPU/issue130120-eliminate-frame-index.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue130120-eliminate-frame-index.ll
@@ -6,16 +6,24 @@ define amdgpu_gfx [13 x i32] @issue130120() {
 ; CHECK:       ; %bb.0: ; %bb
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    s_add_i32 s0, s32, 0xf0
-; CHECK-NEXT:    s_add_i32 s1, s32, 0xf4
-; CHECK-NEXT:    s_add_i32 s2, s32, 0xf8
-; CHECK-NEXT:    s_add_i32 s3, s32, 0xfc
+; CHECK-NEXT:    s_movk_i32 s1, 0xf4
+; CHECK-NEXT:    s_movk_i32 s2, 0xf8
+; CHECK-NEXT:    s_movk_i32 s3, 0xfc
+; CHECK-NEXT:    s_movk_i32 s34, 0x100
 ; CHECK-NEXT:    v_mov_b32_e32 v1, v0
-; CHECK-NEXT:    s_add_i32 s34, s32, 0x100
-; CHECK-NEXT:    s_add_i32 s35, s32, 0x104
-; CHECK-NEXT:    s_add_i32 s36, s32, 0x108
-; CHECK-NEXT:    s_add_i32 s37, s32, 0x110
-; CHECK-NEXT:    s_add_i32 s38, s32, 0x120
+; CHECK-NEXT:    s_movk_i32 s35, 0x104
+; CHECK-NEXT:    s_movk_i32 s36, 0x108
+; CHECK-NEXT:    s_movk_i32 s37, 0x110
+; CHECK-NEXT:    s_movk_i32 s38, 0x120
+; CHECK-NEXT:    s_add_i32 s0, s32, 0xf0
+; CHECK-NEXT:    s_add_i32 s1, s32, s1
+; CHECK-NEXT:    s_add_i32 s2, s32, s2
+; CHECK-NEXT:    s_add_i32 s3, s32, s3
+; CHECK-NEXT:    s_add_i32 s34, s32, s34
+; CHECK-NEXT:    s_add_i32 s35, s32, s35
+; CHECK-NEXT:    s_add_i32 s36, s32, s36
+; CHECK-NEXT:    s_add_i32 s37, s32, s37
+; CHECK-NEXT:    s_add_i32 s38, s32, s38
 ; CHECK-NEXT:    s_or_b32 s39, s32, 4
 ; CHECK-NEXT:    s_or_b32 s40, s32, 8
 ; CHECK-NEXT:    s_or_b32 s41, s32, 12
diff --git a/llvm/test/CodeGen/AMDGPU/issue153808-extract-subvector-legalize.ll b/llvm/test/CodeGen/AMDGPU/issue153808-extract-subvector-legalize.ll
new file mode 100644
index 0000000000000..75c5d206e7933
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/issue153808-extract-subvector-legalize.ll
@@ -0,0 +1,192 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX900 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX942 %s
+
+define <3 x float> @extract_subvector_v3f32_v33f32_elt30_0(ptr addrspace(1) %ptr) #0 {
+; GFX900-LABEL: extract_subvector_v3f32_v33f32_elt30_0:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:96 glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:80 glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:64 glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:48 glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:32 glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:16 glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_load_dword v2, v[0:1], off offset:128 glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[3:6], v[0:1], off offset:112 glc
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v0, v5
+; GFX900-NEXT:    v_mov_b32_e32 v1, v6
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: extract_subvector_v3f32_v33f32_elt30_0:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:96 sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:80 sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:64 sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:48 sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:32 sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:16 sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_load_dword v2, v[0:1], off offset:128 sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[4:7], v[0:1], off offset:112 sc0 sc1
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v0, v6
+; GFX942-NEXT:    v_mov_b32_e32 v1, v7
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+  %val = load volatile <33 x float>, ptr addrspace(1) %ptr, align 4
+  %extract.subvector = shufflevector <33 x float> %val, <33 x float> poison, <3 x i32> <i32 30, i32 31, i32 32>
+  ret <3 x float> %extract.subvector
+}
+
+define <3 x float> @extract_subvector_v3f32_v33f32_elt30_1(ptr addrspace(1) %ptr) #0 {
+; GFX900-LABEL: extract_subvector_v3f32_v33f32_elt30_1:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[3:6], v[0:1], off
+; GFX900-NEXT:    global_load_dwordx4 v[7:10], v[0:1], off offset:112
+; GFX900-NEXT:    global_load_dword v2, v[0:1], off offset:128
+; GFX900-NEXT:    s_mov_b32 s4, 0
+; GFX900-NEXT:    s_mov_b32 s5, s4
+; GFX900-NEXT:    s_mov_b32 s6, s4
+; GFX900-NEXT:    s_mov_b32 s7, s4
+; GFX900-NEXT:    s_waitcnt vmcnt(2)
+; GFX900-NEXT:    buffer_store_dwordx4 v[3:6], off, s[4:7], 0
+; GFX900-NEXT:    s_waitcnt vmcnt(2)
+; GFX900-NEXT:    v_mov_b32_e32 v0, v9
+; GFX900-NEXT:    v_mov_b32_e32 v1, v10
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: extract_subvector_v3f32_v33f32_elt30_1:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[4:7], v[0:1], off
+; GFX942-NEXT:    global_load_dwordx4 v[8:11], v[0:1], off offset:112
+; GFX942-NEXT:    global_load_dword v2, v[0:1], off offset:128
+; GFX942-NEXT:    s_mov_b32 s0, 0
+; GFX942-NEXT:    s_mov_b32 s1, s0
+; GFX942-NEXT:    s_mov_b32 s2, s0
+; GFX942-NEXT:    s_mov_b32 s3, s0
+; GFX942-NEXT:    s_waitcnt vmcnt(2)
+; GFX942-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0
+; GFX942-NEXT:    s_waitcnt vmcnt(2)
+; GFX942-NEXT:    v_mov_b32_e32 v0, v10
+; GFX942-NEXT:    v_mov_b32_e32 v1, v11
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+  %val = load <33 x float>, ptr addrspace(1) %ptr, align 4
+  %val.slice.0 = shufflevector <33 x float> %val, <33 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> %val.slice.0, ptr addrspace(8) null, i32 0, i32 0, i32 0)
+  %val.slice.48 = shufflevector <33 x float> %val, <33 x float> poison, <3 x i32> <i32 30, i32 31, i32 32>
+  ret <3 x float> %val.slice.48
+}
+
+define <6 x float> @extract_subvector_v6f32_v36f32_elt30(ptr addrspace(1) %ptr) #0 {
+; GFX900-LABEL: extract_subvector_v6f32_v36f32_elt30:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    global_load_dwordx4 v[6:9], v[0:1], off
+; GFX900-NEXT:    global_load_dwordx4 v[10:13], v[0:1], off offset:112
+; GFX900-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:128
+; GFX900-NEXT:    s_mov_b32 s4, 0
+; GFX900-NEXT:    s_mov_b32 s5, s4
+; GFX900-NEXT:    s_mov_b32 s6, s4
+; GFX900-NEXT:    s_mov_b32 s7, s4
+; GFX900-NEXT:    s_waitcnt vmcnt(2)
+; GFX900-NEXT:    buffer_store_dwordx4 v[6:9], off, s[4:7], 0
+; GFX900-NEXT:    s_waitcnt vmcnt(2)
+; GFX900-NEXT:    v_mov_b32_e32 v0, v12
+; GFX900-NEXT:    v_mov_b32_e32 v1, v13
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: extract_subvector_v6f32_v36f32_elt30:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[6:9], v[0:1], off
+; GFX942-NEXT:    global_load_dwordx4 v[10:13], v[0:1], off offset:112
+; GFX942-NEXT:    global_load_dwordx4 v[2:5], v[0:1], off offset:128
+; GFX942-NEXT:    s_mov_b32 s0, 0
+; GFX942-NEXT:    s_mov_b32 s1, s0
+; GFX942-NEXT:    s_mov_b32 s2, s0
+; GFX942-NEXT:    s_mov_b32 s3, s0
+; GFX942-NEXT:    s_waitcnt vmcnt(2)
+; GFX942-NEXT:    buffer_store_dwordx4 v[6:9], off, s[0:3], 0
+; GFX942-NEXT:    s_waitcnt vmcnt(2)
+; GFX942-NEXT:    v_mov_b32_e32 v0, v12
+; GFX942-NEXT:    v_mov_b32_e32 v1, v13
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+  %val = load <36 x float>, ptr addrspace(1) %ptr, align 4
+  %val.slice.0 = shufflevector <36 x float> %val, <36 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> %val.slice.0, ptr addrspace(8) null, i32 0, i32 0, i32 0)
+  %val.slice.1 = shufflevector <36 x float> %val, <36 x float> poison, <6 x i32> <i32 30, i32 31, i32 32, i32 33, i32 34, i32 35>
+  ret <6 x float> %val.slice.1
+}
+
+define <3 x float> @issue153808_vector_extract_assert(ptr addrspace(1) %ptr) #0 {
+; GFX900-LABEL: issue153808_vector_extract_assert:
+; GFX900:       ; %bb.0:
+; GFX900-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT:    v_mov_b32_e32 v4, v1
+; GFX900-NEXT:    v_mov_b32_e32 v3, v0
+; GFX900-NEXT:    global_load_dwordx4 v[5:8], v[3:4], off
+; GFX900-NEXT:    global_load_dwordx3 v[0:2], v[3:4], off offset:192
+; GFX900-NEXT:    s_mov_b32 s4, 0
+; GFX900-NEXT:    s_mov_b32 s5, s4
+; GFX900-NEXT:    s_mov_b32 s6, s4
+; GFX900-NEXT:    s_mov_b32 s7, s4
+; GFX900-NEXT:    s_waitcnt vmcnt(1)
+; GFX900-NEXT:    buffer_store_dwordx4 v[5:8], off, s[4:7], 0
+; GFX900-NEXT:    s_waitcnt vmcnt(0)
+; GFX900-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: issue153808_vector_extract_assert:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx4 v[6:9], v[0:1], off
+; GFX942-NEXT:    global_load_dwordx3 v[2:4], v[0:1], off offset:192
+; GFX942-NEXT:    s_mov_b32 s0, 0
+; GFX942-NEXT:    s_mov_b32 s1, s0
+; GFX942-NEXT:    s_mov_b32 s2, s0
+; GFX942-NEXT:    s_mov_b32 s3, s0
+; GFX942-NEXT:    s_waitcnt vmcnt(1)
+; GFX942-NEXT:    buffer_store_dwordx4 v[6:9], off, s[0:3], 0
+; GFX942-NEXT:    s_waitcnt vmcnt(1)
+; GFX942-NEXT:    v_mov_b32_e32 v0, v2
+; GFX942-NEXT:    v_mov_b32_e32 v1, v3
+; GFX942-NEXT:    v_mov_b32_e32 v2, v4
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
+  %val = load <51 x float>, ptr addrspace(1) %ptr, align 4
+  %val.slice.0 = shufflevector <51 x float> %val, <51 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float> %val.slice.0, ptr addrspace(8) null, i32 0, i32 0, i32 0)
+  %val.slice.48 = shufflevector <51 x float> %val, <51 x float> poison, <3 x i32> <i32 48, i32 49, i32 50>
+  ret <3 x float> %val.slice.48
+}
+
+declare void @llvm.amdgcn.raw.ptr.buffer.store.v4f32(<4 x float>, ptr addrspace(8) writeonly captures(none), i32, i32, i32 immarg) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX9: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
index c316ec71863d0..968471287dc4d 100644
--- a/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/itofp.i128.ll
@@ -797,12 +797,11 @@ define double @sitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GISEL-NEXT:  .LBB2_13: ; %Flow4
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT:    v_and_b32_e32 v1, 0x80000000, v6
 ; GISEL-NEXT:    v_mov_b32_e32 v2, 0x3ff00000
-; GISEL-NEXT:    v_mov_b32_e32 v3, 0xfffff
+; GISEL-NEXT:    v_and_b32_e32 v1, 0x80000000, v6
 ; GISEL-NEXT:    v_lshl_add_u32 v2, v8, 20, v2
-; GISEL-NEXT:    v_and_or_b32 v1, v10, v3, v1
-; GISEL-NEXT:    v_or3_b32 v1, v1, v2, 0
+; GISEL-NEXT:    v_and_b32_e32 v3, 0xfffff, v10
+; GISEL-NEXT:    v_or3_b32 v1, v3, v1, v2
 ; GISEL-NEXT:  .LBB2_14: ; %Flow5
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
@@ -1081,8 +1080,8 @@ define double @uitofp_i128_to_f64(i128 %x) {
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, 0x3ff00000
 ; GISEL-NEXT:    v_lshl_add_u32 v0, v7, 20, v0
-; GISEL-NEXT:    v_and_b32_e32 v1, 0xfffff, v9
-; GISEL-NEXT:    v_or3_b32 v5, v1, v0, 0
+; GISEL-NEXT:    v_mov_b32_e32 v1, 0xfffff
+; GISEL-NEXT:    v_and_or_b32 v5, v9, v1, v0
 ; GISEL-NEXT:  .LBB3_14: ; %Flow5
 ; GISEL-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GISEL-NEXT:    v_mov_b32_e32 v0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/lds-limit-diagnostics-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/lds-limit-diagnostics-gfx1250.ll
new file mode 100644
index 0000000000000..da92dcdd7104e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-limit-diagnostics-gfx1250.ll
@@ -0,0 +1,13 @@
+; RUN: not llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s 2>&1 | FileCheck -check-prefix=ERROR %s
+
+; GFX1250 supports upto 320 KB LDS memory.
+; This is a negative test to check when the LDS size exceeds the max usable limit.
+
+; ERROR: error: <unknown>:0:0: local memory (327684) exceeds limit (327680) in function 'test_lds_limit'
+@dst = addrspace(3) global [81921 x i32] undef
+
+define amdgpu_kernel void @test_lds_limit(i32 %val) {
+  %gep = getelementptr [81921 x i32], ptr addrspace(3) @dst, i32 0, i32 100
+  store i32 %val, ptr addrspace(3) %gep
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lds-size-hsa-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/lds-size-hsa-gfx1250.ll
new file mode 100644
index 0000000000000..3db0fa8f21759
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-size-hsa-gfx1250.ll
@@ -0,0 +1,72 @@
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=MESA %s
+
+; GFX1250 supports upto 320 KB configurable LDS memory.
+; This test checks the min and max size of LDS that can be allocated.
+
+@lds.i8 = addrspace(3) global i8 undef
+@lds.array.i8 = addrspace(3) global [327679 x i8] undef
+@lds.i16 = addrspace(3) global i16 undef
+@lds.array.i16 = addrspace(3) global [163839 x i16] undef
+@lds.i32 = addrspace(3) global i32 undef
+@lds.array.i32 = addrspace(3) global [81919 x i32] undef
+
+; GCN-LABEL: test_lds_i8:
+; GCN: .amdhsa_group_segment_fixed_size 1
+; GCN: ; LDSByteSize: 1 bytes/workgroup
+; MESA: granulated_lds_size = 1
+define amdgpu_kernel void @test_lds_i8(i8 %val) {
+  store i8 %val, ptr addrspace(3) @lds.i8
+  ret void
+}
+
+; GCN-LABEL: test_lds_i16:
+; GCN: .amdhsa_group_segment_fixed_size 2
+; GCN: ; LDSByteSize: 2 bytes/workgroup
+; MESA: granulated_lds_size = 1
+define amdgpu_kernel void @test_lds_i16(i16 %val) {
+  store i16 %val, ptr addrspace(3) @lds.i16
+  ret void
+}
+
+; GCN-LABEL: test_lds_i32:
+; GCN: .amdhsa_group_segment_fixed_size 4
+; GCN: ; LDSByteSize: 4 bytes/workgroup
+; MESA: granulated_lds_size = 1
+define amdgpu_kernel void @test_lds_i32(i32 %val) {
+  store i32 %val, ptr addrspace(3) @lds.i32
+  ret void
+}
+
+; GCN-LABEL: test_lds_array_i8:
+; GCN: .amdhsa_group_segment_fixed_size 327680
+; GCN: ; LDSByteSize: 327680 bytes/workgroup
+; MESA: granulated_lds_size = 320
+define amdgpu_kernel void @test_lds_array_i8() {
+  %gep = getelementptr inbounds [327679 x i8], ptr addrspace(3) @lds.array.i8, i32 0, i32 5
+  %val = load i8, ptr addrspace(3) %gep
+  store i8 %val, ptr addrspace(3) @lds.i8
+  ret void
+}
+
+; GCN-LABEL: test_lds_array_i16:
+; GCN: .amdhsa_group_segment_fixed_size 327680
+; GCN: ; LDSByteSize: 327680 bytes/workgroup
+; MESA: granulated_lds_size = 320
+define amdgpu_kernel void @test_lds_array_i16() {
+  %gep = getelementptr inbounds [163839 x i16], ptr addrspace(3) @lds.array.i16, i32 0, i32 10
+  %val = load i16, ptr addrspace(3) %gep
+  store i16 %val, ptr addrspace(3) @lds.i16
+  ret void
+}
+
+; GCN-LABEL: test_lds_array_i32:
+; GCN: .amdhsa_group_segment_fixed_size 327680
+; GCN: ; LDSByteSize: 327680 bytes/workgroup
+; MESA: granulated_lds_size = 320
+define amdgpu_kernel void @test_lds_array_i32() {
+  %gep = getelementptr inbounds [81919 x i32], ptr addrspace(3) @lds.array.i32, i32 0, i32 20
+  %val = load i32, ptr addrspace(3) %gep
+  store i32 %val, ptr addrspace(3) @lds.i32
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lds-size-pal-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/lds-size-pal-gfx1250.ll
new file mode 100644
index 0000000000000..bfa7d37ce63a7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/lds-size-pal-gfx1250.ll
@@ -0,0 +1,61 @@
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1250 < %s | FileCheck -check-prefix=PAL %s
+
+; GFX1250 supports upto 320 KB configurable LDS memory.
+; This test checks the min and max size of LDS that can be allocated.
+
+; PAL: .shader_functions:
+; PAL: test_lds_array_i16:
+; PAL: .lds_size:       0x50000
+; PAL: test_lds_array_i32:
+; PAL: .lds_size:       0x50000
+; PAL: test_lds_array_i8:
+; PAL: .lds_size:       0x50000
+; PAL: test_lds_i16:
+; PAL: .lds_size:       0x2
+; PAL: test_lds_i32:
+; PAL: .lds_size:       0x4
+; PAL: test_lds_i8:
+; PAL: .lds_size:       0x1
+
+@lds.i8 = addrspace(3) global i8 undef
+@lds.array.i8 = addrspace(3) global [327679 x i8] undef
+@lds.i16 = addrspace(3) global i16 undef
+@lds.array.i16 = addrspace(3) global [163839 x i16] undef
+@lds.i32 = addrspace(3) global i32 undef
+@lds.array.i32 = addrspace(3) global [81919 x i32] undef
+
+define amdgpu_gfx void @test_lds_i8(i8 %val) {
+  store i8 %val, ptr addrspace(3) @lds.i8
+  ret void
+}
+
+define amdgpu_gfx void @test_lds_i16(i16 %val) {
+  store i16 %val, ptr addrspace(3) @lds.i16
+  ret void
+}
+
+define amdgpu_gfx void @test_lds_i32(i32 %val) {
+  store i32 %val, ptr addrspace(3) @lds.i32
+  ret void
+}
+
+define amdgpu_gfx void @test_lds_array_i8() {
+  %gep = getelementptr inbounds [327679 x i8], ptr addrspace(3) @lds.array.i8, i32 0, i32 5
+  %val = load i8, ptr addrspace(3) %gep
+  store i8 %val, ptr addrspace(3) @lds.i8
+  ret void
+}
+
+define amdgpu_gfx void @test_lds_array_i16() {
+  %gep = getelementptr inbounds [163839 x i16], ptr addrspace(3) @lds.array.i16, i32 0, i32 10
+  %val = load i16, ptr addrspace(3) %gep
+  store i16 %val, ptr addrspace(3) @lds.i16
+  ret void
+}
+
+define amdgpu_gfx void @test_lds_array_i32() {
+  %gep = getelementptr inbounds [81919 x i32], ptr addrspace(3) @lds.array.i32, i32 0, i32 20
+  %val = load i32, ptr addrspace(3) %gep
+  store i32 %val, ptr addrspace(3) @lds.i32
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/literal64.ll b/llvm/test/CodeGen/AMDGPU/literal64.ll
index 768c9728554df..98691d394abb3 100644
--- a/llvm/test/CodeGen/AMDGPU/literal64.ll
+++ b/llvm/test/CodeGen/AMDGPU/literal64.ll
@@ -67,24 +67,8 @@ define void @v_mov_b64_double(ptr addrspace(1) %ptr) {
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    global_load_b64 v[4:5], v[0:1], off
-; GCN-NEXT:    s_mov_b32 s0, 0
-; GCN-NEXT:  .LBB6_1: ; %atomicrmw.start
-; GCN-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT:    s_wait_loadcnt 0x0
-; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GCN-NEXT:    v_add_f64_e32 v[2:3], lit64(0x4063233333333333), v[4:5]
-; GCN-NEXT:    global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GCN-NEXT:    s_wait_loadcnt 0x0
-; GCN-NEXT:    v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
-; GCN-NEXT:    s_wait_xcnt 0x0
-; GCN-NEXT:    v_mov_b64_e32 v[4:5], v[2:3]
-; GCN-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GCN-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; GCN-NEXT:    s_cbranch_execnz .LBB6_1
-; GCN-NEXT:  ; %bb.2: ; %atomicrmw.end
-; GCN-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GCN-NEXT:    v_mov_b64_e32 v[2:3], lit64(0x4063233333333333)
+; GCN-NEXT:    global_atomic_add_f64 v[0:1], v[2:3], off scope:SCOPE_SYS
 ; GCN-NEXT:    s_set_pc_i64 s[30:31]
   %result = atomicrmw fadd ptr addrspace(1) %ptr, double 153.1 monotonic
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/livevars-implicitdef.mir b/llvm/test/CodeGen/AMDGPU/livevars-implicitdef.mir
new file mode 100644
index 0000000000000..18aeb2527b1a3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/livevars-implicitdef.mir
@@ -0,0 +1,91 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn --run-pass=livevars -o - %s | FileCheck %s
+---
+# Check that super register is defined for an sgpr copy.
+name:            sgpr_copy
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; CHECK-LABEL: name: sgpr_copy
+    ; CHECK: %sval:sreg_32 = S_MOV_B32 0
+    ; CHECK-NEXT: $sgpr0 = COPY %sval
+    ; CHECK-NEXT: $sgpr1 = COPY %sval
+    ; CHECK-NEXT: $sgpr2 = COPY %sval
+    ; CHECK-NEXT: $sgpr3 = COPY killed %sval
+    ; CHECK-NEXT: SI_RETURN implicit killed $sgpr0_sgpr1_sgpr2_sgpr3
+    %sval:sreg_32 = S_MOV_B32 0
+
+    $sgpr0 = COPY %sval
+    $sgpr1 = COPY %sval
+    $sgpr2 = COPY %sval
+    $sgpr3 = COPY %sval
+    SI_RETURN implicit $sgpr0_sgpr1_sgpr2_sgpr3
+
+...
+---
+# Check that super register is defined for a vgpr vector copy.
+name:            vgpr_copy
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; CHECK-LABEL: name: vgpr_copy
+    ; CHECK: %vval:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = COPY %vval
+    ; CHECK-NEXT: $vgpr1 = COPY %vval
+    ; CHECK-NEXT: $vgpr2 = COPY %vval
+    ; CHECK-NEXT: $vgpr3 = COPY killed %vval
+    ; CHECK-NEXT: dead [[COPY:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0_vgpr1_vgpr2_vgpr3
+    %vval:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+
+    $vgpr0 = COPY %vval
+    $vgpr1 = COPY %vval
+    $vgpr2 = COPY %vval
+    $vgpr3 = COPY %vval
+    %0:vgpr_32 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+
+...
+---
+# Check that super register is defined when there is a hole.
+name:            sgpr_copy_hole
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: sgpr_copy_hole
+    ; CHECK: %sval:sreg_32 = S_MOV_B32 0
+    ; CHECK-NEXT: $sgpr0 = COPY %sval
+    ; CHECK-NEXT: $sgpr2 = COPY %sval
+    ; CHECK-NEXT: $sgpr3 = COPY killed %sval
+    ; CHECK-NEXT: SI_RETURN implicit killed $sgpr0_sgpr1_sgpr2_sgpr3
+    %sval:sreg_32 = S_MOV_B32 0
+
+    $sgpr0 = COPY %sval
+    $sgpr2 = COPY %sval
+    $sgpr3 = COPY %sval
+    SI_RETURN implicit $sgpr0_sgpr1_sgpr2_sgpr3
+
+...
+---
+# Check that super register is defined when a pair interrupts the sequence.
+name:            vgpr_copy_pair
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: vgpr_copy_pair
+    ; CHECK: %vval:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: $vgpr0 = COPY %vval
+    ; CHECK-NEXT: $vgpr1 = COPY %vval
+    ; CHECK-NEXT: $vgpr2 = COPY %vval
+    ; CHECK-NEXT: $vgpr3 = COPY killed %vval
+    ; CHECK-NEXT: dead [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1_vgpr2
+    ; CHECK-NEXT: dead [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed $vgpr0_vgpr1_vgpr2_vgpr3
+    %vval:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+
+    $vgpr0 = COPY %vval
+    $vgpr1 = COPY %vval
+    $vgpr2 = COPY %vval
+    $vgpr3 = COPY %vval
+    %0:vgpr_32 = COPY $vgpr1_vgpr2
+    %1:vgpr_32 = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 2a5c65278f7dc..3e17be6b34a57 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -255,6 +255,7 @@
 ; GCN-O1-NEXT:        Uniformity Analysis
 ; GCN-O1-NEXT:        AMDGPU IR late optimizations
 ; GCN-O1-NEXT:        Post-Dominator Tree Construction
+; GCN-O1-NEXT:        Uniformity Analysis
 ; GCN-O1-NEXT:        Unify divergent function exit nodes
 ; GCN-O1-NEXT:        Dominator Tree Construction
 ; GCN-O1-NEXT:        Cycle Info Analysis
@@ -559,6 +560,7 @@
 ; GCN-O1-OPTS-NEXT:        Uniformity Analysis
 ; GCN-O1-OPTS-NEXT:        AMDGPU IR late optimizations
 ; GCN-O1-OPTS-NEXT:        Post-Dominator Tree Construction
+; GCN-O1-OPTS-NEXT:        Uniformity Analysis
 ; GCN-O1-OPTS-NEXT:        Unify divergent function exit nodes
 ; GCN-O1-OPTS-NEXT:        Dominator Tree Construction
 ; GCN-O1-OPTS-NEXT:        Cycle Info Analysis
@@ -875,6 +877,7 @@
 ; GCN-O2-NEXT:        Uniformity Analysis
 ; GCN-O2-NEXT:        AMDGPU IR late optimizations
 ; GCN-O2-NEXT:        Post-Dominator Tree Construction
+; GCN-O2-NEXT:        Uniformity Analysis
 ; GCN-O2-NEXT:        Unify divergent function exit nodes
 ; GCN-O2-NEXT:        Dominator Tree Construction
 ; GCN-O2-NEXT:        Cycle Info Analysis
@@ -1206,6 +1209,7 @@
 ; GCN-O3-NEXT:        Uniformity Analysis
 ; GCN-O3-NEXT:        AMDGPU IR late optimizations
 ; GCN-O3-NEXT:        Post-Dominator Tree Construction
+; GCN-O3-NEXT:        Uniformity Analysis
 ; GCN-O3-NEXT:        Unify divergent function exit nodes
 ; GCN-O3-NEXT:        Dominator Tree Construction
 ; GCN-O3-NEXT:        Cycle Info Analysis
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll
index 3a5507063b834..57967bc1650fe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.sat.pk.ll
@@ -16,7 +16,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
 ; SDAG-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-REAL16-NEXT:    s_wait_kmcnt 0x0
 ; SDAG-REAL16-NEXT:    v_sat_pk4_i4_i8_e32 v0.l, s2
-; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; SDAG-REAL16-NEXT:    s_endpgm
 ;
 ; SDAG-FAKE16-LABEL: sat_pk4_i4_i8_f32_v:
@@ -27,7 +27,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
 ; SDAG-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; SDAG-FAKE16-NEXT:    v_sat_pk4_i4_i8_e32 v1, s2
-; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1]
+; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1] scope:SCOPE_SE
 ; SDAG-FAKE16-NEXT:    s_endpgm
 ;
 ; GISEL-REAL16-LABEL: sat_pk4_i4_i8_f32_v:
@@ -38,7 +38,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
 ; GISEL-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-REAL16-NEXT:    s_wait_kmcnt 0x0
 ; GISEL-REAL16-NEXT:    v_sat_pk4_i4_i8_e32 v0.l, s2
-; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-REAL16-NEXT:    s_endpgm
 ;
 ; GISEL-FAKE16-LABEL: sat_pk4_i4_i8_f32_v:
@@ -49,7 +49,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
 ; GISEL-FAKE16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; GISEL-FAKE16-NEXT:    v_sat_pk4_i4_i8_e32 v0, s2
-; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-FAKE16-NEXT:    s_endpgm
   %cvt = call i16 @llvm.amdgcn.sat.pk4.i4.i8(i32 %src) #0
   store i16 %cvt, ptr %out, align 2
@@ -58,33 +58,21 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_v(i32 %src, ptr %out) #1 {
 
 define amdgpu_kernel void @sat_pk4_i4_i8_f32_s(i32 inreg %src, ptr %out) #1 {
 ; SDAG-REAL16-LABEL: sat_pk4_i4_i8_f32_s:
-; SDAG-REAL16:       ; %bb.1:
-; SDAG-REAL16-NEXT:    s_load_b32 s8, s[4:5], 0x0
-; SDAG-REAL16-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-REAL16-NEXT:    s_branch .LBB1_0
-; SDAG-REAL16-NEXT:    .p2align 8
-; SDAG-REAL16-NEXT:  ; %bb.2:
-; SDAG-REAL16-NEXT:  .LBB1_0:
+; SDAG-REAL16:       ; %bb.0:
 ; SDAG-REAL16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; SDAG-REAL16-NEXT:    v_sat_pk4_i4_i8_e32 v0.l, s8
 ; SDAG-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-REAL16-NEXT:    s_wait_kmcnt 0x0
-; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; SDAG-REAL16-NEXT:    s_endpgm
 ;
 ; SDAG-FAKE16-LABEL: sat_pk4_i4_i8_f32_s:
-; SDAG-FAKE16:       ; %bb.1:
-; SDAG-FAKE16-NEXT:    s_load_b32 s8, s[4:5], 0x0
-; SDAG-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-FAKE16-NEXT:    s_branch .LBB1_0
-; SDAG-FAKE16-NEXT:    .p2align 8
-; SDAG-FAKE16-NEXT:  ; %bb.2:
-; SDAG-FAKE16-NEXT:  .LBB1_0:
+; SDAG-FAKE16:       ; %bb.0:
 ; SDAG-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; SDAG-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-FAKE16-NEXT:    v_sat_pk4_i4_i8_e32 v1, s8
 ; SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1]
+; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1] scope:SCOPE_SE
 ; SDAG-FAKE16-NEXT:    s_endpgm
 ;
 ; GISEL-REAL16-LABEL: sat_pk4_i4_i8_f32_s:
@@ -95,7 +83,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_s(i32 inreg %src, ptr %out) #1 {
 ; GISEL-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-REAL16-NEXT:    s_wait_kmcnt 0x0
 ; GISEL-REAL16-NEXT:    v_sat_pk4_i4_i8_e32 v0.l, s2
-; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-REAL16-NEXT:    s_endpgm
 ;
 ; GISEL-FAKE16-LABEL: sat_pk4_i4_i8_f32_s:
@@ -106,7 +94,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_s(i32 inreg %src, ptr %out) #1 {
 ; GISEL-FAKE16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; GISEL-FAKE16-NEXT:    v_sat_pk4_i4_i8_e32 v0, s2
-; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-FAKE16-NEXT:    s_endpgm
   %cvt = call i16 @llvm.amdgcn.sat.pk4.i4.i8(i32 %src) #0
   store i16 %cvt, ptr %out, align 2
@@ -120,7 +108,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 {
 ; SDAG-REAL16-NEXT:    v_sat_pk4_i4_i8_e32 v0.l, 0x64
 ; SDAG-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-REAL16-NEXT:    s_wait_kmcnt 0x0
-; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; SDAG-REAL16-NEXT:    s_endpgm
 ;
 ; SDAG-FAKE16-LABEL: sat_pk4_i4_i8_f32_i:
@@ -129,7 +117,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 {
 ; SDAG-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-FAKE16-NEXT:    v_sat_pk4_i4_i8_e32 v1, 0x64
 ; SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1]
+; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1] scope:SCOPE_SE
 ; SDAG-FAKE16-NEXT:    s_endpgm
 ;
 ; GISEL-REAL16-LABEL: sat_pk4_i4_i8_f32_i:
@@ -138,7 +126,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 {
 ; GISEL-REAL16-NEXT:    v_sat_pk4_i4_i8_e32 v0.l, 0x64
 ; GISEL-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-REAL16-NEXT:    s_wait_kmcnt 0x0
-; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-REAL16-NEXT:    s_endpgm
 ;
 ; GISEL-FAKE16-LABEL: sat_pk4_i4_i8_f32_i:
@@ -147,7 +135,7 @@ define amdgpu_kernel void @sat_pk4_i4_i8_f32_i(ptr %out) #1 {
 ; GISEL-FAKE16-NEXT:    v_sat_pk4_i4_i8_e32 v0, 0x64
 ; GISEL-FAKE16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-FAKE16-NEXT:    s_endpgm
   %cvt = call i16 @llvm.amdgcn.sat.pk4.i4.i8(i32 100) #0
   store i16 %cvt, ptr %out, align 2
@@ -163,7 +151,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
 ; SDAG-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-REAL16-NEXT:    s_wait_kmcnt 0x0
 ; SDAG-REAL16-NEXT:    v_sat_pk4_u4_u8_e32 v0.l, s2
-; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; SDAG-REAL16-NEXT:    s_endpgm
 ;
 ; SDAG-FAKE16-LABEL: sat_pk4_u4_u8_f32_v:
@@ -174,7 +162,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
 ; SDAG-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; SDAG-FAKE16-NEXT:    v_sat_pk4_u4_u8_e32 v1, s2
-; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1]
+; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1] scope:SCOPE_SE
 ; SDAG-FAKE16-NEXT:    s_endpgm
 ;
 ; GISEL-REAL16-LABEL: sat_pk4_u4_u8_f32_v:
@@ -185,7 +173,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
 ; GISEL-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-REAL16-NEXT:    s_wait_kmcnt 0x0
 ; GISEL-REAL16-NEXT:    v_sat_pk4_u4_u8_e32 v0.l, s2
-; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-REAL16-NEXT:    s_endpgm
 ;
 ; GISEL-FAKE16-LABEL: sat_pk4_u4_u8_f32_v:
@@ -196,7 +184,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
 ; GISEL-FAKE16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; GISEL-FAKE16-NEXT:    v_sat_pk4_u4_u8_e32 v0, s2
-; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-FAKE16-NEXT:    s_endpgm
   %cvt = call i16 @llvm.amdgcn.sat.pk4.u4.u8(i32 %src) #0
   store i16 %cvt, ptr %out, align 2
@@ -205,33 +193,21 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_v(i32 %src, ptr %out) #1 {
 
 define amdgpu_kernel void @sat_pk4_u4_u8_f32_s(i32 inreg %src, ptr %out) #1 {
 ; SDAG-REAL16-LABEL: sat_pk4_u4_u8_f32_s:
-; SDAG-REAL16:       ; %bb.1:
-; SDAG-REAL16-NEXT:    s_load_b32 s8, s[4:5], 0x0
-; SDAG-REAL16-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-REAL16-NEXT:    s_branch .LBB4_0
-; SDAG-REAL16-NEXT:    .p2align 8
-; SDAG-REAL16-NEXT:  ; %bb.2:
-; SDAG-REAL16-NEXT:  .LBB4_0:
+; SDAG-REAL16:       ; %bb.0:
 ; SDAG-REAL16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; SDAG-REAL16-NEXT:    v_sat_pk4_u4_u8_e32 v0.l, s8
 ; SDAG-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-REAL16-NEXT:    s_wait_kmcnt 0x0
-; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; SDAG-REAL16-NEXT:    s_endpgm
 ;
 ; SDAG-FAKE16-LABEL: sat_pk4_u4_u8_f32_s:
-; SDAG-FAKE16:       ; %bb.1:
-; SDAG-FAKE16-NEXT:    s_load_b32 s8, s[4:5], 0x0
-; SDAG-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-FAKE16-NEXT:    s_branch .LBB4_0
-; SDAG-FAKE16-NEXT:    .p2align 8
-; SDAG-FAKE16-NEXT:  ; %bb.2:
-; SDAG-FAKE16-NEXT:  .LBB4_0:
+; SDAG-FAKE16:       ; %bb.0:
 ; SDAG-FAKE16-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
 ; SDAG-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-FAKE16-NEXT:    v_sat_pk4_u4_u8_e32 v1, s8
 ; SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1]
+; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1] scope:SCOPE_SE
 ; SDAG-FAKE16-NEXT:    s_endpgm
 ;
 ; GISEL-REAL16-LABEL: sat_pk4_u4_u8_f32_s:
@@ -242,7 +218,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_s(i32 inreg %src, ptr %out) #1 {
 ; GISEL-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-REAL16-NEXT:    s_wait_kmcnt 0x0
 ; GISEL-REAL16-NEXT:    v_sat_pk4_u4_u8_e32 v0.l, s2
-; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-REAL16-NEXT:    s_endpgm
 ;
 ; GISEL-FAKE16-LABEL: sat_pk4_u4_u8_f32_s:
@@ -253,7 +229,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_s(i32 inreg %src, ptr %out) #1 {
 ; GISEL-FAKE16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
 ; GISEL-FAKE16-NEXT:    v_sat_pk4_u4_u8_e32 v0, s2
-; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-FAKE16-NEXT:    s_endpgm
   %cvt = call i16 @llvm.amdgcn.sat.pk4.u4.u8(i32 %src) #0
   store i16 %cvt, ptr %out, align 2
@@ -267,7 +243,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_i(ptr %out) #1 {
 ; SDAG-REAL16-NEXT:    v_sat_pk4_u4_u8_e32 v0.l, 0x64
 ; SDAG-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; SDAG-REAL16-NEXT:    s_wait_kmcnt 0x0
-; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; SDAG-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; SDAG-REAL16-NEXT:    s_endpgm
 ;
 ; SDAG-FAKE16-LABEL: sat_pk4_u4_u8_f32_i:
@@ -276,7 +252,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_i(ptr %out) #1 {
 ; SDAG-FAKE16-NEXT:    v_mov_b32_e32 v0, 0
 ; SDAG-FAKE16-NEXT:    v_sat_pk4_u4_u8_e32 v1, 0x64
 ; SDAG-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1]
+; SDAG-FAKE16-NEXT:    flat_store_b16 v0, v1, s[0:1] scope:SCOPE_SE
 ; SDAG-FAKE16-NEXT:    s_endpgm
 ;
 ; GISEL-REAL16-LABEL: sat_pk4_u4_u8_f32_i:
@@ -285,7 +261,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_i(ptr %out) #1 {
 ; GISEL-REAL16-NEXT:    v_sat_pk4_u4_u8_e32 v0.l, 0x64
 ; GISEL-REAL16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-REAL16-NEXT:    s_wait_kmcnt 0x0
-; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-REAL16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-REAL16-NEXT:    s_endpgm
 ;
 ; GISEL-FAKE16-LABEL: sat_pk4_u4_u8_f32_i:
@@ -294,7 +270,7 @@ define amdgpu_kernel void @sat_pk4_u4_u8_f32_i(ptr %out) #1 {
 ; GISEL-FAKE16-NEXT:    v_sat_pk4_u4_u8_e32 v0, 0x64
 ; GISEL-FAKE16-NEXT:    v_mov_b32_e32 v1, 0
 ; GISEL-FAKE16-NEXT:    s_wait_kmcnt 0x0
-; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1]
+; GISEL-FAKE16-NEXT:    flat_store_b16 v1, v0, s[0:1] scope:SCOPE_SE
 ; GISEL-FAKE16-NEXT:    s_endpgm
   %cvt = call i16 @llvm.amdgcn.sat.pk4.u4.u8(i32 100) #0
   store i16 %cvt, ptr %out, align 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll
index a9a64318cee0f..18afb7c677207 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -amdgpu-scalarize-global-loads=false  -mtriple=amdgcn < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
 ; XUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
 
@@ -7,123 +8,210 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 declare float @llvm.amdgcn.div.fmas.f32(float, float, float, i1) nounwind readnone
 declare double @llvm.amdgcn.div.fmas.f64(double, double, double, i1) nounwind readnone
 
-; GCN-LABEL: {{^}}test_div_fmas_f32:
-; SI-DAG: s_load_dword [[SA:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x13
-; SI-DAG: s_load_dword [[SB:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x1c
-; SI-DAG: s_load_dword [[SC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x25
-
-; VI-DAG: s_load_dword [[SA:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x4c
-; VI-DAG: s_load_dword [[SB:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x70
-; VI-DAG: s_load_dword [[SC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x94
-
-; GCN-DAG: s_bitcmp1_b32 s{{[0-9]+}}, 0
-
-; GCN-DAG: v_mov_b32_e32 [[VC:v[0-9]+]], [[SC]]
-; GCN-DAG: v_mov_b32_e32 [[VB:v[0-9]+]], [[SB]]
-; GCN-DAG: v_mov_b32_e32 [[VA:v[0-9]+]], [[SA]]
-; GCN: v_div_fmas_f32 [[RESULT:v[0-9]+]], [[VA]], [[VB]], [[VC]]
-; GCN: buffer_store_dword [[RESULT]],
 define amdgpu_kernel void @test_div_fmas_f32(ptr addrspace(1) %out, [8 x i32], float %a, [8 x i32], float %b, [8 x i32], float %c, [8 x i32], i1 %d) nounwind {
+; GCN-LABEL: test_div_fmas_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x13
+; GCN-NEXT:    s_load_dword s2, s[4:5], 0x2e
+; GCN-NEXT:    s_load_dword s7, s[4:5], 0x1c
+; GCN-NEXT:    s_load_dword s4, s[4:5], 0x25
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp1_b32 s2, 0
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NEXT:    v_mov_b32_e32 v1, s7
+; GCN-NEXT:    v_mov_b32_e32 v2, s4
+; GCN-NEXT:    v_div_fmas_f32 v0, v0, v1, v2
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %result = call float @llvm.amdgcn.div.fmas.f32(float %a, float %b, float %c, i1 %d) nounwind readnone
   store float %result, ptr addrspace(1) %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}test_div_fmas_f32_inline_imm_0:
-; SI-DAG: s_load_dword [[SB:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x1c
-; SI-DAG: s_load_dword [[SC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x25
-; SI-DAG: v_mov_b32_e32 [[VC:v[0-9]+]], [[SC]]
-; SI-DAG: v_mov_b32_e32 [[VB:v[0-9]+]], [[SB]]
-; SI: v_div_fmas_f32 [[RESULT:v[0-9]+]], 1.0, [[VB]], [[VC]]
-; SI: buffer_store_dword [[RESULT]],
 define amdgpu_kernel void @test_div_fmas_f32_inline_imm_0(ptr addrspace(1) %out, [8 x i32], float %a, [8 x i32], float %b, [8 x i32], float %c, [8 x i32], i1 %d) nounwind {
+; GCN-LABEL: test_div_fmas_f32_inline_imm_0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT:    s_load_dword s2, s[4:5], 0x2e
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x1c
+; GCN-NEXT:    s_load_dword s4, s[4:5], 0x25
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp1_b32 s2, 0
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NEXT:    v_mov_b32_e32 v1, s4
+; GCN-NEXT:    v_div_fmas_f32 v0, 1.0, v0, v1
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %result = call float @llvm.amdgcn.div.fmas.f32(float 1.0, float %b, float %c, i1 %d) nounwind readnone
   store float %result, ptr addrspace(1) %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}test_div_fmas_f32_inline_imm_1:
-; SI-DAG: s_load_dword [[SA:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
-; SI-DAG: s_load_dword [[SC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xd
-
-; VI-DAG: s_load_dword [[SA:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x4c
-; VI-DAG: s_load_dword [[SC:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x94
-
-; GCN-DAG: v_mov_b32_e32 [[VC:v[0-9]+]], [[SC]]
-; GCN-DAG: v_mov_b32_e32 [[VA:v[0-9]+]], [[SA]]
-; GCN: v_div_fmas_f32 [[RESULT:v[0-9]+]], [[VA]], 1.0, [[VC]]
-; GCN: buffer_store_dword [[RESULT]],
 define amdgpu_kernel void @test_div_fmas_f32_inline_imm_1(ptr addrspace(1) %out, float %a, float %b, float %c, [8 x i32], i1 %d) nounwind {
+; GCN-LABEL: test_div_fmas_f32_inline_imm_1:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT:    s_load_dword s2, s[4:5], 0x16
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0xb
+; GCN-NEXT:    s_load_dword s4, s[4:5], 0xd
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp1_b32 s2, 0
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NEXT:    v_mov_b32_e32 v1, s4
+; GCN-NEXT:    v_div_fmas_f32 v0, v0, 1.0, v1
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %result = call float @llvm.amdgcn.div.fmas.f32(float %a, float 1.0, float %c, i1 %d) nounwind readnone
   store float %result, ptr addrspace(1) %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}test_div_fmas_f32_inline_imm_2:
-; SI-DAG: s_load_dword [[SA:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x13
-; SI-DAG: s_load_dword [[SB:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x1c
-
-; VI-DAG: s_load_dword [[SA:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x4c
-; VI-DAG: s_load_dword [[SB:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x70
-
-; GCN-DAG: v_mov_b32_e32 [[VA:v[0-9]+]], [[SA]]
-; GCN-DAG: v_mov_b32_e32 [[VB:v[0-9]+]], [[SB]]
-; GCN: v_div_fmas_f32 [[RESULT:v[0-9]+]], [[VA]], [[VB]], 1.0
-; GCN: buffer_store_dword [[RESULT]],
 define amdgpu_kernel void @test_div_fmas_f32_inline_imm_2(ptr addrspace(1) %out, [8 x i32], float %a, [8 x i32], float %b, [8 x i32], float %c, [8 x i32], i1 %d) nounwind {
+; GCN-LABEL: test_div_fmas_f32_inline_imm_2:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT:    s_load_dword s2, s[4:5], 0x2e
+; GCN-NEXT:    s_load_dword s6, s[4:5], 0x13
+; GCN-NEXT:    s_load_dword s4, s[4:5], 0x1c
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp1_b32 s2, 0
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NEXT:    v_mov_b32_e32 v1, s4
+; GCN-NEXT:    v_div_fmas_f32 v0, v0, v1, 1.0
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %result = call float @llvm.amdgcn.div.fmas.f32(float %a, float %b, float 1.0, i1 %d) nounwind readnone
   store float %result, ptr addrspace(1) %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}test_div_fmas_f64:
-; GCN: v_div_fmas_f64
 define amdgpu_kernel void @test_div_fmas_f64(ptr addrspace(1) %out, double %a, double %b, double %c, i1 %d) nounwind {
+; GCN-LABEL: test_div_fmas_f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s8, s[4:5], 0x11
+; GCN-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s11, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_bitcmp1_b32 s8, 0
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_mov_b32 s8, s0
+; GCN-NEXT:    s_mov_b32 s9, s1
+; GCN-NEXT:    v_mov_b32_e32 v0, s2
+; GCN-NEXT:    v_mov_b32_e32 v1, s3
+; GCN-NEXT:    v_mov_b32_e32 v2, s4
+; GCN-NEXT:    v_mov_b32_e32 v3, s5
+; GCN-NEXT:    v_mov_b32_e32 v4, s6
+; GCN-NEXT:    v_mov_b32_e32 v5, s7
+; GCN-NEXT:    v_div_fmas_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GCN-NEXT:    s_endpgm
   %result = call double @llvm.amdgcn.div.fmas.f64(double %a, double %b, double %c, i1 %d) nounwind readnone
   store double %result, ptr addrspace(1) %out, align 8
   ret void
 }
 
-; GCN-LABEL: {{^}}test_div_fmas_f32_cond_to_vcc:
-; GCN: s_cmp_eq_u32 s{{[0-9]+}}, 0{{$}}
-; GCN: v_div_fmas_f32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
 define amdgpu_kernel void @test_div_fmas_f32_cond_to_vcc(ptr addrspace(1) %out, float %a, float %b, float %c, i32 %i) nounwind {
+; GCN-LABEL: test_div_fmas_f32_cond_to_vcc:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_cmp_eq_u32 s3, 0
+; GCN-NEXT:    s_cselect_b64 vcc, -1, 0
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    v_div_fmas_f32 v0, v0, v1, v2
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %cmp = icmp eq i32 %i, 0
   %result = call float @llvm.amdgcn.div.fmas.f32(float %a, float %b, float %c, i1 %cmp) nounwind readnone
   store float %result, ptr addrspace(1) %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}test_div_fmas_f32_imm_false_cond_to_vcc:
-; GCN: s_mov_b64 vcc, 0
-; GCN: v_div_fmas_f32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
 define amdgpu_kernel void @test_div_fmas_f32_imm_false_cond_to_vcc(ptr addrspace(1) %out, float %a, float %b, float %c) nounwind {
+; GCN-LABEL: test_div_fmas_f32_imm_false_cond_to_vcc:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    s_mov_b64 vcc, 0
+; GCN-NEXT:    v_div_fmas_f32 v0, v0, v1, v2
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %result = call float @llvm.amdgcn.div.fmas.f32(float %a, float %b, float %c, i1 false) nounwind readnone
   store float %result, ptr addrspace(1) %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}test_div_fmas_f32_imm_true_cond_to_vcc:
-; GCN: s_mov_b64 vcc, -1
-; GCN: v_div_fmas_f32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
 define amdgpu_kernel void @test_div_fmas_f32_imm_true_cond_to_vcc(ptr addrspace(1) %out, float %a, float %b, float %c) nounwind {
+; GCN-LABEL: test_div_fmas_f32_imm_true_cond_to_vcc:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    v_mov_b32_e32 v1, s1
+; GCN-NEXT:    v_mov_b32_e32 v2, s2
+; GCN-NEXT:    s_mov_b64 vcc, -1
+; GCN-NEXT:    v_div_fmas_f32 v0, v0, v1, v2
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_endpgm
   %result = call float @llvm.amdgcn.div.fmas.f32(float %a, float %b, float %c, i1 true) nounwind readnone
   store float %result, ptr addrspace(1) %out, align 4
   ret void
 }
 
-; GCN-LABEL: {{^}}test_div_fmas_f32_logical_cond_to_vcc:
-; SI-DAG: buffer_load_dword [[A:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dword [[B:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:4 glc{{$}}
-; SI-DAG: buffer_load_dword [[C:v[0-9]+]], {{v\[[0-9]+:[0-9]+\]}}, {{s\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-
-; SI-DAG: v_cmp_eq_u32_e32 [[CMP0:vcc]], 0, v{{[0-9]+}}
-; SI-DAG: s_cmp_lg_u32 s{{[0-9]+}}, 0{{$}}
-; SI-DAG: s_cselect_b64 [[CMP1:s\[[0-9]+:[0-9]+\]]], -1, 0
-; SI: s_and_b64 vcc, [[CMP0]], [[CMP1]]
-; SI: v_div_fmas_f32 {{v[0-9]+}}, [[A]], [[B]], [[C]]
-; SI: s_endpgm
 define amdgpu_kernel void @test_div_fmas_f32_logical_cond_to_vcc(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %d) nounwind {
+; GCN-LABEL: test_div_fmas_f32_logical_cond_to_vcc:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT:    s_load_dword s8, s[4:5], 0xd
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GCN-NEXT:    buffer_load_dword v0, v[1:2], s[4:7], 0 addr64 glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_load_dword v3, v[1:2], s[4:7], 0 addr64 offset:4 glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    buffer_load_dword v1, v[1:2], s[4:7], 0 addr64 offset:8 glc
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_cmp_lg_u32 s8, 0
+; GCN-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GCN-NEXT:    s_and_b64 vcc, vcc, s[2:3]
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_mov_b32 s3, s7
+; GCN-NEXT:    v_div_fmas_f32 v0, v0, v3, v1
+; GCN-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:8
+; GCN-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.a = getelementptr float, ptr addrspace(1) %in, i32 %tid
   %gep.b = getelementptr float, ptr addrspace(1) %gep.a, i32 1
@@ -143,26 +231,39 @@ define amdgpu_kernel void @test_div_fmas_f32_logical_cond_to_vcc(ptr addrspace(1
   ret void
 }
 
-; GCN-LABEL: {{^}}test_div_fmas_f32_i1_phi_vcc:
-
-; SI: ; %entry
-; SI:     v_cmp_eq_u32_e64   [[CMP:s\[[0-9]+:[0-9]+\]]], 0, {{v[0-9]+}}
-; SI:     s_mov_b64          vcc, 0
-; SI:     s_and_saveexec_b64 [[SAVE:s\[[0-9]+:[0-9]+\]]], [[CMP]]
-
-; SI: ; %bb
-; SI:     buffer_load_dword  [[LOAD:v[0-9]+]],
-; SI:     v_cmp_ne_u32_e32   vcc, 0, [[LOAD]]
-; SI:     s_and_b64          vcc, vcc, exec
-
-; SI: ; %exit
-; SI:     s_or_b64           exec, exec, [[SAVE]]
-; SI-NOT: vcc
-; SI:     v_div_fmas_f32 {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
-; SI:     buffer_store_dword
-; SI:     s_endpgm
-
 define amdgpu_kernel void @test_div_fmas_f32_i1_phi_vcc(ptr addrspace(1) %out, ptr addrspace(1) %in, ptr addrspace(1) %dummy) nounwind {
+; GCN-LABEL: test_div_fmas_f32_i1_phi_vcc:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GCN-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[0:1], s[10:11]
+; GCN-NEXT:    buffer_load_dwordx2 v[1:2], v[3:4], s[0:3], 0 addr64
+; GCN-NEXT:    buffer_load_dword v3, v[3:4], s[0:3], 0 addr64 offset:8
+; GCN-NEXT:    v_cmp_eq_u32_e64 s[0:1], 0, v0
+; GCN-NEXT:    s_mov_b64 vcc, 0
+; GCN-NEXT:    s_and_saveexec_b64 s[10:11], s[0:1]
+; GCN-NEXT:    s_cbranch_execz .LBB9_2
+; GCN-NEXT:  ; %bb.1: ; %bb
+; GCN-NEXT:    s_mov_b32 s6, -1
+; GCN-NEXT:    s_mov_b32 s7, s3
+; GCN-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GCN-NEXT:    s_and_b64 vcc, vcc, exec
+; GCN-NEXT:  .LBB9_2: ; %exit
+; GCN-NEXT:    s_or_b64 exec, exec, s[10:11]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    v_div_fmas_f32 v0, v1, v2, v3
+; GCN-NEXT:    s_mov_b32 s10, -1
+; GCN-NEXT:    s_mov_b32 s11, s3
+; GCN-NEXT:    buffer_store_dword v0, off, s[8:11], 0 offset:8
+; GCN-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep.out = getelementptr float, ptr addrspace(1) %out, i32 2
@@ -188,3 +289,5 @@ exit:
   store float %result, ptr addrspace(1) %gep.out, align 4
   ret void
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; SI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll
index 89555d3060883..d5fba2df0b828 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.flat.prefetch.ll
@@ -31,7 +31,7 @@ define amdgpu_ps void @flat_prefetch_offset(ptr %ptr) {
 ; GCN-NEXT:    flat_prefetch_b8 v[0:1] offset:512
 ; GCN-NEXT:    s_endpgm
 entry:
-  %gep = getelementptr i32, ptr %ptr, i32 128
+  %gep = getelementptr inbounds i32, ptr %ptr, i32 128
   tail call void @llvm.amdgcn.flat.prefetch(ptr %gep, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll
index 047a6ccf10d91..80f9eeb25ebc0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.global.prefetch.ll
@@ -31,7 +31,7 @@ define amdgpu_ps void @global_prefetch_offset(ptr addrspace(1) %ptr) {
 ; GCN-NEXT:    global_prefetch_b8 v[0:1], off offset:512
 ; GCN-NEXT:    s_endpgm
 entry:
-  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 128
+  %gep = getelementptr inbounds i32, ptr addrspace(1) %ptr, i32 128
   tail call void @llvm.amdgcn.global.prefetch(ptr addrspace(1) %gep, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
index 3b4db4a4075fd..d45705edce2c8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -18,6 +18,16 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -74,6 +84,16 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_cube:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_cube:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -132,6 +152,16 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_2darray:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_2darray:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -190,6 +220,16 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_c_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v1, v2, v1, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_c_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -246,6 +286,16 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_cl_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4_cl v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_cl_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -306,6 +356,16 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_c_cl_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v1, v2, v1, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4_c_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_c_cl_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -362,6 +422,16 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_b_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v1, v2, v1, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_b_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -418,6 +488,16 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_c_b_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v2, v3, v2, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_c_b_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -476,6 +556,16 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_b_cl_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v1, v2, v1, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4_b_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_b_cl_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -538,6 +628,16 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_c_b_cl_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b32 s12, exec_lo
+; GFX10-NEXT:    s_wqm_b32 exec_lo, exec_lo
+; GFX10-NEXT:    v_perm_b32 v2, v3, v2, 0x5040100
+; GFX10-NEXT:    s_and_b32 exec_lo, exec_lo, s12
+; GFX10-NEXT:    image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_c_b_cl_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    s_mov_b32 s12, exec_lo
@@ -591,6 +691,13 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_l_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    image_gather4_l v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_l_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.l
@@ -636,6 +743,13 @@ define amdgpu_ps <4 x float> @gather4_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> in
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_c_l_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_perm_b32 v1, v2, v1, 0x5040100
+; GFX10-NEXT:    image_gather4_c_l v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_c_l_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v2.l
@@ -677,6 +791,13 @@ define amdgpu_ps <4 x float> @gather4_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_lz_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_perm_b32 v0, v1, v0, 0x5040100
+; GFX10-NEXT:    image_gather4_lz v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_lz_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v1.l
@@ -718,6 +839,13 @@ define amdgpu_ps <4 x float> @gather4_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> i
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    ; return to shader part epilog
 ;
+; GFX10-LABEL: gather4_c_lz_2d:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    v_perm_b32 v1, v2, v1, 0x5040100
+; GFX10-NEXT:    image_gather4_c_lz v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    ; return to shader part epilog
+;
 ; GFX11-TRUE16-LABEL: gather4_c_lz_2d:
 ; GFX11-TRUE16:       ; %bb.0: ; %main_body
 ; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v2.l
@@ -773,5 +901,4 @@ attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
 attributes #2 = { nounwind readnone }
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10: {{.*}}
 ; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
index 462090c6e89df..0a2e7afa3d417 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
@@ -1,12 +1,46 @@
-; RUN: llc -mtriple=amdgcn -mcpu=verde < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize64 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+wavefrontsize64 -amdgpu-enable-delay-alu=0 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11-12,GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -amdgpu-enable-delay-alu=0 < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11-12,GFX12 %s
 
-; GCN-LABEL: {{^}}gs_const:
-; GCN-NOT: v_cmpx
-; GCN: s_mov_b64 exec, 0
 define amdgpu_gs void @gs_const() {
+; SI-LABEL: gs_const:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b64 s[0:1], exec
+; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: gs_const:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b64 s[0:1], exec
+; GFX10-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-12-LABEL: gs_const:
+; GFX11-12:       ; %bb.0:
+; GFX11-12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX11-12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], exec
+; GFX11-12-NEXT:    s_mov_b64 exec, 0
+; GFX11-12-NEXT:    s_mov_b32 m0, 0
+; GFX11-12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-12-NEXT:    s_endpgm
+; GFX11-12-NEXT:  ; %bb.1:
+; GFX11-12-NEXT:    s_mov_b64 exec, 0
+; GFX11-12-NEXT:    s_endpgm
   %tmp = icmp ule i32 0, 3
   %tmp1 = select i1 %tmp, float 1.000000e+00, float -1.000000e+00
   %c1 = fcmp oge float %tmp1, 0.0
@@ -19,12 +53,81 @@ define amdgpu_gs void @gs_const() {
   ret void
 }
 
-; GCN-LABEL: {{^}}vcc_implicit_def:
-; GCN: v_cmp_nle_f32_e32 vcc, 0, v{{[0-9]+}}
-; GCN: v_cmp_gt_f32_e64 [[CMP:s\[[0-9]+:[0-9]+\]]], 0, v{{[0-9]+}}
-; GCN: s_and{{n2|_not1}}_b64 exec, exec, vcc
-; GCN: v_cndmask_b32_e64 v{{[0-9]+}}, 0, 1.0, [[CMP]]
 define amdgpu_ps void @vcc_implicit_def(float %arg13, float %arg14) {
+; SI-LABEL: vcc_implicit_def:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v1
+; SI-NEXT:    v_cmp_gt_f32_e64 s[0:1], 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_cbranch_scc0 .LBB1_2
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
+; SI-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB1_2:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    exp null off, off, off, off done vm
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: vcc_implicit_def:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v1
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s[0:1], 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_cbranch_scc0 .LBB1_2
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    exp mrt1 v0, v0, v0, v0 done vm
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  .LBB1_2:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    exp null off, off, off, off done vm
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: vcc_implicit_def:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v1
+; GFX11-NEXT:    v_cmp_gt_f32_e64 s[0:1], 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_cbranch_scc0 .LBB1_2
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s[0:1]
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    exp mrt1 v0, v0, v0, v0 done
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  .LBB1_2:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    exp mrt0 off, off, off, off done
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: vcc_implicit_def:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_le_f32_e64 s[0:1], 0, v1
+; GFX12-NEXT:    s_mov_b64 s[2:3], exec
+; GFX12-NEXT:    v_cmp_gt_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], exec, s[0:1]
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], s[2:3], s[0:1]
+; GFX12-NEXT:    s_cbranch_scc0 .LBB1_2
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_and_b64 exec, exec, s[2:3]
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    export mrt1 v0, v0, v0, v0 done
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  .LBB1_2:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    export mrt0 off, off, off, off done
+; GFX12-NEXT:    s_endpgm
   %tmp0 = fcmp olt float %arg13, 0.000000e+00
   %c1 = fcmp oge float %arg14, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
@@ -34,31 +137,102 @@ define amdgpu_ps void @vcc_implicit_def(float %arg13, float %arg14) {
   ret void
 }
 
-; GCN-LABEL: {{^}}true:
-; GCN-NEXT: %bb.
-; GCN-NEXT: s_endpgm
 define amdgpu_gs void @true() {
+; GCN-LABEL: true:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_endpgm
   call void @llvm.amdgcn.kill(i1 true)
   ret void
 }
 
-; GCN-LABEL: {{^}}false:
-; GCN-NOT: v_cmpx
-; GCN: s_mov_b64 exec, 0
 define amdgpu_gs void @false() {
+; SI-LABEL: false:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_andn2_b64 exec, exec, exec
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: false:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_andn2_b64 exec, exec, exec
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-12-LABEL: false:
+; GFX11-12:       ; %bb.0:
+; GFX11-12-NEXT:    s_and_not1_b64 exec, exec, exec
+; GFX11-12-NEXT:    s_mov_b64 exec, 0
+; GFX11-12-NEXT:    s_mov_b32 m0, 0
+; GFX11-12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-12-NEXT:    s_endpgm
+; GFX11-12-NEXT:  ; %bb.1:
+; GFX11-12-NEXT:    s_mov_b64 exec, 0
+; GFX11-12-NEXT:    s_endpgm
   call void @llvm.amdgcn.kill(i1 false)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}and:
-; GCN: v_cmp_lt_i32
-; GCN: v_cmp_lt_i32
-; GCN: s_or_b64 s[0:1]
-; GCN: s_and{{n2|_not1}}_b64 s[0:1], exec, s[0:1]
-; GCN: s_and{{n2|_not1}}_b64 s[2:3], s[2:3], s[0:1]
-; GCN: s_and_b64 exec, exec, s[2:3]
 define amdgpu_gs void @and(i32 %a, i32 %b, i32 %c, i32 %d) {
+; SI-LABEL: and:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, v0, v1
+; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v3
+; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; SI-NEXT:    s_mov_b64 s[2:3], exec
+; SI-NEXT:    s_andn2_b64 s[0:1], exec, s[0:1]
+; SI-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[0:1]
+; SI-NEXT:    s_and_b64 exec, exec, s[2:3]
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: and:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_lt_i32_e32 vcc, v0, v1
+; GFX10-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v3
+; GFX10-NEXT:    s_mov_b64 s[2:3], exec
+; GFX10-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX10-NEXT:    s_andn2_b64 s[0:1], exec, s[0:1]
+; GFX10-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT:    s_and_b64 exec, exec, s[2:3]
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-12-LABEL: and:
+; GFX11-12:       ; %bb.0:
+; GFX11-12-NEXT:    v_cmp_lt_i32_e32 vcc, v0, v1
+; GFX11-12-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v3
+; GFX11-12-NEXT:    s_mov_b64 s[2:3], exec
+; GFX11-12-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
+; GFX11-12-NEXT:    s_and_not1_b64 s[0:1], exec, s[0:1]
+; GFX11-12-NEXT:    s_and_not1_b64 s[2:3], s[2:3], s[0:1]
+; GFX11-12-NEXT:    s_and_b64 exec, exec, s[2:3]
+; GFX11-12-NEXT:    s_mov_b32 m0, 0
+; GFX11-12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-12-NEXT:    s_endpgm
+; GFX11-12-NEXT:  ; %bb.1:
+; GFX11-12-NEXT:    s_mov_b64 exec, 0
+; GFX11-12-NEXT:    s_endpgm
   %c1 = icmp slt i32 %a, %b
   %c2 = icmp slt i32 %c, %d
   %x = or i1 %c1, %c2
@@ -67,13 +241,52 @@ define amdgpu_gs void @and(i32 %a, i32 %b, i32 %c, i32 %d) {
   ret void
 }
 
-; GCN-LABEL: {{^}}andn2:
-; GCN: v_cmp_lt_i32
-; GCN: v_cmp_lt_i32
-; GCN: s_xor_b64 s[0:1]
-; GCN: s_and{{n2|_not1}}_b64 s[2:3], s[2:3], s[0:1]
-; GCN: s_and_b64 exec, exec, s[2:3]
 define amdgpu_gs void @andn2(i32 %a, i32 %b, i32 %c, i32 %d) {
+; SI-LABEL: andn2:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, v0, v1
+; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v3
+; SI-NEXT:    s_mov_b64 s[2:3], exec
+; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; SI-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[0:1]
+; SI-NEXT:    s_and_b64 exec, exec, s[2:3]
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: andn2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_lt_i32_e32 vcc, v0, v1
+; GFX10-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v3
+; GFX10-NEXT:    s_mov_b64 s[2:3], exec
+; GFX10-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; GFX10-NEXT:    s_andn2_b64 s[2:3], s[2:3], s[0:1]
+; GFX10-NEXT:    s_and_b64 exec, exec, s[2:3]
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-12-LABEL: andn2:
+; GFX11-12:       ; %bb.0:
+; GFX11-12-NEXT:    v_cmp_lt_i32_e32 vcc, v0, v1
+; GFX11-12-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v3
+; GFX11-12-NEXT:    s_mov_b64 s[2:3], exec
+; GFX11-12-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; GFX11-12-NEXT:    s_and_not1_b64 s[2:3], s[2:3], s[0:1]
+; GFX11-12-NEXT:    s_and_b64 exec, exec, s[2:3]
+; GFX11-12-NEXT:    s_mov_b32 m0, 0
+; GFX11-12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-12-NEXT:    s_endpgm
+; GFX11-12-NEXT:  ; %bb.1:
+; GFX11-12-NEXT:    s_mov_b64 exec, 0
+; GFX11-12-NEXT:    s_endpgm
   %c1 = icmp slt i32 %a, %b
   %c2 = icmp slt i32 %c, %d
   %x = xor i1 %c1, %c2
@@ -83,135 +296,854 @@ define amdgpu_gs void @andn2(i32 %a, i32 %b, i32 %c, i32 %d) {
   ret void
 }
 
-; GCN-LABEL: {{^}}oeq:
-; GCN: v_cmp_neq_f32
+; Should use v_cmp_neq_f32
 define amdgpu_gs void @oeq(float %a) {
+; SI-LABEL: oeq:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_neq_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: oeq:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_neq_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: oeq:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_neq_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: oeq:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp oeq float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}ogt:
-; GCN: v_cmp_nlt_f32
+; Should use v_cmp_nlt_f32
 define amdgpu_gs void @ogt(float %a) {
+; SI-LABEL: ogt:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_nlt_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: ogt:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_nlt_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: ogt:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_nlt_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: ogt:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_lt_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp ogt float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}oge:
-; GCN: v_cmp_nle_f32
+; Should use v_cmp_nle_f32
 define amdgpu_gs void @oge(float %a) {
+; SI-LABEL: oge:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: oge:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: oge:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: oge:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_le_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp oge float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}olt:
-; GCN: v_cmp_ngt_f32
+; Should use v_cmp_ngt_f32
 define amdgpu_gs void @olt(float %a) {
+; SI-LABEL: olt:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: olt:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: olt:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_ngt_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: olt:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_gt_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp olt float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}ole:
-; GCN: v_cmp_nge_f32
+; Should use v_cmp_nge_f32
 define amdgpu_gs void @ole(float %a) {
+; SI-LABEL: ole:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_nge_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: ole:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_nge_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: ole:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_nge_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: ole:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp ole float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}one:
-; GCN: v_cmp_nlg_f32
+; Should use v_cmp_nlg_f32
 define amdgpu_gs void @one(float %a) {
+; SI-LABEL: one:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_nlg_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: one:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_nlg_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: one:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_nlg_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: one:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_lg_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp one float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}ord:
-; GCN: v_cmp_o_f32
+; Should use v_cmp_o_f32
 define amdgpu_gs void @ord(float %a) {
+; SI-LABEL: ord:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
+; SI-NEXT:    s_mov_b64 s[0:1], exec
+; SI-NEXT:    s_andn2_b64 s[2:3], exec, vcc
+; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; SI-NEXT:    s_and_b64 exec, exec, s[0:1]
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: ord:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
+; GFX10-NEXT:    s_mov_b64 s[0:1], exec
+; GFX10-NEXT:    s_andn2_b64 s[2:3], exec, vcc
+; GFX10-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-12-LABEL: ord:
+; GFX11-12:       ; %bb.0:
+; GFX11-12-NEXT:    v_cmp_o_f32_e32 vcc, v0, v0
+; GFX11-12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX11-12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX11-12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX11-12-NEXT:    s_mov_b32 m0, 0
+; GFX11-12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-12-NEXT:    s_endpgm
+; GFX11-12-NEXT:  ; %bb.1:
+; GFX11-12-NEXT:    s_mov_b64 exec, 0
+; GFX11-12-NEXT:    s_endpgm
   %c1 = fcmp ord float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}uno:
-; GCN: v_cmp_u_f32
+; Should use v_cmp_u_f32
 define amdgpu_gs void @uno(float %a) {
+; SI-LABEL: uno:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; SI-NEXT:    s_mov_b64 s[0:1], exec
+; SI-NEXT:    s_andn2_b64 s[2:3], exec, vcc
+; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; SI-NEXT:    s_and_b64 exec, exec, s[0:1]
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: uno:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX10-NEXT:    s_mov_b64 s[0:1], exec
+; GFX10-NEXT:    s_andn2_b64 s[2:3], exec, vcc
+; GFX10-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-12-LABEL: uno:
+; GFX11-12:       ; %bb.0:
+; GFX11-12-NEXT:    v_cmp_u_f32_e32 vcc, v0, v0
+; GFX11-12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX11-12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX11-12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX11-12-NEXT:    s_mov_b32 m0, 0
+; GFX11-12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-12-NEXT:    s_endpgm
+; GFX11-12-NEXT:  ; %bb.1:
+; GFX11-12-NEXT:    s_mov_b64 exec, 0
+; GFX11-12-NEXT:    s_endpgm
   %c1 = fcmp uno float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}ueq:
-; GCN: v_cmp_lg_f32
+; Should use v_cmp_lg_f32
 define amdgpu_gs void @ueq(float %a) {
+; SI-LABEL: ueq:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_lg_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: ueq:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_lg_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: ueq:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_lg_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: ueq:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_nlg_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp ueq float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}ugt:
-; GCN: v_cmp_ge_f32
+; Should use v_cmp_ge_f32
 define amdgpu_gs void @ugt(float %a) {
+; SI-LABEL: ugt:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: ugt:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: ugt:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_ge_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: ugt:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_nge_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp ugt float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}uge:
-; GCN: v_cmp_gt_f32_e32 vcc, -1.0
+; Should use v_cmp_gt_f32_e32 vcc, -1.0
 define amdgpu_gs void @uge(float %a) {
+; SI-LABEL: uge:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_gt_f32_e32 vcc, -1.0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: uge:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc, -1.0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: uge:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc, -1.0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: uge:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc, -1.0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp uge float %a, -1.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}ult:
-; GCN: v_cmp_le_f32_e32 vcc, -2.0
+; Should use v_cmp_le_f32_e32 vcc, -2.0
 define amdgpu_gs void @ult(float %a) {
+; SI-LABEL: ult:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_le_f32_e32 vcc, -2.0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: ult:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_le_f32_e32 vcc, -2.0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: ult:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_le_f32_e32 vcc, -2.0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: ult:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_nle_f32_e32 vcc, -2.0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp ult float %a, -2.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}ule:
-; GCN: v_cmp_lt_f32_e32 vcc, 2.0
+; Should use v_cmp_lt_f32_e32 vcc, 2.0
 define amdgpu_gs void @ule(float %a) {
+; SI-LABEL: ule:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_lt_f32_e32 vcc, 2.0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: ule:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc, 2.0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: ule:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc, 2.0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: ule:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_nlt_f32_e32 vcc, 2.0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp ule float %a, 2.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}une:
-; GCN: v_cmp_eq_f32_e32 vcc, 0
+; Should use v_cmp_eq_f32_e32 vcc, 0
 define amdgpu_gs void @une(float %a) {
+; SI-LABEL: une:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: une:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: une:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: une:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_neq_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp une float %a, 0.0
   call void @llvm.amdgcn.kill(i1 %c1)
   call void @llvm.amdgcn.s.sendmsg(i32 3, i32 0)
   ret void
 }
 
-; GCN-LABEL: {{^}}neg_olt:
-; GCN: v_cmp_gt_f32_e32 vcc, 1.0
+; Should use v_cmp_gt_f32_e32 vcc, 1.0
 define amdgpu_gs void @neg_olt(float %a) {
+; SI-LABEL: neg_olt:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_gt_f32_e32 vcc, 1.0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_mov_b32 m0, 0
+; SI-NEXT:    s_nop 0
+; SI-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: neg_olt:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc, 1.0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_mov_b32 m0, 0
+; GFX10-NEXT:    s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP)
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: neg_olt:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_gt_f32_e32 vcc, 1.0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_mov_b32 m0, 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: neg_olt:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_ngt_f32_e32 vcc, 1.0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_mov_b32 m0, 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    s_endpgm
   %c1 = fcmp olt float %a, 1.0
   %c2 = xor i1 %c1, 1
   call void @llvm.amdgcn.kill(i1 %c2)
@@ -219,13 +1151,61 @@ define amdgpu_gs void @neg_olt(float %a) {
   ret void
 }
 
-; GCN-LABEL: {{^}}fcmp_x2:
 ; FIXME: LLVM should be able to combine these fcmp opcodes.
-; SI: v_cmp_lt_f32_e32 vcc, s{{[0-9]+}}, v0
-; GFX10: v_cmp_lt_f32_e32 vcc, 0x3e800000, v0
-; GCN: v_cndmask_b32
-; GCN: v_cmp_nle_f32
 define amdgpu_ps void @fcmp_x2(float %a) #0 {
+; SI-LABEL: fcmp_x2:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_mov_b32 s0, 0x3e800000
+; SI-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; SI-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_cbranch_scc0 .LBB21_1
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB21_1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    exp null off, off, off, off done vm
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: fcmp_x2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc, 0x3e800000, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX10-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_cbranch_scc0 .LBB21_1
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  .LBB21_1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    exp null off, off, off, off done vm
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fcmp_x2:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_lt_f32_e32 vcc, 0x3e800000, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX11-NEXT:    v_cmp_nle_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_cbranch_scc0 .LBB21_1
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  .LBB21_1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    exp mrt0 off, off, off, off done
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: fcmp_x2:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_lt_f32_e32 vcc, 0x3e800000, v0
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, -1.0, vcc
+; GFX12-NEXT:    v_cmp_le_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, vcc
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], exec, s[2:3]
+; GFX12-NEXT:    s_cbranch_scc0 .LBB21_1
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  .LBB21_1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    export mrt0 off, off, off, off done
+; GFX12-NEXT:    s_endpgm
   %ogt = fcmp nsz ogt float %a, 2.500000e-01
   %k = select i1 %ogt, float -1.000000e+00, float 0.000000e+00
   %c = fcmp nsz oge float %k, 0.000000e+00
@@ -234,14 +1214,78 @@ define amdgpu_ps void @fcmp_x2(float %a) #0 {
 }
 
 ; Note: an almost identical test for this exists in llvm.amdgcn.wqm.vote.ll
-; GCN-LABEL: {{^}}wqm:
-; GCN: v_cmp_neq_f32_e32 vcc, 0
-; GCN-DAG: s_wqm_b64 s[2:3], vcc
-; GCN-DAG: s_mov_b64 s[0:1], exec
-; GCN: s_and{{n2|_not1}}_b64 s[2:3], exec, s[2:3]
-; GCN: s_and{{n2|_not1}}_b64 s[0:1], s[0:1], s[2:3]
-; GCN: s_and_b64 exec, exec, s[0:1]
 define amdgpu_ps float @wqm(float %a) {
+; SI-LABEL: wqm:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_neq_f32_e32 vcc, 0, v0
+; SI-NEXT:    s_wqm_b64 s[2:3], vcc
+; SI-NEXT:    s_mov_b64 s[0:1], exec
+; SI-NEXT:    s_andn2_b64 s[2:3], exec, s[2:3]
+; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; SI-NEXT:    s_cbranch_scc0 .LBB22_2
+; SI-NEXT:  ; %bb.1:
+; SI-NEXT:    s_and_b64 exec, exec, s[0:1]
+; SI-NEXT:    v_mov_b32_e32 v0, 0
+; SI-NEXT:    s_branch .LBB22_3
+; SI-NEXT:  .LBB22_2:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    exp null off, off, off, off done vm
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB22_3:
+;
+; GFX10-LABEL: wqm:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_neq_f32_e32 vcc, 0, v0
+; GFX10-NEXT:    s_mov_b64 s[0:1], exec
+; GFX10-NEXT:    s_wqm_b64 s[2:3], vcc
+; GFX10-NEXT:    s_andn2_b64 s[2:3], exec, s[2:3]
+; GFX10-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT:    s_cbranch_scc0 .LBB22_2
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0
+; GFX10-NEXT:    s_branch .LBB22_3
+; GFX10-NEXT:  .LBB22_2:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    exp null off, off, off, off done vm
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  .LBB22_3:
+;
+; GFX11-LABEL: wqm:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_neq_f32_e32 vcc, 0, v0
+; GFX11-NEXT:    s_mov_b64 s[0:1], exec
+; GFX11-NEXT:    s_wqm_b64 s[2:3], vcc
+; GFX11-NEXT:    s_and_not1_b64 s[2:3], exec, s[2:3]
+; GFX11-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-NEXT:    s_cbranch_scc0 .LBB22_2
+; GFX11-NEXT:  ; %bb.1:
+; GFX11-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NEXT:    s_branch .LBB22_3
+; GFX11-NEXT:  .LBB22_2:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    exp mrt0 off, off, off, off done
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  .LBB22_3:
+;
+; GFX12-LABEL: wqm:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cmp_neq_f32_e32 vcc, 0, v0
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_wqm_b64 s[2:3], vcc
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, s[2:3]
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT:    s_cbranch_scc0 .LBB22_2
+; GFX12-NEXT:  ; %bb.1:
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-NEXT:    s_branch .LBB22_3
+; GFX12-NEXT:  .LBB22_2:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    export mrt0 off, off, off, off done
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  .LBB22_3:
   %c1 = fcmp une float %a, 0.0
   %c2 = call i1 @llvm.amdgcn.wqm.vote(i1 %c1)
   call void @llvm.amdgcn.kill(i1 %c2)
@@ -249,28 +1293,212 @@ define amdgpu_ps float @wqm(float %a) {
 }
 
 ; This checks that we use the 64-bit encoding when the operand is a SGPR.
-; GCN-LABEL: {{^}}test_sgpr:
-; GCN: v_cmp_nle_f32_e64
 define amdgpu_ps void @test_sgpr(float inreg %a) #0 {
+; SI-LABEL: test_sgpr:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_cmp_nle_f32_e64 vcc, s0, 1.0
+; SI-NEXT:    s_andn2_b64 exec, exec, vcc
+; SI-NEXT:    s_cbranch_scc0 .LBB23_1
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB23_1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    exp null off, off, off, off done vm
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: test_sgpr:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_nle_f32_e64 vcc, s0, 1.0
+; GFX10-NEXT:    s_andn2_b64 exec, exec, vcc
+; GFX10-NEXT:    s_cbranch_scc0 .LBB23_1
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  .LBB23_1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    exp null off, off, off, off done vm
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: test_sgpr:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_nle_f32_e64 vcc, s0, 1.0
+; GFX11-NEXT:    s_and_not1_b64 exec, exec, vcc
+; GFX11-NEXT:    s_cbranch_scc0 .LBB23_1
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  .LBB23_1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    exp mrt0 off, off, off, off done
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: test_sgpr:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_cmp_le_f32 s0, 1.0
+; GFX12-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], exec, s[0:1]
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, s[0:1]
+; GFX12-NEXT:    s_cbranch_scc0 .LBB23_1
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  .LBB23_1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    export mrt0 off, off, off, off done
+; GFX12-NEXT:    s_endpgm
   %c = fcmp ole float %a, 1.000000e+00
   call void @llvm.amdgcn.kill(i1 %c) #1
   ret void
 }
 
-; GCN-LABEL: {{^}}test_non_inline_imm_sgpr:
-; GCN-NOT: v_cmp_le_f32_e64
 define amdgpu_ps void @test_non_inline_imm_sgpr(float inreg %a) #0 {
+; SI-LABEL: test_non_inline_imm_sgpr:
+; SI:       ; %bb.0:
+; SI-NEXT:    v_mov_b32_e32 v0, 0x3fc00000
+; SI-NEXT:    v_cmp_le_f32_e32 vcc, s0, v0
+; SI-NEXT:    s_andn2_b64 s[0:1], exec, vcc
+; SI-NEXT:    s_andn2_b64 s[2:3], exec, s[0:1]
+; SI-NEXT:    s_cbranch_scc0 .LBB24_1
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB24_1:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    exp null off, off, off, off done vm
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: test_non_inline_imm_sgpr:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cmp_ge_f32_e64 s[0:1], 0x3fc00000, s0
+; GFX10-NEXT:    s_andn2_b64 s[0:1], exec, s[0:1]
+; GFX10-NEXT:    s_andn2_b64 s[2:3], exec, s[0:1]
+; GFX10-NEXT:    s_cbranch_scc0 .LBB24_1
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  .LBB24_1:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    exp null off, off, off, off done vm
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: test_non_inline_imm_sgpr:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cmp_ge_f32_e64 s[0:1], 0x3fc00000, s0
+; GFX11-NEXT:    s_and_not1_b64 s[0:1], exec, s[0:1]
+; GFX11-NEXT:    s_and_not1_b64 s[2:3], exec, s[0:1]
+; GFX11-NEXT:    s_cbranch_scc0 .LBB24_1
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  .LBB24_1:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    exp mrt0 off, off, off, off done
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: test_non_inline_imm_sgpr:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_cmp_le_f32 s0, 0x3fc00000
+; GFX12-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], exec, s[0:1]
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], exec, s[0:1]
+; GFX12-NEXT:    s_cbranch_scc0 .LBB24_1
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  .LBB24_1:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    export mrt0 off, off, off, off done
+; GFX12-NEXT:    s_endpgm
   %c = fcmp ole float %a, 1.500000e+00
   call void @llvm.amdgcn.kill(i1 %c) #1
   ret void
 }
 
-; GCN-LABEL: {{^}}test_scc_liveness:
-; GCN: s_cmp
-; GCN: s_and_b64 exec
-; GCN: s_cmp
-; GCN: s_cbranch_scc
 define amdgpu_ps void @test_scc_liveness() #0 {
+; SI-LABEL: test_scc_liveness:
+; SI:       ; %bb.0: ; %main_body
+; SI-NEXT:    s_mov_b64 s[0:1], exec
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:  .LBB25_1: ; %loop3
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_cmp_gt_i32 s2, 0
+; SI-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT:    s_andn2_b64 s[4:5], exec, s[4:5]
+; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; SI-NEXT:    s_cbranch_scc0 .LBB25_4
+; SI-NEXT:  ; %bb.2: ; %loop3
+; SI-NEXT:    ; in Loop: Header=BB25_1 Depth=1
+; SI-NEXT:    s_and_b64 exec, exec, s[0:1]
+; SI-NEXT:    s_add_i32 s3, s2, 1
+; SI-NEXT:    s_cmp_lt_i32 s2, 1
+; SI-NEXT:    s_mov_b32 s2, s3
+; SI-NEXT:    s_cbranch_scc1 .LBB25_1
+; SI-NEXT:  ; %bb.3: ; %endloop15
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB25_4:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    exp null off, off, off, off done vm
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: test_scc_liveness:
+; GFX10:       ; %bb.0: ; %main_body
+; GFX10-NEXT:    s_mov_b64 s[0:1], exec
+; GFX10-NEXT:    s_mov_b32 s2, 0
+; GFX10-NEXT:  .LBB25_1: ; %loop3
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX10-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX10-NEXT:    s_andn2_b64 s[4:5], exec, s[4:5]
+; GFX10-NEXT:    s_andn2_b64 s[0:1], s[0:1], s[4:5]
+; GFX10-NEXT:    s_cbranch_scc0 .LBB25_4
+; GFX10-NEXT:  ; %bb.2: ; %loop3
+; GFX10-NEXT:    ; in Loop: Header=BB25_1 Depth=1
+; GFX10-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX10-NEXT:    s_add_i32 s3, s2, 1
+; GFX10-NEXT:    s_cmp_lt_i32 s2, 1
+; GFX10-NEXT:    s_mov_b32 s2, s3
+; GFX10-NEXT:    s_cbranch_scc1 .LBB25_1
+; GFX10-NEXT:  ; %bb.3: ; %endloop15
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  .LBB25_4:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    exp null off, off, off, off done vm
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: test_scc_liveness:
+; GFX11:       ; %bb.0: ; %main_body
+; GFX11-NEXT:    s_mov_b64 s[0:1], exec
+; GFX11-NEXT:    s_mov_b32 s2, 0
+; GFX11-NEXT:  .LBB25_1: ; %loop3
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX11-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX11-NEXT:    s_and_not1_b64 s[4:5], exec, s[4:5]
+; GFX11-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX11-NEXT:    s_cbranch_scc0 .LBB25_4
+; GFX11-NEXT:  ; %bb.2: ; %loop3
+; GFX11-NEXT:    ; in Loop: Header=BB25_1 Depth=1
+; GFX11-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX11-NEXT:    s_add_i32 s3, s2, 1
+; GFX11-NEXT:    s_cmp_lt_i32 s2, 1
+; GFX11-NEXT:    s_mov_b32 s2, s3
+; GFX11-NEXT:    s_cbranch_scc1 .LBB25_1
+; GFX11-NEXT:  ; %bb.3: ; %endloop15
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  .LBB25_4:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    exp mrt0 off, off, off, off done
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: test_scc_liveness:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_mov_b64 s[0:1], exec
+; GFX12-NEXT:    s_mov_b32 s2, 0
+; GFX12-NEXT:  .LBB25_1: ; %loop3
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_cmp_gt_i32 s2, 0
+; GFX12-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX12-NEXT:    s_and_not1_b64 s[4:5], exec, s[4:5]
+; GFX12-NEXT:    s_and_not1_b64 s[0:1], s[0:1], s[4:5]
+; GFX12-NEXT:    s_cbranch_scc0 .LBB25_4
+; GFX12-NEXT:  ; %bb.2: ; %loop3
+; GFX12-NEXT:    ; in Loop: Header=BB25_1 Depth=1
+; GFX12-NEXT:    s_and_b64 exec, exec, s[0:1]
+; GFX12-NEXT:    s_add_co_i32 s3, s2, 1
+; GFX12-NEXT:    s_cmp_lt_i32 s2, 1
+; GFX12-NEXT:    s_mov_b32 s2, s3
+; GFX12-NEXT:    s_cbranch_scc1 .LBB25_1
+; GFX12-NEXT:  ; %bb.3: ; %endloop15
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  .LBB25_4:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    export mrt0 off, off, off, off done
+; GFX12-NEXT:    s_endpgm
 main_body:
   br label %loop3
 
@@ -287,11 +1515,139 @@ endloop15:                                        ; preds = %loop3
 
 ; Check this compiles.
 ; If kill is marked as defining VCC then this will fail with live interval issues.
-; GCN-LABEL: {{^}}kill_with_loop_exit:
-; GCN: s_mov_b64 [[LIVE:s\[[0-9]+:[0-9]+\]]], exec
-; GCN: s_and{{n2|_not1}}_b64 [[LIVE]], [[LIVE]], exec
-; GCN-NEXT: s_cbranch_scc0
 define amdgpu_ps void @kill_with_loop_exit(float inreg %inp0, float inreg %inp1, <4 x i32> inreg %inp2, float inreg %inp3) {
+; SI-LABEL: kill_with_loop_exit:
+; SI:       ; %bb.0: ; %.entry
+; SI-NEXT:    v_mov_b32_e32 v0, 0x43000000
+; SI-NEXT:    v_cmp_lt_f32_e32 vcc, s0, v0
+; SI-NEXT:    v_cmp_lt_f32_e64 s[0:1], s1, v0
+; SI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; SI-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-NEXT:    v_mov_b32_e32 v0, 1.0
+; SI-NEXT:    s_cbranch_vccnz .LBB26_5
+; SI-NEXT:  ; %bb.1: ; %.preheader1.preheader
+; SI-NEXT:    v_cmp_ngt_f32_e64 s[0:1], s6, 0
+; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; SI-NEXT:    s_mov_b64 s[2:3], exec
+; SI-NEXT:    v_mov_b32_e32 v0, 0x3fc00000
+; SI-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v1
+; SI-NEXT:  .LBB26_2: ; %bb
+; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
+; SI-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; SI-NEXT:    v_add_f32_e32 v0, 0x3e800000, v0
+; SI-NEXT:    s_cbranch_vccnz .LBB26_2
+; SI-NEXT:  ; %bb.3: ; %bb33
+; SI-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
+; SI-NEXT:    s_cbranch_scc0 .LBB26_6
+; SI-NEXT:  ; %bb.4: ; %bb33
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:  .LBB26_5: ; %bb35
+; SI-NEXT:    exp mrt0 v0, v0, v0, v0 done vm
+; SI-NEXT:    s_endpgm
+; SI-NEXT:  .LBB26_6:
+; SI-NEXT:    s_mov_b64 exec, 0
+; SI-NEXT:    exp null off, off, off, off done vm
+; SI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: kill_with_loop_exit:
+; GFX10:       ; %bb.0: ; %.entry
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s[4:5], 0x43000000, s0
+; GFX10-NEXT:    v_cmp_gt_f32_e64 s[0:1], 0x43000000, s1
+; GFX10-NEXT:    v_mov_b32_e32 v0, 1.0
+; GFX10-NEXT:    s_and_b64 s[0:1], s[4:5], s[0:1]
+; GFX10-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GFX10-NEXT:    s_cbranch_vccnz .LBB26_5
+; GFX10-NEXT:  ; %bb.1: ; %.preheader1.preheader
+; GFX10-NEXT:    v_cmp_ngt_f32_e64 s[0:1], s6, 0
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0x3fc00000
+; GFX10-NEXT:    s_mov_b64 s[2:3], exec
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v1
+; GFX10-NEXT:  .LBB26_2: ; %bb
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_add_f32_e32 v0, 0x3e800000, v0
+; GFX10-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GFX10-NEXT:    s_cbranch_vccnz .LBB26_2
+; GFX10-NEXT:  ; %bb.3: ; %bb33
+; GFX10-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
+; GFX10-NEXT:    s_cbranch_scc0 .LBB26_6
+; GFX10-NEXT:  ; %bb.4: ; %bb33
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:  .LBB26_5: ; %bb35
+; GFX10-NEXT:    exp mrt0 v0, v0, v0, v0 done vm
+; GFX10-NEXT:    s_endpgm
+; GFX10-NEXT:  .LBB26_6:
+; GFX10-NEXT:    s_mov_b64 exec, 0
+; GFX10-NEXT:    exp null off, off, off, off done vm
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: kill_with_loop_exit:
+; GFX11:       ; %bb.0: ; %.entry
+; GFX11-NEXT:    v_cmp_gt_f32_e64 s[4:5], 0x43000000, s0
+; GFX11-NEXT:    v_cmp_gt_f32_e64 s[0:1], 0x43000000, s1
+; GFX11-NEXT:    v_mov_b32_e32 v0, 1.0
+; GFX11-NEXT:    s_and_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GFX11-NEXT:    s_cbranch_vccnz .LBB26_5
+; GFX11-NEXT:  ; %bb.1: ; %.preheader1.preheader
+; GFX11-NEXT:    v_cmp_ngt_f32_e64 s[0:1], s6, 0
+; GFX11-NEXT:    v_mov_b32_e32 v0, 0x3fc00000
+; GFX11-NEXT:    s_mov_b64 s[2:3], exec
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
+; GFX11-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v1
+; GFX11-NEXT:  .LBB26_2: ; %bb
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    v_add_f32_e32 v0, 0x3e800000, v0
+; GFX11-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GFX11-NEXT:    s_cbranch_vccnz .LBB26_2
+; GFX11-NEXT:  ; %bb.3: ; %bb33
+; GFX11-NEXT:    s_and_not1_b64 s[2:3], s[2:3], exec
+; GFX11-NEXT:    s_cbranch_scc0 .LBB26_6
+; GFX11-NEXT:  ; %bb.4: ; %bb33
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:  .LBB26_5: ; %bb35
+; GFX11-NEXT:    exp mrt0 v0, v0, v0, v0 done
+; GFX11-NEXT:    s_endpgm
+; GFX11-NEXT:  .LBB26_6:
+; GFX11-NEXT:    s_mov_b64 exec, 0
+; GFX11-NEXT:    exp mrt0 off, off, off, off done
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: kill_with_loop_exit:
+; GFX12:       ; %bb.0: ; %.entry
+; GFX12-NEXT:    s_cmp_lt_f32 s0, 0x43000000
+; GFX12-NEXT:    s_cselect_b64 s[4:5], -1, 0
+; GFX12-NEXT:    s_cmp_lt_f32 s1, 0x43000000
+; GFX12-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX12-NEXT:    s_and_b64 s[0:1], s[4:5], s[0:1]
+; GFX12-NEXT:    s_mov_b32 s4, 1.0
+; GFX12-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GFX12-NEXT:    s_cbranch_vccnz .LBB26_5
+; GFX12-NEXT:  ; %bb.1: ; %.preheader1.preheader
+; GFX12-NEXT:    s_cmp_ngt_f32 s6, 0
+; GFX12-NEXT:    s_mov_b64 s[2:3], exec
+; GFX12-NEXT:    s_mov_b32 s4, 0x3fc00000
+; GFX12-NEXT:    s_cselect_b64 s[0:1], -1, 0
+; GFX12-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX12-NEXT:    v_cmp_ne_u32_e64 s[0:1], 1, v0
+; GFX12-NEXT:  .LBB26_2: ; %bb
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_add_f32 s4, s4, 0x3e800000
+; GFX12-NEXT:    s_and_b64 vcc, exec, s[0:1]
+; GFX12-NEXT:    s_cbranch_vccnz .LBB26_2
+; GFX12-NEXT:  ; %bb.3: ; %bb33
+; GFX12-NEXT:    s_and_not1_b64 s[2:3], s[2:3], exec
+; GFX12-NEXT:    s_cbranch_scc0 .LBB26_6
+; GFX12-NEXT:  ; %bb.4: ; %bb33
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:  .LBB26_5: ; %bb35
+; GFX12-NEXT:    v_mov_b32_e32 v0, s4
+; GFX12-NEXT:    export mrt0 v0, v0, v0, v0 done
+; GFX12-NEXT:    s_endpgm
+; GFX12-NEXT:  .LBB26_6:
+; GFX12-NEXT:    s_mov_b64 exec, 0
+; GFX12-NEXT:    export mrt0 off, off, off, off done
+; GFX12-NEXT:    s_endpgm
 .entry:
   %tmp24 = fcmp olt float %inp0, 1.280000e+02
   %tmp25 = fcmp olt float %inp1, 1.280000e+02
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll
index 017d402f22b90..3377290ecb1e0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.load.monitor.gfx1250.ll
@@ -104,7 +104,7 @@ define amdgpu_ps void @flat_load_monitor_b32(ptr %addr, ptr addrspace(1) %use) {
 ; GFX1250-NEXT:    global_store_b32 v[2:3], v0, off
 ; GFX1250-NEXT:    s_endpgm
 entry:
-  %gep = getelementptr i64, ptr addrspace(0) %addr, i32 4
+  %gep = getelementptr inbounds i64, ptr addrspace(0) %addr, i32 4
   %val = call i32 @llvm.amdgcn.flat.load.monitor.b32.i32(ptr addrspace(0) %gep, i32 10)
   store i32 %val, ptr addrspace(1) %use
   ret void
@@ -118,7 +118,7 @@ define amdgpu_ps void @flat_load_monitor_b64(ptr %addr, ptr addrspace(1) %use) {
 ; GFX1250-NEXT:    global_store_b64 v[2:3], v[0:1], off
 ; GFX1250-NEXT:    s_endpgm
 entry:
-  %gep = getelementptr i64, ptr addrspace(0) %addr, i32 4
+  %gep = getelementptr inbounds i64, ptr addrspace(0) %addr, i32 4
   %val = call <2 x i32> @llvm.amdgcn.flat.load.monitor.b64.v2i32(ptr addrspace(0) %gep, i32 22)
   store <2 x i32> %val, ptr addrspace(1) %use
   ret void
@@ -132,7 +132,7 @@ define amdgpu_ps void @flat_load_monitor_b128(ptr %addr, ptr addrspace(1) %use)
 ; GFX1250-NEXT:    global_store_b128 v[2:3], v[4:7], off
 ; GFX1250-NEXT:    s_endpgm
 entry:
-  %gep = getelementptr i64, ptr addrspace(0) %addr, i32 4
+  %gep = getelementptr inbounds i64, ptr addrspace(0) %addr, i32 4
   %val = call <4 x i32> @llvm.amdgcn.flat.load.monitor.b128.v4i32(ptr addrspace(0) %gep, i32 27)
   store <4 x i32> %val, ptr addrspace(1) %use
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
index 8081a15b53bb7..284ced1727b7e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.bf16.ll
@@ -39,16 +39,16 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x
 ; GCN-NEXT:    v_accvgpr_write_b32 a13, s21
 ; GCN-NEXT:    v_accvgpr_write_b32 a14, s22
 ; GCN-NEXT:    v_accvgpr_write_b32 a15, s23
-; GCN-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; GCN-NEXT:    v_mov_b32_e32 v16, s16
+; GCN-NEXT:    v_mov_b32_e32 v17, s17
 ; GCN-NEXT:    v_mfma_f32_32x32x16_bf16 a[16:31], v[0:3], v[4:7], a[0:15]
+; GCN-NEXT:    v_mov_b32_e32 v18, s18
+; GCN-NEXT:    v_mov_b32_e32 v19, s19
 ; GCN-NEXT:    v_mov_b32_e32 v0, s20
 ; GCN-NEXT:    v_mov_b32_e32 v1, s21
 ; GCN-NEXT:    v_mov_b32_e32 v2, s22
 ; GCN-NEXT:    v_mov_b32_e32 v3, s23
-; GCN-NEXT:    v_mov_b32_e32 v17, s17
-; GCN-NEXT:    v_mov_b32_e32 v18, s18
-; GCN-NEXT:    v_mov_b32_e32 v19, s19
+; GCN-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; GCN-NEXT:    s_nop 4
 ; GCN-NEXT:    global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
@@ -112,16 +112,16 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_bf16__flags(<8 x bfloat> %arg0
 ; GCN-NEXT:    v_accvgpr_write_b32 a13, s21
 ; GCN-NEXT:    v_accvgpr_write_b32 a14, s22
 ; GCN-NEXT:    v_accvgpr_write_b32 a15, s23
-; GCN-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; GCN-NEXT:    v_mov_b32_e32 v16, s16
+; GCN-NEXT:    v_mov_b32_e32 v17, s17
 ; GCN-NEXT:    v_mfma_f32_32x32x16_bf16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
+; GCN-NEXT:    v_mov_b32_e32 v18, s18
+; GCN-NEXT:    v_mov_b32_e32 v19, s19
 ; GCN-NEXT:    v_mov_b32_e32 v0, s20
 ; GCN-NEXT:    v_mov_b32_e32 v1, s21
 ; GCN-NEXT:    v_mov_b32_e32 v2, s22
 ; GCN-NEXT:    v_mov_b32_e32 v3, s23
-; GCN-NEXT:    v_mov_b32_e32 v17, s17
-; GCN-NEXT:    v_mov_b32_e32 v18, s18
-; GCN-NEXT:    v_mov_b32_e32 v19, s19
+; GCN-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; GCN-NEXT:    s_nop 4
 ; GCN-NEXT:    global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
index d81ec1c61634b..078a043b94604 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx950.ll
@@ -406,16 +406,16 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
 ; SDAG-NEXT:    v_accvgpr_write_b32 a13, s21
 ; SDAG-NEXT:    v_accvgpr_write_b32 a14, s22
 ; SDAG-NEXT:    v_accvgpr_write_b32 a15, s23
-; SDAG-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; SDAG-NEXT:    v_mov_b32_e32 v16, s16
+; SDAG-NEXT:    v_mov_b32_e32 v17, s17
 ; SDAG-NEXT:    v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15]
+; SDAG-NEXT:    v_mov_b32_e32 v18, s18
+; SDAG-NEXT:    v_mov_b32_e32 v19, s19
 ; SDAG-NEXT:    v_mov_b32_e32 v0, s20
 ; SDAG-NEXT:    v_mov_b32_e32 v1, s21
 ; SDAG-NEXT:    v_mov_b32_e32 v2, s22
 ; SDAG-NEXT:    v_mov_b32_e32 v3, s23
-; SDAG-NEXT:    v_mov_b32_e32 v17, s17
-; SDAG-NEXT:    v_mov_b32_e32 v18, s18
-; SDAG-NEXT:    v_mov_b32_e32 v19, s19
+; SDAG-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; SDAG-NEXT:    s_nop 4
 ; SDAG-NEXT:    global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
@@ -449,9 +449,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_load_dwordx8 s[24:31], s[4:5], 0x24
 ; GISEL-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT:    v_mov_b64_e32 v[20:21], 0
-; GISEL-NEXT:    v_mov_b64_e32 v[26:27], 48
-; GISEL-NEXT:    v_mov_b64_e32 v[22:23], 16
+; GISEL-NEXT:    v_mov_b64_e32 v[12:13], 0
+; GISEL-NEXT:    v_mov_b64_e32 v[14:15], 16
+; GISEL-NEXT:    v_mov_b64_e32 v[16:17], 32
 ; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[24:25]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[26:27]
@@ -474,31 +474,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
 ; GISEL-NEXT:    v_accvgpr_write_b32 a14, s22
 ; GISEL-NEXT:    v_accvgpr_write_b32 a15, s23
 ; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[18:19], 48
 ; GISEL-NEXT:    v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[16:17], s[16:17]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[24:25], 32
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
-; GISEL-NEXT:    s_nop 4
-; GISEL-NEXT:    global_store_dwordx4 v[20:21], a[16:19], off sc0 sc1
+; GISEL-NEXT:    s_nop 7
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[22:23], a[20:23], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[24:25], a[24:27], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[26:27], a[28:31], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[20:21], v[8:11], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[16:17]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[18:19]
+; GISEL-NEXT:    global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[22:23]
+; GISEL-NEXT:    global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    s_endpgm
 ;
@@ -530,16 +533,16 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
 ; HEURRC-NEXT:    v_accvgpr_write_b32 a13, s21
 ; HEURRC-NEXT:    v_accvgpr_write_b32 a14, s22
 ; HEURRC-NEXT:    v_accvgpr_write_b32 a15, s23
-; HEURRC-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; HEURRC-NEXT:    v_mov_b32_e32 v16, s16
+; HEURRC-NEXT:    v_mov_b32_e32 v17, s17
 ; HEURRC-NEXT:    v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15]
+; HEURRC-NEXT:    v_mov_b32_e32 v18, s18
+; HEURRC-NEXT:    v_mov_b32_e32 v19, s19
 ; HEURRC-NEXT:    v_mov_b32_e32 v0, s20
 ; HEURRC-NEXT:    v_mov_b32_e32 v1, s21
 ; HEURRC-NEXT:    v_mov_b32_e32 v2, s22
 ; HEURRC-NEXT:    v_mov_b32_e32 v3, s23
-; HEURRC-NEXT:    v_mov_b32_e32 v17, s17
-; HEURRC-NEXT:    v_mov_b32_e32 v18, s18
-; HEURRC-NEXT:    v_mov_b32_e32 v19, s19
+; HEURRC-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; HEURRC-NEXT:    s_nop 4
 ; HEURRC-NEXT:    global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
@@ -589,12 +592,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[10:11], s[18:19]
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[12:13], s[20:21]
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[46:47], 0
 ; VGPRRC-NEXT:    v_mov_b32_e32 v48, s16
-; VGPRRC-NEXT:    v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]
 ; VGPRRC-NEXT:    v_mov_b32_e32 v49, s17
+; VGPRRC-NEXT:    v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15]
 ; VGPRRC-NEXT:    v_mov_b32_e32 v50, s18
 ; VGPRRC-NEXT:    v_mov_b32_e32 v51, s19
+; VGPRRC-NEXT:    v_mov_b64_e32 v[46:47], 0
 ; VGPRRC-NEXT:    s_nop 7
 ; VGPRRC-NEXT:    s_nop 0
 ; VGPRRC-NEXT:    global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1
@@ -605,12 +608,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x hal
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
-; VGPRRC-NEXT:    global_store_dwordx4 v[42:43], v[48:51], off sc0 sc1
-; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    v_mov_b32_e32 v0, s20
 ; VGPRRC-NEXT:    v_mov_b32_e32 v1, s21
 ; VGPRRC-NEXT:    v_mov_b32_e32 v2, s22
 ; VGPRRC-NEXT:    v_mov_b32_e32 v3, s23
+; VGPRRC-NEXT:    global_store_dwordx4 v[42:43], v[48:51], off sc0 sc1
+; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    s_nop 0
@@ -788,16 +791,16 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
 ; SDAG-NEXT:    v_accvgpr_write_b32 a13, s21
 ; SDAG-NEXT:    v_accvgpr_write_b32 a14, s22
 ; SDAG-NEXT:    v_accvgpr_write_b32 a15, s23
-; SDAG-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; SDAG-NEXT:    v_mov_b32_e32 v16, s16
+; SDAG-NEXT:    v_mov_b32_e32 v17, s17
 ; SDAG-NEXT:    v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT:    v_mov_b32_e32 v18, s18
+; SDAG-NEXT:    v_mov_b32_e32 v19, s19
 ; SDAG-NEXT:    v_mov_b32_e32 v0, s20
 ; SDAG-NEXT:    v_mov_b32_e32 v1, s21
 ; SDAG-NEXT:    v_mov_b32_e32 v2, s22
 ; SDAG-NEXT:    v_mov_b32_e32 v3, s23
-; SDAG-NEXT:    v_mov_b32_e32 v17, s17
-; SDAG-NEXT:    v_mov_b32_e32 v18, s18
-; SDAG-NEXT:    v_mov_b32_e32 v19, s19
+; SDAG-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; SDAG-NEXT:    s_nop 4
 ; SDAG-NEXT:    global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
@@ -831,9 +834,9 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_load_dwordx8 s[24:31], s[4:5], 0x24
 ; GISEL-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT:    v_mov_b64_e32 v[20:21], 0
-; GISEL-NEXT:    v_mov_b64_e32 v[26:27], 48
-; GISEL-NEXT:    v_mov_b64_e32 v[22:23], 16
+; GISEL-NEXT:    v_mov_b64_e32 v[12:13], 0
+; GISEL-NEXT:    v_mov_b64_e32 v[14:15], 16
+; GISEL-NEXT:    v_mov_b64_e32 v[16:17], 32
 ; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[24:25]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[26:27]
@@ -856,31 +859,34 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
 ; GISEL-NEXT:    v_accvgpr_write_b32 a14, s22
 ; GISEL-NEXT:    v_accvgpr_write_b32 a15, s23
 ; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[18:19], 48
 ; GISEL-NEXT:    v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[16:17], s[16:17]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[24:25], 32
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
-; GISEL-NEXT:    s_nop 4
-; GISEL-NEXT:    global_store_dwordx4 v[20:21], a[16:19], off sc0 sc1
+; GISEL-NEXT:    s_nop 7
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[22:23], a[20:23], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[24:25], a[24:27], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[26:27], a[28:31], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[20:21], v[8:11], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[16:17]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[18:19]
+; GISEL-NEXT:    global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[22:23]
+; GISEL-NEXT:    global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    s_endpgm
 ;
@@ -912,16 +918,16 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
 ; HEURRC-NEXT:    v_accvgpr_write_b32 a13, s21
 ; HEURRC-NEXT:    v_accvgpr_write_b32 a14, s22
 ; HEURRC-NEXT:    v_accvgpr_write_b32 a15, s23
-; HEURRC-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; HEURRC-NEXT:    v_mov_b32_e32 v16, s16
+; HEURRC-NEXT:    v_mov_b32_e32 v17, s17
 ; HEURRC-NEXT:    v_mfma_f32_32x32x16_f16 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT:    v_mov_b32_e32 v18, s18
+; HEURRC-NEXT:    v_mov_b32_e32 v19, s19
 ; HEURRC-NEXT:    v_mov_b32_e32 v0, s20
 ; HEURRC-NEXT:    v_mov_b32_e32 v1, s21
 ; HEURRC-NEXT:    v_mov_b32_e32 v2, s22
 ; HEURRC-NEXT:    v_mov_b32_e32 v3, s23
-; HEURRC-NEXT:    v_mov_b32_e32 v17, s17
-; HEURRC-NEXT:    v_mov_b32_e32 v18, s18
-; HEURRC-NEXT:    v_mov_b32_e32 v19, s19
+; HEURRC-NEXT:    v_mov_b64_e32 v[14:15], 0
 ; HEURRC-NEXT:    s_nop 4
 ; HEURRC-NEXT:    global_store_dwordx4 v[8:9], a[28:31], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
@@ -971,12 +977,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[10:11], s[18:19]
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[12:13], s[20:21]
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[46:47], 0
 ; VGPRRC-NEXT:    v_mov_b32_e32 v48, s16
-; VGPRRC-NEXT:    v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
 ; VGPRRC-NEXT:    v_mov_b32_e32 v49, s17
+; VGPRRC-NEXT:    v_mfma_f32_32x32x16_f16 v[16:31], v[32:35], v[36:39], v[0:15] cbsz:2 abid:3 blgp:1
 ; VGPRRC-NEXT:    v_mov_b32_e32 v50, s18
 ; VGPRRC-NEXT:    v_mov_b32_e32 v51, s19
+; VGPRRC-NEXT:    v_mov_b64_e32 v[46:47], 0
 ; VGPRRC-NEXT:    s_nop 7
 ; VGPRRC-NEXT:    s_nop 0
 ; VGPRRC-NEXT:    global_store_dwordx4 v[40:41], v[28:31], off sc0 sc1
@@ -987,12 +993,12 @@ define amdgpu_kernel void @test_mfma_f32_32x32x16_f16__flags(<8 x half> %arg0, <
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    global_store_dwordx4 v[46:47], v[16:19], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
-; VGPRRC-NEXT:    global_store_dwordx4 v[42:43], v[48:51], off sc0 sc1
-; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    v_mov_b32_e32 v0, s20
 ; VGPRRC-NEXT:    v_mov_b32_e32 v1, s21
 ; VGPRRC-NEXT:    v_mov_b32_e32 v2, s22
 ; VGPRRC-NEXT:    v_mov_b32_e32 v3, s23
+; VGPRRC-NEXT:    global_store_dwordx4 v[42:43], v[48:51], off sc0 sc1
+; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    global_store_dwordx4 v[40:41], v[0:3], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    s_nop 0
@@ -2978,47 +2984,47 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
 ; SDAG-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
 ; SDAG-NEXT:    v_mov_b64_e32 v[0:1], 48
 ; SDAG-NEXT:    v_mov_b64_e32 v[2:3], 32
-; SDAG-NEXT:    v_mov_b64_e32 v[4:5], 16
 ; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b32_e32 v8, s24
-; SDAG-NEXT:    v_mov_b32_e32 v9, s25
-; SDAG-NEXT:    v_mov_b32_e32 v10, s26
-; SDAG-NEXT:    v_mov_b32_e32 v11, s27
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, s8
-; SDAG-NEXT:    v_mov_b32_e32 v12, s28
-; SDAG-NEXT:    v_mov_b32_e32 v13, s29
-; SDAG-NEXT:    v_mov_b32_e32 v14, s30
-; SDAG-NEXT:    v_mov_b32_e32 v15, s31
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, s9
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, s10
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, s11
-; SDAG-NEXT:    v_accvgpr_write_b32 a4, s12
-; SDAG-NEXT:    v_accvgpr_write_b32 a5, s13
-; SDAG-NEXT:    v_accvgpr_write_b32 a6, s14
-; SDAG-NEXT:    v_accvgpr_write_b32 a7, s15
-; SDAG-NEXT:    v_accvgpr_write_b32 a8, s16
-; SDAG-NEXT:    v_accvgpr_write_b32 a9, s17
-; SDAG-NEXT:    v_accvgpr_write_b32 a10, s18
-; SDAG-NEXT:    v_accvgpr_write_b32 a11, s19
-; SDAG-NEXT:    v_accvgpr_write_b32 a12, s20
-; SDAG-NEXT:    v_accvgpr_write_b32 a13, s21
-; SDAG-NEXT:    v_accvgpr_write_b32 a14, s22
-; SDAG-NEXT:    v_accvgpr_write_b32 a15, s23
+; SDAG-NEXT:    v_mov_b32_e32 v4, s24
+; SDAG-NEXT:    v_mov_b32_e32 v5, s25
+; SDAG-NEXT:    v_mov_b32_e32 v6, s26
+; SDAG-NEXT:    v_mov_b32_e32 v7, s27
+; SDAG-NEXT:    v_accvgpr_write_b32 a31, s23
+; SDAG-NEXT:    v_mov_b32_e32 v8, s28
+; SDAG-NEXT:    v_mov_b32_e32 v9, s29
+; SDAG-NEXT:    v_mov_b32_e32 v10, s30
+; SDAG-NEXT:    v_mov_b32_e32 v11, s31
+; SDAG-NEXT:    v_accvgpr_write_b32 a30, s22
+; SDAG-NEXT:    v_accvgpr_write_b32 a29, s21
+; SDAG-NEXT:    v_accvgpr_write_b32 a28, s20
+; SDAG-NEXT:    v_accvgpr_write_b32 a27, s19
+; SDAG-NEXT:    v_accvgpr_write_b32 a26, s18
+; SDAG-NEXT:    v_accvgpr_write_b32 a25, s17
+; SDAG-NEXT:    v_accvgpr_write_b32 a24, s16
+; SDAG-NEXT:    v_accvgpr_write_b32 a23, s15
+; SDAG-NEXT:    v_accvgpr_write_b32 a22, s14
+; SDAG-NEXT:    v_accvgpr_write_b32 a21, s13
+; SDAG-NEXT:    v_accvgpr_write_b32 a20, s12
+; SDAG-NEXT:    v_accvgpr_write_b32 a19, s11
+; SDAG-NEXT:    v_accvgpr_write_b32 a18, s10
+; SDAG-NEXT:    v_accvgpr_write_b32 a17, s9
+; SDAG-NEXT:    v_accvgpr_write_b32 a16, s8
+; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]
+; SDAG-NEXT:    v_mov_b64_e32 v[4:5], 16
 ; SDAG-NEXT:    v_mov_b64_e32 v[6:7], 0
-; SDAG-NEXT:    s_nop 0
-; SDAG-NEXT:    v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15]
 ; SDAG-NEXT:    v_mov_b32_e32 v8, s16
 ; SDAG-NEXT:    v_mov_b32_e32 v9, s17
 ; SDAG-NEXT:    v_mov_b32_e32 v10, s18
 ; SDAG-NEXT:    v_mov_b32_e32 v11, s19
-; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; SDAG-NEXT:    s_nop 5
+; SDAG-NEXT:    global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
-; SDAG-NEXT:    global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; SDAG-NEXT:    global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
-; SDAG-NEXT:    global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; SDAG-NEXT:    global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
-; SDAG-NEXT:    global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; SDAG-NEXT:    global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
@@ -3047,9 +3053,9 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_load_dwordx8 s[24:31], s[4:5], 0x24
 ; GISEL-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT:    v_mov_b64_e32 v[20:21], 0
-; GISEL-NEXT:    v_mov_b64_e32 v[26:27], 48
-; GISEL-NEXT:    v_mov_b64_e32 v[22:23], 16
+; GISEL-NEXT:    v_mov_b64_e32 v[12:13], 0
+; GISEL-NEXT:    v_mov_b64_e32 v[14:15], 16
+; GISEL-NEXT:    v_mov_b64_e32 v[16:17], 32
 ; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[24:25]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[26:27]
@@ -3072,31 +3078,34 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
 ; GISEL-NEXT:    v_accvgpr_write_b32 a14, s22
 ; GISEL-NEXT:    v_accvgpr_write_b32 a15, s23
 ; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[18:19], 48
 ; GISEL-NEXT:    v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[16:17], s[16:17]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[24:25], 32
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
-; GISEL-NEXT:    s_nop 4
-; GISEL-NEXT:    global_store_dwordx4 v[20:21], a[16:19], off sc0 sc1
+; GISEL-NEXT:    s_nop 7
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[22:23], a[20:23], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[24:25], a[24:27], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[26:27], a[28:31], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[20:21], v[8:11], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[16:17]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[18:19]
+; GISEL-NEXT:    global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[22:23]
+; GISEL-NEXT:    global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    s_endpgm
 ;
@@ -3106,47 +3115,47 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
 ; HEURRC-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
 ; HEURRC-NEXT:    v_mov_b64_e32 v[0:1], 48
 ; HEURRC-NEXT:    v_mov_b64_e32 v[2:3], 32
-; HEURRC-NEXT:    v_mov_b64_e32 v[4:5], 16
 ; HEURRC-NEXT:    s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT:    v_mov_b32_e32 v8, s24
-; HEURRC-NEXT:    v_mov_b32_e32 v9, s25
-; HEURRC-NEXT:    v_mov_b32_e32 v10, s26
-; HEURRC-NEXT:    v_mov_b32_e32 v11, s27
-; HEURRC-NEXT:    v_accvgpr_write_b32 a0, s8
-; HEURRC-NEXT:    v_mov_b32_e32 v12, s28
-; HEURRC-NEXT:    v_mov_b32_e32 v13, s29
-; HEURRC-NEXT:    v_mov_b32_e32 v14, s30
-; HEURRC-NEXT:    v_mov_b32_e32 v15, s31
-; HEURRC-NEXT:    v_accvgpr_write_b32 a1, s9
-; HEURRC-NEXT:    v_accvgpr_write_b32 a2, s10
-; HEURRC-NEXT:    v_accvgpr_write_b32 a3, s11
-; HEURRC-NEXT:    v_accvgpr_write_b32 a4, s12
-; HEURRC-NEXT:    v_accvgpr_write_b32 a5, s13
-; HEURRC-NEXT:    v_accvgpr_write_b32 a6, s14
-; HEURRC-NEXT:    v_accvgpr_write_b32 a7, s15
-; HEURRC-NEXT:    v_accvgpr_write_b32 a8, s16
-; HEURRC-NEXT:    v_accvgpr_write_b32 a9, s17
-; HEURRC-NEXT:    v_accvgpr_write_b32 a10, s18
-; HEURRC-NEXT:    v_accvgpr_write_b32 a11, s19
-; HEURRC-NEXT:    v_accvgpr_write_b32 a12, s20
-; HEURRC-NEXT:    v_accvgpr_write_b32 a13, s21
-; HEURRC-NEXT:    v_accvgpr_write_b32 a14, s22
-; HEURRC-NEXT:    v_accvgpr_write_b32 a15, s23
+; HEURRC-NEXT:    v_mov_b32_e32 v4, s24
+; HEURRC-NEXT:    v_mov_b32_e32 v5, s25
+; HEURRC-NEXT:    v_mov_b32_e32 v6, s26
+; HEURRC-NEXT:    v_mov_b32_e32 v7, s27
+; HEURRC-NEXT:    v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT:    v_mov_b32_e32 v8, s28
+; HEURRC-NEXT:    v_mov_b32_e32 v9, s29
+; HEURRC-NEXT:    v_mov_b32_e32 v10, s30
+; HEURRC-NEXT:    v_mov_b32_e32 v11, s31
+; HEURRC-NEXT:    v_accvgpr_write_b32 a30, s22
+; HEURRC-NEXT:    v_accvgpr_write_b32 a29, s21
+; HEURRC-NEXT:    v_accvgpr_write_b32 a28, s20
+; HEURRC-NEXT:    v_accvgpr_write_b32 a27, s19
+; HEURRC-NEXT:    v_accvgpr_write_b32 a26, s18
+; HEURRC-NEXT:    v_accvgpr_write_b32 a25, s17
+; HEURRC-NEXT:    v_accvgpr_write_b32 a24, s16
+; HEURRC-NEXT:    v_accvgpr_write_b32 a23, s15
+; HEURRC-NEXT:    v_accvgpr_write_b32 a22, s14
+; HEURRC-NEXT:    v_accvgpr_write_b32 a21, s13
+; HEURRC-NEXT:    v_accvgpr_write_b32 a20, s12
+; HEURRC-NEXT:    v_accvgpr_write_b32 a19, s11
+; HEURRC-NEXT:    v_accvgpr_write_b32 a18, s10
+; HEURRC-NEXT:    v_accvgpr_write_b32 a17, s9
+; HEURRC-NEXT:    v_accvgpr_write_b32 a16, s8
+; HEURRC-NEXT:    s_nop 1
+; HEURRC-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31]
+; HEURRC-NEXT:    v_mov_b64_e32 v[4:5], 16
 ; HEURRC-NEXT:    v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT:    s_nop 0
-; HEURRC-NEXT:    v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15]
 ; HEURRC-NEXT:    v_mov_b32_e32 v8, s16
 ; HEURRC-NEXT:    v_mov_b32_e32 v9, s17
 ; HEURRC-NEXT:    v_mov_b32_e32 v10, s18
 ; HEURRC-NEXT:    v_mov_b32_e32 v11, s19
-; HEURRC-NEXT:    s_nop 7
-; HEURRC-NEXT:    global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; HEURRC-NEXT:    s_nop 5
+; HEURRC-NEXT:    global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
-; HEURRC-NEXT:    global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; HEURRC-NEXT:    global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
-; HEURRC-NEXT:    global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; HEURRC-NEXT:    global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
-; HEURRC-NEXT:    global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; HEURRC-NEXT:    global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
 ; HEURRC-NEXT:    global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
@@ -3177,37 +3186,40 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
 ; VGPRRC-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[32:33], 48
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[34:35], 32
-; VGPRRC-NEXT:    v_mov_b64_e32 v[36:37], 16
 ; VGPRRC-NEXT:    s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT:    v_mov_b32_e32 v40, s24
-; VGPRRC-NEXT:    v_mov_b32_e32 v41, s25
-; VGPRRC-NEXT:    v_mov_b32_e32 v42, s26
-; VGPRRC-NEXT:    v_mov_b32_e32 v43, s27
-; VGPRRC-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; VGPRRC-NEXT:    v_mov_b32_e32 v44, s28
-; VGPRRC-NEXT:    v_mov_b32_e32 v45, s29
-; VGPRRC-NEXT:    v_mov_b32_e32 v46, s30
-; VGPRRC-NEXT:    v_mov_b32_e32 v47, s31
-; VGPRRC-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[8:9], s[16:17]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[10:11], s[18:19]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[12:13], s[20:21]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT:    s_nop 0
-; VGPRRC-NEXT:    v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15]
+; VGPRRC-NEXT:    v_mov_b32_e32 v36, s24
+; VGPRRC-NEXT:    v_mov_b32_e32 v37, s25
+; VGPRRC-NEXT:    v_mov_b32_e32 v38, s26
+; VGPRRC-NEXT:    v_mov_b32_e32 v39, s27
+; VGPRRC-NEXT:    v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT:    v_mov_b32_e32 v40, s28
+; VGPRRC-NEXT:    v_mov_b32_e32 v41, s29
+; VGPRRC-NEXT:    v_mov_b32_e32 v42, s30
+; VGPRRC-NEXT:    v_mov_b32_e32 v43, s31
+; VGPRRC-NEXT:    v_mov_b64_e32 v[28:29], s[20:21]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[26:27], s[18:19]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[24:25], s[16:17]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[22:23], s[14:15]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[20:21], s[12:13]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[18:19], s[10:11]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT:    s_nop 1
+; VGPRRC-NEXT:    v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31]
 ; VGPRRC-NEXT:    s_nop 7
 ; VGPRRC-NEXT:    s_nop 3
-; VGPRRC-NEXT:    global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPRRC-NEXT:    global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
-; VGPRRC-NEXT:    global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPRRC-NEXT:    global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
-; VGPRRC-NEXT:    global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPRRC-NEXT:    s_nop 0
+; VGPRRC-NEXT:    v_mov_b64_e32 v[8:9], 16
+; VGPRRC-NEXT:    global_store_dwordx4 v[8:9], v[4:7], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
-; VGPRRC-NEXT:    global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPRRC-NEXT:    s_nop 0
+; VGPRRC-NEXT:    v_mov_b64_e32 v[4:5], 0
+; VGPRRC-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
+; VGPRRC-NEXT:    s_nop 0
 ; VGPRRC-NEXT:    v_mov_b32_e32 v0, s16
 ; VGPRRC-NEXT:    v_mov_b32_e32 v1, s17
 ; VGPRRC-NEXT:    v_mov_b32_e32 v2, s18
@@ -3226,14 +3238,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32>
 ; VGPRRC-NEXT:    v_mov_b32_e32 v1, s9
 ; VGPRRC-NEXT:    v_mov_b32_e32 v2, s10
 ; VGPRRC-NEXT:    v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT:    global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    s_nop 0
 ; VGPRRC-NEXT:    v_mov_b32_e32 v0, s12
 ; VGPRRC-NEXT:    v_mov_b32_e32 v1, s13
 ; VGPRRC-NEXT:    v_mov_b32_e32 v2, s14
 ; VGPRRC-NEXT:    v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT:    global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPRRC-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    s_endpgm
 ; AGPR-LABEL: test_mfma_i32_32x32x32_i8:
@@ -3386,47 +3398,47 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
 ; SDAG-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
 ; SDAG-NEXT:    v_mov_b64_e32 v[0:1], 48
 ; SDAG-NEXT:    v_mov_b64_e32 v[2:3], 32
-; SDAG-NEXT:    v_mov_b64_e32 v[4:5], 16
 ; SDAG-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-NEXT:    v_mov_b32_e32 v8, s24
-; SDAG-NEXT:    v_mov_b32_e32 v9, s25
-; SDAG-NEXT:    v_mov_b32_e32 v10, s26
-; SDAG-NEXT:    v_mov_b32_e32 v11, s27
-; SDAG-NEXT:    v_accvgpr_write_b32 a0, s8
-; SDAG-NEXT:    v_mov_b32_e32 v12, s28
-; SDAG-NEXT:    v_mov_b32_e32 v13, s29
-; SDAG-NEXT:    v_mov_b32_e32 v14, s30
-; SDAG-NEXT:    v_mov_b32_e32 v15, s31
-; SDAG-NEXT:    v_accvgpr_write_b32 a1, s9
-; SDAG-NEXT:    v_accvgpr_write_b32 a2, s10
-; SDAG-NEXT:    v_accvgpr_write_b32 a3, s11
-; SDAG-NEXT:    v_accvgpr_write_b32 a4, s12
-; SDAG-NEXT:    v_accvgpr_write_b32 a5, s13
-; SDAG-NEXT:    v_accvgpr_write_b32 a6, s14
-; SDAG-NEXT:    v_accvgpr_write_b32 a7, s15
-; SDAG-NEXT:    v_accvgpr_write_b32 a8, s16
-; SDAG-NEXT:    v_accvgpr_write_b32 a9, s17
-; SDAG-NEXT:    v_accvgpr_write_b32 a10, s18
-; SDAG-NEXT:    v_accvgpr_write_b32 a11, s19
-; SDAG-NEXT:    v_accvgpr_write_b32 a12, s20
-; SDAG-NEXT:    v_accvgpr_write_b32 a13, s21
-; SDAG-NEXT:    v_accvgpr_write_b32 a14, s22
-; SDAG-NEXT:    v_accvgpr_write_b32 a15, s23
+; SDAG-NEXT:    v_mov_b32_e32 v4, s24
+; SDAG-NEXT:    v_mov_b32_e32 v5, s25
+; SDAG-NEXT:    v_mov_b32_e32 v6, s26
+; SDAG-NEXT:    v_mov_b32_e32 v7, s27
+; SDAG-NEXT:    v_accvgpr_write_b32 a31, s23
+; SDAG-NEXT:    v_mov_b32_e32 v8, s28
+; SDAG-NEXT:    v_mov_b32_e32 v9, s29
+; SDAG-NEXT:    v_mov_b32_e32 v10, s30
+; SDAG-NEXT:    v_mov_b32_e32 v11, s31
+; SDAG-NEXT:    v_accvgpr_write_b32 a30, s22
+; SDAG-NEXT:    v_accvgpr_write_b32 a29, s21
+; SDAG-NEXT:    v_accvgpr_write_b32 a28, s20
+; SDAG-NEXT:    v_accvgpr_write_b32 a27, s19
+; SDAG-NEXT:    v_accvgpr_write_b32 a26, s18
+; SDAG-NEXT:    v_accvgpr_write_b32 a25, s17
+; SDAG-NEXT:    v_accvgpr_write_b32 a24, s16
+; SDAG-NEXT:    v_accvgpr_write_b32 a23, s15
+; SDAG-NEXT:    v_accvgpr_write_b32 a22, s14
+; SDAG-NEXT:    v_accvgpr_write_b32 a21, s13
+; SDAG-NEXT:    v_accvgpr_write_b32 a20, s12
+; SDAG-NEXT:    v_accvgpr_write_b32 a19, s11
+; SDAG-NEXT:    v_accvgpr_write_b32 a18, s10
+; SDAG-NEXT:    v_accvgpr_write_b32 a17, s9
+; SDAG-NEXT:    v_accvgpr_write_b32 a16, s8
+; SDAG-NEXT:    s_nop 1
+; SDAG-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:1
+; SDAG-NEXT:    v_mov_b64_e32 v[4:5], 16
 ; SDAG-NEXT:    v_mov_b64_e32 v[6:7], 0
-; SDAG-NEXT:    s_nop 0
-; SDAG-NEXT:    v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
 ; SDAG-NEXT:    v_mov_b32_e32 v8, s16
 ; SDAG-NEXT:    v_mov_b32_e32 v9, s17
 ; SDAG-NEXT:    v_mov_b32_e32 v10, s18
 ; SDAG-NEXT:    v_mov_b32_e32 v11, s19
-; SDAG-NEXT:    s_nop 7
-; SDAG-NEXT:    global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; SDAG-NEXT:    s_nop 5
+; SDAG-NEXT:    global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
-; SDAG-NEXT:    global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; SDAG-NEXT:    global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
-; SDAG-NEXT:    global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; SDAG-NEXT:    global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
-; SDAG-NEXT:    global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; SDAG-NEXT:    global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-NEXT:    global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
 ; SDAG-NEXT:    s_waitcnt vmcnt(0)
@@ -3455,9 +3467,9 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_load_dwordx8 s[24:31], s[4:5], 0x24
 ; GISEL-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GISEL-NEXT:    v_mov_b64_e32 v[20:21], 0
-; GISEL-NEXT:    v_mov_b64_e32 v[26:27], 48
-; GISEL-NEXT:    v_mov_b64_e32 v[22:23], 16
+; GISEL-NEXT:    v_mov_b64_e32 v[12:13], 0
+; GISEL-NEXT:    v_mov_b64_e32 v[14:15], 16
+; GISEL-NEXT:    v_mov_b64_e32 v[16:17], 32
 ; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[24:25]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[26:27]
@@ -3480,31 +3492,34 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
 ; GISEL-NEXT:    v_accvgpr_write_b32 a14, s22
 ; GISEL-NEXT:    v_accvgpr_write_b32 a15, s23
 ; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[8:9]
-; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[18:19], 48
 ; GISEL-NEXT:    v_mfma_i32_32x32x32_i8 a[16:31], v[0:3], v[4:7], a[0:15] cbsz:2 abid:3 blgp:1
-; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[20:21]
-; GISEL-NEXT:    v_mov_b64_e32 v[16:17], s[16:17]
-; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[22:23]
-; GISEL-NEXT:    v_mov_b64_e32 v[24:25], 32
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[12:13]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[10:11]
-; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[14:15]
-; GISEL-NEXT:    v_mov_b64_e32 v[18:19], s[18:19]
-; GISEL-NEXT:    s_nop 4
-; GISEL-NEXT:    global_store_dwordx4 v[20:21], a[16:19], off sc0 sc1
+; GISEL-NEXT:    s_nop 7
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    global_store_dwordx4 v[12:13], a[16:19], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[22:23], a[20:23], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[14:15], a[20:23], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[24:25], a[24:27], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[16:17], a[24:27], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[26:27], a[28:31], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[18:19], a[28:31], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[20:21], v[8:11], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[12:13], v[8:11], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[22:23], v[12:15], off sc0 sc1
+; GISEL-NEXT:    global_store_dwordx4 v[14:15], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[24:25], v[16:19], off sc0 sc1
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[16:17]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[18:19]
+; GISEL-NEXT:    global_store_dwordx4 v[16:17], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    global_store_dwordx4 v[26:27], v[0:3], off sc0 sc1
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[22:23]
+; GISEL-NEXT:    global_store_dwordx4 v[18:19], v[0:3], off sc0 sc1
 ; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    s_endpgm
 ;
@@ -3514,47 +3529,47 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
 ; HEURRC-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
 ; HEURRC-NEXT:    v_mov_b64_e32 v[0:1], 48
 ; HEURRC-NEXT:    v_mov_b64_e32 v[2:3], 32
-; HEURRC-NEXT:    v_mov_b64_e32 v[4:5], 16
 ; HEURRC-NEXT:    s_waitcnt lgkmcnt(0)
-; HEURRC-NEXT:    v_mov_b32_e32 v8, s24
-; HEURRC-NEXT:    v_mov_b32_e32 v9, s25
-; HEURRC-NEXT:    v_mov_b32_e32 v10, s26
-; HEURRC-NEXT:    v_mov_b32_e32 v11, s27
-; HEURRC-NEXT:    v_accvgpr_write_b32 a0, s8
-; HEURRC-NEXT:    v_mov_b32_e32 v12, s28
-; HEURRC-NEXT:    v_mov_b32_e32 v13, s29
-; HEURRC-NEXT:    v_mov_b32_e32 v14, s30
-; HEURRC-NEXT:    v_mov_b32_e32 v15, s31
-; HEURRC-NEXT:    v_accvgpr_write_b32 a1, s9
-; HEURRC-NEXT:    v_accvgpr_write_b32 a2, s10
-; HEURRC-NEXT:    v_accvgpr_write_b32 a3, s11
-; HEURRC-NEXT:    v_accvgpr_write_b32 a4, s12
-; HEURRC-NEXT:    v_accvgpr_write_b32 a5, s13
-; HEURRC-NEXT:    v_accvgpr_write_b32 a6, s14
-; HEURRC-NEXT:    v_accvgpr_write_b32 a7, s15
-; HEURRC-NEXT:    v_accvgpr_write_b32 a8, s16
-; HEURRC-NEXT:    v_accvgpr_write_b32 a9, s17
-; HEURRC-NEXT:    v_accvgpr_write_b32 a10, s18
-; HEURRC-NEXT:    v_accvgpr_write_b32 a11, s19
-; HEURRC-NEXT:    v_accvgpr_write_b32 a12, s20
-; HEURRC-NEXT:    v_accvgpr_write_b32 a13, s21
-; HEURRC-NEXT:    v_accvgpr_write_b32 a14, s22
-; HEURRC-NEXT:    v_accvgpr_write_b32 a15, s23
+; HEURRC-NEXT:    v_mov_b32_e32 v4, s24
+; HEURRC-NEXT:    v_mov_b32_e32 v5, s25
+; HEURRC-NEXT:    v_mov_b32_e32 v6, s26
+; HEURRC-NEXT:    v_mov_b32_e32 v7, s27
+; HEURRC-NEXT:    v_accvgpr_write_b32 a31, s23
+; HEURRC-NEXT:    v_mov_b32_e32 v8, s28
+; HEURRC-NEXT:    v_mov_b32_e32 v9, s29
+; HEURRC-NEXT:    v_mov_b32_e32 v10, s30
+; HEURRC-NEXT:    v_mov_b32_e32 v11, s31
+; HEURRC-NEXT:    v_accvgpr_write_b32 a30, s22
+; HEURRC-NEXT:    v_accvgpr_write_b32 a29, s21
+; HEURRC-NEXT:    v_accvgpr_write_b32 a28, s20
+; HEURRC-NEXT:    v_accvgpr_write_b32 a27, s19
+; HEURRC-NEXT:    v_accvgpr_write_b32 a26, s18
+; HEURRC-NEXT:    v_accvgpr_write_b32 a25, s17
+; HEURRC-NEXT:    v_accvgpr_write_b32 a24, s16
+; HEURRC-NEXT:    v_accvgpr_write_b32 a23, s15
+; HEURRC-NEXT:    v_accvgpr_write_b32 a22, s14
+; HEURRC-NEXT:    v_accvgpr_write_b32 a21, s13
+; HEURRC-NEXT:    v_accvgpr_write_b32 a20, s12
+; HEURRC-NEXT:    v_accvgpr_write_b32 a19, s11
+; HEURRC-NEXT:    v_accvgpr_write_b32 a18, s10
+; HEURRC-NEXT:    v_accvgpr_write_b32 a17, s9
+; HEURRC-NEXT:    v_accvgpr_write_b32 a16, s8
+; HEURRC-NEXT:    s_nop 1
+; HEURRC-NEXT:    v_mfma_i32_32x32x32_i8 a[0:15], v[4:7], v[8:11], a[16:31] cbsz:2 abid:3 blgp:1
+; HEURRC-NEXT:    v_mov_b64_e32 v[4:5], 16
 ; HEURRC-NEXT:    v_mov_b64_e32 v[6:7], 0
-; HEURRC-NEXT:    s_nop 0
-; HEURRC-NEXT:    v_mfma_i32_32x32x32_i8 a[16:31], v[8:11], v[12:15], a[0:15] cbsz:2 abid:3 blgp:1
 ; HEURRC-NEXT:    v_mov_b32_e32 v8, s16
 ; HEURRC-NEXT:    v_mov_b32_e32 v9, s17
 ; HEURRC-NEXT:    v_mov_b32_e32 v10, s18
 ; HEURRC-NEXT:    v_mov_b32_e32 v11, s19
-; HEURRC-NEXT:    s_nop 7
-; HEURRC-NEXT:    global_store_dwordx4 v[0:1], a[28:31], off sc0 sc1
+; HEURRC-NEXT:    s_nop 5
+; HEURRC-NEXT:    global_store_dwordx4 v[0:1], a[12:15], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
-; HEURRC-NEXT:    global_store_dwordx4 v[2:3], a[24:27], off sc0 sc1
+; HEURRC-NEXT:    global_store_dwordx4 v[2:3], a[8:11], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
-; HEURRC-NEXT:    global_store_dwordx4 v[4:5], a[20:23], off sc0 sc1
+; HEURRC-NEXT:    global_store_dwordx4 v[4:5], a[4:7], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
-; HEURRC-NEXT:    global_store_dwordx4 v[6:7], a[16:19], off sc0 sc1
+; HEURRC-NEXT:    global_store_dwordx4 v[6:7], a[0:3], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
 ; HEURRC-NEXT:    global_store_dwordx4 v[2:3], v[8:11], off sc0 sc1
 ; HEURRC-NEXT:    s_waitcnt vmcnt(0)
@@ -3585,37 +3600,40 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
 ; VGPRRC-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x64
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[32:33], 48
 ; VGPRRC-NEXT:    v_mov_b64_e32 v[34:35], 32
-; VGPRRC-NEXT:    v_mov_b64_e32 v[36:37], 16
 ; VGPRRC-NEXT:    s_waitcnt lgkmcnt(0)
-; VGPRRC-NEXT:    v_mov_b32_e32 v40, s24
-; VGPRRC-NEXT:    v_mov_b32_e32 v41, s25
-; VGPRRC-NEXT:    v_mov_b32_e32 v42, s26
-; VGPRRC-NEXT:    v_mov_b32_e32 v43, s27
-; VGPRRC-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; VGPRRC-NEXT:    v_mov_b32_e32 v44, s28
-; VGPRRC-NEXT:    v_mov_b32_e32 v45, s29
-; VGPRRC-NEXT:    v_mov_b32_e32 v46, s30
-; VGPRRC-NEXT:    v_mov_b32_e32 v47, s31
-; VGPRRC-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[8:9], s[16:17]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[10:11], s[18:19]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[12:13], s[20:21]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[14:15], s[22:23]
-; VGPRRC-NEXT:    v_mov_b64_e32 v[38:39], 0
-; VGPRRC-NEXT:    s_nop 0
-; VGPRRC-NEXT:    v_mfma_i32_32x32x32_i8 v[16:31], v[40:43], v[44:47], v[0:15] cbsz:2 abid:3 blgp:1
+; VGPRRC-NEXT:    v_mov_b32_e32 v36, s24
+; VGPRRC-NEXT:    v_mov_b32_e32 v37, s25
+; VGPRRC-NEXT:    v_mov_b32_e32 v38, s26
+; VGPRRC-NEXT:    v_mov_b32_e32 v39, s27
+; VGPRRC-NEXT:    v_mov_b64_e32 v[30:31], s[22:23]
+; VGPRRC-NEXT:    v_mov_b32_e32 v40, s28
+; VGPRRC-NEXT:    v_mov_b32_e32 v41, s29
+; VGPRRC-NEXT:    v_mov_b32_e32 v42, s30
+; VGPRRC-NEXT:    v_mov_b32_e32 v43, s31
+; VGPRRC-NEXT:    v_mov_b64_e32 v[28:29], s[20:21]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[26:27], s[18:19]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[24:25], s[16:17]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[22:23], s[14:15]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[20:21], s[12:13]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[18:19], s[10:11]
+; VGPRRC-NEXT:    v_mov_b64_e32 v[16:17], s[8:9]
+; VGPRRC-NEXT:    s_nop 1
+; VGPRRC-NEXT:    v_mfma_i32_32x32x32_i8 v[0:15], v[36:39], v[40:43], v[16:31] cbsz:2 abid:3 blgp:1
 ; VGPRRC-NEXT:    s_nop 7
 ; VGPRRC-NEXT:    s_nop 3
-; VGPRRC-NEXT:    global_store_dwordx4 v[32:33], v[28:31], off sc0 sc1
+; VGPRRC-NEXT:    global_store_dwordx4 v[32:33], v[12:15], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
-; VGPRRC-NEXT:    global_store_dwordx4 v[34:35], v[24:27], off sc0 sc1
+; VGPRRC-NEXT:    global_store_dwordx4 v[34:35], v[8:11], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
-; VGPRRC-NEXT:    global_store_dwordx4 v[36:37], v[20:23], off sc0 sc1
+; VGPRRC-NEXT:    s_nop 0
+; VGPRRC-NEXT:    v_mov_b64_e32 v[8:9], 16
+; VGPRRC-NEXT:    global_store_dwordx4 v[8:9], v[4:7], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
-; VGPRRC-NEXT:    global_store_dwordx4 v[38:39], v[16:19], off sc0 sc1
+; VGPRRC-NEXT:    s_nop 0
+; VGPRRC-NEXT:    v_mov_b64_e32 v[4:5], 0
+; VGPRRC-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
+; VGPRRC-NEXT:    s_nop 0
 ; VGPRRC-NEXT:    v_mov_b32_e32 v0, s16
 ; VGPRRC-NEXT:    v_mov_b32_e32 v1, s17
 ; VGPRRC-NEXT:    v_mov_b32_e32 v2, s18
@@ -3634,14 +3652,14 @@ define amdgpu_kernel void @test_mfma_i32_32x32x32_i8__flags(<4 x i32> %arg0, <4
 ; VGPRRC-NEXT:    v_mov_b32_e32 v1, s9
 ; VGPRRC-NEXT:    v_mov_b32_e32 v2, s10
 ; VGPRRC-NEXT:    v_mov_b32_e32 v3, s11
-; VGPRRC-NEXT:    global_store_dwordx4 v[38:39], v[0:3], off sc0 sc1
+; VGPRRC-NEXT:    global_store_dwordx4 v[4:5], v[0:3], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    s_nop 0
 ; VGPRRC-NEXT:    v_mov_b32_e32 v0, s12
 ; VGPRRC-NEXT:    v_mov_b32_e32 v1, s13
 ; VGPRRC-NEXT:    v_mov_b32_e32 v2, s14
 ; VGPRRC-NEXT:    v_mov_b32_e32 v3, s15
-; VGPRRC-NEXT:    global_store_dwordx4 v[36:37], v[0:3], off sc0 sc1
+; VGPRRC-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off sc0 sc1
 ; VGPRRC-NEXT:    s_waitcnt vmcnt(0)
 ; VGPRRC-NEXT:    s_endpgm
 ; AGPR-LABEL: test_mfma_i32_32x32x32_i8__flags:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
index 0b2818f38149d..24af3fa5ff9b7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.32x32x64.f8f6f4.ll
@@ -4784,8 +4784,8 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
 ; GISEL-NEXT:    s_load_dwordx16 s[36:51], s[4:5], 0x0
 ; GISEL-NEXT:    s_load_dwordx16 s[8:23], s[4:5], 0x40
 ; GISEL-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x80
+; GISEL-NEXT:    v_mov_b64_e32 v[16:17], 0
 ; GISEL-NEXT:    v_mov_b64_e32 v[18:19], 16
-; GISEL-NEXT:    v_mov_b64_e32 v[20:21], 32
 ; GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[36:37]
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[38:39]
@@ -4811,16 +4811,16 @@ define amdgpu_kernel void @test_mfma_scale_f32_32x32x64_f8f6f4_0_0__nonmac(<8 x
 ; GISEL-NEXT:    v_accvgpr_write_b32 a13, s21
 ; GISEL-NEXT:    v_accvgpr_write_b32 a14, s22
 ; GISEL-NEXT:    v_accvgpr_write_b32 a15, s23
-; GISEL-NEXT:    v_mov_b32_e32 v16, s1
+; GISEL-NEXT:    v_mov_b32_e32 v20, s1
 ; GISEL-NEXT:    v_mov_b64_e32 v[22:23], 48
 ; GISEL-NEXT:    s_nop 0
-; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], s0, v16 op_sel_hi:[0,0,0]
+; GISEL-NEXT:    v_mfma_scale_f32_32x32x64_f8f6f4 a[0:15], v[0:7], v[8:15], a[0:15], s0, v20 op_sel_hi:[0,0,0]
 ; GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[8:9]
-; GISEL-NEXT:    v_mov_b64_e32 v[16:17], 0
 ; GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[10:11]
 ; GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[12:13]
 ; GISEL-NEXT:    v_mov_b64_e32 v[8:9], s[16:17]
 ; GISEL-NEXT:    v_mov_b64_e32 v[12:13], s[20:21]
+; GISEL-NEXT:    v_mov_b64_e32 v[20:21], 32
 ; GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[14:15]
 ; GISEL-NEXT:    v_mov_b64_e32 v[10:11], s[18:19]
 ; GISEL-NEXT:    v_mov_b64_e32 v[14:15], s[22:23]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
index 6a5dc8f8dd0a6..2daf9c3b472f1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.prng.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck -check-prefixes=GCN %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,SDAG %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefix=GCN %s
 
 declare i32 @llvm.amdgcn.prng.b32(i32) #0
@@ -29,6 +29,13 @@ define amdgpu_kernel void @prng_b32_constant_100(ptr addrspace(1) %out) #1 {
   ret void
 }
 
+; GCN-LABEL: {{^}}prng_undef_i32:
+; SDAG-NOT: v_prng_b32
+define amdgpu_kernel void @prng_undef_i32(ptr addrspace(1) %out) #1 {
+  %prng = call i32 @llvm.amdgcn.prng.b32(i32 undef)
+  store i32 %prng, ptr addrspace(1) %out, align 4
+  ret void
+}
 
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
index 7a20b5c9c51ae..a2c1545743039 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.atomic.buffer.load.ll
@@ -1,27 +1,52 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=0 | FileCheck %s -check-prefixes=CHECK,CHECK-SDAG-TRUE16
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=0 | FileCheck %s -check-prefixes=CHECK,CHECK-FAKE16
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=CHECK,CHECK-GISEL
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=CHECK,CHECK-GISEL
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG-TRUE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL-TRUE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX12,GFX12-SDAG-TRUE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX12,GFX12-GISEL-TRUE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
 
 define amdgpu_kernel void @raw_atomic_buffer_load_i32(<4 x i32> %addr) {
-; CHECK-LABEL: raw_atomic_buffer_load_i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB0_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB0_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_atomic_buffer_load_i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB0_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_atomic_buffer_load_i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB0_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -34,23 +59,42 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_atomic_buffer_load_i32_off(<4 x i32> %addr) {
-; CHECK-LABEL: raw_atomic_buffer_load_i32_off:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB1_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB1_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB1_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_atomic_buffer_load_i32_off:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB1_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -62,23 +106,43 @@ bb2:
   ret void
 }
 define amdgpu_kernel void @raw_atomic_buffer_load_i32_soff(<4 x i32> %addr) {
-; CHECK-LABEL: raw_atomic_buffer_load_i32_soff:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB2_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB2_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB2_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_atomic_buffer_load_i32_soff:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_mov_b32 s5, 4
+; GFX12-NEXT:  .LBB2_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -90,23 +154,42 @@ bb2:
   ret void
 }
 define amdgpu_kernel void @raw_atomic_buffer_load_i32_dlc(<4 x i32> %addr) {
-; CHECK-LABEL: raw_atomic_buffer_load_i32_dlc:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB3_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB3_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB3_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_atomic_buffer_load_i32_dlc:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB3_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -119,24 +202,44 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_nonatomic_buffer_load_i32(<4 x i32> %addr) {
-; CHECK-LABEL: raw_nonatomic_buffer_load_i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
-; CHECK-NEXT:    s_mov_b32 s0, 0
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:  .LBB4_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; CHECK-NEXT:    s_or_b32 s0, s1, s0
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; CHECK-NEXT:    s_cbranch_execnz .LBB4_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:  .LBB4_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_or_b32 s0, s1, s0
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_nonatomic_buffer_load_i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s0, 0
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:  .LBB4_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b32 s0, s1, s0
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -149,23 +252,43 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_atomic_buffer_load_i64(<4 x i32> %addr) {
-; CHECK-LABEL: raw_atomic_buffer_load_i64:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB5_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB5_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_atomic_buffer_load_i64:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB5_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_atomic_buffer_load_i64:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB5_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %id.zext = zext i32 %id to i64
@@ -179,23 +302,42 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_atomic_buffer_load_v2i16(<4 x i32> %addr) {
-; CHECK-LABEL: raw_atomic_buffer_load_v2i16:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB6_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB6_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB6_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB6_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_atomic_buffer_load_v2i16:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB6_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB6_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -209,68 +351,151 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_atomic_buffer_load_v4i16(<4 x i32> %addr) {
-; CHECK-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
-; CHECK-SDAG-TRUE16:       ; %bb.0: ; %bb
-; CHECK-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
-; CHECK-SDAG-TRUE16-NEXT:  .LBB7_1: ; %bb1
-; CHECK-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-SDAG-TRUE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; CHECK-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; CHECK-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
-; CHECK-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
-; CHECK-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
-; CHECK-SDAG-TRUE16-NEXT:    s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-SDAG-TRUE16:       ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT:  .LBB7_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT:    s_endpgm
 ;
-; CHECK-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
-; CHECK-FAKE16:       ; %bb.0: ; %bb
-; CHECK-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-FAKE16-NEXT:    s_mov_b32 s4, 0
-; CHECK-FAKE16-NEXT:  .LBB7_1: ; %bb1
-; CHECK-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-FAKE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; CHECK-FAKE16-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; CHECK-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-FAKE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
-; CHECK-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-FAKE16-NEXT:    s_cbranch_execnz .LBB7_1
-; CHECK-FAKE16-NEXT:  ; %bb.2: ; %bb2
-; CHECK-FAKE16-NEXT:    s_endpgm
+; GFX11-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-FAKE16:       ; %bb.0: ; %bb
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT:  .LBB7_1: ; %bb1
+; GFX11-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-FAKE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
-; CHECK-GISEL-LABEL: raw_atomic_buffer_load_v4i16:
-; CHECK-GISEL:       ; %bb.0: ; %bb
-; CHECK-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-GISEL-NEXT:    s_mov_b32 s4, 0
-; CHECK-GISEL-NEXT:  .LBB7_1: ; %bb1
-; CHECK-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-GISEL-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s5, v1
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s6, v2
-; CHECK-GISEL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; CHECK-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; CHECK-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v0
-; CHECK-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-GISEL-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-GISEL-NEXT:    s_cbranch_execnz .LBB7_1
-; CHECK-GISEL-NEXT:  ; %bb.2: ; %bb2
-; CHECK-GISEL-NEXT:    s_endpgm
+; GFX11-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16:       ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT:  .LBB7_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v2.l
+; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX11-GISEL:       ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT:  .LBB7_1: ; %bb1
+; GFX11-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX11-GISEL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-NEXT:  ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16:       ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT:  .LBB7_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT:    s_endpgm
+;
+; GFX12-FAKE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-FAKE16:       ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT:  .LBB7_1: ; %bb1
+; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX12-FAKE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT:    s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16:       ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT:  .LBB7_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v3.l
+; GFX12-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -285,23 +510,42 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_atomic_buffer_load_v4i32(<4 x i32> %addr) {
-; CHECK-LABEL: raw_atomic_buffer_load_v4i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB8_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v4, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB8_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_atomic_buffer_load_v4i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB8_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_atomic_buffer_load_v4i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB8_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -315,25 +559,46 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_atomic_buffer_load_ptr(<4 x i32> %addr) {
-; CHECK-LABEL: raw_atomic_buffer_load_ptr:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB9_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_load_b32 v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB9_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_atomic_buffer_load_ptr:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB9_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    flat_load_b32 v1, v[1:2]
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_atomic_buffer_load_ptr:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB9_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    flat_load_b32 v1, v[2:3]
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.atomic.fadd.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.atomic.fadd.ll
index 5c0e34c5e2ec0..d51e912a41bf9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.atomic.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.atomic.fadd.ll
@@ -1,58 +1,95 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx908 < %s | FileCheck -check-prefix=CHECK %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX12 %s
 
 define void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
-; CHECK-LABEL: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_atomic_add_f32 v0, v1, s[16:19], s20 offen offset:24
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_add_f32 v0, v1, s[16:19], s20 offen offset:24
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_add_nc_u32_e32 v1, 24, v1
+; GFX12-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen
+; GFX12-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 24
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
 }
 
 define void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset(float %val, <4 x i32> inreg %rsrc, i32 inreg %soffset) {
-; CHECK-LABEL: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_atomic_add_f32 v0, off, s[16:19], s20
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_add_f32 v0, off, s[16:19], s20
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_atomic_add_f32 v0, off, s[0:3], s16
+; GFX12-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 0, i32 %soffset, i32 0)
   ret void
 }
 
 define void @raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(<2 x half> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
-; CHECK-LABEL: raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[16:19], s20 offen
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[16:19], s20 offen
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[0:3], s16 offen
+; GFX12-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
 
 define void @raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset(<2 x half> %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
-; CHECK-LABEL: raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[16:19], s20 offset:92
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[16:19], s20 offset:92
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: raw_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[0:3], s16 offset:92
+; GFX12-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.raw.buffer.atomic.fadd.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 92, i32 %soffset, i32 0)
   ret void
 }
 
 define void @raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc(float %val, <4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
-; CHECK-LABEL: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
-; CHECK:       ; %bb.0:
-; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    buffer_atomic_add_f32 v0, v1, s[16:19], s20 offen slc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    s_setpc_b64 s[30:31]
+; GFX9-LABEL: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_atomic_add_f32 v0, v1, s[16:19], s20 offen slc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: raw_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_NT
+; GFX12-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.raw.buffer.atomic.fadd.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 2)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll
index 8a6594f93bd9d..1a1a1f784464d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.tfe.ll
@@ -6,6 +6,7 @@
 ; RUN: llc -mcpu=gfx1010 -mtriple=amdgcn-- < %s | FileCheck %s -check-prefixes=GFX910,GFX10
 ; RUN: llc -mcpu=gfx1100 -mtriple=amdgcn-- < %s | FileCheck %s -check-prefix=GFX11
 ; RUN: llc -mcpu=gfx1200 -mtriple=amdgcn-- < %s | FileCheck %s -check-prefix=GFX12
+; RUN: llc -mcpu=gfx1250 -mtriple=amdgcn-- < %s | FileCheck %s -check-prefix=GFX12
 
 define amdgpu_ps void @raw_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
 ; GFX67-LABEL: raw_buffer_load_i8_tfe:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
index 89511deaa2bbc..eeea1456792af 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
@@ -3,6 +3,7 @@
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=GFX68,GFX8 %s
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 | FileCheck -check-prefixes=GFX11 %s
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1200 | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 | FileCheck -check-prefixes=GFX12 %s
 
 define amdgpu_ps void @buffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <4 x float>) {
 ; GFX68-LABEL: buffer_store:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
index 561ec7de94bc6..6f7c001e03e26 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
@@ -1,27 +1,52 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=0 | FileCheck %s -check-prefixes=CHECK,CHECK-SDAG-TRUE16
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=0 | FileCheck %s -check-prefixes=CHECK,CHECK-FAKE16
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=CHECK,CHECK-GISEL
-; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=CHECK,CHECK-GISEL
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG-TRUE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL-TRUE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -global-isel=1 -new-reg-bank-select | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX12,GFX12-SDAG-TRUE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -global-isel=0 | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX12,GFX12-GISEL-TRUE16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -global-isel=1 | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
 
 define amdgpu_kernel void @raw_ptr_atomic_buffer_ptr_load_i32(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB0_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB0_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB0_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_ptr_atomic_buffer_ptr_load_i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB0_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -34,23 +59,42 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_off(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: raw_ptr_atomic_buffer_load_i32_off:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB1_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB1_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB1_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_ptr_atomic_buffer_load_i32_off:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB1_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -62,23 +106,43 @@ bb2:
   ret void
 }
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_soff(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB2_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB2_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB2_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 4 offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_ptr_atomic_buffer_load_i32_soff:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_mov_b32 s5, 4
+; GFX12-NEXT:  .LBB2_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], s5 offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -90,23 +154,42 @@ bb2:
   ret void
 }
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB3_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB3_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB3_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_ptr_atomic_buffer_load_i32_dlc:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB3_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT_RT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -119,24 +202,44 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_nonptr_atomic_buffer_load_i32(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: raw_nonptr_atomic_buffer_load_i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
-; CHECK-NEXT:    s_mov_b32 s0, 0
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:  .LBB4_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; CHECK-NEXT:    s_or_b32 s0, s1, s0
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; CHECK-NEXT:    s_cbranch_execnz .LBB4_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 offset:4 glc
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:  .LBB4_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_or_b32 s0, s1, s0
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_nonptr_atomic_buffer_load_i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s0, 0
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:  .LBB4_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b32 s0, s1, s0
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -149,23 +252,43 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: raw_ptr_atomic_buffer_load_i64:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB5_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB5_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB5_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], 0 offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_ptr_atomic_buffer_load_i64:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB5_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %id.zext = zext i32 %id to i64
@@ -179,23 +302,42 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v2i16(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: raw_ptr_atomic_buffer_load_v2i16:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB6_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB6_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB6_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v1, off, s[0:3], 0 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB6_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_ptr_atomic_buffer_load_v2i16:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB6_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v1, off, s[0:3], null th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB6_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -209,68 +351,151 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %ptr) {
-; CHECK-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
-; CHECK-SDAG-TRUE16:       ; %bb.0: ; %bb
-; CHECK-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
-; CHECK-SDAG-TRUE16-NEXT:  .LBB7_1: ; %bb1
-; CHECK-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-SDAG-TRUE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; CHECK-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; CHECK-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
-; CHECK-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
-; CHECK-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
-; CHECK-SDAG-TRUE16-NEXT:    s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-SDAG-TRUE16:       ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT:  .LBB7_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT:    s_endpgm
 ;
-; CHECK-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
-; CHECK-FAKE16:       ; %bb.0: ; %bb
-; CHECK-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-FAKE16-NEXT:    s_mov_b32 s4, 0
-; CHECK-FAKE16-NEXT:  .LBB7_1: ; %bb1
-; CHECK-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-FAKE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; CHECK-FAKE16-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; CHECK-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-FAKE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
-; CHECK-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-FAKE16-NEXT:    s_cbranch_execnz .LBB7_1
-; CHECK-FAKE16-NEXT:  ; %bb.2: ; %bb2
-; CHECK-FAKE16-NEXT:    s_endpgm
+; GFX11-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-FAKE16:       ; %bb.0: ; %bb
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT:  .LBB7_1: ; %bb1
+; GFX11-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshl_or_b32 v1, v2, 16, v1
+; GFX11-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-FAKE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-FAKE16-NEXT:    s_endpgm
 ;
-; CHECK-GISEL-LABEL: raw_ptr_atomic_buffer_load_v4i16:
-; CHECK-GISEL:       ; %bb.0: ; %bb
-; CHECK-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-GISEL-NEXT:    s_mov_b32 s4, 0
-; CHECK-GISEL-NEXT:  .LBB7_1: ; %bb1
-; CHECK-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-GISEL-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s5, v1
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s6, v2
-; CHECK-GISEL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; CHECK-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; CHECK-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v0
-; CHECK-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-GISEL-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-GISEL-NEXT:    s_cbranch_execnz .LBB7_1
-; CHECK-GISEL-NEXT:  ; %bb.2: ; %bb2
-; CHECK-GISEL-NEXT:    s_endpgm
+; GFX11-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16:       ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT:  .LBB7_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v1.h, v2.l
+; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL:       ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT:  .LBB7_1: ; %bb1
+; GFX11-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s6, v2
+; GFX11-GISEL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-GISEL-NEXT:  ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX12-SDAG-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16:       ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT:  .LBB7_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX12-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT:    s_endpgm
+;
+; GFX12-FAKE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-FAKE16:       ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT:  .LBB7_1: ; %bb1
+; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v1, 0xffff, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX12-FAKE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT:    s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: raw_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16:       ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT:  .LBB7_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v3.l
+; GFX12-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX12-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -285,23 +510,42 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_v4i32(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: raw_ptr_atomic_buffer_load_v4i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB8_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v4, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB8_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_ptr_atomic_buffer_load_v4i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB8_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b128 v[1:4], off, s[0:3], 0 offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v4, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_ptr_atomic_buffer_load_v4i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB8_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b128 v[2:5], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -315,25 +559,46 @@ bb2:
 }
 
 define amdgpu_kernel void @raw_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: raw_ptr_atomic_buffer_load_ptr:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB9_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_load_b32 v1, v[1:2]
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB9_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: raw_ptr_atomic_buffer_load_ptr:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB9_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b64 v[1:2], off, s[0:3], 0 offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    flat_load_b32 v1, v[1:2]
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: raw_ptr_atomic_buffer_load_ptr:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB9_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b64 v[2:3], off, s[0:3], null offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    flat_load_b32 v1, v[2:3]
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
index 8b6ba1a3cc094..2c3b5210ae7bd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16.ll
@@ -1,104 +1,174 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; FIXME: Test 90a, 940. 908 should fail to select.
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
 
 define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset(<2 x bfloat> %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen offset:128 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen offset:128 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_add_nc_u32_e32 v1, 0x80, v1
+; GFX1250-NEXT:    buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 128
   %ret = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret <2 x bfloat> %ret
 }
 
 define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc(<2 x bfloat> %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 92, i32 %soffset, i32 2)
   ret <2 x bfloat> %ret
 }
 
 define void @raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset(<2 x bfloat> %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen offset:128
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen offset:128
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_add_nc_u32_e32 v1, 0x80, v1
+; GFX1250-NEXT:    buffer_atomic_pk_add_bf16 v0, v1, s[0:3], s16 offen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 128
   %unused = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
 }
 
 define void @raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc(<2 x bfloat> %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset__slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_pk_add_bf16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_NT
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %unused = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 92, i32 %soffset, i32 2)
   ret void
 }
 
 ; Test waterfall loop
 define <2 x bfloat> @raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset_add__vgpr_soffset(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset) #0 {
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_mov_b32 s2, exec_lo
-; GFX12-NEXT:  .LBB4_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v2
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
-; GFX12-NEXT:    v_readfirstlane_b32 s3, v6
-; GFX12-NEXT:    s_wait_alu 0xf1ff
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX12-NEXT:    v_cmp_eq_u32_e64 s1, s3, v6
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_b32 s0, s0, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_atomic_pk_add_bf16 v0, v5, s[4:7], s3 offen offset:128 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
-; GFX12-NEXT:    ; implicit-def: $vgpr6
-; GFX12-NEXT:    ; implicit-def: $vgpr5
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
-; GFX12-NEXT:  ; %bb.2:
-; GFX12-NEXT:    s_mov_b32 exec_lo, s2
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1200-NEXT:  .LBB4_1: ; =>This Inner Loop Header: Depth=1
+; GFX1200-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1200-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX1200-NEXT:    v_readfirstlane_b32 s3, v6
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
+; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
+; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v6
+; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_and_b32 s0, s0, s1
+; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_pk_add_bf16 v0, v5, s[4:7], s3 offen offset:128 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT:    ; implicit-def: $vgpr6
+; GFX1200-NEXT:    ; implicit-def: $vgpr5
+; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1200-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX1200-NEXT:  ; %bb.2:
+; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2bf16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v11, v4 :: v_dual_mov_b32 v10, v3
+; GFX1250-NEXT:    v_dual_mov_b32 v9, v2 :: v_dual_mov_b32 v8, v1
+; GFX1250-NEXT:    v_add_nc_u32_e32 v1, 0x80, v5
+; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:  .LBB4_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_readfirstlane_b32 s4, v8
+; GFX1250-NEXT:    v_readfirstlane_b32 s5, v9
+; GFX1250-NEXT:    v_readfirstlane_b32 s6, v10
+; GFX1250-NEXT:    v_readfirstlane_b32 s7, v11
+; GFX1250-NEXT:    v_readfirstlane_b32 s3, v6
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[8:9]
+; GFX1250-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[10:11]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_cmp_eq_u32_e64 s1, s3, v6
+; GFX1250-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_and_b32 s0, s0, s1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_pk_add_bf16 v0, v1, s[4:7], s3 offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11
+; GFX1250-NEXT:    ; implicit-def: $vgpr6
+; GFX1250-NEXT:    ; implicit-def: $vgpr1
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX1250-NEXT:  ; %bb.2:
+; GFX1250-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 128
   %ret = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2bf16(<2 x bfloat> %val, ptr addrspace(8) %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
   ret <2 x bfloat> %ret
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd_nortn.ll
index 8141e0df46737..ea8f836ab49a3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd_nortn.ll
@@ -2,7 +2,8 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx908 < %s | FileCheck -check-prefix=GFX908 %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
 
 define void @raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {
 ; GFX908-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
@@ -26,15 +27,22 @@ define void @raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voff
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen scope:SCOPE_SYS
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen scope:SCOPE_SYS
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen scope:SCOPE_SYS
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 24)
   ret void
 }
@@ -61,15 +69,22 @@ define void @raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_add_f32 v0, off, s[0:3], s16
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_add_f32 v0, off, s[0:3], s16
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, off, s[0:3], s16
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 0, i32 %soffset, i32 0)
   ret void
 }
@@ -96,15 +111,22 @@ define void @raw_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[0:3], s16 offen
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[0:3], s16 offen
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[0:3], s16 offen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -131,15 +153,22 @@ define void @raw_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffs
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[0:3], s16 offset:92
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[0:3], s16 offset:92
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[0:3], s16 offset:92
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 92, i32 %soffset, i32 0)
   ret void
 }
@@ -166,15 +195,22 @@ define void @raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voff
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_NT
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_NT
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_NT
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 2)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd_rtn.ll
index 767117dc99fd4..28387403c5048 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.atomic.fadd_rtn.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
 
 define float @raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) #0 {
 ; GFX90A-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
@@ -18,16 +19,24 @@ define float @raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffs
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 24)
   ret float %ret
 }
@@ -47,16 +56,24 @@ define float @raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset_
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_add_f32 v0, off, s[0:3], s16 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_add_f32 v0, off, s[0:3], s16 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, off, s[0:3], s16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 0, i32 %soffset, i32 0)
   ret float %ret
 }
@@ -76,16 +93,24 @@ define <2 x half> @raw_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgp
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_pk_add_f16 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
   ret <2 x half> %ret
 }
@@ -105,16 +130,24 @@ define <2 x half> @raw_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__0_v
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_pk_add_f16 v0, off, s[0:3], s16 offset:92 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 92, i32 %soffset, i32 0)
   ret <2 x half> %ret
 }
@@ -134,16 +167,24 @@ define float @raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffs
 ; GFX942-NEXT:    s_waitcnt vmcnt(0)
 ; GFX942-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_NT_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_NT_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: raw_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 offen th:TH_ATOMIC_NT_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 2)
   ret float %ret
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.bf16.ll
index 3540468566147..4dd258b7bda82 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.load.bf16.ll
@@ -3,7 +3,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck --check-prefix=GFX8 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefix=GFX12 %s
 
 define bfloat @raw_ptr_buffer_load_bf16(ptr addrspace(8) inreg %rsrc) {
 ; GFX7-LABEL: raw_ptr_buffer_load_bf16:
@@ -41,6 +42,14 @@ define bfloat @raw_ptr_buffer_load_bf16(ptr addrspace(8) inreg %rsrc) {
 ; GFX11-NEXT:    buffer_load_u16 v0, off, s[0:3], 0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: raw_ptr_buffer_load_bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_u16 v0, off, s[0:3], null
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_set_pc_i64 s[30:31]
   %val = call bfloat @llvm.amdgcn.raw.ptr.buffer.load.v2bf16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
   ret bfloat %val
 }
@@ -82,6 +91,14 @@ define <2 x bfloat> @raw_ptr_buffer_load_v2bf16(ptr addrspace(8) inreg %rsrc) {
 ; GFX11-NEXT:    buffer_load_b32 v0, off, s[0:3], 0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: raw_ptr_buffer_load_v2bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v0, off, s[0:3], null
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_set_pc_i64 s[30:31]
   %val = call <2 x bfloat> @llvm.amdgcn.raw.ptr.buffer.load.v2bf16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
   ret <2 x bfloat> %val
 }
@@ -125,6 +142,14 @@ define <4 x bfloat> @raw_ptr_buffer_load_v4bf16(ptr addrspace(8) inreg %rsrc) {
 ; GFX11-NEXT:    buffer_load_b64 v[0:1], off, s[0:3], 0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: raw_ptr_buffer_load_v4bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b64 v[0:1], off, s[0:3], null
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_set_pc_i64 s[30:31]
   %val = call <4 x bfloat> @llvm.amdgcn.raw.ptr.buffer.load.v4bf16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
   ret <4 x bfloat> %val
 }
@@ -178,6 +203,14 @@ define <8 x bfloat> @raw_ptr_buffer_load_v8bf16(ptr addrspace(8) inreg %rsrc) {
 ; GFX11-NEXT:    buffer_load_b128 v[0:3], off, s[0:3], 0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: raw_ptr_buffer_load_v8bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b128 v[0:3], off, s[0:3], null
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    s_set_pc_i64 s[30:31]
   %val = call <8 x bfloat> @llvm.amdgcn.raw.ptr.buffer.load.v8bf16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
   ret <8 x bfloat> %val
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.bf16.ll
index e1f84dcbaa607..ec7d7d467ffc6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.bf16.ll
@@ -3,7 +3,8 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck --check-prefix=GFX8 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck --check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck --check-prefix=GFX10 %s
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefix=GFX12 %s
 
 define amdgpu_ps void @buffer_store_bf16(ptr addrspace(8) inreg %rsrc, bfloat %data, i32 %offset) {
 ; GFX7-LABEL: buffer_store_bf16:
@@ -32,6 +33,11 @@ define amdgpu_ps void @buffer_store_bf16(ptr addrspace(8) inreg %rsrc, bfloat %d
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    buffer_store_b16 v0, v1, s[0:3], 0 offen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    buffer_store_b16 v0, v1, s[0:3], null offen
+; GFX12-NEXT:    s_endpgm
   call void @llvm.amdgcn.raw.ptr.buffer.store.bf16(bfloat %data, ptr addrspace(8) %rsrc, i32 %offset, i32 0, i32 0)
   ret void
 }
@@ -65,6 +71,11 @@ define amdgpu_ps void @buffer_store_v2bf16(ptr addrspace(8) inreg %rsrc, <2 x bf
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    buffer_store_b32 v0, v1, s[0:3], 0 offen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_v2bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    buffer_store_b32 v0, v1, s[0:3], null offen
+; GFX12-NEXT:    s_endpgm
   call void @llvm.amdgcn.raw.ptr.buffer.store.v2bf16(<2 x bfloat> %data, ptr addrspace(8) %rsrc, i32 %offset, i32 0, i32 0)
   ret void
 }
@@ -102,6 +113,11 @@ define amdgpu_ps void @buffer_store_v4bf16(ptr addrspace(8) inreg %rsrc, <4 x bf
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    buffer_store_b64 v[0:1], v2, s[0:3], 0 offen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_v4bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    buffer_store_b64 v[0:1], v2, s[0:3], null offen
+; GFX12-NEXT:    s_endpgm
   call void @llvm.amdgcn.raw.ptr.buffer.store.v4bf16(<4 x bfloat> %data, ptr addrspace(8) %rsrc, i32 %offset, i32 0, i32 0)
   ret void
 }
@@ -153,6 +169,11 @@ define amdgpu_ps void @buffer_store_v8bf16(ptr addrspace(8) inreg %rsrc, <8 x bf
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    buffer_store_b128 v[0:3], v4, s[0:3], 0 offen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_v8bf16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v4, s[0:3], null offen
+; GFX12-NEXT:    s_endpgm
   call void @llvm.amdgcn.raw.ptr.buffer.store.v8bf16(<8 x bfloat> %data, ptr addrspace(8) %rsrc, i32 %offset, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
index 90e150c89955b..9003251253740 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
@@ -98,7 +98,6 @@ define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
 ; VARIANT4-NEXT:    s_wait_kmcnt 0x0
 ; VARIANT4-NEXT:    v_xad_u32 v0, v2, -1, s2
 ; VARIANT4-NEXT:    global_store_b32 v3, v2, s[0:1]
-; VARIANT4-NEXT:    s_wait_storecnt 0x0
 ; VARIANT4-NEXT:    s_barrier_signal -1
 ; VARIANT4-NEXT:    s_barrier_wait -1
 ; VARIANT4-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
@@ -145,7 +144,6 @@ define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
 ; VARIANT6-NEXT:    v_sub_nc_u32_e32 v0, s2, v4
 ; VARIANT6-NEXT:    global_store_b32 v5, v4, s[0:1]
 ; VARIANT6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; VARIANT6-NEXT:    s_wait_storecnt 0x0
 ; VARIANT6-NEXT:    s_barrier_signal -1
 ; VARIANT6-NEXT:    s_barrier_wait -1
 ; VARIANT6-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll
index 651d204f65b6c..248e0c716b975 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll
@@ -11,7 +11,6 @@ define i1 @func1() {
 ; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-NEXT:    s_cmp_eq_u32 0, 0
-; GFX12-SDAG-NEXT:    s_wait_storecnt 0x0
 ; GFX12-SDAG-NEXT:    s_barrier_signal_isfirst -1
 ; GFX12-SDAG-NEXT:    s_cselect_b32 s0, -1, 0
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
@@ -27,7 +26,6 @@ define i1 @func1() {
 ; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-GISEL-NEXT:    s_cmp_eq_u32 0, 0
-; GFX12-GISEL-NEXT:    s_wait_storecnt 0x0
 ; GFX12-GISEL-NEXT:    s_barrier_signal_isfirst -1
 ; GFX12-GISEL-NEXT:    s_cselect_b32 s0, 1, 0
 ; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
index f6f614e6e41b5..88963643218a5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
@@ -1,30 +1,58 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-SDAG-TRUE16
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-FAKE16
-; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-GISEL
-; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG-TRUE16
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL-TRUE16
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-SDAG-TRUE16
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-GISEL-TRUE16
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
 
 define amdgpu_kernel void @struct_atomic_buffer_load_i32(<4 x i32> %addr, i32 %index) {
-; CHECK-LABEL: struct_atomic_buffer_load_i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB0_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB0_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_atomic_buffer_load_i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB0_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_atomic_buffer_load_i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB0_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -37,23 +65,43 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_atomic_buffer_load_i32_const_idx(<4 x i32> %addr) {
-; CHECK-LABEL: struct_atomic_buffer_load_i32_const_idx:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB1_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB1_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB1_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_atomic_buffer_load_i32_const_idx:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    v_mov_b32_e32 v1, 15
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB1_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -66,26 +114,48 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_atomic_buffer_load_i32_off(<4 x i32> %addr, i32 %index) {
-; CHECK-LABEL: struct_atomic_buffer_load_i32_off:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB2_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB2_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB2_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_atomic_buffer_load_i32_off:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB2_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -98,26 +168,49 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_atomic_buffer_load_i32_soff(<4 x i32> %addr, i32 %index) {
-; CHECK-LABEL: struct_atomic_buffer_load_i32_soff:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB3_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB3_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB3_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_atomic_buffer_load_i32_soff:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_mov_b32 s5, 4
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB3_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -129,26 +222,48 @@ bb2:
   ret void
 }
 define amdgpu_kernel void @struct_atomic_buffer_load_i32_dlc(<4 x i32> %addr, i32 %index) {
-; CHECK-LABEL: struct_atomic_buffer_load_i32_dlc:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB4_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB4_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB4_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_atomic_buffer_load_i32_dlc:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB4_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -161,26 +276,49 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_nonatomic_buffer_load_i32(<4 x i32> %addr, i32 %index) {
-; CHECK-LABEL: struct_nonatomic_buffer_load_i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
-; CHECK-NEXT:    buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-NEXT:    s_mov_b32 s0, 0
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:  .LBB5_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; CHECK-NEXT:    s_or_b32 s0, s1, s0
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; CHECK-NEXT:    s_cbranch_execnz .LBB5_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT:    buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:  .LBB5_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_or_b32 s0, s1, s0
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_nonatomic_buffer_load_i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:    buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s0, 0
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:  .LBB5_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b32 s0, s1, s0
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -193,26 +331,49 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %index) {
-; CHECK-LABEL: struct_atomic_buffer_load_i64:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v2, s6
-; CHECK-NEXT:  .LBB6_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB6_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_atomic_buffer_load_i64:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v2, s6
+; GFX11-NEXT:  .LBB6_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB6_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_atomic_buffer_load_i64:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v2, s6
+; GFX12-NEXT:  .LBB6_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB6_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %id.zext = zext i32 %id to i64
@@ -226,26 +387,48 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_atomic_buffer_load_v2i16(<4 x i32> %addr, i32 %index) {
-; CHECK-LABEL: struct_atomic_buffer_load_v2i16:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB7_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB7_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB7_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_atomic_buffer_load_v2i16:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB7_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -259,77 +442,172 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32 %index) {
-; CHECK-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
-; CHECK-SDAG-TRUE16:       ; %bb.0: ; %bb
-; CHECK-SDAG-TRUE16-NEXT:    s_clause 0x1
-; CHECK-SDAG-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
-; CHECK-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-SDAG-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-SDAG-TRUE16-NEXT:  .LBB8_1: ; %bb1
-; CHECK-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-SDAG-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; CHECK-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; CHECK-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
-; CHECK-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
-; CHECK-SDAG-TRUE16-NEXT:    s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-SDAG-TRUE16:       ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT:    s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT:  .LBB8_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-FAKE16:       ; %bb.0: ; %bb
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-FAKE16-NEXT:  .LBB8_1: ; %bb1
+; GFX11-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-FAKE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16:       ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT:    s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT:  .LBB8_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v3.l
+; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX11-GISEL:       ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT:  .LBB8_1: ; %bb1
+; GFX11-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX11-GISEL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-NEXT:  ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; CHECK-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
-; CHECK-FAKE16:       ; %bb.0: ; %bb
-; CHECK-FAKE16-NEXT:    s_clause 0x1
-; CHECK-FAKE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-FAKE16-NEXT:    s_mov_b32 s4, 0
-; CHECK-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-FAKE16-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-FAKE16-NEXT:  .LBB8_1: ; %bb1
-; CHECK-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-FAKE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-FAKE16-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; CHECK-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-FAKE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; CHECK-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-FAKE16-NEXT:    s_cbranch_execnz .LBB8_1
-; CHECK-FAKE16-NEXT:  ; %bb.2: ; %bb2
-; CHECK-FAKE16-NEXT:    s_endpgm
+; GFX12-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16:       ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT:    s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT:  .LBB8_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT:    s_endpgm
 ;
-; CHECK-GISEL-LABEL: struct_atomic_buffer_load_v4i16:
-; CHECK-GISEL:       ; %bb.0: ; %bb
-; CHECK-GISEL-NEXT:    s_clause 0x1
-; CHECK-GISEL-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-GISEL-NEXT:    s_mov_b32 s4, 0
-; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-GISEL-NEXT:  .LBB8_1: ; %bb1
-; CHECK-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-GISEL-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s5, v2
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
-; CHECK-GISEL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; CHECK-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; CHECK-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v0
-; CHECK-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-GISEL-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-GISEL-NEXT:    s_cbranch_execnz .LBB8_1
-; CHECK-GISEL-NEXT:  ; %bb.2: ; %bb2
-; CHECK-GISEL-NEXT:    s_endpgm
+; GFX12-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-FAKE16:       ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT:    s_clause 0x1
+; GFX12-FAKE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-FAKE16-NEXT:  .LBB8_1: ; %bb1
+; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX12-FAKE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT:    s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16:       ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT:    s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT:  .LBB8_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v3.l
+; GFX12-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -344,26 +622,48 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_atomic_buffer_load_v4i32(<4 x i32> %addr, i32 %index) {
-; CHECK-LABEL: struct_atomic_buffer_load_v4i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB9_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB9_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_atomic_buffer_load_v4i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB9_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_atomic_buffer_load_v4i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB9_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -377,28 +677,52 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_atomic_buffer_load_ptr(<4 x i32> %addr, i32 %index) {
-; CHECK-LABEL: struct_atomic_buffer_load_ptr:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB10_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_load_b32 v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB10_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_atomic_buffer_load_ptr:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB10_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    flat_load_b32 v2, v[2:3]
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB10_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_atomic_buffer_load_ptr:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB10_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    flat_load_b32 v2, v[2:3]
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB10_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.tfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.tfe.ll
index 13b28d408ba88..9abbc064803da 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.tfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.tfe.ll
@@ -6,6 +6,7 @@
 ; RUN: llc -mcpu=gfx1010 -mtriple=amdgcn-- < %s | FileCheck %s -check-prefixes=GFX910,GFX10
 ; RUN: llc -mcpu=gfx1100 -mtriple=amdgcn-- < %s | FileCheck %s -check-prefix=GFX11
 ; RUN: llc -mcpu=gfx1200 -mtriple=amdgcn-- < %s | FileCheck %s -check-prefix=GFX12
+; RUN: llc -mcpu=gfx1250 -mtriple=amdgcn-- < %s | FileCheck %s -check-prefix=GFX12
 
 define amdgpu_ps void @struct_buffer_load_i8_tfe(<4 x i32> inreg %rsrc, ptr addrspace(1) %data_addr, ptr addrspace(1) %tfe_addr) {
 ; GFX67-LABEL: struct_buffer_load_i8_tfe:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
index 9ce33c68c463f..822016b23c952 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.ll
@@ -3,6 +3,8 @@
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga | FileCheck -check-prefixes=GFX68,GFX8 %s
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
 
 define amdgpu_ps void @buffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <4 x float>) {
 ; GFX68-LABEL: buffer_store:
@@ -21,6 +23,15 @@ define amdgpu_ps void @buffer_store(<4 x i32> inreg, <4 x float>, <4 x float>, <
 ; GFX11-NEXT:    buffer_store_b128 v[4:7], v12, s[0:3], 0 idxen glc
 ; GFX11-NEXT:    buffer_store_b128 v[8:11], v12, s[0:3], 0 idxen slc
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    v_mov_b32_e32 v12, 0
+; GFX12-NEXT:    s_clause 0x2
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v12, s[0:3], null idxen
+; GFX12-NEXT:    buffer_store_b128 v[4:7], v12, s[0:3], null idxen th:TH_STORE_NT
+; GFX12-NEXT:    buffer_store_b128 v[8:11], v12, s[0:3], null idxen th:TH_STORE_HT
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 0, i32 0, i32 0, i32 0)
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %2, <4 x i32> %0, i32 0, i32 0, i32 0, i32 1)
@@ -40,6 +51,12 @@ define amdgpu_ps void @buffer_store_immoffs(<4 x i32> inreg, <4 x float>) {
 ; GFX11-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX11-NEXT:    buffer_store_b128 v[0:3], v4, s[0:3], 0 idxen offset:42
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_immoffs:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    v_mov_b32_e32 v4, 0
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v4, s[0:3], null idxen offset:42
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 0, i32 42, i32 0, i32 0)
   ret void
@@ -55,6 +72,11 @@ define amdgpu_ps void @buffer_store_idx(<4 x i32> inreg, <4 x float>, i32) {
 ; GFX11:       ; %bb.0: ; %main_body
 ; GFX11-NEXT:    buffer_store_b128 v[0:3], v4, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_idx:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v4, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 0, i32 0, i32 0)
   ret void
@@ -76,6 +98,12 @@ define amdgpu_ps void @buffer_store_ofs(<4 x i32> inreg, <4 x float>, i32) {
 ; GFX11-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, s4
 ; GFX11-NEXT:    buffer_store_b128 v[0:3], v[4:5], s[0:3], 0 idxen offen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_ofs:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, 0
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v[4:5], s[0:3], null idxen offen
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 0, i32 %2, i32 0, i32 0)
   ret void
@@ -91,6 +119,11 @@ define amdgpu_ps void @buffer_store_both(<4 x i32> inreg, <4 x float>, i32, i32)
 ; GFX11:       ; %bb.0: ; %main_body
 ; GFX11-NEXT:    buffer_store_b128 v[0:3], v[4:5], s[0:3], 0 idxen offen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_both:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v[4:5], s[0:3], null idxen offen
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 %3, i32 0, i32 0)
   ret void
@@ -108,6 +141,12 @@ define amdgpu_ps void @buffer_store_both_reversed(<4 x i32> inreg, <4 x float>,
 ; GFX11-NEXT:    v_mov_b32_e32 v6, v4
 ; GFX11-NEXT:    buffer_store_b128 v[0:3], v[5:6], s[0:3], 0 idxen offen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_both_reversed:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    v_dual_mov_b32 v6, v5 :: v_dual_mov_b32 v7, v4
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v[6:7], s[0:3], null idxen offen
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %3, i32 %2, i32 0, i32 0)
   ret void
@@ -139,6 +178,15 @@ define amdgpu_ps void @buffer_store_wait(<4 x i32> inreg, <4 x float>, i32, i32,
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    buffer_store_b128 v[0:3], v6, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_wait:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v4, s[0:3], null idxen
+; GFX12-NEXT:    buffer_load_b128 v[0:3], v5, s[0:3], null idxen
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v6, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float> %1, <4 x i32> %0, i32 %2, i32 0, i32 0, i32 0)
   %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %0, i32 %3, i32 0, i32 0, i32 0)
@@ -156,6 +204,11 @@ define amdgpu_ps void @buffer_store_x1(<4 x i32> inreg %rsrc, float %data, i32 %
 ; GFX11:       ; %bb.0: ; %main_body
 ; GFX11-NEXT:    buffer_store_b32 v0, v1, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_x1:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    buffer_store_b32 v0, v1, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.f32(float %data, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
   ret void
@@ -171,6 +224,11 @@ define amdgpu_ps void @buffer_store_x2(<4 x i32> inreg %rsrc, <2 x float> %data,
 ; GFX11:       ; %bb.0: ; %main_body
 ; GFX11-NEXT:    buffer_store_b64 v[0:1], v2, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_x2:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    buffer_store_b64 v[0:1], v2, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.v2f32(<2 x float> %data, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
   ret void
@@ -193,6 +251,15 @@ define amdgpu_ps void @buffer_store_int(<4 x i32> inreg, <4 x i32>, <2 x i32>, i
 ; GFX11-NEXT:    buffer_store_b64 v[4:5], v7, s[0:3], 0 idxen glc
 ; GFX11-NEXT:    buffer_store_b32 v6, v7, s[0:3], 0 idxen slc
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: buffer_store_int:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    v_mov_b32_e32 v7, 0
+; GFX12-NEXT:    s_clause 0x2
+; GFX12-NEXT:    buffer_store_b128 v[0:3], v7, s[0:3], null idxen
+; GFX12-NEXT:    buffer_store_b64 v[4:5], v7, s[0:3], null idxen th:TH_STORE_NT
+; GFX12-NEXT:    buffer_store_b32 v6, v7, s[0:3], null idxen th:TH_STORE_HT
+; GFX12-NEXT:    s_endpgm
 main_body:
   call void @llvm.amdgcn.struct.buffer.store.v4i32(<4 x i32> %1, <4 x i32> %0, i32 0, i32 0, i32 0, i32 0)
   call void @llvm.amdgcn.struct.buffer.store.v2i32(<2 x i32> %2, <4 x i32> %0, i32 0, i32 0, i32 0, i32 1)
@@ -212,6 +279,12 @@ define amdgpu_ps void @struct_buffer_store_byte(<4 x i32> inreg %rsrc, float %v1
 ; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX11-NEXT:    buffer_store_b8 v0, v1, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_buffer_store_byte:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX12-NEXT:    buffer_store_b8 v0, v1, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
 main_body:
   %v2 = fptoui float %v1 to i32
   %v3 = trunc i32 %v2 to i8
@@ -237,6 +310,18 @@ define amdgpu_ps void @struct_buffer_store_f16(<4 x i32> inreg %rsrc, float %v1,
 ; GFX11-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; GFX11-FAKE16-NEXT:    buffer_store_b16 v0, v1, s[0:3], 0 idxen
 ; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX12-TRUE16-LABEL: struct_buffer_store_f16:
+; GFX12-TRUE16:       ; %bb.0:
+; GFX12-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
+; GFX12-TRUE16-NEXT:    buffer_store_b16 v0, v1, s[0:3], null idxen
+; GFX12-TRUE16-NEXT:    s_endpgm
+;
+; GFX12-FAKE16-LABEL: struct_buffer_store_f16:
+; GFX12-FAKE16:       ; %bb.0:
+; GFX12-FAKE16-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; GFX12-FAKE16-NEXT:    buffer_store_b16 v0, v1, s[0:3], null idxen
+; GFX12-FAKE16-NEXT:    s_endpgm
   %v2 = fptrunc float %v1 to half
   call void @llvm.amdgcn.struct.buffer.store.f16(half %v2, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
   ret void
@@ -261,6 +346,11 @@ define amdgpu_ps void @struct_buffer_store_v2f16(<4 x i32> inreg %rsrc, <2 x hal
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    buffer_store_b32 v0, v1, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_buffer_store_v2f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    buffer_store_b32 v0, v1, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
   call void @llvm.amdgcn.struct.buffer.store.v2f16(<2 x half> %v1, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
   ret void
 }
@@ -288,6 +378,11 @@ define amdgpu_ps void @struct_buffer_store_v4f16(<4 x i32> inreg %rsrc, <4 x hal
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    buffer_store_b64 v[0:1], v2, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_buffer_store_v4f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    buffer_store_b64 v[0:1], v2, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
   call void @llvm.amdgcn.struct.buffer.store.v4f16(<4 x half> %v1, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
   ret void
 }
@@ -304,6 +399,12 @@ define amdgpu_ps void @struct_buffer_store_i16(<4 x i32> inreg %rsrc, float %v1,
 ; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX11-NEXT:    buffer_store_b16 v0, v1, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_buffer_store_i16:
+; GFX12:       ; %bb.0: ; %main_body
+; GFX12-NEXT:    v_cvt_u32_f32_e32 v0, v0
+; GFX12-NEXT:    buffer_store_b16 v0, v1, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
 main_body:
   %v2 = fptoui float %v1 to i32
   %v3 = trunc i32 %v2 to i16
@@ -329,6 +430,11 @@ define amdgpu_ps void @struct_buffer_store_vif16(<4 x i32> inreg %rsrc, <2 x i16
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    buffer_store_b32 v0, v1, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_buffer_store_vif16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    buffer_store_b32 v0, v1, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
   call void @llvm.amdgcn.struct.buffer.store.v2i16(<2 x i16> %v1, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
   ret void
 }
@@ -354,6 +460,11 @@ define amdgpu_ps void @struct_buffer_store_v4i16(<4 x i32> inreg %rsrc, <4 x i16
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    buffer_store_b64 v[0:1], v2, s[0:3], 0 idxen
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_buffer_store_v4i16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    buffer_store_b64 v[0:1], v2, s[0:3], null idxen
+; GFX12-NEXT:    s_endpgm
   call void @llvm.amdgcn.struct.buffer.store.v4i16(<4 x i16> %v1, <4 x i32> %rsrc, i32 %index, i32 0, i32 0, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
index 8f33dd6e3a69a..23db2479f66bb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
@@ -1,30 +1,58 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-SDAG-TRUE16
-; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-FAKE16
-; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-GISEL
-; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG-TRUE16
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL-TRUE16
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-FAKE16
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-SDAG-TRUE16
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-GISEL-TRUE16
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 < %s | FileCheck %s -check-prefixes=GFX12,GFX12-FAKE16
 
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-LABEL: struct_ptr_atomic_buffer_load_i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB0_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB0_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB0_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB0_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -37,23 +65,43 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_const_idx(ptr addrspace(8) %ptr) {
-; CHECK-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:  .LBB1_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB1_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_dual_mov_b32 v1, 15 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:  .LBB1_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_const_idx:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    v_mov_b32_e32 v1, 15
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:  .LBB1_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -66,26 +114,48 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_off(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-LABEL: struct_ptr_atomic_buffer_load_i32_off:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB2_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB2_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB2_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_off:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB2_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -98,26 +168,49 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_soff(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB3_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB3_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB3_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_mov_b32 s5, 4
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB3_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB3_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -129,26 +222,48 @@ bb2:
   ret void
 }
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB4_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB4_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB4_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB4_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -161,26 +276,49 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_ptr_nonatomic_buffer_load_i32(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-LABEL: struct_ptr_nonatomic_buffer_load_i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
-; CHECK-NEXT:    buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-NEXT:    s_mov_b32 s0, 0
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; CHECK-NEXT:  .LBB5_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; CHECK-NEXT:    s_or_b32 s0, s1, s0
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
-; CHECK-NEXT:    s_cbranch_execnz .LBB5_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_dual_mov_b32 v1, s6 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT:    buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT:    s_mov_b32 s0, 0
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX11-NEXT:  .LBB5_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT:    s_or_b32 s0, s1, s0
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_nonatomic_buffer_load_i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:    buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s0, 0
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT:  .LBB5_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    s_and_b32 s1, exec_lo, vcc_lo
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    s_or_b32 s0, s1, s0
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -193,26 +331,49 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-LABEL: struct_ptr_atomic_buffer_load_i64:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v2, s6
-; CHECK-NEXT:  .LBB6_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB6_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v2, s6
+; GFX11-NEXT:  .LBB6_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB6_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_atomic_buffer_load_i64:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    v_mov_b32_e32 v1, 0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v2, s6
+; GFX12-NEXT:  .LBB6_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB6_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %id.zext = zext i32 %id to i64
@@ -226,26 +387,48 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v2i16(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-LABEL: struct_ptr_atomic_buffer_load_v2i16:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB7_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB7_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB7_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_atomic_buffer_load_v2i16:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB7_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB7_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -259,77 +442,172 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
-; CHECK-SDAG-TRUE16:       ; %bb.0: ; %bb
-; CHECK-SDAG-TRUE16-NEXT:    s_clause 0x1
-; CHECK-SDAG-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
-; CHECK-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-SDAG-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-SDAG-TRUE16-NEXT:  .LBB8_1: ; %bb1
-; CHECK-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-SDAG-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; CHECK-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; CHECK-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
-; CHECK-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
-; CHECK-SDAG-TRUE16-NEXT:    s_endpgm
+; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-SDAG-TRUE16:       ; %bb.0: ; %bb
+; GFX11-SDAG-TRUE16-NEXT:    s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-SDAG-TRUE16-NEXT:  .LBB8_1: ; %bb1
+; GFX11-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-SDAG-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-SDAG-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-FAKE16:       ; %bb.0: ; %bb
+; GFX11-FAKE16-NEXT:    s_clause 0x1
+; GFX11-FAKE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-FAKE16-NEXT:  .LBB8_1: ; %bb1
+; GFX11-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-FAKE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-FAKE16-NEXT:    s_endpgm
+;
+; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL-TRUE16:       ; %bb.0: ; %bb
+; GFX11-GISEL-TRUE16-NEXT:    s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-GISEL-TRUE16-NEXT:  .LBB8_1: ; %bb1
+; GFX11-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v3.l
+; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX11-GISEL-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX11-GISEL:       ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT:    s_clause 0x1
+; GFX11-GISEL-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-GISEL-NEXT:  .LBB8_1: ; %bb1
+; GFX11-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-GISEL-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX11-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX11-GISEL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-GISEL-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX11-GISEL-NEXT:  ; %bb.2: ; %bb2
+; GFX11-GISEL-NEXT:    s_endpgm
 ;
-; CHECK-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
-; CHECK-FAKE16:       ; %bb.0: ; %bb
-; CHECK-FAKE16-NEXT:    s_clause 0x1
-; CHECK-FAKE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-FAKE16-NEXT:    s_mov_b32 s4, 0
-; CHECK-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-FAKE16-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-FAKE16-NEXT:  .LBB8_1: ; %bb1
-; CHECK-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-FAKE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-FAKE16-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; CHECK-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; CHECK-FAKE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
-; CHECK-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-FAKE16-NEXT:    s_cbranch_execnz .LBB8_1
-; CHECK-FAKE16-NEXT:  ; %bb.2: ; %bb2
-; CHECK-FAKE16-NEXT:    s_endpgm
+; GFX12-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-SDAG-TRUE16:       ; %bb.0: ; %bb
+; GFX12-SDAG-TRUE16-NEXT:    s_clause 0x1
+; GFX12-SDAG-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-SDAG-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT:  .LBB8_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-SDAG-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-SDAG-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-SDAG-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-SDAG-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX12-SDAG-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-SDAG-TRUE16-NEXT:    s_endpgm
 ;
-; CHECK-GISEL-LABEL: struct_ptr_atomic_buffer_load_v4i16:
-; CHECK-GISEL:       ; %bb.0: ; %bb
-; CHECK-GISEL-NEXT:    s_clause 0x1
-; CHECK-GISEL-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-GISEL-NEXT:    s_mov_b32 s4, 0
-; CHECK-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-GISEL-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-GISEL-NEXT:  .LBB8_1: ; %bb1
-; CHECK-GISEL-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-GISEL-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-GISEL-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s5, v2
-; CHECK-GISEL-NEXT:    v_readfirstlane_b32 s6, v3
-; CHECK-GISEL-NEXT:    s_pack_ll_b32_b16 s5, s5, s6
-; CHECK-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; CHECK-GISEL-NEXT:    v_cmp_ne_u32_e32 vcc_lo, s5, v0
-; CHECK-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-GISEL-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-GISEL-NEXT:    s_cbranch_execnz .LBB8_1
-; CHECK-GISEL-NEXT:  ; %bb.2: ; %bb2
-; CHECK-GISEL-NEXT:    s_endpgm
+; GFX12-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-FAKE16:       ; %bb.0: ; %bb
+; GFX12-FAKE16-NEXT:    s_clause 0x1
+; GFX12-FAKE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-FAKE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-FAKE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-FAKE16-NEXT:  .LBB8_1: ; %bb1
+; GFX12-FAKE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-FAKE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT:    v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-FAKE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-FAKE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX12-FAKE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-FAKE16-NEXT:    s_endpgm
+;
+; GFX12-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
+; GFX12-GISEL-TRUE16:       ; %bb.0: ; %bb
+; GFX12-GISEL-TRUE16-NEXT:    s_clause 0x1
+; GFX12-GISEL-TRUE16-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-GISEL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_xcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    s_mov_b32 s4, 0
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT:  .LBB8_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT:    s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v2.h, v3.l
+; GFX12-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-TRUE16-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-GISEL-TRUE16-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-GISEL-TRUE16-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-GISEL-TRUE16-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX12-GISEL-TRUE16-NEXT:  ; %bb.2: ; %bb2
+; GFX12-GISEL-TRUE16-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -344,26 +622,48 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i32(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-LABEL: struct_ptr_atomic_buffer_load_v4i32:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB9_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v5, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB9_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_atomic_buffer_load_v4i32:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB9_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_atomic_buffer_load_v4i32:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB9_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
@@ -377,28 +677,52 @@ bb2:
 }
 
 define amdgpu_kernel void @struct_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %ptr, i32 %index) {
-; CHECK-LABEL: struct_ptr_atomic_buffer_load_ptr:
-; CHECK:       ; %bb.0: ; %bb
-; CHECK-NEXT:    s_clause 0x1
-; CHECK-NEXT:    s_load_b32 s6, s[4:5], 0x34
-; CHECK-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_mov_b32_e32 v1, s6
-; CHECK-NEXT:  .LBB10_1: ; %bb1
-; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
-; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    flat_load_b32 v2, v[2:3]
-; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; CHECK-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
-; CHECK-NEXT:    s_or_b32 s4, vcc_lo, s4
-; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
-; CHECK-NEXT:    s_cbranch_execnz .LBB10_1
-; CHECK-NEXT:  ; %bb.2: ; %bb2
-; CHECK-NEXT:    s_endpgm
+; GFX11-LABEL: struct_ptr_atomic_buffer_load_ptr:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX11-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mov_b32_e32 v1, s6
+; GFX11-NEXT:  .LBB10_1: ; %bb1
+; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    flat_load_b32 v2, v[2:3]
+; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-NEXT:    s_cbranch_execnz .LBB10_1
+; GFX11-NEXT:  ; %bb.2: ; %bb2
+; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: struct_ptr_atomic_buffer_load_ptr:
+; GFX12:       ; %bb.0: ; %bb
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b32 s6, s[4:5], 0x34
+; GFX12-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT:    s_wait_xcnt 0x0
+; GFX12-NEXT:    s_mov_b32 s4, 0
+; GFX12-NEXT:    s_wait_kmcnt 0x0
+; GFX12-NEXT:    v_mov_b32_e32 v1, s6
+; GFX12-NEXT:  .LBB10_1: ; %bb1
+; GFX12-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT:    buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT:    s_wait_loadcnt 0x0
+; GFX12-NEXT:    flat_load_b32 v2, v[2:3]
+; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT:    v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX12-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT:    s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-NEXT:    s_cbranch_execnz .LBB10_1
+; GFX12-NEXT:  ; %bb.2: ; %bb2
+; GFX12-NEXT:    s_endpgm
 bb:
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   br label %bb1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
index 746b8791c39f9..4366472c73a0e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_nortn.ll
@@ -3,6 +3,7 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
 
 define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) #0 {
 ; GFX908-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
@@ -39,6 +40,14 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX1200-NEXT:    s_wait_kmcnt 0x0
 ; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v[1:2], s[0:3], s16 idxen offen
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -75,6 +84,13 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voff
 ; GFX1200-NEXT:    s_wait_kmcnt 0x0
 ; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
 }
@@ -114,6 +130,14 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX1200-NEXT:    s_wait_kmcnt 0x0
 ; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_NT
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
 }
@@ -153,6 +177,14 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr
 ; GFX1200-NEXT:    s_wait_kmcnt 0x0
 ; GFX1200-NEXT:    buffer_atomic_pk_add_f16 v0, v[1:2], s[0:3], s16 idxen offen
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_pk_add_f16 v0, v[2:3], s[0:3], s16 idxen offen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -291,6 +323,42 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_v
 ; GFX1200-NEXT:  ; %bb.2:
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
+; GFX1250-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:  .LBB4_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1250-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1250-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX1250-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
+; GFX1250-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
+; GFX1250-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_and_b32 s0, s0, s1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v[8:9], s[4:7], s3 idxen offen
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT:    ; implicit-def: $vgpr7
+; GFX1250-NEXT:    ; implicit-def: $vgpr0
+; GFX1250-NEXT:    ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX1250-NEXT:  ; %bb.2:
+; GFX1250-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -429,6 +497,42 @@ define void @struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr
 ; GFX1200-NEXT:  ; %bb.2:
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_noret__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
+; GFX1250-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:  .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1250-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1250-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX1250-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
+; GFX1250-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
+; GFX1250-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_and_b32 s0, s0, s1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT:    buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s3 idxen offen
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT:    ; implicit-def: $vgpr7
+; GFX1250-NEXT:    ; implicit-def: $vgpr0
+; GFX1250-NEXT:    ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX1250-NEXT:  ; %bb.2:
+; GFX1250-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
index 71c63bfd69734..0191a85b33888 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fadd_rtn.ll
@@ -2,6 +2,7 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a < %s | FileCheck -check-prefix=GFX90A %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx942 < %s | FileCheck -check-prefix=GFX942 %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
 
 define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) #0 {
 ; GFX90A-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
@@ -32,6 +33,15 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
 ; GFX1200-NEXT:    s_wait_loadcnt 0x0
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret float %ret
 }
@@ -62,6 +72,14 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffs
 ; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN
 ; GFX1200-NEXT:    s_wait_loadcnt 0x0
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret float %ret
 }
@@ -95,6 +113,15 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX1200-NEXT:    buffer_atomic_add_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN
 ; GFX1200-NEXT:    s_wait_loadcnt 0x0
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret float %ret
 }
@@ -128,6 +155,15 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__
 ; GFX1200-NEXT:    buffer_atomic_pk_add_f16 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
 ; GFX1200-NEXT:    s_wait_loadcnt 0x0
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_pk_add_f16 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret <2 x half> %ret
 }
@@ -237,6 +273,43 @@ define float @struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_vo
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX1200-NEXT:    s_wait_loadcnt 0x0
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
+; GFX1250-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:  .LBB4_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1250-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1250-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX1250-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
+; GFX1250-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
+; GFX1250-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_and_b32 s0, s0, s1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_add_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT:    ; implicit-def: $vgpr7
+; GFX1250-NEXT:    ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB4_1
+; GFX1250-NEXT:  ; %bb.2:
+; GFX1250-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret float %ret
 }
@@ -346,6 +419,43 @@ define <2 x half> @struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__
 ; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
 ; GFX1200-NEXT:    s_wait_loadcnt 0x0
 ; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_v2f16_rtn__vgpr_val__vgpr_rsrc__vgpr_voffset__vgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v9, v6 :: v_dual_mov_b32 v8, v5
+; GFX1250-NEXT:    v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:  .LBB5_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1250-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1250-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX1250-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
+; GFX1250-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
+; GFX1250-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_and_b32 s0, s0, s1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_pk_add_f16 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT:    ; implicit-def: $vgpr7
+; GFX1250-NEXT:    ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB5_1
+; GFX1250-NEXT:  ; %bb.2:
+; GFX1250-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half> %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret <2 x half> %ret
 }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
index e3889ab8f5a21..d551d91e5ab12 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32.ll
@@ -4,7 +4,8 @@
 ; Not supported in gfx8 or gfx9
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
 
 define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
 ; GFX6-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
@@ -35,16 +36,25 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret float %ret
 }
@@ -78,16 +88,25 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_add_nc_u32 v5, 0x100, v2
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v[4:5], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
   ret float %ret
@@ -122,16 +141,24 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__0_voffs
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret float %ret
 }
@@ -165,16 +192,25 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret float %ret
 }
@@ -206,15 +242,23 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX11-NEXT:    buffer_atomic_max_f32 v0, v[1:2], s[0:3], s16 idxen offen
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v[2:3], s[0:3], s16 idxen offen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -246,15 +290,23 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX11-NEXT:    buffer_atomic_max_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_add_nc_u32 v5, 0x100, v2
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v[4:5], s[0:3], s16 idxen offen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
@@ -288,15 +340,22 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voff
 ; GFX11-NEXT:    buffer_atomic_max_f32 v0, v1, s[0:3], s16 idxen
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v1, s[0:3], s16 idxen
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v1, s[0:3], s16 idxen
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v1, s[0:3], s16 idxen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
 }
@@ -328,15 +387,23 @@ define void @struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX11-NEXT:    buffer_atomic_max_f32 v0, v[1:2], s[0:3], s16 idxen offen slc
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_NT
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
 }
@@ -442,36 +509,68 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_vo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_mov_b32 s2, exec_lo
-; GFX12-NEXT:  .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v2
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
-; GFX12-NEXT:    s_wait_alu 0xf1ff
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX12-NEXT:    s_and_b32 s1, vcc_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s1, s1
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
-; GFX12-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
-; GFX12-NEXT:  ; %bb.2:
-; GFX12-NEXT:    s_mov_b32 exec_lo, s2
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1200-NEXT:  .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX1200-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1200-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
+; GFX1200-NEXT:    s_and_b32 s1, vcc_lo, s1
+; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_and_saveexec_b32 s1, s1
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s1
+; GFX1200-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX1200-NEXT:  ; %bb.2:
+; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_add__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
+; GFX1250-NEXT:    v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
+; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:  .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1250-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1250-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
+; GFX1250-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[4:5]
+; GFX1250-NEXT:    s_and_b32 s1, vcc_lo, s1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_and_saveexec_b32 s1, s1
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT:    ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_xor_b32 exec_lo, exec_lo, s1
+; GFX1250-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX1250-NEXT:  ; %bb.2:
+; GFX1250-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
   ret float %ret
@@ -595,41 +694,78 @@ define float @struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_vo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_mov_b32 s2, exec_lo
-; GFX12-NEXT:  .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v2
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
-; GFX12-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX12-NEXT:    s_wait_alu 0xf1ff
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX12-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_b32 s0, s0, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
-; GFX12-NEXT:    ; implicit-def: $vgpr7
-; GFX12-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
-; GFX12-NEXT:  ; %bb.2:
-; GFX12-NEXT:    s_mov_b32 exec_lo, s2
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1200-NEXT:  .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1200-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1200-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
+; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
+; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
+; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_and_b32 s0, s0, s1
+; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_max_num_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT:    ; implicit-def: $vgpr7
+; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1200-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX1200-NEXT:  ; %bb.2:
+; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_add_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_add__vgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
+; GFX1250-NEXT:    v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
+; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:  .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1250-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1250-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX1250-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
+; GFX1250-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
+; GFX1250-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_and_b32 s0, s0, s1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_max_num_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT:    ; implicit-def: $vgpr7
+; GFX1250-NEXT:    ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX1250-NEXT:  ; %bb.2:
+; GFX1250-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmax.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
   ret float %ret
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
index f001bf97fcd9e..0096289f39acc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32.ll
@@ -4,7 +4,8 @@
 ; Not supported in gfx8 or gfx9
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX1200 %s
+; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1250 < %s | FileCheck -check-prefix=GFX1250 %s
 
 define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset(float %val, ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
 ; GFX6-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
@@ -35,16 +36,25 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret float %ret
 }
@@ -78,16 +88,25 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_add_nc_u32 v5, 0x100, v2
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v[4:5], s[0:3], s16 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
   ret float %ret
@@ -122,16 +141,24 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__0_voff
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v1, s[0:3], s16 idxen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret float %ret
 }
@@ -165,16 +192,25 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_NT_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret float %ret
 }
@@ -206,15 +242,23 @@ define void @struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_
 ; GFX11-NEXT:    buffer_atomic_min_f32 v0, v[1:2], s[0:3], s16 idxen offen
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v[2:3], s[0:3], s16 idxen offen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
   ret void
 }
@@ -246,15 +290,23 @@ define void @struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_
 ; GFX11-NEXT:    buffer_atomic_min_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen offset:256
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_add_nc_u32 v5, 0x100, v2
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v[4:5], s[0:3], s16 idxen offen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
   ret void
@@ -288,15 +340,22 @@ define void @struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__0_vof
 ; GFX11-NEXT:    buffer_atomic_min_f32 v0, v1, s[0:3], s16 idxen
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v1, s[0:3], s16 idxen
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v1, s[0:3], s16 idxen
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__0_voffset__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v1, s[0:3], s16 idxen
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 0, i32 %soffset, i32 0)
   ret void
 }
@@ -328,15 +387,23 @@ define void @struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_
 ; GFX11-NEXT:    buffer_atomic_min_f32 v0, v[1:2], s[0:3], s16 idxen offen slc
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v[1:2], s[0:3], s16 idxen offen th:TH_ATOMIC_NT
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_noret__vgpr_val__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v[2:3], s[0:3], s16 idxen offen th:TH_ATOMIC_NT
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 2)
   ret void
 }
@@ -442,36 +509,68 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_v
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_mov_b32 s2, exec_lo
-; GFX12-NEXT:  .LBB8_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v2
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
-; GFX12-NEXT:    s_wait_alu 0xf1ff
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
-; GFX12-NEXT:    s_and_b32 s1, vcc_lo, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s1, s1
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
-; GFX12-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT:    s_cbranch_execnz .LBB8_1
-; GFX12-NEXT:  ; %bb.2:
-; GFX12-NEXT:    s_mov_b32 exec_lo, s2
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1200-NEXT:  .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX1200-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1200-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[3:4]
+; GFX1200-NEXT:    s_and_b32 s1, vcc_lo, s1
+; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_and_saveexec_b32 s1, s1
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s0 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s1
+; GFX1200-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX1200-NEXT:  ; %bb.2:
+; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__vgpr_rsrc__vgpr_voffset_fmin__sgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
+; GFX1250-NEXT:    v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
+; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:  .LBB8_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1250-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1250-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
+; GFX1250-NEXT:    v_cmp_eq_u64_e64 s1, s[6:7], v[4:5]
+; GFX1250-NEXT:    s_and_b32 s1, vcc_lo, s1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_and_saveexec_b32 s1, s1
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s0 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT:    ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_xor_b32 exec_lo, exec_lo, s1
+; GFX1250-NEXT:    s_cbranch_execnz .LBB8_1
+; GFX1250-NEXT:  ; %bb.2:
+; GFX1250-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
   ret float %ret
@@ -595,41 +694,78 @@ define float @struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_v
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX12-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__vgpr_soffset:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT:    s_wait_expcnt 0x0
-; GFX12-NEXT:    s_wait_samplecnt 0x0
-; GFX12-NEXT:    s_wait_bvhcnt 0x0
-; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_mov_b32 s2, exec_lo
-; GFX12-NEXT:  .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT:    v_readfirstlane_b32 s4, v1
-; GFX12-NEXT:    v_readfirstlane_b32 s5, v2
-; GFX12-NEXT:    v_readfirstlane_b32 s6, v3
-; GFX12-NEXT:    v_readfirstlane_b32 s7, v4
-; GFX12-NEXT:    v_readfirstlane_b32 s3, v7
-; GFX12-NEXT:    s_wait_alu 0xf1ff
-; GFX12-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
-; GFX12-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
-; GFX12-NEXT:    s_and_b32 s0, vcc_lo, s0
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_b32 s0, s0, s1
-; GFX12-NEXT:    s_wait_alu 0xfffe
-; GFX12-NEXT:    s_and_saveexec_b32 s0, s0
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 th:TH_ATOMIC_RETURN
-; GFX12-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
-; GFX12-NEXT:    ; implicit-def: $vgpr7
-; GFX12-NEXT:    ; implicit-def: $vgpr5_vgpr6
-; GFX12-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT:    s_cbranch_execnz .LBB9_1
-; GFX12-NEXT:  ; %bb.2:
-; GFX12-NEXT:    s_mov_b32 exec_lo, s2
-; GFX12-NEXT:    s_wait_loadcnt 0x0
-; GFX12-NEXT:    s_setpc_b64 s[30:31]
+; GFX1200-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__vgpr_soffset:
+; GFX1200:       ; %bb.0:
+; GFX1200-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1200-NEXT:    s_wait_expcnt 0x0
+; GFX1200-NEXT:    s_wait_samplecnt 0x0
+; GFX1200-NEXT:    s_wait_bvhcnt 0x0
+; GFX1200-NEXT:    s_wait_kmcnt 0x0
+; GFX1200-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1200-NEXT:  .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1200-NEXT:    v_readfirstlane_b32 s4, v1
+; GFX1200-NEXT:    v_readfirstlane_b32 s5, v2
+; GFX1200-NEXT:    v_readfirstlane_b32 s6, v3
+; GFX1200-NEXT:    v_readfirstlane_b32 s7, v4
+; GFX1200-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1200-NEXT:    s_wait_alu 0xf1ff
+; GFX1200-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[1:2]
+; GFX1200-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[3:4]
+; GFX1200-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
+; GFX1200-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_and_b32 s0, s0, s1
+; GFX1200-NEXT:    s_wait_alu 0xfffe
+; GFX1200-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    buffer_atomic_min_num_f32 v0, v[5:6], s[4:7], s3 idxen offen offset:256 th:TH_ATOMIC_RETURN
+; GFX1200-NEXT:    ; implicit-def: $vgpr1_vgpr2_vgpr3_vgpr4
+; GFX1200-NEXT:    ; implicit-def: $vgpr7
+; GFX1200-NEXT:    ; implicit-def: $vgpr5_vgpr6
+; GFX1200-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1200-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX1200-NEXT:  ; %bb.2:
+; GFX1200-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1200-NEXT:    s_wait_loadcnt 0x0
+; GFX1200-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: struct_ptr_buffer_atomic_fmin_f32_ret__vgpr_val__sgpr_rsrc__vgpr_voffset_fmin__vgpr_soffset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v8, v5 :: v_dual_mov_b32 v5, v4
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
+; GFX1250-NEXT:    v_dual_mov_b32 v2, v1 :: v_dual_add_nc_u32 v9, 0x100, v6
+; GFX1250-NEXT:    s_mov_b32 s2, exec_lo
+; GFX1250-NEXT:  .LBB9_1: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_readfirstlane_b32 s4, v2
+; GFX1250-NEXT:    v_readfirstlane_b32 s5, v3
+; GFX1250-NEXT:    v_readfirstlane_b32 s6, v4
+; GFX1250-NEXT:    v_readfirstlane_b32 s7, v5
+; GFX1250-NEXT:    v_readfirstlane_b32 s3, v7
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT:    v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[2:3]
+; GFX1250-NEXT:    v_cmp_eq_u64_e64 s0, s[6:7], v[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_cmp_eq_u32_e64 s1, s3, v7
+; GFX1250-NEXT:    s_and_b32 s0, vcc_lo, s0
+; GFX1250-NEXT:    s_and_b32 s0, s0, s1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_and_saveexec_b32 s0, s0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    buffer_atomic_min_num_f32 v0, v[8:9], s[4:7], s3 idxen offen th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    ; implicit-def: $vgpr2_vgpr3_vgpr4_vgpr5
+; GFX1250-NEXT:    ; implicit-def: $vgpr7
+; GFX1250-NEXT:    ; implicit-def: $vgpr8_vgpr9
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_xor_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT:    s_cbranch_execnz .LBB9_1
+; GFX1250-NEXT:  ; %bb.2:
+; GFX1250-NEXT:    s_mov_b32 exec_lo, s2
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %voffset.add = add i32 %voffset, 256
   %ret = call float @llvm.amdgcn.struct.ptr.buffer.atomic.fmin.f32(float %val, ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset.add, i32 %soffset, i32 0)
   ret float %ret
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll
index f668a116f3c83..cc38d0732399b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.wave.id.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: not --crash llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -global-isel=0 < %s 2>&1 | FileCheck -check-prefix=GFX9-SDAG-ERR %s
-; RUN: not llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -global-isel=1 < %s 2>&1 | FileCheck -check-prefix=GFX9-GISEL-ERR %s
+; RUN: not llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -global-isel=1 -new-reg-bank-select < %s 2>&1 | FileCheck -check-prefix=GFX9-GISEL-ERR %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -mattr=+architected-sgprs -global-isel=0 < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -mattr=+architected-sgprs -global-isel=1 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 -mattr=+architected-sgprs -global-isel=1 -new-reg-bank-select < %s | FileCheck -check-prefix=GFX9 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -global-isel=0 < %s | FileCheck -check-prefix=GFX12 %s
-; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -global-isel=1 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 -global-isel=1 -new-reg-bank-select < %s | FileCheck -check-prefix=GFX12 %s
 
 ; GFX9-SDAG-ERR: LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.wave.id
 ; GFX9-GISEL-ERR: LLVM ERROR: unable to legalize instruction: {{.*}} = G_INTRINSIC intrinsic(@llvm.amdgcn.wave.id)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
index af914bd4043cf..355f77acfd302 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
@@ -76,12 +76,13 @@ define amdgpu_kernel void @v_round_f64(ptr addrspace(1) %out, ptr addrspace(1) %
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
 ; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT:    s_movk_i32 s4, 0xfc01
 ; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_mov_b32 s3, 0xfffff
 ; SI-NEXT:    v_mov_b32_e32 v8, 0x3ff00000
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_bfe_u32 v4, v3, 20, 11
-; SI-NEXT:    v_add_i32_e32 v6, vcc, 0xfffffc01, v4
+; SI-NEXT:    v_add_i32_e32 v6, vcc, s4, v4
 ; SI-NEXT:    v_lshr_b64 v[4:5], s[2:3], v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0x80000000, v3
 ; SI-NEXT:    v_not_b32_e32 v5, v5
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sin.ll b/llvm/test/CodeGen/AMDGPU/llvm.sin.ll
index 2366e39e2eedd..66b01a52da9a6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sin.ll
@@ -16,7 +16,7 @@
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @sin_f32(ptr addrspace(1) %out, float %x) #1 {
+define amdgpu_kernel void @sin_f32(ptr addrspace(1) %out, float %x) {
    %sin = call float @llvm.sin.f32(float %x)
    store float %sin, ptr addrspace(1) %out
    ret void
@@ -29,7 +29,7 @@ define amdgpu_kernel void @sin_f32(ptr addrspace(1) %out, float %x) #1 {
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @safe_sin_3x_f32(ptr addrspace(1) %out, float %x) #1 {
+define amdgpu_kernel void @safe_sin_3x_f32(ptr addrspace(1) %out, float %x) {
   %y = fmul float 3.0, %x
   %sin = call float @llvm.sin.f32(float %y)
   store float %sin, ptr addrspace(1) %out
@@ -44,9 +44,9 @@ define amdgpu_kernel void @safe_sin_3x_f32(ptr addrspace(1) %out, float %x) #1 {
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @unsafe_sin_3x_f32(ptr addrspace(1) %out, float %x) #2 {
-  %y = fmul float 3.0, %x
-  %sin = call float @llvm.sin.f32(float %y)
+define amdgpu_kernel void @unsafe_sin_3x_f32(ptr addrspace(1) %out, float %x) {
+  %y = fmul reassoc float 3.0, %x
+  %sin = call reassoc float @llvm.sin.f32(float %y)
   store float %sin, ptr addrspace(1) %out
   ret void
 }
@@ -59,7 +59,7 @@ define amdgpu_kernel void @unsafe_sin_3x_f32(ptr addrspace(1) %out, float %x) #2
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @fmf_sin_3x_f32(ptr addrspace(1) %out, float %x) #1 {
+define amdgpu_kernel void @fmf_sin_3x_f32(ptr addrspace(1) %out, float %x) {
   %y = fmul reassoc float 3.0, %x
   %sin = call reassoc float @llvm.sin.f32(float %y)
   store float %sin, ptr addrspace(1) %out
@@ -73,7 +73,7 @@ define amdgpu_kernel void @fmf_sin_3x_f32(ptr addrspace(1) %out, float %x) #1 {
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @safe_sin_2x_f32(ptr addrspace(1) %out, float %x) #1 {
+define amdgpu_kernel void @safe_sin_2x_f32(ptr addrspace(1) %out, float %x) {
   %y = fmul float 2.0, %x
   %sin = call float @llvm.sin.f32(float %y)
   store float %sin, ptr addrspace(1) %out
@@ -88,9 +88,9 @@ define amdgpu_kernel void @safe_sin_2x_f32(ptr addrspace(1) %out, float %x) #1 {
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @unsafe_sin_2x_f32(ptr addrspace(1) %out, float %x) #2 {
-  %y = fmul float 2.0, %x
-  %sin = call float @llvm.sin.f32(float %y)
+define amdgpu_kernel void @unsafe_sin_2x_f32(ptr addrspace(1) %out, float %x) {
+  %y = fmul reassoc float 2.0, %x
+  %sin = call reassoc float @llvm.sin.f32(float %y)
   store float %sin, ptr addrspace(1) %out
   ret void
 }
@@ -103,7 +103,7 @@ define amdgpu_kernel void @unsafe_sin_2x_f32(ptr addrspace(1) %out, float %x) #2
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @fmf_sin_2x_f32(ptr addrspace(1) %out, float %x) #1 {
+define amdgpu_kernel void @fmf_sin_2x_f32(ptr addrspace(1) %out, float %x) {
   %y = fmul reassoc float 2.0, %x
   %sin = call reassoc float @llvm.sin.f32(float %y)
   store float %sin, ptr addrspace(1) %out
@@ -117,7 +117,7 @@ define amdgpu_kernel void @fmf_sin_2x_f32(ptr addrspace(1) %out, float %x) #1 {
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @safe_sin_cancel_f32(ptr addrspace(1) %out, float %x) #1 {
+define amdgpu_kernel void @safe_sin_cancel_f32(ptr addrspace(1) %out, float %x) {
   %y = fmul float 0x401921FB60000000, %x
   %sin = call float @llvm.sin.f32(float %y)
   store float %sin, ptr addrspace(1) %out
@@ -131,9 +131,9 @@ define amdgpu_kernel void @safe_sin_cancel_f32(ptr addrspace(1) %out, float %x)
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @unsafe_sin_cancel_f32(ptr addrspace(1) %out, float %x) #2 {
-  %y = fmul float 0x401921FB60000000, %x
-  %sin = call float @llvm.sin.f32(float %y)
+define amdgpu_kernel void @unsafe_sin_cancel_f32(ptr addrspace(1) %out, float %x) {
+  %y = fmul reassoc float 0x401921FB60000000, %x
+  %sin = call reassoc float @llvm.sin.f32(float %y)
   store float %sin, ptr addrspace(1) %out
   ret void
 }
@@ -145,7 +145,7 @@ define amdgpu_kernel void @unsafe_sin_cancel_f32(ptr addrspace(1) %out, float %x
 ; GFX9-NOT: v_fract_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @fmf_sin_cancel_f32(ptr addrspace(1) %out, float %x) #1 {
+define amdgpu_kernel void @fmf_sin_cancel_f32(ptr addrspace(1) %out, float %x) {
   %y = fmul reassoc float 0x401921FB60000000, %x
   %sin = call reassoc float @llvm.sin.f32(float %y)
   store float %sin, ptr addrspace(1) %out
@@ -164,7 +164,7 @@ define amdgpu_kernel void @fmf_sin_cancel_f32(ptr addrspace(1) %out, float %x) #
 ; GCN: v_sin_f32
 ; GCN: v_sin_f32
 ; GCN-NOT: v_sin_f32
-define amdgpu_kernel void @sin_v4f32(ptr addrspace(1) %out, <4 x float> %vx) #1 {
+define amdgpu_kernel void @sin_v4f32(ptr addrspace(1) %out, <4 x float> %vx) {
    %sin = call <4 x float> @llvm.sin.v4f32( <4 x float> %vx)
    store <4 x float> %sin, ptr addrspace(1) %out
    ret void
@@ -174,5 +174,3 @@ declare float @llvm.sin.f32(float) #0
 declare <4 x float> @llvm.sin.v4f32(<4 x float>) #0
 
 attributes #0 = { nounwind readnone }
-attributes #1 = { nounwind "unsafe-fp-math"="false" }
-attributes #2 = { nounwind "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll b/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll
index 91a8446915f66..13ea8b08d2ade 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-always-uniform.ll
@@ -18,10 +18,9 @@ define amdgpu_cs void @test_uniform_load_b96(ptr addrspace(1) %ptr, i32 %arg) "a
 ; GFX11-NEXT:    s_load_b64 s[2:3], s[0:1], 0x0
 ; GFX11-NEXT:    s_load_b32 s0, s[0:1], 0x8
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_or_b32 s1, s2, s3
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_or_b32 s0, s0, s1
-; GFX11-NEXT:    v_mov_b32_e32 v2, s0
+; GFX11-NEXT:    v_mov_b32_e32 v2, s3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_or3_b32 v2, s2, v2, s0
 ; GFX11-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX11-NEXT:    s_endpgm
 ;
@@ -34,14 +33,12 @@ define amdgpu_cs void @test_uniform_load_b96(ptr addrspace(1) %ptr, i32 %arg) "a
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v3, null, v1, v3, vcc_lo
 ; GFX12-NEXT:    v_readfirstlane_b32 s0, v2
-; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
 ; GFX12-NEXT:    v_readfirstlane_b32 s1, v3
 ; GFX12-NEXT:    s_load_b96 s[0:2], s[0:1], 0x0
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    s_or_b32 s0, s0, s1
-; GFX12-NEXT:    s_or_b32 s0, s2, s0
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX12-NEXT:    v_mov_b32_e32 v2, s0
+; GFX12-NEXT:    v_or3_b32 v2, v2, s1, s2
 ; GFX12-NEXT:    global_store_b32 v[0:1], v2, off
 ; GFX12-NEXT:    s_endpgm
 bb:
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
index 0a938b0d2297d..8862cbe6391ea 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
@@ -872,66 +872,66 @@ define amdgpu_kernel void @constant_load_v11i32(ptr addrspace(1) %out, ptr addrs
 ; GFX7-HSA-NEXT:    s_mov_b32 flat_scratch_lo, s13
 ; GFX7-HSA-NEXT:    s_lshr_b32 flat_scratch_hi, s12, 8
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-HSA-NEXT:    s_load_dwordx4 s[12:15], s[10:11], 0x8
 ; GFX7-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
+; GFX7-HSA-NEXT:    s_load_dwordx4 s[12:15], s[10:11], 0x8
 ; GFX7-HSA-NEXT:    s_add_u32 s10, s8, 16
 ; GFX7-HSA-NEXT:    s_addc_u32 s11, s9, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v7, s10
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v8, s11
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v5, s10
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v6, s11
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s6
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, s7
-; GFX7-HSA-NEXT:    flat_store_dwordx4 v[7:8], v[0:3]
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v7, s8
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s2
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, s3
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v8, s9
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GFX7-HSA-NEXT:    flat_store_dwordx4 v[5:6], v[0:3]
 ; GFX7-HSA-NEXT:    s_add_u32 s0, s8, 32
-; GFX7-HSA-NEXT:    flat_store_dwordx4 v[7:8], v[0:3]
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s8
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v6, s2
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v7, s3
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX7-HSA-NEXT:    s_addc_u32 s1, s9, 0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s12
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v5, s13
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v6, s14
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-HSA-NEXT:    flat_store_dwordx3 v[0:1], v[4:6]
+; GFX7-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s12
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s1
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s13
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s14
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, s0
+; GFX7-HSA-NEXT:    flat_store_dwordx3 v[3:4], v[0:2]
 ; GFX7-HSA-NEXT:    s_endpgm
 ;
 ; GFX8-NOHSA-LABEL: constant_load_v11i32:
 ; GFX8-NOHSA:       ; %bb.0: ; %entry
 ; GFX8-NOHSA-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX8-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NOHSA-NEXT:    s_load_dwordx4 s[12:15], s[2:3], 0x20
 ; GFX8-NOHSA-NEXT:    s_load_dwordx8 s[4:11], s[2:3], 0x0
+; GFX8-NOHSA-NEXT:    s_load_dwordx4 s[12:15], s[2:3], 0x20
 ; GFX8-NOHSA-NEXT:    s_add_u32 s2, s0, 16
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s3, s1, 0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s3
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v7, s2
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v6, s3
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v5, s2
 ; GFX8-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v2, s10
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v3, s11
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[7:8], v[0:3]
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s1
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v1, s5
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v2, s6
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v3, s7
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v7, s0
-; GFX8-NOHSA-NEXT:    s_add_u32 s0, s0, 32
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[7:8], v[0:3]
-; GFX8-NOHSA-NEXT:    s_addc_u32 s1, s1, 0
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[5:6], v[0:3]
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v4, s4
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v4, s12
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v5, s13
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v6, s14
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NOHSA-NEXT:    flat_store_dwordx3 v[0:1], v[4:6]
+; GFX8-NOHSA-NEXT:    s_add_u32 s0, s0, 32
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v6, s6
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v7, s7
+; GFX8-NOHSA-NEXT:    s_addc_u32 s1, s1, 0
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v0, s12
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v4, s1
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v1, s13
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v2, s14
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v3, s0
+; GFX8-NOHSA-NEXT:    flat_store_dwordx3 v[3:4], v[0:2]
 ; GFX8-NOHSA-NEXT:    s_endpgm
 ;
 ; EG-LABEL: constant_load_v11i32:
@@ -969,25 +969,25 @@ define amdgpu_kernel void @constant_load_v11i32(ptr addrspace(1) %out, ptr addrs
 ; GFX9-HSA-LABEL: constant_load_v11i32:
 ; GFX9-HSA:       ; %bb.0: ; %entry
 ; GFX9-HSA-NEXT:    s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v7, 0
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX9-HSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-HSA-NEXT:    s_load_dwordx4 s[12:15], s[10:11], 0x20
 ; GFX9-HSA-NEXT:    s_load_dwordx8 s[0:7], s[10:11], 0x0
+; GFX9-HSA-NEXT:    s_load_dwordx4 s[12:15], s[10:11], 0x20
 ; GFX9-HSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v4, s12
 ; GFX9-HSA-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX9-HSA-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX9-HSA-NEXT:    v_mov_b32_e32 v2, s6
 ; GFX9-HSA-NEXT:    v_mov_b32_e32 v3, s7
-; GFX9-HSA-NEXT:    global_store_dwordx4 v7, v[0:3], s[8:9] offset:16
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v5, s13
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v2, s2
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v6, s14
-; GFX9-HSA-NEXT:    global_store_dwordx4 v7, v[0:3], s[8:9]
-; GFX9-HSA-NEXT:    global_store_dwordx3 v7, v[4:6], s[8:9] offset:32
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v4, s0
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v6, s2
+; GFX9-HSA-NEXT:    global_store_dwordx4 v8, v[0:3], s[8:9] offset:16
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v7, s3
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v0, s12
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v1, s13
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v2, s14
+; GFX9-HSA-NEXT:    global_store_dwordx4 v8, v[4:7], s[8:9]
+; GFX9-HSA-NEXT:    global_store_dwordx3 v8, v[0:2], s[8:9] offset:32
 ; GFX9-HSA-NEXT:    s_endpgm
 ;
 ; GFX12-LABEL: constant_load_v11i32:
@@ -995,19 +995,19 @@ define amdgpu_kernel void @constant_load_v11i32(ptr addrspace(1) %out, ptr addrs
 ; GFX12-NEXT:    s_load_b128 s[8:11], s[4:5], 0x24
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    s_clause 0x1
-; GFX12-NEXT:    s_load_b96 s[12:14], s[10:11], 0x20
 ; GFX12-NEXT:    s_load_b256 s[0:7], s[10:11], 0x0
+; GFX12-NEXT:    s_load_b96 s[12:14], s[10:11], 0x20
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
-; GFX12-NEXT:    v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v8, s12
+; GFX12-NEXT:    v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v0, s4
+; GFX12-NEXT:    v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s6
+; GFX12-NEXT:    v_dual_mov_b32 v3, s7 :: v_dual_mov_b32 v4, s0
+; GFX12-NEXT:    v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v6, s2
+; GFX12-NEXT:    v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v8, s12
 ; GFX12-NEXT:    v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v10, s14
-; GFX12-NEXT:    v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
-; GFX12-NEXT:    v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
-; GFX12-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s1
-; GFX12-NEXT:    v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v7, s3
 ; GFX12-NEXT:    s_clause 0x2
-; GFX12-NEXT:    global_store_b96 v11, v[8:10], s[8:9] offset:32
 ; GFX12-NEXT:    global_store_b128 v11, v[0:3], s[8:9] offset:16
 ; GFX12-NEXT:    global_store_b128 v11, v[4:7], s[8:9]
+; GFX12-NEXT:    global_store_b96 v11, v[8:10], s[8:9] offset:32
 ; GFX12-NEXT:    s_endpgm
 entry:
   %ld = load <11 x i32>, ptr addrspace(4) %in
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index a42c71c4849bd..c1a32aafbc71e 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -1259,13 +1259,12 @@ define half @local_atomic_fadd_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v0, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, 4.0, v3.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v0, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -1371,13 +1370,12 @@ define half @local_atomic_fadd_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v0, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, 4.0, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v0, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -1646,13 +1644,12 @@ define half @local_atomic_fadd_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v1, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, 4.0, v3.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v1, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1763,13 +1760,12 @@ define half @local_atomic_fadd_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v1, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, 4.0, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v1, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -2044,13 +2040,12 @@ define void @local_atomic_fadd_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v0, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v4.l, 4.0, v4.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v0, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v4, v2, v3, v4
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -2153,13 +2148,12 @@ define void @local_atomic_fadd_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v0, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v4.l, 4.0, v4.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v4, v2, v3, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -2419,11 +2413,11 @@ define void @local_atomic_fadd_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v4.l, 4.0, v4.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v1, v4
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v4, v3, v2, v4
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2531,11 +2525,11 @@ define void @local_atomic_fadd_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v4.l, 4.0, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v1, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v4, v3, v2, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2795,10 +2789,9 @@ define half @local_atomic_fadd_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v2, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v1.l, 4.0, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, 0xffff0000, v2, v1
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2882,10 +2875,9 @@ define half @local_atomic_fadd_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v1.l, 4.0, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, 0xffff0000, v2, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -3095,8 +3087,8 @@ define void @local_atomic_fadd_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v2.l, 4.0, v1.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v2, 0xffff0000, v1, v2
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -3177,8 +3169,8 @@ define void @local_atomic_fadd_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v2.l, 4.0, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v2, 0xffff0000, v1, v2
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index 8351d28057564..739e86d1928b1 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -803,14 +803,14 @@ define half @local_atomic_fmax_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v0, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v3.l, v3.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, 4.0, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v0, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -918,14 +918,14 @@ define half @local_atomic_fmax_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v0, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v3.l, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, 4.0, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v0, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -1199,14 +1199,14 @@ define half @local_atomic_fmax_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v1, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v3.l, v3.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, 4.0, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v1, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1319,14 +1319,14 @@ define half @local_atomic_fmax_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v1, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v3.l, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, 4.0, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v1, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1606,14 +1606,14 @@ define void @local_atomic_fmax_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v0, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v4.l, v4.l, v4.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v4.l, 4.0, v4.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v0, v4
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v4, v2, v3, v4
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -1718,14 +1718,14 @@ define void @local_atomic_fmax_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v0, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v4.l, v4.l, v4.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v4.l, 4.0, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v0, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v4, v2, v3, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -1990,13 +1990,12 @@ define void @local_atomic_fmax_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v4.l, v4.l, v4.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v4.l, 4.0, v4.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v1, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v4, v3, v2, v4
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2107,13 +2106,12 @@ define void @local_atomic_fmax_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v4.l, v4.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v4.l, 4.0, v4.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v1, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v4, v3, v2, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2379,11 +2377,11 @@ define half @local_atomic_fmax_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v2, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v1.l, v2.l, v2.l
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v1.l, 4.0, v1.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, 0xffff0000, v2, v1
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2469,11 +2467,11 @@ define half @local_atomic_fmax_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v1.l, v2.l, v2.l
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v1.l, 4.0, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, 0xffff0000, v2, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2688,10 +2686,9 @@ define void @local_atomic_fmax_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.l, v1.l, v1.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.l, 4.0, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v2, 0xffff0000, v1, v2
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -2775,10 +2772,9 @@ define void @local_atomic_fmax_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.l, v1.l, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.l, 4.0, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v2, 0xffff0000, v1, v2
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 0c4aca88b3781..6da80262951e5 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -803,14 +803,14 @@ define half @local_atomic_fmin_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v0, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v3.l, v3.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v3.l, 4.0, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v0, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -918,14 +918,14 @@ define half @local_atomic_fmin_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v0, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v3.l, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v3.l, 4.0, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v0, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -1199,14 +1199,14 @@ define half @local_atomic_fmin_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v1, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v3.l, v3.l, v3.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v3.l, 4.0, v3.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v1, v3
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1319,14 +1319,14 @@ define half @local_atomic_fmin_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v1, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v3.l, v3.l, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v3.l, 4.0, v3.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v1, v3
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1606,14 +1606,14 @@ define void @local_atomic_fmin_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v0, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v4.l, v4.l, v4.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v4.l, 4.0, v4.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v0, v4
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v4, v2, v3, v4
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -1718,14 +1718,14 @@ define void @local_atomic_fmin_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v0, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v4.l, v4.l, v4.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v4.l, 4.0, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v0, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v4, v2, v3, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -1990,13 +1990,12 @@ define void @local_atomic_fmin_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v4.l, v4.l, v4.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v4.l, 4.0, v4.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v1, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v4, v3, v2, v4
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2107,13 +2106,12 @@ define void @local_atomic_fmin_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v4.l, v4.l, v4.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v4.l, 4.0, v4.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v1, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v4, v3, v2, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2379,11 +2377,11 @@ define half @local_atomic_fmin_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v2, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v1.l, v2.l, v2.l
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v1.l, 4.0, v1.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, 0xffff0000, v2, v1
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2469,11 +2467,11 @@ define half @local_atomic_fmin_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v1.l, v2.l, v2.l
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v1.l, 4.0, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, 0xffff0000, v2, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2688,10 +2686,9 @@ define void @local_atomic_fmin_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_max_num_f16_e32 v2.l, v1.l, v1.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_min_num_f16_e32 v2.l, 4.0, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v2, 0xffff0000, v1, v2
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -2775,10 +2772,9 @@ define void @local_atomic_fmin_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_max_f16_e32 v2.l, v1.l, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_min_f16_e32 v2.l, 4.0, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v2, 0xffff0000, v1, v2
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 37310b614c0db..786989cc9fb57 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -1721,13 +1721,12 @@ define half @local_atomic_fsub_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v0, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, -4.0, v3.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v0, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -1833,13 +1832,12 @@ define half @local_atomic_fsub_ret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v0, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, -4.0, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v0, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -2108,13 +2106,12 @@ define half @local_atomic_fsub_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v1, v4
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v3.l, -4.0, v3.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v1, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -2225,13 +2222,12 @@ define half @local_atomic_fsub_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, v1, v4
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v3.l, -4.0, v3.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v3, 0xffff, v3
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v3, v1, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v3, v4, v2, v3
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -2506,13 +2502,12 @@ define void @local_atomic_fsub_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX12-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v0, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v4.l, -4.0, v4.l
 ; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v0, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v4, v2, v3, v4
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -2615,13 +2610,12 @@ define void @local_atomic_fsub_noret_f16(ptr addrspace(3) %ptr) nounwind {
 ; GFX11-TRUE16-NEXT:  .LBB10_1: ; %atomicrmw.start
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v0, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v4.l, -4.0, v4.l
 ; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v0, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v4, v2, v3, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -2881,11 +2875,11 @@ define void @local_atomic_fsub_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v4.l, -4.0, v4.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v1, v4
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v4, v3, v2, v4
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2993,11 +2987,11 @@ define void @local_atomic_fsub_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v4.l, -4.0, v4.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b32_e32 v4, v1, v4
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v4, v3, v2, v4
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -3257,10 +3251,9 @@ define half @local_atomic_fsub_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_mov_b32_e32 v2, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v1.l, -4.0, v2.l
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v1, 0xffff0000, v2, v1
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -3344,10 +3337,9 @@ define half @local_atomic_fsub_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v2, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v1.l, -4.0, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v1, 0xffff0000, v2, v1
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -3557,8 +3549,8 @@ define void @local_atomic_fsub_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX12-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX12-TRUE16-NEXT:    s_wait_dscnt 0x0
 ; GFX12-TRUE16-NEXT:    v_add_f16_e32 v2.l, -4.0, v1.l
-; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX12-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-TRUE16-NEXT:    v_and_or_b32 v2, 0xffff0000, v1, v2
 ; GFX12-TRUE16-NEXT:    s_wait_storecnt 0x0
 ; GFX12-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -3639,8 +3631,8 @@ define void @local_atomic_fsub_noret_f16__offset__align4(ptr addrspace(3) %ptr)
 ; GFX11-TRUE16-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v2.l, -4.0, v1.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-TRUE16-NEXT:    v_and_or_b32 v2, 0xffff0000, v1, v2
 ; GFX11-TRUE16-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-TRUE16-NEXT:    ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll b/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
index a3ebaec4811a9..5f0ca7bc42ae0 100644
--- a/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
@@ -74,7 +74,8 @@ define amdgpu_kernel void @local_stack_offset_uses_sp(ptr addrspace(1) %out) {
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_cbranch_scc1 .LBB0_1
 ; FLATSCR-NEXT:  ; %bb.2: ; %split
-; FLATSCR-NEXT:    s_movk_i32 s0, 0x5000
+; FLATSCR-NEXT:    s_movk_i32 s0, 0x2000
+; FLATSCR-NEXT:    s_addk_i32 s0, 0x3000
 ; FLATSCR-NEXT:    scratch_load_dwordx2 v[0:1], off, s0 offset:208 glc
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_movk_i32 s0, 0x3000
@@ -175,7 +176,9 @@ define void @func_local_stack_offset_uses_sp(ptr addrspace(1) %out) {
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_cbranch_scc1 .LBB1_1
 ; FLATSCR-NEXT:  ; %bb.2: ; %split
-; FLATSCR-NEXT:    s_add_i32 s0, s33, 0x5000
+; FLATSCR-NEXT:    s_movk_i32 s0, 0x2000
+; FLATSCR-NEXT:    s_add_i32 s1, s33, s0
+; FLATSCR-NEXT:    s_add_i32 s0, s1, 0x3000
 ; FLATSCR-NEXT:    scratch_load_dwordx2 v[2:3], off, s0 offset:208 glc
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_add_i32 s0, s33, 0x3000
@@ -223,30 +226,35 @@ define amdgpu_kernel void @local_stack_offset_uses_sp_flat(ptr addrspace(1) %out
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    s_cbranch_scc1 .LBB2_1
 ; MUBUF-NEXT:  ; %bb.2: ; %split
+; MUBUF-NEXT:    s_movk_i32 s5, 0x12d4
 ; MUBUF-NEXT:    v_mov_b32_e32 v1, 0x4000
-; MUBUF-NEXT:    v_or_b32_e32 v0, 0x12d4, v1
+; MUBUF-NEXT:    v_or_b32_e32 v0, s5, v1
+; MUBUF-NEXT:    s_movk_i32 s5, 0x12d0
 ; MUBUF-NEXT:    v_mov_b32_e32 v1, 0x4000
 ; MUBUF-NEXT:    s_movk_i32 s4, 0x4000
 ; MUBUF-NEXT:    buffer_load_dword v5, v0, s[0:3], 0 offen glc
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    v_or_b32_e32 v0, 0x12d0, v1
+; MUBUF-NEXT:    v_or_b32_e32 v0, s5, v1
+; MUBUF-NEXT:    s_movk_i32 s5, 0x12c4
 ; MUBUF-NEXT:    v_mov_b32_e32 v1, 0x4000
 ; MUBUF-NEXT:    s_or_b32 s4, s4, 0x12c0
 ; MUBUF-NEXT:    buffer_load_dword v4, v0, s[0:3], 0 offen glc
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    v_or_b32_e32 v0, 0x12c4, v1
-; MUBUF-NEXT:    v_mov_b32_e32 v3, 0x4000
+; MUBUF-NEXT:    v_or_b32_e32 v0, s5, v1
 ; MUBUF-NEXT:    buffer_load_dword v1, v0, s[0:3], 0 offen glc
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    v_mov_b32_e32 v0, s4
-; MUBUF-NEXT:    v_or_b32_e32 v2, 0x12cc, v3
+; MUBUF-NEXT:    s_movk_i32 s4, 0x12cc
+; MUBUF-NEXT:    v_mov_b32_e32 v3, 0x4000
+; MUBUF-NEXT:    v_or_b32_e32 v2, s4, v3
+; MUBUF-NEXT:    s_movk_i32 s4, 0x12c8
 ; MUBUF-NEXT:    v_mov_b32_e32 v6, 0x4000
 ; MUBUF-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen glc
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
 ; MUBUF-NEXT:    v_mov_b32_e32 v7, 0x4000
 ; MUBUF-NEXT:    buffer_load_dword v3, v2, s[0:3], 0 offen glc
 ; MUBUF-NEXT:    s_waitcnt vmcnt(0)
-; MUBUF-NEXT:    v_or_b32_e32 v2, 0x12c8, v6
+; MUBUF-NEXT:    v_or_b32_e32 v2, s4, v6
 ; MUBUF-NEXT:    v_mov_b32_e32 v8, 0x4000
 ; MUBUF-NEXT:    v_mov_b32_e32 v9, 0x4000
 ; MUBUF-NEXT:    buffer_load_dword v2, v2, s[0:3], 0 offen glc
@@ -298,7 +306,8 @@ define amdgpu_kernel void @local_stack_offset_uses_sp_flat(ptr addrspace(1) %out
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    s_cbranch_scc1 .LBB2_1
 ; FLATSCR-NEXT:  ; %bb.2: ; %split
-; FLATSCR-NEXT:    s_movk_i32 s0, 0x3000
+; FLATSCR-NEXT:    s_movk_i32 s0, 0x1000
+; FLATSCR-NEXT:    s_addk_i32 s0, 0x2000
 ; FLATSCR-NEXT:    scratch_load_dwordx2 v[8:9], off, s0 offset:720 glc
 ; FLATSCR-NEXT:    s_waitcnt vmcnt(0)
 ; FLATSCR-NEXT:    scratch_load_dwordx4 v[0:3], off, s0 offset:704 glc
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
index a0c1e573f8fbb..a09e392b89e63 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-constants.ll
@@ -223,3 +223,43 @@ define i32 @fancy_zero() {
   ptr addrspace(7) addrspacecast (ptr addrspace(8) @buf to ptr addrspace(7))
   to i32)
 }
+
+define i32 @load_null() {
+; CHECK-LABEL: define i32 @load_null
+; CHECK-SAME: () #[[ATTR0]] {
+; CHECK-NEXT:    [[X:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0)
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %x = load i32, ptr addrspace(7) null, align 4
+  ret i32 %x
+}
+
+define void @store_null() {
+; CHECK-LABEL: define void @store_null
+; CHECK-SAME: () #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 null, i32 0, i32 0, i32 0)
+; CHECK-NEXT:    ret void
+;
+  store i32 0, ptr addrspace(7) null, align 4
+  ret void
+}
+
+define i32 @load_poison() {
+; CHECK-LABEL: define i32 @load_poison
+; CHECK-SAME: () #[[ATTR0]] {
+; CHECK-NEXT:    [[X:%.*]] = call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) align 4 poison, i32 poison, i32 0, i32 0)
+; CHECK-NEXT:    ret i32 [[X]]
+;
+  %x = load i32, ptr addrspace(7) poison, align 4
+  ret i32 %x
+}
+
+define void @store_poison() {
+; CHECK-LABEL: define void @store_poison
+; CHECK-SAME: () #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 0, ptr addrspace(8) align 4 poison, i32 poison, i32 0, i32 0)
+; CHECK-NEXT:    ret void
+;
+  store i32 0, ptr addrspace(7) poison, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll
index 63c0463149b79..66de953043f10 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-pointer-ops.ll
@@ -255,6 +255,56 @@ define i32 @ptrtoint_offset(ptr addrspace(7) %ptr) {
   ret i32 %ret
 }
 
+define i32 @ptrtoaddr(ptr addrspace(7) %ptr) {
+; CHECK-LABEL: define i32 @ptrtoaddr
+; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
+; CHECK-NEXT:    [[RET:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %ret = ptrtoaddr ptr addrspace(7) %ptr to i32
+  ret i32 %ret
+}
+
+define <2 x i32> @ptrtoaddr_vec(<2 x ptr addrspace(7)> %ptr) {
+; CHECK-LABEL: define <2 x i32> @ptrtoaddr_vec
+; CHECK-SAME: ({ <2 x ptr addrspace(8)>, <2 x i32> } [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[PTR_RSRC:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[PTR]], 0
+; CHECK-NEXT:    [[RET:%.*]] = extractvalue { <2 x ptr addrspace(8)>, <2 x i32> } [[PTR]], 1
+; CHECK-NEXT:    ret <2 x i32> [[RET]]
+;
+  %ret = ptrtoaddr <2 x ptr addrspace(7)> %ptr to <2 x i32>
+  ret <2 x i32> %ret
+}
+
+;; Check that we extend the offset to i160.
+define i160 @ptrtoaddr_ext(ptr addrspace(7) %ptr) {
+; CHECK-LABEL: define i160 @ptrtoaddr_ext
+; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
+; CHECK-NEXT:    [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
+; CHECK-NEXT:    [[RET:%.*]] = zext i32 [[PTR_OFF]] to i160
+; CHECK-NEXT:    ret i160 [[RET]]
+;
+  %addr = ptrtoaddr ptr addrspace(7) %ptr to i32
+  %ext = zext i32 %addr to i160
+  ret i160 %ext
+}
+
+;; Check that we truncate the offset to i16.
+define i16 @ptrtoaddr_trunc(ptr addrspace(7) %ptr) {
+; CHECK-LABEL: define i16 @ptrtoaddr_trunc
+; CHECK-SAME: ({ ptr addrspace(8), i32 } [[PTR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[PTR_RSRC:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 0
+; CHECK-NEXT:    [[PTR_OFF:%.*]] = extractvalue { ptr addrspace(8), i32 } [[PTR]], 1
+; CHECK-NEXT:    [[RET:%.*]] = trunc i32 [[PTR_OFF]] to i16
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %addr = ptrtoaddr ptr addrspace(7) %ptr to i32
+  %trunc = trunc i32 %addr to i16
+  ret i16 %trunc
+}
+
 define ptr addrspace(7) @inttoptr(i160 %v) {
 ; CHECK-LABEL: define { ptr addrspace(8), i32 } @inttoptr
 ; CHECK-SAME: (i160 [[V:%.*]]) #[[ATTR0]] {
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
index 82813200c75ec..42a4829c18784 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-unoptimized-debug-data.ll
@@ -15,51 +15,52 @@ define float @debug_stash_pointer(ptr addrspace(8) %buf, i32 %idx, ptr addrspace
 ; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[BUF]], [[META13]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[META23]])
 ; CHECK-NEXT:    [[BUF_PTR_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[BUF]] to i160, !dbg [[DBG24:![0-9]+]]
 ; CHECK-NEXT:    [[BUF_PTR_INT:%.*]] = shl nuw i160 [[BUF_PTR_INT_RSRC]], 32, !dbg [[DBG24]]
-; CHECK-NEXT:    store i160 [[BUF_PTR_INT]], ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG24]]
-; CHECK-NEXT:      #dbg_value(i32 0, [[META15:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[META25:![0-9]+]])
-; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[AUX]], [[META15]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[META25]])
-; CHECK-NEXT:    [[AUX_PTR_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[AUX]] to i160, !dbg [[DBG26:![0-9]+]]
-; CHECK-NEXT:    [[AUX_PTR_INT:%.*]] = shl nuw i160 [[AUX_PTR_INT_RSRC]], 32, !dbg [[DBG26]]
-; CHECK-NEXT:    store i160 [[AUX_PTR_INT]], ptr addrspace(5) [[AUX_PTR_VAR]], align 32, !dbg [[DBG26]]
-; CHECK-NEXT:    [[BUF_PTR_2:%.*]] = load i160, ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG27:![0-9]+]]
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i160 [[BUF_PTR_2]], 32, !dbg [[DBG27]]
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128, !dbg [[DBG27]]
-; CHECK-NEXT:    [[BUF_PTR_2_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8), !dbg [[DBG27]]
-; CHECK-NEXT:    [[BUF_PTR_2_PTR_OFF:%.*]] = trunc i160 [[BUF_PTR_2]] to i32, !dbg [[DBG27]]
-; CHECK-NEXT:      #dbg_value(i32 [[BUF_PTR_2_PTR_OFF]], [[META16:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[DBG27]])
-; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[BUF_PTR_2_PTR_RSRC]], [[META16]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG27]])
-; CHECK-NEXT:    [[BUF_PTR_3_IDX:%.*]] = mul i32 [[IDX]], 4, !dbg [[DBG28:![0-9]+]]
-; CHECK-NEXT:    [[BUF_PTR_3:%.*]] = add i32 [[BUF_PTR_2_PTR_OFF]], [[BUF_PTR_3_IDX]], !dbg [[DBG28]]
-; CHECK-NEXT:      #dbg_value(i32 [[BUF_PTR_3]], [[META17:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[DBG28]])
-; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[BUF_PTR_2_PTR_RSRC]], [[META17]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG28]])
-; CHECK-NEXT:    [[BUF_PTR_3_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[BUF_PTR_2_PTR_RSRC]] to i160, !dbg [[DBG29:![0-9]+]]
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i160 [[BUF_PTR_3_INT_RSRC]], 32, !dbg [[DBG29]]
-; CHECK-NEXT:    [[BUF_PTR_3_INT_OFF:%.*]] = zext i32 [[BUF_PTR_3]] to i160, !dbg [[DBG29]]
-; CHECK-NEXT:    [[BUF_PTR_3_INT:%.*]] = or i160 [[TMP3]], [[BUF_PTR_3_INT_OFF]], !dbg [[DBG29]]
-; CHECK-NEXT:    store i160 [[BUF_PTR_3_INT]], ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG29]]
-; CHECK-NEXT:    [[BUF_PTR_4:%.*]] = load i160, ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG30:![0-9]+]]
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr i160 [[BUF_PTR_4]], 32, !dbg [[DBG30]]
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i160 [[TMP4]] to i128, !dbg [[DBG30]]
-; CHECK-NEXT:    [[BUF_PTR_4_PTR_RSRC:%.*]] = inttoptr i128 [[TMP5]] to ptr addrspace(8), !dbg [[DBG30]]
-; CHECK-NEXT:    [[BUF_PTR_4_PTR_OFF:%.*]] = trunc i160 [[BUF_PTR_4]] to i32, !dbg [[DBG30]]
-; CHECK-NEXT:      #dbg_value(i32 [[BUF_PTR_4_PTR_OFF]], [[META18:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[DBG30]])
-; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[BUF_PTR_4_PTR_RSRC]], [[META18]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG30]])
-; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF_PTR_4_PTR_RSRC]], i32 [[BUF_PTR_4_PTR_OFF]], i32 0, i32 0), !dbg [[DBG31:![0-9]+]]
-; CHECK-NEXT:      #dbg_value(float [[RET]], [[META19:![0-9]+]], !DIExpression(), [[DBG31]])
-; CHECK-NEXT:    [[AUX_PTR_2:%.*]] = load i160, ptr addrspace(5) [[AUX_PTR_VAR]], align 32, !dbg [[DBG32:![0-9]+]]
-; CHECK-NEXT:    [[TMP6:%.*]] = lshr i160 [[AUX_PTR_2]], 32, !dbg [[DBG32]]
-; CHECK-NEXT:    [[TMP7:%.*]] = trunc i160 [[TMP6]] to i128, !dbg [[DBG32]]
-; CHECK-NEXT:    [[AUX_PTR_2_PTR_RSRC:%.*]] = inttoptr i128 [[TMP7]] to ptr addrspace(8), !dbg [[DBG32]]
-; CHECK-NEXT:    [[AUX_PTR_2_PTR_OFF:%.*]] = trunc i160 [[AUX_PTR_2]] to i32, !dbg [[DBG32]]
-; CHECK-NEXT:      #dbg_value(i32 [[AUX_PTR_2_PTR_OFF]], [[META20:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[DBG32]])
-; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[AUX_PTR_2_PTR_RSRC]], [[META20]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG32]])
-; CHECK-NEXT:    [[BUF_PTR_4_LEGAL:%.*]] = bitcast i160 [[BUF_PTR_4]] to <5 x i32>, !dbg [[DBG33:![0-9]+]]
-; CHECK-NEXT:    [[BUF_PTR_4_SLICE_0:%.*]] = shufflevector <5 x i32> [[BUF_PTR_4_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>, !dbg [[DBG33]]
-; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[BUF_PTR_4_SLICE_0]], ptr addrspace(8) align 32 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_OFF]], i32 0, i32 0), !dbg [[DBG33]]
-; CHECK-NEXT:    [[AUX_PTR_2_PTR_PART_4:%.*]] = add nuw i32 [[AUX_PTR_2_PTR_OFF]], 16, !dbg [[DBG33]]
-; CHECK-NEXT:    [[BUF_PTR_4_SLICE_4:%.*]] = extractelement <5 x i32> [[BUF_PTR_4_LEGAL]], i64 4, !dbg [[DBG33]]
-; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[BUF_PTR_4_SLICE_4]], ptr addrspace(8) align 16 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_PART_4]], i32 0, i32 0), !dbg [[DBG33]]
-; CHECK-NEXT:    ret float [[RET]], !dbg [[DBG34:![0-9]+]]
+; CHECK-NEXT:    store i160 [[BUF_PTR_INT]], ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG24]], !DIAssignID [[DIASSIGNID25:![0-9]+]]
+; CHECK-NEXT:      #dbg_assign(i160 [[BUF_PTR_INT]], [[META13]], !DIExpression(), [[DIASSIGNID25]], ptr addrspace(5) [[BUF_PTR_VAR]], !DIExpression(), [[DBG21]])
+; CHECK-NEXT:      #dbg_value(i32 0, [[META15:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[META26:![0-9]+]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[AUX]], [[META15]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[META26]])
+; CHECK-NEXT:    [[AUX_PTR_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[AUX]] to i160, !dbg [[DBG27:![0-9]+]]
+; CHECK-NEXT:    [[AUX_PTR_INT:%.*]] = shl nuw i160 [[AUX_PTR_INT_RSRC]], 32, !dbg [[DBG27]]
+; CHECK-NEXT:    store i160 [[AUX_PTR_INT]], ptr addrspace(5) [[AUX_PTR_VAR]], align 32, !dbg [[DBG27]]
+; CHECK-NEXT:    [[BUF_PTR_2:%.*]] = load i160, ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG28:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i160 [[BUF_PTR_2]], 32, !dbg [[DBG28]]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i160 [[TMP1]] to i128, !dbg [[DBG28]]
+; CHECK-NEXT:    [[BUF_PTR_2_PTR_RSRC:%.*]] = inttoptr i128 [[TMP2]] to ptr addrspace(8), !dbg [[DBG28]]
+; CHECK-NEXT:    [[BUF_PTR_2_PTR_OFF:%.*]] = trunc i160 [[BUF_PTR_2]] to i32, !dbg [[DBG28]]
+; CHECK-NEXT:      #dbg_value(i32 [[BUF_PTR_2_PTR_OFF]], [[META16:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[DBG28]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[BUF_PTR_2_PTR_RSRC]], [[META16]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG28]])
+; CHECK-NEXT:    [[BUF_PTR_3_IDX:%.*]] = mul i32 [[IDX]], 4, !dbg [[DBG29:![0-9]+]]
+; CHECK-NEXT:    [[BUF_PTR_3:%.*]] = add i32 [[BUF_PTR_2_PTR_OFF]], [[BUF_PTR_3_IDX]], !dbg [[DBG29]]
+; CHECK-NEXT:      #dbg_value(i32 [[BUF_PTR_3]], [[META17:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[DBG29]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[BUF_PTR_2_PTR_RSRC]], [[META17]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG29]])
+; CHECK-NEXT:    [[BUF_PTR_3_INT_RSRC:%.*]] = ptrtoint ptr addrspace(8) [[BUF_PTR_2_PTR_RSRC]] to i160, !dbg [[DBG30:![0-9]+]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i160 [[BUF_PTR_3_INT_RSRC]], 32, !dbg [[DBG30]]
+; CHECK-NEXT:    [[BUF_PTR_3_INT_OFF:%.*]] = zext i32 [[BUF_PTR_3]] to i160, !dbg [[DBG30]]
+; CHECK-NEXT:    [[BUF_PTR_3_INT:%.*]] = or i160 [[TMP3]], [[BUF_PTR_3_INT_OFF]], !dbg [[DBG30]]
+; CHECK-NEXT:    store i160 [[BUF_PTR_3_INT]], ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG30]]
+; CHECK-NEXT:    [[BUF_PTR_4:%.*]] = load i160, ptr addrspace(5) [[BUF_PTR_VAR]], align 32, !dbg [[DBG31:![0-9]+]]
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr i160 [[BUF_PTR_4]], 32, !dbg [[DBG31]]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i160 [[TMP4]] to i128, !dbg [[DBG31]]
+; CHECK-NEXT:    [[BUF_PTR_4_PTR_RSRC:%.*]] = inttoptr i128 [[TMP5]] to ptr addrspace(8), !dbg [[DBG31]]
+; CHECK-NEXT:    [[BUF_PTR_4_PTR_OFF:%.*]] = trunc i160 [[BUF_PTR_4]] to i32, !dbg [[DBG31]]
+; CHECK-NEXT:      #dbg_value(i32 [[BUF_PTR_4_PTR_OFF]], [[META18:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[DBG31]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[BUF_PTR_4_PTR_RSRC]], [[META18]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG31]])
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.amdgcn.raw.ptr.buffer.load.f32(ptr addrspace(8) align 4 [[BUF_PTR_4_PTR_RSRC]], i32 [[BUF_PTR_4_PTR_OFF]], i32 0, i32 0), !dbg [[DBG32:![0-9]+]]
+; CHECK-NEXT:      #dbg_value(float [[RET]], [[META19:![0-9]+]], !DIExpression(), [[DBG32]])
+; CHECK-NEXT:    [[AUX_PTR_2:%.*]] = load i160, ptr addrspace(5) [[AUX_PTR_VAR]], align 32, !dbg [[DBG33:![0-9]+]]
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr i160 [[AUX_PTR_2]], 32, !dbg [[DBG33]]
+; CHECK-NEXT:    [[TMP7:%.*]] = trunc i160 [[TMP6]] to i128, !dbg [[DBG33]]
+; CHECK-NEXT:    [[AUX_PTR_2_PTR_RSRC:%.*]] = inttoptr i128 [[TMP7]] to ptr addrspace(8), !dbg [[DBG33]]
+; CHECK-NEXT:    [[AUX_PTR_2_PTR_OFF:%.*]] = trunc i160 [[AUX_PTR_2]] to i32, !dbg [[DBG33]]
+; CHECK-NEXT:      #dbg_value(i32 [[AUX_PTR_2_PTR_OFF]], [[META20:![0-9]+]], !DIExpression(DW_OP_LLVM_fragment, 128, 32), [[DBG33]])
+; CHECK-NEXT:      #dbg_value(ptr addrspace(8) [[AUX_PTR_2_PTR_RSRC]], [[META20]], !DIExpression(DW_OP_LLVM_fragment, 0, 128), [[DBG33]])
+; CHECK-NEXT:    [[BUF_PTR_4_LEGAL:%.*]] = bitcast i160 [[BUF_PTR_4]] to <5 x i32>, !dbg [[DBG34:![0-9]+]]
+; CHECK-NEXT:    [[BUF_PTR_4_SLICE_0:%.*]] = shufflevector <5 x i32> [[BUF_PTR_4_LEGAL]], <5 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>, !dbg [[DBG34]]
+; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.v4i32(<4 x i32> [[BUF_PTR_4_SLICE_0]], ptr addrspace(8) align 32 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_OFF]], i32 0, i32 0), !dbg [[DBG34]]
+; CHECK-NEXT:    [[AUX_PTR_2_PTR_PART_4:%.*]] = add nuw i32 [[AUX_PTR_2_PTR_OFF]], 16, !dbg [[DBG34]]
+; CHECK-NEXT:    [[BUF_PTR_4_SLICE_4:%.*]] = extractelement <5 x i32> [[BUF_PTR_4_LEGAL]], i64 4, !dbg [[DBG34]]
+; CHECK-NEXT:    call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 [[BUF_PTR_4_SLICE_4]], ptr addrspace(8) align 16 [[AUX_PTR_2_PTR_RSRC]], i32 [[AUX_PTR_2_PTR_PART_4]], i32 0, i32 0), !dbg [[DBG34]]
+; CHECK-NEXT:    ret float [[RET]], !dbg [[DBG35:![0-9]+]]
 ;
   %buf.ptr.var = alloca ptr addrspace(7), align 32, addrspace(5), !dbg !20
   call void @llvm.dbg.value(metadata ptr addrspace(5) %buf.ptr.var, metadata !9, metadata !DIExpression()), !dbg !20
@@ -67,7 +68,8 @@ define float @debug_stash_pointer(ptr addrspace(8) %buf, i32 %idx, ptr addrspace
   call void @llvm.dbg.value(metadata ptr addrspace(5) %aux.ptr.var, metadata !11, metadata !DIExpression()), !dbg !21
   %buf.ptr = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7), !dbg !22
   call void @llvm.dbg.value(metadata ptr addrspace(7) %buf.ptr, metadata !12, metadata !DIExpression()), !dbg !22
-  store ptr addrspace(7) %buf.ptr, ptr addrspace(5) %buf.ptr.var, align 32, !dbg !23
+  store ptr addrspace(7) %buf.ptr, ptr addrspace(5) %buf.ptr.var, align 32, !dbg !23, !DIAssignID !40
+  call void @llvm.dbg.assign(metadata ptr addrspace(7) %buf.ptr, metadata !12, metadata !DIExpression(), metadata !40, metadata ptr addrspace(5) %buf.ptr.var, metadata !DIExpression()), !dbg !20
   %aux.ptr = addrspacecast ptr addrspace(8) %aux to ptr addrspace(7), !dbg !24
   call void @llvm.dbg.value(metadata ptr addrspace(7) %aux.ptr, metadata !14, metadata !DIExpression()), !dbg !24
   store ptr addrspace(7) %aux.ptr, ptr addrspace(5) %aux.ptr.var, align 32, !dbg !25
@@ -129,3 +131,4 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 !31 = !DILocation(line: 12, column: 1, scope: !5)
 !32 = !DILocation(line: 13, column: 1, scope: !5)
 !33 = !DILocation(line: 14, column: 1, scope: !5)
+!40 = distinct !DIAssignID()
diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
index 23412aaeb2e23..3b3ea3f37db80 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats-attr.mir
@@ -347,8 +347,10 @@ body:             |
 ...
 # User-requested maximum number of VGPRs need to be taken into account by
 # the scheduler's rematerialization stage. Register usage above that number
-# is considered like spill; occupancy is "inadvertently" increased when
-# eliminating spill.
+# is considered like spill. On unified RF (gfx90a), the requested number is
+# understood "per-bank", effectively doubling its value, so no rematerialization
+# is necessary.
+---
 name:            small_num_vgprs_as_spill
 tracksRegLiveness: true
 machineFunctionInfo:
@@ -371,36 +373,15 @@ body:             |
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 10, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 11, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 12, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 13, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 14, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 15, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 16, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 17, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 18, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 19, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 20, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 21, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 33, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 15, implicit $exec, implicit $mode
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_5]], implicit [[V_CVT_I32_F64_e32_6]], implicit [[V_CVT_I32_F64_e32_7]], implicit [[V_CVT_I32_F64_e32_8]], implicit [[V_CVT_I32_F64_e32_9]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_10]], implicit [[V_CVT_I32_F64_e32_11]], implicit [[V_CVT_I32_F64_e32_12]], implicit [[V_CVT_I32_F64_e32_13]], implicit [[V_CVT_I32_F64_e32_14]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_15]], implicit [[V_CVT_I32_F64_e32_16]], implicit [[V_CVT_I32_F64_e32_17]], implicit [[V_CVT_I32_F64_e32_18]], implicit [[V_CVT_I32_F64_e32_19]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_20]], implicit [[V_CVT_I32_F64_e32_21]], implicit [[V_CVT_I32_F64_e32_22]], implicit [[V_CVT_I32_F64_e32_23]], implicit [[V_CVT_I32_F64_e32_24]]
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_33:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 32, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_28]], implicit [[V_CVT_I32_F64_e32_29]], implicit [[V_CVT_I32_F64_e32_30]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_31]], implicit [[V_CVT_I32_F64_e32_32]], implicit [[V_CVT_I32_F64_e32_33]], implicit [[V_CVT_I32_F64_e32_27]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 13, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 14, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_10]], implicit [[V_CVT_I32_F64_e32_11]], implicit [[V_CVT_I32_F64_e32_12]], implicit [[V_CVT_I32_F64_e32_14]], implicit [[V_CVT_I32_F64_e32_15]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_13]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
   ; GFX90A-LABEL: name: small_num_vgprs_as_spill
@@ -420,36 +401,15 @@ body:             |
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_10:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 10, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_11:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 11, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_12:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 12, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 13, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 14, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 15, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_16:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 16, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_17:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 17, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_18:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 18, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_19:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 19, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 20, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 21, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 33, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_13:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 13, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_14:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 14, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_15:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 15, implicit $exec, implicit $mode
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.1:
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_5]], implicit [[V_CVT_I32_F64_e32_6]], implicit [[V_CVT_I32_F64_e32_7]], implicit [[V_CVT_I32_F64_e32_8]], implicit [[V_CVT_I32_F64_e32_9]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_10]], implicit [[V_CVT_I32_F64_e32_11]], implicit [[V_CVT_I32_F64_e32_12]], implicit [[V_CVT_I32_F64_e32_13]], implicit [[V_CVT_I32_F64_e32_14]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_15]], implicit [[V_CVT_I32_F64_e32_16]], implicit [[V_CVT_I32_F64_e32_17]], implicit [[V_CVT_I32_F64_e32_18]], implicit [[V_CVT_I32_F64_e32_19]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_20]], implicit [[V_CVT_I32_F64_e32_21]], implicit [[V_CVT_I32_F64_e32_22]], implicit [[V_CVT_I32_F64_e32_23]], implicit [[V_CVT_I32_F64_e32_24]]
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_28]], implicit [[V_CVT_I32_F64_e32_29]], implicit [[V_CVT_I32_F64_e32_30]]
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_33:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 32, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_31]], implicit [[V_CVT_I32_F64_e32_32]], implicit [[V_CVT_I32_F64_e32_33]], implicit [[V_CVT_I32_F64_e32_27]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_15]]
   ; GFX90A-NEXT:   S_ENDPGM 0
   bb.0:
     successors: %bb.1
@@ -467,38 +427,16 @@ body:             |
     %10:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 10, implicit $exec, implicit $mode, implicit-def $m0
     %11:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 11, implicit $exec, implicit $mode, implicit-def $m0
     %12:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 12, implicit $exec, implicit $mode, implicit-def $m0
-    %13:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 13, implicit $exec, implicit $mode, implicit-def $m0
-    %14:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 14, implicit $exec, implicit $mode, implicit-def $m0
-    %15:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 15, implicit $exec, implicit $mode, implicit-def $m0
-    %16:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 16, implicit $exec, implicit $mode, implicit-def $m0
-    %17:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 17, implicit $exec, implicit $mode, implicit-def $m0
-    %18:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 18, implicit $exec, implicit $mode, implicit-def $m0
-    %19:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 19, implicit $exec, implicit $mode, implicit-def $m0
-    %20:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 20, implicit $exec, implicit $mode, implicit-def $m0
-    %21:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 21, implicit $exec, implicit $mode, implicit-def $m0
-    %22:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode, implicit-def $m0
-    %23:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode, implicit-def $m0
-    %24:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
-    %25:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode, implicit-def $m0
-    %26:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0
-    %27:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
-    %28:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
-    %29:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
-    %30:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
-    %31:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
-    %32:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 32, implicit $exec, implicit $mode
-    %33:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 33, implicit $exec, implicit $mode
+    %13:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 13, implicit $exec, implicit $mode
+    %14:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 14, implicit $exec, implicit $mode
+    %15:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 15, implicit $exec, implicit $mode
 
   bb.1:
 
     S_NOP 0, implicit %0,   implicit %1,   implicit %2,   implicit %3,   implicit %4
     S_NOP 0, implicit %5,   implicit %6,   implicit %7,   implicit %8,   implicit %9
     S_NOP 0, implicit %10,  implicit %11,  implicit %12,  implicit %13,  implicit %14
-    S_NOP 0, implicit %15,  implicit %16,  implicit %17,  implicit %18,  implicit %19
-    S_NOP 0, implicit %20,  implicit %21,  implicit %22,  implicit %23,  implicit %24
-    S_NOP 0, implicit %25,  implicit %26,  implicit %27,  implicit %28,  implicit %29
-    S_NOP 0, implicit %30,  implicit %31,  implicit %32,  implicit %33
-
+    S_NOP 0, implicit %15
     S_ENDPGM 0
 ...
 # Min/Max occupancy is 8, but user requests 7, the scheduler's rematerialization
@@ -815,9 +753,9 @@ body:             |
     S_ENDPGM 0
 ...
 # Min/Max waves/EU is 8. For targets with non-unified RF (gfx908) we are able to
-# eliminate both ArchVGPR and AGPR spilling by saving 2 VGPRs. In the unified RF
-# case (gfx90a) the ArchVGPR allocation granule forces us to remat more
-# ArchVGPRs to eliminate spilling.
+# eliminate both ArchVGPR and AGPR spilling by saving one of each. In the
+# unified RF case (gfx90a) the ArchVGPR allocation granule may force us to remat
+# more ArchVGPRs to eliminate spilling.
 ---
 name:            reduce_arch_and_acc_vgrp_spill
 tracksRegLiveness: true
@@ -860,6 +798,7 @@ body:             |
   ; GFX908-NEXT:   [[DEF28:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[DEF29:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[DEF30:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0
@@ -886,12 +825,11 @@ body:             |
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_24:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 24, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_25:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 25, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_26:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 26, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 64, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_27:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 28, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_28:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 29, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_29:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 30, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_30:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 31, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_31:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 64, implicit $exec, implicit $mode
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]]
@@ -899,17 +837,17 @@ body:             |
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_10]], implicit [[V_CVT_I32_F64_e32_11]], implicit [[V_CVT_I32_F64_e32_12]], implicit [[V_CVT_I32_F64_e32_13]], implicit [[V_CVT_I32_F64_e32_14]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_15]], implicit [[V_CVT_I32_F64_e32_16]], implicit [[V_CVT_I32_F64_e32_17]], implicit [[V_CVT_I32_F64_e32_18]], implicit [[V_CVT_I32_F64_e32_19]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_20]], implicit [[V_CVT_I32_F64_e32_21]], implicit [[V_CVT_I32_F64_e32_22]], implicit [[V_CVT_I32_F64_e32_23]], implicit [[V_CVT_I32_F64_e32_24]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_27]], implicit [[V_CVT_I32_F64_e32_28]], implicit [[V_CVT_I32_F64_e32_29]]
-  ; GFX908-NEXT:   [[DEF31:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_32:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 27, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_25]], implicit [[V_CVT_I32_F64_e32_26]], implicit [[V_CVT_I32_F64_e32_32]], implicit [[V_CVT_I32_F64_e32_27]], implicit [[V_CVT_I32_F64_e32_28]]
   ; GFX908-NEXT:   [[DEF32:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_30]], implicit [[V_CVT_I32_F64_e32_31]], implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]], implicit [[V_CVT_I32_F64_e32_32]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF30]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_29]], implicit [[V_CVT_I32_F64_e32_30]], implicit [[DEF32]], implicit [[DEF]], implicit [[DEF1]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]], implicit [[DEF11]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]], implicit [[DEF21]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]], implicit [[DEF30]], implicit [[V_CVT_I32_F64_e32_31]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF31]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
   ; GFX90A-LABEL: name: reduce_arch_and_acc_vgrp_spill
@@ -1358,8 +1296,7 @@ body:             |
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_252:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 252, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_253:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 253, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_254:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 254, implicit $exec, implicit $mode, implicit-def $m0
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_255:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 255, implicit $exec, implicit $mode
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_256:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 256, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_255:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 256, implicit $exec, implicit $mode
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]], implicit [[V_CVT_I32_F64_e32_5]], implicit [[V_CVT_I32_F64_e32_6]], implicit [[V_CVT_I32_F64_e32_7]], implicit [[V_CVT_I32_F64_e32_8]], implicit [[V_CVT_I32_F64_e32_9]]
@@ -1387,7 +1324,8 @@ body:             |
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_220]], implicit [[V_CVT_I32_F64_e32_221]], implicit [[V_CVT_I32_F64_e32_222]], implicit [[V_CVT_I32_F64_e32_223]], implicit [[V_CVT_I32_F64_e32_224]], implicit [[V_CVT_I32_F64_e32_225]], implicit [[V_CVT_I32_F64_e32_226]], implicit [[V_CVT_I32_F64_e32_227]], implicit [[V_CVT_I32_F64_e32_228]], implicit [[V_CVT_I32_F64_e32_229]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_230]], implicit [[V_CVT_I32_F64_e32_231]], implicit [[V_CVT_I32_F64_e32_232]], implicit [[V_CVT_I32_F64_e32_233]], implicit [[V_CVT_I32_F64_e32_234]], implicit [[V_CVT_I32_F64_e32_235]], implicit [[V_CVT_I32_F64_e32_236]], implicit [[V_CVT_I32_F64_e32_237]], implicit [[V_CVT_I32_F64_e32_238]], implicit [[V_CVT_I32_F64_e32_239]]
   ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_240]], implicit [[V_CVT_I32_F64_e32_241]], implicit [[V_CVT_I32_F64_e32_242]], implicit [[V_CVT_I32_F64_e32_243]], implicit [[V_CVT_I32_F64_e32_244]], implicit [[V_CVT_I32_F64_e32_245]], implicit [[V_CVT_I32_F64_e32_246]], implicit [[V_CVT_I32_F64_e32_247]], implicit [[V_CVT_I32_F64_e32_248]], implicit [[V_CVT_I32_F64_e32_249]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_250]], implicit [[V_CVT_I32_F64_e32_251]], implicit [[V_CVT_I32_F64_e32_252]], implicit [[V_CVT_I32_F64_e32_253]], implicit [[V_CVT_I32_F64_e32_254]], implicit [[V_CVT_I32_F64_e32_255]], implicit [[V_CVT_I32_F64_e32_256]], implicit [[DEF]]
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_256:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 255, implicit $exec, implicit $mode
+  ; GFX908-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_250]], implicit [[V_CVT_I32_F64_e32_251]], implicit [[V_CVT_I32_F64_e32_252]], implicit [[V_CVT_I32_F64_e32_253]], implicit [[V_CVT_I32_F64_e32_254]], implicit [[V_CVT_I32_F64_e32_256]], implicit [[V_CVT_I32_F64_e32_255]], implicit [[DEF]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
   ; GFX90A-LABEL: name: reduce_spill_archvgpr_above_addressable_limit
@@ -1395,6 +1333,7 @@ body:             |
   ; GFX90A-NEXT:   successors: %bb.1(0x80000000)
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 0, implicit $exec, implicit $mode, implicit-def $m0
+  ; GFX90A-NEXT:   [[DEF:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_3:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode, implicit-def $m0
@@ -1650,8 +1589,6 @@ body:             |
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_253:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 253, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_254:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 254, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_255:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 256, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[DEF:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_256:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 255, implicit $exec, implicit $mode
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.1:
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]], implicit [[V_CVT_I32_F64_e32_2]], implicit [[V_CVT_I32_F64_e32_3]], implicit [[V_CVT_I32_F64_e32_4]], implicit [[V_CVT_I32_F64_e32_5]], implicit [[V_CVT_I32_F64_e32_6]], implicit [[V_CVT_I32_F64_e32_7]], implicit [[V_CVT_I32_F64_e32_8]], implicit [[V_CVT_I32_F64_e32_9]]
@@ -1679,6 +1616,7 @@ body:             |
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_220]], implicit [[V_CVT_I32_F64_e32_221]], implicit [[V_CVT_I32_F64_e32_222]], implicit [[V_CVT_I32_F64_e32_223]], implicit [[V_CVT_I32_F64_e32_224]], implicit [[V_CVT_I32_F64_e32_225]], implicit [[V_CVT_I32_F64_e32_226]], implicit [[V_CVT_I32_F64_e32_227]], implicit [[V_CVT_I32_F64_e32_228]], implicit [[V_CVT_I32_F64_e32_229]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_230]], implicit [[V_CVT_I32_F64_e32_231]], implicit [[V_CVT_I32_F64_e32_232]], implicit [[V_CVT_I32_F64_e32_233]], implicit [[V_CVT_I32_F64_e32_234]], implicit [[V_CVT_I32_F64_e32_235]], implicit [[V_CVT_I32_F64_e32_236]], implicit [[V_CVT_I32_F64_e32_237]], implicit [[V_CVT_I32_F64_e32_238]], implicit [[V_CVT_I32_F64_e32_239]]
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_240]], implicit [[V_CVT_I32_F64_e32_241]], implicit [[V_CVT_I32_F64_e32_242]], implicit [[V_CVT_I32_F64_e32_243]], implicit [[V_CVT_I32_F64_e32_244]], implicit [[V_CVT_I32_F64_e32_245]], implicit [[V_CVT_I32_F64_e32_246]], implicit [[V_CVT_I32_F64_e32_247]], implicit [[V_CVT_I32_F64_e32_248]], implicit [[V_CVT_I32_F64_e32_249]]
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_256:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 255, implicit $exec, implicit $mode
   ; GFX90A-NEXT:   S_NOP 0, implicit [[V_CVT_I32_F64_e32_250]], implicit [[V_CVT_I32_F64_e32_251]], implicit [[V_CVT_I32_F64_e32_252]], implicit [[V_CVT_I32_F64_e32_253]], implicit [[V_CVT_I32_F64_e32_254]], implicit [[V_CVT_I32_F64_e32_256]], implicit [[V_CVT_I32_F64_e32_255]], implicit [[DEF]]
   ; GFX90A-NEXT:   S_ENDPGM 0
   bb.0:
@@ -2246,35 +2184,35 @@ body:             |
   ; GFX908-NEXT:   [[DEF253:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[DEF254:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT:   [[DEF255:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
-  ; GFX908-NEXT:   [[DEF256:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF128]], implicit [[DEF129]], implicit [[DEF130]], implicit [[DEF131]], implicit [[DEF132]], implicit [[DEF133]], implicit [[DEF134]], implicit [[DEF135]], implicit [[DEF136]], implicit [[DEF137]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF138]], implicit [[DEF139]], implicit [[DEF140]], implicit [[DEF141]], implicit [[DEF142]], implicit [[DEF143]], implicit [[DEF144]], implicit [[DEF145]], implicit [[DEF146]], implicit [[DEF147]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF148]], implicit [[DEF149]], implicit [[DEF150]], implicit [[DEF151]], implicit [[DEF152]], implicit [[DEF153]], implicit [[DEF154]], implicit [[DEF155]], implicit [[DEF156]], implicit [[DEF157]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF158]], implicit [[DEF159]], implicit [[DEF160]], implicit [[DEF161]], implicit [[DEF162]], implicit [[DEF163]], implicit [[DEF164]], implicit [[DEF165]], implicit [[DEF166]], implicit [[DEF167]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF168]], implicit [[DEF169]], implicit [[DEF170]], implicit [[DEF171]], implicit [[DEF172]], implicit [[DEF173]], implicit [[DEF174]], implicit [[DEF175]], implicit [[DEF176]], implicit [[DEF177]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF178]], implicit [[DEF179]], implicit [[DEF180]], implicit [[DEF181]], implicit [[DEF182]], implicit [[DEF183]], implicit [[DEF184]], implicit [[DEF185]], implicit [[DEF186]], implicit [[DEF187]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF188]], implicit [[DEF189]], implicit [[DEF190]], implicit [[DEF191]], implicit [[DEF192]], implicit [[DEF193]], implicit [[DEF194]], implicit [[DEF195]], implicit [[DEF196]], implicit [[DEF197]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF198]], implicit [[DEF199]], implicit [[DEF200]], implicit [[DEF201]], implicit [[DEF202]], implicit [[DEF203]], implicit [[DEF204]], implicit [[DEF205]], implicit [[DEF206]], implicit [[DEF207]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF208]], implicit [[DEF209]], implicit [[DEF210]], implicit [[DEF211]], implicit [[DEF212]], implicit [[DEF213]], implicit [[DEF214]], implicit [[DEF215]], implicit [[DEF216]], implicit [[DEF217]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF218]], implicit [[DEF219]], implicit [[DEF220]], implicit [[DEF221]], implicit [[DEF222]], implicit [[DEF223]], implicit [[DEF224]], implicit [[DEF225]], implicit [[DEF226]], implicit [[DEF227]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF228]], implicit [[DEF229]], implicit [[DEF230]], implicit [[DEF231]], implicit [[DEF232]], implicit [[DEF233]], implicit [[DEF234]], implicit [[DEF235]], implicit [[DEF236]], implicit [[DEF237]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF238]], implicit [[DEF239]], implicit [[DEF240]], implicit [[DEF241]], implicit [[DEF242]], implicit [[DEF243]], implicit [[DEF244]], implicit [[DEF245]], implicit [[DEF246]], implicit [[DEF247]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF248]], implicit [[DEF249]], implicit [[DEF250]], implicit [[DEF251]], implicit [[DEF252]], implicit [[DEF253]], implicit [[DEF254]], implicit [[DEF255]], implicit [[DEF256]], implicit [[DEF]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]], implicit [[DEF30]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF33]], implicit [[DEF34]], implicit [[DEF35]], implicit [[DEF36]], implicit [[DEF37]], implicit [[DEF38]], implicit [[DEF39]], implicit [[DEF40]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF41]], implicit [[DEF42]], implicit [[DEF43]], implicit [[DEF44]], implicit [[DEF45]], implicit [[DEF46]], implicit [[DEF47]], implicit [[DEF48]], implicit [[DEF49]], implicit [[DEF50]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF51]], implicit [[DEF52]], implicit [[DEF53]], implicit [[DEF54]], implicit [[DEF55]], implicit [[DEF56]], implicit [[DEF57]], implicit [[DEF58]], implicit [[DEF59]], implicit [[DEF60]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF61]], implicit [[DEF62]], implicit [[DEF63]], implicit [[DEF64]], implicit [[DEF65]], implicit [[DEF66]], implicit [[DEF67]], implicit [[DEF68]], implicit [[DEF69]], implicit [[DEF70]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF71]], implicit [[DEF72]], implicit [[DEF73]], implicit [[DEF74]], implicit [[DEF75]], implicit [[DEF76]], implicit [[DEF77]], implicit [[DEF78]], implicit [[DEF79]], implicit [[DEF80]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF81]], implicit [[DEF82]], implicit [[DEF83]], implicit [[DEF84]], implicit [[DEF85]], implicit [[DEF86]], implicit [[DEF87]], implicit [[DEF88]], implicit [[DEF89]], implicit [[DEF90]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF91]], implicit [[DEF92]], implicit [[DEF93]], implicit [[DEF94]], implicit [[DEF95]], implicit [[DEF96]], implicit [[DEF97]], implicit [[DEF98]], implicit [[DEF99]], implicit [[DEF100]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF101]], implicit [[DEF102]], implicit [[DEF103]], implicit [[DEF104]], implicit [[DEF105]], implicit [[DEF106]], implicit [[DEF107]], implicit [[DEF108]], implicit [[DEF109]], implicit [[DEF110]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF111]], implicit [[DEF112]], implicit [[DEF113]], implicit [[DEF114]], implicit [[DEF115]], implicit [[DEF116]], implicit [[DEF117]], implicit [[DEF118]], implicit [[DEF119]], implicit [[DEF120]]
-  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF121]], implicit [[DEF122]], implicit [[DEF123]], implicit [[DEF124]], implicit [[DEF125]], implicit [[DEF126]], implicit [[DEF127]], implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]]
+  ; GFX908-NEXT:   [[DEF256:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF256]], implicit [[DEF]], implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF9]], implicit [[DEF10]], implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF19]], implicit [[DEF20]], implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF29]], implicit [[DEF30]], implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF33]], implicit [[DEF34]], implicit [[DEF35]], implicit [[DEF36]], implicit [[DEF37]], implicit [[DEF38]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF39]], implicit [[DEF40]], implicit [[DEF41]], implicit [[DEF42]], implicit [[DEF43]], implicit [[DEF44]], implicit [[DEF45]], implicit [[DEF46]], implicit [[DEF47]], implicit [[DEF48]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF49]], implicit [[DEF50]], implicit [[DEF51]], implicit [[DEF52]], implicit [[DEF53]], implicit [[DEF54]], implicit [[DEF55]], implicit [[DEF56]], implicit [[DEF57]], implicit [[DEF58]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF59]], implicit [[DEF60]], implicit [[DEF61]], implicit [[DEF62]], implicit [[DEF63]], implicit [[DEF64]], implicit [[DEF65]], implicit [[DEF66]], implicit [[DEF67]], implicit [[DEF68]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF69]], implicit [[DEF70]], implicit [[DEF71]], implicit [[DEF72]], implicit [[DEF73]], implicit [[DEF74]], implicit [[DEF75]], implicit [[DEF76]], implicit [[DEF77]], implicit [[DEF78]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF79]], implicit [[DEF80]], implicit [[DEF81]], implicit [[DEF82]], implicit [[DEF83]], implicit [[DEF84]], implicit [[DEF85]], implicit [[DEF86]], implicit [[DEF87]], implicit [[DEF88]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF89]], implicit [[DEF90]], implicit [[DEF91]], implicit [[DEF92]], implicit [[DEF93]], implicit [[DEF94]], implicit [[DEF95]], implicit [[DEF96]], implicit [[DEF97]], implicit [[DEF98]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF99]], implicit [[DEF100]], implicit [[DEF101]], implicit [[DEF102]], implicit [[DEF103]], implicit [[DEF104]], implicit [[DEF105]], implicit [[DEF106]], implicit [[DEF107]], implicit [[DEF108]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF109]], implicit [[DEF110]], implicit [[DEF111]], implicit [[DEF112]], implicit [[DEF113]], implicit [[DEF114]], implicit [[DEF115]], implicit [[DEF116]], implicit [[DEF117]], implicit [[DEF118]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF119]], implicit [[DEF120]], implicit [[DEF121]], implicit [[DEF122]], implicit [[DEF123]], implicit [[DEF124]], implicit [[DEF125]], implicit [[DEF126]], implicit [[DEF127]], implicit [[DEF128]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF129]], implicit [[DEF130]], implicit [[DEF131]], implicit [[DEF132]], implicit [[DEF133]], implicit [[DEF134]], implicit [[DEF135]], implicit [[DEF136]], implicit [[DEF137]], implicit [[DEF138]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF139]], implicit [[DEF140]], implicit [[DEF141]], implicit [[DEF142]], implicit [[DEF143]], implicit [[DEF144]], implicit [[DEF145]], implicit [[DEF146]], implicit [[DEF147]], implicit [[DEF148]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF149]], implicit [[DEF150]], implicit [[DEF151]], implicit [[DEF152]], implicit [[DEF153]], implicit [[DEF154]], implicit [[DEF155]], implicit [[DEF156]], implicit [[DEF157]], implicit [[DEF158]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF159]], implicit [[DEF160]], implicit [[DEF161]], implicit [[DEF162]], implicit [[DEF163]], implicit [[DEF164]], implicit [[DEF165]], implicit [[DEF166]], implicit [[DEF167]], implicit [[DEF168]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF169]], implicit [[DEF170]], implicit [[DEF171]], implicit [[DEF172]], implicit [[DEF173]], implicit [[DEF174]], implicit [[DEF175]], implicit [[DEF176]], implicit [[DEF177]], implicit [[DEF178]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF179]], implicit [[DEF180]], implicit [[DEF181]], implicit [[DEF182]], implicit [[DEF183]], implicit [[DEF184]], implicit [[DEF185]], implicit [[DEF186]], implicit [[DEF187]], implicit [[DEF188]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF189]], implicit [[DEF190]], implicit [[DEF191]], implicit [[DEF192]], implicit [[DEF193]], implicit [[DEF194]], implicit [[DEF195]], implicit [[DEF196]], implicit [[DEF197]], implicit [[DEF198]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF199]], implicit [[DEF200]], implicit [[DEF201]], implicit [[DEF202]], implicit [[DEF203]], implicit [[DEF204]], implicit [[DEF205]], implicit [[DEF206]], implicit [[DEF207]], implicit [[DEF208]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF209]], implicit [[DEF210]], implicit [[DEF211]], implicit [[DEF212]], implicit [[DEF213]], implicit [[DEF214]], implicit [[DEF215]], implicit [[DEF216]], implicit [[DEF217]], implicit [[DEF218]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF219]], implicit [[DEF220]], implicit [[DEF221]], implicit [[DEF222]], implicit [[DEF223]], implicit [[DEF224]], implicit [[DEF225]], implicit [[DEF226]], implicit [[DEF227]], implicit [[DEF228]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF229]], implicit [[DEF230]], implicit [[DEF231]], implicit [[DEF232]], implicit [[DEF233]], implicit [[DEF234]], implicit [[DEF235]], implicit [[DEF236]], implicit [[DEF237]], implicit [[DEF238]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF239]], implicit [[DEF240]], implicit [[DEF241]], implicit [[DEF242]], implicit [[DEF243]], implicit [[DEF244]], implicit [[DEF245]], implicit [[DEF246]], implicit [[DEF247]], implicit [[DEF248]]
+  ; GFX908-NEXT:   S_NOP 0, implicit [[DEF249]], implicit [[DEF250]], implicit [[DEF251]], implicit [[DEF252]], implicit [[DEF253]], implicit [[DEF254]], implicit [[DEF255]], implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]]
   ; GFX908-NEXT:   S_ENDPGM 0
   ;
   ; GFX90A-LABEL: name: reduce_spill_agpr_above_addressable_limit
@@ -2533,41 +2471,41 @@ body:             |
   ; GFX90A-NEXT:   [[DEF249:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF250:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF251:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode
-  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode
   ; GFX90A-NEXT:   [[DEF252:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF253:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF254:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
   ; GFX90A-NEXT:   [[DEF255:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
-  ; GFX90A-NEXT:   [[DEF256:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 257, implicit $exec, implicit $mode
+  ; GFX90A-NEXT:   [[V_CVT_I32_F64_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 258, implicit $exec, implicit $mode
   ; GFX90A-NEXT: {{  $}}
   ; GFX90A-NEXT: bb.1:
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF252]], implicit [[DEF253]], implicit [[DEF254]], implicit [[DEF255]], implicit [[DEF256]], implicit [[DEF]], implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]], implicit [[DEF9]], implicit [[DEF10]], implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]], implicit [[DEF19]], implicit [[DEF20]], implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]], implicit [[DEF29]], implicit [[DEF30]], implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF33]], implicit [[DEF34]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF35]], implicit [[DEF36]], implicit [[DEF37]], implicit [[DEF38]], implicit [[DEF39]], implicit [[DEF40]], implicit [[DEF41]], implicit [[DEF42]], implicit [[DEF43]], implicit [[DEF44]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF45]], implicit [[DEF46]], implicit [[DEF47]], implicit [[DEF48]], implicit [[DEF49]], implicit [[DEF50]], implicit [[DEF51]], implicit [[DEF52]], implicit [[DEF53]], implicit [[DEF54]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF55]], implicit [[DEF56]], implicit [[DEF57]], implicit [[DEF58]], implicit [[DEF59]], implicit [[DEF60]], implicit [[DEF61]], implicit [[DEF62]], implicit [[DEF63]], implicit [[DEF64]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF65]], implicit [[DEF66]], implicit [[DEF67]], implicit [[DEF68]], implicit [[DEF69]], implicit [[DEF70]], implicit [[DEF71]], implicit [[DEF72]], implicit [[DEF73]], implicit [[DEF74]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF75]], implicit [[DEF76]], implicit [[DEF77]], implicit [[DEF78]], implicit [[DEF79]], implicit [[DEF80]], implicit [[DEF81]], implicit [[DEF82]], implicit [[DEF83]], implicit [[DEF84]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF85]], implicit [[DEF86]], implicit [[DEF87]], implicit [[DEF88]], implicit [[DEF89]], implicit [[DEF90]], implicit [[DEF91]], implicit [[DEF92]], implicit [[DEF93]], implicit [[DEF94]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF95]], implicit [[DEF96]], implicit [[DEF97]], implicit [[DEF98]], implicit [[DEF99]], implicit [[DEF100]], implicit [[DEF101]], implicit [[DEF102]], implicit [[DEF103]], implicit [[DEF104]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF105]], implicit [[DEF106]], implicit [[DEF107]], implicit [[DEF108]], implicit [[DEF109]], implicit [[DEF110]], implicit [[DEF111]], implicit [[DEF112]], implicit [[DEF113]], implicit [[DEF114]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF115]], implicit [[DEF116]], implicit [[DEF117]], implicit [[DEF118]], implicit [[DEF119]], implicit [[DEF120]], implicit [[DEF121]], implicit [[DEF122]], implicit [[DEF123]], implicit [[DEF124]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF125]], implicit [[DEF126]], implicit [[DEF127]], implicit [[DEF128]], implicit [[DEF129]], implicit [[DEF130]], implicit [[DEF131]], implicit [[DEF132]], implicit [[DEF133]], implicit [[DEF134]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF135]], implicit [[DEF136]], implicit [[DEF137]], implicit [[DEF138]], implicit [[DEF139]], implicit [[DEF140]], implicit [[DEF141]], implicit [[DEF142]], implicit [[DEF143]], implicit [[DEF144]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF145]], implicit [[DEF146]], implicit [[DEF147]], implicit [[DEF148]], implicit [[DEF149]], implicit [[DEF150]], implicit [[DEF151]], implicit [[DEF152]], implicit [[DEF153]], implicit [[DEF154]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF155]], implicit [[DEF156]], implicit [[DEF157]], implicit [[DEF158]], implicit [[DEF159]], implicit [[DEF160]], implicit [[DEF161]], implicit [[DEF162]], implicit [[DEF163]], implicit [[DEF164]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF165]], implicit [[DEF166]], implicit [[DEF167]], implicit [[DEF168]], implicit [[DEF169]], implicit [[DEF170]], implicit [[DEF171]], implicit [[DEF172]], implicit [[DEF173]], implicit [[DEF174]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF175]], implicit [[DEF176]], implicit [[DEF177]], implicit [[DEF178]], implicit [[DEF179]], implicit [[DEF180]], implicit [[DEF181]], implicit [[DEF182]], implicit [[DEF183]], implicit [[DEF184]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF185]], implicit [[DEF186]], implicit [[DEF187]], implicit [[DEF188]], implicit [[DEF189]], implicit [[DEF190]], implicit [[DEF191]], implicit [[DEF192]], implicit [[DEF193]], implicit [[DEF194]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF195]], implicit [[DEF196]], implicit [[DEF197]], implicit [[DEF198]], implicit [[DEF199]], implicit [[DEF200]], implicit [[DEF201]], implicit [[DEF202]], implicit [[DEF203]], implicit [[DEF204]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF205]], implicit [[DEF206]], implicit [[DEF207]], implicit [[DEF208]], implicit [[DEF209]], implicit [[DEF210]], implicit [[DEF211]], implicit [[DEF212]], implicit [[DEF213]], implicit [[DEF214]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF215]], implicit [[DEF216]], implicit [[DEF217]], implicit [[DEF218]], implicit [[DEF219]], implicit [[DEF220]], implicit [[DEF221]], implicit [[DEF222]], implicit [[DEF223]], implicit [[DEF224]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF225]], implicit [[DEF226]], implicit [[DEF227]], implicit [[DEF228]], implicit [[DEF229]], implicit [[DEF230]], implicit [[DEF231]], implicit [[DEF232]], implicit [[DEF233]], implicit [[DEF234]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF235]], implicit [[DEF236]], implicit [[DEF237]], implicit [[DEF238]], implicit [[DEF239]], implicit [[DEF240]], implicit [[DEF241]], implicit [[DEF242]], implicit [[DEF243]], implicit [[DEF244]]
-  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF245]], implicit [[DEF246]], implicit [[DEF247]], implicit [[DEF248]], implicit [[DEF249]], implicit [[DEF250]], implicit [[DEF251]], implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]]
+  ; GFX90A-NEXT:   [[DEF256:%[0-9]+]]:agpr_32 = IMPLICIT_DEF
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF256]], implicit [[DEF]], implicit [[DEF1]], implicit [[DEF2]], implicit [[DEF3]], implicit [[DEF4]], implicit [[DEF5]], implicit [[DEF6]], implicit [[DEF7]], implicit [[DEF8]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF9]], implicit [[DEF10]], implicit [[DEF11]], implicit [[DEF12]], implicit [[DEF13]], implicit [[DEF14]], implicit [[DEF15]], implicit [[DEF16]], implicit [[DEF17]], implicit [[DEF18]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF19]], implicit [[DEF20]], implicit [[DEF21]], implicit [[DEF22]], implicit [[DEF23]], implicit [[DEF24]], implicit [[DEF25]], implicit [[DEF26]], implicit [[DEF27]], implicit [[DEF28]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF29]], implicit [[DEF30]], implicit [[DEF31]], implicit [[DEF32]], implicit [[DEF33]], implicit [[DEF34]], implicit [[DEF35]], implicit [[DEF36]], implicit [[DEF37]], implicit [[DEF38]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF39]], implicit [[DEF40]], implicit [[DEF41]], implicit [[DEF42]], implicit [[DEF43]], implicit [[DEF44]], implicit [[DEF45]], implicit [[DEF46]], implicit [[DEF47]], implicit [[DEF48]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF49]], implicit [[DEF50]], implicit [[DEF51]], implicit [[DEF52]], implicit [[DEF53]], implicit [[DEF54]], implicit [[DEF55]], implicit [[DEF56]], implicit [[DEF57]], implicit [[DEF58]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF59]], implicit [[DEF60]], implicit [[DEF61]], implicit [[DEF62]], implicit [[DEF63]], implicit [[DEF64]], implicit [[DEF65]], implicit [[DEF66]], implicit [[DEF67]], implicit [[DEF68]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF69]], implicit [[DEF70]], implicit [[DEF71]], implicit [[DEF72]], implicit [[DEF73]], implicit [[DEF74]], implicit [[DEF75]], implicit [[DEF76]], implicit [[DEF77]], implicit [[DEF78]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF79]], implicit [[DEF80]], implicit [[DEF81]], implicit [[DEF82]], implicit [[DEF83]], implicit [[DEF84]], implicit [[DEF85]], implicit [[DEF86]], implicit [[DEF87]], implicit [[DEF88]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF89]], implicit [[DEF90]], implicit [[DEF91]], implicit [[DEF92]], implicit [[DEF93]], implicit [[DEF94]], implicit [[DEF95]], implicit [[DEF96]], implicit [[DEF97]], implicit [[DEF98]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF99]], implicit [[DEF100]], implicit [[DEF101]], implicit [[DEF102]], implicit [[DEF103]], implicit [[DEF104]], implicit [[DEF105]], implicit [[DEF106]], implicit [[DEF107]], implicit [[DEF108]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF109]], implicit [[DEF110]], implicit [[DEF111]], implicit [[DEF112]], implicit [[DEF113]], implicit [[DEF114]], implicit [[DEF115]], implicit [[DEF116]], implicit [[DEF117]], implicit [[DEF118]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF119]], implicit [[DEF120]], implicit [[DEF121]], implicit [[DEF122]], implicit [[DEF123]], implicit [[DEF124]], implicit [[DEF125]], implicit [[DEF126]], implicit [[DEF127]], implicit [[DEF128]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF129]], implicit [[DEF130]], implicit [[DEF131]], implicit [[DEF132]], implicit [[DEF133]], implicit [[DEF134]], implicit [[DEF135]], implicit [[DEF136]], implicit [[DEF137]], implicit [[DEF138]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF139]], implicit [[DEF140]], implicit [[DEF141]], implicit [[DEF142]], implicit [[DEF143]], implicit [[DEF144]], implicit [[DEF145]], implicit [[DEF146]], implicit [[DEF147]], implicit [[DEF148]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF149]], implicit [[DEF150]], implicit [[DEF151]], implicit [[DEF152]], implicit [[DEF153]], implicit [[DEF154]], implicit [[DEF155]], implicit [[DEF156]], implicit [[DEF157]], implicit [[DEF158]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF159]], implicit [[DEF160]], implicit [[DEF161]], implicit [[DEF162]], implicit [[DEF163]], implicit [[DEF164]], implicit [[DEF165]], implicit [[DEF166]], implicit [[DEF167]], implicit [[DEF168]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF169]], implicit [[DEF170]], implicit [[DEF171]], implicit [[DEF172]], implicit [[DEF173]], implicit [[DEF174]], implicit [[DEF175]], implicit [[DEF176]], implicit [[DEF177]], implicit [[DEF178]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF179]], implicit [[DEF180]], implicit [[DEF181]], implicit [[DEF182]], implicit [[DEF183]], implicit [[DEF184]], implicit [[DEF185]], implicit [[DEF186]], implicit [[DEF187]], implicit [[DEF188]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF189]], implicit [[DEF190]], implicit [[DEF191]], implicit [[DEF192]], implicit [[DEF193]], implicit [[DEF194]], implicit [[DEF195]], implicit [[DEF196]], implicit [[DEF197]], implicit [[DEF198]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF199]], implicit [[DEF200]], implicit [[DEF201]], implicit [[DEF202]], implicit [[DEF203]], implicit [[DEF204]], implicit [[DEF205]], implicit [[DEF206]], implicit [[DEF207]], implicit [[DEF208]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF209]], implicit [[DEF210]], implicit [[DEF211]], implicit [[DEF212]], implicit [[DEF213]], implicit [[DEF214]], implicit [[DEF215]], implicit [[DEF216]], implicit [[DEF217]], implicit [[DEF218]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF219]], implicit [[DEF220]], implicit [[DEF221]], implicit [[DEF222]], implicit [[DEF223]], implicit [[DEF224]], implicit [[DEF225]], implicit [[DEF226]], implicit [[DEF227]], implicit [[DEF228]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF229]], implicit [[DEF230]], implicit [[DEF231]], implicit [[DEF232]], implicit [[DEF233]], implicit [[DEF234]], implicit [[DEF235]], implicit [[DEF236]], implicit [[DEF237]], implicit [[DEF238]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF239]], implicit [[DEF240]], implicit [[DEF241]], implicit [[DEF242]], implicit [[DEF243]], implicit [[DEF244]], implicit [[DEF245]], implicit [[DEF246]], implicit [[DEF247]], implicit [[DEF248]]
+  ; GFX90A-NEXT:   S_NOP 0, implicit [[DEF249]], implicit [[DEF250]], implicit [[DEF251]], implicit [[DEF252]], implicit [[DEF253]], implicit [[DEF254]], implicit [[DEF255]], implicit [[V_CVT_I32_F64_e32_]], implicit [[V_CVT_I32_F64_e32_1]]
   ; GFX90A-NEXT:   S_ENDPGM 0
 
   bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
index f69337e67ba8a..06d8474b9054b 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-scheduler-sink-trivial-remats.mir
@@ -2104,13 +2104,9 @@ body:             |
   ; GFX908-NEXT:   [[S_MOV_B32_58:%[0-9]+]]:sgpr_32 = S_MOV_B32 69
   ; GFX908-NEXT:   [[S_MOV_B32_59:%[0-9]+]]:sgpr_32 = S_MOV_B32 70
   ; GFX908-NEXT:   [[S_MOV_B32_60:%[0-9]+]]:sgpr_32 = S_MOV_B32 71
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 20, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[S_MOV_B32_61:%[0-9]+]]:sgpr_32 = S_MOV_B32 72
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 21, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[S_MOV_B32_62:%[0-9]+]]:sgpr_32 = S_MOV_B32 73
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[S_MOV_B32_63:%[0-9]+]]:sgpr_32 = S_MOV_B32 74
-  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
   ; GFX908-NEXT:   [[S_MOV_B32_64:%[0-9]+]]:sgpr_32 = S_MOV_B32 75
   ; GFX908-NEXT:   [[S_MOV_B32_65:%[0-9]+]]:sgpr_32 = S_MOV_B32 76
   ; GFX908-NEXT:   [[S_MOV_B32_66:%[0-9]+]]:sgpr_32 = S_MOV_B32 77
@@ -2120,7 +2116,11 @@ body:             |
   ; GFX908-NEXT:   [[S_MOV_B32_70:%[0-9]+]]:sgpr_32 = S_MOV_B32 81
   ; GFX908-NEXT:   [[S_MOV_B32_71:%[0-9]+]]:sgpr_32 = S_MOV_B32 82
   ; GFX908-NEXT:   [[S_MOV_B32_72:%[0-9]+]]:sgpr_32 = S_MOV_B32 83
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_20:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 20, implicit $exec, implicit $mode, implicit-def $m0
   ; GFX908-NEXT:   [[S_MOV_B32_73:%[0-9]+]]:sgpr_32 = S_MOV_B32 84
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_21:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 21, implicit $exec, implicit $mode, implicit-def $m0
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_22:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 22, implicit $exec, implicit $mode, implicit-def $m0
+  ; GFX908-NEXT:   [[V_CVT_I32_F64_e32_23:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 23, implicit $exec, implicit $mode
   ; GFX908-NEXT: {{  $}}
   ; GFX908-NEXT: bb.1:
   ; GFX908-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
index 553d7e09390fd..680942fcb4d4b 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
@@ -279,11 +279,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v58
 ; CHECK-NEXT:    s_branch .LBB0_7
-; CHECK-NEXT:  .LBB0_16: ; %Flow45
+; CHECK-NEXT:  .LBB0_16: ; %Flow43
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s69
 ; CHECK-NEXT:    v_mov_b32_e32 v57, v0
-; CHECK-NEXT:  .LBB0_17: ; %Flow46
+; CHECK-NEXT:  .LBB0_17: ; %Flow44
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s68
 ; CHECK-NEXT:    s_mov_b32 s55, exec_lo
@@ -330,11 +330,11 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; CHECK-NEXT:    ds_write_b32 v0, v57
 ; CHECK-NEXT:    s_branch .LBB0_19
-; CHECK-NEXT:  .LBB0_22: ; %Flow43
+; CHECK-NEXT:  .LBB0_22: ; %Flow41
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    s_inst_prefetch 0x2
 ; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s68
-; CHECK-NEXT:  .LBB0_23: ; %Flow44
+; CHECK-NEXT:  .LBB0_23: ; %Flow42
 ; CHECK-NEXT:    ; in Loop: Header=BB0_5 Depth=1
 ; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s55
 ; CHECK-NEXT:  ; %bb.24: ; in Loop: Header=BB0_5 Depth=1
@@ -347,7 +347,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_or_b32 s53, s4, s53
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s53
 ; CHECK-NEXT:    s_cbranch_execnz .LBB0_5
-; CHECK-NEXT:  .LBB0_25: ; %Flow51
+; CHECK-NEXT:  .LBB0_25: ; %Flow49
 ; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s52
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 1
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
index 11cda2d4171ed..c96ba754c0811 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
@@ -199,7 +199,6 @@ define float @v_mad_mix_f32_bf16lo_bf16lo_negabsf32(bfloat %src0, bfloat %src1,
   ret float %result
 }
 
-
 define float @v_mad_mix_f32_bf16lo_bf16lo_f32imm1(bfloat %src0, bfloat %src1) #0 {
 ; GFX1250-LABEL: v_mad_mix_f32_bf16lo_bf16lo_f32imm1:
 ; GFX1250:       ; %bb.0:
@@ -230,7 +229,6 @@ define float @v_mad_mix_f32_bf16lo_bf16lo_f32imminv2pi(bfloat %src0, bfloat %src
   ret float %result
 }
 
-
 define float @v_mad_mix_f32_bf16lo_bf16lo_cvtbf16imminv2pi(bfloat %src0, bfloat %src1) #0 {
 ; GFX1250-LABEL: v_mad_mix_f32_bf16lo_bf16lo_cvtbf16imminv2pi:
 ; GFX1250:       ; %bb.0:
@@ -247,7 +245,6 @@ define float @v_mad_mix_f32_bf16lo_bf16lo_cvtbf16imminv2pi(bfloat %src0, bfloat
   ret float %result
 }
 
-
 define float @v_mad_mix_f32_bf16lo_bf16lo_cvtbf16imm63(bfloat %src0, bfloat %src1) #0 {
 ; GFX1250-LABEL: v_mad_mix_f32_bf16lo_bf16lo_cvtbf16imm63:
 ; GFX1250:       ; %bb.0:
@@ -360,7 +357,6 @@ define float @no_mix_simple_fabs(float %src0, float %src1, float %src2) #0 {
   ret float %result
 }
 
-
 define float @v_mad_mix_f32_bf16lo_bf16lo_bf16lo_f32_denormals(bfloat %src0, bfloat %src1, bfloat %src2) #1 {
 ; GFX1250-LABEL: v_mad_mix_f32_bf16lo_bf16lo_bf16lo_f32_denormals:
 ; GFX1250:       ; %bb.0:
@@ -469,7 +465,6 @@ define float @v_mad_mix_f32_negprecvtbf16lo_bf16lo_bf16lo(i32 %src0.arg, bfloat
   ret float %result
 }
 
-
 define float @v_mad_mix_f32_precvtnegbf16hi_abs_bf16lo_bf16lo(i32 %src0.arg, bfloat %src1, bfloat %src2) #0 {
 ; GFX1250-LABEL: v_mad_mix_f32_precvtnegbf16hi_abs_bf16lo_bf16lo:
 ; GFX1250:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
index 1b2eb83ba1726..03304ae3946b3 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
@@ -74,9 +74,7 @@ define bfloat @v_mad_mixlo_bf16_bf16lo_bf16lo_f32_clamp_post_cvt(bfloat %src0, b
 ; GFX1250:       ; %bb.0:
 ; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,0]
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT:    v_pk_max_num_bf16 v0, v0, v0 clamp
+; GFX1250-NEXT:    v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,0] clamp
 ; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %src0.ext = fpext bfloat %src0 to float
   %src1.ext = fpext bfloat %src1 to float
@@ -105,7 +103,6 @@ define bfloat @v_mad_mixlo_bf16_bf16lo_bf16lo_f32_clamp_pre_cvt(bfloat %src0, bf
   ret bfloat %cvt.result
 }
 
-
 define <2 x bfloat> @v_mad_mix_v2f32(<2 x bfloat> %src0, <2 x bfloat> %src1, <2 x bfloat> %src2) #0 {
 ; GFX1250-LABEL: v_mad_mix_v2f32:
 ; GFX1250:       ; %bb.0:
@@ -178,7 +175,6 @@ define <4 x bfloat> @v_mad_mix_v4f32(<4 x bfloat> %src0, <4 x bfloat> %src1, <4
   ret <4 x bfloat> %cvt.result
 }
 
-
 define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt(<2 x bfloat> %src0, <2 x bfloat> %src1, <2 x bfloat> %src2) #0 {
 ; GFX1250-LABEL: v_mad_mix_v2f32_clamp_postcvt:
 ; GFX1250:       ; %bb.0:
@@ -191,9 +187,7 @@ define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt(<2 x bfloat> %src0, <2 x bflo
 ; GFX1250-NEXT:    v_lshlrev_b32_e32 v0, 16, v2
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX1250-NEXT:    v_pk_fma_f32 v[0:1], v[4:5], v[6:7], v[0:1]
-; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v1
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT:    v_pk_max_num_bf16 v0, v0, v0 clamp
+; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v1 clamp
 ; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %src0.ext = fpext <2 x bfloat> %src0 to <2 x float>
   %src1.ext = fpext <2 x bfloat> %src1 to <2 x float>
@@ -205,7 +199,6 @@ define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt(<2 x bfloat> %src0, <2 x bflo
   ret <2 x bfloat> %clamp
 }
 
-
 define <3 x bfloat> @v_mad_mix_v3f32_clamp_postcvt(<3 x bfloat> %src0, <3 x bfloat> %src1, <3 x bfloat> %src2) #0 {
 ; GFX1250-LABEL: v_mad_mix_v3f32_clamp_postcvt:
 ; GFX1250:       ; %bb.0:
@@ -247,11 +240,8 @@ define <4 x bfloat> @v_mad_mix_v4f32_clamp_postcvt(<4 x bfloat> %src0, <4 x bflo
 ; GFX1250-NEXT:    v_pk_fma_f32 v[0:1], v[6:7], v[0:1], v[2:3]
 ; GFX1250-NEXT:    v_pk_fma_f32 v[2:3], v[8:9], v[10:11], v[12:13]
 ; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v1
-; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v1, v2, v3
-; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT:    v_pk_max_num_bf16 v0, v0, v0 clamp
-; GFX1250-NEXT:    v_pk_max_num_bf16 v1, v1, v1 clamp
+; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v0, v0, v1 clamp
+; GFX1250-NEXT:    v_cvt_pk_bf16_f32 v1, v2, v3 clamp
 ; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %src0.ext = fpext <4 x bfloat> %src0 to <4 x float>
   %src1.ext = fpext <4 x bfloat> %src1 to <4 x float>
@@ -323,7 +313,6 @@ define <2 x bfloat> @v_mad_mix_v2f32_clamp_postcvt_hi(<2 x bfloat> %src0, <2 x b
   ret <2 x bfloat> %insert
 }
 
-
 define <2 x bfloat> @v_mad_mix_v2f32_clamp_precvt(<2 x bfloat> %src0, <2 x bfloat> %src1, <2 x bfloat> %src2) #0 {
 ; GFX1250-LABEL: v_mad_mix_v2f32_clamp_precvt:
 ; GFX1250:       ; %bb.0:
@@ -351,7 +340,6 @@ define <2 x bfloat> @v_mad_mix_v2f32_clamp_precvt(<2 x bfloat> %src0, <2 x bfloa
   ret <2 x bfloat> %cvt.result
 }
 
-
 define <3 x bfloat> @v_mad_mix_v3f32_clamp_precvt(<3 x bfloat> %src0, <3 x bfloat> %src1, <3 x bfloat> %src2) #0 {
 ; GFX1250-LABEL: v_mad_mix_v3f32_clamp_precvt:
 ; GFX1250:       ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index 811e25587d3d5..eab92668c536b 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -2382,13 +2382,22 @@ define <4 x half> @v_mad_mix_v4f32_clamp_precvt(<4 x half> %src0, <4 x half> %sr
 }
 
 define i32 @mixlo_zext(float %src0, float %src1, float %src2) #0 {
-; GFX1100-LABEL: mixlo_zext:
-; GFX1100:       ; %bb.0:
-; GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT:    v_fma_mixlo_f16 v0, v0, v1, v2
-; GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX1100-NEXT:    s_setpc_b64 s[30:31]
+; SDAG-GFX1100-TRUE16-LABEL: mixlo_zext:
+; SDAG-GFX1100-TRUE16:       ; %bb.0:
+; SDAG-GFX1100-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-TRUE16-NEXT:    v_fma_mixlo_f16 v1, v0, v1, v2
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
+; SDAG-GFX1100-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; SDAG-GFX1100-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.l
+; SDAG-GFX1100-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; SDAG-GFX1100-FAKE16-LABEL: mixlo_zext:
+; SDAG-GFX1100-FAKE16:       ; %bb.0:
+; SDAG-GFX1100-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-FAKE16-NEXT:    v_fma_mixlo_f16 v0, v0, v1, v2
+; SDAG-GFX1100-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; SDAG-GFX1100-FAKE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX900-LABEL: mixlo_zext:
 ; GFX900:       ; %bb.0:
@@ -2418,6 +2427,14 @@ define i32 @mixlo_zext(float %src0, float %src1, float %src2) #0 {
 ; SDAG-CI-NEXT:    v_cvt_f16_f32_e32 v0, v2
 ; SDAG-CI-NEXT:    s_setpc_b64 s[30:31]
 ;
+; GISEL-GFX1100-LABEL: mixlo_zext:
+; GISEL-GFX1100:       ; %bb.0:
+; GISEL-GFX1100-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX1100-NEXT:    v_fma_mixlo_f16 v0, v0, v1, v2
+; GISEL-GFX1100-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GISEL-GFX1100-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GISEL-GFX1100-NEXT:    s_setpc_b64 s[30:31]
+;
 ; GISEL-CI-LABEL: mixlo_zext:
 ; GISEL-CI:       ; %bb.0:
 ; GISEL-CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/mad.u16.ll b/llvm/test/CodeGen/AMDGPU/mad.u16.ll
index ef80323a98ec0..fbf8011fd40c9 100644
--- a/llvm/test/CodeGen/AMDGPU/mad.u16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad.u16.ll
@@ -179,8 +179,7 @@ define i32 @v_mad_u16_zext(i16 %arg0, i16 %arg1, i16 %arg2) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_mad_u16_zext:
@@ -222,9 +221,9 @@ define i64 @v_mad_u16_zext64(i16 %arg0, i16 %arg1, i16 %arg2) {
 ; GFX11-TRUE16-LABEL: v_mad_u16_zext64:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    v_mad_u16 v0.l, v0.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: v_mad_u16_zext64:
diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index cf9a700cd64f8..b8f9571ccc2ee 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -5,6 +5,7 @@
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11,GFX1100 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1150 < %s | FileCheck -check-prefixes=GFX11,GFX1150 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250 %s
 ; RUN: llc -mtriple=amdgcn -mcpu=gfx11-generic --amdhsa-code-object-version=6 < %s | FileCheck -check-prefixes=GFX11,GFX1100 %s
 
 ; On GFX11, ensure vdst and src2 do not partially overlap. Full overlap is ok.
@@ -54,6 +55,13 @@ define i64 @mad_i64_i32_sextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v0, v1, v[2:3]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_sextops:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_i64_i32 v[0:1], v0, v1, v[2:3]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
@@ -106,6 +114,13 @@ define i64 @mad_i64_i32_sextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v0, v1, v[2:3]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_sextops_commute:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_i64_i32 v[0:1], v0, v1, v[2:3]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
@@ -158,6 +173,13 @@ define i64 @mad_u64_u32_zextops(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v0, v1, v[2:3]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_u64_u32_zextops:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v0, v1, v[2:3]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = zext i32 %arg0 to i64
   %sext1 = zext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
@@ -210,6 +232,13 @@ define i64 @mad_u64_u32_zextops_commute(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v0, v1, v[2:3]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_u64_u32_zextops_commute:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v0, v1, v[2:3]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = zext i32 %arg0 to i64
   %sext1 = zext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
@@ -393,6 +422,38 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; GFX12-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo
 ; GFX12-NEXT:    s_wait_alu 0xfffd
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_sextops_i32_i128:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v9, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v1, v9
+; GFX1250-NEXT:    v_mov_b32_e32 v21, v9
+; GFX1250-NEXT:    v_mul_u64_e32 v[10:11], v[0:1], v[8:9]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_dual_ashrrev_i32 v12, 31, v0 :: v_dual_mov_b32 v8, v11
+; GFX1250-NEXT:    v_dual_ashrrev_i32 v7, 31, v6 :: v_dual_mov_b32 v13, v12
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[14:15], v12, v6, v[8:9]
+; GFX1250-NEXT:    v_mul_u64_e32 v[16:17], v[6:7], v[12:13]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v8, v14
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[18:19], v0, v7, v[8:9]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v8, v15 :: v_dual_mov_b32 v20, v19
+; GFX1250-NEXT:    v_add_nc_u64_e32 v[8:9], v[8:9], v[20:21]
+; GFX1250-NEXT:    v_mad_nc_i64_i32 v[0:1], v7, v0, v[16:17]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[8:9], v12, v7, v[8:9]
+; GFX1250-NEXT:    v_add_nc_u64_e32 v[6:7], v[8:9], v[0:1]
+; GFX1250-NEXT:    v_add_co_u32 v0, vcc_lo, v10, v2
+; GFX1250-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v18, v3, vcc_lo
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v6, v4, vcc_lo
+; GFX1250-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v7, v5, vcc_lo
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = sext i32 %arg0 to i128
   %sext1 = sext i32 %arg1 to i128
   %mul = mul i128 %sext0, %sext1
@@ -445,6 +506,13 @@ define i63 @mad_i64_i32_sextops_i32_i63(i32 %arg0, i32 %arg1, i63 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v0, v1, v[2:3]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_sextops_i32_i63:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_i64_i32 v[0:1], v0, v1, v[2:3]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = sext i32 %arg0 to i63
   %sext1 = sext i32 %arg1 to i63
   %mul = mul i63 %sext0, %sext1
@@ -510,6 +578,16 @@ define i63 @mad_i64_i32_sextops_i31_i63(i31 %arg0, i31 %arg1, i63 %arg2) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v0, v1, v[2:3]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_sextops_i31_i63:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_bfe_i32 v1, v1, 0, 31
+; GFX1250-NEXT:    v_bfe_i32 v0, v0, 0, 31
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_i64_i32 v[0:1], v0, v1, v[2:3]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = sext i31 %arg0 to i63
   %sext1 = sext i31 %arg1 to i63
   %mul = mul i63 %sext0, %sext1
@@ -585,6 +663,17 @@ define i64 @mad_i64_i32_extops_i32_i64(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], null, v2, v4, v[1:2]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_extops_i32_i64:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v5, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v5, v4, v[2:3]
+; GFX1250-NEXT:    v_ashrrev_i32_e32 v2, 31, v5
+; GFX1250-NEXT:    v_mad_u32 v1, v2, v4, v1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ext0 = sext i32 %arg0 to i64
   %ext1 = zext i32 %arg1 to i64
   %mul = mul i64 %ext0, %ext1
@@ -637,6 +726,13 @@ define i64 @mad_u64_u32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v0, v2, v[4:5]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_u64_u32_bitops:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v0, v2, v[4:5]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %trunc.lhs = and i64 %arg0, 4294967295
   %trunc.rhs = and i64 %arg1, 4294967295
   %mul = mul i64 %trunc.lhs, %trunc.rhs
@@ -711,6 +807,17 @@ define i64 @mad_u64_u32_bitops_lhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #
 ; GFX12-NEXT:    v_and_b32_e32 v3, 1, v3
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], null, v3, v2, v[1:2]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_u64_u32_bitops_lhs_mask_small:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v0, v2, v[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX1250-NEXT:    v_mad_u32 v1, v3, v2, v1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %trunc.lhs = and i64 %arg0, 8589934591
   %trunc.rhs = and i64 %arg1, 4294967295
   %mul = mul i64 %trunc.lhs, %trunc.rhs
@@ -786,6 +893,17 @@ define i64 @mad_u64_u32_bitops_rhs_mask_small(i64 %arg0, i64 %arg1, i64 %arg2) #
 ; GFX12-NEXT:    v_and_b32_e32 v2, 1, v3
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[1:2], null, v6, v2, v[1:2]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_u64_u32_bitops_rhs_mask_small:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v6, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v6, v2, v[4:5]
+; GFX1250-NEXT:    v_and_b32_e32 v2, 1, v3
+; GFX1250-NEXT:    v_mad_u32 v1, v6, v2, v1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %trunc.lhs = and i64 %arg0, 4294967295
   %trunc.rhs = and i64 %arg1, 8589934591
   %mul = mul i64 %trunc.lhs, %trunc.rhs
@@ -838,6 +956,13 @@ define i64 @mad_i64_i32_bitops(i64 %arg0, i64 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, v0, v2, v[4:5]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_bitops:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_i64_i32 v[0:1], v0, v2, v[4:5]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %shl.lhs = shl i64 %arg0, 32
   %trunc.lhs = ashr i64 %shl.lhs, 32
   %shl.rhs = shl i64 %arg1, 32
@@ -893,6 +1018,13 @@ define i64 @mad_i64_i32_unpack_i64ops(i64 %arg0) #0 {
 ; GFX12-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, v1, v0, v[0:1]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_unpack_i64ops:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v1, v0, v[0:1]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %tmp4 = lshr i64 %arg0, 32
   %tmp5 = and i64 %arg0, 4294967295
   %mul = mul nuw i64 %tmp4, %tmp5
@@ -982,6 +1114,25 @@ define amdgpu_kernel void @mad_i64_i32_uniform(ptr addrspace(1) %out, i32 %arg0,
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX12-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: mad_i64_i32_uniform:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1250-NEXT:    s_mov_b32 s7, 0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s6, s2
+; GFX1250-NEXT:    s_mov_b32 s2, s3
+; GFX1250-NEXT:    s_mov_b32 s3, s7
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_mul_u64 s[2:3], s[6:7], s[2:3]
+; GFX1250-NEXT:    s_add_nc_u64 s[2:3], s[2:3], s[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-NEXT:    s_endpgm
   %ext0 = zext i32 %arg0 to i64
   %ext1 = zext i32 %arg1 to i64
   %mul = mul i64 %ext0, %ext1
@@ -1055,6 +1206,17 @@ define i64 @mad_i64_i32_twice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3) #0 {
 ; GFX12-NEXT:    v_xor_b32_e32 v0, v2, v0
 ; GFX12-NEXT:    v_xor_b32_e32 v1, v3, v1
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_twice:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_i64_i32 v[2:3], v0, v1, v[2:3]
+; GFX1250-NEXT:    v_mad_nc_i64_i32 v[0:1], v0, v1, v[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_xor_b32_e32 v0, v2, v0
+; GFX1250-NEXT:    v_xor_b32_e32 v1, v3, v1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
@@ -1174,6 +1336,26 @@ define i64 @mad_i64_i32_thrice(i32 %arg0, i32 %arg1, i64 %arg2, i64 %arg3, i64 %
 ; GFX12-NEXT:    v_xor_b32_e32 v0, v2, v0
 ; GFX12-NEXT:    v_xor_b32_e32 v1, v3, v1
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_thrice:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v8, v1 :: v_dual_ashrrev_i32 v1, 31, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GFX1250-NEXT:    v_mul_u64_e32 v[0:1], v[0:1], v[8:9]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_add_nc_u64_e32 v[2:3], v[0:1], v[2:3]
+; GFX1250-NEXT:    v_add_nc_u64_e32 v[4:5], v[0:1], v[4:5]
+; GFX1250-NEXT:    v_add_nc_u64_e32 v[0:1], v[0:1], v[6:7]
+; GFX1250-NEXT:    v_xor_b32_e32 v2, v2, v4
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_xor_b32_e32 v3, v3, v5
+; GFX1250-NEXT:    v_xor_b32_e32 v0, v2, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT:    v_xor_b32_e32 v1, v3, v1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
@@ -1256,6 +1438,21 @@ define i64 @mad_i64_i32_secondary_use(i32 %arg0, i32 %arg1, i64 %arg2) #0 {
 ; GFX12-NEXT:    v_xor_b32_e32 v0, v2, v0
 ; GFX12-NEXT:    v_xor_b32_e32 v1, v3, v1
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i64_i32_secondary_use:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v4, v1 :: v_dual_ashrrev_i32 v1, 31, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
+; GFX1250-NEXT:    v_mul_u64_e32 v[0:1], v[0:1], v[4:5]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_add_nc_u64_e32 v[2:3], v[0:1], v[2:3]
+; GFX1250-NEXT:    v_xor_b32_e32 v0, v2, v0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT:    v_xor_b32_e32 v1, v3, v1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %sext0 = sext i32 %arg0 to i64
   %sext1 = sext i32 %arg1 to i64
   %mul = mul i64 %sext0, %sext1
@@ -1328,6 +1525,18 @@ define i48 @mad_i48_i48(i48 %arg0, i48 %arg1, i48 %arg2) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_add3_u32 v1, v2, v1, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: mad_i48_i48:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, v1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], v6, v2, v[4:5]
+; GFX1250-NEXT:    v_mad_u32 v1, v7, v2, v1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_u32 v1, v6, v3, v1
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %m = mul i48 %arg0, %arg1
   %a = add i48 %m, %arg2
   ret i48 %a
@@ -1391,6 +1600,15 @@ define i64 @lshr_mad_i64_1(i64 %arg0, i64 %arg1) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, 0xfffffc19, v2, v[0:1]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: lshr_mad_i64_1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], 0xfffffc19, v2, v[0:1]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %lsh = lshr i64 %arg0, 32
   %mul = mul i64 %lsh, s0xfffffffffffffc19
   %mad = add i64 %mul, %arg0
@@ -1456,6 +1674,15 @@ define i64 @lshr_mad_i64_2(i64 %arg0) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, 0xd1, v2, v[0:1]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: lshr_mad_i64_2:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], 0xd1, v2, v[0:1]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %lsh = lshr i64 %arg0, 32
   %mul = mul i64 %lsh, s0xffffffff000000d1
   %mad = add i64 %mul, %arg0
@@ -1521,6 +1748,15 @@ define i64 @lshr_mad_i64_3(i64 %arg0) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, 0xfffffc88, v2, v[0:1]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: lshr_mad_i64_3:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], 0xfffffc88, v2, v[0:1]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %lsh = lshr i64 %arg0, 32
   %mul = mul i64 s0xfffffffffffffc88, %lsh
   %mad = add i64 %mul, %arg0
@@ -1602,6 +1838,19 @@ define i64 @lshr_mad_i64_4(i32 %arg0, i64 %arg1) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_u64_u32 v[0:1], null, 0xfffffc88, v0, v[3:4]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: lshr_mad_i64_4:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v2, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_mul_u64_e32 v[2:3], v[2:3], v[0:1]
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[0:1], 0xfffffc88, v3, v[0:1]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %ext = zext i32 %arg0 to i64
   %mul1 = mul i64 %arg1, %ext
   %lsh = lshr i64 %mul1, 32
@@ -1666,6 +1915,15 @@ define i64 @lshr_mad_i64_negative_1(i64 %arg0) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-NEXT:    v_mad_co_i64_i32 v[0:1], null, 0xfffffc19, v2, v[0:1]
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: lshr_mad_i64_negative_1:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_lshrrev_b32_e32 v2, 4, v1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_mad_nc_i64_i32 v[0:1], 0xfffffc19, v2, v[0:1]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %lsh = lshr i64 %arg0, 36
   %mul = mul i64 %lsh, s0xfffffffffffffc19
   %mad = add i64 %mul, %arg0
@@ -1729,6 +1987,16 @@ define i64 @lshr_mad_i64_negative_2(i64 %arg0) #0 {
 ; GFX12-NEXT:    v_sub_nc_u32_e32 v1, v3, v0
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: lshr_mad_i64_negative_2:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[2:3], 0xd1, v1, v[0:1]
+; GFX1250-NEXT:    v_lshlrev_b32_e32 v0, 8, v1
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v2 :: v_dual_sub_nc_u32 v1, v3, v0
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %lsh = lshr i64 %arg0, 32
   %mul = mul i64 %lsh, s0xffffff00000000d1
   %mad = add i64 %mul, %arg0
@@ -1803,6 +2071,18 @@ define i64 @lshr_mad_i64_negative_3(i64 %arg0) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX12-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: lshr_mad_i64_negative_3:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_lshrrev_b64 v[2:3], 22, v[0:1]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT:    v_and_b32_e32 v2, 0xfffffc00, v2
+; GFX1250-NEXT:    v_sub_nc_u64_e32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_add_nc_u64_e32 v[0:1], 1, v[0:1]
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %op = add i64 %arg0, 1
   %lsh = lshr i64 %arg0, 32
   %mul = mul i64 %lsh, s0xfffffffffffffc00
@@ -1878,6 +2158,16 @@ define i64 @lshr_mad_i64_negative_4(i64 %arg0) #0 {
 ; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_3)
 ; GFX12-NEXT:    v_mov_b32_e32 v0, v3
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: lshr_mad_i64_negative_4:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[2:3], v1, v0, v[0:1]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_mad_u32 v1, v1, v1, v3
+; GFX1250-NEXT:    v_mov_b32_e32 v0, v2
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %lsh = lshr i64 %arg0, 32
   %mul = mul i64 %lsh, %arg0
   %mad = add i64 %mul, %arg0
@@ -1938,6 +2228,16 @@ define amdgpu_ps i64 @lshr_mad_i64_sgpr(i64 inreg %arg0) #0 {
 ; GFX12-NEXT:    s_mul_u64 s[2:3], s[2:3], s[4:5]
 ; GFX12-NEXT:    s_add_nc_u64 s[0:1], s[2:3], s[0:1]
 ; GFX12-NEXT:    ; return to shader part epilog
+;
+; GFX1250-LABEL: lshr_mad_i64_sgpr:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_mov_b32 s3, 0
+; GFX1250-NEXT:    s_mov_b32 s2, s1
+; GFX1250-NEXT:    s_mov_b64 s[4:5], lit64(0xffffffffffff1c18)
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_mul_u64 s[2:3], s[2:3], s[4:5]
+; GFX1250-NEXT:    s_add_nc_u64 s[0:1], s[2:3], s[0:1]
+; GFX1250-NEXT:    ; return to shader part epilog
   %lsh = lshr i64 %arg0, 32
   %mul = mul i64 %lsh, s0xffffffffffff1c18
   %mad = add i64 %mul, %arg0
@@ -2018,6 +2318,17 @@ define <2 x i64> @lshr_mad_i64_vec(<2 x i64> %arg0) #0 {
 ; GFX12-NEXT:    v_sub_nc_u32_e32 v3, v7, v3
 ; GFX12-NEXT:    v_mov_b32_e32 v2, v6
 ; GFX12-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX1250-LABEL: lshr_mad_i64_vec:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[4:5], 0xffff1c18, v1, v[0:1]
+; GFX1250-NEXT:    v_mad_nc_u64_u32 v[6:7], 0xffff1118, v3, v[2:3]
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT:    v_dual_sub_nc_u32 v1, v5, v1 :: v_dual_sub_nc_u32 v3, v7, v3
+; GFX1250-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v2, v6
+; GFX1250-NEXT:    s_set_pc_i64 s[30:31]
   %lsh = lshr <2 x i64> %arg0, <i64 32, i64 32>
   %mul = mul <2 x i64> %lsh, <i64 s0xffffffffffff1c18, i64 s0xffffffffffff1118>
   %mad = add <2 x i64> %mul, %arg0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-global.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-global.ll
index 80445f793934b..97d52d5f1f26d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-global.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @workgroup_acquire_fence() {
 ; GFX6-LABEL: workgroup_acquire_fence:
@@ -78,6 +79,10 @@ define amdgpu_kernel void @workgroup_acquire_fence() {
 ; GFX12-CU-LABEL: workgroup_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") acquire, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -145,6 +150,10 @@ define amdgpu_kernel void @workgroup_release_fence() {
 ; GFX12-CU-LABEL: workgroup_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") release, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -217,6 +226,10 @@ define amdgpu_kernel void @workgroup_acq_rel_fence() {
 ; GFX12-CU-LABEL: workgroup_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -289,6 +302,10 @@ define amdgpu_kernel void @workgroup_seq_cst_fence() {
 ; GFX12-CU-LABEL: workgroup_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -359,6 +376,10 @@ define amdgpu_kernel void @workgroup_one_as_acquire_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") acquire, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -426,6 +447,10 @@ define amdgpu_kernel void @workgroup_one_as_release_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") release, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -498,6 +523,10 @@ define amdgpu_kernel void @workgroup_one_as_acq_rel_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -570,6 +599,10 @@ define amdgpu_kernel void @workgroup_one_as_seq_cst_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -662,6 +695,13 @@ define amdgpu_kernel void @agent_acquire_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") acquire, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -744,6 +784,14 @@ define amdgpu_kernel void @agent_release_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") release, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -842,6 +890,15 @@ define amdgpu_kernel void @agent_acq_rel_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -940,6 +997,15 @@ define amdgpu_kernel void @agent_seq_cst_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1032,6 +1098,13 @@ define amdgpu_kernel void @agent_one_as_acquire_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") acquire, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1114,6 +1187,14 @@ define amdgpu_kernel void @agent_one_as_release_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") release, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1212,6 +1293,15 @@ define amdgpu_kernel void @agent_one_as_acq_rel_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1310,6 +1400,15 @@ define amdgpu_kernel void @agent_one_as_seq_cst_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1404,6 +1503,13 @@ define amdgpu_kernel void @system_acquire_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence acquire, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1490,6 +1596,15 @@ define amdgpu_kernel void @system_release_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence release, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1594,6 +1709,16 @@ define amdgpu_kernel void @system_acq_rel_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence acq_rel, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1698,6 +1823,16 @@ define amdgpu_kernel void @system_seq_cst_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence seq_cst, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1792,6 +1927,13 @@ define amdgpu_kernel void @system_one_as_acquire_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") acquire, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1878,6 +2020,15 @@ define amdgpu_kernel void @system_one_as_release_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") release, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -1982,6 +2133,16 @@ define amdgpu_kernel void @system_one_as_acq_rel_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
@@ -2086,6 +2247,16 @@ define amdgpu_kernel void @system_one_as_seq_cst_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"global"}
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-local.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-local.ll
index 7a419a5031ba9..cc42428e1aa06 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-local.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence-mmra-local.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @workgroup_acquire_fence() {
 ; GFX6-LABEL: workgroup_acquire_fence:
@@ -76,6 +77,11 @@ define amdgpu_kernel void @workgroup_acquire_fence() {
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") acquire, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -142,6 +148,10 @@ define amdgpu_kernel void @workgroup_release_fence() {
 ; GFX12-CU-LABEL: workgroup_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") release, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -208,6 +218,10 @@ define amdgpu_kernel void @workgroup_acq_rel_fence() {
 ; GFX12-CU-LABEL: workgroup_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -274,6 +288,10 @@ define amdgpu_kernel void @workgroup_seq_cst_fence() {
 ; GFX12-CU-LABEL: workgroup_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -331,6 +349,10 @@ define amdgpu_kernel void @workgroup_one_as_acquire_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") acquire, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -388,6 +410,10 @@ define amdgpu_kernel void @workgroup_one_as_release_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") release, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -445,6 +471,10 @@ define amdgpu_kernel void @workgroup_one_as_acq_rel_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -502,6 +532,10 @@ define amdgpu_kernel void @workgroup_one_as_seq_cst_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -570,6 +604,11 @@ define amdgpu_kernel void @agent_acquire_fence() {
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") acquire, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -636,6 +675,10 @@ define amdgpu_kernel void @agent_release_fence() {
 ; GFX12-CU-LABEL: agent_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") release, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -702,6 +745,10 @@ define amdgpu_kernel void @agent_acq_rel_fence() {
 ; GFX12-CU-LABEL: agent_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -768,6 +815,10 @@ define amdgpu_kernel void @agent_seq_cst_fence() {
 ; GFX12-CU-LABEL: agent_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -825,6 +876,10 @@ define amdgpu_kernel void @agent_one_as_acquire_fence() {
 ; GFX12-CU-LABEL: agent_one_as_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") acquire, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -882,6 +937,10 @@ define amdgpu_kernel void @agent_one_as_release_fence() {
 ; GFX12-CU-LABEL: agent_one_as_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") release, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -939,6 +998,10 @@ define amdgpu_kernel void @agent_one_as_acq_rel_fence() {
 ; GFX12-CU-LABEL: agent_one_as_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -996,6 +1059,10 @@ define amdgpu_kernel void @agent_one_as_seq_cst_fence() {
 ; GFX12-CU-LABEL: agent_one_as_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -1064,6 +1131,11 @@ define amdgpu_kernel void @system_acquire_fence() {
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence acquire, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -1130,6 +1202,10 @@ define amdgpu_kernel void @system_release_fence() {
 ; GFX12-CU-LABEL: system_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence release, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -1196,6 +1272,10 @@ define amdgpu_kernel void @system_acq_rel_fence() {
 ; GFX12-CU-LABEL: system_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence acq_rel, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -1262,6 +1342,10 @@ define amdgpu_kernel void @system_seq_cst_fence() {
 ; GFX12-CU-LABEL: system_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence seq_cst, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -1319,6 +1403,10 @@ define amdgpu_kernel void @system_one_as_acquire_fence() {
 ; GFX12-CU-LABEL: system_one_as_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") acquire, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -1376,6 +1464,10 @@ define amdgpu_kernel void @system_one_as_release_fence() {
 ; GFX12-CU-LABEL: system_one_as_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") release, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -1433,6 +1525,10 @@ define amdgpu_kernel void @system_one_as_acq_rel_fence() {
 ; GFX12-CU-LABEL: system_one_as_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") acq_rel, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
@@ -1490,6 +1586,10 @@ define amdgpu_kernel void @system_one_as_seq_cst_fence() {
 ; GFX12-CU-LABEL: system_one_as_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") seq_cst, !mmra !{!"amdgpu-synchronize-as", !"local"}
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
index 0e459ed0f1243..b3f6533d43887 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-fence.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @singlethread_acquire_fence() {
 ; GFX6-LABEL: singlethread_acquire_fence:
@@ -65,6 +66,10 @@ define amdgpu_kernel void @singlethread_acquire_fence() {
 ; GFX12-CU-LABEL: singlethread_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: singlethread_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("singlethread") acquire
   ret void
@@ -122,6 +127,10 @@ define amdgpu_kernel void @singlethread_release_fence() {
 ; GFX12-CU-LABEL: singlethread_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: singlethread_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("singlethread") release
   ret void
@@ -179,6 +188,10 @@ define amdgpu_kernel void @singlethread_acq_rel_fence() {
 ; GFX12-CU-LABEL: singlethread_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: singlethread_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("singlethread") acq_rel
   ret void
@@ -236,6 +249,10 @@ define amdgpu_kernel void @singlethread_seq_cst_fence() {
 ; GFX12-CU-LABEL: singlethread_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: singlethread_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("singlethread") seq_cst
   ret void
@@ -293,6 +310,10 @@ define amdgpu_kernel void @singlethread_one_as_acquire_fence() {
 ; GFX12-CU-LABEL: singlethread_one_as_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: singlethread_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("singlethread-one-as") acquire
   ret void
@@ -350,6 +371,10 @@ define amdgpu_kernel void @singlethread_one_as_release_fence() {
 ; GFX12-CU-LABEL: singlethread_one_as_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: singlethread_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("singlethread-one-as") release
   ret void
@@ -407,6 +432,10 @@ define amdgpu_kernel void @singlethread_one_as_acq_rel_fence() {
 ; GFX12-CU-LABEL: singlethread_one_as_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: singlethread_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("singlethread-one-as") acq_rel
   ret void
@@ -464,6 +493,10 @@ define amdgpu_kernel void @singlethread_one_as_seq_cst_fence() {
 ; GFX12-CU-LABEL: singlethread_one_as_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: singlethread_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("singlethread-one-as") seq_cst
   ret void
@@ -521,6 +554,10 @@ define amdgpu_kernel void @wavefront_acquire_fence() {
 ; GFX12-CU-LABEL: wavefront_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: wavefront_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("wavefront") acquire
   ret void
@@ -578,6 +615,10 @@ define amdgpu_kernel void @wavefront_release_fence() {
 ; GFX12-CU-LABEL: wavefront_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: wavefront_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("wavefront") release
   ret void
@@ -635,6 +676,10 @@ define amdgpu_kernel void @wavefront_acq_rel_fence() {
 ; GFX12-CU-LABEL: wavefront_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: wavefront_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("wavefront") acq_rel
   ret void
@@ -692,6 +737,10 @@ define amdgpu_kernel void @wavefront_seq_cst_fence() {
 ; GFX12-CU-LABEL: wavefront_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: wavefront_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("wavefront") seq_cst
   ret void
@@ -749,6 +798,10 @@ define amdgpu_kernel void @wavefront_one_as_acquire_fence() {
 ; GFX12-CU-LABEL: wavefront_one_as_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: wavefront_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("wavefront-one-as") acquire
   ret void
@@ -806,6 +859,10 @@ define amdgpu_kernel void @wavefront_one_as_release_fence() {
 ; GFX12-CU-LABEL: wavefront_one_as_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: wavefront_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("wavefront-one-as") release
   ret void
@@ -863,6 +920,10 @@ define amdgpu_kernel void @wavefront_one_as_acq_rel_fence() {
 ; GFX12-CU-LABEL: wavefront_one_as_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: wavefront_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("wavefront-one-as") acq_rel
   ret void
@@ -920,6 +981,10 @@ define amdgpu_kernel void @wavefront_one_as_seq_cst_fence() {
 ; GFX12-CU-LABEL: wavefront_one_as_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: wavefront_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("wavefront-one-as") seq_cst
   ret void
@@ -998,6 +1063,11 @@ define amdgpu_kernel void @workgroup_acquire_fence() {
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") acquire
   ret void
@@ -1073,6 +1143,11 @@ define amdgpu_kernel void @workgroup_release_fence() {
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") release
   ret void
@@ -1153,6 +1228,11 @@ define amdgpu_kernel void @workgroup_acq_rel_fence() {
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") acq_rel
   ret void
@@ -1233,6 +1313,11 @@ define amdgpu_kernel void @workgroup_seq_cst_fence() {
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup") seq_cst
   ret void
@@ -1303,6 +1388,10 @@ define amdgpu_kernel void @workgroup_one_as_acquire_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_acquire_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") acquire
   ret void
@@ -1370,6 +1459,10 @@ define amdgpu_kernel void @workgroup_one_as_release_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_release_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") release
   ret void
@@ -1442,6 +1535,10 @@ define amdgpu_kernel void @workgroup_one_as_acq_rel_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_acq_rel_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") acq_rel
   ret void
@@ -1514,6 +1611,10 @@ define amdgpu_kernel void @workgroup_one_as_seq_cst_fence() {
 ; GFX12-CU-LABEL: workgroup_one_as_seq_cst_fence:
 ; GFX12-CU:       ; %bb.0: ; %entry
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: workgroup_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("workgroup-one-as") seq_cst
   ret void
@@ -1606,6 +1707,13 @@ define amdgpu_kernel void @agent_acquire_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") acquire
   ret void
@@ -1688,6 +1796,14 @@ define amdgpu_kernel void @agent_release_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") release
   ret void
@@ -1786,6 +1902,15 @@ define amdgpu_kernel void @agent_acq_rel_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") acq_rel
   ret void
@@ -1884,6 +2009,15 @@ define amdgpu_kernel void @agent_seq_cst_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent") seq_cst
   ret void
@@ -1976,6 +2110,13 @@ define amdgpu_kernel void @agent_one_as_acquire_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") acquire
   ret void
@@ -2058,6 +2199,14 @@ define amdgpu_kernel void @agent_one_as_release_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") release
   ret void
@@ -2156,6 +2305,15 @@ define amdgpu_kernel void @agent_one_as_acq_rel_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") acq_rel
   ret void
@@ -2254,6 +2412,15 @@ define amdgpu_kernel void @agent_one_as_seq_cst_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: agent_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("agent-one-as") seq_cst
   ret void
@@ -2348,6 +2515,13 @@ define amdgpu_kernel void @system_acquire_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence acquire
   ret void
@@ -2434,6 +2608,15 @@ define amdgpu_kernel void @system_release_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence release
   ret void
@@ -2538,6 +2721,16 @@ define amdgpu_kernel void @system_acq_rel_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence acq_rel
   ret void
@@ -2642,6 +2835,16 @@ define amdgpu_kernel void @system_seq_cst_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence seq_cst
   ret void
@@ -2736,6 +2939,13 @@ define amdgpu_kernel void @system_one_as_acquire_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_acquire_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") acquire
   ret void
@@ -2822,6 +3032,15 @@ define amdgpu_kernel void @system_one_as_release_fence() {
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_release_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") release
   ret void
@@ -2926,6 +3145,16 @@ define amdgpu_kernel void @system_one_as_acq_rel_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_acq_rel_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") acq_rel
   ret void
@@ -3030,6 +3259,16 @@ define amdgpu_kernel void @system_one_as_seq_cst_fence() {
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: system_one_as_seq_cst_fence:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
 entry:
   fence syncscope("one-as") seq_cst
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
index 07ad8cb0c4a3d..36adbc0011118 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
@@ -11,6 +11,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @flat_agent_unordered_load(
 ; GFX7-LABEL: flat_agent_unordered_load:
@@ -187,6 +188,17 @@ define amdgpu_kernel void @flat_agent_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("agent") unordered, align 4
@@ -369,6 +381,17 @@ define amdgpu_kernel void @flat_agent_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("agent") monotonic, align 4
@@ -566,6 +589,18 @@ define amdgpu_kernel void @flat_agent_acquire_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("agent") acquire, align 4
@@ -789,6 +824,24 @@ define amdgpu_kernel void @flat_agent_seq_cst_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("agent") seq_cst, align 4
@@ -939,6 +992,16 @@ define amdgpu_kernel void @flat_agent_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("agent") unordered, align 4
@@ -1088,6 +1151,16 @@ define amdgpu_kernel void @flat_agent_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("agent") monotonic, align 4
@@ -1261,6 +1334,20 @@ define amdgpu_kernel void @flat_agent_release_store(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("agent") release, align 4
@@ -1434,6 +1521,20 @@ define amdgpu_kernel void @flat_agent_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("agent") seq_cst, align 4
@@ -1583,6 +1684,16 @@ define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") monotonic
@@ -1763,6 +1874,18 @@ define amdgpu_kernel void @flat_agent_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acquire
@@ -1936,6 +2059,20 @@ define amdgpu_kernel void @flat_agent_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") release
@@ -2140,6 +2277,22 @@ define amdgpu_kernel void @flat_agent_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acq_rel
@@ -2344,6 +2497,22 @@ define amdgpu_kernel void @flat_agent_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst
@@ -2552,6 +2721,19 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acquire
@@ -2789,6 +2971,25 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") acq_rel
@@ -3026,6 +3227,25 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent") seq_cst
@@ -3264,6 +3484,20 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3533,6 +3767,22 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3795,6 +4045,24 @@ define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4088,6 +4356,26 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4381,6 +4669,26 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4650,6 +4958,22 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4919,6 +5243,22 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5212,6 +5552,26 @@ define amdgpu_kernel void @flat_agent_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5505,6 +5865,26 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5798,6 +6178,26 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6091,6 +6491,26 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6384,6 +6804,26 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6677,6 +7117,26 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6970,6 +7430,26 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7263,6 +7743,26 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7545,6 +8045,22 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7844,6 +8360,23 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8152,6 +8685,26 @@ define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8479,6 +9032,29 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8806,6 +9382,29 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9109,6 +9708,25 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9408,6 +10026,23 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9735,6 +10370,29 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10062,6 +10720,29 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10389,6 +11070,29 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10716,6 +11420,29 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -11039,6 +11766,27 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -11366,6 +12114,29 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -11693,6 +12464,29 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -12020,6 +12814,29 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -12204,6 +13021,17 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("agent-one-as") unordered, align 4
@@ -12386,6 +13214,17 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("agent-one-as") monotonic, align 4
@@ -12593,6 +13432,19 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("agent-one-as") acquire, align 4
@@ -12826,6 +13678,25 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("agent-one-as") seq_cst, align 4
@@ -12976,6 +13847,16 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("agent-one-as") unordered, align 4
@@ -13125,6 +14006,16 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("agent-one-as") monotonic, align 4
@@ -13298,6 +14189,20 @@ define amdgpu_kernel void @flat_agent_one_as_release_store(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("agent-one-as") release, align 4
@@ -13471,6 +14376,20 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("agent-one-as") seq_cst, align 4
@@ -13620,6 +14539,16 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") monotonic
@@ -13796,6 +14725,18 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acquire
@@ -13969,6 +14910,20 @@ define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") release
@@ -14169,6 +15124,22 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -14369,6 +15340,22 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -14587,6 +15574,20 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acquire
@@ -14834,6 +15835,26 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -15081,6 +16102,26 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -15319,6 +16360,20 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15584,6 +16639,22 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15846,6 +16917,24 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16135,6 +17224,26 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16424,6 +17533,26 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16689,6 +17818,22 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16954,6 +18099,22 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17243,6 +18404,26 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17532,6 +18713,26 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17821,6 +19022,26 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18110,6 +19331,26 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18399,6 +19640,26 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18688,6 +19949,26 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18977,6 +20258,26 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19266,6 +20567,26 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19548,6 +20869,22 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19857,6 +21194,24 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20165,6 +21520,26 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20502,6 +21877,30 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20839,6 +22238,30 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -21152,6 +22575,26 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -21461,6 +22904,24 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -21798,6 +23259,30 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -22135,6 +23620,30 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -22472,6 +23981,30 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -22809,6 +24342,30 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -23142,6 +24699,28 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -23479,6 +25058,30 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -23816,6 +25419,30 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -24153,6 +25780,30 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
index a00af8e5b6582..8d98f532908fe 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-lastuse.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefix=GFX12 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @flat_last_use_load_0(ptr %in, ptr %out) {
 ; GFX12-LABEL: flat_last_use_load_0:
@@ -16,6 +17,17 @@ define amdgpu_kernel void @flat_last_use_load_0(ptr %in, ptr %out) {
 ; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_last_use_load_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] th:TH_LOAD_LU
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
 entry:
   %val = load i32, ptr %in, align 4, !amdgpu.last.use !{}
   store i32 %val, ptr %out
@@ -55,6 +67,21 @@ define amdgpu_kernel void @flat_last_use_load_1(ptr %in, ptr %out) {
 ; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_last_use_load_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v1, v1, s4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v1, s[2:3] scale_offset th:TH_LOAD_LU
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val.gep = getelementptr inbounds i32, ptr %in, i32 %tid
@@ -80,6 +107,19 @@ define amdgpu_kernel void @flat_last_use_and_volatile_load(ptr %in, ptr %out) {
 ; GFX12-NEXT:    s_wait_dscnt 0x0
 ; GFX12-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_last_use_and_volatile_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
 entry:
   %val = load volatile i32, ptr %in, align 4, !amdgpu.last.use !{}
   store i32 %val, ptr %out
@@ -100,6 +140,17 @@ define amdgpu_kernel void @flat_last_use_and_nontemporal_load(ptr %in, ptr %out)
 ; GFX12-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_last_use_and_nontemporal_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] th:TH_LOAD_LU
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
 entry:
   %val = load i32, ptr %in, align 4, !amdgpu.last.use !{}, !nontemporal !0
   store i32 %val, ptr %out
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index 3c24c36ec547d..af48eaf8fcda6 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -11,6 +11,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX7-LABEL: flat_nontemporal_load_0:
@@ -187,6 +188,17 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_nontemporal_load_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] th:TH_LOAD_NT
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load i32, ptr %in, align 4, !nontemporal !0
@@ -555,6 +567,21 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_nontemporal_load_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v1, v1, s4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v1, s[2:3] scale_offset th:TH_LOAD_NT
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -739,6 +766,17 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 th:TH_STORE_NT
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_nontemporal_store_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] th:TH_STORE_NT scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load i32, ptr %in, align 4
@@ -1095,6 +1133,20 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 th:TH_STORE_NT
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_nontemporal_store_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v1, s[2:3]
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s2, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v0, v0, s2
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scale_offset th:TH_STORE_NT scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1293,6 +1345,19 @@ define amdgpu_kernel void @flat_nontemporal_volatile_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_nontemporal_volatile_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] th:TH_LOAD_NT scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load volatile i32, ptr %in, align 4, !nontemporal !0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
index b88a10ab24a98..871c941dd6dca 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
@@ -11,6 +11,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @flat_singlethread_unordered_load(
 ; GFX7-LABEL: flat_singlethread_unordered_load:
@@ -187,6 +188,17 @@ define amdgpu_kernel void @flat_singlethread_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("singlethread") unordered, align 4
@@ -369,6 +381,17 @@ define amdgpu_kernel void @flat_singlethread_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("singlethread") monotonic, align 4
@@ -551,6 +574,17 @@ define amdgpu_kernel void @flat_singlethread_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("singlethread") acquire, align 4
@@ -733,6 +767,17 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("singlethread") seq_cst, align 4
@@ -883,6 +928,16 @@ define amdgpu_kernel void @flat_singlethread_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("singlethread") unordered, align 4
@@ -1032,6 +1087,16 @@ define amdgpu_kernel void @flat_singlethread_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("singlethread") monotonic, align 4
@@ -1181,6 +1246,16 @@ define amdgpu_kernel void @flat_singlethread_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("singlethread") release, align 4
@@ -1330,6 +1405,16 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("singlethread") seq_cst, align 4
@@ -1479,6 +1564,16 @@ define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") monotonic
@@ -1628,6 +1723,16 @@ define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") acquire
@@ -1777,6 +1882,16 @@ define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") release
@@ -1926,6 +2041,16 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") acq_rel
@@ -2075,6 +2200,16 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") seq_cst
@@ -2268,6 +2403,18 @@ define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") acquire
@@ -2462,6 +2609,18 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") acq_rel
@@ -2656,6 +2815,18 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread") seq_cst
@@ -2894,6 +3065,20 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3132,6 +3317,20 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3370,6 +3569,20 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3608,6 +3821,20 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3846,6 +4073,20 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4084,6 +4325,20 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4322,6 +4577,20 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4560,6 +4829,20 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4798,6 +5081,20 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5036,6 +5333,20 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5274,6 +5585,20 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5512,6 +5837,20 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5750,6 +6089,20 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5988,6 +6341,20 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6226,6 +6593,20 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6508,6 +6889,22 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6792,6 +7189,22 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7076,6 +7489,22 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7360,6 +7789,22 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7644,6 +8089,22 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7928,6 +8389,22 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8212,6 +8689,22 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8496,6 +8989,22 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8780,6 +9289,22 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9064,6 +9589,22 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9348,6 +9889,22 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9632,6 +10189,22 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9916,6 +10489,22 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10200,6 +10789,22 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10484,6 +11089,22 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10668,6 +11289,17 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("singlethread-one-as") unordered, align 4
@@ -10850,6 +11482,17 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("singlethread-one-as") monotonic, align 4
@@ -11032,6 +11675,17 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("singlethread-one-as") acquire, align 4
@@ -11214,6 +11868,17 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("singlethread-one-as") seq_cst, align 4
@@ -11364,6 +12029,16 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("singlethread-one-as") unordered, align 4
@@ -11513,6 +12188,16 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("singlethread-one-as") monotonic, align 4
@@ -11662,6 +12347,16 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("singlethread-one-as") release, align 4
@@ -11811,6 +12506,16 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("singlethread-one-as") seq_cst, align 4
@@ -11960,6 +12665,16 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") monotonic
@@ -12109,6 +12824,16 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -12258,6 +12983,16 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") release
@@ -12407,6 +13142,16 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -12556,6 +13301,16 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -12749,6 +13504,18 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -12943,6 +13710,18 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -13137,6 +13916,18 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -13375,6 +14166,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -13613,6 +14418,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -13851,6 +14670,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14089,6 +14922,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14327,6 +15174,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14565,6 +15426,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14803,6 +15678,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15041,6 +15930,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15279,6 +16182,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15517,6 +16434,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15755,6 +16686,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15993,6 +16938,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16231,6 +17190,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16469,6 +17442,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16707,6 +17694,20 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16989,6 +17990,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpx
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17273,6 +18290,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17557,6 +18590,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17841,6 +18890,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18125,6 +19190,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18409,6 +19490,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18693,6 +19790,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18977,6 +20090,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19261,6 +20390,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19545,6 +20690,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19829,6 +20990,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20113,6 +21290,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20397,6 +21590,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20681,6 +21890,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20965,6 +22190,22 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
index 919fc3e8f4e4f..9d70a2437e553 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
@@ -11,6 +11,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @flat_system_unordered_load(
 ; GFX7-LABEL: flat_system_unordered_load:
@@ -187,6 +188,17 @@ define amdgpu_kernel void @flat_system_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in unordered, align 4
@@ -369,6 +381,17 @@ define amdgpu_kernel void @flat_system_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in monotonic, align 4
@@ -568,6 +591,18 @@ define amdgpu_kernel void @flat_system_acquire_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in acquire, align 4
@@ -793,6 +828,24 @@ define amdgpu_kernel void @flat_system_seq_cst_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in seq_cst, align 4
@@ -943,6 +996,16 @@ define amdgpu_kernel void @flat_system_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out unordered, align 4
@@ -1092,6 +1155,16 @@ define amdgpu_kernel void @flat_system_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out monotonic, align 4
@@ -1269,6 +1342,21 @@ define amdgpu_kernel void @flat_system_release_store(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out release, align 4
@@ -1446,6 +1534,21 @@ define amdgpu_kernel void @flat_system_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out seq_cst, align 4
@@ -1595,6 +1698,16 @@ define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in monotonic
@@ -1777,6 +1890,18 @@ define amdgpu_kernel void @flat_system_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in acquire
@@ -1954,6 +2079,21 @@ define amdgpu_kernel void @flat_system_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in release
@@ -2164,6 +2304,23 @@ define amdgpu_kernel void @flat_system_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in acq_rel
@@ -2374,6 +2531,23 @@ define amdgpu_kernel void @flat_system_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in seq_cst
@@ -2584,6 +2758,19 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in acquire
@@ -2827,6 +3014,26 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in acq_rel
@@ -3070,6 +3277,26 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in seq_cst
@@ -3308,6 +3535,20 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3579,6 +3820,22 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3845,6 +4102,25 @@ define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4144,6 +4420,27 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4443,6 +4740,27 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4714,6 +5032,22 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4985,6 +5319,22 @@ define amdgpu_kernel void @flat_system_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5284,6 +5634,27 @@ define amdgpu_kernel void @flat_system_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5583,6 +5954,27 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5882,6 +6274,27 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6181,6 +6594,27 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6480,6 +6914,27 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6779,6 +7234,27 @@ define amdgpu_kernel void @flat_system_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7078,6 +7554,27 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7377,6 +7874,27 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7659,6 +8177,22 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7960,6 +8494,23 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8272,6 +8823,27 @@ define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8605,6 +9177,30 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8938,6 +9534,30 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9243,6 +9863,25 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9544,6 +10183,23 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9877,6 +10533,30 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10210,6 +10890,30 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10543,6 +11247,30 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10876,6 +11604,30 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -11205,6 +11957,28 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -11538,6 +12312,30 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -11871,6 +12669,30 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -12204,6 +13026,30 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -12388,6 +13234,17 @@ define amdgpu_kernel void @flat_system_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("one-as") unordered, align 4
@@ -12570,6 +13427,17 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("one-as") monotonic, align 4
@@ -12779,6 +13647,19 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("one-as") acquire, align 4
@@ -13014,6 +13895,25 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("one-as") seq_cst, align 4
@@ -13164,6 +14064,16 @@ define amdgpu_kernel void @flat_system_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("one-as") unordered, align 4
@@ -13313,6 +14223,16 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("one-as") monotonic, align 4
@@ -13490,6 +14410,21 @@ define amdgpu_kernel void @flat_system_one_as_release_store(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("one-as") release, align 4
@@ -13667,6 +14602,21 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("one-as") seq_cst, align 4
@@ -13816,6 +14766,16 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") monotonic
@@ -13994,6 +14954,18 @@ define amdgpu_kernel void @flat_system_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") acquire
@@ -14171,6 +15143,21 @@ define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") release
@@ -14377,6 +15364,23 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") acq_rel
@@ -14583,6 +15587,23 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") seq_cst
@@ -14803,6 +15824,20 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") acquire
@@ -15056,6 +16091,27 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") acq_rel
@@ -15309,6 +16365,27 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("one-as") seq_cst
@@ -15547,6 +16624,20 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15814,6 +16905,22 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16080,6 +17187,25 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16375,6 +17501,27 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16670,6 +17817,27 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16937,6 +18105,22 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17204,6 +18388,22 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17499,6 +18699,27 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17794,6 +19015,27 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18089,6 +19331,27 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18384,6 +19647,27 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18679,6 +19963,27 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18974,6 +20279,27 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19269,6 +20595,27 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19564,6 +20911,27 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19846,6 +21214,22 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20157,6 +21541,24 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20469,6 +21871,27 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20812,6 +22235,31 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -21155,6 +22603,31 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -21470,6 +22943,26 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -21781,6 +23274,24 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -22124,6 +23635,31 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -22467,6 +24003,31 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -22810,6 +24371,31 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -23153,6 +24739,31 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -23492,6 +25103,29 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -23835,6 +25469,31 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -24178,6 +25837,31 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -24521,6 +26205,31 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
index a88e0e217fdb4..43f015c3a2e0f 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
@@ -7,6 +7,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX7-LABEL: flat_nontemporal_load_0:
@@ -143,6 +144,19 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_nontemporal_load_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load volatile i32, ptr %in, align 4
@@ -415,6 +429,23 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_nontemporal_load_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v1, v1, s4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v1, s[2:3] scale_offset scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -563,6 +594,18 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_nontemporal_store_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load i32, ptr %in, align 4
@@ -831,6 +874,21 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_nontemporal_store_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v1, s[2:3]
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s2, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v0, v0, s2
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scale_offset scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -971,6 +1029,17 @@ define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_volatile_workgroup_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic volatile i32, ptr %in syncscope("workgroup") acquire, align 4
@@ -1090,6 +1159,17 @@ define amdgpu_kernel void @flat_volatile_workgroup_release_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_volatile_workgroup_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
    i32 %in, ptr %out) {
 entry:
   store atomic volatile i32 %in, ptr %out syncscope("workgroup") release, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
index 7c637a20ab47b..f086542b3d1f8 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
@@ -11,6 +11,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @flat_wavefront_unordered_load(
 ; GFX7-LABEL: flat_wavefront_unordered_load:
@@ -187,6 +188,17 @@ define amdgpu_kernel void @flat_wavefront_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("wavefront") unordered, align 4
@@ -369,6 +381,17 @@ define amdgpu_kernel void @flat_wavefront_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("wavefront") monotonic, align 4
@@ -551,6 +574,17 @@ define amdgpu_kernel void @flat_wavefront_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("wavefront") acquire, align 4
@@ -733,6 +767,17 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("wavefront") seq_cst, align 4
@@ -883,6 +928,16 @@ define amdgpu_kernel void @flat_wavefront_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("wavefront") unordered, align 4
@@ -1032,6 +1087,16 @@ define amdgpu_kernel void @flat_wavefront_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("wavefront") monotonic, align 4
@@ -1181,6 +1246,16 @@ define amdgpu_kernel void @flat_wavefront_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("wavefront") release, align 4
@@ -1330,6 +1405,16 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("wavefront") seq_cst, align 4
@@ -1479,6 +1564,16 @@ define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") monotonic
@@ -1628,6 +1723,16 @@ define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acquire
@@ -1777,6 +1882,16 @@ define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") release
@@ -1926,6 +2041,16 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acq_rel
@@ -2075,6 +2200,16 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") seq_cst
@@ -2268,6 +2403,18 @@ define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acquire
@@ -2462,6 +2609,18 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") acq_rel
@@ -2656,6 +2815,18 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront") seq_cst
@@ -2894,6 +3065,20 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3132,6 +3317,20 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3370,6 +3569,20 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3608,6 +3821,20 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3846,6 +4073,20 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4084,6 +4325,20 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4322,6 +4577,20 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4560,6 +4829,20 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4798,6 +5081,20 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5036,6 +5333,20 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5274,6 +5585,20 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5512,6 +5837,20 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5750,6 +6089,20 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5988,6 +6341,20 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6226,6 +6593,20 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6508,6 +6889,22 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6792,6 +7189,22 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7076,6 +7489,22 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7360,6 +7789,22 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7644,6 +8089,22 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7928,6 +8389,22 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8212,6 +8689,22 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8496,6 +8989,22 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8780,6 +9289,22 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9064,6 +9589,22 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9348,6 +9889,22 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9632,6 +10189,22 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9916,6 +10489,22 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10200,6 +10789,22 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10484,6 +11089,22 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10668,6 +11289,17 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("wavefront-one-as") unordered, align 4
@@ -10850,6 +11482,17 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("wavefront-one-as") monotonic, align 4
@@ -11032,6 +11675,17 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("wavefront-one-as") acquire, align 4
@@ -11214,6 +11868,17 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("wavefront-one-as") seq_cst, align 4
@@ -11364,6 +12029,16 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("wavefront-one-as") unordered, align 4
@@ -11513,6 +12188,16 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("wavefront-one-as") monotonic, align 4
@@ -11662,6 +12347,16 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("wavefront-one-as") release, align 4
@@ -11811,6 +12506,16 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("wavefront-one-as") seq_cst, align 4
@@ -11960,6 +12665,16 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") monotonic
@@ -12109,6 +12824,16 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -12258,6 +12983,16 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") release
@@ -12407,6 +13142,16 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -12556,6 +13301,16 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -12749,6 +13504,18 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -12943,6 +13710,18 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -13137,6 +13916,18 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -13375,6 +14166,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -13613,6 +14418,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -13851,6 +14670,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14089,6 +14922,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14327,6 +15174,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14565,6 +15426,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14803,6 +15678,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15041,6 +15930,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15279,6 +16182,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15517,6 +16434,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15755,6 +16686,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15993,6 +16938,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16231,6 +17190,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16469,6 +17442,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16707,6 +17694,20 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16989,6 +17990,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17273,6 +18290,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17557,6 +18590,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17841,6 +18890,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18125,6 +19190,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18409,6 +19490,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18693,6 +19790,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18977,6 +20090,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19261,6 +20390,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19545,6 +20690,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
    ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19829,6 +20990,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
    ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20113,6 +21290,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
    ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20397,6 +21590,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
    ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20681,6 +21890,22 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
index 0fd4aa4a7a93f..d8e6ad043e061 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
@@ -11,6 +11,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @flat_workgroup_unordered_load(
 ; GFX7-LABEL: flat_workgroup_unordered_load:
@@ -187,6 +188,17 @@ define amdgpu_kernel void @flat_workgroup_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("workgroup") unordered, align 4
@@ -369,6 +381,17 @@ define amdgpu_kernel void @flat_workgroup_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("workgroup") monotonic, align 4
@@ -563,6 +586,17 @@ define amdgpu_kernel void @flat_workgroup_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("workgroup") acquire, align 4
@@ -776,6 +810,18 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("workgroup") seq_cst, align 4
@@ -926,6 +972,16 @@ define amdgpu_kernel void @flat_workgroup_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("workgroup") unordered, align 4
@@ -1075,6 +1131,16 @@ define amdgpu_kernel void @flat_workgroup_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("workgroup") monotonic, align 4
@@ -1241,6 +1307,17 @@ define amdgpu_kernel void @flat_workgroup_release_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("workgroup") release, align 4
@@ -1407,6 +1484,17 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("workgroup") seq_cst, align 4
@@ -1556,6 +1644,16 @@ define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") monotonic
@@ -1724,6 +1822,17 @@ define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acquire
@@ -1890,6 +1999,17 @@ define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") release
@@ -2075,6 +2195,18 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acq_rel
@@ -2260,6 +2392,18 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") seq_cst
@@ -2465,6 +2609,18 @@ define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acquire
@@ -2690,6 +2846,19 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") acq_rel
@@ -2915,6 +3084,19 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup") seq_cst
@@ -3153,6 +3335,20 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3410,6 +3606,21 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3665,6 +3876,21 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -3939,6 +4165,22 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4213,6 +4455,22 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4470,6 +4728,21 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -4727,6 +5000,21 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5001,6 +5289,22 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5275,6 +5579,22 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5549,6 +5869,22 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -5823,6 +6159,22 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6105,6 +6457,22 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6401,6 +6769,22 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -6702,6 +7086,23 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7017,6 +7418,23 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7332,6 +7750,23 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7630,6 +8065,22 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -7926,6 +8377,22 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8241,6 +8708,23 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8556,6 +9040,23 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -8871,6 +9372,23 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9186,6 +9704,23 @@ define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9499,6 +10034,23 @@ define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -9814,6 +10366,23 @@ define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10129,6 +10698,23 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10444,6 +11030,23 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -10628,6 +11231,17 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("workgroup-one-as") unordered, align 4
@@ -10810,6 +11424,17 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("workgroup-one-as") monotonic, align 4
@@ -11000,6 +11625,17 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("workgroup-one-as") acquire, align 4
@@ -11202,6 +11838,17 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    flat_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
   %val = load atomic i32, ptr %in syncscope("workgroup-one-as") seq_cst, align 4
@@ -11352,6 +11999,16 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("workgroup-one-as") unordered, align 4
@@ -11501,6 +12158,16 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("workgroup-one-as") monotonic, align 4
@@ -11660,6 +12327,16 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("workgroup-one-as") release, align 4
@@ -11819,6 +12496,16 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
   store atomic i32 %in, ptr %out syncscope("workgroup-one-as") seq_cst, align 4
@@ -11968,6 +12655,16 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") monotonic
@@ -12127,6 +12824,16 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -12286,6 +12993,16 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") release
@@ -12455,6 +13172,16 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -12624,6 +13351,16 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -12825,6 +13562,18 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -13039,6 +13788,18 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -13253,6 +14014,18 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    flat_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -13491,6 +14264,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -13739,6 +14526,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -13987,6 +14788,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14245,6 +15060,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14503,6 +15332,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14751,6 +15594,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -14999,6 +15856,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15257,6 +16128,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15515,6 +16400,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -15773,6 +16672,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16031,6 +16944,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16289,6 +17216,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16547,6 +17488,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -16805,6 +17760,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17063,6 +18032,20 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX12-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17345,6 +18328,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17637,6 +18636,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -17931,6 +18946,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18235,6 +19266,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18539,6 +19586,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -18833,6 +19896,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19125,6 +20204,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19429,6 +20524,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -19733,6 +20844,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20037,6 +21164,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20341,6 +21484,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20643,6 +21802,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -20947,6 +22122,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -21251,6 +22442,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
@@ -21555,6 +22762,22 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    flat_store_b32 v[0:1], v2
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    flat_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    flat_store_b32 v0, v1, s[0:1] scope:SCOPE_SE
+; GFX1250-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
index 74a72e04fa4ae..184e15406bfbc 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-agent.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @global_agent_unordered_load(
 ; GFX6-LABEL: global_agent_unordered_load:
@@ -190,6 +191,17 @@ define amdgpu_kernel void @global_agent_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("agent") unordered, align 4
@@ -374,6 +386,17 @@ define amdgpu_kernel void @global_agent_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("agent") monotonic, align 4
@@ -574,6 +597,18 @@ define amdgpu_kernel void @global_agent_acquire_load(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("agent") acquire, align 4
@@ -793,6 +828,24 @@ define amdgpu_kernel void @global_agent_seq_cst_load(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("agent") seq_cst, align 4
@@ -950,6 +1003,16 @@ define amdgpu_kernel void @global_agent_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("agent") unordered, align 4
@@ -1106,6 +1169,16 @@ define amdgpu_kernel void @global_agent_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("agent") monotonic, align 4
@@ -1287,6 +1360,20 @@ define amdgpu_kernel void @global_agent_release_store(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("agent") release, align 4
@@ -1468,6 +1555,20 @@ define amdgpu_kernel void @global_agent_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("agent") seq_cst, align 4
@@ -1622,6 +1723,16 @@ define amdgpu_kernel void @global_agent_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") monotonic
@@ -1805,6 +1916,18 @@ define amdgpu_kernel void @global_agent_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") acquire
@@ -1984,6 +2107,20 @@ define amdgpu_kernel void @global_agent_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") release
@@ -2192,6 +2329,22 @@ define amdgpu_kernel void @global_agent_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") acq_rel
@@ -2400,6 +2553,22 @@ define amdgpu_kernel void @global_agent_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
@@ -2598,6 +2767,19 @@ define amdgpu_kernel void @global_agent_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") acquire
@@ -2826,6 +3008,25 @@ define amdgpu_kernel void @global_agent_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") acq_rel
@@ -3054,6 +3255,25 @@ define amdgpu_kernel void @global_agent_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent") seq_cst
@@ -3273,6 +3493,20 @@ define amdgpu_kernel void @global_agent_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3521,6 +3755,22 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3765,6 +4015,24 @@ define amdgpu_kernel void @global_agent_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4038,6 +4306,26 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4311,6 +4599,26 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4559,6 +4867,22 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4807,6 +5131,22 @@ define amdgpu_kernel void @global_agent_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5080,6 +5420,26 @@ define amdgpu_kernel void @global_agent_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5353,6 +5713,26 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5626,6 +6006,26 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5899,6 +6299,26 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6172,6 +6592,26 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6445,6 +6885,26 @@ define amdgpu_kernel void @global_agent_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6718,6 +7178,26 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6991,6 +7471,26 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7240,6 +7740,22 @@ define amdgpu_kernel void @global_agent_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7507,6 +8023,23 @@ define amdgpu_kernel void @global_agent_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7783,6 +8316,26 @@ define amdgpu_kernel void @global_agent_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8079,6 +8632,29 @@ define amdgpu_kernel void @global_agent_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8375,6 +8951,29 @@ define amdgpu_kernel void @global_agent_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8646,6 +9245,25 @@ define amdgpu_kernel void @global_agent_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8913,6 +9531,23 @@ define amdgpu_kernel void @global_agent_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9209,6 +9844,29 @@ define amdgpu_kernel void @global_agent_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9505,6 +10163,29 @@ define amdgpu_kernel void @global_agent_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9801,6 +10482,29 @@ define amdgpu_kernel void @global_agent_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10097,6 +10801,29 @@ define amdgpu_kernel void @global_agent_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10389,6 +11116,27 @@ define amdgpu_kernel void @global_agent_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10685,6 +11433,29 @@ define amdgpu_kernel void @global_agent_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10981,6 +11752,29 @@ define amdgpu_kernel void @global_agent_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11277,6 +12071,29 @@ define amdgpu_kernel void @global_agent_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -11463,6 +12280,17 @@ define amdgpu_kernel void @global_agent_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("agent-one-as") unordered, align 4
@@ -11647,6 +12475,17 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("agent-one-as") monotonic, align 4
@@ -11847,6 +12686,18 @@ define amdgpu_kernel void @global_agent_one_as_acquire_load(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("agent-one-as") acquire, align 4
@@ -12066,6 +12917,24 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("agent-one-as") seq_cst, align 4
@@ -12223,6 +13092,16 @@ define amdgpu_kernel void @global_agent_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("agent-one-as") unordered, align 4
@@ -12379,6 +13258,16 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("agent-one-as") monotonic, align 4
@@ -12560,6 +13449,20 @@ define amdgpu_kernel void @global_agent_one_as_release_store(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("agent-one-as") release, align 4
@@ -12741,6 +13644,20 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("agent-one-as") seq_cst, align 4
@@ -12895,6 +13812,16 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") monotonic
@@ -13078,6 +14005,18 @@ define amdgpu_kernel void @global_agent_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") acquire
@@ -13257,6 +14196,20 @@ define amdgpu_kernel void @global_agent_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") release
@@ -13465,6 +14418,22 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -13673,6 +14642,22 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -13871,6 +14856,19 @@ define amdgpu_kernel void @global_agent_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") acquire
@@ -14099,6 +15097,25 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -14327,6 +15344,25 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -14546,6 +15582,20 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14794,6 +15844,22 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15038,6 +16104,24 @@ define amdgpu_kernel void @global_agent_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15311,6 +16395,26 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15584,6 +16688,26 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15832,6 +16956,22 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16080,6 +17220,22 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16353,6 +17509,26 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16626,6 +17802,26 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16899,6 +18095,26 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17172,6 +18388,26 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17445,6 +18681,26 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17718,6 +18974,26 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17991,6 +19267,26 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18264,6 +19560,26 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18513,6 +19829,22 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18780,6 +20112,23 @@ define amdgpu_kernel void @global_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19076,6 +20425,29 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19372,6 +20744,29 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19643,6 +21038,25 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19910,6 +21324,23 @@ define amdgpu_kernel void @global_agent_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20206,6 +21637,29 @@ define amdgpu_kernel void @global_agent_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20502,6 +21956,29 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20798,6 +22275,29 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -21094,6 +22594,29 @@ define amdgpu_kernel void @global_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -21386,6 +22909,27 @@ define amdgpu_kernel void @global_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -21682,6 +23226,29 @@ define amdgpu_kernel void @global_agent_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -21978,6 +23545,29 @@ define amdgpu_kernel void @global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -22274,6 +23864,29 @@ define amdgpu_kernel void @global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_DEV
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_DEV
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
index 5f952b98041f3..ed2d62356f8f2 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-lastuse.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefix=GFX12 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @global_last_use_load_0(ptr addrspace(1) %in, ptr addrspace(1) %out) {
 ; GFX12-LABEL: global_last_use_load_0:
@@ -14,6 +15,18 @@ define amdgpu_kernel void @global_last_use_load_0(ptr addrspace(1) %in, ptr addr
 ; GFX12-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_last_use_load_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[2:3], 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
 entry:
   %val = load i32, ptr addrspace(1) %in, align 4, !amdgpu.last.use !{}
   store i32 %val, ptr addrspace(1) %out
@@ -37,6 +50,21 @@ define amdgpu_kernel void @global_last_use_load_1(ptr addrspace(1) %in, ptr addr
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_last_use_load_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v1, v1, s4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v1, s[2:3] scale_offset th:TH_LOAD_LU
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val.gep = getelementptr inbounds i32, ptr addrspace(1) %in, i32 %tid
@@ -58,6 +86,19 @@ define amdgpu_kernel void @global_last_use_and_volatile_load(ptr addrspace(1) %i
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_last_use_and_volatile_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] th:TH_LOAD_BYPASS scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
 entry:
   %val = load volatile i32, ptr addrspace(1) %in, align 4, !amdgpu.last.use !{}
   store i32 %val, ptr addrspace(1) %out
@@ -81,6 +122,21 @@ define amdgpu_kernel void @global_last_use_and_nontemporal_load(ptr addrspace(1)
 ; GFX12-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_last_use_and_nontemporal_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v1, v1, s4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v1, s[2:3] scale_offset th:TH_LOAD_LU
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %val.gep = getelementptr inbounds i32, ptr addrspace(1) %in, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
index 16e55058e4fc8..0ad64f5599fe7 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-nontemporal.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @global_nontemporal_load_0(
 ; GFX6-LABEL: global_nontemporal_load_0:
@@ -189,6 +190,18 @@ define amdgpu_kernel void @global_nontemporal_load_0(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_nontemporal_load_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[2:3], 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load i32, ptr addrspace(1) %in, align 4, !nontemporal !0
@@ -448,6 +461,21 @@ define amdgpu_kernel void @global_nontemporal_load_1(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_nontemporal_load_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v1, v1, s4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v1, s[2:3] scale_offset th:TH_LOAD_NT
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -633,6 +661,18 @@ define amdgpu_kernel void @global_nontemporal_store_0(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] th:TH_STORE_NT
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_nontemporal_store_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[2:3], 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] th:TH_STORE_NT
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load i32, ptr addrspace(1) %in, align 4
@@ -866,6 +906,20 @@ define amdgpu_kernel void @global_nontemporal_store_1(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] th:TH_STORE_NT
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_nontemporal_store_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[2:3], 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s3, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v0, v0, s3
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scale_offset th:TH_STORE_NT
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1056,6 +1110,19 @@ define amdgpu_kernel void @global_nontemporal_volatile_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_nontemporal_volatile_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] th:TH_LOAD_NT scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load volatile i32, ptr addrspace(1) %in, align 4, !nontemporal !0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
index 8042d38716107..6a5a6e01c741b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-singlethread.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @global_singlethread_unordered_load(
 ; GFX6-LABEL: global_singlethread_unordered_load:
@@ -190,6 +191,17 @@ define amdgpu_kernel void @global_singlethread_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread") unordered, align 4
@@ -374,6 +386,17 @@ define amdgpu_kernel void @global_singlethread_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread") monotonic, align 4
@@ -558,6 +581,17 @@ define amdgpu_kernel void @global_singlethread_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread") acquire, align 4
@@ -742,6 +776,17 @@ define amdgpu_kernel void @global_singlethread_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread") seq_cst, align 4
@@ -899,6 +944,16 @@ define amdgpu_kernel void @global_singlethread_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread") unordered, align 4
@@ -1055,6 +1110,16 @@ define amdgpu_kernel void @global_singlethread_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread") monotonic, align 4
@@ -1211,6 +1276,16 @@ define amdgpu_kernel void @global_singlethread_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread") release, align 4
@@ -1367,6 +1442,16 @@ define amdgpu_kernel void @global_singlethread_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread") seq_cst, align 4
@@ -1521,6 +1606,16 @@ define amdgpu_kernel void @global_singlethread_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") monotonic
@@ -1675,6 +1770,16 @@ define amdgpu_kernel void @global_singlethread_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") acquire
@@ -1829,6 +1934,16 @@ define amdgpu_kernel void @global_singlethread_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") release
@@ -1983,6 +2098,16 @@ define amdgpu_kernel void @global_singlethread_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") acq_rel
@@ -2137,6 +2262,16 @@ define amdgpu_kernel void @global_singlethread_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") seq_cst
@@ -2319,6 +2454,18 @@ define amdgpu_kernel void @global_singlethread_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") acquire
@@ -2502,6 +2649,18 @@ define amdgpu_kernel void @global_singlethread_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") acq_rel
@@ -2685,6 +2844,18 @@ define amdgpu_kernel void @global_singlethread_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread") seq_cst
@@ -2904,6 +3075,20 @@ define amdgpu_kernel void @global_singlethread_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3123,6 +3308,20 @@ define amdgpu_kernel void @global_singlethread_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3342,6 +3541,20 @@ define amdgpu_kernel void @global_singlethread_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3561,6 +3774,20 @@ define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3780,6 +4007,20 @@ define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3999,6 +4240,20 @@ define amdgpu_kernel void @global_singlethread_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4218,6 +4473,20 @@ define amdgpu_kernel void @global_singlethread_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4437,6 +4706,20 @@ define amdgpu_kernel void @global_singlethread_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4656,6 +4939,20 @@ define amdgpu_kernel void @global_singlethread_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4875,6 +5172,20 @@ define amdgpu_kernel void @global_singlethread_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5094,6 +5405,20 @@ define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5313,6 +5638,20 @@ define amdgpu_kernel void @global_singlethread_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5532,6 +5871,20 @@ define amdgpu_kernel void @global_singlethread_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5751,6 +6104,20 @@ define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5970,6 +6337,20 @@ define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6219,6 +6600,22 @@ define amdgpu_kernel void @global_singlethread_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6470,6 +6867,22 @@ define amdgpu_kernel void @global_singlethread_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6721,6 +7134,22 @@ define amdgpu_kernel void @global_singlethread_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6972,6 +7401,22 @@ define amdgpu_kernel void @global_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7223,6 +7668,22 @@ define amdgpu_kernel void @global_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7474,6 +7935,22 @@ define amdgpu_kernel void @global_singlethread_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7725,6 +8202,22 @@ define amdgpu_kernel void @global_singlethread_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7976,6 +8469,22 @@ define amdgpu_kernel void @global_singlethread_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8227,6 +8736,22 @@ define amdgpu_kernel void @global_singlethread_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8478,6 +9003,22 @@ define amdgpu_kernel void @global_singlethread_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8729,6 +9270,22 @@ define amdgpu_kernel void @global_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8980,6 +9537,22 @@ define amdgpu_kernel void @global_singlethread_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9231,6 +9804,22 @@ define amdgpu_kernel void @global_singlethread_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9482,6 +10071,22 @@ define amdgpu_kernel void @global_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9733,6 +10338,22 @@ define amdgpu_kernel void @global_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9919,6 +10540,17 @@ define amdgpu_kernel void @global_singlethread_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread-one-as") unordered, align 4
@@ -10103,6 +10735,17 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread-one-as") monotonic, align 4
@@ -10287,6 +10930,17 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread-one-as") acquire, align 4
@@ -10471,6 +11125,17 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("singlethread-one-as") seq_cst, align 4
@@ -10628,6 +11293,16 @@ define amdgpu_kernel void @global_singlethread_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread-one-as") unordered, align 4
@@ -10784,6 +11459,16 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread-one-as") monotonic, align 4
@@ -10940,6 +11625,16 @@ define amdgpu_kernel void @global_singlethread_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread-one-as") release, align 4
@@ -11096,6 +11791,16 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("singlethread-one-as") seq_cst, align 4
@@ -11250,6 +11955,16 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") monotonic
@@ -11404,6 +12119,16 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -11558,6 +12283,16 @@ define amdgpu_kernel void @global_singlethread_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") release
@@ -11712,6 +12447,16 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -11866,6 +12611,16 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -12048,6 +12803,18 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -12231,6 +12998,18 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -12414,6 +13193,18 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -12633,6 +13424,20 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_cmpxch
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12852,6 +13657,20 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13071,6 +13890,20 @@ define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13290,6 +14123,20 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13509,6 +14356,20 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13728,6 +14589,20 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13947,6 +14822,20 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14166,6 +15055,20 @@ define amdgpu_kernel void @global_singlethread_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14385,6 +15288,20 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14604,6 +15521,20 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14823,6 +15754,20 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15042,6 +15987,20 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15261,6 +16220,20 @@ define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15480,6 +16453,20 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15699,6 +16686,20 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15948,6 +16949,22 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_monotonic_ret_cm
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16199,6 +17216,22 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_monotonic_ret_cmpx
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16450,6 +17483,22 @@ define amdgpu_kernel void @global_singlethread_one_as_release_monotonic_ret_cmpx
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16701,6 +17750,22 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_monotonic_ret_cmpx
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16952,6 +18017,22 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_monotonic_ret_cmpx
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17203,6 +18284,22 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_acquire_ret_cmpx
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17454,6 +18551,22 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_acquire_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17705,6 +18818,22 @@ define amdgpu_kernel void @global_singlethread_one_as_release_acquire_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17956,6 +19085,22 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_acquire_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18207,6 +19352,22 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_acquire_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18458,6 +19619,22 @@ define amdgpu_kernel void @global_singlethread_one_as_monotonic_seq_cst_ret_cmpx
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18709,6 +19886,22 @@ define amdgpu_kernel void @global_singlethread_one_as_acquire_seq_cst_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18960,6 +20153,22 @@ define amdgpu_kernel void @global_singlethread_one_as_release_seq_cst_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19211,6 +20420,22 @@ define amdgpu_kernel void @global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19462,6 +20687,22 @@ define amdgpu_kernel void @global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
index be148464c156e..7ddd515830e11 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @global_system_unordered_load(
 ; GFX6-LABEL: global_system_unordered_load:
@@ -190,6 +191,17 @@ define amdgpu_kernel void @global_system_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in unordered, align 4
@@ -374,6 +386,17 @@ define amdgpu_kernel void @global_system_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in monotonic, align 4
@@ -576,6 +599,18 @@ define amdgpu_kernel void @global_system_acquire_load(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in acquire, align 4
@@ -797,6 +832,24 @@ define amdgpu_kernel void @global_system_seq_cst_load(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in seq_cst, align 4
@@ -954,6 +1007,16 @@ define amdgpu_kernel void @global_system_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out unordered, align 4
@@ -1110,6 +1173,16 @@ define amdgpu_kernel void @global_system_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out monotonic, align 4
@@ -1295,6 +1368,21 @@ define amdgpu_kernel void @global_system_release_store(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out release, align 4
@@ -1480,6 +1568,21 @@ define amdgpu_kernel void @global_system_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out seq_cst, align 4
@@ -1634,6 +1737,16 @@ define amdgpu_kernel void @global_system_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in monotonic
@@ -1819,6 +1932,18 @@ define amdgpu_kernel void @global_system_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in acquire
@@ -2002,6 +2127,21 @@ define amdgpu_kernel void @global_system_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in release
@@ -2216,6 +2356,23 @@ define amdgpu_kernel void @global_system_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in acq_rel
@@ -2430,6 +2587,23 @@ define amdgpu_kernel void @global_system_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in seq_cst
@@ -2630,6 +2804,19 @@ define amdgpu_kernel void @global_system_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in acquire
@@ -2864,6 +3051,26 @@ define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in acq_rel
@@ -3098,6 +3305,26 @@ define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in seq_cst
@@ -3317,6 +3544,20 @@ define amdgpu_kernel void @global_system_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3567,6 +3808,22 @@ define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3815,6 +4072,25 @@ define amdgpu_kernel void @global_system_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4094,6 +4370,27 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4373,6 +4670,27 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4623,6 +4941,22 @@ define amdgpu_kernel void @global_system_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4873,6 +5207,22 @@ define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5152,6 +5502,27 @@ define amdgpu_kernel void @global_system_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5431,6 +5802,27 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5710,6 +6102,27 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5989,6 +6402,27 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6238,6 +6672,22 @@ define amdgpu_kernel void @global_system_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6507,6 +6957,23 @@ define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6809,6 +7276,30 @@ define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7111,6 +7602,30 @@ define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7384,6 +7899,25 @@ define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7653,6 +8187,23 @@ define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7955,6 +8506,30 @@ define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8257,6 +8832,30 @@ define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8559,6 +9158,30 @@ define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8861,6 +9484,30 @@ define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9159,6 +9806,28 @@ define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9461,6 +10130,30 @@ define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9763,6 +10456,30 @@ define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10065,6 +10782,30 @@ define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10251,6 +10992,17 @@ define amdgpu_kernel void @global_system_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("one-as") unordered, align 4
@@ -10435,6 +11187,17 @@ define amdgpu_kernel void @global_system_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("one-as") monotonic, align 4
@@ -10637,6 +11400,18 @@ define amdgpu_kernel void @global_system_one_as_acquire_load(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("one-as") acquire, align 4
@@ -10858,6 +11633,24 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("one-as") seq_cst, align 4
@@ -11015,6 +11808,16 @@ define amdgpu_kernel void @global_system_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("one-as") unordered, align 4
@@ -11171,6 +11974,16 @@ define amdgpu_kernel void @global_system_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("one-as") monotonic, align 4
@@ -11356,6 +12169,21 @@ define amdgpu_kernel void @global_system_one_as_release_store(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("one-as") release, align 4
@@ -11541,6 +12369,21 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("one-as") seq_cst, align 4
@@ -11695,6 +12538,16 @@ define amdgpu_kernel void @global_system_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") monotonic
@@ -11880,6 +12733,18 @@ define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") acquire
@@ -12063,6 +12928,21 @@ define amdgpu_kernel void @global_system_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") release
@@ -12277,6 +13157,23 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") acq_rel
@@ -12491,6 +13388,23 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") seq_cst
@@ -12691,6 +13605,19 @@ define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") acquire
@@ -12925,6 +13852,26 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") acq_rel
@@ -13159,6 +14106,26 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("one-as") seq_cst
@@ -13378,6 +14345,20 @@ define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13628,6 +14609,22 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13876,6 +14873,25 @@ define amdgpu_kernel void @global_system_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16 scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14155,6 +15171,27 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14434,6 +15471,27 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14684,6 +15742,22 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14934,6 +16008,22 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15213,6 +16303,27 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15492,6 +16603,27 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15771,6 +16903,27 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16050,6 +17203,27 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16329,6 +17503,27 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16608,6 +17803,27 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16887,6 +18103,27 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17166,6 +18403,27 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16 scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17415,6 +18673,22 @@ define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17684,6 +18958,23 @@ define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17964,6 +19255,27 @@ define amdgpu_kernel void @global_system_one_as_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18266,6 +19578,30 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18568,6 +19904,30 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18841,6 +20201,25 @@ define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19110,6 +20489,23 @@ define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19412,6 +20808,30 @@ define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19714,6 +21134,30 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20016,6 +21460,30 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20318,6 +21786,30 @@ define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20616,6 +22108,28 @@ define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20918,6 +22432,30 @@ define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -21220,6 +22758,30 @@ define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -21522,6 +23084,30 @@ define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    global_inv scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_wb scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_inv scope:SCOPE_SYS
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
index 8a5c5dda9f79c..0d18963cbfb68 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-volatile.ll
@@ -8,6 +8,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @global_volatile_load_0(
 ; GFX6-LABEL: global_volatile_load_0:
@@ -146,6 +147,19 @@ define amdgpu_kernel void @global_volatile_load_0(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_volatile_load_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load volatile i32, ptr addrspace(1) %in, align 4
@@ -345,6 +359,23 @@ define amdgpu_kernel void @global_volatile_load_1(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_volatile_load_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s4, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v1, v1, s4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v1, s[2:3] scale_offset scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_bvhcnt 0x0
+; GFX1250-NEXT:    s_wait_samplecnt 0x0
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -501,6 +532,19 @@ define amdgpu_kernel void @global_volatile_store_0(
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_volatile_store_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[2:3], 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load i32, ptr addrspace(1) %in, align 4
@@ -693,6 +737,21 @@ define amdgpu_kernel void @global_volatile_store_1(
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
 ; GFX12-CU-NEXT:    s_wait_storecnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_volatile_store_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[2:3], 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s3, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v0, v0, s3
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1] scale_offset scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -838,6 +897,17 @@ define amdgpu_kernel void @global_volatile_workgroup_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_volatile_workgroup_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic volatile i32, ptr addrspace(1) %in syncscope("workgroup") acquire, align 4
@@ -969,6 +1039,17 @@ define amdgpu_kernel void @global_volatile_workgroup_release_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_volatile_workgroup_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
    i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic volatile i32 %in, ptr addrspace(1) %out syncscope("workgroup") release, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
index 151ba07a0b531..1aa8305b1a837 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-wavefront.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @global_wavefront_unordered_load(
 ; GFX6-LABEL: global_wavefront_unordered_load:
@@ -190,6 +191,17 @@ define amdgpu_kernel void @global_wavefront_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront") unordered, align 4
@@ -374,6 +386,17 @@ define amdgpu_kernel void @global_wavefront_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront") monotonic, align 4
@@ -558,6 +581,17 @@ define amdgpu_kernel void @global_wavefront_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront") acquire, align 4
@@ -742,6 +776,17 @@ define amdgpu_kernel void @global_wavefront_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront") seq_cst, align 4
@@ -899,6 +944,16 @@ define amdgpu_kernel void @global_wavefront_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront") unordered, align 4
@@ -1055,6 +1110,16 @@ define amdgpu_kernel void @global_wavefront_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront") monotonic, align 4
@@ -1211,6 +1276,16 @@ define amdgpu_kernel void @global_wavefront_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront") release, align 4
@@ -1367,6 +1442,16 @@ define amdgpu_kernel void @global_wavefront_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront") seq_cst, align 4
@@ -1521,6 +1606,16 @@ define amdgpu_kernel void @global_wavefront_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") monotonic
@@ -1675,6 +1770,16 @@ define amdgpu_kernel void @global_wavefront_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") acquire
@@ -1829,6 +1934,16 @@ define amdgpu_kernel void @global_wavefront_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") release
@@ -1983,6 +2098,16 @@ define amdgpu_kernel void @global_wavefront_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") acq_rel
@@ -2137,6 +2262,16 @@ define amdgpu_kernel void @global_wavefront_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") seq_cst
@@ -2319,6 +2454,18 @@ define amdgpu_kernel void @global_wavefront_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") acquire
@@ -2502,6 +2649,18 @@ define amdgpu_kernel void @global_wavefront_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") acq_rel
@@ -2685,6 +2844,18 @@ define amdgpu_kernel void @global_wavefront_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront") seq_cst
@@ -2904,6 +3075,20 @@ define amdgpu_kernel void @global_wavefront_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3123,6 +3308,20 @@ define amdgpu_kernel void @global_wavefront_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3342,6 +3541,20 @@ define amdgpu_kernel void @global_wavefront_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3561,6 +3774,20 @@ define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3780,6 +4007,20 @@ define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3999,6 +4240,20 @@ define amdgpu_kernel void @global_wavefront_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4218,6 +4473,20 @@ define amdgpu_kernel void @global_wavefront_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4437,6 +4706,20 @@ define amdgpu_kernel void @global_wavefront_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4656,6 +4939,20 @@ define amdgpu_kernel void @global_wavefront_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4875,6 +5172,20 @@ define amdgpu_kernel void @global_wavefront_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5094,6 +5405,20 @@ define amdgpu_kernel void @global_wavefront_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5313,6 +5638,20 @@ define amdgpu_kernel void @global_wavefront_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5532,6 +5871,20 @@ define amdgpu_kernel void @global_wavefront_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5751,6 +6104,20 @@ define amdgpu_kernel void @global_wavefront_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5970,6 +6337,20 @@ define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6219,6 +6600,22 @@ define amdgpu_kernel void @global_wavefront_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
    ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6470,6 +6867,22 @@ define amdgpu_kernel void @global_wavefront_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6721,6 +7134,22 @@ define amdgpu_kernel void @global_wavefront_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
    ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6972,6 +7401,22 @@ define amdgpu_kernel void @global_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7223,6 +7668,22 @@ define amdgpu_kernel void @global_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7474,6 +7935,22 @@ define amdgpu_kernel void @global_wavefront_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7725,6 +8202,22 @@ define amdgpu_kernel void @global_wavefront_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7976,6 +8469,22 @@ define amdgpu_kernel void @global_wavefront_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8227,6 +8736,22 @@ define amdgpu_kernel void @global_wavefront_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8478,6 +9003,22 @@ define amdgpu_kernel void @global_wavefront_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8729,6 +9270,22 @@ define amdgpu_kernel void @global_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8980,6 +9537,22 @@ define amdgpu_kernel void @global_wavefront_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9231,6 +9804,22 @@ define amdgpu_kernel void @global_wavefront_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9482,6 +10071,22 @@ define amdgpu_kernel void @global_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9733,6 +10338,22 @@ define amdgpu_kernel void @global_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9919,6 +10540,17 @@ define amdgpu_kernel void @global_wavefront_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront-one-as") unordered, align 4
@@ -10103,6 +10735,17 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront-one-as") monotonic, align 4
@@ -10287,6 +10930,17 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront-one-as") acquire, align 4
@@ -10471,6 +11125,17 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("wavefront-one-as") seq_cst, align 4
@@ -10628,6 +11293,16 @@ define amdgpu_kernel void @global_wavefront_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront-one-as") unordered, align 4
@@ -10784,6 +11459,16 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront-one-as") monotonic, align 4
@@ -10940,6 +11625,16 @@ define amdgpu_kernel void @global_wavefront_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront-one-as") release, align 4
@@ -11096,6 +11791,16 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("wavefront-one-as") seq_cst, align 4
@@ -11250,6 +11955,16 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") monotonic
@@ -11404,6 +12119,16 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -11558,6 +12283,16 @@ define amdgpu_kernel void @global_wavefront_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") release
@@ -11712,6 +12447,16 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -11866,6 +12611,16 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -12048,6 +12803,18 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -12231,6 +12998,18 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -12414,6 +13193,18 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -12633,6 +13424,20 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -12852,6 +13657,20 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13071,6 +13890,20 @@ define amdgpu_kernel void @global_wavefront_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13290,6 +14123,20 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13509,6 +14356,20 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13728,6 +14589,20 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13947,6 +14822,20 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14166,6 +15055,20 @@ define amdgpu_kernel void @global_wavefront_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14385,6 +15288,20 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14604,6 +15521,20 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14823,6 +15754,20 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15042,6 +15987,20 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15261,6 +16220,20 @@ define amdgpu_kernel void @global_wavefront_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15480,6 +16453,20 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15699,6 +16686,20 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15948,6 +16949,22 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_monotonic_ret_cmpxc
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16199,6 +17216,22 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_monotonic_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16450,6 +17483,22 @@ define amdgpu_kernel void @global_wavefront_one_as_release_monotonic_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16701,6 +17750,22 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16952,6 +18017,22 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17203,6 +18284,22 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_acquire_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17454,6 +18551,22 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17705,6 +18818,22 @@ define amdgpu_kernel void @global_wavefront_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17956,6 +19085,22 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18207,6 +19352,22 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18458,6 +19619,22 @@ define amdgpu_kernel void @global_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18709,6 +19886,22 @@ define amdgpu_kernel void @global_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18960,6 +20153,22 @@ define amdgpu_kernel void @global_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19211,6 +20420,22 @@ define amdgpu_kernel void @global_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19462,6 +20687,22 @@ define amdgpu_kernel void @global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
index 69b0c7f93ab0e..3eab16e6b9713 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-global-workgroup.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @global_workgroup_unordered_load(
 ; GFX6-LABEL: global_workgroup_unordered_load:
@@ -190,6 +191,17 @@ define amdgpu_kernel void @global_workgroup_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") unordered, align 4
@@ -374,6 +386,17 @@ define amdgpu_kernel void @global_workgroup_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") monotonic, align 4
@@ -563,6 +586,17 @@ define amdgpu_kernel void @global_workgroup_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") acquire, align 4
@@ -764,6 +798,18 @@ define amdgpu_kernel void @global_workgroup_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup") seq_cst, align 4
@@ -921,6 +967,16 @@ define amdgpu_kernel void @global_workgroup_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") unordered, align 4
@@ -1077,6 +1133,16 @@ define amdgpu_kernel void @global_workgroup_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") monotonic, align 4
@@ -1251,6 +1317,17 @@ define amdgpu_kernel void @global_workgroup_release_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") release, align 4
@@ -1425,6 +1502,17 @@ define amdgpu_kernel void @global_workgroup_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup") seq_cst, align 4
@@ -1579,6 +1667,16 @@ define amdgpu_kernel void @global_workgroup_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") monotonic
@@ -1743,6 +1841,16 @@ define amdgpu_kernel void @global_workgroup_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acquire
@@ -1915,6 +2023,17 @@ define amdgpu_kernel void @global_workgroup_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") release
@@ -2097,6 +2216,17 @@ define amdgpu_kernel void @global_workgroup_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acq_rel
@@ -2279,6 +2409,17 @@ define amdgpu_kernel void @global_workgroup_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
@@ -2466,6 +2607,18 @@ define amdgpu_kernel void @global_workgroup_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acquire
@@ -2674,6 +2827,19 @@ define amdgpu_kernel void @global_workgroup_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") acq_rel
@@ -2882,6 +3048,19 @@ define amdgpu_kernel void @global_workgroup_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup") seq_cst
@@ -3101,6 +3280,20 @@ define amdgpu_kernel void @global_workgroup_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3330,6 +3523,20 @@ define amdgpu_kernel void @global_workgroup_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3567,6 +3774,21 @@ define amdgpu_kernel void @global_workgroup_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -3814,6 +4036,21 @@ define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4061,6 +4298,21 @@ define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4290,6 +4542,20 @@ define amdgpu_kernel void @global_workgroup_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4519,6 +4785,20 @@ define amdgpu_kernel void @global_workgroup_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -4766,6 +5046,21 @@ define amdgpu_kernel void @global_workgroup_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5013,6 +5308,21 @@ define amdgpu_kernel void @global_workgroup_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5260,6 +5570,21 @@ define amdgpu_kernel void @global_workgroup_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5507,6 +5832,21 @@ define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -5754,6 +6094,21 @@ define amdgpu_kernel void @global_workgroup_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6001,6 +6356,21 @@ define amdgpu_kernel void @global_workgroup_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6248,6 +6618,21 @@ define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6495,6 +6880,21 @@ define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -6744,6 +7144,22 @@ define amdgpu_kernel void @global_workgroup_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7000,6 +7416,22 @@ define amdgpu_kernel void @global_workgroup_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7269,6 +7701,23 @@ define amdgpu_kernel void @global_workgroup_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7545,6 +7994,23 @@ define amdgpu_kernel void @global_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -7821,6 +8287,23 @@ define amdgpu_kernel void @global_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8079,6 +8562,22 @@ define amdgpu_kernel void @global_workgroup_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8335,6 +8834,22 @@ define amdgpu_kernel void @global_workgroup_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8611,6 +9126,23 @@ define amdgpu_kernel void @global_workgroup_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -8887,6 +9419,23 @@ define amdgpu_kernel void @global_workgroup_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9163,6 +9712,23 @@ define amdgpu_kernel void @global_workgroup_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9439,6 +10005,23 @@ define amdgpu_kernel void @global_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9713,6 +10296,23 @@ define amdgpu_kernel void @global_workgroup_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -9989,6 +10589,23 @@ define amdgpu_kernel void @global_workgroup_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10265,6 +10882,23 @@ define amdgpu_kernel void @global_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10541,6 +11175,23 @@ define amdgpu_kernel void @global_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -10727,6 +11378,17 @@ define amdgpu_kernel void @global_workgroup_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") unordered, align 4
@@ -10911,6 +11573,17 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") monotonic, align 4
@@ -11100,6 +11773,17 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") acquire, align 4
@@ -11297,6 +11981,17 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(1) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(1) %in syncscope("workgroup-one-as") seq_cst, align 4
@@ -11454,6 +12149,16 @@ define amdgpu_kernel void @global_workgroup_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") unordered, align 4
@@ -11610,6 +12315,16 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") monotonic, align 4
@@ -11776,6 +12491,16 @@ define amdgpu_kernel void @global_workgroup_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") release, align 4
@@ -11942,6 +12667,16 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(1) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(1) %out syncscope("workgroup-one-as") seq_cst, align 4
@@ -12096,6 +12831,16 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") monotonic
@@ -12260,6 +13005,16 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -12424,6 +13179,16 @@ define amdgpu_kernel void @global_workgroup_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") release
@@ -12598,6 +13363,16 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -12772,6 +13547,16 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX12-CU-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -12959,6 +13744,18 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -13159,6 +13956,18 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -13359,6 +14168,18 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    global_atomic_swap_b32 v1, v0, v1, s[0:1] th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(1) %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -13578,6 +14399,20 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -13807,6 +14642,20 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14036,6 +14885,20 @@ define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14275,6 +15138,20 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14514,6 +15391,20 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14743,6 +15634,20 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -14972,6 +15877,20 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15211,6 +16130,20 @@ define amdgpu_kernel void @global_workgroup_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15450,6 +16383,20 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15689,6 +16636,20 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -15928,6 +16889,20 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16167,6 +17142,20 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16406,6 +17395,20 @@ define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16645,6 +17648,20 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -16884,6 +17901,20 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, v3
 ; GFX12-CU-NEXT:    global_atomic_cmpswap_b32 v0, v[1:2], s[0:1] offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17133,6 +18164,22 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_monotonic_ret_cmpxc
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17389,6 +18436,22 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_monotonic_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17650,6 +18713,22 @@ define amdgpu_kernel void @global_workgroup_one_as_release_monotonic_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -17918,6 +18997,22 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18186,6 +19281,22 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18444,6 +19555,22 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_acquire_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18700,6 +19827,22 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -18968,6 +20111,22 @@ define amdgpu_kernel void @global_workgroup_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19236,6 +20395,22 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19504,6 +20679,22 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -19772,6 +20963,22 @@ define amdgpu_kernel void @global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20038,6 +21245,22 @@ define amdgpu_kernel void @global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20306,6 +21529,22 @@ define amdgpu_kernel void @global_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20574,6 +21813,22 @@ define amdgpu_kernel void @global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
@@ -20842,6 +22097,22 @@ define amdgpu_kernel void @global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_loadcnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: global_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x8
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0xc
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s3
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
+; GFX1250-NEXT:    v_mov_b32_e32 v3, v1
+; GFX1250-NEXT:    global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:16 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(1) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
index 0467c5047a0be..102616b9a2065 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-agent.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @local_agent_unordered_load(
 ; GFX6-LABEL: local_agent_unordered_load:
@@ -177,6 +178,18 @@ define amdgpu_kernel void @local_agent_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("agent") unordered, align 4
@@ -348,6 +361,18 @@ define amdgpu_kernel void @local_agent_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("agent") monotonic, align 4
@@ -524,6 +549,18 @@ define amdgpu_kernel void @local_agent_acquire_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("agent") acquire, align 4
@@ -718,6 +755,19 @@ define amdgpu_kernel void @local_agent_seq_cst_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("agent") seq_cst, align 4
@@ -859,6 +909,16 @@ define amdgpu_kernel void @local_agent_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("agent") unordered, align 4
@@ -999,6 +1059,16 @@ define amdgpu_kernel void @local_agent_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("agent") monotonic, align 4
@@ -1157,6 +1227,17 @@ define amdgpu_kernel void @local_agent_release_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("agent") release, align 4
@@ -1315,6 +1396,17 @@ define amdgpu_kernel void @local_agent_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("agent") seq_cst, align 4
@@ -1455,6 +1547,16 @@ define amdgpu_kernel void @local_agent_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") monotonic
@@ -1611,6 +1713,17 @@ define amdgpu_kernel void @local_agent_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") acquire
@@ -1769,6 +1882,17 @@ define amdgpu_kernel void @local_agent_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") release
@@ -1943,6 +2067,18 @@ define amdgpu_kernel void @local_agent_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") acq_rel
@@ -2117,6 +2253,18 @@ define amdgpu_kernel void @local_agent_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") seq_cst
@@ -2304,6 +2452,19 @@ define amdgpu_kernel void @local_agent_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") acquire
@@ -2510,6 +2671,20 @@ define amdgpu_kernel void @local_agent_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") acq_rel
@@ -2716,6 +2891,20 @@ define amdgpu_kernel void @local_agent_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent") seq_cst
@@ -2883,6 +3072,18 @@ define amdgpu_kernel void @local_agent_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3066,6 +3267,19 @@ define amdgpu_kernel void @local_agent_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3251,6 +3465,19 @@ define amdgpu_kernel void @local_agent_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3452,6 +3679,20 @@ define amdgpu_kernel void @local_agent_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3653,6 +3894,20 @@ define amdgpu_kernel void @local_agent_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3836,6 +4091,19 @@ define amdgpu_kernel void @local_agent_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4019,6 +4287,19 @@ define amdgpu_kernel void @local_agent_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4220,6 +4501,20 @@ define amdgpu_kernel void @local_agent_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4421,6 +4716,20 @@ define amdgpu_kernel void @local_agent_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4622,6 +4931,20 @@ define amdgpu_kernel void @local_agent_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4823,6 +5146,20 @@ define amdgpu_kernel void @local_agent_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5024,6 +5361,20 @@ define amdgpu_kernel void @local_agent_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5225,6 +5576,20 @@ define amdgpu_kernel void @local_agent_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5426,6 +5791,20 @@ define amdgpu_kernel void @local_agent_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5627,6 +6006,20 @@ define amdgpu_kernel void @local_agent_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5836,6 +6229,21 @@ define amdgpu_kernel void @local_agent_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6052,6 +6460,21 @@ define amdgpu_kernel void @local_agent_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6281,6 +6704,22 @@ define amdgpu_kernel void @local_agent_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6515,6 +6954,22 @@ define amdgpu_kernel void @local_agent_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6749,6 +7204,22 @@ define amdgpu_kernel void @local_agent_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6965,6 +7436,21 @@ define amdgpu_kernel void @local_agent_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7181,6 +7667,21 @@ define amdgpu_kernel void @local_agent_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7415,6 +7916,22 @@ define amdgpu_kernel void @local_agent_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7649,6 +8166,22 @@ define amdgpu_kernel void @local_agent_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7883,6 +8416,22 @@ define amdgpu_kernel void @local_agent_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8117,6 +8666,22 @@ define amdgpu_kernel void @local_agent_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8351,6 +8916,22 @@ define amdgpu_kernel void @local_agent_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8585,6 +9166,22 @@ define amdgpu_kernel void @local_agent_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8819,6 +9416,22 @@ define amdgpu_kernel void @local_agent_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9053,6 +9666,22 @@ define amdgpu_kernel void @local_agent_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9226,6 +9855,18 @@ define amdgpu_kernel void @local_agent_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("agent-one-as") unordered, align 4
@@ -9397,6 +10038,18 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("agent-one-as") monotonic, align 4
@@ -9568,6 +10221,18 @@ define amdgpu_kernel void @local_agent_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("agent-one-as") acquire, align 4
@@ -9739,6 +10404,18 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("agent-one-as") seq_cst, align 4
@@ -9880,6 +10557,16 @@ define amdgpu_kernel void @local_agent_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("agent-one-as") unordered, align 4
@@ -10020,6 +10707,16 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("agent-one-as") monotonic, align 4
@@ -10160,6 +10857,16 @@ define amdgpu_kernel void @local_agent_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("agent-one-as") release, align 4
@@ -10300,6 +11007,16 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("agent-one-as") seq_cst, align 4
@@ -10440,6 +11157,16 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") monotonic
@@ -10580,6 +11307,16 @@ define amdgpu_kernel void @local_agent_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") acquire
@@ -10720,6 +11457,16 @@ define amdgpu_kernel void @local_agent_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") release
@@ -10860,6 +11607,16 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -11000,6 +11757,16 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -11182,6 +11949,19 @@ define amdgpu_kernel void @local_agent_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") acquire
@@ -11365,6 +12145,19 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") acq_rel
@@ -11548,6 +12341,19 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("agent-one-as") seq_cst
@@ -11715,6 +12521,18 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11882,6 +12700,18 @@ define amdgpu_kernel void @local_agent_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12049,6 +12879,18 @@ define amdgpu_kernel void @local_agent_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12216,6 +13058,18 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12383,6 +13237,18 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12550,6 +13416,18 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12717,6 +13595,18 @@ define amdgpu_kernel void @local_agent_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12884,6 +13774,18 @@ define amdgpu_kernel void @local_agent_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13051,6 +13953,18 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13218,6 +14132,18 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13385,6 +14311,18 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13552,6 +14490,18 @@ define amdgpu_kernel void @local_agent_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13719,6 +14669,18 @@ define amdgpu_kernel void @local_agent_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13886,6 +14848,18 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14053,6 +15027,18 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14262,6 +15248,21 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14473,6 +15474,21 @@ define amdgpu_kernel void @local_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14684,6 +15700,21 @@ define amdgpu_kernel void @local_agent_one_as_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14895,6 +15926,21 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15106,6 +16152,21 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15317,6 +16378,21 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15528,6 +16604,21 @@ define amdgpu_kernel void @local_agent_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15739,6 +16830,21 @@ define amdgpu_kernel void @local_agent_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15950,6 +17056,21 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16161,6 +17282,21 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16372,6 +17508,21 @@ define amdgpu_kernel void @local_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16583,6 +17734,21 @@ define amdgpu_kernel void @local_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16794,6 +17960,21 @@ define amdgpu_kernel void @local_agent_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -17005,6 +18186,21 @@ define amdgpu_kernel void @local_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -17216,6 +18412,21 @@ define amdgpu_kernel void @local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
index 78209ee34cad4..c6f7ce51f5ea2 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @local_nontemporal_load_0(
 ; GFX6-LABEL: local_nontemporal_load_0:
@@ -193,6 +194,18 @@ define amdgpu_kernel void @local_nontemporal_load_0(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_nontemporal_load_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ds_load_b32 v1, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(1) %out) {
 entry:
   %val = load i32, ptr addrspace(3) %in, align 4, !nontemporal !0
@@ -428,6 +441,22 @@ define amdgpu_kernel void @local_nontemporal_load_1(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_nontemporal_load_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_mov_b32 s2, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v1, v1, s2
+; GFX1250-NEXT:    s_mov_b32 s2, 2
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_lshl_add_u32 v1, v1, s2, s3
+; GFX1250-NEXT:    ds_load_b32 v1, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(1) %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -597,6 +626,18 @@ define amdgpu_kernel void @local_nontemporal_store_0(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_nontemporal_store_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[2:3], 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(3) %out) {
 entry:
   %val = load i32, ptr addrspace(1) %in, align 4
@@ -802,6 +843,22 @@ define amdgpu_kernel void @local_nontemporal_store_1(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_nontemporal_store_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[0:1], 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s1, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v0, v0, s1
+; GFX1250-NEXT:    s_mov_b32 s1, 2
+; GFX1250-NEXT:    v_lshl_add_u32 v0, v0, s1, s2
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(3) %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -991,6 +1048,18 @@ define amdgpu_kernel void @local_nontemporal_volatile_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_nontemporal_volatile_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ds_load_b32 v1, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(1) %out) {
 entry:
   %val = load volatile i32, ptr addrspace(3) %in, align 4, !nontemporal !0
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
index f84d451f8ecb0..1800acbbf605b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-singlethread.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @local_singlethread_unordered_load(
 ; GFX6-LABEL: local_singlethread_unordered_load:
@@ -177,6 +178,18 @@ define amdgpu_kernel void @local_singlethread_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread") unordered, align 4
@@ -348,6 +361,18 @@ define amdgpu_kernel void @local_singlethread_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread") monotonic, align 4
@@ -519,6 +544,18 @@ define amdgpu_kernel void @local_singlethread_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread") acquire, align 4
@@ -690,6 +727,18 @@ define amdgpu_kernel void @local_singlethread_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread") seq_cst, align 4
@@ -831,6 +880,16 @@ define amdgpu_kernel void @local_singlethread_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread") unordered, align 4
@@ -971,6 +1030,16 @@ define amdgpu_kernel void @local_singlethread_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread") monotonic, align 4
@@ -1111,6 +1180,16 @@ define amdgpu_kernel void @local_singlethread_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread") release, align 4
@@ -1251,6 +1330,16 @@ define amdgpu_kernel void @local_singlethread_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread") seq_cst, align 4
@@ -1391,6 +1480,16 @@ define amdgpu_kernel void @local_singlethread_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") monotonic
@@ -1531,6 +1630,16 @@ define amdgpu_kernel void @local_singlethread_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") acquire
@@ -1671,6 +1780,16 @@ define amdgpu_kernel void @local_singlethread_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") release
@@ -1811,6 +1930,16 @@ define amdgpu_kernel void @local_singlethread_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") acq_rel
@@ -1951,6 +2080,16 @@ define amdgpu_kernel void @local_singlethread_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") seq_cst
@@ -2133,6 +2272,19 @@ define amdgpu_kernel void @local_singlethread_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") acquire
@@ -2316,6 +2468,19 @@ define amdgpu_kernel void @local_singlethread_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") acq_rel
@@ -2499,6 +2664,19 @@ define amdgpu_kernel void @local_singlethread_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread") seq_cst
@@ -2666,6 +2844,18 @@ define amdgpu_kernel void @local_singlethread_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2833,6 +3023,18 @@ define amdgpu_kernel void @local_singlethread_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3000,6 +3202,18 @@ define amdgpu_kernel void @local_singlethread_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3167,6 +3381,18 @@ define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3334,6 +3560,18 @@ define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3501,6 +3739,18 @@ define amdgpu_kernel void @local_singlethread_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3668,6 +3918,18 @@ define amdgpu_kernel void @local_singlethread_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3835,6 +4097,18 @@ define amdgpu_kernel void @local_singlethread_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4002,6 +4276,18 @@ define amdgpu_kernel void @local_singlethread_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4169,6 +4455,18 @@ define amdgpu_kernel void @local_singlethread_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4336,6 +4634,18 @@ define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4503,6 +4813,18 @@ define amdgpu_kernel void @local_singlethread_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4670,6 +4992,18 @@ define amdgpu_kernel void @local_singlethread_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4837,6 +5171,18 @@ define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5004,6 +5350,18 @@ define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5213,6 +5571,21 @@ define amdgpu_kernel void @local_singlethread_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5424,6 +5797,21 @@ define amdgpu_kernel void @local_singlethread_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5635,6 +6023,21 @@ define amdgpu_kernel void @local_singlethread_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5846,6 +6249,21 @@ define amdgpu_kernel void @local_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6057,6 +6475,21 @@ define amdgpu_kernel void @local_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6268,6 +6701,21 @@ define amdgpu_kernel void @local_singlethread_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6479,6 +6927,21 @@ define amdgpu_kernel void @local_singlethread_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6690,6 +7153,21 @@ define amdgpu_kernel void @local_singlethread_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6901,6 +7379,21 @@ define amdgpu_kernel void @local_singlethread_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7112,6 +7605,21 @@ define amdgpu_kernel void @local_singlethread_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7323,6 +7831,21 @@ define amdgpu_kernel void @local_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7534,6 +8057,21 @@ define amdgpu_kernel void @local_singlethread_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7745,6 +8283,21 @@ define amdgpu_kernel void @local_singlethread_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7956,6 +8509,21 @@ define amdgpu_kernel void @local_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8167,6 +8735,21 @@ define amdgpu_kernel void @local_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8340,6 +8923,18 @@ define amdgpu_kernel void @local_singlethread_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread-one-as") unordered, align 4
@@ -8511,6 +9106,18 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread-one-as") monotonic, align 4
@@ -8682,6 +9289,18 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread-one-as") acquire, align 4
@@ -8853,6 +9472,18 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("singlethread-one-as") seq_cst, align 4
@@ -8994,6 +9625,16 @@ define amdgpu_kernel void @local_singlethread_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread-one-as") unordered, align 4
@@ -9134,6 +9775,16 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread-one-as") monotonic, align 4
@@ -9274,6 +9925,16 @@ define amdgpu_kernel void @local_singlethread_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread-one-as") release, align 4
@@ -9414,6 +10075,16 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("singlethread-one-as") seq_cst, align 4
@@ -9554,6 +10225,16 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") monotonic
@@ -9694,6 +10375,16 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -9834,6 +10525,16 @@ define amdgpu_kernel void @local_singlethread_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") release
@@ -9974,6 +10675,16 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -10114,6 +10825,16 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -10296,6 +11017,19 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") acquire
@@ -10479,6 +11213,19 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") acq_rel
@@ -10662,6 +11409,19 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("singlethread-one-as") seq_cst
@@ -10829,6 +11589,18 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_cmpxchg
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10996,6 +11768,18 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11163,6 +11947,18 @@ define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11330,6 +12126,18 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11497,6 +12305,18 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11664,6 +12484,18 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11831,6 +12663,18 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11998,6 +12842,18 @@ define amdgpu_kernel void @local_singlethread_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12165,6 +13021,18 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12332,6 +13200,18 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12499,6 +13379,18 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12666,6 +13558,18 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12833,6 +13737,18 @@ define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13000,6 +13916,18 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13167,6 +14095,18 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13376,6 +14316,21 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_monotonic_ret_cmp
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13587,6 +14542,21 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_monotonic_ret_cmpxc
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13798,6 +14768,21 @@ define amdgpu_kernel void @local_singlethread_one_as_release_monotonic_ret_cmpxc
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14009,6 +14994,21 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_monotonic_ret_cmpxc
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14220,6 +15220,21 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_monotonic_ret_cmpxc
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14431,6 +15446,21 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_acquire_ret_cmpxc
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14642,6 +15672,21 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_acquire_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14853,6 +15898,21 @@ define amdgpu_kernel void @local_singlethread_one_as_release_acquire_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15064,6 +16124,21 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15275,6 +16350,21 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15486,6 +16576,21 @@ define amdgpu_kernel void @local_singlethread_one_as_monotonic_seq_cst_ret_cmpxc
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15697,6 +16802,21 @@ define amdgpu_kernel void @local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15908,6 +17028,21 @@ define amdgpu_kernel void @local_singlethread_one_as_release_seq_cst_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16119,6 +17254,21 @@ define amdgpu_kernel void @local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16330,6 +17480,21 @@ define amdgpu_kernel void @local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
index 74a297241d851..1356fe4854170 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-system.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @local_system_unordered_load(
 ; GFX6-LABEL: local_system_unordered_load:
@@ -177,6 +178,18 @@ define amdgpu_kernel void @local_system_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in unordered, align 4
@@ -348,6 +361,18 @@ define amdgpu_kernel void @local_system_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in monotonic, align 4
@@ -524,6 +549,18 @@ define amdgpu_kernel void @local_system_acquire_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in acquire, align 4
@@ -718,6 +755,19 @@ define amdgpu_kernel void @local_system_seq_cst_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in seq_cst, align 4
@@ -859,6 +909,16 @@ define amdgpu_kernel void @local_system_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out unordered, align 4
@@ -999,6 +1059,16 @@ define amdgpu_kernel void @local_system_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out monotonic, align 4
@@ -1157,6 +1227,17 @@ define amdgpu_kernel void @local_system_release_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out release, align 4
@@ -1315,6 +1396,17 @@ define amdgpu_kernel void @local_system_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out seq_cst, align 4
@@ -1455,6 +1547,16 @@ define amdgpu_kernel void @local_system_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in monotonic
@@ -1611,6 +1713,17 @@ define amdgpu_kernel void @local_system_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in acquire
@@ -1769,6 +1882,17 @@ define amdgpu_kernel void @local_system_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in release
@@ -1943,6 +2067,18 @@ define amdgpu_kernel void @local_system_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in acq_rel
@@ -2117,6 +2253,18 @@ define amdgpu_kernel void @local_system_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in seq_cst
@@ -2304,6 +2452,19 @@ define amdgpu_kernel void @local_system_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in acquire
@@ -2510,6 +2671,20 @@ define amdgpu_kernel void @local_system_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in acq_rel
@@ -2716,6 +2891,20 @@ define amdgpu_kernel void @local_system_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in seq_cst
@@ -2883,6 +3072,18 @@ define amdgpu_kernel void @local_system_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3066,6 +3267,19 @@ define amdgpu_kernel void @local_system_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3251,6 +3465,19 @@ define amdgpu_kernel void @local_system_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3452,6 +3679,20 @@ define amdgpu_kernel void @local_system_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3653,6 +3894,20 @@ define amdgpu_kernel void @local_system_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3836,6 +4091,19 @@ define amdgpu_kernel void @local_system_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4019,6 +4287,19 @@ define amdgpu_kernel void @local_system_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4220,6 +4501,20 @@ define amdgpu_kernel void @local_system_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4421,6 +4716,20 @@ define amdgpu_kernel void @local_system_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4622,6 +4931,20 @@ define amdgpu_kernel void @local_system_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4823,6 +5146,20 @@ define amdgpu_kernel void @local_system_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5024,6 +5361,20 @@ define amdgpu_kernel void @local_system_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5225,6 +5576,20 @@ define amdgpu_kernel void @local_system_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5426,6 +5791,20 @@ define amdgpu_kernel void @local_system_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5627,6 +6006,20 @@ define amdgpu_kernel void @local_system_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5836,6 +6229,21 @@ define amdgpu_kernel void @local_system_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6052,6 +6460,21 @@ define amdgpu_kernel void @local_system_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6281,6 +6704,22 @@ define amdgpu_kernel void @local_system_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6515,6 +6954,22 @@ define amdgpu_kernel void @local_system_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6749,6 +7204,22 @@ define amdgpu_kernel void @local_system_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6965,6 +7436,21 @@ define amdgpu_kernel void @local_system_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7181,6 +7667,21 @@ define amdgpu_kernel void @local_system_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7415,6 +7916,22 @@ define amdgpu_kernel void @local_system_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7649,6 +8166,22 @@ define amdgpu_kernel void @local_system_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7883,6 +8416,22 @@ define amdgpu_kernel void @local_system_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8117,6 +8666,22 @@ define amdgpu_kernel void @local_system_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8351,6 +8916,22 @@ define amdgpu_kernel void @local_system_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8585,6 +9166,22 @@ define amdgpu_kernel void @local_system_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8819,6 +9416,22 @@ define amdgpu_kernel void @local_system_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9053,6 +9666,22 @@ define amdgpu_kernel void @local_system_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9226,6 +9855,18 @@ define amdgpu_kernel void @local_system_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("one-as") unordered, align 4
@@ -9397,6 +10038,18 @@ define amdgpu_kernel void @local_system_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("one-as") monotonic, align 4
@@ -9568,6 +10221,18 @@ define amdgpu_kernel void @local_system_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("one-as") acquire, align 4
@@ -9739,6 +10404,18 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("one-as") seq_cst, align 4
@@ -9880,6 +10557,16 @@ define amdgpu_kernel void @local_system_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("one-as") unordered, align 4
@@ -10020,6 +10707,16 @@ define amdgpu_kernel void @local_system_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("one-as") monotonic, align 4
@@ -10160,6 +10857,16 @@ define amdgpu_kernel void @local_system_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("one-as") release, align 4
@@ -10300,6 +11007,16 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("one-as") seq_cst, align 4
@@ -10440,6 +11157,16 @@ define amdgpu_kernel void @local_system_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") monotonic
@@ -10580,6 +11307,16 @@ define amdgpu_kernel void @local_system_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") acquire
@@ -10720,6 +11457,16 @@ define amdgpu_kernel void @local_system_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") release
@@ -10860,6 +11607,16 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") acq_rel
@@ -11000,6 +11757,16 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") seq_cst
@@ -11182,6 +11949,19 @@ define amdgpu_kernel void @local_system_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") acquire
@@ -11365,6 +12145,19 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") acq_rel
@@ -11548,6 +12341,19 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("one-as") seq_cst
@@ -11715,6 +12521,18 @@ define amdgpu_kernel void @local_system_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11882,6 +12700,18 @@ define amdgpu_kernel void @local_system_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12049,6 +12879,18 @@ define amdgpu_kernel void @local_system_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12216,6 +13058,18 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12383,6 +13237,18 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12550,6 +13416,18 @@ define amdgpu_kernel void @local_system_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12717,6 +13595,18 @@ define amdgpu_kernel void @local_system_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12884,6 +13774,18 @@ define amdgpu_kernel void @local_system_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13051,6 +13953,18 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13218,6 +14132,18 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13385,6 +14311,18 @@ define amdgpu_kernel void @local_system_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13552,6 +14490,18 @@ define amdgpu_kernel void @local_system_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13719,6 +14669,18 @@ define amdgpu_kernel void @local_system_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13886,6 +14848,18 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14053,6 +15027,18 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14262,6 +15248,21 @@ define amdgpu_kernel void @local_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14473,6 +15474,21 @@ define amdgpu_kernel void @local_system_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14684,6 +15700,21 @@ define amdgpu_kernel void @local_system_one_as_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14895,6 +15926,21 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15106,6 +16152,21 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15317,6 +16378,21 @@ define amdgpu_kernel void @local_system_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15528,6 +16604,21 @@ define amdgpu_kernel void @local_system_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15739,6 +16830,21 @@ define amdgpu_kernel void @local_system_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15950,6 +17056,21 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16161,6 +17282,21 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16372,6 +17508,21 @@ define amdgpu_kernel void @local_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16583,6 +17734,21 @@ define amdgpu_kernel void @local_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16794,6 +17960,21 @@ define amdgpu_kernel void @local_system_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -17005,6 +18186,21 @@ define amdgpu_kernel void @local_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -17216,6 +18412,21 @@ define amdgpu_kernel void @local_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
index 5e5e3bf83d610..75e28f9008e28 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
@@ -8,6 +8,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @local_volatile_load_0(
 ; GFX6-LABEL: local_volatile_load_0:
@@ -141,6 +142,18 @@ define amdgpu_kernel void @local_volatile_load_0(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_volatile_load_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    ds_load_b32 v1, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(1) %out) {
 entry:
   %val = load volatile i32, ptr addrspace(3) %in, align 4
@@ -308,6 +321,22 @@ define amdgpu_kernel void @local_volatile_load_1(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_volatile_load_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    v_mov_b32_e32 v1, v0
+; GFX1250-NEXT:    s_load_b32 s3, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    s_mov_b32 s2, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v1, v1, s2
+; GFX1250-NEXT:    s_mov_b32 s2, 2
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_lshl_add_u32 v1, v1, s2, s3
+; GFX1250-NEXT:    ds_load_b32 v1, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(1) %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -429,6 +458,18 @@ define amdgpu_kernel void @local_volatile_store_0(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_volatile_store_0:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[2:3], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[2:3], 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(3) %out) {
 entry:
   %val = load i32, ptr addrspace(1) %in, align 4
@@ -570,6 +611,22 @@ define amdgpu_kernel void @local_volatile_store_1(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_volatile_store_1:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[0:1], 0x0
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_mov_b32 s1, 0x3ff
+; GFX1250-NEXT:    v_and_b32_e64 v0, v0, s1
+; GFX1250-NEXT:    s_mov_b32 s1, 2
+; GFX1250-NEXT:    v_lshl_add_u32 v0, v0, s1, s2
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(3) %out) {
 entry:
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -698,6 +755,18 @@ define amdgpu_kernel void @local_volatile_workgroup_acquire_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_volatile_workgroup_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic volatile i32, ptr addrspace(3) %in syncscope("workgroup") acquire, align 4
@@ -813,6 +882,17 @@ define amdgpu_kernel void @local_volatile_workgroup_release_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_volatile_workgroup_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
    i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic volatile i32 %in, ptr addrspace(3) %out syncscope("workgroup") release, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
index b24622a48a16b..7e345ed6e2716 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-wavefront.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @local_wavefront_unordered_load(
 ; GFX6-LABEL: local_wavefront_unordered_load:
@@ -177,6 +178,18 @@ define amdgpu_kernel void @local_wavefront_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront") unordered, align 4
@@ -348,6 +361,18 @@ define amdgpu_kernel void @local_wavefront_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront") monotonic, align 4
@@ -519,6 +544,18 @@ define amdgpu_kernel void @local_wavefront_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront") acquire, align 4
@@ -690,6 +727,18 @@ define amdgpu_kernel void @local_wavefront_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront") seq_cst, align 4
@@ -831,6 +880,16 @@ define amdgpu_kernel void @local_wavefront_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront") unordered, align 4
@@ -971,6 +1030,16 @@ define amdgpu_kernel void @local_wavefront_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront") monotonic, align 4
@@ -1111,6 +1180,16 @@ define amdgpu_kernel void @local_wavefront_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront") release, align 4
@@ -1251,6 +1330,16 @@ define amdgpu_kernel void @local_wavefront_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront") seq_cst, align 4
@@ -1391,6 +1480,16 @@ define amdgpu_kernel void @local_wavefront_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") monotonic
@@ -1531,6 +1630,16 @@ define amdgpu_kernel void @local_wavefront_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") acquire
@@ -1671,6 +1780,16 @@ define amdgpu_kernel void @local_wavefront_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") release
@@ -1811,6 +1930,16 @@ define amdgpu_kernel void @local_wavefront_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") acq_rel
@@ -1951,6 +2080,16 @@ define amdgpu_kernel void @local_wavefront_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") seq_cst
@@ -2133,6 +2272,19 @@ define amdgpu_kernel void @local_wavefront_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") acquire
@@ -2316,6 +2468,19 @@ define amdgpu_kernel void @local_wavefront_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") acq_rel
@@ -2499,6 +2664,19 @@ define amdgpu_kernel void @local_wavefront_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront") seq_cst
@@ -2666,6 +2844,18 @@ define amdgpu_kernel void @local_wavefront_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -2833,6 +3023,18 @@ define amdgpu_kernel void @local_wavefront_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3000,6 +3202,18 @@ define amdgpu_kernel void @local_wavefront_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3167,6 +3381,18 @@ define amdgpu_kernel void @local_wavefront_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3334,6 +3560,18 @@ define amdgpu_kernel void @local_wavefront_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3501,6 +3739,18 @@ define amdgpu_kernel void @local_wavefront_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3668,6 +3918,18 @@ define amdgpu_kernel void @local_wavefront_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3835,6 +4097,18 @@ define amdgpu_kernel void @local_wavefront_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4002,6 +4276,18 @@ define amdgpu_kernel void @local_wavefront_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4169,6 +4455,18 @@ define amdgpu_kernel void @local_wavefront_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4336,6 +4634,18 @@ define amdgpu_kernel void @local_wavefront_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4503,6 +4813,18 @@ define amdgpu_kernel void @local_wavefront_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4670,6 +4992,18 @@ define amdgpu_kernel void @local_wavefront_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4837,6 +5171,18 @@ define amdgpu_kernel void @local_wavefront_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5004,6 +5350,18 @@ define amdgpu_kernel void @local_wavefront_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5213,6 +5571,21 @@ define amdgpu_kernel void @local_wavefront_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5424,6 +5797,21 @@ define amdgpu_kernel void @local_wavefront_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5635,6 +6023,21 @@ define amdgpu_kernel void @local_wavefront_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5846,6 +6249,21 @@ define amdgpu_kernel void @local_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6057,6 +6475,21 @@ define amdgpu_kernel void @local_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6268,6 +6701,21 @@ define amdgpu_kernel void @local_wavefront_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6479,6 +6927,21 @@ define amdgpu_kernel void @local_wavefront_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6690,6 +7153,21 @@ define amdgpu_kernel void @local_wavefront_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6901,6 +7379,21 @@ define amdgpu_kernel void @local_wavefront_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7112,6 +7605,21 @@ define amdgpu_kernel void @local_wavefront_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7323,6 +7831,21 @@ define amdgpu_kernel void @local_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7534,6 +8057,21 @@ define amdgpu_kernel void @local_wavefront_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7745,6 +8283,21 @@ define amdgpu_kernel void @local_wavefront_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7956,6 +8509,21 @@ define amdgpu_kernel void @local_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8167,6 +8735,21 @@ define amdgpu_kernel void @local_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8340,6 +8923,18 @@ define amdgpu_kernel void @local_wavefront_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront-one-as") unordered, align 4
@@ -8511,6 +9106,18 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront-one-as") monotonic, align 4
@@ -8682,6 +9289,18 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront-one-as") acquire, align 4
@@ -8853,6 +9472,18 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("wavefront-one-as") seq_cst, align 4
@@ -8994,6 +9625,16 @@ define amdgpu_kernel void @local_wavefront_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront-one-as") unordered, align 4
@@ -9134,6 +9775,16 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront-one-as") monotonic, align 4
@@ -9274,6 +9925,16 @@ define amdgpu_kernel void @local_wavefront_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront-one-as") release, align 4
@@ -9414,6 +10075,16 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("wavefront-one-as") seq_cst, align 4
@@ -9554,6 +10225,16 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") monotonic
@@ -9694,6 +10375,16 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -9834,6 +10525,16 @@ define amdgpu_kernel void @local_wavefront_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") release
@@ -9974,6 +10675,16 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -10114,6 +10825,16 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -10296,6 +11017,19 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") acquire
@@ -10479,6 +11213,19 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") acq_rel
@@ -10662,6 +11409,19 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("wavefront-one-as") seq_cst
@@ -10829,6 +11589,18 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -10996,6 +11768,18 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11163,6 +11947,18 @@ define amdgpu_kernel void @local_wavefront_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11330,6 +12126,18 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11497,6 +12305,18 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11664,6 +12484,18 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11831,6 +12663,18 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11998,6 +12842,18 @@ define amdgpu_kernel void @local_wavefront_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12165,6 +13021,18 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12332,6 +13200,18 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12499,6 +13379,18 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12666,6 +13558,18 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12833,6 +13737,18 @@ define amdgpu_kernel void @local_wavefront_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13000,6 +13916,18 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13167,6 +14095,18 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13376,6 +14316,21 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_monotonic_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13587,6 +14542,21 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13798,6 +14768,21 @@ define amdgpu_kernel void @local_wavefront_one_as_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14009,6 +14994,21 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14220,6 +15220,21 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14431,6 +15446,21 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14642,6 +15672,21 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14853,6 +15898,21 @@ define amdgpu_kernel void @local_wavefront_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15064,6 +16124,21 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15275,6 +16350,21 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15486,6 +16576,21 @@ define amdgpu_kernel void @local_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15697,6 +16802,21 @@ define amdgpu_kernel void @local_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15908,6 +17028,21 @@ define amdgpu_kernel void @local_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16119,6 +17254,21 @@ define amdgpu_kernel void @local_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16330,6 +17480,21 @@ define amdgpu_kernel void @local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
index 62d7f4801baf8..6aaf9d323b1fd 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-workgroup.ll
@@ -12,6 +12,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1100 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX11-CU %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX12-WGP %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1200 -mattr=+cumode < %s | FileCheck --check-prefixes=GFX12-CU %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @local_workgroup_unordered_load(
 ; GFX6-LABEL: local_workgroup_unordered_load:
@@ -177,6 +178,18 @@ define amdgpu_kernel void @local_workgroup_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") unordered, align 4
@@ -348,6 +361,18 @@ define amdgpu_kernel void @local_workgroup_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") monotonic, align 4
@@ -524,6 +549,18 @@ define amdgpu_kernel void @local_workgroup_acquire_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") acquire, align 4
@@ -718,6 +755,19 @@ define amdgpu_kernel void @local_workgroup_seq_cst_load(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup") seq_cst, align 4
@@ -859,6 +909,16 @@ define amdgpu_kernel void @local_workgroup_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") unordered, align 4
@@ -999,6 +1059,16 @@ define amdgpu_kernel void @local_workgroup_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") monotonic, align 4
@@ -1157,6 +1227,17 @@ define amdgpu_kernel void @local_workgroup_release_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") release, align 4
@@ -1315,6 +1396,17 @@ define amdgpu_kernel void @local_workgroup_seq_cst_store(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup") seq_cst, align 4
@@ -1455,6 +1547,16 @@ define amdgpu_kernel void @local_workgroup_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") monotonic
@@ -1611,6 +1713,17 @@ define amdgpu_kernel void @local_workgroup_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acquire
@@ -1769,6 +1882,17 @@ define amdgpu_kernel void @local_workgroup_release_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") release
@@ -1943,6 +2067,18 @@ define amdgpu_kernel void @local_workgroup_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acq_rel
@@ -2117,6 +2253,18 @@ define amdgpu_kernel void @local_workgroup_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") seq_cst
@@ -2304,6 +2452,19 @@ define amdgpu_kernel void @local_workgroup_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acquire
@@ -2510,6 +2671,20 @@ define amdgpu_kernel void @local_workgroup_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") acq_rel
@@ -2716,6 +2891,20 @@ define amdgpu_kernel void @local_workgroup_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup") seq_cst
@@ -2883,6 +3072,18 @@ define amdgpu_kernel void @local_workgroup_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3066,6 +3267,19 @@ define amdgpu_kernel void @local_workgroup_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3251,6 +3465,19 @@ define amdgpu_kernel void @local_workgroup_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3452,6 +3679,20 @@ define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3653,6 +3894,20 @@ define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -3836,6 +4091,19 @@ define amdgpu_kernel void @local_workgroup_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4019,6 +4287,19 @@ define amdgpu_kernel void @local_workgroup_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4220,6 +4501,20 @@ define amdgpu_kernel void @local_workgroup_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4421,6 +4716,20 @@ define amdgpu_kernel void @local_workgroup_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4622,6 +4931,20 @@ define amdgpu_kernel void @local_workgroup_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -4823,6 +5146,20 @@ define amdgpu_kernel void @local_workgroup_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5024,6 +5361,20 @@ define amdgpu_kernel void @local_workgroup_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5225,6 +5576,20 @@ define amdgpu_kernel void @local_workgroup_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5426,6 +5791,20 @@ define amdgpu_kernel void @local_workgroup_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5627,6 +6006,20 @@ define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -5836,6 +6229,21 @@ define amdgpu_kernel void @local_workgroup_monotonic_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6052,6 +6460,21 @@ define amdgpu_kernel void @local_workgroup_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6281,6 +6704,22 @@ define amdgpu_kernel void @local_workgroup_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6515,6 +6954,22 @@ define amdgpu_kernel void @local_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6749,6 +7204,22 @@ define amdgpu_kernel void @local_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -6965,6 +7436,21 @@ define amdgpu_kernel void @local_workgroup_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7181,6 +7667,21 @@ define amdgpu_kernel void @local_workgroup_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7415,6 +7916,22 @@ define amdgpu_kernel void @local_workgroup_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7649,6 +8166,22 @@ define amdgpu_kernel void @local_workgroup_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -7883,6 +8416,22 @@ define amdgpu_kernel void @local_workgroup_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8117,6 +8666,22 @@ define amdgpu_kernel void @local_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8351,6 +8916,22 @@ define amdgpu_kernel void @local_workgroup_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8585,6 +9166,22 @@ define amdgpu_kernel void @local_workgroup_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -8819,6 +9416,22 @@ define amdgpu_kernel void @local_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9053,6 +9666,22 @@ define amdgpu_kernel void @local_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -9226,6 +9855,18 @@ define amdgpu_kernel void @local_workgroup_one_as_unordered_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_unordered_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") unordered, align 4
@@ -9397,6 +10038,18 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_monotonic_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") monotonic, align 4
@@ -9568,6 +10221,18 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acquire_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") acquire, align 4
@@ -9739,6 +10404,18 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_load(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_load:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    ds_load_b32 v1, v0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %in, ptr addrspace(3) %out) {
 entry:
   %val = load atomic i32, ptr addrspace(3) %in syncscope("workgroup-one-as") seq_cst, align 4
@@ -9880,6 +10557,16 @@ define amdgpu_kernel void @local_workgroup_one_as_unordered_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_unordered_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") unordered, align 4
@@ -10020,6 +10707,16 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_monotonic_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") monotonic, align 4
@@ -10160,6 +10857,16 @@ define amdgpu_kernel void @local_workgroup_one_as_release_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_release_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") release, align 4
@@ -10300,6 +11007,16 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_store(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_store:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     i32 %in, ptr addrspace(3) %out) {
 entry:
   store atomic i32 %in, ptr addrspace(3) %out syncscope("workgroup-one-as") seq_cst, align 4
@@ -10440,6 +11157,16 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_monotonic_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") monotonic
@@ -10580,6 +11307,16 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acquire_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -10720,6 +11457,16 @@ define amdgpu_kernel void @local_workgroup_one_as_release_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_release_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") release
@@ -10860,6 +11607,16 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acq_rel_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -11000,6 +11757,16 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_atomicrmw(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX12-CU-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s0
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v0, v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -11182,6 +11949,19 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acquire_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acquire
@@ -11365,6 +12145,19 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acq_rel_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") acq_rel
@@ -11548,6 +12341,19 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_ret_atomicrmw(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_ret_atomicrmw:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    ds_storexchg_rtn_b32 v1, v0, v1
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in) {
 entry:
   %val = atomicrmw volatile xchg ptr addrspace(3) %out, i32 %in syncscope("workgroup-one-as") seq_cst
@@ -11715,6 +12521,18 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_monotonic_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -11882,6 +12700,18 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acquire_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12049,6 +12879,18 @@ define amdgpu_kernel void @local_workgroup_one_as_release_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_release_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12216,6 +13058,18 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acq_rel_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12383,6 +13237,18 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_monotonic_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12550,6 +13416,18 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_monotonic_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12717,6 +13595,18 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acquire_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -12884,6 +13774,18 @@ define amdgpu_kernel void @local_workgroup_one_as_release_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_release_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13051,6 +13953,18 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acq_rel_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13218,6 +14132,18 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_acquire_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13385,6 +14311,18 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_monotonic_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13552,6 +14490,18 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acquire_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13719,6 +14669,18 @@ define amdgpu_kernel void @local_workgroup_one_as_release_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_release_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -13886,6 +14848,18 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acq_rel_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14053,6 +15027,18 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX12-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX12-CU-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_seq_cst_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s1
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    ds_cmpstore_b32 v0, v1, v2 offset:16
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14262,6 +15248,21 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_monotonic_ret_cmpxch
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_monotonic_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14473,6 +15474,21 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14684,6 +15700,21 @@ define amdgpu_kernel void @local_workgroup_one_as_release_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_release_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -14895,6 +15926,21 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15106,6 +16152,21 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15317,6 +16378,21 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15528,6 +16604,21 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acquire_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15739,6 +16830,21 @@ define amdgpu_kernel void @local_workgroup_one_as_release_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_release_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -15950,6 +17056,21 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16161,6 +17282,21 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16372,6 +17508,21 @@ define amdgpu_kernel void @local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16583,6 +17734,21 @@ define amdgpu_kernel void @local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -16794,6 +17960,21 @@ define amdgpu_kernel void @local_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_release_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -17005,6 +18186,21 @@ define amdgpu_kernel void @local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
@@ -17216,6 +18412,21 @@ define amdgpu_kernel void @local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX12-CU-NEXT:    s_wait_dscnt 0x0
 ; GFX12-CU-NEXT:    ds_store_b32 v0, v1
 ; GFX12-CU-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: local_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
+; GFX1250:       ; %bb.0: ; %entry
+; GFX1250-NEXT:    s_load_b32 s0, s[4:5], 0x0
+; GFX1250-NEXT:    s_load_b32 s2, s[4:5], 0x4
+; GFX1250-NEXT:    s_load_b32 s1, s[4:5], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s2
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:16
+; GFX1250-NEXT:    v_mov_b32_e32 v0, s0
+; GFX1250-NEXT:    s_wait_dscnt 0x0
+; GFX1250-NEXT:    ds_store_b32 v0, v1
+; GFX1250-NEXT:    s_endpgm
     ptr addrspace(3) %out, i32 %in, i32 %old) {
 entry:
   %gep = getelementptr i32, ptr addrspace(3) %out, i32 4
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
index 3751ae1f919eb..d9a6eb62f7a76 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
@@ -90,8 +90,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 4, alignment: 4, stack-id: default,
       isImmutable: false, isAliased: false, callee-saved-register: '' }
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-2.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-2.mir
index 4eec06b799248..8e6d9489c5190 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-2.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-2.mir
@@ -70,8 +70,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 4, alignment: 4, stack-id: default,
       isImmutable: false, isAliased: false, callee-saved-register: '' }
diff --git a/llvm/test/CodeGen/AMDGPU/merge-flat-saddr-load-store.mir b/llvm/test/CodeGen/AMDGPU/merge-flat-saddr-load-store.mir
new file mode 100644
index 0000000000000..1c133c6114ec2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/merge-flat-saddr-load-store.mir
@@ -0,0 +1,338 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -run-pass=si-load-store-opt -o - %s | FileCheck -check-prefix=GCN %s
+
+---
+name:            merge_flat_load_dword_saddr_2
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: merge_flat_load_dword_saddr_2
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[FLAT_LOAD_DWORDX2_SADDR:%[0-9]+]]:vreg_64_align2 = FLAT_LOAD_DWORDX2_SADDR [[DEF]], [[DEF1]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s64) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY [[FLAT_LOAD_DWORDX2_SADDR]].sub0
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed [[FLAT_LOAD_DWORDX2_SADDR]].sub1
+    ; GCN-NEXT: S_NOP 0, implicit [[COPY]], implicit [[COPY1]]
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %3:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 4, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    S_NOP 0, implicit %2, implicit %3
+...
+
+---
+name:            merge_flat_load_dword_saddr_3
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: merge_flat_load_dword_saddr_3
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[FLAT_LOAD_DWORDX3_SADDR:%[0-9]+]]:vreg_96_align2 = FLAT_LOAD_DWORDX3_SADDR [[DEF]], [[DEF1]], 0, 1, implicit $exec, implicit $flat_scr :: (load (s96) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY [[FLAT_LOAD_DWORDX3_SADDR]].sub0_sub1
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed [[FLAT_LOAD_DWORDX3_SADDR]].sub2
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[COPY]].sub1
+    ; GCN-NEXT: S_NOP 0, implicit [[COPY2]], implicit [[COPY3]], implicit [[COPY1]]
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 0, 1, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %3:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 4, 1, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %4:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 8, 1, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    S_NOP 0, implicit %2, implicit %3, implicit %4
+...
+
+---
+name:            merge_flat_load_dword_saddr_4
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: merge_flat_load_dword_saddr_4
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[FLAT_LOAD_DWORDX4_SADDR:%[0-9]+]]:vreg_128_align2 = FLAT_LOAD_DWORDX4_SADDR [[DEF]], [[DEF1]], 0, 2, implicit $exec, implicit $flat_scr :: (load (s128) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_96_align2 = COPY [[FLAT_LOAD_DWORDX4_SADDR]].sub0_sub1_sub2
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed [[FLAT_LOAD_DWORDX4_SADDR]].sub3
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[COPY]].sub0_sub1
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[COPY]].sub2
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]].sub0
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[COPY2]].sub1
+    ; GCN-NEXT: S_NOP 0, implicit [[COPY4]], implicit [[COPY5]], implicit [[COPY3]], implicit [[COPY1]]
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 0, 2, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %3:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 4, 2, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %4:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 8, 2, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %5:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 12, 2, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    S_NOP 0, implicit %2, implicit %3, implicit %4, implicit %5
+...
+
+---
+name:            merge_flat_load_dword_saddr_6
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: merge_flat_load_dword_saddr_6
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[FLAT_LOAD_DWORDX4_SADDR:%[0-9]+]]:vreg_128_align2 = FLAT_LOAD_DWORDX4_SADDR [[DEF]], [[DEF1]], 4, 3, implicit $exec, implicit $flat_scr :: (load (s128) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_96_align2 = COPY [[FLAT_LOAD_DWORDX4_SADDR]].sub0_sub1_sub2
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY killed [[FLAT_LOAD_DWORDX4_SADDR]].sub3
+    ; GCN-NEXT: [[COPY2:%[0-9]+]]:vreg_64_align2 = COPY [[COPY]].sub0_sub1
+    ; GCN-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY killed [[COPY]].sub2
+    ; GCN-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[COPY2]].sub0
+    ; GCN-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY killed [[COPY2]].sub1
+    ; GCN-NEXT: [[FLAT_LOAD_DWORDX2_SADDR:%[0-9]+]]:vreg_64_align2 = FLAT_LOAD_DWORDX2_SADDR [[DEF]], [[DEF1]], 20, 3, implicit $exec, implicit $flat_scr :: (load (s64) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    ; GCN-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[FLAT_LOAD_DWORDX2_SADDR]].sub0
+    ; GCN-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed [[FLAT_LOAD_DWORDX2_SADDR]].sub1
+    ; GCN-NEXT: S_NOP 0, implicit [[COPY4]], implicit [[COPY5]], implicit [[COPY3]], implicit [[COPY1]], implicit [[COPY6]], implicit [[COPY7]]
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 4, 3, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %3:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 8, 3, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %4:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 12, 3, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %5:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 16, 3, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %6:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 20, 3, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %7:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1, 24, 3, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    S_NOP 0, implicit %2, implicit %3, implicit %4, implicit %5, implicit %6, implicit %7
+...
+
+---
+name:            merge_flat_load_dwordx2_saddr
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: merge_flat_load_dwordx2_saddr
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[FLAT_LOAD_DWORDX4_SADDR:%[0-9]+]]:vreg_128_align2 = FLAT_LOAD_DWORDX4_SADDR [[DEF]], [[DEF1]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s128) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY [[FLAT_LOAD_DWORDX4_SADDR]].sub0_sub1
+    ; GCN-NEXT: [[COPY1:%[0-9]+]]:vreg_64_align2 = COPY killed [[FLAT_LOAD_DWORDX4_SADDR]].sub2_sub3
+    ; GCN-NEXT: S_NOP 0, implicit [[COPY]], implicit [[COPY1]]
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vreg_64_align2 = FLAT_LOAD_DWORDX2_SADDR %0, %1, 0, 0, implicit $exec, implicit $flat_scr :: (load (s64) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %3:vreg_64_align2 = FLAT_LOAD_DWORDX2_SADDR %0, %1, 8, 0, implicit $exec, implicit $flat_scr :: (load (s64) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    S_NOP 0, implicit %2, implicit %3
+...
+
+---
+name:            no_merge_flat_load_dword_and_flat_load_dword_saddr
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: no_merge_flat_load_dword_and_flat_load_dword_saddr
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+    ; GCN-NEXT: [[FLAT_LOAD_DWORD:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD [[DEF1]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: [[FLAT_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD_SADDR [[DEF]], [[DEF1]].sub0, 4, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: S_NOP 0, implicit [[FLAT_LOAD_DWORD]], implicit [[FLAT_LOAD_DWORD_SADDR]]
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vreg_64_align2 = IMPLICIT_DEF
+    %2:vgpr_32 = FLAT_LOAD_DWORD %1, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %3:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1.sub0, 4, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    S_NOP 0, implicit %2, implicit %3
+...
+
+---
+name:            no_merge_flat_load_dword_saddr_different_saddr
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: no_merge_flat_load_dword_saddr_different_saddr
+    ; GCN: [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[FLAT_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD_SADDR [[DEF]].sub0_sub1, [[DEF1]], 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: [[FLAT_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD_SADDR [[DEF]].sub2_sub3, [[DEF1]], 4, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: S_NOP 0, implicit [[FLAT_LOAD_DWORD_SADDR]], implicit [[FLAT_LOAD_DWORD_SADDR1]]
+    %0:sgpr_128 = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0.sub0_sub1, %1, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %3:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0.sub2_sub3, %1, 4, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    S_NOP 0, implicit %2, implicit %3
+...
+
+---
+name:            no_merge_flat_load_dword_saddr_different_vaddr
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: no_merge_flat_load_dword_saddr_different_vaddr
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+    ; GCN-NEXT: [[FLAT_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD_SADDR [[DEF]], [[DEF1]].sub0, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: [[FLAT_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = FLAT_LOAD_DWORD_SADDR [[DEF]], [[DEF1]].sub1, 4, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: S_NOP 0, implicit [[FLAT_LOAD_DWORD_SADDR]], implicit [[FLAT_LOAD_DWORD_SADDR1]]
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vreg_64_align2 = IMPLICIT_DEF
+    %2:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1.sub0, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %3:vgpr_32 = FLAT_LOAD_DWORD_SADDR %0, %1.sub1, 4, 0, implicit $exec, implicit $flat_scr :: (load (s32) from `ptr addrspace(1) undef`, align 4, addrspace 1)
+    S_NOP 0, implicit %2, implicit %3
+...
+---
+name:            merge_flat_store_dword_saddr_2
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: merge_flat_store_dword_saddr_2
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[DEF2]], %subreg.sub0, [[DEF3]], %subreg.sub1
+    ; GCN-NEXT: FLAT_STORE_DWORDX2_SADDR [[DEF1]], killed [[REG_SEQUENCE]], [[DEF]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s64) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %3:vgpr_32 = IMPLICIT_DEF
+    FLAT_STORE_DWORD_SADDR %1, %2, %0, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %3, %0, 4, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+...
+
+---
+name:            merge_flat_store_dword_saddr_3
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: merge_flat_store_dword_saddr_3
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[DEF2]], %subreg.sub0, [[DEF3]], %subreg.sub1
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE killed [[REG_SEQUENCE]], %subreg.sub0_sub1, [[DEF4]], %subreg.sub2
+    ; GCN-NEXT: FLAT_STORE_DWORDX3_SADDR [[DEF1]], killed [[REG_SEQUENCE1]], [[DEF]], 4, 1, implicit $exec, implicit $flat_scr :: (store (s96) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %3:vgpr_32 = IMPLICIT_DEF
+    %4:vgpr_32 = IMPLICIT_DEF
+    FLAT_STORE_DWORD_SADDR %1, %2, %0, 4, 1, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %3, %0, 8, 1, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %4, %0, 12, 1, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+...
+
+---
+name:            merge_flat_store_dword_saddr_4
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: merge_flat_store_dword_saddr_4
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[DEF2]], %subreg.sub0, [[DEF3]], %subreg.sub1
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE killed [[REG_SEQUENCE]], %subreg.sub0_sub1, [[DEF4]], %subreg.sub2
+    ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[REG_SEQUENCE1]], %subreg.sub0_sub1_sub2, [[DEF5]], %subreg.sub3
+    ; GCN-NEXT: FLAT_STORE_DWORDX4_SADDR [[DEF1]], killed [[REG_SEQUENCE2]], [[DEF]], 4, 2, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %3:vgpr_32 = IMPLICIT_DEF
+    %4:vgpr_32 = IMPLICIT_DEF
+    %5:vgpr_32 = IMPLICIT_DEF
+    FLAT_STORE_DWORD_SADDR %1, %2, %0, 4, 2, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %3, %0, 8, 2, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %4, %0, 12, 2, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %5, %0, 16, 2, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+...
+
+---
+name:            merge_flat_store_dword_saddr_6
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: merge_flat_store_dword_saddr_6
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF5:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF6:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF7:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[DEF2]], %subreg.sub0, [[DEF3]], %subreg.sub1
+    ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96_align2 = REG_SEQUENCE killed [[REG_SEQUENCE]], %subreg.sub0_sub1, [[DEF4]], %subreg.sub2
+    ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE killed [[REG_SEQUENCE1]], %subreg.sub0_sub1_sub2, [[DEF5]], %subreg.sub3
+    ; GCN-NEXT: FLAT_STORE_DWORDX4_SADDR [[DEF1]], killed [[REG_SEQUENCE2]], [[DEF]], 4, 3, implicit $exec, implicit $flat_scr :: (store (s128) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    ; GCN-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[DEF6]], %subreg.sub0, [[DEF7]], %subreg.sub1
+    ; GCN-NEXT: FLAT_STORE_DWORDX2_SADDR [[DEF1]], killed [[REG_SEQUENCE3]], [[DEF]], 20, 3, implicit $exec, implicit $flat_scr :: (store (s64) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %3:vgpr_32 = IMPLICIT_DEF
+    %4:vgpr_32 = IMPLICIT_DEF
+    %5:vgpr_32 = IMPLICIT_DEF
+    %6:vgpr_32 = IMPLICIT_DEF
+    %7:vgpr_32 = IMPLICIT_DEF
+    FLAT_STORE_DWORD_SADDR %1, %2, %0, 4, 3, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %3, %0, 8, 3, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %4, %0, 12, 3, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %5, %0, 16, 3, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %6, %0, 20, 3, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %7, %0, 24, 3, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+...
+
+---
+name:            no_merge_flat_store_dword_saddr_with_flat_store_dword
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: no_merge_flat_store_dword_saddr_with_flat_store_dword
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: FLAT_STORE_DWORD_SADDR [[DEF1]].sub0, [[DEF2]], [[DEF]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: FLAT_STORE_DWORD [[DEF1]], [[DEF3]], 4, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vreg_64_align2 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %3:vgpr_32 = IMPLICIT_DEF
+    FLAT_STORE_DWORD_SADDR %1.sub0, %2, %0, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD %1, %3, 4, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+...
+
+---
+name:            no_merge_flat_store_dword_saddr_different_vaddr
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: no_merge_flat_store_dword_saddr_different_vaddr
+    ; GCN: [[DEF:%[0-9]+]]:sreg_64_xexec_xnull = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vreg_64_align2 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: FLAT_STORE_DWORD_SADDR [[DEF1]].sub0, [[DEF2]], [[DEF]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: FLAT_STORE_DWORD_SADDR [[DEF1]].sub1, [[DEF3]], [[DEF]], 4, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
+    %0:sreg_64_xexec_xnull = IMPLICIT_DEF
+    %1:vreg_64_align2 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %3:vgpr_32 = IMPLICIT_DEF
+    FLAT_STORE_DWORD_SADDR %1.sub0, %2, %0, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1.sub1, %3, %0, 4, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+...
+
+---
+name:            no_merge_flat_store_dword_saddr_different_saddr
+body:             |
+  bb.0.entry:
+
+    ; GCN-LABEL: name: no_merge_flat_store_dword_saddr_different_saddr
+    ; GCN: [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[DEF3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: FLAT_STORE_DWORD_SADDR [[DEF1]], [[DEF2]], [[DEF]].sub0_sub1, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
+    ; GCN-NEXT: FLAT_STORE_DWORD_SADDR [[DEF1]], [[DEF3]], [[DEF]].sub2_sub3, 4, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, addrspace 1)
+    %0:sgpr_128 = IMPLICIT_DEF
+    %1:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %3:vgpr_32 = IMPLICIT_DEF
+    FLAT_STORE_DWORD_SADDR %1, %2, %0.sub0_sub1, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+    FLAT_STORE_DWORD_SADDR %1, %3, %0.sub2_sub3, 4, 0, implicit $exec, implicit $flat_scr :: (store (s32) into `ptr addrspace(1) undef`, align 4, addrspace 1)
+...
diff --git a/llvm/test/CodeGen/AMDGPU/merge-tbuffer-gfx11.mir b/llvm/test/CodeGen/AMDGPU/merge-tbuffer-gfx11.mir
index 62cc5659fcc6b..f5407a5223166 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-tbuffer-gfx11.mir
+++ b/llvm/test/CodeGen/AMDGPU/merge-tbuffer-gfx11.mir
@@ -1525,3 +1525,621 @@ body:             |
     %8:vgpr_32 = TBUFFER_LOAD_FORMAT_X_BOTHEN_exact %4, %5:sgpr_128, 0, 8, 22, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
     %9:vgpr_32 = TBUFFER_LOAD_FORMAT_X_BOTHEN_exact %4, %5:sgpr_128, 0, 12, 22, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 4)
 ...
+---
+
+name: gfx11_tbuffer_load_x_x_x_idxen_16bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0,$sgpr1,$sgpr2,$sgpr3,$vgpr0
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_x_x_x_idxen_16bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1, %subreg.sub1, $sgpr2, %subreg.sub2, $sgpr3, %subreg.sub3
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZ_IDXEN:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_IDXEN [[COPY]], %rsrc, 0, 0, 57, 0, 0, implicit $exec :: (dereferenceable load (s48), align 2, addrspace 8)
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vreg_64 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_IDXEN]].sub0_sub1
+    ; GFX11-NEXT: %x2:vgpr_32 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZ_IDXEN]].sub2
+    ; GFX11-NEXT: %x0:vgpr_32 = COPY [[COPY1]].sub0
+    ; GFX11-NEXT: %x1:vgpr_32 = COPY killed [[COPY1]].sub1
+    %0:vgpr_32 = COPY $vgpr0
+    %rsrc:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1,%subreg.sub1, $sgpr2, %subreg.sub2, $sgpr3, %subreg.sub3
+    %x0:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %0, %rsrc, 0, 0, 13, 0, 0, implicit $exec :: (dereferenceable load (s16),align 2,addrspace 8)
+    %x1:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %0, %rsrc, 0, 2, 13, 0, 0, implicit $exec :: (dereferenceable load (s16),align 2,addrspace 8)
+    %x2:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %0, %rsrc, 0, 4, 13, 0, 0, implicit $exec :: (dereferenceable load (s16),align 2,addrspace 8)
+...
+---
+
+name: gfx11_tbuffer_load_idxen_16_bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_idxen_16_bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 0, 57, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+    ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub0_sub1_sub2
+    ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub3
+    ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[COPY6]].sub0_sub1
+    ; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY killed [[COPY6]].sub2
+    ; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY8]].sub0
+    ; GFX11-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY killed [[COPY8]].sub1
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 16, 57, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+    ; GFX11-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN1]].sub0_sub1_sub2
+    ; GFX11-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN1]].sub3
+    ; GFX11-NEXT: [[COPY14:%[0-9]+]]:vreg_64 = COPY [[COPY12]].sub0_sub1
+    ; GFX11-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY killed [[COPY12]].sub2
+    ; GFX11-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY14]].sub0
+    ; GFX11-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY killed [[COPY14]].sub1
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 24, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %4:sgpr_32 = COPY $sgpr4
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %8:vgpr_32 = COPY %4:sgpr_32
+    %7:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 0, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %9:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 2, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %11:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 4, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %13:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 6, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %15:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 16, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %17:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 18, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %19:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 20, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %21:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 22, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %22:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 24, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+...
+---
+
+name: gfx11_tbuffer_load_xy_xy_idxen_uint_16_bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_xy_xy_idxen_uint_16_bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1, %subreg.sub1, $sgpr2, %subreg.sub2, $sgpr3, %subreg.sub3
+    ; GFX11-NEXT: %idx:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_IDXEN %idx, %rsrc, 0, 0, 55, 0, 0, implicit $exec :: (dereferenceable load (s64), align 2, addrspace 4)
+    ; GFX11-NEXT: %v0:vreg_64 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub0_sub1
+    ; GFX11-NEXT: %v1:vreg_64 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub2_sub3
+    %rsrc:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1,%subreg.sub1, $sgpr2,%subreg.sub2, $sgpr3,%subreg.sub3
+    %idx:vgpr_32 = COPY $vgpr0
+    %v0:vreg_64 = TBUFFER_LOAD_FORMAT_XY_IDXEN  %idx, %rsrc, 0, 0, 27, 0, 0, implicit $exec :: (dereferenceable load (s32),align 2,addrspace 4)
+    %v1:vreg_64 = TBUFFER_LOAD_FORMAT_XY_IDXEN  %idx, %rsrc, 0, 4, 27, 0, 0, implicit $exec :: (dereferenceable load (s32),align 2,addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_load_xy_xy_idxen_sint_16_bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_xy_xy_idxen_sint_16_bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1, %subreg.sub1, $sgpr2, %subreg.sub2, $sgpr3, %subreg.sub3
+    ; GFX11-NEXT: %idx:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_IDXEN %idx, %rsrc, 0, 0, 56, 0, 0, implicit $exec :: (dereferenceable load (s64), align 2, addrspace 4)
+    ; GFX11-NEXT: %v0:vreg_64 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub0_sub1
+    ; GFX11-NEXT: %v1:vreg_64 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub2_sub3
+    %rsrc:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1,%subreg.sub1, $sgpr2, %subreg.sub2, $sgpr3,%subreg.sub3
+    %idx:vgpr_32 = COPY $vgpr0
+    %v0:vreg_64 = TBUFFER_LOAD_FORMAT_XY_IDXEN  %idx, %rsrc, 0, 0, 28, 0, 0, implicit $exec :: (dereferenceable load (s32),align 2,addrspace 4)
+    %v1:vreg_64 = TBUFFER_LOAD_FORMAT_XY_IDXEN  %idx, %rsrc, 0, 4, 28, 0, 0, implicit $exec :: (dereferenceable load (s32),align 2,addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_load_x_off2_off4_16bit_no_merge
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_x_off2_off4_16bit_no_merge
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 2, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_X_IDXEN1:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 4, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %4:sgpr_32 = COPY $sgpr4
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %8:vgpr_32 = COPY %4:sgpr_32
+    %9:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 2, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    %11:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 4, 13, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+...
+---
+
+name: gfx11_tbuffer_store_x_x_x_idxen_16_bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+    ; GFX11-LABEL: name: gfx11_tbuffer_store_x_x_x_idxen_16_bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE]], %subreg.sub0_sub1, [[COPY2]], %subreg.sub2
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZ_OFFSET_exact killed [[REG_SEQUENCE1]], %rsrc, 0, 0, 57, 0, 0, implicit $exec :: (store (s48), align 2, addrspace 4)
+    %4:vgpr_32 = COPY $vgpr0
+    %5:vgpr_32 = COPY $vgpr1
+    %6:vgpr_32 = COPY $vgpr2
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %rsrc:sgpr_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2,%subreg.sub2,%3, %subreg.sub3
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %4, %rsrc, 0, 0, 13, 0, 0, implicit $exec :: (store (s16),align 2,addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %5, %rsrc, 0, 2, 13, 0, 0, implicit $exec :: (store (s16),align 2,addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %6, %rsrc, 0, 4, 13, 0, 0, implicit $exec :: (store (s16),align 2,addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_store_idxen_16_bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+    ; GFX11-LABEL: name: gfx11_tbuffer_store_idxen_16_bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY11]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY9]], %subreg.sub3
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE]], %subreg.sub0_sub1, [[COPY6]], %subreg.sub2
+    ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[REG_SEQUENCE1]], %subreg.sub0_sub1_sub2, [[COPY5]], %subreg.sub3
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[REG_SEQUENCE2]], %rsrc, 0, 0, 57, 0, 0, implicit $exec :: (store (s64), align 2, addrspace 4)
+    ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE3]], %subreg.sub0_sub1, [[COPY2]], %subreg.sub2
+    ; GFX11-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[REG_SEQUENCE4]], %subreg.sub0_sub1_sub2, [[COPY1]], %subreg.sub3
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[REG_SEQUENCE5]], %rsrc, 0, 8, 57, 0, 0, implicit $exec :: (store (s64), align 2, addrspace 4)
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY]], %rsrc, 0, 16, 13, 0, 0, implicit $exec :: (store (s16), addrspace 4)
+    %12:vgpr_32 = COPY $vgpr8
+    %11:vgpr_32 = COPY $vgpr7
+    %10:vgpr_32 = COPY $vgpr6
+    %9:vgpr_32  = COPY $vgpr5
+    %8:vgpr_32  = COPY $vgpr4
+    %7:vgpr_32  = COPY $vgpr3
+    %6:vgpr_32  = COPY $vgpr2
+    %5:vgpr_32  = COPY $vgpr1
+    %4:vgpr_32  = COPY $vgpr0
+    %3:sgpr_32  = COPY $sgpr3
+    %2:sgpr_32  = COPY $sgpr2
+    %1:sgpr_32  = COPY $sgpr1
+    %0:sgpr_32  = COPY $sgpr0
+    %rsrc:sgpr_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %4,  %rsrc, 0, 0,  13, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %5,  %rsrc, 0, 2,  13, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %6,  %rsrc, 0, 4,  13, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %7,  %rsrc, 0, 6,  13, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %8,  %rsrc, 0, 8,  13, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %9,  %rsrc, 0, 10, 13, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %10, %rsrc, 0, 12, 13, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %11, %rsrc, 0, 14, 13, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %12, %rsrc, 0, 16, 13, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_store_xy_xy_uint_16_bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX11-LABEL: name: gfx11_tbuffer_store_xy_xy_uint_16_bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1, %subreg.sub1, $sgpr2, %subreg.sub2, $sgpr3, %subreg.sub3
+    ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE $vgpr0, %subreg.sub0, $vgpr1, %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE $vgpr2, %subreg.sub0, $vgpr3, %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE1]], %subreg.sub0_sub1, [[REG_SEQUENCE2]], %subreg.sub2_sub3
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]], 0, 0, 55, 0, 0, implicit $exec :: (store (s64), align 2, addrspace 4)
+    %0:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1, %subreg.sub1, $sgpr2, %subreg.sub2, $sgpr3,%subreg.sub3
+    %1:vreg_64 = REG_SEQUENCE $vgpr0, %subreg.sub0, $vgpr1, %subreg.sub1
+    %2:vreg_64 = REG_SEQUENCE $vgpr2, %subreg.sub0, $vgpr3, %subreg.sub1
+    TBUFFER_STORE_FORMAT_XY_OFFSET_exact %1, %0, 0, 0, 27, 0, 0, implicit $exec :: (store (s32),align 2,addrspace 4)
+    TBUFFER_STORE_FORMAT_XY_OFFSET_exact %2, %0, 0, 4, 27, 0, 0, implicit $exec :: (store (s32),align 2,addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_store_xy_xy_sint_16_bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX11-LABEL: name: gfx11_tbuffer_store_xy_xy_sint_16_bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1, %subreg.sub1, $sgpr2, %subreg.sub2, $sgpr3, %subreg.sub3
+    ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE $vgpr0, %subreg.sub0, $vgpr1, %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE $vgpr2, %subreg.sub0, $vgpr3, %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[REG_SEQUENCE1]], %subreg.sub0_sub1, [[REG_SEQUENCE2]], %subreg.sub2_sub3
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[REG_SEQUENCE3]], [[REG_SEQUENCE]], 0, 0, 56, 0, 0, implicit $exec :: (store (s64), align 2, addrspace 4)
+    %0:sgpr_128 = REG_SEQUENCE $sgpr0, %subreg.sub0, $sgpr1, %subreg.sub1, $sgpr2, %subreg.sub2, $sgpr3, %subreg.sub3
+    %1:vreg_64 = REG_SEQUENCE $vgpr0, %subreg.sub0, $vgpr1, %subreg.sub1
+    %2:vreg_64 = REG_SEQUENCE $vgpr2, %subreg.sub0, $vgpr3, %subreg.sub1
+    TBUFFER_STORE_FORMAT_XY_OFFSET_exact %1, %0, 0, 0, 28, 0, 0, implicit $exec :: (store (s32),align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_XY_OFFSET_exact %2, %0, 0, 4, 28, 0, 0, implicit $exec :: (store (s32),align 2, addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_load_x_x_x_idxen_8bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_x_x_x_idxen_8bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZ_IDXEN:%[0-9]+]]:vreg_96 = TBUFFER_LOAD_FORMAT_XYZ_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 0, 46, 0, 0, implicit $exec :: (dereferenceable load (s24), align 1, addrspace 8)
+    ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[TBUFFER_LOAD_FORMAT_XYZ_IDXEN]].sub0_sub1
+    ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZ_IDXEN]].sub2
+    ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY6]].sub0
+    ; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY killed [[COPY6]].sub1
+    %4:sgpr_32 = COPY $sgpr4
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %8:vgpr_32 = COPY %4:sgpr_32
+    %7:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 0, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %9:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 1, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %11:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 2, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+...
+---
+
+name: gfx11_tbuffer_load_idxen_8bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_idxen_8bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 0, 46, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vreg_96 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub0_sub1_sub2
+    ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub3
+    ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[COPY6]].sub0_sub1
+    ; GFX11-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY killed [[COPY6]].sub2
+    ; GFX11-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[COPY8]].sub0
+    ; GFX11-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY killed [[COPY8]].sub1
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 4, 46, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
+    ; GFX11-NEXT: [[COPY12:%[0-9]+]]:vreg_96 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN1]].sub0_sub1_sub2
+    ; GFX11-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN1]].sub3
+    ; GFX11-NEXT: [[COPY14:%[0-9]+]]:vreg_64 = COPY [[COPY12]].sub0_sub1
+    ; GFX11-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY killed [[COPY12]].sub2
+    ; GFX11-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[COPY14]].sub0
+    ; GFX11-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY killed [[COPY14]].sub1
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 8, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+    %4:sgpr_32 = COPY $sgpr4
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %8:vgpr_32 = COPY %4:sgpr_32
+    %7:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 0, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %9:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 1, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %11:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 2, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %13:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 3, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %15:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 4, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %17:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 5, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %19:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 6, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %21:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 7, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %22:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 8, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+...
+---
+
+name: gfx11_tbuffer_load_xy_xy_idxen_uint_8bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_xy_xy_idxen_uint_8bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 0, 43, 0, 0, implicit $exec :: (dereferenceable load (s32), align 2, addrspace 8)
+    ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub0_sub1
+    ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub2_sub3
+    %4:sgpr_32 = COPY $sgpr4
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
+    %6:vgpr_32 = COPY %4
+    %7:vreg_64 = TBUFFER_LOAD_FORMAT_XY_IDXEN %6, %5, 0, 0, 15, 0, 0, implicit $exec :: (dereferenceable load (s16), align 2, addrspace 8)
+    %8:vreg_64 = TBUFFER_LOAD_FORMAT_XY_IDXEN %6, %5, 0, 2, 15, 0, 0, implicit $exec :: (dereferenceable load (s16), align 2, addrspace 8)
+...
+---
+
+name: gfx11_tbuffer_load_xy_xy_idxen_sint_8bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_xy_xy_idxen_sint_8bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 0, 47, 0, 0, implicit $exec :: (dereferenceable load (s32), align 2, addrspace 8)
+    ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vreg_64 = COPY [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub0_sub1
+    ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY killed [[TBUFFER_LOAD_FORMAT_XYZW_IDXEN]].sub2_sub3
+    %4:sgpr_32 = COPY $sgpr4
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
+    %6:vgpr_32 = COPY %4
+    %7:vreg_64 = TBUFFER_LOAD_FORMAT_XY_IDXEN %6, %5, 0, 0, 19, 0, 0, implicit $exec :: (dereferenceable load (s16), align 2, addrspace 8)
+    %8:vreg_64 = TBUFFER_LOAD_FORMAT_XY_IDXEN %6, %5, 0, 2, 19, 0, 0, implicit $exec :: (dereferenceable load (s16), align 2, addrspace 8)
+...
+---
+
+name: gfx11_tbuffer_load_x_off3_off4_8bit_no_merge
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-LABEL: name: gfx11_tbuffer_load_x_off3_off4_8bit_no_merge
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr4
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY1]], %subreg.sub3
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_X_IDXEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 3, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+    ; GFX11-NEXT: [[TBUFFER_LOAD_FORMAT_X_IDXEN1:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN [[COPY5]], [[REG_SEQUENCE]], 0, 4, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), addrspace 8)
+    %4:sgpr_32 = COPY $sgpr4
+    %3:sgpr_32 = COPY $sgpr3
+    %2:sgpr_32 = COPY $sgpr2
+    %1:sgpr_32 = COPY $sgpr1
+    %0:sgpr_32 = COPY $sgpr0
+    %5:sgpr_128 = REG_SEQUENCE %0:sgpr_32, %subreg.sub0, %1:sgpr_32, %subreg.sub1, %2:sgpr_32, %subreg.sub2, %3:sgpr_32, %subreg.sub3
+    %8:vgpr_32 = COPY %4:sgpr_32
+    %7:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 3, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+    %9:vgpr_32 = TBUFFER_LOAD_FORMAT_X_IDXEN %8:vgpr_32, %5:sgpr_128, 0, 4, 5, 0, 0, implicit $exec :: (dereferenceable load (s8), align 1, addrspace 8)
+...
+---
+
+name: gfx11_tbuffer_store_x_x_x_idxen_8bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+    ; GFX11-LABEL: name: gfx11_tbuffer_store_x_x_x_idxen_8bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY6:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY3]], %subreg.sub3
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE]], %subreg.sub0_sub1, [[COPY]], %subreg.sub2
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZ_OFFSET_exact killed [[REG_SEQUENCE1]], %rsrc, 0, 0, 46, 0, 0, implicit $exec :: (store (s24), align 1, addrspace 4)
+    %6:vgpr_32  = COPY $vgpr2
+    %5:vgpr_32  = COPY $vgpr1
+    %4:vgpr_32  = COPY $vgpr0
+    %3:sgpr_32  = COPY $sgpr3
+    %2:sgpr_32  = COPY $sgpr2
+    %1:sgpr_32  = COPY $sgpr1
+    %0:sgpr_32  = COPY $sgpr0
+    %rsrc:sgpr_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %4,  %rsrc, 0, 0, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %5,  %rsrc, 0, 1, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %6,  %rsrc, 0, 2, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_store_idxen_8bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+    ; GFX11-LABEL: name: gfx11_tbuffer_store_idxen_8bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+    ; GFX11-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+    ; GFX11-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY9:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY10:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY11:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY12:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE [[COPY12]], %subreg.sub0, [[COPY11]], %subreg.sub1, [[COPY10]], %subreg.sub2, [[COPY9]], %subreg.sub3
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[COPY7]], %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE]], %subreg.sub0_sub1, [[COPY6]], %subreg.sub2
+    ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[REG_SEQUENCE1]], %subreg.sub0_sub1_sub2, [[COPY5]], %subreg.sub3
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[REG_SEQUENCE2]], %rsrc, 0, 0, 46, 0, 0, implicit $exec :: (store (s32), align 1, addrspace 4)
+    ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY3]], %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:vreg_96 = REG_SEQUENCE killed [[REG_SEQUENCE3]], %subreg.sub0_sub1, [[COPY2]], %subreg.sub2
+    ; GFX11-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:vreg_128 = REG_SEQUENCE killed [[REG_SEQUENCE4]], %subreg.sub0_sub1_sub2, [[COPY1]], %subreg.sub3
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[REG_SEQUENCE5]], %rsrc, 0, 4, 46, 0, 0, implicit $exec :: (store (s32), align 1, addrspace 4)
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY]], %rsrc, 0, 8, 5, 0, 0, implicit $exec :: (store (s8), addrspace 4)
+    %12:vgpr_32 = COPY $vgpr8
+    %11:vgpr_32 = COPY $vgpr7
+    %10:vgpr_32 = COPY $vgpr6
+    %9:vgpr_32  = COPY $vgpr5
+    %8:vgpr_32  = COPY $vgpr4
+    %7:vgpr_32  = COPY $vgpr3
+    %6:vgpr_32  = COPY $vgpr2
+    %5:vgpr_32  = COPY $vgpr1
+    %4:vgpr_32  = COPY $vgpr0
+    %3:sgpr_32  = COPY $sgpr3
+    %2:sgpr_32  = COPY $sgpr2
+    %1:sgpr_32  = COPY $sgpr1
+    %0:sgpr_32  = COPY $sgpr0
+    %rsrc:sgpr_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %4,  %rsrc, 0, 0, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %5,  %rsrc, 0, 1, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %6,  %rsrc, 0, 2, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %7,  %rsrc, 0, 3, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %8,  %rsrc, 0, 4, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %9,  %rsrc, 0, 5, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %10, %rsrc, 0, 6, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %11, %rsrc, 0, 7, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %12, %rsrc, 0, 8, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_store_xy_xy_idxen_uint_8bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX11-LABEL: name: gfx11_tbuffer_store_xy_xy_idxen_uint_8bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: %v0:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: %v1:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: %v2:vgpr_32 = COPY $vgpr2
+    ; GFX11-NEXT: %v3:vgpr_32 = COPY $vgpr3
+    ; GFX11-NEXT: %s3:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: %s2:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: %s1:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: %s0:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE %s0, %subreg.sub0, %s1, %subreg.sub1, %s2, %subreg.sub2, %s3, %subreg.sub3
+    ; GFX11-NEXT: %xy0:vreg_64 = REG_SEQUENCE %v0, %subreg.sub0, %v1, %subreg.sub1
+    ; GFX11-NEXT: %xy1:vreg_64 = REG_SEQUENCE %v2, %subreg.sub0, %v3, %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE %xy0, %subreg.sub0_sub1, %xy1, %subreg.sub2_sub3
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[REG_SEQUENCE]], %rsrc, 0, 0, 43, 0, 0, implicit $exec :: (store (s32), align 2, addrspace 4)
+    %v0:vgpr_32 = COPY $vgpr0
+    %v1:vgpr_32 = COPY $vgpr1
+    %v2:vgpr_32 = COPY $vgpr2
+    %v3:vgpr_32 = COPY $vgpr3
+    %s3:sgpr_32 = COPY $sgpr3
+    %s2:sgpr_32 = COPY $sgpr2
+    %s1:sgpr_32 = COPY $sgpr1
+    %s0:sgpr_32 = COPY $sgpr0
+    %rsrc:sgpr_128 = REG_SEQUENCE %s0, %subreg.sub0, %s1, %subreg.sub1, %s2, %subreg.sub2, %s3, %subreg.sub3
+    %xy0:vreg_64 = REG_SEQUENCE %v0, %subreg.sub0, %v1, %subreg.sub1
+    %xy1:vreg_64 = REG_SEQUENCE %v2, %subreg.sub0, %v3, %subreg.sub1
+    TBUFFER_STORE_FORMAT_XY_OFFSET_exact %xy0, %rsrc, 0, 0, 15, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_XY_OFFSET_exact %xy1, %rsrc, 0, 2, 15, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_store_xy_xy_idxen_sint_8bit
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX11-LABEL: name: gfx11_tbuffer_store_xy_xy_idxen_sint_8bit
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: %v0:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: %v1:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: %v2:vgpr_32 = COPY $vgpr2
+    ; GFX11-NEXT: %v3:vgpr_32 = COPY $vgpr3
+    ; GFX11-NEXT: %s3:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: %s2:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: %s1:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: %s0:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE %s0, %subreg.sub0, %s1, %subreg.sub1, %s2, %subreg.sub2, %s3, %subreg.sub3
+    ; GFX11-NEXT: %xy0:vreg_64 = REG_SEQUENCE %v0, %subreg.sub0, %v1, %subreg.sub1
+    ; GFX11-NEXT: %xy1:vreg_64 = REG_SEQUENCE %v2, %subreg.sub0, %v3, %subreg.sub1
+    ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE %xy0, %subreg.sub0_sub1, %xy1, %subreg.sub2_sub3
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[REG_SEQUENCE]], %rsrc, 0, 0, 42, 0, 0, implicit $exec :: (store (s32), align 2, addrspace 4)
+    %v0:vgpr_32 = COPY $vgpr0
+    %v1:vgpr_32 = COPY $vgpr1
+    %v2:vgpr_32 = COPY $vgpr2
+    %v3:vgpr_32 = COPY $vgpr3
+    %s3:sgpr_32 = COPY $sgpr3
+    %s2:sgpr_32 = COPY $sgpr2
+    %s1:sgpr_32 = COPY $sgpr1
+    %s0:sgpr_32 = COPY $sgpr0
+    %rsrc:sgpr_128 = REG_SEQUENCE %s0, %subreg.sub0, %s1, %subreg.sub1, %s2, %subreg.sub2, %s3, %subreg.sub3
+    %xy0:vreg_64 = REG_SEQUENCE %v0, %subreg.sub0, %v1, %subreg.sub1
+    %xy1:vreg_64 = REG_SEQUENCE %v2, %subreg.sub0, %v3, %subreg.sub1
+    TBUFFER_STORE_FORMAT_XY_OFFSET_exact %xy0, %rsrc, 0, 0, 14, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+    TBUFFER_STORE_FORMAT_XY_OFFSET_exact %xy1, %rsrc, 0, 2, 14, 0, 0, implicit $exec :: (store (s16), align 2, addrspace 4)
+...
+---
+
+name: gfx11_tbuffer_store_x_off3_off4_8bit_no_merge
+body: |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1
+    ; GFX11-LABEL: name: gfx11_tbuffer_store_x_off3_off4_8bit_no_merge
+    ; GFX11: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1
+    ; GFX11-NEXT: {{  $}}
+    ; GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; GFX11-NEXT: [[COPY5:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; GFX11-NEXT: %rsrc:sgpr_128 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY4]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY2]], %subreg.sub3
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY1]], %rsrc, 0, 3, 5, 0, 0, implicit $exec :: (store (s8), addrspace 4)
+    ; GFX11-NEXT: TBUFFER_STORE_FORMAT_X_OFFSET_exact [[COPY]], %rsrc, 0, 4, 5, 0, 0, implicit $exec :: (store (s8), addrspace 4)
+    %5:vgpr_32  = COPY $vgpr1
+    %4:vgpr_32  = COPY $vgpr0
+    %3:sgpr_32  = COPY $sgpr3
+    %2:sgpr_32  = COPY $sgpr2
+    %1:sgpr_32  = COPY $sgpr1
+    %0:sgpr_32  = COPY $sgpr0
+    %rsrc:sgpr_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %4,  %rsrc, 0, 3, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+    TBUFFER_STORE_FORMAT_X_OFFSET_exact %5,  %rsrc, 0, 4, 5, 0, 0, implicit $exec :: (store (s8),  align 1, addrspace 4)
+...
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw.ll
index fcc5584dc353f..2d73a6b015295 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-atomicrmw.ll
@@ -47,7 +47,7 @@ define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, ptr addrspace(1
 
 atomic:
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 100
-  %ret = atomicrmw max ptr addrspace(1) %gep, i32 %y syncscope("workgroup") seq_cst
+  %ret = atomicrmw max ptr addrspace(1) %gep, i32 %y syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   store i32 %ret, ptr addrspace(1) %out
   br label %exit
 
@@ -87,7 +87,7 @@ define amdgpu_kernel void @atomic_max_i32_noret(ptr addrspace(1) %out, ptr addrs
 
 atomic:
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 100
-  %ret = atomicrmw max ptr addrspace(1) %gep, i32 %y syncscope("workgroup") seq_cst
+  %ret = atomicrmw max ptr addrspace(1) %gep, i32 %y syncscope("workgroup") seq_cst, !amdgpu.no.fine.grained.memory !0
   br label %exit
 
 exit:
@@ -96,3 +96,5 @@ exit:
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
+
+!0 = !{}
diff --git a/llvm/test/CodeGen/AMDGPU/no-folding-imm-to-inst-with-fi.ll b/llvm/test/CodeGen/AMDGPU/no-folding-imm-to-inst-with-fi.ll
new file mode 100644
index 0000000000000..6d0aa1e784530
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/no-folding-imm-to-inst-with-fi.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck %s
+
+define protected amdgpu_kernel void @no_folding_imm_to_inst_with_fi(<4 x i64> %val4, <16 x i64> %val16) {
+; CHECK-LABEL: no_folding_imm_to_inst_with_fi:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_clause 0x2
+; CHECK-NEXT:    s_load_b256 s[36:43], s[4:5], 0x24
+; CHECK-NEXT:    s_load_b512 s[16:31], s[4:5], 0xe4
+; CHECK-NEXT:    s_load_b512 s[0:15], s[4:5], 0xa4
+; CHECK-NEXT:    s_mov_b64 s[34:35], src_private_base
+; CHECK-NEXT:    s_movk_i32 s33, 0x70
+; CHECK-NEXT:    s_movk_i32 s34, 0x60
+; CHECK-NEXT:    s_or_b32 s44, 0x80, s33
+; CHECK-NEXT:    s_mov_b32 s45, s35
+; CHECK-NEXT:    s_or_b32 s46, 0x80, s34
+; CHECK-NEXT:    s_mov_b32 s47, s35
+; CHECK-NEXT:    v_dual_mov_b32 v20, s44 :: v_dual_mov_b32 v21, s45
+; CHECK-NEXT:    v_dual_mov_b32 v22, s46 :: v_dual_mov_b32 v23, s47
+; CHECK-NEXT:    s_movk_i32 s34, 0x80
+; CHECK-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; CHECK-NEXT:    v_dual_mov_b32 v34, s34 :: v_dual_mov_b32 v35, s35
+; CHECK-NEXT:    s_wait_kmcnt 0x0
+; CHECK-NEXT:    v_dual_mov_b32 v0, s40 :: v_dual_mov_b32 v1, s41
+; CHECK-NEXT:    v_dual_mov_b32 v2, s42 :: v_dual_mov_b32 v3, s43
+; CHECK-NEXT:    v_dual_mov_b32 v4, s36 :: v_dual_mov_b32 v5, s37
+; CHECK-NEXT:    v_dual_mov_b32 v6, s38 :: v_dual_mov_b32 v7, s39
+; CHECK-NEXT:    scratch_store_b128 off, v[0:3], off offset:16 scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    v_dual_mov_b32 v0, s20 :: v_dual_mov_b32 v1, s21
+; CHECK-NEXT:    s_movk_i32 s20, 0x50
+; CHECK-NEXT:    v_dual_mov_b32 v8, s28 :: v_dual_mov_b32 v9, s29
+; CHECK-NEXT:    v_dual_mov_b32 v10, s30 :: v_dual_mov_b32 v11, s31
+; CHECK-NEXT:    s_wait_alu 0xfffe
+; CHECK-NEXT:    s_or_b32 s20, 0x80, s20
+; CHECK-NEXT:    s_mov_b32 s21, s35
+; CHECK-NEXT:    v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v13, s25
+; CHECK-NEXT:    v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v15, s27
+; CHECK-NEXT:    v_dual_mov_b32 v2, s22 :: v_dual_mov_b32 v3, s23
+; CHECK-NEXT:    s_wait_alu 0xfffe
+; CHECK-NEXT:    v_dual_mov_b32 v25, s21 :: v_dual_mov_b32 v24, s20
+; CHECK-NEXT:    scratch_store_b128 off, v[4:7], off scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    flat_store_b128 v[20:21], v[8:11] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    flat_store_b128 v[22:23], v[12:15] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    flat_store_b128 v[24:25], v[0:3] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v1, s17
+; CHECK-NEXT:    s_or_b32 s16, 0x80, 64
+; CHECK-NEXT:    s_mov_b32 s17, s35
+; CHECK-NEXT:    v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v5, s13
+; CHECK-NEXT:    s_or_b32 s12, 0x80, 48
+; CHECK-NEXT:    s_mov_b32 s13, s35
+; CHECK-NEXT:    v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; CHECK-NEXT:    s_or_b32 s8, 0x80, 32
+; CHECK-NEXT:    s_mov_b32 s9, s35
+; CHECK-NEXT:    v_dual_mov_b32 v12, s4 :: v_dual_mov_b32 v13, s5
+; CHECK-NEXT:    s_or_b32 s4, 0x80, 16
+; CHECK-NEXT:    s_mov_b32 s5, s35
+; CHECK-NEXT:    v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v3, s19
+; CHECK-NEXT:    s_wait_alu 0xfffe
+; CHECK-NEXT:    v_dual_mov_b32 v27, s17 :: v_dual_mov_b32 v26, s16
+; CHECK-NEXT:    v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s15
+; CHECK-NEXT:    v_dual_mov_b32 v29, s13 :: v_dual_mov_b32 v28, s12
+; CHECK-NEXT:    v_dual_mov_b32 v31, s9 :: v_dual_mov_b32 v30, s8
+; CHECK-NEXT:    v_dual_mov_b32 v33, s5 :: v_dual_mov_b32 v32, s4
+; CHECK-NEXT:    v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; CHECK-NEXT:    v_dual_mov_b32 v14, s6 :: v_dual_mov_b32 v15, s7
+; CHECK-NEXT:    v_dual_mov_b32 v16, s0 :: v_dual_mov_b32 v17, s1
+; CHECK-NEXT:    v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s3
+; CHECK-NEXT:    flat_store_b128 v[26:27], v[0:3] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    flat_store_b128 v[28:29], v[4:7] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    flat_store_b128 v[30:31], v[8:11] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    flat_store_b128 v[32:33], v[12:15] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    flat_store_b128 v[34:35], v[16:19] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_storecnt 0x0
+; CHECK-NEXT:    flat_load_b128 v[0:3], v[22:23] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT:    flat_load_b128 v[0:3], v[20:21] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT:    flat_load_b128 v[0:3], v[26:27] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT:    flat_load_b128 v[0:3], v[24:25] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT:    flat_load_b128 v[0:3], v[30:31] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT:    flat_load_b128 v[0:3], v[28:29] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT:    flat_load_b128 v[0:3], v[34:35] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_loadcnt_dscnt 0x0
+; CHECK-NEXT:    flat_load_b128 v[0:3], v[32:33] scope:SCOPE_SYS
+; CHECK-NEXT:    s_wait_loadcnt 0x0
+; CHECK-NEXT:    s_endpgm
+bb:
+  %alloca = alloca <4 x i64>, align 32, addrspace(5)
+  %alloca1 = alloca <16 x i64>, align 128, addrspace(5)
+  store volatile <4 x i64> %val4, ptr addrspace(5) %alloca
+  %ascast = addrspacecast ptr addrspace(5) %alloca1 to ptr
+  store volatile <16 x i64> %val16, ptr %ascast
+  %load = load volatile <16 x i64>, ptr %ascast
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
index d95fc77939e83..a3c38b17abf00 100644
--- a/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
@@ -85,7 +85,7 @@ define i8 @flat_inst_valu_offset_1(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:1
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 1
+  %gep = getelementptr inbounds i8, ptr %p, i64 1
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -160,7 +160,7 @@ define i8 @flat_inst_valu_offset_11bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:2047
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 2047
+  %gep = getelementptr inbounds i8, ptr %p, i64 2047
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -235,7 +235,7 @@ define i8 @flat_inst_valu_offset_12bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:4095
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 4095
+  %gep = getelementptr inbounds i8, ptr %p, i64 4095
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -330,7 +330,7 @@ define i8 @flat_inst_valu_offset_13bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:8191
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 8191
+  %gep = getelementptr inbounds i8, ptr %p, i64 8191
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -425,7 +425,7 @@ define i8 @flat_inst_valu_offset_24bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:8388607
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 8388607
+  %gep = getelementptr inbounds i8, ptr %p, i64 8388607
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -511,7 +511,7 @@ define i8 @flat_inst_valu_offset_neg_11bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:-2048
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 -2048
+  %gep = getelementptr inbounds i8, ptr %p, i64 -2048
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -597,7 +597,7 @@ define i8 @flat_inst_valu_offset_neg_12bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:-4096
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 -4096
+  %gep = getelementptr inbounds i8, ptr %p, i64 -4096
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -683,7 +683,7 @@ define i8 @flat_inst_valu_offset_neg_13bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:-8192
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 -8192
+  %gep = getelementptr inbounds i8, ptr %p, i64 -8192
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -769,7 +769,7 @@ define i8 @flat_inst_valu_offset_neg_24bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:-8388608
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 -8388608
+  %gep = getelementptr inbounds i8, ptr %p, i64 -8388608
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -845,7 +845,7 @@ define i8 @flat_inst_valu_offset_2x_11bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:4095
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 4095
+  %gep = getelementptr inbounds i8, ptr %p, i64 4095
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -940,7 +940,7 @@ define i8 @flat_inst_valu_offset_2x_12bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:8191
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 8191
+  %gep = getelementptr inbounds i8, ptr %p, i64 8191
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -1035,7 +1035,7 @@ define i8 @flat_inst_valu_offset_2x_13bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:16383
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 16383
+  %gep = getelementptr inbounds i8, ptr %p, i64 16383
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -1139,7 +1139,7 @@ define i8 @flat_inst_valu_offset_2x_24bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1]
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 16777214
+  %gep = getelementptr inbounds i8, ptr %p, i64 16777214
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -1225,7 +1225,7 @@ define i8 @flat_inst_valu_offset_2x_neg_11bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:-4096
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 -4096
+  %gep = getelementptr inbounds i8, ptr %p, i64 -4096
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -1311,7 +1311,7 @@ define i8 @flat_inst_valu_offset_2x_neg_12bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:-8192
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 -8192
+  %gep = getelementptr inbounds i8, ptr %p, i64 -8192
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -1397,7 +1397,7 @@ define i8 @flat_inst_valu_offset_2x_neg_13bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    flat_load_u8 v0, v[0:1] offset:-16384
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
-  %gep = getelementptr i8, ptr %p, i64 -16384
+  %gep = getelementptr inbounds i8, ptr %p, i64 -16384
   %load = load i8, ptr %gep, align 4
   ret i8 %load
 }
@@ -2835,7 +2835,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_1(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 1
+  %gep = getelementptr inbounds i8, ptr %p, i64 1
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -2925,7 +2925,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_11bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 2047
+  %gep = getelementptr inbounds i8, ptr %p, i64 2047
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -3015,7 +3015,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_12bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 4095
+  %gep = getelementptr inbounds i8, ptr %p, i64 4095
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -3127,7 +3127,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_13bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 8191
+  %gep = getelementptr inbounds i8, ptr %p, i64 8191
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -3239,7 +3239,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_11bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 -2048
+  %gep = getelementptr inbounds i8, ptr %p, i64 -2048
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -3351,7 +3351,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_12bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 -4096
+  %gep = getelementptr inbounds i8, ptr %p, i64 -4096
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -3463,7 +3463,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_13bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 -8192
+  %gep = getelementptr inbounds i8, ptr %p, i64 -8192
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -3553,7 +3553,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_11bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 4095
+  %gep = getelementptr inbounds i8, ptr %p, i64 4095
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -3665,7 +3665,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_12bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 8191
+  %gep = getelementptr inbounds i8, ptr %p, i64 8191
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -3777,7 +3777,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_13bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 16383
+  %gep = getelementptr inbounds i8, ptr %p, i64 16383
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -3889,7 +3889,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_11bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 -4096
+  %gep = getelementptr inbounds i8, ptr %p, i64 -4096
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -4001,7 +4001,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_12bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 -8192
+  %gep = getelementptr inbounds i8, ptr %p, i64 -8192
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
@@ -4113,7 +4113,7 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_13bit_max(ptr %p) {
 ; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
 ; GFX12-GISEL-NEXT:    flat_store_b8 v[0:1], v0
 ; GFX12-GISEL-NEXT:    s_endpgm
-  %gep = getelementptr i8, ptr %p, i64 -16384
+  %gep = getelementptr inbounds i8, ptr %p, i64 -16384
   %load = load volatile i8, ptr %gep, align 1
   store i8 %load, ptr poison
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index 42401afb6edf2..f78168ba29ef1 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -78,12 +78,14 @@ define amdgpu_kernel void @fadd_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
 ; GFX1250-LABEL: fadd_v2_vs:
 ; GFX1250:       ; %bb.0:
 ; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1250-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -142,13 +144,16 @@ define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
 ; GFX1250-SDAG-NEXT:    s_clause 0x1
 ; GFX1250-SDAG-NEXT:    s_load_b64 s[6:7], s[4:5], 0x24
 ; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
-; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v8, 0x3ff, v0
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v4, s[6:7] scale_offset
+; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v8, s[6:7] scale_offset
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[0:1]
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[2:3], v[2:3], s[2:3]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[0:1]
-; GFX1250-SDAG-NEXT:    global_store_b128 v4, v[0:3], s[6:7] scale_offset
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[2:3], v[2:3], v[4:5]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[6:7]
+; GFX1250-SDAG-NEXT:    global_store_b128 v8, v[0:3], s[6:7] scale_offset
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fadd_v4_vs:
@@ -156,13 +161,16 @@ define amdgpu_kernel void @fadd_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
 ; GFX1250-GISEL-NEXT:    s_clause 0x1
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[6:7], s[4:5], 0x24
 ; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v8, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v4, s[6:7] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v8, s[6:7] scale_offset
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[0:1]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[2:3]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[0:1]
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[2:3], v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b128 v4, v[0:3], s[6:7] scale_offset
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[2:3], v[2:3], v[6:7]
+; GFX1250-GISEL-NEXT:    global_store_b128 v8, v[0:3], s[6:7] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %a, i32 %id
@@ -332,56 +340,69 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
 ;
 ; GFX1250-SDAG-LABEL: fadd_v32_vs:
 ; GFX1250-SDAG:       ; %bb.0:
-; GFX1250-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-SDAG-NEXT:    s_load_b64 s[34:35], s[4:5], 0x24
 ; GFX1250-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT:    v_lshlrev_b32_e32 v32, 7, v0
+; GFX1250-SDAG-NEXT:    v_lshlrev_b32_e32 v56, 7, v0
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX1250-SDAG-NEXT:    s_clause 0x7
-; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v32, s[0:1] offset:16
-; GFX1250-SDAG-NEXT:    global_load_b128 v[4:7], v32, s[0:1]
-; GFX1250-SDAG-NEXT:    global_load_b128 v[8:11], v32, s[0:1] offset:48
-; GFX1250-SDAG-NEXT:    global_load_b128 v[20:23], v32, s[0:1] offset:32
-; GFX1250-SDAG-NEXT:    global_load_b128 v[12:15], v32, s[0:1] offset:80
-; GFX1250-SDAG-NEXT:    global_load_b128 v[16:19], v32, s[0:1] offset:64
-; GFX1250-SDAG-NEXT:    global_load_b128 v[24:27], v32, s[0:1] offset:112
-; GFX1250-SDAG-NEXT:    global_load_b128 v[28:31], v32, s[0:1] offset:96
-; GFX1250-SDAG-NEXT:    s_clause 0x1
-; GFX1250-SDAG-NEXT:    s_load_b512 s[8:23], s[4:5], 0xa4
-; GFX1250-SDAG-NEXT:    s_load_b512 s[36:51], s[4:5], 0xe4
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x7
+; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v56, s[34:35] offset:16
+; GFX1250-SDAG-NEXT:    global_load_b128 v[4:7], v56, s[34:35] offset:48
+; GFX1250-SDAG-NEXT:    global_load_b128 v[8:11], v56, s[34:35] offset:32
+; GFX1250-SDAG-NEXT:    global_load_b128 v[12:15], v56, s[34:35]
+; GFX1250-SDAG-NEXT:    global_load_b128 v[16:19], v56, s[34:35] offset:80
+; GFX1250-SDAG-NEXT:    global_load_b128 v[20:23], v56, s[34:35] offset:96
+; GFX1250-SDAG-NEXT:    global_load_b128 v[24:27], v56, s[34:35] offset:64
+; GFX1250-SDAG-NEXT:    global_load_b128 v[28:31], v56, s[34:35] offset:112
+; GFX1250-SDAG-NEXT:    s_load_b512 s[16:31], s[4:5], 0xa4
+; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT:    s_load_b512 s[0:15], s[4:5], 0xe4
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[12:13]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[2:3], v[2:3], s[14:15]
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v32, s20 :: v_dual_mov_b32 v33, s21
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v34, s22 :: v_dual_mov_b32 v35, s23
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v36, s18 :: v_dual_mov_b32 v39, s29
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v40, s30 :: v_dual_mov_b32 v41, s31
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v42, s24 :: v_dual_mov_b32 v37, s19
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v38, s28 :: v_dual_mov_b32 v55, s15
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v51, s3 :: v_dual_mov_b32 v52, s12
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v53, s13 :: v_dual_mov_b32 v54, s14
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v49, s7 :: v_dual_mov_b32 v50, s2
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v45, s27 :: v_dual_mov_b32 v46, s4
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v47, s5 :: v_dual_mov_b32 v48, s6
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v43, s25 :: v_dual_mov_b32 v44, s26
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x7
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[32:33]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[2:3], v[2:3], v[34:35]
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v32, s8 :: v_dual_mov_b32 v33, s9
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v34, s10 :: v_dual_mov_b32 v35, s11
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x6
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[6:7], v[6:7], s[10:11]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x4
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[20:21], v[20:21], s[16:17]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x3
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[12:13], v[12:13], s[40:41]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[6:7], v[6:7], v[40:41]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[40:41], s[0:1]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[4:5], v[4:5], v[38:39]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[38:39], s[16:17]
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x2
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[18:19], v[18:19], s[38:39]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x1
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[24:25], v[24:25], s[48:49]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[20:21], v[20:21], v[32:33]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[22:23], v[22:23], v[34:35]
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[28:29], v[28:29], s[44:45]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[30:31], v[30:31], s[46:47]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[26:27], v[26:27], s[50:51]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[16:17], v[16:17], s[36:37]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[14:15], v[14:15], s[42:43]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[22:23], v[22:23], s[18:19]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[8:9], v[8:9], s[20:21]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[10:11], v[10:11], s[22:23]
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[4:5], v[4:5], s[8:9]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[28:29], v[28:29], v[52:53]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[30:31], v[30:31], v[54:55]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[26:27], v[26:27], v[50:51]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[24:25], v[24:25], v[40:41]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[16:17], v[16:17], v[46:47]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[18:19], v[18:19], v[48:49]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[8:9], v[8:9], v[42:43]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[10:11], v[10:11], v[44:45]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[14:15], v[14:15], v[36:37]
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[12:13], v[12:13], v[38:39]
 ; GFX1250-SDAG-NEXT:    s_clause 0x7
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[28:31], s[0:1] offset:96
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[24:27], s[0:1] offset:112
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[16:19], s[0:1] offset:64
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[12:15], s[0:1] offset:80
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[8:11], s[0:1] offset:48
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[4:7], s[0:1]
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[0:3], s[0:1] offset:16
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[20:23], s[34:35] offset:96
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[28:31], s[34:35] offset:112
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[24:27], s[34:35] offset:64
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[16:19], s[34:35] offset:80
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[8:11], s[34:35] offset:32
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[4:7], s[34:35] offset:48
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[12:15], s[34:35]
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[0:3], s[34:35] offset:16
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fadd_v32_vs:
@@ -389,54 +410,70 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[34:35], s[4:5], 0x24
 ; GFX1250-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT:    v_lshlrev_b32_e32 v32, 7, v0
+; GFX1250-GISEL-NEXT:    v_lshlrev_b32_e32 v56, 7, v0
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_clause 0x7
-; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v32, s[34:35]
-; GFX1250-GISEL-NEXT:    global_load_b128 v[4:7], v32, s[34:35] offset:16
-; GFX1250-GISEL-NEXT:    global_load_b128 v[8:11], v32, s[34:35] offset:32
-; GFX1250-GISEL-NEXT:    global_load_b128 v[12:15], v32, s[34:35] offset:48
-; GFX1250-GISEL-NEXT:    global_load_b128 v[16:19], v32, s[34:35] offset:64
-; GFX1250-GISEL-NEXT:    global_load_b128 v[20:23], v32, s[34:35] offset:80
-; GFX1250-GISEL-NEXT:    global_load_b128 v[24:27], v32, s[34:35] offset:96
-; GFX1250-GISEL-NEXT:    global_load_b128 v[28:31], v32, s[34:35] offset:112
+; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v56, s[34:35]
+; GFX1250-GISEL-NEXT:    global_load_b128 v[4:7], v56, s[34:35] offset:16
+; GFX1250-GISEL-NEXT:    global_load_b128 v[8:11], v56, s[34:35] offset:32
+; GFX1250-GISEL-NEXT:    global_load_b128 v[12:15], v56, s[34:35] offset:48
+; GFX1250-GISEL-NEXT:    global_load_b128 v[16:19], v56, s[34:35] offset:64
+; GFX1250-GISEL-NEXT:    global_load_b128 v[20:23], v56, s[34:35] offset:80
+; GFX1250-GISEL-NEXT:    global_load_b128 v[24:27], v56, s[34:35] offset:96
+; GFX1250-GISEL-NEXT:    global_load_b128 v[28:31], v56, s[34:35] offset:112
 ; GFX1250-GISEL-NEXT:    s_load_b512 s[16:31], s[4:5], 0xa4
 ; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_load_b512 s[0:15], s[4:5], 0xe4
-; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x7
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[16:17]
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[2:3], v[2:3], s[18:19]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[32:33], s[16:17]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[34:35], s[18:19]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[36:37], s[20:21]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[38:39], s[22:23]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[40:41], s[24:25]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[42:43], s[26:27]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[44:45], s[28:29]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[46:47], s[30:31]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[48:49], s[0:1]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[50:51], s[2:3]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[52:53], s[4:5]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[54:55], s[6:7]
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x7
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[32:33]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[2:3], v[2:3], v[34:35]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[32:33], s[8:9]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[34:35], s[10:11]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x6
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[4:5], v[4:5], s[20:21]
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[6:7], v[6:7], s[22:23]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[4:5], v[4:5], v[36:37]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[6:7], v[6:7], v[38:39]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[36:37], s[12:13]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[38:39], s[14:15]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x5
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[8:9], v[8:9], s[24:25]
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[10:11], v[10:11], s[26:27]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[8:9], v[8:9], v[40:41]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[10:11], v[10:11], v[42:43]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x4
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[12:13], v[12:13], s[28:29]
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[14:15], v[14:15], s[30:31]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[12:13], v[12:13], v[44:45]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[14:15], v[14:15], v[46:47]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x3
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[16:17], v[16:17], s[0:1]
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[18:19], v[18:19], s[2:3]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[16:17], v[16:17], v[48:49]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[18:19], v[18:19], v[50:51]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x2
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[20:21], v[20:21], s[4:5]
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[22:23], v[22:23], s[6:7]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[20:21], v[20:21], v[52:53]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[22:23], v[22:23], v[54:55]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x1
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[24:25], v[24:25], s[8:9]
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[26:27], v[26:27], s[10:11]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[24:25], v[24:25], v[32:33]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[26:27], v[26:27], v[34:35]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[28:29], v[28:29], s[12:13]
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[30:31], v[30:31], s[14:15]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[28:29], v[28:29], v[36:37]
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[30:31], v[30:31], v[38:39]
 ; GFX1250-GISEL-NEXT:    s_clause 0x7
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[0:3], s[34:35]
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[4:7], s[34:35] offset:16
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[8:11], s[34:35] offset:32
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[12:15], s[34:35] offset:48
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[16:19], s[34:35] offset:64
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[20:23], s[34:35] offset:80
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[24:27], s[34:35] offset:96
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[28:31], s[34:35] offset:112
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[0:3], s[34:35]
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[4:7], s[34:35] offset:16
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[8:11], s[34:35] offset:32
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[12:15], s[34:35] offset:48
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[16:19], s[34:35] offset:64
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[20:23], s[34:35] offset:80
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[24:27], s[34:35] offset:96
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[28:31], s[34:35] offset:112
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %a, i32 %id
@@ -502,15 +539,16 @@ define amdgpu_kernel void @fadd_v2_v_imm(ptr addrspace(1) %a) {
 ; GFX1250-GISEL-LABEL: fadd_v2_v_imm:
 ; GFX1250-GISEL:       ; %bb.0:
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s2, 0x42c80000
-; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s3, s2
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -645,15 +683,16 @@ define amdgpu_kernel void @fadd_v2_v_lit_splat(ptr addrspace(1) %a) {
 ; GFX1250-GISEL-LABEL: fadd_v2_v_lit_splat:
 ; GFX1250-GISEL:       ; %bb.0:
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s2, 1.0
-; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s3, s2
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -703,13 +742,15 @@ define amdgpu_kernel void @fadd_v2_v_lit_hi0(ptr addrspace(1) %a) {
 ; GFX1250-GISEL-LABEL: fadd_v2_v_lit_hi0:
 ; GFX1250-GISEL:       ; %bb.0:
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_mov_b64 s[2:3], 0x3f800000
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -746,17 +787,31 @@ define amdgpu_kernel void @fadd_v2_v_lit_lo0(ptr addrspace(1) %a) {
 ; PACKED-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-NEXT:    s_endpgm
 ;
-; GFX1250-LABEL: fadd_v2_v_lit_lo0:
-; GFX1250:       ; %bb.0:
-; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; GFX1250-NEXT:    s_mov_b64 s[2:3], lit64(0x3f80000000000000)
-; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
-; GFX1250-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
-; GFX1250-NEXT:    s_endpgm
+; GFX1250-SDAG-LABEL: fadd_v2_v_lit_lo0:
+; GFX1250-SDAG:       ; %bb.0:
+; GFX1250-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[2:3], lit64(0x3f80000000000000)
+; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: fadd_v2_v_lit_lo0:
+; GFX1250-GISEL:       ; %bb.0:
+; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    s_mov_b64 s[2:3], lit64(0x3f80000000000000)
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
@@ -792,17 +847,31 @@ define amdgpu_kernel void @fadd_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
 ; PACKED-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-NEXT:    s_endpgm
 ;
-; GFX1250-LABEL: fadd_v2_v_unfoldable_lit:
-; GFX1250:       ; %bb.0:
-; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; GFX1250-NEXT:    s_mov_b64 s[2:3], lit64(0x400000003f800000)
-; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
-; GFX1250-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
-; GFX1250-NEXT:    s_endpgm
+; GFX1250-SDAG-LABEL: fadd_v2_v_unfoldable_lit:
+; GFX1250-SDAG:       ; %bb.0:
+; GFX1250-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[2:3], lit64(0x400000003f800000)
+; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: fadd_v2_v_unfoldable_lit:
+; GFX1250-GISEL:       ; %bb.0:
+; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    s_mov_b64 s[2:3], lit64(0x400000003f800000)
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
@@ -1085,12 +1154,14 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, float %x, flo
 ; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_lo2:
 ; GFX1250-SDAG:       ; %bb.0:
 ; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1250-SDAG-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[2:3] neg_lo:[0,1]
-; GFX1250-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3] neg_lo:[0,1]
+; GFX1250-SDAG-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fadd_v2_v_fneg_lo2:
@@ -1159,12 +1230,14 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, float %x, flo
 ; GFX1250-SDAG-LABEL: fadd_v2_v_fneg_hi2:
 ; GFX1250-SDAG:       ; %bb.0:
 ; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1250-SDAG-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], s[2:3] op_sel:[0,1] op_sel_hi:[1,0] neg_hi:[0,1]
-; GFX1250-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3] op_sel:[0,1] op_sel_hi:[1,0] neg_hi:[0,1]
+; GFX1250-SDAG-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fadd_v2_v_fneg_hi2:
@@ -1262,12 +1335,14 @@ define amdgpu_kernel void @fmul_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
 ; GFX1250-LABEL: fmul_v2_vs:
 ; GFX1250:       ; %bb.0:
 ; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1250-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -1326,13 +1401,16 @@ define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
 ; GFX1250-SDAG-NEXT:    s_clause 0x1
 ; GFX1250-SDAG-NEXT:    s_load_b64 s[6:7], s[4:5], 0x24
 ; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
-; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v8, 0x3ff, v0
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v4, s[6:7] scale_offset
+; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v8, s[6:7] scale_offset
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[0:1]
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s3
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[2:3], v[2:3], s[2:3]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], s[0:1]
-; GFX1250-SDAG-NEXT:    global_store_b128 v4, v[0:3], s[6:7] scale_offset
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[2:3], v[2:3], v[4:5]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[6:7]
+; GFX1250-SDAG-NEXT:    global_store_b128 v8, v[0:3], s[6:7] scale_offset
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fmul_v4_vs:
@@ -1340,13 +1418,16 @@ define amdgpu_kernel void @fmul_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
 ; GFX1250-GISEL-NEXT:    s_clause 0x1
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[6:7], s[4:5], 0x24
 ; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v8, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v4, s[6:7] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v8, s[6:7] scale_offset
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[0:1]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[2:3]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], s[0:1]
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[2:3], v[2:3], s[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b128 v4, v[0:3], s[6:7] scale_offset
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[4:5]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[2:3], v[2:3], v[6:7]
+; GFX1250-GISEL-NEXT:    global_store_b128 v8, v[0:3], s[6:7] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %a, i32 %id
@@ -1516,56 +1597,69 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
 ;
 ; GFX1250-SDAG-LABEL: fmul_v32_vs:
 ; GFX1250-SDAG:       ; %bb.0:
-; GFX1250-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-SDAG-NEXT:    s_load_b64 s[34:35], s[4:5], 0x24
 ; GFX1250-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT:    v_lshlrev_b32_e32 v32, 7, v0
+; GFX1250-SDAG-NEXT:    v_lshlrev_b32_e32 v56, 7, v0
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX1250-SDAG-NEXT:    s_clause 0x7
-; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v32, s[0:1] offset:16
-; GFX1250-SDAG-NEXT:    global_load_b128 v[4:7], v32, s[0:1]
-; GFX1250-SDAG-NEXT:    global_load_b128 v[8:11], v32, s[0:1] offset:48
-; GFX1250-SDAG-NEXT:    global_load_b128 v[20:23], v32, s[0:1] offset:32
-; GFX1250-SDAG-NEXT:    global_load_b128 v[12:15], v32, s[0:1] offset:80
-; GFX1250-SDAG-NEXT:    global_load_b128 v[16:19], v32, s[0:1] offset:64
-; GFX1250-SDAG-NEXT:    global_load_b128 v[24:27], v32, s[0:1] offset:112
-; GFX1250-SDAG-NEXT:    global_load_b128 v[28:31], v32, s[0:1] offset:96
-; GFX1250-SDAG-NEXT:    s_clause 0x1
-; GFX1250-SDAG-NEXT:    s_load_b512 s[8:23], s[4:5], 0xa4
-; GFX1250-SDAG-NEXT:    s_load_b512 s[36:51], s[4:5], 0xe4
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x7
+; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v56, s[34:35] offset:16
+; GFX1250-SDAG-NEXT:    global_load_b128 v[4:7], v56, s[34:35] offset:48
+; GFX1250-SDAG-NEXT:    global_load_b128 v[8:11], v56, s[34:35] offset:32
+; GFX1250-SDAG-NEXT:    global_load_b128 v[12:15], v56, s[34:35]
+; GFX1250-SDAG-NEXT:    global_load_b128 v[16:19], v56, s[34:35] offset:80
+; GFX1250-SDAG-NEXT:    global_load_b128 v[20:23], v56, s[34:35] offset:96
+; GFX1250-SDAG-NEXT:    global_load_b128 v[24:27], v56, s[34:35] offset:64
+; GFX1250-SDAG-NEXT:    global_load_b128 v[28:31], v56, s[34:35] offset:112
+; GFX1250-SDAG-NEXT:    s_load_b512 s[16:31], s[4:5], 0xa4
+; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT:    s_load_b512 s[0:15], s[4:5], 0xe4
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], s[12:13]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[2:3], v[2:3], s[14:15]
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v32, s20 :: v_dual_mov_b32 v33, s21
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v34, s22 :: v_dual_mov_b32 v35, s23
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v36, s18 :: v_dual_mov_b32 v39, s29
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v40, s30 :: v_dual_mov_b32 v41, s31
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v42, s24 :: v_dual_mov_b32 v37, s19
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v38, s28 :: v_dual_mov_b32 v55, s15
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v51, s3 :: v_dual_mov_b32 v52, s12
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v53, s13 :: v_dual_mov_b32 v54, s14
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v49, s7 :: v_dual_mov_b32 v50, s2
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v45, s27 :: v_dual_mov_b32 v46, s4
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v47, s5 :: v_dual_mov_b32 v48, s6
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v43, s25 :: v_dual_mov_b32 v44, s26
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x7
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[32:33]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[2:3], v[2:3], v[34:35]
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v32, s8 :: v_dual_mov_b32 v33, s9
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v34, s10 :: v_dual_mov_b32 v35, s11
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x6
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[6:7], v[6:7], s[10:11]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x4
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[20:21], v[20:21], s[16:17]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x3
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[12:13], v[12:13], s[40:41]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[6:7], v[6:7], v[40:41]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[40:41], s[0:1]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[4:5], v[4:5], v[38:39]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[38:39], s[16:17]
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x2
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[18:19], v[18:19], s[38:39]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x1
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[24:25], v[24:25], s[48:49]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[20:21], v[20:21], v[32:33]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[22:23], v[22:23], v[34:35]
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[28:29], v[28:29], s[44:45]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[30:31], v[30:31], s[46:47]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[26:27], v[26:27], s[50:51]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[16:17], v[16:17], s[36:37]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[14:15], v[14:15], s[42:43]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[22:23], v[22:23], s[18:19]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[8:9], v[8:9], s[20:21]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[10:11], v[10:11], s[22:23]
-; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[4:5], v[4:5], s[8:9]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[28:29], v[28:29], v[52:53]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[30:31], v[30:31], v[54:55]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[26:27], v[26:27], v[50:51]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[24:25], v[24:25], v[40:41]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[16:17], v[16:17], v[46:47]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[18:19], v[18:19], v[48:49]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[8:9], v[8:9], v[42:43]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[10:11], v[10:11], v[44:45]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[14:15], v[14:15], v[36:37]
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[12:13], v[12:13], v[38:39]
 ; GFX1250-SDAG-NEXT:    s_clause 0x7
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[28:31], s[0:1] offset:96
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[24:27], s[0:1] offset:112
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[16:19], s[0:1] offset:64
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[12:15], s[0:1] offset:80
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[8:11], s[0:1] offset:48
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[4:7], s[0:1]
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[0:3], s[0:1] offset:16
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[20:23], s[34:35] offset:96
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[28:31], s[34:35] offset:112
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[24:27], s[34:35] offset:64
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[16:19], s[34:35] offset:80
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[8:11], s[34:35] offset:32
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[4:7], s[34:35] offset:48
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[12:15], s[34:35]
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[0:3], s[34:35] offset:16
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fmul_v32_vs:
@@ -1573,54 +1667,70 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[34:35], s[4:5], 0x24
 ; GFX1250-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT:    v_lshlrev_b32_e32 v32, 7, v0
+; GFX1250-GISEL-NEXT:    v_lshlrev_b32_e32 v56, 7, v0
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_clause 0x7
-; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v32, s[34:35]
-; GFX1250-GISEL-NEXT:    global_load_b128 v[4:7], v32, s[34:35] offset:16
-; GFX1250-GISEL-NEXT:    global_load_b128 v[8:11], v32, s[34:35] offset:32
-; GFX1250-GISEL-NEXT:    global_load_b128 v[12:15], v32, s[34:35] offset:48
-; GFX1250-GISEL-NEXT:    global_load_b128 v[16:19], v32, s[34:35] offset:64
-; GFX1250-GISEL-NEXT:    global_load_b128 v[20:23], v32, s[34:35] offset:80
-; GFX1250-GISEL-NEXT:    global_load_b128 v[24:27], v32, s[34:35] offset:96
-; GFX1250-GISEL-NEXT:    global_load_b128 v[28:31], v32, s[34:35] offset:112
+; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v56, s[34:35]
+; GFX1250-GISEL-NEXT:    global_load_b128 v[4:7], v56, s[34:35] offset:16
+; GFX1250-GISEL-NEXT:    global_load_b128 v[8:11], v56, s[34:35] offset:32
+; GFX1250-GISEL-NEXT:    global_load_b128 v[12:15], v56, s[34:35] offset:48
+; GFX1250-GISEL-NEXT:    global_load_b128 v[16:19], v56, s[34:35] offset:64
+; GFX1250-GISEL-NEXT:    global_load_b128 v[20:23], v56, s[34:35] offset:80
+; GFX1250-GISEL-NEXT:    global_load_b128 v[24:27], v56, s[34:35] offset:96
+; GFX1250-GISEL-NEXT:    global_load_b128 v[28:31], v56, s[34:35] offset:112
 ; GFX1250-GISEL-NEXT:    s_load_b512 s[16:31], s[4:5], 0xa4
 ; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_load_b512 s[0:15], s[4:5], 0xe4
-; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x7
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], s[16:17]
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[2:3], v[2:3], s[18:19]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[32:33], s[16:17]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[34:35], s[18:19]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[36:37], s[20:21]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[38:39], s[22:23]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[40:41], s[24:25]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[42:43], s[26:27]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[44:45], s[28:29]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[46:47], s[30:31]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[48:49], s[0:1]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[50:51], s[2:3]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[52:53], s[4:5]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[54:55], s[6:7]
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x7
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[32:33]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[2:3], v[2:3], v[34:35]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[32:33], s[8:9]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[34:35], s[10:11]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x6
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[4:5], v[4:5], s[20:21]
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[6:7], v[6:7], s[22:23]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[4:5], v[4:5], v[36:37]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[6:7], v[6:7], v[38:39]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[36:37], s[12:13]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[38:39], s[14:15]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x5
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[8:9], v[8:9], s[24:25]
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[10:11], v[10:11], s[26:27]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[8:9], v[8:9], v[40:41]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[10:11], v[10:11], v[42:43]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x4
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[12:13], v[12:13], s[28:29]
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[14:15], v[14:15], s[30:31]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[12:13], v[12:13], v[44:45]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[14:15], v[14:15], v[46:47]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x3
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[16:17], v[16:17], s[0:1]
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[18:19], v[18:19], s[2:3]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[16:17], v[16:17], v[48:49]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[18:19], v[18:19], v[50:51]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x2
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[20:21], v[20:21], s[4:5]
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[22:23], v[22:23], s[6:7]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[20:21], v[20:21], v[52:53]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[22:23], v[22:23], v[54:55]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x1
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[24:25], v[24:25], s[8:9]
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[26:27], v[26:27], s[10:11]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[24:25], v[24:25], v[32:33]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[26:27], v[26:27], v[34:35]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[28:29], v[28:29], s[12:13]
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[30:31], v[30:31], s[14:15]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[28:29], v[28:29], v[36:37]
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[30:31], v[30:31], v[38:39]
 ; GFX1250-GISEL-NEXT:    s_clause 0x7
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[0:3], s[34:35]
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[4:7], s[34:35] offset:16
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[8:11], s[34:35] offset:32
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[12:15], s[34:35] offset:48
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[16:19], s[34:35] offset:64
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[20:23], s[34:35] offset:80
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[24:27], s[34:35] offset:96
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[28:31], s[34:35] offset:112
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[0:3], s[34:35]
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[4:7], s[34:35] offset:16
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[8:11], s[34:35] offset:32
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[12:15], s[34:35] offset:48
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[16:19], s[34:35] offset:64
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[20:23], s[34:35] offset:80
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[24:27], s[34:35] offset:96
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[28:31], s[34:35] offset:112
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %a, i32 %id
@@ -1685,15 +1795,16 @@ define amdgpu_kernel void @fmul_v2_v_imm(ptr addrspace(1) %a) {
 ; GFX1250-GISEL-LABEL: fmul_v2_v_imm:
 ; GFX1250-GISEL:       ; %bb.0:
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s2, 0x42c80000
-; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s3, s2
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -1828,15 +1939,16 @@ define amdgpu_kernel void @fmul_v2_v_lit_splat(ptr addrspace(1) %a) {
 ; GFX1250-GISEL-LABEL: fmul_v2_v_lit_splat:
 ; GFX1250-GISEL:       ; %bb.0:
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s2, 4.0
-; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s3, s2
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -1873,17 +1985,31 @@ define amdgpu_kernel void @fmul_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
 ; PACKED-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; PACKED-NEXT:    s_endpgm
 ;
-; GFX1250-LABEL: fmul_v2_v_unfoldable_lit:
-; GFX1250:       ; %bb.0:
-; GFX1250-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; GFX1250-NEXT:    s_mov_b64 s[2:3], lit64(0x4040000040800000)
-; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
-; GFX1250-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], s[2:3]
-; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
-; GFX1250-NEXT:    s_endpgm
+; GFX1250-SDAG-LABEL: fmul_v2_v_unfoldable_lit:
+; GFX1250-SDAG:       ; %bb.0:
+; GFX1250-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[2:3], lit64(0x4040000040800000)
+; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    s_endpgm
+;
+; GFX1250-GISEL-LABEL: fmul_v2_v_unfoldable_lit:
+; GFX1250-GISEL:       ; %bb.0:
+; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    s_mov_b64 s[2:3], lit64(0x4040000040800000)
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
   %load = load <2 x float>, ptr addrspace(1) %gep, align 8
@@ -2040,12 +2166,14 @@ define amdgpu_kernel void @fma_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
 ; GFX1250-LABEL: fma_v2_vs:
 ; GFX1250:       ; %bb.0:
 ; GFX1250-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1250-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
 ; GFX1250-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-NEXT:    global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], s[2:3], s[2:3]
-; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[2:3]
+; GFX1250-NEXT:    global_store_b64 v4, v[0:1], s[0:1] scale_offset
 ; GFX1250-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -2104,13 +2232,16 @@ define amdgpu_kernel void @fma_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
 ; GFX1250-SDAG-NEXT:    s_clause 0x1
 ; GFX1250-SDAG-NEXT:    s_load_b64 s[6:7], s[4:5], 0x24
 ; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
-; GFX1250-SDAG-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v8, 0x3ff, v0
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v4, s[6:7] scale_offset
+; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v8, s[6:7] scale_offset
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[4:5], s[2:3]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[6:7], s[0:1]
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[2:3], v[2:3], s[2:3], s[2:3]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], s[0:1], s[0:1]
-; GFX1250-SDAG-NEXT:    global_store_b128 v4, v[0:3], s[6:7] scale_offset
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[2:3], v[2:3], v[4:5], v[4:5]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[6:7], v[6:7]
+; GFX1250-SDAG-NEXT:    global_store_b128 v8, v[0:3], s[6:7] scale_offset
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fma_v4_vs:
@@ -2118,13 +2249,16 @@ define amdgpu_kernel void @fma_v4_vs(ptr addrspace(1) %a, <4 x float> %x) {
 ; GFX1250-GISEL-NEXT:    s_clause 0x1
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[6:7], s[4:5], 0x24
 ; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x34
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v8, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v4, s[6:7] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v8, s[6:7] scale_offset
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[0:1]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[6:7], s[2:3]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], s[0:1], s[0:1]
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[2:3], v[2:3], s[2:3], s[2:3]
-; GFX1250-GISEL-NEXT:    global_store_b128 v4, v[0:3], s[6:7] scale_offset
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[4:5], v[4:5]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[2:3], v[2:3], v[6:7], v[6:7]
+; GFX1250-GISEL-NEXT:    global_store_b128 v8, v[0:3], s[6:7] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <4 x float>, ptr addrspace(1) %a, i32 %id
@@ -2294,56 +2428,68 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
 ;
 ; GFX1250-SDAG-LABEL: fma_v32_vs:
 ; GFX1250-SDAG:       ; %bb.0:
-; GFX1250-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1250-SDAG-NEXT:    s_load_b64 s[34:35], s[4:5], 0x24
 ; GFX1250-SDAG-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT:    v_lshlrev_b32_e32 v32, 7, v0
+; GFX1250-SDAG-NEXT:    v_lshlrev_b32_e32 v56, 7, v0
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
 ; GFX1250-SDAG-NEXT:    s_clause 0x7
-; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v32, s[0:1] offset:16
-; GFX1250-SDAG-NEXT:    global_load_b128 v[4:7], v32, s[0:1]
-; GFX1250-SDAG-NEXT:    global_load_b128 v[8:11], v32, s[0:1] offset:48
-; GFX1250-SDAG-NEXT:    global_load_b128 v[20:23], v32, s[0:1] offset:32
-; GFX1250-SDAG-NEXT:    global_load_b128 v[12:15], v32, s[0:1] offset:80
-; GFX1250-SDAG-NEXT:    global_load_b128 v[16:19], v32, s[0:1] offset:64
-; GFX1250-SDAG-NEXT:    global_load_b128 v[24:27], v32, s[0:1] offset:112
-; GFX1250-SDAG-NEXT:    global_load_b128 v[28:31], v32, s[0:1] offset:96
-; GFX1250-SDAG-NEXT:    s_clause 0x1
-; GFX1250-SDAG-NEXT:    s_load_b512 s[8:23], s[4:5], 0xa4
-; GFX1250-SDAG-NEXT:    s_load_b512 s[36:51], s[4:5], 0xe4
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x7
+; GFX1250-SDAG-NEXT:    global_load_b128 v[0:3], v56, s[34:35] offset:16
+; GFX1250-SDAG-NEXT:    global_load_b128 v[4:7], v56, s[34:35] offset:48
+; GFX1250-SDAG-NEXT:    global_load_b128 v[8:11], v56, s[34:35] offset:32
+; GFX1250-SDAG-NEXT:    global_load_b128 v[12:15], v56, s[34:35]
+; GFX1250-SDAG-NEXT:    global_load_b128 v[16:19], v56, s[34:35] offset:80
+; GFX1250-SDAG-NEXT:    global_load_b128 v[20:23], v56, s[34:35] offset:96
+; GFX1250-SDAG-NEXT:    global_load_b128 v[24:27], v56, s[34:35] offset:64
+; GFX1250-SDAG-NEXT:    global_load_b128 v[28:31], v56, s[34:35] offset:112
+; GFX1250-SDAG-NEXT:    s_load_b512 s[16:31], s[4:5], 0xa4
+; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT:    s_load_b512 s[0:15], s[4:5], 0xe4
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], s[12:13], s[12:13]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[2:3], v[2:3], s[14:15], s[14:15]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[32:33], s[20:21]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[34:35], s[22:23]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[40:41], s[30:31]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[38:39], s[28:29]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[52:53], s[12:13]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[54:55], s[14:15]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[50:51], s[2:3]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[46:47], s[4:5]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[48:49], s[6:7]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[42:43], s[24:25]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[44:45], s[26:27]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[36:37], s[18:19]
+; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x7
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[32:33], v[32:33]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[2:3], v[2:3], v[34:35], v[34:35]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[32:33], s[8:9]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[34:35], s[10:11]
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x6
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[6:7], v[6:7], s[10:11], s[10:11]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x4
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[20:21], v[20:21], s[16:17], s[16:17]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x3
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[12:13], v[12:13], s[40:41], s[40:41]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x2
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[18:19], v[18:19], s[38:39], s[38:39]
-; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x1
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[24:25], v[24:25], s[48:49], s[48:49]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[6:7], v[6:7], v[40:41], v[40:41]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[40:41], s[0:1]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[4:5], v[4:5], v[38:39], v[38:39]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[38:39], s[16:17]
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[28:29], v[28:29], s[44:45], s[44:45]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[30:31], v[30:31], s[46:47], s[46:47]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[26:27], v[26:27], s[50:51], s[50:51]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[16:17], v[16:17], s[36:37], s[36:37]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[14:15], v[14:15], s[42:43], s[42:43]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[22:23], v[22:23], s[18:19], s[18:19]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[8:9], v[8:9], s[20:21], s[20:21]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[10:11], v[10:11], s[22:23], s[22:23]
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[4:5], v[4:5], s[8:9], s[8:9]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[28:29], v[28:29], v[52:53], v[52:53]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[20:21], v[20:21], v[32:33], v[32:33]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[22:23], v[22:23], v[34:35], v[34:35]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[30:31], v[30:31], v[54:55], v[54:55]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[26:27], v[26:27], v[50:51], v[50:51]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[24:25], v[24:25], v[40:41], v[40:41]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[16:17], v[16:17], v[46:47], v[46:47]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[18:19], v[18:19], v[48:49], v[48:49]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[8:9], v[8:9], v[42:43], v[42:43]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[10:11], v[10:11], v[44:45], v[44:45]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[14:15], v[14:15], v[36:37], v[36:37]
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[12:13], v[12:13], v[38:39], v[38:39]
 ; GFX1250-SDAG-NEXT:    s_clause 0x7
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[28:31], s[0:1] offset:96
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[24:27], s[0:1] offset:112
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[16:19], s[0:1] offset:64
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[12:15], s[0:1] offset:80
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[8:11], s[0:1] offset:48
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[4:7], s[0:1]
-; GFX1250-SDAG-NEXT:    global_store_b128 v32, v[0:3], s[0:1] offset:16
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[20:23], s[34:35] offset:96
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[28:31], s[34:35] offset:112
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[24:27], s[34:35] offset:64
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[16:19], s[34:35] offset:80
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[8:11], s[34:35] offset:32
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[4:7], s[34:35] offset:48
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[12:15], s[34:35]
+; GFX1250-SDAG-NEXT:    global_store_b128 v56, v[0:3], s[34:35] offset:16
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fma_v32_vs:
@@ -2351,54 +2497,70 @@ define amdgpu_kernel void @fma_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[34:35], s[4:5], 0x24
 ; GFX1250-GISEL-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT:    v_lshlrev_b32_e32 v32, 7, v0
+; GFX1250-GISEL-NEXT:    v_lshlrev_b32_e32 v56, 7, v0
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_clause 0x7
-; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v32, s[34:35]
-; GFX1250-GISEL-NEXT:    global_load_b128 v[4:7], v32, s[34:35] offset:16
-; GFX1250-GISEL-NEXT:    global_load_b128 v[8:11], v32, s[34:35] offset:32
-; GFX1250-GISEL-NEXT:    global_load_b128 v[12:15], v32, s[34:35] offset:48
-; GFX1250-GISEL-NEXT:    global_load_b128 v[16:19], v32, s[34:35] offset:64
-; GFX1250-GISEL-NEXT:    global_load_b128 v[20:23], v32, s[34:35] offset:80
-; GFX1250-GISEL-NEXT:    global_load_b128 v[24:27], v32, s[34:35] offset:96
-; GFX1250-GISEL-NEXT:    global_load_b128 v[28:31], v32, s[34:35] offset:112
+; GFX1250-GISEL-NEXT:    global_load_b128 v[0:3], v56, s[34:35]
+; GFX1250-GISEL-NEXT:    global_load_b128 v[4:7], v56, s[34:35] offset:16
+; GFX1250-GISEL-NEXT:    global_load_b128 v[8:11], v56, s[34:35] offset:32
+; GFX1250-GISEL-NEXT:    global_load_b128 v[12:15], v56, s[34:35] offset:48
+; GFX1250-GISEL-NEXT:    global_load_b128 v[16:19], v56, s[34:35] offset:64
+; GFX1250-GISEL-NEXT:    global_load_b128 v[20:23], v56, s[34:35] offset:80
+; GFX1250-GISEL-NEXT:    global_load_b128 v[24:27], v56, s[34:35] offset:96
+; GFX1250-GISEL-NEXT:    global_load_b128 v[28:31], v56, s[34:35] offset:112
 ; GFX1250-GISEL-NEXT:    s_load_b512 s[16:31], s[4:5], 0xa4
 ; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_load_b512 s[0:15], s[4:5], 0xe4
-; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x7
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], s[16:17], s[16:17]
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[2:3], v[2:3], s[18:19], s[18:19]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[32:33], s[16:17]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[34:35], s[18:19]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[36:37], s[20:21]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[38:39], s[22:23]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[40:41], s[24:25]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[42:43], s[26:27]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[44:45], s[28:29]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[46:47], s[30:31]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[48:49], s[0:1]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[50:51], s[2:3]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[52:53], s[4:5]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[54:55], s[6:7]
+; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x7
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[32:33], v[32:33]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[2:3], v[2:3], v[34:35], v[34:35]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[32:33], s[8:9]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[34:35], s[10:11]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x6
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[4:5], v[4:5], s[20:21], s[20:21]
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[6:7], v[6:7], s[22:23], s[22:23]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[4:5], v[4:5], v[36:37], v[36:37]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[6:7], v[6:7], v[38:39], v[38:39]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[36:37], s[12:13]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[38:39], s[14:15]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x5
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[8:9], v[8:9], s[24:25], s[24:25]
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[10:11], v[10:11], s[26:27], s[26:27]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[8:9], v[8:9], v[40:41], v[40:41]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[10:11], v[10:11], v[42:43], v[42:43]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x4
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[12:13], v[12:13], s[28:29], s[28:29]
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[14:15], v[14:15], s[30:31], s[30:31]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[12:13], v[12:13], v[44:45], v[44:45]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[14:15], v[14:15], v[46:47], v[46:47]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x3
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[16:17], v[16:17], s[0:1], s[0:1]
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[18:19], v[18:19], s[2:3], s[2:3]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[16:17], v[16:17], v[48:49], v[48:49]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[18:19], v[18:19], v[50:51], v[50:51]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x2
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[20:21], v[20:21], s[4:5], s[4:5]
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[22:23], v[22:23], s[6:7], s[6:7]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[20:21], v[20:21], v[52:53], v[52:53]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[22:23], v[22:23], v[54:55], v[54:55]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x1
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[24:25], v[24:25], s[8:9], s[8:9]
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[26:27], v[26:27], s[10:11], s[10:11]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[24:25], v[24:25], v[32:33], v[32:33]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[26:27], v[26:27], v[34:35], v[34:35]
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[28:29], v[28:29], s[12:13], s[12:13]
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[30:31], v[30:31], s[14:15], s[14:15]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[28:29], v[28:29], v[36:37], v[36:37]
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[30:31], v[30:31], v[38:39], v[38:39]
 ; GFX1250-GISEL-NEXT:    s_clause 0x7
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[0:3], s[34:35]
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[4:7], s[34:35] offset:16
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[8:11], s[34:35] offset:32
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[12:15], s[34:35] offset:48
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[16:19], s[34:35] offset:64
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[20:23], s[34:35] offset:80
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[24:27], s[34:35] offset:96
-; GFX1250-GISEL-NEXT:    global_store_b128 v32, v[28:31], s[34:35] offset:112
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[0:3], s[34:35]
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[4:7], s[34:35] offset:16
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[8:11], s[34:35] offset:32
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[12:15], s[34:35] offset:48
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[16:19], s[34:35] offset:64
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[20:23], s[34:35] offset:80
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[24:27], s[34:35] offset:96
+; GFX1250-GISEL-NEXT:    global_store_b128 v56, v[28:31], s[34:35] offset:112
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <32 x float>, ptr addrspace(1) %a, i32 %id
@@ -2488,17 +2650,19 @@ define amdgpu_kernel void @fma_v2_v_imm(ptr addrspace(1) %a) {
 ; GFX1250-GISEL-LABEL: fma_v2_v_imm:
 ; GFX1250-GISEL:       ; %bb.0:
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v6, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s2, 0x42c80000
 ; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s4, 0x43480000
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s3, s2
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s5, s4
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v6, s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], s[2:3], s[4:5]
-; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX1250-GISEL-NEXT:    global_store_b64 v6, v[0:1], s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -2653,17 +2817,19 @@ define amdgpu_kernel void @fma_v2_v_lit_splat(ptr addrspace(1) %a) {
 ; GFX1250-GISEL-LABEL: fma_v2_v_lit_splat:
 ; GFX1250-GISEL:       ; %bb.0:
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v6, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s2, 4.0
 ; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s4, 1.0
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s3, s2
 ; GFX1250-GISEL-NEXT:    s_mov_b32 s5, s4
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v6, s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], s[2:3], s[4:5]
-; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX1250-GISEL-NEXT:    global_store_b64 v6, v[0:1], s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -2740,29 +2906,30 @@ define amdgpu_kernel void @fma_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
 ; GFX1250-SDAG-LABEL: fma_v2_v_unfoldable_lit:
 ; GFX1250-SDAG:       ; %bb.0:
 ; GFX1250-SDAG-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-SDAG-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
-; GFX1250-SDAG-NEXT:    s_mov_b64 s[2:3], lit64(0x400000003f800000)
-; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
-; GFX1250-SDAG-NEXT:    s_mov_b64 s[4:5], lit64(0x4040000040800000)
+; GFX1250-SDAG-NEXT:    v_and_b32_e32 v6, 0x3ff, v0
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[2:3], lit64(0x4040000040800000)
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[4:5], lit64(0x400000003f800000)
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    global_load_b64 v[0:1], v6, s[0:1] scale_offset
 ; GFX1250-SDAG-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], s[4:5], s[2:3]
-; GFX1250-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-SDAG-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX1250-SDAG-NEXT:    global_store_b64 v6, v[0:1], s[0:1] scale_offset
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fma_v2_v_unfoldable_lit:
 ; GFX1250-GISEL:       ; %bb.0:
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1250-GISEL-NEXT:    v_and_b32_e32 v2, 0x3ff, v0
+; GFX1250-GISEL-NEXT:    v_and_b32_e32 v6, 0x3ff, v0
 ; GFX1250-GISEL-NEXT:    s_mov_b64 s[2:3], lit64(0x4040000040800000)
 ; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_mov_b64 s[4:5], lit64(0x400000003f800000)
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[4:5], s[4:5]
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v2, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    global_load_b64 v[0:1], v6, s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_wait_loadcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], s[2:3], s[4:5]
-; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT:    v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX1250-GISEL-NEXT:    global_store_b64 v6, v[0:1], s[0:1] scale_offset
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %id = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
@@ -3268,20 +3435,22 @@ define amdgpu_kernel void @fadd_fadd_fsub_0(<2 x float> %arg) {
 ; GFX1250-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
 ; GFX1250-SDAG-NEXT:    s_add_f32 s1, s1, 0
 ; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX1250-SDAG-NEXT:    flat_store_b64 v[0:1], v[0:1]
+; GFX1250-SDAG-NEXT:    flat_store_b64 v[0:1], v[0:1] scope:SCOPE_SE
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fadd_fadd_fsub_0:
 ; GFX1250-GISEL:       ; %bb.0: ; %bb
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[0:1], s[4:5], 0x24
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], s[0:1], 0
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], 0
+; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v0, v1
 ; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v0, v1
 ; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], 0
-; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v3, v0
-; GFX1250-GISEL-NEXT:    flat_store_b64 v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    flat_store_b64 v[0:1], v[2:3] scope:SCOPE_SE
 ; GFX1250-GISEL-NEXT:    s_endpgm
 bb:
   %i12 = fadd <2 x float> zeroinitializer, %arg
@@ -3363,15 +3532,16 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
 ; GFX1250-SDAG-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX1250-SDAG-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-SDAG-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1250-SDAG-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX1250-SDAG-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT:    s_add_f32 s6, s1, s3
-; GFX1250-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], s[2:3], s[6:7] op_sel_hi:[1,0]
-; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, v0
-; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[2:3], s[2:3] neg_lo:[0,1] neg_hi:[0,1]
-; GFX1250-SDAG-NEXT:    global_store_b64 v4, v[0:1], s[4:5]
+; GFX1250-SDAG-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-SDAG-NEXT:    s_add_f32 s2, s1, s3
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_3)
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[2:3], v[0:1], s[2:3] op_sel_hi:[1,0]
+; GFX1250-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT:    v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, v2
+; GFX1250-SDAG-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-SDAG-NEXT:    v_pk_add_f32 v[0:1], v[4:5], v[0:1] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-SDAG-NEXT:    global_store_b64 v2, v[0:1], s[4:5]
 ; GFX1250-SDAG-NEXT:    s_endpgm
 ;
 ; GFX1250-GISEL-LABEL: fadd_fadd_fsub:
@@ -3380,13 +3550,16 @@ define amdgpu_kernel void @fadd_fadd_fsub(<2 x float> %arg, <2 x float> %arg1, p
 ; GFX1250-GISEL-NEXT:    s_wait_xcnt 0x0
 ; GFX1250-GISEL-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], s[0:1], s[2:3]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[2:3], s[2:3]
 ; GFX1250-GISEL-NEXT:    s_sub_f32 s0, s0, s2
-; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_3)
-; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v2, s0
 ; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], s[2:3], v[0:1]
-; GFX1250-GISEL-NEXT:    v_dual_subrev_f32 v3, s3, v0 :: v_dual_mov_b32 v0, 0
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v0, v1
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    v_pk_add_f32 v[0:1], v[2:3], v[0:1]
+; GFX1250-GISEL-NEXT:    v_dual_mov_b32 v2, s0 :: v_dual_subrev_f32 v3, s3, v0
+; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1250-GISEL-NEXT:    global_store_b64 v0, v[2:3], s[4:5]
 ; GFX1250-GISEL-NEXT:    s_endpgm
 bb:
@@ -3593,7 +3766,9 @@ define amdgpu_kernel void @fneg_v2f32_scalar(ptr addrspace(1) %a, <2 x float> %x
 ; GFX1250-GISEL-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
 ; GFX1250-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX1250-GISEL-NEXT:    s_wait_kmcnt 0x0
-; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], 1.0, s[2:3] op_sel_hi:[0,1] neg_lo:[0,1] neg_hi:[0,1]
+; GFX1250-GISEL-NEXT:    v_mov_b64_e32 v[0:1], s[2:3]
+; GFX1250-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT:    v_pk_mul_f32 v[0:1], 1.0, v[0:1] op_sel_hi:[0,1] neg_lo:[0,1] neg_hi:[0,1]
 ; GFX1250-GISEL-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
 ; GFX1250-GISEL-NEXT:    s_endpgm
   %fneg = fsub <2 x float> <float -0.0, float -0.0>, %x
diff --git a/llvm/test/CodeGen/AMDGPU/pal-metadata-3.0.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/pal-metadata-3.0.gfx1250.ll
new file mode 100644
index 0000000000000..f934c85f68e0f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/pal-metadata-3.0.gfx1250.ll
@@ -0,0 +1,208 @@
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1250 <%s | FileCheck %s --check-prefixes=CHECK
+
+; CHECK-LABEL: {{^}}_amdgpu_cs_main:
+; CHECK: ; TotalNumSgprs: 4
+; CHECK: ; NumVgprs: 2
+; CHECK:           .amdgpu_pal_metadata
+; CHECK-NEXT: ---
+; CHECK-NEXT: amdpal.pipelines:
+; CHECK-NEXT:   - .api:            Vulkan
+; CHECK-NEXT:     .compute_registers:
+; CHECK-NEXT:       .tg_size_en:     true
+; CHECK-NEXT:       .tgid_x_en:      false
+; CHECK-NEXT:       .tgid_y_en:      false
+; CHECK-NEXT:       .tgid_z_en:      false
+; CHECK-NEXT:       .tidig_comp_cnt: 0x1
+; CHECK-NEXT:     .graphics_registers:
+; CHECK-NEXT:      .ps_extra_lds_size: 0
+; CHECK-NEXT:      .spi_ps_input_addr:
+; CHECK-NEXT:        .ancillary_ena:  false
+; CHECK-NEXT:        .front_face_ena: true
+; CHECK-NEXT:        .line_stipple_tex_ena: false
+; CHECK-NEXT:        .linear_center_ena: true
+; CHECK-NEXT:        .linear_centroid_ena: true
+; CHECK-NEXT:        .linear_sample_ena: true
+; CHECK-NEXT:        .persp_center_ena: true
+; CHECK-NEXT:        .persp_centroid_ena: true
+; CHECK-NEXT:        .persp_pull_model_ena: false
+; CHECK-NEXT:        .persp_sample_ena: true
+; CHECK-NEXT:        .pos_fixed_pt_ena: true
+; CHECK-NEXT:        .pos_w_float_ena: false
+; CHECK-NEXT:        .pos_x_float_ena: false
+; CHECK-NEXT:        .pos_y_float_ena: false
+; CHECK-NEXT:        .pos_z_float_ena: false
+; CHECK-NEXT:        .sample_coverage_ena: false
+; CHECK-NEXT:      .spi_ps_input_ena:
+; CHECK-NEXT:        .ancillary_ena:  false
+; CHECK-NEXT:        .front_face_ena: false
+; CHECK-NEXT:        .line_stipple_tex_ena: false
+; CHECK-NEXT:        .linear_center_ena: false
+; CHECK-NEXT:        .linear_centroid_ena: false
+; CHECK-NEXT:        .linear_sample_ena: false
+; CHECK-NEXT:        .persp_center_ena: false
+; CHECK-NEXT:        .persp_centroid_ena: false
+; CHECK-NEXT:        .persp_pull_model_ena: false
+; CHECK-NEXT:        .persp_sample_ena: true
+; CHECK-NEXT:        .pos_fixed_pt_ena: false
+; CHECK-NEXT:        .pos_w_float_ena: false
+; CHECK-NEXT:        .pos_x_float_ena: false
+; CHECK-NEXT:        .pos_y_float_ena: false
+; CHECK-NEXT:        .pos_z_float_ena: false
+; CHECK-NEXT:        .sample_coverage_ena: false
+; CHECK-NEXT:    .hardware_stages:
+; CHECK-NEXT:      .cs:
+; CHECK-NEXT:        .checksum_value: 0x9444d7d0
+; CHECK-NEXT:        .debug_mode:     false
+; CHECK-NEXT:        .entry_point:    _amdgpu_cs
+; CHECK-NEXT:        .entry_point_symbol:    _amdgpu_cs_main
+; CHECK-NEXT:        .excp_en:        0
+; CHECK-NEXT:        .float_mode:     0xc0
+; CHECK-NEXT:        .forward_progress: true
+; GFX11-NEXT:        .ieee_mode:      false
+; CHECK-NEXT:        .image_op:       false
+; CHECK-NEXT:        .lds_size:       0
+; CHECK-NEXT:        .mem_ordered:    true
+; CHECK-NEXT:        .scratch_en:     false
+; CHECK-NEXT:        .scratch_memory_size: 0
+; CHECK-NEXT:        .sgpr_count:     0x4
+; CHECK-NEXT:        .sgpr_limit:     0x6a
+; CHECK-NEXT:        .threadgroup_dimensions:
+; CHECK-NEXT:          - 0x1
+; CHECK-NEXT:          - 0x400
+; CHECK-NEXT:          - 0x1
+; CHECK-NEXT:        .trap_present:   false
+; CHECK-NEXT:        .user_data_reg_map:
+; CHECK-NEXT:          - 0x10000000
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:          - 0xffffffff
+; CHECK-NEXT:        .user_sgprs:     0x3
+; CHECK-NEXT:        .vgpr_count:     0x2
+; CHECK-NEXT:        .vgpr_limit:     0x100
+; CHECK-NEXT:        .wavefront_size: 0x20
+; CHECK-NEXT:        .wgp_mode:       false
+; CHECK-NEXT:      .gs:
+; CHECK-NEXT:        .debug_mode:     false
+; CHECK-NEXT:        .entry_point:    _amdgpu_gs
+; CHECK-NEXT:        .entry_point_symbol:    gs_shader
+; CHECK-NEXT:        .forward_progress: true
+; CHECK-NEXT:      .lds_size:       0x400
+; CHECK-NEXT:        .mem_ordered:    true
+; CHECK-NEXT:        .scratch_en:     false
+; CHECK-NEXT:        .scratch_memory_size: 0
+; CHECK-NEXT:        .sgpr_count:     0x1
+; CHECK-NEXT:        .vgpr_count:     0x1
+; CHECK-NEXT:        .wgp_mode:       false
+; CHECK-NEXT:      .hs:
+; CHECK-NEXT:        .debug_mode:     false
+; CHECK-NEXT:        .entry_point:    _amdgpu_hs
+; CHECK-NEXT:        .entry_point_symbol:    hs_shader
+; CHECK-NEXT:        .forward_progress: true
+; CHECK-NEXT:        .lds_size:       0x1000
+; CHECK-NEXT:        .mem_ordered:    true
+; CHECK-NEXT:        .scratch_en:     false
+; CHECK-NEXT:        .scratch_memory_size: 0
+; CHECK-NEXT:        .sgpr_count:     0x1
+; CHECK-NEXT:        .vgpr_count:     0x1
+; CHECK-NEXT:        .wgp_mode:       false
+; CHECK-NEXT:      .ps:
+; CHECK-NEXT:        .debug_mode:     false
+; CHECK-NEXT:        .entry_point:    _amdgpu_ps
+; CHECK-NEXT:        .entry_point_symbol:    ps_shader
+; CHECK-NEXT:        .forward_progress: true
+; CHECK-NEXT:        .lds_size:       0
+; CHECK-NEXT:        .mem_ordered:    true
+; CHECK-NEXT:        .scratch_en:     false
+; CHECK-NEXT:        .scratch_memory_size: 0
+; CHECK-NEXT:        .sgpr_count:     0x1
+; CHECK-NEXT:        .vgpr_count:     0x1
+; CHECK-NEXT:        .wgp_mode:       false
+; CHECK:    .registers:      {}
+; CHECK:amdpal.version:
+; CHECK-NEXT:  - 0x3
+; CHECK-NEXT:  - 0
+; CHECK-NEXT:...
+; CHECK-NEXT:        .end_amdgpu_pal_metadata
+
+define dllexport amdgpu_cs void @_amdgpu_cs_main(i32 inreg %arg1, i32 %arg2) #0 !lgc.shaderstage !1 {
+.entry:
+  %i = call i64 @llvm.amdgcn.s.getpc()
+  %i1 = and i64 %i, -4294967296
+  %i2 = zext i32 %arg1 to i64
+  %i3 = or i64 %i1, %i2
+  %i4 = inttoptr i64 %i3 to ptr addrspace(4)
+  %i5 = and i32 %arg2, 1023
+  %i6 = lshr i32 %arg2, 10
+  %i7 = and i32 %i6, 1023
+  %i8 = add nuw nsw i32 %i7, %i5
+  %i9 = load <4 x i32>, ptr addrspace(4) %i4, align 16
+  %.idx = shl nuw nsw i32 %i8, 2
+  call void @llvm.amdgcn.raw.buffer.store.i32(i32 1, <4 x i32> %i9, i32 %.idx, i32 0, i32 0)
+  ret void
+}
+
+define dllexport amdgpu_ps void @ps_shader() #1 {
+  ret void
+}
+
+@LDS.GS = external addrspace(3) global [1 x i32], align 4
+
+define dllexport amdgpu_gs void @gs_shader() #2 {
+  %ptr = getelementptr i32, ptr addrspace(3) @LDS.GS, i32 0
+  store i32 0, ptr addrspace(3) %ptr, align 4
+  ret void
+}
+
+@LDS.HS = external addrspace(3) global [1024 x i32], align 4
+
+define dllexport amdgpu_hs void @hs_shader() #2 {
+  %ptr = getelementptr i32, ptr addrspace(3) @LDS.HS, i32 0
+  store i32 0, ptr addrspace(3) %ptr, align 4
+  ret void
+}
+
+!amdgpu.pal.metadata.msgpack = !{!0}
+
+; Function Attrs: nounwind willreturn memory(none)
+declare ptr addrspace(7) @lgc.buffer.desc.to.ptr(<4 x i32>) #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare i64 @llvm.amdgcn.s.getpc() #2
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(write)
+declare void @llvm.amdgcn.raw.buffer.store.i32(i32, <4 x i32>, i32, i32, i32 immarg) #3
+
+attributes #0 = { nounwind memory(readwrite) "amdgpu-flat-work-group-size"="1024,1024" "amdgpu-memory-bound"="false" "amdgpu-unroll-threshold"="700" "amdgpu-wave-limiter"="false" "amdgpu-work-group-info-arg-no"="4" "denormal-fp-math-f32"="preserve-sign" }
+
+attributes #1 = { nounwind memory(readwrite) "InitialPSInputAddr"="36983" }
+
+!0 = !{!"\82\B0amdpal.pipelines\91\8A\A4.api\A6Vulkan\B2.compute_registers\85\AB.tg_size_en\C3\AA.tgid_x_en\C2\AA.tgid_y_en\C2\AA.tgid_z_en\C2\AF.tidig_comp_cnt\01\B0.hardware_stages\81\A3.cs\8C\AF.checksum_value\CE\94D\D7\D0\AB.debug_mode\00\AB.float_mode\CC\C0\A9.image_op\C2\AC.mem_ordered\C3\AB.sgpr_limitj\B7.threadgroup_dimensions\93\01\CD\04\00\01\AD.trap_present\00\B2.user_data_reg_map\DC\00 \CE\10\00\00\00\CE\FF\FF\FF\FF\00\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\CE\FF\FF\FF\FF\AB.user_sgprs\03\AB.vgpr_limit\CD\01\00\AF.wavefront_size \B7.internal_pipeline_hash\92\CF\E7\10k\A6:\A6%\F7\CF\B2\1F\1A\D4{\DA\E1T\AA.registers\80\A8.shaders\81\A8.compute\82\B0.api_shader_hash\92\CF\E9Zn7}\1E\B9\E7\00\B1.hardware_mapping\91\A3.cs\B0.spill_threshold\CE\FF\FF\FF\FF\A5.type\A2Cs\B0.user_data_limit\01\AF.xgl_cache_info\82\B3.128_bit_cache_hash\92\CF\B4X\B8\11[\A4\88P\CF\A0;\B0\AF\FF\B4\BE\C0\AD.llpc_version\A461.1\AEamdpal.version\92\03\00"}
+!1 = !{i32 7}
diff --git a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
index 764a1e1090181..21455a9f5074f 100644
--- a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
+++ b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir
@@ -188,8 +188,7 @@ body:             |
 
     ; GCN-LABEL: name: fold_sreg_64_to_sreg_64
     ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 1311768467750121200
-    ; GCN-NEXT: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 1311768467750121200
-    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[S_MOV_B]]
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[S_MOV_B64_]]
     %0:sreg_64 = S_MOV_B64 1311768467750121200
     %1:sreg_64 = COPY killed %0
     SI_RETURN_TO_EPILOG %1
@@ -265,7 +264,7 @@ body:             |
     ; GCN-LABEL: name: fmac_sreg_64_sub0_src0_to_fmamk
     ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[V_FMAMK_F32_:%[0-9]+]]:vgpr_32 = V_FMAMK_F32 [[DEF]], 2882399984, [[DEF1]], implicit $mode, implicit $exec
+    ; GCN-NEXT: [[V_FMAMK_F32_:%[0-9]+]]:vgpr_32 = V_FMAMK_F32 [[DEF]], -1412567312, [[DEF1]], implicit $mode, implicit $exec
     ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_FMAMK_F32_]]
     %0:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
@@ -319,7 +318,7 @@ body:             |
     ; GCN-LABEL: name: fma_sreg_64_sub0_to_fmaak
     ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GCN-NEXT: [[DEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
-    ; GCN-NEXT: [[V_FMAAK_F32_:%[0-9]+]]:vgpr_32 = V_FMAAK_F32 [[DEF]], [[DEF1]], 2882399984, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[V_FMAAK_F32_:%[0-9]+]]:vgpr_32 = V_FMAAK_F32 [[DEF]], [[DEF1]], -1412567312, implicit $mode, implicit $exec
     ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_FMAAK_F32_]]
     %0:vgpr_32 = IMPLICIT_DEF
     %1:vgpr_32 = IMPLICIT_DEF
@@ -761,9 +760,106 @@ body:             |
   bb.0:
     ; GCN-LABEL: name: fold_av_mov_b32_imm_pseudo_inlineimm_to_av
     ; GCN: [[AV_MOV_:%[0-9]+]]:av_32 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
-    ; GCN-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY killed [[AV_MOV_]]
-    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[COPY]]
+    ; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 64, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_MOV_B32_e32_]]
     %0:av_32 = AV_MOV_B32_IMM_PSEUDO 64, implicit $exec
     %1:av_32 = COPY killed %0
     SI_RETURN_TO_EPILOG implicit %1
 ...
+
+---
+name:            fold_av_mov_b64_imm_pseudo_inlineimm_to_vgpr
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: fold_av_mov_b64_imm_pseudo_inlineimm_to_vgpr
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64_align2 = AV_MOV_B64_IMM_PSEUDO 64, implicit $exec
+    ; GCN-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 64, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_MOV_B]]
+    %0:av_64_align2 = AV_MOV_B64_IMM_PSEUDO 64, implicit $exec
+    %1:vreg_64_align2 = COPY killed %0
+    SI_RETURN_TO_EPILOG implicit %1
+...
+
+---
+name:            fold_av_mov_b64_imm_pseudo_inlineimm_to_agpr
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: fold_av_mov_b64_imm_pseudo_inlineimm_to_agpr
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64_align2 = AV_MOV_B64_IMM_PSEUDO 64, implicit $exec
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:areg_64_align2 = COPY killed [[AV_MOV_]]
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[COPY]]
+    %0:av_64_align2 = AV_MOV_B64_IMM_PSEUDO 64, implicit $exec
+    %1:areg_64_align2 = COPY killed %0
+    SI_RETURN_TO_EPILOG implicit %1
+...
+
+---
+name:            fold_av_mov_b64_imm_pseudo_inlineimm_to_av
+body:             |
+  bb.0:
+    ; GCN-LABEL: name: fold_av_mov_b64_imm_pseudo_inlineimm_to_av
+    ; GCN: [[AV_MOV_:%[0-9]+]]:av_64_align2 = AV_MOV_B64_IMM_PSEUDO 64, implicit $exec
+    ; GCN-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 64, implicit $exec
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_MOV_B]]
+    %0:av_64_align2 = AV_MOV_B64_IMM_PSEUDO 64, implicit $exec
+    %1:av_64_align2 = COPY killed %0
+    SI_RETURN_TO_EPILOG implicit %1
+...
+
+---
+name:            fold_simm_16_sub_to_lo_from_mov_64_virt_sgpr16
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_simm_16_sub_to_lo_from_mov_64_virt_sgpr16
+    ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_lo16 = COPY killed [[S_MOV_B64_]].lo16
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[COPY]]
+    %0:sreg_64 = S_MOV_B64 64
+    %1:sgpr_lo16 = COPY killed %0.lo16
+    SI_RETURN_TO_EPILOG %1
+
+...
+---
+name:            fold_simm_16_sub_to_hi_from_mov_64_inline_imm_virt_sgpr16
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_virt_sgpr16
+    ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
+    ; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_lo16 = COPY killed [[S_MOV_B64_]].hi16
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG [[COPY]]
+    %0:sreg_64 = S_MOV_B64 64
+    %1:sgpr_lo16 = COPY killed %0.hi16
+    SI_RETURN_TO_EPILOG %1
+
+...
+
+---
+name:            fold_simm_16_sub_to_lo_from_mov_64_phys_sgpr16_lo
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_simm_16_sub_to_lo_from_mov_64_phys_sgpr16_lo
+    ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 64
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
+    %0:sreg_64 = S_MOV_B64 64
+    $sgpr0_lo16 = COPY killed %0.lo16
+    SI_RETURN_TO_EPILOG $sgpr0_lo16
+
+...
+---
+name:            fold_simm_16_sub_to_hi_from_mov_64_inline_imm_phys_sgpr16_lo
+body:             |
+  bb.0:
+
+    ; GCN-LABEL: name: fold_simm_16_sub_to_hi_from_mov_64_inline_imm_phys_sgpr16_lo
+    ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 64
+    ; GCN-NEXT: $sgpr0 = S_MOV_B32 0
+    ; GCN-NEXT: SI_RETURN_TO_EPILOG $sgpr0_lo16
+    %0:sreg_64 = S_MOV_B64 64
+    $sgpr0_lo16 = COPY killed %0.hi16
+    SI_RETURN_TO_EPILOG $sgpr0_lo16
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir b/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir
index 458afca384911..1e6c00a54a012 100644
--- a/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir
+++ b/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir
@@ -10,13 +10,13 @@ body:             |
     ; GCN-LABEL: name: bundle_memops
     ; GCN: $vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT: S_NOP 0
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit undef $vgpr3_vgpr4, implicit $exec {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr1, implicit undef $vgpr3_vgpr4, implicit $exec {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   $vgpr1 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 4, 0, implicit $exec
     ; GCN-NEXT: }
     ; GCN-NEXT: S_NOP 0
     ; GCN-NEXT: $vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, implicit $exec
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit undef $vgpr0_vgpr1, implicit $exec, implicit undef $vgpr3_vgpr4 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr1, implicit-def $vgpr2, implicit-def $vgpr5, implicit undef $vgpr0_vgpr1, implicit $exec, implicit undef $vgpr3_vgpr4 {
     ; GCN-NEXT:   $vgpr1 = GLOBAL_LOAD_DWORD undef $vgpr0_vgpr1, 4, 0, implicit $exec
     ; GCN-NEXT:   $vgpr2 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 4, 0, implicit $exec
     ; GCN-NEXT:   $vgpr5 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, implicit $exec
@@ -35,7 +35,7 @@ body:             |
     ; GCN-NEXT: $vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT: S_NOP 0
     ; GCN-NEXT: GLOBAL_STORE_DWORD undef $vgpr3_vgpr4, $vgpr0, 4, 0, implicit $exec
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
     ; GCN-NEXT:   $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
     ; GCN-NEXT:   $vgpr3 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec
     ; GCN-NEXT: }
@@ -44,11 +44,11 @@ body:             |
     ; GCN-NEXT:   DS_WRITE_B32_gfx9 $vgpr0, $vgpr3, 4, 0, implicit killed $m0, implicit $exec
     ; GCN-NEXT: }
     ; GCN-NEXT: S_NOP 0
-    ; GCN-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr2_lo16, implicit-def $sgpr2_hi16, implicit-def $sgpr3, implicit-def $sgpr3_lo16, implicit-def $sgpr3_hi16, implicit undef $sgpr0_sgpr1, implicit undef $sgpr10 {
+    ; GCN-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit undef $sgpr0_sgpr1, implicit undef $sgpr10 {
     ; GCN-NEXT:   $sgpr2 = S_LOAD_DWORD_IMM undef $sgpr0_sgpr1, 0, 0
     ; GCN-NEXT:   $sgpr3 = S_LOAD_DWORD_SGPR undef $sgpr0_sgpr1, undef $sgpr10, 0
     ; GCN-NEXT: }
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit $vgpr0, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr2, implicit $exec, implicit $vgpr1 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr2, implicit $exec, implicit $vgpr1 {
     ; GCN-NEXT:   $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr2, 0, 0, 0, implicit $exec
     ; GCN-NEXT:   $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr1, undef $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr2, 0, 0, 0, implicit $exec
     ; GCN-NEXT: }
@@ -56,7 +56,7 @@ body:             |
     ; GCN-NEXT:   BUFFER_STORE_DWORD_ADDR64 $vgpr0, $vgpr2_vgpr3, undef $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
     ; GCN-NEXT:   BUFFER_STORE_DWORD_ADDR64 $vgpr0, $vgpr2_vgpr3, undef $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
     ; GCN-NEXT: }
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit undef $vgpr4_vgpr5_vgpr6_vgpr7, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $exec {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit undef $vgpr4_vgpr5_vgpr6_vgpr7, implicit undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $exec {
     ; GCN-NEXT:   $vgpr2 = IMAGE_LOAD_V1_V4 undef $vgpr4_vgpr5_vgpr6_vgpr7, undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 2, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s32))
     ; GCN-NEXT:   $vgpr3 = IMAGE_LOAD_V1_V4 undef $vgpr4_vgpr5_vgpr6_vgpr7, undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 2, -1, 0, 0, 0, 0, 0, 0, implicit $exec :: (load (s32))
     ; GCN-NEXT: }
@@ -68,7 +68,7 @@ body:             |
     ; GCN-NEXT: $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71 = S_LOAD_DWORDX8_IMM undef $sgpr10_sgpr11, 464, 0
     ; GCN-NEXT: $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75 = S_LOAD_DWORDX8_IMM undef $sgpr10_sgpr11, 128, 0
     ; GCN-NEXT: S_NOP 0
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit-def $vgpr3, implicit-def $vgpr3_lo16, implicit-def $vgpr3_hi16, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
     ; GCN-NEXT:   $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
     ; GCN-NEXT:   $vgpr3 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec
     ; GCN-NEXT: }
@@ -124,7 +124,7 @@ body:             |
     ; GCN-LABEL: name: bundle_dbg_value_0
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6
     ; GCN-NEXT: {{  $}}
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr1, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   DBG_VALUE $vgpr0, 0, 0
     ; GCN-NEXT:   $vgpr1 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
@@ -145,7 +145,7 @@ body:             |
     ; GCN-LABEL: name: bundle_dbg_value_1
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6, $vgpr1
     ; GCN-NEXT: {{  $}}
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr2, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   DBG_VALUE $vgpr0, 0, 0
     ; GCN-NEXT:   DBG_VALUE $vgpr1, 0, 0
@@ -170,7 +170,7 @@ body:             |
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6, $vgpr1
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: DBG_VALUE $vgpr1, 0, 0
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr2, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   DBG_VALUE $vgpr0, 0, 0
     ; GCN-NEXT:   $vgpr2 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
@@ -193,7 +193,7 @@ body:             |
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr1, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr2, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr1, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   KILL $vgpr1
     ; GCN-NEXT:   $vgpr2 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
@@ -215,7 +215,7 @@ body:             |
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr2, implicit-def $vgpr2_lo16, implicit-def $vgpr2_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr2, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   KILL internal $vgpr0
     ; GCN-NEXT:   $vgpr2 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
@@ -239,7 +239,7 @@ body:             |
     ; GCN-LABEL: name: post_bundle_kill
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6
     ; GCN-NEXT: {{  $}}
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr1, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   $vgpr1 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
     ; GCN-NEXT: }
@@ -258,7 +258,7 @@ body:             |
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: $vgpr7 = V_MOV_B32_e32 0, implicit $exec
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr1, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   $vgpr1 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
     ; GCN-NEXT: }
@@ -280,7 +280,7 @@ body:             |
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6
     ; GCN-NEXT: {{  $}}
     ; GCN-NEXT: $vgpr7 = V_MOV_B32_e32 0, implicit $exec
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr1, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   $vgpr1 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
     ; GCN-NEXT: }
@@ -300,7 +300,7 @@ body:             |
     ; GCN-LABEL: name: post_bundle_multi_kill_0
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6
     ; GCN-NEXT: {{  $}}
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr1, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   $vgpr1 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
     ; GCN-NEXT: }
@@ -320,7 +320,7 @@ body:             |
     ; GCN-LABEL: name: post_bundle_multi_kill_1
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6
     ; GCN-NEXT: {{  $}}
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr1, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   $vgpr1 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
     ; GCN-NEXT: }
@@ -341,7 +341,7 @@ body:             |
     ; GCN-LABEL: name: post_bundle_kill_and_null_reg_dbginfo
     ; GCN: liveins: $vgpr3_vgpr4, $vgpr5_vgpr6
     ; GCN-NEXT: {{  $}}
-    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr0_lo16, implicit-def $vgpr0_hi16, implicit-def $vgpr1, implicit-def $vgpr1_lo16, implicit-def $vgpr1_hi16, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
+    ; GCN-NEXT: BUNDLE implicit-def $vgpr0, implicit-def $vgpr1, implicit $vgpr3_vgpr4, implicit $exec, implicit $vgpr5_vgpr6 {
     ; GCN-NEXT:   $vgpr0 = GLOBAL_LOAD_DWORD $vgpr3_vgpr4, 0, 0, implicit $exec
     ; GCN-NEXT:   DBG_VALUE $noreg, $noreg
     ; GCN-NEXT:   $vgpr1 = GLOBAL_LOAD_DWORD $vgpr5_vgpr6, 0, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/postra-bundle-vimage-vsample-gfx12.mir b/llvm/test/CodeGen/AMDGPU/postra-bundle-vimage-vsample-gfx12.mir
index 6ab8a343dd96b..5fea0aee72ec7 100644
--- a/llvm/test/CodeGen/AMDGPU/postra-bundle-vimage-vsample-gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/postra-bundle-vimage-vsample-gfx12.mir
@@ -9,7 +9,7 @@ body:             |
     ; GFX12-LABEL: name: post_bundle_vimage
     ; GFX12: liveins: $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: BUNDLE implicit-def $vgpr5, implicit-def $vgpr5_lo16, implicit-def $vgpr5_hi16, implicit-def $vgpr4, implicit-def $vgpr4_lo16, implicit-def $vgpr4_hi16, implicit killed $vgpr1, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $exec {
+    ; GFX12-NEXT: BUNDLE implicit-def $vgpr5, implicit-def $vgpr4, implicit killed $vgpr1, implicit killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $exec {
     ; GFX12-NEXT:   $vgpr5 = IMAGE_LOAD_V1_V1_gfx12 $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 2, 1, 0, 0, -1, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
     ; GFX12-NEXT:   $vgpr4 = IMAGE_LOAD_V1_V1_gfx12 killed $vgpr1, killed $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 1, 1, 0, 0, -1, 0, 0, implicit $exec :: (dereferenceable invariant load (s32), addrspace 8)
     ; GFX12-NEXT: }
@@ -25,7 +25,7 @@ body:             |
     ; GFX12-LABEL: name: post_bundle_vsample
     ; GFX12: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11
     ; GFX12-NEXT: {{  $}}
-    ; GFX12-NEXT: BUNDLE implicit-def $vgpr6_vgpr7_vgpr8_vgpr9, implicit-def $vgpr6, implicit-def $vgpr6_lo16, implicit-def $vgpr6_hi16, implicit-def $vgpr7, implicit-def $vgpr7_lo16, implicit-def $vgpr7_hi16, implicit-def $vgpr8, implicit-def $vgpr8_lo16, implicit-def $vgpr8_hi16, implicit-def $vgpr9, implicit-def $vgpr9_lo16, implicit-def $vgpr9_hi16, implicit-def $vgpr6_vgpr7, implicit-def $vgpr6_vgpr7_vgpr8, implicit-def $vgpr7_vgpr8, implicit-def $vgpr7_vgpr8_vgpr9, implicit-def $vgpr8_vgpr9, implicit-def $vgpr10_vgpr11_vgpr12_vgpr13, implicit-def $vgpr10, implicit-def $vgpr10_lo16, implicit-def $vgpr10_hi16, implicit-def $vgpr11, implicit-def $vgpr11_lo16, implicit-def $vgpr11_hi16, implicit-def $vgpr12, implicit-def $vgpr12_lo16, implicit-def $vgpr12_hi16, implicit-def $vgpr13, implicit-def $vgpr13_lo16, implicit-def $vgpr13_hi16, implicit-def $vgpr10_vgpr11, implicit-def $vgpr10_vgpr11_vgpr12, implicit-def $vgpr11_vgpr12, implicit-def $vgpr11_vgpr12_vgpr13, implicit-def $vgpr12_vgpr13, implicit killed $vgpr0, implicit killed $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec, implicit killed $vgpr2, implicit killed $vgpr3 {
+    ; GFX12-NEXT: BUNDLE implicit-def $vgpr6_vgpr7_vgpr8_vgpr9, implicit-def $vgpr10_vgpr11_vgpr12_vgpr13, implicit killed $vgpr0, implicit killed $vgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, implicit $sgpr8_sgpr9_sgpr10_sgpr11, implicit $exec, implicit killed $vgpr2, implicit killed $vgpr3 {
     ; GFX12-NEXT:   $vgpr6_vgpr7_vgpr8_vgpr9 = IMAGE_SAMPLE_V4_V2_gfx12 killed $vgpr0, killed $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 8)
     ; GFX12-NEXT:   $vgpr10_vgpr11_vgpr12_vgpr13 = IMAGE_SAMPLE_V4_V2_gfx12 killed $vgpr2, killed $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11, 15, 1, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 8)
     ; GFX12-NEXT: }
diff --git a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
index 79b531e3ce785..546054cba4700 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX942 %s
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX90a %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @preload_block_count_x(ptr addrspace(1) inreg %out) #0 {
 ; GFX942-LABEL: preload_block_count_x:
@@ -30,6 +31,12 @@ define amdgpu_kernel void @preload_block_count_x(ptr addrspace(1) inreg %out) #0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_block_count_x:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
   store i32 %load, ptr addrspace(1) %out
@@ -65,6 +72,12 @@ define amdgpu_kernel void @preload_unused_arg_block_count_x(ptr addrspace(1) inr
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s12
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_unused_arg_block_count_x:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
   store i32 %load, ptr addrspace(1) %out
@@ -101,6 +114,12 @@ define amdgpu_kernel void @no_free_sgprs_block_count_x(ptr addrspace(1) inreg %o
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[14:15]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: no_free_sgprs_block_count_x:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s18
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[8:9]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
   store i32 %load, ptr addrspace(1) %out
@@ -127,6 +146,14 @@ define amdgpu_kernel void @no_inreg_block_count_x(ptr addrspace(1) %out) #0 {
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: no_inreg_block_count_x:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_load_b96 s[0:2], s[0:1], 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
   store i32 %load, ptr addrspace(1) %out
@@ -156,6 +183,16 @@ define amdgpu_kernel void @mixed_inreg_block_count_x(ptr addrspace(1) %out, i32
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: mixed_inreg_block_count_x:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_load_b32 s2, s[0:1], 0x10
+; GFX1250-NEXT:    s_wait_xcnt 0x0
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
   store i32 %load, ptr addrspace(1) %out
@@ -192,6 +229,15 @@ define amdgpu_kernel void @incorrect_type_i64_block_count_x(ptr addrspace(1) inr
 ; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
 ; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: incorrect_type_i64_block_count_x:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[0:1], 0x8
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[0:1]
+; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i64, ptr addrspace(4) %imp_arg_ptr
   store i64 %load, ptr addrspace(1) %out
@@ -228,6 +274,14 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: incorrect_type_i16_block_count_x:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_mov_b32_e32 v0, 0
+; GFX1250-NEXT:    global_load_u16 v1, v0, s[0:1] offset:8
+; GFX1250-NEXT:    s_wait_loadcnt 0x0
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i16, ptr addrspace(4) %imp_arg_ptr
   store i16 %load, ptr addrspace(1) %out
@@ -261,6 +315,12 @@ define amdgpu_kernel void @preload_block_count_y(ptr addrspace(1) inreg %out) #0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_block_count_y:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s5
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 4
   %load = load i32, ptr addrspace(4) %gep
@@ -300,6 +360,14 @@ define amdgpu_kernel void @random_incorrect_offset(ptr addrspace(1) inreg %out)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: random_incorrect_offset:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_load_b32 s0, s[0:1], 0xa
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 2
   %load = load i32, ptr addrspace(4) %gep
@@ -336,6 +404,12 @@ define amdgpu_kernel void @preload_block_count_z(ptr addrspace(1) inreg %out) #0
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s12
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_block_count_z:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 8
   %load = load i32, ptr addrspace(4) %gep
@@ -376,6 +450,15 @@ define amdgpu_kernel void @preload_block_count_x_imparg_align_ptr_i8(ptr addrspa
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_block_count_x_imparg_align_ptr_i8:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_and_b32 s0, s4, 0xff
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_add_co_i32 s0, s6, s0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
   %ext = zext i8 %val to i32
@@ -417,6 +500,13 @@ define amdgpu_kernel void @preload_block_count_xyz(ptr addrspace(1) inreg %out)
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s12
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_block_count_xyz:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s4
+; GFX1250-NEXT:    v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s6
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep_x = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 0
   %load_x = load i32, ptr addrspace(4) %gep_x
@@ -461,6 +551,14 @@ define amdgpu_kernel void @preload_workgroup_size_x(ptr addrspace(1) inreg %out)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_workgroup_size_x:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_and_b32 s0, s7, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 12
   %load = load i16, ptr addrspace(4) %gep
@@ -499,6 +597,14 @@ define amdgpu_kernel void @preload_workgroup_size_y(ptr addrspace(1) inreg %out)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_workgroup_size_y:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_lshr_b32 s0, s7, 16
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 14
   %load = load i16, ptr addrspace(4) %gep
@@ -539,6 +645,14 @@ define amdgpu_kernel void @preload_workgroup_size_z(ptr addrspace(1) inreg %out)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_workgroup_size_z:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_and_b32 s0, s8, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 16
   %load = load i16, ptr addrspace(4) %gep
@@ -587,6 +701,16 @@ define amdgpu_kernel void @preload_workgroup_size_xyz(ptr addrspace(1) inreg %ou
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s2
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_workgroup_size_xyz:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_lshr_b32 s0, s7, 16
+; GFX1250-NEXT:    s_and_b32 s1, s7, 0xffff
+; GFX1250-NEXT:    s_and_b32 s4, s8, 0xffff
+; GFX1250-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s1
+; GFX1250-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s4
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep_x = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 12
   %load_x = load i16, ptr addrspace(4) %gep_x
@@ -636,6 +760,14 @@ define amdgpu_kernel void @preload_remainder_x(ptr addrspace(1) inreg %out) #0 {
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_remainder_x:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_lshr_b32 s0, s8, 16
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 18
   %load = load i16, ptr addrspace(4) %gep
@@ -674,6 +806,14 @@ define amdgpu_kernel void @preloadremainder_y(ptr addrspace(1) inreg %out) #0 {
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preloadremainder_y:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_and_b32 s0, s9, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 20
   %load = load i16, ptr addrspace(4) %gep
@@ -712,6 +852,14 @@ define amdgpu_kernel void @preloadremainder_z(ptr addrspace(1) inreg %out) #0 {
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preloadremainder_z:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_lshr_b32 s0, s9, 16
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 22
   %load = load i16, ptr addrspace(4) %gep
@@ -758,6 +906,16 @@ define amdgpu_kernel void @preloadremainder_xyz(ptr addrspace(1) inreg %out) #0
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preloadremainder_xyz:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_lshr_b32 s0, s9, 16
+; GFX1250-NEXT:    s_lshr_b32 s1, s8, 16
+; GFX1250-NEXT:    s_and_b32 s4, s9, 0xffff
+; GFX1250-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s1
+; GFX1250-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep_x = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 18
   %load_x = load i16, ptr addrspace(4) %gep_x
@@ -805,6 +963,14 @@ define amdgpu_kernel void @no_free_sgprs_preloadremainder_z(ptr addrspace(1) inr
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[14:15]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: no_free_sgprs_preloadremainder_z:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_lshr_b32 s0, s15, 16
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[8:9]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 22
   %load = load i16, ptr addrspace(4) %gep
@@ -845,6 +1011,12 @@ define amdgpu_kernel void @preload_block_max_user_sgprs(ptr addrspace(1) inreg %
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_block_max_user_sgprs:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s12
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %load = load i32, ptr addrspace(4) %imp_arg_ptr
   store i32 %load, ptr addrspace(1) %out
@@ -887,6 +1059,15 @@ define amdgpu_kernel void @preload_block_count_z_workgroup_size_z_remainder_z(pt
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: preload_block_count_z_workgroup_size_z_remainder_z:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_lshr_b32 s0, s9, 16
+; GFX1250-NEXT:    s_and_b32 s1, s8, 0xffff
+; GFX1250-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1250-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s0
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
   %gep0 = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 8
   %gep1 = getelementptr i8, ptr addrspace(4) %imp_arg_ptr, i32 16
diff --git a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
index efe4cfa7e5d2b..be86fd18cd737 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-kernargs.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX942 %s
-
 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX90a %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GFX1250 %s
 
 define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) inreg %out, i8 inreg %arg0) #0 {
 ; GFX942-LABEL: ptr1_i8:
@@ -33,6 +33,14 @@ define amdgpu_kernel void @ptr1_i8(ptr addrspace(1) inreg %out, i8 inreg %arg0)
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: ptr1_i8:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_and_b32 s0, s4, 0xff
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out
   ret void
@@ -68,6 +76,14 @@ define amdgpu_kernel void @ptr1_i8_zext_arg(ptr addrspace(1) inreg %out, i8 zero
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: ptr1_i8_zext_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_and_b32 s0, s4, 0xff
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out, align 4
   ret void
@@ -103,6 +119,14 @@ define amdgpu_kernel void @ptr1_i16_preload_arg(ptr addrspace(1) inreg %out, i16
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: ptr1_i16_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_and_b32 s0, s4, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %ext = zext i16 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out, align 4
   ret void
@@ -136,6 +160,12 @@ define amdgpu_kernel void @ptr1_i32_preload_arg(ptr addrspace(1) inreg %out, i32
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: ptr1_i32_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store i32 %arg0, ptr addrspace(1) %out
   ret void
 }
@@ -172,6 +202,14 @@ define amdgpu_kernel void @i32_ptr1_i32_preload_arg(i32 inreg %arg0, ptr addrspa
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[10:11]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: i32_ptr1_i32_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_add_co_i32 s0, s2, s6
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX1250-NEXT:    s_endpgm
   %add = add i32 %arg0, %arg1
   store i32 %add, ptr addrspace(1) %out
   ret void
@@ -211,6 +249,16 @@ define amdgpu_kernel void @ptr1_i16_i16_preload_arg(ptr addrspace(1) inreg %out,
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: ptr1_i16_i16_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_lshr_b32 s0, s4, 16
+; GFX1250-NEXT:    s_and_b32 s1, s4, 0xffff
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT:    s_add_co_i32 s0, s1, s0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %ext = zext i16 %arg0 to i32
   %ext1 = zext i16 %arg1 to i32
   %add = add i32 %ext, %ext1
@@ -246,6 +294,12 @@ define amdgpu_kernel void @ptr1_v2i8_preload_arg(ptr addrspace(1) inreg %out, <2
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
 ; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: ptr1_v2i8_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <2 x i8> %in, ptr addrspace(1) %out
   ret void
 }
@@ -289,6 +343,18 @@ define amdgpu_kernel void @byref_preload_arg(ptr addrspace(1) inreg %out, ptr ad
 ; GFX90a-NEXT:    global_store_dword v0, v2, s[8:9]
 ; GFX90a-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: byref_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[0:1], 0x100
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v2, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
   %in = load i32, ptr addrspace(4) %in.byref
   store volatile i32 %in, ptr addrspace(1) %out, align 4
   store volatile i32 %after.offset, ptr addrspace(1) %out, align 4
@@ -335,6 +401,18 @@ define amdgpu_kernel void @byref_staggered_preload_arg(ptr addrspace(1) inreg %o
 ; GFX90a-NEXT:    global_store_dword v0, v2, s[8:9]
 ; GFX90a-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: byref_staggered_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_load_b64 s[0:1], s[0:1], 0x100
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s1
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    global_store_b32 v0, v2, s[2:3] scope:SCOPE_SYS
+; GFX1250-NEXT:    s_wait_storecnt 0x0
+; GFX1250-NEXT:    s_endpgm
   %in = load i32, ptr addrspace(4) %in.byref
   store volatile i32 %in, ptr addrspace(1) %out, align 4
   store volatile i32 %after.offset, ptr addrspace(1) %out, align 4
@@ -390,6 +468,18 @@ define amdgpu_kernel void @v8i32_arg(ptr addrspace(1) nocapture inreg %out, <8 x
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
 ; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v8i32_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v0, s14
+; GFX1250-NEXT:    v_dual_mov_b32 v1, s15 :: v_dual_mov_b32 v2, s16
+; GFX1250-NEXT:    v_dual_mov_b32 v3, s17 :: v_dual_mov_b32 v4, s10
+; GFX1250-NEXT:    v_dual_mov_b32 v5, s11 :: v_dual_mov_b32 v6, s12
+; GFX1250-NEXT:    v_mov_b32_e32 v7, s13
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    global_store_b128 v8, v[0:3], s[2:3] offset:16
+; GFX1250-NEXT:    global_store_b128 v8, v[4:7], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <8 x i32> %in, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -425,6 +515,15 @@ define amdgpu_kernel void @v3i16_preload_arg(ptr addrspace(1) nocapture inreg %o
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v3i16_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s5
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3] offset:4
+; GFX1250-NEXT:    global_store_b32 v0, v2, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <3 x i16> %in, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -461,6 +560,13 @@ define amdgpu_kernel void @v3i32_preload_arg(ptr addrspace(1) nocapture inreg %o
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v3i32_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX1250-NEXT:    v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v3, 0
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <3 x i32> %in, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -497,6 +603,13 @@ define amdgpu_kernel void @v3f32_preload_arg(ptr addrspace(1) nocapture inreg %o
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, s14
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v3f32_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s6
+; GFX1250-NEXT:    v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s8
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <3 x float> %in, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -546,6 +659,19 @@ define amdgpu_kernel void @v5i8_preload_arg(ptr addrspace(1) nocapture inreg %ou
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v5i8_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_pack_lh_b32_b16 s0, 0, s4
+; GFX1250-NEXT:    s_and_b32 s1, s4, 0xffff
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s5
+; GFX1250-NEXT:    s_or_b32 s0, s1, s0
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    global_store_b8 v0, v1, s[2:3] offset:4
+; GFX1250-NEXT:    global_store_b32 v0, v2, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <5 x i8> %in, ptr addrspace(1) %out, align 4
   ret void
 }
@@ -604,6 +730,24 @@ define amdgpu_kernel void @v5f64_arg(ptr addrspace(1) nocapture inreg %out, <5 x
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
 ; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v5f64_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    s_load_b64 s[12:13], s[0:1], 0x60
+; GFX1250-NEXT:    s_load_b256 s[4:11], s[0:1], 0x40
+; GFX1250-NEXT:    v_mov_b32_e32 v10, 0
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    v_mov_b64_e32 v[8:9], s[12:13]
+; GFX1250-NEXT:    v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX1250-NEXT:    v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
+; GFX1250-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX1250-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX1250-NEXT:    s_clause 0x2
+; GFX1250-NEXT:    global_store_b64 v10, v[8:9], s[2:3] offset:32
+; GFX1250-NEXT:    global_store_b128 v10, v[0:3], s[2:3] offset:16
+; GFX1250-NEXT:    global_store_b128 v10, v[4:7], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <5 x double> %in, ptr addrspace(1) %out, align 8
   ret void
 }
@@ -665,6 +809,20 @@ define amdgpu_kernel void @v8i8_preload_arg(ptr addrspace(1) inreg %out, <8 x i8
 ; GFX90a-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v8i8_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_pack_lh_b32_b16 s0, 0, s5
+; GFX1250-NEXT:    s_pack_lh_b32_b16 s1, 0, s4
+; GFX1250-NEXT:    s_and_b32 s4, s4, 0xffff
+; GFX1250-NEXT:    s_and_b32 s5, s5, 0xffff
+; GFX1250-NEXT:    s_or_b32 s1, s4, s1
+; GFX1250-NEXT:    s_or_b32 s0, s5, s0
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <8 x i8> %in, ptr addrspace(1) %out
   ret void
 }
@@ -696,6 +854,13 @@ define amdgpu_kernel void @i64_kernel_preload_arg(ptr addrspace(1) inreg %out, i
 ; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[10:11], s[10:11] op_sel:[0,1]
 ; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: i64_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store i64 %a, ptr addrspace(1) %out, align 8
   ret void
 }
@@ -727,6 +892,13 @@ define amdgpu_kernel void @f64_kernel_preload_arg(ptr addrspace(1) inreg %out, d
 ; GFX90a-NEXT:    v_pk_mov_b32 v[0:1], s[10:11], s[10:11] op_sel:[0,1]
 ; GFX90a-NEXT:    global_store_dwordx2 v2, v[0:1], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: f64_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_mov_b64_e32 v[0:1], s[4:5]
+; GFX1250-NEXT:    v_mov_b32_e32 v2, 0
+; GFX1250-NEXT:    global_store_b64 v2, v[0:1], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store double %in, ptr addrspace(1) %out
   ret void
 }
@@ -759,6 +931,12 @@ define amdgpu_kernel void @half_kernel_preload_arg(ptr addrspace(1) inreg %out,
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
 ; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: half_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store half %in, ptr addrspace(1) %out
   ret void
 }
@@ -791,6 +969,12 @@ define amdgpu_kernel void @bfloat_kernel_preload_arg(ptr addrspace(1) inreg %out
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
 ; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: bfloat_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store bfloat %in, ptr addrspace(1) %out
   ret void
 }
@@ -823,6 +1007,12 @@ define amdgpu_kernel void @v2bfloat_kernel_preload_arg(ptr addrspace(1) inreg %o
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v2bfloat_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <2 x bfloat> %in, ptr addrspace(1) %out
   ret void
 }
@@ -858,6 +1048,15 @@ define amdgpu_kernel void @v3bfloat_kernel_preload_arg(ptr addrspace(1) inreg %o
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s10
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v3bfloat_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s5
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s4
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3] offset:4
+; GFX1250-NEXT:    global_store_b32 v0, v2, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <3 x bfloat> %in, ptr addrspace(1) %out
   ret void
 }
@@ -894,6 +1093,13 @@ define amdgpu_kernel void @v6bfloat_kernel_preload_arg(ptr addrspace(1) inreg %o
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v6bfloat_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX1250-NEXT:    v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v3, 0
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <6 x bfloat> %in, ptr addrspace(1) %out
   ret void
 }
@@ -939,6 +1145,17 @@ define amdgpu_kernel void @half_v7bfloat_kernel_preload_arg(ptr addrspace(1) inr
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[0:1]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: half_v7bfloat_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT:    v_dual_mov_b32 v5, s9 :: v_dual_mov_b32 v2, s8
+; GFX1250-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX1250-NEXT:    s_clause 0x2
+; GFX1250-NEXT:    global_store_b16 v3, v4, s[2:3]
+; GFX1250-NEXT:    global_store_b16 v3, v5, s[10:11] offset:12
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[10:11]
+; GFX1250-NEXT:    s_endpgm
   store half %in, ptr addrspace(1) %out
   store <7 x bfloat> %in2, ptr addrspace(1) %out2
   ret void
@@ -974,6 +1191,14 @@ define amdgpu_kernel void @i1_kernel_preload_arg(ptr addrspace(1) inreg %out, i1
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_byte v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: i1_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_and_b32 s0, s4, 1
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b8 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store i1 %in, ptr addrspace(1) %out
   ret void
 }
@@ -1012,6 +1237,14 @@ define amdgpu_kernel void @fp128_kernel_preload_arg(ptr addrspace(1) inreg %out,
 ; GFX90a-NEXT:    v_mov_b32_e32 v3, s15
 ; GFX90a-NEXT:    global_store_dwordx4 v4, v[0:3], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: fp128_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v0, s6
+; GFX1250-NEXT:    v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s8
+; GFX1250-NEXT:    v_mov_b32_e32 v3, s9
+; GFX1250-NEXT:    global_store_b128 v4, v[0:3], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store fp128 %in, ptr addrspace(1) %out
   ret void
 }
@@ -1063,6 +1296,20 @@ define amdgpu_kernel void @v7i8_kernel_preload_arg(ptr addrspace(1) inreg %out,
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v7i8_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_pack_lh_b32_b16 s0, 0, s4
+; GFX1250-NEXT:    s_and_b32 s1, s4, 0xffff
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s5
+; GFX1250-NEXT:    s_or_b32 s0, s1, s0
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s0
+; GFX1250-NEXT:    s_clause 0x2
+; GFX1250-NEXT:    global_store_d16_hi_b8 v0, v1, s[2:3] offset:6
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3] offset:4
+; GFX1250-NEXT:    global_store_b32 v0, v2, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <7 x i8> %in, ptr addrspace(1) %out
   ret void
 }
@@ -1103,6 +1350,16 @@ define amdgpu_kernel void @v7half_kernel_preload_arg(ptr addrspace(1) inreg %out
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s13
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: v7half_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, s9
+; GFX1250-NEXT:    v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v0, s6
+; GFX1250-NEXT:    v_mov_b32_e32 v1, s7
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    global_store_b16 v3, v4, s[2:3] offset:12
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[2:3]
+; GFX1250-NEXT:    s_endpgm
   store <7 x half> %in, ptr addrspace(1) %out
   ret void
 }
@@ -1139,6 +1396,15 @@ define amdgpu_kernel void @i16_i32_kernel_preload_arg(ptr addrspace(1) inreg %ou
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s11
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[12:13]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: i16_i32_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s5
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX1250-NEXT:    global_store_b32 v0, v2, s[6:7]
+; GFX1250-NEXT:    s_endpgm
   store i16 %in, ptr addrspace(1) %out
   store i32 %in2, ptr addrspace(1) %out2
   ret void
@@ -1181,6 +1447,16 @@ define amdgpu_kernel void @i16_v3i32_kernel_preload_arg(ptr addrspace(1) inreg %
 ; GFX90a-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90a-NEXT:    global_store_dwordx3 v3, v[0:2], s[0:1]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: i16_v3i32_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT:    v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX1250-NEXT:    v_mov_b32_e32 v2, s8
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    global_store_b16 v3, v4, s[2:3]
+; GFX1250-NEXT:    global_store_b96 v3, v[0:2], s[10:11]
+; GFX1250-NEXT:    s_endpgm
   store i16 %in, ptr addrspace(1) %out
   store <3 x i32> %in2, ptr addrspace(1) %out2
   ret void
@@ -1216,6 +1492,14 @@ define amdgpu_kernel void @i16_i16_kernel_preload_arg(ptr addrspace(1) inreg %ou
 ; GFX90a-NEXT:    global_store_short v0, v1, s[8:9]
 ; GFX90a-NEXT:    global_store_short_d16_hi v0, v1, s[12:13]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: i16_i16_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX1250-NEXT:    global_store_d16_hi_b16 v0, v1, s[6:7]
+; GFX1250-NEXT:    s_endpgm
   store i16 %in, ptr addrspace(1) %out
   store i16 %in2, ptr addrspace(1) %out2
   ret void
@@ -1261,6 +1545,14 @@ define amdgpu_kernel void @i16_v2i8_kernel_preload_arg(ptr addrspace(1) inreg %o
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_short v0, v1, s[12:13]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: i16_v2i8_kernel_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
+; GFX1250-NEXT:    s_clause 0x1
+; GFX1250-NEXT:    global_store_b16 v0, v1, s[2:3]
+; GFX1250-NEXT:    global_store_d16_hi_b16 v0, v1, s[6:7]
+; GFX1250-NEXT:    s_endpgm
   store i16 %in, ptr addrspace(1) %out
   store <2 x i8> %in2, ptr addrspace(1) %out2
   ret void
@@ -1302,6 +1594,16 @@ define amdgpu_kernel void @i32_ptr1_i32_staggered_preload_arg(i32 inreg %arg0, p
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s2
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: i32_ptr1_i32_staggered_preload_arg:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_load_b96 s[4:6], s[0:1], 0x8
+; GFX1250-NEXT:    s_wait_kmcnt 0x0
+; GFX1250-NEXT:    s_add_co_i32 s0, s2, s6
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX1250-NEXT:    s_endpgm
   %add = add i32 %arg0, %arg1
   store i32 %add, ptr addrspace(1) %out
   ret void
@@ -1336,6 +1638,14 @@ define amdgpu_kernel void @ptr1_i8_trailing_unused(ptr addrspace(1) inreg %out,
 ; GFX90a-NEXT:    v_mov_b32_e32 v1, s0
 ; GFX90a-NEXT:    global_store_dword v0, v1, s[8:9]
 ; GFX90a-NEXT:    s_endpgm
+;
+; GFX1250-LABEL: ptr1_i8_trailing_unused:
+; GFX1250:       ; %bb.0:
+; GFX1250-NEXT:    s_and_b32 s0, s4, 0xff
+; GFX1250-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GFX1250-NEXT:    s_endpgm
   %ext = zext i8 %arg0 to i32
   store i32 %ext, ptr addrspace(1) %out
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
index 3ce09475c0949..79910af5c0434 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
@@ -374,7 +374,7 @@ define i32 @shl_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshlrev_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: shl_i16_zext_i32:
@@ -412,7 +412,7 @@ define i32 @lshr_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_lshrrev_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: lshr_i16_zext_i32:
@@ -450,7 +450,7 @@ define i32 @ashr_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_ashrrev_i16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: ashr_i16_zext_i32:
@@ -488,7 +488,7 @@ define i32 @add_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: add_u16_zext_i32:
@@ -526,7 +526,7 @@ define i32 @sub_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: sub_u16_zext_i32:
@@ -564,7 +564,7 @@ define i32 @mul_lo_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: mul_lo_u16_zext_i32:
@@ -602,7 +602,7 @@ define i32 @min_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: min_u16_zext_i32:
@@ -641,7 +641,7 @@ define i32 @min_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: min_i16_zext_i32:
@@ -680,7 +680,7 @@ define i32 @max_u16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_max_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: max_u16_zext_i32:
@@ -719,7 +719,7 @@ define i32 @max_i16_zext_i32(i16 %x, i16 %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: max_i16_zext_i32:
@@ -758,7 +758,7 @@ define i32 @zext_fadd_f16(half %x, half %y) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: zext_fadd_f16:
@@ -797,8 +797,10 @@ define i32 @zext_fma_f16(half %x, half %y, half %z) {
 ; GFX11-TRUE16-LABEL: zext_fma_f16:
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT:    v_fmac_f16_e32 v2.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, v0.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT:    v_fmac_f16_e32 v0.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: zext_fma_f16:
@@ -838,7 +840,7 @@ define i32 @zext_div_fixup_f16(half %x, half %y, half %z) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_div_fixup_f16 v0.l, v0.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: zext_div_fixup_f16:
@@ -880,7 +882,7 @@ define i32 @zext_fptrunc_f16(float %x) {
 ; GFX11-TRUE16:       ; %bb.0:
 ; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-TRUE16-NEXT:    v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FAKE16-LABEL: zext_fptrunc_f16:
@@ -924,12 +926,20 @@ define i32 @zext_fptrunc_fma_f16(float %x, float %y, float %z) {
 ; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
-; GFX11-LABEL: zext_fptrunc_fma_f16:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    v_fma_mixlo_f16 v0, v0, v1, v2
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: zext_fptrunc_fma_f16:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    v_fma_mixlo_f16 v1, v0, v1, v2
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: zext_fptrunc_fma_f16:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    v_fma_mixlo_f16 v0, v0, v1, v2
+; GFX11-FAKE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT:    s_setpc_b64 s[30:31]
   %fma = call float @llvm.fma.f32(float %x, float %y, float %z)
   %fptrunc = fptrunc float %fma to half
   %cast = bitcast half %fptrunc to i16
@@ -940,3 +950,5 @@ define i32 @zext_fptrunc_fma_f16(float %x, float %y, float %z) {
 declare half @llvm.amdgcn.div.fixup.f16(half, half, half)
 declare half @llvm.fma.f16(half, half, half)
 declare float @llvm.fma.f32(float, float, float)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/ps-shader-arg-count.ll b/llvm/test/CodeGen/AMDGPU/ps-shader-arg-count.ll
index 013b68a40f44b..99e5d0017f30b 100644
--- a/llvm/test/CodeGen/AMDGPU/ps-shader-arg-count.ll
+++ b/llvm/test/CodeGen/AMDGPU/ps-shader-arg-count.ll
@@ -1,5 +1,7 @@
-;RUN: llc < %s -mtriple=amdgcn-pal -mcpu=gfx1010 | FileCheck %s --check-prefixes=CHECK
-;RUN: llc < %s -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 | FileCheck %s --check-prefixes=CHECK
+;RUN: llc -global-isel=1 < %s -mtriple=amdgcn-pal -mcpu=gfx1010 | FileCheck %s --check-prefixes=CHECK
+;RUN: llc -global-isel=1 < %s -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 | FileCheck %s --check-prefixes=CHECK
+;RUN: llc -global-isel=0 < %s -mtriple=amdgcn-pal -mcpu=gfx1010 | FileCheck %s --check-prefixes=CHECK
+;RUN: llc -global-isel=0 < %s -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 | FileCheck %s --check-prefixes=CHECK
 
 ; ;CHECK-LABEL: {{^}}_amdgpu_ps_1_arg:
 ; ;CHECK: NumVgprs: 4
diff --git a/llvm/test/CodeGen/AMDGPU/readcyclecounter.ll b/llvm/test/CodeGen/AMDGPU/readcyclecounter.ll
index 131c5f31585d8..f67cbe381bfad 100644
--- a/llvm/test/CodeGen/AMDGPU/readcyclecounter.ll
+++ b/llvm/test/CodeGen/AMDGPU/readcyclecounter.ll
@@ -10,6 +10,8 @@
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GETREG,GETREG-GISEL -check-prefix=GCN %s
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GCN,GFX12 %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GCN,GFX12 %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GFX1250 %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1250 < %s | FileCheck -check-prefixes=GCN,GFX1250 %s
 
 declare i64 @llvm.readcyclecounter() #0
 
@@ -21,6 +23,7 @@ declare i64 @llvm.readcyclecounter() #0
 ; GFX12:       s_getreg_b32 [[HI2:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_HI)
 ; GFX12:       s_cmp_eq_u32 [[HI1]], [[HI2]]
 ; GFX12:       s_cselect_b32 {{s[0-9]+}}, [[LO1]], 0
+; GFX1250:     s_get_shader_cycles_u64 s{{\[[0-9]+:[0-9]+\]}}
 ; GCN-DAG:     kmcnt
 ; MEMTIME:     store_dwordx2
 ; SIVI-NOT:    kmcnt
@@ -53,6 +56,7 @@ define amdgpu_kernel void @test_readcyclecounter(ptr addrspace(1) %out) #0 {
 ; GFX12:       s_getreg_b32 [[HI1:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_HI)
 ; GFX12:       s_getreg_b32 [[LO1:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_LO)
 ; GFX12:       s_getreg_b32 [[HI2:s[0-9]+]], hwreg(HW_REG_SHADER_CYCLES_HI)
+; GFX1250:     s_get_shader_cycles_u64 s{{\[[0-9]+:[0-9]+\]}}
 ; GCN-DAG:     s_load_{{dword|b32|b64}}
 ; GETREG-DAG:  s_getreg_b32 s{{[0-9]+}}, hwreg(HW_REG_SHADER_CYCLES, 0, 20)
 ; GFX12:       s_cmp_eq_u32 [[HI1]], [[HI2]]
diff --git a/llvm/test/CodeGen/AMDGPU/regcoalesce-64-bit-only-regs.mir b/llvm/test/CodeGen/AMDGPU/regcoalesce-64-bit-only-regs.mir
new file mode 100644
index 0000000000000..038e195742305
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/regcoalesce-64-bit-only-regs.mir
@@ -0,0 +1,57 @@
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -start-before=register-coalescer -show-mc-encoding -o - %s | FileCheck %s
+
+# FIXME: These SRC_*_HI registers do not exist, although defined in the register file
+#        and happily used by the coalescer. The resulting encoding is in fact belong
+#        to the 64-bit register and corresponding *_LO 32-bit part of it.
+
+# CHECK-LABEL: src_private_base:
+# CHECK: s_subb_u32 s0, SRC_PRIVATE_BASE_HI, s1  ; encoding: [0xed,0x01,0x80,0x82]
+---
+name:            src_private_base
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %0:sreg_64 = COPY $src_private_base
+    %1:sreg_64 = IMPLICIT_DEF
+    $scc = IMPLICIT_DEF
+    %2:sreg_32 = S_SUBB_U32 killed %0.sub1:sreg_64, %1.sub1:sreg_64, implicit-def dead $scc, implicit killed $scc
+...
+
+# CHECK-LABEL: src_private_limit:
+# CHECK: s_subb_u32 s0, SRC_PRIVATE_LIMIT_HI, s1  ; encoding: [0xee,0x01,0x80,0x82]
+---
+name:            src_private_limit
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %0:sreg_64 = COPY $src_private_limit
+    %1:sreg_64 = IMPLICIT_DEF
+    $scc = IMPLICIT_DEF
+    %2:sreg_32 = S_SUBB_U32 killed %0.sub1:sreg_64, %1.sub1:sreg_64, implicit-def dead $scc, implicit killed $scc
+...
+
+# CHECK-LABEL: src_shared_base:
+# CHECK: s_subb_u32 s0, SRC_SHARED_BASE_HI, s1  ; encoding: [0xeb,0x01,0x80,0x82]
+---
+name:            src_shared_base
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %0:sreg_64 = COPY $src_shared_base
+    %1:sreg_64 = IMPLICIT_DEF
+    $scc = IMPLICIT_DEF
+    %2:sreg_32 = S_SUBB_U32 killed %0.sub1:sreg_64, %1.sub1:sreg_64, implicit-def dead $scc, implicit killed $scc
+...
+
+# CHECK-LABEL: src_shared_limit:
+# CHECK: s_subb_u32 s0, SRC_SHARED_LIMIT_HI, s1  ; encoding: [0xec,0x01,0x80,0x82]
+---
+name:            src_shared_limit
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    %0:sreg_64 = COPY $src_shared_limit
+    %1:sreg_64 = IMPLICIT_DEF
+    $scc = IMPLICIT_DEF
+    %2:sreg_32 = S_SUBB_U32 killed %0.sub1:sreg_64, %1.sub1:sreg_64, implicit-def dead $scc, implicit killed $scc
+...
diff --git a/llvm/test/CodeGen/AMDGPU/remat-vop.mir b/llvm/test/CodeGen/AMDGPU/remat-vop.mir
index 4f6ea44ccf68b..23cf6f005811e 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-vop.mir
+++ b/llvm/test/CodeGen/AMDGPU/remat-vop.mir
@@ -278,16 +278,16 @@ machineFunctionInfo:
 body:             |
   bb.0:
     ; GCN-LABEL: name: test_remat_v_cvt_i32_f64_e64_undef
-    ; GCN: [[V_CVT_I32_F64_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %1:vreg_64, 0, 0, implicit $exec, implicit $mode
-    ; GCN-NEXT: [[V_CVT_I32_F64_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %1:vreg_64, 0, 0, implicit $exec, implicit $mode
-    ; GCN-NEXT: [[V_CVT_I32_F64_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %1:vreg_64, 0, 0, implicit $exec, implicit $mode
+    ; GCN: [[V_CVT_I32_F64_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %1:vreg_64_align2, 0, 0, implicit $exec, implicit $mode
+    ; GCN-NEXT: [[V_CVT_I32_F64_e64_1:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %1:vreg_64_align2, 0, 0, implicit $exec, implicit $mode
+    ; GCN-NEXT: [[V_CVT_I32_F64_e64_2:%[0-9]+]]:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %1:vreg_64_align2, 0, 0, implicit $exec, implicit $mode
     ; GCN-NEXT: S_NOP 0, implicit [[V_CVT_I32_F64_e64_]]
     ; GCN-NEXT: S_NOP 0, implicit [[V_CVT_I32_F64_e64_1]]
     ; GCN-NEXT: S_NOP 0, implicit [[V_CVT_I32_F64_e64_2]]
     ; GCN-NEXT: S_ENDPGM 0
-    %1:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %0:vreg_64, 0, 0, implicit $exec, implicit $mode
-    %2:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %0:vreg_64, 0, 0, implicit $exec, implicit $mode
-    %3:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %0:vreg_64, 0, 0, implicit $exec, implicit $mode
+    %1:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %0:vreg_64_align2, 0, 0, implicit $exec, implicit $mode
+    %2:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %0:vreg_64_align2, 0, 0, implicit $exec, implicit $mode
+    %3:vgpr_32 = nofpexcept V_CVT_I32_F64_e64 0, undef %0:vreg_64_align2, 0, 0, implicit $exec, implicit $mode
     S_NOP 0, implicit %1
     S_NOP 0, implicit %2
     S_NOP 0, implicit %3
diff --git a/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave64-feature.ll b/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave64-feature.ll
new file mode 100644
index 0000000000000..03dbfdcaf6ff1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/remove-incompatible-wave64-feature.ll
@@ -0,0 +1,34 @@
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32 -stop-after=amdgpu-remove-incompatible-functions\
+; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions < %s 2>%t | FileCheck -check-prefixes=GFX1250 %s
+; RUN: FileCheck --check-prefix=WARN-GFX1250 %s < %t
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32 < %s
+
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 -stop-after=amdgpu-remove-incompatible-functions\
+; RUN:   -pass-remarks=amdgpu-remove-incompatible-functions < %s 2>%t | FileCheck -check-prefixes=GFX1200 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,-wavefrontsize64 < %s
+
+; WARN-GFX1250: removing function 'needs_wavefrontsize64': +wavefrontsize64 is not supported on the current target
+; WARN-GFX1250-NOT: not supported
+
+define void @needs_wavefrontsize64(ptr %out) #0 {
+; GFX1250-NOT:  @needs_wavefrontsize64
+; GFX1200:      define void @needs_wavefrontsize64(
+  %1 = tail call i64 @llvm.read_register.i64(metadata !0)
+  %2 = tail call i64 @llvm.ctpop.i64(i64 %1)
+  store i64 %2, ptr %out, align 4
+  ret void
+}
+
+define void @caller(ptr %out) {
+  ; GFX1250: call void null(
+  ; GFX1200: call void @needs_wavefrontsize64(
+  call void @needs_wavefrontsize64(ptr %out)
+  ret void
+}
+
+declare i64 @llvm.read_register.i64(metadata)
+declare i64 @llvm.ctpop.i64(i64)
+
+!0 = !{!"exec"}
+
+attributes #0 = { "target-features"="+wavefrontsize64" }
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-copy-from.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-copy-from.mir
new file mode 100644
index 0000000000000..6dcfda3117234
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-copy-from.mir
@@ -0,0 +1,261 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=greedy,amdgpu-rewrite-agpr-copy-mfma -verify-machineinstrs -o - %s | FileCheck %s
+
+---
+name:  test_rewrite_mfma_copy_from_agpr_physreg
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $agpr0_agpr1
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_from_agpr_physreg
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $agpr0_agpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY $agpr0_agpr1
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_vgprcd_e64_:%[0-9]+]]:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY3]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = COPY $agpr0_agpr1
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    GLOBAL_STORE_DWORDX2 %0, %4, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
+
+---
+name:  test_rewrite_mfma_copy_from_agpr_unrewritable_use
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_from_agpr_unrewritable_use
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:areg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_vgprcd_e64_:%[0-9]+]]:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY3]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3866633 /* reguse:VReg_64_Align2 */, [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]]
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:areg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %4:vreg_128_align2 = COPY %3
+    %5:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3866633 /* reguse:VReg_64_Align2 */, %5
+    SI_RETURN
+...
+
+---
+name:  test_rewrite_mfma_copy_from_agpr_src2_subreg_use
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_from_agpr_src2_subreg_use
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:areg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_vgprcd_e64_:%[0-9]+]]:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY3]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:areg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %4:vreg_128_align2 = COPY %3
+    %5:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    GLOBAL_STORE_DWORDX2 %0, %5, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
+
+---
+name:  test_rewrite_mfma_copy_from_agpr_vdst_subreg_use
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_from_agpr_vdst_subreg_use
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:areg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY3]].sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:areg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %4:vreg_128_align2 = COPY %3
+    %4.sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4.sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %4, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+...
+
+# A-to-V copy is performed subregister at a time instead.
+---
+name:  test_rewrite_mfma_copy_from_agpr_split_copy
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_from_agpr_split_copy
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0:vreg_64_align2 = COPY [[GLOBAL_LOAD_DWORDX2_]].sub0
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub1:vreg_64_align2 = COPY [[GLOBAL_LOAD_DWORDX2_]].sub1
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_vgprcd_e64_:%[0-9]+]]:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY3]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:areg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    undef %4.sub0:vreg_64_align2 = COPY %3.sub0
+    %4.sub1:vreg_64_align2 = COPY %3.sub1
+    %5:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4, 0, 0, 0, implicit $mode, implicit $exec
+    GLOBAL_STORE_DWORDX2 %0, %5, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
+
+---
+name:  test_rewrite_mfma_copy_from_agpr_copyback
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_from_agpr_copyback
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_64_align2 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[COPY3]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:areg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY4]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:areg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_64_align2 = COPY %3
+    %5:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4, 0, 0, 0, implicit $mode, implicit $exec
+    %6:areg_64_align2 = COPY %5
+    GLOBAL_STORE_DWORDX2 %0, %6, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
+
+# There is a read of the copy from AGPR to VGPR in the dst operand of the MFMA.
+---
+name:  test_rewrite_mfma_copy_from_agpr_vdst_subreg_use_imm_src2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_from_agpr_vdst_subreg_use_imm_src2
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:areg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], 0, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:areg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %4:vreg_128_align2 = COPY %3
+    %4.sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, 0, 0, 0, 0, implicit $mode, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %4, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+...
+
+# Degenerate case. Copy from AGPR to VGPR is dead undef subreg def
+---
+name:  test_rewrite_mfma_copy_from_agpr_undef_vdst_subreg_use_imm_src2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_from_agpr_undef_vdst_subreg_use_imm_src2
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:areg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: dead [[COPY3:%[0-9]+]]:vreg_128_align2 = COPY [[GLOBAL_LOAD_DWORDX4_]]
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_vgprcd_e64_:%[0-9]+]].sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], 0, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:areg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %4:vreg_128_align2 = COPY %3
+    undef %4.sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, 0, 0, 0, 0, implicit $mode, implicit $exec
+    GLOBAL_STORE_DWORDX4 %0, %4, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+...
+
+# Degenerate case. Copy from AGPR to VGPR is dead, but same register
+# is redefined as whole register.
+---
+name:  test_rewrite_mfma_copy_from_agpr_to_vdst_def_imm_src2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_from_agpr_to_vdst_def_imm_src2
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: dead [[COPY3:%[0-9]+]]:vreg_64_align2 = COPY [[GLOBAL_LOAD_DWORDX2_]]
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_vgprcd_e64_:%[0-9]+]]:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], 0, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:areg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_64_align2 = COPY %3
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, 0, 0, 0, 0, implicit $mode, implicit $exec
+    GLOBAL_STORE_DWORDX2 %0, %4, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-phi.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-phi.ll
new file mode 100644
index 0000000000000..cefcd7e0d2651
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-phi.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck %s
+
+target triple = "amdgcn-amd-amdhsa"
+
+define amdgpu_kernel void @test_rewrite_mfma_copy_to_agpr_phi(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, i1 %cond) #0 {
+; CHECK-LABEL: test_rewrite_mfma_copy_to_agpr_phi:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_dword s6, s[4:5], 0x10
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT:    v_lshlrev_b32_e32 v32, 7, v0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_bitcmp0_b32 s6, 0
+; CHECK-NEXT:    s_cbranch_scc0 .LBB0_2
+; CHECK-NEXT:  ; %bb.1: ; %else
+; CHECK-NEXT:    global_load_dwordx4 a[28:31], v32, s[2:3] offset:112
+; CHECK-NEXT:    global_load_dwordx4 a[24:27], v32, s[2:3] offset:96
+; CHECK-NEXT:    global_load_dwordx4 a[20:23], v32, s[2:3] offset:80
+; CHECK-NEXT:    global_load_dwordx4 a[16:19], v32, s[2:3] offset:64
+; CHECK-NEXT:    global_load_dwordx4 a[12:15], v32, s[2:3] offset:48
+; CHECK-NEXT:    global_load_dwordx4 a[8:11], v32, s[2:3] offset:32
+; CHECK-NEXT:    global_load_dwordx4 a[4:7], v32, s[2:3] offset:16
+; CHECK-NEXT:    global_load_dwordx4 a[0:3], v32, s[2:3]
+; CHECK-NEXT:    v_mov_b32_e32 v33, 2.0
+; CHECK-NEXT:    v_mov_b32_e32 v34, 4.0
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 a[0:31], v33, v34, a[0:31]
+; CHECK-NEXT:    ; kill: def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 killed $exec
+; CHECK-NEXT:    s_cbranch_execz .LBB0_3
+; CHECK-NEXT:    s_branch .LBB0_4
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    ; implicit-def: $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31
+; CHECK-NEXT:  .LBB0_3: ; %if
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    global_load_dwordx4 a[28:31], v32, s[0:1] offset:112
+; CHECK-NEXT:    global_load_dwordx4 a[24:27], v32, s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 a[20:23], v32, s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 a[16:19], v32, s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 a[12:15], v32, s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 a[8:11], v32, s[0:1] offset:32
+; CHECK-NEXT:    global_load_dwordx4 a[4:7], v32, s[0:1] offset:16
+; CHECK-NEXT:    global_load_dwordx4 a[0:3], v32, s[0:1]
+; CHECK-NEXT:    v_mov_b32_e32 v32, 2.0
+; CHECK-NEXT:    v_mov_b32_e32 v33, 4.0
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 a[0:31], v32, v33, a[0:31]
+; CHECK-NEXT:    ; kill: def $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 killed $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 killed $exec
+; CHECK-NEXT:  .LBB0_4: ; %endif
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  br i1 %cond, label %if, label %else
+
+if:
+  %gep.0 = getelementptr <32 x float>, ptr addrspace(1) %arg0, i32 %id
+  %in.0 = load <32 x float>, ptr addrspace(1) %gep.0, align 128
+  %mai.0 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float 4.0, <32 x float> %in.0, i32 0, i32 0, i32 0)
+  br label %endif
+
+else:
+  %gep.1 = getelementptr <32 x float>, ptr addrspace(1) %arg1, i32 %id
+  %in.1 = load <32 x float>, ptr addrspace(1) %gep.1, align 128
+  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float 4.0, <32 x float> %in.1, i32 0, i32 0, i32 0)
+  br label %endif
+
+endif:
+  %phi = phi <32 x float> [ %mai.0, %if ], [ %mai.1, %else ]
+  call void asm sideeffect "; use $0", "a"(<32 x float> %phi)
+  ret void
+}
+
+define amdgpu_kernel void @test_rewrite_mfma_copy_to_agpr_phi_loop(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, i32 %n) #0 {
+; CHECK-LABEL: test_rewrite_mfma_copy_to_agpr_phi_loop:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
+; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x10
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT:    v_lshlrev_b32_e32 v32, 7, v0
+; CHECK-NEXT:    s_mov_b32 s1, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 v[28:31], v32, s[2:3] offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[24:27], v32, s[2:3] offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v32, s[2:3] offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v32, s[2:3] offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v32, s[2:3] offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v32, s[2:3] offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v32, s[2:3] offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v32, s[2:3]
+; CHECK-NEXT:    v_mov_b32_e32 v64, 4.0
+; CHECK-NEXT:    v_mov_b32_e32 v65, 2.0
+; CHECK-NEXT:  .LBB1_1: ; %loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    v_mov_b64_e32 v[62:63], v[30:31]
+; CHECK-NEXT:    v_mov_b64_e32 v[60:61], v[28:29]
+; CHECK-NEXT:    v_mov_b64_e32 v[58:59], v[26:27]
+; CHECK-NEXT:    v_mov_b64_e32 v[56:57], v[24:25]
+; CHECK-NEXT:    v_mov_b64_e32 v[54:55], v[22:23]
+; CHECK-NEXT:    v_mov_b64_e32 v[52:53], v[20:21]
+; CHECK-NEXT:    v_mov_b64_e32 v[50:51], v[18:19]
+; CHECK-NEXT:    v_mov_b64_e32 v[48:49], v[16:17]
+; CHECK-NEXT:    v_mov_b64_e32 v[46:47], v[14:15]
+; CHECK-NEXT:    v_mov_b64_e32 v[44:45], v[12:13]
+; CHECK-NEXT:    v_mov_b64_e32 v[42:43], v[10:11]
+; CHECK-NEXT:    v_mov_b64_e32 v[40:41], v[8:9]
+; CHECK-NEXT:    v_mov_b64_e32 v[38:39], v[6:7]
+; CHECK-NEXT:    v_mov_b64_e32 v[36:37], v[4:5]
+; CHECK-NEXT:    v_mov_b64_e32 v[34:35], v[2:3]
+; CHECK-NEXT:    v_mov_b64_e32 v[32:33], v[0:1]
+; CHECK-NEXT:    s_add_i32 s1, s1, 1
+; CHECK-NEXT:    s_cmp_lt_u32 s1, s0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v65, v64, v[32:63]
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v65, v64, v[0:31]
+; CHECK-NEXT:    s_cbranch_scc1 .LBB1_1
+; CHECK-NEXT:  ; %bb.2: ; %endif
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, v32
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, v33
+; CHECK-NEXT:    v_accvgpr_write_b32 a2, v34
+; CHECK-NEXT:    v_accvgpr_write_b32 a3, v35
+; CHECK-NEXT:    v_accvgpr_write_b32 a4, v36
+; CHECK-NEXT:    v_accvgpr_write_b32 a5, v37
+; CHECK-NEXT:    v_accvgpr_write_b32 a6, v38
+; CHECK-NEXT:    v_accvgpr_write_b32 a7, v39
+; CHECK-NEXT:    v_accvgpr_write_b32 a8, v40
+; CHECK-NEXT:    v_accvgpr_write_b32 a9, v41
+; CHECK-NEXT:    v_accvgpr_write_b32 a10, v42
+; CHECK-NEXT:    v_accvgpr_write_b32 a11, v43
+; CHECK-NEXT:    v_accvgpr_write_b32 a12, v44
+; CHECK-NEXT:    v_accvgpr_write_b32 a13, v45
+; CHECK-NEXT:    v_accvgpr_write_b32 a14, v46
+; CHECK-NEXT:    v_accvgpr_write_b32 a15, v47
+; CHECK-NEXT:    v_accvgpr_write_b32 a16, v48
+; CHECK-NEXT:    v_accvgpr_write_b32 a17, v49
+; CHECK-NEXT:    v_accvgpr_write_b32 a18, v50
+; CHECK-NEXT:    v_accvgpr_write_b32 a19, v51
+; CHECK-NEXT:    v_accvgpr_write_b32 a20, v52
+; CHECK-NEXT:    v_accvgpr_write_b32 a21, v53
+; CHECK-NEXT:    v_accvgpr_write_b32 a22, v54
+; CHECK-NEXT:    v_accvgpr_write_b32 a23, v55
+; CHECK-NEXT:    v_accvgpr_write_b32 a24, v56
+; CHECK-NEXT:    v_accvgpr_write_b32 a25, v57
+; CHECK-NEXT:    v_accvgpr_write_b32 a26, v58
+; CHECK-NEXT:    v_accvgpr_write_b32 a27, v59
+; CHECK-NEXT:    v_accvgpr_write_b32 a28, v60
+; CHECK-NEXT:    v_accvgpr_write_b32 a29, v61
+; CHECK-NEXT:    v_accvgpr_write_b32 a30, v62
+; CHECK-NEXT:    v_accvgpr_write_b32 a31, v63
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_endpgm
+entry:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.0 = getelementptr <32 x float>, ptr addrspace(1) %arg0, i32 %id
+  %in.0 = load <32 x float>, ptr addrspace(1) %gep.0, align 128
+  br label %loop
+
+loop:
+  %i.phi = phi i32 [ 0, %entry ], [ %i.inc, %loop ]
+  %phi = phi <32 x float> [ %in.0, %entry ], [ %mai.1, %loop ]
+  %mai.0 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float 4.0, <32 x float> %phi, i32 0, i32 0, i32 0)
+  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float 4.0, <32 x float> %mai.0, i32 0, i32 0, i32 0)
+  %i.inc = add i32 %i.phi, 1
+  %loop.cond = icmp ult i32 %i.inc, %n
+  br i1 %loop.cond, label %loop, label %endif
+
+endif:
+  call void asm sideeffect "; use $0", "a"(<32 x float> %phi)
+  ret void
+}
+
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="1,1" }
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-subreg-insert-extract.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-subreg-insert-extract.mir
new file mode 100644
index 0000000000000..a7877eac7185b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-subreg-insert-extract.mir
@@ -0,0 +1,226 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=greedy,amdgpu-rewrite-agpr-copy-mfma -o - %s | FileCheck %s
+
+# V-to-A copy is a subregister insert
+---
+name:  test_rewrite_mfma_copy_subreg_insert
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_subreg_insert
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0_sub1:areg_128_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub2_sub3:areg_128_align2 = IMPLICIT_DEF
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY3]].sub2_sub3, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    undef %5.sub0_sub1:areg_128_align2 = COPY %4
+    %5.sub2_sub3 = IMPLICIT_DEF
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, %5
+    GLOBAL_STORE_DWORDX4 %0, %5, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX2 %0, %5.sub2_sub3, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+...
+
+# V-to-A copy is a subregister extract
+---
+name:  test_rewrite_mfma_copy_subreg_extract
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_subreg_extract
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr_32 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub0
+    ; CHECK-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    %5:agpr_32 = COPY %4.sub0
+    GLOBAL_STORE_DWORD %0, %5, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+    SI_RETURN
+...
+
+# V-to-A copy is a subregister-to-subregister copy
+---
+name:  test_rewrite_mfma_copy_subreg_insert_extract_same_subreg
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_subreg_insert_extract_same_subreg
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0:areg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub0
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    undef %5.sub0:areg_64_align2 = COPY %4.sub0
+    GLOBAL_STORE_DWORDX2 %0, %5, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
+
+# V-to-A copy is a subregister-to-subregister copy
+---
+name:  test_rewrite_mfma_copy_subreg_insert_extract_different_subreg
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_subreg_insert_extract_different_subreg
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0:areg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub1
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    undef %5.sub0:areg_64_align2 = COPY %4.sub1
+    GLOBAL_STORE_DWORDX2 %0, %5, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
+
+# V-to-A copy is a subregister extract, from a subregister def
+---
+name:  test_rewrite_mfma_copy_subreg_extract_from_subreg_def
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_subreg_extract_from_subreg_def
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: undef [[DEF:%[0-9]+]].sub0:areg_128_align2 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF:%[0-9]+]].sub2_sub3:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr_32 = COPY [[DEF]].sub0
+    ; CHECK-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    undef %4.sub0 = IMPLICIT_DEF
+    %4.sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    %5:agpr_32 = COPY %4.sub0
+    GLOBAL_STORE_DWORD %0, %5, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+    SI_RETURN
+...
+
+# V-to-A copy is a subregister insert from a subregister_def
+---
+name:  test_rewrite_mfma_copy_subreg_insert_from_subreg_def_tuple
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_subreg_insert_from_subreg_def_tuple
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]].sub2_sub3:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0_sub1:areg_128_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub2_sub3
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub2_sub3:areg_128_align2 = IMPLICIT_DEF
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY3]].sub2_sub3, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    undef %4.sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    undef %5.sub0_sub1:areg_128_align2 = COPY %4.sub2_sub3
+    %5.sub2_sub3 = IMPLICIT_DEF
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, %5
+    GLOBAL_STORE_DWORDX4 %0, %5, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX2 %0, %5.sub2_sub3, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+...
+
+# V-to-A copy is a subregister insert of a subregister from a
+# subregister_def
+---
+name:  test_rewrite_mfma_copy_subreg_insert_from_subreg_def_subreg
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_copy_subreg_insert_from_subreg_def_subreg
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_vgprcd_e64_:%[0-9]+]].sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub1:areg_128_align2 = COPY [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]].sub2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]].sub2_sub3:areg_128_align2 = IMPLICIT_DEF
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY3]].sub2_sub3, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    undef %4.sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    undef %5.sub1:areg_128_align2 = COPY %4.sub2
+    %5.sub2_sub3 = IMPLICIT_DEF
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, %5
+    GLOBAL_STORE_DWORDX4 %0, %5, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    GLOBAL_STORE_DWORDX2 %0, %5.sub2_sub3, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-subreg-src2-chain.mir b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-subreg-src2-chain.mir
new file mode 100644
index 0000000000000..e22775eecca24
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr-subreg-src2-chain.mir
@@ -0,0 +1,333 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=greedy,amdgpu-rewrite-agpr-copy-mfma -o - %s | FileCheck %s
+
+---
+name:  test_rewrite_mfma_src2_is_subreg_0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_src2_is_subreg_0
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:areg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX4_]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0_sub1:areg_128_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    undef %5.sub0_sub1:areg_128_align2 = COPY %4
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, %5
+    GLOBAL_STORE_DWORDX4 %0, %5, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+...
+
+---
+name:  test_rewrite_mfma_src2_is_subreg_1
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_src2_is_subreg_1
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:areg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX4_]].sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0_sub1:areg_128_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3.sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    undef %5.sub0_sub1:areg_128_align2 = COPY %4
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, %5
+    GLOBAL_STORE_DWORDX4 %0, %5, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+...
+
+---
+name:  test_rewrite_mfma_src2_is_subreg_chain_mfma_full_def
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_src2_is_subreg_chain_mfma_full_def
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]].sub0_sub1:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: dead %other_use:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub0_sub1
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_1:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F64_4X4X4F64_e64_]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_1]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3735561 /* reguse:AReg_64_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    undef %4.sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    %other_use:vreg_64_align2 = COPY %4.sub0_sub1
+    %5:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    %6:areg_64_align2 = COPY %5
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3735561 /* reguse:AReg_64_Align2 */, %6:areg_64_align2
+    GLOBAL_STORE_DWORDX2 %0, %6, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
+
+---
+name:  test_rewrite_mfma_src2_is_subreg_chain_mfma_sub_def
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_src2_is_subreg_chain_mfma_sub_def
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: dead %other_use:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]]
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_e64_1:%[0-9]+]].sub0_sub1:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F64_4X4X4F64_e64_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_2:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F64_4X4X4F64_e64_1]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_2]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3735561 /* reguse:AReg_64_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    %other_use:vreg_64_align2 = COPY %4
+    undef %5.sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4, 0, 0, 0, implicit $mode, implicit $exec
+    %6:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %5.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    %7:areg_64_align2 = COPY %6
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3735561 /* reguse:AReg_64_Align2 */, %7
+    GLOBAL_STORE_DWORDX2 %0, %7, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+
+...
+
+---
+name:  test_rewrite_mfma_src2_is_subreg_chain_mfma_sub_def_copy_subreg_insert
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_src2_is_subreg_chain_mfma_sub_def_copy_subreg_insert
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_e64_1:%[0-9]+]].sub0_sub1:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F64_4X4X4F64_e64_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: dead %other_use:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_1]].sub0_sub1
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_2:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F64_4X4X4F64_e64_1]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub0_sub1:areg_128_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_2]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    undef %5.sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4, 0, 0, 0, implicit $mode, implicit $exec
+    %other_use:vreg_64_align2 = COPY %5.sub0_sub1
+    %6:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %5.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    undef %8.sub0_sub1:areg_128_align2 = COPY %6
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, %8:areg_128_align2
+    GLOBAL_STORE_DWORDX4 %0, %8, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+
+...
+
+---
+name:  test_rewrite_mfma_src2_is_subreg_chain_mfma_sub_def_copy_subreg_extract
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_src2_is_subreg_chain_mfma_sub_def_copy_subreg_extract
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX2_:%[0-9]+]]:areg_64_align2 = GLOBAL_LOAD_DWORDX2 [[COPY]], 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX2_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: dead %other_use0:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub0
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_e64_1:%[0-9]+]].sub0_sub1:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F64_4X4X4F64_e64_]], 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: dead %other_use1:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_1]].sub0_sub1
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_2:%[0-9]+]]:areg_64_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F64_4X4X4F64_e64_1]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr_32 = COPY [[V_MFMA_F64_4X4X4F64_e64_2]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORD [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_64_align2 = GLOBAL_LOAD_DWORDX2 %0, 0, 0, implicit $exec :: (load (s64), addrspace 1)
+    %4:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3, 0, 0, 0, implicit $mode, implicit $exec
+    %other_use0:vreg_64_align2 = COPY %4.sub0
+    undef %5.sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4, 0, 0, 0, implicit $mode, implicit $exec
+    %other_use1:vreg_64_align2 = COPY %5.sub0_sub1
+    %6:vreg_64_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %5.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    %8:agpr_32 = COPY %6
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 2162697 /* reguse:AGPR_32 */, %8:agpr_32
+    GLOBAL_STORE_DWORD %0, %8, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+    SI_RETURN
+
+...
+
+# Multiple MFMAs in the chain use the same register with different
+# subregisters.
+---
+name:  test_rewrite_mfma_src2_chain_different_subregs_same_reg_full_copy_use
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_src2_chain_different_subregs_same_reg_full_copy_use
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:areg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]].sub2_sub3:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX4_]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]].sub0_sub1:areg_128_align2 = IMPLICIT_DEF
+    ; CHECK-NEXT: dead %other_use0:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub0_sub1
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]].sub0_sub1:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX4_]].sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: dead %other_use1:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub2_sub3
+    ; CHECK-NEXT: dead %other_use2:vreg_64 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub1_sub2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_128_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]]
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX4 [[COPY]], [[COPY3]], 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    undef %4.sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    %4.sub0_sub1 = IMPLICIT_DEF
+    %other_use0:vreg_64_align2 = COPY %4.sub0_sub1
+    %4.sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3.sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    %other_use1:vreg_64_align2 = COPY %4.sub2_sub3
+    %other_use2:vreg_64 = COPY %4.sub1_sub2
+    %6:areg_128_align2 = COPY %4
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 6291465 /* reguse:AReg_128_Align2 */, %6:areg_128_align2
+    GLOBAL_STORE_DWORDX4 %0, %6, 0, 0, implicit $exec :: (store (s128), addrspace 1)
+    SI_RETURN
+...
+
+# Multiple MFMAs in the chain use the same register with different
+# subregisters.
+---
+name:  test_rewrite_mfma_src2_chain_different_subregs_same_reg_extract_use_0
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_src2_chain_different_subregs_same_reg_extract_use_0
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:areg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]].sub2_sub3:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX4_]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]].sub0_sub1:areg_128_align2 = IMPLICIT_DEF
+    ; CHECK-NEXT: dead %other_use0:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub0_sub1
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_e64_:%[0-9]+]].sub0_sub1:areg_128_align2 = V_MFMA_F64_4X4X4F64_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX4_]].sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: %other_use1:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub2_sub3
+    ; CHECK-NEXT: dead %other_use2:vreg_64 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub1_sub2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_64 = COPY [[V_MFMA_F64_4X4X4F64_e64_]].sub1_sub2
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3473417 /* reguse:AReg_64 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], %other_use1, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    undef %4.sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    %4.sub0_sub1 = IMPLICIT_DEF
+    %other_use0:vreg_64_align2 = COPY %4.sub0_sub1
+    %4.sub0_sub1:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3.sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    %other_use1:vreg_64_align2 = COPY %4.sub2_sub3
+    %other_use2:vreg_64 = COPY %4.sub1_sub2
+    %6:areg_64 = COPY %4.sub1_sub2
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3473417 /* reguse:AReg_64 */, %6:areg_64
+    GLOBAL_STORE_DWORDX2 %0, %other_use1, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
+
+---
+name:  test_rewrite_mfma_src2_chain_different_subregs_same_reg_extract_use_1
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    ; CHECK-LABEL: name: test_rewrite_mfma_src2_chain_different_subregs_same_reg_extract_use_1
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]]:vreg_128_align2 = GLOBAL_LOAD_DWORDX4 [[COPY]], 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    ; CHECK-NEXT: undef [[V_MFMA_F64_4X4X4F64_vgprcd_e64_:%[0-9]+]].sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], [[GLOBAL_LOAD_DWORDX4_]].sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: [[V_MFMA_F64_4X4X4F64_vgprcd_e64_:%[0-9]+]].sub0_sub1:vreg_128_align2 = IMPLICIT_DEF
+    ; CHECK-NEXT: dead [[GLOBAL_LOAD_DWORDX4_:%[0-9]+]].sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]].sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: dead %other_use0:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]].sub0_sub1
+    ; CHECK-NEXT: %other_use1:vreg_64_align2 = COPY [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]].sub2_sub3
+    ; CHECK-NEXT: dead %other_use2:vreg_64 = COPY [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]].sub1_sub2
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:areg_64 = COPY [[V_MFMA_F64_4X4X4F64_vgprcd_e64_]].sub1_sub2
+    ; CHECK-NEXT: INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3473417 /* reguse:AReg_64 */, [[COPY3]]
+    ; CHECK-NEXT: GLOBAL_STORE_DWORDX2 [[COPY]], %other_use1, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    ; CHECK-NEXT: SI_RETURN
+    %0:vreg_64_align2 = COPY $vgpr4_vgpr5
+    %1:av_64_align2 = COPY $vgpr0_vgpr1
+    %2:av_64_align2 = COPY $vgpr2_vgpr3
+    %3:vreg_128_align2 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, implicit $exec :: (load (s128), addrspace 1)
+    undef %4.sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %3.sub0_sub1, 0, 0, 0, implicit $mode, implicit $exec
+    %4.sub0_sub1 = IMPLICIT_DEF
+    %3.sub2_sub3:vreg_128_align2 = V_MFMA_F64_4X4X4F64_vgprcd_e64 %1, %2, %4.sub2_sub3, 0, 0, 0, implicit $mode, implicit $exec
+    %other_use0:vreg_64_align2 = COPY %4.sub0_sub1
+    %other_use1:vreg_64_align2 = COPY %4.sub2_sub3
+    %other_use2:vreg_64 = COPY %4.sub1_sub2
+    %6:areg_64 = COPY %4.sub1_sub2
+    INLINEASM &"; use $0", 1 /* sideeffect attdialect */, 3473417 /* reguse:AReg_64 */, %6:areg_64
+    GLOBAL_STORE_DWORDX2 %0, %other_use1, 0, 0, implicit $exec :: (store (s64), addrspace 1)
+    SI_RETURN
+...
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
index 0c6339e4f5121..81613f69c982b 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mcpu=gfx90a < %s | FileCheck %s
+; RUN: llc -mcpu=gfx942 -amdgpu-mfma-vgpr-form < %s | FileCheck %s
 
 target triple = "amdgcn-amd-amdhsa"
 
@@ -7,7 +7,10 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma(ptr addrsp
 ; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma:
 ; CHECK:       ; %bb.0: ; %bb
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; CHECK-NEXT:    v_mov_b32_e32 v32, 1.0
+; CHECK-NEXT:    v_mov_b32_e32 v33, 2.0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    global_load_dwordx4 v[28:31], v0, s[0:1] offset:112
 ; CHECK-NEXT:    global_load_dwordx4 v[24:27], v0, s[0:1] offset:96
@@ -18,126 +21,67 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma(ptr addrsp
 ; CHECK-NEXT:    global_load_dwordx4 v[4:7], v0, s[0:1] offset:16
 ; CHECK-NEXT:    s_nop 0
 ; CHECK-NEXT:    global_load_dwordx4 v[0:3], v0, s[0:1]
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, 1.0
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, 2.0
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
-; CHECK-NEXT:    v_accvgpr_write_b32 a0, v0
-; CHECK-NEXT:    v_accvgpr_write_b32 a1, v1
-; CHECK-NEXT:    v_accvgpr_write_b32 a2, v2
-; CHECK-NEXT:    v_accvgpr_write_b32 a3, v3
-; CHECK-NEXT:    v_accvgpr_write_b32 a4, v4
-; CHECK-NEXT:    v_accvgpr_write_b32 a5, v5
-; CHECK-NEXT:    v_accvgpr_write_b32 a6, v6
-; CHECK-NEXT:    v_accvgpr_write_b32 a7, v7
-; CHECK-NEXT:    v_accvgpr_write_b32 a8, v8
-; CHECK-NEXT:    v_accvgpr_write_b32 a9, v9
-; CHECK-NEXT:    v_accvgpr_write_b32 a10, v10
-; CHECK-NEXT:    v_accvgpr_write_b32 a11, v11
-; CHECK-NEXT:    v_accvgpr_write_b32 a12, v12
-; CHECK-NEXT:    v_accvgpr_write_b32 a13, v13
-; CHECK-NEXT:    v_accvgpr_write_b32 a14, v14
-; CHECK-NEXT:    v_accvgpr_write_b32 a15, v15
-; CHECK-NEXT:    v_accvgpr_write_b32 a16, v16
-; CHECK-NEXT:    v_accvgpr_write_b32 a17, v17
-; CHECK-NEXT:    v_accvgpr_write_b32 a18, v18
-; CHECK-NEXT:    v_accvgpr_write_b32 a19, v19
-; CHECK-NEXT:    v_accvgpr_write_b32 a20, v20
-; CHECK-NEXT:    v_accvgpr_write_b32 a21, v21
-; CHECK-NEXT:    v_accvgpr_write_b32 a22, v22
-; CHECK-NEXT:    v_accvgpr_write_b32 a23, v23
-; CHECK-NEXT:    v_accvgpr_write_b32 a24, v24
-; CHECK-NEXT:    v_accvgpr_write_b32 a25, v25
-; CHECK-NEXT:    v_accvgpr_write_b32 a26, v26
-; CHECK-NEXT:    v_accvgpr_write_b32 a27, v27
-; CHECK-NEXT:    v_accvgpr_write_b32 a28, v28
-; CHECK-NEXT:    v_accvgpr_write_b32 a29, v29
-; CHECK-NEXT:    v_accvgpr_write_b32 a30, v30
-; CHECK-NEXT:    v_accvgpr_write_b32 a31, v31
-; CHECK-NEXT:    v_mov_b32_e32 v0, 1.0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2.0
-; CHECK-NEXT:    s_nop 1
-; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
-; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[32:63], v0, v1, a[0:31]
-; CHECK-NEXT:    s_nop 7
-; CHECK-NEXT:    s_nop 7
-; CHECK-NEXT:    s_nop 2
-; CHECK-NEXT:    v_accvgpr_read_b32 v4, a59
-; CHECK-NEXT:    v_accvgpr_read_b32 v5, a58
-; CHECK-NEXT:    v_accvgpr_read_b32 v6, a57
-; CHECK-NEXT:    v_accvgpr_read_b32 v7, a56
-; CHECK-NEXT:    v_accvgpr_read_b32 v8, a55
-; CHECK-NEXT:    v_accvgpr_read_b32 v9, a54
-; CHECK-NEXT:    v_accvgpr_read_b32 v10, a53
-; CHECK-NEXT:    v_accvgpr_read_b32 v11, a52
-; CHECK-NEXT:    v_accvgpr_read_b32 v12, a51
-; CHECK-NEXT:    v_accvgpr_read_b32 v13, a50
-; CHECK-NEXT:    v_accvgpr_read_b32 v14, a49
-; CHECK-NEXT:    v_accvgpr_read_b32 v15, a48
-; CHECK-NEXT:    v_accvgpr_read_b32 v16, a47
-; CHECK-NEXT:    v_accvgpr_read_b32 v17, a46
-; CHECK-NEXT:    v_accvgpr_read_b32 v18, a45
-; CHECK-NEXT:    v_accvgpr_read_b32 v19, a44
-; CHECK-NEXT:    v_accvgpr_read_b32 v20, a43
-; CHECK-NEXT:    v_accvgpr_read_b32 v21, a42
-; CHECK-NEXT:    v_accvgpr_read_b32 v22, a41
-; CHECK-NEXT:    v_accvgpr_read_b32 v23, a40
-; CHECK-NEXT:    v_accvgpr_read_b32 v24, a39
-; CHECK-NEXT:    v_accvgpr_read_b32 v25, a38
-; CHECK-NEXT:    v_accvgpr_read_b32 v26, a37
-; CHECK-NEXT:    v_accvgpr_read_b32 v27, a36
-; CHECK-NEXT:    v_accvgpr_read_b32 v28, a35
-; CHECK-NEXT:    v_accvgpr_read_b32 v29, a34
-; CHECK-NEXT:    v_accvgpr_mov_b32 a2, a32
-; CHECK-NEXT:    v_accvgpr_mov_b32 a3, a33
-; CHECK-NEXT:    v_accvgpr_write_b32 a4, v29
-; CHECK-NEXT:    v_accvgpr_write_b32 a5, v28
-; CHECK-NEXT:    v_accvgpr_write_b32 a6, v27
-; CHECK-NEXT:    v_accvgpr_write_b32 a7, v26
-; CHECK-NEXT:    v_accvgpr_write_b32 a8, v25
-; CHECK-NEXT:    v_accvgpr_write_b32 a9, v24
-; CHECK-NEXT:    v_accvgpr_write_b32 a10, v23
-; CHECK-NEXT:    v_accvgpr_write_b32 a11, v22
-; CHECK-NEXT:    v_accvgpr_write_b32 a12, v21
-; CHECK-NEXT:    v_accvgpr_write_b32 a13, v20
-; CHECK-NEXT:    v_accvgpr_write_b32 a14, v19
-; CHECK-NEXT:    v_accvgpr_write_b32 a15, v18
-; CHECK-NEXT:    v_accvgpr_write_b32 a16, v17
-; CHECK-NEXT:    v_accvgpr_write_b32 a17, v16
-; CHECK-NEXT:    v_accvgpr_write_b32 a18, v15
-; CHECK-NEXT:    v_accvgpr_write_b32 a19, v14
-; CHECK-NEXT:    v_accvgpr_write_b32 a20, v13
-; CHECK-NEXT:    v_accvgpr_write_b32 a21, v12
-; CHECK-NEXT:    v_accvgpr_write_b32 a22, v11
-; CHECK-NEXT:    v_accvgpr_write_b32 a23, v10
-; CHECK-NEXT:    v_accvgpr_write_b32 a24, v9
-; CHECK-NEXT:    v_accvgpr_write_b32 a25, v8
-; CHECK-NEXT:    v_accvgpr_write_b32 a26, v7
-; CHECK-NEXT:    v_accvgpr_write_b32 a27, v6
-; CHECK-NEXT:    v_accvgpr_write_b32 a28, v5
-; CHECK-NEXT:    v_accvgpr_write_b32 a29, v4
-; CHECK-NEXT:    v_accvgpr_mov_b32 a30, a60
-; CHECK-NEXT:    v_accvgpr_mov_b32 a31, a61
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[32:63], a0, a1, v[0:31]
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
 ; CHECK-NEXT:    s_nop 1
-; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v2, v32
+; CHECK-NEXT:    v_mov_b32_e32 v3, v33
+; CHECK-NEXT:    v_mov_b32_e32 v4, v34
+; CHECK-NEXT:    v_mov_b32_e32 v5, v35
+; CHECK-NEXT:    v_mov_b32_e32 v6, v36
+; CHECK-NEXT:    v_mov_b32_e32 v7, v37
+; CHECK-NEXT:    v_mov_b32_e32 v8, v38
+; CHECK-NEXT:    v_mov_b32_e32 v9, v39
+; CHECK-NEXT:    v_mov_b32_e32 v10, v40
+; CHECK-NEXT:    v_mov_b32_e32 v11, v41
+; CHECK-NEXT:    v_mov_b32_e32 v12, v42
+; CHECK-NEXT:    v_mov_b32_e32 v13, v43
+; CHECK-NEXT:    v_mov_b32_e32 v14, v44
+; CHECK-NEXT:    v_mov_b32_e32 v15, v45
+; CHECK-NEXT:    v_mov_b32_e32 v16, v46
+; CHECK-NEXT:    v_mov_b32_e32 v17, v47
+; CHECK-NEXT:    v_mov_b32_e32 v18, v48
+; CHECK-NEXT:    v_mov_b32_e32 v19, v49
+; CHECK-NEXT:    v_mov_b32_e32 v20, v50
+; CHECK-NEXT:    v_mov_b32_e32 v21, v51
+; CHECK-NEXT:    v_mov_b32_e32 v22, v52
+; CHECK-NEXT:    v_mov_b32_e32 v23, v53
+; CHECK-NEXT:    v_mov_b32_e32 v24, v54
+; CHECK-NEXT:    v_mov_b32_e32 v25, v55
+; CHECK-NEXT:    v_mov_b32_e32 v26, v56
+; CHECK-NEXT:    v_mov_b32_e32 v27, v57
+; CHECK-NEXT:    v_mov_b32_e32 v28, v58
+; CHECK-NEXT:    v_mov_b32_e32 v29, v59
+; CHECK-NEXT:    v_mov_b32_e32 v30, v60
+; CHECK-NEXT:    v_mov_b32_e32 v31, v61
+; CHECK-NEXT:    v_mov_b32_e32 v32, 0
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], a0, a1, v[0:31]
 ; CHECK-NEXT:    s_nop 7
 ; CHECK-NEXT:    s_nop 7
 ; CHECK-NEXT:    s_nop 1
-; CHECK-NEXT:    global_store_dwordx4 v0, a[24:27], s[0:1] offset:96
-; CHECK-NEXT:    global_store_dwordx4 v0, a[28:31], s[0:1] offset:112
-; CHECK-NEXT:    global_store_dwordx4 v0, a[16:19], s[0:1] offset:64
-; CHECK-NEXT:    global_store_dwordx4 v0, a[20:23], s[0:1] offset:80
-; CHECK-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
-; CHECK-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
-; CHECK-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
-; CHECK-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
+; CHECK-NEXT:    global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; CHECK-NEXT:    global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; CHECK-NEXT:    global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; CHECK-NEXT:    global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; CHECK-NEXT:    global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v32, v[0:3], s[0:1]
+; CHECK-NEXT:    global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
 ; CHECK-NEXT:    s_endpgm
 bb:
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id
   %in.1 = load <32 x float>, ptr addrspace(1) %gep, align 128
-  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0)
-  %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.1, i32 0, i32 0, i32 0)
+  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.1, i32 0, i32 0, i32 0)
+  %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %mai.1, i32 0, i32 0, i32 0)
   %tmp.1 = shufflevector <32 x float> %mai.2, <32 x float> %mai.1, <32 x i32> <i32 32, i32 33, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29>
-  %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %tmp.1, i32 0, i32 0, i32 0)
+  %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %tmp.1, i32 0, i32 0, i32 0)
   store <32 x float> %mai.3, ptr addrspace(1) %arg, align 128
   ret void
 }
@@ -146,8 +90,127 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_noshuffle(
 ; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_noshuffle:
 ; CHECK:       ; %bb.0: ; %bb
 ; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2.0
+; CHECK-NEXT:    v_mov_b32_e32 v32, 1.0
+; CHECK-NEXT:    v_mov_b32_e32 v33, 2.0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 v[28:31], v0, s[0:1] offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[24:27], v0, s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v0, s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v0, s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v0, s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v0, s[0:1] offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v0, s[0:1] offset:16
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v0, s[0:1]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; CHECK-NEXT:    v_mov_b32_e32 v32, 0
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; CHECK-NEXT:    global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; CHECK-NEXT:    global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; CHECK-NEXT:    global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; CHECK-NEXT:    global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v32, v[0:3], s[0:1]
+; CHECK-NEXT:    global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id
+  %in.1 = load <32 x float>, ptr addrspace(1) %gep, align 128
+  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %in.1, i32 0, i32 0, i32 0)
+  %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %mai.1, i32 0, i32 0, i32 0)
+  %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %mai.2, i32 0, i32 0, i32 0)
+  store <32 x float> %mai.3, ptr addrspace(1) %arg, align 128
+  ret void
+}
+
+define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_imm0_src2(ptr addrspace(1) %arg) #0 {
+; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_imm0_src2:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    v_mov_b32_e32 v32, 1.0
+; CHECK-NEXT:    v_mov_b32_e32 v33, 2.0
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; CHECK-NEXT:    v_mov_b32_e32 v32, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; CHECK-NEXT:    global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; CHECK-NEXT:    global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; CHECK-NEXT:    global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; CHECK-NEXT:    global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; CHECK-NEXT:    global_store_dwordx4 v32, v[0:3], s[0:1]
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id
+  %in.1 = load <32 x float>, ptr addrspace(1) %gep, align 128
+  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> zeroinitializer, i32 0, i32 0, i32 0)
+  %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %mai.1, i32 0, i32 0, i32 0)
+  %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %mai.2, i32 0, i32 0, i32 0)
+  store <32 x float> %mai.3, ptr addrspace(1) %arg, align 128
+  ret void
+}
+
+define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_imm1_src2(ptr addrspace(1) %arg) #0 {
+; CHECK-LABEL: test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_imm1_src2:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    v_mov_b32_e32 v32, 1.0
+; CHECK-NEXT:    v_mov_b32_e32 v33, 2.0
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, 1.0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v32, v33, v[0:31]
+; CHECK-NEXT:    v_mov_b32_e32 v32, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; CHECK-NEXT:    global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; CHECK-NEXT:    global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; CHECK-NEXT:    global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; CHECK-NEXT:    global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; CHECK-NEXT:    global_store_dwordx4 v32, v[0:3], s[0:1]
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id
+  %in.1 = load <32 x float>, ptr addrspace(1) %gep, align 128
+  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> splat (float 1.000000e+00), i32 0, i32 0, i32 0)
+  %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %mai.1, i32 0, i32 0, i32 0)
+  %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.000000e+00, float 2.000000e+00, <32 x float> %mai.2, i32 0, i32 0, i32 0)
+  store <32 x float> %mai.3, ptr addrspace(1) %arg, align 128
+  ret void
+}
+
+; The inline asm requires the value be copied to an AGPR class, not
+; the AV_* pseudo we usually expect for register allocator live range
+; splits.
+define amdgpu_kernel void @test_rewrite_mfma_direct_copy_to_agpr_class(ptr addrspace(1) %arg) #0 {
+; CHECK-LABEL: test_rewrite_mfma_direct_copy_to_agpr_class:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; CHECK-NEXT:    v_mov_b32_e32 v32, 2.0
+; CHECK-NEXT:    v_mov_b32_e32 v33, 4.0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    global_load_dwordx4 a[28:31], v0, s[0:1] offset:112
 ; CHECK-NEXT:    global_load_dwordx4 a[24:27], v0, s[0:1] offset:96
@@ -157,39 +220,700 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_rewrite_vgpr_mfma_noshuffle(
 ; CHECK-NEXT:    global_load_dwordx4 a[8:11], v0, s[0:1] offset:32
 ; CHECK-NEXT:    global_load_dwordx4 a[4:7], v0, s[0:1] offset:16
 ; CHECK-NEXT:    global_load_dwordx4 a[0:3], v0, s[0:1]
-; CHECK-NEXT:    v_mov_b32_e32 v0, 1.0
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 a[0:31], v32, v33, a[0:31]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_endpgm
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id
+  %in = load <32 x float>, ptr addrspace(1) %gep, align 128
+  %mai = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float 4.0, <32 x float> %in, i32 0, i32 0, i32 0)
+  call void asm sideeffect "; use $0", "a"(<32 x float> %mai)
+  ret void
+}
+
+define void @test_rewrite_mfma_imm_src2(float %arg0, float %arg1) #0 {
+; CHECK-LABEL: test_rewrite_mfma_imm_src2:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, 2.0
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+bb:
+  %mai = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %arg0, float %arg1, <32 x float> splat (float 2.0), i32 0, i32 0, i32 0)
+  call void asm sideeffect "; use $0", "a"(<32 x float> %mai)
+  ret void
+}
+
+define void @test_rewrite_mfma_subreg_extract0(float %arg0, float %arg1, ptr addrspace(1) %ptr) #0 {
+; CHECK-LABEL: test_rewrite_mfma_subreg_extract0:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 a[28:31], v[2:3], off offset:112
+; CHECK-NEXT:    global_load_dwordx4 a[24:27], v[2:3], off offset:96
+; CHECK-NEXT:    global_load_dwordx4 a[20:23], v[2:3], off offset:80
+; CHECK-NEXT:    global_load_dwordx4 a[16:19], v[2:3], off offset:64
+; CHECK-NEXT:    global_load_dwordx4 a[12:15], v[2:3], off offset:48
+; CHECK-NEXT:    global_load_dwordx4 a[8:11], v[2:3], off offset:32
+; CHECK-NEXT:    global_load_dwordx4 a[4:7], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 a[0:3], v[2:3], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:3]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+bb:
+  %src2 = load <32 x float>, ptr addrspace(1) %ptr
+  %mai = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %arg0, float %arg1, <32 x float> %src2, i32 0, i32 0, i32 0)
+  %extract.sub4 = shufflevector <32 x float> %mai, <32 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void asm sideeffect "; use $0", "a"(<4 x float> %extract.sub4)
+  ret void
+}
+
+define void @test_rewrite_mfma_subreg_extract1(float %arg0, float %arg1, ptr addrspace(1) %ptr) #0 {
+; CHECK-LABEL: test_rewrite_mfma_subreg_extract1:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 a[28:31], v[2:3], off offset:112
+; CHECK-NEXT:    global_load_dwordx4 a[24:27], v[2:3], off offset:96
+; CHECK-NEXT:    global_load_dwordx4 a[20:23], v[2:3], off offset:80
+; CHECK-NEXT:    global_load_dwordx4 a[16:19], v[2:3], off offset:64
+; CHECK-NEXT:    global_load_dwordx4 a[12:15], v[2:3], off offset:48
+; CHECK-NEXT:    global_load_dwordx4 a[8:11], v[2:3], off offset:32
+; CHECK-NEXT:    global_load_dwordx4 a[4:7], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 a[0:3], v[2:3], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[4:7]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+bb:
+  %src2 = load <32 x float>, ptr addrspace(1) %ptr
+  %mai = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %arg0, float %arg1, <32 x float> %src2, i32 0, i32 0, i32 0)
+  %extract.sub4 = shufflevector <32 x float> %mai, <32 x float> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  call void asm sideeffect "; use $0", "a"(<4 x float> %extract.sub4)
+  ret void
+}
+
+; odd offset
+define void @test_rewrite_mfma_subreg_extract2(float %arg0, float %arg1, ptr addrspace(1) %ptr) #0 {
+; CHECK-LABEL: test_rewrite_mfma_subreg_extract2:
+; CHECK:       ; %bb.0: ; %bb
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 a[28:31], v[2:3], off offset:112
+; CHECK-NEXT:    global_load_dwordx4 a[24:27], v[2:3], off offset:96
+; CHECK-NEXT:    global_load_dwordx4 a[20:23], v[2:3], off offset:80
+; CHECK-NEXT:    global_load_dwordx4 a[16:19], v[2:3], off offset:64
+; CHECK-NEXT:    global_load_dwordx4 a[12:15], v[2:3], off offset:48
+; CHECK-NEXT:    global_load_dwordx4 a[8:11], v[2:3], off offset:32
+; CHECK-NEXT:    global_load_dwordx4 a[4:7], v[2:3], off offset:16
+; CHECK-NEXT:    global_load_dwordx4 a[0:3], v[2:3], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 a[0:31], v0, v1, a[0:31]
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    v_accvgpr_mov_b32 a0, a1
+; CHECK-NEXT:    v_accvgpr_mov_b32 a1, a2
+; CHECK-NEXT:    v_accvgpr_mov_b32 a2, a3
+; CHECK-NEXT:    v_accvgpr_mov_b32 a3, a4
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:3]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+bb:
+  %src2 = load <32 x float>, ptr addrspace(1) %ptr
+  %mai = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float %arg0, float %arg1, <32 x float> %src2, i32 0, i32 0, i32 0)
+  %extract.sub4 = shufflevector <32 x float> %mai, <32 x float> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+  call void asm sideeffect "; use $0", "a"(<4 x float> %extract.sub4)
+  ret void
+}
+
+define amdgpu_kernel void @illegal_mfma_after_rewrite() #1 {
+; CHECK-LABEL: illegal_mfma_after_rewrite:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_mov_b32 s0, 0
+; CHECK-NEXT:    s_mov_b32 s1, s0
+; CHECK-NEXT:    v_mov_b64_e32 v[8:9], s[0:1]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def s[0:3]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mov_b64_e32 v[6:7], s[2:3]
+; CHECK-NEXT:    v_mov_b64_e32 v[4:5], s[0:1]
+; CHECK-NEXT:    s_mov_b32 s0, 0x3c003c00
+; CHECK-NEXT:    s_mov_b32 s1, s0
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[0:3], v[8:9], v[8:9], v[4:7]
+; CHECK-NEXT:    v_mov_b64_e32 v[12:13], s[0:1]
+; CHECK-NEXT:    s_mov_b32 s0, 0x7e007e00
+; CHECK-NEXT:    s_mov_b32 s1, s0
+; CHECK-NEXT:    v_mov_b64_e32 v[10:11], s[0:1]
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[14:17], v[8:9], v[12:13], v[4:7]
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, v0
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[18:21], v[8:9], v[10:11], v[4:7]
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, v1
+; CHECK-NEXT:    v_accvgpr_write_b32 a2, v2
+; CHECK-NEXT:    v_accvgpr_write_b32 a3, v3
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0x7fc00000
+; CHECK-NEXT:    v_mov_b32_e32 v5, v4
+; CHECK-NEXT:    v_mov_b32_e32 v6, v4
+; CHECK-NEXT:    v_mov_b32_e32 v7, v4
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[14:17], v[8:9], v[8:9], v[14:17]
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[22:25], v[8:9], v[8:9], v[4:7]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def v[4:7]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[0:3], v[8:9], v[12:13], v[4:7]
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[26:29], v[8:9], v[8:9], v[4:7]
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[0:3], v[8:9], v[8:9], v[0:3]
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[22:25], v[8:9], v[8:9], v[22:25]
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[4:7], v[8:9], v[8:9], v[26:29]
+; CHECK-NEXT:    s_nop 5
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v23, v14
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[14:17], v[8:9], v[8:9], v[18:21]
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[0:3], v[12:13], v[8:9], v[0:3]
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    v_accvgpr_read_b32 v19, a3
+; CHECK-NEXT:    v_accvgpr_read_b32 v18, a2
+; CHECK-NEXT:    v_mov_b64_e32 v[20:21], 0
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_accvgpr_read_b32 v17, a1
+; CHECK-NEXT:    v_accvgpr_read_b32 v16, a0
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v15, v22
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v14, v14
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[16:19], v[8:9], v[8:9], v[16:19]
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v12, v0
+; CHECK-NEXT:    global_store_short v[20:21], v23, off
+; CHECK-NEXT:    buffer_wbl2 sc0 sc1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_inv sc0 sc1
+; CHECK-NEXT:    v_mfma_f32_16x16x16_f16 v[0:3], v[10:11], v[8:9], v[4:7]
+; CHECK-NEXT:    global_store_short v[20:21], v15, off
+; CHECK-NEXT:    buffer_wbl2 sc0 sc1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_inv sc0 sc1
+; CHECK-NEXT:    global_store_short v[20:21], v14, off
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v14, v16
+; CHECK-NEXT:    buffer_wbl2 sc0 sc1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_inv sc0 sc1
+; CHECK-NEXT:    global_store_short v[20:21], v14, off
+; CHECK-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; CHECK-NEXT:    buffer_wbl2 sc0 sc1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_inv sc0 sc1
+; CHECK-NEXT:    global_store_short v[20:21], v12, off
+; CHECK-NEXT:    buffer_wbl2 sc0 sc1
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    buffer_inv sc0 sc1
+; CHECK-NEXT:    global_store_short v[20:21], v0, off
+; CHECK-NEXT:    s_endpgm
+entry:
+  %k0 = call <4 x float> asm sideeffect "; def $0", "=s"()
+  %i2 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <4 x float> %k0, i32 0, i32 0, i32 0)
+  %i4 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> splat (half 0xH3C00), <4 x float> %k0, i32 0, i32 0, i32 0)
+  %i6 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> splat (half 0xH7E00), <4 x float> %k0, i32 0, i32 0, i32 0)
+  %i5 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <4 x float> splat (float 0x7FF8000000000000), i32 0, i32 0, i32 0)
+  %k = call <4 x float> asm sideeffect "; def $0", "=v"()
+  %i1 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <4 x float> %k, i32 0, i32 0, i32 0)
+  %i7 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> splat (half 0xH3C00), <4 x float> %k, i32 0, i32 0, i32 0)
+  %i17 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <4 x float> %i1, i32 0, i32 0, i32 0)
+  %i19 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <4 x float> %i4, i32 0, i32 0, i32 0)
+  %c_thread_buf.0 = extractelement <4 x float> %i19, i64 0
+  %conv.0 = fptrunc float %c_thread_buf.0 to half
+  store half %conv.0, ptr addrspace(1) null, align 2
+  fence seq_cst
+  %i22 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <4 x float> %i5, i32 0, i32 0, i32 0)
+  %c_thread_buf.1 = extractelement <4 x float> %i22, i64 0
+  %conv1 = fptrunc float %c_thread_buf.1 to half
+  store half %conv1, ptr addrspace(1) null, align 2
+  fence seq_cst
+  %i23 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <4 x float> %i6, i32 0, i32 0, i32 0)
+  %c_thread_buf.2 = extractelement <4 x float> %i23, i64 0
+  %conv2 = fptrunc float %c_thread_buf.2 to half
+  store half %conv2, ptr addrspace(1) null, align 2
+  fence seq_cst
+  %i25 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <4 x float> %i2, i32 0, i32 0, i32 0)
+  %c_thread_buf.3 = extractelement <4 x float> %i25, i64 0
+  %conv3 = fptrunc float %c_thread_buf.3 to half
+  store half %conv3, ptr addrspace(1) null, align 2
+  fence seq_cst
+  %i26 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> zeroinitializer, <4 x half> zeroinitializer, <4 x float> %i7, i32 0, i32 0, i32 0)
+  %i27 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> splat (half 0xH3C00), <4 x half> zeroinitializer, <4 x float> %i26, i32 0, i32 0, i32 0)
+  %c_thread_buf.4 = extractelement <4 x float> %i27, i64 0
+  %conv4 = fptrunc float %c_thread_buf.4 to half
+  store half %conv4, ptr addrspace(1) null, align 2
+  fence seq_cst
+  %i31 = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half> splat (half 0xH7E00), <4 x half> zeroinitializer, <4 x float> %i17, i32 0, i32 0, i32 0)
+  %c_thread_buf.5 = extractelement <4 x float> %i31, i64 0
+  %conv5 = fptrunc float %c_thread_buf.5 to half
+  store half %conv5, ptr addrspace(1) null, align 2
+  ret void
+}
+
+define void @test_rewrite_mfma_subreg_insert0(float %arg0, float %arg1, ptr addrspace(1) %ptr) #0 {
+; CHECK-LABEL: test_rewrite_mfma_subreg_insert0:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 a[0:3], v[2:3], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_mfma_f32_4x4x1_16b_f32 a[0:3], v0, v1, a[0:3]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:7]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %src2 = load <4 x float>, ptr addrspace(1) %ptr
+  %mai = call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float %arg0, float %arg1, <4 x float> %src2, i32 0, i32 0, i32 0)
+  %insert.sub0 = shufflevector <4 x float> %mai, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+  call void asm sideeffect "; use $0", "a"(<8 x float> %insert.sub0)
+  ret void
+}
+
+; odd offset
+define void @test_rewrite_mfma_subreg_insert1(float %arg0, float %arg1, ptr addrspace(1) %ptr) #0 {
+; CHECK-LABEL: test_rewrite_mfma_subreg_insert1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 v[2:5], v[2:3], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_mfma_f32_4x4x1_16b_f32 v[0:3], v0, v1, v[2:5]
+; CHECK-NEXT:    s_nop 3
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, v0
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, v1
+; CHECK-NEXT:    v_accvgpr_write_b32 a2, v3
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:7]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %src2 = load <4 x float>, ptr addrspace(1) %ptr
+  %mai = call <4 x float> @llvm.amdgcn.mfma.f32.4x4x1f32(float %arg0, float %arg1, <4 x float> %src2, i32 0, i32 0, i32 0)
+  %insert.sub0 = shufflevector <4 x float> %mai, <4 x float> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 poison, i32 poison, i32 poison, i32 poison>
+  call void asm sideeffect "; use $0", "a"(<8 x float> %insert.sub0)
+  ret void
+}
+
+define void @test_rewrite_mfma_subreg_insert2(double %arg0, double %arg1, ptr addrspace(1) %ptr) #0 {
+; CHECK-LABEL: test_rewrite_mfma_subreg_insert2:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx2 a[0:1], v[4:5], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    v_mfma_f64_4x4x4_4b_f64 a[0:1], v[0:1], v[2:3], a[0:1]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:3]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %src2 = load double, ptr addrspace(1) %ptr
+  %mai = call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %arg0, double %arg1, double %src2, i32 0, i32 0, i32 0)
+  %insert.sub0 = insertelement <2 x double> poison, double %mai, i32 0
+  call void asm sideeffect "; use $0", "a"(<2 x double> %insert.sub0)
+  ret void
+}
+
+define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) #0 {
+; CHECK-LABEL: test_rewrite_mfma_direct_copy_from_agpr_class:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_accvgpr_write_b32 a32, v0
+; CHECK-NEXT:    v_accvgpr_read_b32 v63, a31
+; CHECK-NEXT:    v_accvgpr_read_b32 v62, a30
+; CHECK-NEXT:    v_accvgpr_read_b32 v61, a29
+; CHECK-NEXT:    v_accvgpr_read_b32 v60, a28
+; CHECK-NEXT:    v_accvgpr_read_b32 v59, a27
+; CHECK-NEXT:    v_accvgpr_read_b32 v58, a26
+; CHECK-NEXT:    v_accvgpr_read_b32 v57, a25
+; CHECK-NEXT:    v_accvgpr_read_b32 v56, a24
+; CHECK-NEXT:    v_accvgpr_read_b32 v55, a23
+; CHECK-NEXT:    v_accvgpr_read_b32 v54, a22
+; CHECK-NEXT:    v_accvgpr_read_b32 v53, a21
+; CHECK-NEXT:    v_accvgpr_read_b32 v52, a20
+; CHECK-NEXT:    v_accvgpr_read_b32 v51, a19
+; CHECK-NEXT:    v_accvgpr_read_b32 v50, a18
+; CHECK-NEXT:    v_accvgpr_read_b32 v49, a17
+; CHECK-NEXT:    v_accvgpr_read_b32 v48, a16
+; CHECK-NEXT:    v_accvgpr_read_b32 v47, a15
+; CHECK-NEXT:    v_accvgpr_read_b32 v46, a14
+; CHECK-NEXT:    v_accvgpr_read_b32 v45, a13
+; CHECK-NEXT:    v_accvgpr_read_b32 v44, a12
+; CHECK-NEXT:    v_accvgpr_read_b32 v43, a11
+; CHECK-NEXT:    v_accvgpr_read_b32 v42, a10
+; CHECK-NEXT:    v_accvgpr_read_b32 v41, a9
+; CHECK-NEXT:    v_accvgpr_read_b32 v40, a8
+; CHECK-NEXT:    v_accvgpr_read_b32 v39, a7
+; CHECK-NEXT:    v_accvgpr_read_b32 v38, a6
+; CHECK-NEXT:    v_accvgpr_read_b32 v37, a5
+; CHECK-NEXT:    v_accvgpr_read_b32 v36, a4
+; CHECK-NEXT:    v_accvgpr_read_b32 v35, a3
+; CHECK-NEXT:    v_accvgpr_read_b32 v34, a2
+; CHECK-NEXT:    v_accvgpr_read_b32 v33, a1
+; CHECK-NEXT:    v_accvgpr_read_b32 v32, a0
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, 2.0
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, 4.0
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], a0, a1, v[32:63]
+; CHECK-NEXT:    v_accvgpr_write_b32 a0, v32
+; CHECK-NEXT:    v_accvgpr_write_b32 a1, v33
+; CHECK-NEXT:    v_accvgpr_write_b32 a2, v34
+; CHECK-NEXT:    v_accvgpr_write_b32 a3, v35
+; CHECK-NEXT:    v_accvgpr_write_b32 a4, v36
+; CHECK-NEXT:    v_accvgpr_write_b32 a5, v37
+; CHECK-NEXT:    v_accvgpr_write_b32 a6, v38
+; CHECK-NEXT:    v_accvgpr_write_b32 a7, v39
+; CHECK-NEXT:    v_accvgpr_write_b32 a8, v40
+; CHECK-NEXT:    v_accvgpr_write_b32 a9, v41
+; CHECK-NEXT:    v_accvgpr_write_b32 a10, v42
+; CHECK-NEXT:    v_accvgpr_write_b32 a11, v43
+; CHECK-NEXT:    v_accvgpr_write_b32 a12, v44
+; CHECK-NEXT:    v_accvgpr_write_b32 a13, v45
+; CHECK-NEXT:    v_accvgpr_write_b32 a14, v46
+; CHECK-NEXT:    v_accvgpr_write_b32 a15, v47
+; CHECK-NEXT:    v_accvgpr_write_b32 a16, v48
+; CHECK-NEXT:    v_accvgpr_write_b32 a17, v49
+; CHECK-NEXT:    v_accvgpr_write_b32 a18, v50
+; CHECK-NEXT:    v_accvgpr_write_b32 a19, v51
+; CHECK-NEXT:    v_accvgpr_write_b32 a20, v52
+; CHECK-NEXT:    v_accvgpr_write_b32 a21, v53
+; CHECK-NEXT:    v_accvgpr_write_b32 a22, v54
+; CHECK-NEXT:    v_accvgpr_write_b32 a23, v55
+; CHECK-NEXT:    v_accvgpr_write_b32 a24, v56
+; CHECK-NEXT:    v_accvgpr_write_b32 a25, v57
+; CHECK-NEXT:    v_accvgpr_write_b32 a26, v58
+; CHECK-NEXT:    v_accvgpr_write_b32 a27, v59
+; CHECK-NEXT:    v_accvgpr_write_b32 a28, v60
+; CHECK-NEXT:    v_accvgpr_write_b32 a29, v61
+; CHECK-NEXT:    v_accvgpr_write_b32 a30, v62
+; CHECK-NEXT:    v_accvgpr_write_b32 a31, v63
+; CHECK-NEXT:    v_accvgpr_read_b32 v32, a32
+; CHECK-NEXT:    v_mov_b32_e32 v33, 0x41000000
+; CHECK-NEXT:    v_and_b32_e32 v32, 0x3ff, v32
+; CHECK-NEXT:    v_lshlrev_b32_e32 v32, 7, v32
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; CHECK-NEXT:    global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
+; CHECK-NEXT:    global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
+; CHECK-NEXT:    global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
+; CHECK-NEXT:    global_store_dwordx4 v32, v[12:15], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v32, v[8:11], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v32, v[4:7], s[0:1] offset:16
+; CHECK-NEXT:    global_store_dwordx4 v32, v[0:3], s[0:1]
+; CHECK-NEXT:    v_mov_b32_e32 v34, 0x41800000
 ; CHECK-NEXT:    s_nop 0
-; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
-; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
-; CHECK-NEXT:    v_mfma_f32_32x32x1f32 a[0:31], v0, v1, a[0:31]
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_accvgpr_read_b32 v0, a0
+; CHECK-NEXT:    v_accvgpr_read_b32 v1, a1
+; CHECK-NEXT:    v_accvgpr_read_b32 v2, a2
+; CHECK-NEXT:    v_accvgpr_read_b32 v3, a3
+; CHECK-NEXT:    v_accvgpr_read_b32 v4, a4
+; CHECK-NEXT:    v_accvgpr_read_b32 v5, a5
+; CHECK-NEXT:    v_accvgpr_read_b32 v6, a6
+; CHECK-NEXT:    v_accvgpr_read_b32 v7, a7
+; CHECK-NEXT:    v_accvgpr_read_b32 v8, a8
+; CHECK-NEXT:    v_accvgpr_read_b32 v9, a9
+; CHECK-NEXT:    v_accvgpr_read_b32 v10, a10
+; CHECK-NEXT:    v_accvgpr_read_b32 v11, a11
+; CHECK-NEXT:    v_accvgpr_read_b32 v12, a12
+; CHECK-NEXT:    v_accvgpr_read_b32 v13, a13
+; CHECK-NEXT:    v_accvgpr_read_b32 v14, a14
+; CHECK-NEXT:    v_accvgpr_read_b32 v15, a15
+; CHECK-NEXT:    v_accvgpr_read_b32 v16, a16
+; CHECK-NEXT:    v_accvgpr_read_b32 v17, a17
+; CHECK-NEXT:    v_accvgpr_read_b32 v18, a18
+; CHECK-NEXT:    v_accvgpr_read_b32 v19, a19
+; CHECK-NEXT:    v_accvgpr_read_b32 v20, a20
+; CHECK-NEXT:    v_accvgpr_read_b32 v21, a21
+; CHECK-NEXT:    v_accvgpr_read_b32 v22, a22
+; CHECK-NEXT:    v_accvgpr_read_b32 v23, a23
+; CHECK-NEXT:    v_accvgpr_read_b32 v24, a24
+; CHECK-NEXT:    v_accvgpr_read_b32 v25, a25
+; CHECK-NEXT:    v_accvgpr_read_b32 v26, a26
+; CHECK-NEXT:    v_accvgpr_read_b32 v27, a27
+; CHECK-NEXT:    v_accvgpr_read_b32 v28, a28
+; CHECK-NEXT:    v_accvgpr_read_b32 v29, a29
+; CHECK-NEXT:    v_accvgpr_read_b32 v30, a30
+; CHECK-NEXT:    v_accvgpr_read_b32 v31, a31
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v34, v[0:31]
 ; CHECK-NEXT:    s_nop 7
 ; CHECK-NEXT:    s_nop 7
 ; CHECK-NEXT:    s_nop 1
-; CHECK-NEXT:    global_store_dwordx4 v0, a[24:27], s[0:1] offset:96
-; CHECK-NEXT:    global_store_dwordx4 v0, a[28:31], s[0:1] offset:112
-; CHECK-NEXT:    global_store_dwordx4 v0, a[16:19], s[0:1] offset:64
-; CHECK-NEXT:    global_store_dwordx4 v0, a[20:23], s[0:1] offset:80
-; CHECK-NEXT:    global_store_dwordx4 v0, a[8:11], s[0:1] offset:32
-; CHECK-NEXT:    global_store_dwordx4 v0, a[12:15], s[0:1] offset:48
-; CHECK-NEXT:    global_store_dwordx4 v0, a[0:3], s[0:1]
-; CHECK-NEXT:    global_store_dwordx4 v0, a[4:7], s[0:1] offset:16
+; CHECK-NEXT:    global_store_dwordx4 v32, v[24:27], s[2:3] offset:96
+; CHECK-NEXT:    global_store_dwordx4 v32, v[28:31], s[2:3] offset:112
+; CHECK-NEXT:    global_store_dwordx4 v32, v[16:19], s[2:3] offset:64
+; CHECK-NEXT:    global_store_dwordx4 v32, v[20:23], s[2:3] offset:80
+; CHECK-NEXT:    global_store_dwordx4 v32, v[8:11], s[2:3] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v32, v[12:15], s[2:3] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v32, v[0:3], s[2:3]
+; CHECK-NEXT:    global_store_dwordx4 v32, v[4:7], s[2:3] offset:16
 ; CHECK-NEXT:    s_endpgm
-bb:
+  %src2 = call <32 x float> asm sideeffect "; def $0", "=a"()
+  %mai0 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float 4.0, <32 x float> %src2, i32 0, i32 0, i32 0)
+  %mai1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 8.0, float 16.0, <32 x float> %src2, i32 0, i32 0, i32 0)
   %id = call i32 @llvm.amdgcn.workitem.id.x()
-  %gep = getelementptr <32 x float>, ptr addrspace(1) %arg, i32 %id
-  %in.1 = load <32 x float>, ptr addrspace(1) %gep, align 128
-  %mai.1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %in.1, i32 0, i32 0, i32 0)
-  %mai.2 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.1, i32 0, i32 0, i32 0)
-  %mai.3 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> %mai.2, i32 0, i32 0, i32 0)
-  store <32 x float> %mai.3, ptr addrspace(1) %arg, align 128
+  %gep0 = getelementptr <32 x float>, ptr addrspace(1) %arg0, i32 %id
+  store <32 x float> %mai0, ptr addrspace(1) %gep0, align 128
+  %gep1 = getelementptr <32 x float>, ptr addrspace(1) %arg1, i32 %id
+  store <32 x float> %mai1, ptr addrspace(1) %gep1, align 128
+  ret void
+}
+
+define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class_chain(ptr addrspace(1) %arg0) #0 {
+; CHECK-LABEL: test_rewrite_mfma_direct_copy_from_agpr_class_chain:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    v_mov_b32_e32 v1, 2.0
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_mov_b32_e32 v34, 4.0
+; CHECK-NEXT:    v_accvgpr_read_b32 v33, a31
+; CHECK-NEXT:    v_accvgpr_read_b32 v32, a30
+; CHECK-NEXT:    v_accvgpr_read_b32 v31, a29
+; CHECK-NEXT:    v_accvgpr_read_b32 v30, a28
+; CHECK-NEXT:    v_accvgpr_read_b32 v29, a27
+; CHECK-NEXT:    v_accvgpr_read_b32 v28, a26
+; CHECK-NEXT:    v_accvgpr_read_b32 v27, a25
+; CHECK-NEXT:    v_accvgpr_read_b32 v26, a24
+; CHECK-NEXT:    v_accvgpr_read_b32 v25, a23
+; CHECK-NEXT:    v_accvgpr_read_b32 v24, a22
+; CHECK-NEXT:    v_accvgpr_read_b32 v23, a21
+; CHECK-NEXT:    v_accvgpr_read_b32 v22, a20
+; CHECK-NEXT:    v_accvgpr_read_b32 v21, a19
+; CHECK-NEXT:    v_accvgpr_read_b32 v20, a18
+; CHECK-NEXT:    v_accvgpr_read_b32 v19, a17
+; CHECK-NEXT:    v_accvgpr_read_b32 v18, a16
+; CHECK-NEXT:    v_accvgpr_read_b32 v17, a15
+; CHECK-NEXT:    v_accvgpr_read_b32 v16, a14
+; CHECK-NEXT:    v_accvgpr_read_b32 v15, a13
+; CHECK-NEXT:    v_accvgpr_read_b32 v14, a12
+; CHECK-NEXT:    v_accvgpr_read_b32 v13, a11
+; CHECK-NEXT:    v_accvgpr_read_b32 v12, a10
+; CHECK-NEXT:    v_accvgpr_read_b32 v11, a9
+; CHECK-NEXT:    v_accvgpr_read_b32 v10, a8
+; CHECK-NEXT:    v_accvgpr_read_b32 v9, a7
+; CHECK-NEXT:    v_accvgpr_read_b32 v8, a6
+; CHECK-NEXT:    v_accvgpr_read_b32 v7, a5
+; CHECK-NEXT:    v_accvgpr_read_b32 v6, a4
+; CHECK-NEXT:    v_accvgpr_read_b32 v5, a3
+; CHECK-NEXT:    v_accvgpr_read_b32 v4, a2
+; CHECK-NEXT:    v_accvgpr_read_b32 v3, a1
+; CHECK-NEXT:    v_accvgpr_read_b32 v2, a0
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[2:33], v1, v34, v[2:33]
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0x41000000
+; CHECK-NEXT:    v_mov_b32_e32 v34, 0x41800000
+; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 7, v0
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 v[2:33], v1, v34, v[2:33]
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    global_store_dwordx4 v0, v[30:33], s[0:1] offset:112
+; CHECK-NEXT:    global_store_dwordx4 v0, v[26:29], s[0:1] offset:96
+; CHECK-NEXT:    global_store_dwordx4 v0, v[22:25], s[0:1] offset:80
+; CHECK-NEXT:    global_store_dwordx4 v0, v[18:21], s[0:1] offset:64
+; CHECK-NEXT:    global_store_dwordx4 v0, v[14:17], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v0, v[10:13], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
+; CHECK-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
+; CHECK-NEXT:    s_endpgm
+  %src2 = call <32 x float> asm sideeffect "; def $0", "=a"()
+  %mai0 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float 4.0, <32 x float> %src2, i32 0, i32 0, i32 0)
+  %mai1 = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 8.0, float 16.0, <32 x float> %mai0, i32 0, i32 0, i32 0)
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <32 x float>, ptr addrspace(1) %arg0, i32 %id
+  store <32 x float> %mai1, ptr addrspace(1) %gep0, align 128
+  ret void
+}
+
+; Untied case
+define void @test_rewrite_mfma_copy_from_agpr_class_f64_4x4x4f64(double %arg0, double %arg1, ptr addrspace(1) %ptr) #0 {
+; CHECK-LABEL: test_rewrite_mfma_copy_from_agpr_class_f64_4x4x4f64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def a[0:1]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_and_b32_e32 v8, 0x3ff, v31
+; CHECK-NEXT:    v_accvgpr_read_b32 v7, a1
+; CHECK-NEXT:    v_accvgpr_read_b32 v6, a0
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[6:7]
+; CHECK-NEXT:    v_lshlrev_b32_e32 v2, 3, v8
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_lshl_add_u64 v[2:3], v[4:5], 0, v[2:3]
+; CHECK-NEXT:    s_nop 5
+; CHECK-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %src2 = call double asm sideeffect "; def $0", "=a"()
+  %mai = call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %arg0, double %arg1, double %src2, i32 0, i32 0, i32 0)
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr double, ptr addrspace(1) %ptr, i32 %id
+  store double %mai, ptr addrspace(1) %gep0, align 8
+  ret void
+}
+
+define void @test_rewrite_mfma_copy_from_agpr_class_f64_4x4x4f64_chain(double %arg0, double %arg1, double %arg2, double %arg3, ptr addrspace(1) %ptr) #0 {
+; CHECK-LABEL: test_rewrite_mfma_copy_from_agpr_class_f64_4x4x4f64_chain:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def a[0:1]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    v_accvgpr_read_b32 v11, a1
+; CHECK-NEXT:    v_accvgpr_read_b32 v10, a0
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    v_mfma_f64_4x4x4_4b_f64 v[0:1], v[0:1], v[2:3], v[10:11]
+; CHECK-NEXT:    v_and_b32_e32 v2, 0x3ff, v31
+; CHECK-NEXT:    v_lshlrev_b32_e32 v2, 3, v2
+; CHECK-NEXT:    v_mov_b32_e32 v3, 0
+; CHECK-NEXT:    v_lshl_add_u64 v[2:3], v[8:9], 0, v[2:3]
+; CHECK-NEXT:    v_mfma_f64_4x4x4_4b_f64 v[0:1], v[4:5], v[6:7], v[0:1]
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    global_store_dwordx2 v[2:3], v[0:1], off
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    s_setpc_b64 s[30:31]
+  %src2 = call double asm sideeffect "; def $0", "=a"()
+  %mai0 = call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %arg0, double %arg1, double %src2, i32 0, i32 0, i32 0)
+  %mai1 = call double @llvm.amdgcn.mfma.f64.4x4x4f64(double %arg2, double %arg3, double %mai0, i32 0, i32 0, i32 0)
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr double, ptr addrspace(1) %ptr, i32 %id
+  store double %mai1, ptr addrspace(1) %gep0, align 8
+  ret void
+}
+
+define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class_subreg(ptr addrspace(1) %arg) #0 {
+; CHECK-LABEL: test_rewrite_mfma_direct_copy_from_agpr_class_subreg:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    v_mov_b32_e32 v1, 2.0
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_mov_b32_e32 v18, 4.0
+; CHECK-NEXT:    v_accvgpr_read_b32 v17, a15
+; CHECK-NEXT:    v_accvgpr_read_b32 v16, a14
+; CHECK-NEXT:    v_accvgpr_read_b32 v15, a13
+; CHECK-NEXT:    v_accvgpr_read_b32 v14, a12
+; CHECK-NEXT:    v_accvgpr_read_b32 v13, a11
+; CHECK-NEXT:    v_accvgpr_read_b32 v12, a10
+; CHECK-NEXT:    v_accvgpr_read_b32 v11, a9
+; CHECK-NEXT:    v_accvgpr_read_b32 v10, a8
+; CHECK-NEXT:    v_accvgpr_read_b32 v9, a7
+; CHECK-NEXT:    v_accvgpr_read_b32 v8, a6
+; CHECK-NEXT:    v_accvgpr_read_b32 v7, a5
+; CHECK-NEXT:    v_accvgpr_read_b32 v6, a4
+; CHECK-NEXT:    v_accvgpr_read_b32 v5, a3
+; CHECK-NEXT:    v_accvgpr_read_b32 v4, a2
+; CHECK-NEXT:    v_accvgpr_read_b32 v3, a1
+; CHECK-NEXT:    v_accvgpr_read_b32 v2, a0
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT:    v_mfma_f32_16x16x1_4b_f32 v[2:17], v1, v18, v[2:17]
+; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 6, v0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    global_store_dwordx4 v0, v[14:17], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v0, v[10:13], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
+; CHECK-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
+; CHECK-NEXT:    s_endpgm
+  %def = call <32 x float> asm sideeffect "; def $0", "=a"()
+  %src2 = shufflevector <32 x float> %def, <32 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %mai = call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 2.0, float 4.0, <16 x float> %src2, i32 0, i32 0, i32 0)
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id
+  store <16 x float> %mai, ptr addrspace(1) %gep, align 64
+  ret void
+}
+
+define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class_subreg_odd(ptr addrspace(1) %arg) #0 {
+; CHECK-LABEL: test_rewrite_mfma_direct_copy_from_agpr_class_subreg_odd:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    v_mov_b32_e32 v1, 2.0
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_mov_b32_e32 v18, 4.0
+; CHECK-NEXT:    v_accvgpr_read_b32 v17, a16
+; CHECK-NEXT:    v_accvgpr_read_b32 v16, a15
+; CHECK-NEXT:    v_accvgpr_read_b32 v15, a14
+; CHECK-NEXT:    v_accvgpr_read_b32 v14, a13
+; CHECK-NEXT:    v_accvgpr_read_b32 v13, a12
+; CHECK-NEXT:    v_accvgpr_read_b32 v12, a11
+; CHECK-NEXT:    v_accvgpr_read_b32 v11, a10
+; CHECK-NEXT:    v_accvgpr_read_b32 v10, a9
+; CHECK-NEXT:    v_accvgpr_read_b32 v9, a8
+; CHECK-NEXT:    v_accvgpr_read_b32 v8, a7
+; CHECK-NEXT:    v_accvgpr_read_b32 v7, a6
+; CHECK-NEXT:    v_accvgpr_read_b32 v6, a5
+; CHECK-NEXT:    v_accvgpr_read_b32 v5, a4
+; CHECK-NEXT:    v_accvgpr_read_b32 v4, a3
+; CHECK-NEXT:    v_accvgpr_read_b32 v3, a2
+; CHECK-NEXT:    v_accvgpr_read_b32 v2, a1
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; CHECK-NEXT:    v_mfma_f32_16x16x1_4b_f32 v[2:17], v1, v18, v[2:17]
+; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 6, v0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    s_nop 7
+; CHECK-NEXT:    global_store_dwordx4 v0, v[14:17], s[0:1] offset:48
+; CHECK-NEXT:    global_store_dwordx4 v0, v[10:13], s[0:1] offset:32
+; CHECK-NEXT:    global_store_dwordx4 v0, v[6:9], s[0:1] offset:16
+; CHECK-NEXT:    global_store_dwordx4 v0, v[2:5], s[0:1]
+; CHECK-NEXT:    s_endpgm
+  %def = call <32 x float> asm sideeffect "; def $0", "=a"()
+  %src2 = shufflevector <32 x float> %def, <32 x float> poison, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
+  %mai = call <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float 2.0, float 4.0, <16 x float> %src2, i32 0, i32 0, i32 0)
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <16 x float>, ptr addrspace(1) %arg, i32 %id
+  store <16 x float> %mai, ptr addrspace(1) %gep, align 64
+  ret void
+}
+
+; a->v->mfma->a
+define amdgpu_kernel void @test_rewrite_mfma_direct_copy_from_agpr_class_copy_back() #0 {
+; CHECK-LABEL: test_rewrite_mfma_direct_copy_from_agpr_class_copy_back:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    v_mov_b32_e32 v32, 2.0
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; def a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    v_mov_b32_e32 v33, 4.0
+; CHECK-NEXT:    s_nop 1
+; CHECK-NEXT:    v_mfma_f32_32x32x1_2b_f32 a[0:31], v32, v33, a[0:31]
+; CHECK-NEXT:    ;;#ASMSTART
+; CHECK-NEXT:    ; use a[0:31]
+; CHECK-NEXT:    ;;#ASMEND
+; CHECK-NEXT:    s_endpgm
+  %src2 = call <32 x float> asm sideeffect "; def $0", "=a"()
+  %mai = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 2.0, float 4.0, <32 x float> %src2, i32 0, i32 0, i32 0)
+  call void asm sideeffect "; use $0", "a"(<32 x float> %mai)
   ret void
 }
 
-declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32 immarg, i32 immarg, i32 immarg) #1
-declare noundef i32 @llvm.amdgcn.workitem.id.x() #2
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x16f16(<4 x half>, <4 x half>, <4 x float>, i32 immarg, i32 immarg, i32 immarg) #2
+declare <16 x float> @llvm.amdgcn.mfma.f32.16x16x1f32(float, float, <16 x float>, i32 immarg, i32 immarg, i32 immarg) #2
+declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32 immarg, i32 immarg, i32 immarg) #2
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #3
 
-attributes #0 = { "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-implicitarg-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="4,4" }
-attributes #1 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
-attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+attributes #0 = { nounwind "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="4,4" }
+attributes #1 = { mustprogress nofree norecurse nounwind willreturn "amdgpu-waves-per-eu"="8,8" }
+attributes #2 = { convergent nocallback nofree nosync nounwind willreturn memory(none) }
+attributes #3 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/CodeGen/AMDGPU/s-barrier-lowering.ll b/llvm/test/CodeGen/AMDGPU/s-barrier-lowering.ll
index 1f2b3e2c31892..03a666fbe3aea 100644
--- a/llvm/test/CodeGen/AMDGPU/s-barrier-lowering.ll
+++ b/llvm/test/CodeGen/AMDGPU/s-barrier-lowering.ll
@@ -1,31 +1,40 @@
 ; RUN: opt -S -mtriple=amdgcn-- -amdgpu-lower-module-lds < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -verify-machineinstrs -o - %s | FileCheck -check-prefixes=SOUT %s
 
-@bar2 = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison
+%class.ExpAmdWorkgroupWaveBarrier = type { target("amdgcn.named.barrier", 0) }
+
+@bar2 = internal addrspace(3) global [2 x target("amdgcn.named.barrier", 0)] poison
 @bar3 = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison
-@bar1 = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison
+@bar1 = internal addrspace(3) global [4 x %class.ExpAmdWorkgroupWaveBarrier] poison
 
-; CHECK: @bar2 = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison, !absolute_symbol !0
+; CHECK: @bar2 = internal addrspace(3) global [2 x target("amdgcn.named.barrier", 0)] poison, !absolute_symbol !0
 ; CHECK-NEXT: @bar3 = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison, !absolute_symbol !1
-; CHECK-NEXT: @bar1 = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison, !absolute_symbol !2
-; CHECK-NEXT: @bar1.kernel1 = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison, !absolute_symbol !2
+; CHECK-NEXT: @bar1 = internal addrspace(3) global [4 x %class.ExpAmdWorkgroupWaveBarrier] poison, !absolute_symbol !2
+; CHECK-NEXT: @bar1.kernel1 = internal addrspace(3) global [4 x %class.ExpAmdWorkgroupWaveBarrier] poison, !absolute_symbol !2
 
+; SOUT:        .set func1.num_named_barrier, 7
 define void @func1() {
     call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar3, i32 7)
+    call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar3)
     call void @llvm.amdgcn.s.barrier.wait(i16 1)
     ret void
 }
 
+; SOUT:        .set func2.num_named_barrier, 2
 define void @func2() {
     call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar2, i32 7)
+    call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar2)
     call void @llvm.amdgcn.s.barrier.wait(i16 1)
     ret void
 }
 
+; SOUT:                .amdhsa_named_barrier_count 2
+; SOUT:        .set kernel1.num_named_barrier, max(6, func1.num_named_barrier, func2.num_named_barrier)
 define amdgpu_kernel void @kernel1() #0 {
 ; CHECK-DAG: call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar1.kernel1, i32 11)
     call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar1, i32 11)
+    call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar1)
     call void @llvm.amdgcn.s.barrier.wait(i16 1)
-    call void @llvm.amdgcn.s.wakeup.barrier(ptr addrspace(3) @bar1)
     %state = call i32 @llvm.amdgcn.s.get.named.barrier.state(ptr addrspace(3) @bar1)
     call void @llvm.amdgcn.s.barrier()
     call void @func1()
@@ -33,9 +42,12 @@ define amdgpu_kernel void @kernel1() #0 {
     ret void
 }
 
+; SOUT:                .amdhsa_named_barrier_count 2
+; SOUT:        .set kernel2.num_named_barrier, max(6, func2.num_named_barrier)
 define amdgpu_kernel void @kernel2() #0 {
 ; CHECK-DAG: call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar1, i32 9)
     call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar1, i32 9)
+    call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar1)
     call void @llvm.amdgcn.s.barrier.wait(i16 1)
 
     call void @func2()
@@ -47,6 +59,9 @@ declare void @llvm.amdgcn.s.barrier.wait(i16) #1
 declare void @llvm.amdgcn.s.barrier.signal(i32) #1
 declare void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3), i32) #1
 declare i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32) #1
+declare void @llvm.amdgcn.s.barrier.init(ptr addrspace(3), i32) #1
+declare void @llvm.amdgcn.s.barrier.join(ptr addrspace(3)) #1
+declare void @llvm.amdgcn.s.barrier.leave(i16) #1
 declare void @llvm.amdgcn.s.wakeup.barrier(ptr addrspace(3)) #1
 declare i32 @llvm.amdgcn.s.get.named.barrier.state(ptr addrspace(3)) #1
 
@@ -55,5 +70,5 @@ attributes #1 = { convergent nounwind }
 attributes #2 = { nounwind readnone }
 
 ; CHECK: !0 = !{i32 8396816, i32 8396817}
-; CHECK-NEXT: !1 = !{i32 8396848, i32 8396849}
-; CHECK-NEXT: !2 = !{i32 8396832, i32 8396833}
+; CHECK-NEXT: !1 = !{i32 8396912, i32 8396913}
+; CHECK-NEXT: !2 = !{i32 8396848, i32 8396849}
diff --git a/llvm/test/CodeGen/AMDGPU/s-barrier.ll b/llvm/test/CodeGen/AMDGPU/s-barrier.ll
new file mode 100644
index 0000000000000..a4fa8e4b3c8e2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/s-barrier.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12-GISEL %s
+
+@bar = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison
+@bar2 = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison
+@bar3 = internal addrspace(3) global target("amdgcn.named.barrier", 0) poison
+
+define void @func1() {
+; GFX12-SDAG-LABEL: func1:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, 3
+; GFX12-SDAG-NEXT:    s_barrier_join m0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, 0x70003
+; GFX12-SDAG-NEXT:    s_barrier_signal m0
+; GFX12-SDAG-NEXT:    s_barrier_wait 1
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: func1:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_mov_b32 m0, 0x70003
+; GFX12-GISEL-NEXT:    s_barrier_join 3
+; GFX12-GISEL-NEXT:    s_barrier_signal m0
+; GFX12-GISEL-NEXT:    s_barrier_wait 1
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+    call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar3)
+    call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar3, i32 7)
+    call void @llvm.amdgcn.s.barrier.wait(i16 1)
+    ret void
+}
+
+define void @func2() {
+; GFX12-SDAG-LABEL: func2:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, 1
+; GFX12-SDAG-NEXT:    s_barrier_join m0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, 0x70001
+; GFX12-SDAG-NEXT:    s_barrier_signal m0
+; GFX12-SDAG-NEXT:    s_barrier_wait 1
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: func2:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_mov_b32 m0, 0x70001
+; GFX12-GISEL-NEXT:    s_barrier_join 1
+; GFX12-GISEL-NEXT:    s_barrier_signal m0
+; GFX12-GISEL-NEXT:    s_barrier_wait 1
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+    call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar2)
+    call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar2, i32 7)
+    call void @llvm.amdgcn.s.barrier.wait(i16 1)
+    ret void
+}
+
+define amdgpu_kernel void @kernel1(ptr addrspace(1) %out, ptr addrspace(3) %in) #0 {
+; GFX12-SDAG-LABEL: kernel1:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX12-SDAG-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX12-SDAG-NEXT:    s_load_b32 s2, s[4:5], 0x2c
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, 0xc0002
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v31, v0
+; GFX12-SDAG-NEXT:    s_barrier_init m0
+; GFX12-SDAG-NEXT:    s_add_nc_u64 s[8:9], s[4:5], 48
+; GFX12-SDAG-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX12-SDAG-NEXT:    s_mov_b32 s32, 0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_lshr_b32 s2, s2, 4
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-SDAG-NEXT:    s_and_b32 s2, s2, 63
+; GFX12-SDAG-NEXT:    s_or_b32 s3, 0x90000, s2
+; GFX12-SDAG-NEXT:    s_cmp_eq_u32 0, 0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT:    s_barrier_init m0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, 0xc0002
+; GFX12-SDAG-NEXT:    s_barrier_signal m0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, s3
+; GFX12-SDAG-NEXT:    s_barrier_signal m0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT:    s_barrier_signal -1
+; GFX12-SDAG-NEXT:    s_barrier_join m0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, 2
+; GFX12-SDAG-NEXT:    s_barrier_signal_isfirst -1
+; GFX12-SDAG-NEXT:    s_barrier_wait 1
+; GFX12-SDAG-NEXT:    s_barrier_leave
+; GFX12-SDAG-NEXT:    s_get_barrier_state s3, m0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, s2
+; GFX12-SDAG-NEXT:    s_get_barrier_state s2, m0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_getpc_b64 s[2:3]
+; GFX12-SDAG-NEXT:    s_sext_i32_i16 s3, s3
+; GFX12-SDAG-NEXT:    s_add_co_u32 s2, s2, func1@gotpcrel32@lo+8
+; GFX12-SDAG-NEXT:    s_add_co_ci_u32 s3, s3, func1@gotpcrel32@hi+16
+; GFX12-SDAG-NEXT:    s_barrier_signal -1
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[2:3], 0x0
+; GFX12-SDAG-NEXT:    s_barrier_wait -1
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX12-SDAG-NEXT:    s_getpc_b64 s[2:3]
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    s_sext_i32_i16 s3, s3
+; GFX12-SDAG-NEXT:    s_add_co_u32 s2, s2, func2@gotpcrel32@lo+12
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    s_add_co_ci_u32 s3, s3, func2@gotpcrel32@hi+24
+; GFX12-SDAG-NEXT:    s_load_b64 s[2:3], s[2:3], 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_swappc_b64 s[30:31], s[2:3]
+; GFX12-SDAG-NEXT:    s_get_barrier_state s0, -1
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: kernel1:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_mov_b64 s[12:13], s[4:5]
+; GFX12-GISEL-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX12-GISEL-NEXT:    s_load_b32 s0, s[12:13], 0x2c
+; GFX12-GISEL-NEXT:    s_mov_b32 m0, 0xc0002
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v31, v0
+; GFX12-GISEL-NEXT:    s_barrier_init m0
+; GFX12-GISEL-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX12-GISEL-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX12-GISEL-NEXT:    s_mov_b32 s32, 0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_lshr_b32 s0, s0, 4
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-GISEL-NEXT:    s_and_b32 s0, s0, 63
+; GFX12-GISEL-NEXT:    s_or_b32 s1, s0, 0x90000
+; GFX12-GISEL-NEXT:    s_cmp_eq_u32 0, 0
+; GFX12-GISEL-NEXT:    s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT:    s_barrier_init m0
+; GFX12-GISEL-NEXT:    s_mov_b32 m0, 0xc0002
+; GFX12-GISEL-NEXT:    s_barrier_signal m0
+; GFX12-GISEL-NEXT:    s_mov_b32 m0, s1
+; GFX12-GISEL-NEXT:    s_barrier_signal m0
+; GFX12-GISEL-NEXT:    s_mov_b32 m0, s0
+; GFX12-GISEL-NEXT:    s_barrier_signal -1
+; GFX12-GISEL-NEXT:    s_barrier_join m0
+; GFX12-GISEL-NEXT:    s_barrier_signal_isfirst -1
+; GFX12-GISEL-NEXT:    s_add_co_u32 s8, s12, 48
+; GFX12-GISEL-NEXT:    s_barrier_wait 1
+; GFX12-GISEL-NEXT:    s_barrier_leave
+; GFX12-GISEL-NEXT:    s_get_barrier_state s0, 2
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_get_barrier_state s0, m0
+; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s9, s13, 0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_getpc_b64 s[0:1]
+; GFX12-GISEL-NEXT:    s_sext_i32_i16 s1, s1
+; GFX12-GISEL-NEXT:    s_add_co_u32 s0, s0, func1@gotpcrel32@lo+8
+; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, func1@gotpcrel32@hi+16
+; GFX12-GISEL-NEXT:    s_barrier_signal -1
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-GISEL-NEXT:    s_barrier_wait -1
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX12-GISEL-NEXT:    s_add_co_u32 s8, s12, 48
+; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s9, s13, 0
+; GFX12-GISEL-NEXT:    s_getpc_b64 s[0:1]
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    s_sext_i32_i16 s1, s1
+; GFX12-GISEL-NEXT:    s_add_co_u32 s0, s0, func2@gotpcrel32@lo+12
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s1, s1, func2@gotpcrel32@hi+24
+; GFX12-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GFX12-GISEL-NEXT:    s_get_barrier_state s0, -1
+; GFX12-GISEL-NEXT:    s_endpgm
+    call void @llvm.amdgcn.s.barrier.init(ptr addrspace(3) @bar, i32 12)
+    call void @llvm.amdgcn.s.barrier.init(ptr addrspace(3) %in, i32 9)
+    call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar, i32 12)
+    call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) %in, i32 9)
+    call void @llvm.amdgcn.s.barrier.signal(i32 -1)
+    call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) %in)
+    %isfirst = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 -1)
+    call void @llvm.amdgcn.s.barrier.wait(i16 1)
+    call void @llvm.amdgcn.s.barrier.leave(i16 1)
+    %state = call i32 @llvm.amdgcn.s.get.named.barrier.state(ptr addrspace(3) @bar)
+    %state2 = call i32 @llvm.amdgcn.s.get.named.barrier.state(ptr addrspace(3) %in)
+    call void @llvm.amdgcn.s.barrier()
+    call void @func1()
+    call void @func2()
+    %state3 = call i32 @llvm.amdgcn.s.get.barrier.state(i32 -1)
+    ret void
+}
+
+define amdgpu_kernel void @kernel2(ptr addrspace(1) %out, ptr addrspace(3) %in) #0 {
+; GFX12-SDAG-LABEL: kernel2:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX12-SDAG-NEXT:    s_getpc_b64 s[6:7]
+; GFX12-SDAG-NEXT:    s_sext_i32_i16 s7, s7
+; GFX12-SDAG-NEXT:    s_add_co_u32 s6, s6, func2@gotpcrel32@lo+8
+; GFX12-SDAG-NEXT:    s_add_co_ci_u32 s7, s7, func2@gotpcrel32@hi+16
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v31, v0
+; GFX12-SDAG-NEXT:    s_load_b64 s[12:13], s[6:7], 0x0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, 0x70002
+; GFX12-SDAG-NEXT:    s_add_nc_u64 s[8:9], s[4:5], 48
+; GFX12-SDAG-NEXT:    s_barrier_signal m0
+; GFX12-SDAG-NEXT:    s_mov_b32 m0, 2
+; GFX12-SDAG-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX12-SDAG-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX12-SDAG-NEXT:    s_mov_b32 s32, 0
+; GFX12-SDAG-NEXT:    s_barrier_join m0
+; GFX12-SDAG-NEXT:    s_barrier_wait 1
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_swappc_b64 s[30:31], s[12:13]
+; GFX12-SDAG-NEXT:    s_endpgm
+;
+; GFX12-GISEL-LABEL: kernel2:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_add_co_u32 s8, s4, 48
+; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s9, s5, 0
+; GFX12-GISEL-NEXT:    s_getpc_b64 s[4:5]
+; GFX12-GISEL-NEXT:    s_sext_i32_i16 s5, s5
+; GFX12-GISEL-NEXT:    s_add_co_u32 s4, s4, func2@gotpcrel32@lo+8
+; GFX12-GISEL-NEXT:    s_add_co_ci_u32 s5, s5, func2@gotpcrel32@hi+16
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v31, v0
+; GFX12-GISEL-NEXT:    s_load_b64 s[12:13], s[4:5], 0x0
+; GFX12-GISEL-NEXT:    s_mov_b64 s[10:11], s[6:7]
+; GFX12-GISEL-NEXT:    s_mov_b32 m0, 0x70002
+; GFX12-GISEL-NEXT:    s_mov_b64 s[4:5], s[0:1]
+; GFX12-GISEL-NEXT:    s_mov_b64 s[6:7], s[2:3]
+; GFX12-GISEL-NEXT:    s_mov_b32 s32, 0
+; GFX12-GISEL-NEXT:    s_barrier_signal m0
+; GFX12-GISEL-NEXT:    s_barrier_join 2
+; GFX12-GISEL-NEXT:    s_barrier_wait 1
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_swappc_b64 s[30:31], s[12:13]
+; GFX12-GISEL-NEXT:    s_endpgm
+    call void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3) @bar, i32 7)
+    call void @llvm.amdgcn.s.barrier.join(ptr addrspace(3) @bar)
+    call void @llvm.amdgcn.s.barrier.wait(i16 1)
+
+    call void @func2()
+    ret void
+}
+
+declare void @llvm.amdgcn.s.barrier() #1
+declare void @llvm.amdgcn.s.barrier.wait(i16) #1
+declare void @llvm.amdgcn.s.barrier.signal(i32) #1
+declare void @llvm.amdgcn.s.barrier.signal.var(ptr addrspace(3), i32) #1
+declare i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32) #1
+declare void @llvm.amdgcn.s.barrier.init(ptr addrspace(3), i32) #1
+declare void @llvm.amdgcn.s.barrier.join(ptr addrspace(3)) #1
+declare void @llvm.amdgcn.s.barrier.leave(i16) #1
+declare i32 @llvm.amdgcn.s.get.barrier.state(i32) #1
+declare i32 @llvm.amdgcn.s.get.named.barrier.state(ptr addrspace(3)) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { convergent nounwind }
+attributes #2 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/saddsat.ll b/llvm/test/CodeGen/AMDGPU/saddsat.ll
index 019eb2c661edc..4995ce6e57d00 100644
--- a/llvm/test/CodeGen/AMDGPU/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/saddsat.ll
@@ -124,9 +124,8 @@ define i32 @v_saddsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX6-NEXT:    v_add_i32_e64 v1, s[4:5], v0, v1
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v1, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v1
-; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v1, -v0, s[4:5]
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_saddsat_i32:
@@ -136,9 +135,8 @@ define i32 @v_saddsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX8-NEXT:    v_add_u32_e64 v1, s[4:5], v0, v1
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v1, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v1
-; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v1, -v0, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_saddsat_i32:
@@ -383,16 +381,14 @@ define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX6-NEXT:    v_add_i32_e64 v2, s[4:5], v0, v2
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v2
-; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, -v0, s[4:5]
 ; GFX6-NEXT:    v_add_i32_e64 v2, s[4:5], v1, v3
 ; GFX6-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v3
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v1
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v2
-; GFX6-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v2, -v1, s[4:5]
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_saddsat_v2i32:
@@ -402,16 +398,14 @@ define <2 x i32> @v_saddsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v0, v2
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v2
-; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, -v0, s[4:5]
 ; GFX8-NEXT:    v_add_u32_e64 v2, s[4:5], v1, v3
 ; GFX8-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v1
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v2
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v2, -v1, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_saddsat_v2i32:
@@ -442,8 +436,7 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX6-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_saddsat_i64:
@@ -456,8 +449,7 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_saddsat_i64:
@@ -470,8 +462,7 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_saddsat_i64:
@@ -480,12 +471,11 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v2
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_i64_e64 s4, 0, v[2:3]
-; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX10-NEXT:    v_xor_b32_e32 v1, 0x80000000, v6
+; GFX10-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX10-NEXT:    s_xor_b32 vcc_lo, s4, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_saddsat_i64:
@@ -494,11 +484,11 @@ define i64 @v_saddsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, v0, v2
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v5, null, v1, v3, vcc_lo
 ; GFX11-NEXT:    v_cmp_gt_i64_e64 s0, 0, v[2:3]
-; GFX11-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
 ; GFX11-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX11-NEXT:    v_xor_b32_e32 v1, 0x80000000, v6
+; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX11-NEXT:    s_xor_b32 vcc_lo, s0, vcc_lo
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v6 :: v_dual_cndmask_b32 v1, v5, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call i64 @llvm.sadd.sat.i64(i64 %lhs, i64 %rhs)
   ret i64 %result
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-count-graphics.ll b/llvm/test/CodeGen/AMDGPU/sgpr-count-graphics.ll
new file mode 100644
index 0000000000000..3c7b5bf97b879
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-count-graphics.ll
@@ -0,0 +1,38 @@
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=gfx1200 < %s | FileCheck %s --check-prefixes=CHECK,PACKED16
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=tahiti < %s | FileCheck %s --check-prefixes=CHECK,SPLIT16
+
+@global = addrspace(1) global i32 poison, align 4
+
+; The hardware initializes the registers received as arguments by entry points,
+; so they will be counted even if unused.
+
+; Vectors of i1 are always unpacked
+
+; CHECK-LABEL: vec_of_i1:
+; CHECK:  TotalNumSgprs: 8
+define amdgpu_ps void @vec_of_i1(<8 x i1> inreg %v8i1) {
+  ret void
+}
+
+; Vectors of i8 are always unpacked
+
+; CHECK-LABEL: vec_of_i8:
+; CHECK:  TotalNumSgprs: 4
+define amdgpu_ps void @vec_of_i8(<4 x i8> inreg %v4i8) {
+  ret void
+}
+
+; Vectors of 16-bit types are packed for newer architectures and unpacked for older ones.
+
+; CHECK-LABEL: vec_of_16_bit_ty:
+; PACKED16: TotalNumSgprs: 3
+; SPLIT16:  TotalNumSgprs: 6
+define amdgpu_ps void @vec_of_16_bit_ty(<2 x i16> inreg %v2i16, <4 x half> inreg %v4half) {
+  ret void
+}
+
+; CHECK-LABEL: buffer_fat_ptr:
+; CHECK: TotalNumSgprs: 5
+define amdgpu_ps void @buffer_fat_ptr(ptr addrspace(7) inreg %p) {
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/shl_add_ptr_global.ll b/llvm/test/CodeGen/AMDGPU/shl_add_ptr_global.ll
index e83ed89f73c85..91b024727f699 100644
--- a/llvm/test/CodeGen/AMDGPU/shl_add_ptr_global.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl_add_ptr_global.ll
@@ -19,7 +19,7 @@ define void @shl_base_atomicrmw_global_ptr(ptr addrspace(1) %out, ptr addrspace(
   %cast = ptrtoint ptr addrspace(1) %arrayidx0 to i64
   %shl = shl i64 %cast, 2
   %castback = inttoptr i64 %shl to ptr addrspace(1)
-  %val = atomicrmw and ptr addrspace(1) %castback, i32 3 syncscope("agent") seq_cst
+  %val = atomicrmw and ptr addrspace(1) %castback, i32 3 syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   store volatile i64 %cast, ptr addrspace(1) %extra.use, align 4
   ret void
 }
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
index 21aa40d69998e..91c88ec5e718c 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
@@ -1528,10 +1528,9 @@ define amdgpu_kernel void @v_test_i16_x_sub_64_zext_to_i32(ptr addrspace(1) %out
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-SDAG-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-SDAG-TRUE16-NEXT:    v_sub_nc_u16 v0.l, v0.l, 64
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-SDAG-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-SDAG-TRUE16-NEXT:    s_endpgm
 ;
@@ -1560,10 +1559,9 @@ define amdgpu_kernel void @v_test_i16_x_sub_64_zext_to_i32(ptr addrspace(1) %out
 ; GFX11-GISEL-TRUE16-NEXT:    v_lshlrev_b32_e32 v1, 2, v1
 ; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-GISEL-TRUE16-NEXT:    global_load_d16_b16 v0, v0, s[2:3]
+; GFX11-GISEL-TRUE16-NEXT:    v_mov_b16_e32 v0.h, 0
 ; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-GISEL-TRUE16-NEXT:    v_add_nc_u16 v0.l, 0xffc0, v0.l
-; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-GISEL-TRUE16-NEXT:    global_store_b32 v1, v0, s[0:1]
 ; GFX11-GISEL-TRUE16-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/si-pre-allocate-wwwmregs-dbg-noreg.mir b/llvm/test/CodeGen/AMDGPU/si-pre-allocate-wwwmregs-dbg-noreg.mir
new file mode 100644
index 0000000000000..cb515f880d0a0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-pre-allocate-wwwmregs-dbg-noreg.mir
@@ -0,0 +1,71 @@
+# RUN: llc %s -o - -mcpu=gfx1030 -O0 -run-pass=si-pre-allocate-wwm-regs | FileCheck %s
+
+# Simple regression test to make sure DBG_VALUE $noreg does not assert in the pass
+
+# CHECK: $vgpr0 = IMPLICIT_DEF
+# CHECK: $vgpr0 = SI_SPILL_S32_TO_VGPR $sgpr3, 0, $vgpr0
+
+--- |
+  target triple = "amdgcn-amd-amdpal"
+  %dx.types.ResRet.f32 = type { float, float, float, float, i32 }
+  
+  define dllexport amdgpu_cs void @_amdgpu_cs_main(i32 inreg noundef %globalTable, i32 inreg noundef %userdata4, <3 x i32> inreg noundef %WorkgroupId, i32 inreg noundef %MultiDispatchInfo, <3 x i32> noundef %LocalInvocationId) #0 !dbg !5 {
+      #dbg_value(i32 poison, !19, !DIExpression(DW_OP_LLVM_fragment, 0, 32), !20)
+      #dbg_value(%dx.types.ResRet.f32 poison, !21, !DIExpression(), !23)
+    ret void, !dbg !24
+  }
+  
+  attributes #0 = { memory(readwrite) "amdgpu-prealloc-sgpr-spill-vgprs" }
+  
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4}
+  
+  !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "dxcoob 1.7.2308.16 (52da17e29)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !2)
+  !1 = !DIFile(filename: "tests\\basic_var.hlsl", directory: "")
+  !2 = !{}
+  !3 = !{i32 2, !"Dwarf Version", i32 5}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 7, type: !6, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+  !6 = !DISubroutineType(types: !7)
+  !7 = !{null, !8}
+  !8 = !DIDerivedType(tag: DW_TAG_typedef, name: "uint3", file: !1, baseType: !9)
+  !9 = !DICompositeType(tag: DW_TAG_class_type, name: "vector<unsigned int, 3>", file: !1, size: 96, align: 32, elements: !10, templateParams: !15)
+  !10 = !{!11, !13, !14}
+  !11 = !DIDerivedType(tag: DW_TAG_member, name: "x", scope: !9, file: !1, baseType: !12, size: 32, align: 32, flags: DIFlagPublic)
+  !12 = !DIBasicType(name: "unsigned int", size: 32, align: 32, encoding: DW_ATE_unsigned)
+  !13 = !DIDerivedType(tag: DW_TAG_member, name: "y", scope: !9, file: !1, baseType: !12, size: 32, align: 32, offset: 32, flags: DIFlagPublic)
+  !14 = !DIDerivedType(tag: DW_TAG_member, name: "z", scope: !9, file: !1, baseType: !12, size: 32, align: 32, offset: 64, flags: DIFlagPublic)
+  !15 = !{!16, !17}
+  !16 = !DITemplateTypeParameter(name: "element", type: !12)
+  !17 = !DITemplateValueParameter(name: "element_count", type: !18, value: i32 3)
+  !18 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
+  !19 = !DILocalVariable(name: "dtid", arg: 1, scope: !5, file: !1, line: 7, type: !8)
+  !20 = !DILocation(line: 7, column: 17, scope: !5)
+  !21 = !DILocalVariable(name: "my_var", scope: !5, file: !1, line: 11, type: !22)
+  !22 = !DIBasicType(name: "float", size: 32, align: 32, encoding: DW_ATE_float)
+  !23 = !DILocation(line: 11, column: 9, scope: !5)
+  !24 = !DILocation(line: 19, column: 1, scope: !5)
+...
+---
+name:            _amdgpu_cs_main
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  hasSpilledSGPRs: true
+  stackPtrOffsetReg: '$sgpr32'
+  occupancy:       16
+  sgprForEXECCopy: '$sgpr12_sgpr13'
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $sgpr1, $sgpr2, $sgpr3, $sgpr4, $vgpr0, $vgpr1, $vgpr2
+  
+    %0:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = IMPLICIT_DEF
+    %2:vgpr_32 = SI_SPILL_S32_TO_VGPR $sgpr3, 0, %2
+    renamable $sgpr3 = COPY killed $sgpr2
+    renamable $sgpr4 = S_MOV_B32 6
+    %3:vgpr_32 = V_LSHL_ADD_U32_e64 killed $sgpr3, killed $sgpr4, %0, implicit $exec
+    DBG_VALUE %3, $noreg, !19, !DIExpression(DW_OP_LLVM_fragment, 0, 32),  debug-location !20
+    DBG_VALUE $noreg, $noreg, !21, !DIExpression(),  debug-location !23
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
index 6a45b961a61c8..101787abf8ea7 100644
--- a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 < %s | FileCheck -enable-var-scope -check-prefixes=GFX942 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
@@ -32,6 +33,16 @@ define amdgpu_kernel void @sint_to_fp_i32_to_f64(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: sint_to_fp_i32_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_i32_e32 v[0:1], s2
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %result = sitofp i32 %in to double
   store double %result, ptr addrspace(1) %out
   ret void
@@ -73,6 +84,18 @@ define amdgpu_kernel void @sint_to_fp_i1_f64(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: sint_to_fp_i1_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, 0xbff00000, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, s2
+; GFX942-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cmp = icmp eq i32 %in, 0
   %fp = sitofp i1 %cmp to double
   store double %fp, ptr addrspace(1) %out, align 4
@@ -113,6 +136,19 @@ define amdgpu_kernel void @sint_to_fp_i1_f64_load(ptr addrspace(1) %out, i1 %in)
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: sint_to_fp_i1_f64_load:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX942-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX942-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[2:3]
+; GFX942-NEXT:    v_cvt_f64_i32_e32 v[0:1], v0
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %fp = sitofp i1 %in to double
   store double %fp, ptr addrspace(1) %out, align 8
   ret void
@@ -150,6 +186,18 @@ define amdgpu_kernel void @s_sint_to_fp_i64_to_f64(ptr addrspace(1) %out, i64 %i
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_sint_to_fp_i64_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_i32_e32 v[0:1], s3
+; GFX942-NEXT:    v_ldexp_f64 v[0:1], v[0:1], 32
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[2:3], s2
+; GFX942-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX942-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %result = sitofp i64 %in to double
   store double %result, ptr addrspace(1) %out
   ret void
@@ -199,6 +247,22 @@ define amdgpu_kernel void @v_sint_to_fp_i64_to_f64(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: v_sint_to_fp_i64_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_i32_e32 v[2:3], v1
+; GFX942-NEXT:    v_ldexp_f64 v[2:3], v[2:3], 32
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GFX942-NEXT:    v_add_f64 v[0:1], v[2:3], v[0:1]
+; GFX942-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid
   %val = load i64, ptr addrspace(1) %gep, align 8
@@ -238,6 +302,17 @@ define amdgpu_kernel void @s_sint_to_fp_i8_to_f64(ptr addrspace(1) %out, i8 %in)
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_sint_to_fp_i8_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_sext_i32_i8 s2, s2
+; GFX942-NEXT:    v_cvt_f64_i32_e32 v[0:1], s2
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %fp = sitofp i8 %in to double
   store double %fp, ptr addrspace(1) %out
   ret void
@@ -258,6 +333,14 @@ define double @v_sint_to_fp_i8_to_f64(i8 %in) {
 ; VI-NEXT:    v_bfe_i32 v0, v0, 0, 16
 ; VI-NEXT:    v_cvt_f64_i32_e32 v[0:1], v0
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_sint_to_fp_i8_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX942-NEXT:    v_bfe_i32 v0, v0, 0, 16
+; GFX942-NEXT:    v_cvt_f64_i32_e32 v[0:1], v0
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %fp = sitofp i8 %in to double
   ret double %fp
   }
@@ -296,6 +379,18 @@ define amdgpu_kernel void @s_select_sint_to_fp_i1_vals_f64(ptr addrspace(1) %out
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_select_sint_to_fp_i1_vals_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, 0xbff00000, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, s2
+; GFX942-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, double -1.0, double 0.0
   store double %select, ptr addrspace(1) %out, align 8
@@ -313,6 +408,18 @@ define void @v_select_sint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in) {
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[3:4]
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_select_sint_to_fp_i1_vals_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v3, 0xbff00000
+; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_nop 0
+; GFX942-NEXT:    v_cndmask_b32_e32 v5, 0, v3, vcc
+; GFX942-NEXT:    global_store_dwordx2 v[0:1], v[4:5], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, double -1.0, double 0.0
   store double %select, ptr addrspace(1) %out, align 8
@@ -353,6 +460,18 @@ define amdgpu_kernel void @s_select_sint_to_fp_i1_vals_i64(ptr addrspace(1) %out
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_select_sint_to_fp_i1_vals_i64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, 0xbff00000, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, s2
+; GFX942-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, i64 u0xbff0000000000000, i64 0
   store i64 %select, ptr addrspace(1) %out, align 8
@@ -370,6 +489,18 @@ define void @v_select_sint_to_fp_i1_vals_i64(ptr addrspace(1) %out, i32 %in) {
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[3:4]
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_select_sint_to_fp_i1_vals_i64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v3, 0xbff00000
+; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_nop 0
+; GFX942-NEXT:    v_cndmask_b32_e32 v5, 0, v3, vcc
+; GFX942-NEXT:    global_store_dwordx2 v[0:1], v[4:5], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, i64 u0xbff0000000000000, i64 0
   store i64 %select, ptr addrspace(1) %out, align 8
@@ -388,6 +519,18 @@ define void @v_swap_select_sint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[3:4]
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_swap_select_sint_to_fp_i1_vals_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v3, 0xbff00000
+; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_nop 0
+; GFX942-NEXT:    v_cndmask_b32_e64 v5, v3, 0, vcc
+; GFX942-NEXT:    global_store_dwordx2 v[0:1], v[4:5], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, double 0.0, double -1.0
   store double %select, ptr addrspace(1) %out, align 8
@@ -429,6 +572,18 @@ define amdgpu_kernel void @s_swap_select_sint_to_fp_i1_vals_f64(ptr addrspace(1)
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_swap_select_sint_to_fp_i1_vals_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, 0, 0xbff00000
+; GFX942-NEXT:    v_mov_b32_e32 v1, s2
+; GFX942-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, double 0.0, double -1.0
   store double %select, ptr addrspace(1) %out, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
index 40d80f5e83e36..09c0e775f783d 100644
--- a/llvm/test/CodeGen/AMDGPU/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/ssubsat.ll
@@ -124,9 +124,8 @@ define i32 @v_ssubsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX6-NEXT:    v_sub_i32_e64 v1, s[4:5], v0, v1
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v1, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v1
-; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v1, -v0, s[4:5]
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_ssubsat_i32:
@@ -136,9 +135,8 @@ define i32 @v_ssubsat_i32(i32 %lhs, i32 %rhs) {
 ; GFX8-NEXT:    v_sub_u32_e64 v1, s[4:5], v0, v1
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v1, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v1
-; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v1, -v0, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_ssubsat_i32:
@@ -383,16 +381,14 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX6-NEXT:    v_sub_i32_e64 v2, s[4:5], v0, v2
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v2
-; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, -v0, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v2, s[4:5], v1, v3
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v3
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v1
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v2
-; GFX6-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v2, -v1, s[4:5]
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_ssubsat_v2i32:
@@ -402,16 +398,14 @@ define <2 x i32> @v_ssubsat_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {
 ; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v0, v2
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v2
-; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v2, v0, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v2, -v0, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v2, s[4:5], v1, v3
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v3
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v2, v1
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v2
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v2, -v1, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_ssubsat_v2i32:
@@ -439,23 +433,20 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX6-NEXT:    v_sub_i32_e64 v3, s[4:5], v0, v3
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
-; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v3, -v0, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v3, s[4:5], v1, v4
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v4
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v1
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v3
-; GFX6-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v3, -v1, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v3, s[4:5], v2, v5
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v5
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v2
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 31, v3
-; GFX6-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v3, -v2, s[4:5]
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_ssubsat_v3i32:
@@ -465,23 +456,20 @@ define <3 x i32> @v_ssubsat_v3i32(<3 x i32> %lhs, <3 x i32> %rhs) {
 ; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v0, v3
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
-; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v3, v0, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v3, -v0, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v1, v4
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v4
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v1
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v3
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v3, -v1, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v3, s[4:5], v2, v5
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v3, v2
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v3
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v3, -v2, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_ssubsat_v3i32:
@@ -511,30 +499,26 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX6-NEXT:    v_sub_i32_e64 v4, s[4:5], v0, v4
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v4
-; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v4, -v0, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v4, s[4:5], v1, v5
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v5
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v1
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v4
-; GFX6-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v4, -v1, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v4, s[4:5], v2, v6
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v6
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v2
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 31, v4
-; GFX6-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v4, -v2, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v4, s[4:5], v3, v7
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v7
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v3
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 31, v4
-; GFX6-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v4, -v3, s[4:5]
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_ssubsat_v4i32:
@@ -544,30 +528,26 @@ define <4 x i32> @v_ssubsat_v4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
 ; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v0, v4
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v4
-; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, -v0, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v1, v5
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v5
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v1
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v4
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v1, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v4, -v1, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v2, v6
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v6
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v2
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v4
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v4, -v2, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v4, s[4:5], v3, v7
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v7
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v4, v3
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v4
-; GFX8-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v3, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v4, -v3, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_ssubsat_v4i32:
@@ -599,58 +579,50 @@ define <8 x i32> @v_ssubsat_v8i32(<8 x i32> %lhs, <8 x i32> %rhs) {
 ; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v0, v8
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v8
-; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v8, -v0, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v1, v9
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v9
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v1
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v8
-; GFX6-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v8, -v1, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v2, v10
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v10
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v2
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 31, v8
-; GFX6-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v8, -v2, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v3, v11
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v11
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v3
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 31, v8
-; GFX6-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v8, -v3, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v4, v12
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v12
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v4
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v4, 31, v8
-; GFX6-NEXT:    v_xor_b32_e32 v4, 0x80000000, v4
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, v8, -v4, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v5, v13
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v13
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v5
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v5, 31, v8
-; GFX6-NEXT:    v_xor_b32_e32 v5, 0x80000000, v5
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, v8, -v5, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v6, v14
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v14
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v6
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v6, 31, v8
-; GFX6-NEXT:    v_xor_b32_e32 v6, 0x80000000, v6
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, v8, -v6, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v8, s[4:5], v7, v15
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v15
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v7
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v7, 31, v8
-; GFX6-NEXT:    v_xor_b32_e32 v7, 0x80000000, v7
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v7, v8, -v7, s[4:5]
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_ssubsat_v8i32:
@@ -660,58 +632,50 @@ define <8 x i32> @v_ssubsat_v8i32(<8 x i32> %lhs, <8 x i32> %rhs) {
 ; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v0, v8
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v8
-; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v8, v0, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v8, -v0, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v1, v9
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v9
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v1
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v8
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v8, v1, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v8, -v1, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v2, v10
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v10
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v2
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v8
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v8, v2, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v8, -v2, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v3, v11
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v11
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v3
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v8
-; GFX8-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v8, v3, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v8, -v3, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v4, v12
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v12
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v4
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v8
-; GFX8-NEXT:    v_xor_b32_e32 v4, 0x80000000, v4
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v8, v4, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, -v4, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v5, v13
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v13
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v5
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v5, 31, v8
-; GFX8-NEXT:    v_xor_b32_e32 v5, 0x80000000, v5
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v8, v5, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v8, -v5, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v6, v14
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v14
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v6
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v6, 31, v8
-; GFX8-NEXT:    v_xor_b32_e32 v6, 0x80000000, v6
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v8, v6, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v8, -v6, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v8, s[4:5], v7, v15
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v15
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v8, v7
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v7, 31, v8
-; GFX8-NEXT:    v_xor_b32_e32 v7, 0x80000000, v7
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v8, v7, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v8, -v7, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_ssubsat_v8i32:
@@ -751,116 +715,100 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v0, v16
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v0, 31, v16
-; GFX6-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v16, -v0, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v1, v17
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v17
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v1
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v16
-; GFX6-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v16, -v1, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v2, v18
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v18
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v2
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v2, 31, v16
-; GFX6-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v2, v16, -v2, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v3, v19
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v19
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v3
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 31, v16
-; GFX6-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v4, v20
-; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v20
-; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v4
-; GFX6-NEXT:    v_ashrrev_i32_e32 v4, 31, v16
-; GFX6-NEXT:    v_xor_b32_e32 v4, 0x80000000, v4
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v3, v16, -v3, s[4:5]
 ; GFX6-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v4, v20
+; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v20
+; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v4
+; GFX6-NEXT:    v_ashrrev_i32_e32 v4, 31, v17
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, v17, -v4, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v5, v21
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v21
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v5
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v5, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v5, 0x80000000, v5
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v5, v17, -v5, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v6, v22
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v22
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v6
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v6, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v6, 0x80000000, v6
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v6, v17, v6, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v6, v17, -v6, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v7, v23
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v23
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v7
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v7, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v7, 0x80000000, v7
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v7, v17, v7, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v7, v17, -v7, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v8, v24
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v24
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v8
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v8, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v8, 0x80000000, v8
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v8, v17, -v8, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v9, v25
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v25
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v9
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v9, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v9, 0x80000000, v9
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v9, v17, -v9, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v10, v26
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v26
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v10
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v10, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v10, 0x80000000, v10
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v10, v17, -v10, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v11, v27
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v27
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v11
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v11, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v11, 0x80000000, v11
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v11, v17, -v11, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v12, v28
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v28
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v12
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v12, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v12, 0x80000000, v12
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v12, v17, -v12, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v13, v29
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v29
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v13
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v13, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v13, 0x80000000, v13
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v13, v17, -v13, s[4:5]
 ; GFX6-NEXT:    v_sub_i32_e64 v17, s[4:5], v14, v30
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v30
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v14
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v14, 31, v17
-; GFX6-NEXT:    v_xor_b32_e32 v14, 0x80000000, v14
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v14, v17, v14, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v14, v17, -v14, s[4:5]
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v16
 ; GFX6-NEXT:    v_sub_i32_e64 v16, s[4:5], v15, v16
 ; GFX6-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v15
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v15, 31, v16
-; GFX6-NEXT:    v_xor_b32_e32 v15, 0x80000000, v15
-; GFX6-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX6-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
+; GFX6-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v15, v16, -v15, s[4:5]
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_ssubsat_v16i32:
@@ -870,116 +818,100 @@ define <16 x i32> @v_ssubsat_v16i32(<16 x i32> %lhs, <16 x i32> %rhs) {
 ; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v0, v16
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v0, 31, v16
-; GFX8-NEXT:    v_xor_b32_e32 v0, 0x80000000, v0
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v0, v16, v0, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v16, -v0, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v1, v17
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v17
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v1
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v16
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v16, v1, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v16, -v1, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v2, v18
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v18
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v2
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v16
-; GFX8-NEXT:    v_xor_b32_e32 v2, 0x80000000, v2
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v2, v16, v2, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v16, -v2, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v3, v19
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v19
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v3
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v16
-; GFX8-NEXT:    v_xor_b32_e32 v3, 0x80000000, v3
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v3, v16, v3, vcc
-; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v4, v20
-; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v20
-; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v4
-; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v16
-; GFX8-NEXT:    v_xor_b32_e32 v4, 0x80000000, v4
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v4, v16, v4, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v16, -v3, s[4:5]
 ; GFX8-NEXT:    buffer_load_dword v16, off, s[0:3], s32
+; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v4, v20
+; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v20
+; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v4
+; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v17
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v17, -v4, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v5, v21
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v21
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v5
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v5, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v5, 0x80000000, v5
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v5, v17, v5, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v17, -v5, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v6, v22
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v22
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v6
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v6, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v6, 0x80000000, v6
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v6, v17, v6, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v17, -v6, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v7, v23
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v23
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v7
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v7, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v7, 0x80000000, v7
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v7, v17, v7, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v17, -v7, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v8, v24
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v24
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v8
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v8, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v8, 0x80000000, v8
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v8, v17, v8, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v8, v17, -v8, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v9, v25
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v25
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v9
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v9, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v9, 0x80000000, v9
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v9, v17, v9, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v9, v17, -v9, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v10, v26
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v26
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v10
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v10, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v10, 0x80000000, v10
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v10, v17, v10, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v10, v17, -v10, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v11, v27
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v27
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v11
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v11, 0x80000000, v11
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v11, v17, v11, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v11, v17, -v11, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v12, v28
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v28
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v12
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v12, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v12, 0x80000000, v12
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v12, v17, v12, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v12, v17, -v12, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v13, v29
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v29
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v13
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v13, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v13, 0x80000000, v13
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v13, v17, v13, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v13, v17, -v13, s[4:5]
 ; GFX8-NEXT:    v_sub_u32_e64 v17, s[4:5], v14, v30
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v30
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v17, v14
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v14, 31, v17
-; GFX8-NEXT:    v_xor_b32_e32 v14, 0x80000000, v14
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v14, v17, v14, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v14, v17, -v14, s[4:5]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_cmp_lt_i32_e32 vcc, 0, v16
 ; GFX8-NEXT:    v_sub_u32_e64 v16, s[4:5], v15, v16
 ; GFX8-NEXT:    v_cmp_lt_i32_e64 s[4:5], v16, v15
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v15, 31, v16
-; GFX8-NEXT:    v_xor_b32_e32 v15, 0x80000000, v15
-; GFX8-NEXT:    s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e32 v15, v16, v15, vcc
+; GFX8-NEXT:    s_xor_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v15, v16, -v15, s[4:5]
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_ssubsat_v16i32:
@@ -1066,8 +998,7 @@ define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX6-NEXT:    s_xor_b64 vcc, s[4:5], vcc
 ; GFX6-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX6-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX6-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc
 ; GFX6-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: v_ssubsat_i64:
@@ -1080,8 +1011,7 @@ define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX8-NEXT:    s_xor_b64 vcc, s[4:5], vcc
 ; GFX8-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX8-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-LABEL: v_ssubsat_i64:
@@ -1094,8 +1024,7 @@ define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX9-NEXT:    s_xor_b64 vcc, s[4:5], vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc
-; GFX9-NEXT:    v_xor_b32_e32 v1, 0x80000000, v1
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: v_ssubsat_i64:
@@ -1104,12 +1033,11 @@ define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX10-NEXT:    v_sub_co_u32 v4, vcc_lo, v0, v2
 ; GFX10-NEXT:    v_sub_co_ci_u32_e32 v5, vcc_lo, v1, v3, vcc_lo
 ; GFX10-NEXT:    v_cmp_lt_i64_e64 s4, 0, v[2:3]
-; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
 ; GFX10-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX10-NEXT:    v_xor_b32_e32 v1, 0x80000000, v6
+; GFX10-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX10-NEXT:    s_xor_b32 vcc_lo, s4, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v6, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, v5, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: v_ssubsat_i64:
@@ -1118,11 +1046,11 @@ define i64 @v_ssubsat_i64(i64 %lhs, i64 %rhs) {
 ; GFX11-NEXT:    v_sub_co_u32 v4, vcc_lo, v0, v2
 ; GFX11-NEXT:    v_sub_co_ci_u32_e64 v5, null, v1, v3, vcc_lo
 ; GFX11-NEXT:    v_cmp_lt_i64_e64 s0, 0, v[2:3]
-; GFX11-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
 ; GFX11-NEXT:    v_cmp_lt_i64_e32 vcc_lo, v[4:5], v[0:1]
-; GFX11-NEXT:    v_xor_b32_e32 v1, 0x80000000, v6
+; GFX11-NEXT:    v_ashrrev_i32_e32 v1, 31, v5
 ; GFX11-NEXT:    s_xor_b32 vcc_lo, s0, vcc_lo
-; GFX11-NEXT:    v_dual_cndmask_b32 v0, v4, v6 :: v_dual_cndmask_b32 v1, v5, v1
+; GFX11-NEXT:    v_cndmask_b32_e32 v0, v4, v1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v5, -v1, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %result = call i64 @llvm.ssub.sat.i64(i64 %lhs, i64 %rhs)
   ret i64 %result
diff --git a/llvm/test/CodeGen/AMDGPU/store-to-constant-error.ll b/llvm/test/CodeGen/AMDGPU/store-to-constant-error.ll
deleted file mode 100644
index 0bfc45c84b0c4..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/store-to-constant-error.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: not --crash  llc -global-isel=0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o /dev/null %s 2>&1 | FileCheck -check-prefix=SDAG %s
-; RUN: not llc -global-isel=1 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -o /dev/null %s 2>&1 | FileCheck -check-prefix=GISEL %s
-
-; SDAG: LLVM ERROR: Cannot select: {{[a-z0-9]+}}: ch = store<(store (s32) into %ir.ptr.load, addrspace 4)>
-; GISEL: LLVM ERROR: cannot select: G_STORE %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr(p4) :: (store (s32) into %ir.ptr.load, addrspace 4) (in function: store_to_constant_i32)
-define amdgpu_kernel void @store_to_constant_i32(ptr addrspace(4) %ptr) {
-bb:
-  store i32 1, ptr addrspace(4) %ptr, align 4
-  ret void
-}
diff --git a/llvm/test/CodeGen/AMDGPU/store-to-constant.ll b/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
new file mode 100644
index 0000000000000..64d5d01454a37
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
@@ -0,0 +1,186 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a %s -o - | FileCheck %s
+
+; FIXME: We need to test AS6 but the AS6 variants of the following tests fail because of illegal VGPR to SGPR copy.
+; FIXME: We also want to test memset, memcpy, and memmove, but it needs to fix the SelectionDAG store merging issue (#90714).
+
+define amdgpu_kernel void @store_as4_i8(ptr addrspace(4) %p, i8 %v) {
+; CHECK-LABEL: store_as4_i8:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_store_byte v0, v1, s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store i8 %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_i16(ptr addrspace(4) %p, i16 %v) {
+; CHECK-LABEL: store_as4_i16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_store_short v0, v1, s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store i16 %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_i32(ptr addrspace(4) %p, i32 %v) {
+; CHECK-LABEL: store_as4_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store i32 %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_i64(ptr addrspace(4) %p, i64 %v) {
+; CHECK-LABEL: store_as4_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v2, s2
+; CHECK-NEXT:    v_mov_b32_e32 v3, s3
+; CHECK-NEXT:    global_store_dwordx2 v0, v[2:3], s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store i64 %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_float(ptr addrspace(4) %p, float %v) {
+; CHECK-LABEL: store_as4_float:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store float %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_double(ptr addrspace(4) %p, double %v) {
+; CHECK-LABEL: store_as4_double:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v2, s2
+; CHECK-NEXT:    v_mov_b32_e32 v3, s3
+; CHECK-NEXT:    global_store_dwordx2 v0, v[2:3], s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store double %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_half(ptr addrspace(4) %p, half %v) {
+; CHECK-LABEL: store_as4_half:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_store_short v0, v1, s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store half %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_2xi8(ptr addrspace(4) %p, <2 x i8> %v) {
+; CHECK-LABEL: store_as4_2xi8:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_store_short v0, v1, s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store <2 x i8> %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_2xi16(ptr addrspace(4) %p, <2 x i16> %v) {
+; CHECK-LABEL: store_as4_2xi16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store <2 x i16> %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_2xi32(ptr addrspace(4) %p, <2 x i32> %v) {
+; CHECK-LABEL: store_as4_2xi32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store <2 x i32> %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_2xhalf(ptr addrspace(4) %p, <2 x half> %v) {
+; CHECK-LABEL: store_as4_2xhalf:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dword s2, s[8:9], 0x8
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_store_dword v0, v1, s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store <2 x half> %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_2xfloat(ptr addrspace(4) %p, <2 x float> %v) {
+; CHECK-LABEL: store_as4_2xfloat:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; CHECK-NEXT:    s_endpgm
+  store <2 x float> %v, ptr addrspace(4) %p
+  ret void
+}
+
+define amdgpu_kernel void @store_as4_2xdouble(ptr addrspace(4) %p, <2 x double> %v) {
+; CHECK-LABEL: store_as4_2xdouble:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[8:9], 0x10
+; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; CHECK-NEXT:    v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[4:5]
+; CHECK-NEXT:    s_endpgm
+  store <2 x double> %v, ptr addrspace(4) %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll b/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
deleted file mode 100644
index 726e35d6651d0..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/test_isel_single_lane.ll
+++ /dev/null
@@ -1,47 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GCN %s
-
-declare i32 @llvm.amdgcn.atomic.cond.sub.u32.p1(ptr addrspace(1), i32)
-
-
-define amdgpu_kernel void @test_isel_single_lane(ptr addrspace(1) %in, ptr addrspace(1) %out) #0 {
-; GCN-LABEL: test_isel_single_lane:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    s_load_b32 s4, s[0:1], 0x58
-; GCN-NEXT:    s_wait_kmcnt 0x0
-; GCN-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s4
-; GCN-NEXT:    global_atomic_cond_sub_u32 v1, v0, v1, s[0:1] offset:16 th:TH_ATOMIC_RETURN
-; GCN-NEXT:    s_wait_loadcnt 0x0
-; GCN-NEXT:    v_readfirstlane_b32 s0, v1
-; GCN-NEXT:    s_addk_co_i32 s0, 0xf4
-; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT:    s_lshl_b32 s1, s0, 4
-; GCN-NEXT:    s_mul_i32 s0, s0, s1
-; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT:    s_lshl_b32 s0, s0, 12
-; GCN-NEXT:    s_sub_co_i32 s0, s1, s0
-; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    global_store_b32 v0, v1, s[2:3]
-; GCN-NEXT:    s_endpgm
-        %gep0 = getelementptr i32, ptr addrspace(1) %in, i32 22
-        %val0 = load i32, ptr addrspace(1) %gep0, align 4
-        %gep1 = getelementptr i32, ptr addrspace(1) %in, i32 4
-        %val1 = call i32 @llvm.amdgcn.atomic.cond.sub.u32.p0(ptr addrspace(1) %gep1, i32 %val0)
-        %res0 = add i32 %val1, 244
-        %res1 = shl i32 %res0, 4
-        %res2 = mul i32 %res0, %res1
-        %res3 = shl i32 %res2, 12
-        %res4 = sub i32 %res1, %res3
-        store i32 %res4, ptr addrspace(1) %out
-        ret void
-}
-
-
-attributes #0 = {
-  "amdgpu-flat-work-group-size"="1,1"
-  "amdgpu-waves-per-eu"="1,1"
-  "uniform-work-group-size"="true"
-}
diff --git a/llvm/test/CodeGen/AMDGPU/triton_regression_no_waterfall.mir b/llvm/test/CodeGen/AMDGPU/triton_regression_no_waterfall.mir
index 576cf634ae0d0..dc9b9d6440305 100644
--- a/llvm/test/CodeGen/AMDGPU/triton_regression_no_waterfall.mir
+++ b/llvm/test/CodeGen/AMDGPU/triton_regression_no_waterfall.mir
@@ -69,8 +69,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/AMDGPU/twoaddr-constrain.ll b/llvm/test/CodeGen/AMDGPU/twoaddr-constrain.ll
index 8629d548ec0c1..6cc396085280c 100644
--- a/llvm/test/CodeGen/AMDGPU/twoaddr-constrain.ll
+++ b/llvm/test/CodeGen/AMDGPU/twoaddr-constrain.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgcn -mcpu=gfx900 -stop-after twoaddressinstruction < %s | FileCheck %s
+; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx900 -stop-after twoaddressinstruction < %s | FileCheck %s
 
 ; Check that %16 gets constrained to register class sgpr_96_with_sub0_sub1.
 define amdgpu_ps <3 x i32> @s_load_constant_v3i32_align4(ptr addrspace(4) inreg %ptr) {
diff --git a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
index ab278c3b63a3e..983acfc2c0699 100644
--- a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii < %s | FileCheck -enable-var-scope  -check-prefixes=GCN,SI %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 < %s | FileCheck -enable-var-scope -check-prefixes=GFX942 %s
 
 declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
 
@@ -48,6 +49,22 @@ define amdgpu_kernel void @v_uint_to_fp_i64_to_f64(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: v_uint_to_fp_i64_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX942-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    global_load_dwordx2 v[0:1], v0, s[2:3]
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[2:3], v1
+; GFX942-NEXT:    v_ldexp_f64 v[2:3], v[2:3], 32
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GFX942-NEXT:    v_add_f64 v[0:1], v[2:3], v[0:1]
+; GFX942-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
   %gep = getelementptr i64, ptr addrspace(1) %in, i32 %tid
   %val = load i64, ptr addrspace(1) %gep, align 8
@@ -88,6 +105,18 @@ define amdgpu_kernel void @s_uint_to_fp_i64_to_f64(ptr addrspace(1) %out, i64 %i
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_uint_to_fp_i64_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
+; GFX942-NEXT:    v_ldexp_f64 v[0:1], v[0:1], 32
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[2:3], s2
+; GFX942-NEXT:    v_add_f64 v[0:1], v[0:1], v[2:3]
+; GFX942-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cast = uitofp i64 %in to double
   store double %cast, ptr addrspace(1) %out, align 8
   ret void
@@ -136,6 +165,23 @@ define amdgpu_kernel void @s_uint_to_fp_v2i64_to_v2f64(ptr addrspace(1) %out, <2
 ; VI-NEXT:    v_mov_b32_e32 v4, s0
 ; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_uint_to_fp_v2i64_to_v2f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v6, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s3
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[2:3], s2
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[4:5], s1
+; GFX942-NEXT:    v_ldexp_f64 v[0:1], v[0:1], 32
+; GFX942-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
+; GFX942-NEXT:    v_ldexp_f64 v[0:1], v[4:5], 32
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[4:5], s0
+; GFX942-NEXT:    v_add_f64 v[0:1], v[0:1], v[4:5]
+; GFX942-NEXT:    global_store_dwordx4 v6, v[0:3], s[6:7]
+; GFX942-NEXT:    s_endpgm
   %cast = uitofp <2 x i64> %in to <2 x double>
   store <2 x double> %cast, ptr addrspace(1) %out, align 16
   ret void
@@ -210,6 +256,32 @@ define amdgpu_kernel void @s_uint_to_fp_v4i64_to_v4f64(ptr addrspace(1) %out, <4
 ; VI-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
 ; VI-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_uint_to_fp_v4i64_to_v4f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x20
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v10, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s11
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[2:3], s10
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[4:5], s9
+; GFX942-NEXT:    v_ldexp_f64 v[0:1], v[0:1], 32
+; GFX942-NEXT:    v_add_f64 v[2:3], v[0:1], v[2:3]
+; GFX942-NEXT:    v_ldexp_f64 v[0:1], v[4:5], 32
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[4:5], s8
+; GFX942-NEXT:    v_add_f64 v[0:1], v[0:1], v[4:5]
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[4:5], s15
+; GFX942-NEXT:    v_ldexp_f64 v[4:5], v[4:5], 32
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[6:7], s14
+; GFX942-NEXT:    v_add_f64 v[6:7], v[4:5], v[6:7]
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[4:5], s13
+; GFX942-NEXT:    v_ldexp_f64 v[4:5], v[4:5], 32
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[8:9], s12
+; GFX942-NEXT:    v_add_f64 v[4:5], v[4:5], v[8:9]
+; GFX942-NEXT:    global_store_dwordx4 v10, v[4:7], s[0:1] offset:16
+; GFX942-NEXT:    global_store_dwordx4 v10, v[0:3], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cast = uitofp <4 x i64> %in to <4 x double>
   store <4 x double> %cast, ptr addrspace(1) %out, align 16
   ret void
@@ -243,6 +315,16 @@ define amdgpu_kernel void @s_uint_to_fp_i32_to_f64(ptr addrspace(1) %out, i32 %i
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_uint_to_fp_i32_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cast = uitofp i32 %in to double
   store double %cast, ptr addrspace(1) %out, align 8
   ret void
@@ -262,6 +344,16 @@ define amdgpu_kernel void @s_uint_to_fp_v2i32_to_v2f64(ptr addrspace(1) %out, <2
 ; GCN-NEXT:    v_mov_b32_e32 v4, s0
 ; GCN-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GCN-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_uint_to_fp_v2i32_to_v2f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[2:3], s3
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX942-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cast = uitofp <2 x i32> %in to <2 x double>
   store <2 x double> %cast, ptr addrspace(1) %out, align 16
   ret void
@@ -313,6 +405,20 @@ define amdgpu_kernel void @s_uint_to_fp_v4i32_to_v4f64(ptr addrspace(1) %out, <4
 ; VI-NEXT:    v_mov_b32_e32 v5, s5
 ; VI-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_uint_to_fp_v4i32_to_v4f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x10
+; GFX942-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v8, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[6:7], s3
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[4:5], s2
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[2:3], s1
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s0
+; GFX942-NEXT:    global_store_dwordx4 v8, v[4:7], s[6:7] offset:16
+; GFX942-NEXT:    global_store_dwordx4 v8, v[0:3], s[6:7]
+; GFX942-NEXT:    s_endpgm
   %cast = uitofp <4 x i32> %in to <4 x double>
   store <4 x double> %cast, ptr addrspace(1) %out, align 16
   ret void
@@ -354,6 +460,18 @@ define amdgpu_kernel void @uint_to_fp_i1_to_f64(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: uint_to_fp_i1_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, 0x3ff00000, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, s2
+; GFX942-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cmp = icmp eq i32 %in, 0
   %fp = uitofp i1 %cmp to double
   store double %fp, ptr addrspace(1) %out, align 4
@@ -394,6 +512,19 @@ define amdgpu_kernel void @uint_to_fp_i1_to_f64_load(ptr addrspace(1) %out, i1 %
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: uint_to_fp_i1_to_f64_load:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_bitcmp1_b32 s2, 0
+; GFX942-NEXT:    s_cselect_b64 s[2:3], -1, 0
+; GFX942-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[2:3]
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %fp = uitofp i1 %in to double
   store double %fp, ptr addrspace(1) %out, align 8
   ret void
@@ -429,6 +560,17 @@ define amdgpu_kernel void @s_uint_to_fp_i8_to_f64(ptr addrspace(1) %out, i8 %in)
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_uint_to_fp_i8_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v2, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_and_b32 s2, s2, 0xff
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], s2
+; GFX942-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %fp = uitofp i8 %in to double
   store double %fp, ptr addrspace(1) %out
   ret void
@@ -450,6 +592,14 @@ define double @v_uint_to_fp_i8_to_f64(i8 %in) {
 ; VI-NEXT:    v_and_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; VI-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_uint_to_fp_i8_to_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    s_mov_b32 s0, 0xffff
+; GFX942-NEXT:    v_and_b32_sdwa v0, s0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX942-NEXT:    v_cvt_f64_u32_e32 v[0:1], v0
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %fp = uitofp i8 %in to double
   ret double %fp
 }
@@ -488,6 +638,18 @@ define amdgpu_kernel void @s_select_uint_to_fp_i1_vals_f64(ptr addrspace(1) %out
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_select_uint_to_fp_i1_vals_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, 0x3ff00000, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, s2
+; GFX942-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, double 1.0, double 0.0
   store double %select, ptr addrspace(1) %out, align 8
@@ -505,6 +667,18 @@ define void @v_select_uint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in) {
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[3:4]
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_select_uint_to_fp_i1_vals_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v3, 0x3ff00000
+; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_nop 0
+; GFX942-NEXT:    v_cndmask_b32_e32 v5, 0, v3, vcc
+; GFX942-NEXT:    global_store_dwordx2 v[0:1], v[4:5], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, double 1.0, double 0.0
   store double %select, ptr addrspace(1) %out, align 8
@@ -545,6 +719,18 @@ define amdgpu_kernel void @s_select_uint_to_fp_i1_vals_i64(ptr addrspace(1) %out
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_select_uint_to_fp_i1_vals_i64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, 0x3ff00000, 0
+; GFX942-NEXT:    v_mov_b32_e32 v1, s2
+; GFX942-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, i64 u0x3ff0000000000000, i64 0
   store i64 %select, ptr addrspace(1) %out, align 8
@@ -562,6 +748,18 @@ define void @v_select_uint_to_fp_i1_vals_i64(ptr addrspace(1) %out, i32 %in) {
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[3:4]
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_select_uint_to_fp_i1_vals_i64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v3, 0x3ff00000
+; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_nop 0
+; GFX942-NEXT:    v_cndmask_b32_e32 v5, 0, v3, vcc
+; GFX942-NEXT:    global_store_dwordx2 v[0:1], v[4:5], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, i64 u0x3ff0000000000000, i64 0
   store i64 %select, ptr addrspace(1) %out, align 8
@@ -603,6 +801,18 @@ define amdgpu_kernel void @s_swap_select_uint_to_fp_i1_vals_f64(ptr addrspace(1)
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
+;
+; GFX942-LABEL: s_swap_select_uint_to_fp_i1_vals_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_load_dword s2, s[4:5], 0x8
+; GFX942-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT:    v_mov_b32_e32 v0, 0
+; GFX942-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX942-NEXT:    s_cmp_eq_u32 s2, 0
+; GFX942-NEXT:    s_cselect_b32 s2, 0, 0x3ff00000
+; GFX942-NEXT:    v_mov_b32_e32 v1, s2
+; GFX942-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX942-NEXT:    s_endpgm
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, double 0.0, double 1.0
   store double %select, ptr addrspace(1) %out, align 8
@@ -620,6 +830,18 @@ define void @v_swap_select_uint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in
 ; GCN-NEXT:    flat_store_dwordx2 v[0:1], v[3:4]
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_swap_select_uint_to_fp_i1_vals_f64:
+; GFX942:       ; %bb.0:
+; GFX942-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT:    v_mov_b32_e32 v3, 0x3ff00000
+; GFX942-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX942-NEXT:    v_mov_b32_e32 v4, 0
+; GFX942-NEXT:    s_nop 0
+; GFX942-NEXT:    v_cndmask_b32_e64 v5, v3, 0, vcc
+; GFX942-NEXT:    global_store_dwordx2 v[0:1], v[4:5], off
+; GFX942-NEXT:    s_waitcnt vmcnt(0)
+; GFX942-NEXT:    s_setpc_b64 s[30:31]
   %cmp = icmp eq i32 %in, 0
   %select = select i1 %cmp, double 0.0, double 1.0
   store double %select, ptr addrspace(1) %out, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/unify-metadata.ll b/llvm/test/CodeGen/AMDGPU/unify-metadata.ll
deleted file mode 100644
index 455993b16145a..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/unify-metadata.ll
+++ /dev/null
@@ -1,28 +0,0 @@
-; RUN: opt -mtriple=amdgcn--amdhsa -passes=amdgpu-unify-metadata -S < %s | FileCheck -check-prefix=ALL %s
-; RUN: opt -mtriple=amdgcn--amdhsa -passes=amdgpu-unify-metadata -S < %s | FileCheck -check-prefix=ALL %s
-
-; This test check that we have a singe metadata value after linking several
-; modules for records such as opencl.ocl.version, llvm.ident and similar.
-
-; ALL-DAG: !opencl.ocl.version = !{![[OCL_VER:[0-9]+]]}
-; ALL-DAG: !llvm.ident = !{![[LLVM_IDENT_0:[0-9]+]], ![[LLVM_IDENT_1:[0-9]+]]}
-; ALL-DAG: !opencl.used.extensions = !{![[USED_EXT_0:[0-9]+]], ![[USED_EXT_1:[0-9]+]], ![[USED_EXT_2:[0-9]+]]}
-
-; ALL-DAG: ![[OCL_VER]] = !{i32 1, i32 2}
-; ALL-DAG: ![[LLVM_IDENT_0]] = !{!"clang version 4.0"}
-; ALL-DAG: ![[LLVM_IDENT_1]] = !{!"clang version 4.0 (rLXXXXXX)"}
-; ALL-DAG: ![[USED_EXT_0]] = !{!"cl_images"}
-; ALL-DAG: ![[USED_EXT_1]] = !{!"cl_khr_fp16"}
-; ALL-DAG: ![[USED_EXT_2]] = !{!"cl_doubles"}
-
-!opencl.ocl.version = !{!1, !0, !0, !0}
-!llvm.ident = !{!2, !2, !2, !2, !6}
-!opencl.used.extensions = !{!3, !3, !4, !5}
-
-!0 = !{i32 2, i32 0}
-!1 = !{i32 1, i32 2}
-!2 = !{!"clang version 4.0"}
-!3 = !{!"cl_images", !"cl_khr_fp16"}
-!4 = !{!"cl_images", !"cl_doubles"}
-!5 = !{}
-!6 = !{!"clang version 4.0 (rLXXXXXX)"}
diff --git a/llvm/test/CodeGen/AMDGPU/unnamed-function-resource-info.ll b/llvm/test/CodeGen/AMDGPU/unnamed-function-resource-info.ll
index cf5b95a729974..bb0ec0d3ad3f8 100644
--- a/llvm/test/CodeGen/AMDGPU/unnamed-function-resource-info.ll
+++ b/llvm/test/CodeGen/AMDGPU/unnamed-function-resource-info.ll
@@ -16,7 +16,7 @@ entry:
 }
 
 ; CHECK-LABEL: __unnamed_2:
-; CHECK: .set __unnamed_2.num_vgpr, max(32, __unnamed_1.num_vgpr)
+; CHECK: .set __unnamed_2.num_vgpr, max(1, __unnamed_1.num_vgpr)
 ; CHECK: .set __unnamed_2.num_agpr, max(0, __unnamed_1.num_agpr)
 ; CHECK: .set __unnamed_2.numbered_sgpr, max(34, __unnamed_1.numbered_sgpr)
 ; CHECK: .set __unnamed_2.private_seg_size, 16+max(__unnamed_1.private_seg_size)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
index 30ed6ae5484c6..334215125f58a 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
@@ -300,17 +300,15 @@ define i8 @test_vector_reduce_add_v4i8(<4 x i8> %v) {
 ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_add_v4i8:
 ; GFX11-SDAG-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v1.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v0.h
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
+; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
@@ -348,17 +346,15 @@ define i8 @test_vector_reduce_add_v4i8(<4 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v1.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v0.h
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v2
-; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
+; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
@@ -518,21 +514,19 @@ define i8 @test_vector_reduce_add_v8i8(<8 x i8> %v) {
 ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_add_v8i8:
 ; GFX11-SDAG-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v2.l, v6.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.h, v3.l, v7.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v5.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v2.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
@@ -581,21 +575,19 @@ define i8 @test_vector_reduce_add_v8i8(<8 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v2.l, v6.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.h, v3.l, v7.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v5.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v2.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v0.h
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
@@ -832,28 +824,25 @@ define i8 @test_vector_reduce_add_v16i8(<16 x i8> %v) {
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.h, v5.l, v13.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v3.h, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v3.l, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v2.h, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v5.l, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v6.l, v14.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.h, v4.l, v12.l
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v2.h, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v3.l, v4.l, v12.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v1.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v2.l, v2.h
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v2.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v2.l, v2.h, v5.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
 ; GFX11-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
@@ -922,28 +911,25 @@ define i8 @test_vector_reduce_add_v16i8(<16 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.h, v5.l, v13.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v3.h, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v3.l, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v2.h, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v5.l, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v6.l, v14.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.h, v4.l, v12.l
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v2.h, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v3.l, v4.l, v12.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v1.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v2.l, v2.h
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.h, v2.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v2.l, v2.h, v5.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v2.l, 8, v0.h
-; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v1.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v0.h
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
 ; GFX12-SDAG-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
index aab0e76410ccb..1d3b42ee43b0f 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
@@ -374,13 +374,12 @@ define i8 @test_vector_reduce_umin_v4i8(<4 x i8> %v) {
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_min_u16 v1.l, v1.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_min3_u16 v0.l, v0.l, v0.h, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -427,13 +426,12 @@ define i8 @test_vector_reduce_umin_v4i8(<4 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_min_u16 v1.l, v1.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_min3_u16 v0.l, v0.l, v0.h, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -624,22 +622,20 @@ define i8 @test_vector_reduce_umin_v8i8(<8 x i8> %v) {
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v7.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    v_min_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_min_u16 v0.h, v1.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v4.l
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    v_min3_u16 v1.l, v1.l, v3.l, v3.h
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v3
-; GFX11-SDAG-TRUE16-NEXT:    v_min3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_min3_u16 v0.h, v0.h, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_min3_u16 v0.l, v0.l, v2.l, v1.h
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
+; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v3
 ; GFX11-SDAG-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -703,22 +699,20 @@ define i8 @test_vector_reduce_umin_v8i8(<8 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v3.h, 0xff, v7.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v3.l, 0xff, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_min_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v2.l, 0xff, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_min_u16 v0.h, v1.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v4.l
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v6.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT:    v_min3_u16 v1.l, v1.l, v3.l, v3.h
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v2, 0xffff, v3
-; GFX12-SDAG-TRUE16-NEXT:    v_min3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_min3_u16 v0.h, v0.h, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.h, 0
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v3.l, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_min3_u16 v0.l, v0.l, v2.l, v1.h
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
+; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v3
 ; GFX12-SDAG-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1047,14 +1041,12 @@ define i8 @test_vector_reduce_umin_v16i8(<16 x i8> %v) {
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-SDAG-TRUE16-NEXT:    v_min3_u16 v0.l, v0.l, v1.h, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX11-SDAG-TRUE16-NEXT:    v_or_b16 v1.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1176,14 +1168,12 @@ define i8 @test_vector_reduce_umin_v16i8(<16 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX12-SDAG-TRUE16-NEXT:    v_min3_u16 v0.l, v0.l, v1.h, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v0.h, 8, v0.h
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_or_b16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.h, 0
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX12-SDAG-TRUE16-NEXT:    v_or_b16 v1.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_min_u16 v0.l, v0.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
index 2cb5e309c8c21..ee35dc4cddade 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-agpr-limit-gfx90a.ll
@@ -1264,9 +1264,9 @@ define amdgpu_kernel void @k1024_call_no_agprs_ub_callee() #1025 {
 }
 
 ; GCN-LABEL: {{^}}f1024_0:
-; GFX90A: NumVgprs: 32
+; GFX90A: NumVgprs: 1
 ; GFX90A: NumAgprs: 1
-; GFX90A: TotalNumVgprs: 33
+; GFX90A: TotalNumVgprs: 5
 define void @f1024_0() #1024 {
   call void @foo()
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-count-compute.ll b/llvm/test/CodeGen/AMDGPU/vgpr-count-compute.ll
new file mode 100644
index 0000000000000..71d4e6b277320
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-count-compute.ll
@@ -0,0 +1,30 @@
+; RUN: llc -mcpu=gfx1200 < %s | FileCheck %s --check-prefixes=CHECK,PACKED
+; RUN: llc -mcpu=gfx1030 < %s | FileCheck %s --check-prefixes=CHECK,NOTPACKED
+target triple = "amdgcn-amd-amdhsa"
+
+@global = addrspace(1) global i32 poison, align 4
+
+; Carefully crafted kernel that uses v0 but never writes a VGPR or reads another VGPR.
+; Only hardware-initialized VGPRs (v0) are read in this kernel.
+
+; CHECK-LABEL: amdhsa.kernels:
+; CHECK-LABEL: kernel_x
+; CHECK: .vgpr_count:     1
+define amdgpu_kernel void @kernel_x(ptr addrspace(8) %rsrc) #0 {
+entry:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %id, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+  ret void
+}
+
+; CHECK-LABEL: kernel_z
+; PACKED: .vgpr_count:     1
+; NOTPACKED: .vgpr_count:     3
+define amdgpu_kernel void @kernel_z(ptr addrspace(8) %rsrc) {
+entry:
+  %id = call i32 @llvm.amdgcn.workitem.id.z()
+  call void @llvm.amdgcn.raw.ptr.buffer.store.i32(i32 %id, ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0)
+  ret void
+}
+
+attributes #0 = { "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-count-graphics-chain.ll b/llvm/test/CodeGen/AMDGPU/vgpr-count-graphics-chain.ll
new file mode 100644
index 0000000000000..a9f4bda12c557
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-count-graphics-chain.ll
@@ -0,0 +1,26 @@
+; RUN: llc -mcpu=gfx1200 < %s | FileCheck %s
+target triple = "amdgcn--amdpal"
+
+@global = addrspace(1) global i32 poison, align 4
+
+; CHECK-LABEL: amdpal.pipelines:
+
+; Shouldn't report the part of %vgpr_args that's not used
+; CHECK-LABEL: entry_point_symbol: cs_calling_chain
+; CHECK: .vgpr_count:     0xa
+define amdgpu_cs void @cs_calling_chain(i32 %vgpr, i32 inreg %sgpr) {
+  %vgpr_args = insertvalue {i32, i32, i32, i32} poison, i32 %vgpr, 1
+  call void (ptr, i32, i32, {i32, i32, i32, i32}, i32, ...) @llvm.amdgcn.cs.chain.p0.i32.i32.s(
+    ptr @chain_func, i32 0, i32 inreg %sgpr, {i32, i32, i32, i32} %vgpr_args, i32 0)
+  unreachable
+}
+
+; Neither uses not writes a VGPR
+; CHECK-LABEL: chain_func:
+; CHECK: .vgpr_count:     0x1
+define amdgpu_cs_chain void @chain_func([32 x i32] %args) {
+entry:
+  call void (ptr, i32, {}, [32 x i32], i32, ...) @llvm.amdgcn.cs.chain.p0.i32.s.a(
+        ptr @chain_func, i32 0, {} inreg {}, [32 x i32] %args, i32 0)
+  unreachable
+}
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-count-graphics.ll b/llvm/test/CodeGen/AMDGPU/vgpr-count-graphics.ll
new file mode 100644
index 0000000000000..ea19763d61f41
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-count-graphics.ll
@@ -0,0 +1,24 @@
+; RUN: llc -mcpu=gfx1200 < %s | FileCheck %s
+target triple = "amdgcn--amdpal"
+
+@global = addrspace(1) global i32 poison, align 4
+
+; CHECK-LABEL: amdpal.pipelines:
+
+; Neither uses not writes a VGPR, but the hardware initializes the VGPRs that the kernel receives, so they count as used.
+; CHECK-LABEL: .entry_point_symbol: kernel_use
+; CHECK: .vgpr_count:     0x20
+define amdgpu_cs void @kernel_use([32 x i32] %args) {
+entry:
+  %a = extractvalue [32 x i32] %args, 14
+  store i32 %a, ptr addrspace(1) @global
+  ret void
+}
+
+; Neither uses not writes a VGPR
+; CHECK-LABEL: gfx_func:
+; CHECK: .vgpr_count:     0x20
+define amdgpu_gfx [32 x i32] @gfx_func([32 x i32] %args) {
+entry:
+  ret [32 x i32] %args
+}
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-remat.mir b/llvm/test/CodeGen/AMDGPU/vgpr-remat.mir
index 9dad99f307c19..fa4461c1cc387 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-remat.mir
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-remat.mir
@@ -135,3 +135,48 @@ body: |
     S_ENDPGM 0, implicit %4
 
 ...
+
+---
+name: av_mov_imm_b64
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: av_mov_imm_b64
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr0_sgpr1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   undef [[AV_MOV_:%[0-9]+]].sub0_sub1:vreg_192 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+  ; CHECK-NEXT:   [[AV_MOV_:%[0-9]+]].sub2_sub3:vreg_192 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+  ; CHECK-NEXT:   $exec = S_MOV_B64_term [[COPY]]
+  ; CHECK-NEXT:   S_CBRANCH_EXECZ %bb.2, implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[AV_MOV_:%[0-9]+]].sub0_sub1:vreg_192 = V_MUL_F64_e64 0, [[AV_MOV_]].sub0_sub1, 0, [[AV_MOV_]].sub0_sub1, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[AV_MOV_:%[0-9]+]].sub2_sub3:vreg_192 = V_MUL_F64_e64 0, [[AV_MOV_]].sub2_sub3, 0, [[AV_MOV_]].sub2_sub3, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   S_ENDPGM 0, implicit [[AV_MOV_]]
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    %0:av_64 = AV_MOV_B64_IMM_PSEUDO 0, implicit $exec
+    %1:vreg_64 = COPY %0
+    %2:vreg_64 = COPY %0
+    %3:sreg_64 = COPY $sgpr0_sgpr1
+    $exec = S_MOV_B64_term %3:sreg_64
+    S_CBRANCH_EXECZ %bb.2, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    %1:vreg_64 = V_MUL_F64_e64 0, %1:vreg_64, 0, %1:vreg_64, 0, 0, implicit $mode, implicit $exec
+    %2:vreg_64 = V_MUL_F64_e64 0, %2:vreg_64, 0, %2:vreg_64, 0, 0, implicit $mode, implicit $exec
+
+  bb.2:
+    undef %4.sub0_sub1:vreg_192 = COPY %1:vreg_64
+    %4.sub2_sub3:vreg_192 = COPY %2:vreg_64
+    S_ENDPGM 0, implicit %4
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll b/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll
index 77dc32d5fb829..2d7a91f0cd114 100644
--- a/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll
+++ b/llvm/test/CodeGen/AMDGPU/wait-before-stores-with-scope_sys.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
-; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
 
 define amdgpu_ps void @intrinsic_store_system_scope(i32 %val, <4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
 ; GFX12-LABEL: intrinsic_store_system_scope:
diff --git a/llvm/test/CodeGen/AMDGPU/wave_dispatch_regs.ll b/llvm/test/CodeGen/AMDGPU/wave_dispatch_regs.ll
index 76c331cdc8303..e2ef60bb80153 100644
--- a/llvm/test/CodeGen/AMDGPU/wave_dispatch_regs.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave_dispatch_regs.ll
@@ -1,6 +1,9 @@
-; RUN: llc -mtriple=amdgcn--amdpal < %s | FileCheck -check-prefix=GCN -check-prefix=SI -enable-var-scope %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GCN -check-prefix=VI -enable-var-scope %s
-; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -enable-var-scope %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn--amdpal < %s | FileCheck -check-prefix=GCN -check-prefix=SI -enable-var-scope %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn--amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GCN -check-prefix=VI -enable-var-scope %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn--amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -enable-var-scope %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn--amdpal < %s | FileCheck -check-prefix=GCN -check-prefix=SI -enable-var-scope %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn--amdpal -mcpu=tonga < %s | FileCheck -check-prefix=GCN -check-prefix=VI -enable-var-scope %s
+; RUN: llc -global-isel=0 -mtriple=amdgcn--amdpal -mcpu=gfx900 < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -enable-var-scope %s
 
 ; This compute shader has input args that claim that it has 17 sgprs and 5 vgprs
 ; in wave dispatch. Ensure that the sgpr and vgpr counts in COMPUTE_PGM_RSRC1
@@ -17,7 +20,7 @@
 ; GCN-NEXT:         .scratch_memory_size: 0
 ; SI-NEXT:          .sgpr_count:     0x11
 ; VI-NEXT:          .sgpr_count:     0x60
-; GFX9-NEXT:        .sgpr_count:     0x11
+; GFX9-NEXT:        .sgpr_count:     0x15
 ; SI-NEXT:          .vgpr_count:     0x5
 ; VI-NEXT:          .vgpr_count:     0x5
 ; GFX9-NEXT:        .vgpr_count:     0x5
diff --git a/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll b/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
index a13a68a665aee..36e8adb23f1f5 100644
--- a/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
+++ b/llvm/test/CodeGen/AMDGPU/whole-wave-functions.ll
@@ -2412,3 +2412,1427 @@ define amdgpu_gfx_whole_wave <2 x half> @call_gfx_from_whole_wave(i1 %active, <2
   %ret = call amdgpu_gfx <2 x half>(<2 x half>, <2 x half>) @gfx_callee(<2 x half> %y, <2 x half> %x) convergent
   ret <2 x half> %ret
 }
+
+declare amdgpu_gfx_whole_wave float @callee(i1 %active, <8 x float> %x)
+
+define amdgpu_cs void @call_from_entry(<8 x float> %x, ptr %p) {
+; DAGISEL-LABEL: call_from_entry:
+; DAGISEL:       ; %bb.0:
+; DAGISEL-NEXT:    s_mov_b32 s1, callee@abs32@hi
+; DAGISEL-NEXT:    s_mov_b32 s0, callee@abs32@lo
+; DAGISEL-NEXT:    s_mov_b32 s32, 0
+; DAGISEL-NEXT:    v_dual_mov_b32 v41, v9 :: v_dual_mov_b32 v40, v8
+; DAGISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; DAGISEL-NEXT:    flat_store_b32 v[40:41], v0
+; DAGISEL-NEXT:    s_endpgm
+;
+; GISEL-LABEL: call_from_entry:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_mov_b32 s0, callee@abs32@lo
+; GISEL-NEXT:    s_mov_b32 s1, callee@abs32@hi
+; GISEL-NEXT:    s_mov_b32 s32, 0
+; GISEL-NEXT:    v_dual_mov_b32 v40, v8 :: v_dual_mov_b32 v41, v9
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GISEL-NEXT:    flat_store_b32 v[40:41], v0
+; GISEL-NEXT:    s_endpgm
+;
+; DAGISEL64-LABEL: call_from_entry:
+; DAGISEL64:       ; %bb.0:
+; DAGISEL64-NEXT:    s_mov_b32 s1, callee@abs32@hi
+; DAGISEL64-NEXT:    s_mov_b32 s0, callee@abs32@lo
+; DAGISEL64-NEXT:    s_mov_b32 s32, 0
+; DAGISEL64-NEXT:    v_mov_b32_e32 v41, v9
+; DAGISEL64-NEXT:    v_mov_b32_e32 v40, v8
+; DAGISEL64-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; DAGISEL64-NEXT:    flat_store_b32 v[40:41], v0
+; DAGISEL64-NEXT:    s_endpgm
+;
+; GISEL64-LABEL: call_from_entry:
+; GISEL64:       ; %bb.0:
+; GISEL64-NEXT:    s_mov_b32 s0, callee@abs32@lo
+; GISEL64-NEXT:    s_mov_b32 s1, callee@abs32@hi
+; GISEL64-NEXT:    s_mov_b32 s32, 0
+; GISEL64-NEXT:    v_mov_b32_e32 v40, v8
+; GISEL64-NEXT:    v_mov_b32_e32 v41, v9
+; GISEL64-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GISEL64-NEXT:    flat_store_b32 v[40:41], v0
+; GISEL64-NEXT:    s_endpgm
+  %ret = call float(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @callee, <8 x float> %x) convergent
+  store float %ret, ptr %p
+  ret void
+}
+
+define amdgpu_gfx_whole_wave void @call_from_whole_wave(i1 %unused, <8 x float> %x, ptr %p) {
+; DAGISEL-LABEL: call_from_whole_wave:
+; DAGISEL:       ; %bb.0:
+; DAGISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; DAGISEL-NEXT:    s_wait_expcnt 0x0
+; DAGISEL-NEXT:    s_wait_samplecnt 0x0
+; DAGISEL-NEXT:    s_wait_bvhcnt 0x0
+; DAGISEL-NEXT:    s_wait_kmcnt 0x0
+; DAGISEL-NEXT:    s_mov_b32 s0, s33
+; DAGISEL-NEXT:    s_mov_b32 s33, s32
+; DAGISEL-NEXT:    s_xor_saveexec_b32 s4, -1
+; DAGISEL-NEXT:    s_clause 0x1f
+; DAGISEL-NEXT:    scratch_store_b32 off, v0, s33 offset:4
+; DAGISEL-NEXT:    scratch_store_b32 off, v1, s33 offset:8
+; DAGISEL-NEXT:    scratch_store_b32 off, v2, s33 offset:12
+; DAGISEL-NEXT:    scratch_store_b32 off, v3, s33 offset:16
+; DAGISEL-NEXT:    scratch_store_b32 off, v4, s33 offset:20
+; DAGISEL-NEXT:    scratch_store_b32 off, v5, s33 offset:24
+; DAGISEL-NEXT:    scratch_store_b32 off, v6, s33 offset:28
+; DAGISEL-NEXT:    scratch_store_b32 off, v7, s33 offset:32
+; DAGISEL-NEXT:    scratch_store_b32 off, v8, s33 offset:36
+; DAGISEL-NEXT:    scratch_store_b32 off, v9, s33 offset:40
+; DAGISEL-NEXT:    scratch_store_b32 off, v10, s33 offset:44
+; DAGISEL-NEXT:    scratch_store_b32 off, v11, s33 offset:48
+; DAGISEL-NEXT:    scratch_store_b32 off, v12, s33 offset:52
+; DAGISEL-NEXT:    scratch_store_b32 off, v13, s33 offset:56
+; DAGISEL-NEXT:    scratch_store_b32 off, v14, s33 offset:60
+; DAGISEL-NEXT:    scratch_store_b32 off, v15, s33 offset:64
+; DAGISEL-NEXT:    scratch_store_b32 off, v16, s33 offset:68
+; DAGISEL-NEXT:    scratch_store_b32 off, v17, s33 offset:72
+; DAGISEL-NEXT:    scratch_store_b32 off, v18, s33 offset:76
+; DAGISEL-NEXT:    scratch_store_b32 off, v19, s33 offset:80
+; DAGISEL-NEXT:    scratch_store_b32 off, v20, s33 offset:84
+; DAGISEL-NEXT:    scratch_store_b32 off, v21, s33 offset:88
+; DAGISEL-NEXT:    scratch_store_b32 off, v22, s33 offset:92
+; DAGISEL-NEXT:    scratch_store_b32 off, v23, s33 offset:96
+; DAGISEL-NEXT:    scratch_store_b32 off, v24, s33 offset:100
+; DAGISEL-NEXT:    scratch_store_b32 off, v25, s33 offset:104
+; DAGISEL-NEXT:    scratch_store_b32 off, v26, s33 offset:108
+; DAGISEL-NEXT:    scratch_store_b32 off, v27, s33 offset:112
+; DAGISEL-NEXT:    scratch_store_b32 off, v28, s33 offset:116
+; DAGISEL-NEXT:    scratch_store_b32 off, v29, s33 offset:120
+; DAGISEL-NEXT:    scratch_store_b32 off, v30, s33 offset:124
+; DAGISEL-NEXT:    scratch_store_b32 off, v31, s33 offset:128
+; DAGISEL-NEXT:    s_clause 0x1f
+; DAGISEL-NEXT:    scratch_store_b32 off, v32, s33 offset:132
+; DAGISEL-NEXT:    scratch_store_b32 off, v33, s33 offset:136
+; DAGISEL-NEXT:    scratch_store_b32 off, v34, s33 offset:140
+; DAGISEL-NEXT:    scratch_store_b32 off, v35, s33 offset:144
+; DAGISEL-NEXT:    scratch_store_b32 off, v36, s33 offset:148
+; DAGISEL-NEXT:    scratch_store_b32 off, v37, s33 offset:152
+; DAGISEL-NEXT:    scratch_store_b32 off, v38, s33 offset:156
+; DAGISEL-NEXT:    scratch_store_b32 off, v39, s33 offset:160
+; DAGISEL-NEXT:    scratch_store_b32 off, v48, s33 offset:172
+; DAGISEL-NEXT:    scratch_store_b32 off, v49, s33 offset:176
+; DAGISEL-NEXT:    scratch_store_b32 off, v50, s33 offset:180
+; DAGISEL-NEXT:    scratch_store_b32 off, v51, s33 offset:184
+; DAGISEL-NEXT:    scratch_store_b32 off, v52, s33 offset:188
+; DAGISEL-NEXT:    scratch_store_b32 off, v53, s33 offset:192
+; DAGISEL-NEXT:    scratch_store_b32 off, v54, s33 offset:196
+; DAGISEL-NEXT:    scratch_store_b32 off, v55, s33 offset:200
+; DAGISEL-NEXT:    scratch_store_b32 off, v64, s33 offset:204
+; DAGISEL-NEXT:    scratch_store_b32 off, v65, s33 offset:208
+; DAGISEL-NEXT:    scratch_store_b32 off, v66, s33 offset:212
+; DAGISEL-NEXT:    scratch_store_b32 off, v67, s33 offset:216
+; DAGISEL-NEXT:    scratch_store_b32 off, v68, s33 offset:220
+; DAGISEL-NEXT:    scratch_store_b32 off, v69, s33 offset:224
+; DAGISEL-NEXT:    scratch_store_b32 off, v70, s33 offset:228
+; DAGISEL-NEXT:    scratch_store_b32 off, v71, s33 offset:232
+; DAGISEL-NEXT:    scratch_store_b32 off, v80, s33 offset:236
+; DAGISEL-NEXT:    scratch_store_b32 off, v81, s33 offset:240
+; DAGISEL-NEXT:    scratch_store_b32 off, v82, s33 offset:244
+; DAGISEL-NEXT:    scratch_store_b32 off, v83, s33 offset:248
+; DAGISEL-NEXT:    scratch_store_b32 off, v84, s33 offset:252
+; DAGISEL-NEXT:    scratch_store_b32 off, v85, s33 offset:256
+; DAGISEL-NEXT:    scratch_store_b32 off, v86, s33 offset:260
+; DAGISEL-NEXT:    scratch_store_b32 off, v87, s33 offset:264
+; DAGISEL-NEXT:    s_clause 0x1f
+; DAGISEL-NEXT:    scratch_store_b32 off, v96, s33 offset:268
+; DAGISEL-NEXT:    scratch_store_b32 off, v97, s33 offset:272
+; DAGISEL-NEXT:    scratch_store_b32 off, v98, s33 offset:276
+; DAGISEL-NEXT:    scratch_store_b32 off, v99, s33 offset:280
+; DAGISEL-NEXT:    scratch_store_b32 off, v100, s33 offset:284
+; DAGISEL-NEXT:    scratch_store_b32 off, v101, s33 offset:288
+; DAGISEL-NEXT:    scratch_store_b32 off, v102, s33 offset:292
+; DAGISEL-NEXT:    scratch_store_b32 off, v103, s33 offset:296
+; DAGISEL-NEXT:    scratch_store_b32 off, v112, s33 offset:300
+; DAGISEL-NEXT:    scratch_store_b32 off, v113, s33 offset:304
+; DAGISEL-NEXT:    scratch_store_b32 off, v114, s33 offset:308
+; DAGISEL-NEXT:    scratch_store_b32 off, v115, s33 offset:312
+; DAGISEL-NEXT:    scratch_store_b32 off, v116, s33 offset:316
+; DAGISEL-NEXT:    scratch_store_b32 off, v117, s33 offset:320
+; DAGISEL-NEXT:    scratch_store_b32 off, v118, s33 offset:324
+; DAGISEL-NEXT:    scratch_store_b32 off, v119, s33 offset:328
+; DAGISEL-NEXT:    scratch_store_b32 off, v128, s33 offset:332
+; DAGISEL-NEXT:    scratch_store_b32 off, v129, s33 offset:336
+; DAGISEL-NEXT:    scratch_store_b32 off, v130, s33 offset:340
+; DAGISEL-NEXT:    scratch_store_b32 off, v131, s33 offset:344
+; DAGISEL-NEXT:    scratch_store_b32 off, v132, s33 offset:348
+; DAGISEL-NEXT:    scratch_store_b32 off, v133, s33 offset:352
+; DAGISEL-NEXT:    scratch_store_b32 off, v134, s33 offset:356
+; DAGISEL-NEXT:    scratch_store_b32 off, v135, s33 offset:360
+; DAGISEL-NEXT:    scratch_store_b32 off, v144, s33 offset:364
+; DAGISEL-NEXT:    scratch_store_b32 off, v145, s33 offset:368
+; DAGISEL-NEXT:    scratch_store_b32 off, v146, s33 offset:372
+; DAGISEL-NEXT:    scratch_store_b32 off, v147, s33 offset:376
+; DAGISEL-NEXT:    scratch_store_b32 off, v148, s33 offset:380
+; DAGISEL-NEXT:    scratch_store_b32 off, v149, s33 offset:384
+; DAGISEL-NEXT:    scratch_store_b32 off, v150, s33 offset:388
+; DAGISEL-NEXT:    scratch_store_b32 off, v151, s33 offset:392
+; DAGISEL-NEXT:    s_clause 0x1f
+; DAGISEL-NEXT:    scratch_store_b32 off, v160, s33 offset:396
+; DAGISEL-NEXT:    scratch_store_b32 off, v161, s33 offset:400
+; DAGISEL-NEXT:    scratch_store_b32 off, v162, s33 offset:404
+; DAGISEL-NEXT:    scratch_store_b32 off, v163, s33 offset:408
+; DAGISEL-NEXT:    scratch_store_b32 off, v164, s33 offset:412
+; DAGISEL-NEXT:    scratch_store_b32 off, v165, s33 offset:416
+; DAGISEL-NEXT:    scratch_store_b32 off, v166, s33 offset:420
+; DAGISEL-NEXT:    scratch_store_b32 off, v167, s33 offset:424
+; DAGISEL-NEXT:    scratch_store_b32 off, v176, s33 offset:428
+; DAGISEL-NEXT:    scratch_store_b32 off, v177, s33 offset:432
+; DAGISEL-NEXT:    scratch_store_b32 off, v178, s33 offset:436
+; DAGISEL-NEXT:    scratch_store_b32 off, v179, s33 offset:440
+; DAGISEL-NEXT:    scratch_store_b32 off, v180, s33 offset:444
+; DAGISEL-NEXT:    scratch_store_b32 off, v181, s33 offset:448
+; DAGISEL-NEXT:    scratch_store_b32 off, v182, s33 offset:452
+; DAGISEL-NEXT:    scratch_store_b32 off, v183, s33 offset:456
+; DAGISEL-NEXT:    scratch_store_b32 off, v192, s33 offset:460
+; DAGISEL-NEXT:    scratch_store_b32 off, v193, s33 offset:464
+; DAGISEL-NEXT:    scratch_store_b32 off, v194, s33 offset:468
+; DAGISEL-NEXT:    scratch_store_b32 off, v195, s33 offset:472
+; DAGISEL-NEXT:    scratch_store_b32 off, v196, s33 offset:476
+; DAGISEL-NEXT:    scratch_store_b32 off, v197, s33 offset:480
+; DAGISEL-NEXT:    scratch_store_b32 off, v198, s33 offset:484
+; DAGISEL-NEXT:    scratch_store_b32 off, v199, s33 offset:488
+; DAGISEL-NEXT:    scratch_store_b32 off, v208, s33 offset:492
+; DAGISEL-NEXT:    scratch_store_b32 off, v209, s33 offset:496
+; DAGISEL-NEXT:    scratch_store_b32 off, v210, s33 offset:500
+; DAGISEL-NEXT:    scratch_store_b32 off, v211, s33 offset:504
+; DAGISEL-NEXT:    scratch_store_b32 off, v212, s33 offset:508
+; DAGISEL-NEXT:    scratch_store_b32 off, v213, s33 offset:512
+; DAGISEL-NEXT:    scratch_store_b32 off, v214, s33 offset:516
+; DAGISEL-NEXT:    scratch_store_b32 off, v215, s33 offset:520
+; DAGISEL-NEXT:    s_clause 0xf
+; DAGISEL-NEXT:    scratch_store_b32 off, v224, s33 offset:524
+; DAGISEL-NEXT:    scratch_store_b32 off, v225, s33 offset:528
+; DAGISEL-NEXT:    scratch_store_b32 off, v226, s33 offset:532
+; DAGISEL-NEXT:    scratch_store_b32 off, v227, s33 offset:536
+; DAGISEL-NEXT:    scratch_store_b32 off, v228, s33 offset:540
+; DAGISEL-NEXT:    scratch_store_b32 off, v229, s33 offset:544
+; DAGISEL-NEXT:    scratch_store_b32 off, v230, s33 offset:548
+; DAGISEL-NEXT:    scratch_store_b32 off, v231, s33 offset:552
+; DAGISEL-NEXT:    scratch_store_b32 off, v240, s33 offset:556
+; DAGISEL-NEXT:    scratch_store_b32 off, v241, s33 offset:560
+; DAGISEL-NEXT:    scratch_store_b32 off, v242, s33 offset:564
+; DAGISEL-NEXT:    scratch_store_b32 off, v243, s33 offset:568
+; DAGISEL-NEXT:    scratch_store_b32 off, v244, s33 offset:572
+; DAGISEL-NEXT:    scratch_store_b32 off, v245, s33 offset:576
+; DAGISEL-NEXT:    scratch_store_b32 off, v246, s33 offset:580
+; DAGISEL-NEXT:    scratch_store_b32 off, v247, s33 offset:584
+; DAGISEL-NEXT:    s_mov_b32 exec_lo, -1
+; DAGISEL-NEXT:    s_clause 0x2
+; DAGISEL-NEXT:    scratch_store_b32 off, v42, s33
+; DAGISEL-NEXT:    scratch_store_b32 off, v40, s33 offset:164
+; DAGISEL-NEXT:    scratch_store_b32 off, v41, s33 offset:168
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    v_writelane_b32 v42, s0, 3
+; DAGISEL-NEXT:    s_mov_b32 s1, callee@abs32@hi
+; DAGISEL-NEXT:    s_mov_b32 s0, callee@abs32@lo
+; DAGISEL-NEXT:    s_addk_co_i32 s32, 0x250
+; DAGISEL-NEXT:    v_dual_mov_b32 v41, v9 :: v_dual_mov_b32 v40, v8
+; DAGISEL-NEXT:    v_writelane_b32 v42, s4, 0
+; DAGISEL-NEXT:    v_writelane_b32 v42, s30, 1
+; DAGISEL-NEXT:    v_writelane_b32 v42, s31, 2
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; DAGISEL-NEXT:    flat_store_b32 v[40:41], v0
+; DAGISEL-NEXT:    v_readlane_b32 s31, v42, 2
+; DAGISEL-NEXT:    v_readlane_b32 s30, v42, 1
+; DAGISEL-NEXT:    v_readlane_b32 s4, v42, 0
+; DAGISEL-NEXT:    v_readlane_b32 s0, v42, 3
+; DAGISEL-NEXT:    s_clause 0x2
+; DAGISEL-NEXT:    scratch_load_b32 v42, off, s33
+; DAGISEL-NEXT:    scratch_load_b32 v40, off, s33 offset:164
+; DAGISEL-NEXT:    scratch_load_b32 v41, off, s33 offset:168
+; DAGISEL-NEXT:    s_mov_b32 s32, s33
+; DAGISEL-NEXT:    s_xor_b32 exec_lo, s4, -1
+; DAGISEL-NEXT:    s_clause 0x1f
+; DAGISEL-NEXT:    scratch_load_b32 v0, off, s33 offset:4
+; DAGISEL-NEXT:    scratch_load_b32 v1, off, s33 offset:8
+; DAGISEL-NEXT:    scratch_load_b32 v2, off, s33 offset:12
+; DAGISEL-NEXT:    scratch_load_b32 v3, off, s33 offset:16
+; DAGISEL-NEXT:    scratch_load_b32 v4, off, s33 offset:20
+; DAGISEL-NEXT:    scratch_load_b32 v5, off, s33 offset:24
+; DAGISEL-NEXT:    scratch_load_b32 v6, off, s33 offset:28
+; DAGISEL-NEXT:    scratch_load_b32 v7, off, s33 offset:32
+; DAGISEL-NEXT:    scratch_load_b32 v8, off, s33 offset:36
+; DAGISEL-NEXT:    scratch_load_b32 v9, off, s33 offset:40
+; DAGISEL-NEXT:    scratch_load_b32 v10, off, s33 offset:44
+; DAGISEL-NEXT:    scratch_load_b32 v11, off, s33 offset:48
+; DAGISEL-NEXT:    scratch_load_b32 v12, off, s33 offset:52
+; DAGISEL-NEXT:    scratch_load_b32 v13, off, s33 offset:56
+; DAGISEL-NEXT:    scratch_load_b32 v14, off, s33 offset:60
+; DAGISEL-NEXT:    scratch_load_b32 v15, off, s33 offset:64
+; DAGISEL-NEXT:    scratch_load_b32 v16, off, s33 offset:68
+; DAGISEL-NEXT:    scratch_load_b32 v17, off, s33 offset:72
+; DAGISEL-NEXT:    scratch_load_b32 v18, off, s33 offset:76
+; DAGISEL-NEXT:    scratch_load_b32 v19, off, s33 offset:80
+; DAGISEL-NEXT:    scratch_load_b32 v20, off, s33 offset:84
+; DAGISEL-NEXT:    scratch_load_b32 v21, off, s33 offset:88
+; DAGISEL-NEXT:    scratch_load_b32 v22, off, s33 offset:92
+; DAGISEL-NEXT:    scratch_load_b32 v23, off, s33 offset:96
+; DAGISEL-NEXT:    scratch_load_b32 v24, off, s33 offset:100
+; DAGISEL-NEXT:    scratch_load_b32 v25, off, s33 offset:104
+; DAGISEL-NEXT:    scratch_load_b32 v26, off, s33 offset:108
+; DAGISEL-NEXT:    scratch_load_b32 v27, off, s33 offset:112
+; DAGISEL-NEXT:    scratch_load_b32 v28, off, s33 offset:116
+; DAGISEL-NEXT:    scratch_load_b32 v29, off, s33 offset:120
+; DAGISEL-NEXT:    scratch_load_b32 v30, off, s33 offset:124
+; DAGISEL-NEXT:    scratch_load_b32 v31, off, s33 offset:128
+; DAGISEL-NEXT:    s_clause 0x1f
+; DAGISEL-NEXT:    scratch_load_b32 v32, off, s33 offset:132
+; DAGISEL-NEXT:    scratch_load_b32 v33, off, s33 offset:136
+; DAGISEL-NEXT:    scratch_load_b32 v34, off, s33 offset:140
+; DAGISEL-NEXT:    scratch_load_b32 v35, off, s33 offset:144
+; DAGISEL-NEXT:    scratch_load_b32 v36, off, s33 offset:148
+; DAGISEL-NEXT:    scratch_load_b32 v37, off, s33 offset:152
+; DAGISEL-NEXT:    scratch_load_b32 v38, off, s33 offset:156
+; DAGISEL-NEXT:    scratch_load_b32 v39, off, s33 offset:160
+; DAGISEL-NEXT:    scratch_load_b32 v48, off, s33 offset:172
+; DAGISEL-NEXT:    scratch_load_b32 v49, off, s33 offset:176
+; DAGISEL-NEXT:    scratch_load_b32 v50, off, s33 offset:180
+; DAGISEL-NEXT:    scratch_load_b32 v51, off, s33 offset:184
+; DAGISEL-NEXT:    scratch_load_b32 v52, off, s33 offset:188
+; DAGISEL-NEXT:    scratch_load_b32 v53, off, s33 offset:192
+; DAGISEL-NEXT:    scratch_load_b32 v54, off, s33 offset:196
+; DAGISEL-NEXT:    scratch_load_b32 v55, off, s33 offset:200
+; DAGISEL-NEXT:    scratch_load_b32 v64, off, s33 offset:204
+; DAGISEL-NEXT:    scratch_load_b32 v65, off, s33 offset:208
+; DAGISEL-NEXT:    scratch_load_b32 v66, off, s33 offset:212
+; DAGISEL-NEXT:    scratch_load_b32 v67, off, s33 offset:216
+; DAGISEL-NEXT:    scratch_load_b32 v68, off, s33 offset:220
+; DAGISEL-NEXT:    scratch_load_b32 v69, off, s33 offset:224
+; DAGISEL-NEXT:    scratch_load_b32 v70, off, s33 offset:228
+; DAGISEL-NEXT:    scratch_load_b32 v71, off, s33 offset:232
+; DAGISEL-NEXT:    scratch_load_b32 v80, off, s33 offset:236
+; DAGISEL-NEXT:    scratch_load_b32 v81, off, s33 offset:240
+; DAGISEL-NEXT:    scratch_load_b32 v82, off, s33 offset:244
+; DAGISEL-NEXT:    scratch_load_b32 v83, off, s33 offset:248
+; DAGISEL-NEXT:    scratch_load_b32 v84, off, s33 offset:252
+; DAGISEL-NEXT:    scratch_load_b32 v85, off, s33 offset:256
+; DAGISEL-NEXT:    scratch_load_b32 v86, off, s33 offset:260
+; DAGISEL-NEXT:    scratch_load_b32 v87, off, s33 offset:264
+; DAGISEL-NEXT:    s_clause 0x1f
+; DAGISEL-NEXT:    scratch_load_b32 v96, off, s33 offset:268
+; DAGISEL-NEXT:    scratch_load_b32 v97, off, s33 offset:272
+; DAGISEL-NEXT:    scratch_load_b32 v98, off, s33 offset:276
+; DAGISEL-NEXT:    scratch_load_b32 v99, off, s33 offset:280
+; DAGISEL-NEXT:    scratch_load_b32 v100, off, s33 offset:284
+; DAGISEL-NEXT:    scratch_load_b32 v101, off, s33 offset:288
+; DAGISEL-NEXT:    scratch_load_b32 v102, off, s33 offset:292
+; DAGISEL-NEXT:    scratch_load_b32 v103, off, s33 offset:296
+; DAGISEL-NEXT:    scratch_load_b32 v112, off, s33 offset:300
+; DAGISEL-NEXT:    scratch_load_b32 v113, off, s33 offset:304
+; DAGISEL-NEXT:    scratch_load_b32 v114, off, s33 offset:308
+; DAGISEL-NEXT:    scratch_load_b32 v115, off, s33 offset:312
+; DAGISEL-NEXT:    scratch_load_b32 v116, off, s33 offset:316
+; DAGISEL-NEXT:    scratch_load_b32 v117, off, s33 offset:320
+; DAGISEL-NEXT:    scratch_load_b32 v118, off, s33 offset:324
+; DAGISEL-NEXT:    scratch_load_b32 v119, off, s33 offset:328
+; DAGISEL-NEXT:    scratch_load_b32 v128, off, s33 offset:332
+; DAGISEL-NEXT:    scratch_load_b32 v129, off, s33 offset:336
+; DAGISEL-NEXT:    scratch_load_b32 v130, off, s33 offset:340
+; DAGISEL-NEXT:    scratch_load_b32 v131, off, s33 offset:344
+; DAGISEL-NEXT:    scratch_load_b32 v132, off, s33 offset:348
+; DAGISEL-NEXT:    scratch_load_b32 v133, off, s33 offset:352
+; DAGISEL-NEXT:    scratch_load_b32 v134, off, s33 offset:356
+; DAGISEL-NEXT:    scratch_load_b32 v135, off, s33 offset:360
+; DAGISEL-NEXT:    scratch_load_b32 v144, off, s33 offset:364
+; DAGISEL-NEXT:    scratch_load_b32 v145, off, s33 offset:368
+; DAGISEL-NEXT:    scratch_load_b32 v146, off, s33 offset:372
+; DAGISEL-NEXT:    scratch_load_b32 v147, off, s33 offset:376
+; DAGISEL-NEXT:    scratch_load_b32 v148, off, s33 offset:380
+; DAGISEL-NEXT:    scratch_load_b32 v149, off, s33 offset:384
+; DAGISEL-NEXT:    scratch_load_b32 v150, off, s33 offset:388
+; DAGISEL-NEXT:    scratch_load_b32 v151, off, s33 offset:392
+; DAGISEL-NEXT:    s_clause 0x1f
+; DAGISEL-NEXT:    scratch_load_b32 v160, off, s33 offset:396
+; DAGISEL-NEXT:    scratch_load_b32 v161, off, s33 offset:400
+; DAGISEL-NEXT:    scratch_load_b32 v162, off, s33 offset:404
+; DAGISEL-NEXT:    scratch_load_b32 v163, off, s33 offset:408
+; DAGISEL-NEXT:    scratch_load_b32 v164, off, s33 offset:412
+; DAGISEL-NEXT:    scratch_load_b32 v165, off, s33 offset:416
+; DAGISEL-NEXT:    scratch_load_b32 v166, off, s33 offset:420
+; DAGISEL-NEXT:    scratch_load_b32 v167, off, s33 offset:424
+; DAGISEL-NEXT:    scratch_load_b32 v176, off, s33 offset:428
+; DAGISEL-NEXT:    scratch_load_b32 v177, off, s33 offset:432
+; DAGISEL-NEXT:    scratch_load_b32 v178, off, s33 offset:436
+; DAGISEL-NEXT:    scratch_load_b32 v179, off, s33 offset:440
+; DAGISEL-NEXT:    scratch_load_b32 v180, off, s33 offset:444
+; DAGISEL-NEXT:    scratch_load_b32 v181, off, s33 offset:448
+; DAGISEL-NEXT:    scratch_load_b32 v182, off, s33 offset:452
+; DAGISEL-NEXT:    scratch_load_b32 v183, off, s33 offset:456
+; DAGISEL-NEXT:    scratch_load_b32 v192, off, s33 offset:460
+; DAGISEL-NEXT:    scratch_load_b32 v193, off, s33 offset:464
+; DAGISEL-NEXT:    scratch_load_b32 v194, off, s33 offset:468
+; DAGISEL-NEXT:    scratch_load_b32 v195, off, s33 offset:472
+; DAGISEL-NEXT:    scratch_load_b32 v196, off, s33 offset:476
+; DAGISEL-NEXT:    scratch_load_b32 v197, off, s33 offset:480
+; DAGISEL-NEXT:    scratch_load_b32 v198, off, s33 offset:484
+; DAGISEL-NEXT:    scratch_load_b32 v199, off, s33 offset:488
+; DAGISEL-NEXT:    scratch_load_b32 v208, off, s33 offset:492
+; DAGISEL-NEXT:    scratch_load_b32 v209, off, s33 offset:496
+; DAGISEL-NEXT:    scratch_load_b32 v210, off, s33 offset:500
+; DAGISEL-NEXT:    scratch_load_b32 v211, off, s33 offset:504
+; DAGISEL-NEXT:    scratch_load_b32 v212, off, s33 offset:508
+; DAGISEL-NEXT:    scratch_load_b32 v213, off, s33 offset:512
+; DAGISEL-NEXT:    scratch_load_b32 v214, off, s33 offset:516
+; DAGISEL-NEXT:    scratch_load_b32 v215, off, s33 offset:520
+; DAGISEL-NEXT:    s_clause 0xf
+; DAGISEL-NEXT:    scratch_load_b32 v224, off, s33 offset:524
+; DAGISEL-NEXT:    scratch_load_b32 v225, off, s33 offset:528
+; DAGISEL-NEXT:    scratch_load_b32 v226, off, s33 offset:532
+; DAGISEL-NEXT:    scratch_load_b32 v227, off, s33 offset:536
+; DAGISEL-NEXT:    scratch_load_b32 v228, off, s33 offset:540
+; DAGISEL-NEXT:    scratch_load_b32 v229, off, s33 offset:544
+; DAGISEL-NEXT:    scratch_load_b32 v230, off, s33 offset:548
+; DAGISEL-NEXT:    scratch_load_b32 v231, off, s33 offset:552
+; DAGISEL-NEXT:    scratch_load_b32 v240, off, s33 offset:556
+; DAGISEL-NEXT:    scratch_load_b32 v241, off, s33 offset:560
+; DAGISEL-NEXT:    scratch_load_b32 v242, off, s33 offset:564
+; DAGISEL-NEXT:    scratch_load_b32 v243, off, s33 offset:568
+; DAGISEL-NEXT:    scratch_load_b32 v244, off, s33 offset:572
+; DAGISEL-NEXT:    scratch_load_b32 v245, off, s33 offset:576
+; DAGISEL-NEXT:    scratch_load_b32 v246, off, s33 offset:580
+; DAGISEL-NEXT:    scratch_load_b32 v247, off, s33 offset:584
+; DAGISEL-NEXT:    s_mov_b32 exec_lo, s4
+; DAGISEL-NEXT:    s_mov_b32 s33, s0
+; DAGISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; DAGISEL-NEXT:    s_wait_alu 0xfffe
+; DAGISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: call_from_whole_wave:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT:    s_wait_expcnt 0x0
+; GISEL-NEXT:    s_wait_samplecnt 0x0
+; GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GISEL-NEXT:    s_wait_kmcnt 0x0
+; GISEL-NEXT:    s_mov_b32 s0, s33
+; GISEL-NEXT:    s_mov_b32 s33, s32
+; GISEL-NEXT:    s_xor_saveexec_b32 s4, -1
+; GISEL-NEXT:    s_clause 0x1f
+; GISEL-NEXT:    scratch_store_b32 off, v0, s33 offset:4
+; GISEL-NEXT:    scratch_store_b32 off, v1, s33 offset:8
+; GISEL-NEXT:    scratch_store_b32 off, v2, s33 offset:12
+; GISEL-NEXT:    scratch_store_b32 off, v3, s33 offset:16
+; GISEL-NEXT:    scratch_store_b32 off, v4, s33 offset:20
+; GISEL-NEXT:    scratch_store_b32 off, v5, s33 offset:24
+; GISEL-NEXT:    scratch_store_b32 off, v6, s33 offset:28
+; GISEL-NEXT:    scratch_store_b32 off, v7, s33 offset:32
+; GISEL-NEXT:    scratch_store_b32 off, v8, s33 offset:36
+; GISEL-NEXT:    scratch_store_b32 off, v9, s33 offset:40
+; GISEL-NEXT:    scratch_store_b32 off, v10, s33 offset:44
+; GISEL-NEXT:    scratch_store_b32 off, v11, s33 offset:48
+; GISEL-NEXT:    scratch_store_b32 off, v12, s33 offset:52
+; GISEL-NEXT:    scratch_store_b32 off, v13, s33 offset:56
+; GISEL-NEXT:    scratch_store_b32 off, v14, s33 offset:60
+; GISEL-NEXT:    scratch_store_b32 off, v15, s33 offset:64
+; GISEL-NEXT:    scratch_store_b32 off, v16, s33 offset:68
+; GISEL-NEXT:    scratch_store_b32 off, v17, s33 offset:72
+; GISEL-NEXT:    scratch_store_b32 off, v18, s33 offset:76
+; GISEL-NEXT:    scratch_store_b32 off, v19, s33 offset:80
+; GISEL-NEXT:    scratch_store_b32 off, v20, s33 offset:84
+; GISEL-NEXT:    scratch_store_b32 off, v21, s33 offset:88
+; GISEL-NEXT:    scratch_store_b32 off, v22, s33 offset:92
+; GISEL-NEXT:    scratch_store_b32 off, v23, s33 offset:96
+; GISEL-NEXT:    scratch_store_b32 off, v24, s33 offset:100
+; GISEL-NEXT:    scratch_store_b32 off, v25, s33 offset:104
+; GISEL-NEXT:    scratch_store_b32 off, v26, s33 offset:108
+; GISEL-NEXT:    scratch_store_b32 off, v27, s33 offset:112
+; GISEL-NEXT:    scratch_store_b32 off, v28, s33 offset:116
+; GISEL-NEXT:    scratch_store_b32 off, v29, s33 offset:120
+; GISEL-NEXT:    scratch_store_b32 off, v30, s33 offset:124
+; GISEL-NEXT:    scratch_store_b32 off, v31, s33 offset:128
+; GISEL-NEXT:    s_clause 0x1f
+; GISEL-NEXT:    scratch_store_b32 off, v32, s33 offset:132
+; GISEL-NEXT:    scratch_store_b32 off, v33, s33 offset:136
+; GISEL-NEXT:    scratch_store_b32 off, v34, s33 offset:140
+; GISEL-NEXT:    scratch_store_b32 off, v35, s33 offset:144
+; GISEL-NEXT:    scratch_store_b32 off, v36, s33 offset:148
+; GISEL-NEXT:    scratch_store_b32 off, v37, s33 offset:152
+; GISEL-NEXT:    scratch_store_b32 off, v38, s33 offset:156
+; GISEL-NEXT:    scratch_store_b32 off, v39, s33 offset:160
+; GISEL-NEXT:    scratch_store_b32 off, v48, s33 offset:172
+; GISEL-NEXT:    scratch_store_b32 off, v49, s33 offset:176
+; GISEL-NEXT:    scratch_store_b32 off, v50, s33 offset:180
+; GISEL-NEXT:    scratch_store_b32 off, v51, s33 offset:184
+; GISEL-NEXT:    scratch_store_b32 off, v52, s33 offset:188
+; GISEL-NEXT:    scratch_store_b32 off, v53, s33 offset:192
+; GISEL-NEXT:    scratch_store_b32 off, v54, s33 offset:196
+; GISEL-NEXT:    scratch_store_b32 off, v55, s33 offset:200
+; GISEL-NEXT:    scratch_store_b32 off, v64, s33 offset:204
+; GISEL-NEXT:    scratch_store_b32 off, v65, s33 offset:208
+; GISEL-NEXT:    scratch_store_b32 off, v66, s33 offset:212
+; GISEL-NEXT:    scratch_store_b32 off, v67, s33 offset:216
+; GISEL-NEXT:    scratch_store_b32 off, v68, s33 offset:220
+; GISEL-NEXT:    scratch_store_b32 off, v69, s33 offset:224
+; GISEL-NEXT:    scratch_store_b32 off, v70, s33 offset:228
+; GISEL-NEXT:    scratch_store_b32 off, v71, s33 offset:232
+; GISEL-NEXT:    scratch_store_b32 off, v80, s33 offset:236
+; GISEL-NEXT:    scratch_store_b32 off, v81, s33 offset:240
+; GISEL-NEXT:    scratch_store_b32 off, v82, s33 offset:244
+; GISEL-NEXT:    scratch_store_b32 off, v83, s33 offset:248
+; GISEL-NEXT:    scratch_store_b32 off, v84, s33 offset:252
+; GISEL-NEXT:    scratch_store_b32 off, v85, s33 offset:256
+; GISEL-NEXT:    scratch_store_b32 off, v86, s33 offset:260
+; GISEL-NEXT:    scratch_store_b32 off, v87, s33 offset:264
+; GISEL-NEXT:    s_clause 0x1f
+; GISEL-NEXT:    scratch_store_b32 off, v96, s33 offset:268
+; GISEL-NEXT:    scratch_store_b32 off, v97, s33 offset:272
+; GISEL-NEXT:    scratch_store_b32 off, v98, s33 offset:276
+; GISEL-NEXT:    scratch_store_b32 off, v99, s33 offset:280
+; GISEL-NEXT:    scratch_store_b32 off, v100, s33 offset:284
+; GISEL-NEXT:    scratch_store_b32 off, v101, s33 offset:288
+; GISEL-NEXT:    scratch_store_b32 off, v102, s33 offset:292
+; GISEL-NEXT:    scratch_store_b32 off, v103, s33 offset:296
+; GISEL-NEXT:    scratch_store_b32 off, v112, s33 offset:300
+; GISEL-NEXT:    scratch_store_b32 off, v113, s33 offset:304
+; GISEL-NEXT:    scratch_store_b32 off, v114, s33 offset:308
+; GISEL-NEXT:    scratch_store_b32 off, v115, s33 offset:312
+; GISEL-NEXT:    scratch_store_b32 off, v116, s33 offset:316
+; GISEL-NEXT:    scratch_store_b32 off, v117, s33 offset:320
+; GISEL-NEXT:    scratch_store_b32 off, v118, s33 offset:324
+; GISEL-NEXT:    scratch_store_b32 off, v119, s33 offset:328
+; GISEL-NEXT:    scratch_store_b32 off, v128, s33 offset:332
+; GISEL-NEXT:    scratch_store_b32 off, v129, s33 offset:336
+; GISEL-NEXT:    scratch_store_b32 off, v130, s33 offset:340
+; GISEL-NEXT:    scratch_store_b32 off, v131, s33 offset:344
+; GISEL-NEXT:    scratch_store_b32 off, v132, s33 offset:348
+; GISEL-NEXT:    scratch_store_b32 off, v133, s33 offset:352
+; GISEL-NEXT:    scratch_store_b32 off, v134, s33 offset:356
+; GISEL-NEXT:    scratch_store_b32 off, v135, s33 offset:360
+; GISEL-NEXT:    scratch_store_b32 off, v144, s33 offset:364
+; GISEL-NEXT:    scratch_store_b32 off, v145, s33 offset:368
+; GISEL-NEXT:    scratch_store_b32 off, v146, s33 offset:372
+; GISEL-NEXT:    scratch_store_b32 off, v147, s33 offset:376
+; GISEL-NEXT:    scratch_store_b32 off, v148, s33 offset:380
+; GISEL-NEXT:    scratch_store_b32 off, v149, s33 offset:384
+; GISEL-NEXT:    scratch_store_b32 off, v150, s33 offset:388
+; GISEL-NEXT:    scratch_store_b32 off, v151, s33 offset:392
+; GISEL-NEXT:    s_clause 0x1f
+; GISEL-NEXT:    scratch_store_b32 off, v160, s33 offset:396
+; GISEL-NEXT:    scratch_store_b32 off, v161, s33 offset:400
+; GISEL-NEXT:    scratch_store_b32 off, v162, s33 offset:404
+; GISEL-NEXT:    scratch_store_b32 off, v163, s33 offset:408
+; GISEL-NEXT:    scratch_store_b32 off, v164, s33 offset:412
+; GISEL-NEXT:    scratch_store_b32 off, v165, s33 offset:416
+; GISEL-NEXT:    scratch_store_b32 off, v166, s33 offset:420
+; GISEL-NEXT:    scratch_store_b32 off, v167, s33 offset:424
+; GISEL-NEXT:    scratch_store_b32 off, v176, s33 offset:428
+; GISEL-NEXT:    scratch_store_b32 off, v177, s33 offset:432
+; GISEL-NEXT:    scratch_store_b32 off, v178, s33 offset:436
+; GISEL-NEXT:    scratch_store_b32 off, v179, s33 offset:440
+; GISEL-NEXT:    scratch_store_b32 off, v180, s33 offset:444
+; GISEL-NEXT:    scratch_store_b32 off, v181, s33 offset:448
+; GISEL-NEXT:    scratch_store_b32 off, v182, s33 offset:452
+; GISEL-NEXT:    scratch_store_b32 off, v183, s33 offset:456
+; GISEL-NEXT:    scratch_store_b32 off, v192, s33 offset:460
+; GISEL-NEXT:    scratch_store_b32 off, v193, s33 offset:464
+; GISEL-NEXT:    scratch_store_b32 off, v194, s33 offset:468
+; GISEL-NEXT:    scratch_store_b32 off, v195, s33 offset:472
+; GISEL-NEXT:    scratch_store_b32 off, v196, s33 offset:476
+; GISEL-NEXT:    scratch_store_b32 off, v197, s33 offset:480
+; GISEL-NEXT:    scratch_store_b32 off, v198, s33 offset:484
+; GISEL-NEXT:    scratch_store_b32 off, v199, s33 offset:488
+; GISEL-NEXT:    scratch_store_b32 off, v208, s33 offset:492
+; GISEL-NEXT:    scratch_store_b32 off, v209, s33 offset:496
+; GISEL-NEXT:    scratch_store_b32 off, v210, s33 offset:500
+; GISEL-NEXT:    scratch_store_b32 off, v211, s33 offset:504
+; GISEL-NEXT:    scratch_store_b32 off, v212, s33 offset:508
+; GISEL-NEXT:    scratch_store_b32 off, v213, s33 offset:512
+; GISEL-NEXT:    scratch_store_b32 off, v214, s33 offset:516
+; GISEL-NEXT:    scratch_store_b32 off, v215, s33 offset:520
+; GISEL-NEXT:    s_clause 0xf
+; GISEL-NEXT:    scratch_store_b32 off, v224, s33 offset:524
+; GISEL-NEXT:    scratch_store_b32 off, v225, s33 offset:528
+; GISEL-NEXT:    scratch_store_b32 off, v226, s33 offset:532
+; GISEL-NEXT:    scratch_store_b32 off, v227, s33 offset:536
+; GISEL-NEXT:    scratch_store_b32 off, v228, s33 offset:540
+; GISEL-NEXT:    scratch_store_b32 off, v229, s33 offset:544
+; GISEL-NEXT:    scratch_store_b32 off, v230, s33 offset:548
+; GISEL-NEXT:    scratch_store_b32 off, v231, s33 offset:552
+; GISEL-NEXT:    scratch_store_b32 off, v240, s33 offset:556
+; GISEL-NEXT:    scratch_store_b32 off, v241, s33 offset:560
+; GISEL-NEXT:    scratch_store_b32 off, v242, s33 offset:564
+; GISEL-NEXT:    scratch_store_b32 off, v243, s33 offset:568
+; GISEL-NEXT:    scratch_store_b32 off, v244, s33 offset:572
+; GISEL-NEXT:    scratch_store_b32 off, v245, s33 offset:576
+; GISEL-NEXT:    scratch_store_b32 off, v246, s33 offset:580
+; GISEL-NEXT:    scratch_store_b32 off, v247, s33 offset:584
+; GISEL-NEXT:    s_mov_b32 exec_lo, -1
+; GISEL-NEXT:    s_clause 0x2
+; GISEL-NEXT:    scratch_store_b32 off, v42, s33
+; GISEL-NEXT:    scratch_store_b32 off, v40, s33 offset:164
+; GISEL-NEXT:    scratch_store_b32 off, v41, s33 offset:168
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    v_writelane_b32 v42, s0, 3
+; GISEL-NEXT:    s_mov_b32 s0, callee@abs32@lo
+; GISEL-NEXT:    s_mov_b32 s1, callee@abs32@hi
+; GISEL-NEXT:    s_addk_co_i32 s32, 0x250
+; GISEL-NEXT:    v_dual_mov_b32 v40, v8 :: v_dual_mov_b32 v41, v9
+; GISEL-NEXT:    v_writelane_b32 v42, s4, 0
+; GISEL-NEXT:    v_writelane_b32 v42, s30, 1
+; GISEL-NEXT:    v_writelane_b32 v42, s31, 2
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GISEL-NEXT:    flat_store_b32 v[40:41], v0
+; GISEL-NEXT:    v_readlane_b32 s31, v42, 2
+; GISEL-NEXT:    v_readlane_b32 s30, v42, 1
+; GISEL-NEXT:    v_readlane_b32 s4, v42, 0
+; GISEL-NEXT:    v_readlane_b32 s0, v42, 3
+; GISEL-NEXT:    s_clause 0x2
+; GISEL-NEXT:    scratch_load_b32 v42, off, s33
+; GISEL-NEXT:    scratch_load_b32 v40, off, s33 offset:164
+; GISEL-NEXT:    scratch_load_b32 v41, off, s33 offset:168
+; GISEL-NEXT:    s_mov_b32 s32, s33
+; GISEL-NEXT:    s_xor_b32 exec_lo, s4, -1
+; GISEL-NEXT:    s_clause 0x1f
+; GISEL-NEXT:    scratch_load_b32 v0, off, s33 offset:4
+; GISEL-NEXT:    scratch_load_b32 v1, off, s33 offset:8
+; GISEL-NEXT:    scratch_load_b32 v2, off, s33 offset:12
+; GISEL-NEXT:    scratch_load_b32 v3, off, s33 offset:16
+; GISEL-NEXT:    scratch_load_b32 v4, off, s33 offset:20
+; GISEL-NEXT:    scratch_load_b32 v5, off, s33 offset:24
+; GISEL-NEXT:    scratch_load_b32 v6, off, s33 offset:28
+; GISEL-NEXT:    scratch_load_b32 v7, off, s33 offset:32
+; GISEL-NEXT:    scratch_load_b32 v8, off, s33 offset:36
+; GISEL-NEXT:    scratch_load_b32 v9, off, s33 offset:40
+; GISEL-NEXT:    scratch_load_b32 v10, off, s33 offset:44
+; GISEL-NEXT:    scratch_load_b32 v11, off, s33 offset:48
+; GISEL-NEXT:    scratch_load_b32 v12, off, s33 offset:52
+; GISEL-NEXT:    scratch_load_b32 v13, off, s33 offset:56
+; GISEL-NEXT:    scratch_load_b32 v14, off, s33 offset:60
+; GISEL-NEXT:    scratch_load_b32 v15, off, s33 offset:64
+; GISEL-NEXT:    scratch_load_b32 v16, off, s33 offset:68
+; GISEL-NEXT:    scratch_load_b32 v17, off, s33 offset:72
+; GISEL-NEXT:    scratch_load_b32 v18, off, s33 offset:76
+; GISEL-NEXT:    scratch_load_b32 v19, off, s33 offset:80
+; GISEL-NEXT:    scratch_load_b32 v20, off, s33 offset:84
+; GISEL-NEXT:    scratch_load_b32 v21, off, s33 offset:88
+; GISEL-NEXT:    scratch_load_b32 v22, off, s33 offset:92
+; GISEL-NEXT:    scratch_load_b32 v23, off, s33 offset:96
+; GISEL-NEXT:    scratch_load_b32 v24, off, s33 offset:100
+; GISEL-NEXT:    scratch_load_b32 v25, off, s33 offset:104
+; GISEL-NEXT:    scratch_load_b32 v26, off, s33 offset:108
+; GISEL-NEXT:    scratch_load_b32 v27, off, s33 offset:112
+; GISEL-NEXT:    scratch_load_b32 v28, off, s33 offset:116
+; GISEL-NEXT:    scratch_load_b32 v29, off, s33 offset:120
+; GISEL-NEXT:    scratch_load_b32 v30, off, s33 offset:124
+; GISEL-NEXT:    scratch_load_b32 v31, off, s33 offset:128
+; GISEL-NEXT:    s_clause 0x1f
+; GISEL-NEXT:    scratch_load_b32 v32, off, s33 offset:132
+; GISEL-NEXT:    scratch_load_b32 v33, off, s33 offset:136
+; GISEL-NEXT:    scratch_load_b32 v34, off, s33 offset:140
+; GISEL-NEXT:    scratch_load_b32 v35, off, s33 offset:144
+; GISEL-NEXT:    scratch_load_b32 v36, off, s33 offset:148
+; GISEL-NEXT:    scratch_load_b32 v37, off, s33 offset:152
+; GISEL-NEXT:    scratch_load_b32 v38, off, s33 offset:156
+; GISEL-NEXT:    scratch_load_b32 v39, off, s33 offset:160
+; GISEL-NEXT:    scratch_load_b32 v48, off, s33 offset:172
+; GISEL-NEXT:    scratch_load_b32 v49, off, s33 offset:176
+; GISEL-NEXT:    scratch_load_b32 v50, off, s33 offset:180
+; GISEL-NEXT:    scratch_load_b32 v51, off, s33 offset:184
+; GISEL-NEXT:    scratch_load_b32 v52, off, s33 offset:188
+; GISEL-NEXT:    scratch_load_b32 v53, off, s33 offset:192
+; GISEL-NEXT:    scratch_load_b32 v54, off, s33 offset:196
+; GISEL-NEXT:    scratch_load_b32 v55, off, s33 offset:200
+; GISEL-NEXT:    scratch_load_b32 v64, off, s33 offset:204
+; GISEL-NEXT:    scratch_load_b32 v65, off, s33 offset:208
+; GISEL-NEXT:    scratch_load_b32 v66, off, s33 offset:212
+; GISEL-NEXT:    scratch_load_b32 v67, off, s33 offset:216
+; GISEL-NEXT:    scratch_load_b32 v68, off, s33 offset:220
+; GISEL-NEXT:    scratch_load_b32 v69, off, s33 offset:224
+; GISEL-NEXT:    scratch_load_b32 v70, off, s33 offset:228
+; GISEL-NEXT:    scratch_load_b32 v71, off, s33 offset:232
+; GISEL-NEXT:    scratch_load_b32 v80, off, s33 offset:236
+; GISEL-NEXT:    scratch_load_b32 v81, off, s33 offset:240
+; GISEL-NEXT:    scratch_load_b32 v82, off, s33 offset:244
+; GISEL-NEXT:    scratch_load_b32 v83, off, s33 offset:248
+; GISEL-NEXT:    scratch_load_b32 v84, off, s33 offset:252
+; GISEL-NEXT:    scratch_load_b32 v85, off, s33 offset:256
+; GISEL-NEXT:    scratch_load_b32 v86, off, s33 offset:260
+; GISEL-NEXT:    scratch_load_b32 v87, off, s33 offset:264
+; GISEL-NEXT:    s_clause 0x1f
+; GISEL-NEXT:    scratch_load_b32 v96, off, s33 offset:268
+; GISEL-NEXT:    scratch_load_b32 v97, off, s33 offset:272
+; GISEL-NEXT:    scratch_load_b32 v98, off, s33 offset:276
+; GISEL-NEXT:    scratch_load_b32 v99, off, s33 offset:280
+; GISEL-NEXT:    scratch_load_b32 v100, off, s33 offset:284
+; GISEL-NEXT:    scratch_load_b32 v101, off, s33 offset:288
+; GISEL-NEXT:    scratch_load_b32 v102, off, s33 offset:292
+; GISEL-NEXT:    scratch_load_b32 v103, off, s33 offset:296
+; GISEL-NEXT:    scratch_load_b32 v112, off, s33 offset:300
+; GISEL-NEXT:    scratch_load_b32 v113, off, s33 offset:304
+; GISEL-NEXT:    scratch_load_b32 v114, off, s33 offset:308
+; GISEL-NEXT:    scratch_load_b32 v115, off, s33 offset:312
+; GISEL-NEXT:    scratch_load_b32 v116, off, s33 offset:316
+; GISEL-NEXT:    scratch_load_b32 v117, off, s33 offset:320
+; GISEL-NEXT:    scratch_load_b32 v118, off, s33 offset:324
+; GISEL-NEXT:    scratch_load_b32 v119, off, s33 offset:328
+; GISEL-NEXT:    scratch_load_b32 v128, off, s33 offset:332
+; GISEL-NEXT:    scratch_load_b32 v129, off, s33 offset:336
+; GISEL-NEXT:    scratch_load_b32 v130, off, s33 offset:340
+; GISEL-NEXT:    scratch_load_b32 v131, off, s33 offset:344
+; GISEL-NEXT:    scratch_load_b32 v132, off, s33 offset:348
+; GISEL-NEXT:    scratch_load_b32 v133, off, s33 offset:352
+; GISEL-NEXT:    scratch_load_b32 v134, off, s33 offset:356
+; GISEL-NEXT:    scratch_load_b32 v135, off, s33 offset:360
+; GISEL-NEXT:    scratch_load_b32 v144, off, s33 offset:364
+; GISEL-NEXT:    scratch_load_b32 v145, off, s33 offset:368
+; GISEL-NEXT:    scratch_load_b32 v146, off, s33 offset:372
+; GISEL-NEXT:    scratch_load_b32 v147, off, s33 offset:376
+; GISEL-NEXT:    scratch_load_b32 v148, off, s33 offset:380
+; GISEL-NEXT:    scratch_load_b32 v149, off, s33 offset:384
+; GISEL-NEXT:    scratch_load_b32 v150, off, s33 offset:388
+; GISEL-NEXT:    scratch_load_b32 v151, off, s33 offset:392
+; GISEL-NEXT:    s_clause 0x1f
+; GISEL-NEXT:    scratch_load_b32 v160, off, s33 offset:396
+; GISEL-NEXT:    scratch_load_b32 v161, off, s33 offset:400
+; GISEL-NEXT:    scratch_load_b32 v162, off, s33 offset:404
+; GISEL-NEXT:    scratch_load_b32 v163, off, s33 offset:408
+; GISEL-NEXT:    scratch_load_b32 v164, off, s33 offset:412
+; GISEL-NEXT:    scratch_load_b32 v165, off, s33 offset:416
+; GISEL-NEXT:    scratch_load_b32 v166, off, s33 offset:420
+; GISEL-NEXT:    scratch_load_b32 v167, off, s33 offset:424
+; GISEL-NEXT:    scratch_load_b32 v176, off, s33 offset:428
+; GISEL-NEXT:    scratch_load_b32 v177, off, s33 offset:432
+; GISEL-NEXT:    scratch_load_b32 v178, off, s33 offset:436
+; GISEL-NEXT:    scratch_load_b32 v179, off, s33 offset:440
+; GISEL-NEXT:    scratch_load_b32 v180, off, s33 offset:444
+; GISEL-NEXT:    scratch_load_b32 v181, off, s33 offset:448
+; GISEL-NEXT:    scratch_load_b32 v182, off, s33 offset:452
+; GISEL-NEXT:    scratch_load_b32 v183, off, s33 offset:456
+; GISEL-NEXT:    scratch_load_b32 v192, off, s33 offset:460
+; GISEL-NEXT:    scratch_load_b32 v193, off, s33 offset:464
+; GISEL-NEXT:    scratch_load_b32 v194, off, s33 offset:468
+; GISEL-NEXT:    scratch_load_b32 v195, off, s33 offset:472
+; GISEL-NEXT:    scratch_load_b32 v196, off, s33 offset:476
+; GISEL-NEXT:    scratch_load_b32 v197, off, s33 offset:480
+; GISEL-NEXT:    scratch_load_b32 v198, off, s33 offset:484
+; GISEL-NEXT:    scratch_load_b32 v199, off, s33 offset:488
+; GISEL-NEXT:    scratch_load_b32 v208, off, s33 offset:492
+; GISEL-NEXT:    scratch_load_b32 v209, off, s33 offset:496
+; GISEL-NEXT:    scratch_load_b32 v210, off, s33 offset:500
+; GISEL-NEXT:    scratch_load_b32 v211, off, s33 offset:504
+; GISEL-NEXT:    scratch_load_b32 v212, off, s33 offset:508
+; GISEL-NEXT:    scratch_load_b32 v213, off, s33 offset:512
+; GISEL-NEXT:    scratch_load_b32 v214, off, s33 offset:516
+; GISEL-NEXT:    scratch_load_b32 v215, off, s33 offset:520
+; GISEL-NEXT:    s_clause 0xf
+; GISEL-NEXT:    scratch_load_b32 v224, off, s33 offset:524
+; GISEL-NEXT:    scratch_load_b32 v225, off, s33 offset:528
+; GISEL-NEXT:    scratch_load_b32 v226, off, s33 offset:532
+; GISEL-NEXT:    scratch_load_b32 v227, off, s33 offset:536
+; GISEL-NEXT:    scratch_load_b32 v228, off, s33 offset:540
+; GISEL-NEXT:    scratch_load_b32 v229, off, s33 offset:544
+; GISEL-NEXT:    scratch_load_b32 v230, off, s33 offset:548
+; GISEL-NEXT:    scratch_load_b32 v231, off, s33 offset:552
+; GISEL-NEXT:    scratch_load_b32 v240, off, s33 offset:556
+; GISEL-NEXT:    scratch_load_b32 v241, off, s33 offset:560
+; GISEL-NEXT:    scratch_load_b32 v242, off, s33 offset:564
+; GISEL-NEXT:    scratch_load_b32 v243, off, s33 offset:568
+; GISEL-NEXT:    scratch_load_b32 v244, off, s33 offset:572
+; GISEL-NEXT:    scratch_load_b32 v245, off, s33 offset:576
+; GISEL-NEXT:    scratch_load_b32 v246, off, s33 offset:580
+; GISEL-NEXT:    scratch_load_b32 v247, off, s33 offset:584
+; GISEL-NEXT:    s_mov_b32 exec_lo, s4
+; GISEL-NEXT:    s_mov_b32 s33, s0
+; GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL-NEXT:    s_wait_alu 0xfffe
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; DAGISEL64-LABEL: call_from_whole_wave:
+; DAGISEL64:       ; %bb.0:
+; DAGISEL64-NEXT:    s_wait_loadcnt_dscnt 0x0
+; DAGISEL64-NEXT:    s_wait_expcnt 0x0
+; DAGISEL64-NEXT:    s_wait_samplecnt 0x0
+; DAGISEL64-NEXT:    s_wait_bvhcnt 0x0
+; DAGISEL64-NEXT:    s_wait_kmcnt 0x0
+; DAGISEL64-NEXT:    s_mov_b32 s0, s33
+; DAGISEL64-NEXT:    s_mov_b32 s33, s32
+; DAGISEL64-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; DAGISEL64-NEXT:    s_clause 0x1f
+; DAGISEL64-NEXT:    scratch_store_b32 off, v0, s33 offset:4
+; DAGISEL64-NEXT:    scratch_store_b32 off, v1, s33 offset:8
+; DAGISEL64-NEXT:    scratch_store_b32 off, v2, s33 offset:12
+; DAGISEL64-NEXT:    scratch_store_b32 off, v3, s33 offset:16
+; DAGISEL64-NEXT:    scratch_store_b32 off, v4, s33 offset:20
+; DAGISEL64-NEXT:    scratch_store_b32 off, v5, s33 offset:24
+; DAGISEL64-NEXT:    scratch_store_b32 off, v6, s33 offset:28
+; DAGISEL64-NEXT:    scratch_store_b32 off, v7, s33 offset:32
+; DAGISEL64-NEXT:    scratch_store_b32 off, v8, s33 offset:36
+; DAGISEL64-NEXT:    scratch_store_b32 off, v9, s33 offset:40
+; DAGISEL64-NEXT:    scratch_store_b32 off, v10, s33 offset:44
+; DAGISEL64-NEXT:    scratch_store_b32 off, v11, s33 offset:48
+; DAGISEL64-NEXT:    scratch_store_b32 off, v12, s33 offset:52
+; DAGISEL64-NEXT:    scratch_store_b32 off, v13, s33 offset:56
+; DAGISEL64-NEXT:    scratch_store_b32 off, v14, s33 offset:60
+; DAGISEL64-NEXT:    scratch_store_b32 off, v15, s33 offset:64
+; DAGISEL64-NEXT:    scratch_store_b32 off, v16, s33 offset:68
+; DAGISEL64-NEXT:    scratch_store_b32 off, v17, s33 offset:72
+; DAGISEL64-NEXT:    scratch_store_b32 off, v18, s33 offset:76
+; DAGISEL64-NEXT:    scratch_store_b32 off, v19, s33 offset:80
+; DAGISEL64-NEXT:    scratch_store_b32 off, v20, s33 offset:84
+; DAGISEL64-NEXT:    scratch_store_b32 off, v21, s33 offset:88
+; DAGISEL64-NEXT:    scratch_store_b32 off, v22, s33 offset:92
+; DAGISEL64-NEXT:    scratch_store_b32 off, v23, s33 offset:96
+; DAGISEL64-NEXT:    scratch_store_b32 off, v24, s33 offset:100
+; DAGISEL64-NEXT:    scratch_store_b32 off, v25, s33 offset:104
+; DAGISEL64-NEXT:    scratch_store_b32 off, v26, s33 offset:108
+; DAGISEL64-NEXT:    scratch_store_b32 off, v27, s33 offset:112
+; DAGISEL64-NEXT:    scratch_store_b32 off, v28, s33 offset:116
+; DAGISEL64-NEXT:    scratch_store_b32 off, v29, s33 offset:120
+; DAGISEL64-NEXT:    scratch_store_b32 off, v30, s33 offset:124
+; DAGISEL64-NEXT:    scratch_store_b32 off, v31, s33 offset:128
+; DAGISEL64-NEXT:    s_clause 0x1f
+; DAGISEL64-NEXT:    scratch_store_b32 off, v32, s33 offset:132
+; DAGISEL64-NEXT:    scratch_store_b32 off, v33, s33 offset:136
+; DAGISEL64-NEXT:    scratch_store_b32 off, v34, s33 offset:140
+; DAGISEL64-NEXT:    scratch_store_b32 off, v35, s33 offset:144
+; DAGISEL64-NEXT:    scratch_store_b32 off, v36, s33 offset:148
+; DAGISEL64-NEXT:    scratch_store_b32 off, v37, s33 offset:152
+; DAGISEL64-NEXT:    scratch_store_b32 off, v38, s33 offset:156
+; DAGISEL64-NEXT:    scratch_store_b32 off, v39, s33 offset:160
+; DAGISEL64-NEXT:    scratch_store_b32 off, v48, s33 offset:172
+; DAGISEL64-NEXT:    scratch_store_b32 off, v49, s33 offset:176
+; DAGISEL64-NEXT:    scratch_store_b32 off, v50, s33 offset:180
+; DAGISEL64-NEXT:    scratch_store_b32 off, v51, s33 offset:184
+; DAGISEL64-NEXT:    scratch_store_b32 off, v52, s33 offset:188
+; DAGISEL64-NEXT:    scratch_store_b32 off, v53, s33 offset:192
+; DAGISEL64-NEXT:    scratch_store_b32 off, v54, s33 offset:196
+; DAGISEL64-NEXT:    scratch_store_b32 off, v55, s33 offset:200
+; DAGISEL64-NEXT:    scratch_store_b32 off, v64, s33 offset:204
+; DAGISEL64-NEXT:    scratch_store_b32 off, v65, s33 offset:208
+; DAGISEL64-NEXT:    scratch_store_b32 off, v66, s33 offset:212
+; DAGISEL64-NEXT:    scratch_store_b32 off, v67, s33 offset:216
+; DAGISEL64-NEXT:    scratch_store_b32 off, v68, s33 offset:220
+; DAGISEL64-NEXT:    scratch_store_b32 off, v69, s33 offset:224
+; DAGISEL64-NEXT:    scratch_store_b32 off, v70, s33 offset:228
+; DAGISEL64-NEXT:    scratch_store_b32 off, v71, s33 offset:232
+; DAGISEL64-NEXT:    scratch_store_b32 off, v80, s33 offset:236
+; DAGISEL64-NEXT:    scratch_store_b32 off, v81, s33 offset:240
+; DAGISEL64-NEXT:    scratch_store_b32 off, v82, s33 offset:244
+; DAGISEL64-NEXT:    scratch_store_b32 off, v83, s33 offset:248
+; DAGISEL64-NEXT:    scratch_store_b32 off, v84, s33 offset:252
+; DAGISEL64-NEXT:    scratch_store_b32 off, v85, s33 offset:256
+; DAGISEL64-NEXT:    scratch_store_b32 off, v86, s33 offset:260
+; DAGISEL64-NEXT:    scratch_store_b32 off, v87, s33 offset:264
+; DAGISEL64-NEXT:    s_clause 0x1f
+; DAGISEL64-NEXT:    scratch_store_b32 off, v96, s33 offset:268
+; DAGISEL64-NEXT:    scratch_store_b32 off, v97, s33 offset:272
+; DAGISEL64-NEXT:    scratch_store_b32 off, v98, s33 offset:276
+; DAGISEL64-NEXT:    scratch_store_b32 off, v99, s33 offset:280
+; DAGISEL64-NEXT:    scratch_store_b32 off, v100, s33 offset:284
+; DAGISEL64-NEXT:    scratch_store_b32 off, v101, s33 offset:288
+; DAGISEL64-NEXT:    scratch_store_b32 off, v102, s33 offset:292
+; DAGISEL64-NEXT:    scratch_store_b32 off, v103, s33 offset:296
+; DAGISEL64-NEXT:    scratch_store_b32 off, v112, s33 offset:300
+; DAGISEL64-NEXT:    scratch_store_b32 off, v113, s33 offset:304
+; DAGISEL64-NEXT:    scratch_store_b32 off, v114, s33 offset:308
+; DAGISEL64-NEXT:    scratch_store_b32 off, v115, s33 offset:312
+; DAGISEL64-NEXT:    scratch_store_b32 off, v116, s33 offset:316
+; DAGISEL64-NEXT:    scratch_store_b32 off, v117, s33 offset:320
+; DAGISEL64-NEXT:    scratch_store_b32 off, v118, s33 offset:324
+; DAGISEL64-NEXT:    scratch_store_b32 off, v119, s33 offset:328
+; DAGISEL64-NEXT:    scratch_store_b32 off, v128, s33 offset:332
+; DAGISEL64-NEXT:    scratch_store_b32 off, v129, s33 offset:336
+; DAGISEL64-NEXT:    scratch_store_b32 off, v130, s33 offset:340
+; DAGISEL64-NEXT:    scratch_store_b32 off, v131, s33 offset:344
+; DAGISEL64-NEXT:    scratch_store_b32 off, v132, s33 offset:348
+; DAGISEL64-NEXT:    scratch_store_b32 off, v133, s33 offset:352
+; DAGISEL64-NEXT:    scratch_store_b32 off, v134, s33 offset:356
+; DAGISEL64-NEXT:    scratch_store_b32 off, v135, s33 offset:360
+; DAGISEL64-NEXT:    scratch_store_b32 off, v144, s33 offset:364
+; DAGISEL64-NEXT:    scratch_store_b32 off, v145, s33 offset:368
+; DAGISEL64-NEXT:    scratch_store_b32 off, v146, s33 offset:372
+; DAGISEL64-NEXT:    scratch_store_b32 off, v147, s33 offset:376
+; DAGISEL64-NEXT:    scratch_store_b32 off, v148, s33 offset:380
+; DAGISEL64-NEXT:    scratch_store_b32 off, v149, s33 offset:384
+; DAGISEL64-NEXT:    scratch_store_b32 off, v150, s33 offset:388
+; DAGISEL64-NEXT:    scratch_store_b32 off, v151, s33 offset:392
+; DAGISEL64-NEXT:    s_clause 0x1f
+; DAGISEL64-NEXT:    scratch_store_b32 off, v160, s33 offset:396
+; DAGISEL64-NEXT:    scratch_store_b32 off, v161, s33 offset:400
+; DAGISEL64-NEXT:    scratch_store_b32 off, v162, s33 offset:404
+; DAGISEL64-NEXT:    scratch_store_b32 off, v163, s33 offset:408
+; DAGISEL64-NEXT:    scratch_store_b32 off, v164, s33 offset:412
+; DAGISEL64-NEXT:    scratch_store_b32 off, v165, s33 offset:416
+; DAGISEL64-NEXT:    scratch_store_b32 off, v166, s33 offset:420
+; DAGISEL64-NEXT:    scratch_store_b32 off, v167, s33 offset:424
+; DAGISEL64-NEXT:    scratch_store_b32 off, v176, s33 offset:428
+; DAGISEL64-NEXT:    scratch_store_b32 off, v177, s33 offset:432
+; DAGISEL64-NEXT:    scratch_store_b32 off, v178, s33 offset:436
+; DAGISEL64-NEXT:    scratch_store_b32 off, v179, s33 offset:440
+; DAGISEL64-NEXT:    scratch_store_b32 off, v180, s33 offset:444
+; DAGISEL64-NEXT:    scratch_store_b32 off, v181, s33 offset:448
+; DAGISEL64-NEXT:    scratch_store_b32 off, v182, s33 offset:452
+; DAGISEL64-NEXT:    scratch_store_b32 off, v183, s33 offset:456
+; DAGISEL64-NEXT:    scratch_store_b32 off, v192, s33 offset:460
+; DAGISEL64-NEXT:    scratch_store_b32 off, v193, s33 offset:464
+; DAGISEL64-NEXT:    scratch_store_b32 off, v194, s33 offset:468
+; DAGISEL64-NEXT:    scratch_store_b32 off, v195, s33 offset:472
+; DAGISEL64-NEXT:    scratch_store_b32 off, v196, s33 offset:476
+; DAGISEL64-NEXT:    scratch_store_b32 off, v197, s33 offset:480
+; DAGISEL64-NEXT:    scratch_store_b32 off, v198, s33 offset:484
+; DAGISEL64-NEXT:    scratch_store_b32 off, v199, s33 offset:488
+; DAGISEL64-NEXT:    scratch_store_b32 off, v208, s33 offset:492
+; DAGISEL64-NEXT:    scratch_store_b32 off, v209, s33 offset:496
+; DAGISEL64-NEXT:    scratch_store_b32 off, v210, s33 offset:500
+; DAGISEL64-NEXT:    scratch_store_b32 off, v211, s33 offset:504
+; DAGISEL64-NEXT:    scratch_store_b32 off, v212, s33 offset:508
+; DAGISEL64-NEXT:    scratch_store_b32 off, v213, s33 offset:512
+; DAGISEL64-NEXT:    scratch_store_b32 off, v214, s33 offset:516
+; DAGISEL64-NEXT:    scratch_store_b32 off, v215, s33 offset:520
+; DAGISEL64-NEXT:    s_clause 0xf
+; DAGISEL64-NEXT:    scratch_store_b32 off, v224, s33 offset:524
+; DAGISEL64-NEXT:    scratch_store_b32 off, v225, s33 offset:528
+; DAGISEL64-NEXT:    scratch_store_b32 off, v226, s33 offset:532
+; DAGISEL64-NEXT:    scratch_store_b32 off, v227, s33 offset:536
+; DAGISEL64-NEXT:    scratch_store_b32 off, v228, s33 offset:540
+; DAGISEL64-NEXT:    scratch_store_b32 off, v229, s33 offset:544
+; DAGISEL64-NEXT:    scratch_store_b32 off, v230, s33 offset:548
+; DAGISEL64-NEXT:    scratch_store_b32 off, v231, s33 offset:552
+; DAGISEL64-NEXT:    scratch_store_b32 off, v240, s33 offset:556
+; DAGISEL64-NEXT:    scratch_store_b32 off, v241, s33 offset:560
+; DAGISEL64-NEXT:    scratch_store_b32 off, v242, s33 offset:564
+; DAGISEL64-NEXT:    scratch_store_b32 off, v243, s33 offset:568
+; DAGISEL64-NEXT:    scratch_store_b32 off, v244, s33 offset:572
+; DAGISEL64-NEXT:    scratch_store_b32 off, v245, s33 offset:576
+; DAGISEL64-NEXT:    scratch_store_b32 off, v246, s33 offset:580
+; DAGISEL64-NEXT:    scratch_store_b32 off, v247, s33 offset:584
+; DAGISEL64-NEXT:    s_mov_b64 exec, -1
+; DAGISEL64-NEXT:    s_clause 0x2
+; DAGISEL64-NEXT:    scratch_store_b32 off, v42, s33
+; DAGISEL64-NEXT:    scratch_store_b32 off, v40, s33 offset:164
+; DAGISEL64-NEXT:    scratch_store_b32 off, v41, s33 offset:168
+; DAGISEL64-NEXT:    s_wait_alu 0xfffe
+; DAGISEL64-NEXT:    v_writelane_b32 v42, s0, 4
+; DAGISEL64-NEXT:    s_mov_b32 s1, callee@abs32@hi
+; DAGISEL64-NEXT:    s_mov_b32 s0, callee@abs32@lo
+; DAGISEL64-NEXT:    s_addk_co_i32 s32, 0x250
+; DAGISEL64-NEXT:    v_mov_b32_e32 v41, v9
+; DAGISEL64-NEXT:    v_writelane_b32 v42, s4, 0
+; DAGISEL64-NEXT:    v_mov_b32_e32 v40, v8
+; DAGISEL64-NEXT:    v_writelane_b32 v42, s5, 1
+; DAGISEL64-NEXT:    v_writelane_b32 v42, s30, 2
+; DAGISEL64-NEXT:    v_writelane_b32 v42, s31, 3
+; DAGISEL64-NEXT:    s_wait_alu 0xfffe
+; DAGISEL64-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; DAGISEL64-NEXT:    flat_store_b32 v[40:41], v0
+; DAGISEL64-NEXT:    v_readlane_b32 s31, v42, 3
+; DAGISEL64-NEXT:    v_readlane_b32 s30, v42, 2
+; DAGISEL64-NEXT:    v_readlane_b32 s5, v42, 1
+; DAGISEL64-NEXT:    v_readlane_b32 s4, v42, 0
+; DAGISEL64-NEXT:    v_readlane_b32 s0, v42, 4
+; DAGISEL64-NEXT:    s_clause 0x2
+; DAGISEL64-NEXT:    scratch_load_b32 v42, off, s33
+; DAGISEL64-NEXT:    scratch_load_b32 v40, off, s33 offset:164
+; DAGISEL64-NEXT:    scratch_load_b32 v41, off, s33 offset:168
+; DAGISEL64-NEXT:    s_mov_b32 s32, s33
+; DAGISEL64-NEXT:    s_xor_b64 exec, s[4:5], -1
+; DAGISEL64-NEXT:    s_clause 0x1f
+; DAGISEL64-NEXT:    scratch_load_b32 v0, off, s33 offset:4
+; DAGISEL64-NEXT:    scratch_load_b32 v1, off, s33 offset:8
+; DAGISEL64-NEXT:    scratch_load_b32 v2, off, s33 offset:12
+; DAGISEL64-NEXT:    scratch_load_b32 v3, off, s33 offset:16
+; DAGISEL64-NEXT:    scratch_load_b32 v4, off, s33 offset:20
+; DAGISEL64-NEXT:    scratch_load_b32 v5, off, s33 offset:24
+; DAGISEL64-NEXT:    scratch_load_b32 v6, off, s33 offset:28
+; DAGISEL64-NEXT:    scratch_load_b32 v7, off, s33 offset:32
+; DAGISEL64-NEXT:    scratch_load_b32 v8, off, s33 offset:36
+; DAGISEL64-NEXT:    scratch_load_b32 v9, off, s33 offset:40
+; DAGISEL64-NEXT:    scratch_load_b32 v10, off, s33 offset:44
+; DAGISEL64-NEXT:    scratch_load_b32 v11, off, s33 offset:48
+; DAGISEL64-NEXT:    scratch_load_b32 v12, off, s33 offset:52
+; DAGISEL64-NEXT:    scratch_load_b32 v13, off, s33 offset:56
+; DAGISEL64-NEXT:    scratch_load_b32 v14, off, s33 offset:60
+; DAGISEL64-NEXT:    scratch_load_b32 v15, off, s33 offset:64
+; DAGISEL64-NEXT:    scratch_load_b32 v16, off, s33 offset:68
+; DAGISEL64-NEXT:    scratch_load_b32 v17, off, s33 offset:72
+; DAGISEL64-NEXT:    scratch_load_b32 v18, off, s33 offset:76
+; DAGISEL64-NEXT:    scratch_load_b32 v19, off, s33 offset:80
+; DAGISEL64-NEXT:    scratch_load_b32 v20, off, s33 offset:84
+; DAGISEL64-NEXT:    scratch_load_b32 v21, off, s33 offset:88
+; DAGISEL64-NEXT:    scratch_load_b32 v22, off, s33 offset:92
+; DAGISEL64-NEXT:    scratch_load_b32 v23, off, s33 offset:96
+; DAGISEL64-NEXT:    scratch_load_b32 v24, off, s33 offset:100
+; DAGISEL64-NEXT:    scratch_load_b32 v25, off, s33 offset:104
+; DAGISEL64-NEXT:    scratch_load_b32 v26, off, s33 offset:108
+; DAGISEL64-NEXT:    scratch_load_b32 v27, off, s33 offset:112
+; DAGISEL64-NEXT:    scratch_load_b32 v28, off, s33 offset:116
+; DAGISEL64-NEXT:    scratch_load_b32 v29, off, s33 offset:120
+; DAGISEL64-NEXT:    scratch_load_b32 v30, off, s33 offset:124
+; DAGISEL64-NEXT:    scratch_load_b32 v31, off, s33 offset:128
+; DAGISEL64-NEXT:    s_clause 0x1f
+; DAGISEL64-NEXT:    scratch_load_b32 v32, off, s33 offset:132
+; DAGISEL64-NEXT:    scratch_load_b32 v33, off, s33 offset:136
+; DAGISEL64-NEXT:    scratch_load_b32 v34, off, s33 offset:140
+; DAGISEL64-NEXT:    scratch_load_b32 v35, off, s33 offset:144
+; DAGISEL64-NEXT:    scratch_load_b32 v36, off, s33 offset:148
+; DAGISEL64-NEXT:    scratch_load_b32 v37, off, s33 offset:152
+; DAGISEL64-NEXT:    scratch_load_b32 v38, off, s33 offset:156
+; DAGISEL64-NEXT:    scratch_load_b32 v39, off, s33 offset:160
+; DAGISEL64-NEXT:    scratch_load_b32 v48, off, s33 offset:172
+; DAGISEL64-NEXT:    scratch_load_b32 v49, off, s33 offset:176
+; DAGISEL64-NEXT:    scratch_load_b32 v50, off, s33 offset:180
+; DAGISEL64-NEXT:    scratch_load_b32 v51, off, s33 offset:184
+; DAGISEL64-NEXT:    scratch_load_b32 v52, off, s33 offset:188
+; DAGISEL64-NEXT:    scratch_load_b32 v53, off, s33 offset:192
+; DAGISEL64-NEXT:    scratch_load_b32 v54, off, s33 offset:196
+; DAGISEL64-NEXT:    scratch_load_b32 v55, off, s33 offset:200
+; DAGISEL64-NEXT:    scratch_load_b32 v64, off, s33 offset:204
+; DAGISEL64-NEXT:    scratch_load_b32 v65, off, s33 offset:208
+; DAGISEL64-NEXT:    scratch_load_b32 v66, off, s33 offset:212
+; DAGISEL64-NEXT:    scratch_load_b32 v67, off, s33 offset:216
+; DAGISEL64-NEXT:    scratch_load_b32 v68, off, s33 offset:220
+; DAGISEL64-NEXT:    scratch_load_b32 v69, off, s33 offset:224
+; DAGISEL64-NEXT:    scratch_load_b32 v70, off, s33 offset:228
+; DAGISEL64-NEXT:    scratch_load_b32 v71, off, s33 offset:232
+; DAGISEL64-NEXT:    scratch_load_b32 v80, off, s33 offset:236
+; DAGISEL64-NEXT:    scratch_load_b32 v81, off, s33 offset:240
+; DAGISEL64-NEXT:    scratch_load_b32 v82, off, s33 offset:244
+; DAGISEL64-NEXT:    scratch_load_b32 v83, off, s33 offset:248
+; DAGISEL64-NEXT:    scratch_load_b32 v84, off, s33 offset:252
+; DAGISEL64-NEXT:    scratch_load_b32 v85, off, s33 offset:256
+; DAGISEL64-NEXT:    scratch_load_b32 v86, off, s33 offset:260
+; DAGISEL64-NEXT:    scratch_load_b32 v87, off, s33 offset:264
+; DAGISEL64-NEXT:    s_clause 0x1f
+; DAGISEL64-NEXT:    scratch_load_b32 v96, off, s33 offset:268
+; DAGISEL64-NEXT:    scratch_load_b32 v97, off, s33 offset:272
+; DAGISEL64-NEXT:    scratch_load_b32 v98, off, s33 offset:276
+; DAGISEL64-NEXT:    scratch_load_b32 v99, off, s33 offset:280
+; DAGISEL64-NEXT:    scratch_load_b32 v100, off, s33 offset:284
+; DAGISEL64-NEXT:    scratch_load_b32 v101, off, s33 offset:288
+; DAGISEL64-NEXT:    scratch_load_b32 v102, off, s33 offset:292
+; DAGISEL64-NEXT:    scratch_load_b32 v103, off, s33 offset:296
+; DAGISEL64-NEXT:    scratch_load_b32 v112, off, s33 offset:300
+; DAGISEL64-NEXT:    scratch_load_b32 v113, off, s33 offset:304
+; DAGISEL64-NEXT:    scratch_load_b32 v114, off, s33 offset:308
+; DAGISEL64-NEXT:    scratch_load_b32 v115, off, s33 offset:312
+; DAGISEL64-NEXT:    scratch_load_b32 v116, off, s33 offset:316
+; DAGISEL64-NEXT:    scratch_load_b32 v117, off, s33 offset:320
+; DAGISEL64-NEXT:    scratch_load_b32 v118, off, s33 offset:324
+; DAGISEL64-NEXT:    scratch_load_b32 v119, off, s33 offset:328
+; DAGISEL64-NEXT:    scratch_load_b32 v128, off, s33 offset:332
+; DAGISEL64-NEXT:    scratch_load_b32 v129, off, s33 offset:336
+; DAGISEL64-NEXT:    scratch_load_b32 v130, off, s33 offset:340
+; DAGISEL64-NEXT:    scratch_load_b32 v131, off, s33 offset:344
+; DAGISEL64-NEXT:    scratch_load_b32 v132, off, s33 offset:348
+; DAGISEL64-NEXT:    scratch_load_b32 v133, off, s33 offset:352
+; DAGISEL64-NEXT:    scratch_load_b32 v134, off, s33 offset:356
+; DAGISEL64-NEXT:    scratch_load_b32 v135, off, s33 offset:360
+; DAGISEL64-NEXT:    scratch_load_b32 v144, off, s33 offset:364
+; DAGISEL64-NEXT:    scratch_load_b32 v145, off, s33 offset:368
+; DAGISEL64-NEXT:    scratch_load_b32 v146, off, s33 offset:372
+; DAGISEL64-NEXT:    scratch_load_b32 v147, off, s33 offset:376
+; DAGISEL64-NEXT:    scratch_load_b32 v148, off, s33 offset:380
+; DAGISEL64-NEXT:    scratch_load_b32 v149, off, s33 offset:384
+; DAGISEL64-NEXT:    scratch_load_b32 v150, off, s33 offset:388
+; DAGISEL64-NEXT:    scratch_load_b32 v151, off, s33 offset:392
+; DAGISEL64-NEXT:    s_clause 0x1f
+; DAGISEL64-NEXT:    scratch_load_b32 v160, off, s33 offset:396
+; DAGISEL64-NEXT:    scratch_load_b32 v161, off, s33 offset:400
+; DAGISEL64-NEXT:    scratch_load_b32 v162, off, s33 offset:404
+; DAGISEL64-NEXT:    scratch_load_b32 v163, off, s33 offset:408
+; DAGISEL64-NEXT:    scratch_load_b32 v164, off, s33 offset:412
+; DAGISEL64-NEXT:    scratch_load_b32 v165, off, s33 offset:416
+; DAGISEL64-NEXT:    scratch_load_b32 v166, off, s33 offset:420
+; DAGISEL64-NEXT:    scratch_load_b32 v167, off, s33 offset:424
+; DAGISEL64-NEXT:    scratch_load_b32 v176, off, s33 offset:428
+; DAGISEL64-NEXT:    scratch_load_b32 v177, off, s33 offset:432
+; DAGISEL64-NEXT:    scratch_load_b32 v178, off, s33 offset:436
+; DAGISEL64-NEXT:    scratch_load_b32 v179, off, s33 offset:440
+; DAGISEL64-NEXT:    scratch_load_b32 v180, off, s33 offset:444
+; DAGISEL64-NEXT:    scratch_load_b32 v181, off, s33 offset:448
+; DAGISEL64-NEXT:    scratch_load_b32 v182, off, s33 offset:452
+; DAGISEL64-NEXT:    scratch_load_b32 v183, off, s33 offset:456
+; DAGISEL64-NEXT:    scratch_load_b32 v192, off, s33 offset:460
+; DAGISEL64-NEXT:    scratch_load_b32 v193, off, s33 offset:464
+; DAGISEL64-NEXT:    scratch_load_b32 v194, off, s33 offset:468
+; DAGISEL64-NEXT:    scratch_load_b32 v195, off, s33 offset:472
+; DAGISEL64-NEXT:    scratch_load_b32 v196, off, s33 offset:476
+; DAGISEL64-NEXT:    scratch_load_b32 v197, off, s33 offset:480
+; DAGISEL64-NEXT:    scratch_load_b32 v198, off, s33 offset:484
+; DAGISEL64-NEXT:    scratch_load_b32 v199, off, s33 offset:488
+; DAGISEL64-NEXT:    scratch_load_b32 v208, off, s33 offset:492
+; DAGISEL64-NEXT:    scratch_load_b32 v209, off, s33 offset:496
+; DAGISEL64-NEXT:    scratch_load_b32 v210, off, s33 offset:500
+; DAGISEL64-NEXT:    scratch_load_b32 v211, off, s33 offset:504
+; DAGISEL64-NEXT:    scratch_load_b32 v212, off, s33 offset:508
+; DAGISEL64-NEXT:    scratch_load_b32 v213, off, s33 offset:512
+; DAGISEL64-NEXT:    scratch_load_b32 v214, off, s33 offset:516
+; DAGISEL64-NEXT:    scratch_load_b32 v215, off, s33 offset:520
+; DAGISEL64-NEXT:    s_clause 0xf
+; DAGISEL64-NEXT:    scratch_load_b32 v224, off, s33 offset:524
+; DAGISEL64-NEXT:    scratch_load_b32 v225, off, s33 offset:528
+; DAGISEL64-NEXT:    scratch_load_b32 v226, off, s33 offset:532
+; DAGISEL64-NEXT:    scratch_load_b32 v227, off, s33 offset:536
+; DAGISEL64-NEXT:    scratch_load_b32 v228, off, s33 offset:540
+; DAGISEL64-NEXT:    scratch_load_b32 v229, off, s33 offset:544
+; DAGISEL64-NEXT:    scratch_load_b32 v230, off, s33 offset:548
+; DAGISEL64-NEXT:    scratch_load_b32 v231, off, s33 offset:552
+; DAGISEL64-NEXT:    scratch_load_b32 v240, off, s33 offset:556
+; DAGISEL64-NEXT:    scratch_load_b32 v241, off, s33 offset:560
+; DAGISEL64-NEXT:    scratch_load_b32 v242, off, s33 offset:564
+; DAGISEL64-NEXT:    scratch_load_b32 v243, off, s33 offset:568
+; DAGISEL64-NEXT:    scratch_load_b32 v244, off, s33 offset:572
+; DAGISEL64-NEXT:    scratch_load_b32 v245, off, s33 offset:576
+; DAGISEL64-NEXT:    scratch_load_b32 v246, off, s33 offset:580
+; DAGISEL64-NEXT:    scratch_load_b32 v247, off, s33 offset:584
+; DAGISEL64-NEXT:    s_mov_b64 exec, s[4:5]
+; DAGISEL64-NEXT:    s_mov_b32 s33, s0
+; DAGISEL64-NEXT:    s_wait_loadcnt_dscnt 0x0
+; DAGISEL64-NEXT:    s_wait_alu 0xfffe
+; DAGISEL64-NEXT:    s_setpc_b64 s[30:31]
+;
+; GISEL64-LABEL: call_from_whole_wave:
+; GISEL64:       ; %bb.0:
+; GISEL64-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL64-NEXT:    s_wait_expcnt 0x0
+; GISEL64-NEXT:    s_wait_samplecnt 0x0
+; GISEL64-NEXT:    s_wait_bvhcnt 0x0
+; GISEL64-NEXT:    s_wait_kmcnt 0x0
+; GISEL64-NEXT:    s_mov_b32 s0, s33
+; GISEL64-NEXT:    s_mov_b32 s33, s32
+; GISEL64-NEXT:    s_xor_saveexec_b64 s[4:5], -1
+; GISEL64-NEXT:    s_clause 0x1f
+; GISEL64-NEXT:    scratch_store_b32 off, v0, s33 offset:4
+; GISEL64-NEXT:    scratch_store_b32 off, v1, s33 offset:8
+; GISEL64-NEXT:    scratch_store_b32 off, v2, s33 offset:12
+; GISEL64-NEXT:    scratch_store_b32 off, v3, s33 offset:16
+; GISEL64-NEXT:    scratch_store_b32 off, v4, s33 offset:20
+; GISEL64-NEXT:    scratch_store_b32 off, v5, s33 offset:24
+; GISEL64-NEXT:    scratch_store_b32 off, v6, s33 offset:28
+; GISEL64-NEXT:    scratch_store_b32 off, v7, s33 offset:32
+; GISEL64-NEXT:    scratch_store_b32 off, v8, s33 offset:36
+; GISEL64-NEXT:    scratch_store_b32 off, v9, s33 offset:40
+; GISEL64-NEXT:    scratch_store_b32 off, v10, s33 offset:44
+; GISEL64-NEXT:    scratch_store_b32 off, v11, s33 offset:48
+; GISEL64-NEXT:    scratch_store_b32 off, v12, s33 offset:52
+; GISEL64-NEXT:    scratch_store_b32 off, v13, s33 offset:56
+; GISEL64-NEXT:    scratch_store_b32 off, v14, s33 offset:60
+; GISEL64-NEXT:    scratch_store_b32 off, v15, s33 offset:64
+; GISEL64-NEXT:    scratch_store_b32 off, v16, s33 offset:68
+; GISEL64-NEXT:    scratch_store_b32 off, v17, s33 offset:72
+; GISEL64-NEXT:    scratch_store_b32 off, v18, s33 offset:76
+; GISEL64-NEXT:    scratch_store_b32 off, v19, s33 offset:80
+; GISEL64-NEXT:    scratch_store_b32 off, v20, s33 offset:84
+; GISEL64-NEXT:    scratch_store_b32 off, v21, s33 offset:88
+; GISEL64-NEXT:    scratch_store_b32 off, v22, s33 offset:92
+; GISEL64-NEXT:    scratch_store_b32 off, v23, s33 offset:96
+; GISEL64-NEXT:    scratch_store_b32 off, v24, s33 offset:100
+; GISEL64-NEXT:    scratch_store_b32 off, v25, s33 offset:104
+; GISEL64-NEXT:    scratch_store_b32 off, v26, s33 offset:108
+; GISEL64-NEXT:    scratch_store_b32 off, v27, s33 offset:112
+; GISEL64-NEXT:    scratch_store_b32 off, v28, s33 offset:116
+; GISEL64-NEXT:    scratch_store_b32 off, v29, s33 offset:120
+; GISEL64-NEXT:    scratch_store_b32 off, v30, s33 offset:124
+; GISEL64-NEXT:    scratch_store_b32 off, v31, s33 offset:128
+; GISEL64-NEXT:    s_clause 0x1f
+; GISEL64-NEXT:    scratch_store_b32 off, v32, s33 offset:132
+; GISEL64-NEXT:    scratch_store_b32 off, v33, s33 offset:136
+; GISEL64-NEXT:    scratch_store_b32 off, v34, s33 offset:140
+; GISEL64-NEXT:    scratch_store_b32 off, v35, s33 offset:144
+; GISEL64-NEXT:    scratch_store_b32 off, v36, s33 offset:148
+; GISEL64-NEXT:    scratch_store_b32 off, v37, s33 offset:152
+; GISEL64-NEXT:    scratch_store_b32 off, v38, s33 offset:156
+; GISEL64-NEXT:    scratch_store_b32 off, v39, s33 offset:160
+; GISEL64-NEXT:    scratch_store_b32 off, v48, s33 offset:172
+; GISEL64-NEXT:    scratch_store_b32 off, v49, s33 offset:176
+; GISEL64-NEXT:    scratch_store_b32 off, v50, s33 offset:180
+; GISEL64-NEXT:    scratch_store_b32 off, v51, s33 offset:184
+; GISEL64-NEXT:    scratch_store_b32 off, v52, s33 offset:188
+; GISEL64-NEXT:    scratch_store_b32 off, v53, s33 offset:192
+; GISEL64-NEXT:    scratch_store_b32 off, v54, s33 offset:196
+; GISEL64-NEXT:    scratch_store_b32 off, v55, s33 offset:200
+; GISEL64-NEXT:    scratch_store_b32 off, v64, s33 offset:204
+; GISEL64-NEXT:    scratch_store_b32 off, v65, s33 offset:208
+; GISEL64-NEXT:    scratch_store_b32 off, v66, s33 offset:212
+; GISEL64-NEXT:    scratch_store_b32 off, v67, s33 offset:216
+; GISEL64-NEXT:    scratch_store_b32 off, v68, s33 offset:220
+; GISEL64-NEXT:    scratch_store_b32 off, v69, s33 offset:224
+; GISEL64-NEXT:    scratch_store_b32 off, v70, s33 offset:228
+; GISEL64-NEXT:    scratch_store_b32 off, v71, s33 offset:232
+; GISEL64-NEXT:    scratch_store_b32 off, v80, s33 offset:236
+; GISEL64-NEXT:    scratch_store_b32 off, v81, s33 offset:240
+; GISEL64-NEXT:    scratch_store_b32 off, v82, s33 offset:244
+; GISEL64-NEXT:    scratch_store_b32 off, v83, s33 offset:248
+; GISEL64-NEXT:    scratch_store_b32 off, v84, s33 offset:252
+; GISEL64-NEXT:    scratch_store_b32 off, v85, s33 offset:256
+; GISEL64-NEXT:    scratch_store_b32 off, v86, s33 offset:260
+; GISEL64-NEXT:    scratch_store_b32 off, v87, s33 offset:264
+; GISEL64-NEXT:    s_clause 0x1f
+; GISEL64-NEXT:    scratch_store_b32 off, v96, s33 offset:268
+; GISEL64-NEXT:    scratch_store_b32 off, v97, s33 offset:272
+; GISEL64-NEXT:    scratch_store_b32 off, v98, s33 offset:276
+; GISEL64-NEXT:    scratch_store_b32 off, v99, s33 offset:280
+; GISEL64-NEXT:    scratch_store_b32 off, v100, s33 offset:284
+; GISEL64-NEXT:    scratch_store_b32 off, v101, s33 offset:288
+; GISEL64-NEXT:    scratch_store_b32 off, v102, s33 offset:292
+; GISEL64-NEXT:    scratch_store_b32 off, v103, s33 offset:296
+; GISEL64-NEXT:    scratch_store_b32 off, v112, s33 offset:300
+; GISEL64-NEXT:    scratch_store_b32 off, v113, s33 offset:304
+; GISEL64-NEXT:    scratch_store_b32 off, v114, s33 offset:308
+; GISEL64-NEXT:    scratch_store_b32 off, v115, s33 offset:312
+; GISEL64-NEXT:    scratch_store_b32 off, v116, s33 offset:316
+; GISEL64-NEXT:    scratch_store_b32 off, v117, s33 offset:320
+; GISEL64-NEXT:    scratch_store_b32 off, v118, s33 offset:324
+; GISEL64-NEXT:    scratch_store_b32 off, v119, s33 offset:328
+; GISEL64-NEXT:    scratch_store_b32 off, v128, s33 offset:332
+; GISEL64-NEXT:    scratch_store_b32 off, v129, s33 offset:336
+; GISEL64-NEXT:    scratch_store_b32 off, v130, s33 offset:340
+; GISEL64-NEXT:    scratch_store_b32 off, v131, s33 offset:344
+; GISEL64-NEXT:    scratch_store_b32 off, v132, s33 offset:348
+; GISEL64-NEXT:    scratch_store_b32 off, v133, s33 offset:352
+; GISEL64-NEXT:    scratch_store_b32 off, v134, s33 offset:356
+; GISEL64-NEXT:    scratch_store_b32 off, v135, s33 offset:360
+; GISEL64-NEXT:    scratch_store_b32 off, v144, s33 offset:364
+; GISEL64-NEXT:    scratch_store_b32 off, v145, s33 offset:368
+; GISEL64-NEXT:    scratch_store_b32 off, v146, s33 offset:372
+; GISEL64-NEXT:    scratch_store_b32 off, v147, s33 offset:376
+; GISEL64-NEXT:    scratch_store_b32 off, v148, s33 offset:380
+; GISEL64-NEXT:    scratch_store_b32 off, v149, s33 offset:384
+; GISEL64-NEXT:    scratch_store_b32 off, v150, s33 offset:388
+; GISEL64-NEXT:    scratch_store_b32 off, v151, s33 offset:392
+; GISEL64-NEXT:    s_clause 0x1f
+; GISEL64-NEXT:    scratch_store_b32 off, v160, s33 offset:396
+; GISEL64-NEXT:    scratch_store_b32 off, v161, s33 offset:400
+; GISEL64-NEXT:    scratch_store_b32 off, v162, s33 offset:404
+; GISEL64-NEXT:    scratch_store_b32 off, v163, s33 offset:408
+; GISEL64-NEXT:    scratch_store_b32 off, v164, s33 offset:412
+; GISEL64-NEXT:    scratch_store_b32 off, v165, s33 offset:416
+; GISEL64-NEXT:    scratch_store_b32 off, v166, s33 offset:420
+; GISEL64-NEXT:    scratch_store_b32 off, v167, s33 offset:424
+; GISEL64-NEXT:    scratch_store_b32 off, v176, s33 offset:428
+; GISEL64-NEXT:    scratch_store_b32 off, v177, s33 offset:432
+; GISEL64-NEXT:    scratch_store_b32 off, v178, s33 offset:436
+; GISEL64-NEXT:    scratch_store_b32 off, v179, s33 offset:440
+; GISEL64-NEXT:    scratch_store_b32 off, v180, s33 offset:444
+; GISEL64-NEXT:    scratch_store_b32 off, v181, s33 offset:448
+; GISEL64-NEXT:    scratch_store_b32 off, v182, s33 offset:452
+; GISEL64-NEXT:    scratch_store_b32 off, v183, s33 offset:456
+; GISEL64-NEXT:    scratch_store_b32 off, v192, s33 offset:460
+; GISEL64-NEXT:    scratch_store_b32 off, v193, s33 offset:464
+; GISEL64-NEXT:    scratch_store_b32 off, v194, s33 offset:468
+; GISEL64-NEXT:    scratch_store_b32 off, v195, s33 offset:472
+; GISEL64-NEXT:    scratch_store_b32 off, v196, s33 offset:476
+; GISEL64-NEXT:    scratch_store_b32 off, v197, s33 offset:480
+; GISEL64-NEXT:    scratch_store_b32 off, v198, s33 offset:484
+; GISEL64-NEXT:    scratch_store_b32 off, v199, s33 offset:488
+; GISEL64-NEXT:    scratch_store_b32 off, v208, s33 offset:492
+; GISEL64-NEXT:    scratch_store_b32 off, v209, s33 offset:496
+; GISEL64-NEXT:    scratch_store_b32 off, v210, s33 offset:500
+; GISEL64-NEXT:    scratch_store_b32 off, v211, s33 offset:504
+; GISEL64-NEXT:    scratch_store_b32 off, v212, s33 offset:508
+; GISEL64-NEXT:    scratch_store_b32 off, v213, s33 offset:512
+; GISEL64-NEXT:    scratch_store_b32 off, v214, s33 offset:516
+; GISEL64-NEXT:    scratch_store_b32 off, v215, s33 offset:520
+; GISEL64-NEXT:    s_clause 0xf
+; GISEL64-NEXT:    scratch_store_b32 off, v224, s33 offset:524
+; GISEL64-NEXT:    scratch_store_b32 off, v225, s33 offset:528
+; GISEL64-NEXT:    scratch_store_b32 off, v226, s33 offset:532
+; GISEL64-NEXT:    scratch_store_b32 off, v227, s33 offset:536
+; GISEL64-NEXT:    scratch_store_b32 off, v228, s33 offset:540
+; GISEL64-NEXT:    scratch_store_b32 off, v229, s33 offset:544
+; GISEL64-NEXT:    scratch_store_b32 off, v230, s33 offset:548
+; GISEL64-NEXT:    scratch_store_b32 off, v231, s33 offset:552
+; GISEL64-NEXT:    scratch_store_b32 off, v240, s33 offset:556
+; GISEL64-NEXT:    scratch_store_b32 off, v241, s33 offset:560
+; GISEL64-NEXT:    scratch_store_b32 off, v242, s33 offset:564
+; GISEL64-NEXT:    scratch_store_b32 off, v243, s33 offset:568
+; GISEL64-NEXT:    scratch_store_b32 off, v244, s33 offset:572
+; GISEL64-NEXT:    scratch_store_b32 off, v245, s33 offset:576
+; GISEL64-NEXT:    scratch_store_b32 off, v246, s33 offset:580
+; GISEL64-NEXT:    scratch_store_b32 off, v247, s33 offset:584
+; GISEL64-NEXT:    s_mov_b64 exec, -1
+; GISEL64-NEXT:    s_clause 0x2
+; GISEL64-NEXT:    scratch_store_b32 off, v42, s33
+; GISEL64-NEXT:    scratch_store_b32 off, v40, s33 offset:164
+; GISEL64-NEXT:    scratch_store_b32 off, v41, s33 offset:168
+; GISEL64-NEXT:    s_wait_alu 0xfffe
+; GISEL64-NEXT:    v_writelane_b32 v42, s0, 4
+; GISEL64-NEXT:    s_mov_b32 s0, callee@abs32@lo
+; GISEL64-NEXT:    s_mov_b32 s1, callee@abs32@hi
+; GISEL64-NEXT:    s_addk_co_i32 s32, 0x250
+; GISEL64-NEXT:    v_mov_b32_e32 v40, v8
+; GISEL64-NEXT:    v_writelane_b32 v42, s4, 0
+; GISEL64-NEXT:    v_mov_b32_e32 v41, v9
+; GISEL64-NEXT:    v_writelane_b32 v42, s5, 1
+; GISEL64-NEXT:    v_writelane_b32 v42, s30, 2
+; GISEL64-NEXT:    v_writelane_b32 v42, s31, 3
+; GISEL64-NEXT:    s_wait_alu 0xfffe
+; GISEL64-NEXT:    s_swappc_b64 s[30:31], s[0:1]
+; GISEL64-NEXT:    flat_store_b32 v[40:41], v0
+; GISEL64-NEXT:    v_readlane_b32 s31, v42, 3
+; GISEL64-NEXT:    v_readlane_b32 s30, v42, 2
+; GISEL64-NEXT:    v_readlane_b32 s5, v42, 1
+; GISEL64-NEXT:    v_readlane_b32 s4, v42, 0
+; GISEL64-NEXT:    v_readlane_b32 s0, v42, 4
+; GISEL64-NEXT:    s_clause 0x2
+; GISEL64-NEXT:    scratch_load_b32 v42, off, s33
+; GISEL64-NEXT:    scratch_load_b32 v40, off, s33 offset:164
+; GISEL64-NEXT:    scratch_load_b32 v41, off, s33 offset:168
+; GISEL64-NEXT:    s_mov_b32 s32, s33
+; GISEL64-NEXT:    s_xor_b64 exec, s[4:5], -1
+; GISEL64-NEXT:    s_clause 0x1f
+; GISEL64-NEXT:    scratch_load_b32 v0, off, s33 offset:4
+; GISEL64-NEXT:    scratch_load_b32 v1, off, s33 offset:8
+; GISEL64-NEXT:    scratch_load_b32 v2, off, s33 offset:12
+; GISEL64-NEXT:    scratch_load_b32 v3, off, s33 offset:16
+; GISEL64-NEXT:    scratch_load_b32 v4, off, s33 offset:20
+; GISEL64-NEXT:    scratch_load_b32 v5, off, s33 offset:24
+; GISEL64-NEXT:    scratch_load_b32 v6, off, s33 offset:28
+; GISEL64-NEXT:    scratch_load_b32 v7, off, s33 offset:32
+; GISEL64-NEXT:    scratch_load_b32 v8, off, s33 offset:36
+; GISEL64-NEXT:    scratch_load_b32 v9, off, s33 offset:40
+; GISEL64-NEXT:    scratch_load_b32 v10, off, s33 offset:44
+; GISEL64-NEXT:    scratch_load_b32 v11, off, s33 offset:48
+; GISEL64-NEXT:    scratch_load_b32 v12, off, s33 offset:52
+; GISEL64-NEXT:    scratch_load_b32 v13, off, s33 offset:56
+; GISEL64-NEXT:    scratch_load_b32 v14, off, s33 offset:60
+; GISEL64-NEXT:    scratch_load_b32 v15, off, s33 offset:64
+; GISEL64-NEXT:    scratch_load_b32 v16, off, s33 offset:68
+; GISEL64-NEXT:    scratch_load_b32 v17, off, s33 offset:72
+; GISEL64-NEXT:    scratch_load_b32 v18, off, s33 offset:76
+; GISEL64-NEXT:    scratch_load_b32 v19, off, s33 offset:80
+; GISEL64-NEXT:    scratch_load_b32 v20, off, s33 offset:84
+; GISEL64-NEXT:    scratch_load_b32 v21, off, s33 offset:88
+; GISEL64-NEXT:    scratch_load_b32 v22, off, s33 offset:92
+; GISEL64-NEXT:    scratch_load_b32 v23, off, s33 offset:96
+; GISEL64-NEXT:    scratch_load_b32 v24, off, s33 offset:100
+; GISEL64-NEXT:    scratch_load_b32 v25, off, s33 offset:104
+; GISEL64-NEXT:    scratch_load_b32 v26, off, s33 offset:108
+; GISEL64-NEXT:    scratch_load_b32 v27, off, s33 offset:112
+; GISEL64-NEXT:    scratch_load_b32 v28, off, s33 offset:116
+; GISEL64-NEXT:    scratch_load_b32 v29, off, s33 offset:120
+; GISEL64-NEXT:    scratch_load_b32 v30, off, s33 offset:124
+; GISEL64-NEXT:    scratch_load_b32 v31, off, s33 offset:128
+; GISEL64-NEXT:    s_clause 0x1f
+; GISEL64-NEXT:    scratch_load_b32 v32, off, s33 offset:132
+; GISEL64-NEXT:    scratch_load_b32 v33, off, s33 offset:136
+; GISEL64-NEXT:    scratch_load_b32 v34, off, s33 offset:140
+; GISEL64-NEXT:    scratch_load_b32 v35, off, s33 offset:144
+; GISEL64-NEXT:    scratch_load_b32 v36, off, s33 offset:148
+; GISEL64-NEXT:    scratch_load_b32 v37, off, s33 offset:152
+; GISEL64-NEXT:    scratch_load_b32 v38, off, s33 offset:156
+; GISEL64-NEXT:    scratch_load_b32 v39, off, s33 offset:160
+; GISEL64-NEXT:    scratch_load_b32 v48, off, s33 offset:172
+; GISEL64-NEXT:    scratch_load_b32 v49, off, s33 offset:176
+; GISEL64-NEXT:    scratch_load_b32 v50, off, s33 offset:180
+; GISEL64-NEXT:    scratch_load_b32 v51, off, s33 offset:184
+; GISEL64-NEXT:    scratch_load_b32 v52, off, s33 offset:188
+; GISEL64-NEXT:    scratch_load_b32 v53, off, s33 offset:192
+; GISEL64-NEXT:    scratch_load_b32 v54, off, s33 offset:196
+; GISEL64-NEXT:    scratch_load_b32 v55, off, s33 offset:200
+; GISEL64-NEXT:    scratch_load_b32 v64, off, s33 offset:204
+; GISEL64-NEXT:    scratch_load_b32 v65, off, s33 offset:208
+; GISEL64-NEXT:    scratch_load_b32 v66, off, s33 offset:212
+; GISEL64-NEXT:    scratch_load_b32 v67, off, s33 offset:216
+; GISEL64-NEXT:    scratch_load_b32 v68, off, s33 offset:220
+; GISEL64-NEXT:    scratch_load_b32 v69, off, s33 offset:224
+; GISEL64-NEXT:    scratch_load_b32 v70, off, s33 offset:228
+; GISEL64-NEXT:    scratch_load_b32 v71, off, s33 offset:232
+; GISEL64-NEXT:    scratch_load_b32 v80, off, s33 offset:236
+; GISEL64-NEXT:    scratch_load_b32 v81, off, s33 offset:240
+; GISEL64-NEXT:    scratch_load_b32 v82, off, s33 offset:244
+; GISEL64-NEXT:    scratch_load_b32 v83, off, s33 offset:248
+; GISEL64-NEXT:    scratch_load_b32 v84, off, s33 offset:252
+; GISEL64-NEXT:    scratch_load_b32 v85, off, s33 offset:256
+; GISEL64-NEXT:    scratch_load_b32 v86, off, s33 offset:260
+; GISEL64-NEXT:    scratch_load_b32 v87, off, s33 offset:264
+; GISEL64-NEXT:    s_clause 0x1f
+; GISEL64-NEXT:    scratch_load_b32 v96, off, s33 offset:268
+; GISEL64-NEXT:    scratch_load_b32 v97, off, s33 offset:272
+; GISEL64-NEXT:    scratch_load_b32 v98, off, s33 offset:276
+; GISEL64-NEXT:    scratch_load_b32 v99, off, s33 offset:280
+; GISEL64-NEXT:    scratch_load_b32 v100, off, s33 offset:284
+; GISEL64-NEXT:    scratch_load_b32 v101, off, s33 offset:288
+; GISEL64-NEXT:    scratch_load_b32 v102, off, s33 offset:292
+; GISEL64-NEXT:    scratch_load_b32 v103, off, s33 offset:296
+; GISEL64-NEXT:    scratch_load_b32 v112, off, s33 offset:300
+; GISEL64-NEXT:    scratch_load_b32 v113, off, s33 offset:304
+; GISEL64-NEXT:    scratch_load_b32 v114, off, s33 offset:308
+; GISEL64-NEXT:    scratch_load_b32 v115, off, s33 offset:312
+; GISEL64-NEXT:    scratch_load_b32 v116, off, s33 offset:316
+; GISEL64-NEXT:    scratch_load_b32 v117, off, s33 offset:320
+; GISEL64-NEXT:    scratch_load_b32 v118, off, s33 offset:324
+; GISEL64-NEXT:    scratch_load_b32 v119, off, s33 offset:328
+; GISEL64-NEXT:    scratch_load_b32 v128, off, s33 offset:332
+; GISEL64-NEXT:    scratch_load_b32 v129, off, s33 offset:336
+; GISEL64-NEXT:    scratch_load_b32 v130, off, s33 offset:340
+; GISEL64-NEXT:    scratch_load_b32 v131, off, s33 offset:344
+; GISEL64-NEXT:    scratch_load_b32 v132, off, s33 offset:348
+; GISEL64-NEXT:    scratch_load_b32 v133, off, s33 offset:352
+; GISEL64-NEXT:    scratch_load_b32 v134, off, s33 offset:356
+; GISEL64-NEXT:    scratch_load_b32 v135, off, s33 offset:360
+; GISEL64-NEXT:    scratch_load_b32 v144, off, s33 offset:364
+; GISEL64-NEXT:    scratch_load_b32 v145, off, s33 offset:368
+; GISEL64-NEXT:    scratch_load_b32 v146, off, s33 offset:372
+; GISEL64-NEXT:    scratch_load_b32 v147, off, s33 offset:376
+; GISEL64-NEXT:    scratch_load_b32 v148, off, s33 offset:380
+; GISEL64-NEXT:    scratch_load_b32 v149, off, s33 offset:384
+; GISEL64-NEXT:    scratch_load_b32 v150, off, s33 offset:388
+; GISEL64-NEXT:    scratch_load_b32 v151, off, s33 offset:392
+; GISEL64-NEXT:    s_clause 0x1f
+; GISEL64-NEXT:    scratch_load_b32 v160, off, s33 offset:396
+; GISEL64-NEXT:    scratch_load_b32 v161, off, s33 offset:400
+; GISEL64-NEXT:    scratch_load_b32 v162, off, s33 offset:404
+; GISEL64-NEXT:    scratch_load_b32 v163, off, s33 offset:408
+; GISEL64-NEXT:    scratch_load_b32 v164, off, s33 offset:412
+; GISEL64-NEXT:    scratch_load_b32 v165, off, s33 offset:416
+; GISEL64-NEXT:    scratch_load_b32 v166, off, s33 offset:420
+; GISEL64-NEXT:    scratch_load_b32 v167, off, s33 offset:424
+; GISEL64-NEXT:    scratch_load_b32 v176, off, s33 offset:428
+; GISEL64-NEXT:    scratch_load_b32 v177, off, s33 offset:432
+; GISEL64-NEXT:    scratch_load_b32 v178, off, s33 offset:436
+; GISEL64-NEXT:    scratch_load_b32 v179, off, s33 offset:440
+; GISEL64-NEXT:    scratch_load_b32 v180, off, s33 offset:444
+; GISEL64-NEXT:    scratch_load_b32 v181, off, s33 offset:448
+; GISEL64-NEXT:    scratch_load_b32 v182, off, s33 offset:452
+; GISEL64-NEXT:    scratch_load_b32 v183, off, s33 offset:456
+; GISEL64-NEXT:    scratch_load_b32 v192, off, s33 offset:460
+; GISEL64-NEXT:    scratch_load_b32 v193, off, s33 offset:464
+; GISEL64-NEXT:    scratch_load_b32 v194, off, s33 offset:468
+; GISEL64-NEXT:    scratch_load_b32 v195, off, s33 offset:472
+; GISEL64-NEXT:    scratch_load_b32 v196, off, s33 offset:476
+; GISEL64-NEXT:    scratch_load_b32 v197, off, s33 offset:480
+; GISEL64-NEXT:    scratch_load_b32 v198, off, s33 offset:484
+; GISEL64-NEXT:    scratch_load_b32 v199, off, s33 offset:488
+; GISEL64-NEXT:    scratch_load_b32 v208, off, s33 offset:492
+; GISEL64-NEXT:    scratch_load_b32 v209, off, s33 offset:496
+; GISEL64-NEXT:    scratch_load_b32 v210, off, s33 offset:500
+; GISEL64-NEXT:    scratch_load_b32 v211, off, s33 offset:504
+; GISEL64-NEXT:    scratch_load_b32 v212, off, s33 offset:508
+; GISEL64-NEXT:    scratch_load_b32 v213, off, s33 offset:512
+; GISEL64-NEXT:    scratch_load_b32 v214, off, s33 offset:516
+; GISEL64-NEXT:    scratch_load_b32 v215, off, s33 offset:520
+; GISEL64-NEXT:    s_clause 0xf
+; GISEL64-NEXT:    scratch_load_b32 v224, off, s33 offset:524
+; GISEL64-NEXT:    scratch_load_b32 v225, off, s33 offset:528
+; GISEL64-NEXT:    scratch_load_b32 v226, off, s33 offset:532
+; GISEL64-NEXT:    scratch_load_b32 v227, off, s33 offset:536
+; GISEL64-NEXT:    scratch_load_b32 v228, off, s33 offset:540
+; GISEL64-NEXT:    scratch_load_b32 v229, off, s33 offset:544
+; GISEL64-NEXT:    scratch_load_b32 v230, off, s33 offset:548
+; GISEL64-NEXT:    scratch_load_b32 v231, off, s33 offset:552
+; GISEL64-NEXT:    scratch_load_b32 v240, off, s33 offset:556
+; GISEL64-NEXT:    scratch_load_b32 v241, off, s33 offset:560
+; GISEL64-NEXT:    scratch_load_b32 v242, off, s33 offset:564
+; GISEL64-NEXT:    scratch_load_b32 v243, off, s33 offset:568
+; GISEL64-NEXT:    scratch_load_b32 v244, off, s33 offset:572
+; GISEL64-NEXT:    scratch_load_b32 v245, off, s33 offset:576
+; GISEL64-NEXT:    scratch_load_b32 v246, off, s33 offset:580
+; GISEL64-NEXT:    scratch_load_b32 v247, off, s33 offset:584
+; GISEL64-NEXT:    s_mov_b64 exec, s[4:5]
+; GISEL64-NEXT:    s_mov_b32 s33, s0
+; GISEL64-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GISEL64-NEXT:    s_wait_alu 0xfffe
+; GISEL64-NEXT:    s_setpc_b64 s[30:31]
+  %ret = call float(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @callee, <8 x float> %x) convergent
+  store float %ret, ptr %p
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/wqm.mir b/llvm/test/CodeGen/AMDGPU/wqm.mir
index 350b233cb4b62..ceb1b3e16d727 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm.mir
+++ b/llvm/test/CodeGen/AMDGPU/wqm.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 # RUN: llc -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass si-wqm -o -  %s | FileCheck %s
 # RUN: llc -mtriple=amdgcn -mcpu=fiji -passes=si-wqm -o -  %s | FileCheck %s
 
@@ -46,10 +47,6 @@
 
 ---
 # Check for awareness that s_or_saveexec_b64 clobbers SCC
-#
-#CHECK: ENTER_STRICT_WWM
-#CHECK: S_CMP_LT_I32
-#CHECK: S_CSELECT_B32
 name:            test_strict_wwm_scc
 alignment:       1
 exposesReturnsTwice: false
@@ -80,6 +77,21 @@ body:             |
   bb.0:
     liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0
 
+    ; CHECK-LABEL: name: test_strict_wwm_scc
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[ENTER_STRICT_WWM:%[0-9]+]]:sreg_64 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; CHECK-NEXT: S_CMP_LT_I32 0, [[COPY3]], implicit-def $scc
+    ; CHECK-NEXT: [[V_ADD_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 [[COPY]], [[COPY]], implicit-def $vcc, implicit $exec
+    ; CHECK-NEXT: [[S_CSELECT_B32_:%[0-9]+]]:sgpr_32 = S_CSELECT_B32 [[COPY1]], [[COPY2]], implicit $scc
+    ; CHECK-NEXT: [[V_ADD_CO_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 [[S_CSELECT_B32_]], [[V_ADD_CO_U32_e32_]], implicit-def $vcc, implicit $exec
+    ; CHECK-NEXT: $exec = EXIT_STRICT_WWM [[ENTER_STRICT_WWM]]
+    ; CHECK-NEXT: early-clobber $vgpr0 = V_MOV_B32_e32 [[V_ADD_CO_U32_e32_1]], implicit $exec
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG $vgpr0
     %3 = COPY $vgpr0
     %2 = COPY $sgpr2
     %1 = COPY $sgpr1
@@ -96,16 +108,35 @@ body:             |
 ---
 # Second test for awareness that s_or_saveexec_b64 clobbers SCC
 # Because entry block is treated differently.
-#
-#CHECK: %bb.1
-#CHECK: S_CMP_LT_I32
-#CHECK: COPY $scc
-#CHECK: ENTER_STRICT_WWM
-#CHECK: $scc = COPY
-#CHECK: S_CSELECT_B32
 name:            test_strict_wwm_scc2
 tracksRegLiveness: true
 body:             |
+  ; CHECK-LABEL: name: test_strict_wwm_scc2
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[ENTER_STRICT_WWM:%[0-9]+]]:sreg_64 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK-NEXT:   $exec = EXIT_STRICT_WWM [[ENTER_STRICT_WWM]]
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   S_CMP_LT_I32 0, [[COPY3]], implicit-def $scc
+  ; CHECK-NEXT:   [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY]], [[DEF]], 0, 0, 0, 0, implicit $exec
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+  ; CHECK-NEXT:   [[ENTER_STRICT_WWM1:%[0-9]+]]:sreg_64 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK-NEXT:   $scc = COPY [[COPY4]]
+  ; CHECK-NEXT:   [[V_ADD_CO_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 [[COPY]], [[COPY]], implicit-def $vcc, implicit $exec
+  ; CHECK-NEXT:   [[S_CSELECT_B32_:%[0-9]+]]:sgpr_32 = S_CSELECT_B32 [[COPY1]], [[COPY2]], implicit $scc
+  ; CHECK-NEXT:   [[V_ADD_CO_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_CO_U32_e32 [[S_CSELECT_B32_]], [[V_ADD_CO_U32_e32_]], implicit-def $vcc, implicit $exec
+  ; CHECK-NEXT:   $exec = EXIT_STRICT_WWM [[ENTER_STRICT_WWM1]]
+  ; CHECK-NEXT:   early-clobber $vgpr0 = V_MOV_B32_e32 [[V_ADD_CO_U32_e32_1]], implicit $exec
+  ; CHECK-NEXT:   $vgpr1 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
+  ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $vgpr0, $vgpr1
   bb.0:
     liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0
 
@@ -130,7 +161,6 @@ body:             |
 ---
 # V_SET_INACTIVE, when its second operand is undef, is replaced by a
 # COPY by si-wqm. Ensure the instruction is removed.
-#CHECK-NOT: V_SET_INACTIVE
 name:            no_cfg
 alignment:       1
 exposesReturnsTwice: false
@@ -167,6 +197,28 @@ body:             |
   bb.0:
     liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
 
+    ; CHECK-LABEL: name: no_cfg
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
+    ; CHECK-NEXT: dead [[COPY4:%[0-9]+]]:sgpr_128 = COPY [[REG_SEQUENCE]]
+    ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX2_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX2_OFFSET]].sub1
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+    ; CHECK-NEXT: dead [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+    ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY6]], implicit $exec, implicit-def $scc
+    ; CHECK-NEXT: [[ENTER_STRICT_WWM:%[0-9]+]]:sreg_64 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
+    ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+    ; CHECK-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[COPY8]], [[COPY7]], 323, 12, 15, 0, implicit $exec
+    ; CHECK-NEXT: $exec = EXIT_STRICT_WWM [[ENTER_STRICT_WWM]]
+    ; CHECK-NEXT: early-clobber %15:vgpr_32 = V_MOV_B32_e32 [[V_MOV_B32_dpp]], implicit $exec
+    ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET_exact %15, [[REG_SEQUENCE]], [[S_MOV_B32_]], 4, 0, 0, implicit $exec
+    ; CHECK-NEXT: S_ENDPGM 0
     %3:sgpr_32 = COPY $sgpr3
     %2:sgpr_32 = COPY $sgpr2
     %1:sgpr_32 = COPY $sgpr1
@@ -189,18 +241,32 @@ body:             |
 
 ---
 # Ensure that strict_wwm is not put around an EXEC copy
-#CHECK-LABEL: name: copy_exec
-#CHECK: %7:sreg_64 = COPY $exec
-#CHECK-NEXT: %13:sreg_64 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
-#CHECK-NEXT: %8:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-#CHECK-NEXT: $exec = EXIT_STRICT_WWM %13
-#CHECK-NEXT: %9:vgpr_32 = V_MBCNT_LO_U32_B32_e64 %7.sub0, 0, implicit $exec
 name:            copy_exec
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
 
+    ; CHECK-LABEL: name: copy_exec
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_32 = COPY $sgpr3
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr_32 = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+    ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY1]], %subreg.sub2, [[COPY]], %subreg.sub3
+    ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; CHECK-NEXT: dead [[BUFFER_LOAD_DWORDX2_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, implicit $exec
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec
+    ; CHECK-NEXT: [[ENTER_STRICT_WWM:%[0-9]+]]:sreg_64 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
+    ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; CHECK-NEXT: $exec = EXIT_STRICT_WWM [[ENTER_STRICT_WWM]]
+    ; CHECK-NEXT: [[V_MBCNT_LO_U32_B32_e64_:%[0-9]+]]:vgpr_32 = V_MBCNT_LO_U32_B32_e64 [[COPY4]].sub0, 0, implicit $exec
+    ; CHECK-NEXT: [[V_MOV_B32_dpp:%[0-9]+]]:vgpr_32 = V_MOV_B32_dpp [[V_MOV_B32_e32_]], [[V_MBCNT_LO_U32_B32_e64_]], 312, 15, 15, 0, implicit $exec
+    ; CHECK-NEXT: dead [[V_READLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READLANE_B32 [[V_MOV_B32_dpp]], 63
+    ; CHECK-NEXT: early-clobber %12:vgpr_32 = V_MOV_B32_e32 [[V_MOV_B32_e32_]], implicit $exec
+    ; CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET_exact %12, [[REG_SEQUENCE]], [[S_MOV_B32_]], 4, 0, 0, implicit $exec
+    ; CHECK-NEXT: S_ENDPGM 0
     %3:sgpr_32 = COPY $sgpr3
     %2:sgpr_32 = COPY $sgpr2
     %1:sgpr_32 = COPY $sgpr1
@@ -224,20 +290,48 @@ body:             |
 ---
 # Check exit of WQM is still inserted correctly when SCC is live until block end.
 # Critially this tests that compilation does not fail.
-#CHECK-LABEL: name: scc_always_live
-#CHECK: %8:vreg_128 = IMAGE_SAMPLE_V4_V2 %7
-#CHECK-NEXT: S_CMP_EQ_U32 %2, 0, implicit-def $scc
-#CHECK-NEXT: undef %9.sub0:vreg_64 = nsz arcp nofpexcept V_ADD_F32_e64
-#CHECK-NEXT: %9.sub1:vreg_64 = nsz arcp nofpexcept V_MUL_F32_e32
-#CHECK-NEXT: %14:sreg_32_xm0 = COPY $scc
-#CHECK-NEXT: $exec = S_AND_B64 $exec, %13, implicit-def $scc
-#CHECK-NEXT: $scc = COPY %14
-#CHECK-NEXT: %10:vgpr_32 = nsz arcp nofpexcept V_ADD_F32_e64
-#CHECK-NEXT: %11:vreg_128 = IMAGE_SAMPLE_V4_V2
-#CHECK-NEXT: S_CBRANCH_SCC0 %bb.2
 name:            scc_always_live
 tracksRegLiveness: true
 body:             |
+  ; CHECK-LABEL: name: scc_always_live
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; CHECK-NEXT:   liveins: $sgpr1, $sgpr2, $vgpr1, $vgpr2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sreg_64 = COPY $exec
+  ; CHECK-NEXT:   $m0 = COPY $sgpr1
+  ; CHECK-NEXT:   $exec = S_WQM_B64 $exec, implicit-def $scc
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:sgpr_32 = COPY $sgpr2
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[V_INTERP_P1_F32_:%[0-9]+]]:vgpr_32 = V_INTERP_P1_F32 [[COPY1]], 3, 2, implicit $mode, implicit $m0, implicit $exec
+  ; CHECK-NEXT:   [[V_INTERP_P1_F32_1:%[0-9]+]]:vgpr_32 = V_INTERP_P1_F32 [[COPY2]], 3, 2, implicit $mode, implicit $m0, implicit $exec
+  ; CHECK-NEXT:   undef [[COPY4:%[0-9]+]].sub0:vreg_64 = COPY [[V_INTERP_P1_F32_]]
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]].sub1:vreg_64 = COPY [[V_INTERP_P1_F32_1]]
+  ; CHECK-NEXT:   [[IMAGE_SAMPLE_V4_V2_:%[0-9]+]]:vreg_128 = IMAGE_SAMPLE_V4_V2 [[COPY4]], [[DEF]], [[DEF1]], 15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 4, addrspace 4)
+  ; CHECK-NEXT:   S_CMP_EQ_U32 [[COPY3]], 0, implicit-def $scc
+  ; CHECK-NEXT:   undef [[V_ADD_F32_e64_:%[0-9]+]].sub0:vreg_64 = nsz arcp nofpexcept V_ADD_F32_e64 0, [[IMAGE_SAMPLE_V4_V2_]].sub0, 0, [[V_INTERP_P1_F32_1]], 1, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_ADD_F32_e64_:%[0-9]+]].sub1:vreg_64 = nsz arcp nofpexcept V_MUL_F32_e32 [[V_INTERP_P1_F32_]], [[V_INTERP_P1_F32_1]], implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[COPY5:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+  ; CHECK-NEXT:   $exec = S_AND_B64 $exec, [[COPY]], implicit-def $scc
+  ; CHECK-NEXT:   $scc = COPY [[COPY5]]
+  ; CHECK-NEXT:   [[V_ADD_F32_e64_1:%[0-9]+]]:vgpr_32 = nsz arcp nofpexcept V_ADD_F32_e64 0, [[V_INTERP_P1_F32_]], 0, [[V_INTERP_P1_F32_1]], 1, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[IMAGE_SAMPLE_V4_V2_1:%[0-9]+]]:vreg_128 = IMAGE_SAMPLE_V4_V2 [[V_ADD_F32_e64_]], [[DEF]], [[DEF1]], 15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 4, addrspace 4)
+  ; CHECK-NEXT:   S_CBRANCH_SCC0 %bb.2, implicit $scc
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+  ; CHECK-NEXT:   BUFFER_STORE_DWORD_OFFSET_exact [[V_ADD_F32_e64_1]], [[DEF1]], [[S_MOV_B32_]], 4, 0, 0, implicit $exec
+  ; CHECK-NEXT:   S_ENDPGM 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   $vgpr0 = COPY [[IMAGE_SAMPLE_V4_V2_]].sub0
+  ; CHECK-NEXT:   $vgpr1 = COPY [[IMAGE_SAMPLE_V4_V2_]].sub1
+  ; CHECK-NEXT:   $vgpr2 = COPY [[IMAGE_SAMPLE_V4_V2_1]].sub0
+  ; CHECK-NEXT:   $vgpr3 = COPY [[IMAGE_SAMPLE_V4_V2_1]].sub1
+  ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $vgpr0, $vgpr1, $vgpr2, $vgpr3
   bb.0:
     liveins: $sgpr1, $sgpr2, $vgpr1, $vgpr2
 
@@ -281,18 +375,26 @@ body:             |
 ---
 # Check that unnecessary instruction do not get marked for WWM
 #
-#CHECK-NOT: ENTER_STRICT_WWM
-#CHECK: BUFFER_LOAD_DWORDX2
-#CHECK: ENTER_STRICT_WWM
-#CHECK: V_SET_INACTIVE_B32
-#CHECK: V_SET_INACTIVE_B32
-#CHECK-NOT: ENTER_STRICT_WWM
-#CHECK: V_MAX
 name:            test_wwm_set_inactive_propagation
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; CHECK-LABEL: name: test_wwm_set_inactive_propagation
+    ; CHECK: liveins: $sgpr0_sgpr1_sgpr2_sgpr3, $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN [[COPY1]], [[COPY]], 0, 0, 0, 0, implicit $exec
+    ; CHECK-NEXT: [[ENTER_STRICT_WWM:%[0-9]+]]:sreg_64_xexec = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
+    ; CHECK-NEXT: dead [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+    ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]].sub0:vreg_64 = V_SET_INACTIVE_B32 0, [[BUFFER_LOAD_DWORDX2_OFFEN]].sub0, 0, 0, undef [[ENTER_STRICT_WWM]], implicit $exec, implicit-def $scc
+    ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX2_OFFEN:%[0-9]+]].sub1:vreg_64 = V_SET_INACTIVE_B32 0, [[BUFFER_LOAD_DWORDX2_OFFEN]].sub1, 0, 0, undef [[ENTER_STRICT_WWM]], implicit $exec, implicit-def $scc
+    ; CHECK-NEXT: [[V_MAX_F64_e64_:%[0-9]+]]:vreg_64 = nnan nsz arcp contract reassoc nofpexcept V_MAX_F64_e64 0, [[BUFFER_LOAD_DWORDX2_OFFEN]], 0, [[BUFFER_LOAD_DWORDX2_OFFEN]], 0, 0, implicit $mode, implicit $exec
+    ; CHECK-NEXT: $exec = EXIT_STRICT_WWM [[ENTER_STRICT_WWM]]
+    ; CHECK-NEXT: early-clobber $vgpr0 = V_MOV_B32_e32 [[V_MAX_F64_e64_]].sub0, implicit $exec
+    ; CHECK-NEXT: early-clobber $vgpr1 = V_MOV_B32_e32 [[V_MAX_F64_e64_]].sub1, implicit $exec
+    ; CHECK-NEXT: SI_RETURN_TO_EPILOG $vgpr0, $vgpr1
     %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:vgpr_32 = COPY $vgpr0
     %2:vreg_64 = BUFFER_LOAD_DWORDX2_OFFEN %1:vgpr_32, %0:sgpr_128, 0, 0, 0, 0, implicit $exec
@@ -308,15 +410,46 @@ body:             |
 ---
 # Check that WQM marking occurs correctly through phi nodes in live range graph.
 # If not then initial V_MOV will not be in WQM.
-#
-#CHECK-LABEL: name: test_wqm_lr_phi
-#CHECK: COPY $exec
-#CHECK-NEXT: S_WQM
-#CHECK-NEXT: V_MOV_B32_e32 -10
-#CHECK-NEXT: V_MOV_B32_e32 0
 name:            test_wqm_lr_phi
 tracksRegLiveness: true
 body:             |
+  ; CHECK-LABEL: name: test_wqm_lr_phi
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:sreg_64 = COPY $exec
+  ; CHECK-NEXT:   $exec = S_WQM_B64 $exec, implicit-def $scc
+  ; CHECK-NEXT:   undef [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_64 = V_MOV_B32_e32 -10, implicit $exec
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK-NEXT:   [[S_GETPC_B64_:%[0-9]+]]:sreg_64 = S_GETPC_B64
+  ; CHECK-NEXT:   [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[S_GETPC_B64_]], 32, 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $vcc = V_CMP_LT_U32_e64 4, 4, implicit $exec
+  ; CHECK-NEXT:   S_CBRANCH_VCCNZ %bb.3, implicit $vcc
+  ; CHECK-NEXT:   S_BRANCH %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.3(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_64 = V_ADD_U32_e32 1, [[V_MOV_B32_e32_]].sub1, implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.3
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_64 = V_ADD_U32_e32 1, [[V_MOV_B32_e32_]].sub1, implicit $exec
+  ; CHECK-NEXT:   S_BRANCH %bb.4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   $exec = S_AND_B64 $exec, [[COPY]], implicit-def $scc
+  ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+  ; CHECK-NEXT:   [[IMAGE_SAMPLE_V4_V2_:%[0-9]+]]:vreg_128 = IMAGE_SAMPLE_V4_V2 [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX8_IMM]], [[DEF]], 15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), addrspace 7)
+  ; CHECK-NEXT:   $vgpr0 = COPY [[IMAGE_SAMPLE_V4_V2_]].sub0
+  ; CHECK-NEXT:   $vgpr1 = COPY [[IMAGE_SAMPLE_V4_V2_]].sub1
+  ; CHECK-NEXT:   SI_RETURN_TO_EPILOG $vgpr0, $vgpr1
   bb.0:
     undef %0.sub0:vreg_64 = V_MOV_B32_e32 -10, implicit $exec
     %0.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
@@ -345,14 +478,20 @@ body:             |
 ...
 
 ---
-#CHECK-LABEL: name: no_wqm_in_cs
-#CHECK-NOT: S_WQM
 name:            no_wqm_in_cs
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $vgpr1, $vgpr2
 
+    ; CHECK-LABEL: name: no_wqm_in_cs
+    ; CHECK: liveins: $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: undef [[COPY:%[0-9]+]].sub0:vreg_64 = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY:%[0-9]+]].sub1:vreg_64 = COPY $vgpr2
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+    ; CHECK-NEXT: dead [[IMAGE_SAMPLE_V4_V2_:%[0-9]+]]:vreg_128 = IMAGE_SAMPLE_V4_V2 [[COPY]], [[DEF]], [[DEF1]], 15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 4, addrspace 4)
     undef %0.sub0:vreg_64 = COPY $vgpr1
     %0.sub1:vreg_64 = COPY $vgpr2
     %100:sgpr_256 = IMPLICIT_DEF
@@ -362,14 +501,20 @@ body:             |
 ...
 
 ---
-#CHECK-LABEL: name: no_wqm_in_es
-#CHECK-NOT: S_WQM
 name:            no_wqm_in_es
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $vgpr1, $vgpr2
 
+    ; CHECK-LABEL: name: no_wqm_in_es
+    ; CHECK: liveins: $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: undef [[COPY:%[0-9]+]].sub0:vreg_64 = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY:%[0-9]+]].sub1:vreg_64 = COPY $vgpr2
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+    ; CHECK-NEXT: dead [[IMAGE_SAMPLE_V4_V2_:%[0-9]+]]:vreg_128 = IMAGE_SAMPLE_V4_V2 [[COPY]], [[DEF]], [[DEF1]], 15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 4, addrspace 4)
     undef %0.sub0:vreg_64 = COPY $vgpr1
     %0.sub1:vreg_64 = COPY $vgpr2
     %100:sgpr_256 = IMPLICIT_DEF
@@ -379,14 +524,20 @@ body:             |
 ...
 
 ---
-#CHECK-LABEL: name: no_wqm_in_gs
-#CHECK-NOT: S_WQM
 name:            no_wqm_in_gs
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $vgpr1, $vgpr2
 
+    ; CHECK-LABEL: name: no_wqm_in_gs
+    ; CHECK: liveins: $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: undef [[COPY:%[0-9]+]].sub0:vreg_64 = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY:%[0-9]+]].sub1:vreg_64 = COPY $vgpr2
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+    ; CHECK-NEXT: dead [[IMAGE_SAMPLE_V4_V2_:%[0-9]+]]:vreg_128 = IMAGE_SAMPLE_V4_V2 [[COPY]], [[DEF]], [[DEF1]], 15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 4, addrspace 4)
     undef %0.sub0:vreg_64 = COPY $vgpr1
     %0.sub1:vreg_64 = COPY $vgpr2
     %100:sgpr_256 = IMPLICIT_DEF
@@ -396,14 +547,20 @@ body:             |
 ...
 
 ---
-#CHECK-LABEL: name: no_wqm_in_hs
-#CHECK-NOT: S_WQM
 name:            no_wqm_in_hs
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $vgpr1, $vgpr2
 
+    ; CHECK-LABEL: name: no_wqm_in_hs
+    ; CHECK: liveins: $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: undef [[COPY:%[0-9]+]].sub0:vreg_64 = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY:%[0-9]+]].sub1:vreg_64 = COPY $vgpr2
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+    ; CHECK-NEXT: dead [[IMAGE_SAMPLE_V4_V2_:%[0-9]+]]:vreg_128 = IMAGE_SAMPLE_V4_V2 [[COPY]], [[DEF]], [[DEF1]], 15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 4, addrspace 4)
     undef %0.sub0:vreg_64 = COPY $vgpr1
     %0.sub1:vreg_64 = COPY $vgpr2
     %100:sgpr_256 = IMPLICIT_DEF
@@ -413,14 +570,20 @@ body:             |
 ...
 
 ---
-#CHECK-LABEL: name: no_wqm_in_ls
-#CHECK-NOT: S_WQM
 name:            no_wqm_in_ls
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $vgpr1, $vgpr2
 
+    ; CHECK-LABEL: name: no_wqm_in_ls
+    ; CHECK: liveins: $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: undef [[COPY:%[0-9]+]].sub0:vreg_64 = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY:%[0-9]+]].sub1:vreg_64 = COPY $vgpr2
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+    ; CHECK-NEXT: dead [[IMAGE_SAMPLE_V4_V2_:%[0-9]+]]:vreg_128 = IMAGE_SAMPLE_V4_V2 [[COPY]], [[DEF]], [[DEF1]], 15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 4, addrspace 4)
     undef %0.sub0:vreg_64 = COPY $vgpr1
     %0.sub1:vreg_64 = COPY $vgpr2
     %100:sgpr_256 = IMPLICIT_DEF
@@ -430,14 +593,20 @@ body:             |
 ...
 
 ---
-#CHECK-LABEL: name: no_wqm_in_vs
-#CHECK-NOT: S_WQM
 name:            no_wqm_in_vs
 tracksRegLiveness: true
 body:             |
   bb.0:
     liveins: $vgpr1, $vgpr2
 
+    ; CHECK-LABEL: name: no_wqm_in_vs
+    ; CHECK: liveins: $vgpr1, $vgpr2
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: undef [[COPY:%[0-9]+]].sub0:vreg_64 = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY:%[0-9]+]].sub1:vreg_64 = COPY $vgpr2
+    ; CHECK-NEXT: [[DEF:%[0-9]+]]:sgpr_256 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[DEF1:%[0-9]+]]:sgpr_128 = IMPLICIT_DEF
+    ; CHECK-NEXT: dead [[IMAGE_SAMPLE_V4_V2_:%[0-9]+]]:vreg_128 = IMAGE_SAMPLE_V4_V2 [[COPY]], [[DEF]], [[DEF1]], 15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 4, addrspace 4)
     undef %0.sub0:vreg_64 = COPY $vgpr1
     %0.sub1:vreg_64 = COPY $vgpr2
     %100:sgpr_256 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/ARM/2013-05-05-IfConvertBug.ll b/llvm/test/CodeGen/ARM/2013-05-05-IfConvertBug.ll
index 344bb15d2a8b8..8f798fac06f54 100644
--- a/llvm/test/CodeGen/ARM/2013-05-05-IfConvertBug.ll
+++ b/llvm/test/CodeGen/ARM/2013-05-05-IfConvertBug.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=thumbv7-apple-ios -mcpu=cortex-a8 | FileCheck %s
-; RUN: llc < %s -mtriple=thumbv8 | FileCheck -check-prefix=CHECK-V8 %s
-; RUN: llc < %s -mtriple=thumbv7 -arm-restrict-it | FileCheck -check-prefix=CHECK-RESTRICT-IT %s
+; RUN: llc -keep-loops="false" < %s -mtriple=thumbv7-apple-ios -mcpu=cortex-a8 | FileCheck %s
+; RUN: llc -keep-loops="false" < %s -mtriple=thumbv8 | FileCheck -check-prefix=CHECK-V8 %s
+; RUN: llc -keep-loops="false" < %s -mtriple=thumbv7 -arm-restrict-it | FileCheck -check-prefix=CHECK-RESTRICT-IT %s
 
 define i32 @t1(i32 %a, i32 %b, ptr %retaddr) {
 ; CHECK-LABEL: t1:
diff --git a/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir b/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir
index 29429fd5a23eb..9bc335c2ab09a 100644
--- a/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir
+++ b/llvm/test/CodeGen/ARM/cmse-vlldm-no-reorder.mir
@@ -55,8 +55,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/ARM/codesize-ifcvt.mir b/llvm/test/CodeGen/ARM/codesize-ifcvt.mir
index 3acbcf127d8a3..53a2a4ab5b166 100644
--- a/llvm/test/CodeGen/ARM/codesize-ifcvt.mir
+++ b/llvm/test/CodeGen/ARM/codesize-ifcvt.mir
@@ -142,8 +142,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -304,8 +304,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -472,8 +472,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/ARM/constant-island-movwt.mir b/llvm/test/CodeGen/ARM/constant-island-movwt.mir
index 7b3e59eca8472..215d73f6d9774 100644
--- a/llvm/test/CodeGen/ARM/constant-island-movwt.mir
+++ b/llvm/test/CodeGen/ARM/constant-island-movwt.mir
@@ -341,8 +341,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/ARM/constant-islands-cfg.mir b/llvm/test/CodeGen/ARM/constant-islands-cfg.mir
index d85e7bf4bdeb3..c4d6a47854921 100644
--- a/llvm/test/CodeGen/ARM/constant-islands-cfg.mir
+++ b/llvm/test/CodeGen/ARM/constant-islands-cfg.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 
 body:             |
diff --git a/llvm/test/CodeGen/ARM/constant-islands-split-IT.mir b/llvm/test/CodeGen/ARM/constant-islands-split-IT.mir
index 236cd34f7fba5..60f4ec612aaa3 100644
--- a/llvm/test/CodeGen/ARM/constant-islands-split-IT.mir
+++ b/llvm/test/CodeGen/ARM/constant-islands-split-IT.mir
@@ -47,8 +47,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  28
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 callSites:       []
 constants:
diff --git a/llvm/test/CodeGen/ARM/execute-only-save-cpsr.mir b/llvm/test/CodeGen/ARM/execute-only-save-cpsr.mir
index cdd8ed49c6156..6a6d474fd283d 100644
--- a/llvm/test/CodeGen/ARM/execute-only-save-cpsr.mir
+++ b/llvm/test/CodeGen/ARM/execute-only-save-cpsr.mir
@@ -118,8 +118,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  2052
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: var, type: default, offset: 0, size: 4, alignment: 4,
@@ -204,8 +204,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  2052
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: var, type: default, offset: 0, size: 4, alignment: 4,
@@ -296,8 +296,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  2052
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: var, type: default, offset: 0, size: 4, alignment: 4,
@@ -388,8 +388,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  2052
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: var, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/ARM/fp-fast.ll b/llvm/test/CodeGen/ARM/fp-fast.ll
index 7d95a5efe9052..6e1c783bfbe3e 100644
--- a/llvm/test/CodeGen/ARM/fp-fast.ll
+++ b/llvm/test/CodeGen/ARM/fp-fast.ll
@@ -1,5 +1,4 @@
-; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a9 -mattr=+vfp4 -enable-unsafe-fp-math %s -o - \
-; RUN:  | FileCheck %s
+; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a9 -mattr=+vfp4 %s -o - | FileCheck %s
 
 ; CHECK: test1
 define float @test1(float %x) {
@@ -7,7 +6,7 @@ define float @test1(float %x) {
 ; CHECK: vmul.f32
 ; CHECK-NOT: vfma
   %t1 = fmul float %x, 3.0
-  %t2 = call float @llvm.fma.f32(float %x, float 2.0, float %t1)
+  %t2 = call reassoc float @llvm.fma.f32(float %x, float 2.0, float %t1)
   ret float %t2
 }
 
@@ -17,7 +16,7 @@ define float @test2(float %x, float %y) {
 ; CHECK: vfma.f32
 ; CHECK-NOT: vmul
   %t1 = fmul float %x, 3.0
-  %t2 = call float @llvm.fma.f32(float %t1, float 2.0, float %y)
+  %t2 = call reassoc float @llvm.fma.f32(float %t1, float 2.0, float %y)
   ret float %t2
 }
 
@@ -44,7 +43,7 @@ define float @test5(float %x) {
 ; CHECK-NOT: vfma
 ; CHECK: vmul.f32
 ; CHECK-NOT: vfma
-  %t2 = call float @llvm.fma.f32(float %x, float 2.0, float %x)
+  %t2 = call reassoc float @llvm.fma.f32(float %x, float 2.0, float %x)
   ret float %t2
 }
 
@@ -54,7 +53,7 @@ define float @test6(float %x) {
 ; CHECK: vmul.f32
 ; CHECK-NOT: vfma
   %t1 = fsub float -0.0, %x
-  %t2 = call float @llvm.fma.f32(float %x, float 5.0, float %t1)
+  %t2 = call reassoc float @llvm.fma.f32(float %x, float 5.0, float %t1)
   ret float %t2
 }
 
diff --git a/llvm/test/CodeGen/ARM/fp16-litpool2-arm.mir b/llvm/test/CodeGen/ARM/fp16-litpool2-arm.mir
index bd343ebef26ad..46f028bd492db 100644
--- a/llvm/test/CodeGen/ARM/fp16-litpool2-arm.mir
+++ b/llvm/test/CodeGen/ARM/fp16-litpool2-arm.mir
@@ -61,8 +61,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: res, type: default, offset: -2, size: 2, alignment: 2,
diff --git a/llvm/test/CodeGen/ARM/fp16-litpool3-arm.mir b/llvm/test/CodeGen/ARM/fp16-litpool3-arm.mir
index 1f8e6b0ad4216..5a03fcdb7fdf7 100644
--- a/llvm/test/CodeGen/ARM/fp16-litpool3-arm.mir
+++ b/llvm/test/CodeGen/ARM/fp16-litpool3-arm.mir
@@ -62,8 +62,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: res, type: default, offset: -2, size: 2, alignment: 2,
diff --git a/llvm/test/CodeGen/ARM/inlineasmbr-if-cvt.mir b/llvm/test/CodeGen/ARM/inlineasmbr-if-cvt.mir
index 2d53074923151..9a7ac0acdbfab 100644
--- a/llvm/test/CodeGen/ARM/inlineasmbr-if-cvt.mir
+++ b/llvm/test/CodeGen/ARM/inlineasmbr-if-cvt.mir
@@ -69,8 +69,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/ARM/invalidated-save-point.ll b/llvm/test/CodeGen/ARM/invalidated-save-point.ll
index bb602308a1793..0ff153b6799d7 100644
--- a/llvm/test/CodeGen/ARM/invalidated-save-point.ll
+++ b/llvm/test/CodeGen/ARM/invalidated-save-point.ll
@@ -4,8 +4,8 @@
 ; this point. Notably, if it isn't is will be invalid and reference a
 ; deleted block (%bb.-1.if.end)
 
-; CHECK: savePoint: ''
-; CHECK: restorePoint: ''
+; CHECK-NOT: savePoint:
+; CHECK-NOT: restorePoint:
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 target triple = "thumbv7"
diff --git a/llvm/test/CodeGen/ARM/jump-table-dbg-value.mir b/llvm/test/CodeGen/ARM/jump-table-dbg-value.mir
index ec475e1b52ef1..4e7628aaeb434 100644
--- a/llvm/test/CodeGen/ARM/jump-table-dbg-value.mir
+++ b/llvm/test/CodeGen/ARM/jump-table-dbg-value.mir
@@ -98,8 +98,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/ARM/legalize-bitcast.ll b/llvm/test/CodeGen/ARM/legalize-bitcast.ll
index 5b989a099c815..92b77f57915a4 100644
--- a/llvm/test/CodeGen/ARM/legalize-bitcast.ll
+++ b/llvm/test/CodeGen/ARM/legalize-bitcast.ll
@@ -7,7 +7,7 @@ define i32 @vec_to_int() {
 ; CHECK-LABEL: vec_to_int:
 ; CHECK:       @ %bb.0: @ %bb.0
 ; CHECK-NEXT:    push {r4}
-; CHECK-NEXT:    sub sp, sp, #28
+; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    movw r0, :lower16:vec6_p
 ; CHECK-NEXT:    movt r0, :upper16:vec6_p
 ; CHECK-NEXT:    vld1.8 {d16}, [r0]!
@@ -25,7 +25,7 @@ define i32 @vec_to_int() {
 ; CHECK-NEXT:    vrev32.16 q8, q8
 ; CHECK-NEXT:    vmov.f64 d16, d17
 ; CHECK-NEXT:    vmov.32 r0, d16[0]
-; CHECK-NEXT:    add sp, sp, #28
+; CHECK-NEXT:    add sp, sp, #16
 ; CHECK-NEXT:    pop {r4}
 ; CHECK-NEXT:    bx lr
 bb.0:
diff --git a/llvm/test/CodeGen/ARM/llrint-conv.ll b/llvm/test/CodeGen/ARM/llrint-conv.ll
index 017955bb43afb..749ee00a3c68e 100644
--- a/llvm/test/CodeGen/ARM/llrint-conv.ll
+++ b/llvm/test/CodeGen/ARM/llrint-conv.ll
@@ -1,13 +1,23 @@
 ; RUN: llc < %s -mtriple=arm-eabi -float-abi=soft | FileCheck %s --check-prefix=SOFTFP
 ; RUN: llc < %s -mtriple=arm-eabi -float-abi=hard | FileCheck %s --check-prefix=HARDFP
 
+; SOFTFP-LABEL: testmsxh_builtin:
+; SOFTFP:       bl      llrintf
+; HARDFP-LABEL: testmsxh_builtin:
+; HARDFP:       bl      llrintf
+define i64 @testmsxh_builtin(half %x) {
+entry:
+  %0 = tail call i64 @llvm.llrint.i64.f16(half %x)
+  ret i64 %0
+}
+
 ; SOFTFP-LABEL: testmsxs_builtin:
 ; SOFTFP:       bl      llrintf
 ; HARDFP-LABEL: testmsxs_builtin:
 ; HARDFP:       bl      llrintf
 define i64 @testmsxs_builtin(float %x) {
 entry:
-  %0 = tail call i64 @llvm.llrint.f32(float %x)
+  %0 = tail call i64 @llvm.llrint.i64.f32(float %x)
   ret i64 %0
 }
 
@@ -17,9 +27,20 @@ entry:
 ; HARDFP:       bl      llrint
 define i64 @testmsxd_builtin(double %x) {
 entry:
-  %0 = tail call i64 @llvm.llrint.f64(double %x)
+  %0 = tail call i64 @llvm.llrint.i64.f64(double %x)
+  ret i64 %0
+}
+
+; FIXME(#44744): incorrect libcall
+; SOFTFP-LABEL: testmsxq_builtin:
+; SOFTFP:       bl      llrintl
+; HARDFP-LABEL: testmsxq_builtin:
+; HARDFP:       bl      llrintl
+define i64 @testmsxq_builtin(fp128 %x) {
+entry:
+  %0 = tail call i64 @llvm.llrint.i64.f128(fp128 %x)
   ret i64 %0
 }
 
-declare i64 @llvm.llrint.f32(float) nounwind readnone
-declare i64 @llvm.llrint.f64(double) nounwind readnone
+declare i64 @llvm.llrint.i64.f32(float) nounwind readnone
+declare i64 @llvm.llrint.i64.f64(double) nounwind readnone
diff --git a/llvm/test/CodeGen/ARM/lrint-conv.ll b/llvm/test/CodeGen/ARM/lrint-conv.ll
index 192da565c12fd..9aa95112af533 100644
--- a/llvm/test/CodeGen/ARM/lrint-conv.ll
+++ b/llvm/test/CodeGen/ARM/lrint-conv.ll
@@ -1,6 +1,13 @@
 ; RUN: llc < %s -mtriple=arm-eabi -float-abi=soft | FileCheck %s --check-prefix=SOFTFP
 ; RUN: llc < %s -mtriple=arm-eabi -float-abi=hard | FileCheck %s --check-prefix=HARDFP
 
+; FIXME: crash
+; define i32 @testmswh_builtin(half %x) {
+; entry:
+;   %0 = tail call i32 @llvm.lrint.i32.f16(half %x)
+;   ret i32 %0
+; }
+
 ; SOFTFP-LABEL: testmsws_builtin:
 ; SOFTFP:       bl      lrintf
 ; HARDFP-LABEL: testmsws_builtin:
@@ -21,5 +28,16 @@ entry:
   ret i32 %0
 }
 
+; FIXME(#44744): incorrect libcall
+; SOFTFP-LABEL: testmswq_builtin:
+; SOFTFP:       bl      lrintl
+; HARDFP-LABEL: testmswq_builtin:
+; HARDFP:       bl      lrintl
+define i32 @testmswq_builtin(fp128 %x) {
+entry:
+  %0 = tail call i32 @llvm.lrint.i32.f128(fp128 %x)
+  ret i32 %0
+}
+
 declare i32 @llvm.lrint.i32.f32(float) nounwind readnone
 declare i32 @llvm.lrint.i32.f64(double) nounwind readnone
diff --git a/llvm/test/CodeGen/ARM/scmp.ll b/llvm/test/CodeGen/ARM/scmp.ll
index 6e493c993751c..9189aee6aaf43 100644
--- a/llvm/test/CodeGen/ARM/scmp.ll
+++ b/llvm/test/CodeGen/ARM/scmp.ll
@@ -4,12 +4,9 @@
 define i8 @scmp_8_8(i8 signext %x, i8 signext %y) nounwind {
 ; CHECK-LABEL: scmp_8_8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    movwgt r2, #1
-; CHECK-NEXT:    sub r0, r2, r0
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    movwgt r0, #1
+; CHECK-NEXT:    mvnlt r0, #0
 ; CHECK-NEXT:    bx lr
   %1 = call i8 @llvm.scmp(i8 %x, i8 %y)
   ret i8 %1
@@ -18,12 +15,9 @@ define i8 @scmp_8_8(i8 signext %x, i8 signext %y) nounwind {
 define i8 @scmp_8_16(i16 signext %x, i16 signext %y) nounwind {
 ; CHECK-LABEL: scmp_8_16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    movwgt r2, #1
-; CHECK-NEXT:    sub r0, r2, r0
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    movwgt r0, #1
+; CHECK-NEXT:    mvnlt r0, #0
 ; CHECK-NEXT:    bx lr
   %1 = call i8 @llvm.scmp(i16 %x, i16 %y)
   ret i8 %1
@@ -32,12 +26,9 @@ define i8 @scmp_8_16(i16 signext %x, i16 signext %y) nounwind {
 define i8 @scmp_8_32(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: scmp_8_32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    movwgt r2, #1
-; CHECK-NEXT:    sub r0, r2, r0
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    movwgt r0, #1
+; CHECK-NEXT:    mvnlt r0, #0
 ; CHECK-NEXT:    bx lr
   %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
   ret i8 %1
@@ -92,17 +83,26 @@ define i8 @scmp_8_128(i128 %x, i128 %y) nounwind {
 define i32 @scmp_32_32(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: scmp_32_32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlt r0, #1
-; CHECK-NEXT:    movwgt r2, #1
-; CHECK-NEXT:    sub r0, r2, r0
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    movwgt r0, #1
+; CHECK-NEXT:    mvnlt r0, #0
 ; CHECK-NEXT:    bx lr
   %1 = call i32 @llvm.scmp(i32 %x, i32 %y)
   ret i32 %1
 }
 
+define i32 @scmp_neg(i32 %x, i32 %y) nounwind {
+; CHECK-LABEL: scmp_neg:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    adds r0, r0, r1
+; CHECK-NEXT:    movwgt r0, #1
+; CHECK-NEXT:    mvnlt r0, #0
+; CHECK-NEXT:    bx lr
+  %yy = sub nsw i32 0, %y
+  %1 = call i32 @llvm.scmp(i32 %x, i32 %yy)
+  ret i32 %1
+}
+
 define i32 @scmp_32_64(i64 %x, i64 %y) nounwind {
 ; CHECK-LABEL: scmp_32_64:
 ; CHECK:       @ %bb.0:
diff --git a/llvm/test/CodeGen/ARM/stack_frame_offset.mir b/llvm/test/CodeGen/ARM/stack_frame_offset.mir
index e387e079aa20d..423928d8c6b6f 100644
--- a/llvm/test/CodeGen/ARM/stack_frame_offset.mir
+++ b/llvm/test/CodeGen/ARM/stack_frame_offset.mir
@@ -51,8 +51,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 4, alignment: 4,
       stack-id: default, callee-saved-register: '', callee-saved-restored: true,
@@ -103,8 +103,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 4, alignment: 4,
       stack-id: default, callee-saved-register: '', callee-saved-restored: true,
@@ -155,8 +155,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 4, alignment: 4,
       stack-id: default, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/ARM/ucmp.ll b/llvm/test/CodeGen/ARM/ucmp.ll
index ad4af534ee8fe..bb0201454d1ea 100644
--- a/llvm/test/CodeGen/ARM/ucmp.ll
+++ b/llvm/test/CodeGen/ARM/ucmp.ll
@@ -4,12 +4,9 @@
 define i8 @ucmp_8_8(i8 zeroext %x, i8 zeroext %y) nounwind {
 ; CHECK-LABEL: ucmp_8_8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlo r0, #1
-; CHECK-NEXT:    movwhi r2, #1
-; CHECK-NEXT:    sub r0, r2, r0
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    movwhi r0, #1
+; CHECK-NEXT:    mvnlo r0, #0
 ; CHECK-NEXT:    bx lr
   %1 = call i8 @llvm.ucmp(i8 %x, i8 %y)
   ret i8 %1
@@ -18,12 +15,9 @@ define i8 @ucmp_8_8(i8 zeroext %x, i8 zeroext %y) nounwind {
 define i8 @ucmp_8_16(i16 zeroext %x, i16 zeroext %y) nounwind {
 ; CHECK-LABEL: ucmp_8_16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlo r0, #1
-; CHECK-NEXT:    movwhi r2, #1
-; CHECK-NEXT:    sub r0, r2, r0
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    movwhi r0, #1
+; CHECK-NEXT:    mvnlo r0, #0
 ; CHECK-NEXT:    bx lr
   %1 = call i8 @llvm.ucmp(i16 %x, i16 %y)
   ret i8 %1
@@ -32,12 +26,9 @@ define i8 @ucmp_8_16(i16 zeroext %x, i16 zeroext %y) nounwind {
 define i8 @ucmp_8_32(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: ucmp_8_32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlo r0, #1
-; CHECK-NEXT:    movwhi r2, #1
-; CHECK-NEXT:    sub r0, r2, r0
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    movwhi r0, #1
+; CHECK-NEXT:    mvnlo r0, #0
 ; CHECK-NEXT:    bx lr
   %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
   ret i8 %1
@@ -92,12 +83,9 @@ define i8 @ucmp_8_128(i128 %x, i128 %y) nounwind {
 define i32 @ucmp_32_32(i32 %x, i32 %y) nounwind {
 ; CHECK-LABEL: ucmp_32_32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov r0, #0
-; CHECK-NEXT:    mov r2, #0
-; CHECK-NEXT:    movwlo r0, #1
-; CHECK-NEXT:    movwhi r2, #1
-; CHECK-NEXT:    sub r0, r2, r0
+; CHECK-NEXT:    subs r0, r0, r1
+; CHECK-NEXT:    movwhi r0, #1
+; CHECK-NEXT:    mvnlo r0, #0
 ; CHECK-NEXT:    bx lr
   %1 = call i32 @llvm.ucmp(i32 %x, i32 %y)
   ret i32 %1
diff --git a/llvm/test/CodeGen/ARM/vector-llrint.ll b/llvm/test/CodeGen/ARM/vector-llrint.ll
new file mode 100644
index 0000000000000..5f4e39125da12
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/vector-llrint.ll
@@ -0,0 +1,2394 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; RUN: llc %s -o - -mtriple=armv7-unknown-none-eabihf                | FileCheck %s --check-prefixes=LE
+; RUN: llc %s -o - -mtriple=armv7-unknown-none-eabihf -mattr=+neon   | FileCheck %s --check-prefixes=LE
+; RUN: llc %s -o - -mtriple=armebv7-unknown-none-eabihf              | FileCheck %s --check-prefixes=BE
+; RUN: llc %s -o - -mtriple=armebv7-unknown-none-eabihf -mattr=+neon | FileCheck %s --check-prefixes=BE
+
+define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
+; LE-LABEL: llrint_v1i64_v1f16:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r11, lr}
+; LE-NEXT:    push {r11, lr}
+; LE-NEXT:    vmov r0, s0
+; LE-NEXT:    bl __aeabi_f2h
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d0[0], r0
+; LE-NEXT:    vmov.32 d0[1], r1
+; LE-NEXT:    pop {r11, pc}
+;
+; BE-LABEL: llrint_v1i64_v1f16:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r11, lr}
+; BE-NEXT:    push {r11, lr}
+; BE-NEXT:    vmov r0, s0
+; BE-NEXT:    bl __aeabi_f2h
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d0, d16
+; BE-NEXT:    pop {r11, pc}
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half>)
+
+define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
+; LE-LABEL: llrint_v1i64_v2f16:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r11, lr}
+; LE-NEXT:    push {r4, r5, r11, lr}
+; LE-NEXT:    .vsave {d8, d9}
+; LE-NEXT:    vpush {d8, d9}
+; LE-NEXT:    vmov r0, s1
+; LE-NEXT:    vmov.f32 s16, s0
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    mov r4, r0
+; LE-NEXT:    vmov r0, s16
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    vmov.32 d9[0], r4
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    vmov.32 d9[1], r5
+; LE-NEXT:    vmov.32 d8[1], r1
+; LE-NEXT:    vorr q0, q4, q4
+; LE-NEXT:    vpop {d8, d9}
+; LE-NEXT:    pop {r4, r5, r11, pc}
+;
+; BE-LABEL: llrint_v1i64_v2f16:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r11, lr}
+; BE-NEXT:    push {r4, r5, r11, lr}
+; BE-NEXT:    .vsave {d8}
+; BE-NEXT:    vpush {d8}
+; BE-NEXT:    vmov r0, s1
+; BE-NEXT:    vmov.f32 s16, s0
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    mov r4, r0
+; BE-NEXT:    vmov r0, s16
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    vmov.32 d8[0], r4
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov.32 d8[1], r5
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d1, d8
+; BE-NEXT:    vrev64.32 d0, d16
+; BE-NEXT:    vpop {d8}
+; BE-NEXT:    pop {r4, r5, r11, pc}
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half>)
+
+define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
+; LE-LABEL: llrint_v4i64_v4f16:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r11, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; LE-NEXT:    .vsave {d12, d13}
+; LE-NEXT:    vpush {d12, d13}
+; LE-NEXT:    .vsave {d8, d9, d10}
+; LE-NEXT:    vpush {d8, d9, d10}
+; LE-NEXT:    vmov r0, s1
+; LE-NEXT:    vmov.f32 s16, s3
+; LE-NEXT:    vmov.f32 s20, s2
+; LE-NEXT:    vmov.f32 s18, s0
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    mov r5, r0
+; LE-NEXT:    vmov r0, s18
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    mov r7, r0
+; LE-NEXT:    vmov r0, s16
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov s0, r7
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    vmov r0, s20
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    vmov.32 d13[0], r5
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    vmov.32 d13[1], r4
+; LE-NEXT:    vmov.32 d9[1], r6
+; LE-NEXT:    vmov.32 d12[1], r7
+; LE-NEXT:    vmov.32 d8[1], r1
+; LE-NEXT:    vorr q0, q6, q6
+; LE-NEXT:    vorr q1, q4, q4
+; LE-NEXT:    vpop {d8, d9, d10}
+; LE-NEXT:    vpop {d12, d13}
+; LE-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+;
+; BE-LABEL: llrint_v4i64_v4f16:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r11, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r11, lr}
+; BE-NEXT:    .vsave {d8, d9, d10}
+; BE-NEXT:    vpush {d8, d9, d10}
+; BE-NEXT:    vmov r0, s1
+; BE-NEXT:    vmov.f32 s16, s3
+; BE-NEXT:    vmov.f32 s18, s2
+; BE-NEXT:    vmov.f32 s20, s0
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    mov r5, r0
+; BE-NEXT:    vmov r0, s20
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r7, r0
+; BE-NEXT:    vmov r0, s16
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov s0, r7
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d8[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    vmov r0, s18
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    vmov.32 d9[0], r5
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov.32 d9[1], r4
+; BE-NEXT:    vmov.32 d8[1], r6
+; BE-NEXT:    vmov.32 d10[1], r7
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d1, d9
+; BE-NEXT:    vrev64.32 d3, d8
+; BE-NEXT:    vrev64.32 d0, d10
+; BE-NEXT:    vrev64.32 d2, d16
+; BE-NEXT:    vpop {d8, d9, d10}
+; BE-NEXT:    pop {r4, r5, r6, r7, r11, pc}
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half>)
+
+define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
+; LE-LABEL: llrint_v8i64_v8f16:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    .pad #4
+; LE-NEXT:    sub sp, sp, #4
+; LE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    .pad #8
+; LE-NEXT:    sub sp, sp, #8
+; LE-NEXT:    vmov r0, s1
+; LE-NEXT:    vstr s6, [sp, #4] @ 4-byte Spill
+; LE-NEXT:    vmov.f32 s16, s7
+; LE-NEXT:    vmov.f32 s18, s5
+; LE-NEXT:    vmov.f32 s20, s4
+; LE-NEXT:    vmov.f32 s22, s3
+; LE-NEXT:    vmov.f32 s24, s2
+; LE-NEXT:    vmov.f32 s26, s0
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    mov r9, r0
+; LE-NEXT:    vmov r0, s26
+; LE-NEXT:    str r1, [sp] @ 4-byte Spill
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    mov r10, r0
+; LE-NEXT:    vmov r0, s22
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    mov r5, r0
+; LE-NEXT:    vmov r0, s24
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    mov r7, r0
+; LE-NEXT:    vmov r0, s18
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    mov r6, r0
+; LE-NEXT:    vmov r0, s20
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    mov r4, r0
+; LE-NEXT:    vmov r0, s16
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov s0, r4
+; LE-NEXT:    mov r11, r1
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov s0, r6
+; LE-NEXT:    mov r8, r1
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov s0, r7
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov s0, r5
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov s0, r10
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vldr s0, [sp, #4] @ 4-byte Reload
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    vmov r0, s0
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    vmov.32 d9[0], r9
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; LE-NEXT:    vmov.32 d15[1], r5
+; LE-NEXT:    vmov.32 d9[1], r0
+; LE-NEXT:    vmov.32 d13[1], r6
+; LE-NEXT:    vmov.32 d11[1], r11
+; LE-NEXT:    vmov.32 d8[1], r4
+; LE-NEXT:    vmov.32 d14[1], r7
+; LE-NEXT:    vorr q0, q4, q4
+; LE-NEXT:    vmov.32 d12[1], r8
+; LE-NEXT:    vorr q1, q7, q7
+; LE-NEXT:    vmov.32 d10[1], r1
+; LE-NEXT:    vorr q2, q6, q6
+; LE-NEXT:    vorr q3, q5, q5
+; LE-NEXT:    add sp, sp, #8
+; LE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    add sp, sp, #4
+; LE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-LABEL: llrint_v8i64_v8f16:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    .pad #4
+; BE-NEXT:    sub sp, sp, #4
+; BE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14}
+; BE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14}
+; BE-NEXT:    .pad #8
+; BE-NEXT:    sub sp, sp, #8
+; BE-NEXT:    vmov r0, s1
+; BE-NEXT:    vmov.f32 s18, s7
+; BE-NEXT:    vmov.f32 s16, s6
+; BE-NEXT:    vmov.f32 s20, s5
+; BE-NEXT:    vmov.f32 s22, s4
+; BE-NEXT:    vmov.f32 s24, s3
+; BE-NEXT:    vmov.f32 s26, s2
+; BE-NEXT:    vmov.f32 s28, s0
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    mov r9, r0
+; BE-NEXT:    vmov r0, s28
+; BE-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r10, r0
+; BE-NEXT:    vmov r0, s24
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r5, r0
+; BE-NEXT:    vmov r0, s26
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r7, r0
+; BE-NEXT:    vmov r0, s20
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r6, r0
+; BE-NEXT:    vmov r0, s22
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r4, r0
+; BE-NEXT:    vmov r0, s18
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov s0, r4
+; BE-NEXT:    mov r11, r1
+; BE-NEXT:    vmov.32 d9[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov s0, r6
+; BE-NEXT:    mov r8, r1
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov s0, r7
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov s0, r5
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    vmov.32 d12[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov s0, r10
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    vmov r0, s16
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    vmov.32 d8[0], r9
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; BE-NEXT:    vmov.32 d13[1], r5
+; BE-NEXT:    vmov.32 d8[1], r0
+; BE-NEXT:    vmov.32 d11[1], r6
+; BE-NEXT:    vmov.32 d9[1], r11
+; BE-NEXT:    vmov.32 d14[1], r4
+; BE-NEXT:    vmov.32 d12[1], r7
+; BE-NEXT:    vmov.32 d10[1], r8
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d1, d8
+; BE-NEXT:    vrev64.32 d3, d13
+; BE-NEXT:    vrev64.32 d5, d11
+; BE-NEXT:    vrev64.32 d7, d9
+; BE-NEXT:    vrev64.32 d0, d14
+; BE-NEXT:    vrev64.32 d2, d12
+; BE-NEXT:    vrev64.32 d4, d10
+; BE-NEXT:    vrev64.32 d6, d16
+; BE-NEXT:    add sp, sp, #8
+; BE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14}
+; BE-NEXT:    add sp, sp, #4
+; BE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half>)
+
+define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
+; LE-LABEL: llrint_v16i64_v16f16:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    .pad #4
+; LE-NEXT:    sub sp, sp, #4
+; LE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    .pad #120
+; LE-NEXT:    sub sp, sp, #120
+; LE-NEXT:    mov r11, r0
+; LE-NEXT:    vmov r0, s7
+; LE-NEXT:    vstr s15, [sp, #24] @ 4-byte Spill
+; LE-NEXT:    vmov.f32 s23, s13
+; LE-NEXT:    vstr s14, [sp, #100] @ 4-byte Spill
+; LE-NEXT:    vmov.f32 s25, s12
+; LE-NEXT:    vmov.f32 s27, s11
+; LE-NEXT:    vstr s10, [sp, #104] @ 4-byte Spill
+; LE-NEXT:    vstr s9, [sp, #108] @ 4-byte Spill
+; LE-NEXT:    vmov.f32 s24, s8
+; LE-NEXT:    vmov.f32 s19, s6
+; LE-NEXT:    vmov.f32 s29, s5
+; LE-NEXT:    vmov.f32 s17, s4
+; LE-NEXT:    vmov.f32 s16, s3
+; LE-NEXT:    vmov.f32 s21, s2
+; LE-NEXT:    vmov.f32 s26, s1
+; LE-NEXT:    vmov.f32 s18, s0
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    mov r7, r0
+; LE-NEXT:    vmov r0, s25
+; LE-NEXT:    str r1, [sp, #56] @ 4-byte Spill
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    mov r5, r0
+; LE-NEXT:    vmov r0, s27
+; LE-NEXT:    str r1, [sp, #116] @ 4-byte Spill
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    mov r6, r0
+; LE-NEXT:    vmov r0, s29
+; LE-NEXT:    str r1, [sp, #112] @ 4-byte Spill
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    vmov r0, s23
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    add lr, sp, #80
+; LE-NEXT:    vmov.32 d17[0], r6
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    mov r6, r0
+; LE-NEXT:    vmov r0, s17
+; LE-NEXT:    vmov r8, s21
+; LE-NEXT:    str r1, [sp, #76] @ 4-byte Spill
+; LE-NEXT:    vmov r10, s19
+; LE-NEXT:    vmov.32 d10[0], r5
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    add lr, sp, #40
+; LE-NEXT:    vmov.32 d11[0], r6
+; LE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    mov r0, r10
+; LE-NEXT:    mov r9, r1
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    vmov.32 d11[0], r7
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    mov r0, r8
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    mov r6, r0
+; LE-NEXT:    ldr r0, [sp, #56] @ 4-byte Reload
+; LE-NEXT:    vmov.32 d11[1], r0
+; LE-NEXT:    vmov r0, s18
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    mov r5, r0
+; LE-NEXT:    vmov r0, s16
+; LE-NEXT:    vmov.32 d10[1], r7
+; LE-NEXT:    add lr, sp, #56
+; LE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov s0, r0
+; LE-NEXT:    vmov.32 d15[1], r4
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    vmov r0, s26
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    vmov r8, s24
+; LE-NEXT:    vmov.32 d14[1], r9
+; LE-NEXT:    mov r10, r1
+; LE-NEXT:    vmov s24, r5
+; LE-NEXT:    vldr s0, [sp, #24] @ 4-byte Reload
+; LE-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; LE-NEXT:    vmov r7, s0
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov.f32 s0, s24
+; LE-NEXT:    vmov s22, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s22
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    vmov s24, r6
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    mov r0, r7
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov.f32 s0, s24
+; LE-NEXT:    vmov s22, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s22
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    mov r9, r1
+; LE-NEXT:    vmov.32 d15[1], r6
+; LE-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    mov r0, r8
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vldr s0, [sp, #100] @ 4-byte Reload
+; LE-NEXT:    mov r7, r0
+; LE-NEXT:    vmov.32 d14[1], r5
+; LE-NEXT:    vmov r0, s0
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vldr s0, [sp, #104] @ 4-byte Reload
+; LE-NEXT:    vmov s20, r0
+; LE-NEXT:    vmov.32 d13[1], r6
+; LE-NEXT:    vmov r4, s0
+; LE-NEXT:    vldr s0, [sp, #108] @ 4-byte Reload
+; LE-NEXT:    vmov r0, s0
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov.f32 s0, s20
+; LE-NEXT:    vmov s16, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s16
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    vmov s18, r7
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    mov r0, r4
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    bl __aeabi_h2f
+; LE-NEXT:    vmov.f32 s0, s18
+; LE-NEXT:    vmov s16, r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s16
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d11[1], r6
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    add lr, sp, #80
+; LE-NEXT:    vmov.32 d10[1], r4
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #40
+; LE-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    vmov.32 d16[0], r0
+; LE-NEXT:    ldr r0, [sp, #76] @ 4-byte Reload
+; LE-NEXT:    vldmia lr, {d20, d21} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    vmov.32 d19[1], r0
+; LE-NEXT:    ldr r0, [sp, #116] @ 4-byte Reload
+; LE-NEXT:    vmov.32 d21[1], r10
+; LE-NEXT:    vmov.32 d18[1], r0
+; LE-NEXT:    ldr r0, [sp, #112] @ 4-byte Reload
+; LE-NEXT:    vmov.32 d12[1], r5
+; LE-NEXT:    vmov.32 d17[1], r0
+; LE-NEXT:    add r0, r11, #64
+; LE-NEXT:    vmov.32 d16[1], r1
+; LE-NEXT:    vst1.64 {d10, d11}, [r0:128]!
+; LE-NEXT:    vst1.64 {d16, d17}, [r0:128]!
+; LE-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; LE-NEXT:    vmov.32 d20[1], r9
+; LE-NEXT:    vst1.64 {d12, d13}, [r0:128]
+; LE-NEXT:    vst1.64 {d14, d15}, [r11:128]!
+; LE-NEXT:    vst1.64 {d20, d21}, [r11:128]!
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #56
+; LE-NEXT:    vst1.64 {d16, d17}, [r11:128]!
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    vst1.64 {d16, d17}, [r11:128]
+; LE-NEXT:    add sp, sp, #120
+; LE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    add sp, sp, #4
+; LE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-LABEL: llrint_v16i64_v16f16:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    .pad #4
+; BE-NEXT:    sub sp, sp, #4
+; BE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    .pad #112
+; BE-NEXT:    sub sp, sp, #112
+; BE-NEXT:    mov r11, r0
+; BE-NEXT:    vmov r0, s14
+; BE-NEXT:    vmov.f32 s17, s15
+; BE-NEXT:    vstr s13, [sp, #52] @ 4-byte Spill
+; BE-NEXT:    vmov.f32 s21, s12
+; BE-NEXT:    vstr s10, [sp, #68] @ 4-byte Spill
+; BE-NEXT:    vmov.f32 s23, s11
+; BE-NEXT:    vstr s7, [sp, #72] @ 4-byte Spill
+; BE-NEXT:    vmov.f32 s19, s9
+; BE-NEXT:    vstr s4, [sp, #28] @ 4-byte Spill
+; BE-NEXT:    vmov.f32 s26, s8
+; BE-NEXT:    vmov.f32 s24, s6
+; BE-NEXT:    vmov.f32 s18, s5
+; BE-NEXT:    vmov.f32 s25, s3
+; BE-NEXT:    vmov.f32 s16, s2
+; BE-NEXT:    vmov.f32 s27, s1
+; BE-NEXT:    vmov.f32 s29, s0
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    mov r8, r0
+; BE-NEXT:    vmov r0, s29
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r9, r0
+; BE-NEXT:    vmov r0, s27
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r7, r0
+; BE-NEXT:    vmov r0, s21
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r6, r0
+; BE-NEXT:    vmov r0, s25
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r5, r0
+; BE-NEXT:    vmov r0, s23
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov s0, r5
+; BE-NEXT:    str r1, [sp, #108] @ 4-byte Spill
+; BE-NEXT:    vstr d16, [sp, #96] @ 8-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov s0, r6
+; BE-NEXT:    str r1, [sp, #92] @ 4-byte Spill
+; BE-NEXT:    vstr d16, [sp, #80] @ 8-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov s0, r7
+; BE-NEXT:    str r1, [sp, #76] @ 4-byte Spill
+; BE-NEXT:    vstr d16, [sp, #56] @ 8-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov s0, r9
+; BE-NEXT:    mov r10, r1
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d15[0], r0
+; BE-NEXT:    vmov r0, s17
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    vmov.32 d10[0], r8
+; BE-NEXT:    vmov r6, s19
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    mov r0, r6
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r6, r0
+; BE-NEXT:    vmov r0, s18
+; BE-NEXT:    vmov.32 d10[1], r4
+; BE-NEXT:    vstr d10, [sp, #40] @ 8-byte Spill
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    mov r4, r0
+; BE-NEXT:    vmov r0, s16
+; BE-NEXT:    vmov.32 d11[1], r7
+; BE-NEXT:    vstr d11, [sp, #32] @ 8-byte Spill
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov.32 d15[1], r5
+; BE-NEXT:    vmov s0, r0
+; BE-NEXT:    vstr d15, [sp, #16] @ 8-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vldr s0, [sp, #28] @ 4-byte Reload
+; BE-NEXT:    vmov r5, s26
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov s26, r4
+; BE-NEXT:    vmov r0, s0
+; BE-NEXT:    mov r8, r1
+; BE-NEXT:    vmov.32 d14[1], r10
+; BE-NEXT:    vmov r4, s24
+; BE-NEXT:    vstr d16, [sp] @ 8-byte Spill
+; BE-NEXT:    vstr d14, [sp, #8] @ 8-byte Spill
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov.f32 s0, s26
+; BE-NEXT:    vmov s22, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s22
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    vmov s24, r6
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    mov r0, r4
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov.f32 s0, s24
+; BE-NEXT:    vmov s22, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s22
+; BE-NEXT:    mov r9, r1
+; BE-NEXT:    vmov.32 d12[0], r0
+; BE-NEXT:    vmov.32 d14[1], r6
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    mov r0, r5
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vldr s0, [sp, #52] @ 4-byte Reload
+; BE-NEXT:    mov r4, r0
+; BE-NEXT:    vmov.32 d13[1], r7
+; BE-NEXT:    vmov r0, s0
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vldr s0, [sp, #68] @ 4-byte Reload
+; BE-NEXT:    vmov s20, r0
+; BE-NEXT:    vmov.32 d11[1], r6
+; BE-NEXT:    vmov r7, s0
+; BE-NEXT:    vldr s0, [sp, #72] @ 4-byte Reload
+; BE-NEXT:    vmov r0, s0
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov.f32 s0, s20
+; BE-NEXT:    vmov s16, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    vmov s18, r4
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d15[0], r0
+; BE-NEXT:    mov r0, r7
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    bl __aeabi_h2f
+; BE-NEXT:    vmov.f32 s0, s18
+; BE-NEXT:    vmov s16, r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d9[0], r0
+; BE-NEXT:    vmov.32 d15[1], r4
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d24[0], r0
+; BE-NEXT:    ldr r0, [sp, #76] @ 4-byte Reload
+; BE-NEXT:    vldr d23, [sp, #56] @ 8-byte Reload
+; BE-NEXT:    vldr d20, [sp, #8] @ 8-byte Reload
+; BE-NEXT:    vmov.32 d23[1], r0
+; BE-NEXT:    ldr r0, [sp, #92] @ 4-byte Reload
+; BE-NEXT:    vldr d22, [sp, #80] @ 8-byte Reload
+; BE-NEXT:    vldr d26, [sp, #16] @ 8-byte Reload
+; BE-NEXT:    vrev64.32 d21, d20
+; BE-NEXT:    vmov.32 d22[1], r0
+; BE-NEXT:    ldr r0, [sp, #108] @ 4-byte Reload
+; BE-NEXT:    vldr d30, [sp] @ 8-byte Reload
+; BE-NEXT:    vldr d25, [sp, #96] @ 8-byte Reload
+; BE-NEXT:    vrev64.32 d20, d26
+; BE-NEXT:    vldr d26, [sp, #32] @ 8-byte Reload
+; BE-NEXT:    vmov.32 d10[1], r5
+; BE-NEXT:    vmov.32 d12[1], r9
+; BE-NEXT:    vldr d28, [sp, #40] @ 8-byte Reload
+; BE-NEXT:    vrev64.32 d27, d26
+; BE-NEXT:    vmov.32 d25[1], r0
+; BE-NEXT:    add r0, r11, #64
+; BE-NEXT:    vmov.32 d30[1], r8
+; BE-NEXT:    vmov.32 d9[1], r6
+; BE-NEXT:    vrev64.32 d26, d28
+; BE-NEXT:    vrev64.32 d29, d10
+; BE-NEXT:    vmov.32 d24[1], r1
+; BE-NEXT:    vrev64.32 d1, d12
+; BE-NEXT:    vrev64.32 d28, d23
+; BE-NEXT:    vrev64.32 d23, d22
+; BE-NEXT:    vrev64.32 d22, d30
+; BE-NEXT:    vrev64.32 d31, d25
+; BE-NEXT:    vrev64.32 d0, d9
+; BE-NEXT:    vrev64.32 d30, d24
+; BE-NEXT:    vst1.64 {d0, d1}, [r0:128]!
+; BE-NEXT:    vst1.64 {d30, d31}, [r0:128]!
+; BE-NEXT:    vst1.64 {d28, d29}, [r0:128]!
+; BE-NEXT:    vrev64.32 d19, d13
+; BE-NEXT:    vst1.64 {d26, d27}, [r0:128]
+; BE-NEXT:    vst1.64 {d20, d21}, [r11:128]!
+; BE-NEXT:    vrev64.32 d18, d14
+; BE-NEXT:    vst1.64 {d22, d23}, [r11:128]!
+; BE-NEXT:    vrev64.32 d17, d15
+; BE-NEXT:    vrev64.32 d16, d11
+; BE-NEXT:    vst1.64 {d18, d19}, [r11:128]!
+; BE-NEXT:    vst1.64 {d16, d17}, [r11:128]
+; BE-NEXT:    add sp, sp, #112
+; BE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    add sp, sp, #4
+; BE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half>)
+
+define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
+; LE-LABEL: llrint_v1i64_v1f32:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r11, lr}
+; LE-NEXT:    push {r11, lr}
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d0[0], r0
+; LE-NEXT:    vmov.32 d0[1], r1
+; LE-NEXT:    pop {r11, pc}
+;
+; BE-LABEL: llrint_v1i64_v1f32:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r11, lr}
+; BE-NEXT:    push {r11, lr}
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d0, d16
+; BE-NEXT:    pop {r11, pc}
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
+
+define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
+; LE-LABEL: llrint_v2i64_v2f32:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, lr}
+; LE-NEXT:    push {r4, lr}
+; LE-NEXT:    .vsave {d10, d11}
+; LE-NEXT:    vpush {d10, d11}
+; LE-NEXT:    .vsave {d8}
+; LE-NEXT:    vpush {d8}
+; LE-NEXT:    vmov.f64 d8, d0
+; LE-NEXT:    vmov.f32 s0, s17
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s16
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    vmov.32 d11[1], r4
+; LE-NEXT:    vmov.32 d10[1], r1
+; LE-NEXT:    vorr q0, q5, q5
+; LE-NEXT:    vpop {d8}
+; LE-NEXT:    vpop {d10, d11}
+; LE-NEXT:    pop {r4, pc}
+;
+; BE-LABEL: llrint_v2i64_v2f32:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, lr}
+; BE-NEXT:    push {r4, lr}
+; BE-NEXT:    .vsave {d10, d11}
+; BE-NEXT:    vpush {d10, d11}
+; BE-NEXT:    .vsave {d8}
+; BE-NEXT:    vpush {d8}
+; BE-NEXT:    vrev64.32 d8, d0
+; BE-NEXT:    vmov.f32 s0, s17
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    vmov.32 d11[1], r4
+; BE-NEXT:    vmov.32 d10[1], r1
+; BE-NEXT:    vrev64.32 q0, q5
+; BE-NEXT:    vpop {d8}
+; BE-NEXT:    vpop {d10, d11}
+; BE-NEXT:    pop {r4, pc}
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
+
+define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
+; LE-LABEL: llrint_v4i64_v4f32:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, lr}
+; LE-NEXT:    push {r4, r5, r6, lr}
+; LE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; LE-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; LE-NEXT:    vorr q5, q0, q0
+; LE-NEXT:    vmov.f32 s0, s23
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s20
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s21
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s22
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    vmov.32 d13[1], r6
+; LE-NEXT:    vmov.32 d9[1], r4
+; LE-NEXT:    vmov.32 d12[1], r5
+; LE-NEXT:    vmov.32 d8[1], r1
+; LE-NEXT:    vorr q0, q6, q6
+; LE-NEXT:    vorr q1, q4, q4
+; LE-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; LE-NEXT:    pop {r4, r5, r6, pc}
+;
+; BE-LABEL: llrint_v4i64_v4f32:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, lr}
+; BE-NEXT:    push {r4, r5, r6, lr}
+; BE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; BE-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; BE-NEXT:    vrev64.32 d8, d1
+; BE-NEXT:    vrev64.32 d9, d0
+; BE-NEXT:    vmov.f32 s0, s17
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s18
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s19
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vmov.32 d12[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    vmov.32 d13[1], r6
+; BE-NEXT:    vmov.32 d11[1], r4
+; BE-NEXT:    vmov.32 d12[1], r5
+; BE-NEXT:    vmov.32 d10[1], r1
+; BE-NEXT:    vrev64.32 q0, q6
+; BE-NEXT:    vrev64.32 q1, q5
+; BE-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; BE-NEXT:    pop {r4, r5, r6, pc}
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
+
+define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
+; LE-LABEL: llrint_v8i64_v8f32:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    .pad #40
+; LE-NEXT:    sub sp, sp, #40
+; LE-NEXT:    vorr q6, q1, q1
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    vorr q7, q0, q0
+; LE-NEXT:    vstmia lr, {d2, d3} @ 16-byte Spill
+; LE-NEXT:    vmov.f32 s0, s27
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s24
+; LE-NEXT:    mov r8, r1
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s25
+; LE-NEXT:    mov r9, r1
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vorr q6, q7, q7
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    mov r10, r1
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    vmov.f32 s0, s26
+; LE-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s27
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s24
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-NEXT:    vmov.f32 s0, s1
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-NEXT:    vmov.f32 s0, s2
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    vmov.32 d13[1], r6
+; LE-NEXT:    vmov.32 d15[1], r4
+; LE-NEXT:    vmov.32 d11[1], r10
+; LE-NEXT:    vmov.32 d9[1], r8
+; LE-NEXT:    vmov.32 d12[1], r5
+; LE-NEXT:    vmov.32 d14[1], r7
+; LE-NEXT:    vorr q0, q6, q6
+; LE-NEXT:    vmov.32 d10[1], r9
+; LE-NEXT:    vorr q1, q7, q7
+; LE-NEXT:    vmov.32 d8[1], r1
+; LE-NEXT:    vorr q2, q5, q5
+; LE-NEXT:    vorr q3, q4, q4
+; LE-NEXT:    add sp, sp, #40
+; LE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+;
+; BE-LABEL: llrint_v8i64_v8f32:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    .pad #32
+; BE-NEXT:    sub sp, sp, #32
+; BE-NEXT:    vorr q4, q1, q1
+; BE-NEXT:    add lr, sp, #8
+; BE-NEXT:    vorr q5, q0, q0
+; BE-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; BE-NEXT:    vrev64.32 d12, d8
+; BE-NEXT:    vmov.f32 s0, s25
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s24
+; BE-NEXT:    mov r8, r1
+; BE-NEXT:    vmov.32 d15[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vrev64.32 d0, d11
+; BE-NEXT:    mov r9, r1
+; BE-NEXT:    vrev64.32 d8, d9
+; BE-NEXT:    vorr d9, d0, d0
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    vstr d8, [sp, #24] @ 8-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s17
+; BE-NEXT:    mov r10, r1
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    add lr, sp, #8
+; BE-NEXT:    vmov.f32 s0, s19
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    vrev64.32 d8, d16
+; BE-NEXT:    vstr d8, [sp, #8] @ 8-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vldr d0, [sp, #8] @ 8-byte Reload
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vmov.32 d8[0], r0
+; BE-NEXT:    vmov.f32 s0, s1
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vldr d0, [sp, #24] @ 8-byte Reload
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    @ kill: def $s0 killed $s0 killed $d0
+; BE-NEXT:    vmov.32 d9[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d12[0], r0
+; BE-NEXT:    vmov.32 d9[1], r6
+; BE-NEXT:    vmov.32 d11[1], r4
+; BE-NEXT:    vmov.32 d15[1], r8
+; BE-NEXT:    vmov.32 d13[1], r7
+; BE-NEXT:    vmov.32 d8[1], r5
+; BE-NEXT:    vmov.32 d10[1], r10
+; BE-NEXT:    vmov.32 d14[1], r9
+; BE-NEXT:    vmov.32 d12[1], r1
+; BE-NEXT:    vrev64.32 q0, q4
+; BE-NEXT:    vrev64.32 q1, q5
+; BE-NEXT:    vrev64.32 q2, q7
+; BE-NEXT:    vrev64.32 q3, q6
+; BE-NEXT:    add sp, sp, #32
+; BE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
+
+define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
+; LE-LABEL: llrint_v16i64_v16f32:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    .pad #4
+; LE-NEXT:    sub sp, sp, #4
+; LE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    .pad #160
+; LE-NEXT:    sub sp, sp, #160
+; LE-NEXT:    add lr, sp, #112
+; LE-NEXT:    vorr q5, q3, q3
+; LE-NEXT:    vorr q6, q0, q0
+; LE-NEXT:    mov r4, r0
+; LE-NEXT:    vstmia lr, {d4, d5} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #48
+; LE-NEXT:    vorr q7, q1, q1
+; LE-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; LE-NEXT:    vmov.f32 s0, s23
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s24
+; LE-NEXT:    add lr, sp, #144
+; LE-NEXT:    vmov.32 d17[0], r0
+; LE-NEXT:    str r1, [sp, #108] @ 4-byte Spill
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s25
+; LE-NEXT:    str r1, [sp, #84] @ 4-byte Spill
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s28
+; LE-NEXT:    add lr, sp, #128
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    str r1, [sp, #44] @ 4-byte Spill
+; LE-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s29
+; LE-NEXT:    mov r9, r1
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s30
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s31
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    add lr, sp, #112
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    vldmia lr, {d14, d15} @ 16-byte Reload
+; LE-NEXT:    vmov.f32 s0, s29
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s22
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    vmov.32 d17[0], r0
+; LE-NEXT:    mov r11, r1
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    vmov.32 d13[1], r7
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    add lr, sp, #144
+; LE-NEXT:    vmov.f32 s0, s21
+; LE-NEXT:    vmov.32 d12[1], r5
+; LE-NEXT:    str r1, [sp, #40] @ 4-byte Spill
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    vmov.32 d16[0], r0
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #88
+; LE-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s20
+; LE-NEXT:    mov r10, r1
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    vmov.32 d9[1], r6
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s31
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    mov r8, r1
+; LE-NEXT:    vmov.32 d8[1], r9
+; LE-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #64
+; LE-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    add lr, sp, #128
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    ldr r0, [sp, #44] @ 4-byte Reload
+; LE-NEXT:    mov r9, r1
+; LE-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #48
+; LE-NEXT:    vldmia lr, {d12, d13} @ 16-byte Reload
+; LE-NEXT:    vmov.f32 s0, s27
+; LE-NEXT:    vmov.32 d11[1], r0
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.f32 s0, s26
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    ldr r0, [sp, #84] @ 4-byte Reload
+; LE-NEXT:    add lr, sp, #128
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    vmov.32 d10[1], r0
+; LE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    add lr, sp, #144
+; LE-NEXT:    ldr r0, [sp, #108] @ 4-byte Reload
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    vmov.32 d17[1], r0
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #112
+; LE-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; LE-NEXT:    vmov.f32 s0, s20
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    vmov.f32 s0, s22
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    vmov.32 d16[0], r0
+; LE-NEXT:    vmov.32 d17[1], r11
+; LE-NEXT:    vorr q6, q8, q8
+; LE-NEXT:    bl llrintf
+; LE-NEXT:    add lr, sp, #144
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    ldr r0, [sp, #40] @ 4-byte Reload
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #128
+; LE-NEXT:    vmov.32 d9[1], r9
+; LE-NEXT:    vmov.32 d12[1], r6
+; LE-NEXT:    vmov.32 d19[1], r10
+; LE-NEXT:    vmov.32 d8[1], r1
+; LE-NEXT:    vmov.32 d16[1], r0
+; LE-NEXT:    add r0, r4, #64
+; LE-NEXT:    vmov.32 d18[1], r8
+; LE-NEXT:    vst1.64 {d12, d13}, [r0:128]!
+; LE-NEXT:    vst1.64 {d8, d9}, [r0:128]!
+; LE-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; LE-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; LE-NEXT:    vmov.32 d15[1], r7
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #64
+; LE-NEXT:    vmov.32 d14[1], r5
+; LE-NEXT:    vst1.64 {d16, d17}, [r4:128]!
+; LE-NEXT:    vst1.64 {d14, d15}, [r4:128]!
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #88
+; LE-NEXT:    vst1.64 {d16, d17}, [r4:128]!
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    vst1.64 {d16, d17}, [r4:128]
+; LE-NEXT:    add sp, sp, #160
+; LE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    add sp, sp, #4
+; LE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-LABEL: llrint_v16i64_v16f32:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    .pad #4
+; BE-NEXT:    sub sp, sp, #4
+; BE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    .pad #144
+; BE-NEXT:    sub sp, sp, #144
+; BE-NEXT:    vorr q6, q3, q3
+; BE-NEXT:    add lr, sp, #112
+; BE-NEXT:    vorr q7, q0, q0
+; BE-NEXT:    mov r4, r0
+; BE-NEXT:    vstmia lr, {d4, d5} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #96
+; BE-NEXT:    vrev64.32 d8, d13
+; BE-NEXT:    vstmia lr, {d2, d3} @ 16-byte Spill
+; BE-NEXT:    vmov.f32 s0, s17
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    str r1, [sp, #88] @ 4-byte Spill
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vrev64.32 d8, d14
+; BE-NEXT:    add lr, sp, #128
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    str r1, [sp, #92] @ 4-byte Spill
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    vrev64.32 d9, d12
+; BE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-NEXT:    vstr d9, [sp, #64] @ 8-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s19
+; BE-NEXT:    mov r9, r1
+; BE-NEXT:    vmov.32 d12[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s17
+; BE-NEXT:    str r1, [sp, #84] @ 4-byte Spill
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    vrev64.32 d9, d15
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s18
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s19
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vldr d0, [sp, #64] @ 8-byte Reload
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    @ kill: def $s0 killed $s0 killed $d0
+; BE-NEXT:    vmov.32 d15[0], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    add lr, sp, #40
+; BE-NEXT:    str r1, [sp, #60] @ 4-byte Spill
+; BE-NEXT:    vmov.32 d15[1], r7
+; BE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #96
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    vrev64.32 d8, d16
+; BE-NEXT:    vmov.f32 s0, s17
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    vmov.32 d14[1], r5
+; BE-NEXT:    add lr, sp, #64
+; BE-NEXT:    mov r10, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    add lr, sp, #24
+; BE-NEXT:    mov r11, r1
+; BE-NEXT:    vmov.32 d13[1], r6
+; BE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #96
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    vrev64.32 d8, d17
+; BE-NEXT:    vmov.f32 s0, s17
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    vmov.32 d12[1], r9
+; BE-NEXT:    add lr, sp, #96
+; BE-NEXT:    mov r8, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    add lr, sp, #8
+; BE-NEXT:    ldr r0, [sp, #88] @ 4-byte Reload
+; BE-NEXT:    mov r9, r1
+; BE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #112
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #128
+; BE-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; BE-NEXT:    vrev64.32 d8, d16
+; BE-NEXT:    vmov.32 d11[1], r0
+; BE-NEXT:    vmov.f32 s0, s17
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    vmov.32 d15[0], r0
+; BE-NEXT:    ldr r0, [sp, #92] @ 4-byte Reload
+; BE-NEXT:    add lr, sp, #128
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    vmov.32 d10[1], r0
+; BE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    add lr, sp, #112
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    ldr r0, [sp, #84] @ 4-byte Reload
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #40
+; BE-NEXT:    vrev64.32 d8, d17
+; BE-NEXT:    vldmia lr, {d12, d13} @ 16-byte Reload
+; BE-NEXT:    vmov.f32 s0, s17
+; BE-NEXT:    vmov.32 d13[1], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    vmov.f32 s0, s16
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    ldr r0, [sp, #60] @ 4-byte Reload
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d12[1], r0
+; BE-NEXT:    bl llrintf
+; BE-NEXT:    add lr, sp, #24
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    add r0, r4, #64
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #8
+; BE-NEXT:    vmov.32 d17[1], r10
+; BE-NEXT:    vmov.32 d16[1], r11
+; BE-NEXT:    vorr q12, q8, q8
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #128
+; BE-NEXT:    vmov.32 d15[1], r7
+; BE-NEXT:    vmov.32 d11[1], r6
+; BE-NEXT:    vmov.32 d14[1], r5
+; BE-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #96
+; BE-NEXT:    vmov.32 d10[1], r1
+; BE-NEXT:    vmov.32 d17[1], r8
+; BE-NEXT:    vldmia lr, {d20, d21} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #64
+; BE-NEXT:    vmov.32 d16[1], r9
+; BE-NEXT:    vrev64.32 q14, q7
+; BE-NEXT:    vorr q13, q8, q8
+; BE-NEXT:    vrev64.32 q15, q5
+; BE-NEXT:    vldmia lr, {d22, d23} @ 16-byte Reload
+; BE-NEXT:    vrev64.32 q8, q6
+; BE-NEXT:    vst1.64 {d28, d29}, [r0:128]!
+; BE-NEXT:    vst1.64 {d30, d31}, [r0:128]!
+; BE-NEXT:    vrev64.32 q9, q9
+; BE-NEXT:    vrev64.32 q10, q10
+; BE-NEXT:    vst1.64 {d16, d17}, [r0:128]!
+; BE-NEXT:    vrev64.32 q11, q11
+; BE-NEXT:    vrev64.32 q12, q12
+; BE-NEXT:    vst1.64 {d18, d19}, [r0:128]
+; BE-NEXT:    vst1.64 {d20, d21}, [r4:128]!
+; BE-NEXT:    vst1.64 {d22, d23}, [r4:128]!
+; BE-NEXT:    vrev64.32 q13, q13
+; BE-NEXT:    vst1.64 {d24, d25}, [r4:128]!
+; BE-NEXT:    vst1.64 {d26, d27}, [r4:128]
+; BE-NEXT:    add sp, sp, #144
+; BE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    add sp, sp, #4
+; BE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
+
+define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
+; LE-LABEL: llrint_v1i64_v1f64:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r11, lr}
+; LE-NEXT:    push {r11, lr}
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vmov.32 d0[0], r0
+; LE-NEXT:    vmov.32 d0[1], r1
+; LE-NEXT:    pop {r11, pc}
+;
+; BE-LABEL: llrint_v1i64_v1f64:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r11, lr}
+; BE-NEXT:    push {r11, lr}
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d0, d16
+; BE-NEXT:    pop {r11, pc}
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
+
+define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
+; LE-LABEL: llrint_v2i64_v2f64:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, lr}
+; LE-NEXT:    push {r4, lr}
+; LE-NEXT:    .vsave {d8, d9, d10, d11}
+; LE-NEXT:    vpush {d8, d9, d10, d11}
+; LE-NEXT:    vorr q4, q0, q0
+; LE-NEXT:    vorr d0, d9, d9
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d8, d8
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    vmov.32 d11[1], r4
+; LE-NEXT:    vmov.32 d10[1], r1
+; LE-NEXT:    vorr q0, q5, q5
+; LE-NEXT:    vpop {d8, d9, d10, d11}
+; LE-NEXT:    pop {r4, pc}
+;
+; BE-LABEL: llrint_v2i64_v2f64:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, lr}
+; BE-NEXT:    push {r4, lr}
+; BE-NEXT:    .vsave {d8, d9, d10, d11}
+; BE-NEXT:    vpush {d8, d9, d10, d11}
+; BE-NEXT:    vorr q4, q0, q0
+; BE-NEXT:    vorr d0, d9, d9
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d8, d8
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    vmov.32 d11[1], r4
+; BE-NEXT:    vmov.32 d10[1], r1
+; BE-NEXT:    vrev64.32 q0, q5
+; BE-NEXT:    vpop {d8, d9, d10, d11}
+; BE-NEXT:    pop {r4, pc}
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
+
+define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
+; LE-LABEL: llrint_v4i64_v4f64:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, lr}
+; LE-NEXT:    push {r4, r5, r6, lr}
+; LE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    vorr q5, q1, q1
+; LE-NEXT:    vorr q6, q0, q0
+; LE-NEXT:    vorr d0, d11, d11
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d12, d12
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d13, d13
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d10, d10
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    vmov.32 d15[1], r6
+; LE-NEXT:    vmov.32 d9[1], r4
+; LE-NEXT:    vmov.32 d14[1], r5
+; LE-NEXT:    vmov.32 d8[1], r1
+; LE-NEXT:    vorr q0, q7, q7
+; LE-NEXT:    vorr q1, q4, q4
+; LE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    pop {r4, r5, r6, pc}
+;
+; BE-LABEL: llrint_v4i64_v4f64:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, lr}
+; BE-NEXT:    push {r4, r5, r6, lr}
+; BE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    vorr q4, q1, q1
+; BE-NEXT:    vorr q5, q0, q0
+; BE-NEXT:    vorr d0, d9, d9
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d10, d10
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d11, d11
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d8, d8
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d15[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vmov.32 d12[0], r0
+; BE-NEXT:    vmov.32 d15[1], r6
+; BE-NEXT:    vmov.32 d13[1], r4
+; BE-NEXT:    vmov.32 d14[1], r5
+; BE-NEXT:    vmov.32 d12[1], r1
+; BE-NEXT:    vrev64.32 q0, q7
+; BE-NEXT:    vrev64.32 q1, q6
+; BE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    pop {r4, r5, r6, pc}
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
+
+define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
+; LE-LABEL: llrint_v8i64_v8f64:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    .pad #40
+; LE-NEXT:    sub sp, sp, #40
+; LE-NEXT:    vorr q4, q0, q0
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    vorr d0, d7, d7
+; LE-NEXT:    vstmia lr, {d6, d7} @ 16-byte Spill
+; LE-NEXT:    vorr q7, q2, q2
+; LE-NEXT:    vorr q6, q1, q1
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d14, d14
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    vmov.32 d17[0], r0
+; LE-NEXT:    mov r8, r1
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d15, d15
+; LE-NEXT:    mov r9, r1
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d12, d12
+; LE-NEXT:    mov r10, r1
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d13, d13
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d8, d8
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d9, d9
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    vmov.32 d13[1], r6
+; LE-NEXT:    vldmia lr, {d6, d7} @ 16-byte Reload
+; LE-NEXT:    vmov.32 d15[1], r4
+; LE-NEXT:    vmov.32 d11[1], r10
+; LE-NEXT:    vmov.32 d6[0], r0
+; LE-NEXT:    vmov.32 d12[1], r5
+; LE-NEXT:    vmov.32 d14[1], r7
+; LE-NEXT:    vorr q0, q6, q6
+; LE-NEXT:    vmov.32 d10[1], r9
+; LE-NEXT:    vorr q1, q7, q7
+; LE-NEXT:    vmov.32 d7[1], r8
+; LE-NEXT:    vorr q2, q5, q5
+; LE-NEXT:    vmov.32 d6[1], r1
+; LE-NEXT:    add sp, sp, #40
+; LE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+;
+; BE-LABEL: llrint_v8i64_v8f64:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    .pad #40
+; BE-NEXT:    sub sp, sp, #40
+; BE-NEXT:    vorr q4, q0, q0
+; BE-NEXT:    add lr, sp, #24
+; BE-NEXT:    vorr d0, d7, d7
+; BE-NEXT:    vstmia lr, {d6, d7} @ 16-byte Spill
+; BE-NEXT:    vorr q7, q2, q2
+; BE-NEXT:    vorr q6, q1, q1
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d14, d14
+; BE-NEXT:    add lr, sp, #8
+; BE-NEXT:    vmov.32 d17[0], r0
+; BE-NEXT:    mov r8, r1
+; BE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d15, d15
+; BE-NEXT:    mov r9, r1
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d12, d12
+; BE-NEXT:    mov r10, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d13, d13
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d8, d8
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d15[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d9, d9
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vmov.32 d12[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    add lr, sp, #24
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; BE-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    add lr, sp, #8
+; BE-NEXT:    vmov.32 d13[1], r6
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    vmov.32 d15[1], r4
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov.32 d11[1], r10
+; BE-NEXT:    vmov.32 d17[1], r8
+; BE-NEXT:    vmov.32 d12[1], r5
+; BE-NEXT:    vmov.32 d14[1], r7
+; BE-NEXT:    vmov.32 d10[1], r9
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 q0, q6
+; BE-NEXT:    vrev64.32 q1, q7
+; BE-NEXT:    vrev64.32 q2, q5
+; BE-NEXT:    vrev64.32 q3, q8
+; BE-NEXT:    add sp, sp, #40
+; BE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
+
+define <16 x i64> @llrint_v16f64(<16 x double> %x) {
+; LE-LABEL: llrint_v16f64:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    .pad #4
+; LE-NEXT:    sub sp, sp, #4
+; LE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    .pad #176
+; LE-NEXT:    sub sp, sp, #176
+; LE-NEXT:    add lr, sp, #40
+; LE-NEXT:    str r0, [sp, #140] @ 4-byte Spill
+; LE-NEXT:    add r0, sp, #312
+; LE-NEXT:    vorr q6, q2, q2
+; LE-NEXT:    vstmia lr, {d6, d7} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #96
+; LE-NEXT:    vorr q7, q1, q1
+; LE-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #144
+; LE-NEXT:    vorr d0, d1, d1
+; LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-NEXT:    add r0, sp, #280
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #80
+; LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-NEXT:    add r0, sp, #296
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #120
+; LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-NEXT:    add r0, sp, #328
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #56
+; LE-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d14, d14
+; LE-NEXT:    str r1, [sp, #116] @ 4-byte Spill
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d15, d15
+; LE-NEXT:    str r1, [sp, #76] @ 4-byte Spill
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d12, d12
+; LE-NEXT:    add lr, sp, #160
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    str r1, [sp, #72] @ 4-byte Spill
+; LE-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d13, d13
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    add lr, sp, #40
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-NEXT:    vorr d0, d8, d8
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d9, d9
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    add lr, sp, #96
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    add lr, sp, #40
+; LE-NEXT:    mov r10, r1
+; LE-NEXT:    vmov.32 d13[1], r5
+; LE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #56
+; LE-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-NEXT:    vorr d0, d9, d9
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vorr d0, d8, d8
+; LE-NEXT:    vmov.32 d12[1], r7
+; LE-NEXT:    add lr, sp, #96
+; LE-NEXT:    mov r9, r1
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    mov r11, r1
+; LE-NEXT:    vmov.32 d15[1], r4
+; LE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #144
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    vorr d0, d17, d17
+; LE-NEXT:    bl llrint
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    vmov.32 d14[1], r6
+; LE-NEXT:    mov r8, r1
+; LE-NEXT:    vmov.32 d17[0], r0
+; LE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #56
+; LE-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #80
+; LE-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; LE-NEXT:    vorr d0, d11, d11
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    add lr, sp, #160
+; LE-NEXT:    vorr d0, d10, d10
+; LE-NEXT:    ldr r0, [sp, #72] @ 4-byte Reload
+; LE-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    vmov.32 d9[1], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    ldr r0, [sp, #76] @ 4-byte Reload
+; LE-NEXT:    add lr, sp, #160
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d8[1], r0
+; LE-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-NEXT:    add lr, sp, #120
+; LE-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; LE-NEXT:    vorr d0, d11, d11
+; LE-NEXT:    bl llrint
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    add lr, sp, #40
+; LE-NEXT:    vorr d0, d10, d10
+; LE-NEXT:    ldr r0, [sp, #116] @ 4-byte Reload
+; LE-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d9[1], r0
+; LE-NEXT:    bl llrint
+; LE-NEXT:    add lr, sp, #144
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-NEXT:    vmov.32 d8[1], r10
+; LE-NEXT:    bl llrint
+; LE-NEXT:    add lr, sp, #8
+; LE-NEXT:    vmov.32 d15[1], r6
+; LE-NEXT:    vldmia lr, {d20, d21} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #24
+; LE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #160
+; LE-NEXT:    vmov.32 d20[0], r0
+; LE-NEXT:    vmov.32 d21[1], r8
+; LE-NEXT:    vmov.32 d20[1], r1
+; LE-NEXT:    ldr r1, [sp, #140] @ 4-byte Reload
+; LE-NEXT:    vmov.32 d13[1], r5
+; LE-NEXT:    mov r0, r1
+; LE-NEXT:    vst1.64 {d8, d9}, [r0:128]!
+; LE-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #56
+; LE-NEXT:    vmov.32 d14[1], r4
+; LE-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; LE-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-NEXT:    add lr, sp, #96
+; LE-NEXT:    vmov.32 d12[1], r7
+; LE-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; LE-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-NEXT:    vmov.32 d17[1], r9
+; LE-NEXT:    vst1.64 {d18, d19}, [r0:128]
+; LE-NEXT:    add r0, r1, #64
+; LE-NEXT:    vst1.64 {d14, d15}, [r0:128]!
+; LE-NEXT:    vst1.64 {d12, d13}, [r0:128]!
+; LE-NEXT:    vmov.32 d16[1], r11
+; LE-NEXT:    vst1.64 {d20, d21}, [r0:128]!
+; LE-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; LE-NEXT:    add sp, sp, #176
+; LE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    add sp, sp, #4
+; LE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-LABEL: llrint_v16f64:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    .pad #4
+; BE-NEXT:    sub sp, sp, #4
+; BE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    .pad #168
+; BE-NEXT:    sub sp, sp, #168
+; BE-NEXT:    add lr, sp, #64
+; BE-NEXT:    str r0, [sp, #132] @ 4-byte Spill
+; BE-NEXT:    add r0, sp, #304
+; BE-NEXT:    vorr q4, q3, q3
+; BE-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #48
+; BE-NEXT:    vorr d0, d1, d1
+; BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-NEXT:    add r0, sp, #320
+; BE-NEXT:    vorr q6, q2, q2
+; BE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #88
+; BE-NEXT:    vorr q7, q1, q1
+; BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-NEXT:    add r0, sp, #272
+; BE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #112
+; BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-NEXT:    add r0, sp, #288
+; BE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #24
+; BE-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d14, d14
+; BE-NEXT:    add lr, sp, #136
+; BE-NEXT:    vmov.32 d17[0], r0
+; BE-NEXT:    str r1, [sp, #108] @ 4-byte Spill
+; BE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d15, d15
+; BE-NEXT:    str r1, [sp, #84] @ 4-byte Spill
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d12, d12
+; BE-NEXT:    add lr, sp, #152
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    str r1, [sp, #44] @ 4-byte Spill
+; BE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d13, d13
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d8, d8
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d9, d9
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    vmov.32 d12[0], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    add lr, sp, #64
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; BE-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    add lr, sp, #136
+; BE-NEXT:    mov r9, r1
+; BE-NEXT:    vmov.32 d13[1], r5
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #24
+; BE-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-NEXT:    vorr d0, d9, d9
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d8, d8
+; BE-NEXT:    vmov.32 d12[1], r7
+; BE-NEXT:    add lr, sp, #64
+; BE-NEXT:    mov r10, r1
+; BE-NEXT:    vmov.32 d15[0], r0
+; BE-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    add lr, sp, #8
+; BE-NEXT:    mov r11, r1
+; BE-NEXT:    vmov.32 d11[1], r4
+; BE-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #48
+; BE-NEXT:    vorr q6, q5, q5
+; BE-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-NEXT:    vorr d0, d9, d9
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d8, d8
+; BE-NEXT:    vmov.32 d12[1], r6
+; BE-NEXT:    add lr, sp, #24
+; BE-NEXT:    mov r8, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    add lr, sp, #48
+; BE-NEXT:    ldr r0, [sp, #44] @ 4-byte Reload
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-NEXT:    add lr, sp, #152
+; BE-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #88
+; BE-NEXT:    vldmia lr, {d12, d13} @ 16-byte Reload
+; BE-NEXT:    vorr d0, d13, d13
+; BE-NEXT:    vmov.32 d9[1], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vmov.32 d15[0], r0
+; BE-NEXT:    ldr r0, [sp, #84] @ 4-byte Reload
+; BE-NEXT:    vorr d0, d12, d12
+; BE-NEXT:    add lr, sp, #152
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d8[1], r0
+; BE-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; BE-NEXT:    bl llrint
+; BE-NEXT:    add lr, sp, #136
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    ldr r0, [sp, #108] @ 4-byte Reload
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #112
+; BE-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-NEXT:    vorr d0, d9, d9
+; BE-NEXT:    vmov.32 d11[1], r0
+; BE-NEXT:    bl llrint
+; BE-NEXT:    vorr d0, d8, d8
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    vmov.32 d10[1], r9
+; BE-NEXT:    bl llrint
+; BE-NEXT:    add lr, sp, #8
+; BE-NEXT:    vmov.32 d12[0], r0
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #48
+; BE-NEXT:    vmov.32 d17[1], r10
+; BE-NEXT:    vmov.32 d16[1], r11
+; BE-NEXT:    vorr q12, q8, q8
+; BE-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #152
+; BE-NEXT:    vmov.32 d17[1], r8
+; BE-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #24
+; BE-NEXT:    vmov.32 d13[1], r7
+; BE-NEXT:    vmov.32 d16[1], r6
+; BE-NEXT:    vldmia lr, {d20, d21} @ 16-byte Reload
+; BE-NEXT:    add lr, sp, #64
+; BE-NEXT:    vorr q13, q8, q8
+; BE-NEXT:    vmov.32 d12[1], r1
+; BE-NEXT:    ldr r1, [sp, #132] @ 4-byte Reload
+; BE-NEXT:    vrev64.32 q8, q5
+; BE-NEXT:    mov r0, r1
+; BE-NEXT:    vldmia lr, {d22, d23} @ 16-byte Reload
+; BE-NEXT:    vrev64.32 q9, q9
+; BE-NEXT:    vrev64.32 q10, q10
+; BE-NEXT:    vst1.64 {d16, d17}, [r0:128]!
+; BE-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; BE-NEXT:    vrev64.32 q11, q11
+; BE-NEXT:    vmov.32 d15[1], r4
+; BE-NEXT:    vst1.64 {d20, d21}, [r0:128]!
+; BE-NEXT:    vrev64.32 q15, q6
+; BE-NEXT:    vmov.32 d14[1], r5
+; BE-NEXT:    vrev64.32 q12, q12
+; BE-NEXT:    vst1.64 {d22, d23}, [r0:128]
+; BE-NEXT:    add r0, r1, #64
+; BE-NEXT:    vrev64.32 q13, q13
+; BE-NEXT:    vst1.64 {d30, d31}, [r0:128]!
+; BE-NEXT:    vst1.64 {d24, d25}, [r0:128]!
+; BE-NEXT:    vrev64.32 q14, q7
+; BE-NEXT:    vst1.64 {d26, d27}, [r0:128]!
+; BE-NEXT:    vst1.64 {d28, d29}, [r0:128]
+; BE-NEXT:    add sp, sp, #168
+; BE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-NEXT:    add sp, sp, #4
+; BE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+  %a = call <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.llrint.v16i64.v16f64(<16 x double>)
+
+define <1 x i64> @llrint_v1i64_v1f128(<1 x fp128> %x) {
+; LE-LABEL: llrint_v1i64_v1f128:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r11, lr}
+; LE-NEXT:    push {r11, lr}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    vmov.32 d0[0], r0
+; LE-NEXT:    vmov.32 d0[1], r1
+; LE-NEXT:    pop {r11, pc}
+;
+; BE-LABEL: llrint_v1i64_v1f128:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r11, lr}
+; BE-NEXT:    push {r11, lr}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d0, d16
+; BE-NEXT:    pop {r11, pc}
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f128(<1 x fp128> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f128(<1 x fp128>)
+
+define <2 x i64> @llrint_v2i64_v2f128(<2 x fp128> %x) {
+; LE-LABEL: llrint_v2i64_v2f128:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; LE-NEXT:    .vsave {d8, d9}
+; LE-NEXT:    vpush {d8, d9}
+; LE-NEXT:    mov r8, r3
+; LE-NEXT:    add r3, sp, #40
+; LE-NEXT:    mov r5, r2
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    mov r7, r0
+; LE-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    mov r0, r7
+; LE-NEXT:    mov r1, r6
+; LE-NEXT:    mov r2, r5
+; LE-NEXT:    mov r3, r8
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    vmov.32 d9[1], r4
+; LE-NEXT:    vmov.32 d8[1], r1
+; LE-NEXT:    vorr q0, q4, q4
+; LE-NEXT:    vpop {d8, d9}
+; LE-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+;
+; BE-LABEL: llrint_v2i64_v2f128:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; BE-NEXT:    .vsave {d8}
+; BE-NEXT:    vpush {d8}
+; BE-NEXT:    mov r8, r3
+; BE-NEXT:    add r3, sp, #32
+; BE-NEXT:    mov r5, r2
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    mov r7, r0
+; BE-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d8[0], r0
+; BE-NEXT:    mov r0, r7
+; BE-NEXT:    mov r1, r6
+; BE-NEXT:    mov r2, r5
+; BE-NEXT:    mov r3, r8
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov.32 d8[1], r4
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d1, d8
+; BE-NEXT:    vrev64.32 d0, d16
+; BE-NEXT:    vpop {d8}
+; BE-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f128(<2 x fp128> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f128(<2 x fp128>)
+
+define <4 x i64> @llrint_v4i64_v4f128(<4 x fp128> %x) {
+; LE-LABEL: llrint_v4i64_v4f128:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-NEXT:    .vsave {d8, d9, d10, d11}
+; LE-NEXT:    vpush {d8, d9, d10, d11}
+; LE-NEXT:    mov r5, r3
+; LE-NEXT:    add r3, sp, #96
+; LE-NEXT:    mov r7, r2
+; LE-NEXT:    mov r6, r1
+; LE-NEXT:    mov r4, r0
+; LE-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    mov r9, r1
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    mov r0, r4
+; LE-NEXT:    mov r1, r6
+; LE-NEXT:    mov r2, r7
+; LE-NEXT:    mov r3, r5
+; LE-NEXT:    ldr r8, [sp, #80]
+; LE-NEXT:    ldr r10, [sp, #64]
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    add r3, sp, #68
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    mov r0, r10
+; LE-NEXT:    ldm r3, {r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    add r3, sp, #84
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    mov r0, r8
+; LE-NEXT:    ldm r3, {r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    vmov.32 d11[1], r4
+; LE-NEXT:    vmov.32 d9[1], r9
+; LE-NEXT:    vmov.32 d10[1], r5
+; LE-NEXT:    vmov.32 d8[1], r1
+; LE-NEXT:    vorr q0, q5, q5
+; LE-NEXT:    vorr q1, q4, q4
+; LE-NEXT:    vpop {d8, d9, d10, d11}
+; LE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+;
+; BE-LABEL: llrint_v4i64_v4f128:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-NEXT:    .vsave {d8, d9, d10}
+; BE-NEXT:    vpush {d8, d9, d10}
+; BE-NEXT:    mov r5, r3
+; BE-NEXT:    add r3, sp, #88
+; BE-NEXT:    mov r7, r2
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    mov r4, r0
+; BE-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    mov r9, r1
+; BE-NEXT:    vmov.32 d8[0], r0
+; BE-NEXT:    mov r0, r4
+; BE-NEXT:    mov r1, r6
+; BE-NEXT:    mov r2, r7
+; BE-NEXT:    mov r3, r5
+; BE-NEXT:    ldr r8, [sp, #72]
+; BE-NEXT:    ldr r10, [sp, #56]
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    add r3, sp, #60
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    vmov.32 d9[0], r0
+; BE-NEXT:    mov r0, r10
+; BE-NEXT:    ldm r3, {r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    add r3, sp, #76
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    mov r0, r8
+; BE-NEXT:    ldm r3, {r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    vmov.32 d10[1], r4
+; BE-NEXT:    vmov.32 d8[1], r9
+; BE-NEXT:    vmov.32 d9[1], r5
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d1, d10
+; BE-NEXT:    vrev64.32 d3, d8
+; BE-NEXT:    vrev64.32 d0, d9
+; BE-NEXT:    vrev64.32 d2, d16
+; BE-NEXT:    vpop {d8, d9, d10}
+; BE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f128(<4 x fp128> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f128(<4 x fp128>)
+
+define <8 x i64> @llrint_v8i64_v8f128(<8 x fp128> %x) {
+; LE-LABEL: llrint_v8i64_v8f128:
+; LE:       @ %bb.0:
+; LE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-NEXT:    .pad #4
+; LE-NEXT:    sub sp, sp, #4
+; LE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    .pad #8
+; LE-NEXT:    sub sp, sp, #8
+; LE-NEXT:    mov r11, r3
+; LE-NEXT:    add r3, sp, #208
+; LE-NEXT:    mov r10, r2
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    mov r5, r0
+; LE-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    add r7, sp, #164
+; LE-NEXT:    ldr r6, [sp, #160]
+; LE-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; LE-NEXT:    vmov.32 d9[0], r0
+; LE-NEXT:    ldm r7, {r1, r2, r3, r7}
+; LE-NEXT:    mov r0, r6
+; LE-NEXT:    ldr r8, [sp, #128]
+; LE-NEXT:    ldr r9, [sp, #144]
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    add r3, sp, #180
+; LE-NEXT:    str r1, [sp] @ 4-byte Spill
+; LE-NEXT:    vmov.32 d10[0], r0
+; LE-NEXT:    mov r0, r7
+; LE-NEXT:    ldm r3, {r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    add r3, sp, #132
+; LE-NEXT:    mov r7, r1
+; LE-NEXT:    vmov.32 d11[0], r0
+; LE-NEXT:    mov r0, r8
+; LE-NEXT:    ldm r3, {r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    add r3, sp, #148
+; LE-NEXT:    mov r8, r1
+; LE-NEXT:    vmov.32 d12[0], r0
+; LE-NEXT:    mov r0, r9
+; LE-NEXT:    ldm r3, {r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    mov r9, r1
+; LE-NEXT:    vmov.32 d13[0], r0
+; LE-NEXT:    mov r0, r5
+; LE-NEXT:    mov r1, r4
+; LE-NEXT:    mov r2, r10
+; LE-NEXT:    mov r3, r11
+; LE-NEXT:    ldr r6, [sp, #112]
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    add r3, sp, #116
+; LE-NEXT:    mov r4, r1
+; LE-NEXT:    vmov.32 d14[0], r0
+; LE-NEXT:    mov r0, r6
+; LE-NEXT:    ldm r3, {r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    add r3, sp, #196
+; LE-NEXT:    vmov.32 d15[0], r0
+; LE-NEXT:    ldr r0, [sp, #192]
+; LE-NEXT:    mov r5, r1
+; LE-NEXT:    ldm r3, {r1, r2, r3}
+; LE-NEXT:    bl llrintl
+; LE-NEXT:    vmov.32 d8[0], r0
+; LE-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; LE-NEXT:    vmov.32 d11[1], r7
+; LE-NEXT:    vmov.32 d10[1], r0
+; LE-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; LE-NEXT:    vmov.32 d15[1], r5
+; LE-NEXT:    vorr q2, q5, q5
+; LE-NEXT:    vmov.32 d13[1], r9
+; LE-NEXT:    vmov.32 d9[1], r0
+; LE-NEXT:    vmov.32 d14[1], r4
+; LE-NEXT:    vmov.32 d12[1], r8
+; LE-NEXT:    vorr q0, q7, q7
+; LE-NEXT:    vmov.32 d8[1], r1
+; LE-NEXT:    vorr q1, q6, q6
+; LE-NEXT:    vorr q3, q4, q4
+; LE-NEXT:    add sp, sp, #8
+; LE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-NEXT:    add sp, sp, #4
+; LE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-LABEL: llrint_v8i64_v8f128:
+; BE:       @ %bb.0:
+; BE-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-NEXT:    .pad #4
+; BE-NEXT:    sub sp, sp, #4
+; BE-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14}
+; BE-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14}
+; BE-NEXT:    .pad #16
+; BE-NEXT:    sub sp, sp, #16
+; BE-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; BE-NEXT:    add r3, sp, #208
+; BE-NEXT:    mov r11, r2
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    mov r5, r0
+; BE-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    ldr r7, [sp, #176]
+; BE-NEXT:    add r3, sp, #180
+; BE-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; BE-NEXT:    vmov.32 d8[0], r0
+; BE-NEXT:    ldm r3, {r1, r2, r3}
+; BE-NEXT:    mov r0, r7
+; BE-NEXT:    ldr r6, [sp, #128]
+; BE-NEXT:    ldr r8, [sp, #144]
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    add r3, sp, #132
+; BE-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; BE-NEXT:    vmov.32 d9[0], r0
+; BE-NEXT:    mov r0, r6
+; BE-NEXT:    ldm r3, {r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    add r3, sp, #148
+; BE-NEXT:    mov r6, r1
+; BE-NEXT:    vmov.32 d10[0], r0
+; BE-NEXT:    mov r0, r8
+; BE-NEXT:    ldm r3, {r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    add r3, sp, #160
+; BE-NEXT:    mov r9, r0
+; BE-NEXT:    mov r7, r1
+; BE-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; BE-NEXT:    mov r8, r1
+; BE-NEXT:    vmov.32 d11[0], r0
+; BE-NEXT:    mov r0, r5
+; BE-NEXT:    mov r1, r4
+; BE-NEXT:    mov r2, r11
+; BE-NEXT:    ldr r10, [sp, #112]
+; BE-NEXT:    vmov.32 d12[0], r9
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    add r3, sp, #116
+; BE-NEXT:    mov r4, r1
+; BE-NEXT:    vmov.32 d13[0], r0
+; BE-NEXT:    mov r0, r10
+; BE-NEXT:    ldm r3, {r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    add r3, sp, #196
+; BE-NEXT:    vmov.32 d14[0], r0
+; BE-NEXT:    ldr r0, [sp, #192]
+; BE-NEXT:    mov r5, r1
+; BE-NEXT:    ldm r3, {r1, r2, r3}
+; BE-NEXT:    bl llrintl
+; BE-NEXT:    vmov.32 d16[0], r0
+; BE-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; BE-NEXT:    vmov.32 d14[1], r5
+; BE-NEXT:    vmov.32 d9[1], r0
+; BE-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; BE-NEXT:    vmov.32 d12[1], r7
+; BE-NEXT:    vmov.32 d8[1], r0
+; BE-NEXT:    vmov.32 d13[1], r4
+; BE-NEXT:    vmov.32 d10[1], r6
+; BE-NEXT:    vmov.32 d11[1], r8
+; BE-NEXT:    vmov.32 d16[1], r1
+; BE-NEXT:    vrev64.32 d1, d14
+; BE-NEXT:    vrev64.32 d3, d12
+; BE-NEXT:    vrev64.32 d5, d9
+; BE-NEXT:    vrev64.32 d7, d8
+; BE-NEXT:    vrev64.32 d0, d13
+; BE-NEXT:    vrev64.32 d2, d10
+; BE-NEXT:    vrev64.32 d4, d11
+; BE-NEXT:    vrev64.32 d6, d16
+; BE-NEXT:    add sp, sp, #16
+; BE-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14}
+; BE-NEXT:    add sp, sp, #4
+; BE-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f128(<8 x fp128> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f128(<8 x fp128>)
diff --git a/llvm/test/CodeGen/ARM/vector-lrint.ll b/llvm/test/CodeGen/ARM/vector-lrint.ll
new file mode 100644
index 0000000000000..fe5e3cbcdf771
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/vector-lrint.ll
@@ -0,0 +1,3265 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=armv7-unknown-none-eabihf                | FileCheck %s --check-prefixes=LE-I32
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=armv7-unknown-none-eabihf                | FileCheck %s --check-prefixes=LE-I64
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=armv7-unknown-none-eabihf -mattr=+neon   | FileCheck %s --check-prefixes=LE-I32
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=armv7-unknown-none-eabihf -mattr=+neon   | FileCheck %s --check-prefixes=LE-I64
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=armebv7-unknown-none-eabihf              | FileCheck %s --check-prefixes=BE-I32
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=armebv7-unknown-none-eabihf              | FileCheck %s --check-prefixes=BE-I64
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=armebv7-unknown-none-eabihf -mattr=+neon | FileCheck %s --check-prefixes=BE-I32
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=armebv7-unknown-none-eabihf -mattr=+neon | FileCheck %s --check-prefixes=BE-I64
+
+; FIXME: crash "Do not know how to soft promote this operator's operand!"
+; define <1 x iXLen> @lrint_v1f16(<1 x half> %x) {
+;   %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f16(<1 x half> %x)
+;   ret <1 x iXLen> %a
+; }
+; declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f16(<1 x half>)
+
+; define <2 x iXLen> @lrint_v2f16(<2 x half> %x) {
+;   %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f16(<2 x half> %x)
+;   ret <2 x iXLen> %a
+; }
+; declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f16(<2 x half>)
+
+; define <4 x iXLen> @lrint_v4f16(<4 x half> %x) {
+;   %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f16(<4 x half> %x)
+;   ret <4 x iXLen> %a
+; }
+; declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f16(<4 x half>)
+
+; define <8 x iXLen> @lrint_v8f16(<8 x half> %x) {
+;   %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f16(<8 x half> %x)
+;   ret <8 x iXLen> %a
+; }
+; declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f16(<8 x half>)
+
+; define <16 x iXLen> @lrint_v16f16(<16 x half> %x) {
+;   %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f16(<16 x half> %x)
+;   ret <16 x iXLen> %a
+; }
+; declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f16(<16 x half>)
+
+define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
+; LE-I32-LABEL: lrint_v1f32:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v1f32:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r11, lr}
+; LE-I64-NEXT:    push {r11, lr}
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.32 d0[0], r0
+; LE-I64-NEXT:    vmov.32 d0[1], r1
+; LE-I64-NEXT:    pop {r11, pc}
+;
+; BE-I32-LABEL: lrint_v1f32:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v1f32:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r11, lr}
+; BE-I64-NEXT:    push {r11, lr}
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.32 d16[0], r0
+; BE-I64-NEXT:    vmov.32 d16[1], r1
+; BE-I64-NEXT:    vrev64.32 d0, d16
+; BE-I64-NEXT:    pop {r11, pc}
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float>)
+
+define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
+; LE-I32-LABEL: lrint_v2f32:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    .vsave {d8, d9}
+; LE-I32-NEXT:    vpush {d8, d9}
+; LE-I32-NEXT:    vmov.f64 d8, d0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s17
+; LE-I32-NEXT:    vmov.32 d9[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.32 d9[1], r0
+; LE-I32-NEXT:    vorr d0, d9, d9
+; LE-I32-NEXT:    vpop {d8, d9}
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v2f32:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, lr}
+; LE-I64-NEXT:    push {r4, lr}
+; LE-I64-NEXT:    .vsave {d10, d11}
+; LE-I64-NEXT:    vpush {d10, d11}
+; LE-I64-NEXT:    .vsave {d8}
+; LE-I64-NEXT:    vpush {d8}
+; LE-I64-NEXT:    vmov.f64 d8, d0
+; LE-I64-NEXT:    vmov.f32 s0, s17
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s16
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    vmov.32 d11[1], r4
+; LE-I64-NEXT:    vmov.32 d10[1], r1
+; LE-I64-NEXT:    vorr q0, q5, q5
+; LE-I64-NEXT:    vpop {d8}
+; LE-I64-NEXT:    vpop {d10, d11}
+; LE-I64-NEXT:    pop {r4, pc}
+;
+; BE-I32-LABEL: lrint_v2f32:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    .vsave {d8, d9}
+; BE-I32-NEXT:    vpush {d8, d9}
+; BE-I32-NEXT:    vrev64.32 d8, d0
+; BE-I32-NEXT:    vmov.f32 s0, s16
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s17
+; BE-I32-NEXT:    vmov.32 d9[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.32 d9[1], r0
+; BE-I32-NEXT:    vrev64.32 d0, d9
+; BE-I32-NEXT:    vpop {d8, d9}
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v2f32:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, lr}
+; BE-I64-NEXT:    push {r4, lr}
+; BE-I64-NEXT:    .vsave {d10, d11}
+; BE-I64-NEXT:    vpush {d10, d11}
+; BE-I64-NEXT:    .vsave {d8}
+; BE-I64-NEXT:    vpush {d8}
+; BE-I64-NEXT:    vrev64.32 d8, d0
+; BE-I64-NEXT:    vmov.f32 s0, s17
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s16
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    vmov.32 d11[1], r4
+; BE-I64-NEXT:    vmov.32 d10[1], r1
+; BE-I64-NEXT:    vrev64.32 q0, q5
+; BE-I64-NEXT:    vpop {d8}
+; BE-I64-NEXT:    vpop {d10, d11}
+; BE-I64-NEXT:    pop {r4, pc}
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float>)
+
+define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
+; LE-I32-LABEL: lrint_v4f32:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    .vsave {d8, d9, d10, d11}
+; LE-I32-NEXT:    vpush {d8, d9, d10, d11}
+; LE-I32-NEXT:    vorr q4, q0, q0
+; LE-I32-NEXT:    vmov.f32 s0, s18
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s16
+; LE-I32-NEXT:    vmov.32 d11[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s19
+; LE-I32-NEXT:    vmov.32 d10[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s17
+; LE-I32-NEXT:    vmov.32 d11[1], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.32 d10[1], r0
+; LE-I32-NEXT:    vorr q0, q5, q5
+; LE-I32-NEXT:    vpop {d8, d9, d10, d11}
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v4f32:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, lr}
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; LE-I64-NEXT:    vorr q5, q0, q0
+; LE-I64-NEXT:    vmov.f32 s0, s23
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s20
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s21
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    vmov.32 d12[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s22
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    vmov.32 d13[1], r6
+; LE-I64-NEXT:    vmov.32 d9[1], r4
+; LE-I64-NEXT:    vmov.32 d12[1], r5
+; LE-I64-NEXT:    vmov.32 d8[1], r1
+; LE-I64-NEXT:    vorr q0, q6, q6
+; LE-I64-NEXT:    vorr q1, q4, q4
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; LE-I64-NEXT:    pop {r4, r5, r6, pc}
+;
+; BE-I32-LABEL: lrint_v4f32:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    .vsave {d8, d9, d10, d11}
+; BE-I32-NEXT:    vpush {d8, d9, d10, d11}
+; BE-I32-NEXT:    vrev64.32 q4, q0
+; BE-I32-NEXT:    vmov.f32 s0, s18
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s16
+; BE-I32-NEXT:    vmov.32 d11[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s19
+; BE-I32-NEXT:    vmov.32 d10[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s17
+; BE-I32-NEXT:    vmov.32 d11[1], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.32 d10[1], r0
+; BE-I32-NEXT:    vrev64.32 q0, q5
+; BE-I32-NEXT:    vpop {d8, d9, d10, d11}
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v4f32:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, lr}
+; BE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; BE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; BE-I64-NEXT:    vrev64.32 d8, d1
+; BE-I64-NEXT:    vrev64.32 d9, d0
+; BE-I64-NEXT:    vmov.f32 s0, s17
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s18
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s19
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    vmov.32 d12[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s16
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    vmov.32 d13[1], r6
+; BE-I64-NEXT:    vmov.32 d11[1], r4
+; BE-I64-NEXT:    vmov.32 d12[1], r5
+; BE-I64-NEXT:    vmov.32 d10[1], r1
+; BE-I64-NEXT:    vrev64.32 q0, q6
+; BE-I64-NEXT:    vrev64.32 q1, q5
+; BE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; BE-I64-NEXT:    pop {r4, r5, r6, pc}
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float>)
+
+define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
+; LE-I32-LABEL: lrint_v8f32:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    vorr q5, q1, q1
+; LE-I32-NEXT:    vorr q7, q0, q0
+; LE-I32-NEXT:    vmov.f32 s0, s20
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s22
+; LE-I32-NEXT:    vmov.32 d8[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s30
+; LE-I32-NEXT:    vmov.32 d9[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s28
+; LE-I32-NEXT:    vmov.32 d13[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s31
+; LE-I32-NEXT:    vmov.32 d12[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s29
+; LE-I32-NEXT:    vmov.32 d13[1], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s23
+; LE-I32-NEXT:    vmov.32 d12[1], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s21
+; LE-I32-NEXT:    vmov.32 d9[1], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.32 d8[1], r0
+; LE-I32-NEXT:    vorr q0, q6, q6
+; LE-I32-NEXT:    vorr q1, q4, q4
+; LE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v8f32:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    .pad #40
+; LE-I64-NEXT:    sub sp, sp, #40
+; LE-I64-NEXT:    vorr q6, q1, q1
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    vorr q7, q0, q0
+; LE-I64-NEXT:    vstmia lr, {d2, d3} @ 16-byte Spill
+; LE-I64-NEXT:    vmov.f32 s0, s27
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s24
+; LE-I64-NEXT:    mov r8, r1
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s25
+; LE-I64-NEXT:    mov r9, r1
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vorr q6, q7, q7
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    mov r10, r1
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    vmov.f32 s0, s26
+; LE-I64-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s27
+; LE-I64-NEXT:    mov r7, r1
+; LE-I64-NEXT:    vmov.32 d14[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s24
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d15[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    vmov.32 d12[0], r0
+; LE-I64-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.f32 s0, s1
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.f32 s0, s2
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    vmov.32 d13[1], r6
+; LE-I64-NEXT:    vmov.32 d15[1], r4
+; LE-I64-NEXT:    vmov.32 d11[1], r10
+; LE-I64-NEXT:    vmov.32 d9[1], r8
+; LE-I64-NEXT:    vmov.32 d12[1], r5
+; LE-I64-NEXT:    vmov.32 d14[1], r7
+; LE-I64-NEXT:    vorr q0, q6, q6
+; LE-I64-NEXT:    vmov.32 d10[1], r9
+; LE-I64-NEXT:    vorr q1, q7, q7
+; LE-I64-NEXT:    vmov.32 d8[1], r1
+; LE-I64-NEXT:    vorr q2, q5, q5
+; LE-I64-NEXT:    vorr q3, q4, q4
+; LE-I64-NEXT:    add sp, sp, #40
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+;
+; BE-I32-LABEL: lrint_v8f32:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    vrev64.32 q4, q1
+; BE-I32-NEXT:    vrev64.32 q5, q0
+; BE-I32-NEXT:    vmov.f32 s0, s16
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s20
+; BE-I32-NEXT:    vmov.32 d12[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s18
+; BE-I32-NEXT:    vmov.32 d14[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s22
+; BE-I32-NEXT:    vmov.32 d13[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s19
+; BE-I32-NEXT:    vmov.32 d15[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s23
+; BE-I32-NEXT:    vmov.32 d13[1], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s21
+; BE-I32-NEXT:    vmov.32 d15[1], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s17
+; BE-I32-NEXT:    vmov.32 d14[1], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.32 d12[1], r0
+; BE-I32-NEXT:    vrev64.32 q0, q7
+; BE-I32-NEXT:    vrev64.32 q1, q6
+; BE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v8f32:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    .pad #32
+; BE-I64-NEXT:    sub sp, sp, #32
+; BE-I64-NEXT:    vorr q4, q1, q1
+; BE-I64-NEXT:    add lr, sp, #8
+; BE-I64-NEXT:    vorr q5, q0, q0
+; BE-I64-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; BE-I64-NEXT:    vrev64.32 d12, d8
+; BE-I64-NEXT:    vmov.f32 s0, s25
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s24
+; BE-I64-NEXT:    mov r8, r1
+; BE-I64-NEXT:    vmov.32 d15[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vrev64.32 d0, d11
+; BE-I64-NEXT:    mov r9, r1
+; BE-I64-NEXT:    vrev64.32 d8, d9
+; BE-I64-NEXT:    vorr d9, d0, d0
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    vstr d8, [sp, #24] @ 8-byte Spill
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s17
+; BE-I64-NEXT:    mov r10, r1
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    add lr, sp, #8
+; BE-I64-NEXT:    vmov.f32 s0, s19
+; BE-I64-NEXT:    mov r7, r1
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    vrev64.32 d8, d16
+; BE-I64-NEXT:    vstr d8, [sp, #8] @ 8-byte Spill
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s16
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vldr d0, [sp, #8] @ 8-byte Reload
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    vmov.32 d8[0], r0
+; BE-I64-NEXT:    vmov.f32 s0, s1
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vldr d0, [sp, #24] @ 8-byte Reload
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    @ kill: def $s0 killed $s0 killed $d0
+; BE-I64-NEXT:    vmov.32 d9[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.32 d12[0], r0
+; BE-I64-NEXT:    vmov.32 d9[1], r6
+; BE-I64-NEXT:    vmov.32 d11[1], r4
+; BE-I64-NEXT:    vmov.32 d15[1], r8
+; BE-I64-NEXT:    vmov.32 d13[1], r7
+; BE-I64-NEXT:    vmov.32 d8[1], r5
+; BE-I64-NEXT:    vmov.32 d10[1], r10
+; BE-I64-NEXT:    vmov.32 d14[1], r9
+; BE-I64-NEXT:    vmov.32 d12[1], r1
+; BE-I64-NEXT:    vrev64.32 q0, q4
+; BE-I64-NEXT:    vrev64.32 q1, q5
+; BE-I64-NEXT:    vrev64.32 q2, q7
+; BE-I64-NEXT:    vrev64.32 q3, q6
+; BE-I64-NEXT:    add sp, sp, #32
+; BE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float>)
+
+define <16 x iXLen> @lrint_v16f32(<16 x float> %x) {
+; LE-I32-LABEL: lrint_v16f32:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    .pad #80
+; LE-I32-NEXT:    sub sp, sp, #80
+; LE-I32-NEXT:    vorr q5, q3, q3
+; LE-I32-NEXT:    vstmia sp, {d0, d1} @ 16-byte Spill
+; LE-I32-NEXT:    add lr, sp, #32
+; LE-I32-NEXT:    vorr q6, q2, q2
+; LE-I32-NEXT:    vorr q7, q1, q1
+; LE-I32-NEXT:    vmov.f32 s0, s20
+; LE-I32-NEXT:    vstmia lr, {d6, d7} @ 16-byte Spill
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s22
+; LE-I32-NEXT:    vmov.32 d8[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s24
+; LE-I32-NEXT:    add lr, sp, #48
+; LE-I32-NEXT:    vmov.32 d9[0], r0
+; LE-I32-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-I32-NEXT:    add lr, sp, #16
+; LE-I32-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s26
+; LE-I32-NEXT:    vmov.32 d8[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    add lr, sp, #64
+; LE-I32-NEXT:    vmov.32 d9[0], r0
+; LE-I32-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-I32-NEXT:    vorr q4, q7, q7
+; LE-I32-NEXT:    vmov.f32 s0, s16
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s18
+; LE-I32-NEXT:    vmov.32 d10[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vldmia sp, {d12, d13} @ 16-byte Reload
+; LE-I32-NEXT:    vmov.f32 s0, s26
+; LE-I32-NEXT:    vmov.32 d11[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s24
+; LE-I32-NEXT:    vmov.32 d15[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s27
+; LE-I32-NEXT:    vmov.32 d14[0], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s25
+; LE-I32-NEXT:    vmov.32 d15[1], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s19
+; LE-I32-NEXT:    vmov.32 d14[1], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s17
+; LE-I32-NEXT:    vmov.32 d11[1], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    add lr, sp, #16
+; LE-I32-NEXT:    vmov.32 d10[1], r0
+; LE-I32-NEXT:    vldmia lr, {d12, d13} @ 16-byte Reload
+; LE-I32-NEXT:    vmov.f32 s0, s27
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s25
+; LE-I32-NEXT:    add lr, sp, #64
+; LE-I32-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-I32-NEXT:    vmov.32 d9[1], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.32 d8[1], r0
+; LE-I32-NEXT:    add lr, sp, #64
+; LE-I32-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-I32-NEXT:    add lr, sp, #32
+; LE-I32-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-I32-NEXT:    vmov.f32 s0, s19
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    vmov.f32 s0, s17
+; LE-I32-NEXT:    add lr, sp, #48
+; LE-I32-NEXT:    vldmia lr, {d12, d13} @ 16-byte Reload
+; LE-I32-NEXT:    vmov.32 d13[1], r0
+; LE-I32-NEXT:    bl lrintf
+; LE-I32-NEXT:    add lr, sp, #64
+; LE-I32-NEXT:    vmov.32 d12[1], r0
+; LE-I32-NEXT:    vorr q0, q7, q7
+; LE-I32-NEXT:    vldmia lr, {d4, d5} @ 16-byte Reload
+; LE-I32-NEXT:    vorr q1, q5, q5
+; LE-I32-NEXT:    vorr q3, q6, q6
+; LE-I32-NEXT:    add sp, sp, #80
+; LE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v16f32:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I64-NEXT:    .pad #4
+; LE-I64-NEXT:    sub sp, sp, #4
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    .pad #160
+; LE-I64-NEXT:    sub sp, sp, #160
+; LE-I64-NEXT:    add lr, sp, #112
+; LE-I64-NEXT:    vorr q5, q3, q3
+; LE-I64-NEXT:    vorr q6, q0, q0
+; LE-I64-NEXT:    mov r4, r0
+; LE-I64-NEXT:    vstmia lr, {d4, d5} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #48
+; LE-I64-NEXT:    vorr q7, q1, q1
+; LE-I64-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; LE-I64-NEXT:    vmov.f32 s0, s23
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s24
+; LE-I64-NEXT:    add lr, sp, #144
+; LE-I64-NEXT:    vmov.32 d17[0], r0
+; LE-I64-NEXT:    str r1, [sp, #108] @ 4-byte Spill
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s25
+; LE-I64-NEXT:    str r1, [sp, #84] @ 4-byte Spill
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s28
+; LE-I64-NEXT:    add lr, sp, #128
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    str r1, [sp, #44] @ 4-byte Spill
+; LE-I64-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s29
+; LE-I64-NEXT:    mov r9, r1
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s30
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s31
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    vmov.32 d12[0], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    add lr, sp, #112
+; LE-I64-NEXT:    mov r7, r1
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    vldmia lr, {d14, d15} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.f32 s0, s29
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s22
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    vmov.32 d17[0], r0
+; LE-I64-NEXT:    mov r11, r1
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    vmov.32 d13[1], r7
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    add lr, sp, #144
+; LE-I64-NEXT:    vmov.f32 s0, s21
+; LE-I64-NEXT:    vmov.32 d12[1], r5
+; LE-I64-NEXT:    str r1, [sp, #40] @ 4-byte Spill
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.32 d16[0], r0
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #88
+; LE-I64-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s20
+; LE-I64-NEXT:    mov r10, r1
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    vmov.32 d9[1], r6
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s31
+; LE-I64-NEXT:    vmov.32 d12[0], r0
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    mov r8, r1
+; LE-I64-NEXT:    vmov.32 d8[1], r9
+; LE-I64-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #64
+; LE-I64-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    add lr, sp, #128
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #44] @ 4-byte Reload
+; LE-I64-NEXT:    mov r9, r1
+; LE-I64-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #48
+; LE-I64-NEXT:    vldmia lr, {d12, d13} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.f32 s0, s27
+; LE-I64-NEXT:    vmov.32 d11[1], r0
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.f32 s0, s26
+; LE-I64-NEXT:    vmov.32 d15[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #84] @ 4-byte Reload
+; LE-I64-NEXT:    add lr, sp, #128
+; LE-I64-NEXT:    mov r7, r1
+; LE-I64-NEXT:    vmov.32 d10[1], r0
+; LE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    vmov.32 d14[0], r0
+; LE-I64-NEXT:    add lr, sp, #144
+; LE-I64-NEXT:    ldr r0, [sp, #108] @ 4-byte Reload
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.32 d17[1], r0
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #112
+; LE-I64-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.f32 s0, s20
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    vmov.f32 s0, s22
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.32 d16[0], r0
+; LE-I64-NEXT:    vmov.32 d17[1], r11
+; LE-I64-NEXT:    vorr q6, q8, q8
+; LE-I64-NEXT:    bl lrintf
+; LE-I64-NEXT:    add lr, sp, #144
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #40] @ 4-byte Reload
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #128
+; LE-I64-NEXT:    vmov.32 d9[1], r9
+; LE-I64-NEXT:    vmov.32 d12[1], r6
+; LE-I64-NEXT:    vmov.32 d19[1], r10
+; LE-I64-NEXT:    vmov.32 d8[1], r1
+; LE-I64-NEXT:    vmov.32 d16[1], r0
+; LE-I64-NEXT:    add r0, r4, #64
+; LE-I64-NEXT:    vmov.32 d18[1], r8
+; LE-I64-NEXT:    vst1.64 {d12, d13}, [r0:128]!
+; LE-I64-NEXT:    vst1.64 {d8, d9}, [r0:128]!
+; LE-I64-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; LE-I64-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; LE-I64-NEXT:    vmov.32 d15[1], r7
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #64
+; LE-I64-NEXT:    vmov.32 d14[1], r5
+; LE-I64-NEXT:    vst1.64 {d16, d17}, [r4:128]!
+; LE-I64-NEXT:    vst1.64 {d14, d15}, [r4:128]!
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #88
+; LE-I64-NEXT:    vst1.64 {d16, d17}, [r4:128]!
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    vst1.64 {d16, d17}, [r4:128]
+; LE-I64-NEXT:    add sp, sp, #160
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    add sp, sp, #4
+; LE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-I32-LABEL: lrint_v16f32:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    .pad #96
+; BE-I32-NEXT:    sub sp, sp, #96
+; BE-I32-NEXT:    vrev64.32 q3, q3
+; BE-I32-NEXT:    add lr, sp, #64
+; BE-I32-NEXT:    vrev64.32 q4, q0
+; BE-I32-NEXT:    vmov.f32 s0, s12
+; BE-I32-NEXT:    vstmia lr, {d6, d7} @ 16-byte Spill
+; BE-I32-NEXT:    add lr, sp, #32
+; BE-I32-NEXT:    vrev64.32 q5, q1
+; BE-I32-NEXT:    vrev64.32 q7, q2
+; BE-I32-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s16
+; BE-I32-NEXT:    vmov.32 d16[0], r0
+; BE-I32-NEXT:    add lr, sp, #80
+; BE-I32-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s18
+; BE-I32-NEXT:    vmov.32 d12[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s20
+; BE-I32-NEXT:    add lr, sp, #48
+; BE-I32-NEXT:    vmov.32 d13[0], r0
+; BE-I32-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; BE-I32-NEXT:    add lr, sp, #16
+; BE-I32-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s22
+; BE-I32-NEXT:    vmov.32 d8[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s28
+; BE-I32-NEXT:    vmov.32 d9[0], r0
+; BE-I32-NEXT:    vstmia sp, {d8, d9} @ 16-byte Spill
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    add lr, sp, #64
+; BE-I32-NEXT:    vmov.32 d12[0], r0
+; BE-I32-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; BE-I32-NEXT:    vmov.f32 s0, s22
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s30
+; BE-I32-NEXT:    add lr, sp, #80
+; BE-I32-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-I32-NEXT:    vmov.32 d9[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s23
+; BE-I32-NEXT:    vmov.32 d13[0], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s31
+; BE-I32-NEXT:    add lr, sp, #80
+; BE-I32-NEXT:    vmov.32 d9[1], r0
+; BE-I32-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s29
+; BE-I32-NEXT:    vmov.32 d13[1], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    add lr, sp, #16
+; BE-I32-NEXT:    vmov.32 d12[1], r0
+; BE-I32-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-I32-NEXT:    vmov.f32 s0, s19
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s17
+; BE-I32-NEXT:    vldmia sp, {d10, d11} @ 16-byte Reload
+; BE-I32-NEXT:    vmov.32 d11[1], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    add lr, sp, #32
+; BE-I32-NEXT:    vmov.32 d10[1], r0
+; BE-I32-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-I32-NEXT:    vmov.f32 s0, s19
+; BE-I32-NEXT:    vorr q7, q5, q5
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    vmov.f32 s0, s17
+; BE-I32-NEXT:    add lr, sp, #48
+; BE-I32-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; BE-I32-NEXT:    vmov.32 d11[1], r0
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    add lr, sp, #64
+; BE-I32-NEXT:    vmov.32 d10[1], r0
+; BE-I32-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; BE-I32-NEXT:    vmov.f32 s0, s1
+; BE-I32-NEXT:    bl lrintf
+; BE-I32-NEXT:    add lr, sp, #80
+; BE-I32-NEXT:    vrev64.32 q0, q5
+; BE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vrev64.32 q1, q7
+; BE-I32-NEXT:    vmov.32 d16[1], r0
+; BE-I32-NEXT:    vrev64.32 q2, q6
+; BE-I32-NEXT:    vrev64.32 q3, q8
+; BE-I32-NEXT:    add sp, sp, #96
+; BE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v16f32:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I64-NEXT:    .pad #4
+; BE-I64-NEXT:    sub sp, sp, #4
+; BE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    .pad #144
+; BE-I64-NEXT:    sub sp, sp, #144
+; BE-I64-NEXT:    vorr q6, q3, q3
+; BE-I64-NEXT:    add lr, sp, #112
+; BE-I64-NEXT:    vorr q7, q0, q0
+; BE-I64-NEXT:    mov r4, r0
+; BE-I64-NEXT:    vstmia lr, {d4, d5} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #96
+; BE-I64-NEXT:    vrev64.32 d8, d13
+; BE-I64-NEXT:    vstmia lr, {d2, d3} @ 16-byte Spill
+; BE-I64-NEXT:    vmov.f32 s0, s17
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s16
+; BE-I64-NEXT:    str r1, [sp, #88] @ 4-byte Spill
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vrev64.32 d8, d14
+; BE-I64-NEXT:    add lr, sp, #128
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    str r1, [sp, #92] @ 4-byte Spill
+; BE-I64-NEXT:    vmov.f32 s0, s16
+; BE-I64-NEXT:    vrev64.32 d9, d12
+; BE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-I64-NEXT:    vstr d9, [sp, #64] @ 8-byte Spill
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s19
+; BE-I64-NEXT:    mov r9, r1
+; BE-I64-NEXT:    vmov.32 d12[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s17
+; BE-I64-NEXT:    str r1, [sp, #84] @ 4-byte Spill
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    vrev64.32 d9, d15
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s18
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s19
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vldr d0, [sp, #64] @ 8-byte Reload
+; BE-I64-NEXT:    mov r7, r1
+; BE-I64-NEXT:    @ kill: def $s0 killed $s0 killed $d0
+; BE-I64-NEXT:    vmov.32 d15[0], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    add lr, sp, #40
+; BE-I64-NEXT:    str r1, [sp, #60] @ 4-byte Spill
+; BE-I64-NEXT:    vmov.32 d15[1], r7
+; BE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #96
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    vrev64.32 d8, d16
+; BE-I64-NEXT:    vmov.f32 s0, s17
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s16
+; BE-I64-NEXT:    vmov.32 d14[1], r5
+; BE-I64-NEXT:    add lr, sp, #64
+; BE-I64-NEXT:    mov r10, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    add lr, sp, #24
+; BE-I64-NEXT:    mov r11, r1
+; BE-I64-NEXT:    vmov.32 d13[1], r6
+; BE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #96
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    vrev64.32 d8, d17
+; BE-I64-NEXT:    vmov.f32 s0, s17
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s16
+; BE-I64-NEXT:    vmov.32 d12[1], r9
+; BE-I64-NEXT:    add lr, sp, #96
+; BE-I64-NEXT:    mov r8, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    add lr, sp, #8
+; BE-I64-NEXT:    ldr r0, [sp, #88] @ 4-byte Reload
+; BE-I64-NEXT:    mov r9, r1
+; BE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #112
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #128
+; BE-I64-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; BE-I64-NEXT:    vrev64.32 d8, d16
+; BE-I64-NEXT:    vmov.32 d11[1], r0
+; BE-I64-NEXT:    vmov.f32 s0, s17
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s16
+; BE-I64-NEXT:    vmov.32 d15[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #92] @ 4-byte Reload
+; BE-I64-NEXT:    add lr, sp, #128
+; BE-I64-NEXT:    mov r7, r1
+; BE-I64-NEXT:    vmov.32 d10[1], r0
+; BE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    add lr, sp, #112
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #84] @ 4-byte Reload
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #40
+; BE-I64-NEXT:    vrev64.32 d8, d17
+; BE-I64-NEXT:    vldmia lr, {d12, d13} @ 16-byte Reload
+; BE-I64-NEXT:    vmov.f32 s0, s17
+; BE-I64-NEXT:    vmov.32 d13[1], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    vmov.f32 s0, s16
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #60] @ 4-byte Reload
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    vmov.32 d12[1], r0
+; BE-I64-NEXT:    bl lrintf
+; BE-I64-NEXT:    add lr, sp, #24
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    add r0, r4, #64
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #8
+; BE-I64-NEXT:    vmov.32 d17[1], r10
+; BE-I64-NEXT:    vmov.32 d16[1], r11
+; BE-I64-NEXT:    vorr q12, q8, q8
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #128
+; BE-I64-NEXT:    vmov.32 d15[1], r7
+; BE-I64-NEXT:    vmov.32 d11[1], r6
+; BE-I64-NEXT:    vmov.32 d14[1], r5
+; BE-I64-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #96
+; BE-I64-NEXT:    vmov.32 d10[1], r1
+; BE-I64-NEXT:    vmov.32 d17[1], r8
+; BE-I64-NEXT:    vldmia lr, {d20, d21} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #64
+; BE-I64-NEXT:    vmov.32 d16[1], r9
+; BE-I64-NEXT:    vrev64.32 q14, q7
+; BE-I64-NEXT:    vorr q13, q8, q8
+; BE-I64-NEXT:    vrev64.32 q15, q5
+; BE-I64-NEXT:    vldmia lr, {d22, d23} @ 16-byte Reload
+; BE-I64-NEXT:    vrev64.32 q8, q6
+; BE-I64-NEXT:    vst1.64 {d28, d29}, [r0:128]!
+; BE-I64-NEXT:    vst1.64 {d30, d31}, [r0:128]!
+; BE-I64-NEXT:    vrev64.32 q9, q9
+; BE-I64-NEXT:    vrev64.32 q10, q10
+; BE-I64-NEXT:    vst1.64 {d16, d17}, [r0:128]!
+; BE-I64-NEXT:    vrev64.32 q11, q11
+; BE-I64-NEXT:    vrev64.32 q12, q12
+; BE-I64-NEXT:    vst1.64 {d18, d19}, [r0:128]
+; BE-I64-NEXT:    vst1.64 {d20, d21}, [r4:128]!
+; BE-I64-NEXT:    vst1.64 {d22, d23}, [r4:128]!
+; BE-I64-NEXT:    vrev64.32 q13, q13
+; BE-I64-NEXT:    vst1.64 {d24, d25}, [r4:128]!
+; BE-I64-NEXT:    vst1.64 {d26, d27}, [r4:128]
+; BE-I64-NEXT:    add sp, sp, #144
+; BE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    add sp, sp, #4
+; BE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float>)
+
+define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
+; LE-I32-LABEL: lrint_v1f64:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v1f64:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r11, lr}
+; LE-I64-NEXT:    push {r11, lr}
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vmov.32 d0[0], r0
+; LE-I64-NEXT:    vmov.32 d0[1], r1
+; LE-I64-NEXT:    pop {r11, pc}
+;
+; BE-I32-LABEL: lrint_v1f64:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v1f64:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r11, lr}
+; BE-I64-NEXT:    push {r11, lr}
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vmov.32 d16[0], r0
+; BE-I64-NEXT:    vmov.32 d16[1], r1
+; BE-I64-NEXT:    vrev64.32 d0, d16
+; BE-I64-NEXT:    pop {r11, pc}
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double>)
+
+define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
+; LE-I32-LABEL: lrint_v2f64:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    .vsave {d8, d9, d10}
+; LE-I32-NEXT:    vpush {d8, d9, d10}
+; LE-I32-NEXT:    vorr q4, q0, q0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d9, d9
+; LE-I32-NEXT:    vmov.32 d10[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vmov.32 d10[1], r0
+; LE-I32-NEXT:    vorr d0, d10, d10
+; LE-I32-NEXT:    vpop {d8, d9, d10}
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v2f64:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, lr}
+; LE-I64-NEXT:    push {r4, lr}
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11}
+; LE-I64-NEXT:    vorr q4, q0, q0
+; LE-I64-NEXT:    vorr d0, d9, d9
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d8, d8
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    vmov.32 d11[1], r4
+; LE-I64-NEXT:    vmov.32 d10[1], r1
+; LE-I64-NEXT:    vorr q0, q5, q5
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11}
+; LE-I64-NEXT:    pop {r4, pc}
+;
+; BE-I32-LABEL: lrint_v2f64:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    .vsave {d8, d9, d10}
+; BE-I32-NEXT:    vpush {d8, d9, d10}
+; BE-I32-NEXT:    vorr q4, q0, q0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d9, d9
+; BE-I32-NEXT:    vmov.32 d10[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vmov.32 d10[1], r0
+; BE-I32-NEXT:    vrev64.32 d0, d10
+; BE-I32-NEXT:    vpop {d8, d9, d10}
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v2f64:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, lr}
+; BE-I64-NEXT:    push {r4, lr}
+; BE-I64-NEXT:    .vsave {d8, d9, d10, d11}
+; BE-I64-NEXT:    vpush {d8, d9, d10, d11}
+; BE-I64-NEXT:    vorr q4, q0, q0
+; BE-I64-NEXT:    vorr d0, d9, d9
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d8, d8
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    vmov.32 d11[1], r4
+; BE-I64-NEXT:    vmov.32 d10[1], r1
+; BE-I64-NEXT:    vrev64.32 q0, q5
+; BE-I64-NEXT:    vpop {d8, d9, d10, d11}
+; BE-I64-NEXT:    pop {r4, pc}
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double>)
+
+define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
+; LE-I32-LABEL: lrint_v4f64:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; LE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; LE-I32-NEXT:    vorr q4, q1, q1
+; LE-I32-NEXT:    vorr q5, q0, q0
+; LE-I32-NEXT:    vorr d0, d8, d8
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d10, d10
+; LE-I32-NEXT:    vmov.32 d13[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d9, d9
+; LE-I32-NEXT:    vmov.32 d12[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d11, d11
+; LE-I32-NEXT:    vmov.32 d13[1], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vmov.32 d12[1], r0
+; LE-I32-NEXT:    vorr q0, q6, q6
+; LE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v4f64:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, lr}
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    vorr q5, q1, q1
+; LE-I64-NEXT:    vorr q6, q0, q0
+; LE-I64-NEXT:    vorr d0, d11, d11
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d12, d12
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d13, d13
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    vmov.32 d14[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d10, d10
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    vmov.32 d15[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    vmov.32 d15[1], r6
+; LE-I64-NEXT:    vmov.32 d9[1], r4
+; LE-I64-NEXT:    vmov.32 d14[1], r5
+; LE-I64-NEXT:    vmov.32 d8[1], r1
+; LE-I64-NEXT:    vorr q0, q7, q7
+; LE-I64-NEXT:    vorr q1, q4, q4
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    pop {r4, r5, r6, pc}
+;
+; BE-I32-LABEL: lrint_v4f64:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13}
+; BE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13}
+; BE-I32-NEXT:    vorr q4, q1, q1
+; BE-I32-NEXT:    vorr q5, q0, q0
+; BE-I32-NEXT:    vorr d0, d8, d8
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d10, d10
+; BE-I32-NEXT:    vmov.32 d13[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d9, d9
+; BE-I32-NEXT:    vmov.32 d12[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d11, d11
+; BE-I32-NEXT:    vmov.32 d13[1], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vmov.32 d12[1], r0
+; BE-I32-NEXT:    vrev64.32 q0, q6
+; BE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13}
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v4f64:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, lr}
+; BE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    vorr q4, q1, q1
+; BE-I64-NEXT:    vorr q5, q0, q0
+; BE-I64-NEXT:    vorr d0, d9, d9
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d10, d10
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d11, d11
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d8, d8
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    vmov.32 d15[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vmov.32 d12[0], r0
+; BE-I64-NEXT:    vmov.32 d15[1], r6
+; BE-I64-NEXT:    vmov.32 d13[1], r4
+; BE-I64-NEXT:    vmov.32 d14[1], r5
+; BE-I64-NEXT:    vmov.32 d12[1], r1
+; BE-I64-NEXT:    vrev64.32 q0, q7
+; BE-I64-NEXT:    vrev64.32 q1, q6
+; BE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    pop {r4, r5, r6, pc}
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double>)
+
+define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
+; LE-I32-LABEL: lrint_v8f64:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    .pad #32
+; LE-I32-NEXT:    sub sp, sp, #32
+; LE-I32-NEXT:    vorr q5, q0, q0
+; LE-I32-NEXT:    add lr, sp, #16
+; LE-I32-NEXT:    vorr d0, d4, d4
+; LE-I32-NEXT:    vstmia sp, {d6, d7} @ 16-byte Spill
+; LE-I32-NEXT:    vorr q7, q3, q3
+; LE-I32-NEXT:    vstmia lr, {d4, d5} @ 16-byte Spill
+; LE-I32-NEXT:    vorr q6, q1, q1
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d14, d14
+; LE-I32-NEXT:    vmov.32 d8[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d12, d12
+; LE-I32-NEXT:    vmov.32 d9[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d10, d10
+; LE-I32-NEXT:    vmov.32 d15[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d13, d13
+; LE-I32-NEXT:    vmov.32 d14[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d11, d11
+; LE-I32-NEXT:    vmov.32 d15[1], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vldmia sp, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    vmov.32 d14[1], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #16
+; LE-I32-NEXT:    vmov.32 d9[1], r0
+; LE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vmov.32 d8[1], r0
+; LE-I32-NEXT:    vorr q0, q7, q7
+; LE-I32-NEXT:    vorr q1, q4, q4
+; LE-I32-NEXT:    add sp, sp, #32
+; LE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v8f64:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    .pad #40
+; LE-I64-NEXT:    sub sp, sp, #40
+; LE-I64-NEXT:    vorr q4, q0, q0
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    vorr d0, d7, d7
+; LE-I64-NEXT:    vstmia lr, {d6, d7} @ 16-byte Spill
+; LE-I64-NEXT:    vorr q7, q2, q2
+; LE-I64-NEXT:    vorr q6, q1, q1
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d14, d14
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    vmov.32 d17[0], r0
+; LE-I64-NEXT:    mov r8, r1
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d15, d15
+; LE-I64-NEXT:    mov r9, r1
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d12, d12
+; LE-I64-NEXT:    mov r10, r1
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d13, d13
+; LE-I64-NEXT:    mov r7, r1
+; LE-I64-NEXT:    vmov.32 d14[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d8, d8
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d15[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d9, d9
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    vmov.32 d12[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-I64-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    vmov.32 d13[1], r6
+; LE-I64-NEXT:    vldmia lr, {d6, d7} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.32 d15[1], r4
+; LE-I64-NEXT:    vmov.32 d11[1], r10
+; LE-I64-NEXT:    vmov.32 d6[0], r0
+; LE-I64-NEXT:    vmov.32 d12[1], r5
+; LE-I64-NEXT:    vmov.32 d14[1], r7
+; LE-I64-NEXT:    vorr q0, q6, q6
+; LE-I64-NEXT:    vmov.32 d10[1], r9
+; LE-I64-NEXT:    vorr q1, q7, q7
+; LE-I64-NEXT:    vmov.32 d7[1], r8
+; LE-I64-NEXT:    vorr q2, q5, q5
+; LE-I64-NEXT:    vmov.32 d6[1], r1
+; LE-I64-NEXT:    add sp, sp, #40
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+;
+; BE-I32-LABEL: lrint_v8f64:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    .pad #32
+; BE-I32-NEXT:    sub sp, sp, #32
+; BE-I32-NEXT:    vorr q5, q0, q0
+; BE-I32-NEXT:    vstmia sp, {d0, d1} @ 16-byte Spill
+; BE-I32-NEXT:    vorr d0, d4, d4
+; BE-I32-NEXT:    add lr, sp, #16
+; BE-I32-NEXT:    vorr q7, q3, q3
+; BE-I32-NEXT:    vstmia lr, {d4, d5} @ 16-byte Spill
+; BE-I32-NEXT:    vorr q6, q1, q1
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d10, d10
+; BE-I32-NEXT:    vmov.32 d8[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d14, d14
+; BE-I32-NEXT:    vmov.32 d10[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d12, d12
+; BE-I32-NEXT:    vmov.32 d9[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d15, d15
+; BE-I32-NEXT:    vmov.32 d11[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d13, d13
+; BE-I32-NEXT:    vmov.32 d9[1], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vldmia sp, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    vmov.32 d11[1], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #16
+; BE-I32-NEXT:    vmov.32 d10[1], r0
+; BE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vmov.32 d8[1], r0
+; BE-I32-NEXT:    vrev64.32 q0, q5
+; BE-I32-NEXT:    vrev64.32 q1, q4
+; BE-I32-NEXT:    add sp, sp, #32
+; BE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v8f64:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    .pad #40
+; BE-I64-NEXT:    sub sp, sp, #40
+; BE-I64-NEXT:    vorr q4, q0, q0
+; BE-I64-NEXT:    add lr, sp, #24
+; BE-I64-NEXT:    vorr d0, d7, d7
+; BE-I64-NEXT:    vstmia lr, {d6, d7} @ 16-byte Spill
+; BE-I64-NEXT:    vorr q7, q2, q2
+; BE-I64-NEXT:    vorr q6, q1, q1
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d14, d14
+; BE-I64-NEXT:    add lr, sp, #8
+; BE-I64-NEXT:    vmov.32 d17[0], r0
+; BE-I64-NEXT:    mov r8, r1
+; BE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d15, d15
+; BE-I64-NEXT:    mov r9, r1
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d12, d12
+; BE-I64-NEXT:    mov r10, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d13, d13
+; BE-I64-NEXT:    mov r7, r1
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d8, d8
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d15[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d9, d9
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    vmov.32 d12[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    add lr, sp, #24
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; BE-I64-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    add lr, sp, #8
+; BE-I64-NEXT:    vmov.32 d13[1], r6
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    vmov.32 d15[1], r4
+; BE-I64-NEXT:    vmov.32 d16[0], r0
+; BE-I64-NEXT:    vmov.32 d11[1], r10
+; BE-I64-NEXT:    vmov.32 d17[1], r8
+; BE-I64-NEXT:    vmov.32 d12[1], r5
+; BE-I64-NEXT:    vmov.32 d14[1], r7
+; BE-I64-NEXT:    vmov.32 d10[1], r9
+; BE-I64-NEXT:    vmov.32 d16[1], r1
+; BE-I64-NEXT:    vrev64.32 q0, q6
+; BE-I64-NEXT:    vrev64.32 q1, q7
+; BE-I64-NEXT:    vrev64.32 q2, q5
+; BE-I64-NEXT:    vrev64.32 q3, q8
+; BE-I64-NEXT:    add sp, sp, #40
+; BE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double>)
+
+define <16 x iXLen> @lrint_v16f64(<16 x double> %x) {
+; LE-I32-LABEL: lrint_v16f64:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r4, r5, r6, lr}
+; LE-I32-NEXT:    push {r4, r5, r6, lr}
+; LE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    .pad #128
+; LE-I32-NEXT:    sub sp, sp, #128
+; LE-I32-NEXT:    add lr, sp, #80
+; LE-I32-NEXT:    add r0, sp, #240
+; LE-I32-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-I32-NEXT:    add r0, sp, #208
+; LE-I32-NEXT:    vorr q6, q0, q0
+; LE-I32-NEXT:    vstmia lr, {d6, d7} @ 16-byte Spill
+; LE-I32-NEXT:    add lr, sp, #32
+; LE-I32-NEXT:    vorr q5, q1, q1
+; LE-I32-NEXT:    vstmia lr, {d4, d5} @ 16-byte Spill
+; LE-I32-NEXT:    add lr, sp, #16
+; LE-I32-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; LE-I32-NEXT:    add lr, sp, #64
+; LE-I32-NEXT:    vorr d0, d4, d4
+; LE-I32-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I32-NEXT:    add lr, sp, #112
+; LE-I32-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-I32-NEXT:    add r0, sp, #224
+; LE-I32-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I32-NEXT:    add lr, sp, #96
+; LE-I32-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-I32-NEXT:    add r0, sp, #256
+; LE-I32-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I32-NEXT:    add lr, sp, #48
+; LE-I32-NEXT:    vld1.64 {d14, d15}, [r0]
+; LE-I32-NEXT:    vstmia sp, {d2, d3} @ 16-byte Spill
+; LE-I32-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d12, d12
+; LE-I32-NEXT:    vmov.32 d8[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d10, d10
+; LE-I32-NEXT:    vmov.32 d12[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vorr d0, d14, d14
+; LE-I32-NEXT:    vmov.32 d13[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #64
+; LE-I32-NEXT:    mov r4, r0
+; LE-I32-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-I32-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #80
+; LE-I32-NEXT:    vmov.32 d14[0], r0
+; LE-I32-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-I32-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #112
+; LE-I32-NEXT:    vmov.32 d9[0], r0
+; LE-I32-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-I32-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-I32-NEXT:    vmov.32 d15[0], r4
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vldmia sp, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    vmov.32 d10[0], r0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #96
+; LE-I32-NEXT:    vmov.32 d13[1], r0
+; LE-I32-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-I32-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #16
+; LE-I32-NEXT:    vmov.32 d11[0], r0
+; LE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #80
+; LE-I32-NEXT:    vmov.32 d12[1], r0
+; LE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #32
+; LE-I32-NEXT:    vmov.32 d9[1], r0
+; LE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #96
+; LE-I32-NEXT:    vmov.32 d8[1], r0
+; LE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #112
+; LE-I32-NEXT:    vmov.32 d11[1], r0
+; LE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #48
+; LE-I32-NEXT:    vmov.32 d10[1], r0
+; LE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    add lr, sp, #64
+; LE-I32-NEXT:    vmov.32 d15[1], r0
+; LE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I32-NEXT:    vorr d0, d17, d17
+; LE-I32-NEXT:    bl lrint
+; LE-I32-NEXT:    vmov.32 d14[1], r0
+; LE-I32-NEXT:    vorr q0, q6, q6
+; LE-I32-NEXT:    vorr q1, q4, q4
+; LE-I32-NEXT:    vorr q2, q5, q5
+; LE-I32-NEXT:    vorr q3, q7, q7
+; LE-I32-NEXT:    add sp, sp, #128
+; LE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    pop {r4, r5, r6, pc}
+;
+; LE-I64-LABEL: lrint_v16f64:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I64-NEXT:    .pad #4
+; LE-I64-NEXT:    sub sp, sp, #4
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    .pad #176
+; LE-I64-NEXT:    sub sp, sp, #176
+; LE-I64-NEXT:    add lr, sp, #40
+; LE-I64-NEXT:    str r0, [sp, #140] @ 4-byte Spill
+; LE-I64-NEXT:    add r0, sp, #312
+; LE-I64-NEXT:    vorr q6, q2, q2
+; LE-I64-NEXT:    vstmia lr, {d6, d7} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #96
+; LE-I64-NEXT:    vorr q7, q1, q1
+; LE-I64-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #144
+; LE-I64-NEXT:    vorr d0, d1, d1
+; LE-I64-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-I64-NEXT:    add r0, sp, #280
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #80
+; LE-I64-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-I64-NEXT:    add r0, sp, #296
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #120
+; LE-I64-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-I64-NEXT:    add r0, sp, #328
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #56
+; LE-I64-NEXT:    vld1.64 {d16, d17}, [r0]
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d14, d14
+; LE-I64-NEXT:    str r1, [sp, #116] @ 4-byte Spill
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d15, d15
+; LE-I64-NEXT:    str r1, [sp, #76] @ 4-byte Spill
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d12, d12
+; LE-I64-NEXT:    add lr, sp, #160
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    str r1, [sp, #72] @ 4-byte Spill
+; LE-I64-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d13, d13
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    vmov.32 d14[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    add lr, sp, #40
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d15[0], r0
+; LE-I64-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-I64-NEXT:    vorr d0, d8, d8
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d9, d9
+; LE-I64-NEXT:    mov r7, r1
+; LE-I64-NEXT:    vmov.32 d12[0], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    add lr, sp, #96
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-I64-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    add lr, sp, #40
+; LE-I64-NEXT:    mov r10, r1
+; LE-I64-NEXT:    vmov.32 d13[1], r5
+; LE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #56
+; LE-I64-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-I64-NEXT:    vorr d0, d9, d9
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vorr d0, d8, d8
+; LE-I64-NEXT:    vmov.32 d12[1], r7
+; LE-I64-NEXT:    add lr, sp, #96
+; LE-I64-NEXT:    mov r9, r1
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    mov r11, r1
+; LE-I64-NEXT:    vmov.32 d15[1], r4
+; LE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #144
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    vorr d0, d17, d17
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    vmov.32 d14[1], r6
+; LE-I64-NEXT:    mov r8, r1
+; LE-I64-NEXT:    vmov.32 d17[0], r0
+; LE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #56
+; LE-I64-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #80
+; LE-I64-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; LE-I64-NEXT:    vorr d0, d11, d11
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vmov.32 d15[0], r0
+; LE-I64-NEXT:    add lr, sp, #160
+; LE-I64-NEXT:    vorr d0, d10, d10
+; LE-I64-NEXT:    ldr r0, [sp, #72] @ 4-byte Reload
+; LE-I64-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    vmov.32 d9[1], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vmov.32 d14[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #76] @ 4-byte Reload
+; LE-I64-NEXT:    add lr, sp, #160
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d8[1], r0
+; LE-I64-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #120
+; LE-I64-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; LE-I64-NEXT:    vorr d0, d11, d11
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    add lr, sp, #40
+; LE-I64-NEXT:    vorr d0, d10, d10
+; LE-I64-NEXT:    ldr r0, [sp, #116] @ 4-byte Reload
+; LE-I64-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    vmov.32 d9[1], r0
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    add lr, sp, #144
+; LE-I64-NEXT:    mov r7, r1
+; LE-I64-NEXT:    vmov.32 d12[0], r0
+; LE-I64-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; LE-I64-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; LE-I64-NEXT:    vmov.32 d8[1], r10
+; LE-I64-NEXT:    bl lrint
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    vmov.32 d15[1], r6
+; LE-I64-NEXT:    vldmia lr, {d20, d21} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #160
+; LE-I64-NEXT:    vmov.32 d20[0], r0
+; LE-I64-NEXT:    vmov.32 d21[1], r8
+; LE-I64-NEXT:    vmov.32 d20[1], r1
+; LE-I64-NEXT:    ldr r1, [sp, #140] @ 4-byte Reload
+; LE-I64-NEXT:    vmov.32 d13[1], r5
+; LE-I64-NEXT:    mov r0, r1
+; LE-I64-NEXT:    vst1.64 {d8, d9}, [r0:128]!
+; LE-I64-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #56
+; LE-I64-NEXT:    vmov.32 d14[1], r4
+; LE-I64-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; LE-I64-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #96
+; LE-I64-NEXT:    vmov.32 d12[1], r7
+; LE-I64-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; LE-I64-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-I64-NEXT:    vmov.32 d17[1], r9
+; LE-I64-NEXT:    vst1.64 {d18, d19}, [r0:128]
+; LE-I64-NEXT:    add r0, r1, #64
+; LE-I64-NEXT:    vst1.64 {d14, d15}, [r0:128]!
+; LE-I64-NEXT:    vst1.64 {d12, d13}, [r0:128]!
+; LE-I64-NEXT:    vmov.32 d16[1], r11
+; LE-I64-NEXT:    vst1.64 {d20, d21}, [r0:128]!
+; LE-I64-NEXT:    vst1.64 {d16, d17}, [r0:128]
+; LE-I64-NEXT:    add sp, sp, #176
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    add sp, sp, #4
+; LE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-I32-LABEL: lrint_v16f64:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r4, r5, r6, lr}
+; BE-I32-NEXT:    push {r4, r5, r6, lr}
+; BE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    .pad #128
+; BE-I32-NEXT:    sub sp, sp, #128
+; BE-I32-NEXT:    add lr, sp, #64
+; BE-I32-NEXT:    add r0, sp, #240
+; BE-I32-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-I32-NEXT:    add r0, sp, #224
+; BE-I32-NEXT:    vorr q6, q3, q3
+; BE-I32-NEXT:    vstmia lr, {d4, d5} @ 16-byte Spill
+; BE-I32-NEXT:    add lr, sp, #16
+; BE-I32-NEXT:    vorr q5, q1, q1
+; BE-I32-NEXT:    vstmia lr, {d2, d3} @ 16-byte Spill
+; BE-I32-NEXT:    add lr, sp, #32
+; BE-I32-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; BE-I32-NEXT:    add lr, sp, #80
+; BE-I32-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; BE-I32-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I32-NEXT:    add lr, sp, #112
+; BE-I32-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-I32-NEXT:    add r0, sp, #256
+; BE-I32-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I32-NEXT:    add lr, sp, #96
+; BE-I32-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-I32-NEXT:    add r0, sp, #208
+; BE-I32-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I32-NEXT:    add lr, sp, #48
+; BE-I32-NEXT:    vld1.64 {d14, d15}, [r0]
+; BE-I32-NEXT:    vstmia sp, {d6, d7} @ 16-byte Spill
+; BE-I32-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d10, d10
+; BE-I32-NEXT:    vmov.32 d8[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d12, d12
+; BE-I32-NEXT:    vmov.32 d9[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vorr d0, d14, d14
+; BE-I32-NEXT:    vmov.32 d11[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #80
+; BE-I32-NEXT:    mov r4, r0
+; BE-I32-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; BE-I32-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #64
+; BE-I32-NEXT:    vmov.32 d12[0], r0
+; BE-I32-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; BE-I32-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #112
+; BE-I32-NEXT:    vmov.32 d10[0], r0
+; BE-I32-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; BE-I32-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; BE-I32-NEXT:    vmov.32 d14[0], r4
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vldmia sp, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    vmov.32 d15[0], r0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #96
+; BE-I32-NEXT:    vmov.32 d11[1], r0
+; BE-I32-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; BE-I32-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #64
+; BE-I32-NEXT:    vmov.32 d13[0], r0
+; BE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #16
+; BE-I32-NEXT:    vmov.32 d10[1], r0
+; BE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #32
+; BE-I32-NEXT:    vmov.32 d9[1], r0
+; BE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #96
+; BE-I32-NEXT:    vmov.32 d8[1], r0
+; BE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #112
+; BE-I32-NEXT:    vmov.32 d13[1], r0
+; BE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #48
+; BE-I32-NEXT:    vmov.32 d15[1], r0
+; BE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    add lr, sp, #80
+; BE-I32-NEXT:    vmov.32 d14[1], r0
+; BE-I32-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I32-NEXT:    vorr d0, d17, d17
+; BE-I32-NEXT:    bl lrint
+; BE-I32-NEXT:    vmov.32 d12[1], r0
+; BE-I32-NEXT:    vrev64.32 q0, q4
+; BE-I32-NEXT:    vrev64.32 q1, q5
+; BE-I32-NEXT:    vrev64.32 q2, q7
+; BE-I32-NEXT:    vrev64.32 q3, q6
+; BE-I32-NEXT:    add sp, sp, #128
+; BE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    pop {r4, r5, r6, pc}
+;
+; BE-I64-LABEL: lrint_v16f64:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I64-NEXT:    .pad #4
+; BE-I64-NEXT:    sub sp, sp, #4
+; BE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    .pad #168
+; BE-I64-NEXT:    sub sp, sp, #168
+; BE-I64-NEXT:    add lr, sp, #64
+; BE-I64-NEXT:    str r0, [sp, #132] @ 4-byte Spill
+; BE-I64-NEXT:    add r0, sp, #304
+; BE-I64-NEXT:    vorr q4, q3, q3
+; BE-I64-NEXT:    vstmia lr, {d0, d1} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #48
+; BE-I64-NEXT:    vorr d0, d1, d1
+; BE-I64-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-I64-NEXT:    add r0, sp, #320
+; BE-I64-NEXT:    vorr q6, q2, q2
+; BE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #88
+; BE-I64-NEXT:    vorr q7, q1, q1
+; BE-I64-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-I64-NEXT:    add r0, sp, #272
+; BE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #112
+; BE-I64-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-I64-NEXT:    add r0, sp, #288
+; BE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #24
+; BE-I64-NEXT:    vld1.64 {d16, d17}, [r0]
+; BE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d14, d14
+; BE-I64-NEXT:    add lr, sp, #136
+; BE-I64-NEXT:    vmov.32 d17[0], r0
+; BE-I64-NEXT:    str r1, [sp, #108] @ 4-byte Spill
+; BE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d15, d15
+; BE-I64-NEXT:    str r1, [sp, #84] @ 4-byte Spill
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d12, d12
+; BE-I64-NEXT:    add lr, sp, #152
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    str r1, [sp, #44] @ 4-byte Spill
+; BE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d13, d13
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d8, d8
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d9, d9
+; BE-I64-NEXT:    mov r7, r1
+; BE-I64-NEXT:    vmov.32 d12[0], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    add lr, sp, #64
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    vldmia lr, {d0, d1} @ 16-byte Reload
+; BE-I64-NEXT:    @ kill: def $d0 killed $d0 killed $q0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    add lr, sp, #136
+; BE-I64-NEXT:    mov r9, r1
+; BE-I64-NEXT:    vmov.32 d13[1], r5
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    vmov.32 d16[0], r0
+; BE-I64-NEXT:    vstmia lr, {d16, d17} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #24
+; BE-I64-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-I64-NEXT:    vorr d0, d9, d9
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d8, d8
+; BE-I64-NEXT:    vmov.32 d12[1], r7
+; BE-I64-NEXT:    add lr, sp, #64
+; BE-I64-NEXT:    mov r10, r1
+; BE-I64-NEXT:    vmov.32 d15[0], r0
+; BE-I64-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    add lr, sp, #8
+; BE-I64-NEXT:    mov r11, r1
+; BE-I64-NEXT:    vmov.32 d11[1], r4
+; BE-I64-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #48
+; BE-I64-NEXT:    vorr q6, q5, q5
+; BE-I64-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-I64-NEXT:    vorr d0, d9, d9
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d8, d8
+; BE-I64-NEXT:    vmov.32 d12[1], r6
+; BE-I64-NEXT:    add lr, sp, #24
+; BE-I64-NEXT:    mov r8, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    vstmia lr, {d12, d13} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    add lr, sp, #48
+; BE-I64-NEXT:    ldr r0, [sp, #44] @ 4-byte Reload
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; BE-I64-NEXT:    add lr, sp, #152
+; BE-I64-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #88
+; BE-I64-NEXT:    vldmia lr, {d12, d13} @ 16-byte Reload
+; BE-I64-NEXT:    vorr d0, d13, d13
+; BE-I64-NEXT:    vmov.32 d9[1], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vmov.32 d15[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #84] @ 4-byte Reload
+; BE-I64-NEXT:    vorr d0, d12, d12
+; BE-I64-NEXT:    add lr, sp, #152
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d8[1], r0
+; BE-I64-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    add lr, sp, #136
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #108] @ 4-byte Reload
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    vldmia lr, {d10, d11} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #112
+; BE-I64-NEXT:    vldmia lr, {d8, d9} @ 16-byte Reload
+; BE-I64-NEXT:    vorr d0, d9, d9
+; BE-I64-NEXT:    vmov.32 d11[1], r0
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    vorr d0, d8, d8
+; BE-I64-NEXT:    mov r7, r1
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    vmov.32 d10[1], r9
+; BE-I64-NEXT:    bl lrint
+; BE-I64-NEXT:    add lr, sp, #8
+; BE-I64-NEXT:    vmov.32 d12[0], r0
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #48
+; BE-I64-NEXT:    vmov.32 d17[1], r10
+; BE-I64-NEXT:    vmov.32 d16[1], r11
+; BE-I64-NEXT:    vorr q12, q8, q8
+; BE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #152
+; BE-I64-NEXT:    vmov.32 d17[1], r8
+; BE-I64-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #24
+; BE-I64-NEXT:    vmov.32 d13[1], r7
+; BE-I64-NEXT:    vmov.32 d16[1], r6
+; BE-I64-NEXT:    vldmia lr, {d20, d21} @ 16-byte Reload
+; BE-I64-NEXT:    add lr, sp, #64
+; BE-I64-NEXT:    vorr q13, q8, q8
+; BE-I64-NEXT:    vmov.32 d12[1], r1
+; BE-I64-NEXT:    ldr r1, [sp, #132] @ 4-byte Reload
+; BE-I64-NEXT:    vrev64.32 q8, q5
+; BE-I64-NEXT:    mov r0, r1
+; BE-I64-NEXT:    vldmia lr, {d22, d23} @ 16-byte Reload
+; BE-I64-NEXT:    vrev64.32 q9, q9
+; BE-I64-NEXT:    vrev64.32 q10, q10
+; BE-I64-NEXT:    vst1.64 {d16, d17}, [r0:128]!
+; BE-I64-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; BE-I64-NEXT:    vrev64.32 q11, q11
+; BE-I64-NEXT:    vmov.32 d15[1], r4
+; BE-I64-NEXT:    vst1.64 {d20, d21}, [r0:128]!
+; BE-I64-NEXT:    vrev64.32 q15, q6
+; BE-I64-NEXT:    vmov.32 d14[1], r5
+; BE-I64-NEXT:    vrev64.32 q12, q12
+; BE-I64-NEXT:    vst1.64 {d22, d23}, [r0:128]
+; BE-I64-NEXT:    add r0, r1, #64
+; BE-I64-NEXT:    vrev64.32 q13, q13
+; BE-I64-NEXT:    vst1.64 {d30, d31}, [r0:128]!
+; BE-I64-NEXT:    vst1.64 {d24, d25}, [r0:128]!
+; BE-I64-NEXT:    vrev64.32 q14, q7
+; BE-I64-NEXT:    vst1.64 {d26, d27}, [r0:128]!
+; BE-I64-NEXT:    vst1.64 {d28, d29}, [r0:128]
+; BE-I64-NEXT:    add sp, sp, #168
+; BE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    add sp, sp, #4
+; BE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f64(<16 x double> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f64(<16 x double>)
+
+define <1 x iXLen> @lrint_v1fp128(<1 x fp128> %x) {
+; LE-I32-LABEL: lrint_v1fp128:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r11, lr}
+; LE-I32-NEXT:    push {r11, lr}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    pop {r11, pc}
+;
+; LE-I64-LABEL: lrint_v1fp128:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r11, lr}
+; LE-I64-NEXT:    push {r11, lr}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    vmov.32 d0[0], r0
+; LE-I64-NEXT:    vmov.32 d0[1], r1
+; LE-I64-NEXT:    pop {r11, pc}
+;
+; BE-I32-LABEL: lrint_v1fp128:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r11, lr}
+; BE-I32-NEXT:    push {r11, lr}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    pop {r11, pc}
+;
+; BE-I64-LABEL: lrint_v1fp128:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r11, lr}
+; BE-I64-NEXT:    push {r11, lr}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    vmov.32 d16[0], r0
+; BE-I64-NEXT:    vmov.32 d16[1], r1
+; BE-I64-NEXT:    vrev64.32 d0, d16
+; BE-I64-NEXT:    pop {r11, pc}
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1fp128(<1 x fp128> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1fp128(<1 x fp128>)
+
+define <2 x iXLen> @lrint_v2fp128(<2 x fp128> %x) {
+; LE-I32-LABEL: lrint_v2fp128:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; LE-I32-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; LE-I32-NEXT:    mov r8, r3
+; LE-I32-NEXT:    add r3, sp, #24
+; LE-I32-NEXT:    mov r5, r2
+; LE-I32-NEXT:    mov r6, r1
+; LE-I32-NEXT:    mov r7, r0
+; LE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    mov r4, r0
+; LE-I32-NEXT:    mov r0, r7
+; LE-I32-NEXT:    mov r1, r6
+; LE-I32-NEXT:    mov r2, r5
+; LE-I32-NEXT:    mov r3, r8
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    vmov.32 d0[0], r0
+; LE-I32-NEXT:    vmov.32 d0[1], r4
+; LE-I32-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+;
+; LE-I64-LABEL: lrint_v2fp128:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; LE-I64-NEXT:    .vsave {d8, d9}
+; LE-I64-NEXT:    vpush {d8, d9}
+; LE-I64-NEXT:    mov r8, r3
+; LE-I64-NEXT:    add r3, sp, #40
+; LE-I64-NEXT:    mov r5, r2
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    mov r7, r0
+; LE-I64-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    mov r0, r7
+; LE-I64-NEXT:    mov r1, r6
+; LE-I64-NEXT:    mov r2, r5
+; LE-I64-NEXT:    mov r3, r8
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    vmov.32 d9[1], r4
+; LE-I64-NEXT:    vmov.32 d8[1], r1
+; LE-I64-NEXT:    vorr q0, q4, q4
+; LE-I64-NEXT:    vpop {d8, d9}
+; LE-I64-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+;
+; BE-I32-LABEL: lrint_v2fp128:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; BE-I32-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; BE-I32-NEXT:    mov r8, r3
+; BE-I32-NEXT:    add r3, sp, #24
+; BE-I32-NEXT:    mov r5, r2
+; BE-I32-NEXT:    mov r6, r1
+; BE-I32-NEXT:    mov r7, r0
+; BE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    mov r4, r0
+; BE-I32-NEXT:    mov r0, r7
+; BE-I32-NEXT:    mov r1, r6
+; BE-I32-NEXT:    mov r2, r5
+; BE-I32-NEXT:    mov r3, r8
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    vmov.32 d16[0], r0
+; BE-I32-NEXT:    vmov.32 d16[1], r4
+; BE-I32-NEXT:    vrev64.32 d0, d16
+; BE-I32-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+;
+; BE-I64-LABEL: lrint_v2fp128:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, r7, r8, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, r7, r8, lr}
+; BE-I64-NEXT:    .vsave {d8}
+; BE-I64-NEXT:    vpush {d8}
+; BE-I64-NEXT:    mov r8, r3
+; BE-I64-NEXT:    add r3, sp, #32
+; BE-I64-NEXT:    mov r5, r2
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    mov r7, r0
+; BE-I64-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d8[0], r0
+; BE-I64-NEXT:    mov r0, r7
+; BE-I64-NEXT:    mov r1, r6
+; BE-I64-NEXT:    mov r2, r5
+; BE-I64-NEXT:    mov r3, r8
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    vmov.32 d16[0], r0
+; BE-I64-NEXT:    vmov.32 d8[1], r4
+; BE-I64-NEXT:    vmov.32 d16[1], r1
+; BE-I64-NEXT:    vrev64.32 d1, d8
+; BE-I64-NEXT:    vrev64.32 d0, d16
+; BE-I64-NEXT:    vpop {d8}
+; BE-I64-NEXT:    pop {r4, r5, r6, r7, r8, pc}
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2fp128(<2 x fp128> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2fp128(<2 x fp128>)
+
+define <4 x iXLen> @lrint_v4fp128(<4 x fp128> %x) {
+; LE-I32-LABEL: lrint_v4fp128:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r4, lr}
+; LE-I32-NEXT:    push {r4, lr}
+; LE-I32-NEXT:    .vsave {d8, d9}
+; LE-I32-NEXT:    vpush {d8, d9}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #60
+; LE-I32-NEXT:    ldr r12, [sp, #56]
+; LE-I32-NEXT:    vmov.32 d8[0], r0
+; LE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I32-NEXT:    mov r0, r12
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #40
+; LE-I32-NEXT:    mov r4, r0
+; LE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #28
+; LE-I32-NEXT:    ldr r12, [sp, #24]
+; LE-I32-NEXT:    vmov.32 d9[0], r0
+; LE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I32-NEXT:    mov r0, r12
+; LE-I32-NEXT:    vmov.32 d9[1], r4
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    vmov.32 d8[1], r0
+; LE-I32-NEXT:    vorr q0, q4, q4
+; LE-I32-NEXT:    vpop {d8, d9}
+; LE-I32-NEXT:    pop {r4, pc}
+;
+; LE-I64-LABEL: lrint_v4fp128:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11}
+; LE-I64-NEXT:    mov r5, r3
+; LE-I64-NEXT:    add r3, sp, #96
+; LE-I64-NEXT:    mov r7, r2
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    mov r4, r0
+; LE-I64-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    mov r9, r1
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    mov r0, r4
+; LE-I64-NEXT:    mov r1, r6
+; LE-I64-NEXT:    mov r2, r7
+; LE-I64-NEXT:    mov r3, r5
+; LE-I64-NEXT:    ldr r8, [sp, #80]
+; LE-I64-NEXT:    ldr r10, [sp, #64]
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #68
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    mov r0, r10
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #84
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    mov r0, r8
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    vmov.32 d11[1], r4
+; LE-I64-NEXT:    vmov.32 d9[1], r9
+; LE-I64-NEXT:    vmov.32 d10[1], r5
+; LE-I64-NEXT:    vmov.32 d8[1], r1
+; LE-I64-NEXT:    vorr q0, q5, q5
+; LE-I64-NEXT:    vorr q1, q4, q4
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11}
+; LE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+;
+; BE-I32-LABEL: lrint_v4fp128:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r4, lr}
+; BE-I32-NEXT:    push {r4, lr}
+; BE-I32-NEXT:    .vsave {d8, d9}
+; BE-I32-NEXT:    vpush {d8, d9}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #60
+; BE-I32-NEXT:    ldr r12, [sp, #56]
+; BE-I32-NEXT:    vmov.32 d8[0], r0
+; BE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I32-NEXT:    mov r0, r12
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #40
+; BE-I32-NEXT:    mov r4, r0
+; BE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #28
+; BE-I32-NEXT:    ldr r12, [sp, #24]
+; BE-I32-NEXT:    vmov.32 d9[0], r0
+; BE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I32-NEXT:    mov r0, r12
+; BE-I32-NEXT:    vmov.32 d9[1], r4
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    vmov.32 d8[1], r0
+; BE-I32-NEXT:    vrev64.32 q0, q4
+; BE-I32-NEXT:    vpop {d8, d9}
+; BE-I32-NEXT:    pop {r4, pc}
+;
+; BE-I64-LABEL: lrint_v4fp128:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; BE-I64-NEXT:    .vsave {d8, d9, d10}
+; BE-I64-NEXT:    vpush {d8, d9, d10}
+; BE-I64-NEXT:    mov r5, r3
+; BE-I64-NEXT:    add r3, sp, #88
+; BE-I64-NEXT:    mov r7, r2
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    mov r4, r0
+; BE-I64-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    mov r9, r1
+; BE-I64-NEXT:    vmov.32 d8[0], r0
+; BE-I64-NEXT:    mov r0, r4
+; BE-I64-NEXT:    mov r1, r6
+; BE-I64-NEXT:    mov r2, r7
+; BE-I64-NEXT:    mov r3, r5
+; BE-I64-NEXT:    ldr r8, [sp, #72]
+; BE-I64-NEXT:    ldr r10, [sp, #56]
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #60
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    vmov.32 d9[0], r0
+; BE-I64-NEXT:    mov r0, r10
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #76
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    mov r0, r8
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    vmov.32 d16[0], r0
+; BE-I64-NEXT:    vmov.32 d10[1], r4
+; BE-I64-NEXT:    vmov.32 d8[1], r9
+; BE-I64-NEXT:    vmov.32 d9[1], r5
+; BE-I64-NEXT:    vmov.32 d16[1], r1
+; BE-I64-NEXT:    vrev64.32 d1, d10
+; BE-I64-NEXT:    vrev64.32 d3, d8
+; BE-I64-NEXT:    vrev64.32 d0, d9
+; BE-I64-NEXT:    vrev64.32 d2, d16
+; BE-I64-NEXT:    vpop {d8, d9, d10}
+; BE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4fp128(<4 x fp128> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4fp128(<4 x fp128>)
+
+define <8 x iXLen> @lrint_v8fp128(<8 x fp128> %x) {
+; LE-I32-LABEL: lrint_v8fp128:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-I32-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, lr}
+; LE-I32-NEXT:    .vsave {d8, d9, d10, d11}
+; LE-I32-NEXT:    vpush {d8, d9, d10, d11}
+; LE-I32-NEXT:    mov r6, r3
+; LE-I32-NEXT:    add r3, sp, #112
+; LE-I32-NEXT:    mov r7, r2
+; LE-I32-NEXT:    mov r4, r1
+; LE-I32-NEXT:    mov r5, r0
+; LE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    vmov.32 d8[0], r0
+; LE-I32-NEXT:    mov r0, r5
+; LE-I32-NEXT:    mov r1, r4
+; LE-I32-NEXT:    mov r2, r7
+; LE-I32-NEXT:    mov r3, r6
+; LE-I32-NEXT:    ldr r8, [sp, #160]
+; LE-I32-NEXT:    ldr r9, [sp, #64]
+; LE-I32-NEXT:    ldr r10, [sp, #80]
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #84
+; LE-I32-NEXT:    vmov.32 d10[0], r0
+; LE-I32-NEXT:    mov r0, r10
+; LE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    ldr r6, [sp, #96]
+; LE-I32-NEXT:    vmov.32 d11[0], r0
+; LE-I32-NEXT:    ldr r1, [sp, #100]
+; LE-I32-NEXT:    ldr r2, [sp, #104]
+; LE-I32-NEXT:    ldr r3, [sp, #108]
+; LE-I32-NEXT:    mov r0, r6
+; LE-I32-NEXT:    ldr r4, [sp, #68]
+; LE-I32-NEXT:    ldr r5, [sp, #72]
+; LE-I32-NEXT:    ldr r10, [sp, #164]
+; LE-I32-NEXT:    ldr r7, [sp, #168]
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    ldr r3, [sp, #76]
+; LE-I32-NEXT:    vmov.32 d11[1], r0
+; LE-I32-NEXT:    mov r0, r9
+; LE-I32-NEXT:    mov r1, r4
+; LE-I32-NEXT:    mov r2, r5
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    ldr r3, [sp, #172]
+; LE-I32-NEXT:    vmov.32 d10[1], r0
+; LE-I32-NEXT:    mov r0, r8
+; LE-I32-NEXT:    mov r1, r10
+; LE-I32-NEXT:    mov r2, r7
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #144
+; LE-I32-NEXT:    mov r4, r0
+; LE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #132
+; LE-I32-NEXT:    ldr r7, [sp, #128]
+; LE-I32-NEXT:    vmov.32 d9[0], r0
+; LE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I32-NEXT:    mov r0, r7
+; LE-I32-NEXT:    vmov.32 d9[1], r4
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    vmov.32 d8[1], r0
+; LE-I32-NEXT:    vorr q0, q5, q5
+; LE-I32-NEXT:    vorr q1, q4, q4
+; LE-I32-NEXT:    vpop {d8, d9, d10, d11}
+; LE-I32-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, pc}
+;
+; LE-I64-LABEL: lrint_v8fp128:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I64-NEXT:    .pad #4
+; LE-I64-NEXT:    sub sp, sp, #4
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    .pad #8
+; LE-I64-NEXT:    sub sp, sp, #8
+; LE-I64-NEXT:    mov r11, r3
+; LE-I64-NEXT:    add r3, sp, #208
+; LE-I64-NEXT:    mov r10, r2
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    mov r5, r0
+; LE-I64-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r7, sp, #164
+; LE-I64-NEXT:    ldr r6, [sp, #160]
+; LE-I64-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    ldm r7, {r1, r2, r3, r7}
+; LE-I64-NEXT:    mov r0, r6
+; LE-I64-NEXT:    ldr r8, [sp, #128]
+; LE-I64-NEXT:    ldr r9, [sp, #144]
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #180
+; LE-I64-NEXT:    str r1, [sp] @ 4-byte Spill
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    mov r0, r7
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #132
+; LE-I64-NEXT:    mov r7, r1
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    mov r0, r8
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #148
+; LE-I64-NEXT:    mov r8, r1
+; LE-I64-NEXT:    vmov.32 d12[0], r0
+; LE-I64-NEXT:    mov r0, r9
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    mov r9, r1
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    mov r0, r5
+; LE-I64-NEXT:    mov r1, r4
+; LE-I64-NEXT:    mov r2, r10
+; LE-I64-NEXT:    mov r3, r11
+; LE-I64-NEXT:    ldr r6, [sp, #112]
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #116
+; LE-I64-NEXT:    mov r4, r1
+; LE-I64-NEXT:    vmov.32 d14[0], r0
+; LE-I64-NEXT:    mov r0, r6
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #196
+; LE-I64-NEXT:    vmov.32 d15[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #192]
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    ldr r0, [sp] @ 4-byte Reload
+; LE-I64-NEXT:    vmov.32 d11[1], r7
+; LE-I64-NEXT:    vmov.32 d10[1], r0
+; LE-I64-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; LE-I64-NEXT:    vmov.32 d15[1], r5
+; LE-I64-NEXT:    vorr q2, q5, q5
+; LE-I64-NEXT:    vmov.32 d13[1], r9
+; LE-I64-NEXT:    vmov.32 d9[1], r0
+; LE-I64-NEXT:    vmov.32 d14[1], r4
+; LE-I64-NEXT:    vmov.32 d12[1], r8
+; LE-I64-NEXT:    vorr q0, q7, q7
+; LE-I64-NEXT:    vmov.32 d8[1], r1
+; LE-I64-NEXT:    vorr q1, q6, q6
+; LE-I64-NEXT:    vorr q3, q4, q4
+; LE-I64-NEXT:    add sp, sp, #8
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    add sp, sp, #4
+; LE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-I32-LABEL: lrint_v8fp128:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I32-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I32-NEXT:    .pad #4
+; BE-I32-NEXT:    sub sp, sp, #4
+; BE-I32-NEXT:    .vsave {d8, d9, d10, d11}
+; BE-I32-NEXT:    vpush {d8, d9, d10, d11}
+; BE-I32-NEXT:    .pad #8
+; BE-I32-NEXT:    sub sp, sp, #8
+; BE-I32-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; BE-I32-NEXT:    add r3, sp, #128
+; BE-I32-NEXT:    mov r11, r2
+; BE-I32-NEXT:    mov r6, r1
+; BE-I32-NEXT:    mov r7, r0
+; BE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #100
+; BE-I32-NEXT:    ldr r5, [sp, #96]
+; BE-I32-NEXT:    vmov.32 d8[0], r0
+; BE-I32-NEXT:    ldr r4, [sp, #160]
+; BE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I32-NEXT:    mov r0, r5
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #164
+; BE-I32-NEXT:    vmov.32 d11[0], r0
+; BE-I32-NEXT:    mov r0, r4
+; BE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r4, [sp, #176]
+; BE-I32-NEXT:    vmov.32 d9[0], r0
+; BE-I32-NEXT:    ldr r1, [sp, #180]
+; BE-I32-NEXT:    ldr r2, [sp, #184]
+; BE-I32-NEXT:    ldr r3, [sp, #188]
+; BE-I32-NEXT:    mov r0, r4
+; BE-I32-NEXT:    ldr r5, [sp, #116]
+; BE-I32-NEXT:    ldr r8, [sp, #120]
+; BE-I32-NEXT:    ldr r10, [sp, #84]
+; BE-I32-NEXT:    ldr r9, [sp, #88]
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    vmov.32 d9[1], r0
+; BE-I32-NEXT:    ldr r3, [sp, #124]
+; BE-I32-NEXT:    ldr r0, [sp, #112]
+; BE-I32-NEXT:    mov r1, r5
+; BE-I32-NEXT:    mov r2, r8
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    vmov.32 d11[1], r0
+; BE-I32-NEXT:    ldr r3, [sp, #92]
+; BE-I32-NEXT:    ldr r0, [sp, #80]
+; BE-I32-NEXT:    mov r1, r10
+; BE-I32-NEXT:    mov r2, r9
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; BE-I32-NEXT:    mov r4, r0
+; BE-I32-NEXT:    mov r0, r7
+; BE-I32-NEXT:    mov r1, r6
+; BE-I32-NEXT:    mov r2, r11
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #148
+; BE-I32-NEXT:    ldr r7, [sp, #144]
+; BE-I32-NEXT:    vmov.32 d10[0], r0
+; BE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I32-NEXT:    mov r0, r7
+; BE-I32-NEXT:    vmov.32 d10[1], r4
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    vmov.32 d8[1], r0
+; BE-I32-NEXT:    vrev64.32 q0, q5
+; BE-I32-NEXT:    vrev64.32 q1, q4
+; BE-I32-NEXT:    add sp, sp, #8
+; BE-I32-NEXT:    vpop {d8, d9, d10, d11}
+; BE-I32-NEXT:    add sp, sp, #4
+; BE-I32-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-I64-LABEL: lrint_v8fp128:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I64-NEXT:    .pad #4
+; BE-I64-NEXT:    sub sp, sp, #4
+; BE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14}
+; BE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14}
+; BE-I64-NEXT:    .pad #16
+; BE-I64-NEXT:    sub sp, sp, #16
+; BE-I64-NEXT:    str r3, [sp, #4] @ 4-byte Spill
+; BE-I64-NEXT:    add r3, sp, #208
+; BE-I64-NEXT:    mov r11, r2
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    mov r5, r0
+; BE-I64-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    ldr r7, [sp, #176]
+; BE-I64-NEXT:    add r3, sp, #180
+; BE-I64-NEXT:    str r1, [sp, #12] @ 4-byte Spill
+; BE-I64-NEXT:    vmov.32 d8[0], r0
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    mov r0, r7
+; BE-I64-NEXT:    ldr r6, [sp, #128]
+; BE-I64-NEXT:    ldr r8, [sp, #144]
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #132
+; BE-I64-NEXT:    str r1, [sp, #8] @ 4-byte Spill
+; BE-I64-NEXT:    vmov.32 d9[0], r0
+; BE-I64-NEXT:    mov r0, r6
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #148
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    mov r0, r8
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #160
+; BE-I64-NEXT:    mov r9, r0
+; BE-I64-NEXT:    mov r7, r1
+; BE-I64-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    ldr r3, [sp, #4] @ 4-byte Reload
+; BE-I64-NEXT:    mov r8, r1
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    mov r0, r5
+; BE-I64-NEXT:    mov r1, r4
+; BE-I64-NEXT:    mov r2, r11
+; BE-I64-NEXT:    ldr r10, [sp, #112]
+; BE-I64-NEXT:    vmov.32 d12[0], r9
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #116
+; BE-I64-NEXT:    mov r4, r1
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    mov r0, r10
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #196
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #192]
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    vmov.32 d16[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; BE-I64-NEXT:    vmov.32 d14[1], r5
+; BE-I64-NEXT:    vmov.32 d9[1], r0
+; BE-I64-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
+; BE-I64-NEXT:    vmov.32 d12[1], r7
+; BE-I64-NEXT:    vmov.32 d8[1], r0
+; BE-I64-NEXT:    vmov.32 d13[1], r4
+; BE-I64-NEXT:    vmov.32 d10[1], r6
+; BE-I64-NEXT:    vmov.32 d11[1], r8
+; BE-I64-NEXT:    vmov.32 d16[1], r1
+; BE-I64-NEXT:    vrev64.32 d1, d14
+; BE-I64-NEXT:    vrev64.32 d3, d12
+; BE-I64-NEXT:    vrev64.32 d5, d9
+; BE-I64-NEXT:    vrev64.32 d7, d8
+; BE-I64-NEXT:    vrev64.32 d0, d13
+; BE-I64-NEXT:    vrev64.32 d2, d10
+; BE-I64-NEXT:    vrev64.32 d4, d11
+; BE-I64-NEXT:    vrev64.32 d6, d16
+; BE-I64-NEXT:    add sp, sp, #16
+; BE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14}
+; BE-I64-NEXT:    add sp, sp, #4
+; BE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8fp128(<8 x fp128> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8fp128(<8 x fp128>)
+
+define <16 x iXLen> @lrint_v16fp128(<16 x fp128> %x) {
+; LE-I32-LABEL: lrint_v16fp128:
+; LE-I32:       @ %bb.0:
+; LE-I32-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I32-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I32-NEXT:    .pad #4
+; LE-I32-NEXT:    sub sp, sp, #4
+; LE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    mov r8, r3
+; LE-I32-NEXT:    add r3, sp, #280
+; LE-I32-NEXT:    mov r9, r2
+; LE-I32-NEXT:    mov r10, r1
+; LE-I32-NEXT:    mov r6, r0
+; LE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    ldr r4, [sp, #216]
+; LE-I32-NEXT:    vmov.32 d8[0], r0
+; LE-I32-NEXT:    ldr r1, [sp, #220]
+; LE-I32-NEXT:    ldr r2, [sp, #224]
+; LE-I32-NEXT:    ldr r3, [sp, #228]
+; LE-I32-NEXT:    mov r0, r4
+; LE-I32-NEXT:    ldr r7, [sp, #152]
+; LE-I32-NEXT:    ldr r11, [sp, #104]
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #156
+; LE-I32-NEXT:    vmov.32 d10[0], r0
+; LE-I32-NEXT:    mov r0, r7
+; LE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    ldr r7, [sp, #184]
+; LE-I32-NEXT:    vmov.32 d12[0], r0
+; LE-I32-NEXT:    ldr r1, [sp, #188]
+; LE-I32-NEXT:    ldr r2, [sp, #192]
+; LE-I32-NEXT:    ldr r3, [sp, #196]
+; LE-I32-NEXT:    mov r0, r7
+; LE-I32-NEXT:    ldr r4, [sp, #120]
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #124
+; LE-I32-NEXT:    vmov.32 d13[0], r0
+; LE-I32-NEXT:    mov r0, r4
+; LE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    ldr r5, [sp, #136]
+; LE-I32-NEXT:    vmov.32 d15[0], r0
+; LE-I32-NEXT:    ldr r1, [sp, #140]
+; LE-I32-NEXT:    ldr r2, [sp, #144]
+; LE-I32-NEXT:    ldr r3, [sp, #148]
+; LE-I32-NEXT:    mov r0, r5
+; LE-I32-NEXT:    ldr r4, [sp, #108]
+; LE-I32-NEXT:    ldr r7, [sp, #112]
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    ldr r3, [sp, #116]
+; LE-I32-NEXT:    vmov.32 d15[1], r0
+; LE-I32-NEXT:    mov r0, r11
+; LE-I32-NEXT:    mov r1, r4
+; LE-I32-NEXT:    mov r2, r7
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    mov r4, r0
+; LE-I32-NEXT:    mov r0, r6
+; LE-I32-NEXT:    mov r1, r10
+; LE-I32-NEXT:    mov r2, r9
+; LE-I32-NEXT:    mov r3, r8
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    ldr r7, [sp, #200]
+; LE-I32-NEXT:    vmov.32 d14[0], r0
+; LE-I32-NEXT:    ldr r1, [sp, #204]
+; LE-I32-NEXT:    ldr r2, [sp, #208]
+; LE-I32-NEXT:    ldr r3, [sp, #212]
+; LE-I32-NEXT:    mov r0, r7
+; LE-I32-NEXT:    ldr r5, [sp, #172]
+; LE-I32-NEXT:    vmov.32 d14[1], r4
+; LE-I32-NEXT:    ldr r6, [sp, #176]
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    vmov.32 d13[1], r0
+; LE-I32-NEXT:    ldr r3, [sp, #180]
+; LE-I32-NEXT:    ldr r0, [sp, #168]
+; LE-I32-NEXT:    mov r1, r5
+; LE-I32-NEXT:    mov r2, r6
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #248
+; LE-I32-NEXT:    mov r5, r0
+; LE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    ldr r4, [sp, #264]
+; LE-I32-NEXT:    vmov.32 d11[0], r0
+; LE-I32-NEXT:    ldr r1, [sp, #268]
+; LE-I32-NEXT:    ldr r2, [sp, #272]
+; LE-I32-NEXT:    vmov.32 d12[1], r5
+; LE-I32-NEXT:    ldr r3, [sp, #276]
+; LE-I32-NEXT:    mov r0, r4
+; LE-I32-NEXT:    ldr r6, [sp, #236]
+; LE-I32-NEXT:    ldr r7, [sp, #240]
+; LE-I32-NEXT:    ldr r8, [sp, #332]
+; LE-I32-NEXT:    ldr r5, [sp, #336]
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    vmov.32 d11[1], r0
+; LE-I32-NEXT:    ldr r3, [sp, #244]
+; LE-I32-NEXT:    ldr r0, [sp, #232]
+; LE-I32-NEXT:    mov r1, r6
+; LE-I32-NEXT:    mov r2, r7
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    vmov.32 d10[1], r0
+; LE-I32-NEXT:    ldr r3, [sp, #340]
+; LE-I32-NEXT:    ldr r0, [sp, #328]
+; LE-I32-NEXT:    mov r1, r8
+; LE-I32-NEXT:    mov r2, r5
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #312
+; LE-I32-NEXT:    mov r4, r0
+; LE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    add r3, sp, #300
+; LE-I32-NEXT:    ldr r7, [sp, #296]
+; LE-I32-NEXT:    vmov.32 d9[0], r0
+; LE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I32-NEXT:    mov r0, r7
+; LE-I32-NEXT:    vmov.32 d9[1], r4
+; LE-I32-NEXT:    bl lrintl
+; LE-I32-NEXT:    vmov.32 d8[1], r0
+; LE-I32-NEXT:    vorr q0, q7, q7
+; LE-I32-NEXT:    vorr q1, q6, q6
+; LE-I32-NEXT:    vorr q2, q5, q5
+; LE-I32-NEXT:    vorr q3, q4, q4
+; LE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I32-NEXT:    add sp, sp, #4
+; LE-I32-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; LE-I64-LABEL: lrint_v16fp128:
+; LE-I64:       @ %bb.0:
+; LE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; LE-I64-NEXT:    .pad #4
+; LE-I64-NEXT:    sub sp, sp, #4
+; LE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    .pad #72
+; LE-I64-NEXT:    sub sp, sp, #72
+; LE-I64-NEXT:    mov r6, r3
+; LE-I64-NEXT:    add r3, sp, #408
+; LE-I64-NEXT:    mov r7, r2
+; LE-I64-NEXT:    mov r4, r0
+; LE-I64-NEXT:    ldm r3, {r0, r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r5, sp, #176
+; LE-I64-NEXT:    mov r10, r1
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    mov r0, r7
+; LE-I64-NEXT:    ldm r5, {r2, r3, r5}
+; LE-I64-NEXT:    mov r1, r6
+; LE-I64-NEXT:    ldr r8, [sp, #232]
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #188
+; LE-I64-NEXT:    mov r9, r1
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    mov r0, r5
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #236
+; LE-I64-NEXT:    mov r11, r1
+; LE-I64-NEXT:    vmov.32 d9[0], r0
+; LE-I64-NEXT:    mov r0, r8
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #252
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #248]
+; LE-I64-NEXT:    mov r8, r1
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #268
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #264]
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #284
+; LE-I64-NEXT:    vmov.32 d14[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #280]
+; LE-I64-NEXT:    mov r7, r1
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #316
+; LE-I64-NEXT:    vmov.32 d15[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #312]
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    vmov.32 d15[1], r5
+; LE-I64-NEXT:    add lr, sp, #56
+; LE-I64-NEXT:    ldr r5, [sp, #300]
+; LE-I64-NEXT:    vmov.32 d14[1], r7
+; LE-I64-NEXT:    ldr r2, [sp, #304]
+; LE-I64-NEXT:    ldr r3, [sp, #308]
+; LE-I64-NEXT:    vmov.32 d11[1], r6
+; LE-I64-NEXT:    ldr r6, [sp, #200]
+; LE-I64-NEXT:    ldr r7, [sp, #204]
+; LE-I64-NEXT:    vmov.32 d10[1], r8
+; LE-I64-NEXT:    ldr r8, [sp, #344]
+; LE-I64-NEXT:    vmov.32 d9[1], r11
+; LE-I64-NEXT:    ldr r11, [sp, #216]
+; LE-I64-NEXT:    vstmia lr, {d14, d15} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #40
+; LE-I64-NEXT:    vmov.32 d17[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #296]
+; LE-I64-NEXT:    vmov.32 d8[1], r9
+; LE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    vorr q5, q8, q8
+; LE-I64-NEXT:    vstmia lr, {d8, d9} @ 16-byte Spill
+; LE-I64-NEXT:    vorr q4, q6, q6
+; LE-I64-NEXT:    vmov.32 d11[1], r1
+; LE-I64-NEXT:    mov r1, r5
+; LE-I64-NEXT:    vmov.32 d9[1], r10
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    ldr r2, [sp, #208]
+; LE-I64-NEXT:    ldr r3, [sp, #212]
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    mov r9, r1
+; LE-I64-NEXT:    mov r0, r6
+; LE-I64-NEXT:    mov r1, r7
+; LE-I64-NEXT:    vstmia lr, {d10, d11} @ 16-byte Spill
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #220
+; LE-I64-NEXT:    mov r10, r1
+; LE-I64-NEXT:    vmov.32 d10[0], r0
+; LE-I64-NEXT:    mov r0, r11
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #348
+; LE-I64-NEXT:    mov r11, r1
+; LE-I64-NEXT:    vmov.32 d11[0], r0
+; LE-I64-NEXT:    mov r0, r8
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #364
+; LE-I64-NEXT:    vmov.32 d13[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #360]
+; LE-I64-NEXT:    mov r8, r1
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #380
+; LE-I64-NEXT:    vmov.32 d14[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #376]
+; LE-I64-NEXT:    mov r5, r1
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #396
+; LE-I64-NEXT:    vmov.32 d15[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #392]
+; LE-I64-NEXT:    mov r6, r1
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add r3, sp, #332
+; LE-I64-NEXT:    vmov.32 d8[0], r0
+; LE-I64-NEXT:    ldr r0, [sp, #328]
+; LE-I64-NEXT:    mov r7, r1
+; LE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; LE-I64-NEXT:    bl lrintl
+; LE-I64-NEXT:    add lr, sp, #8
+; LE-I64-NEXT:    vmov.32 d12[0], r0
+; LE-I64-NEXT:    add r0, r4, #64
+; LE-I64-NEXT:    vldmia lr, {d18, d19} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #24
+; LE-I64-NEXT:    vmov.32 d13[1], r8
+; LE-I64-NEXT:    vmov.32 d18[1], r9
+; LE-I64-NEXT:    vmov.32 d15[1], r6
+; LE-I64-NEXT:    vmov.32 d12[1], r1
+; LE-I64-NEXT:    vmov.32 d14[1], r5
+; LE-I64-NEXT:    vst1.64 {d18, d19}, [r0:128]!
+; LE-I64-NEXT:    vst1.64 {d12, d13}, [r0:128]!
+; LE-I64-NEXT:    vmov.32 d8[1], r7
+; LE-I64-NEXT:    vst1.64 {d14, d15}, [r0:128]!
+; LE-I64-NEXT:    vst1.64 {d8, d9}, [r0:128]
+; LE-I64-NEXT:    vmov.32 d11[1], r11
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #40
+; LE-I64-NEXT:    vmov.32 d10[1], r10
+; LE-I64-NEXT:    vst1.64 {d16, d17}, [r4:128]!
+; LE-I64-NEXT:    vst1.64 {d10, d11}, [r4:128]!
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    add lr, sp, #56
+; LE-I64-NEXT:    vst1.64 {d16, d17}, [r4:128]!
+; LE-I64-NEXT:    vldmia lr, {d16, d17} @ 16-byte Reload
+; LE-I64-NEXT:    vst1.64 {d16, d17}, [r4:128]
+; LE-I64-NEXT:    add sp, sp, #72
+; LE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; LE-I64-NEXT:    add sp, sp, #4
+; LE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-I32-LABEL: lrint_v16fp128:
+; BE-I32:       @ %bb.0:
+; BE-I32-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I32-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I32-NEXT:    .pad #4
+; BE-I32-NEXT:    sub sp, sp, #4
+; BE-I32-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    .pad #16
+; BE-I32-NEXT:    sub sp, sp, #16
+; BE-I32-NEXT:    stm sp, {r0, r1, r2, r3} @ 16-byte Folded Spill
+; BE-I32-NEXT:    add r3, sp, #264
+; BE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #332
+; BE-I32-NEXT:    ldr r7, [sp, #328]
+; BE-I32-NEXT:    vmov.32 d9[0], r0
+; BE-I32-NEXT:    ldr r10, [sp, #280]
+; BE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I32-NEXT:    mov r0, r7
+; BE-I32-NEXT:    ldr r8, [sp, #168]
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r5, [sp, #344]
+; BE-I32-NEXT:    vmov.32 d11[0], r0
+; BE-I32-NEXT:    ldr r1, [sp, #348]
+; BE-I32-NEXT:    ldr r2, [sp, #352]
+; BE-I32-NEXT:    ldr r3, [sp, #356]
+; BE-I32-NEXT:    mov r0, r5
+; BE-I32-NEXT:    ldr r7, [sp, #284]
+; BE-I32-NEXT:    ldr r4, [sp, #288]
+; BE-I32-NEXT:    ldr r6, [sp, #172]
+; BE-I32-NEXT:    ldr r9, [sp, #176]
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r3, [sp, #292]
+; BE-I32-NEXT:    vmov.32 d11[1], r0
+; BE-I32-NEXT:    mov r0, r10
+; BE-I32-NEXT:    mov r1, r7
+; BE-I32-NEXT:    mov r2, r4
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r3, [sp, #180]
+; BE-I32-NEXT:    vmov.32 d9[1], r0
+; BE-I32-NEXT:    mov r0, r8
+; BE-I32-NEXT:    mov r1, r6
+; BE-I32-NEXT:    mov r2, r9
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #232
+; BE-I32-NEXT:    mov r4, r0
+; BE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #136
+; BE-I32-NEXT:    mov r6, r0
+; BE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r5, [sp, #296]
+; BE-I32-NEXT:    vmov.32 d13[0], r0
+; BE-I32-NEXT:    ldr r1, [sp, #300]
+; BE-I32-NEXT:    ldr r2, [sp, #304]
+; BE-I32-NEXT:    ldr r3, [sp, #308]
+; BE-I32-NEXT:    mov r0, r5
+; BE-I32-NEXT:    ldr r10, [sp, #216]
+; BE-I32-NEXT:    ldr r8, [sp, #220]
+; BE-I32-NEXT:    ldr r9, [sp, #152]
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r7, [sp, #248]
+; BE-I32-NEXT:    vmov.32 d10[0], r0
+; BE-I32-NEXT:    ldr r1, [sp, #252]
+; BE-I32-NEXT:    ldr r2, [sp, #256]
+; BE-I32-NEXT:    vmov.32 d8[0], r6
+; BE-I32-NEXT:    ldr r3, [sp, #260]
+; BE-I32-NEXT:    mov r0, r7
+; BE-I32-NEXT:    ldr r5, [sp, #224]
+; BE-I32-NEXT:    ldr r11, [sp, #120]
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r3, [sp, #228]
+; BE-I32-NEXT:    vmov.32 d8[1], r0
+; BE-I32-NEXT:    mov r0, r10
+; BE-I32-NEXT:    mov r1, r8
+; BE-I32-NEXT:    mov r2, r5
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #200
+; BE-I32-NEXT:    mov r5, r0
+; BE-I32-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    vmov.32 d15[0], r0
+; BE-I32-NEXT:    ldr r0, [sp, #184]
+; BE-I32-NEXT:    ldr r1, [sp, #188]
+; BE-I32-NEXT:    ldr r2, [sp, #192]
+; BE-I32-NEXT:    vmov.32 d14[0], r4
+; BE-I32-NEXT:    ldr r3, [sp, #196]
+; BE-I32-NEXT:    vmov.32 d15[1], r5
+; BE-I32-NEXT:    ldr r7, [sp, #156]
+; BE-I32-NEXT:    ldr r6, [sp, #160]
+; BE-I32-NEXT:    ldr r4, [sp, #124]
+; BE-I32-NEXT:    ldr r5, [sp, #128]
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r3, [sp, #164]
+; BE-I32-NEXT:    vmov.32 d14[1], r0
+; BE-I32-NEXT:    mov r0, r9
+; BE-I32-NEXT:    mov r1, r7
+; BE-I32-NEXT:    mov r2, r6
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    ldr r3, [sp, #132]
+; BE-I32-NEXT:    vmov.32 d13[1], r0
+; BE-I32-NEXT:    mov r0, r11
+; BE-I32-NEXT:    mov r1, r4
+; BE-I32-NEXT:    mov r2, r5
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    mov r4, r0
+; BE-I32-NEXT:    ldm sp, {r0, r1, r2, r3} @ 16-byte Folded Reload
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    add r3, sp, #316
+; BE-I32-NEXT:    ldr r7, [sp, #312]
+; BE-I32-NEXT:    vmov.32 d12[0], r0
+; BE-I32-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I32-NEXT:    mov r0, r7
+; BE-I32-NEXT:    vmov.32 d12[1], r4
+; BE-I32-NEXT:    bl lrintl
+; BE-I32-NEXT:    vmov.32 d10[1], r0
+; BE-I32-NEXT:    vrev64.32 q0, q6
+; BE-I32-NEXT:    vrev64.32 q1, q7
+; BE-I32-NEXT:    vrev64.32 q2, q4
+; BE-I32-NEXT:    vrev64.32 q3, q5
+; BE-I32-NEXT:    add sp, sp, #16
+; BE-I32-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I32-NEXT:    add sp, sp, #4
+; BE-I32-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+;
+; BE-I64-LABEL: lrint_v16fp128:
+; BE-I64:       @ %bb.0:
+; BE-I64-NEXT:    .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I64-NEXT:    push {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; BE-I64-NEXT:    .pad #4
+; BE-I64-NEXT:    sub sp, sp, #4
+; BE-I64-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    .pad #56
+; BE-I64-NEXT:    sub sp, sp, #56
+; BE-I64-NEXT:    mov r5, r3
+; BE-I64-NEXT:    add r3, sp, #376
+; BE-I64-NEXT:    mov r6, r2
+; BE-I64-NEXT:    mov r4, r0
+; BE-I64-NEXT:    ldm r3, {r0, r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    ldr r7, [sp, #392]
+; BE-I64-NEXT:    add r3, sp, #396
+; BE-I64-NEXT:    mov r9, r1
+; BE-I64-NEXT:    vmov.32 d8[0], r0
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    mov r0, r7
+; BE-I64-NEXT:    ldr r11, [sp, #168]
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    ldr r2, [sp, #160]
+; BE-I64-NEXT:    mov r10, r1
+; BE-I64-NEXT:    ldr r3, [sp, #164]
+; BE-I64-NEXT:    vmov.32 d9[0], r0
+; BE-I64-NEXT:    mov r0, r6
+; BE-I64-NEXT:    mov r1, r5
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #172
+; BE-I64-NEXT:    mov r8, r1
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    mov r0, r11
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #220
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #216]
+; BE-I64-NEXT:    mov r11, r1
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #236
+; BE-I64-NEXT:    vmov.32 d12[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #232]
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #252
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #248]
+; BE-I64-NEXT:    mov r7, r1
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #268
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #264]
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    vmov.32 d15[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #280]
+; BE-I64-NEXT:    ldr r2, [sp, #288]
+; BE-I64-NEXT:    vmov.32 d13[1], r7
+; BE-I64-NEXT:    ldr r7, [sp, #284]
+; BE-I64-NEXT:    ldr r3, [sp, #292]
+; BE-I64-NEXT:    vmov.32 d14[1], r5
+; BE-I64-NEXT:    ldr r5, [sp, #328]
+; BE-I64-NEXT:    vmov.32 d12[1], r6
+; BE-I64-NEXT:    ldr r6, [sp, #300]
+; BE-I64-NEXT:    vmov.32 d10[1], r8
+; BE-I64-NEXT:    ldr r8, [sp, #184]
+; BE-I64-NEXT:    vmov.32 d11[1], r11
+; BE-I64-NEXT:    vmov.32 d9[1], r10
+; BE-I64-NEXT:    vmov.32 d8[1], r9
+; BE-I64-NEXT:    vmov.32 d15[1], r1
+; BE-I64-NEXT:    mov r1, r7
+; BE-I64-NEXT:    vstr d14, [sp, #48] @ 8-byte Spill
+; BE-I64-NEXT:    vstr d13, [sp, #40] @ 8-byte Spill
+; BE-I64-NEXT:    vstr d12, [sp, #32] @ 8-byte Spill
+; BE-I64-NEXT:    vstr d11, [sp, #24] @ 8-byte Spill
+; BE-I64-NEXT:    vstr d10, [sp, #16] @ 8-byte Spill
+; BE-I64-NEXT:    vstr d9, [sp, #8] @ 8-byte Spill
+; BE-I64-NEXT:    vstr d8, [sp] @ 8-byte Spill
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    mov r10, r1
+; BE-I64-NEXT:    ldr r1, [sp, #296]
+; BE-I64-NEXT:    ldr r2, [sp, #304]
+; BE-I64-NEXT:    vmov.32 d8[0], r0
+; BE-I64-NEXT:    ldr r3, [sp, #308]
+; BE-I64-NEXT:    mov r0, r1
+; BE-I64-NEXT:    mov r1, r6
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #332
+; BE-I64-NEXT:    mov r11, r1
+; BE-I64-NEXT:    vmov.32 d9[0], r0
+; BE-I64-NEXT:    mov r0, r5
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #188
+; BE-I64-NEXT:    mov r7, r1
+; BE-I64-NEXT:    vmov.32 d10[0], r0
+; BE-I64-NEXT:    mov r0, r8
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #204
+; BE-I64-NEXT:    vmov.32 d11[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #200]
+; BE-I64-NEXT:    mov r8, r1
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #348
+; BE-I64-NEXT:    vmov.32 d12[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #344]
+; BE-I64-NEXT:    mov r5, r1
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #364
+; BE-I64-NEXT:    vmov.32 d13[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #360]
+; BE-I64-NEXT:    mov r9, r1
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    add r3, sp, #316
+; BE-I64-NEXT:    vmov.32 d14[0], r0
+; BE-I64-NEXT:    ldr r0, [sp, #312]
+; BE-I64-NEXT:    mov r6, r1
+; BE-I64-NEXT:    ldm r3, {r1, r2, r3}
+; BE-I64-NEXT:    bl lrintl
+; BE-I64-NEXT:    vldr d18, [sp, #48] @ 8-byte Reload
+; BE-I64-NEXT:    vrev64.32 d17, d15
+; BE-I64-NEXT:    vrev64.32 d16, d18
+; BE-I64-NEXT:    vldr d18, [sp, #40] @ 8-byte Reload
+; BE-I64-NEXT:    vmov.32 d24[0], r0
+; BE-I64-NEXT:    add r0, r4, #64
+; BE-I64-NEXT:    vldr d20, [sp, #32] @ 8-byte Reload
+; BE-I64-NEXT:    vrev64.32 d19, d18
+; BE-I64-NEXT:    vmov.32 d9[1], r11
+; BE-I64-NEXT:    vmov.32 d10[1], r7
+; BE-I64-NEXT:    vrev64.32 d18, d20
+; BE-I64-NEXT:    vldr d20, [sp, #24] @ 8-byte Reload
+; BE-I64-NEXT:    vmov.32 d8[1], r10
+; BE-I64-NEXT:    vmov.32 d14[1], r6
+; BE-I64-NEXT:    vmov.32 d24[1], r1
+; BE-I64-NEXT:    vldr d22, [sp, #16] @ 8-byte Reload
+; BE-I64-NEXT:    vrev64.32 d21, d20
+; BE-I64-NEXT:    vrev64.32 d1, d9
+; BE-I64-NEXT:    vmov.32 d13[1], r9
+; BE-I64-NEXT:    vrev64.32 d31, d10
+; BE-I64-NEXT:    vrev64.32 d20, d22
+; BE-I64-NEXT:    vldr d22, [sp, #8] @ 8-byte Reload
+; BE-I64-NEXT:    vrev64.32 d0, d8
+; BE-I64-NEXT:    vrev64.32 d29, d14
+; BE-I64-NEXT:    vmov.32 d12[1], r5
+; BE-I64-NEXT:    vrev64.32 d30, d24
+; BE-I64-NEXT:    vrev64.32 d27, d22
+; BE-I64-NEXT:    vldr d22, [sp] @ 8-byte Reload
+; BE-I64-NEXT:    vst1.64 {d0, d1}, [r0:128]!
+; BE-I64-NEXT:    vmov.32 d11[1], r8
+; BE-I64-NEXT:    vrev64.32 d28, d13
+; BE-I64-NEXT:    vst1.64 {d30, d31}, [r0:128]!
+; BE-I64-NEXT:    vrev64.32 d26, d22
+; BE-I64-NEXT:    vrev64.32 d23, d12
+; BE-I64-NEXT:    vst1.64 {d28, d29}, [r0:128]!
+; BE-I64-NEXT:    vrev64.32 d22, d11
+; BE-I64-NEXT:    vst1.64 {d26, d27}, [r0:128]
+; BE-I64-NEXT:    vst1.64 {d20, d21}, [r4:128]!
+; BE-I64-NEXT:    vst1.64 {d22, d23}, [r4:128]!
+; BE-I64-NEXT:    vst1.64 {d18, d19}, [r4:128]!
+; BE-I64-NEXT:    vst1.64 {d16, d17}, [r4:128]
+; BE-I64-NEXT:    add sp, sp, #56
+; BE-I64-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; BE-I64-NEXT:    add sp, sp, #4
+; BE-I64-NEXT:    pop {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+  %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16fp128(<16 x fp128> %x)
+  ret <16 x iXLen> %a
+}
+declare <16 x iXLen> @llvm.lrint.v16iXLen.v16fp128(<16 x fp128>)
diff --git a/llvm/test/CodeGen/ARM/vtrn.ll b/llvm/test/CodeGen/ARM/vtrn.ll
index 136fec3ac3167..63774694f8a9a 100644
--- a/llvm/test/CodeGen/ARM/vtrn.ll
+++ b/llvm/test/CodeGen/ARM/vtrn.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=arm-eabi -mattr=+neon %s -o - | FileCheck %s
 
 define <8 x i8> @vtrni8(ptr %A, ptr %B) nounwind {
@@ -20,11 +21,11 @@ define <8 x i8> @vtrni8(ptr %A, ptr %B) nounwind {
 define <16 x i8> @vtrni8_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vtrni8_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr [[LDR1:d[0-9]+]], [r1]
-; CHECK-NEXT:    vldr [[LDR0:d[0-9]+]], [r0]
-; CHECK-NEXT:    vtrn.8 [[LDR0]], [[LDR1]]
-; CHECK-NEXT:    vmov r0, r1, [[LDR0]]
-; CHECK-NEXT:    vmov r2, r3, [[LDR1]]
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.8 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <8 x i8>, ptr %A
   %tmp2 = load <8 x i8>, ptr %B
@@ -52,11 +53,11 @@ define <4 x i16> @vtrni16(ptr %A, ptr %B) nounwind {
 define <8 x i16> @vtrni16_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vtrni16_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr [[LDR1:d[0-9]+]], [r1]
-; CHECK-NEXT:    vldr [[LDR0:d[0-9]+]], [r0]
-; CHECK-NEXT:    vtrn.16 [[LDR0]], [[LDR1]]
-; CHECK-NEXT:    vmov r0, r1, [[LDR0]]
-; CHECK-NEXT:    vmov r2, r3, [[LDR1]]
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.16 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <4 x i16>, ptr %A
   %tmp2 = load <4 x i16>, ptr %B
@@ -84,11 +85,11 @@ define <2 x i32> @vtrni32(ptr %A, ptr %B) nounwind {
 define <4 x i32> @vtrni32_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vtrni32_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr [[LDR1:d[0-9]+]], [r1]
-; CHECK-NEXT:    vldr [[LDR0:d[0-9]+]], [r0]
-; CHECK-NEXT:    vtrn.32 [[LDR0]], [[LDR1]]
-; CHECK-NEXT:    vmov r0, r1, [[LDR0]]
-; CHECK-NEXT:    vmov r2, r3, [[LDR1]]
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.32 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <2 x i32>, ptr %A
   %tmp2 = load <2 x i32>, ptr %B
@@ -116,11 +117,11 @@ define <2 x float> @vtrnf(ptr %A, ptr %B) nounwind {
 define <4 x float> @vtrnf_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vtrnf_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr [[LDR1:d[0-9]+]], [r1]
-; CHECK-NEXT:    vldr [[LDR0:d[0-9]+]], [r0]
-; CHECK-NEXT:    vtrn.32 [[LDR0]], [[LDR1]]
-; CHECK-NEXT:    vmov r0, r1, [[LDR0]]
-; CHECK-NEXT:    vmov r2, r3, [[LDR1]]
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.32 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <2 x float>, ptr %A
   %tmp2 = load <2 x float>, ptr %B
@@ -281,11 +282,11 @@ define <8 x i8> @vtrni8_undef(ptr %A, ptr %B) nounwind {
 define <16 x i8> @vtrni8_undef_Qres(ptr %A, ptr %B) nounwind {
 ; CHECK-LABEL: vtrni8_undef_Qres:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldr [[LDR1:d[0-9]+]], [r1]
-; CHECK-NEXT:    vldr [[LDR0:d[0-9]+]], [r0]
-; CHECK-NEXT:    vtrn.8 [[LDR0]], [[LDR1]]
-; CHECK-NEXT:    vmov r0, r1, [[LDR0]]
-; CHECK-NEXT:    vmov r2, r3, [[LDR1]]
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.8 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d17
+; CHECK-NEXT:    vmov r2, r3, d16
 ; CHECK-NEXT:    mov pc, lr
   %tmp1 = load <8 x i8>, ptr %A
   %tmp2 = load <8 x i8>, ptr %B
@@ -327,9 +328,15 @@ define <16 x i16> @vtrnQi16_undef_QQres(ptr %A, ptr %B) nounwind {
 }
 
 define <8 x i16> @vtrn_lower_shufflemask_undef(ptr %A, ptr %B) {
+; CHECK-LABEL: vtrn_lower_shufflemask_undef:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d17, [r0]
+; CHECK-NEXT:    vtrn.16 d17, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d16
+; CHECK-NEXT:    mov pc, lr
 entry:
-  ; CHECK-LABEL: vtrn_lower_shufflemask_undef
-  ; CHECK: vtrn
 	%tmp1 = load <4 x i16>, ptr %A
 	%tmp2 = load <4 x i16>, ptr %B
   %0 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 5, i32 3, i32 7>
@@ -340,12 +347,26 @@ entry:
 ; values do modify the type. However, we get different input types, as some of
 ; them get truncated from i32 to i8 (from comparing cmp0 with cmp1) and some of
 ; them get truncated from i16 to i8 (from comparing cmp2 with cmp3).
-define <8 x i8> @vtrn_mismatched_builvector0(<8 x i8> %tr0, <8 x i8> %tr1,
-                                             <4 x i32> %cmp0, <4 x i32> %cmp1,
-                                             <4 x i16> %cmp2, <4 x i16> %cmp3) {
-  ; CHECK-LABEL: vtrn_mismatched_builvector0:
-  ; CHECK: vmovn.i32
-  ; CHECK: vbsl
+define <8 x i8> @vtrn_mismatched_builvector0(<8 x i8> %tr0, <8 x i8> %tr1, <4 x i32> %cmp0, <4 x i32> %cmp1, <4 x i16> %cmp2, <4 x i16> %cmp3) {
+; CHECK-LABEL: vtrn_mismatched_builvector0:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    mov r12, sp
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r12]
+; CHECK-NEXT:    add r12, sp, #16
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r12]
+; CHECK-NEXT:    vcgt.u32 q8, q9, q8
+; CHECK-NEXT:    vldr d20, [sp, #32]
+; CHECK-NEXT:    vldr d18, [sp, #40]
+; CHECK-NEXT:    vcgt.u16 d18, d18, d20
+; CHECK-NEXT:    vmovn.i32 d16, q8
+; CHECK-NEXT:    vmov d17, r2, r3
+; CHECK-NEXT:    vtrn.8 d16, d18
+; CHECK-NEXT:    vmov d18, r0, r1
+; CHECK-NEXT:    vshl.i8 d16, d16, #7
+; CHECK-NEXT:    vshr.s8 d16, d16, #7
+; CHECK-NEXT:    vbsl d16, d18, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    mov pc, lr
   %c0 = icmp ult <4 x i32> %cmp0, %cmp1
   %c1 = icmp ult <4 x i16> %cmp2, %cmp3
   %c = shufflevector <4 x i1> %c0, <4 x i1> %c1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
@@ -356,12 +377,30 @@ define <8 x i8> @vtrn_mismatched_builvector0(<8 x i8> %tr0, <8 x i8> %tr1,
 ; Here we get a build_vector node, where half the incoming extract_element
 ; values do not modify the type (the values form cmp2), but half of them do
 ; (from the icmp operation).
-define <8 x i8> @vtrn_mismatched_builvector1(<8 x i8> %tr0, <8 x i8> %tr1,
-                           <4 x i32> %cmp0, <4 x i32> %cmp1, ptr %cmp2_ptr) {
-  ; CHECK-LABEL: vtrn_mismatched_builvector1:
-  ; We need to extend the 4 x i8 to 4 x i16 in order to perform the vtrn
-  ; CHECK: vmovl
-  ; CHECK: vbsl
+; We need to extend the 4 x i8 to 4 x i16 in order to perform the vtrn
+define <8 x i8> @vtrn_mismatched_builvector1(<8 x i8> %tr0, <8 x i8> %tr1, <4 x i32> %cmp0, <4 x i32> %cmp1, ptr %cmp2_ptr) {
+; CHECK-LABEL: vtrn_mismatched_builvector1:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r11, lr}
+; CHECK-NEXT:    push {r11, lr}
+; CHECK-NEXT:    add r12, sp, #8
+; CHECK-NEXT:    add lr, sp, #24
+; CHECK-NEXT:    vld1.64 {d16, d17}, [r12]
+; CHECK-NEXT:    ldr r12, [sp, #40]
+; CHECK-NEXT:    vld1.64 {d18, d19}, [lr]
+; CHECK-NEXT:    vcgt.u32 q8, q9, q8
+; CHECK-NEXT:    vld1.32 {d18[0]}, [r12:32]
+; CHECK-NEXT:    vmovl.u8 q9, d18
+; CHECK-NEXT:    vmovn.i32 d16, q8
+; CHECK-NEXT:    vmov d17, r2, r3
+; CHECK-NEXT:    vtrn.8 d16, d18
+; CHECK-NEXT:    vmov d18, r0, r1
+; CHECK-NEXT:    vshl.i8 d16, d16, #7
+; CHECK-NEXT:    vshr.s8 d16, d16, #7
+; CHECK-NEXT:    vbsl d16, d18, d17
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    pop {r11, lr}
+; CHECK-NEXT:    mov pc, lr
   %cmp2_load = load <4 x i8>, ptr %cmp2_ptr, align 4
   %cmp2 = trunc <4 x i8> %cmp2_load to <4 x i1>
   %c0 = icmp ult <4 x i32> %cmp0, %cmp1
@@ -373,15 +412,15 @@ define <8 x i8> @vtrn_mismatched_builvector1(<8 x i8> %tr0, <8 x i8> %tr1,
 ; The shuffle mask is half a vtrn; we duplicate the half to produce the
 ; full result.
 define void @lower_twice_no_vtrn(ptr %A, ptr %B, ptr %C) {
+; CHECK-LABEL: lower_twice_no_vtrn:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d18, [r0]
+; CHECK-NEXT:    vtrn.16 d18, d16
+; CHECK-NEXT:    vorr d17, d16, d16
+; CHECK-NEXT:    vst1.64 {d16, d17}, [r2]
+; CHECK-NEXT:    mov pc, lr
 entry:
-  ; CHECK-LABEL: lower_twice_no_vtrn:
-  ; CHECK: @ %bb.0:
-  ; CHECK-NEXT: vldr d16, [r1]
-  ; CHECK-NEXT: vldr d18, [r0]
-  ; CHECK-NEXT: vtrn.16 d18, d16
-  ; CHECK-NEXT: vorr d17, d16, d16
-  ; CHECK-NEXT: vst1.64 {d16, d17}, [r2]
-  ; CHECK-NEXT: mov pc, lr
   %tmp1 = load <4 x i16>, ptr %A
   %tmp2 = load <4 x i16>, ptr %B
   %0 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 undef, i32 5, i32 3, i32 7, i32 1, i32 5, i32 3, i32 7>
@@ -392,18 +431,49 @@ entry:
 ; The shuffle mask is half a vtrn; we duplicate the half to produce the
 ; full result.
 define void @upper_twice_no_vtrn(ptr %A, ptr %B, ptr %C) {
+; CHECK-LABEL: upper_twice_no_vtrn:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldr d16, [r1]
+; CHECK-NEXT:    vldr d18, [r0]
+; CHECK-NEXT:    vtrn.16 d18, d16
+; CHECK-NEXT:    vorr d19, d18, d18
+; CHECK-NEXT:    vst1.64 {d18, d19}, [r2]
+; CHECK-NEXT:    mov pc, lr
 entry:
-  ; CHECK-LABEL: upper_twice_no_vtrn:
-  ; CHECK: @ %bb.0:
-  ; CHECK-NEXT: vldr d16, [r1]
-  ; CHECK-NEXT: vldr d18, [r0]
-  ; CHECK-NEXT: vtrn.16 d18, d16
-  ; CHECK-NEXT: vorr d19, d18, d18
-  ; CHECK-NEXT: vst1.64 {d18, d19}, [r2]
-  ; CHECK-NEXT: mov pc, lr
   %tmp1 = load <4 x i16>, ptr %A
   %tmp2 = load <4 x i16>, ptr %B
   %0 = shufflevector <4 x i16> %tmp1, <4 x i16> %tmp2, <8 x i32> <i32 0, i32 undef, i32 2, i32 6, i32 0, i32 4, i32 2, i32 6>
   store <8 x i16> %0, ptr %C
   ret void
 }
+
+define void @test_15xi16(ptr %next.gep, ptr %next.gep13) {
+; CHECK-LABEL: test_15xi16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    add r2, r0, #2
+; CHECK-NEXT:    add r3, r0, #6
+; CHECK-NEXT:    vld1.16 {d16, d17}, [r2]!
+; CHECK-NEXT:    vld1.16 {d18}, [r2]!
+; CHECK-NEXT:    vld1.16 {d20, d21}, [r3]!
+; CHECK-NEXT:    ldr r2, [r2]
+; CHECK-NEXT:    vld1.16 {d22}, [r3]!
+; CHECK-NEXT:    vmov.16 d19[0], r2
+; CHECK-NEXT:    ldr r3, [r3]
+; CHECK-NEXT:    add r2, r0, #30
+; CHECK-NEXT:    add r0, r0, #34
+; CHECK-NEXT:    vmov.16 d19[1], r3
+; CHECK-NEXT:    vld1.16 {d19[2]}, [r2:16]
+; CHECK-NEXT:    vtrn.16 q8, q10
+; CHECK-NEXT:    vld1.16 {d19[3]}, [r0:16]
+; CHECK-NEXT:    vtrn.16 d18, d22
+; CHECK-NEXT:    vst1.16 {d16, d17}, [r1]!
+; CHECK-NEXT:    vst1.16 {d18, d19}, [r1]
+; CHECK-NEXT:    mov pc, lr
+  %a = getelementptr inbounds nuw i8, ptr %next.gep, i32 2
+  %b = load <15 x i16>, ptr %a, align 2
+  %c = getelementptr inbounds nuw i8, ptr %next.gep, i32 6
+  %d = load <15 x i16>, ptr %c, align 2
+  %interleaved.vec = shufflevector <15 x i16> %b, <15 x i16> %d, <16 x i32> <i32 0, i32 15, i32 2, i32 17, i32 4, i32 19, i32 6, i32 21, i32 8, i32 23, i32 10, i32 25, i32 12, i32 27, i32 14, i32 29>
+  store <16 x i16> %interleaved.vec, ptr %next.gep13, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/vuzp.ll b/llvm/test/CodeGen/ARM/vuzp.ll
index 7e1dfba34db2e..d24dadc7fc401 100644
--- a/llvm/test/CodeGen/ARM/vuzp.ll
+++ b/llvm/test/CodeGen/ARM/vuzp.ll
@@ -535,3 +535,59 @@ define %struct.uint8x8x2_t @vuzp_extract_subvector(<16 x i8> %t) #0 {
   %.fca.0.1.insert = insertvalue %struct.uint8x8x2_t %.fca.0.0.insert, <8 x i8> %vuzp1.i, 0, 1
   ret %struct.uint8x8x2_t %.fca.0.1.insert
 }
+
+define void @test_15xi16(ptr %next.gep, ptr %next.gep13) {
+; CHECK-LABEL: test_15xi16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    .save {r4, r5, r6, lr}
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    add r2, r0, #2
+; CHECK-NEXT:    add r3, r0, #6
+; CHECK-NEXT:    vld1.16 {d20, d21}, [r2]!
+; CHECK-NEXT:    vld1.16 {d16}, [r2]!
+; CHECK-NEXT:    vmov.u16 r12, d16[0]
+; CHECK-NEXT:    ldr r2, [r2]
+; CHECK-NEXT:    vmov.u16 r4, d20[0]
+; CHECK-NEXT:    vld1.16 {d22, d23}, [r3]!
+; CHECK-NEXT:    vld1.16 {d24}, [r3]!
+; CHECK-NEXT:    vmov.u16 lr, d16[2]
+; CHECK-NEXT:    vmov.u16 r5, d22[0]
+; CHECK-NEXT:    vmov.u16 r6, d21[0]
+; CHECK-NEXT:    vmov.16 d17[0], r12
+; CHECK-NEXT:    vmov.16 d16[0], r4
+; CHECK-NEXT:    vmov.u16 r4, d24[0]
+; CHECK-NEXT:    vmov.u16 r12, d24[2]
+; CHECK-NEXT:    vmov.16 d17[1], lr
+; CHECK-NEXT:    vmov.16 d18[0], r5
+; CHECK-NEXT:    vmov.u16 r5, d20[2]
+; CHECK-NEXT:    vmov.u16 lr, d23[0]
+; CHECK-NEXT:    vmov.16 d19[0], r4
+; CHECK-NEXT:    vmov.u16 r4, d22[2]
+; CHECK-NEXT:    vmov.16 d16[1], r5
+; CHECK-NEXT:    vmov.u16 r5, d21[2]
+; CHECK-NEXT:    vmov.16 d17[2], r2
+; CHECK-NEXT:    ldr r2, [r3]
+; CHECK-NEXT:    vmov.16 d16[2], r6
+; CHECK-NEXT:    vmov.16 d18[1], r4
+; CHECK-NEXT:    vmov.u16 r4, d23[2]
+; CHECK-NEXT:    vmov.16 d19[1], r12
+; CHECK-NEXT:    vmov.16 d18[2], lr
+; CHECK-NEXT:    vmov.16 d19[2], r2
+; CHECK-NEXT:    add r2, r0, #30
+; CHECK-NEXT:    add r0, r0, #34
+; CHECK-NEXT:    vld1.16 {d17[3]}, [r2:16]
+; CHECK-NEXT:    vmov.16 d16[3], r5
+; CHECK-NEXT:    vmov.16 d18[3], r4
+; CHECK-NEXT:    vld1.16 {d19[3]}, [r0:16]
+; CHECK-NEXT:    vst1.16 {d16, d17}, [r1]!
+; CHECK-NEXT:    vst1.16 {d18, d19}, [r1]
+; CHECK-NEXT:    pop {r4, r5, r6, lr}
+; CHECK-NEXT:    mov pc, lr
+  %a = getelementptr inbounds nuw i8, ptr %next.gep, i32 2
+  %b = load <15 x i16>, ptr %a, align 2
+  %c = getelementptr inbounds nuw i8, ptr %next.gep, i32 6
+  %d = load <15 x i16>, ptr %c, align 2
+  %interleaved.vec = shufflevector <15 x i16> %b, <15 x i16> %d, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29>
+  store <16 x i16> %interleaved.vec, ptr %next.gep13, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/ARM/vzip.ll b/llvm/test/CodeGen/ARM/vzip.ll
index dda774abd8516..ce40a2e48b6e8 100644
--- a/llvm/test/CodeGen/ARM/vzip.ll
+++ b/llvm/test/CodeGen/ARM/vzip.ll
@@ -381,3 +381,22 @@ entry:
   %vzip.i = shufflevector <8 x i8> %lane, <8 x i8> %lane3, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   ret <8 x i8> %vzip.i
 }
+
+define <16 x i16> @test_15xi16(ptr %next.gep, ptr %next.gep13) {
+; CHECK-LABEL: test_15xi16:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    add r1, r1, #2
+; CHECK-NEXT:    mov r2, #4
+; CHECK-NEXT:    vld1.16 {d16, d17}, [r1], r2
+; CHECK-NEXT:    vld1.16 {d18, d19}, [r1]
+; CHECK-NEXT:    vzip.16 q8, q9
+; CHECK-NEXT:    vst1.16 {d16, d17}, [r0:128]!
+; CHECK-NEXT:    vst1.64 {d18, d19}, [r0:128]
+; CHECK-NEXT:    mov pc, lr
+  %a = getelementptr inbounds nuw i8, ptr %next.gep, i32 2
+  %b = load <15 x i16>, ptr %a, align 2
+  %c = getelementptr inbounds nuw i8, ptr %next.gep, i32 6
+  %d = load <15 x i16>, ptr %c, align 2
+  %interleaved.vec = shufflevector <15 x i16> %b, <15 x i16> %d, <16 x i32> <i32 0, i32 15, i32 1, i32 16, i32 2, i32 17, i32 3, i32 18, i32 4, i32 19, i32 5, i32 20, i32 6, i32 21, i32 7, i32 22>
+  ret <16 x i16> %interleaved.vec
+}
diff --git a/llvm/test/CodeGen/AVR/bug-143247.ll b/llvm/test/CodeGen/AVR/bug-143247.ll
index 07c4c6562c950..d4493272af76d 100644
--- a/llvm/test/CodeGen/AVR/bug-143247.ll
+++ b/llvm/test/CodeGen/AVR/bug-143247.ll
@@ -8,18 +8,18 @@ define void @complex_sbi() {
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    push r16
 ; CHECK-NEXT:    push r17
-; CHECK-NEXT:    ldi r24, 0
+; CHECK-NEXT:    ldi r24, 1
 ; CHECK-NEXT:    ldi r25, 0
 ; CHECK-NEXT:  .LBB0_1: ; %while.cond
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    sbi 1, 7
-; CHECK-NEXT:    adiw r24, 1
 ; CHECK-NEXT:    movw r16, r24
 ; CHECK-NEXT:    andi r24, 15
 ; CHECK-NEXT:    andi r25, 0
 ; CHECK-NEXT:    adiw r24, 1
 ; CHECK-NEXT:    call nil
 ; CHECK-NEXT:    movw r24, r16
+; CHECK-NEXT:    adiw r24, 1
 ; CHECK-NEXT:    rjmp .LBB0_1
 entry:
   br label %while.cond
diff --git a/llvm/test/CodeGen/AVR/cmp.ll b/llvm/test/CodeGen/AVR/cmp.ll
index efc9b8da45ba5..c932bda1807f1 100644
--- a/llvm/test/CodeGen/AVR/cmp.ll
+++ b/llvm/test/CodeGen/AVR/cmp.ll
@@ -298,3 +298,18 @@ define i16 @cmp_i16_gt_1023(i16 %0) {
   %3 = zext i1 %2 to i16
   ret i16 %3
 }
+
+define void @cmp_issue152097(i16 %a) addrspace(1) {
+; See: https://github.com/llvm/llvm-project/issues/152097
+; CHECK-LABEL: cmp_issue152097
+; CHECK:      ldi r18, -1
+; CHECK-NEXT: cpi r24, -2
+; CHECK-NEXT: cpc r25, r18
+; CHECK-NEXT: ret
+  %cmp = icmp ugt i16 -2, %a
+  br i1 %cmp, label %if.then, label %if.else
+if.then:
+  ret void
+if.else:
+  ret void
+}
diff --git a/llvm/test/CodeGen/AVR/half.ll b/llvm/test/CodeGen/AVR/half.ll
new file mode 100644
index 0000000000000..c922293e417bd
--- /dev/null
+++ b/llvm/test/CodeGen/AVR/half.ll
@@ -0,0 +1,534 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc %s -o - -mtriple=avr | FileCheck %s
+
+; Tests for various operations on half precison float. Much of the test is
+; copied from test/CodeGen/X86/half.ll.
+
+define void @store(half %x, ptr %p) nounwind {
+; CHECK-LABEL: store:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r22
+; CHECK-NEXT:    mov r31, r23
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    ret
+  store half %x, ptr %p
+  ret void
+}
+
+define half @return(ptr %p) nounwind {
+; CHECK-LABEL: return:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    ret
+  %r = load half, ptr %p
+  ret half %r
+}
+
+define dso_local double @loadd(ptr nocapture readonly %a) local_unnamed_addr nounwind {
+; CHECK-LABEL: loadd:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ldd r24, Z+2
+; CHECK-NEXT:    ldd r25, Z+3
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    rcall __extendsfdf2
+; CHECK-NEXT:    ret
+entry:
+  %arrayidx = getelementptr inbounds i16, ptr %a, i64 1
+  %0 = load i16, ptr %arrayidx, align 2
+  %1 = tail call double @llvm.convert.from.fp16.f64(i16 %0)
+  ret double %1
+}
+
+define dso_local float @loadf(ptr nocapture readonly %a) local_unnamed_addr nounwind {
+; CHECK-LABEL: loadf:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ldd r24, Z+2
+; CHECK-NEXT:    ldd r25, Z+3
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    ret
+entry:
+  %arrayidx = getelementptr inbounds i16, ptr %a, i64 1
+  %0 = load i16, ptr %arrayidx, align 2
+  %1 = tail call float @llvm.convert.from.fp16.f32(i16 %0)
+  ret float %1
+}
+
+define dso_local void @stored(ptr nocapture %a, double %b) local_unnamed_addr nounwind {
+; CHECK-LABEL: stored:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    push r16
+; CHECK-NEXT:    push r17
+; CHECK-NEXT:    mov r30, r22
+; CHECK-NEXT:    mov r31, r23
+; CHECK-NEXT:    mov r22, r20
+; CHECK-NEXT:    mov r23, r21
+; CHECK-NEXT:    mov r20, r18
+; CHECK-NEXT:    mov r21, r19
+; CHECK-NEXT:    mov r18, r16
+; CHECK-NEXT:    mov r19, r17
+; CHECK-NEXT:    mov r16, r24
+; CHECK-NEXT:    mov r17, r25
+; CHECK-NEXT:    mov r24, r30
+; CHECK-NEXT:    mov r25, r31
+; CHECK-NEXT:    rcall __truncdfhf2
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    pop r17
+; CHECK-NEXT:    pop r16
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i16 @llvm.convert.to.fp16.f64(double %b)
+  store i16 %0, ptr %a, align 2
+  ret void
+}
+
+define dso_local void @storef(ptr nocapture %a, float %b) local_unnamed_addr nounwind {
+; CHECK-LABEL: storef:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    push r16
+; CHECK-NEXT:    push r17
+; CHECK-NEXT:    mov r18, r22
+; CHECK-NEXT:    mov r19, r23
+; CHECK-NEXT:    mov r16, r24
+; CHECK-NEXT:    mov r17, r25
+; CHECK-NEXT:    mov r22, r20
+; CHECK-NEXT:    mov r23, r21
+; CHECK-NEXT:    mov r24, r18
+; CHECK-NEXT:    mov r25, r19
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    pop r17
+; CHECK-NEXT:    pop r16
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i16 @llvm.convert.to.fp16.f32(float %b)
+  store i16 %0, ptr %a, align 2
+  ret void
+}
+
+define void @test_load_store(ptr %in, ptr %out) nounwind {
+; CHECK-LABEL: test_load_store:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    mov r30, r22
+; CHECK-NEXT:    mov r31, r23
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    ret
+  %val = load half, ptr %in
+  store half %val, ptr %out
+  ret void
+}
+
+define i16 @test_bitcast_from_half(ptr %addr) nounwind {
+; CHECK-LABEL: test_bitcast_from_half:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    ret
+  %val = load half, ptr %addr
+  %val_int = bitcast half %val to i16
+  ret i16 %val_int
+}
+
+define void @test_bitcast_to_half(ptr %addr, i16 %in) nounwind {
+; CHECK-LABEL: test_bitcast_to_half:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    std Z+1, r23
+; CHECK-NEXT:    st Z, r22
+; CHECK-NEXT:    ret
+  %val_fp = bitcast i16 %in to half
+  store half %val_fp, ptr %addr
+  ret void
+}
+
+define half @from_bits(i16 %x) nounwind {
+; CHECK-LABEL: from_bits:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ret
+  %res = bitcast i16 %x to half
+  ret half %res
+}
+
+define i16 @to_bits(half %x) nounwind {
+; CHECK-LABEL: to_bits:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ret
+    %res = bitcast half %x to i16
+    ret i16 %res
+}
+
+define float @test_extend32(ptr %addr) nounwind {
+; CHECK-LABEL: test_extend32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    ret
+  %val16 = load half, ptr %addr
+  %val32 = fpext half %val16 to float
+  ret float %val32
+}
+
+define double @test_extend64(ptr %addr) nounwind {
+; CHECK-LABEL: test_extend64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    rcall __extendsfdf2
+; CHECK-NEXT:    ret
+  %val16 = load half, ptr %addr
+  %val32 = fpext half %val16 to double
+  ret double %val32
+}
+
+define void @test_trunc32(float %in, ptr %addr) nounwind {
+; CHECK-LABEL: test_trunc32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r16
+; CHECK-NEXT:    push r17
+; CHECK-NEXT:    mov r16, r20
+; CHECK-NEXT:    mov r17, r21
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    pop r17
+; CHECK-NEXT:    pop r16
+; CHECK-NEXT:    ret
+  %val16 = fptrunc float %in to half
+  store half %val16, ptr %addr
+  ret void
+}
+
+define void @test_trunc64(double %in, ptr %addr) nounwind {
+; CHECK-LABEL: test_trunc64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    rcall __truncdfhf2
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    ret
+  %val16 = fptrunc double %in to half
+  store half %val16, ptr %addr
+  ret void
+}
+
+define i64 @test_fptosi_i64(ptr %p) nounwind {
+; CHECK-LABEL: test_fptosi_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    rcall __fixsfdi
+; CHECK-NEXT:    ret
+  %a = load half, ptr %p, align 2
+  %r = fptosi half %a to i64
+  ret i64 %r
+}
+
+define void @test_sitofp_i64(i64 %a, ptr %p) nounwind {
+; CHECK-LABEL: test_sitofp_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    rcall __floatdisf
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    ret
+  %r = sitofp i64 %a to half
+  store half %r, ptr %p
+  ret void
+}
+
+define i64 @test_fptoui_i64(ptr %p) nounwind {
+; CHECK-LABEL: test_fptoui_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    rcall __fixunssfdi
+; CHECK-NEXT:    ret
+  %a = load half, ptr %p, align 2
+  %r = fptoui half %a to i64
+  ret i64 %r
+}
+
+define void @test_uitofp_i64(i64 %a, ptr %p) nounwind {
+; CHECK-LABEL: test_uitofp_i64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    rcall __floatundisf
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    ret
+  %r = uitofp i64 %a to half
+  store half %r, ptr %p
+  ret void
+}
+
+define <2 x float> @test_extend32_vec2(ptr %p) nounwind {
+; CHECK-LABEL: test_extend32_vec2:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r12
+; CHECK-NEXT:    push r13
+; CHECK-NEXT:    push r14
+; CHECK-NEXT:    push r15
+; CHECK-NEXT:    push r16
+; CHECK-NEXT:    push r17
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    mov r12, r30
+; CHECK-NEXT:    mov r13, r31
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    mov r16, r22
+; CHECK-NEXT:    mov r17, r23
+; CHECK-NEXT:    mov r14, r24
+; CHECK-NEXT:    mov r15, r25
+; CHECK-NEXT:    mov r30, r12
+; CHECK-NEXT:    mov r31, r13
+; CHECK-NEXT:    ldd r24, Z+2
+; CHECK-NEXT:    ldd r25, Z+3
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    mov r18, r16
+; CHECK-NEXT:    mov r19, r17
+; CHECK-NEXT:    mov r20, r14
+; CHECK-NEXT:    mov r21, r15
+; CHECK-NEXT:    pop r17
+; CHECK-NEXT:    pop r16
+; CHECK-NEXT:    pop r15
+; CHECK-NEXT:    pop r14
+; CHECK-NEXT:    pop r13
+; CHECK-NEXT:    pop r12
+; CHECK-NEXT:    ret
+  %a = load <2 x half>, ptr %p, align 8
+  %b = fpext <2 x half> %a to <2 x float>
+  ret <2 x float> %b
+}
+
+define <1 x double> @test_extend64_vec1(ptr %p) nounwind {
+; CHECK-LABEL: test_extend64_vec1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r24
+; CHECK-NEXT:    mov r31, r25
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    rcall __extendsfdf2
+; CHECK-NEXT:    ret
+  %a = load <1 x half>, ptr %p, align 8
+  %b = fpext <1 x half> %a to <1 x double>
+  ret <1 x double> %b
+}
+
+define void @test_trunc32_vec2(<2 x float> %a, ptr %p) nounwind {
+; CHECK-LABEL: test_trunc32_vec2:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r12
+; CHECK-NEXT:    push r13
+; CHECK-NEXT:    push r14
+; CHECK-NEXT:    push r15
+; CHECK-NEXT:    mov r14, r20
+; CHECK-NEXT:    mov r15, r21
+; CHECK-NEXT:    mov r12, r18
+; CHECK-NEXT:    mov r13, r19
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    std Z+3, r25
+; CHECK-NEXT:    std Z+2, r24
+; CHECK-NEXT:    mov r22, r12
+; CHECK-NEXT:    mov r23, r13
+; CHECK-NEXT:    mov r24, r14
+; CHECK-NEXT:    mov r25, r15
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    pop r15
+; CHECK-NEXT:    pop r14
+; CHECK-NEXT:    pop r13
+; CHECK-NEXT:    pop r12
+; CHECK-NEXT:    ret
+  %v = fptrunc <2 x float> %a to <2 x half>
+  store <2 x half> %v, ptr %p
+  ret void
+}
+
+define void @test_trunc64_vec1(<1 x double> %a, ptr %p) nounwind {
+; CHECK-LABEL: test_trunc64_vec1:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    rcall __truncdfhf2
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    std Z+1, r25
+; CHECK-NEXT:    st Z, r24
+; CHECK-NEXT:    ret
+  %v = fptrunc <1 x double> %a to <1 x half>
+  store <1 x half> %v, ptr %p
+  ret void
+}
+
+define float @test_sitofp_fadd_i32(i32 %a, ptr %b) nounwind {
+; CHECK-LABEL: test_sitofp_fadd_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r12
+; CHECK-NEXT:    push r13
+; CHECK-NEXT:    push r14
+; CHECK-NEXT:    push r15
+; CHECK-NEXT:    push r16
+; CHECK-NEXT:    push r17
+; CHECK-NEXT:    mov r16, r20
+; CHECK-NEXT:    mov r17, r21
+; CHECK-NEXT:    rcall __floatsisf
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    mov r14, r24
+; CHECK-NEXT:    mov r15, r25
+; CHECK-NEXT:    mov r30, r16
+; CHECK-NEXT:    mov r31, r17
+; CHECK-NEXT:    ld r24, Z
+; CHECK-NEXT:    ldd r25, Z+1
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    mov r16, r22
+; CHECK-NEXT:    mov r17, r23
+; CHECK-NEXT:    mov r12, r24
+; CHECK-NEXT:    mov r13, r25
+; CHECK-NEXT:    mov r24, r14
+; CHECK-NEXT:    mov r25, r15
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    mov r18, r22
+; CHECK-NEXT:    mov r19, r23
+; CHECK-NEXT:    mov r20, r24
+; CHECK-NEXT:    mov r21, r25
+; CHECK-NEXT:    mov r22, r16
+; CHECK-NEXT:    mov r23, r17
+; CHECK-NEXT:    mov r24, r12
+; CHECK-NEXT:    mov r25, r13
+; CHECK-NEXT:    rcall __addsf3
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    pop r17
+; CHECK-NEXT:    pop r16
+; CHECK-NEXT:    pop r15
+; CHECK-NEXT:    pop r14
+; CHECK-NEXT:    pop r13
+; CHECK-NEXT:    pop r12
+; CHECK-NEXT:    ret
+  %tmp0 = load half, ptr %b
+  %tmp1 = sitofp i32 %a to half
+  %tmp2 = fadd half %tmp0, %tmp1
+  %tmp3 = fpext half %tmp2 to float
+  ret float %tmp3
+}
+
+define half @chained_fp_ops(half %x) {
+; CHECK-LABEL: chained_fp_ops:
+; CHECK:       ; %bb.0: ; %start
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    mov r18, r22
+; CHECK-NEXT:    mov r19, r23
+; CHECK-NEXT:    mov r20, r24
+; CHECK-NEXT:    mov r21, r25
+; CHECK-NEXT:    rcall __addsf3
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    ldi r18, 0
+; CHECK-NEXT:    ldi r19, 0
+; CHECK-NEXT:    ldi r20, 0
+; CHECK-NEXT:    ldi r21, 63
+; CHECK-NEXT:    rcall __mulsf3
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    ret
+start:
+  %y = fmul half %x, 0xH4000
+  %z = fdiv half %y, 0xH4000
+  ret half %z
+}
+
+define half @test_select_cc(half) nounwind {
+; CHECK-LABEL: test_select_cc:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    push r16
+; CHECK-NEXT:    push r17
+; CHECK-NEXT:    rcall __extendhfsf2
+; CHECK-NEXT:    ldi r16, 0
+; CHECK-NEXT:    ldi r17, 0
+; CHECK-NEXT:    mov r18, r16
+; CHECK-NEXT:    mov r19, r17
+; CHECK-NEXT:    mov r20, r16
+; CHECK-NEXT:    mov r21, r17
+; CHECK-NEXT:    rcall __nesf2
+; CHECK-NEXT:    cpi r24, 0
+; CHECK-NEXT:    breq .LBB25_2
+; CHECK-NEXT:  ; %bb.1:
+; CHECK-NEXT:    ldi r16, 0
+; CHECK-NEXT:    ldi r17, 60
+; CHECK-NEXT:  .LBB25_2:
+; CHECK-NEXT:    mov r24, r16
+; CHECK-NEXT:    mov r25, r17
+; CHECK-NEXT:    pop r17
+; CHECK-NEXT:    pop r16
+; CHECK-NEXT:    ret
+  %2 = fcmp une half %0, 0xH0000
+  %3 = uitofp i1 %2 to half
+  ret half %3
+}
+
+define half @fabs(half %x) nounwind {
+; CHECK-LABEL: fabs:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    andi r25, 127
+; CHECK-NEXT:    ret
+  %a = call half @llvm.fabs.f16(half %x)
+  ret half %a
+}
+
+define half @fcopysign(half %x, half %y) nounwind {
+; CHECK-LABEL: fcopysign:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    andi r22, 0
+; CHECK-NEXT:    andi r23, 128
+; CHECK-NEXT:    andi r25, 127
+; CHECK-NEXT:    or r24, r22
+; CHECK-NEXT:    or r25, r23
+; CHECK-NEXT:    ret
+  %a = call half @llvm.copysign.f16(half %x, half %y)
+  ret half %a
+}
diff --git a/llvm/test/CodeGen/AVR/issue-151080.ll b/llvm/test/CodeGen/AVR/issue-151080.ll
new file mode 100644
index 0000000000000..9829224283f52
--- /dev/null
+++ b/llvm/test/CodeGen/AVR/issue-151080.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -O=3 -mtriple=avr-none -mcpu=attiny85 -verify-machineinstrs | FileCheck %s
+
+declare dso_local void @foo(i16 noundef) addrspace(1)
+@ci = dso_local global [30 x i16] zeroinitializer, align 1
+define void @loopreduce() {
+; CHECK-LABEL: loopreduce:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    push r14
+; CHECK-NEXT:    push r15
+; CHECK-NEXT:    push r16
+; CHECK-NEXT:    push r17
+; CHECK-NEXT:    ldi r26, lo8(ci)
+; CHECK-NEXT:    ldi r27, hi8(ci)
+; CHECK-NEXT:    ldi r16, lo8(ci+60)
+; CHECK-NEXT:    ldi r17, hi8(ci+60)
+; CHECK-NEXT:  .LBB0_1: ; %for.body
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ld r24, X+
+; CHECK-NEXT:    ld r25, X+
+; CHECK-NEXT:    movw r14, r26
+; CHECK-NEXT:    rcall foo
+; CHECK-NEXT:    movw r26, r14
+; CHECK-NEXT:    cp r26, r16
+; CHECK-NEXT:    cpc r27, r17
+; CHECK-NEXT:    brne .LBB0_1
+; CHECK-NEXT:  ; %bb.2: ; %for.cond.cleanup
+; CHECK-NEXT:    pop r17
+; CHECK-NEXT:    pop r16
+; CHECK-NEXT:    pop r15
+; CHECK-NEXT:    pop r14
+; CHECK-NEXT:    ret
+entry:
+  br label %for.body
+for.body:                                         ; preds = %entry, %for.body
+  %i.03 = phi i16 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds nuw [30 x i16], ptr @ci, i16 0, i16 %i.03
+  %0 = load i16, ptr %arrayidx, align 1
+  tail call addrspace(1) void @foo(i16 noundef %0)
+  %inc = add nuw nsw i16 %i.03, 1
+  %exitcond.not = icmp eq i16 %inc, 30
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+; Exit blocks
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+}
+
+define void @indvar() {
+; CHECK-LABEL: indvar:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    push r12
+; CHECK-NEXT:    push r13
+; CHECK-NEXT:    push r14
+; CHECK-NEXT:    push r15
+; CHECK-NEXT:    push r17
+; CHECK-NEXT:    ldi r24, 8
+; CHECK-NEXT:    ldi r25, 0
+; CHECK-NEXT:    movw r14, r24
+; CHECK-NEXT:    ldi r24, 1
+; CHECK-NEXT:    ldi r25, 0
+; CHECK-NEXT:    movw r12, r24
+; CHECK-NEXT:    ldi r17, 3
+; CHECK-NEXT:  .LBB1_1: ; %for.body
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movw r24, r12
+; CHECK-NEXT:    rcall foo
+; CHECK-NEXT:    movw r22, r14
+; CHECK-NEXT:    movw r24, r22
+; CHECK-NEXT:    rcall __mulhi3
+; CHECK-NEXT:    movw r30, r14
+; CHECK-NEXT:    adiw r30, 1
+; CHECK-NEXT:    movw r14, r30
+; CHECK-NEXT:    cpi r24, -24
+; CHECK-NEXT:    cpc r25, r17
+; CHECK-NEXT:    brlo .LBB1_1
+; CHECK-NEXT:  ; %bb.2: ; %for.cond.cleanup
+; CHECK-NEXT:    pop r17
+; CHECK-NEXT:    pop r15
+; CHECK-NEXT:    pop r14
+; CHECK-NEXT:    pop r13
+; CHECK-NEXT:    pop r12
+; CHECK-NEXT:    ret
+entry:
+  br label %for.body
+for.body:                                         ; preds = %entry, %for.body
+  %i.03 = phi i16 [ 7, %entry ], [ %inc, %for.body ]
+  tail call addrspace(1) void @foo(i16 noundef 1)
+  %inc = add nuw nsw i16 %i.03, 1
+  %mul = mul nuw nsw i16 %inc, %inc
+  %cmp = icmp samesign ult i16 %mul, 1000
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AVR/llrint.ll b/llvm/test/CodeGen/AVR/llrint.ll
index 32b4c7ab12a4b..2980879bb6e3e 100644
--- a/llvm/test/CodeGen/AVR/llrint.ll
+++ b/llvm/test/CodeGen/AVR/llrint.ll
@@ -1,13 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=avr -mcpu=atmega328p | FileCheck %s
 
+; FIXME: crash "Input type needs to be promoted!"
+; define i64 @testmsxh_builtin(half %x) {
+; entry:
+;   %0 = tail call i64 @llvm.llrint.i64.f16(half %x)
+;   ret i64 %0
+; }
+
 define i64 @testmsxs_builtin(float %x) {
 ; CHECK-LABEL: testmsxs_builtin:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    call llrintf
 ; CHECK-NEXT:    ret
 entry:
-  %0 = tail call i64 @llvm.llrint.f32(float %x)
+  %0 = tail call i64 @llvm.llrint.i64.f32(float %x)
   ret i64 %0
 }
 
@@ -17,9 +24,20 @@ define i64 @testmsxd_builtin(double %x) {
 ; CHECK-NEXT:    call llrint
 ; CHECK-NEXT:    ret
 entry:
-  %0 = tail call i64 @llvm.llrint.f64(double %x)
+  %0 = tail call i64 @llvm.llrint.i64.f64(double %x)
+  ret i64 %0
+}
+
+; FIXME(#44744): incorrect libcall
+define i64 @testmsxq_builtin(fp128 %x) {
+; CHECK-LABEL: testmsxq_builtin:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    call llrintl
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i64 @llvm.llrint.i64.fp128(fp128 %x)
   ret i64 %0
 }
 
-declare i64 @llvm.llrint.f32(float) nounwind readnone
-declare i64 @llvm.llrint.f64(double) nounwind readnone
+declare i64 @llvm.llrint.i64.f32(float) nounwind readnone
+declare i64 @llvm.llrint.i64.f64(double) nounwind readnone
diff --git a/llvm/test/CodeGen/AVR/llvm.sincos.ll b/llvm/test/CodeGen/AVR/llvm.sincos.ll
index b70b8d34a8a3b..ff01da9fd167b 100644
--- a/llvm/test/CodeGen/AVR/llvm.sincos.ll
+++ b/llvm/test/CodeGen/AVR/llvm.sincos.ll
@@ -11,8 +11,6 @@ define { half, half } @test_sincos_f16(half %a) #0 {
 ; CHECK-NEXT:    push r15
 ; CHECK-NEXT:    push r16
 ; CHECK-NEXT:    push r17
-; CHECK-NEXT:    mov r24, r22
-; CHECK-NEXT:    mov r25, r23
 ; CHECK-NEXT:    rcall __extendhfsf2
 ; CHECK-NEXT:    mov r16, r22
 ; CHECK-NEXT:    mov r17, r23
@@ -28,10 +26,8 @@ define { half, half } @test_sincos_f16(half %a) #0 {
 ; CHECK-NEXT:    mov r25, r15
 ; CHECK-NEXT:    rcall cos
 ; CHECK-NEXT:    rcall __truncsfhf2
-; CHECK-NEXT:    mov r22, r24
-; CHECK-NEXT:    mov r23, r25
-; CHECK-NEXT:    mov r18, r12
-; CHECK-NEXT:    mov r19, r13
+; CHECK-NEXT:    mov r22, r12
+; CHECK-NEXT:    mov r23, r13
 ; CHECK-NEXT:    pop r17
 ; CHECK-NEXT:    pop r16
 ; CHECK-NEXT:    pop r15
@@ -46,13 +42,9 @@ define { half, half } @test_sincos_f16(half %a) #0 {
 define half @test_sincos_f16_only_use_sin(half %a) #0 {
 ; CHECK-LABEL: test_sincos_f16_only_use_sin:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov r24, r22
-; CHECK-NEXT:    mov r25, r23
 ; CHECK-NEXT:    rcall __extendhfsf2
 ; CHECK-NEXT:    rcall sin
 ; CHECK-NEXT:    rcall __truncsfhf2
-; CHECK-NEXT:    mov r22, r24
-; CHECK-NEXT:    mov r23, r25
 ; CHECK-NEXT:    ret
   %result = call { half, half } @llvm.sincos.f16(half %a)
   %result.0 = extractvalue { half, half } %result, 0
@@ -62,13 +54,9 @@ define half @test_sincos_f16_only_use_sin(half %a) #0 {
 define half @test_sincos_f16_only_use_cos(half %a) #0 {
 ; CHECK-LABEL: test_sincos_f16_only_use_cos:
 ; CHECK:       ; %bb.0:
-; CHECK-NEXT:    mov r24, r22
-; CHECK-NEXT:    mov r25, r23
 ; CHECK-NEXT:    rcall __extendhfsf2
 ; CHECK-NEXT:    rcall cos
 ; CHECK-NEXT:    rcall __truncsfhf2
-; CHECK-NEXT:    mov r22, r24
-; CHECK-NEXT:    mov r23, r25
 ; CHECK-NEXT:    ret
   %result = call { half, half } @llvm.sincos.f16(half %a)
   %result.1 = extractvalue { half, half } %result, 1
@@ -90,48 +78,50 @@ define { <2 x half>, <2 x half> } @test_sincos_v2f16(<2 x half> %a) #0 {
 ; CHECK-NEXT:    push r15
 ; CHECK-NEXT:    push r16
 ; CHECK-NEXT:    push r17
-; CHECK-NEXT:    mov r10, r22
-; CHECK-NEXT:    mov r11, r23
+; CHECK-NEXT:    mov r16, r24
+; CHECK-NEXT:    mov r17, r25
+; CHECK-NEXT:    mov r24, r22
+; CHECK-NEXT:    mov r25, r23
 ; CHECK-NEXT:    rcall __extendhfsf2
-; CHECK-NEXT:    mov r16, r22
-; CHECK-NEXT:    mov r17, r23
-; CHECK-NEXT:    mov r14, r24
-; CHECK-NEXT:    mov r15, r25
-; CHECK-NEXT:    rcall sin
-; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    mov r14, r22
+; CHECK-NEXT:    mov r15, r23
 ; CHECK-NEXT:    mov r12, r24
 ; CHECK-NEXT:    mov r13, r25
-; CHECK-NEXT:    mov r24, r10
-; CHECK-NEXT:    mov r25, r11
+; CHECK-NEXT:    rcall sin
+; CHECK-NEXT:    rcall __truncsfhf2
+; CHECK-NEXT:    mov r10, r24
+; CHECK-NEXT:    mov r11, r25
+; CHECK-NEXT:    mov r24, r16
+; CHECK-NEXT:    mov r25, r17
 ; CHECK-NEXT:    rcall __extendhfsf2
-; CHECK-NEXT:    mov r10, r22
-; CHECK-NEXT:    mov r11, r23
+; CHECK-NEXT:    mov r16, r22
+; CHECK-NEXT:    mov r17, r23
 ; CHECK-NEXT:    mov r8, r24
 ; CHECK-NEXT:    mov r9, r25
-; CHECK-NEXT:    rcall cos
+; CHECK-NEXT:    rcall sin
 ; CHECK-NEXT:    rcall __truncsfhf2
 ; CHECK-NEXT:    mov r6, r24
 ; CHECK-NEXT:    mov r7, r25
-; CHECK-NEXT:    mov r22, r10
-; CHECK-NEXT:    mov r23, r11
-; CHECK-NEXT:    mov r24, r8
-; CHECK-NEXT:    mov r25, r9
-; CHECK-NEXT:    rcall sin
+; CHECK-NEXT:    mov r22, r14
+; CHECK-NEXT:    mov r23, r15
+; CHECK-NEXT:    mov r24, r12
+; CHECK-NEXT:    mov r25, r13
+; CHECK-NEXT:    rcall cos
 ; CHECK-NEXT:    rcall __truncsfhf2
-; CHECK-NEXT:    mov r10, r24
-; CHECK-NEXT:    mov r11, r25
+; CHECK-NEXT:    mov r14, r24
+; CHECK-NEXT:    mov r15, r25
 ; CHECK-NEXT:    mov r22, r16
 ; CHECK-NEXT:    mov r23, r17
-; CHECK-NEXT:    mov r24, r14
-; CHECK-NEXT:    mov r25, r15
+; CHECK-NEXT:    mov r24, r8
+; CHECK-NEXT:    mov r25, r9
 ; CHECK-NEXT:    rcall cos
 ; CHECK-NEXT:    rcall __truncsfhf2
 ; CHECK-NEXT:    mov r18, r10
 ; CHECK-NEXT:    mov r19, r11
-; CHECK-NEXT:    mov r20, r12
-; CHECK-NEXT:    mov r21, r13
-; CHECK-NEXT:    mov r22, r6
-; CHECK-NEXT:    mov r23, r7
+; CHECK-NEXT:    mov r20, r6
+; CHECK-NEXT:    mov r21, r7
+; CHECK-NEXT:    mov r22, r14
+; CHECK-NEXT:    mov r23, r15
 ; CHECK-NEXT:    pop r17
 ; CHECK-NEXT:    pop r16
 ; CHECK-NEXT:    pop r15
diff --git a/llvm/test/CodeGen/AVR/load.ll b/llvm/test/CodeGen/AVR/load.ll
index 5de6b48652914..6a1e067a21aed 100644
--- a/llvm/test/CodeGen/AVR/load.ll
+++ b/llvm/test/CodeGen/AVR/load.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mattr=avr6,sram < %s -mtriple=avr -verify-machineinstrs | FileCheck %s
+; RUN: llc -mattr=avr6,sram < %s -mtriple=avr-none -verify-machineinstrs | FileCheck %s
 
 define i8 @load8(ptr %x) {
 ; CHECK-LABEL: load8:
@@ -98,9 +98,33 @@ while.end:                                        ; preds = %while.body, %entry
   ret i16 %r.0.lcssa
 }
 
+define i16 @load16postincloopreduce(ptr %p, i16 %cnt) {
+; CHECK-LABEL: load16postincloopreduce:
+; CHECK: ld {{.*}}, {{[XYZ]}}+
+; CHECK: ld {{.*}}, {{[XYZ]}}+
+entry:
+  %cmp3 = icmp sgt i16 %cnt, 0
+  br i1 %cmp3, label %for.body, label %for.cond.cleanup
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %sum.0.lcssa = phi i16 [ 0, %entry ], [ %add, %for.body ]
+  ret i16 %sum.0.lcssa
+for.body:                                         ; preds = %entry, %for.body
+  %i.06 = phi i16 [ %inc, %for.body ], [ 0, %entry ]
+  %sum.05 = phi i16 [ %add, %for.body ], [ 0, %entry ]
+  %p.addr.04 = phi ptr [ %incdec.ptr, %for.body ], [ %p, %entry ]
+  %incdec.ptr = getelementptr inbounds nuw i8, ptr %p.addr.04, i16 2
+  %0 = load i16, ptr %p.addr.04, align 1
+  %add = add nsw i16 %0, %sum.05
+  %inc = add nuw nsw i16 %i.06, 1
+  %exitcond.not = icmp eq i16 %inc, %cnt
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
 define i8 @load8predec(ptr %x, i8 %y) {
 ; CHECK-LABEL: load8predec:
-; CHECK: ld {{.*}}, -{{[XYZ]}}
+; TODO: ld {{.*}}, -{{[XYZ]}}
+; CHECK: sbiw r26, 1
+; CHECK: ld {{.*}}, X
 entry:
   %tobool6 = icmp eq i8 %y, 0
   br i1 %tobool6, label %while.end, label %while.body
@@ -121,8 +145,12 @@ while.end:                                        ; preds = %while.body, %entry
 
 define i16 @load16predec(ptr %x, i16 %y) {
 ; CHECK-LABEL: load16predec:
-; CHECK: ld {{.*}}, -{{[XYZ]}}
-; CHECK: ld {{.*}}, -{{[XYZ]}}
+; TODO: ld {{.*}}, -{{[XYZ]}}
+; TODO: ld {{.*}}, -{{[XYZ]}}
+; CHECK: sbiw r24, 2
+; CHECK: movw r30, r24
+; CHECK: ld {{.*}}, Z
+; CHECK: ldd {{.*}}, Z+1
 entry:
   %tobool2 = icmp eq i16 %y, 0
   br i1 %tobool2, label %while.end, label %while.body
diff --git a/llvm/test/CodeGen/AVR/lrint.ll b/llvm/test/CodeGen/AVR/lrint.ll
index d7568305f7b51..4ef656060bd10 100644
--- a/llvm/test/CodeGen/AVR/lrint.ll
+++ b/llvm/test/CodeGen/AVR/lrint.ll
@@ -1,6 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=avr -mcpu=atmega328p | FileCheck %s
 
+; FIXME: crash "Input type needs to be promoted!"
+; define i32 @testmswh_builtin(half %x) {
+; entry:
+;   %0 = tail call i32 @llvm.lrint.i32.f16(half %x)
+;   ret i32 %0
+; }
+
 define i32 @testmsws_builtin(float %x) {
 ; CHECK-LABEL: testmsws_builtin:
 ; CHECK:       ; %bb.0: ; %entry
@@ -21,5 +28,16 @@ entry:
   ret i32 %0
 }
 
+; FIXME(#44744): incorrect libcall
+define i32 @testmswq_builtin(fp128 %x) {
+; CHECK-LABEL: testmswq_builtin:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    call lrint
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i32 @llvm.lrint.i32.fp128(fp128 %x)
+  ret i32 %0
+}
+
 declare i32 @llvm.lrint.i32.f32(float) nounwind readnone
 declare i32 @llvm.lrint.i32.f64(double) nounwind readnone
diff --git a/llvm/test/CodeGen/AVR/shift.ll b/llvm/test/CodeGen/AVR/shift.ll
index 9836f93527b3c..1bd9b45999d7b 100644
--- a/llvm/test/CodeGen/AVR/shift.ll
+++ b/llvm/test/CodeGen/AVR/shift.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=avr -mtriple=avr -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=avr-none -verify-machineinstrs | FileCheck %s
 
 ; Optimize for speed.
 define i8 @shift_i8_i8_speed(i8 %a, i8 %b) {
diff --git a/llvm/test/CodeGen/AVR/store.ll b/llvm/test/CodeGen/AVR/store.ll
index aab02709de7a6..9c4164557721c 100644
--- a/llvm/test/CodeGen/AVR/store.ll
+++ b/llvm/test/CodeGen/AVR/store.ll
@@ -94,7 +94,9 @@ while.end:                                        ; preds = %while.body, %entry
 
 define void @store8predec(ptr %x, i8 %y) {
 ; CHECK-LABEL: store8predec:
-; CHECK: st -{{[XYZ]}}, {{.*}}
+; TODO: st -{{[XYZ]}}, {{.*}}
+; CHECK: sbiw r26, 1
+; CHECK: st X, {{.*}}
 entry:
   %tobool3 = icmp eq i8 %y, 0
   br i1 %tobool3, label %while.end, label %while.body
@@ -112,8 +114,12 @@ while.end:                                        ; preds = %while.body, %entry
 
 define void @store16predec(ptr %x, i16 %y) {
 ; CHECK-LABEL: store16predec:
-; CHECK: st -{{[XYZ]}}, {{.*}}
-; CHECK: st -{{[XYZ]}}, {{.*}}
+; TODO: st -{{[XYZ]}}, {{.*}}
+; TODO: st -{{[XYZ]}}, {{.*}}
+; CHECK: sbiw r24, 2
+; CHECK: movw r30, r24
+; CHECK: std Z+1, {{.*}}
+; CHECK: st Z, {{.*}}
 entry:
   %tobool3 = icmp eq i16 %y, 0
   br i1 %tobool3, label %while.end, label %while.body
diff --git a/llvm/test/CodeGen/BPF/loop-exit-cond.ll b/llvm/test/CodeGen/BPF/loop-exit-cond.ll
index 69fe7148daa74..fa6a4a0254e79 100644
--- a/llvm/test/CodeGen/BPF/loop-exit-cond.ll
+++ b/llvm/test/CodeGen/BPF/loop-exit-cond.ll
@@ -35,14 +35,14 @@ define dso_local i32 @test(i32 %len, ptr %data) #0 {
 ; CHECK-NEXT:    br i1 [[OR_COND]], label [[FOR_BODY:%.*]], label [[IF_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[D]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[D]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[DATA]], align 1, !tbaa [[TBAA3:![0-9]+]]
 ; CHECK-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP1]], 0
 ; CHECK-NEXT:    [[NARROW:%.*]] = select i1 [[TOBOOL_NOT]], i8 48, i8 [[TMP1]]
 ; CHECK-NEXT:    [[CONV2:%.*]] = sext i8 [[NARROW]] to i64
 ; CHECK-NEXT:    store i64 [[CONV2]], ptr [[D]], align 8, !tbaa [[TBAA6:![0-9]+]]
 ; CHECK-NEXT:    call void @foo(ptr nonnull @.str, i32 [[I_05]], ptr nonnull [[D]]) #[[ATTR3]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[D]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[D]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[LEN]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[IF_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -61,7 +61,7 @@ entry:
   br i1 %cmp, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #3
+  call void @llvm.lifetime.start.p0(ptr %i) #3
   store i32 1, ptr %i, align 4, !tbaa !3
   br label %for.cond
 
@@ -73,11 +73,11 @@ for.cond:                                         ; preds = %for.inc, %if.then
 
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #3
+  call void @llvm.lifetime.end.p0(ptr %i) #3
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
-  call void @llvm.lifetime.start.p0(i64 8, ptr %d) #3
+  call void @llvm.lifetime.start.p0(ptr %d) #3
   %3 = load ptr, ptr %data.addr, align 8, !tbaa !7
   %4 = load i8, ptr %3, align 1, !tbaa !9
   %conv = sext i8 %4 to i32
@@ -96,7 +96,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
   store i64 %conv2, ptr %d, align 8, !tbaa !10
   %5 = load i32, ptr %i, align 4, !tbaa !3
   call void @foo(ptr @.str, i32 %5, ptr %d)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %d) #3
+  call void @llvm.lifetime.end.p0(ptr %d) #3
   br label %for.inc
 
 for.inc:                                          ; preds = %cond.end
@@ -113,12 +113,12 @@ if.end:                                           ; preds = %for.end, %entry
 }
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare dso_local void @foo(ptr, i32, ptr) #2
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 attributes #1 = { argmemonly nofree nosync nounwind willreturn }
diff --git a/llvm/test/CodeGen/BPF/vla.ll b/llvm/test/CodeGen/BPF/vla.ll
index 9a2276970bb52..708b41efac257 100644
--- a/llvm/test/CodeGen/BPF/vla.ll
+++ b/llvm/test/CodeGen/BPF/vla.ll
@@ -33,17 +33,17 @@ define dso_local i32 @test1() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[SAVED_STACK:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    store i32 8, ptr [[A]], align 4
 ; CHECK-NEXT:    [[VLA:%.*]] = alloca i8, i64 68, align 1
 ; CHECK-NEXT:    call void @foo(ptr [[VLA]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
   %a = alloca i32, align 4
   %saved_stack = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   store i32 8, ptr %a, align 4
   %0 = call ptr @llvm.stacksave()
   store ptr %0, ptr %saved_stack, align 8
@@ -51,11 +51,11 @@ entry:
   call void @foo(ptr %vla)
   %1 = load ptr, ptr %saved_stack, align 8
   call void @llvm.stackrestore(ptr %1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret i32 0
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare ptr @llvm.stacksave()
 
@@ -63,7 +63,7 @@ declare dso_local void @foo(ptr)
 
 declare void @llvm.stackrestore(ptr)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define dso_local i32 @test2(i32 %b) {
 ; CHECK-LABEL: @test2(
@@ -73,7 +73,7 @@ define dso_local i32 @test2(i32 %b) {
 ; CHECK-NEXT:    [[SAVED_STACK:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[__VLA_EXPR0:%.*]] = alloca i64, align 8
 ; CHECK-NEXT:    store i32 [[B:%.*]], ptr [[B_ADDR]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    store i32 8, ptr [[A]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[B_ADDR]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 8, [[TMP1]]
@@ -81,7 +81,7 @@ define dso_local i32 @test2(i32 %b) {
 ; CHECK-NEXT:    [[VLA:%.*]] = alloca i8, i64 [[TMP2]], align 1
 ; CHECK-NEXT:    store i64 [[TMP2]], ptr [[__VLA_EXPR0]], align 8
 ; CHECK-NEXT:    call void @foo(ptr [[VLA]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -90,7 +90,7 @@ entry:
   %saved_stack = alloca ptr, align 8
   %__vla_expr0 = alloca i64, align 8
   store i32 %b, ptr %b.addr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   store i32 8, ptr %a, align 4
   %0 = load i32, ptr %b.addr, align 4
   %add = add nsw i32 8, %0
@@ -102,6 +102,6 @@ entry:
   call void @foo(ptr %vla)
   %3 = load ptr, ptr %saved_stack, align 8
   call void @llvm.stackrestore(ptr %3)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret i32 0
 }
diff --git a/llvm/test/CodeGen/DirectX/Binding/binding-overlap-7.ll b/llvm/test/CodeGen/DirectX/Binding/binding-overlap-7.ll
new file mode 100644
index 0000000000000..25f81dd26b9db
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/Binding/binding-overlap-7.ll
@@ -0,0 +1,35 @@
+; Use llc for this test so that we don't abort after the first error.
+; RUN: not llc %s -o /dev/null 2>&1 | FileCheck %s
+
+; Check that there is no overlap with unbounded array in different space
+
+  ; Buffer<double> A[2] : register(t2, space4);
+  ; Buffer<double> B : register(t20, space5);  // does not overlap
+  ; Buffer<double> C[] : register(t2, space4); // overlaps with A
+
+; CHECK: error: resource A at register 2 overlaps with resource C at register 2 in space 4
+; CHECK-NOT: error: resource C at register 2 overlaps with resource B at register 20 in space 5
+
+target triple = "dxil-pc-shadermodel6.3-library"
+
+@A.str = private unnamed_addr constant [2 x i8] c"A\00", align 1
+@B.str = private unnamed_addr constant [2 x i8] c"B\00", align 1
+@C.str = private unnamed_addr constant [2 x i8] c"C\00", align 1
+
+define void @test_not_overlapping_in_different_spaces() {
+entry:
+
+  ; Buffer<double> A[2] : register(t2, space4);
+  %h0 = call target("dx.TypedBuffer", double, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 4, i32 2, i32 2, i32 10, i1 false, ptr @A.str)
+
+  ; Buffer<double> B : register(t20, space5);
+  %h1 = call target("dx.TypedBuffer", i64, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 5, i32 20, i32 1, i32 0, i1 false, ptr @B.str)
+
+  ; Buffer<double> C[] : register(t2, space4);
+  %h2 = call target("dx.TypedBuffer", double, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 4, i32 2, i32 -1, i32 10, i1 false, ptr @C.str)
+
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/Metadata/srv_metadata.ll b/llvm/test/CodeGen/DirectX/Metadata/srv_metadata.ll
index abab5c9fb1166..86d69abc75760 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/srv_metadata.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/srv_metadata.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s
 ; RUN: opt -S --passes="dxil-pretty-printer" < %s 2>&1 | FileCheck %s --check-prefix=PRINT
-; RUN: llc %s --filetype=asm -o - < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,PRINT
+; RUN: llc %s --filetype=asm -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,PRINT
 
 target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
 target triple = "dxil-pc-shadermodel6.6-compute"
@@ -14,20 +14,22 @@ target triple = "dxil-pc-shadermodel6.6-compute"
 @Six.str = private unnamed_addr constant [4 x i8] c"Six\00", align 1
 @Seven.str = private unnamed_addr constant [6 x i8] c"Seven\00", align 1
 @Array.str = private unnamed_addr constant [6 x i8] c"Array\00", align 1
+@Array2.str = private unnamed_addr constant [7 x i8] c"Array2\00", align 1
 
 ; PRINT:; Resource Bindings:
 ; PRINT-NEXT:;
-; PRINT-NEXT:; Name                                 Type  Format         Dim      ID      HLSL Bind  Count
-; PRINT-NEXT:; ------------------------------ ---------- ------- ----------- ------- -------------- ------
-; PRINT-NEXT:; Zero                              texture     f16         buf      T0             t0     1
-; PRINT-NEXT:; One                               texture     f32         buf      T1             t1     1
-; PRINT-NEXT:; Two                               texture     f64         buf      T2             t2     1
-; PRINT-NEXT:; Three                             texture     i32         buf      T3             t3     1
-; PRINT-NEXT:; Four                              texture    byte         r/o      T4             t5     1
-; PRINT-NEXT:; Five                              texture  struct         r/o      T5             t6     1
-; PRINT-NEXT:; Six                               texture     u64         buf      T6     t10,space2     1
-; PRINT-NEXT:; Array                             texture     f32         buf      T7      t4,space3   100
-; PRINT-NEXT:; Seven                             texture     u64         buf      T8     t20,space5     1
+; PRINT-NEXT:; Name                                 Type  Format         Dim      ID      HLSL Bind     Count
+; PRINT-NEXT:; ------------------------------ ---------- ------- ----------- ------- -------------- ---------
+; PRINT-NEXT:; Zero                              texture     f16         buf      T0             t0         1
+; PRINT-NEXT:; One                               texture     f32         buf      T1             t1         1
+; PRINT-NEXT:; Two                               texture     f64         buf      T2             t2         1
+; PRINT-NEXT:; Three                             texture     i32         buf      T3             t3         1
+; PRINT-NEXT:; Four                              texture    byte         r/o      T4             t5         1
+; PRINT-NEXT:; Five                              texture  struct         r/o      T5             t6         1
+; PRINT-NEXT:; Six                               texture     u64         buf      T6     t10,space2         1
+; PRINT-NEXT:; Array                             texture     f32         buf      T7      t4,space3       100
+; PRINT-NEXT:; Array2                            texture     f64         buf      T8      t2,space4 unbounded
+; PRINT-NEXT:; Seven                             texture     u64         buf      T9     t20,space5         1
 ;
 
 define void @test() #0 {
@@ -60,19 +62,28 @@ define void @test() #0 {
             @llvm.dx.resource.handlefrombinding(i32 2, i32 10, i32 1, i32 0, i1 false, ptr @Six.str)
 
   ; Same buffer type as Six - should have the same type in metadata
-  ; Buffer<double> Seven : register(t10, space2);
+  ; Buffer<double> Seven : register(t20, space5);
   %Seven_h = call target("dx.TypedBuffer", i64, 0, 0, 0)
             @llvm.dx.resource.handlefrombinding(i32 5, i32 20, i32 1, i32 0, i1 false, ptr @Seven.str)
 
   ; Buffer<float4> Array[100] : register(t4, space3);
   ; Buffer<float4> B1 = Array[30];
-  ; Buffer<float4> B1 = Array[42];
+  ; Buffer<float4> B2 = Array[42];
   ; resource array accesses should produce one metadata entry   
   %Array_30_h = call target("dx.TypedBuffer", <4 x float>, 0, 0, 0)
             @llvm.dx.resource.handlefrombinding(i32 3, i32 4, i32 100, i32 30, i1 false, ptr @Array.str)
   %Array_42_h = call target("dx.TypedBuffer", <4 x float>, 0, 0, 0)
             @llvm.dx.resource.handlefrombinding(i32 3, i32 4, i32 100, i32 42, i1 false, ptr @Array.str)
 
+  ; test unbounded resource array
+  ; Buffer<double> Array2[] : register(t2, space4);
+  ; Buffer<double> C1 = Array[10];
+  ; Buffer<double> C2 = Array[20];
+  %Array2_10_h = call target("dx.TypedBuffer", double, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 4, i32 2, i32 -1, i32 10, i1 false, ptr @Array2.str)
+  %Array2_20_h = call target("dx.TypedBuffer", double, 0, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 4, i32 2, i32 -1, i32 20, i1 false, ptr @Array2.str)
+
   ret void
 }
 
@@ -94,7 +105,8 @@ attributes #0 = { noinline nounwind "hlsl.shader"="compute" }
 ; CHECK: @Four = external constant %ByteAddressBuffer
 ; CHECK: @Five = external constant %"StructuredBuffer<int16_t>"
 ; CHECK: @Six = external constant %"Buffer<uint32_t>"
-; CHECK: @Array = external constant %"Buffer<float4>"
+; CHECK: @Array = external constant [100 x %"Buffer<float4>"]
+; CHECK: @Array2 = external constant [0 x %"Buffer<double>"]
 ; CHECK: @Seven = external constant %"Buffer<uint32_t>"
 
 ; CHECK: !dx.resources = !{[[ResList:[!][0-9]+]]}
@@ -102,7 +114,7 @@ attributes #0 = { noinline nounwind "hlsl.shader"="compute" }
 ; CHECK: [[ResList]] = !{[[SRVList:[!][0-9]+]], null, null, null}
 ; CHECK: [[SRVList]] = !{![[Zero:[0-9]+]], ![[One:[0-9]+]], ![[Two:[0-9]+]],
 ; CHECK-SAME: ![[Three:[0-9]+]], ![[Four:[0-9]+]], ![[Five:[0-9]+]],
-; CHECK-SAME: ![[Six:[0-9]+]], ![[Array:[0-9]+]], ![[Seven:[0-9]+]]}
+; CHECK-SAME: ![[Six:[0-9]+]], ![[Array:[0-9]+]], ![[Array2:[0-9]+]], ![[Seven:[0-9]+]]}
 
 ; CHECK: ![[Zero]] = !{i32 0, ptr @Zero, !"Zero", i32 0, i32 0, i32 1, i32 10, i32 0, ![[Half:[0-9]+]]}
 ; CHECK: ![[Half]] = !{i32 0, i32 8}
@@ -118,4 +130,5 @@ attributes #0 = { noinline nounwind "hlsl.shader"="compute" }
 ; CHECK: ![[Six]] = !{i32 6, ptr @Six, !"Six", i32 2, i32 10, i32 1, i32 10, i32 0, ![[U64:[0-9]+]]}
 ; CHECK: ![[U64]] = !{i32 0, i32 7}
 ; CHECK: ![[Array]] = !{i32 7, ptr @Array, !"Array", i32 3, i32 4, i32 100, i32 10, i32 0, ![[Float]]}
-; CHECK: ![[Seven]] = !{i32 8, ptr @Seven, !"Seven", i32 5, i32 20, i32 1, i32 10, i32 0, ![[U64]]}
+; CHECK: ![[Array2]] = !{i32 8, ptr @Array2, !"Array2", i32 4, i32 2, i32 -1, i32 10, i32 0, ![[Double]]}
+; CHECK: ![[Seven]] = !{i32 9, ptr @Seven, !"Seven", i32 5, i32 20, i32 1, i32 10, i32 0, ![[U64]]}
diff --git a/llvm/test/CodeGen/DirectX/Metadata/uav_metadata.ll b/llvm/test/CodeGen/DirectX/Metadata/uav_metadata.ll
index 9893f8b9ea4d7..4928b1d0bab97 100644
--- a/llvm/test/CodeGen/DirectX/Metadata/uav_metadata.ll
+++ b/llvm/test/CodeGen/DirectX/Metadata/uav_metadata.ll
@@ -1,6 +1,6 @@
 ; RUN: opt -S -dxil-translate-metadata < %s | FileCheck %s
 ; RUN: opt -S --passes="dxil-pretty-printer" < %s 2>&1 | FileCheck %s --check-prefix=PRINT
-; RUN: llc %s --filetype=asm -o - < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,PRINT
+; RUN: llc %s --filetype=asm -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,PRINT
 
 target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
 target triple = "dxil-pc-shadermodel6.6-compute"
@@ -17,23 +17,25 @@ target triple = "dxil-pc-shadermodel6.6-compute"
 @Nine.str = private unnamed_addr constant [5 x i8] c"Nine\00", align 1
 @Ten.str = private unnamed_addr constant [4 x i8] c"Ten\00", align 1
 @Array.str = private unnamed_addr constant [6 x i8] c"Array\00", align 1
+@Array2.str = private unnamed_addr constant [7 x i8] c"Array2\00", align 1
 
 ; PRINT:; Resource Bindings:
 ; PRINT-NEXT:;
-; PRINT-NEXT:; Name                                 Type  Format         Dim      ID      HLSL Bind  Count
-; PRINT-NEXT:; ------------------------------ ---------- ------- ----------- ------- -------------- ------
-; PRINT-NEXT:; Zero                                  UAV     f16         buf      U0             u0     1
-; PRINT-NEXT:; One                                   UAV     f32         buf      U1             u1     1
-; PRINT-NEXT:; Two                                   UAV     f64         buf      U2             u2     1
-; PRINT-NEXT:; Three                                 UAV     i32         buf      U3             u3     1
-; PRINT-NEXT:; Four                                  UAV    byte         r/w      U4             u5     1
-; PRINT-NEXT:; Five                                  UAV  struct         r/w      U5             u6     1
-; PRINT-NEXT:; Six                                   UAV     i32         buf      U6             u7     1
-; PRINT-NEXT:; Seven                                 UAV  struct         r/w      U7             u8     1
-; PRINT-NEXT:; Eight                                 UAV    byte         r/w      U8             u9     1
-; PRINT-NEXT:; Nine                                  UAV     u64         buf      U9     u10,space2     1
-; PRINT-NEXT:; Array                                 UAV     f32         buf     U10      u4,space3   100
-; PRINT-NEXT:; Ten                                   UAV     u64         buf     U11     u22,space5     1
+; PRINT-NEXT:; Name                                 Type  Format         Dim      ID      HLSL Bind     Count
+; PRINT-NEXT:; ------------------------------ ---------- ------- ----------- ------- -------------- ---------
+; PRINT-NEXT:; Zero                                  UAV     f16         buf      U0             u0         1
+; PRINT-NEXT:; One                                   UAV     f32         buf      U1             u1         1
+; PRINT-NEXT:; Two                                   UAV     f64         buf      U2             u2         1
+; PRINT-NEXT:; Three                                 UAV     i32         buf      U3             u3         1
+; PRINT-NEXT:; Four                                  UAV    byte         r/w      U4             u5         1
+; PRINT-NEXT:; Five                                  UAV  struct         r/w      U5             u6         1
+; PRINT-NEXT:; Six                                   UAV     i32         buf      U6             u7         1
+; PRINT-NEXT:; Seven                                 UAV  struct         r/w      U7             u8         1
+; PRINT-NEXT:; Eight                                 UAV    byte         r/w      U8             u9         1
+; PRINT-NEXT:; Nine                                  UAV     u64         buf      U9     u10,space2         1
+; PRINT-NEXT:; Array                                 UAV     f32         buf     U10      u4,space3       100
+; PRINT-NEXT:; Array2                                UAV     f64         buf     U11      u2,space4 unbounded
+; PRINT-NEXT:; Ten                                   UAV     u64         buf     U12     u22,space5         1
 
 define void @test() #0 {
   ; RWBuffer<half4> Zero : register(u0)
@@ -78,13 +80,22 @@ define void @test() #0 {
 
   ; RWBuffer<float4> Array[100] : register(u4, space3);
   ; RWBuffer<float4> B1 = Array[30];
-  ; RWBuffer<float4> B1 = Array[42];
+  ; RWBuffer<float4> B2 = Array[42];
   ; resource array accesses should produce one metadata entry   
   %Array_30_h = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
             @llvm.dx.resource.handlefrombinding(i32 3, i32 4, i32 100, i32 30, i1 false, ptr @Array.str)
   %Array_42_h = call target("dx.TypedBuffer", <4 x float>, 1, 0, 0)
             @llvm.dx.resource.handlefrombinding(i32 3, i32 4, i32 100, i32 42, i1 false, ptr @Array.str)
 
+  ; test unbounded resource array
+  ; RWBuffer<double> Array2[] : register(u2, space4);
+  ; RWBuffer<double> C1 = Array[10];
+  ; RWBuffer<double> C2 = Array[20];
+  %Array2_10_h = call target("dx.TypedBuffer", double, 1, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 4, i32 2, i32 -1, i32 10, i1 false, ptr @Array2.str)
+  %Array2_20_h = call target("dx.TypedBuffer", double, 1, 0, 0)
+            @llvm.dx.resource.handlefrombinding(i32 4, i32 2, i32 -1, i32 20, i1 false, ptr @Array2.str)
+
   ; Same buffer type as Nine - should have the same type in metadata
   ; RWBuffer<double> Ten : register(u2);
   %Ten_h = call target("dx.TypedBuffer", i64, 1, 0, 0)
@@ -117,7 +128,8 @@ attributes #0 = { noinline nounwind "hlsl.shader"="compute" }
 ; CHECK: @Seven = external constant %"RasterizerOrderedStructuredBuffer<int32_t4>"
 ; CHECK: @Eight = external constant %RasterizerOrderedByteAddressBuffer
 ; CHECK: @Nine = external constant %"RWBuffer<uint32_t>"
-; CHECK: @Array = external constant %"RWBuffer<float4>"
+; CHECK: @Array = external constant [100 x %"RWBuffer<float4>"]
+; CHECK: @Array2 = external constant [0 x %"RWBuffer<double>"]
 ; CHECK: @Ten = external constant %"RWBuffer<uint32_t>"
 
 ; CHECK: !dx.resources = !{[[ResList:[!][0-9]+]]}
@@ -126,7 +138,7 @@ attributes #0 = { noinline nounwind "hlsl.shader"="compute" }
 ; CHECK: [[UAVList]] = !{![[Zero:[0-9]+]], ![[One:[0-9]+]], ![[Two:[0-9]+]],
 ; CHECK-SAME: ![[Three:[0-9]+]], ![[Four:[0-9]+]], ![[Five:[0-9]+]],
 ; CHECK-SAME: ![[Six:[0-9]+]], ![[Seven:[0-9]+]], ![[Eight:[0-9]+]],
-; CHECK-SAME: ![[Nine:[0-9]+]], ![[Array:[0-9]+]], ![[Ten:[0-9]+]]}
+; CHECK-SAME: ![[Nine:[0-9]+]], ![[Array:[0-9]+]], ![[Array2:[0-9]+]], ![[Ten:[0-9]+]]}
 
 ; CHECK: ![[Zero]] = !{i32 0, ptr @Zero, !"Zero", i32 0, i32 0, i32 1, i32 10, i1 false, i1 false, i1 false, ![[Half:[0-9]+]]}
 ; CHECK: ![[Half]] = !{i32 0, i32 8}
@@ -146,4 +158,5 @@ attributes #0 = { noinline nounwind "hlsl.shader"="compute" }
 ; CHECK: ![[Nine]] = !{i32 9, ptr @Nine, !"Nine", i32 2, i32 10, i32 1, i32 10, i1 false, i1 false, i1 false, ![[U64:[0-9]+]]}
 ; CHECK: ![[U64]] = !{i32 0, i32 7}
 ; CHECK: ![[Array]] = !{i32 10, ptr @Array, !"Array", i32 3, i32 4, i32 100, i32 10, i1 false, i1 false, i1 false, ![[Float]]}
-; CHECK: ![[Ten]] = !{i32 11, ptr @Ten, !"Ten", i32 5, i32 22, i32 1, i32 10, i1 false, i1 false, i1 false, ![[U64:[0-9]+]]}
+; CHECK: ![[Array2]] = !{i32 11, ptr @Array2, !"Array2", i32 4, i32 2, i32 -1, i32 10, i1 false, i1 false, i1 false, ![[Double]]}
+; CHECK: ![[Ten]] = !{i32 12, ptr @Ten, !"Ten", i32 5, i32 22, i32 1, i32 10, i1 false, i1 false, i1 false, ![[U64:[0-9]+]]}
diff --git a/llvm/test/CodeGen/DirectX/ShaderFlags/lifetimes-noint64op.ll b/llvm/test/CodeGen/DirectX/ShaderFlags/lifetimes-noint64op.ll
index 736c86ebb1299..5cf4fe8223b5d 100644
--- a/llvm/test/CodeGen/DirectX/ShaderFlags/lifetimes-noint64op.ll
+++ b/llvm/test/CodeGen/DirectX/ShaderFlags/lifetimes-noint64op.ll
@@ -15,16 +15,16 @@ target triple = "dxil-pc-shadermodel6.7-library"
 
 define void @lifetimes() #0 {
   %a = alloca [4 x i32], align 8
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %a)
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret void
 }
 
 ; Function Attrs: nounwind memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64, ptr) #1
+declare void @llvm.lifetime.start.p0(ptr) #1
 
 ; Function Attrs: nounwind memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64, ptr) #1
+declare void @llvm.lifetime.end.p0(ptr) #1
 
 attributes #0 = { convergent norecurse nounwind "hlsl.export"}
 attributes #1 = { nounwind memory(argmem: readwrite) }
diff --git a/llvm/test/CodeGen/DirectX/dot2add.ll b/llvm/test/CodeGen/DirectX/dot2add.ll
index 3a2bbcc074f2d..5e1cf405752ba 100644
--- a/llvm/test/CodeGen/DirectX/dot2add.ll
+++ b/llvm/test/CodeGen/DirectX/dot2add.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-compute %s | FileCheck %s
+; RUN: opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.4-compute %s | FileCheck %s
 
 define noundef float @dot2add_simple(<2 x half> noundef %a, <2 x half> noundef %b, float %acc) {
 entry:
@@ -7,7 +7,7 @@ entry:
   %bx = extractelement <2 x half> %b, i32 0
   %by = extractelement <2 x half> %b, i32 1
 
-; CHECK: call float @dx.op.dot2AddHalf(i32 162, float %acc, half %ax, half %ay, half %bx, half %by)
+; CHECK: call float @dx.op.dot2AddHalf.f32(i32 162, float %acc, half %ax, half %ay, half %bx, half %by)
   %ret = call float @llvm.dx.dot2add(float %acc, half %ax, half %ay, half %bx, half %by)
   ret float %ret
 }
diff --git a/llvm/test/CodeGen/DirectX/dot2add_error.ll b/llvm/test/CodeGen/DirectX/dot2add_error.ll
new file mode 100644
index 0000000000000..c45133c8e812f
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/dot2add_error.ll
@@ -0,0 +1,15 @@
+; RUN: not opt -S -dxil-op-lower -mtriple=dxil-pc-shadermodel6.3-compute %s 2>&1 | FileCheck %s
+
+; CHECK: in function f
+; CHECK-SAME: Cannot create Dot2AddHalf operation: No valid overloads for DXIL version 1.3
+
+define noundef float @f(<2 x half> noundef %a, <2 x half> noundef %b, float %acc) {
+entry:
+  %ax = extractelement <2 x half> %a, i32 0
+  %ay = extractelement <2 x half> %a, i32 1
+  %bx = extractelement <2 x half> %b, i32 0
+  %by = extractelement <2 x half> %b, i32 1
+  
+  %ret = call float @llvm.dx.dot2add(float %acc, half %ax, half %ay, half %bx, half %by)
+  ret float %ret
+}
diff --git a/llvm/test/CodeGen/DirectX/finalize_linkage.ll b/llvm/test/CodeGen/DirectX/finalize_linkage.ll
index dc1140f1c9160..f6e54cf907a64 100644
--- a/llvm/test/CodeGen/DirectX/finalize_linkage.ll
+++ b/llvm/test/CodeGen/DirectX/finalize_linkage.ll
@@ -4,7 +4,8 @@
 target triple = "dxilv1.5-pc-shadermodel6.5-compute"
 
 ; DXILFinalizeLinkage changes linkage of all functions that are hidden to
-; internal, and converts private global variables to internal linkage.
+; internal, converts private globals to internal linkage, and converts external globals 
+; with no usage to internal linkage.
 
 ; CHECK: @switch.table = internal unnamed_addr constant [4 x i32]
 @switch.table = private unnamed_addr constant [4 x i32] [i32 1, i32 257, i32 65793, i32 16843009], align 4
@@ -27,6 +28,27 @@ target triple = "dxilv1.5-pc-shadermodel6.5-compute"
 ; CHECK: @hidden_var = hidden global i32
 @hidden_var = hidden global i32 1, align 4
 
+; Running the whole pipeline should remove unused global variables
+
+; CHECK: @aTile = internal addrspace(3) global
+; CHECK-LLC-NOT: @aTile
+@aTile = hidden addrspace(3) global [4 x [1 x i32]] zeroinitializer, align 4
+
+; CHECK: @bTile = internal addrspace(3) global
+; CHECK-LLC-NOT: @bTile
+@bTile = hidden addrspace(3) global [1 x [1 x i32]] zeroinitializer, align 4
+
+define void @anchor_function() #0 {
+entry:
+  %0 = load i32, ptr @switch.table, align 4
+  %1 = load [3 x float], ptr @private_array, align 4
+  %2 = load i32, ptr @private_var, align 4
+  %3 = load i32, ptr @internal_var, align 4
+  %4 = load i32, ptr @external_var, align 4
+  %5 = load i32, ptr @hidden_var, align 4
+  ret void
+}
+
 ; CHECK-NOT: define internal void @"?f1@@YAXXZ"()
 define void @"?f1@@YAXXZ"() #0 {
 entry:
diff --git a/llvm/test/CodeGen/DirectX/forward_handle_on_alloca.ll b/llvm/test/CodeGen/DirectX/forward_handle_on_alloca.ll
new file mode 100644
index 0000000000000..ce5c2d7ca32bf
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/forward_handle_on_alloca.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -dxil-forward-handle-accesses  %s | FileCheck %s  --check-prefixes=CHECK,FHCHECK
+; RUN: opt -S -mtriple=dxil--shadermodel6.3-compute -passes='function(dxil-forward-handle-accesses),dse' %s | FileCheck %s --check-prefix=CHECK
+
+; Note: test to confirm fix for issues: 140819 & 151764
+
+%"class.hlsl::RWStructuredBuffer" = type { target("dx.RawBuffer", i32, 1, 0) }
+@global = internal unnamed_addr global %"class.hlsl::RWStructuredBuffer" poison, align 4
+@name = private unnamed_addr constant [5 x i8] c"dest\00", align 1
+
+
+; NOTE: intent of this test is to confirm load target("dx.RawBuffer", i32, 1, 0)
+;       is replaced with call @llvm.dx.resource.getpointer
+define void @CSMain() local_unnamed_addr {
+; CHECK-LABEL: define void @CSMain() local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; FHCHECK-NEXT:    [[AGG_TMP_I1_SROA_0:%.*]] = alloca target("dx.RawBuffer", i32, 1, 0), align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = tail call target("dx.RawBuffer", i32, 1, 0) @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_i32_1_0t(i32 0, i32 3, i32 1, i32 0, i1 false, ptr nonnull @name)
+; CHECK-NEXT:    store target("dx.RawBuffer", i32, 1, 0) [[TMP0]], ptr @global, align 4
+; FHCHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @global, align 4
+; FHCHECK-NEXT:    store i32 [[TMP2]], ptr [[AGG_TMP_I1_SROA_0]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i32_1_0t(target("dx.RawBuffer", i32, 1, 0) [[TMP0]], i32 0)
+; CHECK-NEXT:    store i32 0, ptr [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %alloca = alloca target("dx.RawBuffer", i32, 1, 0), align 8
+  %handle  = tail call target("dx.RawBuffer", i32, 1, 0) @llvm.dx.resource.handlefrombinding.tdx.RawBuffer_i32_1_0t(i32 0, i32 3, i32 1, i32 0, i1 false, ptr nonnull @name)
+  store target("dx.RawBuffer", i32, 1, 0) %handle , ptr @global, align 4
+  %val  = load i32, ptr @global, align 4
+  call void @llvm.lifetime.start.p0(ptr nonnull %alloca)
+  store i32 %val , ptr  %alloca, align 8
+  %indirect = load target("dx.RawBuffer", i32, 1, 0), ptr  %alloca, align 8
+  %buff = tail call noundef nonnull align 4 dereferenceable(4) ptr @llvm.dx.resource.getpointer.p0.tdx.RawBuffer_i32_1_0t(target("dx.RawBuffer", i32, 1, 0) %indirect, i32 0)
+  store i32 0, ptr %buff, align 4
+  call void @llvm.lifetime.end.p0(ptr nonnull %alloca)
+  ret void
+}
diff --git a/llvm/test/CodeGen/DirectX/imad.ll b/llvm/test/CodeGen/DirectX/imad.ll
index 5d9463d658cf5..2e612f0182ddc 100644
--- a/llvm/test/CodeGen/DirectX/imad.ll
+++ b/llvm/test/CodeGen/DirectX/imad.ll
@@ -1,17 +1,13 @@
-; RUN: opt -S -dxil-op-lower < %s | FileCheck %s
+; RUN: opt -S -scalarizer -dxil-op-lower < %s | FileCheck %s
 
 ; Make sure dxil operation function calls for round are generated for float and half.
-; CHECK:call i16 @dx.op.tertiary.i16(i32 48, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR:]]
-; CHECK:call i32 @dx.op.tertiary.i32(i32 48, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
-; CHECK:call i64 @dx.op.tertiary.i64(i32 48, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
-
-; CHECK: attributes #[[#ATTR]] = {{{.*}} memory(none) {{.*}}}
 
 target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
 target triple = "dxil-pc-shadermodel6.7-library"
 ; Function Attrs: noinline nounwind optnone
 define noundef i16 @imad_short(i16 noundef %p0, i16 noundef %p1, i16 noundef %p2) #0 {
 entry:
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 48, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR:]]
   %p2.addr = alloca i16, align 2
   %p1.addr = alloca i16, align 2
   %p0.addr = alloca i16, align 2
@@ -31,6 +27,7 @@ declare i16 @llvm.dx.imad.i16(i16, i16, i16) #1
 ; Function Attrs: noinline nounwind optnone
 define noundef i32 @imad_int(i32 noundef %p0, i32 noundef %p1, i32 noundef %p2) #0 {
 entry:
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 48, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
   %p2.addr = alloca i32, align 4
   %p1.addr = alloca i32, align 4
   %p0.addr = alloca i32, align 4
@@ -50,6 +47,7 @@ declare i32 @llvm.dx.imad.i32(i32, i32, i32) #1
 ; Function Attrs: noinline nounwind optnone
 define noundef i64 @imad_int64(i64 noundef %p0, i64 noundef %p1, i64 noundef %p2) #0 {
 entry:
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 48, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
   %p2.addr = alloca i64, align 8
   %p1.addr = alloca i64, align 8
   %p0.addr = alloca i64, align 8
@@ -65,3 +63,95 @@ entry:
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn
 declare i64 @llvm.dx.imad.i64(i64, i64, i64) #1
+
+; Function Attrs: noinline nounwind optnone
+define noundef <4 x i16> @imad_int16_t4(<4 x i16> noundef %p0, <4 x i16> noundef %p1, <4 x i16> noundef %p2) #0 {
+entry:
+  ; CHECK: extractelement <4 x i16> %p0, i64 0
+  ; CHECK: extractelement <4 x i16> %p1, i64 0
+  ; CHECK: extractelement <4 x i16> %p2, i64 0
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 48, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i16> %p0, i64 1
+  ; CHECK: extractelement <4 x i16> %p1, i64 1
+  ; CHECK: extractelement <4 x i16> %p2, i64 1
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 48, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i16> %p0, i64 2
+  ; CHECK: extractelement <4 x i16> %p1, i64 2
+  ; CHECK: extractelement <4 x i16> %p2, i64 2
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 48, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i16> %p0, i64 3
+  ; CHECK: extractelement <4 x i16> %p1, i64 3
+  ; CHECK: extractelement <4 x i16> %p2, i64 3
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 48, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR]]
+  ; CHECK: insertelement <4 x i16> poison, i16 %{{.*}}, i64 0
+  ; CHECK: insertelement <4 x i16> %{{.*}}, i16 %{{.*}}, i64 1
+  ; CHECK: insertelement <4 x i16> %{{.*}}, i16 %{{.*}}, i64 2
+  ; CHECK: insertelement <4 x i16> %{{.*}}, i16 %{{.*}}, i64 3
+  %dx.imad = call <4 x i16> @llvm.dx.imad.v4i16(<4 x i16> %p0, <4 x i16> %p1, <4 x i16> %p2)
+  ret <4 x i16> %dx.imad
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare <4 x i16> @llvm.dx.imad.v4i16(<4 x i16>, <4 x i16>, <4 x i16>) #1
+
+; Function Attrs: noinline nounwind optnone
+define noundef <4 x i32> @imad_int4(<4 x i32> noundef %p0, <4 x i32> noundef %p1, <4 x i32> noundef %p2) #0 {
+entry:
+  ; CHECK: extractelement <4 x i32> %p0, i64 0
+  ; CHECK: extractelement <4 x i32> %p1, i64 0
+  ; CHECK: extractelement <4 x i32> %p2, i64 0
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 48, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i32> %p0, i64 1
+  ; CHECK: extractelement <4 x i32> %p1, i64 1
+  ; CHECK: extractelement <4 x i32> %p2, i64 1
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 48, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i32> %p0, i64 2
+  ; CHECK: extractelement <4 x i32> %p1, i64 2
+  ; CHECK: extractelement <4 x i32> %p2, i64 2
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 48, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i32> %p0, i64 3
+  ; CHECK: extractelement <4 x i32> %p1, i64 3
+  ; CHECK: extractelement <4 x i32> %p2, i64 3
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 48, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
+  ; CHECK: insertelement <4 x i32> poison, i32 %{{.*}}, i64 0
+  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i64 1
+  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i64 2
+  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i64 3
+  %dx.imad = call <4 x i32> @llvm.dx.imad.v4i32(<4 x i32> %p0, <4 x i32> %p1, <4 x i32> %p2)
+  ret <4 x i32> %dx.imad
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare <4 x i32> @llvm.dx.imad.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) #1
+
+; Function Attrs: noinline nounwind optnone
+define noundef <4 x i64> @imad_int64_t4(<4 x i64> noundef %p0, <4 x i64> noundef %p1, <4 x i64> noundef %p2) #0 {
+entry:
+  ; CHECK: extractelement <4 x i64> %p0, i64 0
+  ; CHECK: extractelement <4 x i64> %p1, i64 0
+  ; CHECK: extractelement <4 x i64> %p2, i64 0
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 48, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i64> %p0, i64 1
+  ; CHECK: extractelement <4 x i64> %p1, i64 1
+  ; CHECK: extractelement <4 x i64> %p2, i64 1
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 48, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i64> %p0, i64 2
+  ; CHECK: extractelement <4 x i64> %p1, i64 2
+  ; CHECK: extractelement <4 x i64> %p2, i64 2
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 48, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i64> %p0, i64 3
+  ; CHECK: extractelement <4 x i64> %p1, i64 3
+  ; CHECK: extractelement <4 x i64> %p2, i64 3
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 48, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
+  ; CHECK: insertelement <4 x i64> poison, i64 %{{.*}}, i64 0
+  ; CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i64 1
+  ; CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i64 2
+  ; CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i64 3
+  %dx.imad = call <4 x i64> @llvm.dx.imad.v4i64(<4 x i64> %p0, <4 x i64> %p1, <4 x i64> %p2)
+  ret <4 x i64> %dx.imad
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare <4 x i64> @llvm.dx.imad.v4i64(<4 x i64>, <4 x i64>, <4 x i64>) #1
+
+; CHECK: attributes #[[#ATTR]] = {{{.*}} memory(none) {{.*}}}
diff --git a/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.5.ll b/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.5.ll
index e485fa20ddfce..b1eea30cecf09 100644
--- a/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.5.ll
+++ b/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.5.ll
@@ -11,9 +11,9 @@
 define void @test_legal_lifetime()  {
   %accum.i.flat = alloca [1 x i32], align 4
   %gep = getelementptr i32, ptr %accum.i.flat, i32 0
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %accum.i.flat)
+  call void @llvm.lifetime.start.p0(ptr nonnull %accum.i.flat)
   store i32 0, ptr %gep, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %accum.i.flat)
+  call void @llvm.lifetime.end.p0(ptr nonnull %accum.i.flat)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.6.ll b/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.6.ll
index 77133eb729bdc..256fcc04dcdf2 100644
--- a/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.6.ll
+++ b/llvm/test/CodeGen/DirectX/legalize-lifetimes-valver-1.6.ll
@@ -13,12 +13,12 @@
 ; CHECK-NEXT:       [[ACCUM_I_FLAT:%.*]] = alloca [1 x i32], align 4
 ; CHECK-NEXT:       [[GEP:%.*]] = getelementptr i32, ptr [[ACCUM_I_FLAT]], i32 0
 ; CHECK-SM63-NEXT:  store [1 x i32] undef, ptr [[ACCUM_I_FLAT]], align 4
-; CHECK-SM66-NEXT:  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
+; CHECK-SM66-NEXT:  call void @llvm.lifetime.start.p0(ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-EMBED-NOT:  bitcast
 ; CHECK-EMBED-NOT:  lifetime
 ; CHECK-NEXT:       store i32 0, ptr [[GEP]], align 4
 ; CHECK-SM63-NEXT:  store [1 x i32] undef, ptr [[ACCUM_I_FLAT]], align 4
-; CHECK-SM66-NEXT:  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
+; CHECK-SM66-NEXT:  call void @llvm.lifetime.end.p0(ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-EMBED-NOT:  bitcast
 ; CHECK-EMBED-NOT:  lifetime
 ; CHECK-NEXT:       ret void
@@ -26,9 +26,9 @@
 define void @test_legal_lifetime()  {
   %accum.i.flat = alloca [1 x i32], align 4
   %gep = getelementptr i32, ptr %accum.i.flat, i32 0
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %accum.i.flat)
+  call void @llvm.lifetime.start.p0(ptr nonnull %accum.i.flat)
   store i32 0, ptr %gep, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %accum.i.flat)
+  call void @llvm.lifetime.end.p0(ptr nonnull %accum.i.flat)
   ret void
 }
 
diff --git a/llvm/test/CodeGen/DirectX/legalize-memset.ll b/llvm/test/CodeGen/DirectX/legalize-memset.ll
index a73e7378cfb9f..ad45ac67cda00 100644
--- a/llvm/test/CodeGen/DirectX/legalize-memset.ll
+++ b/llvm/test/CodeGen/DirectX/legalize-memset.ll
@@ -5,18 +5,14 @@ define void @replace_float_memset_test() #0 {
 ; CHECK-LABEL: define void @replace_float_memset_test(
 ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    [[ACCUM_I_FLAT:%.*]] = alloca [2 x float], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [2 x float], ptr [[ACCUM_I_FLAT]], i32 0, i32 0
 ; CHECK-NEXT:    store float 0.000000e+00, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr [2 x float], ptr [[ACCUM_I_FLAT]], i32 0, i32 1
 ; CHECK-NEXT:    store float 0.000000e+00, ptr [[GEP1]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    ret void
 ;
   %accum.i.flat = alloca [2 x float], align 4
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %accum.i.flat)
   call void @llvm.memset.p0.i32(ptr nonnull align 4 dereferenceable(8) %accum.i.flat, i8 0, i32 8, i1 false)
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %accum.i.flat)
   ret void
 }
 
@@ -24,18 +20,14 @@ define void @replace_half_memset_test() #0 {
 ; CHECK-LABEL: define void @replace_half_memset_test(
 ; CHECK-SAME: ) #[[ATTR0]] {
 ; CHECK-NEXT:    [[ACCUM_I_FLAT:%.*]] = alloca [2 x half], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [2 x half], ptr [[ACCUM_I_FLAT]], i32 0, i32 0
 ; CHECK-NEXT:    store half 0xH0000, ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr [2 x half], ptr [[ACCUM_I_FLAT]], i32 0, i32 1
 ; CHECK-NEXT:    store half 0xH0000, ptr [[GEP1]], align 2
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    ret void
 ;
   %accum.i.flat = alloca [2 x half], align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %accum.i.flat)
   call void @llvm.memset.p0.i32(ptr nonnull align 4 dereferenceable(8) %accum.i.flat, i8 0, i32 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %accum.i.flat)
   ret void
 }
 
@@ -43,18 +35,14 @@ define void @replace_double_memset_test() #0 {
 ; CHECK-LABEL: define void @replace_double_memset_test(
 ; CHECK-SAME: ) #[[ATTR0]] {
 ; CHECK-NEXT:    [[ACCUM_I_FLAT:%.*]] = alloca [2 x double], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [2 x double], ptr [[ACCUM_I_FLAT]], i32 0, i32 0
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr [2 x double], ptr [[ACCUM_I_FLAT]], i32 0, i32 1
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP1]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    ret void
 ;
   %accum.i.flat = alloca [2 x double], align 4
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %accum.i.flat)
   call void @llvm.memset.p0.i32(ptr nonnull align 4 dereferenceable(8) %accum.i.flat, i8 0, i32 16, i1 false)
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %accum.i.flat)
   ret void
 }
 
@@ -62,18 +50,14 @@ define void @replace_int16_memset_test() #0 {
 ; CHECK-LABEL: define void @replace_int16_memset_test(
 ; CHECK-SAME: ) #[[ATTR0]] {
 ; CHECK-NEXT:    [[CACHE_I:%.*]] = alloca [2 x i16], align 2
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[CACHE_I]])
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [2 x i16], ptr [[CACHE_I]], i32 0, i32 0
 ; CHECK-NEXT:    store i16 0, ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr [2 x i16], ptr [[CACHE_I]], i32 0, i32 1
 ; CHECK-NEXT:    store i16 0, ptr [[GEP1]], align 2
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[CACHE_I]])
 ; CHECK-NEXT:    ret void
 ;
   %cache.i = alloca [2 x i16], align 2
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cache.i)
   call void @llvm.memset.p0.i32(ptr nonnull align 2 dereferenceable(4) %cache.i, i8 0, i32 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cache.i)
   ret void
 }
 
@@ -81,16 +65,12 @@ define void @replace_int_memset_test() #0 {
 ; CHECK-LABEL: define void @replace_int_memset_test(
 ; CHECK-SAME: ) #[[ATTR0]] {
 ; CHECK-NEXT:    [[ACCUM_I_FLAT:%.*]] = alloca [1 x i32], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [1 x i32], ptr [[ACCUM_I_FLAT]], i32 0, i32 0
 ; CHECK-NEXT:    store i32 0, ptr [[GEP]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    ret void
 ;
   %accum.i.flat = alloca [1 x i32], align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %accum.i.flat)
   call void @llvm.memset.p0.i32(ptr nonnull align 4 dereferenceable(8) %accum.i.flat, i8 0, i32 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %accum.i.flat)
   ret void
 }
 
@@ -101,25 +81,19 @@ define void @replace_int_memset_to_var_test() #0 {
 ; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 1, ptr [[I]], align 4
 ; CHECK-NEXT:    [[I8_LOAD:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [1 x i32], ptr [[ACCUM_I_FLAT]], i32 0, i32 0
 ; CHECK-NEXT:    store i32 [[I8_LOAD]], ptr [[GEP]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[ACCUM_I_FLAT]])
 ; CHECK-NEXT:    ret void
 ;
   %accum.i.flat = alloca [1 x i32], align 4
   %i = alloca i8, align 4
   store i8 1, ptr %i
   %i8.load = load i8, ptr %i
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %accum.i.flat)
   call void @llvm.memset.p0.i32(ptr nonnull align 4 dereferenceable(8) %accum.i.flat, i8 %i8.load, i32 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %accum.i.flat)
   ret void
 }
 
 attributes #0 = {"hlsl.export"}
 
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none))
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none))
 declare void @llvm.memset.p0.i32(ptr writeonly captures(none), i8, i32, i1 immarg)
diff --git a/llvm/test/CodeGen/DirectX/llc-pipeline.ll b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
index 151603a7161c5..13c25396ea98f 100644
--- a/llvm/test/CodeGen/DirectX/llc-pipeline.ll
+++ b/llvm/test/CodeGen/DirectX/llc-pipeline.ll
@@ -7,13 +7,14 @@
 ; CHECK-NEXT: Target Library Information
 ; CHECK-NEXT: DXIL Resource Type Analysis
 ; CHECK-NEXT: Target Transform Information
-
+; CHECK-NEXT: Assumption Cache Tracker
 ; CHECK-OBJ-NEXT: Machine Module Information
 ; CHECK-OBJ-NEXT: Machine Branch Probability Analysis
 ; CHECK-OBJ-NEXT: Create Garbage Collector Module Metadata
 
 ; CHECK-NEXT: ModulePass Manager
 ; CHECK-NEXT:   DXIL Finalize Linkage
+; CHECK-NEXT:   Dead Global Elimination
 ; CHECK-NEXT:   FunctionPass Manager
 ; CHECK-NEXT:     DXIL Resource Access
 ; CHECK-NEXT:   DXIL Intrinsic Expansion
@@ -26,6 +27,13 @@
 ; CHECK-NEXT:   FunctionPass Manager
 ; CHECK-NEXT:     Dominator Tree Construction
 ; CHECK-NEXT:     DXIL Forward Handle Accesses
+; CHECK-NEXT:     Dominator Tree Construction
+; CHECK-NEXT:     Basic Alias Analysis (stateless AA impl)
+; CHECK-NEXT:     Function Alias Analysis Results
+; CHECK-NEXT:     Post-Dominator Tree Construction
+; CHECK-NEXT:     Memory SSA
+; CHECK-NEXT:     Natural Loop Information
+; CHECK-NEXT:     Dead Store Elimination
 ; CHECK-NEXT:     DXIL Legalizer
 ; CHECK-NEXT:   DXIL Resource Binding Analysis
 ; CHECK-NEXT:   DXIL Resource Implicit Binding
@@ -33,9 +41,9 @@
 ; CHECK-NEXT:   DXIL Module Metadata analysis
 ; CHECK-NEXT:   DXIL Shader Flag Analysis
 ; CHECK-NEXT:   DXIL Translate Metadata
+; CHECK-NEXT:   DXIL Root Signature Analysis
 ; CHECK-NEXT:   DXIL Post Optimization Validation
 ; CHECK-NEXT:   DXIL Op Lowering
-; CHECK-NEXT:   DXIL Root Signature Analysis
 ; CHECK-NEXT:   DXIL Prepare Module
 
 ; CHECK-ASM-NEXT: DXIL Metadata Pretty Printer
diff --git a/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-cbuffer-range.ll b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-cbuffer-range.ll
new file mode 100644
index 0000000000000..e420225229919
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-cbuffer-range.ll
@@ -0,0 +1,15 @@
+; RUN: not opt -S -passes='dxil-post-optimization-validation' -mtriple=dxil-pc-shadermodel6.6-compute %s 2>&1 | FileCheck %s
+; CHECK: error: resource CBV (space=0, registers=[2, 2]) overlaps with resource CBV (space=0, registers=[0, 2])
+
+define void @CSMain() "hlsl.shader"="compute" {
+entry:
+  ret void
+}
+
+!dx.rootsignatures = !{!0}
+
+!0 = !{ptr @CSMain, !1, i32 2}
+!1 = !{!2, !3}
+!2 = !{!"RootConstants", i32 0, i32 2, i32 0, i32 4}
+!3 = !{!"DescriptorTable", i32 0, !4}
+!4 = !{!"CBV", i32 3, i32 0, i32 0, i32 -1, i32 4}
diff --git a/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-descriptor-table-range.ll b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-descriptor-table-range.ll
new file mode 100644
index 0000000000000..037f8c71f8ef0
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-descriptor-table-range.ll
@@ -0,0 +1,16 @@
+; RUN: not opt -S -passes='dxil-post-optimization-validation' -mtriple=dxil-pc-shadermodel6.6-compute %s 2>&1 | FileCheck %s
+; CHECK: error: resource UAV (space=10, registers=[4294967295, 4294967295]) overlaps with resource UAV (space=10, registers=[4294967295, 4294967295])
+
+define void @CSMain() "hlsl.shader"="compute" {
+entry:
+  ret void
+}
+
+!dx.rootsignatures = !{!0}
+
+!0 = !{ptr @CSMain, !1, i32 2}
+!1 = !{!2, !4}
+!2 = !{!"DescriptorTable", i32 0, !3}
+!3 = !{!"UAV", i32 -1, i32 -1, i32 10, i32 -1, i32 2}
+!4 = !{!"DescriptorTable", i32 0, !5}
+!5 = !{ !"UAV", i32 -1, i32 -1, i32 10, i32 5, i32 2 }
diff --git a/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-root-descriptor-range.ll b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-root-descriptor-range.ll
new file mode 100644
index 0000000000000..7098efbb43f6a
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-root-descriptor-range.ll
@@ -0,0 +1,15 @@
+; RUN: not opt -S -passes='dxil-post-optimization-validation' -mtriple=dxil-pc-shadermodel6.6-compute %s 2>&1 | FileCheck %s
+; CHECK: error: resource UAV (space=1, registers=[3, 3]) overlaps with resource UAV (space=1, registers=[0, 3])
+
+define void @CSMain() "hlsl.shader"="compute" {
+entry:
+  ret void
+}
+
+!dx.rootsignatures = !{!0}
+
+!0 = !{ptr @CSMain, !1, i32 2}
+!1 = !{!2, !4}
+!2 = !{!"RootUAV", i32 0, i32 3, i32 1, i32 4}
+!4 = !{!"DescriptorTable", i32 0, !5}
+!5 = !{!"UAV", i32 4, i32 0, i32 1, i32 -1, i32 2}
diff --git a/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-sampler.ll b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-sampler.ll
new file mode 100644
index 0000000000000..c244095520468
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-sampler.ll
@@ -0,0 +1,15 @@
+; RUN: not opt -S -passes='dxil-post-optimization-validation' -mtriple=dxil-pc-shadermodel6.6-compute %s 2>&1 | FileCheck %s
+; CHECK: error: resource Sampler (space=0, registers=[42, 42]) overlaps with resource Sampler (space=0, registers=[42, 42])
+
+define void @CSMain() "hlsl.shader"="compute" {
+entry:
+  ret void
+}
+
+!dx.rootsignatures = !{!0}
+
+!0 = !{ptr @CSMain, !1, i32 2}
+!1 = !{!2, !3}
+!2 = !{ !"StaticSampler", i32 5, i32 4, i32 5, i32 3, float 0x3FF7CCCCC0000000, i32 10, i32 2, i32 1, float -1.270000e+02, float 1.220000e+02, i32 42, i32 0, i32 0 }
+!3 = !{!"DescriptorTable", i32 0, !4}
+!4 = !{!"Sampler", i32 1, i32 42, i32 0, i32 -1, i32 0}
diff --git a/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-static-sampler-range.ll b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-static-sampler-range.ll
new file mode 100644
index 0000000000000..9ac02ebbc0965
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/rootsignature-validation-fail-static-sampler-range.ll
@@ -0,0 +1,14 @@
+; RUN: not opt -S -passes='dxil-post-optimization-validation' -mtriple=dxil-pc-shadermodel6.6-compute %s 2>&1 | FileCheck %s
+; CHECK: error: resource Sampler (space=0, registers=[42, 42]) overlaps with resource Sampler (space=0, registers=[42, 42])
+
+define void @CSMain() "hlsl.shader"="compute" {
+entry:
+  ret void
+}
+
+!dx.rootsignatures = !{!0}
+
+!0 = !{ptr @CSMain, !1, i32 2}
+!1 = !{!2, !3}
+!2 = !{ !"StaticSampler", i32 5, i32 4, i32 5, i32 3, float 0x3FF7CCCCC0000000, i32 10, i32 2, i32 1, float -1.270000e+02, float 1.220000e+02, i32 42, i32 0, i32 0 }
+!3 = !{ !"StaticSampler", i32 4, i32 2, i32 3, i32 5, float 0x3FF6CCCCC0000000, i32 9, i32 3, i32 2, float -1.280000e+02, float 1.280000e+02, i32 42, i32 0, i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/rootsignature-validation.ll b/llvm/test/CodeGen/DirectX/rootsignature-validation.ll
new file mode 100644
index 0000000000000..0fdba27018cd4
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/rootsignature-validation.ll
@@ -0,0 +1,20 @@
+; RUN: opt -S -passes='dxil-post-optimization-validation' -mtriple=dxil-pc-shadermodel6.6-compute %s 
+; We have a valid root signature, this should compile successfully
+
+define void @CSMain() "hlsl.shader"="compute" {
+entry:
+  ret void
+}
+
+!dx.rootsignatures = !{!0}
+
+!0 = !{ptr @CSMain, !1, i32 2}
+!1 = !{!2, !3, !5, !7, !9}
+!2 = !{!"RootCBV", i32 0, i32 3, i32 1, i32 4}
+!9 = !{!"RootConstants", i32 0, i32 2, i32 0, i32 4}
+!3 = !{!"DescriptorTable", i32 0, !4}
+!4 = !{!"SRV", i32 1, i32 0, i32 0, i32 -1, i32 0}
+!5 = !{!"DescriptorTable", i32 0, !6}
+!6 = !{!"Sampler", i32 5, i32 3, i32 2, i32 -1, i32 0}
+!7 = !{!"DescriptorTable", i32 0, !8}
+!8 = !{!"UAV", i32 -1, i32 0, i32 0, i32 -1, i32 2}
diff --git a/llvm/test/CodeGen/DirectX/scalar-data.ll b/llvm/test/CodeGen/DirectX/scalar-data.ll
index 4861a0890f136..d9c8df9bc169c 100644
--- a/llvm/test/CodeGen/DirectX/scalar-data.ll
+++ b/llvm/test/CodeGen/DirectX/scalar-data.ll
@@ -1,4 +1,4 @@
-; RUN: llc %s -mtriple=dxil-pc-shadermodel6.3-library --filetype=asm -o - | FileCheck %s
+; RUN: opt -S -passes='dxil-data-scalarization,dxil-flatten-arrays' -mtriple=dxil-unknown-shadermodel6.5-compute %s | FileCheck %s
 
 ; Make sure we don't touch arrays without vectors and that can recurse and flatten multiple-dimension arrays of vectors
 
diff --git a/llvm/test/CodeGen/DirectX/umad.ll b/llvm/test/CodeGen/DirectX/umad.ll
index 104d2380af66b..76516a2a64637 100644
--- a/llvm/test/CodeGen/DirectX/umad.ll
+++ b/llvm/test/CodeGen/DirectX/umad.ll
@@ -1,17 +1,13 @@
-; RUN: opt -S -dxil-op-lower < %s | FileCheck %s
+; RUN: opt -S -scalarizer -dxil-op-lower < %s | FileCheck %s
 
 ; Make sure dxil operation function calls for round are generated for float and half.
-; CHECK:call i16 @dx.op.tertiary.i16(i32 49, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR:]]
-; CHECK:call i32 @dx.op.tertiary.i32(i32 49, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
-; CHECK:call i64 @dx.op.tertiary.i64(i32 49, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
-
-; CHECK: attributes #[[#ATTR]] = {{{.*}} memory(none) {{.*}}}
 
 target datalayout = "e-m:e-p:32:32-i1:32-i8:8-i16:16-i32:32-i64:64-f16:16-f32:32-f64:64-n8:16:32:64"
 target triple = "dxil-pc-shadermodel6.7-library"
 ; Function Attrs: noinline nounwind optnone
 define noundef i16 @umad_ushort(i16 noundef %p0, i16 noundef %p1, i16 noundef %p2) #0 {
 entry:
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 49, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR:]]
   %p2.addr = alloca i16, align 2
   %p1.addr = alloca i16, align 2
   %p0.addr = alloca i16, align 2
@@ -31,6 +27,7 @@ declare i16 @llvm.dx.umad.i16(i16, i16, i16) #1
 ; Function Attrs: noinline nounwind optnone
 define noundef i32 @umad_uint(i32 noundef %p0, i32 noundef %p1, i32 noundef %p2) #0 {
 entry:
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 49, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
   %p2.addr = alloca i32, align 4
   %p1.addr = alloca i32, align 4
   %p0.addr = alloca i32, align 4
@@ -50,6 +47,7 @@ declare i32 @llvm.dx.umad.i32(i32, i32, i32) #1
 ; Function Attrs: noinline nounwind optnone
 define noundef i64 @umad_uint64(i64 noundef %p0, i64 noundef %p1, i64 noundef %p2) #0 {
 entry:
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 49, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
   %p2.addr = alloca i64, align 8
   %p1.addr = alloca i64, align 8
   %p0.addr = alloca i64, align 8
@@ -65,3 +63,95 @@ entry:
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn
 declare i64 @llvm.dx.umad.i64(i64, i64, i64) #1
+
+; Function Attrs: noinline nounwind optnone
+define noundef <4 x i16> @umad_uint16_t4(<4 x i16> noundef %p0, <4 x i16> noundef %p1, <4 x i16> noundef %p2) #0 {
+entry:
+  ; CHECK: extractelement <4 x i16> %p0, i64 0
+  ; CHECK: extractelement <4 x i16> %p1, i64 0
+  ; CHECK: extractelement <4 x i16> %p2, i64 0
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 49, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i16> %p0, i64 1
+  ; CHECK: extractelement <4 x i16> %p1, i64 1
+  ; CHECK: extractelement <4 x i16> %p2, i64 1
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 49, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i16> %p0, i64 2
+  ; CHECK: extractelement <4 x i16> %p1, i64 2
+  ; CHECK: extractelement <4 x i16> %p2, i64 2
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 49, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i16> %p0, i64 3
+  ; CHECK: extractelement <4 x i16> %p1, i64 3
+  ; CHECK: extractelement <4 x i16> %p2, i64 3
+  ; CHECK: call i16 @dx.op.tertiary.i16(i32 49, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}) #[[#ATTR]]
+  ; CHECK: insertelement <4 x i16> poison, i16 %{{.*}}, i64 0
+  ; CHECK: insertelement <4 x i16> %{{.*}}, i16 %{{.*}}, i64 1
+  ; CHECK: insertelement <4 x i16> %{{.*}}, i16 %{{.*}}, i64 2
+  ; CHECK: insertelement <4 x i16> %{{.*}}, i16 %{{.*}}, i64 3
+  %dx.umad = call <4 x i16> @llvm.dx.umad.v4i16(<4 x i16> %p0, <4 x i16> %p1, <4 x i16> %p2)
+  ret <4 x i16> %dx.umad
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare <4 x i16> @llvm.dx.umad.v4i16(<4 x i16>, <4 x i16>, <4 x i16>) #1
+
+; Function Attrs: noinline nounwind optnone
+define noundef <4 x i32> @umad_uint4(<4 x i32> noundef %p0, <4 x i32> noundef %p1, <4 x i32> noundef %p2) #0 {
+entry:
+  ; CHECK: extractelement <4 x i32> %p0, i64 0
+  ; CHECK: extractelement <4 x i32> %p1, i64 0
+  ; CHECK: extractelement <4 x i32> %p2, i64 0
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 49, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i32> %p0, i64 1
+  ; CHECK: extractelement <4 x i32> %p1, i64 1
+  ; CHECK: extractelement <4 x i32> %p2, i64 1
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 49, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i32> %p0, i64 2
+  ; CHECK: extractelement <4 x i32> %p1, i64 2
+  ; CHECK: extractelement <4 x i32> %p2, i64 2
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 49, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i32> %p0, i64 3
+  ; CHECK: extractelement <4 x i32> %p1, i64 3
+  ; CHECK: extractelement <4 x i32> %p2, i64 3
+  ; CHECK: call i32 @dx.op.tertiary.i32(i32 49, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}) #[[#ATTR]]
+  ; CHECK: insertelement <4 x i32> poison, i32 %{{.*}}, i64 0
+  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i64 1
+  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i64 2
+  ; CHECK: insertelement <4 x i32> %{{.*}}, i32 %{{.*}}, i64 3
+  %dx.umad = call <4 x i32> @llvm.dx.umad.v4i32(<4 x i32> %p0, <4 x i32> %p1, <4 x i32> %p2)
+  ret <4 x i32> %dx.umad
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare <4 x i32> @llvm.dx.umad.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) #1
+
+; Function Attrs: noinline nounwind optnone
+define noundef <4 x i64> @umad_uint64_t4(<4 x i64> noundef %p0, <4 x i64> noundef %p1, <4 x i64> noundef %p2) #0 {
+entry:
+  ; CHECK: extractelement <4 x i64> %p0, i64 0
+  ; CHECK: extractelement <4 x i64> %p1, i64 0
+  ; CHECK: extractelement <4 x i64> %p2, i64 0
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 49, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i64> %p0, i64 1
+  ; CHECK: extractelement <4 x i64> %p1, i64 1
+  ; CHECK: extractelement <4 x i64> %p2, i64 1
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 49, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i64> %p0, i64 2
+  ; CHECK: extractelement <4 x i64> %p1, i64 2
+  ; CHECK: extractelement <4 x i64> %p2, i64 2
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 49, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
+  ; CHECK: extractelement <4 x i64> %p0, i64 3
+  ; CHECK: extractelement <4 x i64> %p1, i64 3
+  ; CHECK: extractelement <4 x i64> %p2, i64 3
+  ; CHECK: call i64 @dx.op.tertiary.i64(i32 49, i64 %{{.*}}, i64 %{{.*}}, i64 %{{.*}}) #[[#ATTR]]
+  ; CHECK: insertelement <4 x i64> poison, i64 %{{.*}}, i64 0
+  ; CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i64 1
+  ; CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i64 2
+  ; CHECK: insertelement <4 x i64> %{{.*}}, i64 %{{.*}}, i64 3
+  %dx.umad = call <4 x i64> @llvm.dx.umad.v4i64(<4 x i64> %p0, <4 x i64> %p1, <4 x i64> %p2)
+  ret <4 x i64> %dx.umad
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn
+declare <4 x i64> @llvm.dx.umad.v4i64(<4 x i64>, <4 x i64>, <4 x i64>) #1
+
+; CHECK: attributes #[[#ATTR]] = {{{.*}} memory(none) {{.*}}}
diff --git a/llvm/test/CodeGen/Generic/add-with-overflow-128.ll b/llvm/test/CodeGen/Generic/add-with-overflow-128.ll
index 389e6511b6c7c..3db446663e71d 100644
--- a/llvm/test/CodeGen/Generic/add-with-overflow-128.ll
+++ b/llvm/test/CodeGen/Generic/add-with-overflow-128.ll
@@ -1,8 +1,5 @@
 ; RUN: llc < %s
 
-; NVPTX fails to LowerFormalArguments for arg type i96
-; the arg byte size must be one of the {16, 8, 4, 2}
-; XFAIL: target=nvptx{{.*}}
 
 @ok = internal constant [4 x i8] c"%d\0A\00"
 @no = internal constant [4 x i8] c"no\0A\00"
diff --git a/llvm/test/CodeGen/Generic/half.ll b/llvm/test/CodeGen/Generic/half.ll
new file mode 100644
index 0000000000000..ef7bfe2f2d9ce
--- /dev/null
+++ b/llvm/test/CodeGen/Generic/half.ll
@@ -0,0 +1,88 @@
+; Simple cross-platform smoke checks for basic f16 operations.
+;
+; There shouldn't be any architectures that crash when trying to use `half`;
+; check that here. Additionally do a small handful of smoke tests that work
+; well cross-platform.
+
+; RUN: %if aarch64-registered-target     %{ llc %s -o - -mtriple=aarch64-apple-darwin            | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if aarch64-registered-target     %{ llc %s -o - -mtriple=aarch64-pc-windows-msvc         | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if aarch64-registered-target     %{ llc %s -o - -mtriple=aarch64-unknown-linux-gnu       | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if aarch64-registered-target     %{ llc %s -o - -mtriple=arm64ec-pc-windows-msvc         | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if amdgpu-registered-target      %{ llc %s -o - -mtriple=amdgcn-amd-amdhsa               | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if arc-registered-target         %{ llc %s -o - -mtriple=arc-elf                         | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if arm-registered-target         %{ llc %s -o - -mtriple=arm-unknown-linux-gnueabi       | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if arm-registered-target         %{ llc %s -o - -mtriple=thumbv7em-none-eabi             | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if avr-registered-target         %{ llc %s -o - -mtriple=avr-none                        | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if bpf-registered-target         %{ llc %s -o - -mtriple=bpfel                           | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if csky-registered-target        %{ llc %s -o - -mtriple=csky-unknown-linux-gnuabiv2     | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if csky-registered-target        %{ llc %s -o - -mtriple=csky-unknown-linux-gnuabiv2 -mcpu=ck860fv -mattr=+hard-float | FileCheck %s --check-prefixes=ALL,BAD %}
+; RUN: %if directx-registered-target     %{ llc %s -o - -mtriple=dxil-pc-shadermodel6.3-library  | FileCheck %s --check-prefixes=NOCRASH %}
+; RUN: %if hexagon-registered-target     %{ llc %s -o - -mtriple=hexagon-unknown-linux-musl      | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if lanai-registered-target       %{ llc %s -o - -mtriple=lanai-unknown-unknown           | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if loongarch-registered-target   %{ llc %s -o - -mtriple=loongarch32-unknown-linux-gnu   | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if loongarch-registered-target   %{ llc %s -o - -mtriple=loongarch64-unknown-linux-gnu   | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if loongarch-registered-target   %{ llc %s -o - -mtriple=loongarch64-unknown-linux-gnu -mattr=+f | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if m68k-registered-target        %{ llc %s -o - -mtriple=m68k-unknown-linux-gnu          | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if mips-registered-target        %{ llc %s -o - -mtriple=mips-unknown-linux-gnu          | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if mips-registered-target        %{ llc %s -o - -mtriple=mips64-unknown-linux-gnuabi64   | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if mips-registered-target        %{ llc %s -o - -mtriple=mips64el-unknown-linux-gnuabi64 | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if mips-registered-target        %{ llc %s -o - -mtriple=mipsel-unknown-linux-gnu        | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if msp430-registered-target      %{ llc %s -o - -mtriple=msp430-none-elf                 | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if nvptx-registered-target       %{ llc %s -o - -mtriple=nvptx64-nvidia-cuda             | FileCheck %s --check-prefixes=NOCRASH   %}
+; RUN: %if powerpc-registered-target     %{ llc %s -o - -mtriple=powerpc-unknown-linux-gnu       | FileCheck %s --check-prefixes=ALL,BAD   %}
+; RUN: %if powerpc-registered-target     %{ llc %s -o - -mtriple=powerpc64-unknown-linux-gnu     | FileCheck %s --check-prefixes=ALL,BAD   %}
+; RUN: %if powerpc-registered-target     %{ llc %s -o - -mtriple=powerpc64le-unknown-linux-gnu   | FileCheck %s --check-prefixes=ALL,BAD   %}
+; RUN: %if riscv-registered-target       %{ llc %s -o - -mtriple=riscv32-unknown-linux-gnu       | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if riscv-registered-target       %{ llc %s -o - -mtriple=riscv64-unknown-linux-gnu       | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if sparc-registered-target       %{ llc %s -o - -mtriple=sparc-unknown-linux-gnu         | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if sparc-registered-target       %{ llc %s -o - -mtriple=sparc64-unknown-linux-gnu       | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if spirv-registered-target       %{ llc %s -o - -mtriple=spirv-unknown-unknown           | FileCheck %s --check-prefixes=NOCRASH   %}
+; RUN: %if systemz-registered-target     %{ llc %s -o - -mtriple=s390x-unknown-linux-gnu         | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if ve-registered-target          %{ llc %s -o - -mtriple=ve-unknown-unknown              | FileCheck %s --check-prefixes=ALL,BAD   %}
+; RUN: %if webassembly-registered-target %{ llc %s -o - -mtriple=wasm32-unknown-unknown          | FileCheck %s --check-prefixes=ALL,BAD   %}
+; RUN: %if x86-registered-target         %{ llc %s -o - -mtriple=i686-unknown-linux-gnu          | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if x86-registered-target         %{ llc %s -o - -mtriple=x86_64-pc-windows-msvc          | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if x86-registered-target         %{ llc %s -o - -mtriple=x86_64-unknown-linux-gnu        | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if xcore-registered-target       %{ llc %s -o - -mtriple=xcore-unknown-unknown           | FileCheck %s --check-prefixes=ALL,CHECK %}
+; RUN: %if xtensa-registered-target      %{ llc %s -o - -mtriple=xtensa-none-elf                 | FileCheck %s --check-prefixes=ALL,CHECK %}
+
+; Note that arm64ec labels are quoted, hence the `{{"?}}:`.
+
+; Codegen tests don't work the same for graphics targets. Add a dummy directive
+; for filecheck, just make sure we don't crash.
+; NOCRASH: {{.*}}
+
+; All backends need to be able to bitcast without converting to another format,
+; so we assert against __extendhfsf2, __truncsfhf2, __gnu_{h2f,f2h}_ieee. This
+; doesn't catch issues on platforms with hardware f32<->f16, but those tend to
+; work better anyway.
+; Regression test for https://github.com/llvm/llvm-project/issues/97981.
+
+define half @from_bits(i16 %bits) nounwind {
+; ALL-LABEL: from_bits{{"?}}:
+; CHECK-NOT: __extend
+; CHECK-NOT: __trunc
+; CHECK-NOT: __gnu
+; BAD:       __extendhfsf2
+    %f = bitcast i16 %bits to half
+    ret half %f
+}
+
+define i16 @to_bits(half %f) nounwind {
+; ALL-LABEL: to_bits{{"?}}:
+; CHECK-NOT: __extend
+; CHECK-NOT: __trunc
+; CHECK-NOT: __gnu
+; BAD:       __truncsfhf2
+    %bits = bitcast half %f to i16
+    ret i16 %bits
+}
+
+; Some platforms have had problems freezing. Regression test for
+; https://github.com/llvm/llvm-project/issues/117337 and similar issues.
+
+define half @check_freeze(half %f) nounwind {
+; ALL-LABEL: check_freeze{{"?}}:
+  %t0 = freeze half %f
+  ret half %t0
+}
diff --git a/llvm/test/CodeGen/Generic/icmp-illegal.ll b/llvm/test/CodeGen/Generic/icmp-illegal.ll
index 1af42e3443d21..77dd5a59dfd1e 100644
--- a/llvm/test/CodeGen/Generic/icmp-illegal.ll
+++ b/llvm/test/CodeGen/Generic/icmp-illegal.ll
@@ -1,9 +1,5 @@
 ; RUN: llc < %s | FileCheck %s
 
-; NVPTX fails to LowerFormalArguments for arg size > i128
-; the arg byte size must be one of the {16, 8, 4, 2}
-; XFAIL: target=nvptx{{.*}}
-
 ; CHECK-LABEL: test_ult
 define i1 @test_ult(i256 %a) nounwind {
   %1 = icmp ult i256 %a, -6432394258550908438
diff --git a/llvm/test/CodeGen/Hexagon/cext-opt-block-addr.mir b/llvm/test/CodeGen/Hexagon/cext-opt-block-addr.mir
index 807afd7e304b4..5bc3bcc746cf2 100644
--- a/llvm/test/CodeGen/Hexagon/cext-opt-block-addr.mir
+++ b/llvm/test/CodeGen/Hexagon/cext-opt-block-addr.mir
@@ -76,8 +76,8 @@ frameInfo:
   hasTailCall:     true
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -141,8 +141,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/Hexagon/early-if-predicator.mir b/llvm/test/CodeGen/Hexagon/early-if-predicator.mir
index fa5206ca1589b..9373f5419654b 100644
--- a/llvm/test/CodeGen/Hexagon/early-if-predicator.mir
+++ b/llvm/test/CodeGen/Hexagon/early-if-predicator.mir
@@ -53,8 +53,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/Hexagon/hwloop-dist-check.mir b/llvm/test/CodeGen/Hexagon/hwloop-dist-check.mir
index 9f8c14a314309..29ea52e491621 100644
--- a/llvm/test/CodeGen/Hexagon/hwloop-dist-check.mir
+++ b/llvm/test/CodeGen/Hexagon/hwloop-dist-check.mir
@@ -138,8 +138,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -237,8 +237,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/Hexagon/machine-sink-float-usr.mir b/llvm/test/CodeGen/Hexagon/machine-sink-float-usr.mir
index fa05362b8e1e5..2f4df89e367da 100644
--- a/llvm/test/CodeGen/Hexagon/machine-sink-float-usr.mir
+++ b/llvm/test/CodeGen/Hexagon/machine-sink-float-usr.mir
@@ -154,8 +154,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -230,8 +230,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Hexagon/pipeliner/swp-phi-start.mir b/llvm/test/CodeGen/Hexagon/pipeliner/swp-phi-start.mir
index fbbb95a61f004..76e0fd491f6ad 100644
--- a/llvm/test/CodeGen/Hexagon/pipeliner/swp-phi-start.mir
+++ b/llvm/test/CodeGen/Hexagon/pipeliner/swp-phi-start.mir
@@ -100,8 +100,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/Hexagon/rdf-copy-clobber.mir b/llvm/test/CodeGen/Hexagon/rdf-copy-clobber.mir
index 3d1b4345dde11..8f7dae7eefcd4 100644
--- a/llvm/test/CodeGen/Hexagon/rdf-copy-clobber.mir
+++ b/llvm/test/CodeGen/Hexagon/rdf-copy-clobber.mir
@@ -68,8 +68,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 40, size: 8, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Hexagon/rdf-phi-clobber.mir b/llvm/test/CodeGen/Hexagon/rdf-phi-clobber.mir
index 5aebe730e55c4..706795a9f2c73 100644
--- a/llvm/test/CodeGen/Hexagon/rdf-phi-clobber.mir
+++ b/llvm/test/CodeGen/Hexagon/rdf-phi-clobber.mir
@@ -63,8 +63,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 entry_values:    []
 callSites:       []
 debugValueSubstitutions: []
diff --git a/llvm/test/CodeGen/LoongArch/calling-conv-preserve-most.ll b/llvm/test/CodeGen/LoongArch/calling-conv-preserve-most.ll
new file mode 100644
index 0000000000000..26c53d4ef824b
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/calling-conv-preserve-most.ll
@@ -0,0 +1,321 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=loongarch32 < %s | FileCheck %s -check-prefix=LA32
+; RUN: llc -mtriple=loongarch64 < %s | FileCheck %s -check-prefix=LA64
+
+; Check the PreserveMost calling convention works.
+
+declare void @standard_cc_func()
+declare preserve_mostcc void @preserve_mostcc_func()
+
+define preserve_mostcc void @preserve_mostcc1() nounwind {
+; LA32-LABEL: preserve_mostcc1:
+; LA32:       # %bb.0: # %entry
+; LA32-NEXT:    addi.w $sp, $sp, -64
+; LA32-NEXT:    st.w $ra, $sp, 60 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $a0, $sp, 56 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $a1, $sp, 52 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $a2, $sp, 48 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $a3, $sp, 44 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $a4, $sp, 40 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $a5, $sp, 36 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $a6, $sp, 32 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $a7, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $t4, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $t5, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $t6, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $t7, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    bl standard_cc_func
+; LA32-NEXT:    ld.w $t7, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $t6, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $t5, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $t4, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $a7, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $a6, $sp, 32 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $a5, $sp, 36 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $a4, $sp, 40 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $a3, $sp, 44 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $a2, $sp, 48 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $a1, $sp, 52 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $a0, $sp, 56 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 60 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 64
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: preserve_mostcc1:
+; LA64:       # %bb.0: # %entry
+; LA64-NEXT:    addi.d $sp, $sp, -112
+; LA64-NEXT:    st.d $ra, $sp, 104 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $a0, $sp, 96 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $a1, $sp, 88 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $a2, $sp, 80 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $a3, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $a4, $sp, 64 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $a5, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $a6, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $a7, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $t4, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $t5, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $t6, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $t7, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    pcaddu18i $ra, %call36(standard_cc_func)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    ld.d $t7, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $t6, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $t5, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $t4, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $a7, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $a6, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $a5, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $a4, $sp, 64 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $a3, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $a2, $sp, 80 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $a1, $sp, 88 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $a0, $sp, 96 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 104 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 112
+; LA64-NEXT:    ret
+entry:
+  call void @standard_cc_func()
+  ret void
+}
+
+define preserve_mostcc void @preserve_mostcc2() nounwind {
+; LA32-LABEL: preserve_mostcc2:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    bl preserve_mostcc_func
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: preserve_mostcc2:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -16
+; LA64-NEXT:    st.d $ra, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    pcaddu18i $ra, %call36(preserve_mostcc_func)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    ld.d $ra, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 16
+; LA64-NEXT:    ret
+  call preserve_mostcc void @preserve_mostcc_func()
+  ret void
+}
+
+define void @preserve_mostcc3() nounwind {
+; LA32-LABEL: preserve_mostcc3:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -16
+; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 4 # 4-byte Folded Spill
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $a0, $t0
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $a1, $t1
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $a2, $t2
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $a3, $t3
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $a4, $t8
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    bl preserve_mostcc_func
+; LA32-NEXT:    move $t0, $a0
+; LA32-NEXT:    move $t1, $a1
+; LA32-NEXT:    move $t2, $a2
+; LA32-NEXT:    move $t3, $a3
+; LA32-NEXT:    move $t8, $a4
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    ld.w $s1, $sp, 4 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 16
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: preserve_mostcc3:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -96
+; LA64-NEXT:    st.d $ra, $sp, 88 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 80 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s1, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs0, $sp, 64 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs1, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs2, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs3, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs4, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs5, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs6, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    fst.d $fs7, $sp, 8 # 8-byte Folded Spill
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $a0, $t0
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $a1, $t1
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $a2, $t2
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $a3, $t3
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $a4, $t8
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    pcaddu18i $ra, %call36(preserve_mostcc_func)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    move $t0, $a0
+; LA64-NEXT:    move $t1, $a1
+; LA64-NEXT:    move $t2, $a2
+; LA64-NEXT:    move $t3, $a3
+; LA64-NEXT:    move $t8, $a4
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    fld.d $fs7, $sp, 8 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs6, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs5, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs4, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs3, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs2, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs1, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    fld.d $fs0, $sp, 64 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s1, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 80 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 88 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 96
+; LA64-NEXT:    ret
+  %1 = call i32 asm sideeffect "", "={r12}"() nounwind
+  %2 = call i32 asm sideeffect "", "={r13}"() nounwind
+  %3 = call i32 asm sideeffect "", "={r14}"() nounwind
+  %4 = call i32 asm sideeffect "", "={r15}"() nounwind
+  %5 = call i32 asm sideeffect "", "={r20}"() nounwind
+  %6 = call i32 asm sideeffect "", "={r23}"() nounwind
+  %7 = call i32 asm sideeffect "", "={r24}"() nounwind
+  call preserve_mostcc void @preserve_mostcc_func()
+  call void asm sideeffect "", "{r12},{r13},{r14},{r15},{r20},{r23},{r24}"(i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7)
+  ret void
+}
+
+define void @preserve_mostcc4() nounwind {
+; LA32-LABEL: preserve_mostcc4:
+; LA32:       # %bb.0:
+; LA32-NEXT:    addi.w $sp, $sp, -32
+; LA32-NEXT:    st.w $ra, $sp, 28 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $fp, $sp, 24 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s0, $sp, 20 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s1, $sp, 16 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s2, $sp, 12 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s3, $sp, 8 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s4, $sp, 4 # 4-byte Folded Spill
+; LA32-NEXT:    st.w $s5, $sp, 0 # 4-byte Folded Spill
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $fp, $t0
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $s2, $t1
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $s3, $t2
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $s4, $t3
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    move $s5, $t8
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    bl standard_cc_func
+; LA32-NEXT:    move $t0, $fp
+; LA32-NEXT:    move $t1, $s2
+; LA32-NEXT:    move $t2, $s3
+; LA32-NEXT:    move $t3, $s4
+; LA32-NEXT:    move $t8, $s5
+; LA32-NEXT:    #APP
+; LA32-NEXT:    #NO_APP
+; LA32-NEXT:    ld.w $s5, $sp, 0 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s4, $sp, 4 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s3, $sp, 8 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s2, $sp, 12 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s1, $sp, 16 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $s0, $sp, 20 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $fp, $sp, 24 # 4-byte Folded Reload
+; LA32-NEXT:    ld.w $ra, $sp, 28 # 4-byte Folded Reload
+; LA32-NEXT:    addi.w $sp, $sp, 32
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: preserve_mostcc4:
+; LA64:       # %bb.0:
+; LA64-NEXT:    addi.d $sp, $sp, -80
+; LA64-NEXT:    st.d $ra, $sp, 72 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $fp, $sp, 64 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s0, $sp, 56 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s1, $sp, 48 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s2, $sp, 40 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s3, $sp, 32 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s4, $sp, 24 # 8-byte Folded Spill
+; LA64-NEXT:    st.d $s5, $sp, 16 # 8-byte Folded Spill
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $fp, $t0
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $s2, $t1
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $s3, $t2
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $s4, $t3
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    move $s5, $t8
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    pcaddu18i $ra, %call36(standard_cc_func)
+; LA64-NEXT:    jirl $ra, $ra, 0
+; LA64-NEXT:    move $t0, $fp
+; LA64-NEXT:    move $t1, $s2
+; LA64-NEXT:    move $t2, $s3
+; LA64-NEXT:    move $t3, $s4
+; LA64-NEXT:    move $t8, $s5
+; LA64-NEXT:    #APP
+; LA64-NEXT:    #NO_APP
+; LA64-NEXT:    ld.d $s5, $sp, 16 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s4, $sp, 24 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s3, $sp, 32 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s2, $sp, 40 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s1, $sp, 48 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $s0, $sp, 56 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $fp, $sp, 64 # 8-byte Folded Reload
+; LA64-NEXT:    ld.d $ra, $sp, 72 # 8-byte Folded Reload
+; LA64-NEXT:    addi.d $sp, $sp, 80
+; LA64-NEXT:    ret
+  %1 = call i32 asm sideeffect "", "={r12}"() nounwind
+  %2 = call i32 asm sideeffect "", "={r13}"() nounwind
+  %3 = call i32 asm sideeffect "", "={r14}"() nounwind
+  %4 = call i32 asm sideeffect "", "={r15}"() nounwind
+  %5 = call i32 asm sideeffect "", "={r20}"() nounwind
+  %6 = call i32 asm sideeffect "", "={r23}"() nounwind
+  %7 = call i32 asm sideeffect "", "={r24}"() nounwind
+  call void @standard_cc_func()
+  call void asm sideeffect "", "{r12},{r13},{r14},{r15},{r20},{r23},{r24}"(i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i32 %6, i32 %7)
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/fix-addsub-relocs-with-relax.ll b/llvm/test/CodeGen/LoongArch/fix-addsub-relocs-with-relax.ll
new file mode 100644
index 0000000000000..0861fcbfd6738
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/fix-addsub-relocs-with-relax.ll
@@ -0,0 +1,73 @@
+; RUN: llc --filetype=obj --mtriple=loongarch64 %s -o %t.o
+; RUN: llvm-readobj -r %t.o | FileCheck %s
+; RUN: llc --filetype=obj --mtriple=loongarch64 --mattr=+relax %s -o %t.r
+; RUN: llvm-readobj -r %t.r | FileCheck %s
+
+; CHECK:      Relocations [
+; CHECK-NEXT:   Section ({{.*}}) .rela.text {
+; CHECK-NEXT:     0x14 R_LARCH_PCALA_HI20 sym 0x0
+; CHECK-NEXT:     0x14 R_LARCH_RELAX - 0x0
+; CHECK-NEXT:     0x18 R_LARCH_PCALA_LO12 sym 0x0
+; CHECK-NEXT:     0x18 R_LARCH_RELAX - 0x0
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Section ({{.*}}) .rela.debug_info {
+; CHECK-NEXT:     0x8 R_LARCH_32 .debug_abbrev 0x0
+; CHECK-NEXT:     0x11 R_LARCH_32 .L0 0x0
+; CHECK-NEXT:     0x15 R_LARCH_32 .Lline_table_start0 0x0
+; CHECK-NEXT:     0x1B R_LARCH_ADD32 .L0 0x0
+; CHECK-NEXT:     0x1B R_LARCH_SUB32 .L0 0x0
+; CHECK-NEXT:     0x1F R_LARCH_32 .L0 0x0
+; CHECK-NEXT:     0x25 R_LARCH_ADD32 .L0 0x0
+; CHECK-NEXT:     0x25 R_LARCH_SUB32 .L0 0x0
+; CHECK-NEXT:   }
+; CHECK:        Section ({{.*}}) .rela.debug_frame {
+; CHECK-NEXT:     0x1C R_LARCH_32 .L0 0x0
+; CHECK-NEXT:     0x20 R_LARCH_64 .L0 0x0
+; CHECK-NEXT:     0x28 R_LARCH_ADD64 .L0 0x0
+; CHECK-NEXT:     0x28 R_LARCH_SUB64 .L0 0x0
+; CHECK-NEXT:     0x3F R_LARCH_ADD6 .L0 0x0
+; CHECK-NEXT:     0x3F R_LARCH_SUB6 .L0 0x0
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Section ({{.*}}) .rela.debug_line {
+; CHECK-NEXT:     0x22 R_LARCH_32 .debug_line_str 0x0
+; CHECK-NEXT:     0x31 R_LARCH_32 .debug_line_str 0x2
+; CHECK-NEXT:     0x46 R_LARCH_32 .debug_line_str 0x9
+; CHECK-NEXT:     0x4F R_LARCH_64 .L0 0x0
+; CHECK-NEXT:     0x5F R_LARCH_ADD16 .L0 0x0
+; CHECK-NEXT:     0x5F R_LARCH_SUB16 .L0 0x0
+; CHECK-NEXT:   }
+; CHECK-NEXT: ]
+
+; ModuleID = 'test.c'
+source_filename = "test.c"
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
+target triple = "loongarch64"
+
+; Function Attrs: noinline nounwind optnone
+define dso_local signext i32 @foo() #0 !dbg !8 {
+  call void asm sideeffect ".cfi_remember_state\0A\09.cfi_adjust_cfa_offset 16\0A\09nop\0A\09la.pcrel $$t0, sym\0A\09nop\0A\09.cfi_restore_state\0A\09", ""() #1, !dbg !12, !srcloc !13
+  ret i32 0, !dbg !14
+}
+
+attributes #0 = { noinline nounwind optnone "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="loongarch64" "target-features"="+64bit,+d,+f,+ual,+relax" }
+attributes #1 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: ".", checksumkind: CSK_MD5, checksum: "f44d6d71bc4da58b4abe338ca507c007", source: "int foo()\0A{\0A  asm volatile(\0A    \22.cfi_remember_state\\n\\t\22\0A    \22.cfi_adjust_cfa_offset 16\\n\\t\22\0A    \22nop\\n\\t\22\0A    \22la.pcrel $t0, sym\\n\\t\22\0A    \22nop\\n\\t\22\0A    \22.cfi_restore_state\\n\\t\22);\0A  return 0;\0A}\0A")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 7, !"direct-access-external-data", i32 0}
+!6 = !{i32 7, !"frame-pointer", i32 2}
+!7 = !{!"clang"}
+!8 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !9, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DILocation(line: 3, column: 3, scope: !8)
+!13 = !{i64 34, i64 56, i64 92, i64 106, i64 134, i64 148, i64 177}
+!14 = !DILocation(line: 10, column: 3, scope: !8)
diff --git a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll
index c18c637a05bc9..cc9ad633b4273 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/build-vector.ll
@@ -224,96 +224,144 @@ entry:
 define void @buildvector_v32i8(ptr %dst, i8 %a0, i8 %a1, i8 %a2, i8 %a3, i8 %a4, i8 %a5, i8 %a6, i8 %a7, i8 %a8, i8 %a9, i8 %a10, i8 %a11, i8 %a12, i8 %a13, i8 %a14, i8 %a15, i8 %a16, i8 %a17, i8 %a18, i8 %a19, i8 %a20, i8 %a21, i8 %a22, i8 %a23, i8 %a24, i8 %a25, i8 %a26, i8 %a27, i8 %a28, i8 %a29, i8 %a30, i8 %a31) nounwind {
 ; CHECK-LABEL: buildvector_v32i8:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi.d $sp, $sp, -80
+; CHECK-NEXT:    fst.d $fs0, $sp, 72 # 8-byte Folded Spill
+; CHECK-NEXT:    fst.d $fs1, $sp, 64 # 8-byte Folded Spill
+; CHECK-NEXT:    fst.d $fs2, $sp, 56 # 8-byte Folded Spill
+; CHECK-NEXT:    fst.d $fs3, $sp, 48 # 8-byte Folded Spill
+; CHECK-NEXT:    fst.d $fs4, $sp, 40 # 8-byte Folded Spill
+; CHECK-NEXT:    fst.d $fs5, $sp, 32 # 8-byte Folded Spill
+; CHECK-NEXT:    fst.d $fs6, $sp, 24 # 8-byte Folded Spill
+; CHECK-NEXT:    fst.d $fs7, $sp, 16 # 8-byte Folded Spill
 ; CHECK-NEXT:    vinsgr2vr.b $vr0, $a1, 0
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a2, 1
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a3, 2
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a4, 3
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a5, 4
-; CHECK-NEXT:    ld.b $a1, $sp, 0
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a6, 5
-; CHECK-NEXT:    ld.b $a2, $sp, 8
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a7, 6
-; CHECK-NEXT:    ld.b $a3, $sp, 16
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a1, 7
-; CHECK-NEXT:    ld.b $a1, $sp, 24
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a2, 8
-; CHECK-NEXT:    ld.b $a2, $sp, 32
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a3, 9
-; CHECK-NEXT:    ld.b $a3, $sp, 40
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a1, 10
-; CHECK-NEXT:    ld.b $a1, $sp, 48
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a2, 11
-; CHECK-NEXT:    ld.b $a2, $sp, 56
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a3, 12
-; CHECK-NEXT:    ld.b $a3, $sp, 64
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a1, 13
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a2, 14
-; CHECK-NEXT:    ld.b $a1, $sp, 72
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a3, 15
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a2
+; CHECK-NEXT:    xvreplgr2vr.b $xr2, $a3
+; CHECK-NEXT:    xvreplgr2vr.b $xr3, $a4
+; CHECK-NEXT:    ld.b $a1, $sp, 264
+; CHECK-NEXT:    xvreplgr2vr.b $xr4, $a5
 ; CHECK-NEXT:    ld.b $a2, $sp, 80
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 0
-; CHECK-NEXT:    ld.b $a1, $sp, 88
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a2, 1
-; CHECK-NEXT:    ld.b $a2, $sp, 96
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 2
-; CHECK-NEXT:    ld.b $a1, $sp, 104
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a2, 3
-; CHECK-NEXT:    ld.b $a2, $sp, 112
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 4
-; CHECK-NEXT:    ld.b $a1, $sp, 120
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a2, 5
+; CHECK-NEXT:    xvreplgr2vr.b $xr5, $a6
+; CHECK-NEXT:    ld.b $a3, $sp, 88
+; CHECK-NEXT:    xvreplgr2vr.b $xr6, $a7
+; CHECK-NEXT:    ld.b $a4, $sp, 96
+; CHECK-NEXT:    xvreplgr2vr.b $xr7, $a2
+; CHECK-NEXT:    ld.b $a2, $sp, 104
+; CHECK-NEXT:    xvreplgr2vr.b $xr8, $a3
+; CHECK-NEXT:    ld.b $a3, $sp, 112
+; CHECK-NEXT:    xvreplgr2vr.b $xr9, $a4
+; CHECK-NEXT:    ld.b $a4, $sp, 120
+; CHECK-NEXT:    xvreplgr2vr.b $xr10, $a2
 ; CHECK-NEXT:    ld.b $a2, $sp, 128
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 6
-; CHECK-NEXT:    ld.b $a1, $sp, 136
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a2, 7
-; CHECK-NEXT:    ld.b $a2, $sp, 144
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 8
-; CHECK-NEXT:    ld.b $a1, $sp, 152
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a2, 9
-; CHECK-NEXT:    ld.b $a2, $sp, 160
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 10
-; CHECK-NEXT:    ld.b $a1, $sp, 168
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a2, 11
+; CHECK-NEXT:    xvreplgr2vr.b $xr11, $a3
+; CHECK-NEXT:    ld.b $a3, $sp, 136
+; CHECK-NEXT:    xvreplgr2vr.b $xr12, $a4
+; CHECK-NEXT:    ld.b $a4, $sp, 144
+; CHECK-NEXT:    xvreplgr2vr.b $xr13, $a2
+; CHECK-NEXT:    ld.b $a2, $sp, 152
+; CHECK-NEXT:    xvreplgr2vr.b $xr14, $a3
+; CHECK-NEXT:    ld.b $a3, $sp, 160
+; CHECK-NEXT:    xvreplgr2vr.b $xr15, $a4
+; CHECK-NEXT:    ld.b $a4, $sp, 168
+; CHECK-NEXT:    xvreplgr2vr.b $xr16, $a2
 ; CHECK-NEXT:    ld.b $a2, $sp, 176
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 12
-; CHECK-NEXT:    ld.b $a1, $sp, 184
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a2, 13
-; CHECK-NEXT:    ld.b $a2, $sp, 192
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a1, 14
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a2, 15
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    xvreplgr2vr.b $xr17, $a3
+; CHECK-NEXT:    ld.b $a3, $sp, 184
+; CHECK-NEXT:    xvreplgr2vr.b $xr18, $a4
+; CHECK-NEXT:    ld.b $a4, $sp, 192
+; CHECK-NEXT:    xvreplgr2vr.b $xr19, $a2
+; CHECK-NEXT:    ld.b $a2, $sp, 200
+; CHECK-NEXT:    xvreplgr2vr.b $xr20, $a3
+; CHECK-NEXT:    ld.b $a3, $sp, 208
+; CHECK-NEXT:    xvreplgr2vr.b $xr21, $a4
+; CHECK-NEXT:    ld.b $a4, $sp, 216
+; CHECK-NEXT:    xvreplgr2vr.b $xr22, $a2
+; CHECK-NEXT:    ld.b $a2, $sp, 224
+; CHECK-NEXT:    xvreplgr2vr.b $xr23, $a3
+; CHECK-NEXT:    ld.b $a3, $sp, 232
+; CHECK-NEXT:    xvreplgr2vr.b $xr24, $a4
+; CHECK-NEXT:    ld.b $a4, $sp, 240
+; CHECK-NEXT:    xvreplgr2vr.b $xr25, $a2
+; CHECK-NEXT:    ld.b $a2, $sp, 248
+; CHECK-NEXT:    xvreplgr2vr.b $xr26, $a3
+; CHECK-NEXT:    ld.b $a3, $sp, 256
+; CHECK-NEXT:    xvreplgr2vr.b $xr27, $a4
+; CHECK-NEXT:    ld.b $a4, $sp, 272
+; CHECK-NEXT:    xvreplgr2vr.b $xr28, $a2
+; CHECK-NEXT:    xvreplgr2vr.b $xr29, $a3
+; CHECK-NEXT:    xvreplgr2vr.b $xr30, $a1
+; CHECK-NEXT:    xvreplgr2vr.b $xr31, $a4
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 17
+; CHECK-NEXT:    xvpermi.q $xr2, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr2, 34
+; CHECK-NEXT:    xvpermi.q $xr3, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr3, 51
+; CHECK-NEXT:    xvpermi.q $xr4, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr4, 68
+; CHECK-NEXT:    xvpermi.q $xr5, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr5, 85
+; CHECK-NEXT:    xvpermi.q $xr6, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr6, 102
+; CHECK-NEXT:    xvpermi.q $xr7, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr7, 119
+; CHECK-NEXT:    xvpermi.q $xr8, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr8, 136
+; CHECK-NEXT:    xvpermi.q $xr9, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr9, 153
+; CHECK-NEXT:    xvpermi.q $xr10, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr10, 170
+; CHECK-NEXT:    xvpermi.q $xr11, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr11, 187
+; CHECK-NEXT:    xvpermi.q $xr12, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr12, 204
+; CHECK-NEXT:    xvpermi.q $xr13, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr13, 221
+; CHECK-NEXT:    xvpermi.q $xr14, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr14, 238
+; CHECK-NEXT:    xvpermi.q $xr15, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr15, 255
+; CHECK-NEXT:    xvpermi.q $xr16, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr16, 0
+; CHECK-NEXT:    xvpermi.q $xr17, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr17, 17
+; CHECK-NEXT:    xvpermi.q $xr18, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr18, 34
+; CHECK-NEXT:    xvpermi.q $xr19, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr19, 51
+; CHECK-NEXT:    xvpermi.q $xr20, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr20, 68
+; CHECK-NEXT:    xvpermi.q $xr21, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr21, 85
+; CHECK-NEXT:    xvpermi.q $xr22, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr22, 102
+; CHECK-NEXT:    xvpermi.q $xr23, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr23, 119
+; CHECK-NEXT:    xvpermi.q $xr24, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr24, 136
+; CHECK-NEXT:    xvpermi.q $xr25, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr25, 153
+; CHECK-NEXT:    xvpermi.q $xr26, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr26, 170
+; CHECK-NEXT:    xvpermi.q $xr27, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr27, 187
+; CHECK-NEXT:    xvpermi.q $xr28, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr28, 204
+; CHECK-NEXT:    xvpermi.q $xr29, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr29, 221
+; CHECK-NEXT:    xvpermi.q $xr30, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr30, 238
+; CHECK-NEXT:    xvpermi.q $xr31, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr31, 255
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
+; CHECK-NEXT:    fld.d $fs7, $sp, 16 # 8-byte Folded Reload
+; CHECK-NEXT:    fld.d $fs6, $sp, 24 # 8-byte Folded Reload
+; CHECK-NEXT:    fld.d $fs5, $sp, 32 # 8-byte Folded Reload
+; CHECK-NEXT:    fld.d $fs4, $sp, 40 # 8-byte Folded Reload
+; CHECK-NEXT:    fld.d $fs3, $sp, 48 # 8-byte Folded Reload
+; CHECK-NEXT:    fld.d $fs2, $sp, 56 # 8-byte Folded Reload
+; CHECK-NEXT:    fld.d $fs1, $sp, 64 # 8-byte Folded Reload
+; CHECK-NEXT:    fld.d $fs0, $sp, 72 # 8-byte Folded Reload
+; CHECK-NEXT:    addi.d $sp, $sp, 80
 ; CHECK-NEXT:    ret
 entry:
   %ins0  = insertelement <32 x i8> undef,  i8 %a0,  i32 0
@@ -364,36 +412,48 @@ define void @buildvector_v32i8_partial(ptr %dst, i8 %a0, i8 %a1, i8 %a2, i8 %a5,
 ; CHECK-NEXT:    ld.b $t6, $sp, 8
 ; CHECK-NEXT:    ld.b $t7, $sp, 0
 ; CHECK-NEXT:    vinsgr2vr.b $vr0, $a1, 0
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a2, 1
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a3, 2
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a4, 5
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a5, 7
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a6, 8
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a7, 15
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t7, 1
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t6, 2
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t5, 4
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t4, 6
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t3, 7
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t2, 11
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t1, 12
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t0, 15
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 17
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a3
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 34
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a4
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 85
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a5
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 119
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a6
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 136
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a7
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 255
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t7
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 17
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t6
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 34
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t5
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 68
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t4
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 102
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t3
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 119
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 187
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t1
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 204
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t0
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 255
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -445,37 +505,51 @@ define void @buildvector_v32i8_with_constant(ptr %dst, i8 %a0, i8 %a1, i8 %a2, i
 ; CHECK-NEXT:    ld.b $t6, $sp, 8
 ; CHECK-NEXT:    ld.b $t7, $sp, 0
 ; CHECK-NEXT:    xvrepli.b $xr0, 0
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a1, 0
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a2, 1
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a3, 2
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a4, 5
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a5, 8
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a6, 9
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a7, 15
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t7, 1
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t6, 2
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t5, 4
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t4, 6
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t3, 7
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t2, 11
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t1, 12
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $t0, 15
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a1
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 0
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 17
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a3
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 34
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a4
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 85
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a5
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 136
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a6
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 153
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a7
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 255
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t7
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 17
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t6
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 34
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t5
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 68
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t4
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 102
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t3
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 119
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 187
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t1
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 204
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $t0
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 255
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -528,37 +602,51 @@ define void @buildvector_v16i16(ptr %dst, i16 %a0, i16 %a1, i16 %a2, i16 %a3, i1
 ; CHECK-NEXT:    ld.h $t7, $sp, 8
 ; CHECK-NEXT:    ld.h $t8, $sp, 0
 ; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 0
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a2, 1
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a3, 2
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a4, 3
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a5, 4
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a6, 5
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a7, 6
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $t8, 7
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $t7, 0
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $t6, 1
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $t5, 2
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $t4, 3
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $t3, 4
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $t2, 5
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $t1, 6
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $t0, 7
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 17
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a3
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 34
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a4
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 51
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a5
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 68
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a6
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 85
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a7
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 102
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $t8
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 119
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $t7
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 0
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $t6
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 17
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $t5
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 34
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $t4
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 51
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $t3
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 68
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $t2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 85
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $t1
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 102
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $t0
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 119
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -586,16 +674,24 @@ define void @buildvector_v16i16_partial(ptr %dst, i16 %a0, i16 %a2, i16 %a5, i16
 ; CHECK-LABEL: buildvector_v16i16_partial:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 0
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a2, 2
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a3, 5
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a4, 6
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a5, 7
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a6, 4
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a7, 5
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 34
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a3
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 85
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a4
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 102
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a5
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 119
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a6
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 68
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a7
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 85
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
@@ -623,17 +719,27 @@ define void @buildvector_v16i16_with_constant(ptr %dst, i16 %a2, i16 %a3, i16 %a
 ; CHECK-LABEL: buildvector_v16i16_with_constant:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xvrepli.h $xr0, 2
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 2
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a2, 3
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a3, 5
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a4, 6
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a5, 7
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a6, 4
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a7, 5
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a1
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 34
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 51
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a3
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 85
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a4
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 102
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a5
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 119
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a6
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 68
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a7
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 85
 ; CHECK-NEXT:    xvst $xr0, $a0, 0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll
index ac5a2143451d0..ca405314686e6 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insert-extract-element.ll
@@ -6,7 +6,9 @@ define <32 x i8> @insert_extract_v32i8(<32 x i8> %a) nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
 ; CHECK-NEXT:    vpickve2gr.b $a0, $vr1, 15
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a0, 1
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a0
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 17
 ; CHECK-NEXT:    ret
 entry:
   %b = extractelement <32 x i8> %a, i32 31
@@ -19,7 +21,9 @@ define <16 x i16> @insert_extract_v16i16(<16 x i16> %a) nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
 ; CHECK-NEXT:    vpickve2gr.h $a0, $vr1, 7
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 1
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a0
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 17
 ; CHECK-NEXT:    ret
 entry:
   %b = extractelement <16 x i16> %a, i32 15
@@ -30,8 +34,8 @@ entry:
 define <8 x i32> @insert_extract_v8i32(<8 x i32> %a) nounwind {
 ; CHECK-LABEL: insert_extract_v8i32:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xvpickve2gr.w $a0, $xr0, 7
-; CHECK-NEXT:    xvinsgr2vr.w $xr0, $a0, 1
+; CHECK-NEXT:    xvpickve.w $xr1, $xr0, 7
+; CHECK-NEXT:    xvinsve0.w $xr0, $xr1, 1
 ; CHECK-NEXT:    ret
 entry:
   %b = extractelement <8 x i32> %a, i32 7
@@ -39,6 +43,18 @@ entry:
   ret <8 x i32> %c
 }
 
+
+define <8 x i32> @insert_extract0_v8i32(<8 x i32> %a) nounwind {
+; CHECK-LABEL: insert_extract0_v8i32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvinsve0.w $xr0, $xr0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <8 x i32> %a, i32 0
+  %c = insertelement <8 x i32> %a, i32 %b, i32 1
+  ret <8 x i32> %c
+}
+
 define <8 x float> @insert_extract_v8f32(<8 x float> %a) nounwind {
 ; CHECK-LABEL: insert_extract_v8f32:
 ; CHECK:       # %bb.0: # %entry
@@ -54,8 +70,8 @@ entry:
 define <4 x i64> @insert_extract_v4i64(<4 x i64> %a) nounwind {
 ; CHECK-LABEL: insert_extract_v4i64:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 3
-; CHECK-NEXT:    xvinsgr2vr.d $xr0, $a0, 1
+; CHECK-NEXT:    xvpickve.d $xr1, $xr0, 3
+; CHECK-NEXT:    xvinsve0.d $xr0, $xr1, 1
 ; CHECK-NEXT:    ret
 entry:
   %b = extractelement <4 x i64> %a, i32 3
@@ -63,6 +79,17 @@ entry:
   ret <4 x i64> %c
 }
 
+define <4 x i64> @insert_extract0_v4i64(<4 x i64> %a) nounwind {
+; CHECK-LABEL: insert_extract0_v4i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xvinsve0.d $xr0, $xr0, 1
+; CHECK-NEXT:    ret
+entry:
+  %b = extractelement <4 x i64> %a, i32 0
+  %c = insertelement <4 x i64> %a, i64 %b, i32 1
+  ret <4 x i64> %c
+}
+
 define <4 x double> @insert_extract_v4f64(<4 x double> %a) nounwind {
 ; CHECK-LABEL: insert_extract_v4f64:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
index e5a85245710de..72f25095d3837 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/insertelement.ll
@@ -5,7 +5,9 @@ define void @insert_32xi8(ptr %src, ptr %dst, i8 %in) nounwind {
 ; CHECK-LABEL: insert_32xi8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    vinsgr2vr.b $vr0, $a2, 1
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 17
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
 ; CHECK-NEXT:    ret
   %v = load volatile <32 x i8>, ptr %src
@@ -18,9 +20,9 @@ define void @insert_32xi8_upper(ptr %src, ptr %dst, i8 %in) nounwind {
 ; CHECK-LABEL: insert_32xi8_upper:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.b $vr1, $a2, 0
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    xvreplgr2vr.b $xr1, $a2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr1, 0
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
 ; CHECK-NEXT:    ret
   %v = load volatile <32 x i8>, ptr %src
@@ -29,11 +31,37 @@ define void @insert_32xi8_upper(ptr %src, ptr %dst, i8 %in) nounwind {
   ret void
 }
 
+define void @insert_32xi8_undef(ptr %dst, i8 %in) nounwind {
+; CHECK-LABEL: insert_32xi8_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vinsgr2vr.b $vr0, $a1, 1
+; CHECK-NEXT:    xvst $xr0, $a0, 0
+; CHECK-NEXT:    ret
+  %v = insertelement <32 x i8> poison, i8 %in, i32 1
+  store <32 x i8> %v, ptr %dst
+  ret void
+}
+
+define void @insert_32xi8_undef_upper(ptr %dst, i8 %in) nounwind {
+; CHECK-LABEL: insert_32xi8_undef_upper:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvreplgr2vr.b $xr0, $a1
+; CHECK-NEXT:    xvpermi.q $xr0, $xr0, 48
+; CHECK-NEXT:    xvextrins.b $xr0, $xr0, 102
+; CHECK-NEXT:    xvst $xr0, $a0, 0
+; CHECK-NEXT:    ret
+  %v = insertelement <32 x i8> poison, i8 %in, i32 22
+  store <32 x i8> %v, ptr %dst
+  ret void
+}
+
 define void @insert_16xi16(ptr %src, ptr %dst, i16 %in) nounwind {
 ; CHECK-LABEL: insert_16xi16:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    vinsgr2vr.h $vr0, $a2, 1
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 18
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 17
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
 ; CHECK-NEXT:    ret
   %v = load volatile <16 x i16>, ptr %src
@@ -46,9 +74,9 @@ define void @insert_16xi16_upper(ptr %src, ptr %dst, i16 %in) nounwind {
 ; CHECK-LABEL: insert_16xi16_upper:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    xvld $xr0, $a0, 0
-; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 14
-; CHECK-NEXT:    vinsgr2vr.h $vr1, $a2, 0
-; CHECK-NEXT:    xvpermi.q $xr0, $xr1, 2
+; CHECK-NEXT:    xvreplgr2vr.h $xr1, $a2
+; CHECK-NEXT:    xvpermi.q $xr1, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr1, 0
 ; CHECK-NEXT:    xvst $xr0, $a1, 0
 ; CHECK-NEXT:    ret
   %v = load volatile <16 x i16>, ptr %src
@@ -57,6 +85,30 @@ define void @insert_16xi16_upper(ptr %src, ptr %dst, i16 %in) nounwind {
   ret void
 }
 
+define void @insert_16xi16_undef(ptr %dst, i16 %in) nounwind {
+; CHECK-LABEL: insert_16xi16_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a1, 1
+; CHECK-NEXT:    xvst $xr0, $a0, 0
+; CHECK-NEXT:    ret
+  %v = insertelement <16 x i16> poison, i16 %in, i32 1
+  store <16 x i16> %v, ptr %dst
+  ret void
+}
+
+define void @insert_16xi16_undef_upper(ptr %dst, i16 %in) nounwind {
+; CHECK-LABEL: insert_16xi16_undef_upper:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvreplgr2vr.h $xr0, $a1
+; CHECK-NEXT:    xvpermi.q $xr0, $xr0, 48
+; CHECK-NEXT:    xvextrins.h $xr0, $xr0, 34
+; CHECK-NEXT:    xvst $xr0, $a0, 0
+; CHECK-NEXT:    ret
+  %v = insertelement <16 x i16> poison, i16 %in, i32 10
+  store <16 x i16> %v, ptr %dst
+  ret void
+}
+
 define void @insert_8xi32(ptr %src, ptr %dst, i32 %in) nounwind {
 ; CHECK-LABEL: insert_8xi32:
 ; CHECK:       # %bb.0:
@@ -114,8 +166,8 @@ define void @insert_4xdouble(ptr %src, ptr %dst, double %in) nounwind {
 define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_32xi8_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI8_0)
-; CHECK-NEXT:    xvld $xr0, $a4, %pc_lo12(.LCPI8_0)
+; CHECK-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI12_0)
+; CHECK-NEXT:    xvld $xr0, $a4, %pc_lo12(.LCPI12_0)
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
 ; CHECK-NEXT:    bstrpick.d $a0, $a3, 31, 0
 ; CHECK-NEXT:    xvreplgr2vr.b $xr2, $a0
@@ -133,8 +185,8 @@ define void @insert_32xi8_idx(ptr %src, ptr %dst, i8 %in, i32 %idx) nounwind {
 define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_16xi16_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI9_0)
-; CHECK-NEXT:    xvld $xr0, $a4, %pc_lo12(.LCPI9_0)
+; CHECK-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI13_0)
+; CHECK-NEXT:    xvld $xr0, $a4, %pc_lo12(.LCPI13_0)
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
 ; CHECK-NEXT:    bstrpick.d $a0, $a3, 31, 0
 ; CHECK-NEXT:    xvreplgr2vr.h $xr2, $a0
@@ -152,8 +204,8 @@ define void @insert_16xi16_idx(ptr %src, ptr %dst, i16 %in, i32 %idx) nounwind {
 define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_8xi32_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI10_0)
-; CHECK-NEXT:    xvld $xr0, $a4, %pc_lo12(.LCPI10_0)
+; CHECK-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI14_0)
+; CHECK-NEXT:    xvld $xr0, $a4, %pc_lo12(.LCPI14_0)
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
 ; CHECK-NEXT:    bstrpick.d $a0, $a3, 31, 0
 ; CHECK-NEXT:    xvreplgr2vr.w $xr2, $a0
@@ -171,8 +223,8 @@ define void @insert_8xi32_idx(ptr %src, ptr %dst, i32 %in, i32 %idx) nounwind {
 define void @insert_4xi64_idx(ptr %src, ptr %dst, i64 %in, i32 %idx) nounwind {
 ; CHECK-LABEL: insert_4xi64_idx:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI11_0)
-; CHECK-NEXT:    xvld $xr0, $a4, %pc_lo12(.LCPI11_0)
+; CHECK-NEXT:    pcalau12i $a4, %pc_hi20(.LCPI15_0)
+; CHECK-NEXT:    xvld $xr0, $a4, %pc_lo12(.LCPI15_0)
 ; CHECK-NEXT:    xvld $xr1, $a0, 0
 ; CHECK-NEXT:    bstrpick.d $a0, $a3, 31, 0
 ; CHECK-NEXT:    xvreplgr2vr.d $xr2, $a0
@@ -191,8 +243,8 @@ define void @insert_8xfloat_idx(ptr %src, ptr %dst, float %in, i32 %idx) nounwin
 ; CHECK-LABEL: insert_8xfloat_idx:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $f0 killed $f0 def $xr0
-; CHECK-NEXT:    pcalau12i $a3, %pc_hi20(.LCPI12_0)
-; CHECK-NEXT:    xvld $xr1, $a3, %pc_lo12(.LCPI12_0)
+; CHECK-NEXT:    pcalau12i $a3, %pc_hi20(.LCPI16_0)
+; CHECK-NEXT:    xvld $xr1, $a3, %pc_lo12(.LCPI16_0)
 ; CHECK-NEXT:    xvld $xr2, $a0, 0
 ; CHECK-NEXT:    bstrpick.d $a0, $a2, 31, 0
 ; CHECK-NEXT:    xvreplgr2vr.w $xr3, $a0
@@ -211,8 +263,8 @@ define void @insert_4xdouble_idx(ptr %src, ptr %dst, double %in, i32 %idx) nounw
 ; CHECK-LABEL: insert_4xdouble_idx:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    # kill: def $f0_64 killed $f0_64 def $xr0
-; CHECK-NEXT:    pcalau12i $a3, %pc_hi20(.LCPI13_0)
-; CHECK-NEXT:    xvld $xr1, $a3, %pc_lo12(.LCPI13_0)
+; CHECK-NEXT:    pcalau12i $a3, %pc_hi20(.LCPI17_0)
+; CHECK-NEXT:    xvld $xr1, $a3, %pc_lo12(.LCPI17_0)
 ; CHECK-NEXT:    xvld $xr2, $a0, 0
 ; CHECK-NEXT:    bstrpick.d $a0, $a2, 31, 0
 ; CHECK-NEXT:    xvreplgr2vr.d $xr3, $a0
diff --git a/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll b/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll
index 0e172950340e8..fed085843485a 100644
--- a/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll
+++ b/llvm/test/CodeGen/LoongArch/lasx/shuffle-as-permute-and-shuffle.ll
@@ -17,14 +17,25 @@ define <32 x i8> @shuffle_v32i8(<32 x i8> %a) {
   ret <32 x i8> %shuffle
 }
 
+define <32 x i8> @shuffle_v32i8_same_lane(<32 x i8> %a) {
+; CHECK-LABEL: shuffle_v32i8_same_lane:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI1_0)
+; CHECK-NEXT:    xvshuf.h $xr1, $xr0, $xr0
+; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    ret
+  %shuffle = shufflevector <32 x i8> %a, <32 x i8> poison, <32 x i32> <i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <32 x i8> %shuffle
+}
 
 define <16 x i16> @shuffle_v16i16(<16 x i16> %a) {
 ; CHECK-LABEL: shuffle_v16i16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI1_0)
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_1)
-; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI1_1)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI2_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI2_1)
 ; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
 ; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
 ; CHECK-NEXT:    xvshuf.w $xr1, $xr2, $xr0
@@ -34,13 +45,25 @@ define <16 x i16> @shuffle_v16i16(<16 x i16> %a) {
   ret <16 x i16> %shuffle
 }
 
+define <16 x i16> @shuffle_v16i16_same_lane(<16 x i16> %a) {
+; CHECK-LABEL: shuffle_v16i16_same_lane:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    xvshuf.h $xr1, $xr0, $xr0
+; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    ret
+  %shuffle = shufflevector <16 x i16> %a, <16 x i16> poison, <16 x i32> <i32 6, i32 7, i32 0, i32 5, i32 2, i32 3, i32 6, i32 5, i32 8, i32 9, i32 10, i32 13, i32 12, i32 15, i32 13, i32 15>
+  ret <16 x i16> %shuffle
+}
+
 define <8 x i32> @shuffle_v8i32(<8 x i32> %a) {
 ; CHECK-LABEL: shuffle_v8i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI2_0)
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI2_1)
-; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI2_1)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI4_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI4_1)
 ; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
 ; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
 ; CHECK-NEXT:    xvshuf.d $xr1, $xr2, $xr0
@@ -50,13 +73,25 @@ define <8 x i32> @shuffle_v8i32(<8 x i32> %a) {
   ret <8 x i32> %shuffle
 }
 
+define <8 x i32> @shuffle_v8i32_same_lane(<8 x i32> %a) {
+; CHECK-LABEL: shuffle_v8i32_same_lane:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI5_0)
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr0
+; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    ret
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> poison, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i32> %shuffle
+}
+
 define <4 x i64> @shuffle_v4i64(<4 x i64> %a) {
 ; CHECK-LABEL: shuffle_v4i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI3_0)
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_1)
-; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_1)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI6_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI6_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI6_1)
 ; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
 ; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
 ; CHECK-NEXT:    xvshuf.d $xr1, $xr2, $xr0
@@ -66,13 +101,25 @@ define <4 x i64> @shuffle_v4i64(<4 x i64> %a) {
   ret <4 x i64> %shuffle
 }
 
+define <4 x i64> @shuffle_v4i64_same_lane(<4 x i64> %a) {
+; CHECK-LABEL: shuffle_v4i64_same_lane:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI7_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI7_0)
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr0
+; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    ret
+  %shuffle = shufflevector <4 x i64> %a, <4 x i64> poison, <4 x i32> <i32 1, i32 0, i32 2, i32 3>
+  ret <4 x i64> %shuffle
+}
+
 define <8 x float> @shuffle_v8f32(<8 x float> %a) {
 ; CHECK-LABEL: shuffle_v8f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI4_0)
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_1)
-; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI4_1)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI8_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI8_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI8_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI8_1)
 ; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
 ; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
 ; CHECK-NEXT:    xvshuf.d $xr1, $xr2, $xr0
@@ -82,13 +129,26 @@ define <8 x float> @shuffle_v8f32(<8 x float> %a) {
   ret <8 x float> %shuffle
 }
 
+define <8 x float> @shuffle_v8f32_same_lane(<8 x float> %a) {
+; CHECK-LABEL: shuffle_v8f32_same_lane:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI9_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI9_0)
+; CHECK-NEXT:    xvpermi.d $xr0, $xr0, 68
+; CHECK-NEXT:    xvshuf.w $xr1, $xr0, $xr0
+; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    ret
+  %shuffle = shufflevector <8 x float> %a, <8 x float> poison, <8 x i32> <i32 3, i32 2, i32 0, i32 2, i32 3, i32 1, i32 2, i32 3>
+  ret <8 x float> %shuffle
+}
+
 define <4 x double> @shuffle_v4f64(<4 x double> %a) {
 ; CHECK-LABEL: shuffle_v4f64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_0)
-; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI5_0)
-; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI5_1)
-; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI5_1)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI10_0)
+; CHECK-NEXT:    xvld $xr2, $a0, %pc_lo12(.LCPI10_0)
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI10_1)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI10_1)
 ; CHECK-NEXT:    xvpermi.d $xr3, $xr0, 78
 ; CHECK-NEXT:    xvshuf.d $xr2, $xr0, $xr3
 ; CHECK-NEXT:    xvshuf.d $xr1, $xr2, $xr0
@@ -97,3 +157,16 @@ define <4 x double> @shuffle_v4f64(<4 x double> %a) {
   %shuffle = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 1, i32 2, i32 0>
   ret <4 x double> %shuffle
 }
+
+define <4 x double> @shuffle_v4f64_same_lane(<4 x double> %a) {
+; CHECK-LABEL: shuffle_v4f64_same_lane:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI11_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI11_0)
+; CHECK-NEXT:    xvpermi.d $xr0, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr0
+; CHECK-NEXT:    xvori.b $xr0, $xr1, 0
+; CHECK-NEXT:    ret
+  %shuffle = shufflevector <4 x double> %a, <4 x double> poison, <4 x i32> <i32 3, i32 2, i32 0, i32 1>
+  ret <4 x double> %shuffle
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-add.ll b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-add.ll
new file mode 100644
index 0000000000000..7268eb24ee51c
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-add.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx %s -o - | FileCheck %s
+
+define void @vec_reduce_add_v32i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvhaddw.h.b $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvhaddw.w.h $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvhaddw.d.w $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvhaddw.q.d $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 2
+; CHECK-NEXT:    xvadd.d $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
+; CHECK-NEXT:    st.b $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %v = load <32 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v16i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvhaddw.w.h $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvhaddw.d.w $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvhaddw.q.d $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 2
+; CHECK-NEXT:    xvadd.d $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
+; CHECK-NEXT:    st.h $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v8i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvhaddw.d.w $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvhaddw.q.d $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 2
+; CHECK-NEXT:    xvadd.d $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvpickve2gr.d $a0, $xr0, 0
+; CHECK-NEXT:    st.w $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v4i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvhaddw.q.d $xr0, $xr0, $xr0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 2
+; CHECK-NEXT:    xvadd.d $xr0, $xr1, $xr0
+; CHECK-NEXT:    xvstelm.d $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-and.ll b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-and.ll
new file mode 100644
index 0000000000000..a3160f10c8ca8
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-and.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx %s -o - | FileCheck %s
+
+define void @vec_reduce_and_v32i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 228
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvsrli.d $xr1, $xr1, 32
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 14
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.b $xr1, $xr1, 1
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.b $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <32 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.and.v32i8(<32 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v16i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 228
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 14
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.h $xr1, $xr1, 1
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.h $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.and.v16i16(<16 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v8i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 228
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 14
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.w $xr1, $xr1, 1
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.w $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.and.v8i32(<8 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v4i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    xvpermi.d $xr2, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr2
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.d $xr1, $xr1, 1
+; CHECK-NEXT:    xvand.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.d $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-or.ll b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-or.ll
new file mode 100644
index 0000000000000..bc910c23e4b17
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-or.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx %s -o - | FileCheck %s
+
+define void @vec_reduce_or_v32i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 228
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvsrli.d $xr1, $xr1, 32
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 14
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.b $xr1, $xr1, 1
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.b $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <32 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.or.v32i8(<32 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v16i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 228
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 14
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.h $xr1, $xr1, 1
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.h $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.or.v16i16(<16 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v8i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 228
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 14
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.w $xr1, $xr1, 1
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.w $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v4i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    xvpermi.d $xr2, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr2
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.d $xr1, $xr1, 1
+; CHECK-NEXT:    xvor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.d $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-smax.ll b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-smax.ll
new file mode 100644
index 0000000000000..378088c9f8280
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-smax.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx %s -o - | FileCheck %s
+
+define void @vec_reduce_smax_v32i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 228
+; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvsrli.d $xr1, $xr1, 32
+; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 14
+; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.b $xr1, $xr1, 1
+; CHECK-NEXT:    xvmax.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.b $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <32 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smax.v32i8(<32 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v16i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 228
+; CHECK-NEXT:    xvmax.h $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvmax.h $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 14
+; CHECK-NEXT:    xvmax.h $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.h $xr1, $xr1, 1
+; CHECK-NEXT:    xvmax.h $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.h $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.smax.v16i16(<16 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v8i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 228
+; CHECK-NEXT:    xvmax.w $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 14
+; CHECK-NEXT:    xvmax.w $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.w $xr1, $xr1, 1
+; CHECK-NEXT:    xvmax.w $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.w $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v4i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    xvpermi.d $xr2, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr2
+; CHECK-NEXT:    xvmax.d $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.d $xr1, $xr1, 1
+; CHECK-NEXT:    xvmax.d $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.d $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.smax.v4i64(<4 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-smin.ll b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-smin.ll
new file mode 100644
index 0000000000000..1c7f2054cd4e1
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-smin.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx %s -o - | FileCheck %s
+
+define void @vec_reduce_smin_v32i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 228
+; CHECK-NEXT:    xvmin.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvmin.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvsrli.d $xr1, $xr1, 32
+; CHECK-NEXT:    xvmin.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 14
+; CHECK-NEXT:    xvmin.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.b $xr1, $xr1, 1
+; CHECK-NEXT:    xvmin.b $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.b $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <32 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smin.v32i8(<32 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v16i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 228
+; CHECK-NEXT:    xvmin.h $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvmin.h $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 14
+; CHECK-NEXT:    xvmin.h $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.h $xr1, $xr1, 1
+; CHECK-NEXT:    xvmin.h $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.h $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.smin.v16i16(<16 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v8i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 228
+; CHECK-NEXT:    xvmin.w $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 14
+; CHECK-NEXT:    xvmin.w $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.w $xr1, $xr1, 1
+; CHECK-NEXT:    xvmin.w $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.w $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.smin.v8i32(<8 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v4i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    xvpermi.d $xr2, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr2
+; CHECK-NEXT:    xvmin.d $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.d $xr1, $xr1, 1
+; CHECK-NEXT:    xvmin.d $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.d $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.smin.v4i64(<4 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-umax.ll b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-umax.ll
new file mode 100644
index 0000000000000..152f093cbd025
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-umax.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx %s -o - | FileCheck %s
+
+define void @vec_reduce_umax_v32i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 228
+; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvsrli.d $xr1, $xr1, 32
+; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 14
+; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.b $xr1, $xr1, 1
+; CHECK-NEXT:    xvmax.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.b $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <32 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umax.v32i8(<32 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v16i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 228
+; CHECK-NEXT:    xvmax.hu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvmax.hu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 14
+; CHECK-NEXT:    xvmax.hu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.h $xr1, $xr1, 1
+; CHECK-NEXT:    xvmax.hu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.h $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.umax.v16i16(<16 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v8i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 228
+; CHECK-NEXT:    xvmax.wu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 14
+; CHECK-NEXT:    xvmax.wu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.w $xr1, $xr1, 1
+; CHECK-NEXT:    xvmax.wu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.w $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.umax.v8i32(<8 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v4i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    xvpermi.d $xr2, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr2
+; CHECK-NEXT:    xvmax.du $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.d $xr1, $xr1, 1
+; CHECK-NEXT:    xvmax.du $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.d $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.umax.v4i64(<4 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-umin.ll b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-umin.ll
new file mode 100644
index 0000000000000..64ed377535abf
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-umin.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx %s -o - | FileCheck %s
+
+define void @vec_reduce_umin_v32i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 228
+; CHECK-NEXT:    xvmin.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvmin.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvsrli.d $xr1, $xr1, 32
+; CHECK-NEXT:    xvmin.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 14
+; CHECK-NEXT:    xvmin.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.b $xr1, $xr1, 1
+; CHECK-NEXT:    xvmin.bu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.b $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <32 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v16i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 228
+; CHECK-NEXT:    xvmin.hu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvmin.hu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 14
+; CHECK-NEXT:    xvmin.hu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.h $xr1, $xr1, 1
+; CHECK-NEXT:    xvmin.hu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.h $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v8i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 228
+; CHECK-NEXT:    xvmin.wu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 14
+; CHECK-NEXT:    xvmin.wu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.w $xr1, $xr1, 1
+; CHECK-NEXT:    xvmin.wu $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.w $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.umin.v8i32(<8 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v4i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    xvpermi.d $xr2, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr2
+; CHECK-NEXT:    xvmin.du $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.d $xr1, $xr1, 1
+; CHECK-NEXT:    xvmin.du $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.d $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.umin.v4i64(<4 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-xor.ll b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-xor.ll
new file mode 100644
index 0000000000000..5dbf37e732637
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lasx/vec-reduce-xor.ll
@@ -0,0 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lasx %s -o - | FileCheck %s
+
+define void @vec_reduce_xor_v32i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v32i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 228
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvsrli.d $xr1, $xr1, 32
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.b $xr1, $xr1, 14
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.b $xr1, $xr1, 1
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.b $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <32 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v16i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v16i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 228
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvbsrl.v $xr1, $xr1, 8
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.h $xr1, $xr1, 14
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.h $xr1, $xr1, 1
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.h $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.xor.v16i16(<16 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v8i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v8i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 78
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 228
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvshuf4i.w $xr1, $xr1, 14
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.w $xr1, $xr1, 1
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.w $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v4i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v4i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xvld $xr0, $a0, 0
+; CHECK-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI3_0)
+; CHECK-NEXT:    xvld $xr1, $a0, %pc_lo12(.LCPI3_0)
+; CHECK-NEXT:    xvpermi.d $xr2, $xr0, 78
+; CHECK-NEXT:    xvshuf.d $xr1, $xr0, $xr2
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvpermi.d $xr1, $xr0, 68
+; CHECK-NEXT:    xvrepl128vei.d $xr1, $xr1, 1
+; CHECK-NEXT:    xvxor.v $xr0, $xr0, $xr1
+; CHECK-NEXT:    xvstelm.d $xr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.xor.v4i64(<4 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/LoongArch/linker-relaxation.ll b/llvm/test/CodeGen/LoongArch/linker-relaxation.ll
index 2827a95547903..6b197bc578919 100644
--- a/llvm/test/CodeGen/LoongArch/linker-relaxation.ll
+++ b/llvm/test/CodeGen/LoongArch/linker-relaxation.ll
@@ -1,6 +1,6 @@
 ; RUN: llc --mtriple=loongarch64 --filetype=obj -mattr=-relax \
 ; RUN:     --relocation-model=pic --code-model=medium < %s \
-; RUN:     | llvm-readobj -r - | FileCheck --check-prefixes=CHECK-RELOC,PCALA-RELOC %s
+; RUN:     | llvm-readobj -r - | FileCheck --check-prefix=CHECK-RELOC %s
 ; RUN: llc --mtriple=loongarch64 --filetype=obj -mattr=+relax \
 ; RUN:     --relocation-model=pic --code-model=medium < %s \
 ; RUN:     | llvm-readobj -r - | FileCheck --check-prefixes=CHECK-RELOC,RELAX %s
@@ -29,16 +29,12 @@ declare void @callee1() nounwind
 declare dso_local void @callee2() nounwind
 declare dso_local void @callee3() nounwind
 
-define ptr @caller() nounwind {
-; RELAX:            R_LARCH_ALIGN - 0x1C
 ; CHECK-RELOC:      R_LARCH_GOT_PC_HI20 g_e 0x0
-; RELAX-NEXT:       R_LARCH_RELAX - 0x0
+; RELAX:            R_LARCH_RELAX - 0x0
 ; CHECK-RELOC-NEXT: R_LARCH_GOT_PC_LO12 g_e 0x0
 ; RELAX-NEXT:       R_LARCH_RELAX - 0x0
-; PCALA-RELOC:      R_LARCH_PCALA_HI20 .bss 0x0
-; RELAX-NEXT:       R_LARCH_PCALA_HI20 g_i 0x0
-; PCALA-RELOC:      R_LARCH_PCALA_LO12 .bss 0x0
-; RELAX-NEXT:       R_LARCH_PCALA_LO12 g_i 0x0
+; CHECK-RELOC-NEXT: R_LARCH_PCALA_HI20 g_i 0x0
+; CHECK-RELOC-NEXT: R_LARCH_PCALA_LO12 g_i 0x0
 ; CHECK-RELOC:      R_LARCH_TLS_GD_PC_HI20 t_un 0x0
 ; RELAX-NEXT:       R_LARCH_RELAX - 0x0
 ; CHECK-RELOC-NEXT: R_LARCH_GOT_PC_LO12 t_un 0x0
@@ -77,26 +73,33 @@ define ptr @caller() nounwind {
 ; RELAX-NEXT:       R_LARCH_RELAX - 0x0
 ; CHECK-RELOC-NEXT: R_LARCH_TLS_LE_LO12_R t_le 0x0
 ; RELAX-NEXT:       R_LARCH_RELAX - 0x0
+; CHECK-RELOC-NEXT: R_LARCH_PCALA_HI20 g_i1 0x0
+; RELAX-NEXT:       R_LARCH_RELAX - 0x0
+; CHECK-RELOC-NEXT: R_LARCH_PCALA_LO12 g_i1 0x0
+; RELAX-NEXT:       R_LARCH_RELAX - 0x0
+; RELAX-NEXT:       R_LARCH_ALIGN - 0x1C
 ; CHECK-RELOC-NEXT: R_LARCH_CALL36 callee1 0x0
 ; RELAX-NEXT:       R_LARCH_RELAX - 0x0
 ; CHECK-RELOC-NEXT: R_LARCH_CALL36 callee2 0x0
 ; RELAX-NEXT:       R_LARCH_RELAX - 0x0
 ; CHECK-RELOC-NEXT: R_LARCH_CALL36 callee3 0x0
 ; RELAX-NEXT:       R_LARCH_RELAX - 0x0
-; PCALA-RELOC:      R_LARCH_PCALA_HI20 .data 0x0
-; RELAX-NEXT:       R_LARCH_PCALA_HI20 g_i1 0x0
-; RELAX-NEXT:       R_LARCH_RELAX - 0x0
-; PCALA-RELOC:      R_LARCH_PCALA_LO12 .data 0x0
-; RELAX-NEXT:       R_LARCH_PCALA_LO12 g_i1 0x0
-; RELAX-NEXT:       R_LARCH_RELAX - 0x0
+
+;; No ALIGN reloc will emit before the first linker-relaxable instruction.
+define ptr @loader() nounwind {
   %a = load volatile i32, ptr @g_e
   %b = load volatile i32, ptr @g_i
   %c = load volatile i32, ptr @t_un
   %d = load volatile i32, ptr @t_ld
   %e = load volatile i32, ptr @t_ie
   %f = load volatile i32, ptr @t_le
+  ret ptr @g_i1
+}
+
+;; ALIGN reloc will be emitted here.
+define void @caller() nounwind {
   call i32 @callee1()
   call i32 @callee2()
   tail call i32 @callee3()
-  ret ptr @g_i1
+  ret void
 }
diff --git a/llvm/test/CodeGen/LoongArch/lrint-conv.ll b/llvm/test/CodeGen/LoongArch/lrint-conv.ll
new file mode 100644
index 0000000000000..85de820025614
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lrint-conv.ll
@@ -0,0 +1,96 @@
+; Tests for lrint and llrint, with both i32 and i64 checked.
+
+; RUN: sed 's/ITy/i32/g' %s | llc -mtriple=loongarch32 | FileCheck %s --check-prefixes=LA32
+; RUN: sed 's/ITy/i64/g' %s | llc -mtriple=loongarch32 | FileCheck %s --check-prefixes=LA32
+; RUN: sed 's/ITy/i32/g' %s | llc -mtriple=loongarch64 | FileCheck %s --check-prefixes=LA64-I32
+; RUN: sed 's/ITy/i64/g' %s | llc -mtriple=loongarch64 | FileCheck %s --check-prefixes=LA64-I64
+
+; FIXME: crash
+; define ITy @test_lrint_ixx_f16(half %x) nounwind {
+;   %res = tail call ITy @llvm.lrint.ITy.f16(half %x)
+;   ret ITy %res
+; }
+
+; define ITy @test_llrint_ixx_f16(half %x) nounwind {
+;   %res = tail call ITy @llvm.llrint.ITy.f16(half %x)
+;   ret ITy %res
+; }
+
+define ITy @test_lrint_ixx_f32(float %x) nounwind {
+; LA32-LABEL: test_lrint_ixx_f32:
+; LA32:         bl lrintf
+;
+; LA64-I32-LABEL: test_lrint_ixx_f32:
+; LA64-I32:         pcaddu18i $ra, %call36(lrintf)
+;
+; LA64-I64-LABEL: test_lrint_ixx_f32:
+; LA64-I64:         pcaddu18i $t8, %call36(lrintf)
+  %res = tail call ITy @llvm.lrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f32(float %x) nounwind {
+; LA32-LABEL: test_llrint_ixx_f32:
+; LA32:         bl llrintf
+;
+; LA64-I32-LABEL: test_llrint_ixx_f32:
+; LA64-I32:         pcaddu18i $ra, %call36(llrintf)
+;
+; LA64-I64-LABEL: test_llrint_ixx_f32:
+; LA64-I64:         pcaddu18i $t8, %call36(llrintf)
+  %res = tail call ITy @llvm.llrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_lrint_ixx_f64(double %x) nounwind {
+; LA32-LABEL: test_lrint_ixx_f64:
+; LA32:         bl lrint
+;
+; LA64-I32-LABEL: test_lrint_ixx_f64:
+; LA64-I32:         pcaddu18i $ra, %call36(lrint)
+;
+; LA64-I64-LABEL: test_lrint_ixx_f64:
+; LA64-I64:         pcaddu18i $t8, %call36(lrint)
+  %res = tail call ITy @llvm.lrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f64(double %x) nounwind {
+; LA32-LABEL: test_llrint_ixx_f64:
+; LA32:         bl llrint
+;
+; LA64-I32-LABEL: test_llrint_ixx_f64:
+; LA64-I32:         pcaddu18i $ra, %call36(llrint)
+;
+; LA64-I64-LABEL: test_llrint_ixx_f64:
+; LA64-I64:         pcaddu18i $t8, %call36(llrint)
+  %res = tail call ITy @llvm.llrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+; FIXME(#44744): incorrect libcall on loongarch32
+define ITy @test_lrint_ixx_f128(fp128 %x) nounwind {
+; LA32-LABEL: test_lrint_ixx_f128:
+; LA32:         bl lrintl
+;
+; LA64-I32-LABEL: test_lrint_ixx_f128:
+; LA64-I32:         pcaddu18i $ra, %call36(lrintl)
+;
+; LA64-I64-LABEL: test_lrint_ixx_f128:
+; LA64-I64:         pcaddu18i $ra, %call36(lrintl)
+  %res = tail call ITy @llvm.lrint.ITy.f128(fp128 %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f128(fp128 %x) nounwind {
+; LA32-LABEL: test_llrint_ixx_f128:
+; LA32:         bl llrintl
+;
+; LA64-I32-LABEL: test_llrint_ixx_f128:
+; LA64-I32:         pcaddu18i $ra, %call36(llrintl)
+;
+; LA64-I64-LABEL: test_llrint_ixx_f128:
+; LA64-I64:         pcaddu18i $ra, %call36(llrintl)
+  %res = tail call ITy @llvm.llrint.ITy.f128(fp128 %x)
+  ret ITy %res
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-add.ll b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-add.ll
new file mode 100644
index 0000000000000..57fd09ed2e09b
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-add.ll
@@ -0,0 +1,154 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx %s -o - | FileCheck %s
+
+define void @vec_reduce_add_v16i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.h.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.w.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.d.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.q.d $vr0, $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 0
+; CHECK-NEXT:    st.b $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v8i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.h.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.w.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.d.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.w $a0, $vr0, 0
+; CHECK-NEXT:    st.b $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v4i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.h.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.w.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.h $a0, $vr0, 0
+; CHECK-NEXT:    st.b $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v2i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.h.b $vr0, $vr0, $vr0
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v8i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.w.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.d.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.q.d $vr0, $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 0
+; CHECK-NEXT:    st.h $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v4i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.w.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.d.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.w $a0, $vr0, 0
+; CHECK-NEXT:    st.h $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v2i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.w.h $vr0, $vr0, $vr0
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v4i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.d.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vhaddw.q.d $vr0, $vr0, $vr0
+; CHECK-NEXT:    vpickve2gr.d $a0, $vr0, 0
+; CHECK-NEXT:    st.w $a0, $a1, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v2i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.d.w $vr0, $vr0, $vr0
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_add_v2i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_add_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vhaddw.q.d $vr0, $vr0, $vr0
+; CHECK-NEXT:    vstelm.d $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-and.ll b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-and.ll
new file mode 100644
index 0000000000000..c16de10239642
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-and.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx %s -o - | FileCheck %s
+
+define void @vec_reduce_and_v16i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.and.v16i8(<16 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v8i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.and.v8i8(<8 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v4i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v2i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v8i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.and.v8i16(<8 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v4i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.and.v4i16(<4 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v2i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.and.v2i16(<2 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v4i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.w $vr1, $vr0, 14
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v2i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_and_v2i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_and_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.d $vr1, $vr0, 1
+; CHECK-NEXT:    vand.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.d $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.and.v2i64(<2 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-or.ll b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-or.ll
new file mode 100644
index 0000000000000..52f18cce611de
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-or.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx %s -o - | FileCheck %s
+
+define void @vec_reduce_or_v16i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v8i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v4i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v2i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v8i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.or.v8i16(<8 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v4i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v2i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.or.v2i16(<2 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v4i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.w $vr1, $vr0, 14
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v2i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_or_v2i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_or_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.d $vr1, $vr0, 1
+; CHECK-NEXT:    vor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.d $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-smax.ll b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-smax.ll
new file mode 100644
index 0000000000000..5d8c3e36549d6
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-smax.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx %s -o - | FileCheck %s
+
+define void @vec_reduce_smax_v16i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v8i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v4i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v2i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smax.v2i8(<2 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v8i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v4i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v2i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.smax.v2i16(<2 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v4i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.w $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.w $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.w $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v2i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.w $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.smax.v2i32(<2 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smax_v2i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smax_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.d $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.d $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.d $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.smax.v2i64(<2 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-smin.ll b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-smin.ll
new file mode 100644
index 0000000000000..2d53095db89db
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-smin.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx %s -o - | FileCheck %s
+
+define void @vec_reduce_smin_v16i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v8i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v4i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v2i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.b $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.smin.v2i8(<2 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v8i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vmin.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v4i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v2i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.h $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.smin.v2i16(<2 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v4i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.w $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.w $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.w $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v2i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.w $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.smin.v2i32(<2 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_smin_v2i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_smin_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.d $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.d $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.d $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.smin.v2i64(<2 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-umax.ll b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-umax.ll
new file mode 100644
index 0000000000000..abe9ba7dfb246
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-umax.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx %s -o - | FileCheck %s
+
+define void @vec_reduce_umax_v16i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v8i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v4i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v2i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umax.v2i8(<2 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v8i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v4i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v2i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.umax.v2i16(<2 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v4i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.w $vr1, $vr0, 14
+; CHECK-NEXT:    vmax.wu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.wu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v2i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.wu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.umax.v2i32(<2 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umax_v2i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umax_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.d $vr1, $vr0, 1
+; CHECK-NEXT:    vmax.du $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.d $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.umax.v2i64(<2 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-umin.ll b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-umin.ll
new file mode 100644
index 0000000000000..3d396f3692e7d
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-umin.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx %s -o - | FileCheck %s
+
+define void @vec_reduce_umin_v16i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v8i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v4i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umin.v4i8(<4 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v2i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.bu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.umin.v2i8(<2 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v8i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vmin.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v4i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v2i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.hu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.umin.v2i16(<2 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v4i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.w $vr1, $vr0, 14
+; CHECK-NEXT:    vmin.wu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.wu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v2i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.wu $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.umin.v2i32(<2 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_umin_v2i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_umin_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.d $vr1, $vr0, 1
+; CHECK-NEXT:    vmin.du $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.d $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.umin.v2i64(<2 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-xor.ll b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-xor.ll
new file mode 100644
index 0000000000000..1894532d6121d
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/lsx/vec-reduce-xor.ll
@@ -0,0 +1,168 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc --mtriple=loongarch64 --mattr=+lsx %s -o - | FileCheck %s
+
+define void @vec_reduce_xor_v16i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <16 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.xor.v16i8(<16 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v8i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v8i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vsrli.d $vr1, $vr0, 32
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.xor.v8i8(<8 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v4i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v4i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.b $vr1, $vr0, 14
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v2i8(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v2i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.h $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.h $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.b $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.b $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i8>, ptr %src
+  %res = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> %v)
+  store i8 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v8i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vbsrl.v $vr1, $vr0, 8
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <8 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.xor.v8i16(<8 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v4i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v4i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.h $vr1, $vr0, 14
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.xor.v4i16(<4 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v2i16(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.w $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.w $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.h $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.h $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i16>, ptr %src
+  %res = call i16 @llvm.vector.reduce.xor.v2i16(<2 x i16> %v)
+  store i16 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v4i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v4i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vshuf4i.w $vr1, $vr0, 14
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <4 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v2i32(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v2i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ld.d $a0, $a0, 0
+; CHECK-NEXT:    vinsgr2vr.d $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.w $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.w $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i32>, ptr %src
+  %res = call i32 @llvm.vector.reduce.xor.v2i32(<2 x i32> %v)
+  store i32 %res, ptr %dst
+  ret void
+}
+
+define void @vec_reduce_xor_v2i64(ptr %src, ptr %dst) nounwind {
+; CHECK-LABEL: vec_reduce_xor_v2i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vld $vr0, $a0, 0
+; CHECK-NEXT:    vreplvei.d $vr1, $vr0, 1
+; CHECK-NEXT:    vxor.v $vr0, $vr0, $vr1
+; CHECK-NEXT:    vstelm.d $vr0, $a1, 0, 0
+; CHECK-NEXT:    ret
+  %v = load <2 x i64>, ptr %src
+  %res = call i64 @llvm.vector.reduce.xor.v2i64(<2 x i64> %v)
+  store i64 %res, ptr %dst
+  ret void
+}
diff --git a/llvm/test/CodeGen/LoongArch/xray-attribute-instrumentation.ll b/llvm/test/CodeGen/LoongArch/xray-attribute-instrumentation.ll
index 8999c20387003..7838bcea1025d 100644
--- a/llvm/test/CodeGen/LoongArch/xray-attribute-instrumentation.ll
+++ b/llvm/test/CodeGen/LoongArch/xray-attribute-instrumentation.ll
@@ -43,14 +43,14 @@ define i32 @foo() nounwind noinline uwtable "function-instrument"="xray-always"
 ; CHECK-NEXT: .dword 2
 
 ; RELOC:      Section ([[#]]) .relaxray_instr_map {
-; RELOC-NEXT:   0x0 R_LARCH_64_PCREL .text 0x0
-; RELOC-NEXT:   0x8 R_LARCH_64_PCREL .text 0x0
-; RELOC-NEXT:   0x20 R_LARCH_64_PCREL .text 0x34
-; RELOC-NEXT:   0x28 R_LARCH_64_PCREL .text 0x0
+; RELOC-NEXT:   0x0 R_LARCH_64_PCREL .L{{.*}} 0x0
+; RELOC-NEXT:   0x8 R_LARCH_64_PCREL .L{{.*}} 0x0
+; RELOC-NEXT:   0x20 R_LARCH_64_PCREL .L{{.*}} 0x0
+; RELOC-NEXT:   0x28 R_LARCH_64_PCREL .L{{.*}} 0x0
 ; RELOC-NEXT: }
 ; RELOC-NEXT: Section ([[#]]) .relaxray_fn_idx {
-; RELOC-NEXT:   0x0 R_LARCH_64_PCREL xray_instr_map 0x0
+; RELOC-NEXT:   0x0 R_LARCH_64_PCREL .Lxray_sleds_start0 0x0
 ; RELOC-NEXT: }
 ; RELOC-NEXT: Section ([[#]]) .rela.eh_frame {
-; RELOC-NEXT:   0x1C R_LARCH_32_PCREL .text 0x0
+; RELOC-NEXT:   0x1C R_LARCH_32_PCREL .L{{.*}} 0x0
 ; RELOC-NEXT: }
diff --git a/llvm/test/CodeGen/M68k/Arith/add.ll b/llvm/test/CodeGen/M68k/Arith/add.ll
index 417fe8f068479..73220c6eaf43e 100644
--- a/llvm/test/CodeGen/M68k/Arith/add.ll
+++ b/llvm/test/CodeGen/M68k/Arith/add.ll
@@ -85,9 +85,9 @@ define fastcc i32 @test9(i32 %x, i32 %y) nounwind readnone {
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    sub.l #10, %d0
 ; CHECK-NEXT:    seq %d0
-; CHECK-NEXT:    and.l #255, %d0
-; CHECK-NEXT:    sub.l %d0, %d1
-; CHECK-NEXT:    move.l %d1, %d0
+; CHECK-NEXT:    ext.w %d0
+; CHECK-NEXT:    ext.l %d0
+; CHECK-NEXT:    add.l %d1, %d0
 ; CHECK-NEXT:    rts
   %cmp = icmp eq i32 %x, 10
   %sub = sext i1 %cmp to i32
diff --git a/llvm/test/CodeGen/M68k/Arith/smul-with-overflow.ll b/llvm/test/CodeGen/M68k/Arith/smul-with-overflow.ll
index 423431750f20a..ad6843b310f06 100644
--- a/llvm/test/CodeGen/M68k/Arith/smul-with-overflow.ll
+++ b/llvm/test/CodeGen/M68k/Arith/smul-with-overflow.ll
@@ -102,8 +102,9 @@ define fastcc i1 @test2(i32 %v1, i32 %v2) nounwind {
 ; CHECK-NEXT:    suba.l #12, %sp
 ; CHECK-NEXT:    muls.l %d1, %d0
 ; CHECK-NEXT:    svs %d1
-; CHECK-NEXT:    sub.b #1, %d1
-; CHECK-NEXT:    bne .LBB4_2
+; CHECK-NEXT:    and.b #1, %d1
+; CHECK-NEXT:    cmpi.b #0, %d1
+; CHECK-NEXT:    beq .LBB4_2
 ; CHECK-NEXT:  ; %bb.1: ; %overflow
 ; CHECK-NEXT:    lea (no,%pc), %a0
 ; CHECK-NEXT:    move.l %a0, (%sp)
diff --git a/llvm/test/CodeGen/M68k/Atomics/rmw.ll b/llvm/test/CodeGen/M68k/Atomics/rmw.ll
index a277b8fe72ae4..71ab4326cfe7d 100644
--- a/llvm/test/CodeGen/M68k/Atomics/rmw.ll
+++ b/llvm/test/CodeGen/M68k/Atomics/rmw.ll
@@ -37,9 +37,10 @@ define i8 @atomicrmw_add_i8(i8 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.b %d0, %d3
 ; ATOMIC-NEXT:    sub.b %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.b %d0, %d2
-; ATOMIC-NEXT:    bne .LBB0_1
+; ATOMIC-NEXT:    beq .LBB0_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -80,9 +81,10 @@ define i16 @atomicrmw_sub_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.w %d0, %d3
 ; ATOMIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.w %d0, %d2
-; ATOMIC-NEXT:    bne .LBB1_1
+; ATOMIC-NEXT:    beq .LBB1_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -121,9 +123,10 @@ define i32 @atomicrmw_and_i32(i32 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.l %d0, %d3
 ; ATOMIC-NEXT:    sub.l %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.l %d0, %d2
-; ATOMIC-NEXT:    bne .LBB2_1
+; ATOMIC-NEXT:    beq .LBB2_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -194,9 +197,10 @@ define i8 @atomicrmw_or_i8(i8 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.b %d0, %d3
 ; ATOMIC-NEXT:    sub.b %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.b %d0, %d2
-; ATOMIC-NEXT:    bne .LBB4_1
+; ATOMIC-NEXT:    beq .LBB4_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -242,9 +246,10 @@ define i16 @atmoicrmw_nand_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.w %d1, %d3
 ; ATOMIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.w %d1, %d2
-; ATOMIC-NEXT:    bne .LBB5_1
+; ATOMIC-NEXT:    beq .LBB5_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -282,9 +287,10 @@ define i32 @atomicrmw_min_i32(i32 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.l %d0, %d3
 ; ATOMIC-NEXT:    sub.l %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.l %d0, %d2
-; ATOMIC-NEXT:    beq .LBB6_4
+; ATOMIC-NEXT:    bne .LBB6_4
 ; ATOMIC-NEXT:  .LBB6_1: ; %atomicrmw.start
 ; ATOMIC-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ATOMIC-NEXT:    move.l %d2, %d0
@@ -434,9 +440,10 @@ define i8 @atomicrmw_i8_umin(i8 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.b %d0, %d3
 ; ATOMIC-NEXT:    sub.b %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.b %d0, %d2
-; ATOMIC-NEXT:    beq .LBB8_4
+; ATOMIC-NEXT:    bne .LBB8_4
 ; ATOMIC-NEXT:  .LBB8_1: ; %atomicrmw.start
 ; ATOMIC-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ATOMIC-NEXT:    move.b %d2, %d0
@@ -486,9 +493,10 @@ define i16 @atomicrmw_umax_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.w %d0, %d3
 ; ATOMIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.w %d0, %d2
-; ATOMIC-NEXT:    beq .LBB9_4
+; ATOMIC-NEXT:    bne .LBB9_4
 ; ATOMIC-NEXT:  .LBB9_1: ; %atomicrmw.start
 ; ATOMIC-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ATOMIC-NEXT:    move.w %d2, %d0
@@ -537,9 +545,10 @@ define i16 @atomicrmw_xchg_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.w %d0, %d3
 ; ATOMIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.w %d0, %d2
-; ATOMIC-NEXT:    bne .LBB10_1
+; ATOMIC-NEXT:    beq .LBB10_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -577,9 +586,10 @@ define i32 @atomicrmw_xchg_i32(i32 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.l %d0, %d3
 ; ATOMIC-NEXT:    sub.l %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.l %d0, %d2
-; ATOMIC-NEXT:    bne .LBB11_1
+; ATOMIC-NEXT:    beq .LBB11_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -622,9 +632,10 @@ define i8 @atomicrmw_sub_i8_arid(ptr align 2 %self) {
 ; ATOMIC-NEXT:    move.b %d0, %d2
 ; ATOMIC-NEXT:    sub.b %d1, %d2
 ; ATOMIC-NEXT:    seq %d1
-; ATOMIC-NEXT:    sub.b #1, %d1
+; ATOMIC-NEXT:    and.b #1, %d1
+; ATOMIC-NEXT:    cmpi.b #0, %d1
 ; ATOMIC-NEXT:    move.b %d0, %d1
-; ATOMIC-NEXT:    bne .LBB12_1
+; ATOMIC-NEXT:    beq .LBB12_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #4, %sp
@@ -669,9 +680,10 @@ define i16 @atomicrmw_sub_i16_arid(ptr align 2 %self) {
 ; ATOMIC-NEXT:    move.w %d0, %d2
 ; ATOMIC-NEXT:    sub.w %d1, %d2
 ; ATOMIC-NEXT:    seq %d1
-; ATOMIC-NEXT:    sub.b #1, %d1
+; ATOMIC-NEXT:    and.b #1, %d1
+; ATOMIC-NEXT:    cmpi.b #0, %d1
 ; ATOMIC-NEXT:    move.w %d0, %d1
-; ATOMIC-NEXT:    bne .LBB13_1
+; ATOMIC-NEXT:    beq .LBB13_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #4, %sp
@@ -716,9 +728,10 @@ define i32 @atomicrmw_sub_i32_arid(ptr align 2 %self) {
 ; ATOMIC-NEXT:    move.l %d0, %d2
 ; ATOMIC-NEXT:    sub.l %d1, %d2
 ; ATOMIC-NEXT:    seq %d1
-; ATOMIC-NEXT:    sub.b #1, %d1
+; ATOMIC-NEXT:    and.b #1, %d1
+; ATOMIC-NEXT:    cmpi.b #0, %d1
 ; ATOMIC-NEXT:    move.l %d0, %d1
-; ATOMIC-NEXT:    bne .LBB14_1
+; ATOMIC-NEXT:    beq .LBB14_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #4, %sp
diff --git a/llvm/test/CodeGen/M68k/Bits/btst.ll b/llvm/test/CodeGen/M68k/Bits/btst.ll
new file mode 100644
index 0000000000000..6c96f906d1812
--- /dev/null
+++ b/llvm/test/CodeGen/M68k/Bits/btst.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=m68k-freestanding -verify-machineinstrs | FileCheck %s
+
+define fastcc i16 @switch_to_btst(i16 %a) nounwind {
+; CHECK-LABEL: switch_to_btst:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    move.l %d0, %d1
+; CHECK-NEXT:    and.l #65535, %d1
+; CHECK-NEXT:    sub.l #11, %d1
+; CHECK-NEXT:    bhi .LBB0_3
+; CHECK-NEXT:  ; %bb.1: ; %entry
+; CHECK-NEXT:    and.l #65535, %d0
+; CHECK-NEXT:    move.l #3612, %d1
+; CHECK-NEXT:    btst %d0, %d1
+; CHECK-NEXT:    beq .LBB0_3
+; CHECK-NEXT:  ; %bb.2: ; %match
+; CHECK-NEXT:    moveq #1, %d0
+; CHECK-NEXT:    rts
+; CHECK-NEXT:  .LBB0_3: ; %no_match
+; CHECK-NEXT:    moveq #0, %d0
+; CHECK-NEXT:    rts
+  entry:
+    switch i16 %a, label %no_match [
+      i16 11, label %match
+      i16 10, label %match
+      i16 9, label %match
+      i16 4, label %match
+      i16 3, label %match
+      i16 2, label %match
+    ]
+
+  match:
+    ret i16 1
+
+  no_match:
+    ret i16 0
+}
+
+define fastcc i16 @and_mask_to_btst(i8 %a, i8 %b) nounwind {
+; CHECK-LABEL: and_mask_to_btst:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    and.l #7, %d1
+; CHECK-NEXT:    btst %d1, %d0
+; CHECK-NEXT:    beq .LBB1_1
+; CHECK-NEXT:  ; %bb.2: ; %cond_false
+; CHECK-NEXT:    moveq #0, %d0
+; CHECK-NEXT:    rts
+; CHECK-NEXT:  .LBB1_1: ; %cond_true
+; CHECK-NEXT:    moveq #1, %d0
+; CHECK-NEXT:    rts
+  %33 = and i8 %b, 7
+  %34 = shl nuw i8 1, %33
+  %35 = and i8 %34, %a
+  %.not = icmp eq i8 %35, 0
+  br i1 %.not, label %cond_true, label %cond_false
+
+  cond_true:
+    ret i16 1
+
+  cond_false:
+    ret i16 0
+}
diff --git a/llvm/test/CodeGen/M68k/CodeModel/Large/Atomics/rmw.ll b/llvm/test/CodeGen/M68k/CodeModel/Large/Atomics/rmw.ll
index b4c2bb1d223c9..06b89adc597f0 100644
--- a/llvm/test/CodeGen/M68k/CodeModel/Large/Atomics/rmw.ll
+++ b/llvm/test/CodeGen/M68k/CodeModel/Large/Atomics/rmw.ll
@@ -55,9 +55,10 @@ define i8 @atomicrmw_add_i8(i8 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.b %d0, %d3
 ; ATOMIC-NEXT:    sub.b %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.b %d0, %d2
-; ATOMIC-NEXT:    bne .LBB0_1
+; ATOMIC-NEXT:    beq .LBB0_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -81,9 +82,10 @@ define i8 @atomicrmw_add_i8(i8 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.b %d0, %d3
 ; ATOMIC-PIC-NEXT:    sub.b %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.b %d0, %d2
-; ATOMIC-PIC-NEXT:    bne .LBB0_1
+; ATOMIC-PIC-NEXT:    beq .LBB0_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #8, %sp
@@ -137,9 +139,10 @@ define i16 @atomicrmw_sub_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.w %d0, %d3
 ; ATOMIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.w %d0, %d2
-; ATOMIC-NEXT:    bne .LBB1_1
+; ATOMIC-NEXT:    beq .LBB1_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -163,9 +166,10 @@ define i16 @atomicrmw_sub_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.w %d0, %d3
 ; ATOMIC-PIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.w %d0, %d2
-; ATOMIC-PIC-NEXT:    bne .LBB1_1
+; ATOMIC-PIC-NEXT:    beq .LBB1_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #8, %sp
@@ -215,9 +219,10 @@ define i32 @atomicrmw_and_i32(i32 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.l %d0, %d3
 ; ATOMIC-NEXT:    sub.l %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.l %d0, %d2
-; ATOMIC-NEXT:    bne .LBB2_1
+; ATOMIC-NEXT:    beq .LBB2_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -241,9 +246,10 @@ define i32 @atomicrmw_and_i32(i32 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.l %d0, %d3
 ; ATOMIC-PIC-NEXT:    sub.l %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.l %d0, %d2
-; ATOMIC-PIC-NEXT:    bne .LBB2_1
+; ATOMIC-PIC-NEXT:    beq .LBB2_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #8, %sp
@@ -353,9 +359,10 @@ define i8 @atomicrmw_or_i8(i8 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.b %d0, %d3
 ; ATOMIC-NEXT:    sub.b %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.b %d0, %d2
-; ATOMIC-NEXT:    bne .LBB4_1
+; ATOMIC-NEXT:    beq .LBB4_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -379,9 +386,10 @@ define i8 @atomicrmw_or_i8(i8 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.b %d0, %d3
 ; ATOMIC-PIC-NEXT:    sub.b %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.b %d0, %d2
-; ATOMIC-PIC-NEXT:    bne .LBB4_1
+; ATOMIC-PIC-NEXT:    beq .LBB4_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #8, %sp
@@ -444,9 +452,10 @@ define i16 @atmoicrmw_nand_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.w %d1, %d3
 ; ATOMIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.w %d1, %d2
-; ATOMIC-NEXT:    bne .LBB5_1
+; ATOMIC-NEXT:    beq .LBB5_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -471,9 +480,10 @@ define i16 @atmoicrmw_nand_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.w %d1, %d3
 ; ATOMIC-PIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.w %d1, %d2
-; ATOMIC-PIC-NEXT:    bne .LBB5_1
+; ATOMIC-PIC-NEXT:    beq .LBB5_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #8, %sp
@@ -522,9 +532,10 @@ define i32 @atomicrmw_min_i32(i32 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.l %d0, %d3
 ; ATOMIC-NEXT:    sub.l %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.l %d0, %d2
-; ATOMIC-NEXT:    beq .LBB6_4
+; ATOMIC-NEXT:    bne .LBB6_4
 ; ATOMIC-NEXT:  .LBB6_1: ; %atomicrmw.start
 ; ATOMIC-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ATOMIC-NEXT:    move.l %d2, %d0
@@ -557,9 +568,10 @@ define i32 @atomicrmw_min_i32(i32 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.l %d0, %d3
 ; ATOMIC-PIC-NEXT:    sub.l %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.l %d0, %d2
-; ATOMIC-PIC-NEXT:    beq .LBB6_4
+; ATOMIC-PIC-NEXT:    bne .LBB6_4
 ; ATOMIC-PIC-NEXT:  .LBB6_1: ; %atomicrmw.start
 ; ATOMIC-PIC-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ATOMIC-PIC-NEXT:    move.l %d2, %d0
@@ -818,9 +830,10 @@ define i8 @atomicrmw_i8_umin(i8 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.b %d0, %d3
 ; ATOMIC-NEXT:    sub.b %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.b %d0, %d2
-; ATOMIC-NEXT:    beq .LBB8_4
+; ATOMIC-NEXT:    bne .LBB8_4
 ; ATOMIC-NEXT:  .LBB8_1: ; %atomicrmw.start
 ; ATOMIC-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ATOMIC-NEXT:    move.b %d2, %d0
@@ -853,9 +866,10 @@ define i8 @atomicrmw_i8_umin(i8 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.b %d0, %d3
 ; ATOMIC-PIC-NEXT:    sub.b %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.b %d0, %d2
-; ATOMIC-PIC-NEXT:    beq .LBB8_4
+; ATOMIC-PIC-NEXT:    bne .LBB8_4
 ; ATOMIC-PIC-NEXT:  .LBB8_1: ; %atomicrmw.start
 ; ATOMIC-PIC-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ATOMIC-PIC-NEXT:    move.b %d2, %d0
@@ -918,9 +932,10 @@ define i16 @atomicrmw_umax_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.w %d0, %d3
 ; ATOMIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.w %d0, %d2
-; ATOMIC-NEXT:    beq .LBB9_4
+; ATOMIC-NEXT:    bne .LBB9_4
 ; ATOMIC-NEXT:  .LBB9_1: ; %atomicrmw.start
 ; ATOMIC-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ATOMIC-NEXT:    move.w %d2, %d0
@@ -953,9 +968,10 @@ define i16 @atomicrmw_umax_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.w %d0, %d3
 ; ATOMIC-PIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.w %d0, %d2
-; ATOMIC-PIC-NEXT:    beq .LBB9_4
+; ATOMIC-PIC-NEXT:    bne .LBB9_4
 ; ATOMIC-PIC-NEXT:  .LBB9_1: ; %atomicrmw.start
 ; ATOMIC-PIC-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; ATOMIC-PIC-NEXT:    move.w %d2, %d0
@@ -1017,9 +1033,10 @@ define i16 @atomicrmw_xchg_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.w %d0, %d3
 ; ATOMIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.w %d0, %d2
-; ATOMIC-NEXT:    bne .LBB10_1
+; ATOMIC-NEXT:    beq .LBB10_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -1041,9 +1058,10 @@ define i16 @atomicrmw_xchg_i16(i16 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.w %d0, %d3
 ; ATOMIC-PIC-NEXT:    sub.w %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.w %d0, %d2
-; ATOMIC-PIC-NEXT:    bne .LBB10_1
+; ATOMIC-PIC-NEXT:    beq .LBB10_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #8, %sp
@@ -1092,9 +1110,10 @@ define i32 @atomicrmw_xchg_i32(i32 %val, ptr %ptr) {
 ; ATOMIC-NEXT:    move.l %d0, %d3
 ; ATOMIC-NEXT:    sub.l %d2, %d3
 ; ATOMIC-NEXT:    seq %d2
-; ATOMIC-NEXT:    sub.b #1, %d2
+; ATOMIC-NEXT:    and.b #1, %d2
+; ATOMIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-NEXT:    move.l %d0, %d2
-; ATOMIC-NEXT:    bne .LBB11_1
+; ATOMIC-NEXT:    beq .LBB11_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #8, %sp
@@ -1116,9 +1135,10 @@ define i32 @atomicrmw_xchg_i32(i32 %val, ptr %ptr) {
 ; ATOMIC-PIC-NEXT:    move.l %d0, %d3
 ; ATOMIC-PIC-NEXT:    sub.l %d2, %d3
 ; ATOMIC-PIC-NEXT:    seq %d2
-; ATOMIC-PIC-NEXT:    sub.b #1, %d2
+; ATOMIC-PIC-NEXT:    and.b #1, %d2
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d2
 ; ATOMIC-PIC-NEXT:    move.l %d0, %d2
-; ATOMIC-PIC-NEXT:    bne .LBB11_1
+; ATOMIC-PIC-NEXT:    beq .LBB11_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2-%d3 ; 12-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #8, %sp
@@ -1175,9 +1195,10 @@ define i8 @atomicrmw_sub_i8_arid(ptr align 2 %self) {
 ; ATOMIC-NEXT:    move.b %d0, %d2
 ; ATOMIC-NEXT:    sub.b %d1, %d2
 ; ATOMIC-NEXT:    seq %d1
-; ATOMIC-NEXT:    sub.b #1, %d1
+; ATOMIC-NEXT:    and.b #1, %d1
+; ATOMIC-NEXT:    cmpi.b #0, %d1
 ; ATOMIC-NEXT:    move.b %d0, %d1
-; ATOMIC-NEXT:    bne .LBB12_1
+; ATOMIC-NEXT:    beq .LBB12_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #4, %sp
@@ -1201,9 +1222,10 @@ define i8 @atomicrmw_sub_i8_arid(ptr align 2 %self) {
 ; ATOMIC-PIC-NEXT:    move.b %d0, %d2
 ; ATOMIC-PIC-NEXT:    sub.b %d1, %d2
 ; ATOMIC-PIC-NEXT:    seq %d1
-; ATOMIC-PIC-NEXT:    sub.b #1, %d1
+; ATOMIC-PIC-NEXT:    and.b #1, %d1
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d1
 ; ATOMIC-PIC-NEXT:    move.b %d0, %d1
-; ATOMIC-PIC-NEXT:    bne .LBB12_1
+; ATOMIC-PIC-NEXT:    beq .LBB12_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #4, %sp
@@ -1262,9 +1284,10 @@ define i16 @atomicrmw_sub_i16_arid(ptr align 2 %self) {
 ; ATOMIC-NEXT:    move.w %d0, %d2
 ; ATOMIC-NEXT:    sub.w %d1, %d2
 ; ATOMIC-NEXT:    seq %d1
-; ATOMIC-NEXT:    sub.b #1, %d1
+; ATOMIC-NEXT:    and.b #1, %d1
+; ATOMIC-NEXT:    cmpi.b #0, %d1
 ; ATOMIC-NEXT:    move.w %d0, %d1
-; ATOMIC-NEXT:    bne .LBB13_1
+; ATOMIC-NEXT:    beq .LBB13_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #4, %sp
@@ -1288,9 +1311,10 @@ define i16 @atomicrmw_sub_i16_arid(ptr align 2 %self) {
 ; ATOMIC-PIC-NEXT:    move.w %d0, %d2
 ; ATOMIC-PIC-NEXT:    sub.w %d1, %d2
 ; ATOMIC-PIC-NEXT:    seq %d1
-; ATOMIC-PIC-NEXT:    sub.b #1, %d1
+; ATOMIC-PIC-NEXT:    and.b #1, %d1
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d1
 ; ATOMIC-PIC-NEXT:    move.w %d0, %d1
-; ATOMIC-PIC-NEXT:    bne .LBB13_1
+; ATOMIC-PIC-NEXT:    beq .LBB13_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #4, %sp
@@ -1349,9 +1373,10 @@ define i32 @atomicrmw_sub_i32_arid(ptr align 2 %self) {
 ; ATOMIC-NEXT:    move.l %d0, %d2
 ; ATOMIC-NEXT:    sub.l %d1, %d2
 ; ATOMIC-NEXT:    seq %d1
-; ATOMIC-NEXT:    sub.b #1, %d1
+; ATOMIC-NEXT:    and.b #1, %d1
+; ATOMIC-NEXT:    cmpi.b #0, %d1
 ; ATOMIC-NEXT:    move.l %d0, %d1
-; ATOMIC-NEXT:    bne .LBB14_1
+; ATOMIC-NEXT:    beq .LBB14_1
 ; ATOMIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; ATOMIC-NEXT:    adda.l #4, %sp
@@ -1375,9 +1400,10 @@ define i32 @atomicrmw_sub_i32_arid(ptr align 2 %self) {
 ; ATOMIC-PIC-NEXT:    move.l %d0, %d2
 ; ATOMIC-PIC-NEXT:    sub.l %d1, %d2
 ; ATOMIC-PIC-NEXT:    seq %d1
-; ATOMIC-PIC-NEXT:    sub.b #1, %d1
+; ATOMIC-PIC-NEXT:    and.b #1, %d1
+; ATOMIC-PIC-NEXT:    cmpi.b #0, %d1
 ; ATOMIC-PIC-NEXT:    move.l %d0, %d1
-; ATOMIC-PIC-NEXT:    bne .LBB14_1
+; ATOMIC-PIC-NEXT:    beq .LBB14_1
 ; ATOMIC-PIC-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; ATOMIC-PIC-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; ATOMIC-PIC-NEXT:    adda.l #4, %sp
diff --git a/llvm/test/CodeGen/M68k/Control/cmp.ll b/llvm/test/CodeGen/M68k/Control/cmp.ll
index d3a8bbb0b0c8f..01dfaeb1b2ec2 100644
--- a/llvm/test/CodeGen/M68k/Control/cmp.ll
+++ b/llvm/test/CodeGen/M68k/Control/cmp.ll
@@ -84,6 +84,7 @@ define i64 @test3(i64 %x) nounwind {
 ; CHECK-NEXT:    seq %d0
 ; CHECK-NEXT:    move.l %d0, %d1
 ; CHECK-NEXT:    and.l #255, %d1
+; CHECK-NEXT:    and.l #1, %d1
 ; CHECK-NEXT:    moveq #0, %d0
 ; CHECK-NEXT:    rts
   %t = icmp eq i64 %x, 0
@@ -103,6 +104,7 @@ define i64 @test4(i64 %x) nounwind {
 ; CHECK-NEXT:    subx.l %d0, %d1
 ; CHECK-NEXT:    slt %d1
 ; CHECK-NEXT:    and.l #255, %d1
+; CHECK-NEXT:    and.l #1, %d1
 ; CHECK-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
 ; CHECK-NEXT:    adda.l #4, %sp
 ; CHECK-NEXT:    rts
@@ -144,6 +146,7 @@ define i32 @test7(i64 %res) nounwind {
 ; CHECK-NEXT:    cmpi.l #0, (4,%sp)
 ; CHECK-NEXT:    seq %d0
 ; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    and.l #1, %d0
 ; CHECK-NEXT:    rts
 entry:
   %lnot = icmp ult i64 %res, 4294967296
@@ -158,6 +161,7 @@ define i32 @test8(i64 %res) nounwind {
 ; CHECK-NEXT:    sub.l #3, %d0
 ; CHECK-NEXT:    scs %d0
 ; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    and.l #1, %d0
 ; CHECK-NEXT:    rts
 entry:
   %lnot = icmp ult i64 %res, 12884901888
@@ -173,6 +177,7 @@ define i32 @test11(i64 %l) nounwind {
 ; CHECK-NEXT:    sub.l #32768, %d0
 ; CHECK-NEXT:    seq %d0
 ; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    and.l #1, %d0
 ; CHECK-NEXT:    rts
 entry:
   %shr.mask = and i64 %l, -140737488355328
@@ -240,6 +245,7 @@ define zeroext i1 @test15(i32 %bf.load, i32 %n) {
 ; CHECK-NEXT:    or.b %d0, %d1
 ; CHECK-NEXT:    move.l %d1, %d0
 ; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    and.l #1, %d0
 ; CHECK-NEXT:    rts
   %bf.lshr = lshr i32 %bf.load, 16
   %cmp2 = icmp eq i32 %bf.lshr, 0
@@ -288,20 +294,24 @@ define void @test20(i32 %bf.load, i8 %x1, ptr %b_addr) {
 ; CHECK-NEXT:  ; %bb.0:
 ; CHECK-NEXT:    suba.l #4, %sp
 ; CHECK-NEXT:    .cfi_def_cfa_offset -8
-; CHECK-NEXT:    movem.l %d2, (0,%sp) ; 8-byte Folded Spill
+; CHECK-NEXT:    movem.l %d2, (0,%sp)                    ; 8-byte Folded Spill
 ; CHECK-NEXT:    move.l #16777215, %d0
 ; CHECK-NEXT:    and.l (8,%sp), %d0
 ; CHECK-NEXT:    sne %d1
 ; CHECK-NEXT:    and.l #255, %d1
-; CHECK-NEXT:    move.l (16,%sp), %a0
+; CHECK-NEXT:    and.l #1, %d1
 ; CHECK-NEXT:    move.b (15,%sp), %d2
 ; CHECK-NEXT:    and.l #255, %d2
 ; CHECK-NEXT:    add.l %d1, %d2
-; CHECK-NEXT:    sne (%a0)
+; CHECK-NEXT:    sne %d1
+; CHECK-NEXT:    and.b #1, %d1
+; CHECK-NEXT:    move.l (16,%sp), %a0
+; CHECK-NEXT:    move.b %d1, (%a0)
 ; CHECK-NEXT:    cmpi.l #0, %d0
-; CHECK-NEXT:    lea (d,%pc), %a0
-; CHECK-NEXT:    sne (%a0)
-; CHECK-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
+; CHECK-NEXT:    sne %d0
+; CHECK-NEXT:    and.b #1, %d0
+; CHECK-NEXT:    move.b %d0, (d,%pc)
+; CHECK-NEXT:    movem.l (0,%sp), %d2                    ; 8-byte Folded Reload
 ; CHECK-NEXT:    adda.l #4, %sp
 ; CHECK-NEXT:    rts
   %bf.shl = shl i32 %bf.load, 8
diff --git a/llvm/test/CodeGen/M68k/Control/non-cmov-switch.ll b/llvm/test/CodeGen/M68k/Control/non-cmov-switch.ll
index 90d2be017ecdb..9c9033e02f48e 100644
--- a/llvm/test/CodeGen/M68k/Control/non-cmov-switch.ll
+++ b/llvm/test/CodeGen/M68k/Control/non-cmov-switch.ll
@@ -16,6 +16,7 @@ define internal void @select_i32(i32 %self, ptr nonnull %value) {
 ; CHECK-NEXT:    bne .LBB0_2
 ; CHECK-NEXT:  ; %bb.1: ; %start
 ; CHECK-NEXT:    and.l #255, %d1
+; CHECK-NEXT:    and.l #1, %d1
 ; CHECK-NEXT:    cmpi.l #0, %d1
 ; CHECK-NEXT:    bne .LBB0_3
 ; CHECK-NEXT:  .LBB0_2: ; %null
@@ -60,6 +61,7 @@ define internal void @select_i16(i16 %self, ptr nonnull %value) {
 ; CHECK-NEXT:    bne .LBB1_2
 ; CHECK-NEXT:  ; %bb.1: ; %start
 ; CHECK-NEXT:    and.l #255, %d1
+; CHECK-NEXT:    and.w #1, %d1
 ; CHECK-NEXT:    cmpi.w #0, %d1
 ; CHECK-NEXT:    bne .LBB1_3
 ; CHECK-NEXT:  .LBB1_2: ; %null
@@ -93,18 +95,26 @@ define internal void @select_i8(i8 %self, ptr nonnull %value) {
 ; CHECK-LABEL: select_i8:
 ; CHECK:         .cfi_startproc
 ; CHECK-NEXT:  ; %bb.0: ; %start
-; CHECK-NEXT:    move.l (8,%sp), %d0
-; CHECK-NEXT:    cmpi.b #0, (7,%sp)
-; CHECK-NEXT:    sne %d1
-; CHECK-NEXT:    bne .LBB2_2
-; CHECK-NEXT:  ; %bb.1: ; %start
-; CHECK-NEXT:    cmpi.b #0, %d1
-; CHECK-NEXT:    bne .LBB2_3
-; CHECK-NEXT:  .LBB2_2: ; %null
-; CHECK-NEXT:    suba.l %a0, %a0
-; CHECK-NEXT:    move.l %d0, (%a0)
-; CHECK-NEXT:  .LBB2_3: ; %exit
-; CHECK-NEXT:    rts
+; CHECK-NEXT:     suba.l #4, %sp
+; CHECK-NEXT:   .cfi_def_cfa_offset -8
+; CHECK-NEXT:   movem.l %d2, (0,%sp)                    ; 8-byte Folded Spill
+; CHECK-NEXT:   cmpi.b #0, (11,%sp)
+; CHECK-NEXT:   move.w %ccr, %d2
+; CHECK-NEXT:   sne %d1
+; CHECK-NEXT:   move.l (12,%sp), %d0
+; CHECK-NEXT:   move.w %d2, %ccr
+; CHECK-NEXT:   bne .LBB2_2
+; CHECK-NEXT:   ; %bb.1:                                ; %start
+; CHECK-NEXT:   and.b #1, %d1
+; CHECK-NEXT:   cmpi.b #0, %d1
+; CHECK-NEXT:   bne .LBB2_3
+; CHECK-NEXT:   .LBB2_2:                                ; %null
+; CHECK-NEXT:   suba.l %a0, %a0
+; CHECK-NEXT:   move.l %d0, (%a0)
+; CHECK-NEXT:   .LBB2_3:                                ; %exit
+; CHECK-NEXT:   movem.l (0,%sp), %d2                    ; 8-byte Folded Reload
+; CHECK-NEXT:   adda.l #4, %sp
+; CHECK-NEXT:   rts
 start:
   %2 = icmp eq i8 %self, 0
   %3 = select i1 %2, i8 0, i8 1
diff --git a/llvm/test/CodeGen/M68k/Control/setcc.ll b/llvm/test/CodeGen/M68k/Control/setcc.ll
index 9e03f9b90842a..a2c24c94ca4cc 100644
--- a/llvm/test/CodeGen/M68k/Control/setcc.ll
+++ b/llvm/test/CodeGen/M68k/Control/setcc.ll
@@ -11,6 +11,7 @@ define zeroext i16 @t1(i16 zeroext %x) nounwind readnone ssp {
 ; CHECK-NEXT:    sub.l #26, %d0
 ; CHECK-NEXT:    shi %d0
 ; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    and.l #1, %d0
 ; CHECK-NEXT:    lsl.l #5, %d0
 ; CHECK-NEXT:    rts
 entry:
@@ -27,6 +28,7 @@ define zeroext i16 @t2(i16 zeroext %x) nounwind readnone ssp {
 ; CHECK-NEXT:    sub.l #26, %d0
 ; CHECK-NEXT:    scs %d0
 ; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    and.l #1, %d0
 ; CHECK-NEXT:    lsl.l #5, %d0
 ; CHECK-NEXT:    rts
 entry:
@@ -39,16 +41,17 @@ define fastcc i64 @t3(i64 %x) nounwind readnone ssp {
 ; CHECK-LABEL: t3:
 ; CHECK:       ; %bb.0: ; %entry
 ; CHECK-NEXT:    suba.l #4, %sp
-; CHECK-NEXT:    movem.l %d2, (0,%sp) ; 8-byte Folded Spill
+; CHECK-NEXT:    movem.l %d2, (0,%sp)                    ; 8-byte Folded Spill
 ; CHECK-NEXT:    moveq #0, %d2
 ; CHECK-NEXT:    sub.l #18, %d1
 ; CHECK-NEXT:    subx.l %d2, %d0
 ; CHECK-NEXT:    scs %d0
 ; CHECK-NEXT:    move.l %d0, %d1
 ; CHECK-NEXT:    and.l #255, %d1
+; CHECK-NEXT:    and.l #1, %d1
 ; CHECK-NEXT:    lsl.l #6, %d1
 ; CHECK-NEXT:    move.l %d2, %d0
-; CHECK-NEXT:    movem.l (0,%sp), %d2 ; 8-byte Folded Reload
+; CHECK-NEXT:    movem.l (0,%sp), %d2                    ; 8-byte Folded Reload
 ; CHECK-NEXT:    adda.l #4, %sp
 ; CHECK-NEXT:    rts
 entry:
diff --git a/llvm/test/CodeGen/M68k/Data/load-extend.ll b/llvm/test/CodeGen/M68k/Data/load-extend.ll
index 51159730ecc0e..687d3f24523dd 100644
--- a/llvm/test/CodeGen/M68k/Data/load-extend.ll
+++ b/llvm/test/CodeGen/M68k/Data/load-extend.ll
@@ -41,3 +41,45 @@ define i32 @"test_zext_pcd_i16_to_i32"() {
   %val2 = zext i16 %val to i32
   ret i32 %val2
 }
+
+define i16 @test_anyext_pcd_i8_to_i16() nounwind {
+; CHECK-LABEL: test_anyext_pcd_i8_to_i16:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    move.b (__unnamed_1+4,%pc), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    lsl.w #8, %d0
+; CHECK-NEXT:    ; kill: def $wd0 killed $wd0 killed $d0
+; CHECK-NEXT:    rts
+  %copyload = load i8, ptr getelementptr inbounds nuw (i8, ptr @0, i32 4)
+  %insert_ext = zext i8 %copyload to i16
+  %insert_shift = shl i16 %insert_ext, 8
+  ret i16 %insert_shift
+}
+
+define i32 @test_anyext_pcd_i8_to_i32() nounwind {
+; CHECK-LABEL: test_anyext_pcd_i8_to_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    moveq #24, %d1
+; CHECK-NEXT:    move.b (__unnamed_1+4,%pc), %d0
+; CHECK-NEXT:    and.l #255, %d0
+; CHECK-NEXT:    lsl.l %d1, %d0
+; CHECK-NEXT:    rts
+  %copyload = load i8, ptr getelementptr inbounds nuw (i8, ptr @0, i32 4)
+  %insert_ext = zext i8 %copyload to i32
+  %insert_shift = shl i32 %insert_ext, 24
+  ret i32 %insert_shift
+}
+
+define i32 @test_anyext_pcd_i16_to_i32() nounwind {
+; CHECK-LABEL: test_anyext_pcd_i16_to_i32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    moveq #16, %d1
+; CHECK-NEXT:    move.w (__unnamed_1+4,%pc), %d0
+; CHECK-NEXT:    and.l #65535, %d0
+; CHECK-NEXT:    lsl.l %d1, %d0
+; CHECK-NEXT:    rts
+  %copyload = load i16, ptr getelementptr inbounds nuw (i8, ptr @0, i32 4)
+  %insert_ext = zext i16 %copyload to i32
+  %insert_shift = shl i32 %insert_ext, 16
+  ret i32 %insert_shift
+}
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll b/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll
index 278cf0150c2f7..929db4c9be1c7 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/long-branch-reg-all-sgpr-used.ll
@@ -17,6 +17,8 @@
 ; CHECK-NEXT:   waveLimiter:     false
 ; CHECK-NEXT:   hasSpilledSGPRs: false
 ; CHECK-NEXT:   hasSpilledVGPRs: false
+; CHECK-NEXT:   numWaveDispatchSGPRs: 0
+; CHECK-NEXT:   numWaveDispatchVGPRs: 0
 ; CHECK-NEXT:   scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
 ; CHECK-NEXT:   frameOffsetReg:  '$fp_reg'
 ; CHECK-NEXT:   stackPtrOffsetReg: '$sgpr32'
@@ -287,6 +289,8 @@
 ; CHECK-NEXT:   waveLimiter:     false
 ; CHECK-NEXT:   hasSpilledSGPRs: false
 ; CHECK-NEXT:   hasSpilledVGPRs: false
+; CHECK-NEXT:   numWaveDispatchSGPRs: 0
+; CHECK-NEXT:   numWaveDispatchVGPRs: 0
 ; CHECK-NEXT:   scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
 ; CHECK-NEXT:   frameOffsetReg:  '$fp_reg'
 ; CHECK-NEXT:   stackPtrOffsetReg: '$sgpr32'
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll
index 890ea44081ce7..f054bea1f2780 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-after-pei.ll
@@ -16,6 +16,8 @@
 ; AFTER-PEI-NEXT: waveLimiter:     false
 ; AFTER-PEI-NEXT: hasSpilledSGPRs: true
 ; AFTER-PEI-NEXT: hasSpilledVGPRs: false
+; AFTER-PEI-NEXT: numWaveDispatchSGPRs: 0
+; AFTER-PEI-NEXT: numWaveDispatchVGPRs: 0
 ; AFTER-PEI-NEXT: scratchRSrcReg:  '$sgpr68_sgpr69_sgpr70_sgpr71'
 ; AFTER-PEI-NEXT: frameOffsetReg:  '$fp_reg'
 ; AFTER-PEI-NEXT: stackPtrOffsetReg: '$sgpr32'
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll
index f84ef8a3844dd..924216efcc461 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg-debug.ll
@@ -17,6 +17,8 @@
 ; CHECK-NEXT: waveLimiter: false
 ; CHECK-NEXT: hasSpilledSGPRs: false
 ; CHECK-NEXT: hasSpilledVGPRs: false
+; CHECK-NEXT: numWaveDispatchSGPRs: 0
+; CHECK-NEXT: numWaveDispatchVGPRs: 0
 ; CHECK-NEXT: scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
 ; CHECK-NEXT: frameOffsetReg:  '$fp_reg'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
index cc834d017c149..39f1ddd0609d8 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
@@ -17,6 +17,8 @@
 ; CHECK-NEXT: waveLimiter: false
 ; CHECK-NEXT: hasSpilledSGPRs: false
 ; CHECK-NEXT: hasSpilledVGPRs: false
+; CHECK-NEXT: numWaveDispatchSGPRs: 0
+; CHECK-NEXT: numWaveDispatchVGPRs: 0
 ; CHECK-NEXT: scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
 ; CHECK-NEXT: frameOffsetReg:  '$fp_reg'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
index 06c580ec6f6b4..0cb9bc095bc50 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-no-ir.mir
@@ -17,6 +17,8 @@
 # FULL-NEXT: waveLimiter:     true
 # FULL-NEXT: hasSpilledSGPRs: false
 # FULL-NEXT: hasSpilledVGPRs: false
+# FULL-NEXT: numWaveDispatchSGPRs: 0
+# FULL-NEXT: numWaveDispatchVGPRs: 0
 # FULL-NEXT: scratchRSrcReg:  '$sgpr8_sgpr9_sgpr10_sgpr11'
 # FULL-NEXT: frameOffsetReg:  '$sgpr12'
 # FULL-NEXT: stackPtrOffsetReg:  '$sgpr13'
@@ -127,6 +129,8 @@ body:             |
 # FULL-NEXT: waveLimiter:     false
 # FULL-NEXT: hasSpilledSGPRs: false
 # FULL-NEXT: hasSpilledVGPRs: false
+# FULL-NEXT: numWaveDispatchSGPRs: 0
+# FULL-NEXT: numWaveDispatchVGPRs: 0
 # FULL-NEXT: scratchRSrcReg:  '$private_rsrc_reg'
 # FULL-NEXT: frameOffsetReg:  '$fp_reg'
 # FULL-NEXT: stackPtrOffsetReg:  '$sp_reg'
@@ -206,6 +210,8 @@ body:             |
 # FULL-NEXT: waveLimiter:     false
 # FULL-NEXT: hasSpilledSGPRs: false
 # FULL-NEXT: hasSpilledVGPRs: false
+# FULL-NEXT: numWaveDispatchSGPRs: 0
+# FULL-NEXT: numWaveDispatchVGPRs: 0
 # FULL-NEXT: scratchRSrcReg:  '$private_rsrc_reg'
 # FULL-NEXT: frameOffsetReg:  '$fp_reg'
 # FULL-NEXT: stackPtrOffsetReg:  '$sp_reg'
@@ -286,6 +292,8 @@ body:             |
 # FULL-NEXT: waveLimiter:     false
 # FULL-NEXT: hasSpilledSGPRs: false
 # FULL-NEXT: hasSpilledVGPRs: false
+# FULL-NEXT: numWaveDispatchSGPRs: 0
+# FULL-NEXT: numWaveDispatchVGPRs: 0
 # FULL-NEXT: scratchRSrcReg:  '$private_rsrc_reg'
 # FULL-NEXT: frameOffsetReg:  '$fp_reg'
 # FULL-NEXT: stackPtrOffsetReg:  '$sp_reg'
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
index 427154651a381..ab4383b675243 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info.ll
@@ -20,6 +20,8 @@
 ; CHECK-NEXT: waveLimiter: false
 ; CHECK-NEXT: hasSpilledSGPRs: false
 ; CHECK-NEXT: hasSpilledVGPRs: false
+; CHECK-NEXT: numWaveDispatchSGPRs: 0
+; CHECK-NEXT: numWaveDispatchVGPRs: 0
 ; CHECK-NEXT: scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
 ; CHECK-NEXT: frameOffsetReg:  '$fp_reg'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'
@@ -80,6 +82,8 @@ define amdgpu_kernel void @kernel(i32 %arg0, i64 %arg1, <16 x i32> %arg2) {
 ; CHECK-NEXT: waveLimiter: false
 ; CHECK-NEXT: hasSpilledSGPRs: false
 ; CHECK-NEXT: hasSpilledVGPRs: false
+; CHECK-NEXT: numWaveDispatchSGPRs: 3
+; CHECK-NEXT: numWaveDispatchVGPRs: 1
 ; CHECK-NEXT: scratchRSrcReg:  '$sgpr96_sgpr97_sgpr98_sgpr99'
 ; CHECK-NEXT: frameOffsetReg:  '$fp_reg'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'
@@ -144,6 +148,8 @@ define amdgpu_ps void @gds_size_shader(i32 %arg0, i32 inreg %arg1) #5 {
 ; CHECK-NEXT: waveLimiter: false
 ; CHECK-NEXT: hasSpilledSGPRs: false
 ; CHECK-NEXT: hasSpilledVGPRs: false
+; CHECK-NEXT: numWaveDispatchSGPRs: 16
+; CHECK-NEXT: numWaveDispatchVGPRs: 0
 ; CHECK-NEXT: scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
 ; CHECK-NEXT: frameOffsetReg: '$sgpr33'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'
@@ -200,6 +206,8 @@ define void @function() {
 ; CHECK-NEXT: waveLimiter: false
 ; CHECK-NEXT: hasSpilledSGPRs: false
 ; CHECK-NEXT: hasSpilledVGPRs: false
+; CHECK-NEXT: numWaveDispatchSGPRs: 16
+; CHECK-NEXT: numWaveDispatchVGPRs: 0
 ; CHECK-NEXT: scratchRSrcReg: '$sgpr0_sgpr1_sgpr2_sgpr3'
 ; CHECK-NEXT: frameOffsetReg: '$sgpr33'
 ; CHECK-NEXT: stackPtrOffsetReg: '$sgpr32'
diff --git a/llvm/test/CodeGen/MIR/ARM/thumb2-sub-sp-t3.mir b/llvm/test/CodeGen/MIR/ARM/thumb2-sub-sp-t3.mir
index e8e35ef2f4d49..8811aa84dea6b 100644
--- a/llvm/test/CodeGen/MIR/ARM/thumb2-sub-sp-t3.mir
+++ b/llvm/test/CodeGen/MIR/ARM/thumb2-sub-sp-t3.mir
@@ -59,8 +59,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  4004
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: v, type: default, offset: 0, size: 4000, alignment: 1,
diff --git a/llvm/test/CodeGen/MIR/Generic/frame-info.mir b/llvm/test/CodeGen/MIR/Generic/frame-info.mir
index d5e014cf62991..32396b7486fe5 100644
--- a/llvm/test/CodeGen/MIR/Generic/frame-info.mir
+++ b/llvm/test/CodeGen/MIR/Generic/frame-info.mir
@@ -46,8 +46,6 @@ tracksRegLiveness: true
 # CHECK-NEXT: hasTailCall: false
 # CHECK-NEXT: isCalleeSavedInfoValid: false
 # CHECK-NEXT: localFrameSize: 0
-# CHECK-NEXT: savePoint:       ''
-# CHECK-NEXT: restorePoint:    ''
 # CHECK: body
 frameInfo:
   maxAlignment:    4
diff --git a/llvm/test/CodeGen/MIR/Hexagon/addrmode-opt-nonreaching.mir b/llvm/test/CodeGen/MIR/Hexagon/addrmode-opt-nonreaching.mir
index bfdf4b4fbd806..af1bad94a1c21 100644
--- a/llvm/test/CodeGen/MIR/Hexagon/addrmode-opt-nonreaching.mir
+++ b/llvm/test/CodeGen/MIR/Hexagon/addrmode-opt-nonreaching.mir
@@ -127,8 +127,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/MIR/RISCV/machine-function-info.mir b/llvm/test/CodeGen/MIR/RISCV/machine-function-info.mir
index 1861c61c30f63..85ab78a5e4df0 100644
--- a/llvm/test/CodeGen/MIR/RISCV/machine-function-info.mir
+++ b/llvm/test/CodeGen/MIR/RISCV/machine-function-info.mir
@@ -58,8 +58,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: -8, size: 8, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/MIR/X86/branch-folder-with-label.mir b/llvm/test/CodeGen/MIR/X86/branch-folder-with-label.mir
index 27737989dcb43..dd469af0bfc9e 100644
--- a/llvm/test/CodeGen/MIR/X86/branch-folder-with-label.mir
+++ b/llvm/test/CodeGen/MIR/X86/branch-folder-with-label.mir
@@ -173,8 +173,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -216,8 +216,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
   - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
       callee-saved-register: '$rbx', callee-saved-restored: true, debug-info-variable: '', 
@@ -290,8 +290,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
   - { id: 0, type: spill-slot, offset: -24, size: 8, alignment: 8, stack-id: default,
       callee-saved-register: '$rbx', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/CodeGen/MIR/X86/diexpr-win32.mir b/llvm/test/CodeGen/MIR/X86/diexpr-win32.mir
index d8d76758a08a0..054c34996d03e 100644
--- a/llvm/test/CodeGen/MIR/X86/diexpr-win32.mir
+++ b/llvm/test/CodeGen/MIR/X86/diexpr-win32.mir
@@ -174,8 +174,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -8, size: 4, alignment: 4, stack-id: default,
       callee-saved-register: '$esi' }
@@ -232,8 +232,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '' }
diff --git a/llvm/test/CodeGen/MIR/X86/fake-use-tailcall.mir b/llvm/test/CodeGen/MIR/X86/fake-use-tailcall.mir
index 3b833ae73551a..c538a6f149f56 100644
--- a/llvm/test/CodeGen/MIR/X86/fake-use-tailcall.mir
+++ b/llvm/test/CodeGen/MIR/X86/fake-use-tailcall.mir
@@ -87,8 +87,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/MIR/X86/frame-info-multiple-save-restore-points-parse.mir b/llvm/test/CodeGen/MIR/X86/frame-info-multiple-save-restore-points-parse.mir
new file mode 100644
index 0000000000000..4c60ccd573595
--- /dev/null
+++ b/llvm/test/CodeGen/MIR/X86/frame-info-multiple-save-restore-points-parse.mir
@@ -0,0 +1,81 @@
+# RUN: llc -mtriple=x86_64 -run-pass none -o - %s | FileCheck %s
+# This test ensures that the MIR parser parses the save and restore points in
+# the machine frame info correctly.
+
+--- |
+
+  define i32 @foo(i32 %a, i32 %b) {
+  entry:
+    %tmp = alloca i32, align 4
+    %tmp2 = icmp slt i32 %a, %b
+    br i1 %tmp2, label %true, label %false
+
+  true:
+    store i32 %a, ptr %tmp, align 4
+    %tmp4 = call i32 @doSomething(i32 0, ptr %tmp)
+    br label %false
+
+  false:
+    %tmp.0 = phi i32 [ %tmp4, %true ], [ %a, %entry ]
+    ret i32 %tmp.0
+  }
+
+  declare i32 @doSomething(i32, ptr)
+
+...
+---
+name:            foo
+tracksRegLiveness: true
+liveins:
+  - { reg: '$edi' }
+  - { reg: '$esi' }
+# CHECK: frameInfo:
+# CHECK:      savePoint:
+# CHECK-NEXT:   - point:           '%bb.1'
+# CHECK-NEXT:   - point:           '%bb.2'
+# CHECK:      restorePoint:
+# CHECK-NEXT:   - point:           '%bb.2'
+# CHECK-NEXT:   - point:           '%bb.3'
+# CHECK: stack
+frameInfo:
+  maxAlignment:  4
+  hasCalls:      true
+  savePoint:
+    - point:     '%bb.1'
+    - point:     '%bb.2'
+  restorePoint:  
+    - point:     '%bb.2'
+    - point:     '%bb.3'
+stack:
+  - { id: 0, name: tmp, offset: 0, size: 4, alignment: 4 }
+body: |
+  bb.0:
+    successors: %bb.2, %bb.1
+    liveins: $edi, $esi
+
+    $eax = COPY $edi
+    CMP32rr $eax, killed $esi, implicit-def $eflags
+    JCC_1 %bb.2, 12, implicit killed $eflags
+
+  bb.1:
+    successors: %bb.3
+    liveins: $eax
+
+    JMP_1 %bb.3
+
+  bb.2.true:
+    successors: %bb.3
+    liveins: $eax
+
+    MOV32mr %stack.0.tmp, 1, _, 0, _, killed $eax
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $ssp, implicit-def dead $eflags, implicit $rsp, implicit $ssp
+    $rsi = LEA64r %stack.0.tmp, 1, _, 0, _
+    $edi = MOV32r0 implicit-def dead $eflags
+    CALL64pcrel32 @doSomething, csr_64, implicit $rsp, implicit $ssp, implicit $edi, implicit $rsi, implicit-def $rsp, implicit-def $ssp, implicit-def $eax
+    ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $ssp, implicit-def dead $eflags, implicit $rsp, implicit $ssp
+
+  bb.3.false:
+    liveins: $eax
+
+    RET64 $eax
+...
diff --git a/llvm/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir b/llvm/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir
index e26233f946606..6357c5a30da89 100644
--- a/llvm/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir
+++ b/llvm/test/CodeGen/MIR/X86/frame-info-save-restore-points.mir
@@ -30,14 +30,18 @@ liveins:
   - { reg: '$edi' }
   - { reg: '$esi' }
 # CHECK: frameInfo:
-# CHECK:      savePoint: '%bb.2'
-# CHECK-NEXT: restorePoint: '%bb.2'
+# CHECK:      savePoint:
+# CHECK-NEXT:   - point:           '%bb.2'
+# CHECK:      restorePoint:
+# CHECK-NEXT:   - point:           '%bb.2'
 # CHECK: stack
 frameInfo:
   maxAlignment:  4
   hasCalls:      true
-  savePoint:     '%bb.2'
-  restorePoint:  '%bb.2'
+  savePoint:
+   - point:           '%bb.2'
+  restorePoint:
+    - point:           '%bb.2'
 stack:
   - { id: 0, name: tmp, offset: 0, size: 4, alignment: 4 }
 body: |
diff --git a/llvm/test/CodeGen/MIR/X86/inline-asm-rm-exhaustion.mir b/llvm/test/CodeGen/MIR/X86/inline-asm-rm-exhaustion.mir
index e9e35ac8da986..46d0469424c50 100644
--- a/llvm/test/CodeGen/MIR/X86/inline-asm-rm-exhaustion.mir
+++ b/llvm/test/CodeGen/MIR/X86/inline-asm-rm-exhaustion.mir
@@ -69,8 +69,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 4, alignment: 16, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -132,8 +132,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
@@ -200,8 +200,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 4, alignment: 16, stack-id: default,
       isImmutable: false, isAliased: false, callee-saved-register: '',
diff --git a/llvm/test/CodeGen/MSP430/lrint-conv.ll b/llvm/test/CodeGen/MSP430/lrint-conv.ll
new file mode 100644
index 0000000000000..04ab2af6102a0
--- /dev/null
+++ b/llvm/test/CodeGen/MSP430/lrint-conv.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; Tests for lrint and llrint, with both i32 and i64 checked.
+
+; RUN: sed 's/ITy/i32/g' %s | llc -mtriple=msp430-unknown-unknown | FileCheck %s --check-prefixes=CHECK
+; RUN: sed 's/ITy/i64/g' %s | llc -mtriple=msp430-unknown-unknown | FileCheck %s --check-prefixes=CHECK
+
+; FIXME: crash "Input type needs to be promoted!"
+; define ITy @test_lrint_ixx_f16(half %x) nounwind {
+;   %res = tail call ITy @llvm.lrint.ITy.f16(half %x)
+;   ret ITy %res
+; }
+
+; define ITy @test_llrint_ixx_f16(half %x) nounwind {
+;   %res = tail call ITy @llvm.llrint.ITy.f16(half %x)
+;   ret ITy %res
+; }
+
+define ITy @test_lrint_ixx_f32(float %x) nounwind {
+; CHECK-LABEL: test_lrint_ixx_f32:
+; CHECK:         call #lrintf
+  %res = tail call ITy @llvm.lrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f32(float %x) nounwind {
+; CHECK-LABEL: test_llrint_ixx_f32:
+; CHECK:         call #llrintf
+  %res = tail call ITy @llvm.llrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_lrint_ixx_f64(double %x) nounwind {
+; CHECK-LABEL: test_lrint_ixx_f64:
+; CHECK:         call #lrint
+  %res = tail call ITy @llvm.lrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f64(double %x) nounwind {
+; CHECK-LABEL: test_llrint_ixx_f64:
+; CHECK:         call #llrint
+  %res = tail call ITy @llvm.llrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+; FIXME(#44744): incorrect libcall
+define ITy @test_lrint_ixx_f128(fp128 %x) nounwind {
+; CHECK-LABEL: test_lrint_ixx_f128:
+; CHECK:         call #lrintl
+  %res = tail call ITy @llvm.lrint.ITy.f128(fp128 %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f128(fp128 %x) nounwind {
+; CHECK-LABEL: test_llrint_ixx_f128:
+; CHECK:         call #llrintl
+  %res = tail call ITy @llvm.llrint.ITy.f128(fp128 %x)
+  ret ITy %res
+}
diff --git a/llvm/test/CodeGen/Mips/delay-slot-filler-bundled-insts-def-use.mir b/llvm/test/CodeGen/Mips/delay-slot-filler-bundled-insts-def-use.mir
index a86fcfe3920b4..e04a84855ed40 100644
--- a/llvm/test/CodeGen/Mips/delay-slot-filler-bundled-insts-def-use.mir
+++ b/llvm/test/CodeGen/Mips/delay-slot-filler-bundled-insts-def-use.mir
@@ -52,8 +52,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/Mips/delay-slot-filler-bundled-insts.mir b/llvm/test/CodeGen/Mips/delay-slot-filler-bundled-insts.mir
index 9cde85bd211a9..3e300371e8115 100644
--- a/llvm/test/CodeGen/Mips/delay-slot-filler-bundled-insts.mir
+++ b/llvm/test/CodeGen/Mips/delay-slot-filler-bundled-insts.mir
@@ -72,8 +72,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/Mips/fmuladd-soft-float.ll b/llvm/test/CodeGen/Mips/fmuladd-soft-float.ll
index bbfb7cf9ca907..409b1a1f818ab 100644
--- a/llvm/test/CodeGen/Mips/fmuladd-soft-float.ll
+++ b/llvm/test/CodeGen/Mips/fmuladd-soft-float.ll
@@ -49,13 +49,11 @@ define float @fmuladd_intrinsic_f32(float %a, float %b, float %c) #0 {
 ; SOFT-FLOAT-64-NEXT:    sd $16, 0($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64-NEXT:    .cfi_offset 31, -8
 ; SOFT-FLOAT-64-NEXT:    .cfi_offset 16, -16
-; SOFT-FLOAT-64-NEXT:    move $16, $6
-; SOFT-FLOAT-64-NEXT:    sll $4, $4, 0
 ; SOFT-FLOAT-64-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64-NEXT:    sll $5, $5, 0
-; SOFT-FLOAT-64-NEXT:    sll $4, $2, 0
+; SOFT-FLOAT-64-NEXT:    move $16, $6
+; SOFT-FLOAT-64-NEXT:    move $4, $2
 ; SOFT-FLOAT-64-NEXT:    jal __addsf3
-; SOFT-FLOAT-64-NEXT:    sll $5, $16, 0
+; SOFT-FLOAT-64-NEXT:    move $5, $16
 ; SOFT-FLOAT-64-NEXT:    ld $16, 0($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    ld $ra, 8($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    jr $ra
@@ -69,13 +67,11 @@ define float @fmuladd_intrinsic_f32(float %a, float %b, float %c) #0 {
 ; SOFT-FLOAT-64R2-NEXT:    sd $16, 0($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 31, -8
 ; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 16, -16
-; SOFT-FLOAT-64R2-NEXT:    move $16, $6
-; SOFT-FLOAT-64R2-NEXT:    sll $4, $4, 0
 ; SOFT-FLOAT-64R2-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64R2-NEXT:    sll $5, $5, 0
-; SOFT-FLOAT-64R2-NEXT:    sll $4, $2, 0
+; SOFT-FLOAT-64R2-NEXT:    move $16, $6
+; SOFT-FLOAT-64R2-NEXT:    move $4, $2
 ; SOFT-FLOAT-64R2-NEXT:    jal __addsf3
-; SOFT-FLOAT-64R2-NEXT:    sll $5, $16, 0
+; SOFT-FLOAT-64R2-NEXT:    move $5, $16
 ; SOFT-FLOAT-64R2-NEXT:    ld $16, 0($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64R2-NEXT:    ld $ra, 8($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64R2-NEXT:    jr $ra
@@ -203,13 +199,11 @@ define float @fmuladd_contract_f32(float %a, float %b, float %c) #0 {
 ; SOFT-FLOAT-64-NEXT:    sd $16, 0($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64-NEXT:    .cfi_offset 31, -8
 ; SOFT-FLOAT-64-NEXT:    .cfi_offset 16, -16
-; SOFT-FLOAT-64-NEXT:    move $16, $6
-; SOFT-FLOAT-64-NEXT:    sll $4, $4, 0
 ; SOFT-FLOAT-64-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64-NEXT:    sll $5, $5, 0
-; SOFT-FLOAT-64-NEXT:    sll $4, $2, 0
+; SOFT-FLOAT-64-NEXT:    move $16, $6
+; SOFT-FLOAT-64-NEXT:    move $4, $2
 ; SOFT-FLOAT-64-NEXT:    jal __addsf3
-; SOFT-FLOAT-64-NEXT:    sll $5, $16, 0
+; SOFT-FLOAT-64-NEXT:    move $5, $16
 ; SOFT-FLOAT-64-NEXT:    ld $16, 0($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    ld $ra, 8($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    jr $ra
@@ -223,13 +217,11 @@ define float @fmuladd_contract_f32(float %a, float %b, float %c) #0 {
 ; SOFT-FLOAT-64R2-NEXT:    sd $16, 0($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 31, -8
 ; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 16, -16
-; SOFT-FLOAT-64R2-NEXT:    move $16, $6
-; SOFT-FLOAT-64R2-NEXT:    sll $4, $4, 0
 ; SOFT-FLOAT-64R2-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64R2-NEXT:    sll $5, $5, 0
-; SOFT-FLOAT-64R2-NEXT:    sll $4, $2, 0
+; SOFT-FLOAT-64R2-NEXT:    move $16, $6
+; SOFT-FLOAT-64R2-NEXT:    move $4, $2
 ; SOFT-FLOAT-64R2-NEXT:    jal __addsf3
-; SOFT-FLOAT-64R2-NEXT:    sll $5, $16, 0
+; SOFT-FLOAT-64R2-NEXT:    move $5, $16
 ; SOFT-FLOAT-64R2-NEXT:    ld $16, 0($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64R2-NEXT:    ld $ra, 8($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64R2-NEXT:    jr $ra
@@ -443,149 +435,169 @@ define <4 x float> @fmuladd_contract_v4f32(<4 x float> %a, <4 x float> %b, <4 x
 ;
 ; SOFT-FLOAT-64-LABEL: fmuladd_contract_v4f32:
 ; SOFT-FLOAT-64:       # %bb.0:
-; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, -64
-; SOFT-FLOAT-64-NEXT:    .cfi_def_cfa_offset 64
-; SOFT-FLOAT-64-NEXT:    sd $ra, 56($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $22, 48($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $21, 40($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $20, 32($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $19, 24($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $18, 16($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $17, 8($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $16, 0($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, -80
+; SOFT-FLOAT-64-NEXT:    .cfi_def_cfa_offset 80
+; SOFT-FLOAT-64-NEXT:    sd $ra, 72($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sd $23, 64($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sd $22, 56($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sd $21, 48($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sd $20, 40($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sd $19, 32($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sd $18, 24($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sd $17, 16($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sd $16, 8($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64-NEXT:    .cfi_offset 31, -8
-; SOFT-FLOAT-64-NEXT:    .cfi_offset 22, -16
-; SOFT-FLOAT-64-NEXT:    .cfi_offset 21, -24
-; SOFT-FLOAT-64-NEXT:    .cfi_offset 20, -32
-; SOFT-FLOAT-64-NEXT:    .cfi_offset 19, -40
-; SOFT-FLOAT-64-NEXT:    .cfi_offset 18, -48
-; SOFT-FLOAT-64-NEXT:    .cfi_offset 17, -56
-; SOFT-FLOAT-64-NEXT:    .cfi_offset 16, -64
+; SOFT-FLOAT-64-NEXT:    .cfi_offset 23, -16
+; SOFT-FLOAT-64-NEXT:    .cfi_offset 22, -24
+; SOFT-FLOAT-64-NEXT:    .cfi_offset 21, -32
+; SOFT-FLOAT-64-NEXT:    .cfi_offset 20, -40
+; SOFT-FLOAT-64-NEXT:    .cfi_offset 19, -48
+; SOFT-FLOAT-64-NEXT:    .cfi_offset 18, -56
+; SOFT-FLOAT-64-NEXT:    .cfi_offset 17, -64
+; SOFT-FLOAT-64-NEXT:    .cfi_offset 16, -72
 ; SOFT-FLOAT-64-NEXT:    move $16, $9
-; SOFT-FLOAT-64-NEXT:    move $17, $8
-; SOFT-FLOAT-64-NEXT:    move $18, $7
-; SOFT-FLOAT-64-NEXT:    move $19, $6
-; SOFT-FLOAT-64-NEXT:    move $20, $5
+; SOFT-FLOAT-64-NEXT:    move $19, $8
+; SOFT-FLOAT-64-NEXT:    move $17, $7
+; SOFT-FLOAT-64-NEXT:    move $20, $6
+; SOFT-FLOAT-64-NEXT:    move $18, $5
 ; SOFT-FLOAT-64-NEXT:    move $21, $4
-; SOFT-FLOAT-64-NEXT:    sll $4, $4, 0
+; SOFT-FLOAT-64-NEXT:    sll $4, $21, 0
 ; SOFT-FLOAT-64-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64-NEXT:    sll $5, $6, 0
+; SOFT-FLOAT-64-NEXT:    sll $5, $20, 0
+; SOFT-FLOAT-64-NEXT:    move $4, $2
+; SOFT-FLOAT-64-NEXT:    jal __addsf3
+; SOFT-FLOAT-64-NEXT:    sll $5, $19, 0
 ; SOFT-FLOAT-64-NEXT:    move $22, $2
-; SOFT-FLOAT-64-NEXT:    dsra $4, $21, 32
+; SOFT-FLOAT-64-NEXT:    sll $4, $18, 0
 ; SOFT-FLOAT-64-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64-NEXT:    dsra $5, $19, 32
-; SOFT-FLOAT-64-NEXT:    sll $4, $2, 0
-; SOFT-FLOAT-64-NEXT:    jal __addsf3
-; SOFT-FLOAT-64-NEXT:    dsra $5, $17, 32
-; SOFT-FLOAT-64-NEXT:    # kill: def $v0 killed $v0 def $v0_64
-; SOFT-FLOAT-64-NEXT:    sll $4, $22, 0
 ; SOFT-FLOAT-64-NEXT:    sll $5, $17, 0
+; SOFT-FLOAT-64-NEXT:    move $23, $2
+; SOFT-FLOAT-64-NEXT:    dsrl $1, $21, 32
+; SOFT-FLOAT-64-NEXT:    sll $4, $1, 0
+; SOFT-FLOAT-64-NEXT:    dsrl $1, $20, 32
+; SOFT-FLOAT-64-NEXT:    jal __mulsf3
+; SOFT-FLOAT-64-NEXT:    sll $5, $1, 0
+; SOFT-FLOAT-64-NEXT:    move $4, $2
+; SOFT-FLOAT-64-NEXT:    dsll $1, $22, 32
+; SOFT-FLOAT-64-NEXT:    dsrl $2, $19, 32
+; SOFT-FLOAT-64-NEXT:    sll $5, $2, 0
 ; SOFT-FLOAT-64-NEXT:    jal __addsf3
-; SOFT-FLOAT-64-NEXT:    dsll $17, $2, 32
+; SOFT-FLOAT-64-NEXT:    dsrl $19, $1, 32
+; SOFT-FLOAT-64-NEXT:    # kill: def $v0 killed $v0 def $v0_64
 ; SOFT-FLOAT-64-NEXT:    dsll $1, $2, 32
-; SOFT-FLOAT-64-NEXT:    dsrl $1, $1, 32
-; SOFT-FLOAT-64-NEXT:    sll $4, $20, 0
-; SOFT-FLOAT-64-NEXT:    sll $5, $18, 0
-; SOFT-FLOAT-64-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64-NEXT:    or $17, $1, $17
-; SOFT-FLOAT-64-NEXT:    move $19, $2
-; SOFT-FLOAT-64-NEXT:    dsra $4, $20, 32
+; SOFT-FLOAT-64-NEXT:    sll $5, $16, 0
+; SOFT-FLOAT-64-NEXT:    or $19, $19, $1
+; SOFT-FLOAT-64-NEXT:    jal __addsf3
+; SOFT-FLOAT-64-NEXT:    move $4, $23
+; SOFT-FLOAT-64-NEXT:    move $20, $2
+; SOFT-FLOAT-64-NEXT:    dsrl $1, $18, 32
+; SOFT-FLOAT-64-NEXT:    sll $4, $1, 0
+; SOFT-FLOAT-64-NEXT:    dsrl $1, $17, 32
 ; SOFT-FLOAT-64-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64-NEXT:    dsra $5, $18, 32
-; SOFT-FLOAT-64-NEXT:    sll $4, $2, 0
+; SOFT-FLOAT-64-NEXT:    sll $5, $1, 0
+; SOFT-FLOAT-64-NEXT:    move $4, $2
+; SOFT-FLOAT-64-NEXT:    dsll $1, $20, 32
+; SOFT-FLOAT-64-NEXT:    dsrl $17, $1, 32
+; SOFT-FLOAT-64-NEXT:    dsrl $1, $16, 32
 ; SOFT-FLOAT-64-NEXT:    jal __addsf3
-; SOFT-FLOAT-64-NEXT:    dsra $5, $16, 32
+; SOFT-FLOAT-64-NEXT:    sll $5, $1, 0
 ; SOFT-FLOAT-64-NEXT:    # kill: def $v0 killed $v0 def $v0_64
-; SOFT-FLOAT-64-NEXT:    dsll $18, $2, 32
-; SOFT-FLOAT-64-NEXT:    sll $4, $19, 0
-; SOFT-FLOAT-64-NEXT:    jal __addsf3
-; SOFT-FLOAT-64-NEXT:    sll $5, $16, 0
 ; SOFT-FLOAT-64-NEXT:    dsll $1, $2, 32
-; SOFT-FLOAT-64-NEXT:    dsrl $1, $1, 32
-; SOFT-FLOAT-64-NEXT:    or $3, $1, $18
-; SOFT-FLOAT-64-NEXT:    move $2, $17
-; SOFT-FLOAT-64-NEXT:    ld $16, 0($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $17, 8($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $18, 16($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $19, 24($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $20, 32($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $21, 40($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $22, 48($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $ra, 56($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    or $3, $17, $1
+; SOFT-FLOAT-64-NEXT:    move $2, $19
+; SOFT-FLOAT-64-NEXT:    ld $16, 8($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ld $17, 16($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ld $18, 24($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ld $19, 32($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ld $20, 40($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ld $21, 48($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ld $22, 56($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ld $23, 64($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ld $ra, 72($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    jr $ra
-; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, 64
+; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, 80
 ;
 ; SOFT-FLOAT-64R2-LABEL: fmuladd_contract_v4f32:
 ; SOFT-FLOAT-64R2:       # %bb.0:
-; SOFT-FLOAT-64R2-NEXT:    daddiu $sp, $sp, -64
-; SOFT-FLOAT-64R2-NEXT:    .cfi_def_cfa_offset 64
-; SOFT-FLOAT-64R2-NEXT:    sd $ra, 56($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64R2-NEXT:    sd $22, 48($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64R2-NEXT:    sd $21, 40($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64R2-NEXT:    sd $20, 32($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64R2-NEXT:    sd $19, 24($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64R2-NEXT:    sd $18, 16($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64R2-NEXT:    sd $17, 8($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64R2-NEXT:    sd $16, 0($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64R2-NEXT:    daddiu $sp, $sp, -80
+; SOFT-FLOAT-64R2-NEXT:    .cfi_def_cfa_offset 80
+; SOFT-FLOAT-64R2-NEXT:    sd $ra, 72($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64R2-NEXT:    sd $23, 64($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64R2-NEXT:    sd $22, 56($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64R2-NEXT:    sd $21, 48($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64R2-NEXT:    sd $20, 40($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64R2-NEXT:    sd $19, 32($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64R2-NEXT:    sd $18, 24($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64R2-NEXT:    sd $17, 16($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64R2-NEXT:    sd $16, 8($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 31, -8
-; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 22, -16
-; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 21, -24
-; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 20, -32
-; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 19, -40
-; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 18, -48
-; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 17, -56
-; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 16, -64
+; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 23, -16
+; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 22, -24
+; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 21, -32
+; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 20, -40
+; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 19, -48
+; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 18, -56
+; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 17, -64
+; SOFT-FLOAT-64R2-NEXT:    .cfi_offset 16, -72
 ; SOFT-FLOAT-64R2-NEXT:    move $16, $9
-; SOFT-FLOAT-64R2-NEXT:    move $17, $8
-; SOFT-FLOAT-64R2-NEXT:    move $18, $7
-; SOFT-FLOAT-64R2-NEXT:    move $19, $6
-; SOFT-FLOAT-64R2-NEXT:    move $20, $5
+; SOFT-FLOAT-64R2-NEXT:    move $19, $8
+; SOFT-FLOAT-64R2-NEXT:    move $17, $7
+; SOFT-FLOAT-64R2-NEXT:    move $20, $6
+; SOFT-FLOAT-64R2-NEXT:    move $18, $5
 ; SOFT-FLOAT-64R2-NEXT:    move $21, $4
-; SOFT-FLOAT-64R2-NEXT:    dsra $4, $4, 32
-; SOFT-FLOAT-64R2-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64R2-NEXT:    dsra $5, $6, 32
-; SOFT-FLOAT-64R2-NEXT:    move $22, $2
 ; SOFT-FLOAT-64R2-NEXT:    sll $4, $21, 0
 ; SOFT-FLOAT-64R2-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64R2-NEXT:    sll $5, $19, 0
-; SOFT-FLOAT-64R2-NEXT:    sll $4, $2, 0
+; SOFT-FLOAT-64R2-NEXT:    sll $5, $20, 0
+; SOFT-FLOAT-64R2-NEXT:    move $4, $2
 ; SOFT-FLOAT-64R2-NEXT:    jal __addsf3
+; SOFT-FLOAT-64R2-NEXT:    sll $5, $19, 0
+; SOFT-FLOAT-64R2-NEXT:    move $22, $2
+; SOFT-FLOAT-64R2-NEXT:    sll $4, $18, 0
+; SOFT-FLOAT-64R2-NEXT:    jal __mulsf3
 ; SOFT-FLOAT-64R2-NEXT:    sll $5, $17, 0
-; SOFT-FLOAT-64R2-NEXT:    sll $4, $22, 0
-; SOFT-FLOAT-64R2-NEXT:    dsra $5, $17, 32
+; SOFT-FLOAT-64R2-NEXT:    move $23, $2
+; SOFT-FLOAT-64R2-NEXT:    dsrl $1, $21, 32
+; SOFT-FLOAT-64R2-NEXT:    sll $4, $1, 0
+; SOFT-FLOAT-64R2-NEXT:    dsrl $1, $20, 32
+; SOFT-FLOAT-64R2-NEXT:    jal __mulsf3
+; SOFT-FLOAT-64R2-NEXT:    sll $5, $1, 0
+; SOFT-FLOAT-64R2-NEXT:    move $4, $2
+; SOFT-FLOAT-64R2-NEXT:    dsrl $1, $19, 32
+; SOFT-FLOAT-64R2-NEXT:    sll $5, $1, 0
 ; SOFT-FLOAT-64R2-NEXT:    jal __addsf3
-; SOFT-FLOAT-64R2-NEXT:    dext $17, $2, 0, 32
+; SOFT-FLOAT-64R2-NEXT:    dext $19, $22, 0, 32
 ; SOFT-FLOAT-64R2-NEXT:    # kill: def $v0 killed $v0 def $v0_64
 ; SOFT-FLOAT-64R2-NEXT:    dsll $1, $2, 32
-; SOFT-FLOAT-64R2-NEXT:    dsra $4, $20, 32
-; SOFT-FLOAT-64R2-NEXT:    dsra $5, $18, 32
-; SOFT-FLOAT-64R2-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64R2-NEXT:    or $17, $17, $1
-; SOFT-FLOAT-64R2-NEXT:    move $19, $2
-; SOFT-FLOAT-64R2-NEXT:    sll $4, $20, 0
-; SOFT-FLOAT-64R2-NEXT:    jal __mulsf3
-; SOFT-FLOAT-64R2-NEXT:    sll $5, $18, 0
-; SOFT-FLOAT-64R2-NEXT:    sll $4, $2, 0
-; SOFT-FLOAT-64R2-NEXT:    jal __addsf3
 ; SOFT-FLOAT-64R2-NEXT:    sll $5, $16, 0
-; SOFT-FLOAT-64R2-NEXT:    dext $18, $2, 0, 32
-; SOFT-FLOAT-64R2-NEXT:    sll $4, $19, 0
+; SOFT-FLOAT-64R2-NEXT:    or $19, $19, $1
+; SOFT-FLOAT-64R2-NEXT:    jal __addsf3
+; SOFT-FLOAT-64R2-NEXT:    move $4, $23
+; SOFT-FLOAT-64R2-NEXT:    move $20, $2
+; SOFT-FLOAT-64R2-NEXT:    dsrl $1, $18, 32
+; SOFT-FLOAT-64R2-NEXT:    sll $4, $1, 0
+; SOFT-FLOAT-64R2-NEXT:    dsrl $1, $17, 32
+; SOFT-FLOAT-64R2-NEXT:    jal __mulsf3
+; SOFT-FLOAT-64R2-NEXT:    sll $5, $1, 0
+; SOFT-FLOAT-64R2-NEXT:    move $4, $2
+; SOFT-FLOAT-64R2-NEXT:    dext $17, $20, 0, 32
+; SOFT-FLOAT-64R2-NEXT:    dsrl $1, $16, 32
 ; SOFT-FLOAT-64R2-NEXT:    jal __addsf3
-; SOFT-FLOAT-64R2-NEXT:    dsra $5, $16, 32
+; SOFT-FLOAT-64R2-NEXT:    sll $5, $1, 0
 ; SOFT-FLOAT-64R2-NEXT:    # kill: def $v0 killed $v0 def $v0_64
 ; SOFT-FLOAT-64R2-NEXT:    dsll $1, $2, 32
-; SOFT-FLOAT-64R2-NEXT:    or $3, $18, $1
-; SOFT-FLOAT-64R2-NEXT:    move $2, $17
-; SOFT-FLOAT-64R2-NEXT:    ld $16, 0($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64R2-NEXT:    ld $17, 8($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64R2-NEXT:    ld $18, 16($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64R2-NEXT:    ld $19, 24($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64R2-NEXT:    ld $20, 32($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64R2-NEXT:    ld $21, 40($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64R2-NEXT:    ld $22, 48($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64R2-NEXT:    ld $ra, 56($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64R2-NEXT:    or $3, $17, $1
+; SOFT-FLOAT-64R2-NEXT:    move $2, $19
+; SOFT-FLOAT-64R2-NEXT:    ld $16, 8($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64R2-NEXT:    ld $17, 16($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64R2-NEXT:    ld $18, 24($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64R2-NEXT:    ld $19, 32($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64R2-NEXT:    ld $20, 40($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64R2-NEXT:    ld $21, 48($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64R2-NEXT:    ld $22, 56($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64R2-NEXT:    ld $23, 64($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64R2-NEXT:    ld $ra, 72($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64R2-NEXT:    jr $ra
-; SOFT-FLOAT-64R2-NEXT:    daddiu $sp, $sp, 64
+; SOFT-FLOAT-64R2-NEXT:    daddiu $sp, $sp, 80
   %product = fmul contract <4 x float> %a, %b
   %result = fadd contract <4 x float> %product, %c
   ret <4 x float> %result
diff --git a/llvm/test/CodeGen/Mips/indirect-jump-hazard/guards-verify-call.mir b/llvm/test/CodeGen/Mips/indirect-jump-hazard/guards-verify-call.mir
index fe72a1720fb01..bce0ea1cdb356 100644
--- a/llvm/test/CodeGen/Mips/indirect-jump-hazard/guards-verify-call.mir
+++ b/llvm/test/CodeGen/Mips/indirect-jump-hazard/guards-verify-call.mir
@@ -41,8 +41,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/indirect-jump-hazard/guards-verify-tailcall.mir b/llvm/test/CodeGen/Mips/indirect-jump-hazard/guards-verify-tailcall.mir
index e87af5892e34b..7bad3ff8c21fb 100644
--- a/llvm/test/CodeGen/Mips/indirect-jump-hazard/guards-verify-tailcall.mir
+++ b/llvm/test/CodeGen/Mips/indirect-jump-hazard/guards-verify-tailcall.mir
@@ -42,8 +42,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dext-pos.mir b/llvm/test/CodeGen/Mips/instverify/dext-pos.mir
index 1574eeb11a3d5..3f00162604a03 100644
--- a/llvm/test/CodeGen/Mips/instverify/dext-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dext-pos.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dext-size.mir b/llvm/test/CodeGen/Mips/instverify/dext-size.mir
index 2e750aade31a5..ca8d03270bec7 100644
--- a/llvm/test/CodeGen/Mips/instverify/dext-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dext-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dextm-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dextm-pos-size.mir
index 9a0e9a37282cd..a26ca38ca087e 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextm-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextm-pos-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dextm-pos.mir b/llvm/test/CodeGen/Mips/instverify/dextm-pos.mir
index e3862d7d242ea..5f780c45c384b 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextm-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextm-pos.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dextm-size.mir b/llvm/test/CodeGen/Mips/instverify/dextm-size.mir
index 74522e991d97a..2e155f4ca7a4f 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextm-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextm-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dextu-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dextu-pos-size.mir
index f69f20fcfb7e4..2e5da9d57a20a 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextu-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextu-pos-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dextu-pos.mir b/llvm/test/CodeGen/Mips/instverify/dextu-pos.mir
index 52e5b6d0f6921..c91c6f6d194c4 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextu-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextu-pos.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dextu-size-valid.mir b/llvm/test/CodeGen/Mips/instverify/dextu-size-valid.mir
index 75c1c1e44c6c6..7ca5b9c7b64fc 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextu-size-valid.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextu-size-valid.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dextu-size.mir b/llvm/test/CodeGen/Mips/instverify/dextu-size.mir
index b001b42c2cf3c..a030cf3206144 100644
--- a/llvm/test/CodeGen/Mips/instverify/dextu-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dextu-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dins-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dins-pos-size.mir
index c881d65898ee8..510b5521ddc65 100644
--- a/llvm/test/CodeGen/Mips/instverify/dins-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dins-pos-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dins-pos.mir b/llvm/test/CodeGen/Mips/instverify/dins-pos.mir
index b074b87821947..9f47daeab3ed9 100644
--- a/llvm/test/CodeGen/Mips/instverify/dins-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dins-pos.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dins-size.mir b/llvm/test/CodeGen/Mips/instverify/dins-size.mir
index c9a3410aef673..8c6c0cbd9a241 100644
--- a/llvm/test/CodeGen/Mips/instverify/dins-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dins-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsm-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dinsm-pos-size.mir
index 0ae16d1d43d6b..42f1ca0ab3078 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsm-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsm-pos-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsm-pos.mir b/llvm/test/CodeGen/Mips/instverify/dinsm-pos.mir
index fe622e12e7fe7..9109377093e0d 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsm-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsm-pos.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsm-size.mir b/llvm/test/CodeGen/Mips/instverify/dinsm-size.mir
index 2ae4dc7c9f2ba..69839d4ef013a 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsm-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsm-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsu-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/dinsu-pos-size.mir
index 34c509a016fc6..71957d98e4fdd 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsu-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsu-pos-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsu-pos.mir b/llvm/test/CodeGen/Mips/instverify/dinsu-pos.mir
index 7ce2c9509f9e7..0a2428a09e30a 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsu-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsu-pos.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/dinsu-size.mir b/llvm/test/CodeGen/Mips/instverify/dinsu-size.mir
index 9e185cb943f26..f6ce04370a409 100644
--- a/llvm/test/CodeGen/Mips/instverify/dinsu-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/dinsu-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/ext-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/ext-pos-size.mir
index da42bac7140c7..bbac1b0efa924 100644
--- a/llvm/test/CodeGen/Mips/instverify/ext-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ext-pos-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/ext-pos.mir b/llvm/test/CodeGen/Mips/instverify/ext-pos.mir
index 1762d8942aee1..f67a4a5eead5a 100644
--- a/llvm/test/CodeGen/Mips/instverify/ext-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ext-pos.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/ext-size.mir b/llvm/test/CodeGen/Mips/instverify/ext-size.mir
index 42158f63872c2..1089b495779e6 100644
--- a/llvm/test/CodeGen/Mips/instverify/ext-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ext-size.mir
@@ -32,8 +32,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/ins-pos-size.mir b/llvm/test/CodeGen/Mips/instverify/ins-pos-size.mir
index 72d6d690429e4..526df82688637 100644
--- a/llvm/test/CodeGen/Mips/instverify/ins-pos-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ins-pos-size.mir
@@ -35,8 +35,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/ins-pos.mir b/llvm/test/CodeGen/Mips/instverify/ins-pos.mir
index c2464d3d94c24..03f524975ce02 100644
--- a/llvm/test/CodeGen/Mips/instverify/ins-pos.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ins-pos.mir
@@ -35,8 +35,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/instverify/ins-size.mir b/llvm/test/CodeGen/Mips/instverify/ins-size.mir
index 050e909933efb..4375e01023849 100644
--- a/llvm/test/CodeGen/Mips/instverify/ins-size.mir
+++ b/llvm/test/CodeGen/Mips/instverify/ins-size.mir
@@ -35,8 +35,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/llrint-conv.ll b/llvm/test/CodeGen/Mips/llrint-conv.ll
index dcb4e5657e80b..592d40c0f65aa 100644
--- a/llvm/test/CodeGen/Mips/llrint-conv.ll
+++ b/llvm/test/CodeGen/Mips/llrint-conv.ll
@@ -1,10 +1,25 @@
 ; RUN: llc < %s -mtriple=mips64el -mattr=+soft-float | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mattr=+soft-float     | FileCheck %s
+
+; FIXME: crash
+; define signext i32 @testmswh(half %x) {
+; entry:
+;   %0 = tail call i64 @llvm.llrint.i64.f16(half %x)
+;   %conv = trunc i64 %0 to i32
+;   ret i32 %conv
+; }
+
+; define i64 @testmsxh(half %x) {
+; entry:
+;   %0 = tail call i64 @llvm.llrint.i64.f16(half %x)
+;   ret i64 %0
+; }
 
 define signext i32 @testmsws(float %x) {
 ; CHECK-LABEL: testmsws:
 ; CHECK:       jal     llrintf
 entry:
-  %0 = tail call i64 @llvm.llrint.f32(float %x)
+  %0 = tail call i64 @llvm.llrint.i64.f32(float %x)
   %conv = trunc i64 %0 to i32
   ret i32 %conv
 }
@@ -13,7 +28,7 @@ define i64 @testmsxs(float %x) {
 ; CHECK-LABEL: testmsxs:
 ; CHECK:       jal     llrintf
 entry:
-  %0 = tail call i64 @llvm.llrint.f32(float %x)
+  %0 = tail call i64 @llvm.llrint.i64.f32(float %x)
   ret i64 %0
 }
 
@@ -21,7 +36,7 @@ define signext i32 @testmswd(double %x) {
 ; CHECK-LABEL: testmswd:
 ; CHECK:       jal     llrint
 entry:
-  %0 = tail call i64 @llvm.llrint.f64(double %x)
+  %0 = tail call i64 @llvm.llrint.i64.f64(double %x)
   %conv = trunc i64 %0 to i32
   ret i32 %conv
 }
@@ -30,7 +45,7 @@ define i64 @testmsxd(double %x) {
 ; CHECK-LABEL: testmsxd:
 ; CHECK:       jal     llrint
 entry:
-  %0 = tail call i64 @llvm.llrint.f64(double %x)
+  %0 = tail call i64 @llvm.llrint.i64.f64(double %x)
   ret i64 %0
 }
 
@@ -38,7 +53,7 @@ define signext i32 @testmswl(fp128 %x) {
 ; CHECK-LABEL: testmswl:
 ; CHECK:       jal     llrintl
 entry:
-  %0 = tail call i64 @llvm.llrint.f128(fp128 %x)
+  %0 = tail call i64 @llvm.llrint.i64.f128(fp128 %x)
   %conv = trunc i64 %0 to i32
   ret i32 %conv
 }
@@ -47,10 +62,10 @@ define i64 @testmsll(fp128 %x) {
 ; CHECK-LABEL: testmsll:
 ; CHECK:       jal     llrintl
 entry:
-  %0 = tail call i64 @llvm.llrint.f128(fp128 %x)
+  %0 = tail call i64 @llvm.llrint.i64.f128(fp128 %x)
   ret i64 %0
 }
 
-declare i64 @llvm.llrint.f32(float) nounwind readnone
-declare i64 @llvm.llrint.f64(double) nounwind readnone
-declare i64 @llvm.llrint.f128(fp128) nounwind readnone
+declare i64 @llvm.llrint.i64.f32(float) nounwind readnone
+declare i64 @llvm.llrint.i64.f64(double) nounwind readnone
+declare i64 @llvm.llrint.i64.f128(fp128) nounwind readnone
diff --git a/llvm/test/CodeGen/Mips/llvm.frexp.ll b/llvm/test/CodeGen/Mips/llvm.frexp.ll
index 3226766492ed6..69ad5fef7c66e 100644
--- a/llvm/test/CodeGen/Mips/llvm.frexp.ll
+++ b/llvm/test/CodeGen/Mips/llvm.frexp.ll
@@ -493,11 +493,9 @@ define { fp128, i32 } @test_frexp_fp128_i32(fp128 %a) nounwind {
 ; SOFT-FLOAT-64:       # %bb.0:
 ; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, -16
 ; SOFT-FLOAT-64-NEXT:    sd $ra, 8($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    dmfc1 $4, $f12
-; SOFT-FLOAT-64-NEXT:    dmfc1 $5, $f13
 ; SOFT-FLOAT-64-NEXT:    jal frexpl
 ; SOFT-FLOAT-64-NEXT:    daddiu $6, $sp, 4
-; SOFT-FLOAT-64-NEXT:    lw $4, 4($sp)
+; SOFT-FLOAT-64-NEXT:    lw $2, 4($sp)
 ; SOFT-FLOAT-64-NEXT:    ld $ra, 8($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    jr $ra
 ; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, 16
@@ -597,39 +595,39 @@ define { <2 x fp128>, <2 x i32> } @test_frexp_v2fp128_v2i32(<2 x fp128> %a) noun
 ; SOFT-FLOAT-64-LABEL: test_frexp_v2fp128_v2i32:
 ; SOFT-FLOAT-64:       # %bb.0:
 ; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, -64
-; SOFT-FLOAT-64-NEXT:    sd $ra, 56($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $20, 48($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $19, 40($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sdc1 $f25, 56($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sdc1 $f24, 48($sp) # 8-byte Folded Spill
+; SOFT-FLOAT-64-NEXT:    sd $ra, 40($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64-NEXT:    sd $18, 32($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64-NEXT:    sd $17, 24($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64-NEXT:    sd $16, 16($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64-NEXT:    move $16, $6
 ; SOFT-FLOAT-64-NEXT:    move $17, $5
 ; SOFT-FLOAT-64-NEXT:    move $18, $4
-; SOFT-FLOAT-64-NEXT:    daddiu $6, $sp, 12
-; SOFT-FLOAT-64-NEXT:    move $4, $7
+; SOFT-FLOAT-64-NEXT:    dmtc1 $7, $f12
+; SOFT-FLOAT-64-NEXT:    dmtc1 $8, $f13
 ; SOFT-FLOAT-64-NEXT:    jal frexpl
-; SOFT-FLOAT-64-NEXT:    move $5, $8
-; SOFT-FLOAT-64-NEXT:    move $19, $2
-; SOFT-FLOAT-64-NEXT:    move $20, $3
+; SOFT-FLOAT-64-NEXT:    daddiu $6, $sp, 12
+; SOFT-FLOAT-64-NEXT:    mov.d $f24, $f0
+; SOFT-FLOAT-64-NEXT:    mov.d $f25, $f2
+; SOFT-FLOAT-64-NEXT:    dmtc1 $17, $f12
+; SOFT-FLOAT-64-NEXT:    dmtc1 $16, $f13
 ; SOFT-FLOAT-64-NEXT:    daddiu $6, $sp, 8
 ; SOFT-FLOAT-64-NEXT:    lw $1, 12($sp)
-; SOFT-FLOAT-64-NEXT:    sw $1, 36($18)
-; SOFT-FLOAT-64-NEXT:    move $4, $17
 ; SOFT-FLOAT-64-NEXT:    jal frexpl
-; SOFT-FLOAT-64-NEXT:    move $5, $16
+; SOFT-FLOAT-64-NEXT:    sw $1, 36($18)
 ; SOFT-FLOAT-64-NEXT:    lw $1, 8($sp)
 ; SOFT-FLOAT-64-NEXT:    sw $1, 32($18)
-; SOFT-FLOAT-64-NEXT:    sd $20, 24($18)
-; SOFT-FLOAT-64-NEXT:    sd $19, 16($18)
-; SOFT-FLOAT-64-NEXT:    sd $3, 8($18)
-; SOFT-FLOAT-64-NEXT:    sd $2, 0($18)
+; SOFT-FLOAT-64-NEXT:    sdc1 $f25, 24($18)
+; SOFT-FLOAT-64-NEXT:    sdc1 $f24, 16($18)
+; SOFT-FLOAT-64-NEXT:    sdc1 $f2, 8($18)
+; SOFT-FLOAT-64-NEXT:    sdc1 $f0, 0($18)
 ; SOFT-FLOAT-64-NEXT:    ld $16, 16($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    ld $17, 24($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    ld $18, 32($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $19, 40($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $20, 48($sp) # 8-byte Folded Reload
-; SOFT-FLOAT-64-NEXT:    ld $ra, 56($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ld $ra, 40($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ldc1 $f24, 48($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ldc1 $f25, 56($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    jr $ra
 ; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, 64
   %result = call { <2 x fp128>, <2 x i32> } @llvm.frexp.v2fp128.v2i32(<2 x fp128> %a)
diff --git a/llvm/test/CodeGen/Mips/llvm.sincos.ll b/llvm/test/CodeGen/Mips/llvm.sincos.ll
index 046be12c0abe2..58eb427d3a69b 100644
--- a/llvm/test/CodeGen/Mips/llvm.sincos.ll
+++ b/llvm/test/CodeGen/Mips/llvm.sincos.ll
@@ -844,22 +844,13 @@ define { fp128, fp128 } @test_sincos_f128(fp128 %a) #0 {
 ; SOFT-FLOAT-64:       # %bb.0:
 ; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, -48
 ; SOFT-FLOAT-64-NEXT:    sd $ra, 40($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    sd $16, 32($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    move $16, $4
-; SOFT-FLOAT-64-NEXT:    dmfc1 $4, $f13
-; SOFT-FLOAT-64-NEXT:    dmfc1 $5, $f14
 ; SOFT-FLOAT-64-NEXT:    daddiu $6, $sp, 16
 ; SOFT-FLOAT-64-NEXT:    jal sincosl
 ; SOFT-FLOAT-64-NEXT:    daddiu $7, $sp, 0
-; SOFT-FLOAT-64-NEXT:    ld $1, 8($sp)
-; SOFT-FLOAT-64-NEXT:    sd $1, 24($16)
-; SOFT-FLOAT-64-NEXT:    ld $1, 0($sp)
-; SOFT-FLOAT-64-NEXT:    sd $1, 16($16)
-; SOFT-FLOAT-64-NEXT:    ld $1, 24($sp)
-; SOFT-FLOAT-64-NEXT:    sd $1, 8($16)
-; SOFT-FLOAT-64-NEXT:    ld $1, 16($sp)
-; SOFT-FLOAT-64-NEXT:    sd $1, 0($16)
-; SOFT-FLOAT-64-NEXT:    ld $16, 32($sp) # 8-byte Folded Reload
+; SOFT-FLOAT-64-NEXT:    ldc1 $f0, 16($sp)
+; SOFT-FLOAT-64-NEXT:    ldc1 $f2, 24($sp)
+; SOFT-FLOAT-64-NEXT:    ld $2, 0($sp)
+; SOFT-FLOAT-64-NEXT:    ld $3, 8($sp)
 ; SOFT-FLOAT-64-NEXT:    ld $ra, 40($sp) # 8-byte Folded Reload
 ; SOFT-FLOAT-64-NEXT:    jr $ra
 ; SOFT-FLOAT-64-NEXT:    daddiu $sp, $sp, 48
@@ -1001,20 +992,19 @@ define { <2 x fp128>, <2 x fp128> } @test_sincos_v2f128(<2 x fp128> %a) #0 {
 ; SOFT-FLOAT-64-NEXT:    sd $18, 80($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64-NEXT:    sd $17, 72($sp) # 8-byte Folded Spill
 ; SOFT-FLOAT-64-NEXT:    sd $16, 64($sp) # 8-byte Folded Spill
-; SOFT-FLOAT-64-NEXT:    move $1, $7
 ; SOFT-FLOAT-64-NEXT:    move $16, $6
 ; SOFT-FLOAT-64-NEXT:    move $17, $5
 ; SOFT-FLOAT-64-NEXT:    move $18, $4
+; SOFT-FLOAT-64-NEXT:    dmtc1 $7, $f12
+; SOFT-FLOAT-64-NEXT:    dmtc1 $8, $f13
 ; SOFT-FLOAT-64-NEXT:    daddiu $6, $sp, 48
-; SOFT-FLOAT-64-NEXT:    daddiu $7, $sp, 32
-; SOFT-FLOAT-64-NEXT:    move $4, $1
 ; SOFT-FLOAT-64-NEXT:    jal sincosl
-; SOFT-FLOAT-64-NEXT:    move $5, $8
+; SOFT-FLOAT-64-NEXT:    daddiu $7, $sp, 32
+; SOFT-FLOAT-64-NEXT:    dmtc1 $17, $f12
+; SOFT-FLOAT-64-NEXT:    dmtc1 $16, $f13
 ; SOFT-FLOAT-64-NEXT:    daddiu $6, $sp, 16
-; SOFT-FLOAT-64-NEXT:    daddiu $7, $sp, 0
-; SOFT-FLOAT-64-NEXT:    move $4, $17
 ; SOFT-FLOAT-64-NEXT:    jal sincosl
-; SOFT-FLOAT-64-NEXT:    move $5, $16
+; SOFT-FLOAT-64-NEXT:    daddiu $7, $sp, 0
 ; SOFT-FLOAT-64-NEXT:    ld $1, 56($sp)
 ; SOFT-FLOAT-64-NEXT:    ld $2, 0($sp)
 ; SOFT-FLOAT-64-NEXT:    ld $3, 8($sp)
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-micromips.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-micromips.mir
index b686a5e8f1294..4207cac69ddd0 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-micromips.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-micromips.mir
@@ -62,8 +62,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -162,8 +162,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-micromipsr6.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-micromipsr6.mir
index aa6dfed58d5a5..8d4103a1686fa 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-micromipsr6.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-micromipsr6.mir
@@ -62,8 +62,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -154,8 +154,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-mips.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-mips.mir
index 22625d726811a..ea716851ed89d 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-mips.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-mips.mir
@@ -61,8 +61,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -165,8 +165,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-mipsr6.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-mipsr6.mir
index a16573fac3c58..58120af18dc59 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-mipsr6.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-fp-mipsr6.mir
@@ -63,8 +63,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -163,8 +163,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-microMIPS.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-microMIPS.mir
index 9a85a50113b4a..b321a6ff0f1c3 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-microMIPS.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-microMIPS.mir
@@ -123,8 +123,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -214,8 +214,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -309,8 +309,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -404,8 +404,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -499,8 +499,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -594,8 +594,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -685,8 +685,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -780,8 +780,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-micromipsr6.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-micromipsr6.mir
index 67ac46cbe5827..9cae905739ce1 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-micromipsr6.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-micromipsr6.mir
@@ -167,8 +167,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -254,8 +254,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -341,8 +341,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -428,8 +428,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -515,8 +515,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -602,8 +602,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -689,8 +689,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -776,8 +776,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -863,8 +863,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -950,8 +950,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1037,8 +1037,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1124,8 +1124,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mips64.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mips64.mir
index 4f293b70075a2..e126f8749fce3 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mips64.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mips64.mir
@@ -107,8 +107,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -209,8 +209,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -311,8 +311,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -413,8 +413,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -515,8 +515,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -617,8 +617,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mips64r6.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mips64r6.mir
index da4c0f0acd6d4..10fd50a79edb0 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mips64r6.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mips64r6.mir
@@ -179,8 +179,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -256,8 +256,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -333,8 +333,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -431,8 +431,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -529,8 +529,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -627,8 +627,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -725,8 +725,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -823,8 +823,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -921,8 +921,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1019,8 +1019,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1117,8 +1117,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1215,8 +1215,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mipsr6.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mipsr6.mir
index bc110992c571e..79c3b4a8ebc1f 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mipsr6.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int-mipsr6.mir
@@ -167,8 +167,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -254,8 +254,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -341,8 +341,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -428,8 +428,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -515,8 +515,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -602,8 +602,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -689,8 +689,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -776,8 +776,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -863,8 +863,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -950,8 +950,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1037,8 +1037,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1124,8 +1124,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int.mir
index 4174f3321dfea..413a8c2fea74b 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-int.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-int.mir
@@ -101,8 +101,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -200,8 +200,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -299,8 +299,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -398,8 +398,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -497,8 +497,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -596,8 +596,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/longbranch/branch-limits-msa.mir b/llvm/test/CodeGen/Mips/longbranch/branch-limits-msa.mir
index c83859bad0fcb..6c52ce1d86a3f 100644
--- a/llvm/test/CodeGen/Mips/longbranch/branch-limits-msa.mir
+++ b/llvm/test/CodeGen/Mips/longbranch/branch-limits-msa.mir
@@ -248,8 +248,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -365,8 +365,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -479,8 +479,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -593,8 +593,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -704,8 +704,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -815,8 +815,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -932,8 +932,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1046,8 +1046,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1160,8 +1160,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1271,8 +1271,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/lrint-conv.ll b/llvm/test/CodeGen/Mips/lrint-conv.ll
index bd3f7b3babe10..6d2e392675f1c 100644
--- a/llvm/test/CodeGen/Mips/lrint-conv.ll
+++ b/llvm/test/CodeGen/Mips/lrint-conv.ll
@@ -1,4 +1,19 @@
 ; RUN: llc < %s -mtriple=mips64el -mattr=+soft-float | FileCheck %s
+; RUN: llc < %s -mtriple=mips -mattr=+soft-float     | FileCheck %s
+
+; FIXME: crash
+; define signext i32 @testmswh(half %x) {
+; entry:
+;   %0 = tail call i64 @llvm.lrint.i64.f16(half %x)
+;   %conv = trunc i64 %0 to i32
+;   ret i32 %conv
+; }
+
+; define i64 @testmsxh(half %x) {
+; entry:
+;   %0 = tail call i64 @llvm.lrint.i64.f16(half %x)
+;   ret i64 %0
+; }
 
 define signext i32 @testmsws(float %x) {
 ; CHECK-LABEL: testmsws:
diff --git a/llvm/test/CodeGen/Mips/micromips-eva.mir b/llvm/test/CodeGen/Mips/micromips-eva.mir
index 3851a6961e03f..90bcefa431f6e 100644
--- a/llvm/test/CodeGen/Mips/micromips-eva.mir
+++ b/llvm/test/CodeGen/Mips/micromips-eva.mir
@@ -91,8 +91,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -157,8 +157,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: z.addr, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Mips/micromips-short-delay-slot.mir b/llvm/test/CodeGen/Mips/micromips-short-delay-slot.mir
index cadb440cfdab1..4d1ba43f06ead 100644
--- a/llvm/test/CodeGen/Mips/micromips-short-delay-slot.mir
+++ b/llvm/test/CodeGen/Mips/micromips-short-delay-slot.mir
@@ -43,8 +43,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lwp-swp.mir b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lwp-swp.mir
index 7ffdb409fd2b6..5a14ccbb96b31 100644
--- a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lwp-swp.mir
+++ b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-lwp-swp.mir
@@ -44,8 +44,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -112,8 +112,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -180,8 +180,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -248,8 +248,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-no-lwp-swp.mir b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-no-lwp-swp.mir
index 2b136a3ff499a..18eb341de3681 100644
--- a/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-no-lwp-swp.mir
+++ b/llvm/test/CodeGen/Mips/micromips-sizereduction/micromips-no-lwp-swp.mir
@@ -42,8 +42,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -101,8 +101,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -160,8 +160,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -219,8 +219,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir
index 453a0b1b5e27b..3b1fc43e5dc72 100644
--- a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir
+++ b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir
@@ -136,8 +136,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-o32.mir b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-o32.mir
index 8a20b5354f7c4..972899b13810b 100644
--- a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-o32.mir
+++ b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-o32.mir
@@ -63,8 +63,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic.mir b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic.mir
index 5151cb1c89029..7c6b856414563 100644
--- a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic.mir
+++ b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic.mir
@@ -63,8 +63,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/mirparser/target-flags-static-tls.mir b/llvm/test/CodeGen/Mips/mirparser/target-flags-static-tls.mir
index 3cfa28aa7d4fe..b0ea8ea2bb8cc 100644
--- a/llvm/test/CodeGen/Mips/mirparser/target-flags-static-tls.mir
+++ b/llvm/test/CodeGen/Mips/mirparser/target-flags-static-tls.mir
@@ -128,8 +128,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/msa/emergency-spill.mir b/llvm/test/CodeGen/Mips/msa/emergency-spill.mir
index dcc64f58c5646..0111758018413 100644
--- a/llvm/test/CodeGen/Mips/msa/emergency-spill.mir
+++ b/llvm/test/CodeGen/Mips/msa/emergency-spill.mir
@@ -97,8 +97,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: retval, type: default, offset: 0, size: 16, alignment: 16,
diff --git a/llvm/test/CodeGen/Mips/sll-micromips-r6-encoding.mir b/llvm/test/CodeGen/Mips/sll-micromips-r6-encoding.mir
index b87ee412038ca..02c0b3c573d6b 100644
--- a/llvm/test/CodeGen/Mips/sll-micromips-r6-encoding.mir
+++ b/llvm/test/CodeGen/Mips/sll-micromips-r6-encoding.mir
@@ -33,8 +33,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/Mips/unaligned-memops-mapping.mir b/llvm/test/CodeGen/Mips/unaligned-memops-mapping.mir
index df5ef982a19f7..b198ced3fb225 100644
--- a/llvm/test/CodeGen/Mips/unaligned-memops-mapping.mir
+++ b/llvm/test/CodeGen/Mips/unaligned-memops-mapping.mir
@@ -45,8 +45,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -91,8 +91,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/NVPTX/aggregate-return.ll b/llvm/test/CodeGen/NVPTX/aggregate-return.ll
index abc873e2aa706..fab60bdb3f2d1 100644
--- a/llvm/test/CodeGen/NVPTX/aggregate-return.ll
+++ b/llvm/test/CodeGen/NVPTX/aggregate-return.ll
@@ -10,19 +10,20 @@ declare {float, float} @bars({float, float} %input)
 define void @test_v2f32(<2 x float> %input, ptr %output) {
 ; CHECK-LABEL: test_v2f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_v2f32_param_0];
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_v2f32_param_0];
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .align 8 .b8 param0[8];
 ; CHECK-NEXT:    .param .align 8 .b8 retval0[8];
-; CHECK-NEXT:    st.param.b64 [param0], %rd1;
+; CHECK-NEXT:    st.param.v2.b32 [param0], {%r1, %r2};
 ; CHECK-NEXT:    call.uni (retval0), barv, (param0);
-; CHECK-NEXT:    ld.param.b64 %rd2, [retval0];
+; CHECK-NEXT:    ld.param.v2.b32 {%r3, %r4}, [retval0];
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    ld.param.b64 %rd4, [test_v2f32_param_1];
-; CHECK-NEXT:    st.b64 [%rd4], %rd2;
+; CHECK-NEXT:    ld.param.b64 %rd1, [test_v2f32_param_1];
+; CHECK-NEXT:    st.v2.b32 [%rd1], {%r3, %r4};
 ; CHECK-NEXT:    ret;
   %call = tail call <2 x float> @barv(<2 x float> %input)
   store <2 x float> %call, ptr %output, align 8
@@ -32,24 +33,28 @@ define void @test_v2f32(<2 x float> %input, ptr %output) {
 define void @test_v3f32(<3 x float> %input, ptr %output) {
 ; CHECK-LABEL: test_v3f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_v3f32_param_0];
-; CHECK-NEXT:    ld.param.b32 %r1, [test_v3f32_param_0+8];
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_v3f32_param_0];
+; CHECK-NEXT:    ld.param.b32 %r3, [test_v3f32_param_0+8];
 ; CHECK-NEXT:    { // callseq 1, 0
 ; CHECK-NEXT:    .param .align 16 .b8 param0[16];
 ; CHECK-NEXT:    .param .align 16 .b8 retval0[16];
-; CHECK-NEXT:    st.param.b32 [param0+8], %r1;
-; CHECK-NEXT:    st.param.b64 [param0], %rd1;
+; CHECK-NEXT:    st.param.b32 [param0+8], %r3;
+; CHECK-NEXT:    st.param.v2.b32 [param0], {%r1, %r2};
 ; CHECK-NEXT:    call.uni (retval0), barv3, (param0);
-; CHECK-NEXT:    ld.param.b32 %r2, [retval0+8];
-; CHECK-NEXT:    ld.param.b64 %rd2, [retval0];
+; CHECK-NEXT:    ld.param.b32 %r4, [retval0+8];
+; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [retval0];
 ; CHECK-NEXT:    } // callseq 1
-; CHECK-NEXT:    ld.param.b64 %rd4, [test_v3f32_param_1];
-; CHECK-NEXT:    st.b32 [%rd4+8], %r2;
-; CHECK-NEXT:    st.b64 [%rd4], %rd2;
+; CHECK-NEXT:    cvt.u64.u32 %rd1, %r5;
+; CHECK-NEXT:    cvt.u64.u32 %rd2, %r6;
+; CHECK-NEXT:    shl.b64 %rd3, %rd2, 32;
+; CHECK-NEXT:    or.b64 %rd4, %rd1, %rd3;
+; CHECK-NEXT:    ld.param.b64 %rd5, [test_v3f32_param_1];
+; CHECK-NEXT:    st.b32 [%rd5+8], %r4;
+; CHECK-NEXT:    st.b64 [%rd5], %rd4;
 ; CHECK-NEXT:    ret;
   %call = tail call <3 x float> @barv3(<3 x float> %input)
 ; Make sure we don't load more values than than we need to.
@@ -60,7 +65,7 @@ define void @test_v3f32(<3 x float> %input, ptr %output) {
 define void @test_a2f32([2 x float] %input, ptr %output) {
 ; CHECK-LABEL: test_a2f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
@@ -87,7 +92,7 @@ define void @test_a2f32([2 x float] %input, ptr %output) {
 define void @test_s2f32({float, float} %input, ptr %output) {
 ; CHECK-LABEL: test_s2f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
diff --git a/llvm/test/CodeGen/NVPTX/atomics-sm70.ll b/llvm/test/CodeGen/NVPTX/atomics-sm70.ll
index f710d7f883a1b..5f4856acb317c 100644
--- a/llvm/test/CodeGen/NVPTX/atomics-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/atomics-sm70.ll
@@ -47,90 +47,90 @@ define void @test(ptr %dp0, ptr addrspace(1) %dp1, ptr addrspace(3) %dp3, half %
 ; CHECKPTX62:       {
 ; CHECKPTX62-NEXT:    .reg .pred %p<5>;
 ; CHECKPTX62-NEXT:    .reg .b16 %rs<11>;
-; CHECKPTX62-NEXT:    .reg .b32 %r<58>;
+; CHECKPTX62-NEXT:    .reg .b32 %r<50>;
 ; CHECKPTX62-EMPTY:
 ; CHECKPTX62-NEXT:  // %bb.0:
 ; CHECKPTX62-NEXT:    ld.param.b16 %rs1, [test_param_3];
-; CHECKPTX62-NEXT:    ld.param.b32 %r23, [test_param_2];
-; CHECKPTX62-NEXT:    ld.param.b32 %r22, [test_param_1];
-; CHECKPTX62-NEXT:    ld.param.b32 %r24, [test_param_0];
-; CHECKPTX62-NEXT:    and.b32 %r1, %r24, -4;
-; CHECKPTX62-NEXT:    and.b32 %r25, %r24, 3;
-; CHECKPTX62-NEXT:    shl.b32 %r2, %r25, 3;
-; CHECKPTX62-NEXT:    mov.b32 %r26, 65535;
-; CHECKPTX62-NEXT:    shl.b32 %r27, %r26, %r2;
-; CHECKPTX62-NEXT:    not.b32 %r3, %r27;
-; CHECKPTX62-NEXT:    ld.b32 %r54, [%r1];
+; CHECKPTX62-NEXT:    ld.param.b32 %r15, [test_param_2];
+; CHECKPTX62-NEXT:    ld.param.b32 %r14, [test_param_1];
+; CHECKPTX62-NEXT:    ld.param.b32 %r16, [test_param_0];
+; CHECKPTX62-NEXT:    and.b32 %r1, %r16, -4;
+; CHECKPTX62-NEXT:    and.b32 %r17, %r16, 3;
+; CHECKPTX62-NEXT:    shl.b32 %r2, %r17, 3;
+; CHECKPTX62-NEXT:    mov.b32 %r18, 65535;
+; CHECKPTX62-NEXT:    shl.b32 %r19, %r18, %r2;
+; CHECKPTX62-NEXT:    not.b32 %r3, %r19;
+; CHECKPTX62-NEXT:    ld.b32 %r46, [%r1];
 ; CHECKPTX62-NEXT:  $L__BB0_1: // %atomicrmw.start45
 ; CHECKPTX62-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKPTX62-NEXT:    shr.u32 %r28, %r54, %r2;
-; CHECKPTX62-NEXT:    cvt.u16.u32 %rs2, %r28;
+; CHECKPTX62-NEXT:    shr.u32 %r20, %r46, %r2;
+; CHECKPTX62-NEXT:    cvt.u16.u32 %rs2, %r20;
 ; CHECKPTX62-NEXT:    add.rn.f16 %rs3, %rs2, %rs1;
-; CHECKPTX62-NEXT:    cvt.u32.u16 %r29, %rs3;
-; CHECKPTX62-NEXT:    shl.b32 %r30, %r29, %r2;
-; CHECKPTX62-NEXT:    and.b32 %r31, %r54, %r3;
-; CHECKPTX62-NEXT:    or.b32 %r32, %r31, %r30;
-; CHECKPTX62-NEXT:    atom.relaxed.sys.cas.b32 %r6, [%r1], %r54, %r32;
-; CHECKPTX62-NEXT:    setp.ne.b32 %p1, %r6, %r54;
-; CHECKPTX62-NEXT:    mov.b32 %r54, %r6;
+; CHECKPTX62-NEXT:    cvt.u32.u16 %r21, %rs3;
+; CHECKPTX62-NEXT:    shl.b32 %r22, %r21, %r2;
+; CHECKPTX62-NEXT:    and.b32 %r23, %r46, %r3;
+; CHECKPTX62-NEXT:    or.b32 %r24, %r23, %r22;
+; CHECKPTX62-NEXT:    atom.relaxed.sys.cas.b32 %r4, [%r1], %r46, %r24;
+; CHECKPTX62-NEXT:    setp.ne.b32 %p1, %r4, %r46;
+; CHECKPTX62-NEXT:    mov.b32 %r46, %r4;
 ; CHECKPTX62-NEXT:    @%p1 bra $L__BB0_1;
 ; CHECKPTX62-NEXT:  // %bb.2: // %atomicrmw.end44
-; CHECKPTX62-NEXT:    ld.b32 %r55, [%r1];
+; CHECKPTX62-NEXT:    ld.b32 %r47, [%r1];
 ; CHECKPTX62-NEXT:  $L__BB0_3: // %atomicrmw.start27
 ; CHECKPTX62-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKPTX62-NEXT:    shr.u32 %r33, %r55, %r2;
-; CHECKPTX62-NEXT:    cvt.u16.u32 %rs4, %r33;
+; CHECKPTX62-NEXT:    shr.u32 %r25, %r47, %r2;
+; CHECKPTX62-NEXT:    cvt.u16.u32 %rs4, %r25;
 ; CHECKPTX62-NEXT:    mov.b16 %rs5, 0x3C00;
 ; CHECKPTX62-NEXT:    add.rn.f16 %rs6, %rs4, %rs5;
-; CHECKPTX62-NEXT:    cvt.u32.u16 %r34, %rs6;
-; CHECKPTX62-NEXT:    shl.b32 %r35, %r34, %r2;
-; CHECKPTX62-NEXT:    and.b32 %r36, %r55, %r3;
-; CHECKPTX62-NEXT:    or.b32 %r37, %r36, %r35;
-; CHECKPTX62-NEXT:    atom.relaxed.sys.cas.b32 %r9, [%r1], %r55, %r37;
-; CHECKPTX62-NEXT:    setp.ne.b32 %p2, %r9, %r55;
-; CHECKPTX62-NEXT:    mov.b32 %r55, %r9;
+; CHECKPTX62-NEXT:    cvt.u32.u16 %r26, %rs6;
+; CHECKPTX62-NEXT:    shl.b32 %r27, %r26, %r2;
+; CHECKPTX62-NEXT:    and.b32 %r28, %r47, %r3;
+; CHECKPTX62-NEXT:    or.b32 %r29, %r28, %r27;
+; CHECKPTX62-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%r1], %r47, %r29;
+; CHECKPTX62-NEXT:    setp.ne.b32 %p2, %r5, %r47;
+; CHECKPTX62-NEXT:    mov.b32 %r47, %r5;
 ; CHECKPTX62-NEXT:    @%p2 bra $L__BB0_3;
 ; CHECKPTX62-NEXT:  // %bb.4: // %atomicrmw.end26
-; CHECKPTX62-NEXT:    and.b32 %r10, %r22, -4;
-; CHECKPTX62-NEXT:    shl.b32 %r38, %r22, 3;
-; CHECKPTX62-NEXT:    and.b32 %r11, %r38, 24;
-; CHECKPTX62-NEXT:    mov.b32 %r39, 65535;
-; CHECKPTX62-NEXT:    shl.b32 %r40, %r39, %r11;
-; CHECKPTX62-NEXT:    not.b32 %r12, %r40;
-; CHECKPTX62-NEXT:    ld.global.b32 %r56, [%r10];
+; CHECKPTX62-NEXT:    and.b32 %r6, %r14, -4;
+; CHECKPTX62-NEXT:    shl.b32 %r30, %r14, 3;
+; CHECKPTX62-NEXT:    and.b32 %r7, %r30, 24;
+; CHECKPTX62-NEXT:    mov.b32 %r31, 65535;
+; CHECKPTX62-NEXT:    shl.b32 %r32, %r31, %r7;
+; CHECKPTX62-NEXT:    not.b32 %r8, %r32;
+; CHECKPTX62-NEXT:    ld.global.b32 %r48, [%r6];
 ; CHECKPTX62-NEXT:  $L__BB0_5: // %atomicrmw.start9
 ; CHECKPTX62-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKPTX62-NEXT:    shr.u32 %r41, %r56, %r11;
-; CHECKPTX62-NEXT:    cvt.u16.u32 %rs7, %r41;
+; CHECKPTX62-NEXT:    shr.u32 %r33, %r48, %r7;
+; CHECKPTX62-NEXT:    cvt.u16.u32 %rs7, %r33;
 ; CHECKPTX62-NEXT:    add.rn.f16 %rs8, %rs7, %rs1;
-; CHECKPTX62-NEXT:    cvt.u32.u16 %r42, %rs8;
-; CHECKPTX62-NEXT:    shl.b32 %r43, %r42, %r11;
-; CHECKPTX62-NEXT:    and.b32 %r44, %r56, %r12;
-; CHECKPTX62-NEXT:    or.b32 %r45, %r44, %r43;
-; CHECKPTX62-NEXT:    atom.relaxed.sys.global.cas.b32 %r15, [%r10], %r56, %r45;
-; CHECKPTX62-NEXT:    setp.ne.b32 %p3, %r15, %r56;
-; CHECKPTX62-NEXT:    mov.b32 %r56, %r15;
+; CHECKPTX62-NEXT:    cvt.u32.u16 %r34, %rs8;
+; CHECKPTX62-NEXT:    shl.b32 %r35, %r34, %r7;
+; CHECKPTX62-NEXT:    and.b32 %r36, %r48, %r8;
+; CHECKPTX62-NEXT:    or.b32 %r37, %r36, %r35;
+; CHECKPTX62-NEXT:    atom.relaxed.sys.global.cas.b32 %r9, [%r6], %r48, %r37;
+; CHECKPTX62-NEXT:    setp.ne.b32 %p3, %r9, %r48;
+; CHECKPTX62-NEXT:    mov.b32 %r48, %r9;
 ; CHECKPTX62-NEXT:    @%p3 bra $L__BB0_5;
 ; CHECKPTX62-NEXT:  // %bb.6: // %atomicrmw.end8
-; CHECKPTX62-NEXT:    and.b32 %r16, %r23, -4;
-; CHECKPTX62-NEXT:    shl.b32 %r46, %r23, 3;
-; CHECKPTX62-NEXT:    and.b32 %r17, %r46, 24;
-; CHECKPTX62-NEXT:    mov.b32 %r47, 65535;
-; CHECKPTX62-NEXT:    shl.b32 %r48, %r47, %r17;
-; CHECKPTX62-NEXT:    not.b32 %r18, %r48;
-; CHECKPTX62-NEXT:    ld.shared.b32 %r57, [%r16];
+; CHECKPTX62-NEXT:    and.b32 %r10, %r15, -4;
+; CHECKPTX62-NEXT:    shl.b32 %r38, %r15, 3;
+; CHECKPTX62-NEXT:    and.b32 %r11, %r38, 24;
+; CHECKPTX62-NEXT:    mov.b32 %r39, 65535;
+; CHECKPTX62-NEXT:    shl.b32 %r40, %r39, %r11;
+; CHECKPTX62-NEXT:    not.b32 %r12, %r40;
+; CHECKPTX62-NEXT:    ld.shared.b32 %r49, [%r10];
 ; CHECKPTX62-NEXT:  $L__BB0_7: // %atomicrmw.start
 ; CHECKPTX62-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKPTX62-NEXT:    shr.u32 %r49, %r57, %r17;
-; CHECKPTX62-NEXT:    cvt.u16.u32 %rs9, %r49;
+; CHECKPTX62-NEXT:    shr.u32 %r41, %r49, %r11;
+; CHECKPTX62-NEXT:    cvt.u16.u32 %rs9, %r41;
 ; CHECKPTX62-NEXT:    add.rn.f16 %rs10, %rs9, %rs1;
-; CHECKPTX62-NEXT:    cvt.u32.u16 %r50, %rs10;
-; CHECKPTX62-NEXT:    shl.b32 %r51, %r50, %r17;
-; CHECKPTX62-NEXT:    and.b32 %r52, %r57, %r18;
-; CHECKPTX62-NEXT:    or.b32 %r53, %r52, %r51;
-; CHECKPTX62-NEXT:    atom.relaxed.sys.shared.cas.b32 %r21, [%r16], %r57, %r53;
-; CHECKPTX62-NEXT:    setp.ne.b32 %p4, %r21, %r57;
-; CHECKPTX62-NEXT:    mov.b32 %r57, %r21;
+; CHECKPTX62-NEXT:    cvt.u32.u16 %r42, %rs10;
+; CHECKPTX62-NEXT:    shl.b32 %r43, %r42, %r11;
+; CHECKPTX62-NEXT:    and.b32 %r44, %r49, %r12;
+; CHECKPTX62-NEXT:    or.b32 %r45, %r44, %r43;
+; CHECKPTX62-NEXT:    atom.relaxed.sys.shared.cas.b32 %r13, [%r10], %r49, %r45;
+; CHECKPTX62-NEXT:    setp.ne.b32 %p4, %r13, %r49;
+; CHECKPTX62-NEXT:    mov.b32 %r49, %r13;
 ; CHECKPTX62-NEXT:    @%p4 bra $L__BB0_7;
 ; CHECKPTX62-NEXT:  // %bb.8: // %atomicrmw.end
 ; CHECKPTX62-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/atomics-sm90.ll b/llvm/test/CodeGen/NVPTX/atomics-sm90.ll
index f96fd30019025..e560d4386c20d 100644
--- a/llvm/test/CodeGen/NVPTX/atomics-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/atomics-sm90.ll
@@ -47,93 +47,93 @@ define void @test(ptr %dp0, ptr addrspace(1) %dp1, ptr addrspace(3) %dp3, bfloat
 ; CHECKPTX71:       {
 ; CHECKPTX71-NEXT:    .reg .pred %p<5>;
 ; CHECKPTX71-NEXT:    .reg .b16 %rs<14>;
-; CHECKPTX71-NEXT:    .reg .b32 %r<58>;
+; CHECKPTX71-NEXT:    .reg .b32 %r<50>;
 ; CHECKPTX71-EMPTY:
 ; CHECKPTX71-NEXT:  // %bb.0:
 ; CHECKPTX71-NEXT:    ld.param.b16 %rs1, [test_param_3];
-; CHECKPTX71-NEXT:    ld.param.b32 %r23, [test_param_2];
-; CHECKPTX71-NEXT:    ld.param.b32 %r22, [test_param_1];
-; CHECKPTX71-NEXT:    ld.param.b32 %r24, [test_param_0];
-; CHECKPTX71-NEXT:    and.b32 %r1, %r24, -4;
-; CHECKPTX71-NEXT:    and.b32 %r25, %r24, 3;
-; CHECKPTX71-NEXT:    shl.b32 %r2, %r25, 3;
-; CHECKPTX71-NEXT:    mov.b32 %r26, 65535;
-; CHECKPTX71-NEXT:    shl.b32 %r27, %r26, %r2;
-; CHECKPTX71-NEXT:    not.b32 %r3, %r27;
-; CHECKPTX71-NEXT:    ld.b32 %r54, [%r1];
+; CHECKPTX71-NEXT:    ld.param.b32 %r15, [test_param_2];
+; CHECKPTX71-NEXT:    ld.param.b32 %r14, [test_param_1];
+; CHECKPTX71-NEXT:    ld.param.b32 %r16, [test_param_0];
+; CHECKPTX71-NEXT:    and.b32 %r1, %r16, -4;
+; CHECKPTX71-NEXT:    and.b32 %r17, %r16, 3;
+; CHECKPTX71-NEXT:    shl.b32 %r2, %r17, 3;
+; CHECKPTX71-NEXT:    mov.b32 %r18, 65535;
+; CHECKPTX71-NEXT:    shl.b32 %r19, %r18, %r2;
+; CHECKPTX71-NEXT:    not.b32 %r3, %r19;
+; CHECKPTX71-NEXT:    ld.b32 %r46, [%r1];
 ; CHECKPTX71-NEXT:  $L__BB0_1: // %atomicrmw.start45
 ; CHECKPTX71-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKPTX71-NEXT:    shr.u32 %r28, %r54, %r2;
-; CHECKPTX71-NEXT:    cvt.u16.u32 %rs2, %r28;
+; CHECKPTX71-NEXT:    shr.u32 %r20, %r46, %r2;
+; CHECKPTX71-NEXT:    cvt.u16.u32 %rs2, %r20;
 ; CHECKPTX71-NEXT:    mov.b16 %rs3, 0x3F80;
 ; CHECKPTX71-NEXT:    fma.rn.bf16 %rs4, %rs2, %rs3, %rs1;
-; CHECKPTX71-NEXT:    cvt.u32.u16 %r29, %rs4;
-; CHECKPTX71-NEXT:    shl.b32 %r30, %r29, %r2;
-; CHECKPTX71-NEXT:    and.b32 %r31, %r54, %r3;
-; CHECKPTX71-NEXT:    or.b32 %r32, %r31, %r30;
-; CHECKPTX71-NEXT:    atom.relaxed.sys.cas.b32 %r6, [%r1], %r54, %r32;
-; CHECKPTX71-NEXT:    setp.ne.b32 %p1, %r6, %r54;
-; CHECKPTX71-NEXT:    mov.b32 %r54, %r6;
+; CHECKPTX71-NEXT:    cvt.u32.u16 %r21, %rs4;
+; CHECKPTX71-NEXT:    shl.b32 %r22, %r21, %r2;
+; CHECKPTX71-NEXT:    and.b32 %r23, %r46, %r3;
+; CHECKPTX71-NEXT:    or.b32 %r24, %r23, %r22;
+; CHECKPTX71-NEXT:    atom.relaxed.sys.cas.b32 %r4, [%r1], %r46, %r24;
+; CHECKPTX71-NEXT:    setp.ne.b32 %p1, %r4, %r46;
+; CHECKPTX71-NEXT:    mov.b32 %r46, %r4;
 ; CHECKPTX71-NEXT:    @%p1 bra $L__BB0_1;
 ; CHECKPTX71-NEXT:  // %bb.2: // %atomicrmw.end44
-; CHECKPTX71-NEXT:    ld.b32 %r55, [%r1];
+; CHECKPTX71-NEXT:    ld.b32 %r47, [%r1];
 ; CHECKPTX71-NEXT:  $L__BB0_3: // %atomicrmw.start27
 ; CHECKPTX71-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKPTX71-NEXT:    shr.u32 %r33, %r55, %r2;
-; CHECKPTX71-NEXT:    cvt.u16.u32 %rs5, %r33;
+; CHECKPTX71-NEXT:    shr.u32 %r25, %r47, %r2;
+; CHECKPTX71-NEXT:    cvt.u16.u32 %rs5, %r25;
 ; CHECKPTX71-NEXT:    mov.b16 %rs6, 0x3F80;
 ; CHECKPTX71-NEXT:    fma.rn.bf16 %rs7, %rs5, %rs6, %rs6;
-; CHECKPTX71-NEXT:    cvt.u32.u16 %r34, %rs7;
-; CHECKPTX71-NEXT:    shl.b32 %r35, %r34, %r2;
-; CHECKPTX71-NEXT:    and.b32 %r36, %r55, %r3;
-; CHECKPTX71-NEXT:    or.b32 %r37, %r36, %r35;
-; CHECKPTX71-NEXT:    atom.relaxed.sys.cas.b32 %r9, [%r1], %r55, %r37;
-; CHECKPTX71-NEXT:    setp.ne.b32 %p2, %r9, %r55;
-; CHECKPTX71-NEXT:    mov.b32 %r55, %r9;
+; CHECKPTX71-NEXT:    cvt.u32.u16 %r26, %rs7;
+; CHECKPTX71-NEXT:    shl.b32 %r27, %r26, %r2;
+; CHECKPTX71-NEXT:    and.b32 %r28, %r47, %r3;
+; CHECKPTX71-NEXT:    or.b32 %r29, %r28, %r27;
+; CHECKPTX71-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%r1], %r47, %r29;
+; CHECKPTX71-NEXT:    setp.ne.b32 %p2, %r5, %r47;
+; CHECKPTX71-NEXT:    mov.b32 %r47, %r5;
 ; CHECKPTX71-NEXT:    @%p2 bra $L__BB0_3;
 ; CHECKPTX71-NEXT:  // %bb.4: // %atomicrmw.end26
-; CHECKPTX71-NEXT:    and.b32 %r10, %r22, -4;
-; CHECKPTX71-NEXT:    shl.b32 %r38, %r22, 3;
-; CHECKPTX71-NEXT:    and.b32 %r11, %r38, 24;
-; CHECKPTX71-NEXT:    mov.b32 %r39, 65535;
-; CHECKPTX71-NEXT:    shl.b32 %r40, %r39, %r11;
-; CHECKPTX71-NEXT:    not.b32 %r12, %r40;
-; CHECKPTX71-NEXT:    ld.global.b32 %r56, [%r10];
+; CHECKPTX71-NEXT:    and.b32 %r6, %r14, -4;
+; CHECKPTX71-NEXT:    shl.b32 %r30, %r14, 3;
+; CHECKPTX71-NEXT:    and.b32 %r7, %r30, 24;
+; CHECKPTX71-NEXT:    mov.b32 %r31, 65535;
+; CHECKPTX71-NEXT:    shl.b32 %r32, %r31, %r7;
+; CHECKPTX71-NEXT:    not.b32 %r8, %r32;
+; CHECKPTX71-NEXT:    ld.global.b32 %r48, [%r6];
 ; CHECKPTX71-NEXT:  $L__BB0_5: // %atomicrmw.start9
 ; CHECKPTX71-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKPTX71-NEXT:    shr.u32 %r41, %r56, %r11;
-; CHECKPTX71-NEXT:    cvt.u16.u32 %rs8, %r41;
+; CHECKPTX71-NEXT:    shr.u32 %r33, %r48, %r7;
+; CHECKPTX71-NEXT:    cvt.u16.u32 %rs8, %r33;
 ; CHECKPTX71-NEXT:    mov.b16 %rs9, 0x3F80;
 ; CHECKPTX71-NEXT:    fma.rn.bf16 %rs10, %rs8, %rs9, %rs1;
-; CHECKPTX71-NEXT:    cvt.u32.u16 %r42, %rs10;
-; CHECKPTX71-NEXT:    shl.b32 %r43, %r42, %r11;
-; CHECKPTX71-NEXT:    and.b32 %r44, %r56, %r12;
-; CHECKPTX71-NEXT:    or.b32 %r45, %r44, %r43;
-; CHECKPTX71-NEXT:    atom.relaxed.sys.global.cas.b32 %r15, [%r10], %r56, %r45;
-; CHECKPTX71-NEXT:    setp.ne.b32 %p3, %r15, %r56;
-; CHECKPTX71-NEXT:    mov.b32 %r56, %r15;
+; CHECKPTX71-NEXT:    cvt.u32.u16 %r34, %rs10;
+; CHECKPTX71-NEXT:    shl.b32 %r35, %r34, %r7;
+; CHECKPTX71-NEXT:    and.b32 %r36, %r48, %r8;
+; CHECKPTX71-NEXT:    or.b32 %r37, %r36, %r35;
+; CHECKPTX71-NEXT:    atom.relaxed.sys.global.cas.b32 %r9, [%r6], %r48, %r37;
+; CHECKPTX71-NEXT:    setp.ne.b32 %p3, %r9, %r48;
+; CHECKPTX71-NEXT:    mov.b32 %r48, %r9;
 ; CHECKPTX71-NEXT:    @%p3 bra $L__BB0_5;
 ; CHECKPTX71-NEXT:  // %bb.6: // %atomicrmw.end8
-; CHECKPTX71-NEXT:    and.b32 %r16, %r23, -4;
-; CHECKPTX71-NEXT:    shl.b32 %r46, %r23, 3;
-; CHECKPTX71-NEXT:    and.b32 %r17, %r46, 24;
-; CHECKPTX71-NEXT:    mov.b32 %r47, 65535;
-; CHECKPTX71-NEXT:    shl.b32 %r48, %r47, %r17;
-; CHECKPTX71-NEXT:    not.b32 %r18, %r48;
-; CHECKPTX71-NEXT:    ld.shared.b32 %r57, [%r16];
+; CHECKPTX71-NEXT:    and.b32 %r10, %r15, -4;
+; CHECKPTX71-NEXT:    shl.b32 %r38, %r15, 3;
+; CHECKPTX71-NEXT:    and.b32 %r11, %r38, 24;
+; CHECKPTX71-NEXT:    mov.b32 %r39, 65535;
+; CHECKPTX71-NEXT:    shl.b32 %r40, %r39, %r11;
+; CHECKPTX71-NEXT:    not.b32 %r12, %r40;
+; CHECKPTX71-NEXT:    ld.shared.b32 %r49, [%r10];
 ; CHECKPTX71-NEXT:  $L__BB0_7: // %atomicrmw.start
 ; CHECKPTX71-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECKPTX71-NEXT:    shr.u32 %r49, %r57, %r17;
-; CHECKPTX71-NEXT:    cvt.u16.u32 %rs11, %r49;
+; CHECKPTX71-NEXT:    shr.u32 %r41, %r49, %r11;
+; CHECKPTX71-NEXT:    cvt.u16.u32 %rs11, %r41;
 ; CHECKPTX71-NEXT:    mov.b16 %rs12, 0x3F80;
 ; CHECKPTX71-NEXT:    fma.rn.bf16 %rs13, %rs11, %rs12, %rs1;
-; CHECKPTX71-NEXT:    cvt.u32.u16 %r50, %rs13;
-; CHECKPTX71-NEXT:    shl.b32 %r51, %r50, %r17;
-; CHECKPTX71-NEXT:    and.b32 %r52, %r57, %r18;
-; CHECKPTX71-NEXT:    or.b32 %r53, %r52, %r51;
-; CHECKPTX71-NEXT:    atom.relaxed.sys.shared.cas.b32 %r21, [%r16], %r57, %r53;
-; CHECKPTX71-NEXT:    setp.ne.b32 %p4, %r21, %r57;
-; CHECKPTX71-NEXT:    mov.b32 %r57, %r21;
+; CHECKPTX71-NEXT:    cvt.u32.u16 %r42, %rs13;
+; CHECKPTX71-NEXT:    shl.b32 %r43, %r42, %r11;
+; CHECKPTX71-NEXT:    and.b32 %r44, %r49, %r12;
+; CHECKPTX71-NEXT:    or.b32 %r45, %r44, %r43;
+; CHECKPTX71-NEXT:    atom.relaxed.sys.shared.cas.b32 %r13, [%r10], %r49, %r45;
+; CHECKPTX71-NEXT:    setp.ne.b32 %p4, %r13, %r49;
+; CHECKPTX71-NEXT:    mov.b32 %r49, %r13;
 ; CHECKPTX71-NEXT:    @%p4 bra $L__BB0_7;
 ; CHECKPTX71-NEXT:  // %bb.8: // %atomicrmw.end
 ; CHECKPTX71-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/atomics.ll b/llvm/test/CodeGen/NVPTX/atomics.ll
index 04a58cf22cfc5..6ea02f35e9626 100644
--- a/llvm/test/CodeGen/NVPTX/atomics.ll
+++ b/llvm/test/CodeGen/NVPTX/atomics.ll
@@ -425,40 +425,40 @@ define half @atomicrmw_add_f16_generic(ptr %addr, half %val) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
 ; CHECK-NEXT:    .reg .b16 %rs<4>;
-; CHECK-NEXT:    .reg .b32 %r<20>;
+; CHECK-NEXT:    .reg .b32 %r<18>;
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b16 %rs1, [atomicrmw_add_f16_generic_param_1];
 ; CHECK-NEXT:    ld.param.b64 %rd2, [atomicrmw_add_f16_generic_param_0];
 ; CHECK-NEXT:    and.b64 %rd1, %rd2, -4;
-; CHECK-NEXT:    cvt.u32.u64 %r6, %rd2;
-; CHECK-NEXT:    and.b32 %r7, %r6, 3;
-; CHECK-NEXT:    shl.b32 %r1, %r7, 3;
-; CHECK-NEXT:    mov.b32 %r8, 65535;
-; CHECK-NEXT:    shl.b32 %r9, %r8, %r1;
-; CHECK-NEXT:    not.b32 %r2, %r9;
-; CHECK-NEXT:    ld.b32 %r19, [%rd1];
-; CHECK-NEXT:    cvt.f32.f16 %r12, %rs1;
+; CHECK-NEXT:    cvt.u32.u64 %r4, %rd2;
+; CHECK-NEXT:    and.b32 %r5, %r4, 3;
+; CHECK-NEXT:    shl.b32 %r1, %r5, 3;
+; CHECK-NEXT:    mov.b32 %r6, 65535;
+; CHECK-NEXT:    shl.b32 %r7, %r6, %r1;
+; CHECK-NEXT:    not.b32 %r2, %r7;
+; CHECK-NEXT:    ld.b32 %r17, [%rd1];
+; CHECK-NEXT:    cvt.f32.f16 %r10, %rs1;
 ; CHECK-NEXT:  $L__BB24_1: // %atomicrmw.start
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u32 %r10, %r19, %r1;
-; CHECK-NEXT:    cvt.u16.u32 %rs2, %r10;
-; CHECK-NEXT:    cvt.f32.f16 %r11, %rs2;
-; CHECK-NEXT:    add.rn.f32 %r13, %r11, %r12;
-; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r13;
-; CHECK-NEXT:    cvt.u32.u16 %r14, %rs3;
-; CHECK-NEXT:    shl.b32 %r15, %r14, %r1;
-; CHECK-NEXT:    and.b32 %r16, %r19, %r2;
-; CHECK-NEXT:    or.b32 %r17, %r16, %r15;
+; CHECK-NEXT:    shr.u32 %r8, %r17, %r1;
+; CHECK-NEXT:    cvt.u16.u32 %rs2, %r8;
+; CHECK-NEXT:    cvt.f32.f16 %r9, %rs2;
+; CHECK-NEXT:    add.rn.f32 %r11, %r9, %r10;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r11;
+; CHECK-NEXT:    cvt.u32.u16 %r12, %rs3;
+; CHECK-NEXT:    shl.b32 %r13, %r12, %r1;
+; CHECK-NEXT:    and.b32 %r14, %r17, %r2;
+; CHECK-NEXT:    or.b32 %r15, %r14, %r13;
 ; CHECK-NEXT:    membar.sys;
-; CHECK-NEXT:    atom.cas.b32 %r5, [%rd1], %r19, %r17;
-; CHECK-NEXT:    setp.ne.b32 %p1, %r5, %r19;
-; CHECK-NEXT:    mov.b32 %r19, %r5;
+; CHECK-NEXT:    atom.cas.b32 %r3, [%rd1], %r17, %r15;
+; CHECK-NEXT:    setp.ne.b32 %p1, %r3, %r17;
+; CHECK-NEXT:    mov.b32 %r17, %r3;
 ; CHECK-NEXT:    @%p1 bra $L__BB24_1;
 ; CHECK-NEXT:  // %bb.2: // %atomicrmw.end
-; CHECK-NEXT:    shr.u32 %r18, %r5, %r1;
-; CHECK-NEXT:    st.param.b16 [func_retval0], %r18;
+; CHECK-NEXT:    shr.u32 %r16, %r3, %r1;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %r16;
 ; CHECK-NEXT:    ret;
   %ret = atomicrmw fadd ptr %addr, half %val seq_cst
   ret half %ret
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index aee58a044a986..a386e4292777b 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -688,25 +688,25 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b64 %rd1, [test_extload_bf16x8_param_0];
 ; SM70-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
-; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM70-NEXT:    cvt.u32.u16 %r5, %rs2;
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs8;
 ; SM70-NEXT:    shl.b32 %r6, %r5, 16;
-; SM70-NEXT:    cvt.u32.u16 %r7, %rs1;
+; SM70-NEXT:    cvt.u32.u16 %r7, %rs7;
 ; SM70-NEXT:    shl.b32 %r8, %r7, 16;
-; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM70-NEXT:    cvt.u32.u16 %r9, %rs4;
+; SM70-NEXT:    cvt.u32.u16 %r9, %rs6;
 ; SM70-NEXT:    shl.b32 %r10, %r9, 16;
-; SM70-NEXT:    cvt.u32.u16 %r11, %rs3;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs5;
 ; SM70-NEXT:    shl.b32 %r12, %r11, 16;
-; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
-; SM70-NEXT:    cvt.u32.u16 %r13, %rs6;
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs4;
 ; SM70-NEXT:    shl.b32 %r14, %r13, 16;
-; SM70-NEXT:    cvt.u32.u16 %r15, %rs5;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
 ; SM70-NEXT:    shl.b32 %r16, %r15, 16;
-; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r3;
-; SM70-NEXT:    cvt.u32.u16 %r17, %rs8;
+; SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
 ; SM70-NEXT:    shl.b32 %r18, %r17, 16;
-; SM70-NEXT:    cvt.u32.u16 %r19, %rs7;
+; SM70-NEXT:    cvt.u32.u16 %r19, %rs1;
 ; SM70-NEXT:    shl.b32 %r20, %r19, 16;
 ; SM70-NEXT:    st.param.v4.b32 [func_retval0+16], {%r20, %r18, %r16, %r14};
 ; SM70-NEXT:    st.param.v4.b32 [func_retval0], {%r12, %r10, %r8, %r6};
@@ -721,18 +721,18 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM80-NEXT:  // %bb.0:
 ; SM80-NEXT:    ld.param.b64 %rd1, [test_extload_bf16x8_param_0];
 ; SM80-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
-; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-NEXT:    cvt.f32.bf16 %r5, %rs2;
-; SM80-NEXT:    cvt.f32.bf16 %r6, %rs1;
-; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-NEXT:    cvt.f32.bf16 %r7, %rs4;
-; SM80-NEXT:    cvt.f32.bf16 %r8, %rs3;
-; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
-; SM80-NEXT:    cvt.f32.bf16 %r9, %rs6;
-; SM80-NEXT:    cvt.f32.bf16 %r10, %rs5;
-; SM80-NEXT:    mov.b32 {%rs7, %rs8}, %r3;
-; SM80-NEXT:    cvt.f32.bf16 %r11, %rs8;
-; SM80-NEXT:    cvt.f32.bf16 %r12, %rs7;
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; SM80-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %r5, %rs8;
+; SM80-NEXT:    cvt.f32.bf16 %r6, %rs7;
+; SM80-NEXT:    cvt.f32.bf16 %r7, %rs6;
+; SM80-NEXT:    cvt.f32.bf16 %r8, %rs5;
+; SM80-NEXT:    cvt.f32.bf16 %r9, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %r10, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %r11, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %r12, %rs1;
 ; SM80-NEXT:    st.param.v4.b32 [func_retval0+16], {%r12, %r11, %r10, %r9};
 ; SM80-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; SM80-NEXT:    ret;
@@ -746,18 +746,18 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM80-FTZ-NEXT:  // %bb.0:
 ; SM80-FTZ-NEXT:    ld.param.b64 %rd1, [test_extload_bf16x8_param_0];
 ; SM80-FTZ-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
-; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r5, %rs2;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r6, %rs1;
-; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r7, %rs4;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r8, %rs3;
-; SM80-FTZ-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r9, %rs6;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r10, %rs5;
-; SM80-FTZ-NEXT:    mov.b32 {%rs7, %rs8}, %r3;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r11, %rs8;
-; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r12, %rs7;
+; SM80-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; SM80-FTZ-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; SM80-FTZ-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; SM80-FTZ-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r5, %rs8;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r6, %rs7;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r7, %rs6;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r8, %rs5;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r9, %rs4;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r10, %rs3;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r11, %rs2;
+; SM80-FTZ-NEXT:    cvt.ftz.f32.bf16 %r12, %rs1;
 ; SM80-FTZ-NEXT:    st.param.v4.b32 [func_retval0+16], {%r12, %r11, %r10, %r9};
 ; SM80-FTZ-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; SM80-FTZ-NEXT:    ret;
@@ -771,18 +771,18 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b64 %rd1, [test_extload_bf16x8_param_0];
 ; SM90-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
-; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; SM90-NEXT:    cvt.f32.bf16 %r5, %rs2;
-; SM90-NEXT:    cvt.f32.bf16 %r6, %rs1;
-; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; SM90-NEXT:    cvt.f32.bf16 %r7, %rs4;
-; SM90-NEXT:    cvt.f32.bf16 %r8, %rs3;
-; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
-; SM90-NEXT:    cvt.f32.bf16 %r9, %rs6;
-; SM90-NEXT:    cvt.f32.bf16 %r10, %rs5;
-; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r3;
-; SM90-NEXT:    cvt.f32.bf16 %r11, %rs8;
-; SM90-NEXT:    cvt.f32.bf16 %r12, %rs7;
+; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; SM90-NEXT:    cvt.f32.bf16 %r5, %rs8;
+; SM90-NEXT:    cvt.f32.bf16 %r6, %rs7;
+; SM90-NEXT:    cvt.f32.bf16 %r7, %rs6;
+; SM90-NEXT:    cvt.f32.bf16 %r8, %rs5;
+; SM90-NEXT:    cvt.f32.bf16 %r9, %rs4;
+; SM90-NEXT:    cvt.f32.bf16 %r10, %rs3;
+; SM90-NEXT:    cvt.f32.bf16 %r11, %rs2;
+; SM90-NEXT:    cvt.f32.bf16 %r12, %rs1;
 ; SM90-NEXT:    st.param.v4.b32 [func_retval0+16], {%r12, %r11, %r10, %r9};
 ; SM90-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; SM90-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
index 80627a03354a0..e1d4ef1073a78 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions-approx.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx71 --enable-unsafe-fp-math | FileCheck --check-prefixes=CHECK %s
-; RUN: %if ptxas-11.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx71 --enable-unsafe-fp-math | %ptxas-verify -arch=sm_80 %}
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK %s
+; RUN: %if ptxas-11.8 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | %ptxas-verify -arch=sm_80 %}
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
@@ -22,7 +22,7 @@ define <2 x bfloat> @test_sin(<2 x bfloat> %a) #0 #1 {
 ; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
-  %r = call <2 x bfloat> @llvm.sin.f16(<2 x bfloat> %a)
+  %r = call afn <2 x bfloat> @llvm.sin.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
 }
 
@@ -41,7 +41,7 @@ define <2 x bfloat> @test_cos(<2 x bfloat> %a) #0 #1 {
 ; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r5, %r4, %r2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
-  %r = call <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a)
+  %r = call afn <2 x bfloat> @llvm.cos.f16(<2 x bfloat> %a)
   ret <2 x bfloat> %r
 }
 
diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
index b4641d01eb927..bd4c7775354ae 100644
--- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll
@@ -204,7 +204,7 @@ declare <2 x bfloat> @test_callee(<2 x bfloat> %a, <2 x bfloat> %b) #0
 define <2 x bfloat> @test_call(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
 ; CHECK-LABEL: test_call(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_call_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/blocksareclusters-kernel-attr.ll b/llvm/test/CodeGen/NVPTX/blocksareclusters-kernel-attr.ll
new file mode 100644
index 0000000000000..a0a99fe55654f
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/blocksareclusters-kernel-attr.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_90 -mattr=+ptx90 | FileCheck %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+; Test "blocksareclusters" attribute with full "reqntid" and "cluster_dim"
+; attributes.
+define ptx_kernel void @kernel1(ptr %input, ptr %output) #0 #1 #2 {
+; CHECK-LABEL: kernel1(
+; CHECK:       .reqntid 1024, 1, 1
+; CHECK-NEXT:  .reqnctapercluster 2, 2, 2
+; CHECK-NEXT:  .blocksareclusters
+; CHECK-NEXT:  {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ret;
+  ret void
+}
+
+; Test "blocksareclusters" attribute with single dimension "reqntid" and
+; "cluster_dim" attributes.
+define ptx_kernel void @kernel2(ptr %input, ptr %output) #0 #3 #4 {
+; CHECK-LABEL: kernel2(
+; CHECK:       .reqntid 1024
+; CHECK-NEXT:  .reqnctapercluster 2
+; CHECK-NEXT:  .blocksareclusters // @kernel2
+; CHECK-NEXT:  {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ret;
+  ret void
+}
+
+; Test "blocksareclusters" attribute with two dimensions(not z dimension)
+; "reqntid" and "cluster_dim" attributes.
+define ptx_kernel void @kernel3(ptr %input, ptr %output) #0 #5 #6 {
+; CHECK-LABEL: kernel3(
+; CHECK:       .reqntid 512, 2
+; CHECK-NEXT:  .reqnctapercluster 2, 2
+; CHECK-NEXT:  .blocksareclusters // @kernel3
+; CHECK-NEXT:  {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ret;
+  ret void
+}
+
+attributes #0 = { "nvvm.blocksareclusters" }
+
+attributes #1 = { "nvvm.reqntid"="1024,1,1" }
+attributes #2 = { "nvvm.cluster_dim"="2,2,2" }
+
+attributes #3 = { "nvvm.reqntid"="1024" }
+attributes #4 = { "nvvm.cluster_dim"="2" }
+
+attributes #5 = { "nvvm.reqntid"="512,2" }
+attributes #6 = { "nvvm.cluster_dim"="2,2" }
diff --git a/llvm/test/CodeGen/NVPTX/branch-fold.mir b/llvm/test/CodeGen/NVPTX/branch-fold.mir
index ca6f49feff052..c9abe3faf228e 100644
--- a/llvm/test/CodeGen/NVPTX/branch-fold.mir
+++ b/llvm/test/CodeGen/NVPTX/branch-fold.mir
@@ -57,7 +57,7 @@ body:             |
   ; CHECK-NEXT: bb.2.bb1:
   ; CHECK-NEXT:   successors: %bb.2(0x7c000000), %bb.3(0x04000000)
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   [[ADDi64ri:%[0-9]+]]:b64 = ADDi64ri [[ADDi64ri]], 1
+  ; CHECK-NEXT:   [[ADDi64ri:%[0-9]+]]:b64 = ADD64ri [[ADDi64ri]], 1
   ; CHECK-NEXT:   [[SETP_s64ri:%[0-9]+]]:b1 = SETP_i64ri [[ADDi64ri]], 1, 2
   ; CHECK-NEXT:   CBranch [[SETP_s64ri]], %bb.2
   ; CHECK-NEXT: {{  $}}
@@ -76,7 +76,7 @@ body:             |
   bb.2.bb1:
     successors: %bb.2(0x7c000000), %bb.3(0x04000000)
 
-    %5:b64 = ADDi64ri %5, 1
+    %5:b64 = ADD64ri %5, 1
     %4:b1 = SETP_i64ri %5, 1, 2
     CBranch %4, %bb.2
 
diff --git a/llvm/test/CodeGen/NVPTX/byval-arg-vectorize.ll b/llvm/test/CodeGen/NVPTX/byval-arg-vectorize.ll
index 9988d5b122cc1..579f02a9539c6 100644
--- a/llvm/test/CodeGen/NVPTX/byval-arg-vectorize.ll
+++ b/llvm/test/CodeGen/NVPTX/byval-arg-vectorize.ll
@@ -11,7 +11,7 @@ declare %struct.double2 @add(ptr align(16) byval(%struct.double2), ptr align(16)
 define void @call_byval(ptr %out, ptr %in1, ptr %in2) {
 ; CHECK-LABEL: call_byval(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<12>;
+; CHECK-NEXT:    .reg .b64 %rd<10>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [call_byval_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
index 63c389c36e87e..6e480996e7e6a 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm60.ll
@@ -7,41 +7,41 @@ define i8 @monotonic_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_global_cta_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_global_cta_param_1];
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    ld.param.b8 %r7, [monotonic_monotonic_i8_global_cta_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB0_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB0_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB0_1;
 ; SM60-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") monotonic monotonic
     ret i8 %new
@@ -52,42 +52,42 @@ define i8 @monotonic_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_global_cta_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_global_cta_param_1];
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    ld.param.b8 %r7, [monotonic_acquire_i8_global_cta_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB1_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB1_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB1_1;
 ; SM60-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") monotonic acquire
     ret i8 %new
@@ -98,43 +98,43 @@ define i8 @monotonic_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [monotonic_seq_cst_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB2_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB2_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB2_1;
 ; SM60-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") monotonic seq_cst
     ret i8 %new
@@ -145,42 +145,42 @@ define i8 @acquire_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_global_cta_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_global_cta_param_1];
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    ld.param.b8 %r7, [acquire_monotonic_i8_global_cta_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB3_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB3_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB3_1;
 ; SM60-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acquire monotonic
     ret i8 %new
@@ -191,42 +191,42 @@ define i8 @acquire_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_global_cta_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_global_cta_param_1];
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    ld.param.b8 %r7, [acquire_acquire_i8_global_cta_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB4_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB4_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB4_1;
 ; SM60-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acquire acquire
     ret i8 %new
@@ -237,43 +237,43 @@ define i8 @acquire_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [acquire_seq_cst_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB5_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB5_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB5_1;
 ; SM60-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acquire seq_cst
     ret i8 %new
@@ -284,42 +284,42 @@ define i8 @release_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [release_monotonic_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB6_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB6_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB6_1;
 ; SM60-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") release monotonic
     ret i8 %new
@@ -330,43 +330,43 @@ define i8 @release_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [release_acquire_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [release_acquire_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB7_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB7_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB7_1;
 ; SM60-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") release acquire
     ret i8 %new
@@ -377,43 +377,43 @@ define i8 @release_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [release_seq_cst_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB8_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB8_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB8_1;
 ; SM60-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") release seq_cst
     ret i8 %new
@@ -424,43 +424,43 @@ define i8 @acq_rel_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [acq_rel_monotonic_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB9_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB9_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB9_1;
 ; SM60-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel monotonic
     ret i8 %new
@@ -471,43 +471,43 @@ define i8 @acq_rel_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB10_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB10_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB10_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB10_1;
 ; SM60-NEXT:  $L__BB10_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel acquire
     ret i8 %new
@@ -518,43 +518,43 @@ define i8 @acq_rel_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [acq_rel_seq_cst_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB11_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB11_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB11_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB11_1;
 ; SM60-NEXT:  $L__BB11_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel seq_cst
     ret i8 %new
@@ -565,43 +565,43 @@ define i8 @seq_cst_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [seq_cst_monotonic_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB12_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB12_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB12_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB12_1;
 ; SM60-NEXT:  $L__BB12_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") seq_cst monotonic
     ret i8 %new
@@ -612,43 +612,43 @@ define i8 @seq_cst_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [seq_cst_acquire_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB13_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB13_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB13_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB13_1;
 ; SM60-NEXT:  $L__BB13_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") seq_cst acquire
     ret i8 %new
@@ -659,43 +659,43 @@ define i8 @seq_cst_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_global_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [seq_cst_seq_cst_i8_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB14_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB14_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB14_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB14_1;
 ; SM60-NEXT:  $L__BB14_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") seq_cst seq_cst
     ret i8 %new
@@ -706,40 +706,40 @@ define i16 @monotonic_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp,
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [monotonic_monotonic_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i16_global_cta_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    ld.param.b16 %r9, [monotonic_monotonic_i16_global_cta_param_1];
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    ld.param.b16 %r7, [monotonic_monotonic_i16_global_cta_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB15_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB15_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB15_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB15_1;
 ; SM60-NEXT:  $L__BB15_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") monotonic monotonic
     ret i16 %new
@@ -750,41 +750,41 @@ define i16 @monotonic_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [monotonic_acquire_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i16_global_cta_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    ld.param.b16 %r9, [monotonic_acquire_i16_global_cta_param_1];
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    ld.param.b16 %r7, [monotonic_acquire_i16_global_cta_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB16_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB16_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB16_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB16_1;
 ; SM60-NEXT:  $L__BB16_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") monotonic acquire
     ret i16 %new
@@ -795,42 +795,42 @@ define i16 @monotonic_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [monotonic_seq_cst_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [monotonic_seq_cst_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [monotonic_seq_cst_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB17_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB17_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB17_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB17_1;
 ; SM60-NEXT:  $L__BB17_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") monotonic seq_cst
     ret i16 %new
@@ -841,41 +841,41 @@ define i16 @acquire_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [acquire_monotonic_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i16_global_cta_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    ld.param.b16 %r9, [acquire_monotonic_i16_global_cta_param_1];
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    ld.param.b16 %r7, [acquire_monotonic_i16_global_cta_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB18_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB18_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB18_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB18_1;
 ; SM60-NEXT:  $L__BB18_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acquire monotonic
     ret i16 %new
@@ -886,41 +886,41 @@ define i16 @acquire_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [acquire_acquire_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i16_global_cta_param_0];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    ld.param.b16 %r9, [acquire_acquire_i16_global_cta_param_1];
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    ld.param.b16 %r7, [acquire_acquire_i16_global_cta_param_1];
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB19_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB19_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB19_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB19_1;
 ; SM60-NEXT:  $L__BB19_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acquire acquire
     ret i16 %new
@@ -931,42 +931,42 @@ define i16 @acquire_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [acquire_seq_cst_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [acquire_seq_cst_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [acquire_seq_cst_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB20_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB20_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB20_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB20_1;
 ; SM60-NEXT:  $L__BB20_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acquire seq_cst
     ret i16 %new
@@ -977,41 +977,41 @@ define i16 @release_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [release_monotonic_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_monotonic_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [release_monotonic_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [release_monotonic_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB21_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB21_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB21_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB21_1;
 ; SM60-NEXT:  $L__BB21_3: // %partword.cmpxchg.end
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") release monotonic
     ret i16 %new
@@ -1022,42 +1022,42 @@ define i16 @release_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [release_acquire_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_acquire_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [release_acquire_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [release_acquire_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB22_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB22_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB22_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB22_1;
 ; SM60-NEXT:  $L__BB22_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") release acquire
     ret i16 %new
@@ -1068,42 +1068,42 @@ define i16 @release_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [release_seq_cst_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [release_seq_cst_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [release_seq_cst_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB23_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB23_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB23_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB23_1;
 ; SM60-NEXT:  $L__BB23_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") release seq_cst
     ret i16 %new
@@ -1114,42 +1114,42 @@ define i16 @acq_rel_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [acq_rel_monotonic_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [acq_rel_monotonic_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [acq_rel_monotonic_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB24_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB24_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB24_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB24_1;
 ; SM60-NEXT:  $L__BB24_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acq_rel monotonic
     ret i16 %new
@@ -1160,42 +1160,42 @@ define i16 @acq_rel_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [acq_rel_acquire_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [acq_rel_acquire_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [acq_rel_acquire_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB25_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB25_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB25_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB25_1;
 ; SM60-NEXT:  $L__BB25_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acq_rel acquire
     ret i16 %new
@@ -1206,42 +1206,42 @@ define i16 @acq_rel_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [acq_rel_seq_cst_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [acq_rel_seq_cst_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [acq_rel_seq_cst_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB26_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB26_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB26_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB26_1;
 ; SM60-NEXT:  $L__BB26_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acq_rel seq_cst
     ret i16 %new
@@ -1252,42 +1252,42 @@ define i16 @seq_cst_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [seq_cst_monotonic_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [seq_cst_monotonic_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [seq_cst_monotonic_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB27_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB27_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB27_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB27_1;
 ; SM60-NEXT:  $L__BB27_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") seq_cst monotonic
     ret i16 %new
@@ -1298,42 +1298,42 @@ define i16 @seq_cst_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [seq_cst_acquire_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [seq_cst_acquire_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [seq_cst_acquire_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB28_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB28_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB28_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB28_1;
 ; SM60-NEXT:  $L__BB28_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") seq_cst acquire
     ret i16 %new
@@ -1344,42 +1344,42 @@ define i16 @seq_cst_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<20>;
+; SM60-NEXT:    .reg .b32 %r<17>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b16 %rs1, [seq_cst_seq_cst_i16_global_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i16_global_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b16 %r9, [seq_cst_seq_cst_i16_global_cta_param_1];
+; SM60-NEXT:    ld.param.b16 %r7, [seq_cst_seq_cst_i16_global_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 65535;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM60-NEXT:    and.b32 %r19, %r15, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 65535;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM60-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM60-NEXT:  $L__BB29_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r16, %r19, %r3;
-; SM60-NEXT:    or.b32 %r17, %r19, %r4;
-; SM60-NEXT:    atom.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM60-NEXT:    or.b32 %r14, %r16, %r3;
+; SM60-NEXT:    or.b32 %r15, %r16, %r4;
+; SM60-NEXT:    atom.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM60-NEXT:    @%p1 bra $L__BB29_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB29_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM60-NEXT:    mov.b32 %r19, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM60-NEXT:    mov.b32 %r16, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB29_1;
 ; SM60-NEXT:  $L__BB29_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") seq_cst seq_cst
     ret i16 %new
@@ -1899,43 +1899,43 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_param_0];
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_global_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB60_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.sys.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.sys.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB60_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB60_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB60_1;
 ; SM60-NEXT:  $L__BB60_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.sys;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1997,43 +1997,43 @@ define i8 @acq_rel_acquire_i8_generic_cta(ptr %addr, i8 %cmp, i8 %new) {
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_generic_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_generic_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_generic_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_generic_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB64_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB64_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB64_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB64_1;
 ; SM60-NEXT:  $L__BB64_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new syncscope("block") acq_rel acquire
     ret i8 %new
@@ -2044,43 +2044,43 @@ define i8 @acq_rel_acquire_i8_shared_cta(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM60:       {
 ; SM60-NEXT:    .reg .pred %p<3>;
 ; SM60-NEXT:    .reg .b16 %rs<2>;
-; SM60-NEXT:    .reg .b32 %r<21>;
+; SM60-NEXT:    .reg .b32 %r<18>;
 ; SM60-NEXT:    .reg .b64 %rd<3>;
 ; SM60-EMPTY:
 ; SM60-NEXT:  // %bb.0:
 ; SM60-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_shared_cta_param_2];
 ; SM60-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_shared_cta_param_0];
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_shared_cta_param_1];
+; SM60-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_shared_cta_param_1];
 ; SM60-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM60-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM60-NEXT:    and.b32 %r11, %r10, 3;
-; SM60-NEXT:    shl.b32 %r1, %r11, 3;
-; SM60-NEXT:    mov.b32 %r12, 255;
-; SM60-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM60-NEXT:    not.b32 %r2, %r13;
-; SM60-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM60-NEXT:    and.b32 %r15, %r14, 255;
-; SM60-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM60-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM60-NEXT:    ld.shared.b32 %r16, [%rd1];
-; SM60-NEXT:    and.b32 %r20, %r16, %r2;
+; SM60-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM60-NEXT:    and.b32 %r9, %r8, 3;
+; SM60-NEXT:    shl.b32 %r1, %r9, 3;
+; SM60-NEXT:    mov.b32 %r10, 255;
+; SM60-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM60-NEXT:    not.b32 %r2, %r11;
+; SM60-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM60-NEXT:    and.b32 %r13, %r12, 255;
+; SM60-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM60-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM60-NEXT:    ld.shared.b32 %r14, [%rd1];
+; SM60-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM60-NEXT:  $L__BB65_1: // %partword.cmpxchg.loop
 ; SM60-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM60-NEXT:    or.b32 %r17, %r20, %r3;
-; SM60-NEXT:    or.b32 %r18, %r20, %r4;
-; SM60-NEXT:    atom.cta.shared.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM60-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM60-NEXT:    or.b32 %r15, %r17, %r3;
+; SM60-NEXT:    or.b32 %r16, %r17, %r4;
+; SM60-NEXT:    atom.cta.shared.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM60-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM60-NEXT:    @%p1 bra $L__BB65_3;
 ; SM60-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM60-NEXT:    // in Loop: Header=BB65_1 Depth=1
-; SM60-NEXT:    and.b32 %r8, %r7, %r2;
-; SM60-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM60-NEXT:    mov.b32 %r20, %r8;
+; SM60-NEXT:    and.b32 %r6, %r5, %r2;
+; SM60-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM60-NEXT:    mov.b32 %r17, %r6;
 ; SM60-NEXT:    @%p2 bra $L__BB65_1;
 ; SM60-NEXT:  $L__BB65_3: // %partword.cmpxchg.end
 ; SM60-NEXT:    membar.cta;
-; SM60-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM60-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM60-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel acquire
     ret i8 %new
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll
index 5cb344d5ded84..065b89c7ebf74 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm70.ll
@@ -7,41 +7,41 @@ define i8 @monotonic_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_global_cta_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_global_cta_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    ld.param.b8 %r7, [monotonic_monotonic_i8_global_cta_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB0_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB0_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB0_1;
 ; SM70-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") monotonic monotonic
     ret i8 %new
@@ -52,42 +52,42 @@ define i8 @monotonic_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_global_cta_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_global_cta_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    ld.param.b8 %r7, [monotonic_acquire_i8_global_cta_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB1_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB1_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB1_1;
 ; SM70-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") monotonic acquire
     ret i8 %new
@@ -98,43 +98,43 @@ define i8 @monotonic_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [monotonic_seq_cst_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB2_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB2_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB2_1;
 ; SM70-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") monotonic seq_cst
     ret i8 %new
@@ -145,42 +145,42 @@ define i8 @acquire_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_global_cta_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_global_cta_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    ld.param.b8 %r7, [acquire_monotonic_i8_global_cta_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB3_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB3_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB3_1;
 ; SM70-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acquire monotonic
     ret i8 %new
@@ -191,42 +191,42 @@ define i8 @acquire_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_global_cta_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_global_cta_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    ld.param.b8 %r7, [acquire_acquire_i8_global_cta_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB4_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB4_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB4_1;
 ; SM70-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acquire acquire
     ret i8 %new
@@ -237,43 +237,43 @@ define i8 @acquire_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [acquire_seq_cst_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB5_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB5_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB5_1;
 ; SM70-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acquire seq_cst
     ret i8 %new
@@ -284,42 +284,42 @@ define i8 @release_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [release_monotonic_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB6_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB6_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB6_1;
 ; SM70-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") release monotonic
     ret i8 %new
@@ -330,43 +330,43 @@ define i8 @release_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [release_acquire_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [release_acquire_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB7_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB7_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB7_1;
 ; SM70-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") release acquire
     ret i8 %new
@@ -377,43 +377,43 @@ define i8 @release_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [release_seq_cst_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB8_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB8_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB8_1;
 ; SM70-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") release seq_cst
     ret i8 %new
@@ -424,43 +424,43 @@ define i8 @acq_rel_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [acq_rel_monotonic_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB9_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB9_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB9_1;
 ; SM70-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel monotonic
     ret i8 %new
@@ -471,43 +471,43 @@ define i8 @acq_rel_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB10_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB10_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB10_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB10_1;
 ; SM70-NEXT:  $L__BB10_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel acquire
     ret i8 %new
@@ -518,43 +518,43 @@ define i8 @acq_rel_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [acq_rel_seq_cst_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB11_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB11_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB11_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB11_1;
 ; SM70-NEXT:  $L__BB11_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel seq_cst
     ret i8 %new
@@ -565,43 +565,43 @@ define i8 @seq_cst_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [seq_cst_monotonic_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB12_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB12_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB12_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB12_1;
 ; SM70-NEXT:  $L__BB12_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") seq_cst monotonic
     ret i8 %new
@@ -612,43 +612,43 @@ define i8 @seq_cst_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [seq_cst_acquire_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB13_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB13_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB13_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB13_1;
 ; SM70-NEXT:  $L__BB13_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") seq_cst acquire
     ret i8 %new
@@ -659,43 +659,43 @@ define i8 @seq_cst_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_global_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [seq_cst_seq_cst_i8_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB14_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB14_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB14_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB14_1;
 ; SM70-NEXT:  $L__BB14_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") seq_cst seq_cst
     ret i8 %new
@@ -706,40 +706,40 @@ define i16 @monotonic_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp,
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [monotonic_monotonic_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i16_global_cta_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b16 %r9, [monotonic_monotonic_i16_global_cta_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    ld.param.b16 %r7, [monotonic_monotonic_i16_global_cta_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB15_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB15_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB15_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB15_1;
 ; SM70-NEXT:  $L__BB15_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") monotonic monotonic
     ret i16 %new
@@ -750,41 +750,41 @@ define i16 @monotonic_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [monotonic_acquire_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i16_global_cta_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b16 %r9, [monotonic_acquire_i16_global_cta_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    ld.param.b16 %r7, [monotonic_acquire_i16_global_cta_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB16_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB16_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB16_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB16_1;
 ; SM70-NEXT:  $L__BB16_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") monotonic acquire
     ret i16 %new
@@ -795,42 +795,42 @@ define i16 @monotonic_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [monotonic_seq_cst_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [monotonic_seq_cst_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [monotonic_seq_cst_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB17_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB17_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB17_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB17_1;
 ; SM70-NEXT:  $L__BB17_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") monotonic seq_cst
     ret i16 %new
@@ -841,41 +841,41 @@ define i16 @acquire_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [acquire_monotonic_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i16_global_cta_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b16 %r9, [acquire_monotonic_i16_global_cta_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    ld.param.b16 %r7, [acquire_monotonic_i16_global_cta_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB18_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB18_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB18_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB18_1;
 ; SM70-NEXT:  $L__BB18_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acquire monotonic
     ret i16 %new
@@ -886,41 +886,41 @@ define i16 @acquire_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [acquire_acquire_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i16_global_cta_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b16 %r9, [acquire_acquire_i16_global_cta_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    ld.param.b16 %r7, [acquire_acquire_i16_global_cta_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB19_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB19_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB19_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB19_1;
 ; SM70-NEXT:  $L__BB19_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acquire acquire
     ret i16 %new
@@ -931,42 +931,42 @@ define i16 @acquire_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [acquire_seq_cst_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [acquire_seq_cst_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [acquire_seq_cst_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB20_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB20_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB20_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB20_1;
 ; SM70-NEXT:  $L__BB20_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acquire seq_cst
     ret i16 %new
@@ -977,41 +977,41 @@ define i16 @release_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [release_monotonic_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_monotonic_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [release_monotonic_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [release_monotonic_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB21_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB21_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB21_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB21_1;
 ; SM70-NEXT:  $L__BB21_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") release monotonic
     ret i16 %new
@@ -1022,42 +1022,42 @@ define i16 @release_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [release_acquire_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_acquire_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [release_acquire_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [release_acquire_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB22_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB22_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB22_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB22_1;
 ; SM70-NEXT:  $L__BB22_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") release acquire
     ret i16 %new
@@ -1068,42 +1068,42 @@ define i16 @release_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [release_seq_cst_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [release_seq_cst_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [release_seq_cst_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB23_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB23_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB23_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB23_1;
 ; SM70-NEXT:  $L__BB23_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") release seq_cst
     ret i16 %new
@@ -1114,42 +1114,42 @@ define i16 @acq_rel_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [acq_rel_monotonic_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [acq_rel_monotonic_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [acq_rel_monotonic_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB24_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB24_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB24_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB24_1;
 ; SM70-NEXT:  $L__BB24_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acq_rel monotonic
     ret i16 %new
@@ -1160,42 +1160,42 @@ define i16 @acq_rel_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [acq_rel_acquire_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [acq_rel_acquire_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [acq_rel_acquire_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB25_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB25_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB25_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB25_1;
 ; SM70-NEXT:  $L__BB25_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acq_rel acquire
     ret i16 %new
@@ -1206,42 +1206,42 @@ define i16 @acq_rel_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [acq_rel_seq_cst_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [acq_rel_seq_cst_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [acq_rel_seq_cst_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB26_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB26_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB26_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB26_1;
 ; SM70-NEXT:  $L__BB26_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acq_rel seq_cst
     ret i16 %new
@@ -1252,42 +1252,42 @@ define i16 @seq_cst_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [seq_cst_monotonic_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [seq_cst_monotonic_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [seq_cst_monotonic_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB27_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB27_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB27_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB27_1;
 ; SM70-NEXT:  $L__BB27_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") seq_cst monotonic
     ret i16 %new
@@ -1298,42 +1298,42 @@ define i16 @seq_cst_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [seq_cst_acquire_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [seq_cst_acquire_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [seq_cst_acquire_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB28_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB28_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB28_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB28_1;
 ; SM70-NEXT:  $L__BB28_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") seq_cst acquire
     ret i16 %new
@@ -1344,42 +1344,42 @@ define i16 @seq_cst_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [seq_cst_seq_cst_i16_global_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i16_global_cta_param_0];
 ; SM70-NEXT:    fence.sc.cta;
-; SM70-NEXT:    ld.param.b16 %r9, [seq_cst_seq_cst_i16_global_cta_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [seq_cst_seq_cst_i16_global_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB29_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB29_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB29_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB29_1;
 ; SM70-NEXT:  $L__BB29_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") seq_cst seq_cst
     ret i16 %new
@@ -1899,43 +1899,43 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_global_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB60_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.sys.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.sys.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB60_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB60_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB60_1;
 ; SM70-NEXT:  $L__BB60_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -1997,43 +1997,43 @@ define i8 @acq_rel_acquire_i8_generic_cta(ptr %addr, i8 %cmp, i8 %new) {
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_generic_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_generic_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_generic_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_generic_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB64_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB64_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB64_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB64_1;
 ; SM70-NEXT:  $L__BB64_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new syncscope("block") acq_rel acquire
     ret i8 %new
@@ -2044,43 +2044,43 @@ define i8 @acq_rel_acquire_i8_shared_cta(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_shared_cta_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_shared_cta_param_0];
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_shared_cta_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_shared_cta_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.shared.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.shared.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB65_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.cta.shared.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.cta.shared.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB65_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB65_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB65_1;
 ; SM70-NEXT:  $L__BB65_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.cta;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel acquire
     ret i8 %new
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll b/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll
index 7cb259023d6dd..e4433570bdd70 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg-sm90.ll
@@ -7,41 +7,41 @@ define i8 @monotonic_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_monotonic_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i8_global_cta_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    ld.param.b8 %r9, [monotonic_monotonic_i8_global_cta_param_1];
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    ld.param.b8 %r7, [monotonic_monotonic_i8_global_cta_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB0_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB0_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB0_1;
 ; SM90-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") monotonic monotonic
     ret i8 %new
@@ -52,42 +52,42 @@ define i8 @monotonic_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_acquire_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i8_global_cta_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    ld.param.b8 %r9, [monotonic_acquire_i8_global_cta_param_1];
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    ld.param.b8 %r7, [monotonic_acquire_i8_global_cta_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB1_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB1_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB1_1;
 ; SM90-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") monotonic acquire
     ret i8 %new
@@ -98,43 +98,43 @@ define i8 @monotonic_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [monotonic_seq_cst_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [monotonic_seq_cst_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [monotonic_seq_cst_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB2_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB2_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB2_1;
 ; SM90-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") monotonic seq_cst
     ret i8 %new
@@ -145,42 +145,42 @@ define i8 @acquire_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_monotonic_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i8_global_cta_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    ld.param.b8 %r9, [acquire_monotonic_i8_global_cta_param_1];
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    ld.param.b8 %r7, [acquire_monotonic_i8_global_cta_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB3_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB3_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB3_1;
 ; SM90-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acquire monotonic
     ret i8 %new
@@ -191,42 +191,42 @@ define i8 @acquire_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_acquire_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i8_global_cta_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    ld.param.b8 %r9, [acquire_acquire_i8_global_cta_param_1];
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    ld.param.b8 %r7, [acquire_acquire_i8_global_cta_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB4_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB4_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB4_1;
 ; SM90-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acquire acquire
     ret i8 %new
@@ -237,43 +237,43 @@ define i8 @acquire_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [acquire_seq_cst_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [acquire_seq_cst_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [acquire_seq_cst_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB5_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB5_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB5_1;
 ; SM90-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acquire seq_cst
     ret i8 %new
@@ -284,42 +284,42 @@ define i8 @release_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_monotonic_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_monotonic_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [release_monotonic_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [release_monotonic_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB6_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB6_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB6_1;
 ; SM90-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") release monotonic
     ret i8 %new
@@ -330,43 +330,43 @@ define i8 @release_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_acquire_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_acquire_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [release_acquire_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [release_acquire_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB7_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB7_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB7_1;
 ; SM90-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") release acquire
     ret i8 %new
@@ -377,43 +377,43 @@ define i8 @release_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [release_seq_cst_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [release_seq_cst_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [release_seq_cst_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB8_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB8_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB8_1;
 ; SM90-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") release seq_cst
     ret i8 %new
@@ -424,43 +424,43 @@ define i8 @acq_rel_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_monotonic_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_monotonic_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [acq_rel_monotonic_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB9_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB9_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB9_1;
 ; SM90-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel monotonic
     ret i8 %new
@@ -471,43 +471,43 @@ define i8 @acq_rel_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB10_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB10_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB10_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB10_1;
 ; SM90-NEXT:  $L__BB10_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel acquire
     ret i8 %new
@@ -518,43 +518,43 @@ define i8 @acq_rel_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_seq_cst_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_seq_cst_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [acq_rel_seq_cst_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB11_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB11_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB11_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB11_1;
 ; SM90-NEXT:  $L__BB11_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel seq_cst
     ret i8 %new
@@ -565,43 +565,43 @@ define i8 @seq_cst_monotonic_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_monotonic_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_monotonic_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [seq_cst_monotonic_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB12_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB12_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB12_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB12_1;
 ; SM90-NEXT:  $L__BB12_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") seq_cst monotonic
     ret i8 %new
@@ -612,43 +612,43 @@ define i8 @seq_cst_acquire_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_acquire_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_acquire_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [seq_cst_acquire_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB13_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB13_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB13_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB13_1;
 ; SM90-NEXT:  $L__BB13_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") seq_cst acquire
     ret i8 %new
@@ -659,43 +659,43 @@ define i8 @seq_cst_seq_cst_i8_global_cta(ptr addrspace(1) %addr, i8 %cmp, i8 %ne
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [seq_cst_seq_cst_i8_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i8_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [seq_cst_seq_cst_i8_global_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [seq_cst_seq_cst_i8_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB14_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB14_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB14_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB14_1;
 ; SM90-NEXT:  $L__BB14_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new syncscope("block") seq_cst seq_cst
     ret i8 %new
@@ -706,40 +706,40 @@ define i16 @monotonic_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp,
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [monotonic_monotonic_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_monotonic_i16_global_cta_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    ld.param.b16 %r9, [monotonic_monotonic_i16_global_cta_param_1];
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    ld.param.b16 %r7, [monotonic_monotonic_i16_global_cta_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB15_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB15_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB15_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB15_1;
 ; SM90-NEXT:  $L__BB15_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") monotonic monotonic
     ret i16 %new
@@ -750,41 +750,41 @@ define i16 @monotonic_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [monotonic_acquire_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_acquire_i16_global_cta_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    ld.param.b16 %r9, [monotonic_acquire_i16_global_cta_param_1];
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    ld.param.b16 %r7, [monotonic_acquire_i16_global_cta_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB16_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB16_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB16_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB16_1;
 ; SM90-NEXT:  $L__BB16_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") monotonic acquire
     ret i16 %new
@@ -795,42 +795,42 @@ define i16 @monotonic_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [monotonic_seq_cst_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [monotonic_seq_cst_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [monotonic_seq_cst_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [monotonic_seq_cst_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB17_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB17_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB17_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB17_1;
 ; SM90-NEXT:  $L__BB17_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") monotonic seq_cst
     ret i16 %new
@@ -841,41 +841,41 @@ define i16 @acquire_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [acquire_monotonic_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_monotonic_i16_global_cta_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    ld.param.b16 %r9, [acquire_monotonic_i16_global_cta_param_1];
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    ld.param.b16 %r7, [acquire_monotonic_i16_global_cta_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB18_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB18_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB18_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB18_1;
 ; SM90-NEXT:  $L__BB18_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acquire monotonic
     ret i16 %new
@@ -886,41 +886,41 @@ define i16 @acquire_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [acquire_acquire_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_acquire_i16_global_cta_param_0];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    ld.param.b16 %r9, [acquire_acquire_i16_global_cta_param_1];
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    ld.param.b16 %r7, [acquire_acquire_i16_global_cta_param_1];
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB19_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB19_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB19_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB19_1;
 ; SM90-NEXT:  $L__BB19_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acquire acquire
     ret i16 %new
@@ -931,42 +931,42 @@ define i16 @acquire_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [acquire_seq_cst_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acquire_seq_cst_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [acquire_seq_cst_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [acquire_seq_cst_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB20_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB20_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB20_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB20_1;
 ; SM90-NEXT:  $L__BB20_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acquire seq_cst
     ret i16 %new
@@ -977,41 +977,41 @@ define i16 @release_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [release_monotonic_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_monotonic_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [release_monotonic_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [release_monotonic_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB21_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB21_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB21_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB21_1;
 ; SM90-NEXT:  $L__BB21_3: // %partword.cmpxchg.end
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") release monotonic
     ret i16 %new
@@ -1022,42 +1022,42 @@ define i16 @release_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [release_acquire_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_acquire_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [release_acquire_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [release_acquire_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB22_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB22_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB22_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB22_1;
 ; SM90-NEXT:  $L__BB22_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") release acquire
     ret i16 %new
@@ -1068,42 +1068,42 @@ define i16 @release_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [release_seq_cst_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [release_seq_cst_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [release_seq_cst_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [release_seq_cst_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB23_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB23_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB23_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB23_1;
 ; SM90-NEXT:  $L__BB23_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") release seq_cst
     ret i16 %new
@@ -1114,42 +1114,42 @@ define i16 @acq_rel_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [acq_rel_monotonic_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_monotonic_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [acq_rel_monotonic_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [acq_rel_monotonic_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB24_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB24_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB24_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB24_1;
 ; SM90-NEXT:  $L__BB24_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acq_rel monotonic
     ret i16 %new
@@ -1160,42 +1160,42 @@ define i16 @acq_rel_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [acq_rel_acquire_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [acq_rel_acquire_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [acq_rel_acquire_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB25_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB25_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB25_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB25_1;
 ; SM90-NEXT:  $L__BB25_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acq_rel acquire
     ret i16 %new
@@ -1206,42 +1206,42 @@ define i16 @acq_rel_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [acq_rel_seq_cst_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_seq_cst_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [acq_rel_seq_cst_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [acq_rel_seq_cst_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB26_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB26_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB26_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB26_1;
 ; SM90-NEXT:  $L__BB26_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") acq_rel seq_cst
     ret i16 %new
@@ -1252,42 +1252,42 @@ define i16 @seq_cst_monotonic_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [seq_cst_monotonic_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_monotonic_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [seq_cst_monotonic_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [seq_cst_monotonic_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB27_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB27_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB27_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB27_1;
 ; SM90-NEXT:  $L__BB27_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") seq_cst monotonic
     ret i16 %new
@@ -1298,42 +1298,42 @@ define i16 @seq_cst_acquire_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [seq_cst_acquire_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_acquire_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [seq_cst_acquire_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [seq_cst_acquire_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB28_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB28_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB28_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB28_1;
 ; SM90-NEXT:  $L__BB28_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") seq_cst acquire
     ret i16 %new
@@ -1344,42 +1344,42 @@ define i16 @seq_cst_seq_cst_i16_global_cta(ptr addrspace(1) %addr, i16 %cmp, i16
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<20>;
+; SM90-NEXT:    .reg .b32 %r<17>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b16 %rs1, [seq_cst_seq_cst_i16_global_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [seq_cst_seq_cst_i16_global_cta_param_0];
 ; SM90-NEXT:    fence.sc.cta;
-; SM90-NEXT:    ld.param.b16 %r9, [seq_cst_seq_cst_i16_global_cta_param_1];
+; SM90-NEXT:    ld.param.b16 %r7, [seq_cst_seq_cst_i16_global_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 65535;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r15, [%rd1];
-; SM90-NEXT:    and.b32 %r19, %r15, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 65535;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r13, [%rd1];
+; SM90-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM90-NEXT:  $L__BB29_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r16, %r19, %r3;
-; SM90-NEXT:    or.b32 %r17, %r19, %r4;
-; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM90-NEXT:    or.b32 %r14, %r16, %r3;
+; SM90-NEXT:    or.b32 %r15, %r16, %r4;
+; SM90-NEXT:    atom.relaxed.cta.global.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM90-NEXT:    @%p1 bra $L__BB29_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB29_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM90-NEXT:    mov.b32 %r19, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM90-NEXT:    mov.b32 %r16, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB29_1;
 ; SM90-NEXT:  $L__BB29_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i16 %cmp, i16 %new syncscope("block") seq_cst seq_cst
     ret i16 %new
@@ -1899,43 +1899,43 @@ define i8 @acq_rel_acquire_i8_global(ptr addrspace(1) %addr, i8 %cmp, i8 %new) {
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_global_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_global_param_0];
 ; SM90-NEXT:    fence.release.sys;
-; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_global_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_global_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.global.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.global.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB60_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.sys.global.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.sys.global.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB60_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB60_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB60_1;
 ; SM90-NEXT:  $L__BB60_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.sys;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(1) %addr, i8 %cmp, i8 %new acq_rel acquire
     ret i8 %new
@@ -2014,43 +2014,43 @@ define i8 @acq_rel_acquire_i8_generic_cta(ptr %addr, i8 %cmp, i8 %new) {
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_generic_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_generic_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_generic_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_generic_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB65_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB65_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB65_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB65_1;
 ; SM90-NEXT:  $L__BB65_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr %addr, i8 %cmp, i8 %new syncscope("block") acq_rel acquire
     ret i8 %new
@@ -2061,43 +2061,43 @@ define i8 @acq_rel_acquire_i8_shared_cta(ptr addrspace(3) %addr, i8 %cmp, i8 %ne
 ; SM90:       {
 ; SM90-NEXT:    .reg .pred %p<3>;
 ; SM90-NEXT:    .reg .b16 %rs<2>;
-; SM90-NEXT:    .reg .b32 %r<21>;
+; SM90-NEXT:    .reg .b32 %r<18>;
 ; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b8 %rs1, [acq_rel_acquire_i8_shared_cta_param_2];
 ; SM90-NEXT:    ld.param.b64 %rd2, [acq_rel_acquire_i8_shared_cta_param_0];
 ; SM90-NEXT:    fence.release.cta;
-; SM90-NEXT:    ld.param.b8 %r9, [acq_rel_acquire_i8_shared_cta_param_1];
+; SM90-NEXT:    ld.param.b8 %r7, [acq_rel_acquire_i8_shared_cta_param_1];
 ; SM90-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM90-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM90-NEXT:    and.b32 %r11, %r10, 3;
-; SM90-NEXT:    shl.b32 %r1, %r11, 3;
-; SM90-NEXT:    mov.b32 %r12, 255;
-; SM90-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM90-NEXT:    not.b32 %r2, %r13;
-; SM90-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM90-NEXT:    and.b32 %r15, %r14, 255;
-; SM90-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM90-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM90-NEXT:    ld.shared.b32 %r16, [%rd1];
-; SM90-NEXT:    and.b32 %r20, %r16, %r2;
+; SM90-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM90-NEXT:    and.b32 %r9, %r8, 3;
+; SM90-NEXT:    shl.b32 %r1, %r9, 3;
+; SM90-NEXT:    mov.b32 %r10, 255;
+; SM90-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM90-NEXT:    not.b32 %r2, %r11;
+; SM90-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM90-NEXT:    and.b32 %r13, %r12, 255;
+; SM90-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM90-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM90-NEXT:    ld.shared.b32 %r14, [%rd1];
+; SM90-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM90-NEXT:  $L__BB66_1: // %partword.cmpxchg.loop
 ; SM90-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM90-NEXT:    or.b32 %r17, %r20, %r3;
-; SM90-NEXT:    or.b32 %r18, %r20, %r4;
-; SM90-NEXT:    atom.relaxed.cta.shared.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM90-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM90-NEXT:    or.b32 %r15, %r17, %r3;
+; SM90-NEXT:    or.b32 %r16, %r17, %r4;
+; SM90-NEXT:    atom.relaxed.cta.shared.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM90-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM90-NEXT:    @%p1 bra $L__BB66_3;
 ; SM90-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM90-NEXT:    // in Loop: Header=BB66_1 Depth=1
-; SM90-NEXT:    and.b32 %r8, %r7, %r2;
-; SM90-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM90-NEXT:    mov.b32 %r20, %r8;
+; SM90-NEXT:    and.b32 %r6, %r5, %r2;
+; SM90-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM90-NEXT:    mov.b32 %r17, %r6;
 ; SM90-NEXT:    @%p2 bra $L__BB66_1;
 ; SM90-NEXT:  $L__BB66_3: // %partword.cmpxchg.end
 ; SM90-NEXT:    fence.acquire.cta;
-; SM90-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM90-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM90-NEXT:    ret;
     %pairold = cmpxchg ptr addrspace(3) %addr, i8 %cmp, i8 %new syncscope("block") acq_rel acquire
     ret i8 %new
diff --git a/llvm/test/CodeGen/NVPTX/cmpxchg.ll b/llvm/test/CodeGen/NVPTX/cmpxchg.ll
index 237e42394ba2f..997df7a8ad8b8 100644
--- a/llvm/test/CodeGen/NVPTX/cmpxchg.ll
+++ b/llvm/test/CodeGen/NVPTX/cmpxchg.ll
@@ -14,82 +14,82 @@ define i8 @relaxed_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<21>;
+; SM30-NEXT:    .reg .b32 %r<18>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b8 %rs1, [relaxed_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [relaxed_sys_i8_param_0];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    ld.param.b8 %r9, [relaxed_sys_i8_param_1];
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 255;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    and.b32 %r15, %r14, 255;
-; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r16, [%rd1];
-; SM30-NEXT:    and.b32 %r20, %r16, %r2;
+; SM30-NEXT:    ld.param.b8 %r7, [relaxed_sys_i8_param_1];
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 255;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    and.b32 %r13, %r12, 255;
+; SM30-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r14, [%rd1];
+; SM30-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM30-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r17, %r20, %r3;
-; SM30-NEXT:    or.b32 %r18, %r20, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM30-NEXT:    or.b32 %r15, %r17, %r3;
+; SM30-NEXT:    or.b32 %r16, %r17, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM30-NEXT:    @%p1 bra $L__BB0_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB0_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM30-NEXT:    mov.b32 %r20, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM30-NEXT:    mov.b32 %r17, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB0_1;
 ; SM30-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: relaxed_sys_i8(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [relaxed_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [relaxed_sys_i8_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b8 %r9, [relaxed_sys_i8_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    ld.param.b8 %r7, [relaxed_sys_i8_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB0_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB0_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB0_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB0_1;
 ; SM70-NEXT:  $L__BB0_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: relaxed_sys_i8(
 ; SM90:       {
@@ -140,84 +140,84 @@ define i8 @acquire_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<21>;
+; SM30-NEXT:    .reg .b32 %r<18>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b8 %rs1, [acquire_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [acquire_sys_i8_param_0];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    ld.param.b8 %r9, [acquire_sys_i8_param_1];
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 255;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    and.b32 %r15, %r14, 255;
-; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r16, [%rd1];
-; SM30-NEXT:    and.b32 %r20, %r16, %r2;
+; SM30-NEXT:    ld.param.b8 %r7, [acquire_sys_i8_param_1];
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 255;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    and.b32 %r13, %r12, 255;
+; SM30-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r14, [%rd1];
+; SM30-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM30-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r17, %r20, %r3;
-; SM30-NEXT:    or.b32 %r18, %r20, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM30-NEXT:    or.b32 %r15, %r17, %r3;
+; SM30-NEXT:    or.b32 %r16, %r17, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM30-NEXT:    @%p1 bra $L__BB1_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB1_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM30-NEXT:    mov.b32 %r20, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM30-NEXT:    mov.b32 %r17, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB1_1;
 ; SM30-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: acquire_sys_i8(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acquire_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_sys_i8_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b8 %r9, [acquire_sys_i8_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    ld.param.b8 %r7, [acquire_sys_i8_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB1_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB1_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB1_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB1_1;
 ; SM70-NEXT:  $L__BB1_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: acquire_sys_i8(
 ; SM90:       {
@@ -269,84 +269,84 @@ define i8 @release_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<21>;
+; SM30-NEXT:    .reg .b32 %r<18>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b8 %rs1, [release_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [release_sys_i8_param_0];
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    ld.param.b8 %r9, [release_sys_i8_param_1];
+; SM30-NEXT:    ld.param.b8 %r7, [release_sys_i8_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 255;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    and.b32 %r15, %r14, 255;
-; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r16, [%rd1];
-; SM30-NEXT:    and.b32 %r20, %r16, %r2;
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 255;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    and.b32 %r13, %r12, 255;
+; SM30-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r14, [%rd1];
+; SM30-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM30-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r17, %r20, %r3;
-; SM30-NEXT:    or.b32 %r18, %r20, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM30-NEXT:    or.b32 %r15, %r17, %r3;
+; SM30-NEXT:    or.b32 %r16, %r17, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM30-NEXT:    @%p1 bra $L__BB2_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB2_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM30-NEXT:    mov.b32 %r20, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM30-NEXT:    mov.b32 %r17, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB2_1;
 ; SM30-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: release_sys_i8(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [release_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_sys_i8_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    ld.param.b8 %r9, [release_sys_i8_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [release_sys_i8_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB2_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB2_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB2_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB2_1;
 ; SM70-NEXT:  $L__BB2_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: release_sys_i8(
 ; SM90:       {
@@ -398,86 +398,86 @@ define i8 @acq_rel_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<21>;
+; SM30-NEXT:    .reg .b32 %r<18>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b8 %rs1, [acq_rel_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [acq_rel_sys_i8_param_0];
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    ld.param.b8 %r9, [acq_rel_sys_i8_param_1];
+; SM30-NEXT:    ld.param.b8 %r7, [acq_rel_sys_i8_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 255;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    and.b32 %r15, %r14, 255;
-; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r16, [%rd1];
-; SM30-NEXT:    and.b32 %r20, %r16, %r2;
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 255;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    and.b32 %r13, %r12, 255;
+; SM30-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r14, [%rd1];
+; SM30-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM30-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r17, %r20, %r3;
-; SM30-NEXT:    or.b32 %r18, %r20, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM30-NEXT:    or.b32 %r15, %r17, %r3;
+; SM30-NEXT:    or.b32 %r16, %r17, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM30-NEXT:    @%p1 bra $L__BB3_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB3_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM30-NEXT:    mov.b32 %r20, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM30-NEXT:    mov.b32 %r17, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB3_1;
 ; SM30-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: acq_rel_sys_i8(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [acq_rel_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_sys_i8_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    ld.param.b8 %r9, [acq_rel_sys_i8_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [acq_rel_sys_i8_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB3_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB3_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB3_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB3_1;
 ; SM70-NEXT:  $L__BB3_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: acq_rel_sys_i8(
 ; SM90:       {
@@ -530,86 +530,86 @@ define i8 @seq_cst_sys_i8(ptr %addr, i8 %cmp, i8 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<21>;
+; SM30-NEXT:    .reg .b32 %r<18>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b8 %rs1, [seq_cst_sys_i8_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [seq_cst_sys_i8_param_0];
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    ld.param.b8 %r9, [seq_cst_sys_i8_param_1];
+; SM30-NEXT:    ld.param.b8 %r7, [seq_cst_sys_i8_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 255;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    and.b32 %r15, %r14, 255;
-; SM30-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r16, [%rd1];
-; SM30-NEXT:    and.b32 %r20, %r16, %r2;
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 255;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    and.b32 %r13, %r12, 255;
+; SM30-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r14, [%rd1];
+; SM30-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM30-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r17, %r20, %r3;
-; SM30-NEXT:    or.b32 %r18, %r20, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM30-NEXT:    or.b32 %r15, %r17, %r3;
+; SM30-NEXT:    or.b32 %r16, %r17, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM30-NEXT:    @%p1 bra $L__BB4_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB4_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM30-NEXT:    mov.b32 %r20, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM30-NEXT:    mov.b32 %r17, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB4_1;
 ; SM30-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: seq_cst_sys_i8(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .b32 %r<18>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b8 %rs1, [seq_cst_sys_i8_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_sys_i8_param_0];
 ; SM70-NEXT:    fence.sc.sys;
-; SM70-NEXT:    ld.param.b8 %r9, [seq_cst_sys_i8_param_1];
+; SM70-NEXT:    ld.param.b8 %r7, [seq_cst_sys_i8_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 255;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    and.b32 %r15, %r14, 255;
-; SM70-NEXT:    shl.b32 %r3, %r15, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r16, [%rd1];
-; SM70-NEXT:    and.b32 %r20, %r16, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 255;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    and.b32 %r13, %r12, 255;
+; SM70-NEXT:    shl.b32 %r3, %r13, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r14, [%rd1];
+; SM70-NEXT:    and.b32 %r17, %r14, %r2;
 ; SM70-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r17, %r20, %r3;
-; SM70-NEXT:    or.b32 %r18, %r20, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r18, %r17;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r18;
+; SM70-NEXT:    or.b32 %r15, %r17, %r3;
+; SM70-NEXT:    or.b32 %r16, %r17, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r16, %r15;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r16;
 ; SM70-NEXT:    @%p1 bra $L__BB4_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB4_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r20, %r8;
-; SM70-NEXT:    mov.b32 %r20, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r17, %r6;
+; SM70-NEXT:    mov.b32 %r17, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB4_1;
 ; SM70-NEXT:  $L__BB4_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: seq_cst_sys_i8(
 ; SM90:       {
@@ -663,80 +663,80 @@ define i16 @relaxed_sys_i16(ptr %addr, i16 %cmp, i16 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<20>;
+; SM30-NEXT:    .reg .b32 %r<17>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b16 %rs1, [relaxed_sys_i16_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [relaxed_sys_i16_param_0];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    ld.param.b16 %r9, [relaxed_sys_i16_param_1];
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 65535;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r15, [%rd1];
-; SM30-NEXT:    and.b32 %r19, %r15, %r2;
+; SM30-NEXT:    ld.param.b16 %r7, [relaxed_sys_i16_param_1];
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 65535;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r13, [%rd1];
+; SM30-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM30-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r16, %r19, %r3;
-; SM30-NEXT:    or.b32 %r17, %r19, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM30-NEXT:    or.b32 %r14, %r16, %r3;
+; SM30-NEXT:    or.b32 %r15, %r16, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM30-NEXT:    @%p1 bra $L__BB5_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB5_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM30-NEXT:    mov.b32 %r19, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM30-NEXT:    mov.b32 %r16, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB5_1;
 ; SM30-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: relaxed_sys_i16(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [relaxed_sys_i16_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [relaxed_sys_i16_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b16 %r9, [relaxed_sys_i16_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    ld.param.b16 %r7, [relaxed_sys_i16_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB5_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB5_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB5_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB5_1;
 ; SM70-NEXT:  $L__BB5_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: relaxed_sys_i16(
 ; SM90:       {
@@ -786,82 +786,82 @@ define i16 @acquire_sys_i16(ptr %addr, i16 %cmp, i16 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<20>;
+; SM30-NEXT:    .reg .b32 %r<17>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b16 %rs1, [acquire_sys_i16_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [acquire_sys_i16_param_0];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    ld.param.b16 %r9, [acquire_sys_i16_param_1];
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 65535;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r15, [%rd1];
-; SM30-NEXT:    and.b32 %r19, %r15, %r2;
+; SM30-NEXT:    ld.param.b16 %r7, [acquire_sys_i16_param_1];
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 65535;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r13, [%rd1];
+; SM30-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM30-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r16, %r19, %r3;
-; SM30-NEXT:    or.b32 %r17, %r19, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM30-NEXT:    or.b32 %r14, %r16, %r3;
+; SM30-NEXT:    or.b32 %r15, %r16, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM30-NEXT:    @%p1 bra $L__BB6_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB6_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM30-NEXT:    mov.b32 %r19, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM30-NEXT:    mov.b32 %r16, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB6_1;
 ; SM30-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: acquire_sys_i16(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [acquire_sys_i16_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acquire_sys_i16_param_0];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    ld.param.b16 %r9, [acquire_sys_i16_param_1];
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    ld.param.b16 %r7, [acquire_sys_i16_param_1];
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB6_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB6_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB6_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB6_1;
 ; SM70-NEXT:  $L__BB6_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: acquire_sys_i16(
 ; SM90:       {
@@ -912,82 +912,82 @@ define i16 @release_sys_i16(ptr %addr, i16 %cmp, i16 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<20>;
+; SM30-NEXT:    .reg .b32 %r<17>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b16 %rs1, [release_sys_i16_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [release_sys_i16_param_0];
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    ld.param.b16 %r9, [release_sys_i16_param_1];
+; SM30-NEXT:    ld.param.b16 %r7, [release_sys_i16_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 65535;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r15, [%rd1];
-; SM30-NEXT:    and.b32 %r19, %r15, %r2;
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 65535;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r13, [%rd1];
+; SM30-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM30-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r16, %r19, %r3;
-; SM30-NEXT:    or.b32 %r17, %r19, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM30-NEXT:    or.b32 %r14, %r16, %r3;
+; SM30-NEXT:    or.b32 %r15, %r16, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM30-NEXT:    @%p1 bra $L__BB7_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB7_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM30-NEXT:    mov.b32 %r19, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM30-NEXT:    mov.b32 %r16, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB7_1;
 ; SM30-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: release_sys_i16(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [release_sys_i16_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [release_sys_i16_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    ld.param.b16 %r9, [release_sys_i16_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [release_sys_i16_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB7_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB7_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB7_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB7_1;
 ; SM70-NEXT:  $L__BB7_3: // %partword.cmpxchg.end
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: release_sys_i16(
 ; SM90:       {
@@ -1038,84 +1038,84 @@ define i16 @acq_rel_sys_i16(ptr %addr, i16 %cmp, i16 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<20>;
+; SM30-NEXT:    .reg .b32 %r<17>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b16 %rs1, [acq_rel_sys_i16_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [acq_rel_sys_i16_param_0];
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    ld.param.b16 %r9, [acq_rel_sys_i16_param_1];
+; SM30-NEXT:    ld.param.b16 %r7, [acq_rel_sys_i16_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 65535;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r15, [%rd1];
-; SM30-NEXT:    and.b32 %r19, %r15, %r2;
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 65535;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r13, [%rd1];
+; SM30-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM30-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r16, %r19, %r3;
-; SM30-NEXT:    or.b32 %r17, %r19, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM30-NEXT:    or.b32 %r14, %r16, %r3;
+; SM30-NEXT:    or.b32 %r15, %r16, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM30-NEXT:    @%p1 bra $L__BB8_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB8_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM30-NEXT:    mov.b32 %r19, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM30-NEXT:    mov.b32 %r16, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB8_1;
 ; SM30-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: acq_rel_sys_i16(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [acq_rel_sys_i16_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [acq_rel_sys_i16_param_0];
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    ld.param.b16 %r9, [acq_rel_sys_i16_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [acq_rel_sys_i16_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB8_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB8_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB8_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB8_1;
 ; SM70-NEXT:  $L__BB8_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: acq_rel_sys_i16(
 ; SM90:       {
@@ -1168,84 +1168,84 @@ define i16 @seq_cst_sys_i16(ptr %addr, i16 %cmp, i16 %new) {
 ; SM30:       {
 ; SM30-NEXT:    .reg .pred %p<3>;
 ; SM30-NEXT:    .reg .b16 %rs<2>;
-; SM30-NEXT:    .reg .b32 %r<20>;
+; SM30-NEXT:    .reg .b32 %r<17>;
 ; SM30-NEXT:    .reg .b64 %rd<3>;
 ; SM30-EMPTY:
 ; SM30-NEXT:  // %bb.0:
 ; SM30-NEXT:    ld.param.b16 %rs1, [seq_cst_sys_i16_param_2];
 ; SM30-NEXT:    ld.param.b64 %rd2, [seq_cst_sys_i16_param_0];
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    ld.param.b16 %r9, [seq_cst_sys_i16_param_1];
+; SM30-NEXT:    ld.param.b16 %r7, [seq_cst_sys_i16_param_1];
 ; SM30-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM30-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM30-NEXT:    and.b32 %r11, %r10, 3;
-; SM30-NEXT:    shl.b32 %r1, %r11, 3;
-; SM30-NEXT:    mov.b32 %r12, 65535;
-; SM30-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM30-NEXT:    not.b32 %r2, %r13;
-; SM30-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM30-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM30-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM30-NEXT:    ld.b32 %r15, [%rd1];
-; SM30-NEXT:    and.b32 %r19, %r15, %r2;
+; SM30-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM30-NEXT:    and.b32 %r9, %r8, 3;
+; SM30-NEXT:    shl.b32 %r1, %r9, 3;
+; SM30-NEXT:    mov.b32 %r10, 65535;
+; SM30-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM30-NEXT:    not.b32 %r2, %r11;
+; SM30-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM30-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM30-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM30-NEXT:    ld.b32 %r13, [%rd1];
+; SM30-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM30-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
 ; SM30-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM30-NEXT:    or.b32 %r16, %r19, %r3;
-; SM30-NEXT:    or.b32 %r17, %r19, %r4;
-; SM30-NEXT:    atom.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM30-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM30-NEXT:    or.b32 %r14, %r16, %r3;
+; SM30-NEXT:    or.b32 %r15, %r16, %r4;
+; SM30-NEXT:    atom.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM30-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM30-NEXT:    @%p1 bra $L__BB9_3;
 ; SM30-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM30-NEXT:    // in Loop: Header=BB9_1 Depth=1
-; SM30-NEXT:    and.b32 %r8, %r7, %r2;
-; SM30-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM30-NEXT:    mov.b32 %r19, %r8;
+; SM30-NEXT:    and.b32 %r6, %r5, %r2;
+; SM30-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM30-NEXT:    mov.b32 %r16, %r6;
 ; SM30-NEXT:    @%p2 bra $L__BB9_1;
 ; SM30-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM30-NEXT:    membar.sys;
-; SM30-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM30-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM30-NEXT:    ret;
 ;
 ; SM70-LABEL: seq_cst_sys_i16(
 ; SM70:       {
 ; SM70-NEXT:    .reg .pred %p<3>;
 ; SM70-NEXT:    .reg .b16 %rs<2>;
-; SM70-NEXT:    .reg .b32 %r<20>;
+; SM70-NEXT:    .reg .b32 %r<17>;
 ; SM70-NEXT:    .reg .b64 %rd<3>;
 ; SM70-EMPTY:
 ; SM70-NEXT:  // %bb.0:
 ; SM70-NEXT:    ld.param.b16 %rs1, [seq_cst_sys_i16_param_2];
 ; SM70-NEXT:    ld.param.b64 %rd2, [seq_cst_sys_i16_param_0];
 ; SM70-NEXT:    fence.sc.sys;
-; SM70-NEXT:    ld.param.b16 %r9, [seq_cst_sys_i16_param_1];
+; SM70-NEXT:    ld.param.b16 %r7, [seq_cst_sys_i16_param_1];
 ; SM70-NEXT:    and.b64 %rd1, %rd2, -4;
-; SM70-NEXT:    cvt.u32.u64 %r10, %rd2;
-; SM70-NEXT:    and.b32 %r11, %r10, 3;
-; SM70-NEXT:    shl.b32 %r1, %r11, 3;
-; SM70-NEXT:    mov.b32 %r12, 65535;
-; SM70-NEXT:    shl.b32 %r13, %r12, %r1;
-; SM70-NEXT:    not.b32 %r2, %r13;
-; SM70-NEXT:    cvt.u32.u16 %r14, %rs1;
-; SM70-NEXT:    shl.b32 %r3, %r14, %r1;
-; SM70-NEXT:    shl.b32 %r4, %r9, %r1;
-; SM70-NEXT:    ld.b32 %r15, [%rd1];
-; SM70-NEXT:    and.b32 %r19, %r15, %r2;
+; SM70-NEXT:    cvt.u32.u64 %r8, %rd2;
+; SM70-NEXT:    and.b32 %r9, %r8, 3;
+; SM70-NEXT:    shl.b32 %r1, %r9, 3;
+; SM70-NEXT:    mov.b32 %r10, 65535;
+; SM70-NEXT:    shl.b32 %r11, %r10, %r1;
+; SM70-NEXT:    not.b32 %r2, %r11;
+; SM70-NEXT:    cvt.u32.u16 %r12, %rs1;
+; SM70-NEXT:    shl.b32 %r3, %r12, %r1;
+; SM70-NEXT:    shl.b32 %r4, %r7, %r1;
+; SM70-NEXT:    ld.b32 %r13, [%rd1];
+; SM70-NEXT:    and.b32 %r16, %r13, %r2;
 ; SM70-NEXT:  $L__BB9_1: // %partword.cmpxchg.loop
 ; SM70-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM70-NEXT:    or.b32 %r16, %r19, %r3;
-; SM70-NEXT:    or.b32 %r17, %r19, %r4;
-; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r7, [%rd1], %r17, %r16;
-; SM70-NEXT:    setp.eq.b32 %p1, %r7, %r17;
+; SM70-NEXT:    or.b32 %r14, %r16, %r3;
+; SM70-NEXT:    or.b32 %r15, %r16, %r4;
+; SM70-NEXT:    atom.relaxed.sys.cas.b32 %r5, [%rd1], %r15, %r14;
+; SM70-NEXT:    setp.eq.b32 %p1, %r5, %r15;
 ; SM70-NEXT:    @%p1 bra $L__BB9_3;
 ; SM70-NEXT:  // %bb.2: // %partword.cmpxchg.failure
 ; SM70-NEXT:    // in Loop: Header=BB9_1 Depth=1
-; SM70-NEXT:    and.b32 %r8, %r7, %r2;
-; SM70-NEXT:    setp.ne.b32 %p2, %r19, %r8;
-; SM70-NEXT:    mov.b32 %r19, %r8;
+; SM70-NEXT:    and.b32 %r6, %r5, %r2;
+; SM70-NEXT:    setp.ne.b32 %p2, %r16, %r6;
+; SM70-NEXT:    mov.b32 %r16, %r6;
 ; SM70-NEXT:    @%p2 bra $L__BB9_1;
 ; SM70-NEXT:  $L__BB9_3: // %partword.cmpxchg.end
 ; SM70-NEXT:    fence.acq_rel.sys;
-; SM70-NEXT:    st.param.b32 [func_retval0], %r14;
+; SM70-NEXT:    st.param.b32 [func_retval0], %r12;
 ; SM70-NEXT:    ret;
 ; SM90-LABEL: seq_cst_sys_i16(
 ; SM90:       {
diff --git a/llvm/test/CodeGen/NVPTX/combine-mad.ll b/llvm/test/CodeGen/NVPTX/combine-mad.ll
index da303b7c38eb7..e6bce8991a71d 100644
--- a/llvm/test/CodeGen/NVPTX/combine-mad.ll
+++ b/llvm/test/CodeGen/NVPTX/combine-mad.ll
@@ -189,7 +189,7 @@ declare i32 @use(i32 %0, i32 %1)
 define i32 @test_mad_multi_use(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: test_mad_multi_use(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_mad_multi_use_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll b/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll
index dd3e4ecddcd2e..6c80055ef4673 100644
--- a/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll
+++ b/llvm/test/CodeGen/NVPTX/convert-call-to-indirect.ll
@@ -9,7 +9,7 @@ declare i64 @callee_variadic(ptr %p, ...);
 define %struct.64 @test_return_type_mismatch(ptr %p) {
 ; CHECK-LABEL: test_return_type_mismatch(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<40>;
+; CHECK-NEXT:    .reg .b64 %rd<32>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_return_type_mismatch_param_0];
@@ -29,35 +29,35 @@ define %struct.64 @test_return_type_mismatch(ptr %p) {
 ; CHECK-NEXT:    ld.param.b8 %rd9, [retval0+1];
 ; CHECK-NEXT:    ld.param.b8 %rd10, [retval0];
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    shl.b64 %rd13, %rd9, 8;
-; CHECK-NEXT:    or.b64 %rd14, %rd13, %rd10;
-; CHECK-NEXT:    shl.b64 %rd16, %rd8, 16;
-; CHECK-NEXT:    shl.b64 %rd18, %rd7, 24;
-; CHECK-NEXT:    or.b64 %rd19, %rd18, %rd16;
-; CHECK-NEXT:    or.b64 %rd20, %rd19, %rd14;
-; CHECK-NEXT:    shl.b64 %rd23, %rd5, 8;
-; CHECK-NEXT:    or.b64 %rd24, %rd23, %rd6;
-; CHECK-NEXT:    shl.b64 %rd26, %rd4, 16;
-; CHECK-NEXT:    shl.b64 %rd28, %rd3, 24;
-; CHECK-NEXT:    or.b64 %rd29, %rd28, %rd26;
-; CHECK-NEXT:    or.b64 %rd30, %rd29, %rd24;
-; CHECK-NEXT:    shl.b64 %rd31, %rd30, 32;
-; CHECK-NEXT:    or.b64 %rd32, %rd31, %rd20;
+; CHECK-NEXT:    shl.b64 %rd11, %rd9, 8;
+; CHECK-NEXT:    or.b64 %rd12, %rd11, %rd10;
+; CHECK-NEXT:    shl.b64 %rd13, %rd8, 16;
+; CHECK-NEXT:    shl.b64 %rd14, %rd7, 24;
+; CHECK-NEXT:    or.b64 %rd15, %rd14, %rd13;
+; CHECK-NEXT:    or.b64 %rd16, %rd15, %rd12;
+; CHECK-NEXT:    shl.b64 %rd17, %rd5, 8;
+; CHECK-NEXT:    or.b64 %rd18, %rd17, %rd6;
+; CHECK-NEXT:    shl.b64 %rd19, %rd4, 16;
+; CHECK-NEXT:    shl.b64 %rd20, %rd3, 24;
+; CHECK-NEXT:    or.b64 %rd21, %rd20, %rd19;
+; CHECK-NEXT:    or.b64 %rd22, %rd21, %rd18;
+; CHECK-NEXT:    shl.b64 %rd23, %rd22, 32;
+; CHECK-NEXT:    or.b64 %rd24, %rd23, %rd16;
 ; CHECK-NEXT:    st.param.b8 [func_retval0], %rd10;
-; CHECK-NEXT:    shr.u64 %rd33, %rd32, 56;
-; CHECK-NEXT:    st.param.b8 [func_retval0+7], %rd33;
-; CHECK-NEXT:    shr.u64 %rd34, %rd32, 48;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %rd34;
-; CHECK-NEXT:    shr.u64 %rd35, %rd32, 40;
-; CHECK-NEXT:    st.param.b8 [func_retval0+5], %rd35;
-; CHECK-NEXT:    shr.u64 %rd36, %rd32, 32;
-; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rd36;
-; CHECK-NEXT:    shr.u64 %rd37, %rd32, 24;
-; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rd37;
-; CHECK-NEXT:    shr.u64 %rd38, %rd32, 16;
-; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rd38;
-; CHECK-NEXT:    shr.u64 %rd39, %rd32, 8;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rd39;
+; CHECK-NEXT:    shr.u64 %rd25, %rd24, 56;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %rd25;
+; CHECK-NEXT:    shr.u64 %rd26, %rd24, 48;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %rd26;
+; CHECK-NEXT:    shr.u64 %rd27, %rd24, 40;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], %rd27;
+; CHECK-NEXT:    shr.u64 %rd28, %rd24, 32;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rd28;
+; CHECK-NEXT:    shr.u64 %rd29, %rd24, 24;
+; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rd29;
+; CHECK-NEXT:    shr.u64 %rd30, %rd24, 16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rd30;
+; CHECK-NEXT:    shr.u64 %rd31, %rd24, 8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rd31;
 ; CHECK-NEXT:    ret;
   %ret = call %struct.64 @callee(ptr %p)
   ret %struct.64 %ret
@@ -66,7 +66,7 @@ define %struct.64 @test_return_type_mismatch(ptr %p) {
 define i64 @test_param_type_mismatch(ptr %p) {
 ; CHECK-LABEL: test_param_type_mismatch(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    { // callseq 1, 0
@@ -87,7 +87,7 @@ define i64 @test_param_type_mismatch(ptr %p) {
 define i64 @test_param_count_mismatch(ptr %p) {
 ; CHECK-LABEL: test_param_count_mismatch(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_param_count_mismatch_param_0];
@@ -111,7 +111,7 @@ define i64 @test_param_count_mismatch(ptr %p) {
 define %struct.64 @test_return_type_mismatch_variadic(ptr %p) {
 ; CHECK-LABEL: test_return_type_mismatch_variadic(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<40>;
+; CHECK-NEXT:    .reg .b64 %rd<32>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_return_type_mismatch_variadic_param_0];
@@ -131,35 +131,35 @@ define %struct.64 @test_return_type_mismatch_variadic(ptr %p) {
 ; CHECK-NEXT:    ld.param.b8 %rd9, [retval0+1];
 ; CHECK-NEXT:    ld.param.b8 %rd10, [retval0];
 ; CHECK-NEXT:    } // callseq 3
-; CHECK-NEXT:    shl.b64 %rd13, %rd9, 8;
-; CHECK-NEXT:    or.b64 %rd14, %rd13, %rd10;
-; CHECK-NEXT:    shl.b64 %rd16, %rd8, 16;
-; CHECK-NEXT:    shl.b64 %rd18, %rd7, 24;
-; CHECK-NEXT:    or.b64 %rd19, %rd18, %rd16;
-; CHECK-NEXT:    or.b64 %rd20, %rd19, %rd14;
-; CHECK-NEXT:    shl.b64 %rd23, %rd5, 8;
-; CHECK-NEXT:    or.b64 %rd24, %rd23, %rd6;
-; CHECK-NEXT:    shl.b64 %rd26, %rd4, 16;
-; CHECK-NEXT:    shl.b64 %rd28, %rd3, 24;
-; CHECK-NEXT:    or.b64 %rd29, %rd28, %rd26;
-; CHECK-NEXT:    or.b64 %rd30, %rd29, %rd24;
-; CHECK-NEXT:    shl.b64 %rd31, %rd30, 32;
-; CHECK-NEXT:    or.b64 %rd32, %rd31, %rd20;
+; CHECK-NEXT:    shl.b64 %rd11, %rd9, 8;
+; CHECK-NEXT:    or.b64 %rd12, %rd11, %rd10;
+; CHECK-NEXT:    shl.b64 %rd13, %rd8, 16;
+; CHECK-NEXT:    shl.b64 %rd14, %rd7, 24;
+; CHECK-NEXT:    or.b64 %rd15, %rd14, %rd13;
+; CHECK-NEXT:    or.b64 %rd16, %rd15, %rd12;
+; CHECK-NEXT:    shl.b64 %rd17, %rd5, 8;
+; CHECK-NEXT:    or.b64 %rd18, %rd17, %rd6;
+; CHECK-NEXT:    shl.b64 %rd19, %rd4, 16;
+; CHECK-NEXT:    shl.b64 %rd20, %rd3, 24;
+; CHECK-NEXT:    or.b64 %rd21, %rd20, %rd19;
+; CHECK-NEXT:    or.b64 %rd22, %rd21, %rd18;
+; CHECK-NEXT:    shl.b64 %rd23, %rd22, 32;
+; CHECK-NEXT:    or.b64 %rd24, %rd23, %rd16;
 ; CHECK-NEXT:    st.param.b8 [func_retval0], %rd10;
-; CHECK-NEXT:    shr.u64 %rd33, %rd32, 56;
-; CHECK-NEXT:    st.param.b8 [func_retval0+7], %rd33;
-; CHECK-NEXT:    shr.u64 %rd34, %rd32, 48;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %rd34;
-; CHECK-NEXT:    shr.u64 %rd35, %rd32, 40;
-; CHECK-NEXT:    st.param.b8 [func_retval0+5], %rd35;
-; CHECK-NEXT:    shr.u64 %rd36, %rd32, 32;
-; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rd36;
-; CHECK-NEXT:    shr.u64 %rd37, %rd32, 24;
-; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rd37;
-; CHECK-NEXT:    shr.u64 %rd38, %rd32, 16;
-; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rd38;
-; CHECK-NEXT:    shr.u64 %rd39, %rd32, 8;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rd39;
+; CHECK-NEXT:    shr.u64 %rd25, %rd24, 56;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %rd25;
+; CHECK-NEXT:    shr.u64 %rd26, %rd24, 48;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %rd26;
+; CHECK-NEXT:    shr.u64 %rd27, %rd24, 40;
+; CHECK-NEXT:    st.param.b8 [func_retval0+5], %rd27;
+; CHECK-NEXT:    shr.u64 %rd28, %rd24, 32;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rd28;
+; CHECK-NEXT:    shr.u64 %rd29, %rd24, 24;
+; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rd29;
+; CHECK-NEXT:    shr.u64 %rd30, %rd24, 16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rd30;
+; CHECK-NEXT:    shr.u64 %rd31, %rd24, 8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rd31;
 ; CHECK-NEXT:    ret;
   %ret = call %struct.64 (ptr, ...) @callee_variadic(ptr %p)
   ret %struct.64 %ret
@@ -168,7 +168,7 @@ define %struct.64 @test_return_type_mismatch_variadic(ptr %p) {
 define i64 @test_param_type_mismatch_variadic(ptr %p) {
 ; CHECK-LABEL: test_param_type_mismatch_variadic(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_param_type_mismatch_variadic_param_0];
@@ -190,7 +190,7 @@ define i64 @test_param_type_mismatch_variadic(ptr %p) {
 define i64 @test_param_count_mismatch_variadic(ptr %p) {
 ; CHECK-LABEL: test_param_count_mismatch_variadic(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_param_count_mismatch_variadic_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/cse-mov-sym.ll b/llvm/test/CodeGen/NVPTX/cse-mov-sym.ll
new file mode 100644
index 0000000000000..2e68208786d24
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/cse-mov-sym.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s | FileCheck %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+@global_smem = external addrspace(3) global [0 x i8], align 16
+
+
+;; Confirm the mov.b64 of global_smem is CSE'd. We need to make things a bit
+;; complex with a loop to make this interesting.
+define i32 @test_mov_sym(i32 %offset1, i32 %offset2, i1 %cond) {
+; CHECK-LABEL: test_mov_sym(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<4>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0: // %entry
+; CHECK-NEXT:    ld.param.b8 %rs1, [test_mov_sym_param_2];
+; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NEXT:    ld.param.b32 %r1, [test_mov_sym_param_0];
+; CHECK-NEXT:    cvt.s64.s32 %rd1, %r1;
+; CHECK-NEXT:    mov.b64 %rd2, global_smem;
+; CHECK-NEXT:    add.s64 %rd3, %rd2, %rd1;
+; CHECK-NEXT:    ld.shared.b32 %r4, [%rd3];
+; CHECK-NEXT:    not.pred %p2, %p1;
+; CHECK-NEXT:    @%p2 bra $L__BB0_4;
+; CHECK-NEXT:  // %bb.1: // %if1.preheader
+; CHECK-NEXT:    ld.param.b32 %r2, [test_mov_sym_param_1];
+; CHECK-NEXT:    setp.ne.b32 %p3, %r1, %r2;
+; CHECK-NEXT:  $L__BB0_2: // %if1
+; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    @%p3 bra $L__BB0_2;
+; CHECK-NEXT:  // %bb.3: // %if2
+; CHECK-NEXT:    cvt.s64.s32 %rd4, %r2;
+; CHECK-NEXT:    add.s64 %rd5, %rd2, %rd4;
+; CHECK-NEXT:    ld.shared.b32 %r3, [%rd5];
+; CHECK-NEXT:    add.s32 %r4, %r4, %r3;
+; CHECK-NEXT:  $L__BB0_4: // %end
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
+entry:
+    %gep = getelementptr inbounds i8, ptr addrspace(3) @global_smem, i32 %offset1
+    %val = load i32, ptr addrspace(3) %gep
+    br i1 %cond, label %if1, label %end
+if1:
+    %cond2 = icmp eq i32 %offset1, %offset2
+    br i1 %cond2, label %if2, label %if1
+if2:
+    %gep2 = getelementptr inbounds i8, ptr addrspace(3) @global_smem, i32 %offset2
+    %val2 = load i32, ptr addrspace(3) %gep2
+    %add = add i32 %val, %val2
+    br label %end
+end:
+    %ret = phi i32 [ %add, %if2 ], [ %val, %entry ]
+    ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/NVPTX/distributed-shared-cluster.ll b/llvm/test/CodeGen/NVPTX/distributed-shared-cluster.ll
index 2841e6751d029..1d70b9deb6089 100644
--- a/llvm/test/CodeGen/NVPTX/distributed-shared-cluster.ll
+++ b/llvm/test/CodeGen/NVPTX/distributed-shared-cluster.ll
@@ -166,23 +166,23 @@ define void @test_distributed_shared_cluster_cmpxchg(ptr addrspace(7) %dsmem_ptr
 ; CHECK-LABEL: test_distributed_shared_cluster_cmpxchg(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<11>;
-; CHECK-NEXT:    .reg .b32 %r<53>;
+; CHECK-NEXT:    .reg .b32 %r<43>;
 ; CHECK-NEXT:    .reg .b64 %rd<12>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ld.param.b64 %rd2, [test_distributed_shared_cluster_cmpxchg_param_0];
-; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r24, [%rd2], 1, 0;
-; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r25, [%rd2], 1, 0;
-; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r26, [%rd2], 1, 0;
-; CHECK-NEXT:    atom.release.sys.shared::cluster.cas.b32 %r27, [%rd2], 1, 0;
-; CHECK-NEXT:    atom.acq_rel.sys.shared::cluster.cas.b32 %r28, [%rd2], 1, 0;
-; CHECK-NEXT:    atom.acq_rel.sys.shared::cluster.cas.b32 %r29, [%rd2], 1, 0;
+; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r14, [%rd2], 1, 0;
+; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r15, [%rd2], 1, 0;
+; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r16, [%rd2], 1, 0;
+; CHECK-NEXT:    atom.release.sys.shared::cluster.cas.b32 %r17, [%rd2], 1, 0;
+; CHECK-NEXT:    atom.acq_rel.sys.shared::cluster.cas.b32 %r18, [%rd2], 1, 0;
+; CHECK-NEXT:    atom.acq_rel.sys.shared::cluster.cas.b32 %r19, [%rd2], 1, 0;
 ; CHECK-NEXT:    fence.sc.sys;
-; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r30, [%rd2], 1, 0;
+; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r20, [%rd2], 1, 0;
 ; CHECK-NEXT:    fence.sc.sys;
-; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r31, [%rd2], 1, 0;
+; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r21, [%rd2], 1, 0;
 ; CHECK-NEXT:    fence.sc.sys;
-; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r32, [%rd2], 1, 0;
+; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b32 %r22, [%rd2], 1, 0;
 ; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b64 %rd3, [%rd2], 1, 0;
 ; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b64 %rd4, [%rd2], 1, 0;
 ; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b64 %rd5, [%rd2], 1, 0;
@@ -196,92 +196,92 @@ define void @test_distributed_shared_cluster_cmpxchg(ptr addrspace(7) %dsmem_ptr
 ; CHECK-NEXT:    fence.sc.sys;
 ; CHECK-NEXT:    atom.acquire.sys.shared::cluster.cas.b64 %rd11, [%rd2], 1, 0;
 ; CHECK-NEXT:    and.b64 %rd1, %rd2, -4;
-; CHECK-NEXT:    cvt.u32.u64 %r33, %rd2;
-; CHECK-NEXT:    and.b32 %r34, %r33, 3;
-; CHECK-NEXT:    shl.b32 %r1, %r34, 3;
-; CHECK-NEXT:    mov.b32 %r35, 65535;
-; CHECK-NEXT:    shl.b32 %r36, %r35, %r1;
-; CHECK-NEXT:    not.b32 %r2, %r36;
-; CHECK-NEXT:    mov.b32 %r37, 1;
-; CHECK-NEXT:    shl.b32 %r3, %r37, %r1;
-; CHECK-NEXT:    ld.shared::cluster.b32 %r38, [%rd1];
-; CHECK-NEXT:    and.b32 %r48, %r38, %r2;
+; CHECK-NEXT:    cvt.u32.u64 %r23, %rd2;
+; CHECK-NEXT:    and.b32 %r24, %r23, 3;
+; CHECK-NEXT:    shl.b32 %r1, %r24, 3;
+; CHECK-NEXT:    mov.b32 %r25, 65535;
+; CHECK-NEXT:    shl.b32 %r26, %r25, %r1;
+; CHECK-NEXT:    not.b32 %r2, %r26;
+; CHECK-NEXT:    mov.b32 %r27, 1;
+; CHECK-NEXT:    shl.b32 %r3, %r27, %r1;
+; CHECK-NEXT:    ld.shared::cluster.b32 %r28, [%rd1];
+; CHECK-NEXT:    and.b32 %r38, %r28, %r2;
 ; CHECK-NEXT:  $L__BB4_1: // %partword.cmpxchg.loop33
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    or.b32 %r39, %r48, %r3;
-; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r6, [%rd1], %r39, %r48;
-; CHECK-NEXT:    setp.eq.b32 %p1, %r6, %r39;
+; CHECK-NEXT:    or.b32 %r29, %r38, %r3;
+; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r4, [%rd1], %r29, %r38;
+; CHECK-NEXT:    setp.eq.b32 %p1, %r4, %r29;
 ; CHECK-NEXT:    @%p1 bra $L__BB4_3;
 ; CHECK-NEXT:  // %bb.2: // %partword.cmpxchg.failure32
 ; CHECK-NEXT:    // in Loop: Header=BB4_1 Depth=1
-; CHECK-NEXT:    and.b32 %r7, %r6, %r2;
-; CHECK-NEXT:    setp.ne.b32 %p2, %r48, %r7;
-; CHECK-NEXT:    mov.b32 %r48, %r7;
+; CHECK-NEXT:    and.b32 %r5, %r4, %r2;
+; CHECK-NEXT:    setp.ne.b32 %p2, %r38, %r5;
+; CHECK-NEXT:    mov.b32 %r38, %r5;
 ; CHECK-NEXT:    @%p2 bra $L__BB4_1;
 ; CHECK-NEXT:  $L__BB4_3: // %partword.cmpxchg.end31
-; CHECK-NEXT:    ld.shared::cluster.b32 %r40, [%rd1];
-; CHECK-NEXT:    and.b32 %r49, %r40, %r2;
+; CHECK-NEXT:    ld.shared::cluster.b32 %r30, [%rd1];
+; CHECK-NEXT:    and.b32 %r39, %r30, %r2;
 ; CHECK-NEXT:  $L__BB4_4: // %partword.cmpxchg.loop23
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    or.b32 %r41, %r49, %r3;
-; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r10, [%rd1], %r41, %r49;
-; CHECK-NEXT:    setp.eq.b32 %p3, %r10, %r41;
+; CHECK-NEXT:    or.b32 %r31, %r39, %r3;
+; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r6, [%rd1], %r31, %r39;
+; CHECK-NEXT:    setp.eq.b32 %p3, %r6, %r31;
 ; CHECK-NEXT:    @%p3 bra $L__BB4_6;
 ; CHECK-NEXT:  // %bb.5: // %partword.cmpxchg.failure22
 ; CHECK-NEXT:    // in Loop: Header=BB4_4 Depth=1
-; CHECK-NEXT:    and.b32 %r11, %r10, %r2;
-; CHECK-NEXT:    setp.ne.b32 %p4, %r49, %r11;
-; CHECK-NEXT:    mov.b32 %r49, %r11;
+; CHECK-NEXT:    and.b32 %r7, %r6, %r2;
+; CHECK-NEXT:    setp.ne.b32 %p4, %r39, %r7;
+; CHECK-NEXT:    mov.b32 %r39, %r7;
 ; CHECK-NEXT:    @%p4 bra $L__BB4_4;
 ; CHECK-NEXT:  $L__BB4_6: // %partword.cmpxchg.end21
 ; CHECK-NEXT:    fence.acq_rel.sys;
 ; CHECK-NEXT:    fence.acq_rel.sys;
-; CHECK-NEXT:    ld.shared::cluster.b32 %r42, [%rd1];
-; CHECK-NEXT:    and.b32 %r50, %r42, %r2;
+; CHECK-NEXT:    ld.shared::cluster.b32 %r32, [%rd1];
+; CHECK-NEXT:    and.b32 %r40, %r32, %r2;
 ; CHECK-NEXT:  $L__BB4_7: // %partword.cmpxchg.loop13
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    or.b32 %r43, %r50, %r3;
-; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r14, [%rd1], %r43, %r50;
-; CHECK-NEXT:    setp.eq.b32 %p5, %r14, %r43;
+; CHECK-NEXT:    or.b32 %r33, %r40, %r3;
+; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r8, [%rd1], %r33, %r40;
+; CHECK-NEXT:    setp.eq.b32 %p5, %r8, %r33;
 ; CHECK-NEXT:    @%p5 bra $L__BB4_9;
 ; CHECK-NEXT:  // %bb.8: // %partword.cmpxchg.failure12
 ; CHECK-NEXT:    // in Loop: Header=BB4_7 Depth=1
-; CHECK-NEXT:    and.b32 %r15, %r14, %r2;
-; CHECK-NEXT:    setp.ne.b32 %p6, %r50, %r15;
-; CHECK-NEXT:    mov.b32 %r50, %r15;
+; CHECK-NEXT:    and.b32 %r9, %r8, %r2;
+; CHECK-NEXT:    setp.ne.b32 %p6, %r40, %r9;
+; CHECK-NEXT:    mov.b32 %r40, %r9;
 ; CHECK-NEXT:    @%p6 bra $L__BB4_7;
 ; CHECK-NEXT:  $L__BB4_9: // %partword.cmpxchg.end11
 ; CHECK-NEXT:    fence.acq_rel.sys;
-; CHECK-NEXT:    ld.shared::cluster.b32 %r44, [%rd1];
-; CHECK-NEXT:    and.b32 %r51, %r44, %r2;
+; CHECK-NEXT:    ld.shared::cluster.b32 %r34, [%rd1];
+; CHECK-NEXT:    and.b32 %r41, %r34, %r2;
 ; CHECK-NEXT:  $L__BB4_10: // %partword.cmpxchg.loop3
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    or.b32 %r45, %r51, %r3;
-; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r18, [%rd1], %r45, %r51;
-; CHECK-NEXT:    setp.eq.b32 %p7, %r18, %r45;
+; CHECK-NEXT:    or.b32 %r35, %r41, %r3;
+; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r10, [%rd1], %r35, %r41;
+; CHECK-NEXT:    setp.eq.b32 %p7, %r10, %r35;
 ; CHECK-NEXT:    @%p7 bra $L__BB4_12;
 ; CHECK-NEXT:  // %bb.11: // %partword.cmpxchg.failure2
 ; CHECK-NEXT:    // in Loop: Header=BB4_10 Depth=1
-; CHECK-NEXT:    and.b32 %r19, %r18, %r2;
-; CHECK-NEXT:    setp.ne.b32 %p8, %r51, %r19;
-; CHECK-NEXT:    mov.b32 %r51, %r19;
+; CHECK-NEXT:    and.b32 %r11, %r10, %r2;
+; CHECK-NEXT:    setp.ne.b32 %p8, %r41, %r11;
+; CHECK-NEXT:    mov.b32 %r41, %r11;
 ; CHECK-NEXT:    @%p8 bra $L__BB4_10;
 ; CHECK-NEXT:  $L__BB4_12: // %partword.cmpxchg.end1
 ; CHECK-NEXT:    fence.acq_rel.sys;
 ; CHECK-NEXT:    fence.sc.sys;
-; CHECK-NEXT:    ld.shared::cluster.b32 %r46, [%rd1];
-; CHECK-NEXT:    and.b32 %r52, %r46, %r2;
+; CHECK-NEXT:    ld.shared::cluster.b32 %r36, [%rd1];
+; CHECK-NEXT:    and.b32 %r42, %r36, %r2;
 ; CHECK-NEXT:  $L__BB4_13: // %partword.cmpxchg.loop
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    or.b32 %r47, %r52, %r3;
-; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r22, [%rd1], %r47, %r52;
-; CHECK-NEXT:    setp.eq.b32 %p9, %r22, %r47;
+; CHECK-NEXT:    or.b32 %r37, %r42, %r3;
+; CHECK-NEXT:    atom.relaxed.sys.shared::cluster.cas.b32 %r12, [%rd1], %r37, %r42;
+; CHECK-NEXT:    setp.eq.b32 %p9, %r12, %r37;
 ; CHECK-NEXT:    @%p9 bra $L__BB4_15;
 ; CHECK-NEXT:  // %bb.14: // %partword.cmpxchg.failure
 ; CHECK-NEXT:    // in Loop: Header=BB4_13 Depth=1
-; CHECK-NEXT:    and.b32 %r23, %r22, %r2;
-; CHECK-NEXT:    setp.ne.b32 %p10, %r52, %r23;
-; CHECK-NEXT:    mov.b32 %r52, %r23;
+; CHECK-NEXT:    and.b32 %r13, %r12, %r2;
+; CHECK-NEXT:    setp.ne.b32 %p10, %r42, %r13;
+; CHECK-NEXT:    mov.b32 %r42, %r13;
 ; CHECK-NEXT:    @%p10 bra $L__BB4_13;
 ; CHECK-NEXT:  $L__BB4_15: // %partword.cmpxchg.end
 ; CHECK-NEXT:    fence.acq_rel.sys;
diff --git a/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll b/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll
index 06fb8d2c7c54d..ce2f0f32a8748 100644
--- a/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/NVPTX/dynamic_stackalloc.ll
@@ -12,7 +12,7 @@
 define i32 @test_dynamic_stackalloc(i64 %n) {
 ; CHECK-32-LABEL: test_dynamic_stackalloc(
 ; CHECK-32:       {
-; CHECK-32-NEXT:    .reg .b32 %r<8>;
+; CHECK-32-NEXT:    .reg .b32 %r<7>;
 ; CHECK-32-EMPTY:
 ; CHECK-32-NEXT:  // %bb.0:
 ; CHECK-32-NEXT:    ld.param.b32 %r1, [test_dynamic_stackalloc_param_0];
@@ -32,7 +32,7 @@ define i32 @test_dynamic_stackalloc(i64 %n) {
 ;
 ; CHECK-64-LABEL: test_dynamic_stackalloc(
 ; CHECK-64:       {
-; CHECK-64-NEXT:    .reg .b32 %r<3>;
+; CHECK-64-NEXT:    .reg .b32 %r<2>;
 ; CHECK-64-NEXT:    .reg .b64 %rd<6>;
 ; CHECK-64-EMPTY:
 ; CHECK-64-NEXT:  // %bb.0:
diff --git a/llvm/test/CodeGen/NVPTX/extractelement.ll b/llvm/test/CodeGen/NVPTX/extractelement.ll
index d61a63ce24f89..6d67ed0b4d539 100644
--- a/llvm/test/CodeGen/NVPTX/extractelement.ll
+++ b/llvm/test/CodeGen/NVPTX/extractelement.ll
@@ -31,16 +31,16 @@ define i1  @test_v2i8_load(ptr %a) {
 ; CHECK-LABEL: test_v2i8_load(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b16 %rs<7>;
+; CHECK-NEXT:    .reg .b16 %rs<5>;
 ; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_v2i8_load_param_0];
 ; CHECK-NEXT:    ld.v2.b8 {%rs1, %rs2}, [%rd1];
-; CHECK-NEXT:    or.b16 %rs5, %rs1, %rs2;
-; CHECK-NEXT:    and.b16 %rs6, %rs5, 255;
-; CHECK-NEXT:    setp.eq.b16 %p1, %rs6, 0;
+; CHECK-NEXT:    or.b16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    and.b16 %rs4, %rs3, 255;
+; CHECK-NEXT:    setp.eq.b16 %p1, %rs4, 0;
 ; CHECK-NEXT:    selp.b32 %r1, -1, 0, %p1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/f16-instructions.ll b/llvm/test/CodeGen/NVPTX/f16-instructions.ll
index 2b7e4184670c7..d4aec4f16f1ab 100644
--- a/llvm/test/CodeGen/NVPTX/f16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/f16-instructions.ll
@@ -886,8 +886,8 @@ define half @test_sqrt(half %a) #0 {
 ; CHECK:      cvt.rn.f16.f32  [[R:%rs[0-9]+]], [[RF]];
 ; CHECK:      st.param.b16    [func_retval0], [[R]];
 ; CHECK:      ret;
-define half @test_sin(half %a) #0 #1 {
-  %r = call half @llvm.sin.f16(half %a)
+define half @test_sin(half %a) #0 {
+  %r = call afn half @llvm.sin.f16(half %a)
   ret half %r
 }
 
@@ -900,8 +900,8 @@ define half @test_sin(half %a) #0 #1 {
 ; CHECK:      cvt.rn.f16.f32  [[R:%rs[0-9]+]], [[RF]];
 ; CHECK:      st.param.b16    [func_retval0], [[R]];
 ; CHECK:      ret;
-define half @test_cos(half %a) #0 #1 {
-  %r = call half @llvm.cos.f16(half %a)
+define half @test_cos(half %a) #0 {
+  %r = call afn half @llvm.cos.f16(half %a)
   ret half %r
 }
 
@@ -1183,4 +1183,3 @@ define <2 x half> @test_neg_f16x2(<2 x half> noundef %arg) #0 {
 }
 
 attributes #0 = { nounwind }
-attributes #1 = { "unsafe-fp-math" = "true" }
diff --git a/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
index d4fcea320f3ad..7b2126870e319 100644
--- a/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/f16x2-instructions.ll
@@ -455,7 +455,7 @@ declare <2 x half> @test_callee(<2 x half> %a, <2 x half> %b) #0
 define <2 x half> @test_call(<2 x half> %a, <2 x half> %b) #0 {
 ; CHECK-LABEL: test_call(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r2, [test_call_param_1];
@@ -478,7 +478,7 @@ define <2 x half> @test_call(<2 x half> %a, <2 x half> %b) #0 {
 define <2 x half> @test_call_flipped(<2 x half> %a, <2 x half> %b) #0 {
 ; CHECK-LABEL: test_call_flipped(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r2, [test_call_flipped_param_1];
@@ -501,7 +501,7 @@ define <2 x half> @test_call_flipped(<2 x half> %a, <2 x half> %b) #0 {
 define <2 x half> @test_tailcall_flipped(<2 x half> %a, <2 x half> %b) #0 {
 ; CHECK-LABEL: test_tailcall_flipped(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r2, [test_tailcall_flipped_param_1];
@@ -596,18 +596,15 @@ define <2 x float> @test_select_cc_f32_f16(<2 x float> %a, <2 x float> %b,
 ; CHECK-F16:       {
 ; CHECK-F16-NEXT:    .reg .pred %p<3>;
 ; CHECK-F16-NEXT:    .reg .b32 %r<9>;
-; CHECK-F16-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-F16-EMPTY:
 ; CHECK-F16-NEXT:  // %bb.0:
-; CHECK-F16-NEXT:    ld.param.b32 %r2, [test_select_cc_f32_f16_param_3];
-; CHECK-F16-NEXT:    ld.param.b32 %r1, [test_select_cc_f32_f16_param_2];
-; CHECK-F16-NEXT:    ld.param.b64 %rd2, [test_select_cc_f32_f16_param_1];
-; CHECK-F16-NEXT:    ld.param.b64 %rd1, [test_select_cc_f32_f16_param_0];
-; CHECK-F16-NEXT:    setp.neu.f16x2 %p1|%p2, %r1, %r2;
-; CHECK-F16-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-F16-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-F16-NEXT:    selp.f32 %r7, %r6, %r4, %p2;
-; CHECK-F16-NEXT:    selp.f32 %r8, %r5, %r3, %p1;
+; CHECK-F16-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_cc_f32_f16_param_1];
+; CHECK-F16-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f32_f16_param_0];
+; CHECK-F16-NEXT:    ld.param.b32 %r6, [test_select_cc_f32_f16_param_3];
+; CHECK-F16-NEXT:    ld.param.b32 %r5, [test_select_cc_f32_f16_param_2];
+; CHECK-F16-NEXT:    setp.neu.f16x2 %p1|%p2, %r5, %r6;
+; CHECK-F16-NEXT:    selp.f32 %r7, %r2, %r4, %p2;
+; CHECK-F16-NEXT:    selp.f32 %r8, %r1, %r3, %p1;
 ; CHECK-F16-NEXT:    st.param.v2.b32 [func_retval0], {%r8, %r7};
 ; CHECK-F16-NEXT:    ret;
 ;
@@ -616,25 +613,22 @@ define <2 x float> @test_select_cc_f32_f16(<2 x float> %a, <2 x float> %b,
 ; CHECK-NOF16-NEXT:    .reg .pred %p<3>;
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<5>;
 ; CHECK-NOF16-NEXT:    .reg .b32 %r<13>;
-; CHECK-NOF16-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
-; CHECK-NOF16-NEXT:    ld.param.b32 %r2, [test_select_cc_f32_f16_param_3];
-; CHECK-NOF16-NEXT:    ld.param.b32 %r1, [test_select_cc_f32_f16_param_2];
-; CHECK-NOF16-NEXT:    ld.param.b64 %rd2, [test_select_cc_f32_f16_param_1];
-; CHECK-NOF16-NEXT:    ld.param.b64 %rd1, [test_select_cc_f32_f16_param_0];
-; CHECK-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r3, %rs1;
-; CHECK-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r4, %rs3;
-; CHECK-NOF16-NEXT:    setp.neu.f32 %p1, %r4, %r3;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r5, %rs2;
-; CHECK-NOF16-NEXT:    cvt.f32.f16 %r6, %rs4;
-; CHECK-NOF16-NEXT:    setp.neu.f32 %p2, %r6, %r5;
-; CHECK-NOF16-NEXT:    mov.b64 {%r7, %r8}, %rd2;
-; CHECK-NOF16-NEXT:    mov.b64 {%r9, %r10}, %rd1;
-; CHECK-NOF16-NEXT:    selp.f32 %r11, %r10, %r8, %p2;
-; CHECK-NOF16-NEXT:    selp.f32 %r12, %r9, %r7, %p1;
+; CHECK-NOF16-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_cc_f32_f16_param_1];
+; CHECK-NOF16-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f32_f16_param_0];
+; CHECK-NOF16-NEXT:    ld.param.b32 %r6, [test_select_cc_f32_f16_param_3];
+; CHECK-NOF16-NEXT:    ld.param.b32 %r5, [test_select_cc_f32_f16_param_2];
+; CHECK-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r6;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r7, %rs1;
+; CHECK-NOF16-NEXT:    mov.b32 {%rs3, %rs4}, %r5;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r8, %rs3;
+; CHECK-NOF16-NEXT:    setp.neu.f32 %p1, %r8, %r7;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r9, %rs2;
+; CHECK-NOF16-NEXT:    cvt.f32.f16 %r10, %rs4;
+; CHECK-NOF16-NEXT:    setp.neu.f32 %p2, %r10, %r9;
+; CHECK-NOF16-NEXT:    selp.f32 %r11, %r2, %r4, %p2;
+; CHECK-NOF16-NEXT:    selp.f32 %r12, %r1, %r3, %p1;
 ; CHECK-NOF16-NEXT:    st.param.v2.b32 [func_retval0], {%r12, %r11};
 ; CHECK-NOF16-NEXT:    ret;
                                            <2 x half> %c, <2 x half> %d) #0 {
@@ -649,17 +643,14 @@ define <2 x half> @test_select_cc_f16_f32(<2 x half> %a, <2 x half> %b,
 ; CHECK-NEXT:    .reg .pred %p<3>;
 ; CHECK-NEXT:    .reg .b16 %rs<7>;
 ; CHECK-NEXT:    .reg .b32 %r<7>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_select_cc_f16_f32_param_3];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_select_cc_f16_f32_param_2];
+; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_select_cc_f16_f32_param_3];
+; CHECK-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_cc_f16_f32_param_2];
 ; CHECK-NEXT:    ld.param.b32 %r2, [test_select_cc_f16_f32_param_1];
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_select_cc_f16_f32_param_0];
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-NEXT:    setp.neu.f32 %p1, %r5, %r3;
-; CHECK-NEXT:    setp.neu.f32 %p2, %r6, %r4;
+; CHECK-NEXT:    setp.neu.f32 %p1, %r3, %r5;
+; CHECK-NEXT:    setp.neu.f32 %p2, %r4, %r6;
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
 ; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
 ; CHECK-NEXT:    selp.b16 %rs5, %rs4, %rs2, %p2;
@@ -1501,11 +1492,9 @@ define <2 x half> @test_fptrunc_2xfloat(<2 x float> %a) #0 {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fptrunc_2xfloat_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fptrunc_2xfloat_param_0];
 ; CHECK-NEXT:    cvt.rn.f16.f32 %rs1, %r2;
 ; CHECK-NEXT:    cvt.rn.f16.f32 %rs2, %r1;
 ; CHECK-NEXT:    mov.b32 %r3, {%rs2, %rs1};
@@ -1674,7 +1663,7 @@ define <2 x half> @test_sqrt(<2 x half> %a) #0 {
 ;  ret <2 x half> %r
 ;}
 
-define <2 x half> @test_sin(<2 x half> %a) #0 #1 {
+define <2 x half> @test_sin(<2 x half> %a) #0 {
 ; CHECK-LABEL: test_sin(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
@@ -1692,11 +1681,11 @@ define <2 x half> @test_sin(<2 x half> %a) #0 #1 {
 ; CHECK-NEXT:    mov.b32 %r6, {%rs4, %rs3};
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
-  %r = call <2 x half> @llvm.sin.f16(<2 x half> %a)
+  %r = call afn <2 x half> @llvm.sin.f16(<2 x half> %a)
   ret <2 x half> %r
 }
 
-define <2 x half> @test_cos(<2 x half> %a) #0 #1 {
+define <2 x half> @test_cos(<2 x half> %a) #0 {
 ; CHECK-LABEL: test_cos(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
@@ -1714,7 +1703,7 @@ define <2 x half> @test_cos(<2 x half> %a) #0 #1 {
 ; CHECK-NEXT:    mov.b32 %r6, {%rs4, %rs3};
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
-  %r = call <2 x half> @llvm.cos.f16(<2 x half> %a)
+  %r = call afn <2 x half> @llvm.cos.f16(<2 x half> %a)
   ret <2 x half> %r
 }
 
@@ -1928,12 +1917,10 @@ define <2 x half> @test_copysign_f32(<2 x half> %a, <2 x float> %b) #0 {
 ; CHECK-F16:       {
 ; CHECK-F16-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-F16-NEXT:    .reg .b32 %r<8>;
-; CHECK-F16-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-F16-EMPTY:
 ; CHECK-F16-NEXT:  // %bb.0:
-; CHECK-F16-NEXT:    ld.param.b64 %rd1, [test_copysign_f32_param_1];
+; CHECK-F16-NEXT:    ld.param.v2.b32 {%r2, %r3}, [test_copysign_f32_param_1];
 ; CHECK-F16-NEXT:    ld.param.b32 %r1, [test_copysign_f32_param_0];
-; CHECK-F16-NEXT:    mov.b64 {%r2, %r3}, %rd1;
 ; CHECK-F16-NEXT:    cvt.rn.f16.f32 %rs1, %r3;
 ; CHECK-F16-NEXT:    cvt.rn.f16.f32 %rs2, %r2;
 ; CHECK-F16-NEXT:    mov.b32 %r4, {%rs2, %rs1};
@@ -1947,21 +1934,19 @@ define <2 x half> @test_copysign_f32(<2 x half> %a, <2 x float> %b) #0 {
 ; CHECK-NOF16:       {
 ; CHECK-NOF16-NEXT:    .reg .b16 %rs<9>;
 ; CHECK-NOF16-NEXT:    .reg .b32 %r<6>;
-; CHECK-NOF16-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-NOF16-EMPTY:
 ; CHECK-NOF16-NEXT:  // %bb.0:
-; CHECK-NOF16-NEXT:    ld.param.b64 %rd1, [test_copysign_f32_param_1];
+; CHECK-NOF16-NEXT:    ld.param.v2.b32 {%r2, %r3}, [test_copysign_f32_param_1];
 ; CHECK-NOF16-NEXT:    ld.param.b32 %r1, [test_copysign_f32_param_0];
-; CHECK-NOF16-NEXT:    mov.b64 {%r2, %r3}, %rd1;
-; CHECK-NOF16-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
-; CHECK-NOF16-NEXT:    and.b16 %rs3, %rs2, 32767;
 ; CHECK-NOF16-NEXT:    and.b32 %r4, %r3, -2147483648;
-; CHECK-NOF16-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs4}, %r4; }
-; CHECK-NOF16-NEXT:    or.b16 %rs5, %rs3, %rs4;
-; CHECK-NOF16-NEXT:    and.b16 %rs6, %rs1, 32767;
+; CHECK-NOF16-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r4; }
+; CHECK-NOF16-NEXT:    mov.b32 {%rs2, %rs3}, %r1;
+; CHECK-NOF16-NEXT:    and.b16 %rs4, %rs3, 32767;
+; CHECK-NOF16-NEXT:    or.b16 %rs5, %rs4, %rs1;
 ; CHECK-NOF16-NEXT:    and.b32 %r5, %r2, -2147483648;
-; CHECK-NOF16-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r5; }
-; CHECK-NOF16-NEXT:    or.b16 %rs8, %rs6, %rs7;
+; CHECK-NOF16-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs6}, %r5; }
+; CHECK-NOF16-NEXT:    and.b16 %rs7, %rs2, 32767;
+; CHECK-NOF16-NEXT:    or.b16 %rs8, %rs7, %rs6;
 ; CHECK-NOF16-NEXT:    st.param.v2.b16 [func_retval0], {%rs8, %rs5};
 ; CHECK-NOF16-NEXT:    ret;
   %tb = fptrunc <2 x float> %b to <2 x half>
@@ -2330,4 +2315,3 @@ define void @test_store_2xhalf(ptr %p1, ptr %p2, <2 x half> %v) {
 
 
 attributes #0 = { nounwind }
-attributes #1 = { "unsafe-fp-math" = "true" }
diff --git a/llvm/test/CodeGen/NVPTX/f32x2-instructions.ll b/llvm/test/CodeGen/NVPTX/f32x2-instructions.ll
index b84a0ec7155e2..7ca16f702d8f3 100644
--- a/llvm/test/CodeGen/NVPTX/f32x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/f32x2-instructions.ll
@@ -30,12 +30,10 @@ define <2 x float> @test_ret_const() #0 {
 define float @test_extract_0(<2 x float> %a) #0 {
 ; CHECK-NOF32X2-LABEL: test_extract_0(
 ; CHECK-NOF32X2:       {
-; CHECK-NOF32X2-NEXT:    .reg .b32 %r<2>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_extract_0_param_0];
-; CHECK-NOF32X2-NEXT:    { .reg .b32 tmp; mov.b64 {%r1, tmp}, %rd1; }
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_extract_0_param_0];
 ; CHECK-NOF32X2-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -56,13 +54,11 @@ define float @test_extract_0(<2 x float> %a) #0 {
 define float @test_extract_1(<2 x float> %a) #0 {
 ; CHECK-NOF32X2-LABEL: test_extract_1(
 ; CHECK-NOF32X2:       {
-; CHECK-NOF32X2-NEXT:    .reg .b32 %r<2>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_extract_1_param_0];
-; CHECK-NOF32X2-NEXT:    { .reg .b32 tmp; mov.b64 {tmp, %r1}, %rd1; }
-; CHECK-NOF32X2-NEXT:    st.param.b32 [func_retval0], %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_extract_1_param_0];
+; CHECK-NOF32X2-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
 ; CHECK-F32X2-LABEL: test_extract_1(
@@ -79,27 +75,57 @@ define float @test_extract_1(<2 x float> %a) #0 {
   ret float %e
 }
 
-; NOTE: disabled as -O3 miscompiles this into pointer arithmetic on
-; test_extract_i_param_0 where the symbol's address is not taken first (that
-; is, moved to a temporary)
-; define float @test_extract_i(<2 x float> %a, i64 %idx) #0 {
-;   %e = extractelement <2 x float> %a, i64 %idx
-;   ret float %e
-; }
+define float @test_extract_i(<2 x float> %a, i64 %idx) #0 {
+; CHECK-NOF32X2-LABEL: test_extract_i(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .local .align 8 .b8 __local_depot3[8];
+; CHECK-NOF32X2-NEXT:    .reg .b64 %SP;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %SPL;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<4>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    mov.b64 %SPL, __local_depot3;
+; CHECK-NOF32X2-NEXT:    cvta.local.u64 %SP, %SPL;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_extract_i_param_0];
+; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_extract_i_param_1];
+; CHECK-NOF32X2-NEXT:    st.v2.b32 [%SP], {%r1, %r2};
+; CHECK-NOF32X2-NEXT:    and.b64 %rd2, %rd1, 1;
+; CHECK-NOF32X2-NEXT:    shl.b64 %rd3, %rd2, 2;
+; CHECK-NOF32X2-NEXT:    add.u64 %rd4, %SP, 0;
+; CHECK-NOF32X2-NEXT:    or.b64 %rd5, %rd4, %rd3;
+; CHECK-NOF32X2-NEXT:    ld.b32 %r3, [%rd5];
+; CHECK-NOF32X2-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_extract_i(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<2>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<4>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_extract_i_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_extract_i_param_0];
+; CHECK-F32X2-NEXT:    setp.eq.b64 %p1, %rd2, 0;
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    selp.f32 %r3, %r1, %r2, %p1;
+; CHECK-F32X2-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-F32X2-NEXT:    ret;
+  %e = extractelement <2 x float> %a, i64 %idx
+  ret float %e
+}
 
 define <2 x float> @test_fadd(<2 x float> %a, <2 x float> %b) #0 {
 ; CHECK-NOF32X2-LABEL: test_fadd(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_fadd_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fadd_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r5, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r6, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fadd_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fadd_param_0];
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r6, %r1, %r3;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -121,11 +147,9 @@ define <2 x float> @test_fadd_imm_0(<2 x float> %a) #0 {
 ; CHECK-NOF32X2-LABEL: test_fadd_imm_0(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fadd_imm_0_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fadd_imm_0_param_0];
 ; CHECK-NOF32X2-NEXT:    add.rn.f32 %r3, %r2, 0f40000000;
 ; CHECK-NOF32X2-NEXT:    add.rn.f32 %r4, %r1, 0f3F800000;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
@@ -152,11 +176,9 @@ define <2 x float> @test_fadd_imm_1(<2 x float> %a) #0 {
 ; CHECK-NOF32X2-LABEL: test_fadd_imm_1(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fadd_imm_1_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fadd_imm_1_param_0];
 ; CHECK-NOF32X2-NEXT:    add.rn.f32 %r3, %r2, 0f40000000;
 ; CHECK-NOF32X2-NEXT:    add.rn.f32 %r4, %r1, 0f3F800000;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
@@ -183,20 +205,15 @@ define <4 x float> @test_fadd_v4(<4 x float> %a, <4 x float> %b) #0 {
 ; CHECK-NOF32X2-LABEL: test_fadd_v4(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<13>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_fadd_v4_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_fadd_v4_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r5, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r6, %r3, %r1;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r7, %r8}, %rd3;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r9, %r10}, %rd1;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r11, %r10, %r8;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r12, %r9, %r7;
-; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r12, %r11, %r6, %r5};
+; CHECK-NOF32X2-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [test_fadd_v4_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [test_fadd_v4_param_0];
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r9, %r4, %r8;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r10, %r3, %r7;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r11, %r2, %r6;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r12, %r1, %r5;
+; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r12, %r11, %r10, %r9};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
 ; CHECK-F32X2-LABEL: test_fadd_v4(
@@ -218,17 +235,14 @@ define <4 x float> @test_fadd_imm_0_v4(<4 x float> %a) #0 {
 ; CHECK-NOF32X2-LABEL: test_fadd_imm_0_v4(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_fadd_imm_0_v4_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r3, %r2, 0f40800000;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r4, %r1, 0f40400000;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r7, %r6, 0f40000000;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r8, %r5, 0f3F800000;
-; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r4, %r3};
+; CHECK-NOF32X2-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [test_fadd_imm_0_v4_param_0];
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r5, %r4, 0f40800000;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r6, %r3, 0f40400000;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r7, %r2, 0f40000000;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r8, %r1, 0f3F800000;
+; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
 ; CHECK-F32X2-LABEL: test_fadd_imm_0_v4(
@@ -256,17 +270,14 @@ define <4 x float> @test_fadd_imm_1_v4(<4 x float> %a) #0 {
 ; CHECK-NOF32X2-LABEL: test_fadd_imm_1_v4(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_fadd_imm_1_v4_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r3, %r2, 0f40800000;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r4, %r1, 0f40400000;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r7, %r6, 0f40000000;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r8, %r5, 0f3F800000;
-; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r4, %r3};
+; CHECK-NOF32X2-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [test_fadd_imm_1_v4_param_0];
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r5, %r4, 0f40800000;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r6, %r3, 0f40400000;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r7, %r2, 0f40000000;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r8, %r1, 0f3F800000;
+; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
 ; CHECK-F32X2-LABEL: test_fadd_imm_1_v4(
@@ -294,15 +305,12 @@ define <2 x float> @test_fsub(<2 x float> %a, <2 x float> %b) #0 {
 ; CHECK-NOF32X2-LABEL: test_fsub(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_fsub_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fsub_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NOF32X2-NEXT:    sub.rn.f32 %r5, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    sub.rn.f32 %r6, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fsub_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fsub_param_0];
+; CHECK-NOF32X2-NEXT:    sub.rn.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    sub.rn.f32 %r6, %r1, %r3;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -321,18 +329,29 @@ define <2 x float> @test_fsub(<2 x float> %a, <2 x float> %b) #0 {
 }
 
 define <2 x float> @test_fneg(<2 x float> %a) #0 {
-; CHECK-LABEL: test_fneg(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fneg_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    neg.f32 %r3, %r2;
-; CHECK-NEXT:    neg.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fneg(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fneg_param_0];
+; CHECK-NOF32X2-NEXT:    neg.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    neg.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fneg(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fneg_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    neg.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    neg.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = fneg <2 x float> %a
   ret <2 x float> %r
 }
@@ -341,15 +360,12 @@ define <2 x float> @test_fmul(<2 x float> %a, <2 x float> %b) #0 {
 ; CHECK-NOF32X2-LABEL: test_fmul(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_fmul_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fmul_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NOF32X2-NEXT:    mul.rn.f32 %r5, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    mul.rn.f32 %r6, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fmul_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fmul_param_0];
+; CHECK-NOF32X2-NEXT:    mul.rn.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    mul.rn.f32 %r6, %r1, %r3;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -368,50 +384,85 @@ define <2 x float> @test_fmul(<2 x float> %a, <2 x float> %b) #0 {
 }
 
 define <2 x float> @test_fdiv(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fdiv(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<7>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fdiv_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fdiv_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    div.rn.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    div.rn.f32 %r6, %r3, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fdiv(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fdiv_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fdiv_param_0];
+; CHECK-NOF32X2-NEXT:    div.rn.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    div.rn.f32 %r6, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fdiv(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fdiv_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fdiv_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    div.rn.f32 %r5, %r4, %r2;
+; CHECK-F32X2-NEXT:    div.rn.f32 %r6, %r3, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-F32X2-NEXT:    ret;
   %r = fdiv <2 x float> %a, %b
   ret <2 x float> %r
 }
 
 define <2 x float> @test_frem(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_frem(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b32 %r<15>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_frem_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_frem_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    div.rn.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r6, %r5;
-; CHECK-NEXT:    neg.f32 %r7, %r6;
-; CHECK-NEXT:    fma.rn.f32 %r8, %r7, %r2, %r4;
-; CHECK-NEXT:    testp.infinite.f32 %p1, %r2;
-; CHECK-NEXT:    selp.f32 %r9, %r4, %r8, %p1;
-; CHECK-NEXT:    div.rn.f32 %r10, %r3, %r1;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r11, %r10;
-; CHECK-NEXT:    neg.f32 %r12, %r11;
-; CHECK-NEXT:    fma.rn.f32 %r13, %r12, %r1, %r3;
-; CHECK-NEXT:    testp.infinite.f32 %p2, %r1;
-; CHECK-NEXT:    selp.f32 %r14, %r3, %r13, %p2;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r14, %r9};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_frem(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<15>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_frem_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_frem_param_0];
+; CHECK-NOF32X2-NEXT:    div.rn.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.f32.f32 %r6, %r5;
+; CHECK-NOF32X2-NEXT:    neg.f32 %r7, %r6;
+; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r8, %r7, %r4, %r2;
+; CHECK-NOF32X2-NEXT:    testp.infinite.f32 %p1, %r4;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r9, %r2, %r8, %p1;
+; CHECK-NOF32X2-NEXT:    div.rn.f32 %r10, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.f32.f32 %r11, %r10;
+; CHECK-NOF32X2-NEXT:    neg.f32 %r12, %r11;
+; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r13, %r12, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    testp.infinite.f32 %p2, %r3;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r14, %r1, %r13, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r14, %r9};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_frem(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<15>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_frem_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_frem_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    div.rn.f32 %r5, %r4, %r2;
+; CHECK-F32X2-NEXT:    cvt.rzi.f32.f32 %r6, %r5;
+; CHECK-F32X2-NEXT:    neg.f32 %r7, %r6;
+; CHECK-F32X2-NEXT:    fma.rn.f32 %r8, %r7, %r2, %r4;
+; CHECK-F32X2-NEXT:    testp.infinite.f32 %p1, %r2;
+; CHECK-F32X2-NEXT:    selp.f32 %r9, %r4, %r8, %p1;
+; CHECK-F32X2-NEXT:    div.rn.f32 %r10, %r3, %r1;
+; CHECK-F32X2-NEXT:    cvt.rzi.f32.f32 %r11, %r10;
+; CHECK-F32X2-NEXT:    neg.f32 %r12, %r11;
+; CHECK-F32X2-NEXT:    fma.rn.f32 %r13, %r12, %r1, %r3;
+; CHECK-F32X2-NEXT:    testp.infinite.f32 %p2, %r1;
+; CHECK-F32X2-NEXT:    selp.f32 %r14, %r3, %r13, %p2;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r14, %r9};
+; CHECK-F32X2-NEXT:    ret;
   %r = frem <2 x float> %a, %b
   ret <2 x float> %r
 }
@@ -420,15 +471,12 @@ define <2 x float> @test_fadd_ftz(<2 x float> %a, <2 x float> %b) #2 {
 ; CHECK-NOF32X2-LABEL: test_fadd_ftz(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_fadd_ftz_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fadd_ftz_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r5, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r6, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fadd_ftz_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fadd_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r6, %r1, %r3;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -450,11 +498,9 @@ define <2 x float> @test_fadd_imm_0_ftz(<2 x float> %a) #2 {
 ; CHECK-NOF32X2-LABEL: test_fadd_imm_0_ftz(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fadd_imm_0_ftz_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fadd_imm_0_ftz_param_0];
 ; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r3, %r2, 0f40000000;
 ; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r4, %r1, 0f3F800000;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
@@ -481,11 +527,9 @@ define <2 x float> @test_fadd_imm_1_ftz(<2 x float> %a) #2 {
 ; CHECK-NOF32X2-LABEL: test_fadd_imm_1_ftz(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fadd_imm_1_ftz_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fadd_imm_1_ftz_param_0];
 ; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r3, %r2, 0f40000000;
 ; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r4, %r1, 0f3F800000;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
@@ -512,20 +556,15 @@ define <4 x float> @test_fadd_v4_ftz(<4 x float> %a, <4 x float> %b) #2 {
 ; CHECK-NOF32X2-LABEL: test_fadd_v4_ftz(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<13>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_fadd_v4_ftz_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_fadd_v4_ftz_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r5, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r6, %r3, %r1;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r7, %r8}, %rd3;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r9, %r10}, %rd1;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r11, %r10, %r8;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r12, %r9, %r7;
-; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r12, %r11, %r6, %r5};
+; CHECK-NOF32X2-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [test_fadd_v4_ftz_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [test_fadd_v4_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r9, %r4, %r8;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r10, %r3, %r7;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r11, %r2, %r6;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r12, %r1, %r5;
+; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r12, %r11, %r10, %r9};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
 ; CHECK-F32X2-LABEL: test_fadd_v4_ftz(
@@ -547,17 +586,14 @@ define <4 x float> @test_fadd_imm_0_v4_ftz(<4 x float> %a) #2 {
 ; CHECK-NOF32X2-LABEL: test_fadd_imm_0_v4_ftz(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_fadd_imm_0_v4_ftz_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r3, %r2, 0f40800000;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r4, %r1, 0f40400000;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r7, %r6, 0f40000000;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r8, %r5, 0f3F800000;
-; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r4, %r3};
+; CHECK-NOF32X2-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [test_fadd_imm_0_v4_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r5, %r4, 0f40800000;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r6, %r3, 0f40400000;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r7, %r2, 0f40000000;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r8, %r1, 0f3F800000;
+; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
 ; CHECK-F32X2-LABEL: test_fadd_imm_0_v4_ftz(
@@ -585,17 +621,14 @@ define <4 x float> @test_fadd_imm_1_v4_ftz(<4 x float> %a) #2 {
 ; CHECK-NOF32X2-LABEL: test_fadd_imm_1_v4_ftz(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_fadd_imm_1_v4_ftz_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r3, %r2, 0f40800000;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r4, %r1, 0f40400000;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r7, %r6, 0f40000000;
-; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r8, %r5, 0f3F800000;
-; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r4, %r3};
+; CHECK-NOF32X2-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [test_fadd_imm_1_v4_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r5, %r4, 0f40800000;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r6, %r3, 0f40400000;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r7, %r2, 0f40000000;
+; CHECK-NOF32X2-NEXT:    add.rn.ftz.f32 %r8, %r1, 0f3F800000;
+; CHECK-NOF32X2-NEXT:    st.param.v4.b32 [func_retval0], {%r8, %r7, %r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
 ; CHECK-F32X2-LABEL: test_fadd_imm_1_v4_ftz(
@@ -623,15 +656,12 @@ define <2 x float> @test_fsub_ftz(<2 x float> %a, <2 x float> %b) #2 {
 ; CHECK-NOF32X2-LABEL: test_fsub_ftz(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_fsub_ftz_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fsub_ftz_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NOF32X2-NEXT:    sub.rn.ftz.f32 %r5, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    sub.rn.ftz.f32 %r6, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fsub_ftz_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fsub_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    sub.rn.ftz.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    sub.rn.ftz.f32 %r6, %r1, %r3;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -650,18 +680,29 @@ define <2 x float> @test_fsub_ftz(<2 x float> %a, <2 x float> %b) #2 {
 }
 
 define <2 x float> @test_fneg_ftz(<2 x float> %a) #2 {
-; CHECK-LABEL: test_fneg_ftz(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fneg_ftz_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    neg.ftz.f32 %r3, %r2;
-; CHECK-NEXT:    neg.ftz.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fneg_ftz(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fneg_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    neg.ftz.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    neg.ftz.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fneg_ftz(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fneg_ftz_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    neg.ftz.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    neg.ftz.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = fneg <2 x float> %a
   ret <2 x float> %r
 }
@@ -670,15 +711,12 @@ define <2 x float> @test_fmul_ftz(<2 x float> %a, <2 x float> %b) #2 {
 ; CHECK-NOF32X2-LABEL: test_fmul_ftz(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_fmul_ftz_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fmul_ftz_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NOF32X2-NEXT:    mul.rn.ftz.f32 %r5, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    mul.rn.ftz.f32 %r6, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fmul_ftz_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fmul_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    mul.rn.ftz.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    mul.rn.ftz.f32 %r6, %r1, %r3;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -700,17 +738,13 @@ define <2 x float> @test_fma_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c)
 ; CHECK-NOF32X2-LABEL: test_fma_ftz(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd3, [test_fma_ftz_param_2];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_fma_ftz_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fma_ftz_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-NOF32X2-NEXT:    fma.rn.ftz.f32 %r7, %r6, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    fma.rn.ftz.f32 %r8, %r5, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_fma_ftz_param_2];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fma_ftz_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fma_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    fma.rn.ftz.f32 %r7, %r2, %r4, %r6;
+; CHECK-NOF32X2-NEXT:    fma.rn.ftz.f32 %r8, %r1, %r3, %r5;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r8, %r7};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -730,65 +764,112 @@ define <2 x float> @test_fma_ftz(<2 x float> %a, <2 x float> %b, <2 x float> %c)
 }
 
 define <2 x float> @test_fdiv_ftz(<2 x float> %a, <2 x float> %b) #2 {
-; CHECK-LABEL: test_fdiv_ftz(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<7>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fdiv_ftz_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fdiv_ftz_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    div.rn.ftz.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    div.rn.ftz.f32 %r6, %r3, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fdiv_ftz(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fdiv_ftz_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fdiv_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    div.rn.ftz.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    div.rn.ftz.f32 %r6, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fdiv_ftz(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fdiv_ftz_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fdiv_ftz_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    div.rn.ftz.f32 %r5, %r4, %r2;
+; CHECK-F32X2-NEXT:    div.rn.ftz.f32 %r6, %r3, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-F32X2-NEXT:    ret;
   %r = fdiv <2 x float> %a, %b
   ret <2 x float> %r
 }
 
 define <2 x float> @test_frem_ftz(<2 x float> %a, <2 x float> %b) #2 {
-; CHECK-LABEL: test_frem_ftz(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b32 %r<15>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_frem_ftz_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_frem_ftz_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    div.rn.ftz.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    cvt.rzi.ftz.f32.f32 %r6, %r5;
-; CHECK-NEXT:    neg.ftz.f32 %r7, %r6;
-; CHECK-NEXT:    fma.rn.ftz.f32 %r8, %r7, %r2, %r4;
-; CHECK-NEXT:    testp.infinite.f32 %p1, %r2;
-; CHECK-NEXT:    selp.f32 %r9, %r4, %r8, %p1;
-; CHECK-NEXT:    div.rn.ftz.f32 %r10, %r3, %r1;
-; CHECK-NEXT:    cvt.rzi.ftz.f32.f32 %r11, %r10;
-; CHECK-NEXT:    neg.ftz.f32 %r12, %r11;
-; CHECK-NEXT:    fma.rn.ftz.f32 %r13, %r12, %r1, %r3;
-; CHECK-NEXT:    testp.infinite.f32 %p2, %r1;
-; CHECK-NEXT:    selp.f32 %r14, %r3, %r13, %p2;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r14, %r9};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_frem_ftz(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<15>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_frem_ftz_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_frem_ftz_param_0];
+; CHECK-NOF32X2-NEXT:    div.rn.ftz.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.ftz.f32.f32 %r6, %r5;
+; CHECK-NOF32X2-NEXT:    neg.ftz.f32 %r7, %r6;
+; CHECK-NOF32X2-NEXT:    fma.rn.ftz.f32 %r8, %r7, %r4, %r2;
+; CHECK-NOF32X2-NEXT:    testp.infinite.f32 %p1, %r4;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r9, %r2, %r8, %p1;
+; CHECK-NOF32X2-NEXT:    div.rn.ftz.f32 %r10, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.ftz.f32.f32 %r11, %r10;
+; CHECK-NOF32X2-NEXT:    neg.ftz.f32 %r12, %r11;
+; CHECK-NOF32X2-NEXT:    fma.rn.ftz.f32 %r13, %r12, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    testp.infinite.f32 %p2, %r3;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r14, %r1, %r13, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r14, %r9};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_frem_ftz(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<15>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_frem_ftz_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_frem_ftz_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    div.rn.ftz.f32 %r5, %r4, %r2;
+; CHECK-F32X2-NEXT:    cvt.rzi.ftz.f32.f32 %r6, %r5;
+; CHECK-F32X2-NEXT:    neg.ftz.f32 %r7, %r6;
+; CHECK-F32X2-NEXT:    fma.rn.ftz.f32 %r8, %r7, %r2, %r4;
+; CHECK-F32X2-NEXT:    testp.infinite.f32 %p1, %r2;
+; CHECK-F32X2-NEXT:    selp.f32 %r9, %r4, %r8, %p1;
+; CHECK-F32X2-NEXT:    div.rn.ftz.f32 %r10, %r3, %r1;
+; CHECK-F32X2-NEXT:    cvt.rzi.ftz.f32.f32 %r11, %r10;
+; CHECK-F32X2-NEXT:    neg.ftz.f32 %r12, %r11;
+; CHECK-F32X2-NEXT:    fma.rn.ftz.f32 %r13, %r12, %r1, %r3;
+; CHECK-F32X2-NEXT:    testp.infinite.f32 %p2, %r1;
+; CHECK-F32X2-NEXT:    selp.f32 %r14, %r3, %r13, %p2;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r14, %r9};
+; CHECK-F32X2-NEXT:    ret;
   %r = frem <2 x float> %a, %b
   ret <2 x float> %r
 }
 
 define void @test_ldst_v2f32(ptr %a, ptr %b) #0 {
-; CHECK-LABEL: test_ldst_v2f32(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_ldst_v2f32_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_ldst_v2f32_param_0];
-; CHECK-NEXT:    ld.b64 %rd3, [%rd1];
-; CHECK-NEXT:    st.b64 [%rd2], %rd3;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_ldst_v2f32(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_ldst_v2f32_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_ldst_v2f32_param_0];
+; CHECK-NOF32X2-NEXT:    ld.v2.b32 {%r1, %r2}, [%rd1];
+; CHECK-NOF32X2-NEXT:    st.v2.b32 [%rd2], {%r1, %r2};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_ldst_v2f32(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<4>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_ldst_v2f32_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_ldst_v2f32_param_0];
+; CHECK-F32X2-NEXT:    ld.b64 %rd3, [%rd1];
+; CHECK-F32X2-NEXT:    st.b64 [%rd2], %rd3;
+; CHECK-F32X2-NEXT:    ret;
   %t1 = load <2 x float>, ptr %a
   store <2 x float> %t1, ptr %b, align 32
   ret void
@@ -814,34 +895,60 @@ define void @test_ldst_v3f32(ptr %a, ptr %b) #0 {
 }
 
 define void @test_ldst_v4f32(ptr %a, ptr %b) #0 {
-; CHECK-LABEL: test_ldst_v4f32(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_ldst_v4f32_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_ldst_v4f32_param_0];
-; CHECK-NEXT:    ld.v2.b64 {%rd3, %rd4}, [%rd1];
-; CHECK-NEXT:    st.v2.b64 [%rd2], {%rd3, %rd4};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_ldst_v4f32(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_ldst_v4f32_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_ldst_v4f32_param_0];
+; CHECK-NOF32X2-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; CHECK-NOF32X2-NEXT:    st.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_ldst_v4f32(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<5>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_ldst_v4f32_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_ldst_v4f32_param_0];
+; CHECK-F32X2-NEXT:    ld.v2.b64 {%rd3, %rd4}, [%rd1];
+; CHECK-F32X2-NEXT:    st.v2.b64 [%rd2], {%rd3, %rd4};
+; CHECK-F32X2-NEXT:    ret;
   %t1 = load <4 x float>, ptr %a
   store <4 x float> %t1, ptr %b, align 32
   ret void
 }
 
 define void @test_ldst_v8f32(ptr %a, ptr %b) #0 {
-; CHECK-LABEL: test_ldst_v8f32(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<7>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_ldst_v8f32_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_ldst_v8f32_param_0];
-; CHECK-NEXT:    ld.v2.b64 {%rd3, %rd4}, [%rd1];
-; CHECK-NEXT:    ld.v2.b64 {%rd5, %rd6}, [%rd1+16];
-; CHECK-NEXT:    st.v2.b64 [%rd2+16], {%rd5, %rd6};
-; CHECK-NEXT:    st.v2.b64 [%rd2], {%rd3, %rd4};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_ldst_v8f32(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_ldst_v8f32_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_ldst_v8f32_param_0];
+; CHECK-NOF32X2-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; CHECK-NOF32X2-NEXT:    ld.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; CHECK-NOF32X2-NEXT:    st.v4.b32 [%rd2+16], {%r5, %r6, %r7, %r8};
+; CHECK-NOF32X2-NEXT:    st.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_ldst_v8f32(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<7>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_ldst_v8f32_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_ldst_v8f32_param_0];
+; CHECK-F32X2-NEXT:    ld.v2.b64 {%rd3, %rd4}, [%rd1];
+; CHECK-F32X2-NEXT:    ld.v2.b64 {%rd5, %rd6}, [%rd1+16];
+; CHECK-F32X2-NEXT:    st.v2.b64 [%rd2+16], {%rd5, %rd6};
+; CHECK-F32X2-NEXT:    st.v2.b64 [%rd2], {%rd3, %rd4};
+; CHECK-F32X2-NEXT:    ret;
   %t1 = load <8 x float>, ptr %a
   store <8 x float> %t1, ptr %b, align 32
   ret void
@@ -850,571 +957,982 @@ define void @test_ldst_v8f32(ptr %a, ptr %b) #0 {
 declare <2 x float> @test_callee(<2 x float> %a, <2 x float> %b) #0
 
 define <2 x float> @test_call(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_call(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_call_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_call_param_0];
-; CHECK-NEXT:    { // callseq 0, 0
-; CHECK-NEXT:    .param .align 8 .b8 param0[8];
-; CHECK-NEXT:    .param .align 8 .b8 param1[8];
-; CHECK-NEXT:    .param .align 8 .b8 retval0[8];
-; CHECK-NEXT:    st.param.b64 [param1], %rd2;
-; CHECK-NEXT:    st.param.b64 [param0], %rd1;
-; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
-; CHECK-NEXT:    ld.param.b64 %rd3, [retval0];
-; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd3;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_call(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_call_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_call_param_0];
+; CHECK-NOF32X2-NEXT:    { // callseq 0, 0
+; CHECK-NOF32X2-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NOF32X2-NEXT:    .param .align 8 .b8 param1[8];
+; CHECK-NOF32X2-NEXT:    .param .align 8 .b8 retval0[8];
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [param1], {%r3, %r4};
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [param0], {%r1, %r2};
+; CHECK-NOF32X2-NEXT:    call.uni (retval0), test_callee, (param0, param1);
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r5, %r6}, [retval0];
+; CHECK-NOF32X2-NEXT:    } // callseq 0
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_call(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<4>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_call_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_call_param_0];
+; CHECK-F32X2-NEXT:    { // callseq 0, 0
+; CHECK-F32X2-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-F32X2-NEXT:    .param .align 8 .b8 param1[8];
+; CHECK-F32X2-NEXT:    .param .align 8 .b8 retval0[8];
+; CHECK-F32X2-NEXT:    st.param.b64 [param1], %rd2;
+; CHECK-F32X2-NEXT:    st.param.b64 [param0], %rd1;
+; CHECK-F32X2-NEXT:    call.uni (retval0), test_callee, (param0, param1);
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd3, [retval0];
+; CHECK-F32X2-NEXT:    } // callseq 0
+; CHECK-F32X2-NEXT:    st.param.b64 [func_retval0], %rd3;
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @test_callee(<2 x float> %a, <2 x float> %b)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_call_flipped(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_call_flipped(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_call_flipped_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_call_flipped_param_0];
-; CHECK-NEXT:    { // callseq 1, 0
-; CHECK-NEXT:    .param .align 8 .b8 param0[8];
-; CHECK-NEXT:    .param .align 8 .b8 param1[8];
-; CHECK-NEXT:    .param .align 8 .b8 retval0[8];
-; CHECK-NEXT:    st.param.b64 [param1], %rd1;
-; CHECK-NEXT:    st.param.b64 [param0], %rd2;
-; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
-; CHECK-NEXT:    ld.param.b64 %rd3, [retval0];
-; CHECK-NEXT:    } // callseq 1
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd3;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_call_flipped(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_call_flipped_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_call_flipped_param_0];
+; CHECK-NOF32X2-NEXT:    { // callseq 1, 0
+; CHECK-NOF32X2-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NOF32X2-NEXT:    .param .align 8 .b8 param1[8];
+; CHECK-NOF32X2-NEXT:    .param .align 8 .b8 retval0[8];
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [param1], {%r1, %r2};
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [param0], {%r3, %r4};
+; CHECK-NOF32X2-NEXT:    call.uni (retval0), test_callee, (param0, param1);
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r5, %r6}, [retval0];
+; CHECK-NOF32X2-NEXT:    } // callseq 1
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_call_flipped(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<4>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_call_flipped_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_call_flipped_param_0];
+; CHECK-F32X2-NEXT:    { // callseq 1, 0
+; CHECK-F32X2-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-F32X2-NEXT:    .param .align 8 .b8 param1[8];
+; CHECK-F32X2-NEXT:    .param .align 8 .b8 retval0[8];
+; CHECK-F32X2-NEXT:    st.param.b64 [param1], %rd1;
+; CHECK-F32X2-NEXT:    st.param.b64 [param0], %rd2;
+; CHECK-F32X2-NEXT:    call.uni (retval0), test_callee, (param0, param1);
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd3, [retval0];
+; CHECK-F32X2-NEXT:    } // callseq 1
+; CHECK-F32X2-NEXT:    st.param.b64 [func_retval0], %rd3;
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @test_callee(<2 x float> %b, <2 x float> %a)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_tailcall_flipped(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_tailcall_flipped(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_tailcall_flipped_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_tailcall_flipped_param_0];
-; CHECK-NEXT:    { // callseq 2, 0
-; CHECK-NEXT:    .param .align 8 .b8 param0[8];
-; CHECK-NEXT:    .param .align 8 .b8 param1[8];
-; CHECK-NEXT:    .param .align 8 .b8 retval0[8];
-; CHECK-NEXT:    st.param.b64 [param1], %rd1;
-; CHECK-NEXT:    st.param.b64 [param0], %rd2;
-; CHECK-NEXT:    call.uni (retval0), test_callee, (param0, param1);
-; CHECK-NEXT:    ld.param.b64 %rd3, [retval0];
-; CHECK-NEXT:    } // callseq 2
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd3;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_tailcall_flipped(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_tailcall_flipped_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_tailcall_flipped_param_0];
+; CHECK-NOF32X2-NEXT:    { // callseq 2, 0
+; CHECK-NOF32X2-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-NOF32X2-NEXT:    .param .align 8 .b8 param1[8];
+; CHECK-NOF32X2-NEXT:    .param .align 8 .b8 retval0[8];
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [param1], {%r1, %r2};
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [param0], {%r3, %r4};
+; CHECK-NOF32X2-NEXT:    call.uni (retval0), test_callee, (param0, param1);
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r5, %r6}, [retval0];
+; CHECK-NOF32X2-NEXT:    } // callseq 2
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r5, %r6};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_tailcall_flipped(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<4>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_tailcall_flipped_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_tailcall_flipped_param_0];
+; CHECK-F32X2-NEXT:    { // callseq 2, 0
+; CHECK-F32X2-NEXT:    .param .align 8 .b8 param0[8];
+; CHECK-F32X2-NEXT:    .param .align 8 .b8 param1[8];
+; CHECK-F32X2-NEXT:    .param .align 8 .b8 retval0[8];
+; CHECK-F32X2-NEXT:    st.param.b64 [param1], %rd1;
+; CHECK-F32X2-NEXT:    st.param.b64 [param0], %rd2;
+; CHECK-F32X2-NEXT:    call.uni (retval0), test_callee, (param0, param1);
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd3, [retval0];
+; CHECK-F32X2-NEXT:    } // callseq 2
+; CHECK-F32X2-NEXT:    st.param.b64 [func_retval0], %rd3;
+; CHECK-F32X2-NEXT:    ret;
   %r = tail call <2 x float> @test_callee(<2 x float> %b, <2 x float> %a)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_select(<2 x float> %a, <2 x float> %b, i1 zeroext %c) #0 {
-; CHECK-LABEL: test_select(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b8 %rs1, [test_select_param_2];
-; CHECK-NEXT:    and.b16 %rs2, %rs1, 1;
-; CHECK-NEXT:    setp.ne.b16 %p1, %rs2, 0;
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_select_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_select_param_0];
-; CHECK-NEXT:    selp.b64 %rd3, %rd1, %rd2, %p1;
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd3;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_select(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<2>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.b8 %rs1, [test_select_param_2];
+; CHECK-NOF32X2-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-NOF32X2-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_param_0];
+; CHECK-NOF32X2-NEXT:    selp.f32 %r5, %r2, %r4, %p1;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r6, %r1, %r3, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_select(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<2>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<4>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b8 %rs1, [test_select_param_2];
+; CHECK-F32X2-NEXT:    and.b16 %rs2, %rs1, 1;
+; CHECK-F32X2-NEXT:    setp.ne.b16 %p1, %rs2, 0;
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_select_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_select_param_0];
+; CHECK-F32X2-NEXT:    selp.b64 %rd3, %rd1, %rd2, %p1;
+; CHECK-F32X2-NEXT:    st.param.b64 [func_retval0], %rd3;
+; CHECK-F32X2-NEXT:    ret;
   %r = select i1 %c, <2 x float> %a, <2 x float> %b
   ret <2 x float> %r
 }
 
 define <2 x float> @test_select_cc(<2 x float> %a, <2 x float> %b, <2 x float> %c, <2 x float> %d) #0 {
-; CHECK-LABEL: test_select_cc(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b32 %r<11>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd4, [test_select_cc_param_3];
-; CHECK-NEXT:    ld.param.b64 %rd3, [test_select_cc_param_2];
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_select_cc_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_select_cc_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd3;
-; CHECK-NEXT:    setp.neu.f32 %p1, %r3, %r1;
-; CHECK-NEXT:    setp.neu.f32 %p2, %r4, %r2;
-; CHECK-NEXT:    mov.b64 {%r5, %r6}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r7, %r8}, %rd1;
-; CHECK-NEXT:    selp.f32 %r9, %r8, %r6, %p2;
-; CHECK-NEXT:    selp.f32 %r10, %r7, %r5, %p1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r10, %r9};
-; CHECK-NEXT:    ret;
-  %cc = fcmp une <2 x float> %c, %d
-  %r = select <2 x i1> %cc, <2 x float> %a, <2 x float> %b
-  ret <2 x float> %r
+; CHECK-NOF32X2-LABEL: test_select_cc(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<11>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r7, %r8}, [test_select_cc_param_3];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_select_cc_param_2];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_cc_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_param_0];
+; CHECK-NOF32X2-NEXT:    setp.neu.f32 %p1, %r5, %r7;
+; CHECK-NOF32X2-NEXT:    setp.neu.f32 %p2, %r6, %r8;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r9, %r2, %r4, %p2;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r10, %r1, %r3, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r10, %r9};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_select_cc(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<11>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<5>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd4, [test_select_cc_param_3];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd3, [test_select_cc_param_2];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_select_cc_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_select_cc_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd3;
+; CHECK-F32X2-NEXT:    setp.neu.f32 %p1, %r3, %r1;
+; CHECK-F32X2-NEXT:    setp.neu.f32 %p2, %r4, %r2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r5, %r6}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r7, %r8}, %rd1;
+; CHECK-F32X2-NEXT:    selp.f32 %r9, %r8, %r6, %p2;
+; CHECK-F32X2-NEXT:    selp.f32 %r10, %r7, %r5, %p1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r10, %r9};
+; CHECK-F32X2-NEXT:    ret;
+  %cc = fcmp une <2 x float> %c, %d
+  %r = select <2 x i1> %cc, <2 x float> %a, <2 x float> %b
+  ret <2 x float> %r
 }
 
 define <2 x double> @test_select_cc_f64_f32(<2 x double> %a, <2 x double> %b, <2 x float> %c, <2 x float> %d) #0 {
-; CHECK-LABEL: test_select_cc_f64_f32(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<9>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_select_cc_f64_f32_param_1];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_select_cc_f64_f32_param_0];
-; CHECK-NEXT:    ld.param.b64 %rd6, [test_select_cc_f64_f32_param_3];
-; CHECK-NEXT:    ld.param.b64 %rd5, [test_select_cc_f64_f32_param_2];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd6;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd5;
-; CHECK-NEXT:    setp.neu.f32 %p1, %r3, %r1;
-; CHECK-NEXT:    setp.neu.f32 %p2, %r4, %r2;
-; CHECK-NEXT:    selp.f64 %rd7, %rd2, %rd4, %p2;
-; CHECK-NEXT:    selp.f64 %rd8, %rd1, %rd3, %p1;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd8, %rd7};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_select_cc_f64_f32(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_cc_f64_f32_param_3];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f64_f32_param_2];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_select_cc_f64_f32_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_select_cc_f64_f32_param_0];
+; CHECK-NOF32X2-NEXT:    setp.neu.f32 %p1, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    setp.neu.f32 %p2, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    selp.f64 %rd5, %rd2, %rd4, %p2;
+; CHECK-NOF32X2-NEXT:    selp.f64 %rd6, %rd1, %rd3, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd6, %rd5};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_select_cc_f64_f32(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<9>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_select_cc_f64_f32_param_1];
+; CHECK-F32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_select_cc_f64_f32_param_0];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd6, [test_select_cc_f64_f32_param_3];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd5, [test_select_cc_f64_f32_param_2];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd6;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd5;
+; CHECK-F32X2-NEXT:    setp.neu.f32 %p1, %r3, %r1;
+; CHECK-F32X2-NEXT:    setp.neu.f32 %p2, %r4, %r2;
+; CHECK-F32X2-NEXT:    selp.f64 %rd7, %rd2, %rd4, %p2;
+; CHECK-F32X2-NEXT:    selp.f64 %rd8, %rd1, %rd3, %p1;
+; CHECK-F32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd8, %rd7};
+; CHECK-F32X2-NEXT:    ret;
   %cc = fcmp une <2 x float> %c, %d
   %r = select <2 x i1> %cc, <2 x double> %a, <2 x double> %b
   ret <2 x double> %r
 }
 
 define <2 x float> @test_select_cc_f32_f64(<2 x float> %a, <2 x float> %b, <2 x double> %c, <2 x double> %d) #0 {
-; CHECK-LABEL: test_select_cc_f32_f64(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
-; CHECK-NEXT:    .reg .b64 %rd<7>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [test_select_cc_f32_f64_param_3];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_select_cc_f32_f64_param_2];
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_select_cc_f32_f64_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_select_cc_f32_f64_param_0];
-; CHECK-NEXT:    setp.neu.f64 %p1, %rd3, %rd5;
-; CHECK-NEXT:    setp.neu.f64 %p2, %rd4, %rd6;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    selp.f32 %r5, %r4, %r2, %p2;
-; CHECK-NEXT:    selp.f32 %r6, %r3, %r1, %p1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_select_cc_f32_f64(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_select_cc_f32_f64_param_3];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_select_cc_f32_f64_param_2];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_select_cc_f32_f64_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_select_cc_f32_f64_param_0];
+; CHECK-NOF32X2-NEXT:    setp.neu.f64 %p1, %rd1, %rd3;
+; CHECK-NOF32X2-NEXT:    setp.neu.f64 %p2, %rd2, %rd4;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r5, %r2, %r4, %p2;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r6, %r1, %r3, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_select_cc_f32_f64(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<7>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [test_select_cc_f32_f64_param_3];
+; CHECK-F32X2-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_select_cc_f32_f64_param_2];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_select_cc_f32_f64_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_select_cc_f32_f64_param_0];
+; CHECK-F32X2-NEXT:    setp.neu.f64 %p1, %rd3, %rd5;
+; CHECK-F32X2-NEXT:    setp.neu.f64 %p2, %rd4, %rd6;
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    selp.f32 %r5, %r4, %r2, %p2;
+; CHECK-F32X2-NEXT:    selp.f32 %r6, %r3, %r1, %p1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-F32X2-NEXT:    ret;
   %cc = fcmp une <2 x double> %c, %d
   %r = select <2 x i1> %cc, <2 x float> %a, <2 x float> %b
   ret <2 x float> %r
 }
 
 define <2 x i1> @test_fcmp_une(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_une(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_une_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_une_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.neu.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.neu.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_une(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_une_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_une_param_0];
+; CHECK-NOF32X2-NEXT:    setp.neu.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.neu.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_une(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_une_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_une_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.neu.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.neu.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp une <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_ueq(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_ueq(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_ueq_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_ueq_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.equ.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.equ.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_ueq(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_ueq_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_ueq_param_0];
+; CHECK-NOF32X2-NEXT:    setp.equ.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.equ.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_ueq(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_ueq_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_ueq_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.equ.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.equ.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp ueq <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_ugt(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_ugt(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_ugt_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_ugt_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.gtu.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.gtu.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_ugt(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_ugt_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_ugt_param_0];
+; CHECK-NOF32X2-NEXT:    setp.gtu.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.gtu.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_ugt(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_ugt_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_ugt_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.gtu.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.gtu.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp ugt <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_uge(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_uge(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_uge_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_uge_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.geu.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.geu.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_uge(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_uge_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_uge_param_0];
+; CHECK-NOF32X2-NEXT:    setp.geu.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.geu.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_uge(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_uge_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_uge_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.geu.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.geu.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp uge <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_ult(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_ult(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_ult_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_ult_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.ltu.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.ltu.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_ult(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_ult_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_ult_param_0];
+; CHECK-NOF32X2-NEXT:    setp.ltu.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.ltu.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_ult(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_ult_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_ult_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.ltu.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.ltu.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp ult <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_ule(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_ule(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_ule_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_ule_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.leu.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.leu.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_ule(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_ule_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_ule_param_0];
+; CHECK-NOF32X2-NEXT:    setp.leu.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.leu.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_ule(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_ule_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_ule_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.leu.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.leu.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp ule <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_uno(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_uno(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_uno_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_uno_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.nan.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.nan.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_uno(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_uno_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_uno_param_0];
+; CHECK-NOF32X2-NEXT:    setp.nan.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.nan.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_uno(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_uno_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_uno_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.nan.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.nan.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp uno <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_one(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_one(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_one_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_one_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.ne.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.ne.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_one(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_one_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_one_param_0];
+; CHECK-NOF32X2-NEXT:    setp.ne.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.ne.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_one(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_one_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_one_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.ne.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.ne.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp one <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_oeq(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_oeq(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_oeq_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_oeq_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.eq.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.eq.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_oeq(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_oeq_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_oeq_param_0];
+; CHECK-NOF32X2-NEXT:    setp.eq.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.eq.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_oeq(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_oeq_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_oeq_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.eq.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.eq.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp oeq <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_ogt(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_ogt(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_ogt_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_ogt_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.gt.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.gt.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_ogt(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_ogt_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_ogt_param_0];
+; CHECK-NOF32X2-NEXT:    setp.gt.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.gt.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_ogt(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_ogt_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_ogt_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.gt.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.gt.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp ogt <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_oge(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_oge(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_oge_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_oge_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.ge.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.ge.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_oge(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_oge_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_oge_param_0];
+; CHECK-NOF32X2-NEXT:    setp.ge.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.ge.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_oge(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_oge_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_oge_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.ge.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.ge.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp oge <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_olt(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_olt(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_olt_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_olt_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.lt.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.lt.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_olt(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_olt_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_olt_param_0];
+; CHECK-NOF32X2-NEXT:    setp.lt.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.lt.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_olt(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_olt_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_olt_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.lt.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.lt.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp olt <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_ole(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_ole(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_ole_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_ole_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.le.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.le.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_ole(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_ole_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_ole_param_0];
+; CHECK-NOF32X2-NEXT:    setp.le.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.le.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_ole(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_ole_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_ole_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.le.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.le.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp ole <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i1> @test_fcmp_ord(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_fcmp_ord(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_fcmp_ord_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fcmp_ord_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    setp.num.f32 %p1, %r4, %r2;
-; CHECK-NEXT:    setp.num.f32 %p2, %r3, %r1;
-; CHECK-NEXT:    selp.b16 %rs1, -1, 0, %p2;
-; CHECK-NEXT:    st.param.b8 [func_retval0], %rs1;
-; CHECK-NEXT:    selp.b16 %rs2, -1, 0, %p1;
-; CHECK-NEXT:    st.param.b8 [func_retval0+1], %rs2;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fcmp_ord(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fcmp_ord_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fcmp_ord_param_0];
+; CHECK-NOF32X2-NEXT:    setp.num.f32 %p1, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    setp.num.f32 %p2, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-NOF32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-NOF32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fcmp_ord(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b16 %rs<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_fcmp_ord_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fcmp_ord_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    setp.num.f32 %p1, %r4, %r2;
+; CHECK-F32X2-NEXT:    setp.num.f32 %p2, %r3, %r1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs1, -1, 0, %p2;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0], %rs1;
+; CHECK-F32X2-NEXT:    selp.b16 %rs2, -1, 0, %p1;
+; CHECK-F32X2-NEXT:    st.param.b8 [func_retval0+1], %rs2;
+; CHECK-F32X2-NEXT:    ret;
   %r = fcmp ord <2 x float> %a, %b
   ret <2 x i1> %r
 }
 
 define <2 x i32> @test_fptosi_i32(<2 x float> %a) #0 {
-; CHECK-LABEL: test_fptosi_i32(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fptosi_i32_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rzi.s32.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rzi.s32.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fptosi_i32(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fptosi_i32_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rzi.s32.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.s32.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fptosi_i32(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fptosi_i32_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rzi.s32.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    cvt.rzi.s32.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = fptosi <2 x float> %a to <2 x i32>
   ret <2 x i32> %r
 }
 
 define <2 x i64> @test_fptosi_i64(<2 x float> %a) #0 {
-; CHECK-LABEL: test_fptosi_i64(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fptosi_i64_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rzi.s64.f32 %rd2, %r2;
-; CHECK-NEXT:    cvt.rzi.s64.f32 %rd3, %r1;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd2};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fptosi_i64(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fptosi_i64_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rzi.s64.f32 %rd1, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.s64.f32 %rd2, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd2, %rd1};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fptosi_i64(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<4>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fptosi_i64_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rzi.s64.f32 %rd2, %r2;
+; CHECK-F32X2-NEXT:    cvt.rzi.s64.f32 %rd3, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd2};
+; CHECK-F32X2-NEXT:    ret;
   %r = fptosi <2 x float> %a to <2 x i64>
   ret <2 x i64> %r
 }
 
 define <2 x i32> @test_fptoui_2xi32(<2 x float> %a) #0 {
-; CHECK-LABEL: test_fptoui_2xi32(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fptoui_2xi32_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rzi.u32.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rzi.u32.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fptoui_2xi32(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fptoui_2xi32_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rzi.u32.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.u32.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fptoui_2xi32(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fptoui_2xi32_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rzi.u32.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    cvt.rzi.u32.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = fptoui <2 x float> %a to <2 x i32>
   ret <2 x i32> %r
 }
 
 define <2 x i64> @test_fptoui_2xi64(<2 x float> %a) #0 {
-; CHECK-LABEL: test_fptoui_2xi64(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fptoui_2xi64_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rzi.u64.f32 %rd2, %r2;
-; CHECK-NEXT:    cvt.rzi.u64.f32 %rd3, %r1;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd2};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fptoui_2xi64(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fptoui_2xi64_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rzi.u64.f32 %rd1, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.u64.f32 %rd2, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd2, %rd1};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fptoui_2xi64(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<4>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fptoui_2xi64_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rzi.u64.f32 %rd2, %r2;
+; CHECK-F32X2-NEXT:    cvt.rzi.u64.f32 %rd3, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd2};
+; CHECK-F32X2-NEXT:    ret;
   %r = fptoui <2 x float> %a to <2 x i64>
   ret <2 x i64> %r
 }
@@ -1485,16 +2003,14 @@ define <2 x float> @test_uitofp_2xi32_fadd(<2 x i32> %a, <2 x float> %b) #0 {
 ; CHECK-NOF32X2-LABEL: test_uitofp_2xi32_fadd(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_uitofp_2xi32_fadd_param_1];
 ; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_uitofp_2xi32_fadd_param_0];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_uitofp_2xi32_fadd_param_1];
-; CHECK-NOF32X2-NEXT:    cvt.rn.f32.u32 %r3, %r1;
-; CHECK-NOF32X2-NEXT:    cvt.rn.f32.u32 %r4, %r2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r7, %r6, %r4;
-; CHECK-NOF32X2-NEXT:    add.rn.f32 %r8, %r5, %r3;
+; CHECK-NOF32X2-NEXT:    cvt.rn.f32.u32 %r5, %r1;
+; CHECK-NOF32X2-NEXT:    cvt.rn.f32.u32 %r6, %r2;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r7, %r4, %r6;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r8, %r3, %r5;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r8, %r7};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -1518,48 +2034,81 @@ define <2 x float> @test_uitofp_2xi32_fadd(<2 x i32> %a, <2 x float> %b) #0 {
 }
 
 define <2 x float> @test_fptrunc_2xdouble(<2 x double> %a) #0 {
-; CHECK-LABEL: test_fptrunc_2xdouble(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_fptrunc_2xdouble_param_0];
-; CHECK-NEXT:    cvt.rn.f32.f64 %r1, %rd2;
-; CHECK-NEXT:    cvt.rn.f32.f64 %r2, %rd1;
-; CHECK-NEXT:    mov.b64 %rd3, {%r2, %r1};
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd3;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fptrunc_2xdouble(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_fptrunc_2xdouble_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rn.f32.f64 %r1, %rd2;
+; CHECK-NOF32X2-NEXT:    cvt.rn.f32.f64 %r2, %rd1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r2, %r1};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fptrunc_2xdouble(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<4>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_fptrunc_2xdouble_param_0];
+; CHECK-F32X2-NEXT:    cvt.rn.f32.f64 %r1, %rd2;
+; CHECK-F32X2-NEXT:    cvt.rn.f32.f64 %r2, %rd1;
+; CHECK-F32X2-NEXT:    mov.b64 %rd3, {%r2, %r1};
+; CHECK-F32X2-NEXT:    st.param.b64 [func_retval0], %rd3;
+; CHECK-F32X2-NEXT:    ret;
   %r = fptrunc <2 x double> %a to <2 x float>
   ret <2 x float> %r
 }
 
 define <2 x double> @test_fpext_2xdouble(<2 x float> %a) #0 {
-; CHECK-LABEL: test_fpext_2xdouble(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fpext_2xdouble_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.f64.f32 %rd2, %r2;
-; CHECK-NEXT:    cvt.f64.f32 %rd3, %r1;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd2};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fpext_2xdouble(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fpext_2xdouble_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.f64.f32 %rd1, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.f64.f32 %rd2, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd2, %rd1};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fpext_2xdouble(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<4>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fpext_2xdouble_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.f64.f32 %rd2, %r2;
+; CHECK-F32X2-NEXT:    cvt.f64.f32 %rd3, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd2};
+; CHECK-F32X2-NEXT:    ret;
   %r = fpext <2 x float> %a to <2 x double>
   ret <2 x double> %r
 }
 
 define <2 x i32> @test_bitcast_2xfloat_to_2xi32(<2 x float> %a) #0 {
-; CHECK-LABEL: test_bitcast_2xfloat_to_2xi32(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_bitcast_2xfloat_to_2xi32_param_0];
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd1;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_bitcast_2xfloat_to_2xi32(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_bitcast_2xfloat_to_2xi32_param_0];
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r2};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_bitcast_2xfloat_to_2xi32(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_bitcast_2xfloat_to_2xi32_param_0];
+; CHECK-F32X2-NEXT:    st.param.b64 [func_retval0], %rd1;
+; CHECK-F32X2-NEXT:    ret;
   %r = bitcast <2 x float> %a to <2 x i32>
   ret <2 x i32> %r
 }
@@ -1591,31 +2140,51 @@ define <2 x float> @test_bitcast_double_to_2xfloat(double %a) #0 {
 }
 
 define double @test_bitcast_2xfloat_to_double(<2 x float> %a) #0 {
-; CHECK-LABEL: test_bitcast_2xfloat_to_double(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_bitcast_2xfloat_to_double_param_0];
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd1;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_bitcast_2xfloat_to_double(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_bitcast_2xfloat_to_double_param_0];
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r2};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_bitcast_2xfloat_to_double(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_bitcast_2xfloat_to_double_param_0];
+; CHECK-F32X2-NEXT:    st.param.b64 [func_retval0], %rd1;
+; CHECK-F32X2-NEXT:    ret;
   %r = bitcast <2 x float> %a to double
   ret double %r
 }
 
 define <2 x float> @test_sqrt(<2 x float> %a) #0 {
-; CHECK-LABEL: test_sqrt(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_sqrt_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    sqrt.rn.f32 %r3, %r2;
-; CHECK-NEXT:    sqrt.rn.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_sqrt(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_sqrt_param_0];
+; CHECK-NOF32X2-NEXT:    sqrt.rn.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    sqrt.rn.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_sqrt(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_sqrt_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    sqrt.rn.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    sqrt.rn.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.sqrt(<2 x float> %a)
   ret <2 x float> %r
 }
@@ -1627,37 +2196,59 @@ define <2 x float> @test_sqrt(<2 x float> %a) #0 {
 ;  ret <2 x float> %r
 ;}
 
-define <2 x float> @test_sin(<2 x float> %a) #0 #1 {
-; CHECK-LABEL: test_sin(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_sin_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    sin.approx.f32 %r3, %r2;
-; CHECK-NEXT:    sin.approx.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
-  %r = call <2 x float> @llvm.sin(<2 x float> %a)
+define <2 x float> @test_sin(<2 x float> %a) #0 {
+; CHECK-NOF32X2-LABEL: test_sin(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_sin_param_0];
+; CHECK-NOF32X2-NEXT:    sin.approx.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    sin.approx.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_sin(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_sin_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    sin.approx.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    sin.approx.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
+  %r = call afn <2 x float> @llvm.sin(<2 x float> %a)
   ret <2 x float> %r
 }
 
-define <2 x float> @test_cos(<2 x float> %a) #0 #1 {
-; CHECK-LABEL: test_cos(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_cos_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cos.approx.f32 %r3, %r2;
-; CHECK-NEXT:    cos.approx.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
-  %r = call <2 x float> @llvm.cos(<2 x float> %a)
+define <2 x float> @test_cos(<2 x float> %a) #0 {
+; CHECK-NOF32X2-LABEL: test_cos(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_cos_param_0];
+; CHECK-NOF32X2-NEXT:    cos.approx.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    cos.approx.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_cos(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_cos_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cos.approx.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    cos.approx.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
+  %r = call afn <2 x float> @llvm.cos(<2 x float> %a)
   ret <2 x float> %r
 }
 
@@ -1708,17 +2299,13 @@ define <2 x float> @test_fma(<2 x float> %a, <2 x float> %b, <2 x float> %c) #0
 ; CHECK-NOF32X2-LABEL: test_fma(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd3, [test_fma_param_2];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_fma_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fma_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r8, %r5, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_fma_param_2];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fma_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fma_param_0];
+; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r7, %r2, %r4, %r6;
+; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r8, %r1, %r3, %r5;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r8, %r7};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -1738,266 +2325,448 @@ define <2 x float> @test_fma(<2 x float> %a, <2 x float> %b, <2 x float> %c) #0
 }
 
 define <2 x float> @test_fabs(<2 x float> %a) #0 {
-; CHECK-LABEL: test_fabs(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_fabs_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    abs.f32 %r3, %r2;
-; CHECK-NEXT:    abs.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_fabs(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fabs_param_0];
+; CHECK-NOF32X2-NEXT:    abs.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    abs.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_fabs(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_fabs_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    abs.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    abs.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.fabs(<2 x float> %a)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_minnum(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_minnum(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<7>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_minnum_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_minnum_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    min.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    min.f32 %r6, %r3, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_minnum(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_minnum_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_minnum_param_0];
+; CHECK-NOF32X2-NEXT:    min.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    min.f32 %r6, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_minnum(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_minnum_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_minnum_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    min.f32 %r5, %r4, %r2;
+; CHECK-F32X2-NEXT:    min.f32 %r6, %r3, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.minnum(<2 x float> %a, <2 x float> %b)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_maxnum(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_maxnum(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<7>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_maxnum_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_maxnum_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd1;
-; CHECK-NEXT:    max.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    max.f32 %r6, %r3, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_maxnum(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_maxnum_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_maxnum_param_0];
+; CHECK-NOF32X2-NEXT:    max.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    max.f32 %r6, %r1, %r3;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_maxnum(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_maxnum_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_maxnum_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd2;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-F32X2-NEXT:    max.f32 %r5, %r4, %r2;
+; CHECK-F32X2-NEXT:    max.f32 %r6, %r3, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.maxnum(<2 x float> %a, <2 x float> %b)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_copysign(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_copysign(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<7>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_copysign_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_copysign_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    copysign.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    copysign.f32 %r6, %r3, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_copysign(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_copysign_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_copysign_param_0];
+; CHECK-NOF32X2-NEXT:    copysign.f32 %r5, %r4, %r2;
+; CHECK-NOF32X2-NEXT:    copysign.f32 %r6, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_copysign(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_copysign_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_copysign_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd2;
+; CHECK-F32X2-NEXT:    copysign.f32 %r5, %r4, %r2;
+; CHECK-F32X2-NEXT:    copysign.f32 %r6, %r3, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.copysign(<2 x float> %a, <2 x float> %b)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_copysign_f64(<2 x float> %a, <2 x double> %b) #0 {
-; CHECK-LABEL: test_copysign_f64(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b32 %r<9>;
-; CHECK-NEXT:    .reg .b64 %rd<8>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [test_copysign_f64_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_copysign_f64_param_0];
-; CHECK-NEXT:    shr.u64 %rd4, %rd3, 63;
-; CHECK-NEXT:    and.b64 %rd5, %rd4, 1;
-; CHECK-NEXT:    setp.ne.b64 %p1, %rd5, 0;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    abs.f32 %r3, %r2;
-; CHECK-NEXT:    neg.f32 %r4, %r3;
-; CHECK-NEXT:    selp.f32 %r5, %r4, %r3, %p1;
-; CHECK-NEXT:    shr.u64 %rd6, %rd2, 63;
-; CHECK-NEXT:    and.b64 %rd7, %rd6, 1;
-; CHECK-NEXT:    setp.ne.b64 %p2, %rd7, 0;
-; CHECK-NEXT:    abs.f32 %r6, %r1;
-; CHECK-NEXT:    neg.f32 %r7, %r6;
-; CHECK-NEXT:    selp.f32 %r8, %r7, %r6, %p2;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r8, %r5};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_copysign_f64(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<7>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_copysign_f64_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_copysign_f64_param_0];
+; CHECK-NOF32X2-NEXT:    abs.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    neg.f32 %r4, %r3;
+; CHECK-NOF32X2-NEXT:    shr.u64 %rd3, %rd2, 63;
+; CHECK-NOF32X2-NEXT:    and.b64 %rd4, %rd3, 1;
+; CHECK-NOF32X2-NEXT:    setp.ne.b64 %p1, %rd4, 0;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r5, %r4, %r3, %p1;
+; CHECK-NOF32X2-NEXT:    abs.f32 %r6, %r1;
+; CHECK-NOF32X2-NEXT:    neg.f32 %r7, %r6;
+; CHECK-NOF32X2-NEXT:    shr.u64 %rd5, %rd1, 63;
+; CHECK-NOF32X2-NEXT:    and.b64 %rd6, %rd5, 1;
+; CHECK-NOF32X2-NEXT:    setp.ne.b64 %p2, %rd6, 0;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r8, %r7, %r6, %p2;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r8, %r5};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_copysign_f64(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<3>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<9>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<8>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [test_copysign_f64_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_copysign_f64_param_0];
+; CHECK-F32X2-NEXT:    shr.u64 %rd4, %rd3, 63;
+; CHECK-F32X2-NEXT:    and.b64 %rd5, %rd4, 1;
+; CHECK-F32X2-NEXT:    setp.ne.b64 %p1, %rd5, 0;
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    abs.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    neg.f32 %r4, %r3;
+; CHECK-F32X2-NEXT:    selp.f32 %r5, %r4, %r3, %p1;
+; CHECK-F32X2-NEXT:    shr.u64 %rd6, %rd2, 63;
+; CHECK-F32X2-NEXT:    and.b64 %rd7, %rd6, 1;
+; CHECK-F32X2-NEXT:    setp.ne.b64 %p2, %rd7, 0;
+; CHECK-F32X2-NEXT:    abs.f32 %r6, %r1;
+; CHECK-F32X2-NEXT:    neg.f32 %r7, %r6;
+; CHECK-F32X2-NEXT:    selp.f32 %r8, %r7, %r6, %p2;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r8, %r5};
+; CHECK-F32X2-NEXT:    ret;
   %tb = fptrunc <2 x double> %b to <2 x float>
   %r = call <2 x float> @llvm.copysign(<2 x float> %a, <2 x float> %tb)
   ret <2 x float> %r
 }
 
 define <2 x double> @test_copysign_extended(<2 x float> %a, <2 x float> %b) #0 {
-; CHECK-LABEL: test_copysign_extended(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<7>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_copysign_extended_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_copysign_extended_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    copysign.f32 %r5, %r3, %r1;
-; CHECK-NEXT:    copysign.f32 %r6, %r4, %r2;
-; CHECK-NEXT:    cvt.f64.f32 %rd3, %r6;
-; CHECK-NEXT:    cvt.f64.f32 %rd4, %r5;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_copysign_extended(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_copysign_extended_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_copysign_extended_param_0];
+; CHECK-NOF32X2-NEXT:    copysign.f32 %r5, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    copysign.f32 %r6, %r4, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.f64.f32 %rd1, %r6;
+; CHECK-NOF32X2-NEXT:    cvt.f64.f32 %rd2, %r5;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd2, %rd1};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_copysign_extended(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<7>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<5>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_copysign_extended_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_copysign_extended_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    mov.b64 {%r3, %r4}, %rd2;
+; CHECK-F32X2-NEXT:    copysign.f32 %r5, %r3, %r1;
+; CHECK-F32X2-NEXT:    copysign.f32 %r6, %r4, %r2;
+; CHECK-F32X2-NEXT:    cvt.f64.f32 %rd3, %r6;
+; CHECK-F32X2-NEXT:    cvt.f64.f32 %rd4, %r5;
+; CHECK-F32X2-NEXT:    st.param.v2.b64 [func_retval0], {%rd4, %rd3};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.copysign(<2 x float> %a, <2 x float> %b)
   %xr = fpext <2 x float> %r to <2 x double>
   ret <2 x double> %xr
 }
 
 define <2 x float> @test_floor(<2 x float> %a) #0 {
-; CHECK-LABEL: test_floor(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_floor_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rmi.f32.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rmi.f32.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_floor(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_floor_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rmi.f32.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rmi.f32.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_floor(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_floor_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rmi.f32.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    cvt.rmi.f32.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.floor(<2 x float> %a)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_ceil(<2 x float> %a) #0 {
-; CHECK-LABEL: test_ceil(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_ceil_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rpi.f32.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rpi.f32.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_ceil(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_ceil_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rpi.f32.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rpi.f32.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_ceil(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_ceil_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rpi.f32.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    cvt.rpi.f32.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.ceil(<2 x float> %a)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_trunc(<2 x float> %a) #0 {
-; CHECK-LABEL: test_trunc(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_trunc_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_trunc(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_trunc_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.f32.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_trunc(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_trunc_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    cvt.rzi.f32.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.trunc(<2 x float> %a)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_rint(<2 x float> %a) #0 {
-; CHECK-LABEL: test_rint(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_rint_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rni.f32.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rni.f32.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_rint(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_rint_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rni.f32.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rni.f32.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_rint(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_rint_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rni.f32.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    cvt.rni.f32.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.rint(<2 x float> %a)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_nearbyint(<2 x float> %a) #0 {
-; CHECK-LABEL: test_nearbyint(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_nearbyint_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rni.f32.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rni.f32.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_nearbyint(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_nearbyint_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rni.f32.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rni.f32.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_nearbyint(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_nearbyint_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rni.f32.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    cvt.rni.f32.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.nearbyint(<2 x float> %a)
   ret <2 x float> %r
 }
 
 define <2 x float> @test_roundeven(<2 x float> %a) #0 {
-; CHECK-LABEL: test_roundeven(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_roundeven_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rni.f32.f32 %r3, %r2;
-; CHECK-NEXT:    cvt.rni.f32.f32 %r4, %r1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_roundeven(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_roundeven_param_0];
+; CHECK-NOF32X2-NEXT:    cvt.rni.f32.f32 %r3, %r2;
+; CHECK-NOF32X2-NEXT:    cvt.rni.f32.f32 %r4, %r1;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_roundeven(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<5>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_roundeven_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rni.f32.f32 %r3, %r2;
+; CHECK-F32X2-NEXT:    cvt.rni.f32.f32 %r4, %r1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r4, %r3};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.roundeven(<2 x float> %a)
   ret <2 x float> %r
 }
 
 ; check the use of sign mask and 0.5 to implement round
 define <2 x float> @test_round(<2 x float> %a) #0 {
-; CHECK-LABEL: test_round(
-; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<5>;
-; CHECK-NEXT:    .reg .b32 %r<19>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_round_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    and.b32 %r3, %r2, -2147483648;
-; CHECK-NEXT:    or.b32 %r4, %r3, 1056964608;
-; CHECK-NEXT:    add.rn.f32 %r5, %r2, %r4;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r6, %r5;
-; CHECK-NEXT:    abs.f32 %r7, %r2;
-; CHECK-NEXT:    setp.gt.f32 %p1, %r7, 0f4B000000;
-; CHECK-NEXT:    selp.f32 %r8, %r2, %r6, %p1;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r9, %r2;
-; CHECK-NEXT:    setp.lt.f32 %p2, %r7, 0f3F000000;
-; CHECK-NEXT:    selp.f32 %r10, %r9, %r8, %p2;
-; CHECK-NEXT:    and.b32 %r11, %r1, -2147483648;
-; CHECK-NEXT:    or.b32 %r12, %r11, 1056964608;
-; CHECK-NEXT:    add.rn.f32 %r13, %r1, %r12;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r14, %r13;
-; CHECK-NEXT:    abs.f32 %r15, %r1;
-; CHECK-NEXT:    setp.gt.f32 %p3, %r15, 0f4B000000;
-; CHECK-NEXT:    selp.f32 %r16, %r1, %r14, %p3;
-; CHECK-NEXT:    cvt.rzi.f32.f32 %r17, %r1;
-; CHECK-NEXT:    setp.lt.f32 %p4, %r15, 0f3F000000;
-; CHECK-NEXT:    selp.f32 %r18, %r17, %r16, %p4;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r18, %r10};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_round(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .pred %p<5>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<19>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_round_param_0];
+; CHECK-NOF32X2-NEXT:    and.b32 %r3, %r2, -2147483648;
+; CHECK-NOF32X2-NEXT:    or.b32 %r4, %r3, 1056964608;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r5, %r2, %r4;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.f32.f32 %r6, %r5;
+; CHECK-NOF32X2-NEXT:    abs.f32 %r7, %r2;
+; CHECK-NOF32X2-NEXT:    setp.gt.f32 %p1, %r7, 0f4B000000;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r8, %r2, %r6, %p1;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.f32.f32 %r9, %r2;
+; CHECK-NOF32X2-NEXT:    setp.lt.f32 %p2, %r7, 0f3F000000;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r10, %r9, %r8, %p2;
+; CHECK-NOF32X2-NEXT:    and.b32 %r11, %r1, -2147483648;
+; CHECK-NOF32X2-NEXT:    or.b32 %r12, %r11, 1056964608;
+; CHECK-NOF32X2-NEXT:    add.rn.f32 %r13, %r1, %r12;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.f32.f32 %r14, %r13;
+; CHECK-NOF32X2-NEXT:    abs.f32 %r15, %r1;
+; CHECK-NOF32X2-NEXT:    setp.gt.f32 %p3, %r15, 0f4B000000;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r16, %r1, %r14, %p3;
+; CHECK-NOF32X2-NEXT:    cvt.rzi.f32.f32 %r17, %r1;
+; CHECK-NOF32X2-NEXT:    setp.lt.f32 %p4, %r15, 0f3F000000;
+; CHECK-NOF32X2-NEXT:    selp.f32 %r18, %r17, %r16, %p4;
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r18, %r10};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_round(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .pred %p<5>;
+; CHECK-F32X2-NEXT:    .reg .b32 %r<19>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_round_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    and.b32 %r3, %r2, -2147483648;
+; CHECK-F32X2-NEXT:    or.b32 %r4, %r3, 1056964608;
+; CHECK-F32X2-NEXT:    add.rn.f32 %r5, %r2, %r4;
+; CHECK-F32X2-NEXT:    cvt.rzi.f32.f32 %r6, %r5;
+; CHECK-F32X2-NEXT:    abs.f32 %r7, %r2;
+; CHECK-F32X2-NEXT:    setp.gt.f32 %p1, %r7, 0f4B000000;
+; CHECK-F32X2-NEXT:    selp.f32 %r8, %r2, %r6, %p1;
+; CHECK-F32X2-NEXT:    cvt.rzi.f32.f32 %r9, %r2;
+; CHECK-F32X2-NEXT:    setp.lt.f32 %p2, %r7, 0f3F000000;
+; CHECK-F32X2-NEXT:    selp.f32 %r10, %r9, %r8, %p2;
+; CHECK-F32X2-NEXT:    and.b32 %r11, %r1, -2147483648;
+; CHECK-F32X2-NEXT:    or.b32 %r12, %r11, 1056964608;
+; CHECK-F32X2-NEXT:    add.rn.f32 %r13, %r1, %r12;
+; CHECK-F32X2-NEXT:    cvt.rzi.f32.f32 %r14, %r13;
+; CHECK-F32X2-NEXT:    abs.f32 %r15, %r1;
+; CHECK-F32X2-NEXT:    setp.gt.f32 %p3, %r15, 0f4B000000;
+; CHECK-F32X2-NEXT:    selp.f32 %r16, %r1, %r14, %p3;
+; CHECK-F32X2-NEXT:    cvt.rzi.f32.f32 %r17, %r1;
+; CHECK-F32X2-NEXT:    setp.lt.f32 %p4, %r15, 0f3F000000;
+; CHECK-F32X2-NEXT:    selp.f32 %r18, %r17, %r16, %p4;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r18, %r10};
+; CHECK-F32X2-NEXT:    ret;
   %r = call <2 x float> @llvm.round(<2 x float> %a)
   ret <2 x float> %r
 }
@@ -2006,17 +2775,13 @@ define <2 x float> @test_fmuladd(<2 x float> %a, <2 x float> %b, <2 x float> %c)
 ; CHECK-NOF32X2-LABEL: test_fmuladd(
 ; CHECK-NOF32X2:       {
 ; CHECK-NOF32X2-NEXT:    .reg .b32 %r<9>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd3, [test_fmuladd_param_2];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd2, [test_fmuladd_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_fmuladd_param_0];
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NOF32X2-NEXT:    mov.b64 {%r5, %r6}, %rd1;
-; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r7, %r6, %r4, %r2;
-; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r8, %r5, %r3, %r1;
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r5, %r6}, [test_fmuladd_param_2];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_fmuladd_param_1];
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_fmuladd_param_0];
+; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r7, %r2, %r4, %r6;
+; CHECK-NOF32X2-NEXT:    fma.rn.f32 %r8, %r1, %r3, %r5;
 ; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r8, %r7};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
@@ -2036,16 +2801,25 @@ define <2 x float> @test_fmuladd(<2 x float> %a, <2 x float> %b, <2 x float> %c)
 }
 
 define <2 x float> @test_shufflevector(<2 x float> %a) #0 {
-; CHECK-LABEL: test_shufflevector(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_shufflevector_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r2, %r1};
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_shufflevector(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_shufflevector_param_0];
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r2, %r1};
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_shufflevector(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<3>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_shufflevector_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r2, %r1};
+; CHECK-F32X2-NEXT:    ret;
   %s = shufflevector <2 x float> %a, <2 x float> poison, <2 x i32> <i32 1, i32 0>
   ret <2 x float> %s
 }
@@ -2053,14 +2827,12 @@ define <2 x float> @test_shufflevector(<2 x float> %a) #0 {
 define <2 x float> @test_insertelement(<2 x float> %a, float %x) #0 {
 ; CHECK-NOF32X2-LABEL: test_insertelement(
 ; CHECK-NOF32X2:       {
-; CHECK-NOF32X2-NEXT:    .reg .b32 %r<3>;
-; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<4>;
 ; CHECK-NOF32X2-EMPTY:
 ; CHECK-NOF32X2-NEXT:  // %bb.0:
-; CHECK-NOF32X2-NEXT:    ld.param.b32 %r1, [test_insertelement_param_1];
-; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_insertelement_param_0];
-; CHECK-NOF32X2-NEXT:    { .reg .b32 tmp; mov.b64 {%r2, tmp}, %rd1; }
-; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r2, %r1};
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_insertelement_param_0];
+; CHECK-NOF32X2-NEXT:    ld.param.b32 %r3, [test_insertelement_param_1];
+; CHECK-NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r3};
 ; CHECK-NOF32X2-NEXT:    ret;
 ;
 ; CHECK-F32X2-LABEL: test_insertelement(
@@ -2109,36 +2881,60 @@ define <2 x float> @test_uitofp_2xi32_to_2xfloat(<2 x i32> %a) #0 {
 }
 
 define void @test_trunc_to_v2bf16(<2 x float> %a, ptr %p) {
-; CHECK-LABEL: test_trunc_to_v2bf16(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_trunc_to_v2bf16_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_trunc_to_v2bf16_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rn.bf16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.b32 [%rd2], %r3;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_trunc_to_v2bf16(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<4>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_trunc_to_v2bf16_param_0];
+; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_trunc_to_v2bf16_param_1];
+; CHECK-NOF32X2-NEXT:    cvt.rn.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NOF32X2-NEXT:    st.b32 [%rd1], %r3;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_trunc_to_v2bf16(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<4>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_trunc_to_v2bf16_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_trunc_to_v2bf16_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rn.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-F32X2-NEXT:    st.b32 [%rd2], %r3;
+; CHECK-F32X2-NEXT:    ret;
   %trunc = fptrunc <2 x float> %a to <2 x bfloat>
   store <2 x bfloat> %trunc, ptr %p
   ret void
 }
 
 define void @test_trunc_to_v2f16(<2 x float> %a, ptr %p) {
-; CHECK-LABEL: test_trunc_to_v2f16(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd2, [test_trunc_to_v2f16_param_1];
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_trunc_to_v2f16_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd1;
-; CHECK-NEXT:    cvt.rn.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.b32 [%rd2], %r3;
-; CHECK-NEXT:    ret;
+; CHECK-NOF32X2-LABEL: test_trunc_to_v2f16(
+; CHECK-NOF32X2:       {
+; CHECK-NOF32X2-NEXT:    .reg .b32 %r<4>;
+; CHECK-NOF32X2-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NOF32X2-EMPTY:
+; CHECK-NOF32X2-NEXT:  // %bb.0:
+; CHECK-NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_trunc_to_v2f16_param_0];
+; CHECK-NOF32X2-NEXT:    ld.param.b64 %rd1, [test_trunc_to_v2f16_param_1];
+; CHECK-NOF32X2-NEXT:    cvt.rn.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NOF32X2-NEXT:    st.b32 [%rd1], %r3;
+; CHECK-NOF32X2-NEXT:    ret;
+;
+; CHECK-F32X2-LABEL: test_trunc_to_v2f16(
+; CHECK-F32X2:       {
+; CHECK-F32X2-NEXT:    .reg .b32 %r<4>;
+; CHECK-F32X2-NEXT:    .reg .b64 %rd<3>;
+; CHECK-F32X2-EMPTY:
+; CHECK-F32X2-NEXT:  // %bb.0:
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd2, [test_trunc_to_v2f16_param_1];
+; CHECK-F32X2-NEXT:    ld.param.b64 %rd1, [test_trunc_to_v2f16_param_0];
+; CHECK-F32X2-NEXT:    mov.b64 {%r1, %r2}, %rd1;
+; CHECK-F32X2-NEXT:    cvt.rn.f16x2.f32 %r3, %r2, %r1;
+; CHECK-F32X2-NEXT:    st.b32 [%rd2], %r3;
+; CHECK-F32X2-NEXT:    ret;
   %trunc = fptrunc <2 x float> %a to <2 x half>
   store <2 x half> %trunc, ptr %p
   ret void
@@ -2146,5 +2942,4 @@ define void @test_trunc_to_v2f16(<2 x float> %a, ptr %p) {
 
 
 attributes #0 = { nounwind }
-attributes #1 = { "unsafe-fp-math" = "true" }
 attributes #2 = { "denormal-fp-math"="preserve-sign" }
diff --git a/llvm/test/CodeGen/NVPTX/fast-math.ll b/llvm/test/CodeGen/NVPTX/fast-math.ll
index 5eda3a1e2dda1..8561c60a46948 100644
--- a/llvm/test/CodeGen/NVPTX/fast-math.ll
+++ b/llvm/test/CodeGen/NVPTX/fast-math.ll
@@ -22,7 +22,7 @@ define float @sqrt_div(float %a, float %b) {
   ret float %t2
 }
 
-define float @sqrt_div_fast(float %a, float %b) #0 {
+define float @sqrt_div_fast(float %a, float %b) {
 ; CHECK-LABEL: sqrt_div_fast(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<5>;
@@ -34,29 +34,25 @@ define float @sqrt_div_fast(float %a, float %b) #0 {
 ; CHECK-NEXT:    div.approx.f32 %r4, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
-  %t1 = tail call float @llvm.sqrt.f32(float %a)
-  %t2 = fdiv float %t1, %b
+  %t1 = tail call afn float @llvm.sqrt.f32(float %a)
+  %t2 = fdiv afn float %t1, %b
   ret float %t2
 }
 
-define float @sqrt_div_fast_ninf(float %a, float %b) #0 {
+define float @sqrt_div_fast_ninf(float %a, float %b) {
 ; CHECK-LABEL: sqrt_div_fast_ninf(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [sqrt_div_fast_ninf_param_0];
 ; CHECK-NEXT:    sqrt.approx.f32 %r2, %r1;
-; CHECK-NEXT:    abs.f32 %r3, %r1;
-; CHECK-NEXT:    setp.lt.f32 %p1, %r3, 0f00800000;
-; CHECK-NEXT:    selp.f32 %r4, 0f00000000, %r2, %p1;
-; CHECK-NEXT:    ld.param.b32 %r5, [sqrt_div_fast_ninf_param_1];
-; CHECK-NEXT:    div.approx.f32 %r6, %r4, %r5;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ld.param.b32 %r3, [sqrt_div_fast_ninf_param_1];
+; CHECK-NEXT:    div.approx.f32 %r4, %r2, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
   %t1 = tail call ninf afn float @llvm.sqrt.f32(float %a)
-  %t2 = fdiv float %t1, %b
+  %t2 = fdiv afn float %t1, %b
   ret float %t2
 }
 
@@ -77,7 +73,7 @@ define float @sqrt_div_ftz(float %a, float %b) #1 {
   ret float %t2
 }
 
-define float @sqrt_div_fast_ftz(float %a, float %b) #0 #1 {
+define float @sqrt_div_fast_ftz(float %a, float %b) #1 {
 ; CHECK-LABEL: sqrt_div_fast_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<5>;
@@ -89,35 +85,32 @@ define float @sqrt_div_fast_ftz(float %a, float %b) #0 #1 {
 ; CHECK-NEXT:    div.approx.ftz.f32 %r4, %r2, %r3;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
-  %t1 = tail call float @llvm.sqrt.f32(float %a)
-  %t2 = fdiv float %t1, %b
+  %t1 = tail call afn float @llvm.sqrt.f32(float %a)
+  %t2 = fdiv afn float %t1, %b
   ret float %t2
 }
 
-define float @sqrt_div_fast_ftz_ninf(float %a, float %b) #0 #1 {
+define float @sqrt_div_fast_ftz_ninf(float %a, float %b) #1 {
 ; CHECK-LABEL: sqrt_div_fast_ftz_ninf(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b32 %r<6>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [sqrt_div_fast_ftz_ninf_param_0];
-; CHECK-NEXT:    setp.eq.ftz.f32 %p1, %r1, 0f00000000;
 ; CHECK-NEXT:    sqrt.approx.ftz.f32 %r2, %r1;
-; CHECK-NEXT:    selp.f32 %r3, 0f00000000, %r2, %p1;
-; CHECK-NEXT:    ld.param.b32 %r4, [sqrt_div_fast_ftz_ninf_param_1];
-; CHECK-NEXT:    div.approx.ftz.f32 %r5, %r3, %r4;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
+; CHECK-NEXT:    ld.param.b32 %r3, [sqrt_div_fast_ftz_ninf_param_1];
+; CHECK-NEXT:    div.approx.ftz.f32 %r4, %r2, %r3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-NEXT:    ret;
   %t1 = tail call ninf afn float @llvm.sqrt.f32(float %a)
-  %t2 = fdiv float %t1, %b
+  %t2 = fdiv afn float %t1, %b
   ret float %t2
 }
 
 ; There are no fast-math or ftz versions of sqrt and div for f64.  We use
 ; reciprocal(rsqrt(x)) for sqrt(x), and emit a vanilla divide.
 
-define double @sqrt_div_fast_ftz_f64(double %a, double %b) #0 #1 {
+define double @sqrt_div_fast_ftz_f64(double %a, double %b) #1 {
 ; CHECK-LABEL: sqrt_div_fast_ftz_f64(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<5>;
@@ -134,22 +127,17 @@ define double @sqrt_div_fast_ftz_f64(double %a, double %b) #0 #1 {
   ret double %t2
 }
 
-define double @sqrt_div_fast_ftz_f64_ninf(double %a, double %b) #0 #1 {
+define double @sqrt_div_fast_ftz_f64_ninf(double %a, double %b) #1 {
 ; CHECK-LABEL: sqrt_div_fast_ftz_f64_ninf(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b64 %rd<8>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [sqrt_div_fast_ftz_f64_ninf_param_0];
-; CHECK-NEXT:    abs.f64 %rd2, %rd1;
-; CHECK-NEXT:    setp.lt.f64 %p1, %rd2, 0d0010000000000000;
-; CHECK-NEXT:    rsqrt.approx.f64 %rd3, %rd1;
-; CHECK-NEXT:    rcp.approx.ftz.f64 %rd4, %rd3;
-; CHECK-NEXT:    selp.f64 %rd5, 0d0000000000000000, %rd4, %p1;
-; CHECK-NEXT:    ld.param.b64 %rd6, [sqrt_div_fast_ftz_f64_ninf_param_1];
-; CHECK-NEXT:    div.rn.f64 %rd7, %rd5, %rd6;
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd7;
+; CHECK-NEXT:    sqrt.rn.f64 %rd2, %rd1;
+; CHECK-NEXT:    ld.param.b64 %rd3, [sqrt_div_fast_ftz_f64_ninf_param_1];
+; CHECK-NEXT:    div.rn.f64 %rd4, %rd2, %rd3;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd4;
 ; CHECK-NEXT:    ret;
   %t1 = tail call ninf afn double @llvm.sqrt.f64(double %a)
   %t2 = fdiv double %t1, %b
@@ -172,7 +160,7 @@ define float @rsqrt(float %a) {
   ret float %ret
 }
 
-define float @rsqrt_fast(float %a) #0 {
+define float @rsqrt_fast(float %a) {
 ; CHECK-LABEL: rsqrt_fast(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -182,12 +170,12 @@ define float @rsqrt_fast(float %a) #0 {
 ; CHECK-NEXT:    rsqrt.approx.f32 %r2, %r1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
-  %b = tail call float @llvm.sqrt.f32(float %a)
-  %ret = fdiv float 1.0, %b
+  %b = tail call afn float @llvm.sqrt.f32(float %a)
+  %ret = fdiv afn float 1.0, %b
   ret float %ret
 }
 
-define float @rsqrt_fast_ftz(float %a) #0 #1 {
+define float @rsqrt_fast_ftz(float %a) #1 {
 ; CHECK-LABEL: rsqrt_fast_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -197,8 +185,8 @@ define float @rsqrt_fast_ftz(float %a) #0 #1 {
 ; CHECK-NEXT:    rsqrt.approx.ftz.f32 %r2, %r1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
-  %b = tail call float @llvm.sqrt.f32(float %a)
-  %ret = fdiv float 1.0, %b
+  %b = tail call afn float @llvm.sqrt.f32(float %a)
+  %ret = fdiv afn float 1.0, %b
   ret float %ret
 }
 
@@ -263,35 +251,7 @@ define float @fcos_approx_afn(float %a) {
   ret float %r
 }
 
-define float @fsin_approx(float %a) #0 {
-; CHECK-LABEL: fsin_approx(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fsin_approx_param_0];
-; CHECK-NEXT:    sin.approx.f32 %r2, %r1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
-; CHECK-NEXT:    ret;
-  %r = tail call float @llvm.sin.f32(float %a)
-  ret float %r
-}
-
-define float @fcos_approx(float %a) #0 {
-; CHECK-LABEL: fcos_approx(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r1, [fcos_approx_param_0];
-; CHECK-NEXT:    cos.approx.f32 %r2, %r1;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
-; CHECK-NEXT:    ret;
-  %r = tail call float @llvm.cos.f32(float %a)
-  ret float %r
-}
-
-define float @fsin_approx_ftz(float %a) #0 #1 {
+define float @fsin_approx_ftz(float %a) #1 {
 ; CHECK-LABEL: fsin_approx_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -301,11 +261,11 @@ define float @fsin_approx_ftz(float %a) #0 #1 {
 ; CHECK-NEXT:    sin.approx.ftz.f32 %r2, %r1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
-  %r = tail call float @llvm.sin.f32(float %a)
+  %r = tail call afn float @llvm.sin.f32(float %a)
   ret float %r
 }
 
-define float @fcos_approx_ftz(float %a) #0 #1 {
+define float @fcos_approx_ftz(float %a) #1 {
 ; CHECK-LABEL: fcos_approx_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -315,7 +275,7 @@ define float @fcos_approx_ftz(float %a) #0 #1 {
 ; CHECK-NEXT:    cos.approx.ftz.f32 %r2, %r1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r2;
 ; CHECK-NEXT:    ret;
-  %r = tail call float @llvm.cos.f32(float %a)
+  %r = tail call afn float @llvm.cos.f32(float %a)
   ret float %r
 }
 
@@ -423,7 +383,7 @@ define float @repeated_div_recip_allowed_ftz_sel(i1 %pred, float %a, float %b, f
   ret float %w
 }
 
-define float @repeated_div_fast(i1 %pred, float %a, float %b, float %divisor) #0 {
+define float @repeated_div_fast(i1 %pred, float %a, float %b, float %divisor) {
 ; CHECK-LABEL: repeated_div_fast(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -444,14 +404,14 @@ define float @repeated_div_fast(i1 %pred, float %a, float %b, float %divisor) #0
 ; CHECK-NEXT:    selp.f32 %r8, %r7, %r6, %p1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-NEXT:    ret;
-  %x = fdiv float %a, %divisor
-  %y = fdiv float %b, %divisor
-  %z = fmul float %x, %y
+  %x = fdiv afn arcp float %a, %divisor
+  %y = fdiv afn arcp contract float %b, %divisor
+  %z = fmul contract float %x, %y
   %w = select i1 %pred, float %z, float %y
   ret float %w
 }
 
-define float @repeated_div_fast_sel(i1 %pred, float %a, float %b, float %divisor) #0 {
+define float @repeated_div_fast_sel(i1 %pred, float %a, float %b, float %divisor) {
 ; CHECK-LABEL: repeated_div_fast_sel(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -469,13 +429,13 @@ define float @repeated_div_fast_sel(i1 %pred, float %a, float %b, float %divisor
 ; CHECK-NEXT:    div.approx.f32 %r5, %r3, %r4;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
-  %x = fdiv float %a, %divisor
-  %y = fdiv float %b, %divisor
+  %x = fdiv afn float %a, %divisor
+  %y = fdiv afn float %b, %divisor
   %w = select i1 %pred, float %x, float %y
   ret float %w
 }
 
-define float @repeated_div_fast_ftz(i1 %pred, float %a, float %b, float %divisor) #0 #1 {
+define float @repeated_div_fast_ftz(i1 %pred, float %a, float %b, float %divisor) #1 {
 ; CHECK-LABEL: repeated_div_fast_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -496,14 +456,14 @@ define float @repeated_div_fast_ftz(i1 %pred, float %a, float %b, float %divisor
 ; CHECK-NEXT:    selp.f32 %r8, %r7, %r6, %p1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-NEXT:    ret;
-  %x = fdiv float %a, %divisor
-  %y = fdiv float %b, %divisor
-  %z = fmul float %x, %y
+  %x = fdiv afn arcp float %a, %divisor
+  %y = fdiv afn arcp contract float %b, %divisor
+  %z = fmul contract float %x, %y
   %w = select i1 %pred, float %z, float %y
   ret float %w
 }
 
-define float @repeated_div_fast_ftz_sel(i1 %pred, float %a, float %b, float %divisor) #0 #1 {
+define float @repeated_div_fast_ftz_sel(i1 %pred, float %a, float %b, float %divisor) #1 {
 ; CHECK-LABEL: repeated_div_fast_ftz_sel(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -521,13 +481,13 @@ define float @repeated_div_fast_ftz_sel(i1 %pred, float %a, float %b, float %div
 ; CHECK-NEXT:    div.approx.ftz.f32 %r5, %r3, %r4;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
-  %x = fdiv float %a, %divisor
-  %y = fdiv float %b, %divisor
+  %x = fdiv afn float %a, %divisor
+  %y = fdiv afn float %b, %divisor
   %w = select i1 %pred, float %x, float %y
   ret float %w
 }
 
-define float @frem(float %a, float %b) #0 {
+define float @frem(float %a, float %b) {
 ; CHECK-LABEL: frem(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<7>;
@@ -541,11 +501,11 @@ define float @frem(float %a, float %b) #0 {
 ; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
-  %rem = frem float %a, %b
+  %rem = frem afn arcp contract ninf float %a, %b
   ret float %rem
 }
 
-define float @frem_ftz(float %a, float %b) #0 #1 {
+define float @frem_ftz(float %a, float %b) #1 {
 ; CHECK-LABEL: frem_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<7>;
@@ -559,11 +519,11 @@ define float @frem_ftz(float %a, float %b) #0 #1 {
 ; CHECK-NEXT:    fma.rn.ftz.f32 %r6, %r5, %r2, %r1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
-  %rem = frem float %a, %b
+  %rem = frem afn contract ninf float %a, %b
   ret float %rem
 }
 
-define double @frem_f64(double %a, double %b) #0 {
+define double @frem_f64(double %a, double %b) {
 ; CHECK-LABEL: frem_f64(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<7>;
@@ -577,9 +537,8 @@ define double @frem_f64(double %a, double %b) #0 {
 ; CHECK-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
 ; CHECK-NEXT:    ret;
-  %rem = frem double %a, %b
+  %rem = frem ninf double %a, %b
   ret double %rem
 }
 
-attributes #0 = { "unsafe-fp-math" = "true" }
 attributes #1 = { "denormal-fp-math-f32" = "preserve-sign" }
diff --git a/llvm/test/CodeGen/NVPTX/fma-assoc.ll b/llvm/test/CodeGen/NVPTX/fma-assoc.ll
index 1034c3eed3dc0..6693c9044ca2c 100644
--- a/llvm/test/CodeGen/NVPTX/fma-assoc.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-assoc.ll
@@ -20,10 +20,10 @@ define ptx_device float @t1_f32(float %x, float %y, float %z,
 ; CHECK-UNSAFE-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-UNSAFE-NEXT:    ret;
                                 float %u, float %v) {
-  %a = fmul float %x, %y
-  %b = fmul float %u, %v
-  %c = fadd float %a, %b
-  %d = fadd float %c, %z
+  %a = fmul reassoc float %x, %y
+  %b = fmul reassoc float %u, %v
+  %c = fadd reassoc float %a, %b
+  %d = fadd reassoc float %c, %z
   ret float %d
 }
 
@@ -43,10 +43,10 @@ define ptx_device double @t1_f64(double %x, double %y, double %z,
 ; CHECK-UNSAFE-NEXT:    st.param.b64 [func_retval0], %rd7;
 ; CHECK-UNSAFE-NEXT:    ret;
                                  double %u, double %v) {
-  %a = fmul double %x, %y
-  %b = fmul double %u, %v
-  %c = fadd double %a, %b
-  %d = fadd double %c, %z
+  %a = fmul reassoc double %x, %y
+  %b = fmul reassoc double %u, %v
+  %c = fadd reassoc double %a, %b
+  %d = fadd reassoc double %c, %z
   ret double %d
 }
 
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
index 2f1d7d6321438..6d983ba6bf0ff 100644
--- a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
@@ -9,7 +9,7 @@
 ; SM < 80 or (which needs PTX version >= 70) should not emit fma{.ftz}.relu
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_70 | FileCheck %s --check-prefixes=CHECK-SM70
 
-define half @fma_f16_no_nans(half %a, half %b, half %c) #0 {
+define half @fma_f16_no_nans(half %a, half %b, half %c) {
 ; CHECK-LABEL: fma_f16_no_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
@@ -49,14 +49,14 @@ define half @fma_f16_no_nans(half %a, half %b, half %c) #0 {
 ; CHECK-SM70-NEXT:    selp.b16 %rs6, %rs4, 0x0000, %p1;
 ; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs6;
 ; CHECK-SM70-NEXT:    ret;
-  %1 = call half @llvm.fma.f16(half %a, half %b, half %c)
+  %1 = call nnan half @llvm.fma.f16(half %a, half %b, half %c)
   %2 = fcmp ogt half %1, 0.0
-  %3 = select i1 %2, half %1, half 0.0
+  %3 = select nsz i1 %2, half %1, half 0.0
   ret half %3
 }
 
 ; FMA relu shouldn't be selected if the FMA operation has multiple uses
-define half @fma_f16_no_nans_multiple_uses_of_fma(half %a, half %b, half %c) #0 {
+define half @fma_f16_no_nans_multiple_uses_of_fma(half %a, half %b, half %c) {
 ; CHECK-LABEL: fma_f16_no_nans_multiple_uses_of_fma(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<8>;
@@ -103,13 +103,13 @@ define half @fma_f16_no_nans_multiple_uses_of_fma(half %a, half %b, half %c) #0
 ; CHECK-SM70-NEXT:    ret;
   %1 = call half @llvm.fma.f16(half %a, half %b, half %c)
   %2 = fcmp ogt half %1, 0.0
-  %3 = select i1 %2, half %1, half 0.0
-  %4 = fadd half %1, 7.0
-  %5 = fadd half %4, %1
+  %3 = select  i1 %2, half %1, half 0.0
+  %4 = fadd contract half %1, 7.0
+  %5 = fadd contract half %4, %1
   ret half %5
 }
 
-define half @fma_f16_maxnum_no_nans(half %a, half %b, half %c) #0 {
+define half @fma_f16_maxnum_no_nans(half %a, half %b, half %c) {
 ; CHECK-LABEL: fma_f16_maxnum_no_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
@@ -149,12 +149,12 @@ define half @fma_f16_maxnum_no_nans(half %a, half %b, half %c) #0 {
 ; CHECK-SM70-NEXT:    cvt.rn.f16.f32 %rs5, %r2;
 ; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs5;
 ; CHECK-SM70-NEXT:    ret;
-  %1 = call half @llvm.fma.f16(half %a, half %b, half %c)
-  %2 = call half @llvm.maxnum.f16(half %1, half 0.0)
+  %1 = call nnan half @llvm.fma.f16(half %a, half %b, half %c)
+  %2 = call nsz half @llvm.maxnum.f16(half %1, half 0.0)
   ret half %2
 }
 
-define bfloat @fma_bf16_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
+define bfloat @fma_bf16_no_nans(bfloat %a, bfloat %b, bfloat %c) {
 ; CHECK-LABEL: fma_bf16_no_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
@@ -205,14 +205,14 @@ define bfloat @fma_bf16_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
 ; CHECK-SM70-NEXT:    selp.b16 %rs2, %rs1, 0x0000, %p2;
 ; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %rs2;
 ; CHECK-SM70-NEXT:    ret;
-  %1 = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+  %1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
   %2 = fcmp ogt bfloat %1, 0.0
-  %3 = select i1 %2, bfloat %1, bfloat 0.0
+  %3 = select nsz i1 %2, bfloat %1, bfloat 0.0
   ret bfloat %3
 }
 
 ; FMA_relu shouldn't be selected if the FMA operation has multiple uses
-define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloat %c) #0 {
+define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloat %c) {
 ; CHECK-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<9>;
@@ -291,12 +291,12 @@ define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloa
   %1 = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
   %2 = fcmp ogt bfloat %1, 0.0
   %3 = select i1 %2, bfloat %1, bfloat 0.0
-  %4 = fadd bfloat %1, 7.0
-  %5 = fadd bfloat %4, %1
+  %4 = fadd contract bfloat %1, 7.0
+  %5 = fadd contract bfloat %4, %1
   ret bfloat %5
 }
 
-define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
+define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) {
 ; CHECK-LABEL: fma_bf16_maxnum_no_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<5>;
@@ -351,12 +351,12 @@ define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
 ; CHECK-SM70-NEXT:    shr.u32 %r20, %r19, 16;
 ; CHECK-SM70-NEXT:    st.param.b16 [func_retval0], %r20;
 ; CHECK-SM70-NEXT:    ret;
-  %1 = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
-  %2 = call bfloat @llvm.maxnum.bf16(bfloat %1, bfloat 0.0)
+  %1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+  %2 = call nsz bfloat @llvm.maxnum.bf16(bfloat %1, bfloat 0.0)
   ret bfloat %2
 }
 
-define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
 ; CHECK-LABEL: fma_f16x2_no_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<5>;
@@ -399,14 +399,14 @@ define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c
 ; CHECK-SM70-NEXT:    selp.b16 %rs4, %rs1, 0x0000, %p1;
 ; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs4, %rs3};
 ; CHECK-SM70-NEXT:    ret;
-  %1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+  %1 = call nnan <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
   %2 = fcmp ogt <2 x half> %1, <half 0.0, half 0.0>
-  %3 = select <2 x i1> %2, <2 x half> %1, <2 x half> <half 0.0, half 0.0>
+  %3 = select nsz <2 x i1> %2, <2 x half> %1, <2 x half> <half 0.0, half 0.0>
   ret <2 x half> %3
 }
 
 ; FMA relu shouldn't be selected if the FMA operation has multiple uses
-define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
 ; CHECK-LABEL: fma_f16x2_no_nans_multiple_uses_of_fma(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<8>;
@@ -454,12 +454,12 @@ define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x ha
   %1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
   %2 = fcmp ogt <2 x half> %1, <half 0.0, half 0.0>
   %3 = select <2 x i1> %2, <2 x half> %1, <2 x half> <half 0.0, half 0.0>
-  %4 = fadd <2 x half> %1, <half 7.0, half 7.0>
-  %5 = fadd <2 x half> %4, %1
+  %4 = fadd contract <2 x half> %1, <half 7.0, half 7.0>
+  %5 = fadd contract <2 x half> %4, %1
   ret <2 x half> %5
 }
 
-define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
 ; CHECK-LABEL: fma_f16x2_maxnum_no_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<5>;
@@ -504,12 +504,12 @@ define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x h
 ; CHECK-SM70-NEXT:    mov.b32 %r9, {%rs4, %rs3};
 ; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r9;
 ; CHECK-SM70-NEXT:    ret;
-  %1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
-  %2 = call <2 x half> @llvm.maxnum.f16x2(<2 x half> %1, <2 x half> <half 0.0, half 0.0>)
+  %1 = call nnan <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+  %2 = call nsz <2 x half> @llvm.maxnum.f16x2(<2 x half> %1, <2 x half> <half 0.0, half 0.0>)
   ret <2 x half> %2
 }
 
-define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
 ; CHECK-LABEL: fma_bf16x2_no_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<5>;
@@ -580,14 +580,14 @@ define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x b
 ; CHECK-SM70-NEXT:    selp.b16 %rs10, %rs7, 0x0000, %p3;
 ; CHECK-SM70-NEXT:    st.param.v2.b16 [func_retval0], {%rs10, %rs9};
 ; CHECK-SM70-NEXT:    ret;
-  %1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+  %1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
   %2 = fcmp ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
-  %3 = select <2 x i1> %2, <2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+  %3 = select nsz <2 x i1> %2, <2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
   ret <2 x bfloat> %3
 }
 
 ; FMA_relu shouldn't be selected if the FMA operation has multiple uses
-define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
 ; CHECK-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<9>;
@@ -707,12 +707,12 @@ define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2
   %1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
   %2 = fcmp ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
   %3 = select <2 x i1> %2, <2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
-  %4 = fadd <2 x bfloat> %1, <bfloat 7.0, bfloat 7.0>
-  %5 = fadd <2 x bfloat> %4, %1
+  %4 = fadd contract <2 x bfloat> %1, <bfloat 7.0, bfloat 7.0>
+  %5 = fadd contract <2 x bfloat> %4, %1
   ret <2 x bfloat> %5
 }
 
-define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
 ; CHECK-LABEL: fma_bf16x2_maxnum_no_nans(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<5>;
@@ -792,10 +792,7 @@ define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b,
 ; CHECK-SM70-NEXT:    prmt.b32 %r39, %r38, %r31, 0x7632U;
 ; CHECK-SM70-NEXT:    st.param.b32 [func_retval0], %r39;
 ; CHECK-SM70-NEXT:    ret;
-  %1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
-  %2 = call <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
+  %1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+  %2 = call nsz <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
   ret <2 x bfloat> %2
 }
-
-attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "unsafe-fp-math"="true" }
-attributes #1 = { "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/NVPTX/fma.ll b/llvm/test/CodeGen/NVPTX/fma.ll
index 87274aa759bea..ba4bb76d25113 100644
--- a/llvm/test/CodeGen/NVPTX/fma.ll
+++ b/llvm/test/CodeGen/NVPTX/fma.ll
@@ -25,7 +25,7 @@ define ptx_device float @t1_f32(float %x, float %y, float %z) {
 define ptx_device float @t2_f32(float %x, float %y, float %z, float %w) {
 ; CHECK-LABEL: t2_f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [t2_f32_param_0];
@@ -72,7 +72,7 @@ define ptx_device double @t1_f64(double %x, double %y, double %z) {
 define ptx_device double @t2_f64(double %x, double %y, double %z, double %w) {
 ; CHECK-LABEL: t2_f64(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<9>;
+; CHECK-NEXT:    .reg .b64 %rd<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [t2_f64_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/forward-ld-param.ll b/llvm/test/CodeGen/NVPTX/forward-ld-param.ll
index 636e12bf98943..4f1454d3788a4 100644
--- a/llvm/test/CodeGen/NVPTX/forward-ld-param.ll
+++ b/llvm/test/CodeGen/NVPTX/forward-ld-param.ll
@@ -7,7 +7,6 @@ define i32 @test_ld_param_const(ptr byval(i32) %a) {
 ; CHECK-LABEL: test_ld_param_const(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_ld_param_const_param_0+4];
@@ -61,7 +60,6 @@ define void @test_ld_param_byval(ptr byval(i32) %a) {
 ; CHECK-LABEL: test_ld_param_byval(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    { // callseq 1, 0
@@ -98,8 +96,7 @@ define i32 @test_multi_block(ptr byval([10 x i32]) %a, i1 %p) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<3>;
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b8 %rs1, [test_multi_block_param_1];
@@ -108,12 +105,12 @@ define i32 @test_multi_block(ptr byval([10 x i32]) %a, i1 %p) {
 ; CHECK-NEXT:    not.pred %p2, %p1;
 ; CHECK-NEXT:    @%p2 bra $L__BB5_2;
 ; CHECK-NEXT:  // %bb.1: // %if
-; CHECK-NEXT:    ld.param.b32 %r4, [test_multi_block_param_0+4];
+; CHECK-NEXT:    ld.param.b32 %r1, [test_multi_block_param_0+4];
 ; CHECK-NEXT:    bra.uni $L__BB5_3;
 ; CHECK-NEXT:  $L__BB5_2: // %else
-; CHECK-NEXT:    ld.param.b32 %r4, [test_multi_block_param_0+8];
+; CHECK-NEXT:    ld.param.b32 %r1, [test_multi_block_param_0+8];
 ; CHECK-NEXT:  $L__BB5_3: // %end
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r1;
 ; CHECK-NEXT:    ret;
   br i1 %p, label %if, label %else
 if:
diff --git a/llvm/test/CodeGen/NVPTX/frameindex-lifetime.ll b/llvm/test/CodeGen/NVPTX/frameindex-lifetime.ll
index 42655538cc7ad..9c564ffe7037e 100644
--- a/llvm/test/CodeGen/NVPTX/frameindex-lifetime.ll
+++ b/llvm/test/CodeGen/NVPTX/frameindex-lifetime.ll
@@ -44,8 +44,8 @@ declare void @bar(ptr)
 
 define void @foo() {
   %p = alloca i32
-  call void @llvm.lifetime.start(i64 4, ptr %p)
+  call void @llvm.lifetime.start(ptr %p)
   call void @bar(ptr %p)
-  call void @llvm.lifetime.end(i64 4, ptr %p)
+  call void @llvm.lifetime.end(ptr %p)
   ret void
 }
diff --git a/llvm/test/CodeGen/NVPTX/frem.ll b/llvm/test/CodeGen/NVPTX/frem.ll
index 5805aed1bebe6..d30c72cef83d5 100644
--- a/llvm/test/CodeGen/NVPTX/frem.ll
+++ b/llvm/test/CodeGen/NVPTX/frem.ll
@@ -1,313 +1,316 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s --enable-unsafe-fp-math -mcpu=sm_60 | FileCheck %s --check-prefixes=FAST
-; RUN: llc < %s -mcpu=sm_60 | FileCheck %s --check-prefixes=NORMAL
+; RUN: llc < %s -mcpu=sm_60 | FileCheck %s
 
 
 target triple = "nvptx64-unknown-cuda"
 
 define half @frem_f16(half %a, half %b) {
-; FAST-LABEL: frem_f16(
-; FAST:       {
-; FAST-NEXT:    .reg .b16 %rs<4>;
-; FAST-NEXT:    .reg .b32 %r<7>;
-; FAST-EMPTY:
-; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b16 %rs1, [frem_f16_param_0];
-; FAST-NEXT:    ld.param.b16 %rs2, [frem_f16_param_1];
-; FAST-NEXT:    cvt.f32.f16 %r1, %rs2;
-; FAST-NEXT:    cvt.f32.f16 %r2, %rs1;
-; FAST-NEXT:    div.approx.f32 %r3, %r2, %r1;
-; FAST-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; FAST-NEXT:    neg.f32 %r5, %r4;
-; FAST-NEXT:    fma.rn.f32 %r6, %r5, %r1, %r2;
-; FAST-NEXT:    cvt.rn.f16.f32 %rs3, %r6;
-; FAST-NEXT:    st.param.b16 [func_retval0], %rs3;
-; FAST-NEXT:    ret;
-;
-; NORMAL-LABEL: frem_f16(
-; NORMAL:       {
-; NORMAL-NEXT:    .reg .pred %p<2>;
-; NORMAL-NEXT:    .reg .b16 %rs<4>;
-; NORMAL-NEXT:    .reg .b32 %r<8>;
-; NORMAL-EMPTY:
-; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b16 %rs1, [frem_f16_param_0];
-; NORMAL-NEXT:    ld.param.b16 %rs2, [frem_f16_param_1];
-; NORMAL-NEXT:    cvt.f32.f16 %r1, %rs2;
-; NORMAL-NEXT:    cvt.f32.f16 %r2, %rs1;
-; NORMAL-NEXT:    div.rn.f32 %r3, %r2, %r1;
-; NORMAL-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; NORMAL-NEXT:    neg.f32 %r5, %r4;
-; NORMAL-NEXT:    fma.rn.f32 %r6, %r5, %r1, %r2;
-; NORMAL-NEXT:    testp.infinite.f32 %p1, %r1;
-; NORMAL-NEXT:    selp.f32 %r7, %r2, %r6, %p1;
-; NORMAL-NEXT:    cvt.rn.f16.f32 %rs3, %r7;
-; NORMAL-NEXT:    st.param.b16 [func_retval0], %rs3;
-; NORMAL-NEXT:    ret;
+; CHECK-LABEL: frem_f16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [frem_f16_param_0];
+; CHECK-NEXT:    ld.param.b16 %rs2, [frem_f16_param_1];
+; CHECK-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NEXT:    div.rn.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r1, %r2;
+; CHECK-NEXT:    testp.infinite.f32 %p1, %r1;
+; CHECK-NEXT:    selp.f32 %r7, %r2, %r6, %p1;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r7;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
   %r = frem half %a, %b
   ret half %r
 }
 
+define half @frem_f16_fast(half %a, half %b) {
+; CHECK-LABEL: frem_f16_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [frem_f16_fast_param_0];
+; CHECK-NEXT:    ld.param.b16 %rs2, [frem_f16_fast_param_1];
+; CHECK-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NEXT:    div.approx.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r1, %r2;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r6;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %r = frem afn ninf half %a, %b
+  ret half %r
+}
+
 define float @frem_f32(float %a, float %b) {
-; FAST-LABEL: frem_f32(
-; FAST:       {
-; FAST-NEXT:    .reg .b32 %r<7>;
-; FAST-EMPTY:
-; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_param_0];
-; FAST-NEXT:    ld.param.b32 %r2, [frem_f32_param_1];
-; FAST-NEXT:    div.approx.f32 %r3, %r1, %r2;
-; FAST-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; FAST-NEXT:    neg.f32 %r5, %r4;
-; FAST-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
-; FAST-NEXT:    st.param.b32 [func_retval0], %r6;
-; FAST-NEXT:    ret;
-;
-; NORMAL-LABEL: frem_f32(
-; NORMAL:       {
-; NORMAL-NEXT:    .reg .pred %p<2>;
-; NORMAL-NEXT:    .reg .b32 %r<8>;
-; NORMAL-EMPTY:
-; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_param_0];
-; NORMAL-NEXT:    ld.param.b32 %r2, [frem_f32_param_1];
-; NORMAL-NEXT:    div.rn.f32 %r3, %r1, %r2;
-; NORMAL-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; NORMAL-NEXT:    neg.f32 %r5, %r4;
-; NORMAL-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
-; NORMAL-NEXT:    testp.infinite.f32 %p1, %r2;
-; NORMAL-NEXT:    selp.f32 %r7, %r1, %r6, %p1;
-; NORMAL-NEXT:    st.param.b32 [func_retval0], %r7;
-; NORMAL-NEXT:    ret;
+; CHECK-LABEL: frem_f32(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_f32_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [frem_f32_param_1];
+; CHECK-NEXT:    div.rn.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
+; CHECK-NEXT:    testp.infinite.f32 %p1, %r2;
+; CHECK-NEXT:    selp.f32 %r7, %r1, %r6, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT:    ret;
   %r = frem float %a, %b
   ret float %r
 }
 
+define float @frem_f32_fast(float %a, float %b) {
+; CHECK-LABEL: frem_f32_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_f32_fast_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [frem_f32_fast_param_1];
+; CHECK-NEXT:    div.approx.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
+  %r = frem afn ninf float %a, %b
+  ret float %r
+}
+
 define double @frem_f64(double %a, double %b) {
-; FAST-LABEL: frem_f64(
-; FAST:       {
-; FAST-NEXT:    .reg .b64 %rd<7>;
-; FAST-EMPTY:
-; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b64 %rd1, [frem_f64_param_0];
-; FAST-NEXT:    ld.param.b64 %rd2, [frem_f64_param_1];
-; FAST-NEXT:    div.rn.f64 %rd3, %rd1, %rd2;
-; FAST-NEXT:    cvt.rzi.f64.f64 %rd4, %rd3;
-; FAST-NEXT:    neg.f64 %rd5, %rd4;
-; FAST-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
-; FAST-NEXT:    st.param.b64 [func_retval0], %rd6;
-; FAST-NEXT:    ret;
-;
-; NORMAL-LABEL: frem_f64(
-; NORMAL:       {
-; NORMAL-NEXT:    .reg .pred %p<2>;
-; NORMAL-NEXT:    .reg .b64 %rd<8>;
-; NORMAL-EMPTY:
-; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b64 %rd1, [frem_f64_param_0];
-; NORMAL-NEXT:    ld.param.b64 %rd2, [frem_f64_param_1];
-; NORMAL-NEXT:    div.rn.f64 %rd3, %rd1, %rd2;
-; NORMAL-NEXT:    cvt.rzi.f64.f64 %rd4, %rd3;
-; NORMAL-NEXT:    neg.f64 %rd5, %rd4;
-; NORMAL-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
-; NORMAL-NEXT:    testp.infinite.f64 %p1, %rd2;
-; NORMAL-NEXT:    selp.f64 %rd7, %rd1, %rd6, %p1;
-; NORMAL-NEXT:    st.param.b64 [func_retval0], %rd7;
-; NORMAL-NEXT:    ret;
+; CHECK-LABEL: frem_f64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b64 %rd<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [frem_f64_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [frem_f64_param_1];
+; CHECK-NEXT:    div.rn.f64 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    cvt.rzi.f64.f64 %rd4, %rd3;
+; CHECK-NEXT:    neg.f64 %rd5, %rd4;
+; CHECK-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
+; CHECK-NEXT:    testp.infinite.f64 %p1, %rd2;
+; CHECK-NEXT:    selp.f64 %rd7, %rd1, %rd6, %p1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd7;
+; CHECK-NEXT:    ret;
   %r = frem double %a, %b
   ret double %r
 }
 
+define double @frem_f64_fast(double %a, double %b) {
+; CHECK-LABEL: frem_f64_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [frem_f64_fast_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [frem_f64_fast_param_1];
+; CHECK-NEXT:    div.rn.f64 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    cvt.rzi.f64.f64 %rd4, %rd3;
+; CHECK-NEXT:    neg.f64 %rd5, %rd4;
+; CHECK-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
+; CHECK-NEXT:    ret;
+  %r = frem afn ninf double %a, %b
+  ret double %r
+}
+
 define half @frem_f16_ninf(half %a, half %b) {
-; FAST-LABEL: frem_f16_ninf(
-; FAST:       {
-; FAST-NEXT:    .reg .b16 %rs<4>;
-; FAST-NEXT:    .reg .b32 %r<7>;
-; FAST-EMPTY:
-; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b16 %rs1, [frem_f16_ninf_param_0];
-; FAST-NEXT:    ld.param.b16 %rs2, [frem_f16_ninf_param_1];
-; FAST-NEXT:    cvt.f32.f16 %r1, %rs2;
-; FAST-NEXT:    cvt.f32.f16 %r2, %rs1;
-; FAST-NEXT:    div.approx.f32 %r3, %r2, %r1;
-; FAST-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; FAST-NEXT:    neg.f32 %r5, %r4;
-; FAST-NEXT:    fma.rn.f32 %r6, %r5, %r1, %r2;
-; FAST-NEXT:    cvt.rn.f16.f32 %rs3, %r6;
-; FAST-NEXT:    st.param.b16 [func_retval0], %rs3;
-; FAST-NEXT:    ret;
-;
-; NORMAL-LABEL: frem_f16_ninf(
-; NORMAL:       {
-; NORMAL-NEXT:    .reg .b16 %rs<4>;
-; NORMAL-NEXT:    .reg .b32 %r<7>;
-; NORMAL-EMPTY:
-; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b16 %rs1, [frem_f16_ninf_param_0];
-; NORMAL-NEXT:    ld.param.b16 %rs2, [frem_f16_ninf_param_1];
-; NORMAL-NEXT:    cvt.f32.f16 %r1, %rs2;
-; NORMAL-NEXT:    cvt.f32.f16 %r2, %rs1;
-; NORMAL-NEXT:    div.rn.f32 %r3, %r2, %r1;
-; NORMAL-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; NORMAL-NEXT:    neg.f32 %r5, %r4;
-; NORMAL-NEXT:    fma.rn.f32 %r6, %r5, %r1, %r2;
-; NORMAL-NEXT:    cvt.rn.f16.f32 %rs3, %r6;
-; NORMAL-NEXT:    st.param.b16 [func_retval0], %rs3;
-; NORMAL-NEXT:    ret;
+; CHECK-LABEL: frem_f16_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [frem_f16_ninf_param_0];
+; CHECK-NEXT:    ld.param.b16 %rs2, [frem_f16_ninf_param_1];
+; CHECK-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NEXT:    div.rn.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r1, %r2;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r6;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
   %r = frem ninf half %a, %b
   ret half %r
 }
 
+define half @frem_f16_ninf_fast(half %a, half %b) {
+; CHECK-LABEL: frem_f16_ninf_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [frem_f16_ninf_fast_param_0];
+; CHECK-NEXT:    ld.param.b16 %rs2, [frem_f16_ninf_fast_param_1];
+; CHECK-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NEXT:    div.approx.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r1, %r2;
+; CHECK-NEXT:    cvt.rn.f16.f32 %rs3, %r6;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
+  %r = frem afn ninf half %a, %b
+  ret half %r
+}
+
 define float @frem_f32_ninf(float %a, float %b) {
-; FAST-LABEL: frem_f32_ninf(
-; FAST:       {
-; FAST-NEXT:    .reg .b32 %r<7>;
-; FAST-EMPTY:
-; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_ninf_param_0];
-; FAST-NEXT:    ld.param.b32 %r2, [frem_f32_ninf_param_1];
-; FAST-NEXT:    div.approx.f32 %r3, %r1, %r2;
-; FAST-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; FAST-NEXT:    neg.f32 %r5, %r4;
-; FAST-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
-; FAST-NEXT:    st.param.b32 [func_retval0], %r6;
-; FAST-NEXT:    ret;
-;
-; NORMAL-LABEL: frem_f32_ninf(
-; NORMAL:       {
-; NORMAL-NEXT:    .reg .b32 %r<7>;
-; NORMAL-EMPTY:
-; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_ninf_param_0];
-; NORMAL-NEXT:    ld.param.b32 %r2, [frem_f32_ninf_param_1];
-; NORMAL-NEXT:    div.rn.f32 %r3, %r1, %r2;
-; NORMAL-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; NORMAL-NEXT:    neg.f32 %r5, %r4;
-; NORMAL-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
-; NORMAL-NEXT:    st.param.b32 [func_retval0], %r6;
-; NORMAL-NEXT:    ret;
+; CHECK-LABEL: frem_f32_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_f32_ninf_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [frem_f32_ninf_param_1];
+; CHECK-NEXT:    div.rn.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
   %r = frem ninf float %a, %b
   ret float %r
 }
 
+define float @frem_f32_ninf_fast(float %a, float %b) {
+; CHECK-LABEL: frem_f32_ninf_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_f32_ninf_fast_param_0];
+; CHECK-NEXT:    ld.param.b32 %r2, [frem_f32_ninf_fast_param_1];
+; CHECK-NEXT:    div.approx.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
+  %r = frem  afn ninf float %a, %b
+  ret float %r
+}
+
 define double @frem_f64_ninf(double %a, double %b) {
-; FAST-LABEL: frem_f64_ninf(
-; FAST:       {
-; FAST-NEXT:    .reg .b64 %rd<7>;
-; FAST-EMPTY:
-; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b64 %rd1, [frem_f64_ninf_param_0];
-; FAST-NEXT:    ld.param.b64 %rd2, [frem_f64_ninf_param_1];
-; FAST-NEXT:    div.rn.f64 %rd3, %rd1, %rd2;
-; FAST-NEXT:    cvt.rzi.f64.f64 %rd4, %rd3;
-; FAST-NEXT:    neg.f64 %rd5, %rd4;
-; FAST-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
-; FAST-NEXT:    st.param.b64 [func_retval0], %rd6;
-; FAST-NEXT:    ret;
-;
-; NORMAL-LABEL: frem_f64_ninf(
-; NORMAL:       {
-; NORMAL-NEXT:    .reg .b64 %rd<7>;
-; NORMAL-EMPTY:
-; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b64 %rd1, [frem_f64_ninf_param_0];
-; NORMAL-NEXT:    ld.param.b64 %rd2, [frem_f64_ninf_param_1];
-; NORMAL-NEXT:    div.rn.f64 %rd3, %rd1, %rd2;
-; NORMAL-NEXT:    cvt.rzi.f64.f64 %rd4, %rd3;
-; NORMAL-NEXT:    neg.f64 %rd5, %rd4;
-; NORMAL-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
-; NORMAL-NEXT:    st.param.b64 [func_retval0], %rd6;
-; NORMAL-NEXT:    ret;
+; CHECK-LABEL: frem_f64_ninf(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [frem_f64_ninf_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [frem_f64_ninf_param_1];
+; CHECK-NEXT:    div.rn.f64 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    cvt.rzi.f64.f64 %rd4, %rd3;
+; CHECK-NEXT:    neg.f64 %rd5, %rd4;
+; CHECK-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
+; CHECK-NEXT:    ret;
   %r = frem ninf double %a, %b
   ret double %r
 }
 
+define double @frem_f64_ninf_fast(double %a, double %b) {
+; CHECK-LABEL: frem_f64_ninf_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b64 %rd<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b64 %rd1, [frem_f64_ninf_fast_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd2, [frem_f64_ninf_fast_param_1];
+; CHECK-NEXT:    div.rn.f64 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    cvt.rzi.f64.f64 %rd4, %rd3;
+; CHECK-NEXT:    neg.f64 %rd5, %rd4;
+; CHECK-NEXT:    fma.rn.f64 %rd6, %rd5, %rd2, %rd1;
+; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
+; CHECK-NEXT:    ret;
+  %r = frem afn ninf double %a, %b
+  ret double %r
+}
+
 define float @frem_f32_imm1_fast(float %a) {
-; FAST-LABEL: frem_f32_imm1_fast(
-; FAST:       {
-; FAST-NEXT:    .reg .b32 %r<5>;
-; FAST-EMPTY:
-; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_fast_param_0];
-; FAST-NEXT:    mul.f32 %r2, %r1, 0f3E124925;
-; FAST-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
-; FAST-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
-; FAST-NEXT:    st.param.b32 [func_retval0], %r4;
-; FAST-NEXT:    ret;
-;
-; NORMAL-LABEL: frem_f32_imm1_fast(
-; NORMAL:       {
-; NORMAL-NEXT:    .reg .b32 %r<5>;
-; NORMAL-EMPTY:
-; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_fast_param_0];
-; NORMAL-NEXT:    mul.rn.f32 %r2, %r1, 0f3E124925;
-; NORMAL-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
-; NORMAL-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
-; NORMAL-NEXT:    st.param.b32 [func_retval0], %r4;
-; NORMAL-NEXT:    ret;
+; CHECK-LABEL: frem_f32_imm1_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_fast_param_0];
+; CHECK-NEXT:    mul.rn.f32 %r2, %r1, 0f3E124925;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
+; CHECK-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
   %r = frem arcp float %a, 7.0
   ret float %r
 }
 define float @frem_f32_imm1_normal(float %a) {
-; FAST-LABEL: frem_f32_imm1_normal(
-; FAST:       {
-; FAST-NEXT:    .reg .b32 %r<5>;
-; FAST-EMPTY:
-; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_normal_param_0];
-; FAST-NEXT:    div.approx.f32 %r2, %r1, 0f40E00000;
-; FAST-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
-; FAST-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
-; FAST-NEXT:    st.param.b32 [func_retval0], %r4;
-; FAST-NEXT:    ret;
-;
-; NORMAL-LABEL: frem_f32_imm1_normal(
-; NORMAL:       {
-; NORMAL-NEXT:    .reg .b32 %r<5>;
-; NORMAL-EMPTY:
-; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_normal_param_0];
-; NORMAL-NEXT:    div.rn.f32 %r2, %r1, 0f40E00000;
-; NORMAL-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
-; NORMAL-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
-; NORMAL-NEXT:    st.param.b32 [func_retval0], %r4;
-; NORMAL-NEXT:    ret;
+; CHECK-LABEL: frem_f32_imm1_normal(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_f32_imm1_normal_param_0];
+; CHECK-NEXT:    div.rn.f32 %r2, %r1, 0f40E00000;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r3, %r2;
+; CHECK-NEXT:    fma.rn.f32 %r4, %r3, 0fC0E00000, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    ret;
   %r = frem float %a, 7.0
   ret float %r
 }
 
 define float @frem_f32_imm2(float %a) {
-; FAST-LABEL: frem_f32_imm2(
-; FAST:       {
-; FAST-NEXT:    .reg .b32 %r<7>;
-; FAST-EMPTY:
-; FAST-NEXT:  // %bb.0:
-; FAST-NEXT:    ld.param.b32 %r1, [frem_f32_imm2_param_0];
-; FAST-NEXT:    mov.b32 %r2, 0f40E00000;
-; FAST-NEXT:    div.approx.f32 %r3, %r2, %r1;
-; FAST-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; FAST-NEXT:    neg.f32 %r5, %r4;
-; FAST-NEXT:    fma.rn.f32 %r6, %r5, %r1, 0f40E00000;
-; FAST-NEXT:    st.param.b32 [func_retval0], %r6;
-; FAST-NEXT:    ret;
-;
-; NORMAL-LABEL: frem_f32_imm2(
-; NORMAL:       {
-; NORMAL-NEXT:    .reg .pred %p<2>;
-; NORMAL-NEXT:    .reg .b32 %r<8>;
-; NORMAL-EMPTY:
-; NORMAL-NEXT:  // %bb.0:
-; NORMAL-NEXT:    ld.param.b32 %r1, [frem_f32_imm2_param_0];
-; NORMAL-NEXT:    mov.b32 %r2, 0f40E00000;
-; NORMAL-NEXT:    div.rn.f32 %r3, %r2, %r1;
-; NORMAL-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
-; NORMAL-NEXT:    neg.f32 %r5, %r4;
-; NORMAL-NEXT:    fma.rn.f32 %r6, %r5, %r1, 0f40E00000;
-; NORMAL-NEXT:    testp.infinite.f32 %p1, %r1;
-; NORMAL-NEXT:    selp.f32 %r7, 0f40E00000, %r6, %p1;
-; NORMAL-NEXT:    st.param.b32 [func_retval0], %r7;
-; NORMAL-NEXT:    ret;
+; CHECK-LABEL: frem_f32_imm2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_f32_imm2_param_0];
+; CHECK-NEXT:    mov.b32 %r2, 0f40E00000;
+; CHECK-NEXT:    div.rn.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r1, 0f40E00000;
+; CHECK-NEXT:    testp.infinite.f32 %p1, %r1;
+; CHECK-NEXT:    selp.f32 %r7, 0f40E00000, %r6, %p1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT:    ret;
   %r = frem float 7.0, %a
   ret float %r
 }
+
+define float @frem_f32_imm2_fast(float %a) {
+; CHECK-LABEL: frem_f32_imm2_fast(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b32 %r1, [frem_f32_imm2_fast_param_0];
+; CHECK-NEXT:    mov.b32 %r2, 0f40E00000;
+; CHECK-NEXT:    div.approx.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    cvt.rzi.f32.f32 %r4, %r3;
+; CHECK-NEXT:    neg.f32 %r5, %r4;
+; CHECK-NEXT:    fma.rn.f32 %r6, %r5, %r1, 0f40E00000;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT:    ret;
+  %r = frem afn ninf float 7.0, %a
+  ret float %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/i1-select.ll b/llvm/test/CodeGen/NVPTX/i1-select.ll
index df32e2a4cfad2..264f38021e1de 100644
--- a/llvm/test/CodeGen/NVPTX/i1-select.ll
+++ b/llvm/test/CodeGen/NVPTX/i1-select.ll
@@ -66,22 +66,22 @@ define i32 @test_select_i1_basic(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %fals
 ; CHECK-LABEL: test_select_i1_basic(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<4>;
-; CHECK-NEXT:    .reg .b32 %r<12>;
+; CHECK-NEXT:    .reg .b32 %r<10>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_select_i1_basic_param_0];
 ; CHECK-NEXT:    ld.param.b32 %r2, [test_select_i1_basic_param_1];
-; CHECK-NEXT:    or.b32 %r4, %r1, %r2;
+; CHECK-NEXT:    or.b32 %r3, %r1, %r2;
 ; CHECK-NEXT:    setp.ne.b32 %p1, %r1, 0;
-; CHECK-NEXT:    ld.param.b32 %r5, [test_select_i1_basic_param_2];
-; CHECK-NEXT:    setp.eq.b32 %p2, %r5, 0;
-; CHECK-NEXT:    ld.param.b32 %r7, [test_select_i1_basic_param_3];
-; CHECK-NEXT:    setp.eq.b32 %p3, %r4, 0;
-; CHECK-NEXT:    ld.param.b32 %r8, [test_select_i1_basic_param_4];
-; CHECK-NEXT:    selp.b32 %r9, %r7, %r8, %p2;
-; CHECK-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
-; CHECK-NEXT:    selp.b32 %r11, %r7, %r10, %p3;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r11;
+; CHECK-NEXT:    ld.param.b32 %r4, [test_select_i1_basic_param_2];
+; CHECK-NEXT:    setp.eq.b32 %p2, %r4, 0;
+; CHECK-NEXT:    ld.param.b32 %r5, [test_select_i1_basic_param_3];
+; CHECK-NEXT:    setp.eq.b32 %p3, %r3, 0;
+; CHECK-NEXT:    ld.param.b32 %r6, [test_select_i1_basic_param_4];
+; CHECK-NEXT:    selp.b32 %r7, %r5, %r6, %p2;
+; CHECK-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; CHECK-NEXT:    selp.b32 %r9, %r5, %r8, %p3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r9;
 ; CHECK-NEXT:    ret;
   %b1 = icmp eq i32 %v1, 0
   %b2 = icmp eq i32 %v2, 0
@@ -94,7 +94,7 @@ define i32 @test_select_i1_basic(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %fals
 define i32 @test_select_i1_basic_folding(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %false) {
 ; CHECK-LABEL: test_select_i1_basic_folding(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .pred %p<13>;
+; CHECK-NEXT:    .reg .pred %p<11>;
 ; CHECK-NEXT:    .reg .b32 %r<7>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
@@ -106,14 +106,14 @@ define i32 @test_select_i1_basic_folding(i32 %v1, i32 %v2, i32 %v3, i32 %true, i
 ; CHECK-NEXT:    ld.param.b32 %r3, [test_select_i1_basic_folding_param_2];
 ; CHECK-NEXT:    setp.eq.b32 %p4, %r3, 0;
 ; CHECK-NEXT:    ld.param.b32 %r4, [test_select_i1_basic_folding_param_3];
-; CHECK-NEXT:    xor.pred %p6, %p1, %p3;
+; CHECK-NEXT:    xor.pred %p5, %p1, %p3;
 ; CHECK-NEXT:    ld.param.b32 %r5, [test_select_i1_basic_folding_param_4];
-; CHECK-NEXT:    and.pred %p8, %p6, %p4;
-; CHECK-NEXT:    and.pred %p9, %p2, %p4;
-; CHECK-NEXT:    and.pred %p10, %p3, %p8;
-; CHECK-NEXT:    or.pred %p11, %p10, %p9;
-; CHECK-NEXT:    xor.pred %p12, %p11, %p3;
-; CHECK-NEXT:    selp.b32 %r6, %r4, %r5, %p12;
+; CHECK-NEXT:    and.pred %p6, %p5, %p4;
+; CHECK-NEXT:    and.pred %p7, %p2, %p4;
+; CHECK-NEXT:    and.pred %p8, %p3, %p6;
+; CHECK-NEXT:    or.pred %p9, %p8, %p7;
+; CHECK-NEXT:    xor.pred %p10, %p9, %p3;
+; CHECK-NEXT:    selp.b32 %r6, %r4, %r5, %p10;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
   %b1 = icmp eq i32 %v1, 0
diff --git a/llvm/test/CodeGen/NVPTX/i128-array.ll b/llvm/test/CodeGen/NVPTX/i128-array.ll
index 3bb9c6aec51ac..7bd8a0021f1b5 100644
--- a/llvm/test/CodeGen/NVPTX/i128-array.ll
+++ b/llvm/test/CodeGen/NVPTX/i128-array.ll
@@ -27,13 +27,13 @@ define [2 x i128] @foo(i64 %a, i32 %b) {
 define [2 x i128] @foo2(ptr byval([2 x i128]) %a) {
 ; CHECK-LABEL: foo2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<7>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [foo2_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [foo2_param_0+16];
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd4};
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd5, %rd6};
+; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [foo2_param_0];
+; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [foo2_param_0+16];
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd3, %rd4};
 ; CHECK-NEXT:    ret;
   %ptr0 = getelementptr [2 x i128], ptr %a, i64 0, i32 0
   %1 = load i128, i128* %ptr0
diff --git a/llvm/test/CodeGen/NVPTX/i128.ll b/llvm/test/CodeGen/NVPTX/i128.ll
index 44d85589b5056..cdbbabe3e3b05 100644
--- a/llvm/test/CodeGen/NVPTX/i128.ll
+++ b/llvm/test/CodeGen/NVPTX/i128.ll
@@ -7,137 +7,137 @@ define i128 @srem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<20>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<127>;
+; CHECK-NEXT:    .reg .b64 %rd<79>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
-; CHECK-NEXT:    ld.param.v2.b64 {%rd45, %rd46}, [srem_i128_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd49, %rd50}, [srem_i128_param_1];
-; CHECK-NEXT:    shr.s64 %rd2, %rd46, 63;
-; CHECK-NEXT:    sub.cc.s64 %rd51, 0, %rd45;
-; CHECK-NEXT:    subc.cc.s64 %rd52, 0, %rd46;
-; CHECK-NEXT:    setp.lt.s64 %p1, %rd46, 0;
-; CHECK-NEXT:    selp.b64 %rd4, %rd52, %rd46, %p1;
-; CHECK-NEXT:    selp.b64 %rd3, %rd51, %rd45, %p1;
-; CHECK-NEXT:    sub.cc.s64 %rd53, 0, %rd49;
-; CHECK-NEXT:    subc.cc.s64 %rd54, 0, %rd50;
-; CHECK-NEXT:    setp.lt.s64 %p2, %rd50, 0;
-; CHECK-NEXT:    selp.b64 %rd6, %rd54, %rd50, %p2;
-; CHECK-NEXT:    selp.b64 %rd5, %rd53, %rd49, %p2;
-; CHECK-NEXT:    or.b64 %rd55, %rd5, %rd6;
-; CHECK-NEXT:    setp.eq.b64 %p3, %rd55, 0;
-; CHECK-NEXT:    or.b64 %rd56, %rd3, %rd4;
-; CHECK-NEXT:    setp.eq.b64 %p4, %rd56, 0;
+; CHECK-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [srem_i128_param_0];
+; CHECK-NEXT:    ld.param.v2.b64 {%rd10, %rd11}, [srem_i128_param_1];
+; CHECK-NEXT:    shr.s64 %rd1, %rd9, 63;
+; CHECK-NEXT:    sub.cc.s64 %rd12, 0, %rd8;
+; CHECK-NEXT:    subc.cc.s64 %rd13, 0, %rd9;
+; CHECK-NEXT:    setp.lt.s64 %p1, %rd9, 0;
+; CHECK-NEXT:    selp.b64 %rd3, %rd13, %rd9, %p1;
+; CHECK-NEXT:    selp.b64 %rd2, %rd12, %rd8, %p1;
+; CHECK-NEXT:    sub.cc.s64 %rd14, 0, %rd10;
+; CHECK-NEXT:    subc.cc.s64 %rd15, 0, %rd11;
+; CHECK-NEXT:    setp.lt.s64 %p2, %rd11, 0;
+; CHECK-NEXT:    selp.b64 %rd5, %rd15, %rd11, %p2;
+; CHECK-NEXT:    selp.b64 %rd4, %rd14, %rd10, %p2;
+; CHECK-NEXT:    or.b64 %rd16, %rd4, %rd5;
+; CHECK-NEXT:    setp.eq.b64 %p3, %rd16, 0;
+; CHECK-NEXT:    or.b64 %rd17, %rd2, %rd3;
+; CHECK-NEXT:    setp.eq.b64 %p4, %rd17, 0;
 ; CHECK-NEXT:    or.pred %p5, %p3, %p4;
-; CHECK-NEXT:    setp.ne.b64 %p6, %rd6, 0;
-; CHECK-NEXT:    clz.b64 %r1, %rd6;
-; CHECK-NEXT:    cvt.u64.u32 %rd57, %r1;
-; CHECK-NEXT:    clz.b64 %r2, %rd5;
-; CHECK-NEXT:    cvt.u64.u32 %rd58, %r2;
-; CHECK-NEXT:    add.s64 %rd59, %rd58, 64;
-; CHECK-NEXT:    selp.b64 %rd60, %rd57, %rd59, %p6;
-; CHECK-NEXT:    setp.ne.b64 %p7, %rd4, 0;
-; CHECK-NEXT:    clz.b64 %r3, %rd4;
-; CHECK-NEXT:    cvt.u64.u32 %rd61, %r3;
-; CHECK-NEXT:    clz.b64 %r4, %rd3;
-; CHECK-NEXT:    cvt.u64.u32 %rd62, %r4;
-; CHECK-NEXT:    add.s64 %rd63, %rd62, 64;
-; CHECK-NEXT:    selp.b64 %rd64, %rd61, %rd63, %p7;
-; CHECK-NEXT:    mov.b64 %rd117, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd66, %rd60, %rd64;
-; CHECK-NEXT:    subc.cc.s64 %rd67, %rd117, 0;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd66, 127;
-; CHECK-NEXT:    setp.eq.b64 %p9, %rd67, 0;
+; CHECK-NEXT:    setp.ne.b64 %p6, %rd5, 0;
+; CHECK-NEXT:    clz.b64 %r1, %rd5;
+; CHECK-NEXT:    cvt.u64.u32 %rd18, %r1;
+; CHECK-NEXT:    clz.b64 %r2, %rd4;
+; CHECK-NEXT:    cvt.u64.u32 %rd19, %r2;
+; CHECK-NEXT:    add.s64 %rd20, %rd19, 64;
+; CHECK-NEXT:    selp.b64 %rd21, %rd18, %rd20, %p6;
+; CHECK-NEXT:    setp.ne.b64 %p7, %rd3, 0;
+; CHECK-NEXT:    clz.b64 %r3, %rd3;
+; CHECK-NEXT:    cvt.u64.u32 %rd22, %r3;
+; CHECK-NEXT:    clz.b64 %r4, %rd2;
+; CHECK-NEXT:    cvt.u64.u32 %rd23, %r4;
+; CHECK-NEXT:    add.s64 %rd24, %rd23, 64;
+; CHECK-NEXT:    selp.b64 %rd25, %rd22, %rd24, %p7;
+; CHECK-NEXT:    mov.b64 %rd70, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd26, %rd21, %rd25;
+; CHECK-NEXT:    subc.cc.s64 %rd27, %rd70, 0;
+; CHECK-NEXT:    setp.gt.u64 %p8, %rd26, 127;
+; CHECK-NEXT:    setp.eq.b64 %p9, %rd27, 0;
 ; CHECK-NEXT:    and.pred %p10, %p9, %p8;
-; CHECK-NEXT:    setp.ne.b64 %p11, %rd67, 0;
+; CHECK-NEXT:    setp.ne.b64 %p11, %rd27, 0;
 ; CHECK-NEXT:    or.pred %p12, %p10, %p11;
 ; CHECK-NEXT:    or.pred %p13, %p5, %p12;
-; CHECK-NEXT:    xor.b64 %rd68, %rd66, 127;
-; CHECK-NEXT:    or.b64 %rd69, %rd68, %rd67;
-; CHECK-NEXT:    setp.eq.b64 %p14, %rd69, 0;
-; CHECK-NEXT:    selp.b64 %rd126, 0, %rd4, %p13;
-; CHECK-NEXT:    selp.b64 %rd125, 0, %rd3, %p13;
+; CHECK-NEXT:    xor.b64 %rd28, %rd26, 127;
+; CHECK-NEXT:    or.b64 %rd29, %rd28, %rd27;
+; CHECK-NEXT:    setp.eq.b64 %p14, %rd29, 0;
+; CHECK-NEXT:    selp.b64 %rd78, 0, %rd3, %p13;
+; CHECK-NEXT:    selp.b64 %rd77, 0, %rd2, %p13;
 ; CHECK-NEXT:    or.pred %p15, %p13, %p14;
 ; CHECK-NEXT:    @%p15 bra $L__BB0_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd119, %rd66, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd120, %rd67, 0;
-; CHECK-NEXT:    or.b64 %rd72, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.b64 %p16, %rd72, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r5, %rd66;
+; CHECK-NEXT:    add.cc.s64 %rd71, %rd26, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd72, %rd27, 0;
+; CHECK-NEXT:    or.b64 %rd30, %rd71, %rd72;
+; CHECK-NEXT:    setp.eq.b64 %p16, %rd30, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd26;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd73, %rd4, %r6;
+; CHECK-NEXT:    shl.b64 %rd31, %rd3, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd74, %rd3, %r7;
-; CHECK-NEXT:    or.b64 %rd75, %rd73, %rd74;
+; CHECK-NEXT:    shr.u64 %rd32, %rd2, %r7;
+; CHECK-NEXT:    or.b64 %rd33, %rd31, %rd32;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd76, %rd3, %r8;
+; CHECK-NEXT:    shl.b64 %rd34, %rd2, %r8;
 ; CHECK-NEXT:    setp.gt.s32 %p17, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd124, %rd76, %rd75, %p17;
-; CHECK-NEXT:    shl.b64 %rd123, %rd3, %r6;
-; CHECK-NEXT:    mov.b64 %rd114, %rd117;
+; CHECK-NEXT:    selp.b64 %rd76, %rd34, %rd33, %p17;
+; CHECK-NEXT:    shl.b64 %rd75, %rd2, %r6;
+; CHECK-NEXT:    mov.b64 %rd69, %rd70;
 ; CHECK-NEXT:    @%p16 bra $L__BB0_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd119;
-; CHECK-NEXT:    shr.u64 %rd79, %rd3, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd71;
+; CHECK-NEXT:    shr.u64 %rd35, %rd2, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd80, %rd4, %r10;
-; CHECK-NEXT:    or.b64 %rd81, %rd79, %rd80;
+; CHECK-NEXT:    shl.b64 %rd36, %rd3, %r10;
+; CHECK-NEXT:    or.b64 %rd37, %rd35, %rd36;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd82, %rd4, %r11;
+; CHECK-NEXT:    shr.u64 %rd38, %rd3, %r11;
 ; CHECK-NEXT:    setp.gt.s32 %p18, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd121, %rd82, %rd81, %p18;
-; CHECK-NEXT:    shr.u64 %rd122, %rd4, %r9;
-; CHECK-NEXT:    add.cc.s64 %rd35, %rd5, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd36, %rd6, -1;
-; CHECK-NEXT:    mov.b64 %rd114, 0;
-; CHECK-NEXT:    mov.b64 %rd117, %rd114;
+; CHECK-NEXT:    selp.b64 %rd73, %rd38, %rd37, %p18;
+; CHECK-NEXT:    shr.u64 %rd74, %rd3, %r9;
+; CHECK-NEXT:    add.cc.s64 %rd6, %rd4, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd7, %rd5, -1;
+; CHECK-NEXT:    mov.b64 %rd69, 0;
+; CHECK-NEXT:    mov.b64 %rd70, %rd69;
 ; CHECK-NEXT:  $L__BB0_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd83, %rd121, 63;
-; CHECK-NEXT:    shl.b64 %rd84, %rd122, 1;
-; CHECK-NEXT:    or.b64 %rd85, %rd84, %rd83;
-; CHECK-NEXT:    shl.b64 %rd86, %rd121, 1;
-; CHECK-NEXT:    shr.u64 %rd87, %rd124, 63;
-; CHECK-NEXT:    or.b64 %rd88, %rd86, %rd87;
-; CHECK-NEXT:    shr.u64 %rd89, %rd123, 63;
-; CHECK-NEXT:    shl.b64 %rd90, %rd124, 1;
-; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
-; CHECK-NEXT:    shl.b64 %rd92, %rd123, 1;
-; CHECK-NEXT:    or.b64 %rd123, %rd117, %rd92;
-; CHECK-NEXT:    or.b64 %rd124, %rd114, %rd91;
-; CHECK-NEXT:    sub.cc.s64 %rd93, %rd35, %rd88;
-; CHECK-NEXT:    subc.cc.s64 %rd94, %rd36, %rd85;
-; CHECK-NEXT:    shr.s64 %rd95, %rd94, 63;
-; CHECK-NEXT:    and.b64 %rd117, %rd95, 1;
-; CHECK-NEXT:    and.b64 %rd96, %rd95, %rd5;
-; CHECK-NEXT:    and.b64 %rd97, %rd95, %rd6;
-; CHECK-NEXT:    sub.cc.s64 %rd121, %rd88, %rd96;
-; CHECK-NEXT:    subc.cc.s64 %rd122, %rd85, %rd97;
-; CHECK-NEXT:    add.cc.s64 %rd119, %rd119, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd120, %rd120, -1;
-; CHECK-NEXT:    or.b64 %rd98, %rd119, %rd120;
-; CHECK-NEXT:    setp.eq.b64 %p19, %rd98, 0;
+; CHECK-NEXT:    shr.u64 %rd39, %rd73, 63;
+; CHECK-NEXT:    shl.b64 %rd40, %rd74, 1;
+; CHECK-NEXT:    or.b64 %rd41, %rd40, %rd39;
+; CHECK-NEXT:    shl.b64 %rd42, %rd73, 1;
+; CHECK-NEXT:    shr.u64 %rd43, %rd76, 63;
+; CHECK-NEXT:    or.b64 %rd44, %rd42, %rd43;
+; CHECK-NEXT:    shr.u64 %rd45, %rd75, 63;
+; CHECK-NEXT:    shl.b64 %rd46, %rd76, 1;
+; CHECK-NEXT:    or.b64 %rd47, %rd46, %rd45;
+; CHECK-NEXT:    shl.b64 %rd48, %rd75, 1;
+; CHECK-NEXT:    or.b64 %rd75, %rd70, %rd48;
+; CHECK-NEXT:    or.b64 %rd76, %rd69, %rd47;
+; CHECK-NEXT:    sub.cc.s64 %rd49, %rd6, %rd44;
+; CHECK-NEXT:    subc.cc.s64 %rd50, %rd7, %rd41;
+; CHECK-NEXT:    shr.s64 %rd51, %rd50, 63;
+; CHECK-NEXT:    and.b64 %rd70, %rd51, 1;
+; CHECK-NEXT:    and.b64 %rd52, %rd51, %rd4;
+; CHECK-NEXT:    and.b64 %rd53, %rd51, %rd5;
+; CHECK-NEXT:    sub.cc.s64 %rd73, %rd44, %rd52;
+; CHECK-NEXT:    subc.cc.s64 %rd74, %rd41, %rd53;
+; CHECK-NEXT:    add.cc.s64 %rd71, %rd71, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd72, %rd72, -1;
+; CHECK-NEXT:    or.b64 %rd54, %rd71, %rd72;
+; CHECK-NEXT:    setp.eq.b64 %p19, %rd54, 0;
 ; CHECK-NEXT:    @%p19 bra $L__BB0_4;
 ; CHECK-NEXT:    bra.uni $L__BB0_2;
 ; CHECK-NEXT:  $L__BB0_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd99, %rd123, 63;
-; CHECK-NEXT:    shl.b64 %rd100, %rd124, 1;
-; CHECK-NEXT:    or.b64 %rd101, %rd100, %rd99;
-; CHECK-NEXT:    shl.b64 %rd102, %rd123, 1;
-; CHECK-NEXT:    or.b64 %rd125, %rd117, %rd102;
-; CHECK-NEXT:    or.b64 %rd126, %rd114, %rd101;
+; CHECK-NEXT:    shr.u64 %rd55, %rd75, 63;
+; CHECK-NEXT:    shl.b64 %rd56, %rd76, 1;
+; CHECK-NEXT:    or.b64 %rd57, %rd56, %rd55;
+; CHECK-NEXT:    shl.b64 %rd58, %rd75, 1;
+; CHECK-NEXT:    or.b64 %rd77, %rd70, %rd58;
+; CHECK-NEXT:    or.b64 %rd78, %rd69, %rd57;
 ; CHECK-NEXT:  $L__BB0_5: // %udiv-end
-; CHECK-NEXT:    mul.hi.u64 %rd103, %rd5, %rd125;
-; CHECK-NEXT:    mad.lo.s64 %rd104, %rd5, %rd126, %rd103;
-; CHECK-NEXT:    mad.lo.s64 %rd105, %rd6, %rd125, %rd104;
-; CHECK-NEXT:    mul.lo.s64 %rd106, %rd5, %rd125;
-; CHECK-NEXT:    sub.cc.s64 %rd107, %rd3, %rd106;
-; CHECK-NEXT:    subc.cc.s64 %rd108, %rd4, %rd105;
-; CHECK-NEXT:    xor.b64 %rd109, %rd107, %rd2;
-; CHECK-NEXT:    xor.b64 %rd110, %rd108, %rd2;
-; CHECK-NEXT:    sub.cc.s64 %rd111, %rd109, %rd2;
-; CHECK-NEXT:    subc.cc.s64 %rd112, %rd110, %rd2;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd111, %rd112};
+; CHECK-NEXT:    mul.hi.u64 %rd59, %rd4, %rd77;
+; CHECK-NEXT:    mad.lo.s64 %rd60, %rd4, %rd78, %rd59;
+; CHECK-NEXT:    mad.lo.s64 %rd61, %rd5, %rd77, %rd60;
+; CHECK-NEXT:    mul.lo.s64 %rd62, %rd4, %rd77;
+; CHECK-NEXT:    sub.cc.s64 %rd63, %rd2, %rd62;
+; CHECK-NEXT:    subc.cc.s64 %rd64, %rd3, %rd61;
+; CHECK-NEXT:    xor.b64 %rd65, %rd63, %rd1;
+; CHECK-NEXT:    xor.b64 %rd66, %rd64, %rd1;
+; CHECK-NEXT:    sub.cc.s64 %rd67, %rd65, %rd1;
+; CHECK-NEXT:    subc.cc.s64 %rd68, %rd66, %rd1;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd67, %rd68};
 ; CHECK-NEXT:    ret;
   %div = srem i128 %lhs, %rhs
   ret i128 %div
@@ -148,122 +148,122 @@ define i128 @urem_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<18>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<113>;
+; CHECK-NEXT:    .reg .b64 %rd<66>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
-; CHECK-NEXT:    ld.param.v2.b64 {%rd41, %rd42}, [urem_i128_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [urem_i128_param_1];
-; CHECK-NEXT:    or.b64 %rd45, %rd3, %rd4;
-; CHECK-NEXT:    setp.eq.b64 %p1, %rd45, 0;
-; CHECK-NEXT:    or.b64 %rd46, %rd41, %rd42;
-; CHECK-NEXT:    setp.eq.b64 %p2, %rd46, 0;
+; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [urem_i128_param_0];
+; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [urem_i128_param_1];
+; CHECK-NEXT:    or.b64 %rd7, %rd1, %rd2;
+; CHECK-NEXT:    setp.eq.b64 %p1, %rd7, 0;
+; CHECK-NEXT:    or.b64 %rd8, %rd5, %rd6;
+; CHECK-NEXT:    setp.eq.b64 %p2, %rd8, 0;
 ; CHECK-NEXT:    or.pred %p3, %p1, %p2;
-; CHECK-NEXT:    setp.ne.b64 %p4, %rd4, 0;
-; CHECK-NEXT:    clz.b64 %r1, %rd4;
-; CHECK-NEXT:    cvt.u64.u32 %rd47, %r1;
-; CHECK-NEXT:    clz.b64 %r2, %rd3;
-; CHECK-NEXT:    cvt.u64.u32 %rd48, %r2;
-; CHECK-NEXT:    add.s64 %rd49, %rd48, 64;
-; CHECK-NEXT:    selp.b64 %rd50, %rd47, %rd49, %p4;
-; CHECK-NEXT:    setp.ne.b64 %p5, %rd42, 0;
-; CHECK-NEXT:    clz.b64 %r3, %rd42;
-; CHECK-NEXT:    cvt.u64.u32 %rd51, %r3;
-; CHECK-NEXT:    clz.b64 %r4, %rd41;
-; CHECK-NEXT:    cvt.u64.u32 %rd52, %r4;
-; CHECK-NEXT:    add.s64 %rd53, %rd52, 64;
-; CHECK-NEXT:    selp.b64 %rd54, %rd51, %rd53, %p5;
-; CHECK-NEXT:    mov.b64 %rd103, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
-; CHECK-NEXT:    subc.cc.s64 %rd57, %rd103, 0;
-; CHECK-NEXT:    setp.gt.u64 %p6, %rd56, 127;
-; CHECK-NEXT:    setp.eq.b64 %p7, %rd57, 0;
+; CHECK-NEXT:    setp.ne.b64 %p4, %rd2, 0;
+; CHECK-NEXT:    clz.b64 %r1, %rd2;
+; CHECK-NEXT:    cvt.u64.u32 %rd9, %r1;
+; CHECK-NEXT:    clz.b64 %r2, %rd1;
+; CHECK-NEXT:    cvt.u64.u32 %rd10, %r2;
+; CHECK-NEXT:    add.s64 %rd11, %rd10, 64;
+; CHECK-NEXT:    selp.b64 %rd12, %rd9, %rd11, %p4;
+; CHECK-NEXT:    setp.ne.b64 %p5, %rd6, 0;
+; CHECK-NEXT:    clz.b64 %r3, %rd6;
+; CHECK-NEXT:    cvt.u64.u32 %rd13, %r3;
+; CHECK-NEXT:    clz.b64 %r4, %rd5;
+; CHECK-NEXT:    cvt.u64.u32 %rd14, %r4;
+; CHECK-NEXT:    add.s64 %rd15, %rd14, 64;
+; CHECK-NEXT:    selp.b64 %rd16, %rd13, %rd15, %p5;
+; CHECK-NEXT:    mov.b64 %rd57, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd17, %rd12, %rd16;
+; CHECK-NEXT:    subc.cc.s64 %rd18, %rd57, 0;
+; CHECK-NEXT:    setp.gt.u64 %p6, %rd17, 127;
+; CHECK-NEXT:    setp.eq.b64 %p7, %rd18, 0;
 ; CHECK-NEXT:    and.pred %p8, %p7, %p6;
-; CHECK-NEXT:    setp.ne.b64 %p9, %rd57, 0;
+; CHECK-NEXT:    setp.ne.b64 %p9, %rd18, 0;
 ; CHECK-NEXT:    or.pred %p10, %p8, %p9;
 ; CHECK-NEXT:    or.pred %p11, %p3, %p10;
-; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
-; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
-; CHECK-NEXT:    setp.eq.b64 %p12, %rd59, 0;
-; CHECK-NEXT:    selp.b64 %rd112, 0, %rd42, %p11;
-; CHECK-NEXT:    selp.b64 %rd111, 0, %rd41, %p11;
+; CHECK-NEXT:    xor.b64 %rd19, %rd17, 127;
+; CHECK-NEXT:    or.b64 %rd20, %rd19, %rd18;
+; CHECK-NEXT:    setp.eq.b64 %p12, %rd20, 0;
+; CHECK-NEXT:    selp.b64 %rd65, 0, %rd6, %p11;
+; CHECK-NEXT:    selp.b64 %rd64, 0, %rd5, %p11;
 ; CHECK-NEXT:    or.pred %p13, %p11, %p12;
 ; CHECK-NEXT:    @%p13 bra $L__BB1_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd105, %rd56, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd106, %rd57, 0;
-; CHECK-NEXT:    or.b64 %rd62, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.b64 %p14, %rd62, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r5, %rd56;
+; CHECK-NEXT:    add.cc.s64 %rd58, %rd17, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd59, %rd18, 0;
+; CHECK-NEXT:    or.b64 %rd21, %rd58, %rd59;
+; CHECK-NEXT:    setp.eq.b64 %p14, %rd21, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd17;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r6;
+; CHECK-NEXT:    shl.b64 %rd22, %rd6, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r7;
-; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
+; CHECK-NEXT:    shr.u64 %rd23, %rd5, %r7;
+; CHECK-NEXT:    or.b64 %rd24, %rd22, %rd23;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r8;
+; CHECK-NEXT:    shl.b64 %rd25, %rd5, %r8;
 ; CHECK-NEXT:    setp.gt.s32 %p15, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd110, %rd66, %rd65, %p15;
-; CHECK-NEXT:    shl.b64 %rd109, %rd41, %r6;
-; CHECK-NEXT:    mov.b64 %rd100, %rd103;
+; CHECK-NEXT:    selp.b64 %rd63, %rd25, %rd24, %p15;
+; CHECK-NEXT:    shl.b64 %rd62, %rd5, %r6;
+; CHECK-NEXT:    mov.b64 %rd56, %rd57;
 ; CHECK-NEXT:    @%p14 bra $L__BB1_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd105;
-; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd58;
+; CHECK-NEXT:    shr.u64 %rd26, %rd5, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r10;
-; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
+; CHECK-NEXT:    shl.b64 %rd27, %rd6, %r10;
+; CHECK-NEXT:    or.b64 %rd28, %rd26, %rd27;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r11;
+; CHECK-NEXT:    shr.u64 %rd29, %rd6, %r11;
 ; CHECK-NEXT:    setp.gt.s32 %p16, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd107, %rd72, %rd71, %p16;
-; CHECK-NEXT:    shr.u64 %rd108, %rd42, %r9;
-; CHECK-NEXT:    add.cc.s64 %rd33, %rd3, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd34, %rd4, -1;
-; CHECK-NEXT:    mov.b64 %rd100, 0;
-; CHECK-NEXT:    mov.b64 %rd103, %rd100;
+; CHECK-NEXT:    selp.b64 %rd60, %rd29, %rd28, %p16;
+; CHECK-NEXT:    shr.u64 %rd61, %rd6, %r9;
+; CHECK-NEXT:    add.cc.s64 %rd3, %rd1, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd4, %rd2, -1;
+; CHECK-NEXT:    mov.b64 %rd56, 0;
+; CHECK-NEXT:    mov.b64 %rd57, %rd56;
 ; CHECK-NEXT:  $L__BB1_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd73, %rd107, 63;
-; CHECK-NEXT:    shl.b64 %rd74, %rd108, 1;
-; CHECK-NEXT:    or.b64 %rd75, %rd74, %rd73;
-; CHECK-NEXT:    shl.b64 %rd76, %rd107, 1;
-; CHECK-NEXT:    shr.u64 %rd77, %rd110, 63;
-; CHECK-NEXT:    or.b64 %rd78, %rd76, %rd77;
-; CHECK-NEXT:    shr.u64 %rd79, %rd109, 63;
-; CHECK-NEXT:    shl.b64 %rd80, %rd110, 1;
-; CHECK-NEXT:    or.b64 %rd81, %rd80, %rd79;
-; CHECK-NEXT:    shl.b64 %rd82, %rd109, 1;
-; CHECK-NEXT:    or.b64 %rd109, %rd103, %rd82;
-; CHECK-NEXT:    or.b64 %rd110, %rd100, %rd81;
-; CHECK-NEXT:    sub.cc.s64 %rd83, %rd33, %rd78;
-; CHECK-NEXT:    subc.cc.s64 %rd84, %rd34, %rd75;
-; CHECK-NEXT:    shr.s64 %rd85, %rd84, 63;
-; CHECK-NEXT:    and.b64 %rd103, %rd85, 1;
-; CHECK-NEXT:    and.b64 %rd86, %rd85, %rd3;
-; CHECK-NEXT:    and.b64 %rd87, %rd85, %rd4;
-; CHECK-NEXT:    sub.cc.s64 %rd107, %rd78, %rd86;
-; CHECK-NEXT:    subc.cc.s64 %rd108, %rd75, %rd87;
-; CHECK-NEXT:    add.cc.s64 %rd105, %rd105, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd106, %rd106, -1;
-; CHECK-NEXT:    or.b64 %rd88, %rd105, %rd106;
-; CHECK-NEXT:    setp.eq.b64 %p17, %rd88, 0;
+; CHECK-NEXT:    shr.u64 %rd30, %rd60, 63;
+; CHECK-NEXT:    shl.b64 %rd31, %rd61, 1;
+; CHECK-NEXT:    or.b64 %rd32, %rd31, %rd30;
+; CHECK-NEXT:    shl.b64 %rd33, %rd60, 1;
+; CHECK-NEXT:    shr.u64 %rd34, %rd63, 63;
+; CHECK-NEXT:    or.b64 %rd35, %rd33, %rd34;
+; CHECK-NEXT:    shr.u64 %rd36, %rd62, 63;
+; CHECK-NEXT:    shl.b64 %rd37, %rd63, 1;
+; CHECK-NEXT:    or.b64 %rd38, %rd37, %rd36;
+; CHECK-NEXT:    shl.b64 %rd39, %rd62, 1;
+; CHECK-NEXT:    or.b64 %rd62, %rd57, %rd39;
+; CHECK-NEXT:    or.b64 %rd63, %rd56, %rd38;
+; CHECK-NEXT:    sub.cc.s64 %rd40, %rd3, %rd35;
+; CHECK-NEXT:    subc.cc.s64 %rd41, %rd4, %rd32;
+; CHECK-NEXT:    shr.s64 %rd42, %rd41, 63;
+; CHECK-NEXT:    and.b64 %rd57, %rd42, 1;
+; CHECK-NEXT:    and.b64 %rd43, %rd42, %rd1;
+; CHECK-NEXT:    and.b64 %rd44, %rd42, %rd2;
+; CHECK-NEXT:    sub.cc.s64 %rd60, %rd35, %rd43;
+; CHECK-NEXT:    subc.cc.s64 %rd61, %rd32, %rd44;
+; CHECK-NEXT:    add.cc.s64 %rd58, %rd58, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd59, %rd59, -1;
+; CHECK-NEXT:    or.b64 %rd45, %rd58, %rd59;
+; CHECK-NEXT:    setp.eq.b64 %p17, %rd45, 0;
 ; CHECK-NEXT:    @%p17 bra $L__BB1_4;
 ; CHECK-NEXT:    bra.uni $L__BB1_2;
 ; CHECK-NEXT:  $L__BB1_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd89, %rd109, 63;
-; CHECK-NEXT:    shl.b64 %rd90, %rd110, 1;
-; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
-; CHECK-NEXT:    shl.b64 %rd92, %rd109, 1;
-; CHECK-NEXT:    or.b64 %rd111, %rd103, %rd92;
-; CHECK-NEXT:    or.b64 %rd112, %rd100, %rd91;
+; CHECK-NEXT:    shr.u64 %rd46, %rd62, 63;
+; CHECK-NEXT:    shl.b64 %rd47, %rd63, 1;
+; CHECK-NEXT:    or.b64 %rd48, %rd47, %rd46;
+; CHECK-NEXT:    shl.b64 %rd49, %rd62, 1;
+; CHECK-NEXT:    or.b64 %rd64, %rd57, %rd49;
+; CHECK-NEXT:    or.b64 %rd65, %rd56, %rd48;
 ; CHECK-NEXT:  $L__BB1_5: // %udiv-end
-; CHECK-NEXT:    mul.hi.u64 %rd93, %rd3, %rd111;
-; CHECK-NEXT:    mad.lo.s64 %rd94, %rd3, %rd112, %rd93;
-; CHECK-NEXT:    mad.lo.s64 %rd95, %rd4, %rd111, %rd94;
-; CHECK-NEXT:    mul.lo.s64 %rd96, %rd3, %rd111;
-; CHECK-NEXT:    sub.cc.s64 %rd97, %rd41, %rd96;
-; CHECK-NEXT:    subc.cc.s64 %rd98, %rd42, %rd95;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd97, %rd98};
+; CHECK-NEXT:    mul.hi.u64 %rd50, %rd1, %rd64;
+; CHECK-NEXT:    mad.lo.s64 %rd51, %rd1, %rd65, %rd50;
+; CHECK-NEXT:    mad.lo.s64 %rd52, %rd2, %rd64, %rd51;
+; CHECK-NEXT:    mul.lo.s64 %rd53, %rd1, %rd64;
+; CHECK-NEXT:    sub.cc.s64 %rd54, %rd5, %rd53;
+; CHECK-NEXT:    subc.cc.s64 %rd55, %rd6, %rd52;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd54, %rd55};
 ; CHECK-NEXT:    ret;
   %div = urem i128 %lhs, %rhs
   ret i128 %div
@@ -308,132 +308,132 @@ define i128 @sdiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<20>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<122>;
+; CHECK-NEXT:    .reg .b64 %rd<74>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
-; CHECK-NEXT:    ld.param.v2.b64 {%rd45, %rd46}, [sdiv_i128_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd49, %rd50}, [sdiv_i128_param_1];
-; CHECK-NEXT:    sub.cc.s64 %rd51, 0, %rd45;
-; CHECK-NEXT:    subc.cc.s64 %rd52, 0, %rd46;
-; CHECK-NEXT:    setp.lt.s64 %p1, %rd46, 0;
-; CHECK-NEXT:    selp.b64 %rd2, %rd52, %rd46, %p1;
-; CHECK-NEXT:    selp.b64 %rd1, %rd51, %rd45, %p1;
-; CHECK-NEXT:    sub.cc.s64 %rd53, 0, %rd49;
-; CHECK-NEXT:    subc.cc.s64 %rd54, 0, %rd50;
-; CHECK-NEXT:    setp.lt.s64 %p2, %rd50, 0;
-; CHECK-NEXT:    selp.b64 %rd4, %rd54, %rd50, %p2;
-; CHECK-NEXT:    selp.b64 %rd3, %rd53, %rd49, %p2;
-; CHECK-NEXT:    xor.b64 %rd55, %rd50, %rd46;
-; CHECK-NEXT:    shr.s64 %rd5, %rd55, 63;
-; CHECK-NEXT:    or.b64 %rd56, %rd3, %rd4;
-; CHECK-NEXT:    setp.eq.b64 %p3, %rd56, 0;
-; CHECK-NEXT:    or.b64 %rd57, %rd1, %rd2;
-; CHECK-NEXT:    setp.eq.b64 %p4, %rd57, 0;
+; CHECK-NEXT:    ld.param.v2.b64 {%rd8, %rd9}, [sdiv_i128_param_0];
+; CHECK-NEXT:    ld.param.v2.b64 {%rd10, %rd11}, [sdiv_i128_param_1];
+; CHECK-NEXT:    sub.cc.s64 %rd12, 0, %rd8;
+; CHECK-NEXT:    subc.cc.s64 %rd13, 0, %rd9;
+; CHECK-NEXT:    setp.lt.s64 %p1, %rd9, 0;
+; CHECK-NEXT:    selp.b64 %rd2, %rd13, %rd9, %p1;
+; CHECK-NEXT:    selp.b64 %rd1, %rd12, %rd8, %p1;
+; CHECK-NEXT:    sub.cc.s64 %rd14, 0, %rd10;
+; CHECK-NEXT:    subc.cc.s64 %rd15, 0, %rd11;
+; CHECK-NEXT:    setp.lt.s64 %p2, %rd11, 0;
+; CHECK-NEXT:    selp.b64 %rd4, %rd15, %rd11, %p2;
+; CHECK-NEXT:    selp.b64 %rd3, %rd14, %rd10, %p2;
+; CHECK-NEXT:    xor.b64 %rd16, %rd11, %rd9;
+; CHECK-NEXT:    shr.s64 %rd5, %rd16, 63;
+; CHECK-NEXT:    or.b64 %rd17, %rd3, %rd4;
+; CHECK-NEXT:    setp.eq.b64 %p3, %rd17, 0;
+; CHECK-NEXT:    or.b64 %rd18, %rd1, %rd2;
+; CHECK-NEXT:    setp.eq.b64 %p4, %rd18, 0;
 ; CHECK-NEXT:    or.pred %p5, %p3, %p4;
 ; CHECK-NEXT:    setp.ne.b64 %p6, %rd4, 0;
 ; CHECK-NEXT:    clz.b64 %r1, %rd4;
-; CHECK-NEXT:    cvt.u64.u32 %rd58, %r1;
+; CHECK-NEXT:    cvt.u64.u32 %rd19, %r1;
 ; CHECK-NEXT:    clz.b64 %r2, %rd3;
-; CHECK-NEXT:    cvt.u64.u32 %rd59, %r2;
-; CHECK-NEXT:    add.s64 %rd60, %rd59, 64;
-; CHECK-NEXT:    selp.b64 %rd61, %rd58, %rd60, %p6;
+; CHECK-NEXT:    cvt.u64.u32 %rd20, %r2;
+; CHECK-NEXT:    add.s64 %rd21, %rd20, 64;
+; CHECK-NEXT:    selp.b64 %rd22, %rd19, %rd21, %p6;
 ; CHECK-NEXT:    setp.ne.b64 %p7, %rd2, 0;
 ; CHECK-NEXT:    clz.b64 %r3, %rd2;
-; CHECK-NEXT:    cvt.u64.u32 %rd62, %r3;
+; CHECK-NEXT:    cvt.u64.u32 %rd23, %r3;
 ; CHECK-NEXT:    clz.b64 %r4, %rd1;
-; CHECK-NEXT:    cvt.u64.u32 %rd63, %r4;
-; CHECK-NEXT:    add.s64 %rd64, %rd63, 64;
-; CHECK-NEXT:    selp.b64 %rd65, %rd62, %rd64, %p7;
-; CHECK-NEXT:    mov.b64 %rd112, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd67, %rd61, %rd65;
-; CHECK-NEXT:    subc.cc.s64 %rd68, %rd112, 0;
-; CHECK-NEXT:    setp.gt.u64 %p8, %rd67, 127;
-; CHECK-NEXT:    setp.eq.b64 %p9, %rd68, 0;
+; CHECK-NEXT:    cvt.u64.u32 %rd24, %r4;
+; CHECK-NEXT:    add.s64 %rd25, %rd24, 64;
+; CHECK-NEXT:    selp.b64 %rd26, %rd23, %rd25, %p7;
+; CHECK-NEXT:    mov.b64 %rd65, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd27, %rd22, %rd26;
+; CHECK-NEXT:    subc.cc.s64 %rd28, %rd65, 0;
+; CHECK-NEXT:    setp.gt.u64 %p8, %rd27, 127;
+; CHECK-NEXT:    setp.eq.b64 %p9, %rd28, 0;
 ; CHECK-NEXT:    and.pred %p10, %p9, %p8;
-; CHECK-NEXT:    setp.ne.b64 %p11, %rd68, 0;
+; CHECK-NEXT:    setp.ne.b64 %p11, %rd28, 0;
 ; CHECK-NEXT:    or.pred %p12, %p10, %p11;
 ; CHECK-NEXT:    or.pred %p13, %p5, %p12;
-; CHECK-NEXT:    xor.b64 %rd69, %rd67, 127;
-; CHECK-NEXT:    or.b64 %rd70, %rd69, %rd68;
-; CHECK-NEXT:    setp.eq.b64 %p14, %rd70, 0;
-; CHECK-NEXT:    selp.b64 %rd121, 0, %rd2, %p13;
-; CHECK-NEXT:    selp.b64 %rd120, 0, %rd1, %p13;
+; CHECK-NEXT:    xor.b64 %rd29, %rd27, 127;
+; CHECK-NEXT:    or.b64 %rd30, %rd29, %rd28;
+; CHECK-NEXT:    setp.eq.b64 %p14, %rd30, 0;
+; CHECK-NEXT:    selp.b64 %rd73, 0, %rd2, %p13;
+; CHECK-NEXT:    selp.b64 %rd72, 0, %rd1, %p13;
 ; CHECK-NEXT:    or.pred %p15, %p13, %p14;
 ; CHECK-NEXT:    @%p15 bra $L__BB4_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd114, %rd67, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd115, %rd68, 0;
-; CHECK-NEXT:    or.b64 %rd73, %rd114, %rd115;
-; CHECK-NEXT:    setp.eq.b64 %p16, %rd73, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r5, %rd67;
+; CHECK-NEXT:    add.cc.s64 %rd66, %rd27, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd67, %rd28, 0;
+; CHECK-NEXT:    or.b64 %rd31, %rd66, %rd67;
+; CHECK-NEXT:    setp.eq.b64 %p16, %rd31, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd27;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd74, %rd2, %r6;
+; CHECK-NEXT:    shl.b64 %rd32, %rd2, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd75, %rd1, %r7;
-; CHECK-NEXT:    or.b64 %rd76, %rd74, %rd75;
+; CHECK-NEXT:    shr.u64 %rd33, %rd1, %r7;
+; CHECK-NEXT:    or.b64 %rd34, %rd32, %rd33;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd77, %rd1, %r8;
+; CHECK-NEXT:    shl.b64 %rd35, %rd1, %r8;
 ; CHECK-NEXT:    setp.gt.s32 %p17, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd119, %rd77, %rd76, %p17;
-; CHECK-NEXT:    shl.b64 %rd118, %rd1, %r6;
-; CHECK-NEXT:    mov.b64 %rd109, %rd112;
+; CHECK-NEXT:    selp.b64 %rd71, %rd35, %rd34, %p17;
+; CHECK-NEXT:    shl.b64 %rd70, %rd1, %r6;
+; CHECK-NEXT:    mov.b64 %rd64, %rd65;
 ; CHECK-NEXT:    @%p16 bra $L__BB4_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd114;
-; CHECK-NEXT:    shr.u64 %rd80, %rd1, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd66;
+; CHECK-NEXT:    shr.u64 %rd36, %rd1, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd81, %rd2, %r10;
-; CHECK-NEXT:    or.b64 %rd82, %rd80, %rd81;
+; CHECK-NEXT:    shl.b64 %rd37, %rd2, %r10;
+; CHECK-NEXT:    or.b64 %rd38, %rd36, %rd37;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd83, %rd2, %r11;
+; CHECK-NEXT:    shr.u64 %rd39, %rd2, %r11;
 ; CHECK-NEXT:    setp.gt.s32 %p18, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd116, %rd83, %rd82, %p18;
-; CHECK-NEXT:    shr.u64 %rd117, %rd2, %r9;
-; CHECK-NEXT:    add.cc.s64 %rd35, %rd3, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd36, %rd4, -1;
-; CHECK-NEXT:    mov.b64 %rd109, 0;
-; CHECK-NEXT:    mov.b64 %rd112, %rd109;
+; CHECK-NEXT:    selp.b64 %rd68, %rd39, %rd38, %p18;
+; CHECK-NEXT:    shr.u64 %rd69, %rd2, %r9;
+; CHECK-NEXT:    add.cc.s64 %rd6, %rd3, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd7, %rd4, -1;
+; CHECK-NEXT:    mov.b64 %rd64, 0;
+; CHECK-NEXT:    mov.b64 %rd65, %rd64;
 ; CHECK-NEXT:  $L__BB4_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd84, %rd116, 63;
-; CHECK-NEXT:    shl.b64 %rd85, %rd117, 1;
-; CHECK-NEXT:    or.b64 %rd86, %rd85, %rd84;
-; CHECK-NEXT:    shl.b64 %rd87, %rd116, 1;
-; CHECK-NEXT:    shr.u64 %rd88, %rd119, 63;
-; CHECK-NEXT:    or.b64 %rd89, %rd87, %rd88;
-; CHECK-NEXT:    shr.u64 %rd90, %rd118, 63;
-; CHECK-NEXT:    shl.b64 %rd91, %rd119, 1;
-; CHECK-NEXT:    or.b64 %rd92, %rd91, %rd90;
-; CHECK-NEXT:    shl.b64 %rd93, %rd118, 1;
-; CHECK-NEXT:    or.b64 %rd118, %rd112, %rd93;
-; CHECK-NEXT:    or.b64 %rd119, %rd109, %rd92;
-; CHECK-NEXT:    sub.cc.s64 %rd94, %rd35, %rd89;
-; CHECK-NEXT:    subc.cc.s64 %rd95, %rd36, %rd86;
-; CHECK-NEXT:    shr.s64 %rd96, %rd95, 63;
-; CHECK-NEXT:    and.b64 %rd112, %rd96, 1;
-; CHECK-NEXT:    and.b64 %rd97, %rd96, %rd3;
-; CHECK-NEXT:    and.b64 %rd98, %rd96, %rd4;
-; CHECK-NEXT:    sub.cc.s64 %rd116, %rd89, %rd97;
-; CHECK-NEXT:    subc.cc.s64 %rd117, %rd86, %rd98;
-; CHECK-NEXT:    add.cc.s64 %rd114, %rd114, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd115, %rd115, -1;
-; CHECK-NEXT:    or.b64 %rd99, %rd114, %rd115;
-; CHECK-NEXT:    setp.eq.b64 %p19, %rd99, 0;
+; CHECK-NEXT:    shr.u64 %rd40, %rd68, 63;
+; CHECK-NEXT:    shl.b64 %rd41, %rd69, 1;
+; CHECK-NEXT:    or.b64 %rd42, %rd41, %rd40;
+; CHECK-NEXT:    shl.b64 %rd43, %rd68, 1;
+; CHECK-NEXT:    shr.u64 %rd44, %rd71, 63;
+; CHECK-NEXT:    or.b64 %rd45, %rd43, %rd44;
+; CHECK-NEXT:    shr.u64 %rd46, %rd70, 63;
+; CHECK-NEXT:    shl.b64 %rd47, %rd71, 1;
+; CHECK-NEXT:    or.b64 %rd48, %rd47, %rd46;
+; CHECK-NEXT:    shl.b64 %rd49, %rd70, 1;
+; CHECK-NEXT:    or.b64 %rd70, %rd65, %rd49;
+; CHECK-NEXT:    or.b64 %rd71, %rd64, %rd48;
+; CHECK-NEXT:    sub.cc.s64 %rd50, %rd6, %rd45;
+; CHECK-NEXT:    subc.cc.s64 %rd51, %rd7, %rd42;
+; CHECK-NEXT:    shr.s64 %rd52, %rd51, 63;
+; CHECK-NEXT:    and.b64 %rd65, %rd52, 1;
+; CHECK-NEXT:    and.b64 %rd53, %rd52, %rd3;
+; CHECK-NEXT:    and.b64 %rd54, %rd52, %rd4;
+; CHECK-NEXT:    sub.cc.s64 %rd68, %rd45, %rd53;
+; CHECK-NEXT:    subc.cc.s64 %rd69, %rd42, %rd54;
+; CHECK-NEXT:    add.cc.s64 %rd66, %rd66, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd67, %rd67, -1;
+; CHECK-NEXT:    or.b64 %rd55, %rd66, %rd67;
+; CHECK-NEXT:    setp.eq.b64 %p19, %rd55, 0;
 ; CHECK-NEXT:    @%p19 bra $L__BB4_4;
 ; CHECK-NEXT:    bra.uni $L__BB4_2;
 ; CHECK-NEXT:  $L__BB4_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd100, %rd118, 63;
-; CHECK-NEXT:    shl.b64 %rd101, %rd119, 1;
-; CHECK-NEXT:    or.b64 %rd102, %rd101, %rd100;
-; CHECK-NEXT:    shl.b64 %rd103, %rd118, 1;
-; CHECK-NEXT:    or.b64 %rd120, %rd112, %rd103;
-; CHECK-NEXT:    or.b64 %rd121, %rd109, %rd102;
+; CHECK-NEXT:    shr.u64 %rd56, %rd70, 63;
+; CHECK-NEXT:    shl.b64 %rd57, %rd71, 1;
+; CHECK-NEXT:    or.b64 %rd58, %rd57, %rd56;
+; CHECK-NEXT:    shl.b64 %rd59, %rd70, 1;
+; CHECK-NEXT:    or.b64 %rd72, %rd65, %rd59;
+; CHECK-NEXT:    or.b64 %rd73, %rd64, %rd58;
 ; CHECK-NEXT:  $L__BB4_5: // %udiv-end
-; CHECK-NEXT:    xor.b64 %rd104, %rd120, %rd5;
-; CHECK-NEXT:    xor.b64 %rd105, %rd121, %rd5;
-; CHECK-NEXT:    sub.cc.s64 %rd106, %rd104, %rd5;
-; CHECK-NEXT:    subc.cc.s64 %rd107, %rd105, %rd5;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd106, %rd107};
+; CHECK-NEXT:    xor.b64 %rd60, %rd72, %rd5;
+; CHECK-NEXT:    xor.b64 %rd61, %rd73, %rd5;
+; CHECK-NEXT:    sub.cc.s64 %rd62, %rd60, %rd5;
+; CHECK-NEXT:    subc.cc.s64 %rd63, %rd61, %rd5;
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd62, %rd63};
 ; CHECK-NEXT:    ret;
   %div = sdiv i128 %lhs, %rhs
   ret i128 %div
@@ -444,116 +444,116 @@ define i128 @udiv_i128(i128 %lhs, i128 %rhs) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<18>;
 ; CHECK-NEXT:    .reg .b32 %r<12>;
-; CHECK-NEXT:    .reg .b64 %rd<107>;
+; CHECK-NEXT:    .reg .b64 %rd<60>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %_udiv-special-cases
-; CHECK-NEXT:    ld.param.v2.b64 {%rd41, %rd42}, [udiv_i128_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd43, %rd44}, [udiv_i128_param_1];
-; CHECK-NEXT:    or.b64 %rd45, %rd43, %rd44;
-; CHECK-NEXT:    setp.eq.b64 %p1, %rd45, 0;
-; CHECK-NEXT:    or.b64 %rd46, %rd41, %rd42;
-; CHECK-NEXT:    setp.eq.b64 %p2, %rd46, 0;
+; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [udiv_i128_param_0];
+; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [udiv_i128_param_1];
+; CHECK-NEXT:    or.b64 %rd7, %rd5, %rd6;
+; CHECK-NEXT:    setp.eq.b64 %p1, %rd7, 0;
+; CHECK-NEXT:    or.b64 %rd8, %rd3, %rd4;
+; CHECK-NEXT:    setp.eq.b64 %p2, %rd8, 0;
 ; CHECK-NEXT:    or.pred %p3, %p1, %p2;
-; CHECK-NEXT:    setp.ne.b64 %p4, %rd44, 0;
-; CHECK-NEXT:    clz.b64 %r1, %rd44;
-; CHECK-NEXT:    cvt.u64.u32 %rd47, %r1;
-; CHECK-NEXT:    clz.b64 %r2, %rd43;
-; CHECK-NEXT:    cvt.u64.u32 %rd48, %r2;
-; CHECK-NEXT:    add.s64 %rd49, %rd48, 64;
-; CHECK-NEXT:    selp.b64 %rd50, %rd47, %rd49, %p4;
-; CHECK-NEXT:    setp.ne.b64 %p5, %rd42, 0;
-; CHECK-NEXT:    clz.b64 %r3, %rd42;
-; CHECK-NEXT:    cvt.u64.u32 %rd51, %r3;
-; CHECK-NEXT:    clz.b64 %r4, %rd41;
-; CHECK-NEXT:    cvt.u64.u32 %rd52, %r4;
-; CHECK-NEXT:    add.s64 %rd53, %rd52, 64;
-; CHECK-NEXT:    selp.b64 %rd54, %rd51, %rd53, %p5;
-; CHECK-NEXT:    mov.b64 %rd97, 0;
-; CHECK-NEXT:    sub.cc.s64 %rd56, %rd50, %rd54;
-; CHECK-NEXT:    subc.cc.s64 %rd57, %rd97, 0;
-; CHECK-NEXT:    setp.gt.u64 %p6, %rd56, 127;
-; CHECK-NEXT:    setp.eq.b64 %p7, %rd57, 0;
+; CHECK-NEXT:    setp.ne.b64 %p4, %rd6, 0;
+; CHECK-NEXT:    clz.b64 %r1, %rd6;
+; CHECK-NEXT:    cvt.u64.u32 %rd9, %r1;
+; CHECK-NEXT:    clz.b64 %r2, %rd5;
+; CHECK-NEXT:    cvt.u64.u32 %rd10, %r2;
+; CHECK-NEXT:    add.s64 %rd11, %rd10, 64;
+; CHECK-NEXT:    selp.b64 %rd12, %rd9, %rd11, %p4;
+; CHECK-NEXT:    setp.ne.b64 %p5, %rd4, 0;
+; CHECK-NEXT:    clz.b64 %r3, %rd4;
+; CHECK-NEXT:    cvt.u64.u32 %rd13, %r3;
+; CHECK-NEXT:    clz.b64 %r4, %rd3;
+; CHECK-NEXT:    cvt.u64.u32 %rd14, %r4;
+; CHECK-NEXT:    add.s64 %rd15, %rd14, 64;
+; CHECK-NEXT:    selp.b64 %rd16, %rd13, %rd15, %p5;
+; CHECK-NEXT:    mov.b64 %rd51, 0;
+; CHECK-NEXT:    sub.cc.s64 %rd17, %rd12, %rd16;
+; CHECK-NEXT:    subc.cc.s64 %rd18, %rd51, 0;
+; CHECK-NEXT:    setp.gt.u64 %p6, %rd17, 127;
+; CHECK-NEXT:    setp.eq.b64 %p7, %rd18, 0;
 ; CHECK-NEXT:    and.pred %p8, %p7, %p6;
-; CHECK-NEXT:    setp.ne.b64 %p9, %rd57, 0;
+; CHECK-NEXT:    setp.ne.b64 %p9, %rd18, 0;
 ; CHECK-NEXT:    or.pred %p10, %p8, %p9;
 ; CHECK-NEXT:    or.pred %p11, %p3, %p10;
-; CHECK-NEXT:    xor.b64 %rd58, %rd56, 127;
-; CHECK-NEXT:    or.b64 %rd59, %rd58, %rd57;
-; CHECK-NEXT:    setp.eq.b64 %p12, %rd59, 0;
-; CHECK-NEXT:    selp.b64 %rd106, 0, %rd42, %p11;
-; CHECK-NEXT:    selp.b64 %rd105, 0, %rd41, %p11;
+; CHECK-NEXT:    xor.b64 %rd19, %rd17, 127;
+; CHECK-NEXT:    or.b64 %rd20, %rd19, %rd18;
+; CHECK-NEXT:    setp.eq.b64 %p12, %rd20, 0;
+; CHECK-NEXT:    selp.b64 %rd59, 0, %rd4, %p11;
+; CHECK-NEXT:    selp.b64 %rd58, 0, %rd3, %p11;
 ; CHECK-NEXT:    or.pred %p13, %p11, %p12;
 ; CHECK-NEXT:    @%p13 bra $L__BB5_5;
 ; CHECK-NEXT:  // %bb.3: // %udiv-bb1
-; CHECK-NEXT:    add.cc.s64 %rd99, %rd56, 1;
-; CHECK-NEXT:    addc.cc.s64 %rd100, %rd57, 0;
-; CHECK-NEXT:    or.b64 %rd62, %rd99, %rd100;
-; CHECK-NEXT:    setp.eq.b64 %p14, %rd62, 0;
-; CHECK-NEXT:    cvt.u32.u64 %r5, %rd56;
+; CHECK-NEXT:    add.cc.s64 %rd52, %rd17, 1;
+; CHECK-NEXT:    addc.cc.s64 %rd53, %rd18, 0;
+; CHECK-NEXT:    or.b64 %rd21, %rd52, %rd53;
+; CHECK-NEXT:    setp.eq.b64 %p14, %rd21, 0;
+; CHECK-NEXT:    cvt.u32.u64 %r5, %rd17;
 ; CHECK-NEXT:    sub.s32 %r6, 127, %r5;
-; CHECK-NEXT:    shl.b64 %rd63, %rd42, %r6;
+; CHECK-NEXT:    shl.b64 %rd22, %rd4, %r6;
 ; CHECK-NEXT:    sub.s32 %r7, 64, %r6;
-; CHECK-NEXT:    shr.u64 %rd64, %rd41, %r7;
-; CHECK-NEXT:    or.b64 %rd65, %rd63, %rd64;
+; CHECK-NEXT:    shr.u64 %rd23, %rd3, %r7;
+; CHECK-NEXT:    or.b64 %rd24, %rd22, %rd23;
 ; CHECK-NEXT:    sub.s32 %r8, 63, %r5;
-; CHECK-NEXT:    shl.b64 %rd66, %rd41, %r8;
+; CHECK-NEXT:    shl.b64 %rd25, %rd3, %r8;
 ; CHECK-NEXT:    setp.gt.s32 %p15, %r6, 63;
-; CHECK-NEXT:    selp.b64 %rd104, %rd66, %rd65, %p15;
-; CHECK-NEXT:    shl.b64 %rd103, %rd41, %r6;
-; CHECK-NEXT:    mov.b64 %rd94, %rd97;
+; CHECK-NEXT:    selp.b64 %rd57, %rd25, %rd24, %p15;
+; CHECK-NEXT:    shl.b64 %rd56, %rd3, %r6;
+; CHECK-NEXT:    mov.b64 %rd50, %rd51;
 ; CHECK-NEXT:    @%p14 bra $L__BB5_4;
 ; CHECK-NEXT:  // %bb.1: // %udiv-preheader
-; CHECK-NEXT:    cvt.u32.u64 %r9, %rd99;
-; CHECK-NEXT:    shr.u64 %rd69, %rd41, %r9;
+; CHECK-NEXT:    cvt.u32.u64 %r9, %rd52;
+; CHECK-NEXT:    shr.u64 %rd26, %rd3, %r9;
 ; CHECK-NEXT:    sub.s32 %r10, 64, %r9;
-; CHECK-NEXT:    shl.b64 %rd70, %rd42, %r10;
-; CHECK-NEXT:    or.b64 %rd71, %rd69, %rd70;
+; CHECK-NEXT:    shl.b64 %rd27, %rd4, %r10;
+; CHECK-NEXT:    or.b64 %rd28, %rd26, %rd27;
 ; CHECK-NEXT:    add.s32 %r11, %r9, -64;
-; CHECK-NEXT:    shr.u64 %rd72, %rd42, %r11;
+; CHECK-NEXT:    shr.u64 %rd29, %rd4, %r11;
 ; CHECK-NEXT:    setp.gt.s32 %p16, %r9, 63;
-; CHECK-NEXT:    selp.b64 %rd101, %rd72, %rd71, %p16;
-; CHECK-NEXT:    shr.u64 %rd102, %rd42, %r9;
-; CHECK-NEXT:    add.cc.s64 %rd33, %rd43, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd34, %rd44, -1;
-; CHECK-NEXT:    mov.b64 %rd94, 0;
-; CHECK-NEXT:    mov.b64 %rd97, %rd94;
+; CHECK-NEXT:    selp.b64 %rd54, %rd29, %rd28, %p16;
+; CHECK-NEXT:    shr.u64 %rd55, %rd4, %r9;
+; CHECK-NEXT:    add.cc.s64 %rd1, %rd5, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd2, %rd6, -1;
+; CHECK-NEXT:    mov.b64 %rd50, 0;
+; CHECK-NEXT:    mov.b64 %rd51, %rd50;
 ; CHECK-NEXT:  $L__BB5_2: // %udiv-do-while
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    shr.u64 %rd73, %rd101, 63;
-; CHECK-NEXT:    shl.b64 %rd74, %rd102, 1;
-; CHECK-NEXT:    or.b64 %rd75, %rd74, %rd73;
-; CHECK-NEXT:    shl.b64 %rd76, %rd101, 1;
-; CHECK-NEXT:    shr.u64 %rd77, %rd104, 63;
-; CHECK-NEXT:    or.b64 %rd78, %rd76, %rd77;
-; CHECK-NEXT:    shr.u64 %rd79, %rd103, 63;
-; CHECK-NEXT:    shl.b64 %rd80, %rd104, 1;
-; CHECK-NEXT:    or.b64 %rd81, %rd80, %rd79;
-; CHECK-NEXT:    shl.b64 %rd82, %rd103, 1;
-; CHECK-NEXT:    or.b64 %rd103, %rd97, %rd82;
-; CHECK-NEXT:    or.b64 %rd104, %rd94, %rd81;
-; CHECK-NEXT:    sub.cc.s64 %rd83, %rd33, %rd78;
-; CHECK-NEXT:    subc.cc.s64 %rd84, %rd34, %rd75;
-; CHECK-NEXT:    shr.s64 %rd85, %rd84, 63;
-; CHECK-NEXT:    and.b64 %rd97, %rd85, 1;
-; CHECK-NEXT:    and.b64 %rd86, %rd85, %rd43;
-; CHECK-NEXT:    and.b64 %rd87, %rd85, %rd44;
-; CHECK-NEXT:    sub.cc.s64 %rd101, %rd78, %rd86;
-; CHECK-NEXT:    subc.cc.s64 %rd102, %rd75, %rd87;
-; CHECK-NEXT:    add.cc.s64 %rd99, %rd99, -1;
-; CHECK-NEXT:    addc.cc.s64 %rd100, %rd100, -1;
-; CHECK-NEXT:    or.b64 %rd88, %rd99, %rd100;
-; CHECK-NEXT:    setp.eq.b64 %p17, %rd88, 0;
+; CHECK-NEXT:    shr.u64 %rd30, %rd54, 63;
+; CHECK-NEXT:    shl.b64 %rd31, %rd55, 1;
+; CHECK-NEXT:    or.b64 %rd32, %rd31, %rd30;
+; CHECK-NEXT:    shl.b64 %rd33, %rd54, 1;
+; CHECK-NEXT:    shr.u64 %rd34, %rd57, 63;
+; CHECK-NEXT:    or.b64 %rd35, %rd33, %rd34;
+; CHECK-NEXT:    shr.u64 %rd36, %rd56, 63;
+; CHECK-NEXT:    shl.b64 %rd37, %rd57, 1;
+; CHECK-NEXT:    or.b64 %rd38, %rd37, %rd36;
+; CHECK-NEXT:    shl.b64 %rd39, %rd56, 1;
+; CHECK-NEXT:    or.b64 %rd56, %rd51, %rd39;
+; CHECK-NEXT:    or.b64 %rd57, %rd50, %rd38;
+; CHECK-NEXT:    sub.cc.s64 %rd40, %rd1, %rd35;
+; CHECK-NEXT:    subc.cc.s64 %rd41, %rd2, %rd32;
+; CHECK-NEXT:    shr.s64 %rd42, %rd41, 63;
+; CHECK-NEXT:    and.b64 %rd51, %rd42, 1;
+; CHECK-NEXT:    and.b64 %rd43, %rd42, %rd5;
+; CHECK-NEXT:    and.b64 %rd44, %rd42, %rd6;
+; CHECK-NEXT:    sub.cc.s64 %rd54, %rd35, %rd43;
+; CHECK-NEXT:    subc.cc.s64 %rd55, %rd32, %rd44;
+; CHECK-NEXT:    add.cc.s64 %rd52, %rd52, -1;
+; CHECK-NEXT:    addc.cc.s64 %rd53, %rd53, -1;
+; CHECK-NEXT:    or.b64 %rd45, %rd52, %rd53;
+; CHECK-NEXT:    setp.eq.b64 %p17, %rd45, 0;
 ; CHECK-NEXT:    @%p17 bra $L__BB5_4;
 ; CHECK-NEXT:    bra.uni $L__BB5_2;
 ; CHECK-NEXT:  $L__BB5_4: // %udiv-loop-exit
-; CHECK-NEXT:    shr.u64 %rd89, %rd103, 63;
-; CHECK-NEXT:    shl.b64 %rd90, %rd104, 1;
-; CHECK-NEXT:    or.b64 %rd91, %rd90, %rd89;
-; CHECK-NEXT:    shl.b64 %rd92, %rd103, 1;
-; CHECK-NEXT:    or.b64 %rd105, %rd97, %rd92;
-; CHECK-NEXT:    or.b64 %rd106, %rd94, %rd91;
+; CHECK-NEXT:    shr.u64 %rd46, %rd56, 63;
+; CHECK-NEXT:    shl.b64 %rd47, %rd57, 1;
+; CHECK-NEXT:    or.b64 %rd48, %rd47, %rd46;
+; CHECK-NEXT:    shl.b64 %rd49, %rd56, 1;
+; CHECK-NEXT:    or.b64 %rd58, %rd51, %rd49;
+; CHECK-NEXT:    or.b64 %rd59, %rd50, %rd48;
 ; CHECK-NEXT:  $L__BB5_5: // %udiv-end
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd105, %rd106};
+; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd58, %rd59};
 ; CHECK-NEXT:    ret;
   %div = udiv i128 %lhs, %rhs
   ret i128 %div
diff --git a/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll
index 74136bbe478c9..7f48245af4a26 100644
--- a/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i16x2-instructions.ll
@@ -635,7 +635,7 @@ declare <2 x i16> @test_callee(<2 x i16> %a, <2 x i16> %b) #0
 define <2 x i16> @test_call(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-LABEL: test_call(
 ; COMMON:       {
-; COMMON-NEXT:    .reg .b32 %r<5>;
+; COMMON-NEXT:    .reg .b32 %r<4>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r2, [test_call_param_1];
@@ -658,7 +658,7 @@ define <2 x i16> @test_call(<2 x i16> %a, <2 x i16> %b) #0 {
 define <2 x i16> @test_call_flipped(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-LABEL: test_call_flipped(
 ; COMMON:       {
-; COMMON-NEXT:    .reg .b32 %r<5>;
+; COMMON-NEXT:    .reg .b32 %r<4>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r2, [test_call_flipped_param_1];
@@ -681,7 +681,7 @@ define <2 x i16> @test_call_flipped(<2 x i16> %a, <2 x i16> %b) #0 {
 define <2 x i16> @test_tailcall_flipped(<2 x i16> %a, <2 x i16> %b) #0 {
 ; COMMON-LABEL: test_tailcall_flipped(
 ; COMMON:       {
-; COMMON-NEXT:    .reg .b32 %r<5>;
+; COMMON-NEXT:    .reg .b32 %r<4>;
 ; COMMON-EMPTY:
 ; COMMON-NEXT:  // %bb.0:
 ; COMMON-NEXT:    ld.param.b32 %r2, [test_tailcall_flipped_param_1];
diff --git a/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll
index 98f94bb7b3ac1..f4053d84593a5 100644
--- a/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x2-instructions.ll
@@ -69,7 +69,7 @@ define <2 x i8> @test_bitcast_i16_2xi8(i16 %a) {
 define <2 x i8> @test_call_2xi8(<2 x i8> %a) {
 ; O0-LABEL: test_call_2xi8(
 ; O0:       {
-; O0-NEXT:    .reg .b16 %rs<7>;
+; O0-NEXT:    .reg .b16 %rs<5>;
 ; O0-NEXT:    .reg .b32 %r<2>;
 ; O0-EMPTY:
 ; O0-NEXT:  // %bb.0:
@@ -87,7 +87,7 @@ define <2 x i8> @test_call_2xi8(<2 x i8> %a) {
 ;
 ; O3-LABEL: test_call_2xi8(
 ; O3:       {
-; O3-NEXT:    .reg .b16 %rs<7>;
+; O3-NEXT:    .reg .b16 %rs<5>;
 ; O3-EMPTY:
 ; O3-NEXT:  // %bb.0:
 ; O3-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [test_call_2xi8_param_0];
@@ -103,5 +103,34 @@ define <2 x i8> @test_call_2xi8(<2 x i8> %a) {
   %res = call <2 x i8> @test_call_2xi8(<2 x i8> %a)
   ret <2 x i8> %res
 }
+
+define <2 x float> @test_uitofp_2xi8(<2 x i8> %a) {
+; O0-LABEL: test_uitofp_2xi8(
+; O0:       {
+; O0-NEXT:    .reg .b16 %rs<3>;
+; O0-NEXT:    .reg .b32 %r<4>;
+; O0-EMPTY:
+; O0-NEXT:  // %bb.0:
+; O0-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [test_uitofp_2xi8_param_0];
+; O0-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; O0-NEXT:    cvt.rn.f32.u16 %r2, %rs2;
+; O0-NEXT:    cvt.rn.f32.u16 %r3, %rs1;
+; O0-NEXT:    st.param.v2.b32 [func_retval0], {%r3, %r2};
+; O0-NEXT:    ret;
+;
+; O3-LABEL: test_uitofp_2xi8(
+; O3:       {
+; O3-NEXT:    .reg .b16 %rs<3>;
+; O3-NEXT:    .reg .b32 %r<3>;
+; O3-EMPTY:
+; O3-NEXT:  // %bb.0:
+; O3-NEXT:    ld.param.v2.b8 {%rs1, %rs2}, [test_uitofp_2xi8_param_0];
+; O3-NEXT:    cvt.rn.f32.u16 %r1, %rs2;
+; O3-NEXT:    cvt.rn.f32.u16 %r2, %rs1;
+; O3-NEXT:    st.param.v2.b32 [func_retval0], {%r2, %r1};
+; O3-NEXT:    ret;
+  %1 = uitofp <2 x i8> %a to <2 x float>
+  ret <2 x float> %1
+}
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; COMMON: {{.*}}
diff --git a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
index 26336b83c4f96..40d6a07310265 100644
--- a/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/i8x4-instructions.ll
@@ -1298,7 +1298,7 @@ declare <4 x i8> @test_callee(<4 x i8> %a, <4 x i8> %b) #0
 define <4 x i8> @test_call(<4 x i8> %a, <4 x i8> %b) #0 {
 ; O0-LABEL: test_call(
 ; O0:       {
-; O0-NEXT:    .reg .b32 %r<5>;
+; O0-NEXT:    .reg .b32 %r<4>;
 ; O0-EMPTY:
 ; O0-NEXT:  // %bb.0:
 ; O0-NEXT:    ld.param.b32 %r2, [test_call_param_1];
@@ -1317,7 +1317,7 @@ define <4 x i8> @test_call(<4 x i8> %a, <4 x i8> %b) #0 {
 ;
 ; O3-LABEL: test_call(
 ; O3:       {
-; O3-NEXT:    .reg .b32 %r<5>;
+; O3-NEXT:    .reg .b32 %r<4>;
 ; O3-EMPTY:
 ; O3-NEXT:  // %bb.0:
 ; O3-NEXT:    ld.param.b32 %r1, [test_call_param_0];
@@ -1340,7 +1340,7 @@ define <4 x i8> @test_call(<4 x i8> %a, <4 x i8> %b) #0 {
 define <4 x i8> @test_call_flipped(<4 x i8> %a, <4 x i8> %b) #0 {
 ; O0-LABEL: test_call_flipped(
 ; O0:       {
-; O0-NEXT:    .reg .b32 %r<5>;
+; O0-NEXT:    .reg .b32 %r<4>;
 ; O0-EMPTY:
 ; O0-NEXT:  // %bb.0:
 ; O0-NEXT:    ld.param.b32 %r2, [test_call_flipped_param_1];
@@ -1359,7 +1359,7 @@ define <4 x i8> @test_call_flipped(<4 x i8> %a, <4 x i8> %b) #0 {
 ;
 ; O3-LABEL: test_call_flipped(
 ; O3:       {
-; O3-NEXT:    .reg .b32 %r<5>;
+; O3-NEXT:    .reg .b32 %r<4>;
 ; O3-EMPTY:
 ; O3-NEXT:  // %bb.0:
 ; O3-NEXT:    ld.param.b32 %r1, [test_call_flipped_param_0];
@@ -1382,7 +1382,7 @@ define <4 x i8> @test_call_flipped(<4 x i8> %a, <4 x i8> %b) #0 {
 define <4 x i8> @test_tailcall_flipped(<4 x i8> %a, <4 x i8> %b) #0 {
 ; O0-LABEL: test_tailcall_flipped(
 ; O0:       {
-; O0-NEXT:    .reg .b32 %r<5>;
+; O0-NEXT:    .reg .b32 %r<4>;
 ; O0-EMPTY:
 ; O0-NEXT:  // %bb.0:
 ; O0-NEXT:    ld.param.b32 %r2, [test_tailcall_flipped_param_1];
@@ -1401,7 +1401,7 @@ define <4 x i8> @test_tailcall_flipped(<4 x i8> %a, <4 x i8> %b) #0 {
 ;
 ; O3-LABEL: test_tailcall_flipped(
 ; O3:       {
-; O3-NEXT:    .reg .b32 %r<5>;
+; O3-NEXT:    .reg .b32 %r<4>;
 ; O3-EMPTY:
 ; O3-NEXT:  // %bb.0:
 ; O3-NEXT:    ld.param.b32 %r1, [test_tailcall_flipped_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/indirect_byval.ll b/llvm/test/CodeGen/NVPTX/indirect_byval.ll
index 782e6720e5112..673fb73948268 100644
--- a/llvm/test/CodeGen/NVPTX/indirect_byval.ll
+++ b/llvm/test/CodeGen/NVPTX/indirect_byval.ll
@@ -16,8 +16,8 @@ define internal i32 @foo() {
 ; CHECK-NEXT:    .reg .b64 %SP;
 ; CHECK-NEXT:    .reg .b64 %SPL;
 ; CHECK-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    mov.b64 %SPL, __local_depot0;
@@ -29,8 +29,8 @@ define internal i32 @foo() {
 ; CHECK-NEXT:    .param .b32 retval0;
 ; CHECK-NEXT:    add.u64 %rd2, %SP, 0;
 ; CHECK-NEXT:    st.param.b64 [param1], %rd2;
-; CHECK-NEXT:    add.u64 %rd4, %SPL, 1;
-; CHECK-NEXT:    ld.local.b8 %rs1, [%rd4];
+; CHECK-NEXT:    add.u64 %rd3, %SPL, 1;
+; CHECK-NEXT:    ld.local.b8 %rs1, [%rd3];
 ; CHECK-NEXT:    st.param.b8 [param0], %rs1;
 ; CHECK-NEXT:    prototype_0 : .callprototype (.param .b32 _) _ (.param .align 1 .b8 _[1], .param .b64 _);
 ; CHECK-NEXT:    call (retval0), %rd1, (param0, param1), prototype_0;
@@ -53,8 +53,8 @@ define internal i32 @bar() {
 ; CHECK-NEXT:    .local .align 8 .b8 __local_depot1[16];
 ; CHECK-NEXT:    .reg .b64 %SP;
 ; CHECK-NEXT:    .reg .b64 %SPL;
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    mov.b64 %SPL, __local_depot1;
@@ -66,9 +66,9 @@ define internal i32 @bar() {
 ; CHECK-NEXT:    .param .b32 retval0;
 ; CHECK-NEXT:    add.u64 %rd2, %SP, 0;
 ; CHECK-NEXT:    st.param.b64 [param1], %rd2;
-; CHECK-NEXT:    add.u64 %rd4, %SPL, 8;
-; CHECK-NEXT:    ld.local.b64 %rd5, [%rd4];
-; CHECK-NEXT:    st.param.b64 [param0], %rd5;
+; CHECK-NEXT:    add.u64 %rd3, %SPL, 8;
+; CHECK-NEXT:    ld.local.b64 %rd4, [%rd3];
+; CHECK-NEXT:    st.param.b64 [param0], %rd4;
 ; CHECK-NEXT:    prototype_1 : .callprototype (.param .b32 _) _ (.param .align 8 .b8 _[8], .param .b64 _);
 ; CHECK-NEXT:    call (retval0), %rd1, (param0, param1), prototype_1;
 ; CHECK-NEXT:    ld.param.b32 %r1, [retval0];
diff --git a/llvm/test/CodeGen/NVPTX/inline-asm-b128-test3.ll b/llvm/test/CodeGen/NVPTX/inline-asm-b128-test3.ll
index 037d7df1aee59..bf0dd58e27a35 100644
--- a/llvm/test/CodeGen/NVPTX/inline-asm-b128-test3.ll
+++ b/llvm/test/CodeGen/NVPTX/inline-asm-b128-test3.ll
@@ -11,32 +11,32 @@ define void @test_b128_in_loop() {
 ; CHECK-LABEL: test_b128_in_loop(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<3>;
-; CHECK-NEXT:    .reg .b64 %rd<15>;
-; CHECK-NEXT:    .reg .b128 %rq<3>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b128 %rq<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.global.s32 %rd1, [size];
 ; CHECK-NEXT:    setp.eq.b64 %p1, %rd1, 0;
 ; CHECK-NEXT:    @%p1 bra $L__BB0_3;
 ; CHECK-NEXT:  // %bb.1: // %BB1
-; CHECK-NEXT:    ld.global.v2.b64 {%rd12, %rd13}, [x];
-; CHECK-NEXT:    mov.b64 %rd14, 0;
+; CHECK-NEXT:    ld.global.v2.b64 {%rd2, %rd3}, [x];
+; CHECK-NEXT:    mov.b64 %rd4, 0;
 ; CHECK-NEXT:  $L__BB0_2: // %BB2
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    mov.b128 %rq1, {%rd12, %rd13};
+; CHECK-NEXT:    mov.b128 %rq1, {%rd2, %rd3};
 ; CHECK-NEXT:    // begin inline asm
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:    .reg .b64 lo;
 ; CHECK-NEXT:    .reg .b64 hi;
 ; CHECK-NEXT:    mov.b128 {lo, hi}, %rq1;
-; CHECK-NEXT:    add.cc.u64 lo, lo, %rd14;
+; CHECK-NEXT:    add.cc.u64 lo, lo, %rd4;
 ; CHECK-NEXT:    mov.b128 %rq1, {lo, hi};
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    // end inline asm
-; CHECK-NEXT:    mov.b128 {%rd12, %rd13}, %rq1;
-; CHECK-NEXT:    st.global.v2.b64 [x], {%rd12, %rd13};
-; CHECK-NEXT:    add.s64 %rd14, %rd14, 1;
-; CHECK-NEXT:    setp.ne.b64 %p2, %rd1, %rd14;
+; CHECK-NEXT:    mov.b128 {%rd2, %rd3}, %rq1;
+; CHECK-NEXT:    st.global.v2.b64 [x], {%rd2, %rd3};
+; CHECK-NEXT:    add.s64 %rd4, %rd4, 1;
+; CHECK-NEXT:    setp.ne.b64 %p2, %rd1, %rd4;
 ; CHECK-NEXT:    @%p2 bra $L__BB0_2;
 ; CHECK-NEXT:  $L__BB0_3: // %BB3
 ; CHECK-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/intrinsics.ll b/llvm/test/CodeGen/NVPTX/intrinsics.ll
index 4ed50632251cb..6bdb8ead7a64a 100644
--- a/llvm/test/CodeGen/NVPTX/intrinsics.ll
+++ b/llvm/test/CodeGen/NVPTX/intrinsics.ll
@@ -267,6 +267,23 @@ define i64 @test_globaltimer() {
   ret i64 %ret
 }
 
+define i32 @test_globaltimer_lo(){
+; CHECK-LABEL: test_globaltimer_lo(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    mov.u32 %r1, %globaltimer_lo;
+; CHECK-NEXT:    mov.u32 %r2, %globaltimer_lo;
+; CHECK-NEXT:    add.s32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %a = tail call i32 @llvm.nvvm.read.ptx.sreg.globaltimer.lo()
+  %b = tail call i32 @llvm.nvvm.read.ptx.sreg.globaltimer.lo()
+  %ret = add i32 %a, %b
+  ret i32 %ret
+}
+
 define i64 @test_cyclecounter() {
 ; CHECK-LABEL: test_cyclecounter(
 ; CHECK:       {
diff --git a/llvm/test/CodeGen/NVPTX/jump-table.ll b/llvm/test/CodeGen/NVPTX/jump-table.ll
index a6238352179ca..4620c5e01008c 100644
--- a/llvm/test/CodeGen/NVPTX/jump-table.ll
+++ b/llvm/test/CodeGen/NVPTX/jump-table.ll
@@ -10,11 +10,11 @@ define void @foo(i32 %i) {
 ; CHECK-LABEL: foo(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
-; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
-; CHECK-NEXT:    ld.param.b32 %r2, [foo_param_0];
-; CHECK-NEXT:    setp.gt.u32 %p1, %r2, 3;
+; CHECK-NEXT:    ld.param.b32 %r1, [foo_param_0];
+; CHECK-NEXT:    setp.gt.u32 %p1, %r1, 3;
 ; CHECK-NEXT:    @%p1 bra $L__BB0_6;
 ; CHECK-NEXT:  // %bb.1: // %entry
 ; CHECK-NEXT:    $L_brx_0: .branchtargets
@@ -22,7 +22,7 @@ define void @foo(i32 %i) {
 ; CHECK-NEXT:     $L__BB0_3,
 ; CHECK-NEXT:     $L__BB0_4,
 ; CHECK-NEXT:     $L__BB0_5;
-; CHECK-NEXT:    brx.idx %r2, $L_brx_0;
+; CHECK-NEXT:    brx.idx %r1, $L_brx_0;
 ; CHECK-NEXT:  $L__BB0_2: // %case0
 ; CHECK-NEXT:    st.global.b32 [out], 0;
 ; CHECK-NEXT:    bra.uni $L__BB0_6;
diff --git a/llvm/test/CodeGen/NVPTX/ld-param-sink.ll b/llvm/test/CodeGen/NVPTX/ld-param-sink.ll
index 03523a3be50c2..dfb0e80d0907d 100644
--- a/llvm/test/CodeGen/NVPTX/ld-param-sink.ll
+++ b/llvm/test/CodeGen/NVPTX/ld-param-sink.ll
@@ -12,7 +12,7 @@ define ptr @foo(i1 %cond) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
 ; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %entry
 ; CHECK-NEXT:    ld.param.b8 %rs1, [foo_param_0];
@@ -21,14 +21,14 @@ define ptr @foo(i1 %cond) {
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .b64 retval0;
 ; CHECK-NEXT:    call.uni (retval0), baz, ();
-; CHECK-NEXT:    ld.param.b64 %rd2, [retval0];
+; CHECK-NEXT:    ld.param.b64 %rd1, [retval0];
 ; CHECK-NEXT:    } // callseq 0
 ; CHECK-NEXT:    @%p1 bra $L__BB0_2;
 ; CHECK-NEXT:  // %bb.1: // %bb
 ; CHECK-NEXT:    { // callseq 1, 0
 ; CHECK-NEXT:    .param .b64 param0;
 ; CHECK-NEXT:    .param .b64 retval0;
-; CHECK-NEXT:    st.param.b64 [param0], %rd2;
+; CHECK-NEXT:    st.param.b64 [param0], %rd1;
 ; CHECK-NEXT:    call.uni (retval0), bar, (param0);
 ; CHECK-NEXT:    } // callseq 1
 ; CHECK-NEXT:  $L__BB0_2: // %common.ret
diff --git a/llvm/test/CodeGen/NVPTX/ldparam-v4.ll b/llvm/test/CodeGen/NVPTX/ldparam-v4.ll
index efa2666090ccc..cb1d12661ed64 100644
--- a/llvm/test/CodeGen/NVPTX/ldparam-v4.ll
+++ b/llvm/test/CodeGen/NVPTX/ldparam-v4.ll
@@ -7,16 +7,17 @@ declare <4 x float> @bar()
 define void @foo(ptr %ptr) {
 ; CHECK-LABEL: foo(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [foo_param_0];
 ; CHECK-NEXT:    { // callseq 0, 0
 ; CHECK-NEXT:    .param .align 16 .b8 retval0[16];
 ; CHECK-NEXT:    call.uni (retval0), bar, ();
-; CHECK-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [retval0];
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [retval0];
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    st.v2.b64 [%rd1], {%rd2, %rd3};
+; CHECK-NEXT:    st.v4.b32 [%rd1], {%r1, %r2, %r3, %r4};
 ; CHECK-NEXT:    ret;
   %val = tail call <4 x float> @bar()
   store <4 x float> %val, ptr %ptr
diff --git a/llvm/test/CodeGen/NVPTX/load-store-vectors-256.ll b/llvm/test/CodeGen/NVPTX/load-store-vectors-256.ll
index 68c53cde7f9ac..a846607d816c5 100644
--- a/llvm/test/CodeGen/NVPTX/load-store-vectors-256.ll
+++ b/llvm/test/CodeGen/NVPTX/load-store-vectors-256.ll
@@ -137,18 +137,32 @@ define void @generic_4xi64(ptr %a, ptr %b) {
 }
 
 define void @generic_8xfloat(ptr %a, ptr %b) {
-; CHECK-LABEL: generic_8xfloat(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<7>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [generic_8xfloat_param_0];
-; CHECK-NEXT:    ld.v2.b64 {%rd2, %rd3}, [%rd1];
-; CHECK-NEXT:    ld.v2.b64 {%rd4, %rd5}, [%rd1+16];
-; CHECK-NEXT:    ld.param.b64 %rd6, [generic_8xfloat_param_1];
-; CHECK-NEXT:    st.v2.b64 [%rd6+16], {%rd4, %rd5};
-; CHECK-NEXT:    st.v2.b64 [%rd6], {%rd2, %rd3};
-; CHECK-NEXT:    ret;
+; SM90-LABEL: generic_8xfloat(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<9>;
+; SM90-NEXT:    .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b64 %rd1, [generic_8xfloat_param_0];
+; SM90-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    ld.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; SM90-NEXT:    ld.param.b64 %rd2, [generic_8xfloat_param_1];
+; SM90-NEXT:    st.v4.b32 [%rd2+16], {%r5, %r6, %r7, %r8};
+; SM90-NEXT:    st.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
+; SM90-NEXT:    ret;
+;
+; SM100-LABEL: generic_8xfloat(
+; SM100:       {
+; SM100-NEXT:    .reg .b64 %rd<7>;
+; SM100-EMPTY:
+; SM100-NEXT:  // %bb.0:
+; SM100-NEXT:    ld.param.b64 %rd1, [generic_8xfloat_param_0];
+; SM100-NEXT:    ld.v2.b64 {%rd2, %rd3}, [%rd1];
+; SM100-NEXT:    ld.v2.b64 {%rd4, %rd5}, [%rd1+16];
+; SM100-NEXT:    ld.param.b64 %rd6, [generic_8xfloat_param_1];
+; SM100-NEXT:    st.v2.b64 [%rd6+16], {%rd4, %rd5};
+; SM100-NEXT:    st.v2.b64 [%rd6], {%rd2, %rd3};
+; SM100-NEXT:    ret;
   %a.load = load <8 x float>, ptr %a
   store <8 x float> %a.load, ptr %b
   ret void
@@ -288,18 +302,32 @@ define void @generic_volatile_4xi64(ptr %a, ptr %b) {
 }
 
 define void @generic_volatile_8xfloat(ptr %a, ptr %b) {
-; CHECK-LABEL: generic_volatile_8xfloat(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<7>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [generic_volatile_8xfloat_param_0];
-; CHECK-NEXT:    ld.volatile.v2.b64 {%rd2, %rd3}, [%rd1];
-; CHECK-NEXT:    ld.volatile.v2.b64 {%rd4, %rd5}, [%rd1+16];
-; CHECK-NEXT:    ld.param.b64 %rd6, [generic_volatile_8xfloat_param_1];
-; CHECK-NEXT:    st.volatile.v2.b64 [%rd6+16], {%rd4, %rd5};
-; CHECK-NEXT:    st.volatile.v2.b64 [%rd6], {%rd2, %rd3};
-; CHECK-NEXT:    ret;
+; SM90-LABEL: generic_volatile_8xfloat(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<9>;
+; SM90-NEXT:    .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b64 %rd1, [generic_volatile_8xfloat_param_0];
+; SM90-NEXT:    ld.volatile.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    ld.volatile.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; SM90-NEXT:    ld.param.b64 %rd2, [generic_volatile_8xfloat_param_1];
+; SM90-NEXT:    st.volatile.v4.b32 [%rd2+16], {%r5, %r6, %r7, %r8};
+; SM90-NEXT:    st.volatile.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
+; SM90-NEXT:    ret;
+;
+; SM100-LABEL: generic_volatile_8xfloat(
+; SM100:       {
+; SM100-NEXT:    .reg .b64 %rd<7>;
+; SM100-EMPTY:
+; SM100-NEXT:  // %bb.0:
+; SM100-NEXT:    ld.param.b64 %rd1, [generic_volatile_8xfloat_param_0];
+; SM100-NEXT:    ld.volatile.v2.b64 {%rd2, %rd3}, [%rd1];
+; SM100-NEXT:    ld.volatile.v2.b64 {%rd4, %rd5}, [%rd1+16];
+; SM100-NEXT:    ld.param.b64 %rd6, [generic_volatile_8xfloat_param_1];
+; SM100-NEXT:    st.volatile.v2.b64 [%rd6+16], {%rd4, %rd5};
+; SM100-NEXT:    st.volatile.v2.b64 [%rd6], {%rd2, %rd3};
+; SM100-NEXT:    ret;
   %a.load = load volatile <8 x float>, ptr %a
   store volatile <8 x float> %a.load, ptr %b
   ret void
@@ -514,15 +542,16 @@ define void @global_4xi64(ptr addrspace(1) %a, ptr addrspace(1) %b) {
 define void @global_8xfloat(ptr addrspace(1) %a, ptr addrspace(1) %b) {
 ; SM90-LABEL: global_8xfloat(
 ; SM90:       {
-; SM90-NEXT:    .reg .b64 %rd<7>;
+; SM90-NEXT:    .reg .b32 %r<9>;
+; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b64 %rd1, [global_8xfloat_param_0];
-; SM90-NEXT:    ld.global.v2.b64 {%rd2, %rd3}, [%rd1];
-; SM90-NEXT:    ld.global.v2.b64 {%rd4, %rd5}, [%rd1+16];
-; SM90-NEXT:    ld.param.b64 %rd6, [global_8xfloat_param_1];
-; SM90-NEXT:    st.global.v2.b64 [%rd6+16], {%rd4, %rd5};
-; SM90-NEXT:    st.global.v2.b64 [%rd6], {%rd2, %rd3};
+; SM90-NEXT:    ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    ld.global.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; SM90-NEXT:    ld.param.b64 %rd2, [global_8xfloat_param_1];
+; SM90-NEXT:    st.global.v4.b32 [%rd2+16], {%r5, %r6, %r7, %r8};
+; SM90-NEXT:    st.global.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
 ; SM90-NEXT:    ret;
 ;
 ; SM100-LABEL: global_8xfloat(
@@ -758,15 +787,16 @@ define void @global_volatile_4xi64(ptr addrspace(1) %a, ptr addrspace(1) %b) {
 define void @global_volatile_8xfloat(ptr addrspace(1) %a, ptr addrspace(1) %b) {
 ; SM90-LABEL: global_volatile_8xfloat(
 ; SM90:       {
-; SM90-NEXT:    .reg .b64 %rd<7>;
+; SM90-NEXT:    .reg .b32 %r<9>;
+; SM90-NEXT:    .reg .b64 %rd<3>;
 ; SM90-EMPTY:
 ; SM90-NEXT:  // %bb.0:
 ; SM90-NEXT:    ld.param.b64 %rd1, [global_volatile_8xfloat_param_0];
-; SM90-NEXT:    ld.volatile.global.v2.b64 {%rd2, %rd3}, [%rd1];
-; SM90-NEXT:    ld.volatile.global.v2.b64 {%rd4, %rd5}, [%rd1+16];
-; SM90-NEXT:    ld.param.b64 %rd6, [global_volatile_8xfloat_param_1];
-; SM90-NEXT:    st.volatile.global.v2.b64 [%rd6+16], {%rd4, %rd5};
-; SM90-NEXT:    st.volatile.global.v2.b64 [%rd6], {%rd2, %rd3};
+; SM90-NEXT:    ld.volatile.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    ld.volatile.global.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; SM90-NEXT:    ld.param.b64 %rd2, [global_volatile_8xfloat_param_1];
+; SM90-NEXT:    st.volatile.global.v4.b32 [%rd2+16], {%r5, %r6, %r7, %r8};
+; SM90-NEXT:    st.volatile.global.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
 ; SM90-NEXT:    ret;
 ;
 ; SM100-LABEL: global_volatile_8xfloat(
@@ -931,18 +961,32 @@ define void @shared_4xi64(ptr addrspace(3) %a, ptr addrspace(3) %b) {
 }
 
 define void @shared_8xfloat(ptr addrspace(3) %a, ptr addrspace(3) %b) {
-; CHECK-LABEL: shared_8xfloat(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<7>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [shared_8xfloat_param_0];
-; CHECK-NEXT:    ld.shared.v2.b64 {%rd2, %rd3}, [%rd1];
-; CHECK-NEXT:    ld.shared.v2.b64 {%rd4, %rd5}, [%rd1+16];
-; CHECK-NEXT:    ld.param.b64 %rd6, [shared_8xfloat_param_1];
-; CHECK-NEXT:    st.shared.v2.b64 [%rd6+16], {%rd4, %rd5};
-; CHECK-NEXT:    st.shared.v2.b64 [%rd6], {%rd2, %rd3};
-; CHECK-NEXT:    ret;
+; SM90-LABEL: shared_8xfloat(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<9>;
+; SM90-NEXT:    .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b64 %rd1, [shared_8xfloat_param_0];
+; SM90-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    ld.shared.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; SM90-NEXT:    ld.param.b64 %rd2, [shared_8xfloat_param_1];
+; SM90-NEXT:    st.shared.v4.b32 [%rd2+16], {%r5, %r6, %r7, %r8};
+; SM90-NEXT:    st.shared.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
+; SM90-NEXT:    ret;
+;
+; SM100-LABEL: shared_8xfloat(
+; SM100:       {
+; SM100-NEXT:    .reg .b64 %rd<7>;
+; SM100-EMPTY:
+; SM100-NEXT:  // %bb.0:
+; SM100-NEXT:    ld.param.b64 %rd1, [shared_8xfloat_param_0];
+; SM100-NEXT:    ld.shared.v2.b64 {%rd2, %rd3}, [%rd1];
+; SM100-NEXT:    ld.shared.v2.b64 {%rd4, %rd5}, [%rd1+16];
+; SM100-NEXT:    ld.param.b64 %rd6, [shared_8xfloat_param_1];
+; SM100-NEXT:    st.shared.v2.b64 [%rd6+16], {%rd4, %rd5};
+; SM100-NEXT:    st.shared.v2.b64 [%rd6], {%rd2, %rd3};
+; SM100-NEXT:    ret;
   %a.load = load <8 x float>, ptr addrspace(3) %a
   store <8 x float> %a.load, ptr addrspace(3) %b
   ret void
@@ -1082,18 +1126,32 @@ define void @shared_volatile_4xi64(ptr addrspace(3) %a, ptr addrspace(3) %b) {
 }
 
 define void @shared_volatile_8xfloat(ptr addrspace(3) %a, ptr addrspace(3) %b) {
-; CHECK-LABEL: shared_volatile_8xfloat(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<7>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [shared_volatile_8xfloat_param_0];
-; CHECK-NEXT:    ld.volatile.shared.v2.b64 {%rd2, %rd3}, [%rd1];
-; CHECK-NEXT:    ld.volatile.shared.v2.b64 {%rd4, %rd5}, [%rd1+16];
-; CHECK-NEXT:    ld.param.b64 %rd6, [shared_volatile_8xfloat_param_1];
-; CHECK-NEXT:    st.volatile.shared.v2.b64 [%rd6+16], {%rd4, %rd5};
-; CHECK-NEXT:    st.volatile.shared.v2.b64 [%rd6], {%rd2, %rd3};
-; CHECK-NEXT:    ret;
+; SM90-LABEL: shared_volatile_8xfloat(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<9>;
+; SM90-NEXT:    .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b64 %rd1, [shared_volatile_8xfloat_param_0];
+; SM90-NEXT:    ld.volatile.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    ld.volatile.shared.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; SM90-NEXT:    ld.param.b64 %rd2, [shared_volatile_8xfloat_param_1];
+; SM90-NEXT:    st.volatile.shared.v4.b32 [%rd2+16], {%r5, %r6, %r7, %r8};
+; SM90-NEXT:    st.volatile.shared.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
+; SM90-NEXT:    ret;
+;
+; SM100-LABEL: shared_volatile_8xfloat(
+; SM100:       {
+; SM100-NEXT:    .reg .b64 %rd<7>;
+; SM100-EMPTY:
+; SM100-NEXT:  // %bb.0:
+; SM100-NEXT:    ld.param.b64 %rd1, [shared_volatile_8xfloat_param_0];
+; SM100-NEXT:    ld.volatile.shared.v2.b64 {%rd2, %rd3}, [%rd1];
+; SM100-NEXT:    ld.volatile.shared.v2.b64 {%rd4, %rd5}, [%rd1+16];
+; SM100-NEXT:    ld.param.b64 %rd6, [shared_volatile_8xfloat_param_1];
+; SM100-NEXT:    st.volatile.shared.v2.b64 [%rd6+16], {%rd4, %rd5};
+; SM100-NEXT:    st.volatile.shared.v2.b64 [%rd6], {%rd2, %rd3};
+; SM100-NEXT:    ret;
   %a.load = load volatile <8 x float>, ptr addrspace(3) %a
   store volatile <8 x float> %a.load, ptr addrspace(3) %b
   ret void
@@ -1235,18 +1293,32 @@ define void @local_4xi64(ptr addrspace(5) %a, ptr addrspace(5) %b) {
 }
 
 define void @local_8xfloat(ptr addrspace(5) %a, ptr addrspace(5) %b) {
-; CHECK-LABEL: local_8xfloat(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<7>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [local_8xfloat_param_0];
-; CHECK-NEXT:    ld.local.v2.b64 {%rd2, %rd3}, [%rd1];
-; CHECK-NEXT:    ld.local.v2.b64 {%rd4, %rd5}, [%rd1+16];
-; CHECK-NEXT:    ld.param.b64 %rd6, [local_8xfloat_param_1];
-; CHECK-NEXT:    st.local.v2.b64 [%rd6+16], {%rd4, %rd5};
-; CHECK-NEXT:    st.local.v2.b64 [%rd6], {%rd2, %rd3};
-; CHECK-NEXT:    ret;
+; SM90-LABEL: local_8xfloat(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<9>;
+; SM90-NEXT:    .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b64 %rd1, [local_8xfloat_param_0];
+; SM90-NEXT:    ld.local.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    ld.local.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; SM90-NEXT:    ld.param.b64 %rd2, [local_8xfloat_param_1];
+; SM90-NEXT:    st.local.v4.b32 [%rd2+16], {%r5, %r6, %r7, %r8};
+; SM90-NEXT:    st.local.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
+; SM90-NEXT:    ret;
+;
+; SM100-LABEL: local_8xfloat(
+; SM100:       {
+; SM100-NEXT:    .reg .b64 %rd<7>;
+; SM100-EMPTY:
+; SM100-NEXT:  // %bb.0:
+; SM100-NEXT:    ld.param.b64 %rd1, [local_8xfloat_param_0];
+; SM100-NEXT:    ld.local.v2.b64 {%rd2, %rd3}, [%rd1];
+; SM100-NEXT:    ld.local.v2.b64 {%rd4, %rd5}, [%rd1+16];
+; SM100-NEXT:    ld.param.b64 %rd6, [local_8xfloat_param_1];
+; SM100-NEXT:    st.local.v2.b64 [%rd6+16], {%rd4, %rd5};
+; SM100-NEXT:    st.local.v2.b64 [%rd6], {%rd2, %rd3};
+; SM100-NEXT:    ret;
   %a.load = load <8 x float>, ptr addrspace(5) %a
   store <8 x float> %a.load, ptr addrspace(5) %b
   ret void
@@ -1386,18 +1458,32 @@ define void @local_volatile_4xi64(ptr addrspace(5) %a, ptr addrspace(5) %b) {
 }
 
 define void @local_volatile_8xfloat(ptr addrspace(5) %a, ptr addrspace(5) %b) {
-; CHECK-LABEL: local_volatile_8xfloat(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<7>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [local_volatile_8xfloat_param_0];
-; CHECK-NEXT:    ld.local.v2.b64 {%rd2, %rd3}, [%rd1];
-; CHECK-NEXT:    ld.local.v2.b64 {%rd4, %rd5}, [%rd1+16];
-; CHECK-NEXT:    ld.param.b64 %rd6, [local_volatile_8xfloat_param_1];
-; CHECK-NEXT:    st.local.v2.b64 [%rd6+16], {%rd4, %rd5};
-; CHECK-NEXT:    st.local.v2.b64 [%rd6], {%rd2, %rd3};
-; CHECK-NEXT:    ret;
+; SM90-LABEL: local_volatile_8xfloat(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<9>;
+; SM90-NEXT:    .reg .b64 %rd<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b64 %rd1, [local_volatile_8xfloat_param_0];
+; SM90-NEXT:    ld.local.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    ld.local.v4.b32 {%r5, %r6, %r7, %r8}, [%rd1+16];
+; SM90-NEXT:    ld.param.b64 %rd2, [local_volatile_8xfloat_param_1];
+; SM90-NEXT:    st.local.v4.b32 [%rd2+16], {%r5, %r6, %r7, %r8};
+; SM90-NEXT:    st.local.v4.b32 [%rd2], {%r1, %r2, %r3, %r4};
+; SM90-NEXT:    ret;
+;
+; SM100-LABEL: local_volatile_8xfloat(
+; SM100:       {
+; SM100-NEXT:    .reg .b64 %rd<7>;
+; SM100-EMPTY:
+; SM100-NEXT:  // %bb.0:
+; SM100-NEXT:    ld.param.b64 %rd1, [local_volatile_8xfloat_param_0];
+; SM100-NEXT:    ld.local.v2.b64 {%rd2, %rd3}, [%rd1];
+; SM100-NEXT:    ld.local.v2.b64 {%rd4, %rd5}, [%rd1+16];
+; SM100-NEXT:    ld.param.b64 %rd6, [local_volatile_8xfloat_param_1];
+; SM100-NEXT:    st.local.v2.b64 [%rd6+16], {%rd4, %rd5};
+; SM100-NEXT:    st.local.v2.b64 [%rd6], {%rd2, %rd3};
+; SM100-NEXT:    ret;
   %a.load = load volatile <8 x float>, ptr addrspace(5) %a
   store volatile <8 x float> %a.load, ptr addrspace(5) %b
   ret void
diff --git a/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll b/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
index ec8dd0c5c9350..7553c727b09c5 100644
--- a/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
+++ b/llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll
@@ -333,28 +333,30 @@ define ptx_kernel void @foo10(ptr noalias readonly %from, ptr %to) {
 define ptx_kernel void @foo11(ptr noalias readonly %from, ptr %to) {
 ; SM20-LABEL: foo11(
 ; SM20:       {
-; SM20-NEXT:    .reg .b64 %rd<6>;
+; SM20-NEXT:    .reg .b32 %r<3>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.b64 %rd1, [foo11_param_0];
 ; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
 ; SM20-NEXT:    ld.param.b64 %rd3, [foo11_param_1];
 ; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
-; SM20-NEXT:    ld.global.b64 %rd5, [%rd2];
-; SM20-NEXT:    st.global.b64 [%rd4], %rd5;
+; SM20-NEXT:    ld.global.v2.b32 {%r1, %r2}, [%rd2];
+; SM20-NEXT:    st.global.v2.b32 [%rd4], {%r1, %r2};
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: foo11(
 ; SM35:       {
-; SM35-NEXT:    .reg .b64 %rd<6>;
+; SM35-NEXT:    .reg .b32 %r<3>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
 ; SM35-EMPTY:
 ; SM35-NEXT:  // %bb.0:
 ; SM35-NEXT:    ld.param.b64 %rd1, [foo11_param_0];
 ; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
 ; SM35-NEXT:    ld.param.b64 %rd3, [foo11_param_1];
 ; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
-; SM35-NEXT:    ld.global.nc.b64 %rd5, [%rd2];
-; SM35-NEXT:    st.global.b64 [%rd4], %rd5;
+; SM35-NEXT:    ld.global.nc.v2.b32 {%r1, %r2}, [%rd2];
+; SM35-NEXT:    st.global.v2.b32 [%rd4], {%r1, %r2};
 ; SM35-NEXT:    ret;
   %1 = load <2 x float>, ptr %from
   store <2 x float> %1, ptr %to
@@ -494,28 +496,30 @@ define ptx_kernel void @foo15(ptr noalias readonly %from, ptr %to) {
 define ptx_kernel void @foo16(ptr noalias readonly %from, ptr %to) {
 ; SM20-LABEL: foo16(
 ; SM20:       {
-; SM20-NEXT:    .reg .b64 %rd<7>;
+; SM20-NEXT:    .reg .b32 %r<5>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0:
 ; SM20-NEXT:    ld.param.b64 %rd1, [foo16_param_0];
 ; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd1;
 ; SM20-NEXT:    ld.param.b64 %rd3, [foo16_param_1];
 ; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd3;
-; SM20-NEXT:    ld.global.v2.b64 {%rd5, %rd6}, [%rd2];
-; SM20-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM20-NEXT:    ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM20-NEXT:    st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: foo16(
 ; SM35:       {
-; SM35-NEXT:    .reg .b64 %rd<7>;
+; SM35-NEXT:    .reg .b32 %r<5>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
 ; SM35-EMPTY:
 ; SM35-NEXT:  // %bb.0:
 ; SM35-NEXT:    ld.param.b64 %rd1, [foo16_param_0];
 ; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd1;
 ; SM35-NEXT:    ld.param.b64 %rd3, [foo16_param_1];
 ; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd3;
-; SM35-NEXT:    ld.global.nc.v2.b64 {%rd5, %rd6}, [%rd2];
-; SM35-NEXT:    st.global.v2.b64 [%rd4], {%rd5, %rd6};
+; SM35-NEXT:    ld.global.nc.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
+; SM35-NEXT:    st.global.v4.b32 [%rd4], {%r1, %r2, %r3, %r4};
 ; SM35-NEXT:    ret;
   %1 = load <4 x float>, ptr %from
   store <4 x float> %1, ptr %to
@@ -593,51 +597,51 @@ define ptx_kernel void @foo19(ptr noalias readonly %from, ptr %to, i32 %n) {
 ; SM20-LABEL: foo19(
 ; SM20:       {
 ; SM20-NEXT:    .reg .pred %p<2>;
-; SM20-NEXT:    .reg .b32 %r<10>;
-; SM20-NEXT:    .reg .b64 %rd<8>;
+; SM20-NEXT:    .reg .b32 %r<4>;
+; SM20-NEXT:    .reg .b64 %rd<5>;
 ; SM20-EMPTY:
 ; SM20-NEXT:  // %bb.0: // %entry
-; SM20-NEXT:    ld.param.b32 %r8, [foo19_param_2];
-; SM20-NEXT:    ld.param.b64 %rd5, [foo19_param_0];
-; SM20-NEXT:    cvta.to.global.u64 %rd7, %rd5;
-; SM20-NEXT:    ld.param.b64 %rd6, [foo19_param_1];
-; SM20-NEXT:    cvta.to.global.u64 %rd2, %rd6;
-; SM20-NEXT:    mov.b32 %r9, 0f00000000;
+; SM20-NEXT:    ld.param.b32 %r2, [foo19_param_2];
+; SM20-NEXT:    ld.param.b64 %rd2, [foo19_param_0];
+; SM20-NEXT:    cvta.to.global.u64 %rd4, %rd2;
+; SM20-NEXT:    ld.param.b64 %rd3, [foo19_param_1];
+; SM20-NEXT:    cvta.to.global.u64 %rd1, %rd3;
+; SM20-NEXT:    mov.b32 %r3, 0f00000000;
 ; SM20-NEXT:  $L__BB18_1: // %loop
 ; SM20-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM20-NEXT:    ld.global.b32 %r7, [%rd7];
-; SM20-NEXT:    add.rn.f32 %r9, %r7, %r9;
-; SM20-NEXT:    add.s64 %rd7, %rd7, 4;
-; SM20-NEXT:    add.s32 %r8, %r8, -1;
-; SM20-NEXT:    setp.ne.b32 %p1, %r8, 0;
+; SM20-NEXT:    ld.global.b32 %r1, [%rd4];
+; SM20-NEXT:    add.rn.f32 %r3, %r1, %r3;
+; SM20-NEXT:    add.s64 %rd4, %rd4, 4;
+; SM20-NEXT:    add.s32 %r2, %r2, -1;
+; SM20-NEXT:    setp.ne.b32 %p1, %r2, 0;
 ; SM20-NEXT:    @%p1 bra $L__BB18_1;
 ; SM20-NEXT:  // %bb.2: // %exit
-; SM20-NEXT:    st.global.b32 [%rd2], %r9;
+; SM20-NEXT:    st.global.b32 [%rd1], %r3;
 ; SM20-NEXT:    ret;
 ;
 ; SM35-LABEL: foo19(
 ; SM35:       {
 ; SM35-NEXT:    .reg .pred %p<2>;
-; SM35-NEXT:    .reg .b32 %r<10>;
-; SM35-NEXT:    .reg .b64 %rd<8>;
+; SM35-NEXT:    .reg .b32 %r<4>;
+; SM35-NEXT:    .reg .b64 %rd<5>;
 ; SM35-EMPTY:
 ; SM35-NEXT:  // %bb.0: // %entry
-; SM35-NEXT:    ld.param.b32 %r8, [foo19_param_2];
-; SM35-NEXT:    ld.param.b64 %rd5, [foo19_param_0];
-; SM35-NEXT:    cvta.to.global.u64 %rd7, %rd5;
-; SM35-NEXT:    ld.param.b64 %rd6, [foo19_param_1];
-; SM35-NEXT:    cvta.to.global.u64 %rd2, %rd6;
-; SM35-NEXT:    mov.b32 %r9, 0f00000000;
+; SM35-NEXT:    ld.param.b32 %r2, [foo19_param_2];
+; SM35-NEXT:    ld.param.b64 %rd2, [foo19_param_0];
+; SM35-NEXT:    cvta.to.global.u64 %rd4, %rd2;
+; SM35-NEXT:    ld.param.b64 %rd3, [foo19_param_1];
+; SM35-NEXT:    cvta.to.global.u64 %rd1, %rd3;
+; SM35-NEXT:    mov.b32 %r3, 0f00000000;
 ; SM35-NEXT:  $L__BB18_1: // %loop
 ; SM35-NEXT:    // =>This Inner Loop Header: Depth=1
-; SM35-NEXT:    ld.global.nc.b32 %r7, [%rd7];
-; SM35-NEXT:    add.rn.f32 %r9, %r7, %r9;
-; SM35-NEXT:    add.s64 %rd7, %rd7, 4;
-; SM35-NEXT:    add.s32 %r8, %r8, -1;
-; SM35-NEXT:    setp.ne.b32 %p1, %r8, 0;
+; SM35-NEXT:    ld.global.nc.b32 %r1, [%rd4];
+; SM35-NEXT:    add.rn.f32 %r3, %r1, %r3;
+; SM35-NEXT:    add.s64 %rd4, %rd4, 4;
+; SM35-NEXT:    add.s32 %r2, %r2, -1;
+; SM35-NEXT:    setp.ne.b32 %p1, %r2, 0;
 ; SM35-NEXT:    @%p1 bra $L__BB18_1;
 ; SM35-NEXT:  // %bb.2: // %exit
-; SM35-NEXT:    st.global.b32 [%rd2], %r9;
+; SM35-NEXT:    st.global.b32 [%rd1], %r3;
 ; SM35-NEXT:    ret;
 entry:
   br label %loop
diff --git a/llvm/test/CodeGen/NVPTX/local-stack-frame.ll b/llvm/test/CodeGen/NVPTX/local-stack-frame.ll
index ae069cf956c36..f7137e05a5e4f 100644
--- a/llvm/test/CodeGen/NVPTX/local-stack-frame.ll
+++ b/llvm/test/CodeGen/NVPTX/local-stack-frame.ll
@@ -12,13 +12,13 @@ define void @foo(i32 %a) {
 ; PTX32-NEXT:    .local .align 4 .b8 __local_depot0[4];
 ; PTX32-NEXT:    .reg .b32 %SP;
 ; PTX32-NEXT:    .reg .b32 %SPL;
-; PTX32-NEXT:    .reg .b32 %r<4>;
+; PTX32-NEXT:    .reg .b32 %r<3>;
 ; PTX32-EMPTY:
 ; PTX32-NEXT:  // %bb.0:
 ; PTX32-NEXT:    mov.b32 %SPL, __local_depot0;
 ; PTX32-NEXT:    ld.param.b32 %r1, [foo_param_0];
-; PTX32-NEXT:    add.u32 %r3, %SPL, 0;
-; PTX32-NEXT:    st.local.b32 [%r3], %r1;
+; PTX32-NEXT:    add.u32 %r2, %SPL, 0;
+; PTX32-NEXT:    st.local.b32 [%r2], %r1;
 ; PTX32-NEXT:    ret;
 ;
 ; PTX64-LABEL: foo(
@@ -27,13 +27,13 @@ define void @foo(i32 %a) {
 ; PTX64-NEXT:    .reg .b64 %SP;
 ; PTX64-NEXT:    .reg .b64 %SPL;
 ; PTX64-NEXT:    .reg .b32 %r<2>;
-; PTX64-NEXT:    .reg .b64 %rd<3>;
+; PTX64-NEXT:    .reg .b64 %rd<2>;
 ; PTX64-EMPTY:
 ; PTX64-NEXT:  // %bb.0:
 ; PTX64-NEXT:    mov.b64 %SPL, __local_depot0;
 ; PTX64-NEXT:    ld.param.b32 %r1, [foo_param_0];
-; PTX64-NEXT:    add.u64 %rd2, %SPL, 0;
-; PTX64-NEXT:    st.local.b32 [%rd2], %r1;
+; PTX64-NEXT:    add.u64 %rd1, %SPL, 0;
+; PTX64-NEXT:    st.local.b32 [%rd1], %r1;
 ; PTX64-NEXT:    ret;
   %local = alloca i32, align 4
   store volatile i32 %a, ptr %local
@@ -97,15 +97,15 @@ define void @foo3(i32 %a) {
 ; PTX32-NEXT:    .local .align 4 .b8 __local_depot2[12];
 ; PTX32-NEXT:    .reg .b32 %SP;
 ; PTX32-NEXT:    .reg .b32 %SPL;
-; PTX32-NEXT:    .reg .b32 %r<6>;
+; PTX32-NEXT:    .reg .b32 %r<5>;
 ; PTX32-EMPTY:
 ; PTX32-NEXT:  // %bb.0:
 ; PTX32-NEXT:    mov.b32 %SPL, __local_depot2;
 ; PTX32-NEXT:    ld.param.b32 %r1, [foo3_param_0];
-; PTX32-NEXT:    add.u32 %r3, %SPL, 0;
-; PTX32-NEXT:    shl.b32 %r4, %r1, 2;
-; PTX32-NEXT:    add.s32 %r5, %r3, %r4;
-; PTX32-NEXT:    st.local.b32 [%r5], %r1;
+; PTX32-NEXT:    add.u32 %r2, %SPL, 0;
+; PTX32-NEXT:    shl.b32 %r3, %r1, 2;
+; PTX32-NEXT:    add.s32 %r4, %r2, %r3;
+; PTX32-NEXT:    st.local.b32 [%r4], %r1;
 ; PTX32-NEXT:    ret;
 ;
 ; PTX64-LABEL: foo3(
@@ -114,14 +114,14 @@ define void @foo3(i32 %a) {
 ; PTX64-NEXT:    .reg .b64 %SP;
 ; PTX64-NEXT:    .reg .b64 %SPL;
 ; PTX64-NEXT:    .reg .b32 %r<2>;
-; PTX64-NEXT:    .reg .b64 %rd<4>;
+; PTX64-NEXT:    .reg .b64 %rd<3>;
 ; PTX64-EMPTY:
 ; PTX64-NEXT:  // %bb.0:
 ; PTX64-NEXT:    mov.b64 %SPL, __local_depot2;
 ; PTX64-NEXT:    ld.param.b32 %r1, [foo3_param_0];
-; PTX64-NEXT:    add.u64 %rd2, %SPL, 0;
-; PTX64-NEXT:    mad.wide.s32 %rd3, %r1, 4, %rd2;
-; PTX64-NEXT:    st.local.b32 [%rd3], %r1;
+; PTX64-NEXT:    add.u64 %rd1, %SPL, 0;
+; PTX64-NEXT:    mad.wide.s32 %rd2, %r1, 4, %rd1;
+; PTX64-NEXT:    st.local.b32 [%rd2], %r1;
 ; PTX64-NEXT:    ret;
   %local = alloca [3 x i32], align 4
   %1 = getelementptr inbounds i32, ptr %local, i32 %a
diff --git a/llvm/test/CodeGen/NVPTX/lower-args-alignment.ll b/llvm/test/CodeGen/NVPTX/lower-args-alignment.ll
new file mode 100644
index 0000000000000..2051f6305cc03
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/lower-args-alignment.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=nvptx-lower-args,infer-alignment -S | FileCheck %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+; ------------------------------------------------------------------------------
+; Test that alignment can be inferred through llvm.nvvm.internal.addrspace.wrap.p101.p0 intrinsics
+; thanks to the alignment attribute on the intrinsic
+; ------------------------------------------------------------------------------
+
+%struct.S1 = type { i32, i32, i32, i32 }
+define ptx_kernel i32 @test_align8(ptr noundef readonly byval(%struct.S1) align 8 captures(none) %params) {
+; CHECK-LABEL: define ptx_kernel i32 @test_align8(
+; CHECK-SAME: ptr noundef readonly byval([[STRUCT_S1:%.*]]) align 8 captures(none) [[PARAMS:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call align 8 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[PARAMS]])
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(101) [[TMP0]], align 8
+; CHECK-NEXT:    ret i32 [[LOAD]]
+;
+entry:
+  %load = load i32, ptr %params, align 4
+  ret i32 %load
+}
+
+define ptx_kernel i32 @test_align1(ptr noundef readonly byval(%struct.S1) align 1 captures(none) %params) {
+; CHECK-LABEL: define ptx_kernel i32 @test_align1(
+; CHECK-SAME: ptr noundef readonly byval([[STRUCT_S1:%.*]]) align 4 captures(none) [[PARAMS:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call align 1 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[PARAMS]])
+; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr addrspace(101) [[TMP0]], align 4
+; CHECK-NEXT:    ret i32 [[LOAD]]
+;
+entry:
+  %load = load i32, ptr %params, align 4
+  ret i32 %load
+}
diff --git a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
index 045704bdcd3fc..8adde4ceefbf4 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args-gridconstant.ll
@@ -72,7 +72,7 @@ define ptx_kernel void @grid_const_int(ptr byval(i32) align 4 %input1, i32 %inpu
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_int(
 ; OPT-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], i32 [[INPUT2:%.*]], ptr [[OUT:%.*]], i32 [[N:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT11:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; OPT-NEXT:    [[INPUT11:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; OPT-NEXT:    [[TMP:%.*]] = load i32, ptr addrspace(101) [[INPUT11]], align 4
 ; OPT-NEXT:    [[ADD:%.*]] = add i32 [[TMP]], [[INPUT2]]
 ; OPT-NEXT:    store i32 [[ADD]], ptr [[OUT]], align 4
@@ -101,7 +101,7 @@ define ptx_kernel void @grid_const_struct(ptr byval(%struct.s) align 4 %input, p
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_struct(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[INPUT1:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
 ; OPT-NEXT:    [[GEP13:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr addrspace(101) [[INPUT1]], i32 0, i32 0
 ; OPT-NEXT:    [[GEP22:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr addrspace(101) [[INPUT1]], i32 0, i32 1
 ; OPT-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(101) [[GEP13]], align 4
@@ -137,7 +137,7 @@ define ptx_kernel void @grid_const_escape(ptr byval(%struct.s) align 4 %input) {
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_escape(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[TMP1:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
 ; OPT-NEXT:    [[INPUT_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    [[CALL:%.*]] = call i32 @escape(ptr [[INPUT_PARAM_GEN]])
 ; OPT-NEXT:    ret void
@@ -180,9 +180,9 @@ define ptx_kernel void @multiple_grid_const_escape(ptr byval(%struct.s) align 4
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @multiple_grid_const_escape(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]], i32 [[A:%.*]], ptr byval(i32) align 4 [[B:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[B]])
+; OPT-NEXT:    [[TMP1:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[B]])
 ; OPT-NEXT:    [[B_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
-; OPT-NEXT:    [[TMP2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[TMP2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
 ; OPT-NEXT:    [[INPUT_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP2]] to ptr
 ; OPT-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 ; OPT-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
@@ -208,7 +208,7 @@ define ptx_kernel void @grid_const_memory_escape(ptr byval(%struct.s) align 4 %i
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_memory_escape(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]], ptr [[ADDR:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[TMP1:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
 ; OPT-NEXT:    [[INPUT1:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    store ptr [[INPUT1]], ptr [[ADDR]], align 8
 ; OPT-NEXT:    ret void
@@ -235,7 +235,7 @@ define ptx_kernel void @grid_const_inlineasm_escape(ptr byval(%struct.s) align 4
 ; PTX-NOT      .local
 ; OPT-LABEL: define ptx_kernel void @grid_const_inlineasm_escape(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT:%.*]], ptr [[RESULT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[TMP1:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
 ; OPT-NEXT:    [[INPUT1:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    [[TMPPTR1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT1]], i32 0, i32 0
 ; OPT-NEXT:    [[TMPPTR2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[INPUT1]], i32 0, i32 1
@@ -340,24 +340,24 @@ define ptx_kernel void @grid_const_phi(ptr byval(%struct.s) align 4 %input1, ptr
 ; PTX:       {
 ; PTX-NEXT:    .reg .pred %p<2>;
 ; PTX-NEXT:    .reg .b32 %r<3>;
-; PTX-NEXT:    .reg .b64 %rd<7>;
+; PTX-NEXT:    .reg .b64 %rd<4>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
-; PTX-NEXT:    mov.b64 %rd6, grid_const_phi_param_0;
-; PTX-NEXT:    ld.param.b64 %rd5, [grid_const_phi_param_1];
-; PTX-NEXT:    cvta.to.global.u64 %rd1, %rd5;
+; PTX-NEXT:    mov.b64 %rd3, grid_const_phi_param_0;
+; PTX-NEXT:    ld.param.b64 %rd2, [grid_const_phi_param_1];
+; PTX-NEXT:    cvta.to.global.u64 %rd1, %rd2;
 ; PTX-NEXT:    ld.global.b32 %r1, [%rd1];
 ; PTX-NEXT:    setp.lt.s32 %p1, %r1, 0;
 ; PTX-NEXT:    @%p1 bra $L__BB9_2;
 ; PTX-NEXT:  // %bb.1: // %second
-; PTX-NEXT:    add.s64 %rd6, %rd6, 4;
+; PTX-NEXT:    add.s64 %rd3, %rd3, 4;
 ; PTX-NEXT:  $L__BB9_2: // %merge
-; PTX-NEXT:    ld.param.b32 %r2, [%rd6];
+; PTX-NEXT:    ld.param.b32 %r2, [%rd3];
 ; PTX-NEXT:    st.global.b32 [%rd1], %r2;
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_phi(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr [[INOUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; OPT-NEXT:    [[TMP1:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; OPT-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; OPT-NEXT:    [[VAL:%.*]] = load i32, ptr [[INOUT]], align 4
 ; OPT-NEXT:    [[LESS:%.*]] = icmp slt i32 [[VAL]], 0
@@ -396,27 +396,27 @@ define ptx_kernel void @grid_const_phi_ngc(ptr byval(%struct.s) align 4 %input1,
 ; PTX:       {
 ; PTX-NEXT:    .reg .pred %p<2>;
 ; PTX-NEXT:    .reg .b32 %r<3>;
-; PTX-NEXT:    .reg .b64 %rd<8>;
+; PTX-NEXT:    .reg .b64 %rd<5>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
-; PTX-NEXT:    mov.b64 %rd7, grid_const_phi_ngc_param_0;
-; PTX-NEXT:    ld.param.b64 %rd6, [grid_const_phi_ngc_param_2];
-; PTX-NEXT:    cvta.to.global.u64 %rd1, %rd6;
+; PTX-NEXT:    mov.b64 %rd4, grid_const_phi_ngc_param_0;
+; PTX-NEXT:    ld.param.b64 %rd3, [grid_const_phi_ngc_param_2];
+; PTX-NEXT:    cvta.to.global.u64 %rd1, %rd3;
 ; PTX-NEXT:    ld.global.b32 %r1, [%rd1];
 ; PTX-NEXT:    setp.lt.s32 %p1, %r1, 0;
 ; PTX-NEXT:    @%p1 bra $L__BB10_2;
 ; PTX-NEXT:  // %bb.1: // %second
 ; PTX-NEXT:    mov.b64 %rd2, grid_const_phi_ngc_param_1;
-; PTX-NEXT:    add.s64 %rd7, %rd2, 4;
+; PTX-NEXT:    add.s64 %rd4, %rd2, 4;
 ; PTX-NEXT:  $L__BB10_2: // %merge
-; PTX-NEXT:    ld.param.b32 %r2, [%rd7];
+; PTX-NEXT:    ld.param.b32 %r2, [%rd4];
 ; PTX-NEXT:    st.global.b32 [%rd1], %r2;
 ; PTX-NEXT:    ret;
 ; OPT-LABEL: define ptx_kernel void @grid_const_phi_ngc(
 ; OPT-SAME: ptr byval([[STRUCT_S:%.*]]) align 4 [[INPUT1:%.*]], ptr byval([[STRUCT_S]]) [[INPUT2:%.*]], ptr [[INOUT:%.*]]) #[[ATTR0]] {
 ; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; OPT-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
-; OPT-NEXT:    [[TMP2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; OPT-NEXT:    [[TMP2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; OPT-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP2]] to ptr
 ; OPT-NEXT:    [[VAL:%.*]] = load i32, ptr [[INOUT]], align 4
 ; OPT-NEXT:    [[LESS:%.*]] = icmp slt i32 [[VAL]], 0
@@ -471,7 +471,7 @@ define ptx_kernel void @grid_const_select(ptr byval(i32) align 4 %input1, ptr by
 ; OPT-SAME: ptr byval(i32) align 4 [[INPUT1:%.*]], ptr byval(i32) [[INPUT2:%.*]], ptr [[INOUT:%.*]]) #[[ATTR0]] {
 ; OPT-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; OPT-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
-; OPT-NEXT:    [[TMP2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; OPT-NEXT:    [[TMP2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; OPT-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP2]] to ptr
 ; OPT-NEXT:    [[VAL:%.*]] = load i32, ptr [[INOUT]], align 4
 ; OPT-NEXT:    [[LESS:%.*]] = icmp slt i32 [[VAL]], 0
@@ -520,7 +520,7 @@ declare void @device_func(ptr byval(i32) align 4)
 define ptx_kernel void @test_forward_byval_arg(ptr byval(i32) align 4 %input) {
 ; OPT-LABEL: define ptx_kernel void @test_forward_byval_arg(
 ; OPT-SAME: ptr byval(i32) align 4 [[INPUT:%.*]]) #[[ATTR0]] {
-; OPT-NEXT:    [[INPUT_PARAM:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; OPT-NEXT:    [[INPUT_PARAM:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
 ; OPT-NEXT:    [[INPUT_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[INPUT_PARAM]] to ptr
 ; OPT-NEXT:    call void @device_func(ptr byval(i32) align 4 [[INPUT_PARAM_GEN]])
 ; OPT-NEXT:    ret void
diff --git a/llvm/test/CodeGen/NVPTX/lower-args.ll b/llvm/test/CodeGen/NVPTX/lower-args.ll
index 7c029ab516d6e..b4a51035c6610 100644
--- a/llvm/test/CodeGen/NVPTX/lower-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-args.ll
@@ -200,7 +200,7 @@ define ptx_kernel void @ptr_as_int(i64 noundef %i, i32 noundef %v) {
 define ptx_kernel void @ptr_as_int_aggr(ptr nocapture noundef readonly byval(%struct.S) align 8 %s, i32 noundef %v) {
 ; IRC-LABEL: define ptx_kernel void @ptr_as_int_aggr(
 ; IRC-SAME: ptr noundef readonly byval([[STRUCT_S:%.*]]) align 8 captures(none) [[S:%.*]], i32 noundef [[V:%.*]]) {
-; IRC-NEXT:    [[S3:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; IRC-NEXT:    [[S3:%.*]] = call align 8 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; IRC-NEXT:    [[I:%.*]] = load i64, ptr addrspace(101) [[S3]], align 8
 ; IRC-NEXT:    [[P:%.*]] = inttoptr i64 [[I]] to ptr
 ; IRC-NEXT:    [[P1:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(1)
@@ -210,7 +210,7 @@ define ptx_kernel void @ptr_as_int_aggr(ptr nocapture noundef readonly byval(%st
 ;
 ; IRO-LABEL: define ptx_kernel void @ptr_as_int_aggr(
 ; IRO-SAME: ptr noundef readonly byval([[STRUCT_S:%.*]]) align 8 captures(none) [[S:%.*]], i32 noundef [[V:%.*]]) {
-; IRO-NEXT:    [[S1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; IRO-NEXT:    [[S1:%.*]] = call align 8 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; IRO-NEXT:    [[I:%.*]] = load i64, ptr addrspace(101) [[S1]], align 8
 ; IRO-NEXT:    [[P:%.*]] = inttoptr i64 [[I]] to ptr
 ; IRO-NEXT:    store i32 [[V]], ptr [[P]], align 4
diff --git a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
index 20a35198c3c16..21257e21bea9f 100644
--- a/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
+++ b/llvm/test/CodeGen/NVPTX/lower-byval-args.ll
@@ -32,7 +32,7 @@ define dso_local ptx_kernel void @read_only(ptr nocapture noundef writeonly %out
 ; LOWER-ARGS-LABEL: define dso_local ptx_kernel void @read_only(
 ; LOWER-ARGS-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; LOWER-ARGS-NEXT:  [[ENTRY:.*:]]
-; LOWER-ARGS-NEXT:    [[S3:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; LOWER-ARGS-NEXT:    [[S3:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; LOWER-ARGS-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[S3]], align 4
 ; LOWER-ARGS-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
 ; LOWER-ARGS-NEXT:    ret void
@@ -66,7 +66,7 @@ define dso_local ptx_kernel void @read_only_gep(ptr nocapture noundef writeonly
 ; LOWER-ARGS-LABEL: define dso_local ptx_kernel void @read_only_gep(
 ; LOWER-ARGS-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; LOWER-ARGS-NEXT:  [[ENTRY:.*:]]
-; LOWER-ARGS-NEXT:    [[S3:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; LOWER-ARGS-NEXT:    [[S3:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; LOWER-ARGS-NEXT:    [[B4:%.*]] = getelementptr inbounds i8, ptr addrspace(101) [[S3]], i64 4
 ; LOWER-ARGS-NEXT:    [[I:%.*]] = load i32, ptr addrspace(101) [[B4]], align 4
 ; LOWER-ARGS-NEXT:    store i32 [[I]], ptr [[OUT]], align 4
@@ -128,7 +128,7 @@ define dso_local ptx_kernel void @escape_ptr(ptr nocapture noundef readnone %out
 ; COMMON-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; COMMON-NEXT:    [[S2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[S1]]) #[[ATTR6:[0-9]+]]
 ; COMMON-NEXT:    ret void
@@ -167,7 +167,7 @@ define dso_local ptx_kernel void @escape_ptr_gep(ptr nocapture noundef readnone
 ; COMMON-SAME: ptr noundef readnone captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; COMMON-NEXT:    [[S2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
 ; COMMON-NEXT:    call void @_Z6escapePv(ptr noundef nonnull [[B]]) #[[ATTR6]]
@@ -209,7 +209,7 @@ define dso_local ptx_kernel void @escape_ptr_store(ptr nocapture noundef writeon
 ; COMMON-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; COMMON-NEXT:    [[S2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    store ptr [[S1]], ptr [[OUT]], align 8
 ; COMMON-NEXT:    ret void
@@ -246,7 +246,7 @@ define dso_local ptx_kernel void @escape_ptr_gep_store(ptr nocapture noundef wri
 ; COMMON-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; COMMON-NEXT:    [[S2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    [[B:%.*]] = getelementptr inbounds nuw i8, ptr [[S1]], i64 4
 ; COMMON-NEXT:    store ptr [[B]], ptr [[OUT]], align 8
@@ -286,7 +286,7 @@ define dso_local ptx_kernel void @escape_ptrtoint(ptr nocapture noundef writeonl
 ; COMMON-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; COMMON-NEXT:    [[S2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    [[I:%.*]] = ptrtoint ptr [[S1]] to i64
 ; COMMON-NEXT:    store i64 [[I]], ptr [[OUT]], align 8
@@ -324,7 +324,7 @@ define dso_local ptx_kernel void @memcpy_from_param(ptr nocapture noundef writeo
 ; LOWER-ARGS-LABEL: define dso_local ptx_kernel void @memcpy_from_param(
 ; LOWER-ARGS-SAME: ptr noundef writeonly captures(none) [[OUT:%.*]], ptr noundef readonly byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; LOWER-ARGS-NEXT:  [[ENTRY:.*:]]
-; LOWER-ARGS-NEXT:    [[S3:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; LOWER-ARGS-NEXT:    [[S3:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; LOWER-ARGS-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr [[OUT]], ptr addrspace(101) [[S3]], i64 16, i1 true)
 ; LOWER-ARGS-NEXT:    ret void
 ;
@@ -445,7 +445,7 @@ define dso_local ptx_kernel void @memcpy_to_param(ptr nocapture noundef readonly
 ; COMMON-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[ENTRY:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; COMMON-NEXT:    [[S2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    tail call void @llvm.memcpy.p0.p0.i64(ptr [[S1]], ptr [[IN]], i64 16, i1 true)
 ; COMMON-NEXT:    ret void
@@ -456,63 +456,63 @@ define dso_local ptx_kernel void @memcpy_to_param(ptr nocapture noundef readonly
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .b32 %r<3>;
-; PTX-NEXT:    .reg .b64 %rd<48>;
+; PTX-NEXT:    .reg .b64 %rd<47>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %entry
 ; PTX-NEXT:    mov.b64 %SPL, __local_depot9;
 ; PTX-NEXT:    cvta.local.u64 %SP, %SPL;
 ; PTX-NEXT:    ld.param.b64 %rd1, [memcpy_to_param_param_0];
-; PTX-NEXT:    add.u64 %rd3, %SPL, 0;
+; PTX-NEXT:    add.u64 %rd2, %SPL, 0;
 ; PTX-NEXT:    ld.param.b32 %r1, [memcpy_to_param_param_1+4];
-; PTX-NEXT:    st.local.b32 [%rd3+4], %r1;
+; PTX-NEXT:    st.local.b32 [%rd2+4], %r1;
 ; PTX-NEXT:    ld.param.b32 %r2, [memcpy_to_param_param_1];
-; PTX-NEXT:    st.local.b32 [%rd3], %r2;
-; PTX-NEXT:    ld.volatile.b8 %rd4, [%rd1];
-; PTX-NEXT:    ld.volatile.b8 %rd5, [%rd1+1];
-; PTX-NEXT:    shl.b64 %rd6, %rd5, 8;
-; PTX-NEXT:    or.b64 %rd7, %rd6, %rd4;
-; PTX-NEXT:    ld.volatile.b8 %rd8, [%rd1+2];
-; PTX-NEXT:    shl.b64 %rd9, %rd8, 16;
-; PTX-NEXT:    ld.volatile.b8 %rd10, [%rd1+3];
-; PTX-NEXT:    shl.b64 %rd11, %rd10, 24;
-; PTX-NEXT:    or.b64 %rd12, %rd11, %rd9;
-; PTX-NEXT:    or.b64 %rd13, %rd12, %rd7;
-; PTX-NEXT:    ld.volatile.b8 %rd14, [%rd1+4];
-; PTX-NEXT:    ld.volatile.b8 %rd15, [%rd1+5];
-; PTX-NEXT:    shl.b64 %rd16, %rd15, 8;
-; PTX-NEXT:    or.b64 %rd17, %rd16, %rd14;
-; PTX-NEXT:    ld.volatile.b8 %rd18, [%rd1+6];
-; PTX-NEXT:    shl.b64 %rd19, %rd18, 16;
-; PTX-NEXT:    ld.volatile.b8 %rd20, [%rd1+7];
-; PTX-NEXT:    shl.b64 %rd21, %rd20, 24;
-; PTX-NEXT:    or.b64 %rd22, %rd21, %rd19;
-; PTX-NEXT:    or.b64 %rd23, %rd22, %rd17;
-; PTX-NEXT:    shl.b64 %rd24, %rd23, 32;
-; PTX-NEXT:    or.b64 %rd25, %rd24, %rd13;
-; PTX-NEXT:    st.volatile.b64 [%SP], %rd25;
-; PTX-NEXT:    ld.volatile.b8 %rd26, [%rd1+8];
-; PTX-NEXT:    ld.volatile.b8 %rd27, [%rd1+9];
-; PTX-NEXT:    shl.b64 %rd28, %rd27, 8;
-; PTX-NEXT:    or.b64 %rd29, %rd28, %rd26;
-; PTX-NEXT:    ld.volatile.b8 %rd30, [%rd1+10];
-; PTX-NEXT:    shl.b64 %rd31, %rd30, 16;
-; PTX-NEXT:    ld.volatile.b8 %rd32, [%rd1+11];
-; PTX-NEXT:    shl.b64 %rd33, %rd32, 24;
-; PTX-NEXT:    or.b64 %rd34, %rd33, %rd31;
-; PTX-NEXT:    or.b64 %rd35, %rd34, %rd29;
-; PTX-NEXT:    ld.volatile.b8 %rd36, [%rd1+12];
-; PTX-NEXT:    ld.volatile.b8 %rd37, [%rd1+13];
-; PTX-NEXT:    shl.b64 %rd38, %rd37, 8;
-; PTX-NEXT:    or.b64 %rd39, %rd38, %rd36;
-; PTX-NEXT:    ld.volatile.b8 %rd40, [%rd1+14];
-; PTX-NEXT:    shl.b64 %rd41, %rd40, 16;
-; PTX-NEXT:    ld.volatile.b8 %rd42, [%rd1+15];
-; PTX-NEXT:    shl.b64 %rd43, %rd42, 24;
-; PTX-NEXT:    or.b64 %rd44, %rd43, %rd41;
-; PTX-NEXT:    or.b64 %rd45, %rd44, %rd39;
-; PTX-NEXT:    shl.b64 %rd46, %rd45, 32;
-; PTX-NEXT:    or.b64 %rd47, %rd46, %rd35;
-; PTX-NEXT:    st.volatile.b64 [%SP+8], %rd47;
+; PTX-NEXT:    st.local.b32 [%rd2], %r2;
+; PTX-NEXT:    ld.volatile.b8 %rd3, [%rd1];
+; PTX-NEXT:    ld.volatile.b8 %rd4, [%rd1+1];
+; PTX-NEXT:    shl.b64 %rd5, %rd4, 8;
+; PTX-NEXT:    or.b64 %rd6, %rd5, %rd3;
+; PTX-NEXT:    ld.volatile.b8 %rd7, [%rd1+2];
+; PTX-NEXT:    shl.b64 %rd8, %rd7, 16;
+; PTX-NEXT:    ld.volatile.b8 %rd9, [%rd1+3];
+; PTX-NEXT:    shl.b64 %rd10, %rd9, 24;
+; PTX-NEXT:    or.b64 %rd11, %rd10, %rd8;
+; PTX-NEXT:    or.b64 %rd12, %rd11, %rd6;
+; PTX-NEXT:    ld.volatile.b8 %rd13, [%rd1+4];
+; PTX-NEXT:    ld.volatile.b8 %rd14, [%rd1+5];
+; PTX-NEXT:    shl.b64 %rd15, %rd14, 8;
+; PTX-NEXT:    or.b64 %rd16, %rd15, %rd13;
+; PTX-NEXT:    ld.volatile.b8 %rd17, [%rd1+6];
+; PTX-NEXT:    shl.b64 %rd18, %rd17, 16;
+; PTX-NEXT:    ld.volatile.b8 %rd19, [%rd1+7];
+; PTX-NEXT:    shl.b64 %rd20, %rd19, 24;
+; PTX-NEXT:    or.b64 %rd21, %rd20, %rd18;
+; PTX-NEXT:    or.b64 %rd22, %rd21, %rd16;
+; PTX-NEXT:    shl.b64 %rd23, %rd22, 32;
+; PTX-NEXT:    or.b64 %rd24, %rd23, %rd12;
+; PTX-NEXT:    st.volatile.b64 [%SP], %rd24;
+; PTX-NEXT:    ld.volatile.b8 %rd25, [%rd1+8];
+; PTX-NEXT:    ld.volatile.b8 %rd26, [%rd1+9];
+; PTX-NEXT:    shl.b64 %rd27, %rd26, 8;
+; PTX-NEXT:    or.b64 %rd28, %rd27, %rd25;
+; PTX-NEXT:    ld.volatile.b8 %rd29, [%rd1+10];
+; PTX-NEXT:    shl.b64 %rd30, %rd29, 16;
+; PTX-NEXT:    ld.volatile.b8 %rd31, [%rd1+11];
+; PTX-NEXT:    shl.b64 %rd32, %rd31, 24;
+; PTX-NEXT:    or.b64 %rd33, %rd32, %rd30;
+; PTX-NEXT:    or.b64 %rd34, %rd33, %rd28;
+; PTX-NEXT:    ld.volatile.b8 %rd35, [%rd1+12];
+; PTX-NEXT:    ld.volatile.b8 %rd36, [%rd1+13];
+; PTX-NEXT:    shl.b64 %rd37, %rd36, 8;
+; PTX-NEXT:    or.b64 %rd38, %rd37, %rd35;
+; PTX-NEXT:    ld.volatile.b8 %rd39, [%rd1+14];
+; PTX-NEXT:    shl.b64 %rd40, %rd39, 16;
+; PTX-NEXT:    ld.volatile.b8 %rd41, [%rd1+15];
+; PTX-NEXT:    shl.b64 %rd42, %rd41, 24;
+; PTX-NEXT:    or.b64 %rd43, %rd42, %rd40;
+; PTX-NEXT:    or.b64 %rd44, %rd43, %rd38;
+; PTX-NEXT:    shl.b64 %rd45, %rd44, 32;
+; PTX-NEXT:    or.b64 %rd46, %rd45, %rd34;
+; PTX-NEXT:    st.volatile.b64 [%SP+8], %rd46;
 ; PTX-NEXT:    ret;
 entry:
   tail call void @llvm.memcpy.p0.p0.i64(ptr %s, ptr %in, i64 16, i1 true)
@@ -525,7 +525,7 @@ define dso_local ptx_kernel void @copy_on_store(ptr nocapture noundef readonly %
 ; COMMON-SAME: ptr noundef readonly captures(none) [[IN:%.*]], ptr noundef byval([[STRUCT_S:%.*]]) align 4 captures(none) [[S:%.*]], i1 noundef zeroext [[B:%.*]]) local_unnamed_addr #[[ATTR0]] {
 ; COMMON-NEXT:  [[BB:.*:]]
 ; COMMON-NEXT:    [[S1:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[S2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
+; COMMON-NEXT:    [[S2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[S]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[S1]], ptr addrspace(101) align 4 [[S2]], i64 8, i1 false)
 ; COMMON-NEXT:    [[I:%.*]] = load i32, ptr [[IN]], align 4
 ; COMMON-NEXT:    store i32 [[I]], ptr [[S1]], align 4
@@ -551,7 +551,7 @@ define ptx_kernel void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i3
 ; SM_60-NEXT:    [[INPUT25:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT24]], ptr addrspace(101) align 4 [[INPUT25]], i64 4, i1 false)
 ; SM_60-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
-; SM_60-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; SM_60-NEXT:    [[INPUT12:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
 ; SM_60-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT24]]
 ; SM_60-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
@@ -563,7 +563,7 @@ define ptx_kernel void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i3
 ; SM_70-NEXT:  [[BB:.*:]]
 ; SM_70-NEXT:    [[TMP0:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; SM_70-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP0]] to ptr
-; SM_70-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; SM_70-NEXT:    [[TMP1:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; SM_70-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; SM_70-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT1_PARAM_GEN]], ptr [[INPUT2_PARAM_GEN]]
 ; SM_70-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
@@ -577,7 +577,7 @@ define ptx_kernel void @test_select(ptr byval(i32) align 4 %input1, ptr byval(i3
 ; COPY-NEXT:    [[INPUT24:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT23]], ptr addrspace(101) align 4 [[INPUT24]], i64 4, i1 false)
 ; COPY-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
-; COPY-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; COPY-NEXT:    [[INPUT12:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
 ; COPY-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT23]]
 ; COPY-NEXT:    [[VALLOADED:%.*]] = load i32, ptr [[PTRNEW]], align 4
@@ -637,7 +637,7 @@ define ptx_kernel void @test_select_write(ptr byval(i32) align 4 %input1, ptr by
 ; COMMON-NEXT:    [[INPUT24:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT23]], ptr addrspace(101) align 4 [[INPUT24]], i64 4, i1 false)
 ; COMMON-NEXT:    [[INPUT11:%.*]] = alloca i32, align 4
-; COMMON-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; COMMON-NEXT:    [[INPUT12:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 4, i1 false)
 ; COMMON-NEXT:    [[PTRNEW:%.*]] = select i1 [[COND]], ptr [[INPUT11]], ptr [[INPUT23]]
 ; COMMON-NEXT:    store i32 1, ptr [[PTRNEW]], align 4
@@ -651,7 +651,7 @@ define ptx_kernel void @test_select_write(ptr byval(i32) align 4 %input1, ptr by
 ; PTX-NEXT:    .reg .pred %p<2>;
 ; PTX-NEXT:    .reg .b16 %rs<3>;
 ; PTX-NEXT:    .reg .b32 %r<3>;
-; PTX-NEXT:    .reg .b64 %rd<6>;
+; PTX-NEXT:    .reg .b64 %rd<4>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %bb
 ; PTX-NEXT:    mov.b64 %SPL, __local_depot12;
@@ -663,10 +663,10 @@ define ptx_kernel void @test_select_write(ptr byval(i32) align 4 %input1, ptr by
 ; PTX-NEXT:    st.b32 [%SP], %r1;
 ; PTX-NEXT:    ld.param.b32 %r2, [test_select_write_param_0];
 ; PTX-NEXT:    st.b32 [%SP+4], %r2;
-; PTX-NEXT:    add.u64 %rd2, %SPL, 4;
-; PTX-NEXT:    add.u64 %rd4, %SPL, 0;
-; PTX-NEXT:    selp.b64 %rd5, %rd2, %rd4, %p1;
-; PTX-NEXT:    st.local.b32 [%rd5], 1;
+; PTX-NEXT:    add.u64 %rd1, %SPL, 4;
+; PTX-NEXT:    add.u64 %rd2, %SPL, 0;
+; PTX-NEXT:    selp.b64 %rd3, %rd1, %rd2, %p1;
+; PTX-NEXT:    st.local.b32 [%rd3], 1;
 ; PTX-NEXT:    ret;
 bb:
   %ptrnew = select i1 %cond, ptr %input1, ptr %input2
@@ -682,7 +682,7 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; SM_60-NEXT:    [[INPUT25:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[INPUT24]], ptr addrspace(101) align 8 [[INPUT25]], i64 8, i1 false)
 ; SM_60-NEXT:    [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
-; SM_60-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; SM_60-NEXT:    [[INPUT12:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; SM_60-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 8, i1 false)
 ; SM_60-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; SM_60:       [[FIRST]]:
@@ -702,7 +702,7 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; SM_70-NEXT:  [[BB:.*:]]
 ; SM_70-NEXT:    [[TMP0:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; SM_70-NEXT:    [[INPUT2_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP0]] to ptr
-; SM_70-NEXT:    [[TMP1:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; SM_70-NEXT:    [[TMP1:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; SM_70-NEXT:    [[INPUT1_PARAM_GEN:%.*]] = addrspacecast ptr addrspace(101) [[TMP1]] to ptr
 ; SM_70-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; SM_70:       [[FIRST]]:
@@ -724,7 +724,7 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; COPY-NEXT:    [[INPUT24:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[INPUT23]], ptr addrspace(101) align 8 [[INPUT24]], i64 8, i1 false)
 ; COPY-NEXT:    [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
-; COPY-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; COPY-NEXT:    [[INPUT12:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; COPY-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 8, i1 false)
 ; COPY-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; COPY:       [[FIRST]]:
@@ -743,7 +743,7 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; PTX_60:       {
 ; PTX_60-NEXT:    .reg .pred %p<2>;
 ; PTX_60-NEXT:    .reg .b16 %rs<3>;
-; PTX_60-NEXT:    .reg .b32 %r<5>;
+; PTX_60-NEXT:    .reg .b32 %r<2>;
 ; PTX_60-NEXT:    .reg .b64 %rd<3>;
 ; PTX_60-EMPTY:
 ; PTX_60-NEXT:  // %bb.0: // %bb
@@ -752,12 +752,12 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; PTX_60-NEXT:    setp.ne.b16 %p1, %rs2, 0;
 ; PTX_60-NEXT:    ld.param.b64 %rd2, [test_phi_param_2];
 ; PTX_60-NEXT:    cvta.to.global.u64 %rd1, %rd2;
-; PTX_60-NEXT:    ld.param.b32 %r4, [test_phi_param_0];
+; PTX_60-NEXT:    ld.param.b32 %r1, [test_phi_param_0];
 ; PTX_60-NEXT:    @%p1 bra $L__BB13_2;
 ; PTX_60-NEXT:  // %bb.1: // %second
-; PTX_60-NEXT:    ld.param.b32 %r4, [test_phi_param_1+4];
+; PTX_60-NEXT:    ld.param.b32 %r1, [test_phi_param_1+4];
 ; PTX_60-NEXT:  $L__BB13_2: // %merge
-; PTX_60-NEXT:    st.global.b32 [%rd1], %r4;
+; PTX_60-NEXT:    st.global.b32 [%rd1], %r1;
 ; PTX_60-NEXT:    ret;
 ;
 ; PTX_70-LABEL: test_phi(
@@ -765,21 +765,21 @@ define ptx_kernel void @test_phi(ptr byval(%struct.S) align 4 %input1, ptr byval
 ; PTX_70-NEXT:    .reg .pred %p<2>;
 ; PTX_70-NEXT:    .reg .b16 %rs<3>;
 ; PTX_70-NEXT:    .reg .b32 %r<2>;
-; PTX_70-NEXT:    .reg .b64 %rd<8>;
+; PTX_70-NEXT:    .reg .b64 %rd<5>;
 ; PTX_70-EMPTY:
 ; PTX_70-NEXT:  // %bb.0: // %bb
 ; PTX_70-NEXT:    ld.param.b8 %rs1, [test_phi_param_3];
 ; PTX_70-NEXT:    and.b16 %rs2, %rs1, 1;
 ; PTX_70-NEXT:    setp.ne.b16 %p1, %rs2, 0;
-; PTX_70-NEXT:    mov.b64 %rd7, test_phi_param_0;
-; PTX_70-NEXT:    ld.param.b64 %rd6, [test_phi_param_2];
-; PTX_70-NEXT:    cvta.to.global.u64 %rd1, %rd6;
+; PTX_70-NEXT:    mov.b64 %rd4, test_phi_param_0;
+; PTX_70-NEXT:    ld.param.b64 %rd3, [test_phi_param_2];
+; PTX_70-NEXT:    cvta.to.global.u64 %rd1, %rd3;
 ; PTX_70-NEXT:    @%p1 bra $L__BB13_2;
 ; PTX_70-NEXT:  // %bb.1: // %second
 ; PTX_70-NEXT:    mov.b64 %rd2, test_phi_param_1;
-; PTX_70-NEXT:    add.s64 %rd7, %rd2, 4;
+; PTX_70-NEXT:    add.s64 %rd4, %rd2, 4;
 ; PTX_70-NEXT:  $L__BB13_2: // %merge
-; PTX_70-NEXT:    ld.param.b32 %r1, [%rd7];
+; PTX_70-NEXT:    ld.param.b32 %r1, [%rd4];
 ; PTX_70-NEXT:    st.global.b32 [%rd1], %r1;
 ; PTX_70-NEXT:    ret;
 bb:
@@ -808,7 +808,7 @@ define ptx_kernel void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr
 ; COMMON-NEXT:    [[INPUT25:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT2]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 8 [[INPUT24]], ptr addrspace(101) align 8 [[INPUT25]], i64 8, i1 false)
 ; COMMON-NEXT:    [[INPUT11:%.*]] = alloca [[STRUCT_S]], align 4
-; COMMON-NEXT:    [[INPUT12:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
+; COMMON-NEXT:    [[INPUT12:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT1]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT11]], ptr addrspace(101) align 4 [[INPUT12]], i64 8, i1 false)
 ; COMMON-NEXT:    br i1 [[COND]], label %[[FIRST:.*]], label %[[SECOND:.*]]
 ; COMMON:       [[FIRST]]:
@@ -830,7 +830,7 @@ define ptx_kernel void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr
 ; PTX-NEXT:    .reg .pred %p<2>;
 ; PTX-NEXT:    .reg .b16 %rs<3>;
 ; PTX-NEXT:    .reg .b32 %r<3>;
-; PTX-NEXT:    .reg .b64 %rd<7>;
+; PTX-NEXT:    .reg .b64 %rd<3>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0: // %bb
 ; PTX-NEXT:    mov.b64 %SPL, __local_depot14;
@@ -841,14 +841,14 @@ define ptx_kernel void @test_phi_write(ptr byval(%struct.S) align 4 %input1, ptr
 ; PTX-NEXT:    add.u64 %rd1, %SPL, 0;
 ; PTX-NEXT:    ld.param.b32 %r1, [test_phi_write_param_1+4];
 ; PTX-NEXT:    st.b32 [%SP], %r1;
-; PTX-NEXT:    add.u64 %rd6, %SPL, 4;
+; PTX-NEXT:    add.u64 %rd2, %SPL, 4;
 ; PTX-NEXT:    ld.param.b32 %r2, [test_phi_write_param_0];
 ; PTX-NEXT:    st.b32 [%SP+4], %r2;
 ; PTX-NEXT:    @%p1 bra $L__BB14_2;
 ; PTX-NEXT:  // %bb.1: // %second
-; PTX-NEXT:    mov.b64 %rd6, %rd1;
+; PTX-NEXT:    mov.b64 %rd2, %rd1;
 ; PTX-NEXT:  $L__BB14_2: // %merge
-; PTX-NEXT:    st.local.b32 [%rd6], 1;
+; PTX-NEXT:    st.local.b32 [%rd2], 1;
 ; PTX-NEXT:    ret;
 bb:
   br i1 %cond, label %first, label %second
@@ -871,7 +871,7 @@ define ptx_kernel void @test_forward_byval_arg(ptr byval(i32) align 4 %input) {
 ; COMMON-LABEL: define ptx_kernel void @test_forward_byval_arg(
 ; COMMON-SAME: ptr byval(i32) align 4 [[INPUT:%.*]]) #[[ATTR3]] {
 ; COMMON-NEXT:    [[INPUT1:%.*]] = alloca i32, align 4
-; COMMON-NEXT:    [[INPUT2:%.*]] = call ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
+; COMMON-NEXT:    [[INPUT2:%.*]] = call align 4 ptr addrspace(101) @llvm.nvvm.internal.addrspace.wrap.p101.p0(ptr [[INPUT]])
 ; COMMON-NEXT:    call void @llvm.memcpy.p0.p101.i64(ptr align 4 [[INPUT1]], ptr addrspace(101) align 4 [[INPUT2]], i64 4, i1 false)
 ; COMMON-NEXT:    call void @device_func(ptr byval(i32) align 4 [[INPUT1]])
 ; COMMON-NEXT:    ret void
@@ -882,13 +882,13 @@ define ptx_kernel void @test_forward_byval_arg(ptr byval(i32) align 4 %input) {
 ; PTX-NEXT:    .reg .b64 %SP;
 ; PTX-NEXT:    .reg .b64 %SPL;
 ; PTX-NEXT:    .reg .b32 %r<2>;
-; PTX-NEXT:    .reg .b64 %rd<3>;
+; PTX-NEXT:    .reg .b64 %rd<2>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    mov.b64 %SPL, __local_depot15;
-; PTX-NEXT:    add.u64 %rd2, %SPL, 0;
+; PTX-NEXT:    add.u64 %rd1, %SPL, 0;
 ; PTX-NEXT:    ld.param.b32 %r1, [test_forward_byval_arg_param_0];
-; PTX-NEXT:    st.local.b32 [%rd2], %r1;
+; PTX-NEXT:    st.local.b32 [%rd1], %r1;
 ; PTX-NEXT:    { // callseq 2, 0
 ; PTX-NEXT:    .param .align 4 .b8 param0[4];
 ; PTX-NEXT:    st.param.b32 [param0], %r1;
@@ -908,7 +908,6 @@ define void @device_func(ptr byval(i32) align 4 %input) {
 ; PTX-LABEL: device_func(
 ; PTX:       {
 ; PTX-NEXT:    .reg .b32 %r<2>;
-; PTX-NEXT:    .reg .b64 %rd<2>;
 ; PTX-EMPTY:
 ; PTX-NEXT:  // %bb.0:
 ; PTX-NEXT:    { // callseq 3, 0
diff --git a/llvm/test/CodeGen/NVPTX/misaligned-vector-ldst.ll b/llvm/test/CodeGen/NVPTX/misaligned-vector-ldst.ll
index dfdb33852305b..0039370e6dcf5 100644
--- a/llvm/test/CodeGen/NVPTX/misaligned-vector-ldst.ll
+++ b/llvm/test/CodeGen/NVPTX/misaligned-vector-ldst.ll
@@ -8,55 +8,52 @@ target triple = "nvptx64-nvidia-cuda"
 define <4 x float> @t1(ptr %p1) {
 ; CHECK-LABEL: t1(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<46>;
+; CHECK-NEXT:    .reg .b32 %r<41>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [t1_param_0];
-; CHECK-NEXT:    ld.b8 %rd2, [%rd1+8];
-; CHECK-NEXT:    ld.b8 %rd3, [%rd1+9];
-; CHECK-NEXT:    shl.b64 %rd4, %rd3, 8;
-; CHECK-NEXT:    or.b64 %rd5, %rd4, %rd2;
-; CHECK-NEXT:    ld.b8 %rd6, [%rd1+10];
-; CHECK-NEXT:    shl.b64 %rd7, %rd6, 16;
-; CHECK-NEXT:    ld.b8 %rd8, [%rd1+11];
-; CHECK-NEXT:    shl.b64 %rd9, %rd8, 24;
-; CHECK-NEXT:    or.b64 %rd10, %rd9, %rd7;
-; CHECK-NEXT:    or.b64 %rd11, %rd10, %rd5;
-; CHECK-NEXT:    ld.b8 %rd12, [%rd1+12];
-; CHECK-NEXT:    ld.b8 %rd13, [%rd1+13];
-; CHECK-NEXT:    shl.b64 %rd14, %rd13, 8;
-; CHECK-NEXT:    or.b64 %rd15, %rd14, %rd12;
-; CHECK-NEXT:    ld.b8 %rd16, [%rd1+14];
-; CHECK-NEXT:    shl.b64 %rd17, %rd16, 16;
-; CHECK-NEXT:    ld.b8 %rd18, [%rd1+15];
-; CHECK-NEXT:    shl.b64 %rd19, %rd18, 24;
-; CHECK-NEXT:    or.b64 %rd20, %rd19, %rd17;
-; CHECK-NEXT:    or.b64 %rd21, %rd20, %rd15;
-; CHECK-NEXT:    shl.b64 %rd22, %rd21, 32;
-; CHECK-NEXT:    or.b64 %rd23, %rd22, %rd11;
-; CHECK-NEXT:    ld.b8 %rd24, [%rd1];
-; CHECK-NEXT:    ld.b8 %rd25, [%rd1+1];
-; CHECK-NEXT:    shl.b64 %rd26, %rd25, 8;
-; CHECK-NEXT:    or.b64 %rd27, %rd26, %rd24;
-; CHECK-NEXT:    ld.b8 %rd28, [%rd1+2];
-; CHECK-NEXT:    shl.b64 %rd29, %rd28, 16;
-; CHECK-NEXT:    ld.b8 %rd30, [%rd1+3];
-; CHECK-NEXT:    shl.b64 %rd31, %rd30, 24;
-; CHECK-NEXT:    or.b64 %rd32, %rd31, %rd29;
-; CHECK-NEXT:    or.b64 %rd33, %rd32, %rd27;
-; CHECK-NEXT:    ld.b8 %rd34, [%rd1+4];
-; CHECK-NEXT:    ld.b8 %rd35, [%rd1+5];
-; CHECK-NEXT:    shl.b64 %rd36, %rd35, 8;
-; CHECK-NEXT:    or.b64 %rd37, %rd36, %rd34;
-; CHECK-NEXT:    ld.b8 %rd38, [%rd1+6];
-; CHECK-NEXT:    shl.b64 %rd39, %rd38, 16;
-; CHECK-NEXT:    ld.b8 %rd40, [%rd1+7];
-; CHECK-NEXT:    shl.b64 %rd41, %rd40, 24;
-; CHECK-NEXT:    or.b64 %rd42, %rd41, %rd39;
-; CHECK-NEXT:    or.b64 %rd43, %rd42, %rd37;
-; CHECK-NEXT:    shl.b64 %rd44, %rd43, 32;
-; CHECK-NEXT:    or.b64 %rd45, %rd44, %rd33;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd45, %rd23};
+; CHECK-NEXT:    ld.b8 %r1, [%rd1+12];
+; CHECK-NEXT:    ld.b8 %r2, [%rd1+13];
+; CHECK-NEXT:    shl.b32 %r3, %r2, 8;
+; CHECK-NEXT:    or.b32 %r4, %r3, %r1;
+; CHECK-NEXT:    ld.b8 %r5, [%rd1+14];
+; CHECK-NEXT:    shl.b32 %r6, %r5, 16;
+; CHECK-NEXT:    ld.b8 %r7, [%rd1+15];
+; CHECK-NEXT:    shl.b32 %r8, %r7, 24;
+; CHECK-NEXT:    or.b32 %r9, %r8, %r6;
+; CHECK-NEXT:    or.b32 %r10, %r9, %r4;
+; CHECK-NEXT:    ld.b8 %r11, [%rd1+8];
+; CHECK-NEXT:    ld.b8 %r12, [%rd1+9];
+; CHECK-NEXT:    shl.b32 %r13, %r12, 8;
+; CHECK-NEXT:    or.b32 %r14, %r13, %r11;
+; CHECK-NEXT:    ld.b8 %r15, [%rd1+10];
+; CHECK-NEXT:    shl.b32 %r16, %r15, 16;
+; CHECK-NEXT:    ld.b8 %r17, [%rd1+11];
+; CHECK-NEXT:    shl.b32 %r18, %r17, 24;
+; CHECK-NEXT:    or.b32 %r19, %r18, %r16;
+; CHECK-NEXT:    or.b32 %r20, %r19, %r14;
+; CHECK-NEXT:    ld.b8 %r21, [%rd1+4];
+; CHECK-NEXT:    ld.b8 %r22, [%rd1+5];
+; CHECK-NEXT:    shl.b32 %r23, %r22, 8;
+; CHECK-NEXT:    or.b32 %r24, %r23, %r21;
+; CHECK-NEXT:    ld.b8 %r25, [%rd1+6];
+; CHECK-NEXT:    shl.b32 %r26, %r25, 16;
+; CHECK-NEXT:    ld.b8 %r27, [%rd1+7];
+; CHECK-NEXT:    shl.b32 %r28, %r27, 24;
+; CHECK-NEXT:    or.b32 %r29, %r28, %r26;
+; CHECK-NEXT:    or.b32 %r30, %r29, %r24;
+; CHECK-NEXT:    ld.b8 %r31, [%rd1];
+; CHECK-NEXT:    ld.b8 %r32, [%rd1+1];
+; CHECK-NEXT:    shl.b32 %r33, %r32, 8;
+; CHECK-NEXT:    or.b32 %r34, %r33, %r31;
+; CHECK-NEXT:    ld.b8 %r35, [%rd1+2];
+; CHECK-NEXT:    shl.b32 %r36, %r35, 16;
+; CHECK-NEXT:    ld.b8 %r37, [%rd1+3];
+; CHECK-NEXT:    shl.b32 %r38, %r37, 24;
+; CHECK-NEXT:    or.b32 %r39, %r38, %r36;
+; CHECK-NEXT:    or.b32 %r40, %r39, %r34;
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r40, %r30, %r20, %r10};
 ; CHECK-NEXT:    ret;
   %r = load <4 x float>, ptr %p1, align 1
   ret <4 x float> %r
@@ -65,19 +62,16 @@ define <4 x float> @t1(ptr %p1) {
 define <4 x float> @t2(ptr %p1) {
 ; CHECK-LABEL: t2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<10>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [t2_param_0];
-; CHECK-NEXT:    ld.b32 %rd2, [%rd1+8];
-; CHECK-NEXT:    ld.b32 %rd3, [%rd1+12];
-; CHECK-NEXT:    shl.b64 %rd4, %rd3, 32;
-; CHECK-NEXT:    or.b64 %rd5, %rd4, %rd2;
-; CHECK-NEXT:    ld.b32 %rd6, [%rd1];
-; CHECK-NEXT:    ld.b32 %rd7, [%rd1+4];
-; CHECK-NEXT:    shl.b64 %rd8, %rd7, 32;
-; CHECK-NEXT:    or.b64 %rd9, %rd8, %rd6;
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd9, %rd5};
+; CHECK-NEXT:    ld.b32 %r1, [%rd1+12];
+; CHECK-NEXT:    ld.b32 %r2, [%rd1+8];
+; CHECK-NEXT:    ld.b32 %r3, [%rd1+4];
+; CHECK-NEXT:    ld.b32 %r4, [%rd1];
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r4, %r3, %r2, %r1};
 ; CHECK-NEXT:    ret;
   %r = load <4 x float>, ptr %p1, align 4
   ret <4 x float> %r
@@ -86,13 +80,14 @@ define <4 x float> @t2(ptr %p1) {
 define <4 x float> @t3(ptr %p1) {
 ; CHECK-LABEL: t3(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [t3_param_0];
-; CHECK-NEXT:    ld.b64 %rd2, [%rd1+8];
-; CHECK-NEXT:    ld.b64 %rd3, [%rd1];
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd3, %rd2};
+; CHECK-NEXT:    ld.v2.b32 {%r1, %r2}, [%rd1+8];
+; CHECK-NEXT:    ld.v2.b32 {%r3, %r4}, [%rd1];
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r3, %r4, %r1, %r2};
 ; CHECK-NEXT:    ret;
   %r = load <4 x float>, ptr %p1, align 8
   ret <4 x float> %r
@@ -101,12 +96,13 @@ define <4 x float> @t3(ptr %p1) {
 define <4 x float> @t4(ptr %p1) {
 ; CHECK-LABEL: t4(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [t4_param_0];
-; CHECK-NEXT:    ld.v2.b64 {%rd2, %rd3}, [%rd1];
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd2, %rd3};
+; CHECK-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r1, %r2, %r3, %r4};
 ; CHECK-NEXT:    ret;
   %r = load <4 x float>, ptr %p1, align 16
   ret <4 x float> %r
@@ -189,40 +185,43 @@ define void @test_v4halfp0a1(ptr noalias readonly %from, ptr %to) {
 define void @s1(ptr %p1, <4 x float> %v) {
 ; CHECK-LABEL: s1(
 ; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-NEXT:    .reg .b64 %rd<18>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [s1_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [s1_param_1];
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [s1_param_1];
+; CHECK-NEXT:    cvt.u64.u32 %rd2, %r4;
+; CHECK-NEXT:    st.b8 [%rd1+12], %rd2;
+; CHECK-NEXT:    cvt.u64.u32 %rd3, %r3;
 ; CHECK-NEXT:    st.b8 [%rd1+8], %rd3;
-; CHECK-NEXT:    st.b8 [%rd1], %rd2;
-; CHECK-NEXT:    shr.u64 %rd4, %rd3, 56;
-; CHECK-NEXT:    st.b8 [%rd1+15], %rd4;
-; CHECK-NEXT:    shr.u64 %rd5, %rd3, 48;
-; CHECK-NEXT:    st.b8 [%rd1+14], %rd5;
-; CHECK-NEXT:    shr.u64 %rd6, %rd3, 40;
-; CHECK-NEXT:    st.b8 [%rd1+13], %rd6;
-; CHECK-NEXT:    shr.u64 %rd7, %rd3, 32;
-; CHECK-NEXT:    st.b8 [%rd1+12], %rd7;
-; CHECK-NEXT:    shr.u64 %rd8, %rd3, 24;
-; CHECK-NEXT:    st.b8 [%rd1+11], %rd8;
-; CHECK-NEXT:    shr.u64 %rd9, %rd3, 16;
-; CHECK-NEXT:    st.b8 [%rd1+10], %rd9;
-; CHECK-NEXT:    shr.u64 %rd10, %rd3, 8;
-; CHECK-NEXT:    st.b8 [%rd1+9], %rd10;
-; CHECK-NEXT:    shr.u64 %rd11, %rd2, 56;
-; CHECK-NEXT:    st.b8 [%rd1+7], %rd11;
-; CHECK-NEXT:    shr.u64 %rd12, %rd2, 48;
-; CHECK-NEXT:    st.b8 [%rd1+6], %rd12;
-; CHECK-NEXT:    shr.u64 %rd13, %rd2, 40;
-; CHECK-NEXT:    st.b8 [%rd1+5], %rd13;
-; CHECK-NEXT:    shr.u64 %rd14, %rd2, 32;
-; CHECK-NEXT:    st.b8 [%rd1+4], %rd14;
-; CHECK-NEXT:    shr.u64 %rd15, %rd2, 24;
+; CHECK-NEXT:    cvt.u64.u32 %rd4, %r2;
+; CHECK-NEXT:    st.b8 [%rd1+4], %rd4;
+; CHECK-NEXT:    cvt.u64.u32 %rd5, %r1;
+; CHECK-NEXT:    st.b8 [%rd1], %rd5;
+; CHECK-NEXT:    shr.u64 %rd6, %rd2, 24;
+; CHECK-NEXT:    st.b8 [%rd1+15], %rd6;
+; CHECK-NEXT:    shr.u64 %rd7, %rd2, 16;
+; CHECK-NEXT:    st.b8 [%rd1+14], %rd7;
+; CHECK-NEXT:    shr.u64 %rd8, %rd2, 8;
+; CHECK-NEXT:    st.b8 [%rd1+13], %rd8;
+; CHECK-NEXT:    shr.u64 %rd9, %rd3, 24;
+; CHECK-NEXT:    st.b8 [%rd1+11], %rd9;
+; CHECK-NEXT:    shr.u64 %rd10, %rd3, 16;
+; CHECK-NEXT:    st.b8 [%rd1+10], %rd10;
+; CHECK-NEXT:    shr.u64 %rd11, %rd3, 8;
+; CHECK-NEXT:    st.b8 [%rd1+9], %rd11;
+; CHECK-NEXT:    shr.u64 %rd12, %rd4, 24;
+; CHECK-NEXT:    st.b8 [%rd1+7], %rd12;
+; CHECK-NEXT:    shr.u64 %rd13, %rd4, 16;
+; CHECK-NEXT:    st.b8 [%rd1+6], %rd13;
+; CHECK-NEXT:    shr.u64 %rd14, %rd4, 8;
+; CHECK-NEXT:    st.b8 [%rd1+5], %rd14;
+; CHECK-NEXT:    shr.u64 %rd15, %rd5, 24;
 ; CHECK-NEXT:    st.b8 [%rd1+3], %rd15;
-; CHECK-NEXT:    shr.u64 %rd16, %rd2, 16;
+; CHECK-NEXT:    shr.u64 %rd16, %rd5, 16;
 ; CHECK-NEXT:    st.b8 [%rd1+2], %rd16;
-; CHECK-NEXT:    shr.u64 %rd17, %rd2, 8;
+; CHECK-NEXT:    shr.u64 %rd17, %rd5, 8;
 ; CHECK-NEXT:    st.b8 [%rd1+1], %rd17;
 ; CHECK-NEXT:    ret;
   store <4 x float> %v, ptr %p1, align 1
@@ -232,17 +231,16 @@ define void @s1(ptr %p1, <4 x float> %v) {
 define void @s2(ptr %p1, <4 x float> %v) {
 ; CHECK-LABEL: s2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [s2_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [s2_param_1];
-; CHECK-NEXT:    st.b32 [%rd1+8], %rd3;
-; CHECK-NEXT:    st.b32 [%rd1], %rd2;
-; CHECK-NEXT:    shr.u64 %rd4, %rd3, 32;
-; CHECK-NEXT:    st.b32 [%rd1+12], %rd4;
-; CHECK-NEXT:    shr.u64 %rd5, %rd2, 32;
-; CHECK-NEXT:    st.b32 [%rd1+4], %rd5;
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [s2_param_1];
+; CHECK-NEXT:    st.b32 [%rd1+12], %r4;
+; CHECK-NEXT:    st.b32 [%rd1+8], %r3;
+; CHECK-NEXT:    st.b32 [%rd1+4], %r2;
+; CHECK-NEXT:    st.b32 [%rd1], %r1;
 ; CHECK-NEXT:    ret;
   store <4 x float> %v, ptr %p1, align 4
   ret void
@@ -251,13 +249,14 @@ define void @s2(ptr %p1, <4 x float> %v) {
 define void @s3(ptr %p1, <4 x float> %v) {
 ; CHECK-LABEL: s3(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [s3_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [s3_param_1];
-; CHECK-NEXT:    st.b64 [%rd1+8], %rd3;
-; CHECK-NEXT:    st.b64 [%rd1], %rd2;
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [s3_param_1];
+; CHECK-NEXT:    st.v2.b32 [%rd1+8], {%r3, %r4};
+; CHECK-NEXT:    st.v2.b32 [%rd1], {%r1, %r2};
 ; CHECK-NEXT:    ret;
   store <4 x float> %v, ptr %p1, align 8
   ret void
@@ -266,12 +265,13 @@ define void @s3(ptr %p1, <4 x float> %v) {
 define void @s4(ptr %p1, <4 x float> %v) {
 ; CHECK-LABEL: s4(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [s4_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd2, %rd3}, [s4_param_1];
-; CHECK-NEXT:    st.v2.b64 [%rd1], {%rd2, %rd3};
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [s4_param_1];
+; CHECK-NEXT:    st.v4.b32 [%rd1], {%r1, %r2, %r3, %r4};
 ; CHECK-NEXT:    ret;
   store <4 x float> %v, ptr %p1, align 16
   ret void
diff --git a/llvm/test/CodeGen/NVPTX/misched_func_call.ll b/llvm/test/CodeGen/NVPTX/misched_func_call.ll
index b2994c0a97585..62f99e991ea1e 100644
--- a/llvm/test/CodeGen/NVPTX/misched_func_call.ll
+++ b/llvm/test/CodeGen/NVPTX/misched_func_call.ll
@@ -7,14 +7,14 @@ target triple = "nvptx64-nvidia-cuda"
 define ptx_kernel void @wombat(i32 %arg, i32 %arg1, i32 %arg2) {
 ; CHECK-LABEL: wombat(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<11>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %bb
-; CHECK-NEXT:    ld.param.b32 %r4, [wombat_param_2];
-; CHECK-NEXT:    ld.param.b32 %r3, [wombat_param_1];
-; CHECK-NEXT:    ld.param.b32 %r2, [wombat_param_0];
-; CHECK-NEXT:    mov.b32 %r10, 0;
+; CHECK-NEXT:    ld.param.b32 %r3, [wombat_param_2];
+; CHECK-NEXT:    ld.param.b32 %r2, [wombat_param_1];
+; CHECK-NEXT:    ld.param.b32 %r1, [wombat_param_0];
+; CHECK-NEXT:    mov.b32 %r7, 0;
 ; CHECK-NEXT:  $L__BB0_1: // %bb3
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    { // callseq 0, 0
@@ -23,15 +23,15 @@ define ptx_kernel void @wombat(i32 %arg, i32 %arg1, i32 %arg2) {
 ; CHECK-NEXT:    st.param.b64 [param0], 0;
 ; CHECK-NEXT:    call.uni (retval0), quux, (param0);
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    mul.lo.s32 %r7, %r10, %r3;
-; CHECK-NEXT:    or.b32 %r8, %r4, %r7;
-; CHECK-NEXT:    mul.lo.s32 %r9, %r2, %r8;
-; CHECK-NEXT:    cvt.rn.f64.s32 %rd1, %r9;
-; CHECK-NEXT:    cvt.rn.f64.u32 %rd2, %r10;
+; CHECK-NEXT:    mul.lo.s32 %r4, %r7, %r2;
+; CHECK-NEXT:    or.b32 %r5, %r3, %r4;
+; CHECK-NEXT:    mul.lo.s32 %r6, %r1, %r5;
+; CHECK-NEXT:    cvt.rn.f64.s32 %rd1, %r6;
+; CHECK-NEXT:    cvt.rn.f64.u32 %rd2, %r7;
 ; CHECK-NEXT:    add.rn.f64 %rd3, %rd2, %rd1;
 ; CHECK-NEXT:    mov.b64 %rd4, 0;
 ; CHECK-NEXT:    st.global.b64 [%rd4], %rd3;
-; CHECK-NEXT:    mov.b32 %r10, 1;
+; CHECK-NEXT:    mov.b32 %r7, 1;
 ; CHECK-NEXT:    bra.uni $L__BB0_1;
 bb:
   br label %bb3
diff --git a/llvm/test/CodeGen/NVPTX/mulwide.ll b/llvm/test/CodeGen/NVPTX/mulwide.ll
index 666c7a160e1f0..17220340d4b07 100644
--- a/llvm/test/CodeGen/NVPTX/mulwide.ll
+++ b/llvm/test/CodeGen/NVPTX/mulwide.ll
@@ -203,27 +203,35 @@ define i64 @mulwideu32(i32 %a, i32 %b) {
 define i64 @mulwideu7(i7 %a, i7 %b) {
 ; OPT-LABEL: mulwideu7(
 ; OPT:       {
-; OPT-NEXT:    .reg .b32 %r<3>;
+; OPT-NEXT:    .reg .b32 %r<5>;
 ; OPT-NEXT:    .reg .b64 %rd<2>;
 ; OPT-EMPTY:
 ; OPT-NEXT:  // %bb.0:
-; OPT-NEXT:    ld.param.b8 %r1, [mulwideu7_param_0];
-; OPT-NEXT:    ld.param.b8 %r2, [mulwideu7_param_1];
-; OPT-NEXT:    mul.wide.u32 %rd1, %r1, %r2;
+; OPT-NEXT:    ld.param.b8 %r1, [mulwideu7_param_1];
+; OPT-NEXT:    and.b32 %r2, %r1, 127;
+; OPT-NEXT:    ld.param.b8 %r3, [mulwideu7_param_0];
+; OPT-NEXT:    and.b32 %r4, %r3, 127;
+; OPT-NEXT:    mul.wide.u32 %rd1, %r4, %r2;
 ; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
 ; OPT-NEXT:    ret;
 ;
 ; NOOPT-LABEL: mulwideu7(
 ; NOOPT:       {
-; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b16 %rs<9>;
 ; NOOPT-NEXT:    .reg .b64 %rd<6>;
 ; NOOPT-EMPTY:
 ; NOOPT-NEXT:  // %bb.0:
-; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwideu7_param_1];
-; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwideu7_param_0];
-; NOOPT-NEXT:    cvt.u64.u16 %rd1, %rs1;
+; NOOPT-NEXT:    ld.param.b8 %rs3, [mulwideu7_param_0+1];
+; NOOPT-NEXT:    shl.b16 %rs4, %rs3, 8;
+; NOOPT-NEXT:    ld.param.b8 %rs5, [mulwideu7_param_0];
+; NOOPT-NEXT:    or.b16 %rs1, %rs4, %rs5;
+; NOOPT-NEXT:    ld.param.b8 %rs6, [mulwideu7_param_1+1];
+; NOOPT-NEXT:    shl.b16 %rs7, %rs6, 8;
+; NOOPT-NEXT:    ld.param.b8 %rs8, [mulwideu7_param_1];
+; NOOPT-NEXT:    or.b16 %rs2, %rs7, %rs8;
+; NOOPT-NEXT:    cvt.u64.u16 %rd1, %rs5;
 ; NOOPT-NEXT:    and.b64 %rd2, %rd1, 127;
-; NOOPT-NEXT:    cvt.u64.u16 %rd3, %rs2;
+; NOOPT-NEXT:    cvt.u64.u16 %rd3, %rs8;
 ; NOOPT-NEXT:    and.b64 %rd4, %rd3, 127;
 ; NOOPT-NEXT:    mul.lo.s64 %rd5, %rd2, %rd4;
 ; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd5;
@@ -242,26 +250,32 @@ define i64 @mulwides7(i7 %a, i7 %b) {
 ; OPT-EMPTY:
 ; OPT-NEXT:  // %bb.0:
 ; OPT-NEXT:    ld.param.b8 %r1, [mulwides7_param_0];
-; OPT-NEXT:    bfe.s32 %r2, %r1, 0, 7;
-; OPT-NEXT:    ld.param.b8 %r3, [mulwides7_param_1];
-; OPT-NEXT:    bfe.s32 %r4, %r3, 0, 7;
-; OPT-NEXT:    mul.wide.s32 %rd1, %r2, %r4;
+; OPT-NEXT:    ld.param.b8 %r2, [mulwides7_param_1];
+; OPT-NEXT:    bfe.s32 %r3, %r2, 0, 7;
+; OPT-NEXT:    bfe.s32 %r4, %r1, 0, 7;
+; OPT-NEXT:    mul.wide.s32 %rd1, %r4, %r3;
 ; OPT-NEXT:    st.param.b64 [func_retval0], %rd1;
 ; OPT-NEXT:    ret;
 ;
 ; NOOPT-LABEL: mulwides7(
 ; NOOPT:       {
-; NOOPT-NEXT:    .reg .b16 %rs<3>;
+; NOOPT-NEXT:    .reg .b16 %rs<9>;
 ; NOOPT-NEXT:    .reg .b64 %rd<6>;
 ; NOOPT-EMPTY:
 ; NOOPT-NEXT:  // %bb.0:
-; NOOPT-NEXT:    ld.param.b8 %rs2, [mulwides7_param_1];
-; NOOPT-NEXT:    ld.param.b8 %rs1, [mulwides7_param_0];
-; NOOPT-NEXT:    cvt.u64.u16 %rd1, %rs1;
-; NOOPT-NEXT:    bfe.s64 %rd2, %rd1, 0, 7;
-; NOOPT-NEXT:    cvt.u64.u16 %rd3, %rs2;
-; NOOPT-NEXT:    bfe.s64 %rd4, %rd3, 0, 7;
-; NOOPT-NEXT:    mul.lo.s64 %rd5, %rd2, %rd4;
+; NOOPT-NEXT:    ld.param.b8 %rs3, [mulwides7_param_0+1];
+; NOOPT-NEXT:    shl.b16 %rs4, %rs3, 8;
+; NOOPT-NEXT:    ld.param.b8 %rs5, [mulwides7_param_0];
+; NOOPT-NEXT:    or.b16 %rs1, %rs4, %rs5;
+; NOOPT-NEXT:    ld.param.b8 %rs6, [mulwides7_param_1];
+; NOOPT-NEXT:    cvt.u64.u16 %rd1, %rs6;
+; NOOPT-NEXT:    cvt.u64.u16 %rd2, %rs5;
+; NOOPT-NEXT:    ld.param.b8 %rs7, [mulwides7_param_1+1];
+; NOOPT-NEXT:    shl.b16 %rs8, %rs7, 8;
+; NOOPT-NEXT:    or.b16 %rs2, %rs8, %rs6;
+; NOOPT-NEXT:    bfe.s64 %rd3, %rd2, 0, 7;
+; NOOPT-NEXT:    bfe.s64 %rd4, %rd1, 0, 7;
+; NOOPT-NEXT:    mul.lo.s64 %rd5, %rd3, %rd4;
 ; NOOPT-NEXT:    st.param.b64 [func_retval0], %rd5;
 ; NOOPT-NEXT:    ret;
   %val0 = sext i7 %a to i64
diff --git a/llvm/test/CodeGen/NVPTX/no-f32x2.ll b/llvm/test/CodeGen/NVPTX/no-f32x2.ll
new file mode 100644
index 0000000000000..b2b909166a0c6
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/no-f32x2.ll
@@ -0,0 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mcpu=sm_100 | FileCheck %s --check-prefix=F32X2
+; RUN: llc < %s -mcpu=sm_90 | FileCheck %s --check-prefix=NOF32X2
+; RUN: llc < %s -mcpu=sm_100 -nvptx-no-f32x2 | FileCheck %s --check-prefix=NOF32X2
+
+target triple = "nvptx64-nvidia-cuda"
+
+define <2 x float> @test(<2 x float> %a, <2 x float> %b)  {
+; F32X2-LABEL: test(
+; F32X2:       {
+; F32X2-NEXT:    .reg .b64 %rd<4>;
+; F32X2-EMPTY:
+; F32X2-NEXT:  // %bb.0:
+; F32X2-NEXT:    ld.param.b64 %rd1, [test_param_0];
+; F32X2-NEXT:    ld.param.b64 %rd2, [test_param_1];
+; F32X2-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; F32X2-NEXT:    st.param.b64 [func_retval0], %rd3;
+; F32X2-NEXT:    ret;
+;
+; NOF32X2-LABEL: test(
+; NOF32X2:       {
+; NOF32X2-NEXT:    .reg .b32 %r<7>;
+; NOF32X2-EMPTY:
+; NOF32X2-NEXT:  // %bb.0:
+; NOF32X2-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_param_0];
+; NOF32X2-NEXT:    ld.param.v2.b32 {%r3, %r4}, [test_param_1];
+; NOF32X2-NEXT:    add.rn.f32 %r5, %r2, %r4;
+; NOF32X2-NEXT:    add.rn.f32 %r6, %r1, %r3;
+; NOF32X2-NEXT:    st.param.v2.b32 [func_retval0], {%r6, %r5};
+; NOF32X2-NEXT:    ret;
+  %c = fadd <2 x float> %a, %b
+  ret <2 x float> %c
+}
diff --git a/llvm/test/CodeGen/NVPTX/param-add.ll b/llvm/test/CodeGen/NVPTX/param-add.ll
index c5ea9f850ea1f..06d7384200696 100644
--- a/llvm/test/CodeGen/NVPTX/param-add.ll
+++ b/llvm/test/CodeGen/NVPTX/param-add.ll
@@ -14,7 +14,7 @@ declare i32 @callee(%struct.1float %a)
 define i32 @test(%struct.1float alignstack(32) %data) {
 ; CHECK-LABEL: test(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<7>;
+; CHECK-NEXT:    .reg .b32 %r<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/param-load-store.ll b/llvm/test/CodeGen/NVPTX/param-load-store.ll
index db3fbbc1d2c0f..90c8b921009b6 100644
--- a/llvm/test/CodeGen/NVPTX/param-load-store.ll
+++ b/llvm/test/CodeGen/NVPTX/param-load-store.ll
@@ -523,8 +523,7 @@ define <9 x half> @test_v9f16(<9 x half> %a) {
 ; CHECK: .func  (.param .b32 func_retval0)
 ; CHECK-LABEL: test_i19(
 ; CHECK-NEXT: .param .b32 test_i19_param_0
-; CHECK-DAG:  ld.param.b16    {{%r[0-9]+}}, [test_i19_param_0];
-; CHECK-DAG:  ld.param.b8     {{%r[0-9]+}}, [test_i19_param_0+2];
+; CHECK:      ld.param.b32    {{%r[0-9]+}}, [test_i19_param_0];
 ; CHECK:      .param .b32 param0;
 ; CHECK:      .param .b32 retval0;
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
@@ -540,8 +539,7 @@ define i19 @test_i19(i19 %a) {
 ; CHECK: .func  (.param .b32 func_retval0)
 ; CHECK-LABEL: test_i23(
 ; CHECK-NEXT: .param .b32 test_i23_param_0
-; CHECK-DAG:  ld.param.b16    {{%r[0-9]+}}, [test_i23_param_0];
-; CHECK-DAG:  ld.param.b8     {{%r[0-9]+}}, [test_i23_param_0+2];
+; CHECK:      ld.param.b32    {{%r[0-9]+}}, [test_i23_param_0];
 ; CHECK:      .param .b32 param0;
 ; CHECK:      .param .b32 retval0;
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
@@ -557,8 +555,7 @@ define i23 @test_i23(i23 %a) {
 ; CHECK: .func  (.param .b32 func_retval0)
 ; CHECK-LABEL: test_i24(
 ; CHECK-NEXT: .param .b32 test_i24_param_0
-; CHECK-DAG:  ld.param.b8     {{%r[0-9]+}}, [test_i24_param_0+2];
-; CHECK-DAG:  ld.param.b16    {{%r[0-9]+}}, [test_i24_param_0];
+; CHECK:      ld.param.b32    {{%r[0-9]+}}, [test_i24_param_0];
 ; CHECK:      .param .b32 param0;
 ; CHECK:      .param .b32 retval0;
 ; CHECK:      st.param.b32    [param0], {{%r[0-9]+}};
@@ -678,8 +675,7 @@ define float @test_f32(float %a) {
 ; CHECK: .func  (.param .b64 func_retval0)
 ; CHECK-LABEL: test_i40(
 ; CHECK-NEXT: .param .b64 test_i40_param_0
-; CHECK-DAG:  ld.param.b8    {{%rd[0-9]+}}, [test_i40_param_0+4];
-; CHECK-DAG:  ld.param.b32   {{%rd[0-9]+}}, [test_i40_param_0];
+; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [test_i40_param_0];
 ; CHECK:      .param .b64 param0;
 ; CHECK:      .param .b64 retval0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
@@ -695,8 +691,7 @@ define i40 @test_i40(i40 %a) {
 ; CHECK: .func  (.param .b64 func_retval0)
 ; CHECK-LABEL: test_i47(
 ; CHECK-NEXT: .param .b64 test_i47_param_0
-; CHECK-DAG:  ld.param.b16   {{%rd[0-9]+}}, [test_i47_param_0+4];
-; CHECK-DAG:  ld.param.b32   {{%rd[0-9]+}}, [test_i47_param_0];
+; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [test_i47_param_0];
 ; CHECK:      .param .b64 param0;
 ; CHECK:      .param .b64 retval0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
@@ -712,8 +707,7 @@ define i47 @test_i47(i47 %a) {
 ; CHECK: .func  (.param .b64 func_retval0)
 ; CHECK-LABEL: test_i48(
 ; CHECK-NEXT: .param .b64 test_i48_param_0
-; CHECK-DAG:  ld.param.b16   {{%rd[0-9]+}}, [test_i48_param_0+4];
-; CHECK-DAG:  ld.param.b32   {{%rd[0-9]+}}, [test_i48_param_0];
+; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [test_i48_param_0];
 ; CHECK:      .param .b64 param0;
 ; CHECK:      .param .b64 retval0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
@@ -729,9 +723,7 @@ define i48 @test_i48(i48 %a) {
 ; CHECK: .func  (.param .b64 func_retval0)
 ; CHECK-LABEL: test_i51(
 ; CHECK-NEXT: .param .b64 test_i51_param_0
-; CHECK-DAG:  ld.param.b8    {{%rd[0-9]+}}, [test_i51_param_0+6];
-; CHECK-DAG:  ld.param.b16   {{%rd[0-9]+}}, [test_i51_param_0+4];
-; CHECK-DAG:  ld.param.b32   {{%rd[0-9]+}}, [test_i51_param_0];
+; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [test_i51_param_0];
 ; CHECK:      .param .b64 param0;
 ; CHECK:      .param .b64 retval0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
@@ -747,9 +739,7 @@ define i51 @test_i51(i51 %a) {
 ; CHECK: .func  (.param .b64 func_retval0)
 ; CHECK-LABEL: test_i56(
 ; CHECK-NEXT: .param .b64 test_i56_param_0
-; CHECK-DAG:  ld.param.b8    {{%rd[0-9]+}}, [test_i56_param_0+6];
-; CHECK-DAG:  ld.param.b16   {{%rd[0-9]+}}, [test_i56_param_0+4];
-; CHECK-DAG:  ld.param.b32   {{%rd[0-9]+}}, [test_i56_param_0];
+; CHECK:      ld.param.b64    {{%rd[0-9]+}}, [test_i56_param_0];
 ; CHECK:      .param .b64 param0;
 ; CHECK:      .param .b64 retval0;
 ; CHECK:      st.param.b64    [param0], {{%rd[0-9]+}};
diff --git a/llvm/test/CodeGen/NVPTX/param-overalign.ll b/llvm/test/CodeGen/NVPTX/param-overalign.ll
index 2155fb4031c36..8899709d1cf15 100644
--- a/llvm/test/CodeGen/NVPTX/param-overalign.ll
+++ b/llvm/test/CodeGen/NVPTX/param-overalign.ll
@@ -21,7 +21,7 @@ target triple = "nvptx64-nvidia-cuda"
 define float @caller_md(float %a, float %b) {
 ; CHECK-LABEL: caller_md(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [caller_md_param_0];
@@ -62,7 +62,7 @@ define float @callee_md(%struct.float2 alignstack(8) %a) {
 define float @caller(float %a, float %b) {
 ; CHECK-LABEL: caller(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [caller_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/pr126337.ll b/llvm/test/CodeGen/NVPTX/pr126337.ll
index 95258f7a3f360..f56b8eb98077c 100644
--- a/llvm/test/CodeGen/NVPTX/pr126337.ll
+++ b/llvm/test/CodeGen/NVPTX/pr126337.ll
@@ -17,17 +17,16 @@ define ptx_kernel void @Equal_GPU_DT_COMPLEX64_DT_BOOL_kernel(<2 x float> %0) {
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
 ; CHECK-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0: // %.preheader15
-; CHECK-NEXT:    ld.param.b64 %rd1, [Equal_GPU_DT_COMPLEX64_DT_BOOL_kernel_param_0];
-; CHECK-NEXT:    { .reg .b32 tmp; mov.b64 {%r1, tmp}, %rd1; }
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [Equal_GPU_DT_COMPLEX64_DT_BOOL_kernel_param_0];
 ; CHECK-NEXT:    setp.eq.f32 %p1, %r1, 0f00000000;
 ; CHECK-NEXT:    selp.b16 %rs1, 1, 0, %p1;
 ; CHECK-NEXT:  $L__BB0_1: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    mov.b64 %rd2, 0;
-; CHECK-NEXT:    st.b8 [%rd2], %rs1;
+; CHECK-NEXT:    mov.b64 %rd1, 0;
+; CHECK-NEXT:    st.b8 [%rd1], %rs1;
 ; CHECK-NEXT:    bra.uni $L__BB0_1;
 .preheader15:
   br label %1
diff --git a/llvm/test/CodeGen/NVPTX/prefetch-inferas-test.ll b/llvm/test/CodeGen/NVPTX/prefetch-inferas-test.ll
new file mode 100644
index 0000000000000..3efe9be898cc8
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/prefetch-inferas-test.ll
@@ -0,0 +1,80 @@
+; RUN: opt < %s -S -passes=infer-address-spaces | FileCheck %s --check-prefix=INFER
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80 | FileCheck %s --check-prefix=PTX
+; RUN: %if ptxas-12.3 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_90 -mattr=+ptx80 | %ptxas-verify -arch=sm_90 %}
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
+target triple = "nvptx64-unknown-unknown"
+
+@constant_tensormap = addrspace(4) global [64 x i8] zeroinitializer, align 64
+
+; Inference from const address space
+define void @test_infer_const_from_cast() {
+; INFER-LABEL: @test_infer_const_from_cast
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) @constant_tensormap)
+; BOTH: call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) @constant_tensormap)
+; PTX-LABEL: .visible .func test_infer_const_from_cast(
+; PTX: mov.b64 %rd{{[0-9]+}}, constant_tensormap;
+; PTX: cvta.const.u64 %rd{{[0-9]+}}, %rd{{[0-9]+}};
+; PTX: prefetch.tensormap [%rd{{[0-9]+}}];
+entry:
+  %casted = addrspacecast ptr addrspace(4) @constant_tensormap to ptr
+  call void @llvm.nvvm.prefetch.tensormap.p0(ptr %casted)
+  ret void
+}
+
+; Cast from Const space to Generic
+define void @test_const_to_generic_cast(ptr addrspace(4) %const_ptr) {
+; INFER-LABEL: @test_const_to_generic_cast
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) %const_ptr)
+; PTX-LABEL: .visible .func test_const_to_generic_cast(
+; PTX: prefetch.const.tensormap [%rd{{[0-9]+}}];
+entry:
+  %cast = addrspacecast ptr addrspace(4) %const_ptr to ptr
+  call void @llvm.nvvm.prefetch.tensormap.p0(ptr %cast)
+  ret void
+}
+
+; No inference possible 
+define void @test_no_inference_possible(ptr %generic_ptr) {
+; INFER-LABEL: @test_no_inference_possible
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p0(ptr %generic_ptr)
+; PTX-LABEL: .visible .func test_no_inference_possible(
+; PTX: prefetch.tensormap [%rd{{[0-9]+}}]; 
+entry:
+  call void @llvm.nvvm.prefetch.tensormap.p0(ptr %generic_ptr)
+  ret void
+}
+
+; Cast from Parameter space to Generic
+define void @test_param_to_generic_cast(ptr addrspace(101) %param_ptr) {
+; INFER-LABEL: @test_param_to_generic_cast
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101) %param_ptr)
+; PTX-LABEL: .visible .func test_param_to_generic_cast(
+; PTX: prefetch.param.tensormap [%rd{{[0-9]+}}];
+entry:
+  %cast = addrspacecast ptr addrspace(101) %param_ptr to ptr
+  call void @llvm.nvvm.prefetch.tensormap.p0(ptr %cast)
+  ret void
+}
+
+; Multiple casts in sequence
+define void @test_infer_through_multiple_casts() {
+; INFER-LABEL: @test_infer_through_multiple_casts
+; INFER: call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) @constant_tensormap)
+; PTX-LABEL: .visible .func test_infer_through_multiple_casts(
+; PTX: mov.b64 %rd{{[0-9]+}}, constant_tensormap;
+; PTX: cvta.const.u64 %rd{{[0-9]+}}, %rd{{[0-9]+}};
+; PTX: prefetch.tensormap [%rd{{[0-9]+}}];
+entry:
+  %cast1 = addrspacecast ptr addrspace(4) @constant_tensormap to ptr
+  %cast2 = addrspacecast ptr %cast1 to ptr addrspace(4)
+  %cast3 = addrspacecast ptr addrspace(4) %cast2 to ptr
+  call void @llvm.nvvm.prefetch.tensormap(ptr %cast3)
+  ret void
+}
+
+declare void @llvm.nvvm.prefetch.tensormap.p0(ptr)
+declare void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4))
+declare void @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101))
+
+
diff --git a/llvm/test/CodeGen/NVPTX/prefetch.ll b/llvm/test/CodeGen/NVPTX/prefetch.ll
index a64e4fe7a508e..862e26d704679 100644
--- a/llvm/test/CodeGen/NVPTX/prefetch.ll
+++ b/llvm/test/CodeGen/NVPTX/prefetch.ll
@@ -12,6 +12,10 @@ declare void  @llvm.nvvm.prefetch.local.L2(ptr addrspace(5) %local_ptr)
 declare void  @llvm.nvvm.prefetch.L1(ptr %ptr)
 declare void  @llvm.nvvm.prefetch.L2(ptr %ptr)
 
+declare void  @llvm.nvvm.prefetch.tensormap.p0(ptr %ptr)
+declare void  @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) %const_ptr)
+declare void  @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101) %param_ptr)
+
 declare void  @llvm.nvvm.prefetch.global.L2.evict.normal(ptr addrspace(1) %global_ptr)
 declare void  @llvm.nvvm.prefetch.global.L2.evict.last(ptr addrspace(1) %global_ptr)
 
@@ -78,4 +82,43 @@ define void @prefetchu_l1(ptr %ptr) {
 ; CHECK-PTX64-NEXT:    ret;
   tail call void @llvm.nvvm.prefetchu.L1(ptr %ptr)
   ret void
+}
+
+define void @prefetch_tensormap(ptr %ptr) {
+; CHECK-PTX64-LABEL: prefetch_tensormap(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [prefetch_tensormap_param_0];
+; CHECK-PTX64-NEXT:    prefetch.tensormap [%rd1];
+; CHECK-PTX64-NEXT:    ret;
+  tail call void @llvm.nvvm.prefetch.tensormap.p0(ptr %ptr)
+  ret void
+}
+
+define void @prefetch_const_tensormap(ptr addrspace(4) %const_ptr) {
+; CHECK-PTX64-LABEL: prefetch_const_tensormap(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [prefetch_const_tensormap_param_0];
+; CHECK-PTX64-NEXT:    prefetch.const.tensormap [%rd1];
+; CHECK-PTX64-NEXT:    ret;
+  tail call void @llvm.nvvm.prefetch.tensormap.p4(ptr addrspace(4) %const_ptr)
+  ret void
+}
+
+define void @prefetch_param_tensormap(ptr addrspace(101) %param_ptr) {
+; CHECK-PTX64-LABEL: prefetch_param_tensormap(
+; CHECK-PTX64:       {
+; CHECK-PTX64-NEXT:    .reg .b64 %rd<2>;
+; CHECK-PTX64-EMPTY:
+; CHECK-PTX64-NEXT:  // %bb.0:
+; CHECK-PTX64-NEXT:    ld.param.b64 %rd1, [prefetch_param_tensormap_param_0];
+; CHECK-PTX64-NEXT:    prefetch.param.tensormap [%rd1];
+; CHECK-PTX64-NEXT:    ret;
+  tail call void @llvm.nvvm.prefetch.tensormap.p101(ptr addrspace(101) %param_ptr)
+  ret void
 }
\ No newline at end of file
diff --git a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir
index 4a5315203dba1..dfc84177fb0e6 100644
--- a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir
+++ b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure.mir
@@ -66,8 +66,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/NVPTX/read-global-variable-constant.ll b/llvm/test/CodeGen/NVPTX/read-global-variable-constant.ll
index c78fcddb7ed0f..153d677058d9f 100644
--- a/llvm/test/CodeGen/NVPTX/read-global-variable-constant.ll
+++ b/llvm/test/CodeGen/NVPTX/read-global-variable-constant.ll
@@ -25,11 +25,11 @@ define float @test_gv_float() {
 define <2 x float> @test_gv_float2() {
 ; CHECK-LABEL: test_gv_float2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.global.nc.b64 %rd1, [gv_float2];
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd1;
+; CHECK-NEXT:    ld.global.nc.v2.b32 {%r1, %r2}, [gv_float2];
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r2};
 ; CHECK-NEXT:    ret;
   %v = load <2 x float>, ptr @gv_float2
   ret <2 x float> %v
@@ -38,11 +38,11 @@ define <2 x float> @test_gv_float2() {
 define <4 x float> @test_gv_float4() {
 ; CHECK-LABEL: test_gv_float4(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.global.nc.v2.b64 {%rd1, %rd2}, [gv_float4];
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; CHECK-NEXT:    ld.global.nc.v4.b32 {%r1, %r2, %r3, %r4}, [gv_float4];
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r1, %r2, %r3, %r4};
 ; CHECK-NEXT:    ret;
   %v = load <4 x float>, ptr @gv_float4
   ret <4 x float> %v
diff --git a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
index 92cb51b17f0c8..f286928da4481 100644
--- a/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
+++ b/llvm/test/CodeGen/NVPTX/reduction-intrinsics.ll
@@ -2,19 +2,18 @@
 ; RUN: llc < %s -mcpu=sm_80 -mattr=+ptx70 -O0 \
 ; RUN:      -disable-post-ra -verify-machineinstrs \
 ; RUN: | FileCheck -check-prefixes CHECK,CHECK-SM80 %s
-; RUN: %if ptxas-12.8 %{ llc < %s -mcpu=sm_80 -mattr=+ptx70 -O0 \
+; RUN: %if ptxas-12.9 %{ llc < %s -mcpu=sm_80 -mattr=+ptx70 -O0 \
 ; RUN:      -disable-post-ra -verify-machineinstrs \
 ; RUN: | %ptxas-verify -arch=sm_80 %}
-; RUN: llc < %s -mcpu=sm_100 -mattr=+ptx87 -O0 \
+; RUN: llc < %s -mcpu=sm_100 -mattr=+ptx88 -O0 \
 ; RUN:      -disable-post-ra -verify-machineinstrs \
 ; RUN: | FileCheck -check-prefixes CHECK,CHECK-SM100 %s
-; RUN: %if ptxas-12.8 %{ llc < %s -mcpu=sm_100 -mattr=+ptx87 -O0 \
+; RUN: %if ptxas-12.9 %{ llc < %s -mcpu=sm_100 -mattr=+ptx88 -O0 \
 ; RUN:      -disable-post-ra -verify-machineinstrs \
 ; RUN: | %ptxas-verify -arch=sm_100 %}
 target triple = "nvptx64-nvidia-cuda"
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
-; Check straight line reduction.
 define half @reduce_fadd_half(<8 x half> %in) {
 ; CHECK-LABEL: reduce_fadd_half(
 ; CHECK:       {
@@ -43,45 +42,22 @@ define half @reduce_fadd_half(<8 x half> %in) {
 }
 
 define half @reduce_fadd_half_reassoc(<8 x half> %in) {
-; CHECK-SM80-LABEL: reduce_fadd_half_reassoc(
-; CHECK-SM80:       {
-; CHECK-SM80-NEXT:    .reg .b16 %rs<6>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<10>;
-; CHECK-SM80-EMPTY:
-; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fadd_half_reassoc_param_0];
-; CHECK-SM80-NEXT:    add.rn.f16x2 %r5, %r2, %r4;
-; CHECK-SM80-NEXT:    add.rn.f16x2 %r6, %r1, %r3;
-; CHECK-SM80-NEXT:    add.rn.f16x2 %r7, %r6, %r5;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
-; CHECK-SM80-NEXT:    // implicit-def: %rs2
-; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM80-NEXT:    add.rn.f16x2 %r9, %r7, %r8;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
-; CHECK-SM80-NEXT:    mov.b16 %rs4, 0x0000;
-; CHECK-SM80-NEXT:    add.rn.f16 %rs5, %rs3, %rs4;
-; CHECK-SM80-NEXT:    st.param.b16 [func_retval0], %rs5;
-; CHECK-SM80-NEXT:    ret;
-;
-; CHECK-SM100-LABEL: reduce_fadd_half_reassoc(
-; CHECK-SM100:       {
-; CHECK-SM100-NEXT:    .reg .b16 %rs<6>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<10>;
-; CHECK-SM100-EMPTY:
-; CHECK-SM100-NEXT:  // %bb.0:
-; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fadd_half_reassoc_param_0];
-; CHECK-SM100-NEXT:    add.rn.f16x2 %r5, %r2, %r4;
-; CHECK-SM100-NEXT:    add.rn.f16x2 %r6, %r1, %r3;
-; CHECK-SM100-NEXT:    add.rn.f16x2 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    add.rn.f16x2 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    mov.b16 %rs4, 0x0000;
-; CHECK-SM100-NEXT:    add.rn.f16 %rs5, %rs3, %rs4;
-; CHECK-SM100-NEXT:    st.param.b16 [func_retval0], %rs5;
-; CHECK-SM100-NEXT:    ret;
+; CHECK-LABEL: reduce_fadd_half_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<6>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fadd_half_reassoc_param_0];
+; CHECK-NEXT:    add.rn.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    add.rn.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    add.rn.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    add.rn.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    mov.b16 %rs4, 0x0000;
+; CHECK-NEXT:    add.rn.f16 %rs5, %rs3, %rs4;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs5;
+; CHECK-NEXT:    ret;
   %res = call reassoc half @llvm.vector.reduce.fadd(half 0.0, <8 x half> %in)
   ret half %res
 }
@@ -109,30 +85,47 @@ define half @reduce_fadd_half_reassoc_nonpow2(<7 x half> %in) {
   ret half %res
 }
 
-; Check straight-line reduction.
 define float @reduce_fadd_float(<8 x float> %in) {
-; CHECK-LABEL: reduce_fadd_float(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<17>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fadd_float_param_0+16];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd3;
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fadd_float_param_0];
-; CHECK-NEXT:    mov.b64 {%r5, %r6}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r7, %r8}, %rd1;
-; CHECK-NEXT:    add.rn.f32 %r9, %r7, 0f00000000;
-; CHECK-NEXT:    add.rn.f32 %r10, %r9, %r8;
-; CHECK-NEXT:    add.rn.f32 %r11, %r10, %r5;
-; CHECK-NEXT:    add.rn.f32 %r12, %r11, %r6;
-; CHECK-NEXT:    add.rn.f32 %r13, %r12, %r3;
-; CHECK-NEXT:    add.rn.f32 %r14, %r13, %r4;
-; CHECK-NEXT:    add.rn.f32 %r15, %r14, %r1;
-; CHECK-NEXT:    add.rn.f32 %r16, %r15, %r2;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r16;
-; CHECK-NEXT:    ret;
+; CHECK-SM80-LABEL: reduce_fadd_float(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<17>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fadd_float_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fadd_float_param_0];
+; CHECK-SM80-NEXT:    add.rn.f32 %r9, %r1, 0f00000000;
+; CHECK-SM80-NEXT:    add.rn.f32 %r10, %r9, %r2;
+; CHECK-SM80-NEXT:    add.rn.f32 %r11, %r10, %r3;
+; CHECK-SM80-NEXT:    add.rn.f32 %r12, %r11, %r4;
+; CHECK-SM80-NEXT:    add.rn.f32 %r13, %r12, %r5;
+; CHECK-SM80-NEXT:    add.rn.f32 %r14, %r13, %r6;
+; CHECK-SM80-NEXT:    add.rn.f32 %r15, %r14, %r7;
+; CHECK-SM80-NEXT:    add.rn.f32 %r16, %r15, %r8;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r16;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fadd_float(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<17>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fadd_float_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd3;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fadd_float_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd2;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd1;
+; CHECK-SM100-NEXT:    add.rn.f32 %r9, %r7, 0f00000000;
+; CHECK-SM100-NEXT:    add.rn.f32 %r10, %r9, %r8;
+; CHECK-SM100-NEXT:    add.rn.f32 %r11, %r10, %r5;
+; CHECK-SM100-NEXT:    add.rn.f32 %r12, %r11, %r6;
+; CHECK-SM100-NEXT:    add.rn.f32 %r13, %r12, %r3;
+; CHECK-SM100-NEXT:    add.rn.f32 %r14, %r13, %r4;
+; CHECK-SM100-NEXT:    add.rn.f32 %r15, %r14, %r1;
+; CHECK-SM100-NEXT:    add.rn.f32 %r16, %r15, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r16;
+; CHECK-SM100-NEXT:    ret;
   %res = call float @llvm.vector.reduce.fadd(float 0.0, <8 x float> %in)
   ret float %res
 }
@@ -141,22 +134,17 @@ define float @reduce_fadd_float_reassoc(<8 x float> %in) {
 ; CHECK-SM80-LABEL: reduce_fadd_float_reassoc(
 ; CHECK-SM80:       {
 ; CHECK-SM80-NEXT:    .reg .b32 %r<17>;
-; CHECK-SM80-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-SM80-EMPTY:
 ; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fadd_float_reassoc_param_0+16];
-; CHECK-SM80-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fadd_float_reassoc_param_0];
-; CHECK-SM80-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-SM80-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-SM80-NEXT:    add.rn.f32 %r5, %r3, %r1;
-; CHECK-SM80-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-SM80-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-SM80-NEXT:    add.rn.f32 %r10, %r8, %r6;
-; CHECK-SM80-NEXT:    add.rn.f32 %r11, %r4, %r2;
-; CHECK-SM80-NEXT:    add.rn.f32 %r12, %r9, %r7;
-; CHECK-SM80-NEXT:    add.rn.f32 %r13, %r12, %r11;
-; CHECK-SM80-NEXT:    add.rn.f32 %r14, %r10, %r5;
-; CHECK-SM80-NEXT:    add.rn.f32 %r15, %r14, %r13;
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fadd_float_reassoc_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fadd_float_reassoc_param_0];
+; CHECK-SM80-NEXT:    add.rn.f32 %r9, %r4, %r8;
+; CHECK-SM80-NEXT:    add.rn.f32 %r10, %r2, %r6;
+; CHECK-SM80-NEXT:    add.rn.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    add.rn.f32 %r12, %r3, %r7;
+; CHECK-SM80-NEXT:    add.rn.f32 %r13, %r1, %r5;
+; CHECK-SM80-NEXT:    add.rn.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    add.rn.f32 %r15, %r14, %r11;
 ; CHECK-SM80-NEXT:    add.rn.f32 %r16, %r15, 0f00000000;
 ; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r16;
 ; CHECK-SM80-NEXT:    ret;
@@ -164,7 +152,7 @@ define float @reduce_fadd_float_reassoc(<8 x float> %in) {
 ; CHECK-SM100-LABEL: reduce_fadd_float_reassoc(
 ; CHECK-SM100:       {
 ; CHECK-SM100-NEXT:    .reg .b32 %r<5>;
-; CHECK-SM100-NEXT:    .reg .b64 %rd<10>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<8>;
 ; CHECK-SM100-EMPTY:
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fadd_float_reassoc_param_0+16];
@@ -172,11 +160,8 @@ define float @reduce_fadd_float_reassoc(<8 x float> %in) {
 ; CHECK-SM100-NEXT:    add.rn.f32x2 %rd5, %rd2, %rd4;
 ; CHECK-SM100-NEXT:    add.rn.f32x2 %rd6, %rd1, %rd3;
 ; CHECK-SM100-NEXT:    add.rn.f32x2 %rd7, %rd6, %rd5;
-; CHECK-SM100-NEXT:    mov.b64 {_, %r1}, %rd7;
-; CHECK-SM100-NEXT:    // implicit-def: %r2
-; CHECK-SM100-NEXT:    mov.b64 %rd8, {%r1, %r2};
-; CHECK-SM100-NEXT:    add.rn.f32x2 %rd9, %rd7, %rd8;
-; CHECK-SM100-NEXT:    mov.b64 {%r3, _}, %rd9;
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd7;
+; CHECK-SM100-NEXT:    add.rn.f32 %r3, %r1, %r2;
 ; CHECK-SM100-NEXT:    add.rn.f32 %r4, %r3, 0f00000000;
 ; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r4;
 ; CHECK-SM100-NEXT:    ret;
@@ -229,7 +214,6 @@ define float @reduce_fadd_float_reassoc_nonpow2(<7 x float> %in) {
   ret float %res
 }
 
-; Check straight line reduction.
 define half @reduce_fmul_half(<8 x half> %in) {
 ; CHECK-LABEL: reduce_fmul_half(
 ; CHECK:       {
@@ -256,41 +240,20 @@ define half @reduce_fmul_half(<8 x half> %in) {
 }
 
 define half @reduce_fmul_half_reassoc(<8 x half> %in) {
-; CHECK-SM80-LABEL: reduce_fmul_half_reassoc(
-; CHECK-SM80:       {
-; CHECK-SM80-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<10>;
-; CHECK-SM80-EMPTY:
-; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmul_half_reassoc_param_0];
-; CHECK-SM80-NEXT:    mul.rn.f16x2 %r5, %r2, %r4;
-; CHECK-SM80-NEXT:    mul.rn.f16x2 %r6, %r1, %r3;
-; CHECK-SM80-NEXT:    mul.rn.f16x2 %r7, %r6, %r5;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
-; CHECK-SM80-NEXT:    // implicit-def: %rs2
-; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM80-NEXT:    mul.rn.f16x2 %r9, %r7, %r8;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
-; CHECK-SM80-NEXT:    st.param.b16 [func_retval0], %rs3;
-; CHECK-SM80-NEXT:    ret;
-;
-; CHECK-SM100-LABEL: reduce_fmul_half_reassoc(
-; CHECK-SM100:       {
-; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<10>;
-; CHECK-SM100-EMPTY:
-; CHECK-SM100-NEXT:  // %bb.0:
-; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmul_half_reassoc_param_0];
-; CHECK-SM100-NEXT:    mul.rn.f16x2 %r5, %r2, %r4;
-; CHECK-SM100-NEXT:    mul.rn.f16x2 %r6, %r1, %r3;
-; CHECK-SM100-NEXT:    mul.rn.f16x2 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    mul.rn.f16x2 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    st.param.b16 [func_retval0], %rs3;
-; CHECK-SM100-NEXT:    ret;
+; CHECK-LABEL: reduce_fmul_half_reassoc(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmul_half_reassoc_param_0];
+; CHECK-NEXT:    mul.rn.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    mul.rn.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    mul.rn.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    mul.rn.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
+; CHECK-NEXT:    ret;
   %res = call reassoc half @llvm.vector.reduce.fmul(half 1.0, <8 x half> %in)
   ret half %res
 }
@@ -321,29 +284,45 @@ define half @reduce_fmul_half_reassoc_nonpow2(<7 x half> %in) {
   ret half %res
 }
 
-; Check straight-line reduction.
 define float @reduce_fmul_float(<8 x float> %in) {
-; CHECK-LABEL: reduce_fmul_float(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmul_float_param_0+16];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd3;
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmul_float_param_0];
-; CHECK-NEXT:    mov.b64 {%r5, %r6}, %rd2;
-; CHECK-NEXT:    mov.b64 {%r7, %r8}, %rd1;
-; CHECK-NEXT:    mul.rn.f32 %r9, %r7, %r8;
-; CHECK-NEXT:    mul.rn.f32 %r10, %r9, %r5;
-; CHECK-NEXT:    mul.rn.f32 %r11, %r10, %r6;
-; CHECK-NEXT:    mul.rn.f32 %r12, %r11, %r3;
-; CHECK-NEXT:    mul.rn.f32 %r13, %r12, %r4;
-; CHECK-NEXT:    mul.rn.f32 %r14, %r13, %r1;
-; CHECK-NEXT:    mul.rn.f32 %r15, %r14, %r2;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
-; CHECK-NEXT:    ret;
+; CHECK-SM80-LABEL: reduce_fmul_float(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmul_float_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmul_float_param_0];
+; CHECK-SM80-NEXT:    mul.rn.f32 %r9, %r1, %r2;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r10, %r9, %r3;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r11, %r10, %r4;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r12, %r11, %r5;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r13, %r12, %r6;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r14, %r13, %r7;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r15, %r14, %r8;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmul_float(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmul_float_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd3;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmul_float_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd2;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd1;
+; CHECK-SM100-NEXT:    mul.rn.f32 %r9, %r7, %r8;
+; CHECK-SM100-NEXT:    mul.rn.f32 %r10, %r9, %r5;
+; CHECK-SM100-NEXT:    mul.rn.f32 %r11, %r10, %r6;
+; CHECK-SM100-NEXT:    mul.rn.f32 %r12, %r11, %r3;
+; CHECK-SM100-NEXT:    mul.rn.f32 %r13, %r12, %r4;
+; CHECK-SM100-NEXT:    mul.rn.f32 %r14, %r13, %r1;
+; CHECK-SM100-NEXT:    mul.rn.f32 %r15, %r14, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM100-NEXT:    ret;
   %res = call float @llvm.vector.reduce.fmul(float 1.0, <8 x float> %in)
   ret float %res
 }
@@ -352,29 +331,24 @@ define float @reduce_fmul_float_reassoc(<8 x float> %in) {
 ; CHECK-SM80-LABEL: reduce_fmul_float_reassoc(
 ; CHECK-SM80:       {
 ; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
-; CHECK-SM80-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-SM80-EMPTY:
 ; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmul_float_reassoc_param_0+16];
-; CHECK-SM80-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmul_float_reassoc_param_0];
-; CHECK-SM80-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-SM80-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-SM80-NEXT:    mul.rn.f32 %r5, %r3, %r1;
-; CHECK-SM80-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-SM80-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-SM80-NEXT:    mul.rn.f32 %r10, %r8, %r6;
-; CHECK-SM80-NEXT:    mul.rn.f32 %r11, %r4, %r2;
-; CHECK-SM80-NEXT:    mul.rn.f32 %r12, %r9, %r7;
-; CHECK-SM80-NEXT:    mul.rn.f32 %r13, %r12, %r11;
-; CHECK-SM80-NEXT:    mul.rn.f32 %r14, %r10, %r5;
-; CHECK-SM80-NEXT:    mul.rn.f32 %r15, %r14, %r13;
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmul_float_reassoc_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmul_float_reassoc_param_0];
+; CHECK-SM80-NEXT:    mul.rn.f32 %r9, %r4, %r8;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r10, %r2, %r6;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r12, %r3, %r7;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r13, %r1, %r5;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    mul.rn.f32 %r15, %r14, %r11;
 ; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-SM80-NEXT:    ret;
 ;
 ; CHECK-SM100-LABEL: reduce_fmul_float_reassoc(
 ; CHECK-SM100:       {
 ; CHECK-SM100-NEXT:    .reg .b32 %r<4>;
-; CHECK-SM100-NEXT:    .reg .b64 %rd<10>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<8>;
 ; CHECK-SM100-EMPTY:
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmul_float_reassoc_param_0+16];
@@ -382,11 +356,8 @@ define float @reduce_fmul_float_reassoc(<8 x float> %in) {
 ; CHECK-SM100-NEXT:    mul.rn.f32x2 %rd5, %rd2, %rd4;
 ; CHECK-SM100-NEXT:    mul.rn.f32x2 %rd6, %rd1, %rd3;
 ; CHECK-SM100-NEXT:    mul.rn.f32x2 %rd7, %rd6, %rd5;
-; CHECK-SM100-NEXT:    mov.b64 {_, %r1}, %rd7;
-; CHECK-SM100-NEXT:    // implicit-def: %r2
-; CHECK-SM100-NEXT:    mov.b64 %rd8, {%r1, %r2};
-; CHECK-SM100-NEXT:    mul.rn.f32x2 %rd9, %rd7, %rd8;
-; CHECK-SM100-NEXT:    mov.b64 {%r3, _}, %rd9;
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd7;
+; CHECK-SM100-NEXT:    mul.rn.f32 %r3, %r1, %r2;
 ; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-SM100-NEXT:    ret;
   %res = call reassoc float @llvm.vector.reduce.fmul(float 1.0, <8 x float> %in)
@@ -436,7 +407,6 @@ define float @reduce_fmul_float_reassoc_nonpow2(<7 x float> %in) {
   ret float %res
 }
 
-; Check straight line reduction.
 define half @reduce_fmax_half(<8 x half> %in) {
 ; CHECK-LABEL: reduce_fmax_half(
 ; CHECK:       {
@@ -501,84 +471,241 @@ define half @reduce_fmax_half_reassoc_nonpow2(<7 x half> %in) {
   ret half %res
 }
 
-; Check straight-line reduction.
-define float @reduce_fmax_float(<8 x float> %in) {
-;
-; CHECK-LABEL: reduce_fmax_float(
+define half @reduce_fmax_half_nnan(<8 x half> %in) {
+; CHECK-LABEL: reduce_fmax_half_nnan(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmax_float_param_0+16];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmax_float_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    max.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-NEXT:    max.f32 %r10, %r9, %r7;
-; CHECK-NEXT:    max.f32 %r11, %r10, %r5;
-; CHECK-NEXT:    max.f32 %r12, %r3, %r1;
-; CHECK-NEXT:    max.f32 %r13, %r8, %r6;
-; CHECK-NEXT:    max.f32 %r14, %r13, %r12;
-; CHECK-NEXT:    max.f32 %r15, %r14, %r11;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmax_half_nnan_param_0];
+; CHECK-NEXT:    max.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    max.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    max.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    max.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
 ; CHECK-NEXT:    ret;
-  %res = call float @llvm.vector.reduce.fmax(<8 x float> %in)
-  ret float %res
+  %res = call nnan half @llvm.vector.reduce.fmax(<8 x half> %in)
+  ret half %res
 }
 
-define float @reduce_fmax_float_reassoc(<8 x float> %in) {
-;
-; CHECK-LABEL: reduce_fmax_float_reassoc(
+define half @reduce_fmax_half_nnan_nonpow2(<7 x half> %in) {
+; CHECK-LABEL: reduce_fmax_half_nnan_nonpow2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmax_float_reassoc_param_0+16];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmax_float_reassoc_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    max.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-NEXT:    max.f32 %r10, %r9, %r7;
-; CHECK-NEXT:    max.f32 %r11, %r10, %r5;
-; CHECK-NEXT:    max.f32 %r12, %r3, %r1;
-; CHECK-NEXT:    max.f32 %r13, %r8, %r6;
-; CHECK-NEXT:    max.f32 %r14, %r13, %r12;
-; CHECK-NEXT:    max.f32 %r15, %r14, %r11;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmax_half_nnan_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_fmax_half_nnan_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmax_half_nnan_nonpow2_param_0+12];
+; CHECK-NEXT:    max.f16x2 %r4, %r2, %r1;
+; CHECK-NEXT:    mov.b16 %rs8, 0xFC00;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    max.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT:    max.f16x2 %r7, %r4, %r6;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    max.f16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
 ; CHECK-NEXT:    ret;
+  %res = call nnan half @llvm.vector.reduce.fmax(<7 x half> %in)
+  ret half %res
+}
+
+define float @reduce_fmax_float(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmax_float(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmax_float_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmax_float_param_0];
+; CHECK-SM80-NEXT:    max.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    max.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    max.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    max.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    max.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    max.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    max.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmax_float(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmax_float_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmax_float_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    max.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    max.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    max.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    max.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
+  %res = call float @llvm.vector.reduce.fmax(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmax_float_reassoc(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmax_float_reassoc(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmax_float_reassoc_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmax_float_reassoc_param_0];
+; CHECK-SM80-NEXT:    max.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    max.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    max.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    max.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    max.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    max.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    max.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmax_float_reassoc(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmax_float_reassoc_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmax_float_reassoc_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    max.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    max.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    max.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    max.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
   %res = call reassoc float @llvm.vector.reduce.fmax(<8 x float> %in)
   ret float %res
 }
 
 define float @reduce_fmax_float_reassoc_nonpow2(<7 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmax_float_reassoc_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<14>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.b32 %r7, [reduce_fmax_float_reassoc_nonpow2_param_0+24];
+; CHECK-SM80-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmax_float_reassoc_nonpow2_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmax_float_reassoc_nonpow2_param_0];
+; CHECK-SM80-NEXT:    max.f32 %r8, %r5, %r6;
+; CHECK-SM80-NEXT:    max.f32 %r9, %r8, %r7;
+; CHECK-SM80-NEXT:    max.f32 %r10, %r3, %r4;
+; CHECK-SM80-NEXT:    max.f32 %r11, %r1, %r2;
+; CHECK-SM80-NEXT:    max.f32 %r12, %r11, %r10;
+; CHECK-SM80-NEXT:    max.f32 %r13, %r12, %r9;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-SM80-NEXT:    ret;
 ;
-; CHECK-LABEL: reduce_fmax_float_reassoc_nonpow2(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<14>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r7, [reduce_fmax_float_reassoc_nonpow2_param_0+24];
-; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmax_float_reassoc_nonpow2_param_0+16];
-; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmax_float_reassoc_nonpow2_param_0];
-; CHECK-NEXT:    max.f32 %r8, %r3, %r7;
-; CHECK-NEXT:    max.f32 %r9, %r1, %r5;
-; CHECK-NEXT:    max.f32 %r10, %r9, %r8;
-; CHECK-NEXT:    max.f32 %r11, %r2, %r6;
-; CHECK-NEXT:    max.f32 %r12, %r11, %r4;
-; CHECK-NEXT:    max.f32 %r13, %r10, %r12;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
-; CHECK-NEXT:    ret;
+; CHECK-SM100-LABEL: reduce_fmax_float_reassoc_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.b32 %r7, [reduce_fmax_float_reassoc_nonpow2_param_0+24];
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmax_float_reassoc_nonpow2_param_0+16];
+; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmax_float_reassoc_nonpow2_param_0];
+; CHECK-SM100-NEXT:    max.f32 %r8, %r4, %r5, %r6;
+; CHECK-SM100-NEXT:    max.f32 %r9, %r1, %r2, %r3;
+; CHECK-SM100-NEXT:    max.f32 %r10, %r9, %r8, %r7;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
   %res = call reassoc float @llvm.vector.reduce.fmax(<7 x float> %in)
   ret float %res
 }
 
-; Check straight line reduction.
+define float @reduce_fmax_float_nnan(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmax_float_nnan(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmax_float_nnan_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmax_float_nnan_param_0];
+; CHECK-SM80-NEXT:    max.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    max.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    max.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    max.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    max.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    max.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    max.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmax_float_nnan(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmax_float_nnan_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmax_float_nnan_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    max.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    max.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    max.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    max.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
+  %res = call nnan float @llvm.vector.reduce.fmax(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmax_float_nnan_nonpow2(<7 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmax_float_nnan_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<14>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.b32 %r7, [reduce_fmax_float_nnan_nonpow2_param_0+24];
+; CHECK-SM80-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmax_float_nnan_nonpow2_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmax_float_nnan_nonpow2_param_0];
+; CHECK-SM80-NEXT:    max.f32 %r8, %r5, %r6;
+; CHECK-SM80-NEXT:    max.f32 %r9, %r8, %r7;
+; CHECK-SM80-NEXT:    max.f32 %r10, %r3, %r4;
+; CHECK-SM80-NEXT:    max.f32 %r11, %r1, %r2;
+; CHECK-SM80-NEXT:    max.f32 %r12, %r11, %r10;
+; CHECK-SM80-NEXT:    max.f32 %r13, %r12, %r9;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmax_float_nnan_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.b32 %r7, [reduce_fmax_float_nnan_nonpow2_param_0+24];
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmax_float_nnan_nonpow2_param_0+16];
+; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmax_float_nnan_nonpow2_param_0];
+; CHECK-SM100-NEXT:    max.f32 %r8, %r4, %r5, %r6;
+; CHECK-SM100-NEXT:    max.f32 %r9, %r1, %r2, %r3;
+; CHECK-SM100-NEXT:    max.f32 %r10, %r9, %r8, %r7;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call nnan float @llvm.vector.reduce.fmax(<7 x float> %in)
+  ret float %res
+}
+
 define half @reduce_fmin_half(<8 x half> %in) {
 ; CHECK-LABEL: reduce_fmin_half(
 ; CHECK:       {
@@ -643,84 +770,241 @@ define half @reduce_fmin_half_reassoc_nonpow2(<7 x half> %in) {
   ret half %res
 }
 
-; Check straight-line reduction.
-define float @reduce_fmin_float(<8 x float> %in) {
-;
-; CHECK-LABEL: reduce_fmin_float(
+define half @reduce_fmin_half_nnan(<8 x half> %in) {
+; CHECK-LABEL: reduce_fmin_half_nnan(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmin_float_param_0+16];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmin_float_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    min.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-NEXT:    min.f32 %r10, %r9, %r7;
-; CHECK-NEXT:    min.f32 %r11, %r10, %r5;
-; CHECK-NEXT:    min.f32 %r12, %r3, %r1;
-; CHECK-NEXT:    min.f32 %r13, %r8, %r6;
-; CHECK-NEXT:    min.f32 %r14, %r13, %r12;
-; CHECK-NEXT:    min.f32 %r15, %r14, %r11;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmin_half_nnan_param_0];
+; CHECK-NEXT:    min.f16x2 %r5, %r2, %r4;
+; CHECK-NEXT:    min.f16x2 %r6, %r1, %r3;
+; CHECK-NEXT:    min.f16x2 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    min.f16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
 ; CHECK-NEXT:    ret;
-  %res = call float @llvm.vector.reduce.fmin(<8 x float> %in)
-  ret float %res
+  %res = call nnan half @llvm.vector.reduce.fmin(<8 x half> %in)
+  ret half %res
 }
 
-define float @reduce_fmin_float_reassoc(<8 x float> %in) {
-;
-; CHECK-LABEL: reduce_fmin_float_reassoc(
+define half @reduce_fmin_half_nnan_nonpow2(<7 x half> %in) {
+; CHECK-LABEL: reduce_fmin_half_nnan_nonpow2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b16 %rs<12>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmin_float_reassoc_param_0+16];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmin_float_reassoc_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    min.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-NEXT:    min.f32 %r10, %r9, %r7;
-; CHECK-NEXT:    min.f32 %r11, %r10, %r5;
-; CHECK-NEXT:    min.f32 %r12, %r3, %r1;
-; CHECK-NEXT:    min.f32 %r13, %r8, %r6;
-; CHECK-NEXT:    min.f32 %r14, %r13, %r12;
-; CHECK-NEXT:    min.f32 %r15, %r14, %r11;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-NEXT:    ld.param.b32 %r1, [reduce_fmin_half_nnan_nonpow2_param_0+8];
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    ld.param.v2.b32 {%r2, %r3}, [reduce_fmin_half_nnan_nonpow2_param_0];
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NEXT:    ld.param.b16 %rs7, [reduce_fmin_half_nnan_nonpow2_param_0+12];
+; CHECK-NEXT:    min.f16x2 %r4, %r2, %r1;
+; CHECK-NEXT:    mov.b16 %rs8, 0x7C00;
+; CHECK-NEXT:    mov.b32 %r5, {%rs7, %rs8};
+; CHECK-NEXT:    min.f16x2 %r6, %r3, %r5;
+; CHECK-NEXT:    min.f16x2 %r7, %r4, %r6;
+; CHECK-NEXT:    mov.b32 {%rs9, %rs10}, %r7;
+; CHECK-NEXT:    min.f16 %rs11, %rs9, %rs10;
+; CHECK-NEXT:    st.param.b16 [func_retval0], %rs11;
 ; CHECK-NEXT:    ret;
+  %res = call nnan half @llvm.vector.reduce.fmin(<7 x half> %in)
+  ret half %res
+}
+
+define float @reduce_fmin_float(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmin_float(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmin_float_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmin_float_param_0];
+; CHECK-SM80-NEXT:    min.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    min.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    min.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    min.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    min.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    min.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    min.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmin_float(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmin_float_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmin_float_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    min.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    min.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    min.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    min.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
+  %res = call float @llvm.vector.reduce.fmin(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmin_float_reassoc(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmin_float_reassoc(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmin_float_reassoc_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmin_float_reassoc_param_0];
+; CHECK-SM80-NEXT:    min.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    min.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    min.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    min.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    min.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    min.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    min.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmin_float_reassoc(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmin_float_reassoc_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmin_float_reassoc_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    min.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    min.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    min.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    min.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
   %res = call reassoc float @llvm.vector.reduce.fmin(<8 x float> %in)
   ret float %res
 }
 
 define float @reduce_fmin_float_reassoc_nonpow2(<7 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmin_float_reassoc_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<14>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.b32 %r7, [reduce_fmin_float_reassoc_nonpow2_param_0+24];
+; CHECK-SM80-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmin_float_reassoc_nonpow2_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmin_float_reassoc_nonpow2_param_0];
+; CHECK-SM80-NEXT:    min.f32 %r8, %r5, %r6;
+; CHECK-SM80-NEXT:    min.f32 %r9, %r8, %r7;
+; CHECK-SM80-NEXT:    min.f32 %r10, %r3, %r4;
+; CHECK-SM80-NEXT:    min.f32 %r11, %r1, %r2;
+; CHECK-SM80-NEXT:    min.f32 %r12, %r11, %r10;
+; CHECK-SM80-NEXT:    min.f32 %r13, %r12, %r9;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-SM80-NEXT:    ret;
 ;
-; CHECK-LABEL: reduce_fmin_float_reassoc_nonpow2(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<14>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r7, [reduce_fmin_float_reassoc_nonpow2_param_0+24];
-; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmin_float_reassoc_nonpow2_param_0+16];
-; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmin_float_reassoc_nonpow2_param_0];
-; CHECK-NEXT:    min.f32 %r8, %r3, %r7;
-; CHECK-NEXT:    min.f32 %r9, %r1, %r5;
-; CHECK-NEXT:    min.f32 %r10, %r9, %r8;
-; CHECK-NEXT:    min.f32 %r11, %r2, %r6;
-; CHECK-NEXT:    min.f32 %r12, %r11, %r4;
-; CHECK-NEXT:    min.f32 %r13, %r10, %r12;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
-; CHECK-NEXT:    ret;
+; CHECK-SM100-LABEL: reduce_fmin_float_reassoc_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.b32 %r7, [reduce_fmin_float_reassoc_nonpow2_param_0+24];
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmin_float_reassoc_nonpow2_param_0+16];
+; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmin_float_reassoc_nonpow2_param_0];
+; CHECK-SM100-NEXT:    min.f32 %r8, %r4, %r5, %r6;
+; CHECK-SM100-NEXT:    min.f32 %r9, %r1, %r2, %r3;
+; CHECK-SM100-NEXT:    min.f32 %r10, %r9, %r8, %r7;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
   %res = call reassoc float @llvm.vector.reduce.fmin(<7 x float> %in)
   ret float %res
 }
 
-; Check straight-line reduction.
+define float @reduce_fmin_float_nnan(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmin_float_nnan(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmin_float_nnan_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmin_float_nnan_param_0];
+; CHECK-SM80-NEXT:    min.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    min.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    min.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    min.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    min.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    min.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    min.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmin_float_nnan(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmin_float_nnan_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmin_float_nnan_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    min.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    min.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    min.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    min.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
+  %res = call nnan float @llvm.vector.reduce.fmin(<8 x float> %in)
+  ret float %res
+}
+
+define float @reduce_fmin_float_nnan_nonpow2(<7 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmin_float_nnan_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<14>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.b32 %r7, [reduce_fmin_float_nnan_nonpow2_param_0+24];
+; CHECK-SM80-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmin_float_nnan_nonpow2_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmin_float_nnan_nonpow2_param_0];
+; CHECK-SM80-NEXT:    min.f32 %r8, %r5, %r6;
+; CHECK-SM80-NEXT:    min.f32 %r9, %r8, %r7;
+; CHECK-SM80-NEXT:    min.f32 %r10, %r3, %r4;
+; CHECK-SM80-NEXT:    min.f32 %r11, %r1, %r2;
+; CHECK-SM80-NEXT:    min.f32 %r12, %r11, %r10;
+; CHECK-SM80-NEXT:    min.f32 %r13, %r12, %r9;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-SM80-NEXT:    ret;
+;
+; CHECK-SM100-LABEL: reduce_fmin_float_nnan_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.b32 %r7, [reduce_fmin_float_nnan_nonpow2_param_0+24];
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmin_float_nnan_nonpow2_param_0+16];
+; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmin_float_nnan_nonpow2_param_0];
+; CHECK-SM100-NEXT:    min.f32 %r8, %r4, %r5, %r6;
+; CHECK-SM100-NEXT:    min.f32 %r9, %r1, %r2, %r3;
+; CHECK-SM100-NEXT:    min.f32 %r10, %r9, %r8, %r7;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
+  %res = call nnan float @llvm.vector.reduce.fmin(<7 x float> %in)
+  ret float %res
+}
+
 define half @reduce_fmaximum_half(<8 x half> %in) {
 ; CHECK-LABEL: reduce_fmaximum_half(
 ; CHECK:       {
@@ -785,84 +1069,121 @@ define half @reduce_fmaximum_half_reassoc_nonpow2(<7 x half> %in) {
   ret half %res
 }
 
-; Check straight-line reduction.
 define float @reduce_fmaximum_float(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmaximum_float(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmaximum_float_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmaximum_float_param_0];
+; CHECK-SM80-NEXT:    max.NaN.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
 ;
-; CHECK-LABEL: reduce_fmaximum_float(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmaximum_float_param_0+16];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmaximum_float_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    max.NaN.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-NEXT:    max.NaN.f32 %r10, %r9, %r7;
-; CHECK-NEXT:    max.NaN.f32 %r11, %r10, %r5;
-; CHECK-NEXT:    max.NaN.f32 %r12, %r3, %r1;
-; CHECK-NEXT:    max.NaN.f32 %r13, %r8, %r6;
-; CHECK-NEXT:    max.NaN.f32 %r14, %r13, %r12;
-; CHECK-NEXT:    max.NaN.f32 %r15, %r14, %r11;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
-; CHECK-NEXT:    ret;
+; CHECK-SM100-LABEL: reduce_fmaximum_float(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmaximum_float_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmaximum_float_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
   %res = call float @llvm.vector.reduce.fmaximum(<8 x float> %in)
   ret float %res
 }
 
 define float @reduce_fmaximum_float_reassoc(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmaximum_float_reassoc(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fmaximum_float_reassoc_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmaximum_float_reassoc_param_0];
+; CHECK-SM80-NEXT:    max.NaN.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
 ;
-; CHECK-LABEL: reduce_fmaximum_float_reassoc(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmaximum_float_reassoc_param_0+16];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmaximum_float_reassoc_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    max.NaN.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-NEXT:    max.NaN.f32 %r10, %r9, %r7;
-; CHECK-NEXT:    max.NaN.f32 %r11, %r10, %r5;
-; CHECK-NEXT:    max.NaN.f32 %r12, %r3, %r1;
-; CHECK-NEXT:    max.NaN.f32 %r13, %r8, %r6;
-; CHECK-NEXT:    max.NaN.f32 %r14, %r13, %r12;
-; CHECK-NEXT:    max.NaN.f32 %r15, %r14, %r11;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
-; CHECK-NEXT:    ret;
+; CHECK-SM100-LABEL: reduce_fmaximum_float_reassoc(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fmaximum_float_reassoc_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fmaximum_float_reassoc_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
   %res = call reassoc float @llvm.vector.reduce.fmaximum(<8 x float> %in)
   ret float %res
 }
 
 define float @reduce_fmaximum_float_reassoc_nonpow2(<7 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fmaximum_float_reassoc_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<14>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.b32 %r7, [reduce_fmaximum_float_reassoc_nonpow2_param_0+24];
+; CHECK-SM80-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmaximum_float_reassoc_nonpow2_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmaximum_float_reassoc_nonpow2_param_0];
+; CHECK-SM80-NEXT:    max.NaN.f32 %r8, %r5, %r6;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r9, %r8, %r7;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r10, %r3, %r4;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r11, %r1, %r2;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r12, %r11, %r10;
+; CHECK-SM80-NEXT:    max.NaN.f32 %r13, %r12, %r9;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-SM80-NEXT:    ret;
 ;
-; CHECK-LABEL: reduce_fmaximum_float_reassoc_nonpow2(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<14>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r7, [reduce_fmaximum_float_reassoc_nonpow2_param_0+24];
-; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmaximum_float_reassoc_nonpow2_param_0+16];
-; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmaximum_float_reassoc_nonpow2_param_0];
-; CHECK-NEXT:    max.NaN.f32 %r8, %r3, %r7;
-; CHECK-NEXT:    max.NaN.f32 %r9, %r1, %r5;
-; CHECK-NEXT:    max.NaN.f32 %r10, %r9, %r8;
-; CHECK-NEXT:    max.NaN.f32 %r11, %r2, %r6;
-; CHECK-NEXT:    max.NaN.f32 %r12, %r11, %r4;
-; CHECK-NEXT:    max.NaN.f32 %r13, %r10, %r12;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
-; CHECK-NEXT:    ret;
+; CHECK-SM100-LABEL: reduce_fmaximum_float_reassoc_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.b32 %r7, [reduce_fmaximum_float_reassoc_nonpow2_param_0+24];
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fmaximum_float_reassoc_nonpow2_param_0+16];
+; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fmaximum_float_reassoc_nonpow2_param_0];
+; CHECK-SM100-NEXT:    max.NaN.f32 %r8, %r4, %r5, %r6;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r9, %r1, %r2, %r3;
+; CHECK-SM100-NEXT:    max.NaN.f32 %r10, %r9, %r8, %r7;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
   %res = call reassoc float @llvm.vector.reduce.fmaximum(<7 x float> %in)
   ret float %res
 }
 
-; Check straight-line reduction.
 define half @reduce_fminimum_half(<8 x half> %in) {
 ; CHECK-LABEL: reduce_fminimum_half(
 ; CHECK:       {
@@ -927,79 +1248,117 @@ define half @reduce_fminimum_half_reassoc_nonpow2(<7 x half> %in) {
   ret half %res
 }
 
-; Check straight-line reduction.
 define float @reduce_fminimum_float(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fminimum_float(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fminimum_float_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fminimum_float_param_0];
+; CHECK-SM80-NEXT:    min.NaN.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
 ;
-; CHECK-LABEL: reduce_fminimum_float(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fminimum_float_param_0+16];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fminimum_float_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    min.NaN.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-NEXT:    min.NaN.f32 %r10, %r9, %r7;
-; CHECK-NEXT:    min.NaN.f32 %r11, %r10, %r5;
-; CHECK-NEXT:    min.NaN.f32 %r12, %r3, %r1;
-; CHECK-NEXT:    min.NaN.f32 %r13, %r8, %r6;
-; CHECK-NEXT:    min.NaN.f32 %r14, %r13, %r12;
-; CHECK-NEXT:    min.NaN.f32 %r15, %r14, %r11;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
-; CHECK-NEXT:    ret;
+; CHECK-SM100-LABEL: reduce_fminimum_float(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fminimum_float_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fminimum_float_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
   %res = call float @llvm.vector.reduce.fminimum(<8 x float> %in)
   ret float %res
 }
 
 define float @reduce_fminimum_float_reassoc(<8 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fminimum_float_reassoc(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<16>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_fminimum_float_reassoc_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fminimum_float_reassoc_param_0];
+; CHECK-SM80-NEXT:    min.NaN.f32 %r9, %r7, %r8;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r10, %r5, %r6;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r11, %r10, %r9;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r12, %r3, %r4;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r13, %r1, %r2;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r14, %r13, %r12;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r15, %r14, %r11;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r15;
+; CHECK-SM80-NEXT:    ret;
 ;
-; CHECK-LABEL: reduce_fminimum_float_reassoc(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<16>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fminimum_float_reassoc_param_0+16];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fminimum_float_reassoc_param_0];
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd4;
-; CHECK-NEXT:    mov.b64 {%r3, %r4}, %rd2;
-; CHECK-NEXT:    min.NaN.f32 %r5, %r4, %r2;
-; CHECK-NEXT:    mov.b64 {%r6, %r7}, %rd3;
-; CHECK-NEXT:    mov.b64 {%r8, %r9}, %rd1;
-; CHECK-NEXT:    min.NaN.f32 %r10, %r9, %r7;
-; CHECK-NEXT:    min.NaN.f32 %r11, %r10, %r5;
-; CHECK-NEXT:    min.NaN.f32 %r12, %r3, %r1;
-; CHECK-NEXT:    min.NaN.f32 %r13, %r8, %r6;
-; CHECK-NEXT:    min.NaN.f32 %r14, %r13, %r12;
-; CHECK-NEXT:    min.NaN.f32 %r15, %r14, %r11;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
-; CHECK-NEXT:    ret;
+; CHECK-SM100-LABEL: reduce_fminimum_float_reassoc(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<13>;
+; CHECK-SM100-NEXT:    .reg .b64 %rd<5>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [reduce_fminimum_float_reassoc_param_0+16];
+; CHECK-SM100-NEXT:    mov.b64 {%r1, %r2}, %rd4;
+; CHECK-SM100-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [reduce_fminimum_float_reassoc_param_0];
+; CHECK-SM100-NEXT:    mov.b64 {%r3, %r4}, %rd1;
+; CHECK-SM100-NEXT:    mov.b64 {%r5, %r6}, %rd3;
+; CHECK-SM100-NEXT:    mov.b64 {%r7, %r8}, %rd2;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r9, %r8, %r5, %r6;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r10, %r3, %r4, %r7;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r11, %r10, %r9, %r1;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r12, %r11, %r2;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r12;
+; CHECK-SM100-NEXT:    ret;
   %res = call reassoc float @llvm.vector.reduce.fminimum(<8 x float> %in)
   ret float %res
 }
 
 define float @reduce_fminimum_float_reassoc_nonpow2(<7 x float> %in) {
+; CHECK-SM80-LABEL: reduce_fminimum_float_reassoc_nonpow2(
+; CHECK-SM80:       {
+; CHECK-SM80-NEXT:    .reg .b32 %r<14>;
+; CHECK-SM80-EMPTY:
+; CHECK-SM80-NEXT:  // %bb.0:
+; CHECK-SM80-NEXT:    ld.param.b32 %r7, [reduce_fminimum_float_reassoc_nonpow2_param_0+24];
+; CHECK-SM80-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fminimum_float_reassoc_nonpow2_param_0+16];
+; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fminimum_float_reassoc_nonpow2_param_0];
+; CHECK-SM80-NEXT:    min.NaN.f32 %r8, %r5, %r6;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r9, %r8, %r7;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r10, %r3, %r4;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r11, %r1, %r2;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r12, %r11, %r10;
+; CHECK-SM80-NEXT:    min.NaN.f32 %r13, %r12, %r9;
+; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r13;
+; CHECK-SM80-NEXT:    ret;
 ;
-; CHECK-LABEL: reduce_fminimum_float_reassoc_nonpow2(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<14>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b32 %r7, [reduce_fminimum_float_reassoc_nonpow2_param_0+24];
-; CHECK-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fminimum_float_reassoc_nonpow2_param_0+16];
-; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fminimum_float_reassoc_nonpow2_param_0];
-; CHECK-NEXT:    min.NaN.f32 %r8, %r3, %r7;
-; CHECK-NEXT:    min.NaN.f32 %r9, %r1, %r5;
-; CHECK-NEXT:    min.NaN.f32 %r10, %r9, %r8;
-; CHECK-NEXT:    min.NaN.f32 %r11, %r2, %r6;
-; CHECK-NEXT:    min.NaN.f32 %r12, %r11, %r4;
-; CHECK-NEXT:    min.NaN.f32 %r13, %r10, %r12;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r13;
-; CHECK-NEXT:    ret;
+; CHECK-SM100-LABEL: reduce_fminimum_float_reassoc_nonpow2(
+; CHECK-SM100:       {
+; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-EMPTY:
+; CHECK-SM100-NEXT:  // %bb.0:
+; CHECK-SM100-NEXT:    ld.param.b32 %r7, [reduce_fminimum_float_reassoc_nonpow2_param_0+24];
+; CHECK-SM100-NEXT:    ld.param.v2.b32 {%r5, %r6}, [reduce_fminimum_float_reassoc_nonpow2_param_0+16];
+; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_fminimum_float_reassoc_nonpow2_param_0];
+; CHECK-SM100-NEXT:    min.NaN.f32 %r8, %r4, %r5, %r6;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r9, %r1, %r2, %r3;
+; CHECK-SM100-NEXT:    min.NaN.f32 %r10, %r9, %r8, %r7;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    ret;
   %res = call reassoc float @llvm.vector.reduce.fminimum(<7 x float> %in)
   ret float %res
 }
@@ -1014,15 +1373,15 @@ define i16 @reduce_add_i16(<8 x i16> %in) {
 ; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_add_i16_param_0];
 ; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM80-NEXT:    add.s16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    add.s16 %rs5, %rs4, %rs2;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
-; CHECK-SM80-NEXT:    add.s16 %rs10, %rs8, %rs6;
-; CHECK-SM80-NEXT:    add.s16 %rs11, %rs4, %rs2;
-; CHECK-SM80-NEXT:    add.s16 %rs12, %rs9, %rs7;
-; CHECK-SM80-NEXT:    add.s16 %rs13, %rs12, %rs11;
-; CHECK-SM80-NEXT:    add.s16 %rs14, %rs10, %rs5;
-; CHECK-SM80-NEXT:    add.s16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    add.s16 %rs10, %rs9, %rs7;
+; CHECK-SM80-NEXT:    add.s16 %rs11, %rs10, %rs5;
+; CHECK-SM80-NEXT:    add.s16 %rs12, %rs3, %rs1;
+; CHECK-SM80-NEXT:    add.s16 %rs13, %rs8, %rs6;
+; CHECK-SM80-NEXT:    add.s16 %rs14, %rs13, %rs12;
+; CHECK-SM80-NEXT:    add.s16 %rs15, %rs14, %rs11;
 ; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
 ; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-SM80-NEXT:    ret;
@@ -1030,20 +1389,17 @@ define i16 @reduce_add_i16(<8 x i16> %in) {
 ; CHECK-SM100-LABEL: reduce_add_i16(
 ; CHECK-SM100:       {
 ; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
 ; CHECK-SM100-EMPTY:
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_add_i16_param_0];
 ; CHECK-SM100-NEXT:    add.s16x2 %r5, %r2, %r4;
 ; CHECK-SM100-NEXT:    add.s16x2 %r6, %r1, %r3;
 ; CHECK-SM100-NEXT:    add.s16x2 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    add.s16x2 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-SM100-NEXT:    add.s16 %rs3, %rs1, %rs2;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-SM100-NEXT:    ret;
   %res = call i16 @llvm.vector.reduce.add(<8 x i16> %in)
   ret i16 %res
@@ -1103,13 +1459,13 @@ define i32 @reduce_add_i32(<8 x i32> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_add_i32_param_0+16];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_add_i32_param_0];
-; CHECK-NEXT:    add.s32 %r9, %r3, %r7;
-; CHECK-NEXT:    add.s32 %r10, %r1, %r5;
-; CHECK-NEXT:    add.s32 %r11, %r4, %r8;
-; CHECK-NEXT:    add.s32 %r12, %r2, %r6;
-; CHECK-NEXT:    add.s32 %r13, %r12, %r11;
-; CHECK-NEXT:    add.s32 %r14, %r10, %r9;
-; CHECK-NEXT:    add.s32 %r15, %r14, %r13;
+; CHECK-NEXT:    add.s32 %r9, %r4, %r8;
+; CHECK-NEXT:    add.s32 %r10, %r2, %r6;
+; CHECK-NEXT:    add.s32 %r11, %r10, %r9;
+; CHECK-NEXT:    add.s32 %r12, %r3, %r7;
+; CHECK-NEXT:    add.s32 %r13, %r1, %r5;
+; CHECK-NEXT:    add.s32 %r14, %r13, %r12;
+; CHECK-NEXT:    add.s32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-NEXT:    ret;
   %res = call i32 @llvm.vector.reduce.add(<8 x i32> %in)
@@ -1147,15 +1503,15 @@ define i16 @reduce_mul_i16(<8 x i16> %in) {
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_mul_i16_param_0];
 ; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-NEXT:    mul.lo.s16 %rs5, %rs3, %rs1;
+; CHECK-NEXT:    mul.lo.s16 %rs5, %rs4, %rs2;
 ; CHECK-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
 ; CHECK-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
-; CHECK-NEXT:    mul.lo.s16 %rs10, %rs8, %rs6;
-; CHECK-NEXT:    mul.lo.s16 %rs11, %rs4, %rs2;
-; CHECK-NEXT:    mul.lo.s16 %rs12, %rs9, %rs7;
-; CHECK-NEXT:    mul.lo.s16 %rs13, %rs12, %rs11;
-; CHECK-NEXT:    mul.lo.s16 %rs14, %rs10, %rs5;
-; CHECK-NEXT:    mul.lo.s16 %rs15, %rs14, %rs13;
+; CHECK-NEXT:    mul.lo.s16 %rs10, %rs9, %rs7;
+; CHECK-NEXT:    mul.lo.s16 %rs11, %rs10, %rs5;
+; CHECK-NEXT:    mul.lo.s16 %rs12, %rs3, %rs1;
+; CHECK-NEXT:    mul.lo.s16 %rs13, %rs8, %rs6;
+; CHECK-NEXT:    mul.lo.s16 %rs14, %rs13, %rs12;
+; CHECK-NEXT:    mul.lo.s16 %rs15, %rs14, %rs11;
 ; CHECK-NEXT:    cvt.u32.u16 %r5, %rs15;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-NEXT:    ret;
@@ -1194,13 +1550,13 @@ define i32 @reduce_mul_i32(<8 x i32> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_mul_i32_param_0+16];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_mul_i32_param_0];
-; CHECK-NEXT:    mul.lo.s32 %r9, %r3, %r7;
-; CHECK-NEXT:    mul.lo.s32 %r10, %r1, %r5;
-; CHECK-NEXT:    mul.lo.s32 %r11, %r4, %r8;
-; CHECK-NEXT:    mul.lo.s32 %r12, %r2, %r6;
-; CHECK-NEXT:    mul.lo.s32 %r13, %r12, %r11;
-; CHECK-NEXT:    mul.lo.s32 %r14, %r10, %r9;
-; CHECK-NEXT:    mul.lo.s32 %r15, %r14, %r13;
+; CHECK-NEXT:    mul.lo.s32 %r9, %r4, %r8;
+; CHECK-NEXT:    mul.lo.s32 %r10, %r2, %r6;
+; CHECK-NEXT:    mul.lo.s32 %r11, %r10, %r9;
+; CHECK-NEXT:    mul.lo.s32 %r12, %r3, %r7;
+; CHECK-NEXT:    mul.lo.s32 %r13, %r1, %r5;
+; CHECK-NEXT:    mul.lo.s32 %r14, %r13, %r12;
+; CHECK-NEXT:    mul.lo.s32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-NEXT:    ret;
   %res = call i32 @llvm.vector.reduce.mul(<8 x i32> %in)
@@ -1238,15 +1594,15 @@ define i16 @reduce_umax_i16(<8 x i16> %in) {
 ; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_umax_i16_param_0];
 ; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM80-NEXT:    max.u16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    max.u16 %rs5, %rs4, %rs2;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
-; CHECK-SM80-NEXT:    max.u16 %rs10, %rs8, %rs6;
-; CHECK-SM80-NEXT:    max.u16 %rs11, %rs4, %rs2;
-; CHECK-SM80-NEXT:    max.u16 %rs12, %rs9, %rs7;
-; CHECK-SM80-NEXT:    max.u16 %rs13, %rs12, %rs11;
-; CHECK-SM80-NEXT:    max.u16 %rs14, %rs10, %rs5;
-; CHECK-SM80-NEXT:    max.u16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    max.u16 %rs10, %rs9, %rs7;
+; CHECK-SM80-NEXT:    max.u16 %rs11, %rs10, %rs5;
+; CHECK-SM80-NEXT:    max.u16 %rs12, %rs3, %rs1;
+; CHECK-SM80-NEXT:    max.u16 %rs13, %rs8, %rs6;
+; CHECK-SM80-NEXT:    max.u16 %rs14, %rs13, %rs12;
+; CHECK-SM80-NEXT:    max.u16 %rs15, %rs14, %rs11;
 ; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
 ; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-SM80-NEXT:    ret;
@@ -1254,20 +1610,17 @@ define i16 @reduce_umax_i16(<8 x i16> %in) {
 ; CHECK-SM100-LABEL: reduce_umax_i16(
 ; CHECK-SM100:       {
 ; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
 ; CHECK-SM100-EMPTY:
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_umax_i16_param_0];
 ; CHECK-SM100-NEXT:    max.u16x2 %r5, %r2, %r4;
 ; CHECK-SM100-NEXT:    max.u16x2 %r6, %r1, %r3;
 ; CHECK-SM100-NEXT:    max.u16x2 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    max.u16x2 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-SM100-NEXT:    max.u16 %rs3, %rs1, %rs2;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-SM100-NEXT:    ret;
   %res = call i16 @llvm.vector.reduce.umax(<8 x i16> %in)
   ret i16 %res
@@ -1327,13 +1680,13 @@ define i32 @reduce_umax_i32(<8 x i32> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_umax_i32_param_0+16];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_umax_i32_param_0];
-; CHECK-NEXT:    max.u32 %r9, %r3, %r7;
-; CHECK-NEXT:    max.u32 %r10, %r1, %r5;
-; CHECK-NEXT:    max.u32 %r11, %r4, %r8;
-; CHECK-NEXT:    max.u32 %r12, %r2, %r6;
-; CHECK-NEXT:    max.u32 %r13, %r12, %r11;
-; CHECK-NEXT:    max.u32 %r14, %r10, %r9;
-; CHECK-NEXT:    max.u32 %r15, %r14, %r13;
+; CHECK-NEXT:    max.u32 %r9, %r4, %r8;
+; CHECK-NEXT:    max.u32 %r10, %r2, %r6;
+; CHECK-NEXT:    max.u32 %r11, %r10, %r9;
+; CHECK-NEXT:    max.u32 %r12, %r3, %r7;
+; CHECK-NEXT:    max.u32 %r13, %r1, %r5;
+; CHECK-NEXT:    max.u32 %r14, %r13, %r12;
+; CHECK-NEXT:    max.u32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-NEXT:    ret;
   %res = call i32 @llvm.vector.reduce.umax(<8 x i32> %in)
@@ -1371,15 +1724,15 @@ define i16 @reduce_umin_i16(<8 x i16> %in) {
 ; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_umin_i16_param_0];
 ; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM80-NEXT:    min.u16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    min.u16 %rs5, %rs4, %rs2;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
-; CHECK-SM80-NEXT:    min.u16 %rs10, %rs8, %rs6;
-; CHECK-SM80-NEXT:    min.u16 %rs11, %rs4, %rs2;
-; CHECK-SM80-NEXT:    min.u16 %rs12, %rs9, %rs7;
-; CHECK-SM80-NEXT:    min.u16 %rs13, %rs12, %rs11;
-; CHECK-SM80-NEXT:    min.u16 %rs14, %rs10, %rs5;
-; CHECK-SM80-NEXT:    min.u16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    min.u16 %rs10, %rs9, %rs7;
+; CHECK-SM80-NEXT:    min.u16 %rs11, %rs10, %rs5;
+; CHECK-SM80-NEXT:    min.u16 %rs12, %rs3, %rs1;
+; CHECK-SM80-NEXT:    min.u16 %rs13, %rs8, %rs6;
+; CHECK-SM80-NEXT:    min.u16 %rs14, %rs13, %rs12;
+; CHECK-SM80-NEXT:    min.u16 %rs15, %rs14, %rs11;
 ; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
 ; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-SM80-NEXT:    ret;
@@ -1387,20 +1740,17 @@ define i16 @reduce_umin_i16(<8 x i16> %in) {
 ; CHECK-SM100-LABEL: reduce_umin_i16(
 ; CHECK-SM100:       {
 ; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
 ; CHECK-SM100-EMPTY:
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_umin_i16_param_0];
 ; CHECK-SM100-NEXT:    min.u16x2 %r5, %r2, %r4;
 ; CHECK-SM100-NEXT:    min.u16x2 %r6, %r1, %r3;
 ; CHECK-SM100-NEXT:    min.u16x2 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    min.u16x2 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-SM100-NEXT:    min.u16 %rs3, %rs1, %rs2;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-SM100-NEXT:    ret;
   %res = call i16 @llvm.vector.reduce.umin(<8 x i16> %in)
   ret i16 %res
@@ -1460,13 +1810,13 @@ define i32 @reduce_umin_i32(<8 x i32> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_umin_i32_param_0+16];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_umin_i32_param_0];
-; CHECK-NEXT:    min.u32 %r9, %r3, %r7;
-; CHECK-NEXT:    min.u32 %r10, %r1, %r5;
-; CHECK-NEXT:    min.u32 %r11, %r4, %r8;
-; CHECK-NEXT:    min.u32 %r12, %r2, %r6;
-; CHECK-NEXT:    min.u32 %r13, %r12, %r11;
-; CHECK-NEXT:    min.u32 %r14, %r10, %r9;
-; CHECK-NEXT:    min.u32 %r15, %r14, %r13;
+; CHECK-NEXT:    min.u32 %r9, %r4, %r8;
+; CHECK-NEXT:    min.u32 %r10, %r2, %r6;
+; CHECK-NEXT:    min.u32 %r11, %r10, %r9;
+; CHECK-NEXT:    min.u32 %r12, %r3, %r7;
+; CHECK-NEXT:    min.u32 %r13, %r1, %r5;
+; CHECK-NEXT:    min.u32 %r14, %r13, %r12;
+; CHECK-NEXT:    min.u32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-NEXT:    ret;
   %res = call i32 @llvm.vector.reduce.umin(<8 x i32> %in)
@@ -1504,15 +1854,15 @@ define i16 @reduce_smax_i16(<8 x i16> %in) {
 ; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_smax_i16_param_0];
 ; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM80-NEXT:    max.s16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    max.s16 %rs5, %rs4, %rs2;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
-; CHECK-SM80-NEXT:    max.s16 %rs10, %rs8, %rs6;
-; CHECK-SM80-NEXT:    max.s16 %rs11, %rs4, %rs2;
-; CHECK-SM80-NEXT:    max.s16 %rs12, %rs9, %rs7;
-; CHECK-SM80-NEXT:    max.s16 %rs13, %rs12, %rs11;
-; CHECK-SM80-NEXT:    max.s16 %rs14, %rs10, %rs5;
-; CHECK-SM80-NEXT:    max.s16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    max.s16 %rs10, %rs9, %rs7;
+; CHECK-SM80-NEXT:    max.s16 %rs11, %rs10, %rs5;
+; CHECK-SM80-NEXT:    max.s16 %rs12, %rs3, %rs1;
+; CHECK-SM80-NEXT:    max.s16 %rs13, %rs8, %rs6;
+; CHECK-SM80-NEXT:    max.s16 %rs14, %rs13, %rs12;
+; CHECK-SM80-NEXT:    max.s16 %rs15, %rs14, %rs11;
 ; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
 ; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-SM80-NEXT:    ret;
@@ -1520,20 +1870,17 @@ define i16 @reduce_smax_i16(<8 x i16> %in) {
 ; CHECK-SM100-LABEL: reduce_smax_i16(
 ; CHECK-SM100:       {
 ; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
 ; CHECK-SM100-EMPTY:
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_smax_i16_param_0];
 ; CHECK-SM100-NEXT:    max.s16x2 %r5, %r2, %r4;
 ; CHECK-SM100-NEXT:    max.s16x2 %r6, %r1, %r3;
 ; CHECK-SM100-NEXT:    max.s16x2 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    max.s16x2 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-SM100-NEXT:    max.s16 %rs3, %rs1, %rs2;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-SM100-NEXT:    ret;
   %res = call i16 @llvm.vector.reduce.smax(<8 x i16> %in)
   ret i16 %res
@@ -1593,13 +1940,13 @@ define i32 @reduce_smax_i32(<8 x i32> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_smax_i32_param_0+16];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_smax_i32_param_0];
-; CHECK-NEXT:    max.s32 %r9, %r3, %r7;
-; CHECK-NEXT:    max.s32 %r10, %r1, %r5;
-; CHECK-NEXT:    max.s32 %r11, %r4, %r8;
-; CHECK-NEXT:    max.s32 %r12, %r2, %r6;
-; CHECK-NEXT:    max.s32 %r13, %r12, %r11;
-; CHECK-NEXT:    max.s32 %r14, %r10, %r9;
-; CHECK-NEXT:    max.s32 %r15, %r14, %r13;
+; CHECK-NEXT:    max.s32 %r9, %r4, %r8;
+; CHECK-NEXT:    max.s32 %r10, %r2, %r6;
+; CHECK-NEXT:    max.s32 %r11, %r10, %r9;
+; CHECK-NEXT:    max.s32 %r12, %r3, %r7;
+; CHECK-NEXT:    max.s32 %r13, %r1, %r5;
+; CHECK-NEXT:    max.s32 %r14, %r13, %r12;
+; CHECK-NEXT:    max.s32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-NEXT:    ret;
   %res = call i32 @llvm.vector.reduce.smax(<8 x i32> %in)
@@ -1637,15 +1984,15 @@ define i16 @reduce_smin_i16(<8 x i16> %in) {
 ; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_smin_i16_param_0];
 ; CHECK-SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r4;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
-; CHECK-SM80-NEXT:    min.s16 %rs5, %rs3, %rs1;
+; CHECK-SM80-NEXT:    min.s16 %rs5, %rs4, %rs2;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs6, %rs7}, %r3;
 ; CHECK-SM80-NEXT:    mov.b32 {%rs8, %rs9}, %r1;
-; CHECK-SM80-NEXT:    min.s16 %rs10, %rs8, %rs6;
-; CHECK-SM80-NEXT:    min.s16 %rs11, %rs4, %rs2;
-; CHECK-SM80-NEXT:    min.s16 %rs12, %rs9, %rs7;
-; CHECK-SM80-NEXT:    min.s16 %rs13, %rs12, %rs11;
-; CHECK-SM80-NEXT:    min.s16 %rs14, %rs10, %rs5;
-; CHECK-SM80-NEXT:    min.s16 %rs15, %rs14, %rs13;
+; CHECK-SM80-NEXT:    min.s16 %rs10, %rs9, %rs7;
+; CHECK-SM80-NEXT:    min.s16 %rs11, %rs10, %rs5;
+; CHECK-SM80-NEXT:    min.s16 %rs12, %rs3, %rs1;
+; CHECK-SM80-NEXT:    min.s16 %rs13, %rs8, %rs6;
+; CHECK-SM80-NEXT:    min.s16 %rs14, %rs13, %rs12;
+; CHECK-SM80-NEXT:    min.s16 %rs15, %rs14, %rs11;
 ; CHECK-SM80-NEXT:    cvt.u32.u16 %r5, %rs15;
 ; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r5;
 ; CHECK-SM80-NEXT:    ret;
@@ -1653,20 +2000,17 @@ define i16 @reduce_smin_i16(<8 x i16> %in) {
 ; CHECK-SM100-LABEL: reduce_smin_i16(
 ; CHECK-SM100:       {
 ; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
+; CHECK-SM100-NEXT:    .reg .b32 %r<9>;
 ; CHECK-SM100-EMPTY:
 ; CHECK-SM100-NEXT:  // %bb.0:
 ; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_smin_i16_param_0];
 ; CHECK-SM100-NEXT:    min.s16x2 %r5, %r2, %r4;
 ; CHECK-SM100-NEXT:    min.s16x2 %r6, %r1, %r3;
 ; CHECK-SM100-NEXT:    min.s16x2 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    min.s16x2 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
+; CHECK-SM100-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-SM100-NEXT:    min.s16 %rs3, %rs1, %rs2;
+; CHECK-SM100-NEXT:    cvt.u32.u16 %r8, %rs3;
+; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-SM100-NEXT:    ret;
   %res = call i16 @llvm.vector.reduce.smin(<8 x i16> %in)
   ret i16 %res
@@ -1726,13 +2070,13 @@ define i32 @reduce_smin_i32(<8 x i32> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_smin_i32_param_0+16];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_smin_i32_param_0];
-; CHECK-NEXT:    min.s32 %r9, %r3, %r7;
-; CHECK-NEXT:    min.s32 %r10, %r1, %r5;
-; CHECK-NEXT:    min.s32 %r11, %r4, %r8;
-; CHECK-NEXT:    min.s32 %r12, %r2, %r6;
-; CHECK-NEXT:    min.s32 %r13, %r12, %r11;
-; CHECK-NEXT:    min.s32 %r14, %r10, %r9;
-; CHECK-NEXT:    min.s32 %r15, %r14, %r13;
+; CHECK-NEXT:    min.s32 %r9, %r4, %r8;
+; CHECK-NEXT:    min.s32 %r10, %r2, %r6;
+; CHECK-NEXT:    min.s32 %r11, %r10, %r9;
+; CHECK-NEXT:    min.s32 %r12, %r3, %r7;
+; CHECK-NEXT:    min.s32 %r13, %r1, %r5;
+; CHECK-NEXT:    min.s32 %r14, %r13, %r12;
+; CHECK-NEXT:    min.s32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-NEXT:    ret;
   %res = call i32 @llvm.vector.reduce.smin(<8 x i32> %in)
@@ -1761,43 +2105,21 @@ define i32 @reduce_smin_i32_nonpow2(<7 x i32> %in) {
 }
 
 define i16 @reduce_and_i16(<8 x i16> %in) {
-; CHECK-SM80-LABEL: reduce_and_i16(
-; CHECK-SM80:       {
-; CHECK-SM80-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<11>;
-; CHECK-SM80-EMPTY:
-; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_and_i16_param_0];
-; CHECK-SM80-NEXT:    and.b32 %r5, %r2, %r4;
-; CHECK-SM80-NEXT:    and.b32 %r6, %r1, %r3;
-; CHECK-SM80-NEXT:    and.b32 %r7, %r6, %r5;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
-; CHECK-SM80-NEXT:    // implicit-def: %rs2
-; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM80-NEXT:    and.b32 %r9, %r7, %r8;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
-; CHECK-SM80-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r10;
-; CHECK-SM80-NEXT:    ret;
-;
-; CHECK-SM100-LABEL: reduce_and_i16(
-; CHECK-SM100:       {
-; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
-; CHECK-SM100-EMPTY:
-; CHECK-SM100-NEXT:  // %bb.0:
-; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_and_i16_param_0];
-; CHECK-SM100-NEXT:    and.b32 %r5, %r2, %r4;
-; CHECK-SM100-NEXT:    and.b32 %r6, %r1, %r3;
-; CHECK-SM100-NEXT:    and.b32 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    and.b32 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
-; CHECK-SM100-NEXT:    ret;
+; CHECK-LABEL: reduce_and_i16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_and_i16_param_0];
+; CHECK-NEXT:    and.b32 %r5, %r2, %r4;
+; CHECK-NEXT:    and.b32 %r6, %r1, %r3;
+; CHECK-NEXT:    and.b32 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    and.b16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    cvt.u32.u16 %r8, %rs3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
   %res = call i16 @llvm.vector.reduce.and(<8 x i16> %in)
   ret i16 %res
 }
@@ -1837,13 +2159,13 @@ define i32 @reduce_and_i32(<8 x i32> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_and_i32_param_0+16];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_and_i32_param_0];
-; CHECK-NEXT:    and.b32 %r9, %r3, %r7;
-; CHECK-NEXT:    and.b32 %r10, %r1, %r5;
-; CHECK-NEXT:    and.b32 %r11, %r4, %r8;
-; CHECK-NEXT:    and.b32 %r12, %r2, %r6;
-; CHECK-NEXT:    and.b32 %r13, %r12, %r11;
-; CHECK-NEXT:    and.b32 %r14, %r10, %r9;
-; CHECK-NEXT:    and.b32 %r15, %r14, %r13;
+; CHECK-NEXT:    and.b32 %r9, %r4, %r8;
+; CHECK-NEXT:    and.b32 %r10, %r2, %r6;
+; CHECK-NEXT:    and.b32 %r11, %r10, %r9;
+; CHECK-NEXT:    and.b32 %r12, %r3, %r7;
+; CHECK-NEXT:    and.b32 %r13, %r1, %r5;
+; CHECK-NEXT:    and.b32 %r14, %r13, %r12;
+; CHECK-NEXT:    and.b32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-NEXT:    ret;
   %res = call i32 @llvm.vector.reduce.and(<8 x i32> %in)
@@ -1872,43 +2194,21 @@ define i32 @reduce_and_i32_nonpow2(<7 x i32> %in) {
 }
 
 define i16 @reduce_or_i16(<8 x i16> %in) {
-; CHECK-SM80-LABEL: reduce_or_i16(
-; CHECK-SM80:       {
-; CHECK-SM80-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<11>;
-; CHECK-SM80-EMPTY:
-; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_or_i16_param_0];
-; CHECK-SM80-NEXT:    or.b32 %r5, %r2, %r4;
-; CHECK-SM80-NEXT:    or.b32 %r6, %r1, %r3;
-; CHECK-SM80-NEXT:    or.b32 %r7, %r6, %r5;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
-; CHECK-SM80-NEXT:    // implicit-def: %rs2
-; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM80-NEXT:    or.b32 %r9, %r7, %r8;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
-; CHECK-SM80-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r10;
-; CHECK-SM80-NEXT:    ret;
-;
-; CHECK-SM100-LABEL: reduce_or_i16(
-; CHECK-SM100:       {
-; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
-; CHECK-SM100-EMPTY:
-; CHECK-SM100-NEXT:  // %bb.0:
-; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_or_i16_param_0];
-; CHECK-SM100-NEXT:    or.b32 %r5, %r2, %r4;
-; CHECK-SM100-NEXT:    or.b32 %r6, %r1, %r3;
-; CHECK-SM100-NEXT:    or.b32 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    or.b32 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
-; CHECK-SM100-NEXT:    ret;
+; CHECK-LABEL: reduce_or_i16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_or_i16_param_0];
+; CHECK-NEXT:    or.b32 %r5, %r2, %r4;
+; CHECK-NEXT:    or.b32 %r6, %r1, %r3;
+; CHECK-NEXT:    or.b32 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    or.b16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    cvt.u32.u16 %r8, %rs3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
   %res = call i16 @llvm.vector.reduce.or(<8 x i16> %in)
   ret i16 %res
 }
@@ -1948,13 +2248,13 @@ define i32 @reduce_or_i32(<8 x i32> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_or_i32_param_0+16];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_or_i32_param_0];
-; CHECK-NEXT:    or.b32 %r9, %r3, %r7;
-; CHECK-NEXT:    or.b32 %r10, %r1, %r5;
-; CHECK-NEXT:    or.b32 %r11, %r4, %r8;
-; CHECK-NEXT:    or.b32 %r12, %r2, %r6;
-; CHECK-NEXT:    or.b32 %r13, %r12, %r11;
-; CHECK-NEXT:    or.b32 %r14, %r10, %r9;
-; CHECK-NEXT:    or.b32 %r15, %r14, %r13;
+; CHECK-NEXT:    or.b32 %r9, %r4, %r8;
+; CHECK-NEXT:    or.b32 %r10, %r2, %r6;
+; CHECK-NEXT:    or.b32 %r11, %r10, %r9;
+; CHECK-NEXT:    or.b32 %r12, %r3, %r7;
+; CHECK-NEXT:    or.b32 %r13, %r1, %r5;
+; CHECK-NEXT:    or.b32 %r14, %r13, %r12;
+; CHECK-NEXT:    or.b32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-NEXT:    ret;
   %res = call i32 @llvm.vector.reduce.or(<8 x i32> %in)
@@ -1983,43 +2283,21 @@ define i32 @reduce_or_i32_nonpow2(<7 x i32> %in) {
 }
 
 define i16 @reduce_xor_i16(<8 x i16> %in) {
-; CHECK-SM80-LABEL: reduce_xor_i16(
-; CHECK-SM80:       {
-; CHECK-SM80-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM80-NEXT:    .reg .b32 %r<11>;
-; CHECK-SM80-EMPTY:
-; CHECK-SM80-NEXT:  // %bb.0:
-; CHECK-SM80-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_xor_i16_param_0];
-; CHECK-SM80-NEXT:    xor.b32 %r5, %r2, %r4;
-; CHECK-SM80-NEXT:    xor.b32 %r6, %r1, %r3;
-; CHECK-SM80-NEXT:    xor.b32 %r7, %r6, %r5;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
-; CHECK-SM80-NEXT:    // implicit-def: %rs2
-; CHECK-SM80-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM80-NEXT:    xor.b32 %r9, %r7, %r8;
-; CHECK-SM80-NEXT:    { .reg .b16 tmp; mov.b32 {%rs3, tmp}, %r9; }
-; CHECK-SM80-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM80-NEXT:    st.param.b32 [func_retval0], %r10;
-; CHECK-SM80-NEXT:    ret;
-;
-; CHECK-SM100-LABEL: reduce_xor_i16(
-; CHECK-SM100:       {
-; CHECK-SM100-NEXT:    .reg .b16 %rs<4>;
-; CHECK-SM100-NEXT:    .reg .b32 %r<11>;
-; CHECK-SM100-EMPTY:
-; CHECK-SM100-NEXT:  // %bb.0:
-; CHECK-SM100-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_xor_i16_param_0];
-; CHECK-SM100-NEXT:    xor.b32 %r5, %r2, %r4;
-; CHECK-SM100-NEXT:    xor.b32 %r6, %r1, %r3;
-; CHECK-SM100-NEXT:    xor.b32 %r7, %r6, %r5;
-; CHECK-SM100-NEXT:    mov.b32 {_, %rs1}, %r7;
-; CHECK-SM100-NEXT:    // implicit-def: %rs2
-; CHECK-SM100-NEXT:    mov.b32 %r8, {%rs1, %rs2};
-; CHECK-SM100-NEXT:    xor.b32 %r9, %r7, %r8;
-; CHECK-SM100-NEXT:    mov.b32 {%rs3, _}, %r9;
-; CHECK-SM100-NEXT:    cvt.u32.u16 %r10, %rs3;
-; CHECK-SM100-NEXT:    st.param.b32 [func_retval0], %r10;
-; CHECK-SM100-NEXT:    ret;
+; CHECK-LABEL: reduce_xor_i16(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_xor_i16_param_0];
+; CHECK-NEXT:    xor.b32 %r5, %r2, %r4;
+; CHECK-NEXT:    xor.b32 %r6, %r1, %r3;
+; CHECK-NEXT:    xor.b32 %r7, %r6, %r5;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r7;
+; CHECK-NEXT:    xor.b16 %rs3, %rs1, %rs2;
+; CHECK-NEXT:    cvt.u32.u16 %r8, %rs3;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
+; CHECK-NEXT:    ret;
   %res = call i16 @llvm.vector.reduce.xor(<8 x i16> %in)
   ret i16 %res
 }
@@ -2059,13 +2337,13 @@ define i32 @reduce_xor_i32(<8 x i32> %in) {
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [reduce_xor_i32_param_0+16];
 ; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [reduce_xor_i32_param_0];
-; CHECK-NEXT:    xor.b32 %r9, %r3, %r7;
-; CHECK-NEXT:    xor.b32 %r10, %r1, %r5;
-; CHECK-NEXT:    xor.b32 %r11, %r4, %r8;
-; CHECK-NEXT:    xor.b32 %r12, %r2, %r6;
-; CHECK-NEXT:    xor.b32 %r13, %r12, %r11;
-; CHECK-NEXT:    xor.b32 %r14, %r10, %r9;
-; CHECK-NEXT:    xor.b32 %r15, %r14, %r13;
+; CHECK-NEXT:    xor.b32 %r9, %r4, %r8;
+; CHECK-NEXT:    xor.b32 %r10, %r2, %r6;
+; CHECK-NEXT:    xor.b32 %r11, %r10, %r9;
+; CHECK-NEXT:    xor.b32 %r12, %r3, %r7;
+; CHECK-NEXT:    xor.b32 %r13, %r1, %r5;
+; CHECK-NEXT:    xor.b32 %r14, %r13, %r12;
+; CHECK-NEXT:    xor.b32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r15;
 ; CHECK-NEXT:    ret;
   %res = call i32 @llvm.vector.reduce.xor(<8 x i32> %in)
diff --git a/llvm/test/CodeGen/NVPTX/shift-opt.ll b/llvm/test/CodeGen/NVPTX/shift-opt.ll
index e7866b01064c7..e0d22c62993ba 100644
--- a/llvm/test/CodeGen/NVPTX/shift-opt.ll
+++ b/llvm/test/CodeGen/NVPTX/shift-opt.ll
@@ -71,18 +71,17 @@ define <2 x i16> @test_vec(<2 x i16> %x, <2 x i8> %y) {
 ; CHECK-LABEL: test_vec(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b16 %rs<7>;
-; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.v2.b16 {%rs1, %rs2}, [test_vec_param_0];
 ; CHECK-NEXT:    ld.param.v2.b8 {%rs3, %rs4}, [test_vec_param_1];
 ; CHECK-NEXT:    mov.b32 %r1, {%rs3, %rs4};
-; CHECK-NEXT:    and.b32 %r2, %r1, 16711935;
 ; CHECK-NEXT:    shr.u16 %rs5, %rs2, 5;
 ; CHECK-NEXT:    shr.u16 %rs6, %rs1, 5;
-; CHECK-NEXT:    mov.b32 %r3, {%rs6, %rs5};
-; CHECK-NEXT:    or.b32 %r4, %r3, %r2;
-; CHECK-NEXT:    st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT:    mov.b32 %r2, {%rs6, %rs5};
+; CHECK-NEXT:    or.b32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param.b32 [func_retval0], %r3;
 ; CHECK-NEXT:    ret;
   %ext = zext <2 x i8> %y to <2 x i16>
   %shl = shl <2 x i16> %ext, splat(i16 5)
diff --git a/llvm/test/CodeGen/NVPTX/sqrt-approx.ll b/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
index 3989c8e32e458..7e4e701af4cd1 100644
--- a/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
+++ b/llvm/test/CodeGen/NVPTX/sqrt-approx.ll
@@ -13,7 +13,7 @@ declare double @llvm.sqrt.f64(double)
 
 ; -- reciprocal sqrt --
 
-define float @test_rsqrt32(float %a) #0 {
+define float @test_rsqrt32(float %a) {
 ; CHECK-LABEL: test_rsqrt32(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -28,7 +28,7 @@ define float @test_rsqrt32(float %a) #0 {
   ret float %ret
 }
 
-define float @test_rsqrt_ftz(float %a) #0 #1 {
+define float @test_rsqrt_ftz(float %a) #1 {
 ; CHECK-LABEL: test_rsqrt_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -76,7 +76,7 @@ define double @test_rsqrt64_ftz(double %a) #1 {
 
 ; -- sqrt --
 
-define float @test_sqrt32(float %a) #0 {
+define float @test_sqrt32(float %a) {
 ; CHECK-LABEL: test_sqrt32(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -90,7 +90,7 @@ define float @test_sqrt32(float %a) #0 {
   ret float %ret
 }
 
-define float @test_sqrt32_ninf(float %a) #0 {
+define float @test_sqrt32_ninf(float %a) {
 ; CHECK-LABEL: test_sqrt32_ninf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -108,7 +108,7 @@ define float @test_sqrt32_ninf(float %a) #0 {
   ret float %ret
 }
 
-define float @test_sqrt_ftz(float %a) #0 #1 {
+define float @test_sqrt_ftz(float %a) #1 {
 ; CHECK-LABEL: test_sqrt_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -122,7 +122,7 @@ define float @test_sqrt_ftz(float %a) #0 #1 {
   ret float %ret
 }
 
-define float @test_sqrt_ftz_ninf(float %a) #0 #1 {
+define float @test_sqrt_ftz_ninf(float %a) #1 {
 ; CHECK-LABEL: test_sqrt_ftz_ninf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -139,7 +139,7 @@ define float @test_sqrt_ftz_ninf(float %a) #0 #1 {
   ret float %ret
 }
 
-define double @test_sqrt64(double %a) #0 {
+define double @test_sqrt64(double %a) {
 ; CHECK-LABEL: test_sqrt64(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
@@ -156,7 +156,7 @@ define double @test_sqrt64(double %a) #0 {
 ; There's no sqrt.approx.f64 instruction; we emit
 ; reciprocal(rsqrt.approx.f64(x)).  There's no non-ftz approximate reciprocal,
 ; so we just use the ftz version.
-define double @test_sqrt64_ninf(double %a) #0 {
+define double @test_sqrt64_ninf(double %a) {
 ; CHECK-LABEL: test_sqrt64_ninf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -175,7 +175,7 @@ define double @test_sqrt64_ninf(double %a) #0 {
   ret double %ret
 }
 
-define double @test_sqrt64_ftz(double %a) #0 #1 {
+define double @test_sqrt64_ftz(double %a) #1 {
 ; CHECK-LABEL: test_sqrt64_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
@@ -190,7 +190,7 @@ define double @test_sqrt64_ftz(double %a) #0 #1 {
 }
 
 ; There's no sqrt.approx.ftz.f64 instruction; we just use the non-ftz version.
-define double @test_sqrt64_ftz_ninf(double %a) #0 #1 {
+define double @test_sqrt64_ftz_ninf(double %a) #1 {
 ; CHECK-LABEL: test_sqrt64_ftz_ninf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -214,7 +214,7 @@ define double @test_sqrt64_ftz_ninf(double %a) #0 #1 {
 ; The sqrt and rsqrt refinement algorithms both emit an rsqrt.approx, followed
 ; by some math.
 
-define float @test_rsqrt32_refined(float %a) #0 #2 {
+define float @test_rsqrt32_refined(float %a) #2 {
 ; CHECK-LABEL: test_rsqrt32_refined(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<7>;
@@ -229,11 +229,11 @@ define float @test_rsqrt32_refined(float %a) #0 #2 {
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
-  %ret = fdiv arcp float 1.0, %val
+  %ret = fdiv arcp contract float 1.0, %val
   ret float %ret
 }
 
-define float @test_sqrt32_refined(float %a) #0 #2 {
+define float @test_sqrt32_refined(float %a) #2 {
 ; CHECK-LABEL: test_sqrt32_refined(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -247,7 +247,7 @@ define float @test_sqrt32_refined(float %a) #0 #2 {
   ret float %ret
 }
 
-define float @test_sqrt32_refined_ninf(float %a) #0 #2 {
+define float @test_sqrt32_refined_ninf(float %a) #2 {
 ; CHECK-LABEL: test_sqrt32_refined_ninf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -265,11 +265,11 @@ define float @test_sqrt32_refined_ninf(float %a) #0 #2 {
 ; CHECK-NEXT:    selp.f32 %r8, 0f00000000, %r6, %p1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r8;
 ; CHECK-NEXT:    ret;
-  %ret = tail call ninf afn float @llvm.sqrt.f32(float %a)
+  %ret = tail call ninf afn contract float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
-define double @test_rsqrt64_refined(double %a) #0 #2 {
+define double @test_rsqrt64_refined(double %a) #2 {
 ; CHECK-LABEL: test_rsqrt64_refined(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<7>;
@@ -284,11 +284,11 @@ define double @test_rsqrt64_refined(double %a) #0 #2 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv arcp double 1.0, %val
+  %ret = fdiv arcp contract double 1.0, %val
   ret double %ret
 }
 
-define double @test_sqrt64_refined(double %a) #0 #2 {
+define double @test_sqrt64_refined(double %a) #2 {
 ; CHECK-LABEL: test_sqrt64_refined(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
@@ -302,7 +302,7 @@ define double @test_sqrt64_refined(double %a) #0 #2 {
   ret double %ret
 }
 
-define double @test_sqrt64_refined_ninf(double %a) #0 #2 {
+define double @test_sqrt64_refined_ninf(double %a) #2 {
 ; CHECK-LABEL: test_sqrt64_refined_ninf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -320,13 +320,13 @@ define double @test_sqrt64_refined_ninf(double %a) #0 #2 {
 ; CHECK-NEXT:    selp.f64 %rd8, 0d0000000000000000, %rd6, %p1;
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd8;
 ; CHECK-NEXT:    ret;
-  %ret = tail call ninf afn double @llvm.sqrt.f64(double %a)
+  %ret = tail call ninf afn contract double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
 ; -- refined sqrt and rsqrt with ftz enabled --
 
-define float @test_rsqrt32_refined_ftz(float %a) #0 #1 #2 {
+define float @test_rsqrt32_refined_ftz(float %a) #1 #2 {
 ; CHECK-LABEL: test_rsqrt32_refined_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<7>;
@@ -341,11 +341,11 @@ define float @test_rsqrt32_refined_ftz(float %a) #0 #1 #2 {
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r6;
 ; CHECK-NEXT:    ret;
   %val = tail call float @llvm.sqrt.f32(float %a)
-  %ret = fdiv arcp float 1.0, %val
+  %ret = fdiv arcp contract float 1.0, %val
   ret float %ret
 }
 
-define float @test_sqrt32_refined_ftz(float %a) #0 #1 #2 {
+define float @test_sqrt32_refined_ftz(float %a) #1 #2 {
 ; CHECK-LABEL: test_sqrt32_refined_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
@@ -359,7 +359,7 @@ define float @test_sqrt32_refined_ftz(float %a) #0 #1 #2 {
   ret float %ret
 }
 
-define float @test_sqrt32_refined_ftz_ninf(float %a) #0 #1 #2 {
+define float @test_sqrt32_refined_ftz_ninf(float %a) #1 #2 {
 ; CHECK-LABEL: test_sqrt32_refined_ftz_ninf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -376,12 +376,12 @@ define float @test_sqrt32_refined_ftz_ninf(float %a) #0 #1 #2 {
 ; CHECK-NEXT:    selp.f32 %r7, 0f00000000, %r6, %p1;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r7;
 ; CHECK-NEXT:    ret;
-  %ret = tail call ninf afn float @llvm.sqrt.f32(float %a)
+  %ret = tail call ninf afn contract float @llvm.sqrt.f32(float %a)
   ret float %ret
 }
 
 ; There's no rsqrt.approx.ftz.f64, so we just use the non-ftz version.
-define double @test_rsqrt64_refined_ftz(double %a) #0 #1 #2 {
+define double @test_rsqrt64_refined_ftz(double %a) #1 #2 {
 ; CHECK-LABEL: test_rsqrt64_refined_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<7>;
@@ -396,11 +396,11 @@ define double @test_rsqrt64_refined_ftz(double %a) #0 #1 #2 {
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
 ; CHECK-NEXT:    ret;
   %val = tail call double @llvm.sqrt.f64(double %a)
-  %ret = fdiv arcp double 1.0, %val
+  %ret = fdiv arcp contract double 1.0, %val
   ret double %ret
 }
 
-define double @test_sqrt64_refined_ftz(double %a) #0 #1 #2 {
+define double @test_sqrt64_refined_ftz(double %a) #1 #2 {
 ; CHECK-LABEL: test_sqrt64_refined_ftz(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b64 %rd<3>;
@@ -414,7 +414,7 @@ define double @test_sqrt64_refined_ftz(double %a) #0 #1 #2 {
   ret double %ret
 }
 
-define double @test_sqrt64_refined_ftz_ninf(double %a) #0 #1 #2 {
+define double @test_sqrt64_refined_ftz_ninf(double %a) #1 #2 {
 ; CHECK-LABEL: test_sqrt64_refined_ftz_ninf(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .pred %p<2>;
@@ -432,10 +432,9 @@ define double @test_sqrt64_refined_ftz_ninf(double %a) #0 #1 #2 {
 ; CHECK-NEXT:    selp.f64 %rd8, 0d0000000000000000, %rd6, %p1;
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd8;
 ; CHECK-NEXT:    ret;
-  %ret = tail call ninf afn double @llvm.sqrt.f64(double %a)
+  %ret = tail call ninf afn contract double @llvm.sqrt.f64(double %a)
   ret double %ret
 }
 
-attributes #0 = { "unsafe-fp-math" = "true" }
 attributes #1 = { "denormal-fp-math-f32" = "preserve-sign,preserve-sign" }
 attributes #2 = { "reciprocal-estimates" = "rsqrtf:1,rsqrtd:1,sqrtf:1,sqrtd:1" }
diff --git a/llvm/test/CodeGen/NVPTX/surf-read-cuda.ll b/llvm/test/CodeGen/NVPTX/surf-read-cuda.ll
index 8056855a0d539..d443aebf32447 100644
--- a/llvm/test/CodeGen/NVPTX/surf-read-cuda.ll
+++ b/llvm/test/CodeGen/NVPTX/surf-read-cuda.ll
@@ -37,7 +37,7 @@ define ptx_kernel void @bar(ptr %red, i32 %idx) {
 ; CHECK-LABEL: bar(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [bar_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/surf-write-cuda.ll b/llvm/test/CodeGen/NVPTX/surf-write-cuda.ll
index abc2ea89b62cf..c0ced65709610 100644
--- a/llvm/test/CodeGen/NVPTX/surf-write-cuda.ll
+++ b/llvm/test/CodeGen/NVPTX/surf-write-cuda.ll
@@ -34,7 +34,6 @@ define ptx_kernel void @bar(i32 %val, i32 %idx) {
 ; CHECK-LABEL: bar(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [bar_param_0];
diff --git a/llvm/test/CodeGen/NVPTX/tex-read-cuda.ll b/llvm/test/CodeGen/NVPTX/tex-read-cuda.ll
index 3138d7c4c14db..20f6e2ec50c2c 100644
--- a/llvm/test/CodeGen/NVPTX/tex-read-cuda.ll
+++ b/llvm/test/CodeGen/NVPTX/tex-read-cuda.ll
@@ -37,7 +37,7 @@ define ptx_kernel void @bar(ptr %red, i32 %idx) {
 ; CHECK-LABEL: bar(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<6>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [bar_param_0];
@@ -58,7 +58,7 @@ declare float @texfunc(i64)
 define ptx_kernel void @baz(ptr %red, i32 %idx) {
 ; CHECK-LABEL: baz(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<9>;
+; CHECK-NEXT:    .reg .b32 %r<8>;
 ; CHECK-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
@@ -74,8 +74,8 @@ define ptx_kernel void @baz(ptr %red, i32 %idx) {
 ; CHECK-NEXT:    call.uni (retval0), texfunc, (param0);
 ; CHECK-NEXT:    ld.param.b32 %r6, [retval0];
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    add.rn.f32 %r8, %r2, %r6;
-; CHECK-NEXT:    st.global.b32 [%rd2], %r8;
+; CHECK-NEXT:    add.rn.f32 %r7, %r2, %r6;
+; CHECK-NEXT:    st.global.b32 [%rd2], %r7;
 ; CHECK-NEXT:    ret;
   %texHandle = tail call i64 @llvm.nvvm.texsurf.handle.internal.p1(ptr addrspace(1) @tex0)
   %val = tail call { float, float, float, float } @llvm.nvvm.tex.unified.1d.v4f32.s32(i64 %texHandle, i32 %idx)
diff --git a/llvm/test/CodeGen/NVPTX/texsurf-queries.ll b/llvm/test/CodeGen/NVPTX/texsurf-queries.ll
index 4edbec48e6bec..c5299046e1db3 100644
--- a/llvm/test/CodeGen/NVPTX/texsurf-queries.ll
+++ b/llvm/test/CodeGen/NVPTX/texsurf-queries.ll
@@ -35,7 +35,6 @@ define i32 @t1() {
 ; CHECK-LABEL: t1(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    txq.width.b32 %r1, [tex0];
@@ -66,7 +65,6 @@ define i32 @t3() {
 ; CHECK-LABEL: t3(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    txq.height.b32 %r1, [tex0];
@@ -97,7 +95,6 @@ define i32 @s1() {
 ; CHECK-LABEL: s1(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    suq.width.b32 %r1, [surf0];
@@ -128,7 +125,6 @@ define i32 @s3() {
 ; CHECK-LABEL: s3(
 ; CHECK:       {
 ; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    suq.height.b32 %r1, [surf0];
diff --git a/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll b/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll
index 697eb90fb1740..526355247c009 100644
--- a/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll
+++ b/llvm/test/CodeGen/NVPTX/unaligned-param-load-store.ll
@@ -24,9 +24,9 @@
 define %s_i8i16p @test_s_i8i16p(%s_i8i16p %a) {
 ; CHECK-LABEL: test_s_i8i16p(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<13>;
+; CHECK-NEXT:    .reg .b16 %rs<9>;
 ; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_s_i8i16p_param_0];
@@ -45,14 +45,14 @@ define %s_i8i16p @test_s_i8i16p(%s_i8i16p %a) {
 ; CHECK-NEXT:    ld.param.b8 %rs4, [retval0+4];
 ; CHECK-NEXT:    ld.param.b8 %rs5, [retval0+3];
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    shl.b16 %rs8, %rs4, 8;
-; CHECK-NEXT:    or.b16 %rs9, %rs8, %rs5;
+; CHECK-NEXT:    shl.b16 %rs6, %rs4, 8;
+; CHECK-NEXT:    or.b16 %rs7, %rs6, %rs5;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rs5;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+8], %rd2;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rs2;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs3;
-; CHECK-NEXT:    shr.u16 %rs12, %rs9, 8;
-; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs12;
+; CHECK-NEXT:    shr.u16 %rs8, %rs7, 8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs8;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8i16p @test_s_i8i16p(%s_i8i16p %a)
   ret %s_i8i16p %r
@@ -62,9 +62,9 @@ define %s_i8i16p @test_s_i8i16p(%s_i8i16p %a) {
 define %s_i8i32p @test_s_i8i32p(%s_i8i32p %a) {
 ; CHECK-LABEL: test_s_i8i32p(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<4>;
-; CHECK-NEXT:    .reg .b32 %r<24>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<19>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_s_i8i32p_param_0];
@@ -91,22 +91,22 @@ define %s_i8i32p @test_s_i8i32p(%s_i8i32p %a) {
 ; CHECK-NEXT:    ld.param.b8 %r8, [retval0+6];
 ; CHECK-NEXT:    ld.param.b8 %r9, [retval0+5];
 ; CHECK-NEXT:    } // callseq 1
-; CHECK-NEXT:    shl.b32 %r12, %r8, 8;
-; CHECK-NEXT:    or.b32 %r13, %r12, %r9;
-; CHECK-NEXT:    shl.b32 %r15, %r7, 16;
-; CHECK-NEXT:    shl.b32 %r17, %r6, 24;
-; CHECK-NEXT:    or.b32 %r18, %r17, %r15;
-; CHECK-NEXT:    or.b32 %r19, %r18, %r13;
+; CHECK-NEXT:    shl.b32 %r10, %r8, 8;
+; CHECK-NEXT:    or.b32 %r11, %r10, %r9;
+; CHECK-NEXT:    shl.b32 %r12, %r7, 16;
+; CHECK-NEXT:    shl.b32 %r13, %r6, 24;
+; CHECK-NEXT:    or.b32 %r14, %r13, %r12;
+; CHECK-NEXT:    or.b32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r9;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd2;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
-; CHECK-NEXT:    shr.u32 %r21, %r19, 24;
-; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r21;
-; CHECK-NEXT:    shr.u32 %r22, %r19, 16;
-; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r22;
-; CHECK-NEXT:    shr.u32 %r23, %r19, 8;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r23;
+; CHECK-NEXT:    shr.u32 %r16, %r15, 24;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r16;
+; CHECK-NEXT:    shr.u32 %r17, %r15, 16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r17;
+; CHECK-NEXT:    shr.u32 %r18, %r15, 8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r18;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8i32p @test_s_i8i32p(%s_i8i32p %a)
   ret %s_i8i32p %r
@@ -116,8 +116,8 @@ define %s_i8i32p @test_s_i8i32p(%s_i8i32p %a) {
 define %s_i8i64p @test_s_i8i64p(%s_i8i64p %a) {
 ; CHECK-LABEL: test_s_i8i64p(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b64 %rd<46>;
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b64 %rd<36>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_s_i8i64p_param_0];
@@ -144,38 +144,38 @@ define %s_i8i64p @test_s_i8i64p(%s_i8i64p %a) {
 ; CHECK-NEXT:    ld.param.b8 %rd13, [retval0+10];
 ; CHECK-NEXT:    ld.param.b8 %rd14, [retval0+9];
 ; CHECK-NEXT:    } // callseq 2
-; CHECK-NEXT:    shl.b64 %rd17, %rd13, 8;
-; CHECK-NEXT:    or.b64 %rd18, %rd17, %rd14;
-; CHECK-NEXT:    shl.b64 %rd20, %rd12, 16;
-; CHECK-NEXT:    shl.b64 %rd22, %rd11, 24;
-; CHECK-NEXT:    or.b64 %rd23, %rd22, %rd20;
-; CHECK-NEXT:    or.b64 %rd24, %rd23, %rd18;
-; CHECK-NEXT:    shl.b64 %rd27, %rd9, 8;
-; CHECK-NEXT:    or.b64 %rd28, %rd27, %rd10;
-; CHECK-NEXT:    shl.b64 %rd30, %rd8, 16;
-; CHECK-NEXT:    shl.b64 %rd32, %rd7, 24;
-; CHECK-NEXT:    or.b64 %rd33, %rd32, %rd30;
-; CHECK-NEXT:    or.b64 %rd34, %rd33, %rd28;
-; CHECK-NEXT:    shl.b64 %rd35, %rd34, 32;
-; CHECK-NEXT:    or.b64 %rd36, %rd35, %rd24;
+; CHECK-NEXT:    shl.b64 %rd15, %rd13, 8;
+; CHECK-NEXT:    or.b64 %rd16, %rd15, %rd14;
+; CHECK-NEXT:    shl.b64 %rd17, %rd12, 16;
+; CHECK-NEXT:    shl.b64 %rd18, %rd11, 24;
+; CHECK-NEXT:    or.b64 %rd19, %rd18, %rd17;
+; CHECK-NEXT:    or.b64 %rd20, %rd19, %rd16;
+; CHECK-NEXT:    shl.b64 %rd21, %rd9, 8;
+; CHECK-NEXT:    or.b64 %rd22, %rd21, %rd10;
+; CHECK-NEXT:    shl.b64 %rd23, %rd8, 16;
+; CHECK-NEXT:    shl.b64 %rd24, %rd7, 24;
+; CHECK-NEXT:    or.b64 %rd25, %rd24, %rd23;
+; CHECK-NEXT:    or.b64 %rd26, %rd25, %rd22;
+; CHECK-NEXT:    shl.b64 %rd27, %rd26, 32;
+; CHECK-NEXT:    or.b64 %rd28, %rd27, %rd20;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+9], %rd14;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+24], %rd5;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+8], %rs1;
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
-; CHECK-NEXT:    shr.u64 %rd39, %rd36, 56;
-; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rd39;
-; CHECK-NEXT:    shr.u64 %rd40, %rd36, 48;
-; CHECK-NEXT:    st.param.b8 [func_retval0+15], %rd40;
-; CHECK-NEXT:    shr.u64 %rd41, %rd36, 40;
-; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rd41;
-; CHECK-NEXT:    shr.u64 %rd42, %rd36, 32;
-; CHECK-NEXT:    st.param.b8 [func_retval0+13], %rd42;
-; CHECK-NEXT:    shr.u64 %rd43, %rd36, 24;
-; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd43;
-; CHECK-NEXT:    shr.u64 %rd44, %rd36, 16;
-; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd44;
-; CHECK-NEXT:    shr.u64 %rd45, %rd36, 8;
-; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd45;
+; CHECK-NEXT:    shr.u64 %rd29, %rd28, 56;
+; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rd29;
+; CHECK-NEXT:    shr.u64 %rd30, %rd28, 48;
+; CHECK-NEXT:    st.param.b8 [func_retval0+15], %rd30;
+; CHECK-NEXT:    shr.u64 %rd31, %rd28, 40;
+; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rd31;
+; CHECK-NEXT:    shr.u64 %rd32, %rd28, 32;
+; CHECK-NEXT:    st.param.b8 [func_retval0+13], %rd32;
+; CHECK-NEXT:    shr.u64 %rd33, %rd28, 24;
+; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd33;
+; CHECK-NEXT:    shr.u64 %rd34, %rd28, 16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd34;
+; CHECK-NEXT:    shr.u64 %rd35, %rd28, 8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd35;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8i64p @test_s_i8i64p(%s_i8i64p %a)
   ret %s_i8i64p %r
@@ -185,8 +185,8 @@ define %s_i8i64p @test_s_i8i64p(%s_i8i64p %a) {
 define %s_i8f16p @test_s_i8f16p(%s_i8f16p %a) {
 ; CHECK-LABEL: test_s_i8f16p(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<15>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b16 %rs<11>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b16 %rs1, [test_s_i8f16p_param_0];
@@ -207,14 +207,14 @@ define %s_i8f16p @test_s_i8f16p(%s_i8f16p %a) {
 ; CHECK-NEXT:    ld.param.b8 %rs6, [retval0+4];
 ; CHECK-NEXT:    ld.param.b8 %rs7, [retval0+3];
 ; CHECK-NEXT:    } // callseq 3
-; CHECK-NEXT:    shl.b16 %rs10, %rs6, 8;
-; CHECK-NEXT:    or.b16 %rs11, %rs10, %rs7;
+; CHECK-NEXT:    shl.b16 %rs8, %rs6, 8;
+; CHECK-NEXT:    or.b16 %rs9, %rs8, %rs7;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+3], %rs7;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+8], %rd2;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+2], %rs4;
 ; CHECK-NEXT:    st.param.b16 [func_retval0], %rs5;
-; CHECK-NEXT:    shr.u16 %rs14, %rs11, 8;
-; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs14;
+; CHECK-NEXT:    shr.u16 %rs10, %rs9, 8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs10;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8f16p @test_s_i8f16p(%s_i8f16p %a)
   ret %s_i8f16p %r
@@ -224,9 +224,9 @@ define %s_i8f16p @test_s_i8f16p(%s_i8f16p %a) {
 define %s_i8f16x2p @test_s_i8f16x2p(%s_i8f16x2p %a) {
 ; CHECK-LABEL: test_s_i8f16x2p(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<4>;
-; CHECK-NEXT:    .reg .b32 %r<24>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<19>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_s_i8f16x2p_param_0];
@@ -253,22 +253,22 @@ define %s_i8f16x2p @test_s_i8f16x2p(%s_i8f16x2p %a) {
 ; CHECK-NEXT:    ld.param.b8 %r8, [retval0+6];
 ; CHECK-NEXT:    ld.param.b8 %r9, [retval0+5];
 ; CHECK-NEXT:    } // callseq 4
-; CHECK-NEXT:    shl.b32 %r12, %r8, 8;
-; CHECK-NEXT:    or.b32 %r13, %r12, %r9;
-; CHECK-NEXT:    shl.b32 %r15, %r7, 16;
-; CHECK-NEXT:    shl.b32 %r17, %r6, 24;
-; CHECK-NEXT:    or.b32 %r18, %r17, %r15;
-; CHECK-NEXT:    or.b32 %r19, %r18, %r13;
+; CHECK-NEXT:    shl.b32 %r10, %r8, 8;
+; CHECK-NEXT:    or.b32 %r11, %r10, %r9;
+; CHECK-NEXT:    shl.b32 %r12, %r7, 16;
+; CHECK-NEXT:    shl.b32 %r13, %r6, 24;
+; CHECK-NEXT:    or.b32 %r14, %r13, %r12;
+; CHECK-NEXT:    or.b32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r9;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd2;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
-; CHECK-NEXT:    shr.u32 %r21, %r19, 24;
-; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r21;
-; CHECK-NEXT:    shr.u32 %r22, %r19, 16;
-; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r22;
-; CHECK-NEXT:    shr.u32 %r23, %r19, 8;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r23;
+; CHECK-NEXT:    shr.u32 %r16, %r15, 24;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r16;
+; CHECK-NEXT:    shr.u32 %r17, %r15, 16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r17;
+; CHECK-NEXT:    shr.u32 %r18, %r15, 8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r18;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8f16x2p @test_s_i8f16x2p(%s_i8f16x2p %a)
   ret %s_i8f16x2p %r
@@ -278,9 +278,9 @@ define %s_i8f16x2p @test_s_i8f16x2p(%s_i8f16x2p %a) {
 define %s_i8f32p @test_s_i8f32p(%s_i8f32p %a) {
 ; CHECK-LABEL: test_s_i8f32p(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<4>;
-; CHECK-NEXT:    .reg .b32 %r<24>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<19>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b32 %r1, [test_s_i8f32p_param_0];
@@ -307,22 +307,22 @@ define %s_i8f32p @test_s_i8f32p(%s_i8f32p %a) {
 ; CHECK-NEXT:    ld.param.b8 %r8, [retval0+6];
 ; CHECK-NEXT:    ld.param.b8 %r9, [retval0+5];
 ; CHECK-NEXT:    } // callseq 5
-; CHECK-NEXT:    shl.b32 %r12, %r8, 8;
-; CHECK-NEXT:    or.b32 %r13, %r12, %r9;
-; CHECK-NEXT:    shl.b32 %r15, %r7, 16;
-; CHECK-NEXT:    shl.b32 %r17, %r6, 24;
-; CHECK-NEXT:    or.b32 %r18, %r17, %r15;
-; CHECK-NEXT:    or.b32 %r19, %r18, %r13;
+; CHECK-NEXT:    shl.b32 %r10, %r8, 8;
+; CHECK-NEXT:    or.b32 %r11, %r10, %r9;
+; CHECK-NEXT:    shl.b32 %r12, %r7, 16;
+; CHECK-NEXT:    shl.b32 %r13, %r6, 24;
+; CHECK-NEXT:    or.b32 %r14, %r13, %r12;
+; CHECK-NEXT:    or.b32 %r15, %r14, %r11;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+5], %r9;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+16], %rd2;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+4], %rs2;
 ; CHECK-NEXT:    st.param.b32 [func_retval0], %r5;
-; CHECK-NEXT:    shr.u32 %r21, %r19, 24;
-; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r21;
-; CHECK-NEXT:    shr.u32 %r22, %r19, 16;
-; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r22;
-; CHECK-NEXT:    shr.u32 %r23, %r19, 8;
-; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r23;
+; CHECK-NEXT:    shr.u32 %r16, %r15, 24;
+; CHECK-NEXT:    st.param.b8 [func_retval0+8], %r16;
+; CHECK-NEXT:    shr.u32 %r17, %r15, 16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+7], %r17;
+; CHECK-NEXT:    shr.u32 %r18, %r15, 8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+6], %r18;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8f32p @test_s_i8f32p(%s_i8f32p %a)
   ret %s_i8f32p %r
@@ -332,8 +332,8 @@ define %s_i8f32p @test_s_i8f32p(%s_i8f32p %a) {
 define %s_i8f64p @test_s_i8f64p(%s_i8f64p %a) {
 ; CHECK-LABEL: test_s_i8f64p(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b64 %rd<46>;
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NEXT:    .reg .b64 %rd<36>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param.b64 %rd1, [test_s_i8f64p_param_0];
@@ -360,38 +360,38 @@ define %s_i8f64p @test_s_i8f64p(%s_i8f64p %a) {
 ; CHECK-NEXT:    ld.param.b8 %rd13, [retval0+10];
 ; CHECK-NEXT:    ld.param.b8 %rd14, [retval0+9];
 ; CHECK-NEXT:    } // callseq 6
-; CHECK-NEXT:    shl.b64 %rd17, %rd13, 8;
-; CHECK-NEXT:    or.b64 %rd18, %rd17, %rd14;
-; CHECK-NEXT:    shl.b64 %rd20, %rd12, 16;
-; CHECK-NEXT:    shl.b64 %rd22, %rd11, 24;
-; CHECK-NEXT:    or.b64 %rd23, %rd22, %rd20;
-; CHECK-NEXT:    or.b64 %rd24, %rd23, %rd18;
-; CHECK-NEXT:    shl.b64 %rd27, %rd9, 8;
-; CHECK-NEXT:    or.b64 %rd28, %rd27, %rd10;
-; CHECK-NEXT:    shl.b64 %rd30, %rd8, 16;
-; CHECK-NEXT:    shl.b64 %rd32, %rd7, 24;
-; CHECK-NEXT:    or.b64 %rd33, %rd32, %rd30;
-; CHECK-NEXT:    or.b64 %rd34, %rd33, %rd28;
-; CHECK-NEXT:    shl.b64 %rd35, %rd34, 32;
-; CHECK-NEXT:    or.b64 %rd36, %rd35, %rd24;
+; CHECK-NEXT:    shl.b64 %rd15, %rd13, 8;
+; CHECK-NEXT:    or.b64 %rd16, %rd15, %rd14;
+; CHECK-NEXT:    shl.b64 %rd17, %rd12, 16;
+; CHECK-NEXT:    shl.b64 %rd18, %rd11, 24;
+; CHECK-NEXT:    or.b64 %rd19, %rd18, %rd17;
+; CHECK-NEXT:    or.b64 %rd20, %rd19, %rd16;
+; CHECK-NEXT:    shl.b64 %rd21, %rd9, 8;
+; CHECK-NEXT:    or.b64 %rd22, %rd21, %rd10;
+; CHECK-NEXT:    shl.b64 %rd23, %rd8, 16;
+; CHECK-NEXT:    shl.b64 %rd24, %rd7, 24;
+; CHECK-NEXT:    or.b64 %rd25, %rd24, %rd23;
+; CHECK-NEXT:    or.b64 %rd26, %rd25, %rd22;
+; CHECK-NEXT:    shl.b64 %rd27, %rd26, 32;
+; CHECK-NEXT:    or.b64 %rd28, %rd27, %rd20;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+9], %rd14;
 ; CHECK-NEXT:    st.param.b64 [func_retval0+24], %rd5;
 ; CHECK-NEXT:    st.param.b8 [func_retval0+8], %rs1;
 ; CHECK-NEXT:    st.param.b64 [func_retval0], %rd6;
-; CHECK-NEXT:    shr.u64 %rd39, %rd36, 56;
-; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rd39;
-; CHECK-NEXT:    shr.u64 %rd40, %rd36, 48;
-; CHECK-NEXT:    st.param.b8 [func_retval0+15], %rd40;
-; CHECK-NEXT:    shr.u64 %rd41, %rd36, 40;
-; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rd41;
-; CHECK-NEXT:    shr.u64 %rd42, %rd36, 32;
-; CHECK-NEXT:    st.param.b8 [func_retval0+13], %rd42;
-; CHECK-NEXT:    shr.u64 %rd43, %rd36, 24;
-; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd43;
-; CHECK-NEXT:    shr.u64 %rd44, %rd36, 16;
-; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd44;
-; CHECK-NEXT:    shr.u64 %rd45, %rd36, 8;
-; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd45;
+; CHECK-NEXT:    shr.u64 %rd29, %rd28, 56;
+; CHECK-NEXT:    st.param.b8 [func_retval0+16], %rd29;
+; CHECK-NEXT:    shr.u64 %rd30, %rd28, 48;
+; CHECK-NEXT:    st.param.b8 [func_retval0+15], %rd30;
+; CHECK-NEXT:    shr.u64 %rd31, %rd28, 40;
+; CHECK-NEXT:    st.param.b8 [func_retval0+14], %rd31;
+; CHECK-NEXT:    shr.u64 %rd32, %rd28, 32;
+; CHECK-NEXT:    st.param.b8 [func_retval0+13], %rd32;
+; CHECK-NEXT:    shr.u64 %rd33, %rd28, 24;
+; CHECK-NEXT:    st.param.b8 [func_retval0+12], %rd33;
+; CHECK-NEXT:    shr.u64 %rd34, %rd28, 16;
+; CHECK-NEXT:    st.param.b8 [func_retval0+11], %rd34;
+; CHECK-NEXT:    shr.u64 %rd35, %rd28, 8;
+; CHECK-NEXT:    st.param.b8 [func_retval0+10], %rd35;
 ; CHECK-NEXT:    ret;
   %r = tail call %s_i8f64p @test_s_i8f64p(%s_i8f64p %a)
   ret %s_i8f64p %r
diff --git a/llvm/test/CodeGen/NVPTX/variadics-backend.ll b/llvm/test/CodeGen/NVPTX/variadics-backend.ll
index a9b3675b67155..890753b6ac5aa 100644
--- a/llvm/test/CodeGen/NVPTX/variadics-backend.ll
+++ b/llvm/test/CodeGen/NVPTX/variadics-backend.ll
@@ -104,7 +104,7 @@ define dso_local i32 @foo() {
 ; CHECK-PTX-NEXT:    .local .align 8 .b8 __local_depot1[40];
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
-; CHECK-PTX-NEXT:    .reg .b32 %r<3>;
+; CHECK-PTX-NEXT:    .reg .b32 %r<2>;
 ; CHECK-PTX-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
@@ -143,29 +143,29 @@ define dso_local i32 @variadics2(i32 noundef %first, ...) {
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
 ; CHECK-PTX-NEXT:    .reg .b16 %rs<4>;
 ; CHECK-PTX-NEXT:    .reg .b32 %r<6>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<9>;
+; CHECK-PTX-NEXT:    .reg .b64 %rd<8>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
 ; CHECK-PTX-NEXT:    mov.b64 %SPL, __local_depot2;
 ; CHECK-PTX-NEXT:    ld.param.b32 %r1, [variadics2_param_0];
 ; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [variadics2_param_1];
-; CHECK-PTX-NEXT:    add.u64 %rd3, %SPL, 0;
-; CHECK-PTX-NEXT:    add.s64 %rd4, %rd1, 7;
-; CHECK-PTX-NEXT:    and.b64 %rd5, %rd4, -8;
-; CHECK-PTX-NEXT:    ld.b32 %r2, [%rd5];
-; CHECK-PTX-NEXT:    ld.s8 %r3, [%rd5+4];
-; CHECK-PTX-NEXT:    ld.b8 %rs1, [%rd5+7];
-; CHECK-PTX-NEXT:    st.local.b8 [%rd3+2], %rs1;
-; CHECK-PTX-NEXT:    ld.b8 %rs2, [%rd5+6];
-; CHECK-PTX-NEXT:    st.local.b8 [%rd3+1], %rs2;
-; CHECK-PTX-NEXT:    ld.b8 %rs3, [%rd5+5];
-; CHECK-PTX-NEXT:    st.local.b8 [%rd3], %rs3;
-; CHECK-PTX-NEXT:    ld.b64 %rd6, [%rd5+8];
+; CHECK-PTX-NEXT:    add.u64 %rd2, %SPL, 0;
+; CHECK-PTX-NEXT:    add.s64 %rd3, %rd1, 7;
+; CHECK-PTX-NEXT:    and.b64 %rd4, %rd3, -8;
+; CHECK-PTX-NEXT:    ld.b32 %r2, [%rd4];
+; CHECK-PTX-NEXT:    ld.s8 %r3, [%rd4+4];
+; CHECK-PTX-NEXT:    ld.b8 %rs1, [%rd4+7];
+; CHECK-PTX-NEXT:    st.local.b8 [%rd2+2], %rs1;
+; CHECK-PTX-NEXT:    ld.b8 %rs2, [%rd4+6];
+; CHECK-PTX-NEXT:    st.local.b8 [%rd2+1], %rs2;
+; CHECK-PTX-NEXT:    ld.b8 %rs3, [%rd4+5];
+; CHECK-PTX-NEXT:    st.local.b8 [%rd2], %rs3;
+; CHECK-PTX-NEXT:    ld.b64 %rd5, [%rd4+8];
 ; CHECK-PTX-NEXT:    add.s32 %r4, %r1, %r2;
 ; CHECK-PTX-NEXT:    add.s32 %r5, %r4, %r3;
-; CHECK-PTX-NEXT:    cvt.u64.u32 %rd7, %r5;
-; CHECK-PTX-NEXT:    add.s64 %rd8, %rd7, %rd6;
-; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %rd8;
+; CHECK-PTX-NEXT:    cvt.u64.u32 %rd6, %r5;
+; CHECK-PTX-NEXT:    add.s64 %rd7, %rd6, %rd5;
+; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %rd7;
 ; CHECK-PTX-NEXT:    ret;
 entry:
   %vlist = alloca ptr, align 8
@@ -202,19 +202,19 @@ define dso_local i32 @bar() {
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
 ; CHECK-PTX-NEXT:    .reg .b16 %rs<4>;
-; CHECK-PTX-NEXT:    .reg .b32 %r<3>;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<4>;
+; CHECK-PTX-NEXT:    .reg .b32 %r<2>;
+; CHECK-PTX-NEXT:    .reg .b64 %rd<3>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
 ; CHECK-PTX-NEXT:    mov.b64 %SPL, __local_depot3;
 ; CHECK-PTX-NEXT:    cvta.local.u64 %SP, %SPL;
-; CHECK-PTX-NEXT:    add.u64 %rd2, %SPL, 0;
+; CHECK-PTX-NEXT:    add.u64 %rd1, %SPL, 0;
 ; CHECK-PTX-NEXT:    ld.global.nc.b8 %rs1, [__const_$_bar_$_s1+7];
-; CHECK-PTX-NEXT:    st.local.b8 [%rd2+2], %rs1;
+; CHECK-PTX-NEXT:    st.local.b8 [%rd1+2], %rs1;
 ; CHECK-PTX-NEXT:    ld.global.nc.b8 %rs2, [__const_$_bar_$_s1+6];
-; CHECK-PTX-NEXT:    st.local.b8 [%rd2+1], %rs2;
+; CHECK-PTX-NEXT:    st.local.b8 [%rd1+1], %rs2;
 ; CHECK-PTX-NEXT:    ld.global.nc.b8 %rs3, [__const_$_bar_$_s1+5];
-; CHECK-PTX-NEXT:    st.local.b8 [%rd2], %rs3;
+; CHECK-PTX-NEXT:    st.local.b8 [%rd1], %rs3;
 ; CHECK-PTX-NEXT:    st.b32 [%SP+8], 1;
 ; CHECK-PTX-NEXT:    st.b8 [%SP+12], 1;
 ; CHECK-PTX-NEXT:    st.b64 [%SP+16], 1;
@@ -222,8 +222,8 @@ define dso_local i32 @bar() {
 ; CHECK-PTX-NEXT:    .param .b32 param0;
 ; CHECK-PTX-NEXT:    .param .b64 param1;
 ; CHECK-PTX-NEXT:    .param .b32 retval0;
-; CHECK-PTX-NEXT:    add.u64 %rd3, %SP, 8;
-; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd3;
+; CHECK-PTX-NEXT:    add.u64 %rd2, %SP, 8;
+; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd2;
 ; CHECK-PTX-NEXT:    st.param.b32 [param0], 1;
 ; CHECK-PTX-NEXT:    call.uni (retval0), variadics2, (param0, param1);
 ; CHECK-PTX-NEXT:    ld.param.b32 %r1, [retval0];
@@ -282,7 +282,7 @@ define dso_local i32 @baz() {
 ; CHECK-PTX-NEXT:    .local .align 16 .b8 __local_depot5[16];
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
-; CHECK-PTX-NEXT:    .reg .b32 %r<3>;
+; CHECK-PTX-NEXT:    .reg .b32 %r<2>;
 ; CHECK-PTX-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
@@ -309,18 +309,18 @@ entry:
 define dso_local i32 @variadics4(ptr noundef byval(%struct.S2) align 8 %first, ...) {
 ; CHECK-PTX-LABEL: variadics4(
 ; CHECK-PTX:       {
-; CHECK-PTX-NEXT:    .reg .b64 %rd<10>;
+; CHECK-PTX-NEXT:    .reg .b64 %rd<9>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
-; CHECK-PTX-NEXT:    ld.param.b64 %rd2, [variadics4_param_1];
-; CHECK-PTX-NEXT:    add.s64 %rd3, %rd2, 7;
-; CHECK-PTX-NEXT:    and.b64 %rd4, %rd3, -8;
-; CHECK-PTX-NEXT:    ld.b64 %rd5, [%rd4];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd6, [variadics4_param_0];
-; CHECK-PTX-NEXT:    ld.param.b64 %rd7, [variadics4_param_0+8];
-; CHECK-PTX-NEXT:    add.s64 %rd8, %rd6, %rd7;
-; CHECK-PTX-NEXT:    add.s64 %rd9, %rd8, %rd5;
-; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %rd9;
+; CHECK-PTX-NEXT:    ld.param.b64 %rd1, [variadics4_param_1];
+; CHECK-PTX-NEXT:    add.s64 %rd2, %rd1, 7;
+; CHECK-PTX-NEXT:    and.b64 %rd3, %rd2, -8;
+; CHECK-PTX-NEXT:    ld.b64 %rd4, [%rd3];
+; CHECK-PTX-NEXT:    ld.param.b64 %rd5, [variadics4_param_0];
+; CHECK-PTX-NEXT:    ld.param.b64 %rd6, [variadics4_param_0+8];
+; CHECK-PTX-NEXT:    add.s64 %rd7, %rd5, %rd6;
+; CHECK-PTX-NEXT:    add.s64 %rd8, %rd7, %rd4;
+; CHECK-PTX-NEXT:    st.param.b32 [func_retval0], %rd8;
 ; CHECK-PTX-NEXT:    ret;
 entry:
   %vlist = alloca ptr, align 8
@@ -348,27 +348,27 @@ define dso_local void @qux() {
 ; CHECK-PTX-NEXT:    .local .align 8 .b8 __local_depot7[24];
 ; CHECK-PTX-NEXT:    .reg .b64 %SP;
 ; CHECK-PTX-NEXT:    .reg .b64 %SPL;
-; CHECK-PTX-NEXT:    .reg .b64 %rd<8>;
+; CHECK-PTX-NEXT:    .reg .b64 %rd<7>;
 ; CHECK-PTX-EMPTY:
 ; CHECK-PTX-NEXT:  // %bb.0: // %entry
 ; CHECK-PTX-NEXT:    mov.b64 %SPL, __local_depot7;
 ; CHECK-PTX-NEXT:    cvta.local.u64 %SP, %SPL;
-; CHECK-PTX-NEXT:    add.u64 %rd2, %SPL, 0;
-; CHECK-PTX-NEXT:    ld.global.nc.b64 %rd3, [__const_$_qux_$_s+8];
-; CHECK-PTX-NEXT:    st.local.b64 [%rd2+8], %rd3;
-; CHECK-PTX-NEXT:    ld.global.nc.b64 %rd4, [__const_$_qux_$_s];
-; CHECK-PTX-NEXT:    st.local.b64 [%rd2], %rd4;
+; CHECK-PTX-NEXT:    add.u64 %rd1, %SPL, 0;
+; CHECK-PTX-NEXT:    ld.global.nc.b64 %rd2, [__const_$_qux_$_s+8];
+; CHECK-PTX-NEXT:    st.local.b64 [%rd1+8], %rd2;
+; CHECK-PTX-NEXT:    ld.global.nc.b64 %rd3, [__const_$_qux_$_s];
+; CHECK-PTX-NEXT:    st.local.b64 [%rd1], %rd3;
 ; CHECK-PTX-NEXT:    st.b64 [%SP+16], 1;
 ; CHECK-PTX-NEXT:    { // callseq 3, 0
 ; CHECK-PTX-NEXT:    .param .align 8 .b8 param0[16];
 ; CHECK-PTX-NEXT:    .param .b64 param1;
 ; CHECK-PTX-NEXT:    .param .b32 retval0;
-; CHECK-PTX-NEXT:    add.u64 %rd5, %SP, 16;
-; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd5;
-; CHECK-PTX-NEXT:    ld.local.b64 %rd6, [%rd2+8];
-; CHECK-PTX-NEXT:    st.param.b64 [param0+8], %rd6;
-; CHECK-PTX-NEXT:    ld.local.b64 %rd7, [%rd2];
-; CHECK-PTX-NEXT:    st.param.b64 [param0], %rd7;
+; CHECK-PTX-NEXT:    add.u64 %rd4, %SP, 16;
+; CHECK-PTX-NEXT:    st.param.b64 [param1], %rd4;
+; CHECK-PTX-NEXT:    ld.local.b64 %rd5, [%rd1+8];
+; CHECK-PTX-NEXT:    st.param.b64 [param0+8], %rd5;
+; CHECK-PTX-NEXT:    ld.local.b64 %rd6, [%rd1];
+; CHECK-PTX-NEXT:    st.param.b64 [param0], %rd6;
 ; CHECK-PTX-NEXT:    call.uni (retval0), variadics4, (param0, param1);
 ; CHECK-PTX-NEXT:    } // callseq 3
 ; CHECK-PTX-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/variadics-lowering.ll b/llvm/test/CodeGen/NVPTX/variadics-lowering.ll
index 5502980a263b2..1d69f8de2ca8c 100644
--- a/llvm/test/CodeGen/NVPTX/variadics-lowering.ll
+++ b/llvm/test/CodeGen/NVPTX/variadics-lowering.ll
@@ -119,7 +119,7 @@ define dso_local i32 @foo() {
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i8 1 to i32
 ; CHECK-NEXT:    [[CONV1:%.*]] = sext i16 1 to i32
 ; CHECK-NEXT:    [[CONV2:%.*]] = fpext float 1.000000e+00 to double
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 40, ptr [[VARARG_BUFFER]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[FOO_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; CHECK-NEXT:    store i32 [[CONV]], ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[FOO_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
@@ -133,7 +133,7 @@ define dso_local i32 @foo() {
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[FOO_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 6
 ; CHECK-NEXT:    store double 1.000000e+00, ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @variadics1(i32 noundef 1, ptr [[VARARG_BUFFER]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 40, ptr [[VARARG_BUFFER]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
 entry:
@@ -208,7 +208,7 @@ define dso_local i32 @bar() {
 ; CHECK-NEXT:    [[S1_SROA_2_0_COPYLOAD:%.*]] = load i8, ptr getelementptr inbounds (i8, ptr @__const.bar.s1, i64 4), align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[S1_SROA_3]], ptr align 1 getelementptr inbounds (i8, ptr @__const.bar.s1, i64 5), i64 3, i1 false)
 ; CHECK-NEXT:    [[S1_SROA_31_0_COPYLOAD:%.*]] = load i64, ptr getelementptr inbounds (i8, ptr @__const.bar.s1, i64 8), align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[BAR_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; CHECK-NEXT:    store i32 [[S1_SROA_0_0_COPYLOAD]], ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[BAR_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
@@ -216,7 +216,7 @@ define dso_local i32 @bar() {
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[BAR_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 3
 ; CHECK-NEXT:    store i64 [[S1_SROA_31_0_COPYLOAD]], ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @variadics2(i32 noundef 1, ptr [[VARARG_BUFFER]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
 entry:
@@ -274,11 +274,11 @@ define dso_local i32 @baz() {
 ; CHECK-LABEL: define dso_local i32 @baz() {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[BAZ_VARARG:%.*]], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[BAZ_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; CHECK-NEXT:    store <4 x i32> splat (i32 1), ptr [[TMP0]], align 16
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @variadics3(i32 noundef 1, ptr [[VARARG_BUFFER]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
 entry:
@@ -333,11 +333,11 @@ define dso_local void @qux() {
 ; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_S2:%.*]], align 8
 ; CHECK-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[QUX_VARARG:%.*]], align 8
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[S]], ptr align 8 @__const.qux.s, i64 16, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[QUX_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; CHECK-NEXT:    store i64 1, ptr [[TMP0]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @variadics4(ptr noundef byval([[STRUCT_S2]]) align 8 [[S]], ptr [[VARARG_BUFFER]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/CodeGen/NVPTX/vec-param-load.ll b/llvm/test/CodeGen/NVPTX/vec-param-load.ll
index 29939e323b4b1..3c424c9318375 100644
--- a/llvm/test/CodeGen/NVPTX/vec-param-load.ll
+++ b/llvm/test/CodeGen/NVPTX/vec-param-load.ll
@@ -7,17 +7,17 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 define <16 x float> @test_v16f32(<16 x float> %a) {
 ; CHECK-LABEL: test_v16f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<9>;
+; CHECK-NEXT:    .reg .b32 %r<17>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_v16f32_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_v16f32_param_0+16];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [test_v16f32_param_0+32];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd7, %rd8}, [test_v16f32_param_0+48];
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0+48], {%rd7, %rd8};
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0+32], {%rd5, %rd6};
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd3, %rd4};
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [test_v16f32_param_0];
+; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [test_v16f32_param_0+16];
+; CHECK-NEXT:    ld.param.v4.b32 {%r9, %r10, %r11, %r12}, [test_v16f32_param_0+32];
+; CHECK-NEXT:    ld.param.v4.b32 {%r13, %r14, %r15, %r16}, [test_v16f32_param_0+48];
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0+48], {%r13, %r14, %r15, %r16};
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0+32], {%r9, %r10, %r11, %r12};
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0+16], {%r5, %r6, %r7, %r8};
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r1, %r2, %r3, %r4};
 ; CHECK-NEXT:    ret;
   ret <16 x float> %a
 }
@@ -25,13 +25,13 @@ define <16 x float> @test_v16f32(<16 x float> %a) {
 define <8 x float> @test_v8f32(<8 x float> %a) {
 ; CHECK-LABEL: test_v8f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NEXT:    .reg .b32 %r<9>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_v8f32_param_0];
-; CHECK-NEXT:    ld.param.v2.b64 {%rd3, %rd4}, [test_v8f32_param_0+16];
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0+16], {%rd3, %rd4};
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [test_v8f32_param_0];
+; CHECK-NEXT:    ld.param.v4.b32 {%r5, %r6, %r7, %r8}, [test_v8f32_param_0+16];
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0+16], {%r5, %r6, %r7, %r8};
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r1, %r2, %r3, %r4};
 ; CHECK-NEXT:    ret;
   ret <8 x float> %a
 }
@@ -39,11 +39,11 @@ define <8 x float> @test_v8f32(<8 x float> %a) {
 define <4 x float> @test_v4f32(<4 x float> %a) {
 ; CHECK-LABEL: test_v4f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [test_v4f32_param_0];
-; CHECK-NEXT:    st.param.v2.b64 [func_retval0], {%rd1, %rd2};
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [test_v4f32_param_0];
+; CHECK-NEXT:    st.param.v4.b32 [func_retval0], {%r1, %r2, %r3, %r4};
 ; CHECK-NEXT:    ret;
   ret <4 x float> %a
 }
@@ -51,11 +51,11 @@ define <4 x float> @test_v4f32(<4 x float> %a) {
 define <2 x float> @test_v2f32(<2 x float> %a) {
 ; CHECK-LABEL: test_v2f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_v2f32_param_0];
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd1;
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_v2f32_param_0];
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r2};
 ; CHECK-NEXT:    ret;
   ret <2 x float> %a
 }
@@ -64,14 +64,13 @@ define <2 x float> @test_v2f32(<2 x float> %a) {
 define <3 x float> @test_v3f32(<3 x float> %a) {
 ; CHECK-LABEL: test_v3f32(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<2>;
+; CHECK-NEXT:    .reg .b32 %r<4>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [test_v3f32_param_0];
-; CHECK-NEXT:    ld.param.b32 %r1, [test_v3f32_param_0+8];
-; CHECK-NEXT:    st.param.b32 [func_retval0+8], %r1;
-; CHECK-NEXT:    st.param.b64 [func_retval0], %rd1;
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [test_v3f32_param_0];
+; CHECK-NEXT:    ld.param.b32 %r3, [test_v3f32_param_0+8];
+; CHECK-NEXT:    st.param.b32 [func_retval0+8], %r3;
+; CHECK-NEXT:    st.param.v2.b32 [func_retval0], {%r1, %r2};
 ; CHECK-NEXT:    ret;
   ret <3 x float> %a
 }
diff --git a/llvm/test/CodeGen/NVPTX/vector-loads.ll b/llvm/test/CodeGen/NVPTX/vector-loads.ll
index 6f0dff78d5569..ccac7ff8e6472 100644
--- a/llvm/test/CodeGen/NVPTX/vector-loads.ll
+++ b/llvm/test/CodeGen/NVPTX/vector-loads.ll
@@ -206,18 +206,18 @@ define void @extv8f16_global_a16(ptr addrspace(1) noalias readonly align 16 %dst
 ; CHECK-NEXT:    ld.param.b64 %rd1, [extv8f16_global_a16_param_0];
 ; CHECK-NEXT:    ld.param.b64 %rd2, [extv8f16_global_a16_param_1];
 ; CHECK-NEXT:    ld.global.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NEXT:    cvt.f32.f16 %r5, %rs2;
-; CHECK-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NEXT:    cvt.f32.f16 %r7, %rs4;
-; CHECK-NEXT:    cvt.f32.f16 %r8, %rs3;
-; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
-; CHECK-NEXT:    cvt.f32.f16 %r9, %rs6;
-; CHECK-NEXT:    cvt.f32.f16 %r10, %rs5;
-; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r3;
-; CHECK-NEXT:    cvt.f32.f16 %r11, %rs8;
-; CHECK-NEXT:    cvt.f32.f16 %r12, %rs7;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; CHECK-NEXT:    cvt.f32.f16 %r5, %rs8;
+; CHECK-NEXT:    cvt.f32.f16 %r6, %rs7;
+; CHECK-NEXT:    cvt.f32.f16 %r7, %rs6;
+; CHECK-NEXT:    cvt.f32.f16 %r8, %rs5;
+; CHECK-NEXT:    cvt.f32.f16 %r9, %rs4;
+; CHECK-NEXT:    cvt.f32.f16 %r10, %rs3;
+; CHECK-NEXT:    cvt.f32.f16 %r11, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r12, %rs1;
 ; CHECK-NEXT:    st.global.v4.b32 [%rd1+16], {%r12, %r11, %r10, %r9};
 ; CHECK-NEXT:    st.global.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
 ; CHECK-NEXT:    ret;
@@ -270,18 +270,18 @@ define void @extv8f16_generic_a16(ptr noalias readonly align 16 %dst, ptr noalia
 ; CHECK-NEXT:    ld.param.b64 %rd1, [extv8f16_generic_a16_param_0];
 ; CHECK-NEXT:    ld.param.b64 %rd2, [extv8f16_generic_a16_param_1];
 ; CHECK-NEXT:    ld.v4.b32 {%r1, %r2, %r3, %r4}, [%rd2];
-; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
-; CHECK-NEXT:    cvt.f32.f16 %r5, %rs2;
-; CHECK-NEXT:    cvt.f32.f16 %r6, %rs1;
-; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
-; CHECK-NEXT:    cvt.f32.f16 %r7, %rs4;
-; CHECK-NEXT:    cvt.f32.f16 %r8, %rs3;
-; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r4;
-; CHECK-NEXT:    cvt.f32.f16 %r9, %rs6;
-; CHECK-NEXT:    cvt.f32.f16 %r10, %rs5;
-; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r3;
-; CHECK-NEXT:    cvt.f32.f16 %r11, %rs8;
-; CHECK-NEXT:    cvt.f32.f16 %r12, %rs7;
+; CHECK-NEXT:    mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-NEXT:    mov.b32 {%rs3, %rs4}, %r4;
+; CHECK-NEXT:    mov.b32 {%rs5, %rs6}, %r1;
+; CHECK-NEXT:    mov.b32 {%rs7, %rs8}, %r2;
+; CHECK-NEXT:    cvt.f32.f16 %r5, %rs8;
+; CHECK-NEXT:    cvt.f32.f16 %r6, %rs7;
+; CHECK-NEXT:    cvt.f32.f16 %r7, %rs6;
+; CHECK-NEXT:    cvt.f32.f16 %r8, %rs5;
+; CHECK-NEXT:    cvt.f32.f16 %r9, %rs4;
+; CHECK-NEXT:    cvt.f32.f16 %r10, %rs3;
+; CHECK-NEXT:    cvt.f32.f16 %r11, %rs2;
+; CHECK-NEXT:    cvt.f32.f16 %r12, %rs1;
 ; CHECK-NEXT:    st.v4.b32 [%rd1+16], {%r12, %r11, %r10, %r9};
 ; CHECK-NEXT:    st.v4.b32 [%rd1], {%r8, %r7, %r6, %r5};
 ; CHECK-NEXT:    ret;
diff --git a/llvm/test/CodeGen/NVPTX/vector-stores.ll b/llvm/test/CodeGen/NVPTX/vector-stores.ll
index d07c740d32a72..b9bb417aa2c37 100644
--- a/llvm/test/CodeGen/NVPTX/vector-stores.ll
+++ b/llvm/test/CodeGen/NVPTX/vector-stores.ll
@@ -5,12 +5,13 @@
 define void @foo1(<2 x float> %val, ptr %ptr) {
 ; CHECK-LABEL: foo1(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.b64 %rd1, [foo1_param_0];
-; CHECK-NEXT:    ld.param.b64 %rd2, [foo1_param_1];
-; CHECK-NEXT:    st.b64 [%rd2], %rd1;
+; CHECK-NEXT:    ld.param.v2.b32 {%r1, %r2}, [foo1_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo1_param_1];
+; CHECK-NEXT:    st.v2.b32 [%rd1], {%r1, %r2};
 ; CHECK-NEXT:    ret;
   store <2 x float> %val, ptr %ptr
   ret void
@@ -19,12 +20,13 @@ define void @foo1(<2 x float> %val, ptr %ptr) {
 define void @foo2(<4 x float> %val, ptr %ptr) {
 ; CHECK-LABEL: foo2(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
+; CHECK-NEXT:    .reg .b64 %rd<2>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param.v2.b64 {%rd1, %rd2}, [foo2_param_0];
-; CHECK-NEXT:    ld.param.b64 %rd3, [foo2_param_1];
-; CHECK-NEXT:    st.v2.b64 [%rd3], {%rd1, %rd2};
+; CHECK-NEXT:    ld.param.v4.b32 {%r1, %r2, %r3, %r4}, [foo2_param_0];
+; CHECK-NEXT:    ld.param.b64 %rd1, [foo2_param_1];
+; CHECK-NEXT:    st.v4.b32 [%rd1], {%r1, %r2, %r3, %r4};
 ; CHECK-NEXT:    ret;
   store <4 x float> %val, ptr %ptr
   ret void
diff --git a/llvm/test/CodeGen/NVPTX/wmma-ptx87-sm120a.py b/llvm/test/CodeGen/NVPTX/wmma-ptx87-sm120a.py
new file mode 100644
index 0000000000000..ae781df0116fd
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/wmma-ptx87-sm120a.py
@@ -0,0 +1,12 @@
+# Check all variants of instructions supported by PTX87 on SM120a
+# RUN: %python %s --ptx=87 --gpu-arch=120 --aa > %t-ptx87-sm_120a.ll
+# RUN: llc < %t-ptx87-sm_120a.ll -mtriple=nvptx64 -mcpu=sm_120a -mattr=+ptx87 \
+# RUN:           | FileCheck %t-ptx87-sm_120a.ll
+# RUN: %if ptxas-12.7 %{                                                  \
+# RUN: llc < %t-ptx87-sm_120a.ll -mtriple=nvptx64 -mcpu=sm_120a -mattr=+ptx87 \
+# RUN:           | %ptxas-verify -arch=sm_120a                              \
+# RUN: %}
+
+import wmma
+
+wmma.main()
diff --git a/llvm/test/CodeGen/NVPTX/wmma.py b/llvm/test/CodeGen/NVPTX/wmma.py
index 2eb3c3dbb4c39..6d73bce46da7c 100644
--- a/llvm/test/CodeGen/NVPTX/wmma.py
+++ b/llvm/test/CodeGen/NVPTX/wmma.py
@@ -15,6 +15,11 @@ class MMAType:
     def __init__(self, ptx_type):
         self.ptx_type = ptx_type
         self.llvm_type = {
+            "e4m3": "i32",
+            "e5m2": "i32",
+            "e3m2": "i32",
+            "e2m3": "i32",
+            "e2m1": "i32",
             "f16": "<2 x half>",
             "f32": "float",
             "f64": "double",
@@ -43,7 +48,7 @@ def __repr__(self):
 
 
 class MMAFrag:
-    def __init__(self, geom, frag, ptx_elt_type):
+    def __init__(self, geom, frag, ptx_elt_type, is_mma_sparse=False):
         self.geom = geom
         self.frag = frag
         self.mma_type = MMAType(ptx_elt_type)
@@ -79,12 +84,53 @@ def __init__(self, geom, frag, ptx_elt_type):
             "m16n8k16:b:s8": 1,
             "m16n8k16:c:s32": 4,
             "m16n8k16:d:s32": 4,
-            "m16n8k32:a:u8": 4,
-            "m16n8k32:a:s8": 4,
+            "m16n8k32:a:u8": 2 if is_mma_sparse else 4,
+            "m16n8k32:a:s8": 2 if is_mma_sparse else 4,
             "m16n8k32:b:u8": 2,
             "m16n8k32:b:s8": 2,
             "m16n8k32:c:s32": 4,
             "m16n8k32:d:s32": 4,
+            # mma sp
+            "m16n8k32:a:bf16": 4,
+            "m16n8k32:a:f16": 4,
+            "m16n8k32:b:bf16": 4,
+            "m16n8k32:b:f16": 4,
+            "m16n8k32:c:f16": 2,
+            "m16n8k32:c:f32": 4,
+            "m16n8k32:d:f16": 2,
+            "m16n8k32:d:f32": 4,
+            "m16n8k16:a:tf32": 4,
+            "m16n8k16:b:tf32": 4,
+            "m16n8k16:c:tf32": 4,
+            "m16n8k16:d:tf32": 4,
+            "m16n8k64:a:u8": 4,
+            "m16n8k64:a:s8": 4,
+            "m16n8k64:a:e4m3": 4,
+            "m16n8k64:a:e5m2": 4,
+            "m16n8k64:a:e3m2": 4,
+            "m16n8k64:a:e2m3": 4,
+            "m16n8k64:a:e2m1": 4,
+            "m16n8k64:b:u8": 4,
+            "m16n8k64:b:s8": 4,
+            "m16n8k64:b:e4m3": 4,
+            "m16n8k64:b:e5m2": 4,
+            "m16n8k64:b:e3m2": 4,
+            "m16n8k64:b:e2m3": 4,
+            "m16n8k64:b:e2m1": 4,
+            "m16n8k64:c:f16": 2,
+            "m16n8k64:c:f32": 4,
+            "m16n8k64:d:f16": 2,
+            "m16n8k64:d:f32": 4,
+            "m16n8k128:a:u4": 4,
+            "m16n8k128:a:s4": 4,
+            "m16n8k128:a:e2m1": 4,
+            "m16n8k128:b:u4": 4,
+            "m16n8k128:b:s4": 4,
+            "m16n8k128:b:e2m1": 4,
+            "m16n8k128:c:s32": 4,
+            "m16n8k128:c:f32": 4,
+            "m16n8k128:d:s32": 4,
+            "m16n8k128:d:f32": 4,
             # u4/s4 -> s32 @ m8n8k32 (u4/s4)
             "m8n8k32:a:u4": 1,
             "m8n8k32:a:s4": 1,
@@ -98,8 +144,8 @@ def __init__(self, geom, frag, ptx_elt_type):
             "m16n8k32:b:s4": 1,
             "m16n8k32:c:s32": 4,
             "m16n8k32:d:s32": 4,
-            "m16n8k64:a:u4": 4,
-            "m16n8k64:a:s4": 4,
+            "m16n8k64:a:u4": 2 if is_mma_sparse else 4,
+            "m16n8k64:a:s4": 2 if is_mma_sparse else 4,
             "m16n8k64:b:u4": 2,
             "m16n8k64:b:s4": 2,
             "m16n8k64:c:s32": 4,
@@ -124,7 +170,7 @@ def __init__(self, geom, frag, ptx_elt_type):
             "m8n32k16:b:bf16": 8,
             "m32n8k16:a:bf16": 8,
             "m32n8k16:b:bf16": 2,
-            "m16n8k16:a:bf16": 4,
+            "m16n8k16:a:bf16": 2 if is_mma_sparse else 4,
             "m16n8k16:b:bf16": 2,
             "m16n8k16:c:f32": 4,
             "m16n8k16:d:f32": 4,
@@ -143,7 +189,7 @@ def __init__(self, geom, frag, ptx_elt_type):
             "m16n8k4:b:tf32": 1,
             "m16n8k4:c:f32": 4,
             "m16n8k4:d:f32": 4,
-            "m16n8k8:a:tf32": 4,
+            "m16n8k8:a:tf32": 2 if is_mma_sparse else 4,
             "m16n8k8:b:tf32": 2,
             "m16n8k8:c:f32": 4,
             "m16n8k8:d:f32": 4,
@@ -155,7 +201,7 @@ def __init__(self, geom, frag, ptx_elt_type):
             "m16n8k8:d:f16": 2,
             "m16n8k8:c:f32": 4,
             "m16n8k8:d:f32": 4,
-            "m16n8k16:a:f16": 4,
+            "m16n8k16:a:f16": 2 if is_mma_sparse else 4,
             "m16n8k16:b:f16": 2,
             "m16n8k16:c:f16": 2,
             "m16n8k16:d:f16": 2,
@@ -218,7 +264,7 @@ def __repr__(self):
         return "{A:%s, B:%s, C:%s, D:%s}" % (self.a, self.b, self.c, self.d)
 
 
-def make_mma_ops(geoms, types_a, types_b, types_c, types_d):
+def make_mma_ops(geoms, types_a, types_b, types_c, types_d, is_mma_sparse=False):
     ops = []
     for geom, type_a, type_c in product(geoms, types_a, types_c):
         for type_b, type_d in product(
@@ -226,10 +272,10 @@ def make_mma_ops(geoms, types_a, types_b, types_c, types_d):
         ):
             ops.append(
                 MMAOp(
-                    MMAFrag(geom, "a", type_a),
-                    MMAFrag(geom, "b", type_b),
-                    MMAFrag(geom, "c", type_c),
-                    MMAFrag(geom, "d", type_d),
+                    MMAFrag(geom, "a", type_a, is_mma_sparse),
+                    MMAFrag(geom, "b", type_b, is_mma_sparse),
+                    MMAFrag(geom, "c", type_c, is_mma_sparse),
+                    MMAFrag(geom, "d", type_d, is_mma_sparse),
                 )
             )
     return ops
@@ -416,6 +462,10 @@ def is_type_supported(ptx_type):
         return ptx_version >= 65 and gpu_arch >= 75
     if ptx_type in ["bf16", "tf32", "f64"]:
         return ptx_version >= 70
+    if ptx_type in ["e4m3", "e5m2"]:
+        return ptx_version >= 84 and gpu_arch >= 89
+    if ptx_type in ["e3m2", "e2m3", "e2m1"]:
+        return ptx_version >= 87 and gpu_arch >= 120 and aa
     return ptx_version >= 60 and gpu_arch >= 70
 
 
@@ -448,7 +498,7 @@ def is_mma_variant_supported(op, layout_a, layout_b, satf):
     ):
         return False
 
-    if satf and not op.a.mma_type.ptx_type in ["s8", "u8", "s4", "u4"]:
+    if satf and op.a.mma_type.ptx_type not in ["s8", "u8", "s4", "u4"]:
         return False
 
     # If the type of C is f32 then so must the type of D
@@ -825,7 +875,15 @@ def gen_stmatrix_tests():
     return generated_items
 
 def mma_signature(op):
-    if op.a.mma_type.ptx_type == "f16":
+    if op.a.mma_type.ptx_type in ["e4m3", "e5m2", "e3m2", "e2m3", "e2m1"]:
+        # FP8/F8F6F4 ops identified by inputs, accumulator & result types.
+        return "%s.%s.%s.%s" % (
+            op.d.mma_type.ptx_type,
+            op.a.mma_type.ptx_type,
+            op.b.mma_type.ptx_type,
+            op.c.mma_type.ptx_type,
+        )
+    elif op.a.mma_type.ptx_type == "f16":
         # FP16 ops identified by accumulator & result type.
         return "%s.%s" % (op.d.mma_type.ptx_type, op.c.mma_type.ptx_type)
     elif op.a.mma_type.ptx_type != op.b.mma_type.ptx_type:
@@ -980,6 +1038,230 @@ def gen_mma_tests():
     return generated_items
 
 
+def get_mma_sp_ops():
+    return (
+        make_mma_ops(["m16n8k16", "m16n8k32"], ["bf16"], [], ["f32"], [], True)
+        + make_mma_ops(["m16n8k8", "m16n8k16"], ["tf32"], [], ["f32"], [], True)
+        + make_mma_ops(
+            ["m16n8k16", "m16n8k32"],
+            ["f16"],
+            [],
+            ["f16", "f32"],
+            ["f16", "f32"],
+            True,
+        )
+        + make_mma_ops(
+            ["m16n8k64", "m16n8k128"], ["s4", "u4"], ["s4", "u4"], ["s32"], [], True
+        )
+        + make_mma_ops(
+            ["m16n8k32", "m16n8k64"], ["s8", "u8"], ["s8", "u8"], ["s32"], [], True
+        )
+        + make_mma_ops(
+            ["m16n8k64"],
+            ["e4m3", "e5m2", "e3m2", "e2m3", "e2m1"],
+            ["e4m3", "e5m2", "e3m2", "e2m3", "e2m1"],
+            ["f16", "f32"],
+            ["f16", "f32"],
+            True,
+        )
+    )
+
+
+def is_mma_sp_geom_supported(geom):
+    # geometries for FP and ints.
+    if geom in [
+        "m16n8k16",
+        "m16n8k32",
+        "m16n8k8",
+        "m16n8k64",
+        "m16n8k128",
+    ]:
+        return ptx_version >= 71
+    raise ValueError(f"Unexpected sparse MMA geometry: {geom}")
+
+
+def is_mma_sp_variant_supported(op, metadata, kind, satf):
+    if metadata != "sp" and (ptx_version < 85 or gpu_arch < 80):
+        return False
+
+    if kind != "" and (ptx_version < 87 or gpu_arch < 120 or not aa):
+        return False
+
+    if not (
+        is_type_supported(op.a.mma_type.ptx_type)
+        and is_mma_sp_geom_supported(op.a.geom)
+    ):
+        return False
+
+    is_int = op.a.mma_type.ptx_type in ["s8", "u8", "s4", "u4"]
+
+    if satf and not is_int:
+        return False
+
+    # A and B type must be the same
+    if (
+        op.a.mma_type.ptx_type in ["f16", "bf16", "tf32"]
+        and op.a.mma_type.ptx_type != op.b.mma_type.ptx_type
+    ):
+        return False
+
+    # C and D type must be the same for m16n8k16/m16n8k32
+    if (
+        op.a.geom in ["m16n8k16", "m16n8k32"]
+        and op.c.mma_type.ptx_type != op.d.mma_type.ptx_type
+    ):
+        return False
+
+    if kind == "" and (
+        op.a.mma_type.ptx_type in ["e3m2", "e2m3", "e2m1"]
+        or op.b.mma_type.ptx_type in ["e3m2", "e2m3", "e2m1"]
+    ):
+        return False
+
+    if (
+        kind == ""
+        and op.a.geom == "m16n8k64"
+        and (op.c.mma_type.ptx_type == "f16" or op.d.mma_type.ptx_type == "f16")
+    ):
+        return False
+
+    if kind != "" and (metadata == "sp" or op.a.geom != "m16n8k64" or is_int):
+        return False
+
+    return True
+
+
+def sp_selector_gen(op):
+    # (geom, type) -> allowed selector range
+    range_01 = {
+        ("m16n8k32", "bf16"),
+        ("m16n8k32", "f16"),
+        ("m16n8k16", "tf32"),
+        ("m16n8k32", "u8"),
+        ("m16n8k32", "s8"),
+        ("m16n8k64", "u4"),
+        ("m16n8k64", "s4"),
+    }
+
+    if (op.a.geom, op.a.mma_type.ptx_type) in range_01:
+        return range(2)
+    if op.a.geom == "m16n8k64" and op.a.mma_type.ptx_type in [
+        "u8",
+        "s8",
+        "e4m3",
+        "e5m2",
+        "e3m2",
+        "e2m3",
+        "e2m1",
+    ]:
+        return range(1)
+    if op.a.geom == "m16n8k128" and op.a.mma_type.ptx_type in [
+        "u4",
+        "s4",
+    ]:
+        return range(1)
+    return range(4)
+
+
+def common_mma_sp_test_gen(params, op, intrinsic_template, instruction_template):
+    mma_sp_decl_template = """
+declare ${ret_ty} @${intrinsic}(
+        ${args});
+"""
+
+    mma_sp_test_template = """
+; CHECK-LABEL: .func {{.*}}test_${function}_${selector}(
+define ${ret_ty} @test_${function}_${selector}(
+        ${args}) {
+; CHECK: ${instruction}
+; CHECK-NEXT: ${check_d}
+; CHECK-NEXT: ${check_a}
+; CHECK-NEXT: ${check_b}
+; CHECK-NEXT: ${check_c}
+; CHECK-NEXT: ${check_metadata}
+; CHECK-NEXT: ${check_selector}
+  %r = call ${ret_ty} @${intrinsic}(
+        ${call_args});
+  ret ${ret_ty} %r;
+}
+"""
+
+    test_params = params
+    test_params["intrinsic"] = (
+        Template(intrinsic_template)
+        .substitute(params)
+        .replace("::", ".")
+        .replace("_", ".")
+    )
+    test_params["function"] = test_params["intrinsic"].replace(".", "_")
+    test_params["instruction"] = Template(instruction_template).substitute(params)
+    test_params["ret_ty"] = make_wmma_ld_ret_ty(op.d)
+    test_params["check_a"] = check_pattern(op.a)
+    test_params["check_b"] = check_pattern(op.b)
+    test_params["check_c"] = check_pattern(op.c)
+    test_params["check_d"] = check_pattern(op.d)
+    test_params["check_metadata"] = "{{%r[0-9]+}}"
+    args = ",\n        ".join(
+        list(make_wmma_slice_args(frag) for frag in (op.a, op.b, op.c))
+        + ["i32 %metadata", "i32 %selector"]
+    )
+    test_params["args"] = args
+
+    print(Template(mma_sp_decl_template).substitute(test_params))
+
+    for selector in [str(r) for r in sp_selector_gen(op)]:
+        test_params["selector"] = selector
+        test_params["check_selector"] = "{{" + test_params["selector"] + "}}"
+        test_params["call_args"] = test_params["args"].replace(
+            "%selector", test_params["selector"]
+        )
+
+        print(Template(mma_sp_test_template).substitute(test_params))
+
+    return (test_params["intrinsic"], test_params["instruction"])
+
+
+def gen_mma_sp_tests():
+    if ptx_version < 71 or gpu_arch < 80:
+        return []
+
+    mma_sp_intrinsic_template = (
+        "llvm.nvvm.mma.${metadata}.${geom}.row.col${kind}${satf}.${intrinsic_signature}"
+    )
+    mma_sp_instruction_template = (
+        "mma.${metadata}.sync.aligned.${geom}.row.col${kind}${satf}.${ptx_signature}"
+    )
+
+    generated_items = []
+
+    for op, metadata, kind, satf in product(
+        get_mma_sp_ops(),
+        ["sp::ordered_metadata", "sp"],
+        ["", ".kind::f8f6f4"],
+        [".satfinite", ""],
+    ):
+        if not is_mma_sp_variant_supported(op, metadata, kind, satf):
+            continue
+
+        params = {
+            "intrinsic_signature": mma_signature(op),
+            "ptx_signature": mma_ptx_signature(op),
+            "satf": satf,
+            "geom": op.a.geom,
+            "metadata": metadata,
+            "kind": kind,
+        }
+
+        intrinsic_template = mma_sp_intrinsic_template
+        instruction_template = mma_sp_instruction_template
+
+        generated_items.append(
+            common_mma_sp_test_gen(params, op, intrinsic_template, instruction_template)
+        )
+
+    return generated_items
+
+
 # Append complete list of intrinsics and instructions we've generated tests for.
 # Generate set of checks to verify that that we did generate sensible set of
 # tests for the given combination of PTX and SM variants.
@@ -1170,6 +1452,7 @@ def gen_tests():
     items += gen_stmatrix_tests()
     items += gen_wmma_mma_tests()
     items += gen_mma_tests()
+    items += gen_mma_sp_tests()
     gen_check_unsupported_ops(items)
 
 
diff --git a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessNoProfileData.mir b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessNoProfileData.mir
index d3d3b194b3ce5..a23f0af1adcb9 100644
--- a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessNoProfileData.mir
+++ b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessNoProfileData.mir
@@ -123,8 +123,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
index e8ad54b9c6cd4..7d80c025389a5 100644
--- a/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
+++ b/llvm/test/CodeGen/PowerPC/DisableHoistingDueToBlockHotnessProfileData.mir
@@ -168,8 +168,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir b/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
index 41e21248a3f0e..60a399d49bd1f 100644
--- a/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
+++ b/llvm/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
@@ -1,6 +1,12 @@
 # RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -start-after \
 # RUN:   virtregrewriter -ppc-asm-full-reg-names -verify-machineinstrs %s \
 # RUN:   -o - | FileCheck %s
+# RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu -start-after \
+# RUN:   virtregrewriter -ppc-asm-full-reg-names -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+# RUN: llc -mcpu=pwr10 -mtriple=powerpc64le-unknown-linux-gnu -start-after \
+# RUN:   virtregrewriter -ppc-asm-full-reg-names -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
 
 --- |
   ; ModuleID = 'a.ll'
@@ -30,7 +36,7 @@
   ; Function Attrs: nounwind
   declare void @llvm.stackprotector(ptr, ptr) #1
   
-  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { nounwind }
   
   !llvm.ident = !{!0}
@@ -71,8 +77,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           
   - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-abi-mir.ll b/llvm/test/CodeGen/PowerPC/aix-cc-abi-mir.ll
index 9ffb4fd5eae45..258ddf60088c1 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-abi-mir.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-abi-mir.ll
@@ -37,9 +37,9 @@ define signext i8 @test_chars(i8 signext %c1, i8 signext %c2, i8 signext %c3, i8
   ; 32BIT: bb.0.entry:
   ; 32BIT-NEXT:   liveins: $r3, $r4, $r5, $r6
   ; 32BIT-NEXT: {{  $}}
-  ; 32BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, killed renamable $r4
-  ; 32BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, killed renamable $r5
-  ; 32BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, killed renamable $r6
+  ; 32BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4
+  ; 32BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r5
+  ; 32BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r6
   ; 32BIT-NEXT:   renamable $r3 = EXTSB killed renamable $r3
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm, implicit $r3
   ;
@@ -47,9 +47,9 @@ define signext i8 @test_chars(i8 signext %c1, i8 signext %c2, i8 signext %c3, i8
   ; 64BIT: bb.0.entry:
   ; 64BIT-NEXT:   liveins: $x3, $x4, $x5, $x6
   ; 64BIT-NEXT: {{  $}}
-  ; 64BIT-NEXT:   renamable $r3 = ADD4 renamable $r3, renamable $r4, implicit killed $x4, implicit killed $x3
-  ; 64BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, renamable $r5, implicit killed $x5
-  ; 64BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, renamable $r6, implicit killed $x6, implicit-def $x3
+  ; 64BIT-NEXT:   renamable $r3 = nsw ADD4 renamable $r3, renamable $r4, implicit killed $x4, implicit killed $x3
+  ; 64BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r5, implicit killed $x5
+  ; 64BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r6, implicit killed $x6, implicit-def $x3
   ; 64BIT-NEXT:   renamable $x3 = EXTSB8 killed renamable $x3
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm, implicit $x3
 entry:
@@ -96,9 +96,9 @@ define signext i8 @test_chars_mix(i8 signext %c1, i8 zeroext %c2, i8 zeroext %c3
   ; 32BIT: bb.0.entry:
   ; 32BIT-NEXT:   liveins: $r3, $r4, $r5, $r6
   ; 32BIT-NEXT: {{  $}}
-  ; 32BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, killed renamable $r4
-  ; 32BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, killed renamable $r5
-  ; 32BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, killed renamable $r6
+  ; 32BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4
+  ; 32BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r5
+  ; 32BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r6
   ; 32BIT-NEXT:   renamable $r3 = EXTSB killed renamable $r3
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm, implicit $r3
   ;
@@ -106,9 +106,9 @@ define signext i8 @test_chars_mix(i8 signext %c1, i8 zeroext %c2, i8 zeroext %c3
   ; 64BIT: bb.0.entry:
   ; 64BIT-NEXT:   liveins: $x3, $x4, $x5, $x6
   ; 64BIT-NEXT: {{  $}}
-  ; 64BIT-NEXT:   renamable $r3 = ADD4 renamable $r3, renamable $r4, implicit killed $x4, implicit killed $x3
-  ; 64BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, renamable $r5, implicit killed $x5
-  ; 64BIT-NEXT:   renamable $r3 = ADD4 killed renamable $r3, renamable $r6, implicit killed $x6, implicit-def $x3
+  ; 64BIT-NEXT:   renamable $r3 = nsw ADD4 renamable $r3, renamable $r4, implicit killed $x4, implicit killed $x3
+  ; 64BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r5, implicit killed $x5
+  ; 64BIT-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r6, implicit killed $x6, implicit-def $x3
   ; 64BIT-NEXT:   renamable $x3 = EXTSB8 killed renamable $x3
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm, implicit $x3
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/aix-nest-param.ll b/llvm/test/CodeGen/PowerPC/aix-nest-param.ll
index 1863eaf999f14..bfc7fbb374f1b 100644
--- a/llvm/test/CodeGen/PowerPC/aix-nest-param.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-nest-param.ll
@@ -1,5 +1,5 @@
-; RUN: not --crash llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
-; RUN: not --crash llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
 
 define ptr @nest_receiver(ptr nest %arg) nounwind {
   ret ptr %arg
@@ -9,5 +9,10 @@ define ptr @nest_caller(ptr %arg) nounwind {
   %result = call ptr @nest_receiver(ptr nest %arg)
   ret ptr %result
 }
+; CHECK-LABEL: .nest_receiver:
+; CHECK:         mr      3, 11
+; CHECK:         blr
 
-; CHECK: LLVM ERROR: Nest arguments are unimplemented.
+; CHECK-LABEL: .nest_caller:
+; CHECK:         mr      11, 3
+; CHECK:         bl .nest_receiver
diff --git a/llvm/test/CodeGen/PowerPC/aix-trampoline.ll b/llvm/test/CodeGen/PowerPC/aix-trampoline.ll
index b71f6b54587c5..19df220178e3b 100644
--- a/llvm/test/CodeGen/PowerPC/aix-trampoline.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-trampoline.ll
@@ -1,7 +1,7 @@
-; RUN: not --crash llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
-; RUN: not --crash llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 | FileCheck %s
-
-; CHECK: LLVM ERROR: INIT_TRAMPOLINE operation is not supported on AIX.
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff < %s 2>&1 | \
+; RUN: FileCheck %s --check-prefix=32BIT
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff < %s 2>&1 -mattr=-altivec | \
+; RUN: FileCheck %s --check-prefix=64BIT
 
 define void @create_trampoline(ptr %buffer, ptr %nval) nounwind {
 entry:
@@ -12,3 +12,17 @@ entry:
 declare i32 @nested(i32);
 
 declare void @llvm.init.trampoline(ptr, ptr, ptr) nounwind
+
+; 32BIT:     stw 4, 8(3)
+; 32BIT:     lwz [[FuncDesc:[0-9]+]], L..C0(2)
+; 32BIT-DAG: lwz [[SCRATCH1:[0-9]+]], 0([[FuncDesc]])
+; 32BIT-DAG: lwz [[SCRATCH2:[0-9]+]], 4([[FuncDesc]])
+; 32BIT-DAG: stw [[SCRATCH1]], 0(3)
+; 32BIT-DAG: stw [[SCRATCH2]], 4(3)
+
+; 64BIT:     std 4, 16(3)
+; 64BIT-DAG: ld [[FuncDesc:[0-9]+]], L..C0(2)
+; 64BIT-DAG: ld [[SCRATCH1:[0-9]+]], 0([[FuncDesc]])
+; 64BIT-DAG: ld [[SCRATCH2:[0-9]+]], 8([[FuncDesc]])
+; 64BIT-DAG: std [[SCRATCH1]], 0(3)
+; 64BIT-DAG: std [[SCRATCH2]], 8(3)
diff --git a/llvm/test/CodeGen/PowerPC/alignlongjumptest.mir b/llvm/test/CodeGen/PowerPC/alignlongjumptest.mir
index 314844a2f3bf8..f1ae4a2158700 100644
--- a/llvm/test/CodeGen/PowerPC/alignlongjumptest.mir
+++ b/llvm/test/CodeGen/PowerPC/alignlongjumptest.mir
@@ -43,8 +43,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/PowerPC/block-placement-1.mir b/llvm/test/CodeGen/PowerPC/block-placement-1.mir
index f91ab630112ca..a74af48f01dcf 100644
--- a/llvm/test/CodeGen/PowerPC/block-placement-1.mir
+++ b/llvm/test/CodeGen/PowerPC/block-placement-1.mir
@@ -140,8 +140,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -186,8 +186,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -80, size: 8, alignment: 16, stack-id: default, 
       callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/CodeGen/PowerPC/block-placement.mir b/llvm/test/CodeGen/PowerPC/block-placement.mir
index dab8dfbb7c37c..99d399d1e0c22 100644
--- a/llvm/test/CodeGen/PowerPC/block-placement.mir
+++ b/llvm/test/CodeGen/PowerPC/block-placement.mir
@@ -111,8 +111,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/PowerPC/builtins-bcd-format-conversion.ll b/llvm/test/CodeGen/PowerPC/builtins-bcd-format-conversion.ll
new file mode 100644
index 0000000000000..ede86254b1516
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/builtins-bcd-format-conversion.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown -mcpu=pwr9 \
+; RUN:  --ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-unknown -mcpu=pwr9 \
+; RUN:     --ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=pwr9 -mtriple=powerpc64-ibm-aix-xcoff \
+; RUN:   -ppc-asm-full-reg-names  < %s | FileCheck %s
+
+define dso_local <16 x i8> @test_bcdcopysign(<16 x i8> noundef %a, <16 x i8> noundef %b) {
+; CHECK-LABEL: test_bcdcopysign:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdcpsgn. v2, v2, v3
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.bcdcopysign(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %0
+}
+
+define dso_local <16 x i8> @test_bcdsetsign_imm0(<16 x i8> noundef %a) {
+; CHECK-LABEL: test_bcdsetsign_imm0:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdsetsgn. v2, v2, 0
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.bcdsetsign(<16 x i8> %a, i32 0)
+  ret <16 x i8> %0
+}
+
+define dso_local <16 x i8> @test_bcdsetsign_imm1(<16 x i8> noundef %a) {
+; CHECK-LABEL: test_bcdsetsign_imm1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    bcdsetsgn. v2, v2, 1
+; CHECK-NEXT:    blr
+entry:
+  %0 = tail call <16 x i8> @llvm.ppc.bcdsetsign(<16 x i8> %a, i32 1)
+  ret <16 x i8> %0
+}
+
+declare <16 x i8> @llvm.ppc.bcdcopysign(<16 x i8>, <16 x i8>)
+declare <16 x i8> @llvm.ppc.bcdsetsign(<16 x i8>, i32)
diff --git a/llvm/test/CodeGen/PowerPC/collapse-rotates.mir b/llvm/test/CodeGen/PowerPC/collapse-rotates.mir
index 938b27f19d5ca..b30b16108b524 100644
--- a/llvm/test/CodeGen/PowerPC/collapse-rotates.mir
+++ b/llvm/test/CodeGen/PowerPC/collapse-rotates.mir
@@ -45,8 +45,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
index e1d028548803f..fac09d2bdb9ea 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-R0-special-handling.mir
@@ -111,8 +111,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -165,8 +165,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -219,8 +219,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -272,8 +272,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -322,8 +322,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -370,8 +370,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -417,8 +417,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
index cdd6be56b46d5..0b61455b05239 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs-out-of-range.mir
@@ -242,8 +242,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -292,8 +292,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -348,8 +348,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -411,8 +411,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -470,8 +470,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -528,8 +528,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -590,8 +590,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -648,8 +648,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -707,8 +707,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -765,8 +765,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -821,8 +821,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -876,8 +876,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -931,8 +931,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -986,8 +986,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -1040,8 +1040,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -1093,8 +1093,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -1146,8 +1146,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -1199,8 +1199,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -1252,8 +1252,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -1305,8 +1305,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
diff --git a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
index fa06dd551a0d4..61c0da64c001e 100644
--- a/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
+++ b/llvm/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
@@ -1044,8 +1044,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1100,8 +1100,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1160,8 +1160,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1221,8 +1221,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1280,8 +1280,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1335,8 +1335,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1385,8 +1385,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1439,8 +1439,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1494,8 +1494,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1548,8 +1548,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1602,8 +1602,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1655,8 +1655,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1708,8 +1708,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1762,8 +1762,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1818,8 +1818,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1877,8 +1877,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -1938,8 +1938,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2002,8 +2002,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2072,8 +2072,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2149,8 +2149,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2229,8 +2229,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2306,8 +2306,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2384,8 +2384,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2461,8 +2461,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2542,8 +2542,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2621,8 +2621,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2697,8 +2697,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2772,8 +2772,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2845,8 +2845,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2920,8 +2920,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -2993,8 +2993,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3077,8 +3077,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 16, alignment: 16,
@@ -3183,8 +3183,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3256,8 +3256,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3329,8 +3329,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3402,8 +3402,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3465,8 +3465,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3515,8 +3515,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3562,8 +3562,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3608,8 +3608,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3658,8 +3658,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3713,8 +3713,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3768,8 +3768,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3823,8 +3823,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3874,8 +3874,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3920,8 +3920,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -3970,8 +3970,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4024,8 +4024,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4078,8 +4078,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4131,8 +4131,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4183,8 +4183,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4235,8 +4235,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4284,8 +4284,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4339,8 +4339,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4404,8 +4404,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4467,8 +4467,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4522,8 +4522,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4575,8 +4575,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4628,8 +4628,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4681,8 +4681,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4739,8 +4739,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4797,8 +4797,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4859,8 +4859,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4917,8 +4917,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -4976,8 +4976,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5036,8 +5036,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5091,8 +5091,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5144,8 +5144,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5209,8 +5209,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5282,8 +5282,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5357,8 +5357,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5430,8 +5430,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5505,8 +5505,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5578,8 +5578,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5652,8 +5652,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5723,8 +5723,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5794,8 +5794,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5867,8 +5867,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -5938,8 +5938,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6011,8 +6011,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6074,8 +6074,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6126,8 +6126,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6178,8 +6178,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6236,8 +6236,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6297,8 +6297,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6351,8 +6351,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6401,8 +6401,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6448,8 +6448,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -6494,8 +6494,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/PowerPC/ctrloop-do-not-duplicate-mi.mir b/llvm/test/CodeGen/PowerPC/ctrloop-do-not-duplicate-mi.mir
index 651869d667380..668e7fe03ae04 100644
--- a/llvm/test/CodeGen/PowerPC/ctrloop-do-not-duplicate-mi.mir
+++ b/llvm/test/CodeGen/PowerPC/ctrloop-do-not-duplicate-mi.mir
@@ -117,8 +117,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/PowerPC/fp_to_uint_endian.ll b/llvm/test/CodeGen/PowerPC/fp_to_uint_endian.ll
new file mode 100644
index 0000000000000..f8f0611a7ca83
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/fp_to_uint_endian.ll
@@ -0,0 +1,17 @@
+; RUN: llc < %s -mcpu=440 -mtriple=ppc32le-unknown-unknown | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LE
+; RUN: llc < %s -mcpu=440 -mtriple=ppc32-unknown-unknown   | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BE
+
+define i32 @foo(double %a) {
+; CHECK-LABEL: foo:
+; CHECK-DAG:            fctiwz [[FPR_1_i:[0-9]+]], {{[0-9]+}}
+; CHECK-DAG:            stfd [[FPR_1_i]], [[#%u,VAL1_ADDR:]](1)
+; CHECK-LE-DAG:         lwz {{[0-9]+}}, [[#%u,== VAL1_ADDR]](1)
+; CHECK-BE-DAG:         lwz {{[0-9]+}}, [[#%u,== VAL1_ADDR + 4]](1)
+; CHECK-DAG:            fctiwz [[FPR_2:[0-9]+]], {{[0-9]+}}
+; CHECK-DAG:            stfd [[FPR_2]], [[#%u,VAL2_ADDR:]](1)
+; CHECK-LE-DAG:         lwz {{[0-9]+}}, [[#%u,== VAL2_ADDR]](1)
+; CHECK-BE-DAG:         lwz {{[0-9]+}}, [[#%u,== VAL2_ADDR + 4]](1)
+entry:
+        %tmp.1 = fptoui double %a to i32         ; <i32> [#uses=1]
+        ret i32 %tmp.1
+}
diff --git a/llvm/test/CodeGen/PowerPC/livevars-crash2.mir b/llvm/test/CodeGen/PowerPC/livevars-crash2.mir
index deaae3936cefb..c7a98f81b3e25 100644
--- a/llvm/test/CodeGen/PowerPC/livevars-crash2.mir
+++ b/llvm/test/CodeGen/PowerPC/livevars-crash2.mir
@@ -136,8 +136,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/PowerPC/llrint-conv.ll b/llvm/test/CodeGen/PowerPC/llrint-conv.ll
index daadf85b4085a..8e49ddcc6355f 100644
--- a/llvm/test/CodeGen/PowerPC/llrint-conv.ll
+++ b/llvm/test/CodeGen/PowerPC/llrint-conv.ll
@@ -1,10 +1,25 @@
 ; RUN: llc < %s -mtriple=powerpc64le | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc | FileCheck %s
+
+; FIXME: crash "Input type needs to be promoted!"
+; define signext i32 @testmswh(half %x) {
+; entry:
+;   %0 = tail call i64 @llvm.llrint.i64.f16(half %x)
+;   %conv = trunc i64 %0 to i32
+;   ret i32 %conv
+; }
+
+; define i64 @testmsxh(half %x) {
+; entry:
+;   %0 = tail call i64 @llvm.llrint.i64.f16(half %x)
+;   ret i64 %0
+; }
 
 ; CHECK-LABEL: testmsws:
 ; CHECK:       bl      llrintf
 define signext i32 @testmsws(float %x) {
 entry:
-  %0 = tail call i64 @llvm.llrint.f32(float %x)
+  %0 = tail call i64 @llvm.llrint.i64.f32(float %x)
   %conv = trunc i64 %0 to i32
   ret i32 %conv
 }
@@ -13,7 +28,7 @@ entry:
 ; CHECK:       bl      llrintf
 define i64 @testmsxs(float %x) {
 entry:
-  %0 = tail call i64 @llvm.llrint.f32(float %x)
+  %0 = tail call i64 @llvm.llrint.i64.f32(float %x)
   ret i64 %0
 }
 
@@ -21,7 +36,7 @@ entry:
 ; CHECK:       bl      llrint
 define signext i32 @testmswd(double %x) {
 entry:
-  %0 = tail call i64 @llvm.llrint.f64(double %x)
+  %0 = tail call i64 @llvm.llrint.i64.f64(double %x)
   %conv = trunc i64 %0 to i32
   ret i32 %conv
 }
@@ -30,7 +45,7 @@ entry:
 ; CHECK:       bl      llrint
 define i64 @testmsxd(double %x) {
 entry:
-  %0 = tail call i64 @llvm.llrint.f64(double %x)
+  %0 = tail call i64 @llvm.llrint.i64.f64(double %x)
   ret i64 %0
 }
 
@@ -38,7 +53,7 @@ entry:
 ; CHECK:       bl      llrintl
 define signext i32 @testmswl(ppc_fp128 %x) {
 entry:
-  %0 = tail call i64 @llvm.llrint.ppcf128(ppc_fp128 %x)
+  %0 = tail call i64 @llvm.llrint.i64.ppcf128(ppc_fp128 %x)
   %conv = trunc i64 %0 to i32
   ret i32 %conv
 }
@@ -47,10 +62,27 @@ entry:
 ; CHECK:       bl      llrintl
 define i64 @testmsll(ppc_fp128 %x) {
 entry:
-  %0 = tail call i64 @llvm.llrint.ppcf128(ppc_fp128 %x)
+  %0 = tail call i64 @llvm.llrint.i64.ppcf128(ppc_fp128 %x)
+  ret i64 %0
+}
+
+; CHECK-LABEL: testmswq:
+; CHECK:       bl      llrintf128
+define signext i32 @testmswq(fp128 %x) {
+entry:
+  %0 = tail call i64 @llvm.llrint.i64.f128(fp128 %x)
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: testmslq:
+; CHECK:       bl      llrintf128
+define i64 @testmslq(fp128 %x) {
+entry:
+  %0 = tail call i64 @llvm.llrint.i64.f128(fp128 %x)
   ret i64 %0
 }
 
-declare i64 @llvm.llrint.f32(float) nounwind readnone
-declare i64 @llvm.llrint.f64(double) nounwind readnone
-declare i64 @llvm.llrint.ppcf128(ppc_fp128) nounwind readnone
+declare i64 @llvm.llrint.i64.f32(float) nounwind readnone
+declare i64 @llvm.llrint.i64.f64(double) nounwind readnone
+declare i64 @llvm.llrint.i64.ppcf128(ppc_fp128) nounwind readnone
diff --git a/llvm/test/CodeGen/PowerPC/lrint-conv.ll b/llvm/test/CodeGen/PowerPC/lrint-conv.ll
index adfc994497323..bc77a200757f4 100644
--- a/llvm/test/CodeGen/PowerPC/lrint-conv.ll
+++ b/llvm/test/CodeGen/PowerPC/lrint-conv.ll
@@ -1,4 +1,19 @@
 ; RUN: llc < %s -mtriple=powerpc64le | FileCheck %s
+; RUN: llc < %s -mtriple=powerpc | FileCheck %s
+
+; FIXME: crash "Input type needs to be promoted!"
+; define signext i32 @testmswh(half %x) {
+; entry:
+;   %0 = tail call i64 @llvm.lrint.i64.f16(half %x)
+;   %conv = trunc i64 %0 to i32
+;   ret i32 %conv
+; }
+
+; define i64 @testmsxh(half %x) {
+; entry:
+;   %0 = tail call i64 @llvm.lrint.i64.f16(half %x)
+;   ret i64 %0
+; }
 
 ; CHECK-LABEL: testmsws:
 ; CHECK:       bl      lrintf
@@ -51,6 +66,23 @@ entry:
   ret i64 %0
 }
 
+; CHECK-LABEL: testmswq:
+; CHECK:       bl      lrintf128
+define signext i32 @testmswq(fp128 %x) {
+entry:
+  %0 = tail call i64 @llvm.lrint.i64.f128(fp128 %x)
+  %conv = trunc i64 %0 to i32
+  ret i32 %conv
+}
+
+; CHECK-LABEL: testmslq:
+; CHECK:       bl      lrintf128
+define i64 @testmslq(fp128 %x) {
+entry:
+  %0 = tail call i64 @llvm.lrint.i64.f128(fp128 %x)
+  ret i64 %0
+}
+
 declare i64 @llvm.lrint.i64.f32(float) nounwind readnone
 declare i64 @llvm.lrint.i64.f64(double) nounwind readnone
 declare i64 @llvm.lrint.i64.ppcf128(ppc_fp128) nounwind readnone
diff --git a/llvm/test/CodeGen/PowerPC/memintr32.ll b/llvm/test/CodeGen/PowerPC/milicode32.ll
similarity index 55%
rename from llvm/test/CodeGen/PowerPC/memintr32.ll
rename to llvm/test/CodeGen/PowerPC/milicode32.ll
index c07a5af17e48a..a2af6d413b4bf 100644
--- a/llvm/test/CodeGen/PowerPC/memintr32.ll
+++ b/llvm/test/CodeGen/PowerPC/milicode32.ll
@@ -11,7 +11,7 @@ define i32 @memcmp_test(ptr nocapture noundef readonly %ptr1, ptr nocapture noun
 ; CHECK-AIX-32-P9-NEXT:    mflr r0
 ; CHECK-AIX-32-P9-NEXT:    stwu r1, -64(r1)
 ; CHECK-AIX-32-P9-NEXT:    stw r0, 72(r1)
-; CHECK-AIX-32-P9-NEXT:    bl .memcmp[PR]
+; CHECK-AIX-32-P9-NEXT:    bl .___memcmp[PR]
 ; CHECK-AIX-32-P9-NEXT:    nop
 ; CHECK-AIX-32-P9-NEXT:    addi r1, r1, 64
 ; CHECK-AIX-32-P9-NEXT:    lwz r0, 8(r1)
@@ -35,5 +35,37 @@ entry:
 
 declare i32 @memcmp(ptr noundef captures(none), ptr noundef captures(none), i32 noundef) nounwind
 
+define i32 @strlen_test(ptr noundef %str) nounwind {
+; CHECK-AIX-32-P9-LABEL: strlen_test:
+; CHECK-AIX-32-P9:       # %bb.0: # %entry
+; CHECK-AIX-32-P9-NEXT:    mflr r0
+; CHECK-AIX-32-P9-NEXT:    stwu r1, -64(r1)
+; CHECK-AIX-32-P9-NEXT:    stw r0, 72(r1)
+; CHECK-AIX-32-P9-NEXT:    stw r3, 60(r1)
+; CHECK-AIX-32-P9-NEXT:    bl .strlen[PR]
+; CHECK-AIX-32-P9-NEXT:    nop
+; CHECK-AIX-32-P9-NEXT:    addi r1, r1, 64
+; CHECK-AIX-32-P9-NEXT:    lwz r0, 8(r1)
+; CHECK-AIX-32-P9-NEXT:    mtlr r0
+; CHECK-AIX-32-P9-NEXT:    blr
+;
+; CHECK-LINUX32-P9-LABEL: strlen_test:
+; CHECK-LINUX32-P9:       # %bb.0: # %entry
+; CHECK-LINUX32-P9-NEXT:    mflr r0
+; CHECK-LINUX32-P9-NEXT:    stwu r1, -16(r1)
+; CHECK-LINUX32-P9-NEXT:    stw r0, 20(r1)
+; CHECK-LINUX32-P9-NEXT:    stw r3, 12(r1)
+; CHECK-LINUX32-P9-NEXT:    bl strlen
+; CHECK-LINUX32-P9-NEXT:    lwz r0, 20(r1)
+; CHECK-LINUX32-P9-NEXT:    addi r1, r1, 16
+; CHECK-LINUX32-P9-NEXT:    mtlr r0
+; CHECK-LINUX32-P9-NEXT:    blr
+entry:
+  %str.addr = alloca ptr, align 4
+  store ptr %str, ptr %str.addr, align 4
+  %0 = load ptr, ptr %str.addr, align 4
+  %call = call i32 @strlen(ptr noundef %0)
+  ret i32 %call
+}
 
-
+declare i32 @strlen(ptr noundef) nounwind
diff --git a/llvm/test/CodeGen/PowerPC/memintr64.ll b/llvm/test/CodeGen/PowerPC/milicode64.ll
similarity index 57%
rename from llvm/test/CodeGen/PowerPC/memintr64.ll
rename to llvm/test/CodeGen/PowerPC/milicode64.ll
index b3a6650b8f6e6..0f0585d9028a9 100644
--- a/llvm/test/CodeGen/PowerPC/memintr64.ll
+++ b/llvm/test/CodeGen/PowerPC/milicode64.ll
@@ -39,7 +39,7 @@ define noundef i32 @_Z11memcmp_testPKvS0_m(ptr noundef readonly captures(none) %
 ; CHECK-AIX-64-P9-NEXT:    mflr r0
 ; CHECK-AIX-64-P9-NEXT:    stdu r1, -112(r1)
 ; CHECK-AIX-64-P9-NEXT:    std r0, 128(r1)
-; CHECK-AIX-64-P9-NEXT:    bl .memcmp[PR]
+; CHECK-AIX-64-P9-NEXT:    bl .___memcmp64[PR]
 ; CHECK-AIX-64-P9-NEXT:    nop
 ; CHECK-AIX-64-P9-NEXT:    addi r1, r1, 112
 ; CHECK-AIX-64-P9-NEXT:    ld r0, 16(r1)
@@ -52,4 +52,51 @@ entry:
 
 declare i32 @memcmp(ptr noundef captures(none), ptr noundef captures(none), i64 noundef) nounwind
 
+define i64 @strlen_test(ptr noundef %str) nounwind {
+; CHECK-LE-P9-LABEL: strlen_test:
+; CHECK-LE-P9:       # %bb.0: # %entry
+; CHECK-LE-P9-NEXT:    mflr r0
+; CHECK-LE-P9-NEXT:    stdu r1, -48(r1)
+; CHECK-LE-P9-NEXT:    std r0, 64(r1)
+; CHECK-LE-P9-NEXT:    std r3, 40(r1)
+; CHECK-LE-P9-NEXT:    bl strlen
+; CHECK-LE-P9-NEXT:    nop
+; CHECK-LE-P9-NEXT:    addi r1, r1, 48
+; CHECK-LE-P9-NEXT:    ld r0, 16(r1)
+; CHECK-LE-P9-NEXT:    mtlr r0
+; CHECK-LE-P9-NEXT:    blr
+;
+; CHECK-BE-P9-LABEL: strlen_test:
+; CHECK-BE-P9:       # %bb.0: # %entry
+; CHECK-BE-P9-NEXT:    mflr r0
+; CHECK-BE-P9-NEXT:    stdu r1, -128(r1)
+; CHECK-BE-P9-NEXT:    std r0, 144(r1)
+; CHECK-BE-P9-NEXT:    std r3, 120(r1)
+; CHECK-BE-P9-NEXT:    bl strlen
+; CHECK-BE-P9-NEXT:    nop
+; CHECK-BE-P9-NEXT:    addi r1, r1, 128
+; CHECK-BE-P9-NEXT:    ld r0, 16(r1)
+; CHECK-BE-P9-NEXT:    mtlr r0
+; CHECK-BE-P9-NEXT:    blr
+;
+; CHECK-AIX-64-P9-LABEL: strlen_test:
+; CHECK-AIX-64-P9:       # %bb.0: # %entry
+; CHECK-AIX-64-P9-NEXT:    mflr r0
+; CHECK-AIX-64-P9-NEXT:    stdu r1, -128(r1)
+; CHECK-AIX-64-P9-NEXT:    std r0, 144(r1)
+; CHECK-AIX-64-P9-NEXT:    std r3, 120(r1)
+; CHECK-AIX-64-P9-NEXT:    bl .strlen[PR]
+; CHECK-AIX-64-P9-NEXT:    nop
+; CHECK-AIX-64-P9-NEXT:    addi r1, r1, 128
+; CHECK-AIX-64-P9-NEXT:    ld r0, 16(r1)
+; CHECK-AIX-64-P9-NEXT:    mtlr r0
+; CHECK-AIX-64-P9-NEXT:    blr
+entry:
+  %str.addr = alloca ptr, align 8
+  store ptr %str, ptr %str.addr, align 8
+  %0 = load ptr, ptr %str.addr, align 8
+  %call = call i64 @strlen(ptr noundef %0)
+  ret i64 %call
+}
 
+declare i64 @strlen(ptr noundef) nounwind
diff --git a/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll b/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
index 232014db9a012..a9503f77c3090 100644
--- a/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
+++ b/llvm/test/CodeGen/PowerPC/mtvsrbmi.ll
@@ -2,22 +2,87 @@
 ; Verify whether the generated assembly for the following function includes the mtvsrbmi instruction.
 ; vector unsigned char v00FF()
 ; {
-; vector unsigned char x = { 0xFF, 0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
-; return x;
+;   vector unsigned char x = { 0xFF, 0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
+;   return x;
+; }
+; vector unsigned short short00FF()
+; {
+;   vector unsigned short x = { 0xFF, 0,0,0, 0,0,0,0};
+;   return x;
+; }
+; vector unsigned int int00FF()
+; {
+;   vector unsigned int x = { 0xFF, 0,0,0};
+;   return x;
+; }
+; vector unsigned long long  longlong00FF()
+; {
+;   vector unsigned long long x = { 0xFF, 0};
+;   return x;
 ; }
 
 ; RUN: llc < %s -ppc-asm-full-reg-names  -mtriple=powerpc-ibm-aix -mcpu=pwr10  -verify-machineinstrs \
-; RUN:   | FileCheck %s --check-prefix=CHECK
+; RUN:   | FileCheck %s --check-prefixes=CHECK,CHECK-BE
+
+; RUN: llc < %s -ppc-asm-full-reg-names  -mtriple=powerpc64le-unknown-gnu-linux -mcpu=pwr10  -verify-machineinstrs \
+; RUN:   | FileCheck %s --check-prefixes=CHECK,CHECK-LE
+
+; CHECK-NOT:   .byte   255
+; CHECK-NOT:   .byte   0
 
 define dso_local noundef range(i8 -1, 1) <16 x i8> @_Z5v00FFv() {
-; CHECK-NOT:      L..CPI0_0:
-; CHECK-NOT:   .byte   255                             # 0xff
-; CHECK-NOT:   .byte   0                               # 0x0
-
-; CHECK-LABEL: _Z5v00FFv:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    mtvsrbmi v2, 1
-; CHECK-NEXT:    blr
+; CHECK-BE-LABEL: _Z5v00FFv:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mtvsrbmi v2, 32768
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-LE-LABEL: _Z5v00FFv:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mtvsrbmi v2, 1
+; CHECK-LE-NEXT:    blr
+
 entry:
   ret <16 x i8> <i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
 }
+
+define dso_local noundef range(i16 0, 256) <8 x i16> @_Z9short00FFv() {
+; CHECK-BE-LABEL: _Z9short00FFv:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mtvsrbmi v2, 16384
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-LE-LABEL: _Z9short00FFv:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mtvsrbmi v2, 1
+; CHECK-LE-NEXT:    blr
+entry:
+	  ret <8 x i16> <i16 255, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+}
+
+define dso_local noundef range(i32 0, 256) <4 x i32> @_Z7int00FFv() {
+; CHECK-BE-LABEL: _Z7int00FFv:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mtvsrbmi v2, 4096
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-LE-LABEL: _Z7int00FFv:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mtvsrbmi v2, 1
+; CHECK-LE-NEXT:    blr
+entry:
+	  ret <4 x i32> <i32 255, i32 0, i32 0, i32 0>
+}
+
+define dso_local noundef range(i64 0, 256) <2 x i64> @_Z12longlong00FFv() {
+; CHECK-BE-LABEL: _Z12longlong00FFv:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    mtvsrbmi v2, 256
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-LE-LABEL: _Z12longlong00FFv:
+; CHECK-LE:       # %bb.0: # %entry
+; CHECK-LE-NEXT:    mtvsrbmi v2, 1
+; CHECK-LE-NEXT:    blr
+entry:
+	  ret <2 x i64> <i64 255, i64 0>
+}
diff --git a/llvm/test/CodeGen/PowerPC/nofpclass.ll b/llvm/test/CodeGen/PowerPC/nofpclass.ll
new file mode 100644
index 0000000000000..b08e810cd1cca
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/nofpclass.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=powerpc64-ibm-aix-xcoff < %s | FileCheck %s
+
+; TODO: Update this test after adding the proper expansion of nofpclass for
+; ppc_fp128 to test with more masks and to demonstrate preserving nofpclass
+; after legalization.
+
+define ppc_fp128 @f(ppc_fp128 nofpclass(nan) %s) {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    blr
+entry:
+  ret ppc_fp128 %s
+}
diff --git a/llvm/test/CodeGen/PowerPC/p10-spill-crlt.ll b/llvm/test/CodeGen/PowerPC/p10-spill-crlt.ll
index c733a01950603..4b032781c3764 100644
--- a/llvm/test/CodeGen/PowerPC/p10-spill-crlt.ll
+++ b/llvm/test/CodeGen/PowerPC/p10-spill-crlt.ll
@@ -30,16 +30,14 @@ define dso_local void @P10_Spill_CR_LT() local_unnamed_addr {
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    std r0, 16(r1)
 ; CHECK-NEXT:    stw r12, 8(r1)
-; CHECK-NEXT:    stdu r1, -64(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    stdu r1, -48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r29, -24
 ; CHECK-NEXT:    .cfi_offset r30, -16
 ; CHECK-NEXT:    .cfi_offset cr2, 8
 ; CHECK-NEXT:    .cfi_offset cr3, 8
 ; CHECK-NEXT:    .cfi_offset cr4, 8
-; CHECK-NEXT:    std r29, 40(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r30, 48(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    std r30, 32(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    bl call_2@notoc
 ; CHECK-NEXT:    bc 12, 4*cr5+lt, .LBB0_13
 ; CHECK-NEXT:  # %bb.1: # %bb
@@ -67,11 +65,10 @@ define dso_local void @P10_Spill_CR_LT() local_unnamed_addr {
 ; CHECK-NEXT:    bc 12, 4*cr3+eq, .LBB0_11
 ; CHECK-NEXT:  # %bb.6: # %bb32
 ; CHECK-NEXT:    #
+; CHECK-NEXT:    rlwinm r30, r30, 0, 24, 22
 ; CHECK-NEXT:    andi. r3, r30, 2
-; CHECK-NEXT:    rlwinm r29, r30, 0, 24, 22
 ; CHECK-NEXT:    mcrf cr2, cr0
 ; CHECK-NEXT:    bl call_4@notoc
-; CHECK-NEXT:    mr r30, r29
 ; CHECK-NEXT:    beq+ cr2, .LBB0_3
 ; CHECK-NEXT:  # %bb.7: # %bb37
 ; CHECK-NEXT:  .LBB0_8: # %bb22
@@ -92,13 +89,11 @@ define dso_local void @P10_Spill_CR_LT() local_unnamed_addr {
 ; CHECK-BE-NEXT:    stdu r1, -144(r1)
 ; CHECK-BE-NEXT:    .cfi_def_cfa_offset 144
 ; CHECK-BE-NEXT:    .cfi_offset lr, 16
-; CHECK-BE-NEXT:    .cfi_offset r28, -32
 ; CHECK-BE-NEXT:    .cfi_offset r29, -24
 ; CHECK-BE-NEXT:    .cfi_offset r30, -16
 ; CHECK-BE-NEXT:    .cfi_offset cr2, 8
 ; CHECK-BE-NEXT:    .cfi_offset cr2, 8
 ; CHECK-BE-NEXT:    .cfi_offset cr2, 8
-; CHECK-BE-NEXT:    std r28, 112(r1) # 8-byte Folded Spill
 ; CHECK-BE-NEXT:    std r29, 120(r1) # 8-byte Folded Spill
 ; CHECK-BE-NEXT:    std r30, 128(r1) # 8-byte Folded Spill
 ; CHECK-BE-NEXT:    bl call_2
@@ -131,12 +126,11 @@ define dso_local void @P10_Spill_CR_LT() local_unnamed_addr {
 ; CHECK-BE-NEXT:    bc 12, 4*cr3+eq, .LBB0_11
 ; CHECK-BE-NEXT:  # %bb.6: # %bb32
 ; CHECK-BE-NEXT:    #
+; CHECK-BE-NEXT:    rlwinm r29, r29, 0, 24, 22
 ; CHECK-BE-NEXT:    andi. r3, r29, 2
-; CHECK-BE-NEXT:    rlwinm r28, r29, 0, 24, 22
 ; CHECK-BE-NEXT:    mcrf cr2, cr0
 ; CHECK-BE-NEXT:    bl call_4
 ; CHECK-BE-NEXT:    nop
-; CHECK-BE-NEXT:    mr r29, r28
 ; CHECK-BE-NEXT:    beq+ cr2, .LBB0_3
 ; CHECK-BE-NEXT:  # %bb.7: # %bb37
 ; CHECK-BE-NEXT:  .LBB0_8: # %bb22
diff --git a/llvm/test/CodeGen/PowerPC/peephole-phi-acc.mir b/llvm/test/CodeGen/PowerPC/peephole-phi-acc.mir
index f615fcf9f8157..4333473a1f6b3 100644
--- a/llvm/test/CodeGen/PowerPC/peephole-phi-acc.mir
+++ b/llvm/test/CodeGen/PowerPC/peephole-phi-acc.mir
@@ -223,8 +223,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -337,8 +337,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -485,8 +485,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -696,8 +696,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/PowerPC/peephole-replaceInstr-after-eliminate-extsw.mir b/llvm/test/CodeGen/PowerPC/peephole-replaceInstr-after-eliminate-extsw.mir
index 1d898a4d7ce9f..ceecdc5b06fb7 100644
--- a/llvm/test/CodeGen/PowerPC/peephole-replaceInstr-after-eliminate-extsw.mir
+++ b/llvm/test/CodeGen/PowerPC/peephole-replaceInstr-after-eliminate-extsw.mir
@@ -447,8 +447,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/PowerPC/phi-eliminate.mir b/llvm/test/CodeGen/PowerPC/phi-eliminate.mir
index 72f778286abe4..17060a88fb527 100644
--- a/llvm/test/CodeGen/PowerPC/phi-eliminate.mir
+++ b/llvm/test/CodeGen/PowerPC/phi-eliminate.mir
@@ -122,8 +122,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/PowerPC/ppc_reduce_cr_logicals.ll b/llvm/test/CodeGen/PowerPC/ppc_reduce_cr_logicals.ll
new file mode 100644
index 0000000000000..585ce89e9aa46
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/ppc_reduce_cr_logicals.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=powerpc64-unknown-gnu-linux < %s | FileCheck %s -check-prefix=CHECK
+; RUN: llc -mtriple=powerpc-unknown-gnu-linux < %s | FileCheck %s -check-prefix=CHECKBE
+
+define i32 @xe_migrate_copy(ptr %m, ptr %dst, ptr %tile, ptr %0, ptr %primary_gt, i1 %tobool4, i1 %tobool9, i64 %1, i32 %conv55, i1 %tobool37.not) nounwind {
+; CHECK-LABEL: xe_migrate_copy:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr 0
+; CHECK-NEXT:    stdu 1, -128(1)
+; CHECK-NEXT:    lbz 4, 255(1)
+; CHECK-NEXT:    andi. 4, 4, 1
+; CHECK-NEXT:    std 0, 144(1)
+; CHECK-NEXT:    crmove 20, 1
+; CHECK-NEXT:    andi. 4, 9, 1
+; CHECK-NEXT:    lwz 9, 244(1)
+; CHECK-NEXT:    crmove 21, 1
+; CHECK-NEXT:    andi. 4, 8, 1
+; CHECK-NEXT:    li 4, 0
+; CHECK-NEXT:    std 4, 112(1)
+; CHECK-NEXT:    crandc 21, 21, 20
+; CHECK-NEXT:    bc 12, 21, .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %while.body
+; CHECK-NEXT:    crand 20, 20, 1
+; CHECK-NEXT:    li 8, 0
+; CHECK-NEXT:    bc 4, 20, .LBB0_3
+; CHECK-NEXT:  .LBB0_2: # %while.body
+; CHECK-NEXT:    li 8, 1
+; CHECK-NEXT:  .LBB0_3: # %while.body
+; CHECK-NEXT:    li 5, 0
+; CHECK-NEXT:    li 6, 0
+; CHECK-NEXT:    mr 4, 3
+; CHECK-NEXT:    li 7, 0
+; CHECK-NEXT:    li 10, 0
+; CHECK-NEXT:    bl xe_migrate_ccs_copy
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi 1, 1, 128
+; CHECK-NEXT:    ld 0, 16(1)
+; CHECK-NEXT:    mtlr 0
+; CHECK-NEXT:    blr
+;
+; CHECKBE-LABEL: xe_migrate_copy:
+; CHECKBE:       # %bb.0: # %entry
+; CHECKBE-NEXT:    mflr 0
+; CHECKBE-NEXT:    stwu 1, -32(1)
+; CHECKBE-NEXT:    lbz 4, 55(1)
+; CHECKBE-NEXT:    li 5, 0
+; CHECKBE-NEXT:    stw 0, 36(1)
+; CHECKBE-NEXT:    andi. 4, 4, 1
+; CHECKBE-NEXT:    crmove 20, 1
+; CHECKBE-NEXT:    andi. 4, 9, 1
+; CHECKBE-NEXT:    crmove 21, 1
+; CHECKBE-NEXT:    andi. 4, 8, 1
+; CHECKBE-NEXT:    lwz 4, 48(1)
+; CHECKBE-NEXT:    crandc 21, 21, 20
+; CHECKBE-NEXT:    stw 5, 24(1)
+; CHECKBE-NEXT:    stw 5, 20(1)
+; CHECKBE-NEXT:    stw 5, 16(1)
+; CHECKBE-NEXT:    stw 4, 12(1)
+; CHECKBE-NEXT:    bc 12, 21, .LBB0_2
+; CHECKBE-NEXT:  # %bb.1: # %while.body
+; CHECKBE-NEXT:    crand 20, 20, 1
+; CHECKBE-NEXT:    li 8, 0
+; CHECKBE-NEXT:    bc 4, 20, .LBB0_3
+; CHECKBE-NEXT:  .LBB0_2: # %while.body
+; CHECKBE-NEXT:    li 8, 1
+; CHECKBE-NEXT:  .LBB0_3: # %while.body
+; CHECKBE-NEXT:    mr 4, 3
+; CHECKBE-NEXT:    li 6, 0
+; CHECKBE-NEXT:    li 7, 0
+; CHECKBE-NEXT:    li 9, 0
+; CHECKBE-NEXT:    li 10, 0
+; CHECKBE-NEXT:    stw 8, 8(1)
+; CHECKBE-NEXT:    bl xe_migrate_ccs_copy
+; CHECKBE-NEXT:    lwz 0, 36(1)
+; CHECKBE-NEXT:    addi 1, 1, 32
+; CHECKBE-NEXT:    mtlr 0
+; CHECKBE-NEXT:    blr
+
+entry:
+  br label %while.body
+
+while.body:
+  %cond53.in = select i1 %tobool37.not, i1 %tobool4, i1 %tobool9
+  %call57 = call zeroext i32 @xe_migrate_ccs_copy(ptr noundef %m, ptr noundef %m, i64 0, i1 false, i64 0, i1 %cond53.in, i32 %conv55, i64 0, i1 false)
+  ret i32 %call57
+}
+
+declare i32 @xe_migrate_ccs_copy(ptr, ptr, i64, i1, i64, i1, i32, i64, i1)
diff --git a/llvm/test/CodeGen/PowerPC/remove-copy-crunsetcrbit.mir b/llvm/test/CodeGen/PowerPC/remove-copy-crunsetcrbit.mir
index f3ef95bbb79a4..05b30569a2c10 100644
--- a/llvm/test/CodeGen/PowerPC/remove-copy-crunsetcrbit.mir
+++ b/llvm/test/CodeGen/PowerPC/remove-copy-crunsetcrbit.mir
@@ -99,8 +99,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/PowerPC/remove-implicit-use.mir b/llvm/test/CodeGen/PowerPC/remove-implicit-use.mir
index f5b931e1e4238..f89475e21b49f 100644
--- a/llvm/test/CodeGen/PowerPC/remove-implicit-use.mir
+++ b/llvm/test/CodeGen/PowerPC/remove-implicit-use.mir
@@ -56,8 +56,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/PowerPC/remove-redundant-li-skip-imp-kill.mir b/llvm/test/CodeGen/PowerPC/remove-redundant-li-skip-imp-kill.mir
index 913877b0d4318..20404947c3b36 100644
--- a/llvm/test/CodeGen/PowerPC/remove-redundant-li-skip-imp-kill.mir
+++ b/llvm/test/CodeGen/PowerPC/remove-redundant-li-skip-imp-kill.mir
@@ -55,8 +55,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -80, size: 8, alignment: 16, stack-id: default,
       callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '',
diff --git a/llvm/test/CodeGen/PowerPC/remove-self-copies.mir b/llvm/test/CodeGen/PowerPC/remove-self-copies.mir
index b5713a9349f69..6c083902ce5e9 100644
--- a/llvm/test/CodeGen/PowerPC/remove-self-copies.mir
+++ b/llvm/test/CodeGen/PowerPC/remove-self-copies.mir
@@ -65,8 +65,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
diff --git a/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir b/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
index a1d8539dd8cb6..584f2a975b042 100644
--- a/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
+++ b/llvm/test/CodeGen/PowerPC/rlwinm_rldicl_to_andi.mir
@@ -107,8 +107,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -167,8 +167,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -227,8 +227,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -284,8 +284,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -338,8 +338,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
@@ -392,8 +392,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
diff --git a/llvm/test/CodeGen/PowerPC/schedule-addi-load.mir b/llvm/test/CodeGen/PowerPC/schedule-addi-load.mir
index 1717238b2b7a0..4414dfaa43f6c 100644
--- a/llvm/test/CodeGen/PowerPC/schedule-addi-load.mir
+++ b/llvm/test/CodeGen/PowerPC/schedule-addi-load.mir
@@ -68,8 +68,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/PowerPC/setcr_bc.mir b/llvm/test/CodeGen/PowerPC/setcr_bc.mir
index bc8bb5582137f..76f9d5e847dff 100644
--- a/llvm/test/CodeGen/PowerPC/setcr_bc.mir
+++ b/llvm/test/CodeGen/PowerPC/setcr_bc.mir
@@ -64,8 +64,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
   - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
       callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/CodeGen/PowerPC/setcr_bc2.mir b/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
index 5986c885f189e..433ea6301277f 100644
--- a/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
+++ b/llvm/test/CodeGen/PowerPC/setcr_bc2.mir
@@ -64,8 +64,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
   - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
       callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/CodeGen/PowerPC/setcr_bc3.mir b/llvm/test/CodeGen/PowerPC/setcr_bc3.mir
index 2d037d03bb6b7..942ac693db383 100644
--- a/llvm/test/CodeGen/PowerPC/setcr_bc3.mir
+++ b/llvm/test/CodeGen/PowerPC/setcr_bc3.mir
@@ -37,8 +37,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default,
       callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/CodeGen/PowerPC/stack-protector-target.ll b/llvm/test/CodeGen/PowerPC/stack-protector-target.ll
index 03ffa0b4c142b..9b93040acb1c4 100644
--- a/llvm/test/CodeGen/PowerPC/stack-protector-target.ll
+++ b/llvm/test/CodeGen/PowerPC/stack-protector-target.ll
@@ -65,8 +65,8 @@ define void @func() sspreq nounwind {
 ; LINUX32:       # %bb.0:
 ; LINUX32-NEXT:    mflr 0
 ; LINUX32-NEXT:    stwu 1, -16(1)
-; LINUX32-NEXT:    stw 0, 20(1)
 ; LINUX32-NEXT:    lwz 3, -28680(2)
+; LINUX32-NEXT:    stw 0, 20(1)
 ; LINUX32-NEXT:    stw 3, 12(1)
 ; LINUX32-NEXT:    addi 3, 1, 8
 ; LINUX32-NEXT:    bl capture
@@ -86,8 +86,8 @@ define void @func() sspreq nounwind {
 ; LINUX64:       # %bb.0:
 ; LINUX64-NEXT:    mflr 0
 ; LINUX64-NEXT:    stdu 1, -128(1)
-; LINUX64-NEXT:    std 0, 144(1)
 ; LINUX64-NEXT:    ld 3, -28688(13)
+; LINUX64-NEXT:    std 0, 144(1)
 ; LINUX64-NEXT:    std 3, 120(1)
 ; LINUX64-NEXT:    addi 3, 1, 116
 ; LINUX64-NEXT:    bl capture
diff --git a/llvm/test/CodeGen/PowerPC/swaps-le-1.ll b/llvm/test/CodeGen/PowerPC/swaps-le-1.ll
index f3e34101efa29..5d5445f9f473a 100644
--- a/llvm/test/CodeGen/PowerPC/swaps-le-1.ll
+++ b/llvm/test/CodeGen/PowerPC/swaps-le-1.ll
@@ -187,34 +187,34 @@ define void @foo() {
 ; CHECK-P9-NEXT:    .p2align 4
 ; CHECK-P9-NEXT:  .LBB0_1: # %vector.body
 ; CHECK-P9-NEXT:    #
-; CHECK-P9-NEXT:    lxv 2, -32(6)
-; CHECK-P9-NEXT:    lxv 3, -32(5)
-; CHECK-P9-NEXT:    lxv 4, -16(5)
-; CHECK-P9-NEXT:    vadduwm 2, 3, 2
+; CHECK-P9-NEXT:    lxv 2, -32(3)
 ; CHECK-P9-NEXT:    lxv 3, -32(4)
+; CHECK-P9-NEXT:    lxv 4, -16(4)
+; CHECK-P9-NEXT:    vadduwm 2, 3, 2
+; CHECK-P9-NEXT:    lxv 3, -32(5)
 ; CHECK-P9-NEXT:    vmuluwm 2, 2, 3
-; CHECK-P9-NEXT:    lxv 3, -16(6)
-; CHECK-P9-NEXT:    vadduwm 3, 4, 3
-; CHECK-P9-NEXT:    lxv 4, 0(5)
-; CHECK-P9-NEXT:    stxv 2, -32(3)
-; CHECK-P9-NEXT:    lxv 2, -16(4)
-; CHECK-P9-NEXT:    vmuluwm 2, 3, 2
-; CHECK-P9-NEXT:    lxv 3, 0(6)
+; CHECK-P9-NEXT:    lxv 3, -16(3)
 ; CHECK-P9-NEXT:    vadduwm 3, 4, 3
-; CHECK-P9-NEXT:    lxv 4, 16(5)
-; CHECK-P9-NEXT:    addi 5, 5, 64
-; CHECK-P9-NEXT:    stxv 2, -16(3)
-; CHECK-P9-NEXT:    lxv 2, 0(4)
+; CHECK-P9-NEXT:    lxv 4, 0(4)
+; CHECK-P9-NEXT:    stxv 2, -32(6)
+; CHECK-P9-NEXT:    lxv 2, -16(5)
 ; CHECK-P9-NEXT:    vmuluwm 2, 3, 2
-; CHECK-P9-NEXT:    lxv 3, 16(6)
-; CHECK-P9-NEXT:    addi 6, 6, 64
+; CHECK-P9-NEXT:    lxv 3, 0(3)
 ; CHECK-P9-NEXT:    vadduwm 3, 4, 3
-; CHECK-P9-NEXT:    stxv 2, 0(3)
-; CHECK-P9-NEXT:    lxv 2, 16(4)
+; CHECK-P9-NEXT:    lxv 4, 16(4)
 ; CHECK-P9-NEXT:    addi 4, 4, 64
+; CHECK-P9-NEXT:    stxv 2, -16(6)
+; CHECK-P9-NEXT:    lxv 2, 0(5)
 ; CHECK-P9-NEXT:    vmuluwm 2, 3, 2
-; CHECK-P9-NEXT:    stxv 2, 16(3)
+; CHECK-P9-NEXT:    lxv 3, 16(3)
 ; CHECK-P9-NEXT:    addi 3, 3, 64
+; CHECK-P9-NEXT:    vadduwm 3, 4, 3
+; CHECK-P9-NEXT:    stxv 2, 0(6)
+; CHECK-P9-NEXT:    lxv 2, 16(5)
+; CHECK-P9-NEXT:    addi 5, 5, 64
+; CHECK-P9-NEXT:    vmuluwm 2, 3, 2
+; CHECK-P9-NEXT:    stxv 2, 16(6)
+; CHECK-P9-NEXT:    addi 6, 6, 64
 ; CHECK-P9-NEXT:    bdnz .LBB0_1
 ; CHECK-P9-NEXT:  # %bb.2: # %for.end
 ; CHECK-P9-NEXT:    blr
diff --git a/llvm/test/CodeGen/PowerPC/tls_get_addr_fence1.mir b/llvm/test/CodeGen/PowerPC/tls_get_addr_fence1.mir
index 73bd475e9d498..a55cb04137f95 100644
--- a/llvm/test/CodeGen/PowerPC/tls_get_addr_fence1.mir
+++ b/llvm/test/CodeGen/PowerPC/tls_get_addr_fence1.mir
@@ -43,8 +43,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/PowerPC/tls_get_addr_fence2.mir b/llvm/test/CodeGen/PowerPC/tls_get_addr_fence2.mir
index ffeb066b94785..49d4a15ea2d0c 100644
--- a/llvm/test/CodeGen/PowerPC/tls_get_addr_fence2.mir
+++ b/llvm/test/CodeGen/PowerPC/tls_get_addr_fence2.mir
@@ -43,8 +43,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
 constants:       
diff --git a/llvm/test/CodeGen/PowerPC/two-address-crash.mir b/llvm/test/CodeGen/PowerPC/two-address-crash.mir
index cd2e69d8612b9..21f08cb085715 100644
--- a/llvm/test/CodeGen/PowerPC/two-address-crash.mir
+++ b/llvm/test/CodeGen/PowerPC/two-address-crash.mir
@@ -62,8 +62,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/PowerPC/vector-llrint.ll b/llvm/test/CodeGen/PowerPC/vector-llrint.ll
index 9229fefced67e..8a9e48e002381 100644
--- a/llvm/test/CodeGen/PowerPC/vector-llrint.ll
+++ b/llvm/test/CodeGen/PowerPC/vector-llrint.ll
@@ -1,4 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; FIXME: crash "Input type needs to be promoted!"
+; SKIP: llc -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; SKIP:   -mtriple=powerpc-unknown-unknown -verify-machineinstrs < %s | \
+; SKIP:   FileCheck %s --check-prefix=PPC32
 ; RUN: llc -mcpu=pwr7 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
 ; RUN:   -mtriple=powerpc64-unknown-unknown -verify-machineinstrs < %s | \
 ; RUN:   FileCheck %s --check-prefix=BE
@@ -9,14 +13,12 @@
 ; RUN:   -mtriple=powerpc64le-unknown-unknown -verify-machineinstrs < %s \
 ; RUN:   --enable-unsafe-fp-math | FileCheck %s --check-prefix=FAST
 
-define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
+define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) nounwind {
 ; BE-LABEL: llrint_v1i64_v1f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -112(r1)
 ; BE-NEXT:    std r0, 128(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 112
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    bl __truncsfhf2
 ; BE-NEXT:    nop
 ; BE-NEXT:    clrldi r3, r3, 48
@@ -34,8 +36,6 @@ define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -32(r1)
 ; CHECK-NEXT:    std r0, 48(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    clrldi r3, r3, 48
@@ -53,8 +53,6 @@ define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
 ; FAST-NEXT:    mflr r0
 ; FAST-NEXT:    stdu r1, -32(r1)
 ; FAST-NEXT:    std r0, 48(r1)
-; FAST-NEXT:    .cfi_def_cfa_offset 32
-; FAST-NEXT:    .cfi_offset lr, 16
 ; FAST-NEXT:    bl __truncsfhf2
 ; FAST-NEXT:    nop
 ; FAST-NEXT:    clrldi r3, r3, 48
@@ -71,16 +69,12 @@ define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f16(<1 x half>)
 
-define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
+define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) nounwind {
 ; BE-LABEL: llrint_v1i64_v2f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -160(r1)
 ; BE-NEXT:    std r0, 176(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 160
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r30, -24
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f31, 152(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f31, f1
 ; BE-NEXT:    fmr f1, f2
@@ -118,17 +112,12 @@ define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -96(r1)
-; CHECK-NEXT:    std r0, 112(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r30, -24
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v31, -48
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 112(r1)
 ; CHECK-NEXT:    std r30, 72(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f31, 88(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f2
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f31
@@ -153,7 +142,7 @@ define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
 ; CHECK-NEXT:    lfd f31, 88(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r30, 72(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    xxmrghd v2, vs0, v31
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 96
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -162,10 +151,6 @@ define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
 ; FAST-LABEL: llrint_v1i64_v2f16:
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
-; FAST-NEXT:    .cfi_def_cfa_offset 48
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
 ; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stdu r1, -48(r1)
@@ -202,20 +187,12 @@ define <2 x i64> @llrint_v1i64_v2f16(<2 x half> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f16(<2 x half>)
 
-define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
+define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) nounwind {
 ; BE-LABEL: llrint_v4i64_v4f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -208(r1)
 ; BE-NEXT:    std r0, 224(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 208
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r28, -56
-; BE-NEXT:    .cfi_offset r29, -48
-; BE-NEXT:    .cfi_offset r30, -40
-; BE-NEXT:    .cfi_offset f29, -24
-; BE-NEXT:    .cfi_offset f30, -16
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f29, 184(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f29, f1
 ; BE-NEXT:    fmr f1, f2
@@ -289,18 +266,8 @@ define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -144(r1)
-; CHECK-NEXT:    std r0, 160(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 144
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r28, -56
-; CHECK-NEXT:    .cfi_offset r29, -48
-; CHECK-NEXT:    .cfi_offset r30, -40
-; CHECK-NEXT:    .cfi_offset f29, -24
-; CHECK-NEXT:    .cfi_offset f30, -16
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v30, -96
-; CHECK-NEXT:    .cfi_offset v31, -80
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 160(r1)
 ; CHECK-NEXT:    std r28, 88(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r29, 96(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r30, 104(r1) # 8-byte Folded Spill
@@ -308,11 +275,11 @@ define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
 ; CHECK-NEXT:    fmr f29, f2
 ; CHECK-NEXT:    stfd f30, 128(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f30, f3
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    stfd f31, 136(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f4
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f29
@@ -365,11 +332,11 @@ define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
 ; CHECK-NEXT:    lfd f29, 120(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r30, 104(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, 96(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
 ; CHECK-NEXT:    ld r28, 88(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    xxmrghd v3, vs0, v30
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 144
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -378,12 +345,6 @@ define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
 ; FAST-LABEL: llrint_v4i64_v4f16:
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
-; FAST-NEXT:    .cfi_def_cfa_offset 64
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset f28, -32
-; FAST-NEXT:    .cfi_offset f29, -24
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
 ; FAST-NEXT:    stfd f28, -32(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f29, -24(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
@@ -447,28 +408,12 @@ define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f16(<4 x half>)
 
-define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
+define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) nounwind {
 ; BE-LABEL: llrint_v8i64_v8f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -304(r1)
 ; BE-NEXT:    std r0, 320(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 304
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r24, -120
-; BE-NEXT:    .cfi_offset r25, -112
-; BE-NEXT:    .cfi_offset r26, -104
-; BE-NEXT:    .cfi_offset r27, -96
-; BE-NEXT:    .cfi_offset r28, -88
-; BE-NEXT:    .cfi_offset r29, -80
-; BE-NEXT:    .cfi_offset r30, -72
-; BE-NEXT:    .cfi_offset f25, -56
-; BE-NEXT:    .cfi_offset f26, -48
-; BE-NEXT:    .cfi_offset f27, -40
-; BE-NEXT:    .cfi_offset f28, -32
-; BE-NEXT:    .cfi_offset f29, -24
-; BE-NEXT:    .cfi_offset f30, -16
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f25, 248(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f25, f1
 ; BE-NEXT:    fmr f1, f2
@@ -614,44 +559,24 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -240(r1)
-; CHECK-NEXT:    std r0, 256(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 240
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r24, -120
-; CHECK-NEXT:    .cfi_offset r25, -112
-; CHECK-NEXT:    .cfi_offset r26, -104
-; CHECK-NEXT:    .cfi_offset r27, -96
-; CHECK-NEXT:    .cfi_offset r28, -88
-; CHECK-NEXT:    .cfi_offset r29, -80
-; CHECK-NEXT:    .cfi_offset r30, -72
-; CHECK-NEXT:    .cfi_offset f25, -56
-; CHECK-NEXT:    .cfi_offset f26, -48
-; CHECK-NEXT:    .cfi_offset f27, -40
-; CHECK-NEXT:    .cfi_offset f28, -32
-; CHECK-NEXT:    .cfi_offset f29, -24
-; CHECK-NEXT:    .cfi_offset f30, -16
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v28, -192
-; CHECK-NEXT:    .cfi_offset v29, -176
-; CHECK-NEXT:    .cfi_offset v30, -160
-; CHECK-NEXT:    .cfi_offset v31, -144
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 256(r1)
 ; CHECK-NEXT:    std r24, 120(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r25, 128(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r26, 136(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r27, 144(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r28, 152(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r29, 160(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r30, 168(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    std r30, 168(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f25, 184(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f25, f2
 ; CHECK-NEXT:    stfd f26, 192(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f26, f3
 ; CHECK-NEXT:    stfd f27, 200(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f27, f4
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
 ; CHECK-NEXT:    stfd f28, 208(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f28, f5
@@ -659,11 +584,11 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
 ; CHECK-NEXT:    fmr f29, f6
 ; CHECK-NEXT:    stfd f30, 224(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f30, f7
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    stfd f31, 232(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f8
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f25
@@ -766,7 +691,7 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
 ; CHECK-NEXT:    vmr v4, v29
 ; CHECK-NEXT:    lfd f30, 224(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f29, 216(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
 ; CHECK-NEXT:    lfd f28, 208(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f27, 200(r1) # 8-byte Folded Reload
@@ -774,7 +699,7 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
 ; CHECK-NEXT:    lfd f25, 184(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r30, 168(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, 160(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    ld r28, 152(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r27, 144(r1) # 8-byte Folded Reload
@@ -782,9 +707,9 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
 ; CHECK-NEXT:    ld r26, 136(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r25, 128(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r24, 120(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 240
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -793,16 +718,6 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
 ; FAST-LABEL: llrint_v8i64_v8f16:
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
-; FAST-NEXT:    .cfi_def_cfa_offset 96
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset f24, -64
-; FAST-NEXT:    .cfi_offset f25, -56
-; FAST-NEXT:    .cfi_offset f26, -48
-; FAST-NEXT:    .cfi_offset f27, -40
-; FAST-NEXT:    .cfi_offset f28, -32
-; FAST-NEXT:    .cfi_offset f29, -24
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
 ; FAST-NEXT:    stfd f24, -64(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f25, -56(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f26, -48(r1) # 8-byte Folded Spill
@@ -920,44 +835,12 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f16(<8 x half>)
 
-define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
+define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) nounwind {
 ; BE-LABEL: llrint_v16i64_v16f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -496(r1)
 ; BE-NEXT:    std r0, 512(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 496
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r16, -248
-; BE-NEXT:    .cfi_offset r17, -240
-; BE-NEXT:    .cfi_offset r18, -232
-; BE-NEXT:    .cfi_offset r19, -224
-; BE-NEXT:    .cfi_offset r20, -216
-; BE-NEXT:    .cfi_offset r21, -208
-; BE-NEXT:    .cfi_offset r22, -200
-; BE-NEXT:    .cfi_offset r23, -192
-; BE-NEXT:    .cfi_offset r24, -184
-; BE-NEXT:    .cfi_offset r25, -176
-; BE-NEXT:    .cfi_offset r26, -168
-; BE-NEXT:    .cfi_offset r27, -160
-; BE-NEXT:    .cfi_offset r28, -152
-; BE-NEXT:    .cfi_offset r29, -144
-; BE-NEXT:    .cfi_offset r30, -136
-; BE-NEXT:    .cfi_offset f17, -120
-; BE-NEXT:    .cfi_offset f18, -112
-; BE-NEXT:    .cfi_offset f19, -104
-; BE-NEXT:    .cfi_offset f20, -96
-; BE-NEXT:    .cfi_offset f21, -88
-; BE-NEXT:    .cfi_offset f22, -80
-; BE-NEXT:    .cfi_offset f23, -72
-; BE-NEXT:    .cfi_offset f24, -64
-; BE-NEXT:    .cfi_offset f25, -56
-; BE-NEXT:    .cfi_offset f26, -48
-; BE-NEXT:    .cfi_offset f27, -40
-; BE-NEXT:    .cfi_offset f28, -32
-; BE-NEXT:    .cfi_offset f29, -24
-; BE-NEXT:    .cfi_offset f30, -16
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f20, 400(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f20, f1
 ; BE-NEXT:    fmr f1, f2
@@ -1244,105 +1127,65 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -432(r1)
-; CHECK-NEXT:    std r0, 448(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 432
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r16, -248
-; CHECK-NEXT:    .cfi_offset r17, -240
-; CHECK-NEXT:    .cfi_offset r18, -232
-; CHECK-NEXT:    .cfi_offset r19, -224
-; CHECK-NEXT:    .cfi_offset r20, -216
-; CHECK-NEXT:    .cfi_offset r21, -208
-; CHECK-NEXT:    .cfi_offset r22, -200
-; CHECK-NEXT:    .cfi_offset r23, -192
-; CHECK-NEXT:    .cfi_offset r24, -184
-; CHECK-NEXT:    .cfi_offset r25, -176
-; CHECK-NEXT:    .cfi_offset r26, -168
-; CHECK-NEXT:    .cfi_offset r27, -160
-; CHECK-NEXT:    .cfi_offset r28, -152
-; CHECK-NEXT:    .cfi_offset r29, -144
-; CHECK-NEXT:    .cfi_offset r30, -136
-; CHECK-NEXT:    .cfi_offset f17, -120
-; CHECK-NEXT:    .cfi_offset f18, -112
-; CHECK-NEXT:    .cfi_offset f19, -104
-; CHECK-NEXT:    .cfi_offset f20, -96
-; CHECK-NEXT:    .cfi_offset f21, -88
-; CHECK-NEXT:    .cfi_offset f22, -80
-; CHECK-NEXT:    .cfi_offset f23, -72
-; CHECK-NEXT:    .cfi_offset f24, -64
-; CHECK-NEXT:    .cfi_offset f25, -56
-; CHECK-NEXT:    .cfi_offset f26, -48
-; CHECK-NEXT:    .cfi_offset f27, -40
-; CHECK-NEXT:    .cfi_offset f28, -32
-; CHECK-NEXT:    .cfi_offset f29, -24
-; CHECK-NEXT:    .cfi_offset f30, -16
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v24, -384
-; CHECK-NEXT:    .cfi_offset v25, -368
-; CHECK-NEXT:    .cfi_offset v26, -352
-; CHECK-NEXT:    .cfi_offset v27, -336
-; CHECK-NEXT:    .cfi_offset v28, -320
-; CHECK-NEXT:    .cfi_offset v29, -304
-; CHECK-NEXT:    .cfi_offset v30, -288
-; CHECK-NEXT:    .cfi_offset v31, -272
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 448(r1)
 ; CHECK-NEXT:    std r16, 184(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r17, 192(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r18, 200(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r19, 208(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r20, 216(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r21, 224(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r22, 232(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    std r22, 232(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r23, 240(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r24, 248(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r25, 256(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r26, 264(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r27, 272(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r28, 280(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    std r28, 280(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r29, 288(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r30, 296(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f17, 312(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f18, 320(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f19, 328(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    stfd f20, 336(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f20, f2
-; CHECK-NEXT:    stvx v26, r1, r3 # 16-byte Folded Spill
-; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    stfd f21, 344(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f21, f3
 ; CHECK-NEXT:    stfd f22, 352(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f22, f4
+; CHECK-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 112
 ; CHECK-NEXT:    stfd f23, 360(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f23, f5
-; CHECK-NEXT:    stvx v27, r1, r3 # 16-byte Folded Spill
-; CHECK-NEXT:    li r3, 112
 ; CHECK-NEXT:    stfd f24, 368(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f24, f6
 ; CHECK-NEXT:    stfd f25, 376(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f25, f7
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 128
 ; CHECK-NEXT:    stfd f26, 384(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f26, f8
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
-; CHECK-NEXT:    li r3, 128
 ; CHECK-NEXT:    stfd f27, 392(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f27, f9
 ; CHECK-NEXT:    stfd f28, 400(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f28, f10
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 144
 ; CHECK-NEXT:    stfd f29, 408(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f29, f11
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
-; CHECK-NEXT:    li r3, 144
 ; CHECK-NEXT:    stfd f30, 416(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f30, f12
 ; CHECK-NEXT:    stfd f31, 424(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f13
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 160
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f20
@@ -1545,7 +1388,7 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    vmr v4, v29
 ; CHECK-NEXT:    lfd f30, 416(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f29, 408(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 144
 ; CHECK-NEXT:    vmr v5, v28
 ; CHECK-NEXT:    vmr v6, v27
@@ -1553,7 +1396,7 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    vmr v8, v25
 ; CHECK-NEXT:    lfd f28, 400(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f27, 392(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 128
 ; CHECK-NEXT:    lfd f26, 384(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f25, 376(r1) # 8-byte Folded Reload
@@ -1561,7 +1404,7 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    lfd f24, 368(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f23, 360(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f22, 352(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 112
 ; CHECK-NEXT:    lfd f21, 344(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r30, 296(r1) # 8-byte Folded Reload
@@ -1569,7 +1412,7 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    lfd f19, 328(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, 288(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r28, 280(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    lfd f18, 320(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r27, 272(r1) # 8-byte Folded Reload
@@ -1577,7 +1420,7 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    ld r26, 264(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r25, 256(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r24, 248(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
 ; CHECK-NEXT:    ld r23, 240(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r22, 232(r1) # 8-byte Folded Reload
@@ -1585,13 +1428,13 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    ld r20, 216(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r19, 208(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r18, 200(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    ld r17, 192(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r16, 184(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 432
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -1600,24 +1443,6 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 ; FAST-LABEL: llrint_v16i64_v16f16:
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
-; FAST-NEXT:    .cfi_def_cfa_offset 160
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset f16, -128
-; FAST-NEXT:    .cfi_offset f17, -120
-; FAST-NEXT:    .cfi_offset f18, -112
-; FAST-NEXT:    .cfi_offset f19, -104
-; FAST-NEXT:    .cfi_offset f20, -96
-; FAST-NEXT:    .cfi_offset f21, -88
-; FAST-NEXT:    .cfi_offset f22, -80
-; FAST-NEXT:    .cfi_offset f23, -72
-; FAST-NEXT:    .cfi_offset f24, -64
-; FAST-NEXT:    .cfi_offset f25, -56
-; FAST-NEXT:    .cfi_offset f26, -48
-; FAST-NEXT:    .cfi_offset f27, -40
-; FAST-NEXT:    .cfi_offset f28, -32
-; FAST-NEXT:    .cfi_offset f29, -24
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
 ; FAST-NEXT:    stfd f16, -128(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f17, -120(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f18, -112(r1) # 8-byte Folded Spill
@@ -1841,50 +1666,12 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f16(<16 x half>)
 
-define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
+define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) nounwind {
 ; BE-LABEL: llrint_v32i64_v32f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -864(r1)
 ; BE-NEXT:    std r0, 880(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 864
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r14, -288
-; BE-NEXT:    .cfi_offset r15, -280
-; BE-NEXT:    .cfi_offset r16, -272
-; BE-NEXT:    .cfi_offset r17, -264
-; BE-NEXT:    .cfi_offset r18, -256
-; BE-NEXT:    .cfi_offset r19, -248
-; BE-NEXT:    .cfi_offset r20, -240
-; BE-NEXT:    .cfi_offset r21, -232
-; BE-NEXT:    .cfi_offset r22, -224
-; BE-NEXT:    .cfi_offset r23, -216
-; BE-NEXT:    .cfi_offset r24, -208
-; BE-NEXT:    .cfi_offset r25, -200
-; BE-NEXT:    .cfi_offset r26, -192
-; BE-NEXT:    .cfi_offset r27, -184
-; BE-NEXT:    .cfi_offset r28, -176
-; BE-NEXT:    .cfi_offset r29, -168
-; BE-NEXT:    .cfi_offset r30, -160
-; BE-NEXT:    .cfi_offset r31, -152
-; BE-NEXT:    .cfi_offset f14, -144
-; BE-NEXT:    .cfi_offset f15, -136
-; BE-NEXT:    .cfi_offset f16, -128
-; BE-NEXT:    .cfi_offset f17, -120
-; BE-NEXT:    .cfi_offset f18, -112
-; BE-NEXT:    .cfi_offset f19, -104
-; BE-NEXT:    .cfi_offset f20, -96
-; BE-NEXT:    .cfi_offset f21, -88
-; BE-NEXT:    .cfi_offset f22, -80
-; BE-NEXT:    .cfi_offset f23, -72
-; BE-NEXT:    .cfi_offset f24, -64
-; BE-NEXT:    .cfi_offset f25, -56
-; BE-NEXT:    .cfi_offset f26, -48
-; BE-NEXT:    .cfi_offset f27, -40
-; BE-NEXT:    .cfi_offset f28, -32
-; BE-NEXT:    .cfi_offset f29, -24
-; BE-NEXT:    .cfi_offset f30, -16
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f20, 768(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f20, f1
 ; BE-NEXT:    fmr f1, f2
@@ -1924,6 +1711,7 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; BE-NEXT:    stfd f30, 848(r1) # 8-byte Folded Spill
 ; BE-NEXT:    stfd f31, 856(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f31, f13
+; BE-NEXT:    mr r30, r3
 ; BE-NEXT:    fmr f29, f12
 ; BE-NEXT:    fmr f30, f11
 ; BE-NEXT:    fmr f28, f10
@@ -1934,7 +1722,6 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; BE-NEXT:    fmr f23, f5
 ; BE-NEXT:    fmr f22, f4
 ; BE-NEXT:    fmr f21, f3
-; BE-NEXT:    mr r30, r3
 ; BE-NEXT:    bl __truncsfhf2
 ; BE-NEXT:    nop
 ; BE-NEXT:    fmr f1, f20
@@ -2437,98 +2224,48 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -688(r1)
-; CHECK-NEXT:    std r0, 704(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 688
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r14, -288
-; CHECK-NEXT:    .cfi_offset r15, -280
-; CHECK-NEXT:    .cfi_offset r16, -272
-; CHECK-NEXT:    .cfi_offset r17, -264
-; CHECK-NEXT:    .cfi_offset r18, -256
-; CHECK-NEXT:    .cfi_offset r19, -248
-; CHECK-NEXT:    .cfi_offset r20, -240
-; CHECK-NEXT:    .cfi_offset r21, -232
-; CHECK-NEXT:    .cfi_offset r22, -224
-; CHECK-NEXT:    .cfi_offset r23, -216
-; CHECK-NEXT:    .cfi_offset r24, -208
-; CHECK-NEXT:    .cfi_offset r25, -200
-; CHECK-NEXT:    .cfi_offset r26, -192
-; CHECK-NEXT:    .cfi_offset r27, -184
-; CHECK-NEXT:    .cfi_offset r28, -176
-; CHECK-NEXT:    .cfi_offset r29, -168
-; CHECK-NEXT:    .cfi_offset r30, -160
-; CHECK-NEXT:    .cfi_offset r31, -152
-; CHECK-NEXT:    .cfi_offset f14, -144
-; CHECK-NEXT:    .cfi_offset f15, -136
-; CHECK-NEXT:    .cfi_offset f16, -128
-; CHECK-NEXT:    .cfi_offset f17, -120
-; CHECK-NEXT:    .cfi_offset f18, -112
-; CHECK-NEXT:    .cfi_offset f19, -104
-; CHECK-NEXT:    .cfi_offset f20, -96
-; CHECK-NEXT:    .cfi_offset f21, -88
-; CHECK-NEXT:    .cfi_offset f22, -80
-; CHECK-NEXT:    .cfi_offset f23, -72
-; CHECK-NEXT:    .cfi_offset f24, -64
-; CHECK-NEXT:    .cfi_offset f25, -56
-; CHECK-NEXT:    .cfi_offset f26, -48
-; CHECK-NEXT:    .cfi_offset f27, -40
-; CHECK-NEXT:    .cfi_offset f28, -32
-; CHECK-NEXT:    .cfi_offset f29, -24
-; CHECK-NEXT:    .cfi_offset f30, -16
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v20, -480
-; CHECK-NEXT:    .cfi_offset v21, -464
-; CHECK-NEXT:    .cfi_offset v22, -448
-; CHECK-NEXT:    .cfi_offset v23, -432
-; CHECK-NEXT:    .cfi_offset v24, -416
-; CHECK-NEXT:    .cfi_offset v25, -400
-; CHECK-NEXT:    .cfi_offset v26, -384
-; CHECK-NEXT:    .cfi_offset v27, -368
-; CHECK-NEXT:    .cfi_offset v28, -352
-; CHECK-NEXT:    .cfi_offset v29, -336
-; CHECK-NEXT:    .cfi_offset v30, -320
-; CHECK-NEXT:    .cfi_offset v31, -304
 ; CHECK-NEXT:    li r4, 208
+; CHECK-NEXT:    std r0, 704(r1)
 ; CHECK-NEXT:    std r14, 400(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r15, 408(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r16, 416(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r17, 424(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r18, 432(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r19, 440(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r20, 448(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v20, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v20, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 224
+; CHECK-NEXT:    std r20, 448(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r21, 456(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r22, 464(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r23, 472(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r24, 480(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r25, 488(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r26, 496(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v21, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v21, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 240
+; CHECK-NEXT:    std r26, 496(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r27, 504(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r28, 512(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r29, 520(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r30, 528(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    mr r30, r3
-; CHECK-NEXT:    std r31, 536(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v22, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v22, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 256
+; CHECK-NEXT:    std r31, 536(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f14, 544(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f15, 552(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f16, 560(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f17, 568(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f18, 576(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stfd f19, 584(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v23, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v23, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 272
+; CHECK-NEXT:    stfd f19, 584(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f20, 592(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f20, f2
 ; CHECK-NEXT:    stfd f21, 600(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f21, f3
 ; CHECK-NEXT:    stfd f22, 608(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f22, f4
-; CHECK-NEXT:    stvx v24, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v24, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 288
 ; CHECK-NEXT:    stfd f23, 616(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f23, f5
@@ -2536,7 +2273,7 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    fmr f24, f6
 ; CHECK-NEXT:    stfd f25, 632(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f25, f7
-; CHECK-NEXT:    stvx v25, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v25, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 304
 ; CHECK-NEXT:    stfd f26, 640(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f26, f8
@@ -2544,7 +2281,7 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    fmr f27, f9
 ; CHECK-NEXT:    stfd f28, 656(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f28, f10
-; CHECK-NEXT:    stvx v26, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v26, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 320
 ; CHECK-NEXT:    stfd f29, 664(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f29, f11
@@ -2552,15 +2289,15 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    fmr f30, f12
 ; CHECK-NEXT:    stfd f31, 680(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f13
-; CHECK-NEXT:    stvx v27, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v27, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 336
-; CHECK-NEXT:    stvx v28, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v28, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 352
-; CHECK-NEXT:    stvx v29, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 368
-; CHECK-NEXT:    stvx v30, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 384
-; CHECK-NEXT:    stvx v31, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f20
@@ -3039,7 +2776,7 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    li r3, 384
 ; CHECK-NEXT:    xxswapd vs4, vs4
 ; CHECK-NEXT:    stxvd2x vs4, 0, r30
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 368
 ; CHECK-NEXT:    lfd f31, 680(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f30, 672(r1) # 8-byte Folded Reload
@@ -3057,7 +2794,7 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    lfd f18, 576(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f17, 568(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f16, 560(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 352
 ; CHECK-NEXT:    lfd f15, 552(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f14, 544(r1) # 8-byte Folded Reload
@@ -3065,7 +2802,7 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    ld r30, 528(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, 520(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r28, 512(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 336
 ; CHECK-NEXT:    ld r27, 504(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r26, 496(r1) # 8-byte Folded Reload
@@ -3073,7 +2810,7 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    ld r24, 480(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r23, 472(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r22, 464(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 320
 ; CHECK-NEXT:    ld r21, 456(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r20, 448(r1) # 8-byte Folded Reload
@@ -3081,23 +2818,23 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    ld r18, 432(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r17, 424(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r16, 416(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 304
 ; CHECK-NEXT:    ld r15, 408(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r14, 400(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 288
-; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 272
-; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 256
-; CHECK-NEXT:    lvx v23, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v23, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 240
-; CHECK-NEXT:    lvx v22, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v22, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 224
-; CHECK-NEXT:    lvx v21, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v21, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 208
-; CHECK-NEXT:    lvx v20, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v20, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 688
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -3107,95 +2844,62 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
 ; FAST-NEXT:    stdu r1, -480(r1)
-; FAST-NEXT:    std r0, 496(r1)
-; FAST-NEXT:    .cfi_def_cfa_offset 480
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset r30, -160
-; FAST-NEXT:    .cfi_offset f14, -144
-; FAST-NEXT:    .cfi_offset f15, -136
-; FAST-NEXT:    .cfi_offset f16, -128
-; FAST-NEXT:    .cfi_offset f17, -120
-; FAST-NEXT:    .cfi_offset f18, -112
-; FAST-NEXT:    .cfi_offset f19, -104
-; FAST-NEXT:    .cfi_offset f20, -96
-; FAST-NEXT:    .cfi_offset f21, -88
-; FAST-NEXT:    .cfi_offset f22, -80
-; FAST-NEXT:    .cfi_offset f23, -72
-; FAST-NEXT:    .cfi_offset f24, -64
-; FAST-NEXT:    .cfi_offset f25, -56
-; FAST-NEXT:    .cfi_offset f26, -48
-; FAST-NEXT:    .cfi_offset f27, -40
-; FAST-NEXT:    .cfi_offset f28, -32
-; FAST-NEXT:    .cfi_offset f29, -24
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
-; FAST-NEXT:    .cfi_offset v20, -352
-; FAST-NEXT:    .cfi_offset v21, -336
-; FAST-NEXT:    .cfi_offset v22, -320
-; FAST-NEXT:    .cfi_offset v23, -304
-; FAST-NEXT:    .cfi_offset v24, -288
-; FAST-NEXT:    .cfi_offset v25, -272
-; FAST-NEXT:    .cfi_offset v26, -256
-; FAST-NEXT:    .cfi_offset v27, -240
-; FAST-NEXT:    .cfi_offset v28, -224
-; FAST-NEXT:    .cfi_offset v29, -208
-; FAST-NEXT:    .cfi_offset v30, -192
-; FAST-NEXT:    .cfi_offset v31, -176
 ; FAST-NEXT:    li r4, 128
+; FAST-NEXT:    std r0, 496(r1)
 ; FAST-NEXT:    std r30, 320(r1) # 8-byte Folded Spill
+; FAST-NEXT:    mr r30, r3
 ; FAST-NEXT:    stfd f14, 336(r1) # 8-byte Folded Spill
-; FAST-NEXT:    fmr f14, f5
 ; FAST-NEXT:    stfd f15, 344(r1) # 8-byte Folded Spill
+; FAST-NEXT:    fmr f14, f5
 ; FAST-NEXT:    stfd f16, 352(r1) # 8-byte Folded Spill
-; FAST-NEXT:    fmr f16, f4
-; FAST-NEXT:    mr r30, r3
-; FAST-NEXT:    stvx v20, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v20, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 144
+; FAST-NEXT:    fmr f16, f4
 ; FAST-NEXT:    stfd f17, 360(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f18, 368(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f19, 376(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f20, 384(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f21, 392(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stfd f22, 400(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stvx v21, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v21, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 160
+; FAST-NEXT:    stfd f22, 400(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f23, 408(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f24, 416(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f25, 424(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f26, 432(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f27, 440(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stfd f28, 448(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stvx v22, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v22, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 176
 ; FAST-NEXT:    xxlor v22, f3, f3
+; FAST-NEXT:    stfd f28, 448(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f29, 456(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    fmr f29, f9
 ; FAST-NEXT:    stfd f30, 464(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f31, 472(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stvx v23, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v23, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 192
 ; FAST-NEXT:    xxlor v23, f2, f2
-; FAST-NEXT:    stvx v24, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v24, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 208
-; FAST-NEXT:    stvx v25, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v25, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 224
 ; FAST-NEXT:    xxlor v25, f13, f13
-; FAST-NEXT:    stvx v26, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v26, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 240
 ; FAST-NEXT:    xxlor v26, f12, f12
-; FAST-NEXT:    stvx v27, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v27, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 256
 ; FAST-NEXT:    xxlor v27, f11, f11
-; FAST-NEXT:    stvx v28, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v28, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 272
 ; FAST-NEXT:    xxlor v28, f10, f10
-; FAST-NEXT:    stvx v29, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v29, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 288
 ; FAST-NEXT:    xxlor v29, f8, f8
-; FAST-NEXT:    stvx v30, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v30, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 304
 ; FAST-NEXT:    xxlor v30, f7, f7
-; FAST-NEXT:    stvx v31, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v31, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 44
 ; FAST-NEXT:    xxlor v31, f6, f6
 ; FAST-NEXT:    stxsspx f1, r1, r4 # 4-byte Folded Spill
@@ -3624,30 +3328,30 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 ; FAST-NEXT:    lfd f16, 352(r1) # 8-byte Folded Reload
 ; FAST-NEXT:    lfd f15, 344(r1) # 8-byte Folded Reload
 ; FAST-NEXT:    lfd f14, 336(r1) # 8-byte Folded Reload
-; FAST-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 288
 ; FAST-NEXT:    ld r30, 320(r1) # 8-byte Folded Reload
-; FAST-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 272
-; FAST-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 256
-; FAST-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 240
-; FAST-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 224
-; FAST-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 208
-; FAST-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 192
-; FAST-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 176
-; FAST-NEXT:    lvx v23, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v23, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 160
-; FAST-NEXT:    lvx v22, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v22, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 144
-; FAST-NEXT:    lvx v21, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v21, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 128
-; FAST-NEXT:    lvx v20, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v20, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    addi r1, r1, 480
 ; FAST-NEXT:    ld r0, 16(r1)
 ; FAST-NEXT:    mtlr r0
@@ -3657,14 +3361,12 @@ define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
 }
 declare <32 x i64> @llvm.llrint.v32i64.v32f16(<32 x half>)
 
-define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
+define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) nounwind {
 ; BE-LABEL: llrint_v1i64_v1f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -112(r1)
 ; BE-NEXT:    std r0, 128(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 112
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    bl llrintf
 ; BE-NEXT:    nop
 ; BE-NEXT:    addi r1, r1, 112
@@ -3677,8 +3379,6 @@ define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -32(r1)
 ; CHECK-NEXT:    std r0, 48(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    bl llrintf
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
@@ -3696,15 +3396,13 @@ define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
 
-define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
+define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) nounwind {
 ; BE-LABEL: llrint_v2i64_v2f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -144(r1)
-; BE-NEXT:    std r0, 160(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 144
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    std r0, 160(r1)
 ; BE-NEXT:    stxvw4x v2, 0, r3
 ; BE-NEXT:    lfs f1, 116(r1)
 ; BE-NEXT:    bl llrintf
@@ -3725,14 +3423,11 @@ define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -64(r1)
-; CHECK-NEXT:    std r0, 80(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 80(r1)
 ; CHECK-NEXT:    xscvspdpn f1, vs0
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v2
 ; CHECK-NEXT:    bl llrintf
 ; CHECK-NEXT:    nop
@@ -3744,7 +3439,7 @@ define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 ; CHECK-NEXT:    mtfprd f0, r3
 ; CHECK-NEXT:    li r3, 48
 ; CHECK-NEXT:    xxmrghd v2, vs0, v31
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 64
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -3769,15 +3464,13 @@ define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
 
-define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
+define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) nounwind {
 ; BE-LABEL: llrint_v4i64_v4f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -160(r1)
-; BE-NEXT:    std r0, 176(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 160
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    std r0, 176(r1)
 ; BE-NEXT:    stxvw4x v2, 0, r3
 ; BE-NEXT:    lfs f1, 116(r1)
 ; BE-NEXT:    bl llrintf
@@ -3808,17 +3501,13 @@ define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -80(r1)
-; CHECK-NEXT:    std r0, 96(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 96(r1)
 ; CHECK-NEXT:    xscvspdpn f1, vs0
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v2
 ; CHECK-NEXT:    bl llrintf
 ; CHECK-NEXT:    nop
@@ -3841,9 +3530,9 @@ define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    vmr v2, v30
 ; CHECK-NEXT:    xxmrghd v3, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 80
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -3879,15 +3568,13 @@ define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
 
-define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
+define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) nounwind {
 ; BE-LABEL: llrint_v8i64_v8f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -208(r1)
-; BE-NEXT:    std r0, 224(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 208
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    std r0, 224(r1)
 ; BE-NEXT:    stxvw4x v2, 0, r3
 ; BE-NEXT:    addi r3, r1, 128
 ; BE-NEXT:    stxvw4x v3, 0, r3
@@ -3940,24 +3627,18 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -112(r1)
-; CHECK-NEXT:    std r0, 128(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 112
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v28, -64
-; CHECK-NEXT:    .cfi_offset v29, -48
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
 ; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    xscvspdpn f1, vs0
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    vmr v30, v2
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v3
 ; CHECK-NEXT:    bl llrintf
 ; CHECK-NEXT:    nop
@@ -4003,13 +3684,13 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 ; CHECK-NEXT:    vmr v2, v29
 ; CHECK-NEXT:    vmr v4, v28
 ; CHECK-NEXT:    xxmrghd v5, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 112
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4067,15 +3748,13 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
 
-define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
+define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) nounwind {
 ; BE-LABEL: llrint_v16i64_v16f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -304(r1)
-; BE-NEXT:    std r0, 320(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 304
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    std r0, 320(r1)
 ; BE-NEXT:    stxvw4x v2, 0, r3
 ; BE-NEXT:    addi r3, r1, 128
 ; BE-NEXT:    stxvw4x v3, 0, r3
@@ -4172,38 +3851,28 @@ define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -176(r1)
-; CHECK-NEXT:    std r0, 192(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 176
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v24, -128
-; CHECK-NEXT:    .cfi_offset v25, -112
-; CHECK-NEXT:    .cfi_offset v26, -96
-; CHECK-NEXT:    .cfi_offset v27, -80
-; CHECK-NEXT:    .cfi_offset v28, -64
-; CHECK-NEXT:    .cfi_offset v29, -48
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 192(r1)
 ; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-NEXT:    stvx v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    xscvspdpn f1, vs0
-; CHECK-NEXT:    stvx v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    stvx v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    vmr v26, v3
-; CHECK-NEXT:    stvx v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 112
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 128
 ; CHECK-NEXT:    vmr v28, v4
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 144
 ; CHECK-NEXT:    vmr v29, v2
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 160
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v5
 ; CHECK-NEXT:    bl llrintf
 ; CHECK-NEXT:    nop
@@ -4295,21 +3964,21 @@ define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 ; CHECK-NEXT:    vmr v6, v25
 ; CHECK-NEXT:    vmr v8, v24
 ; CHECK-NEXT:    xxmrghd v9, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 144
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 128
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 112
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 96
-; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
-; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 176
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4411,14 +4080,12 @@ define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
 
-define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
+define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) nounwind {
 ; BE-LABEL: llrint_v1i64_v1f64:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -112(r1)
 ; BE-NEXT:    std r0, 128(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 112
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    bl llrint
 ; BE-NEXT:    nop
 ; BE-NEXT:    addi r1, r1, 112
@@ -4431,8 +4098,6 @@ define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -32(r1)
 ; CHECK-NEXT:    std r0, 48(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    bl llrint
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
@@ -4450,16 +4115,13 @@ define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
 
-define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
+define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) nounwind {
 ; BE-LABEL: llrint_v2i64_v2f64:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -160(r1)
-; BE-NEXT:    std r0, 176(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 160
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset v31, -16
 ; BE-NEXT:    li r3, 144
+; BE-NEXT:    std r0, 176(r1)
 ; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    vmr v31, v2
 ; BE-NEXT:    xxlor f1, v31, v31
@@ -4483,12 +4145,9 @@ define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -64(r1)
-; CHECK-NEXT:    std r0, 80(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    std r0, 80(r1)
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v2
 ; CHECK-NEXT:    xxlor f1, v31, v31
 ; CHECK-NEXT:    bl llrint
@@ -4500,7 +4159,7 @@ define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
 ; CHECK-NEXT:    mtfprd f0, r3
 ; CHECK-NEXT:    li r3, 48
 ; CHECK-NEXT:    xxmrghd v2, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 64
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4523,17 +4182,13 @@ define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
 
-define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
+define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) nounwind {
 ; BE-LABEL: llrint_v4i64_v4f64:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -192(r1)
-; BE-NEXT:    std r0, 208(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 192
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset v30, -32
-; BE-NEXT:    .cfi_offset v31, -16
 ; BE-NEXT:    li r3, 160
+; BE-NEXT:    std r0, 208(r1)
 ; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    vmr v30, v2
 ; BE-NEXT:    li r3, 176
@@ -4572,17 +4227,13 @@ define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -80(r1)
-; CHECK-NEXT:    std r0, 96(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    std r0, 96(r1)
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v30, v2
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    xxlor f1, v30, v30
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v3
 ; CHECK-NEXT:    bl llrint
 ; CHECK-NEXT:    nop
@@ -4603,9 +4254,9 @@ define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    vmr v2, v30
 ; CHECK-NEXT:    xxmrghd v3, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 80
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4637,25 +4288,19 @@ define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
 
-define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
+define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) nounwind {
 ; BE-LABEL: llrint_v8i64_v8f64:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -256(r1)
-; BE-NEXT:    std r0, 272(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 256
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset v28, -64
-; BE-NEXT:    .cfi_offset v29, -48
-; BE-NEXT:    .cfi_offset v30, -32
-; BE-NEXT:    .cfi_offset v31, -16
 ; BE-NEXT:    li r3, 192
+; BE-NEXT:    std r0, 272(r1)
 ; BE-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    li r3, 208
 ; BE-NEXT:    vmr v28, v2
-; BE-NEXT:    xxlor f1, v28, v28
 ; BE-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    li r3, 224
+; BE-NEXT:    xxlor f1, v28, v28
 ; BE-NEXT:    vmr v29, v3
 ; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    li r3, 240
@@ -4718,25 +4363,19 @@ define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -112(r1)
-; CHECK-NEXT:    std r0, 128(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 112
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v28, -64
-; CHECK-NEXT:    .cfi_offset v29, -48
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    vmr v28, v2
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
 ; CHECK-NEXT:    xxlor f1, v28, v28
 ; CHECK-NEXT:    vmr v29, v3
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    vmr v30, v4
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v5
 ; CHECK-NEXT:    bl llrint
 ; CHECK-NEXT:    nop
@@ -4777,13 +4416,13 @@ define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
 ; CHECK-NEXT:    vmr v3, v29
 ; CHECK-NEXT:    vmr v2, v28
 ; CHECK-NEXT:    xxmrghd v5, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 112
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4832,3 +4471,536 @@ define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
   ret <8 x i64> %a
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
+
+define <1 x i64> @llrint_v1i64_v1f128(<1 x fp128> %x) nounwind {
+; BE-LABEL: llrint_v1i64_v1f128:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    std r0, 128(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v1i64_v1f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v1i64_v1f128:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -32(r1)
+; FAST-NEXT:    std r0, 48(r1)
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    addi r1, r1, 32
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f128(<1 x fp128> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f128(<1 x fp128>)
+
+define <2 x i64> @llrint_v2i64_v2f128(<2 x fp128> %x) nounwind {
+; BE-LABEL: llrint_v2i64_v2f128:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -160(r1)
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    std r0, 176(r1)
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v2
+; BE-NEXT:    vmr v2, v3
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v31
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 160
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v2i64_v2f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -80(r1)
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 96(r1)
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v3
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxmrghd v2, vs0, v30
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 80
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v2i64_v2f128:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -80(r1)
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    std r0, 96(r1)
+; FAST-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    vmr v31, v3
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v31
+; FAST-NEXT:    mtvsrd v30, r3
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    xxmrghd v2, vs0, v30
+; FAST-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    addi r1, r1, 80
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f128(<2 x fp128> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f128(<2 x fp128>)
+
+define <4 x i64> @llrint_v4i64_v4f128(<4 x fp128> %x) nounwind {
+; BE-LABEL: llrint_v4i64_v4f128:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -208(r1)
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    std r0, 224(r1)
+; BE-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    vmr v29, v2
+; BE-NEXT:    vmr v2, v3
+; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    vmr v30, v4
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v5
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v29
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v31
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v30
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 208
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v4i64_v4f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -112(r1)
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    vmr v29, v3
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v30, v4
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v5
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v29
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    xxmrghd v29, vs0, v28
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v2, v29
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    xxmrghd v3, vs0, v30
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 112
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v4i64_v4f128:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -112(r1)
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    std r0, 128(r1)
+; FAST-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    vmr v29, v3
+; FAST-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    vmr v30, v4
+; FAST-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    vmr v31, v5
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v29
+; FAST-NEXT:    mtvsrd v28, r3
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v30
+; FAST-NEXT:    xxmrghd v29, vs0, v28
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v31
+; FAST-NEXT:    mtvsrd v30, r3
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    vmr v2, v29
+; FAST-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    xxmrghd v3, vs0, v30
+; FAST-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    addi r1, r1, 112
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f128(<4 x fp128> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f128(<4 x fp128>)
+
+define <8 x i64> @llrint_v8i64_v8f128(<8 x fp128> %x) nounwind {
+; BE-LABEL: llrint_v8i64_v8f128:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -304(r1)
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    std r0, 320(r1)
+; BE-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    vmr v25, v2
+; BE-NEXT:    vmr v2, v3
+; BE-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    vmr v26, v4
+; BE-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    vmr v27, v5
+; BE-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 256
+; BE-NEXT:    vmr v28, v6
+; BE-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 272
+; BE-NEXT:    vmr v29, v7
+; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 288
+; BE-NEXT:    vmr v30, v8
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v9
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v25
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v27
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v26
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v29
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v28
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v31
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v30
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    bl llrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    li r3, 288
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 272
+; BE-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 256
+; BE-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 304
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: llrint_v8i64_v8f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -176(r1)
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 192(r1)
+; CHECK-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    vmr v25, v3
+; CHECK-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v26, v4
+; CHECK-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    vmr v27, v5
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    vmr v28, v6
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    vmr v29, v7
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    vmr v30, v8
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v9
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v25
+; CHECK-NEXT:    mtvsrd v24, r3
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v26
+; CHECK-NEXT:    xxmrghd v25, vs0, v24
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v27
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v28
+; CHECK-NEXT:    xxmrghd v27, vs0, v26
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v29
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    xxmrghd v29, vs0, v28
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl llrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    vmr v4, v29
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    vmr v3, v27
+; CHECK-NEXT:    vmr v2, v25
+; CHECK-NEXT:    xxmrghd v5, vs0, v30
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 176
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: llrint_v8i64_v8f128:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -176(r1)
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    std r0, 192(r1)
+; FAST-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    vmr v25, v3
+; FAST-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    vmr v26, v4
+; FAST-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    vmr v27, v5
+; FAST-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    vmr v28, v6
+; FAST-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    vmr v29, v7
+; FAST-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    vmr v30, v8
+; FAST-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    vmr v31, v9
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v25
+; FAST-NEXT:    mtvsrd v24, r3
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v26
+; FAST-NEXT:    xxmrghd v25, vs0, v24
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v27
+; FAST-NEXT:    mtvsrd v26, r3
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v28
+; FAST-NEXT:    xxmrghd v27, vs0, v26
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v29
+; FAST-NEXT:    mtvsrd v28, r3
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v30
+; FAST-NEXT:    xxmrghd v29, vs0, v28
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v31
+; FAST-NEXT:    mtvsrd v30, r3
+; FAST-NEXT:    bl llrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    vmr v4, v29
+; FAST-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    vmr v3, v27
+; FAST-NEXT:    vmr v2, v25
+; FAST-NEXT:    xxmrghd v5, vs0, v30
+; FAST-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    addi r1, r1, 176
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f128(<8 x fp128> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f128(<8 x fp128>)
diff --git a/llvm/test/CodeGen/PowerPC/vector-lrint.ll b/llvm/test/CodeGen/PowerPC/vector-lrint.ll
index c2576d4631db8..f4375362f861c 100644
--- a/llvm/test/CodeGen/PowerPC/vector-lrint.ll
+++ b/llvm/test/CodeGen/PowerPC/vector-lrint.ll
@@ -1,4 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; FIXME: crash "Input type needs to be promoted!"
+; SKIP: sed 's/iXLen/i32/g' %s | llc -ppc-asm-full-reg-names \
+; SKIP:   -ppc-vsr-nums-as-vr -mtriple=powerpc-unknown-unknown \
+; SKIP:   -verify-machineinstrs | FileCheck %s --check-prefixes=PPC32
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mcpu=pwr7 -ppc-asm-full-reg-names \
 ; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64-unknown-unknown \
 ; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=BE
@@ -9,6 +13,10 @@
 ; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64le-unknown-unknown \
 ; RUN:   -verify-machineinstrs --enable-unsafe-fp-math | \
 ; RUN:   FileCheck %s --check-prefixes=FAST
+; FIXME: crash "Input type needs to be promoted!"
+; SKIP: sed 's/iXLen/i64/g' %s | llc -ppc-asm-full-reg-names \
+; SKIP:   -ppc-vsr-nums-as-vr -mtriple=powerpc-unknown-unknown \
+; SKIP:   -verify-machineinstrs | FileCheck %s --check-prefixes=PPC32
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mcpu=pwr7 -ppc-asm-full-reg-names \
 ; RUN:   -ppc-vsr-nums-as-vr -mtriple=powerpc64-unknown-unknown \
 ; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=BE
@@ -20,14 +28,12 @@
 ; RUN:   -verify-machineinstrs --enable-unsafe-fp-math | \
 ; RUN:   FileCheck %s --check-prefixes=FAST
 
-define <1 x i64> @lrint_v1f16(<1 x half> %x) {
+define <1 x i64> @lrint_v1f16(<1 x half> %x) nounwind {
 ; BE-LABEL: lrint_v1f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -112(r1)
 ; BE-NEXT:    std r0, 128(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 112
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    bl __truncsfhf2
 ; BE-NEXT:    nop
 ; BE-NEXT:    clrldi r3, r3, 48
@@ -45,8 +51,6 @@ define <1 x i64> @lrint_v1f16(<1 x half> %x) {
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -32(r1)
 ; CHECK-NEXT:    std r0, 48(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    clrldi r3, r3, 48
@@ -64,8 +68,6 @@ define <1 x i64> @lrint_v1f16(<1 x half> %x) {
 ; FAST-NEXT:    mflr r0
 ; FAST-NEXT:    stdu r1, -32(r1)
 ; FAST-NEXT:    std r0, 48(r1)
-; FAST-NEXT:    .cfi_def_cfa_offset 32
-; FAST-NEXT:    .cfi_offset lr, 16
 ; FAST-NEXT:    bl __truncsfhf2
 ; FAST-NEXT:    nop
 ; FAST-NEXT:    clrldi r3, r3, 48
@@ -82,16 +84,12 @@ define <1 x i64> @lrint_v1f16(<1 x half> %x) {
 }
 declare <1 x i64> @llvm.lrint.v1i64.v1f16(<1 x half>)
 
-define <2 x i64> @lrint_v2f16(<2 x half> %x) {
+define <2 x i64> @lrint_v2f16(<2 x half> %x) nounwind {
 ; BE-LABEL: lrint_v2f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -160(r1)
 ; BE-NEXT:    std r0, 176(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 160
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r30, -24
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f31, 152(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f31, f1
 ; BE-NEXT:    fmr f1, f2
@@ -129,17 +127,12 @@ define <2 x i64> @lrint_v2f16(<2 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -96(r1)
-; CHECK-NEXT:    std r0, 112(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 96
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r30, -24
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v31, -48
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 112(r1)
 ; CHECK-NEXT:    std r30, 72(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f31, 88(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f2
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f31
@@ -164,7 +157,7 @@ define <2 x i64> @lrint_v2f16(<2 x half> %x) {
 ; CHECK-NEXT:    lfd f31, 88(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r30, 72(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    xxmrghd v2, vs0, v31
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 96
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -173,10 +166,6 @@ define <2 x i64> @lrint_v2f16(<2 x half> %x) {
 ; FAST-LABEL: lrint_v2f16:
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
-; FAST-NEXT:    .cfi_def_cfa_offset 48
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
 ; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stdu r1, -48(r1)
@@ -213,20 +202,12 @@ define <2 x i64> @lrint_v2f16(<2 x half> %x) {
 }
 declare <2 x i64> @llvm.lrint.v2i64.v2f16(<2 x half>)
 
-define <4 x i64> @lrint_v4f16(<4 x half> %x) {
+define <4 x i64> @lrint_v4f16(<4 x half> %x) nounwind {
 ; BE-LABEL: lrint_v4f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -208(r1)
 ; BE-NEXT:    std r0, 224(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 208
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r28, -56
-; BE-NEXT:    .cfi_offset r29, -48
-; BE-NEXT:    .cfi_offset r30, -40
-; BE-NEXT:    .cfi_offset f29, -24
-; BE-NEXT:    .cfi_offset f30, -16
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f29, 184(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f29, f1
 ; BE-NEXT:    fmr f1, f2
@@ -300,18 +281,8 @@ define <4 x i64> @lrint_v4f16(<4 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -144(r1)
-; CHECK-NEXT:    std r0, 160(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 144
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r28, -56
-; CHECK-NEXT:    .cfi_offset r29, -48
-; CHECK-NEXT:    .cfi_offset r30, -40
-; CHECK-NEXT:    .cfi_offset f29, -24
-; CHECK-NEXT:    .cfi_offset f30, -16
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v30, -96
-; CHECK-NEXT:    .cfi_offset v31, -80
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 160(r1)
 ; CHECK-NEXT:    std r28, 88(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r29, 96(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r30, 104(r1) # 8-byte Folded Spill
@@ -319,11 +290,11 @@ define <4 x i64> @lrint_v4f16(<4 x half> %x) {
 ; CHECK-NEXT:    fmr f29, f2
 ; CHECK-NEXT:    stfd f30, 128(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f30, f3
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    stfd f31, 136(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f4
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f29
@@ -376,11 +347,11 @@ define <4 x i64> @lrint_v4f16(<4 x half> %x) {
 ; CHECK-NEXT:    lfd f29, 120(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r30, 104(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, 96(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
 ; CHECK-NEXT:    ld r28, 88(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    xxmrghd v3, vs0, v30
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 144
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -389,12 +360,6 @@ define <4 x i64> @lrint_v4f16(<4 x half> %x) {
 ; FAST-LABEL: lrint_v4f16:
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
-; FAST-NEXT:    .cfi_def_cfa_offset 64
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset f28, -32
-; FAST-NEXT:    .cfi_offset f29, -24
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
 ; FAST-NEXT:    stfd f28, -32(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f29, -24(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
@@ -458,28 +423,12 @@ define <4 x i64> @lrint_v4f16(<4 x half> %x) {
 }
 declare <4 x i64> @llvm.lrint.v4i64.v4f16(<4 x half>)
 
-define <8 x i64> @lrint_v8f16(<8 x half> %x) {
+define <8 x i64> @lrint_v8f16(<8 x half> %x) nounwind {
 ; BE-LABEL: lrint_v8f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -304(r1)
 ; BE-NEXT:    std r0, 320(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 304
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r24, -120
-; BE-NEXT:    .cfi_offset r25, -112
-; BE-NEXT:    .cfi_offset r26, -104
-; BE-NEXT:    .cfi_offset r27, -96
-; BE-NEXT:    .cfi_offset r28, -88
-; BE-NEXT:    .cfi_offset r29, -80
-; BE-NEXT:    .cfi_offset r30, -72
-; BE-NEXT:    .cfi_offset f25, -56
-; BE-NEXT:    .cfi_offset f26, -48
-; BE-NEXT:    .cfi_offset f27, -40
-; BE-NEXT:    .cfi_offset f28, -32
-; BE-NEXT:    .cfi_offset f29, -24
-; BE-NEXT:    .cfi_offset f30, -16
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f25, 248(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f25, f1
 ; BE-NEXT:    fmr f1, f2
@@ -625,44 +574,24 @@ define <8 x i64> @lrint_v8f16(<8 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -240(r1)
-; CHECK-NEXT:    std r0, 256(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 240
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r24, -120
-; CHECK-NEXT:    .cfi_offset r25, -112
-; CHECK-NEXT:    .cfi_offset r26, -104
-; CHECK-NEXT:    .cfi_offset r27, -96
-; CHECK-NEXT:    .cfi_offset r28, -88
-; CHECK-NEXT:    .cfi_offset r29, -80
-; CHECK-NEXT:    .cfi_offset r30, -72
-; CHECK-NEXT:    .cfi_offset f25, -56
-; CHECK-NEXT:    .cfi_offset f26, -48
-; CHECK-NEXT:    .cfi_offset f27, -40
-; CHECK-NEXT:    .cfi_offset f28, -32
-; CHECK-NEXT:    .cfi_offset f29, -24
-; CHECK-NEXT:    .cfi_offset f30, -16
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v28, -192
-; CHECK-NEXT:    .cfi_offset v29, -176
-; CHECK-NEXT:    .cfi_offset v30, -160
-; CHECK-NEXT:    .cfi_offset v31, -144
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 256(r1)
 ; CHECK-NEXT:    std r24, 120(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r25, 128(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r26, 136(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r27, 144(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r28, 152(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r29, 160(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r30, 168(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    std r30, 168(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f25, 184(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f25, f2
 ; CHECK-NEXT:    stfd f26, 192(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f26, f3
 ; CHECK-NEXT:    stfd f27, 200(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f27, f4
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
 ; CHECK-NEXT:    stfd f28, 208(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f28, f5
@@ -670,11 +599,11 @@ define <8 x i64> @lrint_v8f16(<8 x half> %x) {
 ; CHECK-NEXT:    fmr f29, f6
 ; CHECK-NEXT:    stfd f30, 224(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f30, f7
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    stfd f31, 232(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f8
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f25
@@ -777,7 +706,7 @@ define <8 x i64> @lrint_v8f16(<8 x half> %x) {
 ; CHECK-NEXT:    vmr v4, v29
 ; CHECK-NEXT:    lfd f30, 224(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f29, 216(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
 ; CHECK-NEXT:    lfd f28, 208(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f27, 200(r1) # 8-byte Folded Reload
@@ -785,7 +714,7 @@ define <8 x i64> @lrint_v8f16(<8 x half> %x) {
 ; CHECK-NEXT:    lfd f25, 184(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r30, 168(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, 160(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    ld r28, 152(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r27, 144(r1) # 8-byte Folded Reload
@@ -793,9 +722,9 @@ define <8 x i64> @lrint_v8f16(<8 x half> %x) {
 ; CHECK-NEXT:    ld r26, 136(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r25, 128(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r24, 120(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 240
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -804,16 +733,6 @@ define <8 x i64> @lrint_v8f16(<8 x half> %x) {
 ; FAST-LABEL: lrint_v8f16:
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
-; FAST-NEXT:    .cfi_def_cfa_offset 96
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset f24, -64
-; FAST-NEXT:    .cfi_offset f25, -56
-; FAST-NEXT:    .cfi_offset f26, -48
-; FAST-NEXT:    .cfi_offset f27, -40
-; FAST-NEXT:    .cfi_offset f28, -32
-; FAST-NEXT:    .cfi_offset f29, -24
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
 ; FAST-NEXT:    stfd f24, -64(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f25, -56(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f26, -48(r1) # 8-byte Folded Spill
@@ -931,44 +850,12 @@ define <8 x i64> @lrint_v8f16(<8 x half> %x) {
 }
 declare <8 x i64> @llvm.lrint.v8i64.v8f16(<8 x half>)
 
-define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
+define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) nounwind {
 ; BE-LABEL: lrint_v16i64_v16f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -496(r1)
 ; BE-NEXT:    std r0, 512(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 496
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r16, -248
-; BE-NEXT:    .cfi_offset r17, -240
-; BE-NEXT:    .cfi_offset r18, -232
-; BE-NEXT:    .cfi_offset r19, -224
-; BE-NEXT:    .cfi_offset r20, -216
-; BE-NEXT:    .cfi_offset r21, -208
-; BE-NEXT:    .cfi_offset r22, -200
-; BE-NEXT:    .cfi_offset r23, -192
-; BE-NEXT:    .cfi_offset r24, -184
-; BE-NEXT:    .cfi_offset r25, -176
-; BE-NEXT:    .cfi_offset r26, -168
-; BE-NEXT:    .cfi_offset r27, -160
-; BE-NEXT:    .cfi_offset r28, -152
-; BE-NEXT:    .cfi_offset r29, -144
-; BE-NEXT:    .cfi_offset r30, -136
-; BE-NEXT:    .cfi_offset f17, -120
-; BE-NEXT:    .cfi_offset f18, -112
-; BE-NEXT:    .cfi_offset f19, -104
-; BE-NEXT:    .cfi_offset f20, -96
-; BE-NEXT:    .cfi_offset f21, -88
-; BE-NEXT:    .cfi_offset f22, -80
-; BE-NEXT:    .cfi_offset f23, -72
-; BE-NEXT:    .cfi_offset f24, -64
-; BE-NEXT:    .cfi_offset f25, -56
-; BE-NEXT:    .cfi_offset f26, -48
-; BE-NEXT:    .cfi_offset f27, -40
-; BE-NEXT:    .cfi_offset f28, -32
-; BE-NEXT:    .cfi_offset f29, -24
-; BE-NEXT:    .cfi_offset f30, -16
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f20, 400(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f20, f1
 ; BE-NEXT:    fmr f1, f2
@@ -1255,105 +1142,65 @@ define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -432(r1)
-; CHECK-NEXT:    std r0, 448(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 432
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r16, -248
-; CHECK-NEXT:    .cfi_offset r17, -240
-; CHECK-NEXT:    .cfi_offset r18, -232
-; CHECK-NEXT:    .cfi_offset r19, -224
-; CHECK-NEXT:    .cfi_offset r20, -216
-; CHECK-NEXT:    .cfi_offset r21, -208
-; CHECK-NEXT:    .cfi_offset r22, -200
-; CHECK-NEXT:    .cfi_offset r23, -192
-; CHECK-NEXT:    .cfi_offset r24, -184
-; CHECK-NEXT:    .cfi_offset r25, -176
-; CHECK-NEXT:    .cfi_offset r26, -168
-; CHECK-NEXT:    .cfi_offset r27, -160
-; CHECK-NEXT:    .cfi_offset r28, -152
-; CHECK-NEXT:    .cfi_offset r29, -144
-; CHECK-NEXT:    .cfi_offset r30, -136
-; CHECK-NEXT:    .cfi_offset f17, -120
-; CHECK-NEXT:    .cfi_offset f18, -112
-; CHECK-NEXT:    .cfi_offset f19, -104
-; CHECK-NEXT:    .cfi_offset f20, -96
-; CHECK-NEXT:    .cfi_offset f21, -88
-; CHECK-NEXT:    .cfi_offset f22, -80
-; CHECK-NEXT:    .cfi_offset f23, -72
-; CHECK-NEXT:    .cfi_offset f24, -64
-; CHECK-NEXT:    .cfi_offset f25, -56
-; CHECK-NEXT:    .cfi_offset f26, -48
-; CHECK-NEXT:    .cfi_offset f27, -40
-; CHECK-NEXT:    .cfi_offset f28, -32
-; CHECK-NEXT:    .cfi_offset f29, -24
-; CHECK-NEXT:    .cfi_offset f30, -16
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v24, -384
-; CHECK-NEXT:    .cfi_offset v25, -368
-; CHECK-NEXT:    .cfi_offset v26, -352
-; CHECK-NEXT:    .cfi_offset v27, -336
-; CHECK-NEXT:    .cfi_offset v28, -320
-; CHECK-NEXT:    .cfi_offset v29, -304
-; CHECK-NEXT:    .cfi_offset v30, -288
-; CHECK-NEXT:    .cfi_offset v31, -272
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 448(r1)
 ; CHECK-NEXT:    std r16, 184(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r17, 192(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r18, 200(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r19, 208(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r20, 216(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r21, 224(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r22, 232(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    std r22, 232(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r23, 240(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r24, 248(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r25, 256(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r26, 264(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r27, 272(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r28, 280(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    std r28, 280(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r29, 288(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r30, 296(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f17, 312(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f18, 320(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f19, 328(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    stfd f20, 336(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f20, f2
-; CHECK-NEXT:    stvx v26, r1, r3 # 16-byte Folded Spill
-; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    stfd f21, 344(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f21, f3
 ; CHECK-NEXT:    stfd f22, 352(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f22, f4
+; CHECK-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 112
 ; CHECK-NEXT:    stfd f23, 360(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f23, f5
-; CHECK-NEXT:    stvx v27, r1, r3 # 16-byte Folded Spill
-; CHECK-NEXT:    li r3, 112
 ; CHECK-NEXT:    stfd f24, 368(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f24, f6
 ; CHECK-NEXT:    stfd f25, 376(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f25, f7
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 128
 ; CHECK-NEXT:    stfd f26, 384(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f26, f8
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
-; CHECK-NEXT:    li r3, 128
 ; CHECK-NEXT:    stfd f27, 392(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f27, f9
 ; CHECK-NEXT:    stfd f28, 400(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f28, f10
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 144
 ; CHECK-NEXT:    stfd f29, 408(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f29, f11
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
-; CHECK-NEXT:    li r3, 144
 ; CHECK-NEXT:    stfd f30, 416(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f30, f12
 ; CHECK-NEXT:    stfd f31, 424(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f13
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 160
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f20
@@ -1556,7 +1403,7 @@ define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    vmr v4, v29
 ; CHECK-NEXT:    lfd f30, 416(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f29, 408(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 144
 ; CHECK-NEXT:    vmr v5, v28
 ; CHECK-NEXT:    vmr v6, v27
@@ -1564,7 +1411,7 @@ define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    vmr v8, v25
 ; CHECK-NEXT:    lfd f28, 400(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f27, 392(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 128
 ; CHECK-NEXT:    lfd f26, 384(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f25, 376(r1) # 8-byte Folded Reload
@@ -1572,7 +1419,7 @@ define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    lfd f24, 368(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f23, 360(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f22, 352(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 112
 ; CHECK-NEXT:    lfd f21, 344(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r30, 296(r1) # 8-byte Folded Reload
@@ -1580,7 +1427,7 @@ define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    lfd f19, 328(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, 288(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r28, 280(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    lfd f18, 320(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r27, 272(r1) # 8-byte Folded Reload
@@ -1588,7 +1435,7 @@ define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    ld r26, 264(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r25, 256(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r24, 248(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
 ; CHECK-NEXT:    ld r23, 240(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r22, 232(r1) # 8-byte Folded Reload
@@ -1596,13 +1443,13 @@ define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
 ; CHECK-NEXT:    ld r20, 216(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r19, 208(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r18, 200(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    ld r17, 192(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r16, 184(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 432
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -1611,24 +1458,6 @@ define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
 ; FAST-LABEL: lrint_v16i64_v16f16:
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
-; FAST-NEXT:    .cfi_def_cfa_offset 160
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset f16, -128
-; FAST-NEXT:    .cfi_offset f17, -120
-; FAST-NEXT:    .cfi_offset f18, -112
-; FAST-NEXT:    .cfi_offset f19, -104
-; FAST-NEXT:    .cfi_offset f20, -96
-; FAST-NEXT:    .cfi_offset f21, -88
-; FAST-NEXT:    .cfi_offset f22, -80
-; FAST-NEXT:    .cfi_offset f23, -72
-; FAST-NEXT:    .cfi_offset f24, -64
-; FAST-NEXT:    .cfi_offset f25, -56
-; FAST-NEXT:    .cfi_offset f26, -48
-; FAST-NEXT:    .cfi_offset f27, -40
-; FAST-NEXT:    .cfi_offset f28, -32
-; FAST-NEXT:    .cfi_offset f29, -24
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
 ; FAST-NEXT:    stfd f16, -128(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f17, -120(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f18, -112(r1) # 8-byte Folded Spill
@@ -1852,50 +1681,12 @@ define <16 x i64> @lrint_v16i64_v16f16(<16 x half> %x) {
 }
 declare <16 x i64> @llvm.lrint.v16i64.v16f16(<16 x half>)
 
-define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
+define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) nounwind {
 ; BE-LABEL: lrint_v32i64_v32f16:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -864(r1)
 ; BE-NEXT:    std r0, 880(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 864
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset r14, -288
-; BE-NEXT:    .cfi_offset r15, -280
-; BE-NEXT:    .cfi_offset r16, -272
-; BE-NEXT:    .cfi_offset r17, -264
-; BE-NEXT:    .cfi_offset r18, -256
-; BE-NEXT:    .cfi_offset r19, -248
-; BE-NEXT:    .cfi_offset r20, -240
-; BE-NEXT:    .cfi_offset r21, -232
-; BE-NEXT:    .cfi_offset r22, -224
-; BE-NEXT:    .cfi_offset r23, -216
-; BE-NEXT:    .cfi_offset r24, -208
-; BE-NEXT:    .cfi_offset r25, -200
-; BE-NEXT:    .cfi_offset r26, -192
-; BE-NEXT:    .cfi_offset r27, -184
-; BE-NEXT:    .cfi_offset r28, -176
-; BE-NEXT:    .cfi_offset r29, -168
-; BE-NEXT:    .cfi_offset r30, -160
-; BE-NEXT:    .cfi_offset r31, -152
-; BE-NEXT:    .cfi_offset f14, -144
-; BE-NEXT:    .cfi_offset f15, -136
-; BE-NEXT:    .cfi_offset f16, -128
-; BE-NEXT:    .cfi_offset f17, -120
-; BE-NEXT:    .cfi_offset f18, -112
-; BE-NEXT:    .cfi_offset f19, -104
-; BE-NEXT:    .cfi_offset f20, -96
-; BE-NEXT:    .cfi_offset f21, -88
-; BE-NEXT:    .cfi_offset f22, -80
-; BE-NEXT:    .cfi_offset f23, -72
-; BE-NEXT:    .cfi_offset f24, -64
-; BE-NEXT:    .cfi_offset f25, -56
-; BE-NEXT:    .cfi_offset f26, -48
-; BE-NEXT:    .cfi_offset f27, -40
-; BE-NEXT:    .cfi_offset f28, -32
-; BE-NEXT:    .cfi_offset f29, -24
-; BE-NEXT:    .cfi_offset f30, -16
-; BE-NEXT:    .cfi_offset f31, -8
 ; BE-NEXT:    stfd f20, 768(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f20, f1
 ; BE-NEXT:    fmr f1, f2
@@ -1935,6 +1726,7 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; BE-NEXT:    stfd f30, 848(r1) # 8-byte Folded Spill
 ; BE-NEXT:    stfd f31, 856(r1) # 8-byte Folded Spill
 ; BE-NEXT:    fmr f31, f13
+; BE-NEXT:    mr r30, r3
 ; BE-NEXT:    fmr f29, f12
 ; BE-NEXT:    fmr f30, f11
 ; BE-NEXT:    fmr f28, f10
@@ -1945,7 +1737,6 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; BE-NEXT:    fmr f23, f5
 ; BE-NEXT:    fmr f22, f4
 ; BE-NEXT:    fmr f21, f3
-; BE-NEXT:    mr r30, r3
 ; BE-NEXT:    bl __truncsfhf2
 ; BE-NEXT:    nop
 ; BE-NEXT:    fmr f1, f20
@@ -2448,98 +2239,48 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -688(r1)
-; CHECK-NEXT:    std r0, 704(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 688
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset r14, -288
-; CHECK-NEXT:    .cfi_offset r15, -280
-; CHECK-NEXT:    .cfi_offset r16, -272
-; CHECK-NEXT:    .cfi_offset r17, -264
-; CHECK-NEXT:    .cfi_offset r18, -256
-; CHECK-NEXT:    .cfi_offset r19, -248
-; CHECK-NEXT:    .cfi_offset r20, -240
-; CHECK-NEXT:    .cfi_offset r21, -232
-; CHECK-NEXT:    .cfi_offset r22, -224
-; CHECK-NEXT:    .cfi_offset r23, -216
-; CHECK-NEXT:    .cfi_offset r24, -208
-; CHECK-NEXT:    .cfi_offset r25, -200
-; CHECK-NEXT:    .cfi_offset r26, -192
-; CHECK-NEXT:    .cfi_offset r27, -184
-; CHECK-NEXT:    .cfi_offset r28, -176
-; CHECK-NEXT:    .cfi_offset r29, -168
-; CHECK-NEXT:    .cfi_offset r30, -160
-; CHECK-NEXT:    .cfi_offset r31, -152
-; CHECK-NEXT:    .cfi_offset f14, -144
-; CHECK-NEXT:    .cfi_offset f15, -136
-; CHECK-NEXT:    .cfi_offset f16, -128
-; CHECK-NEXT:    .cfi_offset f17, -120
-; CHECK-NEXT:    .cfi_offset f18, -112
-; CHECK-NEXT:    .cfi_offset f19, -104
-; CHECK-NEXT:    .cfi_offset f20, -96
-; CHECK-NEXT:    .cfi_offset f21, -88
-; CHECK-NEXT:    .cfi_offset f22, -80
-; CHECK-NEXT:    .cfi_offset f23, -72
-; CHECK-NEXT:    .cfi_offset f24, -64
-; CHECK-NEXT:    .cfi_offset f25, -56
-; CHECK-NEXT:    .cfi_offset f26, -48
-; CHECK-NEXT:    .cfi_offset f27, -40
-; CHECK-NEXT:    .cfi_offset f28, -32
-; CHECK-NEXT:    .cfi_offset f29, -24
-; CHECK-NEXT:    .cfi_offset f30, -16
-; CHECK-NEXT:    .cfi_offset f31, -8
-; CHECK-NEXT:    .cfi_offset v20, -480
-; CHECK-NEXT:    .cfi_offset v21, -464
-; CHECK-NEXT:    .cfi_offset v22, -448
-; CHECK-NEXT:    .cfi_offset v23, -432
-; CHECK-NEXT:    .cfi_offset v24, -416
-; CHECK-NEXT:    .cfi_offset v25, -400
-; CHECK-NEXT:    .cfi_offset v26, -384
-; CHECK-NEXT:    .cfi_offset v27, -368
-; CHECK-NEXT:    .cfi_offset v28, -352
-; CHECK-NEXT:    .cfi_offset v29, -336
-; CHECK-NEXT:    .cfi_offset v30, -320
-; CHECK-NEXT:    .cfi_offset v31, -304
 ; CHECK-NEXT:    li r4, 208
+; CHECK-NEXT:    std r0, 704(r1)
 ; CHECK-NEXT:    std r14, 400(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r15, 408(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r16, 416(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r17, 424(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r18, 432(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r19, 440(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r20, 448(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v20, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v20, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 224
+; CHECK-NEXT:    std r20, 448(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r21, 456(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r22, 464(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r23, 472(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r24, 480(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r25, 488(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    std r26, 496(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v21, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v21, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 240
+; CHECK-NEXT:    std r26, 496(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r27, 504(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r28, 512(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r29, 520(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    std r30, 528(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    mr r30, r3
-; CHECK-NEXT:    std r31, 536(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v22, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v22, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 256
+; CHECK-NEXT:    std r31, 536(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f14, 544(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f15, 552(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f16, 560(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f17, 568(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f18, 576(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stfd f19, 584(r1) # 8-byte Folded Spill
-; CHECK-NEXT:    stvx v23, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v23, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 272
+; CHECK-NEXT:    stfd f19, 584(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    stfd f20, 592(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f20, f2
 ; CHECK-NEXT:    stfd f21, 600(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f21, f3
 ; CHECK-NEXT:    stfd f22, 608(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f22, f4
-; CHECK-NEXT:    stvx v24, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v24, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 288
 ; CHECK-NEXT:    stfd f23, 616(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f23, f5
@@ -2547,7 +2288,7 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    fmr f24, f6
 ; CHECK-NEXT:    stfd f25, 632(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f25, f7
-; CHECK-NEXT:    stvx v25, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v25, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 304
 ; CHECK-NEXT:    stfd f26, 640(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f26, f8
@@ -2555,7 +2296,7 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    fmr f27, f9
 ; CHECK-NEXT:    stfd f28, 656(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f28, f10
-; CHECK-NEXT:    stvx v26, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v26, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 320
 ; CHECK-NEXT:    stfd f29, 664(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f29, f11
@@ -2563,15 +2304,15 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    fmr f30, f12
 ; CHECK-NEXT:    stfd f31, 680(r1) # 8-byte Folded Spill
 ; CHECK-NEXT:    fmr f31, f13
-; CHECK-NEXT:    stvx v27, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v27, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 336
-; CHECK-NEXT:    stvx v28, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v28, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 352
-; CHECK-NEXT:    stvx v29, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 368
-; CHECK-NEXT:    stvx v30, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r4, 384
-; CHECK-NEXT:    stvx v31, r1, r4 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r4 # 16-byte Folded Spill
 ; CHECK-NEXT:    bl __truncsfhf2
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    fmr f1, f20
@@ -3050,7 +2791,7 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    li r3, 384
 ; CHECK-NEXT:    xxswapd vs4, vs4
 ; CHECK-NEXT:    stxvd2x vs4, 0, r30
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 368
 ; CHECK-NEXT:    lfd f31, 680(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f30, 672(r1) # 8-byte Folded Reload
@@ -3068,7 +2809,7 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    lfd f18, 576(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f17, 568(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f16, 560(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 352
 ; CHECK-NEXT:    lfd f15, 552(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    lfd f14, 544(r1) # 8-byte Folded Reload
@@ -3076,7 +2817,7 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    ld r30, 528(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r29, 520(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r28, 512(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 336
 ; CHECK-NEXT:    ld r27, 504(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r26, 496(r1) # 8-byte Folded Reload
@@ -3084,7 +2825,7 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    ld r24, 480(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r23, 472(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r22, 464(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 320
 ; CHECK-NEXT:    ld r21, 456(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r20, 448(r1) # 8-byte Folded Reload
@@ -3092,23 +2833,23 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; CHECK-NEXT:    ld r18, 432(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r17, 424(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r16, 416(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 304
 ; CHECK-NEXT:    ld r15, 408(r1) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld r14, 400(r1) # 8-byte Folded Reload
-; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 288
-; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 272
-; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 256
-; CHECK-NEXT:    lvx v23, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v23, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 240
-; CHECK-NEXT:    lvx v22, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v22, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 224
-; CHECK-NEXT:    lvx v21, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v21, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 208
-; CHECK-NEXT:    lvx v20, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v20, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 688
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -3118,95 +2859,62 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; FAST:       # %bb.0:
 ; FAST-NEXT:    mflr r0
 ; FAST-NEXT:    stdu r1, -480(r1)
-; FAST-NEXT:    std r0, 496(r1)
-; FAST-NEXT:    .cfi_def_cfa_offset 480
-; FAST-NEXT:    .cfi_offset lr, 16
-; FAST-NEXT:    .cfi_offset r30, -160
-; FAST-NEXT:    .cfi_offset f14, -144
-; FAST-NEXT:    .cfi_offset f15, -136
-; FAST-NEXT:    .cfi_offset f16, -128
-; FAST-NEXT:    .cfi_offset f17, -120
-; FAST-NEXT:    .cfi_offset f18, -112
-; FAST-NEXT:    .cfi_offset f19, -104
-; FAST-NEXT:    .cfi_offset f20, -96
-; FAST-NEXT:    .cfi_offset f21, -88
-; FAST-NEXT:    .cfi_offset f22, -80
-; FAST-NEXT:    .cfi_offset f23, -72
-; FAST-NEXT:    .cfi_offset f24, -64
-; FAST-NEXT:    .cfi_offset f25, -56
-; FAST-NEXT:    .cfi_offset f26, -48
-; FAST-NEXT:    .cfi_offset f27, -40
-; FAST-NEXT:    .cfi_offset f28, -32
-; FAST-NEXT:    .cfi_offset f29, -24
-; FAST-NEXT:    .cfi_offset f30, -16
-; FAST-NEXT:    .cfi_offset f31, -8
-; FAST-NEXT:    .cfi_offset v20, -352
-; FAST-NEXT:    .cfi_offset v21, -336
-; FAST-NEXT:    .cfi_offset v22, -320
-; FAST-NEXT:    .cfi_offset v23, -304
-; FAST-NEXT:    .cfi_offset v24, -288
-; FAST-NEXT:    .cfi_offset v25, -272
-; FAST-NEXT:    .cfi_offset v26, -256
-; FAST-NEXT:    .cfi_offset v27, -240
-; FAST-NEXT:    .cfi_offset v28, -224
-; FAST-NEXT:    .cfi_offset v29, -208
-; FAST-NEXT:    .cfi_offset v30, -192
-; FAST-NEXT:    .cfi_offset v31, -176
 ; FAST-NEXT:    li r4, 128
+; FAST-NEXT:    std r0, 496(r1)
 ; FAST-NEXT:    std r30, 320(r1) # 8-byte Folded Spill
+; FAST-NEXT:    mr r30, r3
 ; FAST-NEXT:    stfd f14, 336(r1) # 8-byte Folded Spill
-; FAST-NEXT:    fmr f14, f5
 ; FAST-NEXT:    stfd f15, 344(r1) # 8-byte Folded Spill
+; FAST-NEXT:    fmr f14, f5
 ; FAST-NEXT:    stfd f16, 352(r1) # 8-byte Folded Spill
-; FAST-NEXT:    fmr f16, f4
-; FAST-NEXT:    mr r30, r3
-; FAST-NEXT:    stvx v20, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v20, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 144
+; FAST-NEXT:    fmr f16, f4
 ; FAST-NEXT:    stfd f17, 360(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f18, 368(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f19, 376(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f20, 384(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f21, 392(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stfd f22, 400(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stvx v21, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v21, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 160
+; FAST-NEXT:    stfd f22, 400(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f23, 408(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f24, 416(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f25, 424(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f26, 432(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f27, 440(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stfd f28, 448(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stvx v22, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v22, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 176
 ; FAST-NEXT:    xxlor v22, f3, f3
+; FAST-NEXT:    stfd f28, 448(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f29, 456(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    fmr f29, f9
 ; FAST-NEXT:    stfd f30, 464(r1) # 8-byte Folded Spill
 ; FAST-NEXT:    stfd f31, 472(r1) # 8-byte Folded Spill
-; FAST-NEXT:    stvx v23, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v23, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 192
 ; FAST-NEXT:    xxlor v23, f2, f2
-; FAST-NEXT:    stvx v24, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v24, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 208
-; FAST-NEXT:    stvx v25, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v25, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 224
 ; FAST-NEXT:    xxlor v25, f13, f13
-; FAST-NEXT:    stvx v26, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v26, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 240
 ; FAST-NEXT:    xxlor v26, f12, f12
-; FAST-NEXT:    stvx v27, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v27, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 256
 ; FAST-NEXT:    xxlor v27, f11, f11
-; FAST-NEXT:    stvx v28, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v28, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 272
 ; FAST-NEXT:    xxlor v28, f10, f10
-; FAST-NEXT:    stvx v29, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v29, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 288
 ; FAST-NEXT:    xxlor v29, f8, f8
-; FAST-NEXT:    stvx v30, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v30, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 304
 ; FAST-NEXT:    xxlor v30, f7, f7
-; FAST-NEXT:    stvx v31, r1, r4 # 16-byte Folded Spill
+; FAST-NEXT:    stxvd2x v31, r1, r4 # 16-byte Folded Spill
 ; FAST-NEXT:    li r4, 44
 ; FAST-NEXT:    xxlor v31, f6, f6
 ; FAST-NEXT:    stxsspx f1, r1, r4 # 4-byte Folded Spill
@@ -3635,30 +3343,30 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 ; FAST-NEXT:    lfd f16, 352(r1) # 8-byte Folded Reload
 ; FAST-NEXT:    lfd f15, 344(r1) # 8-byte Folded Reload
 ; FAST-NEXT:    lfd f14, 336(r1) # 8-byte Folded Reload
-; FAST-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 288
 ; FAST-NEXT:    ld r30, 320(r1) # 8-byte Folded Reload
-; FAST-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 272
-; FAST-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 256
-; FAST-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 240
-; FAST-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 224
-; FAST-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 208
-; FAST-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 192
-; FAST-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 176
-; FAST-NEXT:    lvx v23, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v23, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 160
-; FAST-NEXT:    lvx v22, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v22, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 144
-; FAST-NEXT:    lvx v21, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v21, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    li r3, 128
-; FAST-NEXT:    lvx v20, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    lxvd2x v20, r1, r3 # 16-byte Folded Reload
 ; FAST-NEXT:    addi r1, r1, 480
 ; FAST-NEXT:    ld r0, 16(r1)
 ; FAST-NEXT:    mtlr r0
@@ -3668,14 +3376,12 @@ define <32 x i64> @lrint_v32i64_v32f16(<32 x half> %x) {
 }
 declare <32 x i64> @llvm.lrint.v32i64.v32f16(<32 x half>)
 
-define <1 x i64> @lrint_v1f32(<1 x float> %x) {
+define <1 x i64> @lrint_v1f32(<1 x float> %x) nounwind {
 ; BE-LABEL: lrint_v1f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -112(r1)
 ; BE-NEXT:    std r0, 128(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 112
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    bl lrintf
 ; BE-NEXT:    nop
 ; BE-NEXT:    addi r1, r1, 112
@@ -3688,8 +3394,6 @@ define <1 x i64> @lrint_v1f32(<1 x float> %x) {
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -32(r1)
 ; CHECK-NEXT:    std r0, 48(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    bl lrintf
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
@@ -3707,15 +3411,13 @@ define <1 x i64> @lrint_v1f32(<1 x float> %x) {
 }
 declare <1 x i64> @llvm.lrint.v1i64.v1f32(<1 x float>)
 
-define <2 x i64> @lrint_v2f32(<2 x float> %x) {
+define <2 x i64> @lrint_v2f32(<2 x float> %x) nounwind {
 ; BE-LABEL: lrint_v2f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -144(r1)
-; BE-NEXT:    std r0, 160(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 144
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    std r0, 160(r1)
 ; BE-NEXT:    stxvw4x v2, 0, r3
 ; BE-NEXT:    lfs f1, 116(r1)
 ; BE-NEXT:    bl lrintf
@@ -3736,14 +3438,11 @@ define <2 x i64> @lrint_v2f32(<2 x float> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -64(r1)
-; CHECK-NEXT:    std r0, 80(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 80(r1)
 ; CHECK-NEXT:    xscvspdpn f1, vs0
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v2
 ; CHECK-NEXT:    bl lrintf
 ; CHECK-NEXT:    nop
@@ -3755,7 +3454,7 @@ define <2 x i64> @lrint_v2f32(<2 x float> %x) {
 ; CHECK-NEXT:    mtfprd f0, r3
 ; CHECK-NEXT:    li r3, 48
 ; CHECK-NEXT:    xxmrghd v2, vs0, v31
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 64
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -3780,15 +3479,13 @@ define <2 x i64> @lrint_v2f32(<2 x float> %x) {
 }
 declare <2 x i64> @llvm.lrint.v2i64.v2f32(<2 x float>)
 
-define <4 x i64> @lrint_v4f32(<4 x float> %x) {
+define <4 x i64> @lrint_v4f32(<4 x float> %x) nounwind {
 ; BE-LABEL: lrint_v4f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -160(r1)
-; BE-NEXT:    std r0, 176(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 160
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    std r0, 176(r1)
 ; BE-NEXT:    stxvw4x v2, 0, r3
 ; BE-NEXT:    lfs f1, 116(r1)
 ; BE-NEXT:    bl lrintf
@@ -3819,17 +3516,13 @@ define <4 x i64> @lrint_v4f32(<4 x float> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -80(r1)
-; CHECK-NEXT:    std r0, 96(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 96(r1)
 ; CHECK-NEXT:    xscvspdpn f1, vs0
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v2
 ; CHECK-NEXT:    bl lrintf
 ; CHECK-NEXT:    nop
@@ -3852,9 +3545,9 @@ define <4 x i64> @lrint_v4f32(<4 x float> %x) {
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    vmr v2, v30
 ; CHECK-NEXT:    xxmrghd v3, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 80
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -3890,15 +3583,13 @@ define <4 x i64> @lrint_v4f32(<4 x float> %x) {
 }
 declare <4 x i64> @llvm.lrint.v4i64.v4f32(<4 x float>)
 
-define <8 x i64> @lrint_v8f32(<8 x float> %x) {
+define <8 x i64> @lrint_v8f32(<8 x float> %x) nounwind {
 ; BE-LABEL: lrint_v8f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -208(r1)
-; BE-NEXT:    std r0, 224(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 208
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    std r0, 224(r1)
 ; BE-NEXT:    stxvw4x v2, 0, r3
 ; BE-NEXT:    addi r3, r1, 128
 ; BE-NEXT:    stxvw4x v3, 0, r3
@@ -3951,24 +3642,18 @@ define <8 x i64> @lrint_v8f32(<8 x float> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -112(r1)
-; CHECK-NEXT:    std r0, 128(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 112
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v28, -64
-; CHECK-NEXT:    .cfi_offset v29, -48
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
 ; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    xscvspdpn f1, vs0
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    vmr v30, v2
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v3
 ; CHECK-NEXT:    bl lrintf
 ; CHECK-NEXT:    nop
@@ -4014,13 +3699,13 @@ define <8 x i64> @lrint_v8f32(<8 x float> %x) {
 ; CHECK-NEXT:    vmr v2, v29
 ; CHECK-NEXT:    vmr v4, v28
 ; CHECK-NEXT:    xxmrghd v5, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 112
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4078,15 +3763,13 @@ define <8 x i64> @lrint_v8f32(<8 x float> %x) {
 }
 declare <8 x i64> @llvm.lrint.v8i64.v8f32(<8 x float>)
 
-define <16 x i64> @lrint_v16i64_v16f32(<16 x float> %x) {
+define <16 x i64> @lrint_v16i64_v16f32(<16 x float> %x) nounwind {
 ; BE-LABEL: lrint_v16i64_v16f32:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -304(r1)
-; BE-NEXT:    std r0, 320(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 304
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    addi r3, r1, 112
+; BE-NEXT:    std r0, 320(r1)
 ; BE-NEXT:    stxvw4x v2, 0, r3
 ; BE-NEXT:    addi r3, r1, 128
 ; BE-NEXT:    stxvw4x v3, 0, r3
@@ -4183,38 +3866,28 @@ define <16 x i64> @lrint_v16i64_v16f32(<16 x float> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -176(r1)
-; CHECK-NEXT:    std r0, 192(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 176
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v24, -128
-; CHECK-NEXT:    .cfi_offset v25, -112
-; CHECK-NEXT:    .cfi_offset v26, -96
-; CHECK-NEXT:    .cfi_offset v27, -80
-; CHECK-NEXT:    .cfi_offset v28, -64
-; CHECK-NEXT:    .cfi_offset v29, -48
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 192(r1)
 ; CHECK-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-NEXT:    stvx v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    xscvspdpn f1, vs0
-; CHECK-NEXT:    stvx v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    stvx v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    vmr v26, v3
-; CHECK-NEXT:    stvx v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 112
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 128
 ; CHECK-NEXT:    vmr v28, v4
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 144
 ; CHECK-NEXT:    vmr v29, v2
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 160
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v5
 ; CHECK-NEXT:    bl lrintf
 ; CHECK-NEXT:    nop
@@ -4306,21 +3979,21 @@ define <16 x i64> @lrint_v16i64_v16f32(<16 x float> %x) {
 ; CHECK-NEXT:    vmr v6, v25
 ; CHECK-NEXT:    vmr v8, v24
 ; CHECK-NEXT:    xxmrghd v9, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 144
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 128
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 112
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 96
-; CHECK-NEXT:    lvx v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    lvx v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
-; CHECK-NEXT:    lvx v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 176
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4422,14 +4095,12 @@ define <16 x i64> @lrint_v16i64_v16f32(<16 x float> %x) {
 }
 declare <16 x i64> @llvm.lrint.v16i64.v16f32(<16 x float>)
 
-define <1 x i64> @lrint_v1f64(<1 x double> %x) {
+define <1 x i64> @lrint_v1f64(<1 x double> %x) nounwind {
 ; BE-LABEL: lrint_v1f64:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -112(r1)
 ; BE-NEXT:    std r0, 128(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 112
-; BE-NEXT:    .cfi_offset lr, 16
 ; BE-NEXT:    bl lrint
 ; BE-NEXT:    nop
 ; BE-NEXT:    addi r1, r1, 112
@@ -4442,8 +4113,6 @@ define <1 x i64> @lrint_v1f64(<1 x double> %x) {
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -32(r1)
 ; CHECK-NEXT:    std r0, 48(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 32
-; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    bl lrint
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
@@ -4461,16 +4130,13 @@ define <1 x i64> @lrint_v1f64(<1 x double> %x) {
 }
 declare <1 x i64> @llvm.lrint.v1i64.v1f64(<1 x double>)
 
-define <2 x i64> @lrint_v2f64(<2 x double> %x) {
+define <2 x i64> @lrint_v2f64(<2 x double> %x) nounwind {
 ; BE-LABEL: lrint_v2f64:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -160(r1)
-; BE-NEXT:    std r0, 176(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 160
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset v31, -16
 ; BE-NEXT:    li r3, 144
+; BE-NEXT:    std r0, 176(r1)
 ; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    vmr v31, v2
 ; BE-NEXT:    xxlor f1, v31, v31
@@ -4494,12 +4160,9 @@ define <2 x i64> @lrint_v2f64(<2 x double> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -64(r1)
-; CHECK-NEXT:    std r0, 80(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 64
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    std r0, 80(r1)
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v2
 ; CHECK-NEXT:    xxlor f1, v31, v31
 ; CHECK-NEXT:    bl lrint
@@ -4511,7 +4174,7 @@ define <2 x i64> @lrint_v2f64(<2 x double> %x) {
 ; CHECK-NEXT:    mtfprd f0, r3
 ; CHECK-NEXT:    li r3, 48
 ; CHECK-NEXT:    xxmrghd v2, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 64
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4534,17 +4197,13 @@ define <2 x i64> @lrint_v2f64(<2 x double> %x) {
 }
 declare <2 x i64> @llvm.lrint.v2i64.v2f64(<2 x double>)
 
-define <4 x i64> @lrint_v4f64(<4 x double> %x) {
+define <4 x i64> @lrint_v4f64(<4 x double> %x) nounwind {
 ; BE-LABEL: lrint_v4f64:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -192(r1)
-; BE-NEXT:    std r0, 208(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 192
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset v30, -32
-; BE-NEXT:    .cfi_offset v31, -16
 ; BE-NEXT:    li r3, 160
+; BE-NEXT:    std r0, 208(r1)
 ; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    vmr v30, v2
 ; BE-NEXT:    li r3, 176
@@ -4583,17 +4242,13 @@ define <4 x i64> @lrint_v4f64(<4 x double> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -80(r1)
-; CHECK-NEXT:    std r0, 96(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 80
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    std r0, 96(r1)
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v30, v2
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    xxlor f1, v30, v30
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v3
 ; CHECK-NEXT:    bl lrint
 ; CHECK-NEXT:    nop
@@ -4614,9 +4269,9 @@ define <4 x i64> @lrint_v4f64(<4 x double> %x) {
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    vmr v2, v30
 ; CHECK-NEXT:    xxmrghd v3, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 80
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4648,25 +4303,19 @@ define <4 x i64> @lrint_v4f64(<4 x double> %x) {
 }
 declare <4 x i64> @llvm.lrint.v4i64.v4f64(<4 x double>)
 
-define <8 x i64> @lrint_v8f64(<8 x double> %x) {
+define <8 x i64> @lrint_v8f64(<8 x double> %x) nounwind {
 ; BE-LABEL: lrint_v8f64:
 ; BE:       # %bb.0:
 ; BE-NEXT:    mflr r0
 ; BE-NEXT:    stdu r1, -256(r1)
-; BE-NEXT:    std r0, 272(r1)
-; BE-NEXT:    .cfi_def_cfa_offset 256
-; BE-NEXT:    .cfi_offset lr, 16
-; BE-NEXT:    .cfi_offset v28, -64
-; BE-NEXT:    .cfi_offset v29, -48
-; BE-NEXT:    .cfi_offset v30, -32
-; BE-NEXT:    .cfi_offset v31, -16
 ; BE-NEXT:    li r3, 192
+; BE-NEXT:    std r0, 272(r1)
 ; BE-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    li r3, 208
 ; BE-NEXT:    vmr v28, v2
-; BE-NEXT:    xxlor f1, v28, v28
 ; BE-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    li r3, 224
+; BE-NEXT:    xxlor f1, v28, v28
 ; BE-NEXT:    vmr v29, v3
 ; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; BE-NEXT:    li r3, 240
@@ -4729,25 +4378,19 @@ define <8 x i64> @lrint_v8f64(<8 x double> %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    mflr r0
 ; CHECK-NEXT:    stdu r1, -112(r1)
-; CHECK-NEXT:    std r0, 128(r1)
-; CHECK-NEXT:    .cfi_def_cfa_offset 112
-; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    .cfi_offset v28, -64
-; CHECK-NEXT:    .cfi_offset v29, -48
-; CHECK-NEXT:    .cfi_offset v30, -32
-; CHECK-NEXT:    .cfi_offset v31, -16
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    stvx v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 64
 ; CHECK-NEXT:    vmr v28, v2
-; CHECK-NEXT:    stvx v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 80
 ; CHECK-NEXT:    xxlor f1, v28, v28
 ; CHECK-NEXT:    vmr v29, v3
-; CHECK-NEXT:    stvx v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    li r3, 96
 ; CHECK-NEXT:    vmr v30, v4
-; CHECK-NEXT:    stvx v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
 ; CHECK-NEXT:    vmr v31, v5
 ; CHECK-NEXT:    bl lrint
 ; CHECK-NEXT:    nop
@@ -4788,13 +4431,13 @@ define <8 x i64> @lrint_v8f64(<8 x double> %x) {
 ; CHECK-NEXT:    vmr v3, v29
 ; CHECK-NEXT:    vmr v2, v28
 ; CHECK-NEXT:    xxmrghd v5, v31, vs0
-; CHECK-NEXT:    lvx v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 80
-; CHECK-NEXT:    lvx v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 64
-; CHECK-NEXT:    lvx v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    li r3, 48
-; CHECK-NEXT:    lvx v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
 ; CHECK-NEXT:    addi r1, r1, 112
 ; CHECK-NEXT:    ld r0, 16(r1)
 ; CHECK-NEXT:    mtlr r0
@@ -4843,3 +4486,1046 @@ define <8 x i64> @lrint_v8f64(<8 x double> %x) {
   ret <8 x i64> %a
 }
 declare <8 x i64> @llvm.lrint.v8i64.v8f64(<8 x double>)
+
+define <1 x i64> @lrint_v1f128(<1 x fp128> %x) nounwind {
+; BE-LABEL: lrint_v1f128:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -112(r1)
+; BE-NEXT:    std r0, 128(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    addi r1, r1, 112
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v1f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v1f128:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -32(r1)
+; FAST-NEXT:    std r0, 48(r1)
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    addi r1, r1, 32
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <1 x i64> @llvm.lrint.v1i64.v1f128(<1 x fp128> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.lrint.v1i64.v1f128(<1 x fp128>)
+
+define <2 x i64> @lrint_v2f128(<2 x fp128> %x) nounwind {
+; BE-LABEL: lrint_v2f128:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -160(r1)
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    std r0, 176(r1)
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v2
+; BE-NEXT:    vmr v2, v3
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v31
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 160
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v2f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -80(r1)
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 96(r1)
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxmrghd v2, vs0, v30
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 80
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v2f128:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -80(r1)
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    std r0, 96(r1)
+; FAST-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    vmr v31, v3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v31
+; FAST-NEXT:    mtvsrd v30, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    xxmrghd v2, vs0, v30
+; FAST-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    addi r1, r1, 80
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <2 x i64> @llvm.lrint.v2i64.v2f128(<2 x fp128> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.lrint.v2i64.v2f128(<2 x fp128>)
+
+define <4 x i64> @lrint_v4f128(<4 x fp128> %x) nounwind {
+; BE-LABEL: lrint_v4f128:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -208(r1)
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    std r0, 224(r1)
+; BE-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    vmr v29, v2
+; BE-NEXT:    vmr v2, v3
+; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    vmr v30, v4
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v5
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v29
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v31
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v30
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 176
+; BE-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 208
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v4f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -112(r1)
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 128(r1)
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    vmr v29, v3
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v30, v4
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v5
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v29
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    xxmrghd v29, vs0, v28
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v2, v29
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    xxmrghd v3, vs0, v30
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 112
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v4f128:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -112(r1)
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    std r0, 128(r1)
+; FAST-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    vmr v29, v3
+; FAST-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    vmr v30, v4
+; FAST-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    vmr v31, v5
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v29
+; FAST-NEXT:    mtvsrd v28, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v30
+; FAST-NEXT:    xxmrghd v29, vs0, v28
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v31
+; FAST-NEXT:    mtvsrd v30, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    vmr v2, v29
+; FAST-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    xxmrghd v3, vs0, v30
+; FAST-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    addi r1, r1, 112
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <4 x i64> @llvm.lrint.v4i64.v4f128(<4 x fp128> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.lrint.v4i64.v4f128(<4 x fp128>)
+
+define <8 x i64> @lrint_v8f128(<8 x fp128> %x) nounwind {
+; BE-LABEL: lrint_v8f128:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -304(r1)
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    std r0, 320(r1)
+; BE-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    vmr v25, v2
+; BE-NEXT:    vmr v2, v3
+; BE-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    vmr v26, v4
+; BE-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    vmr v27, v5
+; BE-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 256
+; BE-NEXT:    vmr v28, v6
+; BE-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 272
+; BE-NEXT:    vmr v29, v7
+; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 288
+; BE-NEXT:    vmr v30, v8
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    vmr v31, v9
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v25
+; BE-NEXT:    std r3, 136(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v27
+; BE-NEXT:    std r3, 128(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v26
+; BE-NEXT:    std r3, 152(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v29
+; BE-NEXT:    std r3, 144(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v28
+; BE-NEXT:    std r3, 168(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v31
+; BE-NEXT:    std r3, 160(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v30
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    addi r3, r1, 128
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 144
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 160
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    li r3, 288
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 272
+; BE-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 256
+; BE-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 240
+; BE-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 224
+; BE-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 208
+; BE-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 192
+; BE-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 304
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v8f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -176(r1)
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    std r0, 192(r1)
+; CHECK-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    vmr v25, v3
+; CHECK-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    vmr v26, v4
+; CHECK-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    vmr v27, v5
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    vmr v28, v6
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    vmr v29, v7
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    vmr v30, v8
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    vmr v31, v9
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v25
+; CHECK-NEXT:    mtvsrd v24, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v26
+; CHECK-NEXT:    xxmrghd v25, vs0, v24
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v27
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v28
+; CHECK-NEXT:    xxmrghd v27, vs0, v26
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v29
+; CHECK-NEXT:    mtvsrd v28, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    xxmrghd v29, vs0, v28
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    mtvsrd v30, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    vmr v4, v29
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    vmr v3, v27
+; CHECK-NEXT:    vmr v2, v25
+; CHECK-NEXT:    xxmrghd v5, vs0, v30
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 176
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v8f128:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -176(r1)
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    std r0, 192(r1)
+; FAST-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    vmr v25, v3
+; FAST-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    vmr v26, v4
+; FAST-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    vmr v27, v5
+; FAST-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    vmr v28, v6
+; FAST-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    vmr v29, v7
+; FAST-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    vmr v30, v8
+; FAST-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    vmr v31, v9
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v25
+; FAST-NEXT:    mtvsrd v24, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v26
+; FAST-NEXT:    xxmrghd v25, vs0, v24
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v27
+; FAST-NEXT:    mtvsrd v26, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v28
+; FAST-NEXT:    xxmrghd v27, vs0, v26
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v29
+; FAST-NEXT:    mtvsrd v28, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v30
+; FAST-NEXT:    xxmrghd v29, vs0, v28
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v31
+; FAST-NEXT:    mtvsrd v30, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    vmr v4, v29
+; FAST-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    vmr v3, v27
+; FAST-NEXT:    vmr v2, v25
+; FAST-NEXT:    xxmrghd v5, vs0, v30
+; FAST-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    addi r1, r1, 176
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <8 x i64> @llvm.lrint.v8i64.v8f128(<8 x fp128> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.lrint.v8i64.v8f128(<8 x fp128>)
+
+define <16 x i64> @lrint_v16i64_v16f128(<16 x fp128> %x) nounwind {
+; BE-LABEL: lrint_v16i64_v16f128:
+; BE:       # %bb.0:
+; BE-NEXT:    mflr r0
+; BE-NEXT:    stdu r1, -496(r1)
+; BE-NEXT:    li r3, 304
+; BE-NEXT:    std r0, 512(r1)
+; BE-NEXT:    stxvd2x v20, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 320
+; BE-NEXT:    stxvd2x v21, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 336
+; BE-NEXT:    vmr v21, v2
+; BE-NEXT:    vmr v2, v3
+; BE-NEXT:    stxvd2x v22, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 352
+; BE-NEXT:    vmr v22, v4
+; BE-NEXT:    stxvd2x v23, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 368
+; BE-NEXT:    vmr v23, v5
+; BE-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 384
+; BE-NEXT:    vmr v24, v6
+; BE-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 400
+; BE-NEXT:    vmr v25, v7
+; BE-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 416
+; BE-NEXT:    vmr v26, v8
+; BE-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 432
+; BE-NEXT:    vmr v27, v9
+; BE-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 448
+; BE-NEXT:    vmr v28, v11
+; BE-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 464
+; BE-NEXT:    vmr v29, v10
+; BE-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 480
+; BE-NEXT:    vmr v30, v13
+; BE-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    li r3, 128
+; BE-NEXT:    stxvd2x v12, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    addi r3, r1, 768
+; BE-NEXT:    lxvw4x vs0, 0, r3
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    addi r3, r1, 784
+; BE-NEXT:    lxvw4x vs0, 0, r3
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; BE-NEXT:    addi r3, r1, 736
+; BE-NEXT:    lxvw4x v20, 0, r3
+; BE-NEXT:    addi r3, r1, 752
+; BE-NEXT:    lxvw4x v31, 0, r3
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v21
+; BE-NEXT:    std r3, 184(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v23
+; BE-NEXT:    std r3, 176(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v22
+; BE-NEXT:    std r3, 200(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v25
+; BE-NEXT:    std r3, 192(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v24
+; BE-NEXT:    std r3, 216(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v27
+; BE-NEXT:    std r3, 208(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v26
+; BE-NEXT:    std r3, 232(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v28
+; BE-NEXT:    std r3, 224(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v29
+; BE-NEXT:    std r3, 248(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v30
+; BE-NEXT:    std r3, 240(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 264(r1)
+; BE-NEXT:    li r3, 128
+; BE-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v31
+; BE-NEXT:    std r3, 256(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    vmr v2, v20
+; BE-NEXT:    std r3, 280(r1)
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 272(r1)
+; BE-NEXT:    li r3, 144
+; BE-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 296(r1)
+; BE-NEXT:    li r3, 160
+; BE-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    bl lrintf128
+; BE-NEXT:    nop
+; BE-NEXT:    std r3, 288(r1)
+; BE-NEXT:    addi r3, r1, 176
+; BE-NEXT:    lxvd2x v2, 0, r3
+; BE-NEXT:    addi r3, r1, 192
+; BE-NEXT:    lxvd2x v3, 0, r3
+; BE-NEXT:    addi r3, r1, 208
+; BE-NEXT:    lxvd2x v4, 0, r3
+; BE-NEXT:    addi r3, r1, 224
+; BE-NEXT:    lxvd2x v5, 0, r3
+; BE-NEXT:    addi r3, r1, 240
+; BE-NEXT:    lxvd2x v6, 0, r3
+; BE-NEXT:    addi r3, r1, 256
+; BE-NEXT:    lxvd2x v7, 0, r3
+; BE-NEXT:    addi r3, r1, 272
+; BE-NEXT:    lxvd2x v8, 0, r3
+; BE-NEXT:    addi r3, r1, 288
+; BE-NEXT:    lxvd2x v9, 0, r3
+; BE-NEXT:    li r3, 480
+; BE-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 464
+; BE-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 448
+; BE-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 432
+; BE-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 416
+; BE-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 400
+; BE-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 384
+; BE-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 368
+; BE-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 352
+; BE-NEXT:    lxvd2x v23, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 336
+; BE-NEXT:    lxvd2x v22, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 320
+; BE-NEXT:    lxvd2x v21, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    li r3, 304
+; BE-NEXT:    lxvd2x v20, r1, r3 # 16-byte Folded Reload
+; BE-NEXT:    addi r1, r1, 496
+; BE-NEXT:    ld r0, 16(r1)
+; BE-NEXT:    mtlr r0
+; BE-NEXT:    blr
+;
+; CHECK-LABEL: lrint_v16i64_v16f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -304(r1)
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    std r0, 320(r1)
+; CHECK-NEXT:    stxvd2x v20, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    stxvd2x v21, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    vmr v21, v4
+; CHECK-NEXT:    stxvd2x v22, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    vmr v22, v6
+; CHECK-NEXT:    stxvd2x v23, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 176
+; CHECK-NEXT:    vmr v23, v8
+; CHECK-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 192
+; CHECK-NEXT:    vmr v24, v9
+; CHECK-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 208
+; CHECK-NEXT:    vmr v25, v7
+; CHECK-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 224
+; CHECK-NEXT:    vmr v26, v10
+; CHECK-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 240
+; CHECK-NEXT:    vmr v27, v5
+; CHECK-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 256
+; CHECK-NEXT:    vmr v28, v11
+; CHECK-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 272
+; CHECK-NEXT:    vmr v29, v12
+; CHECK-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 288
+; CHECK-NEXT:    vmr v30, v3
+; CHECK-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    stxvd2x v13, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    addi r3, r1, 576
+; CHECK-NEXT:    lxvd2x vs0, 0, r3
+; CHECK-NEXT:    addi r3, r1, 560
+; CHECK-NEXT:    lxvd2x vs1, 0, r3
+; CHECK-NEXT:    addi r3, r1, 544
+; CHECK-NEXT:    lxvd2x vs2, 0, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    xxswapd vs0, vs0
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    xxswapd vs0, vs1
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    xxswapd vs0, vs2
+; CHECK-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; CHECK-NEXT:    addi r3, r1, 528
+; CHECK-NEXT:    lxvd2x vs0, 0, r3
+; CHECK-NEXT:    xxswapd v31, vs0
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    mtvsrd v20, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v21
+; CHECK-NEXT:    xxmrghd v30, vs0, v20
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v27
+; CHECK-NEXT:    mtvsrd v21, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v22
+; CHECK-NEXT:    xxmrghd v27, vs0, v21
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v25
+; CHECK-NEXT:    mtvsrd v22, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v23
+; CHECK-NEXT:    xxmrghd v25, vs0, v22
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v24
+; CHECK-NEXT:    mtvsrd v23, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v26
+; CHECK-NEXT:    xxmrghd v24, vs0, v23
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    vmr v2, v28
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v29
+; CHECK-NEXT:    xxmrghd v28, vs0, v26
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtvsrd v29, r3
+; CHECK-NEXT:    li r3, 64
+; CHECK-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    vmr v2, v31
+; CHECK-NEXT:    xxmrghd v29, vs0, v29
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtvsrd v31, r3
+; CHECK-NEXT:    li r3, 48
+; CHECK-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 80
+; CHECK-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    xxmrghd v31, vs0, v31
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtvsrd v26, r3
+; CHECK-NEXT:    li r3, 96
+; CHECK-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    bl lrintf128
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    mtfprd f0, r3
+; CHECK-NEXT:    li r3, 288
+; CHECK-NEXT:    vmr v8, v31
+; CHECK-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 272
+; CHECK-NEXT:    vmr v2, v30
+; CHECK-NEXT:    vmr v7, v29
+; CHECK-NEXT:    vmr v6, v28
+; CHECK-NEXT:    vmr v3, v27
+; CHECK-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 256
+; CHECK-NEXT:    vmr v4, v25
+; CHECK-NEXT:    vmr v5, v24
+; CHECK-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 240
+; CHECK-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 224
+; CHECK-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 208
+; CHECK-NEXT:    xxmrghd v9, vs0, v26
+; CHECK-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 192
+; CHECK-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 176
+; CHECK-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 160
+; CHECK-NEXT:    lxvd2x v23, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 144
+; CHECK-NEXT:    lxvd2x v22, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 128
+; CHECK-NEXT:    lxvd2x v21, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    li r3, 112
+; CHECK-NEXT:    lxvd2x v20, r1, r3 # 16-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 304
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; FAST-LABEL: lrint_v16i64_v16f128:
+; FAST:       # %bb.0:
+; FAST-NEXT:    mflr r0
+; FAST-NEXT:    stdu r1, -304(r1)
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    std r0, 320(r1)
+; FAST-NEXT:    stxvd2x v20, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    stxvd2x v21, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    vmr v21, v4
+; FAST-NEXT:    stxvd2x v22, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    vmr v22, v6
+; FAST-NEXT:    stxvd2x v23, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 176
+; FAST-NEXT:    vmr v23, v8
+; FAST-NEXT:    stxvd2x v24, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 192
+; FAST-NEXT:    vmr v24, v9
+; FAST-NEXT:    stxvd2x v25, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 208
+; FAST-NEXT:    vmr v25, v7
+; FAST-NEXT:    stxvd2x v26, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 224
+; FAST-NEXT:    vmr v26, v10
+; FAST-NEXT:    stxvd2x v27, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 240
+; FAST-NEXT:    vmr v27, v5
+; FAST-NEXT:    stxvd2x v28, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 256
+; FAST-NEXT:    vmr v28, v11
+; FAST-NEXT:    stxvd2x v29, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 272
+; FAST-NEXT:    vmr v29, v12
+; FAST-NEXT:    stxvd2x v30, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 288
+; FAST-NEXT:    vmr v30, v3
+; FAST-NEXT:    stxvd2x v31, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    stxvd2x v13, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    addi r3, r1, 576
+; FAST-NEXT:    lxvd2x vs0, 0, r3
+; FAST-NEXT:    addi r3, r1, 560
+; FAST-NEXT:    lxvd2x vs1, 0, r3
+; FAST-NEXT:    addi r3, r1, 544
+; FAST-NEXT:    lxvd2x vs2, 0, r3
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    xxswapd vs0, vs0
+; FAST-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    xxswapd vs0, vs1
+; FAST-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    xxswapd vs0, vs2
+; FAST-NEXT:    stxvd2x vs0, r1, r3 # 16-byte Folded Spill
+; FAST-NEXT:    addi r3, r1, 528
+; FAST-NEXT:    lxvd2x vs0, 0, r3
+; FAST-NEXT:    xxswapd v31, vs0
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v30
+; FAST-NEXT:    mtvsrd v20, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v21
+; FAST-NEXT:    xxmrghd v30, vs0, v20
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v27
+; FAST-NEXT:    mtvsrd v21, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v22
+; FAST-NEXT:    xxmrghd v27, vs0, v21
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v25
+; FAST-NEXT:    mtvsrd v22, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v23
+; FAST-NEXT:    xxmrghd v25, vs0, v22
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v24
+; FAST-NEXT:    mtvsrd v23, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v26
+; FAST-NEXT:    xxmrghd v24, vs0, v23
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    vmr v2, v28
+; FAST-NEXT:    mtvsrd v26, r3
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v29
+; FAST-NEXT:    xxmrghd v28, vs0, v26
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtvsrd v29, r3
+; FAST-NEXT:    li r3, 64
+; FAST-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    vmr v2, v31
+; FAST-NEXT:    xxmrghd v29, vs0, v29
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtvsrd v31, r3
+; FAST-NEXT:    li r3, 48
+; FAST-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    li r3, 80
+; FAST-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    xxmrghd v31, vs0, v31
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtvsrd v26, r3
+; FAST-NEXT:    li r3, 96
+; FAST-NEXT:    lxvd2x v2, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    bl lrintf128
+; FAST-NEXT:    nop
+; FAST-NEXT:    mtfprd f0, r3
+; FAST-NEXT:    li r3, 288
+; FAST-NEXT:    vmr v8, v31
+; FAST-NEXT:    lxvd2x v31, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 272
+; FAST-NEXT:    vmr v2, v30
+; FAST-NEXT:    vmr v7, v29
+; FAST-NEXT:    vmr v6, v28
+; FAST-NEXT:    vmr v3, v27
+; FAST-NEXT:    lxvd2x v30, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 256
+; FAST-NEXT:    vmr v4, v25
+; FAST-NEXT:    vmr v5, v24
+; FAST-NEXT:    lxvd2x v29, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 240
+; FAST-NEXT:    lxvd2x v28, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 224
+; FAST-NEXT:    lxvd2x v27, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 208
+; FAST-NEXT:    xxmrghd v9, vs0, v26
+; FAST-NEXT:    lxvd2x v26, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 192
+; FAST-NEXT:    lxvd2x v25, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 176
+; FAST-NEXT:    lxvd2x v24, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 160
+; FAST-NEXT:    lxvd2x v23, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 144
+; FAST-NEXT:    lxvd2x v22, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 128
+; FAST-NEXT:    lxvd2x v21, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    li r3, 112
+; FAST-NEXT:    lxvd2x v20, r1, r3 # 16-byte Folded Reload
+; FAST-NEXT:    addi r1, r1, 304
+; FAST-NEXT:    ld r0, 16(r1)
+; FAST-NEXT:    mtlr r0
+; FAST-NEXT:    blr
+  %a = call <16 x i64> @llvm.lrint.v16i64.v16f128(<16 x fp128> %x)
+  ret <16 x i64> %a
+}
+declare <16 x i64> @llvm.lrint.v16i64.v16f128(<16 x fp128>)
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/double-arith.ll b/llvm/test/CodeGen/RISCV/GlobalISel/double-arith.ll
index cb2037f5fb027..4eb7646d13a39 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/double-arith.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/double-arith.ll
@@ -395,13 +395,9 @@ define double @fmadd_d(double %a, double %b, double %c) nounwind {
 define double @fmsub_d(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fmsub_d:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw zero, 8(sp)
-; RV32IFD-NEXT:    sw zero, 12(sp)
-; RV32IFD-NEXT:    fld fa5, 8(sp)
+; RV32IFD-NEXT:    fcvt.d.w fa5, zero
 ; RV32IFD-NEXT:    fadd.d fa5, fa2, fa5
 ; RV32IFD-NEXT:    fmsub.d fa0, fa0, fa1, fa5
-; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fmsub_d:
@@ -478,14 +474,10 @@ define double @fmsub_d(double %a, double %b, double %c) nounwind {
 define double @fnmadd_d(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fnmadd_d:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw zero, 8(sp)
-; RV32IFD-NEXT:    sw zero, 12(sp)
-; RV32IFD-NEXT:    fld fa5, 8(sp)
+; RV32IFD-NEXT:    fcvt.d.w fa5, zero
 ; RV32IFD-NEXT:    fadd.d fa4, fa0, fa5
 ; RV32IFD-NEXT:    fadd.d fa5, fa2, fa5
 ; RV32IFD-NEXT:    fnmadd.d fa0, fa4, fa1, fa5
-; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fnmadd_d:
@@ -590,14 +582,10 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind {
 define double @fnmadd_d_2(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fnmadd_d_2:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw zero, 8(sp)
-; RV32IFD-NEXT:    sw zero, 12(sp)
-; RV32IFD-NEXT:    fld fa5, 8(sp)
+; RV32IFD-NEXT:    fcvt.d.w fa5, zero
 ; RV32IFD-NEXT:    fadd.d fa4, fa1, fa5
 ; RV32IFD-NEXT:    fadd.d fa5, fa2, fa5
 ; RV32IFD-NEXT:    fnmadd.d fa0, fa4, fa0, fa5
-; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fnmadd_d_2:
@@ -772,13 +760,9 @@ define double @fnmadd_nsz(double %a, double %b, double %c) nounwind {
 define double @fnmsub_d(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fnmsub_d:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw zero, 8(sp)
-; RV32IFD-NEXT:    sw zero, 12(sp)
-; RV32IFD-NEXT:    fld fa5, 8(sp)
+; RV32IFD-NEXT:    fcvt.d.w fa5, zero
 ; RV32IFD-NEXT:    fadd.d fa5, fa0, fa5
 ; RV32IFD-NEXT:    fnmsub.d fa0, fa5, fa1, fa2
-; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fnmsub_d:
@@ -851,13 +835,9 @@ define double @fnmsub_d(double %a, double %b, double %c) nounwind {
 define double @fnmsub_d_2(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fnmsub_d_2:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw zero, 8(sp)
-; RV32IFD-NEXT:    sw zero, 12(sp)
-; RV32IFD-NEXT:    fld fa5, 8(sp)
+; RV32IFD-NEXT:    fcvt.d.w fa5, zero
 ; RV32IFD-NEXT:    fadd.d fa5, fa1, fa5
 ; RV32IFD-NEXT:    fnmsub.d fa0, fa5, fa0, fa2
-; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fnmsub_d_2:
@@ -976,14 +956,10 @@ define double @fmadd_d_contract(double %a, double %b, double %c) nounwind {
 define double @fmsub_d_contract(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fmsub_d_contract:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw zero, 8(sp)
-; RV32IFD-NEXT:    sw zero, 12(sp)
-; RV32IFD-NEXT:    fld fa5, 8(sp)
+; RV32IFD-NEXT:    fcvt.d.w fa5, zero
 ; RV32IFD-NEXT:    fadd.d fa5, fa2, fa5
 ; RV32IFD-NEXT:    fmul.d fa4, fa0, fa1
 ; RV32IFD-NEXT:    fsub.d fa0, fa4, fa5
-; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fmsub_d_contract:
@@ -1069,17 +1045,13 @@ define double @fmsub_d_contract(double %a, double %b, double %c) nounwind {
 define double @fnmadd_d_contract(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fnmadd_d_contract:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw zero, 8(sp)
-; RV32IFD-NEXT:    sw zero, 12(sp)
-; RV32IFD-NEXT:    fld fa5, 8(sp)
+; RV32IFD-NEXT:    fcvt.d.w fa5, zero
 ; RV32IFD-NEXT:    fadd.d fa4, fa0, fa5
 ; RV32IFD-NEXT:    fadd.d fa3, fa1, fa5
 ; RV32IFD-NEXT:    fadd.d fa5, fa2, fa5
 ; RV32IFD-NEXT:    fmul.d fa4, fa4, fa3
 ; RV32IFD-NEXT:    fneg.d fa4, fa4
 ; RV32IFD-NEXT:    fsub.d fa0, fa4, fa5
-; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fnmadd_d_contract:
@@ -1204,14 +1176,10 @@ define double @fnmadd_d_contract(double %a, double %b, double %c) nounwind {
 define double @fnmsub_d_contract(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fnmsub_d_contract:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw zero, 8(sp)
-; RV32IFD-NEXT:    sw zero, 12(sp)
-; RV32IFD-NEXT:    fld fa5, 8(sp)
+; RV32IFD-NEXT:    fcvt.d.w fa5, zero
 ; RV32IFD-NEXT:    fadd.d fa4, fa0, fa5
 ; RV32IFD-NEXT:    fadd.d fa5, fa1, fa5
 ; RV32IFD-NEXT:    fnmsub.d fa0, fa4, fa5, fa2
-; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
 ; RV64IFD-LABEL: fnmsub_d_contract:
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/double-fcmp.ll b/llvm/test/CodeGen/RISCV/GlobalISel/double-fcmp.ll
index dfa76a2e1531b..9ec8c32e989b0 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/double-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/double-fcmp.ll
@@ -138,7 +138,7 @@ define i32 @fcmp_olt(double %a, double %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltdf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -148,8 +148,7 @@ define i32 @fcmp_olt(double %a, double %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltdf2
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srliw a0, a0, 31
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -446,7 +445,7 @@ define i32 @fcmp_ult(double %a, double %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -456,8 +455,7 @@ define i32 @fcmp_ult(double %a, double %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gedf2
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srliw a0, a0, 31
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/float-fcmp.ll b/llvm/test/CodeGen/RISCV/GlobalISel/float-fcmp.ll
index 475b67bda9ae9..380751c907c0d 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/float-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/float-fcmp.ll
@@ -138,7 +138,7 @@ define i32 @fcmp_olt(float %a, float %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltsf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -148,8 +148,7 @@ define i32 @fcmp_olt(float %a, float %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltsf2
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srliw a0, a0, 31
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -431,7 +430,7 @@ define i32 @fcmp_ult(float %a, float %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -441,8 +440,7 @@ define i32 @fcmp_ult(float %a, float %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srliw a0, a0, 31
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant.mir
index e82d4bcec48b1..4db80c6c1141f 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/fp-constant.mir
@@ -164,10 +164,8 @@ body:             |
     ; RV32-LABEL: name: double_positive_zero
     ; RV32: liveins: $x10
     ; RV32-NEXT: {{  $}}
-    ; RV32-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x0
-    ; RV32-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x0
-    ; RV32-NEXT: [[BuildPairF64Pseudo:%[0-9]+]]:fpr64 = BuildPairF64Pseudo [[COPY1]], [[COPY]]
-    ; RV32-NEXT: $f10_d = COPY [[BuildPairF64Pseudo]]
+    ; RV32-NEXT: [[FCVT_D_W:%[0-9]+]]:fpr64 = FCVT_D_W $x0, 0
+    ; RV32-NEXT: $f10_d = COPY [[FCVT_D_W]]
     ; RV32-NEXT: PseudoRET implicit $f10_d
     ;
     ; RV64-LABEL: name: double_positive_zero
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-uadde-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-uadde-rv32.mir
new file mode 100644
index 0000000000000..9f6abf05f49d7
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-uadde-rv32.mir
@@ -0,0 +1,173 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            uadde_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: uadde_i8
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[AND2]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND3]](s32), [[C2]]
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ICMP1]], [[COPY2]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ICMP]], [[AND4]]
+    ; CHECK-NEXT: $x10 = COPY [[ADD1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[OR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s32) = COPY $x10
+    %1:_(s8) = G_TRUNC %0(s32)
+    %2:_(s32) = COPY $x11
+    %3:_(s8) = G_TRUNC %2(s32)
+    %4:_(s32) = COPY $x12
+    %5:_(s1) = G_TRUNC %4(s32)
+    %6:_(s8), %7:_(s1) = G_UADDE %1, %3, %5
+    %8:_(s32) = G_ANYEXT %6(s8)
+    %9:_(s32) = G_ANYEXT %7(s1)
+    $x10 = COPY %8(s32)
+    $x11 = COPY %9(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            uadde_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: uadde_i16
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[AND]](s32), [[AND1]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[AND2]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[AND3]](s32), [[C2]]
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ICMP1]], [[COPY2]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ICMP]], [[AND4]]
+    ; CHECK-NEXT: $x10 = COPY [[ADD1]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[OR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s32) = COPY $x10
+    %1:_(s16) = G_TRUNC %0(s32)
+    %2:_(s32) = COPY $x11
+    %3:_(s16) = G_TRUNC %2(s32)
+    %4:_(s32) = COPY $x12
+    %5:_(s1) = G_TRUNC %4(s32)
+    %6:_(s16), %7:_(s1) = G_UADDE %1, %3, %5
+    %8:_(s32) = G_ANYEXT %6(s16)
+    %9:_(s32) = G_ANYEXT %7(s1)
+    $x10 = COPY %8(s32)
+    $x11 = COPY %9(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            uadde_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: uadde_i32
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[AND]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ADD1]](s32), [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ICMP1]], [[COPY2]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ICMP]], [[AND1]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[ADD1]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[COPY3]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[OR]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = COPY $x12
+    %3:_(s1) = G_TRUNC %2(s32)
+    %4:_(s32), %5:_(s1) = G_UADDE %0, %1, %3
+    %6:_(s32) = G_ANYEXT %5(s1)
+    $x10 = COPY %4(s32)
+    $x11 = COPY %6(s32)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            uadde_i64
+body:             |
+  bb.1:
+    liveins: $x10, $x11, $x12, $x13, $x14
+
+    ; CHECK-LABEL: name: uadde_i64
+    ; CHECK: liveins: $x10, $x11, $x12, $x13, $x14
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD]](s32), [[COPY]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[AND]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ADD1]](s32), [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ICMP1]], [[COPY4]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[ICMP]], [[AND1]]
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ADD1]](s32)
+    ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[COPY3]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[ADD2]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[ADD2]], [[OR]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ADD3]](s32), [[C1]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ICMP3]], [[OR]]
+    ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ICMP2]], [[AND2]]
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY [[ADD3]](s32)
+    ; CHECK-NEXT: $x10 = COPY [[COPY5]](s32)
+    ; CHECK-NEXT: $x11 = COPY [[COPY6]](s32)
+    ; CHECK-NEXT: $x12 = COPY [[OR1]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12
+    %0:_(s32) = COPY $x10
+    %1:_(s32) = COPY $x11
+    %2:_(s32) = COPY $x12
+    %3:_(s32) = COPY $x13
+    %4:_(s32) = COPY $x14
+    %5:_(s1)  = G_TRUNC %4(s32)
+    %6:_(s32), %7:_(s1) = G_UADDE %0, %2, %5
+    %8:_(s32), %9:_(s1) = G_UADDE %1, %3, %7
+    %10:_(s32) = G_ANYEXT %9(s1)
+    $x10 = COPY %6(s32)
+    $x11 = COPY %8(s32)
+    $x12 = COPY %10(s32)
+
+    PseudoRET implicit $x10, implicit $x11, implicit $x12
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-uadde-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-uadde-rv64.mir
new file mode 100644
index 0000000000000..246205029c3fc
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-uadde-rv64.mir
@@ -0,0 +1,214 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            uadde_i8
+body:             |
+  bb.1:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: uadde_i8
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C1]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[ADD]], [[AND2]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[ADD1]], [[C]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND3]](s64), [[C2]]
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s64) = G_AND [[ICMP1]], [[COPY2]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[ICMP]], [[AND4]]
+    ; CHECK-NEXT: $x10 = COPY [[ADD1]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[OR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s64) = COPY $x10
+    %1:_(s8) = G_TRUNC %0(s64)
+    %2:_(s64) = COPY $x11
+    %3:_(s8) = G_TRUNC %2(s64)
+    %4:_(s64) = COPY $x12
+    %5:_(s1) = G_TRUNC %4(s64)
+    %6:_(s8), %7:_(s1) = G_UADDE %1, %3, %5
+    %8:_(s64) = G_ANYEXT %6(s8)
+    %9:_(s64) = G_ANYEXT %7(s1)
+    $x10 = COPY %8(s64)
+    $x11 = COPY %9(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            uadde_i16
+body:             |
+  bb.1:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: uadde_i16
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[C]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[AND]](s64), [[AND1]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C1]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[ADD]], [[AND2]]
+    ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s64) = G_AND [[ADD1]], [[C]]
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[AND3]](s64), [[C2]]
+    ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s64) = G_AND [[ICMP1]], [[COPY2]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[ICMP]], [[AND4]]
+    ; CHECK-NEXT: $x10 = COPY [[ADD1]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[OR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s64) = COPY $x10
+    %1:_(s16) = G_TRUNC %0(s64)
+    %2:_(s64) = COPY $x11
+    %3:_(s16) = G_TRUNC %2(s64)
+    %4:_(s64) = COPY $x12
+    %5:_(s1) = G_TRUNC %4(s64)
+    %6:_(s16), %7:_(s1) = G_UADDE %1, %3, %5
+    %8:_(s64) = G_ANYEXT %6(s16)
+    %9:_(s64) = G_ANYEXT %7(s1)
+    $x10 = COPY %8(s64)
+    $x11 = COPY %9(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            uadde_i32
+body:             |
+  bb.1:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: uadde_i32
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s64) = G_SEXT_INREG [[ADD]], 32
+    ; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s64) = G_SEXT_INREG [[COPY]], 32
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[SEXT_INREG]](s64), [[SEXT_INREG1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[SEXT_INREG]], [[AND]]
+    ; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s64) = G_SEXT_INREG [[ADD1]], 32
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[SEXT_INREG2]](s64), [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[ICMP1]], [[COPY2]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[ICMP]], [[AND1]]
+    ; CHECK-NEXT: $x10 = COPY [[SEXT_INREG2]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[OR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s64) = COPY $x10
+    %1:_(s32) = G_TRUNC %0(s64)
+    %2:_(s64) = COPY $x11
+    %3:_(s32) = G_TRUNC %2(s64)
+    %4:_(s64) = COPY $x12
+    %5:_(s1) = G_TRUNC %4(s64)
+    %6:_(s32), %7:_(s1) = G_UADDE %1, %3, %5
+    %8:_(s64) = G_ANYEXT %6(s32)
+    %9:_(s64) = G_ANYEXT %7(s1)
+    $x10 = COPY %8(s64)
+    $x11 = COPY %9(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            uadde_i64
+body:             |
+  bb.1:
+    liveins: $x10, $x11, $x12
+
+    ; CHECK-LABEL: name: uadde_i64
+    ; CHECK: liveins: $x10, $x11, $x12
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD]](s64), [[COPY]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY2]], [[C]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[ADD]], [[AND]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[ADD1]](s64), [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[ICMP1]], [[COPY2]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[ICMP]], [[AND1]]
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY [[ADD1]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[COPY3]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[OR]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64) = COPY $x12
+    %3:_(s1) = G_TRUNC %2(s64)
+    %4:_(s64), %5:_(s1) = G_UADDE %0, %1, %3
+    %6:_(s64) = G_ANYEXT %5(s1)
+    $x10 = COPY %4(s64)
+    $x11 = COPY %6(s64)
+    PseudoRET implicit $x10, implicit $x11
+
+...
+---
+name:            uadde_i128
+body:             |
+  bb.1:
+    liveins: $x10, $x11, $x12, $x13, $x14
+
+    ; CHECK-LABEL: name: uadde_i128
+    ; CHECK: liveins: $x10, $x11, $x12, $x13, $x14
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14
+    ; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY]], [[COPY2]]
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD]](s64), [[COPY]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY4]], [[C]]
+    ; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[ADD]], [[AND]]
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[ADD1]](s64), [[C1]]
+    ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[ICMP1]], [[COPY4]]
+    ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s64) = G_OR [[ICMP]], [[AND1]]
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY [[ADD1]](s64)
+    ; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s64) = G_ADD [[COPY1]], [[COPY3]]
+    ; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s64) = G_ICMP intpred(ult), [[ADD2]](s64), [[COPY1]]
+    ; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s64) = G_ADD [[ADD2]], [[OR]]
+    ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s64) = G_ICMP intpred(eq), [[ADD3]](s64), [[C1]]
+    ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[ICMP3]], [[OR]]
+    ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s64) = G_OR [[ICMP2]], [[AND2]]
+    ; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY [[ADD3]](s64)
+    ; CHECK-NEXT: $x10 = COPY [[COPY5]](s64)
+    ; CHECK-NEXT: $x11 = COPY [[COPY6]](s64)
+    ; CHECK-NEXT: $x12 = COPY [[OR1]](s64)
+    ; CHECK-NEXT: PseudoRET implicit $x10, implicit $x11, implicit $x12
+    %0:_(s64) = COPY $x10
+    %1:_(s64) = COPY $x11
+    %2:_(s64) = COPY $x12
+    %3:_(s64) = COPY $x13
+    %4:_(s64) = COPY $x14
+    %5:_(s1)  = G_TRUNC %4(s64)
+    %6:_(s64), %7:_(s1) = G_UADDE %0, %2, %5
+    %8:_(s64), %9:_(s1) = G_UADDE %1, %3, %7
+    %10:_(s64) = G_ANYEXT %9(s1)
+    $x10 = COPY %6(s64)
+    $x11 = COPY %8(s64)
+    $x12 = COPY %10(s64)
+
+    PseudoRET implicit $x10, implicit $x11, implicit $x12
+...
diff --git a/llvm/test/CodeGen/RISCV/alu64.ll b/llvm/test/CodeGen/RISCV/alu64.ll
index f032756e007b6..c7938a718de70 100644
--- a/llvm/test/CodeGen/RISCV/alu64.ll
+++ b/llvm/test/CodeGen/RISCV/alu64.ll
@@ -37,7 +37,7 @@ define i64 @slti(i64 %a) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    beqz a1, .LBB1_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slti a0, a1, 0
+; RV32I-NEXT:    srli a0, a1, 31
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB1_2:
diff --git a/llvm/test/CodeGen/RISCV/and-negpow2-cmp.ll b/llvm/test/CodeGen/RISCV/and-negpow2-cmp.ll
index be3de37927564..b16672d3c4a16 100644
--- a/llvm/test/CodeGen/RISCV/and-negpow2-cmp.ll
+++ b/llvm/test/CodeGen/RISCV/and-negpow2-cmp.ll
@@ -1,14 +1,66 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=riscv64 | FileCheck %s
+; RUN: llc < %s -mtriple=riscv32 | FileCheck %s --check-prefixes=CHECK,RV32
+; RUN: llc < %s -mtriple=riscv64 | FileCheck %s --check-prefixes=CHECK,RV64
 
-define i1 @src(i64 %x) {
-; CHECK-LABEL: src:
+define i1 @test1(i64 %x) {
+; RV32-LABEL: test1:
+; RV32:       # %bb.0:
+; RV32-NEXT:    slli a2, a1, 2
+; RV32-NEXT:    srli a0, a0, 30
+; RV32-NEXT:    srai a1, a1, 30
+; RV32-NEXT:    or a0, a0, a2
+; RV32-NEXT:    xori a0, a0, -2
+; RV32-NEXT:    not a1, a1
+; RV32-NEXT:    or a0, a0, a1
+; RV32-NEXT:    seqz a0, a0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: test1:
+; RV64:       # %bb.0:
+; RV64-NEXT:    srai a0, a0, 30
+; RV64-NEXT:    addi a0, a0, 2
+; RV64-NEXT:    seqz a0, a0
+; RV64-NEXT:    ret
+  %a = and i64 %x, -1073741824
+  %b = icmp eq i64 %a, -2147483648
+  ret i1 %b
+}
+
+define i1 @test2(i32 signext %x) {
+; CHECK-LABEL: test2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    srai a0, a0, 30
-; CHECK-NEXT:    addi a0, a0, 2
+; CHECK-NEXT:    srli a0, a0, 30
 ; CHECK-NEXT:    seqz a0, a0
 ; CHECK-NEXT:    ret
-  %a = and i64 %x, -1073741824
-  %b = icmp eq i64 %a, -2147483648
+  %a = and i32 %x, -1073741824
+  %b = icmp eq i32 %a, 0
+  ret i1 %b
+}
+
+define i1 @test3(i32 signext %x) {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srli a0, a0, 29
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    ret
+  %a = and i32 %x, -536870912
+  %b = icmp ne i32 %a, 0
+  ret i1 %b
+}
+
+define i1 @test4(i64 %x) {
+; RV32-LABEL: test4:
+; RV32:       # %bb.0:
+; RV32-NEXT:    srli a1, a1, 14
+; RV32-NEXT:    seqz a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: test4:
+; RV64:       # %bb.0:
+; RV64-NEXT:    srli a0, a0, 46
+; RV64-NEXT:    seqz a0, a0
+; RV64-NEXT:    ret
+  %a = and i64 %x, -70368744177664
+  %b = icmp eq i64 %a, 0
   ret i1 %b
 }
diff --git a/llvm/test/CodeGen/RISCV/arith-with-overflow.ll b/llvm/test/CodeGen/RISCV/arith-with-overflow.ll
index 4efc224ab1ca7..551d8864033f3 100644
--- a/llvm/test/CodeGen/RISCV/arith-with-overflow.ll
+++ b/llvm/test/CodeGen/RISCV/arith-with-overflow.ll
@@ -12,7 +12,7 @@ define i1 @sadd(i32 %a, i32 %b, ptr %c) nounwind {
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    add a3, a0, a1
 ; RV32I-NEXT:    slt a0, a3, a0
-; RV32I-NEXT:    slti a1, a1, 0
+; RV32I-NEXT:    srli a1, a1, 31
 ; RV32I-NEXT:    xor a0, a1, a0
 ; RV32I-NEXT:    sw a3, 0(a2)
 ; RV32I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index a28b818b3db11..362d4dc6eeb58 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -173,8 +173,8 @@
 ; RUN: llc -mtriple=riscv32 -mattr=+smmpm %s -o - | FileCheck --check-prefix=RV32SMMPM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+sspm %s -o - | FileCheck --check-prefix=RV32SSPM %s
 ; RUN: llc -mtriple=riscv32 -mattr=+supm %s -o - | FileCheck --check-prefix=RV32SUPM %s
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-smctr  %s -o - | FileCheck --check-prefix=RV32SMCTR %s
-; RUN: llc -mtriple=riscv32 -mattr=+experimental-ssctr  %s -o - | FileCheck --check-prefix=RV32SSCTR %s
+; RUN: llc -mtriple=riscv32 -mattr=+smctr  %s -o - | FileCheck --check-prefix=RV32SMCTR %s
+; RUN: llc -mtriple=riscv32 -mattr=+ssctr  %s -o - | FileCheck --check-prefix=RV32SSCTR %s
 
 ; RUN: llc -mtriple=riscv64 %s -o - | FileCheck %s
 ; RUN: llc -mtriple=riscv64 -mattr=+m %s -o - | FileCheck --check-prefixes=CHECK,RV64M %s
@@ -336,8 +336,8 @@
 ; RUN: llc -mtriple=riscv64 -mattr=+smmpm %s -o - | FileCheck --check-prefix=RV64SMMPM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+sspm %s -o - | FileCheck --check-prefix=RV64SSPM %s
 ; RUN: llc -mtriple=riscv64 -mattr=+supm %s -o - | FileCheck --check-prefix=RV64SUPM %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-smctr  %s -o - | FileCheck --check-prefix=RV64SMCTR %s
-; RUN: llc -mtriple=riscv64 -mattr=+experimental-ssctr  %s -o - | FileCheck --check-prefix=RV64SSCTR %s
+; RUN: llc -mtriple=riscv64 -mattr=+smctr  %s -o - | FileCheck --check-prefix=RV64SMCTR %s
+; RUN: llc -mtriple=riscv64 -mattr=+ssctr  %s -o - | FileCheck --check-prefix=RV64SSCTR %s
 ; RUN: llc -mtriple=riscv64 -mattr=+sdext  %s -o - | FileCheck --check-prefix=RV64SDEXT %s
 ; RUN: llc -mtriple=riscv64 -mattr=+sdtrig  %s -o - | FileCheck --check-prefix=RV64SDTRIG %s
 ; RUN: llc -mtriple=riscv64 -mattr=+experimental-xqccmp %s -o - | FileCheck --check-prefix=RV64XQCCMP %s
@@ -533,7 +533,7 @@
 ; RV32SUPM: .attribute 5, "rv32i2p1_supm1p0"
 ; RV32SMCTR: .attribute 5, "rv32i2p1_smctr1p0_sscsrind1p0"
 ; RV32SSCTR: .attribute 5, "rv32i2p1_sscsrind1p0_ssctr1p0"
-; RV32P: .attribute 5, "rv32i2p1_p0p14"
+; RV32P: .attribute 5, "rv32i2p1_p0p15"
 
 ; RV64M: .attribute 5, "rv64i2p1_m2p0_zmmul1p0"
 ; RV64ZMMUL: .attribute 5, "rv64i2p1_zmmul1p0"
@@ -698,7 +698,7 @@
 ; RV64SDEXT: .attribute 5, "rv64i2p1_sdext1p0"
 ; RV64SDTRIG: .attribute 5, "rv64i2p1_sdtrig1p0"
 ; RV64XQCCMP: .attribute 5, "rv64i2p1_zca1p0_xqccmp0p3"
-; RV64P: .attribute 5, "rv64i2p1_p0p14"
+; RV64P: .attribute 5, "rv64i2p1_p0p15"
 
 ; RVI20U32: .attribute 5, "rv32i2p1"
 ; RVI20U64: .attribute 5, "rv64i2p1"
diff --git a/llvm/test/CodeGen/RISCV/bittest.ll b/llvm/test/CodeGen/RISCV/bittest.ll
index fa6892be44d97..40e3168d08b27 100644
--- a/llvm/test/CodeGen/RISCV/bittest.ll
+++ b/llvm/test/CodeGen/RISCV/bittest.ll
@@ -187,14 +187,14 @@ define i64 @bittest_31_i64(i64 %a) nounwind {
 ;
 ; RV64ZBS-LABEL: bittest_31_i64:
 ; RV64ZBS:       # %bb.0:
-; RV64ZBS-NEXT:    not a0, a0
-; RV64ZBS-NEXT:    bexti a0, a0, 31
+; RV64ZBS-NEXT:    srliw a0, a0, 31
+; RV64ZBS-NEXT:    xori a0, a0, 1
 ; RV64ZBS-NEXT:    ret
 ;
 ; RV64XTHEADBS-LABEL: bittest_31_i64:
 ; RV64XTHEADBS:       # %bb.0:
-; RV64XTHEADBS-NEXT:    not a0, a0
-; RV64XTHEADBS-NEXT:    th.tst a0, a0, 31
+; RV64XTHEADBS-NEXT:    srliw a0, a0, 31
+; RV64XTHEADBS-NEXT:    xori a0, a0, 1
 ; RV64XTHEADBS-NEXT:    ret
   %shr = lshr i64 %a, 31
   %not = xor i64 %shr, -1
@@ -3507,3 +3507,77 @@ define void @bit_64_1_nz_branch_i64(i64 %0) {
 5:
   ret void
 }
+
+define i32 @bittest_31_andeq0_i64(i64 %x) {
+; RV32-LABEL: bittest_31_andeq0_i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    seqz a0, a0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: bittest_31_andeq0_i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    srliw a0, a0, 31
+; RV64-NEXT:    xori a0, a0, 1
+; RV64-NEXT:    ret
+  %and = and i64 %x, 2147483648
+  %cmp = icmp eq i64 %and, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @bittest_63_andeq0_i64(i64 %x) {
+; RV32-LABEL: bittest_63_andeq0_i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    seqz a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: bittest_63_andeq0_i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    seqz a0, a0
+; RV64-NEXT:    ret
+  %and = and i64 %x, 9223372036854775808
+  %cmp = icmp eq i64 %and, 0
+  %conv = zext i1 %cmp to i32
+  ret i32 %conv
+}
+
+define i32 @bittest_31_slt0_i32(i32 %x, i1 %y) {
+; RV32-LABEL: bittest_31_slt0_i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    and a0, a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: bittest_31_slt0_i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    srliw a0, a0, 31
+; RV64-NEXT:    and a0, a0, a1
+; RV64-NEXT:    ret
+  %cmp = icmp slt i32 %x, 0
+  %and = and i1 %cmp, %y
+  %ext = zext i1 %and to i32
+  ret i32 %ext
+}
+
+define i32 @bittest_63_slt0_i64(i32 %x, i1 %y) {
+; RV32-LABEL: bittest_63_slt0_i64:
+; RV32:       # %bb.0:
+; RV32-NEXT:    srai a0, a0, 31
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    and a0, a0, a1
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: bittest_63_slt0_i64:
+; RV64:       # %bb.0:
+; RV64-NEXT:    srliw a0, a0, 31
+; RV64-NEXT:    and a0, a0, a1
+; RV64-NEXT:    ret
+  %ext = sext i32 %x to i64
+  %cmp = icmp slt i64 %ext, 0
+  %and = and i1 %cmp, %y
+  %cond = zext i1 %and to i32
+  ret i32 %cond
+}
diff --git a/llvm/test/CodeGen/RISCV/condbinops.ll b/llvm/test/CodeGen/RISCV/condbinops.ll
index dc81c13bfb6a3..91052bce9704c 100644
--- a/llvm/test/CodeGen/RISCV/condbinops.ll
+++ b/llvm/test/CodeGen/RISCV/condbinops.ll
@@ -459,7 +459,7 @@ define i64 @shl64(i64 %x, i64 %y, i1 %c) {
 ; RV32ZICOND-NEXT:    addi a4, a2, -32
 ; RV32ZICOND-NEXT:    sll a1, a1, a2
 ; RV32ZICOND-NEXT:    not a2, a2
-; RV32ZICOND-NEXT:    slti a4, a4, 0
+; RV32ZICOND-NEXT:    srli a4, a4, 31
 ; RV32ZICOND-NEXT:    srl a2, a3, a2
 ; RV32ZICOND-NEXT:    czero.nez a3, a0, a4
 ; RV32ZICOND-NEXT:    or a1, a1, a2
@@ -534,7 +534,7 @@ define i64 @ashr64(i64 %x, i64 %y, i1 %c) {
 ; RV32ZICOND-NEXT:    addi a4, a2, -32
 ; RV32ZICOND-NEXT:    srl a0, a0, a2
 ; RV32ZICOND-NEXT:    not a2, a2
-; RV32ZICOND-NEXT:    slti a4, a4, 0
+; RV32ZICOND-NEXT:    srli a4, a4, 31
 ; RV32ZICOND-NEXT:    sll a2, a3, a2
 ; RV32ZICOND-NEXT:    czero.nez a3, a1, a4
 ; RV32ZICOND-NEXT:    or a0, a0, a2
@@ -610,7 +610,7 @@ define i64 @lshr64(i64 %x, i64 %y, i1 %c) {
 ; RV32ZICOND-NEXT:    addi a4, a2, -32
 ; RV32ZICOND-NEXT:    srl a0, a0, a2
 ; RV32ZICOND-NEXT:    not a2, a2
-; RV32ZICOND-NEXT:    slti a4, a4, 0
+; RV32ZICOND-NEXT:    srli a4, a4, 31
 ; RV32ZICOND-NEXT:    sll a2, a3, a2
 ; RV32ZICOND-NEXT:    czero.nez a3, a1, a4
 ; RV32ZICOND-NEXT:    or a0, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index a2e6186e051bf..9c81bc2851347 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -405,7 +405,7 @@ define i32 @fcvt_wu_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    li a2, 0
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi s3, a0, -1
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
@@ -446,8 +446,8 @@ define i32 @fcvt_wu_d_sat(double %a) nounwind {
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    j .LBB6_3
 ; RV64I-NEXT:  .LBB6_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB6_3: # %start
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
@@ -819,7 +819,7 @@ define i64 @fcvt_l_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    mv a3, s0
 ; RV32I-NEXT:    call __unorddf2
 ; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    slti a1, s4, 0
+; RV32I-NEXT:    srli a1, s4, 31
 ; RV32I-NEXT:    sgtz a2, s2
 ; RV32I-NEXT:    addi a0, a0, -1
 ; RV32I-NEXT:    addi a3, a1, -1
@@ -1029,7 +1029,7 @@ define i64 @fcvt_lu_d_sat(double %a) nounwind {
 ; RV32I-NEXT:    li a2, 0
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi s3, a0, -1
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
@@ -1055,7 +1055,7 @@ define i64 @fcvt_lu_d_sat(double %a) nounwind {
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    li a1, 0
 ; RV64I-NEXT:    call __gedf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    addi s1, a0, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __fixunsdfdi
@@ -1898,9 +1898,9 @@ define zeroext i16 @fcvt_wu_s_sat_i16(double %a) nounwind {
 ; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    j .LBB28_3
 ; RV32I-NEXT:  .LBB28_2:
-; RV32I-NEXT:    slti a2, s0, 0
-; RV32I-NEXT:    addi a2, a2, -1
-; RV32I-NEXT:    and a0, a2, a0
+; RV32I-NEXT:    srli s0, s0, 31
+; RV32I-NEXT:    addi s0, s0, -1
+; RV32I-NEXT:    and a0, s0, a0
 ; RV32I-NEXT:  .LBB28_3: # %start
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
@@ -1937,8 +1937,8 @@ define zeroext i16 @fcvt_wu_s_sat_i16(double %a) nounwind {
 ; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    j .LBB28_3
 ; RV64I-NEXT:  .LBB28_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB28_3: # %start
 ; RV64I-NEXT:    and a0, a0, a1
@@ -2271,9 +2271,9 @@ define zeroext i8 @fcvt_wu_s_sat_i8(double %a) nounwind {
 ; RV32I-NEXT:    li a0, 255
 ; RV32I-NEXT:    j .LBB32_3
 ; RV32I-NEXT:  .LBB32_2:
-; RV32I-NEXT:    slti a1, s0, 0
-; RV32I-NEXT:    addi a1, a1, -1
-; RV32I-NEXT:    and a0, a1, a0
+; RV32I-NEXT:    srli s0, s0, 31
+; RV32I-NEXT:    addi s0, s0, -1
+; RV32I-NEXT:    and a0, s0, a0
 ; RV32I-NEXT:  .LBB32_3: # %start
 ; RV32I-NEXT:    zext.b a0, a0
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
@@ -2307,8 +2307,8 @@ define zeroext i8 @fcvt_wu_s_sat_i8(double %a) nounwind {
 ; RV64I-NEXT:    li a0, 255
 ; RV64I-NEXT:    j .LBB32_3
 ; RV64I-NEXT:  .LBB32_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB32_3: # %start
 ; RV64I-NEXT:    zext.b a0, a0
@@ -2386,7 +2386,7 @@ define zeroext i32 @fcvt_wu_d_sat_zext(double %a) nounwind {
 ; RV32I-NEXT:    li a2, 0
 ; RV32I-NEXT:    li a3, 0
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi s3, a0, -1
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s0
@@ -2427,8 +2427,8 @@ define zeroext i32 @fcvt_wu_d_sat_zext(double %a) nounwind {
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    j .LBB33_3
 ; RV64I-NEXT:  .LBB33_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB33_3: # %start
 ; RV64I-NEXT:    slli a0, a0, 32
diff --git a/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll b/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll
index 7c5332f719867..b1c63af3e7e07 100644
--- a/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll
+++ b/llvm/test/CodeGen/RISCV/double-fcmp-strict.ll
@@ -140,7 +140,7 @@ define i32 @fcmp_oge(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -151,7 +151,7 @@ define i32 @fcmp_oge(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gedf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -193,7 +193,7 @@ define i32 @fcmp_olt(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltdf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -203,7 +203,7 @@ define i32 @fcmp_olt(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltdf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -605,7 +605,7 @@ define i32 @fcmp_uge(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltdf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -616,7 +616,7 @@ define i32 @fcmp_uge(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltdf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -661,7 +661,7 @@ define i32 @fcmp_ult(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -671,7 +671,7 @@ define i32 @fcmp_ult(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gedf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -934,7 +934,7 @@ define i32 @fcmps_oge(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -945,7 +945,7 @@ define i32 @fcmps_oge(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gedf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -976,7 +976,7 @@ define i32 @fcmps_olt(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltdf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -986,7 +986,7 @@ define i32 @fcmps_olt(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltdf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -1311,7 +1311,7 @@ define i32 @fcmps_uge(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltdf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1322,7 +1322,7 @@ define i32 @fcmps_uge(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltdf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -1356,7 +1356,7 @@ define i32 @fcmps_ult(double %a, double %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -1366,7 +1366,7 @@ define i32 @fcmps_ult(double %a, double %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gedf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/double-fcmp.ll b/llvm/test/CodeGen/RISCV/double-fcmp.ll
index f73e6865cf47d..31c8589177439 100644
--- a/llvm/test/CodeGen/RISCV/double-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-fcmp.ll
@@ -138,7 +138,7 @@ define i32 @fcmp_oge(double %a, double %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -149,7 +149,7 @@ define i32 @fcmp_oge(double %a, double %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gedf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -180,7 +180,7 @@ define i32 @fcmp_olt(double %a, double %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltdf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -190,7 +190,7 @@ define i32 @fcmp_olt(double %a, double %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltdf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -515,7 +515,7 @@ define i32 @fcmp_uge(double %a, double %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltdf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -526,7 +526,7 @@ define i32 @fcmp_uge(double %a, double %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltdf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -560,7 +560,7 @@ define i32 @fcmp_ult(double %a, double %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gedf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -570,7 +570,7 @@ define i32 @fcmp_ult(double %a, double %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gedf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/early-clobber-tied-def-subreg-liveness.ll b/llvm/test/CodeGen/RISCV/early-clobber-tied-def-subreg-liveness.ll
index 0afdcdccd9246..6a7c73672bf6c 100644
--- a/llvm/test/CodeGen/RISCV/early-clobber-tied-def-subreg-liveness.ll
+++ b/llvm/test/CodeGen/RISCV/early-clobber-tied-def-subreg-liveness.ll
@@ -40,15 +40,7 @@ define void @_Z3foov() {
 ; CHECK-NEXT:    addi a0, a0, %lo(.L__const._Z3foov.var_45)
 ; CHECK-NEXT:    vle16.v v12, (a0)
 ; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    vs2r.v v12, (a0) # vscale x 16-byte Folded Spill
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    vs2r.v v14, (a0) # vscale x 16-byte Folded Spill
+; CHECK-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    lui a0, %hi(.L__const._Z3foov.var_40)
 ; CHECK-NEXT:    addi a0, a0, %lo(.L__const._Z3foov.var_40)
 ; CHECK-NEXT:    #APP
@@ -59,15 +51,7 @@ define void @_Z3foov() {
 ; CHECK-NEXT:    addi a0, a0, 928
 ; CHECK-NEXT:    vmsbc.vx v0, v8, a0
 ; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    vl2r.v v10, (a0) # vscale x 16-byte Folded Reload
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    vl2r.v v12, (a0) # vscale x 16-byte Folded Reload
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    vl2r.v v14, (a0) # vscale x 16-byte Folded Reload
+; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
diff --git a/llvm/test/CodeGen/RISCV/features-info.ll b/llvm/test/CodeGen/RISCV/features-info.ll
index b94665b718ae7..f966f800589b7 100644
--- a/llvm/test/CodeGen/RISCV/features-info.ll
+++ b/llvm/test/CodeGen/RISCV/features-info.ll
@@ -6,21 +6,27 @@
 ; CHECK-NEXT:   32bit                            - Implements RV32.
 ; CHECK-NEXT:   64bit                            - Implements RV64.
 ; CHECK-NEXT:   a                                - 'A' (Atomic Instructions).
+; CHECK-NEXT:   add-load-fusion                  - Enable ADD(.UW) + load macrofusion.
+; CHECK-NEXT:   addi-load-fusion                 - Enable ADDI + load macrofusion.
 ; CHECK-NEXT:   andes45                          - Andes 45-Series processors.
 ; CHECK-NEXT:   auipc-addi-fusion                - Enable AUIPC+ADDI macrofusion.
+; CHECK-NEXT:   auipc-load-fusion                - Enable AUIPC + load macrofusion.
 ; CHECK-NEXT:   b                                - 'B' (the collection of the Zba, Zbb, Zbs extensions).
+; CHECK-NEXT:   bfext-fusion                     - Enable SLLI+SRLI (bitfield extract) macrofusion.
 ; CHECK-NEXT:   c                                - 'C' (Compressed Instructions).
 ; CHECK-NEXT:   conditional-cmv-fusion           - Enable branch+c.mv fusion.
 ; CHECK-NEXT:   d                                - 'D' (Double-Precision Floating-Point).
 ; CHECK-NEXT:   disable-latency-sched-heuristic  - Disable latency scheduling heuristic.
+; CHECK-NEXT:   disable-misched-load-clustering  - Disable load clustering in the machine scheduler.
+; CHECK-NEXT:   disable-misched-store-clustering - Disable store clustering in the machine scheduler.
+; CHECK-NEXT:   disable-postmisched-load-clustering  - Disable PostRA load clustering in the machine scheduler.
+; CHECK-NEXT:   disable-postmisched-store-clustering - Disable PostRA store clustering in the machine scheduler.
 ; CHECK-NEXT:   dlen-factor-2                    - Vector unit DLEN(data path width) is half of VLEN.
 ; CHECK-NEXT:   e                                - 'E' (Embedded Instruction Set with 16 GPRs).
 ; CHECK-NEXT:   exact-asm                        - Enable Exact Assembly (Disables Compression and Relaxation).
 ; CHECK-NEXT:   experimental                     - Experimental intrinsics.
 ; CHECK-NEXT:   experimental-p                   - 'P' ('Base P' (Packed SIMD)).
 ; CHECK-NEXT:   experimental-rvm23u32            - RISC-V experimental-rvm23u32 profile.
-; CHECK-NEXT:   experimental-smctr               - 'Smctr' (Control Transfer Records Machine Level).
-; CHECK-NEXT:   experimental-ssctr               - 'Ssctr' (Control Transfer Records Supervisor Level).
 ; CHECK-NEXT:   experimental-svukte              - 'Svukte' (Address-Independent Latency of User-Mode Faults to Supervisor Addresses).
 ; CHECK-NEXT:   experimental-xqccmp              - 'Xqccmp' (Qualcomm 16-bit Push/Pop and Double Moves).
 ; CHECK-NEXT:   experimental-xqcia               - 'Xqcia' (Qualcomm uC Arithmetic Extension).
@@ -58,6 +64,7 @@
 ; CHECK-NEXT:   ld-add-fusion                    - Enable LD+ADD macrofusion.
 ; CHECK-NEXT:   log-vrgather                     - Has vrgather.vv with LMUL*log2(LMUL) latency
 ; CHECK-NEXT:   lui-addi-fusion                  - Enable LUI+ADDI macro fusion.
+; CHECK-NEXT:   lui-load-fusion                  - Enable LUI + load macrofusion.
 ; CHECK-NEXT:   m                                - 'M' (Integer Multiplication and Division).
 ; CHECK-NEXT:   mips-p8700                       - MIPS p8700 processor.
 ; CHECK-NEXT:   no-default-unroll                - Disable default unroll preference..
@@ -130,11 +137,13 @@
 ; CHECK-NEXT:   shvsatpa                         - 'Shvsatpa' (vsatp supports all modes supported by satp).
 ; CHECK-NEXT:   shvstvala                        - 'Shvstvala' (vstval provides all needed values).
 ; CHECK-NEXT:   shvstvecd                        - 'Shvstvecd' (vstvec supports Direct mode).
+; CHECK-NEXT:   shxadd-load-fusion               - Enable SH(1|2|3)ADD(.UW) + load macrofusion.
 ; CHECK-NEXT:   sifive7                          - SiFive 7-Series processors.
 ; CHECK-NEXT:   smaia                            - 'Smaia' (Advanced Interrupt Architecture Machine Level).
 ; CHECK-NEXT:   smcdeleg                         - 'Smcdeleg' (Counter Delegation Machine Level).
 ; CHECK-NEXT:   smcntrpmf                        - 'Smcntrpmf' (Cycle and Instret Privilege Mode Filtering).
 ; CHECK-NEXT:   smcsrind                         - 'Smcsrind' (Indirect CSR Access Machine Level).
+; CHECK-NEXT:   smctr                            - 'Smctr' (Control Transfer Records Machine Level).
 ; CHECK-NEXT:   smdbltrp                         - 'Smdbltrp' (Double Trap Machine Level).
 ; CHECK-NEXT:   smepmp                           - 'Smepmp' (Enhanced Physical Memory Protection).
 ; CHECK-NEXT:   smmpm                            - 'Smmpm' (Machine-level Pointer Masking for M-mode).
@@ -147,6 +156,7 @@
 ; CHECK-NEXT:   sscofpmf                         - 'Sscofpmf' (Count Overflow and Mode-Based Filtering).
 ; CHECK-NEXT:   sscounterenw                     - 'Sscounterenw' (Support writeable scounteren enable bit for any hpmcounter that is not read-only zero).
 ; CHECK-NEXT:   sscsrind                         - 'Sscsrind' (Indirect CSR Access Supervisor Level).
+; CHECK-NEXT:   ssctr                            - 'Ssctr' (Control Transfer Records Supervisor Level).
 ; CHECK-NEXT:   ssdbltrp                         - 'Ssdbltrp' (Double Trap Supervisor Level).
 ; CHECK-NEXT:   ssnpm                            - 'Ssnpm' (Supervisor-level Pointer Masking for next lower privilege mode).
 ; CHECK-NEXT:   sspm                             - 'Sspm' (Indicates Supervisor-mode Pointer Masking).
@@ -207,6 +217,7 @@
 ; CHECK-NEXT:   xsfvqmaccqoq                     - 'XSfvqmaccqoq' (SiFive Int8 Matrix Multiplication Instructions (4-by-8 and 8-by-4)).
 ; CHECK-NEXT:   xsifivecdiscarddlone             - 'XSiFivecdiscarddlone' (SiFive sf.cdiscard.d.l1 Instruction).
 ; CHECK-NEXT:   xsifivecflushdlone               - 'XSiFivecflushdlone' (SiFive sf.cflush.d.l1 Instruction).
+; CHECK-NEXT:   xsmtvdot                         - 'XSMTVDot' (SpacemiT Vector Dot Product Extension).
 ; CHECK-NEXT:   xtheadba                         - 'XTHeadBa' (T-Head address calculation instructions).
 ; CHECK-NEXT:   xtheadbb                         - 'XTHeadBb' (T-Head basic bit-manipulation instructions).
 ; CHECK-NEXT:   xtheadbs                         - 'XTHeadBs' (T-Head single-bit instructions).
diff --git a/llvm/test/CodeGen/RISCV/float-convert.ll b/llvm/test/CodeGen/RISCV/float-convert.ll
index 60349a0e39953..6e49d479cf0b9 100644
--- a/llvm/test/CodeGen/RISCV/float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert.ll
@@ -278,7 +278,7 @@ define i32 @fcvt_wu_s_sat(float %a) nounwind {
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi s1, a0, -1
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call __fixunssfsi
@@ -320,8 +320,8 @@ define i32 @fcvt_wu_s_sat(float %a) nounwind {
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    j .LBB4_3
 ; RV64I-NEXT:  .LBB4_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB4_3: # %start
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
@@ -736,7 +736,7 @@ define i64 @fcvt_l_s_sat(float %a) nounwind {
 ; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __unordsf2
 ; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    slti a1, s2, 0
+; RV32I-NEXT:    srli a1, s2, 31
 ; RV32I-NEXT:    sgtz a2, s4
 ; RV32I-NEXT:    addi a0, a0, -1
 ; RV32I-NEXT:    addi a3, a1, -1
@@ -932,7 +932,7 @@ define i64 @fcvt_lu_s_sat(float %a) nounwind {
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi s2, a0, -1
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call __fixunssfdi
@@ -971,7 +971,7 @@ define i64 @fcvt_lu_s_sat(float %a) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    li a1, 0
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    addi s2, a0, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __fixunssfdi
@@ -1651,8 +1651,8 @@ define zeroext i16 @fcvt_wu_s_sat_i16(float %a) nounwind {
 ; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    j .LBB26_3
 ; RV32I-NEXT:  .LBB26_2:
-; RV32I-NEXT:    slti a0, s0, 0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    srli s0, s0, 31
+; RV32I-NEXT:    addi a0, s0, -1
 ; RV32I-NEXT:    and a0, a0, s1
 ; RV32I-NEXT:  .LBB26_3: # %start
 ; RV32I-NEXT:    and a0, a0, a1
@@ -1688,8 +1688,8 @@ define zeroext i16 @fcvt_wu_s_sat_i16(float %a) nounwind {
 ; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    j .LBB26_3
 ; RV64I-NEXT:  .LBB26_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB26_3: # %start
 ; RV64I-NEXT:    and a0, a0, a1
@@ -1986,8 +1986,8 @@ define zeroext i8 @fcvt_wu_s_sat_i8(float %a) nounwind {
 ; RV32I-NEXT:    li a0, 255
 ; RV32I-NEXT:    j .LBB30_3
 ; RV32I-NEXT:  .LBB30_2:
-; RV32I-NEXT:    slti a0, s0, 0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    srli s0, s0, 31
+; RV32I-NEXT:    addi a0, s0, -1
 ; RV32I-NEXT:    and a0, a0, s1
 ; RV32I-NEXT:  .LBB30_3: # %start
 ; RV32I-NEXT:    zext.b a0, a0
@@ -2020,8 +2020,8 @@ define zeroext i8 @fcvt_wu_s_sat_i8(float %a) nounwind {
 ; RV64I-NEXT:    li a0, 255
 ; RV64I-NEXT:    j .LBB30_3
 ; RV64I-NEXT:  .LBB30_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB30_3: # %start
 ; RV64I-NEXT:    zext.b a0, a0
@@ -2087,7 +2087,7 @@ define zeroext i32 @fcvt_wu_s_sat_zext(float %a) nounwind {
 ; RV32I-NEXT:    mv s0, a0
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi s1, a0, -1
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call __fixunssfsi
@@ -2129,8 +2129,8 @@ define zeroext i32 @fcvt_wu_s_sat_zext(float %a) nounwind {
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    j .LBB31_3
 ; RV64I-NEXT:  .LBB31_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB31_3: # %start
 ; RV64I-NEXT:    slli a0, a0, 32
diff --git a/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll b/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll
index fd3baa0575250..7cdd1826b4522 100644
--- a/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll
+++ b/llvm/test/CodeGen/RISCV/float-fcmp-strict.ll
@@ -117,7 +117,7 @@ define i32 @fcmp_oge(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -128,7 +128,7 @@ define i32 @fcmp_oge(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -161,7 +161,7 @@ define i32 @fcmp_olt(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltsf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -171,7 +171,7 @@ define i32 @fcmp_olt(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltsf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -492,7 +492,7 @@ define i32 @fcmp_uge(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltsf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -503,7 +503,7 @@ define i32 @fcmp_uge(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltsf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -538,7 +538,7 @@ define i32 @fcmp_ult(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -548,7 +548,7 @@ define i32 @fcmp_ult(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -770,7 +770,7 @@ define i32 @fcmps_oge(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -781,7 +781,7 @@ define i32 @fcmps_oge(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -807,7 +807,7 @@ define i32 @fcmps_olt(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltsf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -817,7 +817,7 @@ define i32 @fcmps_olt(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltsf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -1087,7 +1087,7 @@ define i32 @fcmps_uge(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltsf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1098,7 +1098,7 @@ define i32 @fcmps_uge(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltsf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -1126,7 +1126,7 @@ define i32 @fcmps_ult(float %a, float %b) nounwind strictfp {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -1136,7 +1136,7 @@ define i32 @fcmps_ult(float %a, float %b) nounwind strictfp {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/float-fcmp.ll b/llvm/test/CodeGen/RISCV/float-fcmp.ll
index 2e9c39f331bbc..cec6580247627 100644
--- a/llvm/test/CodeGen/RISCV/float-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-fcmp.ll
@@ -123,7 +123,7 @@ define i32 @fcmp_oge(float %a, float %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -134,7 +134,7 @@ define i32 @fcmp_oge(float %a, float %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -160,7 +160,7 @@ define i32 @fcmp_olt(float %a, float %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltsf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -170,7 +170,7 @@ define i32 @fcmp_olt(float %a, float %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltsf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
@@ -440,7 +440,7 @@ define i32 @fcmp_uge(float %a, float %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __ltsf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    xori a0, a0, 1
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -451,7 +451,7 @@ define i32 @fcmp_uge(float %a, float %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __ltsf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    xori a0, a0, 1
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
@@ -479,7 +479,7 @@ define i32 @fcmp_ult(float %a, float %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -489,7 +489,7 @@ define i32 @fcmp_ult(float %a, float %b) nounwind {
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index ed50042f54ab5..8b8a3257a0027 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -1634,7 +1634,7 @@ define i1 @fpclass(float %x) {
 ; RV32I-NEXT:    add a4, a5, a4
 ; RV32I-NEXT:    addi a5, a5, -1
 ; RV32I-NEXT:    sltu a2, a5, a2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    seqz a1, a1
 ; RV32I-NEXT:    seqz a5, a6
 ; RV32I-NEXT:    srli a4, a4, 24
@@ -1660,8 +1660,7 @@ define i1 @fpclass(float %x) {
 ; RV64I-NEXT:    add a4, a5, a4
 ; RV64I-NEXT:    addi a5, a5, -1
 ; RV64I-NEXT:    sltu a2, a5, a2
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srliw a0, a0, 31
 ; RV64I-NEXT:    seqz a1, a1
 ; RV64I-NEXT:    seqz a5, a6
 ; RV64I-NEXT:    srliw a4, a4, 24
@@ -2092,19 +2091,18 @@ define i1 @isnegfinite_fpclass(float %x) {
 ; RV32I-NEXT:    lui a2, 522240
 ; RV32I-NEXT:    srli a1, a1, 1
 ; RV32I-NEXT:    slt a1, a1, a2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    and a0, a1, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: isnegfinite_fpclass:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    slli a0, a0, 33
+; RV64I-NEXT:    slli a1, a0, 33
 ; RV64I-NEXT:    lui a2, 522240
-; RV64I-NEXT:    srli a0, a0, 33
-; RV64I-NEXT:    slt a0, a0, a2
-; RV64I-NEXT:    slti a1, a1, 0
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    srli a1, a1, 33
+; RV64I-NEXT:    slt a1, a1, a2
+; RV64I-NEXT:    srliw a0, a0, 31
+; RV64I-NEXT:    and a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = call i1 @llvm.is.fpclass.f32(float %x, i32 56)  ; 0x38 = "-finite"
   ret i1 %1
diff --git a/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll b/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
index 477a7d1ce7b6b..aa65ebecbe56a 100644
--- a/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
+++ b/llvm/test/CodeGen/RISCV/fold-addi-loadstore.ll
@@ -909,7 +909,7 @@ define i64 @fold_addi_from_different_bb(i64 %k, i64 %n, ptr %a) nounwind {
 ; RV32I-NEXT:    mv s2, a2
 ; RV32I-NEXT:    beqz a3, .LBB20_3
 ; RV32I-NEXT:  # %bb.1: # %entry
-; RV32I-NEXT:    slti a1, s1, 0
+; RV32I-NEXT:    srli a1, s1, 31
 ; RV32I-NEXT:    beqz a1, .LBB20_4
 ; RV32I-NEXT:  .LBB20_2:
 ; RV32I-NEXT:    li s3, 0
@@ -974,7 +974,7 @@ define i64 @fold_addi_from_different_bb(i64 %k, i64 %n, ptr %a) nounwind {
 ; RV32I-MEDIUM-NEXT:    mv s2, a2
 ; RV32I-MEDIUM-NEXT:    beqz a3, .LBB20_3
 ; RV32I-MEDIUM-NEXT:  # %bb.1: # %entry
-; RV32I-MEDIUM-NEXT:    slti a1, s1, 0
+; RV32I-MEDIUM-NEXT:    srli a1, s1, 31
 ; RV32I-MEDIUM-NEXT:    beqz a1, .LBB20_4
 ; RV32I-MEDIUM-NEXT:  .LBB20_2:
 ; RV32I-MEDIUM-NEXT:    li s3, 0
diff --git a/llvm/test/CodeGen/RISCV/forced-atomics.ll b/llvm/test/CodeGen/RISCV/forced-atomics.ll
index e7719dc70660b..1a69106a485eb 100644
--- a/llvm/test/CodeGen/RISCV/forced-atomics.ll
+++ b/llvm/test/CodeGen/RISCV/forced-atomics.ll
@@ -3475,7 +3475,7 @@ define i64 @rmw64_min_seq_cst(ptr %p) nounwind {
 ; RV32-NEXT:    beqz a1, .LBB50_4
 ; RV32-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32-NEXT:    # in Loop: Header=BB50_2 Depth=1
-; RV32-NEXT:    slti a0, a1, 0
+; RV32-NEXT:    srli a0, a1, 31
 ; RV32-NEXT:    mv a2, a4
 ; RV32-NEXT:    bnez a0, .LBB50_1
 ; RV32-NEXT:    j .LBB50_5
diff --git a/llvm/test/CodeGen/RISCV/fpclamptosat.ll b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
index 519f1e851a832..18d071cc39bb6 100644
--- a/llvm/test/CodeGen/RISCV/fpclamptosat.ll
+++ b/llvm/test/CodeGen/RISCV/fpclamptosat.ll
@@ -22,7 +22,7 @@ define i32 @stest_f64i32(double %x) {
 ; RV32IF-NEXT:    addi a3, a2, -1
 ; RV32IF-NEXT:    beqz a1, .LBB0_2
 ; RV32IF-NEXT:  # %bb.1: # %entry
-; RV32IF-NEXT:    slti a4, a1, 0
+; RV32IF-NEXT:    srli a4, a1, 31
 ; RV32IF-NEXT:    j .LBB0_3
 ; RV32IF-NEXT:  .LBB0_2:
 ; RV32IF-NEXT:    sltu a4, a0, a3
@@ -36,7 +36,7 @@ define i32 @stest_f64i32(double %x) {
 ; RV32IF-NEXT:    li a3, -1
 ; RV32IF-NEXT:    beq a1, a3, .LBB0_7
 ; RV32IF-NEXT:  # %bb.6: # %entry
-; RV32IF-NEXT:    slti a1, a1, 0
+; RV32IF-NEXT:    srli a1, a1, 31
 ; RV32IF-NEXT:    xori a1, a1, 1
 ; RV32IF-NEXT:    beqz a1, .LBB0_8
 ; RV32IF-NEXT:    j .LBB0_9
@@ -185,7 +185,7 @@ define i32 @ustest_f64i32(double %x) {
 ; RV32IF-NEXT:    call __fixdfdi
 ; RV32IF-NEXT:    beqz a1, .LBB2_2
 ; RV32IF-NEXT:  # %bb.1: # %entry
-; RV32IF-NEXT:    slti a2, a1, 0
+; RV32IF-NEXT:    srli a2, a1, 31
 ; RV32IF-NEXT:    j .LBB2_3
 ; RV32IF-NEXT:  .LBB2_2:
 ; RV32IF-NEXT:    sltiu a2, a0, -1
@@ -373,7 +373,7 @@ define i32 @stest_f16i32(half %x) {
 ; RV32-NEXT:    addi a3, a2, -1
 ; RV32-NEXT:    beqz a1, .LBB6_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a4, a1, 0
+; RV32-NEXT:    srli a4, a1, 31
 ; RV32-NEXT:    j .LBB6_3
 ; RV32-NEXT:  .LBB6_2:
 ; RV32-NEXT:    sltu a4, a0, a3
@@ -387,7 +387,7 @@ define i32 @stest_f16i32(half %x) {
 ; RV32-NEXT:    li a3, -1
 ; RV32-NEXT:    beq a1, a3, .LBB6_7
 ; RV32-NEXT:  # %bb.6: # %entry
-; RV32-NEXT:    slti a1, a1, 0
+; RV32-NEXT:    srli a1, a1, 31
 ; RV32-NEXT:    xori a1, a1, 1
 ; RV32-NEXT:    beqz a1, .LBB6_8
 ; RV32-NEXT:    j .LBB6_9
@@ -494,7 +494,7 @@ define i32 @ustest_f16i32(half %x) {
 ; RV32-NEXT:    call __fixsfdi
 ; RV32-NEXT:    beqz a1, .LBB8_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a2, a1, 0
+; RV32-NEXT:    srli a2, a1, 31
 ; RV32-NEXT:    j .LBB8_3
 ; RV32-NEXT:  .LBB8_2:
 ; RV32-NEXT:    sltiu a2, a0, -1
@@ -1108,7 +1108,7 @@ define i64 @stest_f64i64(double %x) {
 ; RV32IF-NEXT:    or a7, a2, a4
 ; RV32IF-NEXT:    beqz a7, .LBB18_4
 ; RV32IF-NEXT:  .LBB18_3: # %entry
-; RV32IF-NEXT:    slti a6, a4, 0
+; RV32IF-NEXT:    srli a6, a4, 31
 ; RV32IF-NEXT:  .LBB18_4: # %entry
 ; RV32IF-NEXT:    neg a7, a6
 ; RV32IF-NEXT:    addi t0, a6, -1
@@ -1130,8 +1130,8 @@ define i64 @stest_f64i64(double %x) {
 ; RV32IF-NEXT:    li a5, -1
 ; RV32IF-NEXT:    beq a2, a5, .LBB18_11
 ; RV32IF-NEXT:  # %bb.10: # %entry
-; RV32IF-NEXT:    slti a0, a4, 0
-; RV32IF-NEXT:    xori a0, a0, 1
+; RV32IF-NEXT:    srli a4, a4, 31
+; RV32IF-NEXT:    xori a0, a4, 1
 ; RV32IF-NEXT:  .LBB18_11: # %entry
 ; RV32IF-NEXT:    bnez a0, .LBB18_13
 ; RV32IF-NEXT:  # %bb.12: # %entry
@@ -1156,7 +1156,7 @@ define i64 @stest_f64i64(double %x) {
 ; RV64IF-NEXT:    srli a3, a2, 1
 ; RV64IF-NEXT:    beqz a1, .LBB18_2
 ; RV64IF-NEXT:  # %bb.1: # %entry
-; RV64IF-NEXT:    slti a4, a1, 0
+; RV64IF-NEXT:    srli a4, a1, 63
 ; RV64IF-NEXT:    j .LBB18_3
 ; RV64IF-NEXT:  .LBB18_2:
 ; RV64IF-NEXT:    sltu a4, a0, a3
@@ -1170,8 +1170,8 @@ define i64 @stest_f64i64(double %x) {
 ; RV64IF-NEXT:    slli a1, a2, 63
 ; RV64IF-NEXT:    beq a5, a2, .LBB18_7
 ; RV64IF-NEXT:  # %bb.6: # %entry
-; RV64IF-NEXT:    slti a2, a5, 0
-; RV64IF-NEXT:    xori a2, a2, 1
+; RV64IF-NEXT:    srli a5, a5, 63
+; RV64IF-NEXT:    xori a2, a5, 1
 ; RV64IF-NEXT:    beqz a2, .LBB18_8
 ; RV64IF-NEXT:    j .LBB18_9
 ; RV64IF-NEXT:  .LBB18_7:
@@ -1211,7 +1211,7 @@ define i64 @stest_f64i64(double %x) {
 ; RV32IFD-NEXT:    or a7, a2, a4
 ; RV32IFD-NEXT:    beqz a7, .LBB18_4
 ; RV32IFD-NEXT:  .LBB18_3: # %entry
-; RV32IFD-NEXT:    slti a6, a4, 0
+; RV32IFD-NEXT:    srli a6, a4, 31
 ; RV32IFD-NEXT:  .LBB18_4: # %entry
 ; RV32IFD-NEXT:    neg a7, a6
 ; RV32IFD-NEXT:    addi t0, a6, -1
@@ -1233,8 +1233,8 @@ define i64 @stest_f64i64(double %x) {
 ; RV32IFD-NEXT:    li a5, -1
 ; RV32IFD-NEXT:    beq a2, a5, .LBB18_11
 ; RV32IFD-NEXT:  # %bb.10: # %entry
-; RV32IFD-NEXT:    slti a0, a4, 0
-; RV32IFD-NEXT:    xori a0, a0, 1
+; RV32IFD-NEXT:    srli a4, a4, 31
+; RV32IFD-NEXT:    xori a0, a4, 1
 ; RV32IFD-NEXT:  .LBB18_11: # %entry
 ; RV32IFD-NEXT:    bnez a0, .LBB18_13
 ; RV32IFD-NEXT:  # %bb.12: # %entry
@@ -1363,7 +1363,7 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IF-NEXT:    lw a0, 16(sp)
 ; RV32IF-NEXT:    beqz a1, .LBB20_2
 ; RV32IF-NEXT:  # %bb.1: # %entry
-; RV32IF-NEXT:    slti a2, a1, 0
+; RV32IF-NEXT:    srli a2, a1, 31
 ; RV32IF-NEXT:    j .LBB20_3
 ; RV32IF-NEXT:  .LBB20_2:
 ; RV32IF-NEXT:    seqz a2, a0
@@ -1446,7 +1446,7 @@ define i64 @ustest_f64i64(double %x) {
 ; RV32IFD-NEXT:    lw a0, 16(sp)
 ; RV32IFD-NEXT:    beqz a1, .LBB20_2
 ; RV32IFD-NEXT:  # %bb.1: # %entry
-; RV32IFD-NEXT:    slti a2, a1, 0
+; RV32IFD-NEXT:    srli a2, a1, 31
 ; RV32IFD-NEXT:    j .LBB20_3
 ; RV32IFD-NEXT:  .LBB20_2:
 ; RV32IFD-NEXT:    seqz a2, a0
@@ -1523,7 +1523,7 @@ define i64 @stest_f32i64(float %x) {
 ; RV32-NEXT:    or a7, a2, a4
 ; RV32-NEXT:    beqz a7, .LBB21_4
 ; RV32-NEXT:  .LBB21_3: # %entry
-; RV32-NEXT:    slti a6, a4, 0
+; RV32-NEXT:    srli a6, a4, 31
 ; RV32-NEXT:  .LBB21_4: # %entry
 ; RV32-NEXT:    neg a7, a6
 ; RV32-NEXT:    addi t0, a6, -1
@@ -1545,8 +1545,8 @@ define i64 @stest_f32i64(float %x) {
 ; RV32-NEXT:    li a5, -1
 ; RV32-NEXT:    beq a2, a5, .LBB21_11
 ; RV32-NEXT:  # %bb.10: # %entry
-; RV32-NEXT:    slti a0, a4, 0
-; RV32-NEXT:    xori a0, a0, 1
+; RV32-NEXT:    srli a4, a4, 31
+; RV32-NEXT:    xori a0, a4, 1
 ; RV32-NEXT:  .LBB21_11: # %entry
 ; RV32-NEXT:    bnez a0, .LBB21_13
 ; RV32-NEXT:  # %bb.12: # %entry
@@ -1643,7 +1643,7 @@ define i64 @ustest_f32i64(float %x) {
 ; RV32-NEXT:    lw a0, 16(sp)
 ; RV32-NEXT:    beqz a1, .LBB23_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a2, a1, 0
+; RV32-NEXT:    srli a2, a1, 31
 ; RV32-NEXT:    j .LBB23_3
 ; RV32-NEXT:  .LBB23_2:
 ; RV32-NEXT:    seqz a2, a0
@@ -1750,7 +1750,7 @@ define i64 @stest_f16i64(half %x) {
 ; RV32-NEXT:    or a7, a2, a4
 ; RV32-NEXT:    beqz a7, .LBB24_4
 ; RV32-NEXT:  .LBB24_3: # %entry
-; RV32-NEXT:    slti a6, a4, 0
+; RV32-NEXT:    srli a6, a4, 31
 ; RV32-NEXT:  .LBB24_4: # %entry
 ; RV32-NEXT:    neg a7, a6
 ; RV32-NEXT:    addi t0, a6, -1
@@ -1772,8 +1772,8 @@ define i64 @stest_f16i64(half %x) {
 ; RV32-NEXT:    li a5, -1
 ; RV32-NEXT:    beq a2, a5, .LBB24_11
 ; RV32-NEXT:  # %bb.10: # %entry
-; RV32-NEXT:    slti a0, a4, 0
-; RV32-NEXT:    xori a0, a0, 1
+; RV32-NEXT:    srli a4, a4, 31
+; RV32-NEXT:    xori a0, a4, 1
 ; RV32-NEXT:  .LBB24_11: # %entry
 ; RV32-NEXT:    bnez a0, .LBB24_13
 ; RV32-NEXT:  # %bb.12: # %entry
@@ -1799,7 +1799,7 @@ define i64 @stest_f16i64(half %x) {
 ; RV64-NEXT:    srli a3, a2, 1
 ; RV64-NEXT:    beqz a1, .LBB24_2
 ; RV64-NEXT:  # %bb.1: # %entry
-; RV64-NEXT:    slti a4, a1, 0
+; RV64-NEXT:    srli a4, a1, 63
 ; RV64-NEXT:    j .LBB24_3
 ; RV64-NEXT:  .LBB24_2:
 ; RV64-NEXT:    sltu a4, a0, a3
@@ -1813,8 +1813,8 @@ define i64 @stest_f16i64(half %x) {
 ; RV64-NEXT:    slli a1, a2, 63
 ; RV64-NEXT:    beq a5, a2, .LBB24_7
 ; RV64-NEXT:  # %bb.6: # %entry
-; RV64-NEXT:    slti a2, a5, 0
-; RV64-NEXT:    xori a2, a2, 1
+; RV64-NEXT:    srli a5, a5, 63
+; RV64-NEXT:    xori a2, a5, 1
 ; RV64-NEXT:    beqz a2, .LBB24_8
 ; RV64-NEXT:    j .LBB24_9
 ; RV64-NEXT:  .LBB24_7:
@@ -1906,7 +1906,7 @@ define i64 @ustest_f16i64(half %x) {
 ; RV32-NEXT:    lw a0, 16(sp)
 ; RV32-NEXT:    beqz a1, .LBB26_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a2, a1, 0
+; RV32-NEXT:    srli a2, a1, 31
 ; RV32-NEXT:    j .LBB26_3
 ; RV32-NEXT:  .LBB26_2:
 ; RV32-NEXT:    seqz a2, a0
@@ -2004,7 +2004,7 @@ define i32 @stest_f64i32_mm(double %x) {
 ; RV32IF-NEXT:    addi a3, a2, -1
 ; RV32IF-NEXT:    beqz a1, .LBB27_2
 ; RV32IF-NEXT:  # %bb.1: # %entry
-; RV32IF-NEXT:    slti a4, a1, 0
+; RV32IF-NEXT:    srli a4, a1, 31
 ; RV32IF-NEXT:    j .LBB27_3
 ; RV32IF-NEXT:  .LBB27_2:
 ; RV32IF-NEXT:    sltu a4, a0, a3
@@ -2018,7 +2018,7 @@ define i32 @stest_f64i32_mm(double %x) {
 ; RV32IF-NEXT:    li a3, -1
 ; RV32IF-NEXT:    beq a1, a3, .LBB27_7
 ; RV32IF-NEXT:  # %bb.6: # %entry
-; RV32IF-NEXT:    slti a1, a1, 0
+; RV32IF-NEXT:    srli a1, a1, 31
 ; RV32IF-NEXT:    xori a1, a1, 1
 ; RV32IF-NEXT:    beqz a1, .LBB27_8
 ; RV32IF-NEXT:    j .LBB27_9
@@ -2171,7 +2171,7 @@ define i32 @ustest_f64i32_mm(double %x) {
 ; RV32IF-NEXT:    neg a2, a2
 ; RV32IF-NEXT:    or a0, a3, a0
 ; RV32IF-NEXT:    and a1, a2, a1
-; RV32IF-NEXT:    slti a1, a1, 0
+; RV32IF-NEXT:    srli a1, a1, 31
 ; RV32IF-NEXT:    addi a1, a1, -1
 ; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -2337,7 +2337,7 @@ define i32 @stest_f16i32_mm(half %x) {
 ; RV32-NEXT:    addi a3, a2, -1
 ; RV32-NEXT:    beqz a1, .LBB33_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a4, a1, 0
+; RV32-NEXT:    srli a4, a1, 31
 ; RV32-NEXT:    j .LBB33_3
 ; RV32-NEXT:  .LBB33_2:
 ; RV32-NEXT:    sltu a4, a0, a3
@@ -2351,7 +2351,7 @@ define i32 @stest_f16i32_mm(half %x) {
 ; RV32-NEXT:    li a3, -1
 ; RV32-NEXT:    beq a1, a3, .LBB33_7
 ; RV32-NEXT:  # %bb.6: # %entry
-; RV32-NEXT:    slti a1, a1, 0
+; RV32-NEXT:    srli a1, a1, 31
 ; RV32-NEXT:    xori a1, a1, 1
 ; RV32-NEXT:    beqz a1, .LBB33_8
 ; RV32-NEXT:    j .LBB33_9
@@ -2462,7 +2462,7 @@ define i32 @ustest_f16i32_mm(half %x) {
 ; RV32-NEXT:    neg a2, a2
 ; RV32-NEXT:    or a0, a3, a0
 ; RV32-NEXT:    and a1, a2, a1
-; RV32-NEXT:    slti a1, a1, 0
+; RV32-NEXT:    srli a1, a1, 31
 ; RV32-NEXT:    addi a1, a1, -1
 ; RV32-NEXT:    and a0, a1, a0
 ; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
@@ -3044,7 +3044,7 @@ define i64 @stest_f64i64_mm(double %x) {
 ; RV32IF-NEXT:    or a7, a2, a4
 ; RV32IF-NEXT:    beqz a7, .LBB45_4
 ; RV32IF-NEXT:  .LBB45_3: # %entry
-; RV32IF-NEXT:    slti a6, a4, 0
+; RV32IF-NEXT:    srli a6, a4, 31
 ; RV32IF-NEXT:  .LBB45_4: # %entry
 ; RV32IF-NEXT:    neg a7, a6
 ; RV32IF-NEXT:    addi t0, a6, -1
@@ -3066,8 +3066,8 @@ define i64 @stest_f64i64_mm(double %x) {
 ; RV32IF-NEXT:    li a5, -1
 ; RV32IF-NEXT:    beq a2, a5, .LBB45_11
 ; RV32IF-NEXT:  # %bb.10: # %entry
-; RV32IF-NEXT:    slti a0, a4, 0
-; RV32IF-NEXT:    xori a0, a0, 1
+; RV32IF-NEXT:    srli a4, a4, 31
+; RV32IF-NEXT:    xori a0, a4, 1
 ; RV32IF-NEXT:  .LBB45_11: # %entry
 ; RV32IF-NEXT:    bnez a0, .LBB45_13
 ; RV32IF-NEXT:  # %bb.12: # %entry
@@ -3092,7 +3092,7 @@ define i64 @stest_f64i64_mm(double %x) {
 ; RV64IF-NEXT:    srli a3, a2, 1
 ; RV64IF-NEXT:    beqz a1, .LBB45_2
 ; RV64IF-NEXT:  # %bb.1: # %entry
-; RV64IF-NEXT:    slti a4, a1, 0
+; RV64IF-NEXT:    srli a4, a1, 63
 ; RV64IF-NEXT:    j .LBB45_3
 ; RV64IF-NEXT:  .LBB45_2:
 ; RV64IF-NEXT:    sltu a4, a0, a3
@@ -3106,8 +3106,8 @@ define i64 @stest_f64i64_mm(double %x) {
 ; RV64IF-NEXT:    slli a1, a2, 63
 ; RV64IF-NEXT:    beq a5, a2, .LBB45_7
 ; RV64IF-NEXT:  # %bb.6: # %entry
-; RV64IF-NEXT:    slti a2, a5, 0
-; RV64IF-NEXT:    xori a2, a2, 1
+; RV64IF-NEXT:    srli a5, a5, 63
+; RV64IF-NEXT:    xori a2, a5, 1
 ; RV64IF-NEXT:    beqz a2, .LBB45_8
 ; RV64IF-NEXT:    j .LBB45_9
 ; RV64IF-NEXT:  .LBB45_7:
@@ -3147,7 +3147,7 @@ define i64 @stest_f64i64_mm(double %x) {
 ; RV32IFD-NEXT:    or a7, a2, a4
 ; RV32IFD-NEXT:    beqz a7, .LBB45_4
 ; RV32IFD-NEXT:  .LBB45_3: # %entry
-; RV32IFD-NEXT:    slti a6, a4, 0
+; RV32IFD-NEXT:    srli a6, a4, 31
 ; RV32IFD-NEXT:  .LBB45_4: # %entry
 ; RV32IFD-NEXT:    neg a7, a6
 ; RV32IFD-NEXT:    addi t0, a6, -1
@@ -3169,8 +3169,8 @@ define i64 @stest_f64i64_mm(double %x) {
 ; RV32IFD-NEXT:    li a5, -1
 ; RV32IFD-NEXT:    beq a2, a5, .LBB45_11
 ; RV32IFD-NEXT:  # %bb.10: # %entry
-; RV32IFD-NEXT:    slti a0, a4, 0
-; RV32IFD-NEXT:    xori a0, a0, 1
+; RV32IFD-NEXT:    srli a4, a4, 31
+; RV32IFD-NEXT:    xori a0, a4, 1
 ; RV32IFD-NEXT:  .LBB45_11: # %entry
 ; RV32IFD-NEXT:    bnez a0, .LBB45_13
 ; RV32IFD-NEXT:  # %bb.12: # %entry
@@ -3298,7 +3298,7 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; RV32IF-NEXT:    lw a3, 16(sp)
 ; RV32IF-NEXT:    beqz a2, .LBB47_2
 ; RV32IF-NEXT:  # %bb.1: # %entry
-; RV32IF-NEXT:    slti a4, a2, 0
+; RV32IF-NEXT:    srli a4, a2, 31
 ; RV32IF-NEXT:    j .LBB47_3
 ; RV32IF-NEXT:  .LBB47_2:
 ; RV32IF-NEXT:    seqz a4, a3
@@ -3312,7 +3312,7 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; RV32IF-NEXT:    and a1, a3, a1
 ; RV32IF-NEXT:    and a0, a3, a0
 ; RV32IF-NEXT:    and a2, a3, a2
-; RV32IF-NEXT:    slti a2, a2, 0
+; RV32IF-NEXT:    srli a2, a2, 31
 ; RV32IF-NEXT:    addi a2, a2, -1
 ; RV32IF-NEXT:    and a0, a2, a0
 ; RV32IF-NEXT:    and a1, a2, a1
@@ -3335,7 +3335,7 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; RV64-NEXT:    li a2, 1
 ; RV64-NEXT:  .LBB47_2: # %entry
 ; RV64-NEXT:    slti a1, a1, 1
-; RV64-NEXT:    slti a2, a2, 0
+; RV64-NEXT:    srli a2, a2, 63
 ; RV64-NEXT:    neg a1, a1
 ; RV64-NEXT:    and a0, a1, a0
 ; RV64-NEXT:    addi a2, a2, -1
@@ -3360,7 +3360,7 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; RV32IFD-NEXT:    lw a3, 16(sp)
 ; RV32IFD-NEXT:    beqz a2, .LBB47_2
 ; RV32IFD-NEXT:  # %bb.1: # %entry
-; RV32IFD-NEXT:    slti a4, a2, 0
+; RV32IFD-NEXT:    srli a4, a2, 31
 ; RV32IFD-NEXT:    j .LBB47_3
 ; RV32IFD-NEXT:  .LBB47_2:
 ; RV32IFD-NEXT:    seqz a4, a3
@@ -3374,7 +3374,7 @@ define i64 @ustest_f64i64_mm(double %x) {
 ; RV32IFD-NEXT:    and a1, a3, a1
 ; RV32IFD-NEXT:    and a0, a3, a0
 ; RV32IFD-NEXT:    and a2, a3, a2
-; RV32IFD-NEXT:    slti a2, a2, 0
+; RV32IFD-NEXT:    srli a2, a2, 31
 ; RV32IFD-NEXT:    addi a2, a2, -1
 ; RV32IFD-NEXT:    and a0, a2, a0
 ; RV32IFD-NEXT:    and a1, a2, a1
@@ -3417,7 +3417,7 @@ define i64 @stest_f32i64_mm(float %x) {
 ; RV32-NEXT:    or a7, a2, a4
 ; RV32-NEXT:    beqz a7, .LBB48_4
 ; RV32-NEXT:  .LBB48_3: # %entry
-; RV32-NEXT:    slti a6, a4, 0
+; RV32-NEXT:    srli a6, a4, 31
 ; RV32-NEXT:  .LBB48_4: # %entry
 ; RV32-NEXT:    neg a7, a6
 ; RV32-NEXT:    addi t0, a6, -1
@@ -3439,8 +3439,8 @@ define i64 @stest_f32i64_mm(float %x) {
 ; RV32-NEXT:    li a5, -1
 ; RV32-NEXT:    beq a2, a5, .LBB48_11
 ; RV32-NEXT:  # %bb.10: # %entry
-; RV32-NEXT:    slti a0, a4, 0
-; RV32-NEXT:    xori a0, a0, 1
+; RV32-NEXT:    srli a4, a4, 31
+; RV32-NEXT:    xori a0, a4, 1
 ; RV32-NEXT:  .LBB48_11: # %entry
 ; RV32-NEXT:    bnez a0, .LBB48_13
 ; RV32-NEXT:  # %bb.12: # %entry
@@ -3536,7 +3536,7 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; RV32-NEXT:    lw a3, 16(sp)
 ; RV32-NEXT:    beqz a2, .LBB50_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a4, a2, 0
+; RV32-NEXT:    srli a4, a2, 31
 ; RV32-NEXT:    j .LBB50_3
 ; RV32-NEXT:  .LBB50_2:
 ; RV32-NEXT:    seqz a4, a3
@@ -3550,7 +3550,7 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; RV32-NEXT:    and a1, a3, a1
 ; RV32-NEXT:    and a0, a3, a0
 ; RV32-NEXT:    and a2, a3, a2
-; RV32-NEXT:    slti a2, a2, 0
+; RV32-NEXT:    srli a2, a2, 31
 ; RV32-NEXT:    addi a2, a2, -1
 ; RV32-NEXT:    and a0, a2, a0
 ; RV32-NEXT:    and a1, a2, a1
@@ -3573,7 +3573,7 @@ define i64 @ustest_f32i64_mm(float %x) {
 ; RV64-NEXT:    li a2, 1
 ; RV64-NEXT:  .LBB50_2: # %entry
 ; RV64-NEXT:    slti a1, a1, 1
-; RV64-NEXT:    slti a2, a2, 0
+; RV64-NEXT:    srli a2, a2, 63
 ; RV64-NEXT:    neg a1, a1
 ; RV64-NEXT:    and a0, a1, a0
 ; RV64-NEXT:    addi a2, a2, -1
@@ -3618,7 +3618,7 @@ define i64 @stest_f16i64_mm(half %x) {
 ; RV32-NEXT:    or a7, a2, a4
 ; RV32-NEXT:    beqz a7, .LBB51_4
 ; RV32-NEXT:  .LBB51_3: # %entry
-; RV32-NEXT:    slti a6, a4, 0
+; RV32-NEXT:    srli a6, a4, 31
 ; RV32-NEXT:  .LBB51_4: # %entry
 ; RV32-NEXT:    neg a7, a6
 ; RV32-NEXT:    addi t0, a6, -1
@@ -3640,8 +3640,8 @@ define i64 @stest_f16i64_mm(half %x) {
 ; RV32-NEXT:    li a5, -1
 ; RV32-NEXT:    beq a2, a5, .LBB51_11
 ; RV32-NEXT:  # %bb.10: # %entry
-; RV32-NEXT:    slti a0, a4, 0
-; RV32-NEXT:    xori a0, a0, 1
+; RV32-NEXT:    srli a4, a4, 31
+; RV32-NEXT:    xori a0, a4, 1
 ; RV32-NEXT:  .LBB51_11: # %entry
 ; RV32-NEXT:    bnez a0, .LBB51_13
 ; RV32-NEXT:  # %bb.12: # %entry
@@ -3667,7 +3667,7 @@ define i64 @stest_f16i64_mm(half %x) {
 ; RV64-NEXT:    srli a3, a2, 1
 ; RV64-NEXT:    beqz a1, .LBB51_2
 ; RV64-NEXT:  # %bb.1: # %entry
-; RV64-NEXT:    slti a4, a1, 0
+; RV64-NEXT:    srli a4, a1, 63
 ; RV64-NEXT:    j .LBB51_3
 ; RV64-NEXT:  .LBB51_2:
 ; RV64-NEXT:    sltu a4, a0, a3
@@ -3681,8 +3681,8 @@ define i64 @stest_f16i64_mm(half %x) {
 ; RV64-NEXT:    slli a1, a2, 63
 ; RV64-NEXT:    beq a5, a2, .LBB51_7
 ; RV64-NEXT:  # %bb.6: # %entry
-; RV64-NEXT:    slti a2, a5, 0
-; RV64-NEXT:    xori a2, a2, 1
+; RV64-NEXT:    srli a5, a5, 63
+; RV64-NEXT:    xori a2, a5, 1
 ; RV64-NEXT:    beqz a2, .LBB51_8
 ; RV64-NEXT:    j .LBB51_9
 ; RV64-NEXT:  .LBB51_7:
@@ -3773,7 +3773,7 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; RV32-NEXT:    lw a3, 16(sp)
 ; RV32-NEXT:    beqz a2, .LBB53_2
 ; RV32-NEXT:  # %bb.1: # %entry
-; RV32-NEXT:    slti a4, a2, 0
+; RV32-NEXT:    srli a4, a2, 31
 ; RV32-NEXT:    j .LBB53_3
 ; RV32-NEXT:  .LBB53_2:
 ; RV32-NEXT:    seqz a4, a3
@@ -3787,7 +3787,7 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; RV32-NEXT:    and a1, a3, a1
 ; RV32-NEXT:    and a0, a3, a0
 ; RV32-NEXT:    and a2, a3, a2
-; RV32-NEXT:    slti a2, a2, 0
+; RV32-NEXT:    srli a2, a2, 31
 ; RV32-NEXT:    addi a2, a2, -1
 ; RV32-NEXT:    and a0, a2, a0
 ; RV32-NEXT:    and a1, a2, a1
@@ -3811,7 +3811,7 @@ define i64 @ustest_f16i64_mm(half %x) {
 ; RV64-NEXT:    li a2, 1
 ; RV64-NEXT:  .LBB53_2: # %entry
 ; RV64-NEXT:    slti a1, a1, 1
-; RV64-NEXT:    slti a2, a2, 0
+; RV64-NEXT:    srli a2, a2, 63
 ; RV64-NEXT:    neg a1, a1
 ; RV64-NEXT:    and a0, a1, a0
 ; RV64-NEXT:    addi a2, a2, -1
diff --git a/llvm/test/CodeGen/RISCV/get-register-reserve.ll b/llvm/test/CodeGen/RISCV/get-register-reserve.ll
index 87acd70ec62f2..7549b4dd3f682 100644
--- a/llvm/test/CodeGen/RISCV/get-register-reserve.ll
+++ b/llvm/test/CodeGen/RISCV/get-register-reserve.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: not --crash llc < %s -mtriple=riscv32 -mattr +reserve-x8 2>&1 \
+; RUN: not llc < %s -mtriple=riscv32 -mattr +reserve-x8 2>&1 \
 ; RUN:   | FileCheck -check-prefix=NO-RESERVE-A1 %s
-; RUN: not --crash llc < %s -mtriple=riscv32 -mattr +reserve-x11 2>&1 \
+; RUN: not llc < %s -mtriple=riscv32 -mattr +reserve-x11 2>&1 \
 ; RUN:   | FileCheck -check-prefix=NO-RESERVE-FP %s
 ; RUN: llc < %s -mtriple=riscv32 -mattr +reserve-x8 -mattr +reserve-x11 \
 ; RUN:   | FileCheck -check-prefix=RESERVE %s
diff --git a/llvm/test/CodeGen/RISCV/half-convert-strict.ll b/llvm/test/CodeGen/RISCV/half-convert-strict.ll
index 675e230816f37..a607893a3735b 100644
--- a/llvm/test/CodeGen/RISCV/half-convert-strict.ll
+++ b/llvm/test/CodeGen/RISCV/half-convert-strict.ll
@@ -1,52 +1,56 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+zfh -verify-machineinstrs \
 ; RUN:   -target-abi ilp32f -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECKIZFH,RV32IZFH %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZFH,CHECK32-IZFH,RV32IZFH %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zfh -verify-machineinstrs \
 ; RUN:   -target-abi lp64f -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECKIZFH,RV64IZFH %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZFH,CHECK64-IZFH,RV64IZFH %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zhinx -verify-machineinstrs \
 ; RUN:   -target-abi ilp32 -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECKIZHINX,RV32IZHINX %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZHINX,CHECK32-IZHINX,RV32IZHINX %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zhinx -verify-machineinstrs \
 ; RUN:   -target-abi lp64 -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECKIZHINX,RV64IZHINX %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZHINX,CHECK64-IZHINX,RV64IZHINX %s
+
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfh -verify-machineinstrs \
 ; RUN:   -target-abi ilp32d -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefix=RV32IDZFH %s
+; RUN:   | FileCheck -check-prefixes=CHECKIDZFH,CHECK32-IZFH %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh -verify-machineinstrs \
 ; RUN:   -target-abi lp64d -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefix=RV64IDZFH %s
+; RUN:   | FileCheck -check-prefixes=CHECKIDZFH,CHECK64-IZFH %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zdinx,+zhinx -verify-machineinstrs \
 ; RUN:   -target-abi ilp32 -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefix=RV32IZDINXZHINX %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZDINXZHINX,CHECK32-IZHINX %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zdinx,+zhinx -verify-machineinstrs \
 ; RUN:   -target-abi lp64 -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefix=RV64IZDINXZHINX %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZDINXZHINX,CHECK64-IZHINX %s
+
 ; RUN: llc -mtriple=riscv32 -mattr=+zfhmin -verify-machineinstrs \
 ; RUN:   -target-abi ilp32f -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK32-IZFHMIN,RV32IFZFHMIN %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZFHMIN,CHECK32-IZFHMIN,RV32IFZFHMIN %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zfhmin -verify-machineinstrs \
 ; RUN:   -target-abi lp64f -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK64-IZFHMIN,RV64IFZFHMIN %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZFHMIN,CHECK64-IZFHMIN,RV64IFZFHMIN %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zhinxmin -verify-machineinstrs \
 ; RUN:   -target-abi ilp32 -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK32-IZHINXMIN %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZHINXMIN,CHECK32-IZHINXMIN,RV32IZHINXMIN %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zhinxmin -verify-machineinstrs \
 ; RUN:   -target-abi lp64 -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK64-IZHINXMIN %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZHINXMIN,CHECK64-IZHINXMIN,RV64IZHINXMIN %s
+
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+zfhmin -verify-machineinstrs \
 ; RUN:   -target-abi ilp32d -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK32-IZFHMIN,RV32IDZFHMIN %s
+; RUN:   | FileCheck -check-prefixes=CHECKIDZFHMIN,CHECK32-IZFHMIN %s
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+zfhmin -verify-machineinstrs \
 ; RUN:   -target-abi lp64d -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK64-IZFHMIN,RV64IDZFHMIN %s
+; RUN:   | FileCheck -check-prefixes=CHECKIDZFHMIN,CHECK64-IZFHMIN %s
 ; RUN: llc -mtriple=riscv32 -mattr=+zdinx,+zhinxmin -verify-machineinstrs \
 ; RUN:   -target-abi ilp32 -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK32-IZDINXZHINXMIN %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZDINXZHINXMIN,CHECK32-IZHINXMIN %s
 ; RUN: llc -mtriple=riscv64 -mattr=+zdinx,+zhinxmin -verify-machineinstrs \
 ; RUN:   -target-abi lp64 -disable-strictnode-mutation < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK64-IZDINXZHINXMIN %s
+; RUN:   | FileCheck -check-prefixes=CHECKIZDINXZHINXMIN,CHECK64-IZHINXMIN %s
+
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs \
 ; RUN:   -target-abi ilp32d -disable-strictnode-mutation < %s \
 ; RUN:   | FileCheck -check-prefixes=CHECK32-D %s
@@ -56,45 +60,25 @@
 ; support rounding mode.
 
 define i16 @fcvt_si_h(half %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_si_h:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    fcvt.w.h a0, fa0, rtz
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_si_h:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_si_h:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    fcvt.w.h a0, a0, rtz
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_si_h:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    fcvt.l.h a0, a0, rtz
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_si_h:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.w.h a0, fa0, rtz
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_si_h:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.l.h a0, fa0, rtz
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_si_h:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.w.h a0, a0, rtz
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_si_h:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.l.h a0, a0, rtz
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_si_h:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    fcvt.w.h a0, fa0, rtz
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_si_h:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_si_h:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    fcvt.w.h a0, a0, rtz
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_si_h:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    fcvt.l.h a0, a0, rtz
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_si_h:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -120,18 +104,6 @@ define i16 @fcvt_si_h(half %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_si_h:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.w.s a0, a0, rtz
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_si_h:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_si_h:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -151,45 +123,25 @@ define i16 @fcvt_si_h(half %a) nounwind strictfp {
 declare i16 @llvm.experimental.constrained.fptosi.i16.f16(half, metadata)
 
 define i16 @fcvt_ui_h(half %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_ui_h:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    fcvt.w.h a0, fa0, rtz
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_ui_h:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_ui_h:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    fcvt.w.h a0, a0, rtz
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_ui_h:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    fcvt.l.h a0, a0, rtz
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_ui_h:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.w.h a0, fa0, rtz
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_ui_h:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.l.h a0, fa0, rtz
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_ui_h:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.w.h a0, a0, rtz
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_ui_h:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.l.h a0, a0, rtz
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_ui_h:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    fcvt.w.h a0, fa0, rtz
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_ui_h:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_ui_h:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    fcvt.w.h a0, a0, rtz
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_ui_h:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    fcvt.l.h a0, a0, rtz
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_ui_h:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -215,18 +167,6 @@ define i16 @fcvt_ui_h(half %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_ui_h:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.w.s a0, a0, rtz
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_ui_h:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_ui_h:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -256,61 +196,39 @@ define i32 @fcvt_w_h(half %a) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.w.h a0, a0, rtz
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_w_h:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.w.h a0, fa0, rtz
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_w_h:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.w.h a0, fa0, rtz
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_w_h:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.w.h a0, a0, rtz
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_w_h:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.w.h a0, a0, rtz
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_w_h:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK32-IZFHMIN-NEXT:    fcvt.w.s a0, fa5, rtz
-; CHECK32-IZFHMIN-NEXT:    ret
-;
-; CHECK64-IZFHMIN-LABEL: fcvt_w_h:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK64-IZFHMIN-NEXT:    fcvt.w.s a0, fa5, rtz
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_w_h:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZHINXMIN-NEXT:    fcvt.w.s a0, a0, rtz
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_w_h:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fcvt.w.s a0, a0, rtz
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_w_h:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.w.s a0, a0, rtz
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_w_h:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.w.s a0, a0, rtz
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_w_h:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.w.h a0, fa0, rtz
+; CHECKIDZFH-NEXT:    ret
+;
+; CHECKIZDINXZHINX-LABEL: fcvt_w_h:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.w.h a0, a0, rtz
+; CHECKIZDINXZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: fcvt_w_h:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; CHECKIZFHMIN-NEXT:    fcvt.w.s a0, fa5, rtz
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: fcvt_w_h:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECKIZHINXMIN-NEXT:    fcvt.w.s a0, a0, rtz
+; CHECKIZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_w_h:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; CHECKIDZFHMIN-NEXT:    fcvt.w.s a0, fa5, rtz
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_w_h:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.w.s a0, a0, rtz
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_w_h:
 ; CHECK32-D:       # %bb.0:
@@ -341,61 +259,39 @@ define i32 @fcvt_wu_h(half %a) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.wu.h a0, a0, rtz
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_wu_h:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_wu_h:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_wu_h:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.wu.h a0, a0, rtz
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_wu_h:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.wu.h a0, a0, rtz
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_wu_h:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK32-IZFHMIN-NEXT:    fcvt.wu.s a0, fa5, rtz
-; CHECK32-IZFHMIN-NEXT:    ret
-;
-; CHECK64-IZFHMIN-LABEL: fcvt_wu_h:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK64-IZFHMIN-NEXT:    fcvt.wu.s a0, fa5, rtz
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_wu_h:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_wu_h:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_wu_h:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_wu_h:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_wu_h:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
+; CHECKIDZFH-NEXT:    ret
+;
+; CHECKIZDINXZHINX-LABEL: fcvt_wu_h:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.wu.h a0, a0, rtz
+; CHECKIZDINXZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: fcvt_wu_h:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; CHECKIZFHMIN-NEXT:    fcvt.wu.s a0, fa5, rtz
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: fcvt_wu_h:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECKIZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
+; CHECKIZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_wu_h:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; CHECKIDZFHMIN-NEXT:    fcvt.wu.s a0, fa5, rtz
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_wu_h:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_wu_h:
 ; CHECK32-D:       # %bb.0:
@@ -433,81 +329,51 @@ define i32 @fcvt_wu_h_multiple_use(half %x, ptr %y) strictfp {
 ; CHECKIZHINX-NEXT:    add a0, a0, a1
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_wu_h_multiple_use:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
-; RV32IDZFH-NEXT:    seqz a1, a0
-; RV32IDZFH-NEXT:    add a0, a0, a1
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_wu_h_multiple_use:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
-; RV64IDZFH-NEXT:    seqz a1, a0
-; RV64IDZFH-NEXT:    add a0, a0, a1
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_wu_h_multiple_use:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.wu.h a0, a0, rtz
-; RV32IZDINXZHINX-NEXT:    seqz a1, a0
-; RV32IZDINXZHINX-NEXT:    add a0, a0, a1
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_wu_h_multiple_use:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.wu.h a0, a0, rtz
-; RV64IZDINXZHINX-NEXT:    seqz a1, a0
-; RV64IZDINXZHINX-NEXT:    add a0, a0, a1
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_wu_h_multiple_use:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK32-IZFHMIN-NEXT:    fcvt.wu.s a0, fa5, rtz
-; CHECK32-IZFHMIN-NEXT:    seqz a1, a0
-; CHECK32-IZFHMIN-NEXT:    add a0, a0, a1
-; CHECK32-IZFHMIN-NEXT:    ret
-;
-; CHECK64-IZFHMIN-LABEL: fcvt_wu_h_multiple_use:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
-; CHECK64-IZFHMIN-NEXT:    fcvt.wu.s a0, fa5, rtz
-; CHECK64-IZFHMIN-NEXT:    seqz a1, a0
-; CHECK64-IZFHMIN-NEXT:    add a0, a0, a1
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_wu_h_multiple_use:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
-; CHECK32-IZHINXMIN-NEXT:    seqz a1, a0
-; CHECK32-IZHINXMIN-NEXT:    add a0, a0, a1
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_wu_h_multiple_use:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
-; CHECK64-IZHINXMIN-NEXT:    seqz a1, a0
-; CHECK64-IZHINXMIN-NEXT:    add a0, a0, a1
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_wu_h_multiple_use:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
-; CHECK32-IZDINXZHINXMIN-NEXT:    seqz a1, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    add a0, a0, a1
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_wu_h_multiple_use:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
-; CHECK64-IZDINXZHINXMIN-NEXT:    seqz a1, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    add a0, a0, a1
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_wu_h_multiple_use:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.wu.h a0, fa0, rtz
+; CHECKIDZFH-NEXT:    seqz a1, a0
+; CHECKIDZFH-NEXT:    add a0, a0, a1
+; CHECKIDZFH-NEXT:    ret
+;
+; CHECKIZDINXZHINX-LABEL: fcvt_wu_h_multiple_use:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.wu.h a0, a0, rtz
+; CHECKIZDINXZHINX-NEXT:    seqz a1, a0
+; CHECKIZDINXZHINX-NEXT:    add a0, a0, a1
+; CHECKIZDINXZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: fcvt_wu_h_multiple_use:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; CHECKIZFHMIN-NEXT:    fcvt.wu.s a0, fa5, rtz
+; CHECKIZFHMIN-NEXT:    seqz a1, a0
+; CHECKIZFHMIN-NEXT:    add a0, a0, a1
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: fcvt_wu_h_multiple_use:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECKIZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
+; CHECKIZHINXMIN-NEXT:    seqz a1, a0
+; CHECKIZHINXMIN-NEXT:    add a0, a0, a1
+; CHECKIZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_wu_h_multiple_use:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.s.h fa5, fa0
+; CHECKIDZFHMIN-NEXT:    fcvt.wu.s a0, fa5, rtz
+; CHECKIDZFHMIN-NEXT:    seqz a1, a0
+; CHECKIDZFHMIN-NEXT:    add a0, a0, a1
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_wu_h_multiple_use:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.wu.s a0, a0, rtz
+; CHECKIZDINXZHINXMIN-NEXT:    seqz a1, a0
+; CHECKIZDINXZHINXMIN-NEXT:    add a0, a0, a1
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_wu_h_multiple_use:
 ; CHECK32-D:       # %bb.0:
@@ -535,61 +401,33 @@ define i32 @fcvt_wu_h_multiple_use(half %x, ptr %y) strictfp {
 }
 
 define i64 @fcvt_l_h(half %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_l_h:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    addi sp, sp, -16
-; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZFH-NEXT:    call __fixhfdi
-; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_l_h:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_l_h:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    addi sp, sp, -16
-; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZHINX-NEXT:    call __fixhfdi
-; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_l_h:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    fcvt.l.h a0, a0, rtz
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_l_h:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    addi sp, sp, -16
-; RV32IDZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IDZFH-NEXT:    call __fixhfdi
-; RV32IDZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IDZFH-NEXT:    addi sp, sp, 16
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_l_h:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.l.h a0, fa0, rtz
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_l_h:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, -16
-; RV32IZDINXZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZDINXZHINX-NEXT:    call __fixhfdi
-; RV32IZDINXZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, 16
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_l_h:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.l.h a0, a0, rtz
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_l_h:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    addi sp, sp, -16
+; CHECK32-IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-IZFH-NEXT:    call __fixhfdi
+; CHECK32-IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-IZFH-NEXT:    addi sp, sp, 16
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_l_h:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    fcvt.l.h a0, fa0, rtz
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_l_h:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    addi sp, sp, -16
+; CHECK32-IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-IZHINX-NEXT:    call __fixhfdi
+; CHECK32-IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-IZHINX-NEXT:    addi sp, sp, 16
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_l_h:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    fcvt.l.h a0, a0, rtz
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_l_h:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -621,21 +459,6 @@ define i64 @fcvt_l_h(half %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_l_h:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK32-IZDINXZHINXMIN-NEXT:    call __fixhfdi
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_l_h:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_l_h:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -655,61 +478,33 @@ define i64 @fcvt_l_h(half %a) nounwind strictfp {
 declare i64 @llvm.experimental.constrained.fptosi.i64.f16(half, metadata)
 
 define i64 @fcvt_lu_h(half %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_lu_h:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    addi sp, sp, -16
-; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZFH-NEXT:    call __fixunshfdi
-; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_lu_h:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_lu_h:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    addi sp, sp, -16
-; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZHINX-NEXT:    call __fixunshfdi
-; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_lu_h:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    fcvt.lu.h a0, a0, rtz
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_lu_h:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    addi sp, sp, -16
-; RV32IDZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IDZFH-NEXT:    call __fixunshfdi
-; RV32IDZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IDZFH-NEXT:    addi sp, sp, 16
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_lu_h:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_lu_h:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, -16
-; RV32IZDINXZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZDINXZHINX-NEXT:    call __fixunshfdi
-; RV32IZDINXZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, 16
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_lu_h:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.lu.h a0, a0, rtz
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_lu_h:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    addi sp, sp, -16
+; CHECK32-IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-IZFH-NEXT:    call __fixunshfdi
+; CHECK32-IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-IZFH-NEXT:    addi sp, sp, 16
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_lu_h:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    fcvt.lu.h a0, fa0, rtz
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_lu_h:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    addi sp, sp, -16
+; CHECK32-IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-IZHINX-NEXT:    call __fixunshfdi
+; CHECK32-IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-IZHINX-NEXT:    addi sp, sp, 16
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_lu_h:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    fcvt.lu.h a0, a0, rtz
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_lu_h:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -741,21 +536,6 @@ define i64 @fcvt_lu_h(half %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.lu.s a0, a0, rtz
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_lu_h:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK32-IZDINXZHINXMIN-NEXT:    call __fixunshfdi
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_lu_h:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.lu.s a0, a0, rtz
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_lu_h:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -775,61 +555,33 @@ define i64 @fcvt_lu_h(half %a) nounwind strictfp {
 declare i64 @llvm.experimental.constrained.fptoui.i64.f16(half, metadata)
 
 define half @fcvt_h_si(i16 %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_h_si:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    slli a0, a0, 16
-; RV32IZFH-NEXT:    srai a0, a0, 16
-; RV32IZFH-NEXT:    fcvt.h.w fa0, a0
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_h_si:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    slli a0, a0, 48
-; RV64IZFH-NEXT:    srai a0, a0, 48
-; RV64IZFH-NEXT:    fcvt.h.w fa0, a0
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_h_si:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    slli a0, a0, 16
-; RV32IZHINX-NEXT:    srai a0, a0, 16
-; RV32IZHINX-NEXT:    fcvt.h.w a0, a0
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_h_si:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    slli a0, a0, 48
-; RV64IZHINX-NEXT:    srai a0, a0, 48
-; RV64IZHINX-NEXT:    fcvt.h.w a0, a0
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_h_si:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    slli a0, a0, 16
-; RV32IDZFH-NEXT:    srai a0, a0, 16
-; RV32IDZFH-NEXT:    fcvt.h.w fa0, a0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_si:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    slli a0, a0, 48
-; RV64IDZFH-NEXT:    srai a0, a0, 48
-; RV64IDZFH-NEXT:    fcvt.h.w fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_si:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    slli a0, a0, 16
-; RV32IZDINXZHINX-NEXT:    srai a0, a0, 16
-; RV32IZDINXZHINX-NEXT:    fcvt.h.w a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_si:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    slli a0, a0, 48
-; RV64IZDINXZHINX-NEXT:    srai a0, a0, 48
-; RV64IZDINXZHINX-NEXT:    fcvt.h.w a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_h_si:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    slli a0, a0, 16
+; CHECK32-IZFH-NEXT:    srai a0, a0, 16
+; CHECK32-IZFH-NEXT:    fcvt.h.w fa0, a0
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_h_si:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    slli a0, a0, 48
+; CHECK64-IZFH-NEXT:    srai a0, a0, 48
+; CHECK64-IZFH-NEXT:    fcvt.h.w fa0, a0
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_h_si:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    slli a0, a0, 16
+; CHECK32-IZHINX-NEXT:    srai a0, a0, 16
+; CHECK32-IZHINX-NEXT:    fcvt.h.w a0, a0
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_h_si:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    slli a0, a0, 48
+; CHECK64-IZHINX-NEXT:    srai a0, a0, 48
+; CHECK64-IZHINX-NEXT:    fcvt.h.w a0, a0
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_h_si:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -863,22 +615,6 @@ define half @fcvt_h_si(i16 %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_si:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    slli a0, a0, 16
-; CHECK32-IZDINXZHINXMIN-NEXT:    srai a0, a0, 16
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_si:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    slli a0, a0, 48
-; CHECK64-IZDINXZHINXMIN-NEXT:    srai a0, a0, 48
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_h_si:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -910,61 +646,39 @@ define half @fcvt_h_si_signext(i16 signext %a) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.h.w a0, a0
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_h_si_signext:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.h.w fa0, a0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_si_signext:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.h.w fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_si_signext:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.h.w a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_si_signext:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.h.w a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_h_si_signext:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.w fa5, a0
-; CHECK32-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK32-IZFHMIN-NEXT:    ret
-;
-; CHECK64-IZFHMIN-LABEL: fcvt_h_si_signext:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.w fa5, a0
-; CHECK64-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_h_si_signext:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK32-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_h_si_signext:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_si_signext:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_si_signext:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_h_si_signext:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.h.w fa0, a0
+; CHECKIDZFH-NEXT:    ret
+;
+; CHECKIZDINXZHINX-LABEL: fcvt_h_si_signext:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.h.w a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: fcvt_h_si_signext:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    fcvt.s.w fa5, a0
+; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: fcvt_h_si_signext:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    fcvt.s.w a0, a0
+; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_h_si_signext:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.s.w fa5, a0
+; CHECKIDZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_h_si_signext:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_h_si_signext:
 ; CHECK32-D:       # %bb.0:
@@ -984,61 +698,33 @@ define half @fcvt_h_si_signext(i16 signext %a) nounwind strictfp {
 }
 
 define half @fcvt_h_ui(i16 %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_h_ui:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    slli a0, a0, 16
-; RV32IZFH-NEXT:    srli a0, a0, 16
-; RV32IZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_h_ui:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    slli a0, a0, 48
-; RV64IZFH-NEXT:    srli a0, a0, 48
-; RV64IZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_h_ui:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    slli a0, a0, 16
-; RV32IZHINX-NEXT:    srli a0, a0, 16
-; RV32IZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_h_ui:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    slli a0, a0, 48
-; RV64IZHINX-NEXT:    srli a0, a0, 48
-; RV64IZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_h_ui:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    slli a0, a0, 16
-; RV32IDZFH-NEXT:    srli a0, a0, 16
-; RV32IDZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_ui:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    slli a0, a0, 48
-; RV64IDZFH-NEXT:    srli a0, a0, 48
-; RV64IDZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_ui:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    slli a0, a0, 16
-; RV32IZDINXZHINX-NEXT:    srli a0, a0, 16
-; RV32IZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_ui:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    slli a0, a0, 48
-; RV64IZDINXZHINX-NEXT:    srli a0, a0, 48
-; RV64IZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_h_ui:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    slli a0, a0, 16
+; CHECK32-IZFH-NEXT:    srli a0, a0, 16
+; CHECK32-IZFH-NEXT:    fcvt.h.wu fa0, a0
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_h_ui:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    slli a0, a0, 48
+; CHECK64-IZFH-NEXT:    srli a0, a0, 48
+; CHECK64-IZFH-NEXT:    fcvt.h.wu fa0, a0
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_h_ui:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    slli a0, a0, 16
+; CHECK32-IZHINX-NEXT:    srli a0, a0, 16
+; CHECK32-IZHINX-NEXT:    fcvt.h.wu a0, a0
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_h_ui:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    slli a0, a0, 48
+; CHECK64-IZHINX-NEXT:    srli a0, a0, 48
+; CHECK64-IZHINX-NEXT:    fcvt.h.wu a0, a0
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_h_ui:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -1072,22 +758,6 @@ define half @fcvt_h_ui(i16 %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_ui:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    slli a0, a0, 16
-; CHECK32-IZDINXZHINXMIN-NEXT:    srli a0, a0, 16
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_ui:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    slli a0, a0, 48
-; CHECK64-IZDINXZHINXMIN-NEXT:    srli a0, a0, 48
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_h_ui:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -1119,61 +789,39 @@ define half @fcvt_h_ui_zeroext(i16 zeroext %a) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.h.wu a0, a0
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_h_ui_zeroext:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_ui_zeroext:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_ui_zeroext:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_ui_zeroext:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_h_ui_zeroext:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.wu fa5, a0
-; CHECK32-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK32-IZFHMIN-NEXT:    ret
-;
-; CHECK64-IZFHMIN-LABEL: fcvt_h_ui_zeroext:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.wu fa5, a0
-; CHECK64-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_h_ui_zeroext:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK32-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_h_ui_zeroext:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_ui_zeroext:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_ui_zeroext:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_h_ui_zeroext:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.h.wu fa0, a0
+; CHECKIDZFH-NEXT:    ret
+;
+; CHECKIZDINXZHINX-LABEL: fcvt_h_ui_zeroext:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: fcvt_h_ui_zeroext:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    fcvt.s.wu fa5, a0
+; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: fcvt_h_ui_zeroext:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_h_ui_zeroext:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.s.wu fa5, a0
+; CHECKIDZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_h_ui_zeroext:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_h_ui_zeroext:
 ; CHECK32-D:       # %bb.0:
@@ -1203,61 +851,39 @@ define half @fcvt_h_w(i32 %a) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.h.w a0, a0
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_h_w:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.h.w fa0, a0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_w:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.h.w fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_w:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.h.w a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_w:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.h.w a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_h_w:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.w fa5, a0
-; CHECK32-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK32-IZFHMIN-NEXT:    ret
-;
-; CHECK64-IZFHMIN-LABEL: fcvt_h_w:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.w fa5, a0
-; CHECK64-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_h_w:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK32-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_h_w:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_w:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_w:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_h_w:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.h.w fa0, a0
+; CHECKIDZFH-NEXT:    ret
+;
+; CHECKIZDINXZHINX-LABEL: fcvt_h_w:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.h.w a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: fcvt_h_w:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    fcvt.s.w fa5, a0
+; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: fcvt_h_w:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    fcvt.s.w a0, a0
+; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_h_w:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.s.w fa5, a0
+; CHECKIDZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_h_w:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_h_w:
 ; CHECK32-D:       # %bb.0:
@@ -1290,71 +916,45 @@ define half @fcvt_h_w_load(ptr %p) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.h.w a0, a0
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_h_w_load:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    lw a0, 0(a0)
-; RV32IDZFH-NEXT:    fcvt.h.w fa0, a0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_w_load:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    lw a0, 0(a0)
-; RV64IDZFH-NEXT:    fcvt.h.w fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_w_load:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    lw a0, 0(a0)
-; RV32IZDINXZHINX-NEXT:    fcvt.h.w a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_w_load:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    lw a0, 0(a0)
-; RV64IZDINXZHINX-NEXT:    fcvt.h.w a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_h_w_load:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    lw a0, 0(a0)
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.w fa5, a0
-; CHECK32-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK32-IZFHMIN-NEXT:    ret
-;
-; CHECK64-IZFHMIN-LABEL: fcvt_h_w_load:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    lw a0, 0(a0)
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.w fa5, a0
-; CHECK64-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_h_w_load:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    lw a0, 0(a0)
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK32-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_h_w_load:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    lw a0, 0(a0)
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_w_load:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a0, 0(a0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_w_load:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    lw a0, 0(a0)
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_h_w_load:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    lw a0, 0(a0)
+; CHECKIDZFH-NEXT:    fcvt.h.w fa0, a0
+; CHECKIDZFH-NEXT:    ret
+;
+; CHECKIZDINXZHINX-LABEL: fcvt_h_w_load:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    lw a0, 0(a0)
+; CHECKIZDINXZHINX-NEXT:    fcvt.h.w a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: fcvt_h_w_load:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    lw a0, 0(a0)
+; CHECKIZFHMIN-NEXT:    fcvt.s.w fa5, a0
+; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: fcvt_h_w_load:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    lw a0, 0(a0)
+; CHECKIZHINXMIN-NEXT:    fcvt.s.w a0, a0
+; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_h_w_load:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    lw a0, 0(a0)
+; CHECKIDZFHMIN-NEXT:    fcvt.s.w fa5, a0
+; CHECKIDZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_h_w_load:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    lw a0, 0(a0)
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.w a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_h_w_load:
 ; CHECK32-D:       # %bb.0:
@@ -1386,61 +986,39 @@ define half @fcvt_h_wu(i32 %a) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.h.wu a0, a0
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_h_wu:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_wu:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_wu:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_wu:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_h_wu:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.wu fa5, a0
-; CHECK32-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK32-IZFHMIN-NEXT:    ret
-;
-; CHECK64-IZFHMIN-LABEL: fcvt_h_wu:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.wu fa5, a0
-; CHECK64-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_h_wu:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK32-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_h_wu:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_wu:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_wu:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_h_wu:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.h.wu fa0, a0
+; CHECKIDZFH-NEXT:    ret
+;
+; CHECKIZDINXZHINX-LABEL: fcvt_h_wu:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: fcvt_h_wu:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    fcvt.s.wu fa5, a0
+; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: fcvt_h_wu:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_h_wu:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.s.wu fa5, a0
+; CHECKIDZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_h_wu:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_h_wu:
 ; CHECK32-D:       # %bb.0:
@@ -1473,71 +1051,45 @@ define half @fcvt_h_wu_load(ptr %p) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.h.wu a0, a0
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_h_wu_load:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    lw a0, 0(a0)
-; RV32IDZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_wu_load:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    lw a0, 0(a0)
-; RV64IDZFH-NEXT:    fcvt.h.wu fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_wu_load:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    lw a0, 0(a0)
-; RV32IZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_wu_load:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    lw a0, 0(a0)
-; RV64IZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_h_wu_load:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    lw a0, 0(a0)
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.wu fa5, a0
-; CHECK32-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK32-IZFHMIN-NEXT:    ret
-;
-; CHECK64-IZFHMIN-LABEL: fcvt_h_wu_load:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    lw a0, 0(a0)
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.wu fa5, a0
-; CHECK64-IZFHMIN-NEXT:    fcvt.h.s fa0, fa5
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_h_wu_load:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    lw a0, 0(a0)
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK32-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_h_wu_load:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    lw a0, 0(a0)
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_wu_load:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a0, 0(a0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_wu_load:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    lw a0, 0(a0)
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_h_wu_load:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    lw a0, 0(a0)
+; CHECKIDZFH-NEXT:    fcvt.h.wu fa0, a0
+; CHECKIDZFH-NEXT:    ret
+;
+; CHECKIZDINXZHINX-LABEL: fcvt_h_wu_load:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    lw a0, 0(a0)
+; CHECKIZDINXZHINX-NEXT:    fcvt.h.wu a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
+;
+; CHECKIZFHMIN-LABEL: fcvt_h_wu_load:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    lw a0, 0(a0)
+; CHECKIZFHMIN-NEXT:    fcvt.s.wu fa5, a0
+; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIZFHMIN-NEXT:    ret
+;
+; CHECKIZHINXMIN-LABEL: fcvt_h_wu_load:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    lw a0, 0(a0)
+; CHECKIZHINXMIN-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_h_wu_load:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    lw a0, 0(a0)
+; CHECKIDZFHMIN-NEXT:    fcvt.s.wu fa5, a0
+; CHECKIDZFHMIN-NEXT:    fcvt.h.s fa0, fa5
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_h_wu_load:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    lw a0, 0(a0)
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.wu a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_h_wu_load:
 ; CHECK32-D:       # %bb.0:
@@ -1559,61 +1111,33 @@ define half @fcvt_h_wu_load(ptr %p) nounwind strictfp {
 }
 
 define half @fcvt_h_l(i64 %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_h_l:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    addi sp, sp, -16
-; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZFH-NEXT:    call __floatdihf
-; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_h_l:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    fcvt.h.l fa0, a0
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_h_l:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    addi sp, sp, -16
-; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZHINX-NEXT:    call __floatdihf
-; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_h_l:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    fcvt.h.l a0, a0
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_h_l:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    addi sp, sp, -16
-; RV32IDZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IDZFH-NEXT:    call __floatdihf
-; RV32IDZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IDZFH-NEXT:    addi sp, sp, 16
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_l:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.h.l fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_l:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, -16
-; RV32IZDINXZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZDINXZHINX-NEXT:    call __floatdihf
-; RV32IZDINXZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, 16
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_l:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.h.l a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_h_l:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    addi sp, sp, -16
+; CHECK32-IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-IZFH-NEXT:    call __floatdihf
+; CHECK32-IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-IZFH-NEXT:    addi sp, sp, 16
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_h_l:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    fcvt.h.l fa0, a0
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_h_l:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    addi sp, sp, -16
+; CHECK32-IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-IZHINX-NEXT:    call __floatdihf
+; CHECK32-IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-IZHINX-NEXT:    addi sp, sp, 16
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_h_l:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    fcvt.h.l a0, a0
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_h_l:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -1645,21 +1169,6 @@ define half @fcvt_h_l(i64 %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_l:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK32-IZDINXZHINXMIN-NEXT:    call __floatdihf
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_l:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.l a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_h_l:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -1679,61 +1188,33 @@ define half @fcvt_h_l(i64 %a) nounwind strictfp {
 declare half @llvm.experimental.constrained.sitofp.f16.i64(i64, metadata, metadata)
 
 define half @fcvt_h_lu(i64 %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_h_lu:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    addi sp, sp, -16
-; RV32IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZFH-NEXT:    call __floatundihf
-; RV32IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZFH-NEXT:    addi sp, sp, 16
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_h_lu:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    fcvt.h.lu fa0, a0
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_h_lu:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    addi sp, sp, -16
-; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZHINX-NEXT:    call __floatundihf
-; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZHINX-NEXT:    addi sp, sp, 16
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_h_lu:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    fcvt.h.lu a0, a0
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_h_lu:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    addi sp, sp, -16
-; RV32IDZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IDZFH-NEXT:    call __floatundihf
-; RV32IDZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IDZFH-NEXT:    addi sp, sp, 16
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_lu:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.h.lu fa0, a0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_lu:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, -16
-; RV32IZDINXZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32IZDINXZHINX-NEXT:    call __floatundihf
-; RV32IZDINXZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, 16
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_lu:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.h.lu a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_h_lu:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    addi sp, sp, -16
+; CHECK32-IZFH-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-IZFH-NEXT:    call __floatundihf
+; CHECK32-IZFH-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-IZFH-NEXT:    addi sp, sp, 16
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_h_lu:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    fcvt.h.lu fa0, a0
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_h_lu:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    addi sp, sp, -16
+; CHECK32-IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; CHECK32-IZHINX-NEXT:    call __floatundihf
+; CHECK32-IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; CHECK32-IZHINX-NEXT:    addi sp, sp, 16
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_h_lu:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    fcvt.h.lu a0, a0
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_h_lu:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -1765,21 +1246,6 @@ define half @fcvt_h_lu(i64 %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_lu:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK32-IZDINXZHINXMIN-NEXT:    call __floatundihf
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_lu:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.lu a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_h_lu:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -1809,55 +1275,35 @@ define half @fcvt_h_s(float %a) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.h.s a0, a0
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_h_s:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.h.s fa0, fa0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_s:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.h.s fa0, fa0
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_s:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.h.s a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_s:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.h.s a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
-;
-; CHECK32-IZFHMIN-LABEL: fcvt_h_s:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
-; CHECK32-IZFHMIN-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_h_s:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.h.s fa0, fa0
+; CHECKIDZFH-NEXT:    ret
 ;
-; CHECK64-IZFHMIN-LABEL: fcvt_h_s:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    fcvt.h.s fa0, fa0
-; CHECK64-IZFHMIN-NEXT:    ret
+; CHECKIZDINXZHINX-LABEL: fcvt_h_s:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.h.s a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
 ;
-; CHECK32-IZHINXMIN-LABEL: fcvt_h_s:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZHINXMIN-NEXT:    ret
+; CHECKIZFHMIN-LABEL: fcvt_h_s:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; CHECKIZFHMIN-NEXT:    ret
 ;
-; CHECK64-IZHINXMIN-LABEL: fcvt_h_s:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZHINXMIN-NEXT:    ret
+; CHECKIZHINXMIN-LABEL: fcvt_h_s:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_s:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIDZFHMIN-LABEL: fcvt_h_s:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.h.s fa0, fa0
+; CHECKIDZFHMIN-NEXT:    ret
 ;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_s:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_h_s:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.h.s a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_h_s:
 ; CHECK32-D:       # %bb.0:
@@ -1887,55 +1333,35 @@ define float @fcvt_s_h(half %a) nounwind strictfp {
 ; CHECKIZHINX-NEXT:    fcvt.s.h a0, a0
 ; CHECKIZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_s_h:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.s.h fa0, fa0
-; RV32IDZFH-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_s_h:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.s.h fa0, fa0
+; CHECKIDZFH-NEXT:    ret
 ;
-; RV64IDZFH-LABEL: fcvt_s_h:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.s.h fa0, fa0
-; RV64IDZFH-NEXT:    ret
+; CHECKIZDINXZHINX-LABEL: fcvt_s_h:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.s.h a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
 ;
-; RV32IZDINXZHINX-LABEL: fcvt_s_h:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
+; CHECKIZFHMIN-LABEL: fcvt_s_h:
+; CHECKIZFHMIN:       # %bb.0:
+; CHECKIZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; CHECKIZFHMIN-NEXT:    ret
 ;
-; RV64IZDINXZHINX-LABEL: fcvt_s_h:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECKIZHINXMIN-LABEL: fcvt_s_h:
+; CHECKIZHINXMIN:       # %bb.0:
+; CHECKIZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECKIZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZFHMIN-LABEL: fcvt_s_h:
-; CHECK32-IZFHMIN:       # %bb.0:
-; CHECK32-IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
-; CHECK32-IZFHMIN-NEXT:    ret
+; CHECKIDZFHMIN-LABEL: fcvt_s_h:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.s.h fa0, fa0
+; CHECKIDZFHMIN-NEXT:    ret
 ;
-; CHECK64-IZFHMIN-LABEL: fcvt_s_h:
-; CHECK64-IZFHMIN:       # %bb.0:
-; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa0, fa0
-; CHECK64-IZFHMIN-NEXT:    ret
-;
-; CHECK32-IZHINXMIN-LABEL: fcvt_s_h:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_s_h:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_s_h:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_s_h:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_s_h:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_s_h:
 ; CHECK32-D:       # %bb.0:
@@ -1991,25 +1417,15 @@ define half @fcvt_h_d(double %a) nounwind strictfp {
 ; RV64IZHINX-NEXT:    addi sp, sp, 16
 ; RV64IZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_h_d:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.h.d fa0, fa0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_d:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.h.d fa0, fa0
-; RV64IDZFH-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_h_d:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.h.d fa0, fa0
+; CHECKIDZFH-NEXT:    ret
 ;
-; RV32IZDINXZHINX-LABEL: fcvt_h_d:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.h.d a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_d:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.h.d a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECKIZDINXZHINX-LABEL: fcvt_h_d:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.h.d a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
 ;
 ; RV32IFZFHMIN-LABEL: fcvt_h_d:
 ; RV32IFZFHMIN:       # %bb.0:
@@ -2029,43 +1445,33 @@ define half @fcvt_h_d(double %a) nounwind strictfp {
 ; RV64IFZFHMIN-NEXT:    addi sp, sp, 16
 ; RV64IFZFHMIN-NEXT:    ret
 ;
-; CHECK32-IZHINXMIN-LABEL: fcvt_h_d:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK32-IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK32-IZHINXMIN-NEXT:    call __truncdfhf2
-; CHECK32-IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_h_d:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK64-IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK64-IZHINXMIN-NEXT:    call __truncdfhf2
-; CHECK64-IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; RV32IDZFHMIN-LABEL: fcvt_h_d:
-; RV32IDZFHMIN:       # %bb.0:
-; RV32IDZFHMIN-NEXT:    fcvt.h.d fa0, fa0
-; RV32IDZFHMIN-NEXT:    ret
-;
-; RV64IDZFHMIN-LABEL: fcvt_h_d:
-; RV64IDZFHMIN:       # %bb.0:
-; RV64IDZFHMIN-NEXT:    fcvt.h.d fa0, fa0
-; RV64IDZFHMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_d:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.d a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_d:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.d a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; RV32IZHINXMIN-LABEL: fcvt_h_d:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    call __truncdfhf2
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: fcvt_h_d:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    call __truncdfhf2
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_h_d:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.h.d fa0, fa0
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_h_d:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.h.d a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_h_d:
 ; CHECK32-D:       # %bb.0:
@@ -2125,25 +1531,15 @@ define double @fcvt_d_h(half %a) nounwind strictfp {
 ; RV64IZHINX-NEXT:    addi sp, sp, 16
 ; RV64IZHINX-NEXT:    ret
 ;
-; RV32IDZFH-LABEL: fcvt_d_h:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    fcvt.d.h fa0, fa0
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_d_h:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    fcvt.d.h fa0, fa0
-; RV64IDZFH-NEXT:    ret
+; CHECKIDZFH-LABEL: fcvt_d_h:
+; CHECKIDZFH:       # %bb.0:
+; CHECKIDZFH-NEXT:    fcvt.d.h fa0, fa0
+; CHECKIDZFH-NEXT:    ret
 ;
-; RV32IZDINXZHINX-LABEL: fcvt_d_h:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    fcvt.d.h a0, a0
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_d_h:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    fcvt.d.h a0, a0
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECKIZDINXZHINX-LABEL: fcvt_d_h:
+; CHECKIZDINXZHINX:       # %bb.0:
+; CHECKIZDINXZHINX-NEXT:    fcvt.d.h a0, a0
+; CHECKIZDINXZHINX-NEXT:    ret
 ;
 ; RV32IFZFHMIN-LABEL: fcvt_d_h:
 ; RV32IFZFHMIN:       # %bb.0:
@@ -2165,45 +1561,35 @@ define double @fcvt_d_h(half %a) nounwind strictfp {
 ; RV64IFZFHMIN-NEXT:    addi sp, sp, 16
 ; RV64IFZFHMIN-NEXT:    ret
 ;
-; CHECK32-IZHINXMIN-LABEL: fcvt_d_h:
-; CHECK32-IZHINXMIN:       # %bb.0:
-; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK32-IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZHINXMIN-NEXT:    call __extendsfdf2
-; CHECK32-IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
-; CHECK32-IZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK32-IZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZHINXMIN-LABEL: fcvt_d_h:
-; CHECK64-IZHINXMIN:       # %bb.0:
-; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK64-IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZHINXMIN-NEXT:    call __extendsfdf2
-; CHECK64-IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK64-IZHINXMIN-NEXT:    ret
-;
-; RV32IDZFHMIN-LABEL: fcvt_d_h:
-; RV32IDZFHMIN:       # %bb.0:
-; RV32IDZFHMIN-NEXT:    fcvt.d.h fa0, fa0
-; RV32IDZFHMIN-NEXT:    ret
-;
-; RV64IDZFHMIN-LABEL: fcvt_d_h:
-; RV64IDZFHMIN:       # %bb.0:
-; RV64IDZFHMIN-NEXT:    fcvt.d.h fa0, fa0
-; RV64IDZFHMIN-NEXT:    ret
-;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_d_h:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.d.h a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_d_h:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.d.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
+; RV32IZHINXMIN-LABEL: fcvt_d_h:
+; RV32IZHINXMIN:       # %bb.0:
+; RV32IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV32IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV32IZHINXMIN-NEXT:    call __extendsfdf2
+; RV32IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV32IZHINXMIN-NEXT:    ret
+;
+; RV64IZHINXMIN-LABEL: fcvt_d_h:
+; RV64IZHINXMIN:       # %bb.0:
+; RV64IZHINXMIN-NEXT:    addi sp, sp, -16
+; RV64IZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64IZHINXMIN-NEXT:    fcvt.s.h a0, a0
+; RV64IZHINXMIN-NEXT:    call __extendsfdf2
+; RV64IZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64IZHINXMIN-NEXT:    addi sp, sp, 16
+; RV64IZHINXMIN-NEXT:    ret
+;
+; CHECKIDZFHMIN-LABEL: fcvt_d_h:
+; CHECKIDZFHMIN:       # %bb.0:
+; CHECKIDZFHMIN-NEXT:    fcvt.d.h fa0, fa0
+; CHECKIDZFHMIN-NEXT:    ret
+;
+; CHECKIZDINXZHINXMIN-LABEL: fcvt_d_h:
+; CHECKIZDINXZHINXMIN:       # %bb.0:
+; CHECKIZDINXZHINXMIN-NEXT:    fcvt.d.h a0, a0
+; CHECKIZDINXZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-D-LABEL: fcvt_d_h:
 ; CHECK32-D:       # %bb.0:
@@ -2225,61 +1611,33 @@ declare double @llvm.experimental.constrained.fpext.f64.f16(half, metadata)
 
 ; Make sure we select W version of addi on RV64.
 define signext i32 @fcvt_h_w_demanded_bits(i32 signext %0, ptr %1) strictfp {
-; RV32IZFH-LABEL: fcvt_h_w_demanded_bits:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    addi a0, a0, 1
-; RV32IZFH-NEXT:    fcvt.h.w fa5, a0
-; RV32IZFH-NEXT:    fsh fa5, 0(a1)
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_h_w_demanded_bits:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    addiw a0, a0, 1
-; RV64IZFH-NEXT:    fcvt.h.w fa5, a0
-; RV64IZFH-NEXT:    fsh fa5, 0(a1)
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_h_w_demanded_bits:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    addi a0, a0, 1
-; RV32IZHINX-NEXT:    fcvt.h.w a2, a0
-; RV32IZHINX-NEXT:    sh a2, 0(a1)
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_h_w_demanded_bits:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    addiw a0, a0, 1
-; RV64IZHINX-NEXT:    fcvt.h.w a2, a0
-; RV64IZHINX-NEXT:    sh a2, 0(a1)
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_h_w_demanded_bits:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    addi a0, a0, 1
-; RV32IDZFH-NEXT:    fcvt.h.w fa5, a0
-; RV32IDZFH-NEXT:    fsh fa5, 0(a1)
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_w_demanded_bits:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    addiw a0, a0, 1
-; RV64IDZFH-NEXT:    fcvt.h.w fa5, a0
-; RV64IDZFH-NEXT:    fsh fa5, 0(a1)
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_w_demanded_bits:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    addi a0, a0, 1
-; RV32IZDINXZHINX-NEXT:    fcvt.h.w a2, a0
-; RV32IZDINXZHINX-NEXT:    sh a2, 0(a1)
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_w_demanded_bits:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    addiw a0, a0, 1
-; RV64IZDINXZHINX-NEXT:    fcvt.h.w a2, a0
-; RV64IZDINXZHINX-NEXT:    sh a2, 0(a1)
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_h_w_demanded_bits:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    addi a0, a0, 1
+; CHECK32-IZFH-NEXT:    fcvt.h.w fa5, a0
+; CHECK32-IZFH-NEXT:    fsh fa5, 0(a1)
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_h_w_demanded_bits:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    addiw a0, a0, 1
+; CHECK64-IZFH-NEXT:    fcvt.h.w fa5, a0
+; CHECK64-IZFH-NEXT:    fsh fa5, 0(a1)
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_h_w_demanded_bits:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    addi a0, a0, 1
+; CHECK32-IZHINX-NEXT:    fcvt.h.w a2, a0
+; CHECK32-IZHINX-NEXT:    sh a2, 0(a1)
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_h_w_demanded_bits:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    addiw a0, a0, 1
+; CHECK64-IZHINX-NEXT:    fcvt.h.w a2, a0
+; CHECK64-IZHINX-NEXT:    sh a2, 0(a1)
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_h_w_demanded_bits:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -2313,22 +1671,6 @@ define signext i32 @fcvt_h_w_demanded_bits(i32 signext %0, ptr %1) strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    sh a2, 0(a1)
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_w_demanded_bits:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi a0, a0, 1
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.w a2, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a2, a2
-; CHECK32-IZDINXZHINXMIN-NEXT:    sh a2, 0(a1)
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_w_demanded_bits:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    addiw a0, a0, 1
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.w a2, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a2, a2
-; CHECK64-IZDINXZHINXMIN-NEXT:    sh a2, 0(a1)
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_h_w_demanded_bits:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -2363,61 +1705,33 @@ define signext i32 @fcvt_h_w_demanded_bits(i32 signext %0, ptr %1) strictfp {
 
 ; Make sure we select W version of addi on RV64.
 define signext i32 @fcvt_h_wu_demanded_bits(i32 signext %0, ptr %1) strictfp {
-; RV32IZFH-LABEL: fcvt_h_wu_demanded_bits:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    addi a0, a0, 1
-; RV32IZFH-NEXT:    fcvt.h.wu fa5, a0
-; RV32IZFH-NEXT:    fsh fa5, 0(a1)
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_h_wu_demanded_bits:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    addiw a0, a0, 1
-; RV64IZFH-NEXT:    fcvt.h.wu fa5, a0
-; RV64IZFH-NEXT:    fsh fa5, 0(a1)
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_h_wu_demanded_bits:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    addi a0, a0, 1
-; RV32IZHINX-NEXT:    fcvt.h.wu a2, a0
-; RV32IZHINX-NEXT:    sh a2, 0(a1)
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_h_wu_demanded_bits:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    addiw a0, a0, 1
-; RV64IZHINX-NEXT:    fcvt.h.wu a2, a0
-; RV64IZHINX-NEXT:    sh a2, 0(a1)
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_h_wu_demanded_bits:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    addi a0, a0, 1
-; RV32IDZFH-NEXT:    fcvt.h.wu fa5, a0
-; RV32IDZFH-NEXT:    fsh fa5, 0(a1)
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_wu_demanded_bits:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    addiw a0, a0, 1
-; RV64IDZFH-NEXT:    fcvt.h.wu fa5, a0
-; RV64IDZFH-NEXT:    fsh fa5, 0(a1)
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_wu_demanded_bits:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    addi a0, a0, 1
-; RV32IZDINXZHINX-NEXT:    fcvt.h.wu a2, a0
-; RV32IZDINXZHINX-NEXT:    sh a2, 0(a1)
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_wu_demanded_bits:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    addiw a0, a0, 1
-; RV64IZDINXZHINX-NEXT:    fcvt.h.wu a2, a0
-; RV64IZDINXZHINX-NEXT:    sh a2, 0(a1)
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_h_wu_demanded_bits:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    addi a0, a0, 1
+; CHECK32-IZFH-NEXT:    fcvt.h.wu fa5, a0
+; CHECK32-IZFH-NEXT:    fsh fa5, 0(a1)
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_h_wu_demanded_bits:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    addiw a0, a0, 1
+; CHECK64-IZFH-NEXT:    fcvt.h.wu fa5, a0
+; CHECK64-IZFH-NEXT:    fsh fa5, 0(a1)
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_h_wu_demanded_bits:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    addi a0, a0, 1
+; CHECK32-IZHINX-NEXT:    fcvt.h.wu a2, a0
+; CHECK32-IZHINX-NEXT:    sh a2, 0(a1)
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_h_wu_demanded_bits:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    addiw a0, a0, 1
+; CHECK64-IZHINX-NEXT:    fcvt.h.wu a2, a0
+; CHECK64-IZHINX-NEXT:    sh a2, 0(a1)
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_h_wu_demanded_bits:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -2451,22 +1765,6 @@ define signext i32 @fcvt_h_wu_demanded_bits(i32 signext %0, ptr %1) strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    sh a2, 0(a1)
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_wu_demanded_bits:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi a0, a0, 1
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a2, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.h.s a2, a2
-; CHECK32-IZDINXZHINXMIN-NEXT:    sh a2, 0(a1)
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_wu_demanded_bits:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    addiw a0, a0, 1
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.wu a2, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.h.s a2, a2
-; CHECK64-IZDINXZHINXMIN-NEXT:    sh a2, 0(a1)
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_h_wu_demanded_bits:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -16
@@ -2500,113 +1798,59 @@ define signext i32 @fcvt_h_wu_demanded_bits(i32 signext %0, ptr %1) strictfp {
 }
 
 define half @fcvt_h_q(fp128 %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_h_q:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    addi sp, sp, -32
-; RV32IZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZFH-NEXT:    lw a1, 0(a0)
-; RV32IZFH-NEXT:    lw a2, 4(a0)
-; RV32IZFH-NEXT:    lw a3, 8(a0)
-; RV32IZFH-NEXT:    lw a4, 12(a0)
-; RV32IZFH-NEXT:    addi a0, sp, 8
-; RV32IZFH-NEXT:    sw a1, 8(sp)
-; RV32IZFH-NEXT:    sw a2, 12(sp)
-; RV32IZFH-NEXT:    sw a3, 16(sp)
-; RV32IZFH-NEXT:    sw a4, 20(sp)
-; RV32IZFH-NEXT:    call __trunctfhf2
-; RV32IZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZFH-NEXT:    addi sp, sp, 32
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_h_q:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    addi sp, sp, -16
-; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFH-NEXT:    call __trunctfhf2
-; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFH-NEXT:    addi sp, sp, 16
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_h_q:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    addi sp, sp, -32
-; RV32IZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZHINX-NEXT:    lw a1, 0(a0)
-; RV32IZHINX-NEXT:    lw a2, 4(a0)
-; RV32IZHINX-NEXT:    lw a3, 8(a0)
-; RV32IZHINX-NEXT:    lw a4, 12(a0)
-; RV32IZHINX-NEXT:    addi a0, sp, 8
-; RV32IZHINX-NEXT:    sw a1, 8(sp)
-; RV32IZHINX-NEXT:    sw a2, 12(sp)
-; RV32IZHINX-NEXT:    sw a3, 16(sp)
-; RV32IZHINX-NEXT:    sw a4, 20(sp)
-; RV32IZHINX-NEXT:    call __trunctfhf2
-; RV32IZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZHINX-NEXT:    addi sp, sp, 32
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_h_q:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    addi sp, sp, -16
-; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZHINX-NEXT:    call __trunctfhf2
-; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZHINX-NEXT:    addi sp, sp, 16
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_h_q:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    addi sp, sp, -32
-; RV32IDZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IDZFH-NEXT:    lw a1, 0(a0)
-; RV32IDZFH-NEXT:    lw a2, 4(a0)
-; RV32IDZFH-NEXT:    lw a3, 8(a0)
-; RV32IDZFH-NEXT:    lw a4, 12(a0)
-; RV32IDZFH-NEXT:    addi a0, sp, 8
-; RV32IDZFH-NEXT:    sw a1, 8(sp)
-; RV32IDZFH-NEXT:    sw a2, 12(sp)
-; RV32IDZFH-NEXT:    sw a3, 16(sp)
-; RV32IDZFH-NEXT:    sw a4, 20(sp)
-; RV32IDZFH-NEXT:    call __trunctfhf2
-; RV32IDZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IDZFH-NEXT:    addi sp, sp, 32
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_h_q:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    addi sp, sp, -16
-; RV64IDZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IDZFH-NEXT:    call __trunctfhf2
-; RV64IDZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IDZFH-NEXT:    addi sp, sp, 16
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_h_q:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, -32
-; RV32IZDINXZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZDINXZHINX-NEXT:    lw a1, 0(a0)
-; RV32IZDINXZHINX-NEXT:    lw a2, 4(a0)
-; RV32IZDINXZHINX-NEXT:    lw a3, 8(a0)
-; RV32IZDINXZHINX-NEXT:    lw a4, 12(a0)
-; RV32IZDINXZHINX-NEXT:    addi a0, sp, 8
-; RV32IZDINXZHINX-NEXT:    sw a1, 8(sp)
-; RV32IZDINXZHINX-NEXT:    sw a2, 12(sp)
-; RV32IZDINXZHINX-NEXT:    sw a3, 16(sp)
-; RV32IZDINXZHINX-NEXT:    sw a4, 20(sp)
-; RV32IZDINXZHINX-NEXT:    call __trunctfhf2
-; RV32IZDINXZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, 32
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_h_q:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    addi sp, sp, -16
-; RV64IZDINXZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZDINXZHINX-NEXT:    call __trunctfhf2
-; RV64IZDINXZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZDINXZHINX-NEXT:    addi sp, sp, 16
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_h_q:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    addi sp, sp, -32
+; CHECK32-IZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZFH-NEXT:    lw a1, 0(a0)
+; CHECK32-IZFH-NEXT:    lw a2, 4(a0)
+; CHECK32-IZFH-NEXT:    lw a3, 8(a0)
+; CHECK32-IZFH-NEXT:    lw a4, 12(a0)
+; CHECK32-IZFH-NEXT:    addi a0, sp, 8
+; CHECK32-IZFH-NEXT:    sw a1, 8(sp)
+; CHECK32-IZFH-NEXT:    sw a2, 12(sp)
+; CHECK32-IZFH-NEXT:    sw a3, 16(sp)
+; CHECK32-IZFH-NEXT:    sw a4, 20(sp)
+; CHECK32-IZFH-NEXT:    call __trunctfhf2
+; CHECK32-IZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZFH-NEXT:    addi sp, sp, 32
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_h_q:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    addi sp, sp, -16
+; CHECK64-IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZFH-NEXT:    call __trunctfhf2
+; CHECK64-IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZFH-NEXT:    addi sp, sp, 16
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_h_q:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    addi sp, sp, -32
+; CHECK32-IZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZHINX-NEXT:    lw a1, 0(a0)
+; CHECK32-IZHINX-NEXT:    lw a2, 4(a0)
+; CHECK32-IZHINX-NEXT:    lw a3, 8(a0)
+; CHECK32-IZHINX-NEXT:    lw a4, 12(a0)
+; CHECK32-IZHINX-NEXT:    addi a0, sp, 8
+; CHECK32-IZHINX-NEXT:    sw a1, 8(sp)
+; CHECK32-IZHINX-NEXT:    sw a2, 12(sp)
+; CHECK32-IZHINX-NEXT:    sw a3, 16(sp)
+; CHECK32-IZHINX-NEXT:    sw a4, 20(sp)
+; CHECK32-IZHINX-NEXT:    call __trunctfhf2
+; CHECK32-IZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZHINX-NEXT:    addi sp, sp, 32
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_h_q:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    addi sp, sp, -16
+; CHECK64-IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZHINX-NEXT:    call __trunctfhf2
+; CHECK64-IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZHINX-NEXT:    addi sp, sp, 16
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_h_q:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -2662,33 +1906,6 @@ define half @fcvt_h_q(fp128 %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, 16
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_h_q:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -32
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a1, 0(a0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a2, 4(a0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a3, 8(a0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a4, 12(a0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi a0, sp, 8
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw a1, 8(sp)
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw a2, 12(sp)
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw a3, 16(sp)
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw a4, 20(sp)
-; CHECK32-IZDINXZHINXMIN-NEXT:    call __trunctfhf2
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 32
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_h_q:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK64-IZDINXZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK64-IZDINXZHINXMIN-NEXT:    call __trunctfhf2
-; CHECK64-IZDINXZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_h_q:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -32
@@ -2715,133 +1932,69 @@ define half @fcvt_h_q(fp128 %a) nounwind strictfp {
 }
 
 define fp128 @fcvt_q_h(half %a) nounwind strictfp {
-; RV32IZFH-LABEL: fcvt_q_h:
-; RV32IZFH:       # %bb.0:
-; RV32IZFH-NEXT:    addi sp, sp, -32
-; RV32IZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZFH-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZFH-NEXT:    mv s0, a0
-; RV32IZFH-NEXT:    fcvt.s.h fa0, fa0
-; RV32IZFH-NEXT:    addi a0, sp, 8
-; RV32IZFH-NEXT:    call __extendsftf2
-; RV32IZFH-NEXT:    lw a0, 8(sp)
-; RV32IZFH-NEXT:    lw a1, 12(sp)
-; RV32IZFH-NEXT:    lw a2, 16(sp)
-; RV32IZFH-NEXT:    lw a3, 20(sp)
-; RV32IZFH-NEXT:    sw a0, 0(s0)
-; RV32IZFH-NEXT:    sw a1, 4(s0)
-; RV32IZFH-NEXT:    sw a2, 8(s0)
-; RV32IZFH-NEXT:    sw a3, 12(s0)
-; RV32IZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZFH-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZFH-NEXT:    addi sp, sp, 32
-; RV32IZFH-NEXT:    ret
-;
-; RV64IZFH-LABEL: fcvt_q_h:
-; RV64IZFH:       # %bb.0:
-; RV64IZFH-NEXT:    addi sp, sp, -16
-; RV64IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZFH-NEXT:    fcvt.s.h fa0, fa0
-; RV64IZFH-NEXT:    call __extendsftf2
-; RV64IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZFH-NEXT:    addi sp, sp, 16
-; RV64IZFH-NEXT:    ret
-;
-; RV32IZHINX-LABEL: fcvt_q_h:
-; RV32IZHINX:       # %bb.0:
-; RV32IZHINX-NEXT:    addi sp, sp, -32
-; RV32IZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZHINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZHINX-NEXT:    mv s0, a0
-; RV32IZHINX-NEXT:    fcvt.s.h a1, a1
-; RV32IZHINX-NEXT:    addi a0, sp, 8
-; RV32IZHINX-NEXT:    call __extendsftf2
-; RV32IZHINX-NEXT:    lw a0, 8(sp)
-; RV32IZHINX-NEXT:    lw a1, 12(sp)
-; RV32IZHINX-NEXT:    lw a2, 16(sp)
-; RV32IZHINX-NEXT:    lw a3, 20(sp)
-; RV32IZHINX-NEXT:    sw a0, 0(s0)
-; RV32IZHINX-NEXT:    sw a1, 4(s0)
-; RV32IZHINX-NEXT:    sw a2, 8(s0)
-; RV32IZHINX-NEXT:    sw a3, 12(s0)
-; RV32IZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZHINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZHINX-NEXT:    addi sp, sp, 32
-; RV32IZHINX-NEXT:    ret
-;
-; RV64IZHINX-LABEL: fcvt_q_h:
-; RV64IZHINX:       # %bb.0:
-; RV64IZHINX-NEXT:    addi sp, sp, -16
-; RV64IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
-; RV64IZHINX-NEXT:    call __extendsftf2
-; RV64IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZHINX-NEXT:    addi sp, sp, 16
-; RV64IZHINX-NEXT:    ret
-;
-; RV32IDZFH-LABEL: fcvt_q_h:
-; RV32IDZFH:       # %bb.0:
-; RV32IDZFH-NEXT:    addi sp, sp, -32
-; RV32IDZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IDZFH-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IDZFH-NEXT:    mv s0, a0
-; RV32IDZFH-NEXT:    fcvt.s.h fa0, fa0
-; RV32IDZFH-NEXT:    addi a0, sp, 8
-; RV32IDZFH-NEXT:    call __extendsftf2
-; RV32IDZFH-NEXT:    lw a0, 8(sp)
-; RV32IDZFH-NEXT:    lw a1, 12(sp)
-; RV32IDZFH-NEXT:    lw a2, 16(sp)
-; RV32IDZFH-NEXT:    lw a3, 20(sp)
-; RV32IDZFH-NEXT:    sw a0, 0(s0)
-; RV32IDZFH-NEXT:    sw a1, 4(s0)
-; RV32IDZFH-NEXT:    sw a2, 8(s0)
-; RV32IDZFH-NEXT:    sw a3, 12(s0)
-; RV32IDZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IDZFH-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IDZFH-NEXT:    addi sp, sp, 32
-; RV32IDZFH-NEXT:    ret
-;
-; RV64IDZFH-LABEL: fcvt_q_h:
-; RV64IDZFH:       # %bb.0:
-; RV64IDZFH-NEXT:    addi sp, sp, -16
-; RV64IDZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IDZFH-NEXT:    fcvt.s.h fa0, fa0
-; RV64IDZFH-NEXT:    call __extendsftf2
-; RV64IDZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IDZFH-NEXT:    addi sp, sp, 16
-; RV64IDZFH-NEXT:    ret
-;
-; RV32IZDINXZHINX-LABEL: fcvt_q_h:
-; RV32IZDINXZHINX:       # %bb.0:
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, -32
-; RV32IZDINXZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; RV32IZDINXZHINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; RV32IZDINXZHINX-NEXT:    mv s0, a0
-; RV32IZDINXZHINX-NEXT:    fcvt.s.h a1, a1
-; RV32IZDINXZHINX-NEXT:    addi a0, sp, 8
-; RV32IZDINXZHINX-NEXT:    call __extendsftf2
-; RV32IZDINXZHINX-NEXT:    lw a0, 8(sp)
-; RV32IZDINXZHINX-NEXT:    lw a1, 12(sp)
-; RV32IZDINXZHINX-NEXT:    lw a2, 16(sp)
-; RV32IZDINXZHINX-NEXT:    lw a3, 20(sp)
-; RV32IZDINXZHINX-NEXT:    sw a0, 0(s0)
-; RV32IZDINXZHINX-NEXT:    sw a1, 4(s0)
-; RV32IZDINXZHINX-NEXT:    sw a2, 8(s0)
-; RV32IZDINXZHINX-NEXT:    sw a3, 12(s0)
-; RV32IZDINXZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; RV32IZDINXZHINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; RV32IZDINXZHINX-NEXT:    addi sp, sp, 32
-; RV32IZDINXZHINX-NEXT:    ret
-;
-; RV64IZDINXZHINX-LABEL: fcvt_q_h:
-; RV64IZDINXZHINX:       # %bb.0:
-; RV64IZDINXZHINX-NEXT:    addi sp, sp, -16
-; RV64IZDINXZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
-; RV64IZDINXZHINX-NEXT:    call __extendsftf2
-; RV64IZDINXZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; RV64IZDINXZHINX-NEXT:    addi sp, sp, 16
-; RV64IZDINXZHINX-NEXT:    ret
+; CHECK32-IZFH-LABEL: fcvt_q_h:
+; CHECK32-IZFH:       # %bb.0:
+; CHECK32-IZFH-NEXT:    addi sp, sp, -32
+; CHECK32-IZFH-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZFH-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZFH-NEXT:    mv s0, a0
+; CHECK32-IZFH-NEXT:    fcvt.s.h fa0, fa0
+; CHECK32-IZFH-NEXT:    addi a0, sp, 8
+; CHECK32-IZFH-NEXT:    call __extendsftf2
+; CHECK32-IZFH-NEXT:    lw a0, 8(sp)
+; CHECK32-IZFH-NEXT:    lw a1, 12(sp)
+; CHECK32-IZFH-NEXT:    lw a2, 16(sp)
+; CHECK32-IZFH-NEXT:    lw a3, 20(sp)
+; CHECK32-IZFH-NEXT:    sw a0, 0(s0)
+; CHECK32-IZFH-NEXT:    sw a1, 4(s0)
+; CHECK32-IZFH-NEXT:    sw a2, 8(s0)
+; CHECK32-IZFH-NEXT:    sw a3, 12(s0)
+; CHECK32-IZFH-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZFH-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZFH-NEXT:    addi sp, sp, 32
+; CHECK32-IZFH-NEXT:    ret
+;
+; CHECK64-IZFH-LABEL: fcvt_q_h:
+; CHECK64-IZFH:       # %bb.0:
+; CHECK64-IZFH-NEXT:    addi sp, sp, -16
+; CHECK64-IZFH-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZFH-NEXT:    fcvt.s.h fa0, fa0
+; CHECK64-IZFH-NEXT:    call __extendsftf2
+; CHECK64-IZFH-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZFH-NEXT:    addi sp, sp, 16
+; CHECK64-IZFH-NEXT:    ret
+;
+; CHECK32-IZHINX-LABEL: fcvt_q_h:
+; CHECK32-IZHINX:       # %bb.0:
+; CHECK32-IZHINX-NEXT:    addi sp, sp, -32
+; CHECK32-IZHINX-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
+; CHECK32-IZHINX-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
+; CHECK32-IZHINX-NEXT:    mv s0, a0
+; CHECK32-IZHINX-NEXT:    fcvt.s.h a1, a1
+; CHECK32-IZHINX-NEXT:    addi a0, sp, 8
+; CHECK32-IZHINX-NEXT:    call __extendsftf2
+; CHECK32-IZHINX-NEXT:    lw a0, 8(sp)
+; CHECK32-IZHINX-NEXT:    lw a1, 12(sp)
+; CHECK32-IZHINX-NEXT:    lw a2, 16(sp)
+; CHECK32-IZHINX-NEXT:    lw a3, 20(sp)
+; CHECK32-IZHINX-NEXT:    sw a0, 0(s0)
+; CHECK32-IZHINX-NEXT:    sw a1, 4(s0)
+; CHECK32-IZHINX-NEXT:    sw a2, 8(s0)
+; CHECK32-IZHINX-NEXT:    sw a3, 12(s0)
+; CHECK32-IZHINX-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
+; CHECK32-IZHINX-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
+; CHECK32-IZHINX-NEXT:    addi sp, sp, 32
+; CHECK32-IZHINX-NEXT:    ret
+;
+; CHECK64-IZHINX-LABEL: fcvt_q_h:
+; CHECK64-IZHINX:       # %bb.0:
+; CHECK64-IZHINX-NEXT:    addi sp, sp, -16
+; CHECK64-IZHINX-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; CHECK64-IZHINX-NEXT:    fcvt.s.h a0, a0
+; CHECK64-IZHINX-NEXT:    call __extendsftf2
+; CHECK64-IZHINX-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; CHECK64-IZHINX-NEXT:    addi sp, sp, 16
+; CHECK64-IZHINX-NEXT:    ret
 ;
 ; CHECK32-IZFHMIN-LABEL: fcvt_q_h:
 ; CHECK32-IZFHMIN:       # %bb.0:
@@ -2907,38 +2060,6 @@ define fp128 @fcvt_q_h(half %a) nounwind strictfp {
 ; CHECK64-IZHINXMIN-NEXT:    addi sp, sp, 16
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
-; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_q_h:
-; CHECK32-IZDINXZHINXMIN:       # %bb.0:
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, -32
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 28(sp) # 4-byte Folded Spill
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw s0, 24(sp) # 4-byte Folded Spill
-; CHECK32-IZDINXZHINXMIN-NEXT:    mv s0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a1, a1
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi a0, sp, 8
-; CHECK32-IZDINXZHINXMIN-NEXT:    call __extendsftf2
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a0, 8(sp)
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a1, 12(sp)
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a2, 16(sp)
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw a3, 20(sp)
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw a0, 0(s0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw a1, 4(s0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw a2, 8(s0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    sw a3, 12(s0)
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
-; CHECK32-IZDINXZHINXMIN-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi sp, sp, 32
-; CHECK32-IZDINXZHINXMIN-NEXT:    ret
-;
-; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_q_h:
-; CHECK64-IZDINXZHINXMIN:       # %bb.0:
-; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, -16
-; CHECK64-IZDINXZHINXMIN-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    call __extendsftf2
-; CHECK64-IZDINXZHINXMIN-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
-; CHECK64-IZDINXZHINXMIN-NEXT:    addi sp, sp, 16
-; CHECK64-IZDINXZHINXMIN-NEXT:    ret
-;
 ; CHECK32-D-LABEL: fcvt_q_h:
 ; CHECK32-D:       # %bb.0:
 ; CHECK32-D-NEXT:    addi sp, sp, -32
diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll
index facb544fb52b6..961c6cd78212b 100644
--- a/llvm/test/CodeGen/RISCV/half-convert.ll
+++ b/llvm/test/CodeGen/RISCV/half-convert.ll
@@ -818,7 +818,7 @@ define i16 @fcvt_ui_h_sat(half %a) nounwind {
 ; RV32I-NEXT:    call __gtsf2
 ; RV32I-NEXT:    bgtz a0, .LBB3_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slti a0, s2, 0
+; RV32I-NEXT:    srli a0, s2, 31
 ; RV32I-NEXT:    addi a0, a0, -1
 ; RV32I-NEXT:    and s0, a0, s1
 ; RV32I-NEXT:  .LBB3_2: # %start
@@ -856,7 +856,7 @@ define i16 @fcvt_ui_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    call __gtsf2
 ; RV64I-NEXT:    bgtz a0, .LBB3_2
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    slti a0, s2, 0
+; RV64I-NEXT:    srli a0, s2, 63
 ; RV64I-NEXT:    addi a0, a0, -1
 ; RV64I-NEXT:    and s0, a0, s1
 ; RV64I-NEXT:  .LBB3_2: # %start
@@ -1788,7 +1788,7 @@ define i32 @fcvt_wu_h_sat(half %a) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi s2, a0, -1
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call __fixunssfsi
@@ -1828,8 +1828,8 @@ define i32 @fcvt_wu_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    j .LBB8_3
 ; RV64I-NEXT:  .LBB8_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB8_3: # %start
 ; RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
@@ -2262,12 +2262,12 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; RV32IZHINX-NEXT:    addi a2, a3, -1
 ; RV32IZHINX-NEXT:  .LBB10_4: # %start
 ; RV32IZHINX-NEXT:    feq.s a3, s0, s0
-; RV32IZHINX-NEXT:    neg a4, a1
-; RV32IZHINX-NEXT:    neg a1, s1
+; RV32IZHINX-NEXT:    neg a4, s1
+; RV32IZHINX-NEXT:    neg a5, a1
 ; RV32IZHINX-NEXT:    neg a3, a3
-; RV32IZHINX-NEXT:    and a0, a1, a0
+; RV32IZHINX-NEXT:    and a0, a4, a0
 ; RV32IZHINX-NEXT:    and a1, a3, a2
-; RV32IZHINX-NEXT:    or a0, a4, a0
+; RV32IZHINX-NEXT:    or a0, a5, a0
 ; RV32IZHINX-NEXT:    and a0, a3, a0
 ; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
@@ -2309,12 +2309,12 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; RV32IZDINXZHINX-NEXT:    addi a2, a3, -1
 ; RV32IZDINXZHINX-NEXT:  .LBB10_4: # %start
 ; RV32IZDINXZHINX-NEXT:    feq.s a3, s0, s0
-; RV32IZDINXZHINX-NEXT:    neg a4, a1
-; RV32IZDINXZHINX-NEXT:    neg a1, s1
+; RV32IZDINXZHINX-NEXT:    neg a4, s1
+; RV32IZDINXZHINX-NEXT:    neg a5, a1
 ; RV32IZDINXZHINX-NEXT:    neg a3, a3
-; RV32IZDINXZHINX-NEXT:    and a0, a1, a0
+; RV32IZDINXZHINX-NEXT:    and a0, a4, a0
 ; RV32IZDINXZHINX-NEXT:    and a1, a3, a2
-; RV32IZDINXZHINX-NEXT:    or a0, a4, a0
+; RV32IZDINXZHINX-NEXT:    or a0, a5, a0
 ; RV32IZDINXZHINX-NEXT:    and a0, a3, a0
 ; RV32IZDINXZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZDINXZHINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
@@ -2369,13 +2369,13 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; RV32I-NEXT:    call __unordsf2
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    sgtz a1, s4
-; RV32I-NEXT:    slti a2, s0, 0
+; RV32I-NEXT:    srli s0, s0, 31
 ; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    neg a3, a1
-; RV32I-NEXT:    addi a2, a2, -1
+; RV32I-NEXT:    neg a2, a1
+; RV32I-NEXT:    addi s0, s0, -1
 ; RV32I-NEXT:    and a1, a0, s3
-; RV32I-NEXT:    and a2, a2, s1
-; RV32I-NEXT:    or a2, a3, a2
+; RV32I-NEXT:    and s0, s0, s1
+; RV32I-NEXT:    or a2, a2, s0
 ; RV32I-NEXT:    and a0, a0, a2
 ; RV32I-NEXT:    lw ra, 28(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    lw s0, 24(sp) # 4-byte Folded Reload
@@ -2653,12 +2653,12 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; CHECK32-IZHINXMIN-NEXT:    addi a2, a3, -1
 ; CHECK32-IZHINXMIN-NEXT:  .LBB10_4: # %start
 ; CHECK32-IZHINXMIN-NEXT:    feq.s a3, s0, s0
-; CHECK32-IZHINXMIN-NEXT:    neg a4, a1
-; CHECK32-IZHINXMIN-NEXT:    neg a1, s1
+; CHECK32-IZHINXMIN-NEXT:    neg a4, s1
+; CHECK32-IZHINXMIN-NEXT:    neg a5, a1
 ; CHECK32-IZHINXMIN-NEXT:    neg a3, a3
-; CHECK32-IZHINXMIN-NEXT:    and a0, a1, a0
+; CHECK32-IZHINXMIN-NEXT:    and a0, a4, a0
 ; CHECK32-IZHINXMIN-NEXT:    and a1, a3, a2
-; CHECK32-IZHINXMIN-NEXT:    or a0, a4, a0
+; CHECK32-IZHINXMIN-NEXT:    or a0, a5, a0
 ; CHECK32-IZHINXMIN-NEXT:    and a0, a3, a0
 ; CHECK32-IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK32-IZHINXMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
@@ -2701,12 +2701,12 @@ define i64 @fcvt_l_h_sat(half %a) nounwind {
 ; CHECK32-IZDINXZHINXMIN-NEXT:    addi a2, a3, -1
 ; CHECK32-IZDINXZHINXMIN-NEXT:  .LBB10_4: # %start
 ; CHECK32-IZDINXZHINXMIN-NEXT:    feq.s a3, s0, s0
-; CHECK32-IZDINXZHINXMIN-NEXT:    neg a4, a1
-; CHECK32-IZDINXZHINXMIN-NEXT:    neg a1, s1
+; CHECK32-IZDINXZHINXMIN-NEXT:    neg a4, s1
+; CHECK32-IZDINXZHINXMIN-NEXT:    neg a5, a1
 ; CHECK32-IZDINXZHINXMIN-NEXT:    neg a3, a3
-; CHECK32-IZDINXZHINXMIN-NEXT:    and a0, a1, a0
+; CHECK32-IZDINXZHINXMIN-NEXT:    and a0, a4, a0
 ; CHECK32-IZDINXZHINXMIN-NEXT:    and a1, a3, a2
-; CHECK32-IZDINXZHINXMIN-NEXT:    or a0, a4, a0
+; CHECK32-IZDINXZHINXMIN-NEXT:    or a0, a5, a0
 ; CHECK32-IZDINXZHINXMIN-NEXT:    and a0, a3, a0
 ; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK32-IZDINXZHINXMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
@@ -2972,18 +2972,19 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind {
 ; RV32IZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IZHINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IZHINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
-; RV32IZHINX-NEXT:    fcvt.s.h a0, a0
-; RV32IZHINX-NEXT:    lui a1, 391168
-; RV32IZHINX-NEXT:    addi a1, a1, -1
-; RV32IZHINX-NEXT:    fle.s a2, zero, a0
-; RV32IZHINX-NEXT:    flt.s a1, a1, a0
-; RV32IZHINX-NEXT:    neg s0, a1
-; RV32IZHINX-NEXT:    neg s1, a2
+; RV32IZHINX-NEXT:    fcvt.s.h s0, a0
+; RV32IZHINX-NEXT:    fle.s a0, zero, s0
+; RV32IZHINX-NEXT:    neg s1, a0
+; RV32IZHINX-NEXT:    mv a0, s0
 ; RV32IZHINX-NEXT:    call __fixunssfdi
 ; RV32IZHINX-NEXT:    and a0, s1, a0
+; RV32IZHINX-NEXT:    lui a2, 391168
 ; RV32IZHINX-NEXT:    and a1, s1, a1
-; RV32IZHINX-NEXT:    or a0, s0, a0
-; RV32IZHINX-NEXT:    or a1, s0, a1
+; RV32IZHINX-NEXT:    addi a2, a2, -1
+; RV32IZHINX-NEXT:    flt.s a2, a2, s0
+; RV32IZHINX-NEXT:    neg a2, a2
+; RV32IZHINX-NEXT:    or a0, a2, a0
+; RV32IZHINX-NEXT:    or a1, a2, a1
 ; RV32IZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZHINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -3005,18 +3006,19 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind {
 ; RV32IZDINXZHINX-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; RV32IZDINXZHINX-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; RV32IZDINXZHINX-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
-; RV32IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
-; RV32IZDINXZHINX-NEXT:    lui a1, 391168
-; RV32IZDINXZHINX-NEXT:    addi a1, a1, -1
-; RV32IZDINXZHINX-NEXT:    fle.s a2, zero, a0
-; RV32IZDINXZHINX-NEXT:    flt.s a1, a1, a0
-; RV32IZDINXZHINX-NEXT:    neg s0, a1
-; RV32IZDINXZHINX-NEXT:    neg s1, a2
+; RV32IZDINXZHINX-NEXT:    fcvt.s.h s0, a0
+; RV32IZDINXZHINX-NEXT:    fle.s a0, zero, s0
+; RV32IZDINXZHINX-NEXT:    neg s1, a0
+; RV32IZDINXZHINX-NEXT:    mv a0, s0
 ; RV32IZDINXZHINX-NEXT:    call __fixunssfdi
 ; RV32IZDINXZHINX-NEXT:    and a0, s1, a0
+; RV32IZDINXZHINX-NEXT:    lui a2, 391168
 ; RV32IZDINXZHINX-NEXT:    and a1, s1, a1
-; RV32IZDINXZHINX-NEXT:    or a0, s0, a0
-; RV32IZDINXZHINX-NEXT:    or a1, s0, a1
+; RV32IZDINXZHINX-NEXT:    addi a2, a2, -1
+; RV32IZDINXZHINX-NEXT:    flt.s a2, a2, s0
+; RV32IZDINXZHINX-NEXT:    neg a2, a2
+; RV32IZDINXZHINX-NEXT:    or a0, a2, a0
+; RV32IZDINXZHINX-NEXT:    or a1, a2, a1
 ; RV32IZDINXZHINX-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32IZDINXZHINX-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; RV32IZDINXZHINX-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -3049,7 +3051,7 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi s2, a0, -1
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call __fixunssfdi
@@ -3083,7 +3085,7 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    li a1, 0
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    addi s2, a0, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __fixunssfdi
@@ -3217,18 +3219,19 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind {
 ; CHECK32-IZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK32-IZHINXMIN-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; CHECK32-IZHINXMIN-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
-; CHECK32-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZHINXMIN-NEXT:    lui a1, 391168
-; CHECK32-IZHINXMIN-NEXT:    addi a1, a1, -1
-; CHECK32-IZHINXMIN-NEXT:    fle.s a2, zero, a0
-; CHECK32-IZHINXMIN-NEXT:    flt.s a1, a1, a0
-; CHECK32-IZHINXMIN-NEXT:    neg s0, a1
-; CHECK32-IZHINXMIN-NEXT:    neg s1, a2
+; CHECK32-IZHINXMIN-NEXT:    fcvt.s.h s0, a0
+; CHECK32-IZHINXMIN-NEXT:    fle.s a0, zero, s0
+; CHECK32-IZHINXMIN-NEXT:    neg s1, a0
+; CHECK32-IZHINXMIN-NEXT:    mv a0, s0
 ; CHECK32-IZHINXMIN-NEXT:    call __fixunssfdi
 ; CHECK32-IZHINXMIN-NEXT:    and a0, s1, a0
+; CHECK32-IZHINXMIN-NEXT:    lui a2, 391168
 ; CHECK32-IZHINXMIN-NEXT:    and a1, s1, a1
-; CHECK32-IZHINXMIN-NEXT:    or a0, s0, a0
-; CHECK32-IZHINXMIN-NEXT:    or a1, s0, a1
+; CHECK32-IZHINXMIN-NEXT:    addi a2, a2, -1
+; CHECK32-IZHINXMIN-NEXT:    flt.s a2, a2, s0
+; CHECK32-IZHINXMIN-NEXT:    neg a2, a2
+; CHECK32-IZHINXMIN-NEXT:    or a0, a2, a0
+; CHECK32-IZHINXMIN-NEXT:    or a1, a2, a1
 ; CHECK32-IZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK32-IZHINXMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; CHECK32-IZHINXMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -3251,18 +3254,19 @@ define i64 @fcvt_lu_h_sat(half %a) nounwind {
 ; CHECK32-IZDINXZHINXMIN-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK32-IZDINXZHINXMIN-NEXT:    sw s0, 8(sp) # 4-byte Folded Spill
 ; CHECK32-IZDINXZHINXMIN-NEXT:    sw s1, 4(sp) # 4-byte Folded Spill
-; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    lui a1, 391168
-; CHECK32-IZDINXZHINXMIN-NEXT:    addi a1, a1, -1
-; CHECK32-IZDINXZHINXMIN-NEXT:    fle.s a2, zero, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    flt.s a1, a1, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    neg s0, a1
-; CHECK32-IZDINXZHINXMIN-NEXT:    neg s1, a2
+; CHECK32-IZDINXZHINXMIN-NEXT:    fcvt.s.h s0, a0
+; CHECK32-IZDINXZHINXMIN-NEXT:    fle.s a0, zero, s0
+; CHECK32-IZDINXZHINXMIN-NEXT:    neg s1, a0
+; CHECK32-IZDINXZHINXMIN-NEXT:    mv a0, s0
 ; CHECK32-IZDINXZHINXMIN-NEXT:    call __fixunssfdi
 ; CHECK32-IZDINXZHINXMIN-NEXT:    and a0, s1, a0
+; CHECK32-IZDINXZHINXMIN-NEXT:    lui a2, 391168
 ; CHECK32-IZDINXZHINXMIN-NEXT:    and a1, s1, a1
-; CHECK32-IZDINXZHINXMIN-NEXT:    or a0, s0, a0
-; CHECK32-IZDINXZHINXMIN-NEXT:    or a1, s0, a1
+; CHECK32-IZDINXZHINXMIN-NEXT:    addi a2, a2, -1
+; CHECK32-IZDINXZHINXMIN-NEXT:    flt.s a2, a2, s0
+; CHECK32-IZDINXZHINXMIN-NEXT:    neg a2, a2
+; CHECK32-IZDINXZHINXMIN-NEXT:    or a0, a2, a0
+; CHECK32-IZDINXZHINXMIN-NEXT:    or a1, a2, a1
 ; CHECK32-IZDINXZHINXMIN-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK32-IZDINXZHINXMIN-NEXT:    lw s0, 8(sp) # 4-byte Folded Reload
 ; CHECK32-IZDINXZHINXMIN-NEXT:    lw s1, 4(sp) # 4-byte Folded Reload
@@ -6908,8 +6912,8 @@ define zeroext i16 @fcvt_wu_s_sat_i16(half %a) nounwind {
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    j .LBB34_3
 ; RV32I-NEXT:  .LBB34_2:
-; RV32I-NEXT:    slti a0, s1, 0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    srli s1, s1, 31
+; RV32I-NEXT:    addi a0, s1, -1
 ; RV32I-NEXT:    and a0, a0, s0
 ; RV32I-NEXT:  .LBB34_3: # %start
 ; RV32I-NEXT:    and a0, a0, s3
@@ -6949,8 +6953,8 @@ define zeroext i16 @fcvt_wu_s_sat_i16(half %a) nounwind {
 ; RV64I-NEXT:    mv a0, s3
 ; RV64I-NEXT:    j .LBB34_3
 ; RV64I-NEXT:  .LBB34_2:
-; RV64I-NEXT:    slti a0, s1, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s1, s1, 63
+; RV64I-NEXT:    addi a0, s1, -1
 ; RV64I-NEXT:    and a0, a0, s0
 ; RV64I-NEXT:  .LBB34_3: # %start
 ; RV64I-NEXT:    and a0, a0, s3
@@ -7852,8 +7856,8 @@ define zeroext i8 @fcvt_wu_s_sat_i8(half %a) nounwind {
 ; RV32I-NEXT:    li a0, 255
 ; RV32I-NEXT:    j .LBB38_3
 ; RV32I-NEXT:  .LBB38_2:
-; RV32I-NEXT:    slti a0, s0, 0
-; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    srli s0, s0, 31
+; RV32I-NEXT:    addi a0, s0, -1
 ; RV32I-NEXT:    and a0, a0, s1
 ; RV32I-NEXT:  .LBB38_3: # %start
 ; RV32I-NEXT:    zext.b a0, a0
@@ -7889,8 +7893,8 @@ define zeroext i8 @fcvt_wu_s_sat_i8(half %a) nounwind {
 ; RV64I-NEXT:    li a0, 255
 ; RV64I-NEXT:    j .LBB38_3
 ; RV64I-NEXT:  .LBB38_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB38_3: # %start
 ; RV64I-NEXT:    zext.b a0, a0
@@ -8126,7 +8130,7 @@ define zeroext i32 @fcvt_wu_h_sat_zext(half %a) nounwind {
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    call __gesf2
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi s2, a0, -1
 ; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    call __fixunssfsi
@@ -8166,8 +8170,8 @@ define zeroext i32 @fcvt_wu_h_sat_zext(half %a) nounwind {
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    j .LBB39_3
 ; RV64I-NEXT:  .LBB39_2:
-; RV64I-NEXT:    slti a0, s0, 0
-; RV64I-NEXT:    addi a0, a0, -1
+; RV64I-NEXT:    srli s0, s0, 63
+; RV64I-NEXT:    addi a0, s0, -1
 ; RV64I-NEXT:    and a0, a0, s1
 ; RV64I-NEXT:  .LBB39_3: # %start
 ; RV64I-NEXT:    slli a0, a0, 32
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-fixed-v-constraint.ll b/llvm/test/CodeGen/RISCV/inline-asm-fixed-v-constraint.ll
new file mode 100644
index 0000000000000..2c698adc201f9
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/inline-asm-fixed-v-constraint.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+define <1 x i8> @constraint_vr_fixed(<1 x i8> %0, <1 x i8> %1) nounwind {
+; RV32I-LABEL: constraint_vr_fixed:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    vadd.vv v8, v8, v9
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: constraint_vr_fixed:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    vadd.vv v8, v8, v9
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %a = tail call <1 x i8> asm "vadd.vv $0, $1, $2", "=^vr,^vr,^vr"(
+    <1 x i8> %0, <1 x i8> %1)
+  ret <1 x i8> %a
+}
+
+define <4 x i32> @constraint_vd_fixed(<4 x i32> %0, <4 x i32> %1) nounwind {
+; RV32I-LABEL: constraint_vd_fixed:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    vadd.vv v8, v8, v9
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: constraint_vd_fixed:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    vadd.vv v8, v8, v9
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %a = tail call <4 x i32> asm "vadd.vv $0, $1, $2", "=^vd,^vr,^vr"(
+    <4 x i32> %0, <4 x i32> %1)
+  ret <4 x i32> %a
+}
+
+define <16 x i1> @constraint_vm_fixed(<16 x i1> %0, <16 x i1> %1) nounwind {
+; RV32I-LABEL: constraint_vm_fixed:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
+; RV32I-NEXT:    vmv1r.v v9, v0
+; RV32I-NEXT:    vmv1r.v v0, v8
+; RV32I-NEXT:    #APP
+; RV32I-NEXT:    vadd.vv v0, v9, v0
+; RV32I-NEXT:    #NO_APP
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: constraint_vm_fixed:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
+; RV64I-NEXT:    vmv1r.v v9, v0
+; RV64I-NEXT:    vmv1r.v v0, v8
+; RV64I-NEXT:    #APP
+; RV64I-NEXT:    vadd.vv v0, v9, v0
+; RV64I-NEXT:    #NO_APP
+; RV64I-NEXT:    ret
+  %a = tail call <16 x i1> asm "vadd.vv $0, $1, $2", "=^vr,^vr,^vm"(
+    <16 x i1> %0, <16 x i1> %1)
+  ret <16 x i1> %a
+}
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr-args-error.ll b/llvm/test/CodeGen/RISCV/interrupt-attr-args-error.ll
index f15f565c39fdd..edc65bffb33ec 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr-args-error.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr-args-error.ll
@@ -1,6 +1,6 @@
-; RUN: not --crash llc -mtriple riscv32-unknown-elf -o - %s \
+; RUN: not llc -mtriple riscv32-unknown-elf -o - %s \
 ; RUN: 2>&1 | FileCheck %s
-; RUN: not --crash llc -mtriple riscv64-unknown-elf -o - %s \
+; RUN: not llc -mtriple riscv64-unknown-elf -o - %s \
 ; RUN: 2>&1 | FileCheck %s
 
 ; CHECK: LLVM ERROR: Functions with the interrupt attribute cannot have arguments!
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr-invalid.ll b/llvm/test/CodeGen/RISCV/interrupt-attr-invalid.ll
index 2bcec1589df02..bddca8af7468f 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr-invalid.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr-invalid.ll
@@ -1,6 +1,6 @@
-; RUN: not --crash llc -mtriple riscv32-unknown-elf -o - %s \
+; RUN: not llc -mtriple riscv32-unknown-elf -o - %s \
 ; RUN: 2>&1 | FileCheck %s
-; RUN: not --crash llc -mtriple riscv64-unknown-elf -o - %s \
+; RUN: not llc -mtriple riscv64-unknown-elf -o - %s \
 ; RUN: 2>&1 | FileCheck %s
 
 ; CHECK: LLVM ERROR: Function interrupt attribute argument not supported!
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr-ret-error.ll b/llvm/test/CodeGen/RISCV/interrupt-attr-ret-error.ll
index 30fd8fb91897d..eec9625825008 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr-ret-error.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr-ret-error.ll
@@ -1,6 +1,6 @@
-; RUN: not --crash llc -mtriple riscv32-unknown-elf -o - %s \
+; RUN: not llc -mtriple riscv32-unknown-elf -o - %s \
 ; RUN: 2>&1 | FileCheck %s
-; RUN: not --crash llc -mtriple riscv64-unknown-elf -o - %s \
+; RUN: not llc -mtriple riscv64-unknown-elf -o - %s \
 ; RUN: 2>&1 | FileCheck %s
 
 ; CHECK: LLVM ERROR: Functions with the interrupt attribute must have void return type!
diff --git a/llvm/test/CodeGen/RISCV/kcfi-mir.ll b/llvm/test/CodeGen/RISCV/kcfi-mir.ll
index 2efdc129f2621..9785fb746a0d2 100644
--- a/llvm/test/CodeGen/RISCV/kcfi-mir.ll
+++ b/llvm/test/CodeGen/RISCV/kcfi-mir.ll
@@ -10,7 +10,7 @@ define void @f1(ptr noundef %x) !kcfi_type !1 {
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION def_cfa_offset 16
   ; CHECK-NEXT:   frame-setup SD killed $x1, $x2, 8 :: (store (s64) into %stack.0)
   ; CHECK-NEXT:   frame-setup CFI_INSTRUCTION offset $x1, -8
-  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x6_w, implicit-def $x6_h, implicit-def $x7, implicit-def $x7_w, implicit-def $x7_h, implicit-def $x28, implicit-def $x28_w, implicit-def $x28_h, implicit-def $x29, implicit-def $x29_w, implicit-def $x29_h, implicit-def $x30, implicit-def $x30_w, implicit-def $x30_h, implicit-def $x31, implicit-def $x31_w, implicit-def $x31_h, implicit-def dead $x1, implicit-def $x2, implicit-def $x2_w, implicit-def $x2_h, implicit killed $x10 {
+  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31, implicit-def dead $x1, implicit-def $x2, implicit killed $x10 {
   ; CHECK-NEXT:     KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
   ; CHECK-NEXT:     PseudoCALLIndirect killed $x10, csr_ilp32_lp64, implicit-def dead $x1, implicit-def $x2
   ; CHECK-NEXT:   }
@@ -28,7 +28,7 @@ define void @f2(ptr noundef %x) #0 {
   ; CHECK: bb.0 (%ir-block.0):
   ; CHECK-NEXT:   liveins: $x10
   ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x6_w, implicit-def $x6_h, implicit-def $x7, implicit-def $x7_w, implicit-def $x7_h, implicit-def $x28, implicit-def $x28_w, implicit-def $x28_h, implicit-def $x29, implicit-def $x29_w, implicit-def $x29_h, implicit-def $x30, implicit-def $x30_w, implicit-def $x30_h, implicit-def $x31, implicit-def $x31_w, implicit-def $x31_h, implicit killed $x10, implicit $x2 {
+  ; CHECK-NEXT:   BUNDLE implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31, implicit killed $x10, implicit $x2 {
   ; CHECK-NEXT:     KCFI_CHECK $x10, 12345678, implicit-def $x6, implicit-def $x7, implicit-def $x28, implicit-def $x29, implicit-def $x30, implicit-def $x31
   ; CHECK-NEXT:     PseudoTAILIndirect killed $x10, csr_ilp32_lp64, implicit $x2
   ; CHECK-NEXT:   }
diff --git a/llvm/test/CodeGen/RISCV/live-sp.mir b/llvm/test/CodeGen/RISCV/live-sp.mir
index 1c4c6e43a9234..6da655ced8125 100644
--- a/llvm/test/CodeGen/RISCV/live-sp.mir
+++ b/llvm/test/CodeGen/RISCV/live-sp.mir
@@ -54,8 +54,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/RISCV/lrint-conv.ll b/llvm/test/CodeGen/RISCV/lrint-conv.ll
new file mode 100644
index 0000000000000..d3af2153588a1
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/lrint-conv.ll
@@ -0,0 +1,76 @@
+; Tests for lrint and llrint, with both i32 and i64 checked.
+
+; RUN: sed 's/ITy/i32/g' %s | llc -mtriple=riscv32 | FileCheck %s --check-prefixes=RV32
+; RUN: sed 's/ITy/i64/g' %s | llc -mtriple=riscv32 | FileCheck %s --check-prefixes=RV32
+; RUN: sed 's/ITy/i32/g' %s | llc -mtriple=riscv64 | FileCheck %s --check-prefixes=RV64
+; RUN: sed 's/ITy/i64/g' %s | llc -mtriple=riscv64 | FileCheck %s --check-prefixes=RV64
+
+; FIXME: crash
+; define ITy @test_lrint_ixx_f16(half %x) nounwind {
+;   %res = tail call ITy @llvm.lrint.ITy.f16(half %x)
+; }
+
+; define ITy @test_llrint_ixx_f16(half %x) nounwind {
+;   %res = tail call ITy @llvm.llrint.ITy.f16(half %x)
+; }
+
+define ITy @test_lrint_ixx_f32(float %x) nounwind {
+; RV32-LABEL: test_lrint_ixx_f32:
+; RV32:         call lrintf
+;
+; RV64-LABEL: test_lrint_ixx_f32:
+; RV64:         call lrintf
+  %res = tail call ITy @llvm.lrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f32(float %x) nounwind {
+; RV32-LABEL: test_llrint_ixx_f32:
+; RV32:         call llrintf
+;
+; RV64-LABEL: test_llrint_ixx_f32:
+; RV64:         call llrintf
+  %res = tail call ITy @llvm.llrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_lrint_ixx_f64(double %x) nounwind {
+; RV32-LABEL: test_lrint_ixx_f64:
+; RV32:         call lrint
+;
+; RV64-LABEL: test_lrint_ixx_f64:
+; RV64:         call lrint
+  %res = tail call ITy @llvm.lrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f64(double %x) nounwind {
+; RV32-LABEL: test_llrint_ixx_f64:
+; RV32:         call llrint
+;
+; RV64-LABEL: test_llrint_ixx_f64:
+; RV64:         call llrint
+  %res = tail call ITy @llvm.llrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+; FIXME(#44744): incorrect libcall on riscv32
+define ITy @test_lrint_ixx_f128(fp128 %x) nounwind {
+; RV32-LABEL: test_lrint_ixx_f128:
+; RV32:         call lrintl
+;
+; RV64-LABEL: test_lrint_ixx_f128:
+; RV64:         call lrintl
+  %res = tail call ITy @llvm.lrint.ITy.f128(fp128 %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f128(fp128 %x) nounwind {
+; RV32-LABEL: test_llrint_ixx_f128:
+; RV32:         call llrintl
+;
+; RV64-LABEL: test_llrint_ixx_f128:
+; RV64:         call llrintl
+  %res = tail call ITy @llvm.llrint.ITy.f128(fp128 %x)
+  ret ITy %res
+}
diff --git a/llvm/test/CodeGen/RISCV/macro-fusions.mir b/llvm/test/CodeGen/RISCV/macro-fusions.mir
index 13464141ce27e..ae5b52da2ac16 100644
--- a/llvm/test/CodeGen/RISCV/macro-fusions.mir
+++ b/llvm/test/CodeGen/RISCV/macro-fusions.mir
@@ -2,7 +2,12 @@
 # RUN: llc -mtriple=riscv64-linux-gnu -x=mir < %s \
 # RUN:   -debug-only=machine-scheduler -start-before=machine-scheduler 2>&1 \
 # RUN:   -mattr=+lui-addi-fusion,+auipc-addi-fusion,+zexth-fusion,+zextw-fusion,+shifted-zextw-fusion,+ld-add-fusion \
+# RUN:   -mattr=+add-load-fusion,+auipc-load-fusion,+lui-load-fusion,+addi-load-fusion \
+# RUN:   -mattr=+zba,+shxadd-load-fusion \
 # RUN:   | FileCheck %s
+# RUN: llc -mtriple=riscv64-linux-gnu -x=mir < %s \
+# RUN:   -debug-only=machine-scheduler -start-before=machine-scheduler 2>&1 \
+# RUN:   -mattr=+zba,+bfext-fusion | FileCheck --check-prefixes=CHECK-BFEXT %s
 
 # CHECK: lui_addi:%bb.0
 # CHECK: Macro fuse: {{.*}}LUI - ADDI
@@ -174,3 +179,1374 @@ body:             |
     $x11 = COPY %5
     PseudoRET
 ...
+
+# CHECK: add_lb
+# CHECK: Macro fuse: {{.*}}ADD - LB
+---
+name: add_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LB %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: add_lh
+# CHECK: Macro fuse: {{.*}}ADD - LH
+---
+name: add_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LH %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: add_lw
+# CHECK: Macro fuse: {{.*}}ADD - LW
+---
+name: add_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LW %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: add_lbu
+# CHECK: Macro fuse: {{.*}}ADD - LBU
+---
+name: add_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LBU %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: add_lhu
+# CHECK: Macro fuse: {{.*}}ADD - LHU
+---
+name: add_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LHU %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: add_lwu
+# CHECK: Macro fuse: {{.*}}ADD - LWU
+---
+name: add_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LWU %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: auipc_lb
+# CHECK: Macro fuse: {{.*}}AUIPC - LB
+---
+name: auipc_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = AUIPC 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LB %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: auipc_lh
+# CHECK: Macro fuse: {{.*}}AUIPC - LH
+---
+name: auipc_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = AUIPC 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LH %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: auipc_lw
+# CHECK: Macro fuse: {{.*}}AUIPC - LW
+---
+name: auipc_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = AUIPC 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LW %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: auipc_ld
+# CHECK: Macro fuse: {{.*}}AUIPC - LD
+---
+name: auipc_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = AUIPC 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LD %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: auipc_lbu
+# CHECK: Macro fuse: {{.*}}AUIPC - LBU
+---
+name: auipc_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = AUIPC 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LBU %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: auipc_lhu
+# CHECK: Macro fuse: {{.*}}AUIPC - LHU
+---
+name: auipc_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = AUIPC 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LHU %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: auipc_lwu
+# CHECK: Macro fuse: {{.*}}AUIPC - LWU
+---
+name: auipc_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = AUIPC 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LWU %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: lui_lb
+# CHECK: Macro fuse: {{.*}}LUI - LB
+---
+name: lui_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = LUI 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LB %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: lui_lh
+# CHECK: Macro fuse: {{.*}}LUI - LH
+---
+name: lui_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = LUI 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LH %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: lui_lw
+# CHECK: Macro fuse: {{.*}}LUI - LW
+---
+name: lui_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = LUI 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LW %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: lui_ld
+# CHECK: Macro fuse: {{.*}}LUI - LD
+---
+name: lui_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = LUI 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LD %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: lui_lbu
+# CHECK: Macro fuse: {{.*}}LUI - LBU
+---
+name: lui_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = LUI 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LBU %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: lui_lhu
+# CHECK: Macro fuse: {{.*}}LUI - LHU
+---
+name: lui_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = LUI 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LHU %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: lui_lwu
+# CHECK: Macro fuse: {{.*}}LUI - LWU
+---
+name: lui_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = LUI 1
+    %3:gpr = XORI %1, 2
+    %4:gpr = LWU %2, 4
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK-BFEXT: bitfield_extract
+# CHECK-BFEXT: Macro fuse: {{.*}}SLLI - SRLI
+---
+name: bitfield_extract
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    %1:gpr = COPY $x10
+    %2:gpr = SLLI %1, 31
+    %3:gpr = XORI %1, 3
+    %4:gpr = SRLI %2, 48
+    $x10 = COPY %3
+    $x11 = COPY %4
+    PseudoRET
+...
+
+# CHECK: addi_lb
+# CHECK: Macro fuse: {{.*}}ADDI - LB
+---
+name: addi_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADDI %1, 8
+    %4:gpr = XORI %2, 3
+    %5:gpr = LB %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: addi_lh
+# CHECK: Macro fuse: {{.*}}ADDI - LH
+---
+name: addi_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADDI %1, 8
+    %4:gpr = XORI %2, 3
+    %5:gpr = LH %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: addi_lw
+# CHECK: Macro fuse: {{.*}}ADDI - LW
+---
+name: addi_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADDI %1, 8
+    %4:gpr = XORI %2, 3
+    %5:gpr = LW %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: addi_ld
+# CHECK: Macro fuse: {{.*}}ADDI - LD
+---
+name: addi_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADDI %1, 8
+    %4:gpr = XORI %2, 3
+    %5:gpr = LD %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: addi_lbu
+# CHECK: Macro fuse: {{.*}}ADDI - LBU
+---
+name: addi_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADDI %1, 8
+    %4:gpr = XORI %2, 3
+    %5:gpr = LBU %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: addi_lhu
+# CHECK: Macro fuse: {{.*}}ADDI - LHU
+---
+name: addi_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADDI %1, 8
+    %4:gpr = XORI %2, 3
+    %5:gpr = LHU %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: addi_lwu
+# CHECK: Macro fuse: {{.*}}ADDI - LWU
+---
+name: addi_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADDI %1, 8
+    %4:gpr = XORI %2, 3
+    %5:gpr = LWU %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: adduw_lb
+# CHECK: Macro fuse: {{.*}}ADD_UW - LB
+---
+name: adduw_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LB %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: adduw_lh
+# CHECK: Macro fuse: {{.*}}ADD_UW - LH
+---
+name: adduw_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LH %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: adduw_lw
+# CHECK: Macro fuse: {{.*}}ADD_UW - LW
+---
+name: adduw_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LW %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: adduw_ld
+# CHECK: Macro fuse: {{.*}}ADD_UW - LD
+---
+name: adduw_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LD %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: adduw_lbu
+# CHECK: Macro fuse: {{.*}}ADD_UW - LBU
+---
+name: adduw_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LBU %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: adduw_lhu
+# CHECK: Macro fuse: {{.*}}ADD_UW - LHU
+---
+name: adduw_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LHU %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: adduw_lwu
+# CHECK: Macro fuse: {{.*}}ADD_UW - LWU
+---
+name: adduw_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LWU %3, 0
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1add_lb
+# CHECK: Macro fuse: {{.*}}SH1ADD - LB
+---
+name: sh1add_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LB %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2add_lb
+# CHECK: Macro fuse: {{.*}}SH2ADD - LB
+---
+name: sh2add_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LB %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3add_lb
+# CHECK: Macro fuse: {{.*}}SH3ADD - LB
+---
+name: sh3add_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LB %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1add_lh
+# CHECK: Macro fuse: {{.*}}SH1ADD - LH
+---
+name: sh1add_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LH %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2add_lh
+# CHECK: Macro fuse: {{.*}}SH2ADD - LH
+---
+name: sh2add_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LH %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3add_lh
+# CHECK: Macro fuse: {{.*}}SH3ADD - LH
+---
+name: sh3add_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LH %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1add_lw
+# CHECK: Macro fuse: {{.*}}SH1ADD - LW
+---
+name: sh1add_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LW %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2add_lw
+# CHECK: Macro fuse: {{.*}}SH2ADD - LW
+---
+name: sh2add_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LW %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3add_lw
+# CHECK: Macro fuse: {{.*}}SH3ADD - LW
+---
+name: sh3add_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LW %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1add_ld
+# CHECK: Macro fuse: {{.*}}SH1ADD - LD
+---
+name: sh1add_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LD %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2add_ld
+# CHECK: Macro fuse: {{.*}}SH2ADD - LD
+---
+name: sh2add_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LD %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3add_ld
+# CHECK: Macro fuse: {{.*}}SH3ADD - LD
+---
+name: sh3add_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LD %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1add_lbu
+# CHECK: Macro fuse: {{.*}}SH1ADD - LBU
+---
+name: sh1add_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LBU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2add_lbu
+# CHECK: Macro fuse: {{.*}}SH2ADD - LBU
+---
+name: sh2add_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LBU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3add_lbu
+# CHECK: Macro fuse: {{.*}}SH3ADD - LBU
+---
+name: sh3add_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LBU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1add_lhu
+# CHECK: Macro fuse: {{.*}}SH1ADD - LHU
+---
+name: sh1add_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LHU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2add_lhu
+# CHECK: Macro fuse: {{.*}}SH2ADD - LHU
+---
+name: sh2add_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LHU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3add_lhu
+# CHECK: Macro fuse: {{.*}}SH3ADD - LHU
+---
+name: sh3add_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LHU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1add_lwu
+# CHECK: Macro fuse: {{.*}}SH1ADD - LWU
+---
+name: sh1add_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LWU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2add_lwu
+# CHECK: Macro fuse: {{.*}}SH2ADD - LWU
+---
+name: sh2add_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LWU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3add_lwu
+# CHECK: Macro fuse: {{.*}}SH3ADD - LWU
+---
+name: sh3add_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LWU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1adduw_lb
+# CHECK: Macro fuse: {{.*}}SH1ADD_UW - LB
+---
+name: sh1adduw_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LB %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2adduw_lb
+# CHECK: Macro fuse: {{.*}}SH2ADD_UW - LB
+---
+name: sh2adduw_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LB %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3adduw_lb
+# CHECK: Macro fuse: {{.*}}SH3ADD_UW - LB
+---
+name: sh3adduw_lb
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LB %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1adduw_lh
+# CHECK: Macro fuse: {{.*}}SH1ADD_UW - LH
+---
+name: sh1adduw_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LH %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2adduw_lh
+# CHECK: Macro fuse: {{.*}}SH2ADD_UW - LH
+---
+name: sh2adduw_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LH %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3adduw_lh
+# CHECK: Macro fuse: {{.*}}SH3ADD_UW - LH
+---
+name: sh3adduw_lh
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LH %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1adduw_lw
+# CHECK: Macro fuse: {{.*}}SH1ADD_UW - LW
+---
+name: sh1adduw_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LW %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2adduw_lw
+# CHECK: Macro fuse: {{.*}}SH2ADD_UW - LW
+---
+name: sh2adduw_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LW %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3adduw_lw
+# CHECK: Macro fuse: {{.*}}SH3ADD_UW - LW
+---
+name: sh3adduw_lw
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LW %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1adduw_ld
+# CHECK: Macro fuse: {{.*}}SH1ADD_UW - LD
+---
+name: sh1adduw_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LD %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2adduw_ld
+# CHECK: Macro fuse: {{.*}}SH2ADD_UW - LD
+---
+name: sh2adduw_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LD %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3adduw_ld
+# CHECK: Macro fuse: {{.*}}SH3ADD_UW - LD
+---
+name: sh3adduw_ld
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LD %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1adduw_lbu
+# CHECK: Macro fuse: {{.*}}SH1ADD_UW - LBU
+---
+name: sh1adduw_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LBU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2adduw_lbu
+# CHECK: Macro fuse: {{.*}}SH2ADD_UW - LBU
+---
+name: sh2adduw_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LBU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3adduw_lbu
+# CHECK: Macro fuse: {{.*}}SH3ADD_UW - LBU
+---
+name: sh3adduw_lbu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LBU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1adduw_lhu
+# CHECK: Macro fuse: {{.*}}SH1ADD_UW - LHU
+---
+name: sh1adduw_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LHU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2adduw_lhu
+# CHECK: Macro fuse: {{.*}}SH2ADD_UW - LHU
+---
+name: sh2adduw_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LHU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3adduw_lhu
+# CHECK: Macro fuse: {{.*}}SH3ADD_UW - LHU
+---
+name: sh3adduw_lhu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LHU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh1adduw_lwu
+# CHECK: Macro fuse: {{.*}}SH1ADD_UW - LWU
+---
+name: sh1adduw_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH1ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LWU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh2adduw_lwu
+# CHECK: Macro fuse: {{.*}}SH2ADD_UW - LWU
+---
+name: sh2adduw_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH2ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LWU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
+
+# CHECK: sh3adduw_lwu
+# CHECK: Macro fuse: {{.*}}SH3ADD_UW - LWU
+---
+name: sh3adduw_lwu
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SH3ADD_UW %1, %2
+    %4:gpr = XORI %2, 3
+    %5:gpr = LWU %3, 8
+    $x10 = COPY %4
+    $x11 = COPY %5
+    PseudoRET
+...
diff --git a/llvm/test/CodeGen/RISCV/memcmp-optsize.ll b/llvm/test/CodeGen/RISCV/memcmp-optsize.ll
index cd9357994742b..afc8e3553f8b1 100644
--- a/llvm/test/CodeGen/RISCV/memcmp-optsize.ll
+++ b/llvm/test/CodeGen/RISCV/memcmp-optsize.ll
@@ -2741,7 +2741,7 @@ define i1 @bcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-NEXT:    ret
@@ -2763,7 +2763,7 @@ define i1 @bcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
@@ -2785,7 +2785,7 @@ define i1 @bcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
@@ -2807,7 +2807,7 @@ define i1 @bcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-V-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-V-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
@@ -5549,7 +5549,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-NEXT:    ret
@@ -5571,7 +5571,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
@@ -5593,7 +5593,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
@@ -5615,7 +5615,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-V-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-V-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-V-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
@@ -5637,7 +5637,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-UNALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-UNALIGNED-RV64-NEXT:    li a2, 4
 ; CHECK-UNALIGNED-RV64-NEXT:    call memcmp
-; CHECK-UNALIGNED-RV64-NEXT:    slti a0, a0, 0
+; CHECK-UNALIGNED-RV64-NEXT:    srli a0, a0, 63
 ; CHECK-UNALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-UNALIGNED-RV64-NEXT:    ret
@@ -5699,7 +5699,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-UNALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 4
 ; CHECK-UNALIGNED-RV64-V-NEXT:    call memcmp
-; CHECK-UNALIGNED-RV64-V-NEXT:    slti a0, a0, 0
+; CHECK-UNALIGNED-RV64-V-NEXT:    srli a0, a0, 63
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/memcmp.ll b/llvm/test/CodeGen/RISCV/memcmp.ll
index a5bdb13d37fb8..c737edb9acce9 100644
--- a/llvm/test/CodeGen/RISCV/memcmp.ll
+++ b/llvm/test/CodeGen/RISCV/memcmp.ll
@@ -3161,7 +3161,7 @@ define i1 @bcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-NEXT:    ret
@@ -3183,7 +3183,7 @@ define i1 @bcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
@@ -3205,7 +3205,7 @@ define i1 @bcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
@@ -3227,7 +3227,7 @@ define i1 @bcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-V-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-V-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
@@ -3454,7 +3454,7 @@ define i1 @bcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-ALIGNED-RV32-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV32-NEXT:    call bcmp
-; CHECK-ALIGNED-RV32-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV32-NEXT:    srli a0, a0, 31
 ; CHECK-ALIGNED-RV32-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
@@ -3466,7 +3466,7 @@ define i1 @bcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
@@ -3478,7 +3478,7 @@ define i1 @bcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    call bcmp
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    srli a0, a0, 31
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
@@ -3490,7 +3490,7 @@ define i1 @bcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
@@ -3502,7 +3502,7 @@ define i1 @bcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call bcmp
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    srli a0, a0, 31
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
@@ -3514,7 +3514,7 @@ define i1 @bcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
@@ -3526,7 +3526,7 @@ define i1 @bcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV32-V-NEXT:    call bcmp
-; CHECK-ALIGNED-RV32-V-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV32-V-NEXT:    srli a0, a0, 31
 ; CHECK-ALIGNED-RV32-V-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
@@ -3538,7 +3538,7 @@ define i1 @bcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-V-NEXT:    call bcmp
-; CHECK-ALIGNED-RV64-V-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-V-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-V-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
@@ -6839,7 +6839,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-NEXT:    ret
@@ -6861,7 +6861,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ret
@@ -6883,7 +6883,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ret
@@ -6905,7 +6905,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-V-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-V-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-V-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
@@ -6927,7 +6927,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-UNALIGNED-RV64-NEXT:    li a2, 4
 ; CHECK-UNALIGNED-RV64-NEXT:    call memcmp
-; CHECK-UNALIGNED-RV64-NEXT:    slti a0, a0, 0
+; CHECK-UNALIGNED-RV64-NEXT:    srli a0, a0, 63
 ; CHECK-UNALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, 16
 ; CHECK-UNALIGNED-RV64-NEXT:    ret
@@ -6989,7 +6989,7 @@ define i1 @memcmp_lt_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 4
 ; CHECK-UNALIGNED-RV64-V-NEXT:    call memcmp
-; CHECK-UNALIGNED-RV64-V-NEXT:    slti a0, a0, 0
+; CHECK-UNALIGNED-RV64-V-NEXT:    srli a0, a0, 63
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
@@ -7366,7 +7366,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-ALIGNED-RV32-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV32-NEXT:    call memcmp
-; CHECK-ALIGNED-RV32-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV32-NEXT:    srli a0, a0, 31
 ; CHECK-ALIGNED-RV32-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-ALIGNED-RV32-NEXT:    addi sp, sp, 16
@@ -7378,7 +7378,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-NEXT:    addi sp, sp, 16
@@ -7390,7 +7390,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    call memcmp
-; CHECK-ALIGNED-RV32-ZBB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV32-ZBB-NEXT:    srli a0, a0, 31
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-ALIGNED-RV32-ZBB-NEXT:    addi sp, sp, 16
@@ -7402,7 +7402,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-ZBB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBB-NEXT:    addi sp, sp, 16
@@ -7414,7 +7414,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    call memcmp
-; CHECK-ALIGNED-RV32-ZBKB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV32-ZBKB-NEXT:    srli a0, a0, 31
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-ALIGNED-RV32-ZBKB-NEXT:    addi sp, sp, 16
@@ -7426,7 +7426,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-ZBKB-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-ZBKB-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-ZBKB-NEXT:    addi sp, sp, 16
@@ -7438,7 +7438,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-ALIGNED-RV32-V-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV32-V-NEXT:    call memcmp
-; CHECK-ALIGNED-RV32-V-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV32-V-NEXT:    srli a0, a0, 31
 ; CHECK-ALIGNED-RV32-V-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-ALIGNED-RV32-V-NEXT:    addi sp, sp, 16
@@ -7450,7 +7450,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-ALIGNED-RV64-V-NEXT:    li a2, 4
 ; CHECK-ALIGNED-RV64-V-NEXT:    call memcmp
-; CHECK-ALIGNED-RV64-V-NEXT:    slti a0, a0, 0
+; CHECK-ALIGNED-RV64-V-NEXT:    srli a0, a0, 63
 ; CHECK-ALIGNED-RV64-V-NEXT:    xori a0, a0, 1
 ; CHECK-ALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
@@ -7462,7 +7462,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-UNALIGNED-RV32-NEXT:    li a2, 4
 ; CHECK-UNALIGNED-RV32-NEXT:    call memcmp
-; CHECK-UNALIGNED-RV32-NEXT:    slti a0, a0, 0
+; CHECK-UNALIGNED-RV32-NEXT:    srli a0, a0, 31
 ; CHECK-UNALIGNED-RV32-NEXT:    xori a0, a0, 1
 ; CHECK-UNALIGNED-RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-UNALIGNED-RV32-NEXT:    addi sp, sp, 16
@@ -7474,7 +7474,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-UNALIGNED-RV64-NEXT:    li a2, 4
 ; CHECK-UNALIGNED-RV64-NEXT:    call memcmp
-; CHECK-UNALIGNED-RV64-NEXT:    slti a0, a0, 0
+; CHECK-UNALIGNED-RV64-NEXT:    srli a0, a0, 63
 ; CHECK-UNALIGNED-RV64-NEXT:    xori a0, a0, 1
 ; CHECK-UNALIGNED-RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-UNALIGNED-RV64-NEXT:    addi sp, sp, 16
@@ -7530,7 +7530,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV32-V-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
 ; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 4
 ; CHECK-UNALIGNED-RV32-V-NEXT:    call memcmp
-; CHECK-UNALIGNED-RV32-V-NEXT:    slti a0, a0, 0
+; CHECK-UNALIGNED-RV32-V-NEXT:    srli a0, a0, 31
 ; CHECK-UNALIGNED-RV32-V-NEXT:    xori a0, a0, 1
 ; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, 16
@@ -7542,7 +7542,7 @@ define i1 @memcmp_ge_zero(ptr %s1, ptr %s2) nounwind {
 ; CHECK-UNALIGNED-RV64-V-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 4
 ; CHECK-UNALIGNED-RV64-V-NEXT:    call memcmp
-; CHECK-UNALIGNED-RV64-V-NEXT:    slti a0, a0, 0
+; CHECK-UNALIGNED-RV64-V-NEXT:    srli a0, a0, 63
 ; CHECK-UNALIGNED-RV64-V-NEXT:    xori a0, a0, 1
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/min-max.ll b/llvm/test/CodeGen/RISCV/min-max.ll
index 0115b48b7124c..acde8adf5d08b 100644
--- a/llvm/test/CodeGen/RISCV/min-max.ll
+++ b/llvm/test/CodeGen/RISCV/min-max.ll
@@ -642,7 +642,7 @@ define signext i32 @smin_i32_negone(i32 signext %a) {
 define i64 @smin_i64_negone(i64 %a) {
 ; RV32I-LABEL: smin_i64_negone:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slti a2, a1, 0
+; RV32I-NEXT:    srli a2, a1, 31
 ; RV32I-NEXT:    addi a2, a2, -1
 ; RV32I-NEXT:    or a0, a2, a0
 ; RV32I-NEXT:    slti a2, a1, -1
@@ -661,7 +661,7 @@ define i64 @smin_i64_negone(i64 %a) {
 ; RV32ZBB:       # %bb.0:
 ; RV32ZBB-NEXT:    li a2, -1
 ; RV32ZBB-NEXT:    min a2, a1, a2
-; RV32ZBB-NEXT:    slti a1, a1, 0
+; RV32ZBB-NEXT:    srli a1, a1, 31
 ; RV32ZBB-NEXT:    addi a1, a1, -1
 ; RV32ZBB-NEXT:    or a0, a1, a0
 ; RV32ZBB-NEXT:    mv a1, a2
diff --git a/llvm/test/CodeGen/RISCV/misched-load-clustering.ll b/llvm/test/CodeGen/RISCV/misched-load-clustering.ll
index 160f0aefa36a7..abdc1bad787ae 100644
--- a/llvm/test/CodeGen/RISCV/misched-load-clustering.ll
+++ b/llvm/test/CodeGen/RISCV/misched-load-clustering.ll
@@ -1,17 +1,42 @@
 ; REQUIRES: asserts
-; RUN: llc -mtriple=riscv32 -verify-misched -riscv-misched-load-store-clustering=false \
+;
+; Disable all misched clustering
+; RUN: llc -mtriple=riscv32 -verify-misched \
+; RUN:     -mattr=+disable-misched-load-clustering,+disable-misched-store-clustering \
 ; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
 ; RUN:   | FileCheck -check-prefix=NOCLUSTER %s
-; RUN: llc -mtriple=riscv64 -verify-misched -riscv-misched-load-store-clustering=false \
+; RUN: llc -mtriple=riscv64 -verify-misched \
+; RUN:     -mattr=+disable-misched-load-clustering,+disable-misched-store-clustering \
 ; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
 ; RUN:   | FileCheck -check-prefix=NOCLUSTER %s
+;
+; ST misched clustering only
+; RUN: llc -mtriple=riscv32 -verify-misched \
+; RUN:     -mattr=+disable-misched-load-clustering \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=STCLUSTER %s
+; RUN: llc -mtriple=riscv64 -verify-misched \
+; RUN:     -mattr=+disable-misched-load-clustering \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=STCLUSTER %s
+;
+; LD misched clustering only
 ; RUN: llc -mtriple=riscv32 -verify-misched \
+; RUN:     -mattr=+disable-misched-store-clustering \
 ; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
 ; RUN:   | FileCheck -check-prefix=LDCLUSTER %s
 ; RUN: llc -mtriple=riscv64 -verify-misched \
+; RUN:     -mattr=+disable-misched-store-clustering \
 ; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
 ; RUN:   | FileCheck -check-prefix=LDCLUSTER %s
-
+;
+; Default misched cluster settings (i.e. both LD and ST clustering)
+; RUN: llc -mtriple=riscv32 -verify-misched \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=DEFAULTCLUSTER %s
+; RUN: llc -mtriple=riscv64 -verify-misched \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=DEFAULTCLUSTER %s
 
 define i32 @load_clustering_1(ptr nocapture %p) {
 ; NOCLUSTER: ********** MI Scheduling **********
@@ -22,6 +47,14 @@ define i32 @load_clustering_1(ptr nocapture %p) {
 ; NOCLUSTER: SU(4): %4:gpr = LW %0:gpr, 4
 ; NOCLUSTER: SU(5): %6:gpr = LW %0:gpr, 16
 ;
+; STCLUSTER: ********** MI Scheduling **********
+; STCLUSTER-LABEL: load_clustering_1:%bb.0
+; STCLUSTER: *** Final schedule for %bb.0 ***
+; STCLUSTER: SU(1): %1:gpr = LW %0:gpr, 12
+; STCLUSTER: SU(2): %2:gpr = LW %0:gpr, 8
+; STCLUSTER: SU(4): %4:gpr = LW %0:gpr, 4
+; STCLUSTER: SU(5): %6:gpr = LW %0:gpr, 16
+;
 ; LDCLUSTER: ********** MI Scheduling **********
 ; LDCLUSTER-LABEL: load_clustering_1:%bb.0
 ; LDCLUSTER: *** Final schedule for %bb.0 ***
@@ -29,6 +62,14 @@ define i32 @load_clustering_1(ptr nocapture %p) {
 ; LDCLUSTER: SU(2): %2:gpr = LW %0:gpr, 8
 ; LDCLUSTER: SU(1): %1:gpr = LW %0:gpr, 12
 ; LDCLUSTER: SU(5): %6:gpr = LW %0:gpr, 16
+;
+; DEFAULTCLUSTER: ********** MI Scheduling **********
+; DEFAULTCLUSTER-LABEL: load_clustering_1:%bb.0
+; DEFAULTCLUSTER: *** Final schedule for %bb.0 ***
+; DEFAULTCLUSTER: SU(4): %4:gpr = LW %0:gpr, 4
+; DEFAULTCLUSTER: SU(2): %2:gpr = LW %0:gpr, 8
+; DEFAULTCLUSTER: SU(1): %1:gpr = LW %0:gpr, 12
+; DEFAULTCLUSTER: SU(5): %6:gpr = LW %0:gpr, 16
 entry:
   %arrayidx0 = getelementptr inbounds i32, ptr %p, i32 3
   %val0 = load i32, ptr %arrayidx0
diff --git a/llvm/test/CodeGen/RISCV/misched-mem-clustering.mir b/llvm/test/CodeGen/RISCV/misched-mem-clustering.mir
index 21398d315ec93..01960f9d99a8b 100644
--- a/llvm/test/CodeGen/RISCV/misched-mem-clustering.mir
+++ b/llvm/test/CodeGen/RISCV/misched-mem-clustering.mir
@@ -1,10 +1,12 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
 # RUN: llc -mtriple=riscv64 -x mir -mcpu=sifive-p470 -verify-misched -enable-post-misched=false \
-# RUN:     -riscv-postmisched-load-store-clustering=false -debug-only=machine-scheduler \
+# RUN:     -mattr=+disable-postmisched-load-clustering \
+# RUN:     -mattr=+disable-postmisched-store-clustering -debug-only=machine-scheduler \
 # RUN:     -start-before=machine-scheduler -stop-after=postmisched -misched-regpressure=false -o - 2>&1 < %s \
 # RUN:   | FileCheck -check-prefix=NOPOSTMISCHED %s
 # RUN: llc -mtriple=riscv64 -x mir -mcpu=sifive-p470 -mattr=+use-postra-scheduler -verify-misched -enable-post-misched=true \
-# RUN:     -riscv-postmisched-load-store-clustering=false -debug-only=machine-scheduler \
+# RUN:     -mattr=+disable-postmisched-load-clustering \
+# RUN:     -mattr=+disable-postmisched-store-clustering -debug-only=machine-scheduler \
 # RUN:     -start-before=machine-scheduler -stop-after=postmisched -misched-regpressure=false -o - 2>&1 < %s \
 # RUN:   | FileCheck -check-prefix=NOCLUSTER %s
 # RUN: llc -mtriple=riscv64 -x mir -mcpu=sifive-p470 -mattr=+use-postra-scheduler -verify-misched -enable-post-misched=true \
diff --git a/llvm/test/CodeGen/RISCV/misched-store-clustering.ll b/llvm/test/CodeGen/RISCV/misched-store-clustering.ll
new file mode 100644
index 0000000000000..02e853d2217cc
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/misched-store-clustering.ll
@@ -0,0 +1,83 @@
+; REQUIRES: asserts
+;
+; Disable all misched clustering
+; RUN: llc -mtriple=riscv32 -verify-misched \
+; RUN:     -mattr=+disable-misched-load-clustering,+disable-misched-store-clustering \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=NOCLUSTER %s
+; RUN: llc -mtriple=riscv64 -verify-misched \
+; RUN:     -mattr=+disable-misched-load-clustering,+disable-misched-store-clustering \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=NOCLUSTER %s
+;
+; ST misched clustering only
+; RUN: llc -mtriple=riscv32 -verify-misched \
+; RUN:     -mattr=+disable-misched-load-clustering \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=STCLUSTER %s
+; RUN: llc -mtriple=riscv64 -verify-misched \
+; RUN:     -mattr=+disable-misched-load-clustering \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=STCLUSTER %s
+;
+; LD misched clustering only
+; RUN: llc -mtriple=riscv32 -verify-misched \
+; RUN:     -mattr=+disable-misched-store-clustering \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=LDCLUSTER %s
+; RUN: llc -mtriple=riscv64 -verify-misched \
+; RUN:     -mattr=+disable-misched-store-clustering \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=LDCLUSTER %s
+;
+; Default misched cluster settings (i.e. both LD and ST clustering)
+; RUN: llc -mtriple=riscv32 -verify-misched \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=DEFAULTCLUSTER %s
+; RUN: llc -mtriple=riscv64 -verify-misched \
+; RUN:     -debug-only=machine-scheduler -o - 2>&1 < %s \
+; RUN:   | FileCheck -check-prefix=DEFAULTCLUSTER %s
+
+define i32 @store_clustering_1(ptr nocapture %p, i32 %v) {
+; NOCLUSTER: ********** MI Scheduling **********
+; NOCLUSTER-LABEL: store_clustering_1:%bb.0
+; NOCLUSTER: *** Final schedule for %bb.0 ***
+; NOCLUSTER: SU(2):   SW %1:gpr, %0:gpr, 12 :: (store (s32) into %ir.arrayidx0)
+; NOCLUSTER: SU(3):   SW %1:gpr, %0:gpr, 8 :: (store (s32) into %ir.arrayidx1)
+; NOCLUSTER: SU(4):   SW %1:gpr, %0:gpr, 4 :: (store (s32) into %ir.arrayidx2)
+; NOCLUSTER: SU(5):   SW %1:gpr, %0:gpr, 16 :: (store (s32) into %ir.arrayidx3)
+;
+; STCLUSTER: ********** MI Scheduling **********
+; STCLUSTER-LABEL: store_clustering_1:%bb.0
+; STCLUSTER: *** Final schedule for %bb.0 ***
+; STCLUSTER: SU(4):   SW %1:gpr, %0:gpr, 4 :: (store (s32) into %ir.arrayidx2)
+; STCLUSTER: SU(3):   SW %1:gpr, %0:gpr, 8 :: (store (s32) into %ir.arrayidx1)
+; STCLUSTER: SU(2):   SW %1:gpr, %0:gpr, 12 :: (store (s32) into %ir.arrayidx0)
+; STCLUSTER: SU(5):   SW %1:gpr, %0:gpr, 16 :: (store (s32) into %ir.arrayidx3)
+;
+; LDCLUSTER: ********** MI Scheduling **********
+; LDCLUSTER-LABEL: store_clustering_1:%bb.0
+; LDCLUSTER: *** Final schedule for %bb.0 ***
+; LDCLUSTER: SU(2):   SW %1:gpr, %0:gpr, 12 :: (store (s32) into %ir.arrayidx0)
+; LDCLUSTER: SU(3):   SW %1:gpr, %0:gpr, 8 :: (store (s32) into %ir.arrayidx1)
+; LDCLUSTER: SU(4):   SW %1:gpr, %0:gpr, 4 :: (store (s32) into %ir.arrayidx2)
+; LDCLUSTER: SU(5):   SW %1:gpr, %0:gpr, 16 :: (store (s32) into %ir.arrayidx3)
+;
+; DEFAULTCLUSTER: ********** MI Scheduling **********
+; DEFAULTCLUSTER-LABEL: store_clustering_1:%bb.0
+; DEFAULTCLUSTER: *** Final schedule for %bb.0 ***
+; DEFAULTCLUSTER: SU(4):   SW %1:gpr, %0:gpr, 4 :: (store (s32) into %ir.arrayidx2)
+; DEFAULTCLUSTER: SU(3):   SW %1:gpr, %0:gpr, 8 :: (store (s32) into %ir.arrayidx1)
+; DEFAULTCLUSTER: SU(2):   SW %1:gpr, %0:gpr, 12 :: (store (s32) into %ir.arrayidx0)
+; DEFAULTCLUSTER: SU(5):   SW %1:gpr, %0:gpr, 16 :: (store (s32) into %ir.arrayidx3)
+entry:
+  %arrayidx0 = getelementptr inbounds i32, ptr %p, i32 3
+  store i32 %v, ptr %arrayidx0
+  %arrayidx1 = getelementptr inbounds i32, ptr %p, i32 2
+  store i32 %v, ptr %arrayidx1
+  %arrayidx2 = getelementptr inbounds i32, ptr %p, i32 1
+  store i32 %v, ptr %arrayidx2
+  %arrayidx3 = getelementptr inbounds i32, ptr %p, i32 4
+  store i32 %v, ptr %arrayidx3
+  ret i32 %v
+}
diff --git a/llvm/test/CodeGen/RISCV/pr153598.mir b/llvm/test/CodeGen/RISCV/pr153598.mir
new file mode 100644
index 0000000000000..a084197fe83cc
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/pr153598.mir
@@ -0,0 +1,23 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv32 -mattr=+zcmp -run-pass=riscv-move-merge -verify-machineinstrs %s -o - | FileCheck %s
+---
+name: mov-merge
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x8, $x9
+    ; CHECK-LABEL: name: mov-merge
+    ; CHECK: liveins: $x8, $x9
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x12 = ADDI $x0, -3
+    ; CHECK-NEXT: SW renamable $x9, $x2, 56
+    ; CHECK-NEXT: CM_MVA01S killed renamable $x9, renamable $x8, implicit-def $x10, implicit-def $x11
+    ; CHECK-NEXT: SW renamable $x8, $x2, 60
+    ; CHECK-NEXT: PseudoRET
+    $x12 = ADDI $x0, -3
+    SW renamable $x9, $x2, 56
+    $x10 = ADDI killed renamable $x9, 0
+    SW renamable $x8, $x2, 60
+    $x11 = ADDI killed renamable $x8, 0
+    PseudoRET
+...
diff --git a/llvm/test/CodeGen/RISCV/pr53662.mir b/llvm/test/CodeGen/RISCV/pr53662.mir
index ce5e5e811f2c6..dccad40368111 100644
--- a/llvm/test/CodeGen/RISCV/pr53662.mir
+++ b/llvm/test/CodeGen/RISCV/pr53662.mir
@@ -10,8 +10,10 @@
 ---
 name:            b
 frameInfo:
-  savePoint:       '%bb.0'
-  restorePoint:    '%bb.1'
+  savePoint:
+    - point:           '%bb.0'
+  restorePoint:
+    - point:           '%bb.1'
 body:             |
   ; CHECK-LABEL: name: b
   ; CHECK: bb.0:
diff --git a/llvm/test/CodeGen/RISCV/pr84653_pr85190.ll b/llvm/test/CodeGen/RISCV/pr84653_pr85190.ll
index 30a9355734772..f84673635fbbd 100644
--- a/llvm/test/CodeGen/RISCV/pr84653_pr85190.ll
+++ b/llvm/test/CodeGen/RISCV/pr84653_pr85190.ll
@@ -39,7 +39,7 @@ define i1 @pr85190(i64 %a) {
 ; CHECK-NOZBB-LABEL: pr85190:
 ; CHECK-NOZBB:       # %bb.0:
 ; CHECK-NOZBB-NEXT:    ori a1, a0, 7
-; CHECK-NOZBB-NEXT:    slti a2, a0, 0
+; CHECK-NOZBB-NEXT:    srli a2, a0, 63
 ; CHECK-NOZBB-NEXT:    li a3, -1
 ; CHECK-NOZBB-NEXT:    slli a3, a3, 63
 ; CHECK-NOZBB-NEXT:    sub a3, a3, a1
diff --git a/llvm/test/CodeGen/RISCV/regalloc-last-chance-recoloring-failure.ll b/llvm/test/CodeGen/RISCV/regalloc-last-chance-recoloring-failure.ll
index 878b180e34c01..f3c88923c15e2 100644
--- a/llvm/test/CodeGen/RISCV/regalloc-last-chance-recoloring-failure.ll
+++ b/llvm/test/CodeGen/RISCV/regalloc-last-chance-recoloring-failure.ll
@@ -32,11 +32,7 @@ define void @last_chance_recoloring_failure() {
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 2
-; CHECK-NEXT:    vs4r.v v16, (a0) # vscale x 32-byte Folded Spill
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    vs4r.v v20, (a0) # vscale x 32-byte Folded Spill
+; CHECK-NEXT:    vs8r.v v16, (a0) # vscale x 64-byte Folded Spill
 ; CHECK-NEXT:    li s0, 36
 ; CHECK-NEXT:    vsetvli zero, s0, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwadd.vv v16, v8, v12, v0.t
@@ -47,11 +43,7 @@ define void @last_chance_recoloring_failure() {
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 2
-; CHECK-NEXT:    vl4r.v v16, (a0) # vscale x 32-byte Folded Reload
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    vl4r.v v20, (a0) # vscale x 32-byte Folded Reload
+; CHECK-NEXT:    vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, s0, e16, m4, ta, ma
@@ -92,11 +84,7 @@ define void @last_chance_recoloring_failure() {
 ; SUBREGLIVENESS-NEXT:    slli a0, a0, 3
 ; SUBREGLIVENESS-NEXT:    add a0, sp, a0
 ; SUBREGLIVENESS-NEXT:    addi a0, a0, 16
-; SUBREGLIVENESS-NEXT:    csrr a1, vlenb
-; SUBREGLIVENESS-NEXT:    slli a1, a1, 2
-; SUBREGLIVENESS-NEXT:    vs4r.v v16, (a0) # vscale x 32-byte Folded Spill
-; SUBREGLIVENESS-NEXT:    add a0, a0, a1
-; SUBREGLIVENESS-NEXT:    vs4r.v v20, (a0) # vscale x 32-byte Folded Spill
+; SUBREGLIVENESS-NEXT:    vs8r.v v16, (a0) # vscale x 64-byte Folded Spill
 ; SUBREGLIVENESS-NEXT:    li s0, 36
 ; SUBREGLIVENESS-NEXT:    vsetvli zero, s0, e16, m4, ta, ma
 ; SUBREGLIVENESS-NEXT:    vfwadd.vv v16, v8, v12, v0.t
@@ -107,11 +95,7 @@ define void @last_chance_recoloring_failure() {
 ; SUBREGLIVENESS-NEXT:    slli a0, a0, 3
 ; SUBREGLIVENESS-NEXT:    add a0, sp, a0
 ; SUBREGLIVENESS-NEXT:    addi a0, a0, 16
-; SUBREGLIVENESS-NEXT:    csrr a1, vlenb
-; SUBREGLIVENESS-NEXT:    slli a1, a1, 2
-; SUBREGLIVENESS-NEXT:    vl4r.v v16, (a0) # vscale x 32-byte Folded Reload
-; SUBREGLIVENESS-NEXT:    add a0, a0, a1
-; SUBREGLIVENESS-NEXT:    vl4r.v v20, (a0) # vscale x 32-byte Folded Reload
+; SUBREGLIVENESS-NEXT:    vl8r.v v16, (a0) # vscale x 64-byte Folded Reload
 ; SUBREGLIVENESS-NEXT:    addi a0, sp, 16
 ; SUBREGLIVENESS-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
 ; SUBREGLIVENESS-NEXT:    vsetvli zero, s0, e16, m4, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rv32xandesperf.ll b/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
index 5cabb8c53e261..4a0cddb618a09 100644
--- a/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
+++ b/llvm/test/CodeGen/RISCV/rv32xandesperf.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O0 -mtriple=riscv32 -mattr=+xandesperf -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv32 -mattr=+xandesperf -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s
 
 ; NDS.BBC
@@ -8,15 +8,14 @@ define i32 @bbc(i32 %a) nounwind {
 ; CHECK-LABEL: bbc:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    nds.bbc a0, 16, .LBB0_2
-; CHECK-NEXT:    j .LBB0_1
-; CHECK-NEXT:  .LBB0_1: # %f
-; CHECK-NEXT:    li a0, 0
-; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB0_2: # %t
+; CHECK-NEXT:  # %bb.1: # %t
 ; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB0_2: # %f
+; CHECK-NEXT:    li a0, 0
 ; CHECK-NEXT:    ret
   %and = and i32 %a, 65536
-  %tst = icmp eq i32 %and, 0
+  %tst = icmp ne i32 %and, 0
   br i1 %tst, label %t, label %f
 f:
   ret i32 0
@@ -27,16 +26,11 @@ t:
 define i32 @select_bbc(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: select_bbc:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    nds.bbc a0, 16, .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    mv a1, a2
 ; CHECK-NEXT:  .LBB1_2:
-; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:    ret
   %and = and i32 %a, 65536
   %tst = icmp eq i32 %and, 0
@@ -50,15 +44,14 @@ define i32 @bbs(i32 %a) nounwind {
 ; CHECK-LABEL: bbs:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    nds.bbs a0, 16, .LBB2_2
-; CHECK-NEXT:    j .LBB2_1
-; CHECK-NEXT:  .LBB2_1: # %f
-; CHECK-NEXT:    li a0, 0
-; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB2_2: # %t
+; CHECK-NEXT:  # %bb.1: # %t
 ; CHECK-NEXT:    li a0, 1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB2_2: # %f
+; CHECK-NEXT:    li a0, 0
 ; CHECK-NEXT:    ret
   %and = and i32 %a, 65536
-  %tst = icmp ne i32 %and, 0
+  %tst = icmp eq i32 %and, 0
   br i1 %tst, label %t, label %f
 f:
   ret i32 0
@@ -69,16 +62,11 @@ t:
 define i32 @select_bbs(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: select_bbs:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    nds.bbs a0, 16, .LBB3_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    mv a1, a2
 ; CHECK-NEXT:  .LBB3_2:
-; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:    ret
   %and = and i32 %a, 65536
   %tst = icmp ne i32 %and, 0
@@ -92,14 +80,13 @@ define i32 @beqc(i32 %a) nounwind {
 ; CHECK-LABEL: beqc:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    nds.beqc a0, 5, .LBB4_2
-; CHECK-NEXT:    j .LBB4_1
-; CHECK-NEXT:  .LBB4_1: # %f
-; CHECK-NEXT:    li a0, 0
-; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB4_2: # %t
+; CHECK-NEXT:  # %bb.1: # %t
 ; CHECK-NEXT:    li a0, 1
 ; CHECK-NEXT:    ret
-  %tst = icmp eq i32 %a, 5
+; CHECK-NEXT:  .LBB4_2: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+  %tst = icmp ne i32 %a, 5
   br i1 %tst, label %t, label %f
 f:
   ret i32 0
@@ -110,16 +97,11 @@ t:
 define i32 @select_beqc(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: select_beqc:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    nds.beqc a0, 5, .LBB5_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    mv a1, a2
 ; CHECK-NEXT:  .LBB5_2:
-; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:    ret
   %tst = icmp eq i32 %a, 5
   %ret = select i1 %tst, i32 %b, i32 %c
@@ -132,14 +114,13 @@ define i32 @bnec(i32 %a) nounwind {
 ; CHECK-LABEL: bnec:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    nds.bnec a0, 5, .LBB6_2
-; CHECK-NEXT:    j .LBB6_1
-; CHECK-NEXT:  .LBB6_1: # %f
-; CHECK-NEXT:    li a0, 0
-; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB6_2: # %t
+; CHECK-NEXT:  # %bb.1: # %t
 ; CHECK-NEXT:    li a0, 1
 ; CHECK-NEXT:    ret
-  %tst = icmp ne i32 %a, 5
+; CHECK-NEXT:  .LBB6_2: # %f
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+  %tst = icmp eq i32 %a, 5
   br i1 %tst, label %t, label %f
 f:
   ret i32 0
@@ -150,16 +131,11 @@ t:
 define i32 @select_bnec(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: select_bnec:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    sw a2, 8(sp) # 4-byte Folded Spill
-; CHECK-NEXT:    sw a1, 12(sp) # 4-byte Folded Spill
 ; CHECK-NEXT:    nds.bnec a0, 5, .LBB7_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    lw a0, 8(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    sw a0, 12(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    mv a1, a2
 ; CHECK-NEXT:  .LBB7_2:
-; CHECK-NEXT:    lw a0, 12(sp) # 4-byte Folded Reload
-; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    mv a0, a1
 ; CHECK-NEXT:    ret
   %tst = icmp ne i32 %a, 5
   %ret = select i1 %tst, i32 %b, i32 %c
@@ -182,7 +158,6 @@ define i32 @bfoz_from_and_i32(i32 %x) {
 define i64 @bfoz_from_and_i64(i64 %x) {
 ; CHECK-LABEL: bfoz_from_and_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfoz a0, a0, 11, 0
 ; CHECK-NEXT:    li a1, 0
 ; CHECK-NEXT:    ret
@@ -203,7 +178,6 @@ define i32 @bfoz_from_and_lshr_i32(i32 %x) {
 define i64 @bfoz_from_and_lshr_i64(i64 %x) {
 ; CHECK-LABEL: bfoz_from_and_lshr_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x12 killed $x11
 ; CHECK-NEXT:    nds.bfoz a0, a1, 25, 14
 ; CHECK-NEXT:    li a1, 0
 ; CHECK-NEXT:    ret
@@ -225,8 +199,6 @@ define i32 @bfoz_from_lshr_and_i32(i32 %x) {
 define i64 @bfoz_from_lshr_and_i64(i64 %x) {
 ; CHECK-LABEL: bfoz_from_lshr_and_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x12 killed $x11
-; CHECK-NEXT:    # kill: def $x12 killed $x10
 ; CHECK-NEXT:    andi a1, a1, 15
 ; CHECK-NEXT:    srli a0, a0, 24
 ; CHECK-NEXT:    slli a1, a1, 8
@@ -299,7 +271,6 @@ define i32 @bfos_from_ashr_shl_i32(i32 %x) {
 define i32 @bfos_from_ashr_sexti8_i32(i8 %x) {
 ; CHECK-LABEL: bfos_from_ashr_sexti8_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfos a0, a0, 7, 5
 ; CHECK-NEXT:    ret
   %sext = sext i8 %x to i32
@@ -310,7 +281,6 @@ define i32 @bfos_from_ashr_sexti8_i32(i8 %x) {
 define i32 @bfos_from_ashr_sexti16_i32(i16 %x) {
 ; CHECK-LABEL: bfos_from_ashr_sexti16_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfos a0, a0, 15, 11
 ; CHECK-NEXT:    ret
   %sext = sext i16 %x to i32
@@ -357,13 +327,50 @@ define i32 @sexti1_i32(i32 %a) {
 define i32 @sexti1_i32_2(i1 %a) {
 ; CHECK-LABEL: sexti1_i32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfos a0, a0, 0, 0
 ; CHECK-NEXT:    ret
   %1 = sext i1 %a to i32
   ret i32 %1
 }
 
+; Make sure we don't use not+nds.bfos
+define zeroext i8 @sexti1_i32_setcc(i32 signext %a) {
+; CHECK-LABEL: sexti1_i32_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srli a0, a0, 31
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    zext.b a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp sgt i32 %a, -1
+  %sext = sext i1 %icmp to i8
+  ret i8 %sext
+}
+
+; Make sure we don't use seqz+nds.bfos instead of snez+addi
+define i32 @sexti1_i32_setcc_2(i32 %a, i32 %b) {
+; CHECK-LABEL: sexti1_i32_setcc_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    ret
+  %icmp = icmp eq i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
+; Make sure we don't use nds.bfos instead of neg.
+define i32 @sexti1_i32_setcc_3(i32 %a, i32 %b) {
+; CHECK-LABEL: sexti1_i32_setcc_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slt a0, a0, a1
+; CHECK-NEXT:    neg a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp slt i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
 define i32 @sexti8_i32(i32 %a) {
 ; CHECK-LABEL: sexti8_i32:
 ; CHECK:       # %bb.0:
@@ -377,7 +384,6 @@ define i32 @sexti8_i32(i32 %a) {
 define i32 @sexti8_i32_2(i8 %a) {
 ; CHECK-LABEL: sexti8_i32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfos a0, a0, 7, 0
 ; CHECK-NEXT:    ret
   %1 = sext i8 %a to i32
@@ -397,7 +403,6 @@ define i32 @sexti16_i32(i32 %a) {
 define i32 @sexti16_i32_2(i16 %a) {
 ; CHECK-LABEL: sexti16_i32_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfos a0, a0, 15, 0
 ; CHECK-NEXT:    ret
   %1 = sext i16 %a to i32
@@ -407,9 +412,8 @@ define i32 @sexti16_i32_2(i16 %a) {
 define i64 @sexti1_i64(i64 %a) {
 ; CHECK-LABEL: sexti1_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
-; CHECK-NEXT:    nds.bfos a1, a0, 0, 0
-; CHECK-NEXT:    mv a0, a1
+; CHECK-NEXT:    nds.bfos a0, a0, 0, 0
+; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    ret
   %shl = shl i64 %a, 63
   %shr = ashr exact i64 %shl, 63
@@ -419,9 +423,8 @@ define i64 @sexti1_i64(i64 %a) {
 define i64 @sexti1_i64_2(i1 %a) {
 ; CHECK-LABEL: sexti1_i64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
-; CHECK-NEXT:    nds.bfos a1, a0, 0, 0
-; CHECK-NEXT:    mv a0, a1
+; CHECK-NEXT:    nds.bfos a0, a0, 0, 0
+; CHECK-NEXT:    mv a1, a0
 ; CHECK-NEXT:    ret
   %1 = sext i1 %a to i64
   ret i64 %1
@@ -430,7 +433,6 @@ define i64 @sexti1_i64_2(i1 %a) {
 define i64 @sexti8_i64(i64 %a) {
 ; CHECK-LABEL: sexti8_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfos a0, a0, 7, 0
 ; CHECK-NEXT:    srai a1, a0, 31
 ; CHECK-NEXT:    ret
@@ -442,7 +444,6 @@ define i64 @sexti8_i64(i64 %a) {
 define i64 @sexti8_i64_2(i8 %a) {
 ; CHECK-LABEL: sexti8_i64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfos a0, a0, 7, 0
 ; CHECK-NEXT:    srai a1, a0, 31
 ; CHECK-NEXT:    ret
@@ -453,7 +454,6 @@ define i64 @sexti8_i64_2(i8 %a) {
 define i64 @sexti16_i64(i64 %a) {
 ; CHECK-LABEL: sexti16_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfos a0, a0, 15, 0
 ; CHECK-NEXT:    srai a1, a0, 31
 ; CHECK-NEXT:    ret
@@ -465,7 +465,6 @@ define i64 @sexti16_i64(i64 %a) {
 define i64 @sexti16_i64_2(i16 %a) {
 ; CHECK-LABEL: sexti16_i64_2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    nds.bfos a0, a0, 15, 0
 ; CHECK-NEXT:    srai a1, a0, 31
 ; CHECK-NEXT:    ret
@@ -476,7 +475,6 @@ define i64 @sexti16_i64_2(i16 %a) {
 define i64 @sexti32_i64(i64 %a) {
 ; CHECK-LABEL: sexti32_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $x11 killed $x10
 ; CHECK-NEXT:    srai a1, a0, 31
 ; CHECK-NEXT:    ret
   %shl = shl i64 %a, 32
diff --git a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
index 723437a610ff8..1360a29a3e10f 100644
--- a/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32xtheadbb.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefixes=RV32I
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV32I
 ; RUN: llc -mtriple=riscv32 -mattr=+xtheadbb -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefixes=RV32XTHEADBB
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV32XTHEADBB,RV32XTHEADBB-NOB
+; RUN: llc -mtriple=riscv32 -mattr=+xtheadbb,+b -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV32XTHEADBB,RV32XTHEADBB-B
 
 declare i32 @llvm.ctlz.i32(i32, i1)
 
@@ -48,10 +50,15 @@ define i32 @ctlz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    li a0, 32
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: ctlz_i32:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    th.ff1 a0, a0
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: ctlz_i32:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    th.ff1 a0, a0
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: ctlz_i32:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    clz a0, a0
+; RV32XTHEADBB-B-NEXT:    ret
   %1 = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
   ret i32 %1
 }
@@ -135,18 +142,31 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: ctlz_i64:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    bnez a1, .LBB1_2
-; RV32XTHEADBB-NEXT:  # %bb.1:
-; RV32XTHEADBB-NEXT:    th.ff1 a0, a0
-; RV32XTHEADBB-NEXT:    addi a0, a0, 32
-; RV32XTHEADBB-NEXT:    li a1, 0
-; RV32XTHEADBB-NEXT:    ret
-; RV32XTHEADBB-NEXT:  .LBB1_2:
-; RV32XTHEADBB-NEXT:    th.ff1 a0, a1
-; RV32XTHEADBB-NEXT:    li a1, 0
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: ctlz_i64:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    bnez a1, .LBB1_2
+; RV32XTHEADBB-NOB-NEXT:  # %bb.1:
+; RV32XTHEADBB-NOB-NEXT:    th.ff1 a0, a0
+; RV32XTHEADBB-NOB-NEXT:    addi a0, a0, 32
+; RV32XTHEADBB-NOB-NEXT:    li a1, 0
+; RV32XTHEADBB-NOB-NEXT:    ret
+; RV32XTHEADBB-NOB-NEXT:  .LBB1_2:
+; RV32XTHEADBB-NOB-NEXT:    th.ff1 a0, a1
+; RV32XTHEADBB-NOB-NEXT:    li a1, 0
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: ctlz_i64:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    bnez a1, .LBB1_2
+; RV32XTHEADBB-B-NEXT:  # %bb.1:
+; RV32XTHEADBB-B-NEXT:    clz a0, a0
+; RV32XTHEADBB-B-NEXT:    addi a0, a0, 32
+; RV32XTHEADBB-B-NEXT:    li a1, 0
+; RV32XTHEADBB-B-NEXT:    ret
+; RV32XTHEADBB-B-NEXT:  .LBB1_2:
+; RV32XTHEADBB-B-NEXT:    clz a0, a1
+; RV32XTHEADBB-B-NEXT:    li a1, 0
+; RV32XTHEADBB-B-NEXT:    ret
   %1 = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
   ret i64 %1
 }
@@ -177,20 +197,25 @@ define i32 @cttz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    li a0, 32
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: cttz_i32:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    beqz a0, .LBB2_2
-; RV32XTHEADBB-NEXT:  # %bb.1: # %cond.false
-; RV32XTHEADBB-NEXT:    addi a1, a0, -1
-; RV32XTHEADBB-NEXT:    not a0, a0
-; RV32XTHEADBB-NEXT:    and a0, a0, a1
-; RV32XTHEADBB-NEXT:    th.ff1 a0, a0
-; RV32XTHEADBB-NEXT:    li a1, 32
-; RV32XTHEADBB-NEXT:    sub a0, a1, a0
-; RV32XTHEADBB-NEXT:    ret
-; RV32XTHEADBB-NEXT:  .LBB2_2:
-; RV32XTHEADBB-NEXT:    li a0, 32
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: cttz_i32:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    beqz a0, .LBB2_2
+; RV32XTHEADBB-NOB-NEXT:  # %bb.1: # %cond.false
+; RV32XTHEADBB-NOB-NEXT:    addi a1, a0, -1
+; RV32XTHEADBB-NOB-NEXT:    not a0, a0
+; RV32XTHEADBB-NOB-NEXT:    and a0, a0, a1
+; RV32XTHEADBB-NOB-NEXT:    th.ff1 a0, a0
+; RV32XTHEADBB-NOB-NEXT:    li a1, 32
+; RV32XTHEADBB-NOB-NEXT:    sub a0, a1, a0
+; RV32XTHEADBB-NOB-NEXT:    ret
+; RV32XTHEADBB-NOB-NEXT:  .LBB2_2:
+; RV32XTHEADBB-NOB-NEXT:    li a0, 32
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: cttz_i32:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    ctz a0, a0
+; RV32XTHEADBB-B-NEXT:    ret
   %1 = call i32 @llvm.cttz.i32(i32 %a, i1 false)
   ret i32 %1
 }
@@ -252,33 +277,46 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: cttz_i64:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    or a2, a0, a1
-; RV32XTHEADBB-NEXT:    beqz a2, .LBB3_3
-; RV32XTHEADBB-NEXT:  # %bb.1: # %cond.false
-; RV32XTHEADBB-NEXT:    bnez a0, .LBB3_4
-; RV32XTHEADBB-NEXT:  # %bb.2: # %cond.false
-; RV32XTHEADBB-NEXT:    addi a0, a1, -1
-; RV32XTHEADBB-NEXT:    not a1, a1
-; RV32XTHEADBB-NEXT:    and a0, a1, a0
-; RV32XTHEADBB-NEXT:    th.ff1 a0, a0
-; RV32XTHEADBB-NEXT:    li a1, 64
-; RV32XTHEADBB-NEXT:    j .LBB3_5
-; RV32XTHEADBB-NEXT:  .LBB3_3:
-; RV32XTHEADBB-NEXT:    li a1, 0
-; RV32XTHEADBB-NEXT:    li a0, 64
-; RV32XTHEADBB-NEXT:    ret
-; RV32XTHEADBB-NEXT:  .LBB3_4:
-; RV32XTHEADBB-NEXT:    addi a1, a0, -1
-; RV32XTHEADBB-NEXT:    not a0, a0
-; RV32XTHEADBB-NEXT:    and a0, a0, a1
-; RV32XTHEADBB-NEXT:    th.ff1 a0, a0
-; RV32XTHEADBB-NEXT:    li a1, 32
-; RV32XTHEADBB-NEXT:  .LBB3_5: # %cond.false
-; RV32XTHEADBB-NEXT:    sub a0, a1, a0
-; RV32XTHEADBB-NEXT:    li a1, 0
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: cttz_i64:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    or a2, a0, a1
+; RV32XTHEADBB-NOB-NEXT:    beqz a2, .LBB3_3
+; RV32XTHEADBB-NOB-NEXT:  # %bb.1: # %cond.false
+; RV32XTHEADBB-NOB-NEXT:    bnez a0, .LBB3_4
+; RV32XTHEADBB-NOB-NEXT:  # %bb.2: # %cond.false
+; RV32XTHEADBB-NOB-NEXT:    addi a0, a1, -1
+; RV32XTHEADBB-NOB-NEXT:    not a1, a1
+; RV32XTHEADBB-NOB-NEXT:    and a0, a1, a0
+; RV32XTHEADBB-NOB-NEXT:    th.ff1 a0, a0
+; RV32XTHEADBB-NOB-NEXT:    li a1, 64
+; RV32XTHEADBB-NOB-NEXT:    j .LBB3_5
+; RV32XTHEADBB-NOB-NEXT:  .LBB3_3:
+; RV32XTHEADBB-NOB-NEXT:    li a1, 0
+; RV32XTHEADBB-NOB-NEXT:    li a0, 64
+; RV32XTHEADBB-NOB-NEXT:    ret
+; RV32XTHEADBB-NOB-NEXT:  .LBB3_4:
+; RV32XTHEADBB-NOB-NEXT:    addi a1, a0, -1
+; RV32XTHEADBB-NOB-NEXT:    not a0, a0
+; RV32XTHEADBB-NOB-NEXT:    and a0, a0, a1
+; RV32XTHEADBB-NOB-NEXT:    th.ff1 a0, a0
+; RV32XTHEADBB-NOB-NEXT:    li a1, 32
+; RV32XTHEADBB-NOB-NEXT:  .LBB3_5: # %cond.false
+; RV32XTHEADBB-NOB-NEXT:    sub a0, a1, a0
+; RV32XTHEADBB-NOB-NEXT:    li a1, 0
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: cttz_i64:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    bnez a0, .LBB3_2
+; RV32XTHEADBB-B-NEXT:  # %bb.1:
+; RV32XTHEADBB-B-NEXT:    ctz a0, a1
+; RV32XTHEADBB-B-NEXT:    addi a0, a0, 32
+; RV32XTHEADBB-B-NEXT:    li a1, 0
+; RV32XTHEADBB-B-NEXT:    ret
+; RV32XTHEADBB-B-NEXT:  .LBB3_2:
+; RV32XTHEADBB-B-NEXT:    ctz a0, a0
+; RV32XTHEADBB-B-NEXT:    li a1, 0
+; RV32XTHEADBB-B-NEXT:    ret
   %1 = call i64 @llvm.cttz.i64(i64 %a, i1 false)
   ret i64 %1
 }
@@ -314,6 +352,44 @@ define i32 @sexti1_i32_2(i1 %a) nounwind {
   ret i32 %sext
 }
 
+; Make sure we don't use not+th.ext
+define zeroext i8 @sexti1_i32_setcc(i32 signext %a) {
+; CHECK-LABEL: sexti1_i32_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srli a0, a0, 31
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    zext.b a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp sgt i32 %a, -1
+  %sext = sext i1 %icmp to i8
+  ret i8 %sext
+}
+
+; Make sure we don't use seqz+th.ext instead of snez+addi
+define i32 @sexti1_i32_setcc_2(i32 %a, i32 %b) {
+; CHECK-LABEL: sexti1_i32_setcc_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    ret
+  %icmp = icmp eq i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
+; Make sure we don't use th.ext instead of neg.
+define i32 @sexti1_i32_setcc_3(i32 %a, i32 %b) {
+; CHECK-LABEL: sexti1_i32_setcc_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slt a0, a0, a1
+; CHECK-NEXT:    neg a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp slt i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
 define i32 @sextb_i32(i32 %a) nounwind {
 ; RV32I-LABEL: sextb_i32:
 ; RV32I:       # %bb.0:
@@ -321,10 +397,15 @@ define i32 @sextb_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    srai a0, a0, 24
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: sextb_i32:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    th.ext a0, a0, 7, 0
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: sextb_i32:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    th.ext a0, a0, 7, 0
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: sextb_i32:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    sext.b a0, a0
+; RV32XTHEADBB-B-NEXT:    ret
   %shl = shl i32 %a, 24
   %shr = ashr exact i32 %shl, 24
   ret i32 %shr
@@ -338,11 +419,17 @@ define i64 @sextb_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    srai a1, a1, 31
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: sextb_i64:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    th.ext a0, a0, 7, 0
-; RV32XTHEADBB-NEXT:    srai a1, a0, 31
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: sextb_i64:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    th.ext a0, a0, 7, 0
+; RV32XTHEADBB-NOB-NEXT:    srai a1, a0, 31
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: sextb_i64:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    sext.b a0, a0
+; RV32XTHEADBB-B-NEXT:    srai a1, a0, 31
+; RV32XTHEADBB-B-NEXT:    ret
   %shl = shl i64 %a, 56
   %shr = ashr exact i64 %shl, 56
   ret i64 %shr
@@ -355,27 +442,26 @@ define i32 @sexth_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    srai a0, a0, 16
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: sexth_i32:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    th.ext a0, a0, 15, 0
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: sexth_i32:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    th.ext a0, a0, 15, 0
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: sexth_i32:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    sext.h a0, a0
+; RV32XTHEADBB-B-NEXT:    ret
   %shl = shl i32 %a, 16
   %shr = ashr exact i32 %shl, 16
   ret i32 %shr
 }
 
 define i32 @no_sexth_i32(i32 %a) nounwind {
-; RV32I-LABEL: no_sexth_i32:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a0, a0, 17
-; RV32I-NEXT:    srai a0, a0, 16
-; RV32I-NEXT:    ret
-;
-; RV32XTHEADBB-LABEL: no_sexth_i32:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    slli a0, a0, 17
-; RV32XTHEADBB-NEXT:    srai a0, a0, 16
-; RV32XTHEADBB-NEXT:    ret
+; CHECK-LABEL: no_sexth_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a0, a0, 17
+; CHECK-NEXT:    srai a0, a0, 16
+; CHECK-NEXT:    ret
   %shl = shl i32 %a, 17
   %shr = ashr exact i32 %shl, 16
   ret i32 %shr
@@ -389,30 +475,29 @@ define i64 @sexth_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    srai a1, a1, 31
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: sexth_i64:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    th.ext a0, a0, 15, 0
-; RV32XTHEADBB-NEXT:    srai a1, a0, 31
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: sexth_i64:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    th.ext a0, a0, 15, 0
+; RV32XTHEADBB-NOB-NEXT:    srai a1, a0, 31
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: sexth_i64:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    sext.h a0, a0
+; RV32XTHEADBB-B-NEXT:    srai a1, a0, 31
+; RV32XTHEADBB-B-NEXT:    ret
   %shl = shl i64 %a, 48
   %shr = ashr exact i64 %shl, 48
   ret i64 %shr
 }
 
 define i64 @no_sexth_i64(i64 %a) nounwind {
-; RV32I-LABEL: no_sexth_i64:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a1, a0, 17
-; RV32I-NEXT:    srai a0, a1, 16
-; RV32I-NEXT:    srai a1, a1, 31
-; RV32I-NEXT:    ret
-;
-; RV32XTHEADBB-LABEL: no_sexth_i64:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    slli a1, a0, 17
-; RV32XTHEADBB-NEXT:    srai a0, a1, 16
-; RV32XTHEADBB-NEXT:    srai a1, a1, 31
-; RV32XTHEADBB-NEXT:    ret
+; CHECK-LABEL: no_sexth_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a1, a0, 17
+; CHECK-NEXT:    srai a0, a1, 16
+; CHECK-NEXT:    srai a1, a1, 31
+; CHECK-NEXT:    ret
   %shl = shl i64 %a, 49
   %shr = ashr exact i64 %shl, 48
   ret i64 %shr
@@ -457,10 +542,15 @@ define i32 @zexth_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    srli a0, a0, 16
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: zexth_i32:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    th.extu a0, a0, 15, 0
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: zexth_i32:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    th.extu a0, a0, 15, 0
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: zexth_i32:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    zext.h a0, a0
+; RV32XTHEADBB-B-NEXT:    ret
   %and = and i32 %a, 65535
   ret i32 %and
 }
@@ -473,11 +563,17 @@ define i64 @zexth_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    li a1, 0
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: zexth_i64:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    th.extu a0, a0, 15, 0
-; RV32XTHEADBB-NEXT:    li a1, 0
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: zexth_i64:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    th.extu a0, a0, 15, 0
+; RV32XTHEADBB-NOB-NEXT:    li a1, 0
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: zexth_i64:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    zext.h a0, a0
+; RV32XTHEADBB-B-NEXT:    li a1, 0
+; RV32XTHEADBB-B-NEXT:    ret
   %and = and i64 %a, 65535
   ret i64 %and
 }
@@ -500,10 +596,15 @@ define i32 @bswap_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: bswap_i32:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    th.rev a0, a0
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: bswap_i32:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    th.rev a0, a0
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: bswap_i32:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    rev8 a0, a0
+; RV32XTHEADBB-B-NEXT:    ret
   %1 = tail call i32 @llvm.bswap.i32(i32 %a)
   ret i32 %1
 }
@@ -535,12 +636,19 @@ define i64 @bswap_i64(i64 %a) {
 ; RV32I-NEXT:    or a1, a3, a4
 ; RV32I-NEXT:    ret
 ;
-; RV32XTHEADBB-LABEL: bswap_i64:
-; RV32XTHEADBB:       # %bb.0:
-; RV32XTHEADBB-NEXT:    th.rev a2, a1
-; RV32XTHEADBB-NEXT:    th.rev a1, a0
-; RV32XTHEADBB-NEXT:    mv a0, a2
-; RV32XTHEADBB-NEXT:    ret
+; RV32XTHEADBB-NOB-LABEL: bswap_i64:
+; RV32XTHEADBB-NOB:       # %bb.0:
+; RV32XTHEADBB-NOB-NEXT:    th.rev a2, a1
+; RV32XTHEADBB-NOB-NEXT:    th.rev a1, a0
+; RV32XTHEADBB-NOB-NEXT:    mv a0, a2
+; RV32XTHEADBB-NOB-NEXT:    ret
+;
+; RV32XTHEADBB-B-LABEL: bswap_i64:
+; RV32XTHEADBB-B:       # %bb.0:
+; RV32XTHEADBB-B-NEXT:    rev8 a2, a1
+; RV32XTHEADBB-B-NEXT:    rev8 a1, a0
+; RV32XTHEADBB-B-NEXT:    mv a0, a2
+; RV32XTHEADBB-B-NEXT:    ret
   %1 = call i64 @llvm.bswap.i64(i64 %a)
   ret i64 %1
 }
diff --git a/llvm/test/CodeGen/RISCV/rv32zbb-zbkb.ll b/llvm/test/CodeGen/RISCV/rv32zbb-zbkb.ll
index 1736074ab1868..7ab3d7c694568 100644
--- a/llvm/test/CodeGen/RISCV/rv32zbb-zbkb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32zbb-zbkb.ll
@@ -431,7 +431,7 @@ define i64 @not_shl_one_i64(i64 %x) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addi a1, a0, -32
 ; CHECK-NEXT:    li a2, 1
-; CHECK-NEXT:    slti a1, a1, 0
+; CHECK-NEXT:    srli a1, a1, 31
 ; CHECK-NEXT:    sll a0, a2, a0
 ; CHECK-NEXT:    neg a2, a1
 ; CHECK-NEXT:    addi a1, a1, -1
diff --git a/llvm/test/CodeGen/RISCV/rv32zbkb.ll b/llvm/test/CodeGen/RISCV/rv32zbkb.ll
index 7ebbd7802b70f..42d326e359d9f 100644
--- a/llvm/test/CodeGen/RISCV/rv32zbkb.ll
+++ b/llvm/test/CodeGen/RISCV/rv32zbkb.ll
@@ -350,10 +350,43 @@ define i32 @pack_lo_packh_hi_packh(i8 zeroext %0, i8 zeroext %1, i8 zeroext %2,
   ret i32 %j
 }
 
+define i32 @pack_lo_packh_hi_packh_2(i8 %0, i8 %1, i8 %2, i8 %3) nounwind {
+; RV32I-LABEL: pack_lo_packh_hi_packh_2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    zext.b a0, a0
+; RV32I-NEXT:    zext.b a1, a1
+; RV32I-NEXT:    zext.b a2, a2
+; RV32I-NEXT:    slli a3, a3, 24
+; RV32I-NEXT:    slli a1, a1, 8
+; RV32I-NEXT:    slli a2, a2, 16
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32ZBKB-LABEL: pack_lo_packh_hi_packh_2:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    packh a0, a0, a1
+; RV32ZBKB-NEXT:    packh a1, a2, a3
+; RV32ZBKB-NEXT:    pack a0, a0, a1
+; RV32ZBKB-NEXT:    ret
+  %a = zext i8 %0 to i32
+  %b = zext i8 %1 to i32
+  %c = zext i8 %2 to i32
+  %d = zext i8 %3 to i32
+  %e = shl i32 %b, 8
+  %f = shl i32 %c, 16
+  %g = shl i32 %d, 24
+  %h = or i32 %a, %e
+  %i = or i32 %h, %f
+  %j = or i32 %i, %g
+  ret i32 %j
+}
+
 define i32 @pack_lo_zext_hi_packh(i16 zeroext %0, i8 zeroext %1, i8 zeroext %2) nounwind {
 ; RV32I-LABEL: pack_lo_zext_hi_packh:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a1, a2, 16
+; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    slli a2, a2, 24
 ; RV32I-NEXT:    or a1, a2, a1
 ; RV32I-NEXT:    or a0, a1, a0
@@ -361,14 +394,14 @@ define i32 @pack_lo_zext_hi_packh(i16 zeroext %0, i8 zeroext %1, i8 zeroext %2)
 ;
 ; RV32ZBKB-LABEL: pack_lo_zext_hi_packh:
 ; RV32ZBKB:       # %bb.0:
-; RV32ZBKB-NEXT:    packh a1, a2, a2
+; RV32ZBKB-NEXT:    packh a1, a1, a2
 ; RV32ZBKB-NEXT:    pack a0, a0, a1
 ; RV32ZBKB-NEXT:    ret
   %a = zext i16 %0 to i32
   %b = zext i8 %1 to i32
   %c = zext i8 %2 to i32
   %d = shl i32 %c, 8
-  %e = or i32 %c, %d
+  %e = or i32 %b, %d
   %f = shl i32 %e, 16
   %g = or i32 %f, %a
   ret i32 %g
@@ -379,7 +412,7 @@ define i32 @pack_lo_zext_hi_packh(i16 zeroext %0, i8 zeroext %1, i8 zeroext %2)
 define i32 @pack_lo_noext_hi_packh(i32 %a, i8 zeroext %1, i8 zeroext %2) nounwind {
 ; RV32I-LABEL: pack_lo_noext_hi_packh:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a1, a2, 16
+; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    slli a2, a2, 24
 ; RV32I-NEXT:    or a1, a2, a1
 ; RV32I-NEXT:    or a0, a1, a0
@@ -387,14 +420,40 @@ define i32 @pack_lo_noext_hi_packh(i32 %a, i8 zeroext %1, i8 zeroext %2) nounwin
 ;
 ; RV32ZBKB-LABEL: pack_lo_noext_hi_packh:
 ; RV32ZBKB:       # %bb.0:
-; RV32ZBKB-NEXT:    packh a1, a2, a2
+; RV32ZBKB-NEXT:    packh a1, a1, a2
+; RV32ZBKB-NEXT:    slli a1, a1, 16
+; RV32ZBKB-NEXT:    or a0, a1, a0
+; RV32ZBKB-NEXT:    ret
+  %b = zext i8 %1 to i32
+  %c = zext i8 %2 to i32
+  %d = shl i32 %c, 8
+  %e = or i32 %b, %d
+  %f = shl i32 %e, 16
+  %g = or i32 %f, %a
+  ret i32 %g
+}
+
+; Make sure we can match packh+slli without having the input bytes zero extended.
+define i32 @pack_lo_noext_hi_packh_nozeroext(i32 %a, i8 %1, i8 %2) nounwind {
+; RV32I-LABEL: pack_lo_noext_hi_packh_nozeroext:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    zext.b a1, a1
+; RV32I-NEXT:    slli a2, a2, 24
+; RV32I-NEXT:    slli a1, a1, 16
+; RV32I-NEXT:    or a0, a2, a0
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBKB-LABEL: pack_lo_noext_hi_packh_nozeroext:
+; RV32ZBKB:       # %bb.0:
+; RV32ZBKB-NEXT:    packh a1, a1, a2
 ; RV32ZBKB-NEXT:    slli a1, a1, 16
 ; RV32ZBKB-NEXT:    or a0, a1, a0
 ; RV32ZBKB-NEXT:    ret
   %b = zext i8 %1 to i32
   %c = zext i8 %2 to i32
   %d = shl i32 %c, 8
-  %e = or i32 %c, %d
+  %e = or i32 %b, %d
   %f = shl i32 %e, 16
   %g = or i32 %f, %a
   ret i32 %g
diff --git a/llvm/test/CodeGen/RISCV/rv32zbs.ll b/llvm/test/CodeGen/RISCV/rv32zbs.ll
index e3728bffacf80..dcb70f88fd4ac 100644
--- a/llvm/test/CodeGen/RISCV/rv32zbs.ll
+++ b/llvm/test/CodeGen/RISCV/rv32zbs.ll
@@ -53,11 +53,11 @@ define i64 @bclr_i64(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    addi a5, a3, -32
 ; RV32I-NEXT:    sll a2, a4, a2
 ; RV32I-NEXT:    sll a3, a4, a3
-; RV32I-NEXT:    slti a4, a5, 0
-; RV32I-NEXT:    neg a5, a4
-; RV32I-NEXT:    addi a4, a4, -1
-; RV32I-NEXT:    and a2, a5, a2
-; RV32I-NEXT:    and a3, a4, a3
+; RV32I-NEXT:    srli a5, a5, 31
+; RV32I-NEXT:    neg a4, a5
+; RV32I-NEXT:    addi a5, a5, -1
+; RV32I-NEXT:    and a2, a4, a2
+; RV32I-NEXT:    and a3, a5, a3
 ; RV32I-NEXT:    not a2, a2
 ; RV32I-NEXT:    not a3, a3
 ; RV32I-NEXT:    and a0, a2, a0
@@ -70,7 +70,7 @@ define i64 @bclr_i64(i64 %a, i64 %b) nounwind {
 ; RV32ZBSNOZBB-NEXT:    bset a2, zero, a2
 ; RV32ZBSNOZBB-NEXT:    addi a4, a3, -32
 ; RV32ZBSNOZBB-NEXT:    bset a3, zero, a3
-; RV32ZBSNOZBB-NEXT:    slti a4, a4, 0
+; RV32ZBSNOZBB-NEXT:    srli a4, a4, 31
 ; RV32ZBSNOZBB-NEXT:    neg a5, a4
 ; RV32ZBSNOZBB-NEXT:    addi a4, a4, -1
 ; RV32ZBSNOZBB-NEXT:    and a2, a5, a2
@@ -87,7 +87,7 @@ define i64 @bclr_i64(i64 %a, i64 %b) nounwind {
 ; RV32ZBSZBB-NEXT:    bset a2, zero, a2
 ; RV32ZBSZBB-NEXT:    bset a4, zero, a3
 ; RV32ZBSZBB-NEXT:    addi a3, a3, -32
-; RV32ZBSZBB-NEXT:    slti a3, a3, 0
+; RV32ZBSZBB-NEXT:    srli a3, a3, 31
 ; RV32ZBSZBB-NEXT:    addi a5, a3, -1
 ; RV32ZBSZBB-NEXT:    neg a3, a3
 ; RV32ZBSZBB-NEXT:    and a4, a5, a4
@@ -188,7 +188,7 @@ define signext i64 @bset_i64_zero(i64 signext %a) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi a1, a0, -32
 ; RV32I-NEXT:    li a2, 1
-; RV32I-NEXT:    slti a1, a1, 0
+; RV32I-NEXT:    srli a1, a1, 31
 ; RV32I-NEXT:    sll a2, a2, a0
 ; RV32I-NEXT:    neg a0, a1
 ; RV32I-NEXT:    addi a1, a1, -1
@@ -200,11 +200,11 @@ define signext i64 @bset_i64_zero(i64 signext %a) nounwind {
 ; RV32ZBS:       # %bb.0:
 ; RV32ZBS-NEXT:    addi a1, a0, -32
 ; RV32ZBS-NEXT:    bset a2, zero, a0
-; RV32ZBS-NEXT:    slti a0, a1, 0
-; RV32ZBS-NEXT:    neg a1, a0
-; RV32ZBS-NEXT:    addi a3, a0, -1
-; RV32ZBS-NEXT:    and a0, a1, a2
-; RV32ZBS-NEXT:    and a1, a3, a2
+; RV32ZBS-NEXT:    srli a1, a1, 31
+; RV32ZBS-NEXT:    neg a0, a1
+; RV32ZBS-NEXT:    addi a1, a1, -1
+; RV32ZBS-NEXT:    and a0, a0, a2
+; RV32ZBS-NEXT:    and a1, a1, a2
 ; RV32ZBS-NEXT:    ret
   %shl = shl i64 1, %a
   ret i64 %shl
diff --git a/llvm/test/CodeGen/RISCV/rv64-double-convert.ll b/llvm/test/CodeGen/RISCV/rv64-double-convert.ll
index dd49d9e3e2dce..caa6c2f8ff96f 100644
--- a/llvm/test/CodeGen/RISCV/rv64-double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-double-convert.ll
@@ -97,7 +97,7 @@ define i128 @fptosi_sat_f64_to_i128(double %a) nounwind {
 ; RV64I-NEXT:    mv a1, s0
 ; RV64I-NEXT:    call __unorddf2
 ; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    slti a1, s2, 0
+; RV64I-NEXT:    srli a1, s2, 63
 ; RV64I-NEXT:    sgtz a2, s4
 ; RV64I-NEXT:    addi a0, a0, -1
 ; RV64I-NEXT:    addi a3, a1, -1
@@ -207,7 +207,7 @@ define i128 @fptoui_sat_f64_to_i128(double %a) nounwind {
 ; RV64I-NEXT:    mv s0, a0
 ; RV64I-NEXT:    li a1, 0
 ; RV64I-NEXT:    call __gedf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    addi s2, a0, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __fixunsdfti
diff --git a/llvm/test/CodeGen/RISCV/rv64-float-convert.ll b/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
index 896e371452db8..ebda78528810f 100644
--- a/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-float-convert.ll
@@ -95,7 +95,7 @@ define i128 @fptosi_sat_f32_to_i128(float %a) nounwind {
 ; RV64I-NEXT:    mv a1, s1
 ; RV64I-NEXT:    call __unordsf2
 ; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    slti a1, s2, 0
+; RV64I-NEXT:    srli a1, s2, 63
 ; RV64I-NEXT:    sgtz a2, s4
 ; RV64I-NEXT:    addi a0, a0, -1
 ; RV64I-NEXT:    addi a3, a1, -1
@@ -209,7 +209,7 @@ define i128 @fptoui_sat_f32_to_i128(float %a) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    li a1, 0
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    addi s2, a0, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __fixunssfti
diff --git a/llvm/test/CodeGen/RISCV/rv64-half-convert.ll b/llvm/test/CodeGen/RISCV/rv64-half-convert.ll
index 57061e1bde83a..648f3789953aa 100644
--- a/llvm/test/CodeGen/RISCV/rv64-half-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-half-convert.ll
@@ -173,13 +173,13 @@ define i128 @fptosi_sat_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:    call __unordsf2
 ; RV64I-NEXT:    snez a0, a0
 ; RV64I-NEXT:    sgtz a1, s4
-; RV64I-NEXT:    slti a2, s0, 0
+; RV64I-NEXT:    srli s0, s0, 63
 ; RV64I-NEXT:    addi a0, a0, -1
-; RV64I-NEXT:    neg a3, a1
-; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    neg a2, a1
+; RV64I-NEXT:    addi s0, s0, -1
 ; RV64I-NEXT:    and a1, a0, s3
-; RV64I-NEXT:    and a2, a2, s1
-; RV64I-NEXT:    or a2, a3, a2
+; RV64I-NEXT:    and s0, s0, s1
+; RV64I-NEXT:    or a2, a2, s0
 ; RV64I-NEXT:    and a0, a0, a2
 ; RV64I-NEXT:    ld ra, 56(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    ld s0, 48(sp) # 8-byte Folded Reload
@@ -253,8 +253,8 @@ define i128 @fptosi_sat_f16_to_i128(half %a) nounwind {
 ; RV64IZHINX-NEXT:    srli a1, a2, 1
 ; RV64IZHINX-NEXT:  .LBB4_4:
 ; RV64IZHINX-NEXT:    feq.s a2, s0, s0
-; RV64IZHINX-NEXT:    neg a3, a3
 ; RV64IZHINX-NEXT:    neg a4, s1
+; RV64IZHINX-NEXT:    neg a3, a3
 ; RV64IZHINX-NEXT:    neg a2, a2
 ; RV64IZHINX-NEXT:    and a0, a4, a0
 ; RV64IZHINX-NEXT:    and a1, a2, a1
@@ -288,7 +288,7 @@ define i128 @fptoui_sat_f16_to_i128(half %a) nounwind {
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    li a1, 0
 ; RV64I-NEXT:    call __gesf2
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    addi s2, a0, -1
 ; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    call __fixunssfti
@@ -334,18 +334,19 @@ define i128 @fptoui_sat_f16_to_i128(half %a) nounwind {
 ; RV64IZHINX-NEXT:    sd ra, 24(sp) # 8-byte Folded Spill
 ; RV64IZHINX-NEXT:    sd s0, 16(sp) # 8-byte Folded Spill
 ; RV64IZHINX-NEXT:    sd s1, 8(sp) # 8-byte Folded Spill
-; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
-; RV64IZHINX-NEXT:    lui a1, 522240
-; RV64IZHINX-NEXT:    addi a1, a1, -1
-; RV64IZHINX-NEXT:    fle.s a2, zero, a0
-; RV64IZHINX-NEXT:    flt.s a1, a1, a0
-; RV64IZHINX-NEXT:    neg s0, a1
-; RV64IZHINX-NEXT:    neg s1, a2
+; RV64IZHINX-NEXT:    fcvt.s.h s0, a0
+; RV64IZHINX-NEXT:    fle.s a0, zero, s0
+; RV64IZHINX-NEXT:    neg s1, a0
+; RV64IZHINX-NEXT:    mv a0, s0
 ; RV64IZHINX-NEXT:    call __fixunssfti
 ; RV64IZHINX-NEXT:    and a0, s1, a0
+; RV64IZHINX-NEXT:    lui a2, 522240
 ; RV64IZHINX-NEXT:    and a1, s1, a1
-; RV64IZHINX-NEXT:    or a0, s0, a0
-; RV64IZHINX-NEXT:    or a1, s0, a1
+; RV64IZHINX-NEXT:    addi a2, a2, -1
+; RV64IZHINX-NEXT:    flt.s a2, a2, s0
+; RV64IZHINX-NEXT:    neg a2, a2
+; RV64IZHINX-NEXT:    or a0, a2, a0
+; RV64IZHINX-NEXT:    or a1, a2, a1
 ; RV64IZHINX-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; RV64IZHINX-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; RV64IZHINX-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/rv64xandesperf.ll b/llvm/test/CodeGen/RISCV/rv64xandesperf.ll
index 98cda42665169..225495817ae16 100644
--- a/llvm/test/CodeGen/RISCV/rv64xandesperf.ll
+++ b/llvm/test/CodeGen/RISCV/rv64xandesperf.ll
@@ -277,6 +277,44 @@ define signext i32 @sexti1_i32_2(i1 %a) {
   ret i32 %1
 }
 
+; Make sure we don't use not+nds.bfos
+define zeroext i8 @sexti1_i32_setcc(i32 signext %a) {
+; CHECK-LABEL: sexti1_i32_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srli a0, a0, 63
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    zext.b a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp sgt i32 %a, -1
+  %sext = sext i1 %icmp to i8
+  ret i8 %sext
+}
+
+; Make sure we don't use seqz+nds.bfos instead of snez+addi
+define signext i32 @sexti1_i32_setcc_2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: sexti1_i32_setcc_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    ret
+  %icmp = icmp eq i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
+; Make sure we don't use nds.bfos instead of neg.
+define signext i32 @sexti1_i32_setcc_3(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: sexti1_i32_setcc_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slt a0, a0, a1
+; CHECK-NEXT:    neg a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp slt i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
 define signext i32 @sexti8_i32(i32 signext %a) {
 ; CHECK-LABEL: sexti8_i32:
 ; CHECK:       # %bb.0:
@@ -334,6 +372,44 @@ define i64 @sexti1_i64_2(i1 %a) {
   ret i64 %1
 }
 
+; Make sure we don't use not+nds.bfos
+define zeroext i8 @sexti1_i64_setcc(i64 %a) {
+; CHECK-LABEL: sexti1_i64_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srli a0, a0, 63
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    zext.b a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp sgt i64 %a, -1
+  %sext = sext i1 %icmp to i8
+  ret i8 %sext
+}
+
+; Make sure we don't use seqz+nds.bfos instead of snez+addi
+define i64 @sexti1_i64_setcc_2(i64 %a, i64 %b) {
+; CHECK-LABEL: sexti1_i64_setcc_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    ret
+  %icmp = icmp eq i64 %a, %b
+  %sext = sext i1 %icmp to i64
+  ret i64 %sext
+}
+
+; Make sure we don't use nds.bfos instead of neg.
+define i64 @sexti1_i64_setcc_3(i64 %a, i64 %b) {
+; CHECK-LABEL: sexti1_i64_setcc_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slt a0, a0, a1
+; CHECK-NEXT:    neg a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp slt i64 %a, %b
+  %sext = sext i1 %icmp to i64
+  ret i64 %sext
+}
+
 define i64 @sexti8_i64(i64 %a) {
 ; CHECK-LABEL: sexti8_i64:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
index 81acb4f724136..24853ebafefcc 100644
--- a/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64xtheadbb.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=RV64I
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV64I
 ; RUN: llc -mtriple=riscv64 -mattr=+xtheadbb -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefix=RV64XTHEADBB
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV64XTHEADBB,RV64XTHEADBB-NOB
+; RUN: llc -mtriple=riscv64 -mattr=+xtheadbb,+b -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV64XTHEADBB,RV64XTHEADBB-B
 
 declare i32 @llvm.ctlz.i32(i32, i1)
 
@@ -48,12 +50,17 @@ define signext i32 @ctlz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    li a0, 32
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: ctlz_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    not a0, a0
-; RV64XTHEADBB-NEXT:    slli a0, a0, 32
-; RV64XTHEADBB-NEXT:    th.ff0 a0, a0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: ctlz_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    not a0, a0
+; RV64XTHEADBB-NOB-NEXT:    slli a0, a0, 32
+; RV64XTHEADBB-NOB-NEXT:    th.ff0 a0, a0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: ctlz_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    clzw a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
   ret i32 %1
 }
@@ -103,14 +110,21 @@ define signext i32 @log2_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sub a0, a1, a0
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: log2_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    not a0, a0
-; RV64XTHEADBB-NEXT:    slli a0, a0, 32
-; RV64XTHEADBB-NEXT:    th.ff0 a0, a0
-; RV64XTHEADBB-NEXT:    li a1, 31
-; RV64XTHEADBB-NEXT:    sub a0, a1, a0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: log2_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    not a0, a0
+; RV64XTHEADBB-NOB-NEXT:    slli a0, a0, 32
+; RV64XTHEADBB-NOB-NEXT:    th.ff0 a0, a0
+; RV64XTHEADBB-NOB-NEXT:    li a1, 31
+; RV64XTHEADBB-NOB-NEXT:    sub a0, a1, a0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: log2_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    clzw a0, a0
+; RV64XTHEADBB-B-NEXT:    li a1, 31
+; RV64XTHEADBB-B-NEXT:    sub a0, a1, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i32 @llvm.ctlz.i32(i32 %a, i1 false)
   %2 = sub i32 31, %1
   ret i32 %2
@@ -160,15 +174,23 @@ define signext i32 @log2_ceil_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    sub a0, a0, a2
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: log2_ceil_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    addi a0, a0, -1
-; RV64XTHEADBB-NEXT:    not a0, a0
-; RV64XTHEADBB-NEXT:    slli a0, a0, 32
-; RV64XTHEADBB-NEXT:    th.ff0 a0, a0
-; RV64XTHEADBB-NEXT:    li a1, 32
-; RV64XTHEADBB-NEXT:    sub a0, a1, a0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: log2_ceil_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    addi a0, a0, -1
+; RV64XTHEADBB-NOB-NEXT:    not a0, a0
+; RV64XTHEADBB-NOB-NEXT:    slli a0, a0, 32
+; RV64XTHEADBB-NOB-NEXT:    th.ff0 a0, a0
+; RV64XTHEADBB-NOB-NEXT:    li a1, 32
+; RV64XTHEADBB-NOB-NEXT:    sub a0, a1, a0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: log2_ceil_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    addi a0, a0, -1
+; RV64XTHEADBB-B-NEXT:    clzw a0, a0
+; RV64XTHEADBB-B-NEXT:    li a1, 32
+; RV64XTHEADBB-B-NEXT:    sub a0, a1, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = sub i32 %a, 1
   %2 = call i32 @llvm.ctlz.i32(i32 %1, i1 false)
   %3 = sub i32 32, %2
@@ -216,16 +238,25 @@ define signext i32 @findLastSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: findLastSet_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    not a1, a0
-; RV64XTHEADBB-NEXT:    snez a0, a0
-; RV64XTHEADBB-NEXT:    slli a1, a1, 32
-; RV64XTHEADBB-NEXT:    th.ff0 a1, a1
-; RV64XTHEADBB-NEXT:    xori a1, a1, 31
-; RV64XTHEADBB-NEXT:    addi a0, a0, -1
-; RV64XTHEADBB-NEXT:    or a0, a0, a1
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: findLastSet_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    not a1, a0
+; RV64XTHEADBB-NOB-NEXT:    snez a0, a0
+; RV64XTHEADBB-NOB-NEXT:    slli a1, a1, 32
+; RV64XTHEADBB-NOB-NEXT:    th.ff0 a1, a1
+; RV64XTHEADBB-NOB-NEXT:    xori a1, a1, 31
+; RV64XTHEADBB-NOB-NEXT:    addi a0, a0, -1
+; RV64XTHEADBB-NOB-NEXT:    or a0, a0, a1
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: findLastSet_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    clzw a1, a0
+; RV64XTHEADBB-B-NEXT:    snez a0, a0
+; RV64XTHEADBB-B-NEXT:    xori a1, a1, 31
+; RV64XTHEADBB-B-NEXT:    addi a0, a0, -1
+; RV64XTHEADBB-B-NEXT:    or a0, a0, a1
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i32 @llvm.ctlz.i32(i32 %a, i1 true)
   %2 = xor i32 31, %1
   %3 = icmp eq i32 %a, 0
@@ -276,13 +307,19 @@ define i32 @ctlz_lshr_i32(i32 signext %a) {
 ; RV64I-NEXT:    li a0, 32
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: ctlz_lshr_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    srliw a0, a0, 1
-; RV64XTHEADBB-NEXT:    not a0, a0
-; RV64XTHEADBB-NEXT:    slli a0, a0, 32
-; RV64XTHEADBB-NEXT:    th.ff0 a0, a0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: ctlz_lshr_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    srliw a0, a0, 1
+; RV64XTHEADBB-NOB-NEXT:    not a0, a0
+; RV64XTHEADBB-NOB-NEXT:    slli a0, a0, 32
+; RV64XTHEADBB-NOB-NEXT:    th.ff0 a0, a0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: ctlz_lshr_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    srliw a0, a0, 1
+; RV64XTHEADBB-B-NEXT:    clzw a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = lshr i32 %a, 1
   %2 = call i32 @llvm.ctlz.i32(i32 %1, i1 false)
   ret i32 %2
@@ -342,10 +379,15 @@ define i64 @ctlz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    li a0, 64
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: ctlz_i64:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    th.ff1 a0, a0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: ctlz_i64:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    th.ff1 a0, a0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: ctlz_i64:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    clz a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i64 @llvm.ctlz.i64(i64 %a, i1 false)
   ret i64 %1
 }
@@ -389,20 +431,25 @@ define signext i32 @cttz_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    li a0, 32
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: cttz_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    beqz a0, .LBB6_2
-; RV64XTHEADBB-NEXT:  # %bb.1: # %cond.false
-; RV64XTHEADBB-NEXT:    addi a1, a0, -1
-; RV64XTHEADBB-NEXT:    not a0, a0
-; RV64XTHEADBB-NEXT:    and a0, a0, a1
-; RV64XTHEADBB-NEXT:    th.ff1 a0, a0
-; RV64XTHEADBB-NEXT:    li a1, 64
-; RV64XTHEADBB-NEXT:    sub a0, a1, a0
-; RV64XTHEADBB-NEXT:    ret
-; RV64XTHEADBB-NEXT:  .LBB6_2:
-; RV64XTHEADBB-NEXT:    li a0, 32
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: cttz_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    beqz a0, .LBB6_2
+; RV64XTHEADBB-NOB-NEXT:  # %bb.1: # %cond.false
+; RV64XTHEADBB-NOB-NEXT:    addi a1, a0, -1
+; RV64XTHEADBB-NOB-NEXT:    not a0, a0
+; RV64XTHEADBB-NOB-NEXT:    and a0, a0, a1
+; RV64XTHEADBB-NOB-NEXT:    th.ff1 a0, a0
+; RV64XTHEADBB-NOB-NEXT:    li a1, 64
+; RV64XTHEADBB-NOB-NEXT:    sub a0, a1, a0
+; RV64XTHEADBB-NOB-NEXT:    ret
+; RV64XTHEADBB-NOB-NEXT:  .LBB6_2:
+; RV64XTHEADBB-NOB-NEXT:    li a0, 32
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: cttz_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    ctzw a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i32 @llvm.cttz.i32(i32 %a, i1 false)
   ret i32 %1
 }
@@ -439,15 +486,20 @@ define signext i32 @cttz_zero_undef_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    lbu a0, 0(a0)
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: cttz_zero_undef_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    addi a1, a0, -1
-; RV64XTHEADBB-NEXT:    not a0, a0
-; RV64XTHEADBB-NEXT:    and a0, a0, a1
-; RV64XTHEADBB-NEXT:    th.ff1 a0, a0
-; RV64XTHEADBB-NEXT:    li a1, 64
-; RV64XTHEADBB-NEXT:    sub a0, a1, a0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: cttz_zero_undef_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    addi a1, a0, -1
+; RV64XTHEADBB-NOB-NEXT:    not a0, a0
+; RV64XTHEADBB-NOB-NEXT:    and a0, a0, a1
+; RV64XTHEADBB-NOB-NEXT:    th.ff1 a0, a0
+; RV64XTHEADBB-NOB-NEXT:    li a1, 64
+; RV64XTHEADBB-NOB-NEXT:    sub a0, a1, a0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: cttz_zero_undef_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    ctzw a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i32 @llvm.cttz.i32(i32 %a, i1 true)
   ret i32 %1
 }
@@ -487,18 +539,26 @@ define signext i32 @findFirstSet_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: findFirstSet_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    addi a1, a0, -1
-; RV64XTHEADBB-NEXT:    not a2, a0
-; RV64XTHEADBB-NEXT:    and a1, a2, a1
-; RV64XTHEADBB-NEXT:    li a2, 64
-; RV64XTHEADBB-NEXT:    snez a0, a0
-; RV64XTHEADBB-NEXT:    th.ff1 a1, a1
-; RV64XTHEADBB-NEXT:    sub a2, a2, a1
-; RV64XTHEADBB-NEXT:    addi a0, a0, -1
-; RV64XTHEADBB-NEXT:    or a0, a0, a2
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: findFirstSet_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    addi a1, a0, -1
+; RV64XTHEADBB-NOB-NEXT:    not a2, a0
+; RV64XTHEADBB-NOB-NEXT:    and a1, a2, a1
+; RV64XTHEADBB-NOB-NEXT:    li a2, 64
+; RV64XTHEADBB-NOB-NEXT:    snez a0, a0
+; RV64XTHEADBB-NOB-NEXT:    th.ff1 a1, a1
+; RV64XTHEADBB-NOB-NEXT:    sub a2, a2, a1
+; RV64XTHEADBB-NOB-NEXT:    addi a0, a0, -1
+; RV64XTHEADBB-NOB-NEXT:    or a0, a0, a2
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: findFirstSet_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    ctzw a1, a0
+; RV64XTHEADBB-B-NEXT:    snez a0, a0
+; RV64XTHEADBB-B-NEXT:    addi a0, a0, -1
+; RV64XTHEADBB-B-NEXT:    or a0, a0, a1
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i32 @llvm.cttz.i32(i32 %a, i1 true)
   %2 = icmp eq i32 %a, 0
   %3 = select i1 %2, i32 -1, i32 %1
@@ -541,18 +601,27 @@ define signext i32 @ffs_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: ffs_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    addi a1, a0, -1
-; RV64XTHEADBB-NEXT:    not a2, a0
-; RV64XTHEADBB-NEXT:    and a1, a2, a1
-; RV64XTHEADBB-NEXT:    li a2, 65
-; RV64XTHEADBB-NEXT:    seqz a0, a0
-; RV64XTHEADBB-NEXT:    th.ff1 a1, a1
-; RV64XTHEADBB-NEXT:    sub a2, a2, a1
-; RV64XTHEADBB-NEXT:    addi a0, a0, -1
-; RV64XTHEADBB-NEXT:    and a0, a0, a2
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: ffs_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    addi a1, a0, -1
+; RV64XTHEADBB-NOB-NEXT:    not a2, a0
+; RV64XTHEADBB-NOB-NEXT:    and a1, a2, a1
+; RV64XTHEADBB-NOB-NEXT:    li a2, 65
+; RV64XTHEADBB-NOB-NEXT:    seqz a0, a0
+; RV64XTHEADBB-NOB-NEXT:    th.ff1 a1, a1
+; RV64XTHEADBB-NOB-NEXT:    sub a2, a2, a1
+; RV64XTHEADBB-NOB-NEXT:    addi a0, a0, -1
+; RV64XTHEADBB-NOB-NEXT:    and a0, a0, a2
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: ffs_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    ctzw a1, a0
+; RV64XTHEADBB-B-NEXT:    seqz a0, a0
+; RV64XTHEADBB-B-NEXT:    addi a1, a1, 1
+; RV64XTHEADBB-B-NEXT:    addi a0, a0, -1
+; RV64XTHEADBB-B-NEXT:    and a0, a0, a1
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i32 @llvm.cttz.i32(i32 %a, i1 true)
   %2 = add i32 %1, 1
   %3 = icmp eq i32 %a, 0
@@ -586,20 +655,25 @@ define i64 @cttz_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    li a0, 64
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: cttz_i64:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    beqz a0, .LBB10_2
-; RV64XTHEADBB-NEXT:  # %bb.1: # %cond.false
-; RV64XTHEADBB-NEXT:    addi a1, a0, -1
-; RV64XTHEADBB-NEXT:    not a0, a0
-; RV64XTHEADBB-NEXT:    and a0, a0, a1
-; RV64XTHEADBB-NEXT:    th.ff1 a0, a0
-; RV64XTHEADBB-NEXT:    li a1, 64
-; RV64XTHEADBB-NEXT:    sub a0, a1, a0
-; RV64XTHEADBB-NEXT:    ret
-; RV64XTHEADBB-NEXT:  .LBB10_2:
-; RV64XTHEADBB-NEXT:    li a0, 64
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: cttz_i64:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    beqz a0, .LBB10_2
+; RV64XTHEADBB-NOB-NEXT:  # %bb.1: # %cond.false
+; RV64XTHEADBB-NOB-NEXT:    addi a1, a0, -1
+; RV64XTHEADBB-NOB-NEXT:    not a0, a0
+; RV64XTHEADBB-NOB-NEXT:    and a0, a0, a1
+; RV64XTHEADBB-NOB-NEXT:    th.ff1 a0, a0
+; RV64XTHEADBB-NOB-NEXT:    li a1, 64
+; RV64XTHEADBB-NOB-NEXT:    sub a0, a1, a0
+; RV64XTHEADBB-NOB-NEXT:    ret
+; RV64XTHEADBB-NOB-NEXT:  .LBB10_2:
+; RV64XTHEADBB-NOB-NEXT:    li a0, 64
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: cttz_i64:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    ctz a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i64 @llvm.cttz.i64(i64 %a, i1 false)
   ret i64 %1
 }
@@ -635,6 +709,44 @@ define signext i32 @sexti1_i32_2(i1 %a) nounwind {
   ret i32 %sext
 }
 
+; Make sure we don't use not+th.ext
+define zeroext i8 @sexti1_i32_setcc(i32 signext %a) {
+; CHECK-LABEL: sexti1_i32_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srli a0, a0, 63
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    zext.b a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp sgt i32 %a, -1
+  %sext = sext i1 %icmp to i8
+  ret i8 %sext
+}
+
+; Make sure we don't use seqz+th.ext instead of snez+addi
+define signext i32 @sexti1_i32_setcc_2(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: sexti1_i32_setcc_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    ret
+  %icmp = icmp eq i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
+; Make sure we don't use th.ext instead of neg.
+define signext i32 @sexti1_i32_setcc_3(i32 signext %a, i32 signext %b) {
+; CHECK-LABEL: sexti1_i32_setcc_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slt a0, a0, a1
+; CHECK-NEXT:    neg a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp slt i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
 define i64 @sexti1_i64(i64 %a) nounwind {
 ; RV64I-LABEL: sexti1_i64:
 ; RV64I:       # %bb.0:
@@ -666,6 +778,44 @@ define i64 @sexti1_i64_2(i1 %a) nounwind {
   ret i64 %sext
 }
 
+; Make sure we don't use not+th.ext
+define zeroext i8 @sexti1_i64_setcc(i64 %a) {
+; CHECK-LABEL: sexti1_i64_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srli a0, a0, 63
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    zext.b a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp sgt i64 %a, -1
+  %sext = sext i1 %icmp to i8
+  ret i8 %sext
+}
+
+; Make sure we don't use seqz+th.ext instead of snez+addi
+define i64 @sexti1_i64_setcc_2(i64 %a, i64 %b) {
+; CHECK-LABEL: sexti1_i64_setcc_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    ret
+  %icmp = icmp eq i64 %a, %b
+  %sext = sext i1 %icmp to i64
+  ret i64 %sext
+}
+
+; Make sure we don't use th.ext instead of neg.
+define i64 @sexti1_i64_setcc_3(i64 %a, i64 %b) {
+; CHECK-LABEL: sexti1_i64_setcc_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slt a0, a0, a1
+; CHECK-NEXT:    neg a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp slt i64 %a, %b
+  %sext = sext i1 %icmp to i64
+  ret i64 %sext
+}
+
 define signext i32 @sextb_i32(i32 signext %a) nounwind {
 ; RV64I-LABEL: sextb_i32:
 ; RV64I:       # %bb.0:
@@ -673,10 +823,15 @@ define signext i32 @sextb_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srai a0, a0, 56
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: sextb_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    th.ext a0, a0, 7, 0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: sextb_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    th.ext a0, a0, 7, 0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: sextb_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    sext.b a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %shl = shl i32 %a, 24
   %shr = ashr exact i32 %shl, 24
   ret i32 %shr
@@ -689,10 +844,15 @@ define i64 @sextb_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srai a0, a0, 56
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: sextb_i64:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    th.ext a0, a0, 7, 0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: sextb_i64:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    th.ext a0, a0, 7, 0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: sextb_i64:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    sext.b a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %shl = shl i64 %a, 56
   %shr = ashr exact i64 %shl, 56
   ret i64 %shr
@@ -705,27 +865,26 @@ define signext i32 @sexth_i32(i32 signext %a) nounwind {
 ; RV64I-NEXT:    srai a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: sexth_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    th.ext a0, a0, 15, 0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: sexth_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    th.ext a0, a0, 15, 0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: sexth_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    sext.h a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %shl = shl i32 %a, 16
   %shr = ashr exact i32 %shl, 16
   ret i32 %shr
 }
 
 define signext i32 @no_sexth_i32(i32 signext %a) nounwind {
-; RV64I-LABEL: no_sexth_i32:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 49
-; RV64I-NEXT:    srai a0, a0, 48
-; RV64I-NEXT:    ret
-;
-; RV64XTHEADBB-LABEL: no_sexth_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    slli a0, a0, 49
-; RV64XTHEADBB-NEXT:    srai a0, a0, 48
-; RV64XTHEADBB-NEXT:    ret
+; CHECK-LABEL: no_sexth_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a0, a0, 49
+; CHECK-NEXT:    srai a0, a0, 48
+; CHECK-NEXT:    ret
   %shl = shl i32 %a, 17
   %shr = ashr exact i32 %shl, 16
   ret i32 %shr
@@ -738,27 +897,26 @@ define i64 @sexth_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srai a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: sexth_i64:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    th.ext a0, a0, 15, 0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: sexth_i64:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    th.ext a0, a0, 15, 0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: sexth_i64:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    sext.h a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %shl = shl i64 %a, 48
   %shr = ashr exact i64 %shl, 48
   ret i64 %shr
 }
 
 define i64 @no_sexth_i64(i64 %a) nounwind {
-; RV64I-LABEL: no_sexth_i64:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 49
-; RV64I-NEXT:    srai a0, a0, 48
-; RV64I-NEXT:    ret
-;
-; RV64XTHEADBB-LABEL: no_sexth_i64:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    slli a0, a0, 49
-; RV64XTHEADBB-NEXT:    srai a0, a0, 48
-; RV64XTHEADBB-NEXT:    ret
+; CHECK-LABEL: no_sexth_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slli a0, a0, 49
+; CHECK-NEXT:    srai a0, a0, 48
+; CHECK-NEXT:    ret
   %shl = shl i64 %a, 49
   %shr = ashr exact i64 %shl, 48
   ret i64 %shr
@@ -771,10 +929,15 @@ define i32 @zexth_i32(i32 %a) nounwind {
 ; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: zexth_i32:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    th.extu a0, a0, 15, 0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: zexth_i32:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    th.extu a0, a0, 15, 0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: zexth_i32:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    zext.h a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %and = and i32 %a, 65535
   ret i32 %and
 }
@@ -786,14 +949,39 @@ define i64 @zexth_i64(i64 %a) nounwind {
 ; RV64I-NEXT:    srli a0, a0, 48
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: zexth_i64:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    th.extu a0, a0, 15, 0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: zexth_i64:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    th.extu a0, a0, 15, 0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: zexth_i64:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    zext.h a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %and = and i64 %a, 65535
   ret i64 %and
 }
 
+define i64 @zextw_i64(i64 %a) nounwind {
+; RV64I-LABEL: zextw_i64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    ret
+;
+; RV64XTHEADBB-NOB-LABEL: zextw_i64:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    th.extu a0, a0, 31, 0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: zextw_i64:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    zext.w a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
+  %and = and i64 %a, 4294967295
+  ret i64 %and
+}
+
 define i64 @zext_bf_i64(i64 %a) nounwind {
 ; RV64I-LABEL: zext_bf_i64:
 ; RV64I:       # %bb.0:
@@ -827,15 +1015,10 @@ define i64 @zext_bf2_i64(i64 %a) nounwind {
 }
 
 define i64 @zext_i64_srliw(i64 %a) nounwind {
-; RV64I-LABEL: zext_i64_srliw:
-; RV64I:       # %bb.0:
-; RV64I-NEXT:    srliw a0, a0, 16
-; RV64I-NEXT:    ret
-;
-; RV64XTHEADBB-LABEL: zext_i64_srliw:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    srliw a0, a0, 16
-; RV64XTHEADBB-NEXT:    ret
+; CHECK-LABEL: zext_i64_srliw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srliw a0, a0, 16
+; CHECK-NEXT:    ret
   %1 = lshr i64 %a, 16
   %and = and i64 %1, 65535
   ret i64 %and
@@ -927,10 +1110,15 @@ define i64 @bswap_i64(i64 %a) {
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
-; RV64XTHEADBB-LABEL: bswap_i64:
-; RV64XTHEADBB:       # %bb.0:
-; RV64XTHEADBB-NEXT:    th.rev a0, a0
-; RV64XTHEADBB-NEXT:    ret
+; RV64XTHEADBB-NOB-LABEL: bswap_i64:
+; RV64XTHEADBB-NOB:       # %bb.0:
+; RV64XTHEADBB-NOB-NEXT:    th.rev a0, a0
+; RV64XTHEADBB-NOB-NEXT:    ret
+;
+; RV64XTHEADBB-B-LABEL: bswap_i64:
+; RV64XTHEADBB-B:       # %bb.0:
+; RV64XTHEADBB-B-NEXT:    rev8 a0, a0
+; RV64XTHEADBB-B-NEXT:    ret
   %1 = call i64 @llvm.bswap.i64(i64 %a)
   ret i64 %1
 }
diff --git a/llvm/test/CodeGen/RISCV/rv64zbkb.ll b/llvm/test/CodeGen/RISCV/rv64zbkb.ll
index 818ea723ca2e1..4537d187a9857 100644
--- a/llvm/test/CodeGen/RISCV/rv64zbkb.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zbkb.ll
@@ -392,3 +392,247 @@ define i64 @zext_i16_to_i64(i16 %a) nounwind {
   %1 = zext i16 %a to i64
   ret i64 %1
 }
+
+define void @pack_lo_packh_hi_packh(i8 zeroext %0, i8 zeroext %1, i8 zeroext %2, i8 zeroext %3, ptr %p) nounwind {
+; RV64I-LABEL: pack_lo_packh_hi_packh:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    slli a2, a2, 16
+; RV64I-NEXT:    slli a3, a3, 24
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    or a2, a2, a3
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    sw a0, 0(a4)
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: pack_lo_packh_hi_packh:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a0, a0, a1
+; RV64ZBKB-NEXT:    packh a1, a2, a3
+; RV64ZBKB-NEXT:    packw a0, a0, a1
+; RV64ZBKB-NEXT:    sw a0, 0(a4)
+; RV64ZBKB-NEXT:    ret
+  %a = zext i8 %0 to i32
+  %b = zext i8 %1 to i32
+  %c = zext i8 %2 to i32
+  %d = zext i8 %3 to i32
+  %e = shl i32 %b, 8
+  %f = shl i32 %c, 16
+  %g = shl i32 %d, 24
+  %h = or i32 %a, %e
+  %i = or i32 %h, %f
+  %j = or i32 %i, %g
+  store i32 %j, ptr %p
+  ret void
+}
+
+define void @pack_lo_packh_hi_packh_2(i8 zeroext %0, i8 zeroext %1, i8 zeroext %2, i8 zeroext %3, ptr %p) nounwind {
+; RV64I-LABEL: pack_lo_packh_hi_packh_2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    slli a2, a2, 16
+; RV64I-NEXT:    slli a3, a3, 24
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    or a2, a2, a3
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    sw a0, 0(a4)
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: pack_lo_packh_hi_packh_2:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a0, a0, a1
+; RV64ZBKB-NEXT:    packh a1, a3, a2
+; RV64ZBKB-NEXT:    packw a0, a0, a1
+; RV64ZBKB-NEXT:    sw a0, 0(a4)
+; RV64ZBKB-NEXT:    ret
+  %a = zext i8 %0 to i32
+  %b = zext i8 %1 to i32
+  %c = zext i8 %2 to i32
+  %d = zext i8 %3 to i32
+  %e = shl i32 %b, 8
+  %f = shl i32 %c, 16
+  %g = shl i32 %d, 24
+  %h = or i32 %a, %e
+  %i = or i32 %g, %h
+  %j = or i32 %f, %i
+  store i32 %j, ptr %p
+  ret void
+}
+
+define void @pack_lo_packh_hi_packh_3(i8 %0, i8 %1, i8 %2, i8 %3, ptr %p) nounwind {
+; RV64I-LABEL: pack_lo_packh_hi_packh_3:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    zext.b a0, a0
+; RV64I-NEXT:    zext.b a1, a1
+; RV64I-NEXT:    zext.b a2, a2
+; RV64I-NEXT:    slli a3, a3, 24
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    slli a2, a2, 16
+; RV64I-NEXT:    or a0, a3, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    sw a0, 0(a4)
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: pack_lo_packh_hi_packh_3:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a0, a0, a1
+; RV64ZBKB-NEXT:    packh a1, a3, a2
+; RV64ZBKB-NEXT:    packw a0, a0, a1
+; RV64ZBKB-NEXT:    sw a0, 0(a4)
+; RV64ZBKB-NEXT:    ret
+  %a = zext i8 %0 to i32
+  %b = zext i8 %1 to i32
+  %c = zext i8 %2 to i32
+  %d = zext i8 %3 to i32
+  %e = shl i32 %b, 8
+  %f = shl i32 %c, 16
+  %g = shl i32 %d, 24
+  %h = or i32 %a, %e
+  %i = or i32 %g, %h
+  %j = or i32 %f, %i
+  store i32 %j, ptr %p
+  ret void
+}
+
+define i32 @pack_lo_packh_hi_packh_4(i8 zeroext %0, i8 zeroext %1, i8 zeroext %2, i8 zeroext %3, ptr %p) nounwind {
+; RV64I-LABEL: pack_lo_packh_hi_packh_4:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a1, 8
+; RV64I-NEXT:    slli a2, a2, 16
+; RV64I-NEXT:    slliw a3, a3, 24
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    or a2, a2, a3
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: pack_lo_packh_hi_packh_4:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a0, a0, a1
+; RV64ZBKB-NEXT:    packh a1, a3, a2
+; RV64ZBKB-NEXT:    packw a0, a0, a1
+; RV64ZBKB-NEXT:    ret
+  %a = zext i8 %0 to i32
+  %b = zext i8 %1 to i32
+  %c = zext i8 %2 to i32
+  %d = zext i8 %3 to i32
+  %e = shl i32 %b, 8
+  %f = shl i32 %c, 16
+  %g = shl i32 %d, 24
+  %h = or i32 %a, %e
+  %i = or i32 %h, %f
+  %j = or i32 %i, %g
+  ret i32 %j
+}
+
+define void @pack_lo_zext_hi_packh(i16 zeroext %0, i8 zeroext %1, i8 zeroext %2, ptr %p) nounwind {
+; RV64I-LABEL: pack_lo_zext_hi_packh:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    slli a2, a2, 24
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    sw a0, 0(a3)
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: pack_lo_zext_hi_packh:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a1, a1, a2
+; RV64ZBKB-NEXT:    packw a0, a0, a1
+; RV64ZBKB-NEXT:    sw a0, 0(a3)
+; RV64ZBKB-NEXT:    ret
+  %a = zext i16 %0 to i32
+  %b = zext i8 %1 to i32
+  %c = zext i8 %2 to i32
+  %d = shl i32 %c, 8
+  %e = or i32 %b, %d
+  %f = shl i32 %e, 16
+  %g = or i32 %f, %a
+  store i32 %g, ptr %p
+  ret void
+}
+
+; Negative test, %a isn't extended so we can't use packw for the outer or, but
+; we can use packh for the high half.
+define void @pack_lo_noext_hi_packh(i32 %a, i8 zeroext %1, i8 zeroext %2, ptr %p) nounwind {
+; RV64I-LABEL: pack_lo_noext_hi_packh:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    slli a2, a2, 24
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    sw a0, 0(a3)
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: pack_lo_noext_hi_packh:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a1, a1, a2
+; RV64ZBKB-NEXT:    slli a1, a1, 16
+; RV64ZBKB-NEXT:    or a0, a1, a0
+; RV64ZBKB-NEXT:    sw a0, 0(a3)
+; RV64ZBKB-NEXT:    ret
+  %b = zext i8 %1 to i32
+  %c = zext i8 %2 to i32
+  %d = shl i32 %c, 8
+  %e = or i32 %b, %d
+  %f = shl i32 %e, 16
+  %g = or i32 %f, %a
+  store i32 %g, ptr %p
+  ret void
+}
+
+; Make sure we can match packh+slli without having the input bytes zero extended.
+define void @pack_i32_lo_noext_hi_packh_nozeroext(i32 %a, i8 %1, i8 %2, ptr %p) nounwind {
+; RV64I-LABEL: pack_i32_lo_noext_hi_packh_nozeroext:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    zext.b a1, a1
+; RV64I-NEXT:    slli a2, a2, 24
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a0, a2, a0
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    sw a0, 0(a3)
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: pack_i32_lo_noext_hi_packh_nozeroext:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a1, a1, a2
+; RV64ZBKB-NEXT:    slli a1, a1, 16
+; RV64ZBKB-NEXT:    or a0, a1, a0
+; RV64ZBKB-NEXT:    sw a0, 0(a3)
+; RV64ZBKB-NEXT:    ret
+  %b = zext i8 %1 to i32
+  %c = zext i8 %2 to i32
+  %d = shl i32 %c, 8
+  %e = or i32 %b, %d
+  %f = shl i32 %e, 16
+  %g = or i32 %f, %a
+  store i32 %g, ptr %p
+  ret void
+}
+
+; Make sure we can match packh+slli without having the input bytes zero extended.
+define i64 @pack_i64_lo_noext_hi_packh_nozeroext(i64 %a, i8 %1, i8 %2, ptr %p) nounwind {
+; RV64I-LABEL: pack_i64_lo_noext_hi_packh_nozeroext:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    zext.b a1, a1
+; RV64I-NEXT:    zext.b a2, a2
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    slli a2, a2, 24
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    or a0, a1, a0
+; RV64I-NEXT:    ret
+;
+; RV64ZBKB-LABEL: pack_i64_lo_noext_hi_packh_nozeroext:
+; RV64ZBKB:       # %bb.0:
+; RV64ZBKB-NEXT:    packh a1, a1, a2
+; RV64ZBKB-NEXT:    slli a1, a1, 16
+; RV64ZBKB-NEXT:    or a0, a1, a0
+; RV64ZBKB-NEXT:    ret
+  %b = zext i8 %1 to i64
+  %c = zext i8 %2 to i64
+  %d = shl i64 %c, 8
+  %e = or i64 %b, %d
+  %f = shl i64 %e, 16
+  %g = or i64 %f, %a
+  ret i64 %g
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/addi-rvv-stack-object.mir b/llvm/test/CodeGen/RISCV/rvv/addi-rvv-stack-object.mir
index 812a29b9f0f4e..64430c916cddb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/addi-rvv-stack-object.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/addi-rvv-stack-object.mir
@@ -33,8 +33,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: local0, type: default, offset: 0, size: 16, alignment: 16, 
diff --git a/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir b/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir
index 9e72382e072c3..a052a95761df7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/emergency-slot.mir
@@ -34,8 +34,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 2048, alignment: 128,
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
index 6eb0b693b5546..c426ee7b7d2b1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-interleaved-access.ll
@@ -205,6 +205,23 @@ define {<4 x i32>, <4 x i32>} @vpload_factor2_interleaved_mask_intrinsic(ptr %pt
   ret {<4 x i32>, <4 x i32>} %res1
 }
 
+; mask = %m, skip the last two fields.
+define {<2 x i32>, <2 x i32>} @vpload_factor4_interleaved_mask_intrinsic_skip_fields(ptr %ptr, <2 x i1> %m) {
+; CHECK-LABEL: vpload_factor4_interleaved_mask_intrinsic_skip_fields:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 16
+; CHECK-NEXT:    vsetivli zero, 4, e32, mf2, ta, ma
+; CHECK-NEXT:    vlsseg2e32.v v8, (a0), a1, v0.t
+; CHECK-NEXT:    ret
+  %interleaved.mask = call <8 x i1> @llvm.vector.interleave4(<2 x i1> %m, <2 x i1> %m, <2 x i1> splat (i1 false), <2 x i1> splat (i1 false))
+  %interleaved.vec = tail call <8 x i32> @llvm.vp.load.v8i32.p0(ptr %ptr, <8 x i1> %interleaved.mask, i32 8)
+  %v0 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <2 x i32> <i32 0, i32 4>
+  %v1 = shufflevector <8 x i32> %interleaved.vec, <8 x i32> poison, <2 x i32> <i32 1, i32 5>
+  %res0 = insertvalue {<2 x i32>, <2 x i32>} undef, <2 x i32> %v0, 0
+  %res1 = insertvalue {<2 x i32>, <2 x i32>} %res0, <2 x i32> %v1, 1
+  ret {<2 x i32>, <2 x i32>} %res1
+}
+
 define {<4 x i32>, <4 x i32>} @vpload_factor2_interleaved_mask_shuffle(ptr %ptr, <4 x i1> %m) {
 ; CHECK-LABEL: vpload_factor2_interleaved_mask_shuffle:
 ; CHECK:       # %bb.0:
@@ -332,6 +349,59 @@ define {<4 x i32>, <4 x i32>, <4 x i32>} @vpload_factor3_poison_shufflemask(ptr
   ret {<4 x i32>, <4 x i32>, <4 x i32>} %res2
 }
 
+define {<4 x i32>, <4 x i32>} @vpload_factor3_skip_fields(ptr %ptr) {
+  ; mask = 1111, skip the last field.
+; CHECK-LABEL: vpload_factor3_skip_fields:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 12
+; CHECK-NEXT:    vsetivli zero, 6, e32, m1, ta, ma
+; CHECK-NEXT:    vlsseg2e32.v v8, (a0), a1
+; CHECK-NEXT:    ret
+  %interleaved.vec = tail call <12 x i32> @llvm.vp.load.v12i32.p0(ptr %ptr, <12 x i1> <i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0, i1 1, i1 1, i1 0>, i32 12)
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 poison, i32 10>
+  %v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  ret {<4 x i32>, <4 x i32>} %res1
+}
+
+define {<4 x i32>, <4 x i32>} @vpload_factor3_mask_skip_fields(ptr %ptr) {
+  ; mask = 0101, skip the last field.
+; CHECK-LABEL: vpload_factor3_mask_skip_fields:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 6, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v0, 10
+; CHECK-NEXT:    li a1, 12
+; CHECK-NEXT:    vlsseg2e32.v v8, (a0), a1, v0.t
+; CHECK-NEXT:    ret
+  %interleaved.vec = tail call <12 x i32> @llvm.vp.load.v12i32.p0(ptr %ptr, <12 x i1> <i1 0, i1 0, i1 0, i1 1, i1 1, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 0>, i32 12)
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 poison, i32 10>
+  %v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  ret {<4 x i32>, <4 x i32>} %res1
+}
+
+define {<4 x i32>, <4 x i32>} @vpload_factor3_combined_mask_skip_field(ptr %ptr, <4 x i1> %mask) {
+; CHECK-LABEL: vpload_factor3_combined_mask_skip_field:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 12
+; CHECK-NEXT:    vsetivli zero, 6, e32, m1, ta, ma
+; CHECK-NEXT:    vlsseg2e32.v v8, (a0), a1, v0.t
+; CHECK-NEXT:    ret
+  %interleaved.mask = shufflevector <4 x i1> %mask, <4 x i1> poison, <12 x i32> <i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3>
+  %combined = and <12 x i1> %interleaved.mask, <i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false>
+  %interleaved.vec = tail call <12 x i32> @llvm.vp.load.v12i32.p0(ptr %ptr, <12 x i1> %combined, i32 12)
+  ; mask = %mask, skip the last field
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  ret {<4 x i32>, <4 x i32>} %res1
+}
+
 define {<4 x i32>, <4 x i32>, <4 x i32>, <4 x i32>} @vpload_factor4(ptr %ptr) {
 ; CHECK-LABEL: vpload_factor4:
 ; CHECK:       # %bb.0:
@@ -479,8 +549,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    li a2, 32
 ; RV32-NEXT:    lui a3, 12
 ; RV32-NEXT:    lui a6, 12291
-; RV32-NEXT:    lui a7, %hi(.LCPI23_0)
-; RV32-NEXT:    addi a7, a7, %lo(.LCPI23_0)
+; RV32-NEXT:    lui a7, %hi(.LCPI27_0)
+; RV32-NEXT:    addi a7, a7, %lo(.LCPI27_0)
 ; RV32-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; RV32-NEXT:    vle32.v v24, (a5)
 ; RV32-NEXT:    vmv.s.x v0, a3
@@ -565,12 +635,12 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    addi a1, a1, 16
 ; RV32-NEXT:    vs4r.v v8, (a1) # vscale x 32-byte Folded Spill
 ; RV32-NEXT:    lui a7, 49164
-; RV32-NEXT:    lui a1, %hi(.LCPI23_1)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI23_1)
+; RV32-NEXT:    lui a1, %hi(.LCPI27_1)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI27_1)
 ; RV32-NEXT:    lui t2, 3
 ; RV32-NEXT:    lui t1, 196656
-; RV32-NEXT:    lui a4, %hi(.LCPI23_3)
-; RV32-NEXT:    addi a4, a4, %lo(.LCPI23_3)
+; RV32-NEXT:    lui a4, %hi(.LCPI27_3)
+; RV32-NEXT:    addi a4, a4, %lo(.LCPI27_3)
 ; RV32-NEXT:    lui t0, 786624
 ; RV32-NEXT:    li a5, 48
 ; RV32-NEXT:    lui a6, 768
@@ -749,8 +819,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vl8r.v v8, (a1) # vscale x 64-byte Folded Reload
 ; RV32-NEXT:    vsetvli zero, zero, e64, m8, ta, ma
 ; RV32-NEXT:    vrgatherei16.vv v24, v8, v2
-; RV32-NEXT:    lui a1, %hi(.LCPI23_2)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI23_2)
+; RV32-NEXT:    lui a1, %hi(.LCPI27_2)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI27_2)
 ; RV32-NEXT:    lui a3, 3073
 ; RV32-NEXT:    addi a3, a3, -1024
 ; RV32-NEXT:    vmv.s.x v0, a3
@@ -814,16 +884,16 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vrgatherei16.vv v28, v8, v3
 ; RV32-NEXT:    vsetivli zero, 10, e32, m4, tu, ma
 ; RV32-NEXT:    vmv.v.v v28, v24
-; RV32-NEXT:    lui a1, %hi(.LCPI23_4)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI23_4)
-; RV32-NEXT:    lui a2, %hi(.LCPI23_5)
-; RV32-NEXT:    addi a2, a2, %lo(.LCPI23_5)
+; RV32-NEXT:    lui a1, %hi(.LCPI27_4)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI27_4)
+; RV32-NEXT:    lui a2, %hi(.LCPI27_5)
+; RV32-NEXT:    addi a2, a2, %lo(.LCPI27_5)
 ; RV32-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; RV32-NEXT:    vle16.v v24, (a2)
 ; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; RV32-NEXT:    vle16.v v8, (a1)
-; RV32-NEXT:    lui a1, %hi(.LCPI23_7)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI23_7)
+; RV32-NEXT:    lui a1, %hi(.LCPI27_7)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI27_7)
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vle16.v v10, (a1)
 ; RV32-NEXT:    csrr a1, vlenb
@@ -851,14 +921,14 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV32-NEXT:    vl8r.v v0, (a1) # vscale x 64-byte Folded Reload
 ; RV32-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV32-NEXT:    vrgatherei16.vv v16, v0, v10
-; RV32-NEXT:    lui a1, %hi(.LCPI23_6)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI23_6)
-; RV32-NEXT:    lui a2, %hi(.LCPI23_8)
-; RV32-NEXT:    addi a2, a2, %lo(.LCPI23_8)
+; RV32-NEXT:    lui a1, %hi(.LCPI27_6)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI27_6)
+; RV32-NEXT:    lui a2, %hi(.LCPI27_8)
+; RV32-NEXT:    addi a2, a2, %lo(.LCPI27_8)
 ; RV32-NEXT:    vsetivli zero, 8, e16, m1, ta, ma
 ; RV32-NEXT:    vle16.v v4, (a1)
-; RV32-NEXT:    lui a1, %hi(.LCPI23_9)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI23_9)
+; RV32-NEXT:    lui a1, %hi(.LCPI27_9)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI27_9)
 ; RV32-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; RV32-NEXT:    vle16.v v6, (a1)
 ; RV32-NEXT:    vsetivli zero, 8, e64, m4, ta, ma
@@ -945,8 +1015,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    li a4, 128
 ; RV64-NEXT:    lui a1, 1
 ; RV64-NEXT:    vle64.v v8, (a3)
-; RV64-NEXT:    lui a3, %hi(.LCPI23_0)
-; RV64-NEXT:    addi a3, a3, %lo(.LCPI23_0)
+; RV64-NEXT:    lui a3, %hi(.LCPI27_0)
+; RV64-NEXT:    addi a3, a3, %lo(.LCPI27_0)
 ; RV64-NEXT:    vmv.s.x v0, a4
 ; RV64-NEXT:    csrr a4, vlenb
 ; RV64-NEXT:    li a5, 61
@@ -1134,8 +1204,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    vl8r.v v16, (a2) # vscale x 64-byte Folded Reload
 ; RV64-NEXT:    vsetivli zero, 8, e64, m4, ta, mu
 ; RV64-NEXT:    vslideup.vi v12, v16, 1, v0.t
-; RV64-NEXT:    lui a2, %hi(.LCPI23_1)
-; RV64-NEXT:    addi a2, a2, %lo(.LCPI23_1)
+; RV64-NEXT:    lui a2, %hi(.LCPI27_1)
+; RV64-NEXT:    addi a2, a2, %lo(.LCPI27_1)
 ; RV64-NEXT:    li a3, 192
 ; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; RV64-NEXT:    vle16.v v6, (a2)
@@ -1169,8 +1239,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    vrgatherei16.vv v24, v16, v6
 ; RV64-NEXT:    addi a2, sp, 16
 ; RV64-NEXT:    vs8r.v v24, (a2) # vscale x 64-byte Folded Spill
-; RV64-NEXT:    lui a2, %hi(.LCPI23_2)
-; RV64-NEXT:    addi a2, a2, %lo(.LCPI23_2)
+; RV64-NEXT:    lui a2, %hi(.LCPI27_2)
+; RV64-NEXT:    addi a2, a2, %lo(.LCPI27_2)
 ; RV64-NEXT:    li a3, 1040
 ; RV64-NEXT:    vmv.s.x v0, a3
 ; RV64-NEXT:    addi a1, a1, -2016
@@ -1254,12 +1324,12 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vs4r.v v8, (a1) # vscale x 32-byte Folded Spill
-; RV64-NEXT:    lui a1, %hi(.LCPI23_3)
-; RV64-NEXT:    addi a1, a1, %lo(.LCPI23_3)
+; RV64-NEXT:    lui a1, %hi(.LCPI27_3)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI27_3)
 ; RV64-NEXT:    vsetivli zero, 16, e16, m2, ta, ma
 ; RV64-NEXT:    vle16.v v20, (a1)
-; RV64-NEXT:    lui a1, %hi(.LCPI23_4)
-; RV64-NEXT:    addi a1, a1, %lo(.LCPI23_4)
+; RV64-NEXT:    lui a1, %hi(.LCPI27_4)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI27_4)
 ; RV64-NEXT:    vle16.v v8, (a1)
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a2, 77
@@ -1310,8 +1380,8 @@ define {<8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>} @load_
 ; RV64-NEXT:    vl2r.v v8, (a1) # vscale x 16-byte Folded Reload
 ; RV64-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; RV64-NEXT:    vrgatherei16.vv v0, v16, v8
-; RV64-NEXT:    lui a1, %hi(.LCPI23_5)
-; RV64-NEXT:    addi a1, a1, %lo(.LCPI23_5)
+; RV64-NEXT:    lui a1, %hi(.LCPI27_5)
+; RV64-NEXT:    addi a1, a1, %lo(.LCPI27_5)
 ; RV64-NEXT:    vle16.v v20, (a1)
 ; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    li a2, 61
@@ -1928,8 +1998,8 @@ define {<4 x i32>, <4 x i32>, <4 x i32>} @invalid_vp_mask(ptr %ptr) {
 ; RV32-NEXT:    vle32.v v12, (a0), v0.t
 ; RV32-NEXT:    li a0, 36
 ; RV32-NEXT:    vmv.s.x v20, a1
-; RV32-NEXT:    lui a1, %hi(.LCPI59_0)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI59_0)
+; RV32-NEXT:    lui a1, %hi(.LCPI63_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI63_0)
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32-NEXT:    vle16.v v21, (a1)
 ; RV32-NEXT:    vcompress.vm v8, v12, v11
@@ -2004,8 +2074,8 @@ define {<4 x i32>, <4 x i32>, <4 x i32>} @invalid_vp_evl(ptr %ptr) {
 ; RV32-NEXT:    vmv.s.x v10, a0
 ; RV32-NEXT:    li a0, 146
 ; RV32-NEXT:    vmv.s.x v11, a0
-; RV32-NEXT:    lui a0, %hi(.LCPI60_0)
-; RV32-NEXT:    addi a0, a0, %lo(.LCPI60_0)
+; RV32-NEXT:    lui a0, %hi(.LCPI64_0)
+; RV32-NEXT:    addi a0, a0, %lo(.LCPI64_0)
 ; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
 ; RV32-NEXT:    vle16.v v20, (a0)
 ; RV32-NEXT:    li a0, 36
@@ -2094,3 +2164,218 @@ define void @maskedstore_factor2(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1) {
   tail call void @llvm.masked.store(<8 x i32> %interleaved.vec, ptr %ptr, i32 4, <8 x i1> splat (i1 true))
   ret void
 }
+
+define {<4 x i32>, <4 x i32>, <4 x i32>} @maskedload_factor3_mask(ptr %ptr) {
+; CHECK-LABEL: maskedload_factor3_mask:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v0, 5
+; CHECK-NEXT:    vlseg3e32.v v8, (a0), v0.t
+; CHECK-NEXT:    ret
+  %interleaved.vec = tail call <12 x i32> @llvm.masked.load.v12i32.p0(ptr %ptr, i32 4, <12 x i1> <i1 1,i1 1,i1 1,i1 0,i1 0,i1 0,i1 1,i1 1,i1 1,i1 0,i1 0,i1 0>, <12 x i32> poison)
+  ; mask = 1010
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  %res2 = insertvalue {<4 x i32>, <4 x i32>, <4 x i32>} %res1, <4 x i32> %v2, 2
+  ret {<4 x i32>, <4 x i32>, <4 x i32>} %res2
+}
+
+define {<4 x i32>, <4 x i32>} @maskedload_factor3_skip_field(ptr %ptr) {
+; CHECK-LABEL: maskedload_factor3_skip_field:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 12
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vlsseg2e32.v v8, (a0), a1
+; CHECK-NEXT:    ret
+  %interleaved.vec = tail call <12 x i32> @llvm.masked.load.v12i32.p0(ptr %ptr, i32 4, <12 x i1> <i1 1,i1 1,i1 0,i1 1,i1 1,i1 0,i1 1,i1 1,i1 0,i1 1,i1 1,i1 0>, <12 x i32> poison)
+  ; mask = 1111, skip last field
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  ret {<4 x i32>, <4 x i32>} %res1
+}
+
+define {<4 x i32>, <4 x i32>} @maskedload_factor3_mask_skip_field(ptr %ptr) {
+; CHECK-LABEL: maskedload_factor3_mask_skip_field:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v0, 5
+; CHECK-NEXT:    li a1, 12
+; CHECK-NEXT:    vlsseg2e32.v v8, (a0), a1, v0.t
+; CHECK-NEXT:    ret
+  %interleaved.vec = tail call <12 x i32> @llvm.masked.load.v12i32.p0(ptr %ptr, i32 4, <12 x i1> <i1 1,i1 1,i1 0,i1 0,i1 0,i1 0,i1 1,i1 1,i1 0,i1 0,i1 0,i1 0>, <12 x i32> poison)
+  ; mask = 1010, skip the last field
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  ret {<4 x i32>, <4 x i32>} %res1
+}
+
+define {<4 x i32>, <4 x i32>} @maskedload_factor3_combined_mask_skip_field(ptr %ptr, <4 x i1> %mask) {
+; CHECK-LABEL: maskedload_factor3_combined_mask_skip_field:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 12
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vlsseg2e32.v v8, (a0), a1, v0.t
+; CHECK-NEXT:    ret
+  %interleaved.mask = shufflevector <4 x i1> %mask, <4 x i1> poison, <12 x i32> <i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3>
+  %combined = and <12 x i1> %interleaved.mask, <i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 true, i1 true, i1 false>
+  %interleaved.vec = tail call <12 x i32> @llvm.masked.load.v12i32.p0(ptr %ptr, i32 4, <12 x i1> %combined, <12 x i32> poison)
+  ; mask = %mask, skip the last field
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  ret {<4 x i32>, <4 x i32>} %res1
+}
+
+define {<4 x i32>, <4 x i32>} @maskedload_factor4_combined_mask_multi_skip_fields(ptr %ptr, <4 x i1> %mask) {
+; CHECK-LABEL: maskedload_factor4_combined_mask_multi_skip_fields:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 16
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vlsseg2e32.v v8, (a0), a1, v0.t
+; CHECK-NEXT:    ret
+  %interleaved.mask = shufflevector <4 x i1> %mask, <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+  %combined = and <16 x i1> %interleaved.mask, <i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false>
+  %combined1 = and <16 x i1> %combined, <i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true>
+  %interleaved.vec = tail call <16 x i32> @llvm.masked.load.v16i32.p0(ptr %ptr, i32 4, <16 x i1> %combined1, <16 x i32> poison)
+  ; mask = %mask, skip the last 2 fields
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  ret {<4 x i32>, <4 x i32>} %res1
+}
+
+define {<4 x i32>, <4 x i32>} @maskedload_factor4_combined_mask_multi_skip_fields_and_masks(ptr %ptr, <4 x i1> %mask, <4 x i1> %mask2) {
+; CHECK-LABEL: maskedload_factor4_combined_mask_multi_skip_fields_and_masks:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmand.mm v0, v0, v8
+; CHECK-NEXT:    li a1, 16
+; CHECK-NEXT:    vlsseg2e32.v v8, (a0), a1, v0.t
+; CHECK-NEXT:    ret
+  %interleaved.mask = shufflevector <4 x i1> %mask, <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+  %combined = and <16 x i1> %interleaved.mask, <i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false>
+
+  %interleaved.mask2 = shufflevector <4 x i1> %mask2, <4 x i1> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+  %combined1 = and <16 x i1> %interleaved.mask2, <i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true>
+
+  %combined2 = and <16 x i1> %combined, %combined1
+  %interleaved.vec = tail call <16 x i32> @llvm.masked.load.v16i32.p0(ptr %ptr, i32 4, <16 x i1> %combined2, <16 x i32> poison)
+  ; mask = %mask & %mask2, skip the last 2 fields
+  %v0 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
+  %v1 = shufflevector <16 x i32> %interleaved.vec, <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  ret {<4 x i32>, <4 x i32>} %res1
+}
+
+; We can only skip the last field for now.
+define {<4 x i32>, <4 x i32>, <4 x i32>} @maskedload_factor3_invalid_skip_field(ptr %ptr) {
+; RV32-LABEL: maskedload_factor3_invalid_skip_field:
+; RV32:       # %bb.0:
+; RV32-NEXT:    li a1, 73
+; RV32-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
+; RV32-NEXT:    vmv.s.x v11, a1
+; RV32-NEXT:    lui a1, 1
+; RV32-NEXT:    vmv.v.i v10, 8
+; RV32-NEXT:    addi a1, a1, -1171
+; RV32-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; RV32-NEXT:    vmv.s.x v0, a1
+; RV32-NEXT:    li a1, 146
+; RV32-NEXT:    vsetivli zero, 12, e32, m4, ta, ma
+; RV32-NEXT:    vle32.v v12, (a0), v0.t
+; RV32-NEXT:    li a0, 36
+; RV32-NEXT:    vmv.s.x v20, a1
+; RV32-NEXT:    lui a1, %hi(.LCPI73_0)
+; RV32-NEXT:    addi a1, a1, %lo(.LCPI73_0)
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV32-NEXT:    vle16.v v21, (a1)
+; RV32-NEXT:    vcompress.vm v8, v12, v11
+; RV32-NEXT:    vsetivli zero, 8, e32, m4, ta, ma
+; RV32-NEXT:    vslidedown.vi v16, v12, 8
+; RV32-NEXT:    vmv1r.v v0, v10
+; RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
+; RV32-NEXT:    vrgather.vi v8, v16, 1, v0.t
+; RV32-NEXT:    vcompress.vm v14, v12, v20
+; RV32-NEXT:    vrgather.vi v14, v16, 2, v0.t
+; RV32-NEXT:    vmv.s.x v0, a0
+; RV32-NEXT:    vmerge.vvm v12, v16, v12, v0
+; RV32-NEXT:    vrgatherei16.vv v10, v12, v21
+; RV32-NEXT:    vmv1r.v v9, v14
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: maskedload_factor3_invalid_skip_field:
+; RV64:       # %bb.0:
+; RV64-NEXT:    li a1, 73
+; RV64-NEXT:    vsetivli zero, 1, e8, m1, ta, ma
+; RV64-NEXT:    vmv.s.x v11, a1
+; RV64-NEXT:    li a1, 146
+; RV64-NEXT:    vmv.s.x v20, a1
+; RV64-NEXT:    lui a1, 1
+; RV64-NEXT:    vmv.v.i v10, 8
+; RV64-NEXT:    addi a1, a1, -1171
+; RV64-NEXT:    vsetvli zero, zero, e16, m2, ta, ma
+; RV64-NEXT:    vmv.s.x v0, a1
+; RV64-NEXT:    li a1, 36
+; RV64-NEXT:    vsetivli zero, 12, e32, m4, ta, ma
+; RV64-NEXT:    vle32.v v12, (a0), v0.t
+; RV64-NEXT:    li a0, 3
+; RV64-NEXT:    slli a0, a0, 32
+; RV64-NEXT:    addi a0, a0, 5
+; RV64-NEXT:    slli a0, a0, 16
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vcompress.vm v8, v12, v11
+; RV64-NEXT:    vsetivli zero, 8, e32, m4, ta, ma
+; RV64-NEXT:    vslidedown.vi v16, v12, 8
+; RV64-NEXT:    vmv1r.v v0, v10
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, mu
+; RV64-NEXT:    vrgather.vi v8, v16, 1, v0.t
+; RV64-NEXT:    vcompress.vm v14, v12, v20
+; RV64-NEXT:    vrgather.vi v14, v16, 2, v0.t
+; RV64-NEXT:    vmv.s.x v0, a1
+; RV64-NEXT:    addi a0, a0, 2
+; RV64-NEXT:    vmerge.vvm v12, v16, v12, v0
+; RV64-NEXT:    vsetivli zero, 2, e64, m1, ta, ma
+; RV64-NEXT:    vmv.v.x v9, a0
+; RV64-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; RV64-NEXT:    vrgatherei16.vv v10, v12, v9
+; RV64-NEXT:    vmv1r.v v9, v14
+; RV64-NEXT:    ret
+  %interleaved.vec = tail call <12 x i32> @llvm.masked.load.v12i32.p0(ptr %ptr, i32 4, <12 x i1> <i1 1,i1 0,i1 1,i1 1,i1 0,i1 1,i1 1,i1 0,i1 1,i1 1,i1 0,i1 1>, <12 x i32> poison)
+  %v0 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
+  %v1 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
+  %v2 = shufflevector <12 x i32> %interleaved.vec, <12 x i32> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  %res2 = insertvalue {<4 x i32>, <4 x i32>, <4 x i32>} %res1, <4 x i32> %v2, 2
+  ret {<4 x i32>, <4 x i32>, <4 x i32>} %res2
+}
+
+define {<4 x i32>, <4 x i32>, <4 x i32>} @maskedload_factor5_skip_fields(ptr %ptr) {
+  ; mask = 1111, skip the last two fields.
+; CHECK-LABEL: maskedload_factor5_skip_fields:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 20
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vlsseg3e32.v v8, (a0), a1
+; CHECK-NEXT:    ret
+  %interleaved.vec = tail call <20 x i32> @llvm.masked.load(ptr %ptr, i32 4, <20 x i1> <i1 1, i1 1, i1 1, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0, i1 1, i1 1, i1 1, i1 0, i1 0>, <20 x i32> poison)
+  %v0 = shufflevector <20 x i32> %interleaved.vec, <20 x i32> poison, <4 x i32> <i32 0, i32 5, i32 10, i32 15>
+  %v1 = shufflevector <20 x i32> %interleaved.vec, <20 x i32> poison, <4 x i32> <i32 1, i32 6, i32 11, i32 16>
+  %v2 = shufflevector <20 x i32> %interleaved.vec, <20 x i32> poison, <4 x i32> <i32 2, i32 7, i32 12, i32 17>
+  %v3 = shufflevector <20 x i32> %interleaved.vec, <20 x i32> poison, <4 x i32> <i32 3, i32 8, i32 13, i32 18>
+  %v4 = shufflevector <20 x i32> %interleaved.vec, <20 x i32> poison, <4 x i32> <i32 4, i32 9, i32 14, i32 19>
+  %res0 = insertvalue {<4 x i32>, <4 x i32>, <4 x i32>} undef, <4 x i32> %v0, 0
+  %res1 = insertvalue {<4 x i32>, <4 x i32>, <4 x i32>} %res0, <4 x i32> %v1, 1
+  %res2 = insertvalue {<4 x i32>, <4 x i32>, <4 x i32>} %res1, <4 x i32> %v2, 2
+  ret {<4 x i32>, <4 x i32>, <4 x i32>} %res2
+}
+
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index 67d55366674f3..4c35b2506d3e4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -2047,19 +2047,17 @@ define <2 x i64> @mgather_v2i32_sextload_v2i64(<2 x ptr> %ptrs, <2 x i1> %m, <2
 ;
 ; RV32ZVE32F-LABEL: mgather_v2i32_sextload_v2i64:
 ; RV32ZVE32F:       # %bb.0:
-; RV32ZVE32F-NEXT:    addi a1, a0, 8
 ; RV32ZVE32F-NEXT:    vsetivli zero, 2, e32, m1, ta, mu
 ; RV32ZVE32F-NEXT:    vluxei32.v v9, (zero), v8, v0.t
 ; RV32ZVE32F-NEXT:    vslidedown.vi v8, v9, 1
-; RV32ZVE32F-NEXT:    vmv.x.s a2, v9
-; RV32ZVE32F-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; RV32ZVE32F-NEXT:    vse32.v v9, (a0)
-; RV32ZVE32F-NEXT:    vmv.x.s a3, v8
-; RV32ZVE32F-NEXT:    srai a2, a2, 31
-; RV32ZVE32F-NEXT:    vse32.v v8, (a1)
-; RV32ZVE32F-NEXT:    srai a3, a3, 31
-; RV32ZVE32F-NEXT:    sw a2, 4(a0)
-; RV32ZVE32F-NEXT:    sw a3, 12(a0)
+; RV32ZVE32F-NEXT:    vmv.x.s a1, v9
+; RV32ZVE32F-NEXT:    vmv.x.s a2, v8
+; RV32ZVE32F-NEXT:    srai a3, a1, 31
+; RV32ZVE32F-NEXT:    srai a4, a2, 31
+; RV32ZVE32F-NEXT:    sw a1, 0(a0)
+; RV32ZVE32F-NEXT:    sw a3, 4(a0)
+; RV32ZVE32F-NEXT:    sw a2, 8(a0)
+; RV32ZVE32F-NEXT:    sw a4, 12(a0)
 ; RV32ZVE32F-NEXT:    ret
 ;
 ; RV64ZVE32F-LABEL: mgather_v2i32_sextload_v2i64:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ssegN-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ssegN-store.ll
new file mode 100644
index 0000000000000..abf289419ec16
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-ssegN-store.ll
@@ -0,0 +1,72 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s
+
+define void @store_factor2(<8 x i8> %v0, <8 x i8> %v1, ptr %ptr, i64 %stride) {
+; CHECK-LABEL: store_factor2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vssseg2e8.v v8, (a0), a1
+; CHECK-NEXT:    ret
+    call void @llvm.riscv.sseg2.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
+    ret void
+}
+
+define void @store_factor3(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, ptr %ptr, i64 %stride) {
+; CHECK-LABEL: store_factor3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vssseg3e8.v v8, (a0), a1
+; CHECK-NEXT:    ret
+    call void @llvm.riscv.sseg3.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
+    ret void
+}
+
+define void @store_factor4(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, ptr %ptr, i64 %stride) {
+; CHECK-LABEL: store_factor4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vssseg4e8.v v8, (a0), a1
+; CHECK-NEXT:    ret
+    call void @llvm.riscv.sseg4.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
+    ret void
+}
+
+define void @store_factor5(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, ptr %ptr, i64 %stride) {
+; CHECK-LABEL: store_factor5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vssseg5e8.v v8, (a0), a1
+; CHECK-NEXT:    ret
+    call void @llvm.riscv.sseg5.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
+    ret void
+}
+
+define void @store_factor6(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, ptr %ptr, i64 %stride) {
+; CHECK-LABEL: store_factor6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vssseg6e8.v v8, (a0), a1
+; CHECK-NEXT:    ret
+    call void @llvm.riscv.sseg6.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
+    ret void
+}
+
+define void @store_factor7(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, <8 x i8> %v6, ptr %ptr, i64 %stride) {
+; CHECK-LABEL: store_factor7:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vssseg7e8.v v8, (a0), a1
+; CHECK-NEXT:    ret
+    call void @llvm.riscv.sseg7.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, <8 x i8> %v6, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
+    ret void
+}
+
+define void @store_factor8(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, <8 x i8> %v6, <8 x i8> %v7, ptr %ptr, i64 %stride) {
+; CHECK-LABEL: store_factor8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vssseg8e8.v v8, (a0), a1
+; CHECK-NEXT:    ret
+    call void @llvm.riscv.sseg8.store.mask.v8i8.i64.i64(<8 x i8> %v0, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3, <8 x i8> %v4, <8 x i8> %v5, <8 x i8> %v6, <8 x i8> %v7, ptr %ptr, i64 %stride, <8 x i1> splat (i1 true), i64 8)
+    ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll
index a04e31a19a4f1..902001a376d6a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll
@@ -434,19 +434,12 @@ define <256 x i8> @vsadd_vi_v258i8_evl129(<256 x i8> %va, <256 x i1> %m) {
   ret <256 x i8> %v
 }
 
-; FIXME: The upper half is doing nothing.
-
 define <256 x i8> @vsadd_vi_v258i8_evl128(<256 x i8> %va, <256 x i1> %m) {
 ; CHECK-LABEL: vsadd_vi_v258i8_evl128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 0, e8, m8, ta, ma
-; CHECK-NEXT:    vlm.v v24, (a0)
 ; CHECK-NEXT:    li a0, 128
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vsadd.vi v8, v8, -1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetivli zero, 0, e8, m8, ta, ma
-; CHECK-NEXT:    vsadd.vi v16, v16, -1, v0.t
 ; CHECK-NEXT:    ret
   %v = call <256 x i8> @llvm.vp.sadd.sat.v258i8(<256 x i8> %va, <256 x i8> splat (i8 -1), <256 x i1> %m, i32 128)
   ret <256 x i8> %v
@@ -1418,13 +1411,8 @@ define <32 x i64> @vsadd_vi_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 define <32 x i64> @vsadd_vx_v32i64_evl12(<32 x i64> %va, <32 x i1> %m) {
 ; CHECK-LABEL: vsadd_vx_v32i64_evl12:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v24, v0, 2
 ; CHECK-NEXT:    vsetivli zero, 12, e64, m8, ta, ma
 ; CHECK-NEXT:    vsadd.vi v8, v8, -1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetivli zero, 0, e64, m8, ta, ma
-; CHECK-NEXT:    vsadd.vi v16, v16, -1, v0.t
 ; CHECK-NEXT:    ret
   %v = call <32 x i64> @llvm.vp.sadd.sat.v32i64(<32 x i64> %va, <32 x i64> splat (i64 -1), <32 x i1> %m, i32 12)
   ret <32 x i64> %v
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll
index 5556b11e9a90c..57292147a0140 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsaddu-vp.ll
@@ -430,19 +430,12 @@ define <256 x i8> @vsaddu_vi_v258i8_evl129(<256 x i8> %va, <256 x i1> %m) {
   ret <256 x i8> %v
 }
 
-; FIXME: The upper half is doing nothing.
-
 define <256 x i8> @vsaddu_vi_v258i8_evl128(<256 x i8> %va, <256 x i1> %m) {
 ; CHECK-LABEL: vsaddu_vi_v258i8_evl128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 0, e8, m8, ta, ma
-; CHECK-NEXT:    vlm.v v24, (a0)
 ; CHECK-NEXT:    li a0, 128
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vsaddu.vi v8, v8, -1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetivli zero, 0, e8, m8, ta, ma
-; CHECK-NEXT:    vsaddu.vi v16, v16, -1, v0.t
 ; CHECK-NEXT:    ret
   %v = call <256 x i8> @llvm.vp.uadd.sat.v258i8(<256 x i8> %va, <256 x i8> splat (i8 -1), <256 x i1> %m, i32 128)
   ret <256 x i8> %v
@@ -1414,13 +1407,8 @@ define <32 x i64> @vsaddu_vi_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 define <32 x i64> @vsaddu_vx_v32i64_evl12(<32 x i64> %va, <32 x i1> %m) {
 ; CHECK-LABEL: vsaddu_vx_v32i64_evl12:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v24, v0, 2
 ; CHECK-NEXT:    vsetivli zero, 12, e64, m8, ta, ma
 ; CHECK-NEXT:    vsaddu.vi v8, v8, -1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetivli zero, 0, e64, m8, ta, ma
-; CHECK-NEXT:    vsaddu.vi v16, v16, -1, v0.t
 ; CHECK-NEXT:    ret
   %v = call <32 x i64> @llvm.vp.uadd.sat.v32i64(<32 x i64> %va, <32 x i64> splat (i64 -1), <32 x i1> %m, i32 12)
   ret <32 x i64> %v
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll
index c28317bf14269..353042fc889e5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssub-vp.ll
@@ -449,20 +449,13 @@ define <256 x i8> @vssub_vi_v258i8_evl129(<256 x i8> %va, <256 x i1> %m) {
   ret <256 x i8> %v
 }
 
-; FIXME: The upper half is doing nothing.
-
 define <256 x i8> @vssub_vi_v258i8_evl128(<256 x i8> %va, <256 x i1> %m) {
 ; CHECK-LABEL: vssub_vi_v258i8_evl128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 0, e8, m8, ta, ma
-; CHECK-NEXT:    vlm.v v24, (a0)
 ; CHECK-NEXT:    li a0, 128
 ; CHECK-NEXT:    li a1, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssub.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetivli zero, 0, e8, m8, ta, ma
-; CHECK-NEXT:    vssub.vx v16, v16, a1, v0.t
 ; CHECK-NEXT:    ret
   %v = call <256 x i8> @llvm.vp.ssub.sat.v258i8(<256 x i8> %va, <256 x i8> splat (i8 -1), <256 x i1> %m, i32 128)
   ret <256 x i8> %v
@@ -1460,14 +1453,9 @@ define <32 x i64> @vssub_vi_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 define <32 x i64> @vssub_vx_v32i64_evl12(<32 x i64> %va, <32 x i1> %m) {
 ; CHECK-LABEL: vssub_vx_v32i64_evl12:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v24, v0, 2
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vsetivli zero, 12, e64, m8, ta, ma
 ; CHECK-NEXT:    vssub.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetivli zero, 0, e64, m8, ta, ma
-; CHECK-NEXT:    vssub.vx v16, v16, a0, v0.t
 ; CHECK-NEXT:    ret
   %v = call <32 x i64> @llvm.vp.ssub.sat.v32i64(<32 x i64> %va, <32 x i64> splat (i64 -1), <32 x i1> %m, i32 12)
   ret <32 x i64> %v
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll
index cbfe1292877ee..c00fb329b2f0c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vssubu-vp.ll
@@ -444,20 +444,13 @@ define <256 x i8> @vssubu_vi_v258i8_evl129(<256 x i8> %va, <256 x i1> %m) {
   ret <256 x i8> %v
 }
 
-; FIXME: The upper half is doing nothing.
-
 define <256 x i8> @vssubu_vi_v258i8_evl128(<256 x i8> %va, <256 x i1> %m) {
 ; CHECK-LABEL: vssubu_vi_v258i8_evl128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 0, e8, m8, ta, ma
-; CHECK-NEXT:    vlm.v v24, (a0)
 ; CHECK-NEXT:    li a0, 128
 ; CHECK-NEXT:    li a1, -1
 ; CHECK-NEXT:    vsetvli zero, a0, e8, m8, ta, ma
 ; CHECK-NEXT:    vssubu.vx v8, v8, a1, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetivli zero, 0, e8, m8, ta, ma
-; CHECK-NEXT:    vssubu.vx v16, v16, a1, v0.t
 ; CHECK-NEXT:    ret
   %v = call <256 x i8> @llvm.vp.usub.sat.v258i8(<256 x i8> %va, <256 x i8> splat (i8 -1), <256 x i1> %m, i32 128)
   ret <256 x i8> %v
@@ -1455,14 +1448,9 @@ define <32 x i64> @vssubu_vi_v32i64_unmasked(<32 x i64> %va, i32 zeroext %evl) {
 define <32 x i64> @vssubu_vx_v32i64_evl12(<32 x i64> %va, <32 x i1> %m) {
 ; CHECK-LABEL: vssubu_vx_v32i64_evl12:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf4, ta, ma
-; CHECK-NEXT:    vslidedown.vi v24, v0, 2
 ; CHECK-NEXT:    li a0, -1
 ; CHECK-NEXT:    vsetivli zero, 12, e64, m8, ta, ma
 ; CHECK-NEXT:    vssubu.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetivli zero, 0, e64, m8, ta, ma
-; CHECK-NEXT:    vssubu.vx v16, v16, a0, v0.t
 ; CHECK-NEXT:    ret
   %v = call <32 x i64> @llvm.vp.usub.sat.v32i64(<32 x i64> %va, <32 x i64> splat (i64 -1), <32 x i1> %m, i32 12)
   ret <32 x i64> %v
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulu.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulu.ll
index 7cedee5068551..90e9ffdcb320a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulu.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vwmulu.ll
@@ -710,6 +710,13 @@ define <4 x i32> @vwmulu_vx_v4i32_i8(ptr %x, ptr %y) {
 }
 
 define <4 x i32> @vwmulu_vx_v4i32_i16(ptr %x, ptr %y) {
+; CHECK-LABEL: vwmulu_vx_v4i32_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vle16.v v9, (a0)
+; CHECK-NEXT:    lhu a0, 0(a1)
+; CHECK-NEXT:    vwmulu.vx v8, v9, a0
+; CHECK-NEXT:    ret
   %a = load <4 x i16>, ptr %x
   %b = load i16, ptr %y
   %c = zext i16 %b to i32
diff --git a/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll b/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
index 6787c8c24c87e..143545ccfd4f6 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fold-binary-reduce.ll
@@ -266,8 +266,7 @@ define float @reduce_fadd3(float %x, <4 x float> %v, ptr %rdxptr) {
 ; CHECK-NEXT:    vfredusum.vs v8, v8, v9
 ; CHECK-NEXT:    vfmv.f.s fa5, v8
 ; CHECK-NEXT:    fadd.s fa0, fa5, fa0
-; CHECK-NEXT:    vsetivli zero, 1, e32, m1, ta, ma
-; CHECK-NEXT:    vse32.v v8, (a0)
+; CHECK-NEXT:    fsw fa5, 0(a0)
 ; CHECK-NEXT:    ret
 entry:
   %rdx = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %v)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
index 9ef7f9441171c..aba9d37bfaa04 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fpclamptosat_vec.ll
@@ -2327,7 +2327,7 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    srli a3, a0, 1
 ; CHECK-NOV-NEXT:    beqz a1, .LBB18_3
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    slti a4, a1, 0
+; CHECK-NOV-NEXT:    srli a4, a1, 63
 ; CHECK-NOV-NEXT:    bnez s1, .LBB18_4
 ; CHECK-NOV-NEXT:  .LBB18_2:
 ; CHECK-NOV-NEXT:    sltu a5, s0, a3
@@ -2337,7 +2337,7 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    sltu a4, a2, a3
 ; CHECK-NOV-NEXT:    beqz s1, .LBB18_2
 ; CHECK-NOV-NEXT:  .LBB18_4: # %entry
-; CHECK-NOV-NEXT:    slti a5, s1, 0
+; CHECK-NOV-NEXT:    srli a5, s1, 63
 ; CHECK-NOV-NEXT:    bnez a5, .LBB18_6
 ; CHECK-NOV-NEXT:  .LBB18_5: # %entry
 ; CHECK-NOV-NEXT:    mv s0, a3
@@ -2353,8 +2353,8 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    slli a1, a0, 63
 ; CHECK-NOV-NEXT:    beq a5, a0, .LBB18_11
 ; CHECK-NOV-NEXT:  # %bb.9: # %entry
-; CHECK-NOV-NEXT:    slti a3, a5, 0
-; CHECK-NOV-NEXT:    xori a3, a3, 1
+; CHECK-NOV-NEXT:    srli a5, a5, 63
+; CHECK-NOV-NEXT:    xori a3, a5, 1
 ; CHECK-NOV-NEXT:    bne a4, a0, .LBB18_12
 ; CHECK-NOV-NEXT:  .LBB18_10:
 ; CHECK-NOV-NEXT:    sltu a0, a1, s0
@@ -2364,8 +2364,8 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    sltu a3, a1, a2
 ; CHECK-NOV-NEXT:    beq a4, a0, .LBB18_10
 ; CHECK-NOV-NEXT:  .LBB18_12: # %entry
-; CHECK-NOV-NEXT:    slti a0, a4, 0
-; CHECK-NOV-NEXT:    xori a0, a0, 1
+; CHECK-NOV-NEXT:    srli a4, a4, 63
+; CHECK-NOV-NEXT:    xori a0, a4, 1
 ; CHECK-NOV-NEXT:    bnez a0, .LBB18_14
 ; CHECK-NOV-NEXT:  .LBB18_13: # %entry
 ; CHECK-NOV-NEXT:    mv s0, a1
@@ -2415,7 +2415,7 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-V-NEXT:    srli a3, a2, 1
 ; CHECK-V-NEXT:    beqz a1, .LBB18_3
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    slti a4, a1, 0
+; CHECK-V-NEXT:    srli a4, a1, 63
 ; CHECK-V-NEXT:    bnez s1, .LBB18_4
 ; CHECK-V-NEXT:  .LBB18_2:
 ; CHECK-V-NEXT:    sltu a5, s0, a3
@@ -2425,7 +2425,7 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-V-NEXT:    sltu a4, a0, a3
 ; CHECK-V-NEXT:    beqz s1, .LBB18_2
 ; CHECK-V-NEXT:  .LBB18_4: # %entry
-; CHECK-V-NEXT:    slti a5, s1, 0
+; CHECK-V-NEXT:    srli a5, s1, 63
 ; CHECK-V-NEXT:    bnez a5, .LBB18_6
 ; CHECK-V-NEXT:  .LBB18_5: # %entry
 ; CHECK-V-NEXT:    mv s0, a3
@@ -2441,8 +2441,8 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-V-NEXT:    slli a1, a2, 63
 ; CHECK-V-NEXT:    beq a5, a2, .LBB18_11
 ; CHECK-V-NEXT:  # %bb.9: # %entry
-; CHECK-V-NEXT:    slti a3, a5, 0
-; CHECK-V-NEXT:    xori a3, a3, 1
+; CHECK-V-NEXT:    srli a5, a5, 63
+; CHECK-V-NEXT:    xori a3, a5, 1
 ; CHECK-V-NEXT:    bne a4, a2, .LBB18_12
 ; CHECK-V-NEXT:  .LBB18_10:
 ; CHECK-V-NEXT:    sltu a2, a1, s0
@@ -2452,8 +2452,8 @@ define <2 x i64> @stest_f64i64(<2 x double> %x) {
 ; CHECK-V-NEXT:    sltu a3, a1, a0
 ; CHECK-V-NEXT:    beq a4, a2, .LBB18_10
 ; CHECK-V-NEXT:  .LBB18_12: # %entry
-; CHECK-V-NEXT:    slti a2, a4, 0
-; CHECK-V-NEXT:    xori a2, a2, 1
+; CHECK-V-NEXT:    srli a4, a4, 63
+; CHECK-V-NEXT:    xori a2, a4, 1
 ; CHECK-V-NEXT:    bnez a2, .LBB18_14
 ; CHECK-V-NEXT:  .LBB18_13: # %entry
 ; CHECK-V-NEXT:    mv s0, a1
@@ -2749,7 +2749,7 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    srli a3, a0, 1
 ; CHECK-NOV-NEXT:    beqz a1, .LBB21_3
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    slti a4, a1, 0
+; CHECK-NOV-NEXT:    srli a4, a1, 63
 ; CHECK-NOV-NEXT:    bnez s1, .LBB21_4
 ; CHECK-NOV-NEXT:  .LBB21_2:
 ; CHECK-NOV-NEXT:    sltu a5, s0, a3
@@ -2759,7 +2759,7 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    sltu a4, a2, a3
 ; CHECK-NOV-NEXT:    beqz s1, .LBB21_2
 ; CHECK-NOV-NEXT:  .LBB21_4: # %entry
-; CHECK-NOV-NEXT:    slti a5, s1, 0
+; CHECK-NOV-NEXT:    srli a5, s1, 63
 ; CHECK-NOV-NEXT:    bnez a5, .LBB21_6
 ; CHECK-NOV-NEXT:  .LBB21_5: # %entry
 ; CHECK-NOV-NEXT:    mv s0, a3
@@ -2775,8 +2775,8 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    slli a1, a0, 63
 ; CHECK-NOV-NEXT:    beq a5, a0, .LBB21_11
 ; CHECK-NOV-NEXT:  # %bb.9: # %entry
-; CHECK-NOV-NEXT:    slti a3, a5, 0
-; CHECK-NOV-NEXT:    xori a3, a3, 1
+; CHECK-NOV-NEXT:    srli a5, a5, 63
+; CHECK-NOV-NEXT:    xori a3, a5, 1
 ; CHECK-NOV-NEXT:    bne a4, a0, .LBB21_12
 ; CHECK-NOV-NEXT:  .LBB21_10:
 ; CHECK-NOV-NEXT:    sltu a0, a1, s0
@@ -2786,8 +2786,8 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    sltu a3, a1, a2
 ; CHECK-NOV-NEXT:    beq a4, a0, .LBB21_10
 ; CHECK-NOV-NEXT:  .LBB21_12: # %entry
-; CHECK-NOV-NEXT:    slti a0, a4, 0
-; CHECK-NOV-NEXT:    xori a0, a0, 1
+; CHECK-NOV-NEXT:    srli a4, a4, 63
+; CHECK-NOV-NEXT:    xori a0, a4, 1
 ; CHECK-NOV-NEXT:    bnez a0, .LBB21_14
 ; CHECK-NOV-NEXT:  .LBB21_13: # %entry
 ; CHECK-NOV-NEXT:    mv s0, a1
@@ -2837,7 +2837,7 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-V-NEXT:    srli a3, a2, 1
 ; CHECK-V-NEXT:    beqz a1, .LBB21_3
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    slti a4, a1, 0
+; CHECK-V-NEXT:    srli a4, a1, 63
 ; CHECK-V-NEXT:    bnez s1, .LBB21_4
 ; CHECK-V-NEXT:  .LBB21_2:
 ; CHECK-V-NEXT:    sltu a5, s0, a3
@@ -2847,7 +2847,7 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-V-NEXT:    sltu a4, a0, a3
 ; CHECK-V-NEXT:    beqz s1, .LBB21_2
 ; CHECK-V-NEXT:  .LBB21_4: # %entry
-; CHECK-V-NEXT:    slti a5, s1, 0
+; CHECK-V-NEXT:    srli a5, s1, 63
 ; CHECK-V-NEXT:    bnez a5, .LBB21_6
 ; CHECK-V-NEXT:  .LBB21_5: # %entry
 ; CHECK-V-NEXT:    mv s0, a3
@@ -2863,8 +2863,8 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-V-NEXT:    slli a1, a2, 63
 ; CHECK-V-NEXT:    beq a5, a2, .LBB21_11
 ; CHECK-V-NEXT:  # %bb.9: # %entry
-; CHECK-V-NEXT:    slti a3, a5, 0
-; CHECK-V-NEXT:    xori a3, a3, 1
+; CHECK-V-NEXT:    srli a5, a5, 63
+; CHECK-V-NEXT:    xori a3, a5, 1
 ; CHECK-V-NEXT:    bne a4, a2, .LBB21_12
 ; CHECK-V-NEXT:  .LBB21_10:
 ; CHECK-V-NEXT:    sltu a2, a1, s0
@@ -2874,8 +2874,8 @@ define <2 x i64> @stest_f32i64(<2 x float> %x) {
 ; CHECK-V-NEXT:    sltu a3, a1, a0
 ; CHECK-V-NEXT:    beq a4, a2, .LBB21_10
 ; CHECK-V-NEXT:  .LBB21_12: # %entry
-; CHECK-V-NEXT:    slti a2, a4, 0
-; CHECK-V-NEXT:    xori a2, a2, 1
+; CHECK-V-NEXT:    srli a4, a4, 63
+; CHECK-V-NEXT:    xori a2, a4, 1
 ; CHECK-V-NEXT:    bnez a2, .LBB21_14
 ; CHECK-V-NEXT:  .LBB21_13: # %entry
 ; CHECK-V-NEXT:    mv s0, a1
@@ -3174,7 +3174,7 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    srli a3, a0, 1
 ; CHECK-NOV-NEXT:    beqz a1, .LBB24_3
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    slti a4, a1, 0
+; CHECK-NOV-NEXT:    srli a4, a1, 63
 ; CHECK-NOV-NEXT:    bnez s1, .LBB24_4
 ; CHECK-NOV-NEXT:  .LBB24_2:
 ; CHECK-NOV-NEXT:    sltu a5, s0, a3
@@ -3184,7 +3184,7 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    sltu a4, a2, a3
 ; CHECK-NOV-NEXT:    beqz s1, .LBB24_2
 ; CHECK-NOV-NEXT:  .LBB24_4: # %entry
-; CHECK-NOV-NEXT:    slti a5, s1, 0
+; CHECK-NOV-NEXT:    srli a5, s1, 63
 ; CHECK-NOV-NEXT:    bnez a5, .LBB24_6
 ; CHECK-NOV-NEXT:  .LBB24_5: # %entry
 ; CHECK-NOV-NEXT:    mv s0, a3
@@ -3200,8 +3200,8 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    slli a1, a0, 63
 ; CHECK-NOV-NEXT:    beq a5, a0, .LBB24_11
 ; CHECK-NOV-NEXT:  # %bb.9: # %entry
-; CHECK-NOV-NEXT:    slti a3, a5, 0
-; CHECK-NOV-NEXT:    xori a3, a3, 1
+; CHECK-NOV-NEXT:    srli a5, a5, 63
+; CHECK-NOV-NEXT:    xori a3, a5, 1
 ; CHECK-NOV-NEXT:    bne a4, a0, .LBB24_12
 ; CHECK-NOV-NEXT:  .LBB24_10:
 ; CHECK-NOV-NEXT:    sltu a0, a1, s0
@@ -3211,8 +3211,8 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    sltu a3, a1, a2
 ; CHECK-NOV-NEXT:    beq a4, a0, .LBB24_10
 ; CHECK-NOV-NEXT:  .LBB24_12: # %entry
-; CHECK-NOV-NEXT:    slti a0, a4, 0
-; CHECK-NOV-NEXT:    xori a0, a0, 1
+; CHECK-NOV-NEXT:    srli a4, a4, 63
+; CHECK-NOV-NEXT:    xori a0, a4, 1
 ; CHECK-NOV-NEXT:    bnez a0, .LBB24_14
 ; CHECK-NOV-NEXT:  .LBB24_13: # %entry
 ; CHECK-NOV-NEXT:    mv s0, a1
@@ -3260,7 +3260,7 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-V-NEXT:    srli a3, a2, 1
 ; CHECK-V-NEXT:    beqz a1, .LBB24_3
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    slti a4, a1, 0
+; CHECK-V-NEXT:    srli a4, a1, 63
 ; CHECK-V-NEXT:    bnez s1, .LBB24_4
 ; CHECK-V-NEXT:  .LBB24_2:
 ; CHECK-V-NEXT:    sltu a5, s0, a3
@@ -3270,7 +3270,7 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-V-NEXT:    sltu a4, a0, a3
 ; CHECK-V-NEXT:    beqz s1, .LBB24_2
 ; CHECK-V-NEXT:  .LBB24_4: # %entry
-; CHECK-V-NEXT:    slti a5, s1, 0
+; CHECK-V-NEXT:    srli a5, s1, 63
 ; CHECK-V-NEXT:    bnez a5, .LBB24_6
 ; CHECK-V-NEXT:  .LBB24_5: # %entry
 ; CHECK-V-NEXT:    mv s0, a3
@@ -3286,8 +3286,8 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-V-NEXT:    slli a1, a2, 63
 ; CHECK-V-NEXT:    beq a5, a2, .LBB24_11
 ; CHECK-V-NEXT:  # %bb.9: # %entry
-; CHECK-V-NEXT:    slti a3, a5, 0
-; CHECK-V-NEXT:    xori a3, a3, 1
+; CHECK-V-NEXT:    srli a5, a5, 63
+; CHECK-V-NEXT:    xori a3, a5, 1
 ; CHECK-V-NEXT:    bne a4, a2, .LBB24_12
 ; CHECK-V-NEXT:  .LBB24_10:
 ; CHECK-V-NEXT:    sltu a2, a1, s0
@@ -3297,8 +3297,8 @@ define <2 x i64> @stest_f16i64(<2 x half> %x) {
 ; CHECK-V-NEXT:    sltu a3, a1, a0
 ; CHECK-V-NEXT:    beq a4, a2, .LBB24_10
 ; CHECK-V-NEXT:  .LBB24_12: # %entry
-; CHECK-V-NEXT:    slti a2, a4, 0
-; CHECK-V-NEXT:    xori a2, a2, 1
+; CHECK-V-NEXT:    srli a4, a4, 63
+; CHECK-V-NEXT:    xori a2, a4, 1
 ; CHECK-V-NEXT:    bnez a2, .LBB24_14
 ; CHECK-V-NEXT:  .LBB24_13: # %entry
 ; CHECK-V-NEXT:    mv s0, a1
@@ -5864,7 +5864,7 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    srli a3, a0, 1
 ; CHECK-NOV-NEXT:    beqz a1, .LBB45_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    slti a4, a1, 0
+; CHECK-NOV-NEXT:    srli a4, a1, 63
 ; CHECK-NOV-NEXT:    beqz a4, .LBB45_3
 ; CHECK-NOV-NEXT:    j .LBB45_4
 ; CHECK-NOV-NEXT:  .LBB45_2:
@@ -5875,7 +5875,7 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NOV-NEXT:  .LBB45_4: # %entry
 ; CHECK-NOV-NEXT:    beqz s1, .LBB45_6
 ; CHECK-NOV-NEXT:  # %bb.5: # %entry
-; CHECK-NOV-NEXT:    slti a6, s1, 0
+; CHECK-NOV-NEXT:    srli a6, s1, 63
 ; CHECK-NOV-NEXT:    j .LBB45_7
 ; CHECK-NOV-NEXT:  .LBB45_6:
 ; CHECK-NOV-NEXT:    sltu a6, s0, a3
@@ -5890,7 +5890,7 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NOV-NEXT:    slli a3, a0, 63
 ; CHECK-NOV-NEXT:    beq a5, a0, .LBB45_11
 ; CHECK-NOV-NEXT:  # %bb.10: # %entry
-; CHECK-NOV-NEXT:    slti a5, a5, 0
+; CHECK-NOV-NEXT:    srli a5, a5, 63
 ; CHECK-NOV-NEXT:    xori a5, a5, 1
 ; CHECK-NOV-NEXT:    and a1, a4, a1
 ; CHECK-NOV-NEXT:    beqz a5, .LBB45_12
@@ -5904,8 +5904,8 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NOV-NEXT:  .LBB45_13: # %entry
 ; CHECK-NOV-NEXT:    beq a1, a0, .LBB45_15
 ; CHECK-NOV-NEXT:  # %bb.14: # %entry
-; CHECK-NOV-NEXT:    slti a0, a1, 0
-; CHECK-NOV-NEXT:    xori a0, a0, 1
+; CHECK-NOV-NEXT:    srli a1, a1, 63
+; CHECK-NOV-NEXT:    xori a0, a1, 1
 ; CHECK-NOV-NEXT:    beqz a0, .LBB45_16
 ; CHECK-NOV-NEXT:    j .LBB45_17
 ; CHECK-NOV-NEXT:  .LBB45_15:
@@ -5955,7 +5955,7 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:    srli a3, a2, 1
 ; CHECK-V-NEXT:    beqz a1, .LBB45_2
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    slti a4, a1, 0
+; CHECK-V-NEXT:    srli a4, a1, 63
 ; CHECK-V-NEXT:    beqz a4, .LBB45_3
 ; CHECK-V-NEXT:    j .LBB45_4
 ; CHECK-V-NEXT:  .LBB45_2:
@@ -5966,7 +5966,7 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:  .LBB45_4: # %entry
 ; CHECK-V-NEXT:    beqz s1, .LBB45_6
 ; CHECK-V-NEXT:  # %bb.5: # %entry
-; CHECK-V-NEXT:    slti a6, s1, 0
+; CHECK-V-NEXT:    srli a6, s1, 63
 ; CHECK-V-NEXT:    j .LBB45_7
 ; CHECK-V-NEXT:  .LBB45_6:
 ; CHECK-V-NEXT:    sltu a6, s0, a3
@@ -5981,7 +5981,7 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:    slli a3, a2, 63
 ; CHECK-V-NEXT:    beq a5, a2, .LBB45_11
 ; CHECK-V-NEXT:  # %bb.10: # %entry
-; CHECK-V-NEXT:    slti a5, a5, 0
+; CHECK-V-NEXT:    srli a5, a5, 63
 ; CHECK-V-NEXT:    xori a5, a5, 1
 ; CHECK-V-NEXT:    and a1, a4, a1
 ; CHECK-V-NEXT:    beqz a5, .LBB45_12
@@ -5995,7 +5995,7 @@ define <2 x i64> @stest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:  .LBB45_13: # %entry
 ; CHECK-V-NEXT:    beq a1, a2, .LBB45_15
 ; CHECK-V-NEXT:  # %bb.14: # %entry
-; CHECK-V-NEXT:    slti a1, a1, 0
+; CHECK-V-NEXT:    srli a1, a1, 63
 ; CHECK-V-NEXT:    xori a1, a1, 1
 ; CHECK-V-NEXT:    beqz a1, .LBB45_16
 ; CHECK-V-NEXT:    j .LBB45_17
@@ -6153,8 +6153,8 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-NOV-NEXT:  .LBB47_4: # %entry
 ; CHECK-NOV-NEXT:    slti a1, a1, 1
 ; CHECK-NOV-NEXT:    slti a4, s1, 1
-; CHECK-NOV-NEXT:    slti a3, a3, 0
-; CHECK-NOV-NEXT:    slti a2, a2, 0
+; CHECK-NOV-NEXT:    srli a3, a3, 63
+; CHECK-NOV-NEXT:    srli a2, a2, 63
 ; CHECK-NOV-NEXT:    neg a1, a1
 ; CHECK-NOV-NEXT:    neg a4, a4
 ; CHECK-NOV-NEXT:    addi a3, a3, -1
@@ -6210,8 +6210,8 @@ define <2 x i64> @ustest_f64i64_mm(<2 x double> %x) {
 ; CHECK-V-NEXT:  .LBB47_4: # %entry
 ; CHECK-V-NEXT:    slti a1, a1, 1
 ; CHECK-V-NEXT:    slti a4, s1, 1
-; CHECK-V-NEXT:    slti a3, a3, 0
-; CHECK-V-NEXT:    slti a2, a2, 0
+; CHECK-V-NEXT:    srli a3, a3, 63
+; CHECK-V-NEXT:    srli a2, a2, 63
 ; CHECK-V-NEXT:    neg a1, a1
 ; CHECK-V-NEXT:    neg a4, a4
 ; CHECK-V-NEXT:    addi a3, a3, -1
@@ -6268,7 +6268,7 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    srli a3, a0, 1
 ; CHECK-NOV-NEXT:    beqz a1, .LBB48_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    slti a4, a1, 0
+; CHECK-NOV-NEXT:    srli a4, a1, 63
 ; CHECK-NOV-NEXT:    beqz a4, .LBB48_3
 ; CHECK-NOV-NEXT:    j .LBB48_4
 ; CHECK-NOV-NEXT:  .LBB48_2:
@@ -6279,7 +6279,7 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB48_4: # %entry
 ; CHECK-NOV-NEXT:    beqz s1, .LBB48_6
 ; CHECK-NOV-NEXT:  # %bb.5: # %entry
-; CHECK-NOV-NEXT:    slti a6, s1, 0
+; CHECK-NOV-NEXT:    srli a6, s1, 63
 ; CHECK-NOV-NEXT:    j .LBB48_7
 ; CHECK-NOV-NEXT:  .LBB48_6:
 ; CHECK-NOV-NEXT:    sltu a6, s0, a3
@@ -6294,7 +6294,7 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NOV-NEXT:    slli a3, a0, 63
 ; CHECK-NOV-NEXT:    beq a5, a0, .LBB48_11
 ; CHECK-NOV-NEXT:  # %bb.10: # %entry
-; CHECK-NOV-NEXT:    slti a5, a5, 0
+; CHECK-NOV-NEXT:    srli a5, a5, 63
 ; CHECK-NOV-NEXT:    xori a5, a5, 1
 ; CHECK-NOV-NEXT:    and a1, a4, a1
 ; CHECK-NOV-NEXT:    beqz a5, .LBB48_12
@@ -6308,8 +6308,8 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB48_13: # %entry
 ; CHECK-NOV-NEXT:    beq a1, a0, .LBB48_15
 ; CHECK-NOV-NEXT:  # %bb.14: # %entry
-; CHECK-NOV-NEXT:    slti a0, a1, 0
-; CHECK-NOV-NEXT:    xori a0, a0, 1
+; CHECK-NOV-NEXT:    srli a1, a1, 63
+; CHECK-NOV-NEXT:    xori a0, a1, 1
 ; CHECK-NOV-NEXT:    beqz a0, .LBB48_16
 ; CHECK-NOV-NEXT:    j .LBB48_17
 ; CHECK-NOV-NEXT:  .LBB48_15:
@@ -6359,7 +6359,7 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:    srli a3, a2, 1
 ; CHECK-V-NEXT:    beqz a1, .LBB48_2
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    slti a4, a1, 0
+; CHECK-V-NEXT:    srli a4, a1, 63
 ; CHECK-V-NEXT:    beqz a4, .LBB48_3
 ; CHECK-V-NEXT:    j .LBB48_4
 ; CHECK-V-NEXT:  .LBB48_2:
@@ -6370,7 +6370,7 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:  .LBB48_4: # %entry
 ; CHECK-V-NEXT:    beqz s1, .LBB48_6
 ; CHECK-V-NEXT:  # %bb.5: # %entry
-; CHECK-V-NEXT:    slti a6, s1, 0
+; CHECK-V-NEXT:    srli a6, s1, 63
 ; CHECK-V-NEXT:    j .LBB48_7
 ; CHECK-V-NEXT:  .LBB48_6:
 ; CHECK-V-NEXT:    sltu a6, s0, a3
@@ -6385,7 +6385,7 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:    slli a3, a2, 63
 ; CHECK-V-NEXT:    beq a5, a2, .LBB48_11
 ; CHECK-V-NEXT:  # %bb.10: # %entry
-; CHECK-V-NEXT:    slti a5, a5, 0
+; CHECK-V-NEXT:    srli a5, a5, 63
 ; CHECK-V-NEXT:    xori a5, a5, 1
 ; CHECK-V-NEXT:    and a1, a4, a1
 ; CHECK-V-NEXT:    beqz a5, .LBB48_12
@@ -6399,7 +6399,7 @@ define <2 x i64> @stest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:  .LBB48_13: # %entry
 ; CHECK-V-NEXT:    beq a1, a2, .LBB48_15
 ; CHECK-V-NEXT:  # %bb.14: # %entry
-; CHECK-V-NEXT:    slti a1, a1, 0
+; CHECK-V-NEXT:    srli a1, a1, 63
 ; CHECK-V-NEXT:    xori a1, a1, 1
 ; CHECK-V-NEXT:    beqz a1, .LBB48_16
 ; CHECK-V-NEXT:    j .LBB48_17
@@ -6557,8 +6557,8 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-NOV-NEXT:  .LBB50_4: # %entry
 ; CHECK-NOV-NEXT:    slti a1, a1, 1
 ; CHECK-NOV-NEXT:    slti a4, s1, 1
-; CHECK-NOV-NEXT:    slti a3, a3, 0
-; CHECK-NOV-NEXT:    slti a2, a2, 0
+; CHECK-NOV-NEXT:    srli a3, a3, 63
+; CHECK-NOV-NEXT:    srli a2, a2, 63
 ; CHECK-NOV-NEXT:    neg a1, a1
 ; CHECK-NOV-NEXT:    neg a4, a4
 ; CHECK-NOV-NEXT:    addi a3, a3, -1
@@ -6614,8 +6614,8 @@ define <2 x i64> @ustest_f32i64_mm(<2 x float> %x) {
 ; CHECK-V-NEXT:  .LBB50_4: # %entry
 ; CHECK-V-NEXT:    slti a1, a1, 1
 ; CHECK-V-NEXT:    slti a4, s1, 1
-; CHECK-V-NEXT:    slti a3, a3, 0
-; CHECK-V-NEXT:    slti a2, a2, 0
+; CHECK-V-NEXT:    srli a3, a3, 63
+; CHECK-V-NEXT:    srli a2, a2, 63
 ; CHECK-V-NEXT:    neg a1, a1
 ; CHECK-V-NEXT:    neg a4, a4
 ; CHECK-V-NEXT:    addi a3, a3, -1
@@ -6675,7 +6675,7 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    srli a3, a0, 1
 ; CHECK-NOV-NEXT:    beqz a1, .LBB51_2
 ; CHECK-NOV-NEXT:  # %bb.1: # %entry
-; CHECK-NOV-NEXT:    slti a4, a1, 0
+; CHECK-NOV-NEXT:    srli a4, a1, 63
 ; CHECK-NOV-NEXT:    beqz a4, .LBB51_3
 ; CHECK-NOV-NEXT:    j .LBB51_4
 ; CHECK-NOV-NEXT:  .LBB51_2:
@@ -6686,7 +6686,7 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB51_4: # %entry
 ; CHECK-NOV-NEXT:    beqz s1, .LBB51_6
 ; CHECK-NOV-NEXT:  # %bb.5: # %entry
-; CHECK-NOV-NEXT:    slti a6, s1, 0
+; CHECK-NOV-NEXT:    srli a6, s1, 63
 ; CHECK-NOV-NEXT:    j .LBB51_7
 ; CHECK-NOV-NEXT:  .LBB51_6:
 ; CHECK-NOV-NEXT:    sltu a6, s0, a3
@@ -6701,7 +6701,7 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NOV-NEXT:    slli a3, a0, 63
 ; CHECK-NOV-NEXT:    beq a5, a0, .LBB51_11
 ; CHECK-NOV-NEXT:  # %bb.10: # %entry
-; CHECK-NOV-NEXT:    slti a5, a5, 0
+; CHECK-NOV-NEXT:    srli a5, a5, 63
 ; CHECK-NOV-NEXT:    xori a5, a5, 1
 ; CHECK-NOV-NEXT:    and a1, a4, a1
 ; CHECK-NOV-NEXT:    beqz a5, .LBB51_12
@@ -6715,8 +6715,8 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB51_13: # %entry
 ; CHECK-NOV-NEXT:    beq a1, a0, .LBB51_15
 ; CHECK-NOV-NEXT:  # %bb.14: # %entry
-; CHECK-NOV-NEXT:    slti a0, a1, 0
-; CHECK-NOV-NEXT:    xori a0, a0, 1
+; CHECK-NOV-NEXT:    srli a1, a1, 63
+; CHECK-NOV-NEXT:    xori a0, a1, 1
 ; CHECK-NOV-NEXT:    beqz a0, .LBB51_16
 ; CHECK-NOV-NEXT:    j .LBB51_17
 ; CHECK-NOV-NEXT:  .LBB51_15:
@@ -6764,7 +6764,7 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-V-NEXT:    srli a3, a2, 1
 ; CHECK-V-NEXT:    beqz a1, .LBB51_2
 ; CHECK-V-NEXT:  # %bb.1: # %entry
-; CHECK-V-NEXT:    slti a4, a1, 0
+; CHECK-V-NEXT:    srli a4, a1, 63
 ; CHECK-V-NEXT:    beqz a4, .LBB51_3
 ; CHECK-V-NEXT:    j .LBB51_4
 ; CHECK-V-NEXT:  .LBB51_2:
@@ -6775,7 +6775,7 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-V-NEXT:  .LBB51_4: # %entry
 ; CHECK-V-NEXT:    beqz s1, .LBB51_6
 ; CHECK-V-NEXT:  # %bb.5: # %entry
-; CHECK-V-NEXT:    slti a6, s1, 0
+; CHECK-V-NEXT:    srli a6, s1, 63
 ; CHECK-V-NEXT:    j .LBB51_7
 ; CHECK-V-NEXT:  .LBB51_6:
 ; CHECK-V-NEXT:    sltu a6, s0, a3
@@ -6790,7 +6790,7 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-V-NEXT:    slli a3, a2, 63
 ; CHECK-V-NEXT:    beq a5, a2, .LBB51_11
 ; CHECK-V-NEXT:  # %bb.10: # %entry
-; CHECK-V-NEXT:    slti a5, a5, 0
+; CHECK-V-NEXT:    srli a5, a5, 63
 ; CHECK-V-NEXT:    xori a5, a5, 1
 ; CHECK-V-NEXT:    and a1, a4, a1
 ; CHECK-V-NEXT:    beqz a5, .LBB51_12
@@ -6804,7 +6804,7 @@ define <2 x i64> @stest_f16i64_mm(<2 x half> %x) {
 ; CHECK-V-NEXT:  .LBB51_13: # %entry
 ; CHECK-V-NEXT:    beq a1, a2, .LBB51_15
 ; CHECK-V-NEXT:  # %bb.14: # %entry
-; CHECK-V-NEXT:    slti a1, a1, 0
+; CHECK-V-NEXT:    srli a1, a1, 63
 ; CHECK-V-NEXT:    xori a1, a1, 1
 ; CHECK-V-NEXT:    beqz a1, .LBB51_16
 ; CHECK-V-NEXT:    j .LBB51_17
@@ -6960,8 +6960,8 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-NOV-NEXT:  .LBB53_4: # %entry
 ; CHECK-NOV-NEXT:    slti a1, a1, 1
 ; CHECK-NOV-NEXT:    slti a4, s1, 1
-; CHECK-NOV-NEXT:    slti a3, a3, 0
-; CHECK-NOV-NEXT:    slti a2, a2, 0
+; CHECK-NOV-NEXT:    srli a3, a3, 63
+; CHECK-NOV-NEXT:    srli a2, a2, 63
 ; CHECK-NOV-NEXT:    neg a1, a1
 ; CHECK-NOV-NEXT:    neg a4, a4
 ; CHECK-NOV-NEXT:    addi a3, a3, -1
@@ -7015,8 +7015,8 @@ define <2 x i64> @ustest_f16i64_mm(<2 x half> %x) {
 ; CHECK-V-NEXT:  .LBB53_4: # %entry
 ; CHECK-V-NEXT:    slti a1, a1, 1
 ; CHECK-V-NEXT:    slti a4, s1, 1
-; CHECK-V-NEXT:    slti a3, a3, 0
-; CHECK-V-NEXT:    slti a2, a2, 0
+; CHECK-V-NEXT:    srli a3, a3, 63
+; CHECK-V-NEXT:    srli a2, a2, 63
 ; CHECK-V-NEXT:    neg a1, a1
 ; CHECK-V-NEXT:    neg a4, a4
 ; CHECK-V-NEXT:    addi a3, a3, -1
diff --git a/llvm/test/CodeGen/RISCV/rvv/incorrect-extract-subvector-combine.ll b/llvm/test/CodeGen/RISCV/rvv/incorrect-extract-subvector-combine.ll
new file mode 100644
index 0000000000000..68d327d20c669
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/incorrect-extract-subvector-combine.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv64 -mattr=+zve64f,+zvl512b < %s | FileCheck %s
+
+; Previously, an incorrect (extract_subvector (extract_subvector X, C), 0) DAG combine crashed
+; this snippet.
+
+define <8 x i16> @gsm_encode(ptr %p) {
+; CHECK-LABEL: gsm_encode:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vsetivli zero, 19, e16, m1, ta, ma
+; CHECK-NEXT:    vle16.v v8, (a0)
+; CHECK-NEXT:    vslidedown.vi v9, v8, 12
+; CHECK-NEXT:    vmv.x.s a0, v9
+; CHECK-NEXT:    vsetivli zero, 8, e16, mf4, ta, ma
+; CHECK-NEXT:    vmv.v.i v9, -1
+; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 9
+; CHECK-NEXT:    vmv.x.s a1, v8
+; CHECK-NEXT:    vsetivli zero, 8, e16, mf4, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vslide1down.vx v9, v9, zero
+; CHECK-NEXT:    vslide1down.vx v8, v8, zero
+; CHECK-NEXT:    vslide1down.vx v8, v8, zero
+; CHECK-NEXT:    vslide1down.vx v8, v8, zero
+; CHECK-NEXT:    vslide1down.vx v8, v8, zero
+; CHECK-NEXT:    vslide1down.vx v8, v8, a1
+; CHECK-NEXT:    vslide1down.vx v8, v8, a0
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
+; CHECK-NEXT:    vand.vv v8, v8, v9
+; CHECK-NEXT:    ret
+entry:
+  %0 = load <19 x i16>, ptr %p, align 2
+  %1 = shufflevector <19 x i16> zeroinitializer, <19 x i16> %0, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 28, i32 31, i32 poison, i32 poison>
+  %2 = shufflevector <9 x i16> %1, <9 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+  ret <8 x i16> %2
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/large-rvv-stack-size.mir b/llvm/test/CodeGen/RISCV/rvv/large-rvv-stack-size.mir
index 89f4950366468..9d02dc79dde2f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/large-rvv-stack-size.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/large-rvv-stack-size.mir
@@ -68,8 +68,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 2048, alignment: 128,
diff --git a/llvm/test/CodeGen/RISCV/rvv/redundant-vfmvsf.ll b/llvm/test/CodeGen/RISCV/rvv/redundant-vfmvsf.ll
new file mode 100644
index 0000000000000..da912bf401ec0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/redundant-vfmvsf.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv64 -mattr='+v,+zvl512b' < %s | FileCheck %s
+
+define <2 x float> @redundant_vfmv(<2 x float> %arg0, <64 x float> %arg1, <64 x float> %arg2) {
+; CHECK-LABEL: redundant_vfmv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfredusum.vs v9, v12, v8
+; CHECK-NEXT:    vsetivli zero, 1, e32, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v8, v8, 1
+; CHECK-NEXT:    vsetvli zero, a0, e32, m4, ta, ma
+; CHECK-NEXT:    vfredusum.vs v8, v16, v8
+; CHECK-NEXT:    vfmv.f.s fa5, v8
+; CHECK-NEXT:    vsetivli zero, 2, e32, mf2, ta, ma
+; CHECK-NEXT:    vrgather.vi v8, v9, 0
+; CHECK-NEXT:    vfslide1down.vf v8, v8, fa5
+; CHECK-NEXT:    ret
+  %s0 = extractelement <2 x float> %arg0, i64 0
+  %r0 = tail call reassoc float @llvm.vector.reduce.fadd.v64f32(float %s0, <64 x float> %arg1)
+  %a0 = insertelement <2 x float> poison, float %r0, i64 0
+  %s1 = extractelement <2 x float> %arg0, i64 1
+  %r1 = tail call reassoc float @llvm.vector.reduce.fadd.v64f32(float %s1, <64 x float> %arg2)
+  %a1 = insertelement <2 x float> %a0, float %r1, i64 1
+  ret <2 x float> %a1
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
index 663bb1fc15517..d69a166b04080 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv32-spill-zvlsseg.ll
@@ -41,14 +41,11 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; SPILL-O2-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
-; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
-; SPILL-O2-NEXT:    addi a0, sp, 16
 ; SPILL-O2-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
@@ -64,15 +61,11 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 16
-; SPILL-O2-VLEN128-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
-; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 16
 ; SPILL-O2-VLEN128-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 16
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 32
@@ -108,14 +101,11 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
-; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
-; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
 ; SPILL-O2-VSETVLI-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
@@ -161,14 +151,11 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; SPILL-O2-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
-; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
-; SPILL-O2-NEXT:    addi a0, sp, 16
 ; SPILL-O2-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
@@ -184,15 +171,11 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 16
-; SPILL-O2-VLEN128-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
-; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 16
 ; SPILL-O2-VLEN128-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 16
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 32
@@ -228,14 +211,11 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
-; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
-; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
 ; SPILL-O2-VSETVLI-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
@@ -283,17 +263,12 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
-; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    slli a1, a1, 1
-; SPILL-O2-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
-; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    slli a1, a1, 1
-; SPILL-O2-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
@@ -309,15 +284,11 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 32
-; SPILL-O2-VLEN128-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
-; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 32
 ; SPILL-O2-VLEN128-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 32
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 64
@@ -353,17 +324,12 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
-; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 1
-; SPILL-O2-VSETVLI-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
-; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
+; SPILL-O2-VSETVLI-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 1
-; SPILL-O2-VSETVLI-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
@@ -411,17 +377,12 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; SPILL-O2-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
-; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    slli a1, a1, 2
-; SPILL-O2-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs4r.v v12, (a0) # vscale x 32-byte Folded Spill
+; SPILL-O2-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
-; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    slli a1, a1, 2
-; SPILL-O2-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
@@ -437,15 +398,11 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 64
-; SPILL-O2-VLEN128-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs4r.v v12, (a0) # vscale x 32-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
-; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 64
 ; SPILL-O2-VLEN128-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 64
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 128
@@ -481,17 +438,12 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
-; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 2
-; SPILL-O2-VSETVLI-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs4r.v v12, (a0) # vscale x 32-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
-; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
+; SPILL-O2-VSETVLI-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 2
-; SPILL-O2-VSETVLI-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
@@ -540,23 +492,19 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    slli a1, a1, 1
-; SPILL-O2-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-NEXT:    slli a1, a1, 2
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vs2r.v v12, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
 ; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    slli a1, a1, 1
-; SPILL-O2-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vl2r.v v10, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
 ; SPILL-O2-NEXT:    li a1, 6
 ; SPILL-O2-NEXT:    mul a0, a0, a1
@@ -571,21 +519,17 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 32
-; SPILL-O2-VLEN128-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    li a1, 64
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vs2r.v v12, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 32
 ; SPILL-O2-VLEN128-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 32
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vl2r.v v10, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 96
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 16
 ; SPILL-O2-VLEN128-NEXT:    ret
@@ -621,23 +565,19 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i32 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
+; SPILL-O2-VSETVLI-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 1
-; SPILL-O2-VSETVLI-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 2
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vs2r.v v12, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
+; SPILL-O2-VSETVLI-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 1
-; SPILL-O2-VSETVLI-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vl2r.v v10, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-VSETVLI-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    li a1, 6
 ; SPILL-O2-VSETVLI-NEXT:    mul a0, a0, a1
diff --git a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
index dc0e8fd987c6d..610443845389a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/rv64-spill-zvlsseg.ll
@@ -41,14 +41,11 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; SPILL-O2-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
-; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
-; SPILL-O2-NEXT:    addi a0, sp, 16
 ; SPILL-O2-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
@@ -64,15 +61,11 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 16
-; SPILL-O2-VLEN128-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
-; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 16
 ; SPILL-O2-VLEN128-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 16
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 32
@@ -108,14 +101,11 @@ define <vscale x 1 x i32> @spill_zvlsseg_nxv1i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, mf2, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
-; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
-; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
 ; SPILL-O2-VSETVLI-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
@@ -161,14 +151,11 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; SPILL-O2-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
-; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
-; SPILL-O2-NEXT:    addi a0, sp, 16
 ; SPILL-O2-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
@@ -184,15 +171,11 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 16
-; SPILL-O2-VLEN128-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
-; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 16
 ; SPILL-O2-VLEN128-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 16
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 32
@@ -228,14 +211,11 @@ define <vscale x 2 x i32> @spill_zvlsseg_nxv2i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
-; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
-; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
 ; SPILL-O2-VSETVLI-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
@@ -283,17 +263,12 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
-; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    slli a1, a1, 1
-; SPILL-O2-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
-; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    slli a1, a1, 1
-; SPILL-O2-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
@@ -309,15 +284,11 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 32
-; SPILL-O2-VLEN128-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
-; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 32
 ; SPILL-O2-VLEN128-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 32
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 64
@@ -353,17 +324,12 @@ define <vscale x 4 x i32> @spill_zvlsseg_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
-; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 1
-; SPILL-O2-VSETVLI-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
-; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
+; SPILL-O2-VSETVLI-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 1
-; SPILL-O2-VSETVLI-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
@@ -411,17 +377,12 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; SPILL-O2-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
-; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    slli a1, a1, 2
-; SPILL-O2-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs4r.v v12, (a0) # vscale x 32-byte Folded Spill
+; SPILL-O2-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
-; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    slli a1, a1, 2
-; SPILL-O2-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
@@ -437,15 +398,11 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 64
-; SPILL-O2-VLEN128-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs4r.v v12, (a0) # vscale x 32-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
-; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 64
 ; SPILL-O2-VLEN128-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 64
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 128
@@ -481,17 +438,12 @@ define <vscale x 8 x i32> @spill_zvlsseg_nxv8i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, m4, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg2e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
-; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 2
-; SPILL-O2-VSETVLI-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs4r.v v12, (a0) # vscale x 32-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    vs8r.v v8, (a0) # vscale x 64-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
-; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
+; SPILL-O2-VSETVLI-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 2
-; SPILL-O2-VSETVLI-NEXT:    vl4r.v v4, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
@@ -540,23 +492,19 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-NEXT:    csrr a1, vlenb
-; SPILL-O2-NEXT:    slli a1, a1, 1
-; SPILL-O2-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-NEXT:    slli a1, a1, 2
 ; SPILL-O2-NEXT:    add a0, a0, a1
 ; SPILL-O2-NEXT:    vs2r.v v12, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-NEXT:    #APP
 ; SPILL-O2-NEXT:    #NO_APP
 ; SPILL-O2-NEXT:    addi a0, sp, 16
+; SPILL-O2-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a1, vlenb
 ; SPILL-O2-NEXT:    slli a1, a1, 1
-; SPILL-O2-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; SPILL-O2-NEXT:    add a0, a0, a1
-; SPILL-O2-NEXT:    vl2r.v v10, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-NEXT:    csrr a0, vlenb
 ; SPILL-O2-NEXT:    li a1, 6
 ; SPILL-O2-NEXT:    mul a0, a0, a1
@@ -571,21 +519,17 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VLEN128-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-VLEN128-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 32
-; SPILL-O2-VLEN128-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
+; SPILL-O2-VLEN128-NEXT:    li a1, 64
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
 ; SPILL-O2-VLEN128-NEXT:    vs2r.v v12, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VLEN128-NEXT:    #APP
 ; SPILL-O2-VLEN128-NEXT:    #NO_APP
 ; SPILL-O2-VLEN128-NEXT:    addi a0, sp, 16
-; SPILL-O2-VLEN128-NEXT:    li a1, 32
 ; SPILL-O2-VLEN128-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    li a1, 32
 ; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; SPILL-O2-VLEN128-NEXT:    add a0, a0, a1
-; SPILL-O2-VLEN128-NEXT:    vl2r.v v10, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-VLEN128-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 96
 ; SPILL-O2-VLEN128-NEXT:    addi sp, sp, 16
 ; SPILL-O2-VLEN128-NEXT:    ret
@@ -621,23 +565,19 @@ define <vscale x 4 x i32> @spill_zvlsseg3_nxv4i32(ptr %base, i64 %vl) nounwind {
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    vlseg3e32.v v8, (a0)
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
+; SPILL-O2-VSETVLI-NEXT:    vs4r.v v8, (a0) # vscale x 32-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
-; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 1
-; SPILL-O2-VSETVLI-NEXT:    vs2r.v v8, (a0) # vscale x 16-byte Folded Spill
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vs2r.v v10, (a0) # vscale x 16-byte Folded Spill
+; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 2
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
 ; SPILL-O2-VSETVLI-NEXT:    vs2r.v v12, (a0) # vscale x 16-byte Folded Spill
 ; SPILL-O2-VSETVLI-NEXT:    #APP
 ; SPILL-O2-VSETVLI-NEXT:    #NO_APP
 ; SPILL-O2-VSETVLI-NEXT:    addi a0, sp, 16
+; SPILL-O2-VSETVLI-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    csrr a1, vlenb
 ; SPILL-O2-VSETVLI-NEXT:    slli a1, a1, 1
-; SPILL-O2-VSETVLI-NEXT:    vl2r.v v6, (a0) # vscale x 16-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vl2r.v v8, (a0) # vscale x 16-byte Folded Reload
-; SPILL-O2-VSETVLI-NEXT:    add a0, a0, a1
-; SPILL-O2-VSETVLI-NEXT:    vl2r.v v10, (a0) # vscale x 16-byte Folded Reload
+; SPILL-O2-VSETVLI-NEXT:    vl4r.v v8, (a0) # vscale x 32-byte Folded Reload
 ; SPILL-O2-VSETVLI-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; SPILL-O2-VSETVLI-NEXT:    li a1, 6
 ; SPILL-O2-VSETVLI-NEXT:    mul a0, a0, a1
diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-stack-align.mir b/llvm/test/CodeGen/RISCV/rvv/rvv-stack-align.mir
index 749bd4c13879b..c0170e7db5f2a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/rvv-stack-align.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/rvv-stack-align.mir
@@ -169,8 +169,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 16, alignment: 8,
@@ -214,8 +214,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 16, alignment: 16,
@@ -259,8 +259,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 16, alignment: 32,
diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll
index ecd098edb30ae..b6aa4affbb10f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/strided-vpload.ll
@@ -939,21 +939,17 @@ define <vscale x 16 x i64> @zero_strided_vadd_nxv16i64(<vscale x 16 x i64> %v, p
 ; CHECK-RV32:       # %bb.0:
 ; CHECK-RV32-NEXT:    csrr a1, vlenb
 ; CHECK-RV32-NEXT:    srli a2, a1, 3
-; CHECK-RV32-NEXT:    sub a3, a2, a1
-; CHECK-RV32-NEXT:    sltu a4, a2, a3
-; CHECK-RV32-NEXT:    addi a4, a4, -1
-; CHECK-RV32-NEXT:    and a3, a4, a3
-; CHECK-RV32-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
-; CHECK-RV32-NEXT:    vlse64.v v24, (a0), zero
-; CHECK-RV32-NEXT:    bltu a2, a1, .LBB61_2
-; CHECK-RV32-NEXT:  # %bb.1:
-; CHECK-RV32-NEXT:    mv a2, a1
-; CHECK-RV32-NEXT:  .LBB61_2:
+; CHECK-RV32-NEXT:    sub a1, a2, a1
+; CHECK-RV32-NEXT:    sltu a3, a2, a1
+; CHECK-RV32-NEXT:    addi a3, a3, -1
+; CHECK-RV32-NEXT:    and a1, a3, a1
 ; CHECK-RV32-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
+; CHECK-RV32-NEXT:    vlse64.v v24, (a0), zero
+; CHECK-RV32-NEXT:    vsetvli zero, a1, e64, m8, ta, ma
 ; CHECK-RV32-NEXT:    vlse64.v v0, (a0), zero
 ; CHECK-RV32-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
-; CHECK-RV32-NEXT:    vadd.vv v16, v16, v24
-; CHECK-RV32-NEXT:    vadd.vv v8, v8, v0
+; CHECK-RV32-NEXT:    vadd.vv v8, v8, v24
+; CHECK-RV32-NEXT:    vadd.vv v16, v16, v0
 ; CHECK-RV32-NEXT:    ret
 ;
 ; CHECK-RV64-LABEL: zero_strided_vadd_nxv16i64:
diff --git a/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.mir b/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.mir
index 98cabe314917c..75e0539843ac5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.mir
@@ -63,8 +63,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll
index 6e9f4d45cd6b4..946c0bbd7ff6f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vadd-vp.ll
@@ -1393,32 +1393,46 @@ define <vscale x 32 x i32> @vadd_vi_nxv32i32_unmasked(<vscale x 32 x i32> %va, i
 
 declare i32 @llvm.vscale.i32()
 
-; FIXME: The upper half of the operation is doing nothing.
-; FIXME: The branches comparing vscale vs. vscale should be constant-foldable.
-
 define <vscale x 32 x i32> @vadd_vi_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, <vscale x 32 x i1> %m) {
-; CHECK-LABEL: vadd_vi_nxv32i32_evl_nx8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv1r.v v24, v0
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    srli a2, a0, 2
-; CHECK-NEXT:    slli a1, a0, 1
-; CHECK-NEXT:    vslidedown.vx v0, v0, a2
-; CHECK-NEXT:    sub a2, a0, a1
-; CHECK-NEXT:    sltu a3, a0, a2
-; CHECK-NEXT:    addi a3, a3, -1
-; CHECK-NEXT:    and a2, a3, a2
-; CHECK-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
-; CHECK-NEXT:    vadd.vi v16, v16, -1, v0.t
-; CHECK-NEXT:    bltu a0, a1, .LBB120_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a0, a1
-; CHECK-NEXT:  .LBB120_2:
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; CHECK-NEXT:    vadd.vi v8, v8, -1, v0.t
-; CHECK-NEXT:    ret
+; RV32-LABEL: vadd_vi_nxv32i32_evl_nx8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    csrr a0, vlenb
+; RV32-NEXT:    srli a1, a0, 2
+; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; RV32-NEXT:    vadd.vi v8, v8, -1, v0.t
+; RV32-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v0, v0, a1
+; RV32-NEXT:    slli a1, a0, 1
+; RV32-NEXT:    sub a1, a0, a1
+; RV32-NEXT:    sltu a0, a0, a1
+; RV32-NEXT:    addi a0, a0, -1
+; RV32-NEXT:    and a0, a0, a1
+; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; RV32-NEXT:    vadd.vi v16, v16, -1, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vadd_vi_nxv32i32_evl_nx8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmv1r.v v24, v0
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    srli a2, a0, 2
+; RV64-NEXT:    slli a1, a0, 1
+; RV64-NEXT:    vslidedown.vx v0, v0, a2
+; RV64-NEXT:    sub a2, a0, a1
+; RV64-NEXT:    sltu a3, a0, a2
+; RV64-NEXT:    addi a3, a3, -1
+; RV64-NEXT:    and a2, a3, a2
+; RV64-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
+; RV64-NEXT:    vadd.vi v16, v16, -1, v0.t
+; RV64-NEXT:    bltu a0, a1, .LBB120_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a0, a1
+; RV64-NEXT:  .LBB120_2:
+; RV64-NEXT:    vmv1r.v v0, v24
+; RV64-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; RV64-NEXT:    vadd.vi v8, v8, -1, v0.t
+; RV64-NEXT:    ret
   %evl = call i32 @llvm.vscale.i32()
   %evl0 = mul i32 %evl, 8
   %v = call <vscale x 32 x i32> @llvm.vp.add.nxv32i32(<vscale x 32 x i32> %va, <vscale x 32 x i32> splat (i32 -1), <vscale x 32 x i1> %m, i32 %evl0)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
index dbe0ecc5b0624..e7baffd8bf8d5 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvbb,+zvfbfwma -verify-machineinstrs | FileCheck %s
-; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvbb,+zvfbfwma -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -mattr=+v,+zvbb,+zvbc,+zvfbfwma -verify-machineinstrs | FileCheck %s
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvbb,+zvbc,+zvfbfwma -verify-machineinstrs | FileCheck %s
 
 ; The purpose of this file is to check the behavior of specific instructions as it relates to the VL optimizer
 
@@ -3398,3 +3398,207 @@ define <vscale x 4 x double> @vfrec7(<vscale x 4 x float> %a) {
   %2 = call <vscale x 4 x double> @llvm.riscv.vfwmacc(<vscale x 4 x double> poison, <vscale x 4 x float> %a, <vscale x 4 x float> %1, iXLen 7, iXLen 6, iXLen 0)
   ret <vscale x 4 x double> %2
 }
+
+define <vscale x 4 x i32> @vandn_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; CHECK-LABEL: vandn_vv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vandn.vv v10, v8, v10
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vandn.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vandn_vx(<vscale x 4 x i32> %a, i32 %b, iXLen %vl) {
+; CHECK-LABEL: vandn_vx:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vandn.vx v10, v8, a0
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vandn.nxv4i32.i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vbrev_v(<vscale x 4 x i32> %a, iXLen %vl) {
+; CHECK-LABEL: vbrev_v:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vbrev.v v10, v8
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vbrev.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vbrev8_v(<vscale x 4 x i32> %a, iXLen %vl) {
+; CHECK-LABEL: vbrev8_v:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vbrev8.v v10, v8
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vbrev8.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vrev8_v(<vscale x 4 x i32> %a, iXLen %vl) {
+; CHECK-LABEL: vrev8_v:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vrev8.v v10, v8
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vrev8.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> undef, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vclz_v(<vscale x 4 x i32> %a, iXLen %vl) {
+; CHECK-LABEL: vclz_v:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vclz.v v10, v8
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vclz.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vcpop_v(<vscale x 4 x i32> %a, iXLen %vl) {
+; CHECK-LABEL: vcpop_v:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vcpop.v v10, v8
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vcpopv.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vctz_v(<vscale x 4 x i32> %a, iXLen %vl) {
+; CHECK-LABEL: vctz_v:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vctz.v v10, v8
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vctz.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vror_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; CHECK-LABEL: vror_vv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vror.vv v10, v8, v10
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vror.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vror_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; CHECK-LABEL: vror_vx:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vror.vx v10, v8, a0
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vror.nxv4i32.iXLen(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vror_vi(<vscale x 4 x i32> %a, iXLen %vl) {
+; CHECK-LABEL: vror_vi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vror.vi v10, v8, 5
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vror.nxv4i32.iXLen(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen 5, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vrol_vv(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen %vl) {
+; CHECK-LABEL: vrol_vv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m2, ta, ma
+; CHECK-NEXT:    vrol.vv v10, v8, v10
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vrol.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 4 x i32> @vrol_vx(<vscale x 4 x i32> %a, iXLen %b, iXLen %vl) {
+; CHECK-LABEL: vrol_vx:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
+; CHECK-NEXT:    vrol.vx v10, v8, a0
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 4 x i32> @llvm.riscv.vrol.nxv4i32.iXLen(<vscale x 4 x i32> poison, <vscale x 4 x i32> %a, iXLen %b, iXLen -1)
+  %2 = call <vscale x 4 x i32> @llvm.riscv.vadd.nxv4i32.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> %1, <vscale x 4 x i32> %a, iXLen %vl)
+  ret <vscale x 4 x i32> %2
+}
+
+define <vscale x 2 x i64> @vclmul_vv(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, iXLen %vl) {
+; CHECK-LABEL: vclmul_vv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vclmul.vv v10, v8, v10
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 2 x i64> @llvm.riscv.vclmul.nxv2i64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b, iXLen -1)
+  %2 = call <vscale x 2 x i64> @llvm.riscv.vadd.nxv2i64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> %1, <vscale x 2 x i64> %a, iXLen %vl)
+  ret <vscale x 2 x i64> %2
+}
+
+define <vscale x 2 x i64> @vclmul_vx(<vscale x 2 x i64> %a, i32 %b, iXLen %vl) {
+; CHECK-LABEL: vclmul_vx:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
+; CHECK-NEXT:    vclmul.vx v10, v8, a0
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 2 x i64> @llvm.riscv.vclmul.nxv2i64.i32(<vscale x 2 x i64> undef, <vscale x 2 x i64> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 2 x i64> @llvm.riscv.vadd.nxv2i64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> %1, <vscale x 2 x i64> %a, iXLen %vl)
+  ret <vscale x 2 x i64> %2
+}
+
+define <vscale x 2 x i64> @vclmulh_vv(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, iXLen %vl) {
+; CHECK-LABEL: vclmulh_vv:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vclmulh.vv v10, v8, v10
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 2 x i64> @llvm.riscv.vclmulh.nxv2i64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b, iXLen -1)
+  %2 = call <vscale x 2 x i64> @llvm.riscv.vadd.nxv2i64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> %1, <vscale x 2 x i64> %a, iXLen %vl)
+  ret <vscale x 2 x i64> %2
+}
+
+define <vscale x 2 x i64> @vclmulh_vx(<vscale x 2 x i64> %a, i32 %b, iXLen %vl) {
+; CHECK-LABEL: vclmulh_vx:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a1, e64, m2, ta, ma
+; CHECK-NEXT:    vclmulh.vx v10, v8, a0
+; CHECK-NEXT:    vadd.vv v8, v10, v8
+; CHECK-NEXT:    ret
+  %1 = call <vscale x 2 x i64> @llvm.riscv.vclmulh.nxv2i64.i32(<vscale x 2 x i64> undef, <vscale x 2 x i64> %a, i32 %b, iXLen -1)
+  %2 = call <vscale x 2 x i64> @llvm.riscv.vadd.nxv2i64.nxv2i64(<vscale x 2 x i64> undef, <vscale x 2 x i64> %1, <vscale x 2 x i64> %a, iXLen %vl)
+  ret <vscale x 2 x i64> %2
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll
index ec95e81b8dd94..d81936354f6f3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmax-vp.ll
@@ -1026,32 +1026,46 @@ define <vscale x 32 x i32> @vmax_vx_nxv32i32_unmasked(<vscale x 32 x i32> %va, i
 
 declare i32 @llvm.vscale.i32()
 
-; FIXME: The upper half of the operation is doing nothing.
-; FIXME: The branches comparing vscale vs. vscale should be constant-foldable.
-
 define <vscale x 32 x i32> @vmax_vx_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m) {
-; CHECK-LABEL: vmax_vx_nxv32i32_evl_nx8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv1r.v v24, v0
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    srli a3, a1, 2
-; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    vslidedown.vx v0, v0, a3
-; CHECK-NEXT:    sub a3, a1, a2
-; CHECK-NEXT:    sltu a4, a1, a3
-; CHECK-NEXT:    addi a4, a4, -1
-; CHECK-NEXT:    and a3, a4, a3
-; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
-; CHECK-NEXT:    vmax.vx v16, v16, a0, v0.t
-; CHECK-NEXT:    bltu a1, a2, .LBB82_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a1, a2
-; CHECK-NEXT:  .LBB82_2:
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
-; CHECK-NEXT:    vmax.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    ret
+; RV32-LABEL: vmax_vx_nxv32i32_evl_nx8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    srli a2, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV32-NEXT:    vmax.vx v8, v8, a0, v0.t
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v0, v0, a2
+; RV32-NEXT:    slli a2, a1, 1
+; RV32-NEXT:    sub a2, a1, a2
+; RV32-NEXT:    sltu a1, a1, a2
+; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    and a1, a1, a2
+; RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV32-NEXT:    vmax.vx v16, v16, a0, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vmax_vx_nxv32i32_evl_nx8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmv1r.v v24, v0
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    srli a3, a1, 2
+; RV64-NEXT:    slli a2, a1, 1
+; RV64-NEXT:    vslidedown.vx v0, v0, a3
+; RV64-NEXT:    sub a3, a1, a2
+; RV64-NEXT:    sltu a4, a1, a3
+; RV64-NEXT:    addi a4, a4, -1
+; RV64-NEXT:    and a3, a4, a3
+; RV64-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
+; RV64-NEXT:    vmax.vx v16, v16, a0, v0.t
+; RV64-NEXT:    bltu a1, a2, .LBB82_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a1, a2
+; RV64-NEXT:  .LBB82_2:
+; RV64-NEXT:    vmv1r.v v0, v24
+; RV64-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV64-NEXT:    vmax.vx v8, v8, a0, v0.t
+; RV64-NEXT:    ret
   %elt.head = insertelement <vscale x 32 x i32> poison, i32 %b, i32 0
   %vb = shufflevector <vscale x 32 x i32> %elt.head, <vscale x 32 x i32> poison, <vscale x 32 x i32> zeroinitializer
   %evl = call i32 @llvm.vscale.i32()
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll
index bd1a6c6e55c70..7603bcef1973e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmaxu-vp.ll
@@ -1025,32 +1025,46 @@ define <vscale x 32 x i32> @vmaxu_vx_nxv32i32_unmasked(<vscale x 32 x i32> %va,
 
 declare i32 @llvm.vscale.i32()
 
-; FIXME: The upper half of the operation is doing nothing.
-; FIXME: The branches comparing vscale vs. vscale should be constant-foldable.
-
 define <vscale x 32 x i32> @vmaxu_vx_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m) {
-; CHECK-LABEL: vmaxu_vx_nxv32i32_evl_nx8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv1r.v v24, v0
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    srli a3, a1, 2
-; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    vslidedown.vx v0, v0, a3
-; CHECK-NEXT:    sub a3, a1, a2
-; CHECK-NEXT:    sltu a4, a1, a3
-; CHECK-NEXT:    addi a4, a4, -1
-; CHECK-NEXT:    and a3, a4, a3
-; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
-; CHECK-NEXT:    vmaxu.vx v16, v16, a0, v0.t
-; CHECK-NEXT:    bltu a1, a2, .LBB82_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a1, a2
-; CHECK-NEXT:  .LBB82_2:
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
-; CHECK-NEXT:    vmaxu.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    ret
+; RV32-LABEL: vmaxu_vx_nxv32i32_evl_nx8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    srli a2, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV32-NEXT:    vmaxu.vx v8, v8, a0, v0.t
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v0, v0, a2
+; RV32-NEXT:    slli a2, a1, 1
+; RV32-NEXT:    sub a2, a1, a2
+; RV32-NEXT:    sltu a1, a1, a2
+; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    and a1, a1, a2
+; RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV32-NEXT:    vmaxu.vx v16, v16, a0, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vmaxu_vx_nxv32i32_evl_nx8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmv1r.v v24, v0
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    srli a3, a1, 2
+; RV64-NEXT:    slli a2, a1, 1
+; RV64-NEXT:    vslidedown.vx v0, v0, a3
+; RV64-NEXT:    sub a3, a1, a2
+; RV64-NEXT:    sltu a4, a1, a3
+; RV64-NEXT:    addi a4, a4, -1
+; RV64-NEXT:    and a3, a4, a3
+; RV64-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
+; RV64-NEXT:    vmaxu.vx v16, v16, a0, v0.t
+; RV64-NEXT:    bltu a1, a2, .LBB82_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a1, a2
+; RV64-NEXT:  .LBB82_2:
+; RV64-NEXT:    vmv1r.v v0, v24
+; RV64-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV64-NEXT:    vmaxu.vx v8, v8, a0, v0.t
+; RV64-NEXT:    ret
   %elt.head = insertelement <vscale x 32 x i32> poison, i32 %b, i32 0
   %vb = shufflevector <vscale x 32 x i32> %elt.head, <vscale x 32 x i32> poison, <vscale x 32 x i32> zeroinitializer
   %evl = call i32 @llvm.vscale.i32()
diff --git a/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll
index 6f1f8e1298321..3922b09f1f02d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vmin-vp.ll
@@ -1026,32 +1026,46 @@ define <vscale x 32 x i32> @vmin_vx_nxv32i32_unmasked(<vscale x 32 x i32> %va, i
 
 declare i32 @llvm.vscale.i32()
 
-; FIXME: The upper half of the operation is doing nothing.
-; FIXME: The branches comparing vscale vs. vscale should be constant-foldable.
-
 define <vscale x 32 x i32> @vmin_vx_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m) {
-; CHECK-LABEL: vmin_vx_nxv32i32_evl_nx8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv1r.v v24, v0
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    srli a3, a1, 2
-; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    vslidedown.vx v0, v0, a3
-; CHECK-NEXT:    sub a3, a1, a2
-; CHECK-NEXT:    sltu a4, a1, a3
-; CHECK-NEXT:    addi a4, a4, -1
-; CHECK-NEXT:    and a3, a4, a3
-; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
-; CHECK-NEXT:    vmin.vx v16, v16, a0, v0.t
-; CHECK-NEXT:    bltu a1, a2, .LBB82_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a1, a2
-; CHECK-NEXT:  .LBB82_2:
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
-; CHECK-NEXT:    vmin.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    ret
+; RV32-LABEL: vmin_vx_nxv32i32_evl_nx8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    srli a2, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV32-NEXT:    vmin.vx v8, v8, a0, v0.t
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v0, v0, a2
+; RV32-NEXT:    slli a2, a1, 1
+; RV32-NEXT:    sub a2, a1, a2
+; RV32-NEXT:    sltu a1, a1, a2
+; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    and a1, a1, a2
+; RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV32-NEXT:    vmin.vx v16, v16, a0, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vmin_vx_nxv32i32_evl_nx8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmv1r.v v24, v0
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    srli a3, a1, 2
+; RV64-NEXT:    slli a2, a1, 1
+; RV64-NEXT:    vslidedown.vx v0, v0, a3
+; RV64-NEXT:    sub a3, a1, a2
+; RV64-NEXT:    sltu a4, a1, a3
+; RV64-NEXT:    addi a4, a4, -1
+; RV64-NEXT:    and a3, a4, a3
+; RV64-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
+; RV64-NEXT:    vmin.vx v16, v16, a0, v0.t
+; RV64-NEXT:    bltu a1, a2, .LBB82_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a1, a2
+; RV64-NEXT:  .LBB82_2:
+; RV64-NEXT:    vmv1r.v v0, v24
+; RV64-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV64-NEXT:    vmin.vx v8, v8, a0, v0.t
+; RV64-NEXT:    ret
   %elt.head = insertelement <vscale x 32 x i32> poison, i32 %b, i32 0
   %vb = shufflevector <vscale x 32 x i32> %elt.head, <vscale x 32 x i32> poison, <vscale x 32 x i32> zeroinitializer
   %evl = call i32 @llvm.vscale.i32()
diff --git a/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll
index 47e7b7d70c07e..59af953fd52d3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vminu-vp.ll
@@ -1025,32 +1025,46 @@ define <vscale x 32 x i32> @vminu_vx_nxv32i32_unmasked(<vscale x 32 x i32> %va,
 
 declare i32 @llvm.vscale.i32()
 
-; FIXME: The upper half of the operation is doing nothing.
-; FIXME: The branches comparing vscale vs. vscale should be constant-foldable.
-
 define <vscale x 32 x i32> @vminu_vx_nxv32i32_evl_nx8(<vscale x 32 x i32> %va, i32 %b, <vscale x 32 x i1> %m) {
-; CHECK-LABEL: vminu_vx_nxv32i32_evl_nx8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv1r.v v24, v0
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    srli a3, a1, 2
-; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    vslidedown.vx v0, v0, a3
-; CHECK-NEXT:    sub a3, a1, a2
-; CHECK-NEXT:    sltu a4, a1, a3
-; CHECK-NEXT:    addi a4, a4, -1
-; CHECK-NEXT:    and a3, a4, a3
-; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
-; CHECK-NEXT:    vminu.vx v16, v16, a0, v0.t
-; CHECK-NEXT:    bltu a1, a2, .LBB82_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a1, a2
-; CHECK-NEXT:  .LBB82_2:
-; CHECK-NEXT:    vmv1r.v v0, v24
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
-; CHECK-NEXT:    vminu.vx v8, v8, a0, v0.t
-; CHECK-NEXT:    ret
+; RV32-LABEL: vminu_vx_nxv32i32_evl_nx8:
+; RV32:       # %bb.0:
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    srli a2, a1, 2
+; RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV32-NEXT:    vminu.vx v8, v8, a0, v0.t
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v0, v0, a2
+; RV32-NEXT:    slli a2, a1, 1
+; RV32-NEXT:    sub a2, a1, a2
+; RV32-NEXT:    sltu a1, a1, a2
+; RV32-NEXT:    addi a1, a1, -1
+; RV32-NEXT:    and a1, a1, a2
+; RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV32-NEXT:    vminu.vx v16, v16, a0, v0.t
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: vminu_vx_nxv32i32_evl_nx8:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmv1r.v v24, v0
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    srli a3, a1, 2
+; RV64-NEXT:    slli a2, a1, 1
+; RV64-NEXT:    vslidedown.vx v0, v0, a3
+; RV64-NEXT:    sub a3, a1, a2
+; RV64-NEXT:    sltu a4, a1, a3
+; RV64-NEXT:    addi a4, a4, -1
+; RV64-NEXT:    and a3, a4, a3
+; RV64-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
+; RV64-NEXT:    vminu.vx v16, v16, a0, v0.t
+; RV64-NEXT:    bltu a1, a2, .LBB82_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a1, a2
+; RV64-NEXT:  .LBB82_2:
+; RV64-NEXT:    vmv1r.v v0, v24
+; RV64-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV64-NEXT:    vminu.vx v8, v8, a0, v0.t
+; RV64-NEXT:    ret
   %elt.head = insertelement <vscale x 32 x i32> poison, i32 %b, i32 0
   %vb = shufflevector <vscale x 32 x i32> %elt.head, <vscale x 32 x i32> poison, <vscale x 32 x i32> zeroinitializer
   %evl = call i32 @llvm.vscale.i32()
diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
index 522c83fd9fa99..2ed3c9bfe2c16 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zvfh,+v -target-abi=ilp32d \
-; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zvfh,+v -target-abi=lp64d \
-; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 ; RUN: llc -mtriple=riscv32 -mattr=+d,+m,+zvfhmin,+v -target-abi=ilp32d \
-; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV32
 ; RUN: llc -mtriple=riscv64 -mattr=+d,+m,+zvfhmin,+v -target-abi=lp64d \
-; RUN:     -verify-machineinstrs < %s | FileCheck %s
+; RUN:     -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,RV64
 
 declare <vscale x 1 x i1> @llvm.vp.select.nxv1i1(<vscale x 1 x i1>, <vscale x 1 x i1>, <vscale x 1 x i1>, i32)
 
@@ -398,48 +398,69 @@ define <vscale x 32 x i32> @select_nxv32i32(<vscale x 32 x i1> %a, <vscale x 32
 declare i32 @llvm.vscale.i32()
 
 define <vscale x 32 x i32> @select_evl_nxv32i32(<vscale x 32 x i1> %a, <vscale x 32 x i32> %b, <vscale x 32 x i32> %c) {
-; CHECK-LABEL: select_evl_nxv32i32:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    addi sp, sp, -16
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    sub sp, sp, a1
-; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
-; CHECK-NEXT:    addi a1, sp, 16
-; CHECK-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
-; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv1r.v v7, v0
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a3, a1, 3
-; CHECK-NEXT:    slli a2, a1, 1
-; CHECK-NEXT:    srli a4, a1, 2
-; CHECK-NEXT:    add a3, a0, a3
-; CHECK-NEXT:    sub a5, a1, a2
-; CHECK-NEXT:    vl8re32.v v24, (a3)
-; CHECK-NEXT:    sltu a3, a1, a5
-; CHECK-NEXT:    addi a3, a3, -1
-; CHECK-NEXT:    vl8re32.v v8, (a0)
-; CHECK-NEXT:    vslidedown.vx v0, v0, a4
-; CHECK-NEXT:    and a3, a3, a5
-; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vvm v16, v24, v16, v0
-; CHECK-NEXT:    bltu a1, a2, .LBB28_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    mv a1, a2
-; CHECK-NEXT:  .LBB28_2:
-; CHECK-NEXT:    vmv1r.v v0, v7
-; CHECK-NEXT:    addi a0, sp, 16
-; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
-; CHECK-NEXT:    vmerge.vvm v8, v8, v24, v0
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    slli a0, a0, 3
-; CHECK-NEXT:    add sp, sp, a0
-; CHECK-NEXT:    .cfi_def_cfa sp, 16
-; CHECK-NEXT:    addi sp, sp, 16
-; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:    ret
+; RV32-LABEL: select_evl_nxv32i32:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vl8re32.v v24, (a0)
+; RV32-NEXT:    csrr a1, vlenb
+; RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV32-NEXT:    vmerge.vvm v8, v24, v8, v0
+; RV32-NEXT:    slli a2, a1, 3
+; RV32-NEXT:    add a0, a0, a2
+; RV32-NEXT:    slli a2, a1, 1
+; RV32-NEXT:    sub a2, a1, a2
+; RV32-NEXT:    vl8re32.v v24, (a0)
+; RV32-NEXT:    sltu a0, a1, a2
+; RV32-NEXT:    addi a0, a0, -1
+; RV32-NEXT:    srli a1, a1, 2
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v0, v0, a1
+; RV32-NEXT:    and a0, a0, a2
+; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; RV32-NEXT:    vmerge.vvm v16, v24, v16, v0
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: select_evl_nxv32i32:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a1, a1, 3
+; RV64-NEXT:    sub sp, sp, a1
+; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb
+; RV64-NEXT:    addi a1, sp, 16
+; RV64-NEXT:    vs8r.v v8, (a1) # vscale x 64-byte Folded Spill
+; RV64-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vmv1r.v v7, v0
+; RV64-NEXT:    csrr a1, vlenb
+; RV64-NEXT:    slli a3, a1, 3
+; RV64-NEXT:    slli a2, a1, 1
+; RV64-NEXT:    srli a4, a1, 2
+; RV64-NEXT:    add a3, a0, a3
+; RV64-NEXT:    sub a5, a1, a2
+; RV64-NEXT:    vl8re32.v v24, (a3)
+; RV64-NEXT:    sltu a3, a1, a5
+; RV64-NEXT:    addi a3, a3, -1
+; RV64-NEXT:    vl8re32.v v8, (a0)
+; RV64-NEXT:    vslidedown.vx v0, v0, a4
+; RV64-NEXT:    and a3, a3, a5
+; RV64-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
+; RV64-NEXT:    vmerge.vvm v16, v24, v16, v0
+; RV64-NEXT:    bltu a1, a2, .LBB28_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    mv a1, a2
+; RV64-NEXT:  .LBB28_2:
+; RV64-NEXT:    vmv1r.v v0, v7
+; RV64-NEXT:    addi a0, sp, 16
+; RV64-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
+; RV64-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; RV64-NEXT:    vmerge.vvm v8, v8, v24, v0
+; RV64-NEXT:    csrr a0, vlenb
+; RV64-NEXT:    slli a0, a0, 3
+; RV64-NEXT:    add sp, sp, a0
+; RV64-NEXT:    .cfi_def_cfa sp, 16
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    .cfi_def_cfa_offset 0
+; RV64-NEXT:    ret
   %evl = call i32 @llvm.vscale.i32()
   %evl0 = mul i32 %evl, 8
   %v = call <vscale x 32 x i32> @llvm.vp.select.nxv32i32(<vscale x 32 x i1> %a, <vscale x 32 x i32> %b, <vscale x 32 x i32> %c, i32 %evl0)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsha2ch.ll b/llvm/test/CodeGen/RISCV/rvv/vsha2ch.ll
index dd5d48a8d2d40..9674b78ab9fa0 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsha2ch.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsha2ch.ll
@@ -3,9 +3,9 @@
 ; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvknhb \
 ; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
-; RUN: sed 's/iXLen/i32/g' %s | not --crash llc -mtriple=riscv32 -mattr=+v,+zvknha 2>&1 \
+; RUN: sed 's/iXLen/i32/g' %s | not llc -mtriple=riscv32 -mattr=+v,+zvknha 2>&1 \
 ; RUN:   | FileCheck --check-prefixes=CHECK-ERROR %s
-; RUN: sed 's/iXLen/i64/g' %s | not --crash llc -mtriple=riscv64 -mattr=+v,+zvknha 2>&1 \
+; RUN: sed 's/iXLen/i64/g' %s | not llc -mtriple=riscv64 -mattr=+v,+zvknha 2>&1 \
 ; RUN:   | FileCheck --check-prefixes=CHECK-ERROR %s
 
 ; CHECK-ERROR: LLVM ERROR: SEW=64 needs Zvknhb to be enabled.
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsha2cl.ll b/llvm/test/CodeGen/RISCV/rvv/vsha2cl.ll
index 697c582dcb38b..b45a768b9ce22 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsha2cl.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsha2cl.ll
@@ -3,9 +3,9 @@
 ; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvknhb \
 ; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
-; RUN: sed 's/iXLen/i32/g' %s | not --crash llc -mtriple=riscv32 -mattr=+v,+zvknha 2>&1 \
+; RUN: sed 's/iXLen/i32/g' %s | not llc -mtriple=riscv32 -mattr=+v,+zvknha 2>&1 \
 ; RUN:   | FileCheck --check-prefixes=CHECK-ERROR %s
-; RUN: sed 's/iXLen/i64/g' %s | not --crash llc -mtriple=riscv64 -mattr=+v,+zvknha 2>&1 \
+; RUN: sed 's/iXLen/i64/g' %s | not llc -mtriple=riscv64 -mattr=+v,+zvknha 2>&1 \
 ; RUN:   | FileCheck --check-prefixes=CHECK-ERROR %s
 
 ; CHECK-ERROR: LLVM ERROR: SEW=64 needs Zvknhb to be enabled.
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsha2ms.ll b/llvm/test/CodeGen/RISCV/rvv/vsha2ms.ll
index e57baa504da7b..ff51acc2f19aa 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsha2ms.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsha2ms.ll
@@ -3,9 +3,9 @@
 ; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -mattr=+v,+zvknha,+zvknhb \
 ; RUN:   -verify-machineinstrs | FileCheck %s --check-prefixes=CHECK
-; RUN: sed 's/iXLen/i32/g' %s | not --crash llc -mtriple=riscv32 -mattr=+v,+zvknha 2>&1 \
+; RUN: sed 's/iXLen/i32/g' %s | not llc -mtriple=riscv32 -mattr=+v,+zvknha 2>&1 \
 ; RUN:   | FileCheck --check-prefixes=CHECK-ERROR %s
-; RUN: sed 's/iXLen/i64/g' %s | not --crash llc -mtriple=riscv64 -mattr=+v,+zvknha 2>&1 \
+; RUN: sed 's/iXLen/i64/g' %s | not llc -mtriple=riscv64 -mattr=+v,+zvknha 2>&1 \
 ; RUN:   | FileCheck --check-prefixes=CHECK-ERROR %s
 
 ; CHECK-ERROR: LLVM ERROR: SEW=64 needs Zvknhb to be enabled.
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
index 7990dfc0880a5..dddcd4f107e3b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
@@ -61,11 +61,11 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
 ; RV32-NEXT:    sltu t3, a4, t3
 ; RV32-NEXT:    and t3, t4, t3
 ; RV32-NEXT:    or t4, a1, a3
-; RV32-NEXT:    slti t4, t4, 0
+; RV32-NEXT:    srli t4, t4, 31
 ; RV32-NEXT:    or t4, t5, t4
 ; RV32-NEXT:    or t5, a1, a5
 ; RV32-NEXT:    sltu t1, a6, t1
-; RV32-NEXT:    slti t5, t5, 0
+; RV32-NEXT:    srli t5, t5, 31
 ; RV32-NEXT:    or t3, t3, t5
 ; RV32-NEXT:    or t3, t4, t3
 ; RV32-NEXT:    or t1, t1, t3
@@ -186,14 +186,14 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
 ; RV64P670-NEXT:    slli t3, t2, 1
 ; RV64P670-NEXT:    and s0, s0, s1
 ; RV64P670-NEXT:    or s1, a1, a3
-; RV64P670-NEXT:    slti s1, s1, 0
+; RV64P670-NEXT:    srli s1, s1, 63
 ; RV64P670-NEXT:    or t6, s0, s1
 ; RV64P670-NEXT:    sltu s1, a0, t5
 ; RV64P670-NEXT:    sltu s0, a4, t4
 ; RV64P670-NEXT:    mv t5, a0
 ; RV64P670-NEXT:    and s0, s0, s1
 ; RV64P670-NEXT:    or s1, a1, a5
-; RV64P670-NEXT:    slti s1, s1, 0
+; RV64P670-NEXT:    srli s1, s1, 63
 ; RV64P670-NEXT:    or s0, s0, s1
 ; RV64P670-NEXT:    li s1, 32
 ; RV64P670-NEXT:    maxu s1, t3, s1
@@ -321,12 +321,12 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
 ; RV64X60-NEXT:    or s2, a1, a3
 ; RV64X60-NEXT:    sltu s0, a0, t5
 ; RV64X60-NEXT:    sltu s1, a4, t3
-; RV64X60-NEXT:    slti t3, s2, 0
+; RV64X60-NEXT:    srli t3, s2, 63
 ; RV64X60-NEXT:    and s0, s0, s1
 ; RV64X60-NEXT:    or s1, a1, a5
 ; RV64X60-NEXT:    or t4, t4, t3
 ; RV64X60-NEXT:    slli t3, t2, 1
-; RV64X60-NEXT:    slti s1, s1, 0
+; RV64X60-NEXT:    srli s1, s1, 63
 ; RV64X60-NEXT:    or s0, s0, s1
 ; RV64X60-NEXT:    maxu s1, t3, t6
 ; RV64X60-NEXT:    or s0, t4, s0
@@ -366,8 +366,8 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
 ; RV64X60-NEXT:    # => This Inner Loop Header: Depth=2
 ; RV64X60-NEXT:    vl2r.v v8, (s2)
 ; RV64X60-NEXT:    vl2r.v v10, (s3)
-; RV64X60-NEXT:    sub s1, s1, t3
 ; RV64X60-NEXT:    vaaddu.vv v8, v8, v10
+; RV64X60-NEXT:    sub s1, s1, t3
 ; RV64X60-NEXT:    vs2r.v v8, (s4)
 ; RV64X60-NEXT:    add s4, s4, t3
 ; RV64X60-NEXT:    add s3, s3, t3
@@ -461,10 +461,10 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
 ; RV64-NEXT:    sltu t5, a4, t5
 ; RV64-NEXT:    and t5, t6, t5
 ; RV64-NEXT:    or t6, a1, a3
-; RV64-NEXT:    slti t6, t6, 0
+; RV64-NEXT:    srli t6, t6, 63
 ; RV64-NEXT:    or t6, s0, t6
 ; RV64-NEXT:    or s0, a1, a5
-; RV64-NEXT:    slti s0, s0, 0
+; RV64-NEXT:    srli s0, s0, 63
 ; RV64-NEXT:    or t5, t5, s0
 ; RV64-NEXT:    or t5, t6, t5
 ; RV64-NEXT:    sltu t4, a6, t4
diff --git a/llvm/test/CodeGen/RISCV/rvv/wrong-stack-offset-for-rvv-object.mir b/llvm/test/CodeGen/RISCV/rvv/wrong-stack-offset-for-rvv-object.mir
index 2ec51911a65f7..6e5b49d543216 100644
--- a/llvm/test/CodeGen/RISCV/rvv/wrong-stack-offset-for-rvv-object.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/wrong-stack-offset-for-rvv-object.mir
@@ -93,8 +93,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: -8, size: 8, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/RISCV/rvv/zvlsseg-spill.mir b/llvm/test/CodeGen/RISCV/rvv/zvlsseg-spill.mir
index 055d9ed630718..dd9960d17af43 100644
--- a/llvm/test/CodeGen/RISCV/rvv/zvlsseg-spill.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/zvlsseg-spill.mir
@@ -2,15 +2,15 @@
 # RUN: llc -mtriple=riscv64 -mattr=+v -stop-after=prologepilog %s -o - 2>&1 | FileCheck %s
 
 --- |
-  target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
-  target triple = "riscv64"
-
-  define void @zvlsseg_spill(ptr %base, i64 %vl) {
+  define void @zvlsseg_spill_0(ptr %base, i64 %vl) {
+    ret void
+  }
+  define void @zvlsseg_spill_1(ptr %base, i64 %vl) {
     ret void
   }
 ...
 ---
-name:    zvlsseg_spill
+name:    zvlsseg_spill_0
 tracksRegLiveness: true
 stack:
   - { id: 0, offset: 0, size: 64, alignment: 8, stack-id: scalable-vector }
@@ -18,7 +18,7 @@ body: |
   bb.0:
     liveins: $x10, $x11
 
-    ; CHECK-LABEL: name: zvlsseg_spill
+    ; CHECK-LABEL: name: zvlsseg_spill_0
     ; CHECK: liveins: $x10, $x11
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: $x2 = frame-setup ADDI $x2, -16
@@ -30,35 +30,22 @@ body: |
     ; CHECK-NEXT: dead $x0 = PseudoVSETVLI killed renamable $x11, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
     ; CHECK-NEXT: $v0_v1_v2_v3_v4_v5_v6 = PseudoVLSEG7E64_V_M1 undef $v0_v1_v2_v3_v4_v5_v6, renamable $x10, $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
     ; CHECK-NEXT: $x11 = ADDI $x2, 16
+    ; CHECK-NEXT: VS4R_V $v0m4, $x11, implicit $v0_v1_v2_v3_v4_v5_v6 :: (store (<vscale x 1 x s256>) into %stack.0, align 8)
     ; CHECK-NEXT: $x12 = PseudoReadVLENB
-    ; CHECK-NEXT: VS1R_V $v0, $x11, implicit $v0_v1_v2_v3_v4_v5_v6 :: (store (<vscale x 1 x s64>) into %stack.0)
-    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: VS1R_V $v1, $x11, implicit $v0_v1_v2_v3_v4_v5_v6 :: (store (<vscale x 1 x s64>) into %stack.0)
-    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: VS1R_V $v2, $x11, implicit $v0_v1_v2_v3_v4_v5_v6 :: (store (<vscale x 1 x s64>) into %stack.0)
-    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: VS1R_V $v3, $x11, implicit $v0_v1_v2_v3_v4_v5_v6 :: (store (<vscale x 1 x s64>) into %stack.0)
-    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: VS1R_V $v4, $x11, implicit $v0_v1_v2_v3_v4_v5_v6 :: (store (<vscale x 1 x s64>) into %stack.0)
-    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: VS1R_V $v5, $x11, implicit $v0_v1_v2_v3_v4_v5_v6 :: (store (<vscale x 1 x s64>) into %stack.0)
+    ; CHECK-NEXT: $x13 = SLLI $x12, 2
+    ; CHECK-NEXT: $x11 = ADD killed $x11, killed $x13
+    ; CHECK-NEXT: VS2R_V $v4m2, $x11, implicit $v0_v1_v2_v3_v4_v5_v6 :: (store (<vscale x 1 x s128>) into %stack.0, align 8)
+    ; CHECK-NEXT: $x12 = SLLI killed $x12, 1
     ; CHECK-NEXT: $x11 = ADD killed $x11, killed $x12
     ; CHECK-NEXT: VS1R_V $v6, killed $x11, implicit $v0_v1_v2_v3_v4_v5_v6 :: (store (<vscale x 1 x s64>) into %stack.0)
     ; CHECK-NEXT: $x11 = ADDI $x2, 16
-    ; CHECK-NEXT: $x12 = PseudoReadVLENB
     ; CHECK-NEXT: $v7 = VL1RE8_V $x11 :: (load (<vscale x 1 x s64>) from %stack.0)
+    ; CHECK-NEXT: $x12 = PseudoReadVLENB
     ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: $v8 = VL1RE8_V $x11 :: (load (<vscale x 1 x s64>) from %stack.0)
-    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: $v9 = VL1RE8_V $x11 :: (load (<vscale x 1 x s64>) from %stack.0)
-    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: $v10 = VL1RE8_V $x11 :: (load (<vscale x 1 x s64>) from %stack.0)
-    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: $v11 = VL1RE8_V $x11 :: (load (<vscale x 1 x s64>) from %stack.0)
-    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
-    ; CHECK-NEXT: $v12 = VL1RE8_V $x11 :: (load (<vscale x 1 x s64>) from %stack.0)
+    ; CHECK-NEXT: $v8m4 = VL4RE8_V $x11 :: (load (<vscale x 1 x s256>) from %stack.0, align 8)
+    ; CHECK-NEXT: $x12 = SLLI killed $x12, 2
     ; CHECK-NEXT: $x11 = ADD killed $x11, killed $x12
-    ; CHECK-NEXT: $v13 = VL1RE8_V killed $x11 :: (load (<vscale x 1 x s64>) from %stack.0)
+    ; CHECK-NEXT: $v12m2 = VL2RE8_V killed $x11 :: (load (<vscale x 1 x s128>) from %stack.0, align 8)
     ; CHECK-NEXT: VS1R_V killed $v8, killed renamable $x10
     ; CHECK-NEXT: $x10 = frame-destroy PseudoReadVLENB
     ; CHECK-NEXT: $x10 = frame-destroy SLLI killed $x10, 3
@@ -75,3 +62,56 @@ body: |
     VS1R_V killed $v8, %0:gpr
     PseudoRET
 ...
+
+---
+name:    zvlsseg_spill_1
+tracksRegLiveness: true
+stack:
+  - { id: 0, offset: 0, size: 64, alignment: 8, stack-id: scalable-vector }
+body: |
+  bb.0:
+    liveins: $x10, $x11
+    ; CHECK-LABEL: name: zvlsseg_spill_1
+    ; CHECK: liveins: $x10, $x11
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: $x2 = frame-setup ADDI $x2, -16
+    ; CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+    ; CHECK-NEXT: $x12 = frame-setup PseudoReadVLENB
+    ; CHECK-NEXT: $x12 = frame-setup SLLI killed $x12, 3
+    ; CHECK-NEXT: $x2 = frame-setup SUB $x2, killed $x12
+    ; CHECK-NEXT: frame-setup CFI_INSTRUCTION escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22
+    ; CHECK-NEXT: dead $x0 = PseudoVSETVLI killed renamable $x11, 216 /* e64, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
+    ; CHECK-NEXT: $v1_v2_v3_v4_v5_v6_v7 = PseudoVLSEG7E64_V_M1 undef $v1_v2_v3_v4_v5_v6_v7, renamable $x10, $noreg, 6 /* e64 */, 0 /* tu, mu */, implicit $vl, implicit $vtype
+    ; CHECK-NEXT: $x11 = ADDI $x2, 16
+    ; CHECK-NEXT: VS1R_V $v1, $x11, implicit $v1_v2_v3_v4_v5_v6_v7 :: (store (<vscale x 1 x s64>) into %stack.0)
+    ; CHECK-NEXT: $x12 = PseudoReadVLENB
+    ; CHECK-NEXT: $x11 = ADD killed $x11, $x12
+    ; CHECK-NEXT: VS2R_V $v2m2, $x11, implicit $v1_v2_v3_v4_v5_v6_v7 :: (store (<vscale x 1 x s128>) into %stack.0, align 8)
+    ; CHECK-NEXT: $x12 = SLLI killed $x12, 1
+    ; CHECK-NEXT: $x11 = ADD killed $x11, killed $x12
+    ; CHECK-NEXT: VS4R_V $v4m4, killed $x11, implicit $v1_v2_v3_v4_v5_v6_v7 :: (store (<vscale x 1 x s256>) into %stack.0, align 8)
+    ; CHECK-NEXT: $x11 = ADDI $x2, 16
+    ; CHECK-NEXT: $v10m2 = VL2RE8_V $x11 :: (load (<vscale x 1 x s128>) from %stack.0, align 8)
+    ; CHECK-NEXT: $x12 = PseudoReadVLENB
+    ; CHECK-NEXT: $x13 = SLLI $x12, 1
+    ; CHECK-NEXT: $x11 = ADD killed $x11, killed $x13
+    ; CHECK-NEXT: $v12m4 = VL4RE8_V $x11 :: (load (<vscale x 1 x s256>) from %stack.0, align 8)
+    ; CHECK-NEXT: $x12 = SLLI killed $x12, 2
+    ; CHECK-NEXT: $x11 = ADD killed $x11, killed $x12
+    ; CHECK-NEXT: $v16 = VL1RE8_V killed $x11 :: (load (<vscale x 1 x s64>) from %stack.0)
+    ; CHECK-NEXT: VS1R_V killed $v10, killed renamable $x10
+    ; CHECK-NEXT: $x10 = frame-destroy PseudoReadVLENB
+    ; CHECK-NEXT: $x10 = frame-destroy SLLI killed $x10, 3
+    ; CHECK-NEXT: $x2 = frame-destroy ADD $x2, killed $x10
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa $x2, 16
+    ; CHECK-NEXT: $x2 = frame-destroy ADDI $x2, 16
+    ; CHECK-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+    ; CHECK-NEXT: PseudoRET
+    %0:gpr = COPY $x10
+    %1:gprnox0 = COPY $x11
+    $v1_v2_v3_v4_v5_v6_v7 = PseudoVLSEG7E64_V_M1 undef $v1_v2_v3_v4_v5_v6_v7, %0, %1, 6, 0
+    PseudoVSPILL7_M1 killed renamable $v1_v2_v3_v4_v5_v6_v7, %stack.0 :: (store (<vscale x 7 x s64>) into %stack.0, align 8)
+    renamable $v10_v11_v12_v13_v14_v15_v16 = PseudoVRELOAD7_M1 %stack.0 :: (load (<vscale x 7 x s64>) from %stack.0, align 8)
+    VS1R_V killed $v10, %0:gpr
+    PseudoRET
+...
diff --git a/llvm/test/CodeGen/RISCV/sadd_sat.ll b/llvm/test/CodeGen/RISCV/sadd_sat.ll
index 04f2436201e9d..1d6d07aa67337 100644
--- a/llvm/test/CodeGen/RISCV/sadd_sat.ll
+++ b/llvm/test/CodeGen/RISCV/sadd_sat.ll
@@ -16,7 +16,7 @@ define signext i32 @func(i32 signext %x, i32 signext %y) nounwind {
 ; RV32-NEXT:    mv a2, a0
 ; RV32-NEXT:    add a0, a0, a1
 ; RV32-NEXT:    slt a2, a0, a2
-; RV32-NEXT:    slti a1, a1, 0
+; RV32-NEXT:    srli a1, a1, 31
 ; RV32-NEXT:    beq a1, a2, .LBB0_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    srai a0, a0, 31
@@ -77,7 +77,7 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
 ; RV64-NEXT:    mv a2, a0
 ; RV64-NEXT:    add a0, a0, a1
 ; RV64-NEXT:    slt a2, a0, a2
-; RV64-NEXT:    slti a1, a1, 0
+; RV64-NEXT:    srli a1, a1, 63
 ; RV64-NEXT:    beq a1, a2, .LBB1_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    srai a0, a0, 63
diff --git a/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll b/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
index 857026cce0d43..9200a77915c56 100644
--- a/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
+++ b/llvm/test/CodeGen/RISCV/sadd_sat_plus.ll
@@ -17,7 +17,7 @@ define i32 @func32(i32 %x, i32 %y, i32 %z) nounwind {
 ; RV32-NEXT:    mul a1, a1, a2
 ; RV32-NEXT:    add a0, a0, a1
 ; RV32-NEXT:    slt a2, a0, a3
-; RV32-NEXT:    slti a1, a1, 0
+; RV32-NEXT:    srli a1, a1, 31
 ; RV32-NEXT:    beq a1, a2, .LBB0_2
 ; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    srai a0, a0, 31
@@ -81,7 +81,7 @@ define i64 @func64(i64 %x, i64 %y, i64 %z) nounwind {
 ; RV64-NEXT:    mv a1, a0
 ; RV64-NEXT:    add a0, a0, a2
 ; RV64-NEXT:    slt a1, a0, a1
-; RV64-NEXT:    slti a2, a2, 0
+; RV64-NEXT:    srli a2, a2, 63
 ; RV64-NEXT:    beq a2, a1, .LBB1_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    srai a0, a0, 63
diff --git a/llvm/test/CodeGen/RISCV/select-binop-identity.ll b/llvm/test/CodeGen/RISCV/select-binop-identity.ll
index 325e4b54c1d6d..8ab66ba3f25f5 100644
--- a/llvm/test/CodeGen/RISCV/select-binop-identity.ll
+++ b/llvm/test/CodeGen/RISCV/select-binop-identity.ll
@@ -260,14 +260,14 @@ define i64 @and_select_all_ones_i64_cmp2(i64 %x, i64 %y, i64 %z) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    beqz a5, .LBB5_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slti a4, a5, 0
+; RV32I-NEXT:    srli a5, a5, 31
 ; RV32I-NEXT:    j .LBB5_3
 ; RV32I-NEXT:  .LBB5_2:
-; RV32I-NEXT:    sltiu a4, a4, 4
+; RV32I-NEXT:    sltiu a5, a4, 4
 ; RV32I-NEXT:  .LBB5_3:
-; RV32I-NEXT:    addi a4, a4, -1
-; RV32I-NEXT:    or a1, a4, a1
-; RV32I-NEXT:    or a0, a4, a0
+; RV32I-NEXT:    addi a5, a5, -1
+; RV32I-NEXT:    or a1, a5, a1
+; RV32I-NEXT:    or a0, a5, a0
 ; RV32I-NEXT:    and a0, a0, a2
 ; RV32I-NEXT:    and a1, a1, a3
 ; RV32I-NEXT:    ret
@@ -300,7 +300,7 @@ define i64 @and_select_all_ones_i64_cmp2(i64 %x, i64 %y, i64 %z) {
 ;
 ; ZICOND32-LABEL: and_select_all_ones_i64_cmp2:
 ; ZICOND32:       # %bb.0:
-; ZICOND32-NEXT:    slti a6, a5, 0
+; ZICOND32-NEXT:    srli a6, a5, 31
 ; ZICOND32-NEXT:    sltiu a4, a4, 4
 ; ZICOND32-NEXT:    czero.eqz a6, a6, a5
 ; ZICOND32-NEXT:    czero.nez a4, a4, a5
diff --git a/llvm/test/CodeGen/RISCV/select-cc.ll b/llvm/test/CodeGen/RISCV/select-cc.ll
index ec1f8aeddcaaf..3df07073e0ed3 100644
--- a/llvm/test/CodeGen/RISCV/select-cc.ll
+++ b/llvm/test/CodeGen/RISCV/select-cc.ll
@@ -200,7 +200,7 @@ define signext i32 @foo(i32 signext %a, ptr %b) nounwind {
 ; RV64I-CCMOV-NEXT:    lw a4, 0(a1)
 ; RV64I-CCMOV-NEXT:    slti a5, a2, 1
 ; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a0, a2
-; RV64I-CCMOV-NEXT:    slti a5, a2, 0
+; RV64I-CCMOV-NEXT:    srli a5, a2, 63
 ; RV64I-CCMOV-NEXT:    mips.ccmov a0, a5, a3, a0
 ; RV64I-CCMOV-NEXT:    lw a1, 0(a1)
 ; RV64I-CCMOV-NEXT:    slti a3, a4, 1025
@@ -384,11 +384,11 @@ define i64 @select_sge_int32min(i64 %x, i64 %y, i64 %z) {
 ; RV32I-NEXT:    li a6, -1
 ; RV32I-NEXT:    bne a1, a6, .LBB3_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    j .LBB3_3
 ; RV32I-NEXT:  .LBB3_2:
-; RV32I-NEXT:    slti a0, a1, 0
-; RV32I-NEXT:    xori a0, a0, 1
+; RV32I-NEXT:    srli a1, a1, 31
+; RV32I-NEXT:    xori a0, a1, 1
 ; RV32I-NEXT:  .LBB3_3:
 ; RV32I-NEXT:    bnez a0, .LBB3_5
 ; RV32I-NEXT:  # %bb.4:
diff --git a/llvm/test/CodeGen/RISCV/select-constant-xor.ll b/llvm/test/CodeGen/RISCV/select-constant-xor.ll
index 72313a82b3d39..f11fb617c3b12 100644
--- a/llvm/test/CodeGen/RISCV/select-constant-xor.ll
+++ b/llvm/test/CodeGen/RISCV/select-constant-xor.ll
@@ -48,8 +48,8 @@ define i64 @selecti64i64(i64 %a) {
 define i32 @selecti64i32(i64 %a) {
 ; RV32-LABEL: selecti64i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    slti a0, a1, 0
-; RV32-NEXT:    xori a0, a0, 1
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    xori a0, a1, 1
 ; RV32-NEXT:    lui a1, 524288
 ; RV32-NEXT:    sub a0, a1, a0
 ; RV32-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/selectcc-to-shiftand.ll b/llvm/test/CodeGen/RISCV/selectcc-to-shiftand.ll
index 3fbaefffac2ef..fa1807cd7d911 100644
--- a/llvm/test/CodeGen/RISCV/selectcc-to-shiftand.ll
+++ b/llvm/test/CodeGen/RISCV/selectcc-to-shiftand.ll
@@ -76,12 +76,19 @@ define i32 @not_pos_sel_same_variable(i32 signext %a) {
 
 ; Compare if positive and select of constants where one constant is zero.
 define i32 @pos_sel_constants(i32 signext %a) {
-; CHECK-LABEL: pos_sel_constants:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    slti a0, a0, 0
-; CHECK-NEXT:    addi a0, a0, -1
-; CHECK-NEXT:    andi a0, a0, 5
-; CHECK-NEXT:    ret
+; RV32-LABEL: pos_sel_constants:
+; RV32:       # %bb.0:
+; RV32-NEXT:    srli a0, a0, 31
+; RV32-NEXT:    addi a0, a0, -1
+; RV32-NEXT:    andi a0, a0, 5
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: pos_sel_constants:
+; RV64:       # %bb.0:
+; RV64-NEXT:    srli a0, a0, 63
+; RV64-NEXT:    addi a0, a0, -1
+; RV64-NEXT:    andi a0, a0, 5
+; RV64-NEXT:    ret
   %tmp.1 = icmp sgt i32 %a, -1
   %retval = select i1 %tmp.1, i32 5, i32 0
   ret i32 %retval
@@ -101,7 +108,7 @@ define i32 @pos_sel_special_constant(i32 signext %a) {
 ;
 ; RV64-LABEL: pos_sel_special_constant:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    slti a0, a0, 0
+; RV64-NEXT:    srli a0, a0, 63
 ; RV64-NEXT:    xori a0, a0, 1
 ; RV64-NEXT:    slli a0, a0, 9
 ; RV64-NEXT:    ret
@@ -114,14 +121,14 @@ define i32 @pos_sel_special_constant(i32 signext %a) {
 define i32 @pos_sel_variable_and_zero(i32 signext %a, i32 signext %b) {
 ; RV32I-LABEL: pos_sel_variable_and_zero:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slti a0, a0, 0
+; RV32I-NEXT:    srli a0, a0, 31
 ; RV32I-NEXT:    addi a0, a0, -1
 ; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: pos_sel_variable_and_zero:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slti a0, a0, 0
+; RV64I-NEXT:    srli a0, a0, 63
 ; RV64I-NEXT:    addi a0, a0, -1
 ; RV64I-NEXT:    and a0, a0, a1
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/stack-folding.ll b/llvm/test/CodeGen/RISCV/stack-folding.ll
index 8373a745e45cb..0e3291167d0ec 100644
--- a/llvm/test/CodeGen/RISCV/stack-folding.ll
+++ b/llvm/test/CodeGen/RISCV/stack-folding.ll
@@ -31,8 +31,8 @@ define i1 @test_sext_w(i64 %x, i32 %y) nounwind {
 ; CHECK-NEXT:    li a0, 0
 ; CHECK-NEXT:    j .LBB0_3
 ; CHECK-NEXT:  .LBB0_2: # %truebb
-; CHECK-NEXT:    lw a0, 8(sp) # 8-byte Folded Reload
-; CHECK-NEXT:    slti a0, a0, 0
+; CHECK-NEXT:    ld a0, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    srliw a0, a0, 31
 ; CHECK-NEXT:  .LBB0_3: # %falsebb
 ; CHECK-NEXT:    ld ra, 120(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld s0, 112(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/stack-probing-frame-setup.mir b/llvm/test/CodeGen/RISCV/stack-probing-frame-setup.mir
index 9ab7f41045c4d..ad7273a29238f 100644
--- a/llvm/test/CodeGen/RISCV/stack-probing-frame-setup.mir
+++ b/llvm/test/CodeGen/RISCV/stack-probing-frame-setup.mir
@@ -63,8 +63,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: v, type: variable-sized, offset: 0, alignment: 1, stack-id: default,
diff --git a/llvm/test/CodeGen/RISCV/stack-slot-coloring.mir b/llvm/test/CodeGen/RISCV/stack-slot-coloring.mir
index 4ea0270c6bc26..869553f8cf051 100644
--- a/llvm/test/CodeGen/RISCV/stack-slot-coloring.mir
+++ b/llvm/test/CodeGen/RISCV/stack-slot-coloring.mir
@@ -49,8 +49,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/RISCV/unaligned-load-store.ll b/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
index c9c49e8f7f532..b396d16b0eb26 100644
--- a/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/unaligned-load-store.ll
@@ -204,18 +204,16 @@ define i64 @load_i64(ptr %p) {
 ; RV64IZBKB-NEXT:    lbu a2, 5(a0)
 ; RV64IZBKB-NEXT:    lbu a3, 6(a0)
 ; RV64IZBKB-NEXT:    lbu a4, 7(a0)
-; RV64IZBKB-NEXT:    lbu a5, 0(a0)
-; RV64IZBKB-NEXT:    lbu a6, 1(a0)
-; RV64IZBKB-NEXT:    lbu a7, 2(a0)
-; RV64IZBKB-NEXT:    lbu a0, 3(a0)
+; RV64IZBKB-NEXT:    lbu a5, 1(a0)
+; RV64IZBKB-NEXT:    lbu a6, 2(a0)
+; RV64IZBKB-NEXT:    lbu a7, 3(a0)
+; RV64IZBKB-NEXT:    lbu a0, 0(a0)
+; RV64IZBKB-NEXT:    packh a3, a3, a4
 ; RV64IZBKB-NEXT:    packh a1, a1, a2
-; RV64IZBKB-NEXT:    packh a2, a3, a4
-; RV64IZBKB-NEXT:    packh a3, a5, a6
-; RV64IZBKB-NEXT:    packh a0, a7, a0
-; RV64IZBKB-NEXT:    slli a2, a2, 16
-; RV64IZBKB-NEXT:    slli a0, a0, 16
-; RV64IZBKB-NEXT:    or a1, a2, a1
-; RV64IZBKB-NEXT:    or a0, a0, a3
+; RV64IZBKB-NEXT:    packh a2, a6, a7
+; RV64IZBKB-NEXT:    packh a0, a0, a5
+; RV64IZBKB-NEXT:    packw a1, a1, a3
+; RV64IZBKB-NEXT:    packw a0, a0, a2
 ; RV64IZBKB-NEXT:    pack a0, a0, a1
 ; RV64IZBKB-NEXT:    ret
 ;
@@ -234,6 +232,69 @@ define i64 @load_i64(ptr %p) {
   ret i64 %res
 }
 
+define i64 @load_i64_align2(ptr %p) {
+; RV32I-LABEL: load_i64_align2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lhu a1, 2(a0)
+; RV32I-NEXT:    lhu a2, 0(a0)
+; RV32I-NEXT:    lhu a3, 6(a0)
+; RV32I-NEXT:    lhu a4, 4(a0)
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    slli a1, a3, 16
+; RV32I-NEXT:    or a1, a1, a4
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: load_i64_align2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lhu a1, 2(a0)
+; RV64I-NEXT:    lhu a2, 0(a0)
+; RV64I-NEXT:    lhu a3, 4(a0)
+; RV64I-NEXT:    lhu a0, 6(a0)
+; RV64I-NEXT:    slli a1, a1, 16
+; RV64I-NEXT:    or a1, a1, a2
+; RV64I-NEXT:    slli a3, a3, 32
+; RV64I-NEXT:    slli a0, a0, 48
+; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV32IZBKB-LABEL: load_i64_align2:
+; RV32IZBKB:       # %bb.0:
+; RV32IZBKB-NEXT:    lhu a1, 0(a0)
+; RV32IZBKB-NEXT:    lhu a2, 2(a0)
+; RV32IZBKB-NEXT:    lhu a3, 4(a0)
+; RV32IZBKB-NEXT:    lhu a4, 6(a0)
+; RV32IZBKB-NEXT:    pack a0, a1, a2
+; RV32IZBKB-NEXT:    pack a1, a3, a4
+; RV32IZBKB-NEXT:    ret
+;
+; RV64IZBKB-LABEL: load_i64_align2:
+; RV64IZBKB:       # %bb.0:
+; RV64IZBKB-NEXT:    lhu a1, 2(a0)
+; RV64IZBKB-NEXT:    lhu a2, 4(a0)
+; RV64IZBKB-NEXT:    lhu a3, 6(a0)
+; RV64IZBKB-NEXT:    lhu a0, 0(a0)
+; RV64IZBKB-NEXT:    packw a2, a2, a3
+; RV64IZBKB-NEXT:    packw a0, a0, a1
+; RV64IZBKB-NEXT:    pack a0, a0, a2
+; RV64IZBKB-NEXT:    ret
+;
+; RV32I-FAST-LABEL: load_i64_align2:
+; RV32I-FAST:       # %bb.0:
+; RV32I-FAST-NEXT:    lw a2, 0(a0)
+; RV32I-FAST-NEXT:    lw a1, 4(a0)
+; RV32I-FAST-NEXT:    mv a0, a2
+; RV32I-FAST-NEXT:    ret
+;
+; RV64I-FAST-LABEL: load_i64_align2:
+; RV64I-FAST:       # %bb.0:
+; RV64I-FAST-NEXT:    ld a0, 0(a0)
+; RV64I-FAST-NEXT:    ret
+  %res = load i64, ptr %p, align 2
+  ret i64 %res
+}
+
 define void @store_i8(ptr %p, i8 %v) {
 ; ALL-LABEL: store_i8:
 ; ALL:       # %bb.0:
@@ -571,8 +632,8 @@ define void @store_large_constant(ptr %x) {
 ;
 ; RV64I-FAST-LABEL: store_large_constant:
 ; RV64I-FAST:       # %bb.0:
-; RV64I-FAST-NEXT:    lui a1, %hi(.LCPI16_0)
-; RV64I-FAST-NEXT:    ld a1, %lo(.LCPI16_0)(a1)
+; RV64I-FAST-NEXT:    lui a1, %hi(.LCPI17_0)
+; RV64I-FAST-NEXT:    ld a1, %lo(.LCPI17_0)(a1)
 ; RV64I-FAST-NEXT:    sd a1, 0(a0)
 ; RV64I-FAST-NEXT:    ret
   store i64 18364758544493064720, ptr %x, align 1
diff --git a/llvm/test/CodeGen/RISCV/xaluo.ll b/llvm/test/CodeGen/RISCV/xaluo.ll
index a30593d7d7afb..2751332c9e3ae 100644
--- a/llvm/test/CodeGen/RISCV/xaluo.ll
+++ b/llvm/test/CodeGen/RISCV/xaluo.ll
@@ -14,7 +14,7 @@ define zeroext i1 @saddo1.i32(i32 signext %v1, i32 signext %v2, ptr %res) {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    add a3, a0, a1
 ; RV32-NEXT:    slt a0, a3, a0
-; RV32-NEXT:    slti a1, a1, 0
+; RV32-NEXT:    srli a1, a1, 31
 ; RV32-NEXT:    xor a0, a1, a0
 ; RV32-NEXT:    sw a3, 0(a2)
 ; RV32-NEXT:    ret
@@ -32,7 +32,7 @@ define zeroext i1 @saddo1.i32(i32 signext %v1, i32 signext %v2, ptr %res) {
 ; RV32ZBA:       # %bb.0: # %entry
 ; RV32ZBA-NEXT:    add a3, a0, a1
 ; RV32ZBA-NEXT:    slt a0, a3, a0
-; RV32ZBA-NEXT:    slti a1, a1, 0
+; RV32ZBA-NEXT:    srli a1, a1, 31
 ; RV32ZBA-NEXT:    xor a0, a1, a0
 ; RV32ZBA-NEXT:    sw a3, 0(a2)
 ; RV32ZBA-NEXT:    ret
@@ -50,7 +50,7 @@ define zeroext i1 @saddo1.i32(i32 signext %v1, i32 signext %v2, ptr %res) {
 ; RV32ZICOND:       # %bb.0: # %entry
 ; RV32ZICOND-NEXT:    add a3, a0, a1
 ; RV32ZICOND-NEXT:    slt a0, a3, a0
-; RV32ZICOND-NEXT:    slti a1, a1, 0
+; RV32ZICOND-NEXT:    srli a1, a1, 31
 ; RV32ZICOND-NEXT:    xor a0, a1, a0
 ; RV32ZICOND-NEXT:    sw a3, 0(a2)
 ; RV32ZICOND-NEXT:    ret
@@ -252,8 +252,8 @@ define zeroext i1 @saddo1.i64(i64 %v1, i64 %v2, ptr %res) {
 ; RV32-NEXT:    not a3, a3
 ; RV32-NEXT:    add a5, a5, a0
 ; RV32-NEXT:    xor a1, a1, a5
-; RV32-NEXT:    and a1, a3, a1
-; RV32-NEXT:    slti a0, a1, 0
+; RV32-NEXT:    and a0, a3, a1
+; RV32-NEXT:    srli a0, a0, 31
 ; RV32-NEXT:    sw a2, 0(a4)
 ; RV32-NEXT:    sw a5, 4(a4)
 ; RV32-NEXT:    ret
@@ -262,7 +262,7 @@ define zeroext i1 @saddo1.i64(i64 %v1, i64 %v2, ptr %res) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    add a3, a0, a1
 ; RV64-NEXT:    slt a0, a3, a0
-; RV64-NEXT:    slti a1, a1, 0
+; RV64-NEXT:    srli a1, a1, 63
 ; RV64-NEXT:    xor a0, a1, a0
 ; RV64-NEXT:    sd a3, 0(a2)
 ; RV64-NEXT:    ret
@@ -276,8 +276,8 @@ define zeroext i1 @saddo1.i64(i64 %v1, i64 %v2, ptr %res) {
 ; RV32ZBA-NEXT:    not a3, a3
 ; RV32ZBA-NEXT:    add a5, a5, a0
 ; RV32ZBA-NEXT:    xor a1, a1, a5
-; RV32ZBA-NEXT:    and a1, a3, a1
-; RV32ZBA-NEXT:    slti a0, a1, 0
+; RV32ZBA-NEXT:    and a0, a3, a1
+; RV32ZBA-NEXT:    srli a0, a0, 31
 ; RV32ZBA-NEXT:    sw a2, 0(a4)
 ; RV32ZBA-NEXT:    sw a5, 4(a4)
 ; RV32ZBA-NEXT:    ret
@@ -286,7 +286,7 @@ define zeroext i1 @saddo1.i64(i64 %v1, i64 %v2, ptr %res) {
 ; RV64ZBA:       # %bb.0: # %entry
 ; RV64ZBA-NEXT:    add a3, a0, a1
 ; RV64ZBA-NEXT:    slt a0, a3, a0
-; RV64ZBA-NEXT:    slti a1, a1, 0
+; RV64ZBA-NEXT:    srli a1, a1, 63
 ; RV64ZBA-NEXT:    xor a0, a1, a0
 ; RV64ZBA-NEXT:    sd a3, 0(a2)
 ; RV64ZBA-NEXT:    ret
@@ -300,8 +300,8 @@ define zeroext i1 @saddo1.i64(i64 %v1, i64 %v2, ptr %res) {
 ; RV32ZICOND-NEXT:    not a3, a3
 ; RV32ZICOND-NEXT:    add a5, a5, a0
 ; RV32ZICOND-NEXT:    xor a1, a1, a5
-; RV32ZICOND-NEXT:    and a1, a3, a1
-; RV32ZICOND-NEXT:    slti a0, a1, 0
+; RV32ZICOND-NEXT:    and a0, a3, a1
+; RV32ZICOND-NEXT:    srli a0, a0, 31
 ; RV32ZICOND-NEXT:    sw a2, 0(a4)
 ; RV32ZICOND-NEXT:    sw a5, 4(a4)
 ; RV32ZICOND-NEXT:    ret
@@ -310,7 +310,7 @@ define zeroext i1 @saddo1.i64(i64 %v1, i64 %v2, ptr %res) {
 ; RV64ZICOND:       # %bb.0: # %entry
 ; RV64ZICOND-NEXT:    add a3, a0, a1
 ; RV64ZICOND-NEXT:    slt a0, a3, a0
-; RV64ZICOND-NEXT:    slti a1, a1, 0
+; RV64ZICOND-NEXT:    srli a1, a1, 63
 ; RV64ZICOND-NEXT:    xor a0, a1, a0
 ; RV64ZICOND-NEXT:    sd a3, 0(a2)
 ; RV64ZICOND-NEXT:    ret
@@ -330,8 +330,8 @@ define zeroext i1 @saddo2.i64(i64 %v1, ptr %res) {
 ; RV32-NEXT:    sltu a0, a3, a0
 ; RV32-NEXT:    add a5, a1, a0
 ; RV32-NEXT:    xor a1, a1, a5
-; RV32-NEXT:    and a1, a4, a1
-; RV32-NEXT:    slti a0, a1, 0
+; RV32-NEXT:    and a0, a4, a1
+; RV32-NEXT:    srli a0, a0, 31
 ; RV32-NEXT:    sw a3, 0(a2)
 ; RV32-NEXT:    sw a5, 4(a2)
 ; RV32-NEXT:    ret
@@ -350,8 +350,8 @@ define zeroext i1 @saddo2.i64(i64 %v1, ptr %res) {
 ; RV32ZBA-NEXT:    sltu a0, a3, a0
 ; RV32ZBA-NEXT:    add a5, a1, a0
 ; RV32ZBA-NEXT:    xor a1, a1, a5
-; RV32ZBA-NEXT:    and a1, a4, a1
-; RV32ZBA-NEXT:    slti a0, a1, 0
+; RV32ZBA-NEXT:    and a0, a4, a1
+; RV32ZBA-NEXT:    srli a0, a0, 31
 ; RV32ZBA-NEXT:    sw a3, 0(a2)
 ; RV32ZBA-NEXT:    sw a5, 4(a2)
 ; RV32ZBA-NEXT:    ret
@@ -370,8 +370,8 @@ define zeroext i1 @saddo2.i64(i64 %v1, ptr %res) {
 ; RV32ZICOND-NEXT:    sltu a0, a3, a0
 ; RV32ZICOND-NEXT:    add a5, a1, a0
 ; RV32ZICOND-NEXT:    xor a1, a1, a5
-; RV32ZICOND-NEXT:    and a1, a4, a1
-; RV32ZICOND-NEXT:    slti a0, a1, 0
+; RV32ZICOND-NEXT:    and a0, a4, a1
+; RV32ZICOND-NEXT:    srli a0, a0, 31
 ; RV32ZICOND-NEXT:    sw a3, 0(a2)
 ; RV32ZICOND-NEXT:    sw a5, 4(a2)
 ; RV32ZICOND-NEXT:    ret
@@ -399,7 +399,7 @@ define zeroext i1 @saddo3.i64(i64 %v1, ptr %res) {
 ; RV32-NEXT:    addi a4, a0, -1
 ; RV32-NEXT:    xor a0, a1, a4
 ; RV32-NEXT:    and a0, a1, a0
-; RV32-NEXT:    slti a0, a0, 0
+; RV32-NEXT:    srli a0, a0, 31
 ; RV32-NEXT:    sw a3, 0(a2)
 ; RV32-NEXT:    sw a4, 4(a2)
 ; RV32-NEXT:    ret
@@ -420,7 +420,7 @@ define zeroext i1 @saddo3.i64(i64 %v1, ptr %res) {
 ; RV32ZBA-NEXT:    addi a4, a0, -1
 ; RV32ZBA-NEXT:    xor a0, a1, a4
 ; RV32ZBA-NEXT:    and a0, a1, a0
-; RV32ZBA-NEXT:    slti a0, a0, 0
+; RV32ZBA-NEXT:    srli a0, a0, 31
 ; RV32ZBA-NEXT:    sw a3, 0(a2)
 ; RV32ZBA-NEXT:    sw a4, 4(a2)
 ; RV32ZBA-NEXT:    ret
@@ -441,7 +441,7 @@ define zeroext i1 @saddo3.i64(i64 %v1, ptr %res) {
 ; RV32ZICOND-NEXT:    addi a4, a0, -1
 ; RV32ZICOND-NEXT:    xor a0, a1, a4
 ; RV32ZICOND-NEXT:    and a0, a1, a0
-; RV32ZICOND-NEXT:    slti a0, a0, 0
+; RV32ZICOND-NEXT:    srli a0, a0, 31
 ; RV32ZICOND-NEXT:    sw a3, 0(a2)
 ; RV32ZICOND-NEXT:    sw a4, 4(a2)
 ; RV32ZICOND-NEXT:    ret
@@ -866,8 +866,8 @@ define zeroext i1 @ssubo.i64(i64 %v1, i64 %v2, ptr %res) {
 ; RV32-NEXT:    sub a2, a0, a2
 ; RV32-NEXT:    sub a5, a6, a5
 ; RV32-NEXT:    xor a1, a1, a5
-; RV32-NEXT:    and a1, a3, a1
-; RV32-NEXT:    slti a0, a1, 0
+; RV32-NEXT:    and a0, a3, a1
+; RV32-NEXT:    srli a0, a0, 31
 ; RV32-NEXT:    sw a2, 0(a4)
 ; RV32-NEXT:    sw a5, 4(a4)
 ; RV32-NEXT:    ret
@@ -889,8 +889,8 @@ define zeroext i1 @ssubo.i64(i64 %v1, i64 %v2, ptr %res) {
 ; RV32ZBA-NEXT:    sub a2, a0, a2
 ; RV32ZBA-NEXT:    sub a5, a6, a5
 ; RV32ZBA-NEXT:    xor a1, a1, a5
-; RV32ZBA-NEXT:    and a1, a3, a1
-; RV32ZBA-NEXT:    slti a0, a1, 0
+; RV32ZBA-NEXT:    and a0, a3, a1
+; RV32ZBA-NEXT:    srli a0, a0, 31
 ; RV32ZBA-NEXT:    sw a2, 0(a4)
 ; RV32ZBA-NEXT:    sw a5, 4(a4)
 ; RV32ZBA-NEXT:    ret
@@ -912,8 +912,8 @@ define zeroext i1 @ssubo.i64(i64 %v1, i64 %v2, ptr %res) {
 ; RV32ZICOND-NEXT:    sub a2, a0, a2
 ; RV32ZICOND-NEXT:    sub a5, a6, a5
 ; RV32ZICOND-NEXT:    xor a1, a1, a5
-; RV32ZICOND-NEXT:    and a1, a3, a1
-; RV32ZICOND-NEXT:    slti a0, a1, 0
+; RV32ZICOND-NEXT:    and a0, a3, a1
+; RV32ZICOND-NEXT:    srli a0, a0, 31
 ; RV32ZICOND-NEXT:    sw a2, 0(a4)
 ; RV32ZICOND-NEXT:    sw a5, 4(a4)
 ; RV32ZICOND-NEXT:    ret
@@ -1963,7 +1963,7 @@ define i32 @saddo.select.i32(i32 signext %v1, i32 signext %v2) {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    add a2, a0, a1
 ; RV32-NEXT:    slt a2, a2, a0
-; RV32-NEXT:    slti a3, a1, 0
+; RV32-NEXT:    srli a3, a1, 31
 ; RV32-NEXT:    bne a3, a2, .LBB28_2
 ; RV32-NEXT:  # %bb.1: # %entry
 ; RV32-NEXT:    mv a0, a1
@@ -1984,7 +1984,7 @@ define i32 @saddo.select.i32(i32 signext %v1, i32 signext %v2) {
 ; RV32ZBA:       # %bb.0: # %entry
 ; RV32ZBA-NEXT:    add a2, a0, a1
 ; RV32ZBA-NEXT:    slt a2, a2, a0
-; RV32ZBA-NEXT:    slti a3, a1, 0
+; RV32ZBA-NEXT:    srli a3, a1, 31
 ; RV32ZBA-NEXT:    bne a3, a2, .LBB28_2
 ; RV32ZBA-NEXT:  # %bb.1: # %entry
 ; RV32ZBA-NEXT:    mv a0, a1
@@ -2004,7 +2004,7 @@ define i32 @saddo.select.i32(i32 signext %v1, i32 signext %v2) {
 ; RV32ZICOND-LABEL: saddo.select.i32:
 ; RV32ZICOND:       # %bb.0: # %entry
 ; RV32ZICOND-NEXT:    add a2, a0, a1
-; RV32ZICOND-NEXT:    slti a3, a1, 0
+; RV32ZICOND-NEXT:    srli a3, a1, 31
 ; RV32ZICOND-NEXT:    slt a2, a2, a0
 ; RV32ZICOND-NEXT:    xor a2, a3, a2
 ; RV32ZICOND-NEXT:    czero.nez a1, a1, a2
@@ -2033,7 +2033,7 @@ define i1 @saddo.not.i32(i32 signext %v1, i32 signext %v2) {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    add a2, a0, a1
 ; RV32-NEXT:    slt a0, a2, a0
-; RV32-NEXT:    slti a1, a1, 0
+; RV32-NEXT:    srli a1, a1, 31
 ; RV32-NEXT:    xor a0, a1, a0
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    ret
@@ -2050,7 +2050,7 @@ define i1 @saddo.not.i32(i32 signext %v1, i32 signext %v2) {
 ; RV32ZBA:       # %bb.0: # %entry
 ; RV32ZBA-NEXT:    add a2, a0, a1
 ; RV32ZBA-NEXT:    slt a0, a2, a0
-; RV32ZBA-NEXT:    slti a1, a1, 0
+; RV32ZBA-NEXT:    srli a1, a1, 31
 ; RV32ZBA-NEXT:    xor a0, a1, a0
 ; RV32ZBA-NEXT:    xori a0, a0, 1
 ; RV32ZBA-NEXT:    ret
@@ -2067,7 +2067,7 @@ define i1 @saddo.not.i32(i32 signext %v1, i32 signext %v2) {
 ; RV32ZICOND:       # %bb.0: # %entry
 ; RV32ZICOND-NEXT:    add a2, a0, a1
 ; RV32ZICOND-NEXT:    slt a0, a2, a0
-; RV32ZICOND-NEXT:    slti a1, a1, 0
+; RV32ZICOND-NEXT:    srli a1, a1, 31
 ; RV32ZICOND-NEXT:    xor a0, a1, a0
 ; RV32ZICOND-NEXT:    xori a0, a0, 1
 ; RV32ZICOND-NEXT:    ret
@@ -2108,7 +2108,7 @@ define i64 @saddo.select.i64(i64 %v1, i64 %v2) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    add a2, a0, a1
 ; RV64-NEXT:    slt a2, a2, a0
-; RV64-NEXT:    slti a3, a1, 0
+; RV64-NEXT:    srli a3, a1, 63
 ; RV64-NEXT:    bne a3, a2, .LBB30_2
 ; RV64-NEXT:  # %bb.1: # %entry
 ; RV64-NEXT:    mv a0, a1
@@ -2136,7 +2136,7 @@ define i64 @saddo.select.i64(i64 %v1, i64 %v2) {
 ; RV64ZBA:       # %bb.0: # %entry
 ; RV64ZBA-NEXT:    add a2, a0, a1
 ; RV64ZBA-NEXT:    slt a2, a2, a0
-; RV64ZBA-NEXT:    slti a3, a1, 0
+; RV64ZBA-NEXT:    srli a3, a1, 63
 ; RV64ZBA-NEXT:    bne a3, a2, .LBB30_2
 ; RV64ZBA-NEXT:  # %bb.1: # %entry
 ; RV64ZBA-NEXT:    mv a0, a1
@@ -2153,7 +2153,7 @@ define i64 @saddo.select.i64(i64 %v1, i64 %v2) {
 ; RV32ZICOND-NEXT:    not a5, a5
 ; RV32ZICOND-NEXT:    xor a4, a1, a4
 ; RV32ZICOND-NEXT:    and a4, a5, a4
-; RV32ZICOND-NEXT:    slti a4, a4, 0
+; RV32ZICOND-NEXT:    srli a4, a4, 31
 ; RV32ZICOND-NEXT:    czero.nez a2, a2, a4
 ; RV32ZICOND-NEXT:    czero.eqz a0, a0, a4
 ; RV32ZICOND-NEXT:    czero.nez a3, a3, a4
@@ -2165,7 +2165,7 @@ define i64 @saddo.select.i64(i64 %v1, i64 %v2) {
 ; RV64ZICOND-LABEL: saddo.select.i64:
 ; RV64ZICOND:       # %bb.0: # %entry
 ; RV64ZICOND-NEXT:    add a2, a0, a1
-; RV64ZICOND-NEXT:    slti a3, a1, 0
+; RV64ZICOND-NEXT:    srli a3, a1, 63
 ; RV64ZICOND-NEXT:    slt a2, a2, a0
 ; RV64ZICOND-NEXT:    xor a2, a3, a2
 ; RV64ZICOND-NEXT:    czero.nez a1, a1, a2
@@ -2190,7 +2190,7 @@ define i1 @saddo.not.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    xor a0, a1, a0
 ; RV32-NEXT:    not a1, a3
 ; RV32-NEXT:    and a0, a1, a0
-; RV32-NEXT:    slti a0, a0, 0
+; RV32-NEXT:    srli a0, a0, 31
 ; RV32-NEXT:    xori a0, a0, 1
 ; RV32-NEXT:    ret
 ;
@@ -2198,7 +2198,7 @@ define i1 @saddo.not.i64(i64 %v1, i64 %v2) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    add a2, a0, a1
 ; RV64-NEXT:    slt a0, a2, a0
-; RV64-NEXT:    slti a1, a1, 0
+; RV64-NEXT:    srli a1, a1, 63
 ; RV64-NEXT:    xor a0, a1, a0
 ; RV64-NEXT:    xori a0, a0, 1
 ; RV64-NEXT:    ret
@@ -2213,7 +2213,7 @@ define i1 @saddo.not.i64(i64 %v1, i64 %v2) {
 ; RV32ZBA-NEXT:    xor a0, a1, a0
 ; RV32ZBA-NEXT:    not a1, a3
 ; RV32ZBA-NEXT:    and a0, a1, a0
-; RV32ZBA-NEXT:    slti a0, a0, 0
+; RV32ZBA-NEXT:    srli a0, a0, 31
 ; RV32ZBA-NEXT:    xori a0, a0, 1
 ; RV32ZBA-NEXT:    ret
 ;
@@ -2221,7 +2221,7 @@ define i1 @saddo.not.i64(i64 %v1, i64 %v2) {
 ; RV64ZBA:       # %bb.0: # %entry
 ; RV64ZBA-NEXT:    add a2, a0, a1
 ; RV64ZBA-NEXT:    slt a0, a2, a0
-; RV64ZBA-NEXT:    slti a1, a1, 0
+; RV64ZBA-NEXT:    srli a1, a1, 63
 ; RV64ZBA-NEXT:    xor a0, a1, a0
 ; RV64ZBA-NEXT:    xori a0, a0, 1
 ; RV64ZBA-NEXT:    ret
@@ -2236,7 +2236,7 @@ define i1 @saddo.not.i64(i64 %v1, i64 %v2) {
 ; RV32ZICOND-NEXT:    xor a0, a1, a0
 ; RV32ZICOND-NEXT:    not a1, a3
 ; RV32ZICOND-NEXT:    and a0, a1, a0
-; RV32ZICOND-NEXT:    slti a0, a0, 0
+; RV32ZICOND-NEXT:    srli a0, a0, 31
 ; RV32ZICOND-NEXT:    xori a0, a0, 1
 ; RV32ZICOND-NEXT:    ret
 ;
@@ -2244,7 +2244,7 @@ define i1 @saddo.not.i64(i64 %v1, i64 %v2) {
 ; RV64ZICOND:       # %bb.0: # %entry
 ; RV64ZICOND-NEXT:    add a2, a0, a1
 ; RV64ZICOND-NEXT:    slt a0, a2, a0
-; RV64ZICOND-NEXT:    slti a1, a1, 0
+; RV64ZICOND-NEXT:    srli a1, a1, 63
 ; RV64ZICOND-NEXT:    xor a0, a1, a0
 ; RV64ZICOND-NEXT:    xori a0, a0, 1
 ; RV64ZICOND-NEXT:    ret
@@ -2713,7 +2713,7 @@ define i64 @ssubo.select.i64(i64 %v1, i64 %v2) {
 ; RV32ZICOND-NEXT:    xor a4, a1, a3
 ; RV32ZICOND-NEXT:    xor a5, a1, a5
 ; RV32ZICOND-NEXT:    and a4, a4, a5
-; RV32ZICOND-NEXT:    slti a4, a4, 0
+; RV32ZICOND-NEXT:    srli a4, a4, 31
 ; RV32ZICOND-NEXT:    czero.nez a2, a2, a4
 ; RV32ZICOND-NEXT:    czero.eqz a0, a0, a4
 ; RV32ZICOND-NEXT:    czero.nez a3, a3, a4
@@ -2748,8 +2748,8 @@ define i1 @ssub.not.i64(i64 %v1, i64 %v2) {
 ; RV32-NEXT:    xor a2, a1, a2
 ; RV32-NEXT:    xor a1, a1, a3
 ; RV32-NEXT:    and a1, a1, a2
-; RV32-NEXT:    slti a0, a1, 0
-; RV32-NEXT:    xori a0, a0, 1
+; RV32-NEXT:    srli a1, a1, 31
+; RV32-NEXT:    xori a0, a1, 1
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ssub.not.i64:
@@ -2769,8 +2769,8 @@ define i1 @ssub.not.i64(i64 %v1, i64 %v2) {
 ; RV32ZBA-NEXT:    xor a2, a1, a2
 ; RV32ZBA-NEXT:    xor a1, a1, a3
 ; RV32ZBA-NEXT:    and a1, a1, a2
-; RV32ZBA-NEXT:    slti a0, a1, 0
-; RV32ZBA-NEXT:    xori a0, a0, 1
+; RV32ZBA-NEXT:    srli a1, a1, 31
+; RV32ZBA-NEXT:    xori a0, a1, 1
 ; RV32ZBA-NEXT:    ret
 ;
 ; RV64ZBA-LABEL: ssub.not.i64:
@@ -2790,8 +2790,8 @@ define i1 @ssub.not.i64(i64 %v1, i64 %v2) {
 ; RV32ZICOND-NEXT:    xor a2, a1, a2
 ; RV32ZICOND-NEXT:    xor a1, a1, a3
 ; RV32ZICOND-NEXT:    and a1, a1, a2
-; RV32ZICOND-NEXT:    slti a0, a1, 0
-; RV32ZICOND-NEXT:    xori a0, a0, 1
+; RV32ZICOND-NEXT:    srli a1, a1, 31
+; RV32ZICOND-NEXT:    xori a0, a1, 1
 ; RV32ZICOND-NEXT:    ret
 ;
 ; RV64ZICOND-LABEL: ssub.not.i64:
@@ -3821,7 +3821,7 @@ define zeroext i1 @saddo.br.i32(i32 signext %v1, i32 signext %v2) {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    add a2, a0, a1
 ; RV32-NEXT:    slt a0, a2, a0
-; RV32-NEXT:    slti a1, a1, 0
+; RV32-NEXT:    srli a1, a1, 31
 ; RV32-NEXT:    beq a1, a0, .LBB52_2
 ; RV32-NEXT:  # %bb.1: # %overflow
 ; RV32-NEXT:    li a0, 0
@@ -3846,7 +3846,7 @@ define zeroext i1 @saddo.br.i32(i32 signext %v1, i32 signext %v2) {
 ; RV32ZBA:       # %bb.0: # %entry
 ; RV32ZBA-NEXT:    add a2, a0, a1
 ; RV32ZBA-NEXT:    slt a0, a2, a0
-; RV32ZBA-NEXT:    slti a1, a1, 0
+; RV32ZBA-NEXT:    srli a1, a1, 31
 ; RV32ZBA-NEXT:    beq a1, a0, .LBB52_2
 ; RV32ZBA-NEXT:  # %bb.1: # %overflow
 ; RV32ZBA-NEXT:    li a0, 0
@@ -3871,7 +3871,7 @@ define zeroext i1 @saddo.br.i32(i32 signext %v1, i32 signext %v2) {
 ; RV32ZICOND:       # %bb.0: # %entry
 ; RV32ZICOND-NEXT:    add a2, a0, a1
 ; RV32ZICOND-NEXT:    slt a0, a2, a0
-; RV32ZICOND-NEXT:    slti a1, a1, 0
+; RV32ZICOND-NEXT:    srli a1, a1, 31
 ; RV32ZICOND-NEXT:    beq a1, a0, .LBB52_2
 ; RV32ZICOND-NEXT:  # %bb.1: # %overflow
 ; RV32ZICOND-NEXT:    li a0, 0
@@ -3927,7 +3927,7 @@ define zeroext i1 @saddo.br.i64(i64 %v1, i64 %v2) {
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    add a2, a0, a1
 ; RV64-NEXT:    slt a0, a2, a0
-; RV64-NEXT:    slti a1, a1, 0
+; RV64-NEXT:    srli a1, a1, 63
 ; RV64-NEXT:    beq a1, a0, .LBB53_2
 ; RV64-NEXT:  # %bb.1: # %overflow
 ; RV64-NEXT:    li a0, 0
@@ -3958,7 +3958,7 @@ define zeroext i1 @saddo.br.i64(i64 %v1, i64 %v2) {
 ; RV64ZBA:       # %bb.0: # %entry
 ; RV64ZBA-NEXT:    add a2, a0, a1
 ; RV64ZBA-NEXT:    slt a0, a2, a0
-; RV64ZBA-NEXT:    slti a1, a1, 0
+; RV64ZBA-NEXT:    srli a1, a1, 63
 ; RV64ZBA-NEXT:    beq a1, a0, .LBB53_2
 ; RV64ZBA-NEXT:  # %bb.1: # %overflow
 ; RV64ZBA-NEXT:    li a0, 0
@@ -3989,7 +3989,7 @@ define zeroext i1 @saddo.br.i64(i64 %v1, i64 %v2) {
 ; RV64ZICOND:       # %bb.0: # %entry
 ; RV64ZICOND-NEXT:    add a2, a0, a1
 ; RV64ZICOND-NEXT:    slt a0, a2, a0
-; RV64ZICOND-NEXT:    slti a1, a1, 0
+; RV64ZICOND-NEXT:    srli a1, a1, 63
 ; RV64ZICOND-NEXT:    beq a1, a0, .LBB53_2
 ; RV64ZICOND-NEXT:  # %bb.1: # %overflow
 ; RV64ZICOND-NEXT:    li a0, 0
diff --git a/llvm/test/CodeGen/RISCV/xqcia.ll b/llvm/test/CodeGen/RISCV/xqcia.ll
index c75bb9daefcf2..3bbf33328f529 100644
--- a/llvm/test/CodeGen/RISCV/xqcia.ll
+++ b/llvm/test/CodeGen/RISCV/xqcia.ll
@@ -11,7 +11,7 @@ define i32 @addsat(i32 %a, i32 %b) {
 ; RV32I-NEXT:    mv a2, a0
 ; RV32I-NEXT:    add a0, a0, a1
 ; RV32I-NEXT:    slt a2, a0, a2
-; RV32I-NEXT:    slti a1, a1, 0
+; RV32I-NEXT:    srli a1, a1, 31
 ; RV32I-NEXT:    beq a1, a2, .LBB0_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    srai a0, a0, 31
diff --git a/llvm/test/CodeGen/RISCV/xqcibi.ll b/llvm/test/CodeGen/RISCV/xqcibi.ll
index f628e0086374c..518ada6c9e631 100644
--- a/llvm/test/CodeGen/RISCV/xqcibi.ll
+++ b/llvm/test/CodeGen/RISCV/xqcibi.ll
@@ -329,7 +329,8 @@ t:
 define i32 @bgeuimm16(i32 %a) {
 ; RV32I-LABEL: bgeuimm16:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    li a1, 99
+; RV32I-NEXT:    lui a1, 16
+; RV32I-NEXT:    addi a1, a1, -2
 ; RV32I-NEXT:    bltu a1, a0, .LBB11_2
 ; RV32I-NEXT:  # %bb.1: # %f
 ; RV32I-NEXT:    li a0, 0
@@ -340,14 +341,14 @@ define i32 @bgeuimm16(i32 %a) {
 ;
 ; RV32IXQCIBI-LABEL: bgeuimm16:
 ; RV32IXQCIBI:       # %bb.0:
-; RV32IXQCIBI-NEXT:    qc.e.bgeui a0, 100, .LBB11_2
+; RV32IXQCIBI-NEXT:    qc.e.bgeui a0, 65535, .LBB11_2
 ; RV32IXQCIBI-NEXT:  # %bb.1: # %f
 ; RV32IXQCIBI-NEXT:    li a0, 0
 ; RV32IXQCIBI-NEXT:    ret
 ; RV32IXQCIBI-NEXT:  .LBB11_2: # %t
 ; RV32IXQCIBI-NEXT:    li a0, 1
 ; RV32IXQCIBI-NEXT:    ret
-  %1 = icmp uge i32 %a, 100
+  %1 = icmp uge i32 %a, 65535
   br i1 %1, label %t, label %f, !prof !0
 f:
   ret i32 0
diff --git a/llvm/test/CodeGen/RISCV/xqcibm-extract.ll b/llvm/test/CodeGen/RISCV/xqcibm-extract.ll
index 481bfdd666430..34657baa47f48 100644
--- a/llvm/test/CodeGen/RISCV/xqcibm-extract.ll
+++ b/llvm/test/CodeGen/RISCV/xqcibm-extract.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefixes=RV32I
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV32I
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcibm -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefixes=RV32XQCIBM
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV32XQCIBM
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcibm,+zbb -verify-machineinstrs < %s \
-; RUN:   | FileCheck %s -check-prefixes=RV32XQCIBMZBB
+; RUN:   | FileCheck %s -check-prefixes=CHECK,RV32XQCIBMZBB
 
 define i32 @sexti1_i32(i1 %a) nounwind {
 ; RV32I-LABEL: sexti1_i32:
@@ -47,6 +47,57 @@ define i32 @sexti1_i32_2(i32 %a) {
   ret i32 %shr
 }
 
+; Make sure we don't use not+qc.ext
+define zeroext i8 @sexti1_i32_setcc(i32 signext %a) {
+; RV32I-LABEL: sexti1_i32_setcc:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    srli a0, a0, 31
+; RV32I-NEXT:    addi a0, a0, -1
+; RV32I-NEXT:    zext.b a0, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: sexti1_i32_setcc:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    srli a0, a0, 31
+; RV32XQCIBM-NEXT:    addi a0, a0, -1
+; RV32XQCIBM-NEXT:    qc.extu a0, a0, 8, 0
+; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti1_i32_setcc:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    srli a0, a0, 31
+; RV32XQCIBMZBB-NEXT:    addi a0, a0, -1
+; RV32XQCIBMZBB-NEXT:    qc.extu a0, a0, 8, 0
+; RV32XQCIBMZBB-NEXT:    ret
+  %icmp = icmp sgt i32 %a, -1
+  %sext = sext i1 %icmp to i8
+  ret i8 %sext
+}
+
+; Make sure we don't use seqz+qc.ext instead of snez+addi
+define i32 @sexti1_i32_setcc_2(i32 %a, i32 %b) {
+; CHECK-LABEL: sexti1_i32_setcc_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xor a0, a0, a1
+; CHECK-NEXT:    snez a0, a0
+; CHECK-NEXT:    addi a0, a0, -1
+; CHECK-NEXT:    ret
+  %icmp = icmp eq i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
+
+; Make sure we don't use qc.ext instead of neg.
+define i32 @sexti1_i32_setcc_3(i32 %a, i32 %b) {
+; CHECK-LABEL: sexti1_i32_setcc_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    slt a0, a0, a1
+; CHECK-NEXT:    neg a0, a0
+; CHECK-NEXT:    ret
+  %icmp = icmp slt i32 %a, %b
+  %sext = sext i1 %icmp to i32
+  ret i32 %sext
+}
 
 define i32 @sexti8_i32(i8 %a) nounwind {
 ; RV32I-LABEL: sexti8_i32:
@@ -272,40 +323,20 @@ define i64 @sexti16_i64_2(i16 %a) {
 }
 
 define i64 @sexti32_i64(i64 %a) {
-; RV32I-LABEL: sexti32_i64:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    srai a1, a0, 31
-; RV32I-NEXT:    ret
-;
-; RV32XQCIBM-LABEL: sexti32_i64:
-; RV32XQCIBM:       # %bb.0:
-; RV32XQCIBM-NEXT:    srai a1, a0, 31
-; RV32XQCIBM-NEXT:    ret
-;
-; RV32XQCIBMZBB-LABEL: sexti32_i64:
-; RV32XQCIBMZBB:       # %bb.0:
-; RV32XQCIBMZBB-NEXT:    srai a1, a0, 31
-; RV32XQCIBMZBB-NEXT:    ret
+; CHECK-LABEL: sexti32_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srai a1, a0, 31
+; CHECK-NEXT:    ret
   %shl = shl i64 %a, 32
   %shr = ashr exact i64 %shl, 32
   ret i64 %shr
 }
 
 define i64 @sexti32_i64_2(i32 %a) {
-; RV32I-LABEL: sexti32_i64_2:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    srai a1, a0, 31
-; RV32I-NEXT:    ret
-;
-; RV32XQCIBM-LABEL: sexti32_i64_2:
-; RV32XQCIBM:       # %bb.0:
-; RV32XQCIBM-NEXT:    srai a1, a0, 31
-; RV32XQCIBM-NEXT:    ret
-;
-; RV32XQCIBMZBB-LABEL: sexti32_i64_2:
-; RV32XQCIBMZBB:       # %bb.0:
-; RV32XQCIBMZBB-NEXT:    srai a1, a0, 31
-; RV32XQCIBMZBB-NEXT:    ret
+; CHECK-LABEL: sexti32_i64_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    srai a1, a0, 31
+; CHECK-NEXT:    ret
   %1 = sext i32 %a to i64
   ret i64 %1
 }
@@ -331,20 +362,10 @@ define i32 @extu_from_and_i32(i32 %x) {
 }
 
 define i32 @no_extu_from_and_i32(i32 %x) {
-; RV32I-LABEL: no_extu_from_and_i32:
-; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 31
-; RV32I-NEXT:    ret
-;
-; RV32XQCIBM-LABEL: no_extu_from_and_i32:
-; RV32XQCIBM:       # %bb.0:
-; RV32XQCIBM-NEXT:    andi a0, a0, 31
-; RV32XQCIBM-NEXT:    ret
-;
-; RV32XQCIBMZBB-LABEL: no_extu_from_and_i32:
-; RV32XQCIBMZBB:       # %bb.0:
-; RV32XQCIBMZBB-NEXT:    andi a0, a0, 31
-; RV32XQCIBMZBB-NEXT:    ret
+; CHECK-LABEL: no_extu_from_and_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    andi a0, a0, 31
+; CHECK-NEXT:    ret
   %a = and i32 %x, 31
   ret i32 %a
 }
diff --git a/llvm/test/CodeGen/RISCV/xqcibm-insbi.ll b/llvm/test/CodeGen/RISCV/xqcibm-insbi.ll
new file mode 100644
index 0000000000000..e4a545169d210
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xqcibm-insbi.ll
@@ -0,0 +1,262 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 --verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32I
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcibm --verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32XQCIBM
+
+define i32 @insbi(i32 %in1) nounwind {
+; RV32I-LABEL: insbi:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    xori a1, a0, 176
+; RV32I-NEXT:    andi a1, a1, 496
+; RV32I-NEXT:    xor a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.insbi a0, 11, 5, 4
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 %in1, 176
+  %and1 = and i32 %xor1, 496
+  %xor2 = xor i32 %and1, %in1
+  ret i32 %xor2
+}
+
+define i32 @insbi_comm_xor(i32 %in1) nounwind {
+; RV32I-LABEL: insbi_comm_xor:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 9
+; RV32I-NEXT:    li a2, 15
+; RV32I-NEXT:    slli a1, a1, 9
+; RV32I-NEXT:    xor a1, a0, a1
+; RV32I-NEXT:    slli a2, a2, 9
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    xor a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_comm_xor:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.insbi a0, 9, 4, 9
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 4608, %in1
+  %and1 = and i32 %xor1, 7680
+  %xor2 = xor i32 %and1, %in1
+  ret i32 %xor2
+}
+
+define i32 @insbi_comm_and(i32 %in1) nounwind {
+; RV32I-LABEL: insbi_comm_and:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    li a2, 15
+; RV32I-NEXT:    slli a1, a1, 9
+; RV32I-NEXT:    xor a1, a0, a1
+; RV32I-NEXT:    slli a2, a2, 9
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    xor a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_comm_and:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.insbi a0, 11, 4, 9
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 %in1, 5632
+  %and1 = and i32 7680, %xor1
+  %xor2 = xor i32 %and1, %in1
+  ret i32 %xor2
+}
+
+define i32 @insbi_comm_xor2(i32 %in1) nounwind {
+; RV32I-LABEL: insbi_comm_xor2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    xori a1, a0, 176
+; RV32I-NEXT:    andi a1, a1, 496
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_comm_xor2:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.insbi a0, 11, 5, 4
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 %in1, 176
+  %and1 = and i32 %xor1, 496
+  %xor2 = xor i32 %in1, %and1
+  ret i32 %xor2
+}
+
+define i32 @insbi_immg(i32 %in1) nounwind {
+; RV32I-LABEL: insbi_immg:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    xori a1, a0, 256
+; RV32I-NEXT:    andi a1, a1, 496
+; RV32I-NEXT:    xor a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_immg:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    li a1, 16
+; RV32XQCIBM-NEXT:    qc.insb a0, a1, 5, 4
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 %in1, 256
+  %and1 = and i32 %xor1, 496
+  %xor2 = xor i32 %and1, %in1
+  ret i32 %xor2
+}
+
+define i32 @insbi_not_shifted_mask(i32 %in1) nounwind {
+; RV32I-LABEL: insbi_not_shifted_mask:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    xori a1, a0, 128
+; RV32I-NEXT:    andi a1, a1, 716
+; RV32I-NEXT:    xor a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_not_shifted_mask:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    xori a1, a0, 128
+; RV32XQCIBM-NEXT:    andi a1, a1, 716
+; RV32XQCIBM-NEXT:    xor a0, a0, a1
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 %in1, 176
+  %and1 = and i32 %xor1, 716
+  %xor2 = xor i32 %and1, %in1
+  ret i32 %xor2
+}
+
+define i32 @insbi_width_z(i32 %in1) nounwind {
+; RV32I-LABEL: insbi_width_z:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a1, a0, 256
+; RV32I-NEXT:    xor a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_width_z:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    andi a1, a0, 256
+; RV32XQCIBM-NEXT:    xor a0, a0, a1
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 %in1, 176
+  %and1 = and i32 %xor1, 256
+  %xor2 = xor i32 %and1, %in1
+  ret i32 %xor2
+}
+
+define i32 @insbi_mul_use_and(i32 %in1, i32 %in2) nounwind {
+; RV32I-LABEL: insbi_mul_use_and:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 11
+; RV32I-NEXT:    li a2, 15
+; RV32I-NEXT:    slli a1, a1, 9
+; RV32I-NEXT:    slli a2, a2, 9
+; RV32I-NEXT:    xor a1, a0, a1
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    xor a2, a1, a0
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_mul_use_and:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    li a1, 11
+; RV32XQCIBM-NEXT:    li a2, 15
+; RV32XQCIBM-NEXT:    slli a1, a1, 9
+; RV32XQCIBM-NEXT:    slli a2, a2, 9
+; RV32XQCIBM-NEXT:    xor a1, a1, a0
+; RV32XQCIBM-NEXT:    and a1, a1, a2
+; RV32XQCIBM-NEXT:    xor a2, a1, a0
+; RV32XQCIBM-NEXT:    add a0, a0, a1
+; RV32XQCIBM-NEXT:    add a0, a0, a2
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 %in1, 5632
+  %and1 = and i32 %xor1, 7680
+  %xor2 = xor i32 %and1, %in1
+  %add1 = add i32 %in1, %and1
+  %add2 = add i32 %add1, %xor2
+  ret i32 %add2
+}
+
+define i32 @insbi_mul_use_xor(i32 %in1, i32 %in2) nounwind {
+; RV32I-LABEL: insbi_mul_use_xor:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    xori a1, a0, 176
+; RV32I-NEXT:    andi a2, a1, 496
+; RV32I-NEXT:    xor a2, a2, a0
+; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_mul_use_xor:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    xori a1, a0, 176
+; RV32XQCIBM-NEXT:    andi a2, a1, 496
+; RV32XQCIBM-NEXT:    xor a2, a2, a0
+; RV32XQCIBM-NEXT:    add a0, a0, a1
+; RV32XQCIBM-NEXT:    add a0, a0, a2
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 %in1, 176
+  %and1 = and i32 %xor1, 496
+  %xor2 = xor i32 %and1, %in1
+  %add1 = add i32 %in1, %xor1
+  %add2 = add i32 %add1, %xor2
+  ret i32 %add2
+}
+
+define i32 @insbi_imm_too_neg(i32 %in1) nounwind {
+; RV32I-LABEL: insbi_imm_too_neg:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    xori a1, a0, -34
+; RV32I-NEXT:    andi a1, a1, -2
+; RV32I-NEXT:    xor a0, a1, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_imm_too_neg:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    li a1, -17
+; RV32XQCIBM-NEXT:    qc.insb a0, a1, 31, 1
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i32 %in1, -34
+  %and1 = and i32 %xor1, -2
+  %xor2 = xor i32 %and1, %in1
+  ret i32 %xor2
+}
+
+define i64 @insbi_i64(i64 %in1) nounwind {
+; RV32I-LABEL: insbi_i64:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 57344
+; RV32I-NEXT:    lui a3, 1044480
+; RV32I-NEXT:    xor a2, a0, a2
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    zext.b a3, a1
+; RV32I-NEXT:    xor a1, a3, a1
+; RV32I-NEXT:    xor a0, a2, a0
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_i64:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.extu a2, a1, 8, 0
+; RV32XQCIBM-NEXT:    xor a1, a1, a2
+; RV32XQCIBM-NEXT:    qc.insbi a0, 14, 8, 24
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i64 %in1, 234881024
+  %and1 = and i64 %xor1, 1099494850560
+  %xor2 = xor i64 %and1, %in1
+  ret i64 %xor2
+}
+define i64 @insbi_i64_large_mask(i64 %in1) nounwind {
+; RV32I-LABEL: insbi_i64_large_mask:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    xori a2, a1, 9
+; RV32I-NEXT:    andi a2, a2, 15
+; RV32I-NEXT:    xor a1, a2, a1
+; RV32I-NEXT:    ret
+;
+; RV32XQCIBM-LABEL: insbi_i64_large_mask:
+; RV32XQCIBM:       # %bb.0:
+; RV32XQCIBM-NEXT:    qc.insbi a1, 9, 4, 0
+; RV32XQCIBM-NEXT:    ret
+  %xor1 = xor i64 %in1, 38654705664
+  %and1 = and i64 %xor1, 64424509440
+  %xor2 = xor i64 %and1, %in1
+  ret i64 %xor2
+}
diff --git a/llvm/test/CodeGen/RISCV/xqcibm-insert.ll b/llvm/test/CodeGen/RISCV/xqcibm-insert.ll
index 88054a691bad1..2a954ae1eb297 100644
--- a/llvm/test/CodeGen/RISCV/xqcibm-insert.ll
+++ b/llvm/test/CodeGen/RISCV/xqcibm-insert.ll
@@ -139,3 +139,208 @@ define i32 @test_single_bit_set(i32 %a) nounwind {
   %or = or i32 %a, 4096
   ret i32 %or
 }
+
+
+; Tests for INSB(I) generation from OR and AND
+
+define i32 @test1(i32 %a) {
+; RV32I-LABEL: test1:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, -16
+; RV32I-NEXT:    addi a0, a0, 5
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test1:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    qc.insbi a0, 5, 4, 0
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test1:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    qc.insbi a0, 5, 4, 0
+; RV32IXQCIBMZBS-NEXT:    ret
+  %1 = and i32 %a, -16 ; 0xfffffff0
+  %2 = or i32 %1, 5    ; 0x00000005
+  ret i32 %2
+}
+
+define i32 @test2(i32 %a) {
+; RV32I-LABEL: test2:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1033216
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    lui a1, 10240
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test2:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    qc.insbi a0, 10, 4, 22
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test2:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    qc.insbi a0, 10, 4, 22
+; RV32IXQCIBMZBS-NEXT:    ret
+  %1 = and i32 %a, -62914561 ; 0xfc3fffff
+  %2 = or i32 %1, 41943040   ; 0x02800000
+  ret i32 %2
+}
+
+define i64 @test3(i64 %a) {
+; RV32I-LABEL: test3:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, -8
+; RV32I-NEXT:    addi a0, a0, 5
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test3:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    qc.insbi a0, 5, 3, 0
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test3:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    qc.insbi a0, 5, 3, 0
+; RV32IXQCIBMZBS-NEXT:    ret
+  %1 = and i64 %a, -8 ; 0xfffffffffffffff8
+  %2 = or i64 %1, 5   ; 0x0000000000000005
+  ret i64 %2
+}
+
+define i64 @test4(i64 %a) {
+; RV32I-LABEL: test4:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, -255
+; RV32I-NEXT:    addi a0, a0, 18
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test4:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    qc.insbi a0, 9, 7, 1
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test4:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    qc.insbi a0, 9, 7, 1
+; RV32IXQCIBMZBS-NEXT:    ret
+  %1 = and i64 %a, -255 ; 0xffffffffffffff01
+  %2 = or i64 %1,  18   ; 0x0000000000000012
+  ret i64 %2
+}
+
+define i32 @test5(i32 %a) {
+; RV32I-LABEL: test5:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    andi a0, a0, -16
+; RV32I-NEXT:    addi a0, a0, 6
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test5:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    qc.insbi a0, 6, 4, 0
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test5:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    qc.insbi a0, 6, 4, 0
+; RV32IXQCIBMZBS-NEXT:    ret
+  %1 = and i32 %a, 4294967280 ; 0xfffffff0
+  %2 = or i32 %1, 6           ; 0x00000006
+  ret i32 %2
+}
+
+define i32 @test6(i32 %a) {
+; RV32I-LABEL: test6:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1048320
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    lui a1, 182
+; RV32I-NEXT:    addi a1, a1, -1326
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test6:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    lui a1, 182
+; RV32IXQCIBM-NEXT:    addi a1, a1, -1326
+; RV32IXQCIBM-NEXT:    qc.insb a0, a1, 20, 0
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test6:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    lui a1, 182
+; RV32IXQCIBMZBS-NEXT:    addi a1, a1, -1326
+; RV32IXQCIBMZBS-NEXT:    qc.insb a0, a1, 20, 0
+; RV32IXQCIBMZBS-NEXT:    ret
+  %1 = and i32 %a, 4293918720 ; 0xfff00000
+  %2 = or i32 %1, 744146      ; 0x000b5ad2
+  ret i32 %2
+}
+
+define i32 @test7(i32 %a) {
+; RV32I-LABEL: test7:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 1048320
+; RV32I-NEXT:    addi a1, a1, 1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    lui a1, 182
+; RV32I-NEXT:    addi a1, a1, -1326
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test7:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    lui a1, 91
+; RV32IXQCIBM-NEXT:    addi a1, a1, -663
+; RV32IXQCIBM-NEXT:    qc.insb a0, a1, 19, 1
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test7:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    lui a1, 91
+; RV32IXQCIBMZBS-NEXT:    addi a1, a1, -663
+; RV32IXQCIBMZBS-NEXT:    qc.insb a0, a1, 19, 1
+; RV32IXQCIBMZBS-NEXT:    ret
+  %1 = and i32 %a, 4293918721 ; 0xfff00001
+  %2 = or i32 %1, 744146      ; 0x000b5ad2
+  ret i32 %2
+}
+
+define i64 @test8(i64 %a) {
+; RV32I-LABEL: test8:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 1044480
+; RV32I-NEXT:    zext.b a0, a0
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    lui a2, 496944
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    lui a2, 9
+; RV32I-NEXT:    addi a2, a2, -170
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV32IXQCIBM-LABEL: test8:
+; RV32IXQCIBM:       # %bb.0:
+; RV32IXQCIBM-NEXT:    lui a2, 1941
+; RV32IXQCIBM-NEXT:    addi a2, a2, 768
+; RV32IXQCIBM-NEXT:    qc.insb a0, a2, 24, 8
+; RV32IXQCIBM-NEXT:    lui a2, 9
+; RV32IXQCIBM-NEXT:    addi a2, a2, -170
+; RV32IXQCIBM-NEXT:    qc.insb a1, a2, 24, 0
+; RV32IXQCIBM-NEXT:    ret
+;
+; RV32IXQCIBMZBS-LABEL: test8:
+; RV32IXQCIBMZBS:       # %bb.0:
+; RV32IXQCIBMZBS-NEXT:    lui a2, 1941
+; RV32IXQCIBMZBS-NEXT:    addi a2, a2, 768
+; RV32IXQCIBMZBS-NEXT:    qc.insb a0, a2, 24, 8
+; RV32IXQCIBMZBS-NEXT:    lui a2, 9
+; RV32IXQCIBMZBS-NEXT:    addi a2, a2, -170
+; RV32IXQCIBMZBS-NEXT:    qc.insb a1, a2, 24, 0
+; RV32IXQCIBMZBS-NEXT:    ret
+  %1 = and i64 %a, -72057594037927681 ; 0xff000000000000ff
+  %2 = or i64 %1, 157601565442048     ; 0x00008f5679530000
+  ret i64 %2
+}
diff --git a/llvm/test/CodeGen/RISCV/zbb-cmp-combine.ll b/llvm/test/CodeGen/RISCV/zbb-cmp-combine.ll
index 2d48f2b49822b..17b35343d0791 100644
--- a/llvm/test/CodeGen/RISCV/zbb-cmp-combine.ll
+++ b/llvm/test/CodeGen/RISCV/zbb-cmp-combine.ll
@@ -224,7 +224,7 @@ define i1 @flo(float %c, float %a, float %b) {
 ; CHECK-RV64I-NEXT:    mv a1, s1
 ; CHECK-RV64I-NEXT:    call __gesf2
 ; CHECK-RV64I-NEXT:    or a0, s2, a0
-; CHECK-RV64I-NEXT:    slti a0, a0, 0
+; CHECK-RV64I-NEXT:    srli a0, a0, 63
 ; CHECK-RV64I-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-RV64I-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; CHECK-RV64I-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
@@ -275,7 +275,7 @@ define i1 @dlo(double %c, double %a, double %b) {
 ; CHECK-NEXT:    mv a1, s1
 ; CHECK-NEXT:    call __gedf2
 ; CHECK-NEXT:    or a0, s2, a0
-; CHECK-NEXT:    slti a0, a0, 0
+; CHECK-NEXT:    srli a0, a0, 63
 ; CHECK-NEXT:    ld ra, 24(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld s0, 16(sp) # 8-byte Folded Reload
 ; CHECK-NEXT:    ld s1, 8(sp) # 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/RISCV/zcmp-prolog-epilog-crash.mir b/llvm/test/CodeGen/RISCV/zcmp-prolog-epilog-crash.mir
index 6e5ac31f192de..979b8a4e17d5a 100644
--- a/llvm/test/CodeGen/RISCV/zcmp-prolog-epilog-crash.mir
+++ b/llvm/test/CodeGen/RISCV/zcmp-prolog-epilog-crash.mir
@@ -38,8 +38,10 @@ liveins:
 frameInfo:
   maxAlignment:    1
   localFrameSize:  32
-  savePoint:       '%bb.2'
-  restorePoint:    '%bb.2'
+  savePoint:
+    - point:           '%bb.2'
+  restorePoint:
+    - point:           '%bb.2'
 stack:
   - { id: 0, size: 32, alignment: 1, local-offset: -32 }
 machineFunctionInfo:
diff --git a/llvm/test/CodeGen/SPARC/float-ua2007.ll b/llvm/test/CodeGen/SPARC/float-ua2007.ll
new file mode 100644
index 0000000000000..252b47943fe43
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/float-ua2007.ll
@@ -0,0 +1,275 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=sparc64 --fp-contract=fast -mattr=-ua2007 < %s | FileCheck %s -check-prefix=NO-UA2007
+; RUN: llc -mtriple=sparc64 --fp-contract=fast -mattr=+ua2007 < %s | FileCheck %s -check-prefix=UA2007
+
+define float @fmadds(float %a, float %b, float %c) nounwind {
+; NO-UA2007-LABEL: fmadds:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuls %f1, %f3, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fadds %f0, %f5, %f0
+;
+; UA2007-LABEL: fmadds:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fmadds %f1, %f3, %f5, %f0
+  %ret = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
+  ret float %ret
+}
+
+define double @fmaddd(double %a, double %b, double %c) nounwind {
+; NO-UA2007-LABEL: fmaddd:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuld %f0, %f2, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    faddd %f0, %f4, %f0
+;
+; UA2007-LABEL: fmaddd:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fmaddd %f0, %f2, %f4, %f0
+  %ret = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
+  ret double %ret
+}
+
+define float @fmsubs(float %a, float %b, float %c) nounwind {
+; NO-UA2007-LABEL: fmsubs:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuls %f1, %f3, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fsubs %f0, %f5, %f0
+;
+; UA2007-LABEL: fmsubs:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fmsubs %f1, %f3, %f5, %f0
+  %neg = fneg float %c
+  %ret = call float @llvm.fmuladd.f32(float %a, float %b, float %neg)
+  ret float %ret
+}
+
+define double @fmsubd(double %a, double %b, double %c) nounwind {
+; NO-UA2007-LABEL: fmsubd:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuld %f0, %f2, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fsubd %f0, %f4, %f0
+;
+; UA2007-LABEL: fmsubd:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fmsubd %f0, %f2, %f4, %f0
+  %neg = fneg double %c
+  %ret = call double @llvm.fmuladd.f64(double %a, double %b, double %neg)
+  ret double %ret
+}
+
+define float @fnmadds(float %a, float %b, float %c) nounwind {
+; NO-UA2007-LABEL: fnmadds:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuls %f1, %f3, %f0
+; NO-UA2007-NEXT:    fadds %f0, %f5, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fnegs %f0, %f0
+;
+; UA2007-LABEL: fnmadds:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fnmadds %f1, %f3, %f5, %f0
+  %fma = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
+  %ret = fneg float %fma
+  ret float %ret
+}
+
+define double @fnmaddd(double %a, double %b, double %c) nounwind {
+; NO-UA2007-LABEL: fnmaddd:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuld %f0, %f2, %f0
+; NO-UA2007-NEXT:    faddd %f0, %f4, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fnegd %f0, %f0
+;
+; UA2007-LABEL: fnmaddd:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fnmaddd %f0, %f2, %f4, %f0
+  %fma = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
+  %ret = fneg double %fma
+  ret double %ret
+}
+
+define float @fnmsubs(float %a, float %b, float %c) nounwind {
+; NO-UA2007-LABEL: fnmsubs:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuls %f1, %f3, %f0
+; NO-UA2007-NEXT:    fsubs %f0, %f5, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fnegs %f0, %f0
+;
+; UA2007-LABEL: fnmsubs:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fnmsubs %f1, %f3, %f5, %f0
+  %neg = fneg float %c
+  %fma = call float @llvm.fmuladd.f32(float %a, float %b, float %neg)
+  %ret = fneg float %fma
+  ret float %ret
+}
+
+define double @fnmsubd(double %a, double %b, double %c) nounwind {
+; NO-UA2007-LABEL: fnmsubd:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuld %f0, %f2, %f0
+; NO-UA2007-NEXT:    fsubd %f0, %f4, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fnegd %f0, %f0
+;
+; UA2007-LABEL: fnmsubd:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fnmsubd %f0, %f2, %f4, %f0
+  %neg = fneg double %c
+  %fma = call double @llvm.fmuladd.f64(double %a, double %b, double %neg)
+  %ret = fneg double %fma
+  ret double %ret
+}
+
+
+define float @combine_madds(float %a, float %b, float %c) nounwind {
+; NO-UA2007-LABEL: combine_madds:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuls %f1, %f3, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fadds %f0, %f5, %f0
+;
+; UA2007-LABEL: combine_madds:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fmadds %f1, %f3, %f5, %f0
+  %mul = fmul float %a, %b
+  %add = fadd float %mul, %c
+  ret float %add
+}
+
+define double @combine_maddd(double %a, double %b, double %c) nounwind {
+; NO-UA2007-LABEL: combine_maddd:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuld %f0, %f2, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    faddd %f0, %f4, %f0
+;
+; UA2007-LABEL: combine_maddd:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fmaddd %f0, %f2, %f4, %f0
+  %mul = fmul double %a, %b
+  %add = fadd double %mul, %c
+  ret double %add
+}
+
+define float @combine_msubs(float %a, float %b, float %c) nounwind {
+; NO-UA2007-LABEL: combine_msubs:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuls %f1, %f3, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fsubs %f0, %f5, %f0
+;
+; UA2007-LABEL: combine_msubs:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fmsubs %f1, %f3, %f5, %f0
+  %mul = fmul float %a, %b
+  %sub = fsub float %mul, %c
+  ret float %sub
+}
+
+define double @combine_msubd(double %a, double %b, double %c) nounwind {
+; NO-UA2007-LABEL: combine_msubd:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuld %f0, %f2, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fsubd %f0, %f4, %f0
+;
+; UA2007-LABEL: combine_msubd:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fmsubd %f0, %f2, %f4, %f0
+  %mul = fmul double %a, %b
+  %sub = fsub double %mul, %c
+  ret double %sub
+}
+
+define float @combine_nmadds(float %a, float %b, float %c) nounwind {
+; NO-UA2007-LABEL: combine_nmadds:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuls %f1, %f3, %f0
+; NO-UA2007-NEXT:    fadds %f0, %f5, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fnegs %f0, %f0
+;
+; UA2007-LABEL: combine_nmadds:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fnmadds %f1, %f3, %f5, %f0
+  %mul = fmul float %a, %b
+  %add = fadd float %mul, %c
+  %neg = fneg float %add
+  ret float %neg
+}
+
+define double @combine_nmaddd(double %a, double %b, double %c) nounwind {
+; NO-UA2007-LABEL: combine_nmaddd:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuld %f0, %f2, %f0
+; NO-UA2007-NEXT:    faddd %f0, %f4, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fnegd %f0, %f0
+;
+; UA2007-LABEL: combine_nmaddd:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fnmaddd %f0, %f2, %f4, %f0
+  %mul = fmul double %a, %b
+  %add = fadd double %mul, %c
+  %neg = fneg double %add
+  ret double %neg
+}
+
+define float @combine_nmsubs(float %a, float %b, float %c) nounwind {
+; NO-UA2007-LABEL: combine_nmsubs:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuls %f1, %f3, %f0
+; NO-UA2007-NEXT:    fsubs %f0, %f5, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fnegs %f0, %f0
+;
+; UA2007-LABEL: combine_nmsubs:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fnmsubs %f1, %f3, %f5, %f0
+  %mul = fmul float %a, %b
+  %sub = fsub float %mul, %c
+  %neg = fneg float %sub
+  ret float %neg
+}
+
+define double @combine_nmsubd(double %a, double %b, double %c) nounwind {
+; NO-UA2007-LABEL: combine_nmsubd:
+; NO-UA2007:       ! %bb.0:
+; NO-UA2007-NEXT:    fmuld %f0, %f2, %f0
+; NO-UA2007-NEXT:    fsubd %f0, %f4, %f0
+; NO-UA2007-NEXT:    retl
+; NO-UA2007-NEXT:    fnegd %f0, %f0
+;
+; UA2007-LABEL: combine_nmsubd:
+; UA2007:       ! %bb.0:
+; UA2007-NEXT:    retl
+; UA2007-NEXT:    fnmsubd %f0, %f2, %f4, %f0
+  %mul = fmul double %a, %b
+  %sub = fsub double %mul, %c
+  %neg = fneg double %sub
+  ret double %neg
+}
+
+declare float @llvm.fmuladd.f32(float, float, float)
+declare double @llvm.fmuladd.f64(double, double, double)
diff --git a/llvm/test/CodeGen/SPARC/fp16-promote.ll b/llvm/test/CodeGen/SPARC/fp16-promote.ll
index efe67b04e8fb3..64873b744de50 100644
--- a/llvm/test/CodeGen/SPARC/fp16-promote.ll
+++ b/llvm/test/CodeGen/SPARC/fp16-promote.ll
@@ -329,13 +329,14 @@ define void @test_fadd(ptr %p, ptr %q) nounwind {
 ; V8-OPT-LABEL: test_fadd:
 ; V8-OPT:       ! %bb.0:
 ; V8-OPT-NEXT:    save %sp, -104, %sp
+; V8-OPT-NEXT:    lduh [%i0], %i2
 ; V8-OPT-NEXT:    call __extendhfsf2
-; V8-OPT-NEXT:    lduh [%i0], %o0
+; V8-OPT-NEXT:    lduh [%i1], %o0
 ; V8-OPT-NEXT:    st %f0, [%fp+-8] ! 4-byte Folded Spill
 ; V8-OPT-NEXT:    call __extendhfsf2
-; V8-OPT-NEXT:    lduh [%i1], %o0
+; V8-OPT-NEXT:    mov %i2, %o0
 ; V8-OPT-NEXT:    ld [%fp+-8], %f1 ! 4-byte Folded Reload
-; V8-OPT-NEXT:    fadds %f1, %f0, %f0
+; V8-OPT-NEXT:    fadds %f0, %f1, %f0
 ; V8-OPT-NEXT:    st %f0, [%fp+-4]
 ; V8-OPT-NEXT:    call __truncsfhf2
 ; V8-OPT-NEXT:    ld [%fp+-4], %o0
@@ -346,13 +347,14 @@ define void @test_fadd(ptr %p, ptr %q) nounwind {
 ; V8-UNOPT-LABEL: test_fadd:
 ; V8-UNOPT:       ! %bb.0:
 ; V8-UNOPT-NEXT:    save %sp, -104, %sp
-; V8-UNOPT-NEXT:    call __extendhfsf2
-; V8-UNOPT-NEXT:    lduh [%i0], %o0
-; V8-UNOPT-NEXT:    st %f0, [%fp+-8] ! 4-byte Folded Spill
+; V8-UNOPT-NEXT:    lduh [%i0], %i2
+; V8-UNOPT-NEXT:    st %i2, [%fp+-12] ! 4-byte Folded Spill
 ; V8-UNOPT-NEXT:    call __extendhfsf2
 ; V8-UNOPT-NEXT:    lduh [%i1], %o0
-; V8-UNOPT-NEXT:    fmovs %f0, %f1
-; V8-UNOPT-NEXT:    ld [%fp+-8], %f0 ! 4-byte Folded Reload
+; V8-UNOPT-NEXT:    ld [%fp+-12], %o0 ! 4-byte Folded Reload
+; V8-UNOPT-NEXT:    call __extendhfsf2
+; V8-UNOPT-NEXT:    st %f0, [%fp+-8]
+; V8-UNOPT-NEXT:    ld [%fp+-8], %f1 ! 4-byte Folded Reload
 ; V8-UNOPT-NEXT:    fadds %f0, %f1, %f0
 ; V8-UNOPT-NEXT:    st %f0, [%fp+-4]
 ; V8-UNOPT-NEXT:    call __truncsfhf2
@@ -364,13 +366,14 @@ define void @test_fadd(ptr %p, ptr %q) nounwind {
 ; V9-LABEL: test_fadd:
 ; V9:       ! %bb.0:
 ; V9-NEXT:    save %sp, -104, %sp
+; V9-NEXT:    lduh [%i0], %i2
 ; V9-NEXT:    call __extendhfsf2
-; V9-NEXT:    lduh [%i0], %o0
+; V9-NEXT:    lduh [%i1], %o0
 ; V9-NEXT:    st %f0, [%fp+-8] ! 4-byte Folded Spill
 ; V9-NEXT:    call __extendhfsf2
-; V9-NEXT:    lduh [%i1], %o0
+; V9-NEXT:    mov %i2, %o0
 ; V9-NEXT:    ld [%fp+-8], %f1 ! 4-byte Folded Reload
-; V9-NEXT:    fadds %f1, %f0, %f0
+; V9-NEXT:    fadds %f0, %f1, %f0
 ; V9-NEXT:    st %f0, [%fp+-4]
 ; V9-NEXT:    call __truncsfhf2
 ; V9-NEXT:    ld [%fp+-4], %o0
@@ -381,14 +384,15 @@ define void @test_fadd(ptr %p, ptr %q) nounwind {
 ; SPARC64-LABEL: test_fadd:
 ; SPARC64:       ! %bb.0:
 ; SPARC64-NEXT:    save %sp, -192, %sp
+; SPARC64-NEXT:    lduh [%i0], %i2
 ; SPARC64-NEXT:    call __extendhfsf2
-; SPARC64-NEXT:    lduh [%i0], %o0
+; SPARC64-NEXT:    lduh [%i1], %o0
 ; SPARC64-NEXT:    st %f0, [%fp+2043] ! 4-byte Folded Spill
 ; SPARC64-NEXT:    call __extendhfsf2
-; SPARC64-NEXT:    lduh [%i1], %o0
+; SPARC64-NEXT:    mov %i2, %o0
 ; SPARC64-NEXT:    ld [%fp+2043], %f1 ! 4-byte Folded Reload
 ; SPARC64-NEXT:    call __truncsfhf2
-; SPARC64-NEXT:    fadds %f1, %f0, %f1
+; SPARC64-NEXT:    fadds %f0, %f1, %f1
 ; SPARC64-NEXT:    sth %o0, [%i0]
 ; SPARC64-NEXT:    ret
 ; SPARC64-NEXT:    restore
@@ -403,13 +407,14 @@ define void @test_fmul(ptr %p, ptr %q) nounwind {
 ; V8-OPT-LABEL: test_fmul:
 ; V8-OPT:       ! %bb.0:
 ; V8-OPT-NEXT:    save %sp, -104, %sp
+; V8-OPT-NEXT:    lduh [%i0], %i2
 ; V8-OPT-NEXT:    call __extendhfsf2
-; V8-OPT-NEXT:    lduh [%i0], %o0
+; V8-OPT-NEXT:    lduh [%i1], %o0
 ; V8-OPT-NEXT:    st %f0, [%fp+-8] ! 4-byte Folded Spill
 ; V8-OPT-NEXT:    call __extendhfsf2
-; V8-OPT-NEXT:    lduh [%i1], %o0
+; V8-OPT-NEXT:    mov %i2, %o0
 ; V8-OPT-NEXT:    ld [%fp+-8], %f1 ! 4-byte Folded Reload
-; V8-OPT-NEXT:    fmuls %f1, %f0, %f0
+; V8-OPT-NEXT:    fmuls %f0, %f1, %f0
 ; V8-OPT-NEXT:    st %f0, [%fp+-4]
 ; V8-OPT-NEXT:    call __truncsfhf2
 ; V8-OPT-NEXT:    ld [%fp+-4], %o0
@@ -420,13 +425,14 @@ define void @test_fmul(ptr %p, ptr %q) nounwind {
 ; V8-UNOPT-LABEL: test_fmul:
 ; V8-UNOPT:       ! %bb.0:
 ; V8-UNOPT-NEXT:    save %sp, -104, %sp
-; V8-UNOPT-NEXT:    call __extendhfsf2
-; V8-UNOPT-NEXT:    lduh [%i0], %o0
-; V8-UNOPT-NEXT:    st %f0, [%fp+-8] ! 4-byte Folded Spill
+; V8-UNOPT-NEXT:    lduh [%i0], %i2
+; V8-UNOPT-NEXT:    st %i2, [%fp+-12] ! 4-byte Folded Spill
 ; V8-UNOPT-NEXT:    call __extendhfsf2
 ; V8-UNOPT-NEXT:    lduh [%i1], %o0
-; V8-UNOPT-NEXT:    fmovs %f0, %f1
-; V8-UNOPT-NEXT:    ld [%fp+-8], %f0 ! 4-byte Folded Reload
+; V8-UNOPT-NEXT:    ld [%fp+-12], %o0 ! 4-byte Folded Reload
+; V8-UNOPT-NEXT:    call __extendhfsf2
+; V8-UNOPT-NEXT:    st %f0, [%fp+-8]
+; V8-UNOPT-NEXT:    ld [%fp+-8], %f1 ! 4-byte Folded Reload
 ; V8-UNOPT-NEXT:    fmuls %f0, %f1, %f0
 ; V8-UNOPT-NEXT:    st %f0, [%fp+-4]
 ; V8-UNOPT-NEXT:    call __truncsfhf2
@@ -438,13 +444,14 @@ define void @test_fmul(ptr %p, ptr %q) nounwind {
 ; V9-LABEL: test_fmul:
 ; V9:       ! %bb.0:
 ; V9-NEXT:    save %sp, -104, %sp
+; V9-NEXT:    lduh [%i0], %i2
 ; V9-NEXT:    call __extendhfsf2
-; V9-NEXT:    lduh [%i0], %o0
+; V9-NEXT:    lduh [%i1], %o0
 ; V9-NEXT:    st %f0, [%fp+-8] ! 4-byte Folded Spill
 ; V9-NEXT:    call __extendhfsf2
-; V9-NEXT:    lduh [%i1], %o0
+; V9-NEXT:    mov %i2, %o0
 ; V9-NEXT:    ld [%fp+-8], %f1 ! 4-byte Folded Reload
-; V9-NEXT:    fmuls %f1, %f0, %f0
+; V9-NEXT:    fmuls %f0, %f1, %f0
 ; V9-NEXT:    st %f0, [%fp+-4]
 ; V9-NEXT:    call __truncsfhf2
 ; V9-NEXT:    ld [%fp+-4], %o0
@@ -455,14 +462,15 @@ define void @test_fmul(ptr %p, ptr %q) nounwind {
 ; SPARC64-LABEL: test_fmul:
 ; SPARC64:       ! %bb.0:
 ; SPARC64-NEXT:    save %sp, -192, %sp
+; SPARC64-NEXT:    lduh [%i0], %i2
 ; SPARC64-NEXT:    call __extendhfsf2
-; SPARC64-NEXT:    lduh [%i0], %o0
+; SPARC64-NEXT:    lduh [%i1], %o0
 ; SPARC64-NEXT:    st %f0, [%fp+2043] ! 4-byte Folded Spill
 ; SPARC64-NEXT:    call __extendhfsf2
-; SPARC64-NEXT:    lduh [%i1], %o0
+; SPARC64-NEXT:    mov %i2, %o0
 ; SPARC64-NEXT:    ld [%fp+2043], %f1 ! 4-byte Folded Reload
 ; SPARC64-NEXT:    call __truncsfhf2
-; SPARC64-NEXT:    fmuls %f1, %f0, %f1
+; SPARC64-NEXT:    fmuls %f0, %f1, %f1
 ; SPARC64-NEXT:    sth %o0, [%i0]
 ; SPARC64-NEXT:    ret
 ; SPARC64-NEXT:    restore
diff --git a/llvm/test/CodeGen/SPARC/half.ll b/llvm/test/CodeGen/SPARC/half.ll
new file mode 100644
index 0000000000000..565160149e715
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/half.ll
@@ -0,0 +1,777 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; RUN: llc %s -o - -mtriple=sparc-linux-gnu           | FileCheck %s -check-prefixes=CHECK,SPARC32,V8
+; RUN: llc %s -o - -mtriple=sparcel-linux-gnu         | FileCheck %s -check-prefixes=CHECK,SPARC32,V8
+; RUN: llc %s -o - -mtriple=sparc-linux-gnu -mattr=v9 | FileCheck %s -check-prefixes=CHECK,SPARC32,V9
+; RUN: llc %s -o - -mtriple=sparc64-unknown-linux     | FileCheck %s -check-prefixes=CHECK,SPARC64
+
+; Tests for various operations on half precison float. Much of the test is
+; copied from test/CodeGen/X86/half.ll.
+
+define void @store(half %x, ptr %p) nounwind {
+; CHECK-LABEL: store:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    sth %o0, [%o1]
+  store half %x, ptr %p
+  ret void
+}
+
+define half @return(ptr %p) nounwind {
+; CHECK-LABEL: return:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    lduh [%o0], %o0
+  %r = load half, ptr %p
+  ret half %r
+}
+
+define dso_local double @loadd(ptr nocapture readonly %a) local_unnamed_addr nounwind {
+; SPARC32-LABEL: loadd:
+; SPARC32:       ! %bb.0: ! %entry
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0+2], %o0
+; SPARC32-NEXT:    fstod %f0, %f0
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: loadd:
+; SPARC64:       ! %bb.0: ! %entry
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0+2], %o0
+; SPARC64-NEXT:    fstod %f0, %f0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+entry:
+  %arrayidx = getelementptr inbounds i16, ptr %a, i64 1
+  %0 = load i16, ptr %arrayidx, align 2
+  %1 = tail call double @llvm.convert.from.fp16.f64(i16 %0)
+  ret double %1
+}
+
+define dso_local float @loadf(ptr nocapture readonly %a) local_unnamed_addr nounwind {
+; SPARC32-LABEL: loadf:
+; SPARC32:       ! %bb.0: ! %entry
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0+2], %o0
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: loadf:
+; SPARC64:       ! %bb.0: ! %entry
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0+2], %o0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+entry:
+  %arrayidx = getelementptr inbounds i16, ptr %a, i64 1
+  %0 = load i16, ptr %arrayidx, align 2
+  %1 = tail call float @llvm.convert.from.fp16.f32(i16 %0)
+  ret float %1
+}
+
+define dso_local void @stored(ptr nocapture %a, double %b) local_unnamed_addr nounwind {
+; SPARC32-LABEL: stored:
+; SPARC32:       ! %bb.0: ! %entry
+; SPARC32-NEXT:    save %sp, -112, %sp
+; SPARC32-NEXT:    mov %i2, %i3
+; SPARC32-NEXT:    mov %i1, %i2
+; SPARC32-NEXT:    std %i2, [%fp+-8]
+; SPARC32-NEXT:    ldd [%fp+-8], %f0
+; SPARC32-NEXT:    std %f0, [%fp+-16]
+; SPARC32-NEXT:    call __truncdfhf2
+; SPARC32-NEXT:    ldd [%fp+-16], %o0
+; SPARC32-NEXT:    sth %o0, [%i0]
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: stored:
+; SPARC64:       ! %bb.0: ! %entry
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    fmovd %f2, %f0
+; SPARC64-NEXT:    call __truncdfhf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    sth %o0, [%i0]
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+entry:
+  %0 = tail call i16 @llvm.convert.to.fp16.f64(double %b)
+  store i16 %0, ptr %a, align 2
+  ret void
+}
+
+define dso_local void @storef(ptr nocapture %a, float %b) local_unnamed_addr nounwind {
+; SPARC32-LABEL: storef:
+; SPARC32:       ! %bb.0: ! %entry
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    mov %i1, %o0
+; SPARC32-NEXT:    sth %o0, [%i0]
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: storef:
+; SPARC64:       ! %bb.0: ! %entry
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    fmovs %f3, %f1
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    sth %o0, [%i0]
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+entry:
+  %0 = tail call i16 @llvm.convert.to.fp16.f32(float %b)
+  store i16 %0, ptr %a, align 2
+  ret void
+}
+
+define void @test_load_store(ptr %in, ptr %out) nounwind {
+; CHECK-LABEL: test_load_store:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    lduh [%o0], %o0
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    sth %o0, [%o1]
+  %val = load half, ptr %in
+  store half %val, ptr %out
+  ret void
+}
+
+define i16 @test_bitcast_from_half(ptr %addr) nounwind {
+; CHECK-LABEL: test_bitcast_from_half:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    lduh [%o0], %o0
+  %val = load half, ptr %addr
+  %val_int = bitcast half %val to i16
+  ret i16 %val_int
+}
+
+define void @test_bitcast_to_half(ptr %addr, i16 %in) nounwind {
+; CHECK-LABEL: test_bitcast_to_half:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    sth %o1, [%o0]
+  %val_fp = bitcast i16 %in to half
+  store half %val_fp, ptr %addr
+  ret void
+}
+
+define half @from_bits(i16 %x) nounwind {
+; CHECK-LABEL: from_bits:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    nop
+  %res = bitcast i16 %x to half
+  ret half %res
+}
+
+define i16 @to_bits(half %x) nounwind {
+; CHECK-LABEL: to_bits:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    nop
+    %res = bitcast half %x to i16
+    ret i16 %res
+}
+
+define float @test_extend32(ptr %addr) nounwind {
+; SPARC32-LABEL: test_extend32:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0], %o0
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_extend32:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0], %o0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %val16 = load half, ptr %addr
+  %val32 = fpext half %val16 to float
+  ret float %val32
+}
+
+define double @test_extend64(ptr %addr) nounwind {
+; SPARC32-LABEL: test_extend64:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0], %o0
+; SPARC32-NEXT:    fstod %f0, %f0
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_extend64:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0], %o0
+; SPARC64-NEXT:    fstod %f0, %f0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %val16 = load half, ptr %addr
+  %val32 = fpext half %val16 to double
+  ret double %val32
+}
+
+define void @test_trunc32(float %in, ptr %addr) nounwind {
+; SPARC32-LABEL: test_trunc32:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    sth %o0, [%i1]
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_trunc32:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    sth %o0, [%i1]
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %val16 = fptrunc float %in to half
+  store half %val16, ptr %addr
+  ret void
+}
+
+define void @test_trunc64(double %in, ptr %addr) nounwind {
+; SPARC32-LABEL: test_trunc64:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -112, %sp
+; SPARC32-NEXT:    ! kill: def $i1 killed $i1 killed $i0_i1 def $i0_i1
+; SPARC32-NEXT:    ! kill: def $i0 killed $i0 killed $i0_i1 def $i0_i1
+; SPARC32-NEXT:    std %i0, [%fp+-8]
+; SPARC32-NEXT:    ldd [%fp+-8], %f0
+; SPARC32-NEXT:    std %f0, [%fp+-16]
+; SPARC32-NEXT:    call __truncdfhf2
+; SPARC32-NEXT:    ldd [%fp+-16], %o0
+; SPARC32-NEXT:    sth %o0, [%i2]
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_trunc64:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    call __truncdfhf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    sth %o0, [%i1]
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %val16 = fptrunc double %in to half
+  store half %val16, ptr %addr
+  ret void
+}
+
+define i64 @test_fptosi_i64(ptr %p) nounwind {
+; SPARC32-LABEL: test_fptosi_i64:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0], %o0
+; SPARC32-NEXT:    st %f0, [%fp+-4]
+; SPARC32-NEXT:    call __fixsfdi
+; SPARC32-NEXT:    ld [%fp+-4], %o0
+; SPARC32-NEXT:    mov %o0, %i0
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore %g0, %o1, %o1
+;
+; SPARC64-LABEL: test_fptosi_i64:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -192, %sp
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0], %o0
+; SPARC64-NEXT:    fstox %f0, %f0
+; SPARC64-NEXT:    std %f0, [%fp+2039]
+; SPARC64-NEXT:    ldx [%fp+2039], %i0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %a = load half, ptr %p, align 2
+  %r = fptosi half %a to i64
+  ret i64 %r
+}
+
+define void @test_sitofp_i64(i64 %a, ptr %p) nounwind {
+; SPARC32-LABEL: test_sitofp_i64:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    mov %i1, %o1
+; SPARC32-NEXT:    call __floatdisf
+; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    st %f0, [%fp+-4]
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-4], %o0
+; SPARC32-NEXT:    sth %o0, [%i2]
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_sitofp_i64:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -192, %sp
+; SPARC64-NEXT:    stx %i0, [%fp+2039]
+; SPARC64-NEXT:    ldd [%fp+2039], %f0
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    fxtos %f0, %f1
+; SPARC64-NEXT:    sth %o0, [%i1]
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %r = sitofp i64 %a to half
+  store half %r, ptr %p
+  ret void
+}
+
+define i64 @test_fptoui_i64(ptr %p) nounwind {
+; SPARC32-LABEL: test_fptoui_i64:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0], %o0
+; SPARC32-NEXT:    st %f0, [%fp+-4]
+; SPARC32-NEXT:    call __fixunssfdi
+; SPARC32-NEXT:    ld [%fp+-4], %o0
+; SPARC32-NEXT:    mov %o0, %i0
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore %g0, %o1, %o1
+;
+; SPARC64-LABEL: test_fptoui_i64:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -192, %sp
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0], %o0
+; SPARC64-NEXT:    sethi %h44(.LCPI17_0), %i0
+; SPARC64-NEXT:    add %i0, %m44(.LCPI17_0), %i0
+; SPARC64-NEXT:    sllx %i0, 12, %i0
+; SPARC64-NEXT:    ld [%i0+%l44(.LCPI17_0)], %f1
+; SPARC64-NEXT:    fsubs %f0, %f1, %f2
+; SPARC64-NEXT:    fstox %f2, %f2
+; SPARC64-NEXT:    std %f2, [%fp+2031]
+; SPARC64-NEXT:    fstox %f0, %f2
+; SPARC64-NEXT:    std %f2, [%fp+2039]
+; SPARC64-NEXT:    ldx [%fp+2031], %i0
+; SPARC64-NEXT:    sethi 0, %i1
+; SPARC64-NEXT:    or %i1, 0, %i1
+; SPARC64-NEXT:    sethi 2097152, %i2
+; SPARC64-NEXT:    or %i2, 0, %i2
+; SPARC64-NEXT:    sllx %i2, 32, %i2
+; SPARC64-NEXT:    ldx [%fp+2039], %i3
+; SPARC64-NEXT:    or %i2, %i1, %i1
+; SPARC64-NEXT:    xor %i0, %i1, %i0
+; SPARC64-NEXT:    fcmps %fcc0, %f0, %f1
+; SPARC64-NEXT:    movl %fcc0, %i3, %i0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %a = load half, ptr %p, align 2
+  %r = fptoui half %a to i64
+  ret i64 %r
+}
+
+define void @test_uitofp_i64(i64 %a, ptr %p) nounwind {
+; SPARC32-LABEL: test_uitofp_i64:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    mov %i1, %o1
+; SPARC32-NEXT:    call __floatundisf
+; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    st %f0, [%fp+-4]
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-4], %o0
+; SPARC32-NEXT:    sth %o0, [%i2]
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_uitofp_i64:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -192, %sp
+; SPARC64-NEXT:    stx %i0, [%fp+2031]
+; SPARC64-NEXT:    srlx %i0, 1, %i2
+; SPARC64-NEXT:    and %i0, 1, %i3
+; SPARC64-NEXT:    or %i3, %i2, %i2
+; SPARC64-NEXT:    stx %i2, [%fp+2039]
+; SPARC64-NEXT:    ldd [%fp+2031], %f0
+; SPARC64-NEXT:    ldd [%fp+2039], %f2
+; SPARC64-NEXT:    fxtos %f0, %f1
+; SPARC64-NEXT:    fxtos %f2, %f0
+; SPARC64-NEXT:    fadds %f0, %f0, %f0
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    fmovrslz %i0, %f0, %f1
+; SPARC64-NEXT:    sth %o0, [%i1]
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %r = uitofp i64 %a to half
+  store half %r, ptr %p
+  ret void
+}
+
+define <4 x float> @test_extend32_vec4(ptr %p) nounwind {
+; SPARC32-LABEL: test_extend32_vec4:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -104, %sp
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0], %o0
+; SPARC32-NEXT:    st %f0, [%fp+-4] ! 4-byte Folded Spill
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0+2], %o0
+; SPARC32-NEXT:    st %f0, [%fp+-8] ! 4-byte Folded Spill
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0+4], %o0
+; SPARC32-NEXT:    st %f0, [%fp+-12] ! 4-byte Folded Spill
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0+6], %o0
+; SPARC32-NEXT:    fmovs %f0, %f3
+; SPARC32-NEXT:    ld [%fp+-4], %f0 ! 4-byte Folded Reload
+; SPARC32-NEXT:    ld [%fp+-8], %f1 ! 4-byte Folded Reload
+; SPARC32-NEXT:    ld [%fp+-12], %f2 ! 4-byte Folded Reload
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_extend32_vec4:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -192, %sp
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0], %o0
+; SPARC64-NEXT:    st %f0, [%fp+2043] ! 4-byte Folded Spill
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0+2], %o0
+; SPARC64-NEXT:    st %f0, [%fp+2039] ! 4-byte Folded Spill
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0+4], %o0
+; SPARC64-NEXT:    st %f0, [%fp+2035] ! 4-byte Folded Spill
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0+6], %o0
+; SPARC64-NEXT:    fmovs %f0, %f3
+; SPARC64-NEXT:    ld [%fp+2043], %f0 ! 4-byte Folded Reload
+; SPARC64-NEXT:    ld [%fp+2039], %f1 ! 4-byte Folded Reload
+; SPARC64-NEXT:    ld [%fp+2035], %f2 ! 4-byte Folded Reload
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %a = load <4 x half>, ptr %p, align 8
+  %b = fpext <4 x half> %a to <4 x float>
+  ret <4 x float> %b
+}
+
+define <4 x double> @test_extend64_vec4(ptr %p) nounwind {
+; SPARC32-LABEL: test_extend64_vec4:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -120, %sp
+; SPARC32-NEXT:    ld [%fp+64], %i1
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0+4], %o0
+; SPARC32-NEXT:    st %f0, [%fp+-8] ! 4-byte Folded Spill
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0+2], %o0
+; SPARC32-NEXT:    st %f0, [%fp+-16] ! 4-byte Folded Spill
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    lduh [%i0], %o0
+; SPARC32-NEXT:    lduh [%i0+6], %o0
+; SPARC32-NEXT:    fstod %f0, %f0
+; SPARC32-NEXT:    std %f0, [%fp+-24] ! 8-byte Folded Spill
+; SPARC32-NEXT:    ld [%fp+-16], %f0 ! 4-byte Folded Reload
+; SPARC32-NEXT:    fstod %f0, %f0
+; SPARC32-NEXT:    std %f0, [%fp+-16] ! 8-byte Folded Spill
+; SPARC32-NEXT:    ld [%fp+-8], %f0 ! 4-byte Folded Reload
+; SPARC32-NEXT:    fstod %f0, %f0
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    std %f0, [%fp+-8]
+; SPARC32-NEXT:    fstod %f0, %f0
+; SPARC32-NEXT:    std %f0, [%i1+24]
+; SPARC32-NEXT:    ldd [%fp+-8], %f0 ! 8-byte Folded Reload
+; SPARC32-NEXT:    std %f0, [%i1+16]
+; SPARC32-NEXT:    ldd [%fp+-16], %f0 ! 8-byte Folded Reload
+; SPARC32-NEXT:    std %f0, [%i1+8]
+; SPARC32-NEXT:    ldd [%fp+-24], %f0 ! 8-byte Folded Reload
+; SPARC32-NEXT:    std %f0, [%i1]
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_extend64_vec4:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -208, %sp
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0], %o0
+; SPARC64-NEXT:    fstod %f0, %f0
+; SPARC64-NEXT:    std %f0, [%fp+2039] ! 8-byte Folded Spill
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0+2], %o0
+; SPARC64-NEXT:    fstod %f0, %f0
+; SPARC64-NEXT:    std %f0, [%fp+2031] ! 8-byte Folded Spill
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0+4], %o0
+; SPARC64-NEXT:    fstod %f0, %f0
+; SPARC64-NEXT:    std %f0, [%fp+2023] ! 8-byte Folded Spill
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    lduh [%i0+6], %o0
+; SPARC64-NEXT:    fstod %f0, %f6
+; SPARC64-NEXT:    ldd [%fp+2039], %f0 ! 8-byte Folded Reload
+; SPARC64-NEXT:    ldd [%fp+2031], %f2 ! 8-byte Folded Reload
+; SPARC64-NEXT:    ldd [%fp+2023], %f4 ! 8-byte Folded Reload
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %a = load <4 x half>, ptr %p, align 8
+  %b = fpext <4 x half> %a to <4 x double>
+  ret <4 x double> %b
+}
+
+define void @test_trunc32_vec4(<4 x float> %a, ptr %p) nounwind {
+; SPARC32-LABEL: test_trunc32_vec4:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -96, %sp
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    mov %o0, %i0
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    mov %i1, %o0
+; SPARC32-NEXT:    mov %o0, %i1
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    mov %i2, %o0
+; SPARC32-NEXT:    mov %o0, %i2
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    mov %i3, %o0
+; SPARC32-NEXT:    sth %o0, [%i4+6]
+; SPARC32-NEXT:    sth %i2, [%i4+4]
+; SPARC32-NEXT:    sth %i1, [%i4+2]
+; SPARC32-NEXT:    sth %i0, [%i4]
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_trunc32_vec4:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -192, %sp
+; SPARC64-NEXT:    st %f7, [%fp+2043] ! 4-byte Folded Spill
+; SPARC64-NEXT:    st %f5, [%fp+2039] ! 4-byte Folded Spill
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    st %f3, [%fp+2035]
+; SPARC64-NEXT:    mov %o0, %i0
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    ld [%fp+2035], %f1
+; SPARC64-NEXT:    mov %o0, %i1
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    ld [%fp+2039], %f1
+; SPARC64-NEXT:    mov %o0, %i2
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    ld [%fp+2043], %f1
+; SPARC64-NEXT:    sth %o0, [%i4+6]
+; SPARC64-NEXT:    sth %i2, [%i4+4]
+; SPARC64-NEXT:    sth %i1, [%i4+2]
+; SPARC64-NEXT:    sth %i0, [%i4]
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %v = fptrunc <4 x float> %a to <4 x half>
+  store <4 x half> %v, ptr %p
+  ret void
+}
+
+define void @test_trunc64_vec4(<4 x double> %a, ptr %p) nounwind {
+; SPARC32-LABEL: test_trunc64_vec4:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -160, %sp
+; SPARC32-NEXT:    ld [%fp+92], %g2
+; SPARC32-NEXT:    ld [%fp+96], %g3
+; SPARC32-NEXT:    ! kill: def $i5 killed $i5 killed $i4_i5 def $i4_i5
+; SPARC32-NEXT:    ! kill: def $i3 killed $i3 killed $i2_i3 def $i2_i3
+; SPARC32-NEXT:    ! kill: def $i1 killed $i1 killed $i0_i1 def $i0_i1
+; SPARC32-NEXT:    std %g2, [%fp+-32]
+; SPARC32-NEXT:    ! kill: def $i4 killed $i4 killed $i4_i5 def $i4_i5
+; SPARC32-NEXT:    std %i4, [%fp+-24]
+; SPARC32-NEXT:    ! kill: def $i2 killed $i2 killed $i2_i3 def $i2_i3
+; SPARC32-NEXT:    std %i2, [%fp+-16]
+; SPARC32-NEXT:    ! kill: def $i0 killed $i0 killed $i0_i1 def $i0_i1
+; SPARC32-NEXT:    std %i0, [%fp+-8]
+; SPARC32-NEXT:    ld [%fp+100], %i3
+; SPARC32-NEXT:    ldd [%fp+-8], %f0
+; SPARC32-NEXT:    ldd [%fp+-16], %f2
+; SPARC32-NEXT:    ldd [%fp+-24], %f4
+; SPARC32-NEXT:    ldd [%fp+-32], %f6
+; SPARC32-NEXT:    std %f0, [%fp+-40]
+; SPARC32-NEXT:    std %f2, [%fp+-48]
+; SPARC32-NEXT:    std %f4, [%fp+-56]
+; SPARC32-NEXT:    std %f6, [%fp+-64]
+; SPARC32-NEXT:    call __truncdfhf2
+; SPARC32-NEXT:    ldd [%fp+-40], %o0
+; SPARC32-NEXT:    mov %o0, %i0
+; SPARC32-NEXT:    call __truncdfhf2
+; SPARC32-NEXT:    ldd [%fp+-48], %o0
+; SPARC32-NEXT:    mov %o0, %i1
+; SPARC32-NEXT:    call __truncdfhf2
+; SPARC32-NEXT:    ldd [%fp+-56], %o0
+; SPARC32-NEXT:    mov %o0, %i2
+; SPARC32-NEXT:    call __truncdfhf2
+; SPARC32-NEXT:    ldd [%fp+-64], %o0
+; SPARC32-NEXT:    sth %o0, [%i3+6]
+; SPARC32-NEXT:    sth %i2, [%i3+4]
+; SPARC32-NEXT:    sth %i1, [%i3+2]
+; SPARC32-NEXT:    sth %i0, [%i3]
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_trunc64_vec4:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -208, %sp
+; SPARC64-NEXT:    std %f6, [%fp+2039] ! 8-byte Folded Spill
+; SPARC64-NEXT:    std %f4, [%fp+2031] ! 8-byte Folded Spill
+; SPARC64-NEXT:    call __truncdfhf2
+; SPARC64-NEXT:    std %f2, [%fp+2023]
+; SPARC64-NEXT:    mov %o0, %i0
+; SPARC64-NEXT:    call __truncdfhf2
+; SPARC64-NEXT:    ldd [%fp+2023], %f0
+; SPARC64-NEXT:    mov %o0, %i1
+; SPARC64-NEXT:    call __truncdfhf2
+; SPARC64-NEXT:    ldd [%fp+2031], %f0
+; SPARC64-NEXT:    mov %o0, %i2
+; SPARC64-NEXT:    call __truncdfhf2
+; SPARC64-NEXT:    ldd [%fp+2039], %f0
+; SPARC64-NEXT:    sth %o0, [%i4+6]
+; SPARC64-NEXT:    sth %i2, [%i4+4]
+; SPARC64-NEXT:    sth %i1, [%i4+2]
+; SPARC64-NEXT:    sth %i0, [%i4]
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %v = fptrunc <4 x double> %a to <4 x half>
+  store <4 x half> %v, ptr %p
+  ret void
+}
+
+define float @test_sitofp_fadd_i32(i32 %a, ptr %b) nounwind {
+; SPARC32-LABEL: test_sitofp_fadd_i32:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    save %sp, -112, %sp
+; SPARC32-NEXT:    lduh [%i1], %i1
+; SPARC32-NEXT:    st %i0, [%fp+-4]
+; SPARC32-NEXT:    ld [%fp+-4], %f0
+; SPARC32-NEXT:    fitos %f0, %f0
+; SPARC32-NEXT:    st %f0, [%fp+-8]
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-8], %o0
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    nop
+; SPARC32-NEXT:    st %f0, [%fp+-16] ! 4-byte Folded Spill
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    mov %i1, %o0
+; SPARC32-NEXT:    ld [%fp+-16], %f1 ! 4-byte Folded Reload
+; SPARC32-NEXT:    fadds %f0, %f1, %f0
+; SPARC32-NEXT:    st %f0, [%fp+-12]
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-12], %o0
+; SPARC32-NEXT:    call __extendhfsf2
+; SPARC32-NEXT:    nop
+; SPARC32-NEXT:    ret
+; SPARC32-NEXT:    restore
+;
+; SPARC64-LABEL: test_sitofp_fadd_i32:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -192, %sp
+; SPARC64-NEXT:    lduh [%i1], %i1
+; SPARC64-NEXT:    st %i0, [%fp+2043]
+; SPARC64-NEXT:    ld [%fp+2043], %f0
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    fitos %f0, %f1
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    st %f0, [%fp+2039] ! 4-byte Folded Spill
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    mov %i1, %o0
+; SPARC64-NEXT:    ld [%fp+2039], %f1 ! 4-byte Folded Reload
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    fadds %f0, %f1, %f1
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %tmp0 = load half, ptr %b
+  %tmp1 = sitofp i32 %a to half
+  %tmp2 = fadd half %tmp0, %tmp1
+  %tmp3 = fpext half %tmp2 to float
+  ret float %tmp3
+}
+
+define half @PR40273(half) nounwind {
+; V8-LABEL: PR40273:
+; V8:       ! %bb.0:
+; V8-NEXT:    save %sp, -96, %sp
+; V8-NEXT:    call __extendhfsf2
+; V8-NEXT:    mov %i0, %o0
+; V8-NEXT:    sethi %hi(.LCPI24_0), %i0
+; V8-NEXT:    ld [%i0+%lo(.LCPI24_0)], %f1
+; V8-NEXT:    fcmps %f0, %f1
+; V8-NEXT:    nop
+; V8-NEXT:    fbne .LBB24_2
+; V8-NEXT:    nop
+; V8-NEXT:  ! %bb.1:
+; V8-NEXT:    ret
+; V8-NEXT:    restore %g0, %g0, %o0
+; V8-NEXT:  .LBB24_2:
+; V8-NEXT:    sethi 15, %i0
+; V8-NEXT:    ret
+; V8-NEXT:    restore
+;
+; V9-LABEL: PR40273:
+; V9:       ! %bb.0:
+; V9-NEXT:    save %sp, -96, %sp
+; V9-NEXT:    call __extendhfsf2
+; V9-NEXT:    mov %i0, %o0
+; V9-NEXT:    sethi %hi(.LCPI24_0), %i0
+; V9-NEXT:    ld [%i0+%lo(.LCPI24_0)], %f1
+; V9-NEXT:    mov %g0, %i0
+; V9-NEXT:    sethi 15, %i1
+; V9-NEXT:    fcmps %fcc0, %f0, %f1
+; V9-NEXT:    movne %fcc0, %i1, %i0
+; V9-NEXT:    ret
+; V9-NEXT:    restore
+;
+; SPARC64-LABEL: PR40273:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    save %sp, -176, %sp
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    srl %i0, 0, %o0
+; SPARC64-NEXT:    sethi %h44(.LCPI24_0), %i0
+; SPARC64-NEXT:    add %i0, %m44(.LCPI24_0), %i0
+; SPARC64-NEXT:    sllx %i0, 12, %i0
+; SPARC64-NEXT:    ld [%i0+%l44(.LCPI24_0)], %f1
+; SPARC64-NEXT:    mov %g0, %i0
+; SPARC64-NEXT:    sethi 15, %i1
+; SPARC64-NEXT:    fcmps %fcc0, %f0, %f1
+; SPARC64-NEXT:    movne %fcc0, %i1, %i0
+; SPARC64-NEXT:    ret
+; SPARC64-NEXT:    restore
+  %2 = fcmp une half %0, 0xH0000
+  %3 = uitofp i1 %2 to half
+  ret half %3
+}
+
+define half @fabs(half %x) nounwind {
+; SPARC32-LABEL: fabs:
+; SPARC32:       ! %bb.0:
+; SPARC32-NEXT:    sethi 4194272, %o1
+; SPARC32-NEXT:    retl
+; SPARC32-NEXT:    andn %o0, %o1, %o0
+;
+; SPARC64-LABEL: fabs:
+; SPARC64:       ! %bb.0:
+; SPARC64-NEXT:    sethi 31, %o1
+; SPARC64-NEXT:    or %o1, 1023, %o1
+; SPARC64-NEXT:    retl
+; SPARC64-NEXT:    and %o0, %o1, %o0
+  %a = call half @llvm.fabs.f16(half %x)
+  ret half %a
+}
+
+define half @fcopysign(half %x, half %y) nounwind {
+; CHECK-LABEL: fcopysign:
+; CHECK:       ! %bb.0:
+; CHECK-NEXT:    sethi 4194272, %o2
+; CHECK-NEXT:    and %o1, %o2, %o1
+; CHECK-NEXT:    andn %o0, %o2, %o0
+; CHECK-NEXT:    retl
+; CHECK-NEXT:    or %o0, %o1, %o0
+  %a = call half @llvm.copysign.f16(half %x, half %y)
+  ret half %a
+}
diff --git a/llvm/test/CodeGen/SPARC/llvm.sincos.ll b/llvm/test/CodeGen/SPARC/llvm.sincos.ll
index 87b9c8e7ba47b..8d0d50f67e3f5 100644
--- a/llvm/test/CodeGen/SPARC/llvm.sincos.ll
+++ b/llvm/test/CodeGen/SPARC/llvm.sincos.ll
@@ -10,74 +10,84 @@ define { half, half } @test_sincos_f16(half %a) #0 {
 ; SPARC32-LABEL: test_sincos_f16:
 ; SPARC32:       ! %bb.0:
 ; SPARC32-NEXT:    save %sp, -104, %sp
-; SPARC32-NEXT:    call __truncsfhf2
-; SPARC32-NEXT:    mov %i0, %o0
 ; SPARC32-NEXT:    call __extendhfsf2
-; SPARC32-NEXT:    nop
-; SPARC32-NEXT:    st %f0, [%fp+-4]
-; SPARC32-NEXT:    ld [%fp+-4], %i0
+; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    st %f0, [%fp+-12]
+; SPARC32-NEXT:    ld [%fp+-12], %i0
 ; SPARC32-NEXT:    call sinf
 ; SPARC32-NEXT:    mov %i0, %o0
-; SPARC32-NEXT:    st %f0, [%fp+-8] ! 4-byte Folded Spill
+; SPARC32-NEXT:    st %f0, [%fp+-8]
 ; SPARC32-NEXT:    call cosf
 ; SPARC32-NEXT:    mov %i0, %o0
-; SPARC32-NEXT:    fmovs %f0, %f1
-; SPARC32-NEXT:    ld [%fp+-8], %f0 ! 4-byte Folded Reload
+; SPARC32-NEXT:    st %f0, [%fp+-4]
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-8], %o0
+; SPARC32-NEXT:    mov %o0, %i0
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-4], %o0
 ; SPARC32-NEXT:    ret
-; SPARC32-NEXT:    restore
+; SPARC32-NEXT:    restore %g0, %o0, %o1
 ;
 ; SPARC64-LABEL: test_sincos_f16:
 ; SPARC64:       ! %bb.0:
 ; SPARC64-NEXT:    save %sp, -192, %sp
-; SPARC64-NEXT:    call __truncsfhf2
-; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    call __extendhfsf2
-; SPARC64-NEXT:    nop
-; SPARC64-NEXT:    st %f0, [%fp+2039] ! 4-byte Folded Spill
+; SPARC64-NEXT:    srl %i0, 0, %o0
+; SPARC64-NEXT:    st %f0, [%fp+2043] ! 4-byte Folded Spill
 ; SPARC64-NEXT:    fmovs %f0, %f1
 ; SPARC64-NEXT:    call sinf
 ; SPARC64-NEXT:    nop
-; SPARC64-NEXT:    st %f0, [%fp+2043] ! 4-byte Folded Spill
+; SPARC64-NEXT:    fmovs %f0, %f1
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    mov %o0, %i0
 ; SPARC64-NEXT:    call cosf
-; SPARC64-NEXT:    ld [%fp+2039], %f1
+; SPARC64-NEXT:    ld [%fp+2043], %f1
 ; SPARC64-NEXT:    fmovs %f0, %f1
-; SPARC64-NEXT:    ld [%fp+2043], %f0 ! 4-byte Folded Reload
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    ret
-; SPARC64-NEXT:    restore
+; SPARC64-NEXT:    restore %g0, %o0, %o1
 ;
 ; GNU32-LABEL: test_sincos_f16:
 ; GNU32:       ! %bb.0:
-; GNU32-NEXT:    save %sp, -104, %sp
-; GNU32-NEXT:    call __truncsfhf2
-; GNU32-NEXT:    mov %i0, %o0
+; GNU32-NEXT:    save %sp, -112, %sp
 ; GNU32-NEXT:    call __extendhfsf2
-; GNU32-NEXT:    nop
+; GNU32-NEXT:    mov %i0, %o0
 ; GNU32-NEXT:    st %f0, [%fp+-12]
 ; GNU32-NEXT:    ld [%fp+-12], %o0
 ; GNU32-NEXT:    add %fp, -4, %o1
 ; GNU32-NEXT:    call sincosf
 ; GNU32-NEXT:    add %fp, -8, %o2
 ; GNU32-NEXT:    ld [%fp+-4], %f0
-; GNU32-NEXT:    ld [%fp+-8], %f1
+; GNU32-NEXT:    st %f0, [%fp+-20]
+; GNU32-NEXT:    ld [%fp+-8], %f0
+; GNU32-NEXT:    st %f0, [%fp+-16]
+; GNU32-NEXT:    call __truncsfhf2
+; GNU32-NEXT:    ld [%fp+-20], %o0
+; GNU32-NEXT:    mov %o0, %i0
+; GNU32-NEXT:    call __truncsfhf2
+; GNU32-NEXT:    ld [%fp+-16], %o0
 ; GNU32-NEXT:    ret
-; GNU32-NEXT:    restore
+; GNU32-NEXT:    restore %g0, %o0, %o1
 ;
 ; GNU64-LABEL: test_sincos_f16:
 ; GNU64:       ! %bb.0:
 ; GNU64-NEXT:    save %sp, -192, %sp
-; GNU64-NEXT:    call __truncsfhf2
-; GNU64-NEXT:    nop
 ; GNU64-NEXT:    call __extendhfsf2
-; GNU64-NEXT:    nop
+; GNU64-NEXT:    srl %i0, 0, %o0
 ; GNU64-NEXT:    add %fp, 2043, %o1
 ; GNU64-NEXT:    add %fp, 2039, %o2
 ; GNU64-NEXT:    fmovs %f0, %f1
 ; GNU64-NEXT:    call sincosf
 ; GNU64-NEXT:    nop
-; GNU64-NEXT:    ld [%fp+2043], %f0
+; GNU64-NEXT:    call __truncsfhf2
+; GNU64-NEXT:    ld [%fp+2043], %f1
+; GNU64-NEXT:    mov %o0, %i0
+; GNU64-NEXT:    call __truncsfhf2
 ; GNU64-NEXT:    ld [%fp+2039], %f1
 ; GNU64-NEXT:    ret
-; GNU64-NEXT:    restore
+; GNU64-NEXT:    restore %g0, %o0, %o1
   %result = call { half, half } @llvm.sincos.f16(half %a)
   ret { half, half } %result
 }
@@ -85,61 +95,63 @@ define { half, half } @test_sincos_f16(half %a) #0 {
 define half @test_sincos_f16_only_use_sin(half %a) #0 {
 ; SPARC32-LABEL: test_sincos_f16_only_use_sin:
 ; SPARC32:       ! %bb.0:
-; SPARC32-NEXT:    save %sp, -96, %sp
-; SPARC32-NEXT:    call __truncsfhf2
-; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    save %sp, -104, %sp
 ; SPARC32-NEXT:    call __extendhfsf2
-; SPARC32-NEXT:    nop
-; SPARC32-NEXT:    st %f0, [%fp+-4]
+; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    st %f0, [%fp+-8]
 ; SPARC32-NEXT:    call sinf
+; SPARC32-NEXT:    ld [%fp+-8], %o0
+; SPARC32-NEXT:    st %f0, [%fp+-4]
+; SPARC32-NEXT:    call __truncsfhf2
 ; SPARC32-NEXT:    ld [%fp+-4], %o0
 ; SPARC32-NEXT:    ret
-; SPARC32-NEXT:    restore
+; SPARC32-NEXT:    restore %g0, %o0, %o0
 ;
 ; SPARC64-LABEL: test_sincos_f16_only_use_sin:
 ; SPARC64:       ! %bb.0:
 ; SPARC64-NEXT:    save %sp, -176, %sp
-; SPARC64-NEXT:    call __truncsfhf2
-; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    call __extendhfsf2
-; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    srl %i0, 0, %o0
 ; SPARC64-NEXT:    fmovs %f0, %f1
 ; SPARC64-NEXT:    call sinf
 ; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    fmovs %f0, %f1
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    ret
-; SPARC64-NEXT:    restore
+; SPARC64-NEXT:    restore %g0, %o0, %o0
 ;
 ; GNU32-LABEL: test_sincos_f16_only_use_sin:
 ; GNU32:       ! %bb.0:
-; GNU32-NEXT:    save %sp, -104, %sp
-; GNU32-NEXT:    call __truncsfhf2
-; GNU32-NEXT:    mov %i0, %o0
+; GNU32-NEXT:    save %sp, -112, %sp
 ; GNU32-NEXT:    call __extendhfsf2
-; GNU32-NEXT:    nop
+; GNU32-NEXT:    mov %i0, %o0
 ; GNU32-NEXT:    st %f0, [%fp+-12]
 ; GNU32-NEXT:    ld [%fp+-12], %o0
 ; GNU32-NEXT:    add %fp, -4, %o1
 ; GNU32-NEXT:    call sincosf
 ; GNU32-NEXT:    add %fp, -8, %o2
 ; GNU32-NEXT:    ld [%fp+-4], %f0
+; GNU32-NEXT:    st %f0, [%fp+-16]
+; GNU32-NEXT:    call __truncsfhf2
+; GNU32-NEXT:    ld [%fp+-16], %o0
 ; GNU32-NEXT:    ret
-; GNU32-NEXT:    restore
+; GNU32-NEXT:    restore %g0, %o0, %o0
 ;
 ; GNU64-LABEL: test_sincos_f16_only_use_sin:
 ; GNU64:       ! %bb.0:
 ; GNU64-NEXT:    save %sp, -192, %sp
-; GNU64-NEXT:    call __truncsfhf2
-; GNU64-NEXT:    nop
 ; GNU64-NEXT:    call __extendhfsf2
-; GNU64-NEXT:    nop
+; GNU64-NEXT:    srl %i0, 0, %o0
 ; GNU64-NEXT:    add %fp, 2043, %o1
 ; GNU64-NEXT:    add %fp, 2039, %o2
 ; GNU64-NEXT:    fmovs %f0, %f1
 ; GNU64-NEXT:    call sincosf
 ; GNU64-NEXT:    nop
-; GNU64-NEXT:    ld [%fp+2043], %f0
+; GNU64-NEXT:    call __truncsfhf2
+; GNU64-NEXT:    ld [%fp+2043], %f1
 ; GNU64-NEXT:    ret
-; GNU64-NEXT:    restore
+; GNU64-NEXT:    restore %g0, %o0, %o0
   %result = call { half, half } @llvm.sincos.f16(half %a)
   %result.0 = extractvalue { half, half } %result, 0
   ret half %result.0
@@ -148,61 +160,63 @@ define half @test_sincos_f16_only_use_sin(half %a) #0 {
 define half @test_sincos_f16_only_use_cos(half %a) #0 {
 ; SPARC32-LABEL: test_sincos_f16_only_use_cos:
 ; SPARC32:       ! %bb.0:
-; SPARC32-NEXT:    save %sp, -96, %sp
-; SPARC32-NEXT:    call __truncsfhf2
-; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    save %sp, -104, %sp
 ; SPARC32-NEXT:    call __extendhfsf2
-; SPARC32-NEXT:    nop
-; SPARC32-NEXT:    st %f0, [%fp+-4]
+; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    st %f0, [%fp+-8]
 ; SPARC32-NEXT:    call cosf
+; SPARC32-NEXT:    ld [%fp+-8], %o0
+; SPARC32-NEXT:    st %f0, [%fp+-4]
+; SPARC32-NEXT:    call __truncsfhf2
 ; SPARC32-NEXT:    ld [%fp+-4], %o0
 ; SPARC32-NEXT:    ret
-; SPARC32-NEXT:    restore
+; SPARC32-NEXT:    restore %g0, %o0, %o0
 ;
 ; SPARC64-LABEL: test_sincos_f16_only_use_cos:
 ; SPARC64:       ! %bb.0:
 ; SPARC64-NEXT:    save %sp, -176, %sp
-; SPARC64-NEXT:    call __truncsfhf2
-; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    call __extendhfsf2
-; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    srl %i0, 0, %o0
 ; SPARC64-NEXT:    fmovs %f0, %f1
 ; SPARC64-NEXT:    call cosf
 ; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    fmovs %f0, %f1
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    ret
-; SPARC64-NEXT:    restore
+; SPARC64-NEXT:    restore %g0, %o0, %o0
 ;
 ; GNU32-LABEL: test_sincos_f16_only_use_cos:
 ; GNU32:       ! %bb.0:
-; GNU32-NEXT:    save %sp, -104, %sp
-; GNU32-NEXT:    call __truncsfhf2
-; GNU32-NEXT:    mov %i0, %o0
+; GNU32-NEXT:    save %sp, -112, %sp
 ; GNU32-NEXT:    call __extendhfsf2
-; GNU32-NEXT:    nop
+; GNU32-NEXT:    mov %i0, %o0
 ; GNU32-NEXT:    st %f0, [%fp+-12]
 ; GNU32-NEXT:    ld [%fp+-12], %o0
 ; GNU32-NEXT:    add %fp, -4, %o1
 ; GNU32-NEXT:    call sincosf
 ; GNU32-NEXT:    add %fp, -8, %o2
 ; GNU32-NEXT:    ld [%fp+-8], %f0
+; GNU32-NEXT:    st %f0, [%fp+-16]
+; GNU32-NEXT:    call __truncsfhf2
+; GNU32-NEXT:    ld [%fp+-16], %o0
 ; GNU32-NEXT:    ret
-; GNU32-NEXT:    restore
+; GNU32-NEXT:    restore %g0, %o0, %o0
 ;
 ; GNU64-LABEL: test_sincos_f16_only_use_cos:
 ; GNU64:       ! %bb.0:
 ; GNU64-NEXT:    save %sp, -192, %sp
-; GNU64-NEXT:    call __truncsfhf2
-; GNU64-NEXT:    nop
 ; GNU64-NEXT:    call __extendhfsf2
-; GNU64-NEXT:    nop
+; GNU64-NEXT:    srl %i0, 0, %o0
 ; GNU64-NEXT:    add %fp, 2043, %o1
 ; GNU64-NEXT:    add %fp, 2039, %o2
 ; GNU64-NEXT:    fmovs %f0, %f1
 ; GNU64-NEXT:    call sincosf
 ; GNU64-NEXT:    nop
-; GNU64-NEXT:    ld [%fp+2039], %f0
+; GNU64-NEXT:    call __truncsfhf2
+; GNU64-NEXT:    ld [%fp+2039], %f1
 ; GNU64-NEXT:    ret
-; GNU64-NEXT:    restore
+; GNU64-NEXT:    restore %g0, %o0, %o0
   %result = call { half, half } @llvm.sincos.f16(half %a)
   %result.1 = extractvalue { half, half } %result, 1
   ret half %result.1
@@ -211,132 +225,157 @@ define half @test_sincos_f16_only_use_cos(half %a) #0 {
 define { <2 x half>, <2 x half> } @test_sincos_v2f16(<2 x half> %a) #0 {
 ; SPARC32-LABEL: test_sincos_v2f16:
 ; SPARC32:       ! %bb.0:
-; SPARC32-NEXT:    save %sp, -112, %sp
-; SPARC32-NEXT:    call __truncsfhf2
-; SPARC32-NEXT:    mov %i1, %o0
+; SPARC32-NEXT:    save %sp, -128, %sp
 ; SPARC32-NEXT:    call __extendhfsf2
-; SPARC32-NEXT:    nop
-; SPARC32-NEXT:    st %f0, [%fp+-12] ! 4-byte Folded Spill
-; SPARC32-NEXT:    call __truncsfhf2
-; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    mov %i1, %o0
+; SPARC32-NEXT:    st %f0, [%fp+-28]
 ; SPARC32-NEXT:    call __extendhfsf2
-; SPARC32-NEXT:    nop
-; SPARC32-NEXT:    st %f0, [%fp+-8]
-; SPARC32-NEXT:    ld [%fp+-12], %f0 ! 4-byte Folded Reload
-; SPARC32-NEXT:    st %f0, [%fp+-4]
-; SPARC32-NEXT:    ld [%fp+-8], %i0
-; SPARC32-NEXT:    call sinf
 ; SPARC32-NEXT:    mov %i0, %o0
-; SPARC32-NEXT:    st %f0, [%fp+-12] ! 4-byte Folded Spill
-; SPARC32-NEXT:    ld [%fp+-4], %i1
-; SPARC32-NEXT:    call sinf
-; SPARC32-NEXT:    mov %i1, %o0
-; SPARC32-NEXT:    st %f0, [%fp+-16] ! 4-byte Folded Spill
+; SPARC32-NEXT:    st %f0, [%fp+-32]
+; SPARC32-NEXT:    ld [%fp+-28], %i0
 ; SPARC32-NEXT:    call cosf
 ; SPARC32-NEXT:    mov %i0, %o0
-; SPARC32-NEXT:    st %f0, [%fp+-20] ! 4-byte Folded Spill
+; SPARC32-NEXT:    st %f0, [%fp+-20]
+; SPARC32-NEXT:    ld [%fp+-32], %i1
 ; SPARC32-NEXT:    call cosf
 ; SPARC32-NEXT:    mov %i1, %o0
-; SPARC32-NEXT:    fmovs %f0, %f3
-; SPARC32-NEXT:    ld [%fp+-12], %f0 ! 4-byte Folded Reload
-; SPARC32-NEXT:    ld [%fp+-16], %f1 ! 4-byte Folded Reload
-; SPARC32-NEXT:    ld [%fp+-20], %f2 ! 4-byte Folded Reload
+; SPARC32-NEXT:    st %f0, [%fp+-12]
+; SPARC32-NEXT:    call sinf
+; SPARC32-NEXT:    mov %i0, %o0
+; SPARC32-NEXT:    st %f0, [%fp+-24]
+; SPARC32-NEXT:    call sinf
+; SPARC32-NEXT:    mov %i1, %o0
+; SPARC32-NEXT:    st %f0, [%fp+-16]
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-20], %o0
+; SPARC32-NEXT:    sethi 63, %i0
+; SPARC32-NEXT:    or %i0, 1023, %i0
+; SPARC32-NEXT:    and %o0, %i0, %i4
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-12], %o0
+; SPARC32-NEXT:    and %o0, %i0, %i2
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-24], %o0
+; SPARC32-NEXT:    and %o0, %i0, %i1
+; SPARC32-NEXT:    call __truncsfhf2
+; SPARC32-NEXT:    ld [%fp+-16], %o0
+; SPARC32-NEXT:    and %o0, %i0, %g2
+; SPARC32-NEXT:    mov %g2, %i0
+; SPARC32-NEXT:    ! kill: def $i2 killed $i2 killed $i2_i3
 ; SPARC32-NEXT:    ret
-; SPARC32-NEXT:    restore
+; SPARC32-NEXT:    restore %g0, %i4, %o3
 ;
 ; SPARC64-LABEL: test_sincos_v2f16:
 ; SPARC64:       ! %bb.0:
 ; SPARC64-NEXT:    save %sp, -192, %sp
-; SPARC64-NEXT:    st %f1, [%fp+2039] ! 4-byte Folded Spill
-; SPARC64-NEXT:    fmovs %f3, %f1
-; SPARC64-NEXT:    call __truncsfhf2
-; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    call __extendhfsf2
-; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    srl %i0, 0, %o0
 ; SPARC64-NEXT:    st %f0, [%fp+2043] ! 4-byte Folded Spill
-; SPARC64-NEXT:    call __truncsfhf2
-; SPARC64-NEXT:    ld [%fp+2039], %f1
-; SPARC64-NEXT:    call __extendhfsf2
-; SPARC64-NEXT:    nop
-; SPARC64-NEXT:    st %f0, [%fp+2031] ! 4-byte Folded Spill
 ; SPARC64-NEXT:    fmovs %f0, %f1
 ; SPARC64-NEXT:    call sinf
 ; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    fmovs %f0, %f1
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    mov %o0, %i0
+; SPARC64-NEXT:    call __extendhfsf2
+; SPARC64-NEXT:    srl %i1, 0, %o0
 ; SPARC64-NEXT:    st %f0, [%fp+2039] ! 4-byte Folded Spill
+; SPARC64-NEXT:    fmovs %f0, %f1
 ; SPARC64-NEXT:    call sinf
-; SPARC64-NEXT:    ld [%fp+2043], %f1
-; SPARC64-NEXT:    st %f0, [%fp+2035] ! 4-byte Folded Spill
-; SPARC64-NEXT:    call cosf
-; SPARC64-NEXT:    ld [%fp+2031], %f1
-; SPARC64-NEXT:    st %f0, [%fp+2031] ! 4-byte Folded Spill
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    fmovs %f0, %f1
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    mov %o0, %i1
 ; SPARC64-NEXT:    call cosf
 ; SPARC64-NEXT:    ld [%fp+2043], %f1
-; SPARC64-NEXT:    fmovs %f0, %f3
-; SPARC64-NEXT:    ld [%fp+2039], %f0 ! 4-byte Folded Reload
-; SPARC64-NEXT:    ld [%fp+2035], %f1 ! 4-byte Folded Reload
-; SPARC64-NEXT:    ld [%fp+2031], %f2 ! 4-byte Folded Reload
+; SPARC64-NEXT:    fmovs %f0, %f1
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
+; SPARC64-NEXT:    mov %o0, %i2
+; SPARC64-NEXT:    call cosf
+; SPARC64-NEXT:    ld [%fp+2039], %f1
+; SPARC64-NEXT:    fmovs %f0, %f1
+; SPARC64-NEXT:    call __truncsfhf2
+; SPARC64-NEXT:    nop
 ; SPARC64-NEXT:    ret
-; SPARC64-NEXT:    restore
+; SPARC64-NEXT:    restore %g0, %o0, %o3
 ;
 ; GNU32-LABEL: test_sincos_v2f16:
 ; GNU32:       ! %bb.0:
-; GNU32-NEXT:    save %sp, -120, %sp
-; GNU32-NEXT:    call __truncsfhf2
-; GNU32-NEXT:    mov %i1, %o0
-; GNU32-NEXT:    call __extendhfsf2
-; GNU32-NEXT:    nop
-; GNU32-NEXT:    st %f0, [%fp+-28] ! 4-byte Folded Spill
-; GNU32-NEXT:    call __truncsfhf2
-; GNU32-NEXT:    mov %i0, %o0
+; GNU32-NEXT:    save %sp, -144, %sp
 ; GNU32-NEXT:    call __extendhfsf2
-; GNU32-NEXT:    nop
-; GNU32-NEXT:    st %f0, [%fp+-20]
-; GNU32-NEXT:    ld [%fp+-20], %o0
+; GNU32-NEXT:    mov %i1, %o0
+; GNU32-NEXT:    st %f0, [%fp+-32]
+; GNU32-NEXT:    ld [%fp+-32], %o0
 ; GNU32-NEXT:    add %fp, -12, %o1
 ; GNU32-NEXT:    call sincosf
 ; GNU32-NEXT:    add %fp, -16, %o2
-; GNU32-NEXT:    ld [%fp+-28], %f0 ! 4-byte Folded Reload
-; GNU32-NEXT:    st %f0, [%fp+-24]
-; GNU32-NEXT:    ld [%fp+-24], %o0
-; GNU32-NEXT:    add %fp, -4, %o1
+; GNU32-NEXT:    call __extendhfsf2
+; GNU32-NEXT:    mov %i0, %o0
+; GNU32-NEXT:    st %f0, [%fp+-28]
+; GNU32-NEXT:    ld [%fp+-28], %o0
+; GNU32-NEXT:    add %fp, -20, %o1
 ; GNU32-NEXT:    call sincosf
-; GNU32-NEXT:    add %fp, -8, %o2
+; GNU32-NEXT:    add %fp, -24, %o2
+; GNU32-NEXT:    ld [%fp+-16], %f0
+; GNU32-NEXT:    st %f0, [%fp+-44]
+; GNU32-NEXT:    ld [%fp+-24], %f0
+; GNU32-NEXT:    st %f0, [%fp+-36]
 ; GNU32-NEXT:    ld [%fp+-12], %f0
-; GNU32-NEXT:    ld [%fp+-4], %f1
-; GNU32-NEXT:    ld [%fp+-16], %f2
-; GNU32-NEXT:    ld [%fp+-8], %f3
+; GNU32-NEXT:    st %f0, [%fp+-48]
+; GNU32-NEXT:    ld [%fp+-20], %f0
+; GNU32-NEXT:    st %f0, [%fp+-40]
+; GNU32-NEXT:    call __truncsfhf2
+; GNU32-NEXT:    ld [%fp+-44], %o0
+; GNU32-NEXT:    sethi 63, %i0
+; GNU32-NEXT:    or %i0, 1023, %i0
+; GNU32-NEXT:    and %o0, %i0, %i4
+; GNU32-NEXT:    call __truncsfhf2
+; GNU32-NEXT:    ld [%fp+-36], %o0
+; GNU32-NEXT:    and %o0, %i0, %i2
+; GNU32-NEXT:    call __truncsfhf2
+; GNU32-NEXT:    ld [%fp+-48], %o0
+; GNU32-NEXT:    and %o0, %i0, %i1
+; GNU32-NEXT:    call __truncsfhf2
+; GNU32-NEXT:    ld [%fp+-40], %o0
+; GNU32-NEXT:    and %o0, %i0, %g2
+; GNU32-NEXT:    mov %g2, %i0
+; GNU32-NEXT:    ! kill: def $i2 killed $i2 killed $i2_i3
 ; GNU32-NEXT:    ret
-; GNU32-NEXT:    restore
+; GNU32-NEXT:    restore %g0, %i4, %o3
 ;
 ; GNU64-LABEL: test_sincos_v2f16:
 ; GNU64:       ! %bb.0:
-; GNU64-NEXT:    save %sp, -208, %sp
-; GNU64-NEXT:    st %f1, [%fp+2023] ! 4-byte Folded Spill
-; GNU64-NEXT:    fmovs %f3, %f1
-; GNU64-NEXT:    call __truncsfhf2
-; GNU64-NEXT:    nop
-; GNU64-NEXT:    call __extendhfsf2
-; GNU64-NEXT:    nop
-; GNU64-NEXT:    st %f0, [%fp+2027] ! 4-byte Folded Spill
-; GNU64-NEXT:    call __truncsfhf2
-; GNU64-NEXT:    ld [%fp+2023], %f1
+; GNU64-NEXT:    save %sp, -192, %sp
 ; GNU64-NEXT:    call __extendhfsf2
-; GNU64-NEXT:    nop
+; GNU64-NEXT:    srl %i0, 0, %o0
 ; GNU64-NEXT:    add %fp, 2035, %o1
 ; GNU64-NEXT:    add %fp, 2031, %o2
 ; GNU64-NEXT:    fmovs %f0, %f1
 ; GNU64-NEXT:    call sincosf
 ; GNU64-NEXT:    nop
+; GNU64-NEXT:    call __extendhfsf2
+; GNU64-NEXT:    srl %i1, 0, %o0
 ; GNU64-NEXT:    add %fp, 2043, %o1
 ; GNU64-NEXT:    add %fp, 2039, %o2
+; GNU64-NEXT:    fmovs %f0, %f1
 ; GNU64-NEXT:    call sincosf
-; GNU64-NEXT:    ld [%fp+2027], %f1
-; GNU64-NEXT:    ld [%fp+2035], %f0
+; GNU64-NEXT:    nop
+; GNU64-NEXT:    call __truncsfhf2
+; GNU64-NEXT:    ld [%fp+2035], %f1
+; GNU64-NEXT:    mov %o0, %i0
+; GNU64-NEXT:    call __truncsfhf2
 ; GNU64-NEXT:    ld [%fp+2043], %f1
-; GNU64-NEXT:    ld [%fp+2031], %f2
-; GNU64-NEXT:    ld [%fp+2039], %f3
+; GNU64-NEXT:    mov %o0, %i1
+; GNU64-NEXT:    call __truncsfhf2
+; GNU64-NEXT:    ld [%fp+2031], %f1
+; GNU64-NEXT:    mov %o0, %i2
+; GNU64-NEXT:    call __truncsfhf2
+; GNU64-NEXT:    ld [%fp+2039], %f1
 ; GNU64-NEXT:    ret
-; GNU64-NEXT:    restore
+; GNU64-NEXT:    restore %g0, %o0, %o3
   %result = call { <2 x half>, <2 x half> } @llvm.sincos.v2f16(<2 x half> %a)
   ret { <2 x half>, <2 x half> } %result
 }
diff --git a/llvm/test/CodeGen/SPARC/lrint-conv.ll b/llvm/test/CodeGen/SPARC/lrint-conv.ll
new file mode 100644
index 0000000000000..81934114f548f
--- /dev/null
+++ b/llvm/test/CodeGen/SPARC/lrint-conv.ll
@@ -0,0 +1,68 @@
+; Tests for lrint and llrint, with both i32 and i64 checked.
+
+; RUN: sed 's/ITy/i32/g' %s | llc -mtriple=sparc   | FileCheck %s --check-prefixes=SPARC32
+; RUN: sed 's/ITy/i64/g' %s | llc -mtriple=sparc   | FileCheck %s --check-prefixes=SPARC32
+; RUN: sed 's/ITy/i32/g' %s | llc -mtriple=sparc64 | FileCheck %s --check-prefixes=SPARC64
+; RUN: sed 's/ITy/i64/g' %s | llc -mtriple=sparc64 | FileCheck %s --check-prefixes=SPARC64
+
+; FIXME: crash "Input type needs to be promoted!"
+; define ITy @test_lrint_ixx_f16(half %x) nounwind {
+;   %res = tail call ITy @llvm.lrint.ITy.f16(half %x)
+;   ret ITy %res
+; }
+
+; define ITy @test_llrint_ixx_f16(half %x) nounwind {
+;   %res = tail call ITy @llvm.llrint.ITy.f16(half %x)
+;   ret ITy %res
+; }
+
+define ITy @test_lrint_ixx_f32(float %x) nounwind {
+; SPARC32-LABEL: test_lrint_ixx_f32:
+; SPARC32:         call lrintf
+;
+; SPARC64-LABEL: test_lrint_ixx_f32:
+; SPARC64:         call lrintf
+  %res = tail call ITy @llvm.lrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f32(float %x) nounwind {
+; SPARC32-LABEL: test_llrint_ixx_f32:
+; SPARC32:         call llrintf
+;
+; SPARC64-LABEL: test_llrint_ixx_f32:
+; SPARC64:         call llrintf
+  %res = tail call ITy @llvm.llrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_lrint_ixx_f64(double %x) nounwind {
+; SPARC32-LABEL: test_lrint_ixx_f64:
+; SPARC32:         call lrint
+;
+; SPARC64-LABEL: test_lrint_ixx_f64:
+; SPARC64:         call lrint
+  %res = tail call ITy @llvm.lrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f64(double %x) nounwind {
+; SPARC32-LABEL: test_llrint_ixx_f64:
+; SPARC32:         call llrint
+;
+; SPARC64-LABEL: test_llrint_ixx_f64:
+; SPARC64:         call llrint
+  %res = tail call ITy @llvm.llrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+; FIXME(#41838): unsupported type
+; define ITy @test_lrint_ixx_f128(fp128 %x) nounwind {
+;   %res = tail call ITy @llvm.lrint.ITy.f128(fp128 %x)
+;   ret ITy %res
+; }
+
+; define ITy @test_llrint_ixx_f128(fp128 %x) nounwind {
+;   %res = tail call ITy @llvm.llrint.ITy.f128(fp128 %x)
+;   ret ITy %res
+; }
diff --git a/llvm/test/CodeGen/SPIRV/AtomicCompareExchange64BitPhiNode.ll b/llvm/test/CodeGen/SPIRV/AtomicCompareExchange64BitPhiNode.ll
new file mode 100644
index 0000000000000..93a7a3ab79dc4
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/AtomicCompareExchange64BitPhiNode.ll
@@ -0,0 +1,30 @@
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; Regression test for issue https://github.com/llvm/llvm-project/issues/152863
+; Ensure OpAtomicCompareExchange returns the correct i64 type when used in phi nodes.
+; Previously, this would generate invalid SPIR-V where the atomic operation returned
+; uint (32-bit) but the phi node expected ulong (64-bit), causing validation errors.
+
+; CHECK-SPIRV-DAG: %[[#Long:]] = OpTypeInt 64 0
+; CHECK-SPIRV-DAG: %[[#Ptr:]] = OpTypePointer CrossWorkgroup %[[#Long]]
+; CHECK-SPIRV-DAG: %[[#Zero64:]] = OpConstantNull %[[#Long]]
+
+; Verify that both the phi node and atomic operation use the same i64 type
+; CHECK-SPIRV: %[[#ValuePhi:]] = OpPhi %[[#Long]] %[[#Zero64]] %[[#]] %[[#AtomicResult:]] %[[#]]
+; CHECK-SPIRV: %[[#AtomicResult]] = OpAtomicCompareExchange %[[#Long]] %[[#]] %[[#]] %[[#]] %[[#]] %[[#Zero64]] %[[#ValuePhi]]
+
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
+target triple = "spirv64-unknown-unknown"
+
+declare i64 @_Z14atomic_cmpxchgPU8CLglobalVlll(ptr addrspace(1), i64, i64)
+
+define spir_kernel void @test_atomic_cmpxchg_phi(ptr addrspace(1) %ptr) {
+conversion:
+  br label %L6
+
+L6:                                               ; preds = %L6, %conversion
+  %value_phi = phi i64 [ 0, %conversion ], [ %1, %L6 ]
+  %1 = call i64 @_Z14atomic_cmpxchgPU8CLglobalVlll(ptr addrspace(1) %ptr, i64 %value_phi, i64 0)
+  br label %L6
+}
diff --git a/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-avoid-invalid.ll b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-avoid-invalid.ll
new file mode 100644
index 0000000000000..2de7fff0bc900
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/extensions/enable-all-extensions-avoid-invalid.ll
@@ -0,0 +1,16 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv1.6-vulkan1.3-compute --spirv-ext=all %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-compute --spirv-ext=all %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+; CHECK-NOT: OpExtension "SPV_KHR_no_integer_wrap_decoration"
+
+define internal void @foo(i32 %i) local_unnamed_addr {
+  %sub.i = sub nsw i32 0, %i
+  ret void
+}
+
+define internal void @main() local_unnamed_addr #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
\ No newline at end of file
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-resources/ImplicitBinding.ll b/llvm/test/CodeGen/SPIRV/hlsl-resources/ImplicitBinding.ll
new file mode 100644
index 0000000000000..00e9185822ad5
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-resources/ImplicitBinding.ll
@@ -0,0 +1,75 @@
+; RUN: llc -O0 -verify-machineinstrs -mtriple=spirv1.6-vulkan1.3-library %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv1.6-vulkan1.3-library %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+@.str = private unnamed_addr constant [2 x i8] c"b\00", align 1
+@.str.2 = private unnamed_addr constant [2 x i8] c"c\00", align 1
+@.str.4 = private unnamed_addr constant [2 x i8] c"d\00", align 1
+@.str.6 = private unnamed_addr constant [2 x i8] c"e\00", align 1
+@.str.8 = private unnamed_addr constant [2 x i8] c"f\00", align 1
+@.str.10 = private unnamed_addr constant [2 x i8] c"g\00", align 1
+@.str.12 = private unnamed_addr constant [2 x i8] c"h\00", align 1
+@.str.14 = private unnamed_addr constant [2 x i8] c"i\00", align 1
+
+; CHECK-DAG: OpName [[b:%[0-9]+]] "b"
+; CHECK-DAG: OpName [[c:%[0-9]+]] "c"
+; CHECK-DAG: OpName [[d:%[0-9]+]] "d"
+; CHECK-DAG: OpName [[e:%[0-9]+]] "e"
+; CHECK-DAG: OpName [[f:%[0-9]+]] "f"
+; CHECK-DAG: OpName [[g:%[0-9]+]] "g"
+; CHECK-DAG: OpName [[h:%[0-9]+]] "h"
+; CHECK-DAG: OpName [[i:%[0-9]+]] "i"
+; CHECK-DAG: OpDecorate [[b]] DescriptorSet 0
+; CHECK-DAG: OpDecorate [[b]] Binding 1
+; CHECK-DAG: OpDecorate [[c]] DescriptorSet 0
+; CHECK-DAG: OpDecorate [[c]] Binding 0
+; CHECK-DAG: OpDecorate [[d]] DescriptorSet 0
+; CHECK-DAG: OpDecorate [[d]] Binding 3
+; CHECK-DAG: OpDecorate [[e]] DescriptorSet 0
+; CHECK-DAG: OpDecorate [[e]] Binding 2
+; CHECK-DAG: OpDecorate [[f]] DescriptorSet 10
+; CHECK-DAG: OpDecorate [[f]] Binding 1
+; CHECK-DAG: OpDecorate [[g]] DescriptorSet 10
+; CHECK-DAG: OpDecorate [[g]] Binding 0
+; CHECK-DAG: OpDecorate [[h]] DescriptorSet 10
+; CHECK-DAG: OpDecorate [[h]] Binding 3
+; CHECK-DAG: OpDecorate [[i]] DescriptorSet 10
+; CHECK-DAG: OpDecorate [[i]] Binding 2
+
+
+define void @main() local_unnamed_addr #0 {
+entry:
+  %0 = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefromimplicitbinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str)
+  %1 = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 0, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str.2)
+  %2 = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefromimplicitbinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 1, i32 0, i32 1, i32 0, i1 false, ptr nonnull @.str.4)
+  %3 = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 0, i32 2, i32 1, i32 0, i1 false, ptr nonnull @.str.6)
+  %4 = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 10, i32 1, i32 1, i32 0, i1 false, ptr nonnull @.str.8)
+  %5 = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefromimplicitbinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 2, i32 10, i32 1, i32 0, i1 false, ptr nonnull @.str.10)
+  %6 = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefromimplicitbinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 3, i32 10, i32 1, i32 0, i1 false, ptr nonnull @.str.12)
+  %7 = tail call target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) @llvm.spv.resource.handlefrombinding.tspirv.SignedImage_i32_5_2_0_0_2_0t(i32 10, i32 2, i32 1, i32 0, i1 false, ptr nonnull @.str.14)
+  %8 = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %1, i32 0)
+  %9 = load i32, ptr addrspace(11) %8, align 4
+  %10 = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %2, i32 0)
+  %11 = load i32, ptr addrspace(11) %10, align 4
+  %add.i = add nsw i32 %11, %9
+  %12 = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %3, i32 0)
+  %13 = load i32, ptr addrspace(11) %12, align 4
+  %add4.i = add nsw i32 %add.i, %13
+  %14 = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %4, i32 0)
+  %15 = load i32, ptr addrspace(11) %14, align 4
+  %add6.i = add nsw i32 %add4.i, %15
+  %16 = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %5, i32 0)
+  %17 = load i32, ptr addrspace(11) %16, align 4
+  %add8.i = add nsw i32 %add6.i, %17
+  %18 = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %6, i32 0)
+  %19 = load i32, ptr addrspace(11) %18, align 4
+  %add10.i = add nsw i32 %add8.i, %19
+  %20 = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %7, i32 0)
+  %21 = load i32, ptr addrspace(11) %20, align 4
+  %add12.i = add nsw i32 %add10.i, %21
+  %22 = tail call noundef align 4 dereferenceable(4) ptr addrspace(11) @llvm.spv.resource.getpointer.p11.tspirv.SignedImage_i32_5_2_0_0_2_0t(target("spirv.SignedImage", i32, 5, 2, 0, 0, 2, 0) %0, i32 0)
+  store i32 %add12.i, ptr addrspace(11) %22, align 4
+  ret void
+}
+
+
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
\ No newline at end of file
diff --git a/llvm/test/CodeGen/SPIRV/instructions/issue-135572-emit-float-opselect.ll b/llvm/test/CodeGen/SPIRV/instructions/issue-135572-emit-float-opselect.ll
new file mode 100644
index 0000000000000..69ea054de1e4f
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/instructions/issue-135572-emit-float-opselect.ll
@@ -0,0 +1,53 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val --target-env spv1.6 %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val --target-env spv1.6 %}
+
+; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#func_type:]] = OpTypeFunction %[[#float_32]] %[[#float_32]] %[[#float_32]]
+; CHECK-DAG: %[[#bool:]] = OpTypeBool
+; CHECK-DAG: %[[#vec4_float_32:]] = OpTypeVector %[[#float_32]] 4
+; CHECK-DAG: %[[#vec_func_type:]] = OpTypeFunction %[[#vec4_float_32]] %[[#vec4_float_32]] %[[#vec4_float_32]]
+; CHECK-DAG:  %[[#vec_4_bool:]] = OpTypeVector %[[#bool]] 4
+
+define spir_func float @opselect_float_scalar_test(float %x, float %y) {
+entry:
+  ; CHECK: %[[#]] = OpFunction %[[#float_32]] None %[[#func_type]]
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#float_32]]
+  ; CHECK: %[[#arg1:]] = OpFunctionParameter %[[#float_32]]
+  ; CHECK: %[[#fcmp:]] = OpFOrdGreaterThan  %[[#bool]]  %[[#arg0]] %[[#arg1]]
+  ; CHECK: %[[#fselect:]] = OpSelect %[[#float_32]] %[[#fcmp]] %[[#arg0]] %[[#arg1]]
+  ; CHECK: OpReturnValue %[[#fselect]]
+  %0 = fcmp ogt float %x, %y
+  %1 = select i1 %0, float %x, float %y
+  ret float %1
+}
+
+define spir_func <4 x float> @opselect_float4_vec_test(<4 x float>  %x, <4 x float>  %y) {
+entry:
+  ; CHECK: %[[#]] = OpFunction %[[#vec4_float_32]] None %[[#vec_func_type]]
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec4_float_32]]
+  ; CHECK: %[[#arg1:]] = OpFunctionParameter %[[#vec4_float_32]]
+  ; CHECK: %[[#fcmp:]] = OpFOrdGreaterThan  %[[#vec_4_bool]]  %[[#arg0]] %[[#arg1]]
+  ; CHECK: %[[#fselect:]] = OpSelect %[[#vec4_float_32]] %[[#fcmp]] %[[#arg0]] %[[#arg1]]
+  ; CHECK: OpReturnValue %[[#fselect]]
+  %0 = fcmp ogt <4 x float> %x, %y
+  %1 = select <4 x i1> %0, <4 x float> %x, <4 x float> %y
+  ret <4 x float>  %1
+}
+
+define spir_func <4 x float> @opselect_scalar_bool_float4_vec_test(float %a, float %b, <4 x float>  %x, <4 x float>  %y) {
+entry:
+  ; CHECK: %[[#]] = OpFunction %[[#vec4_float_32]] None %[[#]]
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#float_32]]
+  ; CHECK: %[[#arg1:]] = OpFunctionParameter %[[#float_32]]
+  ; CHECK: %[[#arg2:]] = OpFunctionParameter %[[#vec4_float_32]]
+  ; CHECK: %[[#arg3:]] = OpFunctionParameter %[[#vec4_float_32]]
+  ; CHECK: %[[#fcmp:]] = OpFOrdGreaterThan  %[[#bool]]  %[[#arg0]] %[[#arg1]]
+  ; CHECK: %[[#fselect:]] = OpSelect %[[#vec4_float_32]] %[[#fcmp]] %[[#arg2]] %[[#arg3]]
+  ; CHECK: OpReturnValue %[[#fselect]]
+  %0 = fcmp ogt float %a, %b
+  %1 = select i1 %0, <4 x float> %x, <4 x float> %y
+  ret <4 x float>  %1
+}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/lifetime.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/lifetime.ll
index 483d7077a66f9..3d93eca72aaed 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/lifetime.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/lifetime.ll
@@ -17,11 +17,11 @@
 ; CL:      OpFunction
 ; CL:      %[[#FooVar:]] = OpVariable
 ; CL-NEXT: %[[#Casted1:]] = OpBitcast %[[#PtrChar]] %[[#FooVar]]
-; CL-NEXT: OpLifetimeStart %[[#Casted1]], 72
+; CL-NEXT: OpLifetimeStart %[[#Casted1]], 16
 ; CL-NEXT: OpBitcast
 ; CL-NEXT: OpInBoundsPtrAccessChain
 ; CL-NEXT: %[[#Casted2:]] = OpBitcast %[[#PtrChar]] %[[#FooVar]]
-; CL-NEXT: OpLifetimeStop %[[#Casted2]], 72
+; CL-NEXT: OpLifetimeStop %[[#Casted2]], 16
 
 ; VK:      OpFunction
 ; VK:      %[[#FooVar:]] = OpVariable
@@ -29,18 +29,20 @@
 ; VK-NEXT: OpReturn
 define spir_func void @foo(ptr noundef byval(%tprange) align 8 %_arg_UserRange) {
   %RoundedRangeKernel = alloca %tprange, align 8
-  call void @llvm.lifetime.start.p0(i64 72, ptr nonnull %RoundedRangeKernel)
+  call void @llvm.lifetime.start.p0(ptr nonnull %RoundedRangeKernel)
   %KernelFunc = getelementptr inbounds i8, ptr %RoundedRangeKernel, i64 8
-  call void @llvm.lifetime.end.p0(i64 72, ptr nonnull %RoundedRangeKernel)
+  call void @llvm.lifetime.end.p0(ptr nonnull %RoundedRangeKernel)
   ret void
 }
 
 ; CL: OpFunction
 ; CL: %[[#BarVar:]] = OpVariable
-; CL-NEXT: OpLifetimeStart %[[#BarVar]], 0
+; CL-NEXT: %[[#Casted1:]] = OpBitcast %[[#PtrChar]] %[[#BarVar]]
+; CL-NEXT: OpLifetimeStart %[[#Casted1]], 16
 ; CL-NEXT: OpBitcast
 ; CL-NEXT: OpInBoundsPtrAccessChain
-; CL-NEXT: OpLifetimeStop %[[#BarVar]], 0
+; CL-NEXT: %[[#Casted2:]] = OpBitcast %[[#PtrChar]] %[[#BarVar]]
+; CL-NEXT: OpLifetimeStop %[[#Casted2]], 16
 
 ; VK:      OpFunction
 ; VK:      %[[#BarVar:]] = OpVariable
@@ -48,9 +50,9 @@ define spir_func void @foo(ptr noundef byval(%tprange) align 8 %_arg_UserRange)
 ; VK-NEXT: OpReturn
 define spir_func void @bar(ptr noundef byval(%tprange) align 8 %_arg_UserRange) {
   %RoundedRangeKernel = alloca %tprange, align 8
-  call void @llvm.lifetime.start.p0(i64 -1, ptr nonnull %RoundedRangeKernel)
+  call void @llvm.lifetime.start.p0(ptr nonnull %RoundedRangeKernel)
   %KernelFunc = getelementptr inbounds i8, ptr %RoundedRangeKernel, i64 8
-  call void @llvm.lifetime.end.p0(i64 -1, ptr nonnull %RoundedRangeKernel)
+  call void @llvm.lifetime.end.p0(ptr nonnull %RoundedRangeKernel)
   ret void
 }
 
@@ -66,12 +68,12 @@ define spir_func void @bar(ptr noundef byval(%tprange) align 8 %_arg_UserRange)
 ; VK-NEXT: OpReturn
 define spir_func void @test(ptr noundef align 8 %_arg) {
   %var = alloca i8, align 8
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %var)
+  call void @llvm.lifetime.start.p0(ptr nonnull %var)
   %KernelFunc = getelementptr inbounds i8, ptr %var, i64 1
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %var)
+  call void @llvm.lifetime.end.p0(ptr nonnull %var)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/add.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/add.ll
index ef9719e64586d..39cbd8726400a 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/add.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/add.ll
@@ -21,7 +21,7 @@
 ; CHECK-DAG: %[[LongVec3:.*]] = OpTypeVector %[[Long]] 3
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpIAdd %[[CharVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2CharR:.*]] = OpCompositeExtract %[[Char]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2CharR]]
@@ -38,7 +38,7 @@
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpIAdd %[[ShortVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2ShortR:.*]] = OpCompositeExtract %[[Short]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2ShortR]]
@@ -55,7 +55,7 @@
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpIAdd %[[IntVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2IntR:.*]] = OpCompositeExtract %[[Int]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2IntR]]
@@ -72,7 +72,7 @@
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpIAdd %[[LongVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2LongR:.*]] = OpCompositeExtract %[[Long]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2LongR]]
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/and.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/and.ll
index e9d9adba8f365..a911cdf094045 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/and.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/and.ll
@@ -21,7 +21,7 @@
 ; CHECK-DAG: %[[LongVec3:.*]] = OpTypeVector %[[Long]] 3
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseAnd %[[CharVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2CharR:.*]] = OpCompositeExtract %[[Char]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2CharR]]
@@ -38,7 +38,7 @@
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseAnd %[[ShortVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2ShortR:.*]] = OpCompositeExtract %[[Short]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2ShortR]]
@@ -55,7 +55,7 @@
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseAnd %[[IntVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2IntR:.*]] = OpCompositeExtract %[[Int]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2IntR]]
@@ -72,7 +72,7 @@
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseAnd %[[LongVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2LongR:.*]] = OpCompositeExtract %[[Long]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2LongR]]
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/mul.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/mul.ll
index 16455f2e21cb6..86e6d42de55d1 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/mul.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/mul.ll
@@ -20,7 +20,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK-DAG: %[[LongVec3:.*]] = OpTypeVector %[[Long]] 3
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpIMul %[[CharVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2CharR:.*]] = OpCompositeExtract %[[Char]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2CharR]]
@@ -37,7 +37,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpIMul %[[ShortVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2ShortR:.*]] = OpCompositeExtract %[[Short]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2ShortR]]
@@ -54,7 +54,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpIMul %[[IntVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2IntR:.*]] = OpCompositeExtract %[[Int]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2IntR]]
@@ -71,7 +71,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpIMul %[[LongVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2LongR:.*]] = OpCompositeExtract %[[Long]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2LongR]]
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/or.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/or.ll
index badfebc27072f..34e5272ecd629 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/or.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/or.ll
@@ -20,7 +20,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK-DAG: %[[LongVec3:.*]] = OpTypeVector %[[Long]] 3
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseOr %[[CharVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2CharR:.*]] = OpCompositeExtract %[[Char]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2CharR]]
@@ -37,7 +37,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseOr %[[ShortVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2ShortR:.*]] = OpCompositeExtract %[[Short]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2ShortR]]
@@ -54,7 +54,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseOr %[[IntVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2IntR:.*]] = OpCompositeExtract %[[Int]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2IntR]]
@@ -71,7 +71,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseOr %[[LongVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2LongR:.*]] = OpCompositeExtract %[[Long]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2LongR]]
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smax.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smax.ll
index 54afb3a25b4d6..eafd4e096a728 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smax.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smax.ll
@@ -20,7 +20,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK-DAG: %[[LongVec3:.*]] = OpTypeVector %[[Long]] 3
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[CharVec2]] %[[#]] s_max %[[#]] %[[#]]
 ; CHECK: %[[Vec2CharR:.*]] = OpCompositeExtract %[[Char]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2CharR]]
@@ -37,7 +37,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[ShortVec2]] %[[#]] s_max %[[#]] %[[#]]
 ; CHECK: %[[Vec2ShortR:.*]] = OpCompositeExtract %[[Short]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2ShortR]]
@@ -54,7 +54,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[IntVec2]] %[[#]] s_max %[[#]] %[[#]]
 ; CHECK: %[[Vec2IntR:.*]] = OpCompositeExtract %[[Int]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2IntR]]
@@ -71,7 +71,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[LongVec2]] %[[#]] s_max %[[#]] %[[#]]
 ; CHECK: %[[Vec2LongR:.*]] = OpCompositeExtract %[[Long]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2LongR]]
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smin.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smin.ll
index a95c2ea0bbbf6..bbd22d4c6d4c5 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smin.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/smin.ll
@@ -20,7 +20,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK-DAG: %[[LongVec3:.*]] = OpTypeVector %[[Long]] 3
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[CharVec2]] %[[#]] s_min %[[#]] %[[#]]
 ; CHECK: %[[Vec2CharR:.*]] = OpCompositeExtract %[[Char]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2CharR]]
@@ -37,7 +37,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[ShortVec2]] %[[#]] s_min %[[#]] %[[#]]
 ; CHECK: %[[Vec2ShortR:.*]] = OpCompositeExtract %[[Short]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2ShortR]]
@@ -54,7 +54,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[IntVec2]] %[[#]] s_min %[[#]] %[[#]]
 ; CHECK: %[[Vec2IntR:.*]] = OpCompositeExtract %[[Int]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2IntR]]
@@ -71,7 +71,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[LongVec2]] %[[#]] s_min %[[#]] %[[#]]
 ; CHECK: %[[Vec2LongR:.*]] = OpCompositeExtract %[[Long]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2LongR]]
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umax.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umax.ll
index a742009cef011..80be288d0b941 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umax.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umax.ll
@@ -20,7 +20,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK-DAG: %[[LongVec3:.*]] = OpTypeVector %[[Long]] 3
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[CharVec2]] %[[#]] u_max %[[#]] %[[#]]
 ; CHECK: %[[Vec2CharR:.*]] = OpCompositeExtract %[[Char]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2CharR]]
@@ -37,7 +37,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[ShortVec2]] %[[#]] u_max %[[#]] %[[#]]
 ; CHECK: %[[Vec2ShortR:.*]] = OpCompositeExtract %[[Short]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2ShortR]]
@@ -54,7 +54,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[IntVec2]] %[[#]] u_max %[[#]] %[[#]]
 ; CHECK: %[[Vec2IntR:.*]] = OpCompositeExtract %[[Int]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2IntR]]
@@ -71,7 +71,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[LongVec2]] %[[#]] u_max %[[#]] %[[#]]
 ; CHECK: %[[Vec2LongR:.*]] = OpCompositeExtract %[[Long]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2LongR]]
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umin.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umin.ll
index 7844c205c7ab0..2c4832c877783 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umin.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/umin.ll
@@ -20,7 +20,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK-DAG: %[[LongVec3:.*]] = OpTypeVector %[[Long]] 3
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[CharVec2]] %[[#]] u_min %[[#]] %[[#]]
 ; CHECK: %[[Vec2CharR:.*]] = OpCompositeExtract %[[Char]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2CharR]]
@@ -37,7 +37,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[ShortVec2]] %[[#]] u_min %[[#]] %[[#]]
 ; CHECK: %[[Vec2ShortR:.*]] = OpCompositeExtract %[[Short]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2ShortR]]
@@ -54,7 +54,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[IntVec2]] %[[#]] u_min %[[#]] %[[#]]
 ; CHECK: %[[Vec2IntR:.*]] = OpCompositeExtract %[[Int]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2IntR]]
@@ -71,7 +71,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpExtInst %[[LongVec2]] %[[#]] u_min %[[#]] %[[#]]
 ; CHECK: %[[Vec2LongR:.*]] = OpCompositeExtract %[[Long]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2LongR]]
diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/xor.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/xor.ll
index 22f45a2c0bd6c..2532154cfc15c 100644
--- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/xor.ll
+++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/llvm-vector-reduce/xor.ll
@@ -20,7 +20,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK-DAG: %[[LongVec3:.*]] = OpTypeVector %[[Long]] 3
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[CharVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseXor %[[CharVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2CharR:.*]] = OpCompositeExtract %[[Char]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2CharR]]
@@ -37,7 +37,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[ShortVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseXor %[[ShortVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2ShortR:.*]] = OpCompositeExtract %[[Short]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2ShortR]]
@@ -54,7 +54,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[IntVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseXor %[[IntVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2IntR:.*]] = OpCompositeExtract %[[Int]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2IntR]]
@@ -71,7 +71,7 @@ target triple = "spir64-unknown-unknown"
 ; CHECK: OpFunctionEnd
 
 ; CHECK: OpFunction
-; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 -1
+; CHECK: %[[Shuffle1:.*]] = OpVectorShuffle %[[LongVec2]] %[[#]] %[[#]] 1 0xFFFFFFFF
 ; CHECK: %[[Added1:.*]] = OpBitwiseXor %[[LongVec2]] %[[#]] %[[#]]
 ; CHECK: %[[Vec2LongR:.*]] = OpCompositeExtract %[[Long]] %[[Added1]] 0
 ; CHECK: OpReturnValue %[[Vec2LongR]]
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
index c088f6d862e7c..9271dc73e2725 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
@@ -363,10 +363,11 @@ define i128 @atomicrmw_uinc_wrap(ptr %src, i128 %b) {
 define i128 @atomicrmw_udec_wrap(ptr %src, i128 %b) {
 ; CHECK-LABEL: atomicrmw_udec_wrap:
 ; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, .LCPI12_0
 ; CHECK-NEXT:    vl %v0, 0(%r4), 3
 ; CHECK-NEXT:    vl %v3, 0(%r3), 4
-; CHECK-NEXT:    vgbm %v1, 65535
-; CHECK-NEXT:    vgbm %v2, 0
+; CHECK-NEXT:    vl %v1, 0(%r1), 3
+; CHECK-NEXT:    vgbm %v2, 65535
 ; CHECK-NEXT:    j .LBB12_2
 ; CHECK-NEXT:  .LBB12_1: # %atomicrmw.start
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
@@ -379,6 +380,9 @@ define i128 @atomicrmw_udec_wrap(ptr %src, i128 %b) {
 ; CHECK-NEXT:    je .LBB12_8
 ; CHECK-NEXT:  .LBB12_2: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vscbiq %v4, %v3, %v1
+; CHECK-NEXT:    vlgvf %r0, %v4, 3
+; CHECK-NEXT:    xilf %r0, 1
 ; CHECK-NEXT:    veclg %v0, %v3
 ; CHECK-NEXT:    jlh .LBB12_4
 ; CHECK-NEXT:  # %bb.3: # %atomicrmw.start
@@ -390,12 +394,11 @@ define i128 @atomicrmw_udec_wrap(ptr %src, i128 %b) {
 ; CHECK-NEXT:    jl .LBB12_6
 ; CHECK-NEXT:  # %bb.5: # %atomicrmw.start
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
-; CHECK-NEXT:    vaq %v4, %v3, %v1
+; CHECK-NEXT:    vaq %v4, %v3, %v2
 ; CHECK-NEXT:  .LBB12_6: # %atomicrmw.start
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
-; CHECK-NEXT:    vceqgs %v5, %v3, %v2
 ; CHECK-NEXT:    vlr %v5, %v0
-; CHECK-NEXT:    je .LBB12_1
+; CHECK-NEXT:    cijlh %r0, 0, .LBB12_1
 ; CHECK-NEXT:  # %bb.7: # %atomicrmw.start
 ; CHECK-NEXT:    # in Loop: Header=BB12_2 Depth=1
 ; CHECK-NEXT:    vlr %v5, %v4
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-65.ll b/llvm/test/CodeGen/SystemZ/int-cmp-65.ll
index b06ab3c1fa3d3..f1d850200fe41 100644
--- a/llvm/test/CodeGen/SystemZ/int-cmp-65.ll
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-65.ll
@@ -1,42 +1,110 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; Test usage of VACC/VSCBI.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s --check-prefix=BASELINE
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s --check-prefix=Z13
 
 define i128 @i128_subc_1(i128 %a, i128 %b) unnamed_addr {
-; CHECK-LABEL: i128_subc_1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl %v0, 0(%r4), 3
-; CHECK-NEXT:    vl %v1, 0(%r3), 3
-; CHECK-NEXT:    vscbiq %v0, %v1, %v0
-; CHECK-NEXT:    vst %v0, 0(%r2), 3
-; CHECK-NEXT:    br %r14
+; BASELINE-LABEL: i128_subc_1:
+; BASELINE:       # %bb.0:
+; BASELINE-NEXT:    stmg %r14, %r15, 112(%r15)
+; BASELINE-NEXT:    .cfi_offset %r14, -48
+; BASELINE-NEXT:    .cfi_offset %r15, -40
+; BASELINE-NEXT:    lg %r5, 0(%r4)
+; BASELINE-NEXT:    lg %r14, 0(%r3)
+; BASELINE-NEXT:    lg %r1, 8(%r3)
+; BASELINE-NEXT:    clgr %r14, %r5
+; BASELINE-NEXT:    ipm %r0
+; BASELINE-NEXT:    clg %r1, 8(%r4)
+; BASELINE-NEXT:    ipm %r1
+; BASELINE-NEXT:    cgrjlh %r14, %r5, .LBB0_2
+; BASELINE-NEXT:  # %bb.1:
+; BASELINE-NEXT:    xilf %r1, 4294967295
+; BASELINE-NEXT:    risbg %r0, %r1, 63, 191, 36
+; BASELINE-NEXT:    j .LBB0_3
+; BASELINE-NEXT:  .LBB0_2:
+; BASELINE-NEXT:    xilf %r0, 4294967295
+; BASELINE-NEXT:    risbg %r0, %r0, 63, 191, 36
+; BASELINE-NEXT:  .LBB0_3:
+; BASELINE-NEXT:    llgfr %r0, %r0
+; BASELINE-NEXT:    stg %r0, 8(%r2)
+; BASELINE-NEXT:    mvghi 0(%r2), 0
+; BASELINE-NEXT:    lmg %r14, %r15, 112(%r15)
+; BASELINE-NEXT:    br %r14
+;
+; Z13-LABEL: i128_subc_1:
+; Z13:       # %bb.0:
+; Z13-NEXT:    vl %v0, 0(%r4), 3
+; Z13-NEXT:    vl %v1, 0(%r3), 3
+; Z13-NEXT:    vscbiq %v0, %v1, %v0
+; Z13-NEXT:    vst %v0, 0(%r2), 3
+; Z13-NEXT:    br %r14
   %cmp = icmp uge i128 %a, %b
   %ext = zext i1 %cmp to i128
   ret i128 %ext
 }
 
 define i128 @i128_subc_2(i128 %a, i128 %b) unnamed_addr {
-; CHECK-LABEL: i128_subc_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl %v0, 0(%r3), 3
-; CHECK-NEXT:    vl %v1, 0(%r4), 3
-; CHECK-NEXT:    vscbiq %v0, %v1, %v0
-; CHECK-NEXT:    vst %v0, 0(%r2), 3
-; CHECK-NEXT:    br %r14
+; BASELINE-LABEL: i128_subc_2:
+; BASELINE:       # %bb.0:
+; BASELINE-NEXT:    stmg %r14, %r15, 112(%r15)
+; BASELINE-NEXT:    .cfi_offset %r14, -48
+; BASELINE-NEXT:    .cfi_offset %r15, -40
+; BASELINE-NEXT:    lg %r5, 0(%r4)
+; BASELINE-NEXT:    lg %r14, 0(%r3)
+; BASELINE-NEXT:    lg %r0, 8(%r3)
+; BASELINE-NEXT:    clgr %r14, %r5
+; BASELINE-NEXT:    ipm %r1
+; BASELINE-NEXT:    clg %r0, 8(%r4)
+; BASELINE-NEXT:    ipm %r0
+; BASELINE-NEXT:    cgrjlh %r14, %r5, .LBB1_2
+; BASELINE-NEXT:  # %bb.1:
+; BASELINE-NEXT:    afi %r0, -536870912
+; BASELINE-NEXT:    srl %r0, 31
+; BASELINE-NEXT:    j .LBB1_3
+; BASELINE-NEXT:  .LBB1_2:
+; BASELINE-NEXT:    afi %r1, -536870912
+; BASELINE-NEXT:    srl %r1, 31
+; BASELINE-NEXT:    lr %r0, %r1
+; BASELINE-NEXT:  .LBB1_3:
+; BASELINE-NEXT:    llgfr %r0, %r0
+; BASELINE-NEXT:    stg %r0, 8(%r2)
+; BASELINE-NEXT:    mvghi 0(%r2), 0
+; BASELINE-NEXT:    lmg %r14, %r15, 112(%r15)
+; BASELINE-NEXT:    br %r14
+;
+; Z13-LABEL: i128_subc_2:
+; Z13:       # %bb.0:
+; Z13-NEXT:    vl %v0, 0(%r3), 3
+; Z13-NEXT:    vl %v1, 0(%r4), 3
+; Z13-NEXT:    vscbiq %v0, %v1, %v0
+; Z13-NEXT:    vst %v0, 0(%r2), 3
+; Z13-NEXT:    br %r14
   %cmp = icmp ule i128 %a, %b
   %ext = zext i1 %cmp to i128
   ret i128 %ext
 }
 
 define i128 @i128_addc_1(i128 %a, i128 %b) {
-; CHECK-LABEL: i128_addc_1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl %v0, 0(%r4), 3
-; CHECK-NEXT:    vl %v1, 0(%r3), 3
-; CHECK-NEXT:    vaccq %v0, %v1, %v0
-; CHECK-NEXT:    vst %v0, 0(%r2), 3
-; CHECK-NEXT:    br %r14
+; BASELINE-LABEL: i128_addc_1:
+; BASELINE:       # %bb.0:
+; BASELINE-NEXT:    lg %r0, 8(%r3)
+; BASELINE-NEXT:    lg %r1, 0(%r3)
+; BASELINE-NEXT:    alg %r0, 8(%r4)
+; BASELINE-NEXT:    alcg %r1, 0(%r4)
+; BASELINE-NEXT:    ipm %r0
+; BASELINE-NEXT:    risbg %r0, %r0, 63, 191, 35
+; BASELINE-NEXT:    stg %r0, 8(%r2)
+; BASELINE-NEXT:    mvghi 0(%r2), 0
+; BASELINE-NEXT:    br %r14
+;
+; Z13-LABEL: i128_addc_1:
+; Z13:       # %bb.0:
+; Z13-NEXT:    vl %v0, 0(%r4), 3
+; Z13-NEXT:    vl %v1, 0(%r3), 3
+; Z13-NEXT:    vaccq %v0, %v1, %v0
+; Z13-NEXT:    vst %v0, 0(%r2), 3
+; Z13-NEXT:    br %r14
   %sum = add i128 %a, %b
   %cmp = icmp ult i128 %sum, %a
   %ext = zext i1 %cmp to i128
@@ -44,13 +112,25 @@ define i128 @i128_addc_1(i128 %a, i128 %b) {
 }
 
 define i128 @i128_addc_2(i128 %a, i128 %b) {
-; CHECK-LABEL: i128_addc_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl %v0, 0(%r4), 3
-; CHECK-NEXT:    vl %v1, 0(%r3), 3
-; CHECK-NEXT:    vaccq %v0, %v1, %v0
-; CHECK-NEXT:    vst %v0, 0(%r2), 3
-; CHECK-NEXT:    br %r14
+; BASELINE-LABEL: i128_addc_2:
+; BASELINE:       # %bb.0:
+; BASELINE-NEXT:    lg %r0, 8(%r3)
+; BASELINE-NEXT:    lg %r1, 0(%r3)
+; BASELINE-NEXT:    alg %r0, 8(%r4)
+; BASELINE-NEXT:    alcg %r1, 0(%r4)
+; BASELINE-NEXT:    ipm %r0
+; BASELINE-NEXT:    risbg %r0, %r0, 63, 191, 35
+; BASELINE-NEXT:    stg %r0, 8(%r2)
+; BASELINE-NEXT:    mvghi 0(%r2), 0
+; BASELINE-NEXT:    br %r14
+;
+; Z13-LABEL: i128_addc_2:
+; Z13:       # %bb.0:
+; Z13-NEXT:    vl %v0, 0(%r4), 3
+; Z13-NEXT:    vl %v1, 0(%r3), 3
+; Z13-NEXT:    vaccq %v0, %v1, %v0
+; Z13-NEXT:    vst %v0, 0(%r2), 3
+; Z13-NEXT:    br %r14
   %sum = add i128 %a, %b
   %cmp = icmp ult i128 %sum, %b
   %ext = zext i1 %cmp to i128
@@ -58,13 +138,25 @@ define i128 @i128_addc_2(i128 %a, i128 %b) {
 }
 
 define i128 @i128_addc_3(i128 %a, i128 %b) {
-; CHECK-LABEL: i128_addc_3:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl %v0, 0(%r4), 3
-; CHECK-NEXT:    vl %v1, 0(%r3), 3
-; CHECK-NEXT:    vaccq %v0, %v1, %v0
-; CHECK-NEXT:    vst %v0, 0(%r2), 3
-; CHECK-NEXT:    br %r14
+; BASELINE-LABEL: i128_addc_3:
+; BASELINE:       # %bb.0:
+; BASELINE-NEXT:    lg %r0, 8(%r3)
+; BASELINE-NEXT:    lg %r1, 0(%r3)
+; BASELINE-NEXT:    alg %r0, 8(%r4)
+; BASELINE-NEXT:    alcg %r1, 0(%r4)
+; BASELINE-NEXT:    ipm %r0
+; BASELINE-NEXT:    risbg %r0, %r0, 63, 191, 35
+; BASELINE-NEXT:    stg %r0, 8(%r2)
+; BASELINE-NEXT:    mvghi 0(%r2), 0
+; BASELINE-NEXT:    br %r14
+;
+; Z13-LABEL: i128_addc_3:
+; Z13:       # %bb.0:
+; Z13-NEXT:    vl %v0, 0(%r4), 3
+; Z13-NEXT:    vl %v1, 0(%r3), 3
+; Z13-NEXT:    vaccq %v0, %v1, %v0
+; Z13-NEXT:    vst %v0, 0(%r2), 3
+; Z13-NEXT:    br %r14
   %sum = add i128 %a, %b
   %cmp = icmp ugt i128 %a, %sum
   %ext = zext i1 %cmp to i128
@@ -72,16 +164,97 @@ define i128 @i128_addc_3(i128 %a, i128 %b) {
 }
 
 define i128 @i128_addc_4(i128 %a, i128 %b) {
-; CHECK-LABEL: i128_addc_4:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl %v0, 0(%r4), 3
-; CHECK-NEXT:    vl %v1, 0(%r3), 3
-; CHECK-NEXT:    vaccq %v0, %v1, %v0
-; CHECK-NEXT:    vst %v0, 0(%r2), 3
-; CHECK-NEXT:    br %r14
+; BASELINE-LABEL: i128_addc_4:
+; BASELINE:       # %bb.0:
+; BASELINE-NEXT:    lg %r0, 8(%r3)
+; BASELINE-NEXT:    lg %r1, 0(%r3)
+; BASELINE-NEXT:    alg %r0, 8(%r4)
+; BASELINE-NEXT:    alcg %r1, 0(%r4)
+; BASELINE-NEXT:    ipm %r0
+; BASELINE-NEXT:    risbg %r0, %r0, 63, 191, 35
+; BASELINE-NEXT:    stg %r0, 8(%r2)
+; BASELINE-NEXT:    mvghi 0(%r2), 0
+; BASELINE-NEXT:    br %r14
+;
+; Z13-LABEL: i128_addc_4:
+; Z13:       # %bb.0:
+; Z13-NEXT:    vl %v0, 0(%r4), 3
+; Z13-NEXT:    vl %v1, 0(%r3), 3
+; Z13-NEXT:    vaccq %v0, %v1, %v0
+; Z13-NEXT:    vst %v0, 0(%r2), 3
+; Z13-NEXT:    br %r14
   %sum = add i128 %a, %b
   %cmp = icmp ugt i128 %b, %sum
   %ext = zext i1 %cmp to i128
   ret i128 %ext
 }
 
+define i128 @i128_addc_xor(i128 %a, i128 %b) {
+; BASELINE-LABEL: i128_addc_xor:
+; BASELINE:       # %bb.0:
+; BASELINE-NEXT:    lg %r0, 8(%r4)
+; BASELINE-NEXT:    lg %r1, 0(%r4)
+; BASELINE-NEXT:    alg %r0, 8(%r3)
+; BASELINE-NEXT:    alcg %r1, 0(%r3)
+; BASELINE-NEXT:    ipm %r0
+; BASELINE-NEXT:    risbg %r0, %r0, 63, 191, 35
+; BASELINE-NEXT:    stg %r0, 8(%r2)
+; BASELINE-NEXT:    mvghi 0(%r2), 0
+; BASELINE-NEXT:    br %r14
+;
+; Z13-LABEL: i128_addc_xor:
+; Z13:       # %bb.0:
+; Z13-NEXT:    vl %v0, 0(%r3), 3
+; Z13-NEXT:    vl %v1, 0(%r4), 3
+; Z13-NEXT:    vaccq %v0, %v1, %v0
+; Z13-NEXT:    vst %v0, 0(%r2), 3
+; Z13-NEXT:    br %r14
+  %b.not = xor i128 %b, -1
+  %cmp = icmp ugt i128 %a, %b.not
+  %ext = zext i1 %cmp to i128
+  ret i128 %ext
+}
+
+define i128 @i128_addc_xor_inv(i128 %a, i128 %b) {
+; BASELINE-LABEL: i128_addc_xor_inv:
+; BASELINE:       # %bb.0:
+; BASELINE-NEXT:    stmg %r14, %r15, 112(%r15)
+; BASELINE-NEXT:    .cfi_offset %r14, -48
+; BASELINE-NEXT:    .cfi_offset %r15, -40
+; BASELINE-NEXT:    lg %r5, 0(%r3)
+; BASELINE-NEXT:    lghi %r14, -1
+; BASELINE-NEXT:    xg %r14, 0(%r4)
+; BASELINE-NEXT:    lghi %r1, -1
+; BASELINE-NEXT:    xg %r1, 8(%r4)
+; BASELINE-NEXT:    clgr %r5, %r14
+; BASELINE-NEXT:    ipm %r0
+; BASELINE-NEXT:    clg %r1, 8(%r3)
+; BASELINE-NEXT:    ipm %r1
+; BASELINE-NEXT:    cgrjlh %r5, %r14, .LBB7_2
+; BASELINE-NEXT:  # %bb.1:
+; BASELINE-NEXT:    xilf %r1, 4294967295
+; BASELINE-NEXT:    risbg %r0, %r1, 63, 191, 36
+; BASELINE-NEXT:    j .LBB7_3
+; BASELINE-NEXT:  .LBB7_2:
+; BASELINE-NEXT:    afi %r0, -536870912
+; BASELINE-NEXT:    srl %r0, 31
+; BASELINE-NEXT:  .LBB7_3:
+; BASELINE-NEXT:    llgfr %r0, %r0
+; BASELINE-NEXT:    stg %r0, 8(%r2)
+; BASELINE-NEXT:    mvghi 0(%r2), 0
+; BASELINE-NEXT:    lmg %r14, %r15, 112(%r15)
+; BASELINE-NEXT:    br %r14
+;
+; Z13-LABEL: i128_addc_xor_inv:
+; Z13:       # %bb.0:
+; Z13-NEXT:    vl %v1, 0(%r4), 3
+; Z13-NEXT:    vl %v0, 0(%r3), 3
+; Z13-NEXT:    vno %v1, %v1, %v1
+; Z13-NEXT:    vscbiq %v0, %v1, %v0
+; Z13-NEXT:    vst %v0, 0(%r2), 3
+; Z13-NEXT:    br %r14
+  %b.not = xor i128 %b, -1
+  %cmp = icmp ule i128 %a, %b.not
+  %ext = zext i1 %cmp to i128
+  ret i128 %ext
+}
diff --git a/llvm/test/CodeGen/Thumb/scmp.ll b/llvm/test/CodeGen/Thumb/scmp.ll
index 661dbe97cdb3c..c0024492b3a6d 100644
--- a/llvm/test/CodeGen/Thumb/scmp.ll
+++ b/llvm/test/CodeGen/Thumb/scmp.ll
@@ -1,151 +1,420 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=thumbv7-apple-darwin %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv6m-eabi %s -o - | FileCheck %s -check-prefix=THUMB1
+; RUN: llc -mtriple=thumbv7-apple-darwin %s -o - | FileCheck %s -check-prefix=THUMB2
+; RUN: llc -mtriple thumbv8.1m.main-none-eabi -o - %s | FileCheck %s --check-prefix=V81M
 
 define i8 @scmp_8_8(i8 signext %x, i8 signext %y) nounwind {
-; CHECK-LABEL: scmp_8_8:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r2, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: scmp_8_8:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    movs r2, #1
+; THUMB1-NEXT:    movs r3, #0
+; THUMB1-NEXT:    cmp r0, r1
+; THUMB1-NEXT:    mov r0, r2
+; THUMB1-NEXT:    bge .LBB0_3
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    ble .LBB0_4
+; THUMB1-NEXT:  .LBB0_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:  .LBB0_3:
+; THUMB1-NEXT:    mov r0, r3
+; THUMB1-NEXT:    bgt .LBB0_2
+; THUMB1-NEXT:  .LBB0_4:
+; THUMB1-NEXT:    mov r2, r3
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    bx lr
+;
+; THUMB2-LABEL: scmp_8_8:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs r0, r0, r1
+; THUMB2-NEXT:    it gt
+; THUMB2-NEXT:    movgt r0, #1
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r0, #-1
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: scmp_8_8:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    cmp r0, r1
+; V81M-NEXT:    cset r0, gt
+; V81M-NEXT:    it lt
+; V81M-NEXT:    movlt.w r0, #-1
+; V81M-NEXT:    bx lr
   %1 = call i8 @llvm.scmp(i8 %x, i8 %y)
   ret i8 %1
 }
 
 define i8 @scmp_8_16(i16 signext %x, i16 signext %y) nounwind {
-; CHECK-LABEL: scmp_8_16:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r2, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: scmp_8_16:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    movs r2, #1
+; THUMB1-NEXT:    movs r3, #0
+; THUMB1-NEXT:    cmp r0, r1
+; THUMB1-NEXT:    mov r0, r2
+; THUMB1-NEXT:    bge .LBB1_3
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    ble .LBB1_4
+; THUMB1-NEXT:  .LBB1_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:  .LBB1_3:
+; THUMB1-NEXT:    mov r0, r3
+; THUMB1-NEXT:    bgt .LBB1_2
+; THUMB1-NEXT:  .LBB1_4:
+; THUMB1-NEXT:    mov r2, r3
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    bx lr
+;
+; THUMB2-LABEL: scmp_8_16:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs r0, r0, r1
+; THUMB2-NEXT:    it gt
+; THUMB2-NEXT:    movgt r0, #1
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r0, #-1
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: scmp_8_16:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    cmp r0, r1
+; V81M-NEXT:    cset r0, gt
+; V81M-NEXT:    it lt
+; V81M-NEXT:    movlt.w r0, #-1
+; V81M-NEXT:    bx lr
   %1 = call i8 @llvm.scmp(i16 %x, i16 %y)
   ret i8 %1
 }
 
 define i8 @scmp_8_32(i32 %x, i32 %y) nounwind {
-; CHECK-LABEL: scmp_8_32:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r2, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: scmp_8_32:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    movs r2, #1
+; THUMB1-NEXT:    movs r3, #0
+; THUMB1-NEXT:    cmp r0, r1
+; THUMB1-NEXT:    mov r0, r2
+; THUMB1-NEXT:    bge .LBB2_3
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    ble .LBB2_4
+; THUMB1-NEXT:  .LBB2_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:  .LBB2_3:
+; THUMB1-NEXT:    mov r0, r3
+; THUMB1-NEXT:    bgt .LBB2_2
+; THUMB1-NEXT:  .LBB2_4:
+; THUMB1-NEXT:    mov r2, r3
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    bx lr
+;
+; THUMB2-LABEL: scmp_8_32:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs r0, r0, r1
+; THUMB2-NEXT:    it gt
+; THUMB2-NEXT:    movgt r0, #1
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r0, #-1
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: scmp_8_32:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    cmp r0, r1
+; V81M-NEXT:    cset r0, gt
+; V81M-NEXT:    it lt
+; V81M-NEXT:    movlt.w r0, #-1
+; V81M-NEXT:    bx lr
   %1 = call i8 @llvm.scmp(i32 %x, i32 %y)
   ret i8 %1
 }
 
 define i8 @scmp_8_64(i64 %x, i64 %y) nounwind {
-; CHECK-LABEL: scmp_8_64:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    subs.w r12, r0, r2
-; CHECK-NEXT:    mov.w r9, #0
-; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt.w r12, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    sbcs.w r0, r3, r1
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt.w r9, #1
-; CHECK-NEXT:    sub.w r0, r9, r12
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: scmp_8_64:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    .save {r4, r5, r6, lr}
+; THUMB1-NEXT:    push {r4, r5, r6, lr}
+; THUMB1-NEXT:    movs r4, #1
+; THUMB1-NEXT:    movs r5, #0
+; THUMB1-NEXT:    subs r6, r0, r2
+; THUMB1-NEXT:    mov r6, r1
+; THUMB1-NEXT:    sbcs r6, r3
+; THUMB1-NEXT:    mov r6, r4
+; THUMB1-NEXT:    blt .LBB3_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    mov r6, r5
+; THUMB1-NEXT:  .LBB3_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    sbcs r3, r1
+; THUMB1-NEXT:    blt .LBB3_4
+; THUMB1-NEXT:  @ %bb.3:
+; THUMB1-NEXT:    mov r4, r5
+; THUMB1-NEXT:  .LBB3_4:
+; THUMB1-NEXT:    subs r0, r4, r6
+; THUMB1-NEXT:    pop {r4, r5, r6, pc}
+;
+; THUMB2-LABEL: scmp_8_64:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs.w r12, r0, r2
+; THUMB2-NEXT:    mov.w r9, #0
+; THUMB2-NEXT:    sbcs.w r12, r1, r3
+; THUMB2-NEXT:    mov.w r12, #0
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r12, #1
+; THUMB2-NEXT:    subs r0, r2, r0
+; THUMB2-NEXT:    sbcs.w r0, r3, r1
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r9, #1
+; THUMB2-NEXT:    sub.w r0, r9, r12
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: scmp_8_64:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    subs.w r12, r0, r2
+; V81M-NEXT:    sbcs.w r12, r1, r3
+; V81M-NEXT:    cset r12, lt
+; V81M-NEXT:    subs r0, r2, r0
+; V81M-NEXT:    sbcs.w r0, r3, r1
+; V81M-NEXT:    cset r0, lt
+; V81M-NEXT:    sub.w r0, r0, r12
+; V81M-NEXT:    bx lr
   %1 = call i8 @llvm.scmp(i64 %x, i64 %y)
   ret i8 %1
 }
 
 define i8 @scmp_8_128(i128 %x, i128 %y) nounwind {
-; CHECK-LABEL: scmp_8_128:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    add.w lr, sp, #16
-; CHECK-NEXT:    ldr r4, [sp, #28]
-; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    ldm.w lr, {r9, r12, lr}
-; CHECK-NEXT:    subs.w r6, r0, r9
-; CHECK-NEXT:    sbcs.w r6, r1, r12
-; CHECK-NEXT:    sbcs.w r6, r2, lr
-; CHECK-NEXT:    sbcs.w r6, r3, r4
-; CHECK-NEXT:    mov.w r6, #0
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r6, #1
-; CHECK-NEXT:    subs.w r0, r9, r0
-; CHECK-NEXT:    sbcs.w r0, r12, r1
-; CHECK-NEXT:    sbcs.w r0, lr, r2
-; CHECK-NEXT:    sbcs.w r0, r4, r3
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r5, #1
-; CHECK-NEXT:    subs r0, r5, r6
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; THUMB1-LABEL: scmp_8_128:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    .save {r4, r5, r6, r7, lr}
+; THUMB1-NEXT:    push {r4, r5, r6, r7, lr}
+; THUMB1-NEXT:    .pad #20
+; THUMB1-NEXT:    sub sp, #20
+; THUMB1-NEXT:    str r3, [sp, #16] @ 4-byte Spill
+; THUMB1-NEXT:    movs r3, #1
+; THUMB1-NEXT:    str r3, [sp] @ 4-byte Spill
+; THUMB1-NEXT:    movs r3, #0
+; THUMB1-NEXT:    str r3, [sp, #12] @ 4-byte Spill
+; THUMB1-NEXT:    ldr r6, [sp, #52]
+; THUMB1-NEXT:    add r7, sp, #40
+; THUMB1-NEXT:    ldm r7, {r3, r5, r7}
+; THUMB1-NEXT:    subs r4, r0, r3
+; THUMB1-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; THUMB1-NEXT:    mov r4, r1
+; THUMB1-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r4, r5
+; THUMB1-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; THUMB1-NEXT:    mov r4, r2
+; THUMB1-NEXT:    sbcs r4, r7
+; THUMB1-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r4, r6
+; THUMB1-NEXT:    mov r2, r1
+; THUMB1-NEXT:    blt .LBB4_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; THUMB1-NEXT:  .LBB4_2:
+; THUMB1-NEXT:    subs r0, r3, r0
+; THUMB1-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r5, r0
+; THUMB1-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r7, r0
+; THUMB1-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r6, r0
+; THUMB1-NEXT:    blt .LBB4_4
+; THUMB1-NEXT:  @ %bb.3:
+; THUMB1-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; THUMB1-NEXT:  .LBB4_4:
+; THUMB1-NEXT:    subs r0, r1, r2
+; THUMB1-NEXT:    add sp, #20
+; THUMB1-NEXT:    pop {r4, r5, r6, r7, pc}
+;
+; THUMB2-LABEL: scmp_8_128:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    push {r4, r5, r6, lr}
+; THUMB2-NEXT:    add.w lr, sp, #16
+; THUMB2-NEXT:    ldr r4, [sp, #28]
+; THUMB2-NEXT:    movs r5, #0
+; THUMB2-NEXT:    ldm.w lr, {r9, r12, lr}
+; THUMB2-NEXT:    subs.w r6, r0, r9
+; THUMB2-NEXT:    sbcs.w r6, r1, r12
+; THUMB2-NEXT:    sbcs.w r6, r2, lr
+; THUMB2-NEXT:    sbcs.w r6, r3, r4
+; THUMB2-NEXT:    mov.w r6, #0
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt r6, #1
+; THUMB2-NEXT:    subs.w r0, r9, r0
+; THUMB2-NEXT:    sbcs.w r0, r12, r1
+; THUMB2-NEXT:    sbcs.w r0, lr, r2
+; THUMB2-NEXT:    sbcs.w r0, r4, r3
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt r5, #1
+; THUMB2-NEXT:    subs r0, r5, r6
+; THUMB2-NEXT:    pop {r4, r5, r6, pc}
+;
+; V81M-LABEL: scmp_8_128:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    .save {r4, r5, r6, lr}
+; V81M-NEXT:    push {r4, r5, r6, lr}
+; V81M-NEXT:    ldrd r5, r4, [sp, #16]
+; V81M-NEXT:    ldrd lr, r12, [sp, #24]
+; V81M-NEXT:    subs r6, r0, r5
+; V81M-NEXT:    sbcs.w r6, r1, r4
+; V81M-NEXT:    sbcs.w r6, r2, lr
+; V81M-NEXT:    sbcs.w r6, r3, r12
+; V81M-NEXT:    cset r6, lt
+; V81M-NEXT:    subs r0, r5, r0
+; V81M-NEXT:    sbcs.w r0, r4, r1
+; V81M-NEXT:    sbcs.w r0, lr, r2
+; V81M-NEXT:    sbcs.w r0, r12, r3
+; V81M-NEXT:    cset r0, lt
+; V81M-NEXT:    subs r0, r0, r6
+; V81M-NEXT:    pop {r4, r5, r6, pc}
   %1 = call i8 @llvm.scmp(i128 %x, i128 %y)
   ret i8 %1
 }
 
 define i32 @scmp_32_32(i32 %x, i32 %y) nounwind {
-; CHECK-LABEL: scmp_32_32:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt r0, #1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r2, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: scmp_32_32:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    movs r2, #1
+; THUMB1-NEXT:    movs r3, #0
+; THUMB1-NEXT:    cmp r0, r1
+; THUMB1-NEXT:    mov r0, r2
+; THUMB1-NEXT:    bge .LBB5_3
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    ble .LBB5_4
+; THUMB1-NEXT:  .LBB5_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    bx lr
+; THUMB1-NEXT:  .LBB5_3:
+; THUMB1-NEXT:    mov r0, r3
+; THUMB1-NEXT:    bgt .LBB5_2
+; THUMB1-NEXT:  .LBB5_4:
+; THUMB1-NEXT:    mov r2, r3
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    bx lr
+;
+; THUMB2-LABEL: scmp_32_32:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs r0, r0, r1
+; THUMB2-NEXT:    it gt
+; THUMB2-NEXT:    movgt r0, #1
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r0, #-1
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: scmp_32_32:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    cmp r0, r1
+; V81M-NEXT:    cset r0, gt
+; V81M-NEXT:    it lt
+; V81M-NEXT:    movlt.w r0, #-1
+; V81M-NEXT:    bx lr
   %1 = call i32 @llvm.scmp(i32 %x, i32 %y)
   ret i32 %1
 }
 
 define i32 @scmp_32_64(i64 %x, i64 %y) nounwind {
-; CHECK-LABEL: scmp_32_64:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    subs.w r12, r0, r2
-; CHECK-NEXT:    mov.w r9, #0
-; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt.w r12, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    sbcs.w r0, r3, r1
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt.w r9, #1
-; CHECK-NEXT:    sub.w r0, r9, r12
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: scmp_32_64:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    .save {r4, r5, r6, lr}
+; THUMB1-NEXT:    push {r4, r5, r6, lr}
+; THUMB1-NEXT:    movs r4, #1
+; THUMB1-NEXT:    movs r5, #0
+; THUMB1-NEXT:    subs r6, r0, r2
+; THUMB1-NEXT:    mov r6, r1
+; THUMB1-NEXT:    sbcs r6, r3
+; THUMB1-NEXT:    mov r6, r4
+; THUMB1-NEXT:    blt .LBB6_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    mov r6, r5
+; THUMB1-NEXT:  .LBB6_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    sbcs r3, r1
+; THUMB1-NEXT:    blt .LBB6_4
+; THUMB1-NEXT:  @ %bb.3:
+; THUMB1-NEXT:    mov r4, r5
+; THUMB1-NEXT:  .LBB6_4:
+; THUMB1-NEXT:    subs r0, r4, r6
+; THUMB1-NEXT:    pop {r4, r5, r6, pc}
+;
+; THUMB2-LABEL: scmp_32_64:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs.w r12, r0, r2
+; THUMB2-NEXT:    mov.w r9, #0
+; THUMB2-NEXT:    sbcs.w r12, r1, r3
+; THUMB2-NEXT:    mov.w r12, #0
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r12, #1
+; THUMB2-NEXT:    subs r0, r2, r0
+; THUMB2-NEXT:    sbcs.w r0, r3, r1
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r9, #1
+; THUMB2-NEXT:    sub.w r0, r9, r12
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: scmp_32_64:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    subs.w r12, r0, r2
+; V81M-NEXT:    sbcs.w r12, r1, r3
+; V81M-NEXT:    cset r12, lt
+; V81M-NEXT:    subs r0, r2, r0
+; V81M-NEXT:    sbcs.w r0, r3, r1
+; V81M-NEXT:    cset r0, lt
+; V81M-NEXT:    sub.w r0, r0, r12
+; V81M-NEXT:    bx lr
   %1 = call i32 @llvm.scmp(i64 %x, i64 %y)
   ret i32 %1
 }
 
 define i64 @scmp_64_64(i64 %x, i64 %y) nounwind {
-; CHECK-LABEL: scmp_64_64:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    subs.w r12, r0, r2
-; CHECK-NEXT:    mov.w r9, #0
-; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt.w r12, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    sbcs.w r0, r3, r1
-; CHECK-NEXT:    it lt
-; CHECK-NEXT:    movlt.w r9, #1
-; CHECK-NEXT:    sub.w r0, r9, r12
-; CHECK-NEXT:    asrs r1, r0, #31
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: scmp_64_64:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    .save {r4, r5, r6, lr}
+; THUMB1-NEXT:    push {r4, r5, r6, lr}
+; THUMB1-NEXT:    movs r4, #1
+; THUMB1-NEXT:    movs r5, #0
+; THUMB1-NEXT:    subs r6, r0, r2
+; THUMB1-NEXT:    mov r6, r1
+; THUMB1-NEXT:    sbcs r6, r3
+; THUMB1-NEXT:    mov r6, r4
+; THUMB1-NEXT:    blt .LBB7_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    mov r6, r5
+; THUMB1-NEXT:  .LBB7_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    sbcs r3, r1
+; THUMB1-NEXT:    blt .LBB7_4
+; THUMB1-NEXT:  @ %bb.3:
+; THUMB1-NEXT:    mov r4, r5
+; THUMB1-NEXT:  .LBB7_4:
+; THUMB1-NEXT:    subs r0, r4, r6
+; THUMB1-NEXT:    asrs r1, r0, #31
+; THUMB1-NEXT:    pop {r4, r5, r6, pc}
+;
+; THUMB2-LABEL: scmp_64_64:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs.w r12, r0, r2
+; THUMB2-NEXT:    mov.w r9, #0
+; THUMB2-NEXT:    sbcs.w r12, r1, r3
+; THUMB2-NEXT:    mov.w r12, #0
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r12, #1
+; THUMB2-NEXT:    subs r0, r2, r0
+; THUMB2-NEXT:    sbcs.w r0, r3, r1
+; THUMB2-NEXT:    it lt
+; THUMB2-NEXT:    movlt.w r9, #1
+; THUMB2-NEXT:    sub.w r0, r9, r12
+; THUMB2-NEXT:    asrs r1, r0, #31
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: scmp_64_64:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    subs.w r12, r0, r2
+; V81M-NEXT:    sbcs.w r12, r1, r3
+; V81M-NEXT:    cset r12, lt
+; V81M-NEXT:    subs r0, r2, r0
+; V81M-NEXT:    sbcs.w r0, r3, r1
+; V81M-NEXT:    cset r0, lt
+; V81M-NEXT:    sub.w r0, r0, r12
+; V81M-NEXT:    asrs r1, r0, #31
+; V81M-NEXT:    bx lr
   %1 = call i64 @llvm.scmp(i64 %x, i64 %y)
   ret i64 %1
 }
diff --git a/llvm/test/CodeGen/Thumb/ucmp.ll b/llvm/test/CodeGen/Thumb/ucmp.ll
index 7e6d0a323b11c..5d0f57e2a9d72 100644
--- a/llvm/test/CodeGen/Thumb/ucmp.ll
+++ b/llvm/test/CodeGen/Thumb/ucmp.ll
@@ -1,151 +1,376 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=thumbv7-apple-darwin %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv6m-eabi %s -o - | FileCheck %s -check-prefix=THUMB1
+; RUN: llc -mtriple=thumbv7-apple-darwin %s -o - | FileCheck %s -check-prefix=THUMB2
+; RUN: llc -mtriple thumbv8.1m.main-none-eabi -o - %s | FileCheck %s --check-prefix=V81M
 
 define i8 @ucmp_8_8(i8 zeroext %x, i8 zeroext %y) nounwind {
-; CHECK-LABEL: ucmp_8_8:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    it hi
-; CHECK-NEXT:    movhi r2, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: ucmp_8_8:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    subs r2, r0, r1
+; THUMB1-NEXT:    sbcs r2, r2
+; THUMB1-NEXT:    cmp r1, r0
+; THUMB1-NEXT:    sbcs r1, r1
+; THUMB1-NEXT:    subs r0, r2, r1
+; THUMB1-NEXT:    bx lr
+;
+; THUMB2-LABEL: ucmp_8_8:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs r0, r0, r1
+; THUMB2-NEXT:    it hi
+; THUMB2-NEXT:    movhi r0, #1
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r0, #-1
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: ucmp_8_8:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    cmp r0, r1
+; V81M-NEXT:    cset r0, hi
+; V81M-NEXT:    it lo
+; V81M-NEXT:    movlo.w r0, #-1
+; V81M-NEXT:    bx lr
   %1 = call i8 @llvm.ucmp(i8 %x, i8 %y)
   ret i8 %1
 }
 
 define i8 @ucmp_8_16(i16 zeroext %x, i16 zeroext %y) nounwind {
-; CHECK-LABEL: ucmp_8_16:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    it hi
-; CHECK-NEXT:    movhi r2, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: ucmp_8_16:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    subs r2, r0, r1
+; THUMB1-NEXT:    sbcs r2, r2
+; THUMB1-NEXT:    cmp r1, r0
+; THUMB1-NEXT:    sbcs r1, r1
+; THUMB1-NEXT:    subs r0, r2, r1
+; THUMB1-NEXT:    bx lr
+;
+; THUMB2-LABEL: ucmp_8_16:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs r0, r0, r1
+; THUMB2-NEXT:    it hi
+; THUMB2-NEXT:    movhi r0, #1
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r0, #-1
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: ucmp_8_16:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    cmp r0, r1
+; V81M-NEXT:    cset r0, hi
+; V81M-NEXT:    it lo
+; V81M-NEXT:    movlo.w r0, #-1
+; V81M-NEXT:    bx lr
   %1 = call i8 @llvm.ucmp(i16 %x, i16 %y)
   ret i8 %1
 }
 
 define i8 @ucmp_8_32(i32 %x, i32 %y) nounwind {
-; CHECK-LABEL: ucmp_8_32:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    it hi
-; CHECK-NEXT:    movhi r2, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: ucmp_8_32:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    subs r2, r0, r1
+; THUMB1-NEXT:    sbcs r2, r2
+; THUMB1-NEXT:    cmp r1, r0
+; THUMB1-NEXT:    sbcs r1, r1
+; THUMB1-NEXT:    subs r0, r2, r1
+; THUMB1-NEXT:    bx lr
+;
+; THUMB2-LABEL: ucmp_8_32:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs r0, r0, r1
+; THUMB2-NEXT:    it hi
+; THUMB2-NEXT:    movhi r0, #1
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r0, #-1
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: ucmp_8_32:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    cmp r0, r1
+; V81M-NEXT:    cset r0, hi
+; V81M-NEXT:    it lo
+; V81M-NEXT:    movlo.w r0, #-1
+; V81M-NEXT:    bx lr
   %1 = call i8 @llvm.ucmp(i32 %x, i32 %y)
   ret i8 %1
 }
 
 define i8 @ucmp_8_64(i64 %x, i64 %y) nounwind {
-; CHECK-LABEL: ucmp_8_64:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    subs.w r12, r0, r2
-; CHECK-NEXT:    mov.w r9, #0
-; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo.w r12, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    sbcs.w r0, r3, r1
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo.w r9, #1
-; CHECK-NEXT:    sub.w r0, r9, r12
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: ucmp_8_64:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    .save {r4, r5, r6, lr}
+; THUMB1-NEXT:    push {r4, r5, r6, lr}
+; THUMB1-NEXT:    movs r4, #1
+; THUMB1-NEXT:    movs r5, #0
+; THUMB1-NEXT:    subs r6, r0, r2
+; THUMB1-NEXT:    mov r6, r1
+; THUMB1-NEXT:    sbcs r6, r3
+; THUMB1-NEXT:    mov r6, r4
+; THUMB1-NEXT:    blo .LBB3_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    mov r6, r5
+; THUMB1-NEXT:  .LBB3_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    sbcs r3, r1
+; THUMB1-NEXT:    blo .LBB3_4
+; THUMB1-NEXT:  @ %bb.3:
+; THUMB1-NEXT:    mov r4, r5
+; THUMB1-NEXT:  .LBB3_4:
+; THUMB1-NEXT:    subs r0, r4, r6
+; THUMB1-NEXT:    pop {r4, r5, r6, pc}
+;
+; THUMB2-LABEL: ucmp_8_64:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs.w r12, r0, r2
+; THUMB2-NEXT:    mov.w r9, #0
+; THUMB2-NEXT:    sbcs.w r12, r1, r3
+; THUMB2-NEXT:    mov.w r12, #0
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r12, #1
+; THUMB2-NEXT:    subs r0, r2, r0
+; THUMB2-NEXT:    sbcs.w r0, r3, r1
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r9, #1
+; THUMB2-NEXT:    sub.w r0, r9, r12
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: ucmp_8_64:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    subs.w r12, r0, r2
+; V81M-NEXT:    sbcs.w r12, r1, r3
+; V81M-NEXT:    cset r12, lo
+; V81M-NEXT:    subs r0, r2, r0
+; V81M-NEXT:    sbcs.w r0, r3, r1
+; V81M-NEXT:    cset r0, lo
+; V81M-NEXT:    sub.w r0, r0, r12
+; V81M-NEXT:    bx lr
   %1 = call i8 @llvm.ucmp(i64 %x, i64 %y)
   ret i8 %1
 }
 
 define i8 @ucmp_8_128(i128 %x, i128 %y) nounwind {
-; CHECK-LABEL: ucmp_8_128:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    push {r4, r5, r6, lr}
-; CHECK-NEXT:    add.w lr, sp, #16
-; CHECK-NEXT:    ldr r4, [sp, #28]
-; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    ldm.w lr, {r9, r12, lr}
-; CHECK-NEXT:    subs.w r6, r0, r9
-; CHECK-NEXT:    sbcs.w r6, r1, r12
-; CHECK-NEXT:    sbcs.w r6, r2, lr
-; CHECK-NEXT:    sbcs.w r6, r3, r4
-; CHECK-NEXT:    mov.w r6, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r6, #1
-; CHECK-NEXT:    subs.w r0, r9, r0
-; CHECK-NEXT:    sbcs.w r0, r12, r1
-; CHECK-NEXT:    sbcs.w r0, lr, r2
-; CHECK-NEXT:    sbcs.w r0, r4, r3
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r5, #1
-; CHECK-NEXT:    subs r0, r5, r6
-; CHECK-NEXT:    pop {r4, r5, r6, pc}
+; THUMB1-LABEL: ucmp_8_128:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    .save {r4, r5, r6, r7, lr}
+; THUMB1-NEXT:    push {r4, r5, r6, r7, lr}
+; THUMB1-NEXT:    .pad #20
+; THUMB1-NEXT:    sub sp, #20
+; THUMB1-NEXT:    str r3, [sp, #16] @ 4-byte Spill
+; THUMB1-NEXT:    movs r3, #1
+; THUMB1-NEXT:    str r3, [sp] @ 4-byte Spill
+; THUMB1-NEXT:    movs r3, #0
+; THUMB1-NEXT:    str r3, [sp, #12] @ 4-byte Spill
+; THUMB1-NEXT:    ldr r6, [sp, #52]
+; THUMB1-NEXT:    add r7, sp, #40
+; THUMB1-NEXT:    ldm r7, {r3, r5, r7}
+; THUMB1-NEXT:    subs r4, r0, r3
+; THUMB1-NEXT:    str r1, [sp, #4] @ 4-byte Spill
+; THUMB1-NEXT:    mov r4, r1
+; THUMB1-NEXT:    ldr r1, [sp] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r4, r5
+; THUMB1-NEXT:    str r2, [sp, #8] @ 4-byte Spill
+; THUMB1-NEXT:    mov r4, r2
+; THUMB1-NEXT:    sbcs r4, r7
+; THUMB1-NEXT:    ldr r4, [sp, #16] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r4, r6
+; THUMB1-NEXT:    mov r2, r1
+; THUMB1-NEXT:    blo .LBB4_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    ldr r2, [sp, #12] @ 4-byte Reload
+; THUMB1-NEXT:  .LBB4_2:
+; THUMB1-NEXT:    subs r0, r3, r0
+; THUMB1-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r5, r0
+; THUMB1-NEXT:    ldr r0, [sp, #8] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r7, r0
+; THUMB1-NEXT:    ldr r0, [sp, #16] @ 4-byte Reload
+; THUMB1-NEXT:    sbcs r6, r0
+; THUMB1-NEXT:    blo .LBB4_4
+; THUMB1-NEXT:  @ %bb.3:
+; THUMB1-NEXT:    ldr r1, [sp, #12] @ 4-byte Reload
+; THUMB1-NEXT:  .LBB4_4:
+; THUMB1-NEXT:    subs r0, r1, r2
+; THUMB1-NEXT:    add sp, #20
+; THUMB1-NEXT:    pop {r4, r5, r6, r7, pc}
+;
+; THUMB2-LABEL: ucmp_8_128:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    push {r4, r5, r6, lr}
+; THUMB2-NEXT:    add.w lr, sp, #16
+; THUMB2-NEXT:    ldr r4, [sp, #28]
+; THUMB2-NEXT:    movs r5, #0
+; THUMB2-NEXT:    ldm.w lr, {r9, r12, lr}
+; THUMB2-NEXT:    subs.w r6, r0, r9
+; THUMB2-NEXT:    sbcs.w r6, r1, r12
+; THUMB2-NEXT:    sbcs.w r6, r2, lr
+; THUMB2-NEXT:    sbcs.w r6, r3, r4
+; THUMB2-NEXT:    mov.w r6, #0
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo r6, #1
+; THUMB2-NEXT:    subs.w r0, r9, r0
+; THUMB2-NEXT:    sbcs.w r0, r12, r1
+; THUMB2-NEXT:    sbcs.w r0, lr, r2
+; THUMB2-NEXT:    sbcs.w r0, r4, r3
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo r5, #1
+; THUMB2-NEXT:    subs r0, r5, r6
+; THUMB2-NEXT:    pop {r4, r5, r6, pc}
+;
+; V81M-LABEL: ucmp_8_128:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    .save {r4, r5, r6, lr}
+; V81M-NEXT:    push {r4, r5, r6, lr}
+; V81M-NEXT:    ldrd r5, r4, [sp, #16]
+; V81M-NEXT:    ldrd lr, r12, [sp, #24]
+; V81M-NEXT:    subs r6, r0, r5
+; V81M-NEXT:    sbcs.w r6, r1, r4
+; V81M-NEXT:    sbcs.w r6, r2, lr
+; V81M-NEXT:    sbcs.w r6, r3, r12
+; V81M-NEXT:    cset r6, lo
+; V81M-NEXT:    subs r0, r5, r0
+; V81M-NEXT:    sbcs.w r0, r4, r1
+; V81M-NEXT:    sbcs.w r0, lr, r2
+; V81M-NEXT:    sbcs.w r0, r12, r3
+; V81M-NEXT:    cset r0, lo
+; V81M-NEXT:    subs r0, r0, r6
+; V81M-NEXT:    pop {r4, r5, r6, pc}
   %1 = call i8 @llvm.ucmp(i128 %x, i128 %y)
   ret i8 %1
 }
 
 define i32 @ucmp_32_32(i32 %x, i32 %y) nounwind {
-; CHECK-LABEL: ucmp_32_32:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r0, #0
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo r0, #1
-; CHECK-NEXT:    it hi
-; CHECK-NEXT:    movhi r2, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: ucmp_32_32:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    subs r2, r0, r1
+; THUMB1-NEXT:    sbcs r2, r2
+; THUMB1-NEXT:    cmp r1, r0
+; THUMB1-NEXT:    sbcs r1, r1
+; THUMB1-NEXT:    subs r0, r2, r1
+; THUMB1-NEXT:    bx lr
+;
+; THUMB2-LABEL: ucmp_32_32:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs r0, r0, r1
+; THUMB2-NEXT:    it hi
+; THUMB2-NEXT:    movhi r0, #1
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r0, #-1
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: ucmp_32_32:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    cmp r0, r1
+; V81M-NEXT:    cset r0, hi
+; V81M-NEXT:    it lo
+; V81M-NEXT:    movlo.w r0, #-1
+; V81M-NEXT:    bx lr
   %1 = call i32 @llvm.ucmp(i32 %x, i32 %y)
   ret i32 %1
 }
 
 define i32 @ucmp_32_64(i64 %x, i64 %y) nounwind {
-; CHECK-LABEL: ucmp_32_64:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    subs.w r12, r0, r2
-; CHECK-NEXT:    mov.w r9, #0
-; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo.w r12, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    sbcs.w r0, r3, r1
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo.w r9, #1
-; CHECK-NEXT:    sub.w r0, r9, r12
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: ucmp_32_64:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    .save {r4, r5, r6, lr}
+; THUMB1-NEXT:    push {r4, r5, r6, lr}
+; THUMB1-NEXT:    movs r4, #1
+; THUMB1-NEXT:    movs r5, #0
+; THUMB1-NEXT:    subs r6, r0, r2
+; THUMB1-NEXT:    mov r6, r1
+; THUMB1-NEXT:    sbcs r6, r3
+; THUMB1-NEXT:    mov r6, r4
+; THUMB1-NEXT:    blo .LBB6_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    mov r6, r5
+; THUMB1-NEXT:  .LBB6_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    sbcs r3, r1
+; THUMB1-NEXT:    blo .LBB6_4
+; THUMB1-NEXT:  @ %bb.3:
+; THUMB1-NEXT:    mov r4, r5
+; THUMB1-NEXT:  .LBB6_4:
+; THUMB1-NEXT:    subs r0, r4, r6
+; THUMB1-NEXT:    pop {r4, r5, r6, pc}
+;
+; THUMB2-LABEL: ucmp_32_64:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs.w r12, r0, r2
+; THUMB2-NEXT:    mov.w r9, #0
+; THUMB2-NEXT:    sbcs.w r12, r1, r3
+; THUMB2-NEXT:    mov.w r12, #0
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r12, #1
+; THUMB2-NEXT:    subs r0, r2, r0
+; THUMB2-NEXT:    sbcs.w r0, r3, r1
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r9, #1
+; THUMB2-NEXT:    sub.w r0, r9, r12
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: ucmp_32_64:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    subs.w r12, r0, r2
+; V81M-NEXT:    sbcs.w r12, r1, r3
+; V81M-NEXT:    cset r12, lo
+; V81M-NEXT:    subs r0, r2, r0
+; V81M-NEXT:    sbcs.w r0, r3, r1
+; V81M-NEXT:    cset r0, lo
+; V81M-NEXT:    sub.w r0, r0, r12
+; V81M-NEXT:    bx lr
   %1 = call i32 @llvm.ucmp(i64 %x, i64 %y)
   ret i32 %1
 }
 
 define i64 @ucmp_64_64(i64 %x, i64 %y) nounwind {
-; CHECK-LABEL: ucmp_64_64:
-; CHECK:       @ %bb.0:
-; CHECK-NEXT:    subs.w r12, r0, r2
-; CHECK-NEXT:    mov.w r9, #0
-; CHECK-NEXT:    sbcs.w r12, r1, r3
-; CHECK-NEXT:    mov.w r12, #0
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo.w r12, #1
-; CHECK-NEXT:    subs r0, r2, r0
-; CHECK-NEXT:    sbcs.w r0, r3, r1
-; CHECK-NEXT:    it lo
-; CHECK-NEXT:    movlo.w r9, #1
-; CHECK-NEXT:    sub.w r0, r9, r12
-; CHECK-NEXT:    asrs r1, r0, #31
-; CHECK-NEXT:    bx lr
+; THUMB1-LABEL: ucmp_64_64:
+; THUMB1:       @ %bb.0:
+; THUMB1-NEXT:    .save {r4, r5, r6, lr}
+; THUMB1-NEXT:    push {r4, r5, r6, lr}
+; THUMB1-NEXT:    movs r4, #1
+; THUMB1-NEXT:    movs r5, #0
+; THUMB1-NEXT:    subs r6, r0, r2
+; THUMB1-NEXT:    mov r6, r1
+; THUMB1-NEXT:    sbcs r6, r3
+; THUMB1-NEXT:    mov r6, r4
+; THUMB1-NEXT:    blo .LBB7_2
+; THUMB1-NEXT:  @ %bb.1:
+; THUMB1-NEXT:    mov r6, r5
+; THUMB1-NEXT:  .LBB7_2:
+; THUMB1-NEXT:    subs r0, r2, r0
+; THUMB1-NEXT:    sbcs r3, r1
+; THUMB1-NEXT:    blo .LBB7_4
+; THUMB1-NEXT:  @ %bb.3:
+; THUMB1-NEXT:    mov r4, r5
+; THUMB1-NEXT:  .LBB7_4:
+; THUMB1-NEXT:    subs r0, r4, r6
+; THUMB1-NEXT:    asrs r1, r0, #31
+; THUMB1-NEXT:    pop {r4, r5, r6, pc}
+;
+; THUMB2-LABEL: ucmp_64_64:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    subs.w r12, r0, r2
+; THUMB2-NEXT:    mov.w r9, #0
+; THUMB2-NEXT:    sbcs.w r12, r1, r3
+; THUMB2-NEXT:    mov.w r12, #0
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r12, #1
+; THUMB2-NEXT:    subs r0, r2, r0
+; THUMB2-NEXT:    sbcs.w r0, r3, r1
+; THUMB2-NEXT:    it lo
+; THUMB2-NEXT:    movlo.w r9, #1
+; THUMB2-NEXT:    sub.w r0, r9, r12
+; THUMB2-NEXT:    asrs r1, r0, #31
+; THUMB2-NEXT:    bx lr
+;
+; V81M-LABEL: ucmp_64_64:
+; V81M:       @ %bb.0:
+; V81M-NEXT:    subs.w r12, r0, r2
+; V81M-NEXT:    sbcs.w r12, r1, r3
+; V81M-NEXT:    cset r12, lo
+; V81M-NEXT:    subs r0, r2, r0
+; V81M-NEXT:    sbcs.w r0, r3, r1
+; V81M-NEXT:    cset r0, lo
+; V81M-NEXT:    sub.w r0, r0, r12
+; V81M-NEXT:    asrs r1, r0, #31
+; V81M-NEXT:    bx lr
   %1 = call i64 @llvm.ucmp(i64 %x, i64 %y)
   ret i64 %1
 }
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/add_reduce.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/add_reduce.mir
index 870ee341a4fcd..f6738c741742d 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/add_reduce.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/add_reduce.mir
@@ -98,8 +98,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 24, size: 4, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/begin-vpt-without-inst.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/begin-vpt-without-inst.mir
index c5f9d32536688..8fae4b675c48b 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/begin-vpt-without-inst.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/begin-vpt-without-inst.mir
@@ -43,8 +43,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-default.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-default.mir
index 647270bc0aad7..384f0c0ec427f 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-default.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-default.mir
@@ -133,8 +133,8 @@ frameInfo:
   stackSize:       76
   offsetAdjustment: 0
   maxAlignment:    4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -40, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize-strd-lr.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize-strd-lr.mir
index d4bc80dde2545..295d6bb6a7328 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize-strd-lr.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize-strd-lr.mir
@@ -136,8 +136,8 @@ frameInfo:
   stackSize:       68
   offsetAdjustment: 0
   maxAlignment:    4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -40, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize.mir
index e7f64cac75b33..0b8af09fbd8f1 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/biquad-cascade-optsize.mir
@@ -138,8 +138,8 @@ frameInfo:
   offsetAdjustment: 0
   maxAlignment:    4
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -40, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-mov.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-mov.mir
index 615dd3f476545..acedecad9cb25 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-mov.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-mov.mir
@@ -69,8 +69,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/disjoint-vcmp.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/disjoint-vcmp.mir
index 9c2434da4b791..4278567248402 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/disjoint-vcmp.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/disjoint-vcmp.mir
@@ -95,8 +95,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -20, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/dont-ignore-vctp.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/dont-ignore-vctp.mir
index 59f5c8e4f14a3..f46f506faae23 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/dont-ignore-vctp.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/dont-ignore-vctp.mir
@@ -72,8 +72,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/dont-remove-loop-update.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/dont-remove-loop-update.mir
index 18fc66eeb262c..15f63965ff48e 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/dont-remove-loop-update.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/dont-remove-loop-update.mir
@@ -88,8 +88,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/emptyblock.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/emptyblock.mir
index 4998b5bafc133..021cb4c89f301 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/emptyblock.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/emptyblock.mir
@@ -300,8 +300,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -76, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/end-positive-offset.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/end-positive-offset.mir
index 9ebb714bc4eea..b373bfc50d326 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/end-positive-offset.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/end-positive-offset.mir
@@ -82,8 +82,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/extract-element.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/extract-element.mir
index cd5292310f168..da5bcee31fd03 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/extract-element.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/extract-element.mir
@@ -89,8 +89,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-16.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-16.mir
index d95db905283c9..cdb7a908c2ee2 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-16.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-16.mir
@@ -80,8 +80,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-32.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-32.mir
index 35effea71350d..6af2d379b5126 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-32.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-32.mir
@@ -88,8 +88,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-8.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-8.mir
index 71f8f20a37f78..3f0d4257241d0 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-8.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/incorrect-sub-8.mir
@@ -81,8 +81,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-1.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-1.mir
index 40b557a5e6dec..70d5d313c02f0 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-1.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-1.mir
@@ -102,8 +102,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-2.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-2.mir
index 380c6957c3b1c..2b3eddf7596f5 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-2.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-2.mir
@@ -102,8 +102,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-3.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-3.mir
index a81f2a557180f..088cdd1a1a85f 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-3.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpnot-3.mir
@@ -102,8 +102,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpsel-1.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpsel-1.mir
index 45f0371660a56..d59ebac12db36 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpsel-1.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpsel-1.mir
@@ -103,8 +103,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 4, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpsel-2.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpsel-2.mir
index 78a05c38b46fa..f477ace3ae3a7 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpsel-2.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/inloop-vpsel-2.mir
@@ -105,8 +105,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 4, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-chain-store.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-chain-store.mir
index 0c6c3330991d8..b691aefa29109 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-chain-store.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-chain-store.mir
@@ -111,8 +111,8 @@ frameInfo:
   offsetAdjustment: 0
   maxAlignment:    4
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -216,8 +216,8 @@ frameInfo:
   stackSize:       8
   offsetAdjustment: 0
   maxAlignment:    4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-chain.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-chain.mir
index 6f021dae228b7..8ae62eab147e0 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-chain.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-chain.mir
@@ -74,8 +74,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-itercount.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-itercount.mir
index 1e1e2a4dda3e7..710ace106d393 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-itercount.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-itercount.mir
@@ -75,8 +75,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-random.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-random.mir
index ea8a8a5b2ab06..251758fa34df2 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-random.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/it-block-random.mir
@@ -76,8 +76,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-chain.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-chain.mir
index 9f027f9164752..6eb70c2ff9d4d 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-chain.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-chain.mir
@@ -182,8 +182,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -40, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-prev-iteration.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-prev-iteration.mir
index e0fd23130baf6..8296f7fce56f3 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-prev-iteration.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-copy-prev-iteration.mir
@@ -183,8 +183,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -40, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-liveout.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-liveout.mir
index 6b29018d28d63..ffc5383ab8032 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-liveout.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/loop-dec-liveout.mir
@@ -183,8 +183,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -40, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/lstp-insertion-position.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/lstp-insertion-position.mir
index 6c67084dd02df..f5da7acb3e970 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/lstp-insertion-position.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/lstp-insertion-position.mir
@@ -118,8 +118,8 @@ frameInfo:
   offsetAdjustment: 0
   maxAlignment:    4
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -256,8 +256,8 @@ frameInfo:
   stackSize:       8
   offsetAdjustment: 0
   maxAlignment:    4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/massive.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/massive.mir
index 9448a1ab00d3e..904aae45889a3 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/massive.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/massive.mir
@@ -87,8 +87,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/matrix-debug.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/matrix-debug.mir
index 2dfaf17326941..b7a62df3cb2aa 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/matrix-debug.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/matrix-debug.mir
@@ -168,8 +168,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/matrix.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/matrix.mir
index cbed22ff10ef8..c1336bab95290 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/matrix.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/matrix.mir
@@ -180,8 +180,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dls.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dls.mir
index 29406adc596f4..fca869caa7dbd 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dls.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dls.mir
@@ -69,8 +69,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dlstp.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dlstp.mir
index cc39f9850d959..c331612faf31f 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dlstp.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dlstp.mir
@@ -125,8 +125,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-lr-terminator.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-lr-terminator.mir
index 4f96e04f0fa89..62f6945a30965 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-lr-terminator.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-lr-terminator.mir
@@ -83,8 +83,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/move-def-before-start.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/move-def-before-start.mir
index 80dc97b52d79b..4261fda217a82 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/move-def-before-start.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/move-def-before-start.mir
@@ -88,8 +88,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/move-start-after-def.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/move-start-after-def.mir
index 9d9170f1e6adf..a64b0b13ee0bb 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/move-start-after-def.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/move-start-after-def.mir
@@ -88,8 +88,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiblock-massive.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiblock-massive.mir
index 06dae765c8e33..78c2ffa9ed83c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiblock-massive.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiblock-massive.mir
@@ -87,8 +87,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiple-do-loops.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiple-do-loops.mir
index 8e5172cadc329..f7f230d81950f 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiple-do-loops.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiple-do-loops.mir
@@ -316,8 +316,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -531,8 +531,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -751,8 +751,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-reduct-livein-arg.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-reduct-livein-arg.mir
index f5b341c381388..0ec0dd9a2af8d 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-reduct-livein-arg.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-reduct-livein-arg.mir
@@ -97,8 +97,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-cbnz.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-cbnz.mir
index 15719baece36f..0a5b47e679bcd 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-cbnz.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-cbnz.mir
@@ -99,8 +99,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-reorder.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-reorder.mir
index fc88475399a74..ceb256c050520 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-reorder.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-reorder.mir
@@ -98,8 +98,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec.mir
index a80f4e9ffae60..585c8bfbac5b2 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec.mir
@@ -102,8 +102,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-vpsel-liveout.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-vpsel-liveout.mir
index ab47e5a181eb0..4114fac4b17a0 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-vpsel-liveout.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-vpsel-liveout.mir
@@ -86,8 +86,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/non-masked-load.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/non-masked-load.mir
index 5279f13bfc876..206e40b484f65 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/non-masked-load.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/non-masked-load.mir
@@ -91,8 +91,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/non-masked-store.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/non-masked-store.mir
index 1b7fec49238ab..824a212e152a2 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/non-masked-store.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/non-masked-store.mir
@@ -83,8 +83,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/out-of-range-cbz.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/out-of-range-cbz.mir
index eb10cd7692cc4..747197066b698 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/out-of-range-cbz.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/out-of-range-cbz.mir
@@ -144,8 +144,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/remove-elem-moves.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/remove-elem-moves.mir
index 345fec361c69c..95ff548ffee9a 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/remove-elem-moves.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/remove-elem-moves.mir
@@ -118,8 +118,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.mir
index 1fe361d478a3e..8ccba6c16822c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.mir
@@ -71,8 +71,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-read.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-read.mir
index cd590c98894e9..8a3384e9d5ece 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-read.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-read.mir
@@ -67,8 +67,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-write.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-write.mir
index 78d52eb4106d9..a466b3d91a2d1 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-write.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-write.mir
@@ -73,8 +73,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-header.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-header.mir
index eec1c3973923f..cbfa646c942fd 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-header.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-header.mir
@@ -136,8 +136,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir
index 1c1d21daf0375..fbfadf408a5d0 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir
@@ -89,8 +89,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-while.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-while.mir
index 3a55b4905ec56..ddda91f12fb52 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-while.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-while.mir
@@ -77,8 +77,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-def-no-mov.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-def-no-mov.mir
index 4ee131d1d46ec..492b99db5c8db 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-def-no-mov.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-def-no-mov.mir
@@ -74,8 +74,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-retaining.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-retaining.mir
index 8e172f1553fc8..cc97588f5c958 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-retaining.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-retaining.mir
@@ -98,7 +98,7 @@ frameInfo:
   stackSize:       8
   offsetAdjustment: 0
   maxAlignment:    4
-  restorePoint:    ''
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 4, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/size-limit.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/size-limit.mir
index 8406720f02928..dcbc203899ba7 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/size-limit.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/size-limit.mir
@@ -87,8 +87,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-debug.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-debug.mir
index 43287b3c5f03b..6d8ae46ea1a6c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-debug.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-debug.mir
@@ -145,8 +145,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-vpt-debug.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-vpt-debug.mir
index a11d5e70876ab..17ef52a002163 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-vpt-debug.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/skip-vpt-debug.mir
@@ -156,8 +156,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/switch.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/switch.mir
index 9b369cb38fe1a..b73794954576f 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/switch.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/switch.mir
@@ -101,8 +101,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unrolled-and-vector.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unrolled-and-vector.mir
index 0b50c2df4c658..bf08a2bb6e678 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unrolled-and-vector.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unrolled-and-vector.mir
@@ -197,8 +197,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-def.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-def.mir
index f2eef49c02f9c..06f010ce7c158 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-def.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-def.mir
@@ -74,8 +74,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-use.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-use.mir
index f847e6afd69ac..c5213d73aae09 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-use.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-cpsr-loop-use.mir
@@ -74,8 +74,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-use-after.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-use-after.mir
index ebae1717fcf81..1c7c0281d50ca 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-use-after.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-use-after.mir
@@ -72,8 +72,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vaddv.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vaddv.mir
index 9acbeb2c1ea07..2019e6bf98a3e 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vaddv.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vaddv.mir
@@ -828,8 +828,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -933,8 +933,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -1038,8 +1038,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -1142,8 +1142,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -1259,8 +1259,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -1376,8 +1376,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -1506,8 +1506,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -1623,8 +1623,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -1753,8 +1753,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 16, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -1882,8 +1882,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 8, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -2021,8 +2021,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 16, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -2149,8 +2149,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 8, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -2288,8 +2288,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 16, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -2416,8 +2416,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 8, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -2555,8 +2555,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 16, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -2683,8 +2683,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 8, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -2822,8 +2822,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -2945,8 +2945,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -3070,8 +3070,8 @@ frameInfo:
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-add-operand-liveout.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-add-operand-liveout.mir
index 08b71c43b51d3..7fa0651316957 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-add-operand-liveout.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-add-operand-liveout.mir
@@ -93,8 +93,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-in-vpt-2.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-in-vpt-2.mir
index d634e6a7ff7b8..ce3d2c8bfa7bc 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-in-vpt-2.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-in-vpt-2.mir
@@ -85,8 +85,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-in-vpt.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-in-vpt.mir
index 7164ff9a9a21e..853854a81bfeb 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-in-vpt.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-in-vpt.mir
@@ -116,8 +116,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4,
@@ -256,8 +256,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4,
@@ -404,8 +404,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4,
@@ -552,8 +552,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subi3.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subi3.mir
index 7d42b407517bd..2e410e06a84c5 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subi3.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subi3.mir
@@ -83,8 +83,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subri.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subri.mir
index 2b2999f81a2e8..15cda656a8b67 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subri.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subri.mir
@@ -82,8 +82,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subri12.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subri12.mir
index 9f8c6dab8c468..be01034473dd8 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subri12.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp-subri12.mir
@@ -82,8 +82,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp16-reduce.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp16-reduce.mir
index 74039faa45e8e..0bf06f689c5c5 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp16-reduce.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vctp16-reduce.mir
@@ -92,8 +92,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vmaxmin_vpred_r.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vmaxmin_vpred_r.mir
index c89ecd3d0bba8..692aa63e71579 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vmaxmin_vpred_r.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vmaxmin_vpred_r.mir
@@ -94,8 +94,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 24, size: 4, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vmldava_in_vpt.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vmldava_in_vpt.mir
index c260c3a89dc4c..966f8217d5408 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vmldava_in_vpt.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vmldava_in_vpt.mir
@@ -95,8 +95,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 20, size: 4, alignment: 4, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-block-debug.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-block-debug.mir
index 26e7b8041a233..4e2a275e213e4 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-block-debug.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-block-debug.mir
@@ -243,8 +243,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 8, size: 4, alignment: 8, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-blocks.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-blocks.mir
index 284282023170a..c8e35390ec1bb 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-blocks.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vpt-blocks.mir
@@ -195,8 +195,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -320,8 +320,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -454,8 +454,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -586,8 +586,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -709,8 +709,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -831,8 +831,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -948,8 +948,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-negative-offset.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-negative-offset.mir
index a6ae8bc75a99e..bea0c3bb6de57 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-negative-offset.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-negative-offset.mir
@@ -90,8 +90,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4, 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.mir
index bc739dea40ef6..5b494bead6371 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.mir
@@ -74,8 +74,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wlstp.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wlstp.mir
index 8076caa563a30..7b256fdc0efd3 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wlstp.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wlstp.mir
@@ -173,8 +173,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -295,8 +295,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
@@ -403,8 +403,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-liveout-lsr-shift.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-liveout-lsr-shift.mir
index 3e1fd796d6ed0..9f5233c13c763 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-liveout-lsr-shift.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-liveout-lsr-shift.mir
@@ -92,8 +92,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-vctp-opcode-liveout.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-vctp-opcode-liveout.mir
index 2aab1ed916e76..eb4b36ca560f0 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-vctp-opcode-liveout.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-vctp-opcode-liveout.mir
@@ -98,8 +98,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-vctp-operand-liveout.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-vctp-operand-liveout.mir
index b0a0ccbd66e12..630e84661d1fb 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-vctp-operand-liveout.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/wrong-vctp-operand-liveout.mir
@@ -90,8 +90,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/bti-pac-replace-1.mir b/llvm/test/CodeGen/Thumb2/bti-pac-replace-1.mir
index a043c42c8644c..0db3867dac8e8 100644
--- a/llvm/test/CodeGen/Thumb2/bti-pac-replace-1.mir
+++ b/llvm/test/CodeGen/Thumb2/bti-pac-replace-1.mir
@@ -44,8 +44,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/Thumb2/ifcvt-neon-deprecated.mir b/llvm/test/CodeGen/Thumb2/ifcvt-neon-deprecated.mir
index 1f5edb0c78b91..98f8604366fa3 100644
--- a/llvm/test/CodeGen/Thumb2/ifcvt-neon-deprecated.mir
+++ b/llvm/test/CodeGen/Thumb2/ifcvt-neon-deprecated.mir
@@ -55,8 +55,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/Thumb2/mve-vector-spill.ll b/llvm/test/CodeGen/Thumb2/mve-vector-spill.ll
index 0af77c4fc8283..dab0f85e39534 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vector-spill.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vector-spill.ll
@@ -8,15 +8,15 @@ define arm_aapcs_vfpcc void @spill_vector_i32(<4 x i32> %v, ptr %p) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #8] @ 16-byte Spill
 ; CHECK-NEXT:    bl external_function
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #8] @ 16-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   call void @external_function()
@@ -29,15 +29,15 @@ define arm_aapcs_vfpcc void @spill_vector_i16(<8 x i16> %v, ptr %p) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #8] @ 16-byte Spill
 ; CHECK-NEXT:    bl external_function
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #8] @ 16-byte Reload
 ; CHECK-NEXT:    vstrh.16 q0, [r0]
-; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   call void @external_function()
@@ -50,15 +50,15 @@ define arm_aapcs_vfpcc void @spill_vector_i8(<16 x i8> %v, ptr %p) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #8] @ 16-byte Spill
 ; CHECK-NEXT:    bl external_function
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #8] @ 16-byte Reload
 ; CHECK-NEXT:    vstrb.8 q0, [r0]
-; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   call void @external_function()
@@ -71,15 +71,15 @@ define arm_aapcs_vfpcc void @spill_vector_i64(<2 x i64> %v, ptr %p) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #8] @ 16-byte Spill
 ; CHECK-NEXT:    bl external_function
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #8] @ 16-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   call void @external_function()
@@ -92,15 +92,15 @@ define arm_aapcs_vfpcc void @spill_vector_f32(<4 x float> %v, ptr %p) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #8] @ 16-byte Spill
 ; CHECK-NEXT:    bl external_function
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #8] @ 16-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   call void @external_function()
@@ -113,15 +113,15 @@ define arm_aapcs_vfpcc void @spill_vector_f16(<8 x half> %v, ptr %p) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #8] @ 16-byte Spill
 ; CHECK-NEXT:    bl external_function
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #8] @ 16-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   call void @external_function()
@@ -134,15 +134,15 @@ define arm_aapcs_vfpcc void @spill_vector_f64(<2 x double> %v, ptr %p) {
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    .save {r7, lr}
 ; CHECK-NEXT:    push {r7, lr}
-; CHECK-NEXT:    .pad #40
-; CHECK-NEXT:    sub sp, #40
-; CHECK-NEXT:    str r0, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT:    vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT:    .pad #24
+; CHECK-NEXT:    sub sp, #24
+; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT:    vstrw.32 q0, [sp, #8] @ 16-byte Spill
 ; CHECK-NEXT:    bl external_function
-; CHECK-NEXT:    ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT:    vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT:    ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT:    vldrw.u32 q0, [sp, #8] @ 16-byte Reload
 ; CHECK-NEXT:    vstrw.32 q0, [r0]
-; CHECK-NEXT:    add sp, #40
+; CHECK-NEXT:    add sp, #24
 ; CHECK-NEXT:    pop {r7, pc}
 entry:
   call void @external_function()
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-1-pred.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-1-pred.mir
index 074c95986e6da..4e817ba957174 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-1-pred.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-1-pred.mir
@@ -53,8 +53,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -68,12 +68,12 @@ body:             |
     ; CHECK: successors: %bb.0(0x80000000)
     ; CHECK: liveins: $lr, $q0, $q1, $q2, $q3, $q4, $q5, $r0, $r1, $r2, $r7, $r8, $r9, $r10, $r11, $r12
     ; CHECK: renamable $r4 = t2ADDrr renamable $r2, renamable $r10, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: BUNDLE implicit-def $vpr, implicit-def $q6, implicit-def $d12, implicit-def $s24, implicit-def $s25, implicit-def $d13, implicit-def $s26, implicit-def $s27, implicit $q1, implicit $q5, implicit killed $r4 {
+    ; CHECK: BUNDLE implicit-def $vpr, implicit-def $q6, implicit $q1, implicit $q5, implicit killed $r4 {
     ; CHECK:   MVE_VPTv4u32 8, renamable $q1, renamable $q5, 2, implicit-def $vpr
     ; CHECK:   renamable $q6 = MVE_VLDRBU32 killed renamable $r4, 0, 1, internal renamable $vpr, $noreg
     ; CHECK: }
     ; CHECK: renamable $r4 = t2ADDrr renamable $r11, renamable $r10, 14 /* CC::al */, $noreg, $noreg
-    ; CHECK: BUNDLE implicit-def $q7, implicit-def $d14, implicit-def $s28, implicit-def $s29, implicit-def $d15, implicit-def $s30, implicit-def $s31, implicit killed $vpr, implicit killed $r4 {
+    ; CHECK: BUNDLE implicit-def $q7, implicit killed $vpr, implicit killed $r4 {
     ; CHECK:   MVE_VPST 8, implicit $vpr
     ; CHECK:   renamable $q7 = MVE_VLDRBU32 killed renamable $r4, 0, 1, killed renamable $vpr, $noreg
     ; CHECK: }
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-2-preds.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-2-preds.mir
index 567f23bed1af4..6b5cbcee6d467 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-2-preds.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-2-preds.mir
@@ -55,8 +55,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -68,12 +68,12 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2, $r0, $r1
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit killed $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
+    ; CHECK: BUNDLE implicit-def $q3, implicit killed $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
     ; CHECK:   MVE_VPST 8, implicit $vpr
     ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, killed renamable $vpr, $noreg, killed renamable $q3
     ; CHECK: }
     ; CHECK: $vpr = VMSR_P0 killed $r1, 14 /* CC::al */, $noreg
-    ; CHECK: BUNDLE implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $vpr, implicit killed $q3, implicit killed $q2, implicit killed $q0 {
+    ; CHECK: BUNDLE implicit-def $q0, implicit killed $vpr, implicit killed $q3, implicit killed $q2, implicit killed $q0 {
     ; CHECK:   MVE_VPST 8, implicit $vpr
     ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q3, killed renamable $q2, 1, killed renamable $vpr, $noreg, killed renamable $q0
     ; CHECK: }
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-ctrl-flow.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-ctrl-flow.mir
index a66f9f9393810..91ccf3b43ec0a 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-ctrl-flow.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-ctrl-flow.mir
@@ -57,8 +57,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -69,14 +69,14 @@ body:             |
   ; CHECK:   liveins: $q0, $q1, $q2, $r0
   ; CHECK:   $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
   ; CHECK:   $q3 = MVE_VORR $q0, $q0, 0, $noreg, $noreg, undef $q3
-  ; CHECK:   BUNDLE implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
+  ; CHECK:   BUNDLE implicit-def $q3, implicit-def $q1, implicit $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
   ; CHECK:     MVE_VPST 4, implicit $vpr
   ; CHECK:     renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, $noreg, killed renamable $q3
   ; CHECK:     renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal renamable $q3, internal renamable $q3, 1, renamable $vpr, $noreg, undef renamable $q1
   ; CHECK:   }
   ; CHECK: bb.1.bb2:
   ; CHECK:   liveins: $q0, $q1, $q2, $q3, $vpr
-  ; CHECK:   BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
+  ; CHECK:   BUNDLE implicit-def dead $q3, implicit-def $q0, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
   ; CHECK:     MVE_VPST 4, implicit $vpr
   ; CHECK:     renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, $noreg, killed renamable $q3
   ; CHECK:     renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, killed renamable $q2, 1, killed renamable $vpr, $noreg, killed renamable $q0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-non-consecutive-ins.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-non-consecutive-ins.mir
index 15508351ea33e..1f19ed9ccd144 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-non-consecutive-ins.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks-non-consecutive-ins.mir
@@ -55,8 +55,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -69,13 +69,13 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
+    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $q1, implicit $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
     ; CHECK:   MVE_VPST 4, implicit $vpr
     ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, internal renamable $q3, 1, renamable $vpr, $noreg, undef renamable $q1
     ; CHECK: }
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
+    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $q0, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
     ; CHECK:   MVE_VPST 4, implicit $vpr
     ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, killed renamable $q2, 1, killed renamable $vpr, $noreg, killed renamable $q0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks.mir
index 1f0b095eec2b3..4f75b01de2803 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-2-blocks.mir
@@ -57,8 +57,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -69,14 +69,14 @@ body:             |
     ; CHECK-LABEL: name: vpt_2_blocks
     ; CHECK: liveins: $q0, $q1, $q2, $q3, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
-    ; CHECK: BUNDLE implicit-def dead $q2, implicit-def $d4, implicit-def $s8, implicit-def $s9, implicit-def $d5, implicit-def $s10, implicit-def $s11, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit $vpr, implicit killed $q2, implicit $q3, implicit killed $q0 {
+    ; CHECK: BUNDLE implicit-def dead $q2, implicit-def $q0, implicit $vpr, implicit killed $q2, implicit $q3, implicit killed $q0 {
     ; CHECK:   MVE_VPST 1, implicit $vpr
     ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q2, renamable $q3, 1, renamable $vpr, $noreg, undef renamable $q2
     ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, internal renamable $q2, 1, renamable $vpr, $noreg, internal undef renamable $q2
     ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, renamable $q3, 1, renamable $vpr, $noreg, killed renamable $q0
     ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q0, renamable $q3, 1, renamable $vpr, $noreg, internal undef renamable $q0
     ; CHECK: }
-    ; CHECK: BUNDLE implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $vpr, implicit killed $q0, implicit killed $q3, implicit killed $q1 {
+    ; CHECK: BUNDLE implicit-def $q1, implicit killed $vpr, implicit killed $q0, implicit killed $q3, implicit killed $q1 {
     ; CHECK:   MVE_VPST 8, implicit $vpr
     ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q0, killed renamable $q3, 1, killed renamable $vpr, $noreg, killed renamable $q1
     ; CHECK: }
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir
index 40c333718ee74..c26838803e356 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-3-blocks-kill-vpr.mir
@@ -55,8 +55,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -68,13 +68,13 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
+    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $q1, implicit $vpr, implicit killed $q1, implicit $q2, implicit killed $q3 {
     ; CHECK:   MVE_VPST 12, implicit $vpr
     ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, internal renamable $q3, 2, renamable $vpr, $noreg, undef renamable $q1
     ; CHECK: }
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
+    ; CHECK: BUNDLE implicit-def dead $q3, implicit-def $q0, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
     ; CHECK:   MVE_VPST 4, implicit $vpr
     ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, killed renamable $q2, 1, killed renamable $vpr, $noreg, killed renamable $q0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block-1-ins.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block-1-ins.mir
index 67469e50a5c3b..1e9e0e3d21ca9 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block-1-ins.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block-1-ins.mir
@@ -53,8 +53,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -65,7 +65,7 @@ body:             |
     ; CHECK-LABEL: name: vpt_block_1_ins
     ; CHECK: liveins: $q0, $q1, $q2, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
-    ; CHECK: BUNDLE implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q0 {
+    ; CHECK: BUNDLE implicit-def $q0, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q0 {
     ; CHECK:   MVE_VPST 8, implicit $vpr
     ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, killed renamable $q2, 1, killed renamable $vpr, $noreg, killed renamable $q0
     ; CHECK: }
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block-2-ins.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block-2-ins.mir
index 54621bd307de6..cb73cdfc4bd55 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block-2-ins.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block-2-ins.mir
@@ -55,8 +55,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -67,7 +67,7 @@ body:             |
     ; CHECK-LABEL: name: vpt_block_2_ins
     ; CHECK: liveins: $q0, $q1, $q2, $q3, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
-    ; CHECK: BUNDLE implicit-def dead $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $vpr, implicit killed $q2, implicit killed $q3, implicit killed $q0, implicit killed $q1 {
+    ; CHECK: BUNDLE implicit-def dead $q0, implicit-def $q1, implicit killed $vpr, implicit killed $q2, implicit killed $q3, implicit killed $q0, implicit killed $q1 {
     ; CHECK:   MVE_VPST 4, implicit $vpr
     ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q2, renamable $q3, 1, renamable $vpr, $noreg, killed renamable $q0
     ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q0, killed renamable $q3, 1, killed renamable $vpr, $noreg, killed renamable $q1
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block-4-ins.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block-4-ins.mir
index 02e50a36284d3..62d7640d3ac5c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block-4-ins.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block-4-ins.mir
@@ -56,8 +56,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -68,7 +68,7 @@ body:             |
     ; CHECK-LABEL: name: vpt_block_4_ins
     ; CHECK: liveins: $q0, $q1, $q2, $q3, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
-    ; CHECK: BUNDLE implicit-def dead $q2, implicit-def $d4, implicit-def $s8, implicit-def $s9, implicit-def $d5, implicit-def $s10, implicit-def $s11, implicit-def dead $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $vpr, implicit killed $q2, implicit killed $q3, implicit killed $q0, implicit killed $q1 {
+    ; CHECK: BUNDLE implicit-def dead $q2, implicit-def dead $q0, implicit-def $q1, implicit killed $vpr, implicit killed $q2, implicit killed $q3, implicit killed $q0, implicit killed $q1 {
     ; CHECK:   MVE_VPST 1, implicit $vpr
     ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q2, renamable $q3, 1, renamable $vpr, $noreg, undef renamable $q2
     ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, internal renamable $q2, 1, renamable $vpr, $noreg, internal undef renamable $q2
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block-debug.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block-debug.mir
index ce42722215b88..bfff0ef735102 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block-debug.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block-debug.mir
@@ -81,7 +81,7 @@ body:             |
     ; CHECK-NEXT: DBG_VALUE $q2, $noreg, !19, !DIExpression(), debug-location !21
     ; CHECK-NEXT: DBG_VALUE $q2, $noreg, !19, !DIExpression(), debug-location !21
     ; CHECK-NEXT: DBG_VALUE $noreg, $noreg, !20, !DIExpression(), debug-location !21
-    ; CHECK-NEXT: BUNDLE implicit-def dead $vpr, implicit-def $q2, implicit-def $d4, implicit-def $s8, implicit-def $s9, implicit-def $d5, implicit-def $s10, implicit-def $s11, implicit killed $q1, implicit killed $q0, implicit killed $q2, debug-location !23 {
+    ; CHECK-NEXT: BUNDLE implicit-def dead $vpr, implicit-def $q2, implicit killed $q1, implicit killed $q0, implicit killed $q2, debug-location !23 {
     ; CHECK-NEXT:   MVE_VPTv4s32 12, renamable $q1, renamable $q0, 10, implicit-def $vpr, debug-location !23
     ; CHECK-NEXT:   renamable $q2 = MVE_VADDi32 renamable $q0, renamable $q1, 1, internal renamable $vpr, $noreg, killed renamable $q2, debug-location !23
     ; CHECK-NEXT:   DBG_VALUE $noreg, $noreg, !20, !DIExpression(), debug-location !21
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block-elses.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block-elses.mir
index aa46ea145f4e2..130c7f4b68de4 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block-elses.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block-elses.mir
@@ -56,8 +56,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -68,7 +68,7 @@ body:             |
     ; CHECK-LABEL: name: vpt_block_else
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit killed $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit killed $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
     ; CHECK:   MVE_VPTv4s32 7, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $vpr = MVE_VCMPs32r killed renamable $q0, $zr, 12, 1, internal killed renamable $vpr, $noreg
@@ -77,7 +77,7 @@ body:             |
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR $q3, $q3, 0, $noreg, $noreg, undef $q0
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit killed $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit killed $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
     ; CHECK:   MVE_VPTv4s32 7, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $vpr = MVE_VCMPs32r killed renamable $q0, $zr, 12, 1, internal killed renamable $vpr, $noreg
@@ -86,7 +86,7 @@ body:             |
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR $q3, $q3, 0, $noreg, $noreg, undef $q0
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
     ; CHECK:   MVE_VPTv4s32 15, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, $noreg, internal renamable $q3
@@ -95,7 +95,7 @@ body:             |
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, $noreg, undef $q0
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
     ; CHECK:   MVE_VPTv4s32 15, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 2, internal killed renamable $vpr, $noreg
@@ -104,7 +104,7 @@ body:             |
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, $noreg, undef $q0
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3 {
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3 {
     ; CHECK:   MVE_VPTv4s32 15, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, $noreg, internal renamable $q3
@@ -113,7 +113,7 @@ body:             |
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, $noreg, undef $q0
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3 {
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3 {
     ; CHECK:   MVE_VPTv4s32 14, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, $noreg, internal renamable $q3
@@ -121,7 +121,7 @@ body:             |
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, $noreg, undef $q0
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3, implicit $zr {
     ; CHECK:   MVE_VPTv4s32 14, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 2, internal killed renamable $vpr, $noreg
@@ -129,7 +129,7 @@ body:             |
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, $noreg, undef $q0
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3 {
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit $q0, implicit $q2, implicit $q1, implicit killed $q3 {
     ; CHECK:   MVE_VPTv4s32 6, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VMAXs32 renamable $q0, renamable $q1, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal renamable $vpr, $noreg, internal killed renamable $q3
@@ -137,7 +137,7 @@ body:             |
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, $noreg, undef $q0
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit killed $q3 {
+    ; CHECK: BUNDLE implicit-def $vpr, implicit-def $q3, implicit $q0, implicit $q2, implicit killed $q3 {
     ; CHECK:   MVE_VPTv4s32 11, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 2, internal renamable $vpr, $noreg, internal killed renamable $q3
@@ -146,7 +146,7 @@ body:             |
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q3, killed $q3, 0, $noreg, $noreg, undef $q0
     ; CHECK: $q3 = MVE_VORR $q2, $q2, 0, $noreg, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit $q0, implicit $q2, implicit killed $q3 {
+    ; CHECK: BUNDLE implicit-def $vpr, implicit-def $q3, implicit $q0, implicit $q2, implicit killed $q3 {
     ; CHECK:   MVE_VPTv4s32 13, renamable $q0, renamable $q2, 10, implicit-def $vpr
     ; CHECK:   renamable $q3 = MVE_VORR renamable $q2, renamable $q2, 1, internal renamable $vpr, $noreg, killed renamable $q3
     ; CHECK:   renamable $vpr = MVE_VCMPs32 renamable $q0, renamable $q2, 11, 2, internal killed renamable $vpr, $noreg
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block-fold-vcmp.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block-fold-vcmp.mir
index e583df903d3ed..0ffed2ed4560f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block-fold-vcmp.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block-fold-vcmp.mir
@@ -66,8 +66,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 12, size: 4, alignment: 4, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -107,14 +107,14 @@ body:             |
     ; CHECK: renamable $r12 = t2LDRi12 $r7, 16, 14 /* CC::al */, $noreg :: (load (s32) from %fixed-stack.2)
     ; CHECK: renamable $lr = t2LDRi12 $r7, 12, 14 /* CC::al */, $noreg :: (load (s32) from %fixed-stack.1)
     ; CHECK: renamable $r3 = t2LDRi12 $r7, 8, 14 /* CC::al */, $noreg :: (load (s32) from %fixed-stack.0)
-    ; CHECK: BUNDLE implicit-def $vpr, implicit-def dead $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit $q0, implicit $zr, implicit killed $r0, implicit killed $r3, implicit killed $r1, implicit killed $lr {
+    ; CHECK: BUNDLE implicit-def $vpr, implicit-def dead $q0, implicit $q0, implicit $zr, implicit killed $r0, implicit killed $r3, implicit killed $r1, implicit killed $lr {
     ; CHECK:   MVE_VPTv4f32r 1, renamable $q0, $zr, 10, implicit-def $vpr
     ; CHECK:   renamable $q0 = MVE_VLDRWU32 killed renamable $r0, 0, 1, internal renamable $vpr, $noreg :: (load (s128) from %ir.src, align 4)
     ; CHECK:   MVE_VSTRWU32 internal killed renamable $q0, killed renamable $r3, 0, 1, internal renamable $vpr, $noreg :: (store (s128) into %ir.dest, align 4)
     ; CHECK:   renamable $q0 = MVE_VLDRWU32 killed renamable $r1, 0, 1, internal renamable $vpr, $noreg :: (load (s128) from %ir.src2, align 4)
     ; CHECK:   MVE_VSTRWU32 internal killed renamable $q0, killed renamable $lr, 0, 1, internal renamable $vpr, $noreg :: (store (s128) into %ir.dest2, align 4)
     ; CHECK: }
-    ; CHECK: BUNDLE implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $vpr, implicit killed $r2, implicit killed $r12 {
+    ; CHECK: BUNDLE implicit-def $q0, implicit killed $vpr, implicit killed $r2, implicit killed $r12 {
     ; CHECK:   MVE_VPST 4, implicit $vpr
     ; CHECK:   renamable $q0 = MVE_VLDRWU32 killed renamable $r2, 0, 1, renamable $vpr, $noreg :: (load (s128) from %ir.src3, align 4)
     ; CHECK:   MVE_VSTRWU32 internal renamable $q0, killed renamable $r12, 0, 1, killed renamable $vpr, $noreg :: (store (s128) into %ir.dest3, align 4)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block-kill.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block-kill.mir
index f37c3fc9920b2..754f54bd78151 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block-kill.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block-kill.mir
@@ -22,7 +22,7 @@ body:             |
     ; CHECK: renamable $q0 = MVE_VMOVimmi32 0, 0, $noreg, $noreg, undef renamable $q0
     ; CHECK: renamable $q1 = MVE_VMOVimmi32 1, 0, $noreg, $noreg, undef renamable $q1
     ; CHECK: renamable $q2 = MVE_VADD_qr_i32 renamable $q1, renamable $r1, 0, $noreg, $noreg, undef renamable $q2
-    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q0, implicit $q1, implicit killed $q2 {
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q0, implicit killed $q0, implicit $q1, implicit killed $q2 {
     ; CHECK:   MVE_VPTv4u32 8, renamable $q0, renamable $q1, 8, implicit-def $vpr
     ; CHECK:   renamable $q0 = MVE_VORR killed renamable $q0, killed renamable $q2, 1, internal killed renamable $vpr, $noreg, renamable $q0
     ; CHECK: }
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block-optnone.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block-optnone.mir
index df56c9506698b..695a8d889cb5f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block-optnone.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block-optnone.mir
@@ -53,8 +53,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -66,7 +66,7 @@ body:             |
     ; CHECK-LABEL: name: test_vminnmq_m_f32_v2
     ; CHECK: liveins: $q0, $q1, $q2, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14 /* CC::al */, $noreg
-    ; CHECK: BUNDLE implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q0 {
+    ; CHECK: BUNDLE implicit-def $q0, implicit killed $vpr, implicit killed $q1, implicit killed $q2, implicit killed $q0 {
     ; CHECK:   MVE_VPST 8, implicit $vpr
     ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, killed renamable $q2, 1, killed renamable $vpr, $noreg, killed renamable $q0
     ; CHECK: }
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-preuse.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-preuse.mir
index 3499916bed912..8be0f74702f1e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-preuse.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-preuse.mir
@@ -51,8 +51,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -67,7 +67,7 @@ body:             |
     ; CHECK: renamable $vpr = MVE_VCMPu32 renamable $q1, renamable $q5, 2, 0, $noreg, $noreg
     ; CHECK: renamable $r4 = t2ADDrr renamable $r2, renamable $r10, 14 /* CC::al */, $noreg, $noreg
     ; CHECK: VSTR_P0_off renamable $vpr, $sp, 0, 14 /* CC::al */, $noreg
-    ; CHECK: BUNDLE implicit-def $q6, implicit-def $d12, implicit-def $s24, implicit-def $s25, implicit-def $d13, implicit-def $s26, implicit-def $s27, implicit $vpr, implicit killed $r4 {
+    ; CHECK: BUNDLE implicit-def $q6, implicit $vpr, implicit killed $r4 {
     ; CHECK:   MVE_VPST 8, implicit $vpr
     ; CHECK:   renamable $q6 = MVE_VLDRBU32 killed renamable $r4, 0, 1, renamable $vpr, $noreg
     ; CHECK: }
diff --git a/llvm/test/CodeGen/Thumb2/pipeliner-preserve-ties.mir b/llvm/test/CodeGen/Thumb2/pipeliner-preserve-ties.mir
index 4554f9f1fa23a..30bc033902a6e 100644
--- a/llvm/test/CodeGen/Thumb2/pipeliner-preserve-ties.mir
+++ b/llvm/test/CodeGen/Thumb2/pipeliner-preserve-ties.mir
@@ -157,8 +157,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/VE/Scalar/fold-imm-addsl.mir b/llvm/test/CodeGen/VE/Scalar/fold-imm-addsl.mir
index a3e374a46dc94..86d8b7a8c67f4 100644
--- a/llvm/test/CodeGen/VE/Scalar/fold-imm-addsl.mir
+++ b/llvm/test/CodeGen/VE/Scalar/fold-imm-addsl.mir
@@ -35,8 +35,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -90,8 +90,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -145,8 +145,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -200,8 +200,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/VE/Scalar/fold-imm-cmpsl.mir b/llvm/test/CodeGen/VE/Scalar/fold-imm-cmpsl.mir
index aab3d5bfcd7d3..34a15ba334a68 100644
--- a/llvm/test/CodeGen/VE/Scalar/fold-imm-cmpsl.mir
+++ b/llvm/test/CodeGen/VE/Scalar/fold-imm-cmpsl.mir
@@ -35,8 +35,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
@@ -90,8 +90,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll b/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll
index a27650f9cd961..7a90d28abeef5 100644
--- a/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll
+++ b/llvm/test/CodeGen/WebAssembly/expand-variadic-call.ll
@@ -37,52 +37,52 @@ define hidden void @copy(ptr noundef %va) {
 ; CHECK-NEXT:    %va.addr = alloca ptr, align 4
 ; CHECK-NEXT:    %cp = alloca ptr, align 4
 ; CHECK-NEXT:    store ptr %va, ptr %va.addr, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cp)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull %cp)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr %cp, ptr %va.addr, i32 4, i1 false)
 ; CHECK-NEXT:    %0 = load ptr, ptr %cp, align 4
 ; CHECK-NEXT:    call void @valist(ptr noundef %0)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cp)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull %cp)
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %va.addr = alloca ptr, align 4
   %cp = alloca ptr, align 4
   store ptr %va, ptr %va.addr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %cp)
   call void @llvm.va_copy.p0(ptr nonnull %cp, ptr nonnull %va.addr)
   %0 = load ptr, ptr %cp, align 4
   call void @valist(ptr noundef %0)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %cp)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @llvm.va_copy.p0(ptr, ptr)
 
 declare void @valist(ptr noundef)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define hidden void @start_once(...) {
 ; CHECK-LABEL: define {{[^@]+}}@start_once(ptr %varargs) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %s = alloca ptr, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull %s)
 ; CHECK-NEXT:    store ptr %varargs, ptr %s, align 4
 ; CHECK-NEXT:    %0 = load ptr, ptr %s, align 4
 ; CHECK-NEXT:    call void @valist(ptr noundef %0)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull %s)
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %s = alloca ptr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s)
+  call void @llvm.lifetime.start.p0(ptr nonnull %s)
   call void @llvm.va_start.p0(ptr nonnull %s)
   %0 = load ptr, ptr %s, align 4
   call void @valist(ptr noundef %0)
   call void @llvm.va_end.p0(ptr %s)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s)
+  call void @llvm.lifetime.end.p0(ptr nonnull %s)
   ret void
 }
 
@@ -95,23 +95,23 @@ define hidden void @start_twice(...) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %s0 = alloca ptr, align 4
 ; CHECK-NEXT:    %s1 = alloca ptr, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s0)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s1)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull %s0)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull %s1)
 ; CHECK-NEXT:    store ptr %varargs, ptr %s0, align 4
 ; CHECK-NEXT:    %0 = load ptr, ptr %s0, align 4
 ; CHECK-NEXT:    call void @valist(ptr noundef %0)
 ; CHECK-NEXT:    store ptr %varargs, ptr %s1, align 4
 ; CHECK-NEXT:    %1 = load ptr, ptr %s1, align 4
 ; CHECK-NEXT:    call void @valist(ptr noundef %1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s0)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull %s1)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull %s0)
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %s0 = alloca ptr, align 4
   %s1 = alloca ptr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s0)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %s0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %s1)
   call void @llvm.va_start.p0(ptr nonnull %s0)
   %0 = load ptr, ptr %s0, align 4
   call void @valist(ptr noundef %0)
@@ -120,8 +120,8 @@ entry:
   %1 = load ptr, ptr %s1, align 4
   call void @valist(ptr noundef %1)
   call void @llvm.va_end.p0(ptr %s1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %s1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %s0)
   ret void
 }
 
@@ -129,11 +129,11 @@ define hidden void @single_i32(i32 noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_i32(i32 noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_i32.vararg, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %0, align 4
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -147,11 +147,11 @@ define hidden void @single_double(double noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_double(double noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_double.vararg, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_double.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store double %x, ptr %0, align 8
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -163,11 +163,11 @@ define hidden void @single_v4f32(<4 x float> noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_v4f32(<4 x float> noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_v4f32.vararg, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_v4f32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <4 x float> %x, ptr %0, align 16
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -179,11 +179,11 @@ define hidden void @single_v8f32(<8 x float> noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_v8f32(<8 x float> noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_v8f32.vararg, align 32
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_v8f32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <8 x float> %x, ptr %0, align 32
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -195,11 +195,11 @@ define hidden void @single_v16f32(<16 x float> noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_v16f32(<16 x float> noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_v16f32.vararg, align 64
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_v16f32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <16 x float> %x, ptr %0, align 64
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -211,11 +211,11 @@ define hidden void @single_v32f32(<32 x float> noundef %x) {
 ; CHECK-LABEL: define {{[^@]+}}@single_v32f32(<32 x float> noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %single_v32f32.vararg, align 128
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 128, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %single_v32f32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <32 x float> %x, ptr %0, align 128
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 128, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -227,13 +227,13 @@ define hidden void @i32_double(i32 noundef %x, double noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_double(i32 noundef %x, double noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_double.vararg, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_double.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_double.vararg, ptr %vararg_buffer, i32 0, i32 2
 ; CHECK-NEXT:    store double %y, ptr %1, align 8
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -245,13 +245,13 @@ define hidden void @double_i32(double noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@double_i32(double noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %double_i32.vararg, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %double_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store double %x, ptr %0, align 8
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %double_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr %1, align 4
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -265,13 +265,13 @@ define hidden void @i32_libcS(i32 noundef %x, ptr noundef byval(%struct.libcS) a
 ; CHECK-NEXT:    %IndirectAlloca = alloca %struct.libcS, align 8
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_libcS.vararg, align 16
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr %IndirectAlloca, ptr %y, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_libcS.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_libcS.vararg, ptr %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store ptr %IndirectAlloca, ptr %1, align 4
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -285,13 +285,13 @@ define hidden void @libcS_i32(ptr noundef byval(%struct.libcS) align 8 %x, i32 n
 ; CHECK-NEXT:    %IndirectAlloca = alloca %struct.libcS, align 8
 ; CHECK-NEXT:    %vararg_buffer = alloca %libcS_i32.vararg, align 16
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr %IndirectAlloca, ptr %x, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %libcS_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store ptr %IndirectAlloca, ptr %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %libcS_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr %1, align 4
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -303,13 +303,13 @@ define hidden void @i32_v4f32(i32 noundef %x, <4 x float> noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_v4f32(i32 noundef %x, <4 x float> noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_v4f32.vararg, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_v4f32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_v4f32.vararg, ptr %vararg_buffer, i32 0, i32 2
 ; CHECK-NEXT:    store <4 x float> %y, ptr %1, align 16
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -321,13 +321,13 @@ define hidden void @v4f32_i32(<4 x float> noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@v4f32_i32(<4 x float> noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %v4f32_i32.vararg, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %v4f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <4 x float> %x, ptr %0, align 16
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %v4f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr %1, align 4
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -339,13 +339,13 @@ define hidden void @i32_v8f32(i32 noundef %x, <8 x float> noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_v8f32(i32 noundef %x, <8 x float> noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_v8f32.vararg, align 32
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_v8f32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_v8f32.vararg, ptr %vararg_buffer, i32 0, i32 2
 ; CHECK-NEXT:    store <8 x float> %y, ptr %1, align 32
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -357,13 +357,13 @@ define hidden void @v8f32_i32(<8 x float> noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@v8f32_i32(<8 x float> noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %v8f32_i32.vararg, align 32
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 36, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %v8f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <8 x float> %x, ptr %0, align 32
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %v8f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr %1, align 4
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 36, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -375,13 +375,13 @@ define hidden void @i32_v16f32(i32 noundef %x, <16 x float> noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_v16f32(i32 noundef %x, <16 x float> noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_v16f32.vararg, align 64
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 128, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_v16f32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_v16f32.vararg, ptr %vararg_buffer, i32 0, i32 2
 ; CHECK-NEXT:    store <16 x float> %y, ptr %1, align 64
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 128, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -393,13 +393,13 @@ define hidden void @v16f32_i32(<16 x float> noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@v16f32_i32(<16 x float> noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %v16f32_i32.vararg, align 64
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 68, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %v16f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <16 x float> %x, ptr %0, align 64
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %v16f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr %1, align 4
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 68, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -411,13 +411,13 @@ define hidden void @i32_v32f32(i32 noundef %x, <32 x float> noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@i32_v32f32(i32 noundef %x, <32 x float> noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %i32_v32f32.vararg, align 128
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 256, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %i32_v32f32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %i32_v32f32.vararg, ptr %vararg_buffer, i32 0, i32 2
 ; CHECK-NEXT:    store <32 x float> %y, ptr %1, align 128
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 256, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -429,13 +429,13 @@ define hidden void @v32f32_i32(<32 x float> noundef %x, i32 noundef %y) {
 ; CHECK-LABEL: define {{[^@]+}}@v32f32_i32(<32 x float> noundef %x, i32 noundef %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %v32f32_i32.vararg, align 128
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 132, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds nuw %v32f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store <32 x float> %x, ptr %0, align 128
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %v32f32_i32.vararg, ptr %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store i32 %y, ptr %1, align 4
 ; CHECK-NEXT:    call void @vararg(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 132, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -448,11 +448,11 @@ define hidden void @fptr_single_i32(i32 noundef %x) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %fptr_single_i32.vararg, align 16
 ; CHECK-NEXT:    %0 = load volatile ptr, ptr @vararg_ptr, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %fptr_single_i32.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %1, align 4
 ; CHECK-NEXT:    call void %0(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -468,11 +468,11 @@ define hidden void @fptr_libcS(ptr noundef byval(%struct.libcS) align 8 %x) {
 ; CHECK-NEXT:    %vararg_buffer = alloca %fptr_libcS.vararg, align 16
 ; CHECK-NEXT:    %0 = load volatile ptr, ptr @vararg_ptr, align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr %IndirectAlloca, ptr %x, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %1 = getelementptr inbounds nuw %fptr_libcS.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store ptr %IndirectAlloca, ptr %1, align 4
 ; CHECK-NEXT:    call void %0(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alloca.ll b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alloca.ll
index 0f968de8a7344..3264fe9a20b4c 100644
--- a/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alloca.ll
+++ b/llvm/test/CodeGen/WebAssembly/lower-em-sjlj-alloca.ll
@@ -18,7 +18,7 @@ define void @test_static() {
 ; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_SPLIT:.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       [[ENTRY_SPLIT]]:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[X]])
 ; CHECK-NEXT:    call void @__wasm_setjmp(ptr @buf, i32 1, ptr [[FUNCTIONINVOCATIONID]])
 ; CHECK-NEXT:    br label %[[ENTRY_SPLIT_SPLIT]]
 ; CHECK:       [[ENTRY_SPLIT_SPLIT]]:
@@ -31,7 +31,7 @@ define void @test_static() {
 ; CHECK:       [[_NOEXC:.*:]]
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[ELSE]]:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[CATCH_DISPATCH_LONGJMP]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = catchswitch within none [label %catch.longjmp] unwind to caller
@@ -53,7 +53,7 @@ define void @test_static() {
 ;
 entry:
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
   %call = call i32 @setjmp(ptr @buf) returns_twice
   %cmp = icmp eq i32 %call, 0
   br i1 %cmp, label %if, label %else
@@ -63,7 +63,7 @@ if:
   ret void
 
 else:
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   ret void
 }
 
@@ -114,7 +114,7 @@ define void @test_dynamic(i32 %size) {
 ;
 entry:
   %x = alloca i32, i32 %size, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
   %call = call i32 @setjmp(ptr @buf) returns_twice
   %cmp = icmp eq i32 %call, 0
   br i1 %cmp, label %if, label %else
@@ -124,6 +124,6 @@ if:
   ret void
 
 else:
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   ret void
 }
diff --git a/llvm/test/CodeGen/WebAssembly/lrint-conv.ll b/llvm/test/CodeGen/WebAssembly/lrint-conv.ll
new file mode 100644
index 0000000000000..0571150cb3505
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/lrint-conv.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+
+; Tests for lrint and llrint, with both i32 and i64 checked.
+
+; RUN: sed 's/ITy/i32/g' %s | llc -mtriple=wasm32-unknown-unknown | FileCheck %s
+; RUN: sed 's/ITy/i64/g' %s | llc -mtriple=wasm32-unknown-unknown | FileCheck %s
+
+define ITy @test_lrint_ixx_f16(half %x) nounwind {
+; CHECK-LABEL: test_lrint_ixx_f16:
+; CHECK:         call lrintf
+  %res = tail call ITy @llvm.lrint.ITy.f16(half %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f16(half %x) nounwind {
+; CHECK-LABEL: test_llrint_ixx_f16:
+; CHECK:         call llrintf
+  %res = tail call ITy @llvm.llrint.ITy.f16(half %x)
+  ret ITy %res
+}
+
+define ITy @test_lrint_ixx_f32(float %x) nounwind {
+; CHECK-LABEL: test_lrint_ixx_f32:
+; CHECK:         call lrintf
+  %res = tail call ITy @llvm.lrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f32(float %x) nounwind {
+; CHECK-LABEL: test_llrint_ixx_f32:
+; CHECK:         call llrintf
+  %res = tail call ITy @llvm.llrint.ITy.f32(float %x)
+  ret ITy %res
+}
+
+define ITy @test_lrint_ixx_f64(double %x) nounwind {
+; CHECK-LABEL: test_lrint_ixx_f64:
+; CHECK:         call lrint
+  %res = tail call ITy @llvm.lrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f64(double %x) nounwind {
+; CHECK-LABEL: test_llrint_ixx_f64:
+; CHECK:         call llrint
+  %res = tail call ITy @llvm.llrint.ITy.f64(double %x)
+  ret ITy %res
+}
+
+define ITy @test_lrint_ixx_f128(fp128 %x) nounwind {
+; CHECK-LABEL: test_lrint_ixx_f128:
+; CHECK:         call lrintl
+  %res = tail call ITy @llvm.lrint.ITy.f128(fp128 %x)
+  ret ITy %res
+}
+
+define ITy @test_llrint_ixx_f128(fp128 %x) nounwind {
+; CHECK-LABEL: test_llrint_ixx_f128:
+; CHECK:         call llrintl
+  %res = tail call ITy @llvm.llrint.ITy.f128(fp128 %x)
+  ret ITy %res
+}
diff --git a/llvm/test/CodeGen/WebAssembly/memcmp-expand.ll b/llvm/test/CodeGen/WebAssembly/memcmp-expand.ll
index 8030438645f82..4357dc5631eb0 100644
--- a/llvm/test/CodeGen/WebAssembly/memcmp-expand.ll
+++ b/llvm/test/CodeGen/WebAssembly/memcmp-expand.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s  -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s
+; RUN: llc < %s  -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
@@ -127,24 +127,15 @@ define i1 @memcmp_expand_8(ptr %a, ptr %b) {
   ret i1 %res
 }
 
-; TODO: Should be using a single load i64x2 or equivalent in bitsizes
 define i1 @memcmp_expand_16(ptr %a, ptr %b) {
 ; CHECK-LABEL: memcmp_expand_16:
 ; CHECK:         .functype memcmp_expand_16 (i32, i32) -> (i32)
 ; CHECK-NEXT:  # %bb.0:
-; CHECK-NEXT:    i64.load $push7=, 0($0):p2align=0
-; CHECK-NEXT:    i64.load $push6=, 0($1):p2align=0
-; CHECK-NEXT:    i64.xor $push8=, $pop7, $pop6
-; CHECK-NEXT:    i32.const $push0=, 8
-; CHECK-NEXT:    i32.add $push3=, $0, $pop0
-; CHECK-NEXT:    i64.load $push4=, 0($pop3):p2align=0
-; CHECK-NEXT:    i32.const $push11=, 8
-; CHECK-NEXT:    i32.add $push1=, $1, $pop11
-; CHECK-NEXT:    i64.load $push2=, 0($pop1):p2align=0
-; CHECK-NEXT:    i64.xor $push5=, $pop4, $pop2
-; CHECK-NEXT:    i64.or $push9=, $pop8, $pop5
-; CHECK-NEXT:    i64.eqz $push10=, $pop9
-; CHECK-NEXT:    return $pop10
+; CHECK-NEXT:    v128.load $push1=, 0($0):p2align=0
+; CHECK-NEXT:    v128.load $push0=, 0($1):p2align=0
+; CHECK-NEXT:    i8x16.eq $push2=, $pop1, $pop0
+; CHECK-NEXT:    i8x16.all_true $push3=, $pop2
+; CHECK-NEXT:    return $pop3
   %cmp_16 = call i32 @memcmp(ptr %a, ptr %b, i32 16)
   %res = icmp eq i32 %cmp_16, 0
   ret i1 %res
diff --git a/llvm/test/CodeGen/WebAssembly/multivalue-dont-move-def-past-use.mir b/llvm/test/CodeGen/WebAssembly/multivalue-dont-move-def-past-use.mir
index 0dfae8059d8b7..14a21a06c8fcd 100644
--- a/llvm/test/CodeGen/WebAssembly/multivalue-dont-move-def-past-use.mir
+++ b/llvm/test/CodeGen/WebAssembly/multivalue-dont-move-def-past-use.mir
@@ -73,8 +73,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/WebAssembly/ref-test-func.ll b/llvm/test/CodeGen/WebAssembly/ref-test-func.ll
index ea2453faaed90..4fda253d39fe3 100644
--- a/llvm/test/CodeGen/WebAssembly/ref-test-func.ll
+++ b/llvm/test/CodeGen/WebAssembly/ref-test-func.ll
@@ -31,7 +31,7 @@ define void @test_fpsig_return_i32(ptr noundef %func) local_unnamed_addr #0 {
 ; CHECK-NEXT:    call use
 ; CHECK-NEXT:    # fallthrough-return
 entry:
-  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, i32 0)
+  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, i32 poison)
   tail call void @use(i32 noundef %res) #3
   ret void
 }
@@ -48,7 +48,7 @@ define void @test_fpsig_return_i64(ptr noundef %func) local_unnamed_addr #0 {
 ; CHECK-NEXT:    call use
 ; CHECK-NEXT:    # fallthrough-return
 entry:
-  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, i64 0)
+  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, i64 poison)
   tail call void @use(i32 noundef %res) #3
   ret void
 }
@@ -65,7 +65,7 @@ define void @test_fpsig_return_f32(ptr noundef %func) local_unnamed_addr #0 {
 ; CHECK-NEXT:    call use
 ; CHECK-NEXT:    # fallthrough-return
 entry:
-  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, float 0.)
+  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, float poison)
   tail call void @use(i32 noundef %res) #3
   ret void
 }
@@ -82,7 +82,7 @@ define void @test_fpsig_return_f64(ptr noundef %func) local_unnamed_addr #0 {
 ; CHECK-NEXT:    call use
 ; CHECK-NEXT:    # fallthrough-return
 entry:
-  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, double 0.)
+  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, double poison)
   tail call void @use(i32 noundef %res) #3
   ret void
 }
@@ -100,7 +100,7 @@ define void @test_fpsig_param_i32(ptr noundef %func) local_unnamed_addr #0 {
 ; CHECK-NEXT:    call use
 ; CHECK-NEXT:    # fallthrough-return
 entry:
-  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, token poison, double 0.)
+  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, token poison, double poison)
   tail call void @use(i32 noundef %res) #3
   ret void
 }
@@ -118,7 +118,7 @@ define void @test_fpsig_multiple_params_and_returns(ptr noundef %func) local_unn
 ; CHECK-NEXT:    call use
 ; CHECK-NEXT:    # fallthrough-return
 entry:
-  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, i32 0, i64 0, float 0., double 0., token poison, i64 0, float 0., i64 0)
+  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, i32 poison, i64 poison, float poison, double poison, token poison, i64 poison, float poison, i64 poison)
   tail call void @use(i32 noundef %res) #3
   ret void
 }
@@ -137,10 +137,26 @@ define void @test_fpsig_ptrs(ptr noundef %func) local_unnamed_addr #0 {
 ; CHECK-NEXT:    call use
 ; CHECK-NEXT:    # fallthrough-return
 entry:
-  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, ptr null, token poison, ptr null, ptr null)
+  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, ptr poison, token poison, ptr poison, ptr poison)
   tail call void @use(i32 noundef %res) #3
   ret void
 }
 
+define void @test_reference_types(ptr noundef %func) local_unnamed_addr #0 {
+; CHECK-LABEL: test_reference_types:
+; CHK32:         .functype test_reference_types (i32) -> ()
+; CHK64:         .functype test_reference_types (i64) -> ()
+; CHECK-NEXT:  # %bb.0: # %entry
+; CHECK-NEXT:    local.get 0
+; CHK64-NEXT:    i32.wrap_i64
+; CHECK-NEXT:    table.get __indirect_function_table
+; CHECK-NEXT:    ref.test (funcref, externref) -> (externref)
+; CHECK-NEXT:    call use
+; CHECK-NEXT:    # fallthrough-return
+entry:
+  %res = tail call i32 (ptr, ...) @llvm.wasm.ref.test.func(ptr %func, ptr addrspace(10) poison, token poison, ptr addrspace(20) poison, ptr addrspace(10) poison)
+  tail call void @use(i32 noundef %res) #3
+  ret void
+}
 
 declare void @use(i32 noundef) local_unnamed_addr #1
diff --git a/llvm/test/CodeGen/WebAssembly/returned.ll b/llvm/test/CodeGen/WebAssembly/returned.ll
index aef75d8bee437..bad9d608b4bf0 100644
--- a/llvm/test/CodeGen/WebAssembly/returned.ll
+++ b/llvm/test/CodeGen/WebAssembly/returned.ll
@@ -99,8 +99,8 @@ define void @test() {
 ; CHECK-NEXT:    return
 entry:
   %a = alloca i32
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   %ret = call ptr @returns_arg(ptr %a)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
diff --git a/llvm/test/CodeGen/WebAssembly/simd-arith.ll b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
index 36637e1d555bd..324a0c49fb413 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-arith.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
@@ -1451,6 +1451,547 @@ define <16 x i8> @avgr_u_v16i8(<16 x i8> %x, <16 x i8> %y) {
   ret <16 x i8> %c
 }
 
+define <16 x i8> @avgr_u_v16i8_zext(<16 x i8> %x, <16 x i8> %y) {
+; SIMD128-LABEL: avgr_u_v16i8_zext:
+; SIMD128:         .functype avgr_u_v16i8_zext (v128, v128) -> (v128)
+; SIMD128-NEXT:  # %bb.0:
+; SIMD128-NEXT:    i8x16.avgr_u $push0=, $0, $1
+; SIMD128-NEXT:    return $pop0
+;
+; SIMD128-FAST-LABEL: avgr_u_v16i8_zext:
+; SIMD128-FAST:         .functype avgr_u_v16i8_zext (v128, v128) -> (v128)
+; SIMD128-FAST-NEXT:  # %bb.0:
+; SIMD128-FAST-NEXT:    i8x16.avgr_u $push0=, $0, $1
+; SIMD128-FAST-NEXT:    return $pop0
+;
+; NO-SIMD128-LABEL: avgr_u_v16i8_zext:
+; NO-SIMD128:         .functype avgr_u_v16i8_zext (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
+; NO-SIMD128-NEXT:  # %bb.0:
+; NO-SIMD128-NEXT:    i32.const $push0=, 255
+; NO-SIMD128-NEXT:    i32.and $push2=, $16, $pop0
+; NO-SIMD128-NEXT:    i32.const $push143=, 255
+; NO-SIMD128-NEXT:    i32.and $push1=, $32, $pop143
+; NO-SIMD128-NEXT:    i32.add $push3=, $pop2, $pop1
+; NO-SIMD128-NEXT:    i32.const $push4=, 1
+; NO-SIMD128-NEXT:    i32.add $push5=, $pop3, $pop4
+; NO-SIMD128-NEXT:    i32.const $push142=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push6=, $pop5, $pop142
+; NO-SIMD128-NEXT:    i32.store8 15($0), $pop6
+; NO-SIMD128-NEXT:    i32.const $push141=, 255
+; NO-SIMD128-NEXT:    i32.and $push8=, $15, $pop141
+; NO-SIMD128-NEXT:    i32.const $push140=, 255
+; NO-SIMD128-NEXT:    i32.and $push7=, $31, $pop140
+; NO-SIMD128-NEXT:    i32.add $push9=, $pop8, $pop7
+; NO-SIMD128-NEXT:    i32.const $push139=, 1
+; NO-SIMD128-NEXT:    i32.add $push10=, $pop9, $pop139
+; NO-SIMD128-NEXT:    i32.const $push138=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push11=, $pop10, $pop138
+; NO-SIMD128-NEXT:    i32.store8 14($0), $pop11
+; NO-SIMD128-NEXT:    i32.const $push137=, 255
+; NO-SIMD128-NEXT:    i32.and $push13=, $14, $pop137
+; NO-SIMD128-NEXT:    i32.const $push136=, 255
+; NO-SIMD128-NEXT:    i32.and $push12=, $30, $pop136
+; NO-SIMD128-NEXT:    i32.add $push14=, $pop13, $pop12
+; NO-SIMD128-NEXT:    i32.const $push135=, 1
+; NO-SIMD128-NEXT:    i32.add $push15=, $pop14, $pop135
+; NO-SIMD128-NEXT:    i32.const $push134=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push16=, $pop15, $pop134
+; NO-SIMD128-NEXT:    i32.store8 13($0), $pop16
+; NO-SIMD128-NEXT:    i32.const $push133=, 255
+; NO-SIMD128-NEXT:    i32.and $push18=, $13, $pop133
+; NO-SIMD128-NEXT:    i32.const $push132=, 255
+; NO-SIMD128-NEXT:    i32.and $push17=, $29, $pop132
+; NO-SIMD128-NEXT:    i32.add $push19=, $pop18, $pop17
+; NO-SIMD128-NEXT:    i32.const $push131=, 1
+; NO-SIMD128-NEXT:    i32.add $push20=, $pop19, $pop131
+; NO-SIMD128-NEXT:    i32.const $push130=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push21=, $pop20, $pop130
+; NO-SIMD128-NEXT:    i32.store8 12($0), $pop21
+; NO-SIMD128-NEXT:    i32.const $push129=, 255
+; NO-SIMD128-NEXT:    i32.and $push23=, $12, $pop129
+; NO-SIMD128-NEXT:    i32.const $push128=, 255
+; NO-SIMD128-NEXT:    i32.and $push22=, $28, $pop128
+; NO-SIMD128-NEXT:    i32.add $push24=, $pop23, $pop22
+; NO-SIMD128-NEXT:    i32.const $push127=, 1
+; NO-SIMD128-NEXT:    i32.add $push25=, $pop24, $pop127
+; NO-SIMD128-NEXT:    i32.const $push126=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push26=, $pop25, $pop126
+; NO-SIMD128-NEXT:    i32.store8 11($0), $pop26
+; NO-SIMD128-NEXT:    i32.const $push125=, 255
+; NO-SIMD128-NEXT:    i32.and $push28=, $11, $pop125
+; NO-SIMD128-NEXT:    i32.const $push124=, 255
+; NO-SIMD128-NEXT:    i32.and $push27=, $27, $pop124
+; NO-SIMD128-NEXT:    i32.add $push29=, $pop28, $pop27
+; NO-SIMD128-NEXT:    i32.const $push123=, 1
+; NO-SIMD128-NEXT:    i32.add $push30=, $pop29, $pop123
+; NO-SIMD128-NEXT:    i32.const $push122=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push31=, $pop30, $pop122
+; NO-SIMD128-NEXT:    i32.store8 10($0), $pop31
+; NO-SIMD128-NEXT:    i32.const $push121=, 255
+; NO-SIMD128-NEXT:    i32.and $push33=, $10, $pop121
+; NO-SIMD128-NEXT:    i32.const $push120=, 255
+; NO-SIMD128-NEXT:    i32.and $push32=, $26, $pop120
+; NO-SIMD128-NEXT:    i32.add $push34=, $pop33, $pop32
+; NO-SIMD128-NEXT:    i32.const $push119=, 1
+; NO-SIMD128-NEXT:    i32.add $push35=, $pop34, $pop119
+; NO-SIMD128-NEXT:    i32.const $push118=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push36=, $pop35, $pop118
+; NO-SIMD128-NEXT:    i32.store8 9($0), $pop36
+; NO-SIMD128-NEXT:    i32.const $push117=, 255
+; NO-SIMD128-NEXT:    i32.and $push38=, $9, $pop117
+; NO-SIMD128-NEXT:    i32.const $push116=, 255
+; NO-SIMD128-NEXT:    i32.and $push37=, $25, $pop116
+; NO-SIMD128-NEXT:    i32.add $push39=, $pop38, $pop37
+; NO-SIMD128-NEXT:    i32.const $push115=, 1
+; NO-SIMD128-NEXT:    i32.add $push40=, $pop39, $pop115
+; NO-SIMD128-NEXT:    i32.const $push114=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push41=, $pop40, $pop114
+; NO-SIMD128-NEXT:    i32.store8 8($0), $pop41
+; NO-SIMD128-NEXT:    i32.const $push113=, 255
+; NO-SIMD128-NEXT:    i32.and $push43=, $8, $pop113
+; NO-SIMD128-NEXT:    i32.const $push112=, 255
+; NO-SIMD128-NEXT:    i32.and $push42=, $24, $pop112
+; NO-SIMD128-NEXT:    i32.add $push44=, $pop43, $pop42
+; NO-SIMD128-NEXT:    i32.const $push111=, 1
+; NO-SIMD128-NEXT:    i32.add $push45=, $pop44, $pop111
+; NO-SIMD128-NEXT:    i32.const $push110=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push46=, $pop45, $pop110
+; NO-SIMD128-NEXT:    i32.store8 7($0), $pop46
+; NO-SIMD128-NEXT:    i32.const $push109=, 255
+; NO-SIMD128-NEXT:    i32.and $push48=, $7, $pop109
+; NO-SIMD128-NEXT:    i32.const $push108=, 255
+; NO-SIMD128-NEXT:    i32.and $push47=, $23, $pop108
+; NO-SIMD128-NEXT:    i32.add $push49=, $pop48, $pop47
+; NO-SIMD128-NEXT:    i32.const $push107=, 1
+; NO-SIMD128-NEXT:    i32.add $push50=, $pop49, $pop107
+; NO-SIMD128-NEXT:    i32.const $push106=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push51=, $pop50, $pop106
+; NO-SIMD128-NEXT:    i32.store8 6($0), $pop51
+; NO-SIMD128-NEXT:    i32.const $push105=, 255
+; NO-SIMD128-NEXT:    i32.and $push53=, $6, $pop105
+; NO-SIMD128-NEXT:    i32.const $push104=, 255
+; NO-SIMD128-NEXT:    i32.and $push52=, $22, $pop104
+; NO-SIMD128-NEXT:    i32.add $push54=, $pop53, $pop52
+; NO-SIMD128-NEXT:    i32.const $push103=, 1
+; NO-SIMD128-NEXT:    i32.add $push55=, $pop54, $pop103
+; NO-SIMD128-NEXT:    i32.const $push102=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push56=, $pop55, $pop102
+; NO-SIMD128-NEXT:    i32.store8 5($0), $pop56
+; NO-SIMD128-NEXT:    i32.const $push101=, 255
+; NO-SIMD128-NEXT:    i32.and $push58=, $5, $pop101
+; NO-SIMD128-NEXT:    i32.const $push100=, 255
+; NO-SIMD128-NEXT:    i32.and $push57=, $21, $pop100
+; NO-SIMD128-NEXT:    i32.add $push59=, $pop58, $pop57
+; NO-SIMD128-NEXT:    i32.const $push99=, 1
+; NO-SIMD128-NEXT:    i32.add $push60=, $pop59, $pop99
+; NO-SIMD128-NEXT:    i32.const $push98=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push61=, $pop60, $pop98
+; NO-SIMD128-NEXT:    i32.store8 4($0), $pop61
+; NO-SIMD128-NEXT:    i32.const $push97=, 255
+; NO-SIMD128-NEXT:    i32.and $push63=, $4, $pop97
+; NO-SIMD128-NEXT:    i32.const $push96=, 255
+; NO-SIMD128-NEXT:    i32.and $push62=, $20, $pop96
+; NO-SIMD128-NEXT:    i32.add $push64=, $pop63, $pop62
+; NO-SIMD128-NEXT:    i32.const $push95=, 1
+; NO-SIMD128-NEXT:    i32.add $push65=, $pop64, $pop95
+; NO-SIMD128-NEXT:    i32.const $push94=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push66=, $pop65, $pop94
+; NO-SIMD128-NEXT:    i32.store8 3($0), $pop66
+; NO-SIMD128-NEXT:    i32.const $push93=, 255
+; NO-SIMD128-NEXT:    i32.and $push68=, $3, $pop93
+; NO-SIMD128-NEXT:    i32.const $push92=, 255
+; NO-SIMD128-NEXT:    i32.and $push67=, $19, $pop92
+; NO-SIMD128-NEXT:    i32.add $push69=, $pop68, $pop67
+; NO-SIMD128-NEXT:    i32.const $push91=, 1
+; NO-SIMD128-NEXT:    i32.add $push70=, $pop69, $pop91
+; NO-SIMD128-NEXT:    i32.const $push90=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push71=, $pop70, $pop90
+; NO-SIMD128-NEXT:    i32.store8 2($0), $pop71
+; NO-SIMD128-NEXT:    i32.const $push89=, 255
+; NO-SIMD128-NEXT:    i32.and $push73=, $2, $pop89
+; NO-SIMD128-NEXT:    i32.const $push88=, 255
+; NO-SIMD128-NEXT:    i32.and $push72=, $18, $pop88
+; NO-SIMD128-NEXT:    i32.add $push74=, $pop73, $pop72
+; NO-SIMD128-NEXT:    i32.const $push87=, 1
+; NO-SIMD128-NEXT:    i32.add $push75=, $pop74, $pop87
+; NO-SIMD128-NEXT:    i32.const $push86=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push76=, $pop75, $pop86
+; NO-SIMD128-NEXT:    i32.store8 1($0), $pop76
+; NO-SIMD128-NEXT:    i32.const $push85=, 255
+; NO-SIMD128-NEXT:    i32.and $push78=, $1, $pop85
+; NO-SIMD128-NEXT:    i32.const $push84=, 255
+; NO-SIMD128-NEXT:    i32.and $push77=, $17, $pop84
+; NO-SIMD128-NEXT:    i32.add $push79=, $pop78, $pop77
+; NO-SIMD128-NEXT:    i32.const $push83=, 1
+; NO-SIMD128-NEXT:    i32.add $push80=, $pop79, $pop83
+; NO-SIMD128-NEXT:    i32.const $push82=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push81=, $pop80, $pop82
+; NO-SIMD128-NEXT:    i32.store8 0($0), $pop81
+; NO-SIMD128-NEXT:    return
+;
+; NO-SIMD128-FAST-LABEL: avgr_u_v16i8_zext:
+; NO-SIMD128-FAST:         .functype avgr_u_v16i8_zext (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
+; NO-SIMD128-FAST-NEXT:  # %bb.0:
+; NO-SIMD128-FAST-NEXT:    i32.const $push0=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push2=, $1, $pop0
+; NO-SIMD128-FAST-NEXT:    i32.const $push143=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push1=, $17, $pop143
+; NO-SIMD128-FAST-NEXT:    i32.add $push3=, $pop2, $pop1
+; NO-SIMD128-FAST-NEXT:    i32.const $push4=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push5=, $pop3, $pop4
+; NO-SIMD128-FAST-NEXT:    i32.const $push142=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push6=, $pop5, $pop142
+; NO-SIMD128-FAST-NEXT:    i32.store8 0($0), $pop6
+; NO-SIMD128-FAST-NEXT:    i32.const $push141=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push8=, $2, $pop141
+; NO-SIMD128-FAST-NEXT:    i32.const $push140=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push7=, $18, $pop140
+; NO-SIMD128-FAST-NEXT:    i32.add $push9=, $pop8, $pop7
+; NO-SIMD128-FAST-NEXT:    i32.const $push139=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push10=, $pop9, $pop139
+; NO-SIMD128-FAST-NEXT:    i32.const $push138=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push11=, $pop10, $pop138
+; NO-SIMD128-FAST-NEXT:    i32.store8 1($0), $pop11
+; NO-SIMD128-FAST-NEXT:    i32.const $push137=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push13=, $3, $pop137
+; NO-SIMD128-FAST-NEXT:    i32.const $push136=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push12=, $19, $pop136
+; NO-SIMD128-FAST-NEXT:    i32.add $push14=, $pop13, $pop12
+; NO-SIMD128-FAST-NEXT:    i32.const $push135=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push15=, $pop14, $pop135
+; NO-SIMD128-FAST-NEXT:    i32.const $push134=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push16=, $pop15, $pop134
+; NO-SIMD128-FAST-NEXT:    i32.store8 2($0), $pop16
+; NO-SIMD128-FAST-NEXT:    i32.const $push133=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push18=, $4, $pop133
+; NO-SIMD128-FAST-NEXT:    i32.const $push132=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push17=, $20, $pop132
+; NO-SIMD128-FAST-NEXT:    i32.add $push19=, $pop18, $pop17
+; NO-SIMD128-FAST-NEXT:    i32.const $push131=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push20=, $pop19, $pop131
+; NO-SIMD128-FAST-NEXT:    i32.const $push130=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push21=, $pop20, $pop130
+; NO-SIMD128-FAST-NEXT:    i32.store8 3($0), $pop21
+; NO-SIMD128-FAST-NEXT:    i32.const $push129=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push23=, $5, $pop129
+; NO-SIMD128-FAST-NEXT:    i32.const $push128=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push22=, $21, $pop128
+; NO-SIMD128-FAST-NEXT:    i32.add $push24=, $pop23, $pop22
+; NO-SIMD128-FAST-NEXT:    i32.const $push127=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push25=, $pop24, $pop127
+; NO-SIMD128-FAST-NEXT:    i32.const $push126=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push26=, $pop25, $pop126
+; NO-SIMD128-FAST-NEXT:    i32.store8 4($0), $pop26
+; NO-SIMD128-FAST-NEXT:    i32.const $push125=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push28=, $6, $pop125
+; NO-SIMD128-FAST-NEXT:    i32.const $push124=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push27=, $22, $pop124
+; NO-SIMD128-FAST-NEXT:    i32.add $push29=, $pop28, $pop27
+; NO-SIMD128-FAST-NEXT:    i32.const $push123=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push30=, $pop29, $pop123
+; NO-SIMD128-FAST-NEXT:    i32.const $push122=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push31=, $pop30, $pop122
+; NO-SIMD128-FAST-NEXT:    i32.store8 5($0), $pop31
+; NO-SIMD128-FAST-NEXT:    i32.const $push121=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push33=, $7, $pop121
+; NO-SIMD128-FAST-NEXT:    i32.const $push120=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push32=, $23, $pop120
+; NO-SIMD128-FAST-NEXT:    i32.add $push34=, $pop33, $pop32
+; NO-SIMD128-FAST-NEXT:    i32.const $push119=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push35=, $pop34, $pop119
+; NO-SIMD128-FAST-NEXT:    i32.const $push118=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push36=, $pop35, $pop118
+; NO-SIMD128-FAST-NEXT:    i32.store8 6($0), $pop36
+; NO-SIMD128-FAST-NEXT:    i32.const $push117=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push38=, $8, $pop117
+; NO-SIMD128-FAST-NEXT:    i32.const $push116=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push37=, $24, $pop116
+; NO-SIMD128-FAST-NEXT:    i32.add $push39=, $pop38, $pop37
+; NO-SIMD128-FAST-NEXT:    i32.const $push115=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push40=, $pop39, $pop115
+; NO-SIMD128-FAST-NEXT:    i32.const $push114=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push41=, $pop40, $pop114
+; NO-SIMD128-FAST-NEXT:    i32.store8 7($0), $pop41
+; NO-SIMD128-FAST-NEXT:    i32.const $push113=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push43=, $9, $pop113
+; NO-SIMD128-FAST-NEXT:    i32.const $push112=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push42=, $25, $pop112
+; NO-SIMD128-FAST-NEXT:    i32.add $push44=, $pop43, $pop42
+; NO-SIMD128-FAST-NEXT:    i32.const $push111=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push45=, $pop44, $pop111
+; NO-SIMD128-FAST-NEXT:    i32.const $push110=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push46=, $pop45, $pop110
+; NO-SIMD128-FAST-NEXT:    i32.store8 8($0), $pop46
+; NO-SIMD128-FAST-NEXT:    i32.const $push109=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push48=, $10, $pop109
+; NO-SIMD128-FAST-NEXT:    i32.const $push108=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push47=, $26, $pop108
+; NO-SIMD128-FAST-NEXT:    i32.add $push49=, $pop48, $pop47
+; NO-SIMD128-FAST-NEXT:    i32.const $push107=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push50=, $pop49, $pop107
+; NO-SIMD128-FAST-NEXT:    i32.const $push106=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push51=, $pop50, $pop106
+; NO-SIMD128-FAST-NEXT:    i32.store8 9($0), $pop51
+; NO-SIMD128-FAST-NEXT:    i32.const $push105=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push53=, $11, $pop105
+; NO-SIMD128-FAST-NEXT:    i32.const $push104=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push52=, $27, $pop104
+; NO-SIMD128-FAST-NEXT:    i32.add $push54=, $pop53, $pop52
+; NO-SIMD128-FAST-NEXT:    i32.const $push103=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push55=, $pop54, $pop103
+; NO-SIMD128-FAST-NEXT:    i32.const $push102=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push56=, $pop55, $pop102
+; NO-SIMD128-FAST-NEXT:    i32.store8 10($0), $pop56
+; NO-SIMD128-FAST-NEXT:    i32.const $push101=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push58=, $12, $pop101
+; NO-SIMD128-FAST-NEXT:    i32.const $push100=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push57=, $28, $pop100
+; NO-SIMD128-FAST-NEXT:    i32.add $push59=, $pop58, $pop57
+; NO-SIMD128-FAST-NEXT:    i32.const $push99=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push60=, $pop59, $pop99
+; NO-SIMD128-FAST-NEXT:    i32.const $push98=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push61=, $pop60, $pop98
+; NO-SIMD128-FAST-NEXT:    i32.store8 11($0), $pop61
+; NO-SIMD128-FAST-NEXT:    i32.const $push97=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push63=, $13, $pop97
+; NO-SIMD128-FAST-NEXT:    i32.const $push96=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push62=, $29, $pop96
+; NO-SIMD128-FAST-NEXT:    i32.add $push64=, $pop63, $pop62
+; NO-SIMD128-FAST-NEXT:    i32.const $push95=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push65=, $pop64, $pop95
+; NO-SIMD128-FAST-NEXT:    i32.const $push94=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push66=, $pop65, $pop94
+; NO-SIMD128-FAST-NEXT:    i32.store8 12($0), $pop66
+; NO-SIMD128-FAST-NEXT:    i32.const $push93=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push68=, $14, $pop93
+; NO-SIMD128-FAST-NEXT:    i32.const $push92=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push67=, $30, $pop92
+; NO-SIMD128-FAST-NEXT:    i32.add $push69=, $pop68, $pop67
+; NO-SIMD128-FAST-NEXT:    i32.const $push91=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push70=, $pop69, $pop91
+; NO-SIMD128-FAST-NEXT:    i32.const $push90=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push71=, $pop70, $pop90
+; NO-SIMD128-FAST-NEXT:    i32.store8 13($0), $pop71
+; NO-SIMD128-FAST-NEXT:    i32.const $push89=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push73=, $15, $pop89
+; NO-SIMD128-FAST-NEXT:    i32.const $push88=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push72=, $31, $pop88
+; NO-SIMD128-FAST-NEXT:    i32.add $push74=, $pop73, $pop72
+; NO-SIMD128-FAST-NEXT:    i32.const $push87=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push75=, $pop74, $pop87
+; NO-SIMD128-FAST-NEXT:    i32.const $push86=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push76=, $pop75, $pop86
+; NO-SIMD128-FAST-NEXT:    i32.store8 14($0), $pop76
+; NO-SIMD128-FAST-NEXT:    i32.const $push85=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push78=, $16, $pop85
+; NO-SIMD128-FAST-NEXT:    i32.const $push84=, 255
+; NO-SIMD128-FAST-NEXT:    i32.and $push77=, $32, $pop84
+; NO-SIMD128-FAST-NEXT:    i32.add $push79=, $pop78, $pop77
+; NO-SIMD128-FAST-NEXT:    i32.const $push83=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push80=, $pop79, $pop83
+; NO-SIMD128-FAST-NEXT:    i32.const $push82=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push81=, $pop80, $pop82
+; NO-SIMD128-FAST-NEXT:    i32.store8 15($0), $pop81
+; NO-SIMD128-FAST-NEXT:    return
+  %x.zext = zext <16 x i8> %x to <16 x i16>
+  %y.zext = zext <16 x i8> %y to <16 x i16>
+  %a = add <16 x i16> %x.zext, %y.zext
+  %b = add <16 x i16> %a, splat (i16 1)
+  %c = lshr <16 x i16> %b, splat (i16 1)
+  %c.trunc = trunc <16 x i16> %c to <16 x i8>
+  ret <16 x i8> %c.trunc
+}
+
+define <8 x i16> @avgr_u_v8i16_zext(<8 x i16> %x, <8 x i16> %y) {
+; SIMD128-LABEL: avgr_u_v8i16_zext:
+; SIMD128:         .functype avgr_u_v8i16_zext (v128, v128) -> (v128)
+; SIMD128-NEXT:  # %bb.0:
+; SIMD128-NEXT:    i16x8.avgr_u $push0=, $0, $1
+; SIMD128-NEXT:    return $pop0
+;
+; SIMD128-FAST-LABEL: avgr_u_v8i16_zext:
+; SIMD128-FAST:         .functype avgr_u_v8i16_zext (v128, v128) -> (v128)
+; SIMD128-FAST-NEXT:  # %bb.0:
+; SIMD128-FAST-NEXT:    i16x8.avgr_u $push0=, $0, $1
+; SIMD128-FAST-NEXT:    return $pop0
+;
+; NO-SIMD128-LABEL: avgr_u_v8i16_zext:
+; NO-SIMD128:         .functype avgr_u_v8i16_zext (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
+; NO-SIMD128-NEXT:  # %bb.0:
+; NO-SIMD128-NEXT:    i32.const $push0=, 65535
+; NO-SIMD128-NEXT:    i32.and $push2=, $8, $pop0
+; NO-SIMD128-NEXT:    i32.const $push71=, 65535
+; NO-SIMD128-NEXT:    i32.and $push1=, $16, $pop71
+; NO-SIMD128-NEXT:    i32.add $push3=, $pop2, $pop1
+; NO-SIMD128-NEXT:    i32.const $push4=, 1
+; NO-SIMD128-NEXT:    i32.add $push5=, $pop3, $pop4
+; NO-SIMD128-NEXT:    i32.const $push70=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push6=, $pop5, $pop70
+; NO-SIMD128-NEXT:    i32.store16 14($0), $pop6
+; NO-SIMD128-NEXT:    i32.const $push69=, 65535
+; NO-SIMD128-NEXT:    i32.and $push8=, $7, $pop69
+; NO-SIMD128-NEXT:    i32.const $push68=, 65535
+; NO-SIMD128-NEXT:    i32.and $push7=, $15, $pop68
+; NO-SIMD128-NEXT:    i32.add $push9=, $pop8, $pop7
+; NO-SIMD128-NEXT:    i32.const $push67=, 1
+; NO-SIMD128-NEXT:    i32.add $push10=, $pop9, $pop67
+; NO-SIMD128-NEXT:    i32.const $push66=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push11=, $pop10, $pop66
+; NO-SIMD128-NEXT:    i32.store16 12($0), $pop11
+; NO-SIMD128-NEXT:    i32.const $push65=, 65535
+; NO-SIMD128-NEXT:    i32.and $push13=, $6, $pop65
+; NO-SIMD128-NEXT:    i32.const $push64=, 65535
+; NO-SIMD128-NEXT:    i32.and $push12=, $14, $pop64
+; NO-SIMD128-NEXT:    i32.add $push14=, $pop13, $pop12
+; NO-SIMD128-NEXT:    i32.const $push63=, 1
+; NO-SIMD128-NEXT:    i32.add $push15=, $pop14, $pop63
+; NO-SIMD128-NEXT:    i32.const $push62=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push16=, $pop15, $pop62
+; NO-SIMD128-NEXT:    i32.store16 10($0), $pop16
+; NO-SIMD128-NEXT:    i32.const $push61=, 65535
+; NO-SIMD128-NEXT:    i32.and $push18=, $5, $pop61
+; NO-SIMD128-NEXT:    i32.const $push60=, 65535
+; NO-SIMD128-NEXT:    i32.and $push17=, $13, $pop60
+; NO-SIMD128-NEXT:    i32.add $push19=, $pop18, $pop17
+; NO-SIMD128-NEXT:    i32.const $push59=, 1
+; NO-SIMD128-NEXT:    i32.add $push20=, $pop19, $pop59
+; NO-SIMD128-NEXT:    i32.const $push58=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push21=, $pop20, $pop58
+; NO-SIMD128-NEXT:    i32.store16 8($0), $pop21
+; NO-SIMD128-NEXT:    i32.const $push57=, 65535
+; NO-SIMD128-NEXT:    i32.and $push23=, $4, $pop57
+; NO-SIMD128-NEXT:    i32.const $push56=, 65535
+; NO-SIMD128-NEXT:    i32.and $push22=, $12, $pop56
+; NO-SIMD128-NEXT:    i32.add $push24=, $pop23, $pop22
+; NO-SIMD128-NEXT:    i32.const $push55=, 1
+; NO-SIMD128-NEXT:    i32.add $push25=, $pop24, $pop55
+; NO-SIMD128-NEXT:    i32.const $push54=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push26=, $pop25, $pop54
+; NO-SIMD128-NEXT:    i32.store16 6($0), $pop26
+; NO-SIMD128-NEXT:    i32.const $push53=, 65535
+; NO-SIMD128-NEXT:    i32.and $push28=, $3, $pop53
+; NO-SIMD128-NEXT:    i32.const $push52=, 65535
+; NO-SIMD128-NEXT:    i32.and $push27=, $11, $pop52
+; NO-SIMD128-NEXT:    i32.add $push29=, $pop28, $pop27
+; NO-SIMD128-NEXT:    i32.const $push51=, 1
+; NO-SIMD128-NEXT:    i32.add $push30=, $pop29, $pop51
+; NO-SIMD128-NEXT:    i32.const $push50=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push31=, $pop30, $pop50
+; NO-SIMD128-NEXT:    i32.store16 4($0), $pop31
+; NO-SIMD128-NEXT:    i32.const $push49=, 65535
+; NO-SIMD128-NEXT:    i32.and $push33=, $2, $pop49
+; NO-SIMD128-NEXT:    i32.const $push48=, 65535
+; NO-SIMD128-NEXT:    i32.and $push32=, $10, $pop48
+; NO-SIMD128-NEXT:    i32.add $push34=, $pop33, $pop32
+; NO-SIMD128-NEXT:    i32.const $push47=, 1
+; NO-SIMD128-NEXT:    i32.add $push35=, $pop34, $pop47
+; NO-SIMD128-NEXT:    i32.const $push46=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push36=, $pop35, $pop46
+; NO-SIMD128-NEXT:    i32.store16 2($0), $pop36
+; NO-SIMD128-NEXT:    i32.const $push45=, 65535
+; NO-SIMD128-NEXT:    i32.and $push38=, $1, $pop45
+; NO-SIMD128-NEXT:    i32.const $push44=, 65535
+; NO-SIMD128-NEXT:    i32.and $push37=, $9, $pop44
+; NO-SIMD128-NEXT:    i32.add $push39=, $pop38, $pop37
+; NO-SIMD128-NEXT:    i32.const $push43=, 1
+; NO-SIMD128-NEXT:    i32.add $push40=, $pop39, $pop43
+; NO-SIMD128-NEXT:    i32.const $push42=, 1
+; NO-SIMD128-NEXT:    i32.shr_u $push41=, $pop40, $pop42
+; NO-SIMD128-NEXT:    i32.store16 0($0), $pop41
+; NO-SIMD128-NEXT:    return
+;
+; NO-SIMD128-FAST-LABEL: avgr_u_v8i16_zext:
+; NO-SIMD128-FAST:         .functype avgr_u_v8i16_zext (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> ()
+; NO-SIMD128-FAST-NEXT:  # %bb.0:
+; NO-SIMD128-FAST-NEXT:    i32.const $push0=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push2=, $1, $pop0
+; NO-SIMD128-FAST-NEXT:    i32.const $push71=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push1=, $9, $pop71
+; NO-SIMD128-FAST-NEXT:    i32.add $push3=, $pop2, $pop1
+; NO-SIMD128-FAST-NEXT:    i32.const $push4=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push5=, $pop3, $pop4
+; NO-SIMD128-FAST-NEXT:    i32.const $push70=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push6=, $pop5, $pop70
+; NO-SIMD128-FAST-NEXT:    i32.store16 0($0), $pop6
+; NO-SIMD128-FAST-NEXT:    i32.const $push69=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push8=, $2, $pop69
+; NO-SIMD128-FAST-NEXT:    i32.const $push68=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push7=, $10, $pop68
+; NO-SIMD128-FAST-NEXT:    i32.add $push9=, $pop8, $pop7
+; NO-SIMD128-FAST-NEXT:    i32.const $push67=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push10=, $pop9, $pop67
+; NO-SIMD128-FAST-NEXT:    i32.const $push66=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push11=, $pop10, $pop66
+; NO-SIMD128-FAST-NEXT:    i32.store16 2($0), $pop11
+; NO-SIMD128-FAST-NEXT:    i32.const $push65=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push13=, $3, $pop65
+; NO-SIMD128-FAST-NEXT:    i32.const $push64=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push12=, $11, $pop64
+; NO-SIMD128-FAST-NEXT:    i32.add $push14=, $pop13, $pop12
+; NO-SIMD128-FAST-NEXT:    i32.const $push63=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push15=, $pop14, $pop63
+; NO-SIMD128-FAST-NEXT:    i32.const $push62=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push16=, $pop15, $pop62
+; NO-SIMD128-FAST-NEXT:    i32.store16 4($0), $pop16
+; NO-SIMD128-FAST-NEXT:    i32.const $push61=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push18=, $4, $pop61
+; NO-SIMD128-FAST-NEXT:    i32.const $push60=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push17=, $12, $pop60
+; NO-SIMD128-FAST-NEXT:    i32.add $push19=, $pop18, $pop17
+; NO-SIMD128-FAST-NEXT:    i32.const $push59=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push20=, $pop19, $pop59
+; NO-SIMD128-FAST-NEXT:    i32.const $push58=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push21=, $pop20, $pop58
+; NO-SIMD128-FAST-NEXT:    i32.store16 6($0), $pop21
+; NO-SIMD128-FAST-NEXT:    i32.const $push57=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push23=, $5, $pop57
+; NO-SIMD128-FAST-NEXT:    i32.const $push56=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push22=, $13, $pop56
+; NO-SIMD128-FAST-NEXT:    i32.add $push24=, $pop23, $pop22
+; NO-SIMD128-FAST-NEXT:    i32.const $push55=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push25=, $pop24, $pop55
+; NO-SIMD128-FAST-NEXT:    i32.const $push54=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push26=, $pop25, $pop54
+; NO-SIMD128-FAST-NEXT:    i32.store16 8($0), $pop26
+; NO-SIMD128-FAST-NEXT:    i32.const $push53=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push28=, $6, $pop53
+; NO-SIMD128-FAST-NEXT:    i32.const $push52=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push27=, $14, $pop52
+; NO-SIMD128-FAST-NEXT:    i32.add $push29=, $pop28, $pop27
+; NO-SIMD128-FAST-NEXT:    i32.const $push51=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push30=, $pop29, $pop51
+; NO-SIMD128-FAST-NEXT:    i32.const $push50=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push31=, $pop30, $pop50
+; NO-SIMD128-FAST-NEXT:    i32.store16 10($0), $pop31
+; NO-SIMD128-FAST-NEXT:    i32.const $push49=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push33=, $7, $pop49
+; NO-SIMD128-FAST-NEXT:    i32.const $push48=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push32=, $15, $pop48
+; NO-SIMD128-FAST-NEXT:    i32.add $push34=, $pop33, $pop32
+; NO-SIMD128-FAST-NEXT:    i32.const $push47=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push35=, $pop34, $pop47
+; NO-SIMD128-FAST-NEXT:    i32.const $push46=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push36=, $pop35, $pop46
+; NO-SIMD128-FAST-NEXT:    i32.store16 12($0), $pop36
+; NO-SIMD128-FAST-NEXT:    i32.const $push45=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push38=, $8, $pop45
+; NO-SIMD128-FAST-NEXT:    i32.const $push44=, 65535
+; NO-SIMD128-FAST-NEXT:    i32.and $push37=, $16, $pop44
+; NO-SIMD128-FAST-NEXT:    i32.add $push39=, $pop38, $pop37
+; NO-SIMD128-FAST-NEXT:    i32.const $push43=, 1
+; NO-SIMD128-FAST-NEXT:    i32.add $push40=, $pop39, $pop43
+; NO-SIMD128-FAST-NEXT:    i32.const $push42=, 1
+; NO-SIMD128-FAST-NEXT:    i32.shr_u $push41=, $pop40, $pop42
+; NO-SIMD128-FAST-NEXT:    i32.store16 14($0), $pop41
+; NO-SIMD128-FAST-NEXT:    return
+  %x.zext = zext <8 x i16> %x to <8 x i32>
+  %y.zext = zext <8 x i16> %y to <8 x i32>
+  %a = add <8 x i32> %x.zext, %y.zext
+  %b = add <8 x i32> %a, splat (i32 1)
+  %c = lshr <8 x i32> %b, splat (i32 1)
+  %c.trunc = trunc <8 x i32> %c to <8 x i16>
+  ret <8 x i16> %c.trunc
+}
 define <16 x i8> @avgr_u_v16i8_wrap(<16 x i8> %x, <16 x i8> %y) {
 ; SIMD128-LABEL: avgr_u_v16i8_wrap:
 ; SIMD128:         .functype avgr_u_v16i8_wrap (v128, v128) -> (v128)
diff --git a/llvm/test/CodeGen/WebAssembly/simd-setcc.ll b/llvm/test/CodeGen/WebAssembly/simd-setcc.ll
new file mode 100644
index 0000000000000..127fd4e963039
--- /dev/null
+++ b/llvm/test/CodeGen/WebAssembly/simd-setcc.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
+
+target triple = "wasm32-unknown-unknown"
+
+declare i32 @memcmp(ptr, ptr, i32)
+
+define i1 @setcc_load(ptr %a, ptr %b) {
+; CHECK-LABEL: setcc_load:
+; CHECK:         .functype setcc_load (i32, i32) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.load $push1=, 0($0):p2align=0
+; CHECK-NEXT:    v128.load $push0=, 0($1):p2align=0
+; CHECK-NEXT:    i8x16.eq $push2=, $pop1, $pop0
+; CHECK-NEXT:    i8x16.all_true $push3=, $pop2
+; CHECK-NEXT:    return $pop3
+  %cmp_16 = call i32 @memcmp(ptr %a, ptr %b, i32 16)
+  %res = icmp eq i32 %cmp_16, 0
+  ret i1 %res
+}
+
+; INFO: Negative test: noimplicitfloat disables simd
+define i1 @setcc_load_should_not_vectorize(ptr %a, ptr %b) noimplicitfloat {
+; CHECK-LABEL: setcc_load_should_not_vectorize:
+; CHECK:         .functype setcc_load_should_not_vectorize (i32, i32) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i64.load $push4=, 0($0):p2align=0
+; CHECK-NEXT:    i64.load $push3=, 0($1):p2align=0
+; CHECK-NEXT:    i64.xor $push5=, $pop4, $pop3
+; CHECK-NEXT:    i64.load $push1=, 8($0):p2align=0
+; CHECK-NEXT:    i64.load $push0=, 8($1):p2align=0
+; CHECK-NEXT:    i64.xor $push2=, $pop1, $pop0
+; CHECK-NEXT:    i64.or $push6=, $pop5, $pop2
+; CHECK-NEXT:    i64.eqz $push7=, $pop6
+; CHECK-NEXT:    return $pop7
+  %cmp_16 = call i32 @memcmp(ptr %a, ptr %b, i32 16)
+  %res = icmp eq i32 %cmp_16, 0
+  ret i1 %res
+}
+
+define i1 @setcc_eq_const_i128(ptr %ptr) {
+; CHECK-LABEL: setcc_eq_const_i128:
+; CHECK:         .functype setcc_eq_const_i128 (i32) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.load $push0=, 0($0)
+; CHECK-NEXT:    v128.const $push1=, 6, 0
+; CHECK-NEXT:    i8x16.eq $push2=, $pop0, $pop1
+; CHECK-NEXT:    i8x16.all_true $push3=, $pop2
+; CHECK-NEXT:    return $pop3
+  %l = load i128, ptr %ptr
+  %res = icmp eq i128 %l, 6
+  ret i1 %res
+}
+
+define i1 @setcc_ne_const_i128(ptr %ptr) {
+; CHECK-LABEL: setcc_ne_const_i128:
+; CHECK:         .functype setcc_ne_const_i128 (i32) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    v128.load $push0=, 0($0)
+; CHECK-NEXT:    v128.const $push1=, 16, 0
+; CHECK-NEXT:    i8x16.ne $push2=, $pop0, $pop1
+; CHECK-NEXT:    v128.any_true $push3=, $pop2
+; CHECK-NEXT:    return $pop3
+  %l = load i128, ptr %ptr
+  %res = icmp ne i128 %l, 16
+  ret i1 %res
+}
+
+; INFO: Negative test: only eq and ne works
+define i1 @setcc_slt_const_i128(ptr %ptr) {
+; CHECK-LABEL: setcc_slt_const_i128:
+; CHECK:         .functype setcc_slt_const_i128 (i32) -> (i32)
+; CHECK-NEXT:  # %bb.0:
+; CHECK-NEXT:    i64.load $push2=, 0($0)
+; CHECK-NEXT:    i64.const $push3=, 25
+; CHECK-NEXT:    i64.lt_u $push4=, $pop2, $pop3
+; CHECK-NEXT:    i64.load $push8=, 8($0)
+; CHECK-NEXT:    local.tee $push7=, $1=, $pop8
+; CHECK-NEXT:    i64.const $push0=, 0
+; CHECK-NEXT:    i64.lt_s $push1=, $pop7, $pop0
+; CHECK-NEXT:    i64.eqz $push5=, $1
+; CHECK-NEXT:    i32.select $push6=, $pop4, $pop1, $pop5
+; CHECK-NEXT:    return $pop6
+  %l = load i128, ptr %ptr
+  %res = icmp slt i128 %l, 25
+  ret i1 %res
+}
diff --git a/llvm/test/CodeGen/X86/GlobalISel/ptrtoaddr.ll b/llvm/test/CodeGen/X86/GlobalISel/ptrtoaddr.ll
new file mode 100644
index 0000000000000..f65d99df53ee3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/ptrtoaddr.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux-gnu -global-isel < %s -o - | FileCheck %s --check-prefix=CHECK
+
+define i1 @ptrtoaddr_1(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    xorb $1, %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %trunc = trunc i64 %addr to i1
+  %ret = xor i1 %trunc, 1
+  ret i1 %ret
+}
+
+define i8 @ptrtoaddr_8(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    notb %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %trunc = trunc i64 %addr to i8
+  %ret = xor i8 %trunc, -1
+  ret i8 %ret
+}
+
+define i16 @ptrtoaddr_16(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    notw %ax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %trunc = trunc i64 %addr to i16
+  %ret = xor i16 %trunc, -1
+  ret i16 %ret
+}
+
+define i32 @ptrtoaddr_32(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    notl %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %trunc = trunc i64 %addr to i32
+  %ret = xor i32 %trunc, -1
+  ret i32 %ret
+}
+
+define i64 @ptrtoaddr_64(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    notq %rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %ret = xor i64 %addr, -1
+  ret i64 %ret
+}
+
+define i128 @ptrtoaddr_128(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_128:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    notq %rax
+; CHECK-NEXT:    notq %rdx
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %ext = zext i64 %addr to i128
+  %ret = xor i128 %ext, -1
+  ret i128 %ret
+}
+
+; TODO: Vector version cannot be handled by GlobalIsel yet (same error as ptrtoint: https://github.com/llvm/llvm-project/issues/150875).
+; define <2 x i64> @ptrtoaddr_vec(<2 x ptr> %p) {
+; entry:
+;  %addr = ptrtoaddr <2 x ptr> %p to <2 x i64>
+;  %ret = xor <2 x i64> %addr, <i64 -1, i64 -1>
+;  ret <2 x i64> %ret
+;}
+
+; UTC_ARGS: --disable
+
+@foo = global [16 x i8] zeroinitializer
+@addr = global i64 ptrtoaddr (ptr @foo to i64)
+; CHECK:      addr:
+; CHECK-NEXT:  .quad	foo
+; CHECK-NEXT:  .size	addr, 8
+@addr_plus_one = global i64 ptrtoaddr (ptr getelementptr (i8, ptr @foo, i64 1) to i64)
+; CHECK:      addr_plus_one:
+; CHECK-NEXT:  .quad	foo+1
+; CHECK-NEXT:  .size	addr_plus_one, 8
+@const_addr = global i64 ptrtoaddr (ptr getelementptr (i8, ptr null, i64 1) to i64)
+; CHECK:      const_addr:
+; CHECK-NEXT:  .quad	0+1
+; CHECK-NEXT:  .size	const_addr, 8
diff --git a/llvm/test/CodeGen/X86/PR37310.mir b/llvm/test/CodeGen/X86/PR37310.mir
index 6c68f79661bec..6a7f5bdfe70ff 100644
--- a/llvm/test/CodeGen/X86/PR37310.mir
+++ b/llvm/test/CodeGen/X86/PR37310.mir
@@ -97,8 +97,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
   - { id: 0, name: q, type: default, offset: 0, size: 512, alignment: 16, 
diff --git a/llvm/test/CodeGen/X86/StackColoring-use-between-allocas.mir b/llvm/test/CodeGen/X86/StackColoring-use-between-allocas.mir
index b820541da2be5..a9922227b194e 100644
--- a/llvm/test/CodeGen/X86/StackColoring-use-between-allocas.mir
+++ b/llvm/test/CodeGen/X86/StackColoring-use-between-allocas.mir
@@ -118,8 +118,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 1, alignment: 8,
diff --git a/llvm/test/CodeGen/X86/align-basic-block-sections.mir b/llvm/test/CodeGen/X86/align-basic-block-sections.mir
index 02ccbcf0c0897..1bf39ec39b2a0 100644
--- a/llvm/test/CodeGen/X86/align-basic-block-sections.mir
+++ b/llvm/test/CodeGen/X86/align-basic-block-sections.mir
@@ -83,8 +83,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/amx_tile_pair_configure_O0.mir b/llvm/test/CodeGen/X86/amx_tile_pair_configure_O0.mir
index 7f862db421686..ab12ab3a4f13d 100644
--- a/llvm/test/CodeGen/X86/amx_tile_pair_configure_O0.mir
+++ b/llvm/test/CodeGen/X86/amx_tile_pair_configure_O0.mir
@@ -43,8 +43,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/X86/amx_tile_pair_configure_O2.mir b/llvm/test/CodeGen/X86/amx_tile_pair_configure_O2.mir
index 3a03fbe6ec72c..c7d241f8a98b6 100644
--- a/llvm/test/CodeGen/X86/amx_tile_pair_configure_O2.mir
+++ b/llvm/test/CodeGen/X86/amx_tile_pair_configure_O2.mir
@@ -93,8 +93,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 64, alignment: 4,
diff --git a/llvm/test/CodeGen/X86/amx_tile_pair_copy.mir b/llvm/test/CodeGen/X86/amx_tile_pair_copy.mir
index a19ff2ad47e7c..66b15aa5b3cde 100644
--- a/llvm/test/CodeGen/X86/amx_tile_pair_copy.mir
+++ b/llvm/test/CodeGen/X86/amx_tile_pair_copy.mir
@@ -46,8 +46,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 43, name: '', type: default, offset: 0, size: 64, alignment: 4,
diff --git a/llvm/test/CodeGen/X86/amx_tile_pair_preconfigure_O0.mir b/llvm/test/CodeGen/X86/amx_tile_pair_preconfigure_O0.mir
index e3f15c6c1f295..1e3b242bca96c 100644
--- a/llvm/test/CodeGen/X86/amx_tile_pair_preconfigure_O0.mir
+++ b/llvm/test/CodeGen/X86/amx_tile_pair_preconfigure_O0.mir
@@ -59,8 +59,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 18, name: '', type: default, offset: 0, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/X86/amx_tile_pair_preconfigure_O2.mir b/llvm/test/CodeGen/X86/amx_tile_pair_preconfigure_O2.mir
index 89f6773157508..ac2cdb4a50568 100644
--- a/llvm/test/CodeGen/X86/amx_tile_pair_preconfigure_O2.mir
+++ b/llvm/test/CodeGen/X86/amx_tile_pair_preconfigure_O2.mir
@@ -57,8 +57,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir
index e478e66c24dbe..0f13a5b195570 100644
--- a/llvm/test/CodeGen/X86/apx/domain-reassignment.mir
+++ b/llvm/test/CodeGen/X86/apx/domain-reassignment.mir
@@ -109,8 +109,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -255,8 +255,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -375,8 +375,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -487,8 +487,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -590,8 +590,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -686,8 +686,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -756,8 +756,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -829,8 +829,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -911,8 +911,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/X86/apx/memfold-nd2rmw.mir b/llvm/test/CodeGen/X86/apx/memfold-nd2rmw.mir
index 54e140686876f..7b4945b3a243f 100644
--- a/llvm/test/CodeGen/X86/apx/memfold-nd2rmw.mir
+++ b/llvm/test/CodeGen/X86/apx/memfold-nd2rmw.mir
@@ -125,8 +125,8 @@ frameInfo:
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 1, alignment: 16, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/X86/attr-dontcall.ll b/llvm/test/CodeGen/X86/attr-dontcall.ll
index 7de44b81dfeaa..9024280fda027 100644
--- a/llvm/test/CodeGen/X86/attr-dontcall.ll
+++ b/llvm/test/CodeGen/X86/attr-dontcall.ll
@@ -1,6 +1,6 @@
 ; RUN: not llc -mtriple=x86_64 -global-isel=0 -fast-isel=0 -stop-after=finalize-isel < %s 2>&1 | FileCheck %s
 ; RUN: not llc -mtriple=x86_64 -global-isel=0 -fast-isel=1 -stop-after=finalize-isel < %s 2>&1 | FileCheck %s
-; RUN: not llc -mtriple=x86_64 -global-isel=1 -fast-isel=0 -stop-after=irtranslator -global-isel-abort=0 < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=x86_64 -global-isel=1 -fast-isel=0 -global-isel-abort=0 < %s 2>&1 | FileCheck %s
 
 declare void @foo() "dontcall-error"="e"
 define void @bar() {
diff --git a/llvm/test/CodeGen/X86/attr-function-return.mir b/llvm/test/CodeGen/X86/attr-function-return.mir
index 91c03e862182f..96db27b3e5a82 100644
--- a/llvm/test/CodeGen/X86/attr-function-return.mir
+++ b/llvm/test/CodeGen/X86/attr-function-return.mir
@@ -45,8 +45,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/avoid-sfb-g-no-change.mir b/llvm/test/CodeGen/X86/avoid-sfb-g-no-change.mir
index 93292990dee02..99fee2756b88e 100644
--- a/llvm/test/CodeGen/X86/avoid-sfb-g-no-change.mir
+++ b/llvm/test/CodeGen/X86/avoid-sfb-g-no-change.mir
@@ -127,8 +127,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
@@ -185,8 +185,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/X86/avoid-sfb-g-no-change2.mir b/llvm/test/CodeGen/X86/avoid-sfb-g-no-change2.mir
index 4da4f03987672..50b2433950e01 100644
--- a/llvm/test/CodeGen/X86/avoid-sfb-g-no-change2.mir
+++ b/llvm/test/CodeGen/X86/avoid-sfb-g-no-change2.mir
@@ -138,8 +138,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/X86/avoid-sfb-g-no-change3.mir b/llvm/test/CodeGen/X86/avoid-sfb-g-no-change3.mir
index 19e3f38c56b78..7a4b99395ea1e 100644
--- a/llvm/test/CodeGen/X86/avoid-sfb-g-no-change3.mir
+++ b/llvm/test/CodeGen/X86/avoid-sfb-g-no-change3.mir
@@ -154,8 +154,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/X86/avoid-sfb-offset.mir b/llvm/test/CodeGen/X86/avoid-sfb-offset.mir
index 978b3e79dcb25..eec19e779d70d 100644
--- a/llvm/test/CodeGen/X86/avoid-sfb-offset.mir
+++ b/llvm/test/CodeGen/X86/avoid-sfb-offset.mir
@@ -67,8 +67,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: a, type: default, offset: 0, size: 144, alignment: 16,
diff --git a/llvm/test/CodeGen/X86/avx-load-store.ll b/llvm/test/CodeGen/X86/avx-load-store.ll
index 3f856d33145d8..cc2cedb6b8b20 100644
--- a/llvm/test/CodeGen/X86/avx-load-store.ll
+++ b/llvm/test/CodeGen/X86/avx-load-store.ll
@@ -34,7 +34,7 @@ define void @test_256_load(ptr nocapture %d, ptr nocapture %f, ptr nocapture %i)
 ;
 ; CHECK_O0-LABEL: test_256_load:
 ; CHECK_O0:       # %bb.0: # %entry
-; CHECK_O0-NEXT:    subq $184, %rsp
+; CHECK_O0-NEXT:    subq $136, %rsp
 ; CHECK_O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK_O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
 ; CHECK_O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
@@ -54,7 +54,7 @@ define void @test_256_load(ptr nocapture %d, ptr nocapture %f, ptr nocapture %i)
 ; CHECK_O0-NEXT:    vmovapd %ymm2, (%rdi)
 ; CHECK_O0-NEXT:    vmovaps %ymm1, (%rsi)
 ; CHECK_O0-NEXT:    vmovdqa %ymm0, (%rdx)
-; CHECK_O0-NEXT:    addq $184, %rsp
+; CHECK_O0-NEXT:    addq $136, %rsp
 ; CHECK_O0-NEXT:    vzeroupper
 ; CHECK_O0-NEXT:    retq
 entry:
diff --git a/llvm/test/CodeGen/X86/avx512f-256-set0.mir b/llvm/test/CodeGen/X86/avx512f-256-set0.mir
index 3915599df524d..a77c724b5063a 100644
--- a/llvm/test/CodeGen/X86/avx512f-256-set0.mir
+++ b/llvm/test/CodeGen/X86/avx512f-256-set0.mir
@@ -52,8 +52,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 constants:       []
diff --git a/llvm/test/CodeGen/X86/avx512f-large-stack.ll b/llvm/test/CodeGen/X86/avx512f-large-stack.ll
new file mode 100644
index 0000000000000..3cb5391c56abf
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512f-large-stack.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp --version 4
+; RUN: llc -O0 -mtriple=x86_64 -mattr=+avx512f -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK
+define void @f(i16 %LGV2, i1 %LGV3) {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0: # %BB
+; CHECK-NEXT:    subq $2147483528, %rsp # imm = 0x7FFFFF88
+; CHECK-NEXT:    .cfi_def_cfa_offset 2147483536
+; CHECK-NEXT:    movb %sil, %cl
+; CHECK-NEXT:    movw %di, %ax
+; CHECK-NEXT:    movswq %ax, %rax
+; CHECK-NEXT:    andb $1, %cl
+; CHECK-NEXT:    movabsq $-2147483768, %rdx # imm = 0xFFFFFFFF7FFFFF88
+; CHECK-NEXT:    movb %cl, (%rsp,%rdx)
+; CHECK-NEXT:    addq $2147483528, %rsp # imm = 0x7FFFFF88
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+BB:
+  %A = alloca i1, i33 2147483648, align 1
+  %G = getelementptr i1, ptr %A, i16 %LGV2
+  %G4 = getelementptr i1, ptr %G, i32 -2147483648
+  store i1 %LGV3, ptr %G4, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-mir-parse.mir b/llvm/test/CodeGen/X86/basic-block-address-map-mir-parse.mir
index a49a4e2726021..34f40b28a846e 100644
--- a/llvm/test/CodeGen/X86/basic-block-address-map-mir-parse.mir
+++ b/llvm/test/CodeGen/X86/basic-block-address-map-mir-parse.mir
@@ -93,8 +93,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default, 
       callee-saved-register: '', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/CodeGen/X86/basic-block-sections-mir-parse.mir b/llvm/test/CodeGen/X86/basic-block-sections-mir-parse.mir
index e49ff140935a3..b71ccc5eb89d2 100644
--- a/llvm/test/CodeGen/X86/basic-block-sections-mir-parse.mir
+++ b/llvm/test/CodeGen/X86/basic-block-sections-mir-parse.mir
@@ -70,8 +70,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default, 
       callee-saved-register: '', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/CodeGen/X86/bitcnt-false-dep.ll b/llvm/test/CodeGen/X86/bitcnt-false-dep.ll
index 5f576c8586285..793cbb8f75bdc 100644
--- a/llvm/test/CodeGen/X86/bitcnt-false-dep.ll
+++ b/llvm/test/CodeGen/X86/bitcnt-false-dep.ll
@@ -1,6 +1,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell | FileCheck %s --check-prefix=HSW
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake | FileCheck %s --check-prefix=SKL
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skx | FileCheck %s --check-prefix=SKL
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=alderlake | FileCheck %s --check-prefix=ADL
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=silvermont -mattr=+lzcnt,+bmi | FileCheck %s --check-prefix=SKL
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=goldmont -mattr=+lzcnt,+bmi | FileCheck %s --check-prefix=SKL
 
@@ -37,6 +38,10 @@ ret:
 ;SKL-LABEL:@loopdep_popcnt32
 ;SKL: xorl [[GPR0:%e[a-d]x]], [[GPR0]]
 ;SKL-NEXT: popcntl {{.*}}, [[GPR0]]
+
+;ADL-LABEL:@loopdep_popcnt32
+;ADL-NOT: xor
+;ADL: popcntl
 }
 
 define i64 @loopdep_popcnt64(ptr nocapture %x, ptr nocapture %y) nounwind {
@@ -63,6 +68,10 @@ ret:
 ;SKL-LABEL:@loopdep_popcnt64
 ;SKL: xorl %e[[GPR0:[a-d]x]], %e[[GPR0]]
 ;SKL-NEXT: popcntq {{.*}}, %r[[GPR0]]
+
+;ADL-LABEL:@loopdep_popcnt64
+;ADL-NOT: xor
+;ADL: popcntq
 }
 
 define i32 @loopdep_tzct32(ptr nocapture %x, ptr nocapture %y) nounwind {
diff --git a/llvm/test/CodeGen/X86/break-false-dep-crash.mir b/llvm/test/CodeGen/X86/break-false-dep-crash.mir
index 713633c322890..febc49da1f832 100644
--- a/llvm/test/CodeGen/X86/break-false-dep-crash.mir
+++ b/llvm/test/CodeGen/X86/break-false-dep-crash.mir
@@ -93,8 +93,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/callbr-asm-loop.ll b/llvm/test/CodeGen/X86/callbr-asm-loop.ll
new file mode 100644
index 0000000000000..0b6898815f8cc
--- /dev/null
+++ b/llvm/test/CodeGen/X86/callbr-asm-loop.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+
+; RUN: llc -O1 -mtriple=i686-- < %s | FileCheck %s
+
+; Test that causes multiple defs of %eax.
+define i32 @loop1() nounwind {
+; CHECK-LABEL: loop1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB0_1: # %tailrecurse
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movl $1, %edx
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    jmp .LBB0_1
+; CHECK-NEXT:  .LBB0_2: # Inline asm indirect target
+; CHECK-NEXT:    # %tailrecurse.tailrecurse_crit_edge
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    # Label of block must be emitted
+; CHECK-NEXT:    jmp .LBB0_1
+; CHECK-NEXT:  .LBB0_3: # Inline asm indirect target
+; CHECK-NEXT:    # %lab2.split
+; CHECK-NEXT:    # Label of block must be emitted
+; CHECK-NEXT:    movl %edx, %eax
+; CHECK-NEXT:    retl
+entry:
+  br label %tailrecurse
+
+tailrecurse:
+  %0 = callbr { i32, i32 } asm "", "={ax},={dx},0,1,!i,!i"(i32 0, i32 1) #1
+          to label %tailrecurse.backedge [label %tailrecurse.backedge, label %lab2.split]
+
+tailrecurse.backedge:
+  br label %tailrecurse
+
+lab2.split:
+  %asmresult5 = extractvalue { i32, i32 } %0, 1
+  ret i32 %asmresult5
+}
diff --git a/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir b/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir
index 1923114d0c297..f5af0ad91f40f 100644
--- a/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir
+++ b/llvm/test/CodeGen/X86/callbr-asm-outputs-regallocfast.mir
@@ -96,8 +96,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: retval, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/X86/cf-opt-memops.mir b/llvm/test/CodeGen/X86/cf-opt-memops.mir
index 44dead87d2e1c..86719a2e2f3b7 100644
--- a/llvm/test/CodeGen/X86/cf-opt-memops.mir
+++ b/llvm/test/CodeGen/X86/cf-opt-memops.mir
@@ -68,8 +68,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/cfi-epilogue-with-return.mir b/llvm/test/CodeGen/X86/cfi-epilogue-with-return.mir
index 583e54b097faf..4f80f4f87dd0d 100644
--- a/llvm/test/CodeGen/X86/cfi-epilogue-with-return.mir
+++ b/llvm/test/CodeGen/X86/cfi-epilogue-with-return.mir
@@ -21,8 +21,10 @@ liveins:
 frameInfo:
   maxAlignment:    1
   hasCalls:        true
-  savePoint:       '%bb.1'
-  restorePoint:    '%bb.1'
+  savePoint:
+    - point:           '%bb.1'
+  restorePoint:
+    - point:           '%bb.1'
 machineFunctionInfo: {}
 body:             |
   bb.0:
diff --git a/llvm/test/CodeGen/X86/cfi-epilogue-without-return.mir b/llvm/test/CodeGen/X86/cfi-epilogue-without-return.mir
index 8f04721489608..38c081c94ab91 100644
--- a/llvm/test/CodeGen/X86/cfi-epilogue-without-return.mir
+++ b/llvm/test/CodeGen/X86/cfi-epilogue-without-return.mir
@@ -28,8 +28,10 @@ liveins:
 frameInfo:
   maxAlignment:    1
   hasCalls:        true
-  savePoint:       '%bb.1'
-  restorePoint:    '%bb.1'
+  savePoint:
+    - point:           '%bb.1'
+  restorePoint:
+    - point:           '%bb.1'
 machineFunctionInfo: {}
 body:             |
   bb.0:
diff --git a/llvm/test/CodeGen/X86/combine-movmsk.ll b/llvm/test/CodeGen/X86/combine-movmsk.ll
index baa0553a34f48..c107a981f1a3d 100644
--- a/llvm/test/CodeGen/X86/combine-movmsk.ll
+++ b/llvm/test/CodeGen/X86/combine-movmsk.ll
@@ -489,6 +489,31 @@ define i32 @or_pmovmskb_pmovmskb(<16 x i8> %a0, <8 x i16> %a1) {
   ret i32 %7
 }
 
+; FREEZE(MOVMSK(X)) -> MOVMSK(FREEZE(X))
+define i32 @movmskps_freeze(<4 x i32> %a0) {
+; SSE-LABEL: movmskps_freeze:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movmskps %xmm0, %eax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: movmskps_freeze:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovmskps %xmm0, %eax
+; AVX-NEXT:    retq
+;
+; ADL-LABEL: movmskps_freeze:
+; ADL:       # %bb.0:
+; ADL-NEXT:    vmovmskps %xmm0, %eax
+; ADL-NEXT:    retq
+  %1 = icmp slt <4 x i32> %a0, zeroinitializer
+  %2 = sext <4 x i1> %1 to <4 x i32>
+  %3 = bitcast <4 x i32> %2 to <4 x float>
+  %4 = tail call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %3)
+  %5 = freeze i32 %4
+  %6 = and i32 %5, 15
+  ret i32 %6
+}
+
 ; We can't fold to ptest if we're not checking every pcmpeq result
 define i32 @movmskps_ptest_numelts_mismatch(<16 x i8> %a0) {
 ; SSE-LABEL: movmskps_ptest_numelts_mismatch:
diff --git a/llvm/test/CodeGen/X86/conditional-tailcall-samedest.mir b/llvm/test/CodeGen/X86/conditional-tailcall-samedest.mir
index e2f5b67f3114b..82572663c0c33 100644
--- a/llvm/test/CodeGen/X86/conditional-tailcall-samedest.mir
+++ b/llvm/test/CodeGen/X86/conditional-tailcall-samedest.mir
@@ -94,8 +94,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/X86/cse-two-preds.mir b/llvm/test/CodeGen/X86/cse-two-preds.mir
index c821b3b94101d..94dc1c494cf45 100644
--- a/llvm/test/CodeGen/X86/cse-two-preds.mir
+++ b/llvm/test/CodeGen/X86/cse-two-preds.mir
@@ -85,8 +85,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/domain-reassignment.mir b/llvm/test/CodeGen/X86/domain-reassignment.mir
index 1889982ac2618..18c505718d987 100644
--- a/llvm/test/CodeGen/X86/domain-reassignment.mir
+++ b/llvm/test/CodeGen/X86/domain-reassignment.mir
@@ -109,8 +109,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -255,8 +255,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -375,8 +375,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -487,8 +487,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -590,8 +590,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -686,8 +686,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -756,8 +756,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -829,8 +829,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
@@ -911,8 +911,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/X86/early-clobber.mir b/llvm/test/CodeGen/X86/early-clobber.mir
new file mode 100644
index 0000000000000..429331524d177
--- /dev/null
+++ b/llvm/test/CodeGen/X86/early-clobber.mir
@@ -0,0 +1,76 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=i386-unknown-linux-gnu -start-before=twoaddressinstruction -stop-after=postrapseudos -verify-machineinstrs -o - %s | FileCheck %s
+
+# Test register live range that is split from rematerializing. The live range should
+# start with Slot_EarlyClobber instead of Slot_Register. Machineverifer can check it.
+
+--- |
+
+  define void @test() nounwind {
+    ret void
+  }
+
+...
+---
+name:            test
+alignment:       16
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    4
+fixedStack:
+  - { id: 0, size: 4, alignment: 16, isImmutable: true }
+machineFunctionInfo: {}
+body:             |
+  ; CHECK-LABEL: name: test
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK-NEXT:   liveins: $ebp, $ebx, $edi, $esi
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   frame-setup PUSH32r killed $ebp, implicit-def $esp, implicit $esp
+  ; CHECK-NEXT:   frame-setup PUSH32r killed $ebx, implicit-def $esp, implicit $esp
+  ; CHECK-NEXT:   frame-setup PUSH32r killed $edi, implicit-def $esp, implicit $esp
+  ; CHECK-NEXT:   frame-setup PUSH32r killed $esi, implicit-def $esp, implicit $esp
+  ; CHECK-NEXT:   $esp = frame-setup SUB32ri $esp, 12, implicit-def dead $eflags
+  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 12 /* clobber */, implicit-def dead early-clobber $ebp, 12 /* clobber */, implicit-def dead early-clobber $eax, 12 /* clobber */, implicit-def dead early-clobber $ebx, 12 /* clobber */, implicit-def dead early-clobber $ecx, 12 /* clobber */, implicit-def dead early-clobber $edx, 12 /* clobber */, implicit-def dead early-clobber $esi, 12 /* clobber */, implicit-def dead early-clobber $edi, 12 /* clobber */, implicit-def dead early-clobber $df, 12 /* clobber */, implicit-def early-clobber $fpsw, 12 /* clobber */, implicit-def dead early-clobber $eflags
+  ; CHECK-NEXT:   early-clobber renamable $eax = XOR32rr undef $eax, undef $eax, implicit-def dead $eflags
+  ; CHECK-NEXT:   TEST8rr renamable $al, renamable $al, implicit-def $eflags
+  ; CHECK-NEXT:   JCC_1 %bb.2, 5, implicit $eflags
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $eax = MOV32ri 1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   liveins: $eax
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $esp = frame-destroy ADD32ri $esp, 12, implicit-def dead $eflags
+  ; CHECK-NEXT:   $esi = frame-destroy POP32r implicit-def $esp, implicit $esp
+  ; CHECK-NEXT:   $edi = frame-destroy POP32r implicit-def $esp, implicit $esp
+  ; CHECK-NEXT:   $ebx = frame-destroy POP32r implicit-def $esp, implicit $esp
+  ; CHECK-NEXT:   $ebp = frame-destroy POP32r implicit-def $esp, implicit $esp
+  ; CHECK-NEXT:   RET 0, $eax
+  bb.0:
+    successors: %bb.1, %bb.2
+
+    early-clobber %0:gr32_abcd = MOV32r0 implicit-def dead $eflags
+    INLINEASM &"", 1 /* sideeffect attdialect */, 12 /* clobber */, implicit-def dead early-clobber $ebp, 12 /* clobber */, implicit-def dead early-clobber $eax, 12 /* clobber */, implicit-def dead early-clobber $ebx, 12 /* clobber */, implicit-def dead early-clobber $ecx, 12 /* clobber */, implicit-def dead early-clobber $edx, 12 /* clobber */, implicit-def dead early-clobber $esi, 12 /* clobber */, implicit-def dead early-clobber $edi, 12 /* clobber */, implicit-def dead early-clobber $df, 12 /* clobber */, implicit-def early-clobber $fpsw, 12 /* clobber */, implicit-def dead early-clobber $eflags
+    %4:gr8 = COPY %0.sub_8bit
+    TEST8rr killed %4, %4, implicit-def $eflags
+    JCC_1 %bb.2, 5, implicit killed $eflags
+    JMP_1 %bb.1
+
+  bb.1:
+    %1:gr32 = MOV32ri 1
+    %5:gr32 = COPY killed %1
+    JMP_1 %bb.3
+
+  bb.2:
+    %5:gr32 = COPY killed %0
+
+  bb.3:
+    %3:gr32 = COPY killed %5
+    $eax = COPY killed %3
+    RET 0, killed $eax
+
+...
diff --git a/llvm/test/CodeGen/X86/fma_patterns.ll b/llvm/test/CodeGen/X86/fma_patterns.ll
index 0ffcb8c46cef9..dc35c8f8dc657 100644
--- a/llvm/test/CodeGen/X86/fma_patterns.ll
+++ b/llvm/test/CodeGen/X86/fma_patterns.ll
@@ -1567,7 +1567,7 @@ define <4 x double> @test_v4f64_interp(<4 x double> %x, <4 x double> %y, <4 x do
 ; Pattern: (fneg (fma x, y, z)) -> (fma x, -y, -z)
 ;
 
-define <4 x float> @test_v4f32_fneg_fmadd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+define <4 x float> @test_v4f32_fneg_fmadd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
 ; FMA-LABEL: test_v4f32_fneg_fmadd:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2
@@ -1582,13 +1582,13 @@ define <4 x float> @test_v4f32_fneg_fmadd(<4 x float> %a0, <4 x float> %a1, <4 x
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2
 ; AVX512-NEXT:    retq
-  %mul = fmul nsz <4 x float> %a0, %a1
-  %add = fadd nsz <4 x float> %mul, %a2
-  %neg = fsub nsz <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %add
+  %mul = fmul contract nsz <4 x float> %a0, %a1
+  %add = fadd contract nsz <4 x float> %mul, %a2
+  %neg = fsub contract nsz <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %add
   ret <4 x float> %neg
 }
 
-define <4 x double> @test_v4f64_fneg_fmsub(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+define <4 x double> @test_v4f64_fneg_fmsub(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) {
 ; FMA-LABEL: test_v4f64_fneg_fmsub:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2
@@ -1609,7 +1609,7 @@ define <4 x double> @test_v4f64_fneg_fmsub(<4 x double> %a0, <4 x double> %a1, <
   ret <4 x double> %neg
 }
 
-define <4 x float> @test_v4f32_fneg_fnmadd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) #0 {
+define <4 x float> @test_v4f32_fneg_fnmadd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
 ; FMA-LABEL: test_v4f32_fneg_fnmadd:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2
@@ -1624,14 +1624,14 @@ define <4 x float> @test_v4f32_fneg_fnmadd(<4 x float> %a0, <4 x float> %a1, <4
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2
 ; AVX512-NEXT:    retq
-  %mul = fmul nsz <4 x float> %a0, %a1
-  %neg0 = fsub nsz <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %mul
-  %add = fadd nsz <4 x float> %neg0, %a2
-  %neg1 = fsub nsz <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %add
+  %mul = fmul contract nsz <4 x float> %a0, %a1
+  %neg0 = fsub contract nsz <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %mul
+  %add = fadd contract nsz <4 x float> %neg0, %a2
+  %neg1 = fsub contract nsz <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %add
   ret <4 x float> %neg1
 }
 
-define <4 x double> @test_v4f64_fneg_fnmsub(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) #0 {
+define <4 x double> @test_v4f64_fneg_fnmsub(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) {
 ; FMA-LABEL: test_v4f64_fneg_fnmsub:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2
@@ -1646,10 +1646,10 @@ define <4 x double> @test_v4f64_fneg_fnmsub(<4 x double> %a0, <4 x double> %a1,
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2
 ; AVX512-NEXT:    retq
-  %mul = fmul nsz  <4 x double> %a0, %a1
-  %neg0 = fsub nsz <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %mul
-  %sub = fsub nsz <4 x double> %neg0, %a2
-  %neg1 = fsub nsz <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %sub
+  %mul = fmul contract nsz  <4 x double> %a0, %a1
+  %neg0 = fsub contract nsz <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %mul
+  %sub = fsub contract nsz <4 x double> %neg0, %a2
+  %neg1 = fsub contract nsz <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %sub
   ret <4 x double> %neg1
 }
 
@@ -1657,7 +1657,7 @@ define <4 x double> @test_v4f64_fneg_fnmsub(<4 x double> %a0, <4 x double> %a1,
 ; Pattern: (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2)
 ;
 
-define <4 x float> @test_v4f32_fma_x_c1_fmul_x_c2(<4 x float> %x) #0 {
+define <4 x float> @test_v4f32_fma_x_c1_fmul_x_c2(<4 x float> %x) {
 ; FMA-LABEL: test_v4f32_fma_x_c1_fmul_x_c2:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
@@ -1672,9 +1672,9 @@ define <4 x float> @test_v4f32_fma_x_c1_fmul_x_c2(<4 x float> %x) #0 {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
-  %m0 = fmul <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 4.0>
-  %m1 = fmul <4 x float> %x, <float 4.0, float 3.0, float 2.0, float 1.0>
-  %a  = fadd <4 x float> %m0, %m1
+  %m0 = fmul contract reassoc <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 4.0>
+  %m1 = fmul contract reassoc <4 x float> %x, <float 4.0, float 3.0, float 2.0, float 1.0>
+  %a  = fadd contract reassoc <4 x float> %m0, %m1
   ret <4 x float> %a
 }
 
@@ -1682,7 +1682,7 @@ define <4 x float> @test_v4f32_fma_x_c1_fmul_x_c2(<4 x float> %x) #0 {
 ; Pattern: (fma (fmul x, c1), c2, y) -> (fma x, c1*c2, y)
 ;
 
-define <4 x float> @test_v4f32_fma_fmul_x_c1_c2_y(<4 x float> %x, <4 x float> %y) #0 {
+define <4 x float> @test_v4f32_fma_fmul_x_c1_c2_y(<4 x float> %x, <4 x float> %y) {
 ; FMA-LABEL: test_v4f32_fma_fmul_x_c1_c2_y:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1
@@ -1697,15 +1697,15 @@ define <4 x float> @test_v4f32_fma_fmul_x_c1_c2_y(<4 x float> %x, <4 x float> %y
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1
 ; AVX512-NEXT:    retq
-  %m0 = fmul <4 x float> %x,  <float 1.0, float 2.0, float 3.0, float 4.0>
-  %m1 = fmul <4 x float> %m0, <float 4.0, float 3.0, float 2.0, float 1.0>
-  %a  = fadd <4 x float> %m1, %y
+  %m0 = fmul contract reassoc <4 x float> %x,  <float 1.0, float 2.0, float 3.0, float 4.0>
+  %m1 = fmul contract reassoc <4 x float> %m0, <float 4.0, float 3.0, float 2.0, float 1.0>
+  %a  = fadd contract reassoc <4 x float> %m1, %y
   ret <4 x float> %a
 }
 
 ; Pattern: (fneg (fmul x, y)) -> (fnmsub x, y, 0)
 
-define double @test_f64_fneg_fmul(double %x, double %y) #0 {
+define double @test_f64_fneg_fmul(double %x, double %y) {
 ; FMA-LABEL: test_f64_fneg_fmul:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
@@ -1723,12 +1723,12 @@ define double @test_f64_fneg_fmul(double %x, double %y) #0 {
 ; AVX512-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2
 ; AVX512-NEXT:    retq
-  %m = fmul nsz double %x, %y
-  %n = fsub double -0.0, %m
+  %m = fmul contract nsz double %x, %y
+  %n = fsub contract double -0.0, %m
   ret double %n
 }
 
-define <4 x float> @test_v4f32_fneg_fmul(<4 x float> %x, <4 x float> %y) #0 {
+define <4 x float> @test_v4f32_fneg_fmul(<4 x float> %x, <4 x float> %y) {
 ; FMA-LABEL: test_v4f32_fneg_fmul:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vxorps %xmm2, %xmm2, %xmm2
@@ -1746,12 +1746,12 @@ define <4 x float> @test_v4f32_fneg_fmul(<4 x float> %x, <4 x float> %y) #0 {
 ; AVX512-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2
 ; AVX512-NEXT:    retq
-  %m = fmul nsz <4 x float> %x, %y
-  %n = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %m
+  %m = fmul contract nsz <4 x float> %x, %y
+  %n = fsub contract <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %m
   ret <4 x float> %n
 }
 
-define <4 x double> @test_v4f64_fneg_fmul(<4 x double> %x, <4 x double> %y) #0 {
+define <4 x double> @test_v4f64_fneg_fmul(<4 x double> %x, <4 x double> %y) {
 ; FMA-LABEL: test_v4f64_fneg_fmul:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
@@ -1769,12 +1769,12 @@ define <4 x double> @test_v4f64_fneg_fmul(<4 x double> %x, <4 x double> %y) #0 {
 ; AVX512-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2
 ; AVX512-NEXT:    retq
-  %m = fmul nsz <4 x double> %x, %y
-  %n = fsub <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %m
+  %m = fmul contract nsz <4 x double> %x, %y
+  %n = fsub contract <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %m
   ret <4 x double> %n
 }
 
-define <4 x double> @test_v4f64_fneg_fmul_no_nsz(<4 x double> %x, <4 x double> %y) #0 {
+define <4 x double> @test_v4f64_fneg_fmul_no_nsz(<4 x double> %x, <4 x double> %y) {
 ; FMA-LABEL: test_v4f64_fneg_fmul_no_nsz:
 ; FMA:       # %bb.0:
 ; FMA-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
@@ -1792,8 +1792,8 @@ define <4 x double> @test_v4f64_fneg_fmul_no_nsz(<4 x double> %x, <4 x double> %
 ; AVX512-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vxorpd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm0
 ; AVX512-NEXT:    retq
-  %m = fmul <4 x double> %x, %y
-  %n = fsub <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %m
+  %m = fmul contract <4 x double> %x, %y
+  %n = fsub contract <4 x double> <double -0.0, double -0.0, double -0.0, double -0.0>, %m
   ret <4 x double> %n
 }
 
@@ -2022,5 +2022,3 @@ define float @fadd_fma_fmul_extra_use_3(float %a, float %b, float %c, float %d,
   %a2 = fadd fast float %n0, %a1
   ret float %a2
 }
-
-attributes #0 = { "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/X86/fma_patterns_wide.ll b/llvm/test/CodeGen/X86/fma_patterns_wide.ll
index fe5ddca67470c..d910110467ee0 100644
--- a/llvm/test/CodeGen/X86/fma_patterns_wide.ll
+++ b/llvm/test/CodeGen/X86/fma_patterns_wide.ll
@@ -1053,9 +1053,9 @@ define <16 x float> @test_v16f32_fma_x_c1_fmul_x_c2(<16 x float> %x) #0 {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
 ; AVX512-NEXT:    retq
-  %m0 = fmul <16 x float> %x, <float 17.0, float 16.0, float 15.0, float 14.0, float 13.0, float 12.0, float 11.0, float 10.0, float 9.0, float 8.0, float 7.0, float 6.0, float 5.0, float 4.0, float 3.0, float 2.0>
-  %m1 = fmul <16 x float> %x, <float 16.0, float 15.0, float 14.0, float 13.0, float 12.0, float 11.0, float 10.0, float 9.0, float 8.0, float 7.0, float 6.0, float 5.0, float 4.0, float 3.0, float 2.0, float 1.0>
-  %a  = fadd <16 x float> %m0, %m1
+  %m0 = fmul contract reassoc <16 x float> %x, <float 17.0, float 16.0, float 15.0, float 14.0, float 13.0, float 12.0, float 11.0, float 10.0, float 9.0, float 8.0, float 7.0, float 6.0, float 5.0, float 4.0, float 3.0, float 2.0>
+  %m1 = fmul contract reassoc <16 x float> %x, <float 16.0, float 15.0, float 14.0, float 13.0, float 12.0, float 11.0, float 10.0, float 9.0, float 8.0, float 7.0, float 6.0, float 5.0, float 4.0, float 3.0, float 2.0, float 1.0>
+  %a  = fadd contract reassoc <16 x float> %m0, %m1
   ret <16 x float> %a
 }
 
@@ -1080,9 +1080,9 @@ define <16 x float> @test_v16f32_fma_fmul_x_c1_c2_y(<16 x float> %x, <16 x float
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * mem) + zmm1
 ; AVX512-NEXT:    retq
-  %m0 = fmul <16 x float> %x,  <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>
-  %m1 = fmul <16 x float> %m0, <float 16.0, float 15.0, float 14.0, float 13.0, float 12.0, float 11.0, float 10.0, float 9.0, float 8.0, float 7.0, float 6.0, float 5.0, float 4.0, float 3.0, float 2.0, float 1.0>
-  %a  = fadd <16 x float> %m1, %y
+  %m0 = fmul contract reassoc <16 x float> %x,  <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>
+  %m1 = fmul contract reassoc <16 x float> %m0, <float 16.0, float 15.0, float 14.0, float 13.0, float 12.0, float 11.0, float 10.0, float 9.0, float 8.0, float 7.0, float 6.0, float 5.0, float 4.0, float 3.0, float 2.0, float 1.0>
+  %a  = fadd contract reassoc <16 x float> %m1, %y
   ret <16 x float> %a
 }
 
diff --git a/llvm/test/CodeGen/X86/fp-double-rounding.ll b/llvm/test/CodeGen/X86/fp-double-rounding.ll
index 543908a10df29..88b20ff0ba8dd 100644
--- a/llvm/test/CodeGen/X86/fp-double-rounding.ll
+++ b/llvm/test/CodeGen/X86/fp-double-rounding.ll
@@ -1,20 +1,53 @@
-; RUN: llc < %s | FileCheck %s --check-prefix=CHECK --check-prefix=SAFE
-; RUN: llc < %s -enable-unsafe-fp-math | FileCheck %s --check-prefix=CHECK --check-prefix=UNSAFE
+; RUN: llc < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64--"
 
+; CHECK-LABEL: double_rounding_safe:
+; CHECK: callq __trunctfdf2
+; CHECK-NEXT: cvtsd2ss %xmm0
+define void @double_rounding_safe(ptr %x, ptr %f) {
+entry:
+  %x.fp128 = load fp128, ptr %x, align 16
+  %x.double = fptrunc fp128 %x.fp128 to double
+  %x.float = fptrunc double %x.double to float
+  store float %x.float, ptr %f, align 4
+  ret void
+}
+
+; CHECK-LABEL: double_rounding_contract_fst:
+; CHECK: callq __trunctfdf2
+; CHECK-NEXT: cvtsd2ss %xmm0
+define void @double_rounding_contract_fst(ptr %x, ptr %f) {
+entry:
+  %x.fp128 = load fp128, ptr %x, align 16
+  %x.double = fptrunc contract fp128 %x.fp128 to double
+  %x.float = fptrunc double %x.double to float
+  store float %x.float, ptr %f, align 4
+  ret void
+}
+
+; CHECK-LABEL: double_rounding_contract_snd:
+; CHECK: callq __trunctfdf2
+; CHECK-NEXT: cvtsd2ss %xmm0
+define void @double_rounding_contract_snd(ptr %x, ptr %f) {
+entry:
+  %x.fp128 = load fp128, ptr %x, align 16
+  %x.double = fptrunc fp128 %x.fp128 to double
+  %x.float = fptrunc contract double %x.double to float
+  store float %x.float, ptr %f, align 4
+  ret void
+}
+
 ; CHECK-LABEL: double_rounding:
-; SAFE: callq __trunctfdf2
-; SAFE-NEXT: cvtsd2ss %xmm0
-; UNSAFE: callq __trunctfsf2
-; UNSAFE-NOT: cvt
+; CHECK: callq __trunctfsf2
+; CHECK-NOT: cvt
 define void @double_rounding(ptr %x, ptr %f) {
 entry:
-  %0 = load fp128, ptr %x, align 16
-  %1 = fptrunc fp128 %0 to double
-  %2 = fptrunc double %1 to float
-  store float %2, ptr %f, align 4
+  %x.fp128 = load fp128, ptr %x, align 16
+  %x.double = fptrunc contract fp128 %x.fp128 to double
+  %x.float = fptrunc contract double %x.double to float
+  store float %x.float, ptr %f, align 4
   ret void
 }
 
diff --git a/llvm/test/CodeGen/X86/fptosi-sat-vector-128.ll b/llvm/test/CodeGen/X86/fptosi-sat-vector-128.ll
index 7f6d64c21724a..a251e43e4295b 100644
--- a/llvm/test/CodeGen/X86/fptosi-sat-vector-128.ll
+++ b/llvm/test/CodeGen/X86/fptosi-sat-vector-128.ll
@@ -57,6 +57,52 @@ define <4 x i1> @test_signed_v4i1_v4f32(<4 x float> %f) nounwind {
   ret <4 x i1> %x
 }
 
+define <4 x i1> @test_freeze_signed_v4i1_v4f32(<4 x float> %f) nounwind {
+; CHECK-LABEL: test_freeze_signed_v4i1_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
+; CHECK-NEXT:    movss {{.*#+}} xmm2 = [-1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    ucomiss %xmm1, %xmm1
+; CHECK-NEXT:    maxss %xmm2, %xmm1
+; CHECK-NEXT:    xorps %xmm3, %xmm3
+; CHECK-NEXT:    minss %xmm3, %xmm1
+; CHECK-NEXT:    cvttss2si %xmm1, %ecx
+; CHECK-NEXT:    cmovpl %eax, %ecx
+; CHECK-NEXT:    movd %ecx, %xmm1
+; CHECK-NEXT:    movaps %xmm0, %xmm4
+; CHECK-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
+; CHECK-NEXT:    ucomiss %xmm4, %xmm4
+; CHECK-NEXT:    maxss %xmm2, %xmm4
+; CHECK-NEXT:    minss %xmm3, %xmm4
+; CHECK-NEXT:    cvttss2si %xmm4, %ecx
+; CHECK-NEXT:    cmovpl %eax, %ecx
+; CHECK-NEXT:    movd %ecx, %xmm4
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    maxss %xmm2, %xmm1
+; CHECK-NEXT:    minss %xmm3, %xmm1
+; CHECK-NEXT:    cvttss2si %xmm1, %ecx
+; CHECK-NEXT:    ucomiss %xmm0, %xmm0
+; CHECK-NEXT:    cmovpl %eax, %ecx
+; CHECK-NEXT:    movd %ecx, %xmm1
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; CHECK-NEXT:    ucomiss %xmm0, %xmm0
+; CHECK-NEXT:    maxss %xmm2, %xmm0
+; CHECK-NEXT:    minss %xmm3, %xmm0
+; CHECK-NEXT:    cvttss2si %xmm0, %ecx
+; CHECK-NEXT:    cmovpl %eax, %ecx
+; CHECK-NEXT:    movd %ecx, %xmm0
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %x = call <4 x i1> @llvm.fptosi.sat.v4i1.v4f32(<4 x float> %f)
+  %y = freeze <4 x i1> %x
+  ret <4 x i1> %y
+}
+
 define <4 x i8> @test_signed_v4i8_v4f32(<4 x float> %f) nounwind {
 ; CHECK-LABEL: test_signed_v4i8_v4f32:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fptoui-sat-vector-128.ll b/llvm/test/CodeGen/X86/fptoui-sat-vector-128.ll
index ffbdd66529f5c..7b1db5c29abaf 100644
--- a/llvm/test/CodeGen/X86/fptoui-sat-vector-128.ll
+++ b/llvm/test/CodeGen/X86/fptoui-sat-vector-128.ll
@@ -48,6 +48,43 @@ define <4 x i1> @test_unsigned_v4i1_v4f32(<4 x float> %f) nounwind {
   ret <4 x i1> %x
 }
 
+define <4 x i1> @test_freeze_unsigned_v4i1_v4f32(<4 x float> %f) nounwind {
+; CHECK-LABEL: test_freeze_unsigned_v4i1_v4f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3],xmm0[3,3]
+; CHECK-NEXT:    xorps %xmm2, %xmm2
+; CHECK-NEXT:    maxss %xmm2, %xmm1
+; CHECK-NEXT:    movss {{.*#+}} xmm3 = [1.0E+0,0.0E+0,0.0E+0,0.0E+0]
+; CHECK-NEXT:    minss %xmm3, %xmm1
+; CHECK-NEXT:    cvttss2si %xmm1, %eax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    movaps %xmm0, %xmm4
+; CHECK-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
+; CHECK-NEXT:    maxss %xmm2, %xmm4
+; CHECK-NEXT:    minss %xmm3, %xmm4
+; CHECK-NEXT:    cvttss2si %xmm4, %eax
+; CHECK-NEXT:    movd %eax, %xmm4
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; CHECK-NEXT:    movaps %xmm0, %xmm1
+; CHECK-NEXT:    maxss %xmm2, %xmm1
+; CHECK-NEXT:    minss %xmm3, %xmm1
+; CHECK-NEXT:    cvttss2si %xmm1, %eax
+; CHECK-NEXT:    movd %eax, %xmm1
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; CHECK-NEXT:    maxss %xmm2, %xmm0
+; CHECK-NEXT:    minss %xmm3, %xmm0
+; CHECK-NEXT:    cvttss2si %xmm0, %eax
+; CHECK-NEXT:    movd %eax, %xmm0
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %x = call <4 x i1> @llvm.fptoui.sat.v4i1.v4f32(<4 x float> %f)
+  %y = freeze <4 x i1> %x
+  ret <4 x i1> %y
+}
+
 define <4 x i8> @test_unsigned_v4i8_v4f32(<4 x float> %f) nounwind {
 ; CHECK-LABEL: test_unsigned_v4i8_v4f32:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/freeze-binary.ll b/llvm/test/CodeGen/X86/freeze-binary.ll
index 962ffe47d0d51..e223765eb887b 100644
--- a/llvm/test/CodeGen/X86/freeze-binary.ll
+++ b/llvm/test/CodeGen/X86/freeze-binary.ll
@@ -924,6 +924,76 @@ define i32 @freeze_usubo(i32 %a0, i32 %a1, i8 %a2, i8 %a3) nounwind {
   ret i32 %r
 }
 
+define i32 @freeze_scmp(i32 %a0) nounwind {
+; X86-LABEL: freeze_scmp:
+; X86:       # %bb.0:
+; X86-NEXT:    movl $2, %eax
+; X86-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    setl %al
+; X86-NEXT:    setg %cl
+; X86-NEXT:    subb %al, %cl
+; X86-NEXT:    movsbl %cl, %eax
+; X86-NEXT:    negl %eax
+; X86-NEXT:    setl %al
+; X86-NEXT:    setg %cl
+; X86-NEXT:    subb %al, %cl
+; X86-NEXT:    movsbl %cl, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_scmp:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $2, %eax
+; X64-NEXT:    cmpl %edi, %eax
+; X64-NEXT:    setl %al
+; X64-NEXT:    setg %cl
+; X64-NEXT:    subb %al, %cl
+; X64-NEXT:    movsbl %cl, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    setl %al
+; X64-NEXT:    setg %cl
+; X64-NEXT:    subb %al, %cl
+; X64-NEXT:    movsbl %cl, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.scmp.i32(i32 2, i32 %a0)
+  %y = freeze i32 %x
+  %z = call i32 @llvm.scmp.i32(i32 0, i32 %y)
+  ret i32 %z
+}
+
+define i32 @freeze_ucmp(i32 %a0) nounwind {
+; X86-LABEL: freeze_ucmp:
+; X86:       # %bb.0:
+; X86-NEXT:    movl $2, %eax
+; X86-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    seta %al
+; X86-NEXT:    sbbb $0, %al
+; X86-NEXT:    movsbl %al, %eax
+; X86-NEXT:    cmpl $2, %eax
+; X86-NEXT:    setae %cl
+; X86-NEXT:    cmpl $1, %eax
+; X86-NEXT:    sbbb $0, %cl
+; X86-NEXT:    movsbl %cl, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: freeze_ucmp:
+; X64:       # %bb.0:
+; X64-NEXT:    movl $2, %eax
+; X64-NEXT:    cmpl %edi, %eax
+; X64-NEXT:    seta %al
+; X64-NEXT:    sbbb $0, %al
+; X64-NEXT:    movsbl %al, %eax
+; X64-NEXT:    cmpl $2, %eax
+; X64-NEXT:    setae %cl
+; X64-NEXT:    cmpl $1, %eax
+; X64-NEXT:    sbbb $0, %cl
+; X64-NEXT:    movsbl %cl, %eax
+; X64-NEXT:    retq
+  %x = call i32 @llvm.ucmp.i32(i32 2, i32 %a0)
+  %y = freeze i32 %x
+  %z = call i32 @llvm.ucmp.i32(i32 %y, i32 1)
+  ret i32 %z
+}
+
 define void @pr59676_frozen(ptr %dst, i32 %x.orig) {
 ; X86-LABEL: pr59676_frozen:
 ; X86:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fshl-fshr-constant.ll b/llvm/test/CodeGen/X86/fshl-fshr-constant.ll
new file mode 100644
index 0000000000000..fdc34f5665b2b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fshl-fshr-constant.ll
@@ -0,0 +1,149 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,CHECK-EXPAND
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512vbmi2 | FileCheck %s --check-prefixes=CHECK,CHECK-UNEXPAND
+
+define <4 x i32> @test_fshl_constants() {
+; CHECK-EXPAND-LABEL: test_fshl_constants:
+; CHECK-EXPAND:       # %bb.0:
+; CHECK-EXPAND-NEXT:    vmovaps {{.*#+}} xmm0 = [0,512,2048,6144]
+; CHECK-EXPAND-NEXT:    retq
+;
+; CHECK-UNEXPAND-LABEL: test_fshl_constants:
+; CHECK-UNEXPAND:       # %bb.0:
+; CHECK-UNEXPAND-NEXT:    vpmovsxwd {{.*#+}} xmm0 = [0,512,2048,6144]
+; CHECK-UNEXPAND-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> <i32 4, i32 5, i32 6, i32 7>, <4 x i32> <i32 8, i32 9, i32 10, i32 11>)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_fshl_splat_constants() {
+; CHECK-LABEL: test_fshl_splat_constants:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [256,256,256,256]
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 4, i32 4, i32 4, i32 4>, <4 x i32> <i32 8, i32 8, i32 8, i32 8>)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_fshl_two_constants(<4 x i32> %a) {
+; CHECK-EXPAND-LABEL: test_fshl_two_constants:
+; CHECK-EXPAND:       # %bb.0:
+; CHECK-EXPAND-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    retq
+;
+; CHECK-UNEXPAND-LABEL: test_fshl_two_constants:
+; CHECK-UNEXPAND:       # %bb.0:
+; CHECK-UNEXPAND-NEXT:    vpmovsxbd {{.*#+}} xmm1 = [4,5,6,7]
+; CHECK-UNEXPAND-NEXT:    vpshldvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0
+; CHECK-UNEXPAND-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> <i32 4, i32 5, i32 6, i32 7>, <4 x i32> <i32 8, i32 9, i32 10, i32 11>)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_fshl_one_constant(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-EXPAND-LABEL: test_fshl_one_constant:
+; CHECK-EXPAND:       # %bb.0:
+; CHECK-EXPAND-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; CHECK-EXPAND-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    retq
+;
+; CHECK-UNEXPAND-LABEL: test_fshl_one_constant:
+; CHECK-UNEXPAND:       # %bb.0:
+; CHECK-UNEXPAND-NEXT:    vpshldvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm0
+; CHECK-UNEXPAND-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 8, i32 9, i32 10, i32 11>)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_fshl_none_constant(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-EXPAND-LABEL: test_fshl_none_constant:
+; CHECK-EXPAND:       # %bb.0:
+; CHECK-EXPAND-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
+; CHECK-EXPAND-NEXT:    vpandn %xmm3, %xmm2, %xmm4
+; CHECK-EXPAND-NEXT:    vpsrld $1, %xmm1, %xmm1
+; CHECK-EXPAND-NEXT:    vpsrlvd %xmm4, %xmm1, %xmm1
+; CHECK-EXPAND-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; CHECK-EXPAND-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    retq
+;
+; CHECK-UNEXPAND-LABEL: test_fshl_none_constant:
+; CHECK-UNEXPAND:       # %bb.0:
+; CHECK-UNEXPAND-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0
+; CHECK-UNEXPAND-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_fshr_constants() {
+; CHECK-LABEL: test_fshr_constants:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [0,8388608,8388608,6291456]
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> <i32 4, i32 5, i32 6, i32 7>, <4 x i32> <i32 8, i32 9, i32 10, i32 11>)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_fshr_two_constants(<4 x i32> %a) {
+; CHECK-EXPAND-LABEL: test_fshr_two_constants:
+; CHECK-EXPAND:       # %bb.0:
+; CHECK-EXPAND-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    retq
+;
+; CHECK-UNEXPAND-LABEL: test_fshr_two_constants:
+; CHECK-UNEXPAND:       # %bb.0:
+; CHECK-UNEXPAND-NEXT:    vpmovsxbd {{.*#+}} xmm1 = [4,5,6,7]
+; CHECK-UNEXPAND-NEXT:    vpshrdvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; CHECK-UNEXPAND-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-UNEXPAND-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> <i32 4, i32 5, i32 6, i32 7>, <4 x i32> <i32 8, i32 9, i32 10, i32 11>)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_fshr_one_constant(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-EXPAND-LABEL: test_fshr_one_constant:
+; CHECK-EXPAND:       # %bb.0:
+; CHECK-EXPAND-NEXT:    vpsrlvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; CHECK-EXPAND-NEXT:    vpsllvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    retq
+;
+; CHECK-UNEXPAND-LABEL: test_fshr_one_constant:
+; CHECK-UNEXPAND:       # %bb.0:
+; CHECK-UNEXPAND-NEXT:    vpshrdvd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; CHECK-UNEXPAND-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-UNEXPAND-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 8, i32 9, i32 10, i32 11>)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_fshr_none_constant(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; CHECK-EXPAND-LABEL: test_fshr_none_constant:
+; CHECK-EXPAND:       # %bb.0:
+; CHECK-EXPAND-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
+; CHECK-EXPAND-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; CHECK-EXPAND-NEXT:    vpsrlvd %xmm4, %xmm1, %xmm1
+; CHECK-EXPAND-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; CHECK-EXPAND-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    vpsllvd %xmm2, %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; CHECK-EXPAND-NEXT:    retq
+;
+; CHECK-UNEXPAND-LABEL: test_fshr_none_constant:
+; CHECK-UNEXPAND:       # %bb.0:
+; CHECK-UNEXPAND-NEXT:    vpshrdvd %xmm2, %xmm0, %xmm1
+; CHECK-UNEXPAND-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-UNEXPAND-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_fshr_splat_constants() {
+; CHECK-LABEL: test_fshr_splat_constants:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vbroadcastss {{.*#+}} xmm0 = [16777216,16777216,16777216,16777216]
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32> <i32 4, i32 4, i32 4, i32 4>, <4 x i32> <i32 8, i32 8, i32 8, i32 8>)
+  ret <4 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/function-align.ll b/llvm/test/CodeGen/X86/function-align.ll
deleted file mode 100644
index 11d0e99929927..0000000000000
--- a/llvm/test/CodeGen/X86/function-align.ll
+++ /dev/null
@@ -1,18 +0,0 @@
-; RUN: llc -function-sections < %s | FileCheck %s
-
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
-; CHECK: .section .text.f1
-; CHECK-NOT: .p2align
-; CHECK: f1:
-define void @f1() align 1 {
-  ret void
-}
-
-; CHECK: .section .text.f2
-; CHECK-NEXT: .globl f2
-; CHECK-NEXT: .p2align 1
-define void @f2() align 2 {
-  ret void
-}
diff --git a/llvm/test/CodeGen/X86/huge-stack.ll b/llvm/test/CodeGen/X86/huge-stack.ll
index 920033ba1182c..41b8a0141b63d 100644
--- a/llvm/test/CodeGen/X86/huge-stack.ll
+++ b/llvm/test/CodeGen/X86/huge-stack.ll
@@ -5,20 +5,70 @@
 define void @foo() unnamed_addr #0 {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movabsq $8589934462, %rax # imm = 0x1FFFFFF7E
+; CHECK-NEXT:    movabsq $8589934472, %rax # imm = 0x1FFFFFF88
 ; CHECK-NEXT:    subq %rax, %rsp
-; CHECK-NEXT:    .cfi_def_cfa_offset 8589934470
-; CHECK-NEXT:    movb $42, -129(%rsp)
-; CHECK-NEXT:    movb $43, -128(%rsp)
-; CHECK-NEXT:    movabsq $8589934462, %rax # imm = 0x1FFFFFF7E
+; CHECK-NEXT:    .cfi_def_cfa_offset 8589934480
+; CHECK-NEXT:    movabsq $4294967177, %rax # imm = 0xFFFFFF89
+; CHECK-NEXT:    movb $42, (%rsp,%rax)
+; CHECK-NEXT:    movb $43, -118(%rsp)
+; CHECK-NEXT:    movabsq $8589934472, %rax # imm = 0x1FFFFFF88
 ; CHECK-NEXT:    addq %rax, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-NEXT:    retq
-  %1 = alloca %large, align 1
-  %2 = alloca %large, align 1
-  %3 = getelementptr inbounds %large, ptr %1, i64 0, i64 0
-  store i8 42, ptr %3, align 1
-  %4 = getelementptr inbounds %large, ptr %2, i64 0, i64 0
-  store i8 43, ptr %4, align 1
+  %large1 = alloca %large, align 1
+  %large2 = alloca %large, align 1
+  %ptrLarge1 = getelementptr inbounds %large, ptr %large1, i64 0, i64 0
+  store i8 42, ptr %ptrLarge1, align 1
+  %ptrLarge2 = getelementptr inbounds %large, ptr %large2, i64 0, i64 0
+  store i8 43, ptr %ptrLarge2, align 1
   ret void
 }
+
+declare ptr @baz(ptr, ptr, ptr, ptr)
+
+define ptr @scavenge_spill() unnamed_addr #0 {
+; CHECK-LABEL: scavenge_spill:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $25769803816, %rax # imm = 0x600000028
+; CHECK-NEXT:    subq %rax, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 25769803824
+; CHECK-NEXT:    movabsq $21474836521, %rax # imm = 0x500000029
+; CHECK-NEXT:    leaq (%rsp,%rax), %rdi
+; CHECK-NEXT:    movabsq $17179869226, %rax # imm = 0x40000002A
+; CHECK-NEXT:    leaq (%rsp,%rax), %rsi
+; CHECK-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movabsq $12884901931, %rax # imm = 0x30000002B
+; CHECK-NEXT:    leaq (%rsp,%rax), %rdx
+; CHECK-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    movabsq $8589934636, %rax # imm = 0x20000002C
+; CHECK-NEXT:    leaq (%rsp,%rax), %rcx
+; CHECK-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    callq baz@PLT
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT:    leaq 46(%rsp), %rdi
+; CHECK-NEXT:    callq baz@PLT
+; CHECK-NEXT:    # kill: def $rcx killed $rax
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    movabsq $25769803816, %rcx # imm = 0x600000028
+; CHECK-NEXT:    addq %rcx, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  %large1 = alloca %large, align 1
+  %ptrLarge1 = getelementptr inbounds %large, ptr %large1, i64 0, i64 0
+  %large2 = alloca %large, align 1
+  %ptrLarge2 = getelementptr inbounds %large, ptr %large2, i64 0, i64 0
+  %large3 = alloca %large, align 1
+  %ptrLarge3 = getelementptr inbounds %large, ptr %large3, i64 0, i64 0
+  %large4 = alloca %large, align 1
+  %ptrLarge4 = getelementptr inbounds %large, ptr %large4, i64 0, i64 0
+  %large5 = alloca %large, align 1
+  %ptrLarge5 = getelementptr inbounds %large, ptr %large5, i64 0, i64 0
+  %ret1 = call ptr @baz(ptr %ptrLarge1, ptr %ptrLarge2, ptr %ptrLarge3, ptr %ptrLarge4)
+  %large6 = alloca %large, align 1
+  %ptrLarge6 = getelementptr inbounds %large, ptr %large6, i64 0, i64 0
+  %ret2 = call ptr @baz(ptr %ptrLarge6, ptr %ptrLarge2, ptr %ptrLarge3, ptr %ptrLarge4)
+  ret ptr %ret1
+}
diff --git a/llvm/test/CodeGen/X86/large-displacements-fastisel.ll b/llvm/test/CodeGen/X86/large-displacements-fastisel.ll
new file mode 100644
index 0000000000000..362b1b5da092d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/large-displacements-fastisel.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64 -O=0 -verify-machineinstrs | FileCheck %s
+@G = global i8 0
+
+; Regression test for PR113856 - incorrect FastISel assert
+
+define i32 @main() {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $-2147483652, %rax # imm = 0xFFFFFFFF7FFFFFFC
+; CHECK-NEXT:    movl $0, (%rsp,%rax)
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %1 = alloca i32, align 4
+  %G = getelementptr i8, ptr %1, i32 -2147483648
+  store i32 0, ptr %G, align 4
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/X86/large-displacements.ll b/llvm/test/CodeGen/X86/large-displacements.ll
new file mode 100644
index 0000000000000..d7085a56edbb9
--- /dev/null
+++ b/llvm/test/CodeGen/X86/large-displacements.ll
@@ -0,0 +1,82 @@
+; RUN: not llc < %s -mtriple=i686 -filetype=null -verify-machineinstrs 2>&1 | FileCheck %s -check-prefix=ERR-i686
+; RUN: llc < %s -mtriple=x86_64 -verify-machineinstrs | FileCheck %s -check-prefix=x86_64
+
+; Regression test for #121932, #113856, #106352, #69365, #25051 which are caused by
+; an incorrectly written assertion for 64-bit offsets when compiling for 32-bit X86.
+
+define i32 @main() #0 {
+; ERR-i686: error: <unknown>:0:0: 64-bit offset calculated but target is 32-bit
+;
+; x86_64-LABEL: main:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    movl $4294967192, %eax # imm = 0xFFFFFF98
+; x86_64-NEXT:    subq %rax, %rsp
+; x86_64-NEXT:    .cfi_def_cfa_offset 4294967200
+; x86_64-NEXT:    movabsq $3221225318, %rax # imm = 0xBFFFFF66
+; x86_64-NEXT:    movb $32, (%rsp,%rax)
+; x86_64-NEXT:    movb $33, 2147483494(%rsp)
+; x86_64-NEXT:    movb $34, 1073741670(%rsp)
+; x86_64-NEXT:    movb $35, -154(%rsp)
+; x86_64-NEXT:    xorl %eax, %eax
+; x86_64-NEXT:    movl $4294967192, %ecx # imm = 0xFFFFFF98
+; x86_64-NEXT:    addq %rcx, %rsp
+; x86_64-NEXT:    .cfi_def_cfa_offset 8
+; x86_64-NEXT:    retq
+entry:
+  %a = alloca [1073741824 x i8], align 16
+  %b = alloca [1073741824 x i8], align 16
+  %c = alloca [1073741824 x i8], align 16
+  %d = alloca [1073741824 x i8], align 16
+
+  %arrayida = getelementptr inbounds [1073741824 x i8], ptr %a, i64 0, i64 -42
+  %arrayidb = getelementptr inbounds [1073741824 x i8], ptr %b, i64 0, i64 -42
+  %arrayidc = getelementptr inbounds [1073741824 x i8], ptr %c, i64 0, i64 -42
+  %arrayidd = getelementptr inbounds [1073741824 x i8], ptr %d, i64 0, i64 -42
+
+  store i8 32, ptr %arrayida, align 2
+  store i8 33, ptr %arrayidb, align 2
+  store i8 34, ptr %arrayidc, align 2
+  store i8 35, ptr %arrayidd, align 2
+
+  ret i32 0
+}
+
+; Same test as above but for an anonymous function.
+define i32 @0() #0 {
+; ERR-i686: error: <unknown>:0:0: 64-bit offset calculated but target is 32-bit
+;
+; x86_64-LABEL: __unnamed_1:
+; x86_64:       # %bb.0: # %entry
+; x86_64-NEXT:    movl $4294967192, %eax # imm = 0xFFFFFF98
+; x86_64-NEXT:    subq %rax, %rsp
+; x86_64-NEXT:    .cfi_def_cfa_offset 4294967200
+; x86_64-NEXT:    movabsq $3221225318, %rax # imm = 0xBFFFFF66
+; x86_64-NEXT:    movb $32, (%rsp,%rax)
+; x86_64-NEXT:    movb $33, 2147483494(%rsp)
+; x86_64-NEXT:    movb $34, 1073741670(%rsp)
+; x86_64-NEXT:    movb $35, -154(%rsp)
+; x86_64-NEXT:    xorl %eax, %eax
+; x86_64-NEXT:    movl $4294967192, %ecx # imm = 0xFFFFFF98
+; x86_64-NEXT:    addq %rcx, %rsp
+; x86_64-NEXT:    .cfi_def_cfa_offset 8
+; x86_64-NEXT:    retq
+entry:
+  %a = alloca [1073741824 x i8], align 16
+  %b = alloca [1073741824 x i8], align 16
+  %c = alloca [1073741824 x i8], align 16
+  %d = alloca [1073741824 x i8], align 16
+
+  %arrayida = getelementptr inbounds [1073741824 x i8], ptr %a, i64 0, i64 -42
+  %arrayidb = getelementptr inbounds [1073741824 x i8], ptr %b, i64 0, i64 -42
+  %arrayidc = getelementptr inbounds [1073741824 x i8], ptr %c, i64 0, i64 -42
+  %arrayidd = getelementptr inbounds [1073741824 x i8], ptr %d, i64 0, i64 -42
+
+  store i8 32, ptr %arrayida, align 2
+  store i8 33, ptr %arrayidb, align 2
+  store i8 34, ptr %arrayidc, align 2
+  store i8 35, ptr %arrayidd, align 2
+
+  ret i32 0
+}
+
+attributes #0 = { optnone noinline }
diff --git a/llvm/test/CodeGen/X86/llrint-conv.ll b/llvm/test/CodeGen/X86/llrint-conv.ll
index 402daf80a15e8..7bcf573118538 100644
--- a/llvm/test/CodeGen/X86/llrint-conv.ll
+++ b/llvm/test/CodeGen/X86/llrint-conv.ll
@@ -7,14 +7,50 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx | FileCheck %s --check-prefixes=X64,X64-AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx512f | FileCheck %s --check-prefixes=X64,X64-AVX
 
-define i64 @testmsxs(float %x) {
+define i64 @testmsxh(half %x) nounwind {
+; X86-NOSSE-LABEL: testmsxh:
+; X86-NOSSE:       # %bb.0: # %entry
+; X86-NOSSE-NEXT:    pushl %eax
+; X86-NOSSE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NOSSE-NEXT:    movl %eax, (%esp)
+; X86-NOSSE-NEXT:    calll __extendhfsf2
+; X86-NOSSE-NEXT:    fstps (%esp)
+; X86-NOSSE-NEXT:    calll llrintf
+; X86-NOSSE-NEXT:    popl %ecx
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: testmsxh:
+; X86-SSE2:       # %bb.0: # %entry
+; X86-SSE2-NEXT:    pushl %eax
+; X86-SSE2-NEXT:    pinsrw $0, {{[0-9]+}}(%esp), %xmm0
+; X86-SSE2-NEXT:    pextrw $0, %xmm0, %eax
+; X86-SSE2-NEXT:    movw %ax, (%esp)
+; X86-SSE2-NEXT:    calll __extendhfsf2
+; X86-SSE2-NEXT:    fstps (%esp)
+; X86-SSE2-NEXT:    calll llrintf
+; X86-SSE2-NEXT:    popl %ecx
+; X86-SSE2-NEXT:    retl
+;
+; X64-SSE-LABEL: testmsxh:
+; X64-SSE:       # %bb.0: # %entry
+; X64-SSE-NEXT:    pushq %rax
+; X64-SSE-NEXT:    callq __extendhfsf2@PLT
+; X64-SSE-NEXT:    callq rintf@PLT
+; X64-SSE-NEXT:    callq __truncsfhf2@PLT
+; X64-SSE-NEXT:    callq __extendhfsf2@PLT
+; X64-SSE-NEXT:    cvttss2si %xmm0, %rax
+; X64-SSE-NEXT:    popq %rcx
+; X64-SSE-NEXT:    retq
+entry:
+  %0 = tail call i64 @llvm.llrint.i64.f16(half %x)
+  ret i64 %0
+}
+
+define i64 @testmsxs(float %x) nounwind {
 ; X86-NOSSE-LABEL: testmsxs:
 ; X86-NOSSE:       # %bb.0: # %entry
 ; X86-NOSSE-NEXT:    pushl %ebp
-; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 8
-; X86-NOSSE-NEXT:    .cfi_offset %ebp, -8
 ; X86-NOSSE-NEXT:    movl %esp, %ebp
-; X86-NOSSE-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-NOSSE-NEXT:    andl $-8, %esp
 ; X86-NOSSE-NEXT:    subl $8, %esp
 ; X86-NOSSE-NEXT:    flds 8(%ebp)
@@ -23,16 +59,12 @@ define i64 @testmsxs(float %x) {
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NOSSE-NEXT:    movl %ebp, %esp
 ; X86-NOSSE-NEXT:    popl %ebp
-; X86-NOSSE-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: testmsxs:
 ; X86-SSE2:       # %bb.0: # %entry
 ; X86-SSE2-NEXT:    pushl %ebp
-; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
-; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
 ; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-SSE2-NEXT:    andl $-8, %esp
 ; X86-SSE2-NEXT:    subl $8, %esp
 ; X86-SSE2-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -43,16 +75,12 @@ define i64 @testmsxs(float %x) {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE2-NEXT:    movl %ebp, %esp
 ; X86-SSE2-NEXT:    popl %ebp
-; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: testmsxs:
 ; X86-AVX:       # %bb.0: # %entry
 ; X86-AVX-NEXT:    pushl %ebp
-; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
-; X86-AVX-NEXT:    .cfi_offset %ebp, -8
 ; X86-AVX-NEXT:    movl %esp, %ebp
-; X86-AVX-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-AVX-NEXT:    andl $-8, %esp
 ; X86-AVX-NEXT:    subl $8, %esp
 ; X86-AVX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
@@ -63,7 +91,6 @@ define i64 @testmsxs(float %x) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX-NEXT:    movl %ebp, %esp
 ; X86-AVX-NEXT:    popl %ebp
-; X86-AVX-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: testmsxs:
@@ -76,18 +103,15 @@ define i64 @testmsxs(float %x) {
 ; X64-AVX-NEXT:    vcvtss2si %xmm0, %rax
 ; X64-AVX-NEXT:    retq
 entry:
-  %0 = tail call i64 @llvm.llrint.f32(float %x)
+  %0 = tail call i64 @llvm.llrint.i64.f32(float %x)
   ret i64 %0
 }
 
-define i64 @testmsxd(double %x) {
+define i64 @testmsxd(double %x) nounwind {
 ; X86-NOSSE-LABEL: testmsxd:
 ; X86-NOSSE:       # %bb.0: # %entry
 ; X86-NOSSE-NEXT:    pushl %ebp
-; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 8
-; X86-NOSSE-NEXT:    .cfi_offset %ebp, -8
 ; X86-NOSSE-NEXT:    movl %esp, %ebp
-; X86-NOSSE-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-NOSSE-NEXT:    andl $-8, %esp
 ; X86-NOSSE-NEXT:    subl $8, %esp
 ; X86-NOSSE-NEXT:    fldl 8(%ebp)
@@ -96,16 +120,12 @@ define i64 @testmsxd(double %x) {
 ; X86-NOSSE-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NOSSE-NEXT:    movl %ebp, %esp
 ; X86-NOSSE-NEXT:    popl %ebp
-; X86-NOSSE-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: testmsxd:
 ; X86-SSE2:       # %bb.0: # %entry
 ; X86-SSE2-NEXT:    pushl %ebp
-; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
-; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
 ; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-SSE2-NEXT:    andl $-8, %esp
 ; X86-SSE2-NEXT:    subl $8, %esp
 ; X86-SSE2-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
@@ -116,16 +136,12 @@ define i64 @testmsxd(double %x) {
 ; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-SSE2-NEXT:    movl %ebp, %esp
 ; X86-SSE2-NEXT:    popl %ebp
-; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX-LABEL: testmsxd:
 ; X86-AVX:       # %bb.0: # %entry
 ; X86-AVX-NEXT:    pushl %ebp
-; X86-AVX-NEXT:    .cfi_def_cfa_offset 8
-; X86-AVX-NEXT:    .cfi_offset %ebp, -8
 ; X86-AVX-NEXT:    movl %esp, %ebp
-; X86-AVX-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-AVX-NEXT:    andl $-8, %esp
 ; X86-AVX-NEXT:    subl $8, %esp
 ; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
@@ -136,7 +152,6 @@ define i64 @testmsxd(double %x) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-AVX-NEXT:    movl %ebp, %esp
 ; X86-AVX-NEXT:    popl %ebp
-; X86-AVX-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-AVX-NEXT:    retl
 ;
 ; X64-SSE-LABEL: testmsxd:
@@ -149,18 +164,15 @@ define i64 @testmsxd(double %x) {
 ; X64-AVX-NEXT:    vcvtsd2si %xmm0, %rax
 ; X64-AVX-NEXT:    retq
 entry:
-  %0 = tail call i64 @llvm.llrint.f64(double %x)
+  %0 = tail call i64 @llvm.llrint.i64.f64(double %x)
   ret i64 %0
 }
 
-define i64 @testmsll(x86_fp80 %x) {
+define i64 @testmsll(x86_fp80 %x) nounwind {
 ; X86-LABEL: testmsll:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %ebp
-; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    .cfi_offset %ebp, -8
 ; X86-NEXT:    movl %esp, %ebp
-; X86-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
 ; X86-NEXT:    fldt 8(%ebp)
@@ -169,7 +181,6 @@ define i64 @testmsll(x86_fp80 %x) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl %ebp, %esp
 ; X86-NEXT:    popl %ebp
-; X86-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: testmsll:
@@ -179,10 +190,65 @@ define i64 @testmsll(x86_fp80 %x) {
 ; X64-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call i64 @llvm.llrint.f80(x86_fp80 %x)
+  %0 = tail call i64 @llvm.llrint.i64.f80(x86_fp80 %x)
+  ret i64 %0
+}
+
+; FIXME(#44744): incorrect libcall
+define i64 @testmslq(fp128 %x) nounwind {
+; X86-NOSSE-LABEL: testmslq:
+; X86-NOSSE:       # %bb.0: # %entry
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-16, %esp
+; X86-NOSSE-NEXT:    subl $16, %esp
+; X86-NOSSE-NEXT:    pushl 20(%ebp)
+; X86-NOSSE-NEXT:    pushl 16(%ebp)
+; X86-NOSSE-NEXT:    pushl 12(%ebp)
+; X86-NOSSE-NEXT:    pushl 8(%ebp)
+; X86-NOSSE-NEXT:    calll llrintl
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: testmslq:
+; X86-SSE2:       # %bb.0: # %entry
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pushl 20(%ebp)
+; X86-SSE2-NEXT:    pushl 16(%ebp)
+; X86-SSE2-NEXT:    pushl 12(%ebp)
+; X86-SSE2-NEXT:    pushl 8(%ebp)
+; X86-SSE2-NEXT:    calll llrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: testmslq:
+; X86-AVX:       # %bb.0: # %entry
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-16, %esp
+; X86-AVX-NEXT:    subl $32, %esp
+; X86-AVX-NEXT:    vmovups 8(%ebp), %xmm0
+; X86-AVX-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX-NEXT:    calll llrintl
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-LABEL: testmslq:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    jmp llrintl@PLT # TAILCALL
+entry:
+  %0 = tail call i64 @llvm.llrint.i64.fp128(fp128 %x)
   ret i64 %0
 }
 
-declare i64 @llvm.llrint.f32(float) nounwind readnone
-declare i64 @llvm.llrint.f64(double) nounwind readnone
-declare i64 @llvm.llrint.f80(x86_fp80) nounwind readnone
+declare i64 @llvm.llrint.i64.f32(float) nounwind readnone
+declare i64 @llvm.llrint.i64.f64(double) nounwind readnone
+declare i64 @llvm.llrint.i64.f80(x86_fp80) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/lrint-conv-i32.ll b/llvm/test/CodeGen/X86/lrint-conv-i32.ll
index 21580f53ec9b3..3c50aea1095f4 100644
--- a/llvm/test/CodeGen/X86/lrint-conv-i32.ll
+++ b/llvm/test/CodeGen/X86/lrint-conv-i32.ll
@@ -7,16 +7,21 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx | FileCheck %s --check-prefixes=X64,X64-AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx512f | FileCheck %s --check-prefixes=X64,X64-AVX
 
-define i32 @testmsws(float %x) {
+; FIXME: crash
+; define i32 @testmswh(half %x) nounwind {
+; entry:
+;   %0 = tail call i32 @llvm.lrint.i32.f16(half %x)
+;   ret i32 %0
+; }
+
+define i32 @testmsws(float %x) nounwind {
 ; X86-NOSSE-LABEL: testmsws:
 ; X86-NOSSE:       # %bb.0: # %entry
 ; X86-NOSSE-NEXT:    pushl %eax
-; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NOSSE-NEXT:    flds {{[0-9]+}}(%esp)
 ; X86-NOSSE-NEXT:    fistpl (%esp)
 ; X86-NOSSE-NEXT:    movl (%esp), %eax
 ; X86-NOSSE-NEXT:    popl %ecx
-; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: testmsws:
@@ -43,16 +48,14 @@ entry:
   ret i32 %0
 }
 
-define i32 @testmswd(double %x) {
+define i32 @testmswd(double %x) nounwind {
 ; X86-NOSSE-LABEL: testmswd:
 ; X86-NOSSE:       # %bb.0: # %entry
 ; X86-NOSSE-NEXT:    pushl %eax
-; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NOSSE-NEXT:    fldl {{[0-9]+}}(%esp)
 ; X86-NOSSE-NEXT:    fistpl (%esp)
 ; X86-NOSSE-NEXT:    movl (%esp), %eax
 ; X86-NOSSE-NEXT:    popl %ecx
-; X86-NOSSE-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NOSSE-NEXT:    retl
 ;
 ; X86-SSE2-LABEL: testmswd:
@@ -79,16 +82,14 @@ entry:
   ret i32 %0
 }
 
-define i32 @testmsll(x86_fp80 %x) {
+define i32 @testmsll(x86_fp80 %x) nounwind {
 ; X86-LABEL: testmsll:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %eax
-; X86-NEXT:    .cfi_def_cfa_offset 8
 ; X86-NEXT:    fldt {{[0-9]+}}(%esp)
 ; X86-NEXT:    fistpl (%esp)
 ; X86-NEXT:    movl (%esp), %eax
 ; X86-NEXT:    popl %ecx
-; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: testmsll:
@@ -102,6 +103,61 @@ entry:
   ret i32 %0
 }
 
+; FIXME(#44744): incorrect libcall
+define i32 @testmswq(fp128 %x) nounwind {
+; X86-NOSSE-LABEL: testmswq:
+; X86-NOSSE:       # %bb.0: # %entry
+; X86-NOSSE-NEXT:    pushl %ebp
+; X86-NOSSE-NEXT:    movl %esp, %ebp
+; X86-NOSSE-NEXT:    andl $-16, %esp
+; X86-NOSSE-NEXT:    subl $16, %esp
+; X86-NOSSE-NEXT:    pushl 20(%ebp)
+; X86-NOSSE-NEXT:    pushl 16(%ebp)
+; X86-NOSSE-NEXT:    pushl 12(%ebp)
+; X86-NOSSE-NEXT:    pushl 8(%ebp)
+; X86-NOSSE-NEXT:    calll lrintl
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    movl %ebp, %esp
+; X86-NOSSE-NEXT:    popl %ebp
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: testmswq:
+; X86-SSE2:       # %bb.0: # %entry
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pushl 20(%ebp)
+; X86-SSE2-NEXT:    pushl 16(%ebp)
+; X86-SSE2-NEXT:    pushl 12(%ebp)
+; X86-SSE2-NEXT:    pushl 8(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: testmswq:
+; X86-AVX:       # %bb.0: # %entry
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-16, %esp
+; X86-AVX-NEXT:    subl $32, %esp
+; X86-AVX-NEXT:    vmovups 8(%ebp), %xmm0
+; X86-AVX-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX-NEXT:    calll lrintl
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-LABEL: testmswq:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    jmp lrintl@PLT # TAILCALL
+entry:
+  %0 = tail call i32 @llvm.lrint.i32.f128(fp128 %x)
+  ret i32 %0
+}
+
 declare i32 @llvm.lrint.i32.f32(float) nounwind readnone
 declare i32 @llvm.lrint.i32.f64(double) nounwind readnone
 declare i32 @llvm.lrint.i32.f80(x86_fp80) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/lrint-conv-i64.ll b/llvm/test/CodeGen/X86/lrint-conv-i64.ll
index 38fa09085e189..2ba1500df0b6e 100644
--- a/llvm/test/CodeGen/X86/lrint-conv-i64.ll
+++ b/llvm/test/CodeGen/X86/lrint-conv-i64.ll
@@ -3,7 +3,23 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx | FileCheck %s --check-prefixes=CHECK,AVX
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx512f | FileCheck %s --check-prefixes=CHECK,AVX
 
-define i64 @testmsxs(float %x) {
+define i64 @testmsxh(half %x) nounwind {
+; SSE-LABEL: testmsxh:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    callq __extendhfsf2@PLT
+; SSE-NEXT:    callq rintf@PLT
+; SSE-NEXT:    callq __truncsfhf2@PLT
+; SSE-NEXT:    callq __extendhfsf2@PLT
+; SSE-NEXT:    cvttss2si %xmm0, %rax
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    retq
+entry:
+  %0 = tail call i64 @llvm.lrint.i64.f16(half %x)
+  ret i64 %0
+}
+
+define i64 @testmsxs(float %x) nounwind {
 ; SSE-LABEL: testmsxs:
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    cvtss2si %xmm0, %rax
@@ -18,7 +34,7 @@ entry:
   ret i64 %0
 }
 
-define i64 @testmsxd(double %x) {
+define i64 @testmsxd(double %x) nounwind {
 ; SSE-LABEL: testmsxd:
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    cvtsd2si %xmm0, %rax
@@ -33,7 +49,7 @@ entry:
   ret i64 %0
 }
 
-define i64 @testmsll(x86_fp80 %x) {
+define i64 @testmsll(x86_fp80 %x) nounwind {
 ; CHECK-LABEL: testmsll:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
@@ -45,7 +61,17 @@ entry:
   ret i64 %0
 }
 
-define i32 @PR125324(float %x) {
+; FIXME(#44744): incorrect libcall
+define i64 @testmsxq(fp128 %x) nounwind {
+; CHECK-LABEL: testmsxq:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    jmp lrintl@PLT # TAILCALL
+entry:
+  %0 = tail call i64 @llvm.lrint.i64.f128(fp128 %x)
+  ret i64 %0
+}
+
+define i32 @PR125324(float %x) nounwind {
 ; SSE-LABEL: PR125324:
 ; SSE:       # %bb.0: # %entry
 ; SSE-NEXT:    cvtss2si %xmm0, %rax
diff --git a/llvm/test/CodeGen/X86/merge-huge-sp-updates.ll b/llvm/test/CodeGen/X86/merge-huge-sp-updates.ll
index b26345e2d5bbc..6920e74d3ec5a 100644
--- a/llvm/test/CodeGen/X86/merge-huge-sp-updates.ll
+++ b/llvm/test/CodeGen/X86/merge-huge-sp-updates.ll
@@ -22,8 +22,8 @@ entry:
   call void @bar(i64 0, i64 0, i64 0, i64 0, i64 0, ptr null, ptr %rhs, ptr null, ptr %rhs)
 ; CHECK: call{{.*}}bar
 ; CHECK: addq{{.*}}$2147483647, %rsp
-; CHECK: addq{{.*}}$372037585, %rsp
-; CHECK: .cfi_adjust_cfa_offset -2519521232
+; CHECK: addq{{.*}}$372037601, %rsp
+; CHECK: .cfi_adjust_cfa_offset -2519521248
   ret void
 }
 
diff --git a/llvm/test/CodeGen/X86/movtopush.mir b/llvm/test/CodeGen/X86/movtopush.mir
index f92c385274be6..075b3c3e52acd 100644
--- a/llvm/test/CodeGen/X86/movtopush.mir
+++ b/llvm/test/CodeGen/X86/movtopush.mir
@@ -83,8 +83,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: p, type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/X86/peephole-test-after-add.mir b/llvm/test/CodeGen/X86/peephole-test-after-add.mir
index aeb9988b90d05..675c8f4720bc9 100644
--- a/llvm/test/CodeGen/X86/peephole-test-after-add.mir
+++ b/llvm/test/CodeGen/X86/peephole-test-after-add.mir
@@ -293,8 +293,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -418,8 +418,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -544,8 +544,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/test/CodeGen/X86/pr140491-sincos-lifetimes.ll b/llvm/test/CodeGen/X86/pr140491-sincos-lifetimes.ll
index 2ca99bdc4b316..58dfd63641d5b 100644
--- a/llvm/test/CodeGen/X86/pr140491-sincos-lifetimes.ll
+++ b/llvm/test/CodeGen/X86/pr140491-sincos-lifetimes.ll
@@ -51,20 +51,20 @@ entry:
   %sincos = tail call { float, float } @llvm.sincos.f32(float %in)
   %sin = extractvalue { float, float } %sincos, 0
   %cos = extractvalue { float, float } %sincos, 1
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %computed)
+  call void @llvm.lifetime.start.p0(ptr nonnull %computed)
   store float %cos, ptr %computed, align 4
   call void @use_ptr(ptr nonnull %computed)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %computed)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %computed1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %computed)
+  call void @llvm.lifetime.start.p0(ptr nonnull %computed1)
   %fneg_sin = fneg float %sin
   store float %fneg_sin, ptr %computed1, align 4
   call void @use_ptr(ptr nonnull %computed1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %computed1)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %computed3)
+  call void @llvm.lifetime.end.p0(ptr nonnull %computed1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %computed3)
   %fneg_cos = fneg float %cos
   store float %fneg_cos, ptr %computed3, align 4
   call void @use_ptr(ptr nonnull %computed3)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %computed3)
+  call void @llvm.lifetime.end.p0(ptr nonnull %computed3)
   ret i32 0
 }
 
diff --git a/llvm/test/CodeGen/X86/pr152150.ll b/llvm/test/CodeGen/X86/pr152150.ll
new file mode 100644
index 0000000000000..6db3e555028cc
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr152150.ll
@@ -0,0 +1,14 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown-eabi-elf | FileCheck %s
+
+; CHECK-LABEL: conv2d
+define dso_local void @conv2d() {
+.preheader:
+  br label %0
+
+0:                                                ; preds = %0, %.preheader
+  %1 = phi [4 x <7 x half>] [ zeroinitializer, %.preheader ], [ %4, %0 ]
+  %2 = extractvalue [4 x <7 x half>] %1, 0
+  %3 = extractvalue [4 x <7 x half>] %1, 1
+  %4 = insertvalue [4 x <7 x half>] poison, <7 x half> poison, 3
+  br label %0
+}
diff --git a/llvm/test/CodeGen/X86/pr152630.ll b/llvm/test/CodeGen/X86/pr152630.ll
new file mode 100644
index 0000000000000..8fa98831c37b0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr152630.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define i32 @pr152630(i1 %cond) nounwind {
+; CHECK-LABEL: pr152630:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    andl $1, %edi
+; CHECK-NEXT:    decl %edi
+; CHECK-NEXT:    cmpl $-1, %edi
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    jne .LBB0_3
+; CHECK-NEXT:  .LBB0_2: # %if.then
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_3: # %if.else
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    retq
+entry:
+  %sel = select i1 %cond, i32 0, i32 -1
+  %conv = trunc nsw i32 %sel to i8
+  switch i8 %conv, label %if.else [
+    i8 -1, label %if.then
+    i8 0, label %if.then
+  ]
+
+if.then:
+  ret i32 0
+
+if.else:
+  ret i32 1
+}
diff --git a/llvm/test/CodeGen/X86/pr154492.ll b/llvm/test/CodeGen/X86/pr154492.ll
new file mode 100644
index 0000000000000..1ba17594976e1
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr154492.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f  | FileCheck %s --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefix=AVX512VL
+
+define <16 x i32> @PR154492() {
+; AVX512F-LABEL: PR154492:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT:    vcvttps2udq %zmm0, %zmm0
+; AVX512F-NEXT:    vmovaps %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: PR154492:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT:    vcvttps2udq %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> zeroinitializer, <16 x i32> zeroinitializer, i16 255, i32 4)
+  ret <16 x i32> %res
+}
diff --git a/llvm/test/CodeGen/X86/pr30821.mir b/llvm/test/CodeGen/X86/pr30821.mir
index dccb49277b519..40761e019fbc0 100644
--- a/llvm/test/CodeGen/X86/pr30821.mir
+++ b/llvm/test/CodeGen/X86/pr30821.mir
@@ -42,8 +42,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: alpha, type: default, offset: 0, size: 1, alignment: 1,
diff --git a/llvm/test/CodeGen/X86/pr33010.ll b/llvm/test/CodeGen/X86/pr33010.ll
index 6f0ce664e06d8..41e44dbc59b9b 100644
--- a/llvm/test/CodeGen/X86/pr33010.ll
+++ b/llvm/test/CodeGen/X86/pr33010.ll
@@ -19,13 +19,10 @@ define ptr addrspace(1) @test(ptr addrspace(1) %a, ptr addrspace(1) %b, i1 %whic
 ; CHECK-NEXT:    callq f@PLT
 ; CHECK-NEXT:  .Ltmp0:
 ; CHECK-NEXT:    testb $1, %bl
-; CHECK-NEXT:    je .LBB0_1
-; CHECK-NEXT:  # %bb.2: # %entry
-; CHECK-NEXT:    movq (%rsp), %rax
-; CHECK-NEXT:    jmp .LBB0_3
-; CHECK-NEXT:  .LBB0_1:
-; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT:  .LBB0_3: # %entry
+; CHECK-NEXT:    movq %rsp, %rax
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    cmovneq %rax, %rcx
+; CHECK-NEXT:    movq (%rcx), %rax
 ; CHECK-NEXT:    addq $16, %rsp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    popq %rbx
diff --git a/llvm/test/CodeGen/X86/pr38952.mir b/llvm/test/CodeGen/X86/pr38952.mir
index 1d18738cf4ad4..e6fab07c5e665 100644
--- a/llvm/test/CodeGen/X86/pr38952.mir
+++ b/llvm/test/CodeGen/X86/pr38952.mir
@@ -60,8 +60,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/CodeGen/X86/pr48064.mir b/llvm/test/CodeGen/X86/pr48064.mir
index 9712a3ca27aa4..eb74edd4f1800 100644
--- a/llvm/test/CodeGen/X86/pr48064.mir
+++ b/llvm/test/CodeGen/X86/pr48064.mir
@@ -236,8 +236,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 4, alignment: 4, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
@@ -292,8 +292,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: zx, type: default, offset: 0, size: 16, alignment: 4,
@@ -403,8 +403,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 0, size: 4, alignment: 4, stack-id: default,
       isImmutable: false, isAliased: false, callee-saved-register: '',
diff --git a/llvm/test/CodeGen/X86/pr62286.ll b/llvm/test/CodeGen/X86/pr62286.ll
index 9728e130333c4..ce03f8fad4a19 100644
--- a/llvm/test/CodeGen/X86/pr62286.ll
+++ b/llvm/test/CodeGen/X86/pr62286.ll
@@ -28,8 +28,9 @@ define i64 @PR62286(i32 %a) {
 ; AVX1-NEXT:    vmovd %edi, %xmm0
 ; AVX1-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
 ; AVX1-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7]
 ; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
 ; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
 ; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm1
@@ -42,10 +43,10 @@ define i64 @PR62286(i32 %a) {
 ; AVX2-LABEL: PR62286:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vmovd %edi, %xmm0
-; AVX2-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
-; AVX2-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
-; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpaddd %xmm0, %xmm0, %xmm1
+; AVX2-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
+; AVX2-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
 ; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
@@ -58,12 +59,13 @@ define i64 @PR62286(i32 %a) {
 ; AVX512-LABEL: PR62286:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmovd %edi, %xmm0
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,1,0]
-; AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
-; AVX512-NEXT:    movw $4369, %ax # imm = 0x1111
+; AVX512-NEXT:    movb $8, %al
 ; AVX512-NEXT:    kmovd %eax, %k1
-; AVX512-NEXT:    vpaddd %zmm0, %zmm0, %zmm1 {%k1}
-; AVX512-NEXT:    vpmovsxdq %ymm1, %zmm0
+; AVX512-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} {z}
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512-NEXT:    vpaddd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX512-NEXT:    vpmovsxdq %ymm0, %zmm0
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
diff --git a/llvm/test/CodeGen/X86/ptrtoaddr.ll b/llvm/test/CodeGen/X86/ptrtoaddr.ll
new file mode 100644
index 0000000000000..24bf9db57d9ec
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ptrtoaddr.ll
@@ -0,0 +1,113 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux-gnu < %s -o - | FileCheck %s --check-prefix=CHECK
+
+define i1 @ptrtoaddr_1(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    xorb $1, %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %trunc = trunc i64 %addr to i1
+  %ret = xor i1 %trunc, 1
+  ret i1 %ret
+}
+
+define i8 @ptrtoaddr_8(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    notb %al
+; CHECK-NEXT:    # kill: def $al killed $al killed $rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %trunc = trunc i64 %addr to i8
+  %ret = xor i8 %trunc, -1
+  ret i8 %ret
+}
+
+define i16 @ptrtoaddr_16(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    notl %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %trunc = trunc i64 %addr to i16
+  %ret = xor i16 %trunc, -1
+  ret i16 %ret
+}
+
+define i32 @ptrtoaddr_32(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_32:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    notl %eax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %trunc = trunc i64 %addr to i32
+  %ret = xor i32 %trunc, -1
+  ret i32 %ret
+}
+
+define i64 @ptrtoaddr_64(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    notq %rax
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %ret = xor i64 %addr, -1
+  ret i64 %ret
+}
+
+define i128 @ptrtoaddr_128(ptr %p) {
+; CHECK-LABEL: ptrtoaddr_128:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    notq %rax
+; CHECK-NEXT:    movq $-1, %rdx
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr ptr %p to i64
+  %ext = zext i64 %addr to i128
+  %ret = xor i128 %ext, -1
+  ret i128 %ret
+}
+
+
+define <2 x i64> @ptrtoaddr_vec(<2 x ptr> %p) {
+; CHECK-LABEL: ptrtoaddr_vec:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
+; CHECK-NEXT:    pxor %xmm1, %xmm0
+; CHECK-NEXT:    retq
+entry:
+  %addr = ptrtoaddr <2 x ptr> %p to <2 x i64>
+  %ret = xor <2 x i64> %addr, <i64 -1, i64 -1>
+  ret <2 x i64> %ret
+}
+
+; UTC_ARGS: --disable
+
+@foo = global [16 x i8] zeroinitializer
+@addr = global i64 ptrtoaddr (ptr @foo to i64)
+; CHECK:      addr:
+; CHECK-NEXT:  .quad	foo
+; CHECK-NEXT:  .size	addr, 8
+@addr_plus_one = global i64 ptrtoaddr (ptr getelementptr (i8, ptr @foo, i64 1) to i64)
+; CHECK:      addr_plus_one:
+; CHECK-NEXT:  .quad	foo+1
+; CHECK-NEXT:  .size	addr_plus_one, 8
+@const_addr = global i64 ptrtoaddr (ptr getelementptr (i8, ptr null, i64 1) to i64)
+; CHECK:      const_addr:
+; CHECK-NEXT:  .quad	0+1
+; CHECK-NEXT:  .size	const_addr, 8
diff --git a/llvm/test/CodeGen/X86/scalarize-strict-fsetcc.ll b/llvm/test/CodeGen/X86/scalarize-strict-fsetcc.ll
new file mode 100644
index 0000000000000..b4c77a573e859
--- /dev/null
+++ b/llvm/test/CodeGen/X86/scalarize-strict-fsetcc.ll
@@ -0,0 +1,293 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s
+
+define <1 x i1> @test_oeq_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_oeq_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    setnp %cl
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    andb %cl, %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"oeq", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ogt_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ogt_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ogt", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_oge_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_oge_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    setae %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"oge", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_olt_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_olt_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm0, %xmm1
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"olt", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ole_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ole_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm0, %xmm1
+; CHECK-NEXT:    setae %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ole", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_one_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_one_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"one", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ord_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ord_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ord", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ueq_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ueq_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ord", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ugt_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ugt_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm0, %xmm1
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ugt", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_uge_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_uge_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm0, %xmm1
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"uge", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ult_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ult_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ult", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ule_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ule_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ule", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_une_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_une_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    setp %cl
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    orb %cl, %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"une", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_uno_q_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_uno_q_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vucomisd %xmm1, %xmm0
+; CHECK-NEXT:    setp %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double> %a, <1 x double> %b, metadata !"uno", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_oeq_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_oeq_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    setnp %cl
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    andb %cl, %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"oeq", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ogt_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ogt_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ogt", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_oge_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_oge_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    setae %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"oge", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_olt_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_olt_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm0, %xmm1
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"olt", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ole_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ole_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm0, %xmm1
+; CHECK-NEXT:    setae %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ole", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_one_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_one_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"one", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ord_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ord_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ord", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ueq_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ueq_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    setnp %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ord", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ugt_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ugt_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm0, %xmm1
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ugt", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_uge_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_uge_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm0, %xmm1
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"uge", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ult_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ult_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ult", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_ule_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_ule_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    setbe %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"ule", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_une_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_une_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    setp %cl
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    orb %cl, %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"une", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+define <1 x i1> @test_uno_s_v1f64(<1 x double> %a, <1 x double> %b) {
+; CHECK-LABEL: test_uno_s_v1f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vcomisd %xmm1, %xmm0
+; CHECK-NEXT:    setp %al
+; CHECK-NEXT:    retq
+  %cond = tail call <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double> %a, <1 x double> %b, metadata !"uno", metadata !"fpexcept.strict")
+  ret <1 x i1> %cond
+}
+
+declare <1 x i1> @llvm.experimental.constrained.fcmp.v1f64(<1 x double>, <1 x double>, metadata, metadata)
+declare <1 x i1> @llvm.experimental.constrained.fcmps.v1f64(<1 x double>, <1 x double>, metadata, metadata)
diff --git a/llvm/test/CodeGen/X86/scheduler-asm-moves.mir b/llvm/test/CodeGen/X86/scheduler-asm-moves.mir
index 87a56a3802145..045b02188888f 100644
--- a/llvm/test/CodeGen/X86/scheduler-asm-moves.mir
+++ b/llvm/test/CodeGen/X86/scheduler-asm-moves.mir
@@ -106,8 +106,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     true
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/select-optimize.ll b/llvm/test/CodeGen/X86/select-optimize.ll
index c7cf9cb221300..6cb49f243ec69 100644
--- a/llvm/test/CodeGen/X86/select-optimize.ll
+++ b/llvm/test/CodeGen/X86/select-optimize.ll
@@ -233,7 +233,7 @@ define i32 @expensive_val_operand5(i32 %b, i32 %y, i1 %cmp) {
 ; CHECK-LABEL: @expensive_val_operand5(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[A]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    [[CMP_FROZEN:%.*]] = freeze i1 [[CMP:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP_FROZEN]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]], !prof [[PROF18]]
 ; CHECK:       select.true.sink:
@@ -245,7 +245,7 @@ define i32 @expensive_val_operand5(i32 %b, i32 %y, i1 %cmp) {
 ;
   %a = alloca i32
   %load = load i32, ptr %a, align 8
-  call void @llvm.lifetime.end.p0(i64 2, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   %x = add i32 %load, %b
   %sel = select i1 %cmp, i32 %x, i32 %y, !prof !17
   ret i32 %sel
@@ -520,7 +520,7 @@ for.body:                                         ; preds = %for.body.preheader,
 declare void @llvm.dbg.value(metadata, metadata, metadata)
 
 ; Function Attrs: argmemonly mustprogress nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @free(ptr nocapture)
 
diff --git a/llvm/test/CodeGen/X86/select-smin-smax.ll b/llvm/test/CodeGen/X86/select-smin-smax.ll
index a7fb60f37ea0c..513983ba54bcf 100644
--- a/llvm/test/CodeGen/X86/select-smin-smax.ll
+++ b/llvm/test/CodeGen/X86/select-smin-smax.ll
@@ -1,66 +1,367 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=-bmi < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NOBMI
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+bmi < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BMI
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=+bmi < %s | FileCheck %s --check-prefixes=X64,X64-BMI
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mattr=-bmi < %s | FileCheck %s --check-prefixes=X64,X64-NOBMI
+; RUN: llc -mtriple=i686-unknown-linux-gnu -mattr=+bmi,+cmov < %s | FileCheck %s --check-prefixes=X86,X86-BMI
+; RUN: llc -mtriple=i686-unknown-linux-gnu -mattr=-bmi < %s | FileCheck %s --check-prefixes=X86,X86-NOBMI
 
+declare i8 @llvm.smax.i8(i8, i8)
+declare i8 @llvm.smin.i8(i8, i8)
+declare i16 @llvm.smax.i16(i16, i16)
+declare i16 @llvm.smin.i16(i16, i16)
 declare i32 @llvm.smax.i32(i32, i32)
 declare i32 @llvm.smin.i32(i32, i32)
 declare i64 @llvm.smax.i64(i64, i64)
 declare i64 @llvm.smin.i64(i64, i64)
+declare i128 @llvm.smax.i128(i128, i128)
+declare i128 @llvm.smin.i128(i128, i128)
+
+define i8 @test_i8_smax(i8 %a) nounwind {
+; X64-LABEL: test_i8_smax:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testb %dil, %dil
+; X64-NEXT:    cmovgl %edi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-BMI-LABEL: test_i8_smax:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    xorl %eax, %eax
+; X86-BMI-NEXT:    testb %cl, %cl
+; X86-BMI-NEXT:    cmovgl %ecx, %eax
+; X86-BMI-NEXT:    # kill: def $al killed $al killed $eax
+; X86-BMI-NEXT:    retl
+;
+; X86-NOBMI-LABEL: test_i8_smax:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    testb %al, %al
+; X86-NOBMI-NEXT:    jg .LBB0_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:  .LBB0_2:
+; X86-NOBMI-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NOBMI-NEXT:    retl
+  %r = call i8 @llvm.smax.i8(i8 %a, i8 0)
+  ret i8 %r
+}
+
+define i8 @test_i8_smin(i8 %a) nounwind {
+; X64-LABEL: test_i8_smin:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testb %dil, %dil
+; X64-NEXT:    cmovsl %edi, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-BMI-LABEL: test_i8_smin:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    xorl %eax, %eax
+; X86-BMI-NEXT:    testb %cl, %cl
+; X86-BMI-NEXT:    cmovsl %ecx, %eax
+; X86-BMI-NEXT:    # kill: def $al killed $al killed $eax
+; X86-BMI-NEXT:    retl
+;
+; X86-NOBMI-LABEL: test_i8_smin:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    testb %al, %al
+; X86-NOBMI-NEXT:    js .LBB1_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:  .LBB1_2:
+; X86-NOBMI-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NOBMI-NEXT:    retl
+  %r = call i8 @llvm.smin.i8(i8 %a, i8 0)
+  ret i8 %r
+}
+
+define i16 @test_i16_smax(i16 %a) nounwind {
+; X64-LABEL: test_i16_smax:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testw %di, %di
+; X64-NEXT:    cmovgl %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-BMI-LABEL: test_i16_smax:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    xorl %eax, %eax
+; X86-BMI-NEXT:    testw %cx, %cx
+; X86-BMI-NEXT:    cmovgl %ecx, %eax
+; X86-BMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-BMI-NEXT:    retl
+;
+; X86-NOBMI-LABEL: test_i16_smax:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    testw %ax, %ax
+; X86-NOBMI-NEXT:    jg .LBB2_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:  .LBB2_2:
+; X86-NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NOBMI-NEXT:    retl
+  %r = call i16 @llvm.smax.i16(i16 %a, i16 0)
+  ret i16 %r
+}
+
+define i16 @test_i16_smin(i16 %a) nounwind {
+; X64-LABEL: test_i16_smin:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    testw %di, %di
+; X64-NEXT:    cmovsl %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+;
+; X86-BMI-LABEL: test_i16_smin:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    xorl %eax, %eax
+; X86-BMI-NEXT:    testw %cx, %cx
+; X86-BMI-NEXT:    cmovsl %ecx, %eax
+; X86-BMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-BMI-NEXT:    retl
+;
+; X86-NOBMI-LABEL: test_i16_smin:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    testw %ax, %ax
+; X86-NOBMI-NEXT:    js .LBB3_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:  .LBB3_2:
+; X86-NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NOBMI-NEXT:    retl
+  %r = call i16 @llvm.smin.i16(i16 %a, i16 0)
+  ret i16 %r
+}
 
 define i32 @test_i32_smax(i32 %a) nounwind {
-; CHECK-NOBMI-LABEL: test_i32_smax:
-; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    xorl %eax, %eax
-; CHECK-NOBMI-NEXT:    testl %edi, %edi
-; CHECK-NOBMI-NEXT:    cmovgl %edi, %eax
-; CHECK-NOBMI-NEXT:    retq
-;
-; CHECK-BMI-LABEL: test_i32_smax:
-; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    movl %edi, %eax
-; CHECK-BMI-NEXT:    sarl $31, %eax
-; CHECK-BMI-NEXT:    andnl %edi, %eax, %eax
-; CHECK-BMI-NEXT:    retq
+; X64-BMI-LABEL: test_i32_smax:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    movl %edi, %eax
+; X64-BMI-NEXT:    sarl $31, %eax
+; X64-BMI-NEXT:    andnl %edi, %eax, %eax
+; X64-BMI-NEXT:    retq
+;
+; X64-NOBMI-LABEL: test_i32_smax:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    xorl %eax, %eax
+; X64-NOBMI-NEXT:    testl %edi, %edi
+; X64-NOBMI-NEXT:    cmovgl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X86-BMI-LABEL: test_i32_smax:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    movl %eax, %ecx
+; X86-BMI-NEXT:    sarl $31, %ecx
+; X86-BMI-NEXT:    andnl %eax, %ecx, %eax
+; X86-BMI-NEXT:    retl
+;
+; X86-NOBMI-LABEL: test_i32_smax:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    testl %eax, %eax
+; X86-NOBMI-NEXT:    jg .LBB4_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:  .LBB4_2:
+; X86-NOBMI-NEXT:    retl
   %r = call i32 @llvm.smax.i32(i32 %a, i32 0)
   ret i32 %r
 }
 
 define i32 @test_i32_smin(i32 %a) nounwind {
-; CHECK-LABEL: test_i32_smin:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    sarl $31, %eax
-; CHECK-NEXT:    andl %edi, %eax
-; CHECK-NEXT:    retq
+; X64-LABEL: test_i32_smin:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    sarl $31, %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i32_smin:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    sarl $31, %eax
+; X86-NEXT:    andl %ecx, %eax
+; X86-NEXT:    retl
   %r = call i32 @llvm.smin.i32(i32 %a, i32 0)
   ret i32 %r
 }
 
 define i64 @test_i64_smax(i64 %a) nounwind {
-; CHECK-NOBMI-LABEL: test_i64_smax:
-; CHECK-NOBMI:       # %bb.0:
-; CHECK-NOBMI-NEXT:    xorl %eax, %eax
-; CHECK-NOBMI-NEXT:    testq %rdi, %rdi
-; CHECK-NOBMI-NEXT:    cmovgq %rdi, %rax
-; CHECK-NOBMI-NEXT:    retq
-;
-; CHECK-BMI-LABEL: test_i64_smax:
-; CHECK-BMI:       # %bb.0:
-; CHECK-BMI-NEXT:    movq %rdi, %rax
-; CHECK-BMI-NEXT:    sarq $63, %rax
-; CHECK-BMI-NEXT:    andnq %rdi, %rax, %rax
-; CHECK-BMI-NEXT:    retq
+; X64-BMI-LABEL: test_i64_smax:
+; X64-BMI:       # %bb.0:
+; X64-BMI-NEXT:    movq %rdi, %rax
+; X64-BMI-NEXT:    sarq $63, %rax
+; X64-BMI-NEXT:    andnq %rdi, %rax, %rax
+; X64-BMI-NEXT:    retq
+;
+; X64-NOBMI-LABEL: test_i64_smax:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    xorl %eax, %eax
+; X64-NOBMI-NEXT:    testq %rdi, %rdi
+; X64-NOBMI-NEXT:    cmovgq %rdi, %rax
+; X64-NOBMI-NEXT:    retq
+;
+; X86-BMI-LABEL: test_i64_smax:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI-NEXT:    xorl %eax, %eax
+; X86-BMI-NEXT:    testl %edx, %edx
+; X86-BMI-NEXT:    cmovlel %eax, %edx
+; X86-BMI-NEXT:    cmovnsl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    retl
+;
+; X86-NOBMI-LABEL: test_i64_smax:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    testl %edx, %edx
+; X86-NOBMI-NEXT:    movl $0, %eax
+; X86-NOBMI-NEXT:    jns .LBB6_1
+; X86-NOBMI-NEXT:  # %bb.2:
+; X86-NOBMI-NEXT:    jle .LBB6_3
+; X86-NOBMI-NEXT:  .LBB6_4:
+; X86-NOBMI-NEXT:    retl
+; X86-NOBMI-NEXT:  .LBB6_1:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    jg .LBB6_4
+; X86-NOBMI-NEXT:  .LBB6_3:
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:    retl
   %r = call i64 @llvm.smax.i64(i64 %a, i64 0)
   ret i64 %r
 }
 
 define i64 @test_i64_smin(i64 %a) nounwind {
-; CHECK-LABEL: test_i64_smin:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    sarq $63, %rax
-; CHECK-NEXT:    andq %rdi, %rax
-; CHECK-NEXT:    retq
+; X64-LABEL: test_i64_smin:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    sarq $63, %rax
+; X64-NEXT:    andq %rdi, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i64_smin:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %edx, %eax
+; X86-NEXT:    sarl $31, %eax
+; X86-NEXT:    andl %eax, %edx
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
   %r = call i64 @llvm.smin.i64(i64 %a, i64 0)
   ret i64 %r
 }
+
+define i128 @test_i128_smax(i128 %a) nounwind {
+; X64-LABEL: test_i128_smax:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    xorl %edx, %edx
+; X64-NEXT:    testq %rsi, %rsi
+; X64-NEXT:    cmovsq %rdx, %rax
+; X64-NEXT:    cmovgq %rsi, %rdx
+; X64-NEXT:    retq
+;
+; X86-BMI-LABEL: test_i128_smax:
+; X86-BMI:       # %bb.0:
+; X86-BMI-NEXT:    pushl %edi
+; X86-BMI-NEXT:    pushl %esi
+; X86-BMI-NEXT:    pushl %eax
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI-NEXT:    xorl %edx, %edx
+; X86-BMI-NEXT:    testl %ecx, %ecx
+; X86-BMI-NEXT:    cmovlel %edx, %ecx
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI-NEXT:    cmovsl %edx, %esi
+; X86-BMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI-NEXT:    cmovsl %edx, %edi
+; X86-BMI-NEXT:    cmovnsl {{[0-9]+}}(%esp), %edx
+; X86-BMI-NEXT:    movl %ecx, 12(%eax)
+; X86-BMI-NEXT:    movl %edx, 8(%eax)
+; X86-BMI-NEXT:    movl %edi, 4(%eax)
+; X86-BMI-NEXT:    movl %esi, (%eax)
+; X86-BMI-NEXT:    addl $4, %esp
+; X86-BMI-NEXT:    popl %esi
+; X86-BMI-NEXT:    popl %edi
+; X86-BMI-NEXT:    retl $4
+;
+; X86-NOBMI-LABEL: test_i128_smax:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NOBMI-NEXT:    testl %ecx, %ecx
+; X86-NOBMI-NEXT:    movl $0, %edx
+; X86-NOBMI-NEXT:    movl $0, %esi
+; X86-NOBMI-NEXT:    movl $0, %edi
+; X86-NOBMI-NEXT:    js .LBB8_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:  .LBB8_2:
+; X86-NOBMI-NEXT:    jg .LBB8_4
+; X86-NOBMI-NEXT:  # %bb.3:
+; X86-NOBMI-NEXT:    xorl %ecx, %ecx
+; X86-NOBMI-NEXT:  .LBB8_4:
+; X86-NOBMI-NEXT:    movl %ecx, 12(%eax)
+; X86-NOBMI-NEXT:    movl %edi, 8(%eax)
+; X86-NOBMI-NEXT:    movl %esi, 4(%eax)
+; X86-NOBMI-NEXT:    movl %edx, (%eax)
+; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    retl $4
+  %r = call i128 @llvm.smax.i128(i128 %a, i128 0)
+  ret i128 %r
+}
+
+define i128 @test_i128_smin(i128 %a) nounwind {
+; X64-LABEL: test_i128_smin:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    movq %rsi, %rdx
+; X64-NEXT:    sarq $63, %rdx
+; X64-NEXT:    andq %rdx, %rax
+; X64-NEXT:    andq %rsi, %rdx
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_i128_smin:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    sarl $31, %edx
+; X86-NEXT:    andl %edx, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    andl %edx, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    andl %edx, %edi
+; X86-NEXT:    andl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %edi, 4(%eax)
+; X86-NEXT:    movl %esi, (%eax)
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+  %r = call i128 @llvm.smin.i128(i128 %a, i128 0)
+  ret i128 %r
+}
diff --git a/llvm/test/CodeGen/X86/shrink_wrap_dbg_value.mir b/llvm/test/CodeGen/X86/shrink_wrap_dbg_value.mir
index af2a18d92b34f..52cbcf22b9988 100644
--- a/llvm/test/CodeGen/X86/shrink_wrap_dbg_value.mir
+++ b/llvm/test/CodeGen/X86/shrink_wrap_dbg_value.mir
@@ -120,10 +120,12 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  # CHECK: savePoint:       '%bb.1'
-  # CHECK: restorePoint:    '%bb.3'
-  savePoint:       ''
-  restorePoint:    ''
+  # CHECK:      savePoint:
+  # CHECK-NEXT:   - point:           '%bb.1'
+  # CHECK:      restorePoint:
+  # CHECK-NEXT:   - point:           '%bb.3'
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
   - { id: 0, type: default, offset: 4, size: 4, alignment: 4, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true }
diff --git a/llvm/test/CodeGen/X86/stack-clash-extra-huge.ll b/llvm/test/CodeGen/X86/stack-clash-extra-huge.ll
index d9b20f50e9a88..4c8bb62e87dcd 100644
--- a/llvm/test/CodeGen/X86/stack-clash-extra-huge.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-extra-huge.ll
@@ -16,13 +16,13 @@ define i32 @foo() local_unnamed_addr #0 {
 ; CHECK-X64-NEXT:    cmpq %r11, %rsp
 ; CHECK-X64-NEXT:    jne .LBB0_1
 ; CHECK-X64-NEXT:  # %bb.2:
-; CHECK-X64-NEXT:    subq $3976, %rsp # imm = 0xF88
+; CHECK-X64-NEXT:    subq $3992, %rsp # imm = 0xF98
 ; CHECK-X64-NEXT:    .cfi_def_cfa_register %rsp
-; CHECK-X64-NEXT:    .cfi_def_cfa_offset 4799999888
-; CHECK-X64-NEXT:    movl $1, 264(%rsp)
-; CHECK-X64-NEXT:    movl $1, 28664(%rsp)
-; CHECK-X64-NEXT:    movl -128(%rsp), %eax
-; CHECK-X64-NEXT:    movabsq $4799999880, %rcx # imm = 0x11E1A2F88
+; CHECK-X64-NEXT:    .cfi_def_cfa_offset 4799999904
+; CHECK-X64-NEXT:    movl $1, 280(%rsp)
+; CHECK-X64-NEXT:    movl $1, 28680(%rsp)
+; CHECK-X64-NEXT:    movl -112(%rsp), %eax
+; CHECK-X64-NEXT:    movabsq $4799999896, %rcx # imm = 0x11E1A2F98
 ; CHECK-X64-NEXT:    addq %rcx, %rsp
 ; CHECK-X64-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-X64-NEXT:    retq
@@ -30,10 +30,10 @@ define i32 @foo() local_unnamed_addr #0 {
 ; CHECK-X86-LABEL: foo:
 ; CHECK-X86:       # %bb.0:
 ; CHECK-X86-NEXT:    ud2
-; CHECK-X86-NEXT:    .cfi_def_cfa_offset 4800000016
-; CHECK-X86-NEXT:    movl $1, 392(%esp)
-; CHECK-X86-NEXT:    movl $1, 28792(%esp)
-; CHECK-X86-NEXT:    movl (%esp), %eax
+; CHECK-X86-NEXT:    .cfi_def_cfa_offset 4800000032
+; CHECK-X86-NEXT:    movl $1, 408(%esp)
+; CHECK-X86-NEXT:    movl $1, 28808(%esp)
+; CHECK-X86-NEXT:    movl 16(%esp), %eax
 ; CHECK-X86-NEXT:    ud2
 ; CHECK-X86-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK-X86-NEXT:    retl
@@ -41,10 +41,10 @@ define i32 @foo() local_unnamed_addr #0 {
 ; CHECK-X32-LABEL: foo:
 ; CHECK-X32:       # %bb.0:
 ; CHECK-X32-NEXT:    ud2
-; CHECK-X32-NEXT:    .cfi_def_cfa_offset 4799999888
-; CHECK-X32-NEXT:    movl $1, 264(%esp)
-; CHECK-X32-NEXT:    movl $1, 28664(%esp)
-; CHECK-X32-NEXT:    movl -128(%esp), %eax
+; CHECK-X32-NEXT:    .cfi_def_cfa_offset 4799999904
+; CHECK-X32-NEXT:    movl $1, 280(%esp)
+; CHECK-X32-NEXT:    movl $1, 28680(%esp)
+; CHECK-X32-NEXT:    movl -112(%esp), %eax
 ; CHECK-X32-NEXT:    ud2
 ; CHECK-X32-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-X32-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/stack-clash-huge.ll b/llvm/test/CodeGen/X86/stack-clash-huge.ll
index c9990773201f0..0e8c2155c6501 100644
--- a/llvm/test/CodeGen/X86/stack-clash-huge.ll
+++ b/llvm/test/CodeGen/X86/stack-clash-huge.ll
@@ -16,13 +16,13 @@ define i32 @foo() local_unnamed_addr #0 {
 ; CHECK-X64-NEXT:    cmpq %r11, %rsp
 ; CHECK-X64-NEXT:    jne .LBB0_1
 ; CHECK-X64-NEXT:  # %bb.2:
-; CHECK-X64-NEXT:    subq $1928, %rsp # imm = 0x788
+; CHECK-X64-NEXT:    subq $1944, %rsp # imm = 0x798
 ; CHECK-X64-NEXT:    .cfi_def_cfa_register %rsp
-; CHECK-X64-NEXT:    .cfi_def_cfa_offset 2399999888
-; CHECK-X64-NEXT:    movl $1, 264(%rsp)
-; CHECK-X64-NEXT:    movl $1, 28664(%rsp)
-; CHECK-X64-NEXT:    movl -128(%rsp), %eax
-; CHECK-X64-NEXT:    movl $2399999880, %ecx # imm = 0x8F0D1788
+; CHECK-X64-NEXT:    .cfi_def_cfa_offset 2399999904
+; CHECK-X64-NEXT:    movl $1, 280(%rsp)
+; CHECK-X64-NEXT:    movl $1, 28680(%rsp)
+; CHECK-X64-NEXT:    movl -112(%rsp), %eax
+; CHECK-X64-NEXT:    movl $2399999896, %ecx # imm = 0x8F0D1798
 ; CHECK-X64-NEXT:    addq %rcx, %rsp
 ; CHECK-X64-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-X64-NEXT:    retq
@@ -39,13 +39,13 @@ define i32 @foo() local_unnamed_addr #0 {
 ; CHECK-X86-NEXT:    cmpl %eax, %esp
 ; CHECK-X86-NEXT:    jne .LBB0_1
 ; CHECK-X86-NEXT:  # %bb.2:
-; CHECK-X86-NEXT:    subl $2060, %esp # imm = 0x80C
+; CHECK-X86-NEXT:    subl $2076, %esp # imm = 0x81C
 ; CHECK-X86-NEXT:    .cfi_def_cfa_register %esp
-; CHECK-X86-NEXT:    .cfi_def_cfa_offset 2400000016
-; CHECK-X86-NEXT:    movl $1, 392(%esp)
-; CHECK-X86-NEXT:    movl $1, 28792(%esp)
-; CHECK-X86-NEXT:    movl (%esp), %eax
-; CHECK-X86-NEXT:    movl $2400000012, %ecx # imm = 0x8F0D180C
+; CHECK-X86-NEXT:    .cfi_def_cfa_offset 2400000032
+; CHECK-X86-NEXT:    movl $1, 408(%esp)
+; CHECK-X86-NEXT:    movl $1, 28808(%esp)
+; CHECK-X86-NEXT:    movl 16(%esp), %eax
+; CHECK-X86-NEXT:    movl $2400000028, %ecx # imm = 0x8F0D181C
 ; CHECK-X86-NEXT:    addl %ecx, %esp
 ; CHECK-X86-NEXT:    .cfi_def_cfa_offset 4
 ; CHECK-X86-NEXT:    retl
@@ -62,13 +62,13 @@ define i32 @foo() local_unnamed_addr #0 {
 ; CHECK-X32-NEXT:    cmpl %r11d, %esp
 ; CHECK-X32-NEXT:    jne .LBB0_1
 ; CHECK-X32-NEXT:  # %bb.2:
-; CHECK-X32-NEXT:    subl $1928, %esp # imm = 0x788
+; CHECK-X32-NEXT:    subl $1944, %esp # imm = 0x798
 ; CHECK-X32-NEXT:    .cfi_def_cfa_register %rsp
-; CHECK-X32-NEXT:    .cfi_def_cfa_offset 2399999888
-; CHECK-X32-NEXT:    movl $1, 264(%esp)
-; CHECK-X32-NEXT:    movl $1, 28664(%esp)
-; CHECK-X32-NEXT:    movl -128(%esp), %eax
-; CHECK-X32-NEXT:    movl $2399999880, %ecx # imm = 0x8F0D1788
+; CHECK-X32-NEXT:    .cfi_def_cfa_offset 2399999904
+; CHECK-X32-NEXT:    movl $1, 280(%esp)
+; CHECK-X32-NEXT:    movl $1, 28680(%esp)
+; CHECK-X32-NEXT:    movl -112(%esp), %eax
+; CHECK-X32-NEXT:    movl $2399999896, %ecx # imm = 0x8F0D1798
 ; CHECK-X32-NEXT:    addl %ecx, %esp
 ; CHECK-X32-NEXT:    .cfi_def_cfa_offset 8
 ; CHECK-X32-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-call.mir b/llvm/test/CodeGen/X86/statepoint-fixup-call.mir
index 2e0efa9d963dc..100d892617cb9 100644
--- a/llvm/test/CodeGen/X86/statepoint-fixup-call.mir
+++ b/llvm/test/CodeGen/X86/statepoint-fixup-call.mir
@@ -58,8 +58,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-copy-prop-neg.mir b/llvm/test/CodeGen/X86/statepoint-fixup-copy-prop-neg.mir
index e87e4d7ae67c8..257ea88616640 100644
--- a/llvm/test/CodeGen/X86/statepoint-fixup-copy-prop-neg.mir
+++ b/llvm/test/CodeGen/X86/statepoint-fixup-copy-prop-neg.mir
@@ -59,8 +59,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir b/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir
index a44b62583460d..16796d5adb8ef 100644
--- a/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir
+++ b/llvm/test/CodeGen/X86/statepoint-fixup-invoke.mir
@@ -76,8 +76,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir b/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir
index a2a194b651b3a..336b55cfb8ad7 100644
--- a/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir
+++ b/llvm/test/CodeGen/X86/statepoint-fixup-shared-ehpad.mir
@@ -93,8 +93,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-undef-def.mir b/llvm/test/CodeGen/X86/statepoint-fixup-undef-def.mir
index 1a88a17213813..8dab329a3e168 100644
--- a/llvm/test/CodeGen/X86/statepoint-fixup-undef-def.mir
+++ b/llvm/test/CodeGen/X86/statepoint-fixup-undef-def.mir
@@ -73,8 +73,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/X86/statepoint-fixup-undef.mir b/llvm/test/CodeGen/X86/statepoint-fixup-undef.mir
index 05d04f5b3efdc..ef053ca95933d 100644
--- a/llvm/test/CodeGen/X86/statepoint-fixup-undef.mir
+++ b/llvm/test/CodeGen/X86/statepoint-fixup-undef.mir
@@ -71,8 +71,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 8, alignment: 8,
diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir
index 5f05270729fde..29c22bb8d3fd1 100644
--- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir
+++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-enter-at-end.mir
@@ -241,8 +241,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir
index cf9128260f196..a50b0a5b69df8 100644
--- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir
+++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-hoist-copies.mir
@@ -408,8 +408,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir
index fcebc69d9b2e3..0c5c3669268c0 100644
--- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir
+++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-inline-spiller.mir
@@ -185,8 +185,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir
index 8bb39a03f7e36..05c7ceb914c93 100644
--- a/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir
+++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra-remove-back-copies.mir
@@ -236,8 +236,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir
index da651039ce21e..5ce76a3e46f66 100644
--- a/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir
+++ b/llvm/test/CodeGen/X86/statepoint-invoke-ra.mir
@@ -182,8 +182,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: default, offset: 0, size: 4, alignment: 4,
diff --git a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir
index d40a9a06d1620..8b6445b7a21e0 100644
--- a/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir
+++ b/llvm/test/CodeGen/X86/statepoint-vreg-folding.mir
@@ -123,8 +123,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: 16, size: 8, alignment: 16, stack-id: default,
       isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true,
diff --git a/llvm/test/CodeGen/X86/tied-depbreak.mir b/llvm/test/CodeGen/X86/tied-depbreak.mir
index 4eca28795516f..cc3d8df2c3a52 100644
--- a/llvm/test/CodeGen/X86/tied-depbreak.mir
+++ b/llvm/test/CodeGen/X86/tied-depbreak.mir
@@ -40,8 +40,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/trunc-nsw-nuw.ll b/llvm/test/CodeGen/X86/trunc-nsw-nuw.ll
index 5c5f7045ea030..6b0789127f5f9 100644
--- a/llvm/test/CodeGen/X86/trunc-nsw-nuw.ll
+++ b/llvm/test/CodeGen/X86/trunc-nsw-nuw.ll
@@ -62,10 +62,11 @@ entry:
 define i32 @simplify_demanded_bits_drop_flag(i1 zeroext %x, i1 zeroext %y) nounwind {
 ; CHECK-LABEL: simplify_demanded_bits_drop_flag:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    negl %edi
+; CHECK-NEXT:    # kill: def $esi killed $esi def $rsi
 ; CHECK-NEXT:    shll $2, %esi
-; CHECK-NEXT:    xorl %edi, %esi
-; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    negq %rax
+; CHECK-NEXT:    xorq %rsi, %rax
 ; CHECK-NEXT:    imulq $-1634202141, %rax, %rax # imm = 0x9E980DE3
 ; CHECK-NEXT:    movq %rax, %rcx
 ; CHECK-NEXT:    shrq $63, %rcx
diff --git a/llvm/test/CodeGen/X86/unfoldMemoryOperand.mir b/llvm/test/CodeGen/X86/unfoldMemoryOperand.mir
index 135b14d6836a0..75953afa09ba9 100644
--- a/llvm/test/CodeGen/X86/unfoldMemoryOperand.mir
+++ b/llvm/test/CodeGen/X86/unfoldMemoryOperand.mir
@@ -72,8 +72,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/X86/vector-llrint-f16.ll b/llvm/test/CodeGen/X86/vector-llrint-f16.ll
index 5e5c5849fc22e..eb7be61b719f2 100644
--- a/llvm/test/CodeGen/X86/vector-llrint-f16.ll
+++ b/llvm/test/CodeGen/X86/vector-llrint-f16.ll
@@ -1,10 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; FIXME: crash "Do not know how to split the result of this operator!"
+; SKIP: sed 's/XRINT/lrint/g' %s | llc -mtriple=i686-unknown | FileCheck %s --check-prefix=X86
+; SKIP: sed 's/XRINT/llrint/g' %s | llc -mtriple=i686-unknown | FileCheck %s --check-prefix=X86
 ; RUN: sed 's/XRINT/lrint/g' %s | llc -mtriple=x86_64-unknown -mattr=avx2,f16c | FileCheck %s --check-prefix=AVX
 ; RUN: sed 's/XRINT/llrint/g' %s | llc -mtriple=x86_64-unknown -mattr=avx2,f16c | FileCheck %s --check-prefix=AVX
 ; RUN: sed 's/XRINT/lrint/g' %s | llc -mtriple=x86_64-unknown -mattr=avx512fp16,avx512vl | FileCheck %s --check-prefix=FP16
 ; RUN: sed 's/XRINT/llrint/g' %s | llc -mtriple=x86_64-unknown -mattr=avx512fp16,avx512vl | FileCheck %s --check-prefix=FP16
 
-define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
+define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) nounwind {
 ; AVX-LABEL: llrint_v1i64_v1f16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm0
@@ -22,7 +25,7 @@ define <1 x i64> @llrint_v1i64_v1f16(<1 x half> %x) {
   ret <1 x i64> %a
 }
 
-define <2 x i64> @llrint_v2i64_v2f16(<2 x half> %x) {
+define <2 x i64> @llrint_v2i64_v2f16(<2 x half> %x) nounwind {
 ; AVX-LABEL: llrint_v2i64_v2f16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vcvtph2ps %xmm0, %xmm1
@@ -49,7 +52,7 @@ define <2 x i64> @llrint_v2i64_v2f16(<2 x half> %x) {
   ret <2 x i64> %a
 }
 
-define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
+define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) nounwind {
 ; AVX-LABEL: llrint_v4i64_v4f16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrlq $48, %xmm0, %xmm1
@@ -92,7 +95,7 @@ define <4 x i64> @llrint_v4i64_v4f16(<4 x half> %x) {
   ret <4 x i64> %a
 }
 
-define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
+define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) nounwind {
 ; AVX-LABEL: llrint_v8i64_v8f16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpsrlq $48, %xmm0, %xmm1
@@ -167,7 +170,7 @@ define <8 x i64> @llrint_v8i64_v8f16(<8 x half> %x) {
   ret <8 x i64> %a
 }
 
-define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
+define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) nounwind {
 ; AVX-LABEL: llrint_v16i64_v16f16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovdqa %ymm0, %ymm2
@@ -307,7 +310,7 @@ define <16 x i64> @llrint_v16i64_v16f16(<16 x half> %x) {
   ret <16 x i64> %a
 }
 
-define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) {
+define <32 x i64> @llrint_v32i64_v32f16(<32 x half> %x) nounwind {
 ; AVX-LABEL: llrint_v32i64_v32f16:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    movq %rdi, %rax
diff --git a/llvm/test/CodeGen/X86/vector-llrint.ll b/llvm/test/CodeGen/X86/vector-llrint.ll
index 7017eb60df41d..6fd1a35505aac 100644
--- a/llvm/test/CodeGen/X86/vector-llrint.ll
+++ b/llvm/test/CodeGen/X86/vector-llrint.ll
@@ -1,10 +1,25 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefix=X86
 ; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx | FileCheck %s --check-prefixes=AVX,AVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx512f | FileCheck %s --check-prefixes=AVX,AVX512
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=avx512dq,avx512vl | FileCheck %s --check-prefixes=AVX512DQ
 
-define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
+define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) nounwind  {
+; X86-LABEL: llrint_v1i64_v1f32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    flds 8(%ebp)
+; X86-NEXT:    fistpll (%esp)
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
 ; SSE-LABEL: llrint_v1i64_v1f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvtss2si %xmm0, %rax
@@ -24,7 +39,34 @@ define <1 x i64> @llrint_v1i64_v1f32(<1 x float> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f32(<1 x float>)
 
-define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
+define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) nounwind {
+; X86-LABEL: llrint_v2i64_v2f32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    flds 16(%ebp)
+; X86-NEXT:    flds 12(%ebp)
+; X86-NEXT:    fistpll (%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    movl (%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 12(%eax)
+; X86-NEXT:    movl %esi, 8(%eax)
+; X86-NEXT:    movl %edx, 4(%eax)
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    leal -8(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
 ; SSE-LABEL: llrint_v2i64_v2f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvtss2si %xmm0, %rax
@@ -55,7 +97,54 @@ define <2 x i64> @llrint_v2i64_v2f32(<2 x float> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f32(<2 x float>)
 
-define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
+define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) nounwind {
+; X86-LABEL: llrint_v4i64_v4f32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $56, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    flds 24(%ebp)
+; X86-NEXT:    flds 20(%ebp)
+; X86-NEXT:    flds 16(%ebp)
+; X86-NEXT:    flds 12(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, 28(%eax)
+; X86-NEXT:    movl %ecx, 24(%eax)
+; X86-NEXT:    movl %edx, 20(%eax)
+; X86-NEXT:    movl %ebx, 16(%eax)
+; X86-NEXT:    movl %edi, 12(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 8(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 4(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
 ; SSE-LABEL: llrint_v4i64_v4f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvtss2si %xmm0, %rax
@@ -121,7 +210,94 @@ define <4 x i64> @llrint_v4i64_v4f32(<4 x float> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f32(<4 x float>)
 
-define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
+define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) nounwind {
+; X86-LABEL: llrint_v8i64_v8f32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $120, %esp
+; X86-NEXT:    flds 12(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 16(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 20(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 24(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 28(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 32(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 36(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 40(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %ebx, 60(%eax)
+; X86-NEXT:    movl %ecx, 56(%eax)
+; X86-NEXT:    movl %edx, 52(%eax)
+; X86-NEXT:    movl %esi, 48(%eax)
+; X86-NEXT:    movl %edi, 44(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 40(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 36(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 32(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 28(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 24(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 20(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 16(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 8(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 4(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
 ; SSE-LABEL: llrint_v8i64_v8f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movaps %xmm0, %xmm2
@@ -235,7 +411,174 @@ define <8 x i64> @llrint_v8i64_v8f32(<8 x float> %x) {
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f32(<8 x float>)
 
-define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
+define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) nounwind {
+; X86-LABEL: llrint_v16i64_v16f32:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $248, %esp
+; X86-NEXT:    flds 12(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 16(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 20(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 24(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 28(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 32(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 36(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 40(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 44(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 48(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 52(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 56(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 60(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 64(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 68(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    flds 72(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %ebx, 124(%eax)
+; X86-NEXT:    movl %ecx, 120(%eax)
+; X86-NEXT:    movl %edx, 116(%eax)
+; X86-NEXT:    movl %esi, 112(%eax)
+; X86-NEXT:    movl %edi, 108(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 104(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 100(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 96(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 92(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 88(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 84(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 80(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 76(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 72(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 68(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 64(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 60(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 56(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 52(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 48(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 44(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 40(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 36(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 32(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 28(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 24(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 20(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 16(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 8(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 4(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
 ; SSE-LABEL: llrint_v16i64_v16f32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movq %rdi, %rax
@@ -451,7 +794,21 @@ define <16 x i64> @llrint_v16i64_v16f32(<16 x float> %x) {
 }
 declare <16 x i64> @llvm.llrint.v16i64.v16f32(<16 x float>)
 
-define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
+define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) nounwind {
+; X86-LABEL: llrint_v1i64_v1f64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    fldl 8(%ebp)
+; X86-NEXT:    fistpll (%esp)
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
 ; SSE-LABEL: llrint_v1i64_v1f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvtsd2si %xmm0, %rax
@@ -471,7 +828,34 @@ define <1 x i64> @llrint_v1i64_v1f64(<1 x double> %x) {
 }
 declare <1 x i64> @llvm.llrint.v1i64.v1f64(<1 x double>)
 
-define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
+define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) nounwind {
+; X86-LABEL: llrint_v2i64_v2f64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    fldl 20(%ebp)
+; X86-NEXT:    fldl 12(%ebp)
+; X86-NEXT:    fistpll (%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    movl (%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %edi, 12(%eax)
+; X86-NEXT:    movl %esi, 8(%eax)
+; X86-NEXT:    movl %edx, 4(%eax)
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    leal -8(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
 ; SSE-LABEL: llrint_v2i64_v2f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvtsd2si %xmm0, %rax
@@ -502,7 +886,54 @@ define <2 x i64> @llrint_v2i64_v2f64(<2 x double> %x) {
 }
 declare <2 x i64> @llvm.llrint.v2i64.v2f64(<2 x double>)
 
-define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
+define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) nounwind {
+; X86-LABEL: llrint_v4i64_v4f64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $56, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    fldl 36(%ebp)
+; X86-NEXT:    fldl 28(%ebp)
+; X86-NEXT:    fldl 20(%ebp)
+; X86-NEXT:    fldl 12(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %esi, 28(%eax)
+; X86-NEXT:    movl %ecx, 24(%eax)
+; X86-NEXT:    movl %edx, 20(%eax)
+; X86-NEXT:    movl %ebx, 16(%eax)
+; X86-NEXT:    movl %edi, 12(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 8(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 4(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
 ; SSE-LABEL: llrint_v4i64_v4f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvtsd2si %xmm0, %rax
@@ -566,7 +997,94 @@ define <4 x i64> @llrint_v4i64_v4f64(<4 x double> %x) {
 }
 declare <4 x i64> @llvm.llrint.v4i64.v4f64(<4 x double>)
 
-define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
+define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) nounwind {
+; X86-LABEL: llrint_v8i64_v8f64:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $120, %esp
+; X86-NEXT:    fldl 12(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl 20(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl 28(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl 36(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl 44(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl 52(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl 60(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    fldl 68(%ebp)
+; X86-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl %ebx, 60(%eax)
+; X86-NEXT:    movl %ecx, 56(%eax)
+; X86-NEXT:    movl %edx, 52(%eax)
+; X86-NEXT:    movl %esi, 48(%eax)
+; X86-NEXT:    movl %edi, 44(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 40(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 36(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 32(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 28(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 24(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 20(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 16(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 8(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, 4(%eax)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT:    movl %ecx, (%eax)
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
 ; SSE-LABEL: llrint_v8i64_v8f64:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    cvtsd2si %xmm0, %rax
@@ -673,3 +1191,655 @@ define <8 x i64> @llrint_v8i64_v8f64(<8 x double> %x) {
   ret <8 x i64> %a
 }
 declare <8 x i64> @llvm.llrint.v8i64.v8f64(<8 x double>)
+
+define <1 x i64> @llrint_v1i64_v1f128(<1 x fp128> %x) nounwind {
+; X86-LABEL: llrint_v1i64_v1f128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    pushl 20(%ebp)
+; X86-NEXT:    pushl 16(%ebp)
+; X86-NEXT:    pushl 12(%ebp)
+; X86-NEXT:    pushl 8(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; SSE-LABEL: llrint_v1i64_v1f128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pushq %rax
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    popq %rcx
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: llrint_v1i64_v1f128:
+; AVX:       # %bb.0:
+; AVX-NEXT:    pushq %rax
+; AVX-NEXT:    callq llrintl@PLT
+; AVX-NEXT:    popq %rcx
+; AVX-NEXT:    retq
+;
+; AVX512DQ-LABEL: llrint_v1i64_v1f128:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    popq %rcx
+; AVX512DQ-NEXT:    retq
+  %a = call <1 x i64> @llvm.llrint.v1i64.v1f128(<1 x fp128> %x)
+  ret <1 x i64> %a
+}
+declare <1 x i64> @llvm.llrint.v1i64.v1f128(<1 x fp128>)
+
+define <2 x i64> @llrint_v2i64_v2f128(<2 x fp128> %x) nounwind {
+; X86-LABEL: llrint_v2i64_v2f128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    movl 8(%ebp), %esi
+; X86-NEXT:    pushl 24(%ebp)
+; X86-NEXT:    pushl 20(%ebp)
+; X86-NEXT:    pushl 16(%ebp)
+; X86-NEXT:    pushl 12(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    pushl 40(%ebp)
+; X86-NEXT:    pushl 36(%ebp)
+; X86-NEXT:    pushl 32(%ebp)
+; X86-NEXT:    pushl 28(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %edx, 12(%esi)
+; X86-NEXT:    movl %eax, 8(%esi)
+; X86-NEXT:    movl %ebx, 4(%esi)
+; X86-NEXT:    movl %edi, (%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
+; SSE-LABEL: llrint_v2i64_v2f128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    subq $40, %rsp
+; SSE-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    punpcklqdq (%rsp), %xmm0 # 16-byte Folded Reload
+; SSE-NEXT:    # xmm0 = xmm0[0],mem[0]
+; SSE-NEXT:    addq $40, %rsp
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: llrint_v2i64_v2f128:
+; AVX:       # %bb.0:
+; AVX-NEXT:    subq $40, %rsp
+; AVX-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:    callq llrintl@PLT
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX-NEXT:    callq llrintl@PLT
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX-NEXT:    addq $40, %rsp
+; AVX-NEXT:    retq
+;
+; AVX512DQ-LABEL: llrint_v2i64_v2f128:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    subq $40, %rsp
+; AVX512DQ-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm1, %xmm0
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-NEXT:    addq $40, %rsp
+; AVX512DQ-NEXT:    retq
+  %a = call <2 x i64> @llvm.llrint.v2i64.v2f128(<2 x fp128> %x)
+  ret <2 x i64> %a
+}
+declare <2 x i64> @llvm.llrint.v2i64.v2f128(<2 x fp128>)
+
+define <4 x i64> @llrint_v4i64_v4f128(<4 x fp128> %x) nounwind {
+; X86-LABEL: llrint_v4i64_v4f128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $32, %esp
+; X86-NEXT:    movl 8(%ebp), %esi
+; X86-NEXT:    movl 36(%ebp), %edi
+; X86-NEXT:    movl 40(%ebp), %ebx
+; X86-NEXT:    pushl 24(%ebp)
+; X86-NEXT:    pushl 20(%ebp)
+; X86-NEXT:    pushl 16(%ebp)
+; X86-NEXT:    pushl 12(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl 32(%ebp)
+; X86-NEXT:    pushl 28(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl 56(%ebp)
+; X86-NEXT:    pushl 52(%ebp)
+; X86-NEXT:    pushl 48(%ebp)
+; X86-NEXT:    pushl 44(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    pushl 72(%ebp)
+; X86-NEXT:    pushl 68(%ebp)
+; X86-NEXT:    pushl 64(%ebp)
+; X86-NEXT:    pushl 60(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %edx, 28(%esi)
+; X86-NEXT:    movl %eax, 24(%esi)
+; X86-NEXT:    movl %ebx, 20(%esi)
+; X86-NEXT:    movl %edi, 16(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 12(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 8(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 4(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
+; SSE-LABEL: llrint_v4i64_v4f128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    subq $72, %rsp
+; SSE-NEXT:    movaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm0, (%rsp) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; SSE-NEXT:    # xmm0 = xmm0[0],mem[0]
+; SSE-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm1
+; SSE-NEXT:    punpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; SSE-NEXT:    # xmm1 = xmm1[0],mem[0]
+; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE-NEXT:    addq $72, %rsp
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: llrint_v4i64_v4f128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    subq $72, %rsp
+; AVX1-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm3, %xmm0
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX1-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX1-NEXT:    vinsertf128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX1-NEXT:    addq $72, %rsp
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: llrint_v4i64_v4f128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    subq $72, %rsp
+; AVX512-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm3, %xmm0
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-NEXT:    vinserti128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512-NEXT:    addq $72, %rsp
+; AVX512-NEXT:    retq
+;
+; AVX512DQ-LABEL: llrint_v4i64_v4f128:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    subq $72, %rsp
+; AVX512DQ-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm3, %xmm0
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-NEXT:    vinserti128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    addq $72, %rsp
+; AVX512DQ-NEXT:    retq
+  %a = call <4 x i64> @llvm.llrint.v4i64.v4f128(<4 x fp128> %x)
+  ret <4 x i64> %a
+}
+declare <4 x i64> @llvm.llrint.v4i64.v4f128(<4 x fp128>)
+
+define <8 x i64> @llrint_v8i64_v8f128(<8 x fp128> %x) nounwind {
+; X86-LABEL: llrint_v8i64_v8f128:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    andl $-16, %esp
+; X86-NEXT:    subl $64, %esp
+; X86-NEXT:    movl 8(%ebp), %esi
+; X86-NEXT:    movl 36(%ebp), %edi
+; X86-NEXT:    movl 40(%ebp), %ebx
+; X86-NEXT:    pushl 24(%ebp)
+; X86-NEXT:    pushl 20(%ebp)
+; X86-NEXT:    pushl 16(%ebp)
+; X86-NEXT:    pushl 12(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl 32(%ebp)
+; X86-NEXT:    pushl 28(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl 56(%ebp)
+; X86-NEXT:    pushl 52(%ebp)
+; X86-NEXT:    pushl 48(%ebp)
+; X86-NEXT:    pushl 44(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl 72(%ebp)
+; X86-NEXT:    pushl 68(%ebp)
+; X86-NEXT:    pushl 64(%ebp)
+; X86-NEXT:    pushl 60(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl 88(%ebp)
+; X86-NEXT:    pushl 84(%ebp)
+; X86-NEXT:    pushl 80(%ebp)
+; X86-NEXT:    pushl 76(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl 104(%ebp)
+; X86-NEXT:    pushl 100(%ebp)
+; X86-NEXT:    pushl 96(%ebp)
+; X86-NEXT:    pushl 92(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT:    pushl 120(%ebp)
+; X86-NEXT:    pushl 116(%ebp)
+; X86-NEXT:    pushl 112(%ebp)
+; X86-NEXT:    pushl 108(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %eax, %edi
+; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    pushl 136(%ebp)
+; X86-NEXT:    pushl 132(%ebp)
+; X86-NEXT:    pushl 128(%ebp)
+; X86-NEXT:    pushl 124(%ebp)
+; X86-NEXT:    calll llrintl
+; X86-NEXT:    addl $16, %esp
+; X86-NEXT:    movl %edx, 60(%esi)
+; X86-NEXT:    movl %eax, 56(%esi)
+; X86-NEXT:    movl %ebx, 52(%esi)
+; X86-NEXT:    movl %edi, 48(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 44(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 40(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 36(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 32(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 28(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 24(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 20(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 16(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 12(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 8(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, 4(%esi)
+; X86-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT:    movl %eax, (%esi)
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    leal -12(%ebp), %esp
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
+; SSE-LABEL: llrint_v8i64_v8f128:
+; SSE:       # %bb.0:
+; SSE-NEXT:    subq $136, %rsp
+; SSE-NEXT:    movaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm3, (%rsp) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; SSE-NEXT:    # xmm0 = xmm0[0],mem[0]
+; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps (%rsp), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    movdqa %xmm0, (%rsp) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    punpcklqdq (%rsp), %xmm0 # 16-byte Folded Reload
+; SSE-NEXT:    # xmm0 = xmm0[0],mem[0]
+; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    punpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; SSE-NEXT:    # xmm0 = xmm0[0],mem[0]
+; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    callq llrintl@PLT
+; SSE-NEXT:    movq %rax, %xmm3
+; SSE-NEXT:    punpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
+; SSE-NEXT:    # xmm3 = xmm3[0],mem[0]
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE-NEXT:    movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; SSE-NEXT:    addq $136, %rsp
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: llrint_v8i64_v8f128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    subq $152, %rsp
+; AVX1-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps %xmm3, %xmm0
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX1-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX1-NEXT:    vinsertf128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX1-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-NEXT:    callq llrintl@PLT
+; AVX1-NEXT:    vmovq %rax, %xmm0
+; AVX1-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX1-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
+; AVX1-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX1-NEXT:    addq $152, %rsp
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: llrint_v8i64_v8f128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    subq $152, %rsp
+; AVX512-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps %xmm7, %xmm0
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-NEXT:    vinserti128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-NEXT:    callq llrintl@PLT
+; AVX512-NEXT:    vmovq %rax, %xmm0
+; AVX512-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512-NEXT:    addq $152, %rsp
+; AVX512-NEXT:    retq
+;
+; AVX512DQ-LABEL: llrint_v8i64_v8f128:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    subq $152, %rsp
+; AVX512DQ-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps %xmm7, %xmm0
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-NEXT:    vinserti128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-NEXT:    callq llrintl@PLT
+; AVX512DQ-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512DQ-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512DQ-NEXT:    addq $152, %rsp
+; AVX512DQ-NEXT:    retq
+  %a = call <8 x i64> @llvm.llrint.v8i64.v8f128(<8 x fp128> %x)
+  ret <8 x i64> %a
+}
+declare <8 x i64> @llvm.llrint.v8i64.v8f128(<8 x fp128>)
diff --git a/llvm/test/CodeGen/X86/vector-lrint-f16.ll b/llvm/test/CodeGen/X86/vector-lrint-f16.ll
index 1316f808aa27e..fa3aeb09eae6f 100644
--- a/llvm/test/CodeGen/X86/vector-lrint-f16.ll
+++ b/llvm/test/CodeGen/X86/vector-lrint-f16.ll
@@ -8,7 +8,7 @@
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=x86_64-unknown -mattr=avx2,f16c | FileCheck %s --check-prefixes=X64-AVX-I32
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=x86_64-unknown -mattr=avx512fp16,avx512vl | FileCheck %s --check-prefixes=X64-FP16-I32
 
-define <1 x iXLen> @lrint_v1f16(<1 x half> %x) {
+define <1 x iXLen> @lrint_v1f16(<1 x half> %x) nounwind {
 ; X86-AVX-I16-LABEL: lrint_v1f16:
 ; X86-AVX-I16:       # %bb.0:
 ; X86-AVX-I16-NEXT:    vcvtph2ps %xmm0, %xmm0
@@ -73,7 +73,7 @@ define <1 x iXLen> @lrint_v1f16(<1 x half> %x) {
 }
 declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f16(<1 x half>)
 
-define <2 x iXLen> @lrint_v2f16(<2 x half> %x) {
+define <2 x iXLen> @lrint_v2f16(<2 x half> %x) nounwind {
 ; X86-AVX-I16-LABEL: lrint_v2f16:
 ; X86-AVX-I16:       # %bb.0:
 ; X86-AVX-I16-NEXT:    vpsrld $16, %xmm0, %xmm1
@@ -250,7 +250,7 @@ define <2 x iXLen> @lrint_v2f16(<2 x half> %x) {
 }
 declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f16(<2 x half>)
 
-define <4 x iXLen> @lrint_v4f16(<4 x half> %x) {
+define <4 x iXLen> @lrint_v4f16(<4 x half> %x) nounwind {
 ; X86-AVX-I16-LABEL: lrint_v4f16:
 ; X86-AVX-I16:       # %bb.0:
 ; X86-AVX-I16-NEXT:    vpsrld $16, %xmm0, %xmm1
@@ -455,7 +455,7 @@ define <4 x iXLen> @lrint_v4f16(<4 x half> %x) {
 }
 declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f16(<4 x half>)
 
-define <8 x iXLen> @lrint_v8f16(<8 x half> %x) {
+define <8 x iXLen> @lrint_v8f16(<8 x half> %x) nounwind {
 ; X86-AVX-I16-LABEL: lrint_v8f16:
 ; X86-AVX-I16:       # %bb.0:
 ; X86-AVX-I16-NEXT:    vpsrld $16, %xmm0, %xmm1
@@ -718,7 +718,7 @@ define <8 x iXLen> @lrint_v8f16(<8 x half> %x) {
 }
 declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f16(<8 x half>)
 
-define <16 x iXLen> @lrint_v16f16(<16 x half> %x) {
+define <16 x iXLen> @lrint_v16f16(<16 x half> %x) nounwind {
 ; X86-AVX-I16-LABEL: lrint_v16f16:
 ; X86-AVX-I16:       # %bb.0:
 ; X86-AVX-I16-NEXT:    vextracti128 $1, %ymm0, %xmm1
@@ -1211,7 +1211,7 @@ define <16 x iXLen> @lrint_v16f16(<16 x half> %x) {
 }
 declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f16(<16 x half>)
 
-define <32 x iXLen> @lrint_v32f32(<32 x half> %x) {
+define <32 x iXLen> @lrint_v32f32(<32 x half> %x) nounwind {
 ; X86-AVX-I16-LABEL: lrint_v32f32:
 ; X86-AVX-I16:       # %bb.0:
 ; X86-AVX-I16-NEXT:    vextracti128 $1, %ymm0, %xmm2
diff --git a/llvm/test/CodeGen/X86/vector-lrint.ll b/llvm/test/CodeGen/X86/vector-lrint.ll
index b1c8d46f497f3..b3e5a0929b7a5 100644
--- a/llvm/test/CodeGen/X86/vector-lrint.ll
+++ b/llvm/test/CodeGen/X86/vector-lrint.ll
@@ -1,4 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=i686-unknown | FileCheck %s --check-prefix=X86-I32
+; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=i686-unknown | FileCheck %s --check-prefix=X86-I64
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=i686-unknown -mattr=sse2 | FileCheck %s --check-prefix=X86-SSE2
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=i686-unknown -mattr=avx | FileCheck %s --check-prefixes=X86-AVX,X86-AVX1
 ; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=i686-unknown -mattr=avx512f | FileCheck %s --check-prefixes=X86-AVX,AVX512-i32
@@ -10,7 +12,30 @@
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=x86_64-unknown -mattr=avx512f | FileCheck %s --check-prefixes=X64-AVX-i64,AVX512-i64
 ; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=x86_64-unknown -mattr=avx512dq,avx512vl | FileCheck %s --check-prefixes=X64-AVX-i64,AVX512DQ-i64
 
-define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
+define <1 x iXLen> @lrint_v1f32(<1 x float> %x) nounwind {
+; X86-I32-LABEL: lrint_v1f32:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %eax
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl (%esp)
+; X86-I32-NEXT:    movl (%esp), %eax
+; X86-I32-NEXT:    popl %ecx
+; X86-I32-NEXT:    retl
+;
+; X86-I64-LABEL: lrint_v1f32:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    andl $-8, %esp
+; X86-I64-NEXT:    subl $8, %esp
+; X86-I64-NEXT:    flds 8(%ebp)
+; X86-I64-NEXT:    fistpll (%esp)
+; X86-I64-NEXT:    movl (%esp), %eax
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I64-NEXT:    movl %ebp, %esp
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl
+;
 ; X86-SSE2-LABEL: lrint_v1f32:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    cvtss2si {{[0-9]+}}(%esp), %eax
@@ -35,7 +60,46 @@ define <1 x iXLen> @lrint_v1f32(<1 x float> %x) {
 }
 declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f32(<1 x float>)
 
-define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
+define <2 x iXLen> @lrint_v2f32(<2 x float> %x) nounwind {
+; X86-I32-LABEL: lrint_v2f32:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    subl $8, %esp
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl (%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    movl (%esp), %eax
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I32-NEXT:    addl $8, %esp
+; X86-I32-NEXT:    retl
+;
+; X86-I64-LABEL: lrint_v2f32:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl %esi
+; X86-I64-NEXT:    andl $-8, %esp
+; X86-I64-NEXT:    subl $16, %esp
+; X86-I64-NEXT:    movl 8(%ebp), %eax
+; X86-I64-NEXT:    flds 16(%ebp)
+; X86-I64-NEXT:    flds 12(%ebp)
+; X86-I64-NEXT:    fistpll (%esp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    movl (%esp), %ecx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I64-NEXT:    movl %edi, 12(%eax)
+; X86-I64-NEXT:    movl %esi, 8(%eax)
+; X86-I64-NEXT:    movl %edx, 4(%eax)
+; X86-I64-NEXT:    movl %ecx, (%eax)
+; X86-I64-NEXT:    leal -8(%ebp), %esp
+; X86-I64-NEXT:    popl %esi
+; X86-I64-NEXT:    popl %edi
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl $4
+;
 ; X86-SSE2-LABEL: lrint_v2f32:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    cvtps2dq %xmm0, %xmm0
@@ -80,7 +144,81 @@ define <2 x iXLen> @lrint_v2f32(<2 x float> %x) {
 }
 declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f32(<2 x float>)
 
-define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
+define <4 x iXLen> @lrint_v4f32(<4 x float> %x) nounwind {
+; X86-I32-LABEL: lrint_v4f32:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl %esi
+; X86-I32-NEXT:    subl $16, %esp
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl (%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    movl (%esp), %ecx
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I32-NEXT:    movl %edi, 12(%eax)
+; X86-I32-NEXT:    movl %esi, 8(%eax)
+; X86-I32-NEXT:    movl %edx, 4(%eax)
+; X86-I32-NEXT:    movl %ecx, (%eax)
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    popl %esi
+; X86-I32-NEXT:    popl %edi
+; X86-I32-NEXT:    retl $4
+;
+; X86-I64-LABEL: lrint_v4f32:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    pushl %ebx
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl %esi
+; X86-I64-NEXT:    andl $-8, %esp
+; X86-I64-NEXT:    subl $56, %esp
+; X86-I64-NEXT:    movl 8(%ebp), %eax
+; X86-I64-NEXT:    flds 24(%ebp)
+; X86-I64-NEXT:    flds 20(%ebp)
+; X86-I64-NEXT:    flds 16(%ebp)
+; X86-I64-NEXT:    flds 12(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I64-NEXT:    movl %esi, 28(%eax)
+; X86-I64-NEXT:    movl %ecx, 24(%eax)
+; X86-I64-NEXT:    movl %edx, 20(%eax)
+; X86-I64-NEXT:    movl %ebx, 16(%eax)
+; X86-I64-NEXT:    movl %edi, 12(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 8(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 4(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, (%eax)
+; X86-I64-NEXT:    leal -12(%ebp), %esp
+; X86-I64-NEXT:    popl %esi
+; X86-I64-NEXT:    popl %edi
+; X86-I64-NEXT:    popl %ebx
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl $4
+;
 ; X86-SSE2-LABEL: lrint_v4f32:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    cvtps2dq %xmm0, %xmm0
@@ -141,7 +279,145 @@ define <4 x iXLen> @lrint_v4f32(<4 x float> %x) {
 }
 declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f32(<4 x float>)
 
-define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
+define <8 x iXLen> @lrint_v8f32(<8 x float> %x) nounwind {
+; X86-I32-LABEL: lrint_v8f32:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %ebp
+; X86-I32-NEXT:    pushl %ebx
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl %esi
+; X86-I32-NEXT:    subl $40, %esp
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I32-NEXT:    movl %edx, 28(%eax)
+; X86-I32-NEXT:    movl %ecx, 24(%eax)
+; X86-I32-NEXT:    movl %ebp, 20(%eax)
+; X86-I32-NEXT:    movl %ebx, 16(%eax)
+; X86-I32-NEXT:    movl %edi, 12(%eax)
+; X86-I32-NEXT:    movl %esi, 8(%eax)
+; X86-I32-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-I32-NEXT:    movl %ecx, 4(%eax)
+; X86-I32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I32-NEXT:    movl %ecx, (%eax)
+; X86-I32-NEXT:    addl $40, %esp
+; X86-I32-NEXT:    popl %esi
+; X86-I32-NEXT:    popl %edi
+; X86-I32-NEXT:    popl %ebx
+; X86-I32-NEXT:    popl %ebp
+; X86-I32-NEXT:    retl $4
+;
+; X86-I64-LABEL: lrint_v8f32:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    pushl %ebx
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl %esi
+; X86-I64-NEXT:    andl $-8, %esp
+; X86-I64-NEXT:    subl $120, %esp
+; X86-I64-NEXT:    flds 12(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    flds 16(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    flds 20(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    flds 24(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    flds 28(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    flds 32(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    flds 36(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    flds 40(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    movl 8(%ebp), %eax
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-I64-NEXT:    movl %ebx, 60(%eax)
+; X86-I64-NEXT:    movl %ecx, 56(%eax)
+; X86-I64-NEXT:    movl %edx, 52(%eax)
+; X86-I64-NEXT:    movl %esi, 48(%eax)
+; X86-I64-NEXT:    movl %edi, 44(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 40(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 36(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 32(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 28(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 24(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 20(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 16(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 12(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 8(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 4(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, (%eax)
+; X86-I64-NEXT:    leal -12(%ebp), %esp
+; X86-I64-NEXT:    popl %esi
+; X86-I64-NEXT:    popl %edi
+; X86-I64-NEXT:    popl %ebx
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl $4
+;
 ; X86-SSE2-LABEL: lrint_v8f32:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    cvtps2dq %xmm0, %xmm0
@@ -235,13 +511,36 @@ define <8 x iXLen> @lrint_v8f32(<8 x float> %x) {
 }
 declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f32(<8 x float>)
 
-define <16 x iXLen> @lrint_v16iXLen_v16f32(<16 x float> %x) {
+define <16 x iXLen> @lrint_v16iXLen_v16f32(<16 x float> %x) nounwind {
   %a = call <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float> %x)
   ret <16 x iXLen> %a
 }
 declare <16 x iXLen> @llvm.lrint.v16iXLen.v16f32(<16 x float>)
 
-define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
+define <1 x iXLen> @lrint_v1f64(<1 x double> %x) nounwind {
+; X86-I32-LABEL: lrint_v1f64:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %eax
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl (%esp)
+; X86-I32-NEXT:    movl (%esp), %eax
+; X86-I32-NEXT:    popl %ecx
+; X86-I32-NEXT:    retl
+;
+; X86-I64-LABEL: lrint_v1f64:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    andl $-8, %esp
+; X86-I64-NEXT:    subl $8, %esp
+; X86-I64-NEXT:    fldl 8(%ebp)
+; X86-I64-NEXT:    fistpll (%esp)
+; X86-I64-NEXT:    movl (%esp), %eax
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I64-NEXT:    movl %ebp, %esp
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl
+;
 ; X86-SSE2-LABEL: lrint_v1f64:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    cvtsd2si {{[0-9]+}}(%esp), %eax
@@ -266,7 +565,46 @@ define <1 x iXLen> @lrint_v1f64(<1 x double> %x) {
 }
 declare <1 x iXLen> @llvm.lrint.v1iXLen.v1f64(<1 x double>)
 
-define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
+define <2 x iXLen> @lrint_v2f64(<2 x double> %x) nounwind {
+; X86-I32-LABEL: lrint_v2f64:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    subl $8, %esp
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl (%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    movl (%esp), %eax
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I32-NEXT:    addl $8, %esp
+; X86-I32-NEXT:    retl
+;
+; X86-I64-LABEL: lrint_v2f64:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl %esi
+; X86-I64-NEXT:    andl $-8, %esp
+; X86-I64-NEXT:    subl $16, %esp
+; X86-I64-NEXT:    movl 8(%ebp), %eax
+; X86-I64-NEXT:    fldl 20(%ebp)
+; X86-I64-NEXT:    fldl 12(%ebp)
+; X86-I64-NEXT:    fistpll (%esp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    movl (%esp), %ecx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I64-NEXT:    movl %edi, 12(%eax)
+; X86-I64-NEXT:    movl %esi, 8(%eax)
+; X86-I64-NEXT:    movl %edx, 4(%eax)
+; X86-I64-NEXT:    movl %ecx, (%eax)
+; X86-I64-NEXT:    leal -8(%ebp), %esp
+; X86-I64-NEXT:    popl %esi
+; X86-I64-NEXT:    popl %edi
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl $4
+;
 ; X86-SSE2-LABEL: lrint_v2f64:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    cvtpd2dq %xmm0, %xmm0
@@ -311,7 +649,81 @@ define <2 x iXLen> @lrint_v2f64(<2 x double> %x) {
 }
 declare <2 x iXLen> @llvm.lrint.v2iXLen.v2f64(<2 x double>)
 
-define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
+define <4 x iXLen> @lrint_v4f64(<4 x double> %x) nounwind {
+; X86-I32-LABEL: lrint_v4f64:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl %esi
+; X86-I32-NEXT:    subl $16, %esp
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl (%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    movl (%esp), %ecx
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I32-NEXT:    movl %edi, 12(%eax)
+; X86-I32-NEXT:    movl %esi, 8(%eax)
+; X86-I32-NEXT:    movl %edx, 4(%eax)
+; X86-I32-NEXT:    movl %ecx, (%eax)
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    popl %esi
+; X86-I32-NEXT:    popl %edi
+; X86-I32-NEXT:    retl $4
+;
+; X86-I64-LABEL: lrint_v4f64:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    pushl %ebx
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl %esi
+; X86-I64-NEXT:    andl $-8, %esp
+; X86-I64-NEXT:    subl $56, %esp
+; X86-I64-NEXT:    movl 8(%ebp), %eax
+; X86-I64-NEXT:    fldl 36(%ebp)
+; X86-I64-NEXT:    fldl 28(%ebp)
+; X86-I64-NEXT:    fldl 20(%ebp)
+; X86-I64-NEXT:    fldl 12(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I64-NEXT:    movl %esi, 28(%eax)
+; X86-I64-NEXT:    movl %ecx, 24(%eax)
+; X86-I64-NEXT:    movl %edx, 20(%eax)
+; X86-I64-NEXT:    movl %ebx, 16(%eax)
+; X86-I64-NEXT:    movl %edi, 12(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 8(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 4(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, (%eax)
+; X86-I64-NEXT:    leal -12(%ebp), %esp
+; X86-I64-NEXT:    popl %esi
+; X86-I64-NEXT:    popl %edi
+; X86-I64-NEXT:    popl %ebx
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl $4
+;
 ; X86-SSE2-LABEL: lrint_v4f64:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    cvtpd2dq %xmm1, %xmm1
@@ -376,14 +788,149 @@ define <4 x iXLen> @lrint_v4f64(<4 x double> %x) {
 }
 declare <4 x iXLen> @llvm.lrint.v4iXLen.v4f64(<4 x double>)
 
-define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
+define <8 x iXLen> @lrint_v8f64(<8 x double> %x) nounwind {
+; X86-I32-LABEL: lrint_v8f64:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %ebp
+; X86-I32-NEXT:    pushl %ebx
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl %esi
+; X86-I32-NEXT:    subl $40, %esp
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fldl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    fistpl {{[0-9]+}}(%esp)
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I32-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I32-NEXT:    movl %ecx, (%esp) # 4-byte Spill
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I32-NEXT:    movl %edx, 28(%eax)
+; X86-I32-NEXT:    movl %ecx, 24(%eax)
+; X86-I32-NEXT:    movl %ebp, 20(%eax)
+; X86-I32-NEXT:    movl %ebx, 16(%eax)
+; X86-I32-NEXT:    movl %edi, 12(%eax)
+; X86-I32-NEXT:    movl %esi, 8(%eax)
+; X86-I32-NEXT:    movl (%esp), %ecx # 4-byte Reload
+; X86-I32-NEXT:    movl %ecx, 4(%eax)
+; X86-I32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I32-NEXT:    movl %ecx, (%eax)
+; X86-I32-NEXT:    addl $40, %esp
+; X86-I32-NEXT:    popl %esi
+; X86-I32-NEXT:    popl %edi
+; X86-I32-NEXT:    popl %ebx
+; X86-I32-NEXT:    popl %ebp
+; X86-I32-NEXT:    retl $4
+;
+; X86-I64-LABEL: lrint_v8f64:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    pushl %ebx
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl %esi
+; X86-I64-NEXT:    andl $-8, %esp
+; X86-I64-NEXT:    subl $120, %esp
+; X86-I64-NEXT:    fldl 12(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fldl 20(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fldl 28(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fldl 36(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fldl 44(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fldl 52(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fldl 60(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    fldl 68(%ebp)
+; X86-I64-NEXT:    fistpll {{[0-9]+}}(%esp)
+; X86-I64-NEXT:    movl 8(%ebp), %eax
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-I64-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-I64-NEXT:    movl %ebx, 60(%eax)
+; X86-I64-NEXT:    movl %ecx, 56(%eax)
+; X86-I64-NEXT:    movl %edx, 52(%eax)
+; X86-I64-NEXT:    movl %esi, 48(%eax)
+; X86-I64-NEXT:    movl %edi, 44(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 40(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 36(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 32(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 28(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 24(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 20(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 16(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 12(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 8(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, 4(%eax)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-I64-NEXT:    movl %ecx, (%eax)
+; X86-I64-NEXT:    leal -12(%ebp), %esp
+; X86-I64-NEXT:    popl %esi
+; X86-I64-NEXT:    popl %edi
+; X86-I64-NEXT:    popl %ebx
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl $4
+;
 ; X86-SSE2-LABEL: lrint_v8f64:
 ; X86-SSE2:       # %bb.0:
 ; X86-SSE2-NEXT:    pushl %ebp
-; X86-SSE2-NEXT:    .cfi_def_cfa_offset 8
-; X86-SSE2-NEXT:    .cfi_offset %ebp, -8
 ; X86-SSE2-NEXT:    movl %esp, %ebp
-; X86-SSE2-NEXT:    .cfi_def_cfa_register %ebp
 ; X86-SSE2-NEXT:    andl $-16, %esp
 ; X86-SSE2-NEXT:    subl $16, %esp
 ; X86-SSE2-NEXT:    cvtpd2dq %xmm1, %xmm1
@@ -394,7 +941,6 @@ define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
 ; X86-SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
 ; X86-SSE2-NEXT:    movl %ebp, %esp
 ; X86-SSE2-NEXT:    popl %ebp
-; X86-SSE2-NEXT:    .cfi_def_cfa %esp, 4
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-AVX1-LABEL: lrint_v8f64:
@@ -490,3 +1036,1145 @@ define <8 x iXLen> @lrint_v8f64(<8 x double> %x) {
   ret <8 x iXLen> %a
 }
 declare <8 x iXLen> @llvm.lrint.v8iXLen.v8f64(<8 x double>)
+
+define <1 x iXLen> @lrint_v1fp128(<1 x fp128> %x) nounwind {
+; X86-I32-LABEL: lrint_v1fp128:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %ebp
+; X86-I32-NEXT:    movl %esp, %ebp
+; X86-I32-NEXT:    andl $-16, %esp
+; X86-I32-NEXT:    subl $16, %esp
+; X86-I32-NEXT:    pushl 20(%ebp)
+; X86-I32-NEXT:    pushl 16(%ebp)
+; X86-I32-NEXT:    pushl 12(%ebp)
+; X86-I32-NEXT:    pushl 8(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %ebp, %esp
+; X86-I32-NEXT:    popl %ebp
+; X86-I32-NEXT:    retl
+;
+; X86-I64-LABEL: lrint_v1fp128:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    andl $-16, %esp
+; X86-I64-NEXT:    subl $16, %esp
+; X86-I64-NEXT:    pushl 20(%ebp)
+; X86-I64-NEXT:    pushl 16(%ebp)
+; X86-I64-NEXT:    pushl 12(%ebp)
+; X86-I64-NEXT:    pushl 8(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %ebp, %esp
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl
+;
+; X86-SSE2-LABEL: lrint_v1fp128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $16, %esp
+; X86-SSE2-NEXT:    pushl 20(%ebp)
+; X86-SSE2-NEXT:    pushl 16(%ebp)
+; X86-SSE2-NEXT:    pushl 12(%ebp)
+; X86-SSE2-NEXT:    pushl 8(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %ebp, %esp
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: lrint_v1fp128:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-16, %esp
+; X86-AVX-NEXT:    subl $32, %esp
+; X86-AVX-NEXT:    vmovups 8(%ebp), %xmm0
+; X86-AVX-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX-NEXT:    calll lrintl
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-AVX-i32-LABEL: lrint_v1fp128:
+; X64-AVX-i32:       # %bb.0:
+; X64-AVX-i32-NEXT:    pushq %rax
+; X64-AVX-i32-NEXT:    callq lrintl@PLT
+; X64-AVX-i32-NEXT:    popq %rcx
+; X64-AVX-i32-NEXT:    retq
+;
+; X64-AVX-i64-LABEL: lrint_v1fp128:
+; X64-AVX-i64:       # %bb.0:
+; X64-AVX-i64-NEXT:    pushq %rax
+; X64-AVX-i64-NEXT:    callq lrintl@PLT
+; X64-AVX-i64-NEXT:    popq %rcx
+; X64-AVX-i64-NEXT:    retq
+  %a = call <1 x iXLen> @llvm.lrint.v1iXLen.v1fp128(<1 x fp128> %x)
+  ret <1 x iXLen> %a
+}
+declare <1 x iXLen> @llvm.lrint.v1iXLen.v1fp128(<1 x fp128>)
+
+define <2 x iXLen> @lrint_v2fp128(<2 x fp128> %x) nounwind {
+; X86-I32-LABEL: lrint_v2fp128:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %ebp
+; X86-I32-NEXT:    movl %esp, %ebp
+; X86-I32-NEXT:    pushl %ebx
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl %esi
+; X86-I32-NEXT:    andl $-16, %esp
+; X86-I32-NEXT:    subl $16, %esp
+; X86-I32-NEXT:    movl 32(%ebp), %edi
+; X86-I32-NEXT:    movl 36(%ebp), %ebx
+; X86-I32-NEXT:    pushl 20(%ebp)
+; X86-I32-NEXT:    pushl 16(%ebp)
+; X86-I32-NEXT:    pushl 12(%ebp)
+; X86-I32-NEXT:    pushl 8(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, %esi
+; X86-I32-NEXT:    pushl %ebx
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl 28(%ebp)
+; X86-I32-NEXT:    pushl 24(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, %edx
+; X86-I32-NEXT:    movl %esi, %eax
+; X86-I32-NEXT:    leal -12(%ebp), %esp
+; X86-I32-NEXT:    popl %esi
+; X86-I32-NEXT:    popl %edi
+; X86-I32-NEXT:    popl %ebx
+; X86-I32-NEXT:    popl %ebp
+; X86-I32-NEXT:    retl
+;
+; X86-I64-LABEL: lrint_v2fp128:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    pushl %ebx
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl %esi
+; X86-I64-NEXT:    andl $-16, %esp
+; X86-I64-NEXT:    subl $16, %esp
+; X86-I64-NEXT:    movl 8(%ebp), %esi
+; X86-I64-NEXT:    pushl 24(%ebp)
+; X86-I64-NEXT:    pushl 20(%ebp)
+; X86-I64-NEXT:    pushl 16(%ebp)
+; X86-I64-NEXT:    pushl 12(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, %edi
+; X86-I64-NEXT:    movl %edx, %ebx
+; X86-I64-NEXT:    pushl 40(%ebp)
+; X86-I64-NEXT:    pushl 36(%ebp)
+; X86-I64-NEXT:    pushl 32(%ebp)
+; X86-I64-NEXT:    pushl 28(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %edx, 12(%esi)
+; X86-I64-NEXT:    movl %eax, 8(%esi)
+; X86-I64-NEXT:    movl %ebx, 4(%esi)
+; X86-I64-NEXT:    movl %edi, (%esi)
+; X86-I64-NEXT:    movl %esi, %eax
+; X86-I64-NEXT:    leal -12(%ebp), %esp
+; X86-I64-NEXT:    popl %esi
+; X86-I64-NEXT:    popl %edi
+; X86-I64-NEXT:    popl %ebx
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl $4
+;
+; X86-SSE2-LABEL: lrint_v2fp128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    pushl %ebx
+; X86-SSE2-NEXT:    pushl %edi
+; X86-SSE2-NEXT:    pushl %esi
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $32, %esp
+; X86-SSE2-NEXT:    movl 12(%ebp), %edi
+; X86-SSE2-NEXT:    movl 16(%ebp), %ebx
+; X86-SSE2-NEXT:    movl 20(%ebp), %esi
+; X86-SSE2-NEXT:    pushl 36(%ebp)
+; X86-SSE2-NEXT:    pushl 32(%ebp)
+; X86-SSE2-NEXT:    pushl 28(%ebp)
+; X86-SSE2-NEXT:    pushl 24(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, (%esp) # 16-byte Spill
+; X86-SSE2-NEXT:    pushl %esi
+; X86-SSE2-NEXT:    pushl %ebx
+; X86-SSE2-NEXT:    pushl %edi
+; X86-SSE2-NEXT:    pushl 8(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    punpckldq (%esp), %xmm0 # 16-byte Folded Reload
+; X86-SSE2-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; X86-SSE2-NEXT:    leal -12(%ebp), %esp
+; X86-SSE2-NEXT:    popl %esi
+; X86-SSE2-NEXT:    popl %edi
+; X86-SSE2-NEXT:    popl %ebx
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: lrint_v2fp128:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    andl $-16, %esp
+; X86-AVX-NEXT:    subl $48, %esp
+; X86-AVX-NEXT:    vmovups 8(%ebp), %xmm0
+; X86-AVX-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX-NEXT:    calll lrintl
+; X86-AVX-NEXT:    vmovups 24(%ebp), %xmm0
+; X86-AVX-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovd %eax, %xmm0
+; X86-AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-AVX-NEXT:    calll lrintl
+; X86-AVX-NEXT:    vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-AVX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
+; X86-AVX-NEXT:    movl %ebp, %esp
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-AVX-i32-LABEL: lrint_v2fp128:
+; X64-AVX-i32:       # %bb.0:
+; X64-AVX-i32-NEXT:    pushq %rbx
+; X64-AVX-i32-NEXT:    subq $16, %rsp
+; X64-AVX-i32-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; X64-AVX-i32-NEXT:    vmovaps %xmm1, %xmm0
+; X64-AVX-i32-NEXT:    callq lrintl@PLT
+; X64-AVX-i32-NEXT:    movl %eax, %ebx
+; X64-AVX-i32-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; X64-AVX-i32-NEXT:    callq lrintl@PLT
+; X64-AVX-i32-NEXT:    vmovd %eax, %xmm0
+; X64-AVX-i32-NEXT:    vpinsrd $1, %ebx, %xmm0, %xmm0
+; X64-AVX-i32-NEXT:    addq $16, %rsp
+; X64-AVX-i32-NEXT:    popq %rbx
+; X64-AVX-i32-NEXT:    retq
+;
+; X64-AVX-i64-LABEL: lrint_v2fp128:
+; X64-AVX-i64:       # %bb.0:
+; X64-AVX-i64-NEXT:    subq $40, %rsp
+; X64-AVX-i64-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX-i64-NEXT:    vmovaps %xmm1, %xmm0
+; X64-AVX-i64-NEXT:    callq lrintl@PLT
+; X64-AVX-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX-i64-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; X64-AVX-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX-i64-NEXT:    callq lrintl@PLT
+; X64-AVX-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX-i64-NEXT:    vpunpcklqdq (%rsp), %xmm0, %xmm0 # 16-byte Folded Reload
+; X64-AVX-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; X64-AVX-i64-NEXT:    addq $40, %rsp
+; X64-AVX-i64-NEXT:    retq
+  %a = call <2 x iXLen> @llvm.lrint.v2iXLen.v2fp128(<2 x fp128> %x)
+  ret <2 x iXLen> %a
+}
+declare <2 x iXLen> @llvm.lrint.v2iXLen.v2fp128(<2 x fp128>)
+
+define <4 x iXLen> @lrint_v4fp128(<4 x fp128> %x) nounwind {
+; X86-I32-LABEL: lrint_v4fp128:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %ebp
+; X86-I32-NEXT:    movl %esp, %ebp
+; X86-I32-NEXT:    pushl %ebx
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl %esi
+; X86-I32-NEXT:    andl $-16, %esp
+; X86-I32-NEXT:    subl $16, %esp
+; X86-I32-NEXT:    movl 8(%ebp), %esi
+; X86-I32-NEXT:    movl 36(%ebp), %ebx
+; X86-I32-NEXT:    movl 40(%ebp), %edi
+; X86-I32-NEXT:    pushl 24(%ebp)
+; X86-I32-NEXT:    pushl 20(%ebp)
+; X86-I32-NEXT:    pushl 16(%ebp)
+; X86-I32-NEXT:    pushl 12(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl %ebx
+; X86-I32-NEXT:    pushl 32(%ebp)
+; X86-I32-NEXT:    pushl 28(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, %ebx
+; X86-I32-NEXT:    pushl 56(%ebp)
+; X86-I32-NEXT:    pushl 52(%ebp)
+; X86-I32-NEXT:    pushl 48(%ebp)
+; X86-I32-NEXT:    pushl 44(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, %edi
+; X86-I32-NEXT:    pushl 72(%ebp)
+; X86-I32-NEXT:    pushl 68(%ebp)
+; X86-I32-NEXT:    pushl 64(%ebp)
+; X86-I32-NEXT:    pushl 60(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, 12(%esi)
+; X86-I32-NEXT:    movl %edi, 8(%esi)
+; X86-I32-NEXT:    movl %ebx, 4(%esi)
+; X86-I32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I32-NEXT:    movl %eax, (%esi)
+; X86-I32-NEXT:    movl %esi, %eax
+; X86-I32-NEXT:    leal -12(%ebp), %esp
+; X86-I32-NEXT:    popl %esi
+; X86-I32-NEXT:    popl %edi
+; X86-I32-NEXT:    popl %ebx
+; X86-I32-NEXT:    popl %ebp
+; X86-I32-NEXT:    retl $4
+;
+; X86-I64-LABEL: lrint_v4fp128:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    pushl %ebx
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl %esi
+; X86-I64-NEXT:    andl $-16, %esp
+; X86-I64-NEXT:    subl $32, %esp
+; X86-I64-NEXT:    movl 8(%ebp), %esi
+; X86-I64-NEXT:    movl 36(%ebp), %edi
+; X86-I64-NEXT:    movl 40(%ebp), %ebx
+; X86-I64-NEXT:    pushl 24(%ebp)
+; X86-I64-NEXT:    pushl 20(%ebp)
+; X86-I64-NEXT:    pushl 16(%ebp)
+; X86-I64-NEXT:    pushl 12(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    pushl %ebx
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl 32(%ebp)
+; X86-I64-NEXT:    pushl 28(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    pushl 56(%ebp)
+; X86-I64-NEXT:    pushl 52(%ebp)
+; X86-I64-NEXT:    pushl 48(%ebp)
+; X86-I64-NEXT:    pushl 44(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, %edi
+; X86-I64-NEXT:    movl %edx, %ebx
+; X86-I64-NEXT:    pushl 72(%ebp)
+; X86-I64-NEXT:    pushl 68(%ebp)
+; X86-I64-NEXT:    pushl 64(%ebp)
+; X86-I64-NEXT:    pushl 60(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %edx, 28(%esi)
+; X86-I64-NEXT:    movl %eax, 24(%esi)
+; X86-I64-NEXT:    movl %ebx, 20(%esi)
+; X86-I64-NEXT:    movl %edi, 16(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 12(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 8(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 4(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, (%esi)
+; X86-I64-NEXT:    movl %esi, %eax
+; X86-I64-NEXT:    leal -12(%ebp), %esp
+; X86-I64-NEXT:    popl %esi
+; X86-I64-NEXT:    popl %edi
+; X86-I64-NEXT:    popl %ebx
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl $4
+;
+; X86-SSE2-LABEL: lrint_v4fp128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    pushl %ebx
+; X86-SSE2-NEXT:    pushl %edi
+; X86-SSE2-NEXT:    pushl %esi
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $48, %esp
+; X86-SSE2-NEXT:    movl 48(%ebp), %edi
+; X86-SSE2-NEXT:    movl 52(%ebp), %ebx
+; X86-SSE2-NEXT:    pushl 36(%ebp)
+; X86-SSE2-NEXT:    pushl 32(%ebp)
+; X86-SSE2-NEXT:    pushl 28(%ebp)
+; X86-SSE2-NEXT:    pushl 24(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %eax, %esi
+; X86-SSE2-NEXT:    pushl %ebx
+; X86-SSE2-NEXT:    pushl %edi
+; X86-SSE2-NEXT:    pushl 44(%ebp)
+; X86-SSE2-NEXT:    pushl 40(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %eax, %edi
+; X86-SSE2-NEXT:    pushl 68(%ebp)
+; X86-SSE2-NEXT:    pushl 64(%ebp)
+; X86-SSE2-NEXT:    pushl 60(%ebp)
+; X86-SSE2-NEXT:    pushl 56(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    movd %edi, %xmm1
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE2-NEXT:    movdqa %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-SSE2-NEXT:    movd %esi, %xmm0
+; X86-SSE2-NEXT:    movdqa %xmm0, (%esp) # 16-byte Spill
+; X86-SSE2-NEXT:    pushl 20(%ebp)
+; X86-SSE2-NEXT:    pushl 16(%ebp)
+; X86-SSE2-NEXT:    pushl 12(%ebp)
+; X86-SSE2-NEXT:    pushl 8(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    punpckldq (%esp), %xmm0 # 16-byte Folded Reload
+; X86-SSE2-NEXT:    # xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; X86-SSE2-NEXT:    punpcklqdq {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; X86-SSE2-NEXT:    # xmm0 = xmm0[0],mem[0]
+; X86-SSE2-NEXT:    leal -12(%ebp), %esp
+; X86-SSE2-NEXT:    popl %esi
+; X86-SSE2-NEXT:    popl %edi
+; X86-SSE2-NEXT:    popl %ebx
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX-LABEL: lrint_v4fp128:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    pushl %ebp
+; X86-AVX-NEXT:    movl %esp, %ebp
+; X86-AVX-NEXT:    pushl %edi
+; X86-AVX-NEXT:    pushl %esi
+; X86-AVX-NEXT:    andl $-16, %esp
+; X86-AVX-NEXT:    subl $32, %esp
+; X86-AVX-NEXT:    vmovups 40(%ebp), %xmm0
+; X86-AVX-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX-NEXT:    calll lrintl
+; X86-AVX-NEXT:    movl %eax, %esi
+; X86-AVX-NEXT:    vmovups 24(%ebp), %xmm0
+; X86-AVX-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX-NEXT:    calll lrintl
+; X86-AVX-NEXT:    movl %eax, %edi
+; X86-AVX-NEXT:    vmovups 8(%ebp), %xmm0
+; X86-AVX-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX-NEXT:    calll lrintl
+; X86-AVX-NEXT:    vmovups 56(%ebp), %xmm0
+; X86-AVX-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX-NEXT:    vmovd %eax, %xmm0
+; X86-AVX-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpinsrd $2, %esi, %xmm0, %xmm0
+; X86-AVX-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-AVX-NEXT:    calll lrintl
+; X86-AVX-NEXT:    vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-AVX-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X86-AVX-NEXT:    leal -8(%ebp), %esp
+; X86-AVX-NEXT:    popl %esi
+; X86-AVX-NEXT:    popl %edi
+; X86-AVX-NEXT:    popl %ebp
+; X86-AVX-NEXT:    retl
+;
+; X64-AVX-i32-LABEL: lrint_v4fp128:
+; X64-AVX-i32:       # %bb.0:
+; X64-AVX-i32-NEXT:    pushq %rbx
+; X64-AVX-i32-NEXT:    subq $48, %rsp
+; X64-AVX-i32-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX-i32-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX-i32-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; X64-AVX-i32-NEXT:    vmovaps %xmm1, %xmm0
+; X64-AVX-i32-NEXT:    callq lrintl@PLT
+; X64-AVX-i32-NEXT:    movl %eax, %ebx
+; X64-AVX-i32-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; X64-AVX-i32-NEXT:    callq lrintl@PLT
+; X64-AVX-i32-NEXT:    vmovd %eax, %xmm0
+; X64-AVX-i32-NEXT:    vpinsrd $1, %ebx, %xmm0, %xmm0
+; X64-AVX-i32-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; X64-AVX-i32-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX-i32-NEXT:    callq lrintl@PLT
+; X64-AVX-i32-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
+; X64-AVX-i32-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
+; X64-AVX-i32-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX-i32-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX-i32-NEXT:    callq lrintl@PLT
+; X64-AVX-i32-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX-i32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X64-AVX-i32-NEXT:    addq $48, %rsp
+; X64-AVX-i32-NEXT:    popq %rbx
+; X64-AVX-i32-NEXT:    retq
+;
+; X64-AVX1-i64-LABEL: lrint_v4fp128:
+; X64-AVX1-i64:       # %bb.0:
+; X64-AVX1-i64-NEXT:    subq $72, %rsp
+; X64-AVX1-i64-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm3, %xmm0
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; X64-AVX1-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; X64-AVX1-i64-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; X64-AVX1-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; X64-AVX1-i64-NEXT:    vinsertf128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; X64-AVX1-i64-NEXT:    addq $72, %rsp
+; X64-AVX1-i64-NEXT:    retq
+;
+; AVX512-i64-LABEL: lrint_v4fp128:
+; AVX512-i64:       # %bb.0:
+; AVX512-i64-NEXT:    subq $72, %rsp
+; AVX512-i64-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm3, %xmm0
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-i64-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-i64-NEXT:    vinserti128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512-i64-NEXT:    addq $72, %rsp
+; AVX512-i64-NEXT:    retq
+;
+; AVX512DQ-i64-LABEL: lrint_v4fp128:
+; AVX512DQ-i64:       # %bb.0:
+; AVX512DQ-i64-NEXT:    subq $72, %rsp
+; AVX512DQ-i64-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm3, %xmm0
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-i64-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-i64-NEXT:    vinserti128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512DQ-i64-NEXT:    addq $72, %rsp
+; AVX512DQ-i64-NEXT:    retq
+  %a = call <4 x iXLen> @llvm.lrint.v4iXLen.v4fp128(<4 x fp128> %x)
+  ret <4 x iXLen> %a
+}
+declare <4 x iXLen> @llvm.lrint.v4iXLen.v4fp128(<4 x fp128>)
+
+define <8 x iXLen> @lrint_v8fp128(<8 x fp128> %x) nounwind {
+; X86-I32-LABEL: lrint_v8fp128:
+; X86-I32:       # %bb.0:
+; X86-I32-NEXT:    pushl %ebp
+; X86-I32-NEXT:    movl %esp, %ebp
+; X86-I32-NEXT:    pushl %ebx
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl %esi
+; X86-I32-NEXT:    andl $-16, %esp
+; X86-I32-NEXT:    subl $32, %esp
+; X86-I32-NEXT:    movl 8(%ebp), %esi
+; X86-I32-NEXT:    movl 36(%ebp), %ebx
+; X86-I32-NEXT:    movl 40(%ebp), %edi
+; X86-I32-NEXT:    pushl 24(%ebp)
+; X86-I32-NEXT:    pushl 20(%ebp)
+; X86-I32-NEXT:    pushl 16(%ebp)
+; X86-I32-NEXT:    pushl 12(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I32-NEXT:    pushl %edi
+; X86-I32-NEXT:    pushl %ebx
+; X86-I32-NEXT:    pushl 32(%ebp)
+; X86-I32-NEXT:    pushl 28(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I32-NEXT:    pushl 56(%ebp)
+; X86-I32-NEXT:    pushl 52(%ebp)
+; X86-I32-NEXT:    pushl 48(%ebp)
+; X86-I32-NEXT:    pushl 44(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I32-NEXT:    pushl 72(%ebp)
+; X86-I32-NEXT:    pushl 68(%ebp)
+; X86-I32-NEXT:    pushl 64(%ebp)
+; X86-I32-NEXT:    pushl 60(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I32-NEXT:    pushl 88(%ebp)
+; X86-I32-NEXT:    pushl 84(%ebp)
+; X86-I32-NEXT:    pushl 80(%ebp)
+; X86-I32-NEXT:    pushl 76(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I32-NEXT:    pushl 104(%ebp)
+; X86-I32-NEXT:    pushl 100(%ebp)
+; X86-I32-NEXT:    pushl 96(%ebp)
+; X86-I32-NEXT:    pushl 92(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, %ebx
+; X86-I32-NEXT:    pushl 120(%ebp)
+; X86-I32-NEXT:    pushl 116(%ebp)
+; X86-I32-NEXT:    pushl 112(%ebp)
+; X86-I32-NEXT:    pushl 108(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, %edi
+; X86-I32-NEXT:    pushl 136(%ebp)
+; X86-I32-NEXT:    pushl 132(%ebp)
+; X86-I32-NEXT:    pushl 128(%ebp)
+; X86-I32-NEXT:    pushl 124(%ebp)
+; X86-I32-NEXT:    calll lrintl
+; X86-I32-NEXT:    addl $16, %esp
+; X86-I32-NEXT:    movl %eax, 28(%esi)
+; X86-I32-NEXT:    movl %edi, 24(%esi)
+; X86-I32-NEXT:    movl %ebx, 20(%esi)
+; X86-I32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I32-NEXT:    movl %eax, 16(%esi)
+; X86-I32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I32-NEXT:    movl %eax, 12(%esi)
+; X86-I32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I32-NEXT:    movl %eax, 8(%esi)
+; X86-I32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I32-NEXT:    movl %eax, 4(%esi)
+; X86-I32-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I32-NEXT:    movl %eax, (%esi)
+; X86-I32-NEXT:    movl %esi, %eax
+; X86-I32-NEXT:    leal -12(%ebp), %esp
+; X86-I32-NEXT:    popl %esi
+; X86-I32-NEXT:    popl %edi
+; X86-I32-NEXT:    popl %ebx
+; X86-I32-NEXT:    popl %ebp
+; X86-I32-NEXT:    retl $4
+;
+; X86-I64-LABEL: lrint_v8fp128:
+; X86-I64:       # %bb.0:
+; X86-I64-NEXT:    pushl %ebp
+; X86-I64-NEXT:    movl %esp, %ebp
+; X86-I64-NEXT:    pushl %ebx
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl %esi
+; X86-I64-NEXT:    andl $-16, %esp
+; X86-I64-NEXT:    subl $64, %esp
+; X86-I64-NEXT:    movl 8(%ebp), %esi
+; X86-I64-NEXT:    movl 36(%ebp), %edi
+; X86-I64-NEXT:    movl 40(%ebp), %ebx
+; X86-I64-NEXT:    pushl 24(%ebp)
+; X86-I64-NEXT:    pushl 20(%ebp)
+; X86-I64-NEXT:    pushl 16(%ebp)
+; X86-I64-NEXT:    pushl 12(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    pushl %ebx
+; X86-I64-NEXT:    pushl %edi
+; X86-I64-NEXT:    pushl 32(%ebp)
+; X86-I64-NEXT:    pushl 28(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    pushl 56(%ebp)
+; X86-I64-NEXT:    pushl 52(%ebp)
+; X86-I64-NEXT:    pushl 48(%ebp)
+; X86-I64-NEXT:    pushl 44(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    pushl 72(%ebp)
+; X86-I64-NEXT:    pushl 68(%ebp)
+; X86-I64-NEXT:    pushl 64(%ebp)
+; X86-I64-NEXT:    pushl 60(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    pushl 88(%ebp)
+; X86-I64-NEXT:    pushl 84(%ebp)
+; X86-I64-NEXT:    pushl 80(%ebp)
+; X86-I64-NEXT:    pushl 76(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    pushl 104(%ebp)
+; X86-I64-NEXT:    pushl 100(%ebp)
+; X86-I64-NEXT:    pushl 96(%ebp)
+; X86-I64-NEXT:    pushl 92(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-I64-NEXT:    pushl 120(%ebp)
+; X86-I64-NEXT:    pushl 116(%ebp)
+; X86-I64-NEXT:    pushl 112(%ebp)
+; X86-I64-NEXT:    pushl 108(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %eax, %edi
+; X86-I64-NEXT:    movl %edx, %ebx
+; X86-I64-NEXT:    pushl 136(%ebp)
+; X86-I64-NEXT:    pushl 132(%ebp)
+; X86-I64-NEXT:    pushl 128(%ebp)
+; X86-I64-NEXT:    pushl 124(%ebp)
+; X86-I64-NEXT:    calll lrintl
+; X86-I64-NEXT:    addl $16, %esp
+; X86-I64-NEXT:    movl %edx, 60(%esi)
+; X86-I64-NEXT:    movl %eax, 56(%esi)
+; X86-I64-NEXT:    movl %ebx, 52(%esi)
+; X86-I64-NEXT:    movl %edi, 48(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 44(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 40(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 36(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 32(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 28(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 24(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 20(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 16(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 12(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 8(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, 4(%esi)
+; X86-I64-NEXT:    movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-I64-NEXT:    movl %eax, (%esi)
+; X86-I64-NEXT:    movl %esi, %eax
+; X86-I64-NEXT:    leal -12(%ebp), %esp
+; X86-I64-NEXT:    popl %esi
+; X86-I64-NEXT:    popl %edi
+; X86-I64-NEXT:    popl %ebx
+; X86-I64-NEXT:    popl %ebp
+; X86-I64-NEXT:    retl $4
+;
+; X86-SSE2-LABEL: lrint_v8fp128:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    pushl %ebp
+; X86-SSE2-NEXT:    movl %esp, %ebp
+; X86-SSE2-NEXT:    pushl %ebx
+; X86-SSE2-NEXT:    pushl %edi
+; X86-SSE2-NEXT:    pushl %esi
+; X86-SSE2-NEXT:    andl $-16, %esp
+; X86-SSE2-NEXT:    subl $64, %esp
+; X86-SSE2-NEXT:    movl 108(%ebp), %esi
+; X86-SSE2-NEXT:    movl 112(%ebp), %edi
+; X86-SSE2-NEXT:    movl 116(%ebp), %ebx
+; X86-SSE2-NEXT:    pushl 100(%ebp)
+; X86-SSE2-NEXT:    pushl 96(%ebp)
+; X86-SSE2-NEXT:    pushl 92(%ebp)
+; X86-SSE2-NEXT:    pushl 88(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-SSE2-NEXT:    pushl %ebx
+; X86-SSE2-NEXT:    pushl %edi
+; X86-SSE2-NEXT:    pushl %esi
+; X86-SSE2-NEXT:    pushl 104(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-SSE2-NEXT:    pushl 132(%ebp)
+; X86-SSE2-NEXT:    pushl 128(%ebp)
+; X86-SSE2-NEXT:    pushl 124(%ebp)
+; X86-SSE2-NEXT:    pushl 120(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-SSE2-NEXT:    pushl 20(%ebp)
+; X86-SSE2-NEXT:    pushl 16(%ebp)
+; X86-SSE2-NEXT:    pushl 12(%ebp)
+; X86-SSE2-NEXT:    pushl 8(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %eax, %esi
+; X86-SSE2-NEXT:    pushl 36(%ebp)
+; X86-SSE2-NEXT:    pushl 32(%ebp)
+; X86-SSE2-NEXT:    pushl 28(%ebp)
+; X86-SSE2-NEXT:    pushl 24(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %eax, %edi
+; X86-SSE2-NEXT:    pushl 52(%ebp)
+; X86-SSE2-NEXT:    pushl 48(%ebp)
+; X86-SSE2-NEXT:    pushl 44(%ebp)
+; X86-SSE2-NEXT:    pushl 40(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movl %eax, %ebx
+; X86-SSE2-NEXT:    pushl 68(%ebp)
+; X86-SSE2-NEXT:    pushl 64(%ebp)
+; X86-SSE2-NEXT:    pushl 60(%ebp)
+; X86-SSE2-NEXT:    pushl 56(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movd %eax, %xmm0
+; X86-SSE2-NEXT:    movd %ebx, %xmm1
+; X86-SSE2-NEXT:    movd %edi, %xmm2
+; X86-SSE2-NEXT:    movd %esi, %xmm4
+; X86-SSE2-NEXT:    movss (%esp), %xmm3 # 4-byte Reload
+; X86-SSE2-NEXT:    # xmm3 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm5 # 4-byte Reload
+; X86-SSE2-NEXT:    # xmm5 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    movss {{[-0-9]+}}(%e{{[sb]}}p), %xmm6 # 4-byte Reload
+; X86-SSE2-NEXT:    # xmm6 = mem[0],zero,zero,zero
+; X86-SSE2-NEXT:    movaps %xmm6, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; X86-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; X86-SSE2-NEXT:    movdqa %xmm4, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-SSE2-NEXT:    unpcklps {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; X86-SSE2-NEXT:    movaps %xmm5, (%esp) # 16-byte Spill
+; X86-SSE2-NEXT:    pushl 84(%ebp)
+; X86-SSE2-NEXT:    pushl 80(%ebp)
+; X86-SSE2-NEXT:    pushl 76(%ebp)
+; X86-SSE2-NEXT:    pushl 72(%ebp)
+; X86-SSE2-NEXT:    calll lrintl
+; X86-SSE2-NEXT:    addl $16, %esp
+; X86-SSE2-NEXT:    movd %eax, %xmm1
+; X86-SSE2-NEXT:    punpckldq {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; X86-SSE2-NEXT:    # xmm1 = xmm1[0],mem[0],xmm1[1],mem[1]
+; X86-SSE2-NEXT:    punpcklqdq (%esp), %xmm1 # 16-byte Folded Reload
+; X86-SSE2-NEXT:    # xmm1 = xmm1[0],mem[0]
+; X86-SSE2-NEXT:    movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-SSE2-NEXT:    leal -12(%ebp), %esp
+; X86-SSE2-NEXT:    popl %esi
+; X86-SSE2-NEXT:    popl %edi
+; X86-SSE2-NEXT:    popl %ebx
+; X86-SSE2-NEXT:    popl %ebp
+; X86-SSE2-NEXT:    retl
+;
+; X86-AVX1-LABEL: lrint_v8fp128:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    pushl %ebp
+; X86-AVX1-NEXT:    movl %esp, %ebp
+; X86-AVX1-NEXT:    pushl %ebx
+; X86-AVX1-NEXT:    pushl %edi
+; X86-AVX1-NEXT:    pushl %esi
+; X86-AVX1-NEXT:    andl $-16, %esp
+; X86-AVX1-NEXT:    subl $80, %esp
+; X86-AVX1-NEXT:    vmovups 40(%ebp), %xmm0
+; X86-AVX1-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX1-NEXT:    calll lrintl
+; X86-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-AVX1-NEXT:    vmovups 24(%ebp), %xmm0
+; X86-AVX1-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX1-NEXT:    calll lrintl
+; X86-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-AVX1-NEXT:    vmovups 8(%ebp), %xmm0
+; X86-AVX1-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX1-NEXT:    calll lrintl
+; X86-AVX1-NEXT:    movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-AVX1-NEXT:    vmovups 120(%ebp), %xmm0
+; X86-AVX1-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX1-NEXT:    calll lrintl
+; X86-AVX1-NEXT:    movl %eax, %esi
+; X86-AVX1-NEXT:    vmovups 104(%ebp), %xmm0
+; X86-AVX1-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX1-NEXT:    calll lrintl
+; X86-AVX1-NEXT:    movl %eax, %edi
+; X86-AVX1-NEXT:    vmovups 88(%ebp), %xmm0
+; X86-AVX1-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX1-NEXT:    calll lrintl
+; X86-AVX1-NEXT:    movl %eax, %ebx
+; X86-AVX1-NEXT:    vmovups 72(%ebp), %xmm0
+; X86-AVX1-NEXT:    vmovups %xmm0, (%esp)
+; X86-AVX1-NEXT:    calll lrintl
+; X86-AVX1-NEXT:    vmovd %eax, %xmm0
+; X86-AVX1-NEXT:    vpinsrd $1, %ebx, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpinsrd $2, %edi, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpinsrd $3, %esi, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-AVX1-NEXT:    vmovd {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 4-byte Folded Reload
+; X86-AVX1-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; X86-AVX1-NEXT:    vpinsrd $1, {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; X86-AVX1-NEXT:    vmovups 56(%ebp), %xmm1
+; X86-AVX1-NEXT:    vmovups %xmm1, (%esp)
+; X86-AVX1-NEXT:    vpinsrd $2, {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm0 # 4-byte Folded Reload
+; X86-AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-AVX1-NEXT:    calll lrintl
+; X86-AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-AVX1-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%e{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; X86-AVX1-NEXT:    leal -12(%ebp), %esp
+; X86-AVX1-NEXT:    popl %esi
+; X86-AVX1-NEXT:    popl %edi
+; X86-AVX1-NEXT:    popl %ebx
+; X86-AVX1-NEXT:    popl %ebp
+; X86-AVX1-NEXT:    retl
+;
+; X64-AVX1-i32-LABEL: lrint_v8fp128:
+; X64-AVX1-i32:       # %bb.0:
+; X64-AVX1-i32-NEXT:    pushq %rbx
+; X64-AVX1-i32-NEXT:    subq $112, %rsp
+; X64-AVX1-i32-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps %xmm4, (%rsp) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps %xmm5, %xmm0
+; X64-AVX1-i32-NEXT:    callq lrintl@PLT
+; X64-AVX1-i32-NEXT:    movl %eax, %ebx
+; X64-AVX1-i32-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    callq lrintl@PLT
+; X64-AVX1-i32-NEXT:    vmovd %eax, %xmm0
+; X64-AVX1-i32-NEXT:    vpinsrd $1, %ebx, %xmm0, %xmm0
+; X64-AVX1-i32-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    callq lrintl@PLT
+; X64-AVX1-i32-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
+; X64-AVX1-i32-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    callq lrintl@PLT
+; X64-AVX1-i32-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X64-AVX1-i32-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    callq lrintl@PLT
+; X64-AVX1-i32-NEXT:    movl %eax, %ebx
+; X64-AVX1-i32-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    callq lrintl@PLT
+; X64-AVX1-i32-NEXT:    vmovd %eax, %xmm0
+; X64-AVX1-i32-NEXT:    vpinsrd $1, %ebx, %xmm0, %xmm0
+; X64-AVX1-i32-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    callq lrintl@PLT
+; X64-AVX1-i32-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    vpinsrd $2, %eax, %xmm0, %xmm0
+; X64-AVX1-i32-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i32-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    callq lrintl@PLT
+; X64-AVX1-i32-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X64-AVX1-i32-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; X64-AVX1-i32-NEXT:    addq $112, %rsp
+; X64-AVX1-i32-NEXT:    popq %rbx
+; X64-AVX1-i32-NEXT:    retq
+;
+; X64-AVX1-i64-LABEL: lrint_v8fp128:
+; X64-AVX1-i64:       # %bb.0:
+; X64-AVX1-i64-NEXT:    subq $152, %rsp
+; X64-AVX1-i64-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps %xmm3, %xmm0
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; X64-AVX1-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; X64-AVX1-i64-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; X64-AVX1-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; X64-AVX1-i64-NEXT:    vinsertf128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; X64-AVX1-i64-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    vzeroupper
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; X64-AVX1-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; X64-AVX1-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; X64-AVX1-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; X64-AVX1-i64-NEXT:    callq lrintl@PLT
+; X64-AVX1-i64-NEXT:    vmovq %rax, %xmm0
+; X64-AVX1-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; X64-AVX1-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; X64-AVX1-i64-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
+; X64-AVX1-i64-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; X64-AVX1-i64-NEXT:    addq $152, %rsp
+; X64-AVX1-i64-NEXT:    retq
+;
+; AVX512-i64-LABEL: lrint_v8fp128:
+; AVX512-i64:       # %bb.0:
+; AVX512-i64-NEXT:    subq $152, %rsp
+; AVX512-i64-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps %xmm7, %xmm0
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-i64-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-i64-NEXT:    vinserti128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512-i64-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    vzeroupper
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512-i64-NEXT:    callq lrintl@PLT
+; AVX512-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512-i64-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512-i64-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512-i64-NEXT:    addq $152, %rsp
+; AVX512-i64-NEXT:    retq
+;
+; AVX512DQ-i64-LABEL: lrint_v8fp128:
+; AVX512DQ-i64:       # %bb.0:
+; AVX512DQ-i64-NEXT:    subq $152, %rsp
+; AVX512DQ-i64-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps %xmm7, %xmm0
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps (%rsp), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-i64-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-i64-NEXT:    vinserti128 $1, (%rsp), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512DQ-i64-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    vzeroupper
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-i64-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512DQ-i64-NEXT:    callq lrintl@PLT
+; AVX512DQ-i64-NEXT:    vmovq %rax, %xmm0
+; AVX512DQ-i64-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; AVX512DQ-i64-NEXT:    # xmm0 = xmm0[0],mem[0]
+; AVX512DQ-i64-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 16-byte Folded Reload
+; AVX512DQ-i64-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 32-byte Folded Reload
+; AVX512DQ-i64-NEXT:    addq $152, %rsp
+; AVX512DQ-i64-NEXT:    retq
+  %a = call <8 x iXLen> @llvm.lrint.v8iXLen.v8fp128(<8 x fp128> %x)
+  ret <8 x iXLen> %a
+}
+declare <8 x iXLen> @llvm.lrint.v8iXLen.v8fp128(<8 x fp128>)
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
index 9cd0f4d12e15a..227e000c6be7f 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-cmp.ll
@@ -903,6 +903,95 @@ define i1 @mask_v8i32(<8 x i32> %a0) {
   ret i1 %3
 }
 
+define i1 @mask_v8i32_2(<8 x i32> %a0) {
+; SSE2-LABEL: mask_v8i32_2:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pslld $1, %xmm0
+; SSE2-NEXT:    movmskps %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    sete %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: mask_v8i32_2:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    ptest {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: mask_v8i32_2:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vptest {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0
+; AVX1-NEXT:    sete %al
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: mask_v8i32_2:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4611686019501129728,4611686019501129728,4611686019501129728,4611686019501129728]
+; AVX2-NEXT:    vptest %ymm1, %ymm0
+; AVX2-NEXT:    sete %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: mask_v8i32_2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4611686019501129728,4611686019501129728,4611686019501129728,4611686019501129728]
+; AVX512-NEXT:    vptest %ymm1, %ymm0
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %1 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %a0)
+  %2 = and i32 %1, 1073741824
+  %3 = icmp eq i32 %2, 0
+  ret i1 %3
+}
+
+
+define i1 @signtest_v8i32(<8 x i32> %a0) {
+; SSE2-LABEL: signtest_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    orps %xmm1, %xmm0
+; SSE2-NEXT:    movmskps %xmm0, %eax
+; SSE2-NEXT:    testl %eax, %eax
+; SSE2-NEXT:    sete %al
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: signtest_v8i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    ptest {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT:    sete %al
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: signtest_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vptest {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0
+; AVX1-NEXT:    sete %al
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: signtest_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372039002259456,9223372039002259456,9223372039002259456,9223372039002259456]
+; AVX2-NEXT:    vptest %ymm1, %ymm0
+; AVX2-NEXT:    sete %al
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: signtest_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372039002259456,9223372039002259456,9223372039002259456,9223372039002259456]
+; AVX512-NEXT:    vptest %ymm1, %ymm0
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %1 = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> %a0)
+  %2 = icmp sgt i32 %1, -1
+  ret i1 %2
+}
+
 define i1 @trunc_v16i16(<16 x i16> %a0) {
 ; SSE2-LABEL: trunc_v16i16:
 ; SSE2:       # %bb.0:
@@ -1073,11 +1162,11 @@ define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
 ; SSE2-NEXT:    movd %xmm0, %eax
 ; SSE2-NEXT:    orl %ecx, %eax
 ; SSE2-NEXT:    testb $1, %al
-; SSE2-NEXT:    je .LBB27_2
+; SSE2-NEXT:    je .LBB29_2
 ; SSE2-NEXT:  # %bb.1:
 ; SSE2-NEXT:    xorl %eax, %eax
 ; SSE2-NEXT:    retq
-; SSE2-NEXT:  .LBB27_2:
+; SSE2-NEXT:  .LBB29_2:
 ; SSE2-NEXT:    movl $1, %eax
 ; SSE2-NEXT:    retq
 ;
@@ -1092,11 +1181,11 @@ define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
 ; SSE41-NEXT:    pextrd $2, %xmm1, %eax
 ; SSE41-NEXT:    orl %ecx, %eax
 ; SSE41-NEXT:    testb $1, %al
-; SSE41-NEXT:    je .LBB27_2
+; SSE41-NEXT:    je .LBB29_2
 ; SSE41-NEXT:  # %bb.1:
 ; SSE41-NEXT:    xorl %eax, %eax
 ; SSE41-NEXT:    retq
-; SSE41-NEXT:  .LBB27_2:
+; SSE41-NEXT:  .LBB29_2:
 ; SSE41-NEXT:    movl $1, %eax
 ; SSE41-NEXT:    retq
 ;
@@ -1111,11 +1200,11 @@ define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
 ; AVX1OR2-NEXT:    vpextrd $2, %xmm0, %eax
 ; AVX1OR2-NEXT:    orl %ecx, %eax
 ; AVX1OR2-NEXT:    testb $1, %al
-; AVX1OR2-NEXT:    je .LBB27_2
+; AVX1OR2-NEXT:    je .LBB29_2
 ; AVX1OR2-NEXT:  # %bb.1:
 ; AVX1OR2-NEXT:    xorl %eax, %eax
 ; AVX1OR2-NEXT:    retq
-; AVX1OR2-NEXT:  .LBB27_2:
+; AVX1OR2-NEXT:  .LBB29_2:
 ; AVX1OR2-NEXT:    movl $1, %eax
 ; AVX1OR2-NEXT:    retq
 ;
@@ -1130,12 +1219,12 @@ define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
 ; AVX512F-NEXT:    korw %k0, %k1, %k0
 ; AVX512F-NEXT:    kmovw %k0, %eax
 ; AVX512F-NEXT:    testb $1, %al
-; AVX512F-NEXT:    je .LBB27_2
+; AVX512F-NEXT:    je .LBB29_2
 ; AVX512F-NEXT:  # %bb.1:
 ; AVX512F-NEXT:    xorl %eax, %eax
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
-; AVX512F-NEXT:  .LBB27_2:
+; AVX512F-NEXT:  .LBB29_2:
 ; AVX512F-NEXT:    movl $1, %eax
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1151,12 +1240,12 @@ define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
 ; AVX512BW-NEXT:    korw %k0, %k1, %k0
 ; AVX512BW-NEXT:    kmovd %k0, %eax
 ; AVX512BW-NEXT:    testb $1, %al
-; AVX512BW-NEXT:    je .LBB27_2
+; AVX512BW-NEXT:    je .LBB29_2
 ; AVX512BW-NEXT:  # %bb.1:
 ; AVX512BW-NEXT:    xorl %eax, %eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
-; AVX512BW-NEXT:  .LBB27_2:
+; AVX512BW-NEXT:  .LBB29_2:
 ; AVX512BW-NEXT:    movl $1, %eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1170,11 +1259,11 @@ define i32 @mask_v3i1(<3 x i32> %a, <3 x i32> %b) {
 ; AVX512BWVL-NEXT:    korw %k0, %k1, %k0
 ; AVX512BWVL-NEXT:    kmovd %k0, %eax
 ; AVX512BWVL-NEXT:    testb $1, %al
-; AVX512BWVL-NEXT:    je .LBB27_2
+; AVX512BWVL-NEXT:    je .LBB29_2
 ; AVX512BWVL-NEXT:  # %bb.1:
 ; AVX512BWVL-NEXT:    xorl %eax, %eax
 ; AVX512BWVL-NEXT:    retq
-; AVX512BWVL-NEXT:  .LBB27_2:
+; AVX512BWVL-NEXT:  .LBB29_2:
 ; AVX512BWVL-NEXT:    movl $1, %eax
 ; AVX512BWVL-NEXT:    retq
   %1 = icmp ne <3 x i32> %a, %b
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
index 87c135ddcec95..ef20cf2a09bba 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v64.ll
@@ -1724,6 +1724,269 @@ define void @PR54562_mem(ptr %src, ptr %dst) {
   ret void
 }
 
+define <512 x i8> @PR153457(<512 x i8> %a0, <512 x i8> %a1) nounwind {
+; AVX512F-LABEL: PR153457:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    pushq %rbp
+; AVX512F-NEXT:    movq %rsp, %rbp
+; AVX512F-NEXT:    andq $-64, %rsp
+; AVX512F-NEXT:    subq $64, %rsp
+; AVX512F-NEXT:    movq %rdi, %rax
+; AVX512F-NEXT:    vpbroadcastq %xmm0, %ymm7
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [u,u,u,u,u,u,u,u,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0]
+; AVX512F-NEXT:    vpblendvb %ymm8, %ymm6, %ymm7, %ymm6
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm2, %ymm7
+; AVX512F-NEXT:    vpbroadcastd %xmm0, %ymm9
+; AVX512F-NEXT:    vpblendvb %ymm8, %ymm7, %ymm9, %ymm8
+; AVX512F-NEXT:    vextracti32x4 $2, %zmm5, %xmm7
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[0,2,4,6,8,10,12,13,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vextracti32x4 $2, %zmm4, %xmm9
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm10 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,128]
+; AVX512F-NEXT:    vpshufb %xmm10, %xmm9, %xmm9
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[5]
+; AVX512F-NEXT:    vpor %xmm11, %xmm9, %xmm9
+; AVX512F-NEXT:    vpshufb %xmm10, %xmm1, %xmm10
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1]
+; AVX512F-NEXT:    vpor %xmm11, %xmm10, %xmm10
+; AVX512F-NEXT:    vpslld $24, %xmm0, %xmm11
+; AVX512F-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm3, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm12 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpblendvb %ymm12, %ymm3, %ymm11, %ymm3
+; AVX512F-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,2,4,6,8,10,12,5,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vmovdqa 16(%rbp), %xmm11
+; AVX512F-NEXT:    vpsrld $16, %xmm11, %xmm12
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm12[0]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm11[0],zero,xmm11[1],zero
+; AVX512F-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm8[6,7]
+; AVX512F-NEXT:    vpsrld $24, %xmm11, %xmm8
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm8, %zmm3
+; AVX512F-NEXT:    vinserti128 $1, %xmm11, %ymm10, %ymm8
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm8[0,1,2,3],zmm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti128 $1, %xmm11, %ymm9, %ymm8
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,21,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm8, %zmm4, %zmm4
+; AVX512F-NEXT:    vpsrlq $48, %xmm11, %xmm8
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm8[0]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm7, %zmm5, %zmm5
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[0,1,2,0]
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,24,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpbroadcastb 16(%rbp), %ymm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm8[6,7]
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm7[0,1,2,3],zmm0[4,5,6,7]
+; AVX512F-NEXT:    vpsrlq $56, %xmm11, %xmm7
+; AVX512F-NEXT:    vmovdqa %ymm7, 416(%rdi)
+; AVX512F-NEXT:    vmovdqa %ymm6, 384(%rdi)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 320(%rdi)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 192(%rdi)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, 128(%rdi)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, 256(%rdi)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, 64(%rdi)
+; AVX512F-NEXT:    movq %rbp, %rsp
+; AVX512F-NEXT:    popq %rbp
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: PR153457:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    pushq %rbp
+; AVX512BW-NEXT:    movq %rsp, %rbp
+; AVX512BW-NEXT:    andq $-64, %rsp
+; AVX512BW-NEXT:    subq $64, %rsp
+; AVX512BW-NEXT:    movq %rdi, %rax
+; AVX512BW-NEXT:    vmovdqa64 16(%rbp), %zmm7
+; AVX512BW-NEXT:    vpbroadcastq %xmm0, %ymm8
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm9 = [u,u,u,u,u,u,u,u,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0]
+; AVX512BW-NEXT:    vpblendvb %ymm9, %ymm6, %ymm8, %ymm6
+; AVX512BW-NEXT:    vextracti64x4 $1, %zmm2, %ymm8
+; AVX512BW-NEXT:    vpbroadcastd %xmm0, %ymm10
+; AVX512BW-NEXT:    vpblendvb %ymm9, %ymm8, %ymm10, %ymm8
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,2,4,6,8,10,12,5,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
+; AVX512BW-NEXT:    vextracti32x4 $2, %zmm5, %xmm8
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3],xmm8[4],xmm0[4],xmm8[5],xmm0[5],xmm8[6],xmm0[6],xmm8[7],xmm0[7]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[0,2,4,6,8,10,12,13,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vinserti32x4 $2, %xmm8, %zmm5, %zmm5
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,128,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,128,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63]
+; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vextracti32x4 $2, %zmm4, %xmm9
+; AVX512BW-NEXT:    vpshufb %xmm8, %xmm9, %xmm9
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[5]
+; AVX512BW-NEXT:    vpor %xmm10, %xmm9, %xmm9
+; AVX512BW-NEXT:    vinserti32x4 $2, %xmm9, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpslld $24, %xmm0, %xmm9
+; AVX512BW-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
+; AVX512BW-NEXT:    vextracti64x4 $1, %zmm3, %ymm3
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm10 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vpblendvb %ymm10, %ymm3, %ymm9, %ymm3
+; AVX512BW-NEXT:    vpshufb %zmm8, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[1,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT:    vporq %zmm8, %zmm1, %zmm1
+; AVX512BW-NEXT:    vinserti128 $1, %xmm7, %ymm1, %ymm8
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm8[0,1,2,3],zmm1[4,5,6,7]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63]
+; AVX512BW-NEXT:    vpmovsxbw {{.*#+}} zmm8 = [0,1,2,3,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,21,22,23,24,25,26,27,28,29,30,31]
+; AVX512BW-NEXT:    vpermi2w %zmm7, %zmm2, %zmm8
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm7[0],zero,xmm7[1],zero
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
+; AVX512BW-NEXT:    vpsrld $24, %xmm7, %xmm3
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm3
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,39,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vinserti32x4 $3, %xmm7, %zmm4, %zmm4
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,53,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vpermq {{.*#+}} ymm6 = ymm0[0,1,2,0]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,24,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    vpbroadcastb 16(%rbp), %ymm9
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm9[6,7]
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm6[0,1,2,3],zmm0[4,5,6,7]
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [16,17,18,19,35,0,0,0,8,9,10,11,12,13,14,15,16,17,18,19,35,0,0,0,8,9,10,11,12,13,14,15]
+; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2w %zmm7, %zmm5, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 320(%rdi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, 384(%rdi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 256(%rdi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 192(%rdi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, 128(%rdi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, 64(%rdi)
+; AVX512BW-NEXT:    movq %rbp, %rsp
+; AVX512BW-NEXT:    popq %rbp
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: PR153457:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    pushq %rbp
+; AVX512DQ-NEXT:    movq %rsp, %rbp
+; AVX512DQ-NEXT:    andq $-64, %rsp
+; AVX512DQ-NEXT:    subq $64, %rsp
+; AVX512DQ-NEXT:    movq %rdi, %rax
+; AVX512DQ-NEXT:    vpbroadcastq %xmm0, %ymm7
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [u,u,u,u,u,u,u,u,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0]
+; AVX512DQ-NEXT:    vpblendvb %ymm8, %ymm6, %ymm7, %ymm6
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm2, %ymm7
+; AVX512DQ-NEXT:    vpbroadcastd %xmm0, %ymm9
+; AVX512DQ-NEXT:    vpblendvb %ymm8, %ymm7, %ymm9, %ymm8
+; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm5, %xmm7
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3],xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[0,2,4,6,8,10,12,13,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm4, %xmm9
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm10 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,128]
+; AVX512DQ-NEXT:    vpshufb %xmm10, %xmm9, %xmm9
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[5]
+; AVX512DQ-NEXT:    vpor %xmm11, %xmm9, %xmm9
+; AVX512DQ-NEXT:    vpshufb %xmm10, %xmm1, %xmm10
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1]
+; AVX512DQ-NEXT:    vpor %xmm11, %xmm10, %xmm10
+; AVX512DQ-NEXT:    vpslld $24, %xmm0, %xmm11
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm3, %ymm3
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpblendvb %ymm12, %ymm3, %ymm11, %ymm3
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,2,4,6,8,10,12,5,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vmovdqa 16(%rbp), %xmm11
+; AVX512DQ-NEXT:    vpsrld $16, %xmm11, %xmm12
+; AVX512DQ-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm12[0]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpmovzxdq {{.*#+}} xmm8 = xmm11[0],zero,xmm11[1],zero
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT:    vpsrld $24, %xmm11, %xmm8
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm8, %zmm3
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm10, %ymm8
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm8[0,1,2,3],zmm1[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm11, %ymm9, %ymm8
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,21,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm8, %zmm4, %zmm4
+; AVX512DQ-NEXT:    vpsrlq $48, %xmm11, %xmm8
+; AVX512DQ-NEXT:    vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm8[0]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm5, %zmm5
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[0,1,2,0]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,24,u,u,u,u,u,u,u,u]
+; AVX512DQ-NEXT:    vpbroadcastb 16(%rbp), %ymm8
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm8[6,7]
+; AVX512DQ-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm7[0,1,2,3],zmm0[4,5,6,7]
+; AVX512DQ-NEXT:    vpsrlq $56, %xmm11, %xmm7
+; AVX512DQ-NEXT:    vmovdqa %ymm7, 416(%rdi)
+; AVX512DQ-NEXT:    vmovdqa %ymm6, 384(%rdi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rdi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm5, 320(%rdi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm3, 192(%rdi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, 128(%rdi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm4, 256(%rdi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, 64(%rdi)
+; AVX512DQ-NEXT:    movq %rbp, %rsp
+; AVX512DQ-NEXT:    popq %rbp
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VBMI-LABEL: PR153457:
+; AVX512VBMI:       # %bb.0:
+; AVX512VBMI-NEXT:    pushq %rbp
+; AVX512VBMI-NEXT:    movq %rsp, %rbp
+; AVX512VBMI-NEXT:    andq $-64, %rsp
+; AVX512VBMI-NEXT:    subq $64, %rsp
+; AVX512VBMI-NEXT:    movq %rdi, %rax
+; AVX512VBMI-NEXT:    vmovdqa64 16(%rbp), %zmm7
+; AVX512VBMI-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [32,33,34,35,36,37,38,70,0,0,0,0,0,0,0,0,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,70,0,0,0,0,0,0,0,0,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
+; AVX512VBMI-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512VBMI-NEXT:    vpermi2b %zmm0, %zmm5, %zmm8
+; AVX512VBMI-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,69,0,0,0,0,0,0,0,0,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,69,0,0,0,0,0,0,0,0,24,25,26,27,28,29,30,31]
+; AVX512VBMI-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512VBMI-NEXT:    vpermi2b %zmm0, %zmm4, %zmm5
+; AVX512VBMI-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,68,0,0,0,0,0,0,0,0,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,68,0,0,0,0,0,0,0,0]
+; AVX512VBMI-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512VBMI-NEXT:    vpermi2b %zmm0, %zmm3, %zmm4
+; AVX512VBMI-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,1,2,3,4,5,6,66,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,67,0,1,2,3,4,5,6,66,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,67]
+; AVX512VBMI-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512VBMI-NEXT:    vpermi2b %zmm0, %zmm2, %zmm3
+; AVX512VBMI-NEXT:    vpshufb {{.*#+}} zmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[1,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI-NEXT:    vpshufb {{.*#+}} zmm1 = zmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14],zero,zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63]
+; AVX512VBMI-NEXT:    vporq %zmm2, %zmm1, %zmm1
+; AVX512VBMI-NEXT:    vmovdqa {{.*#+}} ymm2 = [u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,71]
+; AVX512VBMI-NEXT:    vpermi2b %zmm0, %zmm6, %zmm2
+; AVX512VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,0,64,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,56,57,58,59,60,61,62,63]
+; AVX512VBMI-NEXT:    vpermi2b %zmm7, %zmm0, %zmm6
+; AVX512VBMI-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [0,1,2,3,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,21,22,23,24,25,26,27,28,29,30,31]
+; AVX512VBMI-NEXT:    vpermi2w %zmm7, %zmm3, %zmm0
+; AVX512VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [67,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,68,u,u,u,u,u,u,u]
+; AVX512VBMI-NEXT:    vpermi2b %zmm7, %zmm4, %zmm3
+; AVX512VBMI-NEXT:    vinserti32x4 $3, %xmm7, %zmm5, %zmm4
+; AVX512VBMI-NEXT:    vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,53,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VBMI-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [16,17,18,19,35,0,0,0,8,9,10,11,12,13,14,15,16,17,18,19,35,0,0,0,8,9,10,11,12,13,14,15]
+; AVX512VBMI-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512VBMI-NEXT:    vpermi2w %zmm7, %zmm8, %zmm5
+; AVX512VBMI-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm2
+; AVX512VBMI-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,39,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512VBMI-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,65,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63]
+; AVX512VBMI-NEXT:    vpermi2b %zmm7, %zmm1, %zmm8
+; AVX512VBMI-NEXT:    vmovdqa64 %zmm5, 320(%rdi)
+; AVX512VBMI-NEXT:    vmovdqa64 %zmm4, 256(%rdi)
+; AVX512VBMI-NEXT:    vmovdqa64 %zmm3, 192(%rdi)
+; AVX512VBMI-NEXT:    vmovdqa64 %zmm0, 128(%rdi)
+; AVX512VBMI-NEXT:    vmovdqa64 %zmm8, 64(%rdi)
+; AVX512VBMI-NEXT:    vmovdqa64 %zmm6, (%rdi)
+; AVX512VBMI-NEXT:    vmovdqa64 %zmm2, 384(%rdi)
+; AVX512VBMI-NEXT:    movq %rbp, %rsp
+; AVX512VBMI-NEXT:    popq %rbp
+; AVX512VBMI-NEXT:    vzeroupper
+; AVX512VBMI-NEXT:    retq
+  %shuffle1 = shufflevector <512 x i8> %a0, <512 x i8> zeroinitializer, <512 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 4, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 280, i32 281, i32 282, i32 283, i32 284, i32 285, i32 286, i32 287, i32 288, i32 289, i32 290, i32 291, i32 292, i32 293, i32 294, i32 295, i32 296, i32 297, i32 298, i32 299, i32 300, i32 301, i32 302, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 336, i32 337, i32 338, i32 339, i32 340, i32 341, i32 342, i32 343, i32 344, i32 345, i32 346, i32 347, i32 348, i32 349, i32 350, i32 351, i32 352, i32 353, i32 354, i32 355, i32 356, i32 357, i32 358, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 392, i32 393, i32 394, i32 395, i32 396, i32 397, i32 398, i32 399, i32 400, i32 401, i32 402, i32 403, i32 404, i32 405, i32 406, i32 407, i32 408, i32 409, i32 410, i32 411, i32 412, i32 413, i32 414, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %shuffle2 = shufflevector <512 x i8> %shuffle1, <512 x i8> %a1, <512 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 512, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 64, i32 65, i32 66, i32 67, i32 68, i32 69, i32 70, i32 71, i32 72, i32 73, i32 74, i32 75, i32 76, i32 77, i32 78, i32 79, i32 513, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127, i32 128, i32 129, i32 130, i32 131, i32 132, i32 133, i32 134, i32 135, i32 514, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 168, i32 169, i32 170, i32 171, i32 172, i32 173, i32 174, i32 175, i32 176, i32 177, i32 178, i32 179, i32 180, i32 181, i32 182, i32 183, i32 184, i32 185, i32 186, i32 187, i32 188, i32 189, i32 190, i32 191, i32 515, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 224, i32 225, i32 226, i32 227, i32 228, i32 229, i32 230, i32 231, i32 232, i32 233, i32 234, i32 235, i32 236, i32 237, i32 238, i32 239, i32 240, i32 241, i32 242, i32 243, i32 244, i32 245, i32 246, i32 247, i32 516, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 280, i32 281, i32 282, i32 283, i32 284, i32 285, i32 286, i32 287, i32 288, i32 289, i32 290, i32 291, i32 292, i32 293, i32 294, i32 295, i32 296, i32 297, i32 298, i32 299, i32 300, i32 301, i32 302, i32 303, i32 517, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 336, i32 337, i32 338, i32 339, i32 340, i32 341, i32 342, i32 343, i32 344, i32 345, i32 346, i32 347, i32 348, i32 349, i32 350, i32 351, i32 352, i32 353, i32 354, i32 355, i32 356, i32 357, i32 358, i32 359, i32 518, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 392, i32 393, i32 394, i32 395, i32 396, i32 397, i32 398, i32 399, i32 400, i32 401, i32 402, i32 403, i32 404, i32 405, i32 406, i32 407, i32 408, i32 409, i32 410, i32 411, i32 412, i32 413, i32 414, i32 415, i32 519, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  ret <512 x i8> %shuffle2
+}
+
 define <64 x i8> @shuffle_v32i16_zextinreg_to_v16i32(<64 x i8> %a)  {
 ; ALL-LABEL: shuffle_v32i16_zextinreg_to_v16i32:
 ; ALL:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/win64-eh-empty-block-2.mir b/llvm/test/CodeGen/X86/win64-eh-empty-block-2.mir
index 6f2741faa856d..b97bccd11705d 100644
--- a/llvm/test/CodeGen/X86/win64-eh-empty-block-2.mir
+++ b/llvm/test/CodeGen/X86/win64-eh-empty-block-2.mir
@@ -120,8 +120,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: default, offset: -24, size: 8, alignment: 8, stack-id: default,
       isImmutable: false, isAliased: false, callee-saved-register: '',
diff --git a/llvm/test/CodeGen/X86/win64-eh-unwindv2-errors.mir b/llvm/test/CodeGen/X86/win64-eh-unwindv2-errors.mir
index f099d4fddcb33..de76d90bf6b6c 100644
--- a/llvm/test/CodeGen/X86/win64-eh-unwindv2-errors.mir
+++ b/llvm/test/CodeGen/X86/win64-eh-unwindv2-errors.mir
@@ -97,38 +97,6 @@ body:             |
     RET64
 ...
 
-;--- double_dealloc.mir
-# RUN: not --crash llc -mtriple=x86_64-pc-windows-msvc -o - %t/double_dealloc.mir \
-# RUN:    -run-pass=x86-wineh-unwindv2 2>&1 | FileCheck %s \
-# RUN:    --check-prefix=DOUBLE-DEALLOC
-# RUN: llc -mtriple=x86_64-pc-windows-msvc -o - %t/double_dealloc.mir \
-# RUN:    -run-pass=x86-wineh-unwindv2 -x86-wineh-unwindv2-force-mode=1 | \
-# RUN:    FileCheck %s --check-prefix=BESTEFFORT
-# DOUBLE-DEALLOC: LLVM ERROR: Windows x64 Unwind v2 is required, but LLVM has generated incompatible code in function 'double_dealloc':
-# DOUBLE-DEALLOC-SAME: The epilog is deallocating the stack allocation more than once
-
---- |
-  define dso_local void @double_dealloc() local_unnamed_addr {
-  entry:
-    ret void
-  }
-  !llvm.module.flags = !{!0}
-  !0 = !{i32 1, !"winx64-eh-unwindv2", i32 2}
-...
----
-name:            double_dealloc
-body:             |
-  bb.0.entry:
-    $rsp = frame-setup SUB64ri32 $rsp, 40, implicit-def dead $eflags
-    frame-setup SEH_StackAlloc 40
-    frame-setup SEH_EndPrologue
-    SEH_BeginEpilogue
-    $rsp = frame-destroy ADD64ri32 $rsp, 40, implicit-def dead $eflags
-    $rsp = frame-destroy ADD64ri32 $rsp, 40, implicit-def dead $eflags
-    SEH_EndEpilogue
-    RET64
-...
-
 ;--- dealloc_after_epilog.mir
 # RUN: not --crash llc -mtriple=x86_64-pc-windows-msvc -o - \
 # RUN:    %t/dealloc_after_epilog.mir -run-pass=x86-wineh-unwindv2 2>&1 | \
@@ -138,7 +106,7 @@ body:             |
 # RUN:    -x86-wineh-unwindv2-force-mode=1 |  FileCheck %s \
 # RUN:    --check-prefix=BESTEFFORT
 # DEALLOC-AFTER-EPILOG: LLVM ERROR: Windows x64 Unwind v2 is required, but LLVM has generated incompatible code in function 'dealloc_after_epilog':
-# DEALLOC-AFTER-EPILOG-SAME: Unexpected mov or add instruction after the epilog
+# DEALLOC-AFTER-EPILOG-SAME: Unexpected lea, mov or add instruction after the epilog
 
 --- |
   define dso_local void @dealloc_after_epilog() local_unnamed_addr {
@@ -316,3 +284,38 @@ body:             |
     $ecx = MOV32rr killed $eax
     RET64
 ...
+
+;--- dealloc_pop_dealloc.mir
+# RUN: not --crash llc -mtriple=x86_64-pc-windows-msvc -o - \
+# RUN:    %t/dealloc_pop_dealloc.mir -run-pass=x86-wineh-unwindv2 2>&1 | \
+# RUN:    FileCheck %s --check-prefix=DEALLOC-POP-DEALLOC
+# RUN: llc -mtriple=x86_64-pc-windows-msvc -o - %t/dealloc_pop_dealloc.mir \
+# RUN:    -run-pass=x86-wineh-unwindv2 -x86-wineh-unwindv2-force-mode=1 | \
+# RUN:    FileCheck %s --check-prefix=BESTEFFORT
+# DEALLOC-POP-DEALLOC: LLVM ERROR: Windows x64 Unwind v2 is required, but LLVM has generated incompatible code in function 'dealloc_pop_dealloc':
+# DEALLOC-POP-DEALLOC-SAME: The epilog is deallocating a stack allocation after popping registers
+
+--- |
+  define dso_local void @dealloc_pop_dealloc() local_unnamed_addr {
+  entry:
+    ret void
+  }
+  !llvm.module.flags = !{!0}
+  !0 = !{i32 1, !"winx64-eh-unwindv2", i32 2}
+...
+---
+name:            dealloc_pop_dealloc
+body:             |
+  bb.0.entry:
+    frame-setup PUSH64r killed $rdi, implicit-def $rsp, implicit $rsp
+    frame-setup SEH_PushReg 55
+    $rsp = frame-setup SUB64ri32 $rsp, 40, implicit-def dead $eflags
+    frame-setup SEH_StackAlloc 40
+    frame-setup SEH_EndPrologue
+    SEH_BeginEpilogue
+    $rsp = frame-destroy ADD64ri32 $rsp, 20, implicit-def dead $eflags
+    $rdi = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    $rsp = frame-destroy ADD64ri32 $rsp, 20, implicit-def dead $eflags
+    SEH_EndEpilogue
+    RET64
+...
diff --git a/llvm/test/CodeGen/X86/win64-eh-unwindv2-push-pop-stack-alloc.mir b/llvm/test/CodeGen/X86/win64-eh-unwindv2-push-pop-stack-alloc.mir
new file mode 100644
index 0000000000000..09a839f00814a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/win64-eh-unwindv2-push-pop-stack-alloc.mir
@@ -0,0 +1,199 @@
+# RUN: llc -o - %s -mtriple=x86_64-unknown-windows-msvc \
+# RUN:  -run-pass=x86-wineh-unwindv2 | FileCheck %s
+
+# Regression test for Win x64 unwind v2: in some cases it is better to use
+# push+pop to adjust the stack, rather than sub+add. This is permitted with
+# unwind v2 as the requirement is that the epilog finishes adjusting the stack
+# before popping the registers listed in the unwind table.
+
+# Pushes and pops the same register.
+# CHECK-LABEL:  name:            push_pop_same
+# CHECK:        body:
+# CHECK-NEXT:     bb.0
+# CHECK-NEXT:       SEH_UnwindVersion 2
+# CHECK-NEXT:       frame-setup PUSH64r undef $rax
+# CHECK-NEXT:       frame-setup SEH_StackAlloc 8
+# CHECK-NEXT:       frame-setup SEH_EndPrologue
+# CHECK-NEXT:       SEH_BeginEpilogue
+# CHECK-NEXT:       $rax = frame-destroy
+# CHECK-NEXT:       SEH_UnwindV2Start
+# CHECK-NEXT:       SEH_EndEpilogue
+# CHECK-NEXT:       RET64
+
+# Pushes and pops a different register.
+# CHECK-LABEL:  name:            push_pop_different
+# CHECK:        body:
+# CHECK-NEXT:     bb.0
+# CHECK-NEXT:       SEH_UnwindVersion 2
+# CHECK-NEXT:       frame-setup PUSH64r undef $rax
+# CHECK-NEXT:       frame-setup SEH_StackAlloc 8
+# CHECK-NEXT:       frame-setup SEH_EndPrologue
+# CHECK:            SEH_BeginEpilogue
+# CHECK-NEXT:       $rcx = frame-destroy POP64r
+# CHECK-NEXT:       SEH_UnwindV2Start
+# CHECK-NEXT:       SEH_EndEpilogue
+# CHECK-NEXT:       RET64 $eax
+
+# Pushes in the prolog, adds in the epilog.
+# CHECK-LABEL:  name:            push_add
+# CHECK:        body:
+# CHECK-NEXT:     bb.0
+# CHECK-NEXT:       SEH_UnwindVersion 2
+# CHECK-NEXT:       frame-setup PUSH64r killed $r15
+# CHECK-NEXT:       frame-setup SEH_PushReg 126
+# CHECK-NEXT:       frame-setup PUSH64r killed $r14
+# CHECK-NEXT:       frame-setup SEH_PushReg 125
+# CHECK-NEXT:       frame-setup PUSH64r killed $rsi
+# CHECK-NEXT:       frame-setup SEH_PushReg 60
+# CHECK-NEXT:       frame-setup PUSH64r killed $rdi
+# CHECK-NEXT:       frame-setup SEH_PushReg 55
+# CHECK-NEXT:       frame-setup PUSH64r killed $rbx
+# CHECK-NEXT:       frame-setup SEH_PushReg 53
+# CHECK-NEXT:       frame-setup PUSH64r undef $rax
+# CHECK-NEXT:       frame-setup SEH_StackAlloc 8
+# CHECK-NEXT:       frame-setup SEH_EndPrologue
+# CHECK:            SEH_BeginEpilogue
+# CHECK-NEXT:       $rsp = frame-destroy ADD64ri32 $rsp, 8
+# CHECK-NEXT:       SEH_UnwindV2Start
+# CHECK-NEXT:       $rbx = frame-destroy POP64r
+# CHECK-NEXT:       $rdi = frame-destroy POP64r
+# CHECK-NEXT:       $rsi = frame-destroy POP64r
+# CHECK-NEXT:       $r14 = frame-destroy POP64r
+# CHECK-NEXT:       $r15 = frame-destroy POP64r
+# CHECK-NEXT:       SEH_EndEpilogue
+# CHECK-NEXT:       RET64
+
+--- |
+  define void @push_pop_same() {
+    %small_alloca = alloca i32, align 4
+    ret void
+  }
+  
+  define i32 @push_pop_different(i32 %x, i32 %y) {
+    %small_alloca = alloca i32, align 4
+    %sum = add i32 %x, %y
+    ret i32 %sum
+  }
+  
+  define void @push_add(ptr %a, ptr %b, ptr %out) {
+    %small_alloca = alloca i32, align 4
+    %av = load i256, ptr %a, align 16
+    %bv = load i256, ptr %b, align 16
+    %r = mul i256 %av, %bv
+    store i256 %r, ptr %out, align 16
+    ret void
+  }
+  
+  !llvm.module.flags = !{!0}
+  
+  !0 = !{i32 1, !"winx64-eh-unwindv2", i32 2}
+...
+---
+name:            push_pop_same
+body:             |
+  bb.0 (%ir-block.0):
+    frame-setup PUSH64r undef $rax, implicit-def $rsp, implicit $rsp
+    frame-setup SEH_StackAlloc 8
+    frame-setup SEH_EndPrologue
+    SEH_BeginEpilogue
+    $rax = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    SEH_EndEpilogue
+    RET64
+...
+---
+name:            push_pop_different
+body:             |
+  bb.0 (%ir-block.0):
+    frame-setup PUSH64r undef $rax, implicit-def $rsp, implicit $rsp
+    frame-setup SEH_StackAlloc 8
+    frame-setup SEH_EndPrologue
+    renamable $edx = KILL $edx, implicit-def $rdx
+    renamable $ecx = KILL $ecx, implicit-def $rcx
+    renamable $eax = LEA64_32r killed renamable $rcx, 1, killed renamable $rdx, 0, $noreg
+    SEH_BeginEpilogue
+    $rcx = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    SEH_EndEpilogue
+    RET64 $eax
+...
+---
+name:            push_add
+body:             |
+  bb.0 (%ir-block.0):
+
+    frame-setup PUSH64r killed $r15, implicit-def $rsp, implicit $rsp
+    frame-setup SEH_PushReg 126
+    frame-setup PUSH64r killed $r14, implicit-def $rsp, implicit $rsp
+    frame-setup SEH_PushReg 125
+    frame-setup PUSH64r killed $rsi, implicit-def $rsp, implicit $rsp
+    frame-setup SEH_PushReg 60
+    frame-setup PUSH64r killed $rdi, implicit-def $rsp, implicit $rsp
+    frame-setup SEH_PushReg 55
+    frame-setup PUSH64r killed $rbx, implicit-def $rsp, implicit $rsp
+    frame-setup SEH_PushReg 53
+    frame-setup PUSH64r undef $rax, implicit-def $rsp, implicit $rsp
+    frame-setup SEH_StackAlloc 8
+    frame-setup SEH_EndPrologue
+    $rsi = MOV64rr $rdx
+    renamable $r9 = MOV64rm renamable $rcx, 1, $noreg, 0, $noreg :: (load (s64) from %ir.a, align 16)
+    renamable $rdi = MOV64rm renamable $rcx, 1, $noreg, 8, $noreg :: (load (s64) from %ir.a + 8, basealign 16)
+    renamable $rbx = MOV64rm renamable $rcx, 1, $noreg, 16, $noreg :: (load (s64) from %ir.a + 16, align 16)
+    renamable $r10 = MOV64rm $rdx, 1, $noreg, 16, $noreg :: (load (s64) from %ir.b + 16, align 16)
+    renamable $r11 = MOV64rm $rdx, 1, $noreg, 0, $noreg :: (load (s64) from %ir.b, align 16)
+    renamable $r14 = MOV64rm $rdx, 1, $noreg, 8, $noreg :: (load (s64) from %ir.b + 8, basealign 16)
+    renamable $r15 = MOV64rm killed renamable $rcx, 1, $noreg, 24, $noreg :: (load (s64) from %ir.a + 24, basealign 16)
+    renamable $r15 = IMUL64rr killed renamable $r15, renamable $r11, implicit-def dead $eflags
+    $rax = MOV64rr $r11
+    MUL64r renamable $rbx, implicit-def $rax, implicit-def $rdx, implicit-def dead $eflags, implicit $rax
+    $rcx = MOV64rr $rax
+    renamable $rbx = IMUL64rr killed renamable $rbx, renamable $r14, implicit-def dead $eflags
+    renamable $rbx = ADD64rr killed renamable $rbx, killed renamable $rdx, implicit-def dead $eflags
+    renamable $rbx = ADD64rr killed renamable $rbx, killed renamable $r15, implicit-def dead $eflags
+    $r15 = MOV64rr $r10
+    renamable $r15 = IMUL64rr killed renamable $r15, renamable $rdi, implicit-def dead $eflags
+    $rax = MOV64rr killed $r10
+    MUL64r renamable $r9, implicit-def $rax, implicit-def $rdx, implicit-def dead $eflags, implicit $rax
+    $r10 = MOV64rr $rax
+    renamable $rdx = ADD64rr killed renamable $rdx, killed renamable $r15, implicit-def dead $eflags
+    renamable $r15 = MOV64rm killed renamable $rsi, 1, $noreg, 24, $noreg :: (load (s64) from %ir.b + 24, basealign 16)
+    renamable $r15 = IMUL64rr killed renamable $r15, renamable $r9, implicit-def dead $eflags
+    renamable $r15 = ADD64rr killed renamable $r15, killed renamable $rdx, implicit-def dead $eflags
+    renamable $r10 = ADD64rr killed renamable $r10, killed renamable $rcx, implicit-def $eflags
+    renamable $r15 = ADC64rr killed renamable $r15, killed renamable $rbx, implicit-def dead $eflags, implicit killed $eflags
+    $rax = MOV64rr $r9
+    MUL64r renamable $r11, implicit-def $rax, implicit-def $rdx, implicit-def dead $eflags, implicit $rax
+    $rcx = MOV64rr $rdx
+    $rsi = MOV64rr $rax
+    $rax = MOV64rr $rdi
+    MUL64r killed renamable $r11, implicit-def $rax, implicit-def $rdx, implicit-def dead $eflags, implicit $rax
+    $r11 = MOV64rr $rdx
+    $rbx = MOV64rr $rax
+    renamable $rbx = ADD64rr killed renamable $rbx, killed renamable $rcx, implicit-def $eflags
+    renamable $r11 = ADC64ri32 killed renamable $r11, 0, implicit-def dead $eflags, implicit killed $eflags
+    $rax = MOV64rr killed $r9
+    MUL64r renamable $r14, implicit-def $rax, implicit-def $rdx, implicit-def dead $eflags, implicit $rax
+    $rcx = MOV64rr $rdx
+    $r9 = MOV64rr $rax
+    renamable $r9 = ADD64rr killed renamable $r9, killed renamable $rbx, implicit-def $eflags
+    renamable $rcx = ADC64rr killed renamable $rcx, killed renamable $r11, implicit-def $eflags, implicit killed $eflags
+    renamable $al = SETCCr 2, implicit killed $eflags
+    renamable $r11d = MOVZX32rr8 killed renamable $al, implicit-def $r11
+    $rax = MOV64rr killed $rdi
+    MUL64r killed renamable $r14, implicit-def $rax, implicit-def $rdx, implicit-def dead $eflags, implicit $rax
+    renamable $rax = ADD64rr killed renamable $rax, killed renamable $rcx, implicit-def $eflags
+    renamable $rdx = ADC64rr killed renamable $rdx, killed renamable $r11, implicit-def dead $eflags, implicit killed $eflags
+    renamable $rax = ADD64rr killed renamable $rax, killed renamable $r10, implicit-def $eflags
+    renamable $rdx = ADC64rr killed renamable $rdx, killed renamable $r15, implicit-def dead $eflags, implicit killed $eflags
+    MOV64mr renamable $r8, 1, $noreg, 0, $noreg, killed renamable $rsi :: (store (s64) into %ir.out, align 16)
+    MOV64mr renamable $r8, 1, $noreg, 8, $noreg, killed renamable $r9 :: (store (s64) into %ir.out + 8, basealign 16)
+    MOV64mr renamable $r8, 1, $noreg, 16, $noreg, killed renamable $rax :: (store (s64) into %ir.out + 16, align 16)
+    MOV64mr killed renamable $r8, 1, $noreg, 24, $noreg, killed renamable $rdx :: (store (s64) into %ir.out + 24, basealign 16)
+    SEH_BeginEpilogue
+    $rsp = frame-destroy ADD64ri32 $rsp, 8, implicit-def dead $eflags
+    $rbx = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    $rdi = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    $rsi = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    $r14 = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    $r15 = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    SEH_EndEpilogue
+    RET64
+...
diff --git a/llvm/test/CodeGen/X86/win64-eh-unwindv2.ll b/llvm/test/CodeGen/X86/win64-eh-unwindv2.ll
index a9fd1b9ac2acd..326127a919f3a 100644
--- a/llvm/test/CodeGen/X86/win64-eh-unwindv2.ll
+++ b/llvm/test/CodeGen/X86/win64-eh-unwindv2.ll
@@ -152,6 +152,25 @@ entry:
 ; CHECK-NEXT:   retq
 ; CHECK-NEXT:   .seh_endproc
 
+define dso_local void @large_aligned_alloc() align 16 {
+  %1 = alloca [128 x i8], align 64
+  ret void
+}
+; CHECK-LABEL:  large_aligned_alloc:
+; CHECK:        .seh_unwindversion 2
+; CHECK:        .seh_pushreg %rbp
+; CHECK:        .seh_stackalloc 176
+; CHECK:        .seh_setframe %rbp, 128
+; CHECK:        .seh_endprologue
+; CHECK-NOT:    .seh_endproc
+; CHECK:        .seh_startepilogue
+; CHECK-NEXT:   leaq    48(%rbp), %rsp
+; CHECK-NEXT:   .seh_unwindv2start
+; CHECK-NEXT:   popq    %rbp
+; CHECK-NEXT:   .seh_endepilogue
+; CHECK-NEXT:   retq
+; CHECK-NEXT:   .seh_endproc
+
 declare void @a() local_unnamed_addr
 declare i32 @b() local_unnamed_addr
 declare i32 @c(i32) local_unnamed_addr
diff --git a/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll b/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
index 9555ce032db90..732fc6543e314 100644
--- a/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
+++ b/llvm/test/CodeGen/X86/win64-stackprobe-overflow.ll
@@ -10,5 +10,5 @@ start:
 attributes #0 = { nonlazybind uwtable "probe-stack"="probe_stack" "target-cpu"="x86-64" }
 
 ; CHECK-LABEL: foo:
-; CHECK: movabsq $4294967304, %rax
+; CHECK: movabsq $4294967312, %rax
 ; CHECK-NEXT: callq probe_stack
diff --git a/llvm/test/CodeGen/X86/zero-call-used-regs-debug-info.mir b/llvm/test/CodeGen/X86/zero-call-used-regs-debug-info.mir
index 35eeede9da203..be022e37dfc7f 100644
--- a/llvm/test/CodeGen/X86/zero-call-used-regs-debug-info.mir
+++ b/llvm/test/CodeGen/X86/zero-call-used-regs-debug-info.mir
@@ -117,8 +117,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/CodeGen/Xtensa/atomic-load-store.ll b/llvm/test/CodeGen/Xtensa/atomic-load-store.ll
new file mode 100644
index 0000000000000..bd843a353da2a
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/atomic-load-store.ll
@@ -0,0 +1,498 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=xtensa -mattr=+windowed < %s | FileCheck %s --check-prefixes=XTENSA
+; RUN: llc -mtriple=xtensa -mattr=+windowed,s32c1i < %s | FileCheck %s --check-prefixes=XTENSA-ATOMIC
+
+define i8 @atomic_load_i8_unordered(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i8_unordered:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI0_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i8_unordered:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l8ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i8, ptr %a unordered, align 1
+  ret i8 %1
+}
+
+define i8 @atomic_load_i8_monotonic(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI1_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l8ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i8, ptr %a monotonic, align 1
+  ret i8 %1
+}
+
+define i8 @atomic_load_i8_acquire(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 2
+; XTENSA-NEXT:    l32r a8, .LCPI2_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l8ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i8, ptr %a acquire, align 1
+  ret i8 %1
+}
+
+define i8 @atomic_load_i8_seq_cst(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 5
+; XTENSA-NEXT:    l32r a8, .LCPI3_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l8ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i8, ptr %a seq_cst, align 1
+  ret i8 %1
+}
+
+define i16 @atomic_load_i16_unordered(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i16_unordered:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI4_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i16_unordered:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l16ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i16, ptr %a unordered, align 2
+  ret i16 %1
+}
+
+define i16 @atomic_load_i16_monotonic(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI5_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l16ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i16, ptr %a monotonic, align 2
+  ret i16 %1
+}
+
+define i16 @atomic_load_i16_acquire(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 2
+; XTENSA-NEXT:    l32r a8, .LCPI6_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l16ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i16, ptr %a acquire, align 2
+  ret i16 %1
+}
+
+define i16 @atomic_load_i16_seq_cst(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 5
+; XTENSA-NEXT:    l32r a8, .LCPI7_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l16ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i16, ptr %a seq_cst, align 2
+  ret i16 %1
+}
+
+define i32 @atomic_load_i32_unordered(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i32_unordered:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI8_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i32_unordered:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i32, ptr %a unordered, align 4
+  ret i32 %1
+}
+
+define i32 @atomic_load_i32_monotonic(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI9_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i32, ptr %a monotonic, align 4
+  ret i32 %1
+}
+
+define i32 @atomic_load_i32_acquire(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 2
+; XTENSA-NEXT:    l32r a8, .LCPI10_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i32, ptr %a acquire, align 4
+  ret i32 %1
+}
+
+define i32 @atomic_load_i32_seq_cst(ptr %a) nounwind {
+; XTENSA-LABEL: atomic_load_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 5
+; XTENSA-NEXT:    l32r a8, .LCPI11_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_load_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %1 = load atomic i32, ptr %a seq_cst, align 4
+  ret i32 %1
+}
+
+define void @atomic_store_i8_unordered(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i8_unordered:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI12_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i8_unordered:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    s8i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i8 %b, ptr %a unordered, align 1
+  ret void
+}
+
+define void @atomic_store_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI13_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    s8i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i8 %b, ptr %a monotonic, align 1
+  ret void
+}
+
+define void @atomic_store_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI14_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    s8i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i8 %b, ptr %a release, align 1
+  ret void
+}
+
+define void @atomic_store_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI15_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    s8i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i8 %b, ptr %a seq_cst, align 1
+  ret void
+}
+
+define void @atomic_store_i16_unordered(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i16_unordered:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI16_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i16_unordered:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    s16i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i16 %b, ptr %a unordered, align 2
+  ret void
+}
+
+define void @atomic_store_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI17_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    s16i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i16 %b, ptr %a monotonic, align 2
+  ret void
+}
+
+define void @atomic_store_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI18_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    s16i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i16 %b, ptr %a release, align 2
+  ret void
+}
+
+define void @atomic_store_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI19_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    s16i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i16 %b, ptr %a seq_cst, align 2
+  ret void
+}
+
+define void @atomic_store_i32_unordered(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i32_unordered:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI20_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i32_unordered:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    s32i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i32 %b, ptr %a unordered, align 4
+  ret void
+}
+
+define void @atomic_store_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI21_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    s32i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i32 %b, ptr %a monotonic, align 4
+  ret void
+}
+
+define void @atomic_store_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI22_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    s32i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i32 %b, ptr %a release, align 4
+  ret void
+}
+
+define void @atomic_store_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomic_store_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI23_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomic_store_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    s32i a3, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i32 %b, ptr %a seq_cst, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/Xtensa/atomic-rmw.ll b/llvm/test/CodeGen/Xtensa/atomic-rmw.ll
new file mode 100644
index 0000000000000..81cb2dd5e8184
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/atomic-rmw.ll
@@ -0,0 +1,10298 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=xtensa -mattr=+windowed < %s | FileCheck %s --check-prefixes=XTENSA
+; RUN: llc -mtriple=xtensa -mattr=+windowed,s32c1i < %s | FileCheck %s --check-prefixes=XTENSA-ATOMIC
+
+define i8 @atomicrmw_xchg_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI0_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB0_2
+; XTENSA-ATOMIC-NEXT:  .LBB0_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB0_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB0_4
+; XTENSA-ATOMIC-NEXT:  .LBB0_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB0_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB0_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB0_1
+; XTENSA-ATOMIC-NEXT:  .LBB0_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xchg_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI1_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB1_2
+; XTENSA-ATOMIC-NEXT:  .LBB1_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB1_4
+; XTENSA-ATOMIC-NEXT:  .LBB1_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB1_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB1_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB1_1
+; XTENSA-ATOMIC-NEXT:  .LBB1_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xchg_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI2_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB2_2
+; XTENSA-ATOMIC-NEXT:  .LBB2_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB2_4
+; XTENSA-ATOMIC-NEXT:  .LBB2_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB2_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB2_1
+; XTENSA-ATOMIC-NEXT:  .LBB2_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xchg_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI3_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB3_2
+; XTENSA-ATOMIC-NEXT:  .LBB3_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB3_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB3_4
+; XTENSA-ATOMIC-NEXT:  .LBB3_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB3_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB3_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB3_1
+; XTENSA-ATOMIC-NEXT:  .LBB3_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xchg_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI4_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB4_2
+; XTENSA-ATOMIC-NEXT:  .LBB4_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB4_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB4_4
+; XTENSA-ATOMIC-NEXT:  .LBB4_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB4_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB4_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB4_1
+; XTENSA-ATOMIC-NEXT:  .LBB4_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI5_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB5_2
+; XTENSA-ATOMIC-NEXT:  .LBB5_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB5_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB5_4
+; XTENSA-ATOMIC-NEXT:  .LBB5_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB5_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB5_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB5_1
+; XTENSA-ATOMIC-NEXT:  .LBB5_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI6_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB6_2
+; XTENSA-ATOMIC-NEXT:  .LBB6_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB6_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB6_4
+; XTENSA-ATOMIC-NEXT:  .LBB6_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB6_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB6_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB6_1
+; XTENSA-ATOMIC-NEXT:  .LBB6_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI7_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB7_2
+; XTENSA-ATOMIC-NEXT:  .LBB7_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB7_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB7_4
+; XTENSA-ATOMIC-NEXT:  .LBB7_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB7_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB7_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB7_1
+; XTENSA-ATOMIC-NEXT:  .LBB7_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI8_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB8_2
+; XTENSA-ATOMIC-NEXT:  .LBB8_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB8_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB8_4
+; XTENSA-ATOMIC-NEXT:  .LBB8_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB8_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB8_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB8_1
+; XTENSA-ATOMIC-NEXT:  .LBB8_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI9_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB9_2
+; XTENSA-ATOMIC-NEXT:  .LBB9_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB9_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB9_4
+; XTENSA-ATOMIC-NEXT:  .LBB9_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB9_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB9_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB9_1
+; XTENSA-ATOMIC-NEXT:  .LBB9_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI10_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB10_2
+; XTENSA-ATOMIC-NEXT:  .LBB10_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB10_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB10_4
+; XTENSA-ATOMIC-NEXT:  .LBB10_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB10_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB10_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB10_1
+; XTENSA-ATOMIC-NEXT:  .LBB10_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI11_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB11_2
+; XTENSA-ATOMIC-NEXT:  .LBB11_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB11_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB11_4
+; XTENSA-ATOMIC-NEXT:  .LBB11_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB11_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB11_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB11_1
+; XTENSA-ATOMIC-NEXT:  .LBB11_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI12_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB12_2
+; XTENSA-ATOMIC-NEXT:  .LBB12_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB12_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB12_4
+; XTENSA-ATOMIC-NEXT:  .LBB12_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB12_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB12_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB12_1
+; XTENSA-ATOMIC-NEXT:  .LBB12_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI13_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB13_2
+; XTENSA-ATOMIC-NEXT:  .LBB13_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB13_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB13_4
+; XTENSA-ATOMIC-NEXT:  .LBB13_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB13_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB13_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB13_1
+; XTENSA-ATOMIC-NEXT:  .LBB13_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI14_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB14_2
+; XTENSA-ATOMIC-NEXT:  .LBB14_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB14_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB14_4
+; XTENSA-ATOMIC-NEXT:  .LBB14_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB14_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB14_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB14_1
+; XTENSA-ATOMIC-NEXT:  .LBB14_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI15_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB15_2
+; XTENSA-ATOMIC-NEXT:  .LBB15_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB15_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB15_4
+; XTENSA-ATOMIC-NEXT:  .LBB15_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB15_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB15_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB15_1
+; XTENSA-ATOMIC-NEXT:  .LBB15_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI16_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB16_2
+; XTENSA-ATOMIC-NEXT:  .LBB16_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB16_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB16_4
+; XTENSA-ATOMIC-NEXT:  .LBB16_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB16_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB16_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB16_1
+; XTENSA-ATOMIC-NEXT:  .LBB16_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI17_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB17_2
+; XTENSA-ATOMIC-NEXT:  .LBB17_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB17_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB17_4
+; XTENSA-ATOMIC-NEXT:  .LBB17_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB17_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB17_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB17_1
+; XTENSA-ATOMIC-NEXT:  .LBB17_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI18_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB18_2
+; XTENSA-ATOMIC-NEXT:  .LBB18_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB18_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB18_4
+; XTENSA-ATOMIC-NEXT:  .LBB18_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB18_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB18_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB18_1
+; XTENSA-ATOMIC-NEXT:  .LBB18_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI19_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB19_2
+; XTENSA-ATOMIC-NEXT:  .LBB19_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB19_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB19_4
+; XTENSA-ATOMIC-NEXT:  .LBB19_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB19_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB19_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB19_1
+; XTENSA-ATOMIC-NEXT:  .LBB19_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI20_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB20_2
+; XTENSA-ATOMIC-NEXT:  .LBB20_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB20_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB20_4
+; XTENSA-ATOMIC-NEXT:  .LBB20_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB20_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB20_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB20_1
+; XTENSA-ATOMIC-NEXT:  .LBB20_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI21_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB21_2
+; XTENSA-ATOMIC-NEXT:  .LBB21_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB21_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB21_4
+; XTENSA-ATOMIC-NEXT:  .LBB21_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB21_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB21_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB21_1
+; XTENSA-ATOMIC-NEXT:  .LBB21_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI22_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB22_2
+; XTENSA-ATOMIC-NEXT:  .LBB22_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB22_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB22_4
+; XTENSA-ATOMIC-NEXT:  .LBB22_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB22_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB22_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB22_1
+; XTENSA-ATOMIC-NEXT:  .LBB22_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI23_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB23_2
+; XTENSA-ATOMIC-NEXT:  .LBB23_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB23_4
+; XTENSA-ATOMIC-NEXT:  .LBB23_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB23_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB23_1
+; XTENSA-ATOMIC-NEXT:  .LBB23_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI24_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a10, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB24_2
+; XTENSA-ATOMIC-NEXT:  .LBB24_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB24_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB24_4
+; XTENSA-ATOMIC-NEXT:  .LBB24_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB24_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB24_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB24_1
+; XTENSA-ATOMIC-NEXT:  .LBB24_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI25_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB25_2
+; XTENSA-ATOMIC-NEXT:  .LBB25_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB25_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB25_4
+; XTENSA-ATOMIC-NEXT:  .LBB25_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB25_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB25_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB25_1
+; XTENSA-ATOMIC-NEXT:  .LBB25_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI26_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB26_2
+; XTENSA-ATOMIC-NEXT:  .LBB26_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB26_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB26_4
+; XTENSA-ATOMIC-NEXT:  .LBB26_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB26_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB26_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB26_1
+; XTENSA-ATOMIC-NEXT:  .LBB26_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI27_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB27_2
+; XTENSA-ATOMIC-NEXT:  .LBB27_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB27_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB27_4
+; XTENSA-ATOMIC-NEXT:  .LBB27_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB27_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB27_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB27_1
+; XTENSA-ATOMIC-NEXT:  .LBB27_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI28_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB28_2
+; XTENSA-ATOMIC-NEXT:  .LBB28_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB28_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB28_4
+; XTENSA-ATOMIC-NEXT:  .LBB28_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB28_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB28_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB28_1
+; XTENSA-ATOMIC-NEXT:  .LBB28_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI29_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB29_2
+; XTENSA-ATOMIC-NEXT:  .LBB29_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB29_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB29_4
+; XTENSA-ATOMIC-NEXT:  .LBB29_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB29_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB29_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB29_1
+; XTENSA-ATOMIC-NEXT:  .LBB29_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI30_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB30_2
+; XTENSA-ATOMIC-NEXT:  .LBB30_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB30_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB30_4
+; XTENSA-ATOMIC-NEXT:  .LBB30_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB30_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB30_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB30_1
+; XTENSA-ATOMIC-NEXT:  .LBB30_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI31_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB31_2
+; XTENSA-ATOMIC-NEXT:  .LBB31_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB31_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB31_4
+; XTENSA-ATOMIC-NEXT:  .LBB31_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB31_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB31_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB31_1
+; XTENSA-ATOMIC-NEXT:  .LBB31_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI32_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB32_2
+; XTENSA-ATOMIC-NEXT:  .LBB32_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB32_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB32_4
+; XTENSA-ATOMIC-NEXT:  .LBB32_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB32_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB32_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB32_1
+; XTENSA-ATOMIC-NEXT:  .LBB32_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI33_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB33_2
+; XTENSA-ATOMIC-NEXT:  .LBB33_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB33_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB33_4
+; XTENSA-ATOMIC-NEXT:  .LBB33_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB33_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB33_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB33_1
+; XTENSA-ATOMIC-NEXT:  .LBB33_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI34_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 255
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB34_2
+; XTENSA-ATOMIC-NEXT:  .LBB34_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB34_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB34_4
+; XTENSA-ATOMIC-NEXT:  .LBB34_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB34_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB34_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB34_1
+; XTENSA-ATOMIC-NEXT:  .LBB34_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l8ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    srai a5, a8, 24
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a4, .LCPI35_0
+; XTENSA-NEXT:    j .LBB35_2
+; XTENSA-NEXT:  .LBB35_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l8ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB35_4
+; XTENSA-NEXT:  .LBB35_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a5, a8, .LBB35_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB35_1
+; XTENSA-NEXT:  .LBB35_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB35_2
+; XTENSA-ATOMIC-NEXT:  .LBB35_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB35_6
+; XTENSA-ATOMIC-NEXT:  .LBB35_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a12, a5, .LBB35_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB35_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB35_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB35_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB35_1
+; XTENSA-ATOMIC-NEXT:  .LBB35_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l8ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    srai a5, a8, 24
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a4, .LCPI36_0
+; XTENSA-NEXT:    j .LBB36_2
+; XTENSA-NEXT:  .LBB36_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB36_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l8ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB36_4
+; XTENSA-NEXT:  .LBB36_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a5, a8, .LBB36_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB36_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB36_1
+; XTENSA-NEXT:  .LBB36_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB36_2
+; XTENSA-ATOMIC-NEXT:  .LBB36_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB36_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB36_6
+; XTENSA-ATOMIC-NEXT:  .LBB36_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a12, a5, .LBB36_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB36_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB36_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB36_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB36_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB36_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB36_1
+; XTENSA-ATOMIC-NEXT:  .LBB36_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a9, a2, a2
+; XTENSA-NEXT:    l8ui a2, a9, 0
+; XTENSA-NEXT:    s32i a3, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    or a3, a9, a9
+; XTENSA-NEXT:    srai a4, a8, 24
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a5, .LCPI37_0
+; XTENSA-NEXT:    j .LBB37_2
+; XTENSA-NEXT:  .LBB37_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB37_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    or a10, a3, a3
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB37_4
+; XTENSA-NEXT:  .LBB37_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    l32i a12, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    bge a4, a8, .LBB37_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB37_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB37_1
+; XTENSA-NEXT:  .LBB37_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB37_2
+; XTENSA-ATOMIC-NEXT:  .LBB37_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB37_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB37_6
+; XTENSA-ATOMIC-NEXT:  .LBB37_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a12, a5, .LBB37_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB37_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB37_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB37_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB37_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB37_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB37_1
+; XTENSA-ATOMIC-NEXT:  .LBB37_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a9, a2, a2
+; XTENSA-NEXT:    l8ui a2, a9, 0
+; XTENSA-NEXT:    s32i a3, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    or a3, a9, a9
+; XTENSA-NEXT:    srai a4, a8, 24
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a5, .LCPI38_0
+; XTENSA-NEXT:    j .LBB38_2
+; XTENSA-NEXT:  .LBB38_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB38_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    or a10, a3, a3
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB38_4
+; XTENSA-NEXT:  .LBB38_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    l32i a12, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    bge a4, a8, .LBB38_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB38_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB38_1
+; XTENSA-NEXT:  .LBB38_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB38_2
+; XTENSA-ATOMIC-NEXT:  .LBB38_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB38_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB38_6
+; XTENSA-ATOMIC-NEXT:  .LBB38_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a12, a5, .LBB38_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB38_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB38_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB38_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB38_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB38_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB38_1
+; XTENSA-ATOMIC-NEXT:  .LBB38_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l8ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    srai a5, a8, 24
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a4, .LCPI39_0
+; XTENSA-NEXT:    j .LBB39_2
+; XTENSA-NEXT:  .LBB39_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l8ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB39_4
+; XTENSA-NEXT:  .LBB39_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a5, a8, .LBB39_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB39_1
+; XTENSA-NEXT:  .LBB39_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB39_2
+; XTENSA-ATOMIC-NEXT:  .LBB39_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB39_6
+; XTENSA-ATOMIC-NEXT:  .LBB39_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a12, a5, .LBB39_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB39_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB39_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB39_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB39_1
+; XTENSA-ATOMIC-NEXT:  .LBB39_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l8ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    srai a5, a8, 24
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a4, .LCPI40_0
+; XTENSA-NEXT:    j .LBB40_2
+; XTENSA-NEXT:  .LBB40_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB40_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l8ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB40_4
+; XTENSA-NEXT:  .LBB40_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a5, a8, .LBB40_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB40_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB40_1
+; XTENSA-NEXT:  .LBB40_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB40_2
+; XTENSA-ATOMIC-NEXT:  .LBB40_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB40_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB40_6
+; XTENSA-ATOMIC-NEXT:  .LBB40_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a12, a5, .LBB40_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB40_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB40_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB40_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB40_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB40_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB40_1
+; XTENSA-ATOMIC-NEXT:  .LBB40_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l8ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    srai a5, a8, 24
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a4, .LCPI41_0
+; XTENSA-NEXT:    j .LBB41_2
+; XTENSA-NEXT:  .LBB41_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB41_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l8ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB41_4
+; XTENSA-NEXT:  .LBB41_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a5, a8, .LBB41_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB41_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB41_1
+; XTENSA-NEXT:  .LBB41_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB41_2
+; XTENSA-ATOMIC-NEXT:  .LBB41_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB41_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB41_6
+; XTENSA-ATOMIC-NEXT:  .LBB41_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a12, a5, .LBB41_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB41_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB41_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB41_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB41_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB41_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB41_1
+; XTENSA-ATOMIC-NEXT:  .LBB41_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a9, a2, a2
+; XTENSA-NEXT:    l8ui a2, a9, 0
+; XTENSA-NEXT:    s32i a3, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    or a3, a9, a9
+; XTENSA-NEXT:    srai a4, a8, 24
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a5, .LCPI42_0
+; XTENSA-NEXT:    j .LBB42_2
+; XTENSA-NEXT:  .LBB42_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB42_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    or a10, a3, a3
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB42_4
+; XTENSA-NEXT:  .LBB42_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    l32i a12, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    blt a4, a8, .LBB42_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB42_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB42_1
+; XTENSA-NEXT:  .LBB42_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB42_2
+; XTENSA-ATOMIC-NEXT:  .LBB42_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB42_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB42_6
+; XTENSA-ATOMIC-NEXT:  .LBB42_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a12, a5, .LBB42_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB42_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB42_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB42_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB42_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB42_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB42_1
+; XTENSA-ATOMIC-NEXT:  .LBB42_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a9, a2, a2
+; XTENSA-NEXT:    l8ui a2, a9, 0
+; XTENSA-NEXT:    s32i a3, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    or a3, a9, a9
+; XTENSA-NEXT:    srai a4, a8, 24
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a5, .LCPI43_0
+; XTENSA-NEXT:    j .LBB43_2
+; XTENSA-NEXT:  .LBB43_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    or a10, a3, a3
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB43_4
+; XTENSA-NEXT:  .LBB43_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    l32i a12, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    blt a4, a8, .LBB43_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB43_1
+; XTENSA-NEXT:  .LBB43_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB43_2
+; XTENSA-ATOMIC-NEXT:  .LBB43_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB43_6
+; XTENSA-ATOMIC-NEXT:  .LBB43_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a12, a5, .LBB43_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB43_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB43_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB43_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB43_1
+; XTENSA-ATOMIC-NEXT:  .LBB43_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l8ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 24
+; XTENSA-NEXT:    srai a5, a8, 24
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a4, .LCPI44_0
+; XTENSA-NEXT:    j .LBB44_2
+; XTENSA-NEXT:  .LBB44_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB44_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l8ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB44_4
+; XTENSA-NEXT:  .LBB44_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 24
+; XTENSA-NEXT:    srai a8, a8, 24
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a5, a8, .LBB44_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB44_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB44_1
+; XTENSA-NEXT:  .LBB44_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    slli a12, a3, 24
+; XTENSA-ATOMIC-NEXT:    srai a12, a12, 24
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB44_2
+; XTENSA-ATOMIC-NEXT:  .LBB44_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB44_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB44_6
+; XTENSA-ATOMIC-NEXT:  .LBB44_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    slli a6, a7, 24
+; XTENSA-ATOMIC-NEXT:    srai a5, a6, 24
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a12, a5, .LBB44_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB44_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB44_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB44_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB44_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB44_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB44_1
+; XTENSA-ATOMIC-NEXT:  .LBB44_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a8, a3, a3
+; XTENSA-NEXT:    s32i a2, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a5, 255
+; XTENSA-NEXT:    and a4, a8, a5
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a6, .LCPI45_0
+; XTENSA-NEXT:    j .LBB45_2
+; XTENSA-NEXT:  .LBB45_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB45_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    l32i a10, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB45_4
+; XTENSA-NEXT:  .LBB45_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    and a8, a2, a5
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a4, a8, .LBB45_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB45_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB45_1
+; XTENSA-NEXT:  .LBB45_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB45_2
+; XTENSA-ATOMIC-NEXT:  .LBB45_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB45_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB45_6
+; XTENSA-ATOMIC-NEXT:  .LBB45_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a12, a5, .LBB45_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB45_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB45_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB45_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB45_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB45_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB45_1
+; XTENSA-ATOMIC-NEXT:  .LBB45_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a8, a3, a3
+; XTENSA-NEXT:    s32i a2, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a5, 255
+; XTENSA-NEXT:    and a4, a8, a5
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a6, .LCPI46_0
+; XTENSA-NEXT:    j .LBB46_2
+; XTENSA-NEXT:  .LBB46_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB46_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    l32i a10, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB46_4
+; XTENSA-NEXT:  .LBB46_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    and a8, a2, a5
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a4, a8, .LBB46_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB46_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB46_1
+; XTENSA-NEXT:  .LBB46_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB46_2
+; XTENSA-ATOMIC-NEXT:  .LBB46_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB46_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB46_6
+; XTENSA-ATOMIC-NEXT:  .LBB46_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a12, a5, .LBB46_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB46_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB46_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB46_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB46_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB46_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB46_1
+; XTENSA-ATOMIC-NEXT:  .LBB46_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    s32i a2, a1, 4 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a4, 255
+; XTENSA-NEXT:    or a5, a3, a3
+; XTENSA-NEXT:    and a8, a3, a4
+; XTENSA-NEXT:    s32i a8, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a3, .LCPI47_0
+; XTENSA-NEXT:    j .LBB47_2
+; XTENSA-NEXT:  .LBB47_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 8
+; XTENSA-NEXT:    l32i a10, a1, 4 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a3
+; XTENSA-NEXT:    l8ui a2, a1, 8
+; XTENSA-NEXT:    bnez a10, .LBB47_4
+; XTENSA-NEXT:  .LBB47_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 8
+; XTENSA-NEXT:    and a8, a2, a4
+; XTENSA-NEXT:    or a12, a5, a5
+; XTENSA-NEXT:    l32i a9, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    bgeu a9, a8, .LBB47_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB47_1
+; XTENSA-NEXT:  .LBB47_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB47_2
+; XTENSA-ATOMIC-NEXT:  .LBB47_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB47_6
+; XTENSA-ATOMIC-NEXT:  .LBB47_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a12, a5, .LBB47_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB47_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB47_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB47_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB47_1
+; XTENSA-ATOMIC-NEXT:  .LBB47_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    s32i a2, a1, 4 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a4, 255
+; XTENSA-NEXT:    or a5, a3, a3
+; XTENSA-NEXT:    and a8, a3, a4
+; XTENSA-NEXT:    s32i a8, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a3, .LCPI48_0
+; XTENSA-NEXT:    j .LBB48_2
+; XTENSA-NEXT:  .LBB48_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB48_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 8
+; XTENSA-NEXT:    l32i a10, a1, 4 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a3
+; XTENSA-NEXT:    l8ui a2, a1, 8
+; XTENSA-NEXT:    bnez a10, .LBB48_4
+; XTENSA-NEXT:  .LBB48_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 8
+; XTENSA-NEXT:    and a8, a2, a4
+; XTENSA-NEXT:    or a12, a5, a5
+; XTENSA-NEXT:    l32i a9, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    bgeu a9, a8, .LBB48_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB48_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB48_1
+; XTENSA-NEXT:  .LBB48_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB48_2
+; XTENSA-ATOMIC-NEXT:  .LBB48_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB48_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB48_6
+; XTENSA-ATOMIC-NEXT:  .LBB48_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a12, a5, .LBB48_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB48_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB48_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB48_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB48_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB48_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB48_1
+; XTENSA-ATOMIC-NEXT:  .LBB48_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a8, a3, a3
+; XTENSA-NEXT:    s32i a2, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a5, 255
+; XTENSA-NEXT:    and a4, a8, a5
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a6, .LCPI49_0
+; XTENSA-NEXT:    j .LBB49_2
+; XTENSA-NEXT:  .LBB49_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB49_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    l32i a10, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB49_4
+; XTENSA-NEXT:  .LBB49_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    and a8, a2, a5
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a4, a8, .LBB49_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB49_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB49_1
+; XTENSA-NEXT:  .LBB49_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB49_2
+; XTENSA-ATOMIC-NEXT:  .LBB49_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB49_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB49_6
+; XTENSA-ATOMIC-NEXT:  .LBB49_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a12, a5, .LBB49_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB49_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB49_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB49_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB49_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB49_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB49_1
+; XTENSA-ATOMIC-NEXT:  .LBB49_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_monotonic(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i8_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a8, a3, a3
+; XTENSA-NEXT:    s32i a2, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a5, 255
+; XTENSA-NEXT:    and a4, a8, a5
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a6, .LCPI50_0
+; XTENSA-NEXT:    j .LBB50_2
+; XTENSA-NEXT:  .LBB50_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB50_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    l32i a10, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB50_4
+; XTENSA-NEXT:  .LBB50_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    and a8, a2, a5
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a4, a8, .LBB50_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB50_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB50_1
+; XTENSA-NEXT:  .LBB50_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i8_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB50_2
+; XTENSA-ATOMIC-NEXT:  .LBB50_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB50_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB50_6
+; XTENSA-ATOMIC-NEXT:  .LBB50_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a12, a5, .LBB50_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB50_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB50_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB50_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB50_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB50_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB50_1
+; XTENSA-ATOMIC-NEXT:  .LBB50_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i8_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a8, a3, a3
+; XTENSA-NEXT:    s32i a2, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a5, 255
+; XTENSA-NEXT:    and a4, a8, a5
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a6, .LCPI51_0
+; XTENSA-NEXT:    j .LBB51_2
+; XTENSA-NEXT:  .LBB51_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    l32i a10, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB51_4
+; XTENSA-NEXT:  .LBB51_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    and a8, a2, a5
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a4, a8, .LBB51_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB51_1
+; XTENSA-NEXT:  .LBB51_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i8_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB51_2
+; XTENSA-ATOMIC-NEXT:  .LBB51_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB51_6
+; XTENSA-ATOMIC-NEXT:  .LBB51_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a12, a5, .LBB51_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB51_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB51_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB51_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB51_1
+; XTENSA-ATOMIC-NEXT:  .LBB51_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_release(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i8_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    s32i a2, a1, 4 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a4, 255
+; XTENSA-NEXT:    or a5, a3, a3
+; XTENSA-NEXT:    and a8, a3, a4
+; XTENSA-NEXT:    s32i a8, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a3, .LCPI52_0
+; XTENSA-NEXT:    j .LBB52_2
+; XTENSA-NEXT:  .LBB52_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB52_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 8
+; XTENSA-NEXT:    l32i a10, a1, 4 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a3
+; XTENSA-NEXT:    l8ui a2, a1, 8
+; XTENSA-NEXT:    bnez a10, .LBB52_4
+; XTENSA-NEXT:  .LBB52_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 8
+; XTENSA-NEXT:    and a8, a2, a4
+; XTENSA-NEXT:    or a12, a5, a5
+; XTENSA-NEXT:    l32i a9, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    bltu a9, a8, .LBB52_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB52_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB52_1
+; XTENSA-NEXT:  .LBB52_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i8_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB52_2
+; XTENSA-ATOMIC-NEXT:  .LBB52_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB52_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB52_6
+; XTENSA-ATOMIC-NEXT:  .LBB52_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a12, a5, .LBB52_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB52_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB52_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB52_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB52_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB52_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB52_1
+; XTENSA-ATOMIC-NEXT:  .LBB52_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i8_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    s32i a2, a1, 4 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a4, 255
+; XTENSA-NEXT:    or a5, a3, a3
+; XTENSA-NEXT:    and a8, a3, a4
+; XTENSA-NEXT:    s32i a8, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a3, .LCPI53_0
+; XTENSA-NEXT:    j .LBB53_2
+; XTENSA-NEXT:  .LBB53_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB53_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 8
+; XTENSA-NEXT:    l32i a10, a1, 4 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a3
+; XTENSA-NEXT:    l8ui a2, a1, 8
+; XTENSA-NEXT:    bnez a10, .LBB53_4
+; XTENSA-NEXT:  .LBB53_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 8
+; XTENSA-NEXT:    and a8, a2, a4
+; XTENSA-NEXT:    or a12, a5, a5
+; XTENSA-NEXT:    l32i a9, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    bltu a9, a8, .LBB53_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB53_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB53_1
+; XTENSA-NEXT:  .LBB53_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i8_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB53_2
+; XTENSA-ATOMIC-NEXT:  .LBB53_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB53_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB53_6
+; XTENSA-ATOMIC-NEXT:  .LBB53_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a12, a5, .LBB53_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB53_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB53_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB53_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB53_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB53_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB53_1
+; XTENSA-ATOMIC-NEXT:  .LBB53_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i8_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a8, a3, a3
+; XTENSA-NEXT:    s32i a2, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    l8ui a2, a2, 0
+; XTENSA-NEXT:    movi a5, 255
+; XTENSA-NEXT:    and a4, a8, a5
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a6, .LCPI54_0
+; XTENSA-NEXT:    j .LBB54_2
+; XTENSA-NEXT:  .LBB54_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB54_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    l32i a10, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    l8ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB54_4
+; XTENSA-NEXT:  .LBB54_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s8i a2, a1, 4
+; XTENSA-NEXT:    and a8, a2, a5
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a4, a8, .LBB54_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB54_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB54_1
+; XTENSA-NEXT:  .LBB54_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i8_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    and a12, a3, a9
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB54_2
+; XTENSA-ATOMIC-NEXT:  .LBB54_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB54_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB54_6
+; XTENSA-ATOMIC-NEXT:  .LBB54_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a7, a15
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    or a6, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a12, a5, .LBB54_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB54_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a7, a7
+; XTENSA-ATOMIC-NEXT:  .LBB54_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB54_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a6, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a7, a7
+; XTENSA-ATOMIC-NEXT:    and a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a7
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB54_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB54_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB54_1
+; XTENSA-ATOMIC-NEXT:  .LBB54_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i16 @atomicrmw_xchg_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI55_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI55_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB55_2
+; XTENSA-ATOMIC-NEXT:  .LBB55_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB55_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB55_4
+; XTENSA-ATOMIC-NEXT:  .LBB55_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB55_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB55_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB55_1
+; XTENSA-ATOMIC-NEXT:  .LBB55_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xchg_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI56_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI56_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB56_2
+; XTENSA-ATOMIC-NEXT:  .LBB56_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB56_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB56_4
+; XTENSA-ATOMIC-NEXT:  .LBB56_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB56_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB56_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB56_1
+; XTENSA-ATOMIC-NEXT:  .LBB56_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xchg_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI57_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI57_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB57_2
+; XTENSA-ATOMIC-NEXT:  .LBB57_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB57_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB57_4
+; XTENSA-ATOMIC-NEXT:  .LBB57_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB57_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB57_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB57_1
+; XTENSA-ATOMIC-NEXT:  .LBB57_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xchg_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI58_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI58_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB58_2
+; XTENSA-ATOMIC-NEXT:  .LBB58_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB58_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB58_4
+; XTENSA-ATOMIC-NEXT:  .LBB58_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB58_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB58_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB58_1
+; XTENSA-ATOMIC-NEXT:  .LBB58_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xchg_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI59_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI59_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a10, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a11, -4
+; XTENSA-ATOMIC-NEXT:    and a11, a2, a11
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB59_2
+; XTENSA-ATOMIC-NEXT:  .LBB59_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB59_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB59_4
+; XTENSA-ATOMIC-NEXT:  .LBB59_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a15, a10
+; XTENSA-ATOMIC-NEXT:    or a14, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a11, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB59_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB59_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB59_1
+; XTENSA-ATOMIC-NEXT:  .LBB59_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI60_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI60_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB60_2
+; XTENSA-ATOMIC-NEXT:  .LBB60_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB60_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB60_4
+; XTENSA-ATOMIC-NEXT:  .LBB60_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB60_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB60_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB60_1
+; XTENSA-ATOMIC-NEXT:  .LBB60_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI61_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI61_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB61_2
+; XTENSA-ATOMIC-NEXT:  .LBB61_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB61_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB61_4
+; XTENSA-ATOMIC-NEXT:  .LBB61_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB61_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB61_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB61_1
+; XTENSA-ATOMIC-NEXT:  .LBB61_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI62_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI62_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB62_2
+; XTENSA-ATOMIC-NEXT:  .LBB62_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB62_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB62_4
+; XTENSA-ATOMIC-NEXT:  .LBB62_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB62_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB62_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB62_1
+; XTENSA-ATOMIC-NEXT:  .LBB62_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI63_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI63_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB63_2
+; XTENSA-ATOMIC-NEXT:  .LBB63_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB63_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB63_4
+; XTENSA-ATOMIC-NEXT:  .LBB63_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB63_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB63_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB63_1
+; XTENSA-ATOMIC-NEXT:  .LBB63_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI64_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI64_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB64_2
+; XTENSA-ATOMIC-NEXT:  .LBB64_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB64_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB64_4
+; XTENSA-ATOMIC-NEXT:  .LBB64_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB64_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB64_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB64_1
+; XTENSA-ATOMIC-NEXT:  .LBB64_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI65_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI65_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB65_2
+; XTENSA-ATOMIC-NEXT:  .LBB65_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB65_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB65_4
+; XTENSA-ATOMIC-NEXT:  .LBB65_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB65_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB65_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB65_1
+; XTENSA-ATOMIC-NEXT:  .LBB65_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI66_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI66_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB66_2
+; XTENSA-ATOMIC-NEXT:  .LBB66_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB66_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB66_4
+; XTENSA-ATOMIC-NEXT:  .LBB66_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB66_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB66_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB66_1
+; XTENSA-ATOMIC-NEXT:  .LBB66_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI67_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI67_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB67_2
+; XTENSA-ATOMIC-NEXT:  .LBB67_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB67_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB67_4
+; XTENSA-ATOMIC-NEXT:  .LBB67_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB67_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB67_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB67_1
+; XTENSA-ATOMIC-NEXT:  .LBB67_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI68_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI68_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB68_2
+; XTENSA-ATOMIC-NEXT:  .LBB68_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB68_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB68_4
+; XTENSA-ATOMIC-NEXT:  .LBB68_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB68_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB68_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB68_1
+; XTENSA-ATOMIC-NEXT:  .LBB68_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI69_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI69_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a11, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -4
+; XTENSA-ATOMIC-NEXT:    and a12, a2, a12
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB69_2
+; XTENSA-ATOMIC-NEXT:  .LBB69_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB69_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB69_4
+; XTENSA-ATOMIC-NEXT:  .LBB69_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a11
+; XTENSA-ATOMIC-NEXT:    sub a6, a15, a9
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a10
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a12, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB69_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB69_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a13, a13
+; XTENSA-ATOMIC-NEXT:    j .LBB69_1
+; XTENSA-ATOMIC-NEXT:  .LBB69_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI70_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI70_0
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB70_2
+; XTENSA-ATOMIC-NEXT:  .LBB70_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB70_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB70_4
+; XTENSA-ATOMIC-NEXT:  .LBB70_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB70_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB70_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB70_1
+; XTENSA-ATOMIC-NEXT:  .LBB70_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI71_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI71_0
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB71_2
+; XTENSA-ATOMIC-NEXT:  .LBB71_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB71_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB71_4
+; XTENSA-ATOMIC-NEXT:  .LBB71_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB71_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB71_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB71_1
+; XTENSA-ATOMIC-NEXT:  .LBB71_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI72_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI72_0
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB72_2
+; XTENSA-ATOMIC-NEXT:  .LBB72_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB72_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB72_4
+; XTENSA-ATOMIC-NEXT:  .LBB72_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB72_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB72_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB72_1
+; XTENSA-ATOMIC-NEXT:  .LBB72_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI73_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI73_0
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB73_2
+; XTENSA-ATOMIC-NEXT:  .LBB73_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB73_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB73_4
+; XTENSA-ATOMIC-NEXT:  .LBB73_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB73_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB73_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB73_1
+; XTENSA-ATOMIC-NEXT:  .LBB73_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI74_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI74_0
+; XTENSA-ATOMIC-NEXT:    and a10, a3, a9
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a11
+; XTENSA-ATOMIC-NEXT:    or a9, a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB74_2
+; XTENSA-ATOMIC-NEXT:  .LBB74_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB74_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB74_4
+; XTENSA-ATOMIC-NEXT:  .LBB74_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB74_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB74_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB74_1
+; XTENSA-ATOMIC-NEXT:  .LBB74_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI75_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI75_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB75_2
+; XTENSA-ATOMIC-NEXT:  .LBB75_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB75_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB75_4
+; XTENSA-ATOMIC-NEXT:  .LBB75_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB75_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB75_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB75_1
+; XTENSA-ATOMIC-NEXT:  .LBB75_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI76_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI76_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB76_2
+; XTENSA-ATOMIC-NEXT:  .LBB76_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB76_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB76_4
+; XTENSA-ATOMIC-NEXT:  .LBB76_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB76_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB76_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB76_1
+; XTENSA-ATOMIC-NEXT:  .LBB76_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI77_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI77_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB77_2
+; XTENSA-ATOMIC-NEXT:  .LBB77_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB77_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB77_4
+; XTENSA-ATOMIC-NEXT:  .LBB77_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB77_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB77_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB77_1
+; XTENSA-ATOMIC-NEXT:  .LBB77_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI78_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI78_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB78_2
+; XTENSA-ATOMIC-NEXT:  .LBB78_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB78_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB78_4
+; XTENSA-ATOMIC-NEXT:  .LBB78_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB78_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB78_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB78_1
+; XTENSA-ATOMIC-NEXT:  .LBB78_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_nand_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI79_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_nand_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a10, .LCPI79_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a10
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a11, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a11
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a10
+; XTENSA-ATOMIC-NEXT:    movi a11, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a10, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    movi a15, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB79_2
+; XTENSA-ATOMIC-NEXT:  .LBB79_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB79_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a6, a6
+; XTENSA-ATOMIC-NEXT:    beqi a5, 1, .LBB79_4
+; XTENSA-ATOMIC-NEXT:  .LBB79_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a6, a7, a12
+; XTENSA-ATOMIC-NEXT:    and a5, a7, a9
+; XTENSA-ATOMIC-NEXT:    xor a5, a5, a11
+; XTENSA-ATOMIC-NEXT:    and a5, a5, a10
+; XTENSA-ATOMIC-NEXT:    or a6, a6, a5
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a6, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a5, a15, a15
+; XTENSA-ATOMIC-NEXT:    beq a6, a7, .LBB79_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB79_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a5, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB79_1
+; XTENSA-ATOMIC-NEXT:  .LBB79_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a6
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw nand ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI80_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI80_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB80_2
+; XTENSA-ATOMIC-NEXT:  .LBB80_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB80_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB80_4
+; XTENSA-ATOMIC-NEXT:  .LBB80_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB80_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB80_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB80_1
+; XTENSA-ATOMIC-NEXT:  .LBB80_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI81_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI81_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB81_2
+; XTENSA-ATOMIC-NEXT:  .LBB81_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB81_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB81_4
+; XTENSA-ATOMIC-NEXT:  .LBB81_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB81_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB81_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB81_1
+; XTENSA-ATOMIC-NEXT:  .LBB81_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI82_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI82_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB82_2
+; XTENSA-ATOMIC-NEXT:  .LBB82_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB82_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB82_4
+; XTENSA-ATOMIC-NEXT:  .LBB82_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB82_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB82_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB82_1
+; XTENSA-ATOMIC-NEXT:  .LBB82_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI83_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI83_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB83_2
+; XTENSA-ATOMIC-NEXT:  .LBB83_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB83_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB83_4
+; XTENSA-ATOMIC-NEXT:  .LBB83_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB83_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB83_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB83_1
+; XTENSA-ATOMIC-NEXT:  .LBB83_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI84_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI84_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB84_2
+; XTENSA-ATOMIC-NEXT:  .LBB84_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB84_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB84_4
+; XTENSA-ATOMIC-NEXT:  .LBB84_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB84_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB84_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB84_1
+; XTENSA-ATOMIC-NEXT:  .LBB84_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI85_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI85_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB85_2
+; XTENSA-ATOMIC-NEXT:  .LBB85_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB85_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB85_4
+; XTENSA-ATOMIC-NEXT:  .LBB85_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB85_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB85_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB85_1
+; XTENSA-ATOMIC-NEXT:  .LBB85_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI86_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI86_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB86_2
+; XTENSA-ATOMIC-NEXT:  .LBB86_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB86_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB86_4
+; XTENSA-ATOMIC-NEXT:  .LBB86_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB86_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB86_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB86_1
+; XTENSA-ATOMIC-NEXT:  .LBB86_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI87_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI87_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB87_2
+; XTENSA-ATOMIC-NEXT:  .LBB87_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB87_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB87_4
+; XTENSA-ATOMIC-NEXT:  .LBB87_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB87_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB87_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB87_1
+; XTENSA-ATOMIC-NEXT:  .LBB87_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI88_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI88_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB88_2
+; XTENSA-ATOMIC-NEXT:  .LBB88_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB88_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB88_4
+; XTENSA-ATOMIC-NEXT:  .LBB88_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB88_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB88_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB88_1
+; XTENSA-ATOMIC-NEXT:  .LBB88_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI89_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32r a8, .LCPI89_0
+; XTENSA-ATOMIC-NEXT:    and a9, a3, a8
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a10, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB89_2
+; XTENSA-ATOMIC-NEXT:  .LBB89_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB89_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a13, a13
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB89_4
+; XTENSA-ATOMIC-NEXT:  .LBB89_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a13, a14, a9
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a13, a14, .LBB89_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB89_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB89_1
+; XTENSA-ATOMIC-NEXT:  .LBB89_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    srai a5, a8, 16
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a4, .LCPI90_0
+; XTENSA-NEXT:    j .LBB90_2
+; XTENSA-NEXT:  .LBB90_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB90_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB90_4
+; XTENSA-NEXT:  .LBB90_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a5, a8, .LBB90_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB90_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB90_1
+; XTENSA-NEXT:  .LBB90_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI90_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB90_2
+; XTENSA-ATOMIC-NEXT:  .LBB90_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB90_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB90_6
+; XTENSA-ATOMIC-NEXT:  .LBB90_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a11, a6, .LBB90_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB90_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB90_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB90_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI90_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB90_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB90_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB90_1
+; XTENSA-ATOMIC-NEXT:  .LBB90_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    srai a5, a8, 16
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a4, .LCPI91_0
+; XTENSA-NEXT:    j .LBB91_2
+; XTENSA-NEXT:  .LBB91_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB91_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB91_4
+; XTENSA-NEXT:  .LBB91_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a5, a8, .LBB91_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB91_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB91_1
+; XTENSA-NEXT:  .LBB91_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI91_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB91_2
+; XTENSA-ATOMIC-NEXT:  .LBB91_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB91_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB91_6
+; XTENSA-ATOMIC-NEXT:  .LBB91_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a11, a6, .LBB91_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB91_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB91_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB91_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI91_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB91_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB91_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB91_1
+; XTENSA-ATOMIC-NEXT:  .LBB91_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a9, a2, a2
+; XTENSA-NEXT:    l16ui a2, a9, 0
+; XTENSA-NEXT:    s32i a3, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    or a3, a9, a9
+; XTENSA-NEXT:    srai a4, a8, 16
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a5, .LCPI92_0
+; XTENSA-NEXT:    j .LBB92_2
+; XTENSA-NEXT:  .LBB92_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB92_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    or a10, a3, a3
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB92_4
+; XTENSA-NEXT:  .LBB92_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 4
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    l32i a12, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    bge a4, a8, .LBB92_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB92_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB92_1
+; XTENSA-NEXT:  .LBB92_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI92_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB92_2
+; XTENSA-ATOMIC-NEXT:  .LBB92_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB92_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB92_6
+; XTENSA-ATOMIC-NEXT:  .LBB92_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a11, a6, .LBB92_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB92_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB92_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB92_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI92_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB92_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB92_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB92_1
+; XTENSA-ATOMIC-NEXT:  .LBB92_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a9, a2, a2
+; XTENSA-NEXT:    l16ui a2, a9, 0
+; XTENSA-NEXT:    s32i a3, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    or a3, a9, a9
+; XTENSA-NEXT:    srai a4, a8, 16
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a5, .LCPI93_0
+; XTENSA-NEXT:    j .LBB93_2
+; XTENSA-NEXT:  .LBB93_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB93_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    or a10, a3, a3
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB93_4
+; XTENSA-NEXT:  .LBB93_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 4
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    l32i a12, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    bge a4, a8, .LBB93_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB93_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB93_1
+; XTENSA-NEXT:  .LBB93_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI93_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB93_2
+; XTENSA-ATOMIC-NEXT:  .LBB93_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB93_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB93_6
+; XTENSA-ATOMIC-NEXT:  .LBB93_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a11, a6, .LBB93_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB93_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB93_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB93_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI93_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB93_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB93_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB93_1
+; XTENSA-ATOMIC-NEXT:  .LBB93_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    srai a5, a8, 16
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a4, .LCPI94_0
+; XTENSA-NEXT:    j .LBB94_2
+; XTENSA-NEXT:  .LBB94_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB94_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB94_4
+; XTENSA-NEXT:  .LBB94_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a5, a8, .LBB94_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB94_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB94_1
+; XTENSA-NEXT:  .LBB94_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI94_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB94_2
+; XTENSA-ATOMIC-NEXT:  .LBB94_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB94_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB94_6
+; XTENSA-ATOMIC-NEXT:  .LBB94_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a11, a6, .LBB94_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB94_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB94_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB94_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI94_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB94_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB94_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB94_1
+; XTENSA-ATOMIC-NEXT:  .LBB94_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    srai a5, a8, 16
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a4, .LCPI95_0
+; XTENSA-NEXT:    j .LBB95_2
+; XTENSA-NEXT:  .LBB95_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB95_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB95_4
+; XTENSA-NEXT:  .LBB95_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a5, a8, .LBB95_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB95_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB95_1
+; XTENSA-NEXT:  .LBB95_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI95_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB95_2
+; XTENSA-ATOMIC-NEXT:  .LBB95_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB95_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB95_6
+; XTENSA-ATOMIC-NEXT:  .LBB95_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a11, a6, .LBB95_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB95_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB95_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB95_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI95_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB95_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB95_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB95_1
+; XTENSA-ATOMIC-NEXT:  .LBB95_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    srai a5, a8, 16
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a4, .LCPI96_0
+; XTENSA-NEXT:    j .LBB96_2
+; XTENSA-NEXT:  .LBB96_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB96_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB96_4
+; XTENSA-NEXT:  .LBB96_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a5, a8, .LBB96_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB96_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB96_1
+; XTENSA-NEXT:  .LBB96_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI96_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB96_2
+; XTENSA-ATOMIC-NEXT:  .LBB96_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB96_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB96_6
+; XTENSA-ATOMIC-NEXT:  .LBB96_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a11, a6, .LBB96_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB96_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB96_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB96_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI96_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB96_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB96_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB96_1
+; XTENSA-ATOMIC-NEXT:  .LBB96_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a9, a2, a2
+; XTENSA-NEXT:    l16ui a2, a9, 0
+; XTENSA-NEXT:    s32i a3, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    or a3, a9, a9
+; XTENSA-NEXT:    srai a4, a8, 16
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a5, .LCPI97_0
+; XTENSA-NEXT:    j .LBB97_2
+; XTENSA-NEXT:  .LBB97_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB97_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    or a10, a3, a3
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB97_4
+; XTENSA-NEXT:  .LBB97_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 4
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    l32i a12, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    blt a4, a8, .LBB97_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB97_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB97_1
+; XTENSA-NEXT:  .LBB97_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI97_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB97_2
+; XTENSA-ATOMIC-NEXT:  .LBB97_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB97_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB97_6
+; XTENSA-ATOMIC-NEXT:  .LBB97_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a11, a6, .LBB97_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB97_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB97_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB97_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI97_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB97_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB97_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB97_1
+; XTENSA-ATOMIC-NEXT:  .LBB97_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a9, a2, a2
+; XTENSA-NEXT:    l16ui a2, a9, 0
+; XTENSA-NEXT:    s32i a3, a1, 0 # 4-byte Folded Spill
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    or a3, a9, a9
+; XTENSA-NEXT:    srai a4, a8, 16
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a5, .LCPI98_0
+; XTENSA-NEXT:    j .LBB98_2
+; XTENSA-NEXT:  .LBB98_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB98_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 4
+; XTENSA-NEXT:    or a10, a3, a3
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 4
+; XTENSA-NEXT:    bnez a10, .LBB98_4
+; XTENSA-NEXT:  .LBB98_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 4
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    l32i a12, a1, 0 # 4-byte Folded Reload
+; XTENSA-NEXT:    blt a4, a8, .LBB98_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB98_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB98_1
+; XTENSA-NEXT:  .LBB98_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI98_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB98_2
+; XTENSA-ATOMIC-NEXT:  .LBB98_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB98_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB98_6
+; XTENSA-ATOMIC-NEXT:  .LBB98_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a11, a6, .LBB98_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB98_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB98_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB98_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI98_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB98_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB98_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB98_1
+; XTENSA-ATOMIC-NEXT:  .LBB98_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    slli a8, a3, 16
+; XTENSA-NEXT:    srai a5, a8, 16
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a4, .LCPI99_0
+; XTENSA-NEXT:    j .LBB99_2
+; XTENSA-NEXT:  .LBB99_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB99_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB99_4
+; XTENSA-NEXT:  .LBB99_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    slli a8, a2, 16
+; XTENSA-NEXT:    srai a8, a8, 16
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a5, a8, .LBB99_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB99_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB99_1
+; XTENSA-NEXT:  .LBB99_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI99_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    slli a11, a3, 16
+; XTENSA-ATOMIC-NEXT:    srai a11, a11, 16
+; XTENSA-ATOMIC-NEXT:    movi a12, 0
+; XTENSA-ATOMIC-NEXT:    movi a13, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB99_2
+; XTENSA-ATOMIC-NEXT:  .LBB99_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB99_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a15
+; XTENSA-ATOMIC-NEXT:    beqi a7, 1, .LBB99_6
+; XTENSA-ATOMIC-NEXT:  .LBB99_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a14
+; XTENSA-ATOMIC-NEXT:    slli a7, a15, 16
+; XTENSA-ATOMIC-NEXT:    srai a6, a7, 16
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a11, a6, .LBB99_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB99_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB99_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB99_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a15, .LCPI99_0
+; XTENSA-ATOMIC-NEXT:    and a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a15, a15
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a15
+; XTENSA-ATOMIC-NEXT:    wsr a14, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a15, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:    beq a15, a14, .LBB99_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB99_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB99_1
+; XTENSA-ATOMIC-NEXT:  .LBB99_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a15
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a5, .LCPI100_1
+; XTENSA-NEXT:    j .LBB100_2
+; XTENSA-NEXT:  .LBB100_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB100_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB100_4
+; XTENSA-NEXT:  .LBB100_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI100_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a9, a8, .LBB100_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB100_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB100_1
+; XTENSA-NEXT:  .LBB100_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI100_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB100_2
+; XTENSA-ATOMIC-NEXT:  .LBB100_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB100_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB100_6
+; XTENSA-ATOMIC-NEXT:  .LBB100_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI100_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a6, a5, .LBB100_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB100_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB100_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB100_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB100_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB100_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB100_1
+; XTENSA-ATOMIC-NEXT:  .LBB100_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a5, .LCPI101_1
+; XTENSA-NEXT:    j .LBB101_2
+; XTENSA-NEXT:  .LBB101_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB101_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB101_4
+; XTENSA-NEXT:  .LBB101_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI101_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a9, a8, .LBB101_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB101_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB101_1
+; XTENSA-NEXT:  .LBB101_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI101_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB101_2
+; XTENSA-ATOMIC-NEXT:  .LBB101_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB101_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB101_6
+; XTENSA-ATOMIC-NEXT:  .LBB101_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI101_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a6, a5, .LBB101_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB101_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB101_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB101_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB101_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB101_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB101_1
+; XTENSA-ATOMIC-NEXT:  .LBB101_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l16ui a2, a5, 0
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a4, .LCPI102_1
+; XTENSA-NEXT:    j .LBB102_2
+; XTENSA-NEXT:  .LBB102_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB102_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB102_4
+; XTENSA-NEXT:  .LBB102_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI102_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a9, a8, .LBB102_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB102_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB102_1
+; XTENSA-NEXT:  .LBB102_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI102_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB102_2
+; XTENSA-ATOMIC-NEXT:  .LBB102_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB102_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB102_6
+; XTENSA-ATOMIC-NEXT:  .LBB102_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI102_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a6, a5, .LBB102_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB102_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB102_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB102_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB102_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB102_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB102_1
+; XTENSA-ATOMIC-NEXT:  .LBB102_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l16ui a2, a5, 0
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a4, .LCPI103_1
+; XTENSA-NEXT:    j .LBB103_2
+; XTENSA-NEXT:  .LBB103_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB103_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB103_4
+; XTENSA-NEXT:  .LBB103_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI103_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a9, a8, .LBB103_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB103_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB103_1
+; XTENSA-NEXT:  .LBB103_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI103_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB103_2
+; XTENSA-ATOMIC-NEXT:  .LBB103_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB103_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB103_6
+; XTENSA-ATOMIC-NEXT:  .LBB103_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI103_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a6, a5, .LBB103_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB103_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB103_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB103_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB103_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB103_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB103_1
+; XTENSA-ATOMIC-NEXT:  .LBB103_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI104_1
+; XTENSA-NEXT:    j .LBB104_2
+; XTENSA-NEXT:  .LBB104_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB104_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB104_4
+; XTENSA-NEXT:  .LBB104_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI104_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a9, a8, .LBB104_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB104_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB104_1
+; XTENSA-NEXT:  .LBB104_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI104_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB104_2
+; XTENSA-ATOMIC-NEXT:  .LBB104_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB104_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB104_6
+; XTENSA-ATOMIC-NEXT:  .LBB104_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI104_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a6, a5, .LBB104_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB104_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB104_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB104_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB104_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB104_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB104_1
+; XTENSA-ATOMIC-NEXT:  .LBB104_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_monotonic(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i16_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a5, .LCPI105_1
+; XTENSA-NEXT:    j .LBB105_2
+; XTENSA-NEXT:  .LBB105_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB105_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB105_4
+; XTENSA-NEXT:  .LBB105_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI105_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a9, a8, .LBB105_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB105_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB105_1
+; XTENSA-NEXT:  .LBB105_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i16_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI105_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB105_2
+; XTENSA-ATOMIC-NEXT:  .LBB105_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB105_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB105_6
+; XTENSA-ATOMIC-NEXT:  .LBB105_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI105_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a6, a5, .LBB105_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB105_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB105_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB105_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB105_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB105_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB105_1
+; XTENSA-ATOMIC-NEXT:  .LBB105_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i16_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a5, .LCPI106_1
+; XTENSA-NEXT:    j .LBB106_2
+; XTENSA-NEXT:  .LBB106_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB106_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB106_4
+; XTENSA-NEXT:  .LBB106_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI106_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a9, a8, .LBB106_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB106_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB106_1
+; XTENSA-NEXT:  .LBB106_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i16_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI106_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB106_2
+; XTENSA-ATOMIC-NEXT:  .LBB106_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB106_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB106_6
+; XTENSA-ATOMIC-NEXT:  .LBB106_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI106_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a6, a5, .LBB106_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB106_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB106_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB106_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB106_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB106_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB106_1
+; XTENSA-ATOMIC-NEXT:  .LBB106_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_release(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i16_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l16ui a2, a5, 0
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a4, .LCPI107_1
+; XTENSA-NEXT:    j .LBB107_2
+; XTENSA-NEXT:  .LBB107_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB107_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB107_4
+; XTENSA-NEXT:  .LBB107_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI107_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a9, a8, .LBB107_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB107_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB107_1
+; XTENSA-NEXT:  .LBB107_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i16_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI107_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB107_2
+; XTENSA-ATOMIC-NEXT:  .LBB107_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB107_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB107_6
+; XTENSA-ATOMIC-NEXT:  .LBB107_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI107_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a6, a5, .LBB107_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB107_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB107_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB107_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB107_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB107_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB107_1
+; XTENSA-ATOMIC-NEXT:  .LBB107_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i16_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l16ui a2, a5, 0
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a4, .LCPI108_1
+; XTENSA-NEXT:    j .LBB108_2
+; XTENSA-NEXT:  .LBB108_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB108_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB108_4
+; XTENSA-NEXT:  .LBB108_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI108_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a9, a8, .LBB108_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB108_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB108_1
+; XTENSA-NEXT:  .LBB108_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i16_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI108_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB108_2
+; XTENSA-ATOMIC-NEXT:  .LBB108_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB108_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB108_6
+; XTENSA-ATOMIC-NEXT:  .LBB108_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI108_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a6, a5, .LBB108_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB108_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB108_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB108_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB108_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB108_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB108_1
+; XTENSA-ATOMIC-NEXT:  .LBB108_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i16_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l16ui a2, a6, 0
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI109_1
+; XTENSA-NEXT:    j .LBB109_2
+; XTENSA-NEXT:  .LBB109_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB109_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB109_4
+; XTENSA-NEXT:  .LBB109_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    l32r a8, .LCPI109_0
+; XTENSA-NEXT:    and a9, a3, a8
+; XTENSA-NEXT:    s16i a2, a1, 0
+; XTENSA-NEXT:    and a8, a2, a8
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a9, a8, .LBB109_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB109_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB109_1
+; XTENSA-NEXT:  .LBB109_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i16_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI109_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a13, a10, 0
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB109_2
+; XTENSA-ATOMIC-NEXT:  .LBB109_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB109_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a14, a14
+; XTENSA-ATOMIC-NEXT:    beqi a15, 1, .LBB109_6
+; XTENSA-ATOMIC-NEXT:  .LBB109_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a14, .LCPI109_0
+; XTENSA-ATOMIC-NEXT:    and a6, a3, a14
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a15, a13
+; XTENSA-ATOMIC-NEXT:    and a5, a15, a14
+; XTENSA-ATOMIC-NEXT:    or a7, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a6, a5, .LBB109_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB109_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a15, a15
+; XTENSA-ATOMIC-NEXT:  .LBB109_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB109_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a14, a7, a14
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a14, a14
+; XTENSA-ATOMIC-NEXT:    and a15, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a14
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a15, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a14, a13, .LBB109_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB109_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB109_1
+; XTENSA-ATOMIC-NEXT:  .LBB109_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i32 @atomicrmw_xchg_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI110_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB110_2
+; XTENSA-ATOMIC-NEXT:  .LBB110_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB110_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB110_4
+; XTENSA-ATOMIC-NEXT:  .LBB110_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB110_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB110_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB110_1
+; XTENSA-ATOMIC-NEXT:  .LBB110_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xchg_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI111_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB111_2
+; XTENSA-ATOMIC-NEXT:  .LBB111_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB111_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB111_4
+; XTENSA-ATOMIC-NEXT:  .LBB111_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB111_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB111_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB111_1
+; XTENSA-ATOMIC-NEXT:  .LBB111_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xchg_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI112_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB112_2
+; XTENSA-ATOMIC-NEXT:  .LBB112_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB112_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB112_4
+; XTENSA-ATOMIC-NEXT:  .LBB112_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB112_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB112_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB112_1
+; XTENSA-ATOMIC-NEXT:  .LBB112_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xchg_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI113_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB113_2
+; XTENSA-ATOMIC-NEXT:  .LBB113_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB113_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB113_4
+; XTENSA-ATOMIC-NEXT:  .LBB113_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB113_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB113_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB113_1
+; XTENSA-ATOMIC-NEXT:  .LBB113_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xchg_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xchg_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI114_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xchg_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB114_2
+; XTENSA-ATOMIC-NEXT:  .LBB114_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB114_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB114_4
+; XTENSA-ATOMIC-NEXT:  .LBB114_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB114_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB114_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB114_1
+; XTENSA-ATOMIC-NEXT:  .LBB114_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xchg ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI115_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB115_2
+; XTENSA-ATOMIC-NEXT:  .LBB115_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB115_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB115_4
+; XTENSA-ATOMIC-NEXT:  .LBB115_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    add a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB115_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB115_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB115_1
+; XTENSA-ATOMIC-NEXT:  .LBB115_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI116_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB116_2
+; XTENSA-ATOMIC-NEXT:  .LBB116_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB116_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB116_4
+; XTENSA-ATOMIC-NEXT:  .LBB116_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    add a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB116_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB116_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB116_1
+; XTENSA-ATOMIC-NEXT:  .LBB116_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI117_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB117_2
+; XTENSA-ATOMIC-NEXT:  .LBB117_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB117_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB117_4
+; XTENSA-ATOMIC-NEXT:  .LBB117_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    add a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB117_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB117_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB117_1
+; XTENSA-ATOMIC-NEXT:  .LBB117_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI118_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB118_2
+; XTENSA-ATOMIC-NEXT:  .LBB118_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB118_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB118_4
+; XTENSA-ATOMIC-NEXT:  .LBB118_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    add a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB118_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB118_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB118_1
+; XTENSA-ATOMIC-NEXT:  .LBB118_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_add_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI119_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_add_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB119_2
+; XTENSA-ATOMIC-NEXT:  .LBB119_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB119_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB119_4
+; XTENSA-ATOMIC-NEXT:  .LBB119_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    add a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB119_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB119_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB119_1
+; XTENSA-ATOMIC-NEXT:  .LBB119_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw add ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI120_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB120_2
+; XTENSA-ATOMIC-NEXT:  .LBB120_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB120_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB120_4
+; XTENSA-ATOMIC-NEXT:  .LBB120_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    sub a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB120_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB120_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB120_1
+; XTENSA-ATOMIC-NEXT:  .LBB120_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI121_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB121_2
+; XTENSA-ATOMIC-NEXT:  .LBB121_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB121_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB121_4
+; XTENSA-ATOMIC-NEXT:  .LBB121_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    sub a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB121_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB121_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB121_1
+; XTENSA-ATOMIC-NEXT:  .LBB121_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI122_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB122_2
+; XTENSA-ATOMIC-NEXT:  .LBB122_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB122_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB122_4
+; XTENSA-ATOMIC-NEXT:  .LBB122_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    sub a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB122_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB122_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB122_1
+; XTENSA-ATOMIC-NEXT:  .LBB122_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI123_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB123_2
+; XTENSA-ATOMIC-NEXT:  .LBB123_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB123_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB123_4
+; XTENSA-ATOMIC-NEXT:  .LBB123_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    sub a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB123_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB123_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB123_1
+; XTENSA-ATOMIC-NEXT:  .LBB123_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_sub_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI124_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_sub_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB124_2
+; XTENSA-ATOMIC-NEXT:  .LBB124_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB124_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB124_4
+; XTENSA-ATOMIC-NEXT:  .LBB124_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    sub a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB124_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB124_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB124_1
+; XTENSA-ATOMIC-NEXT:  .LBB124_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw sub ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI125_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB125_2
+; XTENSA-ATOMIC-NEXT:  .LBB125_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB125_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB125_4
+; XTENSA-ATOMIC-NEXT:  .LBB125_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB125_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB125_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB125_1
+; XTENSA-ATOMIC-NEXT:  .LBB125_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI126_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB126_2
+; XTENSA-ATOMIC-NEXT:  .LBB126_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB126_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB126_4
+; XTENSA-ATOMIC-NEXT:  .LBB126_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB126_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB126_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB126_1
+; XTENSA-ATOMIC-NEXT:  .LBB126_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI127_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB127_2
+; XTENSA-ATOMIC-NEXT:  .LBB127_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB127_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB127_4
+; XTENSA-ATOMIC-NEXT:  .LBB127_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB127_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB127_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB127_1
+; XTENSA-ATOMIC-NEXT:  .LBB127_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI128_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB128_2
+; XTENSA-ATOMIC-NEXT:  .LBB128_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB128_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB128_4
+; XTENSA-ATOMIC-NEXT:  .LBB128_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB128_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB128_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB128_1
+; XTENSA-ATOMIC-NEXT:  .LBB128_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_and_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI129_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_and_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB129_2
+; XTENSA-ATOMIC-NEXT:  .LBB129_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB129_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB129_4
+; XTENSA-ATOMIC-NEXT:  .LBB129_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB129_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB129_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB129_1
+; XTENSA-ATOMIC-NEXT:  .LBB129_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw and ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+;define i32 @atomicrmw_nand_i32_monotonic(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b monotonic
+;  ret i32 %res
+;}
+;
+;define i32 @atomicrmw_nand_i32_acquire(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b acquire
+;  ret i32 %res
+;}
+;
+;define i32 @atomicrmw_nand_i32_release(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b release
+;  ret i32 %res
+;}
+;
+;define i32 @atomicrmw_nand_i32_acq_rel(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b acq_rel
+;  ret i32 %res
+;}
+;
+;define i32 @atomicrmw_nand_i32_seq_cst(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b seq_cst
+;  ret i32 %res
+;}
+
+define i32 @atomicrmw_or_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI130_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB130_2
+; XTENSA-ATOMIC-NEXT:  .LBB130_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB130_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB130_4
+; XTENSA-ATOMIC-NEXT:  .LBB130_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB130_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB130_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB130_1
+; XTENSA-ATOMIC-NEXT:  .LBB130_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_or_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI131_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB131_2
+; XTENSA-ATOMIC-NEXT:  .LBB131_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB131_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB131_4
+; XTENSA-ATOMIC-NEXT:  .LBB131_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB131_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB131_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB131_1
+; XTENSA-ATOMIC-NEXT:  .LBB131_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_or_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI132_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB132_2
+; XTENSA-ATOMIC-NEXT:  .LBB132_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB132_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB132_4
+; XTENSA-ATOMIC-NEXT:  .LBB132_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB132_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB132_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB132_1
+; XTENSA-ATOMIC-NEXT:  .LBB132_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_or_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI133_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB133_2
+; XTENSA-ATOMIC-NEXT:  .LBB133_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB133_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB133_4
+; XTENSA-ATOMIC-NEXT:  .LBB133_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB133_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB133_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB133_1
+; XTENSA-ATOMIC-NEXT:  .LBB133_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_or_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_or_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI134_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_or_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB134_2
+; XTENSA-ATOMIC-NEXT:  .LBB134_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB134_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB134_4
+; XTENSA-ATOMIC-NEXT:  .LBB134_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB134_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB134_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB134_1
+; XTENSA-ATOMIC-NEXT:  .LBB134_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw or ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI135_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB135_2
+; XTENSA-ATOMIC-NEXT:  .LBB135_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB135_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB135_4
+; XTENSA-ATOMIC-NEXT:  .LBB135_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB135_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB135_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB135_1
+; XTENSA-ATOMIC-NEXT:  .LBB135_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 2
+; XTENSA-NEXT:    l32r a8, .LCPI136_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB136_2
+; XTENSA-ATOMIC-NEXT:  .LBB136_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB136_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB136_4
+; XTENSA-ATOMIC-NEXT:  .LBB136_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB136_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB136_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB136_1
+; XTENSA-ATOMIC-NEXT:  .LBB136_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI137_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB137_2
+; XTENSA-ATOMIC-NEXT:  .LBB137_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB137_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB137_4
+; XTENSA-ATOMIC-NEXT:  .LBB137_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB137_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB137_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB137_1
+; XTENSA-ATOMIC-NEXT:  .LBB137_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 4
+; XTENSA-NEXT:    l32r a8, .LCPI138_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB138_2
+; XTENSA-ATOMIC-NEXT:  .LBB138_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB138_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB138_4
+; XTENSA-ATOMIC-NEXT:  .LBB138_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB138_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB138_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB138_1
+; XTENSA-ATOMIC-NEXT:  .LBB138_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_xor_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a11, a3, a3
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI139_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_xor_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB139_2
+; XTENSA-ATOMIC-NEXT:  .LBB139_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB139_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB139_4
+; XTENSA-ATOMIC-NEXT:  .LBB139_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a8, a11, a3
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB139_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB139_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB139_1
+; XTENSA-ATOMIC-NEXT:  .LBB139_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw xor ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a5, .LCPI140_0
+; XTENSA-NEXT:    j .LBB140_2
+; XTENSA-NEXT:  .LBB140_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB140_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB140_4
+; XTENSA-NEXT:  .LBB140_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a3, a2, .LBB140_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB140_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB140_1
+; XTENSA-NEXT:  .LBB140_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB140_2
+; XTENSA-ATOMIC-NEXT:  .LBB140_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB140_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB140_6
+; XTENSA-ATOMIC-NEXT:  .LBB140_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a3, a11, .LBB140_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB140_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB140_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB140_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB140_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB140_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB140_1
+; XTENSA-ATOMIC-NEXT:  .LBB140_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a5, .LCPI141_0
+; XTENSA-NEXT:    j .LBB141_2
+; XTENSA-NEXT:  .LBB141_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB141_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB141_4
+; XTENSA-NEXT:  .LBB141_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a3, a2, .LBB141_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB141_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB141_1
+; XTENSA-NEXT:  .LBB141_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB141_2
+; XTENSA-ATOMIC-NEXT:  .LBB141_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB141_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB141_6
+; XTENSA-ATOMIC-NEXT:  .LBB141_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a3, a11, .LBB141_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB141_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB141_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB141_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB141_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB141_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB141_1
+; XTENSA-ATOMIC-NEXT:  .LBB141_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l32i a2, a5, 0
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a4, .LCPI142_0
+; XTENSA-NEXT:    j .LBB142_2
+; XTENSA-NEXT:  .LBB142_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB142_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB142_4
+; XTENSA-NEXT:  .LBB142_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a3, a2, .LBB142_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB142_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB142_1
+; XTENSA-NEXT:  .LBB142_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB142_2
+; XTENSA-ATOMIC-NEXT:  .LBB142_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB142_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB142_6
+; XTENSA-ATOMIC-NEXT:  .LBB142_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a3, a11, .LBB142_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB142_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB142_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB142_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB142_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB142_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB142_1
+; XTENSA-ATOMIC-NEXT:  .LBB142_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l32i a2, a5, 0
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a4, .LCPI143_0
+; XTENSA-NEXT:    j .LBB143_2
+; XTENSA-NEXT:  .LBB143_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB143_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB143_4
+; XTENSA-NEXT:  .LBB143_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a3, a2, .LBB143_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB143_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB143_1
+; XTENSA-NEXT:  .LBB143_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB143_2
+; XTENSA-ATOMIC-NEXT:  .LBB143_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB143_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB143_6
+; XTENSA-ATOMIC-NEXT:  .LBB143_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a3, a11, .LBB143_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB143_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB143_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB143_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB143_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB143_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB143_1
+; XTENSA-ATOMIC-NEXT:  .LBB143_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_max_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI144_0
+; XTENSA-NEXT:    j .LBB144_2
+; XTENSA-NEXT:  .LBB144_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB144_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB144_4
+; XTENSA-NEXT:  .LBB144_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bge a3, a2, .LBB144_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB144_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB144_1
+; XTENSA-NEXT:  .LBB144_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_max_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB144_2
+; XTENSA-ATOMIC-NEXT:  .LBB144_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB144_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB144_6
+; XTENSA-ATOMIC-NEXT:  .LBB144_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bge a3, a11, .LBB144_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB144_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB144_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB144_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB144_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB144_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB144_1
+; XTENSA-ATOMIC-NEXT:  .LBB144_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw max ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a5, .LCPI145_0
+; XTENSA-NEXT:    j .LBB145_2
+; XTENSA-NEXT:  .LBB145_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB145_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB145_4
+; XTENSA-NEXT:  .LBB145_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a3, a2, .LBB145_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB145_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB145_1
+; XTENSA-NEXT:  .LBB145_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB145_2
+; XTENSA-ATOMIC-NEXT:  .LBB145_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB145_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB145_6
+; XTENSA-ATOMIC-NEXT:  .LBB145_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a3, a11, .LBB145_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB145_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB145_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB145_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB145_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB145_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB145_1
+; XTENSA-ATOMIC-NEXT:  .LBB145_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a5, .LCPI146_0
+; XTENSA-NEXT:    j .LBB146_2
+; XTENSA-NEXT:  .LBB146_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB146_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB146_4
+; XTENSA-NEXT:  .LBB146_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a3, a2, .LBB146_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB146_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB146_1
+; XTENSA-NEXT:  .LBB146_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB146_2
+; XTENSA-ATOMIC-NEXT:  .LBB146_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB146_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB146_6
+; XTENSA-ATOMIC-NEXT:  .LBB146_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a3, a11, .LBB146_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB146_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB146_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB146_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB146_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB146_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB146_1
+; XTENSA-ATOMIC-NEXT:  .LBB146_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l32i a2, a5, 0
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a4, .LCPI147_0
+; XTENSA-NEXT:    j .LBB147_2
+; XTENSA-NEXT:  .LBB147_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB147_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB147_4
+; XTENSA-NEXT:  .LBB147_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a3, a2, .LBB147_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB147_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB147_1
+; XTENSA-NEXT:  .LBB147_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB147_2
+; XTENSA-ATOMIC-NEXT:  .LBB147_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB147_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB147_6
+; XTENSA-ATOMIC-NEXT:  .LBB147_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a3, a11, .LBB147_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB147_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB147_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB147_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB147_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB147_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB147_1
+; XTENSA-ATOMIC-NEXT:  .LBB147_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l32i a2, a5, 0
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a4, .LCPI148_0
+; XTENSA-NEXT:    j .LBB148_2
+; XTENSA-NEXT:  .LBB148_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB148_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB148_4
+; XTENSA-NEXT:  .LBB148_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a3, a2, .LBB148_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB148_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB148_1
+; XTENSA-NEXT:  .LBB148_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB148_2
+; XTENSA-ATOMIC-NEXT:  .LBB148_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB148_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB148_6
+; XTENSA-ATOMIC-NEXT:  .LBB148_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a3, a11, .LBB148_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB148_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB148_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB148_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB148_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB148_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB148_1
+; XTENSA-ATOMIC-NEXT:  .LBB148_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_min_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI149_0
+; XTENSA-NEXT:    j .LBB149_2
+; XTENSA-NEXT:  .LBB149_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB149_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB149_4
+; XTENSA-NEXT:  .LBB149_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    blt a3, a2, .LBB149_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB149_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB149_1
+; XTENSA-NEXT:  .LBB149_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_min_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB149_2
+; XTENSA-ATOMIC-NEXT:  .LBB149_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB149_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB149_6
+; XTENSA-ATOMIC-NEXT:  .LBB149_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    blt a3, a11, .LBB149_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB149_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB149_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB149_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB149_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB149_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB149_1
+; XTENSA-ATOMIC-NEXT:  .LBB149_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw min ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a5, .LCPI150_0
+; XTENSA-NEXT:    j .LBB150_2
+; XTENSA-NEXT:  .LBB150_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB150_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB150_4
+; XTENSA-NEXT:  .LBB150_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a3, a2, .LBB150_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB150_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB150_1
+; XTENSA-NEXT:  .LBB150_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB150_2
+; XTENSA-ATOMIC-NEXT:  .LBB150_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB150_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB150_6
+; XTENSA-ATOMIC-NEXT:  .LBB150_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a3, a11, .LBB150_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB150_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB150_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB150_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB150_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB150_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB150_1
+; XTENSA-ATOMIC-NEXT:  .LBB150_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a5, .LCPI151_0
+; XTENSA-NEXT:    j .LBB151_2
+; XTENSA-NEXT:  .LBB151_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB151_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB151_4
+; XTENSA-NEXT:  .LBB151_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a3, a2, .LBB151_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB151_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB151_1
+; XTENSA-NEXT:  .LBB151_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB151_2
+; XTENSA-ATOMIC-NEXT:  .LBB151_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB151_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB151_6
+; XTENSA-ATOMIC-NEXT:  .LBB151_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a3, a11, .LBB151_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB151_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB151_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB151_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB151_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB151_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB151_1
+; XTENSA-ATOMIC-NEXT:  .LBB151_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l32i a2, a5, 0
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a4, .LCPI152_0
+; XTENSA-NEXT:    j .LBB152_2
+; XTENSA-NEXT:  .LBB152_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB152_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB152_4
+; XTENSA-NEXT:  .LBB152_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a3, a2, .LBB152_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB152_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB152_1
+; XTENSA-NEXT:  .LBB152_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB152_2
+; XTENSA-ATOMIC-NEXT:  .LBB152_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB152_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB152_6
+; XTENSA-ATOMIC-NEXT:  .LBB152_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a3, a11, .LBB152_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB152_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB152_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB152_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB152_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB152_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB152_1
+; XTENSA-ATOMIC-NEXT:  .LBB152_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l32i a2, a5, 0
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a4, .LCPI153_0
+; XTENSA-NEXT:    j .LBB153_2
+; XTENSA-NEXT:  .LBB153_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB153_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB153_4
+; XTENSA-NEXT:  .LBB153_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a3, a2, .LBB153_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB153_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB153_1
+; XTENSA-NEXT:  .LBB153_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB153_2
+; XTENSA-ATOMIC-NEXT:  .LBB153_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB153_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB153_6
+; XTENSA-ATOMIC-NEXT:  .LBB153_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a3, a11, .LBB153_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB153_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB153_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB153_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB153_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB153_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB153_1
+; XTENSA-ATOMIC-NEXT:  .LBB153_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umax_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI154_0
+; XTENSA-NEXT:    j .LBB154_2
+; XTENSA-NEXT:  .LBB154_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB154_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB154_4
+; XTENSA-NEXT:  .LBB154_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bgeu a3, a2, .LBB154_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB154_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB154_1
+; XTENSA-NEXT:  .LBB154_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umax_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB154_2
+; XTENSA-ATOMIC-NEXT:  .LBB154_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB154_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB154_6
+; XTENSA-ATOMIC-NEXT:  .LBB154_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bgeu a3, a11, .LBB154_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB154_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB154_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB154_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB154_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB154_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB154_1
+; XTENSA-ATOMIC-NEXT:  .LBB154_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umax ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_monotonic(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 0
+; XTENSA-NEXT:    l32r a5, .LCPI155_0
+; XTENSA-NEXT:    j .LBB155_2
+; XTENSA-NEXT:  .LBB155_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB155_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB155_4
+; XTENSA-NEXT:  .LBB155_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a3, a2, .LBB155_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB155_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB155_1
+; XTENSA-NEXT:  .LBB155_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB155_2
+; XTENSA-ATOMIC-NEXT:  .LBB155_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB155_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB155_6
+; XTENSA-ATOMIC-NEXT:  .LBB155_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a3, a11, .LBB155_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB155_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB155_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB155_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB155_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB155_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB155_1
+; XTENSA-ATOMIC-NEXT:  .LBB155_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_acquire(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 2
+; XTENSA-NEXT:    l32r a5, .LCPI156_0
+; XTENSA-NEXT:    j .LBB156_2
+; XTENSA-NEXT:  .LBB156_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB156_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB156_4
+; XTENSA-NEXT:  .LBB156_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a3, a2, .LBB156_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB156_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB156_1
+; XTENSA-NEXT:  .LBB156_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB156_2
+; XTENSA-ATOMIC-NEXT:  .LBB156_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB156_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB156_6
+; XTENSA-ATOMIC-NEXT:  .LBB156_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a3, a11, .LBB156_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB156_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB156_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB156_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB156_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB156_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB156_1
+; XTENSA-ATOMIC-NEXT:  .LBB156_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_release(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l32i a2, a5, 0
+; XTENSA-NEXT:    movi a7, 3
+; XTENSA-NEXT:    movi a6, 0
+; XTENSA-NEXT:    l32r a4, .LCPI157_0
+; XTENSA-NEXT:    j .LBB157_2
+; XTENSA-NEXT:  .LBB157_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB157_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB157_4
+; XTENSA-NEXT:  .LBB157_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a3, a2, .LBB157_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB157_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB157_1
+; XTENSA-NEXT:  .LBB157_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB157_2
+; XTENSA-ATOMIC-NEXT:  .LBB157_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB157_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB157_6
+; XTENSA-ATOMIC-NEXT:  .LBB157_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a3, a11, .LBB157_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB157_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB157_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB157_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB157_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB157_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB157_1
+; XTENSA-ATOMIC-NEXT:  .LBB157_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i32_acq_rel:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a5, a2, a2
+; XTENSA-NEXT:    l32i a2, a5, 0
+; XTENSA-NEXT:    movi a7, 4
+; XTENSA-NEXT:    movi a6, 2
+; XTENSA-NEXT:    l32r a4, .LCPI158_0
+; XTENSA-NEXT:    j .LBB158_2
+; XTENSA-NEXT:  .LBB158_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB158_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a5, a5
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a6, a6
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB158_4
+; XTENSA-NEXT:  .LBB158_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a3, a2, .LBB158_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB158_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB158_1
+; XTENSA-NEXT:  .LBB158_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i32_acq_rel:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB158_2
+; XTENSA-ATOMIC-NEXT:  .LBB158_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB158_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB158_6
+; XTENSA-ATOMIC-NEXT:  .LBB158_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a3, a11, .LBB158_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB158_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB158_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB158_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB158_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB158_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB158_1
+; XTENSA-ATOMIC-NEXT:  .LBB158_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; XTENSA-LABEL: atomicrmw_umin_i32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI159_0
+; XTENSA-NEXT:    j .LBB159_2
+; XTENSA-NEXT:  .LBB159_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB159_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB159_4
+; XTENSA-NEXT:  .LBB159_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a3, a3
+; XTENSA-NEXT:    bltu a3, a2, .LBB159_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB159_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB159_1
+; XTENSA-NEXT:  .LBB159_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: atomicrmw_umin_i32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB159_2
+; XTENSA-ATOMIC-NEXT:  .LBB159_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB159_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB159_6
+; XTENSA-ATOMIC-NEXT:  .LBB159_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a3, a3
+; XTENSA-ATOMIC-NEXT:    bltu a3, a11, .LBB159_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB159_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB159_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB159_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB159_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB159_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB159_1
+; XTENSA-ATOMIC-NEXT:  .LBB159_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = atomicrmw umin ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
diff --git a/llvm/test/CodeGen/Xtensa/cpus-invalid.ll b/llvm/test/CodeGen/Xtensa/cpus-invalid.ll
new file mode 100644
index 0000000000000..360761cbbbb67
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/cpus-invalid.ll
@@ -0,0 +1,8 @@
+; RUN: llc < %s --mtriple=xtensa --mcpu=invalid 2>&1 | FileCheck %s
+
+; CHECK: {{.*}} is not a recognized processor for this target
+
+define void @f() {
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/Xtensa/cpus.ll b/llvm/test/CodeGen/Xtensa/cpus.ll
new file mode 100644
index 0000000000000..d04ee16882381
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/cpus.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; This tests that llc accepts all valid Xtensa CPUs
+
+; RUN: llc < %s --mtriple=xtensa --mcpu=esp8266 2>&1 | FileCheck -check-prefix=XTENSA-ESP8266  %s
+; RUN: llc < %s --mtriple=xtensa --mcpu=esp32 2>&1 | FileCheck -check-prefix=XTENSA-ESP32 %s
+; RUN: llc < %s --mtriple=xtensa --mcpu=generic 2>&1 | FileCheck -check-prefix=XTENSA-GENERIC %s
+
+define i32 @f(i32 %z) {
+; XTENSA-ESP8266-LABEL: f:
+; XTENSA-ESP8266:         .cfi_startproc
+; XTENSA-ESP8266-NEXT:  # %bb.0:
+; XTENSA-ESP8266-NEXT:    movi a2, 0
+; XTENSA-ESP8266-NEXT:    ret
+;
+; XTENSA-ESP32-LABEL: f:
+; XTENSA-ESP32:         .cfi_startproc
+; XTENSA-ESP32-NEXT:  # %bb.0:
+; XTENSA-ESP32-NEXT:    entry a1, 32
+; XTENSA-ESP32-NEXT:    .cfi_def_cfa_offset 32
+; XTENSA-ESP32-NEXT:    movi a2, 0
+; XTENSA-ESP32-NEXT:    retw.n
+;
+; XTENSA-GENERIC-LABEL: f:
+; XTENSA-GENERIC:         .cfi_startproc
+; XTENSA-GENERIC-NEXT:  # %bb.0:
+; XTENSA-GENERIC-NEXT:    movi a2, 0
+; XTENSA-GENERIC-NEXT:    ret
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/Xtensa/forced-atomics.ll b/llvm/test/CodeGen/Xtensa/forced-atomics.ll
new file mode 100644
index 0000000000000..eeec87b7ab132
--- /dev/null
+++ b/llvm/test/CodeGen/Xtensa/forced-atomics.ll
@@ -0,0 +1,1426 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=xtensa -mattr=+windowed < %s | FileCheck %s --check-prefixes=XTENSA
+; RUN: llc -mtriple=xtensa -mattr=+windowed,s32c1i  -mattr=+forced-atomics < %s | FileCheck %s --check-prefixes=XTENSA-ATOMIC
+
+define i8 @load8(ptr %p) nounwind {
+; XTENSA-LABEL: load8:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 5
+; XTENSA-NEXT:    l32r a8, .LCPI0_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: load8:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l8ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = load atomic i8, ptr %p seq_cst, align 1
+  ret i8 %v
+}
+
+define void @store8(ptr %p) nounwind {
+; XTENSA-LABEL: store8:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI1_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: store8:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    movi a8, 0
+; XTENSA-ATOMIC-NEXT:    s8i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i8 0, ptr %p seq_cst, align 1
+  ret void
+}
+
+define i8 @rmw8(ptr %p) nounwind {
+; XTENSA-LABEL: rmw8:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI2_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw8:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a11, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a11, a12
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    j .LBB2_2
+; XTENSA-ATOMIC-NEXT:  .LBB2_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB2_4
+; XTENSA-ATOMIC-NEXT:  .LBB2_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a12
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a11
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB2_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB2_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB2_1
+; XTENSA-ATOMIC-NEXT:  .LBB2_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw add ptr %p, i8 1 seq_cst, align 1
+  ret i8 %v
+}
+
+define i8 @cmpxchg8(ptr %p) nounwind {
+; XTENSA-LABEL: cmpxchg8:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a8, 0
+; XTENSA-NEXT:    s8i a8, a1, 0
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    movi a12, 1
+; XTENSA-NEXT:    movi a13, 5
+; XTENSA-NEXT:    l32r a8, .LCPI3_0
+; XTENSA-NEXT:    or a14, a13, a13
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    l8ui a2, a1, 0
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: cmpxchg8:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 255
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a10, 0
+; XTENSA-ATOMIC-NEXT:    and a7, a11, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, 1
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a12, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:  .LBB3_1: # %partword.cmpxchg.loop
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a12
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a11, a11
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB3_3
+; XTENSA-ATOMIC-NEXT:  # %bb.2: # %partword.cmpxchg.loop
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB3_1 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:  .LBB3_3: # %partword.cmpxchg.loop
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB3_1 Depth=1
+; XTENSA-ATOMIC-NEXT:    bnez a7, .LBB3_5
+; XTENSA-ATOMIC-NEXT:  # %bb.4: # %partword.cmpxchg.failure
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB3_1 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    bne a15, a7, .LBB3_1
+; XTENSA-ATOMIC-NEXT:  .LBB3_5: # %partword.cmpxchg.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = cmpxchg ptr %p, i8 0, i8 1 seq_cst seq_cst
+  %res.0 = extractvalue { i8, i1 } %res, 0
+  ret i8 %res.0
+}
+
+define i16 @load16(ptr %p) nounwind {
+; XTENSA-LABEL: load16:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 5
+; XTENSA-NEXT:    l32r a8, .LCPI4_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: load16:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l16ui a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = load atomic i16, ptr %p seq_cst, align 2
+  ret i16 %v
+}
+
+define void @store16(ptr %p) nounwind {
+; XTENSA-LABEL: store16:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI5_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: store16:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    movi a8, 0
+; XTENSA-ATOMIC-NEXT:    s16i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i16 0, ptr %p seq_cst, align 2
+  ret void
+}
+
+define i16 @rmw16(ptr %p) nounwind {
+; XTENSA-LABEL: rmw16:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI6_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw16:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a10, a9
+; XTENSA-ATOMIC-NEXT:    l32r a11, .LCPI6_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a11, a11
+; XTENSA-ATOMIC-NEXT:    movi a12, -1
+; XTENSA-ATOMIC-NEXT:    xor a12, a11, a12
+; XTENSA-ATOMIC-NEXT:    movi a13, -4
+; XTENSA-ATOMIC-NEXT:    and a13, a2, a13
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a15, a13, 0
+; XTENSA-ATOMIC-NEXT:    movi a14, 0
+; XTENSA-ATOMIC-NEXT:    j .LBB6_2
+; XTENSA-ATOMIC-NEXT:  .LBB6_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB6_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    beqi a6, 1, .LBB6_4
+; XTENSA-ATOMIC-NEXT:  .LBB6_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a15, a12
+; XTENSA-ATOMIC-NEXT:    add a6, a15, a10
+; XTENSA-ATOMIC-NEXT:    and a6, a6, a11
+; XTENSA-ATOMIC-NEXT:    or a7, a7, a6
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a7, a13, 0
+; XTENSA-ATOMIC-NEXT:    or a6, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a7, a15, .LBB6_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB6_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a6, a14, a14
+; XTENSA-ATOMIC-NEXT:    j .LBB6_1
+; XTENSA-ATOMIC-NEXT:  .LBB6_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a7
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw add ptr %p, i16 1 seq_cst, align 2
+  ret i16 %v
+}
+
+define i16 @cmpxchg16(ptr %p) nounwind {
+; XTENSA-LABEL: cmpxchg16:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a8, 0
+; XTENSA-NEXT:    s16i a8, a1, 0
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    movi a12, 1
+; XTENSA-NEXT:    movi a13, 5
+; XTENSA-NEXT:    l32r a8, .LCPI7_0
+; XTENSA-NEXT:    or a14, a13, a13
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    l16ui a2, a1, 0
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: cmpxchg16:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    slli a8, a2, 3
+; XTENSA-ATOMIC-NEXT:    movi a9, 24
+; XTENSA-ATOMIC-NEXT:    and a8, a8, a9
+; XTENSA-ATOMIC-NEXT:    l32r a9, .LCPI7_0
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a9, a9
+; XTENSA-ATOMIC-NEXT:    movi a10, -1
+; XTENSA-ATOMIC-NEXT:    xor a9, a9, a10
+; XTENSA-ATOMIC-NEXT:    movi a10, -4
+; XTENSA-ATOMIC-NEXT:    and a10, a2, a10
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a10, 0
+; XTENSA-ATOMIC-NEXT:    and a7, a11, a9
+; XTENSA-ATOMIC-NEXT:    movi a11, 1
+; XTENSA-ATOMIC-NEXT:    ssl a8
+; XTENSA-ATOMIC-NEXT:    sll a12, a11
+; XTENSA-ATOMIC-NEXT:    movi a13, 0
+; XTENSA-ATOMIC-NEXT:  .LBB7_1: # %partword.cmpxchg.loop
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a15, a7, a7
+; XTENSA-ATOMIC-NEXT:    or a14, a15, a12
+; XTENSA-ATOMIC-NEXT:    wsr a15, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a14, a10, 0
+; XTENSA-ATOMIC-NEXT:    or a7, a11, a11
+; XTENSA-ATOMIC-NEXT:    beq a14, a15, .LBB7_3
+; XTENSA-ATOMIC-NEXT:  # %bb.2: # %partword.cmpxchg.loop
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB7_1 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a13, a13
+; XTENSA-ATOMIC-NEXT:  .LBB7_3: # %partword.cmpxchg.loop
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB7_1 Depth=1
+; XTENSA-ATOMIC-NEXT:    bnez a7, .LBB7_5
+; XTENSA-ATOMIC-NEXT:  # %bb.4: # %partword.cmpxchg.failure
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB7_1 Depth=1
+; XTENSA-ATOMIC-NEXT:    and a7, a14, a9
+; XTENSA-ATOMIC-NEXT:    bne a15, a7, .LBB7_1
+; XTENSA-ATOMIC-NEXT:  .LBB7_5: # %partword.cmpxchg.end
+; XTENSA-ATOMIC-NEXT:    ssr a8
+; XTENSA-ATOMIC-NEXT:    srl a2, a14
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = cmpxchg ptr %p, i16 0, i16 1 seq_cst seq_cst
+  %res.0 = extractvalue { i16, i1 } %res, 0
+  ret i16 %res.0
+}
+
+define i32 @load32_unordered(ptr %p) nounwind {
+; XTENSA-LABEL: load32_unordered:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI8_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: load32_unordered:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = load atomic i32, ptr %p unordered, align 4
+  ret i32 %v
+}
+
+define i32 @load32_monotonic(ptr %p) nounwind {
+; XTENSA-LABEL: load32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI9_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: load32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = load atomic i32, ptr %p monotonic, align 4
+  ret i32 %v
+}
+
+define i32 @load32_acquire(ptr %p) nounwind {
+; XTENSA-LABEL: load32_acquire:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 2
+; XTENSA-NEXT:    l32r a8, .LCPI10_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: load32_acquire:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = load atomic i32, ptr %p acquire, align 4
+  ret i32 %v
+}
+
+define i32 @load32_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: load32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 5
+; XTENSA-NEXT:    l32r a8, .LCPI11_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: load32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a2, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = load atomic i32, ptr %p seq_cst, align 4
+  ret i32 %v
+}
+
+define void @store32_unordered(ptr %p) nounwind {
+; XTENSA-LABEL: store32_unordered:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI12_0
+; XTENSA-NEXT:    or a12, a11, a11
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: store32_unordered:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 0
+; XTENSA-ATOMIC-NEXT:    s32i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i32 0, ptr %p unordered, align 4
+  ret void
+}
+
+define void @store32_monotonic(ptr %p) nounwind {
+; XTENSA-LABEL: store32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    l32r a8, .LCPI13_0
+; XTENSA-NEXT:    or a12, a11, a11
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: store32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 0
+; XTENSA-ATOMIC-NEXT:    s32i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i32 0, ptr %p monotonic, align 4
+  ret void
+}
+
+define void @store32_release(ptr %p) nounwind {
+; XTENSA-LABEL: store32_release:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    movi a12, 3
+; XTENSA-NEXT:    l32r a8, .LCPI14_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: store32_release:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    movi a8, 0
+; XTENSA-ATOMIC-NEXT:    s32i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i32 0, ptr %p release, align 4
+  ret void
+}
+
+define void @store32_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: store32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 0
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI15_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: store32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    movi a8, 0
+; XTENSA-ATOMIC-NEXT:    s32i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    retw
+  store atomic i32 0, ptr %p seq_cst, align 4
+  ret void
+}
+
+define i32 @rmw32_add_monotonic(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_add_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 0
+; XTENSA-NEXT:    l32r a8, .LCPI16_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_add_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB16_2
+; XTENSA-ATOMIC-NEXT:  .LBB16_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB16_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB16_4
+; XTENSA-ATOMIC-NEXT:  .LBB16_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    addi a8, a11, 1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB16_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB16_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB16_1
+; XTENSA-ATOMIC-NEXT:  .LBB16_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw add ptr %p, i32 1 monotonic, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_add_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_add_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI17_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_add_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB17_2
+; XTENSA-ATOMIC-NEXT:  .LBB17_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB17_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB17_4
+; XTENSA-ATOMIC-NEXT:  .LBB17_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    addi a8, a11, 1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB17_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB17_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB17_1
+; XTENSA-ATOMIC-NEXT:  .LBB17_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw add ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_sub_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_sub_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI18_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_sub_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    movi a10, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB18_2
+; XTENSA-ATOMIC-NEXT:  .LBB18_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB18_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB18_4
+; XTENSA-ATOMIC-NEXT:  .LBB18_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    addi a8, a11, -1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB18_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB18_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    j .LBB18_1
+; XTENSA-ATOMIC-NEXT:  .LBB18_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw sub ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_and_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_and_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI19_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_and_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    movi a10, 0
+; XTENSA-ATOMIC-NEXT:    j .LBB19_2
+; XTENSA-ATOMIC-NEXT:  .LBB19_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB19_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB19_4
+; XTENSA-ATOMIC-NEXT:  .LBB19_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    and a8, a11, a9
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB19_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB19_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    j .LBB19_1
+; XTENSA-ATOMIC-NEXT:  .LBB19_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw and ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_nand_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_nand_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI20_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_nand_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a13, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, -1
+; XTENSA-ATOMIC-NEXT:    movi a10, -2
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    movi a12, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB20_2
+; XTENSA-ATOMIC-NEXT:  .LBB20_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB20_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a14, 1, .LBB20_4
+; XTENSA-ATOMIC-NEXT:  .LBB20_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a8, a13, a9
+; XTENSA-ATOMIC-NEXT:    or a8, a8, a10
+; XTENSA-ATOMIC-NEXT:    wsr a13, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a14, a12, a12
+; XTENSA-ATOMIC-NEXT:    beq a8, a13, .LBB20_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB20_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a14, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB20_1
+; XTENSA-ATOMIC-NEXT:  .LBB20_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw nand ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_or_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_or_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI21_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_or_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    movi a10, 0
+; XTENSA-ATOMIC-NEXT:    j .LBB21_2
+; XTENSA-ATOMIC-NEXT:  .LBB21_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB21_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB21_4
+; XTENSA-ATOMIC-NEXT:  .LBB21_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a9
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB21_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB21_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    j .LBB21_1
+; XTENSA-ATOMIC-NEXT:  .LBB21_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw or ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_xor_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_xor_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI22_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_xor_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    movi a10, 0
+; XTENSA-ATOMIC-NEXT:    j .LBB22_2
+; XTENSA-ATOMIC-NEXT:  .LBB22_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB22_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB22_4
+; XTENSA-ATOMIC-NEXT:  .LBB22_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    xor a8, a11, a9
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB22_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB22_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    j .LBB22_1
+; XTENSA-ATOMIC-NEXT:  .LBB22_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw xor ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_max_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_max_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a5, 1
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a4, .LCPI23_0
+; XTENSA-NEXT:    j .LBB23_2
+; XTENSA-NEXT:  .LBB23_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB23_4
+; XTENSA-NEXT:  .LBB23_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a5, a5
+; XTENSA-NEXT:    bge a5, a2, .LBB23_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB23_1
+; XTENSA-NEXT:  .LBB23_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_max_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    movi a10, 0
+; XTENSA-ATOMIC-NEXT:    j .LBB23_2
+; XTENSA-ATOMIC-NEXT:  .LBB23_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB23_6
+; XTENSA-ATOMIC-NEXT:  .LBB23_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a9, a9
+; XTENSA-ATOMIC-NEXT:    bge a9, a11, .LBB23_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB23_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB23_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB23_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    j .LBB23_1
+; XTENSA-ATOMIC-NEXT:  .LBB23_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw max ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_min_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_min_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    l32i a12, a2, 0
+; XTENSA-NEXT:    movi a6, 1
+; XTENSA-NEXT:    movi a5, 2
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a4, .LCPI24_0
+; XTENSA-NEXT:    j .LBB24_2
+; XTENSA-NEXT:  .LBB24_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB24_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a12, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB24_4
+; XTENSA-NEXT:  .LBB24_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a12, a1, 0
+; XTENSA-NEXT:    blt a12, a5, .LBB24_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB24_2 Depth=1
+; XTENSA-NEXT:    or a12, a6, a6
+; XTENSA-NEXT:    j .LBB24_1
+; XTENSA-NEXT:  .LBB24_4: # %atomicrmw.end
+; XTENSA-NEXT:    or a2, a12, a12
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_min_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a12, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    movi a10, 2
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    or a8, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB24_2
+; XTENSA-ATOMIC-NEXT:  .LBB24_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB24_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a13, 1, .LBB24_6
+; XTENSA-ATOMIC-NEXT:  .LBB24_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    blt a12, a10, .LBB24_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB24_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a9, a9
+; XTENSA-ATOMIC-NEXT:  .LBB24_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB24_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a12, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a13, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a8, a12, .LBB24_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB24_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB24_1
+; XTENSA-ATOMIC-NEXT:  .LBB24_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw min ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_umax_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_umax_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a6, a2, a2
+; XTENSA-NEXT:    l32i a2, a6, 0
+; XTENSA-NEXT:    movi a5, 1
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a4, .LCPI25_0
+; XTENSA-NEXT:    j .LBB25_2
+; XTENSA-NEXT:  .LBB25_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB25_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a6, a6
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB25_4
+; XTENSA-NEXT:  .LBB25_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a2, a1, 0
+; XTENSA-NEXT:    or a12, a5, a5
+; XTENSA-NEXT:    bgeu a5, a2, .LBB25_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB25_2 Depth=1
+; XTENSA-NEXT:    or a12, a2, a2
+; XTENSA-NEXT:    j .LBB25_1
+; XTENSA-NEXT:  .LBB25_4: # %atomicrmw.end
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_umax_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    movi a10, 0
+; XTENSA-ATOMIC-NEXT:    j .LBB25_2
+; XTENSA-ATOMIC-NEXT:  .LBB25_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB25_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB25_6
+; XTENSA-ATOMIC-NEXT:  .LBB25_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a9, a9
+; XTENSA-ATOMIC-NEXT:    bgeu a9, a11, .LBB25_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB25_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a11, a11
+; XTENSA-ATOMIC-NEXT:  .LBB25_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB25_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB25_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB25_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    j .LBB25_1
+; XTENSA-ATOMIC-NEXT:  .LBB25_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw umax ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_umin_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_umin_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    l32i a12, a2, 0
+; XTENSA-NEXT:    movi a6, 1
+; XTENSA-NEXT:    movi a5, 2
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a4, .LCPI26_0
+; XTENSA-NEXT:    j .LBB26_2
+; XTENSA-NEXT:  .LBB26_1: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB26_2 Depth=1
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a4
+; XTENSA-NEXT:    l32i a12, a1, 0
+; XTENSA-NEXT:    bnez a10, .LBB26_4
+; XTENSA-NEXT:  .LBB26_2: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a12, a1, 0
+; XTENSA-NEXT:    bltu a12, a5, .LBB26_1
+; XTENSA-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-NEXT:    # in Loop: Header=BB26_2 Depth=1
+; XTENSA-NEXT:    or a12, a6, a6
+; XTENSA-NEXT:    j .LBB26_1
+; XTENSA-NEXT:  .LBB26_4: # %atomicrmw.end
+; XTENSA-NEXT:    or a2, a12, a12
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_umin_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a12, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    movi a10, 2
+; XTENSA-ATOMIC-NEXT:    movi a11, 0
+; XTENSA-ATOMIC-NEXT:    or a8, a12, a12
+; XTENSA-ATOMIC-NEXT:    j .LBB26_2
+; XTENSA-ATOMIC-NEXT:  .LBB26_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB26_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a13, 1, .LBB26_6
+; XTENSA-ATOMIC-NEXT:  .LBB26_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    bltu a12, a10, .LBB26_4
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB26_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a9, a9
+; XTENSA-ATOMIC-NEXT:  .LBB26_4: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB26_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a12, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a13, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a8, a12, .LBB26_1
+; XTENSA-ATOMIC-NEXT:  # %bb.5: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB26_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a13, a11, a11
+; XTENSA-ATOMIC-NEXT:    j .LBB26_1
+; XTENSA-ATOMIC-NEXT:  .LBB26_6: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw umin ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define i32 @rmw32_xchg_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_xchg_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 32
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a11, 1
+; XTENSA-NEXT:    movi a12, 5
+; XTENSA-NEXT:    l32r a8, .LCPI27_0
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_xchg_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a11, a2, 0
+; XTENSA-ATOMIC-NEXT:    movi a9, 1
+; XTENSA-ATOMIC-NEXT:    movi a10, 0
+; XTENSA-ATOMIC-NEXT:    j .LBB27_2
+; XTENSA-ATOMIC-NEXT:  .LBB27_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB27_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a11, a8, a8
+; XTENSA-ATOMIC-NEXT:    beqi a12, 1, .LBB27_4
+; XTENSA-ATOMIC-NEXT:  .LBB27_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    wsr a11, scompare1
+; XTENSA-ATOMIC-NEXT:    or a8, a9, a9
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a12, a9, a9
+; XTENSA-ATOMIC-NEXT:    beq a8, a11, .LBB27_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB27_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a12, a10, a10
+; XTENSA-ATOMIC-NEXT:    j .LBB27_1
+; XTENSA-ATOMIC-NEXT:  .LBB27_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw xchg ptr %p, i32 1 seq_cst, align 4
+  ret i32 %v
+}
+
+define float @rmw32_fadd_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_fadd_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    l32i a10, a2, 0
+; XTENSA-NEXT:    l32r a6, .LCPI28_1
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI28_2
+; XTENSA-NEXT:  .LBB28_1: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a10, a1, 0
+; XTENSA-NEXT:    l32r a11, .LCPI28_0
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    or a12, a10, a10
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    or a8, a10, a10
+; XTENSA-NEXT:    l32i a10, a1, 0
+; XTENSA-NEXT:    beqz a8, .LBB28_1
+; XTENSA-NEXT:  # %bb.2: # %atomicrmw.end
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_fadd_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a2, 0
+; XTENSA-ATOMIC-NEXT:    l32r a6, .LCPI28_1
+; XTENSA-ATOMIC-NEXT:    movi a5, 0
+; XTENSA-ATOMIC-NEXT:    movi a4, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB28_2
+; XTENSA-ATOMIC-NEXT:  .LBB28_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB28_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a10, a10
+; XTENSA-ATOMIC-NEXT:    beqi a8, 1, .LBB28_4
+; XTENSA-ATOMIC-NEXT:  .LBB28_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a11, .LCPI28_0
+; XTENSA-ATOMIC-NEXT:    or a10, a7, a7
+; XTENSA-ATOMIC-NEXT:    callx8 a6
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a10, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a8, a4, a4
+; XTENSA-ATOMIC-NEXT:    beq a10, a7, .LBB28_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB28_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a5, a5
+; XTENSA-ATOMIC-NEXT:    j .LBB28_1
+; XTENSA-ATOMIC-NEXT:  .LBB28_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a10, a10
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw fadd ptr %p, float 1.0 seq_cst, align 4
+  ret float %v
+}
+
+define float @rmw32_fsub_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_fsub_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    l32i a10, a2, 0
+; XTENSA-NEXT:    l32r a6, .LCPI29_1
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI29_2
+; XTENSA-NEXT:  .LBB29_1: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a10, a1, 0
+; XTENSA-NEXT:    l32r a11, .LCPI29_0
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    or a12, a10, a10
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    or a8, a10, a10
+; XTENSA-NEXT:    l32i a10, a1, 0
+; XTENSA-NEXT:    beqz a8, .LBB29_1
+; XTENSA-NEXT:  # %bb.2: # %atomicrmw.end
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_fsub_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a2, 0
+; XTENSA-ATOMIC-NEXT:    l32r a6, .LCPI29_1
+; XTENSA-ATOMIC-NEXT:    movi a5, 0
+; XTENSA-ATOMIC-NEXT:    movi a4, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB29_2
+; XTENSA-ATOMIC-NEXT:  .LBB29_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB29_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a10, a10
+; XTENSA-ATOMIC-NEXT:    beqi a8, 1, .LBB29_4
+; XTENSA-ATOMIC-NEXT:  .LBB29_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a11, .LCPI29_0
+; XTENSA-ATOMIC-NEXT:    or a10, a7, a7
+; XTENSA-ATOMIC-NEXT:    callx8 a6
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a10, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a8, a4, a4
+; XTENSA-ATOMIC-NEXT:    beq a10, a7, .LBB29_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB29_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a5, a5
+; XTENSA-ATOMIC-NEXT:    j .LBB29_1
+; XTENSA-ATOMIC-NEXT:  .LBB29_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a10, a10
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw fsub ptr %p, float 1.0 seq_cst, align 4
+  ret float %v
+}
+
+define float @rmw32_fmin_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_fmin_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    l32i a10, a2, 0
+; XTENSA-NEXT:    l32r a6, .LCPI30_1
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI30_2
+; XTENSA-NEXT:  .LBB30_1: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a10, a1, 0
+; XTENSA-NEXT:    l32r a11, .LCPI30_0
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    or a12, a10, a10
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    or a8, a10, a10
+; XTENSA-NEXT:    l32i a10, a1, 0
+; XTENSA-NEXT:    beqz a8, .LBB30_1
+; XTENSA-NEXT:  # %bb.2: # %atomicrmw.end
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_fmin_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a2, 0
+; XTENSA-ATOMIC-NEXT:    l32r a6, .LCPI30_1
+; XTENSA-ATOMIC-NEXT:    movi a5, 0
+; XTENSA-ATOMIC-NEXT:    movi a4, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB30_2
+; XTENSA-ATOMIC-NEXT:  .LBB30_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB30_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a10, a10
+; XTENSA-ATOMIC-NEXT:    beqi a8, 1, .LBB30_4
+; XTENSA-ATOMIC-NEXT:  .LBB30_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a11, .LCPI30_0
+; XTENSA-ATOMIC-NEXT:    or a10, a7, a7
+; XTENSA-ATOMIC-NEXT:    callx8 a6
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a10, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a8, a4, a4
+; XTENSA-ATOMIC-NEXT:    beq a10, a7, .LBB30_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB30_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a5, a5
+; XTENSA-ATOMIC-NEXT:    j .LBB30_1
+; XTENSA-ATOMIC-NEXT:  .LBB30_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a10, a10
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw fmin ptr %p, float 1.0 seq_cst, align 4
+  ret float %v
+}
+
+define float @rmw32_fmax_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: rmw32_fmax_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    l32i a10, a2, 0
+; XTENSA-NEXT:    l32r a6, .LCPI31_1
+; XTENSA-NEXT:    movi a7, 5
+; XTENSA-NEXT:    l32r a5, .LCPI31_2
+; XTENSA-NEXT:  .LBB31_1: # %atomicrmw.start
+; XTENSA-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-NEXT:    s32i a10, a1, 0
+; XTENSA-NEXT:    l32r a11, .LCPI31_0
+; XTENSA-NEXT:    callx8 a6
+; XTENSA-NEXT:    or a12, a10, a10
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    or a13, a7, a7
+; XTENSA-NEXT:    or a14, a7, a7
+; XTENSA-NEXT:    callx8 a5
+; XTENSA-NEXT:    or a8, a10, a10
+; XTENSA-NEXT:    l32i a10, a1, 0
+; XTENSA-NEXT:    beqz a8, .LBB31_1
+; XTENSA-NEXT:  # %bb.2: # %atomicrmw.end
+; XTENSA-NEXT:    or a2, a10, a10
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: rmw32_fmax_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    l32i a7, a2, 0
+; XTENSA-ATOMIC-NEXT:    l32r a6, .LCPI31_1
+; XTENSA-ATOMIC-NEXT:    movi a5, 0
+; XTENSA-ATOMIC-NEXT:    movi a4, 1
+; XTENSA-ATOMIC-NEXT:    j .LBB31_2
+; XTENSA-ATOMIC-NEXT:  .LBB31_1: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB31_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a7, a10, a10
+; XTENSA-ATOMIC-NEXT:    beqi a8, 1, .LBB31_4
+; XTENSA-ATOMIC-NEXT:  .LBB31_2: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # =>This Inner Loop Header: Depth=1
+; XTENSA-ATOMIC-NEXT:    l32r a11, .LCPI31_0
+; XTENSA-ATOMIC-NEXT:    or a10, a7, a7
+; XTENSA-ATOMIC-NEXT:    callx8 a6
+; XTENSA-ATOMIC-NEXT:    wsr a7, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a10, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a8, a4, a4
+; XTENSA-ATOMIC-NEXT:    beq a10, a7, .LBB31_1
+; XTENSA-ATOMIC-NEXT:  # %bb.3: # %atomicrmw.start
+; XTENSA-ATOMIC-NEXT:    # in Loop: Header=BB31_2 Depth=1
+; XTENSA-ATOMIC-NEXT:    or a8, a5, a5
+; XTENSA-ATOMIC-NEXT:    j .LBB31_1
+; XTENSA-ATOMIC-NEXT:  .LBB31_4: # %atomicrmw.end
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a10, a10
+; XTENSA-ATOMIC-NEXT:    retw
+  %v = atomicrmw fmax ptr %p, float 1.0 seq_cst, align 4
+  ret float %v
+}
+
+define i32 @cmpxchg32_monotonic(ptr %p) nounwind {
+; XTENSA-LABEL: cmpxchg32_monotonic:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a13, 0
+; XTENSA-NEXT:    s32i a13, a1, 0
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    movi a12, 1
+; XTENSA-NEXT:    l32r a8, .LCPI32_0
+; XTENSA-NEXT:    or a14, a13, a13
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: cmpxchg32_monotonic:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    movi a8, 1
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    wsr a9, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = cmpxchg ptr %p, i32 0, i32 1 monotonic monotonic
+  %res.0 = extractvalue { i32, i1 } %res, 0
+  ret i32 %res.0
+}
+
+define i32 @cmpxchg32_seq_cst(ptr %p) nounwind {
+; XTENSA-LABEL: cmpxchg32_seq_cst:
+; XTENSA:       # %bb.0:
+; XTENSA-NEXT:    entry a1, 48
+; XTENSA-NEXT:    or a10, a2, a2
+; XTENSA-NEXT:    movi a8, 0
+; XTENSA-NEXT:    s32i a8, a1, 0
+; XTENSA-NEXT:    addi a11, a1, 0
+; XTENSA-NEXT:    movi a12, 1
+; XTENSA-NEXT:    movi a13, 5
+; XTENSA-NEXT:    l32r a8, .LCPI33_0
+; XTENSA-NEXT:    or a14, a13, a13
+; XTENSA-NEXT:    callx8 a8
+; XTENSA-NEXT:    l32i a2, a1, 0
+; XTENSA-NEXT:    retw
+;
+; XTENSA-ATOMIC-LABEL: cmpxchg32_seq_cst:
+; XTENSA-ATOMIC:       # %bb.0:
+; XTENSA-ATOMIC-NEXT:    entry a1, 32
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    movi a8, 1
+; XTENSA-ATOMIC-NEXT:    movi a9, 0
+; XTENSA-ATOMIC-NEXT:    wsr a9, scompare1
+; XTENSA-ATOMIC-NEXT:    s32c1i a8, a2, 0
+; XTENSA-ATOMIC-NEXT:    memw
+; XTENSA-ATOMIC-NEXT:    or a2, a8, a8
+; XTENSA-ATOMIC-NEXT:    retw
+  %res = cmpxchg ptr %p, i32 0, i32 1 seq_cst seq_cst
+  %res.0 = extractvalue { i32, i1 } %res, 0
+  ret i32 %res.0
+}
diff --git a/llvm/test/DebugInfo/ARM/machine-cp-updates-dbg-reg.mir b/llvm/test/DebugInfo/ARM/machine-cp-updates-dbg-reg.mir
index e14472af7a4ea..2e520e13eaeb1 100644
--- a/llvm/test/DebugInfo/ARM/machine-cp-updates-dbg-reg.mir
+++ b/llvm/test/DebugInfo/ARM/machine-cp-updates-dbg-reg.mir
@@ -140,8 +140,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4,
diff --git a/llvm/test/DebugInfo/ARM/move-dbg-values-imm-test.mir b/llvm/test/DebugInfo/ARM/move-dbg-values-imm-test.mir
index 99ec658527192..131b4e57a485d 100644
--- a/llvm/test/DebugInfo/ARM/move-dbg-values-imm-test.mir
+++ b/llvm/test/DebugInfo/ARM/move-dbg-values-imm-test.mir
@@ -94,8 +94,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  4
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: c, type: default, offset: 0, size: 4, alignment: 4, 
diff --git a/llvm/test/DebugInfo/KeyInstructions/debugify.ll b/llvm/test/DebugInfo/KeyInstructions/debugify.ll
index 551ae2794c2f3..d3be5131bec4a 100644
--- a/llvm/test/DebugInfo/KeyInstructions/debugify.ll
+++ b/llvm/test/DebugInfo/KeyInstructions/debugify.ll
@@ -1,10 +1,7 @@
 ; RUN: opt -passes=debugify --debugify-atoms -S -o - < %s \
 ; RUN: | FileCheck %s
 
-;; Mirrors llvm/test/DebugInfo/debugify.ll. Split out here because the
-;; test is only supported if LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS is enabled
-;; (which is a condition for running this test directory). Once the conditional
-;; compilation of the feature is removed this can be merged into the original.
+;; Mirrors llvm/test/DebugInfo/debugify.ll
 
 ; CHECK-LABEL: define void @foo
 define void @foo() {
diff --git a/llvm/test/DebugInfo/KeyInstructions/lit.local.cfg b/llvm/test/DebugInfo/KeyInstructions/lit.local.cfg
deleted file mode 100644
index 482bd5c8ac251..0000000000000
--- a/llvm/test/DebugInfo/KeyInstructions/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not config.has_key_instructions:
-    config.unsupported = True
diff --git a/llvm/test/DebugInfo/LoongArch/dwarf-loongarch-relocs.ll b/llvm/test/DebugInfo/LoongArch/dwarf-loongarch-relocs.ll
index d28836d560377..2f5cc373a68f5 100644
--- a/llvm/test/DebugInfo/LoongArch/dwarf-loongarch-relocs.ll
+++ b/llvm/test/DebugInfo/LoongArch/dwarf-loongarch-relocs.ll
@@ -1,5 +1,5 @@
 ; RUN: llc --filetype=obj --mtriple=loongarch64 --mattr=-relax %s -o %t.o
-; RUN: llvm-readobj -r %t.o | FileCheck --check-prefixes=RELOCS-BOTH,RELOCS-NORL %s
+; RUN: llvm-readobj -r %t.o | FileCheck --check-prefix=RELOCS-BOTH %s
 ; RUN: llvm-objdump --source %t.o | FileCheck --check-prefix=SOURCE %s
 ; RUN: llvm-dwarfdump --debug-info --debug-line %t.o | FileCheck --check-prefix=DWARF %s
 
@@ -16,10 +16,8 @@
 ; RELOCS-ENRL-NEXT:      0x18 R_LARCH_RELAX - 0x0
 ; RELOCS-BOTH-NEXT:    }
 ; RELOCS-BOTH:         Section ({{.*}}) .rela.debug_frame {
-; RELOCS-NORL-NEXT:      0x1C R_LARCH_32 .debug_frame 0x0
-; RELOCS-NORL-NEXT:      0x20 R_LARCH_64 .text 0x0
-; RELOCS-ENRL-NEXT:      0x1C R_LARCH_32 .L0  0x0
-; RELOCS-ENRL-NEXT:      0x20 R_LARCH_64 .L0  0x0
+; RELOCS-BOTH-NEXT:      0x1C R_LARCH_32 .L0  0x0
+; RELOCS-BOTH-NEXT:      0x20 R_LARCH_64 .L0  0x0
 ; RELOCS-ENRL-NEXT:      0x28 R_LARCH_ADD64 .L0  0x0
 ; RELOCS-ENRL-NEXT:      0x28 R_LARCH_SUB64 .L0  0x0
 ; RELOCS-ENRL-NEXT:      0x3F R_LARCH_ADD6 .L0  0x0
@@ -29,8 +27,7 @@
 ; RELOCS-BOTH-NEXT:      0x22 R_LARCH_32 .debug_line_str 0x0
 ; RELOCS-BOTH-NEXT:      0x31 R_LARCH_32 .debug_line_str 0x2
 ; RELOCS-BOTH-NEXT:      0x46 R_LARCH_32 .debug_line_str 0x1B
-; RELOCS-NORL-NEXT:      0x4F R_LARCH_64 .text 0x0
-; RELOCS-ENRL-NEXT:      0x4F R_LARCH_64 .L0  0x0
+; RELOCS-BOTH-NEXT:      0x4F R_LARCH_64 .L0  0x0
 ; RELOCS-ENRL-NEXT:      0x5F R_LARCH_ADD16 .L0  0x0
 ; RELOCS-ENRL-NEXT:      0x5F R_LARCH_SUB16 .L0  0x0
 ; RELOCS-BOTH-NEXT:    }
diff --git a/llvm/test/DebugInfo/MIR/AArch64/implicit-def-dead-scope.mir b/llvm/test/DebugInfo/MIR/AArch64/implicit-def-dead-scope.mir
index 113a0035fe7f6..aab561e0ebb2c 100644
--- a/llvm/test/DebugInfo/MIR/AArch64/implicit-def-dead-scope.mir
+++ b/llvm/test/DebugInfo/MIR/AArch64/implicit-def-dead-scope.mir
@@ -169,8 +169,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: bz, type: default, offset: -32, size: 16, alignment: 8,
diff --git a/llvm/test/DebugInfo/MIR/ARM/live-debug-values-reg-copy.mir b/llvm/test/DebugInfo/MIR/ARM/live-debug-values-reg-copy.mir
index c7e19abaa6300..ee6141bd7b5ea 100644
--- a/llvm/test/DebugInfo/MIR/ARM/live-debug-values-reg-copy.mir
+++ b/llvm/test/DebugInfo/MIR/ARM/live-debug-values-reg-copy.mir
@@ -92,8 +92,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
   - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
diff --git a/llvm/test/DebugInfo/MIR/InstrRef/livedebugvalues-transfer-variadic-instr-ref.mir b/llvm/test/DebugInfo/MIR/InstrRef/livedebugvalues-transfer-variadic-instr-ref.mir
index 7ff40aa28b954..2f4abb53860c0 100644
--- a/llvm/test/DebugInfo/MIR/InstrRef/livedebugvalues-transfer-variadic-instr-ref.mir
+++ b/llvm/test/DebugInfo/MIR/InstrRef/livedebugvalues-transfer-variadic-instr-ref.mir
@@ -169,8 +169,8 @@ frameInfo:
   stackProtector:  ''
   functionContext: ''
   cvBytesOfCalleeSavedRegisters: 48
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: default, 
       callee-saved-register: '$rbx', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/DebugInfo/MIR/Mips/last-inst-bundled.mir b/llvm/test/DebugInfo/MIR/Mips/last-inst-bundled.mir
index b8628ac6ae87d..0ce0b0344ae9f 100644
--- a/llvm/test/DebugInfo/MIR/Mips/last-inst-bundled.mir
+++ b/llvm/test/DebugInfo/MIR/Mips/last-inst-bundled.mir
@@ -134,8 +134,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
   - { id: 0, name: condition, type: default, offset: -12, size: 4, alignment: 4,
diff --git a/llvm/test/DebugInfo/MIR/Mips/live-debug-values-reg-copy.mir b/llvm/test/DebugInfo/MIR/Mips/live-debug-values-reg-copy.mir
index 1e848f74512de..aca20ee7a4552 100644
--- a/llvm/test/DebugInfo/MIR/Mips/live-debug-values-reg-copy.mir
+++ b/llvm/test/DebugInfo/MIR/Mips/live-debug-values-reg-copy.mir
@@ -118,8 +118,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
diff --git a/llvm/test/DebugInfo/MIR/X86/dbg-call-site-spilled-arg.mir b/llvm/test/DebugInfo/MIR/X86/dbg-call-site-spilled-arg.mir
index 8ede60c33359d..a5a5f76a3217f 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbg-call-site-spilled-arg.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbg-call-site-spilled-arg.mir
@@ -106,8 +106,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: default, 
       callee-saved-register: '$rbx', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/DebugInfo/MIR/X86/debug-loc-0.mir b/llvm/test/DebugInfo/MIR/X86/debug-loc-0.mir
index 0b007456be1e6..01862f5905f9c 100644
--- a/llvm/test/DebugInfo/MIR/X86/debug-loc-0.mir
+++ b/llvm/test/DebugInfo/MIR/X86/debug-loc-0.mir
@@ -84,8 +84,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: s1.addr, type: default, offset: 0, size: 8, alignment: 8,
diff --git a/llvm/test/DebugInfo/MIR/X86/instr-ref-join-def-vphi.mir b/llvm/test/DebugInfo/MIR/X86/instr-ref-join-def-vphi.mir
index aa6597f782856..431325aea7db0 100644
--- a/llvm/test/DebugInfo/MIR/X86/instr-ref-join-def-vphi.mir
+++ b/llvm/test/DebugInfo/MIR/X86/instr-ref-join-def-vphi.mir
@@ -152,8 +152,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -24, size: 8, alignment: 8, stack-id: default,
       callee-saved-register: '$rdi', callee-saved-restored: true, debug-info-variable: '',
diff --git a/llvm/test/DebugInfo/MIR/X86/kill-after-spill.mir b/llvm/test/DebugInfo/MIR/X86/kill-after-spill.mir
index a28018a0912a0..ba64459faa023 100644
--- a/llvm/test/DebugInfo/MIR/X86/kill-after-spill.mir
+++ b/llvm/test/DebugInfo/MIR/X86/kill-after-spill.mir
@@ -227,8 +227,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: default,
       callee-saved-register: '$rbx', callee-saved-restored: true }
diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-values-reg-copy.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-values-reg-copy.mir
index 6f5c36933cffe..1f9cc7542fd96 100644
--- a/llvm/test/DebugInfo/MIR/X86/live-debug-values-reg-copy.mir
+++ b/llvm/test/DebugInfo/MIR/X86/live-debug-values-reg-copy.mir
@@ -131,8 +131,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
   - { id: 0, type: spill-slot, offset: -48, size: 8, alignment: 16, stack-id: default,
       callee-saved-register: '$rbx', callee-saved-restored: true }
diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-values-restore.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-values-restore.mir
index 73c723b1446de..1189d5414d9c9 100644
--- a/llvm/test/DebugInfo/MIR/X86/live-debug-values-restore.mir
+++ b/llvm/test/DebugInfo/MIR/X86/live-debug-values-restore.mir
@@ -287,8 +287,8 @@ frameInfo:
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
   - { id: 0, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: default,
       callee-saved-register: '$rbx', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir
index 4d48774a78dd7..0f108cd704ed2 100644
--- a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir
+++ b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg-debugonly.mir
@@ -123,8 +123,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir
index e618f48f527b8..425ef4cdfdb5d 100644
--- a/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir
+++ b/llvm/test/DebugInfo/MIR/X86/live-debug-vars-unused-arg.mir
@@ -121,8 +121,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 stack:
 constants:
diff --git a/llvm/test/DebugInfo/X86/DW_AT_alloc_type.ll b/llvm/test/DebugInfo/X86/DW_AT_alloc_type.ll
new file mode 100644
index 0000000000000..33028f2234ceb
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/DW_AT_alloc_type.ll
@@ -0,0 +1,34 @@
+; RUN: llc -O3 -o %t -filetype=obj %s
+; RUN: llvm-dwarfdump %t | FileCheck %s
+
+; based on clang++ output for `int *alloc_int() { return new int; }`
+
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local ptr @alloc_int() !dbg !3 {
+; CHECK: DW_TAG_subprogram
+entry:
+  %call = call ptr @alloc(i64 noundef 4), !heapallocsite !7
+; CHECK: DW_TAG_call_site
+; CHECK: DW_AT_LLVM_alloc_type ([[ALLOCSITE:.*]])
+  ret ptr %call
+}
+
+; CHECK: {{.*}}[[ALLOCSITE]]: DW_TAG_base_type
+; CHECK: DW_AT_name ("int")
+
+declare dso_local ptr @alloc(i64 noundef)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2,!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, emissionKind: FullDebug)
+!1 = !DIFile(filename: "a.cpp", directory: "/")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = distinct !DISubprogram(name: "alloc_int", scope: !1, file: !1, line: 1, type: !4, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition, unit: !0)
+!4 = !DISubroutineType(types: !5)
+!5 = !{!6}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 5}
diff --git a/llvm/test/DebugInfo/X86/instr-ref-track-clobbers.mir b/llvm/test/DebugInfo/X86/instr-ref-track-clobbers.mir
index feeaaa70f008b..d557961cdce55 100644
--- a/llvm/test/DebugInfo/X86/instr-ref-track-clobbers.mir
+++ b/llvm/test/DebugInfo/X86/instr-ref-track-clobbers.mir
@@ -114,8 +114,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:
 - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: default, 
     callee-saved-register: '$r15', callee-saved-restored: true, debug-info-variable: '', 
diff --git a/llvm/test/DebugInfo/X86/live-debug-vars-dse.mir b/llvm/test/DebugInfo/X86/live-debug-vars-dse.mir
index 908889063584c..503fa94988692 100644
--- a/llvm/test/DebugInfo/X86/live-debug-vars-dse.mir
+++ b/llvm/test/DebugInfo/X86/live-debug-vars-dse.mir
@@ -114,8 +114,8 @@ frameInfo:
   hasOpaqueSPAdjustment: false
   hasVAStart:      false
   hasMustTailInVarArgFunc: false
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      
 stack:           
   - { id: 0, name: x.addr, type: default, offset: 0, size: 4, alignment: 4, 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/coro-byval-param.ll b/llvm/test/Instrumentation/AddressSanitizer/coro-byval-param.ll
new file mode 100644
index 0000000000000..290f1cbd38cdf
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/coro-byval-param.ll
@@ -0,0 +1,126 @@
+; RUN: opt < %s -passes='asan,coro-early' -S
+;
+; Regression test to see if ASan breaks the @llvm.coro.id intrinsic
+; (see commit bee2654300a8f524e05dd4cad41411d597246ac0)
+;
+; Forked from llvm/test/Transforms/Coroutines/coro-byval-param.ll
+
+%promise_type = type { i8 }
+%struct.A = type <{ i64, i64, i32, [4 x i8] }>
+
+; Function Attrs: noinline ssp uwtable mustprogress
+define ptr @foo(ptr nocapture readonly byval(%struct.A) align 8 %a1) #0 {
+entry:
+  %__promise = alloca %promise_type, align 1
+  %a2 = alloca %struct.A, align 8
+  %0 = call token @llvm.coro.id(i32 16, ptr nonnull %__promise, ptr @foo, ptr null)
+  %1 = call i1 @llvm.coro.alloc(token %0)
+  br i1 %1, label %coro.alloc, label %coro.init
+
+coro.alloc:                                       ; preds = %entry
+  %2 = call i64 @llvm.coro.size.i64()
+  %call = call noalias nonnull ptr @_Znwm(i64 %2) #9
+  br label %coro.init
+
+coro.init:                                        ; preds = %coro.alloc, %entry
+  %3 = phi ptr [ null, %entry ], [ %call, %coro.alloc ]
+  %4 = call ptr @llvm.coro.begin(token %0, ptr %3) #10
+  call void @llvm.lifetime.start.p0(ptr nonnull %__promise) #2
+  %call2 = call ptr @_ZN4task12promise_type17get_return_objectEv(ptr nonnull dereferenceable(1) %__promise)
+  call void @initial_suspend(ptr nonnull dereferenceable(1) %__promise)
+  %5 = call token @llvm.coro.save(ptr null)
+  call fastcc void @_ZNSt12experimental13coroutines_v116coroutine_handleIN4task12promise_typeEE12from_addressEPv(ptr %4) #2
+  %6 = call i8 @llvm.coro.suspend(token %5, i1 false)
+  switch i8 %6, label %coro.ret [
+    i8 0, label %init.ready
+    i8 1, label %cleanup33
+  ]
+
+init.ready:                                       ; preds = %coro.init
+  call void @llvm.lifetime.start.p0(ptr nonnull %a2) #2
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a2, ptr align 8 %a1, i64 24, i1 false)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a2) #2
+  call void @_ZN4task12promise_type13final_suspendEv(ptr nonnull dereferenceable(1) %__promise) #2
+  %7 = call token @llvm.coro.save(ptr null)
+  call fastcc void @_ZNSt12experimental13coroutines_v116coroutine_handleIN4task12promise_typeEE12from_addressEPv(ptr %4) #2
+  %8 = call i8 @llvm.coro.suspend(token %7, i1 true) #10
+  %switch = icmp ult i8 %8, 2
+  br i1 %switch, label %cleanup33, label %coro.ret
+
+cleanup33:                                        ; preds = %init.ready, %coro.init
+  call void @llvm.lifetime.end.p0(ptr nonnull %__promise) #2
+  %9 = call ptr @llvm.coro.free(token %0, ptr %4)
+  %.not = icmp eq ptr %9, null
+  br i1 %.not, label %coro.ret, label %coro.free
+
+coro.free:                                        ; preds = %cleanup33
+  call void @_ZdlPv(ptr nonnull %9) #2
+  br label %coro.ret
+
+coro.ret:                                         ; preds = %coro.free, %cleanup33, %init.ready, %coro.init
+  %10 = call i1 @llvm.coro.end(ptr null, i1 false, token none) #10
+  ret ptr %call2
+}
+
+; Function Attrs: argmemonly nounwind readonly
+declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
+
+; Function Attrs: nounwind
+declare i1 @llvm.coro.alloc(token) #2
+
+; Function Attrs: nobuiltin nofree allocsize(0)
+declare nonnull ptr @_Znwm(i64) local_unnamed_addr #3
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.coro.size.i64() #4
+
+; Function Attrs: nounwind
+declare ptr @llvm.coro.begin(token, ptr writeonly) #2
+
+; Function Attrs: argmemonly nofree nosync nounwind willreturn
+declare void @llvm.lifetime.start.p0(ptr nocapture) #5
+
+; Function Attrs: argmemonly nofree nounwind willreturn
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #6
+
+; Function Attrs: noinline nounwind ssp uwtable willreturn mustprogress
+declare ptr @_ZN4task12promise_type17get_return_objectEv(ptr nonnull dereferenceable(1)) local_unnamed_addr #7 align 2
+
+; Function Attrs: noinline nounwind ssp uwtable willreturn mustprogress
+declare void @initial_suspend(ptr nonnull dereferenceable(1)) local_unnamed_addr #7 align 2
+
+; Function Attrs: nounwind
+declare token @llvm.coro.save(ptr) #2
+
+; Function Attrs: noinline nounwind ssp uwtable willreturn mustprogress
+declare hidden fastcc void @_ZNSt12experimental13coroutines_v116coroutine_handleIN4task12promise_typeEE12from_addressEPv(ptr) unnamed_addr #7 align 2
+
+; Function Attrs: argmemonly nofree nosync nounwind willreturn
+declare void @llvm.lifetime.end.p0(ptr nocapture) #5
+
+; Function Attrs: nounwind
+declare i8 @llvm.coro.suspend(token, i1) #2
+
+; Function Attrs: noinline nounwind ssp uwtable willreturn mustprogress
+declare void @_ZN4task12promise_type13final_suspendEv(ptr nonnull dereferenceable(1)) local_unnamed_addr #7 align 2
+
+; Function Attrs: nounwind
+declare i1 @llvm.coro.end(ptr, i1, token) #2
+
+; Function Attrs: nobuiltin nounwind
+declare void @_ZdlPv(ptr) local_unnamed_addr #8
+
+; Function Attrs: argmemonly nounwind readonly
+declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1
+
+attributes #0 = { noinline ssp uwtable mustprogress presplitcoroutine sanitize_address "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" }
+attributes #1 = { argmemonly nounwind readonly }
+attributes #2 = { nounwind }
+attributes #3 = { nobuiltin nofree allocsize(0) "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" }
+attributes #4 = { nounwind readnone }
+attributes #5 = { argmemonly nofree nosync nounwind willreturn }
+attributes #6 = { argmemonly nofree nounwind willreturn }
+attributes #7 = { noinline nounwind ssp uwtable willreturn mustprogress "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" }
+attributes #8 = { nobuiltin nounwind "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" }
+attributes #9 = { allocsize(0) }
+attributes #10 = { noduplicate }
diff --git a/llvm/test/Instrumentation/AddressSanitizer/lifetime-uar-uas.ll b/llvm/test/Instrumentation/AddressSanitizer/lifetime-uar-uas.ll
index 3685d8d530bed..aad8940689b4a 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/lifetime-uar-uas.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/lifetime-uar-uas.ll
@@ -8,9 +8,6 @@
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
-
 define i32 @basic_test(i64 %i) sanitize_address {
   ; CHECK-LABEL: define i32 @basic_test(
 
@@ -22,7 +19,7 @@ entry:
   ; CHECK-UAS: store i64 -868082052615769615, ptr %{{[0-9]+}}
   ; CHECK-UAS-SS-NOT: store i64
 
-  call void @llvm.lifetime.start.p0(i64 2, ptr %c)
+  call void @llvm.lifetime.start.p0(ptr %c)
   ; Memory is unpoisoned at llvm.lifetime.start: 01
   ; CHECK-UAS: store i8 2, ptr %{{[0-9]+}}
 
@@ -30,7 +27,7 @@ entry:
   store volatile i32 0, ptr %retval
   store volatile i8 0, ptr %ci, align 1
 
-  call void @llvm.lifetime.end.p0(i64 2, ptr %c)
+  call void @llvm.lifetime.end.p0(ptr %c)
   ; Memory is poisoned at llvm.lifetime.end: F8
   ; CHECK-UAS: store i8 -8, ptr %{{[0-9]+}}
   ; CHECK-UAS-SS-NOT: store i8 -8,
diff --git a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
index 959437001a039..d1e01807f6c26 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/lifetime.ll
@@ -9,73 +9,10 @@ target triple = "x86_64-unknown-linux-gnu"
 declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
 declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
 
-define void @lifetime_no_size(i64 %i) sanitize_address {
-; CHECK-LABEL: define void @lifetime_no_size(
-; CHECK-SAME: i64 [[I:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
-; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 32
-; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr
-; CHECK-NEXT:    [[TMP3:%.*]] = inttoptr i64 [[TMP0]] to ptr
-; CHECK-NEXT:    store i64 1102416563, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP0]], 8
-; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
-; CHECK-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP5]], align 8
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[TMP0]], 16
-; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; CHECK-NEXT:    store i64 ptrtoint (ptr @lifetime_no_size to i64), ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[TMP8:%.*]] = lshr i64 [[TMP0]], 3
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[TMP8]], 2147450880
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = inttoptr i64 [[TMP10]] to ptr
-; CHECK-NEXT:    store i64 -868083117767659023, ptr [[TMP11]], align 1
-; CHECK-NEXT:    [[AI:%.*]] = getelementptr inbounds [2 x i32], ptr [[TMP2]], i64 0, i64 [[I]]
-; CHECK-NEXT:    [[TMP12:%.*]] = ptrtoint ptr [[AI]] to i64
-; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP12]], 3
-; CHECK-NEXT:    [[TMP14:%.*]] = add i64 [[TMP13]], 2147450880
-; CHECK-NEXT:    [[TMP15:%.*]] = inttoptr i64 [[TMP14]] to ptr
-; CHECK-NEXT:    [[TMP16:%.*]] = load i8, ptr [[TMP15]], align 1
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne i8 [[TMP16]], 0
-; CHECK-NEXT:    br i1 [[TMP17]], label %[[BB18:.*]], label %[[BB23:.*]], !prof [[PROF1:![0-9]+]]
-; CHECK:       [[BB18]]:
-; CHECK-NEXT:    [[TMP19:%.*]] = and i64 [[TMP12]], 7
-; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP19]] to i8
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp sge i8 [[TMP20]], [[TMP16]]
-; CHECK-NEXT:    br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB23]]
-; CHECK:       [[BB22]]:
-; CHECK-NEXT:    call void @__asan_report_store1(i64 [[TMP12]]) #[[ATTR4:[0-9]+]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB23]]:
-; CHECK-NEXT:    store volatile i8 0, ptr [[AI]], align 4
-; CHECK-NEXT:    store i64 1172321806, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP24:%.*]] = add i64 [[TMP9]], 0
-; CHECK-NEXT:    [[TMP25:%.*]] = inttoptr i64 [[TMP24]] to ptr
-; CHECK-NEXT:    store i64 0, ptr [[TMP25]], align 1
-; CHECK-NEXT:    ret void
-;
-entry:
-  %a = alloca [2 x i32], align 4
-
-  ; Poison memory in prologue: 0xf3f3f300f1f1f1f1
-
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %a)
-  ; Check that lifetime with no size are ignored.
-
-  %ai = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 %i
-  store volatile i8 0, ptr %ai, align 4
-
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %a)
-  ; Check that lifetime with no size are ignored.
-
-  ; Unpoison stack frame on exit.
-  ret void
-}
-
 ; Generic case of lifetime analysis.
 define void @lifetime() sanitize_address {
 ; CHECK-DEFAULT-LABEL: define void @lifetime(
-; CHECK-DEFAULT-SAME: ) #[[ATTR1]] {
+; CHECK-DEFAULT-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-DEFAULT-NEXT:    [[TMP1:%.*]] = alloca i64, align 32
 ; CHECK-DEFAULT-NEXT:    store i64 0, ptr [[TMP1]], align 8
 ; CHECK-DEFAULT-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
@@ -86,7 +23,7 @@ define void @lifetime() sanitize_address {
 ; CHECK-DEFAULT-NEXT:    store i64 1102416563, ptr [[TMP5]], align 8
 ; CHECK-DEFAULT-NEXT:    [[TMP6:%.*]] = add i64 [[TMP2]], 8
 ; CHECK-DEFAULT-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
-; CHECK-DEFAULT-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP7]], align 8
+; CHECK-DEFAULT-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP7]], align 8
 ; CHECK-DEFAULT-NEXT:    [[TMP8:%.*]] = add i64 [[TMP2]], 16
 ; CHECK-DEFAULT-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-DEFAULT-NEXT:    store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP9]], align 8
@@ -104,14 +41,14 @@ define void @lifetime() sanitize_address {
 ; CHECK-DEFAULT-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
 ; CHECK-DEFAULT-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
 ; CHECK-DEFAULT-NEXT:    [[TMP21:%.*]] = icmp ne i8 [[TMP20]], 0
-; CHECK-DEFAULT-NEXT:    br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB27:.*]], !prof [[PROF1]]
+; CHECK-DEFAULT-NEXT:    br i1 [[TMP21]], label %[[BB22:.*]], label %[[BB27:.*]], !prof [[PROF1:![0-9]+]]
 ; CHECK-DEFAULT:       [[BB22]]:
 ; CHECK-DEFAULT-NEXT:    [[TMP23:%.*]] = and i64 [[TMP16]], 7
 ; CHECK-DEFAULT-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i8
 ; CHECK-DEFAULT-NEXT:    [[TMP25:%.*]] = icmp sge i8 [[TMP24]], [[TMP20]]
 ; CHECK-DEFAULT-NEXT:    br i1 [[TMP25]], label %[[BB26:.*]], label %[[BB27]]
 ; CHECK-DEFAULT:       [[BB26]]:
-; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP16]]) #[[ATTR4]]
+; CHECK-DEFAULT-NEXT:    call void @__asan_report_store1(i64 [[TMP16]]) #[[ATTR4:[0-9]+]]
 ; CHECK-DEFAULT-NEXT:    unreachable
 ; CHECK-DEFAULT:       [[BB27]]:
 ; CHECK-DEFAULT-NEXT:    store volatile i8 0, ptr [[TMP4]], align 1
@@ -182,7 +119,7 @@ define void @lifetime() sanitize_address {
 ; CHECK-DEFAULT-NEXT:    ret void
 ;
 ; CHECK-NO-DYNAMIC-LABEL: define void @lifetime(
-; CHECK-NO-DYNAMIC-SAME: ) #[[ATTR1]] {
+; CHECK-NO-DYNAMIC-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-NO-DYNAMIC-NEXT:    [[MYALLOCA:%.*]] = alloca i8, i64 64, align 32
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[MYALLOCA]] to i64
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 32
@@ -191,7 +128,7 @@ define void @lifetime() sanitize_address {
 ; CHECK-NO-DYNAMIC-NEXT:    store i64 1102416563, ptr [[TMP4]], align 8
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP5:%.*]] = add i64 [[TMP1]], 8
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
-; CHECK-NO-DYNAMIC-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack.1 to i64), ptr [[TMP6]], align 8
+; CHECK-NO-DYNAMIC-NEXT:    store i64 ptrtoint (ptr @___asan_gen_stack to i64), ptr [[TMP6]], align 8
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP7:%.*]] = add i64 [[TMP1]], 16
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
 ; CHECK-NO-DYNAMIC-NEXT:    store i64 ptrtoint (ptr @lifetime to i64), ptr [[TMP8]], align 8
@@ -209,14 +146,14 @@ define void @lifetime() sanitize_address {
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP19]], 0
-; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP20]], label %[[BB21:.*]], label %[[BB26:.*]], !prof [[PROF1]]
+; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP20]], label %[[BB21:.*]], label %[[BB26:.*]], !prof [[PROF1:![0-9]+]]
 ; CHECK-NO-DYNAMIC:       [[BB21]]:
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP22:%.*]] = and i64 [[TMP15]], 7
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP23:%.*]] = trunc i64 [[TMP22]] to i8
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP24:%.*]] = icmp sge i8 [[TMP23]], [[TMP19]]
 ; CHECK-NO-DYNAMIC-NEXT:    br i1 [[TMP24]], label %[[BB25:.*]], label %[[BB26]]
 ; CHECK-NO-DYNAMIC:       [[BB25]]:
-; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP15]]) #[[ATTR4]]
+; CHECK-NO-DYNAMIC-NEXT:    call void @__asan_report_store1(i64 [[TMP15]]) #[[ATTR4:[0-9]+]]
 ; CHECK-NO-DYNAMIC-NEXT:    unreachable
 ; CHECK-NO-DYNAMIC:       [[BB26]]:
 ; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[TMP3]], align 1
@@ -227,7 +164,7 @@ define void @lifetime() sanitize_address {
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP30:%.*]] = inttoptr i64 [[TMP29]] to ptr
 ; CHECK-NO-DYNAMIC-NEXT:    store i8 -8, ptr [[TMP30]], align 1
 ; CHECK-NO-DYNAMIC-NEXT:    [[ARR:%.*]] = alloca [10 x i32], align 16
-; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ARR]])
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP31:%.*]] = ptrtoint ptr [[ARR]] to i64
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP32:%.*]] = lshr i64 [[TMP31]], 3
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP33:%.*]] = add i64 [[TMP32]], 2147450880
@@ -245,7 +182,7 @@ define void @lifetime() sanitize_address {
 ; CHECK-NO-DYNAMIC-NEXT:    unreachable
 ; CHECK-NO-DYNAMIC:       [[BB42]]:
 ; CHECK-NO-DYNAMIC-NEXT:    store volatile i8 0, ptr [[ARR]], align 1
-; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(i64 40, ptr [[ARR]])
+; CHECK-NO-DYNAMIC-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ARR]])
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP43:%.*]] = add i64 [[TMP10]], 4
 ; CHECK-NO-DYNAMIC-NEXT:    [[TMP44:%.*]] = inttoptr i64 [[TMP43]] to ptr
 ; CHECK-NO-DYNAMIC-NEXT:    store i8 4, ptr [[TMP44]], align 1
@@ -318,8 +255,8 @@ define void @zero_sized(i64 %a) #0 {
 ; CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i64, align 8
 ; CHECK-NEXT:    [[B:%.*]] = alloca [0 x i8], align 1
 ; CHECK-NEXT:    store i64 [[A]], ptr [[A_ADDR]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 0, ptr [[B]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 0, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B]])
 ; CHECK-NEXT:    ret void
 ;
 
diff --git a/llvm/test/Instrumentation/AddressSanitizer/remove-memory-effects.ll b/llvm/test/Instrumentation/AddressSanitizer/remove-memory-effects.ll
index 481e780f3acf0..07b28f4d35384 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/remove-memory-effects.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/remove-memory-effects.ll
@@ -10,9 +10,9 @@ declare void @foo(ptr writeonly) memory(argmem: write)
 define void @bar() sanitize_address {
 entry:
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
   call void @foo(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   ret void
 }
 
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
index ac5d8b82099cb..37b280c5bd242 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
@@ -10,8 +10,8 @@ target triple = "riscv64-unknown-linux"
 declare void @mayFail(ptr %x) sanitize_hwaddress
 declare void @onExcept(ptr %x) sanitize_hwaddress
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 declare i32 @__gxx_personality_v0(...)
 
 define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
@@ -46,7 +46,7 @@ define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP19]] to ptr
 ; CHECK-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[X]])
 ; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP15]] to i8
 ; CHECK-NEXT:    [[TMP21:%.*]] = ptrtoint ptr [[X]] to i64
 ; CHECK-NEXT:    [[TMP22:%.*]] = and i64 [[TMP21]], 72057594037927935
@@ -65,7 +65,7 @@ define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:    [[TMP30:%.*]] = lshr i64 [[TMP29]], 4
 ; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP30]]
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP31]], i8 [[TMP27]], i64 1, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       lpad:
 ; CHECK-NEXT:    [[TMP32:%.*]] = landingpad { ptr, i32 }
@@ -81,7 +81,7 @@ define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:    [[TMP38:%.*]] = lshr i64 [[TMP37]], 4
 ; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP38]]
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP39]], i8 [[TMP35]], i64 1, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]])
 ; CHECK-NEXT:    br label [[EH_RESUME:%.*]]
 ; CHECK:       eh.resume:
 ; CHECK-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8
@@ -94,12 +94,12 @@ entry:
   %x = alloca i32, align 8
   %exn.slot = alloca ptr, align 8
   %ehselector.slot = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
   invoke void @mayFail(ptr %x) to label %invoke.cont unwind label %lpad
 
 invoke.cont:                                      ; preds = %entry
 
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   ret void
 
 lpad:                                             ; preds = %entry
@@ -111,7 +111,7 @@ lpad:                                             ; preds = %entry
   %2 = extractvalue { ptr, i32 } %0, 1
   store i32 %2, ptr %ehselector.slot, align 4
   call void @onExcept(ptr %x) #18
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   br label %eh.resume
 
 eh.resume:                                        ; preds = %lpad
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
index db78c1f05b0df..d2949bfb9c1b2 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
@@ -78,13 +78,13 @@ sw.bb1:                                           ; preds = %entry
   br label %return
 
 while.body:                                       ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 4096, ptr nonnull %buf) #10
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf) #10
   store ptr %buf, ptr @stackbuf, align 8
   ; may_jump may call longjmp, going back to the switch (and then the return),
   ; bypassing the lifetime.end. This is why we need to untag on the return,
   ; rather than the lifetime.end.
   call void @may_jump()
-  call void @llvm.lifetime.end.p0(i64 4096, ptr nonnull %buf) #10
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf) #10
   br label %return
 
 return:                                           ; preds = %entry, %while.body, %sw.bb1
@@ -94,5 +94,5 @@ return:                                           ; preds = %entry, %while.body,
 
 declare i32 @setjmp(ptr noundef) returns_twice
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
index 292a565a1e61b..ef86e63aca0d6 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/X86/alloca.ll
@@ -109,7 +109,7 @@ define i32 @test_simple(ptr %a) sanitize_hwaddress {
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP5]], 57
 ; CHECK-NEXT:    [[TMP9:%.*]] = or i64 [[TMP7]], [[TMP8]]
 ; CHECK-NEXT:    [[BUF_SROA_0_HWASAN:%.*]] = inttoptr i64 [[TMP9]] to ptr
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[BUF_SROA_0]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BUF_SROA_0]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i8
 ; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[BUF_SROA_0]], i8 [[TMP10]], i64 16)
 ; CHECK-NEXT:    [[TMP11:%.*]] = ptrtoint ptr [[BUF_SROA_0_HWASAN]] to i64
@@ -117,7 +117,7 @@ define i32 @test_simple(ptr %a) sanitize_hwaddress {
 ; CHECK-NEXT:    store volatile i8 0, ptr [[BUF_SROA_0_HWASAN]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
 ; CHECK-NEXT:    call void @__hwasan_tag_memory(ptr [[BUF_SROA_0]], i8 [[TMP12]], i64 16)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[BUF_SROA_0]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BUF_SROA_0]])
 ; CHECK-NEXT:    ret i32 0
 ;
 ; INLINE-LABEL: define i32 @test_simple
@@ -150,7 +150,7 @@ define i32 @test_simple(ptr %a) sanitize_hwaddress {
 ; INLINE-NEXT:    [[TMP19:%.*]] = shl i64 [[TMP16]], 57
 ; INLINE-NEXT:    [[TMP20:%.*]] = or i64 [[TMP18]], [[TMP19]]
 ; INLINE-NEXT:    [[BUF_SROA_0_HWASAN:%.*]] = inttoptr i64 [[TMP20]] to ptr
-; INLINE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[BUF_SROA_0]])
+; INLINE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BUF_SROA_0]])
 ; INLINE-NEXT:    [[TMP21:%.*]] = trunc i64 [[TMP16]] to i8
 ; INLINE-NEXT:    [[TMP22:%.*]] = ptrtoint ptr [[BUF_SROA_0]] to i64
 ; INLINE-NEXT:    [[TMP23:%.*]] = and i64 [[TMP22]], -9079256848778919937
@@ -197,19 +197,19 @@ define i32 @test_simple(ptr %a) sanitize_hwaddress {
 ; INLINE-NEXT:    [[TMP54:%.*]] = lshr i64 [[TMP53]], 4
 ; INLINE-NEXT:    [[TMP55:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP54]]
 ; INLINE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP55]], i8 [[TMP51]], i64 1, i1 false)
-; INLINE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[BUF_SROA_0]])
+; INLINE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BUF_SROA_0]])
 ; INLINE-NEXT:    ret i32 0
 ;
 entry:
   %buf.sroa.0 = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   store volatile i8 0, ptr %buf.sroa.0, align 4
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
 ; Function Attrs: argmemonly mustprogress nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 ; Function Attrs: argmemonly mustprogress nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/coro-byval-param.ll b/llvm/test/Instrumentation/HWAddressSanitizer/coro-byval-param.ll
new file mode 100644
index 0000000000000..0289b33a45882
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/coro-byval-param.ll
@@ -0,0 +1,126 @@
+; RUN: opt < %s -passes='hwasan,coro-early' -S
+;
+; Regression test to see if HWASan breaks the @llvm.coro.id intrinsic
+; (see commit de64f85f5d224c3304f377dd6e202c9d5415fce1)
+;
+; Forked from llvm/test/Transforms/Coroutines/coro-byval-param.ll
+
+%promise_type = type { i8 }
+%struct.A = type <{ i64, i64, i32, [4 x i8] }>
+
+; Function Attrs: noinline ssp uwtable mustprogress
+define ptr @foo(ptr nocapture readonly byval(%struct.A) align 8 %a1) #0 {
+entry:
+  %__promise = alloca %promise_type, align 1
+  %a2 = alloca %struct.A, align 8
+  %0 = call token @llvm.coro.id(i32 16, ptr nonnull %__promise, ptr @foo, ptr null)
+  %1 = call i1 @llvm.coro.alloc(token %0)
+  br i1 %1, label %coro.alloc, label %coro.init
+
+coro.alloc:                                       ; preds = %entry
+  %2 = call i64 @llvm.coro.size.i64()
+  %call = call noalias nonnull ptr @_Znwm(i64 %2) #9
+  br label %coro.init
+
+coro.init:                                        ; preds = %coro.alloc, %entry
+  %3 = phi ptr [ null, %entry ], [ %call, %coro.alloc ]
+  %4 = call ptr @llvm.coro.begin(token %0, ptr %3) #10
+  call void @llvm.lifetime.start.p0(ptr nonnull %__promise) #2
+  %call2 = call ptr @_ZN4task12promise_type17get_return_objectEv(ptr nonnull dereferenceable(1) %__promise)
+  call void @initial_suspend(ptr nonnull dereferenceable(1) %__promise)
+  %5 = call token @llvm.coro.save(ptr null)
+  call fastcc void @_ZNSt12experimental13coroutines_v116coroutine_handleIN4task12promise_typeEE12from_addressEPv(ptr %4) #2
+  %6 = call i8 @llvm.coro.suspend(token %5, i1 false)
+  switch i8 %6, label %coro.ret [
+    i8 0, label %init.ready
+    i8 1, label %cleanup33
+  ]
+
+init.ready:                                       ; preds = %coro.init
+  call void @llvm.lifetime.start.p0(ptr nonnull %a2) #2
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a2, ptr align 8 %a1, i64 24, i1 false)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a2) #2
+  call void @_ZN4task12promise_type13final_suspendEv(ptr nonnull dereferenceable(1) %__promise) #2
+  %7 = call token @llvm.coro.save(ptr null)
+  call fastcc void @_ZNSt12experimental13coroutines_v116coroutine_handleIN4task12promise_typeEE12from_addressEPv(ptr %4) #2
+  %8 = call i8 @llvm.coro.suspend(token %7, i1 true) #10
+  %switch = icmp ult i8 %8, 2
+  br i1 %switch, label %cleanup33, label %coro.ret
+
+cleanup33:                                        ; preds = %init.ready, %coro.init
+  call void @llvm.lifetime.end.p0(ptr nonnull %__promise) #2
+  %9 = call ptr @llvm.coro.free(token %0, ptr %4)
+  %.not = icmp eq ptr %9, null
+  br i1 %.not, label %coro.ret, label %coro.free
+
+coro.free:                                        ; preds = %cleanup33
+  call void @_ZdlPv(ptr nonnull %9) #2
+  br label %coro.ret
+
+coro.ret:                                         ; preds = %coro.free, %cleanup33, %init.ready, %coro.init
+  %10 = call i1 @llvm.coro.end(ptr null, i1 false, token none) #10
+  ret ptr %call2
+}
+
+; Function Attrs: argmemonly nounwind readonly
+declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
+
+; Function Attrs: nounwind
+declare i1 @llvm.coro.alloc(token) #2
+
+; Function Attrs: nobuiltin nofree allocsize(0)
+declare nonnull ptr @_Znwm(i64) local_unnamed_addr #3
+
+; Function Attrs: nounwind readnone
+declare i64 @llvm.coro.size.i64() #4
+
+; Function Attrs: nounwind
+declare ptr @llvm.coro.begin(token, ptr writeonly) #2
+
+; Function Attrs: argmemonly nofree nosync nounwind willreturn
+declare void @llvm.lifetime.start.p0(ptr nocapture) #5
+
+; Function Attrs: argmemonly nofree nounwind willreturn
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #6
+
+; Function Attrs: noinline nounwind ssp uwtable willreturn mustprogress
+declare ptr @_ZN4task12promise_type17get_return_objectEv(ptr nonnull dereferenceable(1)) local_unnamed_addr #7 align 2
+
+; Function Attrs: noinline nounwind ssp uwtable willreturn mustprogress
+declare void @initial_suspend(ptr nonnull dereferenceable(1)) local_unnamed_addr #7 align 2
+
+; Function Attrs: nounwind
+declare token @llvm.coro.save(ptr) #2
+
+; Function Attrs: noinline nounwind ssp uwtable willreturn mustprogress
+declare hidden fastcc void @_ZNSt12experimental13coroutines_v116coroutine_handleIN4task12promise_typeEE12from_addressEPv(ptr) unnamed_addr #7 align 2
+
+; Function Attrs: argmemonly nofree nosync nounwind willreturn
+declare void @llvm.lifetime.end.p0(ptr nocapture) #5
+
+; Function Attrs: nounwind
+declare i8 @llvm.coro.suspend(token, i1) #2
+
+; Function Attrs: noinline nounwind ssp uwtable willreturn mustprogress
+declare void @_ZN4task12promise_type13final_suspendEv(ptr nonnull dereferenceable(1)) local_unnamed_addr #7 align 2
+
+; Function Attrs: nounwind
+declare i1 @llvm.coro.end(ptr, i1, token) #2
+
+; Function Attrs: nobuiltin nounwind
+declare void @_ZdlPv(ptr) local_unnamed_addr #8
+
+; Function Attrs: argmemonly nounwind readonly
+declare ptr @llvm.coro.free(token, ptr nocapture readonly) #1
+
+attributes #0 = { noinline ssp uwtable mustprogress presplitcoroutine sanitize_hwaddress "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" }
+attributes #1 = { argmemonly nounwind readonly }
+attributes #2 = { nounwind }
+attributes #3 = { nobuiltin nofree allocsize(0) "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" }
+attributes #4 = { nounwind readnone }
+attributes #5 = { argmemonly nofree nosync nounwind willreturn }
+attributes #6 = { argmemonly nofree nounwind willreturn }
+attributes #7 = { noinline nounwind ssp uwtable willreturn mustprogress "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" }
+attributes #8 = { nobuiltin nounwind "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" }
+attributes #9 = { allocsize(0) }
+attributes #10 = { noduplicate }
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll b/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll
index 3e13eb48054d7..f2ba94c92d45e 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/exception-lifetime.ll
@@ -10,8 +10,8 @@ target triple = "aarch64--linux-android"
 declare void @mayFail(ptr %x) sanitize_hwaddress
 declare void @onExcept(ptr %x) sanitize_hwaddress
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 declare i32 @__gxx_personality_v0(...)
 
 define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
@@ -48,7 +48,7 @@ define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:    [[X_HWASAN:%.*]] = inttoptr i64 [[TMP21]] to ptr
 ; CHECK-NEXT:    [[EXN_SLOT:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[EHSELECTOR_SLOT:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[X]])
 ; CHECK-NEXT:    [[TMP22:%.*]] = trunc i64 [[TMP17]] to i8
 ; CHECK-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[X]] to i64
 ; CHECK-NEXT:    [[TMP24:%.*]] = and i64 [[TMP23]], 72057594037927935
@@ -64,7 +64,7 @@ define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:    [[TMP30:%.*]] = lshr i64 [[TMP29]], 4
 ; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP30]]
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP31]], i8 [[TMP27]], i64 1, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       lpad:
 ; CHECK-NEXT:    [[TMP32:%.*]] = landingpad { ptr, i32 }
@@ -82,7 +82,7 @@ define void @test() sanitize_hwaddress personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:    [[TMP39:%.*]] = lshr i64 [[TMP38]], 4
 ; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP39]]
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP40]], i8 [[TMP36]], i64 1, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]])
 ; CHECK-NEXT:    br label [[EH_RESUME:%.*]]
 ; CHECK:       eh.resume:
 ; CHECK-NEXT:    [[EXN:%.*]] = load ptr, ptr [[EXN_SLOT]], align 8
@@ -95,12 +95,12 @@ entry:
   %x = alloca i32, align 8
   %exn.slot = alloca ptr, align 8
   %ehselector.slot = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
   invoke void @mayFail(ptr %x) to label %invoke.cont unwind label %lpad
 
 invoke.cont:                                      ; preds = %entry
 
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   ret void
 
 lpad:                                             ; preds = %entry
@@ -112,7 +112,7 @@ lpad:                                             ; preds = %entry
   %2 = extractvalue { ptr, i32 } %0, 1
   store i32 %2, ptr %ehselector.slot, align 4
   call void @onExcept(ptr %x) #18
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   br label %eh.resume
 
 eh.resume:                                        ; preds = %lpad
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/stack-coloring.ll b/llvm/test/Instrumentation/HWAddressSanitizer/stack-coloring.ll
index ae6fe5776f203..a40d964aa47e3 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/stack-coloring.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/stack-coloring.ll
@@ -16,22 +16,22 @@ define i32 @myCall_w2(i32 %in) sanitize_hwaddress {
 entry:
   %a = alloca [17 x ptr], align 8
   %a2 = alloca [16 x ptr], align 8
-  call void @llvm.lifetime.start.p0(i64 136, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   %t1 = call i32 @foo(i32 %in, ptr %a)
   %t2 = call i32 @foo(i32 %in, ptr %a)
-  call void @llvm.lifetime.end.p0(i64 136, ptr %a)
-  call void @llvm.lifetime.start.p0(i64 128, ptr %a2)
+  call void @llvm.lifetime.end.p0(ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a2)
   %t3 = call i32 @foo(i32 %in, ptr %a2)
   %t4 = call i32 @foo(i32 %in, ptr %a2)
-  call void @llvm.lifetime.end.p0(i64 128, ptr %a2)
+  call void @llvm.lifetime.end.p0(ptr %a2)
   %t5 = add i32 %t1, %t2
   %t6 = add i32 %t3, %t4
   %t7 = add i32 %t5, %t6
   ret i32 %t7
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 
 declare i32 @foo(i32, ptr)
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll b/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll
index 60af551eb458b..a76566ba3d643 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll
@@ -24,9 +24,9 @@ entry:
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: safeAlloca{{[[:space:]]}}Function: test_simple
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_simple
   %buf.sroa.0 = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   store volatile i8 0, ptr %buf.sroa.0, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -43,9 +43,9 @@ entry:
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: safeAlloca{{[[:space:]]}}Function: test_cmpxchg
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_cmpxchg
   %buf.sroa.0 = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   %0 = cmpxchg ptr %buf.sroa.0, i8 1, i8 2 monotonic monotonic, align 4
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -62,9 +62,9 @@ entry:
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: safeAlloca{{[[:space:]]}}Function: test_atomicrwm
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_atomicrwm
   %buf.sroa.0 = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   %0 = atomicrmw add ptr %buf.sroa.0, i8 1 monotonic, align 4
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -82,9 +82,9 @@ entry:
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_use
   %buf.sroa.0 = alloca i8, align 4
   call void @use(ptr nonnull %buf.sroa.0)
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   store volatile i8 0, ptr %buf.sroa.0, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -101,9 +101,9 @@ entry:
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: safeAlloca{{[[:space:]]}}Function: test_in_range
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range
   %buf.sroa.0 = alloca [10 x i8], align 4
-  call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   store volatile i8 0, ptr %buf.sroa.0, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 10, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -121,9 +121,9 @@ entry:
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_in_range2
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
-  call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   store volatile i8 0, ptr %ptr, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 10, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -194,9 +194,9 @@ entry:
   ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 10
-  call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   store volatile i8 0, ptr %ptr, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 10, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -213,9 +213,9 @@ entry:
   ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range2
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 10
-  call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   %0 = cmpxchg ptr %ptr, i8 1, i8 2 monotonic monotonic, align 4
-  call void @llvm.lifetime.end.p0(i64 10, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -268,11 +268,11 @@ entry:
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
   %buf.sroa.1 = alloca [10 x i8], align 4
   %ptr1 = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 9
-  call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
-  call void @llvm.lifetime.end.p0(i64 10, ptr nonnull %buf.sroa.0)
-  call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.1)
   call void @llvm.memmove.p0.p0.i32(ptr %ptr, ptr %ptr1, i32 1, i1 true)
-  call void @llvm.lifetime.end.p0(i64 10, ptr nonnull %buf.sroa.1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.1)
   ret i32 0
 }
 
@@ -289,31 +289,9 @@ entry:
   ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_out_of_range6
   %buf.sroa.0 = alloca [10 x i8], align 4
   %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 10
-  call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   %0 = atomicrmw add ptr %ptr, i32 1 monotonic, align 4
-  call void @llvm.lifetime.end.p0(i64 10, ptr nonnull %buf.sroa.0)
-  ret i32 0
-}
-
-; Check an alloca with potentially out of range GEP to ensure it gets a tag and
-; check.
-define i32 @test_potentially_out_of_range(ptr %a) sanitize_hwaddress {
-entry:
-  ; CHECK-LABEL: @test_potentially_out_of_range
-  ; NOSAFETY: call {{.*}}__hwasan_generate_tag
-  ; NOSAFETY: call {{.*}}__hwasan_store
-  ; SAFETY: call {{.*}}__hwasan_generate_tag
-  ; SAFETY: call {{.*}}__hwasan_store
-  ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
-  ; NOSTACK-NOT: call {{.*}}__hwasan_store
-  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: safeAlloca{{[[:space:]]}}Function: test_potentially_out_of_range
-  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_potentially_out_of_range
-  %buf.sroa.0 = alloca [10 x i8], align 4
-  %off = call i32 @getoffset()
-  %ptr = getelementptr [10 x i8], ptr %buf.sroa.0, i32 0, i32 %off
-  call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %ptr)
-  store volatile i8 0, ptr %ptr, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 10, ptr nonnull %ptr)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -333,26 +311,6 @@ entry:
   call void @llvm.memmove.p0.p0.i32(ptr %ptr, ptr %a, i32 1, i1 true)
   ret i32 0
 }
-; Check an alloca with potentially out of range GEP to ensure it gets a tag and
-; check.
-define i32 @test_unclear(ptr %a) sanitize_hwaddress {
-entry:
-  ; CHECK-LABEL: @test_unclear
-  ; NOSAFETY: call {{.*}}__hwasan_generate_tag
-  ; NOSAFETY: call {{.*}}__hwasan_store
-  ; SAFETY: call {{.*}}__hwasan_generate_tag
-  ; SAFETY: call {{.*}}__hwasan_store
-  ; NOSTACK-NOT: call {{.*}}__hwasan_generate_tag
-  ; NOSTACK: call {{.*}}__hwasan_store
-  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: safeAlloca{{[[:space:]]}}Function: test_unclear
-  ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_unclear
-  %buf.sroa.0 = alloca i8, align 4
-  %ptr = call ptr @getptr(ptr %buf.sroa.0)
-  call void @llvm.lifetime.start.p0(i64 10, ptr nonnull %ptr)
-  store volatile i8 0, ptr %ptr, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 10, ptr nonnull %ptr)
-  ret i32 0
-}
 
 define i32 @test_select(ptr %a) sanitize_hwaddress {
 entry:
@@ -367,11 +325,11 @@ entry:
   ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_select
   %x = call ptr @getptr(ptr %a)
   %buf.sroa.0 = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   %c = call i1 @cond()
   %ptr = select i1 %c, ptr %x, ptr %buf.sroa.0
   store volatile i8 0, ptr %ptr, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -388,10 +346,10 @@ entry:
   ; SAFETY-REMARKS: --- !Missed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: safeAlloca{{[[:space:]]}}Function: test_retptr
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_retptr
   %buf.sroa.0 = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf.sroa.0)
   %ptr = call ptr @retptr(ptr %buf.sroa.0)
   store volatile i8 0, ptr %ptr, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %buf.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf.sroa.0)
   ret i32 0
 }
 
@@ -408,17 +366,17 @@ entry:
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: safeAlloca{{[[:space:]]}}Function: test_lifetime_poison
   ; SAFETY-REMARKS: --- !Passed{{[[:space:]]}}Pass: hwasan{{[[:space:]]}}Name: ignoreAccess{{[[:space:]]}}Function: test_lifetime_poison
   %buf.sroa.0 = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 1, ptr poison)
+  call void @llvm.lifetime.start.p0(ptr poison)
   store volatile i8 0, ptr %buf.sroa.0, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 1, ptr poison)
+  call void @llvm.lifetime.end.p0(ptr poison)
   ret i32 0
 }
 
 ; Function Attrs: argmemonly mustprogress nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 ; Function Attrs: argmemonly mustprogress nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @llvm.memset.p0.i32(ptr, i8, i32, i1)
 declare void @llvm.memcpy.p0.p0.i32(ptr, ptr, i32, i1)
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
index 57d37ca1ef95a..af6411a541e9a 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
@@ -79,13 +79,13 @@ sw.bb1:                                           ; preds = %entry
   br label %return
 
 while.body:                                       ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 4096, ptr nonnull %buf) #10
+  call void @llvm.lifetime.start.p0(ptr nonnull %buf) #10
   store ptr %buf, ptr @stackbuf, align 8
   ; may_jump may call longjmp, going back to the switch (and then the return),
   ; bypassing the lifetime.end. This is why we need to untag on the return,
   ; rather than the lifetime.end.
   call void @may_jump()
-  call void @llvm.lifetime.end.p0(i64 4096, ptr nonnull %buf) #10
+  call void @llvm.lifetime.end.p0(ptr nonnull %buf) #10
   br label %return
 
 return:                                           ; preds = %entry, %while.body, %sw.bb1
@@ -95,5 +95,5 @@ return:                                           ; preds = %entry, %while.body,
 
 declare i32 @setjmp(ptr noundef) returns_twice
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope.ll b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope.ll
index e30b51890e172..cfded025dce7f 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/use-after-scope.ll
@@ -26,13 +26,13 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 ; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; X86-SCOPE-NEXT:    br label [[TMP11:%.*]]
 ; X86-SCOPE:       11:
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP4]])
 ; X86-SCOPE-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP6]] to i8
 ; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP12]], i64 16)
 ; X86-SCOPE-NEXT:    [[TMP13:%.*]] = tail call i1 (...) @cond()
 ; X86-SCOPE-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
 ; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP14]], i64 16)
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP4]])
 ; X86-SCOPE-NEXT:    br i1 [[TMP13]], label [[TMP15:%.*]], label [[TMP11]]
 ; X86-SCOPE:       15:
 ; X86-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
@@ -96,7 +96,7 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
 ; AARCH64-SCOPE-NEXT:    br label [[TMP25:%.*]]
 ; AARCH64-SCOPE:       25:
-; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP20]] to i8
 ; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[TMP18]] to i64
 ; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 72057594037927935
@@ -110,7 +110,7 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
 ; AARCH64-SCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
 ; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SCOPE-NEXT:    br i1 [[TMP31]], label [[TMP37:%.*]], label [[TMP25]]
 ; AARCH64-SCOPE:       37:
 ; AARCH64-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
@@ -198,7 +198,7 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
 ; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP25:%.*]]
 ; AARCH64-SHORT-SCOPE:       25:
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP20]] to i8
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[TMP18]] to i64
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 72057594037927935
@@ -215,7 +215,7 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP37:%.*]] = lshr i64 [[TMP36]], 4
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP37]]
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 [[TMP34]], i64 1, i1 false)
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP33]], label [[TMP39:%.*]], label [[TMP25]]
 ; AARCH64-SHORT-SCOPE:       39:
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
@@ -279,10 +279,10 @@ define dso_local i32 @standard_lifetime() local_unnamed_addr sanitize_hwaddress
 
 2:                                                ; preds = %2, %0
 ; We should tag the memory after the br (in the loop).
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %1)
   %3 = tail call i1 (...) @cond() #2
 ; We should tag the memory before the next br (before the jump back).
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
   br i1 %3, label %4, label %2
 
 4:                                                ; preds = %2
@@ -307,13 +307,13 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 ; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
 ; X86-SCOPE-NEXT:    br label [[TMP11:%.*]]
 ; X86-SCOPE:       11:
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP4]])
 ; X86-SCOPE-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP6]] to i8
 ; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP12]], i64 16)
 ; X86-SCOPE-NEXT:    [[TMP13:%.*]] = tail call i1 (...) @cond()
 ; X86-SCOPE-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
 ; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP14]], i64 16)
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP4]])
 ; X86-SCOPE-NEXT:    br i1 [[TMP13]], label [[TMP15:%.*]], label [[TMP11]]
 ; X86-SCOPE:       15:
 ; X86-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
@@ -377,7 +377,7 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 ; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
 ; AARCH64-SCOPE-NEXT:    br label [[TMP25:%.*]]
 ; AARCH64-SCOPE:       25:
-; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP20]] to i8
 ; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[TMP18]] to i64
 ; AARCH64-SCOPE-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 72057594037927935
@@ -391,7 +391,7 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 ; AARCH64-SCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
 ; AARCH64-SCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
 ; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SCOPE-NEXT:    br i1 [[TMP31]], label [[TMP37:%.*]], label [[TMP25]]
 ; AARCH64-SCOPE:       37:
 ; AARCH64-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
@@ -479,7 +479,7 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 ; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
 ; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP25:%.*]]
 ; AARCH64-SHORT-SCOPE:       25:
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = trunc i64 [[TMP20]] to i8
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = ptrtoint ptr [[TMP18]] to i64
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP28:%.*]] = and i64 [[TMP27]], 72057594037927935
@@ -496,7 +496,7 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP37:%.*]] = lshr i64 [[TMP36]], 4
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP37]]
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 [[TMP34]], i64 1, i1 false)
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SHORT-SCOPE-NEXT:    br i1 [[TMP33]], label [[TMP39:%.*]], label [[TMP25]]
 ; AARCH64-SHORT-SCOPE:       39:
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
@@ -560,10 +560,10 @@ define dso_local i32 @standard_lifetime_optnone() local_unnamed_addr optnone noi
 
 2:                                                ; preds = %2, %0
 ; We should tag the memory after the br (in the loop).
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %1)
   %3 = tail call i1 (...) @cond() #2
 ; We should tag the memory before the next br (before the jump back).
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
   br i1 %3, label %4, label %2
 
 4:                                                ; preds = %2
@@ -809,12 +809,12 @@ define dso_local i32 @multiple_lifetimes() local_unnamed_addr sanitize_hwaddress
   %1 = alloca i8, align 1
 ; We erase lifetime markers if we insert instrumentation outside of the
 ; lifetime.
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %1)
   call void @use(ptr nonnull %1) #2
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %1)
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %1)
   call void @use(ptr nonnull %1) #2
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
   ret i32 0
 }
 
@@ -833,7 +833,7 @@ define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 ; X86-SCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
 ; X86-SCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
 ; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP4]])
 ; X86-SCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
 ; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
 ; X86-SCOPE-NEXT:    [[TMP12:%.*]] = tail call i1 (...) @cond()
@@ -906,7 +906,7 @@ define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
 ; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
 ; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
-; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
 ; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
 ; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
@@ -1019,7 +1019,7 @@ define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
 ; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
@@ -1109,13 +1109,13 @@ define dso_local i32 @unreachable_exit() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-NOSCOPE-NEXT:    ret i32 0
 ;
   %1 = alloca i8, align 1
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %1)
   %2 = tail call i1 (...) @cond() #2
   br i1 %2, label %3, label %4
 
 3:
   call void @use(ptr nonnull %1) #2
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
   ret i32 0
 
 4:
@@ -1137,7 +1137,7 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; X86-SCOPE-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP6]], 57
 ; X86-SCOPE-NEXT:    [[TMP10:%.*]] = or i64 [[TMP8]], [[TMP9]]
 ; X86-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP10]] to ptr
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP4]])
 ; X86-SCOPE-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP6]] to i8
 ; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP11]], i64 16)
 ; X86-SCOPE-NEXT:    [[TMP12:%.*]] = tail call i1 (...) @cond()
@@ -1146,12 +1146,12 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; X86-SCOPE-NEXT:    call void @use(ptr nonnull [[ALLOCA_0_HWASAN]])
 ; X86-SCOPE-NEXT:    [[TMP14:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
 ; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP14]], i64 16)
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP4]])
 ; X86-SCOPE-NEXT:    br label [[TMP17:%.*]]
 ; X86-SCOPE:       15:
 ; X86-SCOPE-NEXT:    [[TMP16:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
 ; X86-SCOPE-NEXT:    call void @__hwasan_tag_memory(ptr [[TMP4]], i8 [[TMP16]], i64 16)
-; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP4]])
+; X86-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP4]])
 ; X86-SCOPE-NEXT:    br label [[TMP17]]
 ; X86-SCOPE:       17:
 ; X86-SCOPE-NEXT:    ret i32 0
@@ -1214,7 +1214,7 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
 ; AARCH64-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
 ; AARCH64-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
-; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
 ; AARCH64-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
 ; AARCH64-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
@@ -1231,7 +1231,7 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SCOPE-NEXT:    [[TMP35:%.*]] = lshr i64 [[TMP34]], 4
 ; AARCH64-SCOPE-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP35]]
 ; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP36]], i8 [[TMP32]], i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SCOPE-NEXT:    br label [[TMP43:%.*]]
 ; AARCH64-SCOPE:       37:
 ; AARCH64-SCOPE-NEXT:    [[TMP38:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
@@ -1240,7 +1240,7 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SCOPE-NEXT:    [[TMP41:%.*]] = lshr i64 [[TMP40]], 4
 ; AARCH64-SCOPE-NEXT:    [[TMP42:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP41]]
 ; AARCH64-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP42]], i8 [[TMP38]], i64 1, i1 false)
-; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SCOPE-NEXT:    br label [[TMP43]]
 ; AARCH64-SCOPE:       43:
 ; AARCH64-SCOPE-NEXT:    ret i32 0
@@ -1327,7 +1327,7 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP20]], 56
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP24:%.*]] = or i64 [[TMP22]], [[TMP23]]
 ; AARCH64-SHORT-SCOPE-NEXT:    [[ALLOCA_0_HWASAN:%.*]] = inttoptr i64 [[TMP24]] to ptr
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP20]] to i8
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP26:%.*]] = ptrtoint ptr [[TMP18]] to i64
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP27:%.*]] = and i64 [[TMP26]], 72057594037927935
@@ -1347,7 +1347,7 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP37:%.*]] = lshr i64 [[TMP36]], 4
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP37]]
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP38]], i8 [[TMP34]], i64 1, i1 false)
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP45:%.*]]
 ; AARCH64-SHORT-SCOPE:       39:
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP40:%.*]] = trunc i64 [[HWASAN_UAR_TAG]] to i8
@@ -1356,7 +1356,7 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP43:%.*]] = lshr i64 [[TMP42]], 4
 ; AARCH64-SHORT-SCOPE-NEXT:    [[TMP44:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP43]]
 ; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.memset.p0.i64(ptr align 1 [[TMP44]], i8 [[TMP40]], i64 1, i1 false)
-; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP18]])
+; AARCH64-SHORT-SCOPE-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP18]])
 ; AARCH64-SHORT-SCOPE-NEXT:    br label [[TMP45]]
 ; AARCH64-SHORT-SCOPE:       45:
 ; AARCH64-SHORT-SCOPE-NEXT:    ret i32 0
@@ -1417,17 +1417,17 @@ define dso_local i32 @diamond_lifetime() local_unnamed_addr sanitize_hwaddress {
 ; AARCH64-SHORT-NOSCOPE-NEXT:    ret i32 0
 ;
   %1 = alloca i8, align 1
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %1)
   %2 = tail call i1 (...) @cond() #2
   br i1 %2, label %3, label %4
 
 3:
   call void @use(ptr nonnull %1) #2
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
   br label %5
 
 4:
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
   br label %5
 
 5:
@@ -1439,7 +1439,7 @@ declare dso_local i1 @cond(...) local_unnamed_addr
 declare dso_local void @use(ptr) local_unnamed_addr
 
 ; Function Attrs: argmemonly mustprogress nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 ; Function Attrs: argmemonly mustprogress nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg-kmsan.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg-kmsan.ll
index 2189424cd76fa..b64dfbfde2dea 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg-kmsan.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg-kmsan.ll
@@ -7,10 +7,10 @@ target triple = "aarch64-unknown-linux-gnu"
 
 define i32 @foo(i32 %guard, ...) {
   %vl = alloca %struct.__va_list, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -45,7 +45,7 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK: [[STACK:%.*]] = getelementptr inbounds i8, ptr {{%.*}}, i32 192
 ; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 16 {{%.*}}, ptr align 16 [[STACK]], i64 {{%.*}}, i1 false)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg.ll
index 0bd0968b0f1a2..f3cceb7c075b2 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg.ll
@@ -7,10 +7,10 @@ target triple = "aarch64-unknown-linux-gnu"
 
 define i32 @foo(i32 %guard, ...) {
   %vl = alloca %struct.__va_list, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -45,10 +45,10 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK: [[STACK:%.*]] = getelementptr inbounds i8, ptr {{%.*}}, i32 192
 ; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 16 {{%.*}}, ptr align 16 [[STACK]], i64 {{%.*}}, i1 false)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
   %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i32 2, double 3.000000e+00, 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg_shadow.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg_shadow.ll
index 9133b329deb26..06a34ac469e8c 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg_shadow.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/vararg_shadow.ll
@@ -749,7 +749,7 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef %t, i32 noundef
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -809,26 +809,26 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef %t, i32 noundef
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.start.p0(ptr nocapture) #3
 
 declare void @llvm.va_start(ptr) #4
 
 declare void @llvm.va_end(ptr) #4
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.end.p0(ptr nocapture) #3
 
 define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef %n, ...) sanitize_memory {
 ; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2IiEvT_iz(
@@ -842,7 +842,7 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -902,16 +902,16 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -927,7 +927,7 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -987,16 +987,16 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -1012,7 +1012,7 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1072,16 +1072,16 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -1097,7 +1097,7 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(fp128 noundef %t, i32 nound
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1157,16 +1157,16 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(fp128 noundef %t, i32 nound
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -1182,7 +1182,7 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1242,16 +1242,16 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -1267,7 +1267,7 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz([2 x i64] %t.coe
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1327,16 +1327,16 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz([2 x i64] %t.coe
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -1352,7 +1352,7 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz([2 x double] a
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1412,16 +1412,16 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz([2 x double] a
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -1437,7 +1437,7 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz([4 x double] alignst
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1497,16 +1497,16 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz([4 x double] alignst
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -1522,7 +1522,7 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz([2 x i64] %t.co
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1582,16 +1582,16 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz([2 x i64] %t.co
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -1607,7 +1607,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz([2 x fp128] ali
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1667,16 +1667,16 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz([2 x fp128] ali
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
@@ -1692,7 +1692,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz([4 x fp128] ali
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca %"struct.std::__va_list", align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 193514046488576
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1752,16 +1752,16 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz([4 x fp128] ali
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca %"struct.std::__va_list", align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #5
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %args) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #5
   ret void
 }
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/ARM32/vararg-arm32.ll b/llvm/test/Instrumentation/MemorySanitizer/ARM32/vararg-arm32.ll
index 52f4901298611..e05018c2d5372 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/ARM32/vararg-arm32.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/ARM32/vararg-arm32.ll
@@ -18,7 +18,7 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], -2147483649
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP7]], i8 0, i64 8, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VL]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[VL]] to i64
 ; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], -2147483649
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
@@ -32,15 +32,15 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP16]], ptr align 8 [[TMP3]], i64 [[TMP2]], i1 false)
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VL]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VL]])
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 0
 ;
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -49,10 +49,10 @@ define i32 @foo(i32 %guard, ...) {
 
 
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
 ; CHECK-LABEL: define i32 @bar() {
diff --git a/llvm/test/Instrumentation/MemorySanitizer/LoongArch/vararg-loongarch64.ll b/llvm/test/Instrumentation/MemorySanitizer/LoongArch/vararg-loongarch64.ll
index 23df3fcd903c1..e6d3a4b2994ad 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/LoongArch/vararg-loongarch64.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/LoongArch/vararg-loongarch64.ll
@@ -4,10 +4,10 @@ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n64-S128"
 target triple = "loongarch64-unknown-linux-gnu"
 
 ;; First, check allocation of the save area.
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 define i32 @foo(i32 %guard, ...) {
 ; CHECK-LABEL: @foo
 ; CHECK:    [[TMP1:%.*]] = load {{.*}} @__msan_va_arg_overflow_size_tls
@@ -17,10 +17,10 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP3]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP4]], i1 false)
 ;
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64.ll b/llvm/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64.ll
index 64a76c5fd436a..69a74a37a1f04 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64.ll
@@ -5,10 +5,10 @@ target triple = "mips64--linux"
 
 define i32 @foo(i32 %guard, ...) {
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -23,10 +23,10 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[A]], i64 800)
 ; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
   %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64el.ll b/llvm/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64el.ll
index 9f3127e9d89ed..b19da8e9ff14b 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64el.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64el.ll
@@ -5,10 +5,10 @@ target triple = "mips64el--linux"
 
 define i32 @foo(i32 %guard, ...) {
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -23,10 +23,10 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[A]], i64 800)
 ; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
   %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mips.ll b/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mips.ll
index 05a88f0c252a0..4d47b02bb2713 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mips.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mips.ll
@@ -18,7 +18,7 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], -2147483649
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP7]], i8 0, i64 8, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VL]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[VL]] to i64
 ; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], -2147483649
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
@@ -32,15 +32,15 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP16]], ptr align 8 [[TMP3]], i64 [[TMP2]], i1 false)
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VL]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VL]])
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 0
 ;
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -49,10 +49,10 @@ define i32 @foo(i32 %guard, ...) {
 
 
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
 ; CHECK-LABEL: define i32 @bar() {
diff --git a/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mipsel.ll b/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mipsel.ll
index 971b25febe0e2..98294e7c0383c 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mipsel.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/Mips32/vararg-mipsel.ll
@@ -18,7 +18,7 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], -2147483649
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP7]], i8 0, i64 8, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VL]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[VL]] to i64
 ; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], -2147483649
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
@@ -32,15 +32,15 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP16]], ptr align 8 [[TMP3]], i64 [[TMP2]], i1 false)
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VL]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VL]])
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 0
 ;
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -49,10 +49,10 @@ define i32 @foo(i32 %guard, ...) {
 
 
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
 ; CHECK-LABEL: define i32 @bar() {
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
index 45e8b2d16854a..9351067969050 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
@@ -5,10 +5,10 @@ target triple = "powerpc64--linux"
 
 define i32 @foo(i32 %guard, ...) {
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -23,10 +23,10 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[A]], i64 800)
 ; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
   %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64le.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64le.ll
index d6b956c821e54..4151f3b223b3a 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64le.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64le.ll
@@ -5,10 +5,10 @@ target triple = "powerpc64le--linux"
 
 define i32 @foo(i32 %guard, ...) {
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -23,10 +23,10 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK: [[D:%.*]] = call i64 @llvm.umin.i64(i64 [[A]], i64 800)
 ; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[C]], ptr align 8 @__msan_va_arg_tls, i64 [[D]], i1 false)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
   %1 = call i32 (i32, ...) @foo(i32 0, i32 1, i64 2, double 3.000000e+00)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppc.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppc.ll
index 246db9dcb8d42..29d1fbd053ecb 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppc.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppc.ll
@@ -18,7 +18,7 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 2147483647
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i32 [[TMP5]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i32(ptr align 8 [[TMP6]], i8 0, i32 4, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VL]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[VL]] to i32
 ; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP7]], 2147483647
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i32 [[TMP8]] to ptr
@@ -50,15 +50,15 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP31:%.*]] = inttoptr i32 [[TMP30]] to ptr
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[TMP17]], ptr align 4 [[TMP31]], i32 [[TMP21]], i1 false)
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VL]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VL]])
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 0
 ;
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -67,10 +67,10 @@ define i32 @foo(i32 %guard, ...) {
 
 
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
 ; CHECK-LABEL: define i32 @bar() {
diff --git a/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppcle.ll b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppcle.ll
index 4a7b7b289a20d..a4d2e165dd3a8 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppcle.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/PowerPC32/vararg-ppcle.ll
@@ -18,7 +18,7 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], 2147483647
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i32 [[TMP5]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i32(ptr align 8 [[TMP6]], i8 0, i32 4, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VL]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[VL]] to i32
 ; CHECK-NEXT:    [[TMP8:%.*]] = and i32 [[TMP7]], 2147483647
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i32 [[TMP8]] to ptr
@@ -50,15 +50,15 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP31:%.*]] = inttoptr i32 [[TMP30]] to ptr
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr align 4 [[TMP17]], ptr align 4 [[TMP31]], i32 [[TMP21]], i1 false)
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VL]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VL]])
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 0
 ;
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -67,10 +67,10 @@ define i32 @foo(i32 %guard, ...) {
 
 
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
 ; CHECK-LABEL: define i32 @bar() {
diff --git a/llvm/test/Instrumentation/MemorySanitizer/RISCV32/vararg-riscv32.ll b/llvm/test/Instrumentation/MemorySanitizer/RISCV32/vararg-riscv32.ll
index 50e7be13049eb..0c6e75c331012 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/RISCV32/vararg-riscv32.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/RISCV32/vararg-riscv32.ll
@@ -18,7 +18,7 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], -2147483649
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP7]], i8 0, i64 8, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VL]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[VL]] to i64
 ; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], -2147483649
 ; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
@@ -32,15 +32,15 @@ define i32 @foo(i32 %guard, ...) {
 ; CHECK-NEXT:    [[TMP16:%.*]] = inttoptr i64 [[TMP15]] to ptr
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP16]], ptr align 8 [[TMP3]], i64 [[TMP2]], i1 false)
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VL]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VL]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VL]])
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 0
 ;
   %vl = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i32 0
 }
 
@@ -49,10 +49,10 @@ define i32 @foo(i32 %guard, ...) {
 
 
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 define i32 @bar() {
 ; CHECK-LABEL: define i32 @bar() {
diff --git a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
index e0b5907719afc..c340d155cad4c 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg-kernel.ll
@@ -4,17 +4,17 @@ target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-a:8:16-n32:64"
 target triple = "s390x-unknown-linux-gnu"
 
 %struct.__va_list = type { i64, i64, ptr, ptr }
-declare void @llvm.lifetime.start.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
 declare void @llvm.va_start(ptr)
 declare void @llvm.va_end(ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 
 define i64 @foo(i64 %guard, ...) #1 {
   %vl = alloca %struct.__va_list
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i64 0
 }
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll
index 009aef970ba70..91b21ea9c37ad 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/SystemZ/vararg.ll
@@ -7,10 +7,10 @@ target triple = "s390x-unknown-linux-gnu"
 
 define i64 @foo(i64 %guard, ...) {
   %vl = alloca %struct.__va_list, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.start.p0(ptr %vl)
   call void @llvm.va_start(ptr %vl)
   call void @llvm.va_end(ptr %vl)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %vl)
+  call void @llvm.lifetime.end.p0(ptr %vl)
   ret i64 0
 }
 
@@ -28,10 +28,10 @@ define i64 @foo(i64 %guard, ...) {
 ; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 {{%.*}}, ptr align 8 {{%.*}}, i64 160, i1 false)
 ; CHECK: call void @llvm.memcpy.p0.p0.i64(ptr align 8 {{%.*}}, ptr align 8 {{%.*}}, i64 [[A]], i1 false)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 declare i32 @random_i32()
 declare i64 @random_i64()
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx10_2_512ni-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx10_2_512ni-intrinsics.ll
new file mode 100644
index 0000000000000..298dc4b2c853a
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx10_2_512ni-intrinsics.ll
@@ -0,0 +1,815 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avx10.2-512 -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avx10_2_512ni-intrinsics.ll
+;
+; Handled strictly:
+; - llvm.x86.avx10.vdpphps.512
+; - llvm.x86.avx10.vmpsadbw.512
+;
+; Handled heuristically: (none)
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <16 x float> @test_mm512_dpph_ps(<16 x float> %__W, <32 x half> %__A, <32 x half> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x float> @test_mm512_dpph_ps(
+; CHECK-SAME: <16 x float> [[__W:%.*]], <32 x half> [[__A:%.*]], <32 x half> [[__B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = tail call <16 x float> @llvm.x86.avx10.vdpphps.512(<16 x float> [[__W]], <32 x half> [[__A]], <32 x half> [[__B]])
+; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %res = tail call <16 x float> @llvm.x86.avx10.vdpphps.512(<16 x float> %__W, <32 x half> %__A, <32 x half> %__B)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_mask_dpph_ps(<16 x float> %__W, i16 zeroext %__U, <32 x half> %__A, <32 x half> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x float> @test_mm512_mask_dpph_ps(
+; CHECK-SAME: <16 x float> [[__W:%.*]], i16 zeroext [[__U:%.*]], <32 x half> [[__A:%.*]], <32 x half> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[DPH:%.*]] = tail call <16 x float> @llvm.x86.avx10.vdpphps.512(<16 x float> [[__W]], <32 x half> [[__A]], <32 x half> [[__B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[BST]], <16 x i32> zeroinitializer, <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[DPH]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <16 x float> [[__W]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <16 x i32> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP16]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x float> [[DPH]], <16 x float> [[__W]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %dph = tail call <16 x float> @llvm.x86.avx10.vdpphps.512(<16 x float> %__W, <32 x half> %__A, <32 x half> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x float> %dph, <16 x float> %__W
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mm512_maskz_dpph_ps(i16 zeroext %__U, <16 x float> %__W, <32 x half> %__A, <32 x half> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x float> @test_mm512_maskz_dpph_ps(
+; CHECK-SAME: i16 zeroext [[__U:%.*]], <16 x float> [[__W:%.*]], <32 x half> [[__A:%.*]], <32 x half> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i16> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[DPH:%.*]] = tail call <16 x float> @llvm.x86.avx10.vdpphps.512(<16 x float> [[__W]], <32 x half> [[__A]], <32 x half> [[__B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <16 x i1> [[BST]], <16 x i32> zeroinitializer, <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x float> [[DPH]] to <16 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP11]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x float> [[DPH]], <16 x float> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x float> [[RES]]
+;
+  %dph = tail call <16 x float> @llvm.x86.avx10.vdpphps.512(<16 x float> %__W, <32 x half> %__A, <32 x half> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x float> %dph, <16 x float> zeroinitializer
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx10.vdpphps.512(<16 x float>, <32 x half>, <32 x half>)
+
+
+define <16 x i32> @test_mm512_dpbssd_epi32(<16 x i32> %__W, <16 x i32> %__A, ptr %pB) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_dpbssd_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], ptr [[PB:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[__B:%.*]] = load <16 x i32>, ptr [[PB]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PB]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <16 x i32> [[__A]] to <64 x i8>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i32> [[__B]] to <64 x i8>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i32> [[_MSLD]] to <64 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <64 x i8> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <64 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <64 x i8> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <64 x i8> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <64 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <64 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <64 x i1> [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or <64 x i1> [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <64 x i1> [[TMP20]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = sext <64 x i1> [[TMP21]] to <64 x i8>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <64 x i8> [[TMP22]] to <32 x i16>
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp ne <32 x i16> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = sext <32 x i1> [[TMP24]] to <32 x i16>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <32 x i16> [[TMP25]] to i512
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast i512 [[TMP26]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP27]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpbssd.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %__B = load <16 x i32>, ptr %pB
+  %res = tail call <16 x i32> @llvm.x86.avx10.vpdpbssd.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_mask_dpbssds_epi32(<16 x i32> %__W, i16 zeroext %__U, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_mask_dpbssds_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], i16 zeroext [[__U:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i32> [[__A]] to <64 x i8>
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <16 x i32> [[__B]] to <64 x i8>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <16 x i32> [[TMP2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp ne <64 x i8> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <64 x i8> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <64 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <64 x i8> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i1> [[TMP28]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i1> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <64 x i1> [[TMP28]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <64 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <64 x i1> [[TMP17]] to <64 x i8>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <64 x i8> [[TMP18]] to <32 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ne <32 x i16> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = sext <32 x i1> [[TMP20]] to <32 x i16>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <32 x i16> [[TMP21]] to i512
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i512 [[TMP22]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP23]], [[TMP1]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpbssds.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> [[__W]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpbssds.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> %__W
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_maskz_dpbssd_epi32(i16 zeroext %__U, <16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_maskz_dpbssd_epi32(
+; CHECK-SAME: i16 zeroext [[__U:%.*]], <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <16 x i32> [[__A]] to <64 x i8>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <16 x i32> [[__B]] to <64 x i8>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <16 x i32> [[TMP2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp ne <64 x i8> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <64 x i8> [[TMP28]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <64 x i8> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <64 x i8> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i1> [[TMP29]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i1> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <64 x i1> [[TMP29]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <64 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <64 x i1> [[TMP17]] to <64 x i8>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <64 x i8> [[TMP18]] to <32 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ne <32 x i16> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = sext <32 x i1> [[TMP20]] to <32 x i16>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <32 x i16> [[TMP21]] to i512
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i512 [[TMP22]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpbssd.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpbssd.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> zeroinitializer
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx10.vpdpbssd.512(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <16 x i32> @llvm.x86.avx10.vpdpbssds.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+define <16 x i32> @test_mm512_dpbsud_epi32(<16 x i32> %__W, <16 x i32> %__A, ptr %pB) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_dpbsud_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], ptr [[PB:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[__B:%.*]] = load <16 x i32>, ptr [[PB]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PB]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[RES:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpbsud.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %__B = load <16 x i32>, ptr %pB
+  %res = tail call <16 x i32> @llvm.x86.avx10.vpdpbsud.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_mask_dpbsuds_epi32(<16 x i32> %__W, i16 zeroext %__U, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_mask_dpbsuds_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], i16 zeroext [[__U:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpbsuds.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> [[__W]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpbsuds.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> %__W
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_maskz_dpbsud_epi32(i16 zeroext %__U, <16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_maskz_dpbsud_epi32(
+; CHECK-SAME: i16 zeroext [[__U:%.*]], <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpbsud.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpbsud.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> zeroinitializer
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx10.vpdpbsud.512(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <16 x i32> @llvm.x86.avx10.vpdpbsuds.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+define <16 x i32> @test_mm512_dpbuud_epi32(<16 x i32> %__W, <16 x i32> %__A, ptr %pB) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_dpbuud_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], ptr [[PB:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[__B:%.*]] = load <16 x i32>, ptr [[PB]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PB]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[RES:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpbuud.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %__B = load <16 x i32>, ptr %pB
+  %res = tail call <16 x i32> @llvm.x86.avx10.vpdpbuud.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_mask_dpbuuds_epi32(<16 x i32> %__W, i16 zeroext %__U, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_mask_dpbuuds_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], i16 zeroext [[__U:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpbuuds.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> [[__W]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpbuuds.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> %__W
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_maskz_dpbuud_epi32(i16 zeroext %__U, <16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_maskz_dpbuud_epi32(
+; CHECK-SAME: i16 zeroext [[__U:%.*]], <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpbuud.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpbuud.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> zeroinitializer
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx10.vpdpbuud.512(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <16 x i32> @llvm.x86.avx10.vpdpbuuds.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+
+define <16 x i32> @test_mm512_dpwsud_epi32(<16 x i32> %__W, <16 x i32> %__A, ptr %pB) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_dpwsud_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], ptr [[PB:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[__B:%.*]] = load <16 x i32>, ptr [[PB]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PB]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[RES:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpwsud.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %__B = load <16 x i32>, ptr %pB
+  %res = tail call <16 x i32> @llvm.x86.avx10.vpdpwsud.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_mask_dpwsuds_epi32(<16 x i32> %__W, i16 zeroext %__U, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_mask_dpwsuds_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], i16 zeroext [[__U:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpwsuds.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> [[__W]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpwsuds.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> %__W
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_maskz_dpwsud_epi32(i16 zeroext %__U, <16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_maskz_dpwsud_epi32(
+; CHECK-SAME: i16 zeroext [[__U:%.*]], <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpwsud.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpwsud.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> zeroinitializer
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx10.vpdpwsud.512(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <16 x i32> @llvm.x86.avx10.vpdpwsuds.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+define <16 x i32> @test_mm512_dpwusd_epi32(<16 x i32> %__W, <16 x i32> %__A, ptr %pB) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_dpwusd_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], ptr [[PB:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[__B:%.*]] = load <16 x i32>, ptr [[PB]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PB]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[RES:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpwusd.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %__B = load <16 x i32>, ptr %pB
+  %res = tail call <16 x i32> @llvm.x86.avx10.vpdpwusd.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_mask_dpwusds_epi32(<16 x i32> %__W, i16 zeroext %__U, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_mask_dpwusds_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], i16 zeroext [[__U:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpwusds.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> [[__W]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpwusds.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> %__W
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_maskz_dpwusd_epi32(i16 zeroext %__U, <16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_maskz_dpwusd_epi32(
+; CHECK-SAME: i16 zeroext [[__U:%.*]], <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpwusd.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpwusd.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> zeroinitializer
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx10.vpdpwusd.512(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <16 x i32> @llvm.x86.avx10.vpdpwusds.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+define <16 x i32> @test_mm512_dpwuud_epi32(<16 x i32> %__W, <16 x i32> %__A, ptr %pB) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_dpwuud_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], ptr [[PB:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[__B:%.*]] = load <16 x i32>, ptr [[PB]], align 64
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[PB]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP8]], align 64
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[RES:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpwuud.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %__B = load <16 x i32>, ptr %pB
+  %res = tail call <16 x i32> @llvm.x86.avx10.vpdpwuud.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_mask_dpwuuds_epi32(<16 x i32> %__W, i16 zeroext %__U, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_mask_dpwuuds_epi32(
+; CHECK-SAME: <16 x i32> [[__W:%.*]], i16 zeroext [[__U:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpwuuds.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> [[__W]]
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpwuuds.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> %__W
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mm512_maskz_dpwuud_epi32(i16 zeroext %__U, <16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_mm512_maskz_dpwuud_epi32(
+; CHECK-SAME: i16 zeroext [[__U:%.*]], <16 x i32> [[__W:%.*]], <16 x i32> [[__A:%.*]], <16 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <16 x i32> @llvm.x86.avx10.vpdpwuud.512(<16 x i32> [[__W]], <16 x i32> [[__A]], <16 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i16 [[__U]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[_MSPROP1]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <16 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <16 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <16 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP9]], <16 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <16 x i1> [[BST]], <16 x i32> [[DPI]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[RES]]
+;
+  %dpi = tail call <16 x i32> @llvm.x86.avx10.vpdpwuud.512(<16 x i32> %__W, <16 x i32> %__A, <16 x i32> %__B)
+  %bst = bitcast i16 %__U to <16 x i1>
+  %res = select <16 x i1> %bst, <16 x i32> %dpi, <16 x i32> zeroinitializer
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx10.vpdpwuud.512(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <16 x i32> @llvm.x86.avx10.vpdpwuuds.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+
+define { <32 x i16>, <32 x i16>, <32 x i16> } @test_mm512_mask_mpsadbw(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x3, i32 %x4) sanitize_memory {
+; CHECK-LABEL: define { <32 x i16>, <32 x i16>, <32 x i16> } @test_mm512_mask_mpsadbw(
+; CHECK-SAME: <64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]], <32 x i16> [[X3:%.*]], i32 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32 [[TMP1]] to <32 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i32 [[X4]] to <32 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <64 x i8> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i512 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RS1:%.*]] = call <32 x i16> @llvm.x86.avx10.vmpsadbw.512(<64 x i8> [[X0]], <64 x i8> [[X1]], i8 2)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <64 x i8> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i512 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[AD2:%.*]] = call <32 x i16> @llvm.x86.avx10.vmpsadbw.512(<64 x i8> [[X0]], <64 x i8> [[X1]], i8 3)
+; CHECK-NEXT:    [[TMP14:%.*]] = select <32 x i1> [[MSK]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <32 x i16> [[AD2]], [[X3]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i16> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i16> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP17]], <32 x i16> [[TMP14]]
+; CHECK-NEXT:    [[RS2:%.*]] = select <32 x i1> [[MSK]], <32 x i16> [[AD2]], <32 x i16> [[X3]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i512 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <64 x i8> [[TMP3]] to i512
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i512 [[TMP19]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    br i1 [[_MSOR8]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR5]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[AD3:%.*]] = call <32 x i16> @llvm.x86.avx10.vmpsadbw.512(<64 x i8> [[X0]], <64 x i8> [[X1]], i8 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = select <32 x i1> [[MSK]], <32 x i16> zeroinitializer, <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <32 x i16> [[AD3]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <32 x i16> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = or <32 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <32 x i1> [[TMP5]], <32 x i16> [[TMP25]], <32 x i16> [[TMP22]]
+; CHECK-NEXT:    [[RS3:%.*]] = select <32 x i1> [[MSK]], <32 x i16> [[AD3]], <32 x i16> zeroinitializer
+; CHECK-NEXT:    [[RS4:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> [[RS1]], 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } { <32 x i16> zeroinitializer, <32 x i16> splat (i16 -1), <32 x i16> splat (i16 -1) }, <32 x i16> [[_MSPROP_SELECT]], 1
+; CHECK-NEXT:    [[RS5:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RS4]], <32 x i16> [[RS2]], 1
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP26]], <32 x i16> [[_MSPROP_SELECT1]], 2
+; CHECK-NEXT:    [[RS6:%.*]] = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } [[RS5]], <32 x i16> [[RS3]], 2
+; CHECK-NEXT:    store { <32 x i16>, <32 x i16>, <32 x i16> } [[TMP27]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <32 x i16>, <32 x i16>, <32 x i16> } [[RS6]]
+;
+  %msk = bitcast i32 %x4 to <32 x i1>
+  %rs1 = call <32 x i16> @llvm.x86.avx10.vmpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i8 2)
+  %ad2 = call <32 x i16> @llvm.x86.avx10.vmpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i8 3)
+  %rs2 = select <32 x i1> %msk, <32 x i16> %ad2, <32 x i16> %x3
+  %ad3 = call <32 x i16> @llvm.x86.avx10.vmpsadbw.512(<64 x i8> %x0, <64 x i8> %x1, i8 4)
+  %rs3 = select <32 x i1> %msk, <32 x i16> %ad3, <32 x i16> zeroinitializer
+  %rs4 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } poison, <32 x i16> %rs1, 0
+  %rs5 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } %rs4, <32 x i16> %rs2, 1
+  %rs6 = insertvalue { <32 x i16>, <32 x i16>, <32 x i16> } %rs5, <32 x i16> %rs3, 2
+  ret { <32 x i16>, <32 x i16>, <32 x i16> } %rs6
+}
+
+declare <32 x i16> @llvm.x86.avx10.vmpsadbw.512(<64 x i8>, <64 x i8>, i8)
+
+
+define <8 x float> @avx_dp_ps(<8 x float> %a, <8 x float> %b) sanitize_memory {
+; CHECK-LABEL: define <8 x float> @avx_dp_ps(
+; CHECK-SAME: <8 x float> [[A:%.*]], <8 x float> [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false>, <8 x i32> [[TMP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP4]])
+; CHECK-NEXT:    [[_MSDPP:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[_MSDPP]], <8 x i1> zeroinitializer, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <8 x i32> [[TMP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.or.v8i32(<8 x i32> [[TMP7]])
+; CHECK-NEXT:    [[_MSDPP1:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[_MSDPP1]], <8 x i1> zeroinitializer, <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i1> [[TMP6]], [[TMP9]]
+; CHECK-NEXT:    [[_MSDPP2:%.*]] = sext <8 x i1> [[TMP10]] to <8 x i32>
+; CHECK-NEXT:    [[R:%.*]] = tail call <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float> [[A]], <8 x float> [[B]], i8 -1)
+; CHECK-NEXT:    store <8 x i32> [[_MSDPP2]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[R]]
+;
+  %r = tail call <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float> %a, <8 x float> %b, i8 -1)
+  ret <8 x float> %r
+}
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx10_2ni-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx10_2ni-intrinsics.ll
new file mode 100644
index 0000000000000..e3a26ae07ac1b
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx10_2ni-intrinsics.ll
@@ -0,0 +1,1176 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avx10.2-256 -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avx10_2ni-intrinsics.ll
+;
+; Handled strictly:
+; - llvm.x86.avx10.vdpphps.128
+; - llvm.x86.avx10.vdpphps.128
+; - llvm.x86.avx10.vdpphps.256
+; - llvm.x86.avx2.mpsadbw
+; - llvm.x86.sse41.mpsadbw
+;
+; Handled heuristically: (none)
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <4 x float> @test_mm_dpph_ps(<4 x float> %__W, <8 x half> %__A, <8 x half> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x float> @test_mm_dpph_ps(
+; CHECK-SAME: <4 x float> [[__W:%.*]], <8 x half> [[__A:%.*]], <8 x half> [[__B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = tail call <4 x float> @llvm.x86.avx10.vdpphps.128(<4 x float> [[__W]], <8 x half> [[__A]], <8 x half> [[__B]])
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %res = tail call <4 x float> @llvm.x86.avx10.vdpphps.128(<4 x float> %__W, <8 x half> %__A, <8 x half> %__B)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm_mask_dpph_ps(<4 x float> %__W, i8 zeroext %__U, <8 x half> %__A, <8 x half> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x float> @test_mm_mask_dpph_ps(
+; CHECK-SAME: <4 x float> [[__W:%.*]], i8 zeroext [[__U:%.*]], <8 x half> [[__A:%.*]], <8 x half> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[DPH:%.*]] = tail call <4 x float> @llvm.x86.avx10.vdpphps.128(<4 x float> [[__W]], <8 x half> [[__A]], <8 x half> [[__B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> splat (i1 true), <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <8 x i1> [[BST]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXT]], <4 x i32> zeroinitializer, <4 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x float> [[DPH]] to <4 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x float> [[__W]] to <4 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <4 x i32> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP16]], <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[EXT]], <4 x float> [[DPH]], <4 x float> [[__W]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %dph = tail call <4 x float> @llvm.x86.avx10.vdpphps.128(<4 x float> %__W, <8 x half> %__A, <8 x half> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %ext = shufflevector <8 x i1> %bst, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res = select <4 x i1> %ext, <4 x float> %dph, <4 x float> %__W
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mm_maskz_dpph_ps(i8 zeroext %__U, <4 x float> %__W, <8 x half> %__A, <8 x half> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x float> @test_mm_maskz_dpph_ps(
+; CHECK-SAME: i8 zeroext [[__U:%.*]], <4 x float> [[__W:%.*]], <8 x half> [[__A:%.*]], <8 x half> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i16> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[DPH:%.*]] = tail call <4 x float> @llvm.x86.avx10.vdpphps.128(<4 x float> [[__W]], <8 x half> [[__A]], <8 x half> [[__B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP:%.*]] = shufflevector <8 x i1> [[TMP10]], <8 x i1> splat (i1 true), <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXT:%.*]] = shufflevector <8 x i1> [[BST]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[EXT]], <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x float> [[DPH]] to <4 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <4 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <4 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <4 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP]], <4 x i32> [[TMP15]], <4 x i32> [[TMP11]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[EXT]], <4 x float> [[DPH]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x float> [[RES]]
+;
+  %dph = tail call <4 x float> @llvm.x86.avx10.vdpphps.128(<4 x float> %__W, <8 x half> %__A, <8 x half> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %ext = shufflevector <8 x i1> %bst, <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res = select <4 x i1> %ext, <4 x float> %dph, <4 x float> zeroinitializer
+  ret <4 x float> %res
+}
+
+define <8 x float> @test_mm256_dpph_ps(<8 x float> %__W, <16 x half> %__A, <16 x half> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x float> @test_mm256_dpph_ps(
+; CHECK-SAME: <8 x float> [[__W:%.*]], <16 x half> [[__A:%.*]], <16 x half> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i16> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB7:.*]], label %[[BB8:.*]], !prof [[PROF1]]
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    [[RES:%.*]] = tail call <8 x float> @llvm.x86.avx10.vdpphps.256(<8 x float> [[__W]], <16 x half> [[__A]], <16 x half> [[__B]])
+; CHECK-NEXT:    store <8 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %res = tail call <8 x float> @llvm.x86.avx10.vdpphps.256(<8 x float> %__W, <16 x half> %__A, <16 x half> %__B)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm256_mask_dpph_ps(<8 x float> %__W, i8 zeroext %__U, <16 x half> %__A, <16 x half> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x float> @test_mm256_mask_dpph_ps(
+; CHECK-SAME: <8 x float> [[__W:%.*]], i8 zeroext [[__U:%.*]], <16 x half> [[__A:%.*]], <16 x half> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i16> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[DPH:%.*]] = tail call <8 x float> @llvm.x86.avx10.vdpphps.256(<8 x float> [[__W]], <16 x half> [[__A]], <16 x half> [[__B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[BST]], <8 x i32> zeroinitializer, <8 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x float> [[DPH]] to <8 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x float> [[__W]] to <8 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = xor <8 x i32> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i32> [[TMP16]], <8 x i32> [[TMP11]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x float> [[DPH]], <8 x float> [[__W]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %dph = tail call <8 x float> @llvm.x86.avx10.vdpphps.256(<8 x float> %__W, <16 x half> %__A, <16 x half> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x float> %dph, <8 x float> %__W
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mm256_maskz_dpph_ps(i8 zeroext %__U, <8 x float> %__W, <16 x half> %__A, <16 x half> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x float> @test_mm256_maskz_dpph_ps(
+; CHECK-SAME: i8 zeroext [[__U:%.*]], <8 x float> [[__W:%.*]], <16 x half> [[__A:%.*]], <16 x half> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i16> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i16> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR3:%.*]] = or i1 [[_MSOR]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR3]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[DPH:%.*]] = tail call <8 x float> @llvm.x86.avx10.vdpphps.256(<8 x float> [[__W]], <16 x half> [[__A]], <16 x half> [[__B]])
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = select <8 x i1> [[BST]], <8 x i32> zeroinitializer, <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x float> [[DPH]] to <8 x i32>
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <8 x i32> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <8 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i32> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP10]], <8 x i32> [[TMP15]], <8 x i32> [[TMP11]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x float> [[DPH]], <8 x float> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x float> [[RES]]
+;
+  %dph = tail call <8 x float> @llvm.x86.avx10.vdpphps.256(<8 x float> %__W, <16 x half> %__A, <16 x half> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x float> %dph, <8 x float> zeroinitializer
+  ret <8 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx10.vdpphps.128(<4 x float>, <8 x half>, <8 x half>)
+declare <8 x float> @llvm.x86.avx10.vdpphps.256(<8 x float>, <16 x half>, <16 x half>)
+
+
+define <4 x i32> @test_mm_mask_dpbssd_epi32(<4 x i32> %__W, i4 zeroext %__U, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_mask_dpbssd_epi32(
+; CHECK-SAME: <4 x i32> [[__W:%.*]], i4 zeroext [[__U:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <4 x i32> [[__A]] to <16 x i8>
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <4 x i32> [[__B]] to <16 x i8>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp ne <16 x i8> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <16 x i8> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP28]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <16 x i1> [[TMP28]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <16 x i1> [[TMP17]] to <16 x i8>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x i8> [[TMP18]] to <8 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ne <8 x i16> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = sext <8 x i1> [[TMP20]] to <8 x i16>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i16> [[TMP21]] to i128
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i128 [[TMP22]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP23]], [[TMP1]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpbssd.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> [[__W]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpbssd.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> %__W
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mm_maskz_dpbssds_epi32(i4 zeroext %__U, <4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_maskz_dpbssds_epi32(
+; CHECK-SAME: i4 zeroext [[__U:%.*]], <4 x i32> [[__W:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <4 x i32> [[__A]] to <16 x i8>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <4 x i32> [[__B]] to <16 x i8>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp ne <16 x i8> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP28]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i8> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <16 x i8> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP29]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <16 x i1> [[TMP29]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <16 x i1> [[TMP17]] to <16 x i8>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x i8> [[TMP18]] to <8 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ne <8 x i16> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = sext <8 x i1> [[TMP20]] to <8 x i16>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i16> [[TMP21]] to i128
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i128 [[TMP22]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpbssds.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpbssds.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> zeroinitializer
+  ret <4 x i32> %res
+}
+
+define <8 x i32> @test_mm256_maskz_dpbssds_epi32(<8 x i32> %__W, i8 zeroext %__U, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_maskz_dpbssds_epi32(
+; CHECK-SAME: <8 x i32> [[__W:%.*]], i8 zeroext [[__U:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i32> [[__A]] to <32 x i8>
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <8 x i32> [[__B]] to <32 x i8>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <8 x i32> [[TMP2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp ne <32 x i8> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <32 x i8> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP28]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <32 x i1> [[TMP28]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <32 x i1> [[TMP17]] to <32 x i8>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <32 x i8> [[TMP18]] to <16 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ne <16 x i16> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = sext <16 x i1> [[TMP20]] to <16 x i16>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <16 x i16> [[TMP21]] to i256
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i256 [[TMP22]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP23]], [[TMP1]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpbssds.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> [[__W]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpbssds.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> %__W
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mm256_mask_dpbssd_epi32(i8 zeroext %__U, <8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_mask_dpbssd_epi32(
+; CHECK-SAME: i8 zeroext [[__U:%.*]], <8 x i32> [[__W:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <8 x i32> [[__A]] to <32 x i8>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <8 x i32> [[__B]] to <32 x i8>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <8 x i32> [[TMP2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp ne <32 x i8> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[TMP28]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i8> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <32 x i8> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP29]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP11]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <32 x i1> [[TMP29]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <32 x i1> [[TMP17]] to <32 x i8>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <32 x i8> [[TMP18]] to <16 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp ne <16 x i16> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = sext <16 x i1> [[TMP20]] to <16 x i16>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <16 x i16> [[TMP21]] to i256
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast i256 [[TMP22]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP23]], [[TMP24]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpbssd.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpbssd.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> zeroinitializer
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpbssd.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.x86.avx2.vpdpbssds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpbssd.256(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpbssds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <4 x i32> @test_mm_mask_dpbsud_epi32(<4 x i32> %__W, i4 zeroext %__U, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_mask_dpbsud_epi32(
+; CHECK-SAME: <4 x i32> [[__W:%.*]], i4 zeroext [[__U:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpbsud.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> [[__W]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpbsud.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> %__W
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mm_maskz_dpbsuds_epi32(i4 zeroext %__U, <4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_maskz_dpbsuds_epi32(
+; CHECK-SAME: i4 zeroext [[__U:%.*]], <4 x i32> [[__W:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpbsuds.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpbsuds.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> zeroinitializer
+  ret <4 x i32> %res
+}
+
+define <8 x i32> @test_mm256_maskz_dpbsuds_epi32(<8 x i32> %__W, i8 zeroext %__U, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_maskz_dpbsuds_epi32(
+; CHECK-SAME: <8 x i32> [[__W:%.*]], i8 zeroext [[__U:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpbsuds.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> [[__W]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpbsuds.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> %__W
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mm256_mask_dpbsud_epi32(i8 zeroext %__U, <8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_mask_dpbsud_epi32(
+; CHECK-SAME: i8 zeroext [[__U:%.*]], <8 x i32> [[__W:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpbsud.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpbsud.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> zeroinitializer
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpbsud.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.x86.avx2.vpdpbsuds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpbsud.256(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpbsuds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <4 x i32> @test_mm_mask_dpbuud_epi32(<4 x i32> %__W, i4 zeroext %__U, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_mask_dpbuud_epi32(
+; CHECK-SAME: <4 x i32> [[__W:%.*]], i4 zeroext [[__U:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpbuud.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> [[__W]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpbuud.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> %__W
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mm_maskz_dpbuuds_epi32(i4 zeroext %__U, <4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_maskz_dpbuuds_epi32(
+; CHECK-SAME: i4 zeroext [[__U:%.*]], <4 x i32> [[__W:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpbuuds.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpbuuds.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> zeroinitializer
+  ret <4 x i32> %res
+}
+
+define <8 x i32> @test_mm256_maskz_dpbuuds_epi32(<8 x i32> %__W, i8 zeroext %__U, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_maskz_dpbuuds_epi32(
+; CHECK-SAME: <8 x i32> [[__W:%.*]], i8 zeroext [[__U:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpbuuds.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> [[__W]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpbuuds.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> %__W
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mm256_mask_dpbuud_epi32(i8 zeroext %__U, <8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_mask_dpbuud_epi32(
+; CHECK-SAME: i8 zeroext [[__U:%.*]], <8 x i32> [[__W:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpbuud.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpbuud.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> zeroinitializer
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpbuud.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.x86.avx2.vpdpbuuds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpbuud.256(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpbuuds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+
+define <4 x i32> @test_mm_mask_dpwsud_epi32(<4 x i32> %__W, i4 zeroext %__U, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_mask_dpwsud_epi32(
+; CHECK-SAME: <4 x i32> [[__W:%.*]], i4 zeroext [[__U:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpwsud.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> [[__W]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpwsud.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> %__W
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mm_maskz_dpwsuds_epi32(i4 zeroext %__U, <4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_maskz_dpwsuds_epi32(
+; CHECK-SAME: i4 zeroext [[__U:%.*]], <4 x i32> [[__W:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpwsuds.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpwsuds.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> zeroinitializer
+  ret <4 x i32> %res
+}
+
+define <8 x i32> @test_mm256_maskz_dpwsuds_epi32(<8 x i32> %__W, i8 zeroext %__U, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_maskz_dpwsuds_epi32(
+; CHECK-SAME: <8 x i32> [[__W:%.*]], i8 zeroext [[__U:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpwsuds.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> [[__W]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpwsuds.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> %__W
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mm256_mask_dpwsud_epi32(i8 zeroext %__U, <8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_mask_dpwsud_epi32(
+; CHECK-SAME: i8 zeroext [[__U:%.*]], <8 x i32> [[__W:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpwsud.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpwsud.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> zeroinitializer
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpwsud.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.x86.avx2.vpdpwsuds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpwsud.256(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpwsuds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <4 x i32> @test_mm_mask_dpwusd_epi32(<4 x i32> %__W, i4 zeroext %__U, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_mask_dpwusd_epi32(
+; CHECK-SAME: <4 x i32> [[__W:%.*]], i4 zeroext [[__U:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpwusd.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> [[__W]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpwusd.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> %__W
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mm_maskz_dpwusds_epi32(i4 zeroext %__U, <4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_maskz_dpwusds_epi32(
+; CHECK-SAME: i4 zeroext [[__U:%.*]], <4 x i32> [[__W:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpwusds.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpwusds.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> zeroinitializer
+  ret <4 x i32> %res
+}
+
+define <8 x i32> @test_mm256_maskz_dpwusds_epi32(<8 x i32> %__W, i8 zeroext %__U, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_maskz_dpwusds_epi32(
+; CHECK-SAME: <8 x i32> [[__W:%.*]], i8 zeroext [[__U:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpwusds.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> [[__W]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpwusds.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> %__W
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mm256_mask_dpwusd_epi32(i8 zeroext %__U, <8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_mask_dpwusd_epi32(
+; CHECK-SAME: i8 zeroext [[__U:%.*]], <8 x i32> [[__W:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpwusd.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpwusd.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> zeroinitializer
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpwusd.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.x86.avx2.vpdpwusds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpwusd.256(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpwusds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <4 x i32> @test_mm_mask_dpwuud_epi32(<4 x i32> %__W, i4 zeroext %__U, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_mask_dpwuud_epi32(
+; CHECK-SAME: <4 x i32> [[__W:%.*]], i4 zeroext [[__U:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpwuud.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> [[__W]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpwuud.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> %__W
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mm_maskz_dpwuuds_epi32(i4 zeroext %__U, <4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_mm_maskz_dpwuuds_epi32(
+; CHECK-SAME: i4 zeroext [[__U:%.*]], <4 x i32> [[__W:%.*]], <4 x i32> [[__A:%.*]], <4 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i4, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <4 x i32> @llvm.x86.avx2.vpdpwuuds.128(<4 x i32> [[__W]], <4 x i32> [[__A]], <4 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i4 [[TMP4]] to <4 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i4 [[__U]] to <4 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[_MSPROP1]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <4 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <4 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <4 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP9]], <4 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <4 x i1> [[BST]], <4 x i32> [[DPI]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %dpi = tail call <4 x i32> @llvm.x86.avx2.vpdpwuuds.128(<4 x i32> %__W, <4 x i32> %__A, <4 x i32> %__B)
+  %bst = bitcast i4 %__U to <4 x i1>
+  %res = select <4 x i1> %bst, <4 x i32> %dpi, <4 x i32> zeroinitializer
+  ret <4 x i32> %res
+}
+
+define <8 x i32> @test_mm256_maskz_dpwuuds_epi32(<8 x i32> %__W, i8 zeroext %__U, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_maskz_dpwuuds_epi32(
+; CHECK-SAME: <8 x i32> [[__W:%.*]], i8 zeroext [[__U:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpwuuds.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], [[__W]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], [[TMP1]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> [[__W]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpwuuds.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> %__W
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mm256_mask_dpwuud_epi32(i8 zeroext %__U, <8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_mm256_mask_dpwuud_epi32(
+; CHECK-SAME: i8 zeroext [[__U:%.*]], <8 x i32> [[__W:%.*]], <8 x i32> [[__A:%.*]], <8 x i32> [[__B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[DPI:%.*]] = tail call <8 x i32> @llvm.x86.avx2.vpdpwuud.256(<8 x i32> [[__W]], <8 x i32> [[__A]], <8 x i32> [[__B]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[BST:%.*]] = bitcast i8 [[__U]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[_MSPROP1]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <8 x i32> [[DPI]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = or <8 x i32> [[TMP7]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = or <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i32> [[TMP9]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[RES:%.*]] = select <8 x i1> [[BST]], <8 x i32> [[DPI]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP_SELECT]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %dpi = tail call <8 x i32> @llvm.x86.avx2.vpdpwuud.256(<8 x i32> %__W, <8 x i32> %__A, <8 x i32> %__B)
+  %bst = bitcast i8 %__U to <8 x i1>
+  %res = select <8 x i1> %bst, <8 x i32> %dpi, <8 x i32> zeroinitializer
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpwuud.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <4 x i32> @llvm.x86.avx2.vpdpwuuds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpwuud.256(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <8 x i32> @llvm.x86.avx2.vpdpwuuds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+
+define { <8 x i16>, <8 x i16>, <8 x i16> } @test_mask_mpsadbw_128(<16 x i8> %x0, <16 x i8> %x1, <8 x i16> %x3, i8 %x4) sanitize_memory {
+; CHECK-LABEL: define { <8 x i16>, <8 x i16>, <8 x i16> } @test_mask_mpsadbw_128(
+; CHECK-SAME: <16 x i8> [[X0:%.*]], <16 x i8> [[X1:%.*]], <8 x i16> [[X3:%.*]], i8 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 48) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8 [[TMP1]] to <8 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i8 [[X4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RS1:%.*]] = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> [[X0]], <16 x i8> [[X1]], i8 2)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[AD2:%.*]] = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> [[X0]], <16 x i8> [[X1]], i8 3)
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[MSK]], <8 x i16> zeroinitializer, <8 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i16> [[AD2]], [[X3]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i16> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i16> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP5]], <8 x i16> [[TMP17]], <8 x i16> [[TMP14]]
+; CHECK-NEXT:    [[RS2:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[AD2]], <8 x i16> [[X3]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i8> [[TMP2]] to i128
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i128 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <16 x i8> [[TMP3]] to i128
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i128 [[TMP19]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    br i1 [[_MSOR8]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[AD3:%.*]] = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> [[X0]], <16 x i8> [[X1]], i8 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[MSK]], <8 x i16> zeroinitializer, <8 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i16> [[AD3]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i16> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <8 x i1> [[TMP5]], <8 x i16> [[TMP25]], <8 x i16> [[TMP22]]
+; CHECK-NEXT:    [[RS3:%.*]] = select <8 x i1> [[MSK]], <8 x i16> [[AD3]], <8 x i16> zeroinitializer
+; CHECK-NEXT:    [[RS4:%.*]] = insertvalue { <8 x i16>, <8 x i16>, <8 x i16> } poison, <8 x i16> [[RS1]], 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertvalue { <8 x i16>, <8 x i16>, <8 x i16> } { <8 x i16> zeroinitializer, <8 x i16> splat (i16 -1), <8 x i16> splat (i16 -1) }, <8 x i16> [[_MSPROP_SELECT]], 1
+; CHECK-NEXT:    [[RS5:%.*]] = insertvalue { <8 x i16>, <8 x i16>, <8 x i16> } [[RS4]], <8 x i16> [[RS2]], 1
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i16>, <8 x i16>, <8 x i16> } [[TMP26]], <8 x i16> [[_MSPROP_SELECT1]], 2
+; CHECK-NEXT:    [[RS6:%.*]] = insertvalue { <8 x i16>, <8 x i16>, <8 x i16> } [[RS5]], <8 x i16> [[RS3]], 2
+; CHECK-NEXT:    store { <8 x i16>, <8 x i16>, <8 x i16> } [[TMP27]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i16>, <8 x i16>, <8 x i16> } [[RS6]]
+;
+  %msk = bitcast i8 %x4 to <8 x i1>
+  %rs1 = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %x0, <16 x i8> %x1, i8 2)
+  %ad2 = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %x0, <16 x i8> %x1, i8 3)
+  %rs2 = select <8 x i1> %msk, <8 x i16> %ad2, <8 x i16> %x3
+  %ad3 = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %x0, <16 x i8> %x1, i8 4)
+  %rs3 = select <8 x i1> %msk, <8 x i16> %ad3, <8 x i16> zeroinitializer
+  %rs4 = insertvalue { <8 x i16>, <8 x i16>, <8 x i16> } poison, <8 x i16> %rs1, 0
+  %rs5 = insertvalue { <8 x i16>, <8 x i16>, <8 x i16> } %rs4, <8 x i16> %rs2, 1
+  %rs6 = insertvalue { <8 x i16>, <8 x i16>, <8 x i16> } %rs5, <8 x i16> %rs3, 2
+  ret { <8 x i16>, <8 x i16>, <8 x i16> } %rs6
+}
+
+define { <16 x i16>, <16 x i16>, <16 x i16> } @test_mask_mpsadbw_256(<32 x i8> %x0, <32 x i8> %x1, <16 x i16> %x3, i16 %x4) sanitize_memory {
+; CHECK-LABEL: define { <16 x i16>, <16 x i16>, <16 x i16> } @test_mask_mpsadbw_256(
+; CHECK-SAME: <32 x i8> [[X0:%.*]], <32 x i8> [[X1:%.*]], <16 x i16> [[X3:%.*]], i16 [[X4:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 96) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i16 [[TMP1]] to <16 x i1>
+; CHECK-NEXT:    [[MSK:%.*]] = bitcast i16 [[X4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP6]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <32 x i8> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP2]]
+; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK:       [[BB8]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB9]]:
+; CHECK-NEXT:    [[RS1:%.*]] = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> [[X0]], <32 x i8> [[X1]], i8 2)
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP10]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <32 x i8> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP11]], 0
+; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    br i1 [[_MSOR5]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK:       [[BB12]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB13]]:
+; CHECK-NEXT:    [[AD2:%.*]] = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> [[X0]], <32 x i8> [[X1]], i8 3)
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[MSK]], <16 x i16> zeroinitializer, <16 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i16> [[AD2]], [[X3]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i16> [[TMP15]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i16> [[TMP16]], [[TMP4]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP5]], <16 x i16> [[TMP17]], <16 x i16> [[TMP14]]
+; CHECK-NEXT:    [[RS2:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[AD2]], <16 x i16> [[X3]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i8> [[TMP2]] to i256
+; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i256 [[TMP18]], 0
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <32 x i8> [[TMP3]] to i256
+; CHECK-NEXT:    [[_MSCMP7:%.*]] = icmp ne i256 [[TMP19]], 0
+; CHECK-NEXT:    [[_MSOR8:%.*]] = or i1 [[_MSCMP6]], [[_MSCMP7]]
+; CHECK-NEXT:    br i1 [[_MSOR8]], label %[[BB20:.*]], label %[[BB21:.*]], !prof [[PROF1]]
+; CHECK:       [[BB20]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB21]]:
+; CHECK-NEXT:    [[AD3:%.*]] = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> [[X0]], <32 x i8> [[X1]], i8 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[MSK]], <16 x i16> zeroinitializer, <16 x i16> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i16> [[AD3]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i16> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT1:%.*]] = select <16 x i1> [[TMP5]], <16 x i16> [[TMP25]], <16 x i16> [[TMP22]]
+; CHECK-NEXT:    [[RS3:%.*]] = select <16 x i1> [[MSK]], <16 x i16> [[AD3]], <16 x i16> zeroinitializer
+; CHECK-NEXT:    [[RS4:%.*]] = insertvalue { <16 x i16>, <16 x i16>, <16 x i16> } poison, <16 x i16> [[RS1]], 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertvalue { <16 x i16>, <16 x i16>, <16 x i16> } { <16 x i16> zeroinitializer, <16 x i16> splat (i16 -1), <16 x i16> splat (i16 -1) }, <16 x i16> [[_MSPROP_SELECT]], 1
+; CHECK-NEXT:    [[RS5:%.*]] = insertvalue { <16 x i16>, <16 x i16>, <16 x i16> } [[RS4]], <16 x i16> [[RS2]], 1
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i16>, <16 x i16>, <16 x i16> } [[TMP26]], <16 x i16> [[_MSPROP_SELECT1]], 2
+; CHECK-NEXT:    [[RS6:%.*]] = insertvalue { <16 x i16>, <16 x i16>, <16 x i16> } [[RS5]], <16 x i16> [[RS3]], 2
+; CHECK-NEXT:    store { <16 x i16>, <16 x i16>, <16 x i16> } [[TMP27]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i16>, <16 x i16>, <16 x i16> } [[RS6]]
+;
+  %msk = bitcast i16 %x4 to <16 x i1>
+  %rs1 = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %x0, <32 x i8> %x1, i8 2)
+  %ad2 = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %x0, <32 x i8> %x1, i8 3)
+  %rs2 = select <16 x i1> %msk, <16 x i16> %ad2, <16 x i16> %x3
+  %ad3 = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %x0, <32 x i8> %x1, i8 4)
+  %rs3 = select <16 x i1> %msk, <16 x i16> %ad3, <16 x i16> zeroinitializer
+  %rs4 = insertvalue { <16 x i16>, <16 x i16>, <16 x i16> } poison, <16 x i16> %rs1, 0
+  %rs5 = insertvalue { <16 x i16>, <16 x i16>, <16 x i16> } %rs4, <16 x i16> %rs2, 1
+  %rs6 = insertvalue { <16 x i16>, <16 x i16>, <16 x i16> } %rs5, <16 x i16> %rs3, 2
+  ret { <16 x i16>, <16 x i16>, <16 x i16> } %rs6
+}
+
+declare <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8>, <16 x i8>, i8)
+declare <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8>, <32 x i8>, i8)
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
index f916130fe53e5..29269ff333771 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
@@ -140,11 +140,20 @@ define <8 x i32> @test_x86_avx2_pmadd_wd(<16 x i16> %a0, <16 x i16> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP3]] to <8 x i32>
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i32> [[TMP4]], zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i32>
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <16 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <16 x i16> [[A0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <16 x i16> [[A1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = and <16 x i1> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP12]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <16 x i1> [[TMP4]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP11]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sext <16 x i1> [[TMP17]] to <16 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i16> [[TMP7]] to <8 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> [[A0]], <16 x i16> [[A1]])
 ; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[RES]]
 ;
@@ -677,11 +686,20 @@ define <16 x i16> @test_x86_avx2_pmadd_ub_sw(<32 x i8> %a0, <32 x i8> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i8> [[TMP3]] to <16 x i16>
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i16> [[TMP4]], zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i16>
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <32 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <32 x i8> [[A0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <32 x i8> [[A1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = and <32 x i1> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP12]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <32 x i1> [[TMP4]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP11]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <32 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sext <32 x i1> [[TMP17]] to <32 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i8> [[TMP7]] to <16 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i16>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0]], <32 x i8> [[A1]])
 ; CHECK-NEXT:    store <16 x i16> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[RES]]
 ;
@@ -706,11 +724,20 @@ define <16 x i16> @test_x86_avx2_pmadd_ub_sw_load_op0(ptr %ptr, <32 x i8> %a1) #
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor i64 [[TMP5]], 87960930222080
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i8>, ptr [[TMP7]], align 32
-; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i8> [[_MSLD]], [[TMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i8> [[TMP8]] to <16 x i16>
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i16> [[TMP9]], zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = sext <16 x i1> [[TMP10]] to <16 x i16>
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i8> [[_MSLD]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <32 x i8> [[A0]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <32 x i8> [[A1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <32 x i1> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <32 x i1> [[TMP17]], [[TMP10]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <32 x i1> [[TMP9]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <32 x i1> [[TMP16]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i1> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sext <32 x i1> [[TMP22]] to <32 x i8>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <32 x i8> [[TMP12]] to <16 x i16>
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp ne <16 x i16> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <16 x i1> [[TMP24]] to <16 x i16>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0]], <32 x i8> [[A1]])
 ; CHECK-NEXT:    store <16 x i16> [[TMP11]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[RES]]
 ;
@@ -740,8 +767,9 @@ define <32 x i8> @test_x86_avx2_pshuf_b(<32 x i8> %a0, <32 x i8> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[TMP1]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1]])
 ; CHECK-NEXT:    store <32 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i8> [[RES]]
 ;
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
index 3c1af6781f0ed..a2f1d65e7cd41 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
@@ -24,7 +24,6 @@
 ; - llvm.x86.avx512.mask.rndscale.pd.512, llvm.x86.avx512.mask.rndscale.ps.512, llvm.x86.avx512.mask.rndscale.sd, llvm.x86.avx512.mask.rndscale.ss
 ; - llvm.x86.avx512.mask.scalef.pd.512, llvm.x86.avx512.mask.scalef.ps.512
 ; - llvm.x86.avx512.mask.sqrt.sd, llvm.x86.avx512.mask.sqrt.ss
-; - llvm.x86.avx512.mask.vcvtps2ph.512
 ; - llvm.x86.avx512.maskz.fixupimm.pd.512, llvm.x86.avx512.maskz.fixupimm.ps.512, llvm.x86.avx512.maskz.fixupimm.sd, llvm.x86.avx512.maskz.fixupimm.ss
 ; - llvm.x86.avx512.mul.pd.512, llvm.x86.avx512.mul.ps.512
 ; - llvm.x86.avx512.permvar.df.512, llvm.x86.avx512.permvar.sf.512
@@ -1903,50 +1902,46 @@ define <16 x i16> @test_x86_vcvtps2ph_256(<16 x float> %a0, <16 x i16> %src, i16
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP7:%.*]], !prof [[PROF1]]
-; CHECK:       6:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
-; CHECK-NEXT:    unreachable
-; CHECK:       7:
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <16 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = sext <16 x i1> [[TMP6]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = select <16 x i1> splat (i1 true), <16 x i16> [[TMP7]], <16 x i16> zeroinitializer
 ; CHECK-NEXT:    [[RES1:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> [[A0:%.*]], i32 2, <16 x i16> zeroinitializer, i16 -1)
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP8]], 0
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = sext <16 x i1> [[TMP11]] to <16 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = select <16 x i1> [[TMP10]], <16 x i16> [[TMP12]], <16 x i16> zeroinitializer
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i16 [[TMP2]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP9:%.*]], label [[TMP10:%.*]], !prof [[PROF1]]
-; CHECK:       9:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label [[TMP9:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
+; CHECK:       7:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       10:
-; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> [[A0]], i32 11, <16 x i16> zeroinitializer, i16 [[MASK:%.*]])
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <16 x i32> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i512 [[TMP11]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <16 x i16> [[TMP3]] to i256
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i256 [[TMP12]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK:       8:
+; CHECK-NEXT:    [[RES2:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> [[A0]], i32 11, <16 x i16> zeroinitializer, i16 [[MASK]])
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast i16 [[MASK]] to <16 x i1>
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp ne <16 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = sext <16 x i1> [[TMP26]] to <16 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = select <16 x i1> [[TMP25]], <16 x i16> [[TMP27]], <16 x i16> [[TMP3]]
 ; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i16 [[TMP2]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label [[TMP13:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
-; CHECK:       13:
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
+; CHECK:       10:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       14:
+; CHECK:       11:
 ; CHECK-NEXT:    [[RES3:%.*]] = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> [[A0]], i32 12, <16 x i16> [[SRC:%.*]], i16 [[MASK]])
 ; CHECK-NEXT:    [[_MSCMP8:%.*]] = icmp ne i64 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP8]], label [[TMP15:%.*]], label [[TMP16:%.*]], !prof [[PROF1]]
-; CHECK:       15:
+; CHECK-NEXT:    br i1 [[_MSCMP8]], label [[TMP21:%.*]], label [[TMP22:%.*]], !prof [[PROF1]]
+; CHECK:       12:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       16:
+; CHECK:       13:
 ; CHECK-NEXT:    [[TMP17:%.*]] = ptrtoint ptr [[DST:%.*]] to i64
 ; CHECK-NEXT:    [[TMP18:%.*]] = xor i64 [[TMP17]], 87960930222080
 ; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
-; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr [[TMP19]], align 32
+; CHECK-NEXT:    store <16 x i16> [[TMP8]], ptr [[TMP19]], align 32
 ; CHECK-NEXT:    store <16 x i16> [[RES1]], ptr [[DST]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <16 x i16> [[TMP13]], [[TMP20]]
 ; CHECK-NEXT:    [[RES:%.*]] = add <16 x i16> [[RES2]], [[RES3]]
-; CHECK-NEXT:    store <16 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <16 x i16> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[RES]]
 ;
   %res1 = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> %a0, i32 2, <16 x i16> zeroinitializer, i16 -1)
@@ -7451,10 +7446,10 @@ define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_512(<16 x float> %x0, <16 x
 ; CHECK-NEXT:    [[TMP6:%.*]] = select <16 x i1> [[TMP3]], <16 x i32> [[TMP5]], <16 x i32> [[TMP2]]
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i16 [[TMP10]], 0
 ; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP11:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
-; CHECK:       8:
+; CHECK:       5:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR10]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       9:
+; CHECK:       6:
 ; CHECK-NEXT:    [[RES:%.*]] = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> [[X0:%.*]], <16 x i32> [[X1:%.*]], i16 [[X2]], i32 10)
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <16 x i32> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = sext <16 x i1> [[TMP7]] to <16 x i32>
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics-upgrade.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics-upgrade.ll
index 02df9c49a010b..51dad35a1edbc 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics-upgrade.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics-upgrade.ll
@@ -7,8 +7,6 @@
 ; - llvm.x86.avx512.dbpsadbw.512
 ; - llvm.x86.avx512.packssdw.512, llvm.x86.avx512.packsswb.512
 ; - llvm.x86.avx512.packusdw.512, llvm.x86.avx512.packuswb.512
-; - llvm.x86.avx512.pmaddubs.w.512
-; - llvm.x86.avx512.pmaddw.d.512
 ;
 ; Heuristically handled:
 ; - llvm.sadd.sat.v32i16, llvm.sadd.sat.v64i8
@@ -22,7 +20,6 @@
 ; - llvm.x86.avx512.pavg.b.512, llvm.x86.avx512.pavg.w.512
 ; - llvm.x86.avx512.permvar.hi.512
 ; - llvm.x86.avx512.pmul.hr.sw.512, llvm.x86.avx512.pmulhu.w.512, llvm.x86.avx512.pmulh.w.512
-; - llvm.x86.avx512.pshuf.b.512
 ; - llvm.x86.avx512.psllv.w.512, llvm.x86.avx512.psrav.w.512,  llvm.x86.avx512.psrlv.w.512
 
 target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -1968,8 +1965,9 @@ define <64 x i8> @test_int_x86_avx512_pshuf_b_512(<64 x i8> %x0, <64 x i8> %x1,
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[TMP1]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1]])
 ; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <64 x i8> [[TMP3]]
 ;
@@ -1984,8 +1982,9 @@ define <64 x i8> @test_int_x86_avx512_mask_pshuf_b_512(<64 x i8> %x0, <64 x i8>
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[TMP1]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[TMP13]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64 [[X3:%.*]] to <64 x i1>
 ; CHECK-NEXT:    [[TMP8:%.*]] = select <64 x i1> [[TMP7]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
@@ -4930,18 +4929,21 @@ define <32 x i16> @test_int_x86_avx512_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
-; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
-; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <64 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <64 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <64 x i8> [[X0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <64 x i8> [[X1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <64 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and <64 x i1> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <64 x i1> [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i1> [[TMP17]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sext <64 x i1> [[TMP11]] to <64 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <64 x i8> [[TMP12]] to <32 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = sext <32 x i1> [[TMP14]] to <32 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0]], <64 x i8> [[X1]])
+; CHECK-NEXT:    store <32 x i16> [[TMP16]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i16> [[TMP7]]
 ;
   %res = call <32 x i16> @llvm.x86.avx512.mask.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1, <32 x i16> %x2, i32 -1)
@@ -4955,22 +4957,25 @@ define <32 x i16> @test_int_x86_avx512_mask_pmaddubs_w_512(<64 x i8> %x0, <64 x
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
-; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <64 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <64 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <64 x i8> [[X0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <64 x i8> [[X1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = and <64 x i1> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <64 x i1> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <64 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <64 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <64 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = sext <64 x i1> [[TMP23]] to <64 x i8>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <64 x i8> [[TMP24]] to <32 x i16>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <32 x i1> [[TMP25]] to <32 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0]], <64 x i8> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP18]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[X2:%.*]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[X2]]
@@ -4988,18 +4993,21 @@ define <16 x i32> @test_int_x86_avx512_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
-; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <32 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <32 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i16> [[X0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <32 x i16> [[X1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <32 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and <32 x i1> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <32 x i1> [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i1> [[TMP17]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sext <32 x i1> [[TMP11]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <32 x i16> [[TMP12]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = sext <16 x i1> [[TMP14]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0]], <32 x i16> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP16]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
 ;
   %res = call <16 x i32> @llvm.x86.avx512.mask.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1, <16 x i32> %x2, i16 -1)
@@ -5013,22 +5021,25 @@ define <16 x i32> @test_int_x86_avx512_mask_pmaddw_d_512(<32 x i16> %x0, <32 x i
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR7]]
-; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <32 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <32 x i16> [[X0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i16> [[X1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = and <32 x i1> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <32 x i1> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <32 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <32 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = sext <32 x i1> [[TMP23]] to <32 x i16>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <32 x i16> [[TMP24]] to <16 x i32>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <16 x i1> [[TMP25]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0]], <32 x i16> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP18]], <16 x i32> [[TMP4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[X2:%.*]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[X2]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics.ll
index 78c272c7b2c5a..c6c7e002213bd 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics.ll
@@ -9,7 +9,6 @@
 ; - llvm.x86.avx512.mask.pmov.wb.mem.512
 ; - llvm.x86.avx512.packssdw.512, llvm.x86.avx512.packsswb.512
 ; - llvm.x86.avx512.packusdw.512, llvm.x86.avx512.packuswb.512
-; - llvm.x86.avx512.pmaddubs.w.512, llvm.x86.avx512.pmaddw.d.512
 ; - llvm.x86.avx512.psad.bw.512
 ;
 ; Heuristically handled:
@@ -17,7 +16,6 @@
 ; - llvm.x86.avx512.pavg.b.512, llvm.x86.avx512.pavg.w.512
 ; - llvm.x86.avx512.permvar.hi.512
 ; - llvm.x86.avx512.pmul.hr.sw.512, llvm.x86.avx512.pmulhu.w.512, llvm.x86.avx512.pmulh.w.512
-; - llvm.x86.avx512.pshuf.b.512
 ; - llvm.x86.avx512.psllv.w.512
 ; - llvm.x86.avx512.psrav.w.512, llvm.x86.avx512.psrlv.w.512
 
@@ -1714,8 +1712,9 @@ define <64 x i8>@test_int_x86_avx512_pshuf_b_512(<64 x i8> %x0, <64 x i8> %x1) #
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[RES:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP3:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[TMP1]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[RES:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1]])
 ; CHECK-NEXT:    store <64 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <64 x i8> [[RES]]
 ;
@@ -1730,8 +1729,9 @@ define <64 x i8>@test_int_x86_avx512_pshuf_b_512_mask(<64 x i8> %x0, <64 x i8> %
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[RES:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[TMP1]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[TMP10]]
+; CHECK-NEXT:    [[RES:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
 ; CHECK-NEXT:    [[TMP6:%.*]] = select <64 x i1> [[MASK_CAST]], <64 x i8> [[_MSPROP]], <64 x i8> [[TMP4]]
@@ -1755,8 +1755,9 @@ define <64 x i8>@test_int_x86_avx512_pshuf_b_512_maskz(<64 x i8> %x0, <64 x i8>
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[RES:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP9:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[TMP1]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <64 x i8> [[TMP2]], [[TMP9]]
+; CHECK-NEXT:    [[RES:%.*]] = call <64 x i8> @llvm.x86.avx512.pshuf.b.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64 [[TMP3]] to <64 x i1>
 ; CHECK-NEXT:    [[MASK_CAST:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <64 x i1> [[MASK_CAST]], <64 x i8> [[_MSPROP]], <64 x i8> zeroinitializer
@@ -2204,18 +2205,21 @@ define <32 x i16> @test_int_x86_avx512_pmaddubs_w_512(<64 x i8> %x0, <64 x i8> %
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <64 x i8>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <64 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
-; CHECK-NEXT:    store <32 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <64 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <64 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <64 x i8> [[X0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <64 x i8> [[X1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <64 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and <64 x i1> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <64 x i1> [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <64 x i1> [[TMP17]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <64 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sext <64 x i1> [[TMP11]] to <64 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <64 x i8> [[TMP12]] to <32 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = sext <32 x i1> [[TMP14]] to <32 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0]], <64 x i8> [[X1]])
+; CHECK-NEXT:    store <32 x i16> [[TMP16]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i16> [[TMP7]]
 ;
   %1 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %x0, <64 x i8> %x1)
@@ -2229,22 +2233,25 @@ define <32 x i16> @test_int_x86_avx512_mask_pmaddubs_w_512(<64 x i8> %x0, <64 x
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <64 x i8> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <64 x i8> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0:%.*]], <64 x i8> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <64 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <64 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <64 x i8> [[X0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <64 x i8> [[X1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = and <64 x i1> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <64 x i1> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <64 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <64 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <64 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = sext <64 x i1> [[TMP23]] to <64 x i8>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <64 x i8> [[TMP24]] to <32 x i16>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <32 x i16> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <32 x i1> [[TMP25]] to <32 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> [[X0]], <64 x i8> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32 [[TMP3]] to <32 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32 [[X3:%.*]] to <32 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> zeroinitializer, <32 x i16> [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP18]], <32 x i16> [[TMP4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <32 x i16> [[TMP9]], [[X2:%.*]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <32 x i16> [[TMP13]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i16> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <32 x i1> [[TMP10]], <32 x i16> [[TMP15]], <32 x i16> [[TMP12]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <32 x i1> [[TMP11]], <32 x i16> [[TMP9]], <32 x i16> [[X2]]
@@ -2264,18 +2271,21 @@ define <16 x i32> @test_int_x86_avx512_pmaddw_d_512(<32 x i16> %x0, <32 x i16> %
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP3]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP4]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP5:%.*]], label [[TMP6:%.*]], !prof [[PROF1]]
-; CHECK:       5:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       6:
-; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
-; CHECK-NEXT:    store <16 x i32> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <32 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <32 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i16> [[X0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <32 x i16> [[X1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <32 x i1> [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and <32 x i1> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and <32 x i1> [[TMP3]], [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <32 x i1> [[TMP17]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = or <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP12:%.*]] = sext <32 x i1> [[TMP11]] to <32 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <32 x i16> [[TMP12]] to <16 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = sext <16 x i1> [[TMP14]] to <16 x i32>
+; CHECK-NEXT:    [[TMP7:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0]], <32 x i16> [[X1]])
+; CHECK-NEXT:    store <16 x i32> [[TMP16]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i32> [[TMP7]]
 ;
   %1 = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> %x0, <32 x i16> %x1)
@@ -2289,22 +2299,25 @@ define <16 x i32> @test_int_x86_avx512_mask_pmaddw_d_512(<32 x i16> %x0, <32 x i
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 192) to ptr), align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <32 x i16> [[TMP1]] to i512
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i512 [[TMP5]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <32 x i16> [[TMP2]] to i512
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i512 [[TMP6]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP]], [[_MSCMP1]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label [[TMP7:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
-; CHECK:       7:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR8]]
-; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0:%.*]], <32 x i16> [[X1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <32 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <32 x i16> [[X0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i16> [[X1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = and <32 x i1> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <32 x i1> [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <32 x i1> [[TMP5]], [[TMP8]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i1> [[TMP19]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <32 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP24:%.*]] = sext <32 x i1> [[TMP23]] to <32 x i16>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <32 x i16> [[TMP24]] to <16 x i32>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <16 x i1> [[TMP25]] to <16 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i32> @llvm.x86.avx512.pmaddw.d.512(<32 x i16> [[X0]], <32 x i16> [[X1]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i16 [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i16 [[X3:%.*]] to <16 x i1>
-; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> zeroinitializer, <16 x i32> [[TMP4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP18]], <16 x i32> [[TMP4]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = xor <16 x i32> [[TMP9]], [[X2:%.*]]
-; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = or <16 x i32> [[TMP13]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i32> [[TMP14]], [[TMP4]]
 ; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP10]], <16 x i32> [[TMP15]], <16 x i32> [[TMP12]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <16 x i1> [[TMP11]], <16 x i32> [[TMP9]], <16 x i32> [[X2]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll
index db0c2e7ae9ed6..e2dc8cbdca968 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll
@@ -1,7 +1,74 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -mattr=+avx512f -passes=msan -S | FileCheck %s
-
+;
 ; Forked from llvm/test/CodeGen/X86/avx512vl-intrinsics.ll
+;
+; Handled strictly:
+; - llvm.x86.avx512.mask.cmp.pd.128, llvm.x86.avx512.mask.cmp.pd.256
+; - llvm.x86.avx512.mask.cmp.ps.128, llvm.x86.avx512.mask.cmp.ps.256
+; - llvm.x86.avx512.mask.compress.v2f64, llvm.x86.avx512.mask.compress.v4f32, llvm.x86.avx512.mask.compress.v4f64, llvm.x86.avx512.mask.compress.v8f32
+; - llvm.x86.avx512.mask.compress.v2i64, llvm.x86.avx512.mask.compress.v4i32, llvm.x86.avx512.mask.compress.v4i64, llvm.x86.avx512.mask.compress.v8i32
+; - llvm.x86.avx512.mask.cvtpd2dq.128
+; - llvm.x86.avx512.mask.cvtpd2ps
+; - llvm.x86.avx512.mask.cvtpd2udq.128, llvm.x86.avx512.mask.cvtpd2udq.256
+; - llvm.x86.avx512.mask.cvtps2dq.128, llvm.x86.avx512.mask.cvtps2dq.256
+; - llvm.x86.avx512.mask.cvtps2udq.128, llvm.x86.avx512.mask.cvtps2udq.256
+; - llvm.x86.avx512.mask.cvttpd2dq.128
+; - llvm.x86.avx512.mask.cvttpd2udq.128, llvm.x86.avx512.mask.cvttpd2udq.256
+; - llvm.x86.avx512.mask.cvttps2udq.128, llvm.x86.avx512.mask.cvttps2udq.256
+; - llvm.x86.avx512.mask.expand.v2f64, llvm.x86.avx512.mask.expand.v4f32, llvm.x86.avx512.mask.expand.v4f64, llvm.x86.avx512.mask.expand.v8f32
+; - llvm.x86.avx512.mask.expand.v2i64, llvm.x86.avx512.mask.expand.v4i32, llvm.x86.avx512.mask.expand.v4i64, llvm.x86.avx512.mask.expand.v8i32
+; - llvm.x86.avx512.mask.fixupimm.pd.128, llvm.x86.avx512.mask.fixupimm.pd.256
+; - llvm.x86.avx512.mask.fixupimm.ps.128 llvm.x86.avx512.mask.fixupimm.ps.256
+; - llvm.x86.avx512.mask.getexp.pd.256
+; - llvm.x86.avx512.mask.getexp.ps.256
+; - llvm.x86.avx512.mask.getmant.pd.128, llvm.x86.avx512.mask.getmant.pd.256
+; - llvm.x86.avx512.mask.getmant.ps.128, llvm.x86.avx512.mask.getmant.ps.256
+; - llvm.x86.avx512.mask.pmov.db.128, llvm.x86.avx512.mask.pmov.db.256
+; - llvm.x86.avx512.mask.pmov.db.mem.128, llvm.x86.avx512.mask.pmov.db.mem.256
+; - llvm.x86.avx512.mask.pmov.dw.128, llvm.x86.avx512.mask.pmov.dw.256
+; - llvm.x86.avx512.mask.pmov.dw.mem.128, llvm.x86.avx512.mask.pmov.dw.mem.256
+; - llvm.x86.avx512.mask.pmov.qb.128, llvm.x86.avx512.mask.pmov.qb.256
+; - llvm.x86.avx512.mask.pmov.qb.mem.128, llvm.x86.avx512.mask.pmov.qb.mem.256
+; - llvm.x86.avx512.mask.pmov.qd.128
+; - llvm.x86.avx512.mask.pmov.qd.mem.128, llvm.x86.avx512.mask.pmov.qd.mem.256
+; - llvm.x86.avx512.mask.pmov.qw.128, llvm.x86.avx512.mask.pmov.qw.256
+; - llvm.x86.avx512.mask.pmov.qw.mem.128, llvm.x86.avx512.mask.pmov.qw.mem.256
+; - llvm.x86.avx512.mask.pmovs.db.128, llvm.x86.avx512.mask.pmovs.db.256
+; - llvm.x86.avx512.mask.pmovs.db.mem.128, llvm.x86.avx512.mask.pmovs.db.mem.256
+; - llvm.x86.avx512.mask.pmovs.dw.128, llvm.x86.avx512.mask.pmovs.dw.256
+; - llvm.x86.avx512.mask.pmovs.dw.mem.128, llvm.x86.avx512.mask.pmovs.dw.mem.256
+; - llvm.x86.avx512.mask.pmovs.qb.128, llvm.x86.avx512.mask.pmovs.qb.256
+; - llvm.x86.avx512.mask.pmovs.qb.mem.128, llvm.x86.avx512.mask.pmovs.qb.mem.256
+; - llvm.x86.avx512.mask.pmovs.qd.128, llvm.x86.avx512.mask.pmovs.qd.256
+; - llvm.x86.avx512.mask.pmovs.qd.mem.128, llvm.x86.avx512.mask.pmovs.qd.mem.256
+; - llvm.x86.avx512.mask.pmovs.qw.128, llvm.x86.avx512.mask.pmovs.qw.256
+; - llvm.x86.avx512.mask.pmovs.qw.mem.128, llvm.x86.avx512.mask.pmovs.qw.mem.256
+; - llvm.x86.avx512.mask.pmovus.db.128, llvm.x86.avx512.mask.pmovus.db.256
+; - llvm.x86.avx512.mask.pmovus.db.mem.128, llvm.x86.avx512.mask.pmovus.db.mem.256
+; - llvm.x86.avx512.mask.pmovus.dw.128, llvm.x86.avx512.mask.pmovus.dw.256
+; - llvm.x86.avx512.mask.pmovus.dw.mem.128, llvm.x86.avx512.mask.pmovus.dw.mem.256
+; - llvm.x86.avx512.mask.pmovus.qb.128, llvm.x86.avx512.mask.pmovus.qb.256
+; - llvm.x86.avx512.mask.pmovus.qb.mem.128, llvm.x86.avx512.mask.pmovus.qb.mem.256
+; - llvm.x86.avx512.mask.pmovus.qd.128, llvm.x86.avx512.mask.pmovus.qd.256
+; - llvm.x86.avx512.mask.pmovus.qd.mem.128, llvm.x86.avx512.mask.pmovus.qd.mem.256
+; - llvm.x86.avx512.mask.pmovus.qw.128, llvm.x86.avx512.mask.pmovus.qw.256
+; - llvm.x86.avx512.mask.pmovus.qw.mem.128, llvm.x86.avx512.mask.pmovus.qw.mem.256
+; - llvm.x86.avx512.mask.rndscale.pd.128, llvm.x86.avx512.mask.rndscale.pd.256
+; - llvm.x86.avx512.mask.rndscale.ps.128, llvm.x86.avx512.mask.rndscale.ps.256
+; - llvm.x86.avx512.mask.scalef.pd.128, llvm.x86.avx512.mask.scalef.pd.256
+; - llvm.x86.avx512.mask.scalef.ps.128, llvm.x86.avx512.mask.scalef.ps.256
+; - llvm.x86.avx512.maskz.fixupimm.pd.128, llvm.x86.avx512.maskz.fixupimm.pd.256
+; - llvm.x86.avx512.maskz.fixupimm.ps.128, llvm.x86.avx512.maskz.fixupimm.ps.256
+; - llvm.x86.avx512.permvar.df.256
+; - llvm.x86.avx512.pternlog.d.128, llvm.x86.avx512.pternlog.d.256
+; - llvm.x86.avx512.pternlog.q.128, llvm.x86.avx512.pternlog.q.256
+; - llvm.x86.avx512.rcp14.pd.128, llvm.x86.avx512.rcp14.pd.256
+; - llvm.x86.avx512.rcp14.ps.128, llvm.x86.avx512.rcp14.ps.256
+; - llvm.x86.avx512.rsqrt14.pd.128, llvm.x86.avx512.rsqrt14.pd.256
+; - llvm.x86.avx512.rsqrt14.ps.128, llvm.x86.avx512.rsqrt14.ps.256
+;
+; Handled heuristically: (none)
 
 target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -7893,40 +7960,44 @@ define <8 x i16> @test_x86_vcvtps2ph_128(<4 x float> %a0, i8 %mask, <8 x i16> %s
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 24) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i128 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
-; CHECK:       [[BB5]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> bitcast (<1 x i8> splat (i8 15) to <8 x i1>), <8 x i16> [[TMP6]], <8 x i16> zeroinitializer
 ; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> [[A0]], i32 2, <8 x i16> zeroinitializer, i8 -1)
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i128 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc i8 [[MASK]] to i4
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i4 [[TMP9]] to i8
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i8 [[TMP10]] to <8 x i1>
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <8 x i32> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = sext <8 x i1> [[TMP12]] to <8 x i16>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP11]], <8 x i16> [[TMP13]], <8 x i16> zeroinitializer
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP2]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
 ; CHECK:       [[BB8]]:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       [[BB9]]:
 ; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> [[A0]], i32 10, <8 x i16> zeroinitializer, i8 [[MASK]])
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[TMP1]] to i128
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i128 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP11]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP18:%.*]] = trunc i8 [[MASK]] to i4
+; CHECK-NEXT:    [[TMP19:%.*]] = zext i4 [[TMP18]] to i8
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP19]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne <8 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = sext <8 x i1> [[TMP21]] to <8 x i16>
+; CHECK-NEXT:    [[TMP23:%.*]] = select <8 x i1> [[TMP20]], <8 x i16> [[TMP22]], <8 x i16> [[TMP3]]
 ; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP2]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
 ; CHECK:       [[BB12]]:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
 ; CHECK:       [[BB13]]:
 ; CHECK-NEXT:    [[RES3:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> [[A0]], i32 11, <8 x i16> [[SRC]], i8 [[MASK]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP7]], [[TMP14]]
 ; CHECK-NEXT:    [[RES0:%.*]] = add <8 x i16> [[RES1]], [[RES2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[TMP23]], [[_MSPROP]]
 ; CHECK-NEXT:    [[RES:%.*]] = add <8 x i16> [[RES3]], [[RES0]]
-; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[RES]]
 ;
   %res1 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> %a0, i32 2, <8 x i16> zeroinitializer, i8 -1)
@@ -7947,40 +8018,37 @@ define <8 x i16> @test_x86_vcvtps2ph_256(<8 x float> %a0, i8 %mask, <8 x i16> %s
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i256 [[TMP4]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
-; CHECK:       [[BB5]]:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = select <8 x i1> splat (i1 true), <8 x i16> [[TMP6]], <8 x i16> zeroinitializer
 ; CHECK-NEXT:    [[RES1:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> [[A0]], i32 2, <8 x i16> zeroinitializer, i8 -1)
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP1:%.*]] = icmp ne i256 [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <8 x i1> [[TMP10]] to <8 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = select <8 x i1> [[TMP9]], <8 x i16> [[TMP11]], <8 x i16> zeroinitializer
 ; CHECK-NEXT:    [[_MSCMP2:%.*]] = icmp ne i8 [[TMP2]], 0
-; CHECK-NEXT:    [[_MSOR:%.*]] = or i1 [[_MSCMP1]], [[_MSCMP2]]
-; CHECK-NEXT:    br i1 [[_MSOR]], label %[[BB8:.*]], label %[[BB9:.*]], !prof [[PROF1]]
-; CHECK:       [[BB8]]:
+; CHECK-NEXT:    br i1 [[_MSCMP2]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB9]]:
+; CHECK:       [[BB7]]:
 ; CHECK-NEXT:    [[RES2:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> [[A0]], i32 11, <8 x i16> zeroinitializer, i8 [[MASK]])
-; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <8 x i32> [[TMP1]] to i256
-; CHECK-NEXT:    [[_MSCMP3:%.*]] = icmp ne i256 [[TMP10]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <8 x i16> [[TMP3]] to i128
-; CHECK-NEXT:    [[_MSCMP4:%.*]] = icmp ne i128 [[TMP11]], 0
-; CHECK-NEXT:    [[_MSOR5:%.*]] = or i1 [[_MSCMP3]], [[_MSCMP4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast i8 [[MASK]] to <8 x i1>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <8 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = sext <8 x i1> [[TMP17]] to <8 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = select <8 x i1> [[TMP16]], <8 x i16> [[TMP18]], <8 x i16> [[TMP3]]
 ; CHECK-NEXT:    [[_MSCMP6:%.*]] = icmp ne i8 [[TMP2]], 0
-; CHECK-NEXT:    [[_MSOR7:%.*]] = or i1 [[_MSOR5]], [[_MSCMP6]]
-; CHECK-NEXT:    br i1 [[_MSOR7]], label %[[BB12:.*]], label %[[BB13:.*]], !prof [[PROF1]]
-; CHECK:       [[BB12]]:
+; CHECK-NEXT:    br i1 [[_MSCMP6]], label %[[BB9:.*]], label %[[BB10:.*]], !prof [[PROF1]]
+; CHECK:       [[BB9]]:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
-; CHECK:       [[BB13]]:
+; CHECK:       [[BB10]]:
 ; CHECK-NEXT:    [[RES3:%.*]] = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> [[A0]], i32 12, <8 x i16> [[SRC]], i8 [[MASK]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i16> [[TMP7]], [[TMP12]]
 ; CHECK-NEXT:    [[RES0:%.*]] = add <8 x i16> [[RES1]], [[RES2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i16> [[TMP19]], [[_MSPROP]]
 ; CHECK-NEXT:    [[RES:%.*]] = add <8 x i16> [[RES3]], [[RES0]]
-; CHECK-NEXT:    store <8 x i16> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <8 x i16> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i16> [[RES]]
 ;
   %res1 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> %a0, i32 2, <8 x i16> zeroinitializer, i8 -1)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl_vnni-intrinsics-upgrade.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl_vnni-intrinsics-upgrade.ll
new file mode 100644
index 0000000000000..822e546c84bca
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl_vnni-intrinsics-upgrade.ll
@@ -0,0 +1,1055 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avx512vnni,+avx512vl -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics-upgrade.ll
+;
+; Handled strictly: (none)
+;
+; Handled heuristically: (none)
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpdpbusd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i8> [[TMP17]] to <16 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <16 x i16> [[TMP20]] to i256
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i256 [[TMP21]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP4]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
+define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpbusd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP10]], align 32
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <8 x i32> [[_MSLD]] to <32 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <32 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <32 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <32 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <32 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <32 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <32 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <32 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <32 x i1> [[TMP61]] to <32 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <32 x i8> [[TMP62]] to <16 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <16 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <16 x i1> [[TMP64]] to <16 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <16 x i16> [[TMP65]] to i256
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i256 [[TMP66]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP17]], <8 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[TMP11]], <8 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i32> [[X4]] to <32 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <8 x i32> [[TMP5]] to <32 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <32 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <32 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <32 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <32 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <32 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <32 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <32 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <32 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <32 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <32 x i1> [[TMP51]] to <32 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <32 x i8> [[TMP52]] to <16 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <16 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <16 x i1> [[TMP54]] to <16 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <16 x i16> [[TMP55]] to i256
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i256 [[TMP56]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[_MSPROP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <8 x i1> [[TMP20]], <8 x i32> [[TMP25]], <8 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i32>, <8 x i32> } { <8 x i32> splat (i32 -1), <8 x i32> splat (i32 -1) }, <8 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[TMP27]], <8 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[RES2]], <8 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <8 x i32>, <8 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i32>, <8 x i32> } [[RES3]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %res0 = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> %res0, 0
+  %res3 = insertvalue { <8 x i32>, <8 x i32> }  %res2, <8 x i32> %res1, 1
+  ret { <8 x i32>, <8 x i32> } %res3
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpdpbusd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i8> [[TMP17]] to <8 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <8 x i16> [[TMP20]] to i128
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i128 [[TMP21]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
+define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpbusd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <4 x i32> [[_MSLD]] to <16 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <16 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <16 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <16 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <16 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <16 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <16 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <16 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <16 x i1> [[TMP61]] to <16 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <16 x i8> [[TMP62]] to <8 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <8 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <8 x i1> [[TMP64]] to <8 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <8 x i16> [[TMP65]] to i128
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i128 [[TMP66]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP2]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP3]], <4 x i32> [[TMP17]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP11]], <4 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <4 x i32> [[X4]] to <16 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <4 x i32> [[TMP5]] to <16 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <16 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <16 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <16 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <16 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <16 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <16 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <16 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <16 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <16 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <16 x i1> [[TMP51]] to <16 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <16 x i8> [[TMP52]] to <8 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <8 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <8 x i1> [[TMP54]] to <8 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <8 x i16> [[TMP55]] to i128
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i128 [[TMP56]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP21]], <8 x i1> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[_MSPROP5]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <4 x i32> [[TMP23]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <4 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT7:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP25]], <4 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[TMP19]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <4 x i32>, <4 x i32> } { <4 x i32> splat (i32 -1), <4 x i32> splat (i32 -1) }, <4 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP27]], <4 x i32> [[_MSPROP_SELECT7]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[RES2]], <4 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <4 x i32>, <4 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <4 x i32>, <4 x i32> } [[RES3]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %res0 = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> %res0, 0
+  %res3 = insertvalue { <4 x i32>, <4 x i32> }  %res2, <4 x i32> %res1, 1
+  ret { <4 x i32>, <4 x i32> } %res3
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpdpbusds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i8> [[TMP17]] to <16 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <16 x i16> [[TMP20]] to i256
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i256 [[TMP21]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP4]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
+define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpbusds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP10]], align 32
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <8 x i32> [[_MSLD]] to <32 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <32 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <32 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <32 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <32 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <32 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <32 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <32 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <32 x i1> [[TMP61]] to <32 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <32 x i8> [[TMP62]] to <16 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <16 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <16 x i1> [[TMP64]] to <16 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <16 x i16> [[TMP65]] to i256
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i256 [[TMP66]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP17]], <8 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[TMP11]], <8 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i32> [[X4]] to <32 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <8 x i32> [[TMP5]] to <32 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <32 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <32 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <32 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <32 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <32 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <32 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <32 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <32 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <32 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <32 x i1> [[TMP51]] to <32 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <32 x i8> [[TMP52]] to <16 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <16 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <16 x i1> [[TMP54]] to <16 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <16 x i16> [[TMP55]] to i256
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i256 [[TMP56]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[_MSPROP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <8 x i1> [[TMP20]], <8 x i32> [[TMP25]], <8 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i32>, <8 x i32> } { <8 x i32> splat (i32 -1), <8 x i32> splat (i32 -1) }, <8 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[TMP27]], <8 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[RES2]], <8 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <8 x i32>, <8 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i32>, <8 x i32> } [[RES3]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %res0 = call <8 x i32> @llvm.x86.avx512.mask.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> %res0, 0
+  %res3 = insertvalue { <8 x i32>, <8 x i32> }  %res2, <8 x i32> %res1, 1
+  ret { <8 x i32>, <8 x i32> } %res3
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpdpbusds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i8> [[TMP17]] to <8 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <8 x i16> [[TMP20]] to i128
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i128 [[TMP21]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
+define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpbusds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <4 x i32> [[_MSLD]] to <16 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <16 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <16 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <16 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <16 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <16 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <16 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <16 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <16 x i1> [[TMP61]] to <16 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <16 x i8> [[TMP62]] to <8 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <8 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <8 x i1> [[TMP64]] to <8 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <8 x i16> [[TMP65]] to i128
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i128 [[TMP66]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP2]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP3]], <4 x i32> [[TMP17]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP11]], <4 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <4 x i32> [[X4]] to <16 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <4 x i32> [[TMP5]] to <16 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <16 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <16 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <16 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <16 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <16 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <16 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <16 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <16 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <16 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <16 x i1> [[TMP51]] to <16 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <16 x i8> [[TMP52]] to <8 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <8 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <8 x i1> [[TMP54]] to <8 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <8 x i16> [[TMP55]] to i128
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i128 [[TMP56]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP21]], <8 x i1> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[_MSPROP5]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <4 x i32> [[TMP23]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <4 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT7:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP25]], <4 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[TMP19]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <4 x i32>, <4 x i32> } { <4 x i32> splat (i32 -1), <4 x i32> splat (i32 -1) }, <4 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP27]], <4 x i32> [[_MSPROP_SELECT7]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[RES2]], <4 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <4 x i32>, <4 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <4 x i32>, <4 x i32> } [[RES3]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %res0 = call <4 x i32> @llvm.x86.avx512.mask.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> %res0, 0
+  %res3 = insertvalue { <4 x i32>, <4 x i32> }  %res2, <4 x i32> %res1, 1
+  ret { <4 x i32>, <4 x i32> } %res3
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpdpwssd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i16> [[TMP17]] to <8 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP4]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
+define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpwssd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP10]], align 32
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[_MSLD]] to <16 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <16 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <16 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <16 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <16 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <16 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <16 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <16 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <16 x i1> [[TMP58]] to <16 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <16 x i16> [[TMP59]] to <8 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <8 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <8 x i1> [[TMP61]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP17]], <8 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[TMP11]], <8 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <8 x i32> [[X4]] to <16 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i32> [[TMP5]] to <16 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <16 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <16 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <16 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <16 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <16 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <16 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <16 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <16 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <16 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <16 x i1> [[TMP49]] to <16 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <16 x i16> [[TMP50]] to <8 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <8 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <8 x i1> [[TMP52]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[_MSPROP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <8 x i1> [[TMP20]], <8 x i32> [[TMP25]], <8 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i32>, <8 x i32> } { <8 x i32> splat (i32 -1), <8 x i32> splat (i32 -1) }, <8 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[TMP27]], <8 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[RES2]], <8 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <8 x i32>, <8 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i32>, <8 x i32> } [[RES3]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %res0 = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> %res0, 0
+  %res3 = insertvalue { <8 x i32>, <8 x i32> }  %res2, <8 x i32> %res1, 1
+  ret { <8 x i32>, <8 x i32> } %res3
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpdpwssd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <8 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <8 x i1> [[TMP16]] to <8 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i16> [[TMP17]] to <4 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <4 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <4 x i1> [[TMP19]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
+define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpwssd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[_MSLD]] to <8 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <8 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <8 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <8 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <8 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <8 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <8 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <8 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <8 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <8 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <8 x i1> [[TMP58]] to <8 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <8 x i16> [[TMP59]] to <4 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <4 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <4 x i1> [[TMP61]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP2]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP3]], <4 x i32> [[TMP17]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP11]], <4 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <4 x i32> [[X4]] to <8 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <4 x i32> [[TMP5]] to <8 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <8 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <8 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <8 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <8 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <8 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <8 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <8 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <8 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <8 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <8 x i1> [[TMP49]] to <8 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <8 x i16> [[TMP50]] to <4 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <4 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <4 x i1> [[TMP52]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP21]], <8 x i1> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[_MSPROP5]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <4 x i32> [[TMP23]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <4 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT7:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP25]], <4 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[TMP19]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <4 x i32>, <4 x i32> } { <4 x i32> splat (i32 -1), <4 x i32> splat (i32 -1) }, <4 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP27]], <4 x i32> [[_MSPROP_SELECT7]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[RES2]], <4 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <4 x i32>, <4 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <4 x i32>, <4 x i32> } [[RES3]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %res0 = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> %res0, 0
+  %res3 = insertvalue { <4 x i32>, <4 x i32> }  %res2, <4 x i32> %res1, 1
+  ret { <4 x i32>, <4 x i32> } %res3
+}
+
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpdpwssds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i16> [[TMP17]] to <8 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP4]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 -1)
+  ret <8 x i32> %res
+}
+
+define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpwssds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP10]], align 32
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[_MSLD]] to <16 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <16 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <16 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <16 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <16 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <16 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <16 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <16 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <16 x i1> [[TMP58]] to <16 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <16 x i16> [[TMP59]] to <8 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <8 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <8 x i1> [[TMP61]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP17]], <8 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[TMP11]], <8 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <8 x i32> [[X4]] to <16 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i32> [[TMP5]] to <16 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <16 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <16 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <16 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <16 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <16 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <16 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <16 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <16 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <16 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <16 x i1> [[TMP49]] to <16 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <16 x i16> [[TMP50]] to <8 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <8 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <8 x i1> [[TMP52]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[_MSPROP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <8 x i1> [[TMP20]], <8 x i32> [[TMP25]], <8 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i32>, <8 x i32> } { <8 x i32> splat (i32 -1), <8 x i32> splat (i32 -1) }, <8 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[TMP27]], <8 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[RES2]], <8 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <8 x i32>, <8 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i32>, <8 x i32> } [[RES3]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %res0 = call <8 x i32> @llvm.x86.avx512.mask.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.maskz.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res2 = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> %res0, 0
+  %res3 = insertvalue { <8 x i32>, <8 x i32> }  %res2, <8 x i32> %res1, 1
+  ret { <8 x i32>, <8 x i32> } %res3
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpdpwssds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <8 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <8 x i1> [[TMP16]] to <8 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i16> [[TMP17]] to <4 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <4 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <4 x i1> [[TMP19]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 -1)
+  ret <4 x i32> %res
+}
+
+define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpwssds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[_MSLD]] to <8 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <8 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <8 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <8 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <8 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <8 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <8 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <8 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <8 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <8 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <8 x i1> [[TMP58]] to <8 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <8 x i16> [[TMP59]] to <4 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <4 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <4 x i1> [[TMP61]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP2]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[_MSPROP2]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP3]], <4 x i32> [[TMP17]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP11]], <4 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <4 x i32> [[X4]] to <8 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <4 x i32> [[TMP5]] to <8 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <8 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <8 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <8 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <8 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <8 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <8 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <8 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <8 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <8 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <8 x i1> [[TMP49]] to <8 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <8 x i16> [[TMP50]] to <4 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <4 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <4 x i1> [[TMP52]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = or <4 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP6:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP21]], <8 x i1> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[_MSPROP5]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <4 x i32> [[TMP23]], [[_MSPROP5]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <4 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT7:%.*]] = select <4 x i1> [[_MSPROP6]], <4 x i32> [[TMP25]], <4 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[TMP19]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <4 x i32>, <4 x i32> } { <4 x i32> splat (i32 -1), <4 x i32> splat (i32 -1) }, <4 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP27]], <4 x i32> [[_MSPROP_SELECT7]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[RES2]], <4 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <4 x i32>, <4 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <4 x i32>, <4 x i32> } [[RES3]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %res0 = call <4 x i32> @llvm.x86.avx512.mask.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.maskz.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res2 = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> %res0, 0
+  %res3 = insertvalue { <4 x i32>, <4 x i32> }  %res2, <4 x i32> %res1, 1
+  ret { <4 x i32>, <4 x i32> } %res3
+}
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl_vnni-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl_vnni-intrinsics.ll
new file mode 100644
index 0000000000000..38f4272ef106f
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vl_vnni-intrinsics.ll
@@ -0,0 +1,1098 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avx512vnni,+avx512vl -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avx512vl_vnni-intrinsics.ll
+;
+; Handled strictly: (none)
+;
+; Handled heuristically: (none)
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx512_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpdpbusd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i8> [[TMP17]] to <16 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <16 x i16> [[TMP20]] to i256
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i256 [[TMP21]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP4]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
+define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpbusd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP10]], align 32
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <8 x i32> [[_MSLD]] to <32 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <32 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <32 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <32 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <32 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <32 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <32 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <32 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <32 x i1> [[TMP61]] to <32 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <32 x i8> [[TMP62]] to <16 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <16 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <16 x i1> [[TMP64]] to <16 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <16 x i16> [[TMP65]] to i256
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i256 [[TMP66]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP17]], <8 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[TMP11]], <8 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i32> [[X4]] to <32 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <8 x i32> [[TMP5]] to <32 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <32 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <32 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <32 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <32 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <32 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <32 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <32 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <32 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <32 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <32 x i1> [[TMP51]] to <32 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <32 x i8> [[TMP52]] to <16 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <16 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <16 x i1> [[TMP54]] to <16 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <16 x i16> [[TMP55]] to i256
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i256 [[TMP56]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[_MSPROP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <8 x i1> [[TMP20]], <8 x i32> [[TMP25]], <8 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i32>, <8 x i32> } { <8 x i32> splat (i32 -1), <8 x i32> splat (i32 -1) }, <8 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[TMP27]], <8 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[RES1]], <8 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <8 x i32>, <8 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i32>, <8 x i32> } [[RES2]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
+  %4 = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res1 = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> %3, 0
+  %res2 = insertvalue { <8 x i32>, <8 x i32> }  %res1, <8 x i32> %6, 1
+  ret { <8 x i32>, <8 x i32> } %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx512_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpdpbusd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i8> [[TMP17]] to <8 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <8 x i16> [[TMP20]] to i128
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i128 [[TMP21]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
+define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpbusd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <4 x i32> [[_MSLD]] to <16 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <16 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <16 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <16 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <16 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <16 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <16 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <16 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <16 x i1> [[TMP61]] to <16 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <16 x i8> [[TMP62]] to <8 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <8 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <8 x i1> [[TMP64]] to <8 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <8 x i16> [[TMP65]] to i128
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i128 [[TMP66]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i32> [[TMP17]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP11]], <4 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <4 x i32> [[X4]] to <16 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <4 x i32> [[TMP5]] to <16 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <16 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <16 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <16 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <16 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <16 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <16 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <16 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <16 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <16 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <16 x i1> [[TMP51]] to <16 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <16 x i8> [[TMP52]] to <8 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <8 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <8 x i1> [[TMP54]] to <8 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <8 x i16> [[TMP55]] to i128
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i128 [[TMP56]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP21]], <8 x i1> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[_MSPROP4]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <4 x i32> [[TMP23]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <4 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT6:%.*]] = select <4 x i1> [[_MSPROP5]], <4 x i32> [[TMP25]], <4 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[TMP19]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <4 x i32>, <4 x i32> } { <4 x i32> splat (i32 -1), <4 x i32> splat (i32 -1) }, <4 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP27]], <4 x i32> [[_MSPROP_SELECT6]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[RES1]], <4 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <4 x i32>, <4 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <4 x i32>, <4 x i32> } [[RES2]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
+  %4 = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res1 = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> %3, 0
+  %res2 = insertvalue { <4 x i32>, <4 x i32> }  %res1, <4 x i32> %6, 1
+  ret { <4 x i32>, <4 x i32> } %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx512_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpdpbusds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i8> [[TMP17]] to <16 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <16 x i16> [[TMP20]] to i256
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i256 [[TMP21]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP4]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
+define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpbusds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP10]], align 32
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <8 x i32> [[_MSLD]] to <32 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <32 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <32 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <32 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <32 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <32 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <32 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <32 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <32 x i1> [[TMP61]] to <32 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <32 x i8> [[TMP62]] to <16 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <16 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <16 x i1> [[TMP64]] to <16 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <16 x i16> [[TMP65]] to i256
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i256 [[TMP66]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP17]], <8 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[TMP11]], <8 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i32> [[X4]] to <32 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <8 x i32> [[TMP5]] to <32 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <32 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <32 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <32 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <32 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <32 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <32 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <32 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <32 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <32 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <32 x i1> [[TMP51]] to <32 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <32 x i8> [[TMP52]] to <16 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <16 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <16 x i1> [[TMP54]] to <16 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <16 x i16> [[TMP55]] to i256
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i256 [[TMP56]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[_MSPROP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <8 x i1> [[TMP20]], <8 x i32> [[TMP25]], <8 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i32>, <8 x i32> } { <8 x i32> splat (i32 -1), <8 x i32> splat (i32 -1) }, <8 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[TMP27]], <8 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[RES1]], <8 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <8 x i32>, <8 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i32>, <8 x i32> } [[RES2]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
+  %4 = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res1 = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> %3, 0
+  %res2 = insertvalue { <8 x i32>, <8 x i32> }  %res1, <8 x i32> %6, 1
+  ret { <8 x i32>, <8 x i32> } %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx512_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpdpbusds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i8> [[TMP17]] to <8 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <8 x i16> [[TMP20]] to i128
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i128 [[TMP21]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
+define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpbusds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <4 x i32> [[_MSLD]] to <16 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <16 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <16 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <16 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <16 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <16 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <16 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <16 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <16 x i1> [[TMP61]] to <16 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <16 x i8> [[TMP62]] to <8 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <8 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <8 x i1> [[TMP64]] to <8 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <8 x i16> [[TMP65]] to i128
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i128 [[TMP66]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i32> [[TMP17]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP11]], <4 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <4 x i32> [[X4]] to <16 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <4 x i32> [[TMP5]] to <16 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <16 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <16 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <16 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <16 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <16 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <16 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <16 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <16 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <16 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <16 x i1> [[TMP51]] to <16 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <16 x i8> [[TMP52]] to <8 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <8 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <8 x i1> [[TMP54]] to <8 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <8 x i16> [[TMP55]] to i128
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i128 [[TMP56]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP21]], <8 x i1> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[_MSPROP4]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <4 x i32> [[TMP23]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <4 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT6:%.*]] = select <4 x i1> [[_MSPROP5]], <4 x i32> [[TMP25]], <4 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[TMP19]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <4 x i32>, <4 x i32> } { <4 x i32> splat (i32 -1), <4 x i32> splat (i32 -1) }, <4 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP27]], <4 x i32> [[_MSPROP_SELECT6]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[RES1]], <4 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <4 x i32>, <4 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <4 x i32>, <4 x i32> } [[RES2]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
+  %4 = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res1 = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> %3, 0
+  %res2 = insertvalue { <4 x i32>, <4 x i32> }  %res1, <4 x i32> %6, 1
+  ret { <4 x i32>, <4 x i32> } %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx512_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpdpwssd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i16> [[TMP17]] to <8 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP4]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
+define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpwssd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP10]], align 32
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[_MSLD]] to <16 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <16 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <16 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <16 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <16 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <16 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <16 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <16 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <16 x i1> [[TMP58]] to <16 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <16 x i16> [[TMP59]] to <8 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <8 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <8 x i1> [[TMP61]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP17]], <8 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[TMP11]], <8 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <8 x i32> [[X4]] to <16 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i32> [[TMP5]] to <16 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <16 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <16 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <16 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <16 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <16 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <16 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <16 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <16 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <16 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <16 x i1> [[TMP49]] to <16 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <16 x i16> [[TMP50]] to <8 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <8 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <8 x i1> [[TMP52]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[_MSPROP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <8 x i1> [[TMP20]], <8 x i32> [[TMP25]], <8 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i32>, <8 x i32> } { <8 x i32> splat (i32 -1), <8 x i32> splat (i32 -1) }, <8 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[TMP27]], <8 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[RES1]], <8 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <8 x i32>, <8 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i32>, <8 x i32> } [[RES2]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
+  %4 = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res1 = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> %3, 0
+  %res2 = insertvalue { <8 x i32>, <8 x i32> }  %res1, <8 x i32> %6, 1
+  ret { <8 x i32>, <8 x i32> } %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx512_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpdpwssd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <8 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <8 x i1> [[TMP16]] to <8 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i16> [[TMP17]] to <4 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <4 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <4 x i1> [[TMP19]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP4]]
+;
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
+define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpwssd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[_MSLD]] to <8 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <8 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <8 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <8 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <8 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <8 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <8 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <8 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <8 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <8 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <8 x i1> [[TMP58]] to <8 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <8 x i16> [[TMP59]] to <4 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <4 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <4 x i1> [[TMP61]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i32> [[TMP17]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP11]], <4 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <4 x i32> [[X4]] to <8 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <4 x i32> [[TMP5]] to <8 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <8 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <8 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <8 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <8 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <8 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <8 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <8 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <8 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <8 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <8 x i1> [[TMP49]] to <8 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <8 x i16> [[TMP50]] to <4 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <4 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <4 x i1> [[TMP52]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP21]], <8 x i1> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[_MSPROP4]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <4 x i32> [[TMP23]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <4 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT6:%.*]] = select <4 x i1> [[_MSPROP5]], <4 x i32> [[TMP25]], <4 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[TMP19]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <4 x i32>, <4 x i32> } { <4 x i32> splat (i32 -1), <4 x i32> splat (i32 -1) }, <4 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP27]], <4 x i32> [[_MSPROP_SELECT6]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[RES1]], <4 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <4 x i32>, <4 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <4 x i32>, <4 x i32> } [[RES2]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
+  %4 = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res1 = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> %3, 0
+  %res2 = insertvalue { <4 x i32>, <4 x i32> }  %res1, <4 x i32> %6, 1
+  ret { <4 x i32>, <4 x i32> } %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx512_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx512_vpdpwssds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i16> [[TMP17]] to <8 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[TMP4]]
+;
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %1
+}
+
+define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <8 x i32>, <8 x i32> } @test_int_x86_avx512_mask_vpdpwssds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 104) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP10]], align 32
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[_MSLD]] to <16 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <16 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <16 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <16 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <16 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <16 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <16 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <16 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <16 x i1> [[TMP58]] to <16 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <16 x i16> [[TMP59]] to <8 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <8 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <8 x i1> [[TMP61]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[_MSPROP1]], <8 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <8 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP17]], <8 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP13]], <8 x i32> [[TMP11]], <8 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <8 x i32> [[X4]] to <16 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <8 x i32> [[TMP5]] to <16 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <16 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <16 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <16 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <16 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <16 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <16 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <16 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <16 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <16 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <16 x i1> [[TMP49]] to <16 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <16 x i16> [[TMP50]] to <8 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <8 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <8 x i1> [[TMP52]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[_MSPROP3]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <8 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <8 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <8 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <8 x i1> [[TMP20]], <8 x i32> [[TMP25]], <8 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <8 x i1> [[TMP21]], <8 x i32> [[TMP19]], <8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <8 x i32>, <8 x i32> } { <8 x i32> splat (i32 -1), <8 x i32> splat (i32 -1) }, <8 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[TMP27]], <8 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <8 x i32>, <8 x i32> } [[RES1]], <8 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <8 x i32>, <8 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <8 x i32>, <8 x i32> } [[RES2]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
+  %4 = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %res1 = insertvalue { <8 x i32>, <8 x i32> } poison, <8 x i32> %3, 0
+  %res2 = insertvalue { <8 x i32>, <8 x i32> }  %res1, <8 x i32> %6, 1
+  ret { <8 x i32>, <8 x i32> } %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx512_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx512_vpdpwssds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB4:.*]], label %[[BB5:.*]], !prof [[PROF1]]
+; CHECK:       [[BB4]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i64 [[TMP6]], 87960930222080
+; CHECK-NEXT:    [[TMP8:%.*]] = inttoptr i64 [[TMP7]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP8]], align 16
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[_MSLD]] to <8 x i16>
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <8 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <8 x i16> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <8 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <8 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = and <8 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = and <8 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <8 x i1> [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP20:%.*]] = or <8 x i1> [[TMP17]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <8 x i1> [[TMP20]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = sext <8 x i1> [[TMP21]] to <8 x i16>
+; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i16> [[TMP22]] to <4 x i32>
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp ne <4 x i32> [[TMP23]], zeroinitializer
+; CHECK-NEXT:    [[TMP25:%.*]] = sext <4 x i1> [[TMP24]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP25]], [[TMP4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[TMP9]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %1
+}
+
+define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4, i8 %x3) sanitize_memory {
+; CHECK-LABEL: define { <4 x i32>, <4 x i32> } @test_int_x86_avx512_mask_vpdpwssds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]], i8 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 56) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[_MSLD]] to <8 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <8 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <8 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <8 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <8 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <8 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <8 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <8 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <8 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <8 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <8 x i1> [[TMP58]] to <8 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <8 x i16> [[TMP59]] to <4 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <4 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <4 x i1> [[TMP61]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = shufflevector <8 x i1> [[TMP12]], <8 x i1> [[TMP12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP13]], <8 x i1> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[_MSPROP1]], <4 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <4 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <4 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <4 x i1> [[_MSPROP2]], <4 x i32> [[TMP17]], <4 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i32> [[TMP11]], <4 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <4 x i32> [[X4]] to <8 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <4 x i32> [[TMP5]] to <8 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <8 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <8 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <8 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <8 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <8 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <8 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <8 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <8 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <8 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <8 x i1> [[TMP49]] to <8 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <8 x i16> [[TMP50]] to <4 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <4 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <4 x i1> [[TMP52]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i8 [[TMP4]] to <8 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i8 [[X3]] to <8 x i1>
+; CHECK-NEXT:    [[_MSPROP5:%.*]] = shufflevector <8 x i1> [[TMP20]], <8 x i1> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[EXTRACT1:%.*]] = shufflevector <8 x i1> [[TMP21]], <8 x i1> [[TMP21]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[_MSPROP4]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <4 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <4 x i32> [[TMP23]], [[_MSPROP4]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <4 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT6:%.*]] = select <4 x i1> [[_MSPROP5]], <4 x i32> [[TMP25]], <4 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <4 x i1> [[EXTRACT1]], <4 x i32> [[TMP19]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <4 x i32>, <4 x i32> } { <4 x i32> splat (i32 -1), <4 x i32> splat (i32 -1) }, <4 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[TMP27]], <4 x i32> [[_MSPROP_SELECT6]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <4 x i32>, <4 x i32> } [[RES1]], <4 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <4 x i32>, <4 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <4 x i32>, <4 x i32> } [[RES2]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
+  %4 = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract1, <4 x i32> %4, <4 x i32> zeroinitializer
+  %res1 = insertvalue { <4 x i32>, <4 x i32> } poison, <4 x i32> %3, 0
+  %res2 = insertvalue { <4 x i32>, <4 x i32> }  %res1, <4 x i32> %6, 1
+  ret { <4 x i32>, <4 x i32> } %res2
+}
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vnni-intrinsics-upgrade.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vnni-intrinsics-upgrade.ll
new file mode 100644
index 0000000000000..f146823b90e03
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vnni-intrinsics-upgrade.ll
@@ -0,0 +1,527 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avx512vnni -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avx512vnni-intrinsics-upgrade.ll
+;
+; Handled strictly: (none)
+;
+; Handled heuristically: (none)
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_vpdpbusd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_vpdpbusd_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[X2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <64 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <64 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <64 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <64 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <64 x i1> [[TMP16]] to <64 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i8> [[TMP17]] to <32 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <32 x i1> [[TMP19]] to <32 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[TMP20]] to i512
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i512 [[TMP21]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4, i16 %x3) sanitize_memory {
+; CHECK-LABEL: define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpbusd_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <16 x i32> [[X4:%.*]], i16 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <16 x i32> [[X2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <16 x i32> [[_MSLD]] to <64 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <64 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <64 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <64 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <64 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <64 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <64 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <64 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <64 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <64 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <64 x i1> [[TMP61]] to <64 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <64 x i8> [[TMP62]] to <32 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <32 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <32 x i1> [[TMP64]] to <32 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <32 x i16> [[TMP65]] to i512
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i512 [[TMP66]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i32> [[X4]] to <64 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <16 x i32> [[TMP5]] to <64 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <64 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <64 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <64 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <64 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <64 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <64 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <64 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <64 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <64 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <64 x i1> [[TMP51]] to <64 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <64 x i8> [[TMP52]] to <32 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <32 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <32 x i1> [[TMP54]] to <32 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <32 x i16> [[TMP55]] to i512
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i512 [[TMP56]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[_MSPROP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <16 x i1> [[TMP20]], <16 x i32> [[TMP25]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[RES2]], <16 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32> } [[RES3]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res3 = insertvalue { <16 x i32>, <16 x i32> }  %res2, <16 x i32> %res1, 1
+  ret { <16 x i32>, <16 x i32> } %res3
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_vpdpbusds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_vpdpbusds_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[X2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <64 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <64 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <64 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <64 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <64 x i1> [[TMP16]] to <64 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i8> [[TMP17]] to <32 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <32 x i1> [[TMP19]] to <32 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[TMP20]] to i512
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i512 [[TMP21]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4, i16 %x3) sanitize_memory {
+; CHECK-LABEL: define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpbusds_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <16 x i32> [[X4:%.*]], i16 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <16 x i32> [[X2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <16 x i32> [[_MSLD]] to <64 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <64 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <64 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <64 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <64 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <64 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <64 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <64 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <64 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <64 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <64 x i1> [[TMP61]] to <64 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <64 x i8> [[TMP62]] to <32 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <32 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <32 x i1> [[TMP64]] to <32 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <32 x i16> [[TMP65]] to i512
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i512 [[TMP66]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i32> [[X4]] to <64 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <16 x i32> [[TMP5]] to <64 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <64 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <64 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <64 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <64 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <64 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <64 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <64 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <64 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <64 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <64 x i1> [[TMP51]] to <64 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <64 x i8> [[TMP52]] to <32 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <32 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <32 x i1> [[TMP54]] to <32 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <32 x i16> [[TMP55]] to i512
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i512 [[TMP56]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[_MSPROP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <16 x i1> [[TMP20]], <16 x i32> [[TMP25]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[RES2]], <16 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32> } [[RES3]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res3 = insertvalue { <16 x i32>, <16 x i32> }  %res2, <16 x i32> %res1, 1
+  ret { <16 x i32>, <16 x i32> } %res3
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_vpdpwssd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_vpdpwssd_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i16> [[TMP17]] to <16 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4, i16 %x3) sanitize_memory {
+; CHECK-LABEL: define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpwssd_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <16 x i32> [[X4:%.*]], i16 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <16 x i32> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <16 x i32> [[_MSLD]] to <32 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <32 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <32 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <32 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <32 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <32 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <32 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <32 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <32 x i1> [[TMP58]] to <32 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <32 x i16> [[TMP59]] to <16 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <16 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <16 x i1> [[TMP61]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <16 x i32> [[X4]] to <32 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i32> [[TMP5]] to <32 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <32 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <32 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <32 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <32 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <32 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <32 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <32 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <32 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <32 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <32 x i1> [[TMP49]] to <32 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <32 x i16> [[TMP50]] to <16 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <16 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <16 x i1> [[TMP52]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[_MSPROP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <16 x i1> [[TMP20]], <16 x i32> [[TMP25]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[RES2]], <16 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32> } [[RES3]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res3 = insertvalue { <16 x i32>, <16 x i32> }  %res2, <16 x i32> %res1, 1
+  ret { <16 x i32>, <16 x i32> } %res3
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_vpdpwssds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_vpdpwssds_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i16> [[TMP17]] to <16 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 -1)
+  ret <16 x i32> %res
+}
+
+define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4, i16 %x3) sanitize_memory {
+; CHECK-LABEL: define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpwssds_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <16 x i32> [[X4:%.*]], i16 [[X3:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <16 x i32> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <16 x i32> [[_MSLD]] to <32 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <32 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <32 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <32 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <32 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <32 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <32 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <32 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <32 x i1> [[TMP58]] to <32 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <32 x i16> [[TMP59]] to <16 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <16 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <16 x i1> [[TMP61]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <16 x i32> [[X4]] to <32 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i32> [[TMP5]] to <32 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <32 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <32 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <32 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <32 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <32 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <32 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <32 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <32 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <32 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <32 x i1> [[TMP49]] to <32 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <32 x i16> [[TMP50]] to <16 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <16 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <16 x i1> [[TMP52]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[_MSPROP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <16 x i1> [[TMP20]], <16 x i32> [[TMP25]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES3:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[RES2]], <16 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32> } [[RES3]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %res0 = call <16 x i32> @llvm.x86.avx512.mask.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res2 = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> %res0, 0
+  %res3 = insertvalue { <16 x i32>, <16 x i32> }  %res2, <16 x i32> %res1, 1
+  ret { <16 x i32>, <16 x i32> } %res3
+}
+
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vnni-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vnni-intrinsics.ll
new file mode 100644
index 0000000000000..7c39ff6bb2be1
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx512vnni-intrinsics.ll
@@ -0,0 +1,538 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avx512vnni -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avx512vnni-intrinsics.ll
+;
+; Handled strictly: (none)
+;
+; Handled heuristically: (none)
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+define <16 x i32> @test_int_x86_avx512_ask_vpdpbusd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_ask_vpdpbusd_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[X2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <64 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <64 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <64 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <64 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <64 x i1> [[TMP16]] to <64 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i8> [[TMP17]] to <32 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <32 x i1> [[TMP19]] to <32 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[TMP20]] to i512
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i512 [[TMP21]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
+define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpbusd_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4, i16 %x3) sanitize_memory {
+; CHECK-LABEL: define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpbusd_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <16 x i32> [[X4:%.*]], i16 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <16 x i32> [[X2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <16 x i32> [[_MSLD]] to <64 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <64 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <64 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <64 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <64 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <64 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <64 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <64 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <64 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <64 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <64 x i1> [[TMP61]] to <64 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <64 x i8> [[TMP62]] to <32 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <32 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <32 x i1> [[TMP64]] to <32 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <32 x i16> [[TMP65]] to i512
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i512 [[TMP66]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i32> [[X4]] to <64 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <16 x i32> [[TMP5]] to <64 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <64 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <64 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <64 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <64 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <64 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <64 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <64 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <64 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <64 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <64 x i1> [[TMP51]] to <64 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <64 x i8> [[TMP52]] to <32 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <32 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <32 x i1> [[TMP54]] to <32 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <32 x i16> [[TMP55]] to i512
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i512 [[TMP56]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[_MSPROP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <16 x i1> [[TMP20]], <16 x i32> [[TMP25]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[RES1]], <16 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32> } [[RES2]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
+  %4 = call <16 x i32> @llvm.x86.avx512.vpdpbusd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res1 = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> %3, 0
+  %res2 = insertvalue { <16 x i32>, <16 x i32> }  %res1, <16 x i32> %6, 1
+  ret { <16 x i32>, <16 x i32> } %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+define <16 x i32>@test_int_x86_avx512_vpdpbusds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_vpdpbusds_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[X2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <64 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <64 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <64 x i8> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <64 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <64 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <64 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <64 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <64 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <64 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <64 x i1> [[TMP16]] to <64 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <64 x i8> [[TMP17]] to <32 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <32 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <32 x i1> [[TMP19]] to <32 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <32 x i16> [[TMP20]] to i512
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i512 [[TMP21]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
+define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpbusds_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4, i16 %x3) sanitize_memory {
+; CHECK-LABEL: define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpbusds_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <16 x i32> [[X4:%.*]], i16 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <16 x i32> [[X2]] to <64 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <16 x i32> [[_MSLD]] to <64 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <64 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <64 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <64 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <64 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = and <64 x i1> [[TMP34]], [[TMP35]]
+; CHECK-NEXT:    [[TMP58:%.*]] = and <64 x i1> [[TMP36]], [[TMP35]]
+; CHECK-NEXT:    [[TMP59:%.*]] = and <64 x i1> [[TMP34]], [[TMP37]]
+; CHECK-NEXT:    [[TMP60:%.*]] = or <64 x i1> [[TMP38]], [[TMP58]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or <64 x i1> [[TMP60]], [[TMP59]]
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <64 x i1> [[TMP61]] to <64 x i8>
+; CHECK-NEXT:    [[TMP63:%.*]] = bitcast <64 x i8> [[TMP62]] to <32 x i16>
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp ne <32 x i16> [[TMP63]], zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = sext <32 x i1> [[TMP64]] to <32 x i16>
+; CHECK-NEXT:    [[TMP66:%.*]] = bitcast <32 x i16> [[TMP65]] to i512
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast i512 [[TMP66]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP29]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i32> [[X1]] to <64 x i8>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i32> [[X4]] to <64 x i8>
+; CHECK-NEXT:    [[TMP41:%.*]] = bitcast <16 x i32> [[TMP3]] to <64 x i8>
+; CHECK-NEXT:    [[TMP42:%.*]] = bitcast <16 x i32> [[TMP5]] to <64 x i8>
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <64 x i8> [[TMP41]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <64 x i8> [[TMP42]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp ne <64 x i8> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <64 x i8> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = and <64 x i1> [[TMP43]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = and <64 x i1> [[TMP45]], [[TMP44]]
+; CHECK-NEXT:    [[TMP49:%.*]] = and <64 x i1> [[TMP43]], [[TMP46]]
+; CHECK-NEXT:    [[TMP50:%.*]] = or <64 x i1> [[TMP47]], [[TMP48]]
+; CHECK-NEXT:    [[TMP51:%.*]] = or <64 x i1> [[TMP50]], [[TMP49]]
+; CHECK-NEXT:    [[TMP52:%.*]] = sext <64 x i1> [[TMP51]] to <64 x i8>
+; CHECK-NEXT:    [[TMP53:%.*]] = bitcast <64 x i8> [[TMP52]] to <32 x i16>
+; CHECK-NEXT:    [[TMP54:%.*]] = icmp ne <32 x i16> [[TMP53]], zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = sext <32 x i1> [[TMP54]] to <32 x i16>
+; CHECK-NEXT:    [[TMP56:%.*]] = bitcast <32 x i16> [[TMP55]] to i512
+; CHECK-NEXT:    [[TMP57:%.*]] = bitcast i512 [[TMP56]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i32> [[TMP57]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[_MSPROP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <16 x i1> [[TMP20]], <16 x i32> [[TMP25]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[RES1]], <16 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32> } [[RES2]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
+  %4 = call <16 x i32> @llvm.x86.avx512.vpdpbusds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res1 = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> %3, 0
+  %res2 = insertvalue { <16 x i32>, <16 x i32> }  %res1, <16 x i32> %6, 1
+  ret { <16 x i32>, <16 x i32> } %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+define <16 x i32>@test_int_x86_avx512_vpdpwssd_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_vpdpwssd_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i16> [[TMP17]] to <16 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
+define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpwssd_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4, i16 %x3) sanitize_memory {
+; CHECK-LABEL: define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpwssd_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <16 x i32> [[X4:%.*]], i16 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <16 x i32> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <16 x i32> [[_MSLD]] to <32 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <32 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <32 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <32 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <32 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <32 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <32 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <32 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <32 x i1> [[TMP58]] to <32 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <32 x i16> [[TMP59]] to <16 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <16 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <16 x i1> [[TMP61]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <16 x i32> [[X4]] to <32 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i32> [[TMP5]] to <32 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <32 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <32 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <32 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <32 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <32 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <32 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <32 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <32 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <32 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <32 x i1> [[TMP49]] to <32 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <32 x i16> [[TMP50]] to <16 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <16 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <16 x i1> [[TMP52]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[_MSPROP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <16 x i1> [[TMP20]], <16 x i32> [[TMP25]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[RES1]], <16 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32> } [[RES2]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
+  %4 = call <16 x i32> @llvm.x86.avx512.vpdpwssd.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res1 = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> %3, 0
+  %res2 = insertvalue { <16 x i32>, <16 x i32> }  %res1, <16 x i32> %6, 1
+  ret { <16 x i32>, <16 x i32> } %res2
+}
+
+declare <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32>, <16 x i32>, <16 x i32>)
+
+define <16 x i32>@test_int_x86_avx512_ask_vpdpwssds_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <16 x i32> @test_int_x86_avx512_ask_vpdpwssds_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], <16 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <16 x i32> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <16 x i32> [[TMP2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i16> [[TMP17]] to <16 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    store <16 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <16 x i32> [[TMP4]]
+;
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  ret <16 x i32> %1
+}
+
+define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpwssds_512(<16 x i32> %x0, <16 x i32> %x1, ptr %x2p, <16 x i32> %x4, i16 %x3) sanitize_memory {
+; CHECK-LABEL: define { <16 x i32>, <16 x i32> } @test_int_x86_avx512_mask_vpdpwssds_512(
+; CHECK-SAME: <16 x i32> [[X0:%.*]], <16 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <16 x i32> [[X4:%.*]], i16 [[X3:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 128) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 200) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 136) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB6:.*]], label %[[BB7:.*]], !prof [[PROF1]]
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB7]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <16 x i32>, ptr [[X2P]], align 64
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i64 [[TMP8]], 87960930222080
+; CHECK-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <16 x i32>, ptr [[TMP10]], align 64
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <16 x i32> [[X2]] to <32 x i16>
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <16 x i32> [[_MSLD]] to <32 x i16>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <32 x i16> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <32 x i16> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i16> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i16> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP54:%.*]] = and <32 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP55:%.*]] = and <32 x i1> [[TMP35]], [[TMP34]]
+; CHECK-NEXT:    [[TMP56:%.*]] = and <32 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP57:%.*]] = or <32 x i1> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or <32 x i1> [[TMP57]], [[TMP56]]
+; CHECK-NEXT:    [[TMP59:%.*]] = sext <32 x i1> [[TMP58]] to <32 x i16>
+; CHECK-NEXT:    [[TMP60:%.*]] = bitcast <32 x i16> [[TMP59]] to <16 x i32>
+; CHECK-NEXT:    [[TMP61:%.*]] = icmp ne <16 x i32> [[TMP60]], zeroinitializer
+; CHECK-NEXT:    [[TMP62:%.*]] = sext <16 x i1> [[TMP61]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <16 x i32> [[TMP62]], [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP14:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[_MSPROP1]], <16 x i32> [[TMP2]]
+; CHECK-NEXT:    [[TMP15:%.*]] = xor <16 x i32> [[TMP11]], [[X0]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i32> [[TMP15]], [[_MSPROP1]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <16 x i32> [[TMP16]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP_SELECT:%.*]] = select <16 x i1> [[TMP12]], <16 x i32> [[TMP17]], <16 x i32> [[TMP14]]
+; CHECK-NEXT:    [[TMP18:%.*]] = select <16 x i1> [[TMP13]], <16 x i32> [[TMP11]], <16 x i32> [[X0]]
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <16 x i32> [[X1]] to <32 x i16>
+; CHECK-NEXT:    [[TMP38:%.*]] = bitcast <16 x i32> [[X4]] to <32 x i16>
+; CHECK-NEXT:    [[TMP39:%.*]] = bitcast <16 x i32> [[TMP3]] to <32 x i16>
+; CHECK-NEXT:    [[TMP40:%.*]] = bitcast <16 x i32> [[TMP5]] to <32 x i16>
+; CHECK-NEXT:    [[TMP41:%.*]] = icmp ne <32 x i16> [[TMP39]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp ne <32 x i16> [[TMP40]], zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = icmp ne <32 x i16> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp ne <32 x i16> [[TMP38]], zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = and <32 x i1> [[TMP41]], [[TMP42]]
+; CHECK-NEXT:    [[TMP46:%.*]] = and <32 x i1> [[TMP43]], [[TMP42]]
+; CHECK-NEXT:    [[TMP47:%.*]] = and <32 x i1> [[TMP41]], [[TMP44]]
+; CHECK-NEXT:    [[TMP48:%.*]] = or <32 x i1> [[TMP45]], [[TMP46]]
+; CHECK-NEXT:    [[TMP49:%.*]] = or <32 x i1> [[TMP48]], [[TMP47]]
+; CHECK-NEXT:    [[TMP50:%.*]] = sext <32 x i1> [[TMP49]] to <32 x i16>
+; CHECK-NEXT:    [[TMP51:%.*]] = bitcast <32 x i16> [[TMP50]] to <16 x i32>
+; CHECK-NEXT:    [[TMP52:%.*]] = icmp ne <16 x i32> [[TMP51]], zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = sext <16 x i1> [[TMP52]] to <16 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <16 x i32> [[TMP53]], [[TMP2]]
+; CHECK-NEXT:    [[TMP19:%.*]] = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> [[X0]], <16 x i32> [[X1]], <16 x i32> [[X4]])
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i16 [[TMP4]] to <16 x i1>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast i16 [[X3]] to <16 x i1>
+; CHECK-NEXT:    [[TMP22:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[_MSPROP3]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = xor <16 x i32> [[TMP19]], zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <16 x i32> [[TMP23]], [[_MSPROP3]]
+; CHECK-NEXT:    [[TMP25:%.*]] = or <16 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[_MSPROP_SELECT4:%.*]] = select <16 x i1> [[TMP20]], <16 x i32> [[TMP25]], <16 x i32> [[TMP22]]
+; CHECK-NEXT:    [[TMP26:%.*]] = select <16 x i1> [[TMP21]], <16 x i32> [[TMP19]], <16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = insertvalue { <16 x i32>, <16 x i32> } { <16 x i32> splat (i32 -1), <16 x i32> splat (i32 -1) }, <16 x i32> [[_MSPROP_SELECT]], 0
+; CHECK-NEXT:    [[RES1:%.*]] = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> [[TMP18]], 0
+; CHECK-NEXT:    [[TMP28:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[TMP27]], <16 x i32> [[_MSPROP_SELECT4]], 1
+; CHECK-NEXT:    [[RES2:%.*]] = insertvalue { <16 x i32>, <16 x i32> } [[RES1]], <16 x i32> [[TMP26]], 1
+; CHECK-NEXT:    store { <16 x i32>, <16 x i32> } [[TMP28]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { <16 x i32>, <16 x i32> } [[RES2]]
+;
+  %x2 = load <16 x i32>, ptr %x2p
+  %1 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
+  %4 = call <16 x i32> @llvm.x86.avx512.vpdpwssds.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %res1 = insertvalue { <16 x i32>, <16 x i32> } poison, <16 x i32> %3, 0
+  %res2 = insertvalue { <16 x i32>, <16 x i32> }  %res1, <16 x i32> %6, 1
+  ret { <16 x i32>, <16 x i32> } %res2
+}
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avx_vnni-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avx_vnni-intrinsics.ll
new file mode 100644
index 0000000000000..678faef203324
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avx_vnni-intrinsics.ll
@@ -0,0 +1,299 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avx512vnni,+avx512vl,+avxvnni -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avx_vnni-intrinsics.ll
+;
+; Handled strictly: (none)
+;
+; Handled heuristically: (none)
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx_vpdpbusd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx_vpdpbusd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i8> [[TMP17]] to <16 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <16 x i16> [[TMP20]] to i256
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i256 [[TMP21]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.vpdpbusd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx_vpdpbusd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx_vpdpbusd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i8> [[TMP17]] to <8 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <8 x i16> [[TMP20]] to i128
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i128 [[TMP21]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.vpdpbusd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx_vpdpbusds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx_vpdpbusds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <32 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <32 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <32 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <32 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <32 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <32 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <32 x i1> [[TMP16]] to <32 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <32 x i8> [[TMP17]] to <16 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <16 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <16 x i16> [[TMP20]] to i256
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i256 [[TMP21]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.vpdpbusds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx_vpdpbusds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx_vpdpbusds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i8> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i8> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i8> [[TMP17]] to <8 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <8 x i16> [[TMP20]] to i128
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast i128 [[TMP21]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.vpdpbusds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx_vpdpwssd_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx_vpdpwssd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i16> [[TMP17]] to <8 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.vpdpwssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx_vpdpwssd_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx_vpdpwssd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <8 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <8 x i1> [[TMP16]] to <8 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i16> [[TMP17]] to <4 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <4 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <4 x i1> [[TMP19]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.vpdpwssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx_vpdpwssds_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx_vpdpwssds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], <8 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i32> [[X1]] to <16 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i32> [[X2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i32> [[TMP2]] to <16 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <8 x i32> [[TMP3]] to <16 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <16 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <16 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <16 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <16 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <16 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <16 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <16 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <16 x i1> [[TMP16]] to <16 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <16 x i16> [[TMP17]] to <8 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <8 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %res = call <8 x i32> @llvm.x86.avx512.vpdpwssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx_vpdpwssds_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx_vpdpwssds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], <4 x i32> [[X2:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i32> [[X1]] to <8 x i16>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[X2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i32> [[TMP2]] to <8 x i16>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i32> [[TMP3]] to <8 x i16>
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne <8 x i16> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <8 x i16> [[TMP7]], zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <8 x i16> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ne <8 x i16> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP10]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP8]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP12]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = sext <8 x i1> [[TMP16]] to <8 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i16> [[TMP17]] to <4 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <4 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = sext <4 x i1> [[TMP19]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP20]], [[TMP21]]
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %res = call <4 x i32> @llvm.x86.avx512.vpdpwssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  ret <4 x i32> %res
+}
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avxvnniint16-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avxvnniint16-intrinsics.ll
new file mode 100644
index 0000000000000..66cbebee80dc3
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avxvnniint16-intrinsics.ll
@@ -0,0 +1,239 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avxvnniint16 -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avxvnniint16-intrinsics.ll
+;
+; Handled strictly: (none)
+;
+; Handled heuristically:
+; - llvm.x86.avx2.vpdpwsud.128
+; - llvm.x86.avx2.vpdpwsud.256
+; - llvm.x86.avx2.vpdpwsuds.128
+; - llvm.x86.avx2.vpdpwsuds.256
+; - llvm.x86.avx2.vpdpwusd.128
+; - llvm.x86.avx2.vpdpwusd.256
+; - llvm.x86.avx2.vpdpwusds.128
+; - llvm.x86.avx2.vpdpwusds.256
+; - llvm.x86.avx2.vpdpwuud.128
+; - llvm.x86.avx2.vpdpwuud.256
+; - llvm.x86.avx2.vpdpwuuds.128
+; - llvm.x86.avx2.vpdpwuuds.256
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define <4 x i32> @test_int_x86_avx2_vpdpwsud_128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpwsud_128(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpwsud.128(<4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> [[C]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RET]]
+;
+  %ret = call <4 x i32> @llvm.x86.avx2.vpdpwsud.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+  ret <4 x i32> %ret
+}
+declare <4 x i32> @llvm.x86.avx2.vpdpwsud.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+
+define <8 x i32> @test_int_x86_avx2_vpdpwsud_256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpwsud_256(
+; CHECK-SAME: <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpwsud.256(<8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> [[C]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RET]]
+;
+  %ret = call <8 x i32> @llvm.x86.avx2.vpdpwsud.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+  ret <8 x i32> %ret
+}
+declare <8 x i32> @llvm.x86.avx2.vpdpwsud.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+
+define <4 x i32> @test_int_x86_avx2_vpdpwsuds_128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpwsuds_128(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpwsuds.128(<4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> [[C]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RET]]
+;
+  %ret = call <4 x i32> @llvm.x86.avx2.vpdpwsuds.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+  ret <4 x i32> %ret
+}
+declare <4 x i32> @llvm.x86.avx2.vpdpwsuds.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+
+define <8 x i32> @test_int_x86_avx2_vpdpwsuds_256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpwsuds_256(
+; CHECK-SAME: <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpwsuds.256(<8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> [[C]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RET]]
+;
+  %ret = call <8 x i32> @llvm.x86.avx2.vpdpwsuds.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+  ret <8 x i32> %ret
+}
+declare <8 x i32> @llvm.x86.avx2.vpdpwsuds.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+
+define <4 x i32> @test_int_x86_avx2_vpdpwusd_128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpwusd_128(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpwusd.128(<4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> [[C]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RET]]
+;
+  %ret = call <4 x i32> @llvm.x86.avx2.vpdpwusd.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+  ret <4 x i32> %ret
+}
+declare <4 x i32> @llvm.x86.avx2.vpdpwusd.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+
+define <8 x i32> @test_int_x86_avx2_vpdpwusd_256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpwusd_256(
+; CHECK-SAME: <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpwusd.256(<8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> [[C]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RET]]
+;
+  %ret = call <8 x i32> @llvm.x86.avx2.vpdpwusd.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+  ret <8 x i32> %ret
+}
+declare <8 x i32> @llvm.x86.avx2.vpdpwusd.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+
+define <4 x i32> @test_int_x86_avx2_vpdpwusds_128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpwusds_128(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpwusds.128(<4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> [[C]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RET]]
+;
+  %ret = call <4 x i32> @llvm.x86.avx2.vpdpwusds.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+  ret <4 x i32> %ret
+}
+declare <4 x i32> @llvm.x86.avx2.vpdpwusds.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+
+define <8 x i32> @test_int_x86_avx2_vpdpwusds_256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpwusds_256(
+; CHECK-SAME: <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpwusds.256(<8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> [[C]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RET]]
+;
+  %ret = call <8 x i32> @llvm.x86.avx2.vpdpwusds.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+  ret <8 x i32> %ret
+}
+declare <8 x i32> @llvm.x86.avx2.vpdpwusds.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+
+define <4 x i32> @test_int_x86_avx2_vpdpwuud_128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpwuud_128(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpwuud.128(<4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> [[C]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RET]]
+;
+  %ret = call <4 x i32> @llvm.x86.avx2.vpdpwuud.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+  ret <4 x i32> %ret
+}
+declare <4 x i32> @llvm.x86.avx2.vpdpwuud.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+
+define <8 x i32> @test_int_x86_avx2_vpdpwuud_256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpwuud_256(
+; CHECK-SAME: <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpwuud.256(<8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> [[C]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RET]]
+;
+  %ret = call <8 x i32> @llvm.x86.avx2.vpdpwuud.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+  ret <8 x i32> %ret
+}
+declare <8 x i32> @llvm.x86.avx2.vpdpwuud.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+
+define <4 x i32> @test_int_x86_avx2_vpdpwuuds_128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpwuuds_128(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpwuuds.128(<4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> [[C]])
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RET]]
+;
+  %ret = call <4 x i32> @llvm.x86.avx2.vpdpwuuds.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+  ret <4 x i32> %ret
+}
+declare <4 x i32> @llvm.x86.avx2.vpdpwuuds.128(<4 x i32> %A, <4 x i32> %B, <4 x i32> %C)
+
+define <8 x i32> @test_int_x86_avx2_vpdpwuuds_256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpwuuds_256(
+; CHECK-SAME: <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[TMP3]]
+; CHECK-NEXT:    [[RET:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpwuuds.256(<8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> [[C]])
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP1]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RET]]
+;
+  %ret = call <8 x i32> @llvm.x86.avx2.vpdpwuuds.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
+  ret <8 x i32> %ret
+}
+declare <8 x i32> @llvm.x86.avx2.vpdpwuuds.256(<8 x i32> %A, <8 x i32> %B, <8 x i32> %C)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/avxvnniint8-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/avxvnniint8-intrinsics.ll
new file mode 100644
index 0000000000000..b36d09bfb5944
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/avxvnniint8-intrinsics.ll
@@ -0,0 +1,626 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mattr=+avxvnniint8 -passes=msan -S | FileCheck %s
+
+; Forked from llvm/test/CodeGen/X86/avxvnniint8-intrinsics.ll
+;
+; Handled strictly: (none)
+;
+; Handled heuristically: (none)
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <4 x i32> @llvm.x86.avx2.vpdpbssd.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx2_vpdpbssd_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpbssd_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP9]], align 16
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> [[_MSLD]] to <16 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <16 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <16 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <16 x i8> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <16 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = and <16 x i1> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <16 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <16 x i1> [[TMP14]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i1> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sext <16 x i1> [[TMP22]] to <16 x i8>
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i8> [[TMP23]] to <8 x i16>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <8 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = sext <8 x i1> [[TMP25]] to <8 x i16>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <8 x i16> [[TMP26]] to i128
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i128 [[TMP27]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP28]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbssd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[X4]] to <16 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast <4 x i32> [[TMP4]] to <16 x i8>
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i8> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <16 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp ne <16 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = and <16 x i1> [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP40:%.*]] = and <16 x i1> [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[TMP41:%.*]] = and <16 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP42:%.*]] = or <16 x i1> [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP43:%.*]] = or <16 x i1> [[TMP42]], [[TMP41]]
+; CHECK-NEXT:    [[TMP44:%.*]] = sext <16 x i1> [[TMP43]] to <16 x i8>
+; CHECK-NEXT:    [[TMP45:%.*]] = bitcast <16 x i8> [[TMP44]] to <8 x i16>
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <8 x i16> [[TMP45]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = sext <8 x i1> [[TMP46]] to <8 x i16>
+; CHECK-NEXT:    [[TMP48:%.*]] = bitcast <8 x i16> [[TMP47]] to i128
+; CHECK-NEXT:    [[TMP49:%.*]] = bitcast i128 [[TMP48]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP49]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbssd.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <4 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx2.vpdpbssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = call <4 x i32> @llvm.x86.avx2.vpdpbssd.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %res = add <4 x i32> %1, %2
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpbssds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx2_vpdpbssds_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpbssds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP9]], align 16
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i32> [[X2]] to <16 x i8>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <4 x i32> [[_MSLD]] to <16 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <16 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <16 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <16 x i8> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <16 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = and <16 x i1> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <16 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <16 x i1> [[TMP14]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <16 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <16 x i1> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sext <16 x i1> [[TMP22]] to <16 x i8>
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <16 x i8> [[TMP23]] to <8 x i16>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <8 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = sext <8 x i1> [[TMP25]] to <8 x i16>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <8 x i16> [[TMP26]] to i128
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i128 [[TMP27]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[TMP28]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbssds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <4 x i32> [[X1]] to <16 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <4 x i32> [[X4]] to <16 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <4 x i32> [[TMP3]] to <16 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast <4 x i32> [[TMP4]] to <16 x i8>
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <16 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <16 x i8> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <16 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp ne <16 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = and <16 x i1> [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP40:%.*]] = and <16 x i1> [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[TMP41:%.*]] = and <16 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP42:%.*]] = or <16 x i1> [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP43:%.*]] = or <16 x i1> [[TMP42]], [[TMP41]]
+; CHECK-NEXT:    [[TMP44:%.*]] = sext <16 x i1> [[TMP43]] to <16 x i8>
+; CHECK-NEXT:    [[TMP45:%.*]] = bitcast <16 x i8> [[TMP44]] to <8 x i16>
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <8 x i16> [[TMP45]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = sext <8 x i1> [[TMP46]] to <8 x i16>
+; CHECK-NEXT:    [[TMP48:%.*]] = bitcast <8 x i16> [[TMP47]] to i128
+; CHECK-NEXT:    [[TMP49:%.*]] = bitcast i128 [[TMP48]] to <4 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[TMP49]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbssds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <4 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx2.vpdpbssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = call <4 x i32> @llvm.x86.avx2.vpdpbssds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %res = add <4 x i32> %1, %2
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx2.vpdpbssd.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx2_vpdpbssd_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpbssd_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP9]], align 32
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i32> [[_MSLD]] to <32 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <32 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <32 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <32 x i8> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <32 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = and <32 x i1> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <32 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <32 x i1> [[TMP14]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <32 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i1> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sext <32 x i1> [[TMP22]] to <32 x i8>
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <32 x i8> [[TMP23]] to <16 x i16>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <16 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = sext <16 x i1> [[TMP25]] to <16 x i16>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <16 x i16> [[TMP26]] to i256
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i256 [[TMP27]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP28]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbssd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[X4]] to <32 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast <8 x i32> [[TMP4]] to <32 x i8>
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i8> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <32 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp ne <32 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = and <32 x i1> [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP40:%.*]] = and <32 x i1> [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[TMP41:%.*]] = and <32 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP42:%.*]] = or <32 x i1> [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP43:%.*]] = or <32 x i1> [[TMP42]], [[TMP41]]
+; CHECK-NEXT:    [[TMP44:%.*]] = sext <32 x i1> [[TMP43]] to <32 x i8>
+; CHECK-NEXT:    [[TMP45:%.*]] = bitcast <32 x i8> [[TMP44]] to <16 x i16>
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <16 x i16> [[TMP45]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = sext <16 x i1> [[TMP46]] to <16 x i16>
+; CHECK-NEXT:    [[TMP48:%.*]] = bitcast <16 x i16> [[TMP47]] to i256
+; CHECK-NEXT:    [[TMP49:%.*]] = bitcast i256 [[TMP48]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP49]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbssd.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx2.vpdpbssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = call <8 x i32> @llvm.x86.avx2.vpdpbssd.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %res = add <8 x i32> %1, %2
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx2.vpdpbssds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx2_vpdpbssds_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpbssds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP9]], align 32
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <8 x i32> [[X2]] to <32 x i8>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <8 x i32> [[_MSLD]] to <32 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <32 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <32 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <32 x i8> [[TMP29]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <32 x i8> [[TMP30]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = and <32 x i1> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = and <32 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <32 x i1> [[TMP14]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = or <32 x i1> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i1> [[TMP21]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sext <32 x i1> [[TMP22]] to <32 x i8>
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <32 x i8> [[TMP23]] to <16 x i16>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <16 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = sext <16 x i1> [[TMP25]] to <16 x i16>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <16 x i16> [[TMP26]] to i256
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast i256 [[TMP27]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[TMP28]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbssds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <8 x i32> [[X1]] to <32 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast <8 x i32> [[X4]] to <32 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = bitcast <8 x i32> [[TMP3]] to <32 x i8>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast <8 x i32> [[TMP4]] to <32 x i8>
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <32 x i8> [[TMP33]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <32 x i8> [[TMP34]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <32 x i8> [[TMP31]], zeroinitializer
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp ne <32 x i8> [[TMP32]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = and <32 x i1> [[TMP35]], [[TMP36]]
+; CHECK-NEXT:    [[TMP40:%.*]] = and <32 x i1> [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[TMP41:%.*]] = and <32 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP42:%.*]] = or <32 x i1> [[TMP39]], [[TMP40]]
+; CHECK-NEXT:    [[TMP43:%.*]] = or <32 x i1> [[TMP42]], [[TMP41]]
+; CHECK-NEXT:    [[TMP44:%.*]] = sext <32 x i1> [[TMP43]] to <32 x i8>
+; CHECK-NEXT:    [[TMP45:%.*]] = bitcast <32 x i8> [[TMP44]] to <16 x i16>
+; CHECK-NEXT:    [[TMP46:%.*]] = icmp ne <16 x i16> [[TMP45]], zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = sext <16 x i1> [[TMP46]] to <16 x i16>
+; CHECK-NEXT:    [[TMP48:%.*]] = bitcast <16 x i16> [[TMP47]] to i256
+; CHECK-NEXT:    [[TMP49:%.*]] = bitcast i256 [[TMP48]] to <8 x i32>
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[TMP49]], [[TMP5]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbssds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx2.vpdpbssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = call <8 x i32> @llvm.x86.avx2.vpdpbssds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %res = add <8 x i32> %1, %2
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpbsud.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx2_vpdpbsud_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpbsud_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP9]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbsud.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbsud.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <4 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx2.vpdpbsud.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = call <4 x i32> @llvm.x86.avx2.vpdpbsud.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %res = add <4 x i32> %1, %2
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpbsuds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx2_vpdpbsuds_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpbsuds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP9]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbsuds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbsuds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <4 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx2.vpdpbsuds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = call <4 x i32> @llvm.x86.avx2.vpdpbsuds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %res = add <4 x i32> %1, %2
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx2.vpdpbsud.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx2_vpdpbsud_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpbsud_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP9]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbsud.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[_MSPROP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbsud.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx2.vpdpbsud.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = call <8 x i32> @llvm.x86.avx2.vpdpbsud.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %res = add <8 x i32> %1, %2
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx2.vpdpbsuds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx2_vpdpbsuds_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpbsuds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP9]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbsuds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[_MSPROP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbsuds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx2.vpdpbsuds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = call <8 x i32> @llvm.x86.avx2.vpdpbsuds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %res = add <8 x i32> %1, %2
+  ret <8 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpbuud.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx2_vpdpbuud_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpbuud_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP9]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbuud.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbuud.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <4 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx2.vpdpbuud.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = call <4 x i32> @llvm.x86.avx2.vpdpbuud.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %res = add <4 x i32> %1, %2
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx2.vpdpbuuds.128(<4 x i32>, <4 x i32>, <4 x i32>)
+
+define <4 x i32>@test_int_x86_avx2_vpdpbuuds_128(<4 x i32> %x0, <4 x i32> %x1, ptr %x2p, <4 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <4 x i32> @test_int_x86_avx2_vpdpbuuds_128(
+; CHECK-SAME: <4 x i32> [[X0:%.*]], <4 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <4 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 40) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <4 x i32>, ptr [[X2P]], align 16
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <4 x i32>, ptr [[TMP9]], align 16
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <4 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbuuds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <4 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <4 x i32> [[_MSPROP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.x86.avx2.vpdpbuuds.128(<4 x i32> [[X0]], <4 x i32> [[X1]], <4 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <4 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <4 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <4 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <4 x i32> [[RES]]
+;
+  %x2 = load <4 x i32>, ptr %x2p
+  %1 = call <4 x i32> @llvm.x86.avx2.vpdpbuuds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = call <4 x i32> @llvm.x86.avx2.vpdpbuuds.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %res = add <4 x i32> %1, %2
+  ret <4 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx2.vpdpbuud.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx2_vpdpbuud_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpbuud_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP9]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbuud.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[_MSPROP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbuud.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx2.vpdpbuud.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = call <8 x i32> @llvm.x86.avx2.vpdpbuud.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %res = add <8 x i32> %1, %2
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx2.vpdpbuuds.256(<8 x i32>, <8 x i32>, <8 x i32>)
+
+define <8 x i32>@test_int_x86_avx2_vpdpbuuds_256(<8 x i32> %x0, <8 x i32> %x1, ptr %x2p, <8 x i32> %x4) sanitize_memory {
+; CHECK-LABEL: define <8 x i32> @test_int_x86_avx2_vpdpbuuds_256(
+; CHECK-SAME: <8 x i32> [[X0:%.*]], <8 x i32> [[X1:%.*]], ptr [[X2P:%.*]], <8 x i32> [[X4:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 64) to ptr), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, ptr @__msan_param_tls, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i32>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 72) to ptr), align 8
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[_MSCMP]], label %[[BB5:.*]], label %[[BB6:.*]], !prof [[PROF1]]
+; CHECK:       [[BB5]]:
+; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR4]]
+; CHECK-NEXT:    unreachable
+; CHECK:       [[BB6]]:
+; CHECK-NEXT:    [[X2:%.*]] = load <8 x i32>, ptr [[X2P]], align 32
+; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[X2P]] to i64
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i64 [[TMP7]], 87960930222080
+; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
+; CHECK-NEXT:    [[_MSLD:%.*]] = load <8 x i32>, ptr [[TMP9]], align 32
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP1:%.*]] = or <8 x i32> [[_MSPROP]], [[_MSLD]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbuuds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X2]])
+; CHECK-NEXT:    [[_MSPROP2:%.*]] = or <8 x i32> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[_MSPROP3:%.*]] = or <8 x i32> [[_MSPROP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call <8 x i32> @llvm.x86.avx2.vpdpbuuds.256(<8 x i32> [[X0]], <8 x i32> [[X1]], <8 x i32> [[X4]])
+; CHECK-NEXT:    [[_MSPROP4:%.*]] = or <8 x i32> [[_MSPROP1]], [[_MSPROP3]]
+; CHECK-NEXT:    [[RES:%.*]] = add <8 x i32> [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    store <8 x i32> [[_MSPROP4]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret <8 x i32> [[RES]]
+;
+  %x2 = load <8 x i32>, ptr %x2p
+  %1 = call <8 x i32> @llvm.x86.avx2.vpdpbuuds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = call <8 x i32> @llvm.x86.avx2.vpdpbuuds.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %res = add <8 x i32> %1, %2
+  ret <8 x i32> %res
+}
+;.
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 1, i32 1048575}
+;.
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/mmx-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/mmx-intrinsics.ll
index ac3bb56719038..3d98f60a8242a 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/mmx-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/mmx-intrinsics.ll
@@ -4,8 +4,6 @@
 ; Handled strictly:
 ; - i32 @llvm.x86.mmx.pmovmskb(<1 x i64> %mmx_var.i) #2
 ; - void @llvm.x86.mmx.maskmovq(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i, ptr %p) #2
-; - <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> %4, i8 3) #5
-; - <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> %4, i8 3) #5
 ; - <2 x double> @llvm.x86.sse.cvtpi2pd(<1 x i64> %4) #5
 ; - <1 x i64> @llvm.x86.sse.cvttpd2pi(<2 x double> %a) #5
 ; - <1 x i64> @llvm.x86.sse.cvtpd2pi(<2 x double> %a) #5
@@ -1687,16 +1685,30 @@ define i64 @test49(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
 ; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[TMP8:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <2 x i32> [[TMP9]], zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = sext <2 x i1> [[TMP10]] to <2 x i32>
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
-; CHECK-NEXT:    [[TMP14:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP12]] to <2 x i32>
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <1 x i64> [[MMX_VAR_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[MMX_VAR1_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP6]] to <4 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp ne <4 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ne <4 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp ne <4 x i16> [[TMP8]], zeroinitializer
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp ne <4 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = and <4 x i1> [[TMP29]], [[TMP30]]
+; CHECK-NEXT:    [[TMP35:%.*]] = and <4 x i1> [[TMP22]], [[TMP30]]
+; CHECK-NEXT:    [[TMP36:%.*]] = and <4 x i1> [[TMP29]], [[TMP32]]
+; CHECK-NEXT:    [[TMP37:%.*]] = or <4 x i1> [[TMP31]], [[TMP35]]
+; CHECK-NEXT:    [[TMP38:%.*]] = or <4 x i1> [[TMP37]], [[TMP36]]
+; CHECK-NEXT:    [[TMP23:%.*]] = sext <4 x i1> [[TMP38]] to <4 x i16>
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <4 x i16> [[TMP23]] to <2 x i32>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <2 x i32> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP27:%.*]] = sext <2 x i1> [[TMP25]] to <2 x i32>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <2 x i32> [[TMP27]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64 [[TMP28]] to <1 x i64>
+; CHECK-NEXT:    [[TMP33:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <1 x i64> [[TMP14]] to <2 x i32>
-; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP3]] to <1 x i64>
-; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <2 x i32> [[TMP20]] to <1 x i64>
+; CHECK-NEXT:    [[TMP34:%.*]] = bitcast <1 x i64> [[TMP33]] to <2 x i32>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP20]] to <1 x i64>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <2 x i32> [[TMP34]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP17]], i32 0
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <1 x i64> [[TMP21]], i32 0
 ; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
@@ -2778,19 +2790,17 @@ define i64 @test21(<1 x i64> %a) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP10]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
-; CHECK:       6:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[TMP13:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP10]], i8 3)
+; CHECK-NEXT:    [[TMP13:%.*]] = or <1 x i64> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP13]] to <4 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP6]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP3]] to <1 x i64>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP12]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
-; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <1 x i64> [[TMP14]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP15]]
 ;
 entry:
   %0 = bitcast <1 x i64> %a to <4 x i16>
@@ -2812,19 +2822,17 @@ define i32 @test21_2(<1 x i64> %a) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP10]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP6:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
-; CHECK:       6:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       7:
-; CHECK-NEXT:    [[TMP13:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP9:%.*]] = call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP10]], i8 3)
+; CHECK-NEXT:    [[TMP13:%.*]] = or <1 x i64> zeroinitializer, [[TMP9]]
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP13]] to <4 x i16>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP6]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP3]] to <2 x i32>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP12]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
-; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i32> [[TMP14]], i32 0
+; CHECK-NEXT:    store i32 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP15]]
 ;
 entry:
   %0 = bitcast <1 x i64> %a to <4 x i16>
@@ -3235,7 +3243,8 @@ define i64 @test9(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[TMP12]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP20:%.*]] = call <1 x i64> @llvm.x86.ssse3.pshuf.b(<1 x i64> [[TMP16]], <1 x i64> [[TMP17]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP8]], [[TMP20]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pshuf.b(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <8 x i8>
@@ -3315,16 +3324,30 @@ define i64 @test7(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[TMP17]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[TMP10:%.*]] = or <1 x i64> [[TMP21]], [[TMP8]]
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i16> [[TMP11]], zeroinitializer
-; CHECK-NEXT:    [[TMP13:%.*]] = sext <4 x i1> [[TMP12]] to <4 x i16>
-; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
-; CHECK-NEXT:    [[TMP24:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64> [[TMP22]], <1 x i64> [[TMP23]]) #[[ATTR5]]
-; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP14]] to <8 x i8>
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP24]] to <8 x i8>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
-; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i8> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[TMP22]] to <8 x i8>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP23]] to <8 x i8>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP21]] to <8 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp ne <8 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <8 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp ne <8 x i8> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp ne <8 x i8> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = and <8 x i1> [[TMP32]], [[TMP33]]
+; CHECK-NEXT:    [[TMP38:%.*]] = and <8 x i1> [[TMP35]], [[TMP33]]
+; CHECK-NEXT:    [[TMP39:%.*]] = and <8 x i1> [[TMP32]], [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or <8 x i1> [[TMP34]], [[TMP38]]
+; CHECK-NEXT:    [[TMP41:%.*]] = or <8 x i1> [[TMP40]], [[TMP39]]
+; CHECK-NEXT:    [[TMP16:%.*]] = sext <8 x i1> [[TMP41]] to <8 x i8>
+; CHECK-NEXT:    [[TMP26:%.*]] = bitcast <8 x i8> [[TMP16]] to <4 x i16>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <4 x i16> [[TMP26]], zeroinitializer
+; CHECK-NEXT:    [[TMP29:%.*]] = sext <4 x i1> [[TMP25]] to <4 x i16>
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <4 x i16> [[TMP29]] to i64
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast i64 [[TMP24]] to <1 x i64>
+; CHECK-NEXT:    [[TMP36:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64> [[TMP22]], <1 x i64> [[TMP23]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP31:%.*]] = bitcast <1 x i64> [[TMP30]] to <8 x i8>
+; CHECK-NEXT:    [[TMP28:%.*]] = bitcast <1 x i64> [[TMP36]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP31]] to <1 x i64>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i8> [[TMP28]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
 ; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <1 x i64> [[TMP19]], i32 0
 ; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/sse2-intrinsics-x86.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/sse2-intrinsics-x86.ll
index 8f915a59db8e5..7048050180792 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/sse2-intrinsics-x86.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/sse2-intrinsics-x86.ll
@@ -762,11 +762,20 @@ define <4 x i32> @test_x86_sse2_pmadd_wd(<8 x i16> %a0, <8 x i16> %a1) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to <4 x i32>
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i32> [[TMP4]], zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i32>
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <8 x i16> [[A0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ne <8 x i16> [[A1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = and <8 x i1> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP12]], [[TMP5]]
+; CHECK-NEXT:    [[TMP15:%.*]] = and <8 x i1> [[TMP4]], [[TMP13]]
+; CHECK-NEXT:    [[TMP16:%.*]] = or <8 x i1> [[TMP11]], [[TMP14]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or <8 x i1> [[TMP16]], [[TMP15]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sext <8 x i1> [[TMP17]] to <8 x i16>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i16> [[TMP7]] to <4 x i32>
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <4 x i32> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <4 x i1> [[TMP19]] to <4 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> [[A0]], <8 x i16> [[A1]])
 ; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[RES]]
 ;
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_call.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_call.ll
index 7a3f0dd88f9c1..b61cb6aebb3ea 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_call.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_call.ll
@@ -29,7 +29,7 @@ entry:
 define dso_local i32 @sum(i32 %n, ...) local_unnamed_addr #0 {
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #2
   call void @llvm.va_start(ptr nonnull %args)
   %cmp9 = icmp sgt i32 %n, 0
   br i1 %cmp9, label %for.body.lr.ph, label %for.end
@@ -85,13 +85,13 @@ vaarg.end:                                        ; preds = %vaarg.in_mem, %vaar
 for.end:                                          ; preds = %vaarg.end, %entry
   %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %vaarg.end ]
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #2
   ret i32 %sum.0.lcssa
 }
 
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind
 declare void @llvm.va_start(ptr) #2
@@ -100,7 +100,7 @@ declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 declare dso_local i80 @sum_i80(i32, ...) local_unnamed_addr
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_shadow.ll b/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_shadow.ll
index 205101564dfe0..4bc14daaca427 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_shadow.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/X86/vararg_shadow.ll
@@ -551,7 +551,7 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef signext %t, i32
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -581,26 +581,26 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef signext %t, i32
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
 
 declare void @llvm.va_start(ptr) #5
 
 declare void @llvm.va_end(ptr) #5
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
 
 define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef %n, ...) sanitize_memory {
 ; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2IiEvT_iz(
@@ -614,7 +614,7 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -644,16 +644,16 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -669,7 +669,7 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -699,16 +699,16 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -724,7 +724,7 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -754,16 +754,16 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -779,7 +779,7 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(x86_fp80 noundef %t, i32 no
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -809,16 +809,16 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(x86_fp80 noundef %t, i32 no
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -834,7 +834,7 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -864,16 +864,16 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -889,7 +889,7 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz(i64 %t.coerce0,
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -919,16 +919,16 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz(i64 %t.coerce0,
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -944,7 +944,7 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz(double %t.coer
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -974,16 +974,16 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz(double %t.coer
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -999,7 +999,7 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz(ptr noundef byval(%s
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1029,16 +1029,16 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz(ptr noundef byval(%s
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -1054,7 +1054,7 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz(double %t.coerc
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1084,16 +1084,16 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz(double %t.coerc
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -1109,7 +1109,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1139,16 +1139,16 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -1164,7 +1164,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP4]], 87960930222080
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1194,16 +1194,16 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/alloca.ll b/llvm/test/Instrumentation/MemorySanitizer/alloca.ll
index 40ade5f40f9af..c05702b0669d3 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/alloca.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/alloca.ll
@@ -125,12 +125,12 @@ entry:
   br label %another_bb
 
 another_bb:
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   store i32 7, ptr %x
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %x)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
   store i32 8, ptr %x
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -158,8 +158,10 @@ another_bb:
 define void @lifetime_start_var(i64 %cnt) sanitize_memory {
 entry:
   %x = alloca i32, i64 %cnt, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
+  call void @llvm.lifetime.start.p0(ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   ret void
 }
 
@@ -176,5 +178,5 @@ entry:
 ; CHECK: call void @llvm.lifetime.end
 ; CHECK: ret void
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
index 5cc56baf0e0de..cd79bcb2233fe 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
@@ -149,11 +149,20 @@ define <8 x i32> @test_x86_avx2_pmadd_wd(<16 x i16> %a0, <16 x i16> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i16> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <16 x i16> [[TMP3]] to <8 x i32>
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i32> [[TMP4]], zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP5]] to <8 x i32>
-; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> [[A0:%.*]], <16 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <16 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <16 x i16> [[A0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <16 x i16> [[A1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <16 x i1> [[TMP5]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = and <16 x i1> [[TMP14]], [[TMP12]]
+; CHECK-NEXT:    [[TMP17:%.*]] = and <16 x i1> [[TMP5]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <16 x i1> [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <16 x i1> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sext <16 x i1> [[TMP19]] to <16 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <16 x i16> [[TMP8]] to <8 x i32>
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne <8 x i32> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <8 x i1> [[TMP21]] to <8 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> [[A0]], <16 x i16> [[A1]])
 ; CHECK-NEXT:    store <8 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <8 x i32> [[RES]]
 ;
@@ -714,11 +723,20 @@ define <16 x i16> @test_x86_avx2_pmadd_ub_sw(<32 x i8> %a0, <32 x i8> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <32 x i8> [[TMP3]] to <16 x i16>
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <16 x i16> [[TMP4]], zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP5]] to <16 x i16>
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <32 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <32 x i8> [[A0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <32 x i8> [[A1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <32 x i1> [[TMP5]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = and <32 x i1> [[TMP14]], [[TMP12]]
+; CHECK-NEXT:    [[TMP17:%.*]] = and <32 x i1> [[TMP5]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <32 x i1> [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <32 x i1> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sext <32 x i1> [[TMP19]] to <32 x i8>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <32 x i8> [[TMP8]] to <16 x i16>
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne <16 x i16> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <16 x i1> [[TMP21]] to <16 x i16>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0]], <32 x i8> [[A1]])
 ; CHECK-NEXT:    store <16 x i16> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[RES]]
 ;
@@ -734,7 +752,7 @@ define <16 x i16> @test_x86_avx2_pmadd_ub_sw_load_op0(ptr %ptr, <32 x i8> %a1) #
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP1]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP12:%.*]], !prof [[PROF1]]
+; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP4:%.*]], label [[TMP8:%.*]], !prof [[PROF1]]
 ; CHECK:       4:
 ; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
 ; CHECK-NEXT:    unreachable
@@ -744,11 +762,20 @@ define <16 x i16> @test_x86_avx2_pmadd_ub_sw_load_op0(ptr %ptr, <32 x i8> %a1) #
 ; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], -2147483649
 ; CHECK-NEXT:    [[TMP7:%.*]] = inttoptr i64 [[TMP6]] to ptr
 ; CHECK-NEXT:    [[_MSLD:%.*]] = load <32 x i8>, ptr [[TMP7]], align 32
-; CHECK-NEXT:    [[TMP8:%.*]] = or <32 x i8> [[_MSLD]], [[TMP2]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <32 x i8> [[TMP8]] to <16 x i16>
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <16 x i16> [[TMP9]], zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = sext <16 x i1> [[TMP10]] to <16 x i16>
-; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <32 x i8> [[_MSLD]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <32 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp ne <32 x i8> [[A0]], zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ne <32 x i8> [[A1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = and <32 x i1> [[TMP10]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = and <32 x i1> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP21:%.*]] = and <32 x i1> [[TMP10]], [[TMP19]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <32 x i1> [[TMP12]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = or <32 x i1> [[TMP22]], [[TMP21]]
+; CHECK-NEXT:    [[TMP13:%.*]] = sext <32 x i1> [[TMP23]] to <32 x i8>
+; CHECK-NEXT:    [[TMP24:%.*]] = bitcast <32 x i8> [[TMP13]] to <16 x i16>
+; CHECK-NEXT:    [[TMP25:%.*]] = icmp ne <16 x i16> [[TMP24]], zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <16 x i1> [[TMP25]] to <16 x i16>
+; CHECK-NEXT:    [[RES:%.*]] = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> [[A0]], <32 x i8> [[A1]])
 ; CHECK-NEXT:    store <16 x i16> [[TMP11]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <16 x i16> [[RES]]
 ;
@@ -780,8 +807,9 @@ define <32 x i8> @test_x86_avx2_pshuf_b(<32 x i8> %a0, <32 x i8> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i8>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 32) to ptr), align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP4:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[TMP1]], <32 x i8> [[A1:%.*]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <32 x i8> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[RES:%.*]] = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> [[A0:%.*]], <32 x i8> [[A1]])
 ; CHECK-NEXT:    store <32 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <32 x i8> [[RES]]
 ;
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll
index 0a3efaaea149f..8052b5e345265 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll
@@ -4,8 +4,6 @@
 ; Handled strictly:
 ; - i32 @llvm.x86.mmx.pmovmskb(<1 x i64> %mmx_var.i) #2
 ; - void @llvm.x86.mmx.maskmovq(<1 x i64> %mmx_var.i, <1 x i64> %mmx_var1.i, ptr %p) #2
-; - <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> %4, i8 3) #5
-; - <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> %4, i8 3) #5
 ; - <2 x double> @llvm.x86.sse.cvtpi2pd(<1 x i64> %4) #5
 ; - <1 x i64> @llvm.x86.sse.cvttpd2pi(<2 x double> %a) #5
 ; - <1 x i64> @llvm.x86.sse.cvtpd2pi(<2 x double> %a) #5
@@ -1730,16 +1728,30 @@ define i64 @test49(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[MMX_VAR_I:%.*]] = bitcast <4 x i16> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast <4 x i16> [[TMP16]] to <1 x i64>
 ; CHECK-NEXT:    [[MMX_VAR1_I:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[TMP8:%.*]] = or <1 x i64> [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP8]] to <2 x i32>
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp ne <2 x i32> [[TMP9]], zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = sext <2 x i1> [[TMP10]] to <2 x i32>
-; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <2 x i32> [[TMP11]] to <1 x i64>
-; CHECK-NEXT:    [[TMP14:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP12]] to <2 x i32>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[MMX_VAR_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <1 x i64> [[MMX_VAR1_I]] to <4 x i16>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP6]] to <4 x i16>
+; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <1 x i64> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = icmp ne <4 x i16> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp ne <4 x i16> [[TMP22]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp ne <4 x i16> [[TMP9]], zeroinitializer
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <4 x i16> [[TMP10]], zeroinitializer
+; CHECK-NEXT:    [[TMP32:%.*]] = and <4 x i1> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP36:%.*]] = and <4 x i1> [[TMP23]], [[TMP31]]
+; CHECK-NEXT:    [[TMP37:%.*]] = and <4 x i1> [[TMP30]], [[TMP33]]
+; CHECK-NEXT:    [[TMP38:%.*]] = or <4 x i1> [[TMP32]], [[TMP36]]
+; CHECK-NEXT:    [[TMP39:%.*]] = or <4 x i1> [[TMP38]], [[TMP37]]
+; CHECK-NEXT:    [[TMP24:%.*]] = sext <4 x i1> [[TMP39]] to <4 x i16>
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <4 x i16> [[TMP24]] to <2 x i32>
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp ne <2 x i32> [[TMP25]], zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = sext <2 x i1> [[TMP26]] to <2 x i32>
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast <2 x i32> [[TMP28]] to i64
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i64 [[TMP29]] to <1 x i64>
+; CHECK-NEXT:    [[TMP34:%.*]] = tail call <1 x i64> @llvm.x86.mmx.pmadd.wd(<1 x i64> [[MMX_VAR_I]], <1 x i64> [[MMX_VAR1_I]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <1 x i64> [[TMP14]] to <2 x i32>
-; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP3]] to <1 x i64>
-; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <2 x i32> [[TMP20]] to <1 x i64>
+; CHECK-NEXT:    [[TMP35:%.*]] = bitcast <1 x i64> [[TMP34]] to <2 x i32>
+; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <2 x i32> [[TMP20]] to <1 x i64>
+; CHECK-NEXT:    [[TMP21:%.*]] = bitcast <2 x i32> [[TMP35]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP17]], i32 0
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <1 x i64> [[TMP21]], i32 0
 ; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
@@ -2849,19 +2861,17 @@ define i64 @test21(<1 x i64> %a) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP10]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP12:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
-; CHECK:       7:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP13:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP10]], i8 3)
+; CHECK-NEXT:    [[TMP13:%.*]] = or <1 x i64> zeroinitializer, [[TMP6]]
+; CHECK-NEXT:    [[TMP14:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP13]] to <4 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP14]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP3]] to <1 x i64>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i16> [[TMP9]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <1 x i64> [[TMP4]], i32 0
-; CHECK-NEXT:    store i64 0, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <1 x i64> [[TMP15]], i32 0
+; CHECK-NEXT:    store i64 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i64 [[TMP12]]
 ;
 entry:
   %0 = bitcast <1 x i64> %a to <4 x i16>
@@ -2884,19 +2894,17 @@ define i32 @test21_2(<1 x i64> %a) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast <1 x i64> [[A]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP10:%.*]] = bitcast <4 x i16> [[TMP8]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <4 x i16> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP10]] to i64
-; CHECK-NEXT:    [[_MSCMP:%.*]] = icmp ne i64 [[TMP9]], 0
-; CHECK-NEXT:    br i1 [[_MSCMP]], label [[TMP12:%.*]], label [[TMP14:%.*]], !prof [[PROF1]]
-; CHECK:       7:
-; CHECK-NEXT:    call void @__msan_warning_noreturn() #[[ATTR6]]
-; CHECK-NEXT:    unreachable
-; CHECK:       8:
-; CHECK-NEXT:    [[TMP13:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP10]], i8 3)
+; CHECK-NEXT:    [[TMP13:%.*]] = or <1 x i64> zeroinitializer, [[TMP6]]
+; CHECK-NEXT:    [[TMP14:%.*]] = tail call <1 x i64> @llvm.x86.sse.pshuf.w(<1 x i64> [[TMP11]], i8 3) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP13]] to <4 x i16>
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast <1 x i64> [[TMP14]] to <4 x i16>
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <4 x i16> [[TMP3]] to <2 x i32>
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast <4 x i16> [[TMP9]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
-; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
-; CHECK-NEXT:    ret i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[TMP15]], i32 0
+; CHECK-NEXT:    store i32 [[TMP5]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret i32 [[TMP12]]
 ;
 entry:
   %0 = bitcast <1 x i64> %a to <4 x i16>
@@ -3319,7 +3327,8 @@ define i64 @test9(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[TMP12]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP17:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP20:%.*]] = call <1 x i64> @llvm.x86.ssse3.pshuf.b(<1 x i64> [[TMP16]], <1 x i64> [[TMP17]])
+; CHECK-NEXT:    [[_MSPROP:%.*]] = or <1 x i64> [[TMP8]], [[TMP20]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pshuf.b(<1 x i64> [[TMP2]], <1 x i64> [[TMP17]]) #[[ATTR5]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[_MSPROP]] to <8 x i8>
 ; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <1 x i64> [[TMP18]] to <8 x i8>
@@ -3401,16 +3410,30 @@ define i64 @test7(<1 x i64> %a, <1 x i64> %b) #0 {
 ; CHECK-NEXT:    [[TMP22:%.*]] = bitcast <8 x i8> [[TMP1]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast <8 x i8> [[TMP17]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP23:%.*]] = bitcast <8 x i8> [[TMP0]] to <1 x i64>
-; CHECK-NEXT:    [[TMP10:%.*]] = or <1 x i64> [[TMP21]], [[TMP8]]
-; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP10]] to <4 x i16>
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i16> [[TMP11]], zeroinitializer
-; CHECK-NEXT:    [[TMP13:%.*]] = sext <4 x i1> [[TMP12]] to <4 x i16>
-; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <4 x i16> [[TMP13]] to <1 x i64>
-; CHECK-NEXT:    [[TMP24:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64> [[TMP22]], <1 x i64> [[TMP23]]) #[[ATTR5]]
-; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <1 x i64> [[TMP14]] to <8 x i8>
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP24]] to <8 x i8>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP16]] to <1 x i64>
-; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i8> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast <1 x i64> [[TMP22]] to <8 x i8>
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast <1 x i64> [[TMP23]] to <8 x i8>
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast <1 x i64> [[TMP21]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast <1 x i64> [[TMP8]] to <8 x i8>
+; CHECK-NEXT:    [[TMP33:%.*]] = icmp ne <8 x i8> [[TMP13]], zeroinitializer
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp ne <8 x i8> [[TMP14]], zeroinitializer
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp ne <8 x i8> [[TMP11]], zeroinitializer
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp ne <8 x i8> [[TMP12]], zeroinitializer
+; CHECK-NEXT:    [[TMP35:%.*]] = and <8 x i1> [[TMP33]], [[TMP34]]
+; CHECK-NEXT:    [[TMP38:%.*]] = and <8 x i1> [[TMP26]], [[TMP34]]
+; CHECK-NEXT:    [[TMP39:%.*]] = and <8 x i1> [[TMP33]], [[TMP36]]
+; CHECK-NEXT:    [[TMP40:%.*]] = or <8 x i1> [[TMP35]], [[TMP38]]
+; CHECK-NEXT:    [[TMP41:%.*]] = or <8 x i1> [[TMP40]], [[TMP39]]
+; CHECK-NEXT:    [[TMP16:%.*]] = sext <8 x i1> [[TMP41]] to <8 x i8>
+; CHECK-NEXT:    [[TMP27:%.*]] = bitcast <8 x i8> [[TMP16]] to <4 x i16>
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp ne <4 x i16> [[TMP27]], zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = sext <4 x i1> [[TMP28]] to <4 x i16>
+; CHECK-NEXT:    [[TMP30:%.*]] = bitcast <4 x i16> [[TMP42]] to i64
+; CHECK-NEXT:    [[TMP32:%.*]] = bitcast i64 [[TMP30]] to <1 x i64>
+; CHECK-NEXT:    [[TMP31:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64> [[TMP22]], <1 x i64> [[TMP23]]) #[[ATTR5]]
+; CHECK-NEXT:    [[TMP25:%.*]] = bitcast <1 x i64> [[TMP32]] to <8 x i8>
+; CHECK-NEXT:    [[TMP37:%.*]] = bitcast <1 x i64> [[TMP31]] to <8 x i8>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <8 x i8> [[TMP25]] to <1 x i64>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <8 x i8> [[TMP37]] to <1 x i64>
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <1 x i64> [[TMP6]], i32 0
 ; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <1 x i64> [[TMP19]], i32 0
 ; CHECK-NEXT:    store i64 [[TMP7]], ptr @__msan_retval_tls, align 8
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/sse2-intrinsics-i386.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/sse2-intrinsics-i386.ll
index e771e60e2f294..3a37eafd78ecb 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/sse2-intrinsics-i386.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/sse2-intrinsics-i386.ll
@@ -800,11 +800,20 @@ define <4 x i32> @test_x86_sse2_pmadd_wd(<8 x i16> %a0, <8 x i16> %a1) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr @__msan_va_arg_overflow_size_tls, align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP3:%.*]] = or <8 x i16> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <8 x i16> [[TMP3]] to <4 x i32>
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i32> [[TMP4]], zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = sext <4 x i1> [[TMP5]] to <4 x i32>
-; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne <8 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <8 x i16> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <8 x i16> [[A0:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <8 x i16> [[A1:%.*]], zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP5]], [[TMP12]]
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i1> [[TMP14]], [[TMP12]]
+; CHECK-NEXT:    [[TMP17:%.*]] = and <8 x i1> [[TMP5]], [[TMP15]]
+; CHECK-NEXT:    [[TMP18:%.*]] = or <8 x i1> [[TMP13]], [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = or <8 x i1> [[TMP18]], [[TMP17]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sext <8 x i1> [[TMP19]] to <8 x i16>
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast <8 x i16> [[TMP8]] to <4 x i32>
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne <4 x i32> [[TMP20]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sext <4 x i1> [[TMP21]] to <4 x i32>
+; CHECK-NEXT:    [[RES:%.*]] = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> [[A0]], <8 x i16> [[A1]])
 ; CHECK-NEXT:    store <4 x i32> [[TMP6]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[RES]]
 ;
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_call.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_call.ll
index b27ef5ddc93bf..27459397b34a3 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_call.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_call.ll
@@ -93,7 +93,7 @@ define dso_local i32 @sum(i32 %n, ...) local_unnamed_addr #0 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP6]], i8 0, i64 24, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = and i64 [[TMP7]], -2147483649
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
@@ -165,7 +165,7 @@ define dso_local i32 @sum(i32 %n, ...) local_unnamed_addr #0 {
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[VAARG_END]] ]
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ;
@@ -186,7 +186,7 @@ define dso_local i32 @sum(i32 %n, ...) local_unnamed_addr #0 {
 ; ORIGIN-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], -4
 ; ORIGIN-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
 ; ORIGIN-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP7]], i8 0, i64 24, i1 false)
-; ORIGIN-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; ORIGIN-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; ORIGIN-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; ORIGIN-NEXT:    [[TMP11:%.*]] = and i64 [[TMP23]], -2147483649
 ; ORIGIN-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -266,7 +266,7 @@ define dso_local i32 @sum(i32 %n, ...) local_unnamed_addr #0 {
 ; ORIGIN:       [[FOR_END]]:
 ; ORIGIN-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[VAARG_END]] ]
 ; ORIGIN-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; ORIGIN-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; ORIGIN-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; ORIGIN-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
 ; ORIGIN-NEXT:    store i32 0, ptr @__msan_retval_origin_tls, align 4
 ; ORIGIN-NEXT:    ret i32 [[SUM_0_LCSSA]]
@@ -288,7 +288,7 @@ define dso_local i32 @sum(i32 %n, ...) local_unnamed_addr #0 {
 ; ORIGIN2-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], -4
 ; ORIGIN2-NEXT:    [[TMP10:%.*]] = inttoptr i64 [[TMP9]] to ptr
 ; ORIGIN2-NEXT:    call void @llvm.memset.p0.i64(ptr align 16 [[TMP7]], i8 0, i64 24, i1 false)
-; ORIGIN2-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; ORIGIN2-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; ORIGIN2-NEXT:    [[TMP23:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; ORIGIN2-NEXT:    [[TMP11:%.*]] = and i64 [[TMP23]], -2147483649
 ; ORIGIN2-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
@@ -368,14 +368,14 @@ define dso_local i32 @sum(i32 %n, ...) local_unnamed_addr #0 {
 ; ORIGIN2:       [[FOR_END]]:
 ; ORIGIN2-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD]], %[[VAARG_END]] ]
 ; ORIGIN2-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; ORIGIN2-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; ORIGIN2-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; ORIGIN2-NEXT:    store i32 0, ptr @__msan_retval_tls, align 8
 ; ORIGIN2-NEXT:    store i32 0, ptr @__msan_retval_origin_tls, align 4
 ; ORIGIN2-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #2
   call void @llvm.va_start(ptr nonnull %args)
   %cmp9 = icmp sgt i32 %n, 0
   br i1 %cmp9, label %for.body.lr.ph, label %for.end
@@ -419,13 +419,13 @@ vaarg.end:                                        ; preds = %vaarg.in_mem, %vaar
 for.end:                                          ; preds = %vaarg.end, %entry
   %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add, %vaarg.end ]
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #2
   ret i32 %sum.0.lcssa
 }
 
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind
 declare void @llvm.va_start(ptr) #2
@@ -434,7 +434,7 @@ declare void @llvm.va_start(ptr) #2
 declare void @llvm.va_end(ptr) #2
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 declare dso_local i80 @sum_i80(i32, ...) local_unnamed_addr
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_shadow.ll b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_shadow.ll
index aedefca2ef2b6..74a62762fc184 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_shadow.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/i386/vararg_shadow.ll
@@ -562,7 +562,7 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef signext %t, i32
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -582,26 +582,26 @@ define linkonce_odr dso_local void @_Z5test2IcEvT_iz(i8 noundef signext %t, i32
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
 
 declare void @llvm.va_start(ptr) #5
 
 declare void @llvm.va_end(ptr) #5
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
 
 define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef %n, ...) sanitize_memory {
 ; CHECK-LABEL: define linkonce_odr dso_local void @_Z5test2IiEvT_iz(
@@ -614,7 +614,7 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -634,16 +634,16 @@ define linkonce_odr dso_local void @_Z5test2IiEvT_iz(i32 noundef %t, i32 noundef
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -658,7 +658,7 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -678,16 +678,16 @@ define linkonce_odr dso_local void @_Z5test2IfEvT_iz(float noundef %t, i32 nound
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -702,7 +702,7 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -722,16 +722,16 @@ define linkonce_odr dso_local void @_Z5test2IdEvT_iz(double noundef %t, i32 noun
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -746,7 +746,7 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(x86_fp80 noundef %t, i32 no
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -766,16 +766,16 @@ define linkonce_odr dso_local void @_Z5test2IeEvT_iz(x86_fp80 noundef %t, i32 no
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -790,7 +790,7 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -810,16 +810,16 @@ define linkonce_odr dso_local void @_Z5test2I6IntIntEvT_iz(i64 %t.coerce, i32 no
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -834,7 +834,7 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz(i64 %t.coerce0,
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -854,16 +854,16 @@ define linkonce_odr dso_local void @_Z5test2I10Int64Int64EvT_iz(i64 %t.coerce0,
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -878,7 +878,7 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz(double %t.coer
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -898,16 +898,16 @@ define linkonce_odr dso_local void @_Z5test2I12DoubleDoubleEvT_iz(double %t.coer
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -922,7 +922,7 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz(ptr noundef byval(%s
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -942,16 +942,16 @@ define linkonce_odr dso_local void @_Z5test2I7Double4EvT_iz(ptr noundef byval(%s
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -966,7 +966,7 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz(double %t.coerc
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -986,16 +986,16 @@ define linkonce_odr dso_local void @_Z5test2I11DoubleFloatEvT_iz(double %t.coerc
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -1010,7 +1010,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1030,16 +1030,16 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble2EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
@@ -1054,7 +1054,7 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP2]], ptr align 8 @__msan_va_arg_tls, i64 [[TMP3]], i1 false)
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[ARGS:%.*]] = alloca [1 x %struct.__va_list_tag], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[ARGS]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = and i64 [[TMP4]], -2147483649
 ; CHECK-NEXT:    [[TMP6:%.*]] = inttoptr i64 [[TMP5]] to ptr
@@ -1074,16 +1074,16 @@ define linkonce_odr dso_local void @_Z5test2I11LongDouble4EvT_iz(ptr noundef byv
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    call void @_Z3usePv(ptr noundef nonnull [[ARGS]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr nonnull [[ARGS]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr nonnull [[ARGS]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[ARGS]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %args = alloca [1 x %struct.__va_list_tag], align 16
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %args) #6
   call void @llvm.va_start(ptr nonnull %args)
   call void @_Z3usePv(ptr noundef nonnull %args)
   call void @llvm.va_end(ptr nonnull %args)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %args) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %args) #6
   ret void
 }
 
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_debug_info.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_debug_info.ll
index f07f3ad06e607..04fdd23aa5a88 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/msan_debug_info.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_debug_info.ll
@@ -664,8 +664,8 @@ entry:
 
 declare i32 @NoSanitizeMemoryUndefHelper(i32 %x)
 
-declare void @llvm.lifetime.start.p0(i64 immarg %0, ptr nocapture %1)
-declare void @llvm.lifetime.end.p0(i64 immarg %0, ptr nocapture %1)
+declare void @llvm.lifetime.start.p0(ptr nocapture %1)
+declare void @llvm.lifetime.end.p0(ptr nocapture %1)
 declare void @foo8(ptr nocapture)
 
 
@@ -674,7 +674,7 @@ define void @msan() sanitize_memory {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    call void @llvm.donothing(), !dbg [[DBG1]]
 ; CHECK-NEXT:    [[TEXT:%.*]] = alloca i8, align 1, !dbg [[DBG1]]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[TEXT]]), !dbg [[DBG7]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TEXT]]), !dbg [[DBG7]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[TEXT]] to i64, !dbg [[DBG7]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = xor i64 [[TMP0]], 87960930222080, !dbg [[DBG7]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = inttoptr i64 [[TMP1]] to ptr, !dbg [[DBG7]]
@@ -685,13 +685,13 @@ define void @msan() sanitize_memory {
 ; CHECK-NEXT:    call void @__msan_set_alloca_origin_with_descr(ptr [[TEXT]], i64 1, ptr @[[GLOB6:[0-9]+]], ptr @[[GLOB7:[0-9]+]]), !dbg [[DBG7]]
 ; CHECK-NEXT:    store i64 0, ptr @__msan_param_tls, align 8, !dbg [[DBG8]]
 ; CHECK-NEXT:    call void @foo8(ptr [[TEXT]]), !dbg [[DBG8]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[TEXT]]), !dbg
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TEXT]]), !dbg
 ; CHECK-NEXT:    ret void, !dbg
 ;
 entry:
   %text = alloca i8, align 1, !dbg !10
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text), !dbg !11
+  call void @llvm.lifetime.start.p0(ptr %text), !dbg !11
   call void @foo8(ptr %text), !dbg !12
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text), !dbg !13
+  call void @llvm.lifetime.end.p0(ptr %text), !dbg !13
   ret void, !dbg !14
 }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll
index 2cc8fd645c596..5779367875388 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll
@@ -12,14 +12,14 @@ target triple = "x86_64-unknown-linux-gnu"
 define dso_local ptr @_Z1fv() local_unnamed_addr #0 {
 entry:
   %p = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %p)
+  call void @llvm.lifetime.start.p0(ptr nonnull %p)
   %0 = load i8, ptr @flag, align 1
   %tobool = icmp ne i8 %0, 0
   %call = call zeroext i1 @_Z2f1PPvb(ptr nonnull %p, i1 zeroext %tobool)
   %1 = load ptr, ptr %p, align 8
   %2 = call ptr @llvm.launder.invariant.group.p0(ptr %1)
   %retval.0 = select i1 %call, ptr %2, ptr null
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %p)
+  call void @llvm.lifetime.end.p0(ptr nonnull %p)
   ret ptr %retval.0
 }
 
@@ -29,8 +29,8 @@ declare dso_local zeroext i1 @_Z2f1PPvb(ptr, i1 zeroext) local_unnamed_addr
 
 declare ptr @llvm.launder.invariant.group.p0(ptr)
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 attributes #0 = { sanitize_memory uwtable }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/vector_arith.ll b/llvm/test/Instrumentation/MemorySanitizer/vector_arith.ll
index d614bb85d8584..d1060fb33e1bc 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/vector_arith.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/vector_arith.ll
@@ -17,10 +17,19 @@ define <4 x i32> @Test_sse2_pmadd_wd(<8 x i16> %a, <8 x i16> %b) sanitize_memory
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i16>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 16) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = or <8 x i16> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <8 x i16> [[TMP2]] to <4 x i32>
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i32> [[TMP3]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <8 x i16> [[TMP0]], zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <8 x i16> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <8 x i16> [[A]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <8 x i16> [[B]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = and <8 x i1> [[TMP2]], [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = and <8 x i1> [[TMP3]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = and <8 x i1> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP15:%.*]] = or <8 x i1> [[TMP6]], [[TMP13]]
+; CHECK-NEXT:    [[TMP10:%.*]] = or <8 x i1> [[TMP15]], [[TMP14]]
+; CHECK-NEXT:    [[TMP11:%.*]] = sext <8 x i1> [[TMP10]] to <8 x i16>
+; CHECK-NEXT:    [[TMP16:%.*]] = bitcast <8 x i16> [[TMP11]] to <4 x i32>
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <4 x i32> [[TMP16]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = sext <4 x i1> [[TMP17]] to <4 x i32>
 ; CHECK-NEXT:    [[C:%.*]] = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> [[A]], <8 x i16> [[B]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP5]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <4 x i32> [[C]]
@@ -39,13 +48,27 @@ define <1 x i64> @Test_ssse3_pmadd_ub_sw(<1 x i64> %a, <1 x i64> %b) sanitize_me
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <1 x i64>, ptr @__msan_param_tls, align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i64>, ptr inttoptr (i64 add (i64 ptrtoint (ptr @__msan_param_tls to i64), i64 8) to ptr), align 8
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    [[TMP2:%.*]] = or <1 x i64> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[TMP2]] to <4 x i16>
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i16> [[TMP3]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = sext <4 x i1> [[TMP4]] to <4 x i16>
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <4 x i16> [[TMP5]] to <1 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <1 x i64> [[A]] to <8 x i8>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast <1 x i64> [[B]] to <8 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast <1 x i64> [[TMP0]] to <8 x i8>
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <1 x i64> [[TMP1]] to <8 x i8>
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <8 x i8> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <8 x i8> [[TMP5]], zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp ne <8 x i8> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp ne <8 x i8> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = and <8 x i1> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP11:%.*]] = and <8 x i1> [[TMP17]], [[TMP15]]
+; CHECK-NEXT:    [[TMP12:%.*]] = and <8 x i1> [[TMP14]], [[TMP21]]
+; CHECK-NEXT:    [[TMP13:%.*]] = or <8 x i1> [[TMP16]], [[TMP11]]
+; CHECK-NEXT:    [[TMP22:%.*]] = or <8 x i1> [[TMP13]], [[TMP12]]
+; CHECK-NEXT:    [[TMP7:%.*]] = sext <8 x i1> [[TMP22]] to <8 x i8>
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast <8 x i8> [[TMP7]] to <4 x i16>
+; CHECK-NEXT:    [[TMP24:%.*]] = icmp ne <4 x i16> [[TMP18]], zeroinitializer
+; CHECK-NEXT:    [[TMP23:%.*]] = sext <4 x i1> [[TMP24]] to <4 x i16>
+; CHECK-NEXT:    [[TMP19:%.*]] = bitcast <4 x i16> [[TMP23]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = bitcast i64 [[TMP19]] to <1 x i64>
 ; CHECK-NEXT:    [[C:%.*]] = tail call <1 x i64> @llvm.x86.ssse3.pmadd.ub.sw(<1 x i64> [[A]], <1 x i64> [[B]]) #[[ATTR2]]
-; CHECK-NEXT:    store <1 x i64> [[TMP6]], ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    store <1 x i64> [[TMP20]], ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <1 x i64> [[C]]
 ;
 entry:
diff --git a/llvm/test/Instrumentation/ThreadSanitizer/capture-no-omit.ll b/llvm/test/Instrumentation/ThreadSanitizer/capture-no-omit.ll
new file mode 100644
index 0000000000000..cae04936002cd
--- /dev/null
+++ b/llvm/test/Instrumentation/ThreadSanitizer/capture-no-omit.ll
@@ -0,0 +1,92 @@
+; RUN: opt < %s -passes=tsan -tsan-omit-by-pointer-capturing=0 -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare void @escape(ptr)
+
+@sink = global ptr null, align 4
+
+
+define void @captured2() nounwind uwtable sanitize_thread {
+entry:
+  %ptr = alloca i32, align 4
+  %tmp = alloca ptr, align 8
+  ; transitive escape
+  store ptr %ptr, ptr %tmp, align 8
+  %0 = load ptr, ptr %tmp, align 8
+  store ptr %0, ptr @sink, align 8
+  store i32 42, ptr %ptr, align 4
+  ret void
+}
+; CHECK-LABEL: define void @captured2
+; CHECK: __tsan_write
+; CHECK: __tsan_read
+; CHECK: __tsan_write
+; CHECK: __tsan_write
+; CHECK: ret void
+
+define void @captured3() nounwind uwtable sanitize_thread {
+entry:
+  %stkobj = alloca [2 x i32], align 8
+  ; escapes due to store into global
+  store ptr %stkobj, ptr @sink, align 8
+  ; derived is captured as its base object is captured
+  %derived = getelementptr inbounds i32, ptr %stkobj, i64 1
+  store i32 42, ptr %derived, align 4
+  ret void
+}
+; CHECK-LABEL: define void @captured3
+; CHECK: __tsan_write
+; CHECK: __tsan_write
+; CHECK: ret void
+
+define void @notcaptured2() nounwind uwtable sanitize_thread {
+entry:
+  %ptr = alloca i32, align 4
+  %tmp = alloca ptr, align 8
+  store i32 42, ptr %ptr, align 4
+  ; transitive escape
+  store ptr %ptr, ptr %tmp, align 8
+  %0 = load ptr, ptr %tmp, align 8
+  store ptr %0, ptr @sink, align 8
+  ret void
+}
+; CHECK-LABEL: define void @notcaptured2
+; CHECK: __tsan_write
+; CHECK: __tsan_write
+; CHECK: __tsan_read
+; CHECK: __tsan_write
+; CHECK: ret void
+
+define void @notcaptured3(i1 %cond) nounwind uwtable sanitize_thread {
+entry:
+  %stkobj = alloca [2 x i32], align 8
+  %derived = getelementptr inbounds i32, ptr %stkobj, i64 1
+  %ptr = select i1 %cond, ptr %derived, ptr %stkobj
+  store i32 42, ptr %ptr, align 4
+  ret void
+}
+; CHECK-LABEL: define void @notcaptured3
+; CHECK: __tsan_write
+; CHECK: ret void
+
+define void @notcaptured4() nounwind uwtable sanitize_thread {
+entry:
+  %stkobj = alloca [10 x i8], align 1
+  br label %loop
+
+exit:
+  ret void
+
+loop:
+  %count = phi i32 [ 0, %entry ], [ %addone, %loop ]
+  %derived = phi ptr [ %stkobj, %entry ], [ %ptraddone, %loop ]
+  store i32 %count, ptr %derived, align 4
+  %ptraddone = getelementptr inbounds i32, ptr %derived, i64 1
+  %addone = add nuw nsw i32 %count, 1
+  %eq10 = icmp eq i32 %addone, 10
+  br i1 %eq10, label %exit, label %loop
+}
+; CHECK-LABEL: define void @notcaptured4
+; CHECK: ret void
+; CHECK: __tsan_write
diff --git a/llvm/test/Instrumentation/ThreadSanitizer/capture.ll b/llvm/test/Instrumentation/ThreadSanitizer/capture.ll
index e1b9e03b88446..5083c790011c6 100644
--- a/llvm/test/Instrumentation/ThreadSanitizer/capture.ll
+++ b/llvm/test/Instrumentation/ThreadSanitizer/capture.ll
@@ -45,6 +45,7 @@ entry:
 ; CHECK-LABEL: define void @captured2
 ; CHECK: __tsan_write
 ; CHECK: __tsan_write
+; CHECK-NOT: __tsan_write
 ; CHECK: ret void
 
 define void @captured3() nounwind uwtable sanitize_thread {
@@ -101,6 +102,7 @@ entry:
 ; CHECK-LABEL: define void @notcaptured2
 ; CHECK: __tsan_write
 ; CHECK: __tsan_write
+; CHECK-NOT: __tsan_write
 ; CHECK: ret void
 
 define void @notcaptured3(i1 %cond) nounwind uwtable sanitize_thread {
diff --git a/llvm/test/Instrumentation/TypeSanitizer/alloca.ll b/llvm/test/Instrumentation/TypeSanitizer/alloca.ll
index c53b00650cdcd..deddecfc19d81 100644
--- a/llvm/test/Instrumentation/TypeSanitizer/alloca.ll
+++ b/llvm/test/Instrumentation/TypeSanitizer/alloca.ll
@@ -48,7 +48,7 @@ define void @alloca_lifetime_test(i1 %c) sanitize_type {
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP7]], [[SHADOW_BASE]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = inttoptr i64 [[TMP8]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP9]], i8 0, i64 80, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 10, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[X]])
 ; CHECK-NEXT:    call void @alloca_test_use(ptr [[X]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[X]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = and i64 [[TMP10]], [[APP_MEM_MASK]]
@@ -56,7 +56,7 @@ define void @alloca_lifetime_test(i1 %c) sanitize_type {
 ; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[TMP12]], [[SHADOW_BASE]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = inttoptr i64 [[TMP13]] to ptr
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP14]], i8 0, i64 80, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 10, ptr [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -66,9 +66,62 @@ entry:
   br label %loop
 
 loop:
-  call void @llvm.lifetime.start.p0(i64 10, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
   call void @alloca_test_use(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 10, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
+  br i1 %c, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @dynamic_alloca_lifetime_test(i1 %c, i64 %n) sanitize_type {
+; CHECK-LABEL: @dynamic_alloca_lifetime_test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[APP_MEM_MASK:%.*]] = load i64, ptr @__tysan_app_memory_mask, align 8
+; CHECK-NEXT:    [[SHADOW_BASE:%.*]] = load i64, ptr @__tysan_shadow_memory_address, align 8
+; CHECK-NEXT:    [[X:%.*]] = alloca i32, i64 [[N:%.*]], align 1
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[N]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = and i64 [[TMP1]], [[APP_MEM_MASK]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], [[SHADOW_BASE]]
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP0]], 3
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP5]], i8 0, i64 [[TMP6]], i1 false)
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[N]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = and i64 [[TMP8]], [[APP_MEM_MASK]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[TMP9]], 3
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], [[SHADOW_BASE]]
+; CHECK-NEXT:    [[TMP12:%.*]] = inttoptr i64 [[TMP11]] to ptr
+; CHECK-NEXT:    [[TMP13:%.*]] = shl i64 [[TMP7]], 3
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP12]], i8 0, i64 [[TMP13]], i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[X]])
+; CHECK-NEXT:    call void @alloca_test_use(ptr [[X]])
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[N]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = and i64 [[TMP15]], [[APP_MEM_MASK]]
+; CHECK-NEXT:    [[TMP17:%.*]] = shl i64 [[TMP16]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], [[SHADOW_BASE]]
+; CHECK-NEXT:    [[TMP19:%.*]] = inttoptr i64 [[TMP18]] to ptr
+; CHECK-NEXT:    [[TMP20:%.*]] = shl i64 [[TMP14]], 3
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[TMP19]], i8 0, i64 [[TMP20]], i1 false)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[X]])
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %x = alloca i32, i64 %n, align 1
+  br label %loop
+
+loop:
+  call void @llvm.lifetime.start.p0(ptr %x)
+  call void @alloca_test_use(ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   br i1 %c, label %loop, label %exit
 
 exit:
diff --git a/llvm/test/Linker/Inputs/ident.b.ll b/llvm/test/Linker/Inputs/ident.b.ll
index d9daf8558a704..bfcff3de577e3 100644
--- a/llvm/test/Linker/Inputs/ident.b.ll
+++ b/llvm/test/Linker/Inputs/ident.b.ll
@@ -1,2 +1,3 @@
-!llvm.ident = !{!0}
-!0 = !{!"Compiler V3"}
+!llvm.ident = !{!0, !1}
+!0 = !{!"Compiler V2"}
+!1 = !{!"Compiler V3"}
diff --git a/llvm/test/Linker/Inputs/opencl.md.a.ll b/llvm/test/Linker/Inputs/opencl.md.a.ll
new file mode 100644
index 0000000000000..e1bfe80ede7f9
--- /dev/null
+++ b/llvm/test/Linker/Inputs/opencl.md.a.ll
@@ -0,0 +1,13 @@
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+!opencl.used.extensions = !{!3, !4}
+!opencl.used.optional.core.features = !{!5}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 3, i32 0}
+!2 = !{!"LLVM.org clang version 20.1.0"}
+!3 = !{!"cl_images", !"cl_khr_fp16"}
+!4 = !{!"cl_images", !"cl_doubles"}
+!5 = !{!"cl_images"}
diff --git a/llvm/test/Linker/Inputs/opencl.md.b.ll b/llvm/test/Linker/Inputs/opencl.md.b.ll
new file mode 100644
index 0000000000000..e062010718858
--- /dev/null
+++ b/llvm/test/Linker/Inputs/opencl.md.b.ll
@@ -0,0 +1,13 @@
+!llvm.module.flags = !{!0}
+!opencl.ocl.version = !{!1}
+!opencl.spir.version = !{!1}
+!opencl.used.extensions = !{!3, !4}
+!opencl.used.optional.core.features = !{!5}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 3, i32 0}
+!2 = !{!"LLVM.org clang version 20.1.0"}
+!3 = !{!"cl_images", !"cl_khr_fp16"}
+!4 = !{!"cl_khr_fp16", !"cl_doubles"}
+!5 = !{!"cl_images"}
diff --git a/llvm/test/Linker/Inputs/opencl.md.c.ll b/llvm/test/Linker/Inputs/opencl.md.c.ll
new file mode 100644
index 0000000000000..03ad621dde7b8
--- /dev/null
+++ b/llvm/test/Linker/Inputs/opencl.md.c.ll
@@ -0,0 +1,6 @@
+!opencl.ocl.version = !{!0}
+!opencl.spir.version = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 2, i32 0}
+!1 = !{!"LLVM.org clang version 20.1.0"}
diff --git a/llvm/test/Linker/dicompositetype-unique.ll b/llvm/test/Linker/dicompositetype-unique.ll
index ab1fdaa3616c9..28b2f33001b9e 100644
--- a/llvm/test/Linker/dicompositetype-unique.ll
+++ b/llvm/test/Linker/dicompositetype-unique.ll
@@ -17,8 +17,8 @@
 
 ; Check that the type map will unique two DICompositeTypes.
 
-; CHECK:   !named = !{!0, !1, !2, !3, !0, !1, !2, !3}
-; NOMAP:   !named = !{!0, !1, !2, !3, !0, !4, !5, !6}
+; CHECK:   !named = !{!0, !1, !2, !3}
+; NOMAP:   !named = !{!0, !1, !2, !3, !4, !5, !6}
 !named = !{!0, !1, !2, !3}
 
 ; Check both directions.
diff --git a/llvm/test/Linker/distinct.ll b/llvm/test/Linker/distinct.ll
index f21c51ce8b54f..0863446f9a33e 100644
--- a/llvm/test/Linker/distinct.ll
+++ b/llvm/test/Linker/distinct.ll
@@ -9,7 +9,7 @@
 ; Add an external reference to @global so that it gets linked in.
 @alias = alias i32, ptr @global
 
-; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !0, !1, !2, !9, !10, !11, !12, !13, !14}
+; CHECK: !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9, !10, !11, !12, !13, !14}
 !named = !{!0, !1, !2, !3, !4, !5, !6, !7, !8}
 
 ; CHECK:      !0 = !{}
diff --git a/llvm/test/Linker/ident.ll b/llvm/test/Linker/ident.ll
index 93bf8c7b2ca72..532a8427346ce 100644
--- a/llvm/test/Linker/ident.ll
+++ b/llvm/test/Linker/ident.ll
@@ -1,6 +1,7 @@
 ; RUN: llvm-link %S/Inputs/ident.a.ll %S/Inputs/ident.b.ll -S | FileCheck %s
 
-; Verify that multiple input llvm.ident metadata are linked together.
+; Verify that multiple input llvm.ident metadata are linked together, and
+; duplicate "Compiler V2" metadata from the two inputs are unified.
 
 ; CHECK-DAG: !llvm.ident = !{!0, !1, !2}
 ; CHECK-DAG: "Compiler V1"
diff --git a/llvm/test/Linker/named-metadata-opencl-version.ll b/llvm/test/Linker/named-metadata-opencl-version.ll
new file mode 100644
index 0000000000000..158091a77ad91
--- /dev/null
+++ b/llvm/test/Linker/named-metadata-opencl-version.ll
@@ -0,0 +1,20 @@
+; RUN: llvm-link %S/Inputs/opencl.md.a.ll %S/Inputs/opencl.md.c.ll -S | FileCheck %s
+
+; OpenCL/SPIR version is different in input modules.
+; Verify that different OpenCL/SPIR version metadata nodes are kept in the
+; destination module.
+
+; CHECK: !opencl.ocl.version = !{![[#MD0:]], ![[#MD1:]]}
+; CHECK: !opencl.spir.version = !{![[#MD0]], ![[#MD1]]}
+; CHECK: !opencl.used.extensions = !{![[#MD2:]], ![[#MD3:]]}
+; CHECK: !opencl.used.optional.core.features = !{![[#MD4:]]}
+; CHECK: !llvm.ident = !{![[#MD5:]]}
+; CHECK: !llvm.module.flags = !{![[#MD6:]]}
+
+; CHECK: ![[#MD0]] = !{i32 3, i32 0}
+; CHECK: ![[#MD1]] = !{i32 2, i32 0}
+; CHECK: ![[#MD2]] = !{!"cl_images", !"cl_khr_fp16"}
+; CHECK: ![[#MD3]] = !{!"cl_images", !"cl_doubles"}
+; CHECK: ![[#MD4]] = !{!"cl_images"}
+; CHECK: ![[#MD5]] = !{!"LLVM.org clang version 20.1.0"}
+; CHECK: ![[#MD6]] = !{i32 1, !"wchar_size", i32 4}
diff --git a/llvm/test/Linker/named-metadata-opencl.ll b/llvm/test/Linker/named-metadata-opencl.ll
new file mode 100644
index 0000000000000..109280ad08c1e
--- /dev/null
+++ b/llvm/test/Linker/named-metadata-opencl.ll
@@ -0,0 +1,18 @@
+; RUN: llvm-link %S/Inputs/opencl.md.a.ll %S/Inputs/opencl.md.b.ll -S | FileCheck %s
+
+; Verify that duplicate named metadata node isn't added to the result module.
+
+; CHECK: !opencl.ocl.version = !{![[#MD0:]]}
+; CHECK: !opencl.spir.version = !{![[#MD0]]}
+; CHECK: !opencl.used.extensions = !{![[#MD1:]], ![[#MD2:]], ![[#MD3:]]}
+; CHECK: !opencl.used.optional.core.features = !{![[#MD4:]]}
+; CHECK: !llvm.ident = !{![[#MD5:]]}
+; CHECK: !llvm.module.flags = !{![[#MD6:]]}
+
+; CHECK: ![[#MD0]] = !{i32 3, i32 0}
+; CHECK: ![[#MD1]] = !{!"cl_images", !"cl_khr_fp16"}
+; CHECK: ![[#MD2]] = !{!"cl_images", !"cl_doubles"}
+; CHECK: ![[#MD3]] = !{!"cl_khr_fp16", !"cl_doubles"}
+; CHECK: ![[#MD4]] = !{!"cl_images"}
+; CHECK: ![[#MD5]] = !{!"LLVM.org clang version 20.1.0"}
+; CHECK: ![[#MD6]] = !{i32 1, !"wchar_size", i32 4}
diff --git a/llvm/test/Linker/unique-fwd-decl-order.ll b/llvm/test/Linker/unique-fwd-decl-order.ll
index e1d8c2e5cf92f..2f83714305b53 100644
--- a/llvm/test/Linker/unique-fwd-decl-order.ll
+++ b/llvm/test/Linker/unique-fwd-decl-order.ll
@@ -8,7 +8,7 @@
 ; Note that these two assembly files number the nodes identically, even though
 ; the nodes are in a different order.  This is for the reader's convenience.
 
-; CHECK: !named = !{!0, !0}
+; CHECK: !named = !{!0}
 !named = !{!0}
 
 ; CHECK: !0 = !{!1}
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfadd.s b/llvm/test/MC/AArch64/SVE2p1/bfadd.s
index 562781c57796c..668b64dcedebe 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfadd.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfadd.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfclamp.s b/llvm/test/MC/AArch64/SVE2p1/bfclamp.s
index b63fbf4f3e368..328f566891f34 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfclamp.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfclamp.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfmax.s b/llvm/test/MC/AArch64/SVE2p1/bfmax.s
index a040361614767..28589738b56ef 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfmax.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfmax.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfmaxnm.s b/llvm/test/MC/AArch64/SVE2p1/bfmaxnm.s
index d6439a649d0ef..e882bd825aa06 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfmaxnm.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfmaxnm.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfmin.s b/llvm/test/MC/AArch64/SVE2p1/bfmin.s
index 8ce308325086d..1e5ac8ccc5aeb 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfmin.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfmin.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfminnm.s b/llvm/test/MC/AArch64/SVE2p1/bfminnm.s
index 5b47dc5f3f69c..07c11e41d1de7 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfminnm.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfminnm.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfmla.s b/llvm/test/MC/AArch64/SVE2p1/bfmla.s
index 909174d6958c6..6d02b65d61846 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfmla.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfmla.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfmls.s b/llvm/test/MC/AArch64/SVE2p1/bfmls.s
index 77e76bc07a361..67672c18bb1ea 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfmls.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfmls.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfmul.s b/llvm/test/MC/AArch64/SVE2p1/bfmul.s
index ff760732311e2..c562ab35126fa 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfmul.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfmul.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/SVE2p1/bfsub.s b/llvm/test/MC/AArch64/SVE2p1/bfsub.s
index 3de0b3bdd6829..ae1d70dec5b36 100644
--- a/llvm/test/MC/AArch64/SVE2p1/bfsub.s
+++ b/llvm/test/MC/AArch64/SVE2p1/bfsub.s
@@ -1,6 +1,6 @@
 // RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
-// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve2 < %s 2>&1 \
 // RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
 // RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve2,+sve-b16b16 < %s \
 // RUN:        | llvm-objdump -d --no-print-imm-hex --mattr=+sve2,+sve-b16b16 - | FileCheck %s --check-prefix=CHECK-INST
diff --git a/llvm/test/MC/AArch64/arm64-aliases.s b/llvm/test/MC/AArch64/arm64-aliases.s
index 3ace7a0f7183b..ae157c676c95f 100644
--- a/llvm/test/MC/AArch64/arm64-aliases.s
+++ b/llvm/test/MC/AArch64/arm64-aliases.s
@@ -512,6 +512,20 @@ foo:
   sys #4, c8, c3, #6
 ; CHECK: tlbi vmalls12e1is
 
+; Check that all 5 register bits are set (0x31):
+;    (from Arm ARM regarding TLBI instructions without operands)
+;    "Rt should be encoded as 0b11111. If the Rt field is not set to 0b11111,
+;     it is CONSTRAINED UNPREDICTABLE whether:
+;       * The instruction is UNDEFINED.
+;       * The instruction behaves as if the Rt field is set to 0b11111."
+;
+; Do not disassemble this to `tlbi` but a SYS alias instead
+;
+  sys #4, c8, c7, #6, x30
+; CHECK: sys #0x4, c8, c7, #0x6, x30
+  sys #4, c8, c7, #6, x31
+; CHECK: tlbi vmalls12e1
+
   ic ialluis
 ; CHECK: ic ialluis                 ; encoding: [0x1f,0x71,0x08,0xd5]
   ic iallu
diff --git a/llvm/test/MC/AArch64/armv8.6a-fgt.s b/llvm/test/MC/AArch64/armv8.6a-fgt.s
index 11002aca5e1a0..632a531bfaee4 100644
--- a/llvm/test/MC/AArch64/armv8.6a-fgt.s
+++ b/llvm/test/MC/AArch64/armv8.6a-fgt.s
@@ -1,75 +1,149 @@
-// RUN:     llvm-mc -triple aarch64 -show-encoding -mattr=+fgt   < %s | FileCheck %s
-// RUN:     llvm-mc -triple aarch64 -show-encoding -mattr=+v8.6a < %s | FileCheck %s
-// RUN: not llvm-mc -triple aarch64 -show-encoding  < %s 2>&1         | FileCheck %s --check-prefix=NOFGT
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v8.6a < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+fgt < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+fgt < %s \
+// RUN:        | llvm-objdump -d --mattr=+fgt - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+fgt < %s \
+// RUN:   | llvm-objdump -d --mattr=-fgt - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+fgt < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+fgt -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 msr HFGRTR_EL2, x0
+// CHECK-INST: msr HFGRTR_EL2, x0
+// CHECK-ENCODING: encoding: [0x80,0x11,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c1180      msr S3_4_C1_C1_4, x0
+
 msr HFGWTR_EL2, x5
+// CHECK-INST: msr HFGWTR_EL2, x5
+// CHECK-ENCODING: encoding: [0xa5,0x11,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c11a5      msr S3_4_C1_C1_5, x5
+
 msr HFGITR_EL2, x10
+// CHECK-INST: msr HFGITR_EL2, x10
+// CHECK-ENCODING: encoding: [0xca,0x11,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c11ca      msr S3_4_C1_C1_6, x10
+
 msr HDFGRTR_EL2, x15
+// CHECK-INST: msr HDFGRTR_EL2, x15
+// CHECK-ENCODING: encoding: [0x8f,0x31,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c318f      msr S3_4_C3_C1_4, x15
+
 msr HDFGWTR_EL2, x20
+// CHECK-INST: msr HDFGWTR_EL2, x20
+// CHECK-ENCODING: encoding: [0xb4,0x31,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c31b4      msr S3_4_C3_C1_5, x20
+
 msr HAFGRTR_EL2, x25
-// CHECK: msr     HFGRTR_EL2, x0          // encoding: [0x80,0x11,0x1c,0xd5]
-// CHECK: msr     HFGWTR_EL2, x5          // encoding: [0xa5,0x11,0x1c,0xd5]
-// CHECK: msr     HFGITR_EL2, x10         // encoding: [0xca,0x11,0x1c,0xd5]
-// CHECK: msr     HDFGRTR_EL2, x15        // encoding: [0x8f,0x31,0x1c,0xd5]
-// CHECK: msr     HDFGWTR_EL2, x20        // encoding: [0xb4,0x31,0x1c,0xd5]
-// CHECK: msr     HAFGRTR_EL2, x25        // encoding: [0xd9,0x31,0x1c,0xd5]
-// NOFGT: error: expected writable system register or pstate
-// NOFGT: error: expected writable system register or pstate
-// NOFGT: error: expected writable system register or pstate
-// NOFGT: error: expected writable system register or pstate
-// NOFGT: error: expected writable system register or pstate
-// NOFGT: error: expected writable system register or pstate
+// CHECK-INST: msr HAFGRTR_EL2, x25
+// CHECK-ENCODING: encoding: [0xd9,0x31,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c31d9      msr S3_4_C3_C1_6, x25
 
 mrs x30,  HFGRTR_EL2
+// CHECK-INST: mrs x30, HFGRTR_EL2
+// CHECK-ENCODING: encoding: [0x9e,0x11,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:11: error: expected readable system register
+// CHECK-UNKNOWN:  d53c119e      mrs x30, S3_4_C1_C1_4
+
 mrs x25,  HFGWTR_EL2
+// CHECK-INST: mrs x25, HFGWTR_EL2
+// CHECK-ENCODING: encoding: [0xb9,0x11,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:11: error: expected readable system register
+// CHECK-UNKNOWN:  d53c11b9      mrs x25, S3_4_C1_C1_5
+
 mrs x20,  HFGITR_EL2
+// CHECK-INST: mrs x20, HFGITR_EL2
+// CHECK-ENCODING: encoding: [0xd4,0x11,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:11: error: expected readable system register
+// CHECK-UNKNOWN:  d53c11d4      mrs x20, S3_4_C1_C1_6
+
 mrs x15,  HDFGRTR_EL2
+// CHECK-INST: mrs x15, HDFGRTR_EL2
+// CHECK-ENCODING: encoding: [0x8f,0x31,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:11: error: expected readable system register
+// CHECK-UNKNOWN:  d53c318f      mrs x15, S3_4_C3_C1_4
+
 mrs x10,  HDFGWTR_EL2
-mrs x5,   HAFGRTR_EL2
-// CHECK: mrs     x30, HFGRTR_EL2         // encoding: [0x9e,0x11,0x3c,0xd5]
-// CHECK: mrs     x25, HFGWTR_EL2         // encoding: [0xb9,0x11,0x3c,0xd5]
-// CHECK: mrs     x20, HFGITR_EL2         // encoding: [0xd4,0x11,0x3c,0xd5]
-// CHECK: mrs     x15, HDFGRTR_EL2        // encoding: [0x8f,0x31,0x3c,0xd5]
-// CHECK: mrs     x10, HDFGWTR_EL2        // encoding: [0xaa,0x31,0x3c,0xd5]
-// CHECK: mrs     x5, HAFGRTR_EL2         // encoding: [0xc5,0x31,0x3c,0xd5]
-// NOFGT: error: expected readable system register
-// NOFGT: error: expected readable system register
-// NOFGT: error: expected readable system register
-// NOFGT: error: expected readable system register
-// NOFGT: error: expected readable system register
-// NOFGT: error: expected readable system register
+// CHECK-INST: mrs x10, HDFGWTR_EL2
+// CHECK-ENCODING: encoding: [0xaa,0x31,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:11: error: expected readable system register
+// CHECK-UNKNOWN:  d53c31aa      mrs x10, S3_4_C3_C1_5
 
+mrs x5,   HAFGRTR_EL2
+// CHECK-INST: mrs x5, HAFGRTR_EL2
+// CHECK-ENCODING: encoding: [0xc5,0x31,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:11: error: expected readable system register
+// CHECK-UNKNOWN:  d53c31c5      mrs x5, S3_4_C3_C1_6
 
 mrs x3, HDFGRTR2_EL2
+// CHECK-INST: mrs x3, HDFGRTR2_EL2
+// CHECK-ENCODING: encoding: [0x03,0x31,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:9: error: expected readable system register
+// CHECK-UNKNOWN:  d53c3103      mrs x3, S3_4_C3_C1_0
+
 mrs x3, HDFGWTR2_EL2
+// CHECK-INST: mrs x3, HDFGWTR2_EL2
+// CHECK-ENCODING: encoding: [0x23,0x31,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:9: error: expected readable system register
+// CHECK-UNKNOWN:  d53c3123      mrs x3, S3_4_C3_C1_1
+
 mrs x3, HFGRTR2_EL2
+// CHECK-INST: mrs x3, HFGRTR2_EL2
+// CHECK-ENCODING: encoding: [0x43,0x31,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:9: error: expected readable system register
+// CHECK-UNKNOWN:  d53c3143      mrs x3, S3_4_C3_C1_2
+
 mrs x3, HFGWTR2_EL2
-mrs x3, HFGITR2_EL2
-// CHECK: mrs     x3, HDFGRTR2_EL2                // encoding: [0x03,0x31,0x3c,0xd5]
-// CHECK: mrs     x3, HDFGWTR2_EL2                // encoding: [0x23,0x31,0x3c,0xd5]
-// CHECK: mrs     x3, HFGRTR2_EL2                 // encoding: [0x43,0x31,0x3c,0xd5]
-// CHECK: mrs     x3, HFGWTR2_EL2                 // encoding: [0x63,0x31,0x3c,0xd5]
-// CHECK: mrs     x3, HFGITR2_EL2                 // encoding: [0xe3,0x31,0x3c,0xd5]
-// NOFGT: error: expected readable system register
-// NOFGT: error: expected readable system register
-// NOFGT: error: expected readable system register
-// NOFGT: error: expected readable system register
-// NOFGT: error: expected readable system register
+// CHECK-INST: mrs x3, HFGWTR2_EL2
+// CHECK-ENCODING: encoding: [0x63,0x31,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:9: error: expected readable system register
+// CHECK-UNKNOWN:  d53c3163      mrs x3, S3_4_C3_C1_3
 
+mrs x3, HFGITR2_EL2
+// CHECK-INST: mrs x3, HFGITR2_EL2
+// CHECK-ENCODING: encoding: [0xe3,0x31,0x3c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:9: error: expected readable system register
+// CHECK-UNKNOWN:  d53c31e3      mrs x3, S3_4_C3_C1_7
 
 msr HDFGRTR2_EL2, x3
+// CHECK-INST: msr HDFGRTR2_EL2, x3
+// CHECK-ENCODING: encoding: [0x03,0x31,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c3103      msr S3_4_C3_C1_0, x3
+
 msr HDFGWTR2_EL2, x3
+// CHECK-INST: msr HDFGWTR2_EL2, x3
+// CHECK-ENCODING: encoding: [0x23,0x31,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c3123      msr S3_4_C3_C1_1, x3
+
 msr HFGRTR2_EL2, x3
+// CHECK-INST: msr HFGRTR2_EL2, x3
+// CHECK-ENCODING: encoding: [0x43,0x31,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c3143      msr S3_4_C3_C1_2, x3
+
 msr HFGWTR2_EL2, x3
+// CHECK-INST: msr HFGWTR2_EL2, x3
+// CHECK-ENCODING: encoding: [0x63,0x31,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c3163      msr S3_4_C3_C1_3, x3
+
 msr HFGITR2_EL2, x3
-// CHECK: msr     HDFGRTR2_EL2, x3                // encoding: [0x03,0x31,0x1c,0xd5]
-// CHECK: msr     HDFGWTR2_EL2, x3                // encoding: [0x23,0x31,0x1c,0xd5]
-// CHECK: msr     HFGRTR2_EL2, x3                 // encoding: [0x43,0x31,0x1c,0xd5]
-// CHECK: msr     HFGWTR2_EL2, x3                 // encoding: [0x63,0x31,0x1c,0xd5]
-// CHECK: msr     HFGITR2_EL2, x3                 // encoding: [0xe3,0x31,0x1c,0xd5]
-// NOFGT: error: expected writable system register
-// NOFGT: error: expected writable system register
-// NOFGT: error: expected writable system register
-// NOFGT: error: expected writable system register
-// NOFGT: error: expected writable system register
+// CHECK-INST: msr HFGITR2_EL2, x3
+// CHECK-ENCODING: encoding: [0xe3,0x31,0x1c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:5: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c31e3      msr S3_4_C3_C1_7, x3
diff --git a/llvm/test/MC/AArch64/armv8.8a-mops-diagnostics.s b/llvm/test/MC/AArch64/armv8.8a-mops-diagnostics.s
new file mode 100644
index 0000000000000..a9a8612bc048f
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv8.8a-mops-diagnostics.s
@@ -0,0 +1,227 @@
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+mops,+mte < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.8a,+mte < %s 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
+
+
+// All operand must be different from each other
+
+// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
+cpyfp [x0]!, [x0]!, x1!
+cpyfp [x0]!, [x1]!, x0!
+cpyfp [x1]!, [x0]!, x0!
+
+// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
+cpyfm [x0]!, [x0]!, x1!
+cpyfm [x0]!, [x1]!, x0!
+cpyfm [x1]!, [x0]!, x0!
+
+// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
+cpyfe [x0]!, [x0]!, x1!
+cpyfe [x0]!, [x1]!, x0!
+cpyfe [x1]!, [x0]!, x0!
+
+// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
+cpyp [x0]!, [x0]!, x1!
+cpyp [x0]!, [x1]!, x0!
+cpyp [x1]!, [x0]!, x0!
+
+// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
+cpym [x0]!, [x0]!, x1!
+cpym [x0]!, [x1]!, x0!
+cpym [x1]!, [x0]!, x0!
+
+// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
+cpye [x0]!, [x0]!, x1!
+cpye [x0]!, [x1]!, x0!
+cpye [x1]!, [x0]!, x0!
+
+// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
+setp [x0]!, x0!, x1
+setp [x0]!, x1!, x0
+setp [x1]!, x0!, x0
+
+// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
+setm [x0]!, x0!, x1
+setm [x0]!, x1!, x0
+setm [x1]!, x0!, x0
+
+// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
+sete [x0]!, x0!, x1
+sete [x0]!, x1!, x0
+sete [x1]!, x0!, x0
+
+// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
+setgp [x0]!, x0!, x1
+setgp [x0]!, x1!, x0
+setgp [x1]!, x0!, x0
+
+// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
+setgm [x0]!, x0!, x1
+setgm [x0]!, x1!, x0
+setgm [x1]!, x0!, x0
+
+// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
+// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
+// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
+setge [x0]!, x0!, x1
+setge [x0]!, x1!, x0
+setge [x1]!, x0!, x0
+
+// SP cannot be used as argument at any position
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpyfp [sp]!, [x1]!, x2!
+cpyfp [x0]!, [sp]!, x2!
+cpyfp [x0]!, [x1]!, sp!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpyfm [sp]!, [x1]!, x2!
+cpyfm [x0]!, [sp]!, x2!
+cpyfm [x0]!, [x1]!, sp!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpyfe [sp]!, [x1]!, x2!
+cpyfe [x0]!, [sp]!, x2!
+cpyfe [x0]!, [x1]!, sp!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpyp [sp]!, [x2]!, x2!
+cpyp [x0]!, [sp]!, x2!
+cpyp [x0]!, [x1]!, sp!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpym [sp]!, [x2]!, x2!
+cpym [x0]!, [sp]!, x2!
+cpym [x0]!, [x1]!, sp!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpye [sp]!, [x2]!, x2!
+cpye [x0]!, [sp]!, x2!
+cpye [x0]!, [x1]!, sp!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+setp [sp]!, x1!, x2
+setp [x0]!, sp!, x2
+setp [x0]!, x1!, sp
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+setm [sp]!, x1!, x2
+setm [x0]!, sp!, x2
+setm [x0]!, x1!, sp
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+sete [sp]!, x1!, x2
+sete [x0]!, sp!, x2
+sete [x0]!, x1!, sp
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+setgp [sp]!, x1!, x2
+setgp [x0]!, sp!, x2
+setgp [x0]!, x1!, sp
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+setgm [sp]!, x1!, x2
+setgm [x0]!, sp!, x2
+setgm [x0]!, x1!, sp
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+setge [sp]!, x1!, x2
+setge [x0]!, sp!, x2
+setge [x0]!, x1!, sp
+
+// XZR can only be used at:
+//  - the size operand in CPY.
+//  - the size or source operands in SET.
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpyfp [xzr]!, [x1]!, x2!
+cpyfp [x0]!, [xzr]!, x2!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpyfm [xzr]!, [x1]!, x2!
+cpyfm [x0]!, [xzr]!, x2!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpyfe [xzr]!, [x1]!, x2!
+cpyfe [x0]!, [xzr]!, x2!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpyp [xzr]!, [x2]!, x2!
+cpyp [x0]!, [xzr]!, x2!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpym [xzr]!, [x2]!, x2!
+cpym [x0]!, [xzr]!, x2!
+
+// CHECK-ERROR: error: invalid operand for instruction
+// CHECK-ERROR: error: invalid operand for instruction
+cpye [xzr]!, [x2]!, x2!
+cpye [x0]!, [xzr]!, x2!
+
+// CHECK-ERROR: error: invalid operand for instruction
+setp [xzr]!, x1!, x2
+
+// CHECK-ERROR: error: invalid operand for instruction
+setm [xzr]!, x1!, x2
+
+// CHECK-ERROR: error: invalid operand for instruction
+sete [xzr]!, x1!, x2
+
+// CHECK-ERROR: error: invalid operand for instruction
+setgp [xzr]!, x1!, x2
+
+// CHECK-ERROR: error: invalid operand for instruction
+setgm [xzr]!, x1!, x2
+
+// CHECK-ERROR: error: invalid operand for instruction
+setge [xzr]!, x1!, x2
diff --git a/llvm/test/MC/AArch64/armv8.8a-mops.s b/llvm/test/MC/AArch64/armv8.8a-mops.s
index f8d75e73d47d5..10a551d052515 100644
--- a/llvm/test/MC/AArch64/armv8.8a-mops.s
+++ b/llvm/test/MC/AArch64/armv8.8a-mops.s
@@ -1,654 +1,849 @@
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+mops,+mte < %s 2> %t | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MTE
-// RUN: FileCheck --check-prefix=CHECK-ERROR %s < %t
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.8a,+mte < %s 2> %t | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-MTE
-// RUN: FileCheck --check-prefix=CHECK-ERROR %s < %t
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+mops < %s 2> %t | FileCheck %s --check-prefix=CHECK
-// RUN: FileCheck --check-prefix=CHECK-NO-MTE-ERR %s < %t
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.8a < %s 2> %t | FileCheck %s --check-prefix=CHECK
-// RUN: FileCheck --check-prefix=CHECK-NO-MTE-ERR %s < %t
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu < %s 2> %t
-// RUN: FileCheck --check-prefix=CHECK-NO-MOPS-ERR --check-prefix=CHECK-NO-MOPSMTE-ERR %s < %t
-
-// CHECK:      [0x40,0x04,0x01,0x19]
-// CHECK-NEXT: [0x40,0x44,0x01,0x19]
-// CHECK-NEXT: [0x40,0x84,0x01,0x19]
-// CHECK-NEXT: [0x40,0xc4,0x01,0x19]
-// CHECK-NEXT: [0x40,0x14,0x01,0x19]
-// CHECK-NEXT: [0x40,0x54,0x01,0x19]
-// CHECK-NEXT: [0x40,0x94,0x01,0x19]
-// CHECK-NEXT: [0x40,0xd4,0x01,0x19]
-// CHECK-NEXT: [0x40,0x24,0x01,0x19]
-// CHECK-NEXT: [0x40,0x64,0x01,0x19]
-// CHECK-NEXT: [0x40,0xa4,0x01,0x19]
-// CHECK-NEXT: [0x40,0xe4,0x01,0x19]
-// CHECK-NEXT: [0x40,0x34,0x01,0x19]
-// CHECK-NEXT: [0x40,0x74,0x01,0x19]
-// CHECK-NEXT: [0x40,0xb4,0x01,0x19]
-// CHECK-NEXT: [0x40,0xf4,0x01,0x19]
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+mops,+mte < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v8.8a,+mte < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+mops,+mte < %s \
+// RUN:        | llvm-objdump -d --mattr=+mops,+mte - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+mops,+mte < %s \
+// RUN:   | llvm-objdump -d --mattr=-mops,-mte - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+mops,+mte < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+mops,+mte -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+
 cpyfp [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfp [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x04,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19010440      <unknown>
+
 cpyfpwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfpwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x44,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19014440      <unknown>
+
 cpyfprn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfprn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x84,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19018440      <unknown>
+
 cpyfpn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfpn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xc4,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1901c440      <unknown>
+
 cpyfpwt [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfpwt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x14,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19011440      <unknown>
+
 cpyfpwtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfpwtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x54,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19015440      <unknown>
+
 cpyfpwtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfpwtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x94,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19019440      <unknown>
+
 cpyfpwtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfpwtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xd4,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1901d440      <unknown>
+
 cpyfprt [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfprt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x24,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19012440      <unknown>
+
 cpyfprtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfprtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x64,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19016440      <unknown>
+
 cpyfprtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfprtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xa4,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1901a440      <unknown>
+
 cpyfprtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfprtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xe4,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1901e440      <unknown>
+
 cpyfpt [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfpt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x34,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19013440      <unknown>
+
 cpyfptwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfptwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x74,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19017440      <unknown>
+
 cpyfptrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfptrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xb4,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1901b440      <unknown>
+
 cpyfptn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfptn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xf4,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1901f440      <unknown>
 
-// CHECK:      [0x40,0x04,0x41,0x19]
-// CHECK-NEXT: [0x40,0x44,0x41,0x19]
-// CHECK-NEXT: [0x40,0x84,0x41,0x19]
-// CHECK-NEXT: [0x40,0xc4,0x41,0x19]
-// CHECK-NEXT: [0x40,0x14,0x41,0x19]
-// CHECK-NEXT: [0x40,0x54,0x41,0x19]
-// CHECK-NEXT: [0x40,0x94,0x41,0x19]
-// CHECK-NEXT: [0x40,0xd4,0x41,0x19]
-// CHECK-NEXT: [0x40,0x24,0x41,0x19]
-// CHECK-NEXT: [0x40,0x64,0x41,0x19]
-// CHECK-NEXT: [0x40,0xa4,0x41,0x19]
-// CHECK-NEXT: [0x40,0xe4,0x41,0x19]
-// CHECK-NEXT: [0x40,0x34,0x41,0x19]
-// CHECK-NEXT: [0x40,0x74,0x41,0x19]
-// CHECK-NEXT: [0x40,0xb4,0x41,0x19]
-// CHECK-NEXT: [0x40,0xf4,0x41,0x19]
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
 cpyfm [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfm [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x04,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19410440      <unknown>
+
 cpyfmwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x44,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19414440      <unknown>
+
 cpyfmrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x84,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19418440      <unknown>
+
 cpyfmn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xc4,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1941c440      <unknown>
+
 cpyfmwt [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmwt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x14,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19411440      <unknown>
+
 cpyfmwtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmwtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x54,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19415440      <unknown>
+
 cpyfmwtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmwtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x94,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19419440      <unknown>
+
 cpyfmwtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmwtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xd4,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1941d440      <unknown>
+
 cpyfmrt [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmrt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x24,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19412440      <unknown>
+
 cpyfmrtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmrtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x64,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19416440      <unknown>
+
 cpyfmrtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmrtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xa4,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1941a440      <unknown>
+
 cpyfmrtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmrtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xe4,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1941e440      <unknown>
+
 cpyfmt [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x34,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19413440      <unknown>
+
 cpyfmtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x74,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19417440      <unknown>
+
 cpyfmtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xb4,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1941b440      <unknown>
+
 cpyfmtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfmtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xf4,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1941f440      <unknown>
 
-// CHECK:      [0x40,0x04,0x81,0x19]
-// CHECK-NEXT: [0x40,0x44,0x81,0x19]
-// CHECK-NEXT: [0x40,0x84,0x81,0x19]
-// CHECK-NEXT: [0x40,0xc4,0x81,0x19]
-// CHECK-NEXT: [0x40,0x14,0x81,0x19]
-// CHECK-NEXT: [0x40,0x54,0x81,0x19]
-// CHECK-NEXT: [0x40,0x94,0x81,0x19]
-// CHECK-NEXT: [0x40,0xd4,0x81,0x19]
-// CHECK-NEXT: [0x40,0x24,0x81,0x19]
-// CHECK-NEXT: [0x40,0x64,0x81,0x19]
-// CHECK-NEXT: [0x40,0xa4,0x81,0x19]
-// CHECK-NEXT: [0x40,0xe4,0x81,0x19]
-// CHECK-NEXT: [0x40,0x34,0x81,0x19]
-// CHECK-NEXT: [0x40,0x74,0x81,0x19]
-// CHECK-NEXT: [0x40,0xb4,0x81,0x19]
-// CHECK-NEXT: [0x40,0xf4,0x81,0x19]
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
 cpyfe [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfe [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x04,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19810440      <unknown>
+
 cpyfewn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfewn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x44,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19814440      <unknown>
+
 cpyfern [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfern [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x84,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19818440      <unknown>
+
 cpyfen [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfen [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xc4,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1981c440      <unknown>
+
 cpyfewt [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfewt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x14,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19811440      <unknown>
+
 cpyfewtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfewtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x54,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19815440      <unknown>
+
 cpyfewtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfewtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x94,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19819440      <unknown>
+
 cpyfewtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfewtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xd4,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1981d440      <unknown>
+
 cpyfert [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfert [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x24,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19812440      <unknown>
+
 cpyfertwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfertwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x64,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19816440      <unknown>
+
 cpyfertrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfertrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xa4,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1981a440      <unknown>
+
 cpyfertn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfertn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xe4,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1981e440      <unknown>
+
 cpyfet [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfet [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x34,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19813440      <unknown>
+
 cpyfetwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfetwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x74,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19817440      <unknown>
+
 cpyfetrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfetrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xb4,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1981b440      <unknown>
+
 cpyfetn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyfetn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xf4,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1981f440      <unknown>
 
-// CHECK:      [0x40,0x04,0x01,0x1d]
-// CHECK-NEXT: [0x40,0x44,0x01,0x1d]
-// CHECK-NEXT: [0x40,0x84,0x01,0x1d]
-// CHECK-NEXT: [0x40,0xc4,0x01,0x1d]
-// CHECK-NEXT: [0x40,0x14,0x01,0x1d]
-// CHECK-NEXT: [0x40,0x54,0x01,0x1d]
-// CHECK-NEXT: [0x40,0x94,0x01,0x1d]
-// CHECK-NEXT: [0x40,0xd4,0x01,0x1d]
-// CHECK-NEXT: [0x40,0x24,0x01,0x1d]
-// CHECK-NEXT: [0x40,0x64,0x01,0x1d]
-// CHECK-NEXT: [0x40,0xa4,0x01,0x1d]
-// CHECK-NEXT: [0x40,0xe4,0x01,0x1d]
-// CHECK-NEXT: [0x40,0x34,0x01,0x1d]
-// CHECK-NEXT: [0x40,0x74,0x01,0x1d]
-// CHECK-NEXT: [0x40,0xb4,0x01,0x1d]
-// CHECK-NEXT: [0x40,0xf4,0x01,0x1d]
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
 cpyp [x0]!, [x1]!, x2!
+// CHECK-INST: cpyp [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x04,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d010440      <unknown>
+
 cpypwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpypwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x44,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d014440      <unknown>
+
 cpyprn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyprn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x84,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d018440      <unknown>
+
 cpypn [x0]!, [x1]!, x2!
+// CHECK-INST: cpypn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xc4,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d01c440      <unknown>
+
 cpypwt [x0]!, [x1]!, x2!
+// CHECK-INST: cpypwt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x14,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d011440      <unknown>
+
 cpypwtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpypwtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x54,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d015440      <unknown>
+
 cpypwtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpypwtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x94,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d019440      <unknown>
+
 cpypwtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpypwtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xd4,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d01d440      <unknown>
+
 cpyprt [x0]!, [x1]!, x2!
+// CHECK-INST: cpyprt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x24,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d012440      <unknown>
+
 cpyprtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyprtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x64,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d016440      <unknown>
+
 cpyprtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyprtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xa4,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d01a440      <unknown>
+
 cpyprtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyprtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xe4,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d01e440      <unknown>
+
 cpypt [x0]!, [x1]!, x2!
+// CHECK-INST: cpypt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x34,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d013440      <unknown>
+
 cpyptwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyptwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x74,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d017440      <unknown>
+
 cpyptrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyptrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xb4,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d01b440      <unknown>
+
 cpyptn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyptn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xf4,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d01f440      <unknown>
 
-// CHECK:      [0x40,0x04,0x41,0x1d]
-// CHECK-NEXT: [0x40,0x44,0x41,0x1d]
-// CHECK-NEXT: [0x40,0x84,0x41,0x1d]
-// CHECK-NEXT: [0x40,0xc4,0x41,0x1d]
-// CHECK-NEXT: [0x40,0x14,0x41,0x1d]
-// CHECK-NEXT: [0x40,0x54,0x41,0x1d]
-// CHECK-NEXT: [0x40,0x94,0x41,0x1d]
-// CHECK-NEXT: [0x40,0xd4,0x41,0x1d]
-// CHECK-NEXT: [0x40,0x24,0x41,0x1d]
-// CHECK-NEXT: [0x40,0x64,0x41,0x1d]
-// CHECK-NEXT: [0x40,0xa4,0x41,0x1d]
-// CHECK-NEXT: [0x40,0xe4,0x41,0x1d]
-// CHECK-NEXT: [0x40,0x34,0x41,0x1d]
-// CHECK-NEXT: [0x40,0x74,0x41,0x1d]
-// CHECK-NEXT: [0x40,0xb4,0x41,0x1d]
-// CHECK-NEXT: [0x40,0xf4,0x41,0x1d]
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
 cpym [x0]!, [x1]!, x2!
+// CHECK-INST: cpym [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x04,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d410440      <unknown>
+
 cpymwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x44,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d414440      <unknown>
+
 cpymrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x84,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d418440      <unknown>
+
 cpymn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xc4,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d41c440      <unknown>
+
 cpymwt [x0]!, [x1]!, x2!
+// CHECK-INST: cpymwt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x14,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d411440      <unknown>
+
 cpymwtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymwtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x54,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d415440      <unknown>
+
 cpymwtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymwtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x94,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d419440      <unknown>
+
 cpymwtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymwtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xd4,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d41d440      <unknown>
+
 cpymrt [x0]!, [x1]!, x2!
+// CHECK-INST: cpymrt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x24,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d412440      <unknown>
+
 cpymrtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymrtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x64,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d416440      <unknown>
+
 cpymrtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymrtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xa4,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d41a440      <unknown>
+
 cpymrtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymrtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xe4,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d41e440      <unknown>
+
 cpymt [x0]!, [x1]!, x2!
+// CHECK-INST: cpymt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x34,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d413440      <unknown>
+
 cpymtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x74,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d417440      <unknown>
+
 cpymtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xb4,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d41b440      <unknown>
+
 cpymtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpymtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xf4,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d41f440      <unknown>
 
-// CHECK:      [0x40,0x04,0x81,0x1d]
-// CHECK-NEXT: [0x40,0x44,0x81,0x1d]
-// CHECK-NEXT: [0x40,0x84,0x81,0x1d]
-// CHECK-NEXT: [0x40,0xc4,0x81,0x1d]
-// CHECK-NEXT: [0x40,0x14,0x81,0x1d]
-// CHECK-NEXT: [0x40,0x54,0x81,0x1d]
-// CHECK-NEXT: [0x40,0x94,0x81,0x1d]
-// CHECK-NEXT: [0x40,0xd4,0x81,0x1d]
-// CHECK-NEXT: [0x40,0x24,0x81,0x1d]
-// CHECK-NEXT: [0x40,0x64,0x81,0x1d]
-// CHECK-NEXT: [0x40,0xa4,0x81,0x1d]
-// CHECK-NEXT: [0x40,0xe4,0x81,0x1d]
-// CHECK-NEXT: [0x40,0x34,0x81,0x1d]
-// CHECK-NEXT: [0x40,0x74,0x81,0x1d]
-// CHECK-NEXT: [0x40,0xb4,0x81,0x1d]
-// CHECK-NEXT: [0x40,0xf4,0x81,0x1d]
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
 cpye [x0]!, [x1]!, x2!
+// CHECK-INST: cpye [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x04,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d810440      <unknown>
+
 cpyewn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyewn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x44,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d814440      <unknown>
+
 cpyern [x0]!, [x1]!, x2!
+// CHECK-INST: cpyern [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x84,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d818440      <unknown>
+
 cpyen [x0]!, [x1]!, x2!
+// CHECK-INST: cpyen [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xc4,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d81c440      <unknown>
+
 cpyewt [x0]!, [x1]!, x2!
+// CHECK-INST: cpyewt [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x14,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d811440      <unknown>
+
 cpyewtwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyewtwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x54,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d815440      <unknown>
+
 cpyewtrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyewtrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x94,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d819440      <unknown>
+
 cpyewtn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyewtn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xd4,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d81d440      <unknown>
+
 cpyert [x0]!, [x1]!, x2!
+// CHECK-INST: cpyert [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x24,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d812440      <unknown>
+
 cpyertwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyertwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x64,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d816440      <unknown>
+
 cpyertrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyertrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xa4,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d81a440      <unknown>
+
 cpyertn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyertn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xe4,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d81e440      <unknown>
+
 cpyet [x0]!, [x1]!, x2!
+// CHECK-INST: cpyet [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x34,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d813440      <unknown>
+
 cpyetwn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyetwn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0x74,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d817440      <unknown>
+
 cpyetrn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyetrn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xb4,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d81b440      <unknown>
+
 cpyetn [x0]!, [x1]!, x2!
+// CHECK-INST: cpyetn [x0]!, [x1]!, x2!
+// CHECK-ENCODING: encoding: [0x40,0xf4,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d81f440      <unknown>
 
-// CHECK:      [0x20,0x04,0xc2,0x19]
-// CHECK-NEXT: [0x20,0x14,0xc2,0x19]
-// CHECK-NEXT: [0x20,0x24,0xc2,0x19]
-// CHECK-NEXT: [0x20,0x34,0xc2,0x19]
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
 setp [x0]!, x1!, x2
+// CHECK-INST: setp [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x04,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c20420      <unknown>
+
 setpt [x0]!, x1!, x2
+// CHECK-INST: setpt [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x14,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c21420      <unknown>
+
 setpn [x0]!, x1!, x2
+// CHECK-INST: setpn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x24,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c22420      <unknown>
+
 setptn [x0]!, x1!, x2
+// CHECK-INST: setptn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x34,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c23420      <unknown>
 
-// CHECK: [0x20,0x44,0xc2,0x19]
-// CHECK: [0x20,0x54,0xc2,0x19]
-// CHECK: [0x20,0x64,0xc2,0x19]
-// CHECK: [0x20,0x74,0xc2,0x19]
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
 setm [x0]!, x1!, x2
+// CHECK-INST: setm [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x44,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c24420      <unknown>
+
 setmt [x0]!, x1!, x2
+// CHECK-INST: setmt [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x54,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c25420      <unknown>
+
 setmn [x0]!, x1!, x2
+// CHECK-INST: setmn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x64,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c26420      <unknown>
+
 setmtn [x0]!, x1!, x2
+// CHECK-INST: setmtn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x74,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c27420      <unknown>
 
-// CHECK: [0x20,0x84,0xc2,0x19]
-// CHECK: [0x20,0x94,0xc2,0x19]
-// CHECK: [0x20,0xa4,0xc2,0x19]
-// CHECK: [0x20,0xb4,0xc2,0x19]
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
-// CHECK-NO-MOPS-ERR: error: instruction requires: mops
 sete [x0]!, x1!, x2
+// CHECK-INST: sete [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x84,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c28420      <unknown>
+
 setet [x0]!, x1!, x2
+// CHECK-INST: setet [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x94,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c29420      <unknown>
+
 seten [x0]!, x1!, x2
+// CHECK-INST: seten [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0xa4,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c2a420      <unknown>
+
 setetn [x0]!, x1!, x2
+// CHECK-INST: setetn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0xb4,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c2b420      <unknown>
 
-// CHECK-MTE: [0x20,0x04,0xc2,0x1d]
-// CHECK-MTE: [0x20,0x14,0xc2,0x1d]
-// CHECK-MTE: [0x20,0x24,0xc2,0x1d]
-// CHECK-MTE: [0x20,0x34,0xc2,0x1d]
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
 setgp [x0]!, x1!, x2
+// CHECK-INST: setgp [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x04,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc20420      <unknown>
+
 setgpt [x0]!, x1!, x2
+// CHECK-INST: setgpt [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x14,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc21420      <unknown>
+
 setgpn [x0]!, x1!, x2
+// CHECK-INST: setgpn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x24,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc22420      <unknown>
+
 setgptn [x0]!, x1!, x2
+// CHECK-INST: setgptn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x34,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc23420      <unknown>
 
-// CHECK-MTE: [0x20,0x44,0xc2,0x1d]
-// CHECK-MTE: [0x20,0x54,0xc2,0x1d]
-// CHECK-MTE: [0x20,0x64,0xc2,0x1d]
-// CHECK-MTE: [0x20,0x74,0xc2,0x1d]
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
 setgm [x0]!, x1!, x2
+// CHECK-INST: setgm [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x44,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc24420      <unknown>
+
 setgmt [x0]!, x1!, x2
+// CHECK-INST: setgmt [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x54,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc25420      <unknown>
+
 setgmn [x0]!, x1!, x2
+// CHECK-INST: setgmn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x64,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc26420      <unknown>
+
 setgmtn [x0]!, x1!, x2
+// CHECK-INST: setgmtn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x74,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc27420      <unknown>
 
-// CHECK-MTE: [0x20,0x84,0xc2,0x1d]
-// CHECK-MTE: [0x20,0x94,0xc2,0x1d]
-// CHECK-MTE: [0x20,0xa4,0xc2,0x1d]
-// CHECK-MTE: [0x20,0xb4,0xc2,0x1d]
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MTE-ERR: error: instruction requires: mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
-// CHECK-NO-MOPSMTE-ERR: error: instruction requires: mops mte
 setge [x0]!, x1!, x2
+// CHECK-INST: setge [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x84,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc28420      <unknown>
+
 setget [x0]!, x1!, x2
+// CHECK-INST: setget [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0x94,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc29420      <unknown>
+
 setgen [x0]!, x1!, x2
-setgetn [x0]!, x1!, x2
+// CHECK-INST: setgen [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0xa4,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc2a420      <unknown>
 
-// All operand must be different from each other
-
-// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
-cpyfp [x0]!, [x0]!, x1!
-cpyfp [x0]!, [x1]!, x0!
-cpyfp [x1]!, [x0]!, x0!
-
-// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
-cpyfm [x0]!, [x0]!, x1!
-cpyfm [x0]!, [x1]!, x0!
-cpyfm [x1]!, [x0]!, x0!
-
-// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
-cpyfe [x0]!, [x0]!, x1!
-cpyfe [x0]!, [x1]!, x0!
-cpyfe [x1]!, [x0]!, x0!
-
-// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
-cpyp [x0]!, [x0]!, x1!
-cpyp [x0]!, [x1]!, x0!
-cpyp [x1]!, [x0]!, x0!
-
-// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
-cpym [x0]!, [x0]!, x1!
-cpym [x0]!, [x1]!, x0!
-cpym [x1]!, [x0]!, x0!
-
-// CHECK-ERROR: error: invalid CPY instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid CPY instruction, source and size registers are the same
-cpye [x0]!, [x0]!, x1!
-cpye [x0]!, [x1]!, x0!
-cpye [x1]!, [x0]!, x0!
-
-// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
-setp [x0]!, x0!, x1
-setp [x0]!, x1!, x0
-setp [x1]!, x0!, x0
-
-// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
-setm [x0]!, x0!, x1
-setm [x0]!, x1!, x0
-setm [x1]!, x0!, x0
-
-// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
-sete [x0]!, x0!, x1
-sete [x0]!, x1!, x0
-sete [x1]!, x0!, x0
-
-// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
-setgp [x0]!, x0!, x1
-setgp [x0]!, x1!, x0
-setgp [x1]!, x0!, x0
-
-// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
-setgm [x0]!, x0!, x1
-setgm [x0]!, x1!, x0
-setgm [x1]!, x0!, x0
-
-// CHECK-ERROR: error: invalid SET instruction, destination and size registers are the same
-// CHECK-ERROR: error: invalid SET instruction, destination and source registers are the same
-// CHECK-ERROR: error: invalid SET instruction, source and size registers are the same
-setge [x0]!, x0!, x1
-setge [x0]!, x1!, x0
-setge [x1]!, x0!, x0
-
-// SP cannot be used as argument at any position
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-cpyfp [sp]!, [x1]!, x2!
-cpyfp [x0]!, [sp]!, x2!
-cpyfp [x0]!, [x1]!, sp!
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-cpyfm [sp]!, [x1]!, x2!
-cpyfm [x0]!, [sp]!, x2!
-cpyfm [x0]!, [x1]!, sp!
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-cpyfe [sp]!, [x1]!, x2!
-cpyfe [x0]!, [sp]!, x2!
-cpyfe [x0]!, [x1]!, sp!
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-cpyp [sp]!, [x2]!, x2!
-cpyp [x0]!, [sp]!, x2!
-cpyp [x0]!, [x1]!, sp!
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-cpym [sp]!, [x2]!, x2!
-cpym [x0]!, [sp]!, x2!
-cpym [x0]!, [x1]!, sp!
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-cpye [sp]!, [x2]!, x2!
-cpye [x0]!, [sp]!, x2!
-cpye [x0]!, [x1]!, sp!
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-setp [sp]!, x1!, x2
-setp [x0]!, sp!, x2
-setp [x0]!, x1!, sp
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-setm [sp]!, x1!, x2
-setm [x0]!, sp!, x2
-setm [x0]!, x1!, sp
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-sete [sp]!, x1!, x2
-sete [x0]!, sp!, x2
-sete [x0]!, x1!, sp
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-setgp [sp]!, x1!, x2
-setgp [x0]!, sp!, x2
-setgp [x0]!, x1!, sp
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-setgm [sp]!, x1!, x2
-setgm [x0]!, sp!, x2
-setgm [x0]!, x1!, sp
-
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-setge [sp]!, x1!, x2
-setge [x0]!, sp!, x2
-setge [x0]!, x1!, sp
+setgetn [x0]!, x1!, x2
+// CHECK-INST: setgetn [x0]!, x1!, x2
+// CHECK-ENCODING: encoding: [0x20,0xb4,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops mte
+// CHECK-UNKNOWN:  1dc2b420      <unknown>
 
 // XZR can only be used at:
 //  - the size operand in CPY.
 //  - the size or source operands in SET.
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK:       cpyfp [x0]!, [x1]!, xzr!
-cpyfp [xzr]!, [x1]!, x2!
-cpyfp [x0]!, [xzr]!, x2!
 cpyfp [x0]!, [x1]!, xzr!
+// CHECK-INST: cpyfp [x0]!, [x1]!, xzr!
+// CHECK-ENCODING: encoding: [0xe0,0x07,0x01,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  190107e0      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK:       cpyfm [x0]!, [x1]!, xzr!
-cpyfm [xzr]!, [x1]!, x2!
-cpyfm [x0]!, [xzr]!, x2!
 cpyfm [x0]!, [x1]!, xzr!
+// CHECK-INST: cpyfm [x0]!, [x1]!, xzr!
+// CHECK-ENCODING: encoding: [0xe0,0x07,0x41,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  194107e0      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK:       cpyfe [x0]!, [x1]!, xzr!
-cpyfe [xzr]!, [x1]!, x2!
-cpyfe [x0]!, [xzr]!, x2!
 cpyfe [x0]!, [x1]!, xzr!
+// CHECK-INST: cpyfe [x0]!, [x1]!, xzr!
+// CHECK-ENCODING: encoding: [0xe0,0x07,0x81,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  198107e0      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK:       cpyp [x0]!, [x1]!, xzr!
-cpyp [xzr]!, [x2]!, x2!
-cpyp [x0]!, [xzr]!, x2!
 cpyp [x0]!, [x1]!, xzr!
+// CHECK-INST: cpyp [x0]!, [x1]!, xzr!
+// CHECK-ENCODING: encoding: [0xe0,0x07,0x01,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d0107e0      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK:       cpym [x0]!, [x1]!, xzr!
-cpym [xzr]!, [x2]!, x2!
-cpym [x0]!, [xzr]!, x2!
 cpym [x0]!, [x1]!, xzr!
+// CHECK-INST: cpym [x0]!, [x1]!, xzr!
+// CHECK-ENCODING: encoding: [0xe0,0x07,0x41,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d4107e0      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK:       cpye [x0]!, [x1]!, xzr!
-cpye [xzr]!, [x2]!, x2!
-cpye [x0]!, [xzr]!, x2!
 cpye [x0]!, [x1]!, xzr!
+// CHECK-INST: cpye [x0]!, [x1]!, xzr!
+// CHECK-ENCODING: encoding: [0xe0,0x07,0x81,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1d8107e0      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK:       setp [x0]!, xzr!, x2
-// CHECK:       setp [x0]!, x1!, xzr
-setp [xzr]!, x1!, x2
 setp [x0]!, xzr!, x2
+// CHECK-INST: setp [x0]!, xzr!, x2
+// CHECK-ENCODING: encoding: [0xe0,0x07,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c207e0      <unknown>
+
 setp [x0]!, x1!, xzr
+// CHECK-INST: setp [x0]!, x1!, xzr
+// CHECK-ENCODING: encoding: [0x20,0x04,0xdf,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19df0420      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK:       setm [x0]!, xzr!, x2
-// CHECK:       setm [x0]!, x1!, xzr
-setm [xzr]!, x1!, x2
 setm [x0]!, xzr!, x2
+// CHECK-INST: setm [x0]!, xzr!, x2
+// CHECK-ENCODING: encoding: [0xe0,0x47,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c247e0      <unknown>
+
 setm [x0]!, x1!, xzr
+// CHECK-INST: setm [x0]!, x1!, xzr
+// CHECK-ENCODING: encoding: [0x20,0x44,0xdf,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19df4420      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK:       sete [x0]!, xzr!, x2
-// CHECK:       sete [x0]!, x1!, xzr
-sete [xzr]!, x1!, x2
 sete [x0]!, xzr!, x2
+// CHECK-INST: sete [x0]!, xzr!, x2
+// CHECK-ENCODING: encoding: [0xe0,0x87,0xc2,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19c287e0      <unknown>
+
 sete [x0]!, x1!, xzr
+// CHECK-INST: sete [x0]!, x1!, xzr
+// CHECK-ENCODING: encoding: [0x20,0x84,0xdf,0x19]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  19df8420      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-MTE:   setgp [x0]!, xzr!, x2
-// CHECK-MTE:   setgp [x0]!, x1!, xzr
-setgp [xzr]!, x1!, x2
 setgp [x0]!, xzr!, x2
+// CHECK-INST: setgp [x0]!, xzr!, x2
+// CHECK-ENCODING: encoding: [0xe0,0x07,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1dc207e0      <unknown>
+
 setgp [x0]!, x1!, xzr
+// CHECK-INST: setgp [x0]!, x1!, xzr
+// CHECK-ENCODING: encoding: [0x20,0x04,0xdf,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1ddf0420      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-MTE:   setgm [x0]!, xzr!, x2
-// CHECK-MTE:   setgm [x0]!, x1!, xzr
-setgm [xzr]!, x1!, x2
 setgm [x0]!, xzr!, x2
+// CHECK-INST: setgm [x0]!, xzr!, x2
+// CHECK-ENCODING: encoding: [0xe0,0x47,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1dc247e0      <unknown>
+
 setgm [x0]!, x1!, xzr
+// CHECK-INST: setgm [x0]!, x1!, xzr
+// CHECK-ENCODING: encoding: [0x20,0x44,0xdf,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1ddf4420      <unknown>
 
-// CHECK-ERROR: error: invalid operand for instruction
-// CHECK-MTE:   setge [x0]!, xzr!, x2
-// CHECK-MTE:   setge [x0]!, x1!, xzr
-setge [xzr]!, x1!, x2
 setge [x0]!, xzr!, x2
+// CHECK-INST: setge [x0]!, xzr!, x2
+// CHECK-ENCODING: encoding: [0xe0,0x87,0xc2,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1dc287e0      <unknown>
+
 setge [x0]!, x1!, xzr
+// CHECK-INST: setge [x0]!, x1!, xzr
+// CHECK-ENCODING: encoding: [0x20,0x84,0xdf,0x1d]
+// CHECK-ERROR: error: instruction requires: mops
+// CHECK-UNKNOWN:  1ddf8420      <unknown>
diff --git a/llvm/test/MC/AArch64/armv8.9a-ats1a.s b/llvm/test/MC/AArch64/armv8.9a-ats1a.s
index a30d206639425..21b960e55d620 100644
--- a/llvm/test/MC/AArch64/armv8.9a-ats1a.s
+++ b/llvm/test/MC/AArch64/armv8.9a-ats1a.s
@@ -1,10 +1,26 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding %s | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefixes=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 at s1e1a, x1
-// CHECK: at s1e1a, x1                        // encoding: [0x41,0x79,0x08,0xd5]
+// CHECK-INST: at s1e1a, x1
+// CHECK-ENCODING: encoding: [0x41,0x79,0x08,0xd5]
+// CHECK-UNKNOWN:  d5087941 at s1e1a, x1
 
 at s1e2a, x1
-// CHECK: at s1e2a, x1                        // encoding: [0x41,0x79,0x0c,0xd5]
+// CHECK-INST: at s1e2a, x1
+// CHECK-ENCODING: encoding: [0x41,0x79,0x0c,0xd5]
+// CHECK-UNKNOWN:  d50c7941 at s1e2a, x1
 
 at s1e3a, x1
-// CHECK: at s1e3a, x1                        // encoding: [0x41,0x79,0x0e,0xd5]
+// CHECK-INST: at s1e3a, x1
+// CHECK-ENCODING: encoding: [0x41,0x79,0x0e,0xd5]
+// CHECK-UNKNOWN:  d50e7941 at s1e3a, x1
diff --git a/llvm/test/MC/AArch64/armv8.9a-clrbhb.s b/llvm/test/MC/AArch64/armv8.9a-clrbhb.s
index 96de61f50f584..9b5bfec86185e 100644
--- a/llvm/test/MC/AArch64/armv8.9a-clrbhb.s
+++ b/llvm/test/MC/AArch64/armv8.9a-clrbhb.s
@@ -2,42 +2,56 @@
 // Assembly is always permitted for instructions in the hint space.
 
 // Optional, off by default
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf < %s | FileCheck %s --check-prefix=HINT_22
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v8a < %s | FileCheck %s --check-prefix=HINT_22
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v8.8a < %s | FileCheck %s --check-prefix=HINT_22
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v9a < %s | FileCheck %s --check-prefix=HINT_22
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v9.3a < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v8a < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v8.8a < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v9a < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v9.3a < %s | FileCheck %s --check-prefix=HINT_22
 
 // Optional, off by default, doubly disabled
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v8a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v8.8a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v9a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v9.3a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v8a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v8.8a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v9a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v9.3a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
 
 // Optional, off by default, manually enabled
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+clrbhb < %s | FileCheck %s --check-prefix=CLRBHB
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v8a,+clrbhb < %s | FileCheck %s --check-prefix=CLRBHB
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v8.8a,+clrbhb < %s | FileCheck %s --check-prefix=CLRBHB
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v9a,+clrbhb < %s | FileCheck %s --check-prefix=CLRBHB
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v9.3a,+clrbhb < %s | FileCheck %s --check-prefix=CLRBHB
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+clrbhb < %s | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v8a,+clrbhb < %s | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v8.8a,+clrbhb < %s | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v9a,+clrbhb < %s | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v9.3a,+clrbhb < %s | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 
 // Mandatory, enabled by default
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v8.9a < %s | FileCheck %s --check-prefix=CLRBHB
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v9.4a < %s | FileCheck %s --check-prefix=CLRBHB
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v8.9a < %s | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v9.4a < %s | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 
 // Mandatory, on by default, doubly enabled
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v8.9a,+clrbhb < %s | FileCheck %s --check-prefix=CLRBHB
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v9.4a,+clrbhb < %s | FileCheck %s --check-prefix=CLRBHB
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v8.9a,+clrbhb < %s | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v9.4a,+clrbhb < %s | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 
 // Mandatory, can't prevent disabling in LLVM
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v8.9a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
-// RUN: llvm-mc -show-encoding -triple aarch64-none-elf -mattr=+v9.4a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
-
-        clrbhb
-        hint #22
-
-// CLRBHB: clrbhb    // encoding: [0xdf,0x22,0x03,0xd5]
-// CLRBHB: clrbhb    // encoding: [0xdf,0x22,0x03,0xd5]
-// HINT_22: hint #22 // encoding: [0xdf,0x22,0x03,0xd5]
-// HINT_22: hint #22 // encoding: [0xdf,0x22,0x03,0xd5]
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v8.9a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
+// RUN: llvm-mc -show-encoding -triple aarch64 -mattr=+v9.4a,-clrbhb < %s | FileCheck %s --check-prefix=HINT_22
+
+// Check Unknown
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+clrbhb < %s \
+// RUN:   | llvm-objdump -d --mattr=-clrbhb --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+clrbhb < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+clrbhb -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+clrbhb
+// HINT_22: hint #22                             // encoding: [0xdf,0x22,0x03,0xd5]
+// CHECK-INST: clrbhb
+// CHECK-ENCODING: encoding: [0xdf,0x22,0x03,0xd5]
+// CHECK-UNKNOWN:  d50322df    hint #22
+
+hint #22
+// HINT_22: hint #22                             // encoding: [0xdf,0x22,0x03,0xd5]
+// CHECK-INST: clrbhb
+// CHECK-ENCODING: encoding: [0xdf,0x22,0x03,0xd5]
+// CHECK-UNKNOWN:  d50322df    hint #22
diff --git a/llvm/test/MC/AArch64/armv8.9a-debug-pmu.s b/llvm/test/MC/AArch64/armv8.9a-debug-pmu.s
index 0b74905c27fb6..db5cb0701036d 100644
--- a/llvm/test/MC/AArch64/armv8.9a-debug-pmu.s
+++ b/llvm/test/MC/AArch64/armv8.9a-debug-pmu.s
@@ -1,485 +1,1785 @@
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding               -mattr=+ite < %s | FileCheck %s
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.8a -mattr=+ite < %s | FileCheck %s
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.9a -mattr=+ite < %s | FileCheck %s
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v9.3a -mattr=+ite < %s | FileCheck %s
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v9.4a -mattr=+ite < %s | FileCheck %s
-
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding               < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-ITE %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.8a < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-ITE %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.9a < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-ITE %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v9.3a < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-ITE %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v9.4a < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-ITE %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v8.8a,+ite < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v8.9a,+ite < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v9.3a,+ite < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v9.4a,+ite < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+ite < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+ite < %s \
+// RUN:        | llvm-objdump -d --mattr=+ite --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+ite < %s \
+// RUN:   | llvm-objdump -d --mattr=-ite --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+ite < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+ite -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+mrs x3, DBGBVR0_EL1
+// CHECK-INST: mrs x3, DBGBVR0_EL1
+// CHECK-ENCODING: encoding: [0x83,0x00,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300083 mrs x3, DBGBVR0_EL1
+
+msr DBGBVR0_EL1, x1
+// CHECK-INST: msr DBGBVR0_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x00,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100081 msr DBGBVR0_EL1, x1
+
+mrs x3, DBGBVR1_EL1
+// CHECK-INST: mrs x3, DBGBVR1_EL1
+// CHECK-ENCODING: encoding: [0x83,0x01,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300183 mrs x3, DBGBVR1_EL1
+
+msr DBGBVR1_EL1, x1
+// CHECK-INST: msr DBGBVR1_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x01,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100181 msr DBGBVR1_EL1, x1
+
+mrs x3, DBGBVR2_EL1
+// CHECK-INST: mrs x3, DBGBVR2_EL1
+// CHECK-ENCODING: encoding: [0x83,0x02,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300283 mrs x3, DBGBVR2_EL1
+
+msr DBGBVR2_EL1, x1
+// CHECK-INST: msr DBGBVR2_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x02,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100281 msr DBGBVR2_EL1, x1
+
+mrs x3, DBGBVR3_EL1
+// CHECK-INST: mrs x3, DBGBVR3_EL1
+// CHECK-ENCODING: encoding: [0x83,0x03,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300383 mrs x3, DBGBVR3_EL1
+
+msr DBGBVR3_EL1, x1
+// CHECK-INST: msr DBGBVR3_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x03,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100381 msr DBGBVR3_EL1, x1
+
+mrs x3, DBGBVR4_EL1
+// CHECK-INST: mrs x3, DBGBVR4_EL1
+// CHECK-ENCODING: encoding: [0x83,0x04,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300483 mrs x3, DBGBVR4_EL1
+
+msr DBGBVR4_EL1, x1
+// CHECK-INST: msr DBGBVR4_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x04,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100481 msr DBGBVR4_EL1, x1
+
+mrs x3, DBGBVR5_EL1
+// CHECK-INST: mrs x3, DBGBVR5_EL1
+// CHECK-ENCODING: encoding: [0x83,0x05,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300583 mrs x3, DBGBVR5_EL1
+
+msr DBGBVR5_EL1, x1
+// CHECK-INST: msr DBGBVR5_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x05,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100581 msr DBGBVR5_EL1, x1
+
+mrs x3, DBGBVR6_EL1
+// CHECK-INST: mrs x3, DBGBVR6_EL1
+// CHECK-ENCODING: encoding: [0x83,0x06,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300683 mrs x3, DBGBVR6_EL1
+
+msr DBGBVR6_EL1, x1
+// CHECK-INST: msr DBGBVR6_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x06,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100681 msr DBGBVR6_EL1, x1
+
+mrs x3, DBGBVR7_EL1
+// CHECK-INST: mrs x3, DBGBVR7_EL1
+// CHECK-ENCODING: encoding: [0x83,0x07,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300783 mrs x3, DBGBVR7_EL1
+
+msr DBGBVR7_EL1, x1
+// CHECK-INST: msr DBGBVR7_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x07,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100781 msr DBGBVR7_EL1, x1
+
+mrs x3, DBGBVR8_EL1
+// CHECK-INST: mrs x3, DBGBVR8_EL1
+// CHECK-ENCODING: encoding: [0x83,0x08,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300883 mrs x3, DBGBVR8_EL1
+
+msr DBGBVR8_EL1, x1
+// CHECK-INST: msr DBGBVR8_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x08,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100881 msr DBGBVR8_EL1, x1
+
+mrs x3, DBGBVR9_EL1
+// CHECK-INST: mrs x3, DBGBVR9_EL1
+// CHECK-ENCODING: encoding: [0x83,0x09,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300983 mrs x3, DBGBVR9_EL1
+
+msr DBGBVR9_EL1, x1
+// CHECK-INST: msr DBGBVR9_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x09,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100981 msr DBGBVR9_EL1, x1
+
+mrs x3, DBGBVR10_EL1
+// CHECK-INST: mrs x3, DBGBVR10_EL1
+// CHECK-ENCODING: encoding: [0x83,0x0a,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300a83 mrs x3, DBGBVR10_EL1
+
+msr DBGBVR10_EL1, x1
+// CHECK-INST: msr DBGBVR10_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x0a,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100a81 msr DBGBVR10_EL1, x1
+
+mrs x3, DBGBVR11_EL1
+// CHECK-INST: mrs x3, DBGBVR11_EL1
+// CHECK-ENCODING: encoding: [0x83,0x0b,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300b83 mrs x3, DBGBVR11_EL1
+
+msr DBGBVR11_EL1, x1
+// CHECK-INST: msr DBGBVR11_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x0b,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100b81 msr DBGBVR11_EL1, x1
+
+mrs x3, DBGBVR12_EL1
+// CHECK-INST: mrs x3, DBGBVR12_EL1
+// CHECK-ENCODING: encoding: [0x83,0x0c,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300c83 mrs x3, DBGBVR12_EL1
+
+msr DBGBVR12_EL1, x1
+// CHECK-INST: msr DBGBVR12_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x0c,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100c81 msr DBGBVR12_EL1, x1
+
+mrs x3, DBGBVR13_EL1
+// CHECK-INST: mrs x3, DBGBVR13_EL1
+// CHECK-ENCODING: encoding: [0x83,0x0d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300d83 mrs x3, DBGBVR13_EL1
+
+msr DBGBVR13_EL1, x1
+// CHECK-INST: msr DBGBVR13_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x0d,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100d81 msr DBGBVR13_EL1, x1
+
+mrs x3, DBGBVR14_EL1
+// CHECK-INST: mrs x3, DBGBVR14_EL1
+// CHECK-ENCODING: encoding: [0x83,0x0e,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300e83 mrs x3, DBGBVR14_EL1
+
+msr DBGBVR14_EL1, x1
+// CHECK-INST: msr DBGBVR14_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x0e,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100e81 msr DBGBVR14_EL1, x1
+
+mrs x3, DBGBVR15_EL1
+// CHECK-INST: mrs x3, DBGBVR15_EL1
+// CHECK-ENCODING: encoding: [0x83,0x0f,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300f83 mrs x3, DBGBVR15_EL1
+
+msr DBGBVR15_EL1, x1
+// CHECK-INST: msr DBGBVR15_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x0f,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100f81 msr DBGBVR15_EL1, x1
+
+mrs x3, DBGBCR0_EL1
+// CHECK-INST: mrs x3, DBGBCR0_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x00,0x30,0xd5]
+// CHECK-UNKNOWN:  d53000a3 mrs x3, DBGBCR0_EL1
+
+msr DBGBCR0_EL1, x1
+// CHECK-INST: msr DBGBCR0_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x00,0x10,0xd5]
+// CHECK-UNKNOWN:  d51000a1 msr DBGBCR0_EL1, x1
+
+mrs x3, DBGBCR1_EL1
+// CHECK-INST: mrs x3, DBGBCR1_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x01,0x30,0xd5]
+// CHECK-UNKNOWN:  d53001a3 mrs x3, DBGBCR1_EL1
+
+msr DBGBCR1_EL1, x1
+// CHECK-INST: msr DBGBCR1_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x01,0x10,0xd5]
+// CHECK-UNKNOWN:  d51001a1 msr DBGBCR1_EL1, x1
+
+mrs x3, DBGBCR2_EL1
+// CHECK-INST: mrs x3, DBGBCR2_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x02,0x30,0xd5]
+// CHECK-UNKNOWN:  d53002a3 mrs x3, DBGBCR2_EL1
+
+msr DBGBCR2_EL1, x1
+// CHECK-INST: msr DBGBCR2_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x02,0x10,0xd5]
+// CHECK-UNKNOWN:  d51002a1 msr DBGBCR2_EL1, x1
+
+mrs x3, DBGBCR3_EL1
+// CHECK-INST: mrs x3, DBGBCR3_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x03,0x30,0xd5]
+// CHECK-UNKNOWN:  d53003a3 mrs x3, DBGBCR3_EL1
+
+msr DBGBCR3_EL1, x1
+// CHECK-INST: msr DBGBCR3_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x03,0x10,0xd5]
+// CHECK-UNKNOWN:  d51003a1 msr DBGBCR3_EL1, x1
+
+mrs x3, DBGBCR4_EL1
+// CHECK-INST: mrs x3, DBGBCR4_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x04,0x30,0xd5]
+// CHECK-UNKNOWN:  d53004a3 mrs x3, DBGBCR4_EL1
+
+msr DBGBCR4_EL1, x1
+// CHECK-INST: msr DBGBCR4_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x04,0x10,0xd5]
+// CHECK-UNKNOWN:  d51004a1 msr DBGBCR4_EL1, x1
+
+mrs x3, DBGBCR5_EL1
+// CHECK-INST: mrs x3, DBGBCR5_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x05,0x30,0xd5]
+// CHECK-UNKNOWN:  d53005a3 mrs x3, DBGBCR5_EL1
+
+msr DBGBCR5_EL1, x1
+// CHECK-INST: msr DBGBCR5_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x05,0x10,0xd5]
+// CHECK-UNKNOWN:  d51005a1 msr DBGBCR5_EL1, x1
+
+mrs x3, DBGBCR6_EL1
+// CHECK-INST: mrs x3, DBGBCR6_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x06,0x30,0xd5]
+// CHECK-UNKNOWN:  d53006a3 mrs x3, DBGBCR6_EL1
+
+msr DBGBCR6_EL1, x1
+// CHECK-INST: msr DBGBCR6_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x06,0x10,0xd5]
+// CHECK-UNKNOWN:  d51006a1 msr DBGBCR6_EL1, x1
+
+mrs x3, DBGBCR7_EL1
+// CHECK-INST: mrs x3, DBGBCR7_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x07,0x30,0xd5]
+// CHECK-UNKNOWN:  d53007a3 mrs x3, DBGBCR7_EL1
+
+msr DBGBCR7_EL1, x1
+// CHECK-INST: msr DBGBCR7_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x07,0x10,0xd5]
+// CHECK-UNKNOWN:  d51007a1 msr DBGBCR7_EL1, x1
+
+mrs x3, DBGBCR8_EL1
+// CHECK-INST: mrs x3, DBGBCR8_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x08,0x30,0xd5]
+// CHECK-UNKNOWN:  d53008a3 mrs x3, DBGBCR8_EL1
+
+msr DBGBCR8_EL1, x1
+// CHECK-INST: msr DBGBCR8_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x08,0x10,0xd5]
+// CHECK-UNKNOWN:  d51008a1 msr DBGBCR8_EL1, x1
+
+mrs x3, DBGBCR9_EL1
+// CHECK-INST: mrs x3, DBGBCR9_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x09,0x30,0xd5]
+// CHECK-UNKNOWN:  d53009a3 mrs x3, DBGBCR9_EL1
+
+msr DBGBCR9_EL1, x1
+// CHECK-INST: msr DBGBCR9_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x09,0x10,0xd5]
+// CHECK-UNKNOWN:  d51009a1 msr DBGBCR9_EL1, x1
+
+mrs x3, DBGBCR10_EL1
+// CHECK-INST: mrs x3, DBGBCR10_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x0a,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300aa3 mrs x3, DBGBCR10_EL1
+
+msr DBGBCR10_EL1, x1
+// CHECK-INST: msr DBGBCR10_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x0a,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100aa1 msr DBGBCR10_EL1, x1
+
+mrs x3, DBGBCR11_EL1
+// CHECK-INST: mrs x3, DBGBCR11_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x0b,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300ba3 mrs x3, DBGBCR11_EL1
+
+msr DBGBCR11_EL1, x1
+// CHECK-INST: msr DBGBCR11_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x0b,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100ba1 msr DBGBCR11_EL1, x1
+
+mrs x3, DBGBCR12_EL1
+// CHECK-INST: mrs x3, DBGBCR12_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x0c,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300ca3 mrs x3, DBGBCR12_EL1
+
+msr DBGBCR12_EL1, x1
+// CHECK-INST: msr DBGBCR12_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x0c,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100ca1 msr DBGBCR12_EL1, x1
+
+mrs x3, DBGBCR13_EL1
+// CHECK-INST: mrs x3, DBGBCR13_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x0d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300da3 mrs x3, DBGBCR13_EL1
+
+msr DBGBCR13_EL1, x1
+// CHECK-INST: msr DBGBCR13_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x0d,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100da1 msr DBGBCR13_EL1, x1
+
+mrs x3, DBGBCR14_EL1
+// CHECK-INST: mrs x3, DBGBCR14_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x0e,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300ea3 mrs x3, DBGBCR14_EL1
+
+msr DBGBCR14_EL1, x1
+// CHECK-INST: msr DBGBCR14_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x0e,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100ea1 msr DBGBCR14_EL1, x1
+
+mrs x3, DBGBCR15_EL1
+// CHECK-INST: mrs x3, DBGBCR15_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x0f,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300fa3 mrs x3, DBGBCR15_EL1
+
+msr DBGBCR15_EL1, x1
+// CHECK-INST: msr DBGBCR15_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x0f,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100fa1 msr DBGBCR15_EL1, x1
+
+mrs x3, DBGWVR0_EL1
+// CHECK-INST: mrs x3, DBGWVR0_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x00,0x30,0xd5]
+// CHECK-UNKNOWN:  d53000c3 mrs x3, DBGWVR0_EL1
+
+msr DBGWVR0_EL1, x1
+// CHECK-INST: msr DBGWVR0_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x00,0x10,0xd5]
+// CHECK-UNKNOWN:  d51000c1 msr DBGWVR0_EL1, x1
+
+mrs x3, DBGWVR1_EL1
+// CHECK-INST: mrs x3, DBGWVR1_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x01,0x30,0xd5]
+// CHECK-UNKNOWN:  d53001c3 mrs x3, DBGWVR1_EL1
+
+msr DBGWVR1_EL1, x1
+// CHECK-INST: msr DBGWVR1_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x01,0x10,0xd5]
+// CHECK-UNKNOWN:  d51001c1 msr DBGWVR1_EL1, x1
+
+mrs x3, DBGWVR2_EL1
+// CHECK-INST: mrs x3, DBGWVR2_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x02,0x30,0xd5]
+// CHECK-UNKNOWN:  d53002c3 mrs x3, DBGWVR2_EL1
+
+msr DBGWVR2_EL1, x1
+// CHECK-INST: msr DBGWVR2_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x02,0x10,0xd5]
+// CHECK-UNKNOWN:  d51002c1 msr DBGWVR2_EL1, x1
+
+mrs x3, DBGWVR3_EL1
+// CHECK-INST: mrs x3, DBGWVR3_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x03,0x30,0xd5]
+// CHECK-UNKNOWN:  d53003c3 mrs x3, DBGWVR3_EL1
+
+msr DBGWVR3_EL1, x1
+// CHECK-INST: msr DBGWVR3_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x03,0x10,0xd5]
+// CHECK-UNKNOWN:  d51003c1 msr DBGWVR3_EL1, x1
+
+mrs x3, DBGWVR4_EL1
+// CHECK-INST: mrs x3, DBGWVR4_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x04,0x30,0xd5]
+// CHECK-UNKNOWN:  d53004c3 mrs x3, DBGWVR4_EL1
+
+msr DBGWVR4_EL1, x1
+// CHECK-INST: msr DBGWVR4_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x04,0x10,0xd5]
+// CHECK-UNKNOWN:  d51004c1 msr DBGWVR4_EL1, x1
+
+mrs x3, DBGWVR5_EL1
+// CHECK-INST: mrs x3, DBGWVR5_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x05,0x30,0xd5]
+// CHECK-UNKNOWN:  d53005c3 mrs x3, DBGWVR5_EL1
+
+msr DBGWVR5_EL1, x1
+// CHECK-INST: msr DBGWVR5_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x05,0x10,0xd5]
+// CHECK-UNKNOWN:  d51005c1 msr DBGWVR5_EL1, x1
+
+mrs x3, DBGWVR6_EL1
+// CHECK-INST: mrs x3, DBGWVR6_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x06,0x30,0xd5]
+// CHECK-UNKNOWN:  d53006c3 mrs x3, DBGWVR6_EL1
+
+msr DBGWVR6_EL1, x1
+// CHECK-INST: msr DBGWVR6_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x06,0x10,0xd5]
+// CHECK-UNKNOWN:  d51006c1 msr DBGWVR6_EL1, x1
+
+mrs x3, DBGWVR7_EL1
+// CHECK-INST: mrs x3, DBGWVR7_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x07,0x30,0xd5]
+// CHECK-UNKNOWN:  d53007c3 mrs x3, DBGWVR7_EL1
+
+msr DBGWVR7_EL1, x1
+// CHECK-INST: msr DBGWVR7_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x07,0x10,0xd5]
+// CHECK-UNKNOWN:  d51007c1 msr DBGWVR7_EL1, x1
+
+mrs x3, DBGWVR8_EL1
+// CHECK-INST: mrs x3, DBGWVR8_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x08,0x30,0xd5]
+// CHECK-UNKNOWN:  d53008c3 mrs x3, DBGWVR8_EL1
+
+msr DBGWVR8_EL1, x1
+// CHECK-INST: msr DBGWVR8_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x08,0x10,0xd5]
+// CHECK-UNKNOWN:  d51008c1 msr DBGWVR8_EL1, x1
+
+mrs x3, DBGWVR9_EL1
+// CHECK-INST: mrs x3, DBGWVR9_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x09,0x30,0xd5]
+// CHECK-UNKNOWN:  d53009c3 mrs x3, DBGWVR9_EL1
+
+msr DBGWVR9_EL1, x1
+// CHECK-INST: msr DBGWVR9_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x09,0x10,0xd5]
+// CHECK-UNKNOWN:  d51009c1 msr DBGWVR9_EL1, x1
+
+mrs x3, DBGWVR10_EL1
+// CHECK-INST: mrs x3, DBGWVR10_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x0a,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300ac3 mrs x3, DBGWVR10_EL1
+
+msr DBGWVR10_EL1, x1
+// CHECK-INST: msr DBGWVR10_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x0a,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100ac1 msr DBGWVR10_EL1, x1
+
+mrs x3, DBGWVR11_EL1
+// CHECK-INST: mrs x3, DBGWVR11_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x0b,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300bc3 mrs x3, DBGWVR11_EL1
+
+msr DBGWVR11_EL1, x1
+// CHECK-INST: msr DBGWVR11_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x0b,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100bc1 msr DBGWVR11_EL1, x1
+
+mrs x3, DBGWVR12_EL1
+// CHECK-INST: mrs x3, DBGWVR12_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x0c,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300cc3 mrs x3, DBGWVR12_EL1
+
+msr DBGWVR12_EL1, x1
+// CHECK-INST: msr DBGWVR12_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x0c,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100cc1 msr DBGWVR12_EL1, x1
+
+mrs x3, DBGWVR13_EL1
+// CHECK-INST: mrs x3, DBGWVR13_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x0d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300dc3 mrs x3, DBGWVR13_EL1
+
+msr DBGWVR13_EL1, x1
+// CHECK-INST: msr DBGWVR13_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x0d,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100dc1 msr DBGWVR13_EL1, x1
+
+mrs x3, DBGWVR14_EL1
+// CHECK-INST: mrs x3, DBGWVR14_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x0e,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300ec3 mrs x3, DBGWVR14_EL1
+
+msr DBGWVR14_EL1, x1
+// CHECK-INST: msr DBGWVR14_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x0e,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100ec1 msr DBGWVR14_EL1, x1
+
+mrs x3, DBGWVR15_EL1
+// CHECK-INST: mrs x3, DBGWVR15_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x0f,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300fc3 mrs x3, DBGWVR15_EL1
+
+msr DBGWVR15_EL1, x1
+// CHECK-INST: msr DBGWVR15_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0x0f,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100fc1 msr DBGWVR15_EL1, x1
+
+mrs x3, DBGWCR0_EL1
+// CHECK-INST: mrs x3, DBGWCR0_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x00,0x30,0xd5]
+// CHECK-UNKNOWN:  d53000e3 mrs x3, DBGWCR0_EL1
+
+msr DBGWCR0_EL1, x1
+// CHECK-INST: msr DBGWCR0_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x00,0x10,0xd5]
+// CHECK-UNKNOWN:  d51000e1 msr DBGWCR0_EL1, x1
+
+mrs x3, DBGWCR1_EL1
+// CHECK-INST: mrs x3, DBGWCR1_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x01,0x30,0xd5]
+// CHECK-UNKNOWN:  d53001e3 mrs x3, DBGWCR1_EL1
+
+msr DBGWCR1_EL1, x1
+// CHECK-INST: msr DBGWCR1_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x01,0x10,0xd5]
+// CHECK-UNKNOWN:  d51001e1 msr DBGWCR1_EL1, x1
+
+mrs x3, DBGWCR2_EL1
+// CHECK-INST: mrs x3, DBGWCR2_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x02,0x30,0xd5]
+// CHECK-UNKNOWN:  d53002e3 mrs x3, DBGWCR2_EL1
+
+msr DBGWCR2_EL1, x1
+// CHECK-INST: msr DBGWCR2_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x02,0x10,0xd5]
+// CHECK-UNKNOWN:  d51002e1 msr DBGWCR2_EL1, x1
+
+mrs x3, DBGWCR3_EL1
+// CHECK-INST: mrs x3, DBGWCR3_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x03,0x30,0xd5]
+// CHECK-UNKNOWN:  d53003e3 mrs x3, DBGWCR3_EL1
+
+msr DBGWCR3_EL1, x1
+// CHECK-INST: msr DBGWCR3_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x03,0x10,0xd5]
+// CHECK-UNKNOWN:  d51003e1 msr DBGWCR3_EL1, x1
+
+mrs x3, DBGWCR4_EL1
+// CHECK-INST: mrs x3, DBGWCR4_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x04,0x30,0xd5]
+// CHECK-UNKNOWN:  d53004e3 mrs x3, DBGWCR4_EL1
+
+msr DBGWCR4_EL1, x1
+// CHECK-INST: msr DBGWCR4_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x04,0x10,0xd5]
+// CHECK-UNKNOWN:  d51004e1 msr DBGWCR4_EL1, x1
+
+mrs x3, DBGWCR5_EL1
+// CHECK-INST: mrs x3, DBGWCR5_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x05,0x30,0xd5]
+// CHECK-UNKNOWN:  d53005e3 mrs x3, DBGWCR5_EL1
+
+msr DBGWCR5_EL1, x1
+// CHECK-INST: msr DBGWCR5_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x05,0x10,0xd5]
+// CHECK-UNKNOWN:  d51005e1 msr DBGWCR5_EL1, x1
+
+mrs x3, DBGWCR6_EL1
+// CHECK-INST: mrs x3, DBGWCR6_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x06,0x30,0xd5]
+// CHECK-UNKNOWN:  d53006e3 mrs x3, DBGWCR6_EL1
+
+msr DBGWCR6_EL1, x1
+// CHECK-INST: msr DBGWCR6_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x06,0x10,0xd5]
+// CHECK-UNKNOWN:  d51006e1 msr DBGWCR6_EL1, x1
+
+mrs x3, DBGWCR7_EL1
+// CHECK-INST: mrs x3, DBGWCR7_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x07,0x30,0xd5]
+// CHECK-UNKNOWN:  d53007e3 mrs x3, DBGWCR7_EL1
+
+msr DBGWCR7_EL1, x1
+// CHECK-INST: msr DBGWCR7_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x07,0x10,0xd5]
+// CHECK-UNKNOWN:  d51007e1 msr DBGWCR7_EL1, x1
+
+mrs x3, DBGWCR8_EL1
+// CHECK-INST: mrs x3, DBGWCR8_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x08,0x30,0xd5]
+// CHECK-UNKNOWN:  d53008e3 mrs x3, DBGWCR8_EL1
+
+msr DBGWCR8_EL1, x1
+// CHECK-INST: msr DBGWCR8_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x08,0x10,0xd5]
+// CHECK-UNKNOWN:  d51008e1 msr DBGWCR8_EL1, x1
+
+mrs x3, DBGWCR9_EL1
+// CHECK-INST: mrs x3, DBGWCR9_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x09,0x30,0xd5]
+// CHECK-UNKNOWN:  d53009e3 mrs x3, DBGWCR9_EL1
+
+msr DBGWCR9_EL1, x1
+// CHECK-INST: msr DBGWCR9_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x09,0x10,0xd5]
+// CHECK-UNKNOWN:  d51009e1 msr DBGWCR9_EL1, x1
+
+mrs x3, DBGWCR10_EL1
+// CHECK-INST: mrs x3, DBGWCR10_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x0a,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300ae3 mrs x3, DBGWCR10_EL1
+
+msr DBGWCR10_EL1, x1
+// CHECK-INST: msr DBGWCR10_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x0a,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100ae1 msr DBGWCR10_EL1, x1
+
+mrs x3, DBGWCR11_EL1
+// CHECK-INST: mrs x3, DBGWCR11_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x0b,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300be3 mrs x3, DBGWCR11_EL1
+
+msr DBGWCR11_EL1, x1
+// CHECK-INST: msr DBGWCR11_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x0b,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100be1 msr DBGWCR11_EL1, x1
+
+mrs x3, DBGWCR12_EL1
+// CHECK-INST: mrs x3, DBGWCR12_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x0c,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300ce3 mrs x3, DBGWCR12_EL1
+
+msr DBGWCR12_EL1, x1
+// CHECK-INST: msr DBGWCR12_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x0c,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100ce1 msr DBGWCR12_EL1, x1
+
+mrs x3, DBGWCR13_EL1
+// CHECK-INST: mrs x3, DBGWCR13_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x0d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300de3 mrs x3, DBGWCR13_EL1
+
+msr DBGWCR13_EL1, x1
+// CHECK-INST: msr DBGWCR13_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x0d,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100de1 msr DBGWCR13_EL1, x1
+
+mrs x3, DBGWCR14_EL1
+// CHECK-INST: mrs x3, DBGWCR14_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x0e,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300ee3 mrs x3, DBGWCR14_EL1
+
+msr DBGWCR14_EL1, x1
+// CHECK-INST: msr DBGWCR14_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x0e,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100ee1 msr DBGWCR14_EL1, x1
+
+mrs x3, DBGWCR15_EL1
+// CHECK-INST: mrs x3, DBGWCR15_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x0f,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300fe3 mrs x3, DBGWCR15_EL1
+
+msr DBGWCR15_EL1, x1
+// CHECK-INST: msr DBGWCR15_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x0f,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100fe1 msr DBGWCR15_EL1, x1
 
 // FEAT_DEBUGv8p9
-            mrs	x3, MDSELR_EL1
-// CHECK:   mrs	x3, MDSELR_EL1                  // encoding: [0x43,0x04,0x30,0xd5]
-            msr MDSELR_EL1, x1
-// CHECK:   msr	MDSELR_EL1, x1                  // encoding: [0x41,0x04,0x10,0xd5]
+mrs x3, MDSELR_EL1
+// CHECK-INST: mrs x3, MDSELR_EL1
+// CHECK-ENCODING: encoding: [0x43,0x04,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300443      mrs x3, MDSELR_EL1
+
+msr MDSELR_EL1, x1
+// CHECK-INST: msr MDSELR_EL1, x1
+// CHECK-ENCODING: encoding: [0x41,0x04,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100441      msr MDSELR_EL1, x1
 
 // FEAT_PMUv3p9
-            mrs	x3, PMUACR_EL1
-// CHECK:   mrs	x3, PMUACR_EL1                  // encoding: [0x83,0x9e,0x38,0xd5]
-            msr	PMUACR_EL1, x1
-// CHECK:   msr	PMUACR_EL1, x1                  // encoding: [0x81,0x9e,0x18,0xd5]
+mrs x3, PMUACR_EL1
+// CHECK-INST: mrs x3, PMUACR_EL1
+// CHECK-ENCODING: encoding: [0x83,0x9e,0x38,0xd5]
+// CHECK-UNKNOWN:  d5389e83      mrs x3, PMUACR_EL1
+
+msr PMUACR_EL1, x1
+// CHECK-INST: msr PMUACR_EL1, x1
+// CHECK-ENCODING: encoding: [0x81,0x9e,0x18,0xd5]
+// CHECK-UNKNOWN:  d5189e81      msr PMUACR_EL1, x1
 
 // FEAT_PMUv3_SS
-            mrs	x3, PMCCNTSVR_EL1
-// CHECK:   mrs	x3, PMCCNTSVR_EL1               // encoding: [0xe3,0xeb,0x30,0xd5]
-            mrs	x3, PMICNTSVR_EL1
-// CHECK:   mrs	x3, PMICNTSVR_EL1               // encoding: [0x03,0xec,0x30,0xd5]
-            mrs	x3, PMSSCR_EL1
-// CHECK:   mrs	x3, PMSSCR_EL1                  // encoding: [0x63,0x9d,0x38,0xd5]
-            msr	PMSSCR_EL1, x1
-// CHECK:   msr	PMSSCR_EL1, x1                  // encoding: [0x61,0x9d,0x18,0xd5]
-            mrs	x3, PMEVCNTSVR0_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR0_EL1             // encoding: [0x03,0xe8,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR1_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR1_EL1             // encoding: [0x23,0xe8,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR2_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR2_EL1             // encoding: [0x43,0xe8,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR3_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR3_EL1             // encoding: [0x63,0xe8,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR4_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR4_EL1             // encoding: [0x83,0xe8,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR5_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR5_EL1             // encoding: [0xa3,0xe8,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR6_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR6_EL1             // encoding: [0xc3,0xe8,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR7_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR7_EL1             // encoding: [0xe3,0xe8,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR8_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR8_EL1             // encoding: [0x03,0xe9,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR9_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR9_EL1             // encoding: [0x23,0xe9,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR10_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR10_EL1            // encoding: [0x43,0xe9,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR11_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR11_EL1            // encoding: [0x63,0xe9,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR12_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR12_EL1            // encoding: [0x83,0xe9,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR13_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR13_EL1            // encoding: [0xa3,0xe9,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR14_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR14_EL1            // encoding: [0xc3,0xe9,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR15_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR15_EL1            // encoding: [0xe3,0xe9,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR16_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR16_EL1            // encoding: [0x03,0xea,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR17_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR17_EL1            // encoding: [0x23,0xea,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR18_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR18_EL1            // encoding: [0x43,0xea,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR19_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR19_EL1            // encoding: [0x63,0xea,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR20_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR20_EL1            // encoding: [0x83,0xea,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR21_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR21_EL1            // encoding: [0xa3,0xea,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR22_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR22_EL1            // encoding: [0xc3,0xea,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR23_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR23_EL1            // encoding: [0xe3,0xea,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR24_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR24_EL1            // encoding: [0x03,0xeb,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR25_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR25_EL1            // encoding: [0x23,0xeb,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR26_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR26_EL1            // encoding: [0x43,0xeb,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR27_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR27_EL1            // encoding: [0x63,0xeb,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR28_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR28_EL1            // encoding: [0x83,0xeb,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR29_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR29_EL1            // encoding: [0xa3,0xeb,0x30,0xd5]
-            mrs	x3, PMEVCNTSVR30_EL1
-// CHECK:   mrs	x3, PMEVCNTSVR30_EL1            // encoding: [0xc3,0xeb,0x30,0xd5]
+mrs x3, PMCCNTSVR_EL1
+// CHECK-INST: mrs x3, PMCCNTSVR_EL1
+// CHECK-ENCODING: encoding: [0xe3,0xeb,0x30,0xd5]
+// CHECK-UNKNOWN:  d530ebe3      mrs x3, PMCCNTSVR_EL1
+
+mrs x3, PMICNTSVR_EL1
+// CHECK-INST: mrs x3, PMICNTSVR_EL1
+// CHECK-ENCODING: encoding: [0x03,0xec,0x30,0xd5]
+// CHECK-UNKNOWN:  d530ec03      mrs x3, PMICNTSVR_EL1
+
+mrs x3, PMSSCR_EL1
+// CHECK-INST: mrs x3, PMSSCR_EL1
+// CHECK-ENCODING: encoding: [0x63,0x9d,0x38,0xd5]
+// CHECK-UNKNOWN:  d5389d63      mrs x3, PMSSCR_EL1
+
+msr PMSSCR_EL1, x1
+// CHECK-INST: msr PMSSCR_EL1, x1
+// CHECK-ENCODING: encoding: [0x61,0x9d,0x18,0xd5]
+// CHECK-UNKNOWN:  d5189d61      msr PMSSCR_EL1, x1
+
+mrs x3, PMEVCNTSVR0_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR0_EL1
+// CHECK-ENCODING: encoding: [0x03,0xe8,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e803      mrs x3, PMEVCNTSVR0_EL1
+
+mrs x3, PMEVCNTSVR1_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR1_EL1
+// CHECK-ENCODING: encoding: [0x23,0xe8,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e823      mrs x3, PMEVCNTSVR1_EL1
+
+mrs x3, PMEVCNTSVR2_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR2_EL1
+// CHECK-ENCODING: encoding: [0x43,0xe8,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e843      mrs x3, PMEVCNTSVR2_EL1
+
+mrs x3, PMEVCNTSVR3_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR3_EL1
+// CHECK-ENCODING: encoding: [0x63,0xe8,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e863      mrs x3, PMEVCNTSVR3_EL1
+
+mrs x3, PMEVCNTSVR4_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR4_EL1
+// CHECK-ENCODING: encoding: [0x83,0xe8,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e883      mrs x3, PMEVCNTSVR4_EL1
+
+mrs x3, PMEVCNTSVR5_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR5_EL1
+// CHECK-ENCODING: encoding: [0xa3,0xe8,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e8a3      mrs x3, PMEVCNTSVR5_EL1
+
+mrs x3, PMEVCNTSVR6_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR6_EL1
+// CHECK-ENCODING: encoding: [0xc3,0xe8,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e8c3      mrs x3, PMEVCNTSVR6_EL1
+
+mrs x3, PMEVCNTSVR7_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR7_EL1
+// CHECK-ENCODING: encoding: [0xe3,0xe8,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e8e3      mrs x3, PMEVCNTSVR7_EL1
+
+mrs x3, PMEVCNTSVR8_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR8_EL1
+// CHECK-ENCODING: encoding: [0x03,0xe9,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e903      mrs x3, PMEVCNTSVR8_EL1
+
+mrs x3, PMEVCNTSVR9_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR9_EL1
+// CHECK-ENCODING: encoding: [0x23,0xe9,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e923      mrs x3, PMEVCNTSVR9_EL1
+
+mrs x3, PMEVCNTSVR10_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR10_EL1
+// CHECK-ENCODING: encoding: [0x43,0xe9,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e943      mrs x3, PMEVCNTSVR10_EL1
+
+mrs x3, PMEVCNTSVR11_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR11_EL1
+// CHECK-ENCODING: encoding: [0x63,0xe9,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e963      mrs x3, PMEVCNTSVR11_EL1
+
+mrs x3, PMEVCNTSVR12_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR12_EL1
+// CHECK-ENCODING: encoding: [0x83,0xe9,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e983      mrs x3, PMEVCNTSVR12_EL1
+
+mrs x3, PMEVCNTSVR13_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR13_EL1
+// CHECK-ENCODING: encoding: [0xa3,0xe9,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e9a3      mrs x3, PMEVCNTSVR13_EL1
+
+mrs x3, PMEVCNTSVR14_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR14_EL1
+// CHECK-ENCODING: encoding: [0xc3,0xe9,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e9c3      mrs x3, PMEVCNTSVR14_EL1
+
+mrs x3, PMEVCNTSVR15_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR15_EL1
+// CHECK-ENCODING: encoding: [0xe3,0xe9,0x30,0xd5]
+// CHECK-UNKNOWN:  d530e9e3      mrs x3, PMEVCNTSVR15_EL1
+
+mrs x3, PMEVCNTSVR16_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR16_EL1
+// CHECK-ENCODING: encoding: [0x03,0xea,0x30,0xd5]
+// CHECK-UNKNOWN:  d530ea03      mrs x3, PMEVCNTSVR16_EL1
+
+mrs x3, PMEVCNTSVR17_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR17_EL1
+// CHECK-ENCODING: encoding: [0x23,0xea,0x30,0xd5]
+// CHECK-UNKNOWN:  d530ea23      mrs x3, PMEVCNTSVR17_EL1
+
+mrs x3, PMEVCNTSVR18_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR18_EL1
+// CHECK-ENCODING: encoding: [0x43,0xea,0x30,0xd5]
+// CHECK-UNKNOWN:  d530ea43      mrs x3, PMEVCNTSVR18_EL1
+
+mrs x3, PMEVCNTSVR19_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR19_EL1
+// CHECK-ENCODING: encoding: [0x63,0xea,0x30,0xd5]
+// CHECK-UNKNOWN:  d530ea63      mrs x3, PMEVCNTSVR19_EL1
+
+mrs x3, PMEVCNTSVR20_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR20_EL1
+// CHECK-ENCODING: encoding: [0x83,0xea,0x30,0xd5]
+// CHECK-UNKNOWN:  d530ea83      mrs x3, PMEVCNTSVR20_EL1
+
+mrs x3, PMEVCNTSVR21_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR21_EL1
+// CHECK-ENCODING: encoding: [0xa3,0xea,0x30,0xd5]
+// CHECK-UNKNOWN:  d530eaa3      mrs x3, PMEVCNTSVR21_EL1
+
+mrs x3, PMEVCNTSVR22_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR22_EL1
+// CHECK-ENCODING: encoding: [0xc3,0xea,0x30,0xd5]
+// CHECK-UNKNOWN:  d530eac3      mrs x3, PMEVCNTSVR22_EL1
+
+mrs x3, PMEVCNTSVR23_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR23_EL1
+// CHECK-ENCODING: encoding: [0xe3,0xea,0x30,0xd5]
+// CHECK-UNKNOWN:  d530eae3      mrs x3, PMEVCNTSVR23_EL1
+
+mrs x3, PMEVCNTSVR24_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR24_EL1
+// CHECK-ENCODING: encoding: [0x03,0xeb,0x30,0xd5]
+// CHECK-UNKNOWN:  d530eb03      mrs x3, PMEVCNTSVR24_EL1
+
+mrs x3, PMEVCNTSVR25_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR25_EL1
+// CHECK-ENCODING: encoding: [0x23,0xeb,0x30,0xd5]
+// CHECK-UNKNOWN:  d530eb23      mrs x3, PMEVCNTSVR25_EL1
+
+mrs x3, PMEVCNTSVR26_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR26_EL1
+// CHECK-ENCODING: encoding: [0x43,0xeb,0x30,0xd5]
+// CHECK-UNKNOWN:  d530eb43      mrs x3, PMEVCNTSVR26_EL1
+
+mrs x3, PMEVCNTSVR27_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR27_EL1
+// CHECK-ENCODING: encoding: [0x63,0xeb,0x30,0xd5]
+// CHECK-UNKNOWN:  d530eb63      mrs x3, PMEVCNTSVR27_EL1
+
+mrs x3, PMEVCNTSVR28_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR28_EL1
+// CHECK-ENCODING: encoding: [0x83,0xeb,0x30,0xd5]
+// CHECK-UNKNOWN:  d530eb83      mrs x3, PMEVCNTSVR28_EL1
+
+mrs x3, PMEVCNTSVR29_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR29_EL1
+// CHECK-ENCODING: encoding: [0xa3,0xeb,0x30,0xd5]
+// CHECK-UNKNOWN:  d530eba3      mrs x3, PMEVCNTSVR29_EL1
+
+mrs x3, PMEVCNTSVR30_EL1
+// CHECK-INST: mrs x3, PMEVCNTSVR30_EL1
+// CHECK-ENCODING: encoding: [0xc3,0xeb,0x30,0xd5]
+// CHECK-UNKNOWN:  d530ebc3      mrs x3, PMEVCNTSVR30_EL1
 
 // FEAT_PMUv3_ICNTR
-            mrs x3, PMICNTR_EL0
-// CHECK:   mrs x3, PMICNTR_EL0                 // encoding: [0x03,0x94,0x3b,0xd5]
-            msr PMICNTR_EL0, x3
-// CHECK:   msr PMICNTR_EL0, x3                 // encoding: [0x03,0x94,0x1b,0xd5]
-            mrs x3, PMICFILTR_EL0
-// CHECK:   mrs x3, PMICFILTR_EL0               // encoding: [0x03,0x96,0x3b,0xd5]
-            msr PMICFILTR_EL0, x3
-// CHECK:   msr PMICFILTR_EL0, x3               // encoding: [0x03,0x96,0x1b,0xd5]
+mrs x3, PMICNTR_EL0
+// CHECK-INST: mrs x3, PMICNTR_EL0
+// CHECK-ENCODING: encoding: [0x03,0x94,0x3b,0xd5]
+// CHECK-UNKNOWN:  d53b9403      mrs x3, PMICNTR_EL0
+
+msr PMICNTR_EL0, x3
+// CHECK-INST: msr PMICNTR_EL0, x3
+// CHECK-ENCODING: encoding: [0x03,0x94,0x1b,0xd5]
+// CHECK-UNKNOWN:  d51b9403      msr PMICNTR_EL0, x3
+
+mrs x3, PMICFILTR_EL0
+// CHECK-INST: mrs x3, PMICFILTR_EL0
+// CHECK-ENCODING: encoding: [0x03,0x96,0x3b,0xd5]
+// CHECK-UNKNOWN:  d53b9603      mrs x3, PMICFILTR_EL0
+
+msr PMICFILTR_EL0, x3
+// CHECK-INST: msr PMICFILTR_EL0, x3
+// CHECK-ENCODING: encoding: [0x03,0x96,0x1b,0xd5]
+// CHECK-UNKNOWN:  d51b9603      msr PMICFILTR_EL0, x3
 
 // FEAT_PMUv3p9/FEAT_PMUV3_ICNTR
-            msr PMZR_EL0, x3
-// CHECK:   msr PMZR_EL0, x3                    // encoding: [0x83,0x9d,0x1b,0xd5]
+msr PMZR_EL0, x3
+// CHECK-INST: msr PMZR_EL0, x3
+// CHECK-ENCODING: encoding: [0x83,0x9d,0x1b,0xd5]
+// CHECK-UNKNOWN:  d51b9d83      msr PMZR_EL0, x3
 
 // FEAT_SEBEP
-            mrs	x3, PMECR_EL1
-// CHECK:   mrs	x3, PMECR_EL1                   // encoding: [0xa3,0x9e,0x38,0xd5]
-            msr	PMECR_EL1, x1
-// CHECK:   msr	PMECR_EL1, x1                   // encoding: [0xa1,0x9e,0x18,0xd5]
-            mrs	x3, PMIAR_EL1
-// CHECK:   mrs	x3, PMIAR_EL1                   // encoding: [0xe3,0x9e,0x38,0xd5]
-            msr	PMIAR_EL1, x1
-// CHECK:   msr	PMIAR_EL1, x1                   // encoding: [0xe1,0x9e,0x18,0xd5]
+mrs x3, PMECR_EL1
+// CHECK-INST: mrs x3, PMECR_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x9e,0x38,0xd5]
+// CHECK-UNKNOWN:  d5389ea3      mrs x3, PMECR_EL1
+
+msr PMECR_EL1, x1
+// CHECK-INST: msr PMECR_EL1, x1
+// CHECK-ENCODING: encoding: [0xa1,0x9e,0x18,0xd5]
+// CHECK-UNKNOWN:  d5189ea1      msr PMECR_EL1, x1
+
+mrs x3, PMIAR_EL1
+// CHECK-INST: mrs x3, PMIAR_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x9e,0x38,0xd5]
+// CHECK-UNKNOWN:  d5389ee3      mrs x3, PMIAR_EL1
+
+msr PMIAR_EL1, x1
+// CHECK-INST: msr PMIAR_EL1, x1
+// CHECK-ENCODING: encoding: [0xe1,0x9e,0x18,0xd5]
+// CHECK-UNKNOWN:  d5189ee1      msr PMIAR_EL1, x1
 
 // FEAT_SPMU
-            mrs	x3, SPMACCESSR_EL1
-// CHECK:   mrs	x3, SPMACCESSR_EL1              // encoding: [0x63,0x9d,0x30,0xd5]
-            msr	SPMACCESSR_EL1, x1
-// CHECK:   msr	SPMACCESSR_EL1, x1              // encoding: [0x61,0x9d,0x10,0xd5]
-            mrs	x3, SPMACCESSR_EL12
-// CHECK:   mrs	x3, SPMACCESSR_EL12             // encoding: [0x63,0x9d,0x35,0xd5]
-            msr	SPMACCESSR_EL12, x1
-// CHECK:   msr	SPMACCESSR_EL12, x1             // encoding: [0x61,0x9d,0x15,0xd5]
-            mrs	x3, SPMACCESSR_EL2
-// CHECK:   mrs	x3, SPMACCESSR_EL2              // encoding: [0x63,0x9d,0x34,0xd5]
-            msr	SPMACCESSR_EL2, x1
-// CHECK:   msr	SPMACCESSR_EL2, x1              // encoding: [0x61,0x9d,0x14,0xd5]
-            mrs	x3, SPMACCESSR_EL3
-// CHECK:   mrs	x3, SPMACCESSR_EL3              // encoding: [0x63,0x9d,0x36,0xd5]
-            msr	SPMACCESSR_EL3, x1
-// CHECK:   msr	SPMACCESSR_EL3, x1              // encoding: [0x61,0x9d,0x16,0xd5]
-            mrs	x3, SPMCNTENCLR_EL0
-// CHECK:   mrs	x3, SPMCNTENCLR_EL0             // encoding: [0x43,0x9c,0x33,0xd5]
-            msr	SPMCNTENCLR_EL0, x1
-// CHECK:   msr	SPMCNTENCLR_EL0, x1             // encoding: [0x41,0x9c,0x13,0xd5]
-            mrs	x3, SPMCNTENSET_EL0
-// CHECK:   mrs	x3, SPMCNTENSET_EL0             // encoding: [0x23,0x9c,0x33,0xd5]
-            msr	SPMCNTENSET_EL0, x1
-// CHECK:   msr	SPMCNTENSET_EL0, x1             // encoding: [0x21,0x9c,0x13,0xd5]
-            mrs	x3, SPMCR_EL0
-// CHECK:   mrs	x3, SPMCR_EL0                   // encoding: [0x03,0x9c,0x33,0xd5]
-            msr	SPMCR_EL0, x1
-// CHECK:   msr	SPMCR_EL0, x1                   // encoding: [0x01,0x9c,0x13,0xd5]
-            mrs	x3, SPMDEVAFF_EL1
-// CHECK:   mrs	x3, SPMDEVAFF_EL1               // encoding: [0xc3,0x9d,0x30,0xd5]
-            mrs	x3, SPMDEVARCH_EL1
-// CHECK:   mrs	x3, SPMDEVARCH_EL1              // encoding: [0xa3,0x9d,0x30,0xd5]
-
-            mrs	x3, SPMEVCNTR0_EL0
-// CHECK:   mrs	x3, SPMEVCNTR0_EL0              // encoding: [0x03,0xe0,0x33,0xd5]
-            msr	SPMEVCNTR0_EL0, x1
-// CHECK:   msr	SPMEVCNTR0_EL0, x1              // encoding: [0x01,0xe0,0x13,0xd5]
-            mrs	x3, SPMEVCNTR1_EL0
-// CHECK:   mrs	x3, SPMEVCNTR1_EL0              // encoding: [0x23,0xe0,0x33,0xd5]
-            msr	SPMEVCNTR1_EL0, x1
-// CHECK:   msr	SPMEVCNTR1_EL0, x1              // encoding: [0x21,0xe0,0x13,0xd5]
-            mrs	x3, SPMEVCNTR2_EL0
-// CHECK:   mrs	x3, SPMEVCNTR2_EL0              // encoding: [0x43,0xe0,0x33,0xd5]
-            msr	SPMEVCNTR2_EL0, x1
-// CHECK:   msr	SPMEVCNTR2_EL0, x1              // encoding: [0x41,0xe0,0x13,0xd5]
-            mrs	x3, SPMEVCNTR3_EL0
-// CHECK:   mrs	x3, SPMEVCNTR3_EL0              // encoding: [0x63,0xe0,0x33,0xd5]
-            msr	SPMEVCNTR3_EL0, x1
-// CHECK:   msr	SPMEVCNTR3_EL0, x1              // encoding: [0x61,0xe0,0x13,0xd5]
-            mrs	x3, SPMEVCNTR4_EL0
-// CHECK:   mrs	x3, SPMEVCNTR4_EL0              // encoding: [0x83,0xe0,0x33,0xd5]
-            msr	SPMEVCNTR4_EL0, x1
-// CHECK:   msr	SPMEVCNTR4_EL0, x1              // encoding: [0x81,0xe0,0x13,0xd5]
-            mrs	x3, SPMEVCNTR5_EL0
-// CHECK:   mrs	x3, SPMEVCNTR5_EL0              // encoding: [0xa3,0xe0,0x33,0xd5]
-            msr	SPMEVCNTR5_EL0, x1
-// CHECK:   msr	SPMEVCNTR5_EL0, x1              // encoding: [0xa1,0xe0,0x13,0xd5]
-            mrs	x3, SPMEVCNTR6_EL0
-// CHECK:   mrs	x3, SPMEVCNTR6_EL0              // encoding: [0xc3,0xe0,0x33,0xd5]
-            msr	SPMEVCNTR6_EL0, x1
-// CHECK:   msr	SPMEVCNTR6_EL0, x1              // encoding: [0xc1,0xe0,0x13,0xd5]
-            mrs	x3, SPMEVCNTR7_EL0
-// CHECK:   mrs	x3, SPMEVCNTR7_EL0              // encoding: [0xe3,0xe0,0x33,0xd5]
-            msr	SPMEVCNTR7_EL0, x1
-// CHECK:   msr	SPMEVCNTR7_EL0, x1              // encoding: [0xe1,0xe0,0x13,0xd5]
-            mrs	x3, SPMEVCNTR8_EL0
-// CHECK:   mrs	x3, SPMEVCNTR8_EL0              // encoding: [0x03,0xe1,0x33,0xd5]
-            msr	SPMEVCNTR8_EL0, x1
-// CHECK:   msr	SPMEVCNTR8_EL0, x1              // encoding: [0x01,0xe1,0x13,0xd5]
-            mrs	x3, SPMEVCNTR9_EL0
-// CHECK:   mrs	x3, SPMEVCNTR9_EL0              // encoding: [0x23,0xe1,0x33,0xd5]
-            msr	SPMEVCNTR9_EL0, x1
-// CHECK:   msr	SPMEVCNTR9_EL0, x1              // encoding: [0x21,0xe1,0x13,0xd5]
-            mrs	x3, SPMEVCNTR10_EL0
-// CHECK:   mrs	x3, SPMEVCNTR10_EL0             // encoding: [0x43,0xe1,0x33,0xd5]
-            msr	SPMEVCNTR10_EL0, x1
-// CHECK:   msr	SPMEVCNTR10_EL0, x1             // encoding: [0x41,0xe1,0x13,0xd5]
-            mrs	x3, SPMEVCNTR11_EL0
-// CHECK:   mrs	x3, SPMEVCNTR11_EL0             // encoding: [0x63,0xe1,0x33,0xd5]
-            msr	SPMEVCNTR11_EL0, x1
-// CHECK:   msr	SPMEVCNTR11_EL0, x1             // encoding: [0x61,0xe1,0x13,0xd5]
-            mrs	x3, SPMEVCNTR12_EL0
-// CHECK:   mrs	x3, SPMEVCNTR12_EL0             // encoding: [0x83,0xe1,0x33,0xd5]
-            msr	SPMEVCNTR12_EL0, x1
-// CHECK:   msr	SPMEVCNTR12_EL0, x1             // encoding: [0x81,0xe1,0x13,0xd5]
-            mrs	x3, SPMEVCNTR13_EL0
-// CHECK:   mrs	x3, SPMEVCNTR13_EL0             // encoding: [0xa3,0xe1,0x33,0xd5]
-            msr	SPMEVCNTR13_EL0, x1
-// CHECK:   msr	SPMEVCNTR13_EL0, x1             // encoding: [0xa1,0xe1,0x13,0xd5]
-            mrs	x3, SPMEVCNTR14_EL0
-// CHECK:   mrs	x3, SPMEVCNTR14_EL0             // encoding: [0xc3,0xe1,0x33,0xd5]
-            msr	SPMEVCNTR14_EL0, x1
-// CHECK:   msr	SPMEVCNTR14_EL0, x1             // encoding: [0xc1,0xe1,0x13,0xd5]
-            mrs	x3, SPMEVCNTR15_EL0
-// CHECK:   mrs	x3, SPMEVCNTR15_EL0             // encoding: [0xe3,0xe1,0x33,0xd5]
-            msr	SPMEVCNTR15_EL0, x1
-// CHECK:   msr	SPMEVCNTR15_EL0, x1             // encoding: [0xe1,0xe1,0x13,0xd5]
-
-            mrs	x3, SPMEVFILT2R0_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R0_EL0            // encoding: [0x03,0xe6,0x33,0xd5]
-            msr	SPMEVFILT2R0_EL0, x1
-// CHECK:   msr	SPMEVFILT2R0_EL0, x1            // encoding: [0x01,0xe6,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R1_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R1_EL0            // encoding: [0x23,0xe6,0x33,0xd5]
-            msr	SPMEVFILT2R1_EL0, x1
-// CHECK:   msr	SPMEVFILT2R1_EL0, x1            // encoding: [0x21,0xe6,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R2_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R2_EL0            // encoding: [0x43,0xe6,0x33,0xd5]
-            msr	SPMEVFILT2R2_EL0, x1
-// CHECK:   msr	SPMEVFILT2R2_EL0, x1            // encoding: [0x41,0xe6,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R3_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R3_EL0            // encoding: [0x63,0xe6,0x33,0xd5]
-            msr	SPMEVFILT2R3_EL0, x1
-// CHECK:   msr	SPMEVFILT2R3_EL0, x1            // encoding: [0x61,0xe6,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R4_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R4_EL0            // encoding: [0x83,0xe6,0x33,0xd5]
-            msr	SPMEVFILT2R4_EL0, x1
-// CHECK:   msr	SPMEVFILT2R4_EL0, x1            // encoding: [0x81,0xe6,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R5_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R5_EL0            // encoding: [0xa3,0xe6,0x33,0xd5]
-            msr	SPMEVFILT2R5_EL0, x1
-// CHECK:   msr	SPMEVFILT2R5_EL0, x1            // encoding: [0xa1,0xe6,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R6_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R6_EL0            // encoding: [0xc3,0xe6,0x33,0xd5]
-            msr	SPMEVFILT2R6_EL0, x1
-// CHECK:   msr	SPMEVFILT2R6_EL0, x1            // encoding: [0xc1,0xe6,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R7_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R7_EL0            // encoding: [0xe3,0xe6,0x33,0xd5]
-            msr	SPMEVFILT2R7_EL0, x1
-// CHECK:   msr	SPMEVFILT2R7_EL0, x1            // encoding: [0xe1,0xe6,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R8_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R8_EL0            // encoding: [0x03,0xe7,0x33,0xd5]
-            msr	SPMEVFILT2R8_EL0, x1
-// CHECK:   msr	SPMEVFILT2R8_EL0, x1            // encoding: [0x01,0xe7,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R9_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R9_EL0            // encoding: [0x23,0xe7,0x33,0xd5]
-            msr	SPMEVFILT2R9_EL0, x1
-// CHECK:   msr	SPMEVFILT2R9_EL0, x1            // encoding: [0x21,0xe7,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R10_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R10_EL0           // encoding: [0x43,0xe7,0x33,0xd5]
-            msr	SPMEVFILT2R10_EL0, x1
-// CHECK:   msr	SPMEVFILT2R10_EL0, x1           // encoding: [0x41,0xe7,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R11_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R11_EL0           // encoding: [0x63,0xe7,0x33,0xd5]
-            msr	SPMEVFILT2R11_EL0, x1
-// CHECK:   msr	SPMEVFILT2R11_EL0, x1           // encoding: [0x61,0xe7,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R12_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R12_EL0           // encoding: [0x83,0xe7,0x33,0xd5]
-            msr	SPMEVFILT2R12_EL0, x1
-// CHECK:   msr	SPMEVFILT2R12_EL0, x1           // encoding: [0x81,0xe7,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R13_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R13_EL0           // encoding: [0xa3,0xe7,0x33,0xd5]
-            msr	SPMEVFILT2R13_EL0, x1
-// CHECK:   msr	SPMEVFILT2R13_EL0, x1           // encoding: [0xa1,0xe7,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R14_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R14_EL0           // encoding: [0xc3,0xe7,0x33,0xd5]
-            msr	SPMEVFILT2R14_EL0, x1
-// CHECK:   msr	SPMEVFILT2R14_EL0, x1           // encoding: [0xc1,0xe7,0x13,0xd5]
-            mrs	x3, SPMEVFILT2R15_EL0
-// CHECK:   mrs	x3, SPMEVFILT2R15_EL0           // encoding: [0xe3,0xe7,0x33,0xd5]
-            msr	SPMEVFILT2R15_EL0, x1
-// CHECK:   msr	SPMEVFILT2R15_EL0, x1           // encoding: [0xe1,0xe7,0x13,0xd5]
-
-            mrs	x3, SPMEVFILTR0_EL0
-// CHECK:   mrs	x3, SPMEVFILTR0_EL0             // encoding: [0x03,0xe4,0x33,0xd5]
-            msr	SPMEVFILTR0_EL0, x1
-// CHECK:   msr	SPMEVFILTR0_EL0, x1             // encoding: [0x01,0xe4,0x13,0xd5]
-            mrs	x3, SPMEVFILTR1_EL0
-// CHECK:   mrs	x3, SPMEVFILTR1_EL0             // encoding: [0x23,0xe4,0x33,0xd5]
-            msr	SPMEVFILTR1_EL0, x1
-// CHECK:   msr	SPMEVFILTR1_EL0, x1             // encoding: [0x21,0xe4,0x13,0xd5]
-            mrs	x3, SPMEVFILTR2_EL0
-// CHECK:   mrs	x3, SPMEVFILTR2_EL0             // encoding: [0x43,0xe4,0x33,0xd5]
-            msr	SPMEVFILTR2_EL0, x1
-// CHECK:   msr	SPMEVFILTR2_EL0, x1             // encoding: [0x41,0xe4,0x13,0xd5]
-            mrs	x3, SPMEVFILTR3_EL0
-// CHECK:   mrs	x3, SPMEVFILTR3_EL0             // encoding: [0x63,0xe4,0x33,0xd5]
-            msr	SPMEVFILTR3_EL0, x1
-// CHECK:   msr	SPMEVFILTR3_EL0, x1             // encoding: [0x61,0xe4,0x13,0xd5]
-            mrs	x3, SPMEVFILTR4_EL0
-// CHECK:   mrs	x3, SPMEVFILTR4_EL0             // encoding: [0x83,0xe4,0x33,0xd5]
-            msr	SPMEVFILTR4_EL0, x1
-// CHECK:   msr	SPMEVFILTR4_EL0, x1             // encoding: [0x81,0xe4,0x13,0xd5]
-            mrs	x3, SPMEVFILTR5_EL0
-// CHECK:   mrs	x3, SPMEVFILTR5_EL0             // encoding: [0xa3,0xe4,0x33,0xd5]
-            msr	SPMEVFILTR5_EL0, x1
-// CHECK:   msr	SPMEVFILTR5_EL0, x1             // encoding: [0xa1,0xe4,0x13,0xd5]
-            mrs	x3, SPMEVFILTR6_EL0
-// CHECK:   mrs	x3, SPMEVFILTR6_EL0             // encoding: [0xc3,0xe4,0x33,0xd5]
-            msr	SPMEVFILTR6_EL0, x1
-// CHECK:   msr	SPMEVFILTR6_EL0, x1             // encoding: [0xc1,0xe4,0x13,0xd5]
-            mrs	x3, SPMEVFILTR7_EL0
-// CHECK:   mrs	x3, SPMEVFILTR7_EL0             // encoding: [0xe3,0xe4,0x33,0xd5]
-            msr	SPMEVFILTR7_EL0, x1
-// CHECK:   msr	SPMEVFILTR7_EL0, x1             // encoding: [0xe1,0xe4,0x13,0xd5]
-            mrs	x3, SPMEVFILTR8_EL0
-// CHECK:   mrs	x3, SPMEVFILTR8_EL0             // encoding: [0x03,0xe5,0x33,0xd5]
-            msr	SPMEVFILTR8_EL0, x1
-// CHECK:   msr	SPMEVFILTR8_EL0, x1             // encoding: [0x01,0xe5,0x13,0xd5]
-            mrs	x3, SPMEVFILTR9_EL0
-// CHECK:   mrs	x3, SPMEVFILTR9_EL0             // encoding: [0x23,0xe5,0x33,0xd5]
-            msr	SPMEVFILTR9_EL0, x1
-// CHECK:   msr	SPMEVFILTR9_EL0, x1             // encoding: [0x21,0xe5,0x13,0xd5]
-            mrs	x3, SPMEVFILTR10_EL0
-// CHECK:   mrs	x3, SPMEVFILTR10_EL0            // encoding: [0x43,0xe5,0x33,0xd5]
-            msr	SPMEVFILTR10_EL0, x1
-// CHECK:   msr	SPMEVFILTR10_EL0, x1            // encoding: [0x41,0xe5,0x13,0xd5]
-            mrs	x3, SPMEVFILTR11_EL0
-// CHECK:   mrs	x3, SPMEVFILTR11_EL0            // encoding: [0x63,0xe5,0x33,0xd5]
-            msr	SPMEVFILTR11_EL0, x1
-// CHECK:   msr	SPMEVFILTR11_EL0, x1            // encoding: [0x61,0xe5,0x13,0xd5]
-            mrs	x3, SPMEVFILTR12_EL0
-// CHECK:   mrs	x3, SPMEVFILTR12_EL0            // encoding: [0x83,0xe5,0x33,0xd5]
-            msr	SPMEVFILTR12_EL0, x1
-// CHECK:   msr	SPMEVFILTR12_EL0, x1            // encoding: [0x81,0xe5,0x13,0xd5]
-            mrs	x3, SPMEVFILTR13_EL0
-// CHECK:   mrs	x3, SPMEVFILTR13_EL0            // encoding: [0xa3,0xe5,0x33,0xd5]
-            msr	SPMEVFILTR13_EL0, x1
-// CHECK:   msr	SPMEVFILTR13_EL0, x1            // encoding: [0xa1,0xe5,0x13,0xd5]
-            mrs	x3, SPMEVFILTR14_EL0
-// CHECK:   mrs	x3, SPMEVFILTR14_EL0            // encoding: [0xc3,0xe5,0x33,0xd5]
-            msr	SPMEVFILTR14_EL0, x1
-// CHECK:   msr	SPMEVFILTR14_EL0, x1            // encoding: [0xc1,0xe5,0x13,0xd5]
-            mrs	x3, SPMEVFILTR15_EL0
-// CHECK:   mrs	x3, SPMEVFILTR15_EL0            // encoding: [0xe3,0xe5,0x33,0xd5]
-            msr	SPMEVFILTR15_EL0, x1
-// CHECK:   msr	SPMEVFILTR15_EL0, x1            // encoding: [0xe1,0xe5,0x13,0xd5]
-
-            mrs	x3, SPMEVTYPER0_EL0
-// CHECK:   mrs	x3, SPMEVTYPER0_EL0             // encoding: [0x03,0xe2,0x33,0xd5]
-            msr	SPMEVTYPER0_EL0, x1
-// CHECK:   msr	SPMEVTYPER0_EL0, x1             // encoding: [0x01,0xe2,0x13,0xd5]
-            mrs	x3, SPMEVTYPER1_EL0
-// CHECK:   mrs	x3, SPMEVTYPER1_EL0             // encoding: [0x23,0xe2,0x33,0xd5]
-            msr	SPMEVTYPER1_EL0, x1
-// CHECK:   msr	SPMEVTYPER1_EL0, x1             // encoding: [0x21,0xe2,0x13,0xd5]
-            mrs	x3, SPMEVTYPER2_EL0
-// CHECK:   mrs	x3, SPMEVTYPER2_EL0             // encoding: [0x43,0xe2,0x33,0xd5]
-            msr	SPMEVTYPER2_EL0, x1
-// CHECK:   msr	SPMEVTYPER2_EL0, x1             // encoding: [0x41,0xe2,0x13,0xd5]
-            mrs	x3, SPMEVTYPER3_EL0
-// CHECK:   mrs	x3, SPMEVTYPER3_EL0             // encoding: [0x63,0xe2,0x33,0xd5]
-            msr	SPMEVTYPER3_EL0, x1
-// CHECK:   msr	SPMEVTYPER3_EL0, x1             // encoding: [0x61,0xe2,0x13,0xd5]
-            mrs	x3, SPMEVTYPER4_EL0
-// CHECK:   mrs	x3, SPMEVTYPER4_EL0             // encoding: [0x83,0xe2,0x33,0xd5]
-            msr	SPMEVTYPER4_EL0, x1
-// CHECK:   msr	SPMEVTYPER4_EL0, x1             // encoding: [0x81,0xe2,0x13,0xd5]
-            mrs	x3, SPMEVTYPER5_EL0
-// CHECK:   mrs	x3, SPMEVTYPER5_EL0             // encoding: [0xa3,0xe2,0x33,0xd5]
-            msr	SPMEVTYPER5_EL0, x1
-// CHECK:   msr	SPMEVTYPER5_EL0, x1             // encoding: [0xa1,0xe2,0x13,0xd5]
-            mrs	x3, SPMEVTYPER6_EL0
-// CHECK:   mrs	x3, SPMEVTYPER6_EL0             // encoding: [0xc3,0xe2,0x33,0xd5]
-            msr	SPMEVTYPER6_EL0, x1
-// CHECK:   msr	SPMEVTYPER6_EL0, x1             // encoding: [0xc1,0xe2,0x13,0xd5]
-            mrs	x3, SPMEVTYPER7_EL0
-// CHECK:   mrs	x3, SPMEVTYPER7_EL0             // encoding: [0xe3,0xe2,0x33,0xd5]
-            msr	SPMEVTYPER7_EL0, x1
-// CHECK:   msr	SPMEVTYPER7_EL0, x1             // encoding: [0xe1,0xe2,0x13,0xd5]
-            mrs	x3, SPMEVTYPER8_EL0
-// CHECK:   mrs	x3, SPMEVTYPER8_EL0             // encoding: [0x03,0xe3,0x33,0xd5]
-            msr	SPMEVTYPER8_EL0, x1
-// CHECK:   msr	SPMEVTYPER8_EL0, x1             // encoding: [0x01,0xe3,0x13,0xd5]
-            mrs	x3, SPMEVTYPER9_EL0
-// CHECK:   mrs	x3, SPMEVTYPER9_EL0             // encoding: [0x23,0xe3,0x33,0xd5]
-            msr	SPMEVTYPER9_EL0, x1
-// CHECK:   msr	SPMEVTYPER9_EL0, x1             // encoding: [0x21,0xe3,0x13,0xd5]
-            mrs	x3, SPMEVTYPER10_EL0
-// CHECK:   mrs	x3, SPMEVTYPER10_EL0            // encoding: [0x43,0xe3,0x33,0xd5]
-            msr	SPMEVTYPER10_EL0, x1
-// CHECK:   msr	SPMEVTYPER10_EL0, x1            // encoding: [0x41,0xe3,0x13,0xd5]
-            mrs	x3, SPMEVTYPER11_EL0
-// CHECK:   mrs	x3, SPMEVTYPER11_EL0            // encoding: [0x63,0xe3,0x33,0xd5]
-            msr	SPMEVTYPER11_EL0, x1
-// CHECK:   msr	SPMEVTYPER11_EL0, x1            // encoding: [0x61,0xe3,0x13,0xd5]
-            mrs	x3, SPMEVTYPER12_EL0
-// CHECK:   mrs	x3, SPMEVTYPER12_EL0            // encoding: [0x83,0xe3,0x33,0xd5]
-            msr	SPMEVTYPER12_EL0, x1
-// CHECK:   msr	SPMEVTYPER12_EL0, x1            // encoding: [0x81,0xe3,0x13,0xd5]
-            mrs	x3, SPMEVTYPER13_EL0
-// CHECK:   mrs	x3, SPMEVTYPER13_EL0            // encoding: [0xa3,0xe3,0x33,0xd5]
-            msr	SPMEVTYPER13_EL0, x1
-// CHECK:   msr	SPMEVTYPER13_EL0, x1            // encoding: [0xa1,0xe3,0x13,0xd5]
-            mrs	x3, SPMEVTYPER14_EL0
-// CHECK:   mrs	x3, SPMEVTYPER14_EL0            // encoding: [0xc3,0xe3,0x33,0xd5]
-            msr	SPMEVTYPER14_EL0, x1
-// CHECK:   msr	SPMEVTYPER14_EL0, x1            // encoding: [0xc1,0xe3,0x13,0xd5]
-            mrs	x3, SPMEVTYPER15_EL0
-// CHECK:   mrs	x3, SPMEVTYPER15_EL0            // encoding: [0xe3,0xe3,0x33,0xd5]
-            msr	SPMEVTYPER15_EL0, x1
-// CHECK:   msr	SPMEVTYPER15_EL0, x1            // encoding: [0xe1,0xe3,0x13,0xd5]
-
-            mrs	x3, SPMIIDR_EL1
-// CHECK:   mrs	x3, SPMIIDR_EL1                 // encoding: [0x83,0x9d,0x30,0xd5]
-            mrs	x3, SPMINTENCLR_EL1
-// CHECK:   mrs	x3, SPMINTENCLR_EL1             // encoding: [0x43,0x9e,0x30,0xd5]
-            msr	SPMINTENCLR_EL1, x1
-// CHECK:   msr	SPMINTENCLR_EL1, x1             // encoding: [0x41,0x9e,0x10,0xd5]
-            mrs	x3, SPMINTENSET_EL1
-// CHECK:   mrs	x3, SPMINTENSET_EL1             // encoding: [0x23,0x9e,0x30,0xd5]
-            msr	SPMINTENSET_EL1, x1
-// CHECK:   msr	SPMINTENSET_EL1, x1             // encoding: [0x21,0x9e,0x10,0xd5]
-            mrs	x3, SPMOVSCLR_EL0
-// CHECK:   mrs	x3, SPMOVSCLR_EL0               // encoding: [0x63,0x9c,0x33,0xd5]
-            msr	SPMOVSCLR_EL0, x1
-// CHECK:   msr	SPMOVSCLR_EL0, x1               // encoding: [0x61,0x9c,0x13,0xd5]
-            mrs	x3, SPMOVSSET_EL0
-// CHECK:   mrs	x3, SPMOVSSET_EL0               // encoding: [0x63,0x9e,0x33,0xd5]
-            msr	SPMOVSSET_EL0, x1
-// CHECK:   msr	SPMOVSSET_EL0, x1               // encoding: [0x61,0x9e,0x13,0xd5]
-            mrs	x3, SPMSELR_EL0
-// CHECK:   mrs	x3, SPMSELR_EL0                 // encoding: [0xa3,0x9c,0x33,0xd5]
-            msr	SPMSELR_EL0, x1
-// CHECK:   msr	SPMSELR_EL0, x1                 // encoding: [0xa1,0x9c,0x13,0xd5]
-            mrs x3, SPMCGCR0_EL1
-// CHECK:   mrs x3, SPMCGCR0_EL1                // encoding: [0x03,0x9d,0x30,0xd5]
-            mrs x3, SPMCGCR1_EL1
-// CHECK:   mrs x3, SPMCGCR1_EL1                // encoding: [0x23,0x9d,0x30,0xd5]
-            mrs x3, SPMCFGR_EL1
-// CHECK:   mrs x3, SPMCFGR_EL1                 // encoding: [0xe3,0x9d,0x30,0xd5]
-            mrs x3, SPMROOTCR_EL3
-// CHECK:   mrs x3, SPMROOTCR_EL3               // encoding: [0xe3,0x9e,0x36,0xd5]
-            msr SPMROOTCR_EL3, x3
-// CHECK:   msr SPMROOTCR_EL3, x3               // encoding: [0xe3,0x9e,0x16,0xd5]
-            mrs x3, SPMSCR_EL1
-// CHECK:   mrs x3, SPMSCR_EL1                  // encoding: [0xe3,0x9e,0x37,0xd5]
-            msr SPMSCR_EL1, x3
-// CHECK:   msr SPMSCR_EL1, x3                  // encoding: [0xe3,0x9e,0x17,0xd5]
+mrs x3, SPMACCESSR_EL1
+// CHECK-INST: mrs x3, SPMACCESSR_EL1
+// CHECK-ENCODING: encoding: [0x63,0x9d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5309d63      mrs x3, SPMACCESSR_EL1
+
+msr SPMACCESSR_EL1, x1
+// CHECK-INST: msr SPMACCESSR_EL1, x1
+// CHECK-ENCODING: encoding: [0x61,0x9d,0x10,0xd5]
+// CHECK-UNKNOWN:  d5109d61      msr SPMACCESSR_EL1, x1
+
+mrs x3, SPMACCESSR_EL12
+// CHECK-INST: mrs x3, SPMACCESSR_EL12
+// CHECK-ENCODING: encoding: [0x63,0x9d,0x35,0xd5]
+// CHECK-UNKNOWN:  d5359d63      mrs x3, SPMACCESSR_EL12
+
+msr SPMACCESSR_EL12, x1
+// CHECK-INST: msr SPMACCESSR_EL12, x1
+// CHECK-ENCODING: encoding: [0x61,0x9d,0x15,0xd5]
+// CHECK-UNKNOWN:  d5159d61      msr SPMACCESSR_EL12, x1
+
+mrs x3, SPMACCESSR_EL2
+// CHECK-INST: mrs x3, SPMACCESSR_EL2
+// CHECK-ENCODING: encoding: [0x63,0x9d,0x34,0xd5]
+// CHECK-UNKNOWN:  d5349d63      mrs x3, SPMACCESSR_EL2
+
+msr SPMACCESSR_EL2, x1
+// CHECK-INST: msr SPMACCESSR_EL2, x1
+// CHECK-ENCODING: encoding: [0x61,0x9d,0x14,0xd5]
+// CHECK-UNKNOWN:  d5149d61      msr SPMACCESSR_EL2, x1
+
+mrs x3, SPMACCESSR_EL3
+// CHECK-INST: mrs x3, SPMACCESSR_EL3
+// CHECK-ENCODING: encoding: [0x63,0x9d,0x36,0xd5]
+// CHECK-UNKNOWN:  d5369d63      mrs x3, SPMACCESSR_EL3
+
+msr SPMACCESSR_EL3, x1
+// CHECK-INST: msr SPMACCESSR_EL3, x1
+// CHECK-ENCODING: encoding: [0x61,0x9d,0x16,0xd5]
+// CHECK-UNKNOWN:  d5169d61      msr SPMACCESSR_EL3, x1
+
+mrs x3, SPMCNTENCLR_EL0
+// CHECK-INST: mrs x3, SPMCNTENCLR_EL0
+// CHECK-ENCODING: encoding: [0x43,0x9c,0x33,0xd5]
+// CHECK-UNKNOWN:  d5339c43      mrs x3, SPMCNTENCLR_EL0
+
+msr SPMCNTENCLR_EL0, x1
+// CHECK-INST: msr SPMCNTENCLR_EL0, x1
+// CHECK-ENCODING: encoding: [0x41,0x9c,0x13,0xd5]
+// CHECK-UNKNOWN:  d5139c41      msr SPMCNTENCLR_EL0, x1
+
+mrs x3, SPMCNTENSET_EL0
+// CHECK-INST: mrs x3, SPMCNTENSET_EL0
+// CHECK-ENCODING: encoding: [0x23,0x9c,0x33,0xd5]
+// CHECK-UNKNOWN:  d5339c23      mrs x3, SPMCNTENSET_EL0
+
+msr SPMCNTENSET_EL0, x1
+// CHECK-INST: msr SPMCNTENSET_EL0, x1
+// CHECK-ENCODING: encoding: [0x21,0x9c,0x13,0xd5]
+// CHECK-UNKNOWN:  d5139c21      msr SPMCNTENSET_EL0, x1
+
+mrs x3, SPMCR_EL0
+// CHECK-INST: mrs x3, SPMCR_EL0
+// CHECK-ENCODING: encoding: [0x03,0x9c,0x33,0xd5]
+// CHECK-UNKNOWN:  d5339c03      mrs x3, SPMCR_EL0
+
+msr SPMCR_EL0, x1
+// CHECK-INST: msr SPMCR_EL0, x1
+// CHECK-ENCODING: encoding: [0x01,0x9c,0x13,0xd5]
+// CHECK-UNKNOWN:  d5139c01      msr SPMCR_EL0, x1
+
+mrs x3, SPMDEVAFF_EL1
+// CHECK-INST: mrs x3, SPMDEVAFF_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x9d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5309dc3      mrs x3, SPMDEVAFF_EL1
+
+mrs x3, SPMDEVARCH_EL1
+// CHECK-INST: mrs x3, SPMDEVARCH_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x9d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5309da3      mrs x3, SPMDEVARCH_EL1
+
+mrs x3, SPMEVCNTR0_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR0_EL0
+// CHECK-ENCODING: encoding: [0x03,0xe0,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e003      mrs x3, SPMEVCNTR0_EL0
+
+msr SPMEVCNTR0_EL0, x1
+// CHECK-INST: msr SPMEVCNTR0_EL0, x1
+// CHECK-ENCODING: encoding: [0x01,0xe0,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e001      msr SPMEVCNTR0_EL0, x1
+
+mrs x3, SPMEVCNTR1_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR1_EL0
+// CHECK-ENCODING: encoding: [0x23,0xe0,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e023      mrs x3, SPMEVCNTR1_EL0
+
+msr SPMEVCNTR1_EL0, x1
+// CHECK-INST: msr SPMEVCNTR1_EL0, x1
+// CHECK-ENCODING: encoding: [0x21,0xe0,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e021      msr SPMEVCNTR1_EL0, x1
+
+mrs x3, SPMEVCNTR2_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR2_EL0
+// CHECK-ENCODING: encoding: [0x43,0xe0,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e043      mrs x3, SPMEVCNTR2_EL0
+
+msr SPMEVCNTR2_EL0, x1
+// CHECK-INST: msr SPMEVCNTR2_EL0, x1
+// CHECK-ENCODING: encoding: [0x41,0xe0,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e041      msr SPMEVCNTR2_EL0, x1
+
+mrs x3, SPMEVCNTR3_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR3_EL0
+// CHECK-ENCODING: encoding: [0x63,0xe0,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e063      mrs x3, SPMEVCNTR3_EL0
+
+msr SPMEVCNTR3_EL0, x1
+// CHECK-INST: msr SPMEVCNTR3_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0xe0,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e061      msr SPMEVCNTR3_EL0, x1
+
+mrs x3, SPMEVCNTR4_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR4_EL0
+// CHECK-ENCODING: encoding: [0x83,0xe0,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e083      mrs x3, SPMEVCNTR4_EL0
+
+msr SPMEVCNTR4_EL0, x1
+// CHECK-INST: msr SPMEVCNTR4_EL0, x1
+// CHECK-ENCODING: encoding: [0x81,0xe0,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e081      msr SPMEVCNTR4_EL0, x1
+
+mrs x3, SPMEVCNTR5_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR5_EL0
+// CHECK-ENCODING: encoding: [0xa3,0xe0,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e0a3      mrs x3, SPMEVCNTR5_EL0
+
+msr SPMEVCNTR5_EL0, x1
+// CHECK-INST: msr SPMEVCNTR5_EL0, x1
+// CHECK-ENCODING: encoding: [0xa1,0xe0,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e0a1      msr SPMEVCNTR5_EL0, x1
+
+mrs x3, SPMEVCNTR6_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR6_EL0
+// CHECK-ENCODING: encoding: [0xc3,0xe0,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e0c3      mrs x3, SPMEVCNTR6_EL0
+
+msr SPMEVCNTR6_EL0, x1
+// CHECK-INST: msr SPMEVCNTR6_EL0, x1
+// CHECK-ENCODING: encoding: [0xc1,0xe0,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e0c1      msr SPMEVCNTR6_EL0, x1
+
+mrs x3, SPMEVCNTR7_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR7_EL0
+// CHECK-ENCODING: encoding: [0xe3,0xe0,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e0e3      mrs x3, SPMEVCNTR7_EL0
+
+msr SPMEVCNTR7_EL0, x1
+// CHECK-INST: msr SPMEVCNTR7_EL0, x1
+// CHECK-ENCODING: encoding: [0xe1,0xe0,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e0e1      msr SPMEVCNTR7_EL0, x1
+
+mrs x3, SPMEVCNTR8_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR8_EL0
+// CHECK-ENCODING: encoding: [0x03,0xe1,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e103      mrs x3, SPMEVCNTR8_EL0
+
+msr SPMEVCNTR8_EL0, x1
+// CHECK-INST: msr SPMEVCNTR8_EL0, x1
+// CHECK-ENCODING: encoding: [0x01,0xe1,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e101      msr SPMEVCNTR8_EL0, x1
+
+mrs x3, SPMEVCNTR9_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR9_EL0
+// CHECK-ENCODING: encoding: [0x23,0xe1,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e123      mrs x3, SPMEVCNTR9_EL0
+
+msr SPMEVCNTR9_EL0, x1
+// CHECK-INST: msr SPMEVCNTR9_EL0, x1
+// CHECK-ENCODING: encoding: [0x21,0xe1,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e121      msr SPMEVCNTR9_EL0, x1
+
+mrs x3, SPMEVCNTR10_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR10_EL0
+// CHECK-ENCODING: encoding: [0x43,0xe1,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e143      mrs x3, SPMEVCNTR10_EL0
+
+msr SPMEVCNTR10_EL0, x1
+// CHECK-INST: msr SPMEVCNTR10_EL0, x1
+// CHECK-ENCODING: encoding: [0x41,0xe1,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e141      msr SPMEVCNTR10_EL0, x1
+
+mrs x3, SPMEVCNTR11_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR11_EL0
+// CHECK-ENCODING: encoding: [0x63,0xe1,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e163      mrs x3, SPMEVCNTR11_EL0
+
+msr SPMEVCNTR11_EL0, x1
+// CHECK-INST: msr SPMEVCNTR11_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0xe1,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e161      msr SPMEVCNTR11_EL0, x1
+
+mrs x3, SPMEVCNTR12_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR12_EL0
+// CHECK-ENCODING: encoding: [0x83,0xe1,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e183      mrs x3, SPMEVCNTR12_EL0
+
+msr SPMEVCNTR12_EL0, x1
+// CHECK-INST: msr SPMEVCNTR12_EL0, x1
+// CHECK-ENCODING: encoding: [0x81,0xe1,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e181      msr SPMEVCNTR12_EL0, x1
+
+mrs x3, SPMEVCNTR13_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR13_EL0
+// CHECK-ENCODING: encoding: [0xa3,0xe1,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e1a3      mrs x3, SPMEVCNTR13_EL0
+
+msr SPMEVCNTR13_EL0, x1
+// CHECK-INST: msr SPMEVCNTR13_EL0, x1
+// CHECK-ENCODING: encoding: [0xa1,0xe1,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e1a1      msr SPMEVCNTR13_EL0, x1
+
+mrs x3, SPMEVCNTR14_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR14_EL0
+// CHECK-ENCODING: encoding: [0xc3,0xe1,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e1c3      mrs x3, SPMEVCNTR14_EL0
+
+msr SPMEVCNTR14_EL0, x1
+// CHECK-INST: msr SPMEVCNTR14_EL0, x1
+// CHECK-ENCODING: encoding: [0xc1,0xe1,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e1c1      msr SPMEVCNTR14_EL0, x1
+
+mrs x3, SPMEVCNTR15_EL0
+// CHECK-INST: mrs x3, SPMEVCNTR15_EL0
+// CHECK-ENCODING: encoding: [0xe3,0xe1,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e1e3      mrs x3, SPMEVCNTR15_EL0
+
+msr SPMEVCNTR15_EL0, x1
+// CHECK-INST: msr SPMEVCNTR15_EL0, x1
+// CHECK-ENCODING: encoding: [0xe1,0xe1,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e1e1      msr SPMEVCNTR15_EL0, x1
+
+mrs x3, SPMEVFILT2R0_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R0_EL0
+// CHECK-ENCODING: encoding: [0x03,0xe6,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e603      mrs x3, SPMEVFILT2R0_EL0
+
+msr SPMEVFILT2R0_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R0_EL0, x1
+// CHECK-ENCODING: encoding: [0x01,0xe6,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e601      msr SPMEVFILT2R0_EL0, x1
+
+mrs x3, SPMEVFILT2R1_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R1_EL0
+// CHECK-ENCODING: encoding: [0x23,0xe6,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e623      mrs x3, SPMEVFILT2R1_EL0
+
+msr SPMEVFILT2R1_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R1_EL0, x1
+// CHECK-ENCODING: encoding: [0x21,0xe6,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e621      msr SPMEVFILT2R1_EL0, x1
+
+mrs x3, SPMEVFILT2R2_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R2_EL0
+// CHECK-ENCODING: encoding: [0x43,0xe6,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e643      mrs x3, SPMEVFILT2R2_EL0
+
+msr SPMEVFILT2R2_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R2_EL0, x1
+// CHECK-ENCODING: encoding: [0x41,0xe6,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e641      msr SPMEVFILT2R2_EL0, x1
+
+mrs x3, SPMEVFILT2R3_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R3_EL0
+// CHECK-ENCODING: encoding: [0x63,0xe6,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e663      mrs x3, SPMEVFILT2R3_EL0
+
+msr SPMEVFILT2R3_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R3_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0xe6,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e661      msr SPMEVFILT2R3_EL0, x1
+
+mrs x3, SPMEVFILT2R4_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R4_EL0
+// CHECK-ENCODING: encoding: [0x83,0xe6,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e683      mrs x3, SPMEVFILT2R4_EL0
+
+msr SPMEVFILT2R4_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R4_EL0, x1
+// CHECK-ENCODING: encoding: [0x81,0xe6,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e681      msr SPMEVFILT2R4_EL0, x1
+
+mrs x3, SPMEVFILT2R5_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R5_EL0
+// CHECK-ENCODING: encoding: [0xa3,0xe6,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e6a3      mrs x3, SPMEVFILT2R5_EL0
+
+msr SPMEVFILT2R5_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R5_EL0, x1
+// CHECK-ENCODING: encoding: [0xa1,0xe6,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e6a1      msr SPMEVFILT2R5_EL0, x1
+
+mrs x3, SPMEVFILT2R6_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R6_EL0
+// CHECK-ENCODING: encoding: [0xc3,0xe6,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e6c3      mrs x3, SPMEVFILT2R6_EL0
+
+msr SPMEVFILT2R6_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R6_EL0, x1
+// CHECK-ENCODING: encoding: [0xc1,0xe6,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e6c1      msr SPMEVFILT2R6_EL0, x1
+
+mrs x3, SPMEVFILT2R7_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R7_EL0
+// CHECK-ENCODING: encoding: [0xe3,0xe6,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e6e3      mrs x3, SPMEVFILT2R7_EL0
+
+msr SPMEVFILT2R7_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R7_EL0, x1
+// CHECK-ENCODING: encoding: [0xe1,0xe6,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e6e1      msr SPMEVFILT2R7_EL0, x1
+
+mrs x3, SPMEVFILT2R8_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R8_EL0
+// CHECK-ENCODING: encoding: [0x03,0xe7,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e703      mrs x3, SPMEVFILT2R8_EL0
+
+msr SPMEVFILT2R8_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R8_EL0, x1
+// CHECK-ENCODING: encoding: [0x01,0xe7,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e701      msr SPMEVFILT2R8_EL0, x1
+
+mrs x3, SPMEVFILT2R9_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R9_EL0
+// CHECK-ENCODING: encoding: [0x23,0xe7,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e723      mrs x3, SPMEVFILT2R9_EL0
+
+msr SPMEVFILT2R9_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R9_EL0, x1
+// CHECK-ENCODING: encoding: [0x21,0xe7,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e721      msr SPMEVFILT2R9_EL0, x1
+
+mrs x3, SPMEVFILT2R10_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R10_EL0
+// CHECK-ENCODING: encoding: [0x43,0xe7,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e743      mrs x3, SPMEVFILT2R10_EL0
+
+msr SPMEVFILT2R10_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R10_EL0, x1
+// CHECK-ENCODING: encoding: [0x41,0xe7,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e741      msr SPMEVFILT2R10_EL0, x1
+
+mrs x3, SPMEVFILT2R11_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R11_EL0
+// CHECK-ENCODING: encoding: [0x63,0xe7,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e763      mrs x3, SPMEVFILT2R11_EL0
+
+msr SPMEVFILT2R11_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R11_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0xe7,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e761      msr SPMEVFILT2R11_EL0, x1
+
+mrs x3, SPMEVFILT2R12_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R12_EL0
+// CHECK-ENCODING: encoding: [0x83,0xe7,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e783      mrs x3, SPMEVFILT2R12_EL0
+
+msr SPMEVFILT2R12_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R12_EL0, x1
+// CHECK-ENCODING: encoding: [0x81,0xe7,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e781      msr SPMEVFILT2R12_EL0, x1
+
+mrs x3, SPMEVFILT2R13_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R13_EL0
+// CHECK-ENCODING: encoding: [0xa3,0xe7,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e7a3      mrs x3, SPMEVFILT2R13_EL0
+
+msr SPMEVFILT2R13_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R13_EL0, x1
+// CHECK-ENCODING: encoding: [0xa1,0xe7,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e7a1      msr SPMEVFILT2R13_EL0, x1
+
+mrs x3, SPMEVFILT2R14_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R14_EL0
+// CHECK-ENCODING: encoding: [0xc3,0xe7,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e7c3      mrs x3, SPMEVFILT2R14_EL0
+
+msr SPMEVFILT2R14_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R14_EL0, x1
+// CHECK-ENCODING: encoding: [0xc1,0xe7,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e7c1      msr SPMEVFILT2R14_EL0, x1
+
+mrs x3, SPMEVFILT2R15_EL0
+// CHECK-INST: mrs x3, SPMEVFILT2R15_EL0
+// CHECK-ENCODING: encoding: [0xe3,0xe7,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e7e3      mrs x3, SPMEVFILT2R15_EL0
+
+msr SPMEVFILT2R15_EL0, x1
+// CHECK-INST: msr SPMEVFILT2R15_EL0, x1
+// CHECK-ENCODING: encoding: [0xe1,0xe7,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e7e1      msr SPMEVFILT2R15_EL0, x1
+
+mrs x3, SPMEVFILTR0_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR0_EL0
+// CHECK-ENCODING: encoding: [0x03,0xe4,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e403      mrs x3, SPMEVFILTR0_EL0
+
+msr SPMEVFILTR0_EL0, x1
+// CHECK-INST: msr SPMEVFILTR0_EL0, x1
+// CHECK-ENCODING: encoding: [0x01,0xe4,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e401      msr SPMEVFILTR0_EL0, x1
+
+mrs x3, SPMEVFILTR1_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR1_EL0
+// CHECK-ENCODING: encoding: [0x23,0xe4,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e423      mrs x3, SPMEVFILTR1_EL0
+
+msr SPMEVFILTR1_EL0, x1
+// CHECK-INST: msr SPMEVFILTR1_EL0, x1
+// CHECK-ENCODING: encoding: [0x21,0xe4,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e421      msr SPMEVFILTR1_EL0, x1
+
+mrs x3, SPMEVFILTR2_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR2_EL0
+// CHECK-ENCODING: encoding: [0x43,0xe4,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e443      mrs x3, SPMEVFILTR2_EL0
+
+msr SPMEVFILTR2_EL0, x1
+// CHECK-INST: msr SPMEVFILTR2_EL0, x1
+// CHECK-ENCODING: encoding: [0x41,0xe4,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e441      msr SPMEVFILTR2_EL0, x1
+
+mrs x3, SPMEVFILTR3_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR3_EL0
+// CHECK-ENCODING: encoding: [0x63,0xe4,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e463      mrs x3, SPMEVFILTR3_EL0
+
+msr SPMEVFILTR3_EL0, x1
+// CHECK-INST: msr SPMEVFILTR3_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0xe4,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e461      msr SPMEVFILTR3_EL0, x1
+
+mrs x3, SPMEVFILTR4_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR4_EL0
+// CHECK-ENCODING: encoding: [0x83,0xe4,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e483      mrs x3, SPMEVFILTR4_EL0
+
+msr SPMEVFILTR4_EL0, x1
+// CHECK-INST: msr SPMEVFILTR4_EL0, x1
+// CHECK-ENCODING: encoding: [0x81,0xe4,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e481      msr SPMEVFILTR4_EL0, x1
+
+mrs x3, SPMEVFILTR5_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR5_EL0
+// CHECK-ENCODING: encoding: [0xa3,0xe4,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e4a3      mrs x3, SPMEVFILTR5_EL0
+
+msr SPMEVFILTR5_EL0, x1
+// CHECK-INST: msr SPMEVFILTR5_EL0, x1
+// CHECK-ENCODING: encoding: [0xa1,0xe4,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e4a1      msr SPMEVFILTR5_EL0, x1
+
+mrs x3, SPMEVFILTR6_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR6_EL0
+// CHECK-ENCODING: encoding: [0xc3,0xe4,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e4c3      mrs x3, SPMEVFILTR6_EL0
+
+msr SPMEVFILTR6_EL0, x1
+// CHECK-INST: msr SPMEVFILTR6_EL0, x1
+// CHECK-ENCODING: encoding: [0xc1,0xe4,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e4c1      msr SPMEVFILTR6_EL0, x1
+
+mrs x3, SPMEVFILTR7_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR7_EL0
+// CHECK-ENCODING: encoding: [0xe3,0xe4,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e4e3      mrs x3, SPMEVFILTR7_EL0
+
+msr SPMEVFILTR7_EL0, x1
+// CHECK-INST: msr SPMEVFILTR7_EL0, x1
+// CHECK-ENCODING: encoding: [0xe1,0xe4,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e4e1      msr SPMEVFILTR7_EL0, x1
+
+mrs x3, SPMEVFILTR8_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR8_EL0
+// CHECK-ENCODING: encoding: [0x03,0xe5,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e503      mrs x3, SPMEVFILTR8_EL0
+
+msr SPMEVFILTR8_EL0, x1
+// CHECK-INST: msr SPMEVFILTR8_EL0, x1
+// CHECK-ENCODING: encoding: [0x01,0xe5,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e501      msr SPMEVFILTR8_EL0, x1
+
+mrs x3, SPMEVFILTR9_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR9_EL0
+// CHECK-ENCODING: encoding: [0x23,0xe5,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e523      mrs x3, SPMEVFILTR9_EL0
+
+msr SPMEVFILTR9_EL0, x1
+// CHECK-INST: msr SPMEVFILTR9_EL0, x1
+// CHECK-ENCODING: encoding: [0x21,0xe5,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e521      msr SPMEVFILTR9_EL0, x1
+
+mrs x3, SPMEVFILTR10_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR10_EL0
+// CHECK-ENCODING: encoding: [0x43,0xe5,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e543      mrs x3, SPMEVFILTR10_EL0
+
+msr SPMEVFILTR10_EL0, x1
+// CHECK-INST: msr SPMEVFILTR10_EL0, x1
+// CHECK-ENCODING: encoding: [0x41,0xe5,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e541      msr SPMEVFILTR10_EL0, x1
+
+mrs x3, SPMEVFILTR11_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR11_EL0
+// CHECK-ENCODING: encoding: [0x63,0xe5,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e563      mrs x3, SPMEVFILTR11_EL0
+
+msr SPMEVFILTR11_EL0, x1
+// CHECK-INST: msr SPMEVFILTR11_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0xe5,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e561      msr SPMEVFILTR11_EL0, x1
+
+mrs x3, SPMEVFILTR12_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR12_EL0
+// CHECK-ENCODING: encoding: [0x83,0xe5,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e583      mrs x3, SPMEVFILTR12_EL0
+
+msr SPMEVFILTR12_EL0, x1
+// CHECK-INST: msr SPMEVFILTR12_EL0, x1
+// CHECK-ENCODING: encoding: [0x81,0xe5,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e581      msr SPMEVFILTR12_EL0, x1
+
+mrs x3, SPMEVFILTR13_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR13_EL0
+// CHECK-ENCODING: encoding: [0xa3,0xe5,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e5a3      mrs x3, SPMEVFILTR13_EL0
+
+msr SPMEVFILTR13_EL0, x1
+// CHECK-INST: msr SPMEVFILTR13_EL0, x1
+// CHECK-ENCODING: encoding: [0xa1,0xe5,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e5a1      msr SPMEVFILTR13_EL0, x1
+
+mrs x3, SPMEVFILTR14_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR14_EL0
+// CHECK-ENCODING: encoding: [0xc3,0xe5,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e5c3      mrs x3, SPMEVFILTR14_EL0
+
+msr SPMEVFILTR14_EL0, x1
+// CHECK-INST: msr SPMEVFILTR14_EL0, x1
+// CHECK-ENCODING: encoding: [0xc1,0xe5,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e5c1      msr SPMEVFILTR14_EL0, x1
+
+mrs x3, SPMEVFILTR15_EL0
+// CHECK-INST: mrs x3, SPMEVFILTR15_EL0
+// CHECK-ENCODING: encoding: [0xe3,0xe5,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e5e3      mrs x3, SPMEVFILTR15_EL0
+
+msr SPMEVFILTR15_EL0, x1
+// CHECK-INST: msr SPMEVFILTR15_EL0, x1
+// CHECK-ENCODING: encoding: [0xe1,0xe5,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e5e1      msr SPMEVFILTR15_EL0, x1
+
+mrs x3, SPMEVTYPER0_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER0_EL0
+// CHECK-ENCODING: encoding: [0x03,0xe2,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e203      mrs x3, SPMEVTYPER0_EL0
+
+msr SPMEVTYPER0_EL0, x1
+// CHECK-INST: msr SPMEVTYPER0_EL0, x1
+// CHECK-ENCODING: encoding: [0x01,0xe2,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e201      msr SPMEVTYPER0_EL0, x1
+
+mrs x3, SPMEVTYPER1_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER1_EL0
+// CHECK-ENCODING: encoding: [0x23,0xe2,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e223      mrs x3, SPMEVTYPER1_EL0
+
+msr SPMEVTYPER1_EL0, x1
+// CHECK-INST: msr SPMEVTYPER1_EL0, x1
+// CHECK-ENCODING: encoding: [0x21,0xe2,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e221      msr SPMEVTYPER1_EL0, x1
+
+mrs x3, SPMEVTYPER2_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER2_EL0
+// CHECK-ENCODING: encoding: [0x43,0xe2,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e243      mrs x3, SPMEVTYPER2_EL0
+
+msr SPMEVTYPER2_EL0, x1
+// CHECK-INST: msr SPMEVTYPER2_EL0, x1
+// CHECK-ENCODING: encoding: [0x41,0xe2,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e241      msr SPMEVTYPER2_EL0, x1
+
+mrs x3, SPMEVTYPER3_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER3_EL0
+// CHECK-ENCODING: encoding: [0x63,0xe2,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e263      mrs x3, SPMEVTYPER3_EL0
+
+msr SPMEVTYPER3_EL0, x1
+// CHECK-INST: msr SPMEVTYPER3_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0xe2,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e261      msr SPMEVTYPER3_EL0, x1
+
+mrs x3, SPMEVTYPER4_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER4_EL0
+// CHECK-ENCODING: encoding: [0x83,0xe2,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e283      mrs x3, SPMEVTYPER4_EL0
+
+msr SPMEVTYPER4_EL0, x1
+// CHECK-INST: msr SPMEVTYPER4_EL0, x1
+// CHECK-ENCODING: encoding: [0x81,0xe2,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e281      msr SPMEVTYPER4_EL0, x1
+
+mrs x3, SPMEVTYPER5_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER5_EL0
+// CHECK-ENCODING: encoding: [0xa3,0xe2,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e2a3      mrs x3, SPMEVTYPER5_EL0
+
+msr SPMEVTYPER5_EL0, x1
+// CHECK-INST: msr SPMEVTYPER5_EL0, x1
+// CHECK-ENCODING: encoding: [0xa1,0xe2,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e2a1      msr SPMEVTYPER5_EL0, x1
+
+mrs x3, SPMEVTYPER6_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER6_EL0
+// CHECK-ENCODING: encoding: [0xc3,0xe2,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e2c3      mrs x3, SPMEVTYPER6_EL0
+
+msr SPMEVTYPER6_EL0, x1
+// CHECK-INST: msr SPMEVTYPER6_EL0, x1
+// CHECK-ENCODING: encoding: [0xc1,0xe2,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e2c1      msr SPMEVTYPER6_EL0, x1
+
+mrs x3, SPMEVTYPER7_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER7_EL0
+// CHECK-ENCODING: encoding: [0xe3,0xe2,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e2e3      mrs x3, SPMEVTYPER7_EL0
+
+msr SPMEVTYPER7_EL0, x1
+// CHECK-INST: msr SPMEVTYPER7_EL0, x1
+// CHECK-ENCODING: encoding: [0xe1,0xe2,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e2e1      msr SPMEVTYPER7_EL0, x1
+
+mrs x3, SPMEVTYPER8_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER8_EL0
+// CHECK-ENCODING: encoding: [0x03,0xe3,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e303      mrs x3, SPMEVTYPER8_EL0
+
+msr SPMEVTYPER8_EL0, x1
+// CHECK-INST: msr SPMEVTYPER8_EL0, x1
+// CHECK-ENCODING: encoding: [0x01,0xe3,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e301      msr SPMEVTYPER8_EL0, x1
+
+mrs x3, SPMEVTYPER9_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER9_EL0
+// CHECK-ENCODING: encoding: [0x23,0xe3,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e323      mrs x3, SPMEVTYPER9_EL0
+
+msr SPMEVTYPER9_EL0, x1
+// CHECK-INST: msr SPMEVTYPER9_EL0, x1
+// CHECK-ENCODING: encoding: [0x21,0xe3,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e321      msr SPMEVTYPER9_EL0, x1
+
+mrs x3, SPMEVTYPER10_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER10_EL0
+// CHECK-ENCODING: encoding: [0x43,0xe3,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e343      mrs x3, SPMEVTYPER10_EL0
+
+msr SPMEVTYPER10_EL0, x1
+// CHECK-INST: msr SPMEVTYPER10_EL0, x1
+// CHECK-ENCODING: encoding: [0x41,0xe3,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e341      msr SPMEVTYPER10_EL0, x1
+
+mrs x3, SPMEVTYPER11_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER11_EL0
+// CHECK-ENCODING: encoding: [0x63,0xe3,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e363      mrs x3, SPMEVTYPER11_EL0
+
+msr SPMEVTYPER11_EL0, x1
+// CHECK-INST: msr SPMEVTYPER11_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0xe3,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e361      msr SPMEVTYPER11_EL0, x1
+
+mrs x3, SPMEVTYPER12_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER12_EL0
+// CHECK-ENCODING: encoding: [0x83,0xe3,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e383      mrs x3, SPMEVTYPER12_EL0
+
+msr SPMEVTYPER12_EL0, x1
+// CHECK-INST: msr SPMEVTYPER12_EL0, x1
+// CHECK-ENCODING: encoding: [0x81,0xe3,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e381      msr SPMEVTYPER12_EL0, x1
+
+mrs x3, SPMEVTYPER13_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER13_EL0
+// CHECK-ENCODING: encoding: [0xa3,0xe3,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e3a3      mrs x3, SPMEVTYPER13_EL0
+
+msr SPMEVTYPER13_EL0, x1
+// CHECK-INST: msr SPMEVTYPER13_EL0, x1
+// CHECK-ENCODING: encoding: [0xa1,0xe3,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e3a1      msr SPMEVTYPER13_EL0, x1
+
+mrs x3, SPMEVTYPER14_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER14_EL0
+// CHECK-ENCODING: encoding: [0xc3,0xe3,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e3c3      mrs x3, SPMEVTYPER14_EL0
+
+msr SPMEVTYPER14_EL0, x1
+// CHECK-INST: msr SPMEVTYPER14_EL0, x1
+// CHECK-ENCODING: encoding: [0xc1,0xe3,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e3c1      msr SPMEVTYPER14_EL0, x1
+
+mrs x3, SPMEVTYPER15_EL0
+// CHECK-INST: mrs x3, SPMEVTYPER15_EL0
+// CHECK-ENCODING: encoding: [0xe3,0xe3,0x33,0xd5]
+// CHECK-UNKNOWN:  d533e3e3      mrs x3, SPMEVTYPER15_EL0
+
+msr SPMEVTYPER15_EL0, x1
+// CHECK-INST: msr SPMEVTYPER15_EL0, x1
+// CHECK-ENCODING: encoding: [0xe1,0xe3,0x13,0xd5]
+// CHECK-UNKNOWN:  d513e3e1      msr SPMEVTYPER15_EL0, x1
+
+mrs x3, SPMIIDR_EL1
+// CHECK-INST: mrs x3, SPMIIDR_EL1
+// CHECK-ENCODING: encoding: [0x83,0x9d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5309d83      mrs x3, SPMIIDR_EL1
+
+mrs x3, SPMINTENCLR_EL1
+// CHECK-INST: mrs x3, SPMINTENCLR_EL1
+// CHECK-ENCODING: encoding: [0x43,0x9e,0x30,0xd5]
+// CHECK-UNKNOWN:  d5309e43      mrs x3, SPMINTENCLR_EL1
+
+msr SPMINTENCLR_EL1, x1
+// CHECK-INST: msr SPMINTENCLR_EL1, x1
+// CHECK-ENCODING: encoding: [0x41,0x9e,0x10,0xd5]
+// CHECK-UNKNOWN:  d5109e41      msr SPMINTENCLR_EL1, x1
+
+mrs x3, SPMINTENSET_EL1
+// CHECK-INST: mrs x3, SPMINTENSET_EL1
+// CHECK-ENCODING: encoding: [0x23,0x9e,0x30,0xd5]
+// CHECK-UNKNOWN:  d5309e23      mrs x3, SPMINTENSET_EL1
+
+msr SPMINTENSET_EL1, x1
+// CHECK-INST: msr SPMINTENSET_EL1, x1
+// CHECK-ENCODING: encoding: [0x21,0x9e,0x10,0xd5]
+// CHECK-UNKNOWN:  d5109e21      msr SPMINTENSET_EL1, x1
+
+mrs x3, SPMOVSCLR_EL0
+// CHECK-INST: mrs x3, SPMOVSCLR_EL0
+// CHECK-ENCODING: encoding: [0x63,0x9c,0x33,0xd5]
+// CHECK-UNKNOWN:  d5339c63      mrs x3, SPMOVSCLR_EL0
+
+msr SPMOVSCLR_EL0, x1
+// CHECK-INST: msr SPMOVSCLR_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0x9c,0x13,0xd5]
+// CHECK-UNKNOWN:  d5139c61      msr SPMOVSCLR_EL0, x1
+
+mrs x3, SPMOVSSET_EL0
+// CHECK-INST: mrs x3, SPMOVSSET_EL0
+// CHECK-ENCODING: encoding: [0x63,0x9e,0x33,0xd5]
+// CHECK-UNKNOWN:  d5339e63      mrs x3, SPMOVSSET_EL0
+
+msr SPMOVSSET_EL0, x1
+// CHECK-INST: msr SPMOVSSET_EL0, x1
+// CHECK-ENCODING: encoding: [0x61,0x9e,0x13,0xd5]
+// CHECK-UNKNOWN:  d5139e61      msr SPMOVSSET_EL0, x1
+
+mrs x3, SPMSELR_EL0
+// CHECK-INST: mrs x3, SPMSELR_EL0
+// CHECK-ENCODING: encoding: [0xa3,0x9c,0x33,0xd5]
+// CHECK-UNKNOWN:  d5339ca3      mrs x3, SPMSELR_EL0
+
+msr SPMSELR_EL0, x1
+// CHECK-INST: msr SPMSELR_EL0, x1
+// CHECK-ENCODING: encoding: [0xa1,0x9c,0x13,0xd5]
+// CHECK-UNKNOWN:  d5139ca1      msr SPMSELR_EL0, x1
+
+mrs x3, SPMCGCR0_EL1
+// CHECK-INST: mrs x3, SPMCGCR0_EL1
+// CHECK-ENCODING: encoding: [0x03,0x9d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5309d03      mrs x3, SPMCGCR0_EL1
+
+mrs x3, SPMCGCR1_EL1
+// CHECK-INST: mrs x3, SPMCGCR1_EL1
+// CHECK-ENCODING: encoding: [0x23,0x9d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5309d23      mrs x3, SPMCGCR1_EL1
+
+mrs x3, SPMCFGR_EL1
+// CHECK-INST: mrs x3, SPMCFGR_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x9d,0x30,0xd5]
+// CHECK-UNKNOWN:  d5309de3      mrs x3, SPMCFGR_EL1
+
+mrs x3, SPMROOTCR_EL3
+// CHECK-INST: mrs x3, SPMROOTCR_EL3
+// CHECK-ENCODING: encoding: [0xe3,0x9e,0x36,0xd5]
+// CHECK-UNKNOWN:  d5369ee3      mrs x3, SPMROOTCR_EL3
+
+msr SPMROOTCR_EL3, x3
+// CHECK-INST: msr SPMROOTCR_EL3, x3
+// CHECK-ENCODING: encoding: [0xe3,0x9e,0x16,0xd5]
+// CHECK-UNKNOWN:  d5169ee3      msr SPMROOTCR_EL3, x3
+
+mrs x3, SPMSCR_EL1
+// CHECK-INST: mrs x3, SPMSCR_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x9e,0x37,0xd5]
+// CHECK-UNKNOWN:  d5379ee3      mrs x3, SPMSCR_EL1
+
+msr SPMSCR_EL1, x3
+// CHECK-INST: msr SPMSCR_EL1, x3
+// CHECK-ENCODING: encoding: [0xe3,0x9e,0x17,0xd5]
+// CHECK-UNKNOWN:  d5179ee3      msr SPMSCR_EL1, x3
 
 // FEAT_ITE
-            mrs x3, TRCITEEDCR
-// CHECK:   mrs x3, TRCITEEDCR                  // encoding: [0x23,0x02,0x31,0xd5]
-// ERROR-NO-ITE: [[@LINE-2]]:21: error: expected readable system register
-            msr TRCITEEDCR, x3
-// CHECK:   msr TRCITEEDCR, x3                  // encoding: [0x23,0x02,0x11,0xd5]
-// ERROR-NO-ITE: [[@LINE-2]]:17: error: expected writable system register
-            mrs	x3, TRCITECR_EL1
-// CHECK:   mrs	x3, TRCITECR_EL1                // encoding: [0x63,0x12,0x38,0xd5]
-// ERROR-NO-ITE: [[@LINE-2]]:21: error: expected readable system register
-            msr	TRCITECR_EL1, x1
-// CHECK:   msr	TRCITECR_EL1, x1                // encoding: [0x61,0x12,0x18,0xd5]
-// ERROR-NO-ITE: [[@LINE-2]]:17: error: expected writable system register or pstate
-            mrs	x3, TRCITECR_EL12
-// CHECK:   mrs	x3, TRCITECR_EL12               // encoding: [0x63,0x12,0x3d,0xd5]
-// ERROR-NO-ITE: [[@LINE-2]]:21: error: expected readable system register
-            msr	TRCITECR_EL12, x1
-// CHECK:   msr	TRCITECR_EL12, x1               // encoding: [0x61,0x12,0x1d,0xd5]
-// ERROR-NO-ITE: [[@LINE-2]]:17: error: expected writable system register or pstate
-            mrs	x3, TRCITECR_EL2
-// CHECK:   mrs	x3, TRCITECR_EL2                // encoding: [0x63,0x12,0x3c,0xd5]
-// ERROR-NO-ITE: [[@LINE-2]]:21: error: expected readable system register
-            msr	TRCITECR_EL2, x1
-// CHECK:   msr	TRCITECR_EL2, x1                // encoding: [0x61,0x12,0x1c,0xd5]
-// ERROR-NO-ITE: [[@LINE-2]]:17: error: expected writable system register or pstate
-            trcit x1
-// CHECK:   trcit x1                            // encoding: [0xe1,0x72,0x0b,0xd5]
-// ERROR-NO-ITE: [[@LINE-2]]:13: error: instruction requires: ite
+mrs x3, TRCITEEDCR
+// CHECK-INST: mrs x3, TRCITEEDCR
+// CHECK-ENCODING: encoding: [0x23,0x02,0x31,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d5310223      mrs x3, S2_1_C0_C2_1
+
+msr TRCITEEDCR, x3
+// CHECK-INST: msr TRCITEEDCR, x3
+// CHECK-ENCODING: encoding: [0x23,0x02,0x11,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d5110223      msr S2_1_C0_C2_1, x3
+
+mrs x3, TRCITECR_EL1
+// CHECK-INST: mrs x3, TRCITECR_EL1
+// CHECK-ENCODING: encoding: [0x63,0x12,0x38,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d5381263      mrs x3, S3_0_C1_C2_3
+
+msr TRCITECR_EL1, x1
+// CHECK-INST: msr TRCITECR_EL1, x1
+// CHECK-ENCODING: encoding: [0x61,0x12,0x18,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d5181261      msr S3_0_C1_C2_3, x1
+
+mrs x3, TRCITECR_EL12
+// CHECK-INST: mrs x3, TRCITECR_EL12
+// CHECK-ENCODING: encoding: [0x63,0x12,0x3d,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53d1263      mrs x3, S3_5_C1_C2_3
+
+msr TRCITECR_EL12, x1
+// CHECK-INST: msr TRCITECR_EL12, x1
+// CHECK-ENCODING: encoding: [0x61,0x12,0x1d,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51d1261      msr S3_5_C1_C2_3, x1
+
+mrs x3, TRCITECR_EL2
+// CHECK-INST: mrs x3, TRCITECR_EL2
+// CHECK-ENCODING: encoding: [0x63,0x12,0x3c,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53c1263      mrs x3, S3_4_C1_C2_3
+
+msr TRCITECR_EL2, x1
+// CHECK-INST: msr TRCITECR_EL2, x1
+// CHECK-ENCODING: encoding: [0x61,0x12,0x1c,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51c1261      msr S3_4_C1_C2_3, x1
+
+trcit x1
+// CHECK-INST: trcit x1
+// CHECK-ENCODING: encoding: [0xe1,0x72,0x0b,0xd5]
+// CHECK-ERROR: error: instruction requires: ite
+// CHECK-UNKNOWN:  d50b72e1      sys #3, c7, c2, #7, x1
 
 // FEAT_SPE_FDS
-            mrs x3, PMSDSFR_EL1
-// CHECK:   mrs x3, PMSDSFR_EL1                 // encoding: [0x83,0x9a,0x38,0xd5]
-            msr PMSDSFR_EL1, x3
-// CHECK:   msr PMSDSFR_EL1, x3                 // encoding: [0x83,0x9a,0x18,0xd5]
+mrs x3, PMSDSFR_EL1
+// CHECK-INST: mrs x3, PMSDSFR_EL1
+// CHECK-ENCODING: encoding: [0x83,0x9a,0x38,0xd5]
+// CHECK-UNKNOWN:  d5389a83      mrs x3, PMSDSFR_EL1
+
+msr PMSDSFR_EL1, x3
+// CHECK-INST: msr PMSDSFR_EL1, x3
+// CHECK-ENCODING: encoding: [0x83,0x9a,0x18,0xd5]
+// CHECK-UNKNOWN:  d5189a83      msr PMSDSFR_EL1, x3
diff --git a/llvm/test/MC/AArch64/armv8.9a-lrcpc3.s b/llvm/test/MC/AArch64/armv8.9a-lrcpc3.s
index 263f200428b68..4ccc800310f28 100644
--- a/llvm/test/MC/AArch64/armv8.9a-lrcpc3.s
+++ b/llvm/test/MC/AArch64/armv8.9a-lrcpc3.s
@@ -1,143 +1,282 @@
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding               -mattr=+rcpc3 < %s | FileCheck %s
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.9a -mattr=+rcpc3 < %s | FileCheck %s
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v9.4a -mattr=+rcpc3 < %s | FileCheck %s
-
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu               < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-RCPC3 %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -mattr=+v8.9a < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-RCPC3 %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -mattr=+v9.4a < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-RCPC3 %s
-
-               stilp   w24, w0, [x16, #-8]!
-// CHECK:      stilp   w24, w0, [x16, #-8]!     // encoding: [0x18,0x0a,0x00,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stilp   w24, w0, [x16,  -8]!
-// CHECK:      stilp   w24, w0, [x16, #-8]!     // encoding: [0x18,0x0a,0x00,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stilp   x25, x1, [x17,  -16]!
-// CHECK:      stilp   x25, x1, [x17, #-16]!    // encoding: [0x39,0x0a,0x01,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stilp   x25, x1, [x17, #-16]!
-// CHECK:      stilp   x25, x1, [x17, #-16]!    // encoding: [0x39,0x0a,0x01,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stilp   w26, w2, [x18]
-// CHECK:      stilp   w26, w2, [x18]           // encoding: [0x5a,0x1a,0x02,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stilp   w26, w2, [x18, #0]
-// CHECK:      stilp   w26, w2, [x18]           // encoding: [0x5a,0x1a,0x02,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stilp   x27, x3, [sp]
-// CHECK:      stilp   x27, x3, [sp]            // encoding: [0xfb,0x1b,0x03,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stilp   x27, x3, [sp, 0]
-// CHECK:      stilp   x27, x3, [sp]            // encoding: [0xfb,0x1b,0x03,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  w28, w4, [x20], #8
-// CHECK:      ldiapp  w28, w4, [x20], #8       // encoding: [0x9c,0x0a,0x44,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  w28, w4, [x20, #0], #8
-// CHECK:      ldiapp  w28, w4, [x20], #8       // encoding: [0x9c,0x0a,0x44,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  w28, w4, [x20],  8
-// CHECK:      ldiapp  w28, w4, [x20], #8       // encoding: [0x9c,0x0a,0x44,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  w28, w4, [x20, 0],  8
-// CHECK:      ldiapp  w28, w4, [x20], #8       // encoding: [0x9c,0x0a,0x44,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  x29, x5, [x21], #16
-// CHECK:      ldiapp  x29, x5, [x21], #16      // encoding: [0xbd,0x0a,0x45,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  x29, x5, [x21],  16
-// CHECK:      ldiapp  x29, x5, [x21], #16      // encoding: [0xbd,0x0a,0x45,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  w30, w6, [sp]
-// CHECK:      ldiapp  w30, w6, [sp]            // encoding: [0xfe,0x1b,0x46,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  w30, w6, [sp, #0]
-// CHECK:      ldiapp  w30, w6, [sp]            // encoding: [0xfe,0x1b,0x46,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  xzr, x7, [x23]
-// CHECK:      ldiapp  xzr, x7, [x23]           // encoding: [0xff,0x1a,0x47,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldiapp  xzr, x7, [x23, 0]
-// CHECK:      ldiapp  xzr, x7, [x23]           // encoding: [0xff,0x1a,0x47,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-
-               stlr w3, [x15, #-4]!
-// CHECK:      stlr w3, [x15, #-4]!    // encoding: [0xe3,0x09,0x80,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stlr w3, [x15,  -4]!
-// CHECK:      stlr w3, [x15, #-4]!    // encoding: [0xe3,0x09,0x80,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stlr x3, [x15, #-8]!
-// CHECK:      stlr x3, [x15, #-8]!    // encoding: [0xe3,0x09,0x80,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stlr x3, [sp,  -8]!
-// CHECK:      stlr x3, [sp, #-8]!     // encoding: [0xe3,0x0b,0x80,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldapr w3, [sp], #4
-// CHECK:      ldapr w3, [sp], #4       // encoding: [0xe3,0x0b,0xc0,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldapr w3, [x15], 4
-// CHECK:      ldapr w3, [x15], #4      // encoding: [0xe3,0x09,0xc0,0x99]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldapr x3, [x15], #8
-// CHECK:      ldapr x3, [x15], #8      // encoding: [0xe3,0x09,0xc0,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldapr x3, [x15], 8
-// CHECK:      ldapr x3, [x15], #8      // encoding: [0xe3,0x09,0xc0,0xd9]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-
-               stlur b3, [x15, #-1]
-// CHECK:      stlur b3, [x15, #-1]  // encoding: [0xe3,0xf9,0x1f,0x1d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stlur h3, [x15, #2]
-// CHECK:      stlur h3, [x15, #2]   // encoding: [0xe3,0x29,0x00,0x5d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stlur s3, [x15, #-3]
-// CHECK:      stlur s3, [x15, #-3]  // encoding: [0xe3,0xd9,0x1f,0x9d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stlur d3, [sp, #4]
-// CHECK:      stlur d3, [sp, #4]    // encoding: [0xe3,0x4b,0x00,0xdd]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stlur q3, [x15, #-5]
-// CHECK:      stlur q3, [x15, #-5]  // encoding: [0xe3,0xb9,0x9f,0x1d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldapur b3, [x15, #6]
-// CHECK:      ldapur b3, [x15, #6]  // encoding: [0xe3,0x69,0x40,0x1d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldapur h3, [x15, #-7]
-// CHECK:      ldapur h3, [x15, #-7] // encoding: [0xe3,0x99,0x5f,0x5d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldapur s3, [x15, #8]
-// CHECK:      ldapur s3, [x15, #8]  // encoding: [0xe3,0x89,0x40,0x9d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldapur d3, [x15, #-9]
-// CHECK:      ldapur d3, [x15, #-9] // encoding: [0xe3,0x79,0x5f,0xdd]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldapur q3, [sp, #10]
-// CHECK:      ldapur q3, [sp, #10]  // encoding: [0xe3,0xab,0xc0,0x1d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-
-               stl1  { v3.d }[0], [x15]
-// CHECK:      stl1  { v3.d }[0], [x15]     // encoding: [0xe3,0x85,0x01,0x0d]
-// ERROR-NO-RCPC3:  [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stl1  { v3.d }[0], [x15, #0]
-// CHECK:      stl1  { v3.d }[0], [x15]     // encoding: [0xe3,0x85,0x01,0x0d]
-// ERROR-NO-RCPC3:  [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stl1  { v3.d }[1], [sp]
-// CHECK:      stl1  { v3.d }[1], [sp]      // encoding: [0xe3,0x87,0x01,0x4d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               stl1  { v3.d }[1], [sp, 0]
-// CHECK:      stl1  { v3.d }[1], [sp]      // encoding: [0xe3,0x87,0x01,0x4d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldap1 { v3.d }[0], [sp]
-// CHECK:      ldap1 { v3.d }[0], [sp]      // encoding: [0xe3,0x87,0x41,0x0d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldap1 { v3.d }[0], [sp, #0]
-// CHECK:      ldap1 { v3.d }[0], [sp]      // encoding: [0xe3,0x87,0x41,0x0d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldap1 { v3.d }[1], [x15]
-// CHECK:      ldap1 { v3.d }[1], [x15]     // encoding: [0xe3,0x85,0x41,0x4d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
-               ldap1 { v3.d }[1], [x15, 0]
-// CHECK:      ldap1 { v3.d }[1], [x15]     // encoding: [0xe3,0x85,0x41,0x4d]
-// ERROR-NO-RCPC3: [[@LINE-2]]:16: error: instruction requires: rcpc3
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+rcpc3 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+rcpc3,+v8.9a < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+rcpc3,+v9.4a < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+rcpc3 < %s \
+// RUN:        | llvm-objdump -d --mattr=+rcpc3 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+rcpc3 < %s \
+// RUN:   | llvm-objdump -d --mattr=-rcpc3 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+rcpc3 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+rcpc3 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+stilp   w24, w0, [x16, #-8]!
+// CHECK-INST: stilp w24, w0, [x16, #-8]!
+// CHECK-ENCODING: encoding: [0x18,0x0a,0x00,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99000a18      <unknown>
+
+stilp   w24, w0, [x16,  -8]!
+// CHECK-INST: stilp w24, w0, [x16, #-8]!
+// CHECK-ENCODING: encoding: [0x18,0x0a,0x00,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99000a18      <unknown>
+
+stilp   x25, x1, [x17,  -16]!
+// CHECK-INST: stilp x25, x1, [x17, #-16]!
+// CHECK-ENCODING: encoding: [0x39,0x0a,0x01,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9010a39      <unknown>
+
+stilp   x25, x1, [x17, #-16]!
+// CHECK-INST: stilp x25, x1, [x17, #-16]!
+// CHECK-ENCODING: encoding: [0x39,0x0a,0x01,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9010a39      <unknown>
+
+stilp   w26, w2, [x18]
+// CHECK-INST: stilp w26, w2, [x18]
+// CHECK-ENCODING: encoding: [0x5a,0x1a,0x02,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99021a5a      <unknown>
+
+stilp   w26, w2, [x18, #0]
+// CHECK-INST: stilp w26, w2, [x18]
+// CHECK-ENCODING: encoding: [0x5a,0x1a,0x02,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99021a5a      <unknown>
+
+stilp   x27, x3, [sp]
+// CHECK-INST: stilp x27, x3, [sp]
+// CHECK-ENCODING: encoding: [0xfb,0x1b,0x03,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9031bfb      <unknown>
+
+stilp   x27, x3, [sp, 0]
+// CHECK-INST: stilp x27, x3, [sp]
+// CHECK-ENCODING: encoding: [0xfb,0x1b,0x03,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9031bfb      <unknown>
+
+ldiapp  w28, w4, [x20], #8
+// CHECK-INST: ldiapp w28, w4, [x20], #8
+// CHECK-ENCODING: encoding: [0x9c,0x0a,0x44,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99440a9c      <unknown>
+
+ldiapp  w28, w4, [x20, #0], #8
+// CHECK-INST: ldiapp w28, w4, [x20], #8
+// CHECK-ENCODING: encoding: [0x9c,0x0a,0x44,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99440a9c      <unknown>
+
+ldiapp  w28, w4, [x20],  8
+// CHECK-INST: ldiapp w28, w4, [x20], #8
+// CHECK-ENCODING: encoding: [0x9c,0x0a,0x44,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99440a9c      <unknown>
+
+ldiapp  w28, w4, [x20, 0],  8
+// CHECK-INST: ldiapp w28, w4, [x20], #8
+// CHECK-ENCODING: encoding: [0x9c,0x0a,0x44,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99440a9c      <unknown>
+
+ldiapp  x29, x5, [x21], #16
+// CHECK-INST: ldiapp x29, x5, [x21], #16
+// CHECK-ENCODING: encoding: [0xbd,0x0a,0x45,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9450abd      <unknown>
+
+ldiapp  x29, x5, [x21],  16
+// CHECK-INST: ldiapp x29, x5, [x21], #16
+// CHECK-ENCODING: encoding: [0xbd,0x0a,0x45,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9450abd      <unknown>
+
+ldiapp  w30, w6, [sp]
+// CHECK-INST: ldiapp w30, w6, [sp]
+// CHECK-ENCODING: encoding: [0xfe,0x1b,0x46,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99461bfe      <unknown>
+
+ldiapp  w30, w6, [sp, #0]
+// CHECK-INST: ldiapp w30, w6, [sp]
+// CHECK-ENCODING: encoding: [0xfe,0x1b,0x46,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99461bfe      <unknown>
+
+ldiapp  xzr, x7, [x23]
+// CHECK-INST: ldiapp xzr, x7, [x23]
+// CHECK-ENCODING: encoding: [0xff,0x1a,0x47,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9471aff      <unknown>
+
+ldiapp  xzr, x7, [x23, 0]
+// CHECK-INST: ldiapp xzr, x7, [x23]
+// CHECK-ENCODING: encoding: [0xff,0x1a,0x47,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9471aff      <unknown>
+
+stlr w3, [x15, #-4]!
+// CHECK-INST: stlr w3, [x15, #-4]!
+// CHECK-ENCODING: encoding: [0xe3,0x09,0x80,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  998009e3      <unknown>
+
+stlr w3, [x15,  -4]!
+// CHECK-INST: stlr w3, [x15, #-4]!
+// CHECK-ENCODING: encoding: [0xe3,0x09,0x80,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  998009e3      <unknown>
+
+stlr x3, [x15, #-8]!
+// CHECK-INST: stlr x3, [x15, #-8]!
+// CHECK-ENCODING: encoding: [0xe3,0x09,0x80,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d98009e3      <unknown>
+
+stlr x3, [sp,  -8]!
+// CHECK-INST: stlr x3, [sp, #-8]!
+// CHECK-ENCODING: encoding: [0xe3,0x0b,0x80,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9800be3      <unknown>
+
+ldapr w3, [sp], #4
+// CHECK-INST: ldapr w3, [sp], #4
+// CHECK-ENCODING: encoding: [0xe3,0x0b,0xc0,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99c00be3      <unknown>
+
+ldapr w3, [x15], 4
+// CHECK-INST: ldapr w3, [x15], #4
+// CHECK-ENCODING: encoding: [0xe3,0x09,0xc0,0x99]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  99c009e3      <unknown>
+
+ldapr x3, [x15], #8
+// CHECK-INST: ldapr x3, [x15], #8
+// CHECK-ENCODING: encoding: [0xe3,0x09,0xc0,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9c009e3      <unknown>
+
+ldapr x3, [x15], 8
+// CHECK-INST: ldapr x3, [x15], #8
+// CHECK-ENCODING: encoding: [0xe3,0x09,0xc0,0xd9]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  d9c009e3      <unknown>
+
+stlur b3, [x15, #-1]
+// CHECK-INST: stlur b3, [x15, #-1]
+// CHECK-ENCODING: encoding: [0xe3,0xf9,0x1f,0x1d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  1d1ff9e3      <unknown>
+
+stlur h3, [x15, #2]
+// CHECK-INST: stlur h3, [x15, #2]
+// CHECK-ENCODING: encoding: [0xe3,0x29,0x00,0x5d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  5d0029e3      <unknown>
+
+stlur s3, [x15, #-3]
+// CHECK-INST: stlur s3, [x15, #-3]
+// CHECK-ENCODING: encoding: [0xe3,0xd9,0x1f,0x9d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  9d1fd9e3      <unknown>
+
+stlur d3, [sp, #4]
+// CHECK-INST: stlur d3, [sp, #4]
+// CHECK-ENCODING: encoding: [0xe3,0x4b,0x00,0xdd]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  dd004be3      <unknown>
+
+stlur q3, [x15, #-5]
+// CHECK-INST: stlur q3, [x15, #-5]
+// CHECK-ENCODING: encoding: [0xe3,0xb9,0x9f,0x1d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  1d9fb9e3      <unknown>
+
+ldapur b3, [x15, #6]
+// CHECK-INST: ldapur b3, [x15, #6]
+// CHECK-ENCODING: encoding: [0xe3,0x69,0x40,0x1d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  1d4069e3      <unknown>
+
+ldapur h3, [x15, #-7]
+// CHECK-INST: ldapur h3, [x15, #-7]
+// CHECK-ENCODING: encoding: [0xe3,0x99,0x5f,0x5d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  5d5f99e3      <unknown>
+
+ldapur s3, [x15, #8]
+// CHECK-INST: ldapur s3, [x15, #8]
+// CHECK-ENCODING: encoding: [0xe3,0x89,0x40,0x9d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  9d4089e3      <unknown>
+
+ldapur d3, [x15, #-9]
+// CHECK-INST: ldapur d3, [x15, #-9]
+// CHECK-ENCODING: encoding: [0xe3,0x79,0x5f,0xdd]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  dd5f79e3      <unknown>
+
+ldapur q3, [sp, #10]
+// CHECK-INST: ldapur q3, [sp, #10]
+// CHECK-ENCODING: encoding: [0xe3,0xab,0xc0,0x1d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  1dc0abe3      <unknown>
+
+stl1  { v3.d }[0], [x15]
+// CHECK-INST: stl1 { v3.d }[0], [x15]
+// CHECK-ENCODING: encoding: [0xe3,0x85,0x01,0x0d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  0d0185e3      <unknown>
+
+stl1  { v3.d }[0], [x15, #0]
+// CHECK-INST: stl1 { v3.d }[0], [x15]
+// CHECK-ENCODING: encoding: [0xe3,0x85,0x01,0x0d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  0d0185e3      <unknown>
+
+stl1  { v3.d }[1], [sp]
+// CHECK-INST: stl1 { v3.d }[1], [sp]
+// CHECK-ENCODING: encoding: [0xe3,0x87,0x01,0x4d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  4d0187e3      <unknown>
+
+stl1  { v3.d }[1], [sp, 0]
+// CHECK-INST: stl1 { v3.d }[1], [sp]
+// CHECK-ENCODING: encoding: [0xe3,0x87,0x01,0x4d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  4d0187e3      <unknown>
+
+ldap1 { v3.d }[0], [sp]
+// CHECK-INST: ldap1 { v3.d }[0], [sp]
+// CHECK-ENCODING: encoding: [0xe3,0x87,0x41,0x0d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  0d4187e3      <unknown>
+
+ldap1 { v3.d }[0], [sp, #0]
+// CHECK-INST: ldap1 { v3.d }[0], [sp]
+// CHECK-ENCODING: encoding: [0xe3,0x87,0x41,0x0d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  0d4187e3      <unknown>
+
+ldap1 { v3.d }[1], [x15]
+// CHECK-INST: ldap1 { v3.d }[1], [x15]
+// CHECK-ENCODING: encoding: [0xe3,0x85,0x41,0x4d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  4d4185e3      <unknown>
+
+ldap1 { v3.d }[1], [x15, 0]
+// CHECK-INST: ldap1 { v3.d }[1], [x15]
+// CHECK-ENCODING: encoding: [0xe3,0x85,0x41,0x4d]
+// CHECK-ERROR:error: instruction requires: rcpc3
+// CHECK-UNKNOWN:  4d4185e3      <unknown>
diff --git a/llvm/test/MC/AArch64/armv8.9a-specres2.s b/llvm/test/MC/AArch64/armv8.9a-specres2.s
index b411ec31580b9..b79124d09e926 100644
--- a/llvm/test/MC/AArch64/armv8.9a-specres2.s
+++ b/llvm/test/MC/AArch64/armv8.9a-specres2.s
@@ -1,13 +1,32 @@
-// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+specres2 < %s      | FileCheck %s
-// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.9a    < %s      | FileCheck %s
-// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v9.4a    < %s      | FileCheck %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=-specres2 < %s 2>&1 | FileCheck %s --check-prefix=NOSPECRES2
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v8.9a < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v9.4a < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+specres2 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+specres2 < %s \
+// RUN:        | llvm-objdump -d --mattr=+specres2 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+specres2 < %s \
+// RUN:   | llvm-objdump -d --mattr=-specres2 --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+specres2 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+specres2 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 cosp rctx, x0
+// CHECK-INST: cosp rctx, x0
+// CHECK-ENCODING: encoding: [0xc0,0x73,0x0b,0xd5]
+// CHECK-ERROR: error: COSP requires: predres2
+// CHECK-UNKNOWN:  d50b73c0      sys #3, c7, c3, #6, x0
+
 sys #3, c7, c3, #6, x0
+// CHECK-INST: cosp rctx, x0
+// CHECK-ENCODING: encoding: [0xc0,0x73,0x0b,0xd5]
+// CHECK-UNKNOWN:  d50b73c0      sys #3, c7, c3, #6, x0
 
-// CHECK: cosp rctx, x0          // encoding: [0xc0,0x73,0x0b,0xd5]
-// CHECK: cosp rctx, x0          // encoding: [0xc0,0x73,0x0b,0xd5]
 
-// NOSPECRES2: COSP requires: predres2
-// NOSPECRES2-NEXT: cosp
diff --git a/llvm/test/MC/AArch64/armv8.9a-the-diagnostics.s b/llvm/test/MC/AArch64/armv8.9a-the-diagnostics.s
new file mode 100644
index 0000000000000..0fccbe1134d03
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv8.9a-the-diagnostics.s
@@ -0,0 +1,103 @@
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+the -mattr=+d128 < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-ZXR %s
+
+rcwswpp   xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwswppa  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwswppal xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwswppl  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwswpp   x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwswppa  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwswppal x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwswppl  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+
+rcwclrp   xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwclrpa  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwclrpal xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwclrpl  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwclrp   x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwclrpa  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwclrpal x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwclrpl  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+
+rcwsetp   xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsetpa  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsetpal xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsetpl  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsetp   x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsetpa  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsetpal x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsetpl  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+
+rcwsswpp   xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsswppa  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsswppal xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsswppl  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsswpp   x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsswppa  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsswppal x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsswppl  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+
+rcwsclrp   xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsclrpa  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsclrpal xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsclrpl  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsclrp   x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsclrpa  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsclrpal x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwsclrpl  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+
+rcwssetp   xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwssetpa  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwssetpal xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwssetpl  xzr, x5, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwssetp   x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwssetpa  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwssetpal x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
+rcwssetpl  x5, xzr, [x4]
+// ERROR-NO-ZXR:   error: invalid operand for instruction
diff --git a/llvm/test/MC/AArch64/armv8.9a-the.s b/llvm/test/MC/AArch64/armv8.9a-the.s
index 33e1b5d27fa76..689b6c9280e7c 100644
--- a/llvm/test/MC/AArch64/armv8.9a-the.s
+++ b/llvm/test/MC/AArch64/armv8.9a-the.s
@@ -1,592 +1,843 @@
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding               -mattr=+the -mattr=+d128 < %s | FileCheck %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.9a -mattr=+the -mattr=+d128 < %s | FileCheck %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v9.4a -mattr=+the -mattr=+d128 < %s | FileCheck %s
-
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu                           < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-THE %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -mattr=+v8.9a             < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-THE %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -mattr=+v9.4a             < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-THE %s
-
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu               -mattr=+the < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-D128 %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -mattr=+v8.9a -mattr=+the < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-D128 %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -mattr=+v9.4a -mattr=+the < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-D128 %s
-
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+the -mattr=+d128 < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-ZXR %s
-
-            mrs x3, RCWMASK_EL1
-// CHECK:   mrs x3, RCWMASK_EL1   // encoding: [0xc3,0xd0,0x38,0xd5]
-// ERROR-NO-THE: [[@LINE-2]]:21: error: expected readable system register
-            msr RCWMASK_EL1, x1
-// CHECK:   msr RCWMASK_EL1, x1   // encoding: [0xc1,0xd0,0x18,0xd5]
-// ERROR-NO-THE: [[@LINE-2]]:17: error: expected writable system register or pstate
-            mrs x3, RCWSMASK_EL1
-// CHECK:   mrs x3, RCWSMASK_EL1  // encoding: [0x63,0xd0,0x38,0xd5]
-// ERROR-NO-THE: [[@LINE-2]]:21: error: expected readable system register
-            msr RCWSMASK_EL1, x1
-// CHECK:   msr RCWSMASK_EL1, x1  // encoding: [0x61,0xd0,0x18,0xd5]
-// ERROR-NO-THE: [[@LINE-2]]:17: error: expected writable system register or pstate
-
-            rcwcas   x0, x1, [x4]
-// CHECK:   rcwcas   x0, x1, [x4] // encoding: [0x81,0x08,0x20,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwcasa  x0, x1, [x4]
-// CHECK:   rcwcasa  x0, x1, [x4] // encoding: [0x81,0x08,0xa0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwcasal x0, x1, [x4]
-// CHECK:   rcwcasal x0, x1, [x4] // encoding: [0x81,0x08,0xe0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwcasl  x0, x1, [x4]
-// CHECK:   rcwcasl  x0, x1, [x4] // encoding: [0x81,0x08,0x60,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwcas   x3, x5, [sp]
-// CHECK:   rcwcas   x3, x5, [sp] // encoding: [0xe5,0x0b,0x23,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwcasa  x3, x5, [sp]
-// CHECK:   rcwcasa  x3, x5, [sp] // encoding: [0xe5,0x0b,0xa3,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwcasal x3, x5, [sp]
-// CHECK:   rcwcasal x3, x5, [sp] // encoding: [0xe5,0x0b,0xe3,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwcasl  x3, x5, [sp]
-// CHECK:   rcwcasl  x3, x5, [sp] // encoding: [0xe5,0x0b,0x63,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-
-            rcwscas   x0, x1, [x4]
-// CHECK:   rcwscas   x0, x1, [x4] // encoding: [0x81,0x08,0x20,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwscasa  x0, x1, [x4]
-// CHECK:   rcwscasa  x0, x1, [x4] // encoding: [0x81,0x08,0xa0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwscasal x0, x1, [x4]
-// CHECK:   rcwscasal x0, x1, [x4] // encoding: [0x81,0x08,0xe0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwscasl  x0, x1, [x4]
-// CHECK:   rcwscasl  x0, x1, [x4] // encoding: [0x81,0x08,0x60,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwscas   x3, x5, [sp]
-// CHECK:   rcwscas   x3, x5, [sp] // encoding: [0xe5,0x0b,0x23,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwscasa  x3, x5, [sp]
-// CHECK:   rcwscasa  x3, x5, [sp] // encoding: [0xe5,0x0b,0xa3,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwscasal x3, x5, [sp]
-// CHECK:   rcwscasal x3, x5, [sp] // encoding: [0xe5,0x0b,0xe3,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwscasl  x3, x5, [sp]
-// CHECK:   rcwscasl  x3, x5, [sp] // encoding: [0xe5,0x0b,0x63,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-
-            rcwcasp   x0, x1, x6, x7, [x4]
-// CHECK:   rcwcasp   x0, x1, x6, x7, [x4] // encoding: [0x86,0x0c,0x20,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwcaspa  x0, x1, x6, x7, [x4]
-// CHECK:   rcwcaspa  x0, x1, x6, x7, [x4] // encoding: [0x86,0x0c,0xa0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwcaspal x0, x1, x6, x7, [x4]
-// CHECK:   rcwcaspal x0, x1, x6, x7, [x4] // encoding: [0x86,0x0c,0xe0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwcaspl  x0, x1, x6, x7, [x4]
-// CHECK:   rcwcaspl  x0, x1, x6, x7, [x4] // encoding: [0x86,0x0c,0x60,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwcasp   x4, x5, x6, x7, [sp]
-// CHECK:   rcwcasp   x4, x5, x6, x7, [sp] // encoding: [0xe6,0x0f,0x24,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwcaspa  x4, x5, x6, x7, [sp]
-// CHECK:   rcwcaspa  x4, x5, x6, x7, [sp] // encoding: [0xe6,0x0f,0xa4,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwcaspal x4, x5, x6, x7, [sp]
-// CHECK:   rcwcaspal x4, x5, x6, x7, [sp] // encoding: [0xe6,0x0f,0xe4,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwcaspl  x4, x5, x6, x7, [sp]
-// CHECK:   rcwcaspl  x4, x5, x6, x7, [sp] // encoding: [0xe6,0x0f,0x64,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-
-            rcwscasp   x0, x1, x6, x7, [x4]
-// CHECK:   rcwscasp   x0, x1, x6, x7, [x4] // encoding: [0x86,0x0c,0x20,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwscaspa  x0, x1, x6, x7, [x4]
-// CHECK:   rcwscaspa  x0, x1, x6, x7, [x4] // encoding: [0x86,0x0c,0xa0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwscaspal x0, x1, x6, x7, [x4]
-// CHECK:   rcwscaspal x0, x1, x6, x7, [x4] // encoding: [0x86,0x0c,0xe0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwscaspl  x0, x1, x6, x7, [x4]
-// CHECK:   rcwscaspl  x0, x1, x6, x7, [x4] // encoding: [0x86,0x0c,0x60,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwscasp   x4, x5, x6, x7, [sp]
-// CHECK:   rcwscasp   x4, x5, x6, x7, [sp] // encoding: [0xe6,0x0f,0x24,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwscaspa  x4, x5, x6, x7, [sp]
-// CHECK:   rcwscaspa  x4, x5, x6, x7, [sp] // encoding: [0xe6,0x0f,0xa4,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwscaspal x4, x5, x6, x7, [sp]
-// CHECK:   rcwscaspal x4, x5, x6, x7, [sp] // encoding: [0xe6,0x0f,0xe4,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwscaspl  x4, x5, x6, x7, [sp]
-// CHECK:   rcwscaspl  x4, x5, x6, x7, [sp] // encoding: [0xe6,0x0f,0x64,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-
-            rcwclr   x0, x1, [x4]
-// CHECK:   rcwclr   x0, x1, [x4] // encoding: [0x81,0x90,0x20,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwclra  x0, x1, [x4]
-// CHECK:   rcwclra  x0, x1, [x4] // encoding: [0x81,0x90,0xa0,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwclral x0, x1, [x4]
-// CHECK:   rcwclral x0, x1, [x4] // encoding: [0x81,0x90,0xe0,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwclrl  x0, x1, [x4]
-// CHECK:   rcwclrl  x0, x1, [x4] // encoding: [0x81,0x90,0x60,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwclr   x3, x5, [sp]
-// CHECK:   rcwclr   x3, x5, [sp] // encoding: [0xe5,0x93,0x23,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwclra  x3, x5, [sp]
-// CHECK:   rcwclra  x3, x5, [sp] // encoding: [0xe5,0x93,0xa3,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwclral x3, x5, [sp]
-// CHECK:   rcwclral x3, x5, [sp] // encoding: [0xe5,0x93,0xe3,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwclrl  x3, x5, [sp]
-// CHECK:   rcwclrl  x3, x5, [sp] // encoding: [0xe5,0x93,0x63,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-
-            rcwsclr   x0, x1, [x4]
-// CHECK:   rcwsclr   x0, x1, [x4] // encoding: [0x81,0x90,0x20,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsclra  x0, x1, [x4]
-// CHECK:   rcwsclra  x0, x1, [x4] // encoding: [0x81,0x90,0xa0,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsclral x0, x1, [x4]
-// CHECK:   rcwsclral x0, x1, [x4] // encoding: [0x81,0x90,0xe0,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsclrl  x0, x1, [x4]
-// CHECK:   rcwsclrl  x0, x1, [x4] // encoding: [0x81,0x90,0x60,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsclr   x3, x5, [sp]
-// CHECK:   rcwsclr   x3, x5, [sp] // encoding: [0xe5,0x93,0x23,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsclra  x3, x5, [sp]
-// CHECK:   rcwsclra  x3, x5, [sp] // encoding: [0xe5,0x93,0xa3,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsclral x3, x5, [sp]
-// CHECK:   rcwsclral x3, x5, [sp] // encoding: [0xe5,0x93,0xe3,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsclrl  x3, x5, [sp]
-// CHECK:   rcwsclrl  x3, x5, [sp] // encoding: [0xe5,0x93,0x63,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-
-            rcwclrp   x1, x0, [x4]
-// CHECK:   rcwclrp   x1, x0, [x4] // encoding: [0x81,0x90,0x20,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwclrpa  x1, x0, [x4]
-// CHECK:   rcwclrpa  x1, x0, [x4] // encoding: [0x81,0x90,0xa0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwclrpal x1, x0, [x4]
-// CHECK:   rcwclrpal x1, x0, [x4] // encoding: [0x81,0x90,0xe0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwclrpl  x1, x0, [x4]
-// CHECK:   rcwclrpl  x1, x0, [x4] // encoding: [0x81,0x90,0x60,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwclrp   x5, x3, [sp]
-// CHECK:   rcwclrp   x5, x3, [sp] // encoding: [0xe5,0x93,0x23,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwclrpa  x5, x3, [sp]
-// CHECK:   rcwclrpa  x5, x3, [sp] // encoding: [0xe5,0x93,0xa3,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwclrpal x5, x3, [sp]
-// CHECK:   rcwclrpal x5, x3, [sp] // encoding: [0xe5,0x93,0xe3,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwclrpl  x5, x3, [sp]
-// CHECK:   rcwclrpl  x5, x3, [sp] // encoding: [0xe5,0x93,0x63,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-
-            rcwsclrp   x1, x0, [x4]
-// CHECK:   rcwsclrp   x1, x0, [x4] // encoding: [0x81,0x90,0x20,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsclrpa  x1, x0, [x4]
-// CHECK:   rcwsclrpa  x1, x0, [x4] // encoding: [0x81,0x90,0xa0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsclrpal x1, x0, [x4]
-// CHECK:   rcwsclrpal x1, x0, [x4] // encoding: [0x81,0x90,0xe0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsclrpl  x1, x0, [x4]
-// CHECK:   rcwsclrpl  x1, x0, [x4] // encoding: [0x81,0x90,0x60,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsclrp   x5, x3, [sp]
-// CHECK:   rcwsclrp   x5, x3, [sp] // encoding: [0xe5,0x93,0x23,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsclrpa  x5, x3, [sp]
-// CHECK:   rcwsclrpa  x5, x3, [sp] // encoding: [0xe5,0x93,0xa3,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsclrpal x5, x3, [sp]
-// CHECK:   rcwsclrpal x5, x3, [sp] // encoding: [0xe5,0x93,0xe3,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsclrpl  x5, x3, [sp]
-// CHECK:   rcwsclrpl  x5, x3, [sp] // encoding: [0xe5,0x93,0x63,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-
-            rcwset   x0, x1, [x4]
-// CHECK:   rcwset   x0, x1, [x4] // encoding: [0x81,0xb0,0x20,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwseta  x0, x1, [x4]
-// CHECK:   rcwseta  x0, x1, [x4] // encoding: [0x81,0xb0,0xa0,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsetal x0, x1, [x4]
-// CHECK:   rcwsetal x0, x1, [x4] // encoding: [0x81,0xb0,0xe0,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsetl  x0, x1, [x4]
-// CHECK:   rcwsetl  x0, x1, [x4] // encoding: [0x81,0xb0,0x60,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwset   x3, x5, [sp]
-// CHECK:   rcwset   x3, x5, [sp] // encoding: [0xe5,0xb3,0x23,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwseta  x3, x5, [sp]
-// CHECK:   rcwseta  x3, x5, [sp] // encoding: [0xe5,0xb3,0xa3,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsetal x3, x5, [sp]
-// CHECK:   rcwsetal x3, x5, [sp] // encoding: [0xe5,0xb3,0xe3,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsetl  x3, x5, [sp]
-// CHECK:   rcwsetl  x3, x5, [sp] // encoding: [0xe5,0xb3,0x63,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-
-            rcwsset   x0, x1, [x4]
-// CHECK:   rcwsset   x0, x1, [x4] // encoding: [0x81,0xb0,0x20,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsseta  x0, x1, [x4]
-// CHECK:   rcwsseta  x0, x1, [x4] // encoding: [0x81,0xb0,0xa0,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwssetal x0, x1, [x4]
-// CHECK:   rcwssetal x0, x1, [x4] // encoding: [0x81,0xb0,0xe0,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwssetl  x0, x1, [x4]
-// CHECK:   rcwssetl  x0, x1, [x4] // encoding: [0x81,0xb0,0x60,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsset   x3, x5, [sp]
-// CHECK:   rcwsset   x3, x5, [sp] // encoding: [0xe5,0xb3,0x23,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsseta  x3, x5, [sp]
-// CHECK:   rcwsseta  x3, x5, [sp] // encoding: [0xe5,0xb3,0xa3,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwssetal x3, x5, [sp]
-// CHECK:   rcwssetal x3, x5, [sp] // encoding: [0xe5,0xb3,0xe3,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwssetl  x3, x5, [sp]
-// CHECK:   rcwssetl  x3, x5, [sp] // encoding: [0xe5,0xb3,0x63,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-
-            rcwsetp   x1, x0, [x4]
-// CHECK:   rcwsetp   x1, x0, [x4] // encoding: [0x81,0xb0,0x20,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsetpa  x1, x0, [x4]
-// CHECK:   rcwsetpa  x1, x0, [x4] // encoding: [0x81,0xb0,0xa0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsetpal x1, x0, [x4]
-// CHECK:   rcwsetpal x1, x0, [x4] // encoding: [0x81,0xb0,0xe0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsetpl  x1, x0, [x4]
-// CHECK:   rcwsetpl  x1, x0, [x4] // encoding: [0x81,0xb0,0x60,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsetp   x5, x3, [sp]
-// CHECK:   rcwsetp   x5, x3, [sp] // encoding: [0xe5,0xb3,0x23,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsetpa  x5, x3, [sp]
-// CHECK:   rcwsetpa  x5, x3, [sp] // encoding: [0xe5,0xb3,0xa3,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsetpal x5, x3, [sp]
-// CHECK:   rcwsetpal x5, x3, [sp] // encoding: [0xe5,0xb3,0xe3,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsetpl  x5, x3, [sp]
-// CHECK:   rcwsetpl  x5, x3, [sp] // encoding: [0xe5,0xb3,0x63,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-
-            rcwssetp   x1, x0, [x4]
-// CHECK:   rcwssetp   x1, x0, [x4] // encoding: [0x81,0xb0,0x20,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwssetpa  x1, x0, [x4]
-// CHECK:   rcwssetpa  x1, x0, [x4] // encoding: [0x81,0xb0,0xa0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwssetpal x1, x0, [x4]
-// CHECK:   rcwssetpal x1, x0, [x4] // encoding: [0x81,0xb0,0xe0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwssetpl  x1, x0, [x4]
-// CHECK:   rcwssetpl  x1, x0, [x4] // encoding: [0x81,0xb0,0x60,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwssetp   x5, x3, [sp]
-// CHECK:   rcwssetp   x5, x3, [sp] // encoding: [0xe5,0xb3,0x23,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwssetpa  x5, x3, [sp]
-// CHECK:   rcwssetpa  x5, x3, [sp] // encoding: [0xe5,0xb3,0xa3,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwssetpal x5, x3, [sp]
-// CHECK:   rcwssetpal x5, x3, [sp] // encoding: [0xe5,0xb3,0xe3,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwssetpl  x5, x3, [sp]
-// CHECK:   rcwssetpl  x5, x3, [sp] // encoding: [0xe5,0xb3,0x63,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-
-            rcwswp   x0, x1, [x4]
-// CHECK:   rcwswp   x0, x1, [x4] // encoding: [0x81,0xa0,0x20,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwswpa  x0, x1, [x4]
-// CHECK:   rcwswpa  x0, x1, [x4] // encoding: [0x81,0xa0,0xa0,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwswpal x0, x1, [x4]
-// CHECK:   rcwswpal x0, x1, [x4] // encoding: [0x81,0xa0,0xe0,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwswpl  x0, x1, [x4]
-// CHECK:   rcwswpl  x0, x1, [x4] // encoding: [0x81,0xa0,0x60,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwswp   x3, x5, [sp]
-// CHECK:   rcwswp   x3, x5, [sp] // encoding: [0xe5,0xa3,0x23,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwswpa  x3, x5, [sp]
-// CHECK:   rcwswpa  x3, x5, [sp] // encoding: [0xe5,0xa3,0xa3,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwswpal x3, x5, [sp]
-// CHECK:   rcwswpal x3, x5, [sp] // encoding: [0xe5,0xa3,0xe3,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwswpl  x3, x5, [sp]
-// CHECK:   rcwswpl  x3, x5, [sp] // encoding: [0xe5,0xa3,0x63,0x38]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-
-            rcwsswp   x0, x1, [x4]
-// CHECK:   rcwsswp   x0, x1, [x4] // encoding: [0x81,0xa0,0x20,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsswpa  x0, x1, [x4]
-// CHECK:   rcwsswpa  x0, x1, [x4] // encoding: [0x81,0xa0,0xa0,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsswpal x0, x1, [x4]
-// CHECK:   rcwsswpal x0, x1, [x4] // encoding: [0x81,0xa0,0xe0,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsswpl  x0, x1, [x4]
-// CHECK:   rcwsswpl  x0, x1, [x4] // encoding: [0x81,0xa0,0x60,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsswp   x3, x5, [sp]
-// CHECK:   rcwsswp   x3, x5, [sp] // encoding: [0xe5,0xa3,0x23,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsswpa  x3, x5, [sp]
-// CHECK:   rcwsswpa  x3, x5, [sp] // encoding: [0xe5,0xa3,0xa3,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsswpal x3, x5, [sp]
-// CHECK:   rcwsswpal x3, x5, [sp] // encoding: [0xe5,0xa3,0xe3,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-            rcwsswpl  x3, x5, [sp]
-// CHECK:   rcwsswpl  x3, x5, [sp] // encoding: [0xe5,0xa3,0x63,0x78]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: the
-
-            rcwswpp   x1, x0, [x4]
-// CHECK:   rcwswpp   x1, x0, [x4] // encoding: [0x81,0xa0,0x20,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwswppa  x1, x0, [x4]
-// CHECK:   rcwswppa  x1, x0, [x4] // encoding: [0x81,0xa0,0xa0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwswppal x1, x0, [x4]
-// CHECK:   rcwswppal x1, x0, [x4] // encoding: [0x81,0xa0,0xe0,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwswppl  x1, x0, [x4]
-// CHECK:   rcwswppl  x1, x0, [x4] // encoding: [0x81,0xa0,0x60,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwswpp   x5, x3, [sp]
-// CHECK:   rcwswpp   x5, x3, [sp] // encoding: [0xe5,0xa3,0x23,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwswppa  x5, x3, [sp]
-// CHECK:   rcwswppa  x5, x3, [sp] // encoding: [0xe5,0xa3,0xa3,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwswppal x5, x3, [sp]
-// CHECK:   rcwswppal x5, x3, [sp] // encoding: [0xe5,0xa3,0xe3,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwswppl  x5, x3, [sp]
-// CHECK:   rcwswppl  x5, x3, [sp] // encoding: [0xe5,0xa3,0x63,0x19]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-
-            rcwsswpp   x1, x0, [x4]
-// CHECK:   rcwsswpp   x1, x0, [x4] // encoding: [0x81,0xa0,0x20,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsswppa  x1, x0, [x4]
-// CHECK:   rcwsswppa  x1, x0, [x4] // encoding: [0x81,0xa0,0xa0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsswppal x1, x0, [x4]
-// CHECK:   rcwsswppal x1, x0, [x4] // encoding: [0x81,0xa0,0xe0,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsswppl  x1, x0, [x4]
-// CHECK:   rcwsswppl  x1, x0, [x4] // encoding: [0x81,0xa0,0x60,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsswpp   x5, x3, [sp]
-// CHECK:   rcwsswpp   x5, x3, [sp] // encoding: [0xe5,0xa3,0x23,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsswppa  x5, x3, [sp]
-// CHECK:   rcwsswppa  x5, x3, [sp] // encoding: [0xe5,0xa3,0xa3,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsswppal x5, x3, [sp]
-// CHECK:   rcwsswppal x5, x3, [sp] // encoding: [0xe5,0xa3,0xe3,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-            rcwsswppl  x5, x3, [sp]
-// CHECK:   rcwsswppl  x5, x3, [sp] // encoding: [0xe5,0xa3,0x63,0x59]
-// ERROR-NO-THE: [[@LINE-2]]:13: error: instruction requires: d128 the
-// ERROR-NO-D128: [[@LINE-3]]:13: error: instruction requires: d128
-
-            rcwswpp   xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwswppa  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwswppal xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwswppl  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwswpp   x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-            rcwswppa  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-            rcwswppal x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-            rcwswppl  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-
-            rcwclrp   xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwclrpa  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwclrpal xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwclrpl  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwclrp   x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-            rcwclrpa  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-            rcwclrpal x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-            rcwclrpl  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-
-            rcwsetp   xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwsetpa  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwsetpal xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwsetpl  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:23: error: invalid operand for instruction
-            rcwsetp   x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-            rcwsetpa  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-            rcwsetpal x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-            rcwsetpl  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:27: error: invalid operand for instruction
-
-            rcwsswpp   xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwsswppa  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwsswppal xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwsswppl  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwsswpp   x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-            rcwsswppa  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-            rcwsswppal x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-            rcwsswppl  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-
-            rcwsclrp   xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwsclrpa  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwsclrpal xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwsclrpl  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwsclrp   x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-            rcwsclrpa  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-            rcwsclrpal x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-            rcwsclrpl  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-
-            rcwssetp   xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwssetpa  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwssetpal xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwssetpl  xzr, x5, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:24: error: invalid operand for instruction
-            rcwssetp   x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-            rcwssetpa  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-            rcwssetpal x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
-            rcwssetpl  x5, xzr, [x4]
-// ERROR-NO-ZXR:   [[@LINE-1]]:28: error: invalid operand for instruction
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+the,+d128 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+the,+d128,v8.9a < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+the,+d128,v9.4a < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+the,+d128 < %s \
+// RUN:        | llvm-objdump -d --mattr=+the,+d128 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+the,+d128 < %s \
+// RUN:   | llvm-objdump -d --mattr=-the,-d128 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+the,+d128 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+the,+d128 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -mattr=+the < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-D128 %s
+
+
+mrs x3, RCWMASK_EL1
+// CHECK-INST: mrs x3, RCWMASK_EL1
+// CHECK-ENCODING: encoding: [0xc3,0xd0,0x38,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d538d0c3      mrs x3, S3_0_C13_C0_6
+
+msr RCWMASK_EL1, x1
+// CHECK-INST: msr RCWMASK_EL1, x1
+// CHECK-ENCODING: encoding: [0xc1,0xd0,0x18,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d518d0c1      msr S3_0_C13_C0_6, x1
+
+mrs x3, RCWSMASK_EL1
+// CHECK-INST: mrs x3, RCWSMASK_EL1
+// CHECK-ENCODING: encoding: [0x63,0xd0,0x38,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d538d063      mrs x3, S3_0_C13_C0_3
+
+msr RCWSMASK_EL1, x1
+// CHECK-INST: msr RCWSMASK_EL1, x1
+// CHECK-ENCODING: encoding: [0x61,0xd0,0x18,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d518d061      msr S3_0_C13_C0_3, x1
+
+rcwcas   x0, x1, [x4]
+// CHECK-INST: rcwcas x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x08,0x20,0x19]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  19200881      <unknown>
+
+rcwcasa  x0, x1, [x4]
+// CHECK-INST: rcwcasa x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x08,0xa0,0x19]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  19a00881      <unknown>
+
+rcwcasal x0, x1, [x4]
+// CHECK-INST: rcwcasal x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x08,0xe0,0x19]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  19e00881      <unknown>
+
+rcwcasl  x0, x1, [x4]
+// CHECK-INST: rcwcasl x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x08,0x60,0x19]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  19600881      <unknown>
+
+rcwcas   x3, x5, [sp]
+// CHECK-INST: rcwcas x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x0b,0x23,0x19]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  19230be5      <unknown>
+
+rcwcasa  x3, x5, [sp]
+// CHECK-INST: rcwcasa x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x0b,0xa3,0x19]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  19a30be5      <unknown>
+
+rcwcasal x3, x5, [sp]
+// CHECK-INST: rcwcasal x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x0b,0xe3,0x19]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  19e30be5      <unknown>
+
+rcwcasl  x3, x5, [sp]
+// CHECK-INST: rcwcasl x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x0b,0x63,0x19]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  19630be5      <unknown>
+
+rcwscas   x0, x1, [x4]
+// CHECK-INST: rcwscas x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x08,0x20,0x59]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  59200881      <unknown>
+
+rcwscasa  x0, x1, [x4]
+// CHECK-INST: rcwscasa x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x08,0xa0,0x59]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  59a00881      <unknown>
+
+rcwscasal x0, x1, [x4]
+// CHECK-INST: rcwscasal x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x08,0xe0,0x59]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  59e00881      <unknown>
+
+rcwscasl  x0, x1, [x4]
+// CHECK-INST: rcwscasl x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x08,0x60,0x59]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  59600881      <unknown>
+
+rcwscas   x3, x5, [sp]
+// CHECK-INST: rcwscas x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x0b,0x23,0x59]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  59230be5      <unknown>
+
+rcwscasa  x3, x5, [sp]
+// CHECK-INST: rcwscasa x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x0b,0xa3,0x59]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  59a30be5      <unknown>
+
+rcwscasal x3, x5, [sp]
+// CHECK-INST: rcwscasal x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x0b,0xe3,0x59]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  59e30be5      <unknown>
+
+rcwscasl  x3, x5, [sp]
+// CHECK-INST: rcwscasl x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x0b,0x63,0x59]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  59630be5      <unknown>
+
+rcwcasp   x0, x1, x6, x7, [x4]
+// CHECK-INST: rcwcasp x0, x1, x6, x7, [x4]
+// CHECK-ENCODING: encoding: [0x86,0x0c,0x20,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19200c86      <unknown>
+
+rcwcaspa  x0, x1, x6, x7, [x4]
+// CHECK-INST: rcwcaspa x0, x1, x6, x7, [x4]
+// CHECK-ENCODING: encoding: [0x86,0x0c,0xa0,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19a00c86      <unknown>
+
+rcwcaspal x0, x1, x6, x7, [x4]
+// CHECK-INST: rcwcaspal x0, x1, x6, x7, [x4]
+// CHECK-ENCODING: encoding: [0x86,0x0c,0xe0,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19e00c86      <unknown>
+
+rcwcaspl  x0, x1, x6, x7, [x4]
+// CHECK-INST: rcwcaspl x0, x1, x6, x7, [x4]
+// CHECK-ENCODING: encoding: [0x86,0x0c,0x60,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19600c86      <unknown>
+
+rcwcasp   x4, x5, x6, x7, [sp]
+// CHECK-INST: rcwcasp x4, x5, x6, x7, [sp]
+// CHECK-ENCODING: encoding: [0xe6,0x0f,0x24,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19240fe6      <unknown>
+
+rcwcaspa  x4, x5, x6, x7, [sp]
+// CHECK-INST: rcwcaspa x4, x5, x6, x7, [sp]
+// CHECK-ENCODING: encoding: [0xe6,0x0f,0xa4,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19a40fe6      <unknown>
+
+rcwcaspal x4, x5, x6, x7, [sp]
+// CHECK-INST: rcwcaspal x4, x5, x6, x7, [sp]
+// CHECK-ENCODING: encoding: [0xe6,0x0f,0xe4,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19e40fe6      <unknown>
+
+rcwcaspl  x4, x5, x6, x7, [sp]
+// CHECK-INST: rcwcaspl x4, x5, x6, x7, [sp]
+// CHECK-ENCODING: encoding: [0xe6,0x0f,0x64,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19640fe6      <unknown>
+
+rcwscasp   x0, x1, x6, x7, [x4]
+// CHECK-INST: rcwscasp x0, x1, x6, x7, [x4]
+// CHECK-ENCODING: encoding: [0x86,0x0c,0x20,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59200c86      <unknown>
+
+rcwscaspa  x0, x1, x6, x7, [x4]
+// CHECK-INST: rcwscaspa x0, x1, x6, x7, [x4]
+// CHECK-ENCODING: encoding: [0x86,0x0c,0xa0,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59a00c86      <unknown>
+
+rcwscaspal x0, x1, x6, x7, [x4]
+// CHECK-INST: rcwscaspal x0, x1, x6, x7, [x4]
+// CHECK-ENCODING: encoding: [0x86,0x0c,0xe0,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59e00c86      <unknown>
+
+rcwscaspl  x0, x1, x6, x7, [x4]
+// CHECK-INST: rcwscaspl x0, x1, x6, x7, [x4]
+// CHECK-ENCODING: encoding: [0x86,0x0c,0x60,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59600c86      <unknown>
+
+rcwscasp   x4, x5, x6, x7, [sp]
+// CHECK-INST: rcwscasp x4, x5, x6, x7, [sp]
+// CHECK-ENCODING: encoding: [0xe6,0x0f,0x24,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59240fe6      <unknown>
+
+rcwscaspa  x4, x5, x6, x7, [sp]
+// CHECK-INST: rcwscaspa x4, x5, x6, x7, [sp]
+// CHECK-ENCODING: encoding: [0xe6,0x0f,0xa4,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59a40fe6      <unknown>
+
+rcwscaspal x4, x5, x6, x7, [sp]
+// CHECK-INST: rcwscaspal x4, x5, x6, x7, [sp]
+// CHECK-ENCODING: encoding: [0xe6,0x0f,0xe4,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59e40fe6      <unknown>
+
+rcwscaspl  x4, x5, x6, x7, [sp]
+// CHECK-INST: rcwscaspl x4, x5, x6, x7, [sp]
+// CHECK-ENCODING: encoding: [0xe6,0x0f,0x64,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59640fe6      <unknown>
+
+rcwclr   x0, x1, [x4]
+// CHECK-INST: rcwclr x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0x20,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38209081      <unknown>
+
+rcwclra  x0, x1, [x4]
+// CHECK-INST: rcwclra x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0xa0,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38a09081      <unknown>
+
+rcwclral x0, x1, [x4]
+// CHECK-INST: rcwclral x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0xe0,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38e09081      <unknown>
+
+rcwclrl  x0, x1, [x4]
+// CHECK-INST: rcwclrl x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0x60,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38609081      <unknown>
+
+rcwclr   x3, x5, [sp]
+// CHECK-INST: rcwclr x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0x23,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  382393e5      <unknown>
+
+rcwclra  x3, x5, [sp]
+// CHECK-INST: rcwclra x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0xa3,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38a393e5      <unknown>
+
+rcwclral x3, x5, [sp]
+// CHECK-INST: rcwclral x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0xe3,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38e393e5      <unknown>
+
+rcwclrl  x3, x5, [sp]
+// CHECK-INST: rcwclrl x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0x63,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  386393e5      <unknown>
+
+rcwsclr   x0, x1, [x4]
+// CHECK-INST: rcwsclr x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0x20,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78209081      <unknown>
+
+rcwsclra  x0, x1, [x4]
+// CHECK-INST: rcwsclra x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0xa0,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78a09081      <unknown>
+
+rcwsclral x0, x1, [x4]
+// CHECK-INST: rcwsclral x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0xe0,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78e09081      <unknown>
+
+rcwsclrl  x0, x1, [x4]
+// CHECK-INST: rcwsclrl x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0x60,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78609081      <unknown>
+
+rcwsclr   x3, x5, [sp]
+// CHECK-INST: rcwsclr x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0x23,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  782393e5      <unknown>
+
+rcwsclra  x3, x5, [sp]
+// CHECK-INST: rcwsclra x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0xa3,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78a393e5      <unknown>
+
+rcwsclral x3, x5, [sp]
+// CHECK-INST: rcwsclral x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0xe3,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78e393e5      <unknown>
+
+rcwsclrl  x3, x5, [sp]
+// CHECK-INST: rcwsclrl x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0x63,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  786393e5      <unknown>
+
+rcwclrp   x1, x0, [x4]
+// CHECK-INST: rcwclrp x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0x20,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19209081      <unknown>
+
+rcwclrpa  x1, x0, [x4]
+// CHECK-INST: rcwclrpa x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0xa0,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19a09081      <unknown>
+
+rcwclrpal x1, x0, [x4]
+// CHECK-INST: rcwclrpal x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0xe0,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19e09081      <unknown>
+
+rcwclrpl  x1, x0, [x4]
+// CHECK-INST: rcwclrpl x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0x60,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19609081      <unknown>
+
+rcwclrp   x5, x3, [sp]
+// CHECK-INST: rcwclrp x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0x23,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  192393e5      <unknown>
+
+rcwclrpa  x5, x3, [sp]
+// CHECK-INST: rcwclrpa x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0xa3,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19a393e5      <unknown>
+
+rcwclrpal x5, x3, [sp]
+// CHECK-INST: rcwclrpal x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0xe3,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19e393e5      <unknown>
+
+rcwclrpl  x5, x3, [sp]
+// CHECK-INST: rcwclrpl x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0x63,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  196393e5      <unknown>
+
+rcwsclrp   x1, x0, [x4]
+// CHECK-INST: rcwsclrp x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0x20,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59209081      <unknown>
+
+rcwsclrpa  x1, x0, [x4]
+// CHECK-INST: rcwsclrpa x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0xa0,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59a09081      <unknown>
+
+rcwsclrpal x1, x0, [x4]
+// CHECK-INST: rcwsclrpal x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0xe0,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59e09081      <unknown>
+
+rcwsclrpl  x1, x0, [x4]
+// CHECK-INST: rcwsclrpl x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0x90,0x60,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59609081      <unknown>
+
+rcwsclrp   x5, x3, [sp]
+// CHECK-INST: rcwsclrp x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0x23,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  592393e5      <unknown>
+
+rcwsclrpa  x5, x3, [sp]
+// CHECK-INST: rcwsclrpa x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0xa3,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59a393e5      <unknown>
+
+rcwsclrpal x5, x3, [sp]
+// CHECK-INST: rcwsclrpal x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0xe3,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59e393e5      <unknown>
+
+rcwsclrpl  x5, x3, [sp]
+// CHECK-INST: rcwsclrpl x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0x93,0x63,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  596393e5      <unknown>
+
+rcwset   x0, x1, [x4]
+// CHECK-INST: rcwset x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0x20,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  3820b081      <unknown>
+
+rcwseta  x0, x1, [x4]
+// CHECK-INST: rcwseta x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0xa0,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38a0b081      <unknown>
+
+rcwsetal x0, x1, [x4]
+// CHECK-INST: rcwsetal x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0xe0,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38e0b081      <unknown>
+
+rcwsetl  x0, x1, [x4]
+// CHECK-INST: rcwsetl x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0x60,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  3860b081      <unknown>
+
+rcwset   x3, x5, [sp]
+// CHECK-INST: rcwset x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0x23,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  3823b3e5      <unknown>
+
+rcwseta  x3, x5, [sp]
+// CHECK-INST: rcwseta x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0xa3,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38a3b3e5      <unknown>
+
+rcwsetal x3, x5, [sp]
+// CHECK-INST: rcwsetal x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0xe3,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38e3b3e5      <unknown>
+
+rcwsetl  x3, x5, [sp]
+// CHECK-INST: rcwsetl x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0x63,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  3863b3e5      <unknown>
+
+rcwsset   x0, x1, [x4]
+// CHECK-INST: rcwsset x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0x20,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  7820b081      <unknown>
+
+rcwsseta  x0, x1, [x4]
+// CHECK-INST: rcwsseta x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0xa0,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78a0b081      <unknown>
+
+rcwssetal x0, x1, [x4]
+// CHECK-INST: rcwssetal x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0xe0,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78e0b081      <unknown>
+
+rcwssetl  x0, x1, [x4]
+// CHECK-INST: rcwssetl x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0x60,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  7860b081      <unknown>
+
+rcwsset   x3, x5, [sp]
+// CHECK-INST: rcwsset x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0x23,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  7823b3e5      <unknown>
+
+rcwsseta  x3, x5, [sp]
+// CHECK-INST: rcwsseta x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0xa3,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78a3b3e5      <unknown>
+
+rcwssetal x3, x5, [sp]
+// CHECK-INST: rcwssetal x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0xe3,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78e3b3e5      <unknown>
+
+rcwssetl  x3, x5, [sp]
+// CHECK-INST: rcwssetl x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0x63,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  7863b3e5      <unknown>
+
+rcwsetp   x1, x0, [x4]
+// CHECK-INST: rcwsetp x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0x20,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  1920b081      <unknown>
+
+rcwsetpa  x1, x0, [x4]
+// CHECK-INST: rcwsetpa x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0xa0,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19a0b081      <unknown>
+
+rcwsetpal x1, x0, [x4]
+// CHECK-INST: rcwsetpal x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0xe0,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19e0b081      <unknown>
+
+rcwsetpl  x1, x0, [x4]
+// CHECK-INST: rcwsetpl x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0x60,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  1960b081      <unknown>
+
+rcwsetp   x5, x3, [sp]
+// CHECK-INST: rcwsetp x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0x23,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  1923b3e5      <unknown>
+
+rcwsetpa  x5, x3, [sp]
+// CHECK-INST: rcwsetpa x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0xa3,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19a3b3e5      <unknown>
+
+rcwsetpal x5, x3, [sp]
+// CHECK-INST: rcwsetpal x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0xe3,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  19e3b3e5      <unknown>
+
+rcwsetpl  x5, x3, [sp]
+// CHECK-INST: rcwsetpl x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0x63,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  1963b3e5      <unknown>
+
+rcwssetp   x1, x0, [x4]
+// CHECK-INST: rcwssetp x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0x20,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  5920b081      <unknown>
+
+rcwssetpa  x1, x0, [x4]
+// CHECK-INST: rcwssetpa x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0xa0,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  59a0b081      <unknown>
+
+rcwssetpal x1, x0, [x4]
+// CHECK-INST: rcwssetpal x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0xe0,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  59e0b081      <unknown>
+
+rcwssetpl  x1, x0, [x4]
+// CHECK-INST: rcwssetpl x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xb0,0x60,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  5960b081      <unknown>
+
+rcwssetp   x5, x3, [sp]
+// CHECK-INST: rcwssetp x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0x23,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  5923b3e5      <unknown>
+
+rcwssetpa  x5, x3, [sp]
+// CHECK-INST: rcwssetpa x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0xa3,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  59a3b3e5      <unknown>
+
+rcwssetpal x5, x3, [sp]
+// CHECK-INST: rcwssetpal x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0xe3,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  59e3b3e5      <unknown>
+
+rcwssetpl  x5, x3, [sp]
+// CHECK-INST: rcwssetpl x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xb3,0x63,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// ERROR-NO-D128: error: instruction requires: d128
+// CHECK-UNKNOWN:  5963b3e5      <unknown>
+
+rcwswp   x0, x1, [x4]
+// CHECK-INST: rcwswp x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0x20,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  3820a081      <unknown>
+
+rcwswpa  x0, x1, [x4]
+// CHECK-INST: rcwswpa x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0xa0,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38a0a081      <unknown>
+
+rcwswpal x0, x1, [x4]
+// CHECK-INST: rcwswpal x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0xe0,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38e0a081      <unknown>
+
+rcwswpl  x0, x1, [x4]
+// CHECK-INST: rcwswpl x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0x60,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  3860a081      <unknown>
+
+rcwswp   x3, x5, [sp]
+// CHECK-INST: rcwswp x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0x23,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  3823a3e5      <unknown>
+
+rcwswpa  x3, x5, [sp]
+// CHECK-INST: rcwswpa x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0xa3,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38a3a3e5      <unknown>
+
+rcwswpal x3, x5, [sp]
+// CHECK-INST: rcwswpal x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0xe3,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  38e3a3e5      <unknown>
+
+rcwswpl  x3, x5, [sp]
+// CHECK-INST: rcwswpl x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0x63,0x38]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  3863a3e5      <unknown>
+
+rcwsswp   x0, x1, [x4]
+// CHECK-INST: rcwsswp x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0x20,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  7820a081      <unknown>
+
+rcwsswpa  x0, x1, [x4]
+// CHECK-INST: rcwsswpa x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0xa0,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78a0a081      <unknown>
+
+rcwsswpal x0, x1, [x4]
+// CHECK-INST: rcwsswpal x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0xe0,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78e0a081      <unknown>
+
+rcwsswpl  x0, x1, [x4]
+// CHECK-INST: rcwsswpl x0, x1, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0x60,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  7860a081      <unknown>
+
+rcwsswp   x3, x5, [sp]
+// CHECK-INST: rcwsswp x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0x23,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  7823a3e5      <unknown>
+
+rcwsswpa  x3, x5, [sp]
+// CHECK-INST: rcwsswpa x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0xa3,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78a3a3e5      <unknown>
+
+rcwsswpal x3, x5, [sp]
+// CHECK-INST: rcwsswpal x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0xe3,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  78e3a3e5      <unknown>
+
+rcwsswpl  x3, x5, [sp]
+// CHECK-INST: rcwsswpl x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0x63,0x78]
+// CHECK-ERROR: error: instruction requires: the
+// CHECK-UNKNOWN:  7863a3e5      <unknown>
+
+rcwswpp   x1, x0, [x4]
+// CHECK-INST: rcwswpp x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0x20,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  1920a081      <unknown>
+
+rcwswppa  x1, x0, [x4]
+// CHECK-INST: rcwswppa x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0xa0,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  19a0a081      <unknown>
+
+rcwswppal x1, x0, [x4]
+// CHECK-INST: rcwswppal x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0xe0,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  19e0a081      <unknown>
+
+rcwswppl  x1, x0, [x4]
+// CHECK-INST: rcwswppl x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0x60,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  1960a081      <unknown>
+
+rcwswpp   x5, x3, [sp]
+// CHECK-INST: rcwswpp x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0x23,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  1923a3e5      <unknown>
+
+rcwswppa  x5, x3, [sp]
+// CHECK-INST: rcwswppa x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0xa3,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  19a3a3e5      <unknown>
+
+rcwswppal x5, x3, [sp]
+// CHECK-INST: rcwswppal x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0xe3,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  19e3a3e5      <unknown>
+
+rcwswppl  x5, x3, [sp]
+// CHECK-INST: rcwswppl x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0x63,0x19]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  1963a3e5      <unknown>
+
+rcwsswpp   x1, x0, [x4]
+// CHECK-INST: rcwsswpp x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0x20,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  5920a081      <unknown>
+
+rcwsswppa  x1, x0, [x4]
+// CHECK-INST: rcwsswppa x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0xa0,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59a0a081      <unknown>
+
+rcwsswppal x1, x0, [x4]
+// CHECK-INST: rcwsswppal x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0xe0,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59e0a081      <unknown>
+
+rcwsswppl  x1, x0, [x4]
+// CHECK-INST: rcwsswppl x1, x0, [x4]
+// CHECK-ENCODING: encoding: [0x81,0xa0,0x60,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  5960a081      <unknown>
+
+rcwsswpp   x5, x3, [sp]
+// CHECK-INST: rcwsswpp x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0x23,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  5923a3e5      <unknown>
+
+rcwsswppa  x5, x3, [sp]
+// CHECK-INST: rcwsswppa x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0xa3,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59a3a3e5      <unknown>
+
+rcwsswppal x5, x3, [sp]
+// CHECK-INST: rcwsswppal x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0xe3,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  59e3a3e5      <unknown>
+
+rcwsswppl  x5, x3, [sp]
+// CHECK-INST: rcwsswppl x5, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe5,0xa3,0x63,0x59]
+// CHECK-ERROR: error: instruction requires: d128 the
+// CHECK-UNKNOWN:  5963a3e5      <unknown>
diff --git a/llvm/test/MC/AArch64/armv9-mrrs-diagnostics.s b/llvm/test/MC/AArch64/armv9-mrrs-diagnostics.s
new file mode 100644
index 0000000000000..4eb8861784653
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9-mrrs-diagnostics.s
@@ -0,0 +1,30 @@
+// +the required for RCWSMASK_EL1, RCWMASK_EL1
+// +el2vmsa required for TTBR0_EL2 (VSCTLR_EL2), VTTBR_EL2
+// +vh required for TTBR1_EL2
+
+// RUN: not llvm-mc -triple=aarch64 -mattr=+d128,+the,+el2vmsa,+vh -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+
+mrrs x0, x2, TTBR0_EL1
+// CHECK-ERROR: error: expected second odd register of a consecutive same-size even/odd register pair
+
+mrrs x0, TTBR0_EL1
+// CHECK-ERROR: error: expected second odd register of a consecutive same-size even/odd register pair
+
+mrrs x1, x2, TTBR0_EL1
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+
+mrrs x31, x0, TTBR0_EL1
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+
+mrrs xzr, x30, TTBR0_EL1
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+
+mrrs xzr, TTBR0_EL1
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+
+mrrs S3_0_c2_c0_1
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+
+mrrs S3_0_c2_c0_1, x0, x1
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
diff --git a/llvm/test/MC/AArch64/armv9-mrrs.s b/llvm/test/MC/AArch64/armv9-mrrs.s
index 870127826cb1f..1fc7274f64d4a 100644
--- a/llvm/test/MC/AArch64/armv9-mrrs.s
+++ b/llvm/test/MC/AArch64/armv9-mrrs.s
@@ -1,100 +1,282 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+d128,+the,+el2vmsa,+vh < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -mattr=+the,+el2vmsa,+vh -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+d128,+the,+el2vmsa,+vh < %s \
+// RUN:        | llvm-objdump -d --mattr=+d128,+the,+el2vmsa,+vh - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+d128,+the,+el2vmsa,+vh < %s \
+// RUN:   | llvm-objdump -d --mattr=-d128,+the,+el2vmsa,+vh - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+d128,+the,+el2vmsa,+vh < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+d128,+the,+el2vmsa,+vh -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 // +the required for RCWSMASK_EL1, RCWMASK_EL1
 // +el2vmsa required for TTBR0_EL2 (VSCTLR_EL2), VTTBR_EL2
 // +vh required for TTBR1_EL2
 
-// RUN: not llvm-mc -triple aarch64 -mattr=+d128,+the,+el2vmsa,+vh -show-encoding %s -o - 2> %t | FileCheck %s
-// RUN: FileCheck %s --input-file=%t --check-prefix=ERRORS
-
-// RUN: not llvm-mc -triple aarch64 -mattr=+the,+el2vmsa,+vh -show-encoding %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR-NO-D128
-
-          mrrs  x0, x1, TTBR0_EL1
-// CHECK: mrrs  x0, x1, TTBR0_EL1           // encoding: [0x00,0x20,0x78,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x0, x1, TTBR1_EL1
-// CHECK: mrrs  x0, x1, TTBR1_EL1           // encoding: [0x20,0x20,0x78,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x0, x1, PAR_EL1
-// CHECK: mrrs  x0, x1, PAR_EL1             // encoding: [0x00,0x74,0x78,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x0, x1, RCWSMASK_EL1
-// CHECK: mrrs  x0, x1, RCWSMASK_EL1        // encoding: [0x60,0xd0,0x78,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x0, x1, RCWMASK_EL1
-// CHECK: mrrs  x0, x1, RCWMASK_EL1         // encoding: [0xc0,0xd0,0x78,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x0, x1, TTBR0_EL2
-// CHECK: mrrs  x0, x1, TTBR0_EL2           // encoding: [0x00,0x20,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x0, x1, TTBR1_EL2
-// CHECK: mrrs  x0, x1, TTBR1_EL2           // encoding: [0x20,0x20,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x0, x1, VTTBR_EL2
-// CHECK: mrrs  x0, x1, VTTBR_EL2           // encoding: [0x00,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          mrrs   x0,  x1, VTTBR_EL2
-// CHECK: mrrs   x0,  x1, VTTBR_EL2           // encoding: [0x00,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs   x2,  x3, VTTBR_EL2
-// CHECK: mrrs   x2,  x3, VTTBR_EL2           // encoding: [0x02,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs   x4,  x5, VTTBR_EL2
-// CHECK: mrrs   x4,  x5, VTTBR_EL2           // encoding: [0x04,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs   x6,  x7, VTTBR_EL2
-// CHECK: mrrs   x6,  x7, VTTBR_EL2           // encoding: [0x06,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs   x8,  x9, VTTBR_EL2
-// CHECK: mrrs   x8,  x9, VTTBR_EL2           // encoding: [0x08,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x10, x11, VTTBR_EL2
-// CHECK: mrrs  x10, x11, VTTBR_EL2           // encoding: [0x0a,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x12, x13, VTTBR_EL2
-// CHECK: mrrs  x12, x13, VTTBR_EL2           // encoding: [0x0c,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x14, x15, VTTBR_EL2
-// CHECK: mrrs  x14, x15, VTTBR_EL2           // encoding: [0x0e,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x16, x17, VTTBR_EL2
-// CHECK: mrrs  x16, x17, VTTBR_EL2           // encoding: [0x10,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x18, x19, VTTBR_EL2
-// CHECK: mrrs  x18, x19, VTTBR_EL2           // encoding: [0x12,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x20, x21, VTTBR_EL2
-// CHECK: mrrs  x20, x21, VTTBR_EL2           // encoding: [0x14,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x22, x23, VTTBR_EL2
-// CHECK: mrrs  x22, x23, VTTBR_EL2           // encoding: [0x16,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x24, x25, VTTBR_EL2
-// CHECK: mrrs  x24, x25, VTTBR_EL2           // encoding: [0x18,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          mrrs  x26, x27, VTTBR_EL2
-// CHECK: mrrs  x26, x27, VTTBR_EL2           // encoding: [0x1a,0x21,0x7c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          mrrs x0, x2, TTBR0_EL1
-// ERRORS: error: expected second odd register of a consecutive same-size even/odd register pair
-
-          mrrs x0, TTBR0_EL1
-// ERRORS: error: expected second odd register of a consecutive same-size even/odd register pair
-
-          mrrs x1, x2, TTBR0_EL1
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          mrrs x31, x0, TTBR0_EL1
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          mrrs xzr, x30, TTBR0_EL1
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          mrrs xzr, TTBR0_EL1
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          mrrs S3_0_c2_c0_1
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          mrrs S3_0_c2_c0_1, x0, x1
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
+mrrs  x0, x1, TTBR0_EL1
+// CHECK-INST: mrrs x0, x1, TTBR0_EL1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x78,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5782000      <unknown>
+
+mrrs  x0, x1, TTBR1_EL1
+// CHECK-INST: mrrs x0, x1, TTBR1_EL1
+// CHECK-ENCODING: encoding: [0x20,0x20,0x78,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5782020      <unknown>
+
+mrrs  x0, x1, PAR_EL1
+// CHECK-INST: mrrs x0, x1, PAR_EL1
+// CHECK-ENCODING: encoding: [0x00,0x74,0x78,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5787400      <unknown>
+
+mrrs  x0, x1, RCWSMASK_EL1
+// CHECK-INST: mrrs x0, x1, RCWSMASK_EL1
+// CHECK-ENCODING: encoding: [0x60,0xd0,0x78,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d578d060      <unknown>
+
+mrrs  x0, x1, RCWMASK_EL1
+// CHECK-INST: mrrs x0, x1, RCWMASK_EL1
+// CHECK-ENCODING: encoding: [0xc0,0xd0,0x78,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d578d0c0      <unknown>
+
+mrrs  x0, x1, TTBR0_EL2
+// CHECK-INST: mrrs x0, x1, TTBR0_EL2
+// CHECK-ENCODING: encoding: [0x00,0x20,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2000      <unknown>
+
+mrrs  x0, x1, TTBR1_EL2
+// CHECK-INST: mrrs x0, x1, TTBR1_EL2
+// CHECK-ENCODING: encoding: [0x20,0x20,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2020      <unknown>
+
+mrrs  x0, x1, VTTBR_EL2
+// CHECK-INST: mrrs x0, x1, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x00,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2100      <unknown>
+
+mrrs   x0,  x1, VTTBR_EL2
+// CHECK-INST: mrrs x0, x1, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x00,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2100      <unknown>
+
+mrrs   x2,  x3, VTTBR_EL2
+// CHECK-INST: mrrs x2, x3, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x02,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2102      <unknown>
+
+mrrs   x4,  x5, VTTBR_EL2
+// CHECK-INST: mrrs x4, x5, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x04,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2104      <unknown>
+
+mrrs   x6,  x7, VTTBR_EL2
+// CHECK-INST: mrrs x6, x7, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x06,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2106      <unknown>
+
+mrrs   x8,  x9, VTTBR_EL2
+// CHECK-INST: mrrs x8, x9, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x08,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2108      <unknown>
+
+mrrs  x10, x11, VTTBR_EL2
+// CHECK-INST: mrrs x10, x11, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x0a,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c210a      <unknown>
+
+mrrs  x12, x13, VTTBR_EL2
+// CHECK-INST: mrrs x12, x13, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x0c,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c210c      <unknown>
+
+mrrs  x14, x15, VTTBR_EL2
+// CHECK-INST: mrrs x14, x15, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x0e,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c210e      <unknown>
+
+mrrs  x16, x17, VTTBR_EL2
+// CHECK-INST: mrrs x16, x17, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x10,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2110      <unknown>
+
+mrrs  x18, x19, VTTBR_EL2
+// CHECK-INST: mrrs x18, x19, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x12,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2112      <unknown>
+
+mrrs  x20, x21, VTTBR_EL2
+// CHECK-INST: mrrs x20, x21, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x14,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2114      <unknown>
+
+mrrs  x22, x23, VTTBR_EL2
+// CHECK-INST: mrrs x22, x23, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x16,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2116      <unknown>
+
+mrrs  x24, x25, VTTBR_EL2
+// CHECK-INST: mrrs x24, x25, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x18,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c2118      <unknown>
+
+mrrs  x26, x27, VTTBR_EL2
+// CHECK-INST: mrrs x26, x27, VTTBR_EL2
+// CHECK-ENCODING: encoding: [0x1a,0x21,0x7c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d57c211a      <unknown>
+
+msrr  TTBR0_EL1, x0, x1
+// CHECK-INST: msrr TTBR0_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5582000      <unknown>
+
+msrr  TTBR1_EL1, x0, x1
+// CHECK-INST: msrr TTBR1_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0x20,0x20,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5582020      <unknown>
+
+msrr  PAR_EL1, x0, x1
+// CHECK-INST: msrr PAR_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x74,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5587400      <unknown>
+
+msrr  RCWSMASK_EL1, x0, x1
+// CHECK-INST: msrr RCWSMASK_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0x60,0xd0,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d558d060      <unknown>
+
+msrr  RCWMASK_EL1, x0, x1
+// CHECK-INST: msrr RCWMASK_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0xc0,0xd0,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d558d0c0      <unknown>
+
+msrr  TTBR0_EL2, x0, x1
+// CHECK-INST: msrr TTBR0_EL2, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2000      <unknown>
+
+msrr  TTBR1_EL2, x0, x1
+// CHECK-INST: msrr TTBR1_EL2, x0, x1
+// CHECK-ENCODING: encoding: [0x20,0x20,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2020      <unknown>
+
+msrr  VTTBR_EL2, x0, x1
+// CHECK-INST: msrr VTTBR_EL2, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2100      <unknown>
+
+msrr   VTTBR_EL2, x0, x1
+// CHECK-INST: msrr VTTBR_EL2, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2100      <unknown>
+
+msrr   VTTBR_EL2, x2, x3
+// CHECK-INST: msrr VTTBR_EL2, x2, x3
+// CHECK-ENCODING: encoding: [0x02,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2102      <unknown>
+
+msrr   VTTBR_EL2, x4, x5
+// CHECK-INST: msrr VTTBR_EL2, x4, x5
+// CHECK-ENCODING: encoding: [0x04,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2104      <unknown>
+
+msrr   VTTBR_EL2, x6, x7
+// CHECK-INST: msrr VTTBR_EL2, x6, x7
+// CHECK-ENCODING: encoding: [0x06,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2106      <unknown>
+
+msrr   VTTBR_EL2, x8, x9
+// CHECK-INST: msrr VTTBR_EL2, x8, x9
+// CHECK-ENCODING: encoding: [0x08,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2108      <unknown>
+
+msrr   VTTBR_EL2, x10, x11
+// CHECK-INST: msrr VTTBR_EL2, x10, x11
+// CHECK-ENCODING: encoding: [0x0a,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c210a      <unknown>
+
+msrr   VTTBR_EL2, x12, x13
+// CHECK-INST: msrr VTTBR_EL2, x12, x13
+// CHECK-ENCODING: encoding: [0x0c,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c210c      <unknown>
+
+msrr   VTTBR_EL2, x14, x15
+// CHECK-INST: msrr VTTBR_EL2, x14, x15
+// CHECK-ENCODING: encoding: [0x0e,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c210e      <unknown>
+
+msrr   VTTBR_EL2, x16, x17
+// CHECK-INST: msrr VTTBR_EL2, x16, x17
+// CHECK-ENCODING: encoding: [0x10,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2110      <unknown>
+
+msrr   VTTBR_EL2, x18, x19
+// CHECK-INST: msrr VTTBR_EL2, x18, x19
+// CHECK-ENCODING: encoding: [0x12,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2112      <unknown>
+
+msrr   VTTBR_EL2, x20, x21
+// CHECK-INST: msrr VTTBR_EL2, x20, x21
+// CHECK-ENCODING: encoding: [0x14,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2114      <unknown>
+
+msrr   VTTBR_EL2, x22, x23
+// CHECK-INST: msrr VTTBR_EL2, x22, x23
+// CHECK-ENCODING: encoding: [0x16,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2116      <unknown>
+
+msrr   VTTBR_EL2, x24, x25
+// CHECK-INST: msrr VTTBR_EL2, x24, x25
+// CHECK-ENCODING: encoding: [0x18,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2118      <unknown>
+
+msrr   VTTBR_EL2, x26, x27
+// CHECK-INST: msrr VTTBR_EL2, x26, x27
+// CHECK-ENCODING: encoding: [0x1a,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c211a      <unknown>
diff --git a/llvm/test/MC/AArch64/armv9-msrr-diagnostics.s b/llvm/test/MC/AArch64/armv9-msrr-diagnostics.s
new file mode 100644
index 0000000000000..d49a3ee68b63f
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9-msrr-diagnostics.s
@@ -0,0 +1,30 @@
+// +the required for RCWSMASK_EL1, RCWMASK_EL1
+// +el2vmsa required for TTBR0_EL2 (VSCTLR_EL2), VTTBR_EL2
+// +vh required for TTBR1_EL2
+
+// RUN: not llvm-mc -triple=aarch64 -mattr=+d128,+the,+el2vmsa,+vh -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+
+msrr TTBR0_EL1, x0, x2
+// CHECK-ERROR: error: expected second odd register of a consecutive same-size even/odd register pair
+
+msrr TTBR0_EL1, x0
+// CHECK-ERROR: error: expected comma
+
+msrr TTBR0_EL1, x1, x2
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+
+msrr TTBR0_EL1, x31, x0
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+
+msrr TTBR0_EL1, xzr, x30
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+
+msrr TTBR0_EL1, xzr
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
+
+msrr S3_0_c2_c0_1
+// CHECK-ERROR: error: too few operands for instruction
+
+msrr x0, x1, S3_0_c2_c0_1
+// CHECK-ERROR: error: expected first even register of a consecutive same-size even/odd register pair
diff --git a/llvm/test/MC/AArch64/armv9-msrr.s b/llvm/test/MC/AArch64/armv9-msrr.s
index 2be17a71e7d47..439e054779b07 100644
--- a/llvm/test/MC/AArch64/armv9-msrr.s
+++ b/llvm/test/MC/AArch64/armv9-msrr.s
@@ -1,100 +1,150 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+d128,+the,+el2vmsa,+vh < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -mattr=+the,+el2vmsa,+vh -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+d128,+the,+el2vmsa,+vh < %s \
+// RUN:        | llvm-objdump -d --mattr=+d128,+the,+el2vmsa,+vh - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+d128,+the,+el2vmsa,+vh < %s \
+// RUN:   | llvm-objdump -d --mattr=-d128,+the,+el2vmsa,+vh - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+d128,+the,+el2vmsa,+vh < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+d128,+the,+el2vmsa,+vh -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 // +the required for RCWSMASK_EL1, RCWMASK_EL1
 // +el2vmsa required for TTBR0_EL2 (VSCTLR_EL2), VTTBR_EL2
 // +vh required for TTBR1_EL2
 
-// RUN: not llvm-mc -triple aarch64 -mattr=+d128,+the,+el2vmsa,+vh -show-encoding %s -o - 2> %t | FileCheck %s
-// RUN: FileCheck %s --input-file=%t --check-prefix=ERRORS
-
-// RUN: not llvm-mc -triple aarch64 -mattr=+the,+el2vmsa,+vh -show-encoding %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR-NO-D128
-
-          msrr  TTBR0_EL1, x0, x1
-// CHECK: msrr  TTBR0_EL1, x0, x1           // encoding: [0x00,0x20,0x58,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr  TTBR1_EL1, x0, x1
-// CHECK: msrr  TTBR1_EL1, x0, x1           // encoding: [0x20,0x20,0x58,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr  PAR_EL1, x0, x1
-// CHECK: msrr  PAR_EL1, x0, x1             // encoding: [0x00,0x74,0x58,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr  RCWSMASK_EL1, x0, x1
-// CHECK: msrr  RCWSMASK_EL1, x0, x1        // encoding: [0x60,0xd0,0x58,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr  RCWMASK_EL1, x0, x1
-// CHECK: msrr  RCWMASK_EL1, x0, x1         // encoding: [0xc0,0xd0,0x58,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr  TTBR0_EL2, x0, x1
-// CHECK: msrr  TTBR0_EL2, x0, x1           // encoding: [0x00,0x20,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr  TTBR1_EL2, x0, x1
-// CHECK: msrr  TTBR1_EL2, x0, x1           // encoding: [0x20,0x20,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr  VTTBR_EL2, x0, x1
-// CHECK: msrr  VTTBR_EL2, x0, x1           // encoding: [0x00,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          msrr   VTTBR_EL2, x0, x1
-// CHECK: msrr   VTTBR_EL2, x0, x1           // encoding: [0x00,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x2, x3
-// CHECK: msrr   VTTBR_EL2, x2, x3           // encoding: [0x02,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x4, x5
-// CHECK: msrr   VTTBR_EL2, x4, x5           // encoding: [0x04,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x6, x7
-// CHECK: msrr   VTTBR_EL2, x6, x7           // encoding: [0x06,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x8, x9
-// CHECK: msrr   VTTBR_EL2, x8, x9           // encoding: [0x08,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x10, x11
-// CHECK: msrr   VTTBR_EL2, x10, x11           // encoding: [0x0a,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x12, x13
-// CHECK: msrr   VTTBR_EL2, x12, x13           // encoding: [0x0c,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x14, x15
-// CHECK: msrr   VTTBR_EL2, x14, x15           // encoding: [0x0e,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x16, x17
-// CHECK: msrr   VTTBR_EL2, x16, x17           // encoding: [0x10,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x18, x19
-// CHECK: msrr   VTTBR_EL2, x18, x19           // encoding: [0x12,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x20, x21
-// CHECK: msrr   VTTBR_EL2, x20, x21           // encoding: [0x14,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x22, x23
-// CHECK: msrr   VTTBR_EL2, x22, x23           // encoding: [0x16,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x24, x25
-// CHECK: msrr   VTTBR_EL2, x24, x25           // encoding: [0x18,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          msrr   VTTBR_EL2, x26, x27
-// CHECK: msrr   VTTBR_EL2, x26, x27           // encoding: [0x1a,0x21,0x5c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          msrr TTBR0_EL1, x0, x2
-// ERRORS: error: expected second odd register of a consecutive same-size even/odd register pair
-
-          msrr TTBR0_EL1, x0
-// ERRORS: error: expected comma
-
-          msrr TTBR0_EL1, x1, x2
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          msrr TTBR0_EL1, x31, x0
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          msrr TTBR0_EL1, xzr, x30
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          msrr TTBR0_EL1, xzr
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          msrr S3_0_c2_c0_1
-// ERRORS: error: too few operands for instruction
-
-          msrr x0, x1, S3_0_c2_c0_1
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
+msrr  TTBR0_EL1, x0, x1
+// CHECK-INST: msrr TTBR0_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5582000      <unknown>
+
+msrr  TTBR1_EL1, x0, x1
+// CHECK-INST: msrr TTBR1_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0x20,0x20,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5582020      <unknown>
+
+msrr  PAR_EL1, x0, x1
+// CHECK-INST: msrr PAR_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x74,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5587400      <unknown>
+
+msrr  RCWSMASK_EL1, x0, x1
+// CHECK-INST: msrr RCWSMASK_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0x60,0xd0,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d558d060      <unknown>
+
+msrr  RCWMASK_EL1, x0, x1
+// CHECK-INST: msrr RCWMASK_EL1, x0, x1
+// CHECK-ENCODING: encoding: [0xc0,0xd0,0x58,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d558d0c0      <unknown>
+
+msrr  TTBR0_EL2, x0, x1
+// CHECK-INST: msrr TTBR0_EL2, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2000      <unknown>
+
+msrr  TTBR1_EL2, x0, x1
+// CHECK-INST: msrr TTBR1_EL2, x0, x1
+// CHECK-ENCODING: encoding: [0x20,0x20,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2020      <unknown>
+
+msrr  VTTBR_EL2, x0, x1
+// CHECK-INST: msrr VTTBR_EL2, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2100      <unknown>
+
+msrr   VTTBR_EL2, x0, x1
+// CHECK-INST: msrr VTTBR_EL2, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2100      <unknown>
+
+msrr   VTTBR_EL2, x2, x3
+// CHECK-INST: msrr VTTBR_EL2, x2, x3
+// CHECK-ENCODING: encoding: [0x02,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2102      <unknown>
+
+msrr   VTTBR_EL2, x4, x5
+// CHECK-INST: msrr VTTBR_EL2, x4, x5
+// CHECK-ENCODING: encoding: [0x04,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2104      <unknown>
+
+msrr   VTTBR_EL2, x6, x7
+// CHECK-INST: msrr VTTBR_EL2, x6, x7
+// CHECK-ENCODING: encoding: [0x06,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2106      <unknown>
+
+msrr   VTTBR_EL2, x8, x9
+// CHECK-INST: msrr VTTBR_EL2, x8, x9
+// CHECK-ENCODING: encoding: [0x08,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2108      <unknown>
+
+msrr   VTTBR_EL2, x10, x11
+// CHECK-INST: msrr VTTBR_EL2, x10, x11
+// CHECK-ENCODING: encoding: [0x0a,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c210a      <unknown>
+
+msrr   VTTBR_EL2, x12, x13
+// CHECK-INST: msrr VTTBR_EL2, x12, x13
+// CHECK-ENCODING: encoding: [0x0c,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c210c      <unknown>
+
+msrr   VTTBR_EL2, x14, x15
+// CHECK-INST: msrr VTTBR_EL2, x14, x15
+// CHECK-ENCODING: encoding: [0x0e,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c210e      <unknown>
+
+msrr   VTTBR_EL2, x16, x17
+// CHECK-INST: msrr VTTBR_EL2, x16, x17
+// CHECK-ENCODING: encoding: [0x10,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2110      <unknown>
+
+msrr   VTTBR_EL2, x18, x19
+// CHECK-INST: msrr VTTBR_EL2, x18, x19
+// CHECK-ENCODING: encoding: [0x12,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2112      <unknown>
+
+msrr   VTTBR_EL2, x20, x21
+// CHECK-INST: msrr VTTBR_EL2, x20, x21
+// CHECK-ENCODING: encoding: [0x14,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2114      <unknown>
+
+msrr   VTTBR_EL2, x22, x23
+// CHECK-INST: msrr VTTBR_EL2, x22, x23
+// CHECK-ENCODING: encoding: [0x16,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2116      <unknown>
+
+msrr   VTTBR_EL2, x24, x25
+// CHECK-INST: msrr VTTBR_EL2, x24, x25
+// CHECK-ENCODING: encoding: [0x18,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c2118      <unknown>
+
+msrr   VTTBR_EL2, x26, x27
+// CHECK-INST: msrr VTTBR_EL2, x26, x27
+// CHECK-ENCODING: encoding: [0x1a,0x21,0x5c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: d128
+// CHECK-UNKNOWN:  d55c211a      <unknown>
diff --git a/llvm/test/MC/AArch64/armv9-sysp-diagnostics.s b/llvm/test/MC/AArch64/armv9-sysp-diagnostics.s
new file mode 100644
index 0000000000000..8b466c10b0aa7
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9-sysp-diagnostics.s
@@ -0,0 +1,35 @@
+// +tbl-rmi required for RIPA*/RVA*
+// +xs required for *NXS
+
+// RUN: not llvm-mc -triple aarch64 -mattr=+d128,+tlb-rmi,+xs -show-encoding %s -o - 2>&1 | FileCheck %s --check-prefix=ERRORS
+
+// sysp #<op1>, <Cn>, <Cm>, #<op2>{, <Xt1>, <Xt2>}
+// registers with 128-bit formats (op0, op1, Cn, Cm, op2)
+// For sysp, op0 is 0
+
+sysp #0, c2, c0, #0, x0, x2
+// ERRORS: error: expected second odd register of a consecutive same-size even/odd register pair
+sysp #0, c2, c0, #0, x0
+// ERRORS: error: expected comma
+sysp #0, c2, c0, #0, x1, x2
+// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
+sysp #0, c2, c0, #0, x31, x0
+// ERRORS: error: xzr must be followed by xzr
+sysp #0, c2, c0, #0, xzr, x30
+// ERRORS: error: xzr must be followed by xzr
+sysp #0, c2, c0, #0, xzr
+// ERRORS: error: expected comma
+sysp #0, c2, c0, #0, xzr,
+// ERRORS: error: expected register operand
+
+
+tlbip RVAE3IS
+// ERRORS: error: expected comma
+tlbip RVAE3IS,
+// ERRORS: error: expected register identifier
+tlbip VAE3,
+// ERRORS: error: expected register identifier
+tlbip IPAS2E1, x4, x8
+// ERRORS: error: specified tlbip op requires a pair of registers
+tlbip RVAE3, x11, x11
+// ERRORS: error: specified tlbip op requires a pair of registers
diff --git a/llvm/test/MC/AArch64/armv9-sysp.s b/llvm/test/MC/AArch64/armv9-sysp.s
deleted file mode 100644
index 908e880ad0aa4..0000000000000
--- a/llvm/test/MC/AArch64/armv9-sysp.s
+++ /dev/null
@@ -1,538 +0,0 @@
-// +tbl-rmi required for RIPA*/RVA*
-// +xs required for *NXS
-
-// RUN: not llvm-mc -triple aarch64 -mattr=+d128,+tlb-rmi,+xs -show-encoding %s -o - 2> %t | FileCheck %s
-// RUN: FileCheck %s --input-file=%t --check-prefix=ERRORS
-
-// RUN: not llvm-mc -triple aarch64 -mattr=+tlb-rmi,+xs -show-encoding %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR-NO-D128
-
-// sysp #<op1>, <Cn>, <Cm>, #<op2>{, <Xt1>, <Xt2>}
-// registers with 128-bit formats (op0, op1, Cn, Cm, op2)
-// For sysp, op0 is 0
-
-          sysp #0, c2, c0, #0, x0, x1          // TTBR0_EL1     3  0  2  0  0
-// CHECK: sysp #0, c2, c0, #0, x0, x1          // encoding: [0x00,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #1, x0, x1          // TTBR1_EL1     3  0  2  0  1
-// CHECK: sysp #0, c2, c0, #1, x0, x1          // encoding: [0x20,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c7, c4, #0, x0, x1          // PAR_EL1       3  0  7  4  0
-// CHECK: sysp #0, c7, c4, #0, x0, x1          // encoding: [0x00,0x74,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c13, c0, #3, x0, x1         // RCWSMASK_EL1  3  0 13  0  3
-// CHECK: sysp #0, c13, c0, #3, x0, x1         // encoding: [0x60,0xd0,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c13, c0, #6, x0, x1         // RCWMASK_EL1   3  0 13  0  6
-// CHECK: sysp #0, c13, c0, #6, x0, x1         // encoding: [0xc0,0xd0,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #4, c2, c0, #0, x0, x1          // TTBR0_EL2     3  4  2  0  0
-// CHECK: sysp #4, c2, c0, #0, x0, x1          // encoding: [0x00,0x20,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #4, c2, c0, #1, x0, x1          // TTBR1_EL2     3  4  2  0  1
-// CHECK: sysp #4, c2, c0, #1, x0, x1          // encoding: [0x20,0x20,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #4, c2, c1, #0, x0, x1          // VTTBR_EL2     3  4  2  1  0
-// CHECK: sysp #4, c2, c1, #0, x0, x1          // encoding: [0x00,0x21,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-
-          sysp #0, c2, c0, #0, x0, x1
-// CHECK: sysp #0, c2, c0, #0, x0, x1          // encoding: [0x00,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #1, x0, x1
-// CHECK: sysp #0, c2, c0, #1, x0, x1          // encoding: [0x20,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c7, c4, #0, x0, x1
-// CHECK: sysp #0, c7, c4, #0, x0, x1          // encoding: [0x00,0x74,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c13, c0, #3, x0, x1
-// CHECK: sysp #0, c13, c0, #3, x0, x1         // encoding: [0x60,0xd0,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c13, c0, #6, x0, x1
-// CHECK: sysp #0, c13, c0, #6, x0, x1         // encoding: [0xc0,0xd0,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #4, c2, c0, #0, x0, x1
-// CHECK: sysp #4, c2, c0, #0, x0, x1          // encoding: [0x00,0x20,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #4, c2, c0, #1, x0, x1
-// CHECK: sysp #4, c2, c0, #1, x0, x1          // encoding: [0x20,0x20,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #4, c2, c1, #0, x0, x1
-// CHECK: sysp #4, c2, c1, #0, x0, x1          // encoding: [0x00,0x21,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          sysp #0, c2, c0, #0, x0, x1
-// CHECK: sysp #0, c2, c0, #0, x0, x1          // encoding: [0x00,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x2, x3
-// CHECK: sysp #0, c2, c0, #0, x2, x3          // encoding: [0x02,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x4, x5
-// CHECK: sysp #0, c2, c0, #0, x4, x5          // encoding: [0x04,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x6, x7
-// CHECK: sysp #0, c2, c0, #0, x6, x7          // encoding: [0x06,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x8, x9
-// CHECK: sysp #0, c2, c0, #0, x8, x9          // encoding: [0x08,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x10, x11
-// CHECK: sysp #0, c2, c0, #0, x10, x11        // encoding: [0x0a,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x12, x13
-// CHECK: sysp #0, c2, c0, #0, x12, x13        // encoding: [0x0c,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x14, x15
-// CHECK: sysp #0, c2, c0, #0, x14, x15        // encoding: [0x0e,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x16, x17
-// CHECK: sysp #0, c2, c0, #0, x16, x17        // encoding: [0x10,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x18, x19
-// CHECK: sysp #0, c2, c0, #0, x18, x19        // encoding: [0x12,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x20, x21
-// CHECK: sysp #0, c2, c0, #0, x20, x21        // encoding: [0x14,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x22, x23
-// CHECK: sysp #0, c2, c0, #0, x22, x23        // encoding: [0x16,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x24, x25
-// CHECK: sysp #0, c2, c0, #0, x24, x25        // encoding: [0x18,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x26, x27
-// CHECK: sysp #0, c2, c0, #0, x26, x27        // encoding: [0x1a,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x28, x29
-// CHECK: sysp #0, c2, c0, #0, x28, x29        // encoding: [0x1c,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x30, x31
-// CHECK: sysp #0, c2, c0, #0, x30, xzr        // encoding: [0x1e,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          sysp #0, c2, c0, #0, x31, x31
-// CHECK: sysp #0, c2, c0, #0                  // encoding: [0x1f,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, xzr, xzr
-// CHECK: sysp #0, c2, c0, #0                  // encoding: [0x1f,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, x31, xzr
-// CHECK: sysp #0, c2, c0, #0                  // encoding: [0x1f,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0, xzr, x31
-// CHECK: sysp #0, c2, c0, #0                  // encoding: [0x1f,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          sysp #0, c2, c0, #0
-// CHECK: sysp #0, c2, c0, #0                  // encoding: [0x1f,0x20,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-
-          sysp #0, c2, c0, #0, x0, x2
-// ERRORS: error: expected second odd register of a consecutive same-size even/odd register pair
-
-          sysp #0, c2, c0, #0, x0
-// ERRORS: error: expected comma
-
-          sysp #0, c2, c0, #0, x1, x2
-// ERRORS: error: expected first even register of a consecutive same-size even/odd register pair
-
-          sysp #0, c2, c0, #0, x31, x0
-// ERRORS: error: xzr must be followed by xzr
-
-          sysp #0, c2, c0, #0, xzr, x30
-// ERRORS: error: xzr must be followed by xzr
-
-          sysp #0, c2, c0, #0, xzr
-// ERRORS: error: expected comma
-
-          sysp #0, c2, c0, #0, xzr,
-// ERRORS: error: expected register operand
-
-
-          tlbip IPAS2E1, x4, x5
-// CHECK: tlbip ipas2e1, x4, x5                 // encoding: [0x24,0x84,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2E1NXS, x4, x5
-// CHECK: tlbip ipas2e1nxs, x4, x5              // encoding: [0x24,0x94,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2E1IS, x4, x5
-// CHECK: tlbip ipas2e1is, x4, x5               // encoding: [0x24,0x80,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2E1ISNXS, x4, x5
-// CHECK: tlbip ipas2e1isnxs, x4, x5            // encoding: [0x24,0x90,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2E1OS, x4, x5
-// CHECK: tlbip ipas2e1os, x4, x5               // encoding: [0x04,0x84,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2E1OSNXS, x4, x5
-// CHECK: tlbip ipas2e1osnxs, x4, x5            // encoding: [0x04,0x94,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2LE1, x4, x5
-// CHECK: tlbip ipas2le1, x4, x5                // encoding: [0xa4,0x84,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2LE1NXS, x4, x5
-// CHECK: tlbip ipas2le1nxs, x4, x5             // encoding: [0xa4,0x94,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2LE1IS, x4, x5
-// CHECK: tlbip ipas2le1is, x4, x5              // encoding: [0xa4,0x80,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2LE1ISNXS, x4, x5
-// CHECK: tlbip ipas2le1isnxs, x4, x5           // encoding: [0xa4,0x90,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2LE1OS, x4, x5
-// CHECK: tlbip ipas2le1os, x4, x5              // encoding: [0x84,0x84,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip IPAS2LE1OSNXS, x4, x5
-// CHECK: tlbip ipas2le1osnxs, x4, x5           // encoding: [0x84,0x94,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-
-          tlbip VAE1, x8, x9
-// CHECK: tlbip vae1, x8, x9                    // encoding: [0x28,0x87,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE1NXS, x8, x9
-// CHECK: tlbip vae1nxs, x8, x9                 // encoding: [0x28,0x97,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE1IS, x8, x9
-// CHECK: tlbip vae1is, x8, x9                  // encoding: [0x28,0x83,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE1ISNXS, x8, x9
-// CHECK: tlbip vae1isnxs, x8, x9               // encoding: [0x28,0x93,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE1OS, x8, x9
-// CHECK: tlbip vae1os, x8, x9                  // encoding: [0x28,0x81,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE1OSNXS, x8, x9
-// CHECK: tlbip vae1osnxs, x8, x9               // encoding: [0x28,0x91,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE1, x8, x9
-// CHECK: tlbip vale1, x8, x9                   // encoding: [0xa8,0x87,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE1NXS, x8, x9
-// CHECK: tlbip vale1nxs, x8, x9                // encoding: [0xa8,0x97,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE1IS, x8, x9
-// CHECK: tlbip vale1is, x8, x9                 // encoding: [0xa8,0x83,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE1ISNXS, x8, x9
-// CHECK: tlbip vale1isnxs, x8, x9              // encoding: [0xa8,0x93,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE1OS, x8, x9
-// CHECK: tlbip vale1os, x8, x9                 // encoding: [0xa8,0x81,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE1OSNXS, x8, x9
-// CHECK: tlbip vale1osnxs, x8, x9              // encoding: [0xa8,0x91,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAAE1, x8, x9
-// CHECK: tlbip vaae1, x8, x9                   // encoding: [0x68,0x87,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAAE1NXS, x8, x9
-// CHECK: tlbip vaae1nxs, x8, x9                // encoding: [0x68,0x97,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAAE1IS, x8, x9
-// CHECK: tlbip vaae1is, x8, x9                 // encoding: [0x68,0x83,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAAE1ISNXS, x8, x9
-// CHECK: tlbip vaae1isnxs, x8, x9              // encoding: [0x68,0x93,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAAE1OS, x8, x9
-// CHECK: tlbip vaae1os, x8, x9                 // encoding: [0x68,0x81,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAAE1OSNXS, x8, x9
-// CHECK: tlbip vaae1osnxs, x8, x9              // encoding: [0x68,0x91,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAALE1, x8, x9
-// CHECK: tlbip vaale1, x8, x9                  // encoding: [0xe8,0x87,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAALE1NXS, x8, x9
-// CHECK: tlbip vaale1nxs, x8, x9               // encoding: [0xe8,0x97,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAALE1IS, x8, x9
-// CHECK: tlbip vaale1is, x8, x9                // encoding: [0xe8,0x83,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAALE1ISNXS, x8, x9
-// CHECK: tlbip vaale1isnxs, x8, x9             // encoding: [0xe8,0x93,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAALE1OS, x8, x9
-// CHECK: tlbip vaale1os, x8, x9                // encoding: [0xe8,0x81,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAALE1OSNXS, x8, x9
-// CHECK: tlbip vaale1osnxs, x8, x9             // encoding: [0xe8,0x91,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          tlbip VAE2, x14, x15
-// CHECK: tlbip vae2, x14, x15                    // encoding: [0x2e,0x87,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE2NXS, x14, x15
-// CHECK: tlbip vae2nxs, x14, x15                 // encoding: [0x2e,0x97,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE2IS, x14, x15
-// CHECK: tlbip vae2is, x14, x15                  // encoding: [0x2e,0x83,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE2ISNXS, x14, x15
-// CHECK: tlbip vae2isnxs, x14, x15               // encoding: [0x2e,0x93,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE2OS, x14, x15
-// CHECK: tlbip vae2os, x14, x15                  // encoding: [0x2e,0x81,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE2OSNXS, x14, x15
-// CHECK: tlbip vae2osnxs, x14, x15               // encoding: [0x2e,0x91,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE2, x14, x15
-// CHECK: tlbip vale2, x14, x15                   // encoding: [0xae,0x87,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE2NXS, x14, x15
-// CHECK: tlbip vale2nxs, x14, x15                // encoding: [0xae,0x97,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE2IS, x14, x15
-// CHECK: tlbip vale2is, x14, x15                 // encoding: [0xae,0x83,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE2ISNXS, x14, x15
-// CHECK: tlbip vale2isnxs, x14, x15              // encoding: [0xae,0x93,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE2OS, x14, x15
-// CHECK: tlbip vale2os, x14, x15                 // encoding: [0xae,0x81,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE2OSNXS, x14, x15
-// CHECK: tlbip vale2osnxs, x14, x15              // encoding: [0xae,0x91,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          tlbip VAE3, x24, x25
-// CHECK: tlbip vae3, x24, x25                    // encoding: [0x38,0x87,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE3NXS, x24, x25
-// CHECK: tlbip vae3nxs, x24, x25                 // encoding: [0x38,0x97,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE3IS, x24, x25
-// CHECK: tlbip vae3is, x24, x25                  // encoding: [0x38,0x83,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE3ISNXS, x24, x25
-// CHECK: tlbip vae3isnxs, x24, x25               // encoding: [0x38,0x93,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE3OS, x24, x25
-// CHECK: tlbip vae3os, x24, x25                  // encoding: [0x38,0x81,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VAE3OSNXS, x24, x25
-// CHECK: tlbip vae3osnxs, x24, x25               // encoding: [0x38,0x91,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE3, x24, x25
-// CHECK: tlbip vale3, x24, x25                   // encoding: [0xb8,0x87,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE3NXS, x24, x25
-// CHECK: tlbip vale3nxs, x24, x25                // encoding: [0xb8,0x97,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE3IS, x24, x25
-// CHECK: tlbip vale3is, x24, x25                 // encoding: [0xb8,0x83,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE3ISNXS, x24, x25
-// CHECK: tlbip vale3isnxs, x24, x25              // encoding: [0xb8,0x93,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE3OS, x24, x25
-// CHECK: tlbip vale3os, x24, x25                 // encoding: [0xb8,0x81,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip VALE3OSNXS, x24, x25
-// CHECK: tlbip vale3osnxs, x24, x25              // encoding: [0xb8,0x91,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-
-          tlbip RVAE1, x18, x19
-// CHECK: tlbip rvae1, x18, x19                   // encoding: [0x32,0x86,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE1NXS, x18, x19
-// CHECK: tlbip rvae1nxs, x18, x19                // encoding: [0x32,0x96,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE1IS, x18, x19
-// CHECK: tlbip rvae1is, x18, x19                 // encoding: [0x32,0x82,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE1ISNXS, x18, x19
-// CHECK: tlbip rvae1isnxs, x18, x19              // encoding: [0x32,0x92,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE1OS, x18, x19
-// CHECK: tlbip rvae1os, x18, x19                 // encoding: [0x32,0x85,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE1OSNXS, x18, x19
-// CHECK: tlbip rvae1osnxs, x18, x19              // encoding: [0x32,0x95,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAAE1, x18, x19
-// CHECK: tlbip rvaae1, x18, x19                  // encoding: [0x72,0x86,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAAE1NXS, x18, x19
-// CHECK: tlbip rvaae1nxs, x18, x19               // encoding: [0x72,0x96,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAAE1IS, x18, x19
-// CHECK: tlbip rvaae1is, x18, x19                // encoding: [0x72,0x82,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAAE1ISNXS, x18, x19
-// CHECK: tlbip rvaae1isnxs, x18, x19             // encoding: [0x72,0x92,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAAE1OS, x18, x19
-// CHECK: tlbip rvaae1os, x18, x19                // encoding: [0x72,0x85,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAAE1OSNXS, x18, x19
-// CHECK: tlbip rvaae1osnxs, x18, x19             // encoding: [0x72,0x95,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE1, x18, x19
-// CHECK: tlbip rvale1, x18, x19                  // encoding: [0xb2,0x86,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE1NXS, x18, x19
-// CHECK: tlbip rvale1nxs, x18, x19               // encoding: [0xb2,0x96,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE1IS, x18, x19
-// CHECK: tlbip rvale1is, x18, x19                // encoding: [0xb2,0x82,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE1ISNXS, x18, x19
-// CHECK: tlbip rvale1isnxs, x18, x19             // encoding: [0xb2,0x92,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE1OS, x18, x19
-// CHECK: tlbip rvale1os, x18, x19                // encoding: [0xb2,0x85,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE1OSNXS, x18, x19
-// CHECK: tlbip rvale1osnxs, x18, x19             // encoding: [0xb2,0x95,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAALE1, x18, x19
-// CHECK: tlbip rvaale1, x18, x19                 // encoding: [0xf2,0x86,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAALE1NXS, x18, x19
-// CHECK: tlbip rvaale1nxs, x18, x19              // encoding: [0xf2,0x96,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAALE1IS, x18, x19
-// CHECK: tlbip rvaale1is, x18, x19               // encoding: [0xf2,0x82,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAALE1ISNXS, x18, x19
-// CHECK: tlbip rvaale1isnxs, x18, x19            // encoding: [0xf2,0x92,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAALE1OS, x18, x19
-// CHECK: tlbip rvaale1os, x18, x19               // encoding: [0xf2,0x85,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAALE1OSNXS, x18, x19
-// CHECK: tlbip rvaale1osnxs, x18, x19            // encoding: [0xf2,0x95,0x48,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          tlbip RVAE2, x28, x29
-// CHECK: tlbip rvae2, x28, x29                   // encoding: [0x3c,0x86,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE2NXS, x28, x29
-// CHECK: tlbip rvae2nxs, x28, x29                // encoding: [0x3c,0x96,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE2IS, x28, x29
-// CHECK: tlbip rvae2is, x28, x29                 // encoding: [0x3c,0x82,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE2ISNXS, x28, x29
-// CHECK: tlbip rvae2isnxs, x28, x29              // encoding: [0x3c,0x92,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE2OS, x28, x29
-// CHECK: tlbip rvae2os, x28, x29                 // encoding: [0x3c,0x85,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE2OSNXS, x28, x29
-// CHECK: tlbip rvae2osnxs, x28, x29              // encoding: [0x3c,0x95,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE2, x28, x29
-// CHECK: tlbip rvale2, x28, x29                  // encoding: [0xbc,0x86,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE2NXS, x28, x29
-// CHECK: tlbip rvale2nxs, x28, x29               // encoding: [0xbc,0x96,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE2IS, x28, x29
-// CHECK: tlbip rvale2is, x28, x29                // encoding: [0xbc,0x82,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE2ISNXS, x28, x29
-// CHECK: tlbip rvale2isnxs, x28, x29             // encoding: [0xbc,0x92,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE2OS, x28, x29
-// CHECK: tlbip rvale2os, x28, x29                // encoding: [0xbc,0x85,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE2OSNXS, x28, x29
-// CHECK: tlbip rvale2osnxs, x28, x29             // encoding: [0xbc,0x95,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          tlbip RVAE3, x10, x11
-// CHECK: tlbip rvae3, x10, x11                   // encoding: [0x2a,0x86,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE3NXS, x10, x11
-// CHECK: tlbip rvae3nxs, x10, x11                // encoding: [0x2a,0x96,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE3IS, x10, x11
-// CHECK: tlbip rvae3is, x10, x11                 // encoding: [0x2a,0x82,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE3ISNXS, x10, x11
-// CHECK: tlbip rvae3isnxs, x10, x11              // encoding: [0x2a,0x92,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE3OS, x10, x11
-// CHECK: tlbip rvae3os, x10, x11                 // encoding: [0x2a,0x85,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE3OSNXS, x10, x11
-// CHECK: tlbip rvae3osnxs, x10, x11              // encoding: [0x2a,0x95,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE3, x10, x11
-// CHECK: tlbip rvale3, x10, x11                  // encoding: [0xaa,0x86,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE3NXS, x10, x11
-// CHECK: tlbip rvale3nxs, x10, x11               // encoding: [0xaa,0x96,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE3IS, x10, x11
-// CHECK: tlbip rvale3is, x10, x11                // encoding: [0xaa,0x82,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE3ISNXS, x10, x11
-// CHECK: tlbip rvale3isnxs, x10, x11             // encoding: [0xaa,0x92,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE3OS, x10, x11
-// CHECK: tlbip rvale3os, x10, x11                // encoding: [0xaa,0x85,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVALE3OSNXS, x10, x11
-// CHECK: tlbip rvale3osnxs, x10, x11             // encoding: [0xaa,0x95,0x4e,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-
-          tlbip RIPAS2E1, x20, x21
-// CHECK: tlbip ripas2e1, x20, x21                // encoding: [0x54,0x84,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2E1NXS, x20, x21
-// CHECK: tlbip ripas2e1nxs, x20, x21             // encoding: [0x54,0x94,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2E1IS, x20, x21
-// CHECK: tlbip ripas2e1is, x20, x21              // encoding: [0x54,0x80,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2E1ISNXS, x20, x21
-// CHECK: tlbip ripas2e1isnxs, x20, x21           // encoding: [0x54,0x90,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2E1OS, x20, x21
-// CHECK: tlbip ripas2e1os, x20, x21              // encoding: [0x74,0x84,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2E1OSNXS, x20, x21
-// CHECK: tlbip ripas2e1osnxs, x20, x21           // encoding: [0x74,0x94,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2LE1, x20, x21
-// CHECK: tlbip ripas2le1, x20, x21               // encoding: [0xd4,0x84,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2LE1NXS, x20, x21
-// CHECK: tlbip ripas2le1nxs, x20, x21            // encoding: [0xd4,0x94,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2LE1IS, x20, x21
-// CHECK: tlbip ripas2le1is, x20, x21             // encoding: [0xd4,0x80,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2LE1ISNXS, x20, x21
-// CHECK: tlbip ripas2le1isnxs, x20, x21          // encoding: [0xd4,0x90,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2LE1OS, x20, x21
-// CHECK: tlbip ripas2le1os, x20, x21             // encoding: [0xf4,0x84,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2LE1OSNXS, x20, x21
-// CHECK: tlbip ripas2le1osnxs, x20, x21          // encoding: [0xf4,0x94,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-
-          tlbip RIPAS2LE1OS, xzr, xzr
-// CHECK: tlbip ripas2le1os, xzr, xzr             // encoding: [0xff,0x84,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RIPAS2LE1OSNXS, xzr, xzr
-// CHECK: tlbip ripas2le1osnxs, xzr, xzr          // encoding: [0xff,0x94,0x4c,0xd5]
-// ERROR-NO-D128: [[@LINE-2]]:11: error: instruction requires: d128
-          tlbip RVAE3IS
-// ERRORS: error: expected comma
-          tlbip RVAE3IS,
-// ERRORS: error: expected register identifier
-          tlbip VAE3,
-// ERRORS: error: expected register identifier
-          tlbip IPAS2E1, x4, x8
-// ERRORS: error: specified tlbip op requires a pair of registers
-          tlbip RVAE3, x11, x11
-// ERRORS: error: specified tlbip op requires a pair of registers
diff --git a/llvm/test/MC/AArch64/armv9.2a-mec.s b/llvm/test/MC/AArch64/armv9.2a-mec.s
index 42e4bf732086e..1998b43d336ec 100644
--- a/llvm/test/MC/AArch64/armv9.2a-mec.s
+++ b/llvm/test/MC/AArch64/armv9.2a-mec.s
@@ -1,55 +1,129 @@
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+mec < %s | FileCheck %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu < %s 2>&1 | FileCheck --check-prefix=CHECK-NO-MEC %s
-
-          mrs x0, MECIDR_EL2
-// CHECK: mrs   x0, MECIDR_EL2       // encoding: [0xe0,0xa8,0x3c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:19: error: expected readable system register
-          mrs x0, MECID_P0_EL2
-// CHECK: mrs   x0, MECID_P0_EL2      // encoding: [0x00,0xa8,0x3c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:19: error: expected readable system register
-          mrs x0, MECID_A0_EL2
-// CHECK: mrs   x0, MECID_A0_EL2      // encoding: [0x20,0xa8,0x3c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:19: error: expected readable system register
-          mrs x0, MECID_P1_EL2
-// CHECK: mrs   x0, MECID_P1_EL2      // encoding: [0x40,0xa8,0x3c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:19: error: expected readable system register
-          mrs x0, MECID_A1_EL2
-// CHECK: mrs   x0, MECID_A1_EL2      // encoding: [0x60,0xa8,0x3c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:19: error: expected readable system register
-          mrs x0, VMECID_P_EL2
-// CHECK: mrs   x0, VMECID_P_EL2     // encoding: [0x00,0xa9,0x3c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:19: error: expected readable system register
-          mrs x0, VMECID_A_EL2
-// CHECK: mrs   x0, VMECID_A_EL2     // encoding: [0x20,0xa9,0x3c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:19: error: expected readable system register
-          mrs x0, MECID_RL_A_EL3
-// CHECK: mrs   x0, MECID_RL_A_EL3   // encoding: [0x20,0xaa,0x3e,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:19: error: expected readable system register
-          msr MECID_P0_EL2,    x0
-// CHECK: msr   MECID_P0_EL2, x0      // encoding: [0x00,0xa8,0x1c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:15: error: expected writable system register or pstate
-          msr MECID_A0_EL2,    x0
-// CHECK: msr   MECID_A0_EL2, x0      // encoding: [0x20,0xa8,0x1c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:15: error: expected writable system register or pstate
-          msr MECID_P1_EL2,    x0
-// CHECK: msr   MECID_P1_EL2, x0      // encoding: [0x40,0xa8,0x1c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:15: error: expected writable system register or pstate
-          msr MECID_A1_EL2,    x0
-// CHECK: msr   MECID_A1_EL2, x0      // encoding: [0x60,0xa8,0x1c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:15: error: expected writable system register or pstate
-          msr VMECID_P_EL2,   x0
-// CHECK: msr   VMECID_P_EL2, x0     // encoding: [0x00,0xa9,0x1c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:15: error: expected writable system register or pstate
-          msr VMECID_A_EL2,   x0
-// CHECK: msr   VMECID_A_EL2, x0     // encoding: [0x20,0xa9,0x1c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:15: error: expected writable system register or pstate
-          msr MECID_RL_A_EL3, x0
-// CHECK: msr   MECID_RL_A_EL3, x0   // encoding: [0x20,0xaa,0x1e,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:15: error: expected writable system register or pstate
-
-          dc cigdpae, x0
-// CHECK: dc cigdpae, x0             // encoding: [0xe0,0x7e,0x0c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:14: error: DC CIGDPAE requires: mec
-          dc cipae, x0
-// CHECK: dc cipae, x0               // encoding: [0x00,0x7e,0x0c,0xd5]
-// CHECK-NO-MEC: [[@LINE-2]]:14: error: DC CIPAE requires: mec
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+mec < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+mec < %s \
+// RUN:        | llvm-objdump -d --mattr=+mec --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+mec < %s \
+// RUN:        | llvm-objdump -d --mattr=-mec --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+mec < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+mec -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple aarch64 -disassemble < %s 2>&1 | FileCheck --check-prefix=CHECK-NO-MEC %s
+
+
+mrs x0, MECIDR_EL2
+// CHECK-INST: mrs x0, MECIDR_EL2
+// CHECK-ENCODING: encoding: [0xe0,0xa8,0x3c,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53ca8e0 mrs x0, S3_4_C10_C8_7
+
+mrs x0, MECID_P0_EL2
+// CHECK-INST: mrs x0, MECID_P0_EL2
+// CHECK-ENCODING: encoding: [0x00,0xa8,0x3c,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53ca800 mrs x0, S3_4_C10_C8_0
+
+mrs x0, MECID_A0_EL2
+// CHECK-INST: mrs x0, MECID_A0_EL2
+// CHECK-ENCODING: encoding: [0x20,0xa8,0x3c,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53ca820 mrs x0, S3_4_C10_C8_1
+
+mrs x0, MECID_P1_EL2
+// CHECK-INST: mrs x0, MECID_P1_EL2
+// CHECK-ENCODING: encoding: [0x40,0xa8,0x3c,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53ca840 mrs x0, S3_4_C10_C8_2
+
+mrs x0, MECID_A1_EL2
+// CHECK-INST: mrs x0, MECID_A1_EL2
+// CHECK-ENCODING: encoding: [0x60,0xa8,0x3c,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53ca860 mrs x0, S3_4_C10_C8_3
+
+mrs x0, VMECID_P_EL2
+// CHECK-INST: mrs x0, VMECID_P_EL2
+// CHECK-ENCODING: encoding: [0x00,0xa9,0x3c,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53ca900 mrs x0, S3_4_C10_C9_0
+
+mrs x0, VMECID_A_EL2
+// CHECK-INST: mrs x0, VMECID_A_EL2
+// CHECK-ENCODING: encoding: [0x20,0xa9,0x3c,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53ca920 mrs x0, S3_4_C10_C9_1
+
+mrs x0, MECID_RL_A_EL3
+// CHECK-INST: mrs x0, MECID_RL_A_EL3
+// CHECK-ENCODING: encoding: [0x20,0xaa,0x3e,0xd5]
+// CHECK-ERROR: error: expected readable system register
+// CHECK-UNKNOWN:  d53eaa20 mrs x0, S3_6_C10_C10_1
+
+msr MECID_P0_EL2,    x0
+// CHECK-INST: msr MECID_P0_EL2, x0
+// CHECK-ENCODING: encoding: [0x00,0xa8,0x1c,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51ca800 msr S3_4_C10_C8_0, x0
+
+msr MECID_A0_EL2,    x0
+// CHECK-INST: msr MECID_A0_EL2, x0
+// CHECK-ENCODING: encoding: [0x20,0xa8,0x1c,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51ca820 msr S3_4_C10_C8_1, x0
+
+msr MECID_P1_EL2,    x0
+// CHECK-INST: msr MECID_P1_EL2, x0
+// CHECK-ENCODING: encoding: [0x40,0xa8,0x1c,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51ca840 msr S3_4_C10_C8_2, x0
+
+msr MECID_A1_EL2,    x0
+// CHECK-INST: msr MECID_A1_EL2, x0
+// CHECK-ENCODING: encoding: [0x60,0xa8,0x1c,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51ca860 msr S3_4_C10_C8_3, x0
+
+msr VMECID_P_EL2,   x0
+// CHECK-INST: msr VMECID_P_EL2, x0
+// CHECK-ENCODING: encoding: [0x00,0xa9,0x1c,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51ca900 msr S3_4_C10_C9_0, x0
+
+msr VMECID_A_EL2,   x0
+// CHECK-INST: msr VMECID_A_EL2, x0
+// CHECK-ENCODING: encoding: [0x20,0xa9,0x1c,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51ca920 msr S3_4_C10_C9_1, x0
+
+msr MECID_RL_A_EL3, x0
+// CHECK-INST: msr MECID_RL_A_EL3, x0
+// CHECK-ENCODING: encoding: [0x20,0xaa,0x1e,0xd5]
+// CHECK-ERROR: error: expected writable system register or pstate
+// CHECK-UNKNOWN:  d51eaa20 msr S3_6_C10_C10_1, x0
+
+dc cigdpae, x0
+// CHECK-INST: dc cigdpae, x0
+// CHECK-ENCODING: encoding: [0xe0,0x7e,0x0c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:4: error: DC CIGDPAE requires: mec
+// CHECK-UNKNOWN:  d50c7ee0 sys #4, c7, c14, #7, x0
+// CHECK-NO-MEC: sys #4, c7, c14, #7, x0
+
+dc cipae, x0
+// CHECK-INST: dc cipae, x0
+// CHECK-ENCODING: encoding: [0x00,0x7e,0x0c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:4: error: DC CIPAE requires: mec
+// CHECK-UNKNOWN:  d50c7e00 sys #4, c7, c14, #0, x0
+// CHECK-NO-MEC: sys #4, c7, c14, #0, x0
+
+sys #4, c7, c14, #7, x0
+// CHECK-INST: dc cigdpae, x0
+// CHECK-ENCODING: encoding: [0xe0,0x7e,0x0c,0xd5]
+// CHECK-UNKNOWN:  d50c7ee0 sys #4, c7, c14, #7, x0
+
+sys #4, c7, c14, #0, x0
+// CHECK-INST: dc cipae, x0
+// CHECK-ENCODING: encoding: [0x00,0x7e,0x0c,0xd5]
+// CHECK-UNKNOWN:  d50c7e00 sys #4, c7, c14, #0, x0
diff --git a/llvm/test/MC/AArch64/armv9.4-lse128.s b/llvm/test/MC/AArch64/armv9.4-lse128.s
deleted file mode 100644
index a639278ec8263..0000000000000
--- a/llvm/test/MC/AArch64/armv9.4-lse128.s
+++ /dev/null
@@ -1,98 +0,0 @@
-// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr +lse128 %s 2>%t | FileCheck %s
-// RUN: FileCheck %s --input-file=%t --check-prefix=ERROR-INVALID-OP
-// RUN: not llvm-mc -triple aarch64 -show-encoding %s 2>&1 | FileCheck --check-prefix=ERROR-NO-LSE128 %s
-
-ldclrp   x1, x2, [x11]
-// CHECK: ldclrp x1, x2, [x11]                   // encoding: [0x61,0x11,0x22,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldclrp   x21, x22, [sp]
-// CHECK: ldclrp x21, x22, [sp]                  // encoding: [0xf5,0x13,0x36,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldclrpa  x1, x2, [x11]
-// CHECK: ldclrpa x1, x2, [x11]                   // encoding: [0x61,0x11,0xa2,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldclrpa  x21, x22, [sp]
-// CHECK: ldclrpa x21, x22, [sp]                  // encoding: [0xf5,0x13,0xb6,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldclrpal x1, x2, [x11]
-// CHECK: ldclrpal x1, x2, [x11]                   // encoding: [0x61,0x11,0xe2,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldclrpal x21, x22, [sp]
-// CHECK: ldclrpal x21, x22, [sp]                  // encoding: [0xf5,0x13,0xf6,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldclrpl  x1, x2, [x11]
-// CHECK: ldclrpl x1, x2, [x11]                   // encoding: [0x61,0x11,0x62,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldclrpl  x21, x22, [sp]
-// CHECK: ldclrpl x21, x22, [sp]                  // encoding: [0xf5,0x13,0x76,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldclrpl  x22, xzr, [sp]
-// ERROR-INVALID-OP: [[@LINE-1]]:15: error: invalid operand for instruction
-// ERROR-NO-LSE128: error: invalid operand for instruction
-ldclrpl  xzr, x22, [sp]
-// ERROR-INVALID-OP: [[@LINE-1]]:10: error: invalid operand for instruction
-// ERROR-NO-LSE128: error: invalid operand for instruction
-
-ldsetp   x1, x2, [x11]
-// CHECK: ldsetp x1, x2, [x11]                   // encoding: [0x61,0x31,0x22,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldsetp   x21, x22, [sp]
-// CHECK: ldsetp x21, x22, [sp]                  // encoding: [0xf5,0x33,0x36,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldsetpa  x1, x2, [x11]
-// CHECK: ldsetpa x1, x2, [x11]                   // encoding: [0x61,0x31,0xa2,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldsetpa  x21, x22, [sp]
-// CHECK: ldsetpa x21, x22, [sp]                  // encoding: [0xf5,0x33,0xb6,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldsetpal x1, x2, [x11]
-// CHECK: ldsetpal x1, x2, [x11]                   // encoding: [0x61,0x31,0xe2,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldsetpal x21, x22, [sp]
-// CHECK: ldsetpal x21, x22, [sp]                  // encoding: [0xf5,0x33,0xf6,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldsetpl  x1, x2, [x11]
-// CHECK: ldsetpl x1, x2, [x11]                   // encoding: [0x61,0x31,0x62,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldsetpl  x21, x22, [sp]
-// CHECK: ldsetpl x21, x22, [sp]                  // encoding: [0xf5,0x33,0x76,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-ldsetpl  x22, xzr, [sp]
-// ERROR-INVALID-OP: [[@LINE-1]]:15: error: invalid operand for instruction
-// ERROR-NO-LSE128: error: invalid operand for instruction
-ldsetpl  xzr, x22, [sp]
-// ERROR-INVALID-OP: [[@LINE-1]]:10: error: invalid operand for instruction
-// ERROR-NO-LSE128: error: invalid operand for instruction
-
-
-swpp     x1, x2, [x11]
-// CHECK: swpp x1, x2, [x11]                   // encoding: [0x61,0x81,0x22,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-swpp     x21, x22, [sp]
-// CHECK: swpp x21, x22, [sp]                  // encoding: [0xf5,0x83,0x36,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-swppa    x1, x2, [x11]
-// CHECK: swppa x1, x2, [x11]                   // encoding: [0x61,0x81,0xa2,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-swppa    x21, x22, [sp]
-// CHECK: swppa x21, x22, [sp]                  // encoding: [0xf5,0x83,0xb6,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-swppal   x1, x2, [x11]
-// CHECK: swppal x1, x2, [x11]                   // encoding: [0x61,0x81,0xe2,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-swppal   x21, x22, [sp]
-// CHECK: swppal x21, x22, [sp]                  // encoding: [0xf5,0x83,0xf6,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-swppl    x1, x2, [x11]
-// CHECK: swppl x1, x2, [x11]                   // encoding: [0x61,0x81,0x62,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-swppl    x21, x22, [sp]
-// CHECK: swppl x21, x22, [sp]                  // encoding: [0xf5,0x83,0x76,0x19]
-// ERROR-NO-LSE128: [[@LINE-2]]:1: error: instruction requires: lse128
-swppl    x22, xzr, [sp]
-// ERROR-INVALID-OP: [[@LINE-1]]:15: error: invalid operand for instruction
-// ERROR-NO-LSE128: error: invalid operand for instruction
-swppl    xzr, x22, [sp]
-// ERROR-INVALID-OP: [[@LINE-1]]:10: error: invalid operand for instruction
-// ERROR-NO-LSE128: error: invalid operand for instruction
-
diff --git a/llvm/test/MC/AArch64/armv9.4a-chk.s b/llvm/test/MC/AArch64/armv9.4a-chk.s
index 95acee370dcb1..14b0c375a7bce 100644
--- a/llvm/test/MC/AArch64/armv9.4a-chk.s
+++ b/llvm/test/MC/AArch64/armv9.4a-chk.s
@@ -1,21 +1,38 @@
-// RUN: llvm-mc -triple aarch64 -mattr=+chk -show-encoding %s | FileCheck %s
-// RUN: llvm-mc -triple aarch64 -mattr=+v8.9a -show-encoding %s | FileCheck %s
-// RUN: llvm-mc -triple aarch64 -mattr=+v9.4a -show-encoding %s | FileCheck %s
-// RUN: llvm-mc -triple aarch64 -mattr=+v8a -show-encoding %s | FileCheck %s --check-prefix=NO-CHK
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v8.9a < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+v9.4a < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+chk < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+chk < %s \
+// RUN:        | llvm-objdump -d --mattr=+chk - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+chk < %s \
+// RUN:   | llvm-objdump -d --mattr=-chk - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+chk < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+chk -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
 
 // FEAT_CHK is mandatory from v8.0-a, but a clang user may not be using the LLVM
 // integrated assembler, so we cannot just print `chkfeat x16` in all
 // circumstances. Thankfully, we can always print `hint #40` when we cannot
 // print `chkfeat x16`.
-//
 // So, in this case, we only print `chkfeat x16` from v8.9-a onwards, as an
 // assembler that understands v8.9-a will understand `chkfeat x16`, and those
 // that understand previous versions may not.
 
 chkfeat x16
-// CHECK: chkfeat x16                       // encoding: [0x1f,0x25,0x03,0xd5]
-// NO-CHK: hint #40                              // encoding: [0x1f,0x25,0x03,0xd5]
+// CHECK-INST: chkfeat x16
+// CHECK-ENCODING: encoding: [0x1f,0x25,0x03,0xd5]
+// CHECK-ERROR: hint #40
+// CHECK-UNKNOWN:  d503251f      hint #40
 
 hint #40
-// CHECK: chkfeat x16                      // encoding: [0x1f,0x25,0x03,0xd5]
-// NO-CHK: hint #40                             // encoding: [0x1f,0x25,0x03,0xd5]
+// CHECK-INST: chkfeat x16
+// CHECK-ENCODING: encoding: [0x1f,0x25,0x03,0xd5]
+// CHECK-ERROR: hint #40
+// CHECK-UNKNOWN:  d503251f      hint #40
diff --git a/llvm/test/MC/AArch64/armv9.4a-ebep.s b/llvm/test/MC/AArch64/armv9.4a-ebep.s
index 7e9f1115d9753..2c7c7147333c0 100644
--- a/llvm/test/MC/AArch64/armv9.4a-ebep.s
+++ b/llvm/test/MC/AArch64/armv9.4a-ebep.s
@@ -1,9 +1,41 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
 
 mrs x2, PM
+// CHECK-INST: mrs x2, PM
+// CHECK-ENCODING: encoding: [0x22,0x43,0x38,0xd5]
+// CHECK-UNKNOWN:  d5384322 mrs x2, PM
+
+mrs x3, PM
+// CHECK-INST: mrs x3, PM
+// CHECK-ENCODING: encoding: [0x23,0x43,0x38,0xd5]
+// CHECK-UNKNOWN:  d5384323 mrs x3, PM
+
 msr PM, x3
+// CHECK-INST: msr PM, x3
+// CHECK-ENCODING: encoding: [0x23,0x43,0x18,0xd5]
+// CHECK-UNKNOWN:  d5184323 msr PM, x3
+
+msr PM, x6
+// CHECK-INST: msr PM, x6
+// CHECK-ENCODING: encoding: [0x26,0x43,0x18,0xd5]
+// CHECK-UNKNOWN:  d5184326 msr PM, x6
+
+msr PM, #0
+// CHECK-INST: msr PM, #0
+// CHECK-ENCODING: encoding: [0x1f,0x42,0x01,0xd5]
+// CHECK-UNKNOWN:  d501421f msr PM, #0
+
 msr PM, #1
+// CHECK-INST: msr PM, #1
+// CHECK-ENCODING: encoding: [0x1f,0x43,0x01,0xd5]
+// CHECK-UNKNOWN:  d501431f msr PM, #1
 
-// CHECK:       mrs x2, {{pm|PM}} // encoding: [0x22,0x43,0x38,0xd5]
-// CHECK:       msr {{pm|PM}}, x3 // encoding: [0x23,0x43,0x18,0xd5]
-// CHECK:       msr {{pm|PM}}, #1 // encoding: [0x1f,0x43,0x01,0xd5]
diff --git a/llvm/test/MC/AArch64/armv9.4a-gcs.s b/llvm/test/MC/AArch64/armv9.4a-gcs.s
index b4af9b5dcb10c..f702c947543ac 100644
--- a/llvm/test/MC/AArch64/armv9.4a-gcs.s
+++ b/llvm/test/MC/AArch64/armv9.4a-gcs.s
@@ -1,115 +1,204 @@
-// RUN: llvm-mc -triple aarch64 -mattr +gcs -show-encoding %s | FileCheck %s
-// RUN: not llvm-mc -triple aarch64 -show-encoding %s 2>%t | FileCheck %s --check-prefix=NO-GCS
-// RUN: FileCheck --check-prefix=ERROR-NO-GCS %s < %t
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+gcs < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+gcs < %s \
+// RUN:        | llvm-objdump -d --mattr=+gcs --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+gcs < %s \
+// RUN:        | llvm-objdump -d --mattr=-gcs --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+gcs < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+gcs -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
 
 msr GCSCR_EL1, x0
+// CHECK-INST: msr GCSCR_EL1, x0
+// CHECK-ENCODING: encoding: [0x00,0x25,0x18,0xd5]
+// CHECK-UNKNOWN:  d5182500 msr GCSCR_EL1, x0
+
 mrs x1, GCSCR_EL1
-// CHECK: msr     GCSCR_EL1, x0                   // encoding: [0x00,0x25,0x18,0xd5]
-// CHECK: mrs     x1, GCSCR_EL1                   // encoding: [0x01,0x25,0x38,0xd5]
+// CHECK-INST: mrs x1, GCSCR_EL1
+// CHECK-ENCODING: encoding: [0x01,0x25,0x38,0xd5]
+// CHECK-UNKNOWN:  d5382501 mrs x1, GCSCR_EL1
 
 msr GCSPR_EL1, x2
+// CHECK-INST: msr GCSPR_EL1, x2
+// CHECK-ENCODING: encoding: [0x22,0x25,0x18,0xd5]
+// CHECK-UNKNOWN:  d5182522 msr GCSPR_EL1, x2
+
 mrs x3, GCSPR_EL1
-// CHECK: msr     GCSPR_EL1, x2                   // encoding: [0x22,0x25,0x18,0xd5]
-// CHECK: mrs     x3, GCSPR_EL1                   // encoding: [0x23,0x25,0x38,0xd5]
+// CHECK-INST: mrs x3, GCSPR_EL1
+// CHECK-ENCODING: encoding: [0x23,0x25,0x38,0xd5]
+// CHECK-UNKNOWN:  d5382523 mrs x3, GCSPR_EL1
 
 msr GCSCRE0_EL1, x4
+// CHECK-INST: msr GCSCRE0_EL1, x4
+// CHECK-ENCODING: encoding: [0x44,0x25,0x18,0xd5]
+// CHECK-UNKNOWN:  d5182544 msr GCSCRE0_EL1, x4
+
 mrs x5, GCSCRE0_EL1
-// CHECK: msr     GCSCRE0_EL1, x4                 // encoding: [0x44,0x25,0x18,0xd5]
-// CHECK: mrs     x5, GCSCRE0_EL1                 // encoding: [0x45,0x25,0x38,0xd5]
+// CHECK-INST: mrs x5, GCSCRE0_EL1
+// CHECK-ENCODING: encoding: [0x45,0x25,0x38,0xd5]
+// CHECK-UNKNOWN:  d5382545 mrs x5, GCSCRE0_EL1
 
 msr GCSPR_EL0, x6
+// CHECK-INST: msr GCSPR_EL0, x6
+// CHECK-ENCODING: encoding: [0x26,0x25,0x1b,0xd5]
+// CHECK-UNKNOWN:  d51b2526 msr GCSPR_EL0, x6
+
 mrs x7, GCSPR_EL0
-// CHECK: msr     GCSPR_EL0, x6                   // encoding: [0x26,0x25,0x1b,0xd5]
-// CHECK: mrs     x7, GCSPR_EL0                   // encoding: [0x27,0x25,0x3b,0xd5]
+// CHECK-INST: mrs x7, GCSPR_EL0
+// CHECK-ENCODING: encoding: [0x27,0x25,0x3b,0xd5]
+// CHECK-UNKNOWN:  d53b2527 mrs x7, GCSPR_EL0
 
 msr GCSCR_EL2, x10
+// CHECK-INST: msr GCSCR_EL2, x10
+// CHECK-ENCODING: encoding: [0x0a,0x25,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c250a msr GCSCR_EL2, x10
+
 mrs x11, GCSCR_EL2
-// CHECK: msr     GCSCR_EL2, x10                  // encoding: [0x0a,0x25,0x1c,0xd5]
-// CHECK: mrs     x11, GCSCR_EL2                  // encoding: [0x0b,0x25,0x3c,0xd5]
+// CHECK-INST: mrs x11, GCSCR_EL2
+// CHECK-ENCODING: encoding: [0x0b,0x25,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c250b mrs x11, GCSCR_EL2
 
 msr GCSPR_EL2, x12
+// CHECK-INST: msr GCSPR_EL2, x12
+// CHECK-ENCODING: encoding: [0x2c,0x25,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c252c msr GCSPR_EL2, x12
+
 mrs x13, GCSPR_EL2
-// CHECK: msr     GCSPR_EL2, x12                  // encoding: [0x2c,0x25,0x1c,0xd5]
-// CHECK: mrs     x13, GCSPR_EL2                  // encoding: [0x2d,0x25,0x3c,0xd5]
+// CHECK-INST: mrs x13, GCSPR_EL2
+// CHECK-ENCODING: encoding: [0x2d,0x25,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c252d mrs x13, GCSPR_EL2
 
 msr GCSCR_EL12, x14
+// CHECK-INST: msr GCSCR_EL12, x14
+// CHECK-ENCODING: encoding: [0x0e,0x25,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d250e msr GCSCR_EL12, x14
+
 mrs x15, GCSCR_EL12
-// CHECK: msr     GCSCR_EL12, x14                 // encoding: [0x0e,0x25,0x1d,0xd5]
-// CHECK: mrs     x15, GCSCR_EL12                 // encoding: [0x0f,0x25,0x3d,0xd5]
+// CHECK-INST: mrs x15, GCSCR_EL12
+// CHECK-ENCODING: encoding: [0x0f,0x25,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d250f mrs x15, GCSCR_EL12
 
 msr GCSPR_EL12, x16
+// CHECK-INST: msr GCSPR_EL12, x16
+// CHECK-ENCODING: encoding: [0x30,0x25,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d2530 msr GCSPR_EL12, x16
+
 mrs x17, GCSPR_EL12
-// CHECK: msr     GCSPR_EL12, x16                 // encoding: [0x30,0x25,0x1d,0xd5]
-// CHECK: mrs     x17, GCSPR_EL12                 // encoding: [0x31,0x25,0x3d,0xd5]
+// CHECK-INST: mrs x17, GCSPR_EL12
+// CHECK-ENCODING: encoding: [0x31,0x25,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d2531 mrs x17, GCSPR_EL12
 
 msr GCSCR_EL3, x18
+// CHECK-INST: msr GCSCR_EL3, x18
+// CHECK-ENCODING: encoding: [0x12,0x25,0x1e,0xd5]
+// CHECK-UNKNOWN:  d51e2512 msr GCSCR_EL3, x18
+
 mrs x19, GCSCR_EL3
-// CHECK: msr     GCSCR_EL3, x18                  // encoding: [0x12,0x25,0x1e,0xd5]
-// CHECK: mrs     x19, GCSCR_EL3                  // encoding: [0x13,0x25,0x3e,0xd5]
+// CHECK-INST: mrs x19, GCSCR_EL3
+// CHECK-ENCODING: encoding: [0x13,0x25,0x3e,0xd5]
+// CHECK-UNKNOWN:  d53e2513 mrs x19, GCSCR_EL3
 
 msr GCSPR_EL3, x20
+// CHECK-INST: msr GCSPR_EL3, x20
+// CHECK-ENCODING: encoding: [0x34,0x25,0x1e,0xd5]
+// CHECK-UNKNOWN:  d51e2534 msr GCSPR_EL3, x20
+
 mrs x21, GCSPR_EL3
-// CHECK: msr     GCSPR_EL3, x20                  // encoding: [0x34,0x25,0x1e,0xd5]
-// CHECK: mrs     x21, GCSPR_EL3                  // encoding: [0x35,0x25,0x3e,0xd5]
+// CHECK-INST: mrs x21, GCSPR_EL3
+// CHECK-ENCODING: encoding: [0x35,0x25,0x3e,0xd5]
+// CHECK-UNKNOWN:  d53e2535 mrs x21, GCSPR_EL3
 
 gcsss1 x21
-// CHECK: gcsss1  x21                        // encoding: [0x55,0x77,0x0b,0xd5]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcsss1 x21
+// CHECK-ENCODING: encoding: [0x55,0x77,0x0b,0xd5]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d50b7755 sys #3, c7, c7, #2, x21
 
 gcsss2 x22
-// CHECK: gcsss2  x22                        // encoding: [0x76,0x77,0x2b,0xd5]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcsss2 x22
+// CHECK-ENCODING: encoding: [0x76,0x77,0x2b,0xd5]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d52b7776 sysl x22, #3, c7, c7, #3
 
 gcspushm x25
-// CHECK: gcspushm x25                       // encoding: [0x19,0x77,0x0b,0xd5]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcspushm x25
+// CHECK-ENCODING: encoding: [0x19,0x77,0x0b,0xd5]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d50b7719 sys #3, c7, c7, #0, x25
 
 gcspopm
-// CHECK: gcspopm                             // encoding: [0x3f,0x77,0x2b,0xd5]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcspopm
+// CHECK-ENCODING: encoding: [0x3f,0x77,0x2b,0xd5]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d52b773f sysl xzr, #3, c7, c7, #1
 
 gcspopm xzr
-// CHECK: gcspopm                            // encoding: [0x3f,0x77,0x2b,0xd5]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcspopm
+// CHECK-ENCODING: encoding: [0x3f,0x77,0x2b,0xd5]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d52b773f sysl xzr, #3, c7, c7, #1
 
 gcspopm x25
-// CHECK: gcspopm  x25                        // encoding: [0x39,0x77,0x2b,0xd5]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
-
-gcsb dsync
-// CHECK: gcsb    dsync                           // encoding: [0x7f,0x22,0x03,0xd5]
-// ERROR-NO-GCS-NOT: [[@LINE-2]]:1: error: instruction requires: gcs
-// NO-GCS: hint #19                              // encoding: [0x7f,0x22,0x03,0xd5]
-
-hint #19
-// CHECK: gcsb    dsync                           // encoding: [0x7f,0x22,0x03,0xd5]
-// ERROR-NO-GCS-NOT: [[@LINE-2]]:1: error: instruction requires: gcs
-// NO-GCS: hint #19                              // encoding: [0x7f,0x22,0x03,0xd5]
+// CHECK-INST: gcspopm x25
+// CHECK-ENCODING: encoding: [0x39,0x77,0x2b,0xd5]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d52b7739 sysl x25, #3, c7, c7, #1
 
 gcsstr x26, [x27]
-// CHECK: gcsstr x26, [x27]                        // encoding: [0x7a,0x0f,0x1f,0xd9]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcsstr x26, [x27]
+// CHECK-ENCODING: encoding: [0x7a,0x0f,0x1f,0xd9]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d91f0f7a <unknown>
 
 gcsstr x26, [sp]
-// CHECK: gcsstr x26, [sp]                         // encoding: [0xfa,0x0f,0x1f,0xd9]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcsstr x26, [sp]
+// CHECK-ENCODING: encoding: [0xfa,0x0f,0x1f,0xd9]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d91f0ffa <unknown>
 
 gcssttr x26, [x27]
-// CHECK: gcssttr x26, [x27]                       // encoding: [0x7a,0x1f,0x1f,0xd9]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcssttr x26, [x27]
+// CHECK-ENCODING: encoding: [0x7a,0x1f,0x1f,0xd9]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d91f1f7a <unknown>
 
 gcssttr x26, [sp]
-// CHECK: gcssttr x26, [sp]                        // encoding: [0xfa,0x1f,0x1f,0xd9]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcssttr x26, [sp]
+// CHECK-ENCODING: encoding: [0xfa,0x1f,0x1f,0xd9]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d91f1ffa <unknown>
 
 gcspushx
-// CHECK: gcspushx                          // encoding: [0x9f,0x77,0x08,0xd5]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcspushx
+// CHECK-ENCODING: encoding: [0x9f,0x77,0x08,0xd5]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d508779f sys #0, c7, c7, #4
 
 gcspopcx
-// CHECK: gcspopcx                          // encoding: [0xbf,0x77,0x08,0xd5]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcspopcx
+// CHECK-ENCODING: encoding: [0xbf,0x77,0x08,0xd5]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d50877bf sys #0, c7, c7, #5
 
 gcspopx
-// CHECK: gcspopx                           // encoding: [0xdf,0x77,0x08,0xd5]
-// ERROR-NO-GCS: [[@LINE-2]]:1: error: instruction requires: gcs
+// CHECK-INST: gcspopx
+// CHECK-ENCODING: encoding: [0xdf,0x77,0x08,0xd5]
+// CHECK-ERROR: error: instruction requires: gcs
+// CHECK-UNKNOWN:  d50877df sys #0, c7, c7, #6
+
+gcsb dsync
+// CHECK-INST: gcsb dsync
+// CHECK-ENCODING: encoding: [0x7f,0x22,0x03,0xd5]
+// CHECK-UNKNOWN:  d503227f hint #19
+// CHECK-ERROR: hint #19                              // encoding: [0x7f,0x22,0x03,0xd5]
+
+hint #19
+// CHECK-INST: gcsb dsync
+// CHECK-ENCODING: encoding: [0x7f,0x22,0x03,0xd5]
+// CHECK-UNKNOWN:  d503227f hint #19
+// CHECK-ERROR: hint #19                              // encoding: [0x7f,0x22,0x03,0xd5]
diff --git a/llvm/test/MC/AArch64/armv9.4a-lse128-diagnostics.s b/llvm/test/MC/AArch64/armv9.4a-lse128-diagnostics.s
new file mode 100644
index 0000000000000..059b18f1e9153
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9.4a-lse128-diagnostics.s
@@ -0,0 +1,20 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+
+ldclrpl  x22, xzr, [sp]
+// CHECK-ERROR: error: invalid operand for instruction
+
+ldclrpl  xzr, x22, [sp]
+// CHECK-ERROR: error: invalid operand for instruction
+
+ldsetpl  x22, xzr, [sp]
+// CHECK-ERROR: error: invalid operand for instruction
+
+ldsetpl  xzr, x22, [sp]
+// CHECK-ERROR: error: invalid operand for instruction
+
+swppl    x22, xzr, [sp]
+// CHECK-ERROR: error: invalid operand for instruction
+
+swppl    xzr, x22, [sp]
+// CHECK-ERROR: error: invalid operand for instruction
diff --git a/llvm/test/MC/AArch64/armv9.4a-lse128.s b/llvm/test/MC/AArch64/armv9.4a-lse128.s
new file mode 100644
index 0000000000000..25dcb04c2ae63
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9.4a-lse128.s
@@ -0,0 +1,159 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+lse128 < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+lse128 < %s \
+// RUN:        | llvm-objdump -d --mattr=+lse128 - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+lse128 < %s \
+// RUN:        | llvm-objdump -d --mattr=-lse128 - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+lse128 < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+lse128 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+
+ldclrp   x1, x2, [x11]
+// CHECK-INST: ldclrp x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x11,0x22,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19221161 <unknown>
+
+ldclrp   x21, x22, [sp]
+// CHECK-INST: ldclrp x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x13,0x36,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  193613f5 <unknown>
+
+ldclrpa  x1, x2, [x11]
+// CHECK-INST: ldclrpa x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x11,0xa2,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19a21161 <unknown>
+
+ldclrpa  x21, x22, [sp]
+// CHECK-INST: ldclrpa x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x13,0xb6,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19b613f5 <unknown>
+
+ldclrpal x1, x2, [x11]
+// CHECK-INST: ldclrpal x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x11,0xe2,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19e21161 <unknown>
+
+ldclrpal x21, x22, [sp]
+// CHECK-INST: ldclrpal x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x13,0xf6,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19f613f5 <unknown>
+
+ldclrpl  x1, x2, [x11]
+// CHECK-INST: ldclrpl x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x11,0x62,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19621161 <unknown>
+
+ldclrpl  x21, x22, [sp]
+// CHECK-INST: ldclrpl x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x13,0x76,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  197613f5 <unknown>
+
+ldsetp   x1, x2, [x11]
+// CHECK-INST: ldsetp x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x31,0x22,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19223161 <unknown>
+
+ldsetp   x21, x22, [sp]
+// CHECK-INST: ldsetp x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x33,0x36,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  193633f5 <unknown>
+
+ldsetpa  x1, x2, [x11]
+// CHECK-INST: ldsetpa x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x31,0xa2,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19a23161 <unknown>
+
+ldsetpa  x21, x22, [sp]
+// CHECK-INST: ldsetpa x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x33,0xb6,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19b633f5 <unknown>
+
+ldsetpal x1, x2, [x11]
+// CHECK-INST: ldsetpal x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x31,0xe2,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19e23161 <unknown>
+
+ldsetpal x21, x22, [sp]
+// CHECK-INST: ldsetpal x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x33,0xf6,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19f633f5 <unknown>
+
+ldsetpl  x1, x2, [x11]
+// CHECK-INST: ldsetpl x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x31,0x62,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19623161 <unknown>
+
+ldsetpl  x21, x22, [sp]
+// CHECK-INST: ldsetpl x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x33,0x76,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  197633f5 <unknown>
+
+swpp     x1, x2, [x11]
+// CHECK-INST: swpp x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x81,0x22,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19228161 <unknown>
+
+swpp     x21, x22, [sp]
+// CHECK-INST: swpp x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x83,0x36,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  193683f5 <unknown>
+
+swppa    x1, x2, [x11]
+// CHECK-INST: swppa x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x81,0xa2,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19a28161 <unknown>
+
+swppa    x21, x22, [sp]
+// CHECK-INST: swppa x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x83,0xb6,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19b683f5 <unknown>
+
+swppal   x1, x2, [x11]
+// CHECK-INST: swppal x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x81,0xe2,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19e28161 <unknown>
+
+swppal   x21, x22, [sp]
+// CHECK-INST: swppal x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x83,0xf6,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19f683f5 <unknown>
+
+swppl    x1, x2, [x11]
+// CHECK-INST: swppl x1, x2, [x11]
+// CHECK-ENCODING: encoding: [0x61,0x81,0x62,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  19628161 <unknown>
+
+swppl    x21, x22, [sp]
+// CHECK-INST: swppl x21, x22, [sp]
+// CHECK-ENCODING: encoding: [0xf5,0x83,0x76,0x19]
+// CHECK-ERROR: :[[@LINE-3]]:1: error: instruction requires: lse128
+// CHECK-UNKNOWN:  197683f5 <unknown>
diff --git a/llvm/test/MC/AArch64/armv9.5a-cpa.s b/llvm/test/MC/AArch64/armv9.5a-cpa.s
index 1c338eccf6cac..d239224b502fe 100644
--- a/llvm/test/MC/AArch64/armv9.5a-cpa.s
+++ b/llvm/test/MC/AArch64/armv9.5a-cpa.s
@@ -1,50 +1,87 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding -mattr=+cpa < %s | FileCheck %s
-// RUN: not llvm-mc -triple aarch64 < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-CPA %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+cpa < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+cpa < %s \
+// RUN:        | llvm-objdump -d --mattr=+cpa - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+cpa < %s \
+// RUN:        | llvm-objdump -d --mattr=-cpa - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+cpa < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+cpa -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 addpt x0, x1, x2
-// CHECK: addpt x0, x1, x2               // encoding: [0x20,0x20,0x02,0x9a]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: addpt x0, x1, x2
+// CHECK-ENCODING: encoding: [0x20,0x20,0x02,0x9a]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  9a022020 <unknown>
 
 addpt sp, sp, x2
-// CHECK: addpt sp, sp, x2               // encoding: [0xff,0x23,0x02,0x9a]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: addpt sp, sp, x2
+// CHECK-ENCODING: encoding: [0xff,0x23,0x02,0x9a]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  9a0223ff <unknown>
 
 addpt x0, x1, x2, lsl #0
-// CHECK: addpt x0, x1, x2               // encoding: [0x20,0x20,0x02,0x9a]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: addpt x0, x1, x2
+// CHECK-ENCODING: encoding: [0x20,0x20,0x02,0x9a]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  9a022020 <unknown>
 
 addpt x0, x1, x2, lsl #7
-// CHECK: addpt x0, x1, x2, lsl #7       // encoding: [0x20,0x3c,0x02,0x9a]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: addpt x0, x1, x2, lsl #7
+// CHECK-ENCODING: encoding: [0x20,0x3c,0x02,0x9a]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  9a023c20 <unknown>
 
 addpt sp, sp, x2, lsl #7
-// CHECK: addpt sp, sp, x2, lsl #7       // encoding: [0xff,0x3f,0x02,0x9a]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: addpt sp, sp, x2, lsl #7
+// CHECK-ENCODING: encoding: [0xff,0x3f,0x02,0x9a]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  9a023fff <unknown>
 
 subpt x0, x1, x2
-// CHECK: subpt x0, x1, x2               // encoding: [0x20,0x20,0x02,0xda]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: subpt x0, x1, x2
+// CHECK-ENCODING: encoding: [0x20,0x20,0x02,0xda]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  da022020 <unknown>
 
 subpt sp, sp, x2
-// CHECK: subpt sp, sp, x2               // encoding: [0xff,0x23,0x02,0xda]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: subpt sp, sp, x2
+// CHECK-ENCODING: encoding: [0xff,0x23,0x02,0xda]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  da0223ff <unknown>
 
 subpt x0, x1, x2, lsl #0
-// CHECK: subpt x0, x1, x2               // encoding: [0x20,0x20,0x02,0xda]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: subpt x0, x1, x2
+// CHECK-ENCODING: encoding: [0x20,0x20,0x02,0xda]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  da022020 <unknown>
 
 subpt x0, x1, x2, lsl #7
-// CHECK: subpt x0, x1, x2, lsl #7       // encoding: [0x20,0x3c,0x02,0xda]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: subpt x0, x1, x2, lsl #7
+// CHECK-ENCODING: encoding: [0x20,0x3c,0x02,0xda]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  da023c20 <unknown>
 
 subpt sp, sp, x2, lsl #7
-// CHECK: subpt sp, sp, x2, lsl #7       // encoding: [0xff,0x3f,0x02,0xda]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: subpt sp, sp, x2, lsl #7
+// CHECK-ENCODING: encoding: [0xff,0x3f,0x02,0xda]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  da023fff <unknown>
 
 maddpt x0, x1, x2, x3
-// CHECK: maddpt x0, x1, x2, x3          // encoding: [0x20,0x0c,0x62,0x9b]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: maddpt x0, x1, x2, x3
+// CHECK-ENCODING: encoding: [0x20,0x0c,0x62,0x9b]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  9b620c20 <unknown>
 
 msubpt x0, x1, x2, x3
-// CHECK: msubpt x0, x1, x2, x3          // encoding: [0x20,0x8c,0x62,0x9b]
-// ERROR-NO-CPA: error: instruction requires: cpa
+// CHECK-INST: msubpt x0, x1, x2, x3
+// CHECK-ENCODING: encoding: [0x20,0x8c,0x62,0x9b]
+// CHECK-ERROR: error: instruction requires: cpa
+// CHECK-UNKNOWN:  9b628c20 <unknown>
diff --git a/llvm/test/MC/AArch64/armv9.5a-e3dse.s b/llvm/test/MC/AArch64/armv9.5a-e3dse.s
index b69d49ab4e9e5..9d9798a32c8c0 100644
--- a/llvm/test/MC/AArch64/armv9.5a-e3dse.s
+++ b/llvm/test/MC/AArch64/armv9.5a-e3dse.s
@@ -1,13 +1,31 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 mrs x0, VDISR_EL3
-// CHECK: mrs x0, VDISR_EL3                  // encoding: [0x20,0xc1,0x3e,0xd5]
+// CHECK-INST: mrs x0, VDISR_EL3
+// CHECK-ENCODING: encoding: [0x20,0xc1,0x3e,0xd5]
+// CHECK-UNKNOWN:  d53ec120 mrs x0, VDISR_EL3
 
 msr VDISR_EL3, x0
-// CHECK: msr VDISR_EL3, x0                  // encoding: [0x20,0xc1,0x1e,0xd5]
+// CHECK-INST: msr VDISR_EL3, x0
+// CHECK-ENCODING: encoding: [0x20,0xc1,0x1e,0xd5]
+// CHECK-UNKNOWN:  d51ec120 msr VDISR_EL3, x0
 
 mrs x0, VSESR_EL3
-// CHECK: mrs x0, VSESR_EL3                  // encoding: [0x60,0x52,0x3e,0xd5]
+// CHECK-INST: mrs x0, VSESR_EL3
+// CHECK-ENCODING: encoding: [0x60,0x52,0x3e,0xd5]
+// CHECK-UNKNOWN:  d53e5260 mrs x0, VSESR_EL3
 
 msr VSESR_EL3, x0
-// CHECK: msr VSESR_EL3, x0                  // encoding: [0x60,0x52,0x1e,0xd5]
+// CHECK-INST: msr VSESR_EL3, x0
+// CHECK-ENCODING: encoding: [0x60,0x52,0x1e,0xd5]
+// CHECK-UNKNOWN:  d51e5260 msr VSESR_EL3, x0
diff --git a/llvm/test/MC/AArch64/armv9.5a-fgwte3.s b/llvm/test/MC/AArch64/armv9.5a-fgwte3.s
index 2352bc7e1ca71..6546d517d208a 100644
--- a/llvm/test/MC/AArch64/armv9.5a-fgwte3.s
+++ b/llvm/test/MC/AArch64/armv9.5a-fgwte3.s
@@ -1,6 +1,20 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
 
 mrs x0, FGWTE3_EL3
-// CHECK: mrs x0, FGWTE3_EL3                  // encoding: [0xa0,0x11,0x3e,0xd5]
+// CHECK-INST: mrs x0, FGWTE3_EL3
+// CHECK-ENCODING: encoding: [0xa0,0x11,0x3e,0xd5]
+// CHECK-UNKNOWN:  d53e11a0 mrs x0, FGWTE3_EL3
+
 msr FGWTE3_EL3, x0
-// CHECK: msr FGWTE3_EL3, x0                  // encoding: [0xa0,0x11,0x1e,0xd5]
+// CHECK-INST: msr FGWTE3_EL3, x0
+// CHECK-ENCODING: encoding: [0xa0,0x11,0x1e,0xd5]
+// CHECK-UNKNOWN:  d51e11a0 msr FGWTE3_EL3, x0
diff --git a/llvm/test/MC/AArch64/armv9.5a-hacdbs.s b/llvm/test/MC/AArch64/armv9.5a-hacdbs.s
index 8ccba29beb444..e1d1aaace574f 100644
--- a/llvm/test/MC/AArch64/armv9.5a-hacdbs.s
+++ b/llvm/test/MC/AArch64/armv9.5a-hacdbs.s
@@ -1,12 +1,31 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
 
 mrs x0, HACDBSBR_EL2
-// CHECK: mrs x0, HACDBSBR_EL2                  // encoding: [0x80,0x23,0x3c,0xd5]
+// CHECK-INST: mrs x0, HACDBSBR_EL2
+// CHECK-ENCODING: encoding: [0x80,0x23,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c2380 mrs x0, HACDBSBR_EL2
+
 msr HACDBSBR_EL2, x0
-// CHECK: msr HACDBSBR_EL2, x0                  // encoding: [0x80,0x23,0x1c,0xd5]
+// CHECK-INST: msr HACDBSBR_EL2, x0
+// CHECK-ENCODING: encoding: [0x80,0x23,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c2380 msr HACDBSBR_EL2, x0
 
 mrs x0, HACDBSCONS_EL2
-// CHECK: mrs x0, HACDBSCONS_EL2                  // encoding: [0xa0,0x23,0x3c,0xd5]
+// CHECK-INST: mrs x0, HACDBSCONS_EL2
+// CHECK-ENCODING: encoding: [0xa0,0x23,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c23a0 mrs x0, HACDBSCONS_EL2
+
 msr HACDBSCONS_EL2, x0
-// CHECK: msr HACDBSCONS_EL2, x0                  // encoding: [0xa0,0x23,0x1c,0xd5]
+// CHECK-INST: msr HACDBSCONS_EL2, x0
+// CHECK-ENCODING: encoding: [0xa0,0x23,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c23a0 msr HACDBSCONS_EL2, x0
 
diff --git a/llvm/test/MC/AArch64/armv9.5a-hdbss.s b/llvm/test/MC/AArch64/armv9.5a-hdbss.s
index c4505c9d70e7f..3e18fe32aa620 100644
--- a/llvm/test/MC/AArch64/armv9.5a-hdbss.s
+++ b/llvm/test/MC/AArch64/armv9.5a-hdbss.s
@@ -1,12 +1,32 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 mrs x0, HDBSSBR_EL2
-// CHECK: mrs x0, HDBSSBR_EL2                  // encoding: [0x40,0x23,0x3c,0xd5]
+// CHECK-INST: mrs x0, HDBSSBR_EL2
+// CHECK-ENCODING: encoding: [0x40,0x23,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c2340 mrs x0, HDBSSBR_EL2
+
 msr HDBSSBR_EL2, x0
-// CHECK: msr HDBSSBR_EL2, x0                  // encoding: [0x40,0x23,0x1c,0xd5]
+// CHECK-INST: msr HDBSSBR_EL2, x0
+// CHECK-ENCODING: encoding: [0x40,0x23,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c2340 msr HDBSSBR_EL2, x0
 
 mrs x0, HDBSSPROD_EL2
-// CHECK: mrs x0, HDBSSPROD_EL2                  // encoding: [0x60,0x23,0x3c,0xd5]
+// CHECK-INST: mrs x0, HDBSSPROD_EL2
+// CHECK-ENCODING: encoding: [0x60,0x23,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c2360 mrs x0, HDBSSPROD_EL2
+
 msr HDBSSPROD_EL2, x0
-// CHECK: msr HDBSSPROD_EL2, x0                  // encoding: [0x60,0x23,0x1c,0xd5]
+// CHECK-INST: msr HDBSSPROD_EL2, x0
+// CHECK-ENCODING: encoding: [0x60,0x23,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c2360 msr HDBSSPROD_EL2, x0
 
diff --git a/llvm/test/MC/AArch64/armv9.5a-spmu2.s b/llvm/test/MC/AArch64/armv9.5a-spmu2.s
index b7febdb9d248e..51770981ef9bf 100644
--- a/llvm/test/MC/AArch64/armv9.5a-spmu2.s
+++ b/llvm/test/MC/AArch64/armv9.5a-spmu2.s
@@ -1,4 +1,16 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 msr SPMZR_EL0, x0
-// CHECK: msr SPMZR_EL0, x0                  // encoding: [0x80,0x9c,0x13,0xd5]
+// CHECK-INST: msr SPMZR_EL0, x0
+// CHECK-ENCODING: encoding: [0x80,0x9c,0x13,0xd5]
+// CHECK-UNKNOWN:  d5139c80 msr SPMZR_EL0, x0
diff --git a/llvm/test/MC/AArch64/armv9.5a-step2.s b/llvm/test/MC/AArch64/armv9.5a-step2.s
index c5f226bda3178..5d0768502785c 100644
--- a/llvm/test/MC/AArch64/armv9.5a-step2.s
+++ b/llvm/test/MC/AArch64/armv9.5a-step2.s
@@ -1,7 +1,21 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding < %s | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 mrs x0, MDSTEPOP_EL1
-// CHECK: mrs x0, MDSTEPOP_EL1                  // encoding: [0x40,0x05,0x30,0xd5]
+// CHECK-INST: mrs x0, MDSTEPOP_EL1
+// CHECK-ENCODING: encoding: [0x40,0x05,0x30,0xd5]
+// CHECK-UNKNOWN:  d5300540 mrs x0, MDSTEPOP_EL1
 
 msr MDSTEPOP_EL1, x0
-// CHECK: msr MDSTEPOP_EL1, x0                  // encoding: [0x40,0x05,0x10,0xd5]
+// CHECK-INST: msr MDSTEPOP_EL1, x0
+// CHECK-ENCODING: encoding: [0x40,0x05,0x10,0xd5]
+// CHECK-UNKNOWN:  d5100540 msr MDSTEPOP_EL1, x0
diff --git a/llvm/test/MC/AArch64/armv9.5a-tlbiw.s b/llvm/test/MC/AArch64/armv9.5a-tlbiw.s
index 435ed06b33c8c..efd410c1c2c59 100644
--- a/llvm/test/MC/AArch64/armv9.5a-tlbiw.s
+++ b/llvm/test/MC/AArch64/armv9.5a-tlbiw.s
@@ -1,27 +1,50 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding -mattr=+tlbiw -mattr=+xs < %s | FileCheck --check-prefix=CHECK-TLBIW --check-prefix=CHECK-XS %s
-// RUN: not llvm-mc -triple aarch64 -show-encoding -mattr=+tlbiw < %s 2> %t | FileCheck --check-prefix=CHECK-TLBIW %s && FileCheck --check-prefix=ERROR-NO-XS-TLBIW %s < %t
-// RUN: not llvm-mc -triple aarch64 < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-TLBIW --check-prefix=ERROR-NO-XS-TLBIW %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+tlbiw,+xs < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+tlbiw,+xs < %s \
+// RUN:        | llvm-objdump -d --mattr=+tlbiw,+xs --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+tlbiw,+xs < %s \
+// RUN:   | llvm-objdump -d --mattr=-tlbiw,-xs --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+tlbiw,+xs < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+tlbiw,+xs -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
 
 tlbi VMALLWS2E1
-// CHECK-TLBIW: tlbi vmallws2e1                  // encoding: [0x5f,0x86,0x0c,0xd5]
-// ERROR-NO-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1 requires: tlbiw
+// CHECK-INST: tlbi vmallws2e1
+// CHECK-ENCODING: encoding: [0x5f,0x86,0x0c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:6: error: TLBI VMALLWS2E1 requires: tlbiw
+// CHECK-UNKNOWN:  d50c865f      sys #4, c8, c6, #2
 
 tlbi VMALLWS2E1IS
-// CHECK-TLBIW: tlbi vmallws2e1is                // encoding: [0x5f,0x82,0x0c,0xd5]
-// ERROR-NO-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1IS requires: tlbiw
+// CHECK-INST: tlbi vmallws2e1is
+// CHECK-ENCODING: encoding: [0x5f,0x82,0x0c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:6: error: TLBI VMALLWS2E1IS requires: tlbiw
+// CHECK-UNKNOWN:  d50c825f      sys #4, c8, c2, #2
 
 tlbi VMALLWS2E1OS
-// CHECK-TLBIW: tlbi vmallws2e1os                // encoding: [0x5f,0x85,0x0c,0xd5]
-// ERROR-NO-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1OS requires: tlbiw
+// CHECK-INST: tlbi vmallws2e1os
+// CHECK-ENCODING: encoding: [0x5f,0x85,0x0c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:6: error: TLBI VMALLWS2E1OS requires: tlbiw
+// CHECK-UNKNOWN:  d50c855f      sys #4, c8, c5, #2
 
 tlbi VMALLWS2E1nXS
-// CHECK-XS: tlbi vmallws2e1nxs                  // encoding: [0x5f,0x96,0x0c,0xd5]
-// ERROR-NO-XS-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1nXS requires: xs, tlbiw
+// CHECK-INST: tlbi vmallws2e1nxs
+// CHECK-ENCODING: encoding: [0x5f,0x96,0x0c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:6: error: TLBI VMALLWS2E1nXS requires: xs, tlbiw
+// CHECK-UNKNOWN:  d50c965f      sys #4, c9, c6, #2
 
 tlbi VMALLWS2E1ISnXS
-// CHECK-XS: tlbi vmallws2e1isnxs                // encoding: [0x5f,0x92,0x0c,0xd5]
-// ERROR-NO-XS-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1ISnXS requires: xs, tlbiw
+// CHECK-INST: tlbi vmallws2e1isnxs
+// CHECK-ENCODING: encoding: [0x5f,0x92,0x0c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:6: error: TLBI VMALLWS2E1ISnXS requires: xs, tlbiw
+// CHECK-UNKNOWN:  d50c925f      sys #4, c9, c2, #2
 
 tlbi VMALLWS2E1OSnXS
-// CHECK-XS: tlbi vmallws2e1osnxs                // encoding: [0x5f,0x95,0x0c,0xd5]
-// ERROR-NO-XS-TLBIW: [[@LINE-2]]:6: error: TLBI VMALLWS2E1OSnXS requires: xs, tlbiw
+// CHECK-INST: tlbi vmallws2e1osnxs
+// CHECK-ENCODING: encoding: [0x5f,0x95,0x0c,0xd5]
+// CHECK-ERROR: :[[@LINE-3]]:6: error: TLBI VMALLWS2E1OSnXS requires: xs, tlbiw
+// CHECK-UNKNOWN:  d50c955f      sys #4, c9, c5, #2
diff --git a/llvm/test/MC/AArch64/armv9.6a-lsui.s b/llvm/test/MC/AArch64/armv9.6a-lsui.s
index d4a5e1f980560..63a188921ace7 100644
--- a/llvm/test/MC/AArch64/armv9.6a-lsui.s
+++ b/llvm/test/MC/AArch64/armv9.6a-lsui.s
@@ -1,408 +1,714 @@
-// RUN: llvm-mc -triple aarch64 -mattr=+lsui -show-encoding %s  | FileCheck %s
-// RUN: not llvm-mc -triple aarch64 -show-encoding %s 2>&1  | FileCheck %s --check-prefix=ERROR
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+lsui < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+lsui < %s \
+// RUN:        | llvm-objdump -d --mattr=+lsui --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+lsui < %s \
+// RUN:        | llvm-objdump -d --mattr=-lsui --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+lsui < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+lsui -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
-_func:
-// CHECK: _func:
 //------------------------------------------------------------------------------
 // Unprivileged load/store operations
 //------------------------------------------------------------------------------
-  ldtxr       x9, [sp]
-// CHECK: ldtxr	x9, [sp]                        // encoding: [0xe9,0x7f,0x5f,0xc9]
-// ERROR: error: instruction requires: lsui
-  ldtxr       x9, [sp, #0]
-// CHECK: ldtxr	x9, [sp]                        // encoding: [0xe9,0x7f,0x5f,0xc9]
-// ERROR: error: instruction requires: lsui
-  ldtxr       x10, [x11]
-// CHECK: ldtxr	x10, [x11]                      // encoding: [0x6a,0x7d,0x5f,0xc9]
-// ERROR: error: instruction requires: lsui
-  ldtxr       x10, [x11, #0]
-// CHECK: ldtxr	x10, [x11]                      // encoding: [0x6a,0x7d,0x5f,0xc9]
-// ERROR: error: instruction requires: lsui
-
-  ldatxr      x9, [sp]
-// CHECK: ldatxr	x9, [sp]                        // encoding: [0xe9,0xff,0x5f,0xc9]
-// ERROR: error: instruction requires: lsui
-  ldatxr      x10, [x11]
-// CHECK: ldatxr	x10, [x11]                      // encoding: [0x6a,0xfd,0x5f,0xc9]
-// ERROR: error: instruction requires: lsui
-
-  sttxr       wzr, w4, [sp]
-// CHECK: sttxr	wzr, w4, [sp]                   // encoding: [0xe4,0x7f,0x1f,0x89]
-// ERROR: error: instruction requires: lsui
-  sttxr       wzr, w4, [sp, #0]
-// CHECK: sttxr	wzr, w4, [sp]                   // encoding: [0xe4,0x7f,0x1f,0x89]
-// ERROR: error: instruction requires: lsui
-  sttxr       w5, x6, [x7]
-// CHECK: sttxr	w5, x6, [x7]                    // encoding: [0xe6,0x7c,0x05,0xc9]
-// ERROR: error: instruction requires: lsui
-  sttxr       w5, x6, [x7, #0]
-// CHECK: sttxr	w5, x6, [x7]                    // encoding: [0xe6,0x7c,0x05,0xc9]
-// ERROR: error: instruction requires: lsui
-
-  stltxr      w2, w4, [sp]
-// CHECK: stltxr	w2, w4, [sp]                    // encoding: [0xe4,0xff,0x02,0x89]
-// ERROR: error: instruction requires: lsui
-  stltxr      w5, x6, [x7]
-// CHECK: stltxr	w5, x6, [x7]                    // encoding: [0xe6,0xfc,0x05,0xc9]
-// ERROR: error: instruction requires: lsui
+ldtxr x9, [sp]
+// CHECK-INST: ldtxr x9, [sp]
+// CHECK-ENCODING: encoding: [0xe9,0x7f,0x5f,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c95f7fe9 <unknown>
+
+ldtxr x9, [sp, #0]
+// CHECK-INST: ldtxr x9, [sp]
+// CHECK-ENCODING: encoding: [0xe9,0x7f,0x5f,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c95f7fe9 <unknown>
+
+ldtxr x10, [x11]
+// CHECK-INST: ldtxr x10, [x11]
+// CHECK-ENCODING: encoding: [0x6a,0x7d,0x5f,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c95f7d6a <unknown>
+
+ldtxr x10, [x11, #0]
+// CHECK-INST: ldtxr x10, [x11]
+// CHECK-ENCODING: encoding: [0x6a,0x7d,0x5f,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c95f7d6a <unknown>
+
+ldatxr x9, [sp]
+// CHECK-INST: ldatxr x9, [sp]
+// CHECK-ENCODING: encoding: [0xe9,0xff,0x5f,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c95fffe9 <unknown>
+
+ldatxr x10, [x11]
+// CHECK-INST: ldatxr x10, [x11]
+// CHECK-ENCODING: encoding: [0x6a,0xfd,0x5f,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c95ffd6a <unknown>
+
+sttxr wzr, w4, [sp]
+// CHECK-INST: sttxr wzr, w4, [sp]
+// CHECK-ENCODING: encoding: [0xe4,0x7f,0x1f,0x89]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  891f7fe4 <unknown>
+
+sttxr wzr, w4, [sp, #0]
+// CHECK-INST: sttxr wzr, w4, [sp]
+// CHECK-ENCODING: encoding: [0xe4,0x7f,0x1f,0x89]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  891f7fe4 <unknown>
+
+sttxr w5, x6, [x7]
+// CHECK-INST: sttxr w5, x6, [x7]
+// CHECK-ENCODING: encoding: [0xe6,0x7c,0x05,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c9057ce6 <unknown>
+
+sttxr w5, x6, [x7, #0]
+// CHECK-INST: sttxr w5, x6, [x7]
+// CHECK-ENCODING: encoding: [0xe6,0x7c,0x05,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c9057ce6 <unknown>
+
+stltxr w2, w4, [sp]
+// CHECK-INST: stltxr w2, w4, [sp]
+// CHECK-ENCODING: encoding: [0xe4,0xff,0x02,0x89]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  8902ffe4 <unknown>
+
+stltxr w5, x6, [x7]
+// CHECK-INST: stltxr w5, x6, [x7]
+// CHECK-ENCODING: encoding: [0xe6,0xfc,0x05,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c905fce6 <unknown>
 
 //------------------------------------------------------------------------------
 // Unprivileged load/store register pair (offset)
 //------------------------------------------------------------------------------
 
-  ldtp       x21, x29, [x2, #504]
-// CHECK: ldtp	x21, x29, [x2, #504]            // encoding: [0x55,0xf4,0x5f,0xe9]
-// ERROR: instruction requires: lsui
-  ldtp       x22, x23, [x3, #-512]
-// CHECK: ldtp	x22, x23, [x3, #-512]           // encoding: [0x76,0x5c,0x60,0xe9]
-// ERROR: instruction requires: lsui
-  ldtp       x24, x25, [x4, #8]
-// CHECK: ldtp	x24, x25, [x4, #8]              // encoding: [0x98,0xe4,0x40,0xe9]
-// ERROR: instruction requires: lsui
-
-  sttp       x3, x5, [sp], #16
-// CHECK: sttp	x3, x5, [sp], #16               // encoding: [0xe3,0x17,0x81,0xe8]
-// ERROR: instruction requires: lsui
-  sttp       x3, x5, [sp, #8]!
-// CHECK: sttp	x3, x5, [sp, #8]!               // encoding: [0xe3,0x97,0x80,0xe9]
-// ERROR: instruction requires: lsui
-
-  sttp       q3, q5, [sp]
-// CHECK: sttp	q3, q5, [sp]                    // encoding: [0xe3,0x17,0x00,0xed]
-// ERROR: instruction requires: lsui
-  sttp       q17, q19, [sp, #1008]
-// CHECK: sttp	q17, q19, [sp, #1008]           // encoding: [0xf1,0xcf,0x1f,0xed]
-// ERROR: instruction requires: lsui
+ldtp x21, x29, [x2, #504]
+// CHECK-INST: ldtp x21, x29, [x2, #504]
+// CHECK-ENCODING: encoding: [0x55,0xf4,0x5f,0xe9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e95ff455 <unknown>
+
+ldtp x22, x23, [x3, #-512]
+// CHECK-INST: ldtp x22, x23, [x3, #-512]
+// CHECK-ENCODING: encoding: [0x76,0x5c,0x60,0xe9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e9605c76 <unknown>
+
+ldtp x24, x25, [x4, #8]
+// CHECK-INST: ldtp x24, x25, [x4, #8]
+// CHECK-ENCODING: encoding: [0x98,0xe4,0x40,0xe9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e940e498 <unknown>
+
+sttp x3, x5, [sp], #16
+// CHECK-INST: sttp x3, x5, [sp], #16
+// CHECK-ENCODING: encoding: [0xe3,0x17,0x81,0xe8]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e88117e3 <unknown>
+
+sttp x3, x5, [sp, #8]!
+// CHECK-INST: sttp x3, x5, [sp, #8]!
+// CHECK-ENCODING: encoding: [0xe3,0x97,0x80,0xe9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e98097e3 <unknown>
+
+sttp q3, q5, [sp]
+// CHECK-INST: sttp q3, q5, [sp]
+// CHECK-ENCODING: encoding: [0xe3,0x17,0x00,0xed]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ed0017e3 <unknown>
+
+sttp q17, q19, [sp, #1008]
+// CHECK-INST: sttp q17, q19, [sp, #1008]
+// CHECK-ENCODING: encoding: [0xf1,0xcf,0x1f,0xed]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ed1fcff1 <unknown>
 
 //------------------------------------------------------------------------------
 // Load/store register pair (post-indexed)
 //------------------------------------------------------------------------------
 
-  ldtp       x21, x29, [x2], #504
-// CHECK: ldtp	x21, x29, [x2], #504            // encoding: [0x55,0xf4,0xdf,0xe8]
-// ERROR: instruction requires: lsui
-  ldtp       x22, x23, [x3], #-512
-// CHECK: ldtp	x22, x23, [x3], #-512           // encoding: [0x76,0x5c,0xe0,0xe8]
-// ERROR: instruction requires: lsui
-  ldtp       x24, x25, [x4], #8
-// CHECK: ldtp	x24, x25, [x4], #8              // encoding: [0x98,0xe4,0xc0,0xe8]
-// ERROR: instruction requires: lsui
-
-  sttp       q3, q5, [sp], #0
-// CHECK: sttp	q3, q5, [sp], #0                // encoding: [0xe3,0x17,0x80,0xec]
-// ERROR: instruction requires: lsui
-  sttp       q17, q19, [sp], #1008
-// CHECK: sttp	q17, q19, [sp], #1008           // encoding: [0xf1,0xcf,0x9f,0xec]
-// ERROR: instruction requires: lsui
-  ldtp       q23, q29, [x1], #-1024
-// CHECK: ldtp	q23, q29, [x1], #-1024          // encoding: [0x37,0x74,0xe0,0xec]
-// ERROR: instruction requires: lsui
+ldtp x21, x29, [x2], #504
+// CHECK-INST: ldtp x21, x29, [x2], #504
+// CHECK-ENCODING: encoding: [0x55,0xf4,0xdf,0xe8]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e8dff455 <unknown>
+
+ldtp x22, x23, [x3], #-512
+// CHECK-INST: ldtp x22, x23, [x3], #-512
+// CHECK-ENCODING: encoding: [0x76,0x5c,0xe0,0xe8]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e8e05c76 <unknown>
+
+ldtp x24, x25, [x4], #8
+// CHECK-INST: ldtp x24, x25, [x4], #8
+// CHECK-ENCODING: encoding: [0x98,0xe4,0xc0,0xe8]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e8c0e498 <unknown>
+
+sttp q3, q5, [sp], #0
+// CHECK-INST: sttp q3, q5, [sp], #0
+// CHECK-ENCODING: encoding: [0xe3,0x17,0x80,0xec]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ec8017e3 <unknown>
+
+sttp q17, q19, [sp], #1008
+// CHECK-INST: sttp q17, q19, [sp], #1008
+// CHECK-ENCODING: encoding: [0xf1,0xcf,0x9f,0xec]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ec9fcff1 <unknown>
+
+ldtp q23, q29, [x1], #-1024
+// CHECK-INST: ldtp q23, q29, [x1], #-1024
+// CHECK-ENCODING: encoding: [0x37,0x74,0xe0,0xec]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ece07437 <unknown>
 
 //------------------------------------------------------------------------------
 // Load/store register pair (pre-indexed)
 //------------------------------------------------------------------------------
-  ldtp       x21, x29, [x2, #504]!
-// CHECK: ldtp	x21, x29, [x2, #504]!           // encoding: [0x55,0xf4,0xdf,0xe9]
-// ERROR: instruction requires: lsui
-  ldtp       x22, x23, [x3, #-512]!
-// CHECK: ldtp	x22, x23, [x3, #-512]!          // encoding: [0x76,0x5c,0xe0,0xe9]
-// ERROR: instruction requires: lsui
-  ldtp       x24, x25, [x4, #8]!
-// CHECK: ldtp	x24, x25, [x4, #8]!             // encoding: [0x98,0xe4,0xc0,0xe9]
-// ERROR: instruction requires: lsui
-
-  sttp       q3, q5, [sp, #0]!
-// CHECK: sttp	q3, q5, [sp, #0]!               // encoding: [0xe3,0x17,0x80,0xed]
-// ERROR: instruction requires: lsui
-  sttp       q17, q19, [sp, #1008]!
-// CHECK: sttp	q17, q19, [sp, #1008]!          // encoding: [0xf1,0xcf,0x9f,0xed]
-// ERROR: instruction requires: lsui
-  ldtp       q23, q29, [x1, #-1024]!
-// CHECK: ldtp	q23, q29, [x1, #-1024]!         // encoding: [0x37,0x74,0xe0,0xed]
-// ERROR: instruction requires: lsui
+ldtp x21, x29, [x2, #504]!
+// CHECK-INST: ldtp x21, x29, [x2, #504]!
+// CHECK-ENCODING: encoding: [0x55,0xf4,0xdf,0xe9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e9dff455 <unknown>
+
+ldtp x22, x23, [x3, #-512]!
+// CHECK-INST: ldtp x22, x23, [x3, #-512]!
+// CHECK-ENCODING: encoding: [0x76,0x5c,0xe0,0xe9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e9e05c76 <unknown>
+
+ldtp x24, x25, [x4, #8]!
+// CHECK-INST: ldtp x24, x25, [x4, #8]!
+// CHECK-ENCODING: encoding: [0x98,0xe4,0xc0,0xe9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e9c0e498 <unknown>
+
+sttp q3, q5, [sp, #0]!
+// CHECK-INST: sttp q3, q5, [sp, #0]!
+// CHECK-ENCODING: encoding: [0xe3,0x17,0x80,0xed]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ed8017e3 <unknown>
+
+sttp q17, q19, [sp, #1008]!
+// CHECK-INST: sttp q17, q19, [sp, #1008]!
+// CHECK-ENCODING: encoding: [0xf1,0xcf,0x9f,0xed]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ed9fcff1 <unknown>
+
+ldtp q23, q29, [x1, #-1024]!
+// CHECK-INST: ldtp q23, q29, [x1, #-1024]!
+// CHECK-ENCODING: encoding: [0x37,0x74,0xe0,0xed]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ede07437 <unknown>
 
 //------------------------------------------------------------------------------
 // CAS(P)T instructions
 //------------------------------------------------------------------------------
   //64 bits
-  cast       x0, x1, [x2]
-// CHECK: cast	x0, x1, [x2]                    // encoding: [0x41,0x7c,0x80,0xc9]
-// ERROR: instruction requires: lsui
-  cast       x0, x1, [sp, #0]
-// CHECK: cast	x0, x1, [sp]                    // encoding: [0xe1,0x7f,0x80,0xc9]
-// ERROR: instruction requires: lsui
-  casat      x0, x1, [x2]
-// CHECK: casat	x0, x1, [x2]                    // encoding: [0x41,0x7c,0xc0,0xc9]
-// ERROR: instruction requires: lsui
-  casat      x0, x1, [sp, #0]
-// CHECK: casat	x0, x1, [sp]                    // encoding: [0xe1,0x7f,0xc0,0xc9]
-// ERROR: instruction requires: lsui
-  casalt     x0, x1, [x2]
-// CHECK: casalt	x0, x1, [x2]                    // encoding: [0x41,0xfc,0xc0,0xc9]
-// ERROR: instruction requires: lsui
-  casalt     x0, x1, [sp, #0]
-// CHECK: casalt	x0, x1, [sp]                    // encoding: [0xe1,0xff,0xc0,0xc9]
-// ERROR: instruction requires: lsui
-  caslt      x0, x1, [x2]
-// CHECK: caslt	x0, x1, [x2]                    // encoding: [0x41,0xfc,0x80,0xc9]
-// ERROR: instruction requires: lsui
-  caslt      x0, x1, [sp, #0]
-// CHECK: caslt	x0, x1, [sp]                    // encoding: [0xe1,0xff,0x80,0xc9]
-// ERROR: instruction requires: lsui
+  cast x0, x1, [x2]
+// CHECK-INST: cast x0, x1, [x2]
+// CHECK-ENCODING: encoding: [0x41,0x7c,0x80,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c9807c41 <unknown>
+
+  cast x0, x1, [sp, #0]
+// CHECK-INST: cast x0, x1, [sp]
+// CHECK-ENCODING: encoding: [0xe1,0x7f,0x80,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c9807fe1 <unknown>
+
+  casat x0, x1, [x2]
+// CHECK-INST: casat x0, x1, [x2]
+// CHECK-ENCODING: encoding: [0x41,0x7c,0xc0,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c9c07c41 <unknown>
+
+  casat x0, x1, [sp, #0]
+// CHECK-INST: casat x0, x1, [sp]
+// CHECK-ENCODING: encoding: [0xe1,0x7f,0xc0,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c9c07fe1 <unknown>
+
+  casalt x0, x1, [x2]
+// CHECK-INST: casalt x0, x1, [x2]
+// CHECK-ENCODING: encoding: [0x41,0xfc,0xc0,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c9c0fc41 <unknown>
+
+  casalt x0, x1, [sp, #0]
+// CHECK-INST: casalt x0, x1, [sp]
+// CHECK-ENCODING: encoding: [0xe1,0xff,0xc0,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c9c0ffe1 <unknown>
+
+  caslt x0, x1, [x2]
+// CHECK-INST: caslt x0, x1, [x2]
+// CHECK-ENCODING: encoding: [0x41,0xfc,0x80,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c980fc41 <unknown>
+
+  caslt x0, x1, [sp, #0]
+// CHECK-INST: caslt x0, x1, [sp]
+// CHECK-ENCODING: encoding: [0xe1,0xff,0x80,0xc9]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  c980ffe1 <unknown>
 
   //CASP instruction
-  caspt      x0, x1, x2, x3, [x4]
-// CHECK: caspt	x0, x1, x2, x3, [x4]            // encoding: [0x82,0x7c,0x80,0x49]
-// ERROR: instruction requires: lsui
-  caspt      x0, x1, x2, x3, [sp, #0]
-// CHECK: caspt	x0, x1, x2, x3, [sp]            // encoding: [0xe2,0x7f,0x80,0x49]
-// ERROR: instruction requires: lsui
-  caspat     x0, x1, x2, x3, [x4]
-// CHECK: caspat	x0, x1, x2, x3, [x4]            // encoding: [0x82,0x7c,0xc0,0x49]
-// ERROR: instruction requires: lsui
-  caspat     x0, x1, x2, x3, [sp, #0]
-// CHECK: caspat	x0, x1, x2, x3, [sp]            // encoding: [0xe2,0x7f,0xc0,0x49]
-// ERROR: instruction requires: lsui
-  casplt     x0, x1, x2, x3, [x4]
-// CHECK: casplt	x0, x1, x2, x3, [x4]            // encoding: [0x82,0xfc,0x80,0x49]
-// ERROR: instruction requires: lsui
-  casplt     x0, x1, x2, x3, [sp, #0]
-// CHECK: casplt	x0, x1, x2, x3, [sp]            // encoding: [0xe2,0xff,0x80,0x49]
-// ERROR: instruction requires: lsui
-  caspalt    x0, x1, x2, x3, [x4]
-// CHECK: caspalt	x0, x1, x2, x3, [x4]            // encoding: [0x82,0xfc,0xc0,0x49]
-// ERROR: instruction requires: lsui
-  caspalt    x0, x1, x2, x3, [sp, #0]
-// CHECK: caspalt	x0, x1, x2, x3, [sp]            // encoding: [0xe2,0xff,0xc0,0x49]
-// ERROR: instruction requires: lsui
+caspt x0, x1, x2, x3, [x4]
+// CHECK-INST: caspt x0, x1, x2, x3, [x4]
+// CHECK-ENCODING: encoding: [0x82,0x7c,0x80,0x49]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  49807c82 <unknown>
+
+caspt x0, x1, x2, x3, [sp, #0]
+// CHECK-INST: caspt x0, x1, x2, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe2,0x7f,0x80,0x49]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  49807fe2 <unknown>
+
+caspat x0, x1, x2, x3, [x4]
+// CHECK-INST: caspat x0, x1, x2, x3, [x4]
+// CHECK-ENCODING: encoding: [0x82,0x7c,0xc0,0x49]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  49c07c82 <unknown>
+
+caspat x0, x1, x2, x3, [sp, #0]
+// CHECK-INST: caspat x0, x1, x2, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe2,0x7f,0xc0,0x49]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  49c07fe2 <unknown>
+
+casplt x0, x1, x2, x3, [x4]
+// CHECK-INST: casplt x0, x1, x2, x3, [x4]
+// CHECK-ENCODING: encoding: [0x82,0xfc,0x80,0x49]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  4980fc82 <unknown>
+
+casplt x0, x1, x2, x3, [sp, #0]
+// CHECK-INST: casplt x0, x1, x2, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe2,0xff,0x80,0x49]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  4980ffe2 <unknown>
+
+caspalt x0, x1, x2, x3, [x4]
+// CHECK-INST: caspalt x0, x1, x2, x3, [x4]
+// CHECK-ENCODING: encoding: [0x82,0xfc,0xc0,0x49]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  49c0fc82 <unknown>
+
+caspalt x0, x1, x2, x3, [sp, #0]
+// CHECK-INST: caspalt x0, x1, x2, x3, [sp]
+// CHECK-ENCODING: encoding: [0xe2,0xff,0xc0,0x49]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  49c0ffe2 <unknown>
 
 //------------------------------------------------------------------------------
 // SWP(A|L)T instructions
 //------------------------------------------------------------------------------
-  swpt       w7, wzr, [x5]
-// CHECK: swpt	w7, wzr, [x5]                   // encoding: [0xbf,0x84,0x27,0x19]
-// ERROR: instruction requires: lsui
-  swpt       x9, xzr, [sp]
-// CHECK: swpt	x9, xzr, [sp]                   // encoding: [0xff,0x87,0x29,0x59]
-// ERROR: instruction requires: lsui
-
-  swpta      w7, wzr, [x5]
-// CHECK: swpta	w7, wzr, [x5]                   // encoding: [0xbf,0x84,0xa7,0x19]
-// ERROR: instruction requires: lsui
-  swpta      x9, xzr, [sp]
-// CHECK: swpta	x9, xzr, [sp]                   // encoding: [0xff,0x87,0xa9,0x59]
-// ERROR: instruction requires: lsui
-
-  swptl      w7, wzr, [x5]
-// CHECK: swptl	w7, wzr, [x5]                   // encoding: [0xbf,0x84,0x67,0x19]
-// ERROR: instruction requires: lsui
-  swptl      x9, xzr, [sp]
-// CHECK: swptl	x9, xzr, [sp]                   // encoding: [0xff,0x87,0x69,0x59]
-// ERROR: instruction requires: lsui
-
-  swptal     w7, wzr, [x5]
-// CHECK: swptal	w7, wzr, [x5]                   // encoding: [0xbf,0x84,0xe7,0x19]
-// ERROR: instruction requires: lsui
-  swptal     x9, xzr, [sp]
-// CHECK: swptal	x9, xzr, [sp]                   // encoding: [0xff,0x87,0xe9,0x59]
-// ERROR: instruction requires: lsui
+swpt w7, wzr, [x5]
+// CHECK-INST: swpt w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x84,0x27,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  192784bf <unknown>
+
+swpt x9, xzr, [sp]
+// CHECK-INST: swpt x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x87,0x29,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  592987ff <unknown>
+
+swpta w7, wzr, [x5]
+// CHECK-INST: swpta w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x84,0xa7,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  19a784bf <unknown>
+
+swpta x9, xzr, [sp]
+// CHECK-INST: swpta x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x87,0xa9,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  59a987ff <unknown>
+
+swptl w7, wzr, [x5]
+// CHECK-INST: swptl w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x84,0x67,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  196784bf <unknown>
+
+swptl x9, xzr, [sp]
+// CHECK-INST: swptl x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x87,0x69,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  596987ff <unknown>
+
+swptal w7, wzr, [x5]
+// CHECK-INST: swptal w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x84,0xe7,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  19e784bf <unknown>
+
+swptal x9, xzr, [sp]
+// CHECK-INST: swptal x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x87,0xe9,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  59e987ff <unknown>
 
 //------------------------------------------------------------------------------
 // LD{ADD|CLR|SET)(A|L|AL)T instructions
 //------------------------------------------------------------------------------
 
-  ldtadd     w7, wzr, [x5]
-// CHECK: ldtadd	w7, wzr, [x5]                   // encoding: [0xbf,0x04,0x27,0x19]
-// ERROR: instruction requires: lsui
-  ldtadd     x9, xzr, [sp]
-// CHECK: ldtadd	x9, xzr, [sp]                   // encoding: [0xff,0x07,0x29,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtadda    w7, wzr, [x5]
-// CHECK: ldtadda	w7, wzr, [x5]                   // encoding: [0xbf,0x04,0xa7,0x19]
-// ERROR: instruction requires: lsui
-  ldtadda    x9, xzr, [sp]
-// CHECK: ldtadda	x9, xzr, [sp]                   // encoding: [0xff,0x07,0xa9,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtaddl    w7, wzr, [x5]
-// CHECK: ldtaddl	w7, wzr, [x5]                   // encoding: [0xbf,0x04,0x67,0x19]
-// ERROR: instruction requires: lsui
-  ldtaddl    x9, xzr, [sp]
-// CHECK: ldtaddl	x9, xzr, [sp]                   // encoding: [0xff,0x07,0x69,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtaddal   w7, wzr, [x5]
-// CHECK: ldtaddal	w7, wzr, [x5]                   // encoding: [0xbf,0x04,0xe7,0x19]
-// ERROR: instruction requires: lsui
-  ldtaddal   x9, xzr, [sp]
-// CHECK: ldtaddal	x9, xzr, [sp]                   // encoding: [0xff,0x07,0xe9,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtclr     w7, wzr, [x5]
-// CHECK: ldtclr	w7, wzr, [x5]                   // encoding: [0xbf,0x14,0x27,0x19]
-// ERROR: instruction requires: lsui
-  ldtclr     x9, xzr, [sp]
-// CHECK: ldtclr	x9, xzr, [sp]                   // encoding: [0xff,0x17,0x29,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtclrl    w7, wzr, [x5]
-// CHECK: ldtclrl	w7, wzr, [x5]                   // encoding: [0xbf,0x14,0x67,0x19]
-// ERROR: instruction requires: lsui
-  ldtclrl    x9, xzr, [sp]
-// CHECK: ldtclrl	x9, xzr, [sp]                   // encoding: [0xff,0x17,0x69,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtclra    w7, wzr, [x5]
-// CHECK: ldtclra	w7, wzr, [x5]                   // encoding: [0xbf,0x14,0xa7,0x19]
-// ERROR: instruction requires: lsui
-  ldtclra    x9, xzr, [sp]
-// CHECK: ldtclra	x9, xzr, [sp]                   // encoding: [0xff,0x17,0xa9,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtclral   w7, wzr, [x5]
-// CHECK: ldtclral	w7, wzr, [x5]                   // encoding: [0xbf,0x14,0xe7,0x19]
-// ERROR: instruction requires: lsui
-  ldtclral   x9, xzr, [sp]
-// CHECK: ldtclral	x9, xzr, [sp]                   // encoding: [0xff,0x17,0xe9,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtset     w7, wzr, [x5]
-// CHECK: ldtset	w7, wzr, [x5]                   // encoding: [0xbf,0x34,0x27,0x19]
-// ERROR: instruction requires: lsui
-  ldtset     x9, xzr, [sp]
-// CHECK: ldtset	x9, xzr, [sp]                   // encoding: [0xff,0x37,0x29,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtsetl    w7, wzr, [x5]
-// CHECK: ldtsetl	w7, wzr, [x5]                   // encoding: [0xbf,0x34,0x67,0x19]
-// ERROR: instruction requires: lsui
-  ldtsetl    x9, xzr, [sp]
-// CHECK: ldtsetl	x9, xzr, [sp]                   // encoding: [0xff,0x37,0x69,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtseta    w7, wzr, [x5]
-// CHECK: ldtseta	w7, wzr, [x5]                   // encoding: [0xbf,0x34,0xa7,0x19]
-// ERROR: instruction requires: lsui
-  ldtseta    x9, xzr, [sp]
-// CHECK: ldtseta	x9, xzr, [sp]                   // encoding: [0xff,0x37,0xa9,0x59]
-// ERROR: instruction requires: lsui
-
-  ldtsetal   w7, wzr, [x5]
-// CHECK: ldtsetal	w7, wzr, [x5]                   // encoding: [0xbf,0x34,0xe7,0x19]
-// ERROR: instruction requires: lsui
-  ldtsetal   x9, xzr, [sp]
-// CHECK: ldtsetal	x9, xzr, [sp]                   // encoding: [0xff,0x37,0xe9,0x59]
-// ERROR: instruction requires: lsui
+ldtadd w7, wzr, [x5]
+// CHECK-INST: sttadd w7, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x04,0x27,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  192704bf <unknown>
+
+ldtadd x9, xzr, [sp]
+// CHECK-INST: sttadd x9, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x07,0x29,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  592907ff <unknown>
+
+ldtadda w7, wzr, [x5]
+// CHECK-INST: ldtadda w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x04,0xa7,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  19a704bf <unknown>
+
+ldtadda x9, xzr, [sp]
+// CHECK-INST: ldtadda x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x07,0xa9,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  59a907ff <unknown>
+
+ldtaddl w7, wzr, [x5]
+// CHECK-INST: sttaddl w7, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x04,0x67,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  196704bf <unknown>
+
+ldtaddl x9, xzr, [sp]
+// CHECK-INST: sttaddl x9, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x07,0x69,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  596907ff <unknown>
+
+ldtaddal w7, wzr, [x5]
+// CHECK-INST: ldtaddal w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x04,0xe7,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  19e704bf <unknown>
+
+ldtaddal x9, xzr, [sp]
+// CHECK-INST: ldtaddal x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x07,0xe9,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  59e907ff <unknown>
+
+ldtclr w7, wzr, [x5]
+// CHECK-INST: sttclr w7, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x14,0x27,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  192714bf <unknown>
+
+ldtclr x9, xzr, [sp]
+// CHECK-INST: sttclr x9, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x17,0x29,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  592917ff <unknown>
+
+ldtclrl w7, wzr, [x5]
+// CHECK-INST: sttclrl w7, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x14,0x67,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  196714bf <unknown>
+
+ldtclrl x9, xzr, [sp]
+// CHECK-INST: sttclrl x9, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x17,0x69,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  596917ff <unknown>
+
+ldtclra w7, wzr, [x5]
+// CHECK-INST: ldtclra w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x14,0xa7,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  19a714bf <unknown>
+
+ldtclra x9, xzr, [sp]
+// CHECK-INST: ldtclra x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x17,0xa9,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  59a917ff <unknown>
+
+ldtclral w7, wzr, [x5]
+// CHECK-INST: ldtclral w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x14,0xe7,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  19e714bf <unknown>
+
+ldtclral x9, xzr, [sp]
+// CHECK-INST: ldtclral x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x17,0xe9,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  59e917ff <unknown>
+
+ldtset w7, wzr, [x5]
+// CHECK-INST: sttset w7, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x34,0x27,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  192734bf <unknown>
+
+ldtset x9, xzr, [sp]
+// CHECK-INST: sttset x9, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x37,0x29,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  592937ff <unknown>
+
+ldtsetl w7, wzr, [x5]
+// CHECK-INST: sttsetl w7, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x34,0x67,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  196734bf <unknown>
+
+ldtsetl x9, xzr, [sp]
+// CHECK-INST: sttsetl x9, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x37,0x69,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  596937ff <unknown>
+
+ldtseta w7, wzr, [x5]
+// CHECK-INST: ldtseta w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x34,0xa7,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  19a734bf <unknown>
+
+ldtseta x9, xzr, [sp]
+// CHECK-INST: ldtseta x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x37,0xa9,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  59a937ff <unknown>
+
+ldtsetal w7, wzr, [x5]
+// CHECK-INST: ldtsetal w7, wzr, [x5]
+// CHECK-ENCODING: encoding: [0xbf,0x34,0xe7,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  19e734bf <unknown>
+
+ldtsetal x9, xzr, [sp]
+// CHECK-INST: ldtsetal x9, xzr, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x37,0xe9,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  59e937ff <unknown>
 
 //------------------------------------------------------------------------------
 // ST{ADD|CLR|SET)(A|L|AL)T instructions
 //------------------------------------------------------------------------------
 
-  sttadd     w0, [x2]
-// CHECK: ldtadd	w0, wzr, [x2]                   // encoding: [0x5f,0x04,0x20,0x19]
-// ERROR: instruction requires: lsui
-  sttadd     w2, [sp]
-// CHECK: ldtadd	w2, wzr, [sp]                   // encoding: [0xff,0x07,0x22,0x19]
-// ERROR: instruction requires: lsui
-  sttadd     x0, [x2]
-// CHECK: ldtadd	x0, xzr, [x2]                   // encoding: [0x5f,0x04,0x20,0x59]
-// ERROR: instruction requires: lsui
-  sttadd     x2, [sp]
-// CHECK: ldtadd	x2, xzr, [sp]                   // encoding: [0xff,0x07,0x22,0x59]
-// ERROR: instruction requires: lsui
-
-  sttaddl    w0, [x2]
-// CHECK: ldtaddl	w0, wzr, [x2]                   // encoding: [0x5f,0x04,0x60,0x19]
-// ERROR: instruction requires: lsui
-  sttaddl    w2, [sp]
-// CHECK: ldtaddl	w2, wzr, [sp]                   // encoding: [0xff,0x07,0x62,0x19]
-// ERROR: instruction requires: lsui
-  sttaddl    x0, [x2]
-// CHECK: ldtaddl	x0, xzr, [x2]                   // encoding: [0x5f,0x04,0x60,0x59]
-// ERROR: instruction requires: lsui
-  sttaddl    x2, [sp]
-// CHECK: ldtaddl	x2, xzr, [sp]                   // encoding: [0xff,0x07,0x62,0x59]
-// ERROR: instruction requires: lsui
-
-  sttclr     w0, [x2]
-// CHECK: ldtclr	w0, wzr, [x2]                   // encoding: [0x5f,0x14,0x20,0x19]
-// ERROR: instruction requires: lsui
-  sttclr     w2, [sp]
-// CHECK: ldtclr	w2, wzr, [sp]                   // encoding: [0xff,0x17,0x22,0x19]
-// ERROR: instruction requires: lsui
-  sttclr     x0, [x2]
-// CHECK: ldtclr	x0, xzr, [x2]                   // encoding: [0x5f,0x14,0x20,0x59]
-// ERROR: instruction requires: lsui
-  sttclr     x2, [sp]
-// CHECK: ldtclr	x2, xzr, [sp]                   // encoding: [0xff,0x17,0x22,0x59]
-// ERROR: instruction requires: lsui
-
-  sttclrl    w0, [x2]
-// CHECK: ldtclrl	w0, wzr, [x2]                   // encoding: [0x5f,0x14,0x60,0x19]
-// ERROR: instruction requires: lsui
-  sttclrl    w2, [sp]
-// CHECK: ldtclrl	w2, wzr, [sp]                   // encoding: [0xff,0x17,0x62,0x19]
-// ERROR: instruction requires: lsui
-  sttclrl    x0, [x2]
-// CHECK: ldtclrl	x0, xzr, [x2]                   // encoding: [0x5f,0x14,0x60,0x59]
-// ERROR: instruction requires: lsui
-  sttclrl    x2, [sp]
-// CHECK: ldtclrl	x2, xzr, [sp]                   // encoding: [0xff,0x17,0x62,0x59]
-// ERROR: instruction requires: lsui
-
-  sttset     w0, [x2]
-// CHECK: ldtset	w0, wzr, [x2]                   // encoding: [0x5f,0x34,0x20,0x19]
-// ERROR: instruction requires: lsui
-  sttset     w2, [sp]
-// CHECK: ldtset	w2, wzr, [sp]                   // encoding: [0xff,0x37,0x22,0x19]
-// ERROR: instruction requires: lsui
-  sttset     x0, [x2]
-// CHECK: ldtset	x0, xzr, [x2]                   // encoding: [0x5f,0x34,0x20,0x59]
-// ERROR: instruction requires: lsui
-  sttset     x2, [sp]
-// CHECK: ldtset	x2, xzr, [sp]                   // encoding: [0xff,0x37,0x22,0x59]
-// ERROR: instruction requires: lsui
-
-  sttsetl    w0, [x2]
-// CHECK: ldtsetl	w0, wzr, [x2]                   // encoding: [0x5f,0x34,0x60,0x19]
-// ERROR: instruction requires: lsui
-  sttsetl    w2, [sp]
-// CHECK: ldtsetl	w2, wzr, [sp]                   // encoding: [0xff,0x37,0x62,0x19]
-// ERROR: instruction requires: lsui
-  sttsetl    x0, [x2]
-// CHECK: ldtsetl	x0, xzr, [x2]                   // encoding: [0x5f,0x34,0x60,0x59]
-// ERROR: instruction requires: lsui
-  sttsetl    x2, [sp]
-// CHECK: ldtsetl	x2, xzr, [sp]                   // encoding: [0xff,0x37,0x62,0x59]
-// ERROR: instruction requires: lsui
+sttadd w0, [x2]
+// CHECK-INST: sttadd w0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x04,0x20,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  1920045f <unknown>
+
+sttadd w2, [sp]
+// CHECK-INST: sttadd w2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x07,0x22,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  192207ff <unknown>
+
+sttadd x0, [x2]
+// CHECK-INST: sttadd x0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x04,0x20,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  5920045f <unknown>
+
+sttadd x2, [sp]
+// CHECK-INST: sttadd x2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x07,0x22,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  592207ff <unknown>
+
+sttaddl w0, [x2]
+// CHECK-INST: sttaddl w0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x04,0x60,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  1960045f <unknown>
+
+sttaddl w2, [sp]
+// CHECK-INST: sttaddl w2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x07,0x62,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  196207ff <unknown>
+
+sttaddl x0, [x2]
+// CHECK-INST: sttaddl x0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x04,0x60,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  5960045f <unknown>
+
+sttaddl x2, [sp]
+// CHECK-INST: sttaddl x2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x07,0x62,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  596207ff <unknown>
+
+sttclr w0, [x2]
+// CHECK-INST: sttclr w0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x14,0x20,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  1920145f <unknown>
+
+sttclr w2, [sp]
+// CHECK-INST: sttclr w2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x17,0x22,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  192217ff <unknown>
+
+sttclr x0, [x2]
+// CHECK-INST: sttclr x0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x14,0x20,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  5920145f <unknown>
+
+sttclr x2, [sp]
+// CHECK-INST: sttclr x2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x17,0x22,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  592217ff <unknown>
+
+sttclrl w0, [x2]
+// CHECK-INST: sttclrl w0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x14,0x60,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  1960145f <unknown>
+
+sttclrl w2, [sp]
+// CHECK-INST: sttclrl w2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x17,0x62,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  196217ff <unknown>
+
+sttclrl x0, [x2]
+// CHECK-INST: sttclrl x0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x14,0x60,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  5960145f <unknown>
+
+sttclrl x2, [sp]
+// CHECK-INST: sttclrl x2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x17,0x62,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  596217ff <unknown>
+
+sttset w0, [x2]
+// CHECK-INST: sttset w0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x34,0x20,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  1920345f <unknown>
+
+sttset w2, [sp]
+// CHECK-INST: sttset w2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x37,0x22,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  192237ff <unknown>
+
+sttset x0, [x2]
+// CHECK-INST: sttset x0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x34,0x20,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  5920345f <unknown>
+
+sttset x2, [sp]
+// CHECK-INST: sttset x2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x37,0x22,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  592237ff <unknown>
+
+sttsetl w0, [x2]
+// CHECK-INST: sttsetl w0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x34,0x60,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  1960345f <unknown>
+
+sttsetl w2, [sp]
+// CHECK-INST: sttsetl w2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x37,0x62,0x19]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  196237ff <unknown>
+
+sttsetl x0, [x2]
+// CHECK-INST: sttsetl x0, [x2]
+// CHECK-ENCODING: encoding: [0x5f,0x34,0x60,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  5960345f <unknown>
+
+sttsetl x2, [sp]
+// CHECK-INST: sttsetl x2, [sp]
+// CHECK-ENCODING: encoding: [0xff,0x37,0x62,0x59]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  596237ff <unknown>
 
 //------------------------------------------------------------------------------
 // Load/store non-temporal register pair (offset)
 //------------------------------------------------------------------------------
-  ldtnp      x21, x29, [x2, #504]
-// CHECK: ldtnp	x21, x29, [x2, #504]            // encoding: [0x55,0xf4,0x5f,0xe8]
-// ERROR: instruction requires: lsui
-  ldtnp      x22, x23, [x3, #-512]
-// CHECK: ldtnp	x22, x23, [x3, #-512]           // encoding: [0x76,0x5c,0x60,0xe8]
-// ERROR: instruction requires: lsui
-  ldtnp      x24, x25, [x4, #8]
-// CHECK: ldtnp	x24, x25, [x4, #8]              // encoding: [0x98,0xe4,0x40,0xe8]
-// ERROR: instruction requires: lsui
-  ldtnp      q23, q29, [x1, #-1024]
-// CHECK: ldtnp	q23, q29, [x1, #-1024]          // encoding: [0x37,0x74,0x60,0xec]
-// ERROR: instruction requires: lsui
-
-  sttnp      x3, x5, [sp]
-// CHECK: sttnp	x3, x5, [sp]                    // encoding: [0xe3,0x17,0x00,0xe8]
-// ERROR: instruction requires: lsui
-  sttnp      x17, x19, [sp, #64]
-// CHECK: sttnp	x17, x19, [sp, #64]             // encoding: [0xf1,0x4f,0x04,0xe8]
-// ERROR: instruction requires: lsui
-  sttnp      q3, q5, [sp]
-// CHECK: sttnp	q3, q5, [sp]                    // encoding: [0xe3,0x17,0x00,0xec]
-// ERROR: instruction requires: lsui
-  sttnp      q17, q19, [sp, #1008]
-// CHECK: sttnp	q17, q19, [sp, #1008]           // encoding: [0xf1,0xcf,0x1f,0xec]
-// ERROR: instruction requires: lsui
-
+ldtnp x21, x29, [x2, #504]
+// CHECK-INST: ldtnp x21, x29, [x2, #504]
+// CHECK-ENCODING: encoding: [0x55,0xf4,0x5f,0xe8]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e85ff455 <unknown>
+
+ldtnp x22, x23, [x3, #-512]
+// CHECK-INST: ldtnp x22, x23, [x3, #-512]
+// CHECK-ENCODING: encoding: [0x76,0x5c,0x60,0xe8]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e8605c76 <unknown>
+
+ldtnp x24, x25, [x4, #8]
+// CHECK-INST: ldtnp x24, x25, [x4, #8]
+// CHECK-ENCODING: encoding: [0x98,0xe4,0x40,0xe8]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e840e498 <unknown>
+
+ldtnp q23, q29, [x1, #-1024]
+// CHECK-INST: ldtnp q23, q29, [x1, #-1024]
+// CHECK-ENCODING: encoding: [0x37,0x74,0x60,0xec]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ec607437 <unknown>
+
+sttnp x3, x5, [sp]
+// CHECK-INST: sttnp x3, x5, [sp]
+// CHECK-ENCODING: encoding: [0xe3,0x17,0x00,0xe8]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e80017e3 <unknown>
+
+sttnp x17, x19, [sp, #64]
+// CHECK-INST: sttnp x17, x19, [sp, #64]
+// CHECK-ENCODING: encoding: [0xf1,0x4f,0x04,0xe8]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  e8044ff1 <unknown>
+
+sttnp q3, q5, [sp]
+// CHECK-INST: sttnp q3, q5, [sp]
+// CHECK-ENCODING: encoding: [0xe3,0x17,0x00,0xec]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ec0017e3 <unknown>
+
+sttnp q17, q19, [sp, #1008]
+// CHECK-INST: sttnp q17, q19, [sp, #1008]
+// CHECK-ENCODING: encoding: [0xf1,0xcf,0x1f,0xec]
+// CHECK-ERROR: error: instruction requires: lsui
+// CHECK-UNKNOWN:  ec1fcff1 <unknown>
diff --git a/llvm/test/MC/AArch64/armv9.6a-mpam-diagnostics.s b/llvm/test/MC/AArch64/armv9.6a-mpam-diagnostics.s
new file mode 100644
index 0000000000000..a39eaef2a8df4
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9.6a-mpam-diagnostics.s
@@ -0,0 +1,5 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+
+msr MPAMBWIDR_EL1, x0
+// CHECK-ERROR: error: expected writable system register or pstate
\ No newline at end of file
diff --git a/llvm/test/MC/AArch64/armv9.6a-mpam.s b/llvm/test/MC/AArch64/armv9.6a-mpam.s
index c0696efd3cceb..82603caed5545 100644
--- a/llvm/test/MC/AArch64/armv9.6a-mpam.s
+++ b/llvm/test/MC/AArch64/armv9.6a-mpam.s
@@ -1,45 +1,94 @@
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding < %s 2> %t | FileCheck %s --check-prefix=CHECK
-// RUN: FileCheck --check-prefix=CHECK-RO < %t %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 //------------------------------------------------------------------------------
 // Armv9.6-A FEAT_MPAM Extensions
 //------------------------------------------------------------------------------
 
-msr MPAMBWIDR_EL1, x0
 msr MPAMBW3_EL3, x0
+// CHECK-INST: msr MPAMBW3_EL3, x0
+// CHECK-ENCODING: encoding: [0x80,0xa5,0x1e,0xd5]
+// CHECK-UNKNOWN:  d51ea580 msr MPAMBW3_EL3, x0
+
 msr MPAMBW2_EL2, x0
+// CHECK-INST: msr MPAMBW2_EL2, x0
+// CHECK-ENCODING: encoding: [0x80,0xa5,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51ca580 msr MPAMBW2_EL2, x0
+
 msr MPAMBW1_EL1, x0
+// CHECK-INST: msr MPAMBW1_EL1, x0
+// CHECK-ENCODING: encoding: [0x80,0xa5,0x18,0xd5]
+// CHECK-UNKNOWN:  d518a580 msr MPAMBW1_EL1, x0
+
 msr MPAMBW1_EL12, x0
+// CHECK-INST: msr MPAMBW1_EL12, x0
+// CHECK-ENCODING: encoding: [0x80,0xa5,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51da580 msr MPAMBW1_EL12, x0
+
 msr MPAMBW0_EL1, x0
+// CHECK-INST: msr MPAMBW0_EL1, x0
+// CHECK-ENCODING: encoding: [0xa0,0xa5,0x18,0xd5]
+// CHECK-UNKNOWN:  d518a5a0 msr MPAMBW0_EL1, x0
+
 msr MPAMBWCAP_EL2, x0
+// CHECK-INST: msr MPAMBWCAP_EL2, x0
+// CHECK-ENCODING: encoding: [0xc0,0xa5,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51ca5c0 msr MPAMBWCAP_EL2, x0
+
 msr MPAMBWSM_EL1, x0
+// CHECK-INST: msr MPAMBWSM_EL1, x0
+// CHECK-ENCODING: encoding: [0xe0,0xa5,0x18,0xd5]
+// CHECK-UNKNOWN:  d518a5e0 msr MPAMBWSM_EL1, x0
 
 mrs x0, MPAMBWIDR_EL1
+// CHECK-INST: mrs x0, MPAMBWIDR_EL1
+// CHECK-ENCODING: encoding: [0xa0,0xa4,0x38,0xd5]
+// CHECK-UNKNOWN:  d538a4a0 mrs x0, MPAMBWIDR_EL1
+
 mrs x0, MPAMBW3_EL3
+// CHECK-INST: mrs x0, MPAMBW3_EL3
+// CHECK-ENCODING: encoding: [0x80,0xa5,0x3e,0xd5]
+// CHECK-UNKNOWN:  d53ea580 mrs x0, MPAMBW3_EL3
+
 mrs x0, MPAMBW2_EL2
+// CHECK-INST: mrs x0, MPAMBW2_EL2
+// CHECK-ENCODING: encoding: [0x80,0xa5,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53ca580 mrs x0, MPAMBW2_EL2
+
 mrs x0, MPAMBW1_EL1
+// CHECK-INST: mrs x0, MPAMBW1_EL1
+// CHECK-ENCODING: encoding: [0x80,0xa5,0x38,0xd5]
+// CHECK-UNKNOWN:  d538a580 mrs x0, MPAMBW1_EL1
+
 mrs x0, MPAMBW1_EL12
+// CHECK-INST: mrs x0, MPAMBW1_EL12
+// CHECK-ENCODING: encoding: [0x80,0xa5,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53da580 mrs x0, MPAMBW1_EL12
+
 mrs x0, MPAMBW0_EL1
+// CHECK-INST: mrs x0, MPAMBW0_EL1
+// CHECK-ENCODING: encoding: [0xa0,0xa5,0x38,0xd5]
+// CHECK-UNKNOWN:  d538a5a0 mrs x0, MPAMBW0_EL1
+
 mrs x0, MPAMBWCAP_EL2
+// CHECK-INST: mrs x0, MPAMBWCAP_EL2
+// CHECK-ENCODING: encoding: [0xc0,0xa5,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53ca5c0 mrs x0, MPAMBWCAP_EL2
+
 mrs x0, MPAMBWSM_EL1
+// CHECK-INST: mrs x0, MPAMBWSM_EL1
+// CHECK-ENCODING: encoding: [0xe0,0xa5,0x38,0xd5]
+// CHECK-UNKNOWN:  d538a5e0 mrs x0, MPAMBWSM_EL1
+
 
-//CHECK: msr     MPAMBW3_EL3, x0                 // encoding: [0x80,0xa5,0x1e,0xd5]
-//CHECK: msr     MPAMBW2_EL2, x0                 // encoding: [0x80,0xa5,0x1c,0xd5]
-//CHECK: msr     MPAMBW1_EL1, x0                 // encoding: [0x80,0xa5,0x18,0xd5]
-//CHECK: msr     MPAMBW1_EL12, x0                // encoding: [0x80,0xa5,0x1d,0xd5]
-//CHECK: msr     MPAMBW0_EL1, x0                 // encoding: [0xa0,0xa5,0x18,0xd5]
-//CHECK: msr     MPAMBWCAP_EL2, x0               // encoding: [0xc0,0xa5,0x1c,0xd5]
-//CHECK: msr     MPAMBWSM_EL1, x0                // encoding: [0xe0,0xa5,0x18,0xd5]
-
-//CHECK-RO: error: expected writable system register or pstate
-//CHECK-RO: msr MPAMBWIDR_EL1, x0
-//CHECK-RO:     ^
-
-//CHECK: mrs     x0, MPAMBWIDR_EL1               // encoding: [0xa0,0xa4,0x38,0xd5]
-//CHECK: mrs     x0, MPAMBW3_EL3                 // encoding: [0x80,0xa5,0x3e,0xd5]
-//CHECK: mrs     x0, MPAMBW2_EL2                 // encoding: [0x80,0xa5,0x3c,0xd5]
-//CHECK: mrs     x0, MPAMBW1_EL1                 // encoding: [0x80,0xa5,0x38,0xd5]
-//CHECK: mrs     x0, MPAMBW1_EL12                // encoding: [0x80,0xa5,0x3d,0xd5]
-//CHECK: mrs     x0, MPAMBW0_EL1                 // encoding: [0xa0,0xa5,0x38,0xd5]
-//CHECK: mrs     x0, MPAMBWCAP_EL2               // encoding: [0xc0,0xa5,0x3c,0xd5]
-//CHECK: mrs     x0, MPAMBWSM_EL1                // encoding: [0xe0,0xa5,0x38,0xd5]
diff --git a/llvm/test/MC/AArch64/armv9.6a-occmo.s b/llvm/test/MC/AArch64/armv9.6a-occmo.s
index d6548f98645a5..9f25642009910 100644
--- a/llvm/test/MC/AArch64/armv9.6a-occmo.s
+++ b/llvm/test/MC/AArch64/armv9.6a-occmo.s
@@ -1,17 +1,39 @@
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+occmo -mattr=+mte %s | FileCheck %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding %s -mattr=+mte 2>&1 | FileCheck --check-prefix=ERROR %s
-.func:
-// CHECK: .func:
-  dc civaoc, x12
-// CHECK: dc	civaoc, x12                     // encoding: [0x0c,0x7f,0x0b,0xd5]
-// ERROR: error: DC CIVAOC requires: occmo
-  dc cigdvaoc, x0
-// CHECK: dc	cigdvaoc, x0                    // encoding: [0xe0,0x7f,0x0b,0xd5]
-// ERROR: error: DC CIGDVAOC requires: mte, memtag, occmo
-  dc cvaoc, x13
-// CHECK: dc	cvaoc, x13                      // encoding: [0x0d,0x7b,0x0b,0xd5]
-// ERROR: error: DC CVAOC requires: occmo
-  dc cgdvaoc, x1
-// CHECK: dc	cgdvaoc, x1                     // encoding: [0xe1,0x7b,0x0b,0xd5]
-// ERROR: error: DC CGDVAOC requires: mte, memtag, occmo
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+occmo,+mte,+memtag < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+occmo,+mte,+memtag < %s \
+// RUN:        | llvm-objdump -d --mattr=+occmo,+mte,+memtag --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+occmo,+mte,+memtag < %s \
+// RUN:        | llvm-objdump -d --mattr=-occmo,-mte,-memtag --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+occmo,+mte,+memtag < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+occmo,+mte,+memtag -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 
+
+
+dc civaoc, x12
+// CHECK-INST: dc civaoc, x12
+// CHECK-ENCODING: encoding: [0x0c,0x7f,0x0b,0xd5]
+// CHECK-ERROR: error: DC CIVAOC requires: occmo
+// CHECK-UNKNOWN:  d50b7f0c      sys #3, c7, c15, #0, x12
+
+dc cigdvaoc, x0
+// CHECK-INST: dc cigdvaoc, x0
+// CHECK-ENCODING: encoding: [0xe0,0x7f,0x0b,0xd5]
+// CHECK-ERROR: error: DC CIGDVAOC requires: mte, memtag, occmo
+// CHECK-UNKNOWN:  d50b7fe0      sys #3, c7, c15, #7, x0
+
+dc cvaoc, x13
+// CHECK-INST: dc cvaoc, x13
+// CHECK-ENCODING: encoding: [0x0d,0x7b,0x0b,0xd5]
+// CHECK-ERROR: error: DC CVAOC requires: occmo
+// CHECK-UNKNOWN:  d50b7b0d      sys #3, c7, c11, #0, x13
+
+dc cgdvaoc, x1
+// CHECK-INST: dc cgdvaoc, x1
+// CHECK-ENCODING: encoding: [0xe1,0x7b,0x0b,0xd5]
+// CHECK-ERROR: error: DC CGDVAOC requires: mte, memtag, occmo
+// CHECK-UNKNOWN:  d50b7be1      sys #3, c7, c11, #7, x1
diff --git a/llvm/test/MC/AArch64/armv9.6a-pcdphint.s b/llvm/test/MC/AArch64/armv9.6a-pcdphint.s
index 6314e534318c4..8394171740501 100644
--- a/llvm/test/MC/AArch64/armv9.6a-pcdphint.s
+++ b/llvm/test/MC/AArch64/armv9.6a-pcdphint.s
@@ -1,13 +1,25 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding -mattr=+pcdphint %s | FileCheck %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding %s 2>&1 | FileCheck --check-prefix=ERROR %s
-
-.func:
-// CHECK: .func:
-  stshh keep
-// CHECK: stshh	keep                            // encoding: [0x1f,0x96,0x01,0xd5]
-// ERROR: error: instruction requires: pcdphint
-  stshh strm
-// CHECK: stshh	strm                            // encoding: [0x3f,0x96,0x01,0xd5]
-// ERROR: error: instruction requires: pcdphint
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+pcdphint < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+pcdphint < %s \
+// RUN:        | llvm-objdump -d --mattr=+pcdphint - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+pcdphint < %s \
+// RUN:        | llvm-objdump -d --mattr=-pcdphint - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+pcdphint < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+pcdphint -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
 
+stshh keep
+// CHECK-INST: stshh keep
+// CHECK-ENCODING: encoding: [0x1f,0x96,0x01,0xd5]
+// CHECK-ERROR: error: instruction requires: pcdphint
+// CHECK-UNKNOWN:  d501961f      msr S0_1_C9_C6_0, xzr
 
+stshh strm
+// CHECK-INST: stshh strm
+// CHECK-ENCODING: encoding: [0x3f,0x96,0x01,0xd5]
+// CHECK-ERROR: error: instruction requires: pcdphint
+// CHECK-UNKNOWN:  d501963f      msr S0_1_C9_C6_1, xzr
diff --git a/llvm/test/MC/AArch64/armv9.6a-rme-gpc3.s b/llvm/test/MC/AArch64/armv9.6a-rme-gpc3.s
index 093101b6cd812..2a1943ac22a27 100644
--- a/llvm/test/MC/AArch64/armv9.6a-rme-gpc3.s
+++ b/llvm/test/MC/AArch64/armv9.6a-rme-gpc3.s
@@ -1,19 +1,45 @@
-# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
-// RUN: llvm-mc -triple aarch64 -show-encoding %s  | FileCheck %s
-.func:
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
   apas x0
+// CHECK-INST: apas x0
+// CHECK-ENCODING: encoding: [0x00,0x70,0x0e,0xd5]
+// CHECK-UNKNOWN:  d50e7000      apas x0
+
   apas x1
+// CHECK-INST: apas x1
+// CHECK-ENCODING: encoding: [0x01,0x70,0x0e,0xd5]
+// CHECK-UNKNOWN:  d50e7001      apas x1
+
   apas x2
+// CHECK-INST: apas x2
+// CHECK-ENCODING: encoding: [0x02,0x70,0x0e,0xd5]
+// CHECK-UNKNOWN:  d50e7002      apas x2
+
   apas x17
+// CHECK-INST: apas x17
+// CHECK-ENCODING: encoding: [0x11,0x70,0x0e,0xd5]
+// CHECK-UNKNOWN:  d50e7011      apas x17
+
   apas x30
+// CHECK-INST: apas x30
+// CHECK-ENCODING: encoding: [0x1e,0x70,0x0e,0xd5]
+// CHECK-UNKNOWN:  d50e701e      apas x30
+
   mrs x3, GPCBW_EL3
-  msr GPCBW_EL3, x4
+// CHECK-INST: mrs x3, GPCBW_EL3
+// CHECK-ENCODING: encoding: [0xa3,0x21,0x3e,0xd5]
+// CHECK-UNKNOWN:  d53e21a3      mrs x3, GPCBW_EL3
 
-# CHECK:      .func:
-# CHECK-NEXT:	apas    x0                              // encoding: [0x00,0x70,0x0e,0xd5]
-# CHECK-NEXT:	apas    x1                              // encoding: [0x01,0x70,0x0e,0xd5]
-# CHECK-NEXT:	apas    x2                              // encoding: [0x02,0x70,0x0e,0xd5]
-# CHECK-NEXT:	apas    x17                             // encoding: [0x11,0x70,0x0e,0xd5]
-# CHECK-NEXT:	apas    x30                             // encoding: [0x1e,0x70,0x0e,0xd5]
-# CHECK-NEXT: 	mrs	x3, GPCBW_EL3                   // encoding: [0xa3,0x21,0x3e,0xd5]
-# CHECK-NEXT: 	msr	GPCBW_EL3, x4                   // encoding: [0xa4,0x21,0x1e,0xd5]
+  msr GPCBW_EL3, x4
+// CHECK-INST: msr GPCBW_EL3, x4
+// CHECK-ENCODING: encoding: [0xa4,0x21,0x1e,0xd5]
+// CHECK-UNKNOWN:  d51e21a4      msr GPCBW_EL3, x4
diff --git a/llvm/test/MC/AArch64/armv9.6a-srmask.s b/llvm/test/MC/AArch64/armv9.6a-srmask.s
index 40f0e98494d4c..fb91993bad135 100644
--- a/llvm/test/MC/AArch64/armv9.6a-srmask.s
+++ b/llvm/test/MC/AArch64/armv9.6a-srmask.s
@@ -1,102 +1,254 @@
-// RUN: llvm-mc -triple aarch64 -show-encoding %s   | FileCheck %s
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
 
 mrs x3, SCTLRMASK_EL1
-// CHECK: mrs	x3, SCTLRMASK_EL1               // encoding: [0x03,0x14,0x38,0xd5]
+// CHECK-INST: mrs x3, SCTLRMASK_EL1
+// CHECK-ENCODING: encoding: [0x03,0x14,0x38,0xd5]
+// CHECK-UNKNOWN:  d5381403 mrs x3, SCTLRMASK_EL1
+
 mrs x3, SCTLRMASK_EL2
-// CHECK: mrs	x3, SCTLRMASK_EL2               // encoding: [0x03,0x14,0x3c,0xd5]
+// CHECK-INST: mrs x3, SCTLRMASK_EL2
+// CHECK-ENCODING: encoding: [0x03,0x14,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c1403 mrs x3, SCTLRMASK_EL2
+
 mrs x3, SCTLRMASK_EL12
-// CHECK: mrs	x3, SCTLRMASK_EL12              // encoding: [0x03,0x14,0x3d,0xd5]
+// CHECK-INST: mrs x3, SCTLRMASK_EL12
+// CHECK-ENCODING: encoding: [0x03,0x14,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d1403 mrs x3, SCTLRMASK_EL12
+
 mrs x3, CPACRMASK_EL1
-// CHECK: mrs	x3, CPACRMASK_EL1               // encoding: [0x43,0x14,0x38,0xd5]
+// CHECK-INST: mrs x3, CPACRMASK_EL1
+// CHECK-ENCODING: encoding: [0x43,0x14,0x38,0xd5]
+// CHECK-UNKNOWN:  d5381443 mrs x3, CPACRMASK_EL1
+
 mrs x3, CPTRMASK_EL2
-// CHECK: mrs	x3, CPTRMASK_EL2                // encoding: [0x43,0x14,0x3c,0xd5]
+// CHECK-INST: mrs x3, CPTRMASK_EL2
+// CHECK-ENCODING: encoding: [0x43,0x14,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c1443 mrs x3, CPTRMASK_EL2
+
 mrs x3, CPACRMASK_EL12
-// CHECK: mrs	x3, CPACRMASK_EL12              // encoding: [0x43,0x14,0x3d,0xd5]
+// CHECK-INST: mrs x3, CPACRMASK_EL12
+// CHECK-ENCODING: encoding: [0x43,0x14,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d1443 mrs x3, CPACRMASK_EL12
+
 mrs x3, SCTLR2MASK_EL1
-// CHECK: mrs	x3, SCTLR2MASK_EL1              // encoding: [0x63,0x14,0x38,0xd5]
+// CHECK-INST: mrs x3, SCTLR2MASK_EL1
+// CHECK-ENCODING: encoding: [0x63,0x14,0x38,0xd5]
+// CHECK-UNKNOWN:  d5381463 mrs x3, SCTLR2MASK_EL1
+
 mrs x3, SCTLR2MASK_EL2
-// CHECK: mrs	x3, SCTLR2MASK_EL2              // encoding: [0x63,0x14,0x3c,0xd5]
+// CHECK-INST: mrs x3, SCTLR2MASK_EL2
+// CHECK-ENCODING: encoding: [0x63,0x14,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c1463 mrs x3, SCTLR2MASK_EL2
+
 mrs x3, SCTLR2MASK_EL12
-// CHECK: mrs	x3, SCTLR2MASK_EL12             // encoding: [0x63,0x14,0x3d,0xd5]
+// CHECK-INST: mrs x3, SCTLR2MASK_EL12
+// CHECK-ENCODING: encoding: [0x63,0x14,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d1463 mrs x3, SCTLR2MASK_EL12
+
 mrs x3, CPACRALIAS_EL1
-// CHECK: mrs	x3, CPACRALIAS_EL1              // encoding: [0x83,0x14,0x38,0xd5]
+// CHECK-INST: mrs x3, CPACRALIAS_EL1
+// CHECK-ENCODING: encoding: [0x83,0x14,0x38,0xd5]
+// CHECK-UNKNOWN:  d5381483 mrs x3, CPACRALIAS_EL1
+
 mrs x3, SCTLRALIAS_EL1
-// CHECK: mrs	x3, SCTLRALIAS_EL1              // encoding: [0xc3,0x14,0x38,0xd5]
+// CHECK-INST: mrs x3, SCTLRALIAS_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x14,0x38,0xd5]
+// CHECK-UNKNOWN:  d53814c3 mrs x3, SCTLRALIAS_EL1
+
 mrs x3, SCTLR2ALIAS_EL1
-// CHECK: mrs	x3, SCTLR2ALIAS_EL1             // encoding: [0xe3,0x14,0x38,0xd5]
+// CHECK-INST: mrs x3, SCTLR2ALIAS_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x14,0x38,0xd5]
+// CHECK-UNKNOWN:  d53814e3 mrs x3, SCTLR2ALIAS_EL1
+
 mrs x3, TCRMASK_EL1
-// CHECK: mrs	x3, TCRMASK_EL1                 // encoding: [0x43,0x27,0x38,0xd5]
+// CHECK-INST: mrs x3, TCRMASK_EL1
+// CHECK-ENCODING: encoding: [0x43,0x27,0x38,0xd5]
+// CHECK-UNKNOWN:  d5382743 mrs x3, TCRMASK_EL1
+
 mrs x3, TCRMASK_EL2
-// CHECK: mrs	x3, TCRMASK_EL2                 // encoding: [0x43,0x27,0x3c,0xd5]
+// CHECK-INST: mrs x3, TCRMASK_EL2
+// CHECK-ENCODING: encoding: [0x43,0x27,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c2743 mrs x3, TCRMASK_EL2
+
 mrs x3, TCRMASK_EL12
-// CHECK: mrs	x3, TCRMASK_EL12                // encoding: [0x43,0x27,0x3d,0xd5]
+// CHECK-INST: mrs x3, TCRMASK_EL12
+// CHECK-ENCODING: encoding: [0x43,0x27,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d2743 mrs x3, TCRMASK_EL12
+
 mrs x3, TCR2MASK_EL1
-// CHECK: mrs	x3, TCR2MASK_EL1                // encoding: [0x63,0x27,0x38,0xd5]
+// CHECK-INST: mrs x3, TCR2MASK_EL1
+// CHECK-ENCODING: encoding: [0x63,0x27,0x38,0xd5]
+// CHECK-UNKNOWN:  d5382763 mrs x3, TCR2MASK_EL1
+
 mrs x3, TCR2MASK_EL2
-// CHECK: mrs	x3, TCR2MASK_EL2                // encoding: [0x63,0x27,0x3c,0xd5]
+// CHECK-INST: mrs x3, TCR2MASK_EL2
+// CHECK-ENCODING: encoding: [0x63,0x27,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c2763 mrs x3, TCR2MASK_EL2
+
 mrs x3, TCR2MASK_EL12
-// CHECK: mrs	x3, TCR2MASK_EL12               // encoding: [0x63,0x27,0x3d,0xd5]
+// CHECK-INST: mrs x3, TCR2MASK_EL12
+// CHECK-ENCODING: encoding: [0x63,0x27,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d2763 mrs x3, TCR2MASK_EL12
+
 mrs x3, TCRALIAS_EL1
-// CHECK: mrs	x3, TCRALIAS_EL1                // encoding: [0xc3,0x27,0x38,0xd5]
+// CHECK-INST: mrs x3, TCRALIAS_EL1
+// CHECK-ENCODING: encoding: [0xc3,0x27,0x38,0xd5]
+// CHECK-UNKNOWN:  d53827c3 mrs x3, TCRALIAS_EL1
+
 mrs x3, TCR2ALIAS_EL1
-// CHECK: mrs	x3, TCR2ALIAS_EL1               // encoding: [0xe3,0x27,0x38,0xd5]
+// CHECK-INST: mrs x3, TCR2ALIAS_EL1
+// CHECK-ENCODING: encoding: [0xe3,0x27,0x38,0xd5]
+// CHECK-UNKNOWN:  d53827e3 mrs x3, TCR2ALIAS_EL1
+
 mrs x3, ACTLRMASK_EL1
-// CHECK: mrs	x3, ACTLRMASK_EL1               // encoding: [0x23,0x14,0x38,0xd5]
+// CHECK-INST: mrs x3, ACTLRMASK_EL1
+// CHECK-ENCODING: encoding: [0x23,0x14,0x38,0xd5]
+// CHECK-UNKNOWN:  d5381423 mrs x3, ACTLRMASK_EL1
+
 mrs x3, ACTLRMASK_EL2
-// CHECK: mrs	x3, ACTLRMASK_EL2               // encoding: [0x23,0x14,0x3c,0xd5]
+// CHECK-INST: mrs x3, ACTLRMASK_EL2
+// CHECK-ENCODING: encoding: [0x23,0x14,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c1423 mrs x3, ACTLRMASK_EL2
+
 mrs x3, ACTLRMASK_EL12
-// CHECK: mrs	x3, ACTLRMASK_EL12              // encoding: [0x23,0x14,0x3d,0xd5]
+// CHECK-INST: mrs x3, ACTLRMASK_EL12
+// CHECK-ENCODING: encoding: [0x23,0x14,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d1423 mrs x3, ACTLRMASK_EL12
+
 mrs x3, ACTLRALIAS_EL1
-// CHECK: mrs	x3, ACTLRALIAS_EL1              // encoding: [0xa3,0x14,0x38,0xd5]
+// CHECK-INST: mrs x3, ACTLRALIAS_EL1
+// CHECK-ENCODING: encoding: [0xa3,0x14,0x38,0xd5]
+// CHECK-UNKNOWN:  d53814a3 mrs x3, ACTLRALIAS_EL1
 
 msr SCTLRMASK_EL1, x3
-// CHECK: msr	SCTLRMASK_EL1, x3               // encoding: [0x03,0x14,0x18,0xd5]
+// CHECK-INST: msr SCTLRMASK_EL1, x3
+// CHECK-ENCODING: encoding: [0x03,0x14,0x18,0xd5]
+// CHECK-UNKNOWN:  d5181403 msr SCTLRMASK_EL1, x3
+
 msr SCTLRMASK_EL2, x3
-// CHECK: msr	SCTLRMASK_EL2, x3               // encoding: [0x03,0x14,0x1c,0xd5]
+// CHECK-INST: msr SCTLRMASK_EL2, x3
+// CHECK-ENCODING: encoding: [0x03,0x14,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c1403 msr SCTLRMASK_EL2, x3
+
 msr SCTLRMASK_EL12, x3
-// CHECK: msr	SCTLRMASK_EL12, x3              // encoding: [0x03,0x14,0x1d,0xd5]
+// CHECK-INST: msr SCTLRMASK_EL12, x3
+// CHECK-ENCODING: encoding: [0x03,0x14,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d1403 msr SCTLRMASK_EL12, x3
+
 msr CPACRMASK_EL1, x3
-// CHECK: msr	CPACRMASK_EL1, x3               // encoding: [0x43,0x14,0x18,0xd5]
+// CHECK-INST: msr CPACRMASK_EL1, x3
+// CHECK-ENCODING: encoding: [0x43,0x14,0x18,0xd5]
+// CHECK-UNKNOWN:  d5181443 msr CPACRMASK_EL1, x3
+
 msr CPTRMASK_EL2, x3
-// CHECK: msr	CPTRMASK_EL2, x3                // encoding: [0x43,0x14,0x1c,0xd5]
+// CHECK-INST: msr CPTRMASK_EL2, x3
+// CHECK-ENCODING: encoding: [0x43,0x14,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c1443 msr CPTRMASK_EL2, x3
+
 msr CPACRMASK_EL12, x3
-// CHECK: msr	CPACRMASK_EL12, x3              // encoding: [0x43,0x14,0x1d,0xd5]
+// CHECK-INST: msr CPACRMASK_EL12, x3
+// CHECK-ENCODING: encoding: [0x43,0x14,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d1443 msr CPACRMASK_EL12, x3
+
 msr SCTLR2MASK_EL1, x3
-// CHECK: msr	SCTLR2MASK_EL1, x3              // encoding: [0x63,0x14,0x18,0xd5]
+// CHECK-INST: msr SCTLR2MASK_EL1, x3
+// CHECK-ENCODING: encoding: [0x63,0x14,0x18,0xd5]
+// CHECK-UNKNOWN:  d5181463 msr SCTLR2MASK_EL1, x3
+
 msr SCTLR2MASK_EL2, x3
-// CHECK: msr	SCTLR2MASK_EL2, x3              // encoding: [0x63,0x14,0x1c,0xd5]
+// CHECK-INST: msr SCTLR2MASK_EL2, x3
+// CHECK-ENCODING: encoding: [0x63,0x14,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c1463 msr SCTLR2MASK_EL2, x3
+
 msr SCTLR2MASK_EL12, x3
-// CHECK: msr	SCTLR2MASK_EL12, x3             // encoding: [0x63,0x14,0x1d,0xd5]
+// CHECK-INST: msr SCTLR2MASK_EL12, x3
+// CHECK-ENCODING: encoding: [0x63,0x14,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d1463 msr SCTLR2MASK_EL12, x3
+
 msr CPACRALIAS_EL1, x3
-// CHECK: msr	CPACRALIAS_EL1, x3              // encoding: [0x83,0x14,0x18,0xd5]
+// CHECK-INST: msr CPACRALIAS_EL1, x3
+// CHECK-ENCODING: encoding: [0x83,0x14,0x18,0xd5]
+// CHECK-UNKNOWN:  d5181483 msr CPACRALIAS_EL1, x3
+
 msr SCTLRALIAS_EL1, x3
-// CHECK: msr	SCTLRALIAS_EL1, x3              // encoding: [0xc3,0x14,0x18,0xd5]
+// CHECK-INST: msr SCTLRALIAS_EL1, x3
+// CHECK-ENCODING: encoding: [0xc3,0x14,0x18,0xd5]
+// CHECK-UNKNOWN:  d51814c3 msr SCTLRALIAS_EL1, x3
+
 msr SCTLR2ALIAS_EL1, x3
-// CHECK: msr	SCTLR2ALIAS_EL1, x3             // encoding: [0xe3,0x14,0x18,0xd5]
+// CHECK-INST: msr SCTLR2ALIAS_EL1, x3
+// CHECK-ENCODING: encoding: [0xe3,0x14,0x18,0xd5]
+// CHECK-UNKNOWN:  d51814e3 msr SCTLR2ALIAS_EL1, x3
+
 msr TCRMASK_EL1, x3
-// CHECK: msr	TCRMASK_EL1, x3                 // encoding: [0x43,0x27,0x18,0xd5]
+// CHECK-INST: msr TCRMASK_EL1, x3
+// CHECK-ENCODING: encoding: [0x43,0x27,0x18,0xd5]
+// CHECK-UNKNOWN:  d5182743 msr TCRMASK_EL1, x3
+
 msr TCRMASK_EL2, x3
-// CHECK: msr	TCRMASK_EL2, x3                 // encoding: [0x43,0x27,0x1c,0xd5]
+// CHECK-INST: msr TCRMASK_EL2, x3
+// CHECK-ENCODING: encoding: [0x43,0x27,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c2743 msr TCRMASK_EL2, x3
+
 msr TCRMASK_EL12, x3
-// CHECK: msr	TCRMASK_EL12, x3                // encoding: [0x43,0x27,0x1d,0xd5]
+// CHECK-INST: msr TCRMASK_EL12, x3
+// CHECK-ENCODING: encoding: [0x43,0x27,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d2743 msr TCRMASK_EL12, x3
+
 msr TCR2MASK_EL1, x3
-// CHECK: msr	TCR2MASK_EL1, x3                // encoding: [0x63,0x27,0x18,0xd5]
+// CHECK-INST: msr TCR2MASK_EL1, x3
+// CHECK-ENCODING: encoding: [0x63,0x27,0x18,0xd5]
+// CHECK-UNKNOWN:  d5182763 msr TCR2MASK_EL1, x3
+
 msr TCR2MASK_EL2, x3
-// CHECK: msr	TCR2MASK_EL2, x3                // encoding: [0x63,0x27,0x1c,0xd5]
+// CHECK-INST: msr TCR2MASK_EL2, x3
+// CHECK-ENCODING: encoding: [0x63,0x27,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c2763 msr TCR2MASK_EL2, x3
+
 msr TCR2MASK_EL12, x3
-// CHECK: msr	TCR2MASK_EL12, x3               // encoding: [0x63,0x27,0x1d,0xd5]
+// CHECK-INST: msr TCR2MASK_EL12, x3
+// CHECK-ENCODING: encoding: [0x63,0x27,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d2763 msr TCR2MASK_EL12, x3
+
 msr TCRALIAS_EL1, x3
-// CHECK: msr	TCRALIAS_EL1, x3                // encoding: [0xc3,0x27,0x18,0xd5]
+// CHECK-INST: msr TCRALIAS_EL1, x3
+// CHECK-ENCODING: encoding: [0xc3,0x27,0x18,0xd5]
+// CHECK-UNKNOWN:  d51827c3 msr TCRALIAS_EL1, x3
+
 msr TCR2ALIAS_EL1, x3
-// CHECK: msr	TCR2ALIAS_EL1, x3               // encoding: [0xe3,0x27,0x18,0xd5]
+// CHECK-INST: msr TCR2ALIAS_EL1, x3
+// CHECK-ENCODING: encoding: [0xe3,0x27,0x18,0xd5]
+// CHECK-UNKNOWN:  d51827e3 msr TCR2ALIAS_EL1, x3
+
 msr ACTLRMASK_EL1, x3
-// CHECK: msr	ACTLRMASK_EL1, x3               // encoding: [0x23,0x14,0x18,0xd5]
+// CHECK-INST: msr ACTLRMASK_EL1, x3
+// CHECK-ENCODING: encoding: [0x23,0x14,0x18,0xd5]
+// CHECK-UNKNOWN:  d5181423 msr ACTLRMASK_EL1, x3
+
 msr ACTLRMASK_EL2, x3
-// CHECK: msr	ACTLRMASK_EL2, x3               // encoding: [0x23,0x14,0x1c,0xd5]
+// CHECK-INST: msr ACTLRMASK_EL2, x3
+// CHECK-ENCODING: encoding: [0x23,0x14,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c1423 msr ACTLRMASK_EL2, x3
+
 msr ACTLRMASK_EL12, x3
-// CHECK: msr	ACTLRMASK_EL12, x3              // encoding: [0x23,0x14,0x1d,0xd5]
+// CHECK-INST: msr ACTLRMASK_EL12, x3
+// CHECK-ENCODING: encoding: [0x23,0x14,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d1423 msr ACTLRMASK_EL12, x3
+
 msr ACTLRALIAS_EL1, x3
-// CHECK: msr	ACTLRALIAS_EL1, x3              // encoding: [0xa3,0x14,0x18,0xd5]
+// CHECK-INST: msr ACTLRALIAS_EL1, x3
+// CHECK-ENCODING: encoding: [0xa3,0x14,0x18,0xd5]
+// CHECK-UNKNOWN:  d51814a3 msr ACTLRALIAS_EL1, x3
 
 
 
diff --git a/llvm/test/MC/AArch64/armv9.6a-statistical-profiling.s b/llvm/test/MC/AArch64/armv9.6a-statistical-profiling.s
index 2314c414d4d96..4ef6367c62ea0 100644
--- a/llvm/test/MC/AArch64/armv9.6a-statistical-profiling.s
+++ b/llvm/test/MC/AArch64/armv9.6a-statistical-profiling.s
@@ -1,19 +1,51 @@
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding     < %s 2> %t | FileCheck %s
-
-  msr pmbmar_el1, x0
-  msr pmbsr_el12, x0
-  msr pmbsr_el2, x0
-  msr pmbsr_el3, x0
-// CHECK:     msr PMBMAR_EL1, x0          // encoding: [0xa0,0x9a,0x18,0xd5]
-// CHECK:     msr PMBSR_EL12, x0          // encoding: [0x60,0x9a,0x1d,0xd5]
-// CHECK:     msr PMBSR_EL2, x0           // encoding: [0x60,0x9a,0x1c,0xd5]
-// CHECK:     msr PMBSR_EL3, x0           // encoding: [0x60,0x9a,0x1e,0xd5]
-
-  mrs x0, pmbmar_el1
-  mrs x0, pmbsr_el12
-  mrs x0, pmbsr_el2
-  mrs x0, pmbsr_el3
-// CHECK:    mrs x0, PMBMAR_EL1          // encoding: [0xa0,0x9a,0x38,0xd5]
-// CHECK:    mrs x0, PMBSR_EL12          // encoding: [0x60,0x9a,0x3d,0xd5]
-// CHECK:    mrs x0, PMBSR_EL2           // encoding: [0x60,0x9a,0x3c,0xd5]
-// CHECK:    mrs x0, PMBSR_EL3           // encoding: [0x60,0x9a,0x3e,0xd5]
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+
+msr pmbmar_el1, x0
+// CHECK-INST: msr PMBMAR_EL1, x0
+// CHECK-ENCODING: encoding: [0xa0,0x9a,0x18,0xd5]
+// CHECK-UNKNOWN:  d5189aa0 msr PMBMAR_EL1, x0
+
+msr pmbsr_el12, x0
+// CHECK-INST: msr PMBSR_EL12, x0
+// CHECK-ENCODING: encoding: [0x60,0x9a,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d9a60 msr PMBSR_EL12, x0
+
+msr pmbsr_el2, x0
+// CHECK-INST: msr PMBSR_EL2, x0
+// CHECK-ENCODING: encoding: [0x60,0x9a,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c9a60 msr PMBSR_EL2, x0
+
+msr pmbsr_el3, x0
+// CHECK-INST: msr PMBSR_EL3, x0
+// CHECK-ENCODING: encoding: [0x60,0x9a,0x1e,0xd5]
+// CHECK-UNKNOWN:  d51e9a60 msr PMBSR_EL3, x0
+
+mrs x0, pmbmar_el1
+// CHECK-INST: mrs x0, PMBMAR_EL1
+// CHECK-ENCODING: encoding: [0xa0,0x9a,0x38,0xd5]
+// CHECK-UNKNOWN:  d5389aa0 mrs x0, PMBMAR_EL1
+
+mrs x0, pmbsr_el12
+// CHECK-INST: mrs x0, PMBSR_EL12
+// CHECK-ENCODING: encoding: [0x60,0x9a,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d9a60 mrs x0, PMBSR_EL12
+
+mrs x0, pmbsr_el2
+// CHECK-INST: mrs x0, PMBSR_EL2
+// CHECK-ENCODING: encoding: [0x60,0x9a,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c9a60 mrs x0, PMBSR_EL2
+
+mrs x0, pmbsr_el3
+// CHECK-INST: mrs x0, PMBSR_EL3
+// CHECK-ENCODING: encoding: [0x60,0x9a,0x3e,0xd5]
+// CHECK-UNKNOWN:  d53e9a60 mrs x0, PMBSR_EL3
diff --git a/llvm/test/MC/AArch64/armv9.6a-trbe-exception.s b/llvm/test/MC/AArch64/armv9.6a-trbe-exception.s
index a8ba7c442e4ce..fb795aa8c64ca 100644
--- a/llvm/test/MC/AArch64/armv9.6a-trbe-exception.s
+++ b/llvm/test/MC/AArch64/armv9.6a-trbe-exception.s
@@ -1,15 +1,41 @@
-// RUN: llvm-mc -triple aarch64-none-linux-gnu -show-encoding      < %s 2> %t | FileCheck %s
-
-  msr trbsr_el12, x0
-  msr trbsr_el2, x0
-  msr trbsr_el3, x0
-// CHECK:     msr TRBSR_EL12, x0          // encoding: [0x60,0x9b,0x1d,0xd5]
-// CHECK:     msr TRBSR_EL2, x0           // encoding: [0x60,0x9b,0x1c,0xd5]
-// CHECK:     msr TRBSR_EL3, x0           // encoding: [0x60,0x9b,0x1e,0xd5]
-
-  mrs x0, trbsr_el12
-  mrs x0, trbsr_el2
-  mrs x0, trbsr_el3
-// CHECK:    mrs x0, TRBSR_EL12          // encoding: [0x60,0x9b,0x3d,0xd5]
-// CHECK:    mrs x0, TRBSR_EL2           // encoding: [0x60,0x9b,0x3c,0xd5]
-// CHECK:    mrs x0, TRBSR_EL3           // encoding: [0x60,0x9b,0x3e,0xd5]
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+
+msr trbsr_el12, x0
+// CHECK-INST: msr TRBSR_EL12, x0
+// CHECK-ENCODING: encoding: [0x60,0x9b,0x1d,0xd5]
+// CHECK-UNKNOWN:  d51d9b60 msr TRBSR_EL12, x0
+
+msr trbsr_el2, x0
+// CHECK-INST: msr TRBSR_EL2, x0
+// CHECK-ENCODING: encoding: [0x60,0x9b,0x1c,0xd5]
+// CHECK-UNKNOWN:  d51c9b60 msr TRBSR_EL2, x0
+
+msr trbsr_el3, x0
+// CHECK-INST: msr TRBSR_EL3, x0
+// CHECK-ENCODING: encoding: [0x60,0x9b,0x1e,0xd5]
+// CHECK-UNKNOWN:  d51e9b60 msr TRBSR_EL3, x0
+
+mrs x0, trbsr_el12
+// CHECK-INST: mrs x0, TRBSR_EL12
+// CHECK-ENCODING: encoding: [0x60,0x9b,0x3d,0xd5]
+// CHECK-UNKNOWN:  d53d9b60 mrs x0, TRBSR_EL12
+
+mrs x0, trbsr_el2
+// CHECK-INST: mrs x0, TRBSR_EL2
+// CHECK-ENCODING: encoding: [0x60,0x9b,0x3c,0xd5]
+// CHECK-UNKNOWN:  d53c9b60 mrs x0, TRBSR_EL2
+
+mrs x0, trbsr_el3
+// CHECK-INST: mrs x0, TRBSR_EL3
+// CHECK-ENCODING: encoding: [0x60,0x9b,0x3e,0xd5]
+// CHECK-UNKNOWN:  d53e9b60 mrs x0, TRBSR_EL3
diff --git a/llvm/test/MC/AArch64/armv9a-sysp.s b/llvm/test/MC/AArch64/armv9a-sysp.s
new file mode 100644
index 0000000000000..600657595e8aa
--- /dev/null
+++ b/llvm/test/MC/AArch64/armv9a-sysp.s
@@ -0,0 +1,978 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+d128,+tlb-rmi,+xs < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+tlb-rmi,+xs < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+d128,+tlb-rmi,+xs < %s \
+// RUN:        | llvm-objdump -d --mattr=+d128,+tlb-rmi,+xs --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+d128,+tlb-rmi,+xs < %s \
+// RUN:   | llvm-objdump -d --mattr=-d128,+tlb-rmi,+xs --no-print-imm-hex - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+// Disassemble encoding and check the re-encoding (-show-encoding) matches.
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+d128,+tlb-rmi,+xs < %s \
+// RUN:        | sed '/.text/d' | sed 's/.*encoding: //g' \
+// RUN:        | llvm-mc -triple=aarch64 -mattr=+d128,+tlb-rmi,+xs -disassemble -show-encoding \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+
+
+// +tbl-rmi required for RIPA*/RVA*
+// +xs required for *NXS
+
+// sysp #<op1>, <Cn>, <Cm>, #<op2>{, <Xt1>, <Xt2>}
+// registers with 128-bit formats (op0, op1, Cn, Cm, op2)
+// For sysp, op0 is 0
+
+sysp #0, c2, c0, #0, x0, x1// TTBR0_EL1     3  0  2  0  0
+// CHECK-INST: sysp #0, c2, c0, #0, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482000      <unknown>
+
+sysp #0, c2, c0, #1, x0, x1// TTBR1_EL1     3  0  2  0  1
+// CHECK-INST: sysp #0, c2, c0, #1, x0, x1
+// CHECK-ENCODING: encoding: [0x20,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482020      <unknown>
+
+sysp #0, c7, c4, #0, x0, x1// PAR_EL1       3  0  7  4  0
+// CHECK-INST: sysp #0, c7, c4, #0, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x74,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5487400      <unknown>
+
+sysp #0, c13, c0, #3, x0, x1         // RCWSMASK_EL1  3  0 13  0  3
+// CHECK-INST: sysp #0, c13, c0, #3, x0, x1
+// CHECK-ENCODING: encoding: [0x60,0xd0,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548d060      <unknown>
+
+sysp #0, c13, c0, #6, x0, x1         // RCWMASK_EL1   3  0 13  0  6
+// CHECK-INST: sysp #0, c13, c0, #6, x0, x1
+// CHECK-ENCODING: encoding: [0xc0,0xd0,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548d0c0      <unknown>
+
+sysp #4, c2, c0, #0, x0, x1// TTBR0_EL2     3  4  2  0  0
+// CHECK-INST: sysp #4, c2, c0, #0, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c2000      <unknown>
+
+sysp #4, c2, c0, #1, x0, x1// TTBR1_EL2     3  4  2  0  1
+// CHECK-INST: sysp #4, c2, c0, #1, x0, x1
+// CHECK-ENCODING: encoding: [0x20,0x20,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c2020      <unknown>
+
+sysp #4, c2, c1, #0, x0, x1// VTTBR_EL2     3  4  2  1  0
+// CHECK-INST: sysp #4, c2, c1, #0, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x21,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c2100      <unknown>
+
+
+
+sysp #0, c2, c0, #0, x0, x1
+// CHECK-INST: sysp #0, c2, c0, #0, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482000      <unknown>
+
+sysp #0, c2, c0, #1, x0, x1
+// CHECK-INST: sysp #0, c2, c0, #1, x0, x1
+// CHECK-ENCODING: encoding: [0x20,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482020      <unknown>
+
+sysp #0, c7, c4, #0, x0, x1
+// CHECK-INST: sysp #0, c7, c4, #0, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x74,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5487400      <unknown>
+
+sysp #0, c13, c0, #3, x0, x1
+// CHECK-INST: sysp #0, c13, c0, #3, x0, x1
+// CHECK-ENCODING: encoding: [0x60,0xd0,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548d060      <unknown>
+
+sysp #0, c13, c0, #6, x0, x1
+// CHECK-INST: sysp #0, c13, c0, #6, x0, x1
+// CHECK-ENCODING: encoding: [0xc0,0xd0,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548d0c0      <unknown>
+
+sysp #4, c2, c0, #0, x0, x1
+// CHECK-INST: sysp #4, c2, c0, #0, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c2000      <unknown>
+
+sysp #4, c2, c0, #1, x0, x1
+// CHECK-INST: sysp #4, c2, c0, #1, x0, x1
+// CHECK-ENCODING: encoding: [0x20,0x20,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c2020      <unknown>
+
+sysp #4, c2, c1, #0, x0, x1
+// CHECK-INST: sysp #4, c2, c1, #0, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x21,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c2100      <unknown>
+
+sysp #0, c2, c0, #0, x0, x1
+// CHECK-INST: sysp #0, c2, c0, #0, x0, x1
+// CHECK-ENCODING: encoding: [0x00,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482000      <unknown>
+
+sysp #0, c2, c0, #0, x2, x3
+// CHECK-INST: sysp #0, c2, c0, #0, x2, x3
+// CHECK-ENCODING: encoding: [0x02,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482002      <unknown>
+
+sysp #0, c2, c0, #0, x4, x5
+// CHECK-INST: sysp #0, c2, c0, #0, x4, x5
+// CHECK-ENCODING: encoding: [0x04,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482004      <unknown>
+
+sysp #0, c2, c0, #0, x6, x7
+// CHECK-INST: sysp #0, c2, c0, #0, x6, x7
+// CHECK-ENCODING: encoding: [0x06,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482006      <unknown>
+
+sysp #0, c2, c0, #0, x8, x9
+// CHECK-INST: sysp #0, c2, c0, #0, x8, x9
+// CHECK-ENCODING: encoding: [0x08,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482008      <unknown>
+
+sysp #0, c2, c0, #0, x10, x11
+// CHECK-INST: sysp #0, c2, c0, #0, x10, x11
+// CHECK-ENCODING: encoding: [0x0a,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548200a      <unknown>
+
+sysp #0, c2, c0, #0, x12, x13
+// CHECK-INST: sysp #0, c2, c0, #0, x12, x13
+// CHECK-ENCODING: encoding: [0x0c,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548200c      <unknown>
+
+sysp #0, c2, c0, #0, x14, x15
+// CHECK-INST: sysp #0, c2, c0, #0, x14, x15
+// CHECK-ENCODING: encoding: [0x0e,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548200e      <unknown>
+
+sysp #0, c2, c0, #0, x16, x17
+// CHECK-INST: sysp #0, c2, c0, #0, x16, x17
+// CHECK-ENCODING: encoding: [0x10,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482010      <unknown>
+
+sysp #0, c2, c0, #0, x18, x19
+// CHECK-INST: sysp #0, c2, c0, #0, x18, x19
+// CHECK-ENCODING: encoding: [0x12,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482012      <unknown>
+
+sysp #0, c2, c0, #0, x20, x21
+// CHECK-INST: sysp #0, c2, c0, #0, x20, x21
+// CHECK-ENCODING: encoding: [0x14,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482014      <unknown>
+
+sysp #0, c2, c0, #0, x22, x23
+// CHECK-INST: sysp #0, c2, c0, #0, x22, x23
+// CHECK-ENCODING: encoding: [0x16,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482016      <unknown>
+
+sysp #0, c2, c0, #0, x24, x25
+// CHECK-INST: sysp #0, c2, c0, #0, x24, x25
+// CHECK-ENCODING: encoding: [0x18,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5482018      <unknown>
+
+sysp #0, c2, c0, #0, x26, x27
+// CHECK-INST: sysp #0, c2, c0, #0, x26, x27
+// CHECK-ENCODING: encoding: [0x1a,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548201a      <unknown>
+
+sysp #0, c2, c0, #0, x28, x29
+// CHECK-INST: sysp #0, c2, c0, #0, x28, x29
+// CHECK-ENCODING: encoding: [0x1c,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548201c      <unknown>
+
+sysp #0, c2, c0, #0, x30, x31
+// CHECK-INST: sysp #0, c2, c0, #0, x30, xzr
+// CHECK-ENCODING: encoding: [0x1e,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548201e      <unknown>
+
+
+sysp #0, c2, c0, #0, x31, x31
+// CHECK-INST: sysp #0, c2, c0, #0
+// CHECK-ENCODING: encoding: [0x1f,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548201f      <unknown>
+
+sysp #0, c2, c0, #0, xzr, xzr
+// CHECK-INST: sysp #0, c2, c0, #0
+// CHECK-ENCODING: encoding: [0x1f,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548201f      <unknown>
+
+sysp #0, c2, c0, #0, x31, xzr
+// CHECK-INST: sysp #0, c2, c0, #0
+// CHECK-ENCODING: encoding: [0x1f,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548201f      <unknown>
+
+sysp #0, c2, c0, #0, xzr, x31
+// CHECK-INST: sysp #0, c2, c0, #0
+// CHECK-ENCODING: encoding: [0x1f,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548201f      <unknown>
+
+sysp #0, c2, c0, #0
+// CHECK-INST: sysp #0, c2, c0, #0
+// CHECK-ENCODING: encoding: [0x1f,0x20,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d548201f      <unknown>
+
+tlbip IPAS2E1, x4, x5
+// CHECK-INST: tlbip ipas2e1, x4, x5
+// CHECK-ENCODING: encoding: [0x24,0x84,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c8424      <unknown>
+
+tlbip IPAS2E1NXS, x4, x5
+// CHECK-INST: tlbip ipas2e1nxs, x4, x5
+// CHECK-ENCODING: encoding: [0x24,0x94,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c9424      <unknown>
+
+tlbip IPAS2E1IS, x4, x5
+// CHECK-INST: tlbip ipas2e1is, x4, x5
+// CHECK-ENCODING: encoding: [0x24,0x80,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c8024      <unknown>
+
+tlbip IPAS2E1ISNXS, x4, x5
+// CHECK-INST: tlbip ipas2e1isnxs, x4, x5
+// CHECK-ENCODING: encoding: [0x24,0x90,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c9024      <unknown>
+
+tlbip IPAS2E1OS, x4, x5
+// CHECK-INST: tlbip ipas2e1os, x4, x5
+// CHECK-ENCODING: encoding: [0x04,0x84,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c8404      <unknown>
+
+tlbip IPAS2E1OSNXS, x4, x5
+// CHECK-INST: tlbip ipas2e1osnxs, x4, x5
+// CHECK-ENCODING: encoding: [0x04,0x94,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c9404      <unknown>
+
+tlbip IPAS2LE1, x4, x5
+// CHECK-INST: tlbip ipas2le1, x4, x5
+// CHECK-ENCODING: encoding: [0xa4,0x84,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c84a4      <unknown>
+
+tlbip IPAS2LE1NXS, x4, x5
+// CHECK-INST: tlbip ipas2le1nxs, x4, x5
+// CHECK-ENCODING: encoding: [0xa4,0x94,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c94a4      <unknown>
+
+tlbip IPAS2LE1IS, x4, x5
+// CHECK-INST: tlbip ipas2le1is, x4, x5
+// CHECK-ENCODING: encoding: [0xa4,0x80,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c80a4      <unknown>
+
+tlbip IPAS2LE1ISNXS, x4, x5
+// CHECK-INST: tlbip ipas2le1isnxs, x4, x5
+// CHECK-ENCODING: encoding: [0xa4,0x90,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c90a4      <unknown>
+
+tlbip IPAS2LE1OS, x4, x5
+// CHECK-INST: tlbip ipas2le1os, x4, x5
+// CHECK-ENCODING: encoding: [0x84,0x84,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c8484      <unknown>
+
+tlbip IPAS2LE1OSNXS, x4, x5
+// CHECK-INST: tlbip ipas2le1osnxs, x4, x5
+// CHECK-ENCODING: encoding: [0x84,0x94,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c9484      <unknown>
+
+tlbip VAE1, x8, x9
+// CHECK-INST: tlbip vae1, x8, x9
+// CHECK-ENCODING: encoding: [0x28,0x87,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488728      <unknown>
+
+tlbip VAE1NXS, x8, x9
+// CHECK-INST: tlbip vae1nxs, x8, x9
+// CHECK-ENCODING: encoding: [0x28,0x97,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489728      <unknown>
+
+tlbip VAE1IS, x8, x9
+// CHECK-INST: tlbip vae1is, x8, x9
+// CHECK-ENCODING: encoding: [0x28,0x83,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488328      <unknown>
+
+tlbip VAE1ISNXS, x8, x9
+// CHECK-INST: tlbip vae1isnxs, x8, x9
+// CHECK-ENCODING: encoding: [0x28,0x93,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489328      <unknown>
+
+tlbip VAE1OS, x8, x9
+// CHECK-INST: tlbip vae1os, x8, x9
+// CHECK-ENCODING: encoding: [0x28,0x81,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488128      <unknown>
+
+tlbip VAE1OSNXS, x8, x9
+// CHECK-INST: tlbip vae1osnxs, x8, x9
+// CHECK-ENCODING: encoding: [0x28,0x91,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489128      <unknown>
+
+tlbip VALE1, x8, x9
+// CHECK-INST: tlbip vale1, x8, x9
+// CHECK-ENCODING: encoding: [0xa8,0x87,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54887a8      <unknown>
+
+tlbip VALE1NXS, x8, x9
+// CHECK-INST: tlbip vale1nxs, x8, x9
+// CHECK-ENCODING: encoding: [0xa8,0x97,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54897a8      <unknown>
+
+tlbip VALE1IS, x8, x9
+// CHECK-INST: tlbip vale1is, x8, x9
+// CHECK-ENCODING: encoding: [0xa8,0x83,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54883a8      <unknown>
+
+tlbip VALE1ISNXS, x8, x9
+// CHECK-INST: tlbip vale1isnxs, x8, x9
+// CHECK-ENCODING: encoding: [0xa8,0x93,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54893a8      <unknown>
+
+tlbip VALE1OS, x8, x9
+// CHECK-INST: tlbip vale1os, x8, x9
+// CHECK-ENCODING: encoding: [0xa8,0x81,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54881a8      <unknown>
+
+tlbip VALE1OSNXS, x8, x9
+// CHECK-INST: tlbip vale1osnxs, x8, x9
+// CHECK-ENCODING: encoding: [0xa8,0x91,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54891a8      <unknown>
+
+tlbip VAAE1, x8, x9
+// CHECK-INST: tlbip vaae1, x8, x9
+// CHECK-ENCODING: encoding: [0x68,0x87,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488768      <unknown>
+
+tlbip VAAE1NXS, x8, x9
+// CHECK-INST: tlbip vaae1nxs, x8, x9
+// CHECK-ENCODING: encoding: [0x68,0x97,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489768      <unknown>
+
+tlbip VAAE1IS, x8, x9
+// CHECK-INST: tlbip vaae1is, x8, x9
+// CHECK-ENCODING: encoding: [0x68,0x83,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488368      <unknown>
+
+tlbip VAAE1ISNXS, x8, x9
+// CHECK-INST: tlbip vaae1isnxs, x8, x9
+// CHECK-ENCODING: encoding: [0x68,0x93,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489368      <unknown>
+
+tlbip VAAE1OS, x8, x9
+// CHECK-INST: tlbip vaae1os, x8, x9
+// CHECK-ENCODING: encoding: [0x68,0x81,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488168      <unknown>
+
+tlbip VAAE1OSNXS, x8, x9
+// CHECK-INST: tlbip vaae1osnxs, x8, x9
+// CHECK-ENCODING: encoding: [0x68,0x91,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489168      <unknown>
+
+tlbip VAALE1, x8, x9
+// CHECK-INST: tlbip vaale1, x8, x9
+// CHECK-ENCODING: encoding: [0xe8,0x87,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54887e8      <unknown>
+
+tlbip VAALE1NXS, x8, x9
+// CHECK-INST: tlbip vaale1nxs, x8, x9
+// CHECK-ENCODING: encoding: [0xe8,0x97,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54897e8      <unknown>
+
+tlbip VAALE1IS, x8, x9
+// CHECK-INST: tlbip vaale1is, x8, x9
+// CHECK-ENCODING: encoding: [0xe8,0x83,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54883e8      <unknown>
+
+tlbip VAALE1ISNXS, x8, x9
+// CHECK-INST: tlbip vaale1isnxs, x8, x9
+// CHECK-ENCODING: encoding: [0xe8,0x93,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54893e8      <unknown>
+
+tlbip VAALE1OS, x8, x9
+// CHECK-INST: tlbip vaale1os, x8, x9
+// CHECK-ENCODING: encoding: [0xe8,0x81,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54881e8      <unknown>
+
+tlbip VAALE1OSNXS, x8, x9
+// CHECK-INST: tlbip vaale1osnxs, x8, x9
+// CHECK-ENCODING: encoding: [0xe8,0x91,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54891e8      <unknown>
+
+tlbip VAE2, x14, x15
+// CHECK-INST: tlbip vae2, x14, x15
+// CHECK-ENCODING: encoding: [0x2e,0x87,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c872e      <unknown>
+
+tlbip VAE2NXS, x14, x15
+// CHECK-INST: tlbip vae2nxs, x14, x15
+// CHECK-ENCODING: encoding: [0x2e,0x97,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c972e      <unknown>
+
+tlbip VAE2IS, x14, x15
+// CHECK-INST: tlbip vae2is, x14, x15
+// CHECK-ENCODING: encoding: [0x2e,0x83,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c832e      <unknown>
+
+tlbip VAE2ISNXS, x14, x15
+// CHECK-INST: tlbip vae2isnxs, x14, x15
+// CHECK-ENCODING: encoding: [0x2e,0x93,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c932e      <unknown>
+
+tlbip VAE2OS, x14, x15
+// CHECK-INST: tlbip vae2os, x14, x15
+// CHECK-ENCODING: encoding: [0x2e,0x81,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c812e      <unknown>
+
+tlbip VAE2OSNXS, x14, x15
+// CHECK-INST: tlbip vae2osnxs, x14, x15
+// CHECK-ENCODING: encoding: [0x2e,0x91,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c912e      <unknown>
+
+tlbip VALE2, x14, x15
+// CHECK-INST: tlbip vale2, x14, x15
+// CHECK-ENCODING: encoding: [0xae,0x87,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c87ae      <unknown>
+
+tlbip VALE2NXS, x14, x15
+// CHECK-INST: tlbip vale2nxs, x14, x15
+// CHECK-ENCODING: encoding: [0xae,0x97,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c97ae      <unknown>
+
+tlbip VALE2IS, x14, x15
+// CHECK-INST: tlbip vale2is, x14, x15
+// CHECK-ENCODING: encoding: [0xae,0x83,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c83ae      <unknown>
+
+tlbip VALE2ISNXS, x14, x15
+// CHECK-INST: tlbip vale2isnxs, x14, x15
+// CHECK-ENCODING: encoding: [0xae,0x93,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c93ae      <unknown>
+
+tlbip VALE2OS, x14, x15
+// CHECK-INST: tlbip vale2os, x14, x15
+// CHECK-ENCODING: encoding: [0xae,0x81,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c81ae      <unknown>
+
+tlbip VALE2OSNXS, x14, x15
+// CHECK-INST: tlbip vale2osnxs, x14, x15
+// CHECK-ENCODING: encoding: [0xae,0x91,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c91ae      <unknown>
+
+tlbip VAE3, x24, x25
+// CHECK-INST: tlbip vae3, x24, x25
+// CHECK-ENCODING: encoding: [0x38,0x87,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e8738      <unknown>
+
+tlbip VAE3NXS, x24, x25
+// CHECK-INST: tlbip vae3nxs, x24, x25
+// CHECK-ENCODING: encoding: [0x38,0x97,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e9738      <unknown>
+
+tlbip VAE3IS, x24, x25
+// CHECK-INST: tlbip vae3is, x24, x25
+// CHECK-ENCODING: encoding: [0x38,0x83,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e8338      <unknown>
+
+tlbip VAE3ISNXS, x24, x25
+// CHECK-INST: tlbip vae3isnxs, x24, x25
+// CHECK-ENCODING: encoding: [0x38,0x93,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e9338      <unknown>
+
+tlbip VAE3OS, x24, x25
+// CHECK-INST: tlbip vae3os, x24, x25
+// CHECK-ENCODING: encoding: [0x38,0x81,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e8138      <unknown>
+
+tlbip VAE3OSNXS, x24, x25
+// CHECK-INST: tlbip vae3osnxs, x24, x25
+// CHECK-ENCODING: encoding: [0x38,0x91,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e9138      <unknown>
+
+tlbip VALE3, x24, x25
+// CHECK-INST: tlbip vale3, x24, x25
+// CHECK-ENCODING: encoding: [0xb8,0x87,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e87b8      <unknown>
+
+tlbip VALE3NXS, x24, x25
+// CHECK-INST: tlbip vale3nxs, x24, x25
+// CHECK-ENCODING: encoding: [0xb8,0x97,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e97b8      <unknown>
+
+tlbip VALE3IS, x24, x25
+// CHECK-INST: tlbip vale3is, x24, x25
+// CHECK-ENCODING: encoding: [0xb8,0x83,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e83b8      <unknown>
+
+tlbip VALE3ISNXS, x24, x25
+// CHECK-INST: tlbip vale3isnxs, x24, x25
+// CHECK-ENCODING: encoding: [0xb8,0x93,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e93b8      <unknown>
+
+tlbip VALE3OS, x24, x25
+// CHECK-INST: tlbip vale3os, x24, x25
+// CHECK-ENCODING: encoding: [0xb8,0x81,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e81b8      <unknown>
+
+tlbip VALE3OSNXS, x24, x25
+// CHECK-INST: tlbip vale3osnxs, x24, x25
+// CHECK-ENCODING: encoding: [0xb8,0x91,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e91b8      <unknown>
+
+tlbip RVAE1, x18, x19
+// CHECK-INST: tlbip rvae1, x18, x19
+// CHECK-ENCODING: encoding: [0x32,0x86,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488632      <unknown>
+
+tlbip RVAE1NXS, x18, x19
+// CHECK-INST: tlbip rvae1nxs, x18, x19
+// CHECK-ENCODING: encoding: [0x32,0x96,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489632      <unknown>
+
+tlbip RVAE1IS, x18, x19
+// CHECK-INST: tlbip rvae1is, x18, x19
+// CHECK-ENCODING: encoding: [0x32,0x82,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488232      <unknown>
+
+tlbip RVAE1ISNXS, x18, x19
+// CHECK-INST: tlbip rvae1isnxs, x18, x19
+// CHECK-ENCODING: encoding: [0x32,0x92,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489232      <unknown>
+
+tlbip RVAE1OS, x18, x19
+// CHECK-INST: tlbip rvae1os, x18, x19
+// CHECK-ENCODING: encoding: [0x32,0x85,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488532      <unknown>
+
+tlbip RVAE1OSNXS, x18, x19
+// CHECK-INST: tlbip rvae1osnxs, x18, x19
+// CHECK-ENCODING: encoding: [0x32,0x95,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489532      <unknown>
+
+tlbip RVAAE1, x18, x19
+// CHECK-INST: tlbip rvaae1, x18, x19
+// CHECK-ENCODING: encoding: [0x72,0x86,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488672      <unknown>
+
+tlbip RVAAE1NXS, x18, x19
+// CHECK-INST: tlbip rvaae1nxs, x18, x19
+// CHECK-ENCODING: encoding: [0x72,0x96,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489672      <unknown>
+
+tlbip RVAAE1IS, x18, x19
+// CHECK-INST: tlbip rvaae1is, x18, x19
+// CHECK-ENCODING: encoding: [0x72,0x82,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488272      <unknown>
+
+tlbip RVAAE1ISNXS, x18, x19
+// CHECK-INST: tlbip rvaae1isnxs, x18, x19
+// CHECK-ENCODING: encoding: [0x72,0x92,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489272      <unknown>
+
+tlbip RVAAE1OS, x18, x19
+// CHECK-INST: tlbip rvaae1os, x18, x19
+// CHECK-ENCODING: encoding: [0x72,0x85,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5488572      <unknown>
+
+tlbip RVAAE1OSNXS, x18, x19
+// CHECK-INST: tlbip rvaae1osnxs, x18, x19
+// CHECK-ENCODING: encoding: [0x72,0x95,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d5489572      <unknown>
+
+tlbip RVALE1, x18, x19
+// CHECK-INST: tlbip rvale1, x18, x19
+// CHECK-ENCODING: encoding: [0xb2,0x86,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54886b2      <unknown>
+
+tlbip RVALE1NXS, x18, x19
+// CHECK-INST: tlbip rvale1nxs, x18, x19
+// CHECK-ENCODING: encoding: [0xb2,0x96,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54896b2      <unknown>
+
+tlbip RVALE1IS, x18, x19
+// CHECK-INST: tlbip rvale1is, x18, x19
+// CHECK-ENCODING: encoding: [0xb2,0x82,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54882b2      <unknown>
+
+tlbip RVALE1ISNXS, x18, x19
+// CHECK-INST: tlbip rvale1isnxs, x18, x19
+// CHECK-ENCODING: encoding: [0xb2,0x92,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54892b2      <unknown>
+
+tlbip RVALE1OS, x18, x19
+// CHECK-INST: tlbip rvale1os, x18, x19
+// CHECK-ENCODING: encoding: [0xb2,0x85,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54885b2      <unknown>
+
+tlbip RVALE1OSNXS, x18, x19
+// CHECK-INST: tlbip rvale1osnxs, x18, x19
+// CHECK-ENCODING: encoding: [0xb2,0x95,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54895b2      <unknown>
+
+tlbip RVAALE1, x18, x19
+// CHECK-INST: tlbip rvaale1, x18, x19
+// CHECK-ENCODING: encoding: [0xf2,0x86,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54886f2      <unknown>
+
+tlbip RVAALE1NXS, x18, x19
+// CHECK-INST: tlbip rvaale1nxs, x18, x19
+// CHECK-ENCODING: encoding: [0xf2,0x96,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54896f2      <unknown>
+
+tlbip RVAALE1IS, x18, x19
+// CHECK-INST: tlbip rvaale1is, x18, x19
+// CHECK-ENCODING: encoding: [0xf2,0x82,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54882f2      <unknown>
+
+tlbip RVAALE1ISNXS, x18, x19
+// CHECK-INST: tlbip rvaale1isnxs, x18, x19
+// CHECK-ENCODING: encoding: [0xf2,0x92,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54892f2      <unknown>
+
+tlbip RVAALE1OS, x18, x19
+// CHECK-INST: tlbip rvaale1os, x18, x19
+// CHECK-ENCODING: encoding: [0xf2,0x85,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54885f2      <unknown>
+
+tlbip RVAALE1OSNXS, x18, x19
+// CHECK-INST: tlbip rvaale1osnxs, x18, x19
+// CHECK-ENCODING: encoding: [0xf2,0x95,0x48,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54895f2      <unknown>
+
+tlbip RVAE2, x28, x29
+// CHECK-INST: tlbip rvae2, x28, x29
+// CHECK-ENCODING: encoding: [0x3c,0x86,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c863c      <unknown>
+
+tlbip RVAE2NXS, x28, x29
+// CHECK-INST: tlbip rvae2nxs, x28, x29
+// CHECK-ENCODING: encoding: [0x3c,0x96,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c963c      <unknown>
+
+tlbip RVAE2IS, x28, x29
+// CHECK-INST: tlbip rvae2is, x28, x29
+// CHECK-ENCODING: encoding: [0x3c,0x82,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c823c      <unknown>
+
+tlbip RVAE2ISNXS, x28, x29
+// CHECK-INST: tlbip rvae2isnxs, x28, x29
+// CHECK-ENCODING: encoding: [0x3c,0x92,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c923c      <unknown>
+
+tlbip RVAE2OS, x28, x29
+// CHECK-INST: tlbip rvae2os, x28, x29
+// CHECK-ENCODING: encoding: [0x3c,0x85,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c853c      <unknown>
+
+tlbip RVAE2OSNXS, x28, x29
+// CHECK-INST: tlbip rvae2osnxs, x28, x29
+// CHECK-ENCODING: encoding: [0x3c,0x95,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c953c      <unknown>
+
+tlbip RVALE2, x28, x29
+// CHECK-INST: tlbip rvale2, x28, x29
+// CHECK-ENCODING: encoding: [0xbc,0x86,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c86bc      <unknown>
+
+tlbip RVALE2NXS, x28, x29
+// CHECK-INST: tlbip rvale2nxs, x28, x29
+// CHECK-ENCODING: encoding: [0xbc,0x96,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c96bc      <unknown>
+
+tlbip RVALE2IS, x28, x29
+// CHECK-INST: tlbip rvale2is, x28, x29
+// CHECK-ENCODING: encoding: [0xbc,0x82,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c82bc      <unknown>
+
+tlbip RVALE2ISNXS, x28, x29
+// CHECK-INST: tlbip rvale2isnxs, x28, x29
+// CHECK-ENCODING: encoding: [0xbc,0x92,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c92bc      <unknown>
+
+tlbip RVALE2OS, x28, x29
+// CHECK-INST: tlbip rvale2os, x28, x29
+// CHECK-ENCODING: encoding: [0xbc,0x85,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c85bc      <unknown>
+
+tlbip RVALE2OSNXS, x28, x29
+// CHECK-INST: tlbip rvale2osnxs, x28, x29
+// CHECK-ENCODING: encoding: [0xbc,0x95,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c95bc      <unknown>
+
+tlbip RVAE3, x10, x11
+// CHECK-INST: tlbip rvae3, x10, x11
+// CHECK-ENCODING: encoding: [0x2a,0x86,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e862a      <unknown>
+
+tlbip RVAE3NXS, x10, x11
+// CHECK-INST: tlbip rvae3nxs, x10, x11
+// CHECK-ENCODING: encoding: [0x2a,0x96,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e962a      <unknown>
+
+tlbip RVAE3IS, x10, x11
+// CHECK-INST: tlbip rvae3is, x10, x11
+// CHECK-ENCODING: encoding: [0x2a,0x82,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e822a      <unknown>
+
+tlbip RVAE3ISNXS, x10, x11
+// CHECK-INST: tlbip rvae3isnxs, x10, x11
+// CHECK-ENCODING: encoding: [0x2a,0x92,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e922a      <unknown>
+
+tlbip RVAE3OS, x10, x11
+// CHECK-INST: tlbip rvae3os, x10, x11
+// CHECK-ENCODING: encoding: [0x2a,0x85,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e852a      <unknown>
+
+tlbip RVAE3OSNXS, x10, x11
+// CHECK-INST: tlbip rvae3osnxs, x10, x11
+// CHECK-ENCODING: encoding: [0x2a,0x95,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e952a      <unknown>
+
+tlbip RVALE3, x10, x11
+// CHECK-INST: tlbip rvale3, x10, x11
+// CHECK-ENCODING: encoding: [0xaa,0x86,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e86aa      <unknown>
+
+tlbip RVALE3NXS, x10, x11
+// CHECK-INST: tlbip rvale3nxs, x10, x11
+// CHECK-ENCODING: encoding: [0xaa,0x96,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e96aa      <unknown>
+
+tlbip RVALE3IS, x10, x11
+// CHECK-INST: tlbip rvale3is, x10, x11
+// CHECK-ENCODING: encoding: [0xaa,0x82,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e82aa      <unknown>
+
+tlbip RVALE3ISNXS, x10, x11
+// CHECK-INST: tlbip rvale3isnxs, x10, x11
+// CHECK-ENCODING: encoding: [0xaa,0x92,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e92aa      <unknown>
+
+tlbip RVALE3OS, x10, x11
+// CHECK-INST: tlbip rvale3os, x10, x11
+// CHECK-ENCODING: encoding: [0xaa,0x85,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e85aa      <unknown>
+
+tlbip RVALE3OSNXS, x10, x11
+// CHECK-INST: tlbip rvale3osnxs, x10, x11
+// CHECK-ENCODING: encoding: [0xaa,0x95,0x4e,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54e95aa      <unknown>
+
+tlbip RIPAS2E1, x20, x21
+// CHECK-INST: tlbip ripas2e1, x20, x21
+// CHECK-ENCODING: encoding: [0x54,0x84,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c8454      <unknown>
+
+tlbip RIPAS2E1NXS, x20, x21
+// CHECK-INST: tlbip ripas2e1nxs, x20, x21
+// CHECK-ENCODING: encoding: [0x54,0x94,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c9454      <unknown>
+
+tlbip RIPAS2E1IS, x20, x21
+// CHECK-INST: tlbip ripas2e1is, x20, x21
+// CHECK-ENCODING: encoding: [0x54,0x80,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c8054      <unknown>
+
+tlbip RIPAS2E1ISNXS, x20, x21
+// CHECK-INST: tlbip ripas2e1isnxs, x20, x21
+// CHECK-ENCODING: encoding: [0x54,0x90,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c9054      <unknown>
+
+tlbip RIPAS2E1OS, x20, x21
+// CHECK-INST: tlbip ripas2e1os, x20, x21
+// CHECK-ENCODING: encoding: [0x74,0x84,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c8474      <unknown>
+
+tlbip RIPAS2E1OSNXS, x20, x21
+// CHECK-INST: tlbip ripas2e1osnxs, x20, x21
+// CHECK-ENCODING: encoding: [0x74,0x94,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c9474      <unknown>
+
+tlbip RIPAS2LE1, x20, x21
+// CHECK-INST: tlbip ripas2le1, x20, x21
+// CHECK-ENCODING: encoding: [0xd4,0x84,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c84d4      <unknown>
+
+tlbip RIPAS2LE1NXS, x20, x21
+// CHECK-INST: tlbip ripas2le1nxs, x20, x21
+// CHECK-ENCODING: encoding: [0xd4,0x94,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c94d4      <unknown>
+
+tlbip RIPAS2LE1IS, x20, x21
+// CHECK-INST: tlbip ripas2le1is, x20, x21
+// CHECK-ENCODING: encoding: [0xd4,0x80,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c80d4      <unknown>
+
+tlbip RIPAS2LE1ISNXS, x20, x21
+// CHECK-INST: tlbip ripas2le1isnxs, x20, x21
+// CHECK-ENCODING: encoding: [0xd4,0x90,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c90d4      <unknown>
+
+tlbip RIPAS2LE1OS, x20, x21
+// CHECK-INST: tlbip ripas2le1os, x20, x21
+// CHECK-ENCODING: encoding: [0xf4,0x84,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c84f4      <unknown>
+
+tlbip RIPAS2LE1OSNXS, x20, x21
+// CHECK-INST: tlbip ripas2le1osnxs, x20, x21
+// CHECK-ENCODING: encoding: [0xf4,0x94,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c94f4      <unknown>
+
+tlbip RIPAS2LE1OS, xzr, xzr
+// CHECK-INST: tlbip ripas2le1os, xzr, xzr
+// CHECK-ENCODING: encoding: [0xff,0x84,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c84ff      <unknown>
+
+tlbip RIPAS2LE1OSNXS, xzr, xzr
+// CHECK-INST: tlbip ripas2le1osnxs, xzr, xzr
+// CHECK-ENCODING: encoding: [0xff,0x94,0x4c,0xd5]
+// CHECK-ERROR: error: instruction requires: d128
+// CHECK-UNKNOWN:  d54c94ff      <unknown>
diff --git a/llvm/test/MC/AArch64/build-attributes-asm-aeabi-switch_using_name_only.s b/llvm/test/MC/AArch64/build-attributes-asm-aeabi-switch_using_name_only.s
new file mode 100644
index 0000000000000..cf89d3e376f3b
--- /dev/null
+++ b/llvm/test/MC/AArch64/build-attributes-asm-aeabi-switch_using_name_only.s
@@ -0,0 +1,39 @@
+// RUN: llvm-mc -triple=aarch64 %s -o - | FileCheck %s --check-prefix=ASM
+// RUN: llvm-mc -triple=aarch64 -filetype=obj %s -o - | llvm-readelf --hex-dump=.ARM.attributes - | FileCheck %s --check-prefix=ELF
+
+// ASM: .aeabi_subsection aeabi_pauthabi, required, uleb128
+// ASM: .aeabi_subsection aeabi_feature_and_bits, optional, uleb128
+// ASM: .aeabi_attribute	0, 1 // Tag_Feature_BTI
+// ASM: .aeabi_attribute	2, 1 // Tag_PAuth_Schema
+// ASM: .aeabi_attribute	1, 1 // Tag_PAuth_Platform
+// ASM: .aeabi_attribute	2, 1 // Tag_Feature_GCS
+// ASM: .aeabi_attribute	1, 0 // Tag_Feature_PAC
+// ASM: .aeabi_attribute	7, 1
+// ASM: .aeabi_attribute	7, 0
+
+// ELF: Hex dump of section '.ARM.attributes':
+// ELF-NEXT: 0x00000000 411b0000 00616561 62695f70 61757468 A....aeabi_pauth
+// ELF-NEXT: 0x00000010 61626900 00000201 01010700 25000000 abi.........%...
+// ELF-NEXT: 0x00000020 61656162 695f6665 61747572 655f616e aeabi_feature_an
+// ELF-NEXT: 0x00000030 645f6269 74730001 00000102 01010007 d_bits..........
+// ELF-NEXT: 0x00000040 01
+
+
+.aeabi_subsection aeabi_pauthabi, required, uleb128
+.aeabi_subsection aeabi_feature_and_bits, optional, uleb128
+.aeabi_attribute Tag_Feature_BTI, 1
+.aeabi_subsection aeabi_feature_and_bits
+.aeabi_subsection aeabi_pauthabi
+.aeabi_attribute Tag_PAuth_Schema, 1
+.aeabi_subsection aeabi_pauthabi
+.aeabi_attribute Tag_PAuth_Platform, 1
+.aeabi_subsection aeabi_pauthabi
+.aeabi_subsection aeabi_feature_and_bits
+.aeabi_attribute Tag_Feature_GCS, 1
+.aeabi_subsection aeabi_pauthabi
+.aeabi_subsection aeabi_feature_and_bits
+.aeabi_attribute Tag_Feature_PAC, 0
+.aeabi_subsection aeabi_feature_and_bits
+.aeabi_attribute 7, 1
+.aeabi_subsection aeabi_pauthabi
+.aeabi_attribute 7, 0
diff --git a/llvm/test/MC/AArch64/patchinst.s b/llvm/test/MC/AArch64/patchinst.s
new file mode 100644
index 0000000000000..fbfb66ac3f27f
--- /dev/null
+++ b/llvm/test/MC/AArch64/patchinst.s
@@ -0,0 +1,12 @@
+// RUN: llvm-mc -triple aarch64-elf -filetype=obj %s -o - | llvm-objdump -r - | FileCheck %s
+
+// Test that PATCHINST appears after JUMP26.
+// CHECK:      R_AARCH64_JUMP26
+// CHECK-NEXT: R_AARCH64_JUMP26
+// CHECK-NEXT: R_AARCH64_PATCHINST
+// CHECK-NEXT: R_AARCH64_PATCHINST
+.reloc ., R_AARCH64_PATCHINST, ds
+b f1
+.balign 8
+.reloc ., R_AARCH64_PATCHINST, ds
+b f2
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s b/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s
index f1641fc693b1c..b46189b15d9dc 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_ds.s
@@ -1,6 +1,1917 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR %s
 
+ds_nop
+// GFX1250: ds_nop                                  ; encoding: [0x00,0x00,0x50,0xd8,0x00,0x00,0x00,0x00]
+
+ds_add_f32 v1, v2
+// GFX1250: ds_add_f32 v1, v2                       ; encoding: [0x00,0x00,0x54,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_f32 v1, v2 offset:65535
+// GFX1250: ds_add_f32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x54,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_f32 v1, v2 offset:0
+// GFX1250: ds_add_f32 v1, v2                       ; encoding: [0x00,0x00,0x54,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_f32 v255, v255 offset:4
+// GFX1250: ds_add_f32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x54,0xd8,0xff,0xff,0x00,0x00]
+
+ds_add_rtn_f32 v5, v1, v2
+// GFX1250: ds_add_rtn_f32 v5, v1, v2               ; encoding: [0x00,0x00,0xe4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_f32 v5, v1, v2 offset:65535
+// GFX1250: ds_add_rtn_f32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xe4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_f32 v5, v1, v2 offset:0
+// GFX1250: ds_add_rtn_f32 v5, v1, v2               ; encoding: [0x00,0x00,0xe4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_f32 v255, v255, v255 offset:4
+// GFX1250: ds_add_rtn_f32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xe4,0xd9,0xff,0xff,0x00,0xff]
+
+ds_add_rtn_u32 v5, v1, v2
+// GFX1250: ds_add_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x80,0xd8,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_u32 v5, v1, v2 offset:65535
+// GFX1250: ds_add_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x80,0xd8,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_u32 v5, v1, v2 offset:0
+// GFX1250: ds_add_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x80,0xd8,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_u32 v255, v255, v255 offset:4
+// GFX1250: ds_add_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x80,0xd8,0xff,0xff,0x00,0xff]
+
+ds_add_rtn_u64 v[6:7], v1, v[2:3]
+// GFX1250: ds_add_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x80,0xd9,0x01,0x02,0x00,0x06]
+
+ds_add_rtn_u64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_add_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x80,0xd9,0x01,0x02,0x00,0x06]
+
+ds_add_rtn_u64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_add_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x80,0xd9,0x01,0x02,0x00,0x06]
+
+ds_add_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_add_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x80,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_add_u32 v1, v2
+// GFX1250: ds_add_u32 v1, v2                       ; encoding: [0x00,0x00,0x00,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_u32 v1, v2 offset:65535
+// GFX1250: ds_add_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x00,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_u32 v1, v2 offset:0
+// GFX1250: ds_add_u32 v1, v2                       ; encoding: [0x00,0x00,0x00,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_u32 v255, v255 offset:4
+// GFX1250: ds_add_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x00,0xd8,0xff,0xff,0x00,0x00]
+
+ds_add_u64 v1, v[2:3]
+// GFX1250: ds_add_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x00,0xd9,0x01,0x02,0x00,0x00]
+
+ds_add_u64 v1, v[2:3] offset:65535
+// GFX1250: ds_add_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x00,0xd9,0x01,0x02,0x00,0x00]
+
+ds_add_u64 v1, v[2:3] offset:0
+// GFX1250: ds_add_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x00,0xd9,0x01,0x02,0x00,0x00]
+
+ds_add_u64 v255, v[254:255] offset:4
+// GFX1250: ds_add_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x00,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_and_b32 v1, v2
+// GFX1250: ds_and_b32 v1, v2                       ; encoding: [0x00,0x00,0x24,0xd8,0x01,0x02,0x00,0x00]
+
+ds_and_b32 v1, v2 offset:65535
+// GFX1250: ds_and_b32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x24,0xd8,0x01,0x02,0x00,0x00]
+
+ds_and_b32 v1, v2 offset:0
+// GFX1250: ds_and_b32 v1, v2                       ; encoding: [0x00,0x00,0x24,0xd8,0x01,0x02,0x00,0x00]
+
+ds_and_b32 v255, v255 offset:4
+// GFX1250: ds_and_b32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x24,0xd8,0xff,0xff,0x00,0x00]
+
+ds_and_b64 v1, v[2:3]
+// GFX1250: ds_and_b64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x24,0xd9,0x01,0x02,0x00,0x00]
+
+ds_and_b64 v1, v[2:3] offset:65535
+// GFX1250: ds_and_b64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x24,0xd9,0x01,0x02,0x00,0x00]
+
+ds_and_b64 v1, v[2:3] offset:0
+// GFX1250: ds_and_b64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x24,0xd9,0x01,0x02,0x00,0x00]
+
+ds_and_b64 v255, v[254:255] offset:4
+// GFX1250: ds_and_b64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x24,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_and_rtn_b32 v5, v1, v2
+// GFX1250: ds_and_rtn_b32 v5, v1, v2               ; encoding: [0x00,0x00,0xa4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_and_rtn_b32 v5, v1, v2 offset:65535
+// GFX1250: ds_and_rtn_b32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xa4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_and_rtn_b32 v5, v1, v2 offset:0
+// GFX1250: ds_and_rtn_b32 v5, v1, v2               ; encoding: [0x00,0x00,0xa4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_and_rtn_b32 v255, v255, v255 offset:4
+// GFX1250: ds_and_rtn_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xa4,0xd8,0xff,0xff,0x00,0xff]
+
+ds_and_rtn_b64 v[6:7], v1, v[2:3]
+// GFX1250: ds_and_rtn_b64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0xa4,0xd9,0x01,0x02,0x00,0x06]
+
+ds_and_rtn_b64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_and_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa4,0xd9,0x01,0x02,0x00,0x06]
+
+ds_and_rtn_b64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_and_rtn_b64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0xa4,0xd9,0x01,0x02,0x00,0x06]
+
+ds_and_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_and_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xa4,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_append v5
+// GFX1250: ds_append v5                            ; encoding: [0x00,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05]
+
+ds_append v5 offset:65535
+// GFX1250: ds_append v5 offset:65535               ; encoding: [0xff,0xff,0xf8,0xd8,0x00,0x00,0x00,0x05]
+
+ds_append v5 offset:0
+// GFX1250: ds_append v5                            ; encoding: [0x00,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05]
+
+ds_append v255 offset:4
+// GFX1250: ds_append v255 offset:4                 ; encoding: [0x04,0x00,0xf8,0xd8,0x00,0x00,0x00,0xff]
+
+ds_bpermute_b32 v5, v1, v2
+// GFX1250: ds_bpermute_b32 v5, v1, v2              ; encoding: [0x00,0x00,0xcc,0xda,0x01,0x02,0x00,0x05]
+
+ds_bpermute_b32 v5, v1, v2 offset:65535
+// GFX1250: ds_bpermute_b32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xcc,0xda,0x01,0x02,0x00,0x05]
+
+ds_bpermute_b32 v5, v1, v2 offset:0
+// GFX1250: ds_bpermute_b32 v5, v1, v2              ; encoding: [0x00,0x00,0xcc,0xda,0x01,0x02,0x00,0x05]
+
+ds_bpermute_b32 v255, v255, v255 offset:4
+// GFX1250: ds_bpermute_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xcc,0xda,0xff,0xff,0x00,0xff]
+
+ds_cmpstore_b32 v1, v2, v3
+// GFX1250: ds_cmpstore_b32 v1, v2, v3              ; encoding: [0x00,0x00,0x40,0xd8,0x01,0x02,0x03,0x00]
+
+ds_cmpstore_b32 v1, v2, v3 offset:65535
+// GFX1250: ds_cmpstore_b32 v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0x40,0xd8,0x01,0x02,0x03,0x00]
+
+ds_cmpstore_b32 v1, v2, v3 offset:0
+// GFX1250: ds_cmpstore_b32 v1, v2, v3              ; encoding: [0x00,0x00,0x40,0xd8,0x01,0x02,0x03,0x00]
+
+ds_cmpstore_b32 v255, v255, v255 offset:4
+// GFX1250: ds_cmpstore_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x40,0xd8,0xff,0xff,0xff,0x00]
+
+ds_cmpstore_b64 v1, v[2:3], v[4:5]
+// GFX1250: ds_cmpstore_b64 v1, v[2:3], v[4:5]      ; encoding: [0x00,0x00,0x40,0xd9,0x01,0x02,0x04,0x00]
+
+ds_cmpstore_b64 v1, v[2:3], v[4:5] offset:65535
+// GFX1250: ds_cmpstore_b64 v1, v[2:3], v[4:5] offset:65535 ; encoding: [0xff,0xff,0x40,0xd9,0x01,0x02,0x04,0x00]
+
+ds_cmpstore_b64 v1, v[2:3], v[4:5] offset:0
+// GFX1250: ds_cmpstore_b64 v1, v[2:3], v[4:5]      ; encoding: [0x00,0x00,0x40,0xd9,0x01,0x02,0x04,0x00]
+
+ds_cmpstore_b64 v255, v[254:255], v[254:255] offset:4
+// GFX1250: ds_cmpstore_b64 v255, v[254:255], v[254:255] offset:4 ; encoding: [0x04,0x00,0x40,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_cmpstore_rtn_b32 v5, v1, v2, v3
+// GFX1250: ds_cmpstore_rtn_b32 v5, v1, v2, v3      ; encoding: [0x00,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_cmpstore_rtn_b32 v5, v1, v2, v3 offset:65535
+// GFX1250: ds_cmpstore_rtn_b32 v5, v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xc0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_cmpstore_rtn_b32 v5, v1, v2, v3 offset:0
+// GFX1250: ds_cmpstore_rtn_b32 v5, v1, v2, v3      ; encoding: [0x00,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_cmpstore_rtn_b32 v255, v255, v255, v255 offset:4
+// GFX1250: ds_cmpstore_rtn_b32 v255, v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xc0,0xd8,0xff,0xff,0xff,0xff]
+
+ds_cmpstore_rtn_b64 v[6:7], v1, v[2:3], v[4:5]
+// GFX1250: ds_cmpstore_rtn_b64 v[6:7], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xc0,0xd9,0x01,0x02,0x04,0x06]
+
+ds_cmpstore_rtn_b64 v[6:7], v1, v[2:3], v[4:5] offset:65535
+// GFX1250: ds_cmpstore_rtn_b64 v[6:7], v1, v[2:3], v[4:5] offset:65535 ; encoding: [0xff,0xff,0xc0,0xd9,0x01,0x02,0x04,0x06]
+
+ds_cmpstore_rtn_b64 v[6:7], v1, v[2:3], v[4:5] offset:0
+// GFX1250: ds_cmpstore_rtn_b64 v[6:7], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xc0,0xd9,0x01,0x02,0x04,0x06]
+
+ds_cmpstore_rtn_b64 v[254:255], v255, v[254:255], v[254:255] offset:4
+// GFX1250: ds_cmpstore_rtn_b64 v[254:255], v255, v[254:255], v[254:255] offset:4 ; encoding: [0x04,0x00,0xc0,0xd9,0xff,0xfe,0xfe,0xfe]
+
+ds_condxchg32_rtn_b64 v[6:7], v1, v[2:3]
+// GFX1250: ds_condxchg32_rtn_b64 v[6:7], v1, v[2:3] ; encoding: [0x00,0x00,0xf8,0xd9,0x01,0x02,0x00,0x06]
+
+ds_condxchg32_rtn_b64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_condxchg32_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xf8,0xd9,0x01,0x02,0x00,0x06]
+
+ds_condxchg32_rtn_b64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_condxchg32_rtn_b64 v[6:7], v1, v[2:3] ; encoding: [0x00,0x00,0xf8,0xd9,0x01,0x02,0x00,0x06]
+
+ds_condxchg32_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_condxchg32_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xf8,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_consume v5
+// GFX1250: ds_consume v5                           ; encoding: [0x00,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05]
+
+ds_consume v5 offset:65535
+// GFX1250: ds_consume v5 offset:65535              ; encoding: [0xff,0xff,0xf4,0xd8,0x00,0x00,0x00,0x05]
+
+ds_consume v5 offset:0
+// GFX1250: ds_consume v5                           ; encoding: [0x00,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05]
+
+ds_consume v255 offset:4
+// GFX1250: ds_consume v255 offset:4                ; encoding: [0x04,0x00,0xf4,0xd8,0x00,0x00,0x00,0xff]
+
+ds_dec_rtn_u32 v5, v1, v2
+// GFX1250: ds_dec_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x90,0xd8,0x01,0x02,0x00,0x05]
+
+ds_dec_rtn_u32 v5, v1, v2 offset:65535
+// GFX1250: ds_dec_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x90,0xd8,0x01,0x02,0x00,0x05]
+
+ds_dec_rtn_u32 v5, v1, v2 offset:0
+// GFX1250: ds_dec_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x90,0xd8,0x01,0x02,0x00,0x05]
+
+ds_dec_rtn_u32 v255, v255, v255 offset:4
+// GFX1250: ds_dec_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x90,0xd8,0xff,0xff,0x00,0xff]
+
+ds_dec_rtn_u64 v[6:7], v1, v[2:3]
+// GFX1250: ds_dec_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x90,0xd9,0x01,0x02,0x00,0x06]
+
+ds_dec_rtn_u64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_dec_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x90,0xd9,0x01,0x02,0x00,0x06]
+
+ds_dec_rtn_u64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_dec_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x90,0xd9,0x01,0x02,0x00,0x06]
+
+ds_dec_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_dec_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x90,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_dec_u32 v1, v2
+// GFX1250: ds_dec_u32 v1, v2                       ; encoding: [0x00,0x00,0x10,0xd8,0x01,0x02,0x00,0x00]
+
+ds_dec_u32 v1, v2 offset:65535
+// GFX1250: ds_dec_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x10,0xd8,0x01,0x02,0x00,0x00]
+
+ds_dec_u32 v1, v2 offset:0
+// GFX1250: ds_dec_u32 v1, v2                       ; encoding: [0x00,0x00,0x10,0xd8,0x01,0x02,0x00,0x00]
+
+ds_dec_u32 v255, v255 offset:4
+// GFX1250: ds_dec_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x10,0xd8,0xff,0xff,0x00,0x00]
+
+ds_dec_u64 v1, v[2:3]
+// GFX1250: ds_dec_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x10,0xd9,0x01,0x02,0x00,0x00]
+
+ds_dec_u64 v1, v[2:3] offset:65535
+// GFX1250: ds_dec_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x10,0xd9,0x01,0x02,0x00,0x00]
+
+ds_dec_u64 v1, v[2:3] offset:0
+// GFX1250: ds_dec_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x10,0xd9,0x01,0x02,0x00,0x00]
+
+ds_dec_u64 v255, v[254:255] offset:4
+// GFX1250: ds_dec_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x10,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_inc_rtn_u32 v5, v1, v2
+// GFX1250: ds_inc_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x8c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_inc_rtn_u32 v5, v1, v2 offset:65535
+// GFX1250: ds_inc_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x8c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_inc_rtn_u32 v5, v1, v2 offset:0
+// GFX1250: ds_inc_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x8c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_inc_rtn_u32 v255, v255, v255 offset:4
+// GFX1250: ds_inc_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x8c,0xd8,0xff,0xff,0x00,0xff]
+
+ds_inc_rtn_u64 v[6:7], v1, v[2:3]
+// GFX1250: ds_inc_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x8c,0xd9,0x01,0x02,0x00,0x06]
+
+ds_inc_rtn_u64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_inc_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x8c,0xd9,0x01,0x02,0x00,0x06]
+
+ds_inc_rtn_u64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_inc_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x8c,0xd9,0x01,0x02,0x00,0x06]
+
+ds_inc_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_inc_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x8c,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_inc_u32 v1, v2
+// GFX1250: ds_inc_u32 v1, v2                       ; encoding: [0x00,0x00,0x0c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_inc_u32 v1, v2 offset:65535
+// GFX1250: ds_inc_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x0c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_inc_u32 v1, v2 offset:0
+// GFX1250: ds_inc_u32 v1, v2                       ; encoding: [0x00,0x00,0x0c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_inc_u32 v255, v255 offset:4
+// GFX1250: ds_inc_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x0c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_inc_u64 v1, v[2:3]
+// GFX1250: ds_inc_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_inc_u64 v1, v[2:3] offset:65535
+// GFX1250: ds_inc_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x0c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_inc_u64 v1, v[2:3] offset:0
+// GFX1250: ds_inc_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_inc_u64 v255, v[254:255] offset:4
+// GFX1250: ds_inc_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x0c,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_load_2addr_b32 v[6:7], v1
+// GFX1250: ds_load_2addr_b32 v[6:7], v1            ; encoding: [0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_b32 v[6:7], v1 offset0:127 offset1:255
+// GFX1250: ds_load_2addr_b32 v[6:7], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_b32 v[6:7], v1 offset0:0 offset1:0
+// GFX1250: ds_load_2addr_b32 v[6:7], v1            ; encoding: [0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_b32 v[254:255], v255 offset0:16 offset1:1
+// GFX1250: ds_load_2addr_b32 v[254:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xdc,0xd8,0xff,0x00,0x00,0xfe]
+
+ds_load_2addr_b64 v[6:9], v1
+// GFX1250: ds_load_2addr_b64 v[6:9], v1            ; encoding: [0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_b64 v[6:9], v1 offset0:127 offset1:255
+// GFX1250: ds_load_2addr_b64 v[6:9], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_b64 v[6:9], v1 offset0:0 offset1:0
+// GFX1250: ds_load_2addr_b64 v[6:9], v1            ; encoding: [0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_b64 v[252:255], v255 offset0:16 offset1:1
+// GFX1250: ds_load_2addr_b64 v[252:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xdc,0xd9,0xff,0x00,0x00,0xfc]
+
+ds_load_2addr_stride64_b32 v[6:7], v1
+// GFX1250: ds_load_2addr_stride64_b32 v[6:7], v1   ; encoding: [0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_stride64_b32 v[6:7], v1 offset0:127 offset1:255
+// GFX1250: ds_load_2addr_stride64_b32 v[6:7], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_stride64_b32 v[6:7], v1 offset0:0 offset1:0
+// GFX1250: ds_load_2addr_stride64_b32 v[6:7], v1   ; encoding: [0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_stride64_b32 v[254:255], v255 offset0:16 offset1:1
+// GFX1250: ds_load_2addr_stride64_b32 v[254:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xe0,0xd8,0xff,0x00,0x00,0xfe]
+
+ds_load_2addr_stride64_b64 v[6:9], v1
+// GFX1250: ds_load_2addr_stride64_b64 v[6:9], v1   ; encoding: [0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_stride64_b64 v[6:9], v1 offset0:127 offset1:255
+// GFX1250: ds_load_2addr_stride64_b64 v[6:9], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_stride64_b64 v[6:9], v1 offset0:0 offset1:0
+// GFX1250: ds_load_2addr_stride64_b64 v[6:9], v1   ; encoding: [0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x06]
+
+ds_load_2addr_stride64_b64 v[252:255], v255 offset0:16 offset1:1
+// GFX1250: ds_load_2addr_stride64_b64 v[252:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xe0,0xd9,0xff,0x00,0x00,0xfc]
+
+ds_load_addtid_b32 v5
+// GFX1250: ds_load_addtid_b32 v5                   ; encoding: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_load_addtid_b32 v5 offset:65535
+// GFX1250: ds_load_addtid_b32 v5 offset:65535      ; encoding: [0xff,0xff,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_load_addtid_b32 v5 offset:0
+// GFX1250: ds_load_addtid_b32 v5                   ; encoding: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_load_addtid_b32 v255 offset:4
+// GFX1250: ds_load_addtid_b32 v255 offset:4        ; encoding: [0x04,0x00,0xc4,0xda,0x00,0x00,0x00,0xff]
+
+ds_load_b128 v[6:9], v1
+// GFX1250: ds_load_b128 v[6:9], v1                 ; encoding: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x06]
+
+ds_load_b128 v[6:9], v1 offset:65535
+// GFX1250: ds_load_b128 v[6:9], v1 offset:65535    ; encoding: [0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0x06]
+
+ds_load_b128 v[6:9], v1 offset:0
+// GFX1250: ds_load_b128 v[6:9], v1                 ; encoding: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x06]
+
+ds_load_b128 v[252:255], v255 offset:4
+// GFX1250: ds_load_b128 v[252:255], v255 offset:4  ; encoding: [0x04,0x00,0xfc,0xdb,0xff,0x00,0x00,0xfc]
+
+ds_load_b32 v5, v1
+// GFX1250: ds_load_b32 v5, v1                      ; encoding: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_b32 v5, v1 offset:65535
+// GFX1250: ds_load_b32 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_b32 v5, v1 offset:0
+// GFX1250: ds_load_b32 v5, v1                      ; encoding: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_b32 v255, v255 offset:4
+// GFX1250: ds_load_b32 v255, v255 offset:4         ; encoding: [0x04,0x00,0xd8,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_b64 v[6:7], v1
+// GFX1250: ds_load_b64 v[6:7], v1                  ; encoding: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x06]
+
+ds_load_b64 v[6:7], v1 offset:65535
+// GFX1250: ds_load_b64 v[6:7], v1 offset:65535     ; encoding: [0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0x06]
+
+ds_load_b64 v[6:7], v1 offset:0
+// GFX1250: ds_load_b64 v[6:7], v1                  ; encoding: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x06]
+
+ds_load_b64 v[254:255], v255 offset:4
+// GFX1250: ds_load_b64 v[254:255], v255 offset:4   ; encoding: [0x04,0x00,0xd8,0xd9,0xff,0x00,0x00,0xfe]
+
+ds_load_b96 v[6:8], v1
+// GFX1250: ds_load_b96 v[6:8], v1                  ; encoding: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x06]
+
+ds_load_b96 v[6:8], v1 offset:65535
+// GFX1250: ds_load_b96 v[6:8], v1 offset:65535     ; encoding: [0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0x06]
+
+ds_load_b96 v[6:8], v1 offset:0
+// GFX1250: ds_load_b96 v[6:8], v1                  ; encoding: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x06]
+
+ds_load_b96 v[252:254], v255 offset:4
+// GFX1250: ds_load_b96 v[252:254], v255 offset:4   ; encoding: [0x04,0x00,0xf8,0xdb,0xff,0x00,0x00,0xfc]
+
+ds_load_i16 v5, v1
+// GFX1250: ds_load_i16 v5, v1                      ; encoding: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i16 v5, v1 offset:65535
+// GFX1250: ds_load_i16 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i16 v5, v1 offset:0
+// GFX1250: ds_load_i16 v5, v1                      ; encoding: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i16 v255, v255 offset:4
+// GFX1250: ds_load_i16 v255, v255 offset:4         ; encoding: [0x04,0x00,0xec,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_i8 v5, v1
+// GFX1250: ds_load_i8 v5, v1                       ; encoding: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i8 v5, v1 offset:65535
+// GFX1250: ds_load_i8 v5, v1 offset:65535          ; encoding: [0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i8 v5, v1 offset:0
+// GFX1250: ds_load_i8 v5, v1                       ; encoding: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i8 v255, v255 offset:4
+// GFX1250: ds_load_i8 v255, v255 offset:4          ; encoding: [0x04,0x00,0xe4,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_i8_d16 v5, v1
+// GFX1250: ds_load_i8_d16 v5, v1                   ; encoding: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16 v5, v1 offset:65535
+// GFX1250: ds_load_i8_d16 v5, v1 offset:65535      ; encoding: [0xff,0xff,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16 v5, v1 offset:0
+// GFX1250: ds_load_i8_d16 v5, v1                   ; encoding: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16 v255, v255 offset:4
+// GFX1250: ds_load_i8_d16 v255, v255 offset:4      ; encoding: [0x04,0x00,0x90,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_i8_d16_hi v5, v1
+// GFX1250: ds_load_i8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16_hi v5, v1 offset:65535
+// GFX1250: ds_load_i8_d16_hi v5, v1 offset:65535   ; encoding: [0xff,0xff,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16_hi v5, v1 offset:0
+// GFX1250: ds_load_i8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16_hi v255, v255 offset:4
+// GFX1250: ds_load_i8_d16_hi v255, v255 offset:4   ; encoding: [0x04,0x00,0x94,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_u16 v5, v1
+// GFX1250: ds_load_u16 v5, v1                      ; encoding: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u16 v5, v1 offset:65535
+// GFX1250: ds_load_u16 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u16 v5, v1 offset:0
+// GFX1250: ds_load_u16 v5, v1                      ; encoding: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u16 v255, v255 offset:4
+// GFX1250: ds_load_u16 v255, v255 offset:4         ; encoding: [0x04,0x00,0xf0,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_u16_d16 v5, v1
+// GFX1250: ds_load_u16_d16 v5, v1                  ; encoding: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16 v5, v1 offset:65535
+// GFX1250: ds_load_u16_d16 v5, v1 offset:65535     ; encoding: [0xff,0xff,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16 v5, v1 offset:0
+// GFX1250: ds_load_u16_d16 v5, v1                  ; encoding: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16 v255, v255 offset:4
+// GFX1250: ds_load_u16_d16 v255, v255 offset:4     ; encoding: [0x04,0x00,0x98,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_u16_d16_hi v5, v1
+// GFX1250: ds_load_u16_d16_hi v5, v1               ; encoding: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16_hi v5, v1 offset:65535
+// GFX1250: ds_load_u16_d16_hi v5, v1 offset:65535  ; encoding: [0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16_hi v5, v1 offset:0
+// GFX1250: ds_load_u16_d16_hi v5, v1               ; encoding: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16_hi v255, v255 offset:4
+// GFX1250: ds_load_u16_d16_hi v255, v255 offset:4  ; encoding: [0x04,0x00,0x9c,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_u8 v5, v1
+// GFX1250: ds_load_u8 v5, v1                       ; encoding: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u8 v5, v1 offset:65535
+// GFX1250: ds_load_u8 v5, v1 offset:65535          ; encoding: [0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u8 v5, v1 offset:0
+// GFX1250: ds_load_u8 v5, v1                       ; encoding: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u8 v255, v255 offset:4
+// GFX1250: ds_load_u8 v255, v255 offset:4          ; encoding: [0x04,0x00,0xe8,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_u8_d16 v5, v1
+// GFX1250: ds_load_u8_d16 v5, v1                   ; encoding: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16 v5, v1 offset:65535
+// GFX1250: ds_load_u8_d16 v5, v1 offset:65535      ; encoding: [0xff,0xff,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16 v5, v1 offset:0
+// GFX1250: ds_load_u8_d16 v5, v1                   ; encoding: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16 v255, v255 offset:4
+// GFX1250: ds_load_u8_d16 v255, v255 offset:4      ; encoding: [0x04,0x00,0x88,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_u8_d16_hi v5, v1
+// GFX1250: ds_load_u8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16_hi v5, v1 offset:65535
+// GFX1250: ds_load_u8_d16_hi v5, v1 offset:65535   ; encoding: [0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16_hi v5, v1 offset:0
+// GFX1250: ds_load_u8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16_hi v255, v255 offset:4
+// GFX1250: ds_load_u8_d16_hi v255, v255 offset:4   ; encoding: [0x04,0x00,0x8c,0xda,0xff,0x00,0x00,0xff]
+
+ds_max_num_f32 v1, v2
+// GFX1250: ds_max_num_f32 v1, v2                   ; encoding: [0x00,0x00,0x4c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_num_f32 v1, v2 offset:65535
+// GFX1250: ds_max_num_f32 v1, v2 offset:65535      ; encoding: [0xff,0xff,0x4c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_num_f32 v1, v2 offset:0
+// GFX1250: ds_max_num_f32 v1, v2                   ; encoding: [0x00,0x00,0x4c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_num_f32 v255, v255 offset:4
+// GFX1250: ds_max_num_f32 v255, v255 offset:4      ; encoding: [0x04,0x00,0x4c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_max_num_f64 v1, v[2:3]
+// GFX1250: ds_max_num_f64 v1, v[2:3]               ; encoding: [0x00,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_num_f64 v1, v[2:3] offset:65535
+// GFX1250: ds_max_num_f64 v1, v[2:3] offset:65535  ; encoding: [0xff,0xff,0x4c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_num_f64 v1, v[2:3] offset:0
+// GFX1250: ds_max_num_f64 v1, v[2:3]               ; encoding: [0x00,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_num_f64 v255, v[254:255] offset:4
+// GFX1250: ds_max_num_f64 v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x4c,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_max_i32 v1, v2
+// GFX1250: ds_max_i32 v1, v2                       ; encoding: [0x00,0x00,0x18,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_i32 v1, v2 offset:65535
+// GFX1250: ds_max_i32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x18,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_i32 v1, v2 offset:0
+// GFX1250: ds_max_i32 v1, v2                       ; encoding: [0x00,0x00,0x18,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_i32 v255, v255 offset:4
+// GFX1250: ds_max_i32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x18,0xd8,0xff,0xff,0x00,0x00]
+
+ds_max_i64 v1, v[2:3]
+// GFX1250: ds_max_i64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x18,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_i64 v1, v[2:3] offset:65535
+// GFX1250: ds_max_i64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x18,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_i64 v1, v[2:3] offset:0
+// GFX1250: ds_max_i64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x18,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_i64 v255, v[254:255] offset:4
+// GFX1250: ds_max_i64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x18,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_max_num_rtn_f32 v5, v1, v2
+// GFX1250: ds_max_num_rtn_f32 v5, v1, v2           ; encoding: [0x00,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_num_rtn_f32 v5, v1, v2 offset:65535
+// GFX1250: ds_max_num_rtn_f32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xcc,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_num_rtn_f32 v5, v1, v2 offset:0
+// GFX1250: ds_max_num_rtn_f32 v5, v1, v2           ; encoding: [0x00,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_num_rtn_f32 v255, v255, v255 offset:4
+// GFX1250: ds_max_num_rtn_f32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xcc,0xd8,0xff,0xff,0x00,0xff]
+
+ds_max_num_rtn_f64 v[6:7], v1, v[2:3]
+// GFX1250: ds_max_num_rtn_f64 v[6:7], v1, v[2:3]   ; encoding: [0x00,0x00,0xcc,0xd9,0x01,0x02,0x00,0x06]
+
+ds_max_num_rtn_f64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_max_num_rtn_f64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xcc,0xd9,0x01,0x02,0x00,0x06]
+
+ds_max_num_rtn_f64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_max_num_rtn_f64 v[6:7], v1, v[2:3]   ; encoding: [0x00,0x00,0xcc,0xd9,0x01,0x02,0x00,0x06]
+
+ds_max_num_rtn_f64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_max_num_rtn_f64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xcc,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_max_rtn_i32 v5, v1, v2
+// GFX1250: ds_max_rtn_i32 v5, v1, v2               ; encoding: [0x00,0x00,0x98,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_i32 v5, v1, v2 offset:65535
+// GFX1250: ds_max_rtn_i32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x98,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_i32 v5, v1, v2 offset:0
+// GFX1250: ds_max_rtn_i32 v5, v1, v2               ; encoding: [0x00,0x00,0x98,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_i32 v255, v255, v255 offset:4
+// GFX1250: ds_max_rtn_i32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x98,0xd8,0xff,0xff,0x00,0xff]
+
+ds_max_rtn_i64 v[6:7], v1, v[2:3]
+// GFX1250: ds_max_rtn_i64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x98,0xd9,0x01,0x02,0x00,0x06]
+
+ds_max_rtn_i64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_max_rtn_i64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x98,0xd9,0x01,0x02,0x00,0x06]
+
+ds_max_rtn_i64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_max_rtn_i64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x98,0xd9,0x01,0x02,0x00,0x06]
+
+ds_max_rtn_i64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_max_rtn_i64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x98,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_max_rtn_u32 v5, v1, v2
+// GFX1250: ds_max_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0xa0,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_u32 v5, v1, v2 offset:65535
+// GFX1250: ds_max_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xa0,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_u32 v5, v1, v2 offset:0
+// GFX1250: ds_max_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0xa0,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_u32 v255, v255, v255 offset:4
+// GFX1250: ds_max_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xa0,0xd8,0xff,0xff,0x00,0xff]
+
+ds_max_rtn_u64 v[6:7], v1, v[2:3]
+// GFX1250: ds_max_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0xa0,0xd9,0x01,0x02,0x00,0x06]
+
+ds_max_rtn_u64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_max_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa0,0xd9,0x01,0x02,0x00,0x06]
+
+ds_max_rtn_u64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_max_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0xa0,0xd9,0x01,0x02,0x00,0x06]
+
+ds_max_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_max_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xa0,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_max_u32 v1, v2
+// GFX1250: ds_max_u32 v1, v2                       ; encoding: [0x00,0x00,0x20,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_u32 v1, v2 offset:65535
+// GFX1250: ds_max_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x20,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_u32 v1, v2 offset:0
+// GFX1250: ds_max_u32 v1, v2                       ; encoding: [0x00,0x00,0x20,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_u32 v255, v255 offset:4
+// GFX1250: ds_max_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x20,0xd8,0xff,0xff,0x00,0x00]
+
+ds_max_u64 v1, v[2:3]
+// GFX1250: ds_max_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x20,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_u64 v1, v[2:3] offset:65535
+// GFX1250: ds_max_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x20,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_u64 v1, v[2:3] offset:0
+// GFX1250: ds_max_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x20,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_u64 v255, v[254:255] offset:4
+// GFX1250: ds_max_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x20,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_min_num_f32 v1, v2
+// GFX1250: ds_min_num_f32 v1, v2                   ; encoding: [0x00,0x00,0x48,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_num_f32 v1, v2 offset:65535
+// GFX1250: ds_min_num_f32 v1, v2 offset:65535      ; encoding: [0xff,0xff,0x48,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_num_f32 v1, v2 offset:0
+// GFX1250: ds_min_num_f32 v1, v2                   ; encoding: [0x00,0x00,0x48,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_num_f32 v255, v255 offset:4
+// GFX1250: ds_min_num_f32 v255, v255 offset:4      ; encoding: [0x04,0x00,0x48,0xd8,0xff,0xff,0x00,0x00]
+
+ds_min_num_f64 v1, v[2:3]
+// GFX1250: ds_min_num_f64 v1, v[2:3]               ; encoding: [0x00,0x00,0x48,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_num_f64 v1, v[2:3] offset:65535
+// GFX1250: ds_min_num_f64 v1, v[2:3] offset:65535  ; encoding: [0xff,0xff,0x48,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_num_f64 v1, v[2:3] offset:0
+// GFX1250: ds_min_num_f64 v1, v[2:3]               ; encoding: [0x00,0x00,0x48,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_num_f64 v255, v[254:255] offset:4
+// GFX1250: ds_min_num_f64 v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x48,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_min_i32 v1, v2
+// GFX1250: ds_min_i32 v1, v2                       ; encoding: [0x00,0x00,0x14,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_i32 v1, v2 offset:65535
+// GFX1250: ds_min_i32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x14,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_i32 v1, v2 offset:0
+// GFX1250: ds_min_i32 v1, v2                       ; encoding: [0x00,0x00,0x14,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_i32 v255, v255 offset:4
+// GFX1250: ds_min_i32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x14,0xd8,0xff,0xff,0x00,0x00]
+
+ds_min_i64 v1, v[2:3]
+// GFX1250: ds_min_i64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x14,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_i64 v1, v[2:3] offset:65535
+// GFX1250: ds_min_i64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x14,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_i64 v1, v[2:3] offset:0
+// GFX1250: ds_min_i64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x14,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_i64 v255, v[254:255] offset:4
+// GFX1250: ds_min_i64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x14,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_min_num_rtn_f32 v5, v1, v2
+// GFX1250: ds_min_num_rtn_f32 v5, v1, v2           ; encoding: [0x00,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_num_rtn_f32 v5, v1, v2 offset:65535
+// GFX1250: ds_min_num_rtn_f32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xc8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_num_rtn_f32 v5, v1, v2 offset:0
+// GFX1250: ds_min_num_rtn_f32 v5, v1, v2           ; encoding: [0x00,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_num_rtn_f32 v255, v255, v255 offset:4
+// GFX1250: ds_min_num_rtn_f32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xc8,0xd8,0xff,0xff,0x00,0xff]
+
+ds_min_num_rtn_f64 v[6:7], v1, v[2:3]
+// GFX1250: ds_min_num_rtn_f64 v[6:7], v1, v[2:3]   ; encoding: [0x00,0x00,0xc8,0xd9,0x01,0x02,0x00,0x06]
+
+ds_min_num_rtn_f64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_min_num_rtn_f64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xc8,0xd9,0x01,0x02,0x00,0x06]
+
+ds_min_num_rtn_f64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_min_num_rtn_f64 v[6:7], v1, v[2:3]   ; encoding: [0x00,0x00,0xc8,0xd9,0x01,0x02,0x00,0x06]
+
+ds_min_num_rtn_f64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_min_num_rtn_f64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xc8,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_min_rtn_i32 v5, v1, v2
+// GFX1250: ds_min_rtn_i32 v5, v1, v2               ; encoding: [0x00,0x00,0x94,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_i32 v5, v1, v2 offset:65535
+// GFX1250: ds_min_rtn_i32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x94,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_i32 v5, v1, v2 offset:0
+// GFX1250: ds_min_rtn_i32 v5, v1, v2               ; encoding: [0x00,0x00,0x94,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_i32 v255, v255, v255 offset:4
+// GFX1250: ds_min_rtn_i32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x94,0xd8,0xff,0xff,0x00,0xff]
+
+ds_min_rtn_i64 v[6:7], v1, v[2:3]
+// GFX1250: ds_min_rtn_i64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x94,0xd9,0x01,0x02,0x00,0x06]
+
+ds_min_rtn_i64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_min_rtn_i64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x94,0xd9,0x01,0x02,0x00,0x06]
+
+ds_min_rtn_i64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_min_rtn_i64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x94,0xd9,0x01,0x02,0x00,0x06]
+
+ds_min_rtn_i64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_min_rtn_i64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x94,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_min_rtn_u32 v5, v1, v2
+// GFX1250: ds_min_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x9c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_u32 v5, v1, v2 offset:65535
+// GFX1250: ds_min_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x9c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_u32 v5, v1, v2 offset:0
+// GFX1250: ds_min_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x9c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_u32 v255, v255, v255 offset:4
+// GFX1250: ds_min_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x9c,0xd8,0xff,0xff,0x00,0xff]
+
+ds_min_rtn_u64 v[6:7], v1, v[2:3]
+// GFX1250: ds_min_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x9c,0xd9,0x01,0x02,0x00,0x06]
+
+ds_min_rtn_u64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_min_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x9c,0xd9,0x01,0x02,0x00,0x06]
+
+ds_min_rtn_u64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_min_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x9c,0xd9,0x01,0x02,0x00,0x06]
+
+ds_min_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_min_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x9c,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_min_u32 v1, v2
+// GFX1250: ds_min_u32 v1, v2                       ; encoding: [0x00,0x00,0x1c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_u32 v1, v2 offset:65535
+// GFX1250: ds_min_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x1c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_u32 v1, v2 offset:0
+// GFX1250: ds_min_u32 v1, v2                       ; encoding: [0x00,0x00,0x1c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_u32 v255, v255 offset:4
+// GFX1250: ds_min_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x1c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_min_u64 v1, v[2:3]
+// GFX1250: ds_min_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_u64 v1, v[2:3] offset:65535
+// GFX1250: ds_min_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x1c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_u64 v1, v[2:3] offset:0
+// GFX1250: ds_min_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_u64 v255, v[254:255] offset:4
+// GFX1250: ds_min_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x1c,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_mskor_b32 v1, v2, v3
+// GFX1250: ds_mskor_b32 v1, v2, v3                 ; encoding: [0x00,0x00,0x30,0xd8,0x01,0x02,0x03,0x00]
+
+ds_mskor_b32 v1, v2, v3 offset:65535
+// GFX1250: ds_mskor_b32 v1, v2, v3 offset:65535    ; encoding: [0xff,0xff,0x30,0xd8,0x01,0x02,0x03,0x00]
+
+ds_mskor_b32 v1, v2, v3 offset:0
+// GFX1250: ds_mskor_b32 v1, v2, v3                 ; encoding: [0x00,0x00,0x30,0xd8,0x01,0x02,0x03,0x00]
+
+ds_mskor_b32 v255, v255, v255 offset:4
+// GFX1250: ds_mskor_b32 v255, v255, v255 offset:4  ; encoding: [0x04,0x00,0x30,0xd8,0xff,0xff,0xff,0x00]
+
+ds_mskor_b64 v1, v[2:3], v[4:5]
+// GFX1250: ds_mskor_b64 v1, v[2:3], v[4:5]         ; encoding: [0x00,0x00,0x30,0xd9,0x01,0x02,0x04,0x00]
+
+ds_mskor_b64 v1, v[2:3], v[4:5] offset:65535
+// GFX1250: ds_mskor_b64 v1, v[2:3], v[4:5] offset:65535 ; encoding: [0xff,0xff,0x30,0xd9,0x01,0x02,0x04,0x00]
+
+ds_mskor_b64 v1, v[2:3], v[4:5] offset:0
+// GFX1250: ds_mskor_b64 v1, v[2:3], v[4:5]         ; encoding: [0x00,0x00,0x30,0xd9,0x01,0x02,0x04,0x00]
+
+ds_mskor_b64 v255, v[254:255], v[254:255] offset:4
+// GFX1250: ds_mskor_b64 v255, v[254:255], v[254:255] offset:4 ; encoding: [0x04,0x00,0x30,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_mskor_rtn_b32 v5, v1, v2, v3
+// GFX1250: ds_mskor_rtn_b32 v5, v1, v2, v3         ; encoding: [0x00,0x00,0xb0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_mskor_rtn_b32 v5, v1, v2, v3 offset:65535
+// GFX1250: ds_mskor_rtn_b32 v5, v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xb0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_mskor_rtn_b32 v5, v1, v2, v3 offset:0
+// GFX1250: ds_mskor_rtn_b32 v5, v1, v2, v3         ; encoding: [0x00,0x00,0xb0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_mskor_rtn_b32 v255, v255, v255, v255 offset:4
+// GFX1250: ds_mskor_rtn_b32 v255, v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xb0,0xd8,0xff,0xff,0xff,0xff]
+
+ds_mskor_rtn_b64 v[6:7], v1, v[2:3], v[4:5]
+// GFX1250: ds_mskor_rtn_b64 v[6:7], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xb0,0xd9,0x01,0x02,0x04,0x06]
+
+ds_mskor_rtn_b64 v[6:7], v1, v[2:3], v[4:5] offset:65535
+// GFX1250: ds_mskor_rtn_b64 v[6:7], v1, v[2:3], v[4:5] offset:65535 ; encoding: [0xff,0xff,0xb0,0xd9,0x01,0x02,0x04,0x06]
+
+ds_mskor_rtn_b64 v[6:7], v1, v[2:3], v[4:5] offset:0
+// GFX1250: ds_mskor_rtn_b64 v[6:7], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xb0,0xd9,0x01,0x02,0x04,0x06]
+
+ds_mskor_rtn_b64 v[254:255], v255, v[254:255], v[254:255] offset:4
+// GFX1250: ds_mskor_rtn_b64 v[254:255], v255, v[254:255], v[254:255] offset:4 ; encoding: [0x04,0x00,0xb0,0xd9,0xff,0xfe,0xfe,0xfe]
+
+ds_or_b32 v1, v2
+// GFX1250: ds_or_b32 v1, v2                        ; encoding: [0x00,0x00,0x28,0xd8,0x01,0x02,0x00,0x00]
+
+ds_or_b32 v1, v2 offset:65535
+// GFX1250: ds_or_b32 v1, v2 offset:65535           ; encoding: [0xff,0xff,0x28,0xd8,0x01,0x02,0x00,0x00]
+
+ds_or_b32 v1, v2 offset:0
+// GFX1250: ds_or_b32 v1, v2                        ; encoding: [0x00,0x00,0x28,0xd8,0x01,0x02,0x00,0x00]
+
+ds_or_b32 v255, v255 offset:4
+// GFX1250: ds_or_b32 v255, v255 offset:4           ; encoding: [0x04,0x00,0x28,0xd8,0xff,0xff,0x00,0x00]
+
+ds_or_b64 v1, v[2:3]
+// GFX1250: ds_or_b64 v1, v[2:3]                    ; encoding: [0x00,0x00,0x28,0xd9,0x01,0x02,0x00,0x00]
+
+ds_or_b64 v1, v[2:3] offset:65535
+// GFX1250: ds_or_b64 v1, v[2:3] offset:65535       ; encoding: [0xff,0xff,0x28,0xd9,0x01,0x02,0x00,0x00]
+
+ds_or_b64 v1, v[2:3] offset:0
+// GFX1250: ds_or_b64 v1, v[2:3]                    ; encoding: [0x00,0x00,0x28,0xd9,0x01,0x02,0x00,0x00]
+
+ds_or_b64 v255, v[254:255] offset:4
+// GFX1250: ds_or_b64 v255, v[254:255] offset:4     ; encoding: [0x04,0x00,0x28,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_or_rtn_b32 v5, v1, v2
+// GFX1250: ds_or_rtn_b32 v5, v1, v2                ; encoding: [0x00,0x00,0xa8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_or_rtn_b32 v5, v1, v2 offset:65535
+// GFX1250: ds_or_rtn_b32 v5, v1, v2 offset:65535   ; encoding: [0xff,0xff,0xa8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_or_rtn_b32 v5, v1, v2 offset:0
+// GFX1250: ds_or_rtn_b32 v5, v1, v2                ; encoding: [0x00,0x00,0xa8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_or_rtn_b32 v255, v255, v255 offset:4
+// GFX1250: ds_or_rtn_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xa8,0xd8,0xff,0xff,0x00,0xff]
+
+ds_or_rtn_b64 v[6:7], v1, v[2:3]
+// GFX1250: ds_or_rtn_b64 v[6:7], v1, v[2:3]        ; encoding: [0x00,0x00,0xa8,0xd9,0x01,0x02,0x00,0x06]
+
+ds_or_rtn_b64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_or_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa8,0xd9,0x01,0x02,0x00,0x06]
+
+ds_or_rtn_b64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_or_rtn_b64 v[6:7], v1, v[2:3]        ; encoding: [0x00,0x00,0xa8,0xd9,0x01,0x02,0x00,0x06]
+
+ds_or_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_or_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xa8,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_permute_b32 v5, v1, v2
+// GFX1250: ds_permute_b32 v5, v1, v2               ; encoding: [0x00,0x00,0xc8,0xda,0x01,0x02,0x00,0x05]
+
+ds_permute_b32 v5, v1, v2 offset:65535
+// GFX1250: ds_permute_b32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xc8,0xda,0x01,0x02,0x00,0x05]
+
+ds_permute_b32 v5, v1, v2 offset:0
+// GFX1250: ds_permute_b32 v5, v1, v2               ; encoding: [0x00,0x00,0xc8,0xda,0x01,0x02,0x00,0x05]
+
+ds_permute_b32 v255, v255, v255 offset:4
+// GFX1250: ds_permute_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xc8,0xda,0xff,0xff,0x00,0xff]
+
+ds_pk_add_f16 v2, v1
+// GFX1250: ds_pk_add_f16 v2, v1                    ; encoding: [0x00,0x00,0x68,0xda,0x02,0x01,0x00,0x00]
+
+ds_pk_add_f16 v2, v1 offset:0
+// GFX1250: ds_pk_add_f16 v2, v1                    ; encoding: [0x00,0x00,0x68,0xda,0x02,0x01,0x00,0x00]
+
+ds_pk_add_f16 v2, v1 offset:4660
+// GFX1250: ds_pk_add_f16 v2, v1 offset:4660        ; encoding: [0x34,0x12,0x68,0xda,0x02,0x01,0x00,0x00]
+
+ds_pk_add_f16 v2, v1 offset:65535
+// GFX1250: ds_pk_add_f16 v2, v1 offset:65535       ; encoding: [0xff,0xff,0x68,0xda,0x02,0x01,0x00,0x00]
+
+ds_pk_add_f16 v255, v255
+// GFX1250: ds_pk_add_f16 v255, v255                ; encoding: [0x00,0x00,0x68,0xda,0xff,0xff,0x00,0x00]
+
+ds_pk_add_f16 v255, v255 offset:0
+// GFX1250: ds_pk_add_f16 v255, v255                ; encoding: [0x00,0x00,0x68,0xda,0xff,0xff,0x00,0x00]
+
+ds_pk_add_f16 v255, v255 offset:4660
+// GFX1250: ds_pk_add_f16 v255, v255 offset:4660    ; encoding: [0x34,0x12,0x68,0xda,0xff,0xff,0x00,0x00]
+
+ds_pk_add_f16 v255, v255 offset:65535
+// GFX1250: ds_pk_add_f16 v255, v255 offset:65535   ; encoding: [0xff,0xff,0x68,0xda,0xff,0xff,0x00,0x00]
+
+ds_pk_add_f16 v0, v0
+// GFX1250: ds_pk_add_f16 v0, v0                    ; encoding: [0x00,0x00,0x68,0xda,0x00,0x00,0x00,0x00]
+
+ds_pk_add_bf16 v2, v1
+// GFX1250: ds_pk_add_bf16 v2, v1                   ; encoding: [0x00,0x00,0x6c,0xda,0x02,0x01,0x00,0x00]
+
+ds_pk_add_bf16 v2, v1 offset:0
+// GFX1250: ds_pk_add_bf16 v2, v1                   ; encoding: [0x00,0x00,0x6c,0xda,0x02,0x01,0x00,0x00]
+
+ds_pk_add_bf16 v255, v255
+// GFX1250: ds_pk_add_bf16 v255, v255               ; encoding: [0x00,0x00,0x6c,0xda,0xff,0xff,0x00,0x00]
+
+ds_pk_add_bf16 v255, v255 offset:4660
+// GFX1250: ds_pk_add_bf16 v255, v255 offset:4660   ; encoding: [0x34,0x12,0x6c,0xda,0xff,0xff,0x00,0x00]
+
+ds_pk_add_bf16 v0, v0
+// GFX1250: ds_pk_add_bf16 v0, v0                   ; encoding: [0x00,0x00,0x6c,0xda,0x00,0x00,0x00,0x00]
+
+ds_pk_add_bf16 v0, v0 offset:65535
+// GFX1250: ds_pk_add_bf16 v0, v0 offset:65535      ; encoding: [0xff,0xff,0x6c,0xda,0x00,0x00,0x00,0x00]
+
+ds_pk_add_rtn_f16 v3, v2, v1
+// GFX1250: ds_pk_add_rtn_f16 v3, v2, v1            ; encoding: [0x00,0x00,0xa8,0xda,0x02,0x01,0x00,0x03]
+
+ds_pk_add_rtn_f16 v3, v2, v1 offset:4660
+// GFX1250: ds_pk_add_rtn_f16 v3, v2, v1 offset:4660 ; encoding: [0x34,0x12,0xa8,0xda,0x02,0x01,0x00,0x03]
+
+ds_pk_add_rtn_f16 v255, v0, v200
+// GFX1250: ds_pk_add_rtn_f16 v255, v0, v200        ; encoding: [0x00,0x00,0xa8,0xda,0x00,0xc8,0x00,0xff]
+
+ds_pk_add_rtn_f16 v255, v0, v200 offset:65535
+// GFX1250: ds_pk_add_rtn_f16 v255, v0, v200 offset:65535 ; encoding: [0xff,0xff,0xa8,0xda,0x00,0xc8,0x00,0xff]
+
+ds_pk_add_rtn_f16 v255, v255, v255
+// GFX1250: ds_pk_add_rtn_f16 v255, v255, v255      ; encoding: [0x00,0x00,0xa8,0xda,0xff,0xff,0x00,0xff]
+
+ds_pk_add_rtn_bf16 v3, v2, v1
+// GFX1250: ds_pk_add_rtn_bf16 v3, v2, v1           ; encoding: [0x00,0x00,0xac,0xda,0x02,0x01,0x00,0x03]
+
+ds_pk_add_rtn_bf16 v3, v2, v1 offset:4660
+// GFX1250: ds_pk_add_rtn_bf16 v3, v2, v1 offset:4660 ; encoding: [0x34,0x12,0xac,0xda,0x02,0x01,0x00,0x03]
+
+ds_pk_add_rtn_bf16 v255, v0, v200
+// GFX1250: ds_pk_add_rtn_bf16 v255, v0, v200       ; encoding: [0x00,0x00,0xac,0xda,0x00,0xc8,0x00,0xff]
+
+ds_pk_add_rtn_bf16 v255, v255, v255
+// GFX1250: ds_pk_add_rtn_bf16 v255, v255, v255     ; encoding: [0x00,0x00,0xac,0xda,0xff,0xff,0x00,0xff]
+
+ds_pk_add_rtn_bf16 v255, v255, v255 offset:65535
+// GFX1250: ds_pk_add_rtn_bf16 v255, v255, v255 offset:65535 ; encoding: [0xff,0xff,0xac,0xda,0xff,0xff,0x00,0xff]
+
+ds_read2_b32 v[6:7], v1
+// GFX1250: ds_load_2addr_b32 v[6:7], v1            ; encoding: [0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x06]
+
+ds_read2_b32 v[6:7], v1 offset0:127 offset1:255
+// GFX1250: ds_load_2addr_b32 v[6:7], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0x06]
+
+ds_read2_b32 v[6:7], v1 offset0:0 offset1:0
+// GFX1250: ds_load_2addr_b32 v[6:7], v1            ; encoding: [0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x06]
+
+ds_read2_b32 v[254:255], v255 offset0:16 offset1:1
+// GFX1250: ds_load_2addr_b32 v[254:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xdc,0xd8,0xff,0x00,0x00,0xfe]
+
+ds_read2_b64 v[6:9], v1
+// GFX1250: ds_load_2addr_b64 v[6:9], v1            ; encoding: [0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x06]
+
+ds_read2_b64 v[6:9], v1 offset0:127 offset1:255
+// GFX1250: ds_load_2addr_b64 v[6:9], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0x06]
+
+ds_read2_b64 v[6:9], v1 offset0:0 offset1:0
+// GFX1250: ds_load_2addr_b64 v[6:9], v1            ; encoding: [0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x06]
+
+ds_read2_b64 v[252:255], v255 offset0:16 offset1:1
+// GFX1250: ds_load_2addr_b64 v[252:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xdc,0xd9,0xff,0x00,0x00,0xfc]
+
+ds_read2st64_b32 v[6:7], v1
+// GFX1250: ds_load_2addr_stride64_b32 v[6:7], v1   ; encoding: [0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x06]
+
+ds_read2st64_b32 v[6:7], v1 offset0:127 offset1:255
+// GFX1250: ds_load_2addr_stride64_b32 v[6:7], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0x06]
+
+ds_read2st64_b32 v[6:7], v1 offset0:0 offset1:0
+// GFX1250: ds_load_2addr_stride64_b32 v[6:7], v1   ; encoding: [0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x06]
+
+ds_read2st64_b32 v[254:255], v255 offset0:16 offset1:1
+// GFX1250: ds_load_2addr_stride64_b32 v[254:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xe0,0xd8,0xff,0x00,0x00,0xfe]
+
+ds_read2st64_b64 v[6:9], v1
+// GFX1250: ds_load_2addr_stride64_b64 v[6:9], v1   ; encoding: [0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x06]
+
+ds_read2st64_b64 v[6:9], v1 offset0:127 offset1:255
+// GFX1250: ds_load_2addr_stride64_b64 v[6:9], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0x06]
+
+ds_read2st64_b64 v[6:9], v1 offset0:0 offset1:0
+// GFX1250: ds_load_2addr_stride64_b64 v[6:9], v1   ; encoding: [0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x06]
+
+ds_read2st64_b64 v[252:255], v255 offset0:16 offset1:1
+// GFX1250: ds_load_2addr_stride64_b64 v[252:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xe0,0xd9,0xff,0x00,0x00,0xfc]
+
+ds_read_addtid_b32 v5
+// GFX1250: ds_load_addtid_b32 v5                   ; encoding: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_read_addtid_b32 v5 offset:65535
+// GFX1250: ds_load_addtid_b32 v5 offset:65535      ; encoding: [0xff,0xff,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_read_addtid_b32 v5 offset:0
+// GFX1250: ds_load_addtid_b32 v5                   ; encoding: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_read_addtid_b32 v255 offset:4
+// GFX1250: ds_load_addtid_b32 v255 offset:4        ; encoding: [0x04,0x00,0xc4,0xda,0x00,0x00,0x00,0xff]
+
+ds_read_b128 v[6:9], v1
+// GFX1250: ds_load_b128 v[6:9], v1                 ; encoding: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x06]
+
+ds_read_b128 v[6:9], v1 offset:65535
+// GFX1250: ds_load_b128 v[6:9], v1 offset:65535    ; encoding: [0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0x06]
+
+ds_read_b128 v[6:9], v1 offset:0
+// GFX1250: ds_load_b128 v[6:9], v1                 ; encoding: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x06]
+
+ds_read_b128 v[252:255], v255 offset:4
+// GFX1250: ds_load_b128 v[252:255], v255 offset:4  ; encoding: [0x04,0x00,0xfc,0xdb,0xff,0x00,0x00,0xfc]
+
+ds_read_b32 v5, v1
+// GFX1250: ds_load_b32 v5, v1                      ; encoding: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_b32 v5, v1 offset:65535
+// GFX1250: ds_load_b32 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_b32 v5, v1 offset:0
+// GFX1250: ds_load_b32 v5, v1                      ; encoding: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_b32 v255, v255 offset:4
+// GFX1250: ds_load_b32 v255, v255 offset:4         ; encoding: [0x04,0x00,0xd8,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_b64 v[6:7], v1
+// GFX1250: ds_load_b64 v[6:7], v1                  ; encoding: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x06]
+
+ds_read_b64 v[6:7], v1 offset:65535
+// GFX1250: ds_load_b64 v[6:7], v1 offset:65535     ; encoding: [0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0x06]
+
+ds_read_b64 v[6:7], v1 offset:0
+// GFX1250: ds_load_b64 v[6:7], v1                  ; encoding: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x06]
+
+ds_read_b64 v[254:255], v255 offset:4
+// GFX1250: ds_load_b64 v[254:255], v255 offset:4   ; encoding: [0x04,0x00,0xd8,0xd9,0xff,0x00,0x00,0xfe]
+
+ds_read_b96 v[6:8], v1
+// GFX1250: ds_load_b96 v[6:8], v1                  ; encoding: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x06]
+
+ds_read_b96 v[6:8], v1 offset:65535
+// GFX1250: ds_load_b96 v[6:8], v1 offset:65535     ; encoding: [0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0x06]
+
+ds_read_b96 v[6:8], v1 offset:0
+// GFX1250: ds_load_b96 v[6:8], v1                  ; encoding: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x06]
+
+ds_read_b96 v[252:254], v255 offset:4
+// GFX1250: ds_load_b96 v[252:254], v255 offset:4   ; encoding: [0x04,0x00,0xf8,0xdb,0xff,0x00,0x00,0xfc]
+
+ds_read_i16 v5, v1
+// GFX1250: ds_load_i16 v5, v1                      ; encoding: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i16 v5, v1 offset:65535
+// GFX1250: ds_load_i16 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i16 v5, v1 offset:0
+// GFX1250: ds_load_i16 v5, v1                      ; encoding: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i16 v255, v255 offset:4
+// GFX1250: ds_load_i16 v255, v255 offset:4         ; encoding: [0x04,0x00,0xec,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_i8 v5, v1
+// GFX1250: ds_load_i8 v5, v1                       ; encoding: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i8 v5, v1 offset:65535
+// GFX1250: ds_load_i8 v5, v1 offset:65535          ; encoding: [0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i8 v5, v1 offset:0
+// GFX1250: ds_load_i8 v5, v1                       ; encoding: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i8 v255, v255 offset:4
+// GFX1250: ds_load_i8 v255, v255 offset:4          ; encoding: [0x04,0x00,0xe4,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_i8_d16 v5, v1
+// GFX1250: ds_load_i8_d16 v5, v1                   ; encoding: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16 v5, v1 offset:65535
+// GFX1250: ds_load_i8_d16 v5, v1 offset:65535      ; encoding: [0xff,0xff,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16 v5, v1 offset:0
+// GFX1250: ds_load_i8_d16 v5, v1                   ; encoding: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16 v255, v255 offset:4
+// GFX1250: ds_load_i8_d16 v255, v255 offset:4      ; encoding: [0x04,0x00,0x90,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_i8_d16_hi v5, v1
+// GFX1250: ds_load_i8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16_hi v5, v1 offset:65535
+// GFX1250: ds_load_i8_d16_hi v5, v1 offset:65535   ; encoding: [0xff,0xff,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16_hi v5, v1 offset:0
+// GFX1250: ds_load_i8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16_hi v255, v255 offset:4
+// GFX1250: ds_load_i8_d16_hi v255, v255 offset:4   ; encoding: [0x04,0x00,0x94,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_u16 v5, v1
+// GFX1250: ds_load_u16 v5, v1                      ; encoding: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u16 v5, v1 offset:65535
+// GFX1250: ds_load_u16 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u16 v5, v1 offset:0
+// GFX1250: ds_load_u16 v5, v1                      ; encoding: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u16 v255, v255 offset:4
+// GFX1250: ds_load_u16 v255, v255 offset:4         ; encoding: [0x04,0x00,0xf0,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_u16_d16 v5, v1
+// GFX1250: ds_load_u16_d16 v5, v1                  ; encoding: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16 v5, v1 offset:65535
+// GFX1250: ds_load_u16_d16 v5, v1 offset:65535     ; encoding: [0xff,0xff,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16 v5, v1 offset:0
+// GFX1250: ds_load_u16_d16 v5, v1                  ; encoding: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16 v255, v255 offset:4
+// GFX1250: ds_load_u16_d16 v255, v255 offset:4     ; encoding: [0x04,0x00,0x98,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_u16_d16_hi v5, v1
+// GFX1250: ds_load_u16_d16_hi v5, v1               ; encoding: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16_hi v5, v1 offset:65535
+// GFX1250: ds_load_u16_d16_hi v5, v1 offset:65535  ; encoding: [0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16_hi v5, v1 offset:0
+// GFX1250: ds_load_u16_d16_hi v5, v1               ; encoding: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16_hi v255, v255 offset:4
+// GFX1250: ds_load_u16_d16_hi v255, v255 offset:4  ; encoding: [0x04,0x00,0x9c,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_u8 v5, v1
+// GFX1250: ds_load_u8 v5, v1                       ; encoding: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u8 v5, v1 offset:65535
+// GFX1250: ds_load_u8 v5, v1 offset:65535          ; encoding: [0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u8 v5, v1 offset:0
+// GFX1250: ds_load_u8 v5, v1                       ; encoding: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u8 v255, v255 offset:4
+// GFX1250: ds_load_u8 v255, v255 offset:4          ; encoding: [0x04,0x00,0xe8,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_u8_d16 v5, v1
+// GFX1250: ds_load_u8_d16 v5, v1                   ; encoding: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16 v5, v1 offset:65535
+// GFX1250: ds_load_u8_d16 v5, v1 offset:65535      ; encoding: [0xff,0xff,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16 v5, v1 offset:0
+// GFX1250: ds_load_u8_d16 v5, v1                   ; encoding: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16 v255, v255 offset:4
+// GFX1250: ds_load_u8_d16 v255, v255 offset:4      ; encoding: [0x04,0x00,0x88,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_u8_d16_hi v5, v1
+// GFX1250: ds_load_u8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16_hi v5, v1 offset:65535
+// GFX1250: ds_load_u8_d16_hi v5, v1 offset:65535   ; encoding: [0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16_hi v5, v1 offset:0
+// GFX1250: ds_load_u8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16_hi v255, v255 offset:4
+// GFX1250: ds_load_u8_d16_hi v255, v255 offset:4   ; encoding: [0x04,0x00,0x8c,0xda,0xff,0x00,0x00,0xff]
+
+ds_rsub_rtn_u32 v5, v1, v2
+// GFX1250: ds_rsub_rtn_u32 v5, v1, v2              ; encoding: [0x00,0x00,0x88,0xd8,0x01,0x02,0x00,0x05]
+
+ds_rsub_rtn_u32 v5, v1, v2 offset:65535
+// GFX1250: ds_rsub_rtn_u32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0x88,0xd8,0x01,0x02,0x00,0x05]
+
+ds_rsub_rtn_u32 v5, v1, v2 offset:0
+// GFX1250: ds_rsub_rtn_u32 v5, v1, v2              ; encoding: [0x00,0x00,0x88,0xd8,0x01,0x02,0x00,0x05]
+
+ds_rsub_rtn_u32 v255, v255, v255 offset:4
+// GFX1250: ds_rsub_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x88,0xd8,0xff,0xff,0x00,0xff]
+
+ds_rsub_rtn_u64 v[6:7], v1, v[2:3]
+// GFX1250: ds_rsub_rtn_u64 v[6:7], v1, v[2:3]      ; encoding: [0x00,0x00,0x88,0xd9,0x01,0x02,0x00,0x06]
+
+ds_rsub_rtn_u64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_rsub_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x88,0xd9,0x01,0x02,0x00,0x06]
+
+ds_rsub_rtn_u64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_rsub_rtn_u64 v[6:7], v1, v[2:3]      ; encoding: [0x00,0x00,0x88,0xd9,0x01,0x02,0x00,0x06]
+
+ds_rsub_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_rsub_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x88,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_rsub_u32 v1, v2
+// GFX1250: ds_rsub_u32 v1, v2                      ; encoding: [0x00,0x00,0x08,0xd8,0x01,0x02,0x00,0x00]
+
+ds_rsub_u32 v1, v2 offset:65535
+// GFX1250: ds_rsub_u32 v1, v2 offset:65535         ; encoding: [0xff,0xff,0x08,0xd8,0x01,0x02,0x00,0x00]
+
+ds_rsub_u32 v1, v2 offset:0
+// GFX1250: ds_rsub_u32 v1, v2                      ; encoding: [0x00,0x00,0x08,0xd8,0x01,0x02,0x00,0x00]
+
+ds_rsub_u32 v255, v255 offset:4
+// GFX1250: ds_rsub_u32 v255, v255 offset:4         ; encoding: [0x04,0x00,0x08,0xd8,0xff,0xff,0x00,0x00]
+
+ds_rsub_u64 v1, v[2:3]
+// GFX1250: ds_rsub_u64 v1, v[2:3]                  ; encoding: [0x00,0x00,0x08,0xd9,0x01,0x02,0x00,0x00]
+
+ds_rsub_u64 v1, v[2:3] offset:65535
+// GFX1250: ds_rsub_u64 v1, v[2:3] offset:65535     ; encoding: [0xff,0xff,0x08,0xd9,0x01,0x02,0x00,0x00]
+
+ds_rsub_u64 v1, v[2:3] offset:0
+// GFX1250: ds_rsub_u64 v1, v[2:3]                  ; encoding: [0x00,0x00,0x08,0xd9,0x01,0x02,0x00,0x00]
+
+ds_rsub_u64 v255, v[254:255] offset:4
+// GFX1250: ds_rsub_u64 v255, v[254:255] offset:4   ; encoding: [0x04,0x00,0x08,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_store_2addr_b32 v1, v2, v3
+// GFX1250: ds_store_2addr_b32 v1, v2, v3           ; encoding: [0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_b32 v1, v2, v3 offset0:127 offset1:255
+// GFX1250: ds_store_2addr_b32 v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_b32 v1, v2, v3 offset0:0 offset1:0
+// GFX1250: ds_store_2addr_b32 v1, v2, v3           ; encoding: [0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_b32 v255, v255, v255 offset0:16 offset1:1
+// GFX1250: ds_store_2addr_b32 v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0x38,0xd8,0xff,0xff,0xff,0x00]
+
+ds_store_2addr_b64 v1, v[2:3], v[4:5]
+// GFX1250: ds_store_2addr_b64 v1, v[2:3], v[4:5]   ; encoding: [0x00,0x00,0x38,0xd9,0x01,0x02,0x04,0x00]
+
+ds_store_2addr_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255
+// GFX1250: ds_store_2addr_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd9,0x01,0x02,0x04,0x00]
+
+ds_store_2addr_b64 v1, v[2:3], v[4:5] offset0:0 offset1:0
+// GFX1250: ds_store_2addr_b64 v1, v[2:3], v[4:5]   ; encoding: [0x00,0x00,0x38,0xd9,0x01,0x02,0x04,0x00]
+
+ds_store_2addr_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX1250: ds_store_2addr_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0x38,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_store_2addr_stride64_b32 v1, v2, v3
+// GFX1250: ds_store_2addr_stride64_b32 v1, v2, v3  ; encoding: [0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_stride64_b32 v1, v2, v3 offset0:127 offset1:255
+// GFX1250: ds_store_2addr_stride64_b32 v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_stride64_b32 v1, v2, v3 offset0:0 offset1:0
+// GFX1250: ds_store_2addr_stride64_b32 v1, v2, v3  ; encoding: [0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_stride64_b32 v255, v255, v255 offset0:16 offset1:1
+// GFX1250: ds_store_2addr_stride64_b32 v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0x3c,0xd8,0xff,0xff,0xff,0x00]
+
+ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5]
+// GFX1250: ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0x3c,0xd9,0x01,0x02,0x04,0x00]
+
+ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255
+// GFX1250: ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd9,0x01,0x02,0x04,0x00]
+
+ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] offset0:0 offset1:0
+// GFX1250: ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0x3c,0xd9,0x01,0x02,0x04,0x00]
+
+ds_store_2addr_stride64_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX1250: ds_store_2addr_stride64_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0x3c,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_store_addtid_b32 v1
+// GFX1250: ds_store_addtid_b32 v1                  ; encoding: [0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_store_addtid_b32 v1 offset:65535
+// GFX1250: ds_store_addtid_b32 v1 offset:65535     ; encoding: [0xff,0xff,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_store_addtid_b32 v1 offset:0
+// GFX1250: ds_store_addtid_b32 v1                  ; encoding: [0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_store_addtid_b32 v255 offset:4
+// GFX1250: ds_store_addtid_b32 v255 offset:4       ; encoding: [0x04,0x00,0xc0,0xda,0x00,0xff,0x00,0x00]
+
+ds_store_b128 v1, v[2:5]
+// GFX1250: ds_store_b128 v1, v[2:5]                ; encoding: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b128 v1, v[2:5] offset:65535
+// GFX1250: ds_store_b128 v1, v[2:5] offset:65535   ; encoding: [0xff,0xff,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b128 v1, v[2:5] offset:0
+// GFX1250: ds_store_b128 v1, v[2:5]                ; encoding: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b128 v255, v[252:255] offset:4
+// GFX1250: ds_store_b128 v255, v[252:255] offset:4 ; encoding: [0x04,0x00,0x7c,0xdb,0xff,0xfc,0x00,0x00]
+
+ds_store_b16 v1, v2
+// GFX1250: ds_store_b16 v1, v2                     ; encoding: [0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b16 v1, v2 offset:65535
+// GFX1250: ds_store_b16 v1, v2 offset:65535        ; encoding: [0xff,0xff,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b16 v1, v2 offset:0
+// GFX1250: ds_store_b16 v1, v2                     ; encoding: [0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b16 v255, v255 offset:4
+// GFX1250: ds_store_b16 v255, v255 offset:4        ; encoding: [0x04,0x00,0x7c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_store_b16_d16_hi v1, v2
+// GFX1250: ds_store_b16_d16_hi v1, v2              ; encoding: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b16_d16_hi v1, v2 offset:65535
+// GFX1250: ds_store_b16_d16_hi v1, v2 offset:65535 ; encoding: [0xff,0xff,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b16_d16_hi v1, v2 offset:0
+// GFX1250: ds_store_b16_d16_hi v1, v2              ; encoding: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b16_d16_hi v255, v255 offset:4
+// GFX1250: ds_store_b16_d16_hi v255, v255 offset:4 ; encoding: [0x04,0x00,0x84,0xda,0xff,0xff,0x00,0x00]
+
+ds_store_b32 v1, v2
+// GFX1250: ds_store_b32 v1, v2                     ; encoding: [0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b32 v1, v2 offset:65535
+// GFX1250: ds_store_b32 v1, v2 offset:65535        ; encoding: [0xff,0xff,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b32 v1, v2 offset:0
+// GFX1250: ds_store_b32 v1, v2                     ; encoding: [0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b32 v255, v255 offset:4
+// GFX1250: ds_store_b32 v255, v255 offset:4        ; encoding: [0x04,0x00,0x34,0xd8,0xff,0xff,0x00,0x00]
+
+ds_store_b64 v1, v[2:3]
+// GFX1250: ds_store_b64 v1, v[2:3]                 ; encoding: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_store_b64 v1, v[2:3] offset:65535
+// GFX1250: ds_store_b64 v1, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_store_b64 v1, v[2:3] offset:0
+// GFX1250: ds_store_b64 v1, v[2:3]                 ; encoding: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_store_b64 v255, v[254:255] offset:4
+// GFX1250: ds_store_b64 v255, v[254:255] offset:4  ; encoding: [0x04,0x00,0x34,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_store_b8 v1, v2
+// GFX1250: ds_store_b8 v1, v2                      ; encoding: [0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b8 v1, v2 offset:65535
+// GFX1250: ds_store_b8 v1, v2 offset:65535         ; encoding: [0xff,0xff,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b8 v1, v2 offset:0
+// GFX1250: ds_store_b8 v1, v2                      ; encoding: [0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b8 v255, v255 offset:4
+// GFX1250: ds_store_b8 v255, v255 offset:4         ; encoding: [0x04,0x00,0x78,0xd8,0xff,0xff,0x00,0x00]
+
+ds_store_b8_d16_hi v1, v2
+// GFX1250: ds_store_b8_d16_hi v1, v2               ; encoding: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b8_d16_hi v1, v2 offset:65535
+// GFX1250: ds_store_b8_d16_hi v1, v2 offset:65535  ; encoding: [0xff,0xff,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b8_d16_hi v1, v2 offset:0
+// GFX1250: ds_store_b8_d16_hi v1, v2               ; encoding: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b8_d16_hi v255, v255 offset:4
+// GFX1250: ds_store_b8_d16_hi v255, v255 offset:4  ; encoding: [0x04,0x00,0x80,0xda,0xff,0xff,0x00,0x00]
+
+ds_store_b96 v1, v[2:4]
+// GFX1250: ds_store_b96 v1, v[2:4]                 ; encoding: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b96 v1, v[2:4] offset:65535
+// GFX1250: ds_store_b96 v1, v[2:4] offset:65535    ; encoding: [0xff,0xff,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b96 v1, v[2:4] offset:0
+// GFX1250: ds_store_b96 v1, v[2:4]                 ; encoding: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b96 v255, v[252:254] offset:4
+// GFX1250: ds_store_b96 v255, v[252:254] offset:4  ; encoding: [0x04,0x00,0x78,0xdb,0xff,0xfc,0x00,0x00]
+
+ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3
+// GFX1250: ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x06]
+
+ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255
+// GFX1250: ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0x06]
+
+ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 offset0:0 offset1:0
+// GFX1250: ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x06]
+
+ds_storexchg_2addr_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1
+// GFX1250: ds_storexchg_2addr_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xb8,0xd8,0xff,0xff,0xff,0xfe]
+
+ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5]
+// GFX1250: ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xb8,0xd9,0x01,0x02,0x04,0x06]
+
+ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255
+// GFX1250: ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd9,0x01,0x02,0x04,0x06]
+
+ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:0 offset1:0
+// GFX1250: ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xb8,0xd9,0x01,0x02,0x04,0x06]
+
+ds_storexchg_2addr_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX1250: ds_storexchg_2addr_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0xb8,0xd9,0xff,0xfe,0xfe,0xfc]
+
+ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x06]
+
+ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0x06]
+
+ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 offset0:0 offset1:0
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x06]
+
+ds_storexchg_2addr_stride64_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xbc,0xd8,0xff,0xff,0xff,0xfe]
+
+ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5]
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xbc,0xd9,0x01,0x02,0x04,0x06]
+
+ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd9,0x01,0x02,0x04,0x06]
+
+ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:0 offset1:0
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xbc,0xd9,0x01,0x02,0x04,0x06]
+
+ds_storexchg_2addr_stride64_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0xbc,0xd9,0xff,0xfe,0xfe,0xfc]
+
+ds_storexchg_rtn_b32 v5, v1, v2
+// GFX1250: ds_storexchg_rtn_b32 v5, v1, v2         ; encoding: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_storexchg_rtn_b32 v5, v1, v2 offset:65535
+// GFX1250: ds_storexchg_rtn_b32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_storexchg_rtn_b32 v5, v1, v2 offset:0
+// GFX1250: ds_storexchg_rtn_b32 v5, v1, v2         ; encoding: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_storexchg_rtn_b32 v255, v255, v255 offset:4
+// GFX1250: ds_storexchg_rtn_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xb4,0xd8,0xff,0xff,0x00,0xff]
+
+ds_storexchg_rtn_b64 v[6:7], v1, v[2:3]
+// GFX1250: ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] ; encoding: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x06]
+
+ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0x06]
+
+ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] ; encoding: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x06]
+
+ds_storexchg_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_storexchg_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xb4,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_cond_sub_rtn_u32 v5, v1, v2
+// GFX1250: ds_cond_sub_rtn_u32 v5, v1, v2          ; encoding: [0x00,0x00,0xa0,0xda,0x01,0x02,0x00,0x05]
+
+ds_cond_sub_rtn_u32 v5, v1, v2 offset:65535
+// GFX1250: ds_cond_sub_rtn_u32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xa0,0xda,0x01,0x02,0x00,0x05]
+
+ds_cond_sub_rtn_u32 v5, v1, v2 offset:0
+// GFX1250: ds_cond_sub_rtn_u32 v5, v1, v2          ; encoding: [0x00,0x00,0xa0,0xda,0x01,0x02,0x00,0x05]
+
+ds_cond_sub_u32 v1, v2
+// GFX1250: ds_cond_sub_u32 v1, v2                  ; encoding: [0x00,0x00,0x60,0xda,0x01,0x02,0x00,0x00]
+
+ds_cond_sub_u32 v1, v2 offset:65535
+// GFX1250: ds_cond_sub_u32 v1, v2 offset:65535     ; encoding: [0xff,0xff,0x60,0xda,0x01,0x02,0x00,0x00]
+
+ds_cond_sub_u32 v1, v2 offset:0
+// GFX1250: ds_cond_sub_u32 v1, v2                  ; encoding: [0x00,0x00,0x60,0xda,0x01,0x02,0x00,0x00]
+
+ds_sub_clamp_rtn_u32 v5, v1, v2
+// GFX1250: ds_sub_clamp_rtn_u32 v5, v1, v2         ; encoding: [0x00,0x00,0xa4,0xda,0x01,0x02,0x00,0x05]
+
+ds_sub_clamp_rtn_u32 v5, v1, v2 offset:65535
+// GFX1250: ds_sub_clamp_rtn_u32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xa4,0xda,0x01,0x02,0x00,0x05]
+
+ds_sub_clamp_rtn_u32 v5, v1, v2 offset:0
+// GFX1250: ds_sub_clamp_rtn_u32 v5, v1, v2         ; encoding: [0x00,0x00,0xa4,0xda,0x01,0x02,0x00,0x05]
+
+ds_sub_clamp_rtn_u32 v255, v255, v255 offset:4
+// GFX1250: ds_sub_clamp_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xa4,0xda,0xff,0xff,0x00,0xff]
+
+ds_sub_clamp_u32 v1, v2
+// GFX1250: ds_sub_clamp_u32 v1, v2                 ; encoding: [0x00,0x00,0x64,0xda,0x01,0x02,0x00,0x00]
+
+ds_sub_clamp_u32 v1, v2 offset:65535
+// GFX1250: ds_sub_clamp_u32 v1, v2 offset:65535    ; encoding: [0xff,0xff,0x64,0xda,0x01,0x02,0x00,0x00]
+
+ds_sub_clamp_u32 v1, v2 offset:0
+// GFX1250: ds_sub_clamp_u32 v1, v2                 ; encoding: [0x00,0x00,0x64,0xda,0x01,0x02,0x00,0x00]
+
+ds_sub_clamp_u32 v255, v255 offset:4
+// GFX1250: ds_sub_clamp_u32 v255, v255 offset:4    ; encoding: [0x04,0x00,0x64,0xda,0xff,0xff,0x00,0x00]
+
+ds_sub_rtn_u32 v5, v1, v2
+// GFX1250: ds_sub_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x84,0xd8,0x01,0x02,0x00,0x05]
+
+ds_sub_rtn_u32 v5, v1, v2 offset:65535
+// GFX1250: ds_sub_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x84,0xd8,0x01,0x02,0x00,0x05]
+
+ds_sub_rtn_u32 v5, v1, v2 offset:0
+// GFX1250: ds_sub_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x84,0xd8,0x01,0x02,0x00,0x05]
+
+ds_sub_rtn_u32 v255, v255, v255 offset:4
+// GFX1250: ds_sub_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x84,0xd8,0xff,0xff,0x00,0xff]
+
+ds_sub_rtn_u64 v[6:7], v1, v[2:3]
+// GFX1250: ds_sub_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x84,0xd9,0x01,0x02,0x00,0x06]
+
+ds_sub_rtn_u64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_sub_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x84,0xd9,0x01,0x02,0x00,0x06]
+
+ds_sub_rtn_u64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_sub_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x84,0xd9,0x01,0x02,0x00,0x06]
+
+ds_sub_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_sub_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x84,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_sub_u32 v1, v2
+// GFX1250: ds_sub_u32 v1, v2                       ; encoding: [0x00,0x00,0x04,0xd8,0x01,0x02,0x00,0x00]
+
+ds_sub_u32 v1, v2 offset:65535
+// GFX1250: ds_sub_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x04,0xd8,0x01,0x02,0x00,0x00]
+
+ds_sub_u32 v1, v2 offset:0
+// GFX1250: ds_sub_u32 v1, v2                       ; encoding: [0x00,0x00,0x04,0xd8,0x01,0x02,0x00,0x00]
+
+ds_sub_u32 v255, v255 offset:4
+// GFX1250: ds_sub_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x04,0xd8,0xff,0xff,0x00,0x00]
+
+ds_sub_u64 v1, v[2:3]
+// GFX1250: ds_sub_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x04,0xd9,0x01,0x02,0x00,0x00]
+
+ds_sub_u64 v1, v[2:3] offset:65535
+// GFX1250: ds_sub_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x04,0xd9,0x01,0x02,0x00,0x00]
+
+ds_sub_u64 v1, v[2:3] offset:0
+// GFX1250: ds_sub_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x04,0xd9,0x01,0x02,0x00,0x00]
+
+ds_sub_u64 v255, v[254:255] offset:4
+// GFX1250: ds_sub_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x04,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_write2_b32 v1, v2, v3
+// GFX1250: ds_store_2addr_b32 v1, v2, v3           ; encoding: [0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2_b32 v1, v2, v3 offset0:127 offset1:255
+// GFX1250: ds_store_2addr_b32 v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2_b32 v1, v2, v3 offset0:0 offset1:0
+// GFX1250: ds_store_2addr_b32 v1, v2, v3           ; encoding: [0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2_b32 v255, v255, v255 offset0:16 offset1:1
+// GFX1250: ds_store_2addr_b32 v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0x38,0xd8,0xff,0xff,0xff,0x00]
+
+ds_write2_b64 v1, v[2:3], v[4:5]
+// GFX1250: ds_store_2addr_b64 v1, v[2:3], v[4:5]   ; encoding: [0x00,0x00,0x38,0xd9,0x01,0x02,0x04,0x00]
+
+ds_write2_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255
+// GFX1250: ds_store_2addr_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd9,0x01,0x02,0x04,0x00]
+
+ds_write2_b64 v1, v[2:3], v[4:5] offset0:0 offset1:0
+// GFX1250: ds_store_2addr_b64 v1, v[2:3], v[4:5]   ; encoding: [0x00,0x00,0x38,0xd9,0x01,0x02,0x04,0x00]
+
+ds_write2_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX1250: ds_store_2addr_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0x38,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_write2st64_b32 v1, v2, v3
+// GFX1250: ds_store_2addr_stride64_b32 v1, v2, v3  ; encoding: [0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2st64_b32 v1, v2, v3 offset0:127 offset1:255
+// GFX1250: ds_store_2addr_stride64_b32 v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2st64_b32 v1, v2, v3 offset0:0 offset1:0
+// GFX1250: ds_store_2addr_stride64_b32 v1, v2, v3  ; encoding: [0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2st64_b32 v255, v255, v255 offset0:16 offset1:1
+// GFX1250: ds_store_2addr_stride64_b32 v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0x3c,0xd8,0xff,0xff,0xff,0x00]
+
+ds_write2st64_b64 v1, v[2:3], v[4:5]
+// GFX1250: ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0x3c,0xd9,0x01,0x02,0x04,0x00]
+
+ds_write2st64_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255
+// GFX1250: ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd9,0x01,0x02,0x04,0x00]
+
+ds_write2st64_b64 v1, v[2:3], v[4:5] offset0:0 offset1:0
+// GFX1250: ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0x3c,0xd9,0x01,0x02,0x04,0x00]
+
+ds_write2st64_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX1250: ds_store_2addr_stride64_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0x3c,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_write_addtid_b32 v1
+// GFX1250: ds_store_addtid_b32 v1                  ; encoding: [0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_write_addtid_b32 v1 offset:65535
+// GFX1250: ds_store_addtid_b32 v1 offset:65535     ; encoding: [0xff,0xff,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_write_addtid_b32 v1 offset:0
+// GFX1250: ds_store_addtid_b32 v1                  ; encoding: [0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_write_addtid_b32 v255 offset:4
+// GFX1250: ds_store_addtid_b32 v255 offset:4       ; encoding: [0x04,0x00,0xc0,0xda,0x00,0xff,0x00,0x00]
+
+ds_write_b128 v1, v[2:5]
+// GFX1250: ds_store_b128 v1, v[2:5]                ; encoding: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b128 v1, v[2:5] offset:65535
+// GFX1250: ds_store_b128 v1, v[2:5] offset:65535   ; encoding: [0xff,0xff,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b128 v1, v[2:5] offset:0
+// GFX1250: ds_store_b128 v1, v[2:5]                ; encoding: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b128 v255, v[252:255] offset:4
+// GFX1250: ds_store_b128 v255, v[252:255] offset:4 ; encoding: [0x04,0x00,0x7c,0xdb,0xff,0xfc,0x00,0x00]
+
+ds_write_b16 v1, v2
+// GFX1250: ds_store_b16 v1, v2                     ; encoding: [0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b16 v1, v2 offset:65535
+// GFX1250: ds_store_b16 v1, v2 offset:65535        ; encoding: [0xff,0xff,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b16 v1, v2 offset:0
+// GFX1250: ds_store_b16 v1, v2                     ; encoding: [0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b16 v255, v255 offset:4
+// GFX1250: ds_store_b16 v255, v255 offset:4        ; encoding: [0x04,0x00,0x7c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_write_b16_d16_hi v1, v2
+// GFX1250: ds_store_b16_d16_hi v1, v2              ; encoding: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b16_d16_hi v1, v2 offset:65535
+// GFX1250: ds_store_b16_d16_hi v1, v2 offset:65535 ; encoding: [0xff,0xff,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b16_d16_hi v1, v2 offset:0
+// GFX1250: ds_store_b16_d16_hi v1, v2              ; encoding: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b16_d16_hi v255, v255 offset:4
+// GFX1250: ds_store_b16_d16_hi v255, v255 offset:4 ; encoding: [0x04,0x00,0x84,0xda,0xff,0xff,0x00,0x00]
+
+ds_write_b32 v1, v2
+// GFX1250: ds_store_b32 v1, v2                     ; encoding: [0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b32 v1, v2 offset:65535
+// GFX1250: ds_store_b32 v1, v2 offset:65535        ; encoding: [0xff,0xff,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b32 v1, v2 offset:0
+// GFX1250: ds_store_b32 v1, v2                     ; encoding: [0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b32 v255, v255 offset:4
+// GFX1250: ds_store_b32 v255, v255 offset:4        ; encoding: [0x04,0x00,0x34,0xd8,0xff,0xff,0x00,0x00]
+
+ds_write_b64 v1, v[2:3]
+// GFX1250: ds_store_b64 v1, v[2:3]                 ; encoding: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_write_b64 v1, v[2:3] offset:65535
+// GFX1250: ds_store_b64 v1, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_write_b64 v1, v[2:3] offset:0
+// GFX1250: ds_store_b64 v1, v[2:3]                 ; encoding: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_write_b64 v255, v[254:255] offset:4
+// GFX1250: ds_store_b64 v255, v[254:255] offset:4  ; encoding: [0x04,0x00,0x34,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_write_b8 v1, v2
+// GFX1250: ds_store_b8 v1, v2                      ; encoding: [0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b8 v1, v2 offset:65535
+// GFX1250: ds_store_b8 v1, v2 offset:65535         ; encoding: [0xff,0xff,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b8 v1, v2 offset:0
+// GFX1250: ds_store_b8 v1, v2                      ; encoding: [0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b8 v255, v255 offset:4
+// GFX1250: ds_store_b8 v255, v255 offset:4         ; encoding: [0x04,0x00,0x78,0xd8,0xff,0xff,0x00,0x00]
+
+ds_write_b8_d16_hi v1, v2
+// GFX1250: ds_store_b8_d16_hi v1, v2               ; encoding: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b8_d16_hi v1, v2 offset:65535
+// GFX1250: ds_store_b8_d16_hi v1, v2 offset:65535  ; encoding: [0xff,0xff,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b8_d16_hi v1, v2 offset:0
+// GFX1250: ds_store_b8_d16_hi v1, v2               ; encoding: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b8_d16_hi v255, v255 offset:4
+// GFX1250: ds_store_b8_d16_hi v255, v255 offset:4  ; encoding: [0x04,0x00,0x80,0xda,0xff,0xff,0x00,0x00]
+
+ds_write_b96 v1, v[2:4]
+// GFX1250: ds_store_b96 v1, v[2:4]                 ; encoding: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b96 v1, v[2:4] offset:65535
+// GFX1250: ds_store_b96 v1, v[2:4] offset:65535    ; encoding: [0xff,0xff,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b96 v1, v[2:4] offset:0
+// GFX1250: ds_store_b96 v1, v[2:4]                 ; encoding: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b96 v255, v[252:254] offset:4
+// GFX1250: ds_store_b96 v255, v[252:254] offset:4  ; encoding: [0x04,0x00,0x78,0xdb,0xff,0xfc,0x00,0x00]
+
+ds_wrxchg2_rtn_b32 v[6:7], v1, v2, v3
+// GFX1250: ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x06]
+
+ds_wrxchg2_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255
+// GFX1250: ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0x06]
+
+ds_wrxchg2_rtn_b32 v[6:7], v1, v2, v3 offset0:0 offset1:0
+// GFX1250: ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x06]
+
+ds_wrxchg2_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1
+// GFX1250: ds_storexchg_2addr_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xb8,0xd8,0xff,0xff,0xff,0xfe]
+
+ds_wrxchg2_rtn_b64 v[6:9], v1, v[2:3], v[4:5]
+// GFX1250: ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xb8,0xd9,0x01,0x02,0x04,0x06]
+
+ds_wrxchg2_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255
+// GFX1250: ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd9,0x01,0x02,0x04,0x06]
+
+ds_wrxchg2_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:0 offset1:0
+// GFX1250: ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xb8,0xd9,0x01,0x02,0x04,0x06]
+
+ds_wrxchg2_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX1250: ds_storexchg_2addr_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0xb8,0xd9,0xff,0xfe,0xfe,0xfc]
+
+ds_wrxchg2st64_rtn_b32 v[6:7], v1, v2, v3
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x06]
+
+ds_wrxchg2st64_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0x06]
+
+ds_wrxchg2st64_rtn_b32 v[6:7], v1, v2, v3 offset0:0 offset1:0
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x06]
+
+ds_wrxchg2st64_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xbc,0xd8,0xff,0xff,0xff,0xfe]
+
+ds_wrxchg2st64_rtn_b64 v[6:9], v1, v[2:3], v[4:5]
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xbc,0xd9,0x01,0x02,0x04,0x06]
+
+ds_wrxchg2st64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd9,0x01,0x02,0x04,0x06]
+
+ds_wrxchg2st64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:0 offset1:0
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xbc,0xd9,0x01,0x02,0x04,0x06]
+
+ds_wrxchg2st64_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0xbc,0xd9,0xff,0xfe,0xfe,0xfc]
+
+ds_wrxchg_rtn_b32 v5, v1, v2
+// GFX1250: ds_storexchg_rtn_b32 v5, v1, v2         ; encoding: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_wrxchg_rtn_b32 v5, v1, v2 offset:65535
+// GFX1250: ds_storexchg_rtn_b32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_wrxchg_rtn_b32 v5, v1, v2 offset:0
+// GFX1250: ds_storexchg_rtn_b32 v5, v1, v2         ; encoding: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_wrxchg_rtn_b32 v255, v255, v255 offset:4
+// GFX1250: ds_storexchg_rtn_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xb4,0xd8,0xff,0xff,0x00,0xff]
+
+ds_wrxchg_rtn_b64 v[6:7], v1, v[2:3]
+// GFX1250: ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] ; encoding: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x06]
+
+ds_wrxchg_rtn_b64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0x06]
+
+ds_wrxchg_rtn_b64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] ; encoding: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x06]
+
+ds_wrxchg_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_storexchg_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xb4,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_xor_b32 v1, v2
+// GFX1250: ds_xor_b32 v1, v2                       ; encoding: [0x00,0x00,0x2c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_xor_b32 v1, v2 offset:65535
+// GFX1250: ds_xor_b32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x2c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_xor_b32 v1, v2 offset:0
+// GFX1250: ds_xor_b32 v1, v2                       ; encoding: [0x00,0x00,0x2c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_xor_b32 v255, v255 offset:4
+// GFX1250: ds_xor_b32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x2c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_xor_b64 v1, v[2:3]
+// GFX1250: ds_xor_b64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_xor_b64 v1, v[2:3] offset:65535
+// GFX1250: ds_xor_b64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x2c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_xor_b64 v1, v[2:3] offset:0
+// GFX1250: ds_xor_b64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_xor_b64 v255, v[254:255] offset:4
+// GFX1250: ds_xor_b64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x2c,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_xor_rtn_b32 v5, v1, v2
+// GFX1250: ds_xor_rtn_b32 v5, v1, v2               ; encoding: [0x00,0x00,0xac,0xd8,0x01,0x02,0x00,0x05]
+
+ds_xor_rtn_b32 v5, v1, v2 offset:65535
+// GFX1250: ds_xor_rtn_b32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xac,0xd8,0x01,0x02,0x00,0x05]
+
+ds_xor_rtn_b32 v5, v1, v2 offset:0
+// GFX1250: ds_xor_rtn_b32 v5, v1, v2               ; encoding: [0x00,0x00,0xac,0xd8,0x01,0x02,0x00,0x05]
+
+ds_xor_rtn_b32 v255, v255, v255 offset:4
+// GFX1250: ds_xor_rtn_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xac,0xd8,0xff,0xff,0x00,0xff]
+
+ds_xor_rtn_b64 v[6:7], v1, v[2:3]
+// GFX1250: ds_xor_rtn_b64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0xac,0xd9,0x01,0x02,0x00,0x06]
+
+ds_xor_rtn_b64 v[6:7], v1, v[2:3] offset:65535
+// GFX1250: ds_xor_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xac,0xd9,0x01,0x02,0x00,0x06]
+
+ds_xor_rtn_b64 v[6:7], v1, v[2:3] offset:0
+// GFX1250: ds_xor_rtn_b64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0xac,0xd9,0x01,0x02,0x00,0x06]
+
+ds_xor_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX1250: ds_xor_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xac,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_swizzle_b32 v8, v2
+// GFX1250: ds_swizzle_b32 v8, v2                   ; encoding: [0x00,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:0
+// GFX1250: ds_swizzle_b32 v8, v2                   ; encoding: [0x00,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:0xFFFF
+// GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(QUAD_PERM, 0, 1, 2, 3)
+// GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(QUAD_PERM,0,1,2,3) ; encoding: [0xe4,0x80,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(SWAP,16)
+// GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(SWAP,16) ; encoding: [0x1f,0x40,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(REVERSE,8)
+// GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(REVERSE,8) ; encoding: [0x1f,0x1c,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,4,1)
+// GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,4,1) ; encoding: [0x3c,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,8,7)
+// GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,8,7) ; encoding: [0xf8,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(BITMASK_PERM, "01pip")
+// GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(BITMASK_PERM,"01pip") ; encoding: [0x07,0x09,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
 ds_atomic_async_barrier_arrive_b64 v1 offset:65407
 // GFX1250: ds_atomic_async_barrier_arrive_b64 v1 offset:65407 ; encoding: [0x7f,0xff,0x58,0xd9,0x01,0x00,0x00,0x00]
 // GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_features.s b/llvm/test/MC/AMDGPU/gfx1250_asm_features.s
new file mode 100644
index 0000000000000..013b790b2b4b6
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_features.s
@@ -0,0 +1,32 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefixes=GFX1250 %s
+
+//
+// Elements of CPol operand can be given in any order
+//
+
+s_load_b32 s4, s[2:3], 10 th:TH_LOAD_NT scope:SCOPE_SE nv
+// GFX1250: encoding: [0x01,0x01,0xb0,0xf4,0x0a,0x00,0x00,0xf8]
+
+s_load_b32 s4, s[2:3], 10 scope:SCOPE_SE nv th:TH_LOAD_NT
+// GFX1250: encoding: [0x01,0x01,0xb0,0xf4,0x0a,0x00,0x00,0xf8]
+
+s_load_b32 s4, s[2:3], 10 nv scope:SCOPE_SE th:TH_LOAD_NT
+// GFX1250: encoding: [0x01,0x01,0xb0,0xf4,0x0a,0x00,0x00,0xf8]
+
+buffer_load_b32 v5, v1, s[8:11], s3 offen offset:4095 th:TH_LOAD_NT scope:SCOPE_SE nv
+// GFX1250: encoding: [0x83,0x00,0x05,0xc4,0x05,0x10,0x94,0x40,0x01,0xff,0x0f,0x00]
+
+buffer_load_b32 v5, v1, s[8:11], s3 offen offset:4095 scope:SCOPE_SE nv th:TH_LOAD_NT
+// GFX1250: encoding: [0x83,0x00,0x05,0xc4,0x05,0x10,0x94,0x40,0x01,0xff,0x0f,0x00]
+
+buffer_load_b32 v5, v1, s[8:11], s3 offen offset:4095 nv scope:SCOPE_SE th:TH_LOAD_NT
+// GFX1250: encoding: [0x83,0x00,0x05,0xc4,0x05,0x10,0x94,0x40,0x01,0xff,0x0f,0x00]
+
+global_load_b32 v0, v[2:3], off th:TH_LOAD_NT scope:SCOPE_SE nv
+// GFX1250: encoding: [0xfc,0x00,0x05,0xee,0x00,0x00,0x14,0x00,0x02,0x00,0x00,0x00]
+
+global_load_b32 v0, v[2:3], off scope:SCOPE_SE nv th:TH_LOAD_NT
+// GFX1250: encoding: [0xfc,0x00,0x05,0xee,0x00,0x00,0x14,0x00,0x02,0x00,0x00,0x00]
+
+global_load_b32 v0, v[2:3], off nv scope:SCOPE_SE th:TH_LOAD_NT
+// GFX1250: encoding: [0xfc,0x00,0x05,0xee,0x00,0x00,0x14,0x00,0x02,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_operands.s b/llvm/test/MC/AMDGPU/gfx1250_asm_operands.s
index 8b7465b5df574..91b3fcb9f660b 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_operands.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_operands.s
@@ -27,3 +27,32 @@ s_mov_b64 s[0:1], src_shared_limit
 
 s_getreg_b32 s1, hwreg(33)
 // GFX1250: encoding: [0x21,0xf8,0x81,0xb8]
+
+s_getreg_b32 s1, hwreg(HW_REG_XNACK_STATE_PRIV)
+// GFX1200-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid hardware register: not supported on this GPU
+// GFX1250: encoding: [0x21,0xf8,0x81,0xb8]
+
+s_getreg_b32 s1, hwreg(34)
+// GFX1250: encoding: [0x22,0xf8,0x81,0xb8]
+
+s_getreg_b32 s1, hwreg(HW_REG_XNACK_MASK)
+// GFX1200-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid hardware register: not supported on this GPU
+// GFX1250: encoding: [0x22,0xf8,0x81,0xb8]
+
+s_setreg_b32 hwreg(33), s1
+// GFX1250: encoding: [0x21,0xf8,0x01,0xb9]
+
+s_setreg_b32 hwreg(HW_REG_XNACK_STATE_PRIV), s1
+// GFX1200-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid hardware register: not supported on this GPU
+// GFX1250: encoding: [0x21,0xf8,0x01,0xb9]
+
+s_setreg_b32 hwreg(34), s1
+// GFX1250: encoding: [0x22,0xf8,0x01,0xb9]
+
+s_setreg_b32 hwreg(HW_REG_XNACK_MASK), s1
+// GFX1200-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid hardware register: not supported on this GPU
+// GFX1250: encoding: [0x22,0xf8,0x01,0xb9]
+
+s_setreg_b32 hwreg(HW_REG_IB_STS2), s1
+// GFX1200-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid hardware register: not supported on this GPU
+// GFX1250: encoding: [0x1c,0xf8,0x01,0xb9]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_sop1.s b/llvm/test/MC/AMDGPU/gfx1250_asm_sop1.s
index 41b6e93357a3f..aab8d9a2fcbfd 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_sop1.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_sop1.s
@@ -45,6 +45,10 @@ s_rfe_i64 s[2:3]
 s_rfe_b64 s[2:3]
 // GFX1250: s_rfe_i64 s[2:3]                        ; encoding: [0x02,0x4a,0x80,0xbe]
 
+s_get_shader_cycles_u64 s[2:3]
+// GFX1250: s_get_shader_cycles_u64 s[2:3]          ; encoding: [0x00,0x06,0x82,0xbe]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
 s_barrier_signal -3
 // GFX1250: s_barrier_signal -3                     ; encoding: [0xc3,0x4e,0x80,0xbe]
 
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
index 234c2ed0de793..bfc3544ac1b15 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
@@ -48,3 +48,10 @@ s_monitor_sleep 32768
 s_monitor_sleep 0
 // GFX1250: s_monitor_sleep 0                       ; encoding: [0x00,0x00,0x84,0xbf]
 // GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_sendmsg sendmsg(MSG_SAVEWAVE_HAS_TDM)
+// GFX1250: s_sendmsg sendmsg(MSG_SAVEWAVE_HAS_TDM)     ; encoding: [0x0a,0x00,0xb6,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU
+
+s_barrier_wait -3
+// GFX1250: s_barrier_wait -3                       ; encoding: [0xfd,0xff,0x94,0xbf]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_sopp_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp_err.s
new file mode 100644
index 0000000000000..d151c9ac2f804
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp_err.s
@@ -0,0 +1,4 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX1250-ERR --implicit-check-not=error: -strict-whitespace %s
+
+s_setkill 0
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_unsupported.s b/llvm/test/MC/AMDGPU/gfx1250_asm_unsupported.s
index 89bd507942a22..7681a32dce7dc 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_unsupported.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_unsupported.s
@@ -97,6 +97,20 @@ v_interp_p10_rtz_f16_f32 v0, v1, v2, v3
 v_interp_p2_rtz_f16_f32 v0, v1, v2, v3
 // GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
+;; *xf32
+
+v_mfma_f32_16x16x8_xf32 a[0:3], v[2:3], v[4:5], a[2:5]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_mfma_f32_16x16x8xf32 a[0:3], v[2:3], v[4:5], a[2:5]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_mfma_f32_32x32x4_xf32 a[0:15], v[2:3], v[4:5], a[18:33]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_mfma_f32_32x32x4xf32 a[0:15], v[2:3], v[4:5], a[18:33]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
 ;; Export, S_WAIT_EXPCNT and S_WAIT_EVENT
 
 export mrt0 off, off, off, off
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vbuffer_mubuf.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vbuffer_mubuf.s
index 7a4da255b5594..0b8f190a7ae0d 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vbuffer_mubuf.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vbuffer_mubuf.s
@@ -1,6 +1,2310 @@
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefix=GFX1250 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
 
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], s3
+// GFX1250: buffer_load_b32 v5, off, s[8:11], s3    ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_b32 v5, off, s[8:11], s3    ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[6:8], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[252:254], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_b96 v[252:254], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[6:8], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_b96 v[6:8], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[6:8], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_b96 v[6:8], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[6:8], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[6:8], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_b96 v[6:8], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[6:8], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_b96 v[6:8], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[6:8], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_b96 v[6:8], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[6:8], off, s[8:11], s3
+// GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b96 v[6:8], off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b96 v[6:8], off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_b96 v[6:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[6:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[6:9], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[252:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_b128 v[252:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[6:9], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_b128 v[6:9], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[6:9], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_b128 v[6:9], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[6:9], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[6:9], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_b128 v[6:9], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[6:9], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_b128 v[6:9], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[6:9], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_b128 v[6:9], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[6:9], off, s[8:11], s3
+// GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b128 v[6:9], off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b128 v[6:9], off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_b128 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_b16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_d16_b16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_d16_b16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_d16_b16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_d16_b16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_d16_b16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3
+// GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_b16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_i8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_u8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_i8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_d16_i8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_d16_i8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_d16_i8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_d16_i8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_d16_i8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3
+// GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_d16_u8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_d16_u8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_d16_u8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_d16_u8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_d16_u8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_d16_u8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3
+// GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_i8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_i8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_i8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_i8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_i8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_i8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_i8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], s3
+// GFX1250: buffer_load_i8 v5, off, s[8:11], s3     ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_i8 v5, off, s[8:11], s3     ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_i8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_i16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_i16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_i16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_i16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_i16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_i16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_i16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], s3
+// GFX1250: buffer_load_i16 v5, off, s[8:11], s3    ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_i16 v5, off, s[8:11], s3    ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_i16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_u8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_u8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_u8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_u8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_u8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_u8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_u8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], s3
+// GFX1250: buffer_load_u8 v5, off, s[8:11], s3     ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_u8 v5, off, s[8:11], s3     ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_u8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_load_u16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_load_u16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_load_u16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_load_u16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_load_u16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_load_u16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_load_u16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], s3
+// GFX1250: buffer_load_u16 v5, off, s[8:11], s3    ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_load_u16 v5, off, s[8:11], s3    ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_load_u16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v255, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b8 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[16:19], s4 offset:8388607
+// GFX1250: buffer_store_b8 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[96:99], s4 offset:8388607
+// GFX1250: buffer_store_b8 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], s101 offset:8388607
+// GFX1250: buffer_store_b8 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], m0 offset:8388607
+// GFX1250: buffer_store_b8 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX1250: buffer_store_b8 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX1250: buffer_store_b8 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], s4
+// GFX1250: buffer_store_b8 v1, off, s[12:15], s4   ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:0
+// GFX1250: buffer_store_b8 v1, off, s[12:15], s4   ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:7
+// GFX1250: buffer_store_b8 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v255, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b16 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[16:19], s4 offset:8388607
+// GFX1250: buffer_store_b16 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[96:99], s4 offset:8388607
+// GFX1250: buffer_store_b16 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], s101 offset:8388607
+// GFX1250: buffer_store_b16 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], m0 offset:8388607
+// GFX1250: buffer_store_b16 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX1250: buffer_store_b16 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX1250: buffer_store_b16 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], s4
+// GFX1250: buffer_store_b16 v1, off, s[12:15], s4  ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:0
+// GFX1250: buffer_store_b16 v1, off, s[12:15], s4  ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:7
+// GFX1250: buffer_store_b16 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v255, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b32 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[16:19], s4 offset:8388607
+// GFX1250: buffer_store_b32 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[96:99], s4 offset:8388607
+// GFX1250: buffer_store_b32 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], s101 offset:8388607
+// GFX1250: buffer_store_b32 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], m0 offset:8388607
+// GFX1250: buffer_store_b32 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX1250: buffer_store_b32 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX1250: buffer_store_b32 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], s4
+// GFX1250: buffer_store_b32 v1, off, s[12:15], s4  ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:0
+// GFX1250: buffer_store_b32 v1, off, s[12:15], s4  ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:7
+// GFX1250: buffer_store_b32 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[2:3], off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[254:255], off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b64 v[254:255], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[2:3], off, s[16:19], s4 offset:8388607
+// GFX1250: buffer_store_b64 v[2:3], off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[2:3], off, s[96:99], s4 offset:8388607
+// GFX1250: buffer_store_b64 v[2:3], off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[2:3], off, s[12:15], s101 offset:8388607
+// GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[2:3], off, s[12:15], m0 offset:8388607
+// GFX1250: buffer_store_b64 v[2:3], off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[2:3], v0, s[12:15], s4 idxen offset:8388607
+// GFX1250: buffer_store_b64 v[2:3], v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[2:3], v0, s[12:15], s4 offen offset:8388607
+// GFX1250: buffer_store_b64 v[2:3], v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[2:3], off, s[12:15], s4
+// GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b64 v[2:3], off, s[12:15], s4 offset:0
+// GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b64 v[2:3], off, s[12:15], s4 offset:7
+// GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 offset:7 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b64 v[2:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[2:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[2:4], off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[252:254], off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b96 v[252:254], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[2:4], off, s[16:19], s4 offset:8388607
+// GFX1250: buffer_store_b96 v[2:4], off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[2:4], off, s[96:99], s4 offset:8388607
+// GFX1250: buffer_store_b96 v[2:4], off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[2:4], off, s[12:15], s101 offset:8388607
+// GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[2:4], off, s[12:15], m0 offset:8388607
+// GFX1250: buffer_store_b96 v[2:4], off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[2:4], v0, s[12:15], s4 idxen offset:8388607
+// GFX1250: buffer_store_b96 v[2:4], v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[2:4], v0, s[12:15], s4 offen offset:8388607
+// GFX1250: buffer_store_b96 v[2:4], v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[2:4], off, s[12:15], s4
+// GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b96 v[2:4], off, s[12:15], s4 offset:0
+// GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b96 v[2:4], off, s[12:15], s4 offset:7
+// GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 offset:7 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b96 v[2:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[2:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[2:5], off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[252:255], off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_b128 v[252:255], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[2:5], off, s[16:19], s4 offset:8388607
+// GFX1250: buffer_store_b128 v[2:5], off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[2:5], off, s[96:99], s4 offset:8388607
+// GFX1250: buffer_store_b128 v[2:5], off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[2:5], off, s[12:15], s101 offset:8388607
+// GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[2:5], off, s[12:15], m0 offset:8388607
+// GFX1250: buffer_store_b128 v[2:5], off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[2:5], v0, s[12:15], s4 idxen offset:8388607
+// GFX1250: buffer_store_b128 v[2:5], v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[2:5], v0, s[12:15], s4 offen offset:8388607
+// GFX1250: buffer_store_b128 v[2:5], v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[2:5], off, s[12:15], s4
+// GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b128 v[2:5], off, s[12:15], s4 offset:0
+// GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b128 v[2:5], off, s[12:15], s4 offset:7
+// GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 offset:7 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b128 v[2:5], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[2:5], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v255, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_d16_hi_b8 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[16:19], s4 offset:8388607
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[96:99], s4 offset:8388607
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s101 offset:8388607
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], m0 offset:8388607
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX1250: buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX1250: buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:0
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:7
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v255, off, s[12:15], s4 offset:8388607
+// GFX1250: buffer_store_d16_hi_b16 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[16:19], s4 offset:8388607
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[96:99], s4 offset:8388607
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s101 offset:8388607
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], m0 offset:8388607
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX1250: buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX1250: buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:0
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:7
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_pk_add_f16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_pk_add_bf16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_pk_add_bf16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_pk_add_bf16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_add_f32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_add_f32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_add_f32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_add_f32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_add_f32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_add_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_add_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_add_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_add_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_add_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_add_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_add_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_add_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_and_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_and_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_and_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_and_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_and_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_and_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_and_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_and_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b32 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[252:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b64 v[252:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:0 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:0 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:0 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_cond_sub_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_cond_sub_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_cond_sub_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_dec_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_dec_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_dec_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_dec_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_dec_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_dec_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_inc_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_inc_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_inc_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_inc_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_inc_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_inc_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_num_f32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_i32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_max_i32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_max_i32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_max_i32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_max_i32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_i64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_max_i64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_max_i64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_max_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_max_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_max_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_max_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_max_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_max_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_max_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_num_f32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_i32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_min_i32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_min_i32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_min_i32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_min_i32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_i64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_min_i64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_min_i64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_min_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_min_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_min_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_min_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_min_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_min_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_min_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_or_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_or_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_or_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_or_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_or_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_or_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_or_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_or_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_sub_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_sub_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_sub_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_sub_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_sub_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_sub_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_swap_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_swap_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_swap_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_swap_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_swap_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_swap_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_xor_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_xor_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_xor_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX1250: buffer_atomic_xor_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], off, s[12:15], s3 offset:8388607
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], off, s[96:99], s3 offset:8388607
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], off, s[8:11], s101 offset:8388607
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], off, s[8:11], m0 offset:8388607
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX1250: buffer_atomic_xor_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607
+// GFX1250: buffer_atomic_xor_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:0
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:7
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
 buffer_load_b32 v5, v1, s[8:11], s3 offen offset:4095 nv
 // GFX1250: buffer_load_b32 v5, v1, s[8:11], s3 offen offset:4095 nv ; encoding: [0x83,0x00,0x05,0xc4,0x05,0x10,0x80,0x40,0x01,0xff,0x0f,0x00]
 // GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: nv is not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s
index 3b2e03f1d4b9e..5ce9da1019498 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vimage_err.s
@@ -23,3 +23,63 @@ tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] r128
 
 tensor_store_from_lds s[0:3], s[4:11], s[12:15], s[16:19] th:TH_LOAD_NT_HT scope:SCOPE_DEV
 // GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid th value for store instructions
+
+tensor_load_to_lds null, s[4:11]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_load_to_lds null, s[4:11]
+// GFX1250-ERR:                    ^
+
+tensor_load_to_lds s[0:3], null
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_load_to_lds s[0:3], null
+// GFX1250-ERR:                            ^
+
+tensor_load_to_lds null, s[4:11], s[12:15], s[16:19]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_load_to_lds null, s[4:11], s[12:15], s[16:19]
+// GFX1250-ERR:                    ^
+
+tensor_load_to_lds s[0:3], null, s[12:15], s[16:19]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_load_to_lds s[0:3], null, s[12:15], s[16:19]
+// GFX1250-ERR:                            ^
+
+tensor_load_to_lds s[0:3], s[4:11], null, s[16:19]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_load_to_lds s[0:3], s[4:11], null, s[16:19]
+// GFX1250-ERR:                                     ^
+
+tensor_load_to_lds s[0:3], s[4:11], s[12:15], null
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_load_to_lds s[0:3], s[4:11], s[12:15], null
+// GFX1250-ERR:                                               ^
+
+tensor_store_from_lds null, s[4:11]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_store_from_lds null, s[4:11]
+// GFX1250-ERR:                       ^
+
+tensor_store_from_lds s[0:3], null
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_store_from_lds s[0:3], null
+// GFX1250-ERR:                               ^
+
+tensor_store_from_lds null, s[4:11], s[12:15], s[16:19]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_store_from_lds null, s[4:11], s[12:15], s[16:19]
+// GFX1250-ERR:                       ^
+
+tensor_store_from_lds s[0:3], null, s[12:15], s[16:19]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_store_from_lds s[0:3], null, s[12:15], s[16:19]
+// GFX1250-ERR:                               ^
+
+tensor_store_from_lds s[0:3], s[4:11], null, s[16:19]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_store_from_lds s[0:3], s[4:11], null, s[16:19]
+// GFX1250-ERR:                                        ^
+
+tensor_store_from_lds s[0:3], s[4:11], s[12:15], null
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: tensor_store_from_lds s[0:3], s[4:11], s[12:15], null
+// GFX1250-ERR:                                                  ^
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_err.s
index c5bd00c004a43..e87943224e8f5 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_err.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_err.s
@@ -5,6 +5,76 @@ v_lshl_add_u64 v[2:3], v[4:5], v7, v[8:9] dpp8:[7,6,5,4,3,2,1,0]
 // GFX125X-ERR-NEXT:{{^}}v_lshl_add_u64 v[2:3], v[4:5], v7, v[8:9] dpp8:[7,6,5,4,3,2,1,0]
 // GFX125X-ERR-NEXT:{{^}}                                          ^
 
+v_fma_f64 v[4:5], v[2:3], v[6:7], v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_fma_f64 v[4:5], v[2:3], v[6:7], v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                         ^
+
+v_div_fixup_f64 v[4:5], v[2:3], v[6:7], v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_div_fixup_f64 v[4:5], v[2:3], v[6:7], v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                               ^
+
+v_div_fmas_f64 v[4:5], v[2:3], v[6:7], v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_div_fmas_f64 v[4:5], v[2:3], v[6:7], v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                              ^
+
+v_div_scale_f64 v[4:5], s2, v[2:3], v[6:7], v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_div_scale_f64 v[4:5], s2, v[2:3], v[6:7], v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                                   ^
+
+v_mad_co_u64_u32 v[4:5], s2, v2, v6, v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_mad_co_u64_u32 v[4:5], s2, v2, v6, v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                            ^
+
+v_mad_co_i64_i32 v[4:5], s2, v2, v6, v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_mad_co_i64_i32 v[4:5], s2, v2, v6, v[8:9] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                            ^
+
+v_minimum_f64 v[4:5], v[2:3], v[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_minimum_f64 v[4:5], v[2:3], v[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                     ^
+
+v_maximum_f64 v[4:5], v[2:3], v[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_maximum_f64 v[4:5], v[2:3], v[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                     ^
+
+v_ldexp_f64 v[4:5], v[2:3], v6 dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_ldexp_f64 v[4:5], v[2:3], v6 dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                               ^
+
+v_mul_lo_u32 v4, v2, v6 dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_mul_lo_u32 v4, v2, v6 dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                        ^
+
+v_mul_hi_u32 v4, v2, v6 dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_mul_hi_u32 v4, v2, v6 dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                        ^
+
+v_mul_hi_i32 v4, v2, v6 dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_mul_hi_i32 v4, v2, v6 dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                        ^
+
+v_lshrrev_b64 v[4:5], v2, v[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_lshrrev_b64 v[4:5], v2, v[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                 ^
+
+v_ashrrev_i64 v[4:5], v2, v[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_ashrrev_i64 v[4:5], v2, v[6:7] dpp8:[7,6,5,4,3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                 ^
+
 v_mad_u32 v2, v4, v7, v8 dpp8:[7,6,5,4,3,2,1,0]
 // GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
 // GFX125X-ERR-NEXT:{{^}}v_mad_u32 v2, v4, v7, v8 dpp8:[7,6,5,4,3,2,1,0]
@@ -42,9 +112,94 @@ v_mad_nc_i64_i32 v[4:5], v2, v5, v[6:7] dpp8:[7,6,5,4,3,2,1,0]
 
 v_lshl_add_u64 v[2:3], v[4:5], v7, v[8:9] quad_perm:[3,2,1,0]
 // GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
 // GFX125X-ERR-NEXT:{{^}}v_lshl_add_u64 v[2:3], v[4:5], v7, v[8:9] quad_perm:[3,2,1,0]
 // GFX125X-ERR-NEXT:{{^}}                                          ^
 
+v_fma_f64 v[4:5], v[2:3], v[6:7], v[8:9] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_fma_f64 v[4:5], v[2:3], v[6:7], v[8:9] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                         ^
+
+v_div_fixup_f64 v[4:5], v[2:3], v[6:7], v[8:9] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_div_fixup_f64 v[4:5], v[2:3], v[6:7], v[8:9] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                               ^
+
+v_div_fmas_f64 v[4:5], v[2:3], v[6:7], v[8:9] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_div_fmas_f64 v[4:5], v[2:3], v[6:7], v[8:9] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                              ^
+
+v_div_scale_f64 v[4:5], s2, v[2:3], v[6:7], v[8:9] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_div_scale_f64 v[4:5], s2, v[2:3], v[6:7], v[8:9] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                                   ^
+
+v_mad_co_u64_u32 v[4:5], s2, v2, v6, v[8:9] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_mad_co_u64_u32 v[4:5], s2, v2, v6, v[8:9] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                            ^
+
+v_mad_co_i64_i32 v[4:5], s2, v2, v6, v[8:9] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_mad_co_i64_i32 v[4:5], s2, v2, v6, v[8:9] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                            ^
+
+v_minimum_f64 v[4:5], v[2:3], v[6:7] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_minimum_f64 v[4:5], v[2:3], v[6:7] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                     ^
+
+v_maximum_f64 v[4:5], v[2:3], v[6:7] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_maximum_f64 v[4:5], v[2:3], v[6:7] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                     ^
+
+v_ldexp_f64 v[4:5], v[2:3], v6 quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_ldexp_f64 v[4:5], v[2:3], v6 quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                               ^
+
+v_mul_lo_u32 v4, v2, v6 quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_mul_lo_u32 v4, v2, v6 quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                        ^
+
+v_mul_hi_u32 v4, v2, v6 quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_mul_hi_u32 v4, v2, v6 quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                        ^
+
+v_mul_hi_i32 v4, v2, v6 quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_mul_hi_i32 v4, v2, v6 quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                        ^
+
+v_lshrrev_b64 v[4:5], v2, v[6:7] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_lshrrev_b64 v[4:5], v2, v[6:7] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                 ^
+
+v_ashrrev_i64 v[4:5], v2, v[6:7] quad_perm:[3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
+// GFX125X-ERR-NEXT:{{^}}v_ashrrev_i64 v[4:5], v2, v[6:7] quad_perm:[3,2,1,0]
+// GFX125X-ERR-NEXT:{{^}}                                 ^
+
 v_mad_u32 v2, v4, v7, v8 quad_perm:[3,2,1,0]
 // GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
 // GFX1251-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: DP ALU dpp only supports row_share
@@ -87,6 +242,11 @@ v_mad_nc_i64_i32 v[4:5], v2, v5, v[6:7] quad_perm:[3,2,1,0]
 // GFX125X-ERR-NEXT:{{^}}v_mad_nc_i64_i32 v[4:5], v2, v5, v[6:7] quad_perm:[3,2,1,0]
 // GFX125X-ERR-NEXT:{{^}}                                        ^
 
+v_trig_preop_f64 v[4:5], v[8:9], v2 row_share:1
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX125X-ERR-NEXT:{{^}}v_trig_preop_f64 v[4:5], v[8:9], v2 row_share:1
+// GFX125X-ERR-NEXT:{{^}}                                    ^
+
 v_ashr_pk_i8_i32 v1, v2, v3, v4 clamp
 // GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
 // GFX125X-ERR-NEXT:{{^}}v_ashr_pk_i8_i32 v1, v2, v3, v4 clamp
@@ -161,3 +321,8 @@ v_cvt_scale_pk8_f32_fp4 v[10:17], s20, v8
 // GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
 // GFX125X-ERR-NEXT:{{^}}v_cvt_scale_pk8_f32_fp4 v[10:17], s20, v8
 // GFX125X-ERR-NEXT:{{^}}                                  ^
+
+v_cvt_scale_pk16_bf16_bf6 v[10:17], s[20:22], 0xcf00
+// GFX125X-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX125X-ERR-NEXT:{{^}}v_cvt_scale_pk16_bf16_bf6 v[10:17], s[20:22], 0xcf00
+// GFX125X-ERR-NEXT:{{^}}                                    ^
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop2_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop2_err.s
new file mode 100644
index 0000000000000..157b4d6af9c05
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3_from_vop2_err.s
@@ -0,0 +1,13 @@
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX1250-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_fmaak_f32_e64_dpp v4, v2, v6, 3 row_share:1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: e64_dpp variant of this instruction is not supported
+
+v_fmamk_f32_e64_dpp v4, v2, 3, v6 row_share:1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: e64_dpp variant of this instruction is not supported
+
+v_fmaak_f16_e64_dpp v4, v2, v6, 3 row_share:1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: e64_dpp variant of this instruction is not supported
+
+v_fmamk_f16_e64_dpp v4, v2, 3, v6 row_share:1
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: e64_dpp variant of this instruction is not supported
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3cx.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3cx.s
new file mode 100644
index 0000000000000..4aea7b32f13e5
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3cx.s
@@ -0,0 +1,3413 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
+
+v_cmpx_class_f16_e64 v1, v2
+// GFX1250: v_cmpx_class_f16_e64 v1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_class_f16_e64 v255, v2
+// GFX1250: v_cmpx_class_f16_e64 v255, v2           ; encoding: [0x7e,0x00,0xfd,0xd4,0xff,0x05,0x02,0x00]
+
+v_cmpx_class_f16_e64 s1, v2
+// GFX1250: v_cmpx_class_f16_e64 s1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x04,0x02,0x00]
+
+v_cmpx_class_f16_e64 s105, v255
+// GFX1250: v_cmpx_class_f16_e64 s105, v255         ; encoding: [0x7e,0x00,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+
+v_cmpx_class_f16_e64 vcc_lo, s2
+// GFX1250: v_cmpx_class_f16_e64 vcc_lo, s2         ; encoding: [0x7e,0x00,0xfd,0xd4,0x6a,0x04,0x00,0x00]
+
+v_cmpx_class_f16_e64 vcc_hi, s105
+// GFX1250: v_cmpx_class_f16_e64 vcc_hi, s105       ; encoding: [0x7e,0x00,0xfd,0xd4,0x6b,0xd2,0x00,0x00]
+
+v_cmpx_class_f16_e64 ttmp15, ttmp15
+// GFX1250: v_cmpx_class_f16_e64 ttmp15, ttmp15     ; encoding: [0x7e,0x00,0xfd,0xd4,0x7b,0xf6,0x00,0x00]
+
+v_cmpx_class_f16_e64 m0, src_scc
+// GFX1250: v_cmpx_class_f16_e64 m0, src_scc        ; encoding: [0x7e,0x00,0xfd,0xd4,0x7d,0xfa,0x01,0x00]
+
+v_cmpx_class_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_class_f16_e64 exec_lo, -1        ; encoding: [0x7e,0x00,0xfd,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_class_f16_e64 exec_hi, null
+// GFX1250: v_cmpx_class_f16_e64 exec_hi, null      ; encoding: [0x7e,0x00,0xfd,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_class_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_class_f16_e64 null, exec_lo      ; encoding: [0x7e,0x00,0xfd,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_class_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_class_f16_e64 -1, exec_hi        ; encoding: [0x7e,0x00,0xfd,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_class_f16_e64 0.5, m0
+// GFX1250: v_cmpx_class_f16_e64 0.5, m0            ; encoding: [0x7e,0x00,0xfd,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_class_f16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_class_f16_e64 src_scc, vcc_lo    ; encoding: [0x7e,0x00,0xfd,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi
+// GFX1250: v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi  ; encoding: [0x7e,0x01,0xfd,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_class_f32_e64 v1, v2
+// GFX1250: v_cmpx_class_f32_e64 v1, v2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_class_f32_e64 v255, v255
+// GFX1250: v_cmpx_class_f32_e64 v255, v255         ; encoding: [0x7e,0x00,0xfe,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_class_f32_e64 s1, s2
+// GFX1250: v_cmpx_class_f32_e64 s1, s2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_class_f32_e64 s105, s105
+// GFX1250: v_cmpx_class_f32_e64 s105, s105         ; encoding: [0x7e,0x00,0xfe,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_class_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_class_f32_e64 vcc_lo, ttmp15     ; encoding: [0x7e,0x00,0xfe,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_class_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_class_f32_e64 vcc_hi, 0xaf123456 ; encoding: [0x7e,0x00,0xfe,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_class_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_class_f32_e64 ttmp15, src_scc    ; encoding: [0x7e,0x00,0xfe,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_class_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_class_f32_e64 m0, 0.5            ; encoding: [0x7e,0x00,0xfe,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_class_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_class_f32_e64 exec_lo, -1        ; encoding: [0x7e,0x00,0xfe,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_class_f32_e64 exec_hi, null
+// GFX1250: v_cmpx_class_f32_e64 exec_hi, null      ; encoding: [0x7e,0x00,0xfe,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_class_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_class_f32_e64 null, exec_lo      ; encoding: [0x7e,0x00,0xfe,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_class_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_class_f32_e64 -1, exec_hi        ; encoding: [0x7e,0x00,0xfe,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_class_f32_e64 0.5, m0
+// GFX1250: v_cmpx_class_f32_e64 0.5, m0            ; encoding: [0x7e,0x00,0xfe,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_class_f32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_class_f32_e64 src_scc, vcc_lo    ; encoding: [0x7e,0x00,0xfe,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_class_f32_e64 -|0xaf123456|, vcc_hi
+// GFX1250: v_cmpx_class_f32_e64 -|0xaf123456|, vcc_hi ; encoding: [0x7e,0x01,0xfe,0xd4,0xff,0xd6,0x00,0x20,0x56,0x34,0x12,0xaf]
+
+v_cmpx_class_f64_e64 v[2:3], v2
+// GFX1250: v_cmpx_class_f64_e64 v[2:3], v2         ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_class_f64_e64 v[2:3], v255
+// GFX1250: v_cmpx_class_f64_e64 v[2:3], v255       ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0xff,0x03,0x00]
+
+v_cmpx_class_f64_e64 v[2:3], s2
+// GFX1250: v_cmpx_class_f64_e64 v[2:3], s2         ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0x05,0x00,0x00]
+
+v_cmpx_class_f64_e64 v[2:3], s105
+// GFX1250: v_cmpx_class_f64_e64 v[2:3], s105       ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0xd3,0x00,0x00]
+
+v_cmpx_class_f64_e64 v[254:255], ttmp15
+// GFX1250: v_cmpx_class_f64_e64 v[254:255], ttmp15 ; encoding: [0x7e,0x00,0xff,0xd4,0xfe,0xf7,0x00,0x00]
+
+v_cmpx_class_f64_e64 s[2:3], vcc_hi
+// GFX1250: v_cmpx_class_f64_e64 s[2:3], vcc_hi     ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0xd6,0x00,0x00]
+
+v_cmpx_class_f64_e64 s[104:105], vcc_lo
+// GFX1250: v_cmpx_class_f64_e64 s[104:105], vcc_lo ; encoding: [0x7e,0x00,0xff,0xd4,0x68,0xd4,0x00,0x00]
+
+v_cmpx_class_f64_e64 vcc, m0
+// GFX1250: v_cmpx_class_f64_e64 vcc, m0            ; encoding: [0x7e,0x00,0xff,0xd4,0x6a,0xfa,0x00,0x00]
+
+v_cmpx_class_f64_e64 ttmp[14:15], exec_hi
+// GFX1250: v_cmpx_class_f64_e64 ttmp[14:15], exec_hi ; encoding: [0x7e,0x00,0xff,0xd4,0x7a,0xfe,0x00,0x00]
+
+v_cmpx_class_f64_e64 exec, exec_lo
+// GFX1250: v_cmpx_class_f64_e64 exec, exec_lo      ; encoding: [0x7e,0x00,0xff,0xd4,0x7e,0xfc,0x00,0x00]
+
+v_cmpx_class_f64_e64 null, null
+// GFX1250: v_cmpx_class_f64_e64 null, null         ; encoding: [0x7e,0x00,0xff,0xd4,0x7c,0xf8,0x00,0x00]
+
+v_cmpx_class_f64_e64 -1, -1
+// GFX1250: v_cmpx_class_f64_e64 -1, -1             ; encoding: [0x7e,0x00,0xff,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_class_f64_e64 0.5, 0.5
+// GFX1250: v_cmpx_class_f64_e64 0.5, 0.5           ; encoding: [0x7e,0x00,0xff,0xd4,0xf0,0xe0,0x01,0x00]
+
+v_cmpx_class_f64_e64 -|src_scc|, src_scc
+// GFX1250: v_cmpx_class_f64_e64 -|src_scc|, src_scc ; encoding: [0x7e,0x01,0xff,0xd4,0xfd,0xfa,0x01,0x20]
+
+v_cmpx_class_f64_e64 0xaf123456, 0xaf123456
+// GFX1250: v_cmpx_class_f64_e64 0xaf123456, 0xaf123456 ; encoding: [0x7e,0x00,0xff,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f16_e64 v1, v2
+// GFX1250: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_f16_e64 v255, v255
+// GFX1250: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_f16_e64 s1, s2
+// GFX1250: v_cmpx_eq_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_f16_e64 s105, s105
+// GFX1250: v_cmpx_eq_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x82,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_eq_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x82,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_eq_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x82,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_eq_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x82,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_eq_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x82,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_eq_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x82,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_eq_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x82,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_eq_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x82,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_eq_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x82,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_eq_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x82,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_eq_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_eq_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x82,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_f32_e64 v1, v2
+// GFX1250: v_cmpx_eq_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_f32_e64 v255, v255
+// GFX1250: v_cmpx_eq_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x92,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_f32_e64 s1, s2
+// GFX1250: v_cmpx_eq_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_f32_e64 s105, s105
+// GFX1250: v_cmpx_eq_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x92,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_eq_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x92,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_eq_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x92,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_eq_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x92,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_eq_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x92,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_eq_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x92,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_eq_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x92,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_eq_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x92,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_eq_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x92,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_eq_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x92,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_eq_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_eq_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x92,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_eq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_eq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x92,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_eq_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa2,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_eq_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_eq_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa2,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_eq_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_eq_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa2,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_eq_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_eq_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa2,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_eq_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_eq_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa2,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_eq_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_eq_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa2,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_eq_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa2,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_eq_f64_e64 null, 0.5
+// GFX1250: v_cmpx_eq_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa2,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_eq_f64_e64 -1, -1
+// GFX1250: v_cmpx_eq_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa2,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_eq_f64_e64 0.5, null
+// GFX1250: v_cmpx_eq_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa2,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_eq_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_eq_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa2,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_eq_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_eq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa2,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i16_e64 v1, v2
+// GFX1250: v_cmpx_eq_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_i16_e64 v255, v255
+// GFX1250: v_cmpx_eq_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_i16_e64 s1, s2
+// GFX1250: v_cmpx_eq_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_i16_e64 s105, s105
+// GFX1250: v_cmpx_eq_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb2,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_i16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_eq_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb2,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_i16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_eq_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb2,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_i16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_eq_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb2,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_i16_e64 m0, 0.5
+// GFX1250: v_cmpx_eq_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_i16_e64 exec_lo, -1
+// GFX1250: v_cmpx_eq_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb2,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_i16_e64 exec_hi, null
+// GFX1250: v_cmpx_eq_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb2,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_i16_e64 null, exec_lo
+// GFX1250: v_cmpx_eq_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb2,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_i16_e64 -1, exec_hi
+// GFX1250: v_cmpx_eq_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb2,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_i16_e64 0.5, m0
+// GFX1250: v_cmpx_eq_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb2,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_eq_i16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_eq_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb2,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_eq_i16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_i32_e64 v1, v2
+// GFX1250: v_cmpx_eq_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_i32_e64 v255, v255
+// GFX1250: v_cmpx_eq_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc2,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_i32_e64 s1, s2
+// GFX1250: v_cmpx_eq_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_i32_e64 s105, s105
+// GFX1250: v_cmpx_eq_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc2,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_i32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_eq_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc2,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_i32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_eq_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc2,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_eq_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc2,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_i32_e64 m0, 0.5
+// GFX1250: v_cmpx_eq_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc2,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_i32_e64 exec_lo, -1
+// GFX1250: v_cmpx_eq_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc2,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_i32_e64 exec_hi, null
+// GFX1250: v_cmpx_eq_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc2,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_i32_e64 null, exec_lo
+// GFX1250: v_cmpx_eq_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc2,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_i32_e64 -1, exec_hi
+// GFX1250: v_cmpx_eq_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc2,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_i32_e64 0.5, m0
+// GFX1250: v_cmpx_eq_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc2,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_eq_i32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_eq_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc2,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_eq_i32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_eq_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc2,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_i64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_eq_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd2,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_eq_i64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_eq_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd2,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_eq_i64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_eq_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd2,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_eq_i64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_eq_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd2,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_eq_i64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_eq_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd2,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_eq_i64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_eq_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd2,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_eq_i64_e64 exec, src_scc
+// GFX1250: v_cmpx_eq_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd2,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_eq_i64_e64 null, 0.5
+// GFX1250: v_cmpx_eq_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd2,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_eq_i64_e64 -1, -1
+// GFX1250: v_cmpx_eq_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd2,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_eq_i64_e64 0.5, null
+// GFX1250: v_cmpx_eq_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd2,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_eq_i64_e64 src_scc, exec
+// GFX1250: v_cmpx_eq_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd2,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_eq_i64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_eq_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd2,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_eq_u16_e64 v1, v2
+// GFX1250: v_cmpx_eq_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_u16_e64 v255, v255
+// GFX1250: v_cmpx_eq_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_u16_e64 s1, s2
+// GFX1250: v_cmpx_eq_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_u16_e64 s105, s105
+// GFX1250: v_cmpx_eq_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xba,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_u16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_eq_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xba,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_u16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_eq_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xba,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_u16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_eq_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xba,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_u16_e64 m0, 0.5
+// GFX1250: v_cmpx_eq_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_u16_e64 exec_lo, -1
+// GFX1250: v_cmpx_eq_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xba,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_u16_e64 exec_hi, null
+// GFX1250: v_cmpx_eq_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xba,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_u16_e64 null, exec_lo
+// GFX1250: v_cmpx_eq_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xba,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_u16_e64 -1, exec_hi
+// GFX1250: v_cmpx_eq_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xba,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_u16_e64 0.5, m0
+// GFX1250: v_cmpx_eq_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xba,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_eq_u16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_eq_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xba,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_eq_u16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_eq_u32_e64 v1, v2
+// GFX1250: v_cmpx_eq_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_eq_u32_e64 v255, v255
+// GFX1250: v_cmpx_eq_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xca,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_eq_u32_e64 s1, s2
+// GFX1250: v_cmpx_eq_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_eq_u32_e64 s105, s105
+// GFX1250: v_cmpx_eq_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xca,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_eq_u32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_eq_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xca,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_eq_u32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_eq_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xca,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_eq_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xca,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_eq_u32_e64 m0, 0.5
+// GFX1250: v_cmpx_eq_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xca,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_eq_u32_e64 exec_lo, -1
+// GFX1250: v_cmpx_eq_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xca,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_eq_u32_e64 exec_hi, null
+// GFX1250: v_cmpx_eq_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xca,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_eq_u32_e64 null, exec_lo
+// GFX1250: v_cmpx_eq_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xca,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_eq_u32_e64 -1, exec_hi
+// GFX1250: v_cmpx_eq_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xca,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_eq_u32_e64 0.5, m0
+// GFX1250: v_cmpx_eq_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xca,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_eq_u32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_eq_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xca,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_eq_u32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_eq_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xca,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_eq_u64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_eq_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xda,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_eq_u64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_eq_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xda,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_eq_u64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_eq_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xda,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_eq_u64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_eq_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xda,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_eq_u64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_eq_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xda,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_eq_u64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_eq_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xda,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_eq_u64_e64 exec, src_scc
+// GFX1250: v_cmpx_eq_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xda,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_eq_u64_e64 null, 0.5
+// GFX1250: v_cmpx_eq_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xda,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_eq_u64_e64 -1, -1
+// GFX1250: v_cmpx_eq_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xda,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_eq_u64_e64 0.5, null
+// GFX1250: v_cmpx_eq_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xda,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_eq_u64_e64 src_scc, exec
+// GFX1250: v_cmpx_eq_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xda,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_eq_u64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_eq_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xda,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_ge_f16_e64 v1, v2
+// GFX1250: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_f16_e64 v255, v255
+// GFX1250: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_f16_e64 s1, s2
+// GFX1250: v_cmpx_ge_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_f16_e64 s105, s105
+// GFX1250: v_cmpx_ge_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x86,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ge_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x86,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_ge_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x86,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ge_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x86,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_ge_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x86,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_ge_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x86,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_ge_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x86,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_ge_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x86,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_ge_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x86,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_ge_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x86,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_ge_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_ge_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x86,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_f32_e64 v1, v2
+// GFX1250: v_cmpx_ge_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_f32_e64 v255, v255
+// GFX1250: v_cmpx_ge_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x96,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_f32_e64 s1, s2
+// GFX1250: v_cmpx_ge_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_f32_e64 s105, s105
+// GFX1250: v_cmpx_ge_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x96,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ge_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x96,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_ge_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x96,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ge_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x96,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_ge_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x96,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_ge_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x96,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_ge_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x96,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_ge_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x96,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_ge_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x96,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_ge_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x96,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_ge_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_ge_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x96,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_ge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_ge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x96,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_ge_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa6,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_ge_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_ge_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa6,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ge_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_ge_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa6,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ge_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_ge_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa6,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ge_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_ge_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa6,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ge_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_ge_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa6,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_ge_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa6,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_ge_f64_e64 null, 0.5
+// GFX1250: v_cmpx_ge_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa6,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ge_f64_e64 -1, -1
+// GFX1250: v_cmpx_ge_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa6,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ge_f64_e64 0.5, null
+// GFX1250: v_cmpx_ge_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa6,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ge_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_ge_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa6,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_ge_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_ge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa6,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i16_e64 v1, v2
+// GFX1250: v_cmpx_ge_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_i16_e64 v255, v255
+// GFX1250: v_cmpx_ge_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_i16_e64 s1, s2
+// GFX1250: v_cmpx_ge_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_i16_e64 s105, s105
+// GFX1250: v_cmpx_ge_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb6,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_i16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ge_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb6,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_i16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_ge_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb6,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_i16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ge_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb6,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_i16_e64 m0, 0.5
+// GFX1250: v_cmpx_ge_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_i16_e64 exec_lo, -1
+// GFX1250: v_cmpx_ge_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb6,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_i16_e64 exec_hi, null
+// GFX1250: v_cmpx_ge_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb6,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_i16_e64 null, exec_lo
+// GFX1250: v_cmpx_ge_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb6,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_i16_e64 -1, exec_hi
+// GFX1250: v_cmpx_ge_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb6,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_i16_e64 0.5, m0
+// GFX1250: v_cmpx_ge_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb6,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ge_i16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_ge_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb6,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ge_i16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_i32_e64 v1, v2
+// GFX1250: v_cmpx_ge_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_i32_e64 v255, v255
+// GFX1250: v_cmpx_ge_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc6,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_i32_e64 s1, s2
+// GFX1250: v_cmpx_ge_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_i32_e64 s105, s105
+// GFX1250: v_cmpx_ge_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc6,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_i32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ge_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc6,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_i32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_ge_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc6,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ge_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc6,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_i32_e64 m0, 0.5
+// GFX1250: v_cmpx_ge_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc6,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_i32_e64 exec_lo, -1
+// GFX1250: v_cmpx_ge_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc6,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_i32_e64 exec_hi, null
+// GFX1250: v_cmpx_ge_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc6,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_i32_e64 null, exec_lo
+// GFX1250: v_cmpx_ge_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc6,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_i32_e64 -1, exec_hi
+// GFX1250: v_cmpx_ge_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc6,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_i32_e64 0.5, m0
+// GFX1250: v_cmpx_ge_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc6,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ge_i32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_ge_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc6,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ge_i32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_ge_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc6,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_i64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_ge_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd6,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_ge_i64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_ge_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd6,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ge_i64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_ge_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd6,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ge_i64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_ge_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd6,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ge_i64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_ge_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd6,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ge_i64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_ge_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd6,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_ge_i64_e64 exec, src_scc
+// GFX1250: v_cmpx_ge_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd6,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_ge_i64_e64 null, 0.5
+// GFX1250: v_cmpx_ge_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd6,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ge_i64_e64 -1, -1
+// GFX1250: v_cmpx_ge_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd6,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ge_i64_e64 0.5, null
+// GFX1250: v_cmpx_ge_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd6,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ge_i64_e64 src_scc, exec
+// GFX1250: v_cmpx_ge_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd6,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_ge_i64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_ge_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd6,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_ge_u16_e64 v1, v2
+// GFX1250: v_cmpx_ge_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_u16_e64 v255, v255
+// GFX1250: v_cmpx_ge_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_u16_e64 s1, s2
+// GFX1250: v_cmpx_ge_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_u16_e64 s105, s105
+// GFX1250: v_cmpx_ge_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbe,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_u16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ge_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbe,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_u16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_ge_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbe,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_u16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ge_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbe,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_u16_e64 m0, 0.5
+// GFX1250: v_cmpx_ge_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_u16_e64 exec_lo, -1
+// GFX1250: v_cmpx_ge_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbe,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_u16_e64 exec_hi, null
+// GFX1250: v_cmpx_ge_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbe,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_u16_e64 null, exec_lo
+// GFX1250: v_cmpx_ge_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbe,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_u16_e64 -1, exec_hi
+// GFX1250: v_cmpx_ge_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbe,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_u16_e64 0.5, m0
+// GFX1250: v_cmpx_ge_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xbe,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ge_u16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_ge_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbe,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ge_u16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ge_u32_e64 v1, v2
+// GFX1250: v_cmpx_ge_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ge_u32_e64 v255, v255
+// GFX1250: v_cmpx_ge_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xce,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ge_u32_e64 s1, s2
+// GFX1250: v_cmpx_ge_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ge_u32_e64 s105, s105
+// GFX1250: v_cmpx_ge_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xce,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ge_u32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ge_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xce,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ge_u32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_ge_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xce,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ge_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xce,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ge_u32_e64 m0, 0.5
+// GFX1250: v_cmpx_ge_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xce,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ge_u32_e64 exec_lo, -1
+// GFX1250: v_cmpx_ge_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xce,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ge_u32_e64 exec_hi, null
+// GFX1250: v_cmpx_ge_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xce,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ge_u32_e64 null, exec_lo
+// GFX1250: v_cmpx_ge_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xce,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ge_u32_e64 -1, exec_hi
+// GFX1250: v_cmpx_ge_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xce,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ge_u32_e64 0.5, m0
+// GFX1250: v_cmpx_ge_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xce,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ge_u32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_ge_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xce,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ge_u32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_ge_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xce,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ge_u64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_ge_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xde,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_ge_u64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_ge_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xde,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ge_u64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_ge_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xde,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ge_u64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_ge_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xde,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ge_u64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_ge_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xde,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ge_u64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_ge_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xde,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_ge_u64_e64 exec, src_scc
+// GFX1250: v_cmpx_ge_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xde,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_ge_u64_e64 null, 0.5
+// GFX1250: v_cmpx_ge_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xde,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ge_u64_e64 -1, -1
+// GFX1250: v_cmpx_ge_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xde,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ge_u64_e64 0.5, null
+// GFX1250: v_cmpx_ge_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xde,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ge_u64_e64 src_scc, exec
+// GFX1250: v_cmpx_ge_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xde,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_ge_u64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_ge_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xde,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_gt_f16_e64 v1, v2
+// GFX1250: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_f16_e64 v255, v255
+// GFX1250: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_f16_e64 s1, s2
+// GFX1250: v_cmpx_gt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_f16_e64 s105, s105
+// GFX1250: v_cmpx_gt_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x84,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_gt_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x84,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_gt_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x84,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_gt_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x84,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_gt_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x84,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_gt_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x84,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_gt_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x84,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_gt_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x84,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_gt_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x84,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_gt_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x84,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_gt_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_gt_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x84,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_f32_e64 v1, v2
+// GFX1250: v_cmpx_gt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_f32_e64 v255, v255
+// GFX1250: v_cmpx_gt_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x94,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_f32_e64 s1, s2
+// GFX1250: v_cmpx_gt_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_f32_e64 s105, s105
+// GFX1250: v_cmpx_gt_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x94,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_gt_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x94,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_gt_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x94,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_gt_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x94,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_gt_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x94,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_gt_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x94,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_gt_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x94,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_gt_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x94,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_gt_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x94,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_gt_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x94,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_gt_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_gt_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x94,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_gt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_gt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x94,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_gt_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa4,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_gt_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_gt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa4,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_gt_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_gt_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa4,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_gt_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_gt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa4,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_gt_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_gt_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa4,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_gt_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_gt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa4,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_gt_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa4,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_gt_f64_e64 null, 0.5
+// GFX1250: v_cmpx_gt_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa4,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_gt_f64_e64 -1, -1
+// GFX1250: v_cmpx_gt_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa4,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_gt_f64_e64 0.5, null
+// GFX1250: v_cmpx_gt_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa4,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_gt_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_gt_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa4,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_gt_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_gt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa4,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i16_e64 v1, v2
+// GFX1250: v_cmpx_gt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_i16_e64 v255, v255
+// GFX1250: v_cmpx_gt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_i16_e64 s1, s2
+// GFX1250: v_cmpx_gt_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_i16_e64 s105, s105
+// GFX1250: v_cmpx_gt_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb4,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_i16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_gt_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb4,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_i16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_gt_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb4,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_i16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_gt_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb4,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_i16_e64 m0, 0.5
+// GFX1250: v_cmpx_gt_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_i16_e64 exec_lo, -1
+// GFX1250: v_cmpx_gt_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb4,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_i16_e64 exec_hi, null
+// GFX1250: v_cmpx_gt_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb4,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_i16_e64 null, exec_lo
+// GFX1250: v_cmpx_gt_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb4,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_i16_e64 -1, exec_hi
+// GFX1250: v_cmpx_gt_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb4,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_i16_e64 0.5, m0
+// GFX1250: v_cmpx_gt_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb4,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_gt_i16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_gt_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb4,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_gt_i16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_i32_e64 v1, v2
+// GFX1250: v_cmpx_gt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_i32_e64 v255, v255
+// GFX1250: v_cmpx_gt_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc4,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_i32_e64 s1, s2
+// GFX1250: v_cmpx_gt_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_i32_e64 s105, s105
+// GFX1250: v_cmpx_gt_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc4,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_i32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_gt_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc4,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_i32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_gt_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc4,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_gt_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc4,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_i32_e64 m0, 0.5
+// GFX1250: v_cmpx_gt_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc4,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_i32_e64 exec_lo, -1
+// GFX1250: v_cmpx_gt_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc4,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_i32_e64 exec_hi, null
+// GFX1250: v_cmpx_gt_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc4,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_i32_e64 null, exec_lo
+// GFX1250: v_cmpx_gt_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc4,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_i32_e64 -1, exec_hi
+// GFX1250: v_cmpx_gt_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc4,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_i32_e64 0.5, m0
+// GFX1250: v_cmpx_gt_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc4,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_gt_i32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_gt_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc4,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_gt_i32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_gt_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc4,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_i64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_gt_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd4,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_gt_i64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_gt_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd4,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_gt_i64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_gt_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd4,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_gt_i64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_gt_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd4,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_gt_i64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_gt_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd4,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_gt_i64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_gt_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd4,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_gt_i64_e64 exec, src_scc
+// GFX1250: v_cmpx_gt_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd4,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_gt_i64_e64 null, 0.5
+// GFX1250: v_cmpx_gt_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd4,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_gt_i64_e64 -1, -1
+// GFX1250: v_cmpx_gt_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd4,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_gt_i64_e64 0.5, null
+// GFX1250: v_cmpx_gt_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd4,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_gt_i64_e64 src_scc, exec
+// GFX1250: v_cmpx_gt_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd4,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_gt_i64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_gt_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd4,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_gt_u16_e64 v1, v2
+// GFX1250: v_cmpx_gt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_u16_e64 v255, v255
+// GFX1250: v_cmpx_gt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_u16_e64 s1, s2
+// GFX1250: v_cmpx_gt_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_u16_e64 s105, s105
+// GFX1250: v_cmpx_gt_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbc,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_u16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_gt_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbc,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_u16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_gt_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbc,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_u16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_gt_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbc,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_u16_e64 m0, 0.5
+// GFX1250: v_cmpx_gt_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_u16_e64 exec_lo, -1
+// GFX1250: v_cmpx_gt_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbc,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_u16_e64 exec_hi, null
+// GFX1250: v_cmpx_gt_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbc,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_u16_e64 null, exec_lo
+// GFX1250: v_cmpx_gt_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbc,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_u16_e64 -1, exec_hi
+// GFX1250: v_cmpx_gt_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbc,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_u16_e64 0.5, m0
+// GFX1250: v_cmpx_gt_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xbc,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_gt_u16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_gt_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbc,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_gt_u16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_gt_u32_e64 v1, v2
+// GFX1250: v_cmpx_gt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_gt_u32_e64 v255, v255
+// GFX1250: v_cmpx_gt_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xcc,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_gt_u32_e64 s1, s2
+// GFX1250: v_cmpx_gt_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_gt_u32_e64 s105, s105
+// GFX1250: v_cmpx_gt_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xcc,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_gt_u32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_gt_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xcc,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_gt_u32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_gt_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xcc,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_gt_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xcc,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_gt_u32_e64 m0, 0.5
+// GFX1250: v_cmpx_gt_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xcc,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_gt_u32_e64 exec_lo, -1
+// GFX1250: v_cmpx_gt_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xcc,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_gt_u32_e64 exec_hi, null
+// GFX1250: v_cmpx_gt_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xcc,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_gt_u32_e64 null, exec_lo
+// GFX1250: v_cmpx_gt_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xcc,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_gt_u32_e64 -1, exec_hi
+// GFX1250: v_cmpx_gt_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xcc,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_gt_u32_e64 0.5, m0
+// GFX1250: v_cmpx_gt_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xcc,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_gt_u32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_gt_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xcc,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_gt_u32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_gt_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xcc,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_gt_u64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_gt_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xdc,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_gt_u64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_gt_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdc,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_gt_u64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_gt_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xdc,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_gt_u64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_gt_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdc,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_gt_u64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_gt_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xdc,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_gt_u64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_gt_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xdc,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_gt_u64_e64 exec, src_scc
+// GFX1250: v_cmpx_gt_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xdc,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_gt_u64_e64 null, 0.5
+// GFX1250: v_cmpx_gt_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xdc,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_gt_u64_e64 -1, -1
+// GFX1250: v_cmpx_gt_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xdc,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_gt_u64_e64 0.5, null
+// GFX1250: v_cmpx_gt_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xdc,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_gt_u64_e64 src_scc, exec
+// GFX1250: v_cmpx_gt_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xdc,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_gt_u64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_gt_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xdc,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_le_f16_e64 v1, v2
+// GFX1250: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_f16_e64 v255, v255
+// GFX1250: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_f16_e64 s1, s2
+// GFX1250: v_cmpx_le_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_f16_e64 s105, s105
+// GFX1250: v_cmpx_le_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x83,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_le_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x83,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_le_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x83,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_le_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x83,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_le_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x83,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_le_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x83,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_le_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x83,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_le_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x83,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_le_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x83,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_le_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x83,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_le_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_le_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x83,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_f32_e64 v1, v2
+// GFX1250: v_cmpx_le_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_f32_e64 v255, v255
+// GFX1250: v_cmpx_le_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x93,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_f32_e64 s1, s2
+// GFX1250: v_cmpx_le_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_f32_e64 s105, s105
+// GFX1250: v_cmpx_le_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x93,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_le_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x93,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_le_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x93,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_le_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x93,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_le_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x93,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_le_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x93,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_le_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x93,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_le_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x93,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_le_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x93,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_le_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x93,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_le_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_le_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x93,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_le_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_le_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x93,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_le_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa3,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_le_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_le_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa3,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_le_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_le_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa3,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_le_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_le_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa3,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_le_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_le_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa3,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_le_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_le_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa3,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_le_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa3,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_le_f64_e64 null, 0.5
+// GFX1250: v_cmpx_le_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa3,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_le_f64_e64 -1, -1
+// GFX1250: v_cmpx_le_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa3,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_le_f64_e64 0.5, null
+// GFX1250: v_cmpx_le_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa3,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_le_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_le_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa3,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_le_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_le_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa3,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i16_e64 v1, v2
+// GFX1250: v_cmpx_le_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_i16_e64 v255, v255
+// GFX1250: v_cmpx_le_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_i16_e64 s1, s2
+// GFX1250: v_cmpx_le_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_i16_e64 s105, s105
+// GFX1250: v_cmpx_le_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb3,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_i16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_le_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb3,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_i16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_le_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb3,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_i16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_le_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb3,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_i16_e64 m0, 0.5
+// GFX1250: v_cmpx_le_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_i16_e64 exec_lo, -1
+// GFX1250: v_cmpx_le_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb3,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_i16_e64 exec_hi, null
+// GFX1250: v_cmpx_le_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb3,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_i16_e64 null, exec_lo
+// GFX1250: v_cmpx_le_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb3,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_i16_e64 -1, exec_hi
+// GFX1250: v_cmpx_le_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb3,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_i16_e64 0.5, m0
+// GFX1250: v_cmpx_le_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb3,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_le_i16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_le_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb3,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_le_i16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_i32_e64 v1, v2
+// GFX1250: v_cmpx_le_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_i32_e64 v255, v255
+// GFX1250: v_cmpx_le_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc3,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_i32_e64 s1, s2
+// GFX1250: v_cmpx_le_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_i32_e64 s105, s105
+// GFX1250: v_cmpx_le_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc3,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_i32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_le_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc3,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_i32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_le_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc3,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_le_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc3,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_i32_e64 m0, 0.5
+// GFX1250: v_cmpx_le_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc3,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_i32_e64 exec_lo, -1
+// GFX1250: v_cmpx_le_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc3,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_i32_e64 exec_hi, null
+// GFX1250: v_cmpx_le_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc3,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_i32_e64 null, exec_lo
+// GFX1250: v_cmpx_le_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc3,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_i32_e64 -1, exec_hi
+// GFX1250: v_cmpx_le_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc3,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_i32_e64 0.5, m0
+// GFX1250: v_cmpx_le_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc3,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_le_i32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_le_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc3,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_le_i32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_le_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc3,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_i64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_le_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd3,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_le_i64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_le_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd3,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_le_i64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_le_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd3,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_le_i64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_le_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd3,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_le_i64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_le_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd3,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_le_i64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_le_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd3,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_le_i64_e64 exec, src_scc
+// GFX1250: v_cmpx_le_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd3,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_le_i64_e64 null, 0.5
+// GFX1250: v_cmpx_le_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd3,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_le_i64_e64 -1, -1
+// GFX1250: v_cmpx_le_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd3,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_le_i64_e64 0.5, null
+// GFX1250: v_cmpx_le_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd3,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_le_i64_e64 src_scc, exec
+// GFX1250: v_cmpx_le_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd3,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_le_i64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_le_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd3,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_le_u16_e64 v1, v2
+// GFX1250: v_cmpx_le_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_u16_e64 v255, v255
+// GFX1250: v_cmpx_le_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_u16_e64 s1, s2
+// GFX1250: v_cmpx_le_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_u16_e64 s105, s105
+// GFX1250: v_cmpx_le_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbb,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_u16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_le_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbb,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_u16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_le_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbb,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_u16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_le_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbb,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_u16_e64 m0, 0.5
+// GFX1250: v_cmpx_le_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_u16_e64 exec_lo, -1
+// GFX1250: v_cmpx_le_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbb,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_u16_e64 exec_hi, null
+// GFX1250: v_cmpx_le_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbb,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_u16_e64 null, exec_lo
+// GFX1250: v_cmpx_le_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbb,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_u16_e64 -1, exec_hi
+// GFX1250: v_cmpx_le_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbb,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_u16_e64 0.5, m0
+// GFX1250: v_cmpx_le_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xbb,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_le_u16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_le_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbb,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_le_u16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_le_u32_e64 v1, v2
+// GFX1250: v_cmpx_le_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_le_u32_e64 v255, v255
+// GFX1250: v_cmpx_le_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xcb,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_le_u32_e64 s1, s2
+// GFX1250: v_cmpx_le_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_le_u32_e64 s105, s105
+// GFX1250: v_cmpx_le_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xcb,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_le_u32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_le_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xcb,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_le_u32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_le_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xcb,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_le_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xcb,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_le_u32_e64 m0, 0.5
+// GFX1250: v_cmpx_le_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xcb,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_le_u32_e64 exec_lo, -1
+// GFX1250: v_cmpx_le_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xcb,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_le_u32_e64 exec_hi, null
+// GFX1250: v_cmpx_le_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xcb,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_le_u32_e64 null, exec_lo
+// GFX1250: v_cmpx_le_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xcb,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_le_u32_e64 -1, exec_hi
+// GFX1250: v_cmpx_le_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xcb,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_le_u32_e64 0.5, m0
+// GFX1250: v_cmpx_le_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xcb,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_le_u32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_le_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xcb,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_le_u32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_le_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xcb,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_le_u64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_le_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xdb,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_le_u64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_le_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdb,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_le_u64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_le_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xdb,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_le_u64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_le_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdb,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_le_u64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_le_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xdb,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_le_u64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_le_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xdb,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_le_u64_e64 exec, src_scc
+// GFX1250: v_cmpx_le_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xdb,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_le_u64_e64 null, 0.5
+// GFX1250: v_cmpx_le_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xdb,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_le_u64_e64 -1, -1
+// GFX1250: v_cmpx_le_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xdb,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_le_u64_e64 0.5, null
+// GFX1250: v_cmpx_le_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xdb,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_le_u64_e64 src_scc, exec
+// GFX1250: v_cmpx_le_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xdb,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_le_u64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_le_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xdb,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_lg_f16_e64 v1, v2
+// GFX1250: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lg_f16_e64 v255, v255
+// GFX1250: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lg_f16_e64 s1, s2
+// GFX1250: v_cmpx_lg_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lg_f16_e64 s105, s105
+// GFX1250: v_cmpx_lg_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x85,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lg_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_lg_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x85,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lg_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_lg_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x85,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lg_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_lg_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x85,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lg_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_lg_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x85,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lg_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_lg_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x85,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lg_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_lg_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x85,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lg_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_lg_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x85,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lg_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_lg_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x85,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lg_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_lg_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x85,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_lg_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_lg_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x85,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lg_f32_e64 v1, v2
+// GFX1250: v_cmpx_lg_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lg_f32_e64 v255, v255
+// GFX1250: v_cmpx_lg_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x95,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lg_f32_e64 s1, s2
+// GFX1250: v_cmpx_lg_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lg_f32_e64 s105, s105
+// GFX1250: v_cmpx_lg_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x95,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lg_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_lg_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x95,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lg_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_lg_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x95,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_lg_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x95,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lg_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_lg_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x95,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lg_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_lg_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x95,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lg_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_lg_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x95,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lg_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_lg_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x95,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lg_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_lg_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x95,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lg_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_lg_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x95,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_lg_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_lg_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x95,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_lg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_lg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x95,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_lg_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa5,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_lg_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_lg_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa5,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_lg_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_lg_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa5,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_lg_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_lg_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa5,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_lg_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_lg_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa5,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_lg_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_lg_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa5,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lg_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_lg_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa5,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_lg_f64_e64 null, 0.5
+// GFX1250: v_cmpx_lg_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa5,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_lg_f64_e64 -1, -1
+// GFX1250: v_cmpx_lg_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa5,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_lg_f64_e64 0.5, null
+// GFX1250: v_cmpx_lg_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa5,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_lg_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_lg_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa5,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_lg_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_lg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa5,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f16_e64 v1, v2
+// GFX1250: v_cmpx_lt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_f16_e64 v255, v255
+// GFX1250: v_cmpx_lt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x81,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_f16_e64 s1, s2
+// GFX1250: v_cmpx_lt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_f16_e64 s105, s105
+// GFX1250: v_cmpx_lt_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x81,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_lt_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x81,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_lt_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x81,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_lt_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x81,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_lt_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x81,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_lt_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x81,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_lt_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x81,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_lt_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x81,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_lt_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x81,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_lt_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x81,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_lt_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_lt_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x81,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_f32_e64 v1, v2
+// GFX1250: v_cmpx_lt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_f32_e64 v255, v255
+// GFX1250: v_cmpx_lt_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x91,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_f32_e64 s1, s2
+// GFX1250: v_cmpx_lt_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_f32_e64 s105, s105
+// GFX1250: v_cmpx_lt_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x91,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_lt_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x91,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_lt_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x91,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_lt_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x91,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_lt_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x91,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_lt_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x91,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_lt_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x91,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_lt_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x91,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_lt_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x91,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_lt_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x91,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_lt_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_lt_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x91,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_lt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_lt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x91,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_lt_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa1,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_lt_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_lt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa1,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_lt_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_lt_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa1,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_lt_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_lt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa1,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_lt_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_lt_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa1,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_lt_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_lt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa1,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_lt_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa1,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_lt_f64_e64 null, 0.5
+// GFX1250: v_cmpx_lt_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa1,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_lt_f64_e64 -1, -1
+// GFX1250: v_cmpx_lt_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa1,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_lt_f64_e64 0.5, null
+// GFX1250: v_cmpx_lt_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa1,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_lt_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_lt_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa1,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_lt_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_lt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa1,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i16_e64 v1, v2
+// GFX1250: v_cmpx_lt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_i16_e64 v255, v255
+// GFX1250: v_cmpx_lt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_i16_e64 s1, s2
+// GFX1250: v_cmpx_lt_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_i16_e64 s105, s105
+// GFX1250: v_cmpx_lt_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb1,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_i16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_lt_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb1,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_i16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_lt_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb1,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_i16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_lt_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb1,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_i16_e64 m0, 0.5
+// GFX1250: v_cmpx_lt_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_i16_e64 exec_lo, -1
+// GFX1250: v_cmpx_lt_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb1,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_i16_e64 exec_hi, null
+// GFX1250: v_cmpx_lt_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb1,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_i16_e64 null, exec_lo
+// GFX1250: v_cmpx_lt_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb1,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_i16_e64 -1, exec_hi
+// GFX1250: v_cmpx_lt_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb1,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_i16_e64 0.5, m0
+// GFX1250: v_cmpx_lt_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb1,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_lt_i16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_lt_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb1,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_lt_i16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_i32_e64 v1, v2
+// GFX1250: v_cmpx_lt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_i32_e64 v255, v255
+// GFX1250: v_cmpx_lt_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc1,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_i32_e64 s1, s2
+// GFX1250: v_cmpx_lt_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_i32_e64 s105, s105
+// GFX1250: v_cmpx_lt_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc1,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_i32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_lt_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc1,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_i32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_lt_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc1,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_lt_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc1,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_i32_e64 m0, 0.5
+// GFX1250: v_cmpx_lt_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc1,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_i32_e64 exec_lo, -1
+// GFX1250: v_cmpx_lt_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc1,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_i32_e64 exec_hi, null
+// GFX1250: v_cmpx_lt_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc1,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_i32_e64 null, exec_lo
+// GFX1250: v_cmpx_lt_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc1,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_i32_e64 -1, exec_hi
+// GFX1250: v_cmpx_lt_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc1,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_i32_e64 0.5, m0
+// GFX1250: v_cmpx_lt_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc1,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_lt_i32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_lt_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc1,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_lt_i32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_lt_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc1,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_i64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_lt_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd1,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_lt_i64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_lt_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd1,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_lt_i64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_lt_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd1,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_lt_i64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_lt_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd1,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_lt_i64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_lt_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd1,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_lt_i64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_lt_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd1,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_lt_i64_e64 exec, src_scc
+// GFX1250: v_cmpx_lt_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd1,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_lt_i64_e64 null, 0.5
+// GFX1250: v_cmpx_lt_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd1,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_lt_i64_e64 -1, -1
+// GFX1250: v_cmpx_lt_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd1,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_lt_i64_e64 0.5, null
+// GFX1250: v_cmpx_lt_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd1,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_lt_i64_e64 src_scc, exec
+// GFX1250: v_cmpx_lt_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd1,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_lt_i64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_lt_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd1,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_lt_u16_e64 v1, v2
+// GFX1250: v_cmpx_lt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_u16_e64 v255, v255
+// GFX1250: v_cmpx_lt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_u16_e64 s1, s2
+// GFX1250: v_cmpx_lt_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_u16_e64 s105, s105
+// GFX1250: v_cmpx_lt_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb9,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_u16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_lt_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb9,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_u16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_lt_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb9,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_u16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_lt_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb9,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_u16_e64 m0, 0.5
+// GFX1250: v_cmpx_lt_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_u16_e64 exec_lo, -1
+// GFX1250: v_cmpx_lt_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb9,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_u16_e64 exec_hi, null
+// GFX1250: v_cmpx_lt_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb9,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_u16_e64 null, exec_lo
+// GFX1250: v_cmpx_lt_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb9,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_u16_e64 -1, exec_hi
+// GFX1250: v_cmpx_lt_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb9,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_u16_e64 0.5, m0
+// GFX1250: v_cmpx_lt_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb9,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_lt_u16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_lt_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb9,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_lt_u16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_lt_u32_e64 v1, v2
+// GFX1250: v_cmpx_lt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_lt_u32_e64 v255, v255
+// GFX1250: v_cmpx_lt_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc9,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_lt_u32_e64 s1, s2
+// GFX1250: v_cmpx_lt_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_lt_u32_e64 s105, s105
+// GFX1250: v_cmpx_lt_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc9,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_lt_u32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_lt_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc9,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_lt_u32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_lt_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc9,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_lt_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc9,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_lt_u32_e64 m0, 0.5
+// GFX1250: v_cmpx_lt_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc9,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_lt_u32_e64 exec_lo, -1
+// GFX1250: v_cmpx_lt_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc9,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_lt_u32_e64 exec_hi, null
+// GFX1250: v_cmpx_lt_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc9,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_lt_u32_e64 null, exec_lo
+// GFX1250: v_cmpx_lt_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc9,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_lt_u32_e64 -1, exec_hi
+// GFX1250: v_cmpx_lt_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc9,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_lt_u32_e64 0.5, m0
+// GFX1250: v_cmpx_lt_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc9,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_lt_u32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_lt_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc9,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_lt_u32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_lt_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc9,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_lt_u64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_lt_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd9,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_lt_u64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_lt_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd9,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_lt_u64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_lt_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd9,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_lt_u64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_lt_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd9,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_lt_u64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_lt_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd9,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_lt_u64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_lt_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd9,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_lt_u64_e64 exec, src_scc
+// GFX1250: v_cmpx_lt_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd9,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_lt_u64_e64 null, 0.5
+// GFX1250: v_cmpx_lt_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd9,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_lt_u64_e64 -1, -1
+// GFX1250: v_cmpx_lt_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd9,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_lt_u64_e64 0.5, null
+// GFX1250: v_cmpx_lt_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd9,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_lt_u64_e64 src_scc, exec
+// GFX1250: v_cmpx_lt_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd9,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_lt_u64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_lt_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd9,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_ne_i16_e64 v1, v2
+// GFX1250: v_cmpx_ne_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_i16_e64 v255, v255
+// GFX1250: v_cmpx_ne_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ne_i16_e64 s1, s2
+// GFX1250: v_cmpx_ne_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ne_i16_e64 s105, s105
+// GFX1250: v_cmpx_ne_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb5,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ne_i16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ne_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb5,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ne_i16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_ne_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb5,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_i16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ne_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb5,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ne_i16_e64 m0, 0.5
+// GFX1250: v_cmpx_ne_i16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ne_i16_e64 exec_lo, -1
+// GFX1250: v_cmpx_ne_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb5,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ne_i16_e64 exec_hi, null
+// GFX1250: v_cmpx_ne_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb5,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ne_i16_e64 null, exec_lo
+// GFX1250: v_cmpx_ne_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb5,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ne_i16_e64 -1, exec_hi
+// GFX1250: v_cmpx_ne_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb5,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ne_i16_e64 0.5, m0
+// GFX1250: v_cmpx_ne_i16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xb5,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ne_i16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_ne_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb5,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ne_i16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_i32_e64 v1, v2
+// GFX1250: v_cmpx_ne_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_i32_e64 v255, v255
+// GFX1250: v_cmpx_ne_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc5,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ne_i32_e64 s1, s2
+// GFX1250: v_cmpx_ne_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ne_i32_e64 s105, s105
+// GFX1250: v_cmpx_ne_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc5,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ne_i32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ne_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc5,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ne_i32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_ne_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc5,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ne_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc5,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ne_i32_e64 m0, 0.5
+// GFX1250: v_cmpx_ne_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc5,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ne_i32_e64 exec_lo, -1
+// GFX1250: v_cmpx_ne_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc5,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ne_i32_e64 exec_hi, null
+// GFX1250: v_cmpx_ne_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc5,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ne_i32_e64 null, exec_lo
+// GFX1250: v_cmpx_ne_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc5,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ne_i32_e64 -1, exec_hi
+// GFX1250: v_cmpx_ne_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc5,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ne_i32_e64 0.5, m0
+// GFX1250: v_cmpx_ne_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc5,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ne_i32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_ne_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc5,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ne_i32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_ne_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc5,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_i64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_ne_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd5,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_ne_i64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_ne_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd5,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ne_i64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_ne_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd5,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ne_i64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_ne_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd5,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ne_i64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_ne_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd5,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ne_i64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_ne_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd5,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_ne_i64_e64 exec, src_scc
+// GFX1250: v_cmpx_ne_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd5,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_ne_i64_e64 null, 0.5
+// GFX1250: v_cmpx_ne_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd5,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ne_i64_e64 -1, -1
+// GFX1250: v_cmpx_ne_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd5,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ne_i64_e64 0.5, null
+// GFX1250: v_cmpx_ne_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd5,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ne_i64_e64 src_scc, exec
+// GFX1250: v_cmpx_ne_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd5,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_ne_i64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_ne_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd5,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_ne_u16_e64 v1, v2
+// GFX1250: v_cmpx_ne_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_u16_e64 v255, v255
+// GFX1250: v_cmpx_ne_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ne_u16_e64 s1, s2
+// GFX1250: v_cmpx_ne_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ne_u16_e64 s105, s105
+// GFX1250: v_cmpx_ne_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbd,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ne_u16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ne_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbd,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ne_u16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_ne_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbd,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_u16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ne_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbd,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ne_u16_e64 m0, 0.5
+// GFX1250: v_cmpx_ne_u16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ne_u16_e64 exec_lo, -1
+// GFX1250: v_cmpx_ne_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbd,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ne_u16_e64 exec_hi, null
+// GFX1250: v_cmpx_ne_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbd,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ne_u16_e64 null, exec_lo
+// GFX1250: v_cmpx_ne_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbd,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ne_u16_e64 -1, exec_hi
+// GFX1250: v_cmpx_ne_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbd,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ne_u16_e64 0.5, m0
+// GFX1250: v_cmpx_ne_u16_e64 0.5, m0               ; encoding: [0x7e,0x00,0xbd,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ne_u16_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_ne_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbd,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ne_u16_e64 0xfe0b, vcc_hi
+// GFX1250: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ne_u32_e64 v1, v2
+// GFX1250: v_cmpx_ne_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ne_u32_e64 v255, v255
+// GFX1250: v_cmpx_ne_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xcd,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ne_u32_e64 s1, s2
+// GFX1250: v_cmpx_ne_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ne_u32_e64 s105, s105
+// GFX1250: v_cmpx_ne_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xcd,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ne_u32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ne_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xcd,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ne_u32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_ne_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xcd,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ne_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xcd,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ne_u32_e64 m0, 0.5
+// GFX1250: v_cmpx_ne_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xcd,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ne_u32_e64 exec_lo, -1
+// GFX1250: v_cmpx_ne_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xcd,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ne_u32_e64 exec_hi, null
+// GFX1250: v_cmpx_ne_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xcd,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ne_u32_e64 null, exec_lo
+// GFX1250: v_cmpx_ne_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xcd,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ne_u32_e64 -1, exec_hi
+// GFX1250: v_cmpx_ne_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xcd,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ne_u32_e64 0.5, m0
+// GFX1250: v_cmpx_ne_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xcd,0xd4,0xf0,0xfa,0x00,0x00]
+
+v_cmpx_ne_u32_e64 src_scc, vcc_lo
+// GFX1250: v_cmpx_ne_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xcd,0xd4,0xfd,0xd4,0x00,0x00]
+
+v_cmpx_ne_u32_e64 0xaf123456, vcc_hi
+// GFX1250: v_cmpx_ne_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xcd,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ne_u64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_ne_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xdd,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_ne_u64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_ne_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdd,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ne_u64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_ne_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xdd,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ne_u64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_ne_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdd,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ne_u64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_ne_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xdd,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ne_u64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_ne_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xdd,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_ne_u64_e64 exec, src_scc
+// GFX1250: v_cmpx_ne_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xdd,0xd4,0x7e,0xfa,0x01,0x00]
+
+v_cmpx_ne_u64_e64 null, 0.5
+// GFX1250: v_cmpx_ne_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xdd,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ne_u64_e64 -1, -1
+// GFX1250: v_cmpx_ne_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xdd,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ne_u64_e64 0.5, null
+// GFX1250: v_cmpx_ne_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xdd,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ne_u64_e64 src_scc, exec
+// GFX1250: v_cmpx_ne_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xdd,0xd4,0xfd,0xfc,0x00,0x00]
+
+v_cmpx_ne_u64_e64 0xaf123456, vcc
+// GFX1250: v_cmpx_ne_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xdd,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+v_cmpx_neq_f16_e64 v1, v2
+// GFX1250: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_neq_f16_e64 v255, v255
+// GFX1250: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_neq_f16_e64 s1, s2
+// GFX1250: v_cmpx_neq_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_neq_f16_e64 s105, s105
+// GFX1250: v_cmpx_neq_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8d,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_neq_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_neq_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8d,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_neq_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_neq_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_neq_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_neq_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8d,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_neq_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_neq_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8d,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_neq_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_neq_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8d,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_neq_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_neq_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8d,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_neq_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_neq_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8d,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_neq_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_neq_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8d,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_neq_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_neq_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8d,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_neq_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_neq_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8d,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_neq_f32_e64 v1, v2
+// GFX1250: v_cmpx_neq_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_neq_f32_e64 v255, v255
+// GFX1250: v_cmpx_neq_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9d,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_neq_f32_e64 s1, s2
+// GFX1250: v_cmpx_neq_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_neq_f32_e64 s105, s105
+// GFX1250: v_cmpx_neq_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9d,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_neq_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_neq_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9d,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_neq_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_neq_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_neq_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9d,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_neq_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_neq_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9d,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_neq_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_neq_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9d,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_neq_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_neq_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9d,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_neq_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_neq_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9d,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_neq_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_neq_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9d,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_neq_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_neq_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9d,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_neq_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_neq_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9d,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_neq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_neq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9d,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_neq_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xad,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_neq_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_neq_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xad,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_neq_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_neq_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xad,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_neq_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_neq_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xad,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_neq_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_neq_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xad,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_neq_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_neq_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xad,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_neq_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_neq_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xad,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_neq_f64_e64 null, 0.5
+// GFX1250: v_cmpx_neq_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xad,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_neq_f64_e64 -1, -1
+// GFX1250: v_cmpx_neq_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xad,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_neq_f64_e64 0.5, null
+// GFX1250: v_cmpx_neq_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xad,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_neq_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_neq_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xad,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xad,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f16_e64 v1, v2
+// GFX1250: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nge_f16_e64 v255, v255
+// GFX1250: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nge_f16_e64 s1, s2
+// GFX1250: v_cmpx_nge_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nge_f16_e64 s105, s105
+// GFX1250: v_cmpx_nge_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x89,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nge_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_nge_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x89,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nge_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_nge_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x89,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nge_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_nge_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x89,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nge_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_nge_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x89,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nge_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_nge_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x89,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nge_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_nge_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x89,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nge_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_nge_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x89,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nge_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_nge_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x89,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nge_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_nge_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x89,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nge_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_nge_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x89,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nge_f32_e64 v1, v2
+// GFX1250: v_cmpx_nge_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nge_f32_e64 v255, v255
+// GFX1250: v_cmpx_nge_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x99,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nge_f32_e64 s1, s2
+// GFX1250: v_cmpx_nge_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nge_f32_e64 s105, s105
+// GFX1250: v_cmpx_nge_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x99,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nge_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_nge_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x99,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nge_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_nge_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x99,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_nge_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x99,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nge_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_nge_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x99,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nge_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_nge_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x99,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nge_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_nge_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x99,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nge_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_nge_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x99,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nge_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_nge_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x99,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nge_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_nge_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x99,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nge_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_nge_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x99,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_nge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x99,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_nge_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xa9,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_nge_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_nge_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa9,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_nge_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_nge_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xa9,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_nge_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_nge_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa9,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_nge_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_nge_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xa9,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_nge_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_nge_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa9,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nge_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_nge_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xa9,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_nge_f64_e64 null, 0.5
+// GFX1250: v_cmpx_nge_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xa9,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_nge_f64_e64 -1, -1
+// GFX1250: v_cmpx_nge_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xa9,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_nge_f64_e64 0.5, null
+// GFX1250: v_cmpx_nge_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xa9,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_nge_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_nge_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xa9,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa9,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f16_e64 v1, v2
+// GFX1250: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ngt_f16_e64 v255, v255
+// GFX1250: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ngt_f16_e64 s1, s2
+// GFX1250: v_cmpx_ngt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 s105, s105
+// GFX1250: v_cmpx_ngt_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8b,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ngt_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8b,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_ngt_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ngt_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8b,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ngt_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_ngt_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8b,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ngt_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_ngt_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8b,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ngt_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_ngt_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8b,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_ngt_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8b,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_ngt_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8b,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_ngt_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8b,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_ngt_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_ngt_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8b,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 v1, v2
+// GFX1250: v_cmpx_ngt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_ngt_f32_e64 v255, v255
+// GFX1250: v_cmpx_ngt_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9b,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_ngt_f32_e64 s1, s2
+// GFX1250: v_cmpx_ngt_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 s105, s105
+// GFX1250: v_cmpx_ngt_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9b,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_ngt_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9b,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_ngt_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_ngt_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9b,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_ngt_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_ngt_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9b,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_ngt_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_ngt_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9b,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_ngt_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_ngt_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9b,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_ngt_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9b,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_ngt_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9b,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_ngt_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_ngt_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9b,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_ngt_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_ngt_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9b,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_ngt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_ngt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9b,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_ngt_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xab,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_ngt_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_ngt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xab,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_ngt_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_ngt_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xab,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_ngt_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_ngt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xab,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_ngt_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_ngt_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xab,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_ngt_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_ngt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xab,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_ngt_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_ngt_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xab,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_ngt_f64_e64 null, 0.5
+// GFX1250: v_cmpx_ngt_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xab,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_ngt_f64_e64 -1, -1
+// GFX1250: v_cmpx_ngt_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xab,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_ngt_f64_e64 0.5, null
+// GFX1250: v_cmpx_ngt_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xab,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_ngt_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_ngt_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xab,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xab,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f16_e64 v1, v2
+// GFX1250: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nle_f16_e64 v255, v255
+// GFX1250: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nle_f16_e64 s1, s2
+// GFX1250: v_cmpx_nle_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nle_f16_e64 s105, s105
+// GFX1250: v_cmpx_nle_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8c,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nle_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_nle_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8c,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nle_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_nle_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nle_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_nle_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8c,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nle_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_nle_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8c,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nle_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_nle_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8c,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nle_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_nle_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8c,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nle_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_nle_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8c,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nle_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_nle_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8c,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nle_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_nle_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8c,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nle_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_nle_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8c,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nle_f32_e64 v1, v2
+// GFX1250: v_cmpx_nle_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nle_f32_e64 v255, v255
+// GFX1250: v_cmpx_nle_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9c,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nle_f32_e64 s1, s2
+// GFX1250: v_cmpx_nle_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nle_f32_e64 s105, s105
+// GFX1250: v_cmpx_nle_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9c,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nle_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_nle_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9c,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nle_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_nle_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_nle_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9c,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nle_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_nle_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9c,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nle_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_nle_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9c,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nle_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_nle_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9c,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nle_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_nle_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9c,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nle_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_nle_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9c,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nle_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_nle_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9c,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nle_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_nle_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9c,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nle_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_nle_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9c,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_nle_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xac,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_nle_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_nle_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xac,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_nle_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_nle_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xac,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_nle_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_nle_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xac,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_nle_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_nle_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xac,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_nle_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_nle_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xac,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nle_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_nle_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xac,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_nle_f64_e64 null, 0.5
+// GFX1250: v_cmpx_nle_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xac,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_nle_f64_e64 -1, -1
+// GFX1250: v_cmpx_nle_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xac,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_nle_f64_e64 0.5, null
+// GFX1250: v_cmpx_nle_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xac,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_nle_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_nle_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xac,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xac,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f16_e64 v1, v2
+// GFX1250: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlg_f16_e64 v255, v255
+// GFX1250: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nlg_f16_e64 s1, s2
+// GFX1250: v_cmpx_nlg_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 s105, s105
+// GFX1250: v_cmpx_nlg_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8a,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_nlg_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8a,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_nlg_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_nlg_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8a,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nlg_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_nlg_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8a,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nlg_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_nlg_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8a,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nlg_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_nlg_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8a,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_nlg_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8a,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_nlg_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8a,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_nlg_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8a,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nlg_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_nlg_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8a,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 v1, v2
+// GFX1250: v_cmpx_nlg_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlg_f32_e64 v255, v255
+// GFX1250: v_cmpx_nlg_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9a,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nlg_f32_e64 s1, s2
+// GFX1250: v_cmpx_nlg_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 s105, s105
+// GFX1250: v_cmpx_nlg_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9a,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_nlg_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9a,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_nlg_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_nlg_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9a,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nlg_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_nlg_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9a,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nlg_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_nlg_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9a,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nlg_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_nlg_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9a,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_nlg_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9a,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_nlg_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9a,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nlg_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_nlg_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9a,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nlg_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_nlg_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9a,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nlg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_nlg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9a,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_nlg_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xaa,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_nlg_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_nlg_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xaa,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_nlg_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_nlg_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xaa,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_nlg_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_nlg_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xaa,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_nlg_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_nlg_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xaa,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_nlg_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_nlg_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xaa,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlg_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_nlg_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xaa,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_nlg_f64_e64 null, 0.5
+// GFX1250: v_cmpx_nlg_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xaa,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_nlg_f64_e64 -1, -1
+// GFX1250: v_cmpx_nlg_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xaa,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_nlg_f64_e64 0.5, null
+// GFX1250: v_cmpx_nlg_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xaa,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_nlg_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_nlg_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xaa,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaa,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f16_e64 v1, v2
+// GFX1250: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlt_f16_e64 v255, v255
+// GFX1250: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nlt_f16_e64 s1, s2
+// GFX1250: v_cmpx_nlt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 s105, s105
+// GFX1250: v_cmpx_nlt_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8e,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_nlt_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8e,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_nlt_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_nlt_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8e,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nlt_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_nlt_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8e,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nlt_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_nlt_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8e,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nlt_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_nlt_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8e,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_nlt_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8e,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_nlt_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8e,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_nlt_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8e,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nlt_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_nlt_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8e,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 v1, v2
+// GFX1250: v_cmpx_nlt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_nlt_f32_e64 v255, v255
+// GFX1250: v_cmpx_nlt_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9e,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_nlt_f32_e64 s1, s2
+// GFX1250: v_cmpx_nlt_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 s105, s105
+// GFX1250: v_cmpx_nlt_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9e,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_nlt_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9e,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_nlt_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_nlt_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9e,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_nlt_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_nlt_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9e,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_nlt_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_nlt_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9e,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_nlt_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_nlt_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9e,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_nlt_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9e,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_nlt_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9e,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_nlt_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_nlt_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9e,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_nlt_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_nlt_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9e,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_nlt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_nlt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9e,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_nlt_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xae,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_nlt_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_nlt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xae,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_nlt_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_nlt_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xae,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_nlt_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_nlt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xae,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_nlt_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_nlt_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xae,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_nlt_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_nlt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xae,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_nlt_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_nlt_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xae,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_nlt_f64_e64 null, 0.5
+// GFX1250: v_cmpx_nlt_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xae,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_nlt_f64_e64 -1, -1
+// GFX1250: v_cmpx_nlt_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xae,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_nlt_f64_e64 0.5, null
+// GFX1250: v_cmpx_nlt_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xae,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_nlt_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_nlt_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xae,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xae,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f16_e64 v1, v2
+// GFX1250: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_o_f16_e64 v255, v255
+// GFX1250: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_o_f16_e64 s1, s2
+// GFX1250: v_cmpx_o_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_o_f16_e64 s105, s105
+// GFX1250: v_cmpx_o_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x87,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_o_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_o_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x87,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_o_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_o_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x87,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_o_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_o_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x87,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_o_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_o_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x87,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_o_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_o_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x87,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_o_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_o_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x87,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_o_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_o_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x87,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_o_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_o_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x87,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_o_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_o_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x87,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_o_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_o_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x87,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_o_f32_e64 v1, v2
+// GFX1250: v_cmpx_o_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_o_f32_e64 v255, v255
+// GFX1250: v_cmpx_o_f32_e64 v255, v255             ; encoding: [0x7e,0x00,0x97,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_o_f32_e64 s1, s2
+// GFX1250: v_cmpx_o_f32_e64 s1, s2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_o_f32_e64 s105, s105
+// GFX1250: v_cmpx_o_f32_e64 s105, s105             ; encoding: [0x7e,0x00,0x97,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_o_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_o_f32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x97,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_o_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_o_f32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0x97,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_o_f32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x97,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_o_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_o_f32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x97,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_o_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_o_f32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x97,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_o_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_o_f32_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x97,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_o_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_o_f32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x97,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_o_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_o_f32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x97,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_o_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_o_f32_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x97,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_o_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_o_f32_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x97,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_o_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_o_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x97,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_o_f64_e64 v[2:3], v[2:3]         ; encoding: [0x7e,0x00,0xa7,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_o_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_o_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa7,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_o_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_o_f64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xa7,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_o_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_o_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa7,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_o_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_o_f64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xa7,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_o_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_o_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa7,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_o_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_o_f64_e64 -|exec|, src_scc       ; encoding: [0x7e,0x01,0xa7,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_o_f64_e64 null, 0.5
+// GFX1250: v_cmpx_o_f64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xa7,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_o_f64_e64 -1, -1
+// GFX1250: v_cmpx_o_f64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xa7,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_o_f64_e64 0.5, null
+// GFX1250: v_cmpx_o_f64_e64 0.5, null              ; encoding: [0x7e,0x00,0xa7,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_o_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_o_f64_e64 -|src_scc|, -|exec|    ; encoding: [0x7e,0x03,0xa7,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa7,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f16_e64 v1, v2
+// GFX1250: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_u_f16_e64 v255, v255
+// GFX1250: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_u_f16_e64 s1, s2
+// GFX1250: v_cmpx_u_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_u_f16_e64 s105, s105
+// GFX1250: v_cmpx_u_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x88,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_u_f16_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_u_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x88,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_u_f16_e64 vcc_hi, 0xfe0b
+// GFX1250: v_cmpx_u_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x88,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_u_f16_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_u_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x88,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_u_f16_e64 m0, 0.5
+// GFX1250: v_cmpx_u_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x88,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_u_f16_e64 exec_lo, -1
+// GFX1250: v_cmpx_u_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x88,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_u_f16_e64 |exec_hi|, null
+// GFX1250: v_cmpx_u_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x88,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_u_f16_e64 null, exec_lo
+// GFX1250: v_cmpx_u_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x88,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_u_f16_e64 -1, exec_hi
+// GFX1250: v_cmpx_u_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x88,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_u_f16_e64 0.5, -m0
+// GFX1250: v_cmpx_u_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x88,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_u_f16_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_u_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x88,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+v_cmpx_u_f32_e64 v1, v2
+// GFX1250: v_cmpx_u_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00]
+
+v_cmpx_u_f32_e64 v255, v255
+// GFX1250: v_cmpx_u_f32_e64 v255, v255             ; encoding: [0x7e,0x00,0x98,0xd4,0xff,0xff,0x03,0x00]
+
+v_cmpx_u_f32_e64 s1, s2
+// GFX1250: v_cmpx_u_f32_e64 s1, s2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x04,0x00,0x00]
+
+v_cmpx_u_f32_e64 s105, s105
+// GFX1250: v_cmpx_u_f32_e64 s105, s105             ; encoding: [0x7e,0x00,0x98,0xd4,0x69,0xd2,0x00,0x00]
+
+v_cmpx_u_f32_e64 vcc_lo, ttmp15
+// GFX1250: v_cmpx_u_f32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x98,0xd4,0x6a,0xf6,0x00,0x00]
+
+v_cmpx_u_f32_e64 vcc_hi, 0xaf123456
+// GFX1250: v_cmpx_u_f32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0x98,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f32_e64 ttmp15, src_scc
+// GFX1250: v_cmpx_u_f32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x98,0xd4,0x7b,0xfa,0x01,0x00]
+
+v_cmpx_u_f32_e64 m0, 0.5
+// GFX1250: v_cmpx_u_f32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x98,0xd4,0x7d,0xe0,0x01,0x00]
+
+v_cmpx_u_f32_e64 exec_lo, -1
+// GFX1250: v_cmpx_u_f32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x98,0xd4,0x7e,0x82,0x01,0x00]
+
+v_cmpx_u_f32_e64 |exec_hi|, null
+// GFX1250: v_cmpx_u_f32_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x98,0xd4,0x7f,0xf8,0x00,0x00]
+
+v_cmpx_u_f32_e64 null, exec_lo
+// GFX1250: v_cmpx_u_f32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x98,0xd4,0x7c,0xfc,0x00,0x00]
+
+v_cmpx_u_f32_e64 -1, exec_hi
+// GFX1250: v_cmpx_u_f32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x98,0xd4,0xc1,0xfe,0x00,0x00]
+
+v_cmpx_u_f32_e64 0.5, -m0
+// GFX1250: v_cmpx_u_f32_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x98,0xd4,0xf0,0xfa,0x00,0x40]
+
+v_cmpx_u_f32_e64 -src_scc, |vcc_lo|
+// GFX1250: v_cmpx_u_f32_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x98,0xd4,0xfd,0xd4,0x00,0x20]
+
+v_cmpx_u_f32_e64 -|0xaf123456|, -|vcc_hi| clamp
+// GFX1250: v_cmpx_u_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x98,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f64_e64 v[2:3], v[2:3]
+// GFX1250: v_cmpx_u_f64_e64 v[2:3], v[2:3]         ; encoding: [0x7e,0x00,0xa8,0xd4,0x02,0x05,0x02,0x00]
+
+v_cmpx_u_f64_e64 v[254:255], v[254:255]
+// GFX1250: v_cmpx_u_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa8,0xd4,0xfe,0xfd,0x03,0x00]
+
+v_cmpx_u_f64_e64 s[2:3], s[4:5]
+// GFX1250: v_cmpx_u_f64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xa8,0xd4,0x02,0x08,0x00,0x00]
+
+v_cmpx_u_f64_e64 s[104:105], s[104:105]
+// GFX1250: v_cmpx_u_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa8,0xd4,0x68,0xd0,0x00,0x00]
+
+v_cmpx_u_f64_e64 vcc, ttmp[14:15]
+// GFX1250: v_cmpx_u_f64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xa8,0xd4,0x6a,0xf4,0x00,0x00]
+
+v_cmpx_u_f64_e64 ttmp[14:15], 0xaf123456
+// GFX1250: v_cmpx_u_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa8,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_cmpx_u_f64_e64 -|exec|, src_scc
+// GFX1250: v_cmpx_u_f64_e64 -|exec|, src_scc       ; encoding: [0x7e,0x01,0xa8,0xd4,0x7e,0xfa,0x01,0x20]
+
+v_cmpx_u_f64_e64 null, 0.5
+// GFX1250: v_cmpx_u_f64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xa8,0xd4,0x7c,0xe0,0x01,0x00]
+
+v_cmpx_u_f64_e64 -1, -1
+// GFX1250: v_cmpx_u_f64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xa8,0xd4,0xc1,0x82,0x01,0x00]
+
+v_cmpx_u_f64_e64 0.5, null
+// GFX1250: v_cmpx_u_f64_e64 0.5, null              ; encoding: [0x7e,0x00,0xa8,0xd4,0xf0,0xf8,0x00,0x00]
+
+v_cmpx_u_f64_e64 -|src_scc|, -|exec|
+// GFX1250: v_cmpx_u_f64_e64 -|src_scc|, -|exec|    ; encoding: [0x7e,0x03,0xa8,0xd4,0xfd,0xfc,0x00,0x60]
+
+v_cmpx_u_f64_e64 0xaf123456, -|vcc| clamp
+// GFX1250: v_cmpx_u_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa8,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_dpp16.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_dpp16.s
new file mode 100644
index 0000000000000..2875d3ec8e069
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_dpp16.s
@@ -0,0 +1,14 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_fma_mix_f32_bf16 v0, v1, v2, v3 op_sel:[0,0,0] row_ror:7 bank_mask:0x1 bound_ctrl:0
+// GFX1250: v_fma_mix_f32_bf16_e64_dpp v0, v1, v2, v3 row_ror:7 row_mask:0xf bank_mask:0x1 ; encoding: [0x00,0x00,0x3d,0xcc,0xfa,0x04,0x0e,0x04,0x01,0x27,0x01,0xf1]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_fma_mixlo_bf16 v0, v1, v2, v3 op_sel_hi:[1,1,1] clamp quad_perm:[0,2,3,1] row_mask:0x0
+// GFX1250: v_fma_mixlo_bf16_e64_dpp v0, v1, v2, v3 op_sel_hi:[1,1,1] clamp quad_perm:[0,2,3,1] row_mask:0x0 bank_mask:0xf ; encoding: [0x00,0xc0,0x3e,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x78,0x00,0x0f]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_fma_mixhi_bf16 v0, v1, v2, v3 op_sel_hi:[1,1,1] clamp quad_perm:[0,2,3,1] row_mask:0x0
+// GFX1250: v_fma_mixhi_bf16_e64_dpp v0, v1, v2, v3 op_sel_hi:[1,1,1] clamp quad_perm:[0,2,3,1] row_mask:0x0 bank_mask:0xf ; encoding: [0x00,0xc0,0x3f,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x78,0x00,0x0f]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_dpp8.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_dpp8.s
new file mode 100644
index 0000000000000..13b8e211c821e
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_dpp8.s
@@ -0,0 +1,26 @@
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding < %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_fma_mix_f32_bf16 v0, v1, v2, v3 dpp8:[2,2,2,2,4,4,4,4]
+// GFX1250: v_fma_mix_f32_bf16_e64_dpp v0, v1, v2, v3 dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x00,0x3d,0xcc,0xe9,0x04,0x0e,0x04,0x01,0x92,0x44,0x92]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_fma_mix_f32_bf16 v0, v1, v2, v3 clamp dpp8:[2,2,2,2,4,4,4,4] fi:1
+// GFX1250: v_fma_mix_f32_bf16_e64_dpp v0, v1, v2, v3 clamp dpp8:[2,2,2,2,4,4,4,4] fi:1 ; encoding: [0x00,0x80,0x3d,0xcc,0xea,0x04,0x0e,0x04,0x01,0x92,0x44,0x92]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_fma_mixlo_bf16 v0, abs(v1), -v2, abs(v3) dpp8:[2,2,2,2,4,4,4,4]
+// GFX1250: v_fma_mixlo_bf16_e64_dpp v0, |v1|, -v2, |v3| dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x05,0x3e,0xcc,0xe9,0x04,0x0e,0x44,0x01,0x92,0x44,0x92]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_fma_mixlo_bf16 v0, abs(v1), -v2, abs(v3) op_sel:[1,0,0] op_sel_hi:[1,0,0] dpp8:[2,2,2,2,4,4,4,4]
+// GFX1250: v_fma_mixlo_bf16_e64_dpp v0, |v1|, -v2, |v3| op_sel:[1,0,0] op_sel_hi:[1,0,0] dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x0d,0x3e,0xcc,0xe9,0x04,0x0e,0x4c,0x01,0x92,0x44,0x92]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_fma_mixhi_bf16 v0, abs(v1), -v2, abs(v3) dpp8:[2,2,2,2,4,4,4,4]
+// GFX1250: v_fma_mixhi_bf16_e64_dpp v0, |v1|, -v2, |v3| dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x05,0x3f,0xcc,0xe9,0x04,0x0e,0x44,0x01,0x92,0x44,0x92]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_fma_mixhi_bf16 v0, abs(v1), -v2, abs(v3) op_sel:[1,0,0] op_sel_hi:[1,0,0] dpp8:[2,2,2,2,4,4,4,4]
+// GFX1250: v_fma_mixhi_bf16_e64_dpp v0, |v1|, -v2, |v3| op_sel:[1,0,0] op_sel_hi:[1,0,0] dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x0d,0x3f,0xcc,0xe9,0x04,0x0e,0x4c,0x01,0x92,0x44,0x92]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_err.s
new file mode 100644
index 0000000000000..1ea64de5cbc9e
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vop3p_err.s
@@ -0,0 +1,74 @@
+// NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX12-ERR --implicit-check-not=error: --strict-whitespace %s
+
+v_pk_fma_f32 v[8:9], s[0:1], v[0:1], v[4:5]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_fma_f32 v[8:9], v[0:1], s[0:1], v[4:5]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_fma_f32 v[8:9], v[0:1], v[4:5], s[0:1]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_fma_f32 v[8:9], s[0:1], v[0:1], v[4:5] op_sel:[1,0,0] op_sel_hi:[0,0,0]
+// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
+
+v_pk_fma_f32 v[8:9], s[0:1], v[0:1], v[4:5] op_sel:[1,0,0] op_sel_hi:[1,0,0]
+// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
+
+v_pk_fma_f32 v[8:9], v[0:1], s[0:1], v[4:5] op_sel:[0,1,0] op_sel_hi:[0,0,0]
+// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
+
+v_pk_fma_f32 v[8:9], v[0:1], v[4:5], s[0:1] op_sel:[0,0,1] op_sel_hi:[0,0,0]
+// GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
+
+v_pk_mul_f32 v[8:9], s[0:1], v[0:1]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_mul_f32 v[8:9], v[0:1], s[0:1]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_mul_f32 v[8:9], s[0:1], v[0:1] op_sel:[1,0] op_sel_hi:[0,0]
+// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
+
+v_pk_mul_f32 v[8:9], v[0:1], s[0:1] op_sel:[0,1] op_sel_hi:[0,0]
+// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
+
+v_pk_mul_f32 v[8:9], v[0:1], s[0:1] op_sel:[0,1] op_sel_hi:[0,1]
+// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
+
+v_pk_add_f32 v[8:9], s[0:1], v[0:1]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_add_f32 v[8:9], v[0:1], s[0:1]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_add_f32 v[8:9], s[0:1], v[0:1] op_sel:[1,0] op_sel_hi:[0,0]
+// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
+
+v_pk_add_f32 v[8:9], v[0:1], s[0:1] op_sel:[0,1] op_sel_hi:[0,0]
+// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
+
+v_pk_add_f32 v[8:9], v[0:1], s[0:1] op_sel:[0,1] op_sel_hi:[0,1]
+// GFX12-ERR: :[[@LINE-1]]:37: error: invalid op_sel operand
+
+v_pk_fma_f32 v[8:9], exec, v[0:1], v[4:5]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_fma_f32 v[8:9], v[0:1], exec, v[4:5]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_fma_f32 v[8:9], v[0:1], v[4:5], exec
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_mul_f32 v[8:9], exec, v[0:1]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_mul_f32 v[8:9], v[0:1], exec
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_add_f32 v[8:9], exec, v[0:1]
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
+
+v_pk_add_f32 v[8:9], v[0:1], exec
+// GFX12-ERR: :[[@LINE-1]]:1: error: invalid op_sel operand
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_vsample_err.s b/llvm/test/MC/AMDGPU/gfx1250_asm_vsample_err.s
new file mode 100644
index 0000000000000..50766f13fd226
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_vsample_err.s
@@ -0,0 +1,175 @@
+; RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefix=GFX1250-ERR --implicit-check-not=error: --strict-whitespace %s
+
+image_sample v64, v32, s[4:11], s[100:103] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_l v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_b v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_lz v64, v32, s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_l v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_b v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_lz v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_o v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_l_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_b_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_lz_o v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_o v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_l_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_b_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_lz_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4 v[64:67], [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_l v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_b v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_lz v[64:67], [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_lz v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_o v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_lz_o v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_lz_o v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_get_lod v64, v32, s[4:11], s[100:103] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_g16 v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_g16 v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_o_g16 v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_o_g16 v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_cl v64, [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_cl v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_b_cl v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_cl v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_cl v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_b_cl v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_cl_o v64, [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_cl_o v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_b_cl_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_cl_o v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_cl_o v64, [v32, v33, v34, v[35:37]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_b_cl_o v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_cl_g16 v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_cl_o_g16 v64, [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_c_d_cl_o_g16 v64, [v32, v33, v34, v[35:37]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_sample_d_cl_g16 v64, [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_1D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_cl v[64:67], [v32, v33, v34], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_b_cl v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_cl v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_l v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_b v[64:67], [v32, v33, v34, v35], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4_c_b_cl v[64:67], [v32, v33, v34, v[35:36]], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_gather4h v[64:67], [v32, v33], s[4:11], s[4:7] dmask:0x1 dim:SQ_RSRC_IMG_2D
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+image_msaa_load v[1:4], [v5, v6, v7], s[8:15] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx1250_err.s b/llvm/test/MC/AMDGPU/gfx1250_err.s
index e4598fe91a00b..676eb48cc5a7f 100644
--- a/llvm/test/MC/AMDGPU/gfx1250_err.s
+++ b/llvm/test/MC/AMDGPU/gfx1250_err.s
@@ -1,5 +1,30 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX1250-ERR --implicit-check-not=error: -strict-whitespace %s
 
+s_load_b32 s4, s[2:3], 10 th:TH_LOAD_NT th:TH_LOAD_NT
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: s_load_b32 s4, s[2:3], 10 th:TH_LOAD_NT th:TH_LOAD_NT
+// GFX1250-ERR:                                         ^
+
+s_load_b32 s4, s[2:3], 10 scope:SCOPE_SE scope:SCOPE_SE
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: s_load_b32 s4, s[2:3], 10 scope:SCOPE_SE scope:SCOPE_SE
+// GFX1250-ERR:                                          ^
+
+s_load_b32 s4, s[2:3], 10 nv nv
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+// GFX1250-ERR: s_load_b32 s4, s[2:3], 10 nv nv
+// GFX1250-ERR:                              ^
+
+v_mov_b64 v[4:5], v[2:3] quad_perm:[1,1,1,1]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1250-ERR: v_mov_b64 v[4:5], v[2:3] quad_perm:[1,1,1,1]
+// GFX1250-ERR:                          ^
+
+v_mov_b64 v[4:5], v[2:3] dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1250-ERR: v_mov_b64 v[4:5], v[2:3] dpp8:[7,6,5,4,3,2,1,0]
+// GFX1250-ERR:                          ^
+
 // For v_dual_cndmask_b32 use of the explicit src2 forces VOPD3 form even if it is vcc_lo.
 // If src2 is omitted then it forces VOPD form. As a result a proper form of the instruction
 // has to be used if the other component of the dual instruction cannot be used if that
@@ -137,6 +162,11 @@ v_fmaak_f64 v[4:5], 0x7e8, v[8:9], lit64(0x7e8)
 // GFX1250-ERR: v_fmaak_f64 v[4:5], 0x7e8, v[8:9], lit64(0x7e8)
 // GFX1250-ERR:                     ^
 
+v_pk_add_min_i16 v10, |v1|, v2, v3
+// GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
+// GFX1250-ERR: v_pk_add_min_i16 v10, |v1|, v2, v3
+// GFX1250-ERR:                       ^
+
 v_pk_add_min_i16 v10, -v1, v2, v3
 // GFX1250-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: not a valid operand.
 // GFX1250-ERR: v_pk_add_min_i16 v10, -v1, v2, v3
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s
index b7d4764335a53..949847e2dec3b 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s
@@ -725,6 +725,24 @@ s_barrier_signal_isfirst -1
 s_barrier_signal_isfirst m0
 // GFX12: s_barrier_signal_isfirst m0             ; encoding: [0x7d,0x4f,0x80,0xbe]
 
+s_barrier_init -1
+// GFX12: s_barrier_init -1                       ; encoding: [0xc1,0x51,0x80,0xbe]
+
+s_barrier_init -2
+// GFX12: s_barrier_init -2                       ; encoding: [0xc2,0x51,0x80,0xbe]
+
+s_barrier_init m0
+// GFX12: s_barrier_init m0                       ; encoding: [0x7d,0x51,0x80,0xbe]
+
+s_barrier_join -1
+// GFX12: s_barrier_join -1                       ; encoding: [0xc1,0x52,0x80,0xbe]
+
+s_barrier_join -2
+// GFX12: s_barrier_join -2                       ; encoding: [0xc2,0x52,0x80,0xbe]
+
+s_barrier_join m0
+// GFX12: s_barrier_join m0                       ; encoding: [0x7d,0x52,0x80,0xbe]
+
 s_get_barrier_state s3, -1
 // GFX12: s_get_barrier_state s3, -1              ; encoding: [0xc1,0x50,0x83,0xbe]
 
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s
index bc762ed28f17f..2ecec4c1f7131 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s
@@ -1,5 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck -check-prefix=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefixes=GFX12,GFX1200 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1250 %s | FileCheck --check-prefixes=GFX12,GFX1250 %s
 
 s_add_nc_u64 s[0:1], s[2:3], s[4:5]
 // GFX12: s_add_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x80,0xa9]
@@ -53,7 +54,8 @@ s_add_nc_u64 s[0:1], 0x3f717273, s[2:3]
 // GFX12: s_add_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x73,0x72,0x71,0x3f]
 
 s_add_nc_u64 s[0:1], 0xaf123456, s[2:3]
-// GFX12: s_add_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf]
+// GFX1200: s_add_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf]
+// GFX1250: s_add_nc_u64 s[0:1], lit64(0xaf123456), s[2:3] ; encoding: [0xfe,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_add_nc_u64 s[0:1], s[2:3], exec
 // GFX12: s_add_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x80,0xa9]
@@ -77,7 +79,8 @@ s_add_nc_u64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_add_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0xa9,0x73,0x72,0x71,0x3f]
 
 s_add_nc_u64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_add_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf]
+// GFX1200: s_add_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf]
+// GFX1250: s_add_nc_u64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0xa9,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_sub_nc_u64 s[0:1], s[2:3], s[4:5]
 // GFX12: s_sub_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x00,0xaa]
@@ -131,7 +134,8 @@ s_sub_nc_u64 s[0:1], 0x3f717273, s[2:3]
 // GFX12: s_sub_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x73,0x72,0x71,0x3f]
 
 s_sub_nc_u64 s[0:1], 0xaf123456, s[2:3]
-// GFX12: s_sub_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf]
+// GFX1200: s_sub_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf]
+// GFX1250: s_sub_nc_u64 s[0:1], lit64(0xaf123456), s[2:3] ; encoding: [0xfe,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_sub_nc_u64 s[0:1], s[2:3], exec
 // GFX12: s_sub_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x00,0xaa]
@@ -155,7 +159,8 @@ s_sub_nc_u64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x00,0xaa,0x73,0x72,0x71,0x3f]
 
 s_sub_nc_u64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf]
+// GFX1200: s_sub_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf]
+// GFX1250: s_sub_nc_u64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x00,0xaa,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_mul_u64 s[0:1], s[2:3], s[4:5]
 // GFX12: s_mul_u64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0xaa]
@@ -209,7 +214,8 @@ s_mul_u64 s[0:1], 0x3f717273, s[2:3]
 // GFX12: s_mul_u64 s[0:1], 0x3f717273, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x73,0x72,0x71,0x3f]
 
 s_mul_u64 s[0:1], 0xaf123456, s[2:3]
-// GFX12: s_mul_u64 s[0:1], 0xaf123456, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf]
+// GFX1200: s_mul_u64 s[0:1], 0xaf123456, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf]
+// GFX1250: s_mul_u64 s[0:1], lit64(0xaf123456), s[2:3] ; encoding: [0xfe,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_mul_u64 s[0:1], s[2:3], exec
 // GFX12: s_mul_u64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0xaa]
@@ -233,7 +239,8 @@ s_mul_u64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_mul_u64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0xaa,0x73,0x72,0x71,0x3f]
 
 s_mul_u64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_mul_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf]
+// GFX1200: s_mul_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf]
+// GFX1250: s_mul_u64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0xaa,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_add_f32 s5, s1, s2
 // GFX12: s_add_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa0]
@@ -2350,7 +2357,8 @@ s_cselect_b64 s[0:1], 0x3f717273, s[4:5]
 // GFX12: s_cselect_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x73,0x72,0x71,0x3f]
 
 s_cselect_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: s_cselect_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf]
+// GFX1200: s_cselect_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf]
+// GFX1250: s_cselect_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x98,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_cselect_b64 s[0:1], s[2:3], exec
 // GFX12: s_cselect_b64 s[0:1], s[2:3], exec      ; encoding: [0x02,0x7e,0x80,0x98]
@@ -2374,7 +2382,8 @@ s_cselect_b64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_cselect_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x98,0x73,0x72,0x71,0x3f]
 
 s_cselect_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_cselect_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf]
+// GFX1200: s_cselect_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf]
+// GFX1250: s_cselect_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x98,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_and_b32 s0, s1, s2
 // GFX12: s_and_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8b]
@@ -2542,7 +2551,8 @@ s_and_b64 s[0:1], 0x3f717273, s[4:5]
 // GFX12: s_and_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x73,0x72,0x71,0x3f]
 
 s_and_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: s_and_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf]
+// GFX1200: s_and_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf]
+// GFX1250: s_and_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_and_b64 s[0:1], s[2:3], exec
 // GFX12: s_and_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8b]
@@ -2566,7 +2576,8 @@ s_and_b64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_and_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8b,0x73,0x72,0x71,0x3f]
 
 s_and_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_and_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf]
+// GFX1200: s_and_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf]
+// GFX1250: s_and_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8b,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_or_b32 s0, s1, s2
 // GFX12: s_or_b32 s0, s1, s2                     ; encoding: [0x01,0x02,0x00,0x8c]
@@ -2725,7 +2736,8 @@ s_or_b64 s[0:1], 0x3f717273, s[4:5]
 // GFX12: s_or_b64 s[0:1], 0x3f717273, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x73,0x72,0x71,0x3f]
 
 s_or_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: s_or_b64 s[0:1], 0xaf123456, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf]
+// GFX1200: s_or_b64 s[0:1], 0xaf123456, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf]
+// GFX1250: s_or_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_or_b64 s[0:1], s[2:3], exec
 // GFX12: s_or_b64 s[0:1], s[2:3], exec           ; encoding: [0x02,0x7e,0x80,0x8c]
@@ -2749,7 +2761,8 @@ s_or_b64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_or_b64 s[0:1], s[2:3], 0x3f717273     ; encoding: [0x02,0xff,0x80,0x8c,0x73,0x72,0x71,0x3f]
 
 s_or_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_or_b64 s[0:1], s[2:3], 0xaf123456     ; encoding: [0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf]
+// GFX1200: s_or_b64 s[0:1], s[2:3], 0xaf123456     ; encoding: [0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf]
+// GFX1250: s_or_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8c,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_xor_b32 s0, s1, s2
 // GFX12: s_xor_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8d]
@@ -2908,7 +2921,8 @@ s_xor_b64 s[0:1], 0x3f717273, s[4:5]
 // GFX12: s_xor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x73,0x72,0x71,0x3f]
 
 s_xor_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: s_xor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf]
+// GFX1200: s_xor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf]
+// GFX1250: s_xor_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_xor_b64 s[0:1], s[2:3], exec
 // GFX12: s_xor_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8d]
@@ -2932,7 +2946,8 @@ s_xor_b64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_xor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8d,0x73,0x72,0x71,0x3f]
 
 s_xor_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_xor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf]
+// GFX1200: s_xor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf]
+// GFX1250: s_xor_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8d,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_andn2_b32 s0, s1, s2
 // GFX12: s_and_not1_b32 s0, s1, s2               ; encoding: [0x01,0x02,0x00,0x91]
@@ -3091,7 +3106,8 @@ s_andn2_b64 s[0:1], 0x3f717273, s[4:5]
 // GFX12: s_and_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x73,0x72,0x71,0x3f]
 
 s_andn2_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: s_and_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf]
+// GFX1200: s_and_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf]
+// GFX1250: s_and_not1_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x91,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_andn2_b64 s[0:1], s[2:3], exec
 // GFX12: s_and_not1_b64 s[0:1], s[2:3], exec     ; encoding: [0x02,0x7e,0x80,0x91]
@@ -3115,7 +3131,8 @@ s_andn2_b64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_and_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x91,0x73,0x72,0x71,0x3f]
 
 s_andn2_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_and_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf]
+// GFX1200: s_and_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf]
+// GFX1250: s_and_not1_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x91,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_orn2_b32 s0, s1, s2
 // GFX12: s_or_not1_b32 s0, s1, s2                ; encoding: [0x01,0x02,0x00,0x92]
@@ -3274,7 +3291,8 @@ s_orn2_b64 s[0:1], 0x3f717273, s[4:5]
 // GFX12: s_or_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x73,0x72,0x71,0x3f]
 
 s_orn2_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: s_or_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf]
+// GFX1200: s_or_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf]
+// GFX1250: s_or_not1_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x92,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_orn2_b64 s[0:1], s[2:3], exec
 // GFX12: s_or_not1_b64 s[0:1], s[2:3], exec      ; encoding: [0x02,0x7e,0x80,0x92]
@@ -3298,7 +3316,8 @@ s_orn2_b64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_or_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x92,0x73,0x72,0x71,0x3f]
 
 s_orn2_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_or_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf]
+// GFX1200: s_or_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf]
+// GFX1250: s_or_not1_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x92,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_nand_b32 s0, s1, s2
 // GFX12: s_nand_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x8e]
@@ -3457,7 +3476,8 @@ s_nand_b64 s[0:1], 0x3f717273, s[4:5]
 // GFX12: s_nand_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x73,0x72,0x71,0x3f]
 
 s_nand_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: s_nand_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf]
+// GFX1200: s_nand_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf]
+// GFX1250: s_nand_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_nand_b64 s[0:1], s[2:3], exec
 // GFX12: s_nand_b64 s[0:1], s[2:3], exec         ; encoding: [0x02,0x7e,0x80,0x8e]
@@ -3481,7 +3501,8 @@ s_nand_b64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_nand_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x8e,0x73,0x72,0x71,0x3f]
 
 s_nand_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_nand_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf]
+// GFX1200: s_nand_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf]
+// GFX1250: s_nand_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8e,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_nor_b32 s0, s1, s2
 // GFX12: s_nor_b32 s0, s1, s2                    ; encoding: [0x01,0x02,0x00,0x8f]
@@ -3640,7 +3661,8 @@ s_nor_b64 s[0:1], 0x3f717273, s[4:5]
 // GFX12: s_nor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x73,0x72,0x71,0x3f]
 
 s_nor_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: s_nor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf]
+// GFX1200: s_nor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf]
+// GFX1250: s_nor_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_nor_b64 s[0:1], s[2:3], exec
 // GFX12: s_nor_b64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0x8f]
@@ -3664,7 +3686,8 @@ s_nor_b64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_nor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8f,0x73,0x72,0x71,0x3f]
 
 s_nor_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_nor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf]
+// GFX1200: s_nor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf]
+// GFX1250: s_nor_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8f,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_xnor_b32 s0, s1, s2
 // GFX12: s_xnor_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x90]
@@ -3823,7 +3846,8 @@ s_xnor_b64 s[0:1], 0x3f717273, s[4:5]
 // GFX12: s_xnor_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x73,0x72,0x71,0x3f]
 
 s_xnor_b64 s[0:1], 0xaf123456, s[4:5]
-// GFX12: s_xnor_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf]
+// GFX1200: s_xnor_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf]
+// GFX1250: s_xnor_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x90,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_xnor_b64 s[0:1], s[2:3], exec
 // GFX12: s_xnor_b64 s[0:1], s[2:3], exec         ; encoding: [0x02,0x7e,0x80,0x90]
@@ -3847,7 +3871,8 @@ s_xnor_b64 s[0:1], s[2:3], 0x3f717273
 // GFX12: s_xnor_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x90,0x73,0x72,0x71,0x3f]
 
 s_xnor_b64 s[0:1], s[2:3], 0xaf123456
-// GFX12: s_xnor_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf]
+// GFX1200: s_xnor_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf]
+// GFX1250: s_xnor_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x90,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_lshl_b32 s0, s1, s2
 // GFX12: s_lshl_b32 s0, s1, s2                   ; encoding: [0x01,0x02,0x00,0x84]
@@ -4006,7 +4031,8 @@ s_lshl_b64 s[0:1], 0x3f717273, s4
 // GFX12: s_lshl_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x84,0x73,0x72,0x71,0x3f]
 
 s_lshl_b64 s[0:1], 0xaf123456, s4
-// GFX12: s_lshl_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf]
+// GFX1200: s_lshl_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf]
+// GFX1250: s_lshl_b64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x80,0x84,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_lshl_b64 s[0:1], s[2:3], exec_lo
 // GFX12: s_lshl_b64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x84]
@@ -4189,7 +4215,8 @@ s_lshr_b64 s[0:1], 0x3f717273, s4
 // GFX12: s_lshr_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x85,0x73,0x72,0x71,0x3f]
 
 s_lshr_b64 s[0:1], 0xaf123456, s4
-// GFX12: s_lshr_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf]
+// GFX1200: s_lshr_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf]
+// GFX1250: s_lshr_b64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x80,0x85,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_lshr_b64 s[0:1], s[2:3], exec_lo
 // GFX12: s_lshr_b64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x85]
@@ -4372,7 +4399,8 @@ s_ashr_i64 s[0:1], 0x3f717273, s4
 // GFX12: s_ashr_i64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x86,0x73,0x72,0x71,0x3f]
 
 s_ashr_i64 s[0:1], 0xaf123456, s4
-// GFX12: s_ashr_i64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf]
+// GFX1200: s_ashr_i64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf]
+// GFX1250: s_ashr_i64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x80,0x86,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_ashr_i64 s[0:1], s[2:3], exec_lo
 // GFX12: s_ashr_i64 s[0:1], s[2:3], exec_lo      ; encoding: [0x02,0x7e,0x80,0x86]
@@ -4966,7 +4994,8 @@ s_bfe_u64 s[0:1], 0x3f717273, s4
 // GFX12: s_bfe_u64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x00,0x94,0x73,0x72,0x71,0x3f]
 
 s_bfe_u64 s[0:1], 0xaf123456, s4
-// GFX12: s_bfe_u64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf]
+// GFX1200: s_bfe_u64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf]
+// GFX1250: s_bfe_u64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x00,0x94,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_bfe_u64 s[0:1], s[2:3], exec_lo
 // GFX12: s_bfe_u64 s[0:1], s[2:3], exec_lo       ; encoding: [0x02,0x7e,0x00,0x94]
@@ -5044,7 +5073,8 @@ s_bfe_i64 s[0:1], 0x3f717273, s4
 // GFX12: s_bfe_i64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x80,0x94,0x73,0x72,0x71,0x3f]
 
 s_bfe_i64 s[0:1], 0xaf123456, s4
-// GFX12: s_bfe_i64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf]
+// GFX1200: s_bfe_i64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf]
+// GFX1250: s_bfe_i64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x80,0x94,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_bfe_i64 s[0:1], s[2:3], exec_lo
 // GFX12: s_bfe_i64 s[0:1], s[2:3], exec_lo       ; encoding: [0x02,0x7e,0x80,0x94]
@@ -6247,7 +6277,8 @@ s_and_not1_b64 s[10:11], vcc, ttmp[14:15]
 // GFX12: s_and_not1_b64 s[10:11], vcc, ttmp[14:15] ; encoding: [0x6a,0x7a,0x8a,0x91]
 
 s_and_not1_b64 s[10:11], ttmp[14:15], 0xaf123456
-// GFX12: s_and_not1_b64 s[10:11], ttmp[14:15], 0xaf123456 ; encoding: [0x7a,0xff,0x8a,0x91,0x56,0x34,0x12,0xaf]
+// GFX1200: s_and_not1_b64 s[10:11], ttmp[14:15], 0xaf123456 ; encoding: [0x7a,0xff,0x8a,0x91,0x56,0x34,0x12,0xaf]
+// GFX1250: s_and_not1_b64 s[10:11], ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7a,0xfe,0x8a,0x91,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_and_not1_b64 s[10:11], exec, src_scc
 // GFX12: s_and_not1_b64 s[10:11], exec, src_scc  ; encoding: [0x7e,0xfd,0x8a,0x91]
@@ -6265,7 +6296,8 @@ s_and_not1_b64 exec, src_scc, exec
 // GFX12: s_and_not1_b64 exec, src_scc, exec      ; encoding: [0xfd,0x7e,0xfe,0x91]
 
 s_and_not1_b64 null, 0xaf123456, vcc
-// GFX12: s_and_not1_b64 null, 0xaf123456, vcc    ; encoding: [0xff,0x6a,0xfc,0x91,0x56,0x34,0x12,0xaf]
+// GFX1200: s_and_not1_b64 null, 0xaf123456, vcc    ; encoding: [0xff,0x6a,0xfc,0x91,0x56,0x34,0x12,0xaf]
+// GFX1250: s_and_not1_b64 null, lit64(0xaf123456), vcc ; encoding: [0xfe,0x6a,0xfc,0x91,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_or_not1_b64 s[10:11], s[2:3], s[4:5]
 // GFX12: s_or_not1_b64 s[10:11], s[2:3], s[4:5]  ; encoding: [0x02,0x04,0x8a,0x92]
@@ -6277,7 +6309,8 @@ s_or_not1_b64 s[10:11], vcc, ttmp[14:15]
 // GFX12: s_or_not1_b64 s[10:11], vcc, ttmp[14:15] ; encoding: [0x6a,0x7a,0x8a,0x92]
 
 s_or_not1_b64 s[10:11], ttmp[14:15], 0xaf123456
-// GFX12: s_or_not1_b64 s[10:11], ttmp[14:15], 0xaf123456 ; encoding: [0x7a,0xff,0x8a,0x92,0x56,0x34,0x12,0xaf]
+// GFX1200: s_or_not1_b64 s[10:11], ttmp[14:15], 0xaf123456 ; encoding: [0x7a,0xff,0x8a,0x92,0x56,0x34,0x12,0xaf]
+// GFX1250: s_or_not1_b64 s[10:11], ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7a,0xfe,0x8a,0x92,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_or_not1_b64 s[10:11], exec, src_scc
 // GFX12: s_or_not1_b64 s[10:11], exec, src_scc   ; encoding: [0x7e,0xfd,0x8a,0x92]
@@ -6295,4 +6328,5 @@ s_or_not1_b64 exec, src_scc, exec
 // GFX12: s_or_not1_b64 exec, src_scc, exec       ; encoding: [0xfd,0x7e,0xfe,0x92]
 
 s_or_not1_b64 null, 0xaf123456, vcc
-// GFX12: s_or_not1_b64 null, 0xaf123456, vcc     ; encoding: [0xff,0x6a,0xfc,0x92,0x56,0x34,0x12,0xaf]
+// GFX1200: s_or_not1_b64 null, 0xaf123456, vcc     ; encoding: [0xff,0x6a,0xfc,0x92,0x56,0x34,0x12,0xaf]
+// GFX1250: s_or_not1_b64 null, lit64(0xaf123456), vcc ; encoding: [0xfe,0x6a,0xfc,0x92,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s
index 6d8d6dc16fd09..cedba66d9d51e 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopc.s
@@ -1,5 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefixes=GFX12,GFX1200 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -show-encoding %s | FileCheck --check-prefixes=GFX12,GFX1250 %s
 
 s_cmp_lt_f32 s1, s2
 // GFX12: s_cmp_lt_f32 s1, s2                     ; encoding: [0x01,0x02,0x41,0xbf]
@@ -2117,7 +2118,8 @@ s_cmp_eq_u64 s[0:1], 0x3f717273
 // GFX12: s_cmp_eq_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x10,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_eq_u64 s[0:1], 0xaf123456
-// GFX12: s_cmp_eq_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf]
+// GFX1200: s_cmp_eq_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf]
+// GFX1250: s_cmp_eq_u64 s[0:1], lit64(0xaf123456)  ; encoding: [0x00,0xfe,0x10,0xbf,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 s_cmp_lg_u64 s[0:1], s[2:3]
 // GFX12: s_cmp_lg_u64 s[0:1], s[2:3]             ; encoding: [0x00,0x02,0x11,0xbf]
@@ -2159,4 +2161,5 @@ s_cmp_lg_u64 s[0:1], 0x3f717273
 // GFX12: s_cmp_lg_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x11,0xbf,0x73,0x72,0x71,0x3f]
 
 s_cmp_lg_u64 s[0:1], 0xaf123456
-// GFX12: s_cmp_lg_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf]
+// GFX1200: s_cmp_lg_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf]
+// GFX1250: s_cmp_lg_u64 s[0:1], lit64(0xaf123456)  ; encoding: [0x00,0xfe,0x11,0xbf,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s
index a58d68cb30cb1..75c3d6e6368df 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s
@@ -76,6 +76,9 @@ s_barrier_wait 0xffff
 s_barrier_wait 1
 // GFX12: encoding: [0x01,0x00,0x94,0xbf]
 
+s_barrier_leave
+// GFX12: encoding: [0x00,0x00,0x95,0xbf]
+
 //===----------------------------------------------------------------------===//
 // s_waitcnt
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_err.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_err.s
index eb96a0f31f1c1..7a26bae94c788 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_err.s
@@ -67,3 +67,9 @@ v_permlane16_var_b32 v5, v1, v2 op_sel:[0, 0, 1]
 
 v_cvt_sr_bf8_f32 v1, v2, v3 byte_sel:4
 // GFX12: :[[@LINE-1]]:29: error: invalid byte_sel value.
+
+v_cvt_pk_fp8_f32 v1, v2, v3 clamp
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
+
+v_cvt_sr_fp8_f32 v1, v2, v3 clamp
+// GFX12: :[[@LINE-1]]:29: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx12_err.s b/llvm/test/MC/AMDGPU/gfx12_err.s
index 9ddb91e25afe6..6406b0647835a 100644
--- a/llvm/test/MC/AMDGPU/gfx12_err.s
+++ b/llvm/test/MC/AMDGPU/gfx12_err.s
@@ -125,3 +125,6 @@ s_alloc_vgpr exec
 
 s_alloc_vgpr vcc
 // GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+v_cvt_f32_fp8 v1, v3 clamp
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx12_unsupported.s b/llvm/test/MC/AMDGPU/gfx12_unsupported.s
index c9c15e6507019..be42e64d2f05a 100644
--- a/llvm/test/MC/AMDGPU/gfx12_unsupported.s
+++ b/llvm/test/MC/AMDGPU/gfx12_unsupported.s
@@ -223,6 +223,18 @@ v_cmpx_f_u64 v[0:1], v[2:3]
 v_cmpx_t_u64 v[0:1], v[2:3]
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
+v_mfma_f32_16x16x8_xf32 a[0:3], v[2:3], v[4:5], a[2:5]
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_mfma_f32_16x16x8xf32 a[0:3], v[2:3], v[4:5], a[2:5]
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_mfma_f32_32x32x4_xf32 a[0:15], v[2:3], v[4:5], a[18:33]
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+v_mfma_f32_32x32x4xf32 a[0:15], v[2:3], v[4:5], a[18:33]
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
 buffer_atomic_cmpswap_f32 v[5:6], off, s[96:99], s3
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
 
diff --git a/llvm/test/MC/AMDGPU/hsa-diag-v4.s b/llvm/test/MC/AMDGPU/hsa-diag-v4.s
index 9ab177cf2b125..44fe55ef6e9ba 100644
--- a/llvm/test/MC/AMDGPU/hsa-diag-v4.s
+++ b/llvm/test/MC/AMDGPU/hsa-diag-v4.s
@@ -1,9 +1,10 @@
-// RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=ALL,GCN,GFX8,PREGFX10,AMDHSA
+// RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd-amdhsa -mcpu=gfx810 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s -DMCPU=gfx810 --check-prefixes=ALL,GCN,GFX8,PREGFX10,NOWGP,AMDHSA
 // RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=ALL,GCN,GFX10PLUS,GFX10,AMDHSA
 // RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd-amdhsa -mcpu=gfx1100 -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=ALL,GCN,GFX10PLUS,GFX11,AMDHSA
-// RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd-amdhsa -mcpu=gfx1200 -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=ALL,GCN,GFX10PLUS,GFX12,AMDHSA
+// RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd-amdhsa -mcpu=gfx1200 -show-encoding %s 2>&1 >/dev/null | FileCheck %s -DMCPU=gfx1200 --check-prefixes=ALL,GCN,GFX10PLUS,GFX12,AMDHSA
 // RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd- -mcpu=gfx810 -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=ALL,GCN,NONAMDHSA
-// RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s --check-prefixes=ALL,GFX90A,PREGFX10,AMDHSA
+// RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+xnack -show-encoding %s 2>&1 >/dev/null | FileCheck %s -DMCPU=gfx90a --check-prefixes=ALL,GFX90A,PREGFX10,NOWGP,AMDHSA
+// RUN: not llvm-mc --amdhsa-code-object-version=4 -triple amdgcn-amd-amdhsa -mcpu=gfx1250 -show-encoding %s 2>&1 >/dev/null | FileCheck %s -DMCPU=gfx1250 --check-prefixes=ALL,GCN,GFX10PLUS,GFX12,NOWGP,AMDHSA
 
 .text
 
@@ -11,7 +12,7 @@
 // GFX8-NOT: error:
 // GFX10: error: .amdgcn_target directive's target id amdgcn-amd-amdhsa--gfx810:xnack+ does not match the specified target id amdgcn-amd-amdhsa--gfx1010:xnack+
 // GFX11: error: .amdgcn_target directive's target id amdgcn-amd-amdhsa--gfx810:xnack+ does not match the specified target id amdgcn-amd-amdhsa--gfx1100
-// GFX12: error: .amdgcn_target directive's target id amdgcn-amd-amdhsa--gfx810:xnack+ does not match the specified target id amdgcn-amd-amdhsa--gfx1200
+// GFX12: error: .amdgcn_target directive's target id amdgcn-amd-amdhsa--gfx810:xnack+ does not match the specified target id amdgcn-amd-amdhsa--[[MCPU]]
 // NONAMDHSA: error: .amdgcn_target directive's target id amdgcn-amd-amdhsa--gfx810:xnack+ does not match the specified target id amdgcn-amd-unknown--gfx810
 .warning "test_target"
 .amdgcn_target "amdgcn-amd-amdhsa--gfx810:xnack+"
@@ -176,8 +177,7 @@
 .end_amdhsa_kernel
 
 // GCN-LABEL: warning: test_amdhsa_workgroup_processor_mode
-// PREGFX10: error: directive requires gfx10+
-// GFX10PLUS: error: .amdhsa_next_free_vgpr directive is required
+// NOWGP: error: directive unsupported on [[MCPU]]
 // NONAMDHSA: error: unknown directive
 .warning "test_amdhsa_workgroup_processor_mode"
 .amdhsa_kernel test_amdhsa_workgroup_processor_mode
@@ -185,8 +185,7 @@
 .end_amdhsa_kernel
 
 // GCN-LABEL: warning: test_amdhsa_workgroup_processor_mode_invalid
-// PREGFX10: error: directive requires gfx10+
-// GFX10PLUS: error: value out of range
+// NOWGP: error: directive unsupported on [[MCPU]]
 // NONAMDHSA: error: unknown directive
 .warning "test_amdhsa_workgroup_processor_mode_invalid"
 .amdhsa_kernel test_amdhsa_workgroup_processor_mode_invalid
@@ -290,6 +289,24 @@
   .amdhsa_inst_pref_size 15
 .end_amdhsa_kernel
 
+// GCN-LABEL: warning: test_amdhsa_dx10_clamp_bit
+// GFX12: error: directive unsupported on gfx12+
+.warning "test_amdhsa_dx10_clamp_bit"
+.amdhsa_kernel test_amdhsa_dx10_clamp_bit
+  .amdhsa_next_free_vgpr 32
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_dx10_clamp 1
+.end_amdhsa_kernel
+
+// GCN-LABEL: warning: test_amdhsa_ieee_mode_bit
+// GFX12: error: directive unsupported on gfx12+
+.warning "test_amdhsa_ieee_mode_bit"
+.amdhsa_kernel test_amdhsa_ieee_mode_bit
+  .amdhsa_next_free_vgpr 32
+  .amdhsa_next_free_sgpr 0
+  .amdhsa_ieee_mode 1
+.end_amdhsa_kernel
+
 // GCN-LABEL: warning: test_next_free_vgpr_invalid
 // AMDHSA: error: .amdgcn.next_free_{v,s}gpr symbols must be absolute expressions
 // NONAMDHSA-NOT: error:
diff --git a/llvm/test/MC/AMDGPU/hsa-gfx1250-v4.s b/llvm/test/MC/AMDGPU/hsa-gfx1250-v4.s
new file mode 100644
index 0000000000000..3cd1d877600e2
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-gfx1250-v4.s
@@ -0,0 +1,323 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1250 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefixes=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1250 --amdhsa-code-object-version=4 -filetype=obj < %s > %t
+// RUN: llvm-readelf -S -r -s %t | FileCheck --check-prefix=READOBJ %s
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// READOBJ: Section Headers
+// READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
+// READOBJ: .rodata PROGBITS {{[0-9a-f]+}}        000540 {{[0-9a-f]+}} {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
+
+// READOBJ: Relocation section '.rela.rodata' at offset
+// READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000090 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 210
+// READOBJ: 00000000000000d0 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 310
+// READOBJ: 0000000000000110 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 410
+
+// READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
+// READOBJ:      0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
+// READOBJ-NEXT: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
+// READOBJ-NEXT: 0000000000000200  0 FUNC    LOCAL  PROTECTED 2 special_sgpr
+// READOBJ-NEXT: 0000000000000300  0 FUNC    LOCAL  PROTECTED 2 disabled_user_sgpr
+// READOBJ-NEXT: 0000000000000400  0 FUNC    LOCAL  PROTECTED 2 max_lds_size
+// READOBJ-NEXT: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
+// READOBJ-NEXT: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
+// READOBJ-NEXT: 0000000000000080 64 OBJECT  LOCAL  DEFAULT   3 special_sgpr.kd
+// READOBJ-NEXT: 00000000000000c0 64 OBJECT  LOCAL  DEFAULT   3 disabled_user_sgpr.kd
+// READOBJ-NEXT: 0000000000000100 64 OBJECT  LOCAL  DEFAULT   3 max_lds_size.kd
+
+// OBJDUMP: Contents of section .rodata
+// Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
+// minimal
+// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0030 00000cc0 80000000 00040000 00000000
+// complete
+// OBJDUMP-NEXT: 0040 01000000 01000000 0c000000 00000000
+// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00c00000
+// OBJDUMP-NEXT: 0070 015021c4 410f007f 5e068200 00000000
+// special_sgpr
+// OBJDUMP-NEXT: 0080 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0090 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00a0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00b0 000000c0 80000000 00040000 00000000
+// disabled_user_sgpr
+// OBJDUMP-NEXT: 00c0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00d0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00e0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00f0 00000cc0 80000000 00040000 00000000
+// max_lds_size
+// OBJDUMP-NEXT: 0100 00000600 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0110 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0120 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0130 00000cc0 80000000 00040000 00000000
+
+.text
+
+.amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
+// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
+
+.p2align 8
+.type minimal,@function
+minimal:
+  s_endpgm
+
+.p2align 8
+.type complete,@function
+complete:
+  s_endpgm
+
+.p2align 8
+.type special_sgpr,@function
+special_sgpr:
+  s_endpgm
+
+.p2align 8
+.type disabled_user_sgpr,@function
+disabled_user_sgpr:
+  s_endpgm
+
+.p2align 8
+.type max_lds_size,@function
+max_lds_size:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+// Test that only specifying required directives is allowed, and that defaulted
+// values are omitted.
+.p2align 6
+.amdhsa_kernel minimal
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel minimal
+// ASM: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM: .end_amdhsa_kernel
+
+// Test that we can specify all available directives with non-default values.
+.p2align 6
+.amdhsa_kernel complete
+  .amdhsa_group_segment_fixed_size 1
+  .amdhsa_private_segment_fixed_size 1
+  .amdhsa_kernarg_size 12
+  .amdhsa_user_sgpr_count 32
+  .amdhsa_user_sgpr_dispatch_ptr 1
+  .amdhsa_user_sgpr_queue_ptr 1
+  .amdhsa_user_sgpr_kernarg_segment_ptr 1
+  .amdhsa_user_sgpr_dispatch_id 1
+  .amdhsa_user_sgpr_kernarg_preload_length 2
+  .amdhsa_user_sgpr_kernarg_preload_offset 1
+  .amdhsa_user_sgpr_private_segment_size 1
+  .amdhsa_uses_cu_stores 1
+  .amdhsa_wavefront_size32 1
+  .amdhsa_enable_private_segment 1
+  .amdhsa_system_sgpr_workgroup_id_x 0
+  .amdhsa_system_sgpr_workgroup_id_y 1
+  .amdhsa_system_sgpr_workgroup_id_z 1
+  .amdhsa_system_sgpr_workgroup_info 1
+  .amdhsa_system_vgpr_workitem_id 1
+  .amdhsa_next_free_vgpr 9
+  .amdhsa_next_free_sgpr 27
+  .amdhsa_named_barrier_count 3
+  .amdhsa_reserve_vcc 0
+  .amdhsa_float_round_mode_32 1
+  .amdhsa_float_round_mode_16_64 1
+  .amdhsa_float_denorm_mode_32 1
+  .amdhsa_float_denorm_mode_16_64 0
+  .amdhsa_fp16_overflow 1
+  .amdhsa_memory_ordered 1
+  .amdhsa_forward_progress 1
+  .amdhsa_round_robin_scheduling 1
+  .amdhsa_exception_fp_ieee_invalid_op 1
+  .amdhsa_exception_fp_denorm_src 1
+  .amdhsa_exception_fp_ieee_div_zero 1
+  .amdhsa_exception_fp_ieee_overflow 1
+  .amdhsa_exception_fp_ieee_underflow 1
+  .amdhsa_exception_fp_ieee_inexact 1
+  .amdhsa_exception_int_div_zero 1
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel complete
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 1
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 1
+// ASM-NEXT: .amdhsa_kernarg_size 12
+// ASM-NEXT: .amdhsa_user_sgpr_count 32
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_length 2
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_preload_offset 1
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
+// ASM-NEXT: .amdhsa_uses_cu_stores 1
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_enable_private_segment 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1
+// ASM-NEXT: .amdhsa_next_free_vgpr 9
+// ASM-NEXT: .amdhsa_next_free_sgpr 32
+// ASM-NEXT: .amdhsa_named_barrier_count 3
+// ASM-NEXT: .amdhsa_reserve_vcc 0
+// ASM-NEXT: .amdhsa_float_round_mode_32 1
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 1
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 1
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 0
+// ASM-NEXT: .amdhsa_fp16_overflow 1
+// ASM-NEXT: .amdhsa_memory_ordered 1
+// ASM-NEXT: .amdhsa_forward_progress 1
+// ASM-NEXT: .amdhsa_inst_pref_size 0
+// ASM-NEXT: .amdhsa_round_robin_scheduling 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1
+// ASM-NEXT: .amdhsa_exception_int_div_zero 1
+// ASM-NEXT: .end_amdhsa_kernel
+
+// Test that we are including special SGPR usage in the granulated count.
+.p2align 6
+.amdhsa_kernel special_sgpr
+  .amdhsa_next_free_sgpr 27
+
+  .amdhsa_reserve_vcc 0
+
+  .amdhsa_float_denorm_mode_16_64 0
+  .amdhsa_next_free_vgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel special_sgpr
+// ASM: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 27
+// ASM-NEXT: .amdhsa_named_barrier_count 0
+// ASM-NEXT: .amdhsa_reserve_vcc 0
+// ASM: .amdhsa_float_denorm_mode_16_64 0
+// ASM: .end_amdhsa_kernel
+
+// Test that explicitly disabling user_sgpr's does not affect the user_sgpr
+// count, i.e. this should produce the same descriptor as minimal.
+.p2align 6
+.amdhsa_kernel disabled_user_sgpr
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel disabled_user_sgpr
+// ASM: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM: .end_amdhsa_kernel
+
+.p2align 6
+.amdhsa_kernel max_lds_size
+ .amdhsa_group_segment_fixed_size 393216
+ .amdhsa_next_free_vgpr 1
+ .amdhsa_next_free_sgpr 1
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel max_lds_size
+// ASM: .amdhsa_group_segment_fixed_size 393216
+// ASM: .end_amdhsa_kernel
+
+.section .foo
+
+.byte .amdgcn.gfx_generation_number
+// ASM: .byte 12
+
+.byte .amdgcn.gfx_generation_minor
+// ASM: .byte 5
+
+.byte .amdgcn.gfx_generation_stepping
+// ASM: .byte 0
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 0
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 0
+
+v_mov_b32_e32 v16, s3
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 17
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 4
+
+.set .amdgcn.next_free_vgpr, 0
+.set .amdgcn.next_free_sgpr, 0
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 0
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 0
+
+v_mov_b32_e32 v16, s3
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 17
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 4
+
+// Metadata
+
+.amdgpu_metadata
+  amdhsa.version:
+    - 3
+    - 0
+  amdhsa.kernels:
+    - .name:       amd_kernel_code_t_test_all
+      .symbol: amd_kernel_code_t_test_all@kd
+      .kernarg_segment_size: 8
+      .group_segment_fixed_size: 16
+      .private_segment_fixed_size: 32
+      .kernarg_segment_align: 64
+      .wavefront_size: 128
+      .sgpr_count: 14
+      .vgpr_count: 1024
+      .max_flat_workgroup_size: 256
+    - .name:       amd_kernel_code_t_minimal
+      .symbol: amd_kernel_code_t_minimal@kd
+      .kernarg_segment_size: 8
+      .group_segment_fixed_size: 16
+      .private_segment_fixed_size: 32
+      .kernarg_segment_align: 64
+      .wavefront_size: 128
+      .sgpr_count: 14
+      .vgpr_count: 40
+      .max_flat_workgroup_size: 256
+.end_amdgpu_metadata
+
+// ASM:      	.amdgpu_metadata
+// ASM:      amdhsa.kernels:
+// ASM:        - .group_segment_fixed_size: 16
+// ASM:          .kernarg_segment_align: 64
+// ASM:          .kernarg_segment_size: 8
+// ASM:          .max_flat_workgroup_size: 256
+// ASM:          .name:           amd_kernel_code_t_test_all
+// ASM:          .private_segment_fixed_size: 32
+// ASM:          .sgpr_count:     14
+// ASM:          .symbol:         'amd_kernel_code_t_test_all@kd'
+// ASM:          .vgpr_count:     1024
+// ASM:          .wavefront_size: 128
+// ASM:        - .group_segment_fixed_size: 16
+// ASM:          .kernarg_segment_align: 64
+// ASM:          .kernarg_segment_size: 8
+// ASM:          .max_flat_workgroup_size: 256
+// ASM:          .name:           amd_kernel_code_t_minimal
+// ASM:          .private_segment_fixed_size: 32
+// ASM:          .sgpr_count:     14
+// ASM:          .symbol:         'amd_kernel_code_t_minimal@kd'
+// ASM:          .vgpr_count:     40
+// ASM:          .wavefront_size: 128
+// ASM:      amdhsa.version:
+// ASM-NEXT:   - 3
+// ASM-NEXT:   - 0
+// ASM:      	.end_amdgpu_metadata
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.6a-fgt.txt b/llvm/test/MC/Disassembler/AArch64/armv8.6a-fgt.txt
deleted file mode 100644
index 5b8d8170145fe..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv8.6a-fgt.txt
+++ /dev/null
@@ -1,75 +0,0 @@
-# RUN: llvm-mc -triple=aarch64  -mattr=+fgt -disassemble < %s      | FileCheck %s
-# RUN: llvm-mc -triple=aarch64              -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOFGT
-
-[0x80,0x11,0x1c,0xd5]
-[0xa0,0x11,0x1c,0xd5]
-[0xc0,0x11,0x1c,0xd5]
-[0x80,0x31,0x1c,0xd5]
-[0xa0,0x31,0x1c,0xd5]
-[0xc0,0x31,0x1c,0xd5]
-
-# CHECK: msr HFGRTR_EL2, x0
-# CHECK: msr HFGWTR_EL2, x0
-# CHECK: msr HFGITR_EL2, x0
-# CHECK: msr HDFGRTR_EL2, x0
-# CHECK: msr HDFGWTR_EL2, x0
-# CHECK: msr HAFGRTR_EL2, x0
-# NOFGT: msr S3_4_C1_C1_4, x0
-# NOFGT: msr S3_4_C1_C1_5, x0
-# NOFGT: msr S3_4_C1_C1_6, x0
-# NOFGT: msr S3_4_C3_C1_4, x0
-# NOFGT: msr S3_4_C3_C1_5, x0
-# NOFGT: msr S3_4_C3_C1_6, x0
-
-[0x80,0x11,0x3c,0xd5]
-[0xa0,0x11,0x3c,0xd5]
-[0xc0,0x11,0x3c,0xd5]
-[0x80,0x31,0x3c,0xd5]
-[0xa0,0x31,0x3c,0xd5]
-[0xc0,0x31,0x3c,0xd5]
-
-# CHECK: mrs x0, HFGRTR_EL2
-# CHECK: mrs x0, HFGWTR_EL2
-# CHECK: mrs x0, HFGITR_EL2
-# CHECK: mrs x0, HDFGRTR_EL2
-# CHECK: mrs x0, HDFGWTR_EL2
-# CHECK: mrs x0, HAFGRTR_EL2
-# NOFGT: mrs x0, S3_4_C1_C1_4
-# NOFGT: mrs x0, S3_4_C1_C1_5
-# NOFGT: mrs x0, S3_4_C1_C1_6
-# NOFGT: mrs x0, S3_4_C3_C1_4
-# NOFGT: mrs x0, S3_4_C3_C1_5
-# NOFGT: mrs x0, S3_4_C3_C1_6
-
-[0x03,0x31,0x3c,0xd5]
-[0x23,0x31,0x3c,0xd5]
-[0x43,0x31,0x3c,0xd5]
-[0x63,0x31,0x3c,0xd5]
-[0xe3,0x31,0x3c,0xd5]
-# CHECK: mrs x3, HDFGRTR2_EL2
-# CHECK: mrs x3, HDFGWTR2_EL2
-# CHECK: mrs x3, HFGRTR2_EL2
-# CHECK: mrs x3, HFGWTR2_EL2
-# CHECK: mrs x3, HFGITR2_EL2
-# NOFGT: mrs x3, S3_4_C3_C1_0
-# NOFGT: mrs x3, S3_4_C3_C1_1
-# NOFGT: mrs x3, S3_4_C3_C1_2
-# NOFGT: mrs x3, S3_4_C3_C1_3
-# NOFGT: mrs x3, S3_4_C3_C1_7
-
-
-[0x03,0x31,0x1c,0xd5]
-[0x23,0x31,0x1c,0xd5]
-[0x43,0x31,0x1c,0xd5]
-[0x63,0x31,0x1c,0xd5]
-[0xe3,0x31,0x1c,0xd5]
-# CHECK: msr HDFGRTR2_EL2, x3
-# CHECK: msr HDFGWTR2_EL2, x3
-# CHECK: msr HFGRTR2_EL2, x3
-# CHECK: msr HFGWTR2_EL2, x3
-# CHECK: msr HFGITR2_EL2, x3
-# NOFGT: msr S3_4_C3_C1_0, x3
-# NOFGT: msr S3_4_C3_C1_1, x3
-# NOFGT: msr S3_4_C3_C1_2, x3
-# NOFGT: msr S3_4_C3_C1_3, x3
-# NOFGT: msr S3_4_C3_C1_7, x3
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.8a-mops.txt b/llvm/test/MC/Disassembler/AArch64/armv8.8a-mops.txt
deleted file mode 100644
index de7121c3b3972..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv8.8a-mops.txt
+++ /dev/null
@@ -1,434 +0,0 @@
-# RUN: not llvm-mc -triple aarch64 -mattr=+mops,+mte -disassemble < %s 2> %t | FileCheck %s --check-prefixes=CHECK-MOPS,CHECK-MTE
-# RUN: FileCheck %s --check-prefix=CHECK-INVALID < %t
-# RUN: not llvm-mc -triple aarch64 -mattr=+v8.8a,+mte -disassemble < %s 2> %t | FileCheck %s --check-prefixes=CHECK-MOPS,CHECK-MTE
-# RUN: FileCheck %s --check-prefix=CHECK-INVALID < %t
-# RUN: not llvm-mc -triple aarch64 -mattr=+mops -disassemble < %s 2> %t | FileCheck %s --check-prefix=CHECK-MOPS
-# RUN: FileCheck %s --check-prefixes=CHECK-INVALID,CHECK-NO-MTE < %t
-# RUN: not llvm-mc -triple aarch64 -mattr=+v8.8a -disassemble < %s 2> %t | FileCheck %s --check-prefix=CHECK-MOPS
-# RUN: FileCheck %s --check-prefixes=CHECK-INVALID,CHECK-NO-MTE < %t
-# RUN: not llvm-mc -triple aarch64 -disassemble < %s 2> %t
-# RUN: FileCheck %s --check-prefixes=CHECK-INVALID,CHECK-NO-MOPS,CHECK-NO-MTE < %t
-
-
-[0x40,0x04,0x01,0x19]
-[0x40,0x44,0x01,0x19]
-[0x40,0x84,0x01,0x19]
-[0x40,0xc4,0x01,0x19]
-[0x40,0x14,0x01,0x19]
-[0x40,0x54,0x01,0x19]
-[0x40,0x94,0x01,0x19]
-[0x40,0xd4,0x01,0x19]
-[0x40,0x24,0x01,0x19]
-[0x40,0x64,0x01,0x19]
-[0x40,0xa4,0x01,0x19]
-[0x40,0xe4,0x01,0x19]
-[0x40,0x34,0x01,0x19]
-[0x40,0x74,0x01,0x19]
-[0x40,0xb4,0x01,0x19]
-[0x40,0xf4,0x01,0x19]
-# CHECK-MOPS: cpyfp	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfpwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfprn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfpn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfpwt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfpwtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfpwtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfpwtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfprt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfprtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfprtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfprtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfpt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfptwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfptrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfptn	[x0]!, [x1]!, x2!
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-
-[0x40,0x04,0x41,0x19]
-[0x40,0x44,0x41,0x19]
-[0x40,0x84,0x41,0x19]
-[0x40,0xc4,0x41,0x19]
-[0x40,0x14,0x41,0x19]
-[0x40,0x54,0x41,0x19]
-[0x40,0x94,0x41,0x19]
-[0x40,0xd4,0x41,0x19]
-[0x40,0x24,0x41,0x19]
-[0x40,0x64,0x41,0x19]
-[0x40,0xa4,0x41,0x19]
-[0x40,0xe4,0x41,0x19]
-[0x40,0x34,0x41,0x19]
-[0x40,0x74,0x41,0x19]
-[0x40,0xb4,0x41,0x19]
-[0x40,0xf4,0x41,0x19]
-# CHECK-MOPS: cpyfm	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmwt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmwtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmwtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmwtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmrt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmrtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmrtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmrtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfmtn	[x0]!, [x1]!, x2!
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-
-[0x40,0x04,0x81,0x19]
-[0x40,0x44,0x81,0x19]
-[0x40,0x84,0x81,0x19]
-[0x40,0xc4,0x81,0x19]
-[0x40,0x14,0x81,0x19]
-[0x40,0x54,0x81,0x19]
-[0x40,0x94,0x81,0x19]
-[0x40,0xd4,0x81,0x19]
-[0x40,0x24,0x81,0x19]
-[0x40,0x64,0x81,0x19]
-[0x40,0xa4,0x81,0x19]
-[0x40,0xe4,0x81,0x19]
-[0x40,0x34,0x81,0x19]
-[0x40,0x74,0x81,0x19]
-[0x40,0xb4,0x81,0x19]
-[0x40,0xf4,0x81,0x19]
-# CHECK-MOPS: cpyfe	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfewn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfern	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfen	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfewt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfewtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfewtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfewtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfert	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfertwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfertrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfertn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfet	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfetwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfetrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyfetn	[x0]!, [x1]!, x2!
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-
-[0x40,0x04,0x01,0x1d]
-[0x40,0x44,0x01,0x1d]
-[0x40,0x84,0x01,0x1d]
-[0x40,0xc4,0x01,0x1d]
-[0x40,0x14,0x01,0x1d]
-[0x40,0x54,0x01,0x1d]
-[0x40,0x94,0x01,0x1d]
-[0x40,0xd4,0x01,0x1d]
-[0x40,0x24,0x01,0x1d]
-[0x40,0x64,0x01,0x1d]
-[0x40,0xa4,0x01,0x1d]
-[0x40,0xe4,0x01,0x1d]
-[0x40,0x34,0x01,0x1d]
-[0x40,0x74,0x01,0x1d]
-[0x40,0xb4,0x01,0x1d]
-[0x40,0xf4,0x01,0x1d]
-# CHECK-MOPS: cpyp	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpypwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyprn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpypn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpypwt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpypwtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpypwtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpypwtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyprt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyprtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyprtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyprtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpypt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyptwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyptrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyptn	[x0]!, [x1]!, x2!
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-
-[0x40,0x04,0x41,0x1d]
-[0x40,0x44,0x41,0x1d]
-[0x40,0x84,0x41,0x1d]
-[0x40,0xc4,0x41,0x1d]
-[0x40,0x14,0x41,0x1d]
-[0x40,0x54,0x41,0x1d]
-[0x40,0x94,0x41,0x1d]
-[0x40,0xd4,0x41,0x1d]
-[0x40,0x24,0x41,0x1d]
-[0x40,0x64,0x41,0x1d]
-[0x40,0xa4,0x41,0x1d]
-[0x40,0xe4,0x41,0x1d]
-[0x40,0x34,0x41,0x1d]
-[0x40,0x74,0x41,0x1d]
-[0x40,0xb4,0x41,0x1d]
-[0x40,0xf4,0x41,0x1d]
-# CHECK-MOPS: cpym	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymwt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymwtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymwtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymwtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymrt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymrtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymrtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymrtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpymtn	[x0]!, [x1]!, x2!
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-
-[0x40,0x04,0x81,0x1d]
-[0x40,0x44,0x81,0x1d]
-[0x40,0x84,0x81,0x1d]
-[0x40,0xc4,0x81,0x1d]
-[0x40,0x14,0x81,0x1d]
-[0x40,0x54,0x81,0x1d]
-[0x40,0x94,0x81,0x1d]
-[0x40,0xd4,0x81,0x1d]
-[0x40,0x24,0x81,0x1d]
-[0x40,0x64,0x81,0x1d]
-[0x40,0xa4,0x81,0x1d]
-[0x40,0xe4,0x81,0x1d]
-[0x40,0x34,0x81,0x1d]
-[0x40,0x74,0x81,0x1d]
-[0x40,0xb4,0x81,0x1d]
-[0x40,0xf4,0x81,0x1d]
-# CHECK-MOPS: cpye	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyewn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyern	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyen	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyewt	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyewtwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyewtrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyewtn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyert	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyertwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyertrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyertn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyet	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyetwn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyetrn	[x0]!, [x1]!, x2!
-# CHECK-MOPS: cpyetn	[x0]!, [x1]!, x2!
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-
-[0x20,0x04,0xc2,0x19]
-[0x20,0x14,0xc2,0x19]
-[0x20,0x24,0xc2,0x19]
-[0x20,0x34,0xc2,0x19]
-# CHECK-MOPS: setp	[x0]!, x1!, x2
-# CHECK-MOPS: setpt	[x0]!, x1!, x2
-# CHECK-MOPS: setpn	[x0]!, x1!, x2
-# CHECK-MOPS: setptn	[x0]!, x1!, x2
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-
-[0x20,0x44,0xc2,0x19]
-[0x20,0x54,0xc2,0x19]
-[0x20,0x64,0xc2,0x19]
-[0x20,0x74,0xc2,0x19]
-# CHECK-MOPS: setm	[x0]!, x1!, x2
-# CHECK-MOPS: setmt	[x0]!, x1!, x2
-# CHECK-MOPS: setmn	[x0]!, x1!, x2
-# CHECK-MOPS: setmtn	[x0]!, x1!, x2
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-
-[0x20,0x84,0xc2,0x19]
-[0x20,0x94,0xc2,0x19]
-[0x20,0xa4,0xc2,0x19]
-[0x20,0xb4,0xc2,0x19]
-# CHECK-MOPS: sete	[x0]!, x1!, x2
-# CHECK-MOPS: setet	[x0]!, x1!, x2
-# CHECK-MOPS: seten	[x0]!, x1!, x2
-# CHECK-MOPS: setetn	[x0]!, x1!, x2
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-# CHECK-NO-MOPS: warning: invalid instruction encoding
-
-[0x20,0x04,0xc2,0x1d]
-[0x20,0x14,0xc2,0x1d]
-[0x20,0x24,0xc2,0x1d]
-[0x20,0x34,0xc2,0x1d]
-# CHECK-MTE: setgp	[x0]!, x1!, x2
-# CHECK-MTE: setgpt [x0]!, x1!, x2
-# CHECK-MTE: setgpn	[x0]!, x1!, x2
-# CHECK-MTE: setgptn	[x0]!, x1!, x2
-# CHECK-NO-MTE: warning: invalid instruction encoding
-# CHECK-NO-MTE: warning: invalid instruction encoding
-# CHECK-NO-MTE: warning: invalid instruction encoding
-# CHECK-NO-MTE: warning: invalid instruction encoding
-
-[0x20,0x44,0xc2,0x1d]
-[0x20,0x54,0xc2,0x1d]
-[0x20,0x64,0xc2,0x1d]
-[0x20,0x74,0xc2,0x1d]
-# CHECK-MTE: setgm	[x0]!, x1!, x2
-# CHECK-MTE: setgmt	[x0]!, x1!, x2
-# CHECK-MTE: setgmn	[x0]!, x1!, x2
-# CHECK-MTE: setgmtn	[x0]!, x1!, x2
-# CHECK-NO-MTE: warning: invalid instruction encoding
-# CHECK-NO-MTE: warning: invalid instruction encoding
-# CHECK-NO-MTE: warning: invalid instruction encoding
-# CHECK-NO-MTE: warning: invalid instruction encoding
-
-[0x20,0x84,0xc2,0x1d]
-[0x20,0x94,0xc2,0x1d]
-[0x20,0xa4,0xc2,0x1d]
-[0x20,0xb4,0xc2,0x1d]
-# CHECK-MTE: setge	[x0]!, x1!, x2
-# CHECK-MTE: setget	[x0]!, x1!, x2
-# CHECK-MTE: setgen	[x0]!, x1!, x2
-# CHECK-MTE: setgetn	[x0]!, x1!, x2
-# CHECK-NO-MTE: warning: invalid instruction encoding
-# CHECK-NO-MTE: warning: invalid instruction encoding
-# CHECK-NO-MTE: warning: invalid instruction encoding
-# CHECK-NO-MTE: warning: invalid instruction encoding
-
-
-# Register number 31 (SP or XZR) is not allowed in address positions.
-# cpyfp
-[0x5f,0x04,0x01,0x19]
-[0x40,0x04,0x1f,0x19]
-# cpyfm
-[0x5f,0x04,0x41,0x19]
-[0x40,0x04,0x5f,0x19]
-# cpyfe
-[0x5f,0x04,0x81,0x19]
-[0x40,0x04,0x9f,0x19]
-# cpyp
-[0x5f,0x04,0x01,0x1d]
-[0x40,0x04,0x1f,0x1d]
-# cpym
-[0x5f,0x04,0x41,0x1d]
-[0x40,0x04,0x5f,0x1d]
-# cpye
-[0x5f,0x04,0x81,0x1d]
-[0x40,0x04,0x9f,0x1d]
-# setp
-[0x5f,0x04,0xc2,0x19]
-# setm
-[0x5f,0x44,0xc2,0x19]
-# sete
-[0x5f,0x84,0xc2,0x19]
-# setgp
-[0x5f,0x04,0xc2,0x1d]
-# setgm
-[0x5f,0x44,0xc2,0x1d]
-# setge
-[0x5f,0x84,0xc2,0x1d]
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
-# CHECK-INVALID: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.9a-ats1a.txt b/llvm/test/MC/Disassembler/AArch64/armv8.9a-ats1a.txt
deleted file mode 100644
index 03aca5e916db0..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv8.9a-ats1a.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-# RUN: llvm-mc -triple=aarch64 -disassemble %s | FileCheck %s
-
-[0x41,0x79,0x08,0xd5]
-# CHECK: at s1e1a, x1
-
-[0x41,0x79,0x0c,0xd5]
-# CHECK: at s1e2a, x1
-
-[0x41,0x79,0x0e,0xd5]
-# CHECK: at s1e3a, x1
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.9a-clrbhb.txt b/llvm/test/MC/Disassembler/AArch64/armv8.9a-clrbhb.txt
deleted file mode 100644
index f8c7e9fa1a15a..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv8.9a-clrbhb.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-# CLRBHB is optional for all v8a/v9a, mandatory for 8.9a/9.4a.
-# Should disassemble to hint #22 if the feature is not present.
-# RUN: llvm-mc -triple=aarch64 -disassemble %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v8a %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v8.9a,-clrbhb %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v9.3a %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v9.4a,-clrbhb %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+clrbhb %s | FileCheck %s --check-prefix=CLRBHB
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v8a,+clrbhb %s | FileCheck %s --check-prefix=CLRBHB
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v8.9a %s | FileCheck %s --check-prefix=CLRBHB
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v9.3a,+clrbhb %s | FileCheck %s --check-prefix=CLRBHB
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v9.4a %s | FileCheck %s --check-prefix=CLRBHB
-
-[0xdf,0x22,0x03,0xd5]
-# CLRBHB: clrbhb
-# HINT_22: hint #22
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.9a-debug-pmu.txt b/llvm/test/MC/Disassembler/AArch64/armv8.9a-debug-pmu.txt
deleted file mode 100644
index ff898fec0c92f..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv8.9a-debug-pmu.txt
+++ /dev/null
@@ -1,730 +0,0 @@
-# RUN: llvm-mc -triple=aarch64               -mattr=+ite -disassemble %s | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=+v8.8a -mattr=+ite -disassemble %s | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=+v9.3a -mattr=+ite -disassemble %s | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=+v8.9a -mattr=+ite -disassemble %s | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=+v9.4a -mattr=+ite -disassemble %s | FileCheck %s
-
-# RUN: llvm-mc -triple=aarch64                           -disassemble %s | FileCheck %s --check-prefix=ERROR-NO-ITE
-# RUN: llvm-mc -triple=aarch64 -mattr=+v8.8a             -disassemble %s | FileCheck %s --check-prefix=ERROR-NO-ITE
-# RUN: llvm-mc -triple=aarch64 -mattr=+v9.3a             -disassemble %s | FileCheck %s --check-prefix=ERROR-NO-ITE
-# RUN: llvm-mc -triple=aarch64 -mattr=+v8.9a             -disassemble %s | FileCheck %s --check-prefix=ERROR-NO-ITE
-# RUN: llvm-mc -triple=aarch64 -mattr=+v9.4a             -disassemble %s | FileCheck %s --check-prefix=ERROR-NO-ITE
-
-[0x83,0x00,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR0_EL1
-[0x81,0x00,0x10,0xd5]
-# CHECK:    msr DBGBVR0_EL1, x1
-[0x83,0x01,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR1_EL1
-[0x81,0x01,0x10,0xd5]
-# CHECK:    msr DBGBVR1_EL1, x1
-[0x83,0x02,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR2_EL1
-[0x81,0x02,0x10,0xd5]
-# CHECK:    msr DBGBVR2_EL1, x1
-[0x83,0x03,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR3_EL1
-[0x81,0x03,0x10,0xd5]
-# CHECK:    msr DBGBVR3_EL1, x1
-[0x83,0x04,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR4_EL1
-[0x81,0x04,0x10,0xd5]
-# CHECK:    msr DBGBVR4_EL1, x1
-[0x83,0x05,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR5_EL1
-[0x81,0x05,0x10,0xd5]
-# CHECK:    msr DBGBVR5_EL1, x1
-[0x83,0x06,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR6_EL1
-[0x81,0x06,0x10,0xd5]
-# CHECK:    msr DBGBVR6_EL1, x1
-[0x83,0x07,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR7_EL1
-[0x81,0x07,0x10,0xd5]
-# CHECK:    msr DBGBVR7_EL1, x1
-[0x83,0x08,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR8_EL1
-[0x81,0x08,0x10,0xd5]
-# CHECK:    msr DBGBVR8_EL1, x1
-[0x83,0x09,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR9_EL1
-[0x81,0x09,0x10,0xd5]
-# CHECK:    msr DBGBVR9_EL1, x1
-[0x83,0x0a,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR10_EL1
-[0x81,0x0a,0x10,0xd5]
-# CHECK:    msr DBGBVR10_EL1, x1
-[0x83,0x0b,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR11_EL1
-[0x81,0x0b,0x10,0xd5]
-# CHECK:    msr DBGBVR11_EL1, x1
-[0x83,0x0c,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR12_EL1
-[0x81,0x0c,0x10,0xd5]
-# CHECK:    msr DBGBVR12_EL1, x1
-[0x83,0x0d,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR13_EL1
-[0x81,0x0d,0x10,0xd5]
-# CHECK:    msr DBGBVR13_EL1, x1
-[0x83,0x0e,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR14_EL1
-[0x81,0x0e,0x10,0xd5]
-# CHECK:    msr DBGBVR14_EL1, x1
-[0x83,0x0f,0x30,0xd5]
-# CHECK:    mrs x3, DBGBVR15_EL1
-[0x81,0x0f,0x10,0xd5]
-# CHECK:    msr DBGBVR15_EL1, x1
-
-[0xa3,0x00,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR0_EL1
-[0xa1,0x00,0x10,0xd5]
-# CHECK:    msr DBGBCR0_EL1, x1
-[0xa3,0x01,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR1_EL1
-[0xa1,0x01,0x10,0xd5]
-# CHECK:    msr DBGBCR1_EL1, x1
-[0xa3,0x02,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR2_EL1
-[0xa1,0x02,0x10,0xd5]
-# CHECK:    msr DBGBCR2_EL1, x1
-[0xa3,0x03,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR3_EL1
-[0xa1,0x03,0x10,0xd5]
-# CHECK:    msr DBGBCR3_EL1, x1
-[0xa3,0x04,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR4_EL1
-[0xa1,0x04,0x10,0xd5]
-# CHECK:    msr DBGBCR4_EL1, x1
-[0xa3,0x05,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR5_EL1
-[0xa1,0x05,0x10,0xd5]
-# CHECK:    msr DBGBCR5_EL1, x1
-[0xa3,0x06,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR6_EL1
-[0xa1,0x06,0x10,0xd5]
-# CHECK:    msr DBGBCR6_EL1, x1
-[0xa3,0x07,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR7_EL1
-[0xa1,0x07,0x10,0xd5]
-# CHECK:    msr DBGBCR7_EL1, x1
-[0xa3,0x08,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR8_EL1
-[0xa1,0x08,0x10,0xd5]
-# CHECK:    msr DBGBCR8_EL1, x1
-[0xa3,0x09,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR9_EL1
-[0xa1,0x09,0x10,0xd5]
-# CHECK:    msr DBGBCR9_EL1, x1
-[0xa3,0x0a,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR10_EL1
-[0xa1,0x0a,0x10,0xd5]
-# CHECK:    msr DBGBCR10_EL1, x1
-[0xa3,0x0b,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR11_EL1
-[0xa1,0x0b,0x10,0xd5]
-# CHECK:    msr DBGBCR11_EL1, x1
-[0xa3,0x0c,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR12_EL1
-[0xa1,0x0c,0x10,0xd5]
-# CHECK:    msr DBGBCR12_EL1, x1
-[0xa3,0x0d,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR13_EL1
-[0xa1,0x0d,0x10,0xd5]
-# CHECK:    msr DBGBCR13_EL1, x1
-[0xa3,0x0e,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR14_EL1
-[0xa1,0x0e,0x10,0xd5]
-# CHECK:    msr DBGBCR14_EL1, x1
-[0xa3,0x0f,0x30,0xd5]
-# CHECK:    mrs x3, DBGBCR15_EL1
-[0xa1,0x0f,0x10,0xd5]
-# CHECK:    msr DBGBCR15_EL1, x1
-
-[0xc3,0x00,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR0_EL1
-[0xc1,0x00,0x10,0xd5]
-# CHECK:    msr DBGWVR0_EL1, x1
-[0xc3,0x01,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR1_EL1
-[0xc1,0x01,0x10,0xd5]
-# CHECK:    msr DBGWVR1_EL1, x1
-[0xc3,0x02,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR2_EL1
-[0xc1,0x02,0x10,0xd5]
-# CHECK:    msr DBGWVR2_EL1, x1
-[0xc3,0x03,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR3_EL1
-[0xc1,0x03,0x10,0xd5]
-# CHECK:    msr DBGWVR3_EL1, x1
-[0xc3,0x04,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR4_EL1
-[0xc1,0x04,0x10,0xd5]
-# CHECK:    msr DBGWVR4_EL1, x1
-[0xc3,0x05,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR5_EL1
-[0xc1,0x05,0x10,0xd5]
-# CHECK:    msr DBGWVR5_EL1, x1
-[0xc3,0x06,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR6_EL1
-[0xc1,0x06,0x10,0xd5]
-# CHECK:    msr DBGWVR6_EL1, x1
-[0xc3,0x07,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR7_EL1
-[0xc1,0x07,0x10,0xd5]
-# CHECK:    msr DBGWVR7_EL1, x1
-[0xc3,0x08,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR8_EL1
-[0xc1,0x08,0x10,0xd5]
-# CHECK:    msr DBGWVR8_EL1, x1
-[0xc3,0x09,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR9_EL1
-[0xc1,0x09,0x10,0xd5]
-# CHECK:    msr DBGWVR9_EL1, x1
-[0xc3,0x0a,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR10_EL1
-[0xc1,0x0a,0x10,0xd5]
-# CHECK:    msr DBGWVR10_EL1, x1
-[0xc3,0x0b,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR11_EL1
-[0xc1,0x0b,0x10,0xd5]
-# CHECK:    msr DBGWVR11_EL1, x1
-[0xc3,0x0c,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR12_EL1
-[0xc1,0x0c,0x10,0xd5]
-# CHECK:    msr DBGWVR12_EL1, x1
-[0xc3,0x0d,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR13_EL1
-[0xc1,0x0d,0x10,0xd5]
-# CHECK:    msr DBGWVR13_EL1, x1
-[0xc3,0x0e,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR14_EL1
-[0xc1,0x0e,0x10,0xd5]
-# CHECK:    msr DBGWVR14_EL1, x1
-[0xc3,0x0f,0x30,0xd5]
-# CHECK:    mrs x3, DBGWVR15_EL1
-[0xc1,0x0f,0x10,0xd5]
-# CHECK:    msr DBGWVR15_EL1, x1
-
-[0xe3,0x00,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR0_EL1
-[0xe1,0x00,0x10,0xd5]
-# CHECK:    msr DBGWCR0_EL1, x1
-[0xe3,0x01,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR1_EL1
-[0xe1,0x01,0x10,0xd5]
-# CHECK:    msr DBGWCR1_EL1, x1
-[0xe3,0x02,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR2_EL1
-[0xe1,0x02,0x10,0xd5]
-# CHECK:    msr DBGWCR2_EL1, x1
-[0xe3,0x03,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR3_EL1
-[0xe1,0x03,0x10,0xd5]
-# CHECK:    msr DBGWCR3_EL1, x1
-[0xe3,0x04,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR4_EL1
-[0xe1,0x04,0x10,0xd5]
-# CHECK:    msr DBGWCR4_EL1, x1
-[0xe3,0x05,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR5_EL1
-[0xe1,0x05,0x10,0xd5]
-# CHECK:    msr DBGWCR5_EL1, x1
-[0xe3,0x06,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR6_EL1
-[0xe1,0x06,0x10,0xd5]
-# CHECK:    msr DBGWCR6_EL1, x1
-[0xe3,0x07,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR7_EL1
-[0xe1,0x07,0x10,0xd5]
-# CHECK:    msr DBGWCR7_EL1, x1
-[0xe3,0x08,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR8_EL1
-[0xe1,0x08,0x10,0xd5]
-# CHECK:    msr DBGWCR8_EL1, x1
-[0xe3,0x09,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR9_EL1
-[0xe1,0x09,0x10,0xd5]
-# CHECK:    msr DBGWCR9_EL1, x1
-[0xe3,0x0a,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR10_EL1
-[0xe1,0x0a,0x10,0xd5]
-# CHECK:    msr DBGWCR10_EL1, x1
-[0xe3,0x0b,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR11_EL1
-[0xe1,0x0b,0x10,0xd5]
-# CHECK:    msr DBGWCR11_EL1, x1
-[0xe3,0x0c,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR12_EL1
-[0xe1,0x0c,0x10,0xd5]
-# CHECK:    msr DBGWCR12_EL1, x1
-[0xe3,0x0d,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR13_EL1
-[0xe1,0x0d,0x10,0xd5]
-# CHECK:    msr DBGWCR13_EL1, x1
-[0xe3,0x0e,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR14_EL1
-[0xe1,0x0e,0x10,0xd5]
-# CHECK:    msr DBGWCR14_EL1, x1
-[0xe3,0x0f,0x30,0xd5]
-# CHECK:    mrs x3, DBGWCR15_EL1
-[0xe1,0x0f,0x10,0xd5]
-# CHECK:    msr DBGWCR15_EL1, x1
-
-[0x43,0x04,0x30,0xd5]
-# CHECK:	mrs	x3, MDSELR_EL1
-[0x41,0x04,0x10,0xd5]
-# CHECK:	msr	MDSELR_EL1, x1
-
-[0x83,0x9e,0x38,0xd5]
-# CHECK:	mrs	x3, PMUACR_EL1
-[0x81,0x9e,0x18,0xd5]
-# CHECK:	msr	PMUACR_EL1, x1
-
-[0xe3,0xeb,0x30,0xd5]
-# CHECK:	mrs	x3, PMCCNTSVR_EL1
-[0x03,0xec,0x30,0xd5]
-# CHECK:	mrs	x3, PMICNTSVR_EL1
-[0x63,0x9d,0x38,0xd5]
-# CHECK:	mrs	x3, PMSSCR_EL1
-[0x61,0x9d,0x18,0xd5]
-# CHECK:	msr	PMSSCR_EL1, x1
-[0x03,0xe8,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR0_EL1
-[0x23,0xe8,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR1_EL1
-[0x43,0xe8,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR2_EL1
-[0x63,0xe8,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR3_EL1
-[0x83,0xe8,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR4_EL1
-[0xa3,0xe8,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR5_EL1
-[0xc3,0xe8,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR6_EL1
-[0xe3,0xe8,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR7_EL1
-[0x03,0xe9,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR8_EL1
-[0x23,0xe9,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR9_EL1
-[0x43,0xe9,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR10_EL1
-[0x63,0xe9,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR11_EL1
-[0x83,0xe9,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR12_EL1
-[0xa3,0xe9,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR13_EL1
-[0xc3,0xe9,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR14_EL1
-[0xe3,0xe9,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR15_EL1
-[0x03,0xea,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR16_EL1
-[0x23,0xea,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR17_EL1
-[0x43,0xea,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR18_EL1
-[0x63,0xea,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR19_EL1
-[0x83,0xea,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR20_EL1
-[0xa3,0xea,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR21_EL1
-[0xc3,0xea,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR22_EL1
-[0xe3,0xea,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR23_EL1
-[0x03,0xeb,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR24_EL1
-[0x23,0xeb,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR25_EL1
-[0x43,0xeb,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR26_EL1
-[0x63,0xeb,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR27_EL1
-[0x83,0xeb,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR28_EL1
-[0xa3,0xeb,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR29_EL1
-[0xc3,0xeb,0x30,0xd5]
-# CHECK:	mrs	x3, PMEVCNTSVR30_EL1
-
-[0x03,0x94,0x3b,0xd5]
-# CHECK:  mrs x3, PMICNTR_EL0
-[0x03,0x94,0x1b,0xd5]
-# CHECK:  msr PMICNTR_EL0, x3
-[0x03,0x96,0x3b,0xd5]
-# CHECK:  mrs x3, PMICFILTR_EL0
-[0x03,0x96,0x1b,0xd5]
-# CHECK:  msr PMICFILTR_EL0, x3
-
-[0x83,0x9d,0x1b,0xd5]
-# CHECK:  msr PMZR_EL0, x3
-
-[0xa3,0x9e,0x38,0xd5]
-# CHECK:	mrs	x3, PMECR_EL1
-[0xa1,0x9e,0x18,0xd5]
-# CHECK:	msr	PMECR_EL1, x1
-[0xe3,0x9e,0x38,0xd5]
-# CHECK:	mrs	x3, PMIAR_EL1
-[0xe1,0x9e,0x18,0xd5]
-# CHECK:	msr	PMIAR_EL1, x1
-
-[0x63,0x9d,0x30,0xd5]
-# CHECK:	mrs	x3, SPMACCESSR_EL1
-[0x61,0x9d,0x10,0xd5]
-# CHECK:	msr	SPMACCESSR_EL1, x1
-[0x63,0x9d,0x35,0xd5]
-# CHECK:	mrs	x3, SPMACCESSR_EL12
-[0x61,0x9d,0x15,0xd5]
-# CHECK:	msr	SPMACCESSR_EL12, x1
-[0x63,0x9d,0x34,0xd5]
-# CHECK:	mrs	x3, SPMACCESSR_EL2
-[0x61,0x9d,0x14,0xd5]
-# CHECK:	msr	SPMACCESSR_EL2, x1
-[0x63,0x9d,0x36,0xd5]
-# CHECK:	mrs	x3, SPMACCESSR_EL3
-[0x61,0x9d,0x16,0xd5]
-# CHECK:	msr	SPMACCESSR_EL3, x1
-[0x43,0x9c,0x33,0xd5]
-# CHECK:	mrs	x3, SPMCNTENCLR_EL0
-[0x41,0x9c,0x13,0xd5]
-# CHECK:	msr	SPMCNTENCLR_EL0, x1
-[0x23,0x9c,0x33,0xd5]
-# CHECK:	mrs	x3, SPMCNTENSET_EL0
-[0x21,0x9c,0x13,0xd5]
-# CHECK:	msr	SPMCNTENSET_EL0, x1
-[0x03,0x9c,0x33,0xd5]
-# CHECK:	mrs	x3, SPMCR_EL0
-[0x01,0x9c,0x13,0xd5]
-# CHECK:	msr	SPMCR_EL0, x1
-[0xc3,0x9d,0x30,0xd5]
-# CHECK:	mrs	x3, SPMDEVAFF_EL1
-[0xa3,0x9d,0x30,0xd5]
-# CHECK:	mrs	x3, SPMDEVARCH_EL1
-
-[0x03,0xe0,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR0_EL0
-[0x01,0xe0,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR0_EL0, x1
-[0x23,0xe0,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR1_EL0
-[0x21,0xe0,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR1_EL0, x1
-[0x43,0xe0,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR2_EL0
-[0x41,0xe0,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR2_EL0, x1
-[0x63,0xe0,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR3_EL0
-[0x61,0xe0,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR3_EL0, x1
-[0x83,0xe0,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR4_EL0
-[0x81,0xe0,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR4_EL0, x1
-[0xa3,0xe0,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR5_EL0
-[0xa1,0xe0,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR5_EL0, x1
-[0xc3,0xe0,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR6_EL0
-[0xc1,0xe0,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR6_EL0, x1
-[0xe3,0xe0,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR7_EL0
-[0xe1,0xe0,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR7_EL0, x1
-[0x03,0xe1,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR8_EL0
-[0x01,0xe1,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR8_EL0, x1
-[0x23,0xe1,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR9_EL0
-[0x21,0xe1,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR9_EL0, x1
-[0x43,0xe1,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR10_EL0
-[0x41,0xe1,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR10_EL0, x1
-[0x63,0xe1,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR11_EL0
-[0x61,0xe1,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR11_EL0, x1
-[0x83,0xe1,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR12_EL0
-[0x81,0xe1,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR12_EL0, x1
-[0xa3,0xe1,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR13_EL0
-[0xa1,0xe1,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR13_EL0, x1
-[0xc3,0xe1,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR14_EL0
-[0xc1,0xe1,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR14_EL0, x1
-[0xe3,0xe1,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVCNTR15_EL0
-[0xe1,0xe1,0x13,0xd5]
-# CHECK:	msr	SPMEVCNTR15_EL0, x1
-
-[0x03,0xe6,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R0_EL0
-[0x01,0xe6,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R0_EL0, x1
-[0x23,0xe6,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R1_EL0
-[0x21,0xe6,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R1_EL0, x1
-[0x43,0xe6,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R2_EL0
-[0x41,0xe6,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R2_EL0, x1
-[0x63,0xe6,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R3_EL0
-[0x61,0xe6,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R3_EL0, x1
-[0x83,0xe6,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R4_EL0
-[0x81,0xe6,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R4_EL0, x1
-[0xa3,0xe6,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R5_EL0
-[0xa1,0xe6,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R5_EL0, x1
-[0xc3,0xe6,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R6_EL0
-[0xc1,0xe6,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R6_EL0, x1
-[0xe3,0xe6,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R7_EL0
-[0xe1,0xe6,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R7_EL0, x1
-[0x03,0xe7,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R8_EL0
-[0x01,0xe7,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R8_EL0, x1
-[0x23,0xe7,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R9_EL0
-[0x21,0xe7,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R9_EL0, x1
-[0x43,0xe7,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R10_EL0
-[0x41,0xe7,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R10_EL0, x1
-[0x63,0xe7,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R11_EL0
-[0x61,0xe7,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R11_EL0, x1
-[0x83,0xe7,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R12_EL0
-[0x81,0xe7,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R12_EL0, x1
-[0xa3,0xe7,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R13_EL0
-[0xa1,0xe7,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R13_EL0, x1
-[0xc3,0xe7,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R14_EL0
-[0xc1,0xe7,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R14_EL0, x1
-[0xe3,0xe7,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILT2R15_EL0
-[0xe1,0xe7,0x13,0xd5]
-# CHECK:	msr	SPMEVFILT2R15_EL0, x1
-
-[0x03,0xe4,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR0_EL0
-[0x01,0xe4,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR0_EL0, x1
-[0x23,0xe4,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR1_EL0
-[0x21,0xe4,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR1_EL0, x1
-[0x43,0xe4,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR2_EL0
-[0x41,0xe4,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR2_EL0, x1
-[0x63,0xe4,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR3_EL0
-[0x61,0xe4,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR3_EL0, x1
-[0x83,0xe4,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR4_EL0
-[0x81,0xe4,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR4_EL0, x1
-[0xa3,0xe4,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR5_EL0
-[0xa1,0xe4,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR5_EL0, x1
-[0xc3,0xe4,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR6_EL0
-[0xc1,0xe4,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR6_EL0, x1
-[0xe3,0xe4,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR7_EL0
-[0xe1,0xe4,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR7_EL0, x1
-[0x03,0xe5,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR8_EL0
-[0x01,0xe5,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR8_EL0, x1
-[0x23,0xe5,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR9_EL0
-[0x21,0xe5,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR9_EL0, x1
-[0x43,0xe5,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR10_EL0
-[0x41,0xe5,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR10_EL0, x1
-[0x63,0xe5,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR11_EL0
-[0x61,0xe5,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR11_EL0, x1
-[0x83,0xe5,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR12_EL0
-[0x81,0xe5,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR12_EL0, x1
-[0xa3,0xe5,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR13_EL0
-[0xa1,0xe5,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR13_EL0, x1
-[0xc3,0xe5,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR14_EL0
-[0xc1,0xe5,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR14_EL0, x1
-[0xe3,0xe5,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVFILTR15_EL0
-[0xe1,0xe5,0x13,0xd5]
-# CHECK:	msr	SPMEVFILTR15_EL0, x1
-
-[0x03,0xe2,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER0_EL0
-[0x01,0xe2,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER0_EL0, x1
-[0x23,0xe2,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER1_EL0
-[0x21,0xe2,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER1_EL0, x1
-[0x43,0xe2,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER2_EL0
-[0x41,0xe2,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER2_EL0, x1
-[0x63,0xe2,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER3_EL0
-[0x61,0xe2,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER3_EL0, x1
-[0x83,0xe2,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER4_EL0
-[0x81,0xe2,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER4_EL0, x1
-[0xa3,0xe2,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER5_EL0
-[0xa1,0xe2,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER5_EL0, x1
-[0xc3,0xe2,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER6_EL0
-[0xc1,0xe2,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER6_EL0, x1
-[0xe3,0xe2,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER7_EL0
-[0xe1,0xe2,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER7_EL0, x1
-[0x03,0xe3,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER8_EL0
-[0x01,0xe3,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER8_EL0, x1
-[0x23,0xe3,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER9_EL0
-[0x21,0xe3,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER9_EL0, x1
-[0x43,0xe3,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER10_EL0
-[0x41,0xe3,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER10_EL0, x1
-[0x63,0xe3,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER11_EL0
-[0x61,0xe3,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER11_EL0, x1
-[0x83,0xe3,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER12_EL0
-[0x81,0xe3,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER12_EL0, x1
-[0xa3,0xe3,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER13_EL0
-[0xa1,0xe3,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER13_EL0, x1
-[0xc3,0xe3,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER14_EL0
-[0xc1,0xe3,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER14_EL0, x1
-[0xe3,0xe3,0x33,0xd5]
-# CHECK:	mrs	x3, SPMEVTYPER15_EL0
-[0xe1,0xe3,0x13,0xd5]
-# CHECK:	msr	SPMEVTYPER15_EL0, x1
-
-[0x83,0x9d,0x30,0xd5]
-# CHECK:	mrs	x3, SPMIIDR_EL1
-[0x43,0x9e,0x30,0xd5]
-# CHECK:	mrs	x3, SPMINTENCLR_EL1
-[0x41,0x9e,0x10,0xd5]
-# CHECK:	msr	SPMINTENCLR_EL1, x1
-[0x23,0x9e,0x30,0xd5]
-# CHECK:	mrs	x3, SPMINTENSET_EL1
-[0x21,0x9e,0x10,0xd5]
-# CHECK:	msr	SPMINTENSET_EL1, x1
-[0x63,0x9c,0x33,0xd5]
-# CHECK:	mrs	x3, SPMOVSCLR_EL0
-[0x61,0x9c,0x13,0xd5]
-# CHECK:	msr	SPMOVSCLR_EL0, x1
-[0x63,0x9e,0x33,0xd5]
-# CHECK:	mrs	x3, SPMOVSSET_EL0
-[0x61,0x9e,0x13,0xd5]
-# CHECK:	msr	SPMOVSSET_EL0, x1
-[0xa3,0x9c,0x33,0xd5]
-# CHECK:	mrs	x3, SPMSELR_EL0
-[0xa1,0x9c,0x13,0xd5]
-# CHECK:	msr	SPMSELR_EL0, x1
-[0x03,0x9d,0x30,0xd5]
-# CHECK:  mrs x3, SPMCGCR0_EL1
-[0x23,0x9d,0x30,0xd5]
-# CHECK:  mrs x3, SPMCGCR1_EL1
-[0xe3,0x9d,0x30,0xd5]
-# CHECK:  mrs x3, SPMCFGR_EL1
-[0xe3,0x9e,0x36,0xd5]
-# CHECK:  mrs x3, SPMROOTCR_EL3
-[0xe3,0x9e,0x16,0xd5]
-# CHECK:  msr SPMROOTCR_EL3, x3
-[0xe3,0x9e,0x37,0xd5]
-# CHECK:  mrs x3, SPMSCR_EL1
-[0xe3,0x9e,0x17,0xd5]
-# CHECK:  msr SPMSCR_EL1, x3
-
-[0x63,0x12,0x38,0xd5]
-# CHECK:	mrs	x3, TRCITECR_EL1
-# ERROR-NO-ITE:    mrs x3, S3_0_C1_C2_3
-[0x61,0x12,0x18,0xd5]
-# CHECK:	msr	TRCITECR_EL1, x1
-# ERROR-NO-ITE:    msr S3_0_C1_C2_3, x1
-[0x63,0x12,0x3d,0xd5]
-# CHECK:	mrs	x3, TRCITECR_EL12
-# ERROR-NO-ITE:    mrs x3, S3_5_C1_C2_3
-[0x61,0x12,0x1d,0xd5]
-# CHECK:	msr	TRCITECR_EL12, x1
-# ERROR-NO-ITE:    msr S3_5_C1_C2_3, x1
-[0x63,0x12,0x3c,0xd5]
-# CHECK:	mrs	x3, TRCITECR_EL2
-# ERROR-NO-ITE:    mrs x3, S3_4_C1_C2_3
-[0x61,0x12,0x1c,0xd5]
-# CHECK:	msr	TRCITECR_EL2, x1
-# ERROR-NO-ITE:    msr S3_4_C1_C2_3, x1
-[0xe1,0x72,0x0b,0xd5]
-# CHECK:    trcit x1
-# ERROR-NO-ITE:    sys #3, c7, c2, #7, x1
-
-[0x83,0x9a,0x38,0xd5]
-# CHECK:    mrs x3, PMSDSFR_EL1
-[0x83,0x9a,0x18,0xd5]
-# CHECK:    msr PMSDSFR_EL1, x3
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.9a-lrcpc3.txt b/llvm/test/MC/Disassembler/AArch64/armv8.9a-lrcpc3.txt
deleted file mode 100644
index 644e032cd79cd..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv8.9a-lrcpc3.txt
+++ /dev/null
@@ -1,113 +0,0 @@
-# RUN: llvm-mc -triple aarch64-none-linux-gnu -disassemble -show-encoding               -mattr=+rcpc3 < %s | FileCheck %s
-# RUN: llvm-mc -triple aarch64-none-linux-gnu -disassemble -show-encoding -mattr=+v8.9a -mattr=+rcpc3 < %s | FileCheck %s
-# RUN: llvm-mc -triple aarch64-none-linux-gnu -disassemble -show-encoding -mattr=+v9.4a -mattr=+rcpc3 < %s | FileCheck %s
-
-# RUN: not llvm-mc -triple aarch64-none-linux-gnu -disassemble               < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-RCPC3 %s
-# RUN: not llvm-mc -triple aarch64-none-linux-gnu -disassemble -mattr=+v8.9a < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-RCPC3 %s
-# RUN: not llvm-mc -triple aarch64-none-linux-gnu -disassemble -mattr=+v9.4a < %s 2>&1 | FileCheck --check-prefix=ERROR-NO-RCPC3 %s
-
-[0x18,0x0a,0x00,0x99]
-# CHECK:      stilp   w24, w0, [x16, #-8]!     // encoding: [0x18,0x0a,0x00,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x18,0x0a,0x00,0x99]
-# CHECK:      stilp   w24, w0, [x16, #-8]!     // encoding: [0x18,0x0a,0x00,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x39,0x0a,0x01,0xd9]
-# CHECK:      stilp   x25, x1, [x17, #-16]!    // encoding: [0x39,0x0a,0x01,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x39,0x0a,0x01,0xd9]
-# CHECK:      stilp   x25, x1, [x17, #-16]!    // encoding: [0x39,0x0a,0x01,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x5a,0x1a,0x02,0x99]
-# CHECK:      stilp   w26, w2, [x18]           // encoding: [0x5a,0x1a,0x02,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xfb,0x1b,0x03,0xd9]
-# CHECK:      stilp   x27, x3, [sp]            // encoding: [0xfb,0x1b,0x03,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x9c,0x0a,0x44,0x99]
-# CHECK:      ldiapp  w28, w4, [x20], #8       // encoding: [0x9c,0x0a,0x44,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x9c,0x0a,0x44,0x99]
-# CHECK:      ldiapp  w28, w4, [x20], #8       // encoding: [0x9c,0x0a,0x44,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xbd,0x0a,0x45,0xd9]
-# CHECK:      ldiapp  x29, x5, [x21], #16      // encoding: [0xbd,0x0a,0x45,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xbd,0x0a,0x45,0xd9]
-# CHECK:      ldiapp  x29, x5, [x21], #16      // encoding: [0xbd,0x0a,0x45,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xfe,0x1b,0x46,0x99]
-# CHECK:      ldiapp  w30, w6, [sp]            // encoding: [0xfe,0x1b,0x46,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xff,0x1a,0x47,0xd9]
-# CHECK:      ldiapp  xzr, x7, [x23]           // encoding: [0xff,0x1a,0x47,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0xe3,0x09,0x80,0x99]
-# CHECK:      stlr w3, [x15, #-4]!    // encoding: [0xe3,0x09,0x80,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x09,0x80,0x99]
-# CHECK:      stlr w3, [x15, #-4]!    // encoding: [0xe3,0x09,0x80,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x09,0x80,0xd9]
-# CHECK:      stlr x3, [x15, #-8]!    // encoding: [0xe3,0x09,0x80,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x0b,0x80,0xd9]
-# CHECK:      stlr x3, [sp, #-8]!     // encoding: [0xe3,0x0b,0x80,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x0b,0xc0,0x99]
-# CHECK:      ldapr w3, [sp], #4       // encoding: [0xe3,0x0b,0xc0,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x09,0xc0,0x99]
-# CHECK:      ldapr w3, [x15], #4      // encoding: [0xe3,0x09,0xc0,0x99]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x09,0xc0,0xd9]
-# CHECK:      ldapr x3, [x15], #8      // encoding: [0xe3,0x09,0xc0,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x09,0xc0,0xd9]
-# CHECK:      ldapr x3, [x15], #8      // encoding: [0xe3,0x09,0xc0,0xd9]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0xe3,0xf9,0x1f,0x1d]
-# CHECK:      stlur b3, [x15, #-1]  // encoding: [0xe3,0xf9,0x1f,0x1d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x29,0x00,0x5d]
-# CHECK:      stlur h3, [x15, #2]   // encoding: [0xe3,0x29,0x00,0x5d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0xd9,0x1f,0x9d]
-# CHECK:      stlur s3, [x15, #-3]  // encoding: [0xe3,0xd9,0x1f,0x9d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x4b,0x00,0xdd]
-# CHECK:      stlur d3, [sp, #4]    // encoding: [0xe3,0x4b,0x00,0xdd]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0xb9,0x9f,0x1d]
-# CHECK:      stlur q3, [x15, #-5]  // encoding: [0xe3,0xb9,0x9f,0x1d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x69,0x40,0x1d]
-# CHECK:      ldapur b3, [x15, #6]  // encoding: [0xe3,0x69,0x40,0x1d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x99,0x5f,0x5d]
-# CHECK:      ldapur h3, [x15, #-7] // encoding: [0xe3,0x99,0x5f,0x5d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x89,0x40,0x9d]
-# CHECK:      ldapur s3, [x15, #8]  // encoding: [0xe3,0x89,0x40,0x9d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x79,0x5f,0xdd]
-# CHECK:      ldapur d3, [x15, #-9] // encoding: [0xe3,0x79,0x5f,0xdd]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0xab,0xc0,0x1d]
-# CHECK:      ldapur q3, [sp, #10]  // encoding: [0xe3,0xab,0xc0,0x1d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0xe3,0x85,0x01,0x0d]
-# CHECK:      stl1  { v3.d }[0], [x15] // encoding: [0xe3,0x85,0x01,0x0d]
-# ERROR-NO-RCPC3:  [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x87,0x01,0x4d]
-# CHECK:      stl1  { v3.d }[1], [sp]  // encoding: [0xe3,0x87,0x01,0x4d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x87,0x41,0x0d]
-# CHECK:      ldap1 { v3.d }[0], [sp]  // encoding: [0xe3,0x87,0x41,0x0d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe3,0x85,0x41,0x4d]
-# CHECK:      ldap1 { v3.d }[1], [x15] // encoding: [0xe3,0x85,0x41,0x4d]
-# ERROR-NO-RCPC3: [[@LINE-2]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.9a-specres2.txt b/llvm/test/MC/Disassembler/AArch64/armv8.9a-specres2.txt
deleted file mode 100644
index a114cd3493787..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv8.9a-specres2.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-# FEAT_SPECRES2 is optional for all v8a/v9a, mandatory for 8.9a/9.4a.
-# Should disassemble to hint #22 if the feature is not present.
-# RUN: llvm-mc -triple=aarch64 -disassemble %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v8a %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v8.9a -mattr=-specres2 %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v9.3a %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v9.4a -mattr=-specres2 %s | FileCheck %s --check-prefix=HINT_22
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+specres2 %s | FileCheck %s --check-prefix=FEAT_SPECRES2
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v8a -mattr=+specres2 %s | FileCheck %s --check-prefix=FEAT_SPECRES2
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v8.9a %s | FileCheck %s --check-prefix=FEAT_SPECRES2
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v9.3a -mattr=+specres2 %s | FileCheck %s --check-prefix=FEAT_SPECRES2
-# RUN: llvm-mc -triple=aarch64 -disassemble -mattr=+v9.4a %s | FileCheck %s --check-prefix=FEAT_SPECRES2
-
-[0xc0,0x73,0x0b,0xd5]
-# FEAT_SPECRES2: cosp rctx, x0
-# HINT_22: sys #3, c7, c3, #6, x0
diff --git a/llvm/test/MC/Disassembler/AArch64/armv8.9a-the.txt b/llvm/test/MC/Disassembler/AArch64/armv8.9a-the.txt
deleted file mode 100644
index f3b313a36487b..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv8.9a-the.txt
+++ /dev/null
@@ -1,482 +0,0 @@
-# RUN: llvm-mc -triple=aarch64                   -mattr=+the -mattr=+d128 -disassemble %s      | FileCheck %s
-# RUN: llvm-mc -triple=aarch64     -mattr=+v8.9a -mattr=+the -mattr=+d128 -disassemble %s      | FileCheck %s
-# RUN: llvm-mc -triple=aarch64     -mattr=+v9.4a -mattr=+the -mattr=+d128 -disassemble %s      | FileCheck %s
-# RUN: not llvm-mc -triple=aarch64                                          -disassemble %s 2>&1 | FileCheck %s --check-prefix=ERROR-NO-THE
-# RUN: not llvm-mc -triple=aarch64 -mattr=+v8.9a                            -disassemble %s 2>&1 | FileCheck %s --check-prefix=ERROR-NO-THE
-# RUN: not llvm-mc -triple=aarch64 -mattr=+v9.4a                            -disassemble %s 2>&1 | FileCheck %s --check-prefix=ERROR-NO-THE
-# RUN: not llvm-mc -triple=aarch64               -mattr=+the                -disassemble %s 2>&1 | FileCheck %s --check-prefix=ERROR-NO-D128
-# RUN: not llvm-mc -triple=aarch64 -mattr=+v8.9a -mattr=+the                -disassemble %s 2>&1 | FileCheck %s --check-prefix=ERROR-NO-D128
-# RUN: not llvm-mc -triple=aarch64 -mattr=+v9.4a -mattr=+the                -disassemble %s 2>&1 | FileCheck %s --check-prefix=ERROR-NO-D128
-
-[0xc3,0xd0,0x38,0xd5]
-# CHECK: mrs x3, RCWMASK_EL1
-[0xc1,0xd0,0x18,0xd5]
-# CHECK: msr RCWMASK_EL1, x1
-[0x63,0xd0,0x38,0xd5]
-# CHECK: mrs x3, RCWSMASK_EL1
-[0x61,0xd0,0x18,0xd5]
-# CHECK: msr RCWSMASK_EL1, x1
-
-[0x81,0x08,0x20,0x19]
-# CHECK: rcwcas   x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x08,0xa0,0x19]
-# CHECK: rcwcasa  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x08,0xe0,0x19]
-# CHECK: rcwcasal x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x08,0x60,0x19]
-# CHECK: rcwcasl  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x0b,0x23,0x19]
-# CHECK: rcwcas   x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x0b,0xa3,0x19]
-# CHECK: rcwcasa  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x0b,0xe3,0x19]
-# CHECK: rcwcasal x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x0b,0x63,0x19]
-# CHECK: rcwcasl  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0x81,0x08,0x20,0x59]
-# CHECK: rcwscas   x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x08,0xa0,0x59]
-# CHECK: rcwscasa  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x08,0xe0,0x59]
-# CHECK: rcwscasal x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x08,0x60,0x59]
-# CHECK: rcwscasl  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x0b,0x23,0x59]
-# CHECK: rcwscas   x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x0b,0xa3,0x59]
-# CHECK: rcwscasa  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x0b,0xe3,0x59]
-# CHECK: rcwscasal x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x0b,0x63,0x59]
-# CHECK: rcwscasl  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0x86,0x0c,0x20,0x19]
-# CHECK: rcwcasp   x0, x1, x6, x7, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x86,0x0c,0xa0,0x19]
-# CHECK: rcwcaspa  x0, x1, x6, x7, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x86,0x0c,0xe0,0x19]
-# CHECK: rcwcaspal x0, x1, x6, x7, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x86,0x0c,0x60,0x19]
-# CHECK: rcwcaspl  x0, x1, x6, x7, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe6,0x0f,0x24,0x19]
-# CHECK: rcwcasp   x4, x5, x6, x7, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe6,0x0f,0xa4,0x19]
-# CHECK: rcwcaspa  x4, x5, x6, x7, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe6,0x0f,0xe4,0x19]
-# CHECK: rcwcaspal x4, x5, x6, x7, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe6,0x0f,0x64,0x19]
-# CHECK: rcwcaspl  x4, x5, x6, x7, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-
-[0x86,0x0c,0x20,0x59]
-# CHECK: rcwscasp   x0, x1, x6, x7, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x86,0x0c,0xa0,0x59]
-# CHECK: rcwscaspa  x0, x1, x6, x7, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x86,0x0c,0xe0,0x59]
-# CHECK: rcwscaspal x0, x1, x6, x7, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x86,0x0c,0x60,0x59]
-# CHECK: rcwscaspl  x0, x1, x6, x7, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe6,0x0f,0x24,0x59]
-# CHECK: rcwscasp   x4, x5, x6, x7, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe6,0x0f,0xa4,0x59]
-# CHECK: rcwscaspa  x4, x5, x6, x7, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe6,0x0f,0xe4,0x59]
-# CHECK: rcwscaspal x4, x5, x6, x7, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe6,0x0f,0x64,0x59]
-# CHECK: rcwscaspl  x4, x5, x6, x7, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-
-[0x81,0x90,0x20,0x38]
-# CHECK: rcwclr   x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x90,0xa0,0x38]
-# CHECK: rcwclra  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x90,0xe0,0x38]
-# CHECK: rcwclral x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x90,0x60,0x38]
-# CHECK: rcwclrl  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0x23,0x38]
-# CHECK: rcwclr   x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0xa3,0x38]
-# CHECK: rcwclra  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0xe3,0x38]
-# CHECK: rcwclral x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0x63,0x38]
-# CHECK: rcwclrl  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0x81,0x90,0x20,0x78]
-# CHECK: rcwsclr   x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x90,0xa0,0x78]
-# CHECK: rcwsclra  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x90,0xe0,0x78]
-# CHECK: rcwsclral x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0x90,0x60,0x78]
-# CHECK: rcwsclrl  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0x23,0x78]
-# CHECK: rcwsclr   x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0xa3,0x78]
-# CHECK: rcwsclra  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0xe3,0x78]
-# CHECK: rcwsclral x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0x63,0x78]
-# CHECK: rcwsclrl  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0x81,0x90,0x20,0x19]
-# CHECK: rcwclrp   x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0x90,0xa0,0x19]
-# CHECK: rcwclrpa  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0x90,0xe0,0x19]
-# CHECK: rcwclrpal x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0x90,0x60,0x19]
-# CHECK: rcwclrpl  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0x23,0x19]
-# CHECK: rcwclrp   x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0xa3,0x19]
-# CHECK: rcwclrpa  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0xe3,0x19]
-# CHECK: rcwclrpal x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0x63,0x19]
-# CHECK: rcwclrpl  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-
-[0x81,0x90,0x20,0x59]
-# CHECK: rcwsclrp   x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0x90,0xa0,0x59]
-# CHECK: rcwsclrpa  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0x90,0xe0,0x59]
-# CHECK: rcwsclrpal x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0x90,0x60,0x59]
-# CHECK: rcwsclrpl  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0x23,0x59]
-# CHECK: rcwsclrp   x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0xa3,0x59]
-# CHECK: rcwsclrpa  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0xe3,0x59]
-# CHECK: rcwsclrpal x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0x93,0x63,0x59]
-# CHECK: rcwsclrpl  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-
-[0x81,0xb0,0x20,0x38]
-# CHECK: rcwset   x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0xa0,0x38]
-# CHECK: rcwseta  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0xe0,0x38]
-# CHECK: rcwsetal x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0x60,0x38]
-# CHECK: rcwsetl  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0x23,0x38]
-# CHECK: rcwset   x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0xa3,0x38]
-# CHECK: rcwseta  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0xe3,0x38]
-# CHECK: rcwsetal x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0x63,0x38]
-# CHECK: rcwsetl  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0x81,0xb0,0x20,0x78]
-# CHECK: rcwsset   x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0xa0,0x78]
-# CHECK: rcwsseta  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0xe0,0x78]
-# CHECK: rcwssetal x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0x60,0x78]
-# CHECK: rcwssetl  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0x23,0x78]
-# CHECK: rcwsset   x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0xa3,0x78]
-# CHECK: rcwsseta  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0xe3,0x78]
-# CHECK: rcwssetal x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0x63,0x78]
-# CHECK: rcwssetl  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0x81,0xb0,0x20,0x19]
-# CHECK: rcwsetp   x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0xa0,0x19]
-# CHECK: rcwsetpa  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0xe0,0x19]
-# CHECK: rcwsetpal x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0x60,0x19]
-# CHECK: rcwsetpl  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0x23,0x19]
-# CHECK: rcwsetp   x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0xa3,0x19]
-# CHECK: rcwsetpa  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0xe3,0x19]
-# CHECK: rcwsetpal x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0x63,0x19]
-# CHECK: rcwsetpl  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-
-[0x81,0xb0,0x20,0x59]
-# CHECK: rcwssetp   x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0xa0,0x59]
-# CHECK: rcwssetpa  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0xe0,0x59]
-# CHECK: rcwssetpal x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xb0,0x60,0x59]
-# CHECK: rcwssetpl  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0x23,0x59]
-# CHECK: rcwssetp   x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0xa3,0x59]
-# CHECK: rcwssetpa  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0xe3,0x59]
-# CHECK: rcwssetpal x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xb3,0x63,0x59]
-# CHECK: rcwssetpl  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-
-[0x81,0xa0,0x20,0x38]
-# CHECK: rcwswp   x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0xa0,0x38]
-# CHECK: rcwswpa  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0xe0,0x38]
-# CHECK: rcwswpal x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0x60,0x38]
-# CHECK: rcwswpl  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0x23,0x38]
-# CHECK: rcwswp   x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0xa3,0x38]
-# CHECK: rcwswpa  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0xe3,0x38]
-# CHECK: rcwswpal x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0x63,0x38]
-# CHECK: rcwswpl  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0x81,0xa0,0x20,0x78]
-# CHECK: rcwsswp   x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0xa0,0x78]
-# CHECK: rcwsswpa  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0xe0,0x78]
-# CHECK: rcwsswpal x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0x60,0x78]
-# CHECK: rcwsswpl  x0, x1, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0x23,0x78]
-# CHECK: rcwsswp   x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0xa3,0x78]
-# CHECK: rcwsswpa  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0xe3,0x78]
-# CHECK: rcwsswpal x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0x63,0x78]
-# CHECK: rcwsswpl  x3, x5, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-
-[0x81,0xa0,0x20,0x19]
-# CHECK: rcwswpp   x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0xa0,0x19]
-# CHECK: rcwswppa  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0xe0,0x19]
-# CHECK: rcwswppal x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0x60,0x19]
-# CHECK: rcwswppl  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0x23,0x19]
-# CHECK: rcwswpp   x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0xa3,0x19]
-# CHECK: rcwswppa  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0xe3,0x19]
-# CHECK: rcwswppal x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0x63,0x19]
-# CHECK: rcwswppl  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-
-[0x81,0xa0,0x20,0x59]
-# CHECK: rcwsswpp   x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0xa0,0x59]
-# CHECK: rcwsswppa  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0xe0,0x59]
-# CHECK: rcwsswppal x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0x81,0xa0,0x60,0x59]
-# CHECK: rcwsswppl  x1, x0, [x4]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0x23,0x59]
-# CHECK: rcwsswpp   x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0xa3,0x59]
-# CHECK: rcwsswppa  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0xe3,0x59]
-# CHECK: rcwsswppal x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
-[0xe5,0xa3,0x63,0x59]
-# CHECK: rcwsswppl  x5, x3, [sp]
-# ERROR-NO-THE: [[@LINE-2]]:2: warning: invalid instruction encoding
-# ERROR-NO-D128: [[@LINE-3]]:2: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9-sysp.txt b/llvm/test/MC/Disassembler/AArch64/armv9-sysp.txt
deleted file mode 100644
index 2bbdef4d5518e..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9-sysp.txt
+++ /dev/null
@@ -1,562 +0,0 @@
-# RUN: llvm-mc -triple aarch64 --disassemble -mattr=+d128,+tlb-rmi,+xs %s -o - 2> %t | FileCheck %s
-# RUN: FileCheck %s --check-prefix=INVALID --input-file=%t
-
-# RUN: llvm-mc -triple aarch64 --disassemble %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR-NO-D128
-
-0x00 0x20 0x48 0xd5
-0x20 0x20 0x48 0xd5
-0x00 0x74 0x48 0xd5
-0x60 0xd0 0x48 0xd5
-0xc0 0xd0 0x48 0xd5
-0x00 0x20 0x4c 0xd5
-0x20 0x20 0x4c 0xd5
-0x00 0x21 0x4c 0xd5
-0x00 0x20 0x48 0xd5
-0x20 0x20 0x48 0xd5
-0x00 0x74 0x48 0xd5
-0x60 0xd0 0x48 0xd5
-0xc0 0xd0 0x48 0xd5
-0x00 0x20 0x4c 0xd5
-0x20 0x20 0x4c 0xd5
-0x00 0x21 0x4c 0xd5
-0x00 0x20 0x48 0xd5
-0x02 0x20 0x48 0xd5
-0x04 0x20 0x48 0xd5
-0x06 0x20 0x48 0xd5
-0x08 0x20 0x48 0xd5
-0x0a 0x20 0x48 0xd5
-0x0c 0x20 0x48 0xd5
-0x0e 0x20 0x48 0xd5
-0x10 0x20 0x48 0xd5
-0x12 0x20 0x48 0xd5
-0x14 0x20 0x48 0xd5
-0x16 0x20 0x48 0xd5
-0x18 0x20 0x48 0xd5
-0x1a 0x20 0x48 0xd5
-0x1c 0x20 0x48 0xd5
-0x1e 0x20 0x48 0xd5
-0x1f 0x20 0x48 0xd5
-
-# CHECK: sysp #0, c2, c0, #0, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #1, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c7, c4, #0, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c13, c0, #3, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c13, c0, #6, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #4, c2, c0, #0, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #4, c2, c0, #1, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #4, c2, c1, #0, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #1, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c7, c4, #0, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c13, c0, #3, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c13, c0, #6, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #4, c2, c0, #0, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #4, c2, c0, #1, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #4, c2, c1, #0, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x0, x1
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x2, x3
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x6, x7
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x12, x13
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x16, x17
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x22, x23
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x26, x27
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0, x30, xzr
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK-NEXT: sysp #0, c2, c0, #0
-# ERROR-NO-D128: warning: invalid instruction encoding
-
-
-
-0x01 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x1, x2
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x01 0x20 0x48 0xd5
-
-0x03 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x3, x4
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x03 0x20 0x48 0xd5
-
-0x05 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x5, x6
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x05 0x20 0x48 0xd5
-
-0x07 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x7, x8
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x07 0x20 0x48 0xd5
-
-0x09 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x9, x10
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x09 0x20 0x48 0xd5
-
-0x0b 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x11, x12
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x0b 0x20 0x48 0xd5
-
-0x0d 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x13, x14
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x0d 0x20 0x48 0xd5
-
-0x0f 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x15, x16
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x0f 0x20 0x48 0xd5
-
-0x11 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x17, x18
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x11 0x20 0x48 0xd5
-
-0x13 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x19, x20
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x13 0x20 0x48 0xd5
-
-0x15 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x21, x22
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x15 0x20 0x48 0xd5
-
-0x17 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x23, x24
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x17 0x20 0x48 0xd5
-
-0x19 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x25, x26
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x19 0x20 0x48 0xd5
-
-0x1b 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x27, x28
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x1b 0x20 0x48 0xd5
-
-0x1d 0x20 0x48 0xd5 # sysp #0, c2, c0, #0, x29, x30
-# INVALID: warning: invalid instruction encoding
-# INVALID-NEXT: 0x1d 0x20 0x48 0xd5
-
-0x24 0x80 0x4c 0xd5
-# CHECK: tlbip   ipas2e1is, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x24 0x90 0x4c 0xd5
-# CHECK: tlbip   ipas2e1isnxs, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x24 0x84 0x4c 0xd5
-# CHECK: tlbip   ipas2e1, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x24 0x94 0x4c 0xd5
-# CHECK: tlbip   ipas2e1nxs, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x04 0x84 0x4c 0xd5
-# CHECK: tlbip   ipas2e1os, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x04 0x94 0x4c 0xd5
-# CHECK: tlbip   ipas2e1osnxs, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa4 0x84 0x4c 0xd5
-# CHECK: tlbip   ipas2le1, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa4 0x94 0x4c 0xd5
-# CHECK: tlbip   ipas2le1nxs, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa4 0x80 0x4c 0xd5
-# CHECK: tlbip   ipas2le1is, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa4 0x90 0x4c 0xd5
-# CHECK: tlbip   ipas2le1isnxs, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x84 0x84 0x4c 0xd5
-# CHECK: tlbip   ipas2le1os, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x84 0x94 0x4c 0xd5
-# CHECK: tlbip   ipas2le1osnxs, x4, x5
-# ERROR-NO-D128: warning: invalid instruction encoding
-
-
-0x28 0x83 0x48 0xd5
-# CHECK: tlbip   vae1is, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x28 0x93 0x48 0xd5
-# CHECK: tlbip   vae1isnxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x28 0x87 0x48 0xd5
-# CHECK: tlbip   vae1, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x28 0x97 0x48 0xd5
-# CHECK: tlbip   vae1nxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x28 0x83 0x48 0xd5
-# CHECK: tlbip   vae1is, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x28 0x93 0x48 0xd5
-# CHECK: tlbip   vae1isnxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x28 0x81 0x48 0xd5
-# CHECK: tlbip   vae1os, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x28 0x91 0x48 0xd5
-# CHECK: tlbip   vae1osnxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa8 0x87 0x48 0xd5
-# CHECK: tlbip   vale1, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa8 0x97 0x48 0xd5
-# CHECK: tlbip   vale1nxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa8 0x83 0x48 0xd5
-# CHECK: tlbip   vale1is, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa8 0x93 0x48 0xd5
-# CHECK: tlbip   vale1isnxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa8 0x81 0x48 0xd5
-# CHECK: tlbip   vale1os, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xa8 0x91 0x48 0xd5
-# CHECK: tlbip   vale1osnxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x68 0x87 0x48 0xd5
-# CHECK: tlbip   vaae1, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x68 0x97 0x48 0xd5
-# CHECK: tlbip   vaae1nxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x68 0x83 0x48 0xd5
-# CHECK: tlbip   vaae1is, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x68 0x93 0x48 0xd5
-# CHECK: tlbip   vaae1isnxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x68 0x81 0x48 0xd5
-# CHECK: tlbip   vaae1os, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x68 0x91 0x48 0xd5
-# CHECK: tlbip   vaae1osnxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xe8 0x87 0x48 0xd5
-# CHECK: tlbip   vaale1, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xe8 0x97 0x48 0xd5
-# CHECK: tlbip   vaale1nxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xe8 0x83 0x48 0xd5
-# CHECK: tlbip   vaale1is, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xe8 0x93 0x48 0xd5
-# CHECK: tlbip   vaale1isnxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xe8 0x81 0x48 0xd5
-# CHECK: tlbip   vaale1os, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xe8 0x91 0x48 0xd5
-# CHECK: tlbip   vaale1osnxs, x8, x9
-# ERROR-NO-D128: warning: invalid instruction encoding
-
-0x2e 0x87 0x4c 0xd5
-# CHECK: tlbip   vae2, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2e 0x97 0x4c 0xd5
-# CHECK: tlbip   vae2nxs, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2e 0x83 0x4c 0xd5
-# CHECK: tlbip   vae2is, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2e 0x93 0x4c 0xd5
-# CHECK: tlbip   vae2isnxs, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2e 0x81 0x4c 0xd5
-# CHECK: tlbip   vae2os, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2e 0x91 0x4c 0xd5
-# CHECK: tlbip   vae2osnxs, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xae 0x87 0x4c 0xd5
-# CHECK: tlbip   vale2, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xae 0x97 0x4c 0xd5
-# CHECK: tlbip   vale2nxs, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xae 0x83 0x4c 0xd5
-# CHECK: tlbip   vale2is, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xae 0x93 0x4c 0xd5
-# CHECK: tlbip   vale2isnxs, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xae 0x81 0x4c 0xd5
-# CHECK: tlbip   vale2os, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xae 0x91 0x4c 0xd5
-# CHECK: tlbip   vale2osnxs, x14, x15
-# ERROR-NO-D128: warning: invalid instruction encoding
-
-0x38 0x87 0x4e 0xd5
-# CHECK: tlbip   vae3, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x38 0x97 0x4e 0xd5
-# CHECK: tlbip   vae3nxs, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x38 0x83 0x4e 0xd5
-# CHECK: tlbip   vae3is, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x38 0x93 0x4e 0xd5
-# CHECK: tlbip   vae3isnxs, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x38 0x81 0x4e 0xd5
-# CHECK: tlbip   vae3os, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x38 0x91 0x4e 0xd5
-# CHECK: tlbip   vae3osnxs, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb8 0x87 0x4e 0xd5
-# CHECK: tlbip   vale3, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb8 0x97 0x4e 0xd5
-# CHECK: tlbip   vale3nxs, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb8 0x83 0x4e 0xd5
-# CHECK: tlbip   vale3is, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb8 0x93 0x4e 0xd5
-# CHECK: tlbip   vale3isnxs, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb8 0x81 0x4e 0xd5
-# CHECK: tlbip   vale3os, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb8 0x91 0x4e 0xd5
-# CHECK: tlbip   vale3osnxs, x24, x25
-# ERROR-NO-D128: warning: invalid instruction encoding
-
-
-0x32 0x86 0x48 0xd5
-# CHECK: tlbip   rvae1, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x32 0x96 0x48 0xd5
-# CHECK: tlbip   rvae1nxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x32 0x86 0x48 0xd5
-# CHECK: tlbip   rvae1, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x32 0x96 0x48 0xd5
-# CHECK: tlbip   rvae1nxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x32 0x82 0x48 0xd5
-# CHECK: tlbip   rvae1is, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x32 0x92 0x48 0xd5
-# CHECK: tlbip   rvae1isnxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x32 0x85 0x48 0xd5
-# CHECK: tlbip   rvae1os, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x32 0x95 0x48 0xd5
-# CHECK: tlbip   rvae1osnxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x72 0x86 0x48 0xd5
-# CHECK: tlbip   rvaae1, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x72 0x96 0x48 0xd5
-# CHECK: tlbip   rvaae1nxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x72 0x82 0x48 0xd5
-# CHECK: tlbip   rvaae1is, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x72 0x92 0x48 0xd5
-# CHECK: tlbip   rvaae1isnxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x72 0x85 0x48 0xd5
-# CHECK: tlbip   rvaae1os, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x72 0x95 0x48 0xd5
-# CHECK: tlbip   rvaae1osnxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb2 0x86 0x48 0xd5
-# CHECK: tlbip   rvale1, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb2 0x96 0x48 0xd5
-# CHECK: tlbip   rvale1nxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb2 0x82 0x48 0xd5
-# CHECK: tlbip   rvale1is, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb2 0x92 0x48 0xd5
-# CHECK: tlbip   rvale1isnxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb2 0x85 0x48 0xd5
-# CHECK: tlbip   rvale1os, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xb2 0x95 0x48 0xd5
-# CHECK: tlbip   rvale1osnxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xf2 0x86 0x48 0xd5
-# CHECK: tlbip   rvaale1, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xf2 0x96 0x48 0xd5
-# CHECK: tlbip   rvaale1nxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xf2 0x82 0x48 0xd5
-# CHECK: tlbip   rvaale1is, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xf2 0x92 0x48 0xd5
-# CHECK: tlbip   rvaale1isnxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xf2 0x85 0x48 0xd5
-# CHECK: tlbip   rvaale1os, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xf2 0x95 0x48 0xd5
-# CHECK: tlbip   rvaale1osnxs, x18, x19
-# ERROR-NO-D128: warning: invalid instruction encoding
-
-0x3c 0x86 0x4c 0xd5
-# CHECK: tlbip   rvae2, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x3c 0x96 0x4c 0xd5
-# CHECK: tlbip   rvae2nxs, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x3c 0x82 0x4c 0xd5
-# CHECK: tlbip   rvae2is, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x3c 0x92 0x4c 0xd5
-# CHECK: tlbip   rvae2isnxs, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x3c 0x85 0x4c 0xd5
-# CHECK: tlbip   rvae2os, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x3c 0x95 0x4c 0xd5
-# CHECK: tlbip   rvae2osnxs, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xbc 0x86 0x4c 0xd5
-# CHECK: tlbip   rvale2, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xbc 0x96 0x4c 0xd5
-# CHECK: tlbip   rvale2nxs, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xbc 0x82 0x4c 0xd5
-# CHECK: tlbip   rvale2is, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xbc 0x92 0x4c 0xd5
-# CHECK: tlbip   rvale2isnxs, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xbc 0x85 0x4c 0xd5
-# CHECK: tlbip   rvale2os, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xbc 0x95 0x4c 0xd5
-# CHECK: tlbip   rvale2osnxs, x28, x29
-# ERROR-NO-D128: warning: invalid instruction encoding
-
-0x2a 0x86 0x4e 0xd5
-# CHECK: tlbip   rvae3, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2a 0x96 0x4e 0xd5
-# CHECK: tlbip   rvae3nxs, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2a 0x82 0x4e 0xd5
-# CHECK: tlbip   rvae3is, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2a 0x92 0x4e 0xd5
-# CHECK: tlbip   rvae3isnxs, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2a 0x85 0x4e 0xd5
-# CHECK: tlbip   rvae3os, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x2a 0x95 0x4e 0xd5
-# CHECK: tlbip   rvae3osnxs, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xaa 0x86 0x4e 0xd5
-# CHECK: tlbip   rvale3, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xaa 0x96 0x4e 0xd5
-# CHECK: tlbip   rvale3nxs, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xaa 0x82 0x4e 0xd5
-# CHECK: tlbip   rvale3is, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xaa 0x92 0x4e 0xd5
-# CHECK: tlbip   rvale3isnxs, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xaa 0x85 0x4e 0xd5
-# CHECK: tlbip   rvale3os, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xaa 0x95 0x4e 0xd5
-# CHECK: tlbip   rvale3osnxs, x10, x11
-# ERROR-NO-D128: warning: invalid instruction encoding
-
-
-0x54 0x80 0x4c 0xd5
-# CHECK: tlbip   ripas2e1is, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x54 0x90 0x4c 0xd5
-# CHECK: tlbip   ripas2e1isnxs, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x54 0x84 0x4c 0xd5
-# CHECK: tlbip   ripas2e1, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x54 0x94 0x4c 0xd5
-# CHECK: tlbip   ripas2e1nxs, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x54 0x80 0x4c 0xd5
-# CHECK: tlbip   ripas2e1is, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x54 0x90 0x4c 0xd5
-# CHECK: tlbip   ripas2e1isnxs, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x74 0x84 0x4c 0xd5
-# CHECK: tlbip   ripas2e1os, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0x74 0x94 0x4c 0xd5
-# CHECK: tlbip   ripas2e1osnxs, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xd4 0x84 0x4c 0xd5
-# CHECK: tlbip   ripas2le1, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xd4 0x94 0x4c 0xd5
-# CHECK: tlbip   ripas2le1nxs, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xd4 0x80 0x4c 0xd5
-# CHECK: tlbip   ripas2le1is, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xd4 0x90 0x4c 0xd5
-# CHECK: tlbip   ripas2le1isnxs, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xf4 0x84 0x4c 0xd5
-# CHECK: tlbip   ripas2le1os, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-0xf4 0x94 0x4c 0xd5
-# CHECK: tlbip   ripas2le1osnxs, x20, x21
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK: tlbip ripas2le1os, xzr, xzr
-0xff 0x84 0x4c 0xd5
-# ERROR-NO-D128: warning: invalid instruction encoding
-# CHECK: tlbip ripas2le1osnxs, xzr, xzr
-0xff 0x94 0x4c 0xd5
-# ERROR-NO-D128: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9-sysreg128.txt b/llvm/test/MC/Disassembler/AArch64/armv9-sysreg128.txt
deleted file mode 100644
index 4ab37a0200738..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9-sysreg128.txt
+++ /dev/null
@@ -1,147 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -mattr=+d128                   --disassemble -show-encoding %s -o - | FileCheck %s --check-prefix=WITHOUT
-# RUN: llvm-mc -triple aarch64 -mattr=+d128,+the,+el2vmsa,+vh --disassemble -show-encoding %s -o - | FileCheck %s --check-prefix=W_FEATS
-
-# RUN: llvm-mc -triple aarch64 --disassemble -show-encoding %s -o - 2>&1 | FileCheck %s --check-prefix=ERROR-NO-D128
-
-0x00,0x20,0x78,0xd5
-0x20,0x20,0x78,0xd5
-0x00,0x74,0x78,0xd5
-0x60,0xd0,0x78,0xd5
-0xc0,0xd0,0x78,0xd5
-0x00,0x20,0x7c,0xd5
-0x20,0x20,0x7c,0xd5
-0x00,0x21,0x7c,0xd5
-0x00,0x21,0x7c,0xd5
-0x02,0x21,0x7c,0xd5
-0x04,0x21,0x7c,0xd5
-0x06,0x21,0x7c,0xd5
-0x08,0x21,0x7c,0xd5
-0x0a,0x21,0x7c,0xd5
-0x0c,0x21,0x7c,0xd5
-0x0e,0x21,0x7c,0xd5
-0x10,0x21,0x7c,0xd5
-0x12,0x21,0x7c,0xd5
-0x14,0x21,0x7c,0xd5
-0x16,0x21,0x7c,0xd5
-0x18,0x21,0x7c,0xd5
-0x1a,0x21,0x7c,0xd5
-
-# WITHOUT: mrrs	x0, x1, TTBR0_EL1               // encoding: [0x00,0x20,0x78,0xd5]
-# WITHOUT: mrrs	x0, x1, TTBR1_EL1               // encoding: [0x20,0x20,0x78,0xd5]
-# WITHOUT: mrrs	x0, x1, PAR_EL1                 // encoding: [0x00,0x74,0x78,0xd5]
-# WITHOUT: mrrs	x0, x1, S3_0_C13_C0_3           // encoding: [0x60,0xd0,0x78,0xd5]
-# WITHOUT: mrrs	x0, x1, S3_0_C13_C0_6           // encoding: [0xc0,0xd0,0x78,0xd5]
-# WITHOUT: mrrs	x0, x1, S3_4_C2_C0_0            // encoding: [0x00,0x20,0x7c,0xd5]
-# WITHOUT: mrrs	x0, x1, S3_4_C2_C0_1            // encoding: [0x20,0x20,0x7c,0xd5]
-# WITHOUT: mrrs	x0, x1, S3_4_C2_C1_0            // encoding: [0x00,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x0, x1, S3_4_C2_C1_0            // encoding: [0x00,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x2, x3, S3_4_C2_C1_0            // encoding: [0x02,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x4, x5, S3_4_C2_C1_0            // encoding: [0x04,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x6, x7, S3_4_C2_C1_0            // encoding: [0x06,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x8, x9, S3_4_C2_C1_0            // encoding: [0x08,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x10, x11, S3_4_C2_C1_0          // encoding: [0x0a,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x12, x13, S3_4_C2_C1_0          // encoding: [0x0c,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x14, x15, S3_4_C2_C1_0          // encoding: [0x0e,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x16, x17, S3_4_C2_C1_0          // encoding: [0x10,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x18, x19, S3_4_C2_C1_0          // encoding: [0x12,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x20, x21, S3_4_C2_C1_0          // encoding: [0x14,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x22, x23, S3_4_C2_C1_0          // encoding: [0x16,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x24, x25, S3_4_C2_C1_0          // encoding: [0x18,0x21,0x7c,0xd5]
-# WITHOUT: mrrs	x26, x27, S3_4_C2_C1_0          // encoding: [0x1a,0x21,0x7c,0xd5]
-
-# W_FEATS: mrrs	x0, x1, TTBR0_EL1               // encoding: [0x00,0x20,0x78,0xd5]
-# W_FEATS: mrrs	x0, x1, TTBR1_EL1               // encoding: [0x20,0x20,0x78,0xd5]
-# W_FEATS: mrrs	x0, x1, PAR_EL1                 // encoding: [0x00,0x74,0x78,0xd5]
-# W_FEATS: mrrs	x0, x1, RCWSMASK_EL1            // encoding: [0x60,0xd0,0x78,0xd5]
-# W_FEATS: mrrs	x0, x1, RCWMASK_EL1             // encoding: [0xc0,0xd0,0x78,0xd5]
-# W_FEATS: mrrs	x0, x1, TTBR0_EL2               // encoding: [0x00,0x20,0x7c,0xd5]
-# W_FEATS: mrrs	x0, x1, TTBR1_EL2               // encoding: [0x20,0x20,0x7c,0xd5]
-# W_FEATS: mrrs	x0, x1, VTTBR_EL2               // encoding: [0x00,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x0, x1, VTTBR_EL2               // encoding: [0x00,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x2, x3, VTTBR_EL2               // encoding: [0x02,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x4, x5, VTTBR_EL2               // encoding: [0x04,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x6, x7, VTTBR_EL2               // encoding: [0x06,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x8, x9, VTTBR_EL2               // encoding: [0x08,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x10, x11, VTTBR_EL2             // encoding: [0x0a,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x12, x13, VTTBR_EL2             // encoding: [0x0c,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x14, x15, VTTBR_EL2             // encoding: [0x0e,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x16, x17, VTTBR_EL2             // encoding: [0x10,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x18, x19, VTTBR_EL2             // encoding: [0x12,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x20, x21, VTTBR_EL2             // encoding: [0x14,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x22, x23, VTTBR_EL2             // encoding: [0x16,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x24, x25, VTTBR_EL2             // encoding: [0x18,0x21,0x7c,0xd5]
-# W_FEATS: mrrs	x26, x27, VTTBR_EL2             // encoding: [0x1a,0x21,0x7c,0xd5]
-
-# ERROR-NO-D128: warning: invalid instruction encoding
-
-
-0x00,0x20,0x58,0xd5
-0x20,0x20,0x58,0xd5
-0x00,0x74,0x58,0xd5
-0x60,0xd0,0x58,0xd5
-0xc0,0xd0,0x58,0xd5
-0x00,0x20,0x5c,0xd5
-0x20,0x20,0x5c,0xd5
-0x00,0x21,0x5c,0xd5
-0x00,0x21,0x5c,0xd5
-0x02,0x21,0x5c,0xd5
-0x04,0x21,0x5c,0xd5
-0x06,0x21,0x5c,0xd5
-0x08,0x21,0x5c,0xd5
-0x0a,0x21,0x5c,0xd5
-0x0c,0x21,0x5c,0xd5
-0x0e,0x21,0x5c,0xd5
-0x10,0x21,0x5c,0xd5
-0x12,0x21,0x5c,0xd5
-0x14,0x21,0x5c,0xd5
-0x16,0x21,0x5c,0xd5
-0x18,0x21,0x5c,0xd5
-0x1a,0x21,0x5c,0xd5
-
-# WITHOUT: msrr	TTBR0_EL1, x0, x1               // encoding: [0x00,0x20,0x58,0xd5]
-# WITHOUT: msrr	TTBR1_EL1, x0, x1               // encoding: [0x20,0x20,0x58,0xd5]
-# WITHOUT: msrr	PAR_EL1, x0, x1                 // encoding: [0x00,0x74,0x58,0xd5]
-# WITHOUT: msrr	S3_0_C13_C0_3, x0, x1           // encoding: [0x60,0xd0,0x58,0xd5]
-# WITHOUT: msrr	S3_0_C13_C0_6, x0, x1           // encoding: [0xc0,0xd0,0x58,0xd5]
-# WITHOUT: msrr	S3_4_C2_C0_0, x0, x1            // encoding: [0x00,0x20,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C0_1, x0, x1            // encoding: [0x20,0x20,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x0, x1            // encoding: [0x00,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x0, x1            // encoding: [0x00,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x2, x3            // encoding: [0x02,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x4, x5            // encoding: [0x04,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x6, x7            // encoding: [0x06,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x8, x9            // encoding: [0x08,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x10, x11          // encoding: [0x0a,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x12, x13          // encoding: [0x0c,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x14, x15          // encoding: [0x0e,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x16, x17          // encoding: [0x10,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x18, x19          // encoding: [0x12,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x20, x21          // encoding: [0x14,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x22, x23          // encoding: [0x16,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x24, x25          // encoding: [0x18,0x21,0x5c,0xd5]
-# WITHOUT: msrr	S3_4_C2_C1_0, x26, x27          // encoding: [0x1a,0x21,0x5c,0xd5]
-
-# W_FEATS: msrr	TTBR0_EL1, x0, x1               // encoding: [0x00,0x20,0x58,0xd5]
-# W_FEATS: msrr	TTBR1_EL1, x0, x1               // encoding: [0x20,0x20,0x58,0xd5]
-# W_FEATS: msrr	PAR_EL1, x0, x1                 // encoding: [0x00,0x74,0x58,0xd5]
-# W_FEATS: msrr	RCWSMASK_EL1, x0, x1            // encoding: [0x60,0xd0,0x58,0xd5]
-# W_FEATS: msrr	RCWMASK_EL1, x0, x1             // encoding: [0xc0,0xd0,0x58,0xd5]
-# W_FEATS: msrr	TTBR0_EL2, x0, x1               // encoding: [0x00,0x20,0x5c,0xd5]
-# W_FEATS: msrr	TTBR1_EL2, x0, x1               // encoding: [0x20,0x20,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x0, x1               // encoding: [0x00,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x0, x1               // encoding: [0x00,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x2, x3               // encoding: [0x02,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x4, x5               // encoding: [0x04,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x6, x7               // encoding: [0x06,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x8, x9               // encoding: [0x08,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x10, x11             // encoding: [0x0a,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x12, x13             // encoding: [0x0c,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x14, x15             // encoding: [0x0e,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x16, x17             // encoding: [0x10,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x18, x19             // encoding: [0x12,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x20, x21             // encoding: [0x14,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x22, x23             // encoding: [0x16,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x24, x25             // encoding: [0x18,0x21,0x5c,0xd5]
-# W_FEATS: msrr	VTTBR_EL2, x26, x27             // encoding: [0x1a,0x21,0x5c,0xd5]
-
-# ERROR-NO-D128: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.4a-chk.txt b/llvm/test/MC/Disassembler/AArch64/armv9.4a-chk.txt
deleted file mode 100644
index 730f4440402e1..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.4a-chk.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# RUN: llvm-mc -triple=aarch64 -mattr=+v8.9a -disassemble %s 2> %t | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=+v9.4a -disassemble %s 2> %t | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=+chk   -disassemble %s 2> %t | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=+v8a   -disassemble %s 2> %t | FileCheck %s --check-prefix=NO-CHK
-
-[0x1f,0x25,0x03,0xd5]
-// CHECK: chkfeat x16
-// NO-CHK: hint #40
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.4a-ebep.txt b/llvm/test/MC/Disassembler/AArch64/armv9.4a-ebep.txt
deleted file mode 100644
index aa9c95fb2082b..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.4a-ebep.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-# RUN: llvm-mc -triple=aarch64 -disassemble %s | FileCheck %s
-
-[0x23,0x43,0x38,0xd5]
-# CHECK:  mrs x3, PM
-
-[0x26,0x43,0x18,0xd5]
-# CHECK:  msr PM, x6
-
-[0x1f,0x42,0x01,0xd5]
-# CHECK:  msr PM, #0
-
-[0x1f,0x43,0x01,0xd5]
-# CHECK:  msr PM, #1
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.4a-gcs.txt b/llvm/test/MC/Disassembler/AArch64/armv9.4a-gcs.txt
deleted file mode 100644
index 512f4027d9761..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.4a-gcs.txt
+++ /dev/null
@@ -1,90 +0,0 @@
-# RUN: llvm-mc -triple=aarch64 -mattr +gcs -disassemble %s 2> %t | FileCheck %s
-
-[0x00,0x25,0x18,0xd5]
-[0x01,0x25,0x38,0xd5]
-// CHECK: msr     GCSCR_EL1, x0
-// CHECK: mrs     x1, GCSCR_EL1
-
-[0x22,0x25,0x18,0xd5]
-[0x23,0x25,0x38,0xd5]
-// CHECK: msr     GCSPR_EL1, x2
-// CHECK: mrs     x3, GCSPR_EL1
-
-[0x44,0x25,0x18,0xd5]
-[0x45,0x25,0x38,0xd5]
-// CHECK: msr     GCSCRE0_EL1, x4
-// CHECK: mrs     x5, GCSCRE0_EL1
-
-[0x26,0x25,0x1b,0xd5]
-[0x27,0x25,0x3b,0xd5]
-// CHECK: msr     GCSPR_EL0, x6
-// CHECK: mrs     x7, GCSPR_EL0
-
-[0x0a,0x25,0x1c,0xd5]
-[0x0b,0x25,0x3c,0xd5]
-// CHECK: msr     GCSCR_EL2, x10
-// CHECK: mrs     x11, GCSCR_EL2
-
-[0x2c,0x25,0x1c,0xd5]
-[0x2d,0x25,0x3c,0xd5]
-// CHECK: msr     GCSPR_EL2, x12
-// CHECK: mrs     x13, GCSPR_EL2
-
-[0x0e,0x25,0x1d,0xd5]
-[0x0f,0x25,0x3d,0xd5]
-// CHECK: msr     GCSCR_EL12, x14
-// CHECK: mrs     x15, GCSCR_EL12
-
-[0x30,0x25,0x1d,0xd5]
-[0x31,0x25,0x3d,0xd5]
-// CHECK: msr     GCSPR_EL12, x16
-// CHECK: mrs     x17, GCSPR_EL12
-
-[0x12,0x25,0x1e,0xd5]
-[0x13,0x25,0x3e,0xd5]
-// CHECK: msr     GCSCR_EL3, x18
-// CHECK: mrs     x19, GCSCR_EL3
-
-[0x34,0x25,0x1e,0xd5]
-[0x35,0x25,0x3e,0xd5]
-// CHECK: msr     GCSPR_EL3, x20
-// CHECK: mrs     x21, GCSPR_EL3
-
-[0x55,0x77,0x0b,0xd5]
-// CHECK: gcsss1 x21
-
-[0x76,0x77,0x2b,0xd5]
-// CHECK: gcsss2    x22
-
-[0x19,0x77,0x0b,0xd5]
-// CHECK: gcspushm x25
-
-[0x3f,0x77,0x2b,0xd5]
-// CHECK: gcspopm
-
-[0x39,0x77,0x2b,0xd5]
-// CHECK: gcspopm    x25
-
-[0x7f,0x22,0x03,0xd5]
-// CHECK: gcsb    dsync
-
-[0x7a,0x0f,0x1f,0xd9]
-// CHECK: gcsstr   x26, [x27]
-
-[0xfa,0x0f,0x1f,0xd9]
-// CHECK: gcsstr   x26, [sp]
-
-[0x7a,0x1f,0x1f,0xd9]
-// CHECK: gcssttr  x26, [x27]
-
-[0xfa,0x1f,0x1f,0xd9]
-// CHECK: gcssttr  x26, [sp]
-
-[0x9f,0x77,0x08,0xd5]
-// CHECK: gcspushx
-
-[0xbf,0x77,0x08,0xd5]
-// CHECK: gcspopcx
-
-[0xdf,0x77,0x08,0xd5]
-// CHECK: gcspopx
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.4a-lse128.txt b/llvm/test/MC/Disassembler/AArch64/armv9.4a-lse128.txt
deleted file mode 100644
index d4dffa0b3a9b6..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.4a-lse128.txt
+++ /dev/null
@@ -1,98 +0,0 @@
-# RUN: llvm-mc -triple=aarch64 -mattr=+lse128 -disassemble %s | FileCheck %s
-# RUN: not llvm-mc -triple=aarch64 -disassemble %s 2>&1 | FileCheck --check-prefix=NO-LSE128 %s
-
-[0x61,0x11,0x22,0x19]
-# CHECK: ldclrp x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x13,0x36,0x19]
-# CHECK: ldclrp x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x11,0xa2,0x19]
-# CHECK: ldclrpa x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x13,0xb6,0x19]
-# CHECK: ldclrpa x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x11,0xe2,0x19]
-# CHECK: ldclrpal x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x13,0xf6,0x19]
-# CHECK: ldclrpal x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x11,0x62,0x19]
-# CHECK: ldclrpl x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x13,0x76,0x19]
-# CHECK: ldclrpl x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x31,0x22,0x19]
-# CHECK: ldsetp x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x33,0x36,0x19]
-# CHECK: ldsetp x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x31,0xa2,0x19]
-# CHECK: ldsetpa x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x33,0xb6,0x19]
-# CHECK: ldsetpa x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x31,0xe2,0x19]
-# CHECK: ldsetpal x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x33,0xf6,0x19]
-# CHECK: ldsetpal x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x31,0x62,0x19]
-# CHECK: ldsetpl x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x33,0x76,0x19]
-# CHECK: ldsetpl x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x81,0x22,0x19]
-# CHECK: swpp x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x83,0x36,0x19]
-# CHECK: swpp x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x81,0xa2,0x19]
-# CHECK: swppa x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x83,0xb6,0x19]
-# CHECK: swppa x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x81,0xe2,0x19]
-# CHECK: swppal x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x83,0xf6,0x19]
-# CHECK: swppal x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0x61,0x81,0x62,0x19]
-# CHECK: swppl x1, x2, [x11]
-# NO-LSE128: warning: invalid instruction encoding
-
-[0xf5,0x83,0x76,0x19]
-# CHECK: swppl x21, x22, [sp]
-# NO-LSE128: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-cpa.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-cpa.txt
deleted file mode 100644
index bf61782f912a3..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.5a-cpa.txt
+++ /dev/null
@@ -1,42 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble -mattr=+cpa < %s | FileCheck %s
-# RUN: not llvm-mc -triple aarch64 -disassemble < %s 2>&1 | FileCheck --check-prefix=NO-CPA %s
-
-[0x20,0x20,0x02,0x9a]
-# CHECK: addpt x0, x1, x2
-# NO-CPA: warning: invalid instruction encoding
-
-[0xff,0x23,0x02,0x9a]
-# CHECK: addpt sp, sp, x2
-# NO-CPA: warning: invalid instruction encoding
-
-[0x20,0x3c,0x02,0x9a]
-# CHECK: addpt x0, x1, x2, lsl #7
-# NO-CPA: warning: invalid instruction encoding
-
-[0xff,0x3f,0x02,0x9a]
-# CHECK: addpt sp, sp, x2, lsl #7
-# NO-CPA: warning: invalid instruction encoding
-
-[0x20,0x20,0x02,0xda]
-# CHECK: subpt x0, x1, x2
-# NO-CPA: warning: invalid instruction encoding
-
-[0xff,0x23,0x02,0xda]
-# CHECK: subpt sp, sp, x2
-# NO-CPA: warning: invalid instruction encoding
-
-[0x20,0x3c,0x02,0xda]
-# CHECK: subpt x0, x1, x2, lsl #7
-# NO-CPA: warning: invalid instruction encoding
-
-[0xff,0x3f,0x02,0xda]
-# CHECK: subpt sp, sp, x2, lsl #7
-# NO-CPA: warning: invalid instruction encoding
-
-[0x20,0x0c,0x62,0x9b]
-# CHECK: maddpt x0, x1, x2, x3
-# NO-CPA: warning: invalid instruction encoding
-
-[0x20,0x8c,0x62,0x9b]
-# CHECK: msubpt x0, x1, x2, x3
-# NO-CPA: warning: invalid instruction encoding
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-e3dse.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-e3dse.txt
deleted file mode 100644
index d2476dbf876d4..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.5a-e3dse.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
-
-[0x20,0xc1,0x3e,0xd5]
-# CHECK: mrs x0, VDISR_EL3
-
-[0x20,0xc1,0x1e,0xd5]
-# CHECK: msr VDISR_EL3, x0
-
-[0x60,0x52,0x3e,0xd5]
-# CHECK: mrs x0, VSESR_EL3
-
-[0x60,0x52,0x1e,0xd5]
-# CHECK: msr VSESR_EL3, x0
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-fgwte3.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-fgwte3.txt
deleted file mode 100644
index f7e355a700af0..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.5a-fgwte3.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
-
-[0xa0,0x11,0x3e,0xd5]
-# CHECK: mrs x0, FGWTE3_EL3
-
-[0xa0,0x11,0x1e,0xd5]
-# CHECK: msr FGWTE3_EL3, x0
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-hacdbs.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-hacdbs.txt
deleted file mode 100644
index d9be7e5ba4432..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.5a-hacdbs.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
-
-[0x80,0x23,0x3c,0xd5]
-# CHECK: mrs x0, HACDBSBR_EL2
-
-[0x80,0x23,0x1c,0xd5]
-# CHECK: msr HACDBSBR_EL2, x0
-
-[0xa0,0x23,0x3c,0xd5]
-# CHECK: mrs x0, HACDBSCONS_EL2
-
-[0xa0,0x23,0x1c,0xd5]
-# CHECK: msr HACDBSCONS_EL2, x0
-
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-hdbss.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-hdbss.txt
deleted file mode 100644
index 999f322548f46..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.5a-hdbss.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
-
-[0x40,0x23,0x3c,0xd5]
-# CHECK: mrs x0, HDBSSBR_EL2
-
-[0x40,0x23,0x1c,0xd5]
-# CHECK: msr HDBSSBR_EL2, x0
-
-[0x60,0x23,0x3c,0xd5]
-# CHECK: mrs x0, HDBSSPROD_EL2
-
-[0x60,0x23,0x1c,0xd5]
-# CHECK: msr HDBSSPROD_EL2, x0
-
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-spmu2.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-spmu2.txt
deleted file mode 100644
index 9d4fa1b1a078a..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.5a-spmu2.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
-
-[0x80,0x9c,0x13,0xd5]
-# CHECK: msr SPMZR_EL0, x0
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-step2.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-step2.txt
deleted file mode 100644
index 473c16d096012..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.5a-step2.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
-
-[0x40,0x05,0x30,0xd5]
-# CHECK: mrs x0, MDSTEPOP_EL1
-
-[0x40,0x05,0x10,0xd5]
-# CHECK: msr MDSTEPOP_EL1, x0
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.5a-tlbiw.txt b/llvm/test/MC/Disassembler/AArch64/armv9.5a-tlbiw.txt
deleted file mode 100644
index df5e894a929e4..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.5a-tlbiw.txt
+++ /dev/null
@@ -1,27 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble -mattr=+tlbiw -mattr=+xs < %s | FileCheck --check-prefix=CHECK-TLBIW --check-prefix=CHECK-XS %s
-# RUN: llvm-mc -triple aarch64 -disassemble -mattr=+tlbiw < %s | FileCheck --check-prefix=CHECK-TLBIW --check-prefix=CHECK-NO-XS-TLBIW %s
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck --check-prefix=CHECK-NO-TLBIW --check-prefix=CHECK-NO-XS-TLBIW %s
-
-[0x5f,0x86,0x0c,0xd5]
-# CHECK-TLBIW: tlbi vmallws2e1
-# CHECK-NO-TLBIW: sys #4, c8, c6, #2
-
-[0x5f,0x82,0x0c,0xd5]
-# CHECK-TLBIW: tlbi vmallws2e1is
-# CHECK-NO-TLBIW: sys #4, c8, c2, #2
-
-[0x5f,0x85,0x0c,0xd5]
-# CHECK-TLBIW: tlbi vmallws2e1os
-# CHECK-NO-TLBIW: sys #4, c8, c5, #2
-
-[0x5f,0x96,0x0c,0xd5]
-# CHECK-XS: tlbi vmallws2e1nxs
-# CHECK-NO-XS-TLBIW: sys #4, c9, c6, #2
-
-[0x5f,0x92,0x0c,0xd5]
-# CHECK-XS: tlbi vmallws2e1isnxs
-# CHECK-NO-XS-TLBIW: sys #4, c9, c2, #2
-
-[0x5f,0x95,0x0c,0xd5]
-# CHECK-XS: tlbi vmallws2e1osnxs
-# CHECK-NO-XS-TLBIW: sys #4, c9, c5, #2
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-lsui.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-lsui.txt
deleted file mode 100644
index 4cde11f38dde1..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.6a-lsui.txt
+++ /dev/null
@@ -1,323 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
-# RUN: llvm-mc -triple aarch64 -mattr=+lsui -disassemble %s  | FileCheck %s
-
-# LDTXR and STTXR
-[0xe9,0x7f,0x5f,0xc9]
-[0xe9,0x7f,0x5f,0xc9]
-[0x6a,0x7d,0x5f,0xc9]
-[0x6a,0x7d,0x5f,0xc9]
-
-[0xe4,0x7f,0x1f,0x89]
-[0xe4,0x7f,0x1f,0x89]
-[0xe6,0x7c,0x05,0xc9]
-[0xe6,0x7c,0x05,0xc9]
-
-# LDATXR and STLTXR
-[0xe9,0xff,0x5f,0xc9]
-[0x6a,0xfd,0x5f,0xc9]
-
-[0xe4,0xff,0x02,0x89]
-[0xe6,0xfc,0x05,0xc9]
-
-# STTP and LDTP
-[0x55,0xf4,0x5f,0xe9]
-[0x76,0x5c,0x60,0xe9]
-[0x98,0xe4,0x40,0xe9]
-
-[0xe3,0x17,0x81,0xe8]
-[0xe3,0x97,0x80,0xe9]
-[0xe3,0x17,0x00,0xed]
-[0xf1,0xcf,0x1f,0xed]
-
-[0x55,0xf4,0xdf,0xe8]
-[0x76,0x5c,0xe0,0xe8]
-[0x98,0xe4,0xc0,0xe8]
-
-[0xe3,0x17,0x80,0xec]
-[0xf1,0xcf,0x9f,0xec]
-[0x37,0x74,0xe0,0xec]
-
-[0x55,0xf4,0xdf,0xe9]
-[0x76,0x5c,0xe0,0xe9]
-[0x98,0xe4,0xc0,0xe9]
-
-[0xe3,0x17,0x80,0xed]
-[0xf1,0xcf,0x9f,0xed]
-[0x37,0x74,0xe0,0xed]
-
-[0x55,0xf4,0x5f,0xe8]
-[0x76,0x5c,0x60,0xe8]
-[0x98,0xe4,0x40,0xe8]
-[0x37,0x74,0x60,0xec]
-
-[0xe3,0x17,0x00,0xe8]
-[0xf1,0x4f,0x04,0xe8]
-[0xe3,0x17,0x00,0xec]
-[0xf1,0xcf,0x1f,0xec]
-
-# SWPT{A|L}
-[0xbf,0x84,0x27,0x19]
-[0xff,0x87,0x29,0x59]
-
-[0xbf,0x84,0xa7,0x19]
-[0xff,0x87,0xa9,0x59]
-
-[0xbf,0x84,0x67,0x19]
-[0xff,0x87,0x69,0x59]
-
-[0xbf,0x84,0xe7,0x19]
-[0xff,0x87,0xe9,0x59]
-
-# CAS{A|L}T
-[0x41,0x7c,0x80,0xc9]
-[0xe1,0x7f,0x80,0xc9]
-[0x41,0x7c,0xc0,0xc9]
-[0xe1,0x7f,0xc0,0xc9]
-[0x41,0xfc,0xc0,0xc9]
-[0xe1,0xff,0xc0,0xc9]
-[0x41,0xfc,0x80,0xc9]
-[0xe1,0xff,0x80,0xc9]
-
-# CASP{A|L}T
-[0x82,0x7c,0x80,0x49]
-[0xe2,0x7f,0x80,0x49]
-[0x82,0x7c,0xc0,0x49]
-[0xe2,0x7f,0xc0,0x49]
-[0x82,0xfc,0x80,0x49]
-[0xe2,0xff,0x80,0x49]
-[0x82,0xfc,0xc0,0x49]
-[0xe2,0xff,0xc0,0x49]
-
-#LDT{SET|ADD|CLR}{A|L} and STT{ADD|SET|CLR}{L}
-
-[0xbf,0x04,0x27,0x19]
-[0xff,0x07,0x29,0x59]
-
-[0xbf,0x04,0xa7,0x19]
-[0xff,0x07,0xa9,0x59]
-
-[0xbf,0x04,0x67,0x19]
-[0xff,0x07,0x69,0x59]
-
-[0xbf,0x04,0xe7,0x19]
-[0xff,0x07,0xe9,0x59]
-
-[0xbf,0x14,0x27,0x19]
-[0xff,0x17,0x29,0x59]
-
-[0xbf,0x14,0x67,0x19]
-[0xff,0x17,0x69,0x59]
-
-[0xbf,0x14,0xa7,0x19]
-[0xff,0x17,0xa9,0x59]
-
-[0xbf,0x14,0xe7,0x19]
-[0xff,0x17,0xe9,0x59]
-
-[0xbf,0x34,0x27,0x19]
-[0xff,0x37,0x29,0x59]
-
-[0xbf,0x34,0x67,0x19]
-[0xff,0x37,0x69,0x59]
-
-[0xbf,0x34,0xa7,0x19]
-[0xff,0x37,0xa9,0x59]
-
-[0xbf,0x34,0xe7,0x19]
-[0xff,0x37,0xe9,0x59]
-
-[0x5f,0x04,0x20,0x19]
-[0xff,0x07,0x22,0x19]
-[0x5f,0x04,0x20,0x59]
-[0xff,0x07,0x22,0x59]
-
-[0x5f,0x04,0x20,0x19]
-[0xff,0x07,0x22,0x19]
-[0x5f,0x04,0x20,0x59]
-[0xff,0x07,0x22,0x59]
-
-[0x5f,0x04,0x20,0x19]
-[0xff,0x07,0x22,0x19]
-[0x5f,0x04,0x20,0x59]
-[0xff,0x07,0x22,0x59]
-
-[0x5f,0x04,0x20,0x19]
-[0xff,0x07,0x22,0x19]
-[0x5f,0x04,0x20,0x59]
-[0xff,0x07,0x22,0x59]
-
-[0x5f,0x14,0x20,0x19]
-[0xff,0x17,0x22,0x19]
-[0x5f,0x14,0x20,0x59]
-[0xff,0x17,0x22,0x59]
-
-[0x5f,0x14,0x20,0x19]
-[0xff,0x17,0x22,0x19]
-[0x5f,0x14,0x20,0x59]
-[0xff,0x17,0x22,0x59]
-
-[0x5f,0x14,0x20,0x19]
-[0xff,0x17,0x22,0x19]
-[0x5f,0x14,0x20,0x59]
-[0xff,0x17,0x22,0x59]
-
-[0x5f,0x14,0x20,0x19]
-[0xff,0x17,0x22,0x59]
-[0x5f,0x14,0x20,0x59]
-[0xff,0x17,0x22,0x59]
-
-[0x5f,0x34,0x20,0x19]
-[0xff,0x37,0x22,0x19]
-[0x5f,0x34,0x20,0x59]
-[0xff,0x37,0x22,0x59]
-
-[0x5f,0x34,0x20,0x19]
-[0xff,0x37,0x22,0x19]
-[0x5f,0x34,0x20,0x59]
-[0xff,0x37,0x22,0x59]
-
-[0x5f,0x34,0x20,0x19]
-[0xff,0x37,0x22,0x19]
-[0x5f,0x34,0x20,0x59]
-[0xff,0x37,0x22,0x59]
-
-[0x5f,0x34,0x20,0x19]
-[0xff,0x37,0x22,0x59]
-[0x5f,0x34,0x20,0x59]
-[0xff,0x37,0x22,0x59]
-
-# CHECK:      	ldtxr	x9, [sp]
-# CHECK-NEXT: 	ldtxr	x9, [sp]
-# CHECK-NEXT: 	ldtxr	x10, [x11]
-# CHECK-NEXT: 	ldtxr	x10, [x11]
-# CHECK-NEXT: 	sttxr	wzr, w4, [sp]
-# CHECK-NEXT: 	sttxr	wzr, w4, [sp]
-# CHECK-NEXT: 	sttxr	w5, x6, [x7]
-# CHECK-NEXT: 	sttxr	w5, x6, [x7]
-# CHECK-NEXT: 	ldatxr	x9, [sp]
-# CHECK-NEXT: 	ldatxr	x10, [x11]
-# CHECK-NEXT: 	stltxr	w2, w4, [sp]
-# CHECK-NEXT: 	stltxr	w5, x6, [x7]
-# CHECK-NEXT: 	ldtp	x21, x29, [x2, #504]
-# CHECK-NEXT: 	ldtp	x22, x23, [x3, #-512]
-# CHECK-NEXT: 	ldtp	x24, x25, [x4, #8]
-# CHECK-NEXT:	sttp	x3, x5, [sp], #16
-# CHECK-NEXT:	sttp	x3, x5, [sp, #8]!
-# CHECK-NEXT: 	sttp	q3, q5, [sp]
-# CHECK-NEXT: 	sttp	q17, q19, [sp, #1008]
-# CHECK-NEXT: 	ldtp	x21, x29, [x2], #504
-# CHECK-NEXT: 	ldtp	x22, x23, [x3], #-512
-# CHECK-NEXT: 	ldtp	x24, x25, [x4], #8
-# CHECK-NEXT: 	sttp	q3, q5, [sp], #0
-# CHECK-NEXT: 	sttp	q17, q19, [sp], #1008
-# CHECK-NEXT: 	ldtp	q23, q29, [x1], #-1024
-# CHECK-NEXT: 	ldtp	x21, x29, [x2, #504]!
-# CHECK-NEXT: 	ldtp	x22, x23, [x3, #-512]!
-# CHECK-NEXT: 	ldtp	x24, x25, [x4, #8]!
-# CHECK-NEXT: 	sttp	q3, q5, [sp, #0]!
-# CHECK-NEXT: 	sttp	q17, q19, [sp, #1008]!
-# CHECK-NEXT: 	ldtp	q23, q29, [x1, #-1024]!
-# CHECK-NEXT: 	ldtnp	x21, x29, [x2, #504]
-# CHECK-NEXT: 	ldtnp	x22, x23, [x3, #-512]
-# CHECK-NEXT: 	ldtnp	x24, x25, [x4, #8]
-# CHECK-NEXT: 	ldtnp	q23, q29, [x1, #-1024]
-# CHECK-NEXT: 	sttnp	x3, x5, [sp]
-# CHECK-NEXT: 	sttnp	x17, x19, [sp, #64]
-# CHECK-NEXT: 	sttnp	q3, q5, [sp]
-# CHECK-NEXT: 	sttnp	q17, q19, [sp, #1008]
-# CHECK-NEXT: 	swpt	w7, wzr, [x5]
-# CHECK-NEXT: 	swpt	x9, xzr, [sp]
-# CHECK-NEXT: 	swpta	w7, wzr, [x5]
-# CHECK-NEXT: 	swpta	x9, xzr, [sp]
-# CHECK-NEXT: 	swptl	w7, wzr, [x5]
-# CHECK-NEXT: 	swptl	x9, xzr, [sp]
-# CHECK-NEXT: 	swptal	w7, wzr, [x5]
-# CHECK-NEXT: 	swptal	x9, xzr, [sp]
-# CHECK-NEXT: 	cast	x0, x1, [x2]
-# CHECK-NEXT: 	cast	x0, x1, [sp]
-# CHECK-NEXT: 	casat	x0, x1, [x2]
-# CHECK-NEXT: 	casat	x0, x1, [sp]
-# CHECK-NEXT: 	casalt	x0, x1, [x2]
-# CHECK-NEXT: 	casalt	x0, x1, [sp]
-# CHECK-NEXT: 	caslt	x0, x1, [x2]
-# CHECK-NEXT: 	caslt	x0, x1, [sp]
-# CHECK-NEXT: 	caspt	x0, x1, x2, x3, [x4]
-# CHECK-NEXT: 	caspt	x0, x1, x2, x3, [sp]
-# CHECK-NEXT: 	caspat	x0, x1, x2, x3, [x4]
-# CHECK-NEXT: 	caspat	x0, x1, x2, x3, [sp]
-# CHECK-NEXT: 	casplt	x0, x1, x2, x3, [x4]
-# CHECK-NEXT: 	casplt	x0, x1, x2, x3, [sp]
-# CHECK-NEXT: 	caspalt	x0, x1, x2, x3, [x4]
-# CHECK-NEXT: 	caspalt	x0, x1, x2, x3, [sp]
-# CHECK-NEXT: 	ldtadd	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtadd	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtadda	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtadda	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtaddl	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtaddl	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtaddal	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtaddal	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtclr	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtclr	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtclrl	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtclrl	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtclra	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtclra	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtclral	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtclral	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtset	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtset	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtsetl	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtsetl	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtseta	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtseta	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtsetal	w7, wzr, [x5]
-# CHECK-NEXT: 	ldtsetal	x9, xzr, [sp]
-# CHECK-NEXT: 	ldtadd	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtadd	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtadd	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtadd	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtadd	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtadd	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtadd	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtadd	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtadd	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtadd	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtadd	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtadd	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtadd	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtadd	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtadd	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtadd	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtclr	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtclr	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtclr	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtclr	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtclr	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtclr	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtclr	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtclr	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtclr	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtclr	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtclr	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtclr	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtclr	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtclr	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtclr	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtclr	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtset	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtset	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtset	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtset	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtset	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtset	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtset	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtset	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtset	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtset	w2, wzr, [sp]
-# CHECK-NEXT: 	ldtset	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtset	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtset	w0, wzr, [x2]
-# CHECK-NEXT: 	ldtset	x2, xzr, [sp]
-# CHECK-NEXT: 	ldtset	x0, xzr, [x2]
-# CHECK-NEXT: 	ldtset	x2, xzr, [sp]
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-mpam.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-mpam.txt
deleted file mode 100644
index b9ff0a4d209b2..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.6a-mpam.txt
+++ /dev/null
@@ -1,50 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
-
-#------------------------------------------------------------------------------
-# Armv9.6-A FEAT_MPAM Extensions
-#------------------------------------------------------------------------------
-
-[0x80,0xa5,0x1e,0xd5]
-# CHECK: msr MPAMBW3_EL3, x0
-
-[0x80,0xa5,0x1c,0xd5]
-# CHECK: msr MPAMBW2_EL2, x0
-
-[0x80,0xa5,0x18,0xd5]
-# CHECK: msr MPAMBW1_EL1, x0
-
-[0x80,0xa5,0x1d,0xd5]
-# CHECK: msr MPAMBW1_EL12, x0
-
-[0xa0,0xa5,0x18,0xd5]
-# CHECK: msr MPAMBW0_EL1, x0
-
-[0xc0,0xa5,0x1c,0xd5]
-# CHECK: msr MPAMBWCAP_EL2, x0
-
-[0xe0,0xa5,0x18,0xd5]
-# CHECK: msr MPAMBWSM_EL1, x0
-
-[0xa0,0xa4,0x38,0xd5]
-# CHECK: mrs x0, MPAMBWIDR_EL1
-
-[0x80,0xa5,0x3e,0xd5]
-# CHECK: mrs x0, MPAMBW3_EL3
-
-[0x80,0xa5,0x3c,0xd5]
-# CHECK: mrs x0, MPAMBW2_EL2
-
-[0x80,0xa5,0x38,0xd5]
-# CHECK: mrs x0, MPAMBW1_EL1
-
-[0x80,0xa5,0x3d,0xd5]
-# CHECK: mrs x0, MPAMBW1_EL12
-
-[0xa0,0xa5,0x38,0xd5]
-# CHECK: mrs x0, MPAMBW0_EL1
-
-[0xc0,0xa5,0x3c,0xd5]
-# CHECK: mrs x0, MPAMBWCAP_EL2
-
-[0xe0,0xa5,0x38,0xd5]
-# CHECK: mrs x0, MPAMBWSM_EL1
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-occmo.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-occmo.txt
deleted file mode 100644
index 5c3b57a871b9e..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.6a-occmo.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
-# RUN: llvm-mc -triple aarch64-none-linux-gnu -mattr=+occmo -mattr=+mte -disassemble < %s | FileCheck %s
-[0x0c,0x7f,0x0b,0xd5]
-[0xe0,0x7f,0x0b,0xd5]
-[0x0d,0x7b,0x0b,0xd5]
-[0xe1,0x7b,0x0b,0xd5]
-
-# CHECK:      	dc	civaoc, x12
-# CHECK-NEXT: 	dc	cigdvaoc, x0
-# CHECK-NEXT: 	dc	cvaoc, x13
-# CHECK-NEXT: 	dc	cgdvaoc, x1
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-pcdphint.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-pcdphint.txt
deleted file mode 100644
index 3855ce035a4c1..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.6a-pcdphint.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
-# RUN: llvm-mc -triple aarch64 -disassemble -mattr=+pcdphint %s | FileCheck %s
-
-[0x1f,0x96,0x01,0xd5]
-[0x3f,0x96,0x01,0xd5]
-
-# CHECK:      	stshh	keep
-# CHECK-NEXT: 	stshh	strm
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-rme-gpc3.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-rme-gpc3.txt
deleted file mode 100644
index 75129ac48566e..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.6a-rme-gpc3.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
-# RUN: llvm-mc -triple aarch64 -disassemble %s  | FileCheck %s
-
-[0x00,0x70,0x0e,0xd5]
-[0x01,0x70,0x0e,0xd5]
-[0x02,0x70,0x0e,0xd5]
-[0x11,0x70,0x0e,0xd5]
-[0x1e,0x70,0x0e,0xd5]
-[0xa3,0x21,0x3e,0xd5]
-[0xa4,0x21,0x1e,0xd5]
-
-# CHECK:      	apas x0
-# CHECK-NEXT: 	apas x1
-# CHECK-NEXT: 	apas x2
-# CHECK-NEXT: 	apas x17
-# CHECK-NEXT: 	apas x30
-# CHECK-NEXT: 	mrs	x3, GPCBW_EL3
-# CHECK-NEXT: 	msr	GPCBW_EL3, x4
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-srmask.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-srmask.txt
deleted file mode 100644
index 30d0a60321881..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.6a-srmask.txt
+++ /dev/null
@@ -1,101 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
-# RUN: llvm-mc -triple aarch64 -disassemble < %s 2> %t | FileCheck %s
-
-[0x03,0x14,0x38,0xd5]
-[0x03,0x14,0x3c,0xd5]
-[0x03,0x14,0x3d,0xd5]
-[0x43,0x14,0x38,0xd5]
-[0x43,0x14,0x3c,0xd5]
-[0x43,0x14,0x3d,0xd5]
-[0x63,0x14,0x38,0xd5]
-[0x63,0x14,0x3c,0xd5]
-[0x63,0x14,0x3d,0xd5]
-[0x83,0x14,0x38,0xd5]
-[0xc3,0x14,0x38,0xd5]
-[0xe3,0x14,0x38,0xd5]
-[0x43,0x27,0x38,0xd5]
-[0x43,0x27,0x3c,0xd5]
-[0x43,0x27,0x3d,0xd5]
-[0x63,0x27,0x38,0xd5]
-[0x63,0x27,0x3c,0xd5]
-[0x63,0x27,0x3d,0xd5]
-[0xc3,0x27,0x38,0xd5]
-[0xe3,0x27,0x38,0xd5]
-[0x23,0x14,0x38,0xd5]
-[0x23,0x14,0x3c,0xd5]
-[0x23,0x14,0x3d,0xd5]
-[0xa3,0x14,0x38,0xd5]
-
-[0x03,0x14,0x18,0xd5]
-[0x03,0x14,0x1c,0xd5]
-[0x03,0x14,0x1d,0xd5]
-[0x43,0x14,0x18,0xd5]
-[0x43,0x14,0x1c,0xd5]
-[0x43,0x14,0x1d,0xd5]
-[0x63,0x14,0x18,0xd5]
-[0x63,0x14,0x1c,0xd5]
-[0x63,0x14,0x1d,0xd5]
-[0x83,0x14,0x18,0xd5]
-[0xc3,0x14,0x18,0xd5]
-[0xe3,0x14,0x18,0xd5]
-[0x43,0x27,0x18,0xd5]
-[0x43,0x27,0x1c,0xd5]
-[0x43,0x27,0x1d,0xd5]
-[0x63,0x27,0x18,0xd5]
-[0x63,0x27,0x1c,0xd5]
-[0x63,0x27,0x1d,0xd5]
-[0xc3,0x27,0x18,0xd5]
-[0xe3,0x27,0x18,0xd5]
-[0x23,0x14,0x18,0xd5]
-[0x23,0x14,0x1c,0xd5]
-[0x23,0x14,0x1d,0xd5]
-[0xa3,0x14,0x18,0xd5]
-
-# CHECK:      	mrs	x3, SCTLRMASK_EL1
-# CHECK-NEXT: 	mrs	x3, SCTLRMASK_EL2
-# CHECK-NEXT: 	mrs	x3, SCTLRMASK_EL12
-# CHECK-NEXT: 	mrs	x3, CPACRMASK_EL1
-# CHECK-NEXT: 	mrs	x3, CPTRMASK_EL2
-# CHECK-NEXT: 	mrs	x3, CPACRMASK_EL12
-# CHECK-NEXT: 	mrs	x3, SCTLR2MASK_EL1
-# CHECK-NEXT: 	mrs	x3, SCTLR2MASK_EL2
-# CHECK-NEXT: 	mrs	x3, SCTLR2MASK_EL12
-# CHECK-NEXT: 	mrs	x3, CPACRALIAS_EL1
-# CHECK-NEXT: 	mrs	x3, SCTLRALIAS_EL1
-# CHECK-NEXT: 	mrs	x3, SCTLR2ALIAS_EL1
-# CHECK-NEXT: 	mrs	x3, TCRMASK_EL1
-# CHECK-NEXT: 	mrs	x3, TCRMASK_EL2
-# CHECK-NEXT: 	mrs	x3, TCRMASK_EL12
-# CHECK-NEXT: 	mrs	x3, TCR2MASK_EL1
-# CHECK-NEXT: 	mrs	x3, TCR2MASK_EL2
-# CHECK-NEXT: 	mrs	x3, TCR2MASK_EL12
-# CHECK-NEXT: 	mrs	x3, TCRALIAS_EL1
-# CHECK-NEXT: 	mrs	x3, TCR2ALIAS_EL1
-# CHECK-NEXT: 	mrs	x3, ACTLRMASK_EL1
-# CHECK-NEXT: 	mrs	x3, ACTLRMASK_EL2
-# CHECK-NEXT: 	mrs	x3, ACTLRMASK_EL12
-# CHECK-NEXT: 	mrs	x3, ACTLRALIAS_EL1
-# CHECK-NEXT: 	msr	SCTLRMASK_EL1, x3
-# CHECK-NEXT: 	msr	SCTLRMASK_EL2, x3
-# CHECK-NEXT: 	msr	SCTLRMASK_EL12, x3
-# CHECK-NEXT: 	msr	CPACRMASK_EL1, x3
-# CHECK-NEXT: 	msr	CPTRMASK_EL2, x3
-# CHECK-NEXT: 	msr	CPACRMASK_EL12, x3
-# CHECK-NEXT: 	msr	SCTLR2MASK_EL1, x3
-# CHECK-NEXT: 	msr	SCTLR2MASK_EL2, x3
-# CHECK-NEXT: 	msr	SCTLR2MASK_EL12, x3
-# CHECK-NEXT: 	msr	CPACRALIAS_EL1, x3
-# CHECK-NEXT: 	msr	SCTLRALIAS_EL1, x3
-# CHECK-NEXT: 	msr	SCTLR2ALIAS_EL1, x3
-# CHECK-NEXT: 	msr	TCRMASK_EL1, x3
-# CHECK-NEXT: 	msr	TCRMASK_EL2, x3
-# CHECK-NEXT: 	msr	TCRMASK_EL12, x3
-# CHECK-NEXT: 	msr	TCR2MASK_EL1, x3
-# CHECK-NEXT: 	msr	TCR2MASK_EL2, x3
-# CHECK-NEXT: 	msr	TCR2MASK_EL12, x3
-# CHECK-NEXT: 	msr	TCRALIAS_EL1, x3
-# CHECK-NEXT: 	msr	TCR2ALIAS_EL1, x3
-# CHECK-NEXT: 	msr	ACTLRMASK_EL1, x3
-# CHECK-NEXT: 	msr	ACTLRMASK_EL2, x3
-# CHECK-NEXT: 	msr	ACTLRMASK_EL12, x3
-# CHECK-NEXT: 	msr	ACTLRALIAS_EL1, x3
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-statistical-profiling.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-statistical-profiling.txt
deleted file mode 100644
index 446e2f0eb05c4..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.6a-statistical-profiling.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
-
-[0x60,0x9a,0x1d,0xd5]
-# CHECK: msr PMBSR_EL12, x0
-[0x60,0x9a,0x1c,0xd5]
-# CHECK: msr PMBSR_EL2, x0
-[0x60,0x9a,0x1e,0xd5]
-# CHECK: msr PMBSR_EL3, x0
-
-[0x60,0x9a,0x3d,0xd5]
-# CHECK: mrs x0, PMBSR_EL12
-[0x60,0x9a,0x3c,0xd5]
-# CHECK: mrs x0, PMBSR_EL2
-[0x60,0x9a,0x3e,0xd5]
-# CHECK: mrs x0, PMBSR_EL3
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9.6a-trbe-exception.txt b/llvm/test/MC/Disassembler/AArch64/armv9.6a-trbe-exception.txt
deleted file mode 100644
index 4b39e107538d9..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9.6a-trbe-exception.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-# RUN: llvm-mc -triple aarch64 -disassemble < %s | FileCheck %s
-
-[0x60,0x9b,0x1d,0xd5]
-# CHECK: msr TRBSR_EL12, x0
-[0x60,0x9b,0x1c,0xd5]
-# CHECK: msr TRBSR_EL2, x0
-[0x60,0x9b,0x1e,0xd5]
-# CHECK: msr TRBSR_EL3, x0
-
-[0x60,0x9b,0x3d,0xd5]
-# CHECK: mrs x0, TRBSR_EL12
-[0x60,0x9b,0x3c,0xd5]
-# CHECK: mrs x0, TRBSR_EL2
-[0x60,0x9b,0x3e,0xd5]
-# CHECK: mrs x0, TRBSR_EL3
diff --git a/llvm/test/MC/Disassembler/AArch64/armv9a-mec.txt b/llvm/test/MC/Disassembler/AArch64/armv9a-mec.txt
deleted file mode 100644
index c5d931d46ca6d..0000000000000
--- a/llvm/test/MC/Disassembler/AArch64/armv9a-mec.txt
+++ /dev/null
@@ -1,54 +0,0 @@
-# RUN: llvm-mc -triple=aarch64 -mattr=+mec -disassemble %s      | FileCheck %s
-# RUN: llvm-mc -triple=aarch64             -disassemble %s 2>&1 | FileCheck %s --check-prefix=CHECK-NO-MEC
-
-[0xe0,0xa8,0x3c,0xd5]
-# CHECK: mrs x0, MECIDR_EL2
-# CHECK-NO-MEC: mrs x0, S3_4_C10_C8_7
-[0x00,0xa8,0x3c,0xd5]
-# CHECK: mrs x0, MECID_P0_EL2
-# CHECK-NO-MEC: mrs x0, S3_4_C10_C8_0
-[0x20,0xa8,0x3c,0xd5]
-# CHECK: mrs x0, MECID_A0_EL2
-# CHECK-NO-MEC: mrs x0, S3_4_C10_C8_1
-[0x40,0xa8,0x3c,0xd5]
-# CHECK: mrs x0, MECID_P1_EL2
-# CHECK-NO-MEC: mrs x0, S3_4_C10_C8_2
-[0x60,0xa8,0x3c,0xd5]
-# CHECK: mrs x0, MECID_A1_EL2
-# CHECK-NO-MEC: mrs x0, S3_4_C10_C8_3
-[0x00,0xa9,0x3c,0xd5]
-# CHECK: mrs x0, VMECID_P_EL2
-# CHECK-NO-MEC: mrs x0, S3_4_C10_C9_0
-[0x20,0xa9,0x3c,0xd5]
-# CHECK: mrs x0, VMECID_A_EL2
-# CHECK-NO-MEC: mrs x0, S3_4_C10_C9_1
-[0x20,0xaa,0x3e,0xd5]
-# CHECK: mrs x0, MECID_RL_A_EL3
-# CHECK-NO-MEC: mrs x0, S3_6_C10_C10_1
-[0x00,0xa8,0x1c,0xd5]
-# CHECK: msr MECID_P0_EL2,    x0
-# CHECK-NO-MEC: msr S3_4_C10_C8_0, x0
-[0x20,0xa8,0x1c,0xd5]
-# CHECK: msr MECID_A0_EL2,    x0
-# CHECK-NO-MEC: msr S3_4_C10_C8_1, x0
-[0x40,0xa8,0x1c,0xd5]
-# CHECK: msr MECID_P1_EL2,    x0
-# CHECK-NO-MEC: msr S3_4_C10_C8_2, x0
-[0x60,0xa8,0x1c,0xd5]
-# CHECK: msr MECID_A1_EL2,    x0
-# CHECK-NO-MEC: msr S3_4_C10_C8_3, x0
-[0x00,0xa9,0x1c,0xd5]
-# CHECK: msr VMECID_P_EL2,   x0
-# CHECK-NO-MEC: msr S3_4_C10_C9_0, x0
-[0x20,0xa9,0x1c,0xd5]
-# CHECK: msr VMECID_A_EL2,   x0
-# CHECK-NO-MEC: msr S3_4_C10_C9_1, x0
-[0x20,0xaa,0x1e,0xd5]
-# CHECK: msr MECID_RL_A_EL3, x0
-# CHECK-NO-MEC: msr S3_6_C10_C10_1, x0
-[0xe0,0x7e,0x0c,0xd5]
-# CHECK: dc cigdpae, x0
-# CHECK-NO-MEC: sys #4, c7, c14, #7, x0
-[0x00,0x7e,0x0c,0xd5]
-# CHECK: dc cipae, x0
-# CHECK-NO-MEC: sys #4, c7, c14, #0, x0
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt
index 0870aa7ba3dc2..13440a06032be 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_ds.txt
@@ -1,5 +1,1109 @@
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX1250 %s
 
+# GFX1250: ds_add_f32 v1, v2                       ; encoding: [0x00,0x00,0x54,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x54,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_add_f32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x54,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x54,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_add_f32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x54,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x54,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_add_rtn_f32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xe4,0xd9,0xff,0xff,0x00,0xff]
+0x04,0x00,0xe4,0xd9,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_add_rtn_f32 v5, v1, v2               ; encoding: [0x00,0x00,0xe4,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0xe4,0xd9,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_add_rtn_f32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xe4,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0xe4,0xd9,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_add_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x80,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0x80,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_add_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x80,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0x80,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_add_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x80,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0x80,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_add_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x80,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0x80,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_add_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x80,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0x80,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_add_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x80,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0x80,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_add_u32 v1, v2                       ; encoding: [0x00,0x00,0x00,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x00,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_add_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x00,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x00,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_add_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x00,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x00,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_add_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x00,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x00,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_add_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x00,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x00,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_add_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x00,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x00,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_and_b32 v1, v2                       ; encoding: [0x00,0x00,0x24,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x24,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_and_b32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x24,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x24,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_and_b32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x24,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x24,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_and_b64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x24,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x24,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_and_b64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x24,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x24,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_and_b64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x24,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x24,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_and_rtn_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xa4,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0xa4,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_and_rtn_b32 v5, v1, v2               ; encoding: [0x00,0x00,0xa4,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0xa4,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_and_rtn_b32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xa4,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0xa4,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_and_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xa4,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0xa4,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_and_rtn_b64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0xa4,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0xa4,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_and_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa4,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0xa4,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_append v255 offset:4                 ; encoding: [0x04,0x00,0xf8,0xd8,0x00,0x00,0x00,0xff]
+0x04,0x00,0xf8,0xd8,0x00,0x00,0x00,0xff
+
+# GFX1250: ds_append v5                            ; encoding: [0x00,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05]
+0x00,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05
+
+# GFX1250: ds_append v5 offset:65535               ; encoding: [0xff,0xff,0xf8,0xd8,0x00,0x00,0x00,0x05]
+0xff,0xff,0xf8,0xd8,0x00,0x00,0x00,0x05
+
+# GFX1250: ds_bpermute_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xcc,0xda,0xff,0xff,0x00,0xff]
+0x04,0x00,0xcc,0xda,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_bpermute_b32 v5, v1, v2              ; encoding: [0x00,0x00,0xcc,0xda,0x01,0x02,0x00,0x05]
+0x00,0x00,0xcc,0xda,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_bpermute_b32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xcc,0xda,0x01,0x02,0x00,0x05]
+0xff,0xff,0xcc,0xda,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_cmpstore_b32 v1, v2, v3              ; encoding: [0x00,0x00,0x40,0xd8,0x01,0x02,0x03,0x00]
+0x00,0x00,0x40,0xd8,0x01,0x02,0x03,0x00
+
+# GFX1250: ds_cmpstore_b32 v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0x40,0xd8,0x01,0x02,0x03,0x00]
+0xff,0xff,0x40,0xd8,0x01,0x02,0x03,0x00
+
+# GFX1250: ds_cmpstore_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x40,0xd8,0xff,0xff,0xff,0x00]
+0x04,0x00,0x40,0xd8,0xff,0xff,0xff,0x00
+
+# GFX1250: ds_cmpstore_b64 v1, v[2:3], v[4:5]      ; encoding: [0x00,0x00,0x40,0xd9,0x01,0x02,0x04,0x00]
+0x00,0x00,0x40,0xd9,0x01,0x02,0x04,0x00
+
+# GFX1250: ds_cmpstore_b64 v1, v[2:3], v[4:5] offset:65535 ; encoding: [0xff,0xff,0x40,0xd9,0x01,0x02,0x04,0x00]
+0xff,0xff,0x40,0xd9,0x01,0x02,0x04,0x00
+
+# GFX1250: ds_cmpstore_b64 v255, v[254:255], v[254:255] offset:4 ; encoding: [0x04,0x00,0x40,0xd9,0xff,0xfe,0xfe,0x00]
+0x04,0x00,0x40,0xd9,0xff,0xfe,0xfe,0x00
+
+# GFX1250: ds_cmpstore_rtn_b32 v255, v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xc0,0xd8,0xff,0xff,0xff,0xff]
+0x04,0x00,0xc0,0xd8,0xff,0xff,0xff,0xff
+
+# GFX1250: ds_cmpstore_rtn_b32 v5, v1, v2, v3      ; encoding: [0x00,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05]
+0x00,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05
+
+# GFX1250: ds_cmpstore_rtn_b32 v5, v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xc0,0xd8,0x01,0x02,0x03,0x05]
+0xff,0xff,0xc0,0xd8,0x01,0x02,0x03,0x05
+
+# GFX1250: ds_cmpstore_rtn_b64 v[254:255], v255, v[254:255], v[254:255] offset:4 ; encoding: [0x04,0x00,0xc0,0xd9,0xff,0xfe,0xfe,0xfe]
+0x04,0x00,0xc0,0xd9,0xff,0xfe,0xfe,0xfe
+
+# GFX1250: ds_cmpstore_rtn_b64 v[6:7], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xc0,0xd9,0x01,0x02,0x04,0x06]
+0x00,0x00,0xc0,0xd9,0x01,0x02,0x04,0x06
+
+# GFX1250: ds_cmpstore_rtn_b64 v[6:7], v1, v[2:3], v[4:5] offset:65535 ; encoding: [0xff,0xff,0xc0,0xd9,0x01,0x02,0x04,0x06]
+0xff,0xff,0xc0,0xd9,0x01,0x02,0x04,0x06
+
+# GFX1250: ds_cond_sub_rtn_u32 v5, v1, v2          ; encoding: [0x00,0x00,0xa0,0xda,0x01,0x02,0x00,0x05]
+0x00,0x00,0xa0,0xda,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_cond_sub_rtn_u32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xa0,0xda,0x01,0x02,0x00,0x05]
+0xff,0xff,0xa0,0xda,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_cond_sub_u32 v1, v2                  ; encoding: [0x00,0x00,0x60,0xda,0x01,0x02,0x00,0x00]
+0x00,0x00,0x60,0xda,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_cond_sub_u32 v1, v2 offset:65535     ; encoding: [0xff,0xff,0x60,0xda,0x01,0x02,0x00,0x00]
+0xff,0xff,0x60,0xda,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_condxchg32_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xf8,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0xf8,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_condxchg32_rtn_b64 v[6:7], v1, v[2:3] ; encoding: [0x00,0x00,0xf8,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0xf8,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_condxchg32_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xf8,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0xf8,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_consume v255 offset:4                ; encoding: [0x04,0x00,0xf4,0xd8,0x00,0x00,0x00,0xff]
+0x04,0x00,0xf4,0xd8,0x00,0x00,0x00,0xff
+
+# GFX1250: ds_consume v5                           ; encoding: [0x00,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05]
+0x00,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05
+
+# GFX1250: ds_consume v5 offset:65535              ; encoding: [0xff,0xff,0xf4,0xd8,0x00,0x00,0x00,0x05]
+0xff,0xff,0xf4,0xd8,0x00,0x00,0x00,0x05
+
+# GFX1250: ds_dec_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x90,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0x90,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_dec_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x90,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0x90,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_dec_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x90,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0x90,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_dec_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x90,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0x90,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_dec_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x90,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0x90,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_dec_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x90,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0x90,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_dec_u32 v1, v2                       ; encoding: [0x00,0x00,0x10,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x10,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_dec_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x10,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x10,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_dec_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x10,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x10,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_dec_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x10,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x10,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_dec_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x10,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x10,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_dec_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x10,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x10,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_inc_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x8c,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0x8c,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_inc_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x8c,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0x8c,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_inc_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x8c,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0x8c,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_inc_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x8c,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0x8c,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_inc_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x8c,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0x8c,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_inc_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x8c,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0x8c,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_inc_u32 v1, v2                       ; encoding: [0x00,0x00,0x0c,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x0c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_inc_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x0c,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x0c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_inc_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x0c,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x0c,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_inc_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_inc_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x0c,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x0c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_inc_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x0c,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x0c,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_load_2addr_b32 v[254:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xdc,0xd8,0xff,0x00,0x00,0xfe]
+0x10,0x01,0xdc,0xd8,0xff,0x00,0x00,0xfe
+
+# GFX1250: ds_load_2addr_b32 v[6:7], v1            ; encoding: [0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x06]
+0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_2addr_b32 v[6:7], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0x06]
+0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_2addr_b64 v[252:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xdc,0xd9,0xff,0x00,0x00,0xfc]
+0x10,0x01,0xdc,0xd9,0xff,0x00,0x00,0xfc
+
+# GFX1250: ds_load_2addr_b64 v[6:9], v1            ; encoding: [0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x06]
+0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_2addr_b64 v[6:9], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0x06]
+0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_2addr_stride64_b32 v[254:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xe0,0xd8,0xff,0x00,0x00,0xfe]
+0x10,0x01,0xe0,0xd8,0xff,0x00,0x00,0xfe
+
+# GFX1250: ds_load_2addr_stride64_b32 v[6:7], v1   ; encoding: [0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x06]
+0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_2addr_stride64_b32 v[6:7], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0x06]
+0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_2addr_stride64_b64 v[252:255], v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xe0,0xd9,0xff,0x00,0x00,0xfc]
+0x10,0x01,0xe0,0xd9,0xff,0x00,0x00,0xfc
+
+# GFX1250: ds_load_2addr_stride64_b64 v[6:9], v1   ; encoding: [0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x06]
+0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_2addr_stride64_b64 v[6:9], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0x06]
+0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_addtid_b32 v255 offset:4        ; encoding: [0x04,0x00,0xc4,0xda,0x00,0x00,0x00,0xff]
+0x04,0x00,0xc4,0xda,0x00,0x00,0x00,0xff
+
+# GFX1250: ds_load_addtid_b32 v5                   ; encoding: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05
+
+# GFX1250: ds_load_addtid_b32 v5 offset:65535      ; encoding: [0xff,0xff,0xc4,0xda,0x00,0x00,0x00,0x05]
+0xff,0xff,0xc4,0xda,0x00,0x00,0x00,0x05
+
+# GFX1250: ds_load_b128 v[252:255], v255 offset:4  ; encoding: [0x04,0x00,0xfc,0xdb,0xff,0x00,0x00,0xfc]
+0x04,0x00,0xfc,0xdb,0xff,0x00,0x00,0xfc
+
+# GFX1250: ds_load_b128 v[6:9], v1                 ; encoding: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x06]
+0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_b128 v[6:9], v1 offset:65535    ; encoding: [0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0x06]
+0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_b32 v255, v255 offset:4         ; encoding: [0x04,0x00,0xd8,0xd8,0xff,0x00,0x00,0xff]
+0x04,0x00,0xd8,0xd8,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_b32 v5, v1                      ; encoding: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_b32 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_b64 v[254:255], v255 offset:4   ; encoding: [0x04,0x00,0xd8,0xd9,0xff,0x00,0x00,0xfe]
+0x04,0x00,0xd8,0xd9,0xff,0x00,0x00,0xfe
+
+# GFX1250: ds_load_b64 v[6:7], v1                  ; encoding: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x06]
+0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_b64 v[6:7], v1 offset:65535     ; encoding: [0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0x06]
+0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_b96 v[252:254], v255 offset:4   ; encoding: [0x04,0x00,0xf8,0xdb,0xff,0x00,0x00,0xfc]
+0x04,0x00,0xf8,0xdb,0xff,0x00,0x00,0xfc
+
+# GFX1250: ds_load_b96 v[6:8], v1                  ; encoding: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x06]
+0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_b96 v[6:8], v1 offset:65535     ; encoding: [0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0x06]
+0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0x06
+
+# GFX1250: ds_load_i16 v255, v255 offset:4         ; encoding: [0x04,0x00,0xec,0xd8,0xff,0x00,0x00,0xff]
+0x04,0x00,0xec,0xd8,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_i16 v5, v1                      ; encoding: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_i16 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_i8 v255, v255 offset:4          ; encoding: [0x04,0x00,0xe4,0xd8,0xff,0x00,0x00,0xff]
+0x04,0x00,0xe4,0xd8,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_i8 v5, v1                       ; encoding: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_i8 v5, v1 offset:65535          ; encoding: [0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_i8_d16 v255, v255 offset:4      ; encoding: [0x04,0x00,0x90,0xda,0xff,0x00,0x00,0xff]
+0x04,0x00,0x90,0xda,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_i8_d16 v5, v1                   ; encoding: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_i8_d16 v5, v1 offset:65535      ; encoding: [0xff,0xff,0x90,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x90,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_i8_d16_hi v255, v255 offset:4   ; encoding: [0x04,0x00,0x94,0xda,0xff,0x00,0x00,0xff]
+0x04,0x00,0x94,0xda,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_i8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_i8_d16_hi v5, v1 offset:65535   ; encoding: [0xff,0xff,0x94,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x94,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u16 v255, v255 offset:4         ; encoding: [0x04,0x00,0xf0,0xd8,0xff,0x00,0x00,0xff]
+0x04,0x00,0xf0,0xd8,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_u16 v5, v1                      ; encoding: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u16 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u16_d16 v255, v255 offset:4     ; encoding: [0x04,0x00,0x98,0xda,0xff,0x00,0x00,0xff]
+0x04,0x00,0x98,0xda,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_u16_d16 v5, v1                  ; encoding: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u16_d16 v5, v1 offset:65535     ; encoding: [0xff,0xff,0x98,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x98,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u16_d16_hi v255, v255 offset:4  ; encoding: [0x04,0x00,0x9c,0xda,0xff,0x00,0x00,0xff]
+0x04,0x00,0x9c,0xda,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_u16_d16_hi v5, v1               ; encoding: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u16_d16_hi v5, v1 offset:65535  ; encoding: [0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u8 v255, v255 offset:4          ; encoding: [0x04,0x00,0xe8,0xd8,0xff,0x00,0x00,0xff]
+0x04,0x00,0xe8,0xd8,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_u8 v5, v1                       ; encoding: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u8 v5, v1 offset:65535          ; encoding: [0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u8_d16 v255, v255 offset:4      ; encoding: [0x04,0x00,0x88,0xda,0xff,0x00,0x00,0xff]
+0x04,0x00,0x88,0xda,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_u8_d16 v5, v1                   ; encoding: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u8_d16 v5, v1 offset:65535      ; encoding: [0xff,0xff,0x88,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x88,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u8_d16_hi v255, v255 offset:4   ; encoding: [0x04,0x00,0x8c,0xda,0xff,0x00,0x00,0xff]
+0x04,0x00,0x8c,0xda,0xff,0x00,0x00,0xff
+
+# GFX1250: ds_load_u8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_load_u8_d16_hi v5, v1 offset:65535   ; encoding: [0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0x05
+
+# GFX1250: ds_max_i32 v1, v2                       ; encoding: [0x00,0x00,0x18,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x18,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_i32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x18,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x18,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_i32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x18,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x18,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_max_i64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x18,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x18,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_i64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x18,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x18,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_i64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x18,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x18,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_max_num_f32 v1, v2                   ; encoding: [0x00,0x00,0x4c,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x4c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_num_f32 v1, v2 offset:65535      ; encoding: [0xff,0xff,0x4c,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x4c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_num_f32 v255, v255 offset:4      ; encoding: [0x04,0x00,0x4c,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x4c,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_max_num_f64 v1, v[2:3]               ; encoding: [0x00,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_num_f64 v1, v[2:3] offset:65535  ; encoding: [0xff,0xff,0x4c,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x4c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_num_f64 v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x4c,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x4c,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_max_num_rtn_f32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xcc,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0xcc,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_max_num_rtn_f32 v5, v1, v2           ; encoding: [0x00,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_max_num_rtn_f32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xcc,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0xcc,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_max_num_rtn_f64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xcc,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0xcc,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_max_num_rtn_f64 v[6:7], v1, v[2:3]   ; encoding: [0x00,0x00,0xcc,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0xcc,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_max_num_rtn_f64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xcc,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0xcc,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_max_rtn_i32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x98,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0x98,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_max_rtn_i32 v5, v1, v2               ; encoding: [0x00,0x00,0x98,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0x98,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_max_rtn_i32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x98,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0x98,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_max_rtn_i64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x98,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0x98,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_max_rtn_i64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x98,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0x98,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_max_rtn_i64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x98,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0x98,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_max_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xa0,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0xa0,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_max_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0xa0,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0xa0,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_max_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xa0,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0xa0,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_max_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xa0,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0xa0,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_max_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0xa0,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0xa0,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_max_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa0,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0xa0,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_max_u32 v1, v2                       ; encoding: [0x00,0x00,0x20,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x20,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x20,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x20,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x20,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x20,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_max_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x20,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x20,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x20,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x20,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_max_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x20,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x20,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_min_i32 v1, v2                       ; encoding: [0x00,0x00,0x14,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x14,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_i32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x14,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x14,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_i32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x14,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x14,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_min_i64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x14,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x14,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_i64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x14,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x14,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_i64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x14,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x14,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_min_num_f32 v1, v2                   ; encoding: [0x00,0x00,0x48,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x48,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_num_f32 v1, v2 offset:65535      ; encoding: [0xff,0xff,0x48,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x48,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_num_f32 v255, v255 offset:4      ; encoding: [0x04,0x00,0x48,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x48,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_min_num_f64 v1, v[2:3]               ; encoding: [0x00,0x00,0x48,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x48,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_num_f64 v1, v[2:3] offset:65535  ; encoding: [0xff,0xff,0x48,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x48,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_num_f64 v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x48,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x48,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_min_num_rtn_f32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xc8,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0xc8,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_min_num_rtn_f32 v5, v1, v2           ; encoding: [0x00,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_min_num_rtn_f32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xc8,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0xc8,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_min_num_rtn_f64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xc8,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0xc8,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_min_num_rtn_f64 v[6:7], v1, v[2:3]   ; encoding: [0x00,0x00,0xc8,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0xc8,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_min_num_rtn_f64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xc8,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0xc8,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_min_rtn_i32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x94,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0x94,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_min_rtn_i32 v5, v1, v2               ; encoding: [0x00,0x00,0x94,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0x94,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_min_rtn_i32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x94,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0x94,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_min_rtn_i64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x94,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0x94,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_min_rtn_i64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x94,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0x94,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_min_rtn_i64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x94,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0x94,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_min_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x9c,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0x9c,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_min_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x9c,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0x9c,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_min_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x9c,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0x9c,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_min_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x9c,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0x9c,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_min_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x9c,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0x9c,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_min_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x9c,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0x9c,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_min_u32 v1, v2                       ; encoding: [0x00,0x00,0x1c,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x1c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x1c,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x1c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x1c,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x1c,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_min_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x1c,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x1c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_min_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x1c,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x1c,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_mskor_b32 v1, v2, v3                 ; encoding: [0x00,0x00,0x30,0xd8,0x01,0x02,0x03,0x00]
+0x00,0x00,0x30,0xd8,0x01,0x02,0x03,0x00
+
+# GFX1250: ds_mskor_b32 v1, v2, v3 offset:65535    ; encoding: [0xff,0xff,0x30,0xd8,0x01,0x02,0x03,0x00]
+0xff,0xff,0x30,0xd8,0x01,0x02,0x03,0x00
+
+# GFX1250: ds_mskor_b32 v255, v255, v255 offset:4  ; encoding: [0x04,0x00,0x30,0xd8,0xff,0xff,0xff,0x00]
+0x04,0x00,0x30,0xd8,0xff,0xff,0xff,0x00
+
+# GFX1250: ds_mskor_b64 v1, v[2:3], v[4:5]         ; encoding: [0x00,0x00,0x30,0xd9,0x01,0x02,0x04,0x00]
+0x00,0x00,0x30,0xd9,0x01,0x02,0x04,0x00
+
+# GFX1250: ds_mskor_b64 v1, v[2:3], v[4:5] offset:65535 ; encoding: [0xff,0xff,0x30,0xd9,0x01,0x02,0x04,0x00]
+0xff,0xff,0x30,0xd9,0x01,0x02,0x04,0x00
+
+# GFX1250: ds_mskor_b64 v255, v[254:255], v[254:255] offset:4 ; encoding: [0x04,0x00,0x30,0xd9,0xff,0xfe,0xfe,0x00]
+0x04,0x00,0x30,0xd9,0xff,0xfe,0xfe,0x00
+
+# GFX1250: ds_mskor_rtn_b32 v255, v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xb0,0xd8,0xff,0xff,0xff,0xff]
+0x04,0x00,0xb0,0xd8,0xff,0xff,0xff,0xff
+
+# GFX1250: ds_mskor_rtn_b32 v5, v1, v2, v3         ; encoding: [0x00,0x00,0xb0,0xd8,0x01,0x02,0x03,0x05]
+0x00,0x00,0xb0,0xd8,0x01,0x02,0x03,0x05
+
+# GFX1250: ds_mskor_rtn_b32 v5, v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xb0,0xd8,0x01,0x02,0x03,0x05]
+0xff,0xff,0xb0,0xd8,0x01,0x02,0x03,0x05
+
+# GFX1250: ds_mskor_rtn_b64 v[254:255], v255, v[254:255], v[254:255] offset:4 ; encoding: [0x04,0x00,0xb0,0xd9,0xff,0xfe,0xfe,0xfe]
+0x04,0x00,0xb0,0xd9,0xff,0xfe,0xfe,0xfe
+
+# GFX1250: ds_mskor_rtn_b64 v[6:7], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xb0,0xd9,0x01,0x02,0x04,0x06]
+0x00,0x00,0xb0,0xd9,0x01,0x02,0x04,0x06
+
+# GFX1250: ds_mskor_rtn_b64 v[6:7], v1, v[2:3], v[4:5] offset:65535 ; encoding: [0xff,0xff,0xb0,0xd9,0x01,0x02,0x04,0x06]
+0xff,0xff,0xb0,0xd9,0x01,0x02,0x04,0x06
+
+# GFX1250: ds_nop                                  ; encoding: [0x00,0x00,0x50,0xd8,0x00,0x00,0x00,0x00]
+0x00,0x00,0x50,0xd8,0x00,0x00,0x00,0x00
+
+# GFX1250: ds_or_b32 v1, v2                        ; encoding: [0x00,0x00,0x28,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x28,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_or_b32 v1, v2 offset:65535           ; encoding: [0xff,0xff,0x28,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x28,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_or_b32 v255, v255 offset:4           ; encoding: [0x04,0x00,0x28,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x28,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_or_b64 v1, v[2:3]                    ; encoding: [0x00,0x00,0x28,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x28,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_or_b64 v1, v[2:3] offset:65535       ; encoding: [0xff,0xff,0x28,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x28,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_or_b64 v255, v[254:255] offset:4     ; encoding: [0x04,0x00,0x28,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x28,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_or_rtn_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xa8,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0xa8,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_or_rtn_b32 v5, v1, v2                ; encoding: [0x00,0x00,0xa8,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0xa8,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_or_rtn_b32 v5, v1, v2 offset:65535   ; encoding: [0xff,0xff,0xa8,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0xa8,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_or_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xa8,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0xa8,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_or_rtn_b64 v[6:7], v1, v[2:3]        ; encoding: [0x00,0x00,0xa8,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0xa8,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_or_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa8,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0xa8,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_permute_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xc8,0xda,0xff,0xff,0x00,0xff]
+0x04,0x00,0xc8,0xda,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_permute_b32 v5, v1, v2               ; encoding: [0x00,0x00,0xc8,0xda,0x01,0x02,0x00,0x05]
+0x00,0x00,0xc8,0xda,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_permute_b32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xc8,0xda,0x01,0x02,0x00,0x05]
+0xff,0xff,0xc8,0xda,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_pk_add_bf16 v0, v0                   ; encoding: [0x00,0x00,0x6c,0xda,0x00,0x00,0x00,0x00]
+0x00,0x00,0x6c,0xda,0x00,0x00,0x00,0x00
+
+# GFX1250: ds_pk_add_bf16 v0, v0 offset:65535      ; encoding: [0xff,0xff,0x6c,0xda,0x00,0x00,0x00,0x00]
+0xff,0xff,0x6c,0xda,0x00,0x00,0x00,0x00
+
+# GFX1250: ds_pk_add_bf16 v2, v1                   ; encoding: [0x00,0x00,0x6c,0xda,0x02,0x01,0x00,0x00]
+0x00,0x00,0x6c,0xda,0x02,0x01,0x00,0x00
+
+# GFX1250: ds_pk_add_bf16 v255, v255               ; encoding: [0x00,0x00,0x6c,0xda,0xff,0xff,0x00,0x00]
+0x00,0x00,0x6c,0xda,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_pk_add_bf16 v255, v255 offset:4660   ; encoding: [0x34,0x12,0x6c,0xda,0xff,0xff,0x00,0x00]
+0x34,0x12,0x6c,0xda,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_pk_add_f16 v0, v0                    ; encoding: [0x00,0x00,0x68,0xda,0x00,0x00,0x00,0x00]
+0x00,0x00,0x68,0xda,0x00,0x00,0x00,0x00
+
+# GFX1250: ds_pk_add_f16 v2, v1                    ; encoding: [0x00,0x00,0x68,0xda,0x02,0x01,0x00,0x00]
+0x00,0x00,0x68,0xda,0x02,0x01,0x00,0x00
+
+# GFX1250: ds_pk_add_f16 v2, v1 offset:4660        ; encoding: [0x34,0x12,0x68,0xda,0x02,0x01,0x00,0x00]
+0x34,0x12,0x68,0xda,0x02,0x01,0x00,0x00
+
+# GFX1250: ds_pk_add_f16 v2, v1 offset:65535       ; encoding: [0xff,0xff,0x68,0xda,0x02,0x01,0x00,0x00]
+0xff,0xff,0x68,0xda,0x02,0x01,0x00,0x00
+
+# GFX1250: ds_pk_add_f16 v255, v255                ; encoding: [0x00,0x00,0x68,0xda,0xff,0xff,0x00,0x00]
+0x00,0x00,0x68,0xda,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_pk_add_f16 v255, v255 offset:4660    ; encoding: [0x34,0x12,0x68,0xda,0xff,0xff,0x00,0x00]
+0x34,0x12,0x68,0xda,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_pk_add_f16 v255, v255 offset:65535   ; encoding: [0xff,0xff,0x68,0xda,0xff,0xff,0x00,0x00]
+0xff,0xff,0x68,0xda,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_pk_add_rtn_bf16 v255, v0, v200       ; encoding: [0x00,0x00,0xac,0xda,0x00,0xc8,0x00,0xff]
+0x00,0x00,0xac,0xda,0x00,0xc8,0x00,0xff
+
+# GFX1250: ds_pk_add_rtn_bf16 v255, v255, v255     ; encoding: [0x00,0x00,0xac,0xda,0xff,0xff,0x00,0xff]
+0x00,0x00,0xac,0xda,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_pk_add_rtn_bf16 v255, v255, v255 offset:65535 ; encoding: [0xff,0xff,0xac,0xda,0xff,0xff,0x00,0xff]
+0xff,0xff,0xac,0xda,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_pk_add_rtn_bf16 v3, v2, v1           ; encoding: [0x00,0x00,0xac,0xda,0x02,0x01,0x00,0x03]
+0x00,0x00,0xac,0xda,0x02,0x01,0x00,0x03
+
+# GFX1250: ds_pk_add_rtn_bf16 v3, v2, v1 offset:4660 ; encoding: [0x34,0x12,0xac,0xda,0x02,0x01,0x00,0x03]
+0x34,0x12,0xac,0xda,0x02,0x01,0x00,0x03
+
+# GFX1250: ds_pk_add_rtn_f16 v255, v0, v200        ; encoding: [0x00,0x00,0xa8,0xda,0x00,0xc8,0x00,0xff]
+0x00,0x00,0xa8,0xda,0x00,0xc8,0x00,0xff
+
+# GFX1250: ds_pk_add_rtn_f16 v255, v0, v200 offset:65535 ; encoding: [0xff,0xff,0xa8,0xda,0x00,0xc8,0x00,0xff]
+0xff,0xff,0xa8,0xda,0x00,0xc8,0x00,0xff
+
+# GFX1250: ds_pk_add_rtn_f16 v255, v255, v255      ; encoding: [0x00,0x00,0xa8,0xda,0xff,0xff,0x00,0xff]
+0x00,0x00,0xa8,0xda,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_pk_add_rtn_f16 v3, v2, v1            ; encoding: [0x00,0x00,0xa8,0xda,0x02,0x01,0x00,0x03]
+0x00,0x00,0xa8,0xda,0x02,0x01,0x00,0x03
+
+# GFX1250: ds_pk_add_rtn_f16 v3, v2, v1 offset:4660 ; encoding: [0x34,0x12,0xa8,0xda,0x02,0x01,0x00,0x03]
+0x34,0x12,0xa8,0xda,0x02,0x01,0x00,0x03
+
+# GFX1250: ds_rsub_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x88,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0x88,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_rsub_rtn_u32 v5, v1, v2              ; encoding: [0x00,0x00,0x88,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0x88,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_rsub_rtn_u32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0x88,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0x88,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_rsub_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x88,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0x88,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_rsub_rtn_u64 v[6:7], v1, v[2:3]      ; encoding: [0x00,0x00,0x88,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0x88,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_rsub_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x88,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0x88,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_rsub_u32 v1, v2                      ; encoding: [0x00,0x00,0x08,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x08,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_rsub_u32 v1, v2 offset:65535         ; encoding: [0xff,0xff,0x08,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x08,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_rsub_u32 v255, v255 offset:4         ; encoding: [0x04,0x00,0x08,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x08,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_rsub_u64 v1, v[2:3]                  ; encoding: [0x00,0x00,0x08,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x08,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_rsub_u64 v1, v[2:3] offset:65535     ; encoding: [0xff,0xff,0x08,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x08,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_rsub_u64 v255, v[254:255] offset:4   ; encoding: [0x04,0x00,0x08,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x08,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_store_2addr_b32 v1, v2, v3           ; encoding: [0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00]
+0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00
+
+# GFX1250: ds_store_2addr_b32 v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd8,0x01,0x02,0x03,0x00]
+0x7f,0xff,0x38,0xd8,0x01,0x02,0x03,0x00
+
+# GFX1250: ds_store_2addr_b32 v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0x38,0xd8,0xff,0xff,0xff,0x00]
+0x10,0x01,0x38,0xd8,0xff,0xff,0xff,0x00
+
+# GFX1250: ds_store_2addr_b64 v1, v[2:3], v[4:5]   ; encoding: [0x00,0x00,0x38,0xd9,0x01,0x02,0x04,0x00]
+0x00,0x00,0x38,0xd9,0x01,0x02,0x04,0x00
+
+# GFX1250: ds_store_2addr_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd9,0x01,0x02,0x04,0x00]
+0x7f,0xff,0x38,0xd9,0x01,0x02,0x04,0x00
+
+# GFX1250: ds_store_2addr_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0x38,0xd9,0xff,0xfe,0xfe,0x00]
+0x10,0x01,0x38,0xd9,0xff,0xfe,0xfe,0x00
+
+# GFX1250: ds_store_2addr_stride64_b32 v1, v2, v3  ; encoding: [0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00]
+0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00
+
+# GFX1250: ds_store_2addr_stride64_b32 v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd8,0x01,0x02,0x03,0x00]
+0x7f,0xff,0x3c,0xd8,0x01,0x02,0x03,0x00
+
+# GFX1250: ds_store_2addr_stride64_b32 v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0x3c,0xd8,0xff,0xff,0xff,0x00]
+0x10,0x01,0x3c,0xd8,0xff,0xff,0xff,0x00
+
+# GFX1250: ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0x3c,0xd9,0x01,0x02,0x04,0x00]
+0x00,0x00,0x3c,0xd9,0x01,0x02,0x04,0x00
+
+# GFX1250: ds_store_2addr_stride64_b64 v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd9,0x01,0x02,0x04,0x00]
+0x7f,0xff,0x3c,0xd9,0x01,0x02,0x04,0x00
+
+# GFX1250: ds_store_2addr_stride64_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0x3c,0xd9,0xff,0xfe,0xfe,0x00]
+0x10,0x01,0x3c,0xd9,0xff,0xfe,0xfe,0x00
+
+# GFX1250: ds_store_addtid_b32 v1                  ; encoding: [0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00]
+0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00
+
+# GFX1250: ds_store_addtid_b32 v1 offset:65535     ; encoding: [0xff,0xff,0xc0,0xda,0x00,0x01,0x00,0x00]
+0xff,0xff,0xc0,0xda,0x00,0x01,0x00,0x00
+
+# GFX1250: ds_store_addtid_b32 v255 offset:4       ; encoding: [0x04,0x00,0xc0,0xda,0x00,0xff,0x00,0x00]
+0x04,0x00,0xc0,0xda,0x00,0xff,0x00,0x00
+
+# GFX1250: ds_store_b128 v1, v[2:5]                ; encoding: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b128 v1, v[2:5] offset:65535   ; encoding: [0xff,0xff,0x7c,0xdb,0x01,0x02,0x00,0x00]
+0xff,0xff,0x7c,0xdb,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b128 v255, v[252:255] offset:4 ; encoding: [0x04,0x00,0x7c,0xdb,0xff,0xfc,0x00,0x00]
+0x04,0x00,0x7c,0xdb,0xff,0xfc,0x00,0x00
+
+# GFX1250: ds_store_b16 v1, v2                     ; encoding: [0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b16 v1, v2 offset:65535        ; encoding: [0xff,0xff,0x7c,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x7c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b16 v255, v255 offset:4        ; encoding: [0x04,0x00,0x7c,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x7c,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_store_b16_d16_hi v1, v2              ; encoding: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b16_d16_hi v1, v2 offset:65535 ; encoding: [0xff,0xff,0x84,0xda,0x01,0x02,0x00,0x00]
+0xff,0xff,0x84,0xda,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b16_d16_hi v255, v255 offset:4 ; encoding: [0x04,0x00,0x84,0xda,0xff,0xff,0x00,0x00]
+0x04,0x00,0x84,0xda,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_store_b32 v1, v2                     ; encoding: [0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b32 v1, v2 offset:65535        ; encoding: [0xff,0xff,0x34,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x34,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b32 v255, v255 offset:4        ; encoding: [0x04,0x00,0x34,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x34,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_store_b64 v1, v[2:3]                 ; encoding: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b64 v1, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x34,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x34,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b64 v255, v[254:255] offset:4  ; encoding: [0x04,0x00,0x34,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x34,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_store_b8 v1, v2                      ; encoding: [0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b8 v1, v2 offset:65535         ; encoding: [0xff,0xff,0x78,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x78,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b8 v255, v255 offset:4         ; encoding: [0x04,0x00,0x78,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x78,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_store_b8_d16_hi v1, v2               ; encoding: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b8_d16_hi v1, v2 offset:65535  ; encoding: [0xff,0xff,0x80,0xda,0x01,0x02,0x00,0x00]
+0xff,0xff,0x80,0xda,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b8_d16_hi v255, v255 offset:4  ; encoding: [0x04,0x00,0x80,0xda,0xff,0xff,0x00,0x00]
+0x04,0x00,0x80,0xda,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_store_b96 v1, v[2:4]                 ; encoding: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b96 v1, v[2:4] offset:65535    ; encoding: [0xff,0xff,0x78,0xdb,0x01,0x02,0x00,0x00]
+0xff,0xff,0x78,0xdb,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_store_b96 v255, v[252:254] offset:4  ; encoding: [0x04,0x00,0x78,0xdb,0xff,0xfc,0x00,0x00]
+0x04,0x00,0x78,0xdb,0xff,0xfc,0x00,0x00
+
+# GFX1250: ds_storexchg_2addr_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xb8,0xd8,0xff,0xff,0xff,0xfe]
+0x10,0x01,0xb8,0xd8,0xff,0xff,0xff,0xfe
+
+# GFX1250: ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x06]
+0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x06
+
+# GFX1250: ds_storexchg_2addr_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0x06]
+0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0x06
+
+# GFX1250: ds_storexchg_2addr_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0xb8,0xd9,0xff,0xfe,0xfe,0xfc]
+0x10,0x01,0xb8,0xd9,0xff,0xfe,0xfe,0xfc
+
+# GFX1250: ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xb8,0xd9,0x01,0x02,0x04,0x06]
+0x00,0x00,0xb8,0xd9,0x01,0x02,0x04,0x06
+
+# GFX1250: ds_storexchg_2addr_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd9,0x01,0x02,0x04,0x06]
+0x7f,0xff,0xb8,0xd9,0x01,0x02,0x04,0x06
+
+# GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1 ; encoding: [0x10,0x01,0xbc,0xd8,0xff,0xff,0xff,0xfe]
+0x10,0x01,0xbc,0xd8,0xff,0xff,0xff,0xfe
+
+# GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 ; encoding: [0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x06]
+0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x06
+
+# GFX1250: ds_storexchg_2addr_stride64_rtn_b32 v[6:7], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0x06]
+0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0x06
+
+# GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1 ; encoding: [0x10,0x01,0xbc,0xd9,0xff,0xfe,0xfe,0xfc]
+0x10,0x01,0xbc,0xd9,0xff,0xfe,0xfe,0xfc
+
+# GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] ; encoding: [0x00,0x00,0xbc,0xd9,0x01,0x02,0x04,0x06]
+0x00,0x00,0xbc,0xd9,0x01,0x02,0x04,0x06
+
+# GFX1250: ds_storexchg_2addr_stride64_rtn_b64 v[6:9], v1, v[2:3], v[4:5] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd9,0x01,0x02,0x04,0x06]
+0x7f,0xff,0xbc,0xd9,0x01,0x02,0x04,0x06
+
+# GFX1250: ds_storexchg_rtn_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xb4,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0xb4,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_storexchg_rtn_b32 v5, v1, v2         ; encoding: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_storexchg_rtn_b32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xb4,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0xb4,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_storexchg_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xb4,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0xb4,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] ; encoding: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_storexchg_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_sub_clamp_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xa4,0xda,0xff,0xff,0x00,0xff]
+0x04,0x00,0xa4,0xda,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_sub_clamp_rtn_u32 v5, v1, v2         ; encoding: [0x00,0x00,0xa4,0xda,0x01,0x02,0x00,0x05]
+0x00,0x00,0xa4,0xda,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_sub_clamp_rtn_u32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xa4,0xda,0x01,0x02,0x00,0x05]
+0xff,0xff,0xa4,0xda,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_sub_clamp_u32 v1, v2                 ; encoding: [0x00,0x00,0x64,0xda,0x01,0x02,0x00,0x00]
+0x00,0x00,0x64,0xda,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_sub_clamp_u32 v1, v2 offset:65535    ; encoding: [0xff,0xff,0x64,0xda,0x01,0x02,0x00,0x00]
+0xff,0xff,0x64,0xda,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_sub_clamp_u32 v255, v255 offset:4    ; encoding: [0x04,0x00,0x64,0xda,0xff,0xff,0x00,0x00]
+0x04,0x00,0x64,0xda,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_sub_rtn_u32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0x84,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0x84,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_sub_rtn_u32 v5, v1, v2               ; encoding: [0x00,0x00,0x84,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0x84,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_sub_rtn_u32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x84,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0x84,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_sub_rtn_u64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0x84,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0x84,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_sub_rtn_u64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0x84,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0x84,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_sub_rtn_u64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x84,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0x84,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_sub_u32 v1, v2                       ; encoding: [0x00,0x00,0x04,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x04,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_sub_u32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x04,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x04,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_sub_u32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x04,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x04,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_sub_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x04,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x04,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_sub_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x04,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x04,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_sub_u64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x04,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x04,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_swizzle_b32 v8, v2                   ; encoding: [0x00,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0x00,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(FFT,31) ; encoding: [0xff,0xff,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0xff,0xff,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(BITMASK_PERM,"01pip") ; encoding: [0x07,0x09,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0x07,0x09,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,4,1) ; encoding: [0x3c,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0x3c,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,8,7) ; encoding: [0xf8,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0xf8,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(QUAD_PERM,0,1,2,3) ; encoding: [0xe4,0x80,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0xe4,0x80,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(REVERSE,8) ; encoding: [0x1f,0x1c,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0x1f,0x1c,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX1250: ds_swizzle_b32 v8, v2 offset:swizzle(SWAP,16) ; encoding: [0x1f,0x40,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0x1f,0x40,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX1250: ds_xor_b32 v1, v2                       ; encoding: [0x00,0x00,0x2c,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x2c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_xor_b32 v1, v2 offset:65535          ; encoding: [0xff,0xff,0x2c,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x2c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_xor_b32 v255, v255 offset:4          ; encoding: [0x04,0x00,0x2c,0xd8,0xff,0xff,0x00,0x00]
+0x04,0x00,0x2c,0xd8,0xff,0xff,0x00,0x00
+
+# GFX1250: ds_xor_b64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_xor_b64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x2c,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x2c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX1250: ds_xor_b64 v255, v[254:255] offset:4    ; encoding: [0x04,0x00,0x2c,0xd9,0xff,0xfe,0x00,0x00]
+0x04,0x00,0x2c,0xd9,0xff,0xfe,0x00,0x00
+
+# GFX1250: ds_xor_rtn_b32 v255, v255, v255 offset:4 ; encoding: [0x04,0x00,0xac,0xd8,0xff,0xff,0x00,0xff]
+0x04,0x00,0xac,0xd8,0xff,0xff,0x00,0xff
+
+# GFX1250: ds_xor_rtn_b32 v5, v1, v2               ; encoding: [0x00,0x00,0xac,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0xac,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_xor_rtn_b32 v5, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xac,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0xac,0xd8,0x01,0x02,0x00,0x05
+
+# GFX1250: ds_xor_rtn_b64 v[254:255], v255, v[254:255] offset:4 ; encoding: [0x04,0x00,0xac,0xd9,0xff,0xfe,0x00,0xfe]
+0x04,0x00,0xac,0xd9,0xff,0xfe,0x00,0xfe
+
+# GFX1250: ds_xor_rtn_b64 v[6:7], v1, v[2:3]       ; encoding: [0x00,0x00,0xac,0xd9,0x01,0x02,0x00,0x06]
+0x00,0x00,0xac,0xd9,0x01,0x02,0x00,0x06
+
+# GFX1250: ds_xor_rtn_b64 v[6:7], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xac,0xd9,0x01,0x02,0x00,0x06]
+0xff,0xff,0xac,0xd9,0x01,0x02,0x00,0x06
+
 # GFX1250: ds_atomic_async_barrier_arrive_b64 v1 offset:65407 ; encoding: [0x7f,0xff,0x58,0xd9,0x01,0x00,0x00,0x00]
 0x7f,0xff,0x58,0xd9,0x01,0x00,0x00,0x00
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_operands.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_operands.txt
index a3e7e570ab9b6..361c49b344805 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_operands.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_operands.txt
@@ -20,3 +20,18 @@
 
 # GFX1250: s_mov_b64 s[0:1], src_shared_limit      ; encoding: [0xec,0x01,0x80,0xbe]
 0xec,0x01,0x80,0xbe
+
+# GFX1250: s_getreg_b32 s1, hwreg(HW_REG_XNACK_STATE_PRIV) ; encoding: [0x21,0xf8,0x81,0xb8]
+0x21,0xf8,0x81,0xb8
+
+# GFX1250: s_getreg_b32 s1, hwreg(HW_REG_XNACK_MASK) ; encoding: [0x22,0xf8,0x81,0xb8]
+0x22,0xf8,0x81,0xb8
+
+# GFX1250: s_setreg_b32 hwreg(HW_REG_XNACK_STATE_PRIV), s1 ; encoding: [0x21,0xf8,0x01,0xb9]
+0x21,0xf8,0x01,0xb9
+
+# GFX1250: s_setreg_b32 hwreg(HW_REG_XNACK_MASK), s1 ; encoding: [0x22,0xf8,0x01,0xb9]
+0x22,0xf8,0x01,0xb9
+
+# GFX1250: s_setreg_b32 hwreg(HW_REG_IB_STS2), s1  ; encoding: [0x1c,0xf8,0x01,0xb9]
+0x1c,0xf8,0x01,0xb9
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sop1.txt
index 83fa647696d6c..07aca1e40b071 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sop1.txt
@@ -12,6 +12,9 @@
 # GFX1250: s_add_pc_i64 s[2:3]                     ; encoding: [0x02,0x4b,0x80,0xbe]
 0x02,0x4b,0x80,0xbe
 
+# GFX1250: s_get_shader_cycles_u64 s[2:3]          ; encoding: [0x00,0x06,0x82,0xbe]
+0x00,0x06,0x82,0xbe
+
 # GFX1250: s_barrier_signal -3                     ; encoding: [0xc3,0x4e,0x80,0xbe]
 0xc3,0x4e,0x80,0xbe
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
index e7026df3c0e2b..af94fbc7824a5 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
@@ -35,3 +35,9 @@
 
 # GFX1250: s_monitor_sleep 1                       ; encoding: [0x01,0x00,0x84,0xbf]
 0x01,0x00,0x84,0xbf
+
+# GFX1250: s_sendmsg sendmsg(MSG_SAVEWAVE_HAS_TDM)     ; encoding: [0x0a,0x00,0xb6,0xbf]
+0x0a,0x00,0xb6,0xbf
+
+# GFX1250: s_barrier_wait 0xfffd                   ; encoding: [0xfd,0xff,0x94,0xbf]
+0xfd,0xff,0x94,0xbf
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vbuffer_mubuf.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vbuffer_mubuf.txt
index 2499225626acc..ddf779adc9e15 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vbuffer_mubuf.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vbuffer_mubuf.txt
@@ -1,5 +1,2138 @@
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
 
+# GFX1250: buffer_atomic_add_f32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_f32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_add_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_and_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b32 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[252:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cmpswap_b64 v[6:9], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_cond_sub_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_dec_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_inc_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_i64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_max_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_i64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_min_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_or_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x16,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x16,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x16,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x16,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x16,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x16,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_bf16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x16,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x16,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x16,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x16,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x16,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x16,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x16,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_pk_add_f16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x16,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0xff,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0xff,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0xff,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x18,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x18,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0xc0,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_sub_u64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_swap_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x06,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x06,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_atomic_xor_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[252:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[6:9], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[6:9], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x06,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[6:9], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[6:9], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[6:9], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b128 v[6:9], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v5, off, s[8:11], s3    ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_b32 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[6:7], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[6:7], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x06,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[6:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[6:7], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[6:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b64 v[6:7], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[252:254], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[6:8], off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x06,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[6:8], off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x06,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[6:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x06,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[6:8], off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x06,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[6:8], v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_b96 v[6:8], v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x06,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_b16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_i8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_d16_u8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v5, off, s[8:11], s3    ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_i16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v5, off, s[8:11], s3     ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_i8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_i8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v5, off, s[8:11], s3    ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_u16 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v255, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v5, off, s[12:15], s3 offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v5, off, s[8:11], m0 offset:8388607 ; encoding: [0x7d,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v5, off, s[8:11], s101 offset:8388607 ; encoding: [0x65,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v5, off, s[8:11], s3     ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_load_u8 v5, off, s[8:11], s3 offset:7 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v5, off, s[96:99], s3 offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_load_u8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[252:255], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[2:5], off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 offset:7 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x02,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[2:5], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x02,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[2:5], off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x02,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[2:5], off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x02,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[2:5], v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b128 v[2:5], v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x02,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v1, off, s[12:15], s4  ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_store_b16 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b16 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v1, off, s[12:15], s4  ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_store_b32 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b32 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x80,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[254:255], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[2:3], off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 offset:7 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x02,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[2:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x02,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[2:3], off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x02,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[2:3], off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x02,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[2:3], v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b64 v[2:3], v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x02,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v1, off, s[12:15], s4   ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_store_b8 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b8 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[252:254], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[2:4], off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 offset:7 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x02,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[2:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x02,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[2:4], off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x02,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[2:4], off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x02,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[2:4], v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_b96 v[2:4], v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x02,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b16 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x40,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], m0 offset:8388607 ; encoding: [0x7d,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s101 offset:8388607 ; encoding: [0x65,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:7 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v1, off, s[16:19], s4 offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v1, off, s[96:99], s4 offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX1250: buffer_store_d16_hi_b8 v255, off, s[12:15], s4 offset:8388607 ; encoding: [0x04,0x00,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
 # GFX1250: buffer_atomic_and_b32 v5, v1, s[8:11], s3 offen offset:4095 nv ; encoding: [0x83,0x00,0x0f,0xc4,0x05,0x10,0x80,0x40,0x01,0xff,0x0f,0x00]
 0x83,0x00,0x0f,0xc4,0x05,0x10,0x80,0x40,0x01,0xff,0x0f,0x00
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_dpp16.txt
index 5fa7bc8470d39..9a731319177c6 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_dpp16.txt
@@ -1,6 +1,172 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
+
+0xff,0x04,0x33,0xd6,0xfa,0xfe,0xf7,0xab,0xff,0x6f,0x0d,0x30
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v255.l, v255.l, v255.l, src_scc bitop3:0x65 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x04,0x33,0xd6,0xfa,0xfe,0xf7,0xab,0xff,0x6f,0x0d,0x30]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v255, v255, v255, src_scc bitop3:0x65 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x04,0x33,0xd6,0xfa,0xfe,0xf7,0xab,0xff,0x6f,0x0d,0x30]
+
+0xff,0x45,0x33,0xd6,0xfa,0xfe,0xf7,0x0b,0xff,0x6f,0x0d,0x30
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v255.h, v255.l, v255.l, src_scc bitop3:0x68 op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x45,0x33,0xd6,0xfa,0xfe,0xf7,0x0b,0xff,0x6f,0x0d,0x30]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v255, v255, v255, src_scc bitop3:0x68 op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x45,0x33,0xd6,0xfa,0xfe,0xf7,0x0b,0xff,0x6f,0x0d,0x30]
+
+0x05,0x04,0x33,0xd6,0xfa,0x04,0x06,0x6b,0x01,0x60,0x09,0x13
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, -1 bitop3:0x63 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0x06,0x6b,0x01,0x60,0x09,0x13]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, -1 bitop3:0x63 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0x06,0x6b,0x01,0x60,0x09,0x13]
+
+0x05,0x25,0x33,0xd6,0xfa,0x04,0x06,0x0b,0x01,0x60,0x01,0x13
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, -1 bitop3:0x68 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x25,0x33,0xd6,0xfa,0x04,0x06,0x0b,0x01,0x60,0x01,0x13]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, -1 bitop3:0x68 op_sel:[0,0,1,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x25,0x33,0xd6,0xfa,0x04,0x06,0x0b,0x01,0x60,0x01,0x13]
+
+0x05,0x7c,0x33,0xd6,0xfa,0x04,0xfe,0xc9,0x01,0x2f,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.h, v1.h, v2.h, exec_hi bitop3:0x66 op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x33,0xd6,0xfa,0x04,0xfe,0xc9,0x01,0x2f,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_hi bitop3:0x66 op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x33,0xd6,0xfa,0x04,0xfe,0xc9,0x01,0x2f,0x01,0xff]
+
+0x05,0x00,0x33,0xd6,0xfa,0x04,0xfe,0xc1,0x01,0x2f,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, exec_hi bitop3:6 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0xfe,0xc1,0x01,0x2f,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_hi bitop3:6 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0xfe,0xc1,0x01,0x2f,0x01,0xff]
+
+0x05,0x78,0x33,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x33,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x78,0x33,0xd6,0xfa,0x04,0xfe,0x01,0x01,0x2f,0x01,0xff]
+
+0x05,0x01,0x33,0xd6,0xfa,0x04,0xfa,0xa9,0x01,0x50,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, exec_lo bitop3:0x4d row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x33,0xd6,0xfa,0x04,0xfa,0xa9,0x01,0x50,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_lo bitop3:0x4d row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x33,0xd6,0xfa,0x04,0xfa,0xa9,0x01,0x50,0x01,0xff]
+
+0x05,0x0c,0x33,0xd6,0xfa,0x04,0xfa,0xe9,0x01,0x50,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.h, v2.l, exec_lo bitop3:0x67 op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x33,0xd6,0xfa,0x04,0xfa,0xe9,0x01,0x50,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_lo bitop3:0x67 op_sel:[1,0,0,0] row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x0c,0x33,0xd6,0xfa,0x04,0xfa,0xe9,0x01,0x50,0x01,0xff]
+
+0x05,0x00,0x33,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_lo row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0xfa,0x01,0x01,0x50,0x01,0xff]
+
+0x05,0x03,0x33,0xd6,0xfa,0x04,0xf2,0x09,0x01,0x5f,0x01,0x01
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, null bitop3:0x58 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x03,0x33,0xd6,0xfa,0x04,0xf2,0x09,0x01,0x5f,0x01,0x01]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, null bitop3:0x58 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x03,0x33,0xd6,0xfa,0x04,0xf2,0x09,0x01,0x5f,0x01,0x01]
+
+0x05,0x15,0x33,0xd6,0xfa,0x04,0xf2,0x09,0x01,0x5f,0x01,0x01
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.h, null bitop3:0x68 op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x33,0xd6,0xfa,0x04,0xf2,0x09,0x01,0x5f,0x01,0x01]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, null bitop3:0x68 op_sel:[0,1,0,0] row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x15,0x33,0xd6,0xfa,0x04,0xf2,0x09,0x01,0x5f,0x01,0x01]
+
+0x05,0x02,0x33,0xd6,0xfa,0x04,0xa6,0xc1,0x01,0x0f,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, s105 bitop3:0x16 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x33,0xd6,0xfa,0x04,0xa6,0xc1,0x01,0x0f,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, s105 bitop3:0x16 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x33,0xd6,0xfa,0x04,0xa6,0xc1,0x01,0x0f,0x01,0xff]
+
+0x05,0x00,0x33,0xd6,0xfa,0x04,0xee,0xa1,0x01,0x21,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, ttmp15 bitop3:5 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0xee,0xa1,0x01,0x21,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, ttmp15 bitop3:5 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0xee,0xa1,0x01,0x21,0x01,0xff]
+
+0x05,0x00,0x33,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, v255.l row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, v255 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x01,0x01,0xff]
+
+0x05,0x04,0x33,0xd6,0xfa,0x04,0x0e,0xe4,0x01,0x40,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, v3.l bitop3:0x27 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0x0e,0xe4,0x01,0x40,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, v3 bitop3:0x27 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0x0e,0xe4,0x01,0x40,0x01,0xff]
+
+0x05,0x04,0x33,0xd6,0xfa,0x04,0x0e,0x8c,0x01,0x41,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, v3.l bitop3:0x64 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0x0e,0x8c,0x01,0x41,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, v3 bitop3:0x64 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0x0e,0x8c,0x01,0x41,0x01,0xff]
+
+0x05,0x04,0x33,0xd6,0xfa,0x04,0x0e,0x34,0x01,0xe4,0x00,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, v3.l bitop3:0xa1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0x0e,0x34,0x01,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, v3 bitop3:0xa1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0x0e,0x34,0x01,0xe4,0x00,0xff]
+
+0x05,0x00,0x33,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x33,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+0x05,0x07,0x33,0xd6,0xfa,0x04,0xae,0xe1,0x01,0x11,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, vcc_hi bitop3:0x3f row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x07,0x33,0xd6,0xfa,0x04,0xae,0xe1,0x01,0x11,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, vcc_hi bitop3:0x3f row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x07,0x33,0xd6,0xfa,0x04,0xae,0xe1,0x01,0x11,0x01,0xff]
+
+0x05,0x04,0x33,0xd6,0xfa,0x04,0xaa,0x81,0x01,0x1f,0x01,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, vcc_lo bitop3:0x24 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0xaa,0x81,0x01,0x1f,0x01,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, vcc_lo bitop3:0x24 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x33,0xd6,0xfa,0x04,0xaa,0x81,0x01,0x1f,0x01,0xff]
+
+0x05,0x7c,0x33,0xd6,0xfa,0x04,0x0e,0xcc,0x01,0xe4,0x00,0xff
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.h, v1.h, v2.h, v3.h bitop3:0x66 op_sel:[1,1,1,1] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x33,0xd6,0xfa,0x04,0x0e,0xcc,0x01,0xe4,0x00,0xff]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, v3 bitop3:0x66 op_sel:[1,1,1,1] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x7c,0x33,0xd6,0xfa,0x04,0x0e,0xcc,0x01,0xe4,0x00,0xff]
+
+0xff,0x04,0x34,0xd6,0xfa,0xfe,0xf7,0xab,0xff,0x6f,0x0d,0x30
+# GFX1250: v_bitop3_b32_e64_dpp v255, v255, v255, src_scc bitop3:0x65 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x04,0x34,0xd6,0xfa,0xfe,0xf7,0xab,0xff,0x6f,0x0d,0x30]
+
+0x05,0x03,0x34,0xd6,0xfa,0x04,0x06,0x0b,0x01,0x5f,0x01,0x01
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, -1 bitop3:0x58 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x03,0x34,0xd6,0xfa,0x04,0x06,0x0b,0x01,0x5f,0x01,0x01]
+
+0x05,0x04,0x34,0xd6,0xfa,0x04,0xc2,0x6b,0x01,0x60,0x09,0x13
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, 0.5 bitop3:0x63 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x04,0x34,0xd6,0xfa,0x04,0xc2,0x6b,0x01,0x60,0x09,0x13]
+
+0x05,0x00,0x34,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x21,0x01,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, exec_hi bitop3:5 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd6,0xfa,0x04,0xfe,0xa1,0x01,0x21,0x01,0xff]
+
+0x05,0x00,0x34,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x2f,0x01,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, exec_lo bitop3:6 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd6,0xfa,0x04,0xfa,0xc1,0x01,0x2f,0x01,0xff]
+
+0x05,0x01,0x34,0xd6,0xfa,0x04,0xf2,0xa9,0x01,0x50,0x01,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, null bitop3:0x4d row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x34,0xd6,0xfa,0x04,0xf2,0xa9,0x01,0x50,0x01,0xff]
+
+0x05,0x00,0x34,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+0x05,0x04,0x34,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, ttmp15 bitop3:0x24 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x34,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+0x05,0x04,0x34,0xd6,0xfa,0x04,0xfe,0x8f,0x01,0x41,0x01,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, v255 bitop3:0x64 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x34,0xd6,0xfa,0x04,0xfe,0x8f,0x01,0x41,0x01,0xff]
+
+0x05,0x04,0x34,0xd6,0xfa,0x04,0x0e,0xe4,0x01,0x40,0x01,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, v3 bitop3:0x27 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x34,0xd6,0xfa,0x04,0x0e,0xe4,0x01,0x40,0x01,0xff]
+
+0x05,0x04,0x34,0xd6,0xfa,0x04,0x0e,0x34,0x01,0xe4,0x00,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, v3 bitop3:0xa1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x34,0xd6,0xfa,0x04,0x0e,0x34,0x01,0xe4,0x00,0xff]
+
+0x05,0x00,0x34,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x34,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+0x05,0x02,0x34,0xd6,0xfa,0x04,0xae,0xa1,0x01,0x0f,0x01,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, vcc_hi bitop3:0x15 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x34,0xd6,0xfa,0x04,0xae,0xa1,0x01,0x0f,0x01,0xff]
+
+0x05,0x07,0x34,0xd6,0xfa,0x04,0xaa,0xe1,0x01,0x11,0x01,0xff
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, vcc_lo bitop3:0x3f row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x07,0x34,0xd6,0xfa,0x04,0xaa,0xe1,0x01,0x11,0x01,0xff]
+
+0x02,0x00,0x60,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x79,0x00,0xff
+# GFX1250: v_add_min_i32_e64_dpp v2, v4, v7, v8 quad_perm:[1,2,3,1] row_mask:0xf bank_mask:0xf ; encoding: [0x02,0x00,0x60,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x79,0x00,0xff]
+
+0x02,0x00,0x60,0xd6,0xfa,0x0e,0x06,0x02,0x04,0x50,0x01,0xff
+# GFX1250: v_add_min_i32_e64_dpp v2, v4, v7, 1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x02,0x00,0x60,0xd6,0xfa,0x0e,0x06,0x02,0x04,0x50,0x01,0xff]
+
+0x02,0x00,0x60,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x53,0x05,0xff
+# GFX1250: v_add_min_i32_e64_dpp v2, v4, v7, v8 row_share:3 row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x02,0x00,0x60,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x53,0x05,0xff]
+
+0x02,0x00,0x5e,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x1b,0x00,0xff
+# GFX1250: v_add_max_i32_e64_dpp v2, v4, v7, v8 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x02,0x00,0x5e,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x1b,0x00,0xff]
+
+0x02,0x00,0x5e,0xd6,0xfa,0x0e,0x06,0x02,0x04,0x50,0x01,0xff
+# GFX1250: v_add_max_i32_e64_dpp v2, v4, v7, 1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x02,0x00,0x5e,0xd6,0xfa,0x0e,0x06,0x02,0x04,0x50,0x01,0xff]
+
+0x02,0x00,0x5e,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x53,0x05,0xff
+# GFX1250: v_add_max_i32_e64_dpp v2, v4, v7, v8 row_share:3 row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x02,0x00,0x5e,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x53,0x05,0xff]
+
+0x02,0x00,0x61,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x1b,0x00,0xff
+# GFX1250: v_add_min_u32_e64_dpp v2, v4, v7, v8 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x02,0x00,0x61,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x1b,0x00,0xff]
+
+0x02,0x00,0x61,0xd6,0xfa,0x0e,0x06,0x02,0x04,0x50,0x01,0xff
+# GFX1250: v_add_min_u32_e64_dpp v2, v4, v7, 1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x02,0x00,0x61,0xd6,0xfa,0x0e,0x06,0x02,0x04,0x50,0x01,0xff]
+
+0x02,0x00,0x61,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x53,0x05,0xff
+# GFX1250: v_add_min_u32_e64_dpp v2, v4, v7, v8 row_share:3 row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x02,0x00,0x61,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x53,0x05,0xff]
+
+0x02,0x00,0x5f,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x1b,0x00,0xff
+# GFX1250: v_add_max_u32_e64_dpp v2, v4, v7, v8 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x02,0x00,0x5f,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x1b,0x00,0xff]
+
+0x02,0x00,0x5f,0xd6,0xfa,0x0e,0x06,0x02,0x04,0x50,0x01,0xff
+# GFX1250: v_add_max_u32_e64_dpp v2, v4, v7, 1 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x02,0x00,0x5f,0xd6,0xfa,0x0e,0x06,0x02,0x04,0x50,0x01,0xff]
+
+0x02,0x00,0x5f,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x53,0x05,0xff
+# GFX1250: v_add_max_u32_e64_dpp v2, v4, v7, v8 row_share:3 row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x02,0x00,0x5f,0xd6,0xfa,0x0e,0x22,0x04,0x04,0x53,0x05,0xff]
 
 0xff,0x04,0x33,0xd6,0xfa,0xfe,0xf7,0xab,0xff,0x6f,0x0d,0x30
 # GFX1250-REAL16: v_bitop3_b16_e64_dpp v255.l, v255.l, v255.l, src_scc bitop3:0x65 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x04,0x33,0xd6,0xfa,0xfe,0xf7,0xab,0xff,0x6f,0x0d,0x30]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_dpp8.txt
index faeff45a9992d..93c7f00da1c12 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3_dpp8.txt
@@ -1,6 +1,138 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
+
+0xff,0x01,0x33,0xd6,0xe9,0xfe,0xf7,0xab,0xff,0x00,0x00,0x00
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v255.l, v255.l, v255.l, src_scc bitop3:0x4d dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x01,0x33,0xd6,0xe9,0xfe,0xf7,0xab,0xff,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v255, v255, v255, src_scc bitop3:0x4d dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x01,0x33,0xd6,0xe9,0xfe,0xf7,0xab,0xff,0x00,0x00,0x00]
+
+0xff,0x44,0x33,0xd6,0xea,0xfe,0xf7,0xeb,0xff,0x00,0x00,0x00
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v255.h, v255.l, v255.l, src_scc bitop3:0x67 op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x44,0x33,0xd6,0xea,0xfe,0xf7,0xeb,0xff,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v255, v255, v255, src_scc bitop3:0x67 op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x44,0x33,0xd6,0xea,0xfe,0xf7,0xeb,0xff,0x00,0x00,0x00]
+
+0x05,0x24,0x33,0xd6,0xe9,0x04,0x06,0xcb,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, -1 bitop3:0x66 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x24,0x33,0xd6,0xe9,0x04,0x06,0xcb,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, -1 bitop3:0x66 op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x24,0x33,0xd6,0xe9,0x04,0x06,0xcb,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x33,0xd6,0xea,0x04,0x06,0xc3,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, -1 bitop3:6 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x33,0xd6,0xea,0x04,0x06,0xc3,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, -1 bitop3:6 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x33,0xd6,0xea,0x04,0x06,0xc3,0x01,0x77,0x39,0x05]
+
+0x05,0x07,0x33,0xd6,0xe9,0x04,0xfe,0xe1,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, exec_hi bitop3:0x3f dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x07,0x33,0xd6,0xe9,0x04,0xfe,0xe1,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_hi bitop3:0x3f dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x07,0x33,0xd6,0xe9,0x04,0xfe,0xe1,0x01,0x77,0x39,0x05]
+
+0x05,0x7b,0x33,0xd6,0xe9,0x04,0xfe,0x09,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.h, v1.h, v2.h, exec_hi bitop3:0x58 op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7b,0x33,0xd6,0xe9,0x04,0xfe,0x09,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_hi bitop3:0x58 op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x7b,0x33,0xd6,0xe9,0x04,0xfe,0x09,0x01,0x77,0x39,0x05]
+
+0x05,0x78,0x33,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.h, v1.h, v2.h, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x33,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_hi op_sel:[1,1,1,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x78,0x33,0xd6,0xe9,0x04,0xfe,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x33,0xd6,0xe9,0x04,0xfa,0x81,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, exec_lo bitop3:0x24 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x33,0xd6,0xe9,0x04,0xfa,0x81,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_lo bitop3:0x24 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x33,0xd6,0xe9,0x04,0xfa,0x81,0x01,0x77,0x39,0x05]
+
+0x05,0x0c,0x33,0xd6,0xe9,0x04,0xfa,0x69,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.h, v2.l, exec_lo bitop3:0x63 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x33,0xd6,0xe9,0x04,0xfa,0x69,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, exec_lo bitop3:0x63 op_sel:[1,0,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x0c,0x33,0xd6,0xe9,0x04,0xfa,0x69,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x33,0xd6,0xe9,0x04,0xf2,0xa1,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, null bitop3:5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd6,0xe9,0x04,0xf2,0xa1,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, null bitop3:5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd6,0xe9,0x04,0xf2,0xa1,0x01,0x77,0x39,0x05]
+
+0x05,0x10,0x33,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.h, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x33,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, null op_sel:[0,1,0,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x10,0x33,0xd6,0xe9,0x04,0xf2,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x33,0xd6,0xe9,0x04,0xa6,0xe1,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, s105 bitop3:0x27 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x33,0xd6,0xe9,0x04,0xa6,0xe1,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, s105 bitop3:0x27 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x33,0xd6,0xe9,0x04,0xa6,0xe1,0x01,0x77,0x39,0x05]
+
+0x05,0x01,0x33,0xd6,0xe9,0x04,0xee,0xe1,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, ttmp15 bitop3:0xf dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x33,0xd6,0xe9,0x04,0xee,0xe1,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, ttmp15 bitop3:0xf dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x33,0xd6,0xe9,0x04,0xee,0xe1,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x33,0xd6,0xe9,0x04,0xfe,0x37,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, v255.l bitop3:0xa1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x33,0xd6,0xe9,0x04,0xfe,0x37,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, v255 bitop3:0xa1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x33,0xd6,0xe9,0x04,0xfe,0x37,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x33,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, v3.l dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x33,0xd6,0xe9,0x04,0xae,0x89,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, vcc_hi bitop3:0x64 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x33,0xd6,0xe9,0x04,0xae,0x89,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, vcc_hi bitop3:0x64 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x33,0xd6,0xe9,0x04,0xae,0x89,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x33,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.l, v1.l, v2.l, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x33,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x7c,0x33,0xd6,0xe9,0x04,0x0e,0xcc,0x01,0x00,0x00,0x00
+# GFX1250-REAL16: v_bitop3_b16_e64_dpp v5.h, v1.h, v2.h, v3.h bitop3:0x66 op_sel:[1,1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x05,0x7c,0x33,0xd6,0xe9,0x04,0x0e,0xcc,0x01,0x00,0x00,0x00]
+# GFX1250-FAKE16: v_bitop3_b16_e64_dpp v5, v1, v2, v3 bitop3:0x66 op_sel:[1,1,1,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0x05,0x7c,0x33,0xd6,0xe9,0x04,0x0e,0xcc,0x01,0x00,0x00,0x00]
+
+0xff,0x03,0x34,0xd6,0xe9,0xfe,0xf7,0x0b,0xff,0x00,0x00,0x00
+# GFX1250: v_bitop3_b32_e64_dpp v255, v255, v255, src_scc bitop3:0x58 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x34,0xd6,0xe9,0xfe,0xf7,0x0b,0xff,0x00,0x00,0x00]
+
+0x05,0x00,0x34,0xd6,0xe9,0x04,0x06,0xc3,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, -1 bitop3:6 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd6,0xe9,0x04,0x06,0xc3,0x01,0x77,0x39,0x05]
+
+0x05,0x01,0x34,0xd6,0xea,0x04,0xc2,0xab,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, 0.5 bitop3:0x4d dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x01,0x34,0xd6,0xea,0x04,0xc2,0xab,0x01,0x77,0x39,0x05]
+
+0x05,0x07,0x34,0xd6,0xe9,0x04,0xfe,0xe1,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, exec_hi bitop3:0x3f dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x07,0x34,0xd6,0xe9,0x04,0xfe,0xe1,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x34,0xd6,0xe9,0x04,0xfa,0x81,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, exec_lo bitop3:0x24 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x34,0xd6,0xe9,0x04,0xfa,0x81,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x34,0xd6,0xe9,0x04,0xf2,0xa1,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, null bitop3:5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd6,0xe9,0x04,0xf2,0xa1,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x34,0xd6,0xe9,0x04,0xa6,0xe1,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, s105 bitop3:0x27 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x34,0xd6,0xe9,0x04,0xa6,0xe1,0x01,0x77,0x39,0x05]
+
+0x05,0x02,0x34,0xd6,0xe9,0x04,0xee,0xa1,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, ttmp15 bitop3:0x15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x34,0xd6,0xe9,0x04,0xee,0xa1,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x34,0xd6,0xe9,0x04,0xfe,0x37,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, v255 bitop3:0xa1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x34,0xd6,0xe9,0x04,0xfe,0x37,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x34,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+0x05,0x04,0x34,0xd6,0xe9,0x04,0xae,0x89,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, vcc_hi bitop3:0x64 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x34,0xd6,0xe9,0x04,0xae,0x89,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x34,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+# GFX1250: v_bitop3_b32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x34,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x60,0xd6,0xea,0x54,0x0d,0x00,0x01,0x77,0x39,0x05
+# GFX1250: v_add_min_i32_e64_dpp v5, v1, 42, s3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x60,0xd6,0xea,0x54,0x0d,0x00,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x60,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX1250: v_add_min_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x60,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x5e,0xd6,0xea,0x54,0x0d,0x00,0x01,0x77,0x39,0x05
+# GFX1250: v_add_max_i32_e64_dpp v5, v1, 42, s3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x5e,0xd6,0xea,0x54,0x0d,0x00,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x5e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX1250: v_add_max_i32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x61,0xd6,0xea,0x54,0x0d,0x00,0x01,0x77,0x39,0x05
+# GFX1250: v_add_min_u32_e64_dpp v5, v1, 42, s3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x61,0xd6,0xea,0x54,0x0d,0x00,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x61,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX1250: v_add_min_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x61,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x5f,0xd6,0xea,0x54,0x0d,0x00,0x01,0x77,0x39,0x05
+# GFX1250: v_add_max_u32_e64_dpp v5, v1, 42, s3 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x00,0x5f,0xd6,0xea,0x54,0x0d,0x00,0x01,0x77,0x39,0x05]
+
+0x05,0x00,0x5f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+# GFX1250: v_add_max_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x5f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
 
 0xff,0x01,0x33,0xd6,0xe9,0xfe,0xf7,0xab,0xff,0x00,0x00,0x00
 # GFX1250-REAL16: v_bitop3_b16_e64_dpp v255.l, v255.l, v255.l, src_scc bitop3:0x4d dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x01,0x33,0xd6,0xe9,0xfe,0xf7,0xab,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3cx.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3cx.txt
new file mode 100644
index 0000000000000..e419e4583acfc
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3cx.txt
@@ -0,0 +1,3413 @@
+# NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+
+0x7e,0x00,0xfd,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_class_f16_e64 -1, exec_hi        ; encoding: [0x7e,0x00,0xfd,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x01,0xfd,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_class_f16_e64 -|0xfe0b|, vcc_hi  ; encoding: [0x7e,0x01,0xfd,0xd4,0xff,0xd6,0x00,0x20,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xfd,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_class_f16_e64 0.5, m0            ; encoding: [0x7e,0x00,0xfd,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_class_f16_e64 exec_hi, null      ; encoding: [0x7e,0x00,0xfd,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_class_f16_e64 exec_lo, -1        ; encoding: [0x7e,0x00,0xfd,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x7d,0xfa,0x01,0x00
+# GFX1250: v_cmpx_class_f16_e64 m0, src_scc        ; encoding: [0x7e,0x00,0xfd,0xd4,0x7d,0xfa,0x01,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_class_f16_e64 null, exec_lo      ; encoding: [0x7e,0x00,0xfd,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x01,0x04,0x02,0x00
+# GFX1250: v_cmpx_class_f16_e64 s1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x04,0x02,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x69,0xfe,0x03,0x00
+# GFX1250: v_cmpx_class_f16_e64 s105, v255         ; encoding: [0x7e,0x00,0xfd,0xd4,0x69,0xfe,0x03,0x00]
+
+0x7e,0x00,0xfd,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_class_f16_e64 src_scc, vcc_lo    ; encoding: [0x7e,0x00,0xfd,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x7b,0xf6,0x00,0x00
+# GFX1250: v_cmpx_class_f16_e64 ttmp15, ttmp15     ; encoding: [0x7e,0x00,0xfd,0xd4,0x7b,0xf6,0x00,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_class_f16_e64 v1, v2             ; encoding: [0x7e,0x00,0xfd,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xfd,0xd4,0xff,0x05,0x02,0x00
+# GFX1250: v_cmpx_class_f16_e64 v255, v2           ; encoding: [0x7e,0x00,0xfd,0xd4,0xff,0x05,0x02,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x6b,0xd2,0x00,0x00
+# GFX1250: v_cmpx_class_f16_e64 vcc_hi, s105       ; encoding: [0x7e,0x00,0xfd,0xd4,0x6b,0xd2,0x00,0x00]
+
+0x7e,0x00,0xfd,0xd4,0x6a,0x04,0x00,0x00
+# GFX1250: v_cmpx_class_f16_e64 vcc_lo, s2         ; encoding: [0x7e,0x00,0xfd,0xd4,0x6a,0x04,0x00,0x00]
+
+0x7e,0x00,0xfe,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_class_f32_e64 -1, exec_hi        ; encoding: [0x7e,0x00,0xfe,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x01,0xfe,0xd4,0xff,0xd6,0x00,0x20,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_class_f32_e64 -|0xaf123456|, vcc_hi ; encoding: [0x7e,0x01,0xfe,0xd4,0xff,0xd6,0x00,0x20,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xfe,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_class_f32_e64 0.5, m0            ; encoding: [0x7e,0x00,0xfe,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xfe,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_class_f32_e64 exec_hi, null      ; encoding: [0x7e,0x00,0xfe,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xfe,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_class_f32_e64 exec_lo, -1        ; encoding: [0x7e,0x00,0xfe,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xfe,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_class_f32_e64 m0, 0.5            ; encoding: [0x7e,0x00,0xfe,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xfe,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_class_f32_e64 null, exec_lo      ; encoding: [0x7e,0x00,0xfe,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xfe,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_class_f32_e64 s1, s2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xfe,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_class_f32_e64 s105, s105         ; encoding: [0x7e,0x00,0xfe,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xfe,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_class_f32_e64 src_scc, vcc_lo    ; encoding: [0x7e,0x00,0xfe,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xfe,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_class_f32_e64 ttmp15, src_scc    ; encoding: [0x7e,0x00,0xfe,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_class_f32_e64 v1, v2             ; encoding: [0x7e,0x00,0xfe,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xfe,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_class_f32_e64 v255, v255         ; encoding: [0x7e,0x00,0xfe,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xfe,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_class_f32_e64 vcc_hi, 0xaf123456 ; encoding: [0x7e,0x00,0xfe,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xfe,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_class_f32_e64 vcc_lo, ttmp15     ; encoding: [0x7e,0x00,0xfe,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xff,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_class_f64_e64 -1, -1             ; encoding: [0x7e,0x00,0xff,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xff,0xd4,0xfd,0xfa,0x01,0x20
+# GFX1250: v_cmpx_class_f64_e64 -|src_scc|, src_scc ; encoding: [0x7e,0x01,0xff,0xd4,0xfd,0xfa,0x01,0x20]
+
+0x7e,0x00,0xff,0xd4,0xf0,0xe0,0x01,0x00
+# GFX1250: v_cmpx_class_f64_e64 0.5, 0.5           ; encoding: [0x7e,0x00,0xff,0xd4,0xf0,0xe0,0x01,0x00]
+
+0x7e,0x00,0xff,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_class_f64_e64 0xaf123456, 0xaf123456 ; encoding: [0x7e,0x00,0xff,0xd4,0xff,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xff,0xd4,0x7e,0xfc,0x00,0x00
+# GFX1250: v_cmpx_class_f64_e64 exec, exec_lo      ; encoding: [0x7e,0x00,0xff,0xd4,0x7e,0xfc,0x00,0x00]
+
+0x7e,0x00,0xff,0xd4,0x7c,0xf8,0x00,0x00
+# GFX1250: v_cmpx_class_f64_e64 null, null         ; encoding: [0x7e,0x00,0xff,0xd4,0x7c,0xf8,0x00,0x00]
+
+0x7e,0x00,0xff,0xd4,0x68,0xd4,0x00,0x00
+# GFX1250: v_cmpx_class_f64_e64 s[104:105], vcc_lo ; encoding: [0x7e,0x00,0xff,0xd4,0x68,0xd4,0x00,0x00]
+
+0x7e,0x00,0xff,0xd4,0x02,0xd6,0x00,0x00
+# GFX1250: v_cmpx_class_f64_e64 s[2:3], vcc_hi     ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0xd6,0x00,0x00]
+
+0x7e,0x00,0xff,0xd4,0x7a,0xfe,0x00,0x00
+# GFX1250: v_cmpx_class_f64_e64 ttmp[14:15], exec_hi ; encoding: [0x7e,0x00,0xff,0xd4,0x7a,0xfe,0x00,0x00]
+
+0x7e,0x00,0xff,0xd4,0xfe,0xf7,0x00,0x00
+# GFX1250: v_cmpx_class_f64_e64 v[254:255], ttmp15 ; encoding: [0x7e,0x00,0xff,0xd4,0xfe,0xf7,0x00,0x00]
+
+0x7e,0x00,0xff,0xd4,0x02,0xd3,0x00,0x00
+# GFX1250: v_cmpx_class_f64_e64 v[2:3], s105       ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0xd3,0x00,0x00]
+
+0x7e,0x00,0xff,0xd4,0x02,0x05,0x00,0x00
+# GFX1250: v_cmpx_class_f64_e64 v[2:3], s2         ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0x05,0x00,0x00]
+
+0x7e,0x00,0xff,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_class_f64_e64 v[2:3], v2         ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xff,0xd4,0x02,0xff,0x03,0x00
+# GFX1250: v_cmpx_class_f64_e64 v[2:3], v255       ; encoding: [0x7e,0x00,0xff,0xd4,0x02,0xff,0x03,0x00]
+
+0x7e,0x00,0xff,0xd4,0x6a,0xfa,0x00,0x00
+# GFX1250: v_cmpx_class_f64_e64 vcc, m0            ; encoding: [0x7e,0x00,0xff,0xd4,0x6a,0xfa,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x82,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x82,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_eq_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x82,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x82,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_eq_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x82,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x82,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_eq_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x82,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x82,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_eq_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x82,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x82,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_eq_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x82,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_eq_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_eq_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x82,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_eq_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x82,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_eq_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x82,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_eq_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x82,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x82,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x82,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x82,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_eq_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x82,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x82,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_eq_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x82,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x92,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x92,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x92,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_eq_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x92,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x92,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x92,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x92,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_eq_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x92,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x92,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_eq_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x92,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x92,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_eq_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x92,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x92,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_eq_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x92,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x92,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_eq_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x92,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_eq_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x92,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x92,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_eq_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x92,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_eq_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x92,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x92,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_eq_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x92,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x92,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x92,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x92,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_eq_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x92,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x92,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_eq_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x92,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xa2,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_eq_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa2,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xa2,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_eq_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa2,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xa2,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_eq_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa2,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xa2,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_eq_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa2,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xa2,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa2,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa2,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_eq_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa2,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xa2,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_eq_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa2,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xa2,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_eq_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa2,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xa2,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa2,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa2,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_eq_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa2,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xa2,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_eq_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa2,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xa2,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_eq_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa2,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb2,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb2,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_eq_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb2,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xb2,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb2,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb2,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb2,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_eq_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb2,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_eq_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb2,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xb2,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_eq_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb2,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb2,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb2,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_eq_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb2,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xc2,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc2,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xc2,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_eq_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc2,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xc2,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc2,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc2,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_eq_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc2,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xc2,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_eq_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc2,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xc2,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_eq_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc2,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xc2,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_eq_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc2,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xc2,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_eq_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xc2,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_eq_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc2,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xc2,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_eq_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc2,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xc2,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_eq_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc2,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_eq_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc2,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xc2,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_eq_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc2,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xc2,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc2,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc2,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_eq_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc2,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xd2,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_eq_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd2,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xd2,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_eq_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd2,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xd2,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd2,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd2,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_eq_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd2,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xd2,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_eq_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd2,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xd2,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_eq_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd2,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xd2,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_eq_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd2,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xd2,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_eq_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd2,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xd2,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd2,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd2,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_eq_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd2,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xd2,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_eq_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd2,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xd2,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_eq_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd2,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xba,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xba,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_eq_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xba,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xba,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xba,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xba,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xba,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xba,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_eq_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xba,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xba,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_eq_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xba,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xba,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_eq_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xba,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xba,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xba,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xba,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_eq_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xba,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xca,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_eq_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xca,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xca,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_eq_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xca,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xca,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xca,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xca,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_eq_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xca,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xca,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_eq_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xca,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xca,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_eq_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xca,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xca,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_eq_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xca,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xca,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_eq_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xca,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_eq_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xca,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xca,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_eq_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xca,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xca,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_eq_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xca,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_eq_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xca,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xca,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_eq_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xca,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xca,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xca,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xca,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_eq_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xca,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xda,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_eq_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xda,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xda,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_eq_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xda,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xda,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xda,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xda,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_eq_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xda,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xda,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_eq_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xda,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xda,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_eq_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xda,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xda,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_eq_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xda,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xda,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_eq_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xda,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xda,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_eq_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xda,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xda,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_eq_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xda,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xda,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_eq_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xda,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xda,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_eq_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xda,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x86,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x86,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_ge_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x86,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x86,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_ge_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x86,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x86,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ge_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x86,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x86,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ge_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x86,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x86,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ge_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x86,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ge_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ge_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x86,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ge_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x86,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ge_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x86,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ge_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x86,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x86,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x86,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x86,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ge_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x86,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x86,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ge_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x86,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x96,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x96,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x96,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_ge_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x96,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x96,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x96,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x96,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_ge_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x96,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x96,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ge_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x96,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x96,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ge_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x96,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x96,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ge_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x96,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x96,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ge_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x96,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ge_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x96,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x96,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ge_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x96,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ge_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x96,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x96,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ge_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x96,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x96,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x96,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x96,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ge_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x96,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x96,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ge_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x96,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xa6,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_ge_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa6,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xa6,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_ge_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa6,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xa6,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_ge_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa6,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xa6,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ge_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa6,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xa6,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa6,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa6,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ge_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa6,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xa6,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_ge_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa6,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xa6,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_ge_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa6,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xa6,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa6,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa6,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_ge_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa6,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xa6,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_ge_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa6,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xa6,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_ge_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa6,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb6,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb6,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ge_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb6,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xb6,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb6,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb6,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb6,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ge_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb6,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ge_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb6,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xb6,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ge_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb6,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb6,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb6,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ge_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb6,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xc6,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc6,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xc6,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_ge_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc6,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xc6,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc6,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc6,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ge_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc6,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xc6,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ge_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc6,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xc6,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ge_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc6,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xc6,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ge_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc6,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xc6,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ge_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xc6,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ge_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc6,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xc6,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_ge_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc6,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xc6,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ge_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc6,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ge_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc6,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xc6,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ge_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc6,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xc6,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc6,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc6,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ge_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc6,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xd6,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_ge_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd6,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xd6,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ge_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd6,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xd6,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd6,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd6,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ge_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd6,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xd6,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ge_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd6,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xd6,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_ge_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd6,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xd6,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_ge_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd6,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xd6,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ge_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd6,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xd6,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd6,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd6,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_ge_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd6,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xd6,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_ge_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd6,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xd6,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_ge_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd6,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbe,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbe,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ge_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbe,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xbe,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbe,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbe,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbe,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ge_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbe,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ge_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbe,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xbe,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ge_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbe,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbe,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbe,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ge_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbe,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xce,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ge_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xce,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xce,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_ge_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xce,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xce,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xce,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xce,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ge_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xce,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xce,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ge_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xce,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xce,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ge_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xce,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xce,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ge_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xce,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xce,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ge_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xce,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ge_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xce,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xce,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_ge_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xce,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xce,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ge_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xce,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ge_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xce,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xce,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ge_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xce,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xce,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xce,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xce,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ge_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xce,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xde,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_ge_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xde,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xde,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ge_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xde,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xde,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xde,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xde,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ge_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xde,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xde,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ge_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xde,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xde,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_ge_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xde,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xde,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_ge_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xde,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xde,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ge_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xde,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xde,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ge_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xde,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xde,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_ge_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xde,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xde,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_ge_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xde,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xde,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_ge_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xde,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x84,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x84,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_gt_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x84,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x84,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_gt_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x84,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x84,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_gt_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x84,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x84,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_gt_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x84,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x84,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_gt_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x84,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_gt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_gt_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x84,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_gt_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x84,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_gt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x84,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_gt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x84,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x84,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x84,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x84,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_gt_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x84,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x84,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_gt_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x84,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x94,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x94,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x94,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_gt_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x94,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x94,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x94,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x94,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_gt_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x94,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x94,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_gt_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x94,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x94,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_gt_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x94,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x94,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_gt_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x94,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x94,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_gt_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x94,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_gt_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x94,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x94,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_gt_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x94,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_gt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x94,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x94,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_gt_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x94,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x94,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x94,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x94,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_gt_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x94,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x94,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_gt_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x94,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xa4,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_gt_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa4,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xa4,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_gt_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa4,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xa4,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_gt_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa4,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xa4,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_gt_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa4,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xa4,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa4,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa4,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_gt_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa4,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xa4,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_gt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa4,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xa4,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_gt_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa4,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xa4,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa4,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa4,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_gt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa4,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xa4,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_gt_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa4,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xa4,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_gt_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa4,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb4,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb4,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_gt_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb4,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xb4,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb4,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb4,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb4,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_gt_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb4,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_gt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb4,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xb4,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_gt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb4,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb4,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb4,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_gt_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb4,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xc4,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc4,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xc4,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_gt_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc4,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xc4,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc4,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc4,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_gt_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc4,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xc4,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_gt_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc4,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xc4,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_gt_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc4,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xc4,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_gt_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc4,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xc4,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_gt_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xc4,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_gt_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc4,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xc4,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_gt_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc4,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xc4,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_gt_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc4,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_gt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc4,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xc4,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_gt_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc4,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xc4,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc4,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc4,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_gt_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc4,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xd4,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_gt_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd4,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xd4,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_gt_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd4,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xd4,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd4,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd4,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_gt_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd4,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xd4,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_gt_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd4,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xd4,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_gt_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd4,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xd4,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_gt_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd4,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xd4,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_gt_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd4,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xd4,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd4,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd4,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_gt_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd4,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xd4,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_gt_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd4,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xd4,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_gt_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd4,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbc,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbc,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_gt_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbc,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xbc,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbc,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbc,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbc,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_gt_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbc,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_gt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbc,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xbc,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_gt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbc,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbc,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbc,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_gt_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbc,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xcc,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_gt_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xcc,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xcc,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_gt_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xcc,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xcc,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xcc,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xcc,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_gt_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xcc,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xcc,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_gt_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xcc,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xcc,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_gt_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xcc,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xcc,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_gt_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xcc,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xcc,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_gt_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xcc,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_gt_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xcc,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xcc,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_gt_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xcc,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xcc,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_gt_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xcc,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_gt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcc,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xcc,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_gt_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xcc,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xcc,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xcc,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xcc,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_gt_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xcc,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xdc,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_gt_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xdc,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xdc,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_gt_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xdc,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xdc,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xdc,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xdc,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_gt_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xdc,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xdc,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_gt_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xdc,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xdc,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_gt_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdc,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xdc,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_gt_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xdc,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xdc,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_gt_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xdc,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xdc,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_gt_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xdc,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xdc,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_gt_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdc,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xdc,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_gt_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xdc,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xdc,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_gt_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xdc,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x83,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x83,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_le_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x83,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x83,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_le_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x83,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x83,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_le_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x83,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x83,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_le_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x83,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x83,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_le_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x83,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_le_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_le_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x83,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_le_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x83,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_le_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x83,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_le_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x83,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x83,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x83,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x83,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_le_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x83,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x83,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_le_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x83,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x93,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x93,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x93,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_le_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x93,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x93,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x93,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x93,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_le_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x93,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x93,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_le_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x93,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x93,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_le_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x93,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x93,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_le_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x93,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x93,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_le_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x93,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_le_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x93,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x93,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_le_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x93,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_le_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x93,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x93,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_le_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x93,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x93,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x93,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x93,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_le_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x93,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x93,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_le_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x93,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xa3,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_le_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa3,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xa3,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_le_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa3,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xa3,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_le_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa3,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xa3,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_le_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa3,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xa3,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa3,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa3,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_le_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa3,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xa3,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_le_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa3,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xa3,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_le_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa3,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xa3,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa3,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa3,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_le_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa3,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xa3,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_le_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa3,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xa3,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_le_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa3,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb3,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb3,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_le_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb3,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xb3,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb3,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb3,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb3,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_le_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb3,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_le_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb3,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xb3,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_le_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb3,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb3,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb3,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_le_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb3,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xc3,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc3,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xc3,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_le_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc3,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xc3,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc3,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc3,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_le_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc3,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xc3,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_le_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc3,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xc3,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_le_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc3,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xc3,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_le_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc3,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xc3,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_le_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xc3,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_le_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc3,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xc3,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_le_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc3,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xc3,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_le_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc3,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_le_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc3,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xc3,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_le_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc3,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xc3,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc3,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc3,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_le_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc3,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xd3,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_le_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd3,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xd3,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_le_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd3,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xd3,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd3,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd3,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_le_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd3,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xd3,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_le_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd3,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xd3,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_le_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd3,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xd3,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_le_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd3,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xd3,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_le_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd3,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xd3,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd3,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd3,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_le_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd3,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xd3,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_le_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd3,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xd3,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_le_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd3,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbb,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbb,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_le_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbb,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xbb,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbb,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbb,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbb,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_le_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbb,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_le_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbb,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xbb,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_le_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbb,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbb,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbb,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_le_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbb,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xcb,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_le_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xcb,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xcb,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_le_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xcb,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xcb,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xcb,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xcb,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_le_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xcb,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xcb,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_le_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xcb,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xcb,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_le_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xcb,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xcb,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_le_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xcb,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xcb,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_le_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xcb,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_le_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xcb,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xcb,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_le_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xcb,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xcb,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_le_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xcb,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_le_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcb,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xcb,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_le_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xcb,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xcb,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xcb,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xcb,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_le_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xcb,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xdb,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_le_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xdb,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xdb,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_le_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xdb,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xdb,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xdb,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xdb,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_le_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xdb,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xdb,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_le_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xdb,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xdb,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_le_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdb,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xdb,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_le_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xdb,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xdb,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_le_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xdb,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xdb,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_le_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xdb,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xdb,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_le_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdb,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xdb,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_le_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xdb,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xdb,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_le_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xdb,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lg_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x85,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x85,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_lg_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x85,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x85,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_lg_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x85,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x85,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_lg_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x85,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x85,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lg_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x85,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x85,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lg_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x85,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_lg_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_lg_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x85,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lg_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x85,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_lg_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x85,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_lg_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x85,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x85,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lg_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x85,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x85,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_lg_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x85,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x85,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lg_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x85,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x95,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lg_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x95,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x95,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_lg_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x95,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x95,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x95,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x95,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_lg_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x95,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x95,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_lg_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x95,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x95,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lg_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x95,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x95,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lg_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x95,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x95,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_lg_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x95,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_lg_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x95,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x95,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lg_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x95,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_lg_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x95,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x95,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_lg_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x95,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x95,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lg_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x95,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x95,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_lg_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x95,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x95,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lg_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x95,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xa5,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_lg_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa5,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xa5,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_lg_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa5,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xa5,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_lg_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa5,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xa5,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lg_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa5,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xa5,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa5,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa5,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lg_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa5,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xa5,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_lg_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa5,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xa5,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_lg_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa5,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xa5,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lg_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa5,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa5,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_lg_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa5,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xa5,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_lg_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa5,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xa5,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_lg_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa5,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x81,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_f16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x81,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x81,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_lt_f16_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x81,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x81,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x81,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_lt_f16_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x81,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x81,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_lt_f16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x81,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x81,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lt_f16_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x81,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x81,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lt_f16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x81,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x81,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_lt_f16_e64 s1, s2                ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x81,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_lt_f16_e64 s105, s105            ; encoding: [0x7e,0x00,0x81,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x81,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lt_f16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x81,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x81,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_lt_f16_e64 v1, v2                ; encoding: [0x7e,0x00,0x81,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x81,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_lt_f16_e64 v255, v255            ; encoding: [0x7e,0x00,0x81,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x81,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_f16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0x81,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x81,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_lt_f16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x81,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x81,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lt_f16_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x81,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x91,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_f32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0x91,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x91,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_lt_f32_e64 -src_scc, |vcc_lo|    ; encoding: [0x7e,0x02,0x91,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x91,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x91,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x91,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_lt_f32_e64 0.5, -m0              ; encoding: [0x7e,0x00,0x91,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x91,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_lt_f32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0x91,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x91,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lt_f32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0x91,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x91,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lt_f32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0x91,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x91,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_lt_f32_e64 s1, s2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x91,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_lt_f32_e64 s105, s105            ; encoding: [0x7e,0x00,0x91,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x91,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lt_f32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0x91,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_lt_f32_e64 v1, v2                ; encoding: [0x7e,0x00,0x91,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x91,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_lt_f32_e64 v255, v255            ; encoding: [0x7e,0x00,0x91,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x91,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_f32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0x91,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x91,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_lt_f32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0x91,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x91,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lt_f32_e64 |exec_hi|, null       ; encoding: [0x7e,0x01,0x91,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xa1,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_lt_f64_e64 -1, -1                ; encoding: [0x7e,0x00,0xa1,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xa1,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_lt_f64_e64 -|exec|, src_scc      ; encoding: [0x7e,0x01,0xa1,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xa1,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_lt_f64_e64 -|src_scc|, -|exec|   ; encoding: [0x7e,0x03,0xa1,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xa1,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lt_f64_e64 0.5, null             ; encoding: [0x7e,0x00,0xa1,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xa1,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa1,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa1,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lt_f64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xa1,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xa1,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_lt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa1,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xa1,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_lt_f64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xa1,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xa1,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa1,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa1,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_lt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa1,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xa1,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_lt_f64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xa1,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xa1,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_lt_f64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xa1,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb1,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb1,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_lt_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb1,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xb1,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb1,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb1,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb1,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lt_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb1,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_lt_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb1,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xb1,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_lt_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb1,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb1,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb1,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_lt_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb1,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xc1,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc1,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xc1,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_lt_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc1,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xc1,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc1,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc1,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lt_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc1,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xc1,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_lt_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc1,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xc1,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lt_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc1,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xc1,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lt_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc1,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xc1,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_lt_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xc1,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_lt_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc1,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xc1,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_lt_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc1,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xc1,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lt_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc1,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_lt_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc1,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xc1,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_lt_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc1,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xc1,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc1,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc1,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_lt_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc1,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xd1,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_lt_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd1,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xd1,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lt_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd1,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xd1,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd1,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd1,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lt_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd1,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xd1,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lt_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd1,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xd1,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_lt_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd1,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xd1,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_lt_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd1,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xd1,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lt_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd1,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xd1,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd1,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd1,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_lt_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd1,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xd1,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_lt_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd1,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xd1,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_lt_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd1,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb9,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb9,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_lt_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb9,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xb9,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb9,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb9,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb9,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lt_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb9,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_lt_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb9,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xb9,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_lt_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb9,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb9,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb9,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_lt_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb9,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xc9,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_lt_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc9,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xc9,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_lt_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc9,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xc9,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc9,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc9,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lt_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc9,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xc9,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_lt_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc9,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xc9,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lt_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc9,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xc9,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lt_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc9,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xc9,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_lt_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xc9,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_lt_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc9,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xc9,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_lt_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc9,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xc9,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lt_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc9,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_lt_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc9,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xc9,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_lt_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc9,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xc9,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc9,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc9,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_lt_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc9,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xd9,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_lt_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd9,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xd9,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_lt_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd9,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xd9,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd9,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd9,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_lt_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd9,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xd9,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_lt_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd9,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xd9,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_lt_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd9,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xd9,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_lt_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd9,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xd9,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_lt_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd9,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xd9,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_lt_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd9,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd9,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_lt_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd9,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xd9,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_lt_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd9,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xd9,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_lt_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd9,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xb5,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xb5,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ne_i16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xb5,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xb5,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xb5,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 s1, s2                ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 s105, s105            ; encoding: [0x7e,0x00,0xb5,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xb5,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ne_i16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xb5,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ne_i16_e64 v1, v2                ; encoding: [0x7e,0x00,0xb5,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xb5,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ne_i16_e64 v255, v255            ; encoding: [0x7e,0x00,0xb5,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xb5,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xb5,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ne_i16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xb5,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xc5,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ne_i32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xc5,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xc5,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_ne_i32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xc5,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xc5,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ne_i32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xc5,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc5,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ne_i32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xc5,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xc5,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ne_i32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xc5,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xc5,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ne_i32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xc5,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xc5,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ne_i32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xc5,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xc5,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ne_i32_e64 s1, s2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xc5,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ne_i32_e64 s105, s105            ; encoding: [0x7e,0x00,0xc5,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xc5,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_ne_i32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xc5,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xc5,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ne_i32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xc5,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ne_i32_e64 v1, v2                ; encoding: [0x7e,0x00,0xc5,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xc5,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ne_i32_e64 v255, v255            ; encoding: [0x7e,0x00,0xc5,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xc5,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ne_i32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xc5,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xc5,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ne_i32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xc5,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xd5,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_ne_i64_e64 -1, -1                ; encoding: [0x7e,0x00,0xd5,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xd5,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ne_i64_e64 0.5, null             ; encoding: [0x7e,0x00,0xd5,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xd5,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ne_i64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xd5,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd5,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ne_i64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xd5,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xd5,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ne_i64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xd5,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xd5,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_ne_i64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xd5,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xd5,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_ne_i64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xd5,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xd5,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ne_i64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xd5,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xd5,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ne_i64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xd5,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xd5,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_ne_i64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xd5,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xd5,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_ne_i64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xd5,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xd5,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_ne_i64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xd5,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xbd,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 0x3800, m0            ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xfa,0x00,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 0xfe0b, vcc_hi        ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xbd,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ne_u16_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xbd,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 m0, 0x3800            ; encoding: [0x7e,0x00,0xbd,0xd4,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xbd,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 s1, s2                ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 s105, s105            ; encoding: [0x7e,0x00,0xbd,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xbd,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ne_u16_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xbd,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ne_u16_e64 v1, v2                ; encoding: [0x7e,0x00,0xbd,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xbd,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ne_u16_e64 v255, v255            ; encoding: [0x7e,0x00,0xbd,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 vcc_hi, 0xfe0b        ; encoding: [0x7e,0x00,0xbd,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0xbd,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ne_u16_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xbd,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xcd,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ne_u32_e64 -1, exec_hi           ; encoding: [0x7e,0x00,0xcd,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x00,0xcd,0xd4,0xf0,0xfa,0x00,0x00
+# GFX1250: v_cmpx_ne_u32_e64 0.5, m0               ; encoding: [0x7e,0x00,0xcd,0xd4,0xf0,0xfa,0x00,0x00]
+
+0x7e,0x00,0xcd,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ne_u32_e64 0xaf123456, vcc_hi    ; encoding: [0x7e,0x00,0xcd,0xd4,0xff,0xd6,0x00,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xcd,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ne_u32_e64 exec_hi, null         ; encoding: [0x7e,0x00,0xcd,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xcd,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ne_u32_e64 exec_lo, -1           ; encoding: [0x7e,0x00,0xcd,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0xcd,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ne_u32_e64 m0, 0.5               ; encoding: [0x7e,0x00,0xcd,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0xcd,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ne_u32_e64 null, exec_lo         ; encoding: [0x7e,0x00,0xcd,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0xcd,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ne_u32_e64 s1, s2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0xcd,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ne_u32_e64 s105, s105            ; encoding: [0x7e,0x00,0xcd,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0xcd,0xd4,0xfd,0xd4,0x00,0x00
+# GFX1250: v_cmpx_ne_u32_e64 src_scc, vcc_lo       ; encoding: [0x7e,0x00,0xcd,0xd4,0xfd,0xd4,0x00,0x00]
+
+0x7e,0x00,0xcd,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ne_u32_e64 ttmp15, src_scc       ; encoding: [0x7e,0x00,0xcd,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ne_u32_e64 v1, v2                ; encoding: [0x7e,0x00,0xcd,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0xcd,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ne_u32_e64 v255, v255            ; encoding: [0x7e,0x00,0xcd,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0xcd,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ne_u32_e64 vcc_hi, 0xaf123456    ; encoding: [0x7e,0x00,0xcd,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xcd,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ne_u32_e64 vcc_lo, ttmp15        ; encoding: [0x7e,0x00,0xcd,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x00,0xdd,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_ne_u64_e64 -1, -1                ; encoding: [0x7e,0x00,0xdd,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x00,0xdd,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ne_u64_e64 0.5, null             ; encoding: [0x7e,0x00,0xdd,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x00,0xdd,0xd4,0xff,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ne_u64_e64 lit64(0xaf123456), vcc ; encoding: [0x7e,0x00,0xdd,0xd4,0xfe,0xd4,0x00,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xdd,0xd4,0x7e,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ne_u64_e64 exec, src_scc         ; encoding: [0x7e,0x00,0xdd,0xd4,0x7e,0xfa,0x01,0x00]
+
+0x7e,0x00,0xdd,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ne_u64_e64 null, 0.5             ; encoding: [0x7e,0x00,0xdd,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xdd,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_ne_u64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xdd,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xdd,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_ne_u64_e64 s[2:3], s[4:5]        ; encoding: [0x7e,0x00,0xdd,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xdd,0xd4,0xfd,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ne_u64_e64 src_scc, exec         ; encoding: [0x7e,0x00,0xdd,0xd4,0xfd,0xfc,0x00,0x00]
+
+0x7e,0x00,0xdd,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ne_u64_e64 ttmp[14:15], lit64(0xaf123456) ; encoding: [0x7e,0x00,0xdd,0xd4,0x7a,0xfc,0x01,0x00,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
+
+0x7e,0x00,0xdd,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_ne_u64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xdd,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xdd,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_ne_u64_e64 v[2:3], v[2:3]        ; encoding: [0x7e,0x00,0xdd,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xdd,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_ne_u64_e64 vcc, ttmp[14:15]      ; encoding: [0x7e,0x00,0xdd,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_neq_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8d,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x8d,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_neq_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8d,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_neq_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8d,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_neq_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8d,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x8d,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_neq_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8d,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_neq_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8d,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_neq_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8d,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_neq_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_neq_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8d,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_neq_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8d,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_neq_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8d,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_neq_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8d,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_neq_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8d,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8d,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_neq_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8d,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x8d,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_neq_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8d,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x9d,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_neq_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9d,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x9d,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_neq_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9d,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x9d,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_neq_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9d,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9d,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_neq_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9d,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x9d,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_neq_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9d,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x9d,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_neq_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9d,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x9d,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_neq_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9d,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x9d,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_neq_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x9d,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_neq_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9d,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x9d,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_neq_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9d,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_neq_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9d,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x9d,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_neq_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9d,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x9d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_neq_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9d,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9d,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_neq_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9d,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x9d,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_neq_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9d,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xad,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_neq_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xad,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xad,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_neq_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xad,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xad,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_neq_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xad,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xad,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_neq_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xad,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xad,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_neq_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xad,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xad,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_neq_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xad,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xad,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_neq_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xad,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xad,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_neq_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xad,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xad,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_neq_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xad,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xad,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_neq_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xad,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xad,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_neq_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xad,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xad,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_neq_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xad,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nge_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x89,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x89,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_nge_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x89,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nge_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x89,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_nge_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x89,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x89,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_nge_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x89,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x89,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nge_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x89,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x89,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_nge_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x89,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_nge_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_nge_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x89,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_nge_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x89,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_nge_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x89,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_nge_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x89,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x89,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nge_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x89,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x89,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_nge_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x89,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x89,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nge_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x89,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x99,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nge_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x99,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x99,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_nge_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x99,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x99,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nge_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x99,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x99,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_nge_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x99,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x99,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_nge_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x99,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x99,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nge_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x99,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x99,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_nge_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x99,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x99,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_nge_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x99,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_nge_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x99,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x99,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_nge_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x99,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_nge_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x99,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x99,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_nge_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x99,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x99,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nge_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x99,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x99,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_nge_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x99,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x99,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nge_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x99,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xa9,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_nge_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xa9,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xa9,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_nge_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xa9,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xa9,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_nge_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xa9,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xa9,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nge_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xa9,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xa9,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nge_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa9,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa9,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nge_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xa9,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xa9,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_nge_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa9,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xa9,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_nge_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xa9,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xa9,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nge_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa9,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa9,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_nge_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa9,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xa9,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_nge_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xa9,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xa9,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_nge_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xa9,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8b,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x8b,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_ngt_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8b,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8b,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_ngt_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8b,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x8b,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8b,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8b,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8b,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8b,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8b,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8b,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8b,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8b,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8b,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8b,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x8b,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ngt_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8b,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x9b,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9b,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x9b,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_ngt_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9b,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x9b,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ngt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9b,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9b,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_ngt_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9b,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x9b,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9b,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x9b,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9b,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x9b,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9b,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x9b,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x9b,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9b,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x9b,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9b,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9b,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x9b,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9b,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x9b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ngt_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9b,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9b,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9b,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x9b,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ngt_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9b,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xab,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_ngt_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xab,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xab,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_ngt_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xab,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xab,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_ngt_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xab,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xab,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_ngt_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xab,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xab,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ngt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xab,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xab,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_ngt_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xab,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xab,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_ngt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xab,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xab,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_ngt_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xab,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xab,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_ngt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xab,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xab,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_ngt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xab,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xab,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_ngt_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xab,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xab,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_ngt_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xab,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nle_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8c,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x8c,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_nle_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8c,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nle_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8c,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_nle_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8c,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x8c,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_nle_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8c,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nle_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8c,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_nle_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8c,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_nle_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_nle_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8c,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_nle_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8c,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_nle_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8c,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_nle_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8c,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nle_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8c,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8c,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_nle_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8c,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x8c,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nle_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8c,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x9c,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nle_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9c,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x9c,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_nle_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9c,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x9c,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nle_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9c,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9c,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_nle_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9c,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x9c,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_nle_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9c,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x9c,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nle_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9c,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x9c,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_nle_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9c,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x9c,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_nle_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x9c,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_nle_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9c,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x9c,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_nle_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9c,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_nle_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9c,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x9c,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_nle_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9c,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x9c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nle_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9c,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9c,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_nle_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9c,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x9c,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nle_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9c,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xac,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_nle_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xac,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xac,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_nle_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xac,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xac,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_nle_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xac,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xac,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nle_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xac,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xac,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nle_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xac,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xac,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nle_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xac,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xac,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_nle_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xac,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xac,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_nle_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xac,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xac,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nle_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xac,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xac,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_nle_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xac,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xac,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_nle_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xac,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xac,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_nle_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xac,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8a,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x8a,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_nlg_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8a,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8a,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_nlg_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8a,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x8a,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8a,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8a,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8a,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8a,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8a,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8a,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8a,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8a,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8a,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8a,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x8a,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nlg_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8a,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x9a,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9a,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x9a,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_nlg_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9a,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x9a,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nlg_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9a,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9a,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_nlg_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9a,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x9a,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9a,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x9a,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9a,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x9a,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9a,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x9a,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x9a,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9a,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x9a,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9a,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9a,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x9a,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9a,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x9a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nlg_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9a,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9a,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9a,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x9a,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nlg_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9a,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xaa,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_nlg_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xaa,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xaa,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_nlg_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xaa,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xaa,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_nlg_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xaa,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xaa,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nlg_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xaa,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xaa,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nlg_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xaa,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xaa,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nlg_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xaa,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xaa,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_nlg_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xaa,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xaa,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_nlg_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xaa,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xaa,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nlg_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xaa,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xaa,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_nlg_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xaa,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xaa,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_nlg_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xaa,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xaa,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_nlg_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xaa,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x8e,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x8e,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_nlt_f16_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x8e,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x8e,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_nlt_f16_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x8e,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x8e,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x8e,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x8e,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x8e,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 s1, s2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 s105, s105           ; encoding: [0x7e,0x00,0x8e,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x8e,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 v1, v2               ; encoding: [0x7e,0x00,0x8e,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 v255, v255           ; encoding: [0x7e,0x00,0x8e,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 vcc_hi, 0xfe0b       ; encoding: [0x7e,0x00,0x8e,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x8e,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x8e,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x8e,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nlt_f16_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x8e,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x9e,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 -1, exec_hi          ; encoding: [0x7e,0x00,0x9e,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x9e,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_nlt_f32_e64 -src_scc, |vcc_lo|   ; encoding: [0x7e,0x02,0x9e,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x9e,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nlt_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x9e,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9e,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_nlt_f32_e64 0.5, -m0             ; encoding: [0x7e,0x00,0x9e,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x9e,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 exec_lo, -1          ; encoding: [0x7e,0x00,0x9e,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x9e,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 m0, 0.5              ; encoding: [0x7e,0x00,0x9e,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x9e,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 null, exec_lo        ; encoding: [0x7e,0x00,0x9e,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x9e,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 s1, s2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x9e,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 s105, s105           ; encoding: [0x7e,0x00,0x9e,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x9e,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 ttmp15, src_scc      ; encoding: [0x7e,0x00,0x9e,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 v1, v2               ; encoding: [0x7e,0x00,0x9e,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x9e,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 v255, v255           ; encoding: [0x7e,0x00,0x9e,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x9e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nlt_f32_e64 vcc_hi, 0xaf123456   ; encoding: [0x7e,0x00,0x9e,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x9e,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 vcc_lo, ttmp15       ; encoding: [0x7e,0x00,0x9e,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x9e,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nlt_f32_e64 |exec_hi|, null      ; encoding: [0x7e,0x01,0x9e,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xae,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_nlt_f64_e64 -1, -1               ; encoding: [0x7e,0x00,0xae,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xae,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_nlt_f64_e64 -|exec|, src_scc     ; encoding: [0x7e,0x01,0xae,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xae,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_nlt_f64_e64 -|src_scc|, -|exec|  ; encoding: [0x7e,0x03,0xae,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xae,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_nlt_f64_e64 0.5, null            ; encoding: [0x7e,0x00,0xae,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xae,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nlt_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xae,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xae,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_nlt_f64_e64 null, 0.5            ; encoding: [0x7e,0x00,0xae,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xae,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_nlt_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xae,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xae,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_nlt_f64_e64 s[2:3], s[4:5]       ; encoding: [0x7e,0x00,0xae,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xae,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_nlt_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xae,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xae,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_nlt_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xae,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xae,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_nlt_f64_e64 v[2:3], v[2:3]       ; encoding: [0x7e,0x00,0xae,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xae,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_nlt_f64_e64 vcc, ttmp[14:15]     ; encoding: [0x7e,0x00,0xae,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_o_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x87,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x87,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_o_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x87,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_o_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x87,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_o_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x87,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x87,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_o_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x87,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x87,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_o_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x87,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x87,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_o_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x87,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_o_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_o_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x87,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_o_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x87,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_o_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x87,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_o_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x87,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x87,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_o_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x87,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x87,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_o_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x87,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x87,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_o_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x87,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x97,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_o_f32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x97,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x97,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_o_f32_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x97,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x97,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_o_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x97,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x97,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_o_f32_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x97,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x97,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_o_f32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x97,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x97,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_o_f32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x97,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x97,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_o_f32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x97,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x97,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_o_f32_e64 s1, s2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x97,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_o_f32_e64 s105, s105             ; encoding: [0x7e,0x00,0x97,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x97,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_o_f32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x97,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_o_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x97,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x97,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_o_f32_e64 v255, v255             ; encoding: [0x7e,0x00,0x97,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x97,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_o_f32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0x97,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x97,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_o_f32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x97,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x97,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_o_f32_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x97,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xa7,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_o_f64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xa7,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xa7,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_o_f64_e64 -|exec|, src_scc       ; encoding: [0x7e,0x01,0xa7,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xa7,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_o_f64_e64 -|src_scc|, -|exec|    ; encoding: [0x7e,0x03,0xa7,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xa7,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_o_f64_e64 0.5, null              ; encoding: [0x7e,0x00,0xa7,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xa7,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_o_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa7,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa7,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_o_f64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xa7,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xa7,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_o_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa7,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xa7,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_o_f64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xa7,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xa7,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_o_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa7,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa7,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_o_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa7,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xa7,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_o_f64_e64 v[2:3], v[2:3]         ; encoding: [0x7e,0x00,0xa7,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xa7,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_o_f64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xa7,0xd4,0x6a,0xf4,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_u_f16_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x88,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x88,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_u_f16_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x88,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_u_f16_e64 -|0xfe0b|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x88,0xd4,0xff,0xd6,0x00,0x60,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_u_f16_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x88,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x88,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_u_f16_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x88,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x88,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_u_f16_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x88,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x88,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_u_f16_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x88,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_u_f16_e64 s1, s2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_u_f16_e64 s105, s105             ; encoding: [0x7e,0x00,0x88,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_u_f16_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x88,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_u_f16_e64 v1, v2                 ; encoding: [0x7e,0x00,0x88,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_u_f16_e64 v255, v255             ; encoding: [0x7e,0x00,0x88,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x88,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00
+# GFX1250: v_cmpx_u_f16_e64 vcc_hi, 0xfe0b         ; encoding: [0x7e,0x00,0x88,0xd4,0x6b,0xfe,0x01,0x00,0x0b,0xfe,0x00,0x00]
+
+0x7e,0x00,0x88,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_u_f16_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x88,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x88,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_u_f16_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x88,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0x98,0xd4,0xc1,0xfe,0x00,0x00
+# GFX1250: v_cmpx_u_f32_e64 -1, exec_hi            ; encoding: [0x7e,0x00,0x98,0xd4,0xc1,0xfe,0x00,0x00]
+
+0x7e,0x02,0x98,0xd4,0xfd,0xd4,0x00,0x20
+# GFX1250: v_cmpx_u_f32_e64 -src_scc, |vcc_lo|     ; encoding: [0x7e,0x02,0x98,0xd4,0xfd,0xd4,0x00,0x20]
+
+0x7e,0x83,0x98,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_u_f32_e64 -|0xaf123456|, -|vcc_hi| clamp ; encoding: [0x7e,0x83,0x98,0xd4,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x98,0xd4,0xf0,0xfa,0x00,0x40
+# GFX1250: v_cmpx_u_f32_e64 0.5, -m0               ; encoding: [0x7e,0x00,0x98,0xd4,0xf0,0xfa,0x00,0x40]
+
+0x7e,0x00,0x98,0xd4,0x7e,0x82,0x01,0x00
+# GFX1250: v_cmpx_u_f32_e64 exec_lo, -1            ; encoding: [0x7e,0x00,0x98,0xd4,0x7e,0x82,0x01,0x00]
+
+0x7e,0x00,0x98,0xd4,0x7d,0xe0,0x01,0x00
+# GFX1250: v_cmpx_u_f32_e64 m0, 0.5                ; encoding: [0x7e,0x00,0x98,0xd4,0x7d,0xe0,0x01,0x00]
+
+0x7e,0x00,0x98,0xd4,0x7c,0xfc,0x00,0x00
+# GFX1250: v_cmpx_u_f32_e64 null, exec_lo          ; encoding: [0x7e,0x00,0x98,0xd4,0x7c,0xfc,0x00,0x00]
+
+0x7e,0x00,0x98,0xd4,0x01,0x04,0x00,0x00
+# GFX1250: v_cmpx_u_f32_e64 s1, s2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x04,0x00,0x00]
+
+0x7e,0x00,0x98,0xd4,0x69,0xd2,0x00,0x00
+# GFX1250: v_cmpx_u_f32_e64 s105, s105             ; encoding: [0x7e,0x00,0x98,0xd4,0x69,0xd2,0x00,0x00]
+
+0x7e,0x00,0x98,0xd4,0x7b,0xfa,0x01,0x00
+# GFX1250: v_cmpx_u_f32_e64 ttmp15, src_scc        ; encoding: [0x7e,0x00,0x98,0xd4,0x7b,0xfa,0x01,0x00]
+
+0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00
+# GFX1250: v_cmpx_u_f32_e64 v1, v2                 ; encoding: [0x7e,0x00,0x98,0xd4,0x01,0x05,0x02,0x00]
+
+0x7e,0x00,0x98,0xd4,0xff,0xff,0x03,0x00
+# GFX1250: v_cmpx_u_f32_e64 v255, v255             ; encoding: [0x7e,0x00,0x98,0xd4,0xff,0xff,0x03,0x00]
+
+0x7e,0x00,0x98,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_u_f32_e64 vcc_hi, 0xaf123456     ; encoding: [0x7e,0x00,0x98,0xd4,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0x98,0xd4,0x6a,0xf6,0x00,0x00
+# GFX1250: v_cmpx_u_f32_e64 vcc_lo, ttmp15         ; encoding: [0x7e,0x00,0x98,0xd4,0x6a,0xf6,0x00,0x00]
+
+0x7e,0x01,0x98,0xd4,0x7f,0xf8,0x00,0x00
+# GFX1250: v_cmpx_u_f32_e64 |exec_hi|, null        ; encoding: [0x7e,0x01,0x98,0xd4,0x7f,0xf8,0x00,0x00]
+
+0x7e,0x00,0xa8,0xd4,0xc1,0x82,0x01,0x00
+# GFX1250: v_cmpx_u_f64_e64 -1, -1                 ; encoding: [0x7e,0x00,0xa8,0xd4,0xc1,0x82,0x01,0x00]
+
+0x7e,0x01,0xa8,0xd4,0x7e,0xfa,0x01,0x20
+# GFX1250: v_cmpx_u_f64_e64 -|exec|, src_scc       ; encoding: [0x7e,0x01,0xa8,0xd4,0x7e,0xfa,0x01,0x20]
+
+0x7e,0x03,0xa8,0xd4,0xfd,0xfc,0x00,0x60
+# GFX1250: v_cmpx_u_f64_e64 -|src_scc|, -|exec|    ; encoding: [0x7e,0x03,0xa8,0xd4,0xfd,0xfc,0x00,0x60]
+
+0x7e,0x00,0xa8,0xd4,0xf0,0xf8,0x00,0x00
+# GFX1250: v_cmpx_u_f64_e64 0.5, null              ; encoding: [0x7e,0x00,0xa8,0xd4,0xf0,0xf8,0x00,0x00]
+
+0x7e,0x82,0xa8,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_u_f64_e64 0xaf123456, -|vcc| clamp ; encoding: [0x7e,0x82,0xa8,0xd4,0xff,0xd4,0x00,0x40,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa8,0xd4,0x7c,0xe0,0x01,0x00
+# GFX1250: v_cmpx_u_f64_e64 null, 0.5              ; encoding: [0x7e,0x00,0xa8,0xd4,0x7c,0xe0,0x01,0x00]
+
+0x7e,0x00,0xa8,0xd4,0x68,0xd0,0x00,0x00
+# GFX1250: v_cmpx_u_f64_e64 s[104:105], s[104:105] ; encoding: [0x7e,0x00,0xa8,0xd4,0x68,0xd0,0x00,0x00]
+
+0x7e,0x00,0xa8,0xd4,0x02,0x08,0x00,0x00
+# GFX1250: v_cmpx_u_f64_e64 s[2:3], s[4:5]         ; encoding: [0x7e,0x00,0xa8,0xd4,0x02,0x08,0x00,0x00]
+
+0x7e,0x00,0xa8,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+# GFX1250: v_cmpx_u_f64_e64 ttmp[14:15], 0xaf123456 ; encoding: [0x7e,0x00,0xa8,0xd4,0x7a,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+0x7e,0x00,0xa8,0xd4,0xfe,0xfd,0x03,0x00
+# GFX1250: v_cmpx_u_f64_e64 v[254:255], v[254:255] ; encoding: [0x7e,0x00,0xa8,0xd4,0xfe,0xfd,0x03,0x00]
+
+0x7e,0x00,0xa8,0xd4,0x02,0x05,0x02,0x00
+# GFX1250: v_cmpx_u_f64_e64 v[2:3], v[2:3]         ; encoding: [0x7e,0x00,0xa8,0xd4,0x02,0x05,0x02,0x00]
+
+0x7e,0x00,0xa8,0xd4,0x6a,0xf4,0x00,0x00
+# GFX1250: v_cmpx_u_f64_e64 vcc, ttmp[14:15]       ; encoding: [0x7e,0x00,0xa8,0xd4,0x6a,0xf4,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3p_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3p_dpp16.txt
new file mode 100644
index 0000000000000..73e9d731646ba
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3p_dpp16.txt
@@ -0,0 +1,10 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+
+# GFX1250: v_fma_mix_f32_bf16_e64_dpp v0, v1, v2, v3 row_ror:7 row_mask:0xf bank_mask:0x1 ; encoding: [0x00,0x00,0x3d,0xcc,0xfa,0x04,0x0e,0x04,0x01,0x27,0x01,0xf1]
+0x00,0x00,0x3d,0xcc,0xfa,0x04,0x0e,0x04,0x01,0x27,0x01,0xf1
+
+# GFX1250: v_fma_mixlo_bf16_e64_dpp v0, v1, v2, v3 op_sel_hi:[1,1,1] clamp quad_perm:[0,2,3,1] row_mask:0x0 bank_mask:0xf ; encoding: [0x00,0xc0,0x3e,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x78,0x00,0x0f]
+0x00,0xc0,0x3e,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x78,0x00,0x0f
+
+# GFX1250: v_fma_mixhi_bf16_e64_dpp v0, v1, v2, v3 op_sel_hi:[1,1,1] clamp quad_perm:[0,2,3,1] row_mask:0x0 bank_mask:0xf ; encoding: [0x00,0xc0,0x3f,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x78,0x00,0x0f]
+0x00,0xc0,0x3f,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x78,0x00,0x0f
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3p_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3p_dpp8.txt
new file mode 100644
index 0000000000000..27e7025978315
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_vop3p_dpp8.txt
@@ -0,0 +1,19 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+
+# GFX1250: v_fma_mix_f32_bf16_e64_dpp v0, v1, v2, v3 clamp dpp8:[2,2,2,2,4,4,4,4] fi:1 ; encoding: [0x00,0x80,0x3d,0xcc,0xea,0x04,0x0e,0x04,0x01,0x92,0x44,0x92]
+0x00,0x80,0x3d,0xcc,0xea,0x04,0x0e,0x04,0x01,0x92,0x44,0x92
+
+# GFX1250: v_fma_mix_f32_bf16_e64_dpp v0, v1, v2, v3 dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x00,0x3d,0xcc,0xe9,0x04,0x0e,0x04,0x01,0x92,0x44,0x92]
+0x00,0x00,0x3d,0xcc,0xe9,0x04,0x0e,0x04,0x01,0x92,0x44,0x92
+
+# GFX1250: v_fma_mixlo_bf16_e64_dpp v0, |v1|, -v2, |v3| dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x05,0x3e,0xcc,0xe9,0x04,0x0e,0x44,0x01,0x92,0x44,0x92]
+0x00,0x05,0x3e,0xcc,0xe9,0x04,0x0e,0x44,0x01,0x92,0x44,0x92
+
+# GFX1250: v_fma_mixlo_bf16_e64_dpp v0, |v1|, -v2, |v3| op_sel:[1,0,0] op_sel_hi:[1,0,0] dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x0d,0x3e,0xcc,0xe9,0x04,0x0e,0x4c,0x01,0x92,0x44,0x92]
+0x00,0x0d,0x3e,0xcc,0xe9,0x04,0x0e,0x4c,0x01,0x92,0x44,0x92
+
+# GFX1250: v_fma_mixhi_bf16_e64_dpp v0, |v1|, -v2, |v3| dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x05,0x3f,0xcc,0xe9,0x04,0x0e,0x44,0x01,0x92,0x44,0x92]
+0x00,0x05,0x3f,0xcc,0xe9,0x04,0x0e,0x44,0x01,0x92,0x44,0x92
+
+# GFX1250: v_fma_mixhi_bf16_e64_dpp v0, |v1|, -v2, |v3| op_sel:[1,0,0] op_sel_hi:[1,0,0] dpp8:[2,2,2,2,4,4,4,4] ; encoding: [0x00,0x0d,0x3f,0xcc,0xe9,0x04,0x0e,0x4c,0x01,0x92,0x44,0x92]
+0x00,0x0d,0x3f,0xcc,0xe9,0x04,0x0e,0x4c,0x01,0x92,0x44,0x92
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt
index 1016d07df4995..c88fbc2fc6c4f 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt
@@ -719,6 +719,24 @@
 0x7d,0x4f,0x80,0xbe
 # GFX12: s_barrier_signal_isfirst m0             ; encoding: [0x7d,0x4f,0x80,0xbe]
 
+0xc1,0x51,0x80,0xbe
+# GFX12: s_barrier_init -1                       ; encoding: [0xc1,0x51,0x80,0xbe]
+
+0xc2,0x51,0x80,0xbe
+# GFX12: s_barrier_init -2                       ; encoding: [0xc2,0x51,0x80,0xbe]
+
+0x7d,0x51,0x80,0xbe
+# GFX12: s_barrier_init m0                       ; encoding: [0x7d,0x51,0x80,0xbe]
+
+0xc1,0x52,0x80,0xbe
+# GFX12: s_barrier_join -1                       ; encoding: [0xc1,0x52,0x80,0xbe]
+
+0xc2,0x52,0x80,0xbe
+# GFX12: s_barrier_join -2                       ; encoding: [0xc2,0x52,0x80,0xbe]
+
+0x7d,0x52,0x80,0xbe
+# GFX12: s_barrier_join m0                       ; encoding: [0x7d,0x52,0x80,0xbe]
+
 0xc1,0x50,0x83,0xbe
 # GFX12: s_get_barrier_state s3, -1              ; encoding: [0xc1,0x50,0x83,0xbe]
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt
index 5be288300c2f2..d8899316f6415 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt
@@ -1,6 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace --check-prefixes=GFX12,GFX1200 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+WavefrontSize64 -disassemble -show-encoding < %s | FileCheck -strict-whitespace --check-prefixes=GFX12,GFX1200 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -strict-whitespace --check-prefixes=GFX12,GFX1250 %s
 
 0x02,0x04,0x80,0xa9
 # GFX12: s_add_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x80,0xa9]
@@ -54,7 +55,8 @@
 # GFX12: s_add_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x73,0x72,0x71,0x3f]
 
 0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf
-# GFX12: s_add_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf]
+# GFX1200: s_add_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf]
+# GFX1250: s_add_nc_u64 s[0:1], lit64(0xaf123456), s[2:3] ; encoding: [0xfe,0x02,0x80,0xa9,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0x7e,0x80,0xa9
 # GFX12: s_add_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x80,0xa9]
@@ -78,7 +80,8 @@
 # GFX12: s_add_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0xa9,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf
-# GFX12: s_add_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf]
+# GFX1200: s_add_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0xa9,0x56,0x34,0x12,0xaf]
+# GFX1250: s_add_nc_u64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0xa9,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0x04,0x00,0xaa
 # GFX12: s_sub_nc_u64 s[0:1], s[2:3], s[4:5]     ; encoding: [0x02,0x04,0x00,0xaa]
@@ -132,7 +135,8 @@
 # GFX12: s_sub_nc_u64 s[0:1], 0x3f717273, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x73,0x72,0x71,0x3f]
 
 0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf
-# GFX12: s_sub_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf]
+# GFX1200: s_sub_nc_u64 s[0:1], 0xaf123456, s[2:3] ; encoding: [0xff,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf]
+# GFX1250: s_sub_nc_u64 s[0:1], lit64(0xaf123456), s[2:3] ; encoding: [0xfe,0x02,0x00,0xaa,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0x7e,0x00,0xaa
 # GFX12: s_sub_nc_u64 s[0:1], s[2:3], exec       ; encoding: [0x02,0x7e,0x00,0xaa]
@@ -156,7 +160,8 @@
 # GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x00,0xaa,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf
-# GFX12: s_sub_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf]
+# GFX1200: s_sub_nc_u64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x00,0xaa,0x56,0x34,0x12,0xaf]
+# GFX1250: s_sub_nc_u64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x00,0xaa,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0x04,0x80,0xaa
 # GFX12: s_mul_u64 s[0:1], s[2:3], s[4:5]        ; encoding: [0x02,0x04,0x80,0xaa]
@@ -210,7 +215,8 @@
 # GFX12: s_mul_u64 s[0:1], 0x3f717273, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x73,0x72,0x71,0x3f]
 
 0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf
-# GFX12: s_mul_u64 s[0:1], 0xaf123456, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf]
+# GFX1200: s_mul_u64 s[0:1], 0xaf123456, s[2:3]    ; encoding: [0xff,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf]
+# GFX1250: s_mul_u64 s[0:1], lit64(0xaf123456), s[2:3] ; encoding: [0xfe,0x02,0x80,0xaa,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0x7e,0x80,0xaa
 # GFX12: s_mul_u64 s[0:1], s[2:3], exec          ; encoding: [0x02,0x7e,0x80,0xaa]
@@ -234,7 +240,8 @@
 # GFX12: s_mul_u64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0xaa,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf
-# GFX12: s_mul_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf]
+# GFX1200: s_mul_u64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0xaa,0x56,0x34,0x12,0xaf]
+# GFX1250: s_mul_u64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0xaa,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x01,0x02,0x05,0xa0
 # GFX12: s_add_f32 s5, s1, s2                    ; encoding: [0x01,0x02,0x05,0xa0]
@@ -1689,7 +1696,8 @@
 # GFX12: s_and_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf
-# GFX12: s_and_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf]
+# GFX1200: s_and_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf]
+# GFX1250: s_and_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8b,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x8b
 # GFX12: s_and_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8b]
@@ -1716,7 +1724,8 @@
 # GFX12: s_and_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8b,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf
-# GFX12: s_and_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf]
+# GFX1200: s_and_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8b,0x56,0x34,0x12,0xaf]
+# GFX1250: s_and_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8b,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0xc1,0x80,0x8b
 # GFX12: s_and_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8b]
@@ -1872,7 +1881,8 @@
 # GFX12: s_and_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf
-# GFX12: s_and_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf]
+# GFX1200: s_and_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x91,0x56,0x34,0x12,0xaf]
+# GFX1250: s_and_not1_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x91,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x91
 # GFX12: s_and_not1_b64 s[0:1], -1, s[4:5]       ; encoding: [0xc1,0x04,0x80,0x91]
@@ -1899,7 +1909,8 @@
 # GFX12: s_and_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x91,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf
-# GFX12: s_and_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf]
+# GFX1200: s_and_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x91,0x56,0x34,0x12,0xaf]
+# GFX1250: s_and_not1_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x91,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0xc1,0x80,0x91
 # GFX12: s_and_not1_b64 s[0:1], s[2:3], -1       ; encoding: [0x02,0xc1,0x80,0x91]
@@ -2055,7 +2066,8 @@
 # GFX12: s_ashr_i64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x86,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf
-# GFX12: s_ashr_i64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf]
+# GFX1200: s_ashr_i64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x86,0x56,0x34,0x12,0xaf]
+# GFX1250: s_ashr_i64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x80,0x86,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x86
 # GFX12: s_ashr_i64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x86]
@@ -2238,7 +2250,8 @@
 # GFX12: s_bfe_i64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x80,0x94,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf
-# GFX12: s_bfe_i64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf]
+# GFX1200: s_bfe_i64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x80,0x94,0x56,0x34,0x12,0xaf]
+# GFX1250: s_bfe_i64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x80,0x94,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x94
 # GFX12: s_bfe_i64 s[0:1], -1, s4                ; encoding: [0xc1,0x04,0x80,0x94]
@@ -2421,7 +2434,8 @@
 # GFX12: s_bfe_u64 s[0:1], 0x3f717273, s4        ; encoding: [0xff,0x04,0x00,0x94,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf
-# GFX12: s_bfe_u64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf]
+# GFX1200: s_bfe_u64 s[0:1], 0xaf123456, s4        ; encoding: [0xff,0x04,0x00,0x94,0x56,0x34,0x12,0xaf]
+# GFX1250: s_bfe_u64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x00,0x94,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x00,0x94
 # GFX12: s_bfe_u64 s[0:1], -1, s4                ; encoding: [0xc1,0x04,0x00,0x94]
@@ -2805,7 +2819,8 @@
 # GFX12: s_cselect_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf
-# GFX12: s_cselect_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf]
+# GFX1200: s_cselect_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x98,0x56,0x34,0x12,0xaf]
+# GFX1250: s_cselect_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x98,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x98
 # GFX12: s_cselect_b64 s[0:1], -1, s[4:5]        ; encoding: [0xc1,0x04,0x80,0x98]
@@ -2832,7 +2847,8 @@
 # GFX12: s_cselect_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x98,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf
-# GFX12: s_cselect_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf]
+# GFX1200: s_cselect_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x98,0x56,0x34,0x12,0xaf]
+# GFX1250: s_cselect_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x98,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0xc1,0x80,0x98
 # GFX12: s_cselect_b64 s[0:1], s[2:3], -1        ; encoding: [0x02,0xc1,0x80,0x98]
@@ -3408,7 +3424,8 @@
 # GFX12: s_lshl_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x84,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf
-# GFX12: s_lshl_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf]
+# GFX1200: s_lshl_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x84,0x56,0x34,0x12,0xaf]
+# GFX1250: s_lshl_b64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x80,0x84,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x84
 # GFX12: s_lshl_b64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x84]
@@ -3591,7 +3608,8 @@
 # GFX12: s_lshr_b64 s[0:1], 0x3f717273, s4       ; encoding: [0xff,0x04,0x80,0x85,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf
-# GFX12: s_lshr_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf]
+# GFX1200: s_lshr_b64 s[0:1], 0xaf123456, s4       ; encoding: [0xff,0x04,0x80,0x85,0x56,0x34,0x12,0xaf]
+# GFX1250: s_lshr_b64 s[0:1], lit64(0xaf123456), s4 ; encoding: [0xfe,0x04,0x80,0x85,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x85
 # GFX12: s_lshr_b64 s[0:1], -1, s4               ; encoding: [0xc1,0x04,0x80,0x85]
@@ -4509,7 +4527,8 @@
 # GFX12: s_nand_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf
-# GFX12: s_nand_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf]
+# GFX1200: s_nand_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf]
+# GFX1250: s_nand_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8e,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x8e
 # GFX12: s_nand_b64 s[0:1], -1, s[4:5]           ; encoding: [0xc1,0x04,0x80,0x8e]
@@ -4536,7 +4555,8 @@
 # GFX12: s_nand_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x8e,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf
-# GFX12: s_nand_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf]
+# GFX1200: s_nand_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x8e,0x56,0x34,0x12,0xaf]
+# GFX1250: s_nand_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8e,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0xc1,0x80,0x8e
 # GFX12: s_nand_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x8e]
@@ -4692,7 +4712,8 @@
 # GFX12: s_nor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf
-# GFX12: s_nor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf]
+# GFX1200: s_nor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf]
+# GFX1250: s_nor_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8f,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x8f
 # GFX12: s_nor_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8f]
@@ -4719,7 +4740,8 @@
 # GFX12: s_nor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8f,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf
-# GFX12: s_nor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf]
+# GFX1200: s_nor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8f,0x56,0x34,0x12,0xaf]
+# GFX1250: s_nor_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8f,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0xc1,0x80,0x8f
 # GFX12: s_nor_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8f]
@@ -4875,7 +4897,8 @@
 # GFX12: s_or_b64 s[0:1], 0x3f717273, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf
-# GFX12: s_or_b64 s[0:1], 0xaf123456, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf]
+# GFX1200: s_or_b64 s[0:1], 0xaf123456, s[4:5]     ; encoding: [0xff,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf]
+# GFX1250: s_or_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8c,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x8c
 # GFX12: s_or_b64 s[0:1], -1, s[4:5]             ; encoding: [0xc1,0x04,0x80,0x8c]
@@ -4902,7 +4925,8 @@
 # GFX12: s_or_b64 s[0:1], s[2:3], 0x3f717273     ; encoding: [0x02,0xff,0x80,0x8c,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf
-# GFX12: s_or_b64 s[0:1], s[2:3], 0xaf123456     ; encoding: [0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf]
+# GFX1200: s_or_b64 s[0:1], s[2:3], 0xaf123456     ; encoding: [0x02,0xff,0x80,0x8c,0x56,0x34,0x12,0xaf]
+# GFX1250: s_or_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8c,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0xc1,0x80,0x8c
 # GFX12: s_or_b64 s[0:1], s[2:3], -1             ; encoding: [0x02,0xc1,0x80,0x8c]
@@ -5058,7 +5082,8 @@
 # GFX12: s_or_not1_b64 s[0:1], 0x3f717273, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf
-# GFX12: s_or_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf]
+# GFX1200: s_or_not1_b64 s[0:1], 0xaf123456, s[4:5] ; encoding: [0xff,0x04,0x80,0x92,0x56,0x34,0x12,0xaf]
+# GFX1250: s_or_not1_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x92,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x92
 # GFX12: s_or_not1_b64 s[0:1], -1, s[4:5]        ; encoding: [0xc1,0x04,0x80,0x92]
@@ -5085,7 +5110,8 @@
 # GFX12: s_or_not1_b64 s[0:1], s[2:3], 0x3f717273 ; encoding: [0x02,0xff,0x80,0x92,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf
-# GFX12: s_or_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf]
+# GFX1200: s_or_not1_b64 s[0:1], s[2:3], 0xaf123456 ; encoding: [0x02,0xff,0x80,0x92,0x56,0x34,0x12,0xaf]
+# GFX1250: s_or_not1_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x92,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0xc1,0x80,0x92
 # GFX12: s_or_not1_b64 s[0:1], s[2:3], -1        ; encoding: [0x02,0xc1,0x80,0x92]
@@ -5871,7 +5897,8 @@
 # GFX12: s_xnor_b64 s[0:1], 0x3f717273, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf
-# GFX12: s_xnor_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf]
+# GFX1200: s_xnor_b64 s[0:1], 0xaf123456, s[4:5]   ; encoding: [0xff,0x04,0x80,0x90,0x56,0x34,0x12,0xaf]
+# GFX1250: s_xnor_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x90,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x90
 # GFX12: s_xnor_b64 s[0:1], -1, s[4:5]           ; encoding: [0xc1,0x04,0x80,0x90]
@@ -5898,7 +5925,8 @@
 # GFX12: s_xnor_b64 s[0:1], s[2:3], 0x3f717273   ; encoding: [0x02,0xff,0x80,0x90,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf
-# GFX12: s_xnor_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf]
+# GFX1200: s_xnor_b64 s[0:1], s[2:3], 0xaf123456   ; encoding: [0x02,0xff,0x80,0x90,0x56,0x34,0x12,0xaf]
+# GFX1250: s_xnor_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x90,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0xc1,0x80,0x90
 # GFX12: s_xnor_b64 s[0:1], s[2:3], -1           ; encoding: [0x02,0xc1,0x80,0x90]
@@ -6054,7 +6082,8 @@
 # GFX12: s_xor_b64 s[0:1], 0x3f717273, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x73,0x72,0x71,0x3f]
 
 0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf
-# GFX12: s_xor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf]
+# GFX1200: s_xor_b64 s[0:1], 0xaf123456, s[4:5]    ; encoding: [0xff,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf]
+# GFX1250: s_xor_b64 s[0:1], lit64(0xaf123456), s[4:5] ; encoding: [0xfe,0x04,0x80,0x8d,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0xc1,0x04,0x80,0x8d
 # GFX12: s_xor_b64 s[0:1], -1, s[4:5]            ; encoding: [0xc1,0x04,0x80,0x8d]
@@ -6081,7 +6110,8 @@
 # GFX12: s_xor_b64 s[0:1], s[2:3], 0x3f717273    ; encoding: [0x02,0xff,0x80,0x8d,0x73,0x72,0x71,0x3f]
 
 0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf
-# GFX12: s_xor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf]
+# GFX1200: s_xor_b64 s[0:1], s[2:3], 0xaf123456    ; encoding: [0x02,0xff,0x80,0x8d,0x56,0x34,0x12,0xaf]
+# GFX1250: s_xor_b64 s[0:1], s[2:3], lit64(0xaf123456) ; encoding: [0x02,0xfe,0x80,0x8d,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x02,0xc1,0x80,0x8d
 # GFX12: s_xor_b64 s[0:1], s[2:3], -1            ; encoding: [0x02,0xc1,0x80,0x8d]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt
index e59006c322b4e..a8da16fbb42cc 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopc.txt
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck --check-prefixes=GFX12,GFX1200 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck --check-prefixes=GFX12,GFX1250 %s
 
 0x01,0x02,0x41,0xbf
 # GFX12: s_cmp_lt_f32 s1, s2                     ; encoding: [0x01,0x02,0x41,0xbf]
@@ -1490,7 +1491,8 @@
 # GFX12: s_cmp_eq_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x10,0xbf,0x73,0x72,0x71,0x3f]
 
 0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf
-# GFX12: s_cmp_eq_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf]
+# GFX1200: s_cmp_eq_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x10,0xbf,0x56,0x34,0x12,0xaf]
+# GFX1250: s_cmp_eq_u64 s[0:1], lit64(0xaf123456)  ; encoding: [0x00,0xfe,0x10,0xbf,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x00,0xc1,0x10,0xbf
 # GFX12: s_cmp_eq_u64 s[0:1], -1                 ; encoding: [0x00,0xc1,0x10,0xbf]
@@ -2012,7 +2014,8 @@
 # GFX12: s_cmp_lg_u64 s[0:1], 0x3f717273         ; encoding: [0x00,0xff,0x11,0xbf,0x73,0x72,0x71,0x3f]
 
 0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf
-# GFX12: s_cmp_lg_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf]
+# GFX1200: s_cmp_lg_u64 s[0:1], 0xaf123456         ; encoding: [0x00,0xff,0x11,0xbf,0x56,0x34,0x12,0xaf]
+# GFX1250: s_cmp_lg_u64 s[0:1], lit64(0xaf123456)  ; encoding: [0x00,0xfe,0x11,0xbf,0x56,0x34,0x12,0xaf,0x00,0x00,0x00,0x00]
 
 0x00,0xc1,0x11,0xbf
 # GFX12: s_cmp_lg_u64 s[0:1], -1                 ; encoding: [0x00,0xc1,0x11,0xbf]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt
index f8aef72678623..1e40ea27c47d3 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt
@@ -68,6 +68,9 @@
 # GFX12: s_barrier_wait 1                        ; encoding: [0x01,0x00,0x94,0xbf]
 0x01,0x00,0x94,0xbf
 
+# GFX12: s_barrier_leave                         ; encoding: [0x00,0x00,0x95,0xbf]
+0x00,0x00,0x95,0xbf
+
 # GFX12: s_branch 0                              ; encoding: [0x00,0x00,0xa0,0xbf]
 0x00,0x00,0xa0,0xbf
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/kernel-descriptor-rsrc-errors.test b/llvm/test/MC/Disassembler/AMDGPU/kernel-descriptor-rsrc-errors.test
index 1a54bac5fa40f..be92aade6b531 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/kernel-descriptor-rsrc-errors.test
+++ b/llvm/test/MC/Disassembler/AMDGPU/kernel-descriptor-rsrc-errors.test
@@ -39,9 +39,14 @@
 
 # RUN: yaml2obj %s -DGPU=GFX1100 -DSRC1=0300AC60 -DSRC2=80000000 -DSRC3=00000100 \
 # RUN:   | llvm-objdump --disassemble-symbols=test.kd - | FileCheck %s --check-prefix=RSRC3_10
-# RSRC3_10: ; error decoding test.kd: kernel descriptor COMPUTE_PGM_RSRC3 reserved bits in range (30:14) set, must be zero on gfx10+
+# RSRC3_10: ; error decoding test.kd: kernel descriptor COMPUTE_PGM_RSRC3 reserved bits in range (21:14) set, must be zero on gfx10+
 # RSRC3_10-NEXT: ; decoding failed region as bytes
 
+# RUN: yaml2obj %s -DGPU=GFX1100 -DSRC1=0300AC60 -DSRC2=80000000 -DSRC3=00004000 \
+# RUN:   | llvm-objdump --disassemble-symbols=test.kd - | FileCheck %s --check-prefix=RSRC3_10_1
+# RSRC3_10_1: ; error decoding test.kd: kernel descriptor COMPUTE_PGM_RSRC3 reserved bits in range (30:22) set, must be zero on gfx10+
+# RSRC3_10_1-NEXT: ; decoding failed region as bytes
+
 # RUN: yaml2obj %s -DGPU=GFX801 -DSRC1=0300AC60 -DSRC2=80000000 -DSRC3=00000001 \
 # RUN:   | llvm-objdump --disassemble-symbols=test.kd - | FileCheck %s --check-prefix=RSRC3_PRE_9
 # RSRC3_PRE_9: ; error decoding test.kd: kernel descriptor COMPUTE_PGM_RSRC3 must be all zero before gfx9
diff --git a/llvm/test/MC/Disassembler/RISCV/bigendian.txt b/llvm/test/MC/Disassembler/RISCV/bigendian.txt
new file mode 100644
index 0000000000000..54930f94b75ed
--- /dev/null
+++ b/llvm/test/MC/Disassembler/RISCV/bigendian.txt
@@ -0,0 +1,29 @@
+# RUN: llvm-mc --disassemble %s -triple=riscv32be -mattr=+c 2>&1 | FileCheck %s
+# RUN: llvm-mc --disassemble %s -triple=riscv64be -mattr=+c 2>&1 | FileCheck %s
+
+# Test basic disassembly for big-endian RISC-V
+# Instructions are always little-endian encoded in RISC-V
+
+[0x13,0x05,0x45,0x06]
+# CHECK: addi a0, a0, 100
+
+[0xb7,0x52,0x34,0x12]
+# CHECK: lui t0, 74565
+
+[0x03,0x26,0x05,0x00]
+# CHECK: lw a2, 0(a0)
+
+[0x23,0x22,0xc5,0x00]
+# CHECK: sw a2, 4(a0)
+
+[0xef,0x00,0x00,0x00]
+# CHECK: jal 0
+
+[0x63,0x00,0xb5,0x00]
+# CHECK: beq a0, a1, 0
+
+[0x01,0x00]
+# CHECK: nop
+
+[0x05,0x05]
+# CHECK: addi a0, a0, 1
diff --git a/llvm/test/MC/Disassembler/RISCV/riscv-mapping-symbols.s b/llvm/test/MC/Disassembler/RISCV/riscv-mapping-symbols.s
new file mode 100644
index 0000000000000..ff15008b8e3f8
--- /dev/null
+++ b/llvm/test/MC/Disassembler/RISCV/riscv-mapping-symbols.s
@@ -0,0 +1,20 @@
+# RUN: llvm-mc --triple=riscv32-unknown-none-elf %s -filetype=obj -o - \
+# RUN: | llvm-objdump -dr - \
+# RUN: | FileCheck %s
+# RUN: llvm-mc --triple=riscv64-unknown-none-elf %s -filetype=obj -o - \
+# RUN: | llvm-objdump -dr - \
+# RUN: | FileCheck %s
+
+
+  # CHECK: 00000013 nop
+  nop
+
+  # CHECK-NEXT: 55 55 55 55 .word 0x55555555
+  .word 0x55555555
+
+  # CHECK-NEXT: 00 00 00 00 .word 0x00000000
+  # CHECK-NEXT: R_RISCV_32 foo
+  .word foo
+
+  # CHECK-NEXT: 00000013 nop
+  nop
diff --git a/llvm/test/MC/ELF/cfi-sframe.s b/llvm/test/MC/ELF/cfi-sframe.s
new file mode 100644
index 0000000000000..66b9ee61bdaff
--- /dev/null
+++ b/llvm/test/MC/ELF/cfi-sframe.s
@@ -0,0 +1,72 @@
+// TODO: Add other architectures as they gain sframe support
+// REQUIRES: x86-registered-target
+// RUN: llvm-mc --assemble --filetype=obj --gsframe -triple x86_64 %s -o %t.o
+// RUN: llvm-readelf --sframe %t.o | FileCheck %s
+
+	.cfi_sections .sframe
+f1:
+	.cfi_startproc
+	nop
+        .cfi_endproc
+
+f2:
+	.cfi_startproc
+	nop
+	nop
+        .cfi_endproc
+
+// CHECK: SFrame section '.sframe' {
+// CHECK-NEXT:  Header {
+// CHECK-NEXT:    Magic: 0xDEE2
+// CHECK-NEXT:    Version: V2 (0x2)
+// CHECK-NEXT:    Flags [ (0x4)
+// CHECK:    ABI: AMD64EndianLittle (0x3)
+// CHECK-NEXT:    CFA fixed FP offset (unused): 0
+// CHECK-NEXT:    CFA fixed RA offset: 0
+// CHECK-NEXT:    Auxiliary header length: 0
+// CHECK-NEXT:    Num FDEs: 2
+// CHECK-NEXT:    Num FREs: 0
+// CHECK-NEXT:    FRE subsection length: 0
+// CHECK-NEXT:    FDE subsection offset: 0
+// CHECK-NEXT:    FRE subsection offset: 40
+// CHECK:    Function Index [
+// CHECK-NEXT:        FuncDescEntry [0] {
+// CHECK-NEXT:          PC {
+// CHECK-NEXT:            Relocation: {{.*}}32{{.*}}
+// CHECK-NEXT:            Symbol Name: .text
+// CHECK-NEXT:            Start Address: 0x0
+// CHECK-NEXT:          }
+// CHECK-NEXT:          Size: 0x1
+// CHECK-NEXT:          Start FRE Offset: 0x0
+// CHECK-NEXT:          Num FREs: 0
+// CHECK-NEXT:          Info {
+// CHECK-NEXT:            FRE Type: Addr1 (0x0)
+// CHECK-NEXT:            FDE Type: PCInc (0x0)
+// CHECK-NEXT:            Raw: 0x0
+// CHECK-NEXT:          }
+// CHECK-NEXT:          Repetitive block size (unused): 0x0
+// CHECK-NEXT:          Padding2: 0x0
+// CHECK-NEXT:          FREs [
+// CHECK-NEXT:          ]
+// CHECK-NEXT:        }
+// CHECK-NEXT:        FuncDescEntry [1] {
+// CHECK-NEXT:          PC {
+// CHECK-NEXT:            Relocation: R_X86_64_PC32
+// CHECK-NEXT:            Symbol Name: .text
+// CHECK-NEXT:            Start Address: {{.*}}
+// CHECK-NEXT:          }
+// CHECK-NEXT:          Size: 0x2
+// CHECK-NEXT:          Start FRE Offset: 0x0
+// CHECK-NEXT:          Num FREs: 0
+// CHECK-NEXT:          Info {
+// CHECK-NEXT:            FRE Type: Addr1 (0x0)
+// CHECK-NEXT:            FDE Type: PCInc (0x0)
+// CHECK-NEXT:            Raw: 0x0
+// CHECK-NEXT:          }
+// CHECK-NEXT:          Repetitive block size (unused): 0x0
+// CHECK-NEXT:          Padding2: 0x0
+// CHECK-NEXT:          FREs [
+// CHECK-NEXT:          ]
+// CHECK-NEXT:        }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:    }
diff --git a/llvm/test/MC/ELF/many-instructions.s b/llvm/test/MC/ELF/many-instructions.s
index 843d35fe6a3a4..7c13c0d63d536 100644
--- a/llvm/test/MC/ELF/many-instructions.s
+++ b/llvm/test/MC/ELF/many-instructions.s
@@ -1,4 +1,5 @@
-# REQUIRES: asserts
+## Checks the size of an internal MC structure that is different on 32-bit.
+# REQUIRES: asserts, llvm-64-bits
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o /dev/null -debug-only=mc-dump 2>&1 | grep -E -o '[0-9]+ Data Size:[0-9]+' | FileCheck %s
 
 ## Test that encodeInstruction may cause a new fragment to be created.
diff --git a/llvm/test/MC/ELF/mc-dump.s b/llvm/test/MC/ELF/mc-dump.s
index a590e1c3d9be4..51b3ff4774cf3 100644
--- a/llvm/test/MC/ELF/mc-dump.s
+++ b/llvm/test/MC/ELF/mc-dump.s
@@ -30,7 +30,7 @@
 # CHECK-NEXT:5 LEB Size:0+1 [15] Value:.Ltmp0-_start Signed:0
 #      CHECK:]
 
-# CHECK:  3 assembler         - Number of fixup evaluations for relaxation
+# CHECK:  2 assembler         - Number of fixup evaluations for relaxation
 # CHECK:  8 assembler         - Number of fixups
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t -debug-only=mc-dump -save-temp-labels -g 2>&1 | FileCheck %s --check-prefix=CHECK2
diff --git a/llvm/test/MC/Hexagon/system-inst.s b/llvm/test/MC/Hexagon/system-inst.s
index 7bc1533598532..07f7ca0acb2dc 100644
--- a/llvm/test/MC/Hexagon/system-inst.s
+++ b/llvm/test/MC/Hexagon/system-inst.s
@@ -89,6 +89,9 @@ crswap(r12,sgp0)
 #CHECK: 652dc000 { crswap(r13,sgp1) }
 crswap(r13,sgp1)
 
+#CHECK: 6d8ec000 { crswap(r15:14,s1:0) }
+crswap(r15:14,sgp1:0)
+
 #CHECK: 660fc00e { r14 = getimask(r15) }
 r14=getimask(r15)
 
diff --git a/llvm/test/MC/LoongArch/Misc/cfi-advance.s b/llvm/test/MC/LoongArch/Misc/cfi-advance.s
index 38eba7caf6106..86b36a38c3f15 100644
--- a/llvm/test/MC/LoongArch/Misc/cfi-advance.s
+++ b/llvm/test/MC/LoongArch/Misc/cfi-advance.s
@@ -5,13 +5,13 @@
 # RUN:     | llvm-readobj -r - | FileCheck --check-prefix=RELAX %s
 
 # RELOC:       Relocations [
-# RELOC-NEXT:    .rela.eh_frame {
-# RELOC-NEXT:       0x1C R_LARCH_32_PCREL .text 0x0
+# RELOC:         .rela.eh_frame {
+# RELOC-NEXT:       0x1C R_LARCH_32_PCREL .L{{.*}} 0x0
 # RELOC-NEXT:    }
 # RELOC-NEXT:  ]
-# DWARFDUMP:       DW_CFA_advance_loc: 4
+# DWARFDUMP:       DW_CFA_advance_loc: 8
 # DWARFDUMP-NEXT:  DW_CFA_def_cfa_offset: +8
-# DWARFDUMP-NEXT:  DW_CFA_advance_loc: 8
+# DWARFDUMP-NEXT:  DW_CFA_advance_loc: 4
 # DWARFDUMP-NEXT:  DW_CFA_def_cfa_offset: +8
 
 # RELAX:       Relocations [
@@ -19,6 +19,8 @@
 # RELAX-NEXT:       0x1C R_LARCH_32_PCREL .L{{.*}} 0x0
 # RELAX-NEXT:       0x20 R_LARCH_ADD32 .L{{.*}} 0x0
 # RELAX-NEXT:       0x20 R_LARCH_SUB32 .L{{.*}} 0x0
+# RELAX-NEXT:       0x25 R_LARCH_ADD6 .L{{.*}} 0x0
+# RELAX-NEXT:       0x25 R_LARCH_SUB6 .L{{.*}} 0x0
 # RELAX-NEXT:       0x28 R_LARCH_ADD6 .L{{.*}} 0x0
 # RELAX-NEXT:       0x28 R_LARCH_SUB6 .L{{.*}} 0x0
 # RELAX-NEXT:    }
@@ -30,7 +32,7 @@
         .type   test,@function
 test:
         .cfi_startproc
-        nop
+        call36 foo
         .cfi_def_cfa_offset 8
         .p2align 3
         nop
diff --git a/llvm/test/MC/LoongArch/Relocations/align-after-relax.s b/llvm/test/MC/LoongArch/Relocations/align-after-relax.s
new file mode 100644
index 0000000000000..199c3fcfa0afd
--- /dev/null
+++ b/llvm/test/MC/LoongArch/Relocations/align-after-relax.s
@@ -0,0 +1,57 @@
+## The file testing R_LARCH_ALIGN emitting when linker-relaxation enabled.
+
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=-relax %s -o %t.n
+# RUN: llvm-objdump -dr %t.n | FileCheck %s --check-prefix=NORELAX
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax %s -o %t.r
+# RUN: llvm-objdump -dr %t.r | FileCheck %s --check-prefix=RELAX
+
+# NORELAX:         pcaddu18i $ra, 0
+# NORELAX-NEXT:        R_LARCH_CALL36 f
+# NORELAX-NEXT:    jirl $ra, $ra, 0
+# NORELAX-COUNT-6: nop
+# NORELAX:         pcaddu18i $ra, 0
+# NORELAX-NEXT:        R_LARCH_CALL36 f
+# NORELAX-NEXT:    jirl $ra, $ra, 0
+# NORELAX-COUNT-6: nop
+# NORELAX:         pcaddu18i $ra, 0
+# NORELAX-NEXT:        R_LARCH_CALL36 f
+# NORELAX-NEXT:    jirl $ra, $ra, 0
+
+# RELAX:         pcaddu18i $ra, 0
+# RELAX-NEXT:        R_LARCH_CALL36 f
+# RELAX-NEXT:        R_LARCH_RELAX *ABS*
+# RELAX-NEXT:    jirl $ra, $ra, 0
+# RELAX-NEXT:    nop
+# RELAX-NEXT:        R_LARCH_ALIGN *ABS*+0x1c
+# RELAX-COUNT-6: nop
+# RELAX:         pcaddu18i $ra, 0
+# RELAX-NEXT:        R_LARCH_CALL36 f
+# RELAX-NEXT:        R_LARCH_RELAX *ABS*
+# RELAX-NEXT:    jirl $ra, $ra, 0
+# RELAX-NEXT:    nop
+# RELAX-NEXT:        R_LARCH_ALIGN *ABS*+0x1c
+# RELAX-COUNT-6: nop
+# RELAX:         pcaddu18i $ra, 0
+# RELAX-NEXT:        R_LARCH_CALL36 f
+# RELAX-NEXT:    jirl $ra, $ra, 0
+
+.text
+## No R_LARCH_ALIGN before the first linker-relaxable instruction.
+.p2align 5
+foo:
+  call36 f
+
+## R_LARCH_ALIGN is required after the first linker-relaxable instruction.
+.p2align 5
+bar:
+  call36 f
+
+.option push
+.option norelax
+## R_LARCH_ALIGN is required even if norelax, because it is after a
+## linker-relaxable instruction. No R_LARCH_RELAX for call36 because
+## of the norelax.
+.p2align 5
+baz:
+  call36 f
+.option pop
diff --git a/llvm/test/MC/LoongArch/Relocations/fde-reloc.s b/llvm/test/MC/LoongArch/Relocations/fde-reloc.s
index ab911d1853a87..3b9f4003950f8 100644
--- a/llvm/test/MC/LoongArch/Relocations/fde-reloc.s
+++ b/llvm/test/MC/LoongArch/Relocations/fde-reloc.s
@@ -1,7 +1,7 @@
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=-relax < %s \
 # RUN:     | llvm-readobj -r - | FileCheck %s
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax < %s \
-# RUN:     | llvm-readobj -r - | FileCheck %s --check-prefix=RELAX
+# RUN:     | llvm-readobj -r - | FileCheck %s
 
 ## Ensure that the eh_frame records the symbolic difference with
 ## the R_LARCH_32_PCREL relocation.
@@ -11,9 +11,6 @@ func:
   ret
  .cfi_endproc
 
-# CHECK:   Section (4) .rela.eh_frame {
-# CHECK-NEXT:   0x1C R_LARCH_32_PCREL .text 0x0
+# CHECK:   Section ({{.*}}) .rela.eh_frame {
+# CHECK-NEXT:   0x1C R_LARCH_32_PCREL .L{{.*}} 0x0
 # CHECK-NEXT: }
-# RELAX:   Section ({{.*}}) .rela.eh_frame {
-# RELAX-NEXT:   0x1C R_LARCH_32_PCREL .L{{.*}} 0x0
-# RELAX-NEXT: }
diff --git a/llvm/test/MC/LoongArch/Relocations/relax-addsub.s b/llvm/test/MC/LoongArch/Relocations/relax-addsub.s
index f2524b29d230b..67c643c076895 100644
--- a/llvm/test/MC/LoongArch/Relocations/relax-addsub.s
+++ b/llvm/test/MC/LoongArch/Relocations/relax-addsub.s
@@ -5,34 +5,37 @@
 
 # NORELAX:       Relocations [
 # NORELAX-NEXT:    Section ({{.*}}) .rela.text {
-# NORELAX-NEXT:      0x10 R_LARCH_PCALA_HI20 .text 0x0
-# NORELAX-NEXT:      0x14 R_LARCH_PCALA_LO12 .text 0x0
+# NORELAX-NEXT:      0x0 R_LARCH_CALL36 foo 0x0
+# NORELAX-NEXT:      0x10 R_LARCH_PCALA_HI20 .L1 0x0
+# NORELAX-NEXT:      0x14 R_LARCH_PCALA_LO12 .L1 0x0
 # NORELAX-NEXT:    }
 # NORELAX-NEXT:    Section ({{.*}}) .rela.data {
 # NORELAX-NEXT:      0x30 R_LARCH_ADD8 foo 0x0
-# NORELAX-NEXT:      0x30 R_LARCH_SUB8 .text 0x10
+# NORELAX-NEXT:      0x30 R_LARCH_SUB8 .L3 0x0
 # NORELAX-NEXT:      0x31 R_LARCH_ADD16 foo 0x0
-# NORELAX-NEXT:      0x31 R_LARCH_SUB16 .text 0x10
+# NORELAX-NEXT:      0x31 R_LARCH_SUB16 .L3 0x0
 # NORELAX-NEXT:      0x33 R_LARCH_ADD32 foo 0x0
-# NORELAX-NEXT:      0x33 R_LARCH_SUB32 .text 0x10
+# NORELAX-NEXT:      0x33 R_LARCH_SUB32 .L3 0x0
 # NORELAX-NEXT:      0x37 R_LARCH_ADD64 foo 0x0
-# NORELAX-NEXT:      0x37 R_LARCH_SUB64 .text 0x10
+# NORELAX-NEXT:      0x37 R_LARCH_SUB64 .L3 0x0
 # NORELAX-NEXT:    }
 # NORELAX-NEXT:  ]
 
 # NORELAX:      Hex dump of section '.data':
 # NORELAX-NEXT: 0x00000000 04040004 00000004 00000000 00000004
-# NORELAX-NEXT: 0x00000010 0c0c000c 0000000c 00000000 0000000c
+# NORELAX-NEXT: 0x00000010 04040004 00000004 00000000 00000004
 # NORELAX-NEXT: 0x00000020 08080008 00000008 00000000 00000008
 # NORELAX-NEXT: 0x00000030 00000000 00000000 00000000 000000
 
 # RELAX:       Relocations [
 # RELAX-NEXT:    Section ({{.*}}) .rela.text {
-# RELAX-NEXT:      0x4 R_LARCH_ALIGN - 0xC
-# RELAX-NEXT:      0x10 R_LARCH_PCALA_HI20 .L1 0x0
-# RELAX-NEXT:      0x10 R_LARCH_RELAX - 0x0
-# RELAX-NEXT:      0x14 R_LARCH_PCALA_LO12 .L1 0x0
-# RELAX-NEXT:      0x14 R_LARCH_RELAX - 0x0
+# RELAX-NEXT:      0x0 R_LARCH_CALL36 foo 0x0
+# RELAX-NEXT:      0x0 R_LARCH_RELAX - 0x0
+# RELAX-NEXT:      0xC R_LARCH_ALIGN - 0xC
+# RELAX-NEXT:      0x18 R_LARCH_PCALA_HI20 .L1 0x0
+# RELAX-NEXT:      0x18 R_LARCH_RELAX - 0x0
+# RELAX-NEXT:      0x1C R_LARCH_PCALA_LO12 .L1 0x0
+# RELAX-NEXT:      0x1C R_LARCH_RELAX - 0x0
 # RELAX-NEXT:    }
 # RELAX-NEXT:    Section ({{.*}}) .rela.data {
 # RELAX-NEXT:      0x10 R_LARCH_ADD8 .L3 0x0
@@ -73,6 +76,7 @@
 # RELAX-NEXT: 0x00000030 00000000 00000000 00000000 000000
 
 .text
+  call36 foo
 .L1:
   nop
 .L2:
diff --git a/llvm/test/MC/LoongArch/Relocations/relax-align-in-subsection.s b/llvm/test/MC/LoongArch/Relocations/relax-align-in-subsection.s
new file mode 100644
index 0000000000000..92fceeb14faee
--- /dev/null
+++ b/llvm/test/MC/LoongArch/Relocations/relax-align-in-subsection.s
@@ -0,0 +1,35 @@
+## The file testing R_LARCH_ALIGN emitting when linker-relaxation enabled.
+
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax %s -o %t.n
+# RUN: llvm-objdump -dr %t.n | FileCheck %s --check-prefix=RELAX
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax --defsym FILL=1 %s -o %t.f
+# RUN: llvm-objdump -dr %t.f | FileCheck %s --check-prefixes=RELAX,ALIGN
+
+# ALIGN:         nop
+# ALIGN-NEXT:        R_LARCH_ALIGN *ABS*+0x1c
+# ALIGN-COUNT-6: nop
+# RELAX:         ret
+# RELAX:         pcaddu18i $ra, 0
+# RELAX-NEXT:        R_LARCH_CALL36 f
+# RELAX-NEXT:        R_LARCH_RELAX *ABS*
+# RELAX-NEXT:    jirl $ra, $ra, 0
+
+.text
+.option push
+.option norelax
+## When FILL is defined, the order of Alignment directive in this lower-numbered
+## subsection will be larger, and even larger than the section order of the first
+## linker-relaxable call36 instruction. It should conservatively be treated as
+## linker-relaxable even has norelax.
+.ifdef FILL
+  .space 0
+.endif
+.p2align 5
+foo:
+  ret
+.option pop
+
+.text 1
+  .space 0
+bar:
+  call36 f
diff --git a/llvm/test/MC/LoongArch/Relocations/relax-align.s b/llvm/test/MC/LoongArch/Relocations/relax-align.s
index 477d5ca24ec7d..e9d7f31b68393 100644
--- a/llvm/test/MC/LoongArch/Relocations/relax-align.s
+++ b/llvm/test/MC/LoongArch/Relocations/relax-align.s
@@ -8,8 +8,9 @@
 # RUN: llvm-readobj -r %t.r | FileCheck %s --check-prefixes=RELOC,RELAX-RELOC
 
 .text
-break 0
-# INSTR: break 0
+call36 foo
+# INSTR:      pcaddu18i $ra, 0
+# INSTR-NEXT: jirl $ra, $ra, 0
 
 ## Not emit R_LARCH_ALIGN if alignment directive is less than or equal to
 ## minimum code alignment(a.k.a 4).
@@ -24,8 +25,8 @@ break 0
 ## The behavior is the same as GNU assembler.
 break 1
 .p2align 4, 1
-# INSTR-NEXT:    break 1
-# INSTR-COUNT-2: 01 01 01 01
+# INSTR-NEXT: break 1
+# INSTR-NEXT: 01 01 01 01
 
 break 2
 .p2align 4, 1, 12
@@ -62,20 +63,25 @@ ret
 
 ## Test the symbol index is different from .text.
 .section .text2, "ax"
+call36 foo
 .p2align 4
 .p2align 4, , 4
 break 7
 
 # RELOC:            Relocations [
-# RELAX-RELOC-NEXT:   Section ({{.*}}) .rela.text {
+# RELOC-NEXT:         Section ({{.*}}) .rela.text {
+# RELOC-NEXT:           0x0 R_LARCH_CALL36 foo 0x0
+# RELAX-RELOC-NEXT:     0x0 R_LARCH_RELAX - 0x0
 # RELAX-RELOC-NEXT:     0x24 R_LARCH_ALIGN - 0xC
 # RELAX-RELOC-NEXT:     0x34 R_LARCH_ALIGN - 0x1C
 # RELAX-RELOC-NEXT:     0x50 R_LARCH_ALIGN - 0xC
 # RELAX-RELOC-NEXT:     0x60 R_LARCH_ALIGN .Lla-relax-align0 0xB04
 # RELAX-RELOC-NEXT:     0x70 R_LARCH_ALIGN - 0xC
-# RELAX-RELOC-NEXT:   }
-# RELAX-RELOC-NEXT:   Section ({{.*}}) .rela.text2 {
-# RELAX-RELOC-NEXT:     0x0 R_LARCH_ALIGN - 0xC
-# RELAX-RELOC-NEXT:     0xC R_LARCH_ALIGN .Lla-relax-align1 0x404
-# RELAX-RELOC-NEXT:   }
+# RELOC-NEXT:         }
+# RELOC-NEXT:         Section ({{.*}}) .rela.text2 {
+# RELOC-NEXT:           0x0 R_LARCH_CALL36 foo 0x0
+# RELAX-RELOC-NEXT:     0x0 R_LARCH_RELAX - 0x0
+# RELAX-RELOC-NEXT:     0x8 R_LARCH_ALIGN - 0xC
+# RELAX-RELOC-NEXT:     0x14 R_LARCH_ALIGN .Lla-relax-align1 0x404
+# RELOC-NEXT:         }
 # RELOC-NEXT:       ]
diff --git a/llvm/test/MC/LoongArch/Relocations/relax-attr.s b/llvm/test/MC/LoongArch/Relocations/relax-attr.s
index b1e648d850bb9..7cc8dda07e333 100644
--- a/llvm/test/MC/LoongArch/Relocations/relax-attr.s
+++ b/llvm/test/MC/LoongArch/Relocations/relax-attr.s
@@ -1,19 +1,24 @@
-# RUN: llvm-mc --filetype=obj --triple=loongarch64 %s -o %t
-# RUN: llvm-readobj -r %t | FileCheck %s
-# RUN: llvm-mc --filetype=obj --triple=loongarch64 -mattr=+relax %s -o %t
-# RUN: llvm-readobj -r %t | FileCheck %s --check-prefix=CHECKR
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 -mattr=-relax %s -o %t.n
+# RUN: llvm-readobj -r %t.n | FileCheck %s
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 -mattr=+relax %s -o %t.r
+# RUN: llvm-readobj -r %t.r | FileCheck %s --check-prefix=CHECKR
 
 # CHECK:      Relocations [
+# CHECK-NEXT:   Section ({{.*}}) .rela.text {
+# CHECK-NEXT:     0x4 R_LARCH_CALL36 foo 0x0
+# CHECK-NEXT:   }
 # CHECK-NEXT:   Section ({{.*}}) .rela.data {
-# CHECK-NEXT:     0x0 R_LARCH_64 .text 0x4
+# CHECK-NEXT:     0x0 R_LARCH_64 .L1 0x0
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
 # CHECKR:      Relocations [
 # CHECKR-NEXT:   Section ({{.*}}) .rela.text {
-# CHECKR-NEXT:     0x8 R_LARCH_B21 .L1 0x0
-# CHECKR-NEXT:     0xC R_LARCH_B16 .L1 0x0
-# CHECKR-NEXT:     0x10 R_LARCH_B26 .L1 0x0
+# CHECKR-NEXT:     0x4 R_LARCH_CALL36 foo 0x0
+# CHECKR-NEXT:     0x4 R_LARCH_RELAX - 0x0
+# CHECKR-NEXT:     0x10 R_LARCH_B21 .L0 0x0
+# CHECKR-NEXT:     0x18 R_LARCH_B16 .L0 0x0
+# CHECKR-NEXT:     0x20 R_LARCH_B26 .L0 0x0
 # CHECKR-NEXT:   }
 # CHECKR-NEXT:   Section ({{.*}}) .rela.data {
 # CHECKR-NEXT:     0x0 R_LARCH_64 .L1 0x0
@@ -22,10 +27,21 @@
 
 .text
   nop
+
+.L0:
+  call36 foo
+
 .L1:
   nop
+## Relocations for branches to .L0 must be reserved and be fixed up by linker
+## when linker relaxation enabled, because of the relaxable call36 instruction.
+## Branches to .L1 can be resolved correctly at compile time, so their
+## relocations can simply be removed.
+  bnez $a0, .L0
   beqz $a0, .L1
+  beq  $a0, $a1, .L0
   blt  $a0, $a1, .L1
+  bl   .L0
   b    .L1
 
 .data
diff --git a/llvm/test/MC/LoongArch/Relocations/relocation-specifier.s b/llvm/test/MC/LoongArch/Relocations/relocation-specifier.s
index d0898aaab92fe..c2526a6ecd701 100644
--- a/llvm/test/MC/LoongArch/Relocations/relocation-specifier.s
+++ b/llvm/test/MC/LoongArch/Relocations/relocation-specifier.s
@@ -6,10 +6,10 @@
 ## This test is similar to test/MC/CSKY/relocation-specifier.s.
 
 # RELOC32: '.rela.data'
-# RELOC32: R_LARCH_32 00000000 .data + 0
+# RELOC32: R_LARCH_32 00000000 local
 
 # RELOC64: '.rela.data'
-# RELOC64: R_LARCH_32 0000000000000000 .data + 0
+# RELOC64: R_LARCH_32 0000000000000000 local
 
 # CHECK: TLS GLOBAL DEFAULT UND gd
 # CHECK: TLS GLOBAL DEFAULT UND ld
diff --git a/llvm/test/MC/LoongArch/Relocations/sub-expr.s b/llvm/test/MC/LoongArch/Relocations/sub-expr.s
index 8bf046acc6975..4554101200818 100644
--- a/llvm/test/MC/LoongArch/Relocations/sub-expr.s
+++ b/llvm/test/MC/LoongArch/Relocations/sub-expr.s
@@ -1,75 +1,57 @@
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=-relax %s \
-# RUN:     | llvm-readobj -r - | FileCheck %s
+# RUN:     | llvm-readobj -r - | FileCheck %s --check-prefixes=CHECK,NORELAX
 # RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax %s \
-# RUN:     | llvm-readobj -r - | FileCheck %s --check-prefix=RELAX
+# RUN:     | llvm-readobj -r - | FileCheck %s --check-prefixes=CHECK,RELAX
 
 ## Check that subtraction expressions emit R_LARCH_32_PCREL and R_LARCH_64_PCREL relocations.
 
 ## TODO: 1- or 2-byte data relocations are not supported for now.
 
-# CHECK:      Relocations [
-# CHECK-NEXT:     Section ({{.*}}) .rela.sx {
-# CHECK-NEXT:       0x4 R_LARCH_PCALA_HI20 z 0x0
-# CHECK-NEXT:       0x8 R_LARCH_PCALA_LO12 z 0x0
-# CHECK-NEXT:       0xC R_LARCH_32_PCREL .sy 0xC
-# CHECK-NEXT:     }
+# CHECK:        Relocations [
+# NORELAX-NEXT:   Section ({{.*}}) .rela.sx {
+# NORELAX-NEXT:     0x4 R_LARCH_PCALA_HI20 z 0x0
+# NORELAX-NEXT:     0x8 R_LARCH_PCALA_LO12 z 0x0
+# NORELAX-NEXT:     0xC R_LARCH_32_PCREL y 0x8
+# NORELAX-NEXT:   }
+# RELAX-NEXT:     Section ({{.*}}) .rela.sx {
+# RELAX-NEXT:       0x4 R_LARCH_PCALA_HI20 z 0x0
+# RELAX-NEXT:       0x4 R_LARCH_RELAX - 0x0
+# RELAX-NEXT:       0x8 R_LARCH_PCALA_LO12 z 0x0
+# RELAX-NEXT:       0x8 R_LARCH_RELAX - 0x0
+# RELAX-NEXT:       0xC R_LARCH_ADD32 y 0x0
+# RELAX-NEXT:       0xC R_LARCH_SUB32 x 0x0
+# RELAX-NEXT:     }
 # CHECK-NEXT:     Section ({{.*}}) .rela.data {
-# CHECK-NEXT:       0x0 R_LARCH_64_PCREL .sx 0x4
-# CHECK-NEXT:       0x8 R_LARCH_64_PCREL .sy 0x4
-# CHECK-NEXT:       0x10 R_LARCH_32_PCREL .sx 0x4
-# CHECK-NEXT:       0x14 R_LARCH_32_PCREL .sy 0x4
-# CHECK-NEXT:       0x18 R_LARCH_ADD64 .sx 0x4
-# CHECK-NEXT:       0x18 R_LARCH_SUB64 .sy 0x4
-# CHECK-NEXT:       0x20 R_LARCH_ADD64 .sy 0x4
-# CHECK-NEXT:       0x20 R_LARCH_SUB64 .sx 0x4
-# CHECK-NEXT:       0x28 R_LARCH_ADD32 .sx 0x4
-# CHECK-NEXT:       0x28 R_LARCH_SUB32 .sy 0x4
-# CHECK-NEXT:       0x2C R_LARCH_ADD32 .sy 0x4
-# CHECK-NEXT:       0x2C R_LARCH_SUB32 .sx 0x4
-# CHECK-NEXT:       0x30 R_LARCH_ADD64 .data 0x30
-# CHECK-NEXT:       0x30 R_LARCH_SUB64 .sx 0x4
-# CHECK-NEXT:       0x38 R_LARCH_ADD32 .data 0x38
-# CHECK-NEXT:       0x38 R_LARCH_SUB32 .sy 0x4
-# CHECK-NEXT:     }
-# CHECK-NEXT:     Section ({{.*}}) .rela.sy {
-# CHECK-NEXT:       0x10 R_LARCH_32_PCREL .sx 0x10
+# CHECK-NEXT:       0x0 R_LARCH_64_PCREL x 0x0
+# CHECK-NEXT:       0x8 R_LARCH_64_PCREL y 0x0
+# CHECK-NEXT:       0x10 R_LARCH_32_PCREL x 0x0
+# CHECK-NEXT:       0x14 R_LARCH_32_PCREL y 0x0
+# CHECK-NEXT:       0x18 R_LARCH_ADD64 x 0x0
+# CHECK-NEXT:       0x18 R_LARCH_SUB64 y 0x0
+# CHECK-NEXT:       0x20 R_LARCH_ADD64 y 0x0
+# CHECK-NEXT:       0x20 R_LARCH_SUB64 x 0x0
+# CHECK-NEXT:       0x28 R_LARCH_ADD32 x 0x0
+# CHECK-NEXT:       0x28 R_LARCH_SUB32 y 0x0
+# CHECK-NEXT:       0x2C R_LARCH_ADD32 y 0x0
+# CHECK-NEXT:       0x2C R_LARCH_SUB32 x 0x0
+# CHECK-NEXT:       0x30 R_LARCH_ADD64 {{.*}} 0x0
+# CHECK-NEXT:       0x30 R_LARCH_SUB64 x 0x0
+# CHECK-NEXT:       0x38 R_LARCH_ADD32 {{.*}} 0x0
+# CHECK-NEXT:       0x38 R_LARCH_SUB32 y 0x0
 # CHECK-NEXT:     }
+# NORELAX-NEXT:   Section ({{.*}}) .rela.sy {
+# NORELAX-NEXT:     0x0 R_LARCH_CALL36 foo 0x0
+# NORELAX-NEXT:     0x10 R_LARCH_32_PCREL x 0x8
+# NORELAX-NEXT:   }
+# RELAX-NEXT:     Section ({{.*}}) .rela.sy {
+# RELAX-NEXT:       0x0 R_LARCH_CALL36 foo 0x0
+# RELAX-NEXT:       0x0 R_LARCH_RELAX - 0x0
+# RELAX-NEXT:       0x8 R_LARCH_ALIGN - 0xC
+# RELAX-NEXT:       0x14 R_LARCH_ADD32 x 0x0
+# RELAX-NEXT:       0x14 R_LARCH_SUB32 y 0x0
+# RELAX-NEXT:     }
 # CHECK-NEXT:   ]
 
-# RELAX:      Relocations [
-# RELAX-NEXT:   Section ({{.*}}) .rela.sx {
-# RELAX-NEXT:     0x4 R_LARCH_PCALA_HI20 z 0x0
-# RELAX-NEXT:     0x4 R_LARCH_RELAX - 0x0
-# RELAX-NEXT:     0x8 R_LARCH_PCALA_LO12 z 0x0
-# RELAX-NEXT:     0x8 R_LARCH_RELAX - 0x0
-# RELAX-NEXT:     0xC R_LARCH_ADD32 y 0x0
-# RELAX-NEXT:     0xC R_LARCH_SUB32 x 0x0
-# RELAX-NEXT:   }
-# RELAX-NEXT:   Section ({{.*}}) .rela.data {
-# RELAX-NEXT:     0x0 R_LARCH_64_PCREL x 0x0
-# RELAX-NEXT:     0x8 R_LARCH_64_PCREL y 0x0
-# RELAX-NEXT:     0x10 R_LARCH_32_PCREL x 0x0
-# RELAX-NEXT:     0x14 R_LARCH_32_PCREL y 0x0
-# RELAX-NEXT:     0x18 R_LARCH_ADD64 x 0x0
-# RELAX-NEXT:     0x18 R_LARCH_SUB64 y 0x0
-# RELAX-NEXT:     0x20 R_LARCH_ADD64 y 0x0
-# RELAX-NEXT:     0x20 R_LARCH_SUB64 x 0x0
-# RELAX-NEXT:     0x28 R_LARCH_ADD32 x 0x0
-# RELAX-NEXT:     0x28 R_LARCH_SUB32 y 0x0
-# RELAX-NEXT:     0x2C R_LARCH_ADD32 y 0x0
-# RELAX-NEXT:     0x2C R_LARCH_SUB32 x 0x0
-# RELAX-NEXT:     0x30 R_LARCH_ADD64 {{.*}} 0x0
-# RELAX-NEXT:     0x30 R_LARCH_SUB64 x 0x0
-# RELAX-NEXT:     0x38 R_LARCH_ADD32 {{.*}} 0x0
-# RELAX-NEXT:     0x38 R_LARCH_SUB32 y 0x0
-# RELAX-NEXT:   }
-# RELAX-NEXT:   Section ({{.*}}) .rela.sy {
-# RELAX-NEXT:     0x4 R_LARCH_ALIGN - 0xC
-# RELAX-NEXT:     0x10 R_LARCH_ADD32 x 0x0
-# RELAX-NEXT:     0x10 R_LARCH_SUB32 y 0x0
-# RELAX-NEXT:   }
-# RELAX-NEXT: ]
-
 .section .sx,"ax"
 nop
 x:
@@ -89,7 +71,7 @@ la.pcrel $a0, z
 .4byte .-y
 
 .section .sy,"ax"
-nop
+call36 foo
 y:
 .p2align 4
 .4byte x-y
diff --git a/llvm/test/MC/M68k/Atomics/cas.s b/llvm/test/MC/M68k/Atomics/cas.s
index c6cae72c03f09..32b210aacb9c5 100644
--- a/llvm/test/MC/M68k/Atomics/cas.s
+++ b/llvm/test/MC/M68k/Atomics/cas.s
@@ -15,39 +15,39 @@ cas.l %d6, %d7, (%a4)
 
 ; Address Register Indirect with Displacement
 ; CHECK: cas.b %d3, %d2, (5,%a2)
-; CHECK-SAME: ; encoding: [0x0a,0xea,0x00,0x83]
+; CHECK-SAME: ; encoding: [0x0a,0xea,0x00,0x83,0x00,0x05]
 cas.b %d3, %d2, (5, %a2)
 
 ; CHECK: cas.w %d4, %d5, (6,%a3)
-; CHECK-SAME: ; encoding: [0x0c,0xeb,0x01,0x44]
+; CHECK-SAME: ; encoding: [0x0c,0xeb,0x01,0x44,0x00,0x06]
 cas.w %d4, %d5, (6, %a3)
 
 ; CHECK: cas.l %d6, %d7, (7,%a4)
-; CHECK-SAME: ; encoding: [0x0e,0xec,0x01,0xc6]
+; CHECK-SAME: ; encoding: [0x0e,0xec,0x01,0xc6,0x00,0x07]
 cas.l %d6, %d7, (7, %a4)
 
 ; Address Register Indirect with Index (Scale = 1)
 ; CHECK: cas.b %d3, %d2, (5,%a2,%d1)
-; CHECK-SAME: ; encoding: [0x0a,0xf2,0x00,0x83]
+; CHECK-SAME: ; encoding: [0x0a,0xf2,0x00,0x83,0x18,0x05]
 cas.b %d3, %d2, (5, %a2, %d1)
 
 ; CHECK: cas.w %d4, %d5, (6,%a3,%d1)
-; CHECK-SAME: ; encoding: [0x0c,0xf3,0x01,0x44]
+; CHECK-SAME: ; encoding: [0x0c,0xf3,0x01,0x44,0x18,0x06]
 cas.w %d4, %d5, (6, %a3, %d1)
 
 ; CHECK: cas.l %d6, %d7, (7,%a4,%d1)
-; CHECK-SAME: ; encoding: [0x0e,0xf4,0x01,0xc6]
+; CHECK-SAME: ; encoding: [0x0e,0xf4,0x01,0xc6,0x18,0x07]
 cas.l %d6, %d7, (7, %a4, %d1)
 
 ; Absolute Long Address
 ; CHECK: cas.b %d3, %d2, $ffffffffffffffff
-; CHECK-SAME: ; encoding: [0x0a,0xf8,0x00,0x83]
+; CHECK-SAME: ; encoding: [0x0a,0xf9,0x00,0x83,0xff,0xff,0xff,0xff]
 cas.b %d3, %d2, $ffffffffffffffff
 
 ; CHECK: cas.w %d4, %d5, $ffffffffffffffff
-; CHECK-SAME: ; encoding: [0x0c,0xf8,0x01,0x44]
+; CHECK-SAME: ; encoding: [0x0c,0xf9,0x01,0x44,0xff,0xff,0xff,0xff]
 cas.w %d4, %d5, $ffffffffffffffff
 
 ; CHECK: cas.l %d6, %d7, $ffffffffffffffff
-; CHECK-SAME: ; encoding: [0x0e,0xf8,0x01,0xc6]
+; CHECK-SAME: ; encoding: [0x0e,0xf9,0x01,0xc6,0xff,0xff,0xff,0xff]
 cas.l %d6, %d7, $ffffffffffffffff
diff --git a/llvm/test/MC/RISCV/Relocations/align-after-relax.s b/llvm/test/MC/RISCV/Relocations/align-after-relax.s
new file mode 100644
index 0000000000000..95bef513dff97
--- /dev/null
+++ b/llvm/test/MC/RISCV/Relocations/align-after-relax.s
@@ -0,0 +1,50 @@
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c,+relax %s --defsym LATE=1 -o %t1
+# RUN: llvm-objdump -dr --no-show-raw-insn -M no-aliases %t1 | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+c,+relax %s -o %t0
+# RUN: llvm-objdump -dr --no-show-raw-insn -M no-aliases %t0 | FileCheck %s --check-prefix=CHECK0
+
+# CHECK:            4: 00 00 01 00 .word 0x00010000
+# CHECK-EMPTY:
+# CHECK:            8: 78 56 34 12 .word 0x12345678
+# CHECK-NEXT:       c: 00 00 00 00 .word 0x00000000
+# CHECK:           10: auipc   ra, 0x0
+# CHECK-NEXT:                R_RISCV_CALL_PLT     foo
+# CHECK-NEXT:                R_RISCV_RELAX        *ABS*
+# CHECK:           18: c.nop
+# CHECK-NEXT:                R_RISCV_ALIGN        *ABS*+0x6
+
+## Alignment directives in a lower-numbered subsection may be conservatively treated as linker-relaxable.
+# CHECK0:           4: 00 00 01 00 .word 0x00010000
+# CHECK0-NEXT:               000000006: R_RISCV_ALIGN        *ABS*+0x6
+# CHECK0-NEXT:      8: 13 00 00 00 .word 0x00000013
+# CHECK0:          14: auipc   ra, 0x0
+# CHECK0:          1c: c.nop
+# CHECK0-NEXT:               R_RISCV_ALIGN        *ABS*+0x6
+
+.text 2
+.option push
+.option norelax
+## R_RISCV_ALIGN is required even if norelax, because it is after a linker-relaxable instruction.
+.balign 8
+l2:
+  .word 0x12345678
+.option pop
+
+.text 1
+  .org .+1
+  .org .+3
+.ifdef LATE
+  .org .+0
+.endif
+  call foo
+
+.text 0
+_start:
+  .space 6
+.option push
+.option norelax
+.balign 8
+l0:
+  .word 0x12345678
+.option pop
diff --git a/llvm/test/MC/RISCV/Relocations/align-norvc.s b/llvm/test/MC/RISCV/Relocations/align-norvc.s
new file mode 100644
index 0000000000000..c3fe71e65a205
--- /dev/null
+++ b/llvm/test/MC/RISCV/Relocations/align-norvc.s
@@ -0,0 +1,23 @@
+## To ensure ALIGN relocations in norvc code can adapt to shrinking of preceding rvc code,
+## we generate $alignment-2 bytes of NOPs regardless of rvc.
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+relax %s -o %t
+# RUN: llvm-objdump -dr -M no-aliases %t | FileCheck %s
+
+# RUN: llvm-mc -filetype=obj -triple=riscv64 -mattr=+relax -riscv-align-rvc=0 %s -o %t0
+# RUN: llvm-objdump -dr -M no-aliases %t0 | FileCheck %s --check-prefix=CHECK0
+
+# CHECK:               00000000: R_RISCV_RELAX        *ABS*
+# CHECK-NEXT:       4: 0001      <unknown>
+# CHECK-NEXT:          00000004: R_RISCV_ALIGN        *ABS*+0x6
+# CHECK-NEXT:       6: 00000013  addi zero, zero, 0x0
+# CHECK-NEXT:       a: 00000537  lui a0, 0x0
+
+# CHECK0:              00000000: R_RISCV_RELAX        *ABS*
+# CHECK0-NEXT:      4: 00000013  addi zero, zero, 0x0
+# CHECK0-NEXT:         00000004: R_RISCV_ALIGN        *ABS*+0x4
+# CHECK0-NEXT:      8: 00000537  lui a0, 0x0
+
+  lui a0, %hi(foo)
+  .option norvc
+.balign 8
+  lui a0, %hi(foo)
diff --git a/llvm/test/MC/RISCV/Relocations/bigendian-fixups.s b/llvm/test/MC/RISCV/Relocations/bigendian-fixups.s
new file mode 100644
index 0000000000000..b79a282de932b
--- /dev/null
+++ b/llvm/test/MC/RISCV/Relocations/bigendian-fixups.s
@@ -0,0 +1,101 @@
+# RUN: llvm-mc --triple=riscv32be %s --show-encoding \
+# RUN:     | FileCheck --check-prefixes=CHECK-FIXUP,CHECK-ENCODING %s
+# RUN: llvm-mc --filetype=obj --triple=riscv32be %s \
+# RUN:     | llvm-objdump -d - | FileCheck --check-prefix=CHECK-INSTR %s
+# RUN: llvm-mc --filetype=obj --triple=riscv32be %s \
+# RUN:     | llvm-readobj -r - | FileCheck --check-prefix=CHECK-REL %s
+
+# RUN: llvm-mc --triple=riscv64be %s --show-encoding \
+# RUN:     | FileCheck --check-prefixes=CHECK-FIXUP,CHECK-ENCODING %s
+# RUN: llvm-mc --filetype=obj --triple=riscv64be %s \
+# RUN:     | llvm-objdump -d - | FileCheck --check-prefix=CHECK-INSTR %s
+# RUN: llvm-mc --filetype=obj --triple=riscv64be %s \
+# RUN:     | llvm-readobj -r - | FileCheck --check-prefix=CHECK-REL %s
+
+## Checks that fixups that can be resolved within the same object file are
+## applied correctly on big-endian RISC-V targets.
+##
+## This test verifies that RISC-V instructions remain little-endian even on
+## big-endian systems. This is a fundamental property of RISC-V:
+## - Instructions are always little-endian
+## - Data can be big-endian or little-endian depending on the system
+
+.LBB0:
+addi t0, t0, 1
+# CHECK-ENCODING: encoding: [0x93,0x82,0x12,0x00]
+# CHECK-INSTR: addi t0, t0, 0x1
+
+lui t1, %hi(val)
+# CHECK-ENCODING: encoding: [0x37,0bAAAA0011,A,A]
+# CHECK-FIXUP: fixup A - offset: 0, value: %hi(val), kind: fixup_riscv_hi20
+# CHECK-INSTR: lui t1, 0x12345
+
+lw a0, %lo(val)(t1)
+# CHECK-ENCODING: encoding: [0x03,0x25,0bAAAA0011,A]
+# CHECK-FIXUP: fixup A - offset: 0, value: %lo(val), kind: fixup_riscv_lo12_i
+# CHECK-INSTR: lw a0, 0x678(t1)
+
+addi a1, t1, %lo(val)
+# CHECK-ENCODING: encoding: [0x93,0x05,0bAAAA0011,A]
+# CHECK-FIXUP: fixup A - offset: 0, value: %lo(val), kind: fixup_riscv_lo12_i
+# CHECK-INSTR: addi a1, t1, 0x678
+
+sw a0, %lo(val)(t1)
+# CHECK-ENCODING: encoding: [0x23'A',0x20'A',0xa3'A',A]
+# CHECK-FIXUP: fixup A - offset: 0, value: %lo(val), kind: fixup_riscv_lo12_s
+# CHECK-INSTR: sw a0, 0x678(t1)
+
+1:
+auipc t1, %pcrel_hi(.LBB0)
+# CHECK-ENCODING: encoding: [0x17,0bAAAA0011,A,A]
+# CHECK-FIXUP: fixup A - offset: 0, value: %pcrel_hi(.LBB0), kind: fixup_riscv_pcrel_hi20
+# CHECK-INSTR: auipc t1, 0
+
+addi t1, t1, %pcrel_lo(1b)
+# CHECK-ENCODING: encoding: [0x13,0x03,0bAAAA0011,A]
+# CHECK-FIXUP: fixup A - offset: 0, value: %pcrel_lo({{.*}}), kind: fixup_riscv_pcrel_lo12_i
+# CHECK-INSTR: addi t1, t1, -0x14
+
+sw t1, %pcrel_lo(1b)(t1)
+# CHECK-ENCODING: encoding: [0x23'A',0x20'A',0x63'A',A]
+# CHECK-FIXUP: fixup A - offset: 0, value: %pcrel_lo({{.*}}), kind: fixup_riscv_pcrel_lo12_s
+# CHECK-INSTR: sw t1, -0x14(t1)
+
+jal zero, .LBB0
+# CHECK-ENCODING: encoding: [0x6f,0bAAAA0000,A,A]
+# CHECK-FIXUP: fixup A - offset: 0, value: .LBB0, kind: fixup_riscv_jal
+# CHECK-INSTR: j 0x0 <.text>
+
+jal zero, .LBB2
+# CHECK-ENCODING: encoding: [0x6f,0bAAAA0000,A,A]
+# CHECK-FIXUP: fixup A - offset: 0, value: .LBB2, kind: fixup_riscv_jal
+# CHECK-INSTR: j 0x50d18 <.text+0x50d18>
+
+beq a0, a1, .LBB0
+# CHECK-ENCODING: encoding: [0x63'A',A,0xb5'A',A]
+# CHECK-FIXUP: fixup A - offset: 0, value: .LBB0, kind: fixup_riscv_branch
+# CHECK-INSTR: beq a0, a1, 0x0 <.text>
+
+blt a0, a1, .LBB1
+# CHECK-ENCODING: encoding: [0x63'A',0x40'A',0xb5'A',A]
+# CHECK-FIXUP: fixup A - offset: 0, value: .LBB1, kind: fixup_riscv_branch
+# CHECK-INSTR: blt a0, a1, 0x480 <.text+0x480>
+
+.fill 1104
+
+.LBB1:
+
+.fill 329876
+addi zero, zero, 0
+.LBB2:
+
+.set val, 0x12345678
+
+# CHECK-REL-NOT: R_RISCV
+
+.data
+.align 3
+data_label:
+  .word val  # On BE: 0x12345678 stored as [0x12, 0x34, 0x56, 0x78]
+  .long val  # On BE: 0x12345678 stored as [0x12, 0x34, 0x56, 0x78]
+  .quad val  # On BE: 0x0000000012345678 stored as [0x00, 0x00, 0x00, 0x00, 0x12, 0x34, 0x56, 0x78]
diff --git a/llvm/test/MC/RISCV/Relocations/mc-dump.s b/llvm/test/MC/RISCV/Relocations/mc-dump.s
index ddc0c7d2fe6b1..bd8acf9dd41e0 100644
--- a/llvm/test/MC/RISCV/Relocations/mc-dump.s
+++ b/llvm/test/MC/RISCV/Relocations/mc-dump.s
@@ -2,19 +2,19 @@
 # RUN: llvm-mc -filetype=obj --triple=riscv64 --mattr=+relax %s -debug-only=mc-dump -o /dev/null 2>&1 | FileCheck %s
 
 #      CHECK:Sections:[
-# CHECK-NEXT:MCSection Name:.text
+# CHECK-NEXT:MCSection Name:.text FirstLinkerRelaxable:1
 # CHECK-NEXT:0 Align Size:0+0 []
 # CHECK-NEXT:  Align:4 Fill:0 FillLen:1 MaxBytesToEmit:4 Nops
 # CHECK-NEXT:  Symbol @0 .text
 # CHECK-NEXT:0 Data LinkerRelaxable Size:8 [97,00,00,00,e7,80,00,00]
-# CHECK-NEXT:  Fixup @0 Value:specifier(19,ext) Kind:4023
+# CHECK-NEXT:  Fixup @0 Value:specifier(19,ext) Kind:4023 LinkerRelaxable
 # CHECK-NEXT:  Symbol @0 $x
-# CHECK-NEXT:8 Align LinkerRelaxable Size:0+4 []
+# CHECK-NEXT:8 Align LinkerRelaxable Size:0+6 []
 # CHECK-NEXT:  Align:8 Fill:0 FillLen:1 MaxBytesToEmit:8 Nops
-# CHECK-NEXT:  Fixup @0 Value:4 Kind:[[#]]
-# CHECK-NEXT:12 Align LinkerRelaxable Size:4+4 [13,05,30,00]
+# CHECK-NEXT:  Fixup @0 Value:6 Kind:[[#]]
+# CHECK-NEXT:14 Align LinkerRelaxable Size:4+6 [13,05,30,00]
 # CHECK-NEXT:  Align:8 Fill:0 FillLen:1 MaxBytesToEmit:8 Nops
-# CHECK-NEXT:  Fixup @4 Value:4 Kind:[[#]]
+# CHECK-NEXT:  Fixup @4 Value:6 Kind:[[#]]
 # CHECK-NEXT:]
 
 call ext
diff --git a/llvm/test/MC/RISCV/align-option-relax.s b/llvm/test/MC/RISCV/align-option-relax.s
index 890e1e72d7706..60cd55f5a8b4d 100644
--- a/llvm/test/MC/RISCV/align-option-relax.s
+++ b/llvm/test/MC/RISCV/align-option-relax.s
@@ -1,8 +1,21 @@
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=-relax < %s \
 # RUN:     | llvm-readobj -r - | FileCheck %s
 
-# Check that .option relax overrides -mno-relax and enables R_RISCV_ALIGN
-# relocations.
-# CHECK: R_RISCV_ALIGN
-	.option relax
-	.align 4
+## .option relax overrides -mno-relax and enables R_RISCV_ALIGN/R_RISCV_RELAX relocations.
+# CHECK:      .rela.text
+# CHECK:       R_RISCV_CALL_PLT
+# CHECK-NEXT:  R_RISCV_RELAX
+# CHECK-NEXT:  R_RISCV_ALIGN
+.option relax
+call foo
+.align 4
+
+## Alignments before the first linker-relaxable instruction do not need relocations.
+# CHECK-NOT: .rela.text1
+.section .text1,"ax"
+.align 4
+nop
+
+# CHECK: .rela.text2
+.section .text2,"ax"
+call foo
diff --git a/llvm/test/MC/RISCV/align.s b/llvm/test/MC/RISCV/align.s
index da3b1aa9b637a..4d4d998c992fd 100644
--- a/llvm/test/MC/RISCV/align.s
+++ b/llvm/test/MC/RISCV/align.s
@@ -46,20 +46,21 @@
 # type for .align N directive when linker relaxation enabled.
 # Linker could satisfy alignment by removing NOPs after linker relaxation.
 
-# The first R_RISCV_ALIGN come from
-# MCELFStreamer::InitSections() emitCodeAlignment(getTextSectionAligntment()).
-# C-OR-ZCA-EXT-RELAX-RELOC: R_RISCV_ALIGN - 0x2
-# C-OR-ZCA-EXT-RELAX-INST:  c.nop
 test:
+## Start with a linker-relaxable instruction so that the following alignment can be relaxable.
+	call foo
+# NORELAX-RELOC:              R_RISCV_CALL_PLT
+# C-OR-ZCA-EXT-NORELAX-RELOC: R_RISCV_CALL_PLT
+
 	.p2align 2
 # If the +c extension is enabled, the text section will be 2-byte aligned, so
 # one c.nop instruction is sufficient.
-# C-OR-ZCA-EXT-RELAX-RELOC-NOT: R_RISCV_ALIGN - 0x2
-# C-OR-ZCA-EXT-RELAX-INST-NOT:  c.nop
+# C-OR-ZCA-EXT-RELAX-RELOC: R_RISCV_ALIGN - 0x2
+# C-OR-ZCA-EXT-RELAX-INST:  c.nop
 	bne     zero, a0, .LBB0_2
 	mv	a0, zero
 	.p2align 3
-# RELAX-RELOC: R_RISCV_ALIGN - 0x4
+# RELAX-RELOC: R_RISCV_ALIGN - 0x6
 # RELAX-INST:  addi    zero, zero, 0
 # C-OR-ZCA-EXT-RELAX-RELOC: R_RISCV_ALIGN - 0x6
 # C-OR-ZCA-EXT-RELAX-INST:  c.nop
@@ -68,7 +69,7 @@ test:
 	add	a0, a0, a1
 	.align 4
 .LBB0_2:
-# RELAX-RELOC: R_RISCV_ALIGN - 0xC
+# RELAX-RELOC: R_RISCV_ALIGN - 0xE
 # RELAX-INST:  addi    zero, zero, 0
 # RELAX-INST:  addi    zero, zero, 0
 # RELAX-INST:  addi    zero, zero, 0
@@ -84,7 +85,7 @@ test:
 	.p2align 3
 .constant_pool:
 .long	3126770193
-# RELAX-RELOC: R_RISCV_ALIGN - 0x4
+# RELAX-RELOC: R_RISCV_ALIGN - 0x6
 # RELAX-INST:  addi    zero, zero, 0
 # NORELAX-INST: addi    zero, zero, 0
 # C-OR-ZCA-EXT-RELAX-RELOC: R_RISCV_ALIGN - 0x6
@@ -136,16 +137,8 @@ data2:
 	add	a0, a0, a1
 
 ## Branches crossing the linker-relaxable R_RISCV_ALIGN need relocations.
-# RELAX-RELOC:      .rela.text3 {
-# RELAX-RELOC-NEXT:    0x4 R_RISCV_BRANCH .Ltmp[[#]] 0x0
-# RELAX-RELOC-NEXT:    0x8 R_RISCV_ALIGN - 0x4
-# RELAX-RELOC-NEXT:    0xC R_RISCV_BRANCH .Ltmp[[#]] 0x0
-# RELAX-RELOC-NEXT: }
-# C-OR-ZCA-EXT-RELAX-RELOC:      .rela.text3 {
-# C-OR-ZCA-EXT-RELAX-RELOC-NEXT:    0x4 R_RISCV_BRANCH .Ltmp[[#]] 0x0
-# C-OR-ZCA-EXT-RELAX-RELOC-NEXT:    0x8 R_RISCV_ALIGN - 0x4
-# C-OR-ZCA-EXT-RELAX-RELOC-NEXT:    0xC R_RISCV_BRANCH .Ltmp[[#]] 0x0
-# C-OR-ZCA-EXT-RELAX-RELOC-NEXT: }
+# RELAX-RELOC-NOT:  .rela.text3 {
+# C-OR-ZCA-EXT-RELAX-RELOC-NOT:  .rela.text3 {
 	.section .text3, "ax"
 	bnez t1, 1f
 	bnez t2, 2f
@@ -157,14 +150,15 @@ data2:
 
 ## .text3 with a call at the start
 # NORELAX-RELOC:    .rela.text3a
-# C-OR-ZCA-EXT-NORELAX-RELOC: .rela.text3a
 # RELAX-RELOC:      .rela.text3a {
 # RELAX-RELOC-NEXT:    0x0  R_RISCV_CALL_PLT foo 0x0
 # RELAX-RELOC-NEXT:    0x0  R_RISCV_RELAX - 0x0
 # RELAX-RELOC-NEXT:    0xC  R_RISCV_BRANCH .Ltmp[[#]] 0x0
-# RELAX-RELOC-NEXT:    0x10 R_RISCV_ALIGN - 0x4
-# RELAX-RELOC-NEXT:    0x14 R_RISCV_BRANCH .Ltmp[[#]] 0x0
+# RELAX-RELOC-NEXT:    0x10 R_RISCV_ALIGN - 0x6
+# RELAX-RELOC-NEXT:    0x16 R_RISCV_BRANCH .Ltmp[[#]] 0x0
 # RELAX-RELOC-NEXT: }
+# C-OR-ZCA-EXT-NORELAX-RELOC: .rela.text3a
+# C-OR-ZCA-EXT-RELAX-RELOC: .rela.text3a
 .section .text3a, "ax"
 call foo
 bnez t1, 1f
@@ -177,11 +171,8 @@ bnez t1, 2b
 
 ## .text3 with a call at the end
 # RELAX-RELOC:      .rela.text3b {
-# RELAX-RELOC-NEXT:    0x4  R_RISCV_BRANCH .Ltmp[[#]] 0x0
-# RELAX-RELOC-NEXT:    0x8  R_RISCV_ALIGN - 0x4
-# RELAX-RELOC-NEXT:    0xC  R_RISCV_BRANCH .Ltmp[[#]] 0x0
-# RELAX-RELOC-NEXT:    0x14 R_RISCV_CALL_PLT foo 0x0
-# RELAX-RELOC-NEXT:    0x14 R_RISCV_RELAX - 0x0
+# RELAX-RELOC-NEXT:    0x10 R_RISCV_CALL_PLT foo 0x0
+# RELAX-RELOC-NEXT:    0x10 R_RISCV_RELAX - 0x0
 # RELAX-RELOC-NEXT: }
 .section .text3b, "ax"
 bnez t1, 1f
diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s
index 19cc4d5ba1675..03a8957b5b5ca 100644
--- a/llvm/test/MC/RISCV/attribute-arch.s
+++ b/llvm/test/MC/RISCV/attribute-arch.s
@@ -492,8 +492,8 @@
 .attribute arch, "rv32i_sdtrig1p0"
 # CHECK: attribute      5, "rv32i2p1_sdtrig1p0"
 
-.attribute arch, "rv32i_p0p14"
-# CHECK: attribute      5, "rv32i2p1_p0p14"
+.attribute arch, "rv32i_p0p15"
+# CHECK: attribute      5, "rv32i2p1_p0p15"
 
-.attribute arch, "rv64i_p0p14"
-# CHECK: attribute      5, "rv64i2p1_p0p14"
+.attribute arch, "rv64i_p0p15"
+# CHECK: attribute      5, "rv64i2p1_p0p15"
diff --git a/llvm/test/MC/RISCV/bigendian-data-directives.s b/llvm/test/MC/RISCV/bigendian-data-directives.s
new file mode 100644
index 0000000000000..54dd9211e2b64
--- /dev/null
+++ b/llvm/test/MC/RISCV/bigendian-data-directives.s
@@ -0,0 +1,36 @@
+# RUN: llvm-mc -filetype=obj -triple=riscv32be %s -o %t.32be.o
+# RUN: llvm-objdump -s %t.32be.o | FileCheck -check-prefix=RV32BE %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64be %s -o %t.64be.o
+# RUN: llvm-objdump -s %t.64be.o | FileCheck -check-prefix=RV64BE %s
+# RUN: llvm-mc -filetype=obj -triple=riscv32 %s -o %t.32le.o
+# RUN: llvm-objdump -s %t.32le.o | FileCheck -check-prefix=RV32LE %s
+# RUN: llvm-mc -filetype=obj -triple=riscv64 %s -o %t.64le.o
+# RUN: llvm-objdump -s %t.64le.o | FileCheck -check-prefix=RV64LE %s
+
+# Test that data directives are properly byte-swapped on big-endian RISC-V
+
+.data
+
+byte_data:
+  .byte 0x11
+  .byte 0x22
+  .half 0x3344
+  .word 0x55667788
+  .long 0x99aabbcc
+  .quad 0x1122334455667788
+
+# RV32BE: Contents of section .data:
+# RV32BE-NEXT:  0000 11223344 55667788 99aabbcc 11223344
+# RV32BE-NEXT:  0010 55667788
+
+# RV64BE: Contents of section .data:
+# RV64BE-NEXT:  0000 11223344 55667788 99aabbcc 11223344
+# RV64BE-NEXT:  0010 55667788
+
+# RV32LE: Contents of section .data:
+# RV32LE-NEXT:  0000 11224433 88776655 ccbbaa99 88776655
+# RV32LE-NEXT:  0010 44332211
+
+# RV64LE: Contents of section .data:
+# RV64LE-NEXT:  0000 11224433 88776655 ccbbaa99 88776655
+# RV64LE-NEXT:  0010 44332211
diff --git a/llvm/test/MC/RISCV/cfi-advance.s b/llvm/test/MC/RISCV/cfi-advance.s
index f3f8530c419f0..7643e019c30af 100644
--- a/llvm/test/MC/RISCV/cfi-advance.s
+++ b/llvm/test/MC/RISCV/cfi-advance.s
@@ -7,7 +7,7 @@
 
 # NORELAX:      Relocation section '.rela.text1' at offset {{.*}} contains 1 entries:
 # NORELAX-NEXT:  Offset     Info    Type                Sym. Value  Symbol's Name + Addend
-# NORELAX-NEXT: 00000000  00000313 R_RISCV_CALL_PLT       00000004   .L0 + 0
+# NORELAX-NEXT: 00000000  00000313 R_RISCV_CALL_PLT       00000008   .L0 + 0
 # NORELAX-EMPTY:
 # RELAX:        Relocation section '.rela.text1' at offset {{.*}} contains 2 entries:
 # RELAX:        R_RISCV_CALL_PLT
@@ -16,23 +16,25 @@
 # NORELAX-NEXT: Relocation section '.rela.eh_frame' at offset {{.*}} contains 1 entries:
 # NORELAX:       Offset     Info    Type                Sym. Value  Symbol's Name + Addend
 # NORELAX-NEXT: 0000001c  00000139 R_RISCV_32_PCREL       00000000   .L0 + 0
-# RELAX-NEXT:   Relocation section '.rela.eh_frame' at offset {{.*}} contains 5 entries:
+# RELAX-NEXT:   Relocation section '.rela.eh_frame' at offset {{.*}} contains 7 entries:
 # RELAX:         Offset     Info    Type                Sym. Value  Symbol's Name + Addend
 # RELAX-NEXT:   0000001c  00000139 R_RISCV_32_PCREL       00000000   .L0  + 0
-# RELAX-NEXT:   00000020  00000c23 R_RISCV_ADD32          0001017a   .L0  + 0
+# RELAX-NEXT:   00000020  00000d23 R_RISCV_ADD32          0001017c   .L0  + 0
 # RELAX-NEXT:   00000020  00000127 R_RISCV_SUB32          00000000   .L0  + 0
-# RELAX-NEXT:   00000035  00000b35 R_RISCV_SET6           00010176   .L0  + 0
-# RELAX-NEXT:   00000035  00000934 R_RISCV_SUB6           0001016e   .L0  + 0
+# RELAX-NEXT:   00000026  00000536 R_RISCV_SET8           00000068   .L0  + 0
+# RELAX-NEXT:   00000026  00000125 R_RISCV_SUB8           00000000   .L0  + 0
+# RELAX-NEXT:   00000035  00000c35 R_RISCV_SET6           00010178   .L0  + 0
+# RELAX-NEXT:   00000035  00000a34 R_RISCV_SUB6           0001016e   .L0  + 0
 # CHECK-EMPTY:
-# NORELAX:      Symbol table '.symtab' contains 13 entries:
-# RELAX:        Symbol table '.symtab' contains 16 entries:
+# NORELAX:      Symbol table '.symtab' contains 14 entries:
+# RELAX:        Symbol table '.symtab' contains 18 entries:
 # RELAX-NEXT:      Num:    Value  Size Type    Bind   Vis       Ndx Name
 # RELAX-NEXT:        0: 00000000     0 NOTYPE  LOCAL  DEFAULT   UND
 # RELAX-NEXT:        1: 00000000     0 NOTYPE  LOCAL  DEFAULT     2 .L0 {{$}}
-# RELAX:             3: 00000004     0 NOTYPE  LOCAL  DEFAULT     2 .L0{{$}}
-# RELAX:             9: 0001016e     0 NOTYPE  LOCAL  DEFAULT     2 .L0 {{$}}
-# RELAX:            11: 00010176     0 NOTYPE  LOCAL  DEFAULT     2 .L0 {{$}}
-# RELAX:            12: 0001017a     0 NOTYPE  LOCAL  DEFAULT     2 .L0 {{$}}
+# RELAX:             3: 00000008     0 NOTYPE  LOCAL  DEFAULT     2 .L0{{$}}
+# RELAX:            10: 0001016e     0 NOTYPE  LOCAL  DEFAULT     2 .L0 {{$}}
+# RELAX:            12: 00010178     0 NOTYPE  LOCAL  DEFAULT     2 .L0 {{$}}
+# RELAX:            13: 0001017c     0 NOTYPE  LOCAL  DEFAULT     2 .L0 {{$}}
 
 # CHECK-DWARFDUMP: DW_CFA_advance_loc1: 104
 # CHECK-DWARFDUMP-NEXT: DW_CFA_def_cfa_offset: +8
@@ -48,11 +50,11 @@
         .type   test,@function
 test:
         .cfi_startproc
-        nop
+        call foo
 ## This looks similar to fake label names ".L0 ". Even if this is ".L0 ",
 ## the assembler will not conflate it with fake labels.
 .L0:
-        .zero 100, 0x90
+        .zero 96, 0x90
         .cfi_def_cfa_offset 8
         nop
         .zero 255, 0x90
diff --git a/llvm/test/MC/RISCV/elf-bigendian.s b/llvm/test/MC/RISCV/elf-bigendian.s
new file mode 100644
index 0000000000000..da12a14aea142
--- /dev/null
+++ b/llvm/test/MC/RISCV/elf-bigendian.s
@@ -0,0 +1,46 @@
+# RUN: llvm-mc %s -filetype=obj -triple=riscv32be | llvm-readobj -h - \
+# RUN:     | FileCheck -check-prefix=RV32BE %s
+# RUN: llvm-mc %s -filetype=obj -triple=riscv64be | llvm-readobj -h - \
+# RUN:     | FileCheck -check-prefix=RV64BE %s
+
+# Test that RISC-V big-endian targets produce correct ELF headers
+
+# RV32BE: Format: elf32-bigriscv
+# RV32BE: Arch: riscv32
+# RV32BE: AddressSize: 32bit
+# RV32BE: ElfHeader {
+# RV32BE:   Ident {
+# RV32BE:     Magic: (7F 45 4C 46)
+# RV32BE:     Class: 32-bit (0x1)
+# RV32BE:     DataEncoding: BigEndian (0x2)
+# RV32BE:     FileVersion: 1
+# RV32BE:     OS/ABI: SystemV (0x0)
+# RV32BE:     ABIVersion: 0
+# RV32BE:   }
+# RV32BE:   Type: Relocatable (0x1)
+# RV32BE:   Machine: EM_RISCV (0xF3)
+# RV32BE:   Version: 1
+# RV32BE:   Flags [ (0x0)
+# RV32BE:   ]
+# RV32BE: }
+
+# RV64BE: Format: elf64-bigriscv
+# RV64BE: Arch: riscv64
+# RV64BE: AddressSize: 64bit
+# RV64BE: ElfHeader {
+# RV64BE:   Ident {
+# RV64BE:     Magic: (7F 45 4C 46)
+# RV64BE:     Class: 64-bit (0x2)
+# RV64BE:     DataEncoding: BigEndian (0x2)
+# RV64BE:     FileVersion: 1
+# RV64BE:     OS/ABI: SystemV (0x0)
+# RV64BE:     ABIVersion: 0
+# RV64BE:   }
+# RV64BE:   Type: Relocatable (0x1)
+# RV64BE:   Machine: EM_RISCV (0xF3)
+# RV64BE:   Version: 1
+# RV64BE:   Flags [ (0x0)
+# RV64BE:   ]
+# RV64BE: }
+
+nop
diff --git a/llvm/test/MC/RISCV/large-instructions.s b/llvm/test/MC/RISCV/large-instructions.s
deleted file mode 100644
index b50dbde17d380..0000000000000
--- a/llvm/test/MC/RISCV/large-instructions.s
+++ /dev/null
@@ -1,29 +0,0 @@
-# RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:     | llvm-objdump -d - | FileCheck %s
-
-# CHECK: 011f 4523 8967 <unknown>
-.byte 0x1f, 0x01, 0x23, 0x45, 0x67, 0x89
-
-# CHECK: 4523013f cdab8967 <unknown>
-.byte 0x3f, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd
-
-# CHECK: 007f 4523 8967 cdab feef <unknown>
-.byte 0x7f, 0x00, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe
-
-# CHECK: 4523107f cdab8967 badcfeef <unknown>
-.byte 0x7f, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba
-
-# CHECK: 207f 4523 8967 cdab feef badc 7698 <unknown>
-.byte 0x7f, 0x20, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76
-
-# CHECK: 4523307f cdab8967 badcfeef 32547698 <unknown>
-.byte 0x7f, 0x30, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32
-
-# CHECK: 407f 4523 8967 cdab feef badc 7698 3254 1210 <unknown>
-.byte 0x7f, 0x40, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, 0x12
-
-# CHECK: 4523507f cdab8967 badcfeef 32547698 56341210 <unknown>
-.byte 0x7f, 0x50, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, 0x12, 0x34, 0x56
-
-# CHECK: 607f 4523 8967 cdab feef badc 7698 3254 1210 5634 9a78 <unknown>
-.byte 0x7f, 0x60, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, 0x12, 0x34, 0x56, 0x78, 0x9a
diff --git a/llvm/test/MC/RISCV/large-instructions.test b/llvm/test/MC/RISCV/large-instructions.test
new file mode 100644
index 0000000000000..b8396a9379e8b
--- /dev/null
+++ b/llvm/test/MC/RISCV/large-instructions.test
@@ -0,0 +1,60 @@
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objdump -d %t | FileCheck %s
+
+## This CHECKs objdump's handling of wide instruction encodings, and how it
+## groups the instruction bytes when disassembling.
+##
+## This is written in YAML because using `.byte` emits the wrong mapping
+## symbols.
+
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_RISCV
+  SectionHeaderStringTable: .strtab
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    0x1
+    ContentArray:    [
+      # CHECK: 011f 4523 8967 <unknown>
+      0x1f, 0x01, 0x23, 0x45, 0x67, 0x89,
+
+      # CHECK: 4523013f cdab8967 <unknown>
+      0x3f, 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd,
+
+      # CHECK: 007f 4523 8967 cdab feef <unknown>
+      0x7f, 0x00, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe,
+
+      # CHECK: 4523107f cdab8967 badcfeef <unknown>
+      0x7f, 0x10, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba,
+
+      # CHECK: 207f 4523 8967 cdab feef badc 7698 <unknown>
+      0x7f, 0x20, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76,
+
+      # CHECK: 4523307f cdab8967 badcfeef 32547698 <unknown>
+      0x7f, 0x30, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32,
+
+      # CHECK: 407f 4523 8967 cdab feef badc 7698 3254 1210 <unknown>
+      0x7f, 0x40, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, 0x12,
+
+      # CHECK: 4523507f cdab8967 badcfeef 32547698 56341210 <unknown>
+      0x7f, 0x50, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, 0x12, 0x34, 0x56,
+
+      # CHECK: 607f 4523 8967 cdab feef badc 7698 3254 1210 5634 9a78 <unknown>
+      0x7f, 0x60, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10, 0x12, 0x34, 0x56, 0x78, 0x9a,
+    ]
+
+  - Type:            SectionHeaderTable
+    Sections:
+      - Name:            .strtab
+      - Name:            .symtab
+      - Name:            .text
+Symbols:
+  - Name:    "$x"
+    Section: .text
+    Value:   0x0
+...
diff --git a/llvm/test/MC/RISCV/nop-slide.s b/llvm/test/MC/RISCV/nop-slide.s
index 4dc888b3ba777..9e7401dc929d3 100644
--- a/llvm/test/MC/RISCV/nop-slide.s
+++ b/llvm/test/MC/RISCV/nop-slide.s
@@ -1,5 +1,5 @@
-# RUN: llvm-mc -triple riscv64 -mattr +c,-relax -filetype obj -o - %s | llvm-objdump -d - | FileCheck %s -check-prefix CHECK-RVC-NORELAX
-# RUN: llvm-mc -triple riscv64 -mattr +c,+relax -filetype obj -o - %s | llvm-objdump -d - | FileCheck %s -check-prefix CHECK-RVC-RELAX
+# RUN: llvm-mc -triple riscv64 -mattr +c,-relax -filetype obj -o - %s | llvm-objdump -d - | FileCheck %s
+# RUN: llvm-mc -triple riscv64 -mattr +c,+relax -filetype obj -o - %s | llvm-objdump -d - | FileCheck %s
 # RUN: llvm-mc -triple riscv64 -mattr -c,-relax -filetype obj -o - %s | llvm-objdump -d - | FileCheck %s
 # RUN: llvm-mc -triple riscv64 -mattr -c,+relax -filetype obj -o - %s | llvm-objdump -d - | FileCheck %s
 
@@ -9,19 +9,6 @@
 .balign 4
 auipc a0, 0
 
-# CHECK-RVC-NORELAX: 0000000000000000 <.text>:
-# CHECK-RVC-NORELAX-NEXT: 0: 0000      	unimp
-# CHECK-RVC-NORELAX-NEXT: 2: 0001      	nop
-# CHECK-RVC-NORELAX-NEXT: 4: 00000517  	auipc	a0, 0x0
-
-# CHECK-RVC-RELAX: 0000000000000000 <.text>:
-# CHECK-RVC-RELAX-NEXT:   0: 0001      	nop
-# CHECK-RVC-RELAX-NEXT:   2: 0100      	addi	s0, sp, 0x80
-# CHECK-RVC-RELAX-NEXT:   4: 1700      	addi	s0, sp, 0x3a0
-# CHECK-RVC-RELAX-NEXT:   6: 0005      	c.nop	0x1
-# CHECK-RVC-RELAX-NEXT:   8: 00        	<unknown>
-
 # CHECK: 0000000000000000 <.text>:
-# CHECK-NEXT: 0: 0000      	<unknown>
-# CHECK-NEXT: 2: 0000      	<unknown>
+# CHECK-NEXT: 0: 00 00 01 00   .word   0x00010000
 # CHECK-NEXT: 4: 00000517  	auipc	a0, 0x0
diff --git a/llvm/test/MC/RISCV/rv32p-invalid.s b/llvm/test/MC/RISCV/rv32p-invalid.s
index da3c67b73d0af..b00c39b8811dc 100644
--- a/llvm/test/MC/RISCV/rv32p-invalid.s
+++ b/llvm/test/MC/RISCV/rv32p-invalid.s
@@ -1,19 +1,116 @@
 # RUN: not llvm-mc -triple=riscv32 --mattr=+experimental-p %s 2>&1 \
-# RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+# RUN:     | FileCheck %s
 
 # Imm overflow
-pli.h a0, 0x400
-# CHECK-ERROR: immediate must be an integer in the range [-512, 511]
-plui.h a1, 0x400
-# CHECK-ERROR: immediate must be an integer in the range [-512, 1023]
-pli.b a0, 0x200
-# CHECK-ERROR: immediate must be an integer in the range [0, 255]
-
-pslli.b a6, a7, 100
-# CHECK-ERROR: immediate must be an integer in the range [0, 7]
-pslli.h ra, sp, 100
-# CHECK-ERROR: immediate must be an integer in the range [0, 15]
-psslai.h t0, t1, 100
-# CHECK-ERROR: immediate must be an integer in the range [0, 15]
-sslai a4, a5, -1
-# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+pli.h a0, 0x400 # CHECK: :[[@LINE]]:11: error: immediate must be an integer in the range [-512, 511]
+plui.h a1, 0x400 # CHECK: :[[@LINE]]:12: error: immediate must be an integer in the range [-512, 1023]
+pli.b a0, 0x200 # CHECK: :[[@LINE]]:11: error: immediate must be an integer in the range [-128, 255]
+
+pslli.b a6, a7, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 7]
+pslli.h ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 15]
+pslli.w ra, sp, 12 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+psslai.h t0, t1, 100 # CHECK: :[[@LINE]]:18: error: immediate must be an integer in the range [0, 15]
+psslai.w t0, t1, 27 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+sslai a4, a5, -1 # CHECK: :[[@LINE]]:15: error: immediate must be an integer in the range [0, 31]
+
+psll.ws a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+padd.ws a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pssha.ws a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+psshar.ws a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+sha a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+shar a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+psrli.b a6, a7, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 7]
+psrli.h ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 15]
+psrli.w ra, sp, 31 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+pusati.h ra, sp, 100 # CHECK: :[[@LINE]]:18: error: immediate must be an integer in the range [0, 15]
+pusati.w ra, sp, 0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+usati ra, sp, 100 # CHECK: :[[@LINE]]:15: error: immediate must be an integer in the range [0, 31]
+
+psrai.b a6, a7, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 7]
+psrai.h ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 15]
+psrai.w ra, sp, 10 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+psrari.h ra, sp, 100 # CHECK: :[[@LINE]]:18: error: immediate must be an integer in the range [0, 15]
+psrari.w ra, sp, 15 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+srari ra, sp, 100 # CHECK: :[[@LINE]]:15: error: immediate must be an integer in the range [0, 31]
+
+psati.h ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 15]
+psati.w ra, sp, 24 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+sati ra, sp, 100 # CHECK: :[[@LINE]]:14: error: immediate must be an integer in the range [0, 31]
+
+psrl.ws a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+predsum.ws a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+predsumu.ws a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+psra.ws a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+padd.w t3, s0, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+psadd.w t3, t1, s2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+paadd.w t5, t1, a4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+psaddu.w s0, s2, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+paaddu.w s0, t1, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+psub.w t3, a0, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pssub.w t3, a4, t1 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pasub.w t3, a2, a4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pssubu.w a0, a4, t3 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pasubu.w a0, t3, a4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+pmul.w.h01 s2, t5, t3 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+mul.w01 t5, a4, t1 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmacc.w.h01 t1, t1, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+macc.w01 s2, a0, t3 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmulu.w.h01 t1, a4, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+mulu.w01 t5, t1, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmaccu.w.h01 t5, t5, a4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+maccu.w01 a0, a0, t1 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+psh1add.w s2, t5, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pssh1sadd.w a4, t3, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+unzip8p a4, t3, t1 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+unzip16p t5, a4, t3 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+unzip8hp s0, a0, t1 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+unzip16hp a0, a0, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+zip8p t5, t3, t3 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+zip16p a0, t5, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+zip8hp t5, a0, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+zip16hp t1, t5, a4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+pmul.w.h00 s2, t1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+mul.w00 a4, a0, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmacc.w.h00 s2, t5, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+macc.w00 t1, a0, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmul.w.h11 s0, a4, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+mul.w11 a0, t3, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmacc.w.h11 a4, a4, t1 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+macc.w11 t3, s2, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmulu.w.h00 a2, t3, t1 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+mulu.w00 a0, t5, s2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmaccu.w.h00 t3, t3, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+maccu.w00 s2, t1, s2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmulu.w.h11 s0, t5, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+mulu.w11 s0, t1, a4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmaccu.w.h11 a0, s0, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+maccu.w11 s2, t3, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmulsu.w.h00 t5, t5, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+mulsu.w00 t1, s0, a4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmulsu.w.h11 t1, t3, s2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+mulsu.w11 a2, s2, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmaccsu.w.h00 a4, a0, t1 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+maccsu.w00 a4, s2, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+pmaccsu.w.h11 a0, a2, t3 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+maccsu.w11 t5, a4, s2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+ppack.w t5, a2, a4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+ppackbt.w t5, s0, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+ppacktb.w t5, t1, t1 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+ppackt.w t3, a0, s2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV64I Base Instruction Set
+
+pli.dh a1, 1 # CHECK: :[[@LINE]]:8: error: register must be even
+pli.db s1, 1 # CHECK: :[[@LINE]]:8: error: register must be even
+plui.dh t2, 1 # CHECK: :[[@LINE]]:9: error: register must be even
+
+pli.dh a0, 0x400 # CHECK: :[[@LINE]]:12: error: immediate must be an integer in the range [-512, 511]
+pli.db a0, 0x200 # CHECK: :[[@LINE]]:12: error: immediate must be an integer in the range [-128, 255]
+plui.dh a0, 0x400 # CHECK: :[[@LINE]]:13: error: immediate must be an integer in the range [-512, 1023]
diff --git a/llvm/test/MC/RISCV/rv32p-valid.s b/llvm/test/MC/RISCV/rv32p-valid.s
index ffff0f25642a3..432fd05d231ee 100644
--- a/llvm/test/MC/RISCV/rv32p-valid.s
+++ b/llvm/test/MC/RISCV/rv32p-valid.s
@@ -61,6 +61,9 @@ pli.h a5, 16
 # CHECK-ASM-AND-OBJ: pli.b a6, 16
 # CHECK-ASM: encoding: [0x1b,0x28,0x10,0xb4]
 pli.b a6, 16
+# CHECK-ASM-AND-OBJ: pli.b a6, -128
+# CHECK-ASM: encoding: [0x1b,0x28,0x80,0xb4]
+pli.b a6, -128
 # CHECK-ASM-AND-OBJ: psext.h.b a7, a0
 # CHECK-ASM: encoding: [0x9b,0x28,0x45,0xe0]
 psext.h.b a7, a0
@@ -76,3 +79,597 @@ plui.h gp, 32
 # CHECK-ASM-AND-OBJ: plui.h gp, -412
 # CHECK-ASM: encoding: [0x9b,0x21,0x99,0xf0]
 plui.h gp, 612
+# CHECK-ASM-AND-OBJ: psll.hs a0, a1, a2
+# CHECK-ASM: encoding: [0x1b,0xa5,0xc5,0x88]
+psll.hs a0, a1, a2
+# CHECK-ASM-AND-OBJ: psll.bs a3, a4, a5
+# CHECK-ASM: encoding: [0x9b,0x26,0xf7,0x8c]
+psll.bs a3, a4, a5
+# CHECK-ASM-AND-OBJ: padd.hs t0, t1, t2
+# CHECK-ASM: encoding: [0x9b,0x22,0x73,0x98]
+padd.hs t0, t1, t2
+# CHECK-ASM-AND-OBJ: padd.bs ra, a1, a2
+# CHECK-ASM: encoding: [0x9b,0xa0,0xc5,0x9c]
+padd.bs ra, a1, a2
+# CHECK-ASM-AND-OBJ: pssha.hs a3, a4, a5
+# CHECK-ASM: encoding: [0x9b,0x26,0xf7,0xe8]
+pssha.hs a3, a4, a5
+# CHECK-ASM-AND-OBJ: ssha gp, a4, a5
+# CHECK-ASM: encoding: [0x9b,0x21,0xf7,0xea]
+ssha gp, a4, a5
+# CHECK-ASM-AND-OBJ: psshar.hs a6, a7, a0
+# CHECK-ASM: encoding: [0x1b,0xa8,0xa8,0xf8]
+psshar.hs a6, a7, a0
+# CHECK-ASM-AND-OBJ: sshar t1, a7, a0
+# CHECK-ASM: encoding: [0x1b,0xa3,0xa8,0xfa]
+sshar t1, a7, a0
+# CHECK-ASM-AND-OBJ: psrli.b a6, a7, 0
+# CHECK-ASM: encoding: [0x1b,0xc8,0x88,0x80]
+psrli.b a6, a7, 0
+# CHECK-ASM-AND-OBJ: psrli.h ra, sp, 1
+# CHECK-ASM: encoding: [0x9b,0x40,0x11,0x81]
+psrli.h ra, sp, 1
+# CHECK-ASM-AND-OBJ: pusati.h t2, t3, 4
+# CHECK-ASM: encoding: [0x9b,0x43,0x4e,0xa1]
+pusati.h t2, t3, 4
+# CHECK-ASM-AND-OBJ: usati t3, t4, 5
+# CHECK-ASM: encoding: [0x1b,0xce,0x5e,0xa2]
+usati t3, t4, 5
+# CHECK-ASM-AND-OBJ: psrai.b a6, a7, 0
+# CHECK-ASM: encoding: [0x1b,0xc8,0x88,0xc0]
+psrai.b a6, a7, 0
+# CHECK-ASM-AND-OBJ: psrai.h ra, sp, 1
+# CHECK-ASM: encoding: [0x9b,0x40,0x11,0xc1]
+psrai.h ra, sp, 1
+# CHECK-ASM-AND-OBJ: psrari.h t4, t5, 6
+# CHECK-ASM: encoding: [0x9b,0x4e,0x6f,0xd1]
+psrari.h t4, t5, 6
+# CHECK-ASM-AND-OBJ: srari t5, t6, 7
+# CHECK-ASM: encoding: [0x1b,0xcf,0x7f,0xd2]
+srari t5, t6, 7
+# CHECK-ASM-AND-OBJ: psati.h t6, s11, 8
+# CHECK-ASM: encoding: [0x9b,0xcf,0x8d,0xe1]
+psati.h t6, s11, 8
+# CHECK-ASM-AND-OBJ: sati s11, s10, 9
+# CHECK-ASM: encoding: [0x9b,0x4d,0x9d,0xe2]
+sati s11, s10, 9
+# CHECK-ASM-AND-OBJ: psrl.hs a6, a7, a1
+# CHECK-ASM: encoding: [0x1b,0xc8,0xb8,0x88]
+psrl.hs a6, a7, a1
+# CHECK-ASM-AND-OBJ: psrl.bs a1, a2, a3
+# CHECK-ASM: encoding: [0x9b,0x45,0xd6,0x8c]
+psrl.bs a1, a2, a3
+# CHECK-ASM-AND-OBJ: predsum.hs a4, a5, a6
+# CHECK-ASM: encoding: [0x1b,0xc7,0x07,0x99]
+predsum.hs a4, a5, a6
+# CHECK-ASM-AND-OBJ: predsum.bs a7, a1, a1
+# CHECK-ASM: encoding: [0x9b,0xc8,0xb5,0x9c]
+predsum.bs a7, a1, a1
+# CHECK-ASM-AND-OBJ: predsumu.hs t0, t1, t2
+# CHECK-ASM: encoding: [0x9b,0x42,0x73,0xb8]
+predsumu.hs t0, t1, t2
+# CHECK-ASM-AND-OBJ: predsumu.bs t3, t4, t5
+# CHECK-ASM: encoding: [0x1b,0xce,0xee,0xbd]
+predsumu.bs t3, t4, t5
+# CHECK-ASM-AND-OBJ: psra.hs ra, a1, a2
+# CHECK-ASM: encoding: [0x9b,0xc0,0xc5,0xc8]
+psra.hs ra, a1, a2
+# CHECK-ASM-AND-OBJ: psra.bs sp, a2, a3
+# CHECK-ASM: encoding: [0x1b,0x41,0xd6,0xcc]
+psra.bs sp, a2, a3
+# CHECK-ASM-AND-OBJ: padd.h t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0x80]
+padd.h t0, t1, t2
+# CHECK-ASM-AND-OBJ: padd.b ra, a1, a2
+# CHECK-ASM: encoding: [0xbb,0x80,0xc5,0x84]
+padd.b ra, a1, a2
+# CHECK-ASM-AND-OBJ: psadd.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0x91]
+psadd.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: sadd t1, a7, a0
+# CHECK-ASM: encoding: [0x3b,0x83,0xa8,0x92]
+sadd t1, a7, a0
+# CHECK-ASM-AND-OBJ: psadd.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0x94]
+psadd.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: paadd.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0x99]
+paadd.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: aadd t1, a7, a0
+# CHECK-ASM: encoding: [0x3b,0x83,0xa8,0x9a]
+aadd t1, a7, a0
+# CHECK-ASM-AND-OBJ: paadd.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0x9c]
+paadd.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: psaddu.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0xb1]
+psaddu.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: saddu t1, a7, a0
+# CHECK-ASM: encoding: [0x3b,0x83,0xa8,0xb2]
+saddu t1, a7, a0
+# CHECK-ASM-AND-OBJ: psaddu.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0xb4]
+psaddu.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: paaddu.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0xb9]
+paaddu.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: aaddu t1, a7, a0
+# CHECK-ASM: encoding: [0x3b,0x83,0xa8,0xba]
+aaddu t1, a7, a0
+# CHECK-ASM-AND-OBJ: paaddu.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0xbc]
+paaddu.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: psub.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0xc1]
+psub.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: psub.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0xc4]
+psub.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: pdif.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0xc9]
+pdif.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: pdif.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0xcc]
+pdif.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: pssub.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0xd1]
+pssub.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: ssub t1, a7, a0
+# CHECK-ASM: encoding: [0x3b,0x83,0xa8,0xd2]
+ssub t1, a7, a0
+# CHECK-ASM-AND-OBJ: pssub.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0xd4]
+pssub.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: pasub.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0xd9]
+pasub.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: asub t1, a7, a0
+# CHECK-ASM: encoding: [0x3b,0x83,0xa8,0xda]
+asub t1, a7, a0
+# CHECK-ASM-AND-OBJ: pasub.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0xdc]
+pasub.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: pdifu.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0xe9]
+pdifu.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: pdifu.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0xec]
+pdifu.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: pssubu.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0xf1]
+pssubu.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: ssubu t1, a7, a0
+# CHECK-ASM: encoding: [0x3b,0x83,0xa8,0xf2]
+ssubu t1, a7, a0
+# CHECK-ASM-AND-OBJ: pssubu.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0xf4]
+pssubu.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: pasubu.h t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x8e,0xee,0xf9]
+pasubu.h t3, t4, t5
+# CHECK-ASM-AND-OBJ: asubu t1, a7, a0
+# CHECK-ASM: encoding: [0x3b,0x83,0xa8,0xfa]
+asubu t1, a7, a0
+# CHECK-ASM-AND-OBJ: pasubu.b t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x02,0x73,0xfc]
+pasubu.b t0, t1, t2
+# CHECK-ASM-AND-OBJ: slx gp, a4, a5
+# CHECK-ASM: encoding: [0xbb,0x11,0xf7,0x8e]
+slx gp, a4, a5
+# CHECK-ASM-AND-OBJ: pmul.h.b01 t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x9e,0xee,0x91]
+pmul.h.b01 t3, t4, t5
+# CHECK-ASM-AND-OBJ: mul.h01 t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x12,0x73,0x92]
+mul.h01 t0, t1, t2
+# CHECK-ASM-AND-OBJ: macc.h01 t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x9e,0xee,0x9b]
+macc.h01 t3, t4, t5
+# CHECK-ASM-AND-OBJ: mvm a0, a1, a2
+# CHECK-ASM: encoding: [0x3b,0x95,0xc5,0xa8]
+mvm a0, a1, a2
+# CHECK-ASM-AND-OBJ: mvmn gp, a4, a5
+# CHECK-ASM: encoding: [0xbb,0x11,0xf7,0xaa]
+mvmn gp, a4, a5
+# CHECK-ASM-AND-OBJ: merge t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x12,0x73,0xac]
+merge t0, t1, t2
+# CHECK-ASM-AND-OBJ: srx gp, a4, a5
+# CHECK-ASM: encoding: [0xbb,0x11,0xf7,0xae]
+srx gp, a4, a5
+# CHECK-ASM-AND-OBJ: pmulu.h.b01 t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x9e,0xee,0xb1]
+pmulu.h.b01 t3, t4, t5
+# CHECK-ASM-AND-OBJ: mulu.h01 t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x12,0x73,0xb2]
+mulu.h01 t0, t1, t2
+# CHECK-ASM-AND-OBJ: pdifsumu.b t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x9e,0xee,0xb5]
+pdifsumu.b t3, t4, t5
+# CHECK-ASM-AND-OBJ: maccu.h01 t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x9e,0xee,0xbb]
+maccu.h01 t3, t4, t5
+# CHECK-ASM-AND-OBJ: pdifsumau.b t3, t4, t5
+# CHECK-ASM: encoding: [0x3b,0x9e,0xee,0xbd]
+pdifsumau.b t3, t4, t5
+# CHECK-ASM-AND-OBJ: psh1add.h a0, a1, a2
+# CHECK-ASM: encoding: [0x3b,0xa5,0xc5,0xa0]
+psh1add.h a0, a1, a2
+# CHECK-ASM-AND-OBJ: pssh1sadd.h a3, a4, a5
+# CHECK-ASM: encoding: [0xbb,0x26,0xf7,0xb0]
+pssh1sadd.h a3, a4, a5
+# CHECK-ASM-AND-OBJ: ssh1sadd t0, t1, t2
+# CHECK-ASM: encoding: [0xbb,0x22,0x73,0xb2]
+ssh1sadd t0, t1, t2
+# CHECK-ASM-AND-OBJ: pmul.h.b00 s6, t4, s8
+# CHECK-ASM: encoding: [0x3b,0xbb,0x8e,0x81]
+pmul.h.b00 s6, t4, s8
+# CHECK-ASM-AND-OBJ: mul.h00 a4, t4, s2
+# CHECK-ASM: encoding: [0x3b,0xb7,0x2e,0x83]
+mul.h00 a4, t4, s2
+# CHECK-ASM-AND-OBJ: macc.h00 a4, a0, s0
+# CHECK-ASM: encoding: [0x3b,0x37,0x85,0x8a]
+macc.h00 a4, a0, s0
+# CHECK-ASM-AND-OBJ: pmul.h.b11 t0, t4, s4
+# CHECK-ASM: encoding: [0xbb,0xb2,0x4e,0x91]
+pmul.h.b11 t0, t4, s4
+# CHECK-ASM-AND-OBJ: mul.h11 a0, a4, a0
+# CHECK-ASM: encoding: [0x3b,0x35,0xa7,0x92]
+mul.h11 a0, a4, a0
+# CHECK-ASM-AND-OBJ: macc.h11 s6, a4, s4
+# CHECK-ASM: encoding: [0x3b,0x3b,0x47,0x9b]
+macc.h11 s6, a4, s4
+# CHECK-ASM-AND-OBJ: pmulu.h.b00 t2, s6, s8
+# CHECK-ASM: encoding: [0xbb,0x33,0x8b,0xa1]
+pmulu.h.b00 t2, s6, s8
+# CHECK-ASM-AND-OBJ: mulu.h00 s6, s0, a0
+# CHECK-ASM: encoding: [0x3b,0x3b,0xa4,0xa2]
+mulu.h00 s6, s0, a0
+# CHECK-ASM-AND-OBJ: maccu.h00 s0, s6, s0
+# CHECK-ASM: encoding: [0x3b,0x34,0x8b,0xaa]
+maccu.h00 s0, s6, s0
+# CHECK-ASM-AND-OBJ: pmulu.h.b11 t2, s6, a0
+# CHECK-ASM: encoding: [0xbb,0x33,0xab,0xb0]
+pmulu.h.b11 t2, s6, a0
+# CHECK-ASM-AND-OBJ: mulu.h11 s0, s4, s6
+# CHECK-ASM: encoding: [0x3b,0x34,0x6a,0xb3]
+mulu.h11 s0, s4, s6
+# CHECK-ASM-AND-OBJ: maccu.h11 s0, t4, t4
+# CHECK-ASM: encoding: [0x3b,0xb4,0xde,0xbb]
+maccu.h11 s0, t4, t4
+# CHECK-ASM-AND-OBJ: pmulsu.h.b00 s4, a4, s8
+# CHECK-ASM: encoding: [0x3b,0x3a,0x87,0xe1]
+pmulsu.h.b00 s4, a4, s8
+# CHECK-ASM-AND-OBJ: mulsu.h00 a4, s4, s6
+# CHECK-ASM: encoding: [0x3b,0x37,0x6a,0xe3]
+mulsu.h00 a4, s4, s6
+# CHECK-ASM-AND-OBJ: maccsu.h00 s4, s4, s0
+# CHECK-ASM: encoding: [0x3b,0x3a,0x8a,0xea]
+maccsu.h00 s4, s4, s0
+# CHECK-ASM-AND-OBJ: pmulsu.h.b11 s6, a2, s4
+# CHECK-ASM: encoding: [0x3b,0x3b,0x46,0xf1]
+pmulsu.h.b11 s6, a2, s4
+# CHECK-ASM-AND-OBJ: mulsu.h11 s8, s4, s0
+# CHECK-ASM: encoding: [0x3b,0x3c,0x8a,0xf2]
+mulsu.h11 s8, s4, s0
+# CHECK-ASM-AND-OBJ: maccsu.h11 s0, a2, s6
+# CHECK-ASM: encoding: [0x3b,0x34,0x66,0xfb]
+maccsu.h11 s0, a2, s6
+# CHECK-ASM-AND-OBJ: ppack.h t1, a2, t5
+# CHECK-ASM: encoding: [0x3b,0x43,0xe6,0x81]
+ppack.h t1, a2, t5
+# CHECK-ASM-AND-OBJ: ppackbt.h t5, t3, s2
+# CHECK-ASM: encoding: [0x3b,0x4f,0x2e,0x91]
+ppackbt.h t5, t3, s2
+# CHECK-ASM-AND-OBJ: packbt t1, t1, s2
+# CHECK-ASM: encoding: [0x3b,0x43,0x23,0x93]
+packbt t1, t1, s2
+# CHECK-ASM-AND-OBJ: ppacktb.h t1, t1, s0
+# CHECK-ASM: encoding: [0x3b,0x43,0x83,0xa0]
+ppacktb.h t1, t1, s0
+# CHECK-ASM-AND-OBJ: packtb t5, s0, a2
+# CHECK-ASM: encoding: [0x3b,0x4f,0xc4,0xa2]
+packtb t5, s0, a2
+# CHECK-ASM-AND-OBJ: ppackt.h t3, s0, s0
+# CHECK-ASM: encoding: [0x3b,0x4e,0x84,0xb0]
+ppackt.h t3, s0, s0
+# CHECK-ASM-AND-OBJ: packt a2, t3, t1
+# CHECK-ASM: encoding: [0x3b,0x46,0x6e,0xb2]
+packt a2, t3, t1
+# CHECK-ASM: encoding: [0x3b,0x5e,0x83,0x80]
+pm2add.h t3, t1, s0
+# CHECK-ASM-AND-OBJ: pm4add.b t1, s2, t5
+# CHECK-ASM: encoding: [0x3b,0x53,0xe9,0x85]
+pm4add.b t1, s2, t5
+# CHECK-ASM-AND-OBJ: pm2adda.h t1, s2, a4
+# CHECK-ASM: encoding: [0x3b,0x53,0xe9,0x88]
+pm2adda.h t1, s2, a4
+# CHECK-ASM-AND-OBJ: pm4adda.b t3, a0, t5
+# CHECK-ASM: encoding: [0x3b,0x5e,0xe5,0x8d]
+pm4adda.b t3, a0, t5
+# CHECK-ASM-AND-OBJ: pm2add.hx s0, t5, a4
+# CHECK-ASM: encoding: [0x3b,0x54,0xef,0x90]
+pm2add.hx s0, t5, a4
+# CHECK-ASM-AND-OBJ: pm2adda.hx a0, a0, t1
+# CHECK-ASM: encoding: [0x3b,0x55,0x65,0x98]
+pm2adda.hx a0, a0, t1
+# CHECK-ASM-AND-OBJ: pm2addu.h s2, t5, a2
+# CHECK-ASM: encoding: [0x3b,0x59,0xcf,0xa0]
+pm2addu.h s2, t5, a2
+# CHECK-ASM-AND-OBJ: pm4addu.b t5, s0, t1
+# CHECK-ASM: encoding: [0x3b,0x5f,0x64,0xa4]
+pm4addu.b t5, s0, t1
+# CHECK-ASM-AND-OBJ: pm2addau.h t3, t1, t5
+# CHECK-ASM: encoding: [0x3b,0x5e,0xe3,0xa9]
+pm2addau.h t3, t1, t5
+# CHECK-ASM-AND-OBJ: pm4addau.b a4, t3, a0
+# CHECK-ASM: encoding: [0x3b,0x57,0xae,0xac]
+pm4addau.b a4, t3, a0
+# CHECK-ASM-AND-OBJ: pmq2add.h t1, a2, t1
+# CHECK-ASM: encoding: [0x3b,0x53,0x66,0xb0]
+pmq2add.h t1, a2, t1
+# CHECK-ASM-AND-OBJ: pmqr2add.h a2, a4, s2
+# CHECK-ASM: encoding: [0x3b,0x56,0x27,0xb5]
+pmqr2add.h a2, a4, s2
+# CHECK-ASM-AND-OBJ: pmq2adda.h a2, s2, t1
+# CHECK-ASM: encoding: [0x3b,0x56,0x69,0xb8]
+pmq2adda.h a2, s2, t1
+# CHECK-ASM-AND-OBJ: pmqr2adda.h t1, s0, a2
+# CHECK-ASM: encoding: [0x3b,0x53,0xc4,0xbc]
+pmqr2adda.h t1, s0, a2
+# CHECK-ASM-AND-OBJ: pm2sub.h t1, a0, a4
+# CHECK-ASM: encoding: [0x3b,0x53,0xe5,0xc0]
+pm2sub.h t1, a0, a4
+# CHECK-ASM-AND-OBJ: pm2sadd.h s2, s2, t3
+# CHECK-ASM: encoding: [0x3b,0x59,0xc9,0xc5]
+pm2sadd.h s2, s2, t3
+# CHECK-ASM-AND-OBJ: pm2suba.h s0, s0, t3
+# CHECK-ASM: encoding: [0x3b,0x54,0xc4,0xc9]
+pm2suba.h s0, s0, t3
+# CHECK-ASM-AND-OBJ: pm2sub.hx a2, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x56,0xa6,0xd0]
+pm2sub.hx a2, a2, a0
+# CHECK-ASM-AND-OBJ: pm2sadd.hx t5, s2, a0
+# CHECK-ASM: encoding: [0x3b,0x5f,0xa9,0xd4]
+pm2sadd.hx t5, s2, a0
+# CHECK-ASM-AND-OBJ: pm2suba.hx a4, a2, t5
+# CHECK-ASM: encoding: [0x3b,0x57,0xe6,0xd9]
+pm2suba.hx a4, a2, t5
+# CHECK-ASM-AND-OBJ: pm2addsu.h s0, s0, s2
+# CHECK-ASM: encoding: [0x3b,0x54,0x24,0xe1]
+pm2addsu.h s0, s0, s2
+# CHECK-ASM-AND-OBJ: pm4addsu.b a0, s0, t3
+# CHECK-ASM: encoding: [0x3b,0x55,0xc4,0xe5]
+pm4addsu.b a0, s0, t3
+# CHECK-ASM-AND-OBJ: pm2addasu.h a4, t1, a2
+# CHECK-ASM: encoding: [0x3b,0x57,0xc3,0xe8]
+pm2addasu.h a4, t1, a2
+# CHECK-ASM-AND-OBJ: pm4addasu.b s2, s0, a2
+# CHECK-ASM: encoding: [0x3b,0x59,0xc4,0xec]
+pm4addasu.b s2, s0, a2
+# CHECK-ASM-AND-OBJ: mqacc.h01 a4, t1, a0
+# CHECK-ASM: encoding: [0x3b,0x57,0xa3,0xf8]
+mqacc.h01 a4, t1, a0
+# CHECK-ASM-AND-OBJ: mqracc.h01 s0, a0, t5
+# CHECK-ASM: encoding: [0x3b,0x54,0xe5,0xfd]
+mqracc.h01 s0, a0, t5
+# CHECK-ASM-AND-OBJ: pas.hx t5, s2, t5
+# CHECK-ASM: encoding: [0x3b,0x6f,0xe9,0x81]
+pas.hx t5, s2, t5
+# CHECK-ASM-AND-OBJ: psa.hx s0, t1, t3
+# CHECK-ASM: encoding: [0x3b,0x64,0xc3,0x85]
+psa.hx s0, t1, t3
+# CHECK-ASM-AND-OBJ: psas.hx t5, s2, a2
+# CHECK-ASM: encoding: [0x3b,0x6f,0xc9,0x90]
+psas.hx t5, s2, a2
+# CHECK-ASM-AND-OBJ: pssa.hx s2, t3, t5
+# CHECK-ASM: encoding: [0x3b,0x69,0xee,0x95]
+pssa.hx s2, t3, t5
+# CHECK-ASM-AND-OBJ: paas.hx t1, a2, s0
+# CHECK-ASM: encoding: [0x3b,0x63,0x86,0x98]
+paas.hx t1, a2, s0
+# CHECK-ASM-AND-OBJ: pasa.hx t1, s2, t3
+# CHECK-ASM: encoding: [0x3b,0x63,0xc9,0x9d]
+pasa.hx t1, s2, t3
+# CHECK-ASM-AND-OBJ: pmseq.h a0, t1, s0
+# CHECK-ASM: encoding: [0x3b,0x65,0x83,0xc0]
+pmseq.h a0, t1, s0
+# CHECK-ASM-AND-OBJ: pmseq.b s0, s2, s0
+# CHECK-ASM: encoding: [0x3b,0x64,0x89,0xc4]
+pmseq.b s0, s2, s0
+# CHECK-ASM-AND-OBJ: pmslt.h t3, a2, a4
+# CHECK-ASM: encoding: [0x3b,0x6e,0xe6,0xd0]
+pmslt.h t3, a2, a4
+# CHECK-ASM-AND-OBJ: pmslt.b t5, a4, t1
+# CHECK-ASM: encoding: [0x3b,0x6f,0x67,0xd4]
+pmslt.b t5, a4, t1
+# CHECK-ASM-AND-OBJ: pmsltu.h s2, s0, s2
+# CHECK-ASM: encoding: [0x3b,0x69,0x24,0xd9]
+pmsltu.h s2, s0, s2
+# CHECK-ASM-AND-OBJ: pmsltu.b s0, s2, t5
+# CHECK-ASM: encoding: [0x3b,0x64,0xe9,0xdd]
+pmsltu.b s0, s2, t5
+# CHECK-ASM-AND-OBJ: pmin.h s2, t3, s2
+# CHECK-ASM: encoding: [0x3b,0x69,0x2e,0xe1]
+pmin.h s2, t3, s2
+# CHECK-ASM-AND-OBJ: pmin.b a2, a2, t5
+# CHECK-ASM: encoding: [0x3b,0x66,0xe6,0xe5]
+pmin.b a2, a2, t5
+# CHECK-ASM-AND-OBJ: pminu.h a0, a4, a2
+# CHECK-ASM: encoding: [0x3b,0x65,0xc7,0xe8]
+pminu.h a0, a4, a2
+# CHECK-ASM-AND-OBJ: pminu.b a4, t3, s2
+# CHECK-ASM: encoding: [0x3b,0x67,0x2e,0xed]
+pminu.b a4, t3, s2
+# CHECK-ASM-AND-OBJ: pmax.h a4, s2, a0
+# CHECK-ASM: encoding: [0x3b,0x67,0xa9,0xf0]
+pmax.h a4, s2, a0
+# CHECK-ASM-AND-OBJ: pmax.b t1, s0, s0
+# CHECK-ASM: encoding: [0x3b,0x63,0x84,0xf4]
+pmax.b t1, s0, s0
+# CHECK-ASM-AND-OBJ: pmaxu.h a0, s0, s0
+# CHECK-ASM: encoding: [0x3b,0x65,0x84,0xf8]
+pmaxu.h a0, s0, s0
+# CHECK-ASM-AND-OBJ: pmaxu.b t3, a0, t1
+# CHECK-ASM: encoding: [0x3b,0x6e,0x65,0xfc]
+pmaxu.b t3, a0, t1
+# CHECK-ASM-AND-OBJ: mseq a4, t1, s0
+# CHECK-ASM: encoding: [0x3b,0x67,0x83,0xc2]
+mseq a4, t1, s0
+# CHECK-ASM-AND-OBJ: mslt t5, t5, t1
+# CHECK-ASM: encoding: [0x3b,0x6f,0x6f,0xd2]
+mslt t5, t5, t1
+# CHECK-ASM-AND-OBJ: msltu s2, a0, t3
+# CHECK-ASM: encoding: [0x3b,0x69,0xc5,0xdb]
+msltu s2, a0, t3
+# CHECK-ASM-AND-OBJ: pmulh.h s0, t3, t3
+# CHECK-ASM: encoding: [0x3b,0x74,0xce,0x81]
+pmulh.h s0, t3, t3
+# CHECK-ASM-AND-OBJ: pmulhr.h t1, t5, s0
+# CHECK-ASM: encoding: [0x3b,0x73,0x8f,0x84]
+pmulhr.h t1, t5, s0
+# CHECK-ASM-AND-OBJ: pmhacc.h s0, t5, s2
+# CHECK-ASM: encoding: [0x3b,0x74,0x2f,0x89]
+pmhacc.h s0, t5, s2
+# CHECK-ASM-AND-OBJ: pmhracc.h a4, t3, a2
+# CHECK-ASM: encoding: [0x3b,0x77,0xce,0x8c]
+pmhracc.h a4, t3, a2
+# CHECK-ASM-AND-OBJ: pmulhu.h a4, t5, a2
+# CHECK-ASM: encoding: [0x3b,0x77,0xcf,0x90]
+pmulhu.h a4, t5, a2
+# CHECK-ASM-AND-OBJ: pmulhru.h a4, a2, a4
+# CHECK-ASM: encoding: [0x3b,0x77,0xe6,0x94]
+pmulhru.h a4, a2, a4
+# CHECK-ASM-AND-OBJ: pmhaccu.h a4, t1, t3
+# CHECK-ASM: encoding: [0x3b,0x77,0xc3,0x99]
+pmhaccu.h a4, t1, t3
+# CHECK-ASM-AND-OBJ: pmhraccu.h s2, s0, t1
+# CHECK-ASM: encoding: [0x3b,0x79,0x64,0x9c]
+pmhraccu.h s2, s0, t1
+# CHECK-ASM-AND-OBJ: pmulh.h.b0 a0, t5, a4
+# CHECK-ASM: encoding: [0x3b,0x75,0xef,0xa0]
+pmulh.h.b0 a0, t5, a4
+# CHECK-ASM-AND-OBJ: pmulhsu.h.b0 s0, a4, s2
+# CHECK-ASM: encoding: [0x3b,0x74,0x27,0xa5]
+pmulhsu.h.b0 s0, a4, s2
+# CHECK-ASM-AND-OBJ: pmhacc.h.b0 s0, a0, t5
+# CHECK-ASM: encoding: [0x3b,0x74,0xe5,0xa9]
+pmhacc.h.b0 s0, a0, t5
+# CHECK-ASM-AND-OBJ: pmhaccsu.h.b0 t3, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x7e,0xe4,0xac]
+pmhaccsu.h.b0 t3, s0, a4
+# CHECK-ASM-AND-OBJ: pmulh.h.b1 a0, a4, s2
+# CHECK-ASM: encoding: [0x3b,0x75,0x27,0xb1]
+pmulh.h.b1 a0, a4, s2
+# CHECK-ASM-AND-OBJ: pmulhsu.h.b1 t5, t3, t1
+# CHECK-ASM: encoding: [0x3b,0x7f,0x6e,0xb4]
+pmulhsu.h.b1 t5, t3, t1
+# CHECK-ASM-AND-OBJ: pmhacc.h.b1 t1, t5, a2
+# CHECK-ASM: encoding: [0x3b,0x73,0xcf,0xb8]
+pmhacc.h.b1 t1, t5, a2
+# CHECK-ASM-AND-OBJ: pmhaccsu.h.b1 a2, a0, a4
+# CHECK-ASM: encoding: [0x3b,0x76,0xe5,0xbc]
+pmhaccsu.h.b1 a2, a0, a4
+# CHECK-ASM-AND-OBJ: pmulhsu.h s0, a0, t1
+# CHECK-ASM: encoding: [0x3b,0x74,0x65,0xc0]
+pmulhsu.h s0, a0, t1
+# CHECK-ASM-AND-OBJ: pmulhrsu.h t3, t5, a4
+# CHECK-ASM: encoding: [0x3b,0x7e,0xef,0xc4]
+pmulhrsu.h t3, t5, a4
+# CHECK-ASM-AND-OBJ: pmhaccsu.h s0, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x74,0xe4,0xc8]
+pmhaccsu.h s0, s0, a4
+# CHECK-ASM-AND-OBJ: pmhraccsu.h a2, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x76,0xa6,0xcc]
+pmhraccsu.h a2, a2, a0
+# CHECK-ASM-AND-OBJ: pmulq.h a0, t1, t1
+# CHECK-ASM: encoding: [0x3b,0x75,0x63,0xd0]
+pmulq.h a0, t1, t1
+# CHECK-ASM-AND-OBJ: pmulqr.h s2, s0, s2
+# CHECK-ASM: encoding: [0x3b,0x79,0x24,0xd5]
+pmulqr.h s2, s0, s2
+# CHECK-ASM-AND-OBJ: mulhr a4, s2, t5
+# CHECK-ASM: encoding: [0x3b,0x77,0xe9,0x87]
+mulhr a4, s2, t5
+# CHECK-ASM-AND-OBJ: mhacc t1, s0, a2
+# CHECK-ASM: encoding: [0x3b,0x73,0xc4,0x8a]
+mhacc t1, s0, a2
+# CHECK-ASM-AND-OBJ: mhracc t1, t5, s0
+# CHECK-ASM: encoding: [0x3b,0x73,0x8f,0x8e]
+mhracc t1, t5, s0
+# CHECK-ASM-AND-OBJ: mulhru t1, t5, s0
+# CHECK-ASM: encoding: [0x3b,0x73,0x8f,0x96]
+mulhru t1, t5, s0
+# CHECK-ASM-AND-OBJ: mhaccu t3, a2, s2
+# CHECK-ASM: encoding: [0x3b,0x7e,0x26,0x9b]
+mhaccu t3, a2, s2
+# CHECK-ASM-AND-OBJ: mhraccu a0, t1, a4
+# CHECK-ASM: encoding: [0x3b,0x75,0xe3,0x9e]
+mhraccu a0, t1, a4
+# CHECK-ASM-AND-OBJ: mulh.h0 t3, a4, t1
+# CHECK-ASM: encoding: [0x3b,0x7e,0x67,0xa2]
+mulh.h0 t3, a4, t1
+# CHECK-ASM-AND-OBJ: mulhsu.h0 t1, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x73,0xa6,0xa6]
+mulhsu.h0 t1, a2, a0
+# CHECK-ASM-AND-OBJ: mhacc.h0 s0, a2, t3
+# CHECK-ASM: encoding: [0x3b,0x74,0xc6,0xab]
+mhacc.h0 s0, a2, t3
+# CHECK-ASM-AND-OBJ: mhaccsu.h0 a2, t1, s0
+# CHECK-ASM: encoding: [0x3b,0x76,0x83,0xae]
+mhaccsu.h0 a2, t1, s0
+# CHECK-ASM-AND-OBJ: mulh.h1 t1, t1, t3
+# CHECK-ASM: encoding: [0x3b,0x73,0xc3,0xb3]
+mulh.h1 t1, t1, t3
+# CHECK-ASM-AND-OBJ: mulhsu.h1 t3, a2, t1
+# CHECK-ASM: encoding: [0x3b,0x7e,0x66,0xb6]
+mulhsu.h1 t3, a2, t1
+# CHECK-ASM-AND-OBJ: mhacc.h1 t3, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x7e,0xa6,0xba]
+mhacc.h1 t3, a2, a0
+# CHECK-ASM-AND-OBJ: mhaccsu.h1 s0, s2, s0
+# CHECK-ASM: encoding: [0x3b,0x74,0x89,0xbe]
+mhaccsu.h1 s0, s2, s0
+# CHECK-ASM-AND-OBJ: mulhrsu t5, a4, t5
+# CHECK-ASM: encoding: [0x3b,0x7f,0xe7,0xc7]
+mulhrsu t5, a4, t5
+# CHECK-ASM-AND-OBJ: mhaccsu a0, a2, s2
+# CHECK-ASM: encoding: [0x3b,0x75,0x26,0xcb]
+mhaccsu a0, a2, s2
+# CHECK-ASM-AND-OBJ: mhraccsu a0, a0, t1
+# CHECK-ASM: encoding: [0x3b,0x75,0x65,0xce]
+mhraccsu a0, a0, t1
+# CHECK-ASM-AND-OBJ: mulq t1, a2, a2
+# CHECK-ASM: encoding: [0x3b,0x73,0xc6,0xd2]
+mulq t1, a2, a2
+# CHECK-ASM-AND-OBJ: mulqr a4, a4, t3
+# CHECK-ASM: encoding: [0x3b,0x77,0xc7,0xd7]
+mulqr a4, a4, t3
+# CHECK-ASM-AND-OBJ: mqacc.h00 a2, t3, t3
+# CHECK-ASM: encoding: [0x3b,0x76,0xce,0xe9]
+mqacc.h00 a2, t3, t3
+# CHECK-ASM-AND-OBJ: mqracc.h00 t5, a4, t3
+# CHECK-ASM: encoding: [0x3b,0x7f,0xc7,0xed]
+mqracc.h00 t5, a4, t3
+# CHECK-ASM-AND-OBJ: mqacc.h11 t5, t5, s0
+# CHECK-ASM: encoding: [0x3b,0x7f,0x8f,0xf8]
+mqacc.h11 t5, t5, s0
+# CHECK-ASM-AND-OBJ: mqracc.h11 s0, t5, s2
+# CHECK-ASM: encoding: [0x3b,0x74,0x2f,0xfd]
+mqracc.h11 s0, t5, s2
+
+# CHECK-ASM-AND-OBJ: pli.dh a4, 16
+# CHECK-ASM: encoding: [0x1b,0x27,0x10,0x30]
+pli.dh a4, 16
+# CHECK-ASM-AND-OBJ: pli.db a6, 16
+# CHECK-ASM: encoding: [0x1b,0x28,0x10,0x34]
+pli.db a6, 16
+# CHECK-ASM-AND-OBJ: pli.db a6, -128
+# CHECK-ASM: encoding: [0x1b,0x28,0x80,0x34]
+pli.db a6, -128
+# CHECK-ASM-AND-OBJ: plui.dh tp, 32
+# CHECK-ASM: encoding: [0x1b,0x22,0x08,0x70]
+plui.dh tp, 32
+# CHECK-ASM-AND-OBJ: plui.dh tp, -412
+# CHECK-ASM: encoding: [0x1b,0x22,0x99,0x70]
+plui.dh tp, 612
diff --git a/llvm/test/MC/RISCV/rv64p-invalid.s b/llvm/test/MC/RISCV/rv64p-invalid.s
index 572a099fba4d4..e18c9ec0e29ea 100644
--- a/llvm/test/MC/RISCV/rv64p-invalid.s
+++ b/llvm/test/MC/RISCV/rv64p-invalid.s
@@ -1,21 +1,72 @@
 # RUN: not llvm-mc -triple=riscv64 --mattr=+experimental-p %s 2>&1 \
-# RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+# RUN:     | FileCheck %s
 
 # Imm overflow
-pli.h a0, 0x400
-# CHECK-ERROR: immediate must be an integer in the range [-512, 511]
-plui.h a1, 0x400
-# CHECK-ERROR: immediate must be an integer in the range [-512, 1023]
-pli.w a1, -0x201
-# CHECK-ERROR: immediate must be an integer in the range [-512, 511]
-
-pslli.b a6, a7, 100
-# CHECK-ERROR: immediate must be an integer in the range [0, 7]
-pslli.h ra, sp, 100
-# CHECK-ERROR: immediate must be an integer in the range [0, 15]
-pslli.w ra, sp, 100
-# CHECK-ERROR: immediate must be an integer in the range [0, 31]
-psslai.h t0, t1, 100
-# CHECK-ERROR: immediate must be an integer in the range [0, 15]
-psslai.w a4, a5, -1
-# CHECK-ERROR: error: immediate must be an integer in the range [0, 31]
+pli.h a0, 0x400 # CHECK: :[[@LINE]]:11: error: immediate must be an integer in the range [-512, 511]
+plui.h a1, 0x400 # CHECK: :[[@LINE]]:12: error: immediate must be an integer in the range [-512, 1023]
+pli.w a1, -0x201 # CHECK: :[[@LINE]]:11: error: immediate must be an integer in the range [-512, 511]
+
+pslli.b a6, a7, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 7]
+pslli.h ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 15]
+pslli.w ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 31]
+
+ssha a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+sshar a0, a1, a2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+
+psslai.h t0, t1, 100 # CHECK: :[[@LINE]]:18: error: immediate must be an integer in the range [0, 15]
+psslai.w a4, a5, -1 # CHECK: :[[@LINE]]:18: error: immediate must be an integer in the range [0, 31]
+sslai ra, sp, 10 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+
+psrli.b a6, a7, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 7]
+psrli.h ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 15]
+psrli.w ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 31]
+
+pusati.h ra, sp, 100 # CHECK: :[[@LINE]]:18: error: immediate must be an integer in the range [0, 15]
+pusati.w ra, sp, 100 # CHECK: :[[@LINE]]:18: error: immediate must be an integer in the range [0, 31]
+usati ra, sp, 100 # CHECK: :[[@LINE]]:15: error: immediate must be an integer in the range [0, 63]
+
+psrai.b a6, a7, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 7]
+psrai.h ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 15]
+psrai.w ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 31]
+
+psrari.h ra, sp, 100 # CHECK: :[[@LINE]]:18: error: immediate must be an integer in the range [0, 15]
+psrari.w ra, sp, 100 # CHECK: :[[@LINE]]:18: error: immediate must be an integer in the range [0, 31]
+srari ra, sp, 100 # CHECK: :[[@LINE]]:15: error: immediate must be an integer in the range [0, 63]
+
+psati.h ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 15]
+psati.w ra, sp, 100 # CHECK: :[[@LINE]]:17: error: immediate must be an integer in the range [0, 31]
+sati ra, sp, 100 # CHECK: :[[@LINE]]:14: error: immediate must be an integer in the range [0, 63]
+
+sadd t1, a7, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+aadd t1, a7, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+saddu t1, a7, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+aaddu t1, a7, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+ssub t1, a7, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+asub t1, a7, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+ssubu t1, a7, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+asubu t1, a7, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+
+mul.h01 t0, t1, t2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+macc.h01 t3, t4, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+mulu.h01 t0, t1, t2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+maccu.h01 t3, t4, t5 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+
+ssh1sadd t0, t1, t2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+
+mul.h00 a4, t4, s2 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+macc.h00 a4, a0, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+mul.h11 a0, a4, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+macc.h11 s6, a4, s4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+mulu.h00 s6, s0, a0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+maccu.h00 s0, s6, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+mulu.h11 s0, s4, s6 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+maccu.h11 s0, t4, t4 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+mulsu.h00 a4, s4, s6 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+maccsu.h00 s4, s4, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+mulsu.h11 s8, s4, s0 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+maccsu.h11 s0, a2, s6 # CHECK: :[[@LINE]]:1: error: instruction requires the following: RV32I Base Instruction Set
+
+# FIXME: This error doesn't make sense. Should say that we need RV32I.
+pli.dh a0, 1 # CHECK: :[[@LINE]]:8: error: invalid operand for instruction
+pli.db s0, 1 # CHECK: :[[@LINE]]:8: error: invalid operand for instruction
+plui.dh t1, 1 # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/rv64p-valid.s b/llvm/test/MC/RISCV/rv64p-valid.s
index a0d6eadfb6c30..a789f506da499 100644
--- a/llvm/test/MC/RISCV/rv64p-valid.s
+++ b/llvm/test/MC/RISCV/rv64p-valid.s
@@ -55,7 +55,7 @@ max t3, t4, t5
 # CHECK-ASM-AND-OBJ: maxu a4, a5, a6
 # CHECK-ASM: encoding: [0x33,0xf7,0x07,0x0b]
 maxu a4, a5, a6
-# CHECK-ASM-AND-OBJ: pslli.b a6, a7
+# CHECK-ASM-AND-OBJ: pslli.b a6, a7, 0
 # CHECK-ASM: encoding: [0x1b,0xa8,0x88,0x80]
 pslli.b a6, a7, 0
 # CHECK-ASM-AND-OBJ: pslli.h ra, sp, 1
@@ -79,6 +79,12 @@ pli.w a5, 5
 # CHECK-ASM-AND-OBJ: pli.b a6, 6
 # CHECK-ASM: encoding: [0x1b,0x28,0x06,0xb4]
 pli.b a6, 6
+# CHECK-ASM-AND-OBJ: pli.b a6, -1
+# CHECK-ASM: encoding: [0x1b,0x28,0xff,0xb4]
+pli.b a6, -1
+# CHECK-ASM-AND-OBJ: pli.b a6, -1
+# CHECK-ASM: encoding: [0x1b,0x28,0xff,0xb4]
+pli.b a6, 255
 # CHECK-ASM-AND-OBJ: psext.h.b t3, a2
 # CHECK-ASM: encoding: [0x1b,0x2e,0x46,0xe0]
 psext.h.b t3, a2
@@ -106,3 +112,825 @@ plui.w a2, 1
 # CHECK-ASM-AND-OBJ: plui.w a2, -1
 # CHECK-ASM: encoding: [0x1b,0xa6,0xff,0xf3]
 plui.w a2, 1023
+# CHECK-ASM-AND-OBJ: psll.hs s0, a2, s2
+# CHECK-ASM: encoding: [0x1b,0x24,0x26,0x89]
+psll.hs s0, a2, s2
+# CHECK-ASM-AND-OBJ: psll.bs a0, t3, t5
+# CHECK-ASM: encoding: [0x1b,0x25,0xee,0x8d]
+psll.bs a0, t3, t5
+# CHECK-ASM-AND-OBJ: padd.hs t1, a2, s0
+# CHECK-ASM: encoding: [0x1b,0x23,0x86,0x98]
+padd.hs t1, a2, s0
+# CHECK-ASM-AND-OBJ: padd.bs t3, t1, t3
+# CHECK-ASM: encoding: [0x1b,0x2e,0xc3,0x9d]
+padd.bs t3, t1, t3
+# CHECK-ASM-AND-OBJ: pssha.hs s0, t1, a2
+# CHECK-ASM: encoding: [0x1b,0x24,0xc3,0xe8]
+pssha.hs s0, t1, a2
+# CHECK-ASM-AND-OBJ: psshar.hs s2, t5, t3
+# CHECK-ASM: encoding: [0x1b,0x29,0xcf,0xf9]
+psshar.hs s2, t5, t3
+# CHECK-ASM-AND-OBJ: psll.ws s0, t1, a0
+# CHECK-ASM: encoding: [0x1b,0x24,0xa3,0x8a]
+psll.ws s0, t1, a0
+# CHECK-ASM-AND-OBJ: padd.ws s2, a2, a0
+# CHECK-ASM: encoding: [0x1b,0x29,0xa6,0x9a]
+padd.ws s2, a2, a0
+# CHECK-ASM-AND-OBJ: pssha.ws a4, a2, t1
+# CHECK-ASM: encoding: [0x1b,0x27,0x66,0xea]
+pssha.ws a4, a2, t1
+# CHECK-ASM-AND-OBJ: psshar.ws a2, a0, a4
+# CHECK-ASM: encoding: [0x1b,0x26,0xe5,0xfa]
+psshar.ws a2, a0, a4
+# CHECK-ASM-AND-OBJ: sha a0, t5, t5
+# CHECK-ASM: encoding: [0x1b,0x25,0xef,0xef]
+sha a0, t5, t5
+# CHECK-ASM-AND-OBJ: shar t5, t5, t3
+# CHECK-ASM: encoding: [0x1b,0x2f,0xcf,0xff]
+shar t5, t5, t3
+# CHECK-ASM-AND-OBJ: psrli.b a6, a7
+# CHECK-ASM: encoding: [0x1b,0xc8,0x88,0x80]
+psrli.b a6, a7, 0
+# CHECK-ASM-AND-OBJ: psrli.h ra, sp, 1
+# CHECK-ASM: encoding: [0x9b,0x40,0x11,0x81]
+psrli.h ra, sp, 1
+# CHECK-ASM-AND-OBJ: psrli.w ra, sp, 2
+# CHECK-ASM: encoding: [0x9b,0x40,0x21,0x82]
+psrli.w ra, sp, 2
+# CHECK-ASM-AND-OBJ: pusati.h t2, t3, 4
+# CHECK-ASM: encoding: [0x9b,0x43,0x4e,0xa1]
+pusati.h t2, t3, 4
+# CHECK-ASM-AND-OBJ: pusati.w t2, t3, 5
+# CHECK-ASM: encoding: [0x9b,0x43,0x5e,0xa2]
+pusati.w t2, t3, 5
+# CHECK-ASM-AND-OBJ: usati t3, t4, 5
+# CHECK-ASM: encoding: [0x1b,0xce,0x5e,0xa4]
+usati t3, t4, 5
+# CHECK-ASM-AND-OBJ: psrai.b a6, a7, 0
+# CHECK-ASM: encoding: [0x1b,0xc8,0x88,0xc0]
+psrai.b a6, a7, 0
+# CHECK-ASM-AND-OBJ: psrai.h ra, sp, 1
+# CHECK-ASM: encoding: [0x9b,0x40,0x11,0xc1]
+psrai.h ra, sp, 1
+# CHECK-ASM-AND-OBJ: psrai.w ra, sp, 2
+# CHECK-ASM: encoding: [0x9b,0x40,0x21,0xc2]
+psrai.w ra, sp, 2
+# CHECK-ASM-AND-OBJ: psrari.h t4, t5, 6
+# CHECK-ASM: encoding: [0x9b,0x4e,0x6f,0xd1]
+psrari.h t4, t5, 6
+# CHECK-ASM-AND-OBJ: psrari.w t5, t6, 7
+# CHECK-ASM: encoding: [0x1b,0xcf,0x7f,0xd2]
+psrari.w t5, t6, 7
+# CHECK-ASM-AND-OBJ: srari t6, s11, 63
+# CHECK-ASM: encoding: [0x9b,0xcf,0xfd,0xd7]
+srari t6, s11, 63
+# CHECK-ASM-AND-OBJ: psati.h s11, s10, 9
+# CHECK-ASM: encoding: [0x9b,0x4d,0x9d,0xe1]
+psati.h s11, s10, 9
+# CHECK-ASM-AND-OBJ: psati.w s10, s9, 10
+# CHECK-ASM: encoding: [0x1b,0xcd,0xac,0xe2]
+psati.w s10, s9, 10
+# CHECK-ASM-AND-OBJ: sati s9, s8, 32
+# CHECK-ASM: encoding: [0x9b,0x4c,0x0c,0xe6]
+sati s9, s8, 32
+# CHECK-ASM-AND-OBJ: psrl.hs a6, a7, a1
+# CHECK-ASM: encoding: [0x1b,0xc8,0xb8,0x88]
+psrl.hs a6, a7, a1
+# CHECK-ASM-AND-OBJ: psrl.bs a1, a2, a3
+# CHECK-ASM: encoding: [0x9b,0x45,0xd6,0x8c]
+psrl.bs a1, a2, a3
+# CHECK-ASM-AND-OBJ: predsum.hs a4, a5, a6
+# CHECK-ASM: encoding: [0x1b,0xc7,0x07,0x99]
+predsum.hs a4, a5, a6
+# CHECK-ASM-AND-OBJ: predsum.bs a7, a1, a1
+# CHECK-ASM: encoding: [0x9b,0xc8,0xb5,0x9c]
+predsum.bs a7, a1, a1
+# CHECK-ASM-AND-OBJ: predsumu.hs t0, t1, t2
+# CHECK-ASM: encoding: [0x9b,0x42,0x73,0xb8]
+predsumu.hs t0, t1, t2
+# CHECK-ASM-AND-OBJ: predsumu.bs t3, t4, t5
+# CHECK-ASM: encoding: [0x1b,0xce,0xee,0xbd]
+predsumu.bs t3, t4, t5
+# CHECK-ASM-AND-OBJ: psra.hs ra, a1, a2
+# CHECK-ASM: encoding: [0x9b,0xc0,0xc5,0xc8]
+psra.hs ra, a1, a2
+# CHECK-ASM-AND-OBJ: psra.bs sp, a2, a3
+# CHECK-ASM: encoding: [0x1b,0x41,0xd6,0xcc]
+psra.bs sp, a2, a3
+# CHECK-ASM-AND-OBJ: padd.h t1, t5, s2
+# CHECK-ASM: encoding: [0x3b,0x03,0x2f,0x81]
+padd.h t1, t5, s2
+# CHECK-ASM-AND-OBJ: padd.w t3, s0, a0
+# CHECK-ASM: encoding: [0x3b,0x0e,0xa4,0x82]
+padd.w t3, s0, a0
+# CHECK-ASM-AND-OBJ: padd.b t5, s0, t1
+# CHECK-ASM: encoding: [0x3b,0x0f,0x64,0x84]
+padd.b t5, s0, t1
+# CHECK-ASM-AND-OBJ: psadd.h a2, a2, s2
+# CHECK-ASM: encoding: [0x3b,0x06,0x26,0x91]
+psadd.h a2, a2, s2
+# CHECK-ASM-AND-OBJ: psadd.w t3, t1, s2
+# CHECK-ASM: encoding: [0x3b,0x0e,0x23,0x93]
+psadd.w t3, t1, s2
+# CHECK-ASM-AND-OBJ: psadd.b t1, a0, s0
+# CHECK-ASM: encoding: [0x3b,0x03,0x85,0x94]
+psadd.b t1, a0, s0
+# CHECK-ASM-AND-OBJ: paadd.h t5, s0, t3
+# CHECK-ASM: encoding: [0x3b,0x0f,0xc4,0x99]
+paadd.h t5, s0, t3
+# CHECK-ASM-AND-OBJ: paadd.w t5, t1, a4
+# CHECK-ASM: encoding: [0x3b,0x0f,0xe3,0x9a]
+paadd.w t5, t1, a4
+# CHECK-ASM-AND-OBJ: paadd.b a4, s2, a4
+# CHECK-ASM: encoding: [0x3b,0x07,0xe9,0x9c]
+paadd.b a4, s2, a4
+# CHECK-ASM-AND-OBJ: psaddu.h a0, t1, t1
+# CHECK-ASM: encoding: [0x3b,0x05,0x63,0xb0]
+psaddu.h a0, t1, t1
+# CHECK-ASM-AND-OBJ: psaddu.w s0, s2, t5
+# CHECK-ASM: encoding: [0x3b,0x04,0xe9,0xb3]
+psaddu.w s0, s2, t5
+# CHECK-ASM-AND-OBJ: psaddu.b t3, a2, a4
+# CHECK-ASM: encoding: [0x3b,0x0e,0xe6,0xb4]
+psaddu.b t3, a2, a4
+# CHECK-ASM-AND-OBJ: paaddu.h t3, s2, a2
+# CHECK-ASM: encoding: [0x3b,0x0e,0xc9,0xb8]
+paaddu.h t3, s2, a2
+# CHECK-ASM-AND-OBJ: paaddu.w s0, t1, s0
+# CHECK-ASM: encoding: [0x3b,0x04,0x83,0xba]
+paaddu.w s0, t1, s0
+# CHECK-ASM-AND-OBJ: paaddu.b t3, a0, t3
+# CHECK-ASM: encoding: [0x3b,0x0e,0xc5,0xbd]
+paaddu.b t3, a0, t3
+# CHECK-ASM-AND-OBJ: psub.h s0, s2, t3
+# CHECK-ASM: encoding: [0x3b,0x04,0xc9,0xc1]
+psub.h s0, s2, t3
+# CHECK-ASM-AND-OBJ: psub.w t3, a0, s0
+# CHECK-ASM: encoding: [0x3b,0x0e,0x85,0xc2]
+psub.w t3, a0, s0
+# CHECK-ASM-AND-OBJ: psub.b t5, t1, a4
+# CHECK-ASM: encoding: [0x3b,0x0f,0xe3,0xc4]
+psub.b t5, t1, a4
+# CHECK-ASM-AND-OBJ: pdif.h t1, a4, a2
+# CHECK-ASM: encoding: [0x3b,0x03,0xc7,0xc8]
+pdif.h t1, a4, a2
+# CHECK-ASM-AND-OBJ: pdif.b t3, t1, t5
+# CHECK-ASM: encoding: [0x3b,0x0e,0xe3,0xcd]
+pdif.b t3, t1, t5
+# CHECK-ASM-AND-OBJ: pssub.h a0, a2, t3
+# CHECK-ASM: encoding: [0x3b,0x05,0xc6,0xd1]
+pssub.h a0, a2, t3
+# CHECK-ASM-AND-OBJ: pssub.w t3, a4, t1
+# CHECK-ASM: encoding: [0x3b,0x0e,0x67,0xd2]
+pssub.w t3, a4, t1
+# CHECK-ASM-AND-OBJ: pssub.b a2, t5, a4
+# CHECK-ASM: encoding: [0x3b,0x06,0xef,0xd4]
+pssub.b a2, t5, a4
+# CHECK-ASM-AND-OBJ: pasub.h t5, t3, t3
+# CHECK-ASM: encoding: [0x3b,0x0f,0xce,0xd9]
+pasub.h t5, t3, t3
+# CHECK-ASM-AND-OBJ: pasub.w t3, a2, a4
+# CHECK-ASM: encoding: [0x3b,0x0e,0xe6,0xda]
+pasub.w t3, a2, a4
+# CHECK-ASM-AND-OBJ: pasub.b s0, t3, s2
+# CHECK-ASM: encoding: [0x3b,0x04,0x2e,0xdd]
+pasub.b s0, t3, s2
+# CHECK-ASM-AND-OBJ: pdifu.h t5, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x0f,0xe4,0xe8]
+pdifu.h t5, s0, a4
+# CHECK-ASM-AND-OBJ: pdifu.b t3, a0, t5
+# CHECK-ASM: encoding: [0x3b,0x0e,0xe5,0xed]
+pdifu.b t3, a0, t5
+# CHECK-ASM-AND-OBJ: pssubu.h t3, s2, a0
+# CHECK-ASM: encoding: [0x3b,0x0e,0xa9,0xf0]
+pssubu.h t3, s2, a0
+# CHECK-ASM-AND-OBJ: pssubu.w a0, a4, t3
+# CHECK-ASM: encoding: [0x3b,0x05,0xc7,0xf3]
+pssubu.w a0, a4, t3
+# CHECK-ASM-AND-OBJ: pssubu.b t3, a4, t3
+# CHECK-ASM: encoding: [0x3b,0x0e,0xc7,0xf5]
+pssubu.b t3, a4, t3
+# CHECK-ASM-AND-OBJ: pasubu.h a2, s0, t5
+# CHECK-ASM: encoding: [0x3b,0x06,0xe4,0xf9]
+pasubu.h a2, s0, t5
+# CHECK-ASM-AND-OBJ: pasubu.w a0, t3, a4
+# CHECK-ASM: encoding: [0x3b,0x05,0xee,0xfa]
+pasubu.w a0, t3, a4
+# CHECK-ASM-AND-OBJ: pasubu.b s0, t5, a4
+# CHECK-ASM: encoding: [0x3b,0x04,0xef,0xfc]
+pasubu.b s0, t5, a4
+# CHECK-ASM-AND-OBJ: slx a0, a2, s2
+# CHECK-ASM: encoding: [0x3b,0x15,0x26,0x8f]
+slx a0, a2, s2
+# CHECK-ASM-AND-OBJ: pmul.h.b01 a2, a4, a2
+# CHECK-ASM: encoding: [0x3b,0x16,0xc7,0x90]
+pmul.h.b01 a2, a4, a2
+# CHECK-ASM-AND-OBJ: pmul.w.h01 s2, t5, t3
+# CHECK-ASM: encoding: [0x3b,0x19,0xcf,0x93]
+pmul.w.h01 s2, t5, t3
+# CHECK-ASM-AND-OBJ: mul.w01 t5, a4, t1
+# CHECK-ASM: encoding: [0x3b,0x1f,0x67,0x96]
+mul.w01 t5, a4, t1
+# CHECK-ASM-AND-OBJ: pmacc.w.h01 t1, t1, a0
+# CHECK-ASM: encoding: [0x3b,0x13,0xa3,0x9a]
+pmacc.w.h01 t1, t1, a0
+# CHECK-ASM-AND-OBJ: macc.w01 s2, a0, t3
+# CHECK-ASM: encoding: [0x3b,0x19,0xc5,0x9f]
+macc.w01 s2, a0, t3
+# CHECK-ASM-AND-OBJ: mvm s0, t1, a2
+# CHECK-ASM: encoding: [0x3b,0x14,0xc3,0xa8]
+mvm s0, t1, a2
+# CHECK-ASM-AND-OBJ: mvmn a2, a4, a0
+# CHECK-ASM: encoding: [0x3b,0x16,0xa7,0xaa]
+mvmn a2, a4, a0
+# CHECK-ASM-AND-OBJ: merge a4, a2, a2
+# CHECK-ASM: encoding: [0x3b,0x17,0xc6,0xac]
+merge a4, a2, a2
+# CHECK-ASM-AND-OBJ: srx t1, t3, a4
+# CHECK-ASM: encoding: [0x3b,0x13,0xee,0xae]
+srx t1, t3, a4
+# CHECK-ASM-AND-OBJ: pmulu.h.b01 s2, a4, a0
+# CHECK-ASM: encoding: [0x3b,0x19,0xa7,0xb0]
+pmulu.h.b01 s2, a4, a0
+# CHECK-ASM-AND-OBJ: pmulu.w.h01 t1, a4, a2
+# CHECK-ASM: encoding: [0x3b,0x13,0xc7,0xb2]
+pmulu.w.h01 t1, a4, a2
+# CHECK-ASM-AND-OBJ: pdifsumu.b t3, t5, t3
+# CHECK-ASM: encoding: [0x3b,0x1e,0xcf,0xb5]
+pdifsumu.b t3, t5, t3
+# CHECK-ASM-AND-OBJ: mulu.w01 t5, t1, t5
+# CHECK-ASM: encoding: [0x3b,0x1f,0xe3,0xb7]
+mulu.w01 t5, t1, t5
+# CHECK-ASM-AND-OBJ: pmaccu.w.h01 t5, t5, a4
+# CHECK-ASM: encoding: [0x3b,0x1f,0xef,0xba]
+pmaccu.w.h01 t5, t5, a4
+# CHECK-ASM-AND-OBJ: pdifsumau.b s2, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x19,0xa6,0xbc]
+pdifsumau.b s2, a2, a0
+# CHECK-ASM-AND-OBJ: maccu.w01 a0, a0, t1
+# CHECK-ASM: encoding: [0x3b,0x15,0x65,0xbe]
+maccu.w01 a0, a0, t1
+# CHECK-ASM-AND-OBJ: psh1add.h a2, a2, t3
+# CHECK-ASM: encoding: [0x3b,0x26,0xc6,0xa1]
+psh1add.h a2, a2, t3
+# CHECK-ASM-AND-OBJ: pssh1sadd.h a2, t3, t3
+# CHECK-ASM: encoding: [0x3b,0x26,0xce,0xb1]
+pssh1sadd.h a2, t3, t3
+# CHECK-ASM-AND-OBJ: psh1add.w s2, t5, a2
+# CHECK-ASM: encoding: [0x3b,0x29,0xcf,0xa2]
+psh1add.w s2, t5, a2
+# CHECK-ASM-AND-OBJ: pssh1sadd.w a4, t3, s0
+# CHECK-ASM: encoding: [0x3b,0x27,0x8e,0xb2]
+pssh1sadd.w a4, t3, s0
+# CHECK-ASM-AND-OBJ: unzip8p a4, t3, t1
+# CHECK-ASM: encoding: [0x3b,0x27,0x6e,0xe0]
+unzip8p a4, t3, t1
+# CHECK-ASM-AND-OBJ: unzip16p t5, a4, t3
+# CHECK-ASM: encoding: [0x3b,0x2f,0xc7,0xe3]
+unzip16p t5, a4, t3
+# CHECK-ASM-AND-OBJ: unzip8hp s0, a0, t1
+# CHECK-ASM: encoding: [0x3b,0x24,0x65,0xe4]
+unzip8hp s0, a0, t1
+# CHECK-ASM-AND-OBJ: unzip16hp a0, a0, a2
+# CHECK-ASM: encoding: [0x3b,0x25,0xc5,0xe6]
+unzip16hp a0, a0, a2
+# CHECK-ASM-AND-OBJ: zip8p t5, t3, t3
+# CHECK-ASM: encoding: [0x3b,0x2f,0xce,0xf1]
+zip8p t5, t3, t3
+# CHECK-ASM-AND-OBJ: zip16p a0, t5, a0
+# CHECK-ASM: encoding: [0x3b,0x25,0xaf,0xf2]
+zip16p a0, t5, a0
+# CHECK-ASM-AND-OBJ: zip8hp t5, a0, a2
+# CHECK-ASM: encoding: [0x3b,0x2f,0xc5,0xf4]
+zip8hp t5, a0, a2
+# CHECK-ASM-AND-OBJ: zip16hp t1, t5, a4
+# CHECK-ASM: encoding: [0x3b,0x23,0xef,0xf6]
+zip16hp t1, t5, a4
+# CHECK-ASM-AND-OBJ: pmul.h.b00 a4, a4, s2
+# CHECK-ASM: encoding: [0x3b,0x37,0x27,0x81]
+pmul.h.b00 a4, a4, s2
+# CHECK-ASM-AND-OBJ: pmul.w.h00 s2, t1, a2
+# CHECK-ASM: encoding: [0x3b,0x39,0xc3,0x82]
+pmul.w.h00 s2, t1, a2
+# CHECK-ASM-AND-OBJ: mul.w00 a4, a0, a2
+# CHECK-ASM: encoding: [0x3b,0x37,0xc5,0x86]
+mul.w00 a4, a0, a2
+# CHECK-ASM-AND-OBJ: pmacc.w.h00 s2, t5, a2
+# CHECK-ASM: encoding: [0x3b,0x39,0xcf,0x8a]
+pmacc.w.h00 s2, t5, a2
+# CHECK-ASM-AND-OBJ: macc.w00 t1, a0, t5
+# CHECK-ASM: encoding: [0x3b,0x33,0xe5,0x8f]
+macc.w00 t1, a0, t5
+# CHECK-ASM-AND-OBJ: pmul.h.b11 t1, s2, s0
+# CHECK-ASM: encoding: [0x3b,0x33,0x89,0x90]
+pmul.h.b11 t1, s2, s0
+# CHECK-ASM-AND-OBJ: pmul.w.h11 s0, a4, s0
+# CHECK-ASM: encoding: [0x3b,0x34,0x87,0x92]
+pmul.w.h11 s0, a4, s0
+# CHECK-ASM-AND-OBJ: mul.w11 a0, t3, a0
+# CHECK-ASM: encoding: [0x3b,0x35,0xae,0x96]
+mul.w11 a0, t3, a0
+# CHECK-ASM-AND-OBJ: pmacc.w.h11 a4, a4, t1
+# CHECK-ASM: encoding: [0x3b,0x37,0x67,0x9a]
+pmacc.w.h11 a4, a4, t1
+# CHECK-ASM-AND-OBJ: macc.w11 t3, s2, a2
+# CHECK-ASM: encoding: [0x3b,0x3e,0xc9,0x9e]
+macc.w11 t3, s2, a2
+# CHECK-ASM-AND-OBJ: pmulu.h.b00 a4, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x37,0xa6,0xa0]
+pmulu.h.b00 a4, a2, a0
+# CHECK-ASM-AND-OBJ: pmulu.w.h00 a2, t3, t1
+# CHECK-ASM: encoding: [0x3b,0x36,0x6e,0xa2]
+pmulu.w.h00 a2, t3, t1
+# CHECK-ASM-AND-OBJ: mulu.w00 a0, t5, s2
+# CHECK-ASM: encoding: [0x3b,0x35,0x2f,0xa7]
+mulu.w00 a0, t5, s2
+# CHECK-ASM-AND-OBJ: pmaccu.w.h00 t3, t3, t5
+# CHECK-ASM: encoding: [0x3b,0x3e,0xee,0xab]
+pmaccu.w.h00 t3, t3, t5
+# CHECK-ASM-AND-OBJ: maccu.w00 s2, t1, s2
+# CHECK-ASM: encoding: [0x3b,0x39,0x23,0xaf]
+maccu.w00 s2, t1, s2
+# CHECK-ASM-AND-OBJ: pmulu.h.b11 a4, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x37,0xe4,0xb0]
+pmulu.h.b11 a4, s0, a4
+# CHECK-ASM-AND-OBJ: pmulu.w.h11 s0, t5, t5
+# CHECK-ASM: encoding: [0x3b,0x34,0xef,0xb3]
+pmulu.w.h11 s0, t5, t5
+# CHECK-ASM-AND-OBJ: mulu.w11 s0, t1, a4
+# CHECK-ASM: encoding: [0x3b,0x34,0xe3,0xb6]
+mulu.w11 s0, t1, a4
+# CHECK-ASM-AND-OBJ: pmaccu.w.h11 a0, s0, t5
+# CHECK-ASM: encoding: [0x3b,0x35,0xe4,0xbb]
+pmaccu.w.h11 a0, s0, t5
+# CHECK-ASM-AND-OBJ: maccu.w11 s2, t3, t5
+# CHECK-ASM: encoding: [0x3b,0x39,0xee,0xbf]
+maccu.w11 s2, t3, t5
+# CHECK-ASM-AND-OBJ: pmulsu.h.b00 a2, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x36,0xe4,0xe0]
+pmulsu.h.b00 a2, s0, a4
+# CHECK-ASM-AND-OBJ: pmulsu.w.h00 t5, t5, t5
+# CHECK-ASM: encoding: [0x3b,0x3f,0xef,0xe3]
+pmulsu.w.h00 t5, t5, t5
+# CHECK-ASM-AND-OBJ: mulsu.w00 t1, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x33,0xe4,0xe6]
+mulsu.w00 t1, s0, a4
+# CHECK-ASM-AND-OBJ: pmulsu.h.b11 t3, t1, a4
+# CHECK-ASM: encoding: [0x3b,0x3e,0xe3,0xf0]
+pmulsu.h.b11 t3, t1, a4
+# CHECK-ASM-AND-OBJ: pmulsu.w.h11 t1, t3, s2
+# CHECK-ASM: encoding: [0x3b,0x33,0x2e,0xf3]
+pmulsu.w.h11 t1, t3, s2
+# CHECK-ASM-AND-OBJ: mulsu.w11 a2, s2, a2
+# CHECK-ASM: encoding: [0x3b,0x36,0xc9,0xf6]
+mulsu.w11 a2, s2, a2
+# CHECK-ASM-AND-OBJ: pmaccsu.w.h00 a4, a0, t1
+# CHECK-ASM: encoding: [0x3b,0x37,0x65,0xea]
+pmaccsu.w.h00 a4, a0, t1
+# CHECK-ASM-AND-OBJ: maccsu.w00 a4, s2, s0
+# CHECK-ASM: encoding: [0x3b,0x37,0x89,0xee]
+maccsu.w00 a4, s2, s0
+# CHECK-ASM-AND-OBJ: pmaccsu.w.h11 a0, a2, t3
+# CHECK-ASM: encoding: [0x3b,0x35,0xc6,0xfb]
+pmaccsu.w.h11 a0, a2, t3
+# CHECK-ASM-AND-OBJ: maccsu.w11 t5, a4, s2
+# CHECK-ASM: encoding: [0x3b,0x3f,0x27,0xff]
+maccsu.w11 t5, a4, s2
+# CHECK-ASM-AND-OBJ: ppack.h s0, s0, s2
+# CHECK-ASM: encoding: [0x3b,0x44,0x24,0x81]
+ppack.h s0, s0, s2
+# CHECK-ASM-AND-OBJ: ppack.w t5, a2, a4
+# CHECK-ASM: encoding: [0x3b,0x4f,0xe6,0x82]
+ppack.w t5, a2, a4
+# CHECK-ASM-AND-OBJ: ppackbt.h a4, s2, t3
+# CHECK-ASM: encoding: [0x3b,0x47,0xc9,0x91]
+ppackbt.h a4, s2, t3
+# CHECK-ASM-AND-OBJ: ppackbt.w t5, s0, t5
+# CHECK-ASM: encoding: [0x3b,0x4f,0xe4,0x93]
+ppackbt.w t5, s0, t5
+# CHECK-ASM-AND-OBJ: packbt a0, t5, a0
+# CHECK-ASM: encoding: [0x3b,0x45,0xaf,0x96]
+packbt a0, t5, a0
+# CHECK-ASM-AND-OBJ: ppacktb.h t3, s0, t3
+# CHECK-ASM: encoding: [0x3b,0x4e,0xc4,0xa1]
+ppacktb.h t3, s0, t3
+# CHECK-ASM-AND-OBJ: ppacktb.w t5, t1, t1
+# CHECK-ASM: encoding: [0x3b,0x4f,0x63,0xa2]
+ppacktb.w t5, t1, t1
+# CHECK-ASM-AND-OBJ: packtb t5, a4, a4
+# CHECK-ASM: encoding: [0x3b,0x4f,0xe7,0xa6]
+packtb t5, a4, a4
+# CHECK-ASM-AND-OBJ: ppackt.h a0, t1, t5
+# CHECK-ASM: encoding: [0x3b,0x45,0xe3,0xb1]
+ppackt.h a0, t1, t5
+# CHECK-ASM-AND-OBJ: ppackt.w t3, a0, s2
+# CHECK-ASM: encoding: [0x3b,0x4e,0x25,0xb3]
+ppackt.w t3, a0, s2
+# CHECK-ASM-AND-OBJ: packt a2, t3, t1
+# CHECK-ASM: encoding: [0x3b,0x46,0x6e,0xb6]
+packt a2, t3, t1
+# CHECK-ASM-AND-OBJ: pm2add.h a4, t3, t5
+# CHECK-ASM: encoding: [0x3b,0x57,0xee,0x81]
+pm2add.h a4, t3, t5
+# CHECK-ASM-AND-OBJ: pm4add.b t1, s2, s2
+# CHECK-ASM: encoding: [0x3b,0x53,0x29,0x85]
+pm4add.b t1, s2, s2
+# CHECK-ASM-AND-OBJ: pm2adda.h a0, t5, s0
+# CHECK-ASM: encoding: [0x3b,0x55,0x8f,0x88]
+pm2adda.h a0, t5, s0
+# CHECK-ASM-AND-OBJ: pm4adda.b a0, t5, a2
+# CHECK-ASM: encoding: [0x3b,0x55,0xcf,0x8c]
+pm4adda.b a0, t5, a2
+# CHECK-ASM-AND-OBJ: pm2add.hx a0, s0, t3
+# CHECK-ASM: encoding: [0x3b,0x55,0xc4,0x91]
+pm2add.hx a0, s0, t3
+# CHECK-ASM-AND-OBJ: pm2adda.hx s0, a0, s2
+# CHECK-ASM: encoding: [0x3b,0x54,0x25,0x99]
+pm2adda.hx s0, a0, s2
+# CHECK-ASM-AND-OBJ: pm2addu.h t1, a4, a0
+# CHECK-ASM: encoding: [0x3b,0x53,0xa7,0xa0]
+pm2addu.h t1, a4, a0
+# CHECK-ASM-AND-OBJ: pm4addu.b a0, t1, t3
+# CHECK-ASM: encoding: [0x3b,0x55,0xc3,0xa5]
+pm4addu.b a0, t1, t3
+# CHECK-ASM-AND-OBJ: pm2addau.h s2, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x59,0xa6,0xa8]
+pm2addau.h s2, a2, a0
+# CHECK-ASM-AND-OBJ: pm4addau.b s2, s2, t5
+# CHECK-ASM: encoding: [0x3b,0x59,0xe9,0xad]
+pm4addau.b s2, s2, t5
+# CHECK-ASM-AND-OBJ: pmq2add.h s2, s2, t3
+# CHECK-ASM: encoding: [0x3b,0x59,0xc9,0xb1]
+pmq2add.h s2, s2, t3
+# CHECK-ASM-AND-OBJ: pmqr2add.h a4, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x57,0xa6,0xb4]
+pmqr2add.h a4, a2, a0
+# CHECK-ASM-AND-OBJ: pmq2adda.h a2, s2, t1
+# CHECK-ASM: encoding: [0x3b,0x56,0x69,0xb8]
+pmq2adda.h a2, s2, t1
+# CHECK-ASM-AND-OBJ: pmqr2adda.h a2, a0, a0
+# CHECK-ASM: encoding: [0x3b,0x56,0xa5,0xbc]
+pmqr2adda.h a2, a0, a0
+# CHECK-ASM-AND-OBJ: pm2sub.h t5, s0, s0
+# CHECK-ASM: encoding: [0x3b,0x5f,0x84,0xc0]
+pm2sub.h t5, s0, s0
+# CHECK-ASM-AND-OBJ: pm2sadd.h a0, t5, a0
+# CHECK-ASM: encoding: [0x3b,0x55,0xaf,0xc4]
+pm2sadd.h a0, t5, a0
+# CHECK-ASM-AND-OBJ: pm2suba.h s0, a4, t5
+# CHECK-ASM: encoding: [0x3b,0x54,0xe7,0xc9]
+pm2suba.h s0, a4, t5
+# CHECK-ASM-AND-OBJ: pm2sub.hx t3, t5, a0
+# CHECK-ASM: encoding: [0x3b,0x5e,0xaf,0xd0]
+pm2sub.hx t3, t5, a0
+# CHECK-ASM-AND-OBJ: pm2sadd.hx t1, a0, t3
+# CHECK-ASM: encoding: [0x3b,0x53,0xc5,0xd5]
+pm2sadd.hx t1, a0, t3
+# CHECK-ASM-AND-OBJ: pm2suba.hx t3, a0, a4
+# CHECK-ASM: encoding: [0x3b,0x5e,0xe5,0xd8]
+pm2suba.hx t3, a0, a4
+# CHECK-ASM-AND-OBJ: pm2addsu.h t3, a0, t3
+# CHECK-ASM: encoding: [0x3b,0x5e,0xc5,0xe1]
+pm2addsu.h t3, a0, t3
+# CHECK-ASM-AND-OBJ: pm4addsu.b t1, t5, s0
+# CHECK-ASM: encoding: [0x3b,0x53,0x8f,0xe4]
+pm4addsu.b t1, t5, s0
+# CHECK-ASM-AND-OBJ: pm2addasu.h t1, t3, t3
+# CHECK-ASM: encoding: [0x3b,0x53,0xce,0xe9]
+pm2addasu.h t1, t3, t3
+# CHECK-ASM-AND-OBJ: pm4addasu.b t3, a0, a0
+# CHECK-ASM: encoding: [0x3b,0x5e,0xa5,0xec]
+pm4addasu.b t3, a0, a0
+# CHECK-ASM-AND-OBJ: pm2add.w t3, s0, t5
+# CHECK-ASM: encoding: [0x3b,0x5e,0xe4,0x83]
+pm2add.w t3, s0, t5
+# CHECK-ASM-AND-OBJ: pm4add.h s2, s2, t3
+# CHECK-ASM: encoding: [0x3b,0x59,0xc9,0x87]
+pm4add.h s2, s2, t3
+# CHECK-ASM-AND-OBJ: pm2adda.w a2, a0, s0
+# CHECK-ASM: encoding: [0x3b,0x56,0x85,0x8a]
+pm2adda.w a2, a0, s0
+# CHECK-ASM-AND-OBJ: pm4adda.h s2, s0, t1
+# CHECK-ASM: encoding: [0x3b,0x59,0x64,0x8e]
+pm4adda.h s2, s0, t1
+# CHECK-ASM-AND-OBJ: pm2add.wx t1, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x53,0xe4,0x92]
+pm2add.wx t1, s0, a4
+# CHECK-ASM-AND-OBJ: pm2adda.wx t5, t3, s0
+# CHECK-ASM: encoding: [0x3b,0x5f,0x8e,0x9a]
+pm2adda.wx t5, t3, s0
+# CHECK-ASM-AND-OBJ: pm2addu.w s2, a0, s2
+# CHECK-ASM: encoding: [0x3b,0x59,0x25,0xa3]
+pm2addu.w s2, a0, s2
+# CHECK-ASM-AND-OBJ: pm4addu.h a4, a2, a2
+# CHECK-ASM: encoding: [0x3b,0x57,0xc6,0xa6]
+pm4addu.h a4, a2, a2
+# CHECK-ASM-AND-OBJ: pm2addau.w s0, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x54,0xe4,0xaa]
+pm2addau.w s0, s0, a4
+# CHECK-ASM-AND-OBJ: pm4addau.h a2, a4, t5
+# CHECK-ASM: encoding: [0x3b,0x56,0xe7,0xaf]
+pm4addau.h a2, a4, t5
+# CHECK-ASM-AND-OBJ: pmq2add.w t5, t1, t1
+# CHECK-ASM: encoding: [0x3b,0x5f,0x63,0xb2]
+pmq2add.w t5, t1, t1
+# CHECK-ASM-AND-OBJ: pmqr2add.w s0, t1, a4
+# CHECK-ASM: encoding: [0x3b,0x54,0xe3,0xb6]
+pmqr2add.w s0, t1, a4
+# CHECK-ASM-AND-OBJ: pmq2adda.w a4, s0, t1
+# CHECK-ASM: encoding: [0x3b,0x57,0x64,0xba]
+pmq2adda.w a4, s0, t1
+# CHECK-ASM-AND-OBJ: pmqr2adda.w a4, t5, s0
+# CHECK-ASM: encoding: [0x3b,0x57,0x8f,0xbe]
+pmqr2adda.w a4, t5, s0
+# CHECK-ASM-AND-OBJ: pm2sub.w t3, a2, t1
+# CHECK-ASM: encoding: [0x3b,0x5e,0x66,0xc2]
+pm2sub.w t3, a2, t1
+# CHECK-ASM-AND-OBJ: pm2suba.w t5, t5, t3
+# CHECK-ASM: encoding: [0x3b,0x5f,0xcf,0xcb]
+pm2suba.w t5, t5, t3
+# CHECK-ASM-AND-OBJ: pm2sub.wx t5, s2, s0
+# CHECK-ASM: encoding: [0x3b,0x5f,0x89,0xd2]
+pm2sub.wx t5, s2, s0
+# CHECK-ASM-AND-OBJ: pm2suba.wx a2, a0, a4
+# CHECK-ASM: encoding: [0x3b,0x56,0xe5,0xda]
+pm2suba.wx a2, a0, a4
+# CHECK-ASM-AND-OBJ: pm2addsu.w s0, s0, s2
+# CHECK-ASM: encoding: [0x3b,0x54,0x24,0xe3]
+pm2addsu.w s0, s0, s2
+# CHECK-ASM-AND-OBJ: pm4addsu.h a2, s2, s0
+# CHECK-ASM: encoding: [0x3b,0x56,0x89,0xe6]
+pm4addsu.h a2, s2, s0
+# CHECK-ASM-AND-OBJ: pm2addasu.w a0, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x55,0xa6,0xea]
+pm2addasu.w a0, a2, a0
+# CHECK-ASM-AND-OBJ: pm4addasu.h a0, s0, t5
+# CHECK-ASM: encoding: [0x3b,0x55,0xe4,0xef]
+pm4addasu.h a0, s0, t5
+# CHECK-ASM-AND-OBJ: pmqacc.w.h01 t5, t1, a4
+# CHECK-ASM: encoding: [0x3b,0x5f,0xe3,0xf8]
+pmqacc.w.h01 t5, t1, a4
+# CHECK-ASM-AND-OBJ: mqacc.w01 a0, a2, t3
+# CHECK-ASM: encoding: [0x3b,0x55,0xc6,0xfb]
+mqacc.w01 a0, a2, t3
+# CHECK-ASM-AND-OBJ: pmqracc.w.h01 a4, t1, s2
+# CHECK-ASM: encoding: [0x3b,0x57,0x23,0xfd]
+pmqracc.w.h01 a4, t1, s2
+# CHECK-ASM-AND-OBJ: mqracc.w01 s0, t5, a4
+# CHECK-ASM: encoding: [0x3b,0x54,0xef,0xfe]
+mqracc.w01 s0, t5, a4
+# CHECK-ASM-AND-OBJ: pas.hx a0, t5, t1
+# CHECK-ASM: encoding: [0x3b,0x65,0x6f,0x80]
+pas.hx a0, t5, t1
+# CHECK-ASM-AND-OBJ: psa.hx a2, t3, a0
+# CHECK-ASM: encoding: [0x3b,0x66,0xae,0x84]
+psa.hx a2, t3, a0
+# CHECK-ASM-AND-OBJ: psas.hx s0, a0, a2
+# CHECK-ASM: encoding: [0x3b,0x64,0xc5,0x90]
+psas.hx s0, a0, a2
+# CHECK-ASM-AND-OBJ: pssa.hx a0, t1, t5
+# CHECK-ASM: encoding: [0x3b,0x65,0xe3,0x95]
+pssa.hx a0, t1, t5
+# CHECK-ASM-AND-OBJ: paas.hx t1, a2, s0
+# CHECK-ASM: encoding: [0x3b,0x63,0x86,0x98]
+paas.hx t1, a2, s0
+# CHECK-ASM-AND-OBJ: pasa.hx t1, s2, t3
+# CHECK-ASM: encoding: [0x3b,0x63,0xc9,0x9d]
+pasa.hx t1, s2, t3
+# CHECK-ASM-AND-OBJ: pmseq.h t3, s0, t1
+# CHECK-ASM: encoding: [0x3b,0x6e,0x64,0xc0]
+pmseq.h t3, s0, t1
+# CHECK-ASM-AND-OBJ: pmseq.b t5, s2, a2
+# CHECK-ASM: encoding: [0x3b,0x6f,0xc9,0xc4]
+pmseq.b t5, s2, a2
+# CHECK-ASM-AND-OBJ: pmslt.h t1, a0, a4
+# CHECK-ASM: encoding: [0x3b,0x63,0xe5,0xd0]
+pmslt.h t1, a0, a4
+# CHECK-ASM-AND-OBJ: pmslt.b s2, t3, t1
+# CHECK-ASM: encoding: [0x3b,0x69,0x6e,0xd4]
+pmslt.b s2, t3, t1
+# CHECK-ASM-AND-OBJ: pmsltu.h t1, a0, t5
+# CHECK-ASM: encoding: [0x3b,0x63,0xe5,0xd9]
+pmsltu.h t1, a0, t5
+# CHECK-ASM-AND-OBJ: pmsltu.b t3, a4, s2
+# CHECK-ASM: encoding: [0x3b,0x6e,0x27,0xdd]
+pmsltu.b t3, a4, s2
+# CHECK-ASM-AND-OBJ: pmin.h a2, a2, s2
+# CHECK-ASM: encoding: [0x3b,0x66,0x26,0xe1]
+pmin.h a2, a2, s2
+# CHECK-ASM-AND-OBJ: pmin.b t3, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x6e,0xa6,0xe4]
+pmin.b t3, a2, a0
+# CHECK-ASM-AND-OBJ: pminu.h a2, s2, t1
+# CHECK-ASM: encoding: [0x3b,0x66,0x69,0xe8]
+pminu.h a2, s2, t1
+# CHECK-ASM-AND-OBJ: pminu.b a0, t1, a0
+# CHECK-ASM: encoding: [0x3b,0x65,0xa3,0xec]
+pminu.b a0, t1, a0
+# CHECK-ASM-AND-OBJ: pmax.h a0, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x65,0xe4,0xf0]
+pmax.h a0, s0, a4
+# CHECK-ASM-AND-OBJ: pmax.b t1, t3, a2
+# CHECK-ASM: encoding: [0x3b,0x63,0xce,0xf4]
+pmax.b t1, t3, a2
+# CHECK-ASM-AND-OBJ: pmaxu.h t1, a0, s2
+# CHECK-ASM: encoding: [0x3b,0x63,0x25,0xf9]
+pmaxu.h t1, a0, s2
+# CHECK-ASM-AND-OBJ: pmaxu.b t1, t3, s2
+# CHECK-ASM: encoding: [0x3b,0x63,0x2e,0xfd]
+pmaxu.b t1, t3, s2
+# CHECK-ASM-AND-OBJ: pas.wx t1, a2, s0
+# CHECK-ASM: encoding: [0x3b,0x63,0x86,0x82]
+pas.wx t1, a2, s0
+# CHECK-ASM-AND-OBJ: psa.wx t3, t3, a2
+# CHECK-ASM: encoding: [0x3b,0x6e,0xce,0x86]
+psa.wx t3, t3, a2
+# CHECK-ASM-AND-OBJ: psas.wx a4, a4, s2
+# CHECK-ASM: encoding: [0x3b,0x67,0x27,0x93]
+psas.wx a4, a4, s2
+# CHECK-ASM-AND-OBJ: pssa.wx a0, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x65,0xa6,0x96]
+pssa.wx a0, a2, a0
+# CHECK-ASM-AND-OBJ: paas.wx s0, a4, t1
+# CHECK-ASM: encoding: [0x3b,0x64,0x67,0x9a]
+paas.wx s0, a4, t1
+# CHECK-ASM-AND-OBJ: pasa.wx t1, s2, s2
+# CHECK-ASM: encoding: [0x3b,0x63,0x29,0x9f]
+pasa.wx t1, s2, s2
+# CHECK-ASM-AND-OBJ: pmseq.w t1, a4, s2
+# CHECK-ASM: encoding: [0x3b,0x63,0x27,0xc3]
+pmseq.w t1, a4, s2
+# CHECK-ASM-AND-OBJ: pmslt.w t5, t5, t3
+# CHECK-ASM: encoding: [0x3b,0x6f,0xcf,0xd3]
+pmslt.w t5, t5, t3
+# CHECK-ASM-AND-OBJ: pmsltu.w s2, a4, t1
+# CHECK-ASM: encoding: [0x3b,0x69,0x67,0xda]
+pmsltu.w s2, a4, t1
+# CHECK-ASM-AND-OBJ: pmin.w t5, a4, t5
+# CHECK-ASM: encoding: [0x3b,0x6f,0xe7,0xe3]
+pmin.w t5, a4, t5
+# CHECK-ASM-AND-OBJ: pminu.w a0, a2, a0
+# CHECK-ASM: encoding: [0x3b,0x65,0xa6,0xea]
+pminu.w a0, a2, a0
+# CHECK-ASM-AND-OBJ: pmax.w a0, s2, t1
+# CHECK-ASM: encoding: [0x3b,0x65,0x69,0xf2]
+pmax.w a0, s2, t1
+# CHECK-ASM-AND-OBJ: pmaxu.w a0, a4, a4
+# CHECK-ASM: encoding: [0x3b,0x65,0xe7,0xfa]
+pmaxu.w a0, a4, a4
+# CHECK-ASM-AND-OBJ: pmulh.h a0, t5, t1
+# CHECK-ASM: encoding: [0x3b,0x75,0x6f,0x80]
+pmulh.h a0, t5, t1
+# CHECK-ASM-AND-OBJ: pmulhr.h s2, t5, t3
+# CHECK-ASM: encoding: [0x3b,0x79,0xcf,0x85]
+pmulhr.h s2, t5, t3
+# CHECK-ASM-AND-OBJ: pmhacc.h t5, t3, a4
+# CHECK-ASM: encoding: [0x3b,0x7f,0xee,0x88]
+pmhacc.h t5, t3, a4
+# CHECK-ASM-AND-OBJ: pmhracc.h s2, t5, s2
+# CHECK-ASM: encoding: [0x3b,0x79,0x2f,0x8d]
+pmhracc.h s2, t5, s2
+# CHECK-ASM-AND-OBJ: pmulhu.h t3, t1, t1
+# CHECK-ASM: encoding: [0x3b,0x7e,0x63,0x90]
+pmulhu.h t3, t1, t1
+# CHECK-ASM-AND-OBJ: pmulhru.h s0, t5, t1
+# CHECK-ASM: encoding: [0x3b,0x74,0x6f,0x94]
+pmulhru.h s0, t5, t1
+# CHECK-ASM-AND-OBJ: pmhaccu.h t3, a0, t3
+# CHECK-ASM: encoding: [0x3b,0x7e,0xc5,0x99]
+pmhaccu.h t3, a0, t3
+# CHECK-ASM-AND-OBJ: pmhraccu.h t5, t3, a2
+# CHECK-ASM: encoding: [0x3b,0x7f,0xce,0x9c]
+pmhraccu.h t5, t3, a2
+# CHECK-ASM-AND-OBJ: pmulh.h.b0 t1, a0, a0
+# CHECK-ASM: encoding: [0x3b,0x73,0xa5,0xa0]
+pmulh.h.b0 t1, a0, a0
+# CHECK-ASM-AND-OBJ: pmulhsu.h.b0 t3, s0, a4
+# CHECK-ASM: encoding: [0x3b,0x7e,0xe4,0xa4]
+pmulhsu.h.b0 t3, s0, a4
+# CHECK-ASM-AND-OBJ: pmhacc.h.b0 t1, a0, a4
+# CHECK-ASM: encoding: [0x3b,0x73,0xe5,0xa8]
+pmhacc.h.b0 t1, a0, a4
+# CHECK-ASM-AND-OBJ: pmhaccsu.h.b0 s2, t5, t3
+# CHECK-ASM: encoding: [0x3b,0x79,0xcf,0xad]
+pmhaccsu.h.b0 s2, t5, t3
+# CHECK-ASM-AND-OBJ: pmulh.h.b1 a0, s0, a2
+# CHECK-ASM: encoding: [0x3b,0x75,0xc4,0xb0]
+pmulh.h.b1 a0, s0, a2
+# CHECK-ASM-AND-OBJ: pmulhsu.h.b1 t1, t3, t3
+# CHECK-ASM: encoding: [0x3b,0x73,0xce,0xb5]
+pmulhsu.h.b1 t1, t3, t3
+# CHECK-ASM-AND-OBJ: pmhacc.h.b1 t3, t5, s2
+# CHECK-ASM: encoding: [0x3b,0x7e,0x2f,0xb9]
+pmhacc.h.b1 t3, t5, s2
+# CHECK-ASM-AND-OBJ: pmhaccsu.h.b1 t5, t5, t1
+# CHECK-ASM: encoding: [0x3b,0x7f,0x6f,0xbc]
+pmhaccsu.h.b1 t5, t5, t1
+# CHECK-ASM-AND-OBJ: pmulhsu.h s2, t3, a4
+# CHECK-ASM: encoding: [0x3b,0x79,0xee,0xc0]
+pmulhsu.h s2, t3, a4
+# CHECK-ASM-AND-OBJ: pmulhrsu.h a0, a0, t5
+# CHECK-ASM: encoding: [0x3b,0x75,0xe5,0xc5]
+pmulhrsu.h a0, a0, t5
+# CHECK-ASM-AND-OBJ: pmhaccsu.h s0, t3, t1
+# CHECK-ASM: encoding: [0x3b,0x74,0x6e,0xc8]
+pmhaccsu.h s0, t3, t1
+# CHECK-ASM-AND-OBJ: pmhraccsu.h s0, t5, a4
+# CHECK-ASM: encoding: [0x3b,0x74,0xef,0xcc]
+pmhraccsu.h s0, t5, a4
+# CHECK-ASM-AND-OBJ: pmulq.h t3, t1, s0
+# CHECK-ASM: encoding: [0x3b,0x7e,0x83,0xd0]
+pmulq.h t3, t1, s0
+# CHECK-ASM-AND-OBJ: pmulqr.h t1, s2, s0
+# CHECK-ASM: encoding: [0x3b,0x73,0x89,0xd4]
+pmulqr.h t1, s2, s0
+# CHECK-ASM-AND-OBJ: pmulh.w t5, a4, a4
+# CHECK-ASM: encoding: [0x3b,0x7f,0xe7,0x82]
+pmulh.w t5, a4, a4
+# CHECK-ASM-AND-OBJ: pmulhr.w t1, t5, t1
+# CHECK-ASM: encoding: [0x3b,0x73,0x6f,0x86]
+pmulhr.w t1, t5, t1
+# CHECK-ASM-AND-OBJ: pmhacc.w t5, s0, t5
+# CHECK-ASM: encoding: [0x3b,0x7f,0xe4,0x8b]
+pmhacc.w t5, s0, t5
+# CHECK-ASM-AND-OBJ: pmhracc.w s0, s2, t5
+# CHECK-ASM: encoding: [0x3b,0x74,0xe9,0x8f]
+pmhracc.w s0, s2, t5
+# CHECK-ASM-AND-OBJ: pmulhu.w a2, a0, a4
+# CHECK-ASM: encoding: [0x3b,0x76,0xe5,0x92]
+pmulhu.w a2, a0, a4
+# CHECK-ASM-AND-OBJ: pmulhru.w t1, t1, a4
+# CHECK-ASM: encoding: [0x3b,0x73,0xe3,0x96]
+pmulhru.w t1, t1, a4
+# CHECK-ASM-AND-OBJ: pmhaccu.w a0, s0, a0
+# CHECK-ASM: encoding: [0x3b,0x75,0xa4,0x9a]
+pmhaccu.w a0, s0, a0
+# CHECK-ASM-AND-OBJ: pmhraccu.w s2, s0, t3
+# CHECK-ASM: encoding: [0x3b,0x79,0xc4,0x9f]
+pmhraccu.w s2, s0, t3
+# CHECK-ASM-AND-OBJ: pmulh.w.h0 t5, s0, t5
+# CHECK-ASM: encoding: [0x3b,0x7f,0xe4,0xa3]
+pmulh.w.h0 t5, s0, t5
+# CHECK-ASM-AND-OBJ: pmulhsu.w.h0 a2, t3, a2
+# CHECK-ASM: encoding: [0x3b,0x76,0xce,0xa6]
+pmulhsu.w.h0 a2, t3, a2
+# CHECK-ASM-AND-OBJ: pmhacc.w.h0 a2, a0, t1
+# CHECK-ASM: encoding: [0x3b,0x76,0x65,0xaa]
+pmhacc.w.h0 a2, a0, t1
+# CHECK-ASM-AND-OBJ: pmhaccsu.w.h0 t1, a4, t1
+# CHECK-ASM: encoding: [0x3b,0x73,0x67,0xae]
+pmhaccsu.w.h0 t1, a4, t1
+# CHECK-ASM-AND-OBJ: pmulh.w.h1 t1, a0, t3
+# CHECK-ASM: encoding: [0x3b,0x73,0xc5,0xb3]
+pmulh.w.h1 t1, a0, t3
+# CHECK-ASM-AND-OBJ: pmulhsu.w.h1 s2, t3, a4
+# CHECK-ASM: encoding: [0x3b,0x79,0xee,0xb6]
+pmulhsu.w.h1 s2, t3, a4
+# CHECK-ASM-AND-OBJ: pmhacc.w.h1 s0, t5, a2
+# CHECK-ASM: encoding: [0x3b,0x74,0xcf,0xba]
+pmhacc.w.h1 s0, t5, a2
+# CHECK-ASM-AND-OBJ: pmhaccsu.w.h1 a0, a0, a0
+# CHECK-ASM: encoding: [0x3b,0x75,0xa5,0xbe]
+pmhaccsu.w.h1 a0, a0, a0
+# CHECK-ASM-AND-OBJ: pmulhsu.w t3, a2, a4
+# CHECK-ASM: encoding: [0x3b,0x7e,0xe6,0xc2]
+pmulhsu.w t3, a2, a4
+# CHECK-ASM-AND-OBJ: pmulhrsu.w t5, a0, a0
+# CHECK-ASM: encoding: [0x3b,0x7f,0xa5,0xc6]
+pmulhrsu.w t5, a0, a0
+# CHECK-ASM-AND-OBJ: pmhaccsu.w a4, a0, a0
+# CHECK-ASM: encoding: [0x3b,0x77,0xa5,0xca]
+pmhaccsu.w a4, a0, a0
+# CHECK-ASM-AND-OBJ: pmhraccsu.w t5, t1, t3
+# CHECK-ASM: encoding: [0x3b,0x7f,0xc3,0xcf]
+pmhraccsu.w t5, t1, t3
+# CHECK-ASM-AND-OBJ: pmulq.w a2, a2, t5
+# CHECK-ASM: encoding: [0x3b,0x76,0xe6,0xd3]
+pmulq.w a2, a2, t5
+# CHECK-ASM-AND-OBJ: pmulqr.w a0, t3, t5
+# CHECK-ASM: encoding: [0x3b,0x75,0xee,0xd7]
+pmulqr.w a0, t3, t5
+# CHECK-ASM-AND-OBJ: pmqacc.w.h00 t5, a4, t1
+# CHECK-ASM: encoding: [0x3b,0x7f,0x67,0xe8]
+pmqacc.w.h00 t5, a4, t1
+# CHECK-ASM-AND-OBJ: mqacc.w00 t1, t1, a0
+# CHECK-ASM: encoding: [0x3b,0x73,0xa3,0xea]
+mqacc.w00 t1, t1, a0
+# CHECK-ASM-AND-OBJ: pmqracc.w.h00 t1, a2, t5
+# CHECK-ASM: encoding: [0x3b,0x73,0xe6,0xed]
+pmqracc.w.h00 t1, a2, t5
+# CHECK-ASM-AND-OBJ: mqracc.w00 s2, s2, a2
+# CHECK-ASM: encoding: [0x3b,0x79,0xc9,0xee]
+mqracc.w00 s2, s2, a2
+# CHECK-ASM-AND-OBJ: pmqacc.w.h11 a2, a0, a0
+# CHECK-ASM: encoding: [0x3b,0x76,0xa5,0xf8]
+pmqacc.w.h11 a2, a0, a0
+# CHECK-ASM-AND-OBJ: mqacc.w11 a4, a2, a2
+# CHECK-ASM: encoding: [0x3b,0x77,0xc6,0xfa]
+mqacc.w11 a4, a2, a2
+# CHECK-ASM-AND-OBJ: pmqracc.w.h11 s0, t1, t3
+# CHECK-ASM: encoding: [0x3b,0x74,0xc3,0xfd]
+pmqracc.w.h11 s0, t1, t3
+# CHECK-ASM-AND-OBJ: mqracc.w11 s2, t1, a4
+# CHECK-ASM: encoding: [0x3b,0x79,0xe3,0xfe]
+mqracc.w11 s2, t1, a4
diff --git a/llvm/test/MC/RISCV/rvv/vsetvl-invalid.s b/llvm/test/MC/RISCV/rvv/vsetvl-invalid.s
index b45f3f2dfc854..d97b53803b97a 100644
--- a/llvm/test/MC/RISCV/rvv/vsetvl-invalid.s
+++ b/llvm/test/MC/RISCV/rvv/vsetvl-invalid.s
@@ -4,37 +4,37 @@
 # RUN:     | llvm-objdump -d --mattr=+v - | FileCheck %s
 
 # CHECK: vsetvli a1, a0, e64, m1, tu, mu
-.word 0x018575d7
+.insn 4, 0x018575d7
 
 # CHECK: vsetvli a1, a0, 0x1c
-.word 0x01c575d7
+.insn 4, 0x01c575d7
 
 # CHECK: vsetvli a1, a0, 0x24
-.word 0x024575d7
+.insn 4, 0x024575d7
 
 # CHECK: vsetvli a1, a0, 0x29
-.word 0x029575d7
+.insn 4, 0x029575d7
 
 # CHECK: vsetvli a1, a0, 0x110
-.word 0x110575d7
+.insn 4, 0x110575d7
 
 # CHECK: vsetvli a1, a0, e64, mf8, tu, mu
-.word 0x01d575d7
+.insn 4, 0x01d575d7
 
 # CHECK: vsetivli a1, 0x10, e8, m4, tu, mu
-.word 0xc02875d7
+.insn 4, 0xc02875d7
 
 # CHECK: vsetivli a1, 0x10, 0xc
-.word 0xc0c875d7
+.insn 4, 0xc0c875d7
 
 # CHECK: vsetivli a1, 0x10, 0x14
-.word 0xc14875d7
+.insn 4, 0xc14875d7
 
 # CHECK: vsetivli a1, 0x10, 0x38
-.word 0xc38875d7
+.insn 4, 0xc38875d7
 
 # CHECK: vsetivli a1, 0x10, 0x103
-.word 0xd03875d7
+.insn 4, 0xd03875d7
 
 # CHECK: vsetivli a1, 0x10, e8, mf4, tu, mu
-.word 0xc06875d7
+.insn 4, 0xc06875d7
diff --git a/llvm/test/MC/RISCV/smctr-ssctr-valid.s b/llvm/test/MC/RISCV/smctr-ssctr-valid.s
index 8bbd5a426b8ee..072231a9b546e 100644
--- a/llvm/test/MC/RISCV/smctr-ssctr-valid.s
+++ b/llvm/test/MC/RISCV/smctr-ssctr-valid.s
@@ -1,22 +1,22 @@
-# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-smctr -M no-aliases -show-encoding \
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+smctr -M no-aliases -show-encoding \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
-# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-smctr -M no-aliases -show-encoding \
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+smctr -M no-aliases -show-encoding \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
-# RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-ssctr -M no-aliases -show-encoding \
+# RUN: llvm-mc %s -triple=riscv32 -mattr=+ssctr -M no-aliases -show-encoding \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
-# RUN: llvm-mc %s -triple=riscv64 -mattr=+experimental-ssctr -M no-aliases -show-encoding \
+# RUN: llvm-mc %s -triple=riscv64 -mattr=+ssctr -M no-aliases -show-encoding \
 # RUN:     | FileCheck -check-prefixes=CHECK,CHECK-INST %s
-# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-smctr < %s \
-# RUN:     | llvm-objdump --mattr=+experimental-smctr -M no-aliases -d - \
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+smctr < %s \
+# RUN:     | llvm-objdump --mattr=+smctr -M no-aliases -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
-# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+experimental-smctr < %s \
-# RUN:     | llvm-objdump --mattr=+experimental-smctr -M no-aliases -d - \
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+smctr < %s \
+# RUN:     | llvm-objdump --mattr=+smctr -M no-aliases -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
-# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-ssctr < %s \
-# RUN:     | llvm-objdump --mattr=+experimental-ssctr -M no-aliases -d - \
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+ssctr < %s \
+# RUN:     | llvm-objdump --mattr=+ssctr -M no-aliases -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
-# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+experimental-ssctr < %s \
-# RUN:     | llvm-objdump --mattr=+experimental-ssctr -M no-aliases -d - \
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+ssctr < %s \
+# RUN:     | llvm-objdump --mattr=+ssctr -M no-aliases -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 
 # RUN: not llvm-mc -triple riscv32 -M no-aliases -show-encoding < %s 2>&1 \
diff --git a/llvm/test/MC/RISCV/xqcibi-linker-relaxation.s b/llvm/test/MC/RISCV/xqcibi-linker-relaxation.s
new file mode 100644
index 0000000000000..2066b55b41ad3
--- /dev/null
+++ b/llvm/test/MC/RISCV/xqcibi-linker-relaxation.s
@@ -0,0 +1,84 @@
+# RUN: llvm-mc --triple=riscv32 -mattr=+relax,+experimental-xqcilb,+experimental-xqcibi \
+# RUN:    %s -filetype=obj -o - -riscv-add-build-attributes \
+# RUN:    | llvm-objdump -dr -M no-aliases - \
+# RUN:    | FileCheck %s
+
+## This tests that we correctly emit relocations for linker relaxation when
+## relaxing `JAL` to `QC.E.JAL`.
+
+## PR150071
+
+.global foo
+
+# CHECK-LABEL: <branch_over_relaxable>:
+branch_over_relaxable:
+  jal x1, foo
+# CHECK: qc.e.jal 0x0 <branch_over_relaxable>
+# CHECK-NEXT: R_RISCV_VENDOR QUALCOMM
+# CHECK-NEXT: R_RISCV_CUSTOM195 foo
+# CHECK-NEXT: R_RISCV_RELAX *ABS*
+  bne a0, a1, branch_over_relaxable
+# CHECK-NEXT: bne a0, a1, 0x6 <branch_over_relaxable+0x6>
+# CHECK-NEXT: R_RISCV_BRANCH branch_over_relaxable
+# CHECK-NOT: R_RISCV_RELAX
+  qc.e.bnei a0, 0x21, branch_over_relaxable
+# CHECK-NEXT: qc.e.bnei a0, 0x21, 0xa <branch_over_relaxable+0xa>
+# CHECK-NEXT: R_RISCV_VENDOR QUALCOMM
+# CHECK-NEXT: R_RISCV_CUSTOM193 branch_over_relaxable
+# CHECK-NOT: R_RISCV_RELAX
+  ret
+# CHECK-NEXT: c.jr ra
+
+# CHECK-LABEL: <short_jump_over_fixed>:
+short_jump_over_fixed:
+  nop
+# CHECK: c.nop
+  j short_jump_over_fixed
+# CHECK-NEXT: c.j 0x12 <short_jump_over_fixed>
+# CHECK-NOT: R_RISCV_RVC_JUMP
+# CHECK-NOT: R_RISCV_RELAX
+  ret
+# CHECK-NEXT: c.jr ra
+
+# CHECK-LABEL: <short_jump_over_relaxable>:
+short_jump_over_relaxable:
+  call foo
+# CHECK: auipc ra, 0x0
+# CHECK-NEXT: R_RISCV_CALL_PLT foo
+# CHECK-NEXT: R_RISCV_RELAX *ABS*
+# CHECK-NEXT: jalr ra, 0x0(ra) <short_jump_over_relaxable>
+  j short_jump_over_relaxable
+# CHECK-NEXT: c.j 0x20 <short_jump_over_relaxable+0x8>
+# CHECK-NEXT: R_RISCV_RVC_JUMP short_jump_over_relaxable
+# CHECK-NOT: R_RISCV_RELAX
+  ret
+# CHECK-NEXT: c.jr ra
+
+# CHECK-LABEL: <mid_jump_over_fixed>:
+mid_jump_over_fixed:
+  nop
+# CHECK: c.nop
+  .space 0x1000
+# CHECK-NEXT: ...
+  j mid_jump_over_fixed
+# CHECK-NEXT: jal zero, 0x24 <mid_jump_over_fixed>
+# CHECK-NOT: R_RISCV_JAL
+# CHECK-NOT: R_RISCV_RELAX
+  ret
+# CHECK-NEXT: c.jr ra
+
+# CHECK-LABEL: <mid_jump_over_relaxable>:
+mid_jump_over_relaxable:
+  call foo
+# CHECK: auipc ra, 0x0
+# CHECK-NEXT: R_RISCV_CALL_PLT foo
+# CHECK-NEXT: R_RISCV_RELAX *ABS*
+# CHECK-NEXT: jalr ra, 0x0(ra) <mid_jump_over_relaxable>
+  .space 0x1000
+# CHECK-NEXT: ...
+  j mid_jump_over_relaxable
+# CHECK-NEXT: jal zero, 0x2034 <mid_jump_over_relaxable+0x1008>
+# CHECK-NEXT: R_RISCV_JAL mid_jump_over_relaxable
+# CHECK-NOT: R_RISCV_RELAX
+  ret
+# CHECK-NEXT: c.jr ra
diff --git a/llvm/test/MC/RISCV/xsmtvdot-invalid.s b/llvm/test/MC/RISCV/xsmtvdot-invalid.s
new file mode 100644
index 0000000000000..e4c48c5702888
--- /dev/null
+++ b/llvm/test/MC/RISCV/xsmtvdot-invalid.s
@@ -0,0 +1,52 @@
+# RUN: not llvm-mc -triple riscv32 -mattr=+xsmtvdot < %s 2>&1 \
+# RUN:     | FileCheck %s
+# RUN: not llvm-mc -triple riscv64 -mattr=+xsmtvdot < %s 2>&1 \
+# RUN:     | FileCheck %s
+
+# NoSlide
+smt.vmadot   v1, v2, v2 # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
+smt.vmadotu  v1, v2, v2 # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
+smt.vmadotsu v1, v2, v2 # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
+smt.vmadotus v1, v2, v2 # CHECK: :[[@LINE]]:14: error: invalid operand for instruction
+
+# Slide = 1
+smt.vmadot1   v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot1u  v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot1su v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot1us v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot1   v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot1u  v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot1su v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot1us v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot1   v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot1u  v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot1su v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot1us v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+
+# Slide = 2
+smt.vmadot2   v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot2u  v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot2su v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot2us v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot2   v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot2u  v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot2su v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot2us v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot2   v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot2u  v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot2su v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot2us v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+
+# Slide = 3
+smt.vmadot3   v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot3u  v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot3su v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot3us v1, v2, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot3   v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot3u  v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot3su v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot3us v2, v1, v2 # CHECK: :[[@LINE]]:19: error: invalid operand for instruction
+smt.vmadot3   v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot3u  v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot3su v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
+smt.vmadot3us v1, v3, v2 # CHECK: :[[@LINE]]:15: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/xsmtvdot-valid.s b/llvm/test/MC/RISCV/xsmtvdot-valid.s
new file mode 100644
index 0000000000000..18df55d21d93a
--- /dev/null
+++ b/llvm/test/MC/RISCV/xsmtvdot-valid.s
@@ -0,0 +1,114 @@
+# RUN: llvm-mc -triple=riscv32 -show-encoding --mattr=+xsmtvdot %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -show-encoding --mattr=+xsmtvdot %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+# RUN: not llvm-mc -triple=riscv32 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: not llvm-mc -triple=riscv64 -show-encoding %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+xsmtvdot %s \
+# RUN:        | llvm-objdump -d  --mattr=+xsmtvdot - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+xsmtvdot %s \
+# RUN:        | llvm-objdump -d  --mattr=+xsmtvdot - \
+# RUN:        | FileCheck %s --check-prefix=CHECK-INST
+# RUN: llvm-mc -triple=riscv32 -filetype=obj --mattr=+xsmtvdot %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+# RUN: llvm-mc -triple=riscv64 -filetype=obj --mattr=+xsmtvdot %s \
+# RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+# CHECK-INST: smt.vmadot  v16, v0, v8
+# CHECK-ENCODING: [0x2b,0x38,0x80,0xe2]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e280382b <unknown>
+smt.vmadot   v16, v0, v8
+
+# CHECK-INST: smt.vmadotu v18, v1, v9
+# CHECK-ENCODING: [0x2b,0x89,0x90,0xe2]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e290892b <unknown>
+smt.vmadotu  v18, v1, v9
+
+# CHECK-INST: smt.vmadotsu        v20, v2, v10
+# CHECK-ENCODING: [0x2b,0x2a,0xa1,0xe2]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e2a12a2b <unknown>
+smt.vmadotsu v20, v2, v10
+
+# CHECK-INST: smt.vmadotus        v22, v3, v11
+# CHECK-ENCODING: [0x2b,0x9b,0xb1,0xe2]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e2b19b2b <unknown>
+smt.vmadotus v22, v3, v11
+
+# CHECK-INST: smt.vmadot1 v24, v16, v12
+# CHECK-ENCODING: [0x2b,0x3c,0xc8,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e6c83c2b <unknown>
+smt.vmadot1   v24, v16, v12
+
+# CHECK-INST: smt.vmadot1u        v26, v18, v13
+# CHECK-ENCODING: [0x2b,0x0d,0xd9,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e6d90d2b <unknown>
+smt.vmadot1u  v26, v18, v13
+
+# CHECK-INST: smt.vmadot1su       v28, v20, v14
+# CHECK-ENCODING: [0x2b,0x2e,0xea,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e6ea2e2b <unknown>
+smt.vmadot1su v28, v20, v14
+
+# CHECK-INST: smt.vmadot1us       v30, v22, v15
+# CHECK-ENCODING: [0x2b,0x1f,0xfb,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e6fb1f2b <unknown>
+smt.vmadot1us v30, v22, v15
+
+# CHECK-INST: smt.vmadot2 v0, v24, v4
+# CHECK-ENCODING: [0x2b,0x70,0x4c,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e64c702b <unknown>
+smt.vmadot2   v0, v24, v4
+
+# CHECK-INST: smt.vmadot2u        v2, v26, v5
+# CHECK-ENCODING: [0x2b,0x41,0x5d,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e65d412b <unknown>
+smt.vmadot2u  v2, v26, v5
+
+# CHECK-INST: smt.vmadot2su       v4, v28, v6
+# CHECK-ENCODING: [0x2b,0x62,0x6e,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e66e622b <unknown>
+smt.vmadot2su v4, v28, v6
+
+# CHECK-INST: smt.vmadot2us       v6, v30, v7
+# CHECK-ENCODING: [0x2b,0x53,0x7f,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e67f532b <unknown>
+smt.vmadot2us v6, v30, v7
+
+# CHECK-INST: smt.vmadot3 v8, v0, v8
+# CHECK-ENCODING: [0x2b,0xb4,0x80,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e680b42b <unknown>
+smt.vmadot3   v8, v0, v8
+
+# CHECK-INST: smt.vmadot3u        v10, v2, v9
+# CHECK-ENCODING: [0x2b,0x85,0x91,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e691852b <unknown>
+smt.vmadot3u  v10, v2, v9
+
+# CHECK-INST: smt.vmadot3su       v12, v4, v10
+# CHECK-ENCODING: [0x2b,0xa6,0xa2,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e6a2a62b <unknown>
+smt.vmadot3su v12, v4, v10
+
+# CHECK-INST: smt.vmadot3us       v14, v6, v11
+# CHECK-ENCODING: [0x2b,0x97,0xb3,0xe6]
+# CHECK-ERROR: instruction requires the following: 'XSMTVDot' (SpacemiT Vector Dot Product Extension){{$}}
+# CHECK-UNKNOWN: e6b3972b <unknown>
+smt.vmadot3us v14, v6, v11
diff --git a/llvm/test/MC/WebAssembly/reloc-directive.s b/llvm/test/MC/WebAssembly/reloc-directive.s
new file mode 100644
index 0000000000000..bc234b928839a
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/reloc-directive.s
@@ -0,0 +1,28 @@
+# RUN: llvm-mc -triple=wasm32 %s | FileCheck --check-prefix=PRINT %s
+# RUN: llvm-mc -filetype=obj -triple=wasm32 %s | llvm-readobj -r - | FileCheck %s
+
+get_addr_func:
+  .functype get_addr_func () -> (i32)
+  i32.const 0
+  nop # 4 NOPs in addition to one zero in i32.const 0 for a canonical 5 byte relocatable [S]LEB.
+  nop
+  nop
+  nop
+  end_function
+
+# PRINT: .reloc get_addr_func+2, R_WASM_MEMORY_ADDR_SLEB, function_index_data+1
+# CHECK:      Section ({{.*}}) CODE {
+# CHECK-NEXT:   0x4 R_WASM_MEMORY_ADDR_SLEB function_index_data 1
+# CHECK-NEXT: }
+.reloc get_addr_func + 2, R_WASM_MEMORY_ADDR_SLEB, function_index_data + 1
+
+.section .data,"",@
+function_index_data:
+  .int32 0
+.size function_index_data, 4
+
+# PRINT: .reloc function_index_data, R_WASM_FUNCTION_INDEX_I32, get_addr_func
+# CHECK:      Section ({{.*}}) DATA {
+# CHECK-NEXT:   0x6 R_WASM_FUNCTION_INDEX_I32 get_addr_func
+# CHECK-NEXT: }
+.reloc function_index_data, R_WASM_FUNCTION_INDEX_I32, get_addr_func
diff --git a/llvm/test/MachineVerifier/AMDGPU/av_mov_b64_imm_pseudo.mir b/llvm/test/MachineVerifier/AMDGPU/av_mov_b64_imm_pseudo.mir
new file mode 100644
index 0000000000000..be3cf3de24d51
--- /dev/null
+++ b/llvm/test/MachineVerifier/AMDGPU/av_mov_b64_imm_pseudo.mir
@@ -0,0 +1,25 @@
+# RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 -verify-machineinstrs -run-pass=none -filetype=null %s 2>&1 | FileCheck %s
+
+# FIXME: Value error messages are misleading.
+---
+name: test
+body: |
+  bb.0:
+    %0:av_64 = IMPLICIT_DEF
+
+    ; CHECK: *** Bad machine code: Expected immediate, but got non-immediate ***
+    %1:av_64 = AV_MOV_B64_IMM_PSEUDO %1, implicit $exec
+
+    ; Low half isn't inline imm
+    ; CHECK: *** Bad machine code: VOP3 instruction uses literal ***
+    %2:av_64 = AV_MOV_B64_IMM_PSEUDO 65, implicit $exec
+
+    ; High half isn't inline imm
+    ; CHECK: *** Bad machine code: VOP3 instruction uses literal ***
+    %2:av_64 = AV_MOV_B64_IMM_PSEUDO 279172874240, implicit $exec
+
+    ; Neither half is inline imm
+    ; CHECK: *** Bad machine code: VOP3 instruction uses literal ***
+    %3:av_64 = AV_MOV_B64_IMM_PSEUDO 279172874306, implicit $exec
+
+...
diff --git a/llvm/test/MachineVerifier/AMDGPU/unsupported-unaligned-vgpr-check-vsrc-operand.mir b/llvm/test/MachineVerifier/AMDGPU/unsupported-unaligned-vgpr-check-vsrc-operand.mir
new file mode 100644
index 0000000000000..b4652f2b519e7
--- /dev/null
+++ b/llvm/test/MachineVerifier/AMDGPU/unsupported-unaligned-vgpr-check-vsrc-operand.mir
@@ -0,0 +1,34 @@
+# RUN: not --crash llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -run-pass=none -filetype=null %s 2>&1 | FileCheck -implicit-check-not="Bad machine code" %s
+
+# 64-bit vsrc operands were not correctly diagnosed with unaligned registers.
+
+---
+name: uses_unaligned_physreg
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr1_vgpr2
+
+    ; CHECK: *** Bad machine code: Subtarget requires even aligned vector registers ***
+    ; CHECK: - instruction: $vcc = V_CMP_NE_U64_e64 0, $vgpr1_vgpr2, implicit $exec
+
+    $vcc = V_CMP_NE_U64_e64 0, $vgpr1_vgpr2, implicit $exec
+
+    ; CHECK: *** Bad machine code: Subtarget requires even aligned vector registers ***
+    ; CHECK: *** Bad machine code: Illegal physical register for instruction ***
+    ; CHECK: - instruction: V_CMP_NE_U64_e32 0, $vgpr1_vgpr2, implicit-def $vcc, implicit $exec
+    V_CMP_NE_U64_e32 0, $vgpr1_vgpr2, implicit-def $vcc, implicit $exec
+
+    %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %2:vreg_64 = IMPLICIT_DEF
+
+    ; CHECK: *** Bad machine code: Subtarget requires even aligned vector registers ***
+    ; CHECK: *** Bad machine code: Illegal virtual register for instruction ***
+    %3:areg_128_align2 = V_MFMA_F32_4X4X1F32_e64 %0, %1, %2, 0, 0, 0, implicit $mode, implicit $exec
+    %4:vreg_64 = IMPLICIT_DEF
+
+    ; CHECK: *** Bad machine code: Subtarget requires even aligned vector registers ***
+    ; CHECK: *** Bad machine code: Illegal virtual register for instruction ***
+    %5:vreg_128_align2 = V_MFMA_F32_4X4X1F32_vgprcd_e64 %0, %1, %4, 0, 0, 0, implicit $mode, implicit $exec
+...
diff --git a/llvm/test/MachineVerifier/verify-inlineasmbr.mir b/llvm/test/MachineVerifier/verify-inlineasmbr.mir
index d03582e16881c..aaf3f97fcdee0 100644
--- a/llvm/test/MachineVerifier/verify-inlineasmbr.mir
+++ b/llvm/test/MachineVerifier/verify-inlineasmbr.mir
@@ -107,8 +107,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:
   - { id: 0, name: skip.i.i, type: default, offset: 0, size: 1, alignment: 4,
diff --git a/llvm/test/Other/debugcounter-predicateinfo.ll b/llvm/test/Other/debugcounter-predicateinfo.ll
index 981bd1514f6e9..4f9440875e21a 100644
--- a/llvm/test/Other/debugcounter-predicateinfo.ll
+++ b/llvm/test/Other/debugcounter-predicateinfo.ll
@@ -8,10 +8,10 @@ define fastcc void @barney() {
 ; CHECK-NEXT:    br label [[BB22:%.*]]
 ; CHECK:       bb22:
 ; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i32 undef, 2
-; CHECK:         [[TMP23_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP23]])
+; CHECK:         [[TMP23_0:%.*]] = bitcast i1 [[TMP23]] to i1
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[BB29:%.*]], label [[BB35:%.*]]
 ; CHECK:       bb29:
-; CHECK:         [[TMP23_0_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP23_0]])
+; CHECK:         [[TMP23_0_1:%.*]] = bitcast i1 [[TMP23_0]] to i1
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[BB33:%.*]], label [[BB35]]
 ; CHECK:       bb33:
 ; CHECK-NEXT:    br i1 [[TMP23_0_1]], label [[BB35]], label [[BB35]]
diff --git a/llvm/test/Other/offload-wrapper.ll b/llvm/test/Other/offload-wrapper.ll
new file mode 100644
index 0000000000000..9107a141ad201
--- /dev/null
+++ b/llvm/test/Other/offload-wrapper.ll
@@ -0,0 +1,52 @@
+; RUN: llvm-offload-wrapper --triple=x86-64 -kind=hip %s -o %t.bc
+; RUN: llvm-dis %t.bc -o - | FileCheck %s --check-prefix=HIP
+
+;      HIP: @__start_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry], section "llvm_offload_entries$OA"
+; HIP-NEXT: @__stop_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry], section "llvm_offload_entries$OZ"
+; HIP-NEXT: @.fatbin_image = internal constant {{.*}}, section ".hip_fatbin"
+; HIP-NEXT: @.fatbin_wrapper = internal constant %fatbin_wrapper { i32 1212764230, i32 1, ptr @.fatbin_image, ptr null }, section ".hipFatBinSegment", align 8
+; HIP-NEXT: @.hip.binary_handle = internal global ptr null
+; HIP-NEXT: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.hip.fatbin_reg, ptr null }]
+
+;      HIP: define internal void @.hip.fatbin_reg() section ".text.startup" {
+; HIP-NEXT: entry:
+; HIP-NEXT:   %0 = call ptr @__hipRegisterFatBinary(ptr @.fatbin_wrapper)
+; HIP-NEXT:   store ptr %0, ptr @.hip.binary_handle, align 8
+; HIP-NEXT:   call void @.hip.globals_reg(ptr %0)
+; HIP-NEXT:   %1 = call i32 @atexit(ptr @.hip.fatbin_unreg)
+; HIP-NEXT:   ret void
+; HIP-NEXT: }
+
+;      HIP: define internal void @.hip.fatbin_unreg() section ".text.startup" {
+; HIP-NEXT: entry:
+; HIP-NEXT:   %0 = load ptr, ptr @.hip.binary_handle, align 8
+; HIP-NEXT:   call void @__hipUnregisterFatBinary(ptr %0)
+; HIP-NEXT:   ret void
+; HIP-NEXT: }
+
+; RUN: llvm-offload-wrapper --triple=x86-64 -kind=cuda %s -o %t.bc
+; RUN: llvm-dis %t.bc -o - | FileCheck %s --check-prefix=CUDA
+
+;      CUDA: @__start_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry], section "llvm_offload_entries$OA"
+; CUDA-NEXT: @__stop_llvm_offload_entries = external hidden constant [0 x %struct.__tgt_offload_entry], section "llvm_offload_entries$OZ"
+; CUDA-NEXT: @.fatbin_image = internal constant {{.*}}, section ".nv_fatbin"
+; CUDA-NEXT: @.fatbin_wrapper = internal constant %fatbin_wrapper { i32 1180844977, i32 1, ptr @.fatbin_image, ptr null }, section ".nvFatBinSegment", align 8
+; CUDA-NEXT: @.cuda.binary_handle = internal global ptr null
+; CUDA-NEXT: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.cuda.fatbin_reg, ptr null }]
+
+;      CUDA: define internal void @.cuda.fatbin_reg() section ".text.startup" {
+; CUDA-NEXT: entry:
+; CUDA-NEXT:   %0 = call ptr @__cudaRegisterFatBinary(ptr @.fatbin_wrapper)
+; CUDA-NEXT:   store ptr %0, ptr @.cuda.binary_handle, align 8
+; CUDA-NEXT:   call void @.cuda.globals_reg(ptr %0)
+; CUDA-NEXT:   call void @__cudaRegisterFatBinaryEnd(ptr %0)
+; CUDA-NEXT:   %1 = call i32 @atexit(ptr @.cuda.fatbin_unreg)
+; CUDA-NEXT:   ret void
+; CUDA-NEXT: }
+
+;      CUDA: define internal void @.cuda.fatbin_unreg() section ".text.startup" {
+; CUDA-NEXT: entry:
+; CUDA-NEXT:   %0 = load ptr, ptr @.cuda.binary_handle, align 8
+; CUDA-NEXT:   call void @__cudaUnregisterFatBinary(ptr %0)
+; CUDA-NEXT:   ret void
+; CUDA-NEXT: }
diff --git a/llvm/test/TableGen/FixedLenDecoderEmitter/InvalidEncoding.td b/llvm/test/TableGen/FixedLenDecoderEmitter/InvalidEncoding.td
new file mode 100644
index 0000000000000..074c7c53e85c4
--- /dev/null
+++ b/llvm/test/TableGen/FixedLenDecoderEmitter/InvalidEncoding.td
@@ -0,0 +1,53 @@
+// RUN: not llvm-tblgen -gen-disassembler -I %p/../../../include %s -DTEST1 2>&1 | FileCheck %s --check-prefix=CHECK-TEST1
+// RUN: not llvm-tblgen -gen-disassembler -I %p/../../../include %s -DTEST2 2>&1 | FileCheck %s --check-prefix=CHECK-TEST2
+// RUN: not llvm-tblgen -gen-disassembler -I %p/../../../include %s -DTEST3 2>&1 | FileCheck %s --check-prefix=CHECK-TEST3
+
+include "llvm/Target/Target.td"
+
+def archInstrInfo : InstrInfo { }
+
+def arch : Target {
+    let InstructionSet = archInstrInfo;
+}
+
+#ifdef TEST1
+// CHECK-TEST1: [[#@LINE+1]]:5: error: foo: Size is 16 bits, but Inst bits beyond that are not zero/unset
+def foo : Instruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins i32imm:$factor);
+  let Size = 2;
+  field bits<24> Inst;
+  field bits<24> SoftFail = 0;
+  bits<8> factor;
+  let Inst{15...8} = factor{7...0};
+  let Inst{20} = 1;
+}
+#endif
+
+#ifdef TEST2
+// CHECK-TEST2: [[#@LINE+1]]:5: error: foo: Size is 16 bits, but SoftFail bits beyond that are not zero/unset
+def foo : Instruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins i32imm:$factor);
+  let Size = 2;
+  field bits<24> Inst;
+  field bits<24> SoftFail = 0;
+  bits<8> factor;
+  let Inst{15...8} = factor{7...0};
+  let Inst{23...16} = 0;
+  let SoftFail{20} = 1;
+}
+#endif
+
+#ifdef TEST3
+// CHECK-TEST3: [[#@LINE+1]]:5: error: bar: Size is 16 bits, but Inst specifies only 8 bits
+def bar : Instruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins i32imm:$factor);
+  let Size = 2;
+  field bits<8> Inst;
+  field bits<8> SoftFail = 0;
+  bits<4> factor;
+  let Inst{4...1} = factor{3...0};
+}
+#endif
diff --git a/llvm/test/TableGen/FixedLenDecoderEmitter/additional-encoding.td b/llvm/test/TableGen/FixedLenDecoderEmitter/additional-encoding.td
new file mode 100644
index 0000000000000..192cb3c3f5858
--- /dev/null
+++ b/llvm/test/TableGen/FixedLenDecoderEmitter/additional-encoding.td
@@ -0,0 +1,67 @@
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../../include %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+class Enc {
+  int Size = 2;
+  bits<16> Inst;
+}
+
+class EncSHIFT<bits<2> opc> : Enc {
+  bits<6> shamt;
+  let Inst{15...14} = {0, 0};
+  let Inst{13...12} = opc;
+  let Inst{11...6} = shamt;
+}
+
+class EncNOP<bits<2> opc> : Enc {
+  let Inst{15...14} = {0, 0};
+  let Inst{13...12} = opc;
+  let Inst{11...6} = {0, 0, 0, 0, 0, 0};
+}
+
+def ShAmtOp : Operand<i32> {
+  let DecoderMethod = "decodeShAmt";
+  let hasCompleteDecoder = false;
+}
+
+class I<dag out_ops, dag in_ops> : Instruction {
+  let InOperandList = in_ops;
+  let OutOperandList = out_ops;
+}
+
+// CHECK:      /* 0 */  MCD::OPC_ExtractField, 12, 4,                // Inst{15-12} ...
+// CHECK-NEXT: /* 3 */  MCD::OPC_FilterValue, 0, 14, 0,              // Skip to: 21
+// CHECK-NEXT: /* 7 */  MCD::OPC_CheckField, 6, 6, 0, 4, 0,          // Skip to: 17
+// CHECK-NEXT: /* 13 */ MCD::OPC_Decode, {{[0-9]+}}, 2, 0,           // Opcode: {{.*}}:NOP
+// CHECK-NEXT: /* 17 */ MCD::OPC_TryDecodeOrFail, {{[0-9]+}}, 2, 1,
+// CHECK-NEXT: /* 21 */ MCD::OPC_FilterValue, 1, 14, 0,              // Skip to: 39
+// CHECK-NEXT: /* 25 */ MCD::OPC_CheckField, 6, 6, 0, 4, 0,          // Skip to: 35
+// CHECK-NEXT: /* 31 */ MCD::OPC_Decode, {{[0-9]+}}, 2, 0,           // Opcode: {{.*}}:NOP
+// CHECK-NEXT: /* 35 */ MCD::OPC_TryDecodeOrFail, {{[0-9]+}}, 2, 1,
+// CHECK-NEXT: /* 39 */ MCD::OPC_FilterValue, 2, 14, 0,              // Skip to: 57
+// CHECK-NEXT: /* 43 */ MCD::OPC_CheckField, 6, 6, 0, 4, 0,          // Skip to: 53
+// CHECK-NEXT: /* 49 */ MCD::OPC_Decode, {{[0-9]+}}, 2, 0,           // Opcode: {{.*}}:NOP
+// CHECK-NEXT: /* 53 */ MCD::OPC_TryDecodeOrFail, {{[0-9]+}}, 2, 1,
+// CHECK-NEXT: /* 57 */ MCD::OPC_FilterValueOrFail, 3,
+// CHECK-NEXT: /* 59 */ MCD::OPC_CheckField, 6, 6, 0, 4, 0,          // Skip to: 69
+// CHECK-NEXT: /* 65 */ MCD::OPC_Decode, {{[0-9]+}}, 2, 0,           // Opcode: {{.*}}:NOP
+// CHECK-NEXT: /* 69 */ MCD::OPC_TryDecodeOrFail, {{[0-9]+}}, 2, 1,
+// CHECK-NEXT: /* 73 */ MCD::OPC_Fail,
+
+class SHIFT<bits<2> opc> : I<(outs), (ins ShAmtOp:$shamt)>, EncSHIFT<opc>;
+def SHIFT0 : SHIFT<0>;
+def SHIFT1 : SHIFT<1>;
+def SHIFT2 : SHIFT<2>;
+def SHIFT3 : SHIFT<3>;
+
+def NOP : I<(outs), (ins)>, EncNOP<0>;
+def : AdditionalEncoding<NOP>, EncNOP<1>;
+def : AdditionalEncoding<NOP>, EncNOP<2>;
+def : AdditionalEncoding<NOP>, EncNOP<3>;
+
+def II : InstrInfo;
+
+def MyTarget : Target {
+  let InstructionSet = II;
+}
diff --git a/llvm/test/TableGen/FixedLenDecoderEmitter/big-filter.td b/llvm/test/TableGen/FixedLenDecoderEmitter/big-filter.td
new file mode 100644
index 0000000000000..e525b4e7219df
--- /dev/null
+++ b/llvm/test/TableGen/FixedLenDecoderEmitter/big-filter.td
@@ -0,0 +1,39 @@
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../../include %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+class I : Instruction {
+  let InOperandList = (ins);
+  let OutOperandList = (outs);
+  let Size = 16;
+  bits<128> Inst;
+}
+
+// Check that a 64-bit filter with all bits set does not confuse DecoderEmitter.
+//
+// CHECK-LABEL: static const uint8_t DecoderTable128[35] = {
+// CHECK-NEXT:    MCD::OPC_ExtractField, 0, 64,
+// CHECK-NEXT:    MCD::OPC_FilterValue, 1, 8, 0,
+// CHECK-NEXT:    MCD::OPC_CheckFieldOrFail, 127, 1, 1,
+// CHECK-NEXT:    MCD::OPC_Decode, 187, 2, 0,
+// CHECK-NEXT:    MCD::OPC_FilterValueOrFail, 255, 255, 255, 255, 255, 255, 255, 255, 255, 1,
+// CHECK-NEXT:    MCD::OPC_CheckFieldOrFail, 127, 1, 0,
+// CHECK-NEXT:    MCD::OPC_Decode, 186, 2, 0,
+// CHECK-NEXT:    MCD::OPC_Fail,
+// CHECK-NEXT:  };
+
+def I1 : I {
+  let Inst{63...0} = -1;
+  let Inst{127} = 0;
+}
+
+def I2 : I {
+  let Inst{63...0} = 1;
+  let Inst{127} = 1;
+}
+
+def II : InstrInfo;
+
+def MyTarget : Target {
+  let InstructionSet = II;
+}
diff --git a/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td b/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td
index 7399ef726d0e2..853a68d22d1d9 100644
--- a/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td
+++ b/llvm/test/TableGen/FixedLenDecoderEmitter/conflict.td
@@ -29,7 +29,7 @@ def B : I<(outs GPR32:$dst), (ins GPR32:$src1), []> {
 }
 
 // CHECK: Decoding Conflict:
-// CHECK:   00000000000000000000000000000000
-// CHECK:   ................................
-// CHECK: A 00000000000000000000000000000000
-// CHECK: B 00000000000000000000000000000000
+// CHECK:     ................................
+// CHECK:     00000000000000000000000000000000
+// CHECK:     00000000000000000000000000000000  A
+// CHECK:     00000000000000000000000000000000  B
diff --git a/llvm/test/TableGen/FixedLenDecoderEmitter/var-len-conflict-1.td b/llvm/test/TableGen/FixedLenDecoderEmitter/var-len-conflict-1.td
new file mode 100644
index 0000000000000..eda714eef1048
--- /dev/null
+++ b/llvm/test/TableGen/FixedLenDecoderEmitter/var-len-conflict-1.td
@@ -0,0 +1,44 @@
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../../include %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+class I : Instruction {
+  let InOperandList = (ins i32imm:$op);
+  let OutOperandList = (outs);
+}
+
+// Check that we don't try to read the second byte without ruling out
+// 1-byte encodings first. This should actually be a decoding conflict,
+// but DecoderEmitter heuristics decide that I8_0 and I8_1 are more specific
+// than the rest and give them priority.
+
+//          _______0  I8_0
+//          _______1  I8_1
+// 00000000 ________  I16_0
+// 00000001 ________  I16_1
+// 00000010 ________  I16_2
+
+// CHECK:      MCD::OPC_ExtractField, 0, 1,       // Inst{0} ...
+// CHECK-NEXT: MCD::OPC_FilterValue, 0, 4, 0,     // Skip to: 11
+// CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, 2, 0, // Opcode: I8_0, DecodeIdx: 0
+// CHECK-NEXT: MCD::OPC_FilterValue, 1, 4, 0,     // Skip to: 19
+// CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, 2, 0, // Opcode: I8_1, DecodeIdx: 0
+// CHECK-NEXT: MCD::OPC_ExtractField, 8, 8,       // Inst{15-8} ...
+// CHECK-NEXT: MCD::OPC_FilterValue, 0, 4, 0,     // Skip to: 30
+// CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, 2, 1, // Opcode: I16_0, DecodeIdx: 1
+// CHECK-NEXT: MCD::OPC_FilterValue, 1, 4, 0,     // Skip to: 38
+// CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, 2, 1, // Opcode: I16_1, DecodeIdx: 1
+// CHECK-NEXT: MCD::OPC_FilterValueOrFail, 2,
+// CHECK-NEXT: MCD::OPC_Decode, {{[0-9]+}}, 2, 1, // Opcode: I16_2, DecodeIdx: 1
+
+def I8_0  : I { dag Inst = (descend (operand "$op", 7), 0b0); }
+def I8_1  : I { dag Inst = (descend (operand "$op", 7), 0b1); }
+def I16_0 : I { dag Inst = (descend 0b00000000, (operand "$op", 8)); }
+def I16_1 : I { dag Inst = (descend 0b00000001, (operand "$op", 8)); }
+def I16_2 : I { dag Inst = (descend 0b00000010, (operand "$op", 8)); }
+
+def II : InstrInfo;
+
+def MyTarget : Target {
+  let InstructionSet = II;
+}
diff --git a/llvm/test/TableGen/FixedLenDecoderEmitter/var-len-conflict-2.td b/llvm/test/TableGen/FixedLenDecoderEmitter/var-len-conflict-2.td
new file mode 100644
index 0000000000000..8a848a986ce24
--- /dev/null
+++ b/llvm/test/TableGen/FixedLenDecoderEmitter/var-len-conflict-2.td
@@ -0,0 +1,35 @@
+// RUN: not llvm-tblgen -gen-disassembler -I %p/../../../include %s -o /dev/null 2>&1 \
+// RUN:   | FileCheck --strict-whitespace %s
+
+include "llvm/Target/Target.td"
+
+class I : Instruction {
+  let InOperandList = (ins i32imm:$op, i32imm:$op2, i32imm:$op3);
+  let OutOperandList = (outs);
+}
+
+// Check pretty printing of decoding conflicts.
+
+// CHECK:      {{^}}Decoding Conflict:
+// CHECK-NEXT: {{^}}                    ........
+// CHECK-NEXT: {{^}}            ..............11
+// CHECK-NEXT: {{^}}            .......0......11
+// CHECK-NEXT: {{^}}            _______0______11  I16_0
+// CHECK-NEXT: {{^}}            _______0______11  I16_1
+// CHECK-NEXT: {{^}}    _______0_______0______11  I24_0
+
+def I8_0  : I { dag Inst = (descend (operand "$op", 6), 0b00); }
+def I8_1  : I { dag Inst = (descend (operand "$op", 6), 0b01); }
+def I16_0 : I { dag Inst = (descend (operand "$op2", 7), 0b0,
+                                    (operand "$op", 6), 0b11); }
+def I16_1 : I { dag Inst = (descend (operand "$op2", 7), 0b0,
+                                    (operand "$op", 6), 0b11); }
+def I24_0 : I { dag Inst = (descend (operand "$op3", 7), 0b0,
+                                    (operand "$op2", 7), 0b0,
+                                    (operand "$op", 6), 0b11); }
+
+def II : InstrInfo;
+
+def MyTarget : Target {
+  let InstructionSet = II;
+}
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-typeof.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-typeof.td
index 7fe63b1298ae7..ef918d47b9d88 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-typeof.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-typeof.td
@@ -20,18 +20,18 @@ def Test0 : GICombineRule<
 // CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_MUL),
 // CHECK-NEXT:       // MIs[0] dst
-// CHECK-NEXT:       GIM_RecordRegType, /*MI*/0, /*Op*/0, /*TempTypeIdx*/uint8_t(-1),
+// CHECK-NEXT:       GIM_RecordRegType, /*MI*/0, /*Op*/0, /*TempTypeIdx*/255,
 // CHECK-NEXT:       // MIs[0] src
-// CHECK-NEXT:       GIM_RecordRegType, /*MI*/0, /*Op*/1, /*TempTypeIdx*/uint8_t(-2),
+// CHECK-NEXT:       GIM_RecordRegType, /*MI*/0, /*Op*/1, /*TempTypeIdx*/254,
 // CHECK-NEXT:       // MIs[0] Operand 2
-// CHECK-NEXT:       GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-1),
-// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/uint8_t(-2),
+// CHECK-NEXT:       GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 255,
+// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/254,
 // CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/1, /*Val*/GIMT_Encode8(0),
-// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/uint8_t(-1),
+// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/255,
 // CHECK-NEXT:       // Combiner Rule #0: Test0
 // CHECK-NEXT:       GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::G_CONSTANT),
 // CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
-// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/0, /*Type*/uint8_t(-2), /*Imm*/GIMT_Encode8(42),
+// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/0, /*Type*/254, /*Imm*/GIMT_Encode8(42),
 // CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_SUB),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // dst
 // CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/1,
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
index 92baab91c620e..8907cfe811abd 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
@@ -181,7 +181,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
 // CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_CONSTANT),
 // CHECK-NEXT:       // MIs[1] z
-// CHECK-NEXT:       GIM_CheckLiteralInt, /*MI*/1, /*Op*/1, GIMT_Encode8(-42),
+// CHECK-NEXT:       GIM_CheckLiteralInt, /*MI*/1, /*Op*/1, GIMT_Encode8(18446744073709551574u),
 // CHECK-NEXT:       GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 43,
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*NumInsns*/1,
 // CHECK-NEXT:       // Combiner Rule #5: InOutInstTest1
diff --git a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
index 4a516c6e0235c..7a86b5b726a82 100644
--- a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td
@@ -617,11 +617,11 @@ def MOV : I<(outs GPR32:$dst), (ins GPR32:$src1),
 // R02N-NEXT:    // MIs[0] Operand 2
 // R02N-NEXT:    GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
 //
-// R02C-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-2)
+// R02C-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 254,
 // R02C-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -2:{ *:[i32] }) => (XORI:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
 // R02C-NEXT:    GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::XORI),
 // R02C-NEXT:    GIR_RootToRootCopy, /*OpIdx*/0, // DstI[dst]
-// R02C-NEXT:    GIR_AddImm8, /*InsnID*/0, /*Imm*/uint8_t(-1),
+// R02C-NEXT:    GIR_AddImm8, /*InsnID*/0, /*Imm*/255,
 // R02C-NEXT:    GIR_RootToRootCopy, /*OpIdx*/1, // src1
 // R02C-NEXT:    GIR_RootConstrainSelectedInstOperands,
 // R02C-NEXT:    // GIR_Coverage, 2,
@@ -648,7 +648,7 @@ def XORI : I<(outs GPR32:$dst), (ins m1:$src2, GPR32:$src1),
 // NOOPT-NEXT:    GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-3)
+// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 253,
 // NOOPT-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -3:{ *:[i32] }) => (XOR:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
 // NOOPT-NEXT:    GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::XOR),
 // NOOPT-NEXT:    GIR_RootToRootCopy, /*OpIdx*/0, //  DstI[dst]
@@ -676,11 +676,11 @@ def XOR : I<(outs GPR32:$dst), (ins Z:$src2, GPR32:$src1),
 // NOOPT-NEXT:    GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-4)
+// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 252,
 // NOOPT-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -4:{ *:[i32] }) => (XORlike:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
 // NOOPT-NEXT:    GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::XORlike),
 // NOOPT-NEXT:    GIR_RootToRootCopy, /*OpIdx*/0, // DstI[dst]
-// NOOPT-NEXT:    GIR_AddImm8, /*InsnID*/0, /*Imm*/uint8_t(-1),
+// NOOPT-NEXT:    GIR_AddImm8, /*InsnID*/0, /*Imm*/255,
 // NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, GIMT_Encode2(MyTarget::R0),
 // NOOPT-NEXT:    GIR_RootToRootCopy, /*OpIdx*/1, // src1
 // NOOPT-NEXT:    GIR_RootConstrainSelectedInstOperands,
@@ -705,11 +705,11 @@ def XORlike : I<(outs GPR32:$dst), (ins m1Z:$src2, GPR32:$src1),
 // NOOPT-NEXT:    GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-5),
+// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 251,
 // NOOPT-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -5:{ *:[i32] }) => (XORManyDefaults:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
 // NOOPT-NEXT:    GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::XORManyDefaults),
 // NOOPT-NEXT:    GIR_RootToRootCopy, /*OpIdx*/0, // DstI[dst]
-// NOOPT-NEXT:    GIR_AddImm8, /*InsnID*/0, /*Imm*/uint8_t(-1),
+// NOOPT-NEXT:    GIR_AddImm8, /*InsnID*/0, /*Imm*/255,
 // NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, GIMT_Encode2(MyTarget::R0),
 // NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, GIMT_Encode2(MyTarget::R0),
 // NOOPT-NEXT:    GIR_RootToRootCopy, /*OpIdx*/1, // src1
@@ -735,7 +735,7 @@ def XORManyDefaults : I<(outs GPR32:$dst), (ins m1Z:$src3, Z:$src2, GPR32:$src1)
 // NOOPT-NEXT:    GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-6)
+// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 250,
 // NOOPT-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -6:{ *:[i32] }) => (XORIb:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
 // NOOPT-NEXT:    GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::XORIb),
 // NOOPT-NEXT:    GIR_RootToRootCopy, /*OpIdx*/0, // DstI[dst]
@@ -766,7 +766,7 @@ def XORIb : I<(outs GPR32:$dst), (ins mb:$src2, GPR32:$src1),
 // NOOPT-NEXT:    GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-1),
+// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 255,
 // NOOPT-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$Wm, -1:{ *:[i32] }) => (ORN:{ *:[i32] } R0:{ *:[i32] }, GPR32:{ *:[i32] }:$Wm)
 // NOOPT-NEXT:    GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::ORN),
 // NOOPT-NEXT:    GIR_RootToRootCopy, /*OpIdx*/0, //  DstI[dst]
diff --git a/llvm/test/TableGen/GlobalISelEmitter/int64min.td b/llvm/test/TableGen/GlobalISelEmitter/int64min.td
new file mode 100644
index 0000000000000..ccdb749f2c148
--- /dev/null
+++ b/llvm/test/TableGen/GlobalISelEmitter/int64min.td
@@ -0,0 +1,30 @@
+// RUN: llvm-tblgen -gen-global-isel -optimize-match-table=false -I %p/../../../include -I %p/../Common %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+include "GlobalISelEmitterCommon.td"
+
+def GPR : RegisterClass<"MyTarget", [i64], 64, (add R0)>;
+def ANDI : I<(outs GPR:$dst), (ins GPR:$src1, i64imm:$src2), []>;
+
+// CHECK-LABEL: GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(59), // Rule ID 0 //
+// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_AND),
+// CHECK-NEXT: // MIs[0] DstI[dst]
+// CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s64,
+// CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
+// CHECK-NEXT: // MIs[0] rs1
+// CHECK-NEXT: GIM_RootCheckType, /*Op*/1, /*Type*/GILLT_s64,
+// CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
+// CHECK-NEXT: // MIs[0] Operand 2
+// CHECK-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s64,
+// CHECK-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, GIMT_Encode8(9223372036854775808u),
+// CHECK-NEXT: // (and:{ *:[i64] } GPR:{ *:[i64] }:$rs1, -9223372036854775808:{ *:[i64] })  =>  (ANDI:{ *:[i64] } GPR:{ *:[i64] }:$rs1, -9223372036854775808:{ *:[i64] })
+// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::ANDI),
+// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/1, // rs1
+// CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/GIMT_Encode8(9223372036854775808u),
+// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands,
+// CHECK-NEXT: // GIR_Coverage, 0,
+// CHECK-NEXT: GIR_EraseRootFromParent_Done,
+def : Pat<(and GPR:$rs1, 0x8000000000000000),
+          (ANDI GPR:$rs1, 0x8000000000000000)>;
diff --git a/llvm/test/TableGen/HwModeEncodeDecode.td b/llvm/test/TableGen/HwModeEncodeDecode.td
index aaacd7d3f6590..991054f8a279f 100644
--- a/llvm/test/TableGen/HwModeEncodeDecode.td
+++ b/llvm/test/TableGen/HwModeEncodeDecode.td
@@ -70,10 +70,10 @@ def baz : Instruction {
 }
 }
 
-// DECODER-LABEL: DecoderTable_ModeA32[] =
+// DECODER-LABEL: DecoderTable_ModeA32[22] =
 // DECODER-DAG: Opcode: fooTypeEncA:foo
 // DECODER-DAG: Opcode: bar
-// DECODER-LABEL: DecoderTable_ModeB32[] =
+// DECODER-LABEL: DecoderTable_ModeB32[30] =
 // DECODER-DAG: Opcode: fooTypeEncB:foo
 // DECODER-DAG: Opcode: fooTypeEncA:baz
 // DECODER-DAG: Opcode: bar
diff --git a/llvm/test/TableGen/HwModeEncodeDecode2.td b/llvm/test/TableGen/HwModeEncodeDecode2.td
index cf96dda6c8bf3..a62d723837c8f 100644
--- a/llvm/test/TableGen/HwModeEncodeDecode2.td
+++ b/llvm/test/TableGen/HwModeEncodeDecode2.td
@@ -93,26 +93,26 @@ let OutOperandList = (outs) in {
   }
 }
 
-// DECODER-LABEL: DecoderTableAlt_ModeA32[] =
-// DECODER-DAG: Opcode: unrelated
-// DECODER-LABEL: DecoderTableAlt_ModeB32[] =
-// DECODER-DAG: Opcode: unrelated
-// DECODER-LABEL: DecoderTable_ModeA32[] =
+// DECODER-LABEL: DecoderTable_ModeA32[22] =
 // DECODER-DAG: Opcode: fooTypeEncA:foo
 // DECODER-DAG: Opcode: bar
-// DECODER-LABEL: DecoderTable_ModeB32[] =
+// DECODER-LABEL: DecoderTable_ModeB32[30] =
 // DECODER-DAG: Opcode: fooTypeEncB:foo
 // DECODER-DAG: Opcode: fooTypeEncA:baz
 // DECODER-DAG: Opcode: bar
+// DECODER-LABEL: DecoderTableAlt_ModeA32[9] =
+// DECODER-DAG: Opcode: unrelated
+// DECODER-LABEL: DecoderTableAlt_ModeB32[9] =
+// DECODER-DAG: Opcode: unrelated
 
-// DECODER-SUPPRESS-LABEL: DecoderTable32[] =
+// DECODER-SUPPRESS-LABEL: DecoderTable32[9] =
 // DECODER-SUPPRESS-DAG: Opcode: bar
-// DECODER-SUPPRESS-LABEL: DecoderTableAlt32[] =
-// DECODER-SUPPRESS-DAG: Opcode: unrelated
-// DECODER-SUPPRESS-LABEL: DecoderTable_ModeA32[] =
+// DECODER-SUPPRESS-LABEL: DecoderTable_ModeA32[9] =
 // DECODER-SUPPRESS-DAG: Opcode: fooTypeEncA:foo
 // DECODER-SUPPRESS-NOT: Opcode: bar
-// DECODER-SUPPRESS-LABEL: DecoderTable_ModeB32[] =
+// DECODER-SUPPRESS-LABEL: DecoderTable_ModeB32[22] =
 // DECODER-SUPPRESS-DAG: Opcode: fooTypeEncB:foo
 // DECODER-SUPPRESS-DAG: Opcode: fooTypeEncA:baz
 // DECODER-SUPPRESS-NOT: Opcode: bar
+// DECODER-SUPPRESS-LABEL: DecoderTableAlt32[9] =
+// DECODER-SUPPRESS-DAG: Opcode: unrelated
diff --git a/llvm/test/TableGen/HwModeEncodeDecode3.td b/llvm/test/TableGen/HwModeEncodeDecode3.td
index c4d488d9d5f8f..a7366949f4017 100644
--- a/llvm/test/TableGen/HwModeEncodeDecode3.td
+++ b/llvm/test/TableGen/HwModeEncodeDecode3.td
@@ -116,47 +116,47 @@ def unrelated: Instruction {
 // significant duplication of DecoderTables. The four tables ‘DecoderTableAlt32’,
 // ‘DecoderTableAlt_ModeA32’, ‘DecoderTableAlt_ModeB32’ and 'DecoderTable_ModeC32' are
 // exact duplicates and could effectively be merged into one.
-// DECODER-LABEL: DecoderTable32[] =
+// DECODER-LABEL: DecoderTable32[9] =
 // DECODER-DAG: Opcode: bar
-// DECODER-LABEL: DecoderTable64[] =
+// DECODER-LABEL: DecoderTable64[9] =
 // DECODER-DAG: Opcode: fooTypeEncDefault:foo
-// DECODER-LABEL: DecoderTableAlt32[] =
-// DECODER-DAG: Opcode: unrelated
-// DECODER-LABEL: DecoderTableAlt_ModeA32[] =
-// DECODER-DAG: Opcode: unrelated
-// DECODER-LABEL: DecoderTableAlt_ModeB32[] =
-// DECODER-DAG: Opcode: unrelated
-// DECODER-LABEL: DecoderTableAlt_ModeC32[] =
-// DECODER-DAG: Opcode: unrelated
-// DECODER-LABEL: DecoderTable_ModeA32[] =
+// DECODER-LABEL: DecoderTable_ModeA32[22] =
 // DECODER-DAG: Opcode: fooTypeEncA:foo
 // DECODER-DAG: Opcode: bar
-// DECODER-LABEL: DecoderTable_ModeB32[] =
+// DECODER-LABEL: DecoderTable_ModeB32[30] =
 // DECODER-DAG: Opcode: fooTypeEncB:foo
 // DECODER-DAG: Opcode: fooTypeEncA:baz
 // DECODER-DAG: Opcode: bar
-// DECODER-LABEL: DecoderTable_ModeC32[] =
+// DECODER-LABEL: DecoderTable_ModeC32[22] =
 // DECODER-DAG: Opcode: fooTypeEncC:foo
 // DECODER-DAG: Opcode: bar
+// DECODER-LABEL: DecoderTableAlt32[9] =
+// DECODER-DAG: Opcode: unrelated
+// DECODER-LABEL: DecoderTableAlt_ModeA32[9] =
+// DECODER-DAG: Opcode: unrelated
+// DECODER-LABEL: DecoderTableAlt_ModeB32[9] =
+// DECODER-DAG: Opcode: unrelated
+// DECODER-LABEL: DecoderTableAlt_ModeC32[9] =
+// DECODER-DAG: Opcode: unrelated
 
 // Under the 'O1' optimization level, unnecessary duplicate tables will be eliminated,
 // reducing the four ‘Alt’ tables down to just one.
-// DECODER-SUPPRESS-O1-LABEL: DecoderTable32[] =
+// DECODER-SUPPRESS-O1-LABEL: DecoderTable32[9] =
 // DECODER-SUPPRESS-O1-DAG: Opcode: bar
-// DECODER-SUPPRESS-O1-LABEL: DecoderTable64[] =
+// DECODER-SUPPRESS-O1-LABEL: DecoderTable64[9] =
 // DECODER-SUPPRESS-O1-DAG: Opcode: fooTypeEncDefault:foo
-// DECODER-SUPPRESS-O1-LABEL: DecoderTableAlt32[] =
-// DECODER-SUPPRESS-O1-DAG: Opcode: unrelated
-// DECODER-SUPPRESS-O1-LABEL: DecoderTable_ModeA32[] =
+// DECODER-SUPPRESS-O1-LABEL: DecoderTable_ModeA32[22] =
 // DECODER-SUPPRESS-O1-DAG: Opcode: fooTypeEncA:foo
 // DECODER-SUPPRESS-O1-DAG: Opcode: bar
-// DECODER-SUPPRESS-O1-LABEL: DecoderTable_ModeB32[] =
+// DECODER-SUPPRESS-O1-LABEL: DecoderTable_ModeB32[30] =
 // DECODER-SUPPRESS-O1-DAG: Opcode: fooTypeEncB:foo
 // DECODER-SUPPRESS-O1-DAG: Opcode: fooTypeEncA:baz
 // DECODER-SUPPRESS-O1-DAG: Opcode: bar
-// DECODER-SUPPRESS-O1-LABEL: DecoderTable_ModeC32[] =
+// DECODER-SUPPRESS-O1-LABEL: DecoderTable_ModeC32[22] =
 // DECODER-SUPPRESS-O1-DAG: Opcode: fooTypeEncC:foo
 // DECODER-SUPPRESS-O1-DAG: Opcode: bar
+// DECODER-SUPPRESS-O1-LABEL: DecoderTableAlt32[9] =
+// DECODER-SUPPRESS-O1-DAG: Opcode: unrelated
 
 // Under the 'O2' optimization condition, instructions possessing the 'EncodingByHwMode'
 // attribute will be extracted from their original DecoderNamespace and placed into their
@@ -164,23 +164,23 @@ def unrelated: Instruction {
 // attribute but are within the same DecoderNamespace will be stored in the 'Default' table. This
 // approach will significantly reduce instruction redundancy, but it necessitates users to thoroughly
 // consider the interplay between HwMode and DecoderNamespace for their instructions.
-// DECODER-SUPPRESS-O2-LABEL: DecoderTable32[] =
+// DECODER-SUPPRESS-O2-LABEL: DecoderTable32[9] =
 // DECODER-SUPPRESS-O2-DAG: Opcode: bar
-// DECODER-SUPPRESS-O2-LABEL: DecoderTable64[] =
+// DECODER-SUPPRESS-O2-LABEL: DecoderTable64[9] =
 // DECODER-SUPPRESS-O2-NOT: Opcode: bar
 // DECODER-SUPPRESS-O2-DAG: Opcode: fooTypeEncDefault:foo
-// DECODER-SUPPRESS-O2-LABEL: DecoderTableAlt32[] =
-// DECODER-SUPPRESS-O2-DAG: Opcode: unrelated
-// DECODER-SUPPRESS-O2-LABEL: DecoderTable_ModeA32[] =
+// DECODER-SUPPRESS-O2-LABEL: DecoderTable_ModeA32[9] =
 // DECODER-SUPPRESS-O2-DAG: Opcode: fooTypeEncA:foo
 // DECODER-SUPPRESS-O2-NOT: Opcode: bar
-// DECODER-SUPPRESS-O2-LABEL: DecoderTable_ModeB32[] =
+// DECODER-SUPPRESS-O2-LABEL: DecoderTable_ModeB32[22] =
 // DECODER-SUPPRESS-O2-DAG: Opcode: fooTypeEncB:foo
 // DECODER-SUPPRESS-O2-DAG: Opcode: fooTypeEncA:baz
 // DECODER-SUPPRESS-O2-NOT: Opcode: bar
-// DECODER-SUPPRESS-O2-LABEL: DecoderTable_ModeC32[] =
+// DECODER-SUPPRESS-O2-LABEL: DecoderTable_ModeC32[13] =
 // DECODER-SUPPRESS-O2-DAG: Opcode: fooTypeEncC:foo
 // DECODER-SUPPRESS-O2-NOT: Opcode: bar
+// DECODER-SUPPRESS-O2-LABEL: DecoderTableAlt32[9] =
+// DECODER-SUPPRESS-O2-DAG: Opcode: unrelated
 
 // For 'bar' and 'unrelated', we didn't assign any HwModes for them,
 // they should keep the same in the following four tables.
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter-calling-conv.td b/llvm/test/TableGen/RuntimeLibcallEmitter-calling-conv.td
index 49d5ecaa0e5c5..7d1e830c912a5 100644
--- a/llvm/test/TableGen/RuntimeLibcallEmitter-calling-conv.td
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter-calling-conv.td
@@ -41,86 +41,54 @@ def MSP430LibraryWithCondCC : SystemRuntimeLibrary<isMSP430,
 >;
 
 
-// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets(const llvm::Triple &TT, FloatABI::ABIType FloatABI) {
+// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets(const llvm::Triple &TT, ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, EABI EABIVersion, StringRef ABIName) {
 // CHECK: if (TT.getArch() == Triple::avr && TT.isOSHurd()) {
 // CHECK-NEXT:   const CallingConv::ID DefaultCC = isFoo() ? CallingConv::Fast : CallingConv::GHC;
 // CHECK-NEXT:   for (CallingConv::ID &Entry : LibcallImplCallingConvs) {
 // CHECK-NEXT:     Entry = DefaultCC;
 // CHECK-NEXT:   }
 // CHECK-EMPTY:
-// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
-// CHECK-NEXT:        {RTLIB::MALLOC, RTLIB::malloc}, // malloc
-// CHECK-NEXT:    };
-// CHECK-EMPTY:
-// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
-// CHECK-NEXT:      setLibcallImpl(Func, Impl);
-// CHECK-NEXT:    }
+// CHECK-NEXT:    setLibcallsImpl({
+// CHECK-NEXT:      {RTLIB::MALLOC, RTLIB::malloc}, // malloc
+// CHECK-NEXT:    });
 // CHECK-EMPTY:
-// CHECK-NEXT:    static const LibcallImplPair LibraryCalls_AlwaysAvailable[] = {
+// CHECK-NEXT:    setLibcallsImpl({
 // CHECK-NEXT:        {RTLIB::SDIVREM_I8, RTLIB::__divmodqi4}, // __divmodqi4
 // CHECK-NEXT:        {RTLIB::UDIVREM_I16, RTLIB::__udivmodhi4}, // __udivmodhi4
-// CHECK-NEXT:    };
-// CHECK-EMPTY:
-// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls_AlwaysAvailable) {
-// CHECK-NEXT:      setLibcallImpl(Func, Impl);
-// CHECK-NEXT:      setLibcallImplCallingConv(Impl, CallingConv::AVR_BUILTIN);
-// CHECK-NEXT:    }
+// CHECK-NEXT:    }, CallingConv::AVR_BUILTIN);
 // CHECK-EMPTY:
 // CHECK-NEXT:    return;
 // CHECK-NEXT:  }
 // CHECK-EMPTY:
 // CHECK-NEXT: if (TT.getArch() == Triple::avr) {
-// CHECK-NEXT:   static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:   setLibcallsImpl({
 // CHECK-NEXT:       {RTLIB::MALLOC, RTLIB::malloc}, // malloc
-// CHECK-NEXT:   };
-// CHECK-EMPTY:
-// CHECK-NEXT:   for (const auto [Func, Impl] : LibraryCalls) {
-// CHECK-NEXT:     setLibcallImpl(Func, Impl);
-// CHECK-NEXT:   }
+// CHECK-NEXT:   });
 // CHECK-EMPTY:
-// CHECK-NEXT:   static const LibcallImplPair LibraryCalls_AlwaysAvailable[] = {
+// CHECK-NEXT:   setLibcallsImpl({
 // CHECK-NEXT:       {RTLIB::SDIVREM_I8, RTLIB::__divmodqi4}, // __divmodqi4
 // CHECK-NEXT:       {RTLIB::UDIVREM_I16, RTLIB::__udivmodhi4}, // __udivmodhi4
-// CHECK-NEXT:   };
-// CHECK-EMPTY:
-// CHECK-NEXT:   for (const auto [Func, Impl] : LibraryCalls_AlwaysAvailable) {
-// CHECK-NEXT:     setLibcallImpl(Func, Impl);
-// CHECK-NEXT:     setLibcallImplCallingConv(Impl, CallingConv::AVR_BUILTIN);
-// CHECK-NEXT:   }
+// CHECK-NEXT:   }, CallingConv::AVR_BUILTIN);
 // CHECK-EMPTY:
 // CHECK-NEXT:   return;
 // CHECK-NEXT:  }
 // CHECK-EMPTY:
 // CHECK-NEXT:  if (TT.getArch() == Triple::msp430) {
-// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:    setLibcallsImpl({
 // CHECK-NEXT:        {RTLIB::MALLOC, RTLIB::malloc}, // malloc
-// CHECK-NEXT:    };
-// CHECK-EMPTY:
-// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
-// CHECK-NEXT:      setLibcallImpl(Func, Impl);
-// CHECK-NEXT:    }
+// CHECK-NEXT:    });
 // CHECK-EMPTY:
 // CHECK-NEXT:    if ( isFoo() ) {
-// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_anonymous_3[] = {
-// CHECK-NEXT:          {RTLIB::SDIVREM_I8, RTLIB::__divmodqi4}, // __divmodqi4
-// CHECK-NEXT:      };
-// CHECK-EMPTY:
-// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_anonymous_3) {
-// CHECK-NEXT:        setLibcallImpl(Func, Impl);
-// CHECK-NEXT:        setLibcallImplCallingConv(Impl, CallingConv::AVR_BUILTIN);
-// CHECK-NEXT:      }
+// CHECK-NEXT:      setLibcallsImpl({
+// CHECK-NEXT:        {RTLIB::SDIVREM_I8, RTLIB::__divmodqi4}, // __divmodqi4
+// CHECK-NEXT:      }, CallingConv::AVR_BUILTIN);
 // CHECK-EMPTY:
 // CHECK-NEXT:    }
 // CHECK-EMPTY:
 // CHECK-NEXT:    if ( isBar() ) {
-// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_anonymous_5[] = {
+// CHECK-NEXT:      setLibcallsImpl({
 // CHECK-NEXT:          {RTLIB::UDIVREM_I16, RTLIB::__udivmodhi4}, // __udivmodhi4
-// CHECK-NEXT:      };
-// CHECK-EMPTY:
-// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_anonymous_5) {
-// CHECK-NEXT:        setLibcallImpl(Func, Impl);
-// CHECK-NEXT:        setLibcallImplCallingConv(Impl, CallingConv::MSP430_BUILTIN);
-// CHECK-NEXT:      }
+// CHECK-NEXT:      }, CallingConv::MSP430_BUILTIN);
 // CHECK-EMPTY:
 // CHECK-NEXT:    }
 // CHECK-EMPTY:
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter-conflict-warning.td b/llvm/test/TableGen/RuntimeLibcallEmitter-conflict-warning.td
index 00c9d53846326..2608faf2f75f0 100644
--- a/llvm/test/TableGen/RuntimeLibcallEmitter-conflict-warning.td
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter-conflict-warning.td
@@ -25,9 +25,9 @@ def dup1 : RuntimeLibcallImpl<ANOTHER_DUP>;
 // func_a and func_b both provide SOME_FUNC.
 
 // CHECK: if (isTargetArchA()) {
-// CHECK-NEXT: static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT: setLibcallsImpl({
 // CHECK-NEXT:   {RTLIB::SOME_FUNC, RTLIB::func_b}, // func_b
-// CHECK-NEXT: };
+// CHECK-NEXT: });
 
 // ERR: :[[@LINE+1]]:5: warning: conflicting implementations for libcall SOME_FUNC: func_b, func_a
 def TheSystemLibraryA : SystemRuntimeLibrary<isTargetArchA,
@@ -35,10 +35,10 @@ def TheSystemLibraryA : SystemRuntimeLibrary<isTargetArchA,
 >;
 
 // CHECK: if (isTargetArchB()) {
-// CHECK-NEXT: static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT: setLibcallsImpl({
 // CHECK-NEXT:   {RTLIB::OTHER_FUNC, RTLIB::other_func}, // other_func
-// CHECK-NEXT:  {RTLIB::SOME_FUNC, RTLIB::func_a}, // func_a
-// CHECK-NEXT: };
+// CHECK-NEXT:   {RTLIB::SOME_FUNC, RTLIB::func_a}, // func_a
+// CHECK-NEXT: });
 
 // ERR: :[[@LINE+1]]:5: warning: conflicting implementations for libcall SOME_FUNC: func_a, func_b
 def TheSystemLibraryB : SystemRuntimeLibrary<isTargetArchB,
@@ -46,11 +46,11 @@ def TheSystemLibraryB : SystemRuntimeLibrary<isTargetArchB,
 >;
 
 // CHECK: if (isTargetArchC()) {
-// CHECK-NEXT: static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT: setLibcallsImpl({
 // CHECK-NEXT:   {RTLIB::ANOTHER_DUP, RTLIB::dup1}, // dup1
 // CHECK-NEXT:   {RTLIB::OTHER_FUNC, RTLIB::other_func}, // other_func
 // CHECK-NEXT:   {RTLIB::SOME_FUNC, RTLIB::func_a}, // func_a
-// CHECK-NEXT: };
+// CHECK-NEXT: });
 
 // ERR: :[[@LINE+3]]:5: warning: conflicting implementations for libcall ANOTHER_DUP: dup1, dup0
 // ERR: :[[@LINE+2]]:5: warning: conflicting implementations for libcall SOME_FUNC: func_a, func_b
diff --git a/llvm/test/TableGen/RuntimeLibcallEmitter.td b/llvm/test/TableGen/RuntimeLibcallEmitter.td
index 642f8b85a89c6..07c8abcaab567 100644
--- a/llvm/test/TableGen/RuntimeLibcallEmitter.td
+++ b/llvm/test/TableGen/RuntimeLibcallEmitter.td
@@ -137,6 +137,19 @@ def BlahLibrary : SystemRuntimeLibrary<isBlahArch, (add calloc, LibraryWithCondi
 // CHECK-NEXT:   54, // sqrtl
 // CHECK-NEXT:   54, // sqrtl
 // CHECK-NEXT: };
+// CHECK-EMPTY:
+// CHECK-NEXT: const uint8_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameSizeTable[] = {
+// CHECK-NEXT:   0,
+// CHECK-NEXT:   9,
+// CHECK-NEXT:   9,
+// CHECK-NEXT:   9,
+// CHECK-NEXT:   9,
+// CHECK-NEXT:   5,
+// CHECK-NEXT:   6,
+// CHECK-NEXT:   5,
+// CHECK-NEXT:   5,
+// CHECK-NEXT: };
+// CHECK-EMPTY:
 // CHECK-NEXT: const RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::ImplToLibcall[RTLIB::NumLibcallImpls] = {
 // CHECK-NEXT: RTLIB::UNKNOWN_LIBCALL, // RTLIB::Unsupported
 // CHECK-NEXT: RTLIB::MEMCPY, // RTLIB::___memcpy
@@ -149,44 +162,68 @@ def BlahLibrary : SystemRuntimeLibrary<isBlahArch, (add calloc, LibraryWithCondi
 // CHECK-NEXT: RTLIB::SQRT_F80, // RTLIB::sqrtl_f80
 // CHECK-NEXT: };
 
+// CHECK: #ifdef GET_LOOKUP_LIBCALL_IMPL_NAME_BODY
+// CHECK-NEXT: size_t Size = Name.size();
+// CHECK-NEXT: if (Size == 0 || Size > 9)
+// CHECK-NEXT:   return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);
+// CHECK-NEXT: return lookupLibcallImplNameImpl(Name);
+// CHECK-NEXT: #endif
+
+// CHECK: #ifdef DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME
+// CHECK-NEXT: static inline uint64_t hash(StringRef Str) {
+// CHECK-NEXT: return static_cast<uint32_t>(xxh3_64bits(Str));
+// CHECK-NEXT: }
+
+// CHECK: iota_range<RTLIB::LibcallImpl> RTLIB::RuntimeLibcallsInfo::lookupLibcallImplNameImpl(StringRef Name) {
+// CHECK: static constexpr uint16_t HashTableNameToEnum[16] = {
+// CHECK: 2,
+// CHECK: 0,
+// CHECK: 6,
+// CHECK: 0,
+// CHECK: };
 
-// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets(const llvm::Triple &TT, FloatABI::ABIType FloatABI) {
+// CHECK: unsigned Idx = (hash(Name) % 8) * 2;
+// CHECK: for (int I = 0; I != 2; ++I) {
+// CHECK: return libcallImplNameHit(Entry, StrOffset);
+
+// CHECK: return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);
+// CHECK-NEXT: }
+
+// CHECK: void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets(const llvm::Triple &TT, ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, EABI EABIVersion, StringRef ABIName) {
 // CHECK-NEXT:  struct LibcallImplPair {
 // CHECK-NEXT:    RTLIB::Libcall Func;
 // CHECK-NEXT:    RTLIB::LibcallImpl Impl;
 // CHECK-NEXT:  };
+// CHECK-NEXT:  auto setLibcallsImpl = [this](
+// CHECK-NEXT:    ArrayRef<LibcallImplPair> Libcalls,
+// CHECK-NEXT:    std::optional<llvm::CallingConv::ID> CC = {})
+// CHECK-NEXT:  {
+// CHECK-NEXT:    for (const auto [Func, Impl] : Libcalls) {
+// CHECK-NEXT:      setLibcallImpl(Func, Impl);
+// CHECK-NEXT:      if (CC)
+// CHECK-NEXT:        setLibcallImplCallingConv(Impl, *CC);
+// CHECK-NEXT:    }
+// CHECK-NEXT:  };
 // CHECK-EMPTY:
 // CHECK-NEXT: if (TT.getArch() == Triple::blah) {
-// CHECK-NEXT:     static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:     setLibcallsImpl({
 // CHECK-NEXT:         {RTLIB::BZERO, RTLIB::bzero}, // bzero
 // CHECK-NEXT:         {RTLIB::CALLOC, RTLIB::calloc}, // calloc
 // CHECK-NEXT:         {RTLIB::SQRT_F128, RTLIB::sqrtl_f128}, // sqrtl
-// CHECK-NEXT:     };
-// CHECK-EMPTY:
-// CHECK-NEXT:     for (const auto [Func, Impl] : LibraryCalls) {
-// CHECK-NEXT:       setLibcallImpl(Func, Impl);
-// CHECK-NEXT:     }
+// CHECK-NEXT:     });
 // CHECK-EMPTY:
 // CHECK-NEXT:    if (TT.hasCompilerRT()) {
-// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_hasCompilerRT[] = {
+// CHECK-NEXT:      setLibcallsImpl({
 // CHECK-NEXT:          {RTLIB::SHL_I32, RTLIB::__ashlsi3}, // __ashlsi3
 // CHECK-NEXT:          {RTLIB::SRL_I64, RTLIB::__lshrdi3}, // __lshrdi3
-// CHECK-NEXT:      };
-// CHECK-EMPTY:
-// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_hasCompilerRT) {
-// CHECK-NEXT:        setLibcallImpl(Func, Impl);
-// CHECK-NEXT:      }
+// CHECK-NEXT:      });
 // CHECK-EMPTY:
 // CHECK-NEXT:    }
 // CHECK-EMPTY:
 // CHECK-NEXT:    if (TT.getOS() == Triple::bar) {
-// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_isBarOS[] = {
+// CHECK-NEXT:      setLibcallsImpl({
 // CHECK-NEXT:          {RTLIB::MEMSET, RTLIB::___memset}, // ___memset
-// CHECK-NEXT:      };
-// CHECK-EMPTY:
-// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_isBarOS) {
-// CHECK-NEXT:        setLibcallImpl(Func, Impl);
-// CHECK-NEXT:      }
+// CHECK-NEXT:      });
 // CHECK-EMPTY:
 // CHECK-NEXT:    }
 // CHECK-EMPTY:
@@ -194,37 +231,25 @@ def BlahLibrary : SystemRuntimeLibrary<isBlahArch, (add calloc, LibraryWithCondi
 // CHECK-NEXT: }
 // CHECK-EMPTY:
 // CHECK-NEXT: if (TT.getArch() == Triple::buzz) {
-// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:    setLibcallsImpl({
 // CHECK-NEXT:        {RTLIB::SHL_I32, RTLIB::__ashlsi3}, // __ashlsi3
 // CHECK-NEXT:        {RTLIB::SQRT_F80, RTLIB::sqrtl_f80}, // sqrtl
 // CHECK-NEXT:        {RTLIB::SRL_I64, RTLIB::__lshrdi3}, // __lshrdi3
-// CHECK-NEXT:    };
-// CHECK-EMPTY:
-// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
-// CHECK-NEXT:      setLibcallImpl(Func, Impl);
-// CHECK-NEXT:    }
+// CHECK-NEXT:    });
 // CHECK-EMPTY:
 // CHECK-NEXT:   return;
 // CHECK-NEXT: }
 // CHECK-EMPTY:
 // CHECK-NEXT: if (TT.getArch() == Triple::foo) {
-// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:    setLibcallsImpl({
 // CHECK-NEXT:        {RTLIB::BZERO, RTLIB::bzero}, // bzero
 // CHECK-NEXT:        {RTLIB::SQRT_F128, RTLIB::sqrtl_f128}, // sqrtl
-// CHECK-NEXT:    };
-// CHECK-EMPTY:
-// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
-// CHECK-NEXT:      setLibcallImpl(Func, Impl);
-// CHECK-NEXT:    }
+// CHECK-NEXT:    });
 // CHECK-EMPTY:
 // CHECK-NEXT:    if (TT.getOS() == Triple::bar) {
-// CHECK-NEXT:      static const LibcallImplPair LibraryCalls_isBarOS[] = {
+// CHECK-NEXT:      setLibcallsImpl({
 // CHECK-NEXT:          {RTLIB::MEMSET, RTLIB::___memset}, // ___memset
-// CHECK-NEXT:      };
-// CHECK-EMPTY:
-// CHECK-NEXT:      for (const auto [Func, Impl] : LibraryCalls_isBarOS) {
-// CHECK-NEXT:        setLibcallImpl(Func, Impl);
-// CHECK-NEXT:      }
+// CHECK-NEXT:      });
 // CHECK-EMPTY:
 // CHECK-NEXT:    }
 // CHECK-EMPTY:
@@ -232,16 +257,12 @@ def BlahLibrary : SystemRuntimeLibrary<isBlahArch, (add calloc, LibraryWithCondi
 // CHECK-NEXT:  }
 // CHECK-EMPTY:
 // CHECK-NEXT: if (TT.getArch() == Triple::simple) {
-// CHECK-NEXT:    static const LibcallImplPair LibraryCalls[] = {
+// CHECK-NEXT:    setLibcallsImpl({
 // CHECK-NEXT:        {RTLIB::CALLOC, RTLIB::calloc}, // calloc
 // CHECK-NEXT:        {RTLIB::SHL_I32, RTLIB::__ashlsi3}, // __ashlsi3
 // CHECK-NEXT:        {RTLIB::SQRT_F80, RTLIB::sqrtl_f80}, // sqrtl
 // CHECK-NEXT:        {RTLIB::SRL_I64, RTLIB::__lshrdi3}, // __lshrdi3
-// CHECK-NEXT:    };
-// CHECK-EMPTY:
-// CHECK-NEXT:    for (const auto [Func, Impl] : LibraryCalls) {
-// CHECK-NEXT:      setLibcallImpl(Func, Impl);
-// CHECK-NEXT:    }
+// CHECK-NEXT:    });
 // CHECK-EMPTY:
 // CHECK-NEXT:   return;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/get-named-operand-idx.td b/llvm/test/TableGen/get-named-operand-idx.td
index ab23edd54c723..e6f6331cd9c48 100644
--- a/llvm/test/TableGen/get-named-operand-idx.td
+++ b/llvm/test/TableGen/get-named-operand-idx.td
@@ -48,34 +48,70 @@ def InstD : InstBase {
   let UseNamedOperandTable = 0;
 }
 
-// CHECK: #ifdef GET_INSTRINFO_OPERAND_ENUM
-// CHECK: #undef GET_INSTRINFO_OPERAND_ENUM
-// CHECK: namespace llvm::MyNamespace {
-// CHECK: enum class OpName {
-// CHECK:   a = 0,
-// CHECK:   b = 1,
-// CHECK:   c = 2,
-// CHECK:   d = 3,
-// CHECK:   x = 4,
-// CHECK:   NUM_OPERAND_NAMES = 5,
-// CHECK: }; // enum class OpName
-// CHECK: } // end namespace llvm::MyNamespace
-// CHECK: #endif //GET_INSTRINFO_OPERAND_ENUM
+// CHECK-LABEL: #ifdef GET_INSTRINFO_OPERAND_ENUM
+// CHECK-NEXT:  #undef GET_INSTRINFO_OPERAND_ENUM
+// CHECK-NEXT:  namespace llvm::MyNamespace {
+// CHECK-NEXT:  enum class OpName : uint8_t {
+// CHECK-NEXT:    a = 0,
+// CHECK-NEXT:    b = 1,
+// CHECK-NEXT:    c = 2,
+// CHECK-NEXT:    d = 3,
+// CHECK-NEXT:    x = 4,
+// CHECK-NEXT:    NUM_OPERAND_NAMES = 5,
+// CHECK-NEXT:  }; // enum class OpName
+// CHECK-EMPTY:
+// CHECK-NEXT:  LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, OpName Name);
+// CHECK-NEXT:  LLVM_READONLY OpName getOperandIdxName(uint16_t Opcode, int16_t Idx);
+// CHECK-NEXT:  } // end namespace llvm::MyNamespace
+// CHECK-NEXT:  #endif //GET_INSTRINFO_OPERAND_ENUM
 
-// CHECK: #ifdef GET_INSTRINFO_NAMED_OPS
-// CHECK: #undef GET_INSTRINFO_NAMED_OPS
-// CHECK: namespace llvm::MyNamespace {
-// CHECK: LLVM_READONLY
-// CHECK: int16_t getNamedOperandIdx(uint16_t Opcode, OpName Name) {
-// CHECK:  assert(Name != OpName::NUM_OPERAND_NAMES);
-// CHECK:  static constexpr int8_t OperandMap[][5] = {
-// CHECK:      {0, 1, 2, -1, -1, },
-// CHECK:      {-1, -1, -1, 0, 1, },
-// CHECK:    };
-// CHECK:    static constexpr uint8_t InstructionIndex[] = {
-// CHECK:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK:    };
-// CHECK:    return OperandMap[InstructionIndex[Opcode]][(unsigned)Name];
-// CHECK:  }
-// CHECK:  } // end namespace llvm::MyNamespace
-// CHECK:  #endif //GET_INSTRINFO_NAMED_OPS
+// CHECK-LABEL: #ifdef GET_INSTRINFO_NAMED_OPS
+// CHECK-NEXT:  #undef GET_INSTRINFO_NAMED_OPS
+// CHECK-NEXT:  namespace llvm::MyNamespace {
+// CHECK-NEXT:  LLVM_READONLY static uint8_t getInstructionIndexForOpLookup(uint16_t Opcode) {
+// CHECK-NEXT:    static constexpr uint8_t InstructionIndex[] = {
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+// CHECK-NEXT:      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0,
+// CHECK-NEXT:    };
+// CHECK-NEXT:    return InstructionIndex[Opcode];
+// CHECK-NEXT:  }
+// CHECK-NEXT:  LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, OpName Name) {
+// CHECK-NEXT:    assert(Name != OpName::NUM_OPERAND_NAMES);
+// CHECK-NEXT:    static constexpr int8_t OperandMap[][5] = {
+// CHECK-NEXT:      {-1, -1, -1, -1, -1, },
+// CHECK-NEXT:      {0, 1, 2, -1, -1, },
+// CHECK-NEXT:      {-1, -1, -1, 0, 1, },
+// CHECK-NEXT:    };
+// CHECK-NEXT:    unsigned InstrIdx = getInstructionIndexForOpLookup(Opcode);
+// CHECK-NEXT:    return OperandMap[InstrIdx][(unsigned)Name];
+// CHECK-NEXT:  }
+// CHECK-NEXT:  LLVM_READONLY OpName getOperandIdxName(uint16_t Opcode, int16_t Idx) {
+// CHECK-NEXT:    assert(Idx >= 0 && Idx < 3);
+// CHECK-NEXT:    static constexpr OpName OperandMap[][3] = {
+// CHECK-NEXT:      {OpName::NUM_OPERAND_NAMES, OpName::NUM_OPERAND_NAMES, OpName::NUM_OPERAND_NAMES, },
+// CHECK-NEXT:      {OpName::a, OpName::b, OpName::c, },
+// CHECK-NEXT:      {OpName::d, OpName::x, OpName::NUM_OPERAND_NAMES, },
+// CHECK-NEXT:    };
+// CHECK-NEXT:    unsigned InstrIdx = getInstructionIndexForOpLookup(Opcode);
+// CHECK-NEXT:    return OperandMap[InstrIdx][(unsigned)Idx];
+// CHECK-NEXT:  }
+// CHECK-NEXT:  } // end namespace llvm::MyNamespace
+// CHECK-NEXT:  #endif //GET_INSTRINFO_NAMED_OPS
diff --git a/llvm/test/TableGen/intrinsic-attrs.td b/llvm/test/TableGen/intrinsic-attrs.td
index 18309d7419994..bcded0cd2e9f1 100644
--- a/llvm/test/TableGen/intrinsic-attrs.td
+++ b/llvm/test/TableGen/intrinsic-attrs.td
@@ -27,15 +27,16 @@ def int_deref_ptr_ret : Intrinsic<[llvm_ptr_ty], [], [Dereferenceable<RetIndex,
 // CHECK: static constexpr uint16_t IntrinsicsToAttributesMap[] = {
 // CHECK: 0 << 8 | 0, // llvm.deref.ptr.ret
 // CHECK: 1 << 8 | 1, // llvm.random.gen
+// CHECK: }; // IntrinsicsToAttributesMap
+
+// CHECK: static constexpr ArgNoAttrIDPair ArgAttrIdTable[] = {
+// CHECK-NEXT:   {0, 0},
+// CHECK: }; // ArgAttrIdTable
+
+// CHECK: static constexpr ArgAttributesInfo ArgAttributesInfoTable[] = {
+// CHECK-NEXT:  {0, 1},
+// CHECK-NEXT:  {0, 0},
+// CHECK-NEXT: }; // ArgAttributesInfoTable
 
 // CHECK: getAttributes(LLVMContext &C, ID id,
 // CHECK-NEXT: FunctionType *FT) {
-// CHECK: case 1:
-// CHECK-NEXT: return AttributeList::get(C, {
-// CHECK-NEXT:   {AttributeList::FunctionIndex, getIntrinsicFnAttributeSet(C, FnAttrID)}
-// CHECK-NEXT: });
-// CHECK-NEXT: case 0:
-// CHECK-NEXT: return AttributeList::get(C, {
-// CHECK-NEXT:   {0, getIntrinsicArgAttributeSet(C, 0, FT->getContainedType(0))},
-// CHECK-NEXT:   {AttributeList::FunctionIndex, getIntrinsicFnAttributeSet(C, FnAttrID)}
-// CHECK-NEXT: });
diff --git a/llvm/test/TableGen/math.td b/llvm/test/TableGen/math.td
index 64151a5ca0075..fe8faf0a79e33 100644
--- a/llvm/test/TableGen/math.td
+++ b/llvm/test/TableGen/math.td
@@ -3,6 +3,9 @@
 // RUN: not llvm-tblgen -DERROR2 %s 2>&1 | FileCheck --check-prefix=ERROR2 %s
 // RUN: not llvm-tblgen -DERROR3 %s 2>&1 | FileCheck --check-prefix=ERROR3 %s
 // RUN: not llvm-tblgen -DERROR4 %s 2>&1 | FileCheck --check-prefix=ERROR4 %s
+// RUN: not llvm-tblgen -DERROR5 %s 2>&1 | FileCheck --check-prefix=ERROR5 %s
+// RUN: not llvm-tblgen -DERROR6 %s 2>&1 | FileCheck --check-prefix=ERROR6 %s
+// RUN: not llvm-tblgen -DERROR7 %s 2>&1 | FileCheck --check-prefix=ERROR7 %s
 // XFAIL: vg_leak
 
 // CHECK: def shifts
@@ -143,3 +146,18 @@ def v954 : Int<!logtwo(-1)>;
 // CHECK: def vneg
 // CHECK: Value = -2
 def vneg : Int<!sub(v925.Value, 927)>;
+
+#ifdef ERROR5
+// ERROR5: error: Illegal operation: out of bounds shift
+def vshl_neg : Int<!shl(1, -1)>;
+#endif
+
+#ifdef ERROR6
+// ERROR6: error: Illegal operation: out of bounds shift
+def vsra_large : Int<!sra(1, 64)>;
+#endif
+
+#ifdef ERROR7
+// ERROR7: error: Illegal operation: out of bounds shift
+def vsrl_large : Int<!srl(1, 100)>;
+#endif
diff --git a/llvm/test/ThinLTO/AArch64/cgdata-merge-write.ll b/llvm/test/ThinLTO/AArch64/cgdata-merge-write.ll
index a4022eb885b43..47042d23cc2ca 100644
--- a/llvm/test/ThinLTO/AArch64/cgdata-merge-write.ll
+++ b/llvm/test/ThinLTO/AArch64/cgdata-merge-write.ll
@@ -36,9 +36,11 @@
 
 ; Merge the cgdata using llvm-cgdata.
 ; We now validate the content of the merged cgdata.
-; Two functions have the same hash with only one different constnat at a same location.
+; Two functions have the same hash with only one different constant at the same location.
 ; RUN: llvm-cgdata --merge -o %tout.cgdata %tout-nowrite.1 %tout-nowrite.2
 ; RUN: llvm-cgdata --convert %tout.cgdata   -o - | FileCheck %s
+; RUN: llvm-cgdata --merge -o %tout-lazy.cgdata %tout-nowrite.1 %tout-nowrite.2 -indexed-codegen-data-lazy-loading
+; RUN: llvm-cgdata --convert %tout-lazy.cgdata -indexed-codegen-data-lazy-loading -o - | FileCheck %s
 
 ; CHECK:      - Hash: [[#%d,HASH:]]
 ; CHECK-NEXT:   FunctionName: f1
diff --git a/llvm/test/ThinLTO/X86/import-metadata.ll b/llvm/test/ThinLTO/X86/import-metadata.ll
index f938fdd5c93c9..7175fb6bb2715 100644
--- a/llvm/test/ThinLTO/X86/import-metadata.ll
+++ b/llvm/test/ThinLTO/X86/import-metadata.ll
@@ -12,11 +12,12 @@
 
 ; CHECK: !llvm.dbg.cu = !{![[#CU1:]], ![[#CU2:]]}
 ;; Note that MD1 comes from the current module. MD2 is from the imported module. 
-;; We are checking if the imported MD2 doesn't end up having a null operand.
-; CHECK: !llvm.md = !{![[#MD1:]], ![[#MD2:]]}
+;; We are checking that MD2 is unified with MD1 and the imported MD2 doesn't
+;; end up having a null operand.
+; CHECK: !llvm.md = !{![[#MD1:]]}
 ; CHECK: ![[#MD3:]] = !{}
 ; CHECK: ![[#CU2]] = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: ![[#FILE2:]], isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug)
-; CHECK: ![[#MD2]] = !{![[#MD3]]}
+; CHECK: ![[#MD1]] = !{![[#MD3]]}
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-scei-ps4"
diff --git a/llvm/test/Transforms/AddDiscriminators/call.ll b/llvm/test/Transforms/AddDiscriminators/call.ll
index d093c65c4befa..93d3aa4c4a1c2 100644
--- a/llvm/test/Transforms/AddDiscriminators/call.ll
+++ b/llvm/test/Transforms/AddDiscriminators/call.ll
@@ -12,8 +12,8 @@ define void @_Z3foov() #0 !dbg !4 {
   call void @_Z3barv(), !dbg !10
 ; CHECK:  call void @_Z3barv(), !dbg ![[CALL0:[0-9]+]]
   %a = alloca [100 x i8], align 16
-  call void @llvm.lifetime.start.p0(i64 100, ptr %a), !dbg !11
-  call void @llvm.lifetime.end.p0(i64 100, ptr %a), !dbg !11
+  call void @llvm.lifetime.start.p0(ptr %a), !dbg !11
+  call void @llvm.lifetime.end.p0(ptr %a), !dbg !11
   call void @_Z3barv(), !dbg !11
 ; CHECK:  call void @_Z3barv(), !dbg ![[CALL1:[0-9]+]]
   call void @_Z3barv(), !dbg !12
@@ -22,8 +22,8 @@ define void @_Z3foov() #0 !dbg !4 {
 }
 
 declare void @_Z3barv() #1
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind argmemonly
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind argmemonly
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind argmemonly
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind argmemonly
 
 attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll b/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll
index f6716c817b739..46ec9e0a50842 100644
--- a/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll
+++ b/llvm/test/Transforms/AggressiveInstCombine/X86/or-load.ll
@@ -101,6 +101,107 @@ define i32 @loadCombine_4consecutive(ptr %p) {
   ret i32 %o3
 }
 
+define i32 @loadCombine_4consecutive_commuted(ptr %p) {
+; LE-LABEL: @loadCombine_4consecutive_commuted(
+; LE-NEXT:    [[O3:%.*]] = load i32, ptr [[P:%.*]], align 1
+; LE-NEXT:    ret i32 [[O3]]
+;
+; BE-LABEL: @loadCombine_4consecutive_commuted(
+; BE-NEXT:    [[P1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 1
+; BE-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P]], i32 2
+; BE-NEXT:    [[P3:%.*]] = getelementptr i8, ptr [[P]], i32 3
+; BE-NEXT:    [[L1:%.*]] = load i8, ptr [[P]], align 1
+; BE-NEXT:    [[L2:%.*]] = load i8, ptr [[P1]], align 1
+; BE-NEXT:    [[L3:%.*]] = load i8, ptr [[P2]], align 1
+; BE-NEXT:    [[L4:%.*]] = load i8, ptr [[P3]], align 1
+; BE-NEXT:    [[E1:%.*]] = zext i8 [[L1]] to i32
+; BE-NEXT:    [[E2:%.*]] = zext i8 [[L2]] to i32
+; BE-NEXT:    [[E3:%.*]] = zext i8 [[L3]] to i32
+; BE-NEXT:    [[E4:%.*]] = zext i8 [[L4]] to i32
+; BE-NEXT:    [[S2:%.*]] = shl i32 [[E2]], 8
+; BE-NEXT:    [[S3:%.*]] = shl i32 [[E3]], 16
+; BE-NEXT:    [[S4:%.*]] = shl i32 [[E4]], 24
+; BE-NEXT:    [[O1:%.*]] = or i32 [[S2]], [[S3]]
+; BE-NEXT:    [[O2:%.*]] = or i32 [[S4]], [[O1]]
+; BE-NEXT:    [[O3:%.*]] = or i32 [[E1]], [[O2]]
+; BE-NEXT:    ret i32 [[O3]]
+;
+  %p1 = getelementptr i8, ptr %p, i32 1
+  %p2 = getelementptr i8, ptr %p, i32 2
+  %p3 = getelementptr i8, ptr %p, i32 3
+  %l1 = load i8, ptr %p
+  %l2 = load i8, ptr %p1
+  %l3 = load i8, ptr %p2
+  %l4 = load i8, ptr %p3
+
+  %e1 = zext i8 %l1 to i32
+  %e2 = zext i8 %l2 to i32
+  %e3 = zext i8 %l3 to i32
+  %e4 = zext i8 %l4 to i32
+
+  %s2 = shl i32 %e2, 8
+  %s3 = shl i32 %e3, 16
+  %s4 = shl i32 %e4, 24
+
+  %o1 = or i32 %s2, %s3
+  %o2 = or i32 %s4, %o1
+  %o3 = or i32 %e1, %o2
+  ret i32 %o3
+}
+
+define i32 @loadCombine_4consecutive_multiuse(ptr %p) {
+; LE-LABEL: @loadCombine_4consecutive_multiuse(
+; LE-NEXT:    [[P3:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 3
+; LE-NEXT:    [[O3:%.*]] = load i32, ptr [[P]], align 1
+; LE-NEXT:    [[L4:%.*]] = load i8, ptr [[P3]], align 1
+; LE-NEXT:    call void @use(i8 [[L4]])
+; LE-NEXT:    ret i32 [[O3]]
+;
+; BE-LABEL: @loadCombine_4consecutive_multiuse(
+; BE-NEXT:    [[P1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 1
+; BE-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P]], i32 2
+; BE-NEXT:    [[P3:%.*]] = getelementptr i8, ptr [[P]], i32 3
+; BE-NEXT:    [[L1:%.*]] = load i8, ptr [[P]], align 1
+; BE-NEXT:    [[L2:%.*]] = load i8, ptr [[P1]], align 1
+; BE-NEXT:    [[L3:%.*]] = load i8, ptr [[P2]], align 1
+; BE-NEXT:    [[L4:%.*]] = load i8, ptr [[P3]], align 1
+; BE-NEXT:    call void @use(i8 [[L4]])
+; BE-NEXT:    [[E1:%.*]] = zext i8 [[L1]] to i32
+; BE-NEXT:    [[E2:%.*]] = zext i8 [[L2]] to i32
+; BE-NEXT:    [[E3:%.*]] = zext i8 [[L3]] to i32
+; BE-NEXT:    [[E4:%.*]] = zext i8 [[L4]] to i32
+; BE-NEXT:    [[S2:%.*]] = shl i32 [[E2]], 8
+; BE-NEXT:    [[S3:%.*]] = shl i32 [[E3]], 16
+; BE-NEXT:    [[S4:%.*]] = shl i32 [[E4]], 24
+; BE-NEXT:    [[O1:%.*]] = or i32 [[E1]], [[S2]]
+; BE-NEXT:    [[O2:%.*]] = or i32 [[O1]], [[S3]]
+; BE-NEXT:    [[O3:%.*]] = or i32 [[O2]], [[S4]]
+; BE-NEXT:    ret i32 [[O3]]
+;
+  %p1 = getelementptr i8, ptr %p, i32 1
+  %p2 = getelementptr i8, ptr %p, i32 2
+  %p3 = getelementptr i8, ptr %p, i32 3
+  %l1 = load i8, ptr %p
+  %l2 = load i8, ptr %p1
+  %l3 = load i8, ptr %p2
+  %l4 = load i8, ptr %p3
+  call void @use(i8 %l4)
+
+  %e1 = zext i8 %l1 to i32
+  %e2 = zext i8 %l2 to i32
+  %e3 = zext i8 %l3 to i32
+  %e4 = zext i8 %l4 to i32
+
+  %s2 = shl i32 %e2, 8
+  %s3 = shl i32 %e3, 16
+  %s4 = shl i32 %e4, 24
+
+  %o1 = or i32 %e1, %s2
+  %o2 = or i32 %o1, %s3
+  %o3 = or i32 %o2, %s4
+  ret i32 %o3
+}
+
 define i32 @loadCombine_4consecutive_BE(ptr %p) {
 ; LE-LABEL: @loadCombine_4consecutive_BE(
 ; LE-NEXT:    [[P1:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 1
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll
index 19cb7d21471d9..d821caa5b1c6b 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll
@@ -151,8 +151,17 @@ define i16 @test_atomicrmw_and_i16_global_system(ptr addrspace(1) %ptr, i16 %val
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
 ; CHECK-NEXT:    ret i16 [[EXTRACTED]]
 ;
@@ -204,8 +213,17 @@ define i16 @test_atomicrmw_or_i16_global_system(ptr addrspace(1) %ptr, i16 %valu
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw or ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
 ; CHECK-NEXT:    ret i16 [[EXTRACTED]]
 ;
@@ -224,8 +242,17 @@ define i16 @test_atomicrmw_xor_i16_global_system(ptr addrspace(1) %ptr, i16 %val
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw xor ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
 ; CHECK-NEXT:    ret i16 [[EXTRACTED]]
 ;
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
index c49909597c72c..72fc4f468543a 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
@@ -140,33 +140,74 @@ define i16 @test_atomicrmw_sub_i16_global_agent(ptr addrspace(1) %ptr, i16 %valu
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent(
-; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
-; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; R600-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; R600-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; R600-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; R600-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent_align4(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_align4(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4(
+; GCN-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; GCN-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4(
+; R600-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; R600-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4
   ret i16 %res
@@ -174,21 +215,46 @@ define i16 @test_atomicrmw_and_i16_global_agent_align4(ptr addrspace(1) %ptr, i1
 
 ; Drop unknown metadata and noundef
 define i16 @test_atomicrmw_and_i16_global_agent_drop_md(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_drop_md(
-; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
-; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_drop_md(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_drop_md(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; R600-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; R600-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; R600-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; R600-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !noundef !0, !some.unknown.md !0
   ret i16 %res
@@ -196,12 +262,28 @@ define i16 @test_atomicrmw_and_i16_global_agent_drop_md(ptr addrspace(1) %ptr, i
 
 ; Drop unknown metadata
 define i16 @test_atomicrmw_and_i16_global_agent_align4_drop_md(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_align4_drop_md(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_drop_md(
+; GCN-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; GCN-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_drop_md(
+; R600-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; R600-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !noundef !0, !some.unknown.md !0
   ret i16 %res
@@ -209,21 +291,46 @@ define i16 @test_atomicrmw_and_i16_global_agent_align4_drop_md(ptr addrspace(1)
 
 ; Drop noundef, preserve mmra
 define i16 @test_atomicrmw_and_i16_global_agent_preserve_mmra(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_preserve_mmra(
-; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
-; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !mmra [[META0:![0-9]+]]
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_preserve_mmra(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4, !mmra [[META0:![0-9]+]]
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4, !mmra [[META0]]
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_preserve_mmra(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; R600-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; R600-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; R600-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; R600-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !mmra [[META0:![0-9]+]]
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !noundef !0, !mmra !1
   ret i16 %res
@@ -231,126 +338,252 @@ define i16 @test_atomicrmw_and_i16_global_agent_preserve_mmra(ptr addrspace(1) %
 
 ; Drop noundef, preserve mmra
 define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_mmra(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_mmra(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !mmra [[META0]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_mmra(
+; GCN-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; GCN-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4, !mmra [[META0]]
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4, !mmra [[META0]]
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_mmra(
+; R600-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; R600-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !mmra [[META0]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !noundef !0, !mmra !1
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_alias_scope(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_alias_scope(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !alias.scope [[META1:![0-9]+]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_alias_scope(
+; GCN-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; GCN-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_alias_scope(
+; R600-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; R600-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !alias.scope [[META1:![0-9]+]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !alias.scope !2
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_noalias(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_noalias(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !noalias [[META1]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_noalias(
+; GCN-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; GCN-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_noalias(
+; R600-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; R600-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !noalias [[META1]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !noalias !2
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa_struct(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa_struct(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !tbaa.struct [[TBAA_STRUCT4:![0-9]+]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa_struct(
+; GCN-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; GCN-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa_struct(
+; R600-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; R600-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !tbaa.struct [[TBAA_STRUCT4:![0-9]+]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !tbaa.struct !5
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !tbaa [[TBAA5:![0-9]+]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa(
+; GCN-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; GCN-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(1) [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4_preserve_tbaa(
+; R600-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; R600-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !tbaa [[TBAA5:![0-9]+]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !tbaa !6
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_remote_memory(
-; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
-; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META8:![0-9]+]]
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_remote_memory(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; GCN-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META1:![0-9]+]]
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_remote_memory(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; R600-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; R600-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; R600-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; R600-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META8:![0-9]+]]
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_remote_memory(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META8]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_remote_memory(
+; GCN-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; GCN-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META1]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_remote_memory(
+; R600-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; R600-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory [[META8]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !amdgpu.no.remote.memory !0
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_fine_grained_memory(
-; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
-; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META8]]
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_fine_grained_memory(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; GCN-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META1]]
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent__amdgpu_no_fine_grained_memory(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; R600-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; R600-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; R600-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
+; R600-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META8]]
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_fine_grained_memory(
-; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
-; CHECK-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META8]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_fine_grained_memory(
+; GCN-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; GCN-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META1]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_and_i16_global_agent_align4__amdgpu_no_fine_grained_memory(
+; R600-NEXT:    [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[TMP1]], -65536
+; R600-NEXT:    [[TMP2:%.*]] = atomicrmw and ptr addrspace(1) [[PTR:%.*]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META8]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[TMP2]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
   ret i16 %res
@@ -390,40 +623,88 @@ define i16 @test_atomicrmw_nand_i16_global_agent(ptr addrspace(1) %ptr, i16 %val
 }
 
 define i16 @test_atomicrmw_or_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_or_i16_global_agent(
-; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
-; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw or ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] syncscope("agent") seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_or_i16_global_agent(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_or_i16_global_agent(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; R600-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; R600-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; R600-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; R600-NEXT:    [[TMP4:%.*]] = atomicrmw or ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] syncscope("agent") seq_cst, align 4
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst
   ret i16 %res
 }
 
 define i16 @test_atomicrmw_xor_i16_global_agent(ptr addrspace(1) %ptr, i16 %value) {
-; CHECK-LABEL: @test_atomicrmw_xor_i16_global_agent(
-; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
-; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
-; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
-; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
-; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
-; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw xor ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] syncscope("agent") seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
-; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+; GCN-LABEL: @test_atomicrmw_xor_i16_global_agent(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; GCN-NEXT:    ret i16 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_xor_i16_global_agent(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; R600-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; R600-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; R600-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; R600-NEXT:    [[TMP4:%.*]] = atomicrmw xor ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] syncscope("agent") seq_cst, align 4
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; R600-NEXT:    ret i16 [[EXTRACTED]]
 ;
   %res = atomicrmw xor ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst
   ret i16 %res
@@ -1362,6 +1643,102 @@ define i16 @test_atomicrmw_add_i16_buffer_fat_agent_align4(ptr addrspace(7) %ptr
   ret i16 %res
 }
 
+define i16 @test_atomicrmw_sub_i16_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i16 %value) {
+; CHECK-LABEL: @test_atomicrmw_sub_i16_global_agent__amdgpu_no_remote_memory(
+; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+;
+  %res = atomicrmw sub ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+  ret i16 %res
+}
+
+define i16 @test_atomicrmw_sub_i16_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i16 %value) {
+; CHECK-LABEL: @test_atomicrmw_sub_i16_global_agent__amdgpu_no_fine_grained_memory(
+; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+;
+  %res = atomicrmw sub ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+  ret i16 %res
+}
+
+define i16 @test_atomicrmw_sub_i16_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i16 %value) {
+; CHECK-LABEL: @test_atomicrmw_sub_i16_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; CHECK-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; CHECK-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; CHECK-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT:    [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
+; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
+; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
+; CHECK-NEXT:    ret i16 [[EXTRACTED]]
+;
+  %res = atomicrmw sub ptr addrspace(1) %ptr, i16 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+  ret i16 %res
+}
+
 !0 = !{}
 !1 = !{!"foo", !"bar"}
 !2 = !{!3}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-agent.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-agent.ll
index a1007bacd522f..1440045d11e2d 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-agent.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-agent.ll
@@ -110,10 +110,99 @@ define i32 @test_atomicrmw_add_i32_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, sub is not supported over PCIe
 define i32 @test_atomicrmw_sub_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_sub_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_sub_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_sub_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_sub_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_sub_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_sub_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_sub_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_sub_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_sub_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw sub ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -155,10 +244,99 @@ define i32 @test_atomicrmw_sub_i32_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, operation not supported over PCIe
 define i32 @test_atomicrmw_and_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_and_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_and_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_and_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_and_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_and_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_and_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_and_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_and_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_and_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -285,10 +463,99 @@ define i32 @test_atomicrmw_nand_i32_global_agent__amdgpu_no_fine_grained_memory_
 
 ; expansion is necessary, operation not supported over PCIe
 define i32 @test_atomicrmw_or_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_or_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_or_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_or_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_or_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_or_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_or_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_or_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_or_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_or_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -330,10 +597,99 @@ define i32 @test_atomicrmw_or_i32_global_agent__amdgpu_no_fine_grained_memory__a
 
 ; expansion is necessary, operation not supported over PCIe
 define i32 @test_atomicrmw_xor_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_xor_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_xor_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_xor_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_xor_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_xor_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_xor_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_xor_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_xor_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_xor_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw xor ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -375,10 +731,105 @@ define i32 @test_atomicrmw_xor_i32_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, operation not supported over PCIe
 define i32 @test_atomicrmw_max_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_max_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_max_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX803-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_max_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX906-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_max_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_max_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX90A-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_max_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_max_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX10-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_max_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_max_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw max ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -420,10 +871,105 @@ define i32 @test_atomicrmw_max_i32_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, operation not supported over PCIe
 define i32 @test_atomicrmw_min_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_min_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_min_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX803-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_min_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX906-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_min_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_min_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX90A-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_min_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_min_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX10-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_min_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_min_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw min ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -465,10 +1011,105 @@ define i32 @test_atomicrmw_min_i32_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, operation not supported over PCIe
 define i32 @test_atomicrmw_umax_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_umax_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_umax_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX803-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_umax_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX906-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_umax_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_umax_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX90A-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_umax_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_umax_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX10-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_umax_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_umax_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw umax ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -510,10 +1151,105 @@ define i32 @test_atomicrmw_umax_i32_global_agent__amdgpu_no_fine_grained_memory_
 
 ; expansion is necessary, operation not supported over PCIe
 define i32 @test_atomicrmw_umin_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_umin_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_umin_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX803-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_umin_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX906-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_umin_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_umin_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX90A-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_umin_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_umin_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX10-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_umin_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_umin_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw umin ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -555,10 +1291,111 @@ define i32 @test_atomicrmw_umin_i32_global_agent__amdgpu_no_fine_grained_memory_
 
 ; expansion is necessary, operation not supported over PCIe
 define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; GFX803-NEXT:    [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; GFX906-NEXT:    [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; GFX908-NEXT:    [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; GFX90A-NEXT:    [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; GFX90A-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; GFX10-NEXT:    [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; GFX11-NEXT:    [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -600,10 +1437,123 @@ define i32 @test_atomicrmw_uinc_wrap_i32_global_agent__amdgpu_no_fine_grained_me
 
 ; expansion is necessary, operation not supported over PCIe
 define i32 @test_atomicrmw_udec_wrap_i32_global_agent(ptr addrspace(1) %ptr, i32 %value) {
-; COMMON-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
-; COMMON-NEXT:    ret i32 [[NEWLOADED]]
+; GFX803-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; GFX803-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; GFX803-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VALUE]], i32 [[TMP2]]
+; GFX803-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; GFX906-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; GFX906-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VALUE]], i32 [[TMP2]]
+; GFX906-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; GFX908-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; GFX908-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VALUE]], i32 [[TMP2]]
+; GFX908-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; GFX90A-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; GFX90A-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VALUE]], i32 [[TMP2]]
+; GFX90A-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX940-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; GFX10-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; GFX10-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VALUE]], i32 [[TMP2]]
+; GFX10-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; GFX11-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; GFX11-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VALUE]], i32 [[TMP2]]
+; GFX11-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i32 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i32 [[VALUE]] syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %value syncscope("agent") seq_cst
   ret i32 %res
@@ -657,12 +1607,3 @@ define i32 @test_atomicrmw_udec_wrap_i32_global_agent__amdgpu_no_fine_grained_me
 ;.
 ; GFX12: [[META0]] = !{}
 ;.
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10: {{.*}}
-; GFX11: {{.*}}
-; GFX12: {{.*}}
-; GFX803: {{.*}}
-; GFX906: {{.*}}
-; GFX908: {{.*}}
-; GFX90A: {{.*}}
-; GFX942: {{.*}}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-system.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-system.ll
index 08288848efd66..b8e7c3eb4673a 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-system.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-system.ll
@@ -112,7 +112,16 @@ define i32 @test_atomicrmw_add_i32_global_system__amdgpu_no_fine_grained_memory_
 define i32 @test_atomicrmw_sub_i32_global_system(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_sub_i32_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4
+; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw sub ptr addrspace(1) %ptr, i32 %value seq_cst
@@ -157,7 +166,16 @@ define i32 @test_atomicrmw_sub_i32_global_system__amdgpu_no_fine_grained_memory_
 define i32 @test_atomicrmw_and_i32_global_system(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_and_i32_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4
+; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i32 %value seq_cst
@@ -287,7 +305,16 @@ define i32 @test_atomicrmw_nand_i32_global_system__amdgpu_no_fine_grained_memory
 define i32 @test_atomicrmw_or_i32_global_system(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_or_i32_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4
+; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i32 %value seq_cst
@@ -332,7 +359,16 @@ define i32 @test_atomicrmw_or_i32_global_system__amdgpu_no_fine_grained_memory__
 define i32 @test_atomicrmw_xor_i32_global_system(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_xor_i32_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4
+; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP2]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw xor ptr addrspace(1) %ptr, i32 %value seq_cst
@@ -397,17 +433,7 @@ define i32 @test_atomicrmw_max_i32_global_system(ptr addrspace(1) %ptr, i32 %val
 define i32 @test_atomicrmw_max_i32_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_max_i32_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw max ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -417,17 +443,7 @@ define i32 @test_atomicrmw_max_i32_global_system__amdgpu_no_fine_grained_memory(
 define i32 @test_atomicrmw_max_i32_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_max_i32_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw max ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.remote.memory !0
@@ -437,17 +453,7 @@ define i32 @test_atomicrmw_max_i32_global_system__amdgpu_no_remote_memory(ptr ad
 define i32 @test_atomicrmw_max_i32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_max_i32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw max ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -482,17 +488,7 @@ define i32 @test_atomicrmw_min_i32_global_system(ptr addrspace(1) %ptr, i32 %val
 define i32 @test_atomicrmw_min_i32_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_min_i32_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw min ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -502,17 +498,7 @@ define i32 @test_atomicrmw_min_i32_global_system__amdgpu_no_fine_grained_memory(
 define i32 @test_atomicrmw_min_i32_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_min_i32_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw min ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.remote.memory !0
@@ -522,17 +508,7 @@ define i32 @test_atomicrmw_min_i32_global_system__amdgpu_no_remote_memory(ptr ad
 define i32 @test_atomicrmw_min_i32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_min_i32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw min ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -567,17 +543,7 @@ define i32 @test_atomicrmw_umax_i32_global_system(ptr addrspace(1) %ptr, i32 %va
 define i32 @test_atomicrmw_umax_i32_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_umax_i32_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw umax ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -587,17 +553,7 @@ define i32 @test_atomicrmw_umax_i32_global_system__amdgpu_no_fine_grained_memory
 define i32 @test_atomicrmw_umax_i32_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_umax_i32_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw umax ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.remote.memory !0
@@ -607,17 +563,7 @@ define i32 @test_atomicrmw_umax_i32_global_system__amdgpu_no_remote_memory(ptr a
 define i32 @test_atomicrmw_umax_i32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_umax_i32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw umax ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -652,17 +598,7 @@ define i32 @test_atomicrmw_umin_i32_global_system(ptr addrspace(1) %ptr, i32 %va
 define i32 @test_atomicrmw_umin_i32_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_umin_i32_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw umin ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -672,17 +608,7 @@ define i32 @test_atomicrmw_umin_i32_global_system__amdgpu_no_fine_grained_memory
 define i32 @test_atomicrmw_umin_i32_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_umin_i32_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw umin ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.remote.memory !0
@@ -692,17 +618,7 @@ define i32 @test_atomicrmw_umin_i32_global_system__amdgpu_no_remote_memory(ptr a
 define i32 @test_atomicrmw_umin_i32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_umin_i32_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ule i32 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i32 [[LOADED]], i32 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw umin ptr addrspace(1) %ptr, i32 %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -717,7 +633,18 @@ define i32 @test_atomicrmw_umin_i32_global_system__amdgpu_no_fine_grained_memory
 define i32 @test_atomicrmw_uinc_wrap_i32_global_system(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_uinc_wrap_i32_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4
+; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[TMP2:%.*]] = add i32 [[LOADED]], 1
+; COMMON-NEXT:    [[TMP3:%.*]] = icmp uge i32 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 0, i32 [[TMP2]]
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP4]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i32 %value seq_cst
@@ -762,7 +689,20 @@ define i32 @test_atomicrmw_uinc_wrap_i32_global_system__amdgpu_no_fine_grained_m
 define i32 @test_atomicrmw_udec_wrap_i32_global_system(ptr addrspace(1) %ptr, i32 %value) {
 ; COMMON-LABEL: define i32 @test_atomicrmw_udec_wrap_i32_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i32 [[VALUE]] seq_cst, align 4
+; COMMON-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[TMP2:%.*]] = sub i32 [[LOADED]], 1
+; COMMON-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[LOADED]], 0
+; COMMON-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i32 [[VALUE]], i32 [[TMP2]]
+; COMMON-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i32 [[NEWLOADED]]
 ;
   %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i32 %value seq_cst
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-agent.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-agent.ll
index 7586a0af43c95..8bc481408fe73 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-agent.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-agent.ll
@@ -110,10 +110,99 @@ define i64 @test_atomicrmw_add_i64_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, sub is not supported over PCIe
 define i64 @test_atomicrmw_sub_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_sub_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw sub ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -155,10 +244,99 @@ define i64 @test_atomicrmw_sub_i64_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, operation not supported over PCIe
 define i64 @test_atomicrmw_and_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_and_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_and_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_and_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_and_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_and_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_and_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_and_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_and_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_and_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -285,10 +463,99 @@ define i64 @test_atomicrmw_nand_i64_global_agent__amdgpu_no_fine_grained_memory_
 
 ; expansion is necessary, operation not supported over PCIe
 define i64 @test_atomicrmw_or_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_or_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_or_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_or_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_or_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_or_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_or_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_or_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_or_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_or_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -330,10 +597,99 @@ define i64 @test_atomicrmw_or_i64_global_agent__amdgpu_no_fine_grained_memory__a
 
 ; expansion is necessary, operation not supported over PCIe
 define i64 @test_atomicrmw_xor_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_xor_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw xor ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -375,10 +731,105 @@ define i64 @test_atomicrmw_xor_i64_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, operation not supported over PCIe
 define i64 @test_atomicrmw_max_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_max_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_max_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX803-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_max_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX906-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_max_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_max_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX90A-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_max_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_max_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX10-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_max_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_max_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw max ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -420,10 +871,105 @@ define i64 @test_atomicrmw_max_i64_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, operation not supported over PCIe
 define i64 @test_atomicrmw_min_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_min_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_min_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX803-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_min_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX906-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_min_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_min_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX90A-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_min_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_min_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX10-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_min_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_min_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw min ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -465,10 +1011,105 @@ define i64 @test_atomicrmw_min_i64_global_agent__amdgpu_no_fine_grained_memory__
 
 ; expansion is necessary, operation not supported over PCIe
 define i64 @test_atomicrmw_umax_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX803-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX906-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX90A-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX10-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_umax_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw umax ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -510,10 +1151,105 @@ define i64 @test_atomicrmw_umax_i64_global_agent__amdgpu_no_fine_grained_memory_
 
 ; expansion is necessary, operation not supported over PCIe
 define i64 @test_atomicrmw_umin_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX803-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX906-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX908-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX90A-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX10-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
+; GFX11-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_umin_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw umin ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -555,10 +1291,111 @@ define i64 @test_atomicrmw_umin_i64_global_agent__amdgpu_no_fine_grained_memory_
 
 ; expansion is necessary, operation not supported over PCIe
 define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = add i64 [[LOADED]], 1
+; GFX803-NEXT:    [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]]
+; GFX803-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = add i64 [[LOADED]], 1
+; GFX906-NEXT:    [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]]
+; GFX906-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = add i64 [[LOADED]], 1
+; GFX908-NEXT:    [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]]
+; GFX908-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = add i64 [[LOADED]], 1
+; GFX90A-NEXT:    [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]]
+; GFX90A-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = add i64 [[LOADED]], 1
+; GFX10-NEXT:    [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]]
+; GFX10-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = add i64 [[LOADED]], 1
+; GFX11-NEXT:    [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]]
+; GFX11-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -600,10 +1437,123 @@ define i64 @test_atomicrmw_uinc_wrap_i64_global_agent__amdgpu_no_fine_grained_me
 
 ; expansion is necessary, operation not supported over PCIe
 define i64 @test_atomicrmw_udec_wrap_i64_global_agent(ptr addrspace(1) %ptr, i64 %value) {
-; COMMON-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent(
-; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
-; COMMON-NEXT:    ret i64 [[NEWLOADED]]
+; GFX803-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX803-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX803:       atomicrmw.start:
+; GFX803-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX803-NEXT:    [[TMP2:%.*]] = sub i64 [[LOADED]], 1
+; GFX803-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0
+; GFX803-NEXT:    [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX803-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX803-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]]
+; GFX803-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX803-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; GFX803-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0
+; GFX803-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX803:       atomicrmw.end:
+; GFX803-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX906-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent(
+; GFX906-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX906-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX906-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX906:       atomicrmw.start:
+; GFX906-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX906-NEXT:    [[TMP2:%.*]] = sub i64 [[LOADED]], 1
+; GFX906-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0
+; GFX906-NEXT:    [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX906-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX906-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]]
+; GFX906-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX906-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; GFX906-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0
+; GFX906-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX906:       atomicrmw.end:
+; GFX906-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent(
+; GFX908-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX908-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX908:       atomicrmw.start:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[TMP2:%.*]] = sub i64 [[LOADED]], 1
+; GFX908-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0
+; GFX908-NEXT:    [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX908-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]]
+; GFX908-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX908:       atomicrmw.end:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX90A-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX90A:       atomicrmw.start:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[TMP2:%.*]] = sub i64 [[LOADED]], 1
+; GFX90A-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0
+; GFX90A-NEXT:    [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX90A-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]]
+; GFX90A-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX90A:       atomicrmw.end:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent(
+; GFX940-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX10-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX10-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX10:       atomicrmw.start:
+; GFX10-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX10-NEXT:    [[TMP2:%.*]] = sub i64 [[LOADED]], 1
+; GFX10-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0
+; GFX10-NEXT:    [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX10-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX10-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]]
+; GFX10-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX10-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; GFX10-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0
+; GFX10-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX10:       atomicrmw.end:
+; GFX10-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX11-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; GFX11-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX11:       atomicrmw.start:
+; GFX11-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GFX11-NEXT:    [[TMP2:%.*]] = sub i64 [[LOADED]], 1
+; GFX11-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0
+; GFX11-NEXT:    [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; GFX11-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; GFX11-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]]
+; GFX11-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8
+; GFX11-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; GFX11-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0
+; GFX11-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX11:       atomicrmw.end:
+; GFX11-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX12-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %value syncscope("agent") seq_cst
   ret i64 %res
@@ -657,12 +1607,3 @@ define i64 @test_atomicrmw_udec_wrap_i64_global_agent__amdgpu_no_fine_grained_me
 ;.
 ; GFX12: [[META0]] = !{}
 ;.
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10: {{.*}}
-; GFX11: {{.*}}
-; GFX12: {{.*}}
-; GFX803: {{.*}}
-; GFX906: {{.*}}
-; GFX908: {{.*}}
-; GFX90A: {{.*}}
-; GFX942: {{.*}}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-system.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-system.ll
index 4f3979f25076e..c895eebcf0f8d 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-system.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-system.ll
@@ -112,7 +112,16 @@ define i64 @test_atomicrmw_add_i64_global_system__amdgpu_no_fine_grained_memory_
 define i64 @test_atomicrmw_sub_i64_global_system(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_sub_i64_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8
+; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw sub ptr addrspace(1) %ptr, i64 %value seq_cst
@@ -157,7 +166,16 @@ define i64 @test_atomicrmw_sub_i64_global_system__amdgpu_no_fine_grained_memory_
 define i64 @test_atomicrmw_and_i64_global_system(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_and_i64_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw and ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8
+; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw and ptr addrspace(1) %ptr, i64 %value seq_cst
@@ -287,7 +305,16 @@ define i64 @test_atomicrmw_nand_i64_global_system__amdgpu_no_fine_grained_memory
 define i64 @test_atomicrmw_or_i64_global_system(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_or_i64_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8
+; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = or i64 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i64 %value seq_cst
@@ -332,7 +359,16 @@ define i64 @test_atomicrmw_or_i64_global_system__amdgpu_no_fine_grained_memory__
 define i64 @test_atomicrmw_xor_i64_global_system(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_xor_i64_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8
+; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[NEW:%.*]] = xor i64 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw xor ptr addrspace(1) %ptr, i64 %value seq_cst
@@ -397,17 +433,7 @@ define i64 @test_atomicrmw_max_i64_global_system(ptr addrspace(1) %ptr, i64 %val
 define i64 @test_atomicrmw_max_i64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_max_i64_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw max ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -417,17 +443,7 @@ define i64 @test_atomicrmw_max_i64_global_system__amdgpu_no_fine_grained_memory(
 define i64 @test_atomicrmw_max_i64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_max_i64_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw max ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.remote.memory !0
@@ -437,17 +453,7 @@ define i64 @test_atomicrmw_max_i64_global_system__amdgpu_no_remote_memory(ptr ad
 define i64 @test_atomicrmw_max_i64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_max_i64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sgt i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw max ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw max ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -482,17 +488,7 @@ define i64 @test_atomicrmw_min_i64_global_system(ptr addrspace(1) %ptr, i64 %val
 define i64 @test_atomicrmw_min_i64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_min_i64_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw min ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -502,17 +498,7 @@ define i64 @test_atomicrmw_min_i64_global_system__amdgpu_no_fine_grained_memory(
 define i64 @test_atomicrmw_min_i64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_min_i64_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw min ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.remote.memory !0
@@ -522,17 +508,7 @@ define i64 @test_atomicrmw_min_i64_global_system__amdgpu_no_remote_memory(ptr ad
 define i64 @test_atomicrmw_min_i64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_min_i64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp sle i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw min ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw min ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -567,17 +543,7 @@ define i64 @test_atomicrmw_umax_i64_global_system(ptr addrspace(1) %ptr, i64 %va
 define i64 @test_atomicrmw_umax_i64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_umax_i64_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw umax ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -587,17 +553,7 @@ define i64 @test_atomicrmw_umax_i64_global_system__amdgpu_no_fine_grained_memory
 define i64 @test_atomicrmw_umax_i64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_umax_i64_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw umax ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.remote.memory !0
@@ -607,17 +563,7 @@ define i64 @test_atomicrmw_umax_i64_global_system__amdgpu_no_remote_memory(ptr a
 define i64 @test_atomicrmw_umax_i64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_umax_i64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umax ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw umax ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -652,17 +598,7 @@ define i64 @test_atomicrmw_umin_i64_global_system(ptr addrspace(1) %ptr, i64 %va
 define i64 @test_atomicrmw_umin_i64_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_umin_i64_global_system__amdgpu_no_fine_grained_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw umin ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -672,17 +608,7 @@ define i64 @test_atomicrmw_umin_i64_global_system__amdgpu_no_fine_grained_memory
 define i64 @test_atomicrmw_umin_i64_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_umin_i64_global_system__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw umin ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.remote.memory !0
@@ -692,17 +618,7 @@ define i64 @test_atomicrmw_umin_i64_global_system__amdgpu_no_remote_memory(ptr a
 define i64 @test_atomicrmw_umin_i64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_umin_i64_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
-; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
-; COMMON:       atomicrmw.start:
-; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
-; COMMON-NEXT:    [[TMP2:%.*]] = icmp ule i64 [[LOADED]], [[VALUE]]
-; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP2]], i64 [[LOADED]], i64 [[VALUE]]
-; COMMON-NEXT:    [[TMP3:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
-; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP3]], 1
-; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP3]], 0
-; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
-; COMMON:       atomicrmw.end:
+; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw umin ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw umin ptr addrspace(1) %ptr, i64 %value seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -717,7 +633,18 @@ define i64 @test_atomicrmw_umin_i64_global_system__amdgpu_no_fine_grained_memory
 define i64 @test_atomicrmw_uinc_wrap_i64_global_system(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_uinc_wrap_i64_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw uinc_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8
+; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[TMP2:%.*]] = add i64 [[LOADED]], 1
+; COMMON-NEXT:    [[TMP3:%.*]] = icmp uge i64 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i64 0, i64 [[TMP2]]
+; COMMON-NEXT:    [[TMP4:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP4]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw uinc_wrap ptr addrspace(1) %ptr, i64 %value seq_cst
@@ -762,7 +689,20 @@ define i64 @test_atomicrmw_uinc_wrap_i64_global_system__amdgpu_no_fine_grained_m
 define i64 @test_atomicrmw_udec_wrap_i64_global_system(ptr addrspace(1) %ptr, i64 %value) {
 ; COMMON-LABEL: define i64 @test_atomicrmw_udec_wrap_i64_global_system(
 ; COMMON-SAME: ptr addrspace(1) [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; COMMON-NEXT:    [[NEWLOADED:%.*]] = atomicrmw udec_wrap ptr addrspace(1) [[PTR]], i64 [[VALUE]] seq_cst, align 8
+; COMMON-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspace(1) [[PTR]], align 8
+; COMMON-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; COMMON:       atomicrmw.start:
+; COMMON-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; COMMON-NEXT:    [[TMP2:%.*]] = sub i64 [[LOADED]], 1
+; COMMON-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[LOADED]], 0
+; COMMON-NEXT:    [[TMP4:%.*]] = icmp ugt i64 [[LOADED]], [[VALUE]]
+; COMMON-NEXT:    [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]]
+; COMMON-NEXT:    [[NEW:%.*]] = select i1 [[TMP5]], i64 [[VALUE]], i64 [[TMP2]]
+; COMMON-NEXT:    [[TMP6:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i64 [[LOADED]], i64 [[NEW]] seq_cst seq_cst, align 8
+; COMMON-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP6]], 1
+; COMMON-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP6]], 0
+; COMMON-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; COMMON:       atomicrmw.end:
 ; COMMON-NEXT:    ret i64 [[NEWLOADED]]
 ;
   %res = atomicrmw udec_wrap ptr addrspace(1) %ptr, i64 %value seq_cst
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8-system.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8-system.ll
index 088371f461ec1..b548943a326b8 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8-system.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8-system.ll
@@ -162,8 +162,17 @@ define i8 @test_atomicrmw_and_i8_global_system(ptr addrspace(1) %ptr, i8 %value)
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
 ; CHECK-NEXT:    ret i8 [[EXTRACTED]]
 ;
@@ -215,8 +224,17 @@ define i8 @test_atomicrmw_or_i8_global_system(ptr addrspace(1) %ptr, i8 %value)
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw or ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
 ; CHECK-NEXT:    ret i8 [[EXTRACTED]]
 ;
@@ -235,8 +253,17 @@ define i8 @test_atomicrmw_xor_i8_global_system(ptr addrspace(1) %ptr, i8 %value)
 ; CHECK-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
 ; CHECK-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; CHECK-NEXT:    [[TMP4:%.*]] = atomicrmw xor ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst, align 4
-; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
 ; CHECK-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
 ; CHECK-NEXT:    ret i8 [[EXTRACTED]]
 ;
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll
index 590ee63001615..b19a717b56938 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll
@@ -265,8 +265,17 @@ define i8 @test_atomicrmw_and_i8_global_agent(ptr addrspace(1) %ptr, i8 %value)
 ; GCN-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
 ; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
 ; GCN-NEXT:    [[ANDOPERAND:%.*]] = or i32 [[VALOPERAND_SHIFTED]], [[INV_MASK]]
-; GCN-NEXT:    [[TMP4:%.*]] = atomicrmw and ptr addrspace(1) [[ALIGNEDADDR]], i32 [[ANDOPERAND]] syncscope("agent") seq_cst, align 4
-; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = and i32 [[LOADED]], [[ANDOPERAND]]
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
 ; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
 ; GCN-NEXT:    ret i8 [[EXTRACTED]]
 ;
@@ -360,8 +369,17 @@ define i8 @test_atomicrmw_or_i8_global_agent(ptr addrspace(1) %ptr, i8 %value) {
 ; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
 ; GCN-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
 ; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; GCN-NEXT:    [[TMP4:%.*]] = atomicrmw or ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] syncscope("agent") seq_cst, align 4
-; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
 ; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
 ; GCN-NEXT:    ret i8 [[EXTRACTED]]
 ;
@@ -394,8 +412,17 @@ define i8 @test_atomicrmw_xor_i8_global_agent(ptr addrspace(1) %ptr, i8 %value)
 ; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
 ; GCN-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
 ; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
-; GCN-NEXT:    [[TMP4:%.*]] = atomicrmw xor ptr addrspace(1) [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] syncscope("agent") seq_cst, align 4
-; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[TMP4]], [[SHIFTAMT]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = xor i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[NEW]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
 ; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
 ; GCN-NEXT:    ret i8 [[EXTRACTED]]
 ;
@@ -1712,3 +1739,179 @@ define i8 @test_atomicrmw_add_i8_buffer_fat_agent_align4(ptr addrspace(7) %ptr,
   %res = atomicrmw add ptr addrspace(7) %ptr, i8 %value syncscope("agent") seq_cst, align 4
   ret i8 %res
 }
+
+define i8 @test_atomicrmw_sub_i8_global_agent__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i8 %value) {
+; GCN-LABEL: @test_atomicrmw_sub_i8_global_agent__amdgpu_no_remote_memory(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; GCN-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
+; GCN-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; GCN-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
+; GCN-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; GCN-NEXT:    ret i8 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_sub_i8_global_agent__amdgpu_no_remote_memory(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i32(ptr addrspace(1) [[PTR:%.*]], i32 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i32
+; R600-NEXT:    [[PTRLSB:%.*]] = and i32 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3
+; R600-NEXT:    [[MASK:%.*]] = shl i32 255, [[TMP2]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]
+; R600-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; R600-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; R600:       atomicrmw.start:
+; R600-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; R600-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; R600-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
+; R600-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; R600-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
+; R600-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
+; R600-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; R600-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
+; R600-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; R600:       atomicrmw.end:
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; R600-NEXT:    ret i8 [[EXTRACTED]]
+;
+  %res = atomicrmw sub ptr addrspace(1) %ptr, i8 %value syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
+  ret i8 %res
+}
+
+define i8 @test_atomicrmw_sub_i8_global_agent__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, i8 %value) {
+; GCN-LABEL: @test_atomicrmw_sub_i8_global_agent__amdgpu_no_fine_grained_memory(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; GCN-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
+; GCN-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; GCN-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
+; GCN-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; GCN-NEXT:    ret i8 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_sub_i8_global_agent__amdgpu_no_fine_grained_memory(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i32(ptr addrspace(1) [[PTR:%.*]], i32 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i32
+; R600-NEXT:    [[PTRLSB:%.*]] = and i32 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3
+; R600-NEXT:    [[MASK:%.*]] = shl i32 255, [[TMP2]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]
+; R600-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; R600-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; R600:       atomicrmw.start:
+; R600-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; R600-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; R600-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
+; R600-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; R600-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
+; R600-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
+; R600-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; R600-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
+; R600-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; R600:       atomicrmw.end:
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; R600-NEXT:    ret i8 [[EXTRACTED]]
+;
+  %res = atomicrmw sub ptr addrspace(1) %ptr, i8 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+  ret i8 %res
+}
+
+define i8 @test_atomicrmw_sub_i8_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr, i8 %value) {
+; GCN-LABEL: @test_atomicrmw_sub_i8_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; GCN-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1) [[PTR:%.*]], i64 -4)
+; GCN-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i64
+; GCN-NEXT:    [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
+; GCN-NEXT:    [[TMP2:%.*]] = shl i64 [[PTRLSB]], 3
+; GCN-NEXT:    [[SHIFTAMT:%.*]] = trunc i64 [[TMP2]] to i32
+; GCN-NEXT:    [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
+; GCN-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; GCN-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
+; GCN-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[SHIFTAMT]]
+; GCN-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; GCN-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
+; GCN-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; GCN-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
+; GCN-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; GCN-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
+; GCN-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; GCN-NEXT:    ret i8 [[EXTRACTED]]
+;
+; R600-LABEL: @test_atomicrmw_sub_i8_global_agent__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
+; R600-NEXT:    [[ALIGNEDADDR:%.*]] = call ptr addrspace(1) @llvm.ptrmask.p1.i32(ptr addrspace(1) [[PTR:%.*]], i32 -4)
+; R600-NEXT:    [[TMP1:%.*]] = ptrtoint ptr addrspace(1) [[PTR]] to i32
+; R600-NEXT:    [[PTRLSB:%.*]] = and i32 [[TMP1]], 3
+; R600-NEXT:    [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3
+; R600-NEXT:    [[MASK:%.*]] = shl i32 255, [[TMP2]]
+; R600-NEXT:    [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
+; R600-NEXT:    [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
+; R600-NEXT:    [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]
+; R600-NEXT:    [[TMP4:%.*]] = load i32, ptr addrspace(1) [[ALIGNEDADDR]], align 4
+; R600-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; R600:       atomicrmw.start:
+; R600-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; R600-NEXT:    [[NEW:%.*]] = sub i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
+; R600-NEXT:    [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
+; R600-NEXT:    [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
+; R600-NEXT:    [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
+; R600-NEXT:    [[TMP8:%.*]] = cmpxchg ptr addrspace(1) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
+; R600-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
+; R600-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
+; R600-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; R600:       atomicrmw.end:
+; R600-NEXT:    [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]
+; R600-NEXT:    [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
+; R600-NEXT:    ret i8 [[EXTRACTED]]
+;
+  %res = atomicrmw sub ptr addrspace(1) %ptr, i8 %value syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
+  ret i8 %res
+}
+
+!0 = !{}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-flat-noalias-addrspace.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-flat-noalias-addrspace.ll
index 1bf821fd53eab..f8eb16f61e3b7 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-flat-noalias-addrspace.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-flat-noalias-addrspace.ll
@@ -332,10 +332,71 @@ define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5(ptr %ptr, i64
 }
 
 define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, i64 %value) {
-; ALL-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
-; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; ALL-NEXT:    [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
-; ALL-NEXT:    ret i64 [[RES]]
+; GFX7-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX7-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT:    [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
+; GFX7-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX7:       [[ATOMICRMW_START]]:
+; GFX7-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX7-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX7-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX7-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX7-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX7-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX7:       [[ATOMICRMW_END]]:
+; GFX7-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX900-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX900-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX900-NEXT:    [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
+; GFX900-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX900:       [[ATOMICRMW_START]]:
+; GFX900-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX900-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX900-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX900-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX900-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX900-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX900:       [[ATOMICRMW_END]]:
+; GFX900-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX908-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
+; GFX908-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX908:       [[ATOMICRMW_START]]:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX908:       [[ATOMICRMW_END]]:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX90A-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
+; GFX90A-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX90A:       [[ATOMICRMW_START]]:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = and i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX90A:       [[ATOMICRMW_END]]:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX940-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX12-LABEL: define i64 @test_flat_atomicrmw_and_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX12-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw and ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw and ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1
   ret i64 %res
@@ -424,10 +485,71 @@ define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5(ptr %ptr, i64
 }
 
 define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(ptr %ptr, i64 %value) {
-; ALL-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
-; ALL-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
-; ALL-NEXT:    [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
-; ALL-NEXT:    ret i64 [[RES]]
+; GFX7-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX7-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT:    [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
+; GFX7-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX7:       [[ATOMICRMW_START]]:
+; GFX7-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX7-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX7-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX7-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX7-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX7-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX7:       [[ATOMICRMW_END]]:
+; GFX7-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX900-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX900-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX900-NEXT:    [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
+; GFX900-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX900:       [[ATOMICRMW_START]]:
+; GFX900-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX900-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX900-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX900-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX900-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX900-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX900:       [[ATOMICRMW_END]]:
+; GFX900-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX908-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX908-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX908-NEXT:    [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
+; GFX908-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX908:       [[ATOMICRMW_START]]:
+; GFX908-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX908-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX908-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX908-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX908-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX908-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX908:       [[ATOMICRMW_END]]:
+; GFX908-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX90A-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX90A-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[TMP1:%.*]] = load i64, ptr [[PTR]], align 8
+; GFX90A-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; GFX90A:       [[ATOMICRMW_START]]:
+; GFX90A-NEXT:    [[LOADED:%.*]] = phi i64 [ [[TMP1]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; GFX90A-NEXT:    [[NEW:%.*]] = sub i64 [[LOADED]], [[VALUE]]
+; GFX90A-NEXT:    [[TMP2:%.*]] = cmpxchg ptr [[PTR]], i64 [[LOADED]], i64 [[NEW]] syncscope("agent") seq_cst seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX90A-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP2]], 1
+; GFX90A-NEXT:    [[NEWLOADED]] = extractvalue { i64, i1 } [[TMP2]], 0
+; GFX90A-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; GFX90A:       [[ATOMICRMW_END]]:
+; GFX90A-NEXT:    ret i64 [[NEWLOADED]]
+;
+; GFX940-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX940-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX940-NEXT:    [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX940-NEXT:    ret i64 [[RES]]
+;
+; GFX12-LABEL: define i64 @test_flat_atomicrmw_sub_i64_agent__noalias_addrspace_5__maybe_fine_grained(
+; GFX12-SAME: ptr [[PTR:%.*]], i64 [[VALUE:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw sub ptr [[PTR]], i64 [[VALUE]] syncscope("agent") seq_cst, align 8, !noalias.addrspace [[META0]]
+; GFX12-NEXT:    ret i64 [[RES]]
 ;
   %res = atomicrmw sub ptr %ptr, i64 %value syncscope("agent") seq_cst, !noalias.addrspace !1
   ret i64 %res
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll
index dc1107d9130d5..185f42ebc028c 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll
@@ -69,10 +69,40 @@ define i32 @test_atomicrmw_or_0_as999_system(ptr addrspace(999) %ptr) {
 
 ; Leave as-is, only system scope should be changed.
 define i32 @test_atomicrmw_or_0_global_agent(ptr addrspace(1) %ptr) {
-; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
-; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
-; CHECK-NEXT:    ret i32 [[RES]]
+; GFX803-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
+; GFX803-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX803-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
+; GFX803-NEXT:    ret i32 [[RES]]
+;
+; GFX900-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
+; GFX900-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX900-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
+; GFX900-NEXT:    ret i32 [[RES]]
+;
+; GFX90A-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
+; GFX90A-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX90A-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
+; GFX90A-NEXT:    ret i32 [[RES]]
+;
+; GFX10-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
+; GFX10-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX10-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
+; GFX10-NEXT:    ret i32 [[RES]]
+;
+; GFX11-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
+; GFX11-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX11-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
+; GFX11-NEXT:    ret i32 [[RES]]
+;
+; GFX942-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
+; GFX942-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX942-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
+; GFX942-NEXT:    ret i32 [[RES]]
+;
+; GFX12-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
+; GFX12-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
+; GFX12-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
+; GFX12-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 syncscope("agent") seq_cst
   ret i32 %res
@@ -93,7 +123,16 @@ define i32 @test_atomicrmw_or_0_local(ptr addrspace(3) %ptr) {
 define i32 @test_atomicrmw_or_1_global_system(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_or_1_global_system(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 1 seq_cst, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    [[RES]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i32 1 seq_cst
@@ -103,7 +142,16 @@ define i32 @test_atomicrmw_or_1_global_system(ptr addrspace(1) %ptr) {
 define i32 @test_atomicrmw_or_var_global_system(ptr addrspace(1) %ptr, i32 %val) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_or_var_global_system(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP1]], [[TMP0:%.*]] ], [ [[RES:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = or i32 [[LOADED]], [[VAL]]
+; CHECK-NEXT:    [[TMP2:%.*]] = cmpxchg ptr addrspace(1) [[PTR]], i32 [[LOADED]], i32 [[NEW]] seq_cst seq_cst, align 4
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP2]], 1
+; CHECK-NEXT:    [[RES]] = extractvalue { i32, i1 } [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i32 %val seq_cst
@@ -146,7 +194,7 @@ define i32 @test_atomicrmw_xor_0_global_system(ptr addrspace(1) %ptr) {
 define i32 @test_atomicrmw_or_0_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_system__amdgpu_no_fine_grained_memory(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -156,7 +204,7 @@ define i32 @test_atomicrmw_or_0_global_system__amdgpu_no_fine_grained_memory(ptr
 define i32 @test_atomicrmw_or_0_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_system__amdgpu_no_remote_memory(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 seq_cst, !amdgpu.no.remote.memory !0
@@ -166,7 +214,7 @@ define i32 @test_atomicrmw_or_0_global_system__amdgpu_no_remote_memory(ptr addrs
 define i32 @test_atomicrmw_or_0_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -176,7 +224,7 @@ define i32 @test_atomicrmw_or_0_global_system__amdgpu_no_fine_grained_memory__am
 define i32 @test_atomicrmw_xor_0_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_xor_0_global_system__amdgpu_no_fine_grained_memory(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw xor ptr addrspace(1) %ptr, i32 0 seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -186,7 +234,7 @@ define i32 @test_atomicrmw_xor_0_global_system__amdgpu_no_fine_grained_memory(pt
 define i32 @test_atomicrmw_xor_0_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_xor_0_global_system__amdgpu_no_remote_memory(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw xor ptr addrspace(1) %ptr, i32 0 seq_cst, !amdgpu.no.remote.memory !0
@@ -196,7 +244,7 @@ define i32 @test_atomicrmw_xor_0_global_system__amdgpu_no_remote_memory(ptr addr
 define i32 @test_atomicrmw_xor_0_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_xor_0_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw xor ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw xor ptr addrspace(1) %ptr, i32 0 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -206,7 +254,7 @@ define i32 @test_atomicrmw_xor_0_global_system__amdgpu_no_fine_grained_memory__a
 define i32 @test_atomicrmw_sub_0_global_system__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_sub_0_global_system__amdgpu_no_fine_grained_memory(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw sub ptr addrspace(1) %ptr, i32 0 seq_cst, !amdgpu.no.fine.grained.memory !0
@@ -216,7 +264,7 @@ define i32 @test_atomicrmw_sub_0_global_system__amdgpu_no_fine_grained_memory(pt
 define i32 @test_atomicrmw_sub_0_global_system__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_sub_0_global_system__amdgpu_no_remote_memory(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.remote.memory [[META0]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw sub ptr addrspace(1) %ptr, i32 0 seq_cst, !amdgpu.no.remote.memory !0
@@ -226,7 +274,7 @@ define i32 @test_atomicrmw_sub_0_global_system__amdgpu_no_remote_memory(ptr addr
 define i32 @test_atomicrmw_sub_0_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(ptr addrspace(1) %ptr) {
 ; CHECK-LABEL: define i32 @test_atomicrmw_sub_0_global_system__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory(
 ; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw sub ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !amdgpu.no.fine.grained.memory [[META0]], !amdgpu.no.remote.memory [[META0]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %res = atomicrmw sub ptr addrspace(1) %ptr, i32 0 seq_cst, !amdgpu.no.fine.grained.memory !0, !amdgpu.no.remote.memory !0
@@ -324,12 +372,3 @@ define i32 @test_atomicrmw_sub_0_global_agent__amdgpu_no_fine_grained_memory__am
 }
 
 !0 = !{}
-
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10: {{.*}}
-; GFX11: {{.*}}
-; GFX12: {{.*}}
-; GFX803: {{.*}}
-; GFX900: {{.*}}
-; GFX90A: {{.*}}
-; GFX942: {{.*}}
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-cmpxchg-flat-maybe-private.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-cmpxchg-flat-maybe-private.ll
index 4b4994958e66a..a9e4f10dc9a53 100644
--- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-cmpxchg-flat-maybe-private.ll
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-cmpxchg-flat-maybe-private.ll
@@ -308,24 +308,7 @@ define { i64, i1 } @cmpxchg_flat_agent_i64__noalias_addrspace_edge_case0(ptr %pt
 define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_6(ptr %ptr, i64 %val, i64 %swap) {
 ; CHECK-LABEL: define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_6(
 ; CHECK-SAME: ptr [[PTR:%.*]], i64 [[VAL:%.*]], i64 [[SWAP:%.*]]) {
-; CHECK-NEXT:    [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]])
-; CHECK-NEXT:    br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]]
-; CHECK:       [[ATOMICRMW_PRIVATE]]:
-; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5)
-; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[TMP2]], [[VAL]]
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[SWAP]], i64 [[TMP2]]
-; CHECK-NEXT:    store i64 [[TMP4]], ptr addrspace(5) [[TMP1]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = insertvalue { i64, i1 } poison, i64 [[TMP2]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i64, i1 } [[TMP5]], i1 [[TMP3]], 1
-; CHECK-NEXT:    br label %[[ATOMICRMW_PHI:.*]]
-; CHECK:       [[ATOMICRMW_GLOBAL]]:
-; CHECK-NEXT:    [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[VAL]], i64 [[SWAP]] syncscope("agent") monotonic seq_cst, align 8, !noalias.addrspace [[META0]]
-; CHECK-NEXT:    br label %[[ATOMICRMW_PHI]]
-; CHECK:       [[ATOMICRMW_PHI]]:
-; CHECK-NEXT:    [[RESULT:%.*]] = phi { i64, i1 } [ [[TMP6]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP7]], %[[ATOMICRMW_GLOBAL]] ]
-; CHECK-NEXT:    br label %[[ATOMICRMW_END:.*]]
-; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    [[RESULT:%.*]] = cmpxchg ptr [[PTR]], i64 [[VAL]], i64 [[SWAP]] syncscope("agent") monotonic seq_cst, align 8, !noalias.addrspace [[META4:![0-9]+]]
 ; CHECK-NEXT:    ret { i64, i1 } [[RESULT]]
 ;
   %result = cmpxchg ptr %ptr, i64 %val, i64 %swap syncscope("agent") monotonic seq_cst, !noalias.addrspace !7
@@ -335,7 +318,7 @@ define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_6(ptr %ptr, i64 %val, i64 %swap
 define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_3_5(ptr %ptr, i64 %val, i64 %swap) {
 ; CHECK-LABEL: define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_3_5(
 ; CHECK-SAME: ptr [[PTR:%.*]], i64 [[VAL:%.*]], i64 [[SWAP:%.*]]) {
-; CHECK-NEXT:    [[RESULT:%.*]] = cmpxchg ptr [[PTR]], i64 [[VAL]], i64 [[SWAP]] syncscope("agent") monotonic seq_cst, align 8, !noalias.addrspace [[META4:![0-9]+]]
+; CHECK-NEXT:    [[RESULT:%.*]] = cmpxchg ptr [[PTR]], i64 [[VAL]], i64 [[SWAP]] syncscope("agent") monotonic seq_cst, align 8, !noalias.addrspace [[META5:![0-9]+]]
 ; CHECK-NEXT:    ret { i64, i1 } [[RESULT]]
 ;
   %result = cmpxchg ptr %ptr, i64 %val, i64 %swap syncscope("agent") monotonic seq_cst, !noalias.addrspace !8
@@ -357,5 +340,6 @@ define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_3_5(ptr %ptr, i64 %val, i64 %sw
 ; CHECK: [[META1]] = !{[[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK: [[META2]] = !{!"foo", !"bar"}
 ; CHECK: [[META3]] = !{!"bux", !"baz"}
-; CHECK: [[META4]] = !{i32 2, i32 4, i32 5, i32 6}
+; CHECK: [[META4]] = !{i32 2, i32 6}
+; CHECK: [[META5]] = !{i32 2, i32 4, i32 5, i32 6}
 ;.
diff --git a/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll b/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll
index cc51a00db415c..9bf8a5149b3d5 100644
--- a/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll
+++ b/llvm/test/Transforms/AtomicExpand/PowerPC/cmpxchg.ll
@@ -31,11 +31,11 @@ define i1 @test_cmpxchg_seq_cst(ptr %addr, i128 %desire, i128 %new) {
 ; PWR7-LABEL: @test_cmpxchg_seq_cst(
 ; PWR7-NEXT:  entry:
 ; PWR7-NEXT:    [[TMP0:%.*]] = alloca i128, align 16
-; PWR7-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[TMP0]])
+; PWR7-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP0]])
 ; PWR7-NEXT:    store i128 [[DESIRE:%.*]], ptr [[TMP0]], align 16
 ; PWR7-NEXT:    [[TMP1:%.*]] = call zeroext i1 @__atomic_compare_exchange_16(ptr [[ADDR:%.*]], ptr [[TMP0]], i128 [[NEW:%.*]], i32 5, i32 5)
 ; PWR7-NEXT:    [[TMP2:%.*]] = load i128, ptr [[TMP0]], align 16
-; PWR7-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[TMP0]])
+; PWR7-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP0]])
 ; PWR7-NEXT:    [[TMP3:%.*]] = insertvalue { i128, i1 } poison, i128 [[TMP2]], 0
 ; PWR7-NEXT:    [[TMP4:%.*]] = insertvalue { i128, i1 } [[TMP3]], i1 [[TMP1]], 1
 ; PWR7-NEXT:    [[SUCC:%.*]] = extractvalue { i128, i1 } [[TMP4]], 1
diff --git a/llvm/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll b/llvm/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll
index 2cbb1794b4340..60fb2487faa13 100644
--- a/llvm/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll
+++ b/llvm/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll
@@ -9,12 +9,12 @@ define float @test_atomicrmw_fadd_f32(ptr %ptr, float %value) {
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
 ; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
 ; CHECK-NEXT:    store float [[LOADED]], ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
 ; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[PTR]], ptr [[TMP1]], i32 [[TMP3]], i32 5, i32 5)
 ; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[TMP1]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { float, i1 } poison, float [[TMP5]], 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { float, i1 } [[TMP6]], i1 [[TMP4]], 1
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { float, i1 } [[TMP7]], 1
@@ -35,12 +35,12 @@ define float @test_atomicrmw_fsub_f32(ptr %ptr, float %value) {
 ; CHECK:       atomicrmw.start:
 ; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
 ; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
 ; CHECK-NEXT:    store float [[LOADED]], ptr [[TMP1]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
 ; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[PTR]], ptr [[TMP1]], i32 [[TMP3]], i32 5, i32 5)
 ; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[TMP1]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { float, i1 } poison, float [[TMP5]], 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { float, i1 } [[TMP6]], i1 [[TMP4]], 1
 ; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { float, i1 } [[TMP7]], 1
diff --git a/llvm/test/Transforms/AtomicExpand/SPARC/libcalls.ll b/llvm/test/Transforms/AtomicExpand/SPARC/libcalls.ll
index 682c1e6848b31..1d6a32ca093f1 100644
--- a/llvm/test/Transforms/AtomicExpand/SPARC/libcalls.ll
+++ b/llvm/test/Transforms/AtomicExpand/SPARC/libcalls.ll
@@ -38,11 +38,11 @@ define i16 @test_exchange_i16(ptr %arg, i16 %val) {
 
 ; CHECK-LABEL: @test_cmpxchg_i16(
 ; CHECK:  %1 = alloca i16, align 2
-; CHECK:  call void @llvm.lifetime.start.p0(i64 2, ptr %1)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %1)
 ; CHECK:  store i16 %old, ptr %1, align 2
 ; CHECK:  %2 = call zeroext i1 @__atomic_compare_exchange_2(ptr %arg, ptr %1, i16 %new, i32 5, i32 0)
 ; CHECK:  %3 = load i16, ptr %1, align 2
-; CHECK:  call void @llvm.lifetime.end.p0(i64 2, ptr %1)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %1)
 ; CHECK:  %4 = insertvalue { i16, i1 } poison, i16 %3, 0
 ; CHECK:  %5 = insertvalue { i16, i1 } %4, i1 %2, 1
 ; CHECK:  %ret = extractvalue { i16, i1 } %5, 0
@@ -68,10 +68,10 @@ define i16 @test_add_i16(ptr %arg, i16 %val) {
 
 ; CHECK-LABEL: @test_load_i128(
 ; CHECK:  %1 = alloca i128, align 8
-; CHECK:  call void @llvm.lifetime.start.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %1)
 ; CHECK:  call void @__atomic_load(i32 16, ptr %arg, ptr %1, i32 5)
 ; CHECK:  %2 = load i128, ptr %1, align 8
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %1)
 ; CHECK:  ret i128 %2
 define i128 @test_load_i128(ptr %arg) {
   %ret = load atomic i128, ptr %arg seq_cst, align 16
@@ -80,10 +80,10 @@ define i128 @test_load_i128(ptr %arg) {
 
 ; CHECK-LABEL: @test_store_i128(
 ; CHECK:  %1 = alloca i128, align 8
-; CHECK:  call void @llvm.lifetime.start.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %1)
 ; CHECK:  store i128 %val, ptr %1, align 8
 ; CHECK:  call void @__atomic_store(i32 16, ptr %arg, ptr %1, i32 5)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %1)
 ; CHECK:  ret void
 define void @test_store_i128(ptr %arg, i128 %val) {
   store atomic i128 %val, ptr %arg seq_cst, align 16
@@ -92,14 +92,14 @@ define void @test_store_i128(ptr %arg, i128 %val) {
 
 ; CHECK-LABEL: @test_exchange_i128(
 ; CHECK:  %1 = alloca i128, align 8
-; CHECK:  call void @llvm.lifetime.start.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %1)
 ; CHECK:  store i128 %val, ptr %1, align 8
 ; CHECK:  %2 = alloca i128, align 8
-; CHECK:  call void @llvm.lifetime.start.p0(i64 16, ptr %2)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %2)
 ; CHECK:  call void @__atomic_exchange(i32 16, ptr %arg, ptr %1, ptr %2, i32 5)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %1)
 ; CHECK:  %3 = load i128, ptr %2, align 8
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr %2)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %2)
 ; CHECK:  ret i128 %3
 define i128 @test_exchange_i128(ptr %arg, i128 %val) {
   %ret = atomicrmw xchg ptr %arg, i128 %val seq_cst
@@ -108,15 +108,15 @@ define i128 @test_exchange_i128(ptr %arg, i128 %val) {
 
 ; CHECK-LABEL: @test_cmpxchg_i128(
 ; CHECK:  %1 = alloca i128, align 8
-; CHECK:  call void @llvm.lifetime.start.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %1)
 ; CHECK:  store i128 %old, ptr %1, align 8
 ; CHECK:  %2 = alloca i128, align 8
-; CHECK:  call void @llvm.lifetime.start.p0(i64 16, ptr %2)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %2)
 ; CHECK:  store i128 %new, ptr %2, align 8
 ; CHECK:  %3 = call zeroext i1 @__atomic_compare_exchange(i32 16, ptr %arg, ptr %1, ptr %2, i32 5, i32 0)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr %2)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %2)
 ; CHECK:  %4 = load i128, ptr %1, align 8
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %1)
 ; CHECK:  %5 = insertvalue { i128, i1 } poison, i128 %4, 0
 ; CHECK:  %6 = insertvalue { i128, i1 } %5, i1 %3, 1
 ; CHECK:  %ret = extractvalue { i128, i1 } %6, 0
@@ -139,14 +139,14 @@ define i128 @test_cmpxchg_i128(ptr %arg, i128 %old, i128 %new) {
 ; CHECK:atomicrmw.start:
 ; CHECK:  %loaded = phi i128 [ %3, %0 ], [ %newloaded, %atomicrmw.start ]
 ; CHECK:  %new = add i128 %loaded, %val
-; CHECK:  call void @llvm.lifetime.start.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %1)
 ; CHECK:  store i128 %loaded, ptr %1, align 8
-; CHECK:  call void @llvm.lifetime.start.p0(i64 16, ptr %2)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %2)
 ; CHECK:  store i128 %new, ptr %2, align 8
 ; CHECK:  %4 = call zeroext i1 @__atomic_compare_exchange(i32 16, ptr %arg, ptr %1, ptr %2, i32 5, i32 5)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr %2)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %2)
 ; CHECK:  %5 = load i128, ptr %1, align 8
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %1)
 ; CHECK:  %6 = insertvalue { i128, i1 } poison, i128 %5, 0
 ; CHECK:  %7 = insertvalue { i128, i1 } %6, i1 %4, 1
 ; CHECK:  %success = extractvalue { i128, i1 } %7, 1
@@ -181,12 +181,12 @@ define void @test_store_double(ptr %arg, double %val) {
 
 ; CHECK-LABEL: @test_cmpxchg_ptr(
 ; CHECK:   %1 = alloca ptr, align 4
-; CHECK:   call void @llvm.lifetime.start.p0(i64 4, ptr %1)
+; CHECK:   call void @llvm.lifetime.start.p0(ptr %1)
 ; CHECK:   store ptr %old, ptr %1, align 4
 ; CHECK:   %2 = ptrtoint ptr %new to i32
 ; CHECK:   %3 = call zeroext i1 @__atomic_compare_exchange_4(ptr %arg, ptr %1, i32 %2, i32 5, i32 2)
 ; CHECK:   %4 = load ptr, ptr %1, align 4
-; CHECK:   call void @llvm.lifetime.end.p0(i64 4, ptr %1)
+; CHECK:   call void @llvm.lifetime.end.p0(ptr %1)
 ; CHECK:   %5 = insertvalue { ptr, i1 } poison, ptr %4, 0
 ; CHECK:   %6 = insertvalue { ptr, i1 } %5, i1 %3, 1
 ; CHECK:   %ret = extractvalue { ptr, i1 } %6, 0
@@ -202,10 +202,10 @@ define ptr @test_cmpxchg_ptr(ptr %arg, ptr %old, ptr %new) {
 
 ; CHECK-LABEL: @test_store_fp128
 ; CHECK:  %1 = alloca fp128, align 8
-; CHECK:  call void @llvm.lifetime.start.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.start.p0(ptr %1)
 ; CHECK:  store fp128 %val, ptr %1, align 8
 ; CHECK:  call void @__atomic_store(i32 16, ptr %arg, ptr %1, i32 5)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 16, ptr %1)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr %1)
 ; CHECK:  ret void
 define void @test_store_fp128(ptr %arg, fp128 %val) {
   store atomic fp128 %val, ptr %arg seq_cst, align 16
diff --git a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-libcall.ll b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-libcall.ll
index 20a9e9f6cb86a..fda296be02a71 100644
--- a/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-libcall.ll
+++ b/llvm/test/Transforms/AtomicExpand/X86/expand-atomic-libcall.ll
@@ -5,10 +5,10 @@
 define i256 @atomic_load256_libcall(ptr %ptr) nounwind {
 ; CHECK-LABEL: @atomic_load256_libcall(
 ; CHECK-NEXT:    [[TMP1:%.*]] = alloca i256, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
 ; CHECK-NEXT:    call void @__atomic_load(i32 32, ptr [[PTR:%.*]], ptr [[TMP1]], i32 0)
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i256, ptr [[TMP1]], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
 ; CHECK-NEXT:    ret i256 [[TMP2]]
 ;
   %result = load atomic i256, ptr %ptr unordered, align 16
@@ -19,10 +19,10 @@ define i256 @atomic_load256_libcall_as1(ptr addrspace(1) %ptr) nounwind {
 ; CHECK-LABEL: @atomic_load256_libcall_as1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr addrspace(1) [[PTR:%.*]] to ptr
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca i256, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[TMP2]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP2]])
 ; CHECK-NEXT:    call void @__atomic_load(i32 32, ptr [[TMP1]], ptr [[TMP2]], i32 0)
 ; CHECK-NEXT:    [[TMP3:%.*]] = load i256, ptr [[TMP2]], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[TMP2]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP2]])
 ; CHECK-NEXT:    ret i256 [[TMP3]]
 ;
   %result = load atomic i256, ptr addrspace(1) %ptr unordered, align 16
diff --git a/llvm/test/Transforms/AtomicExpand/Xtensa/atomicrmw-expand.ll b/llvm/test/Transforms/AtomicExpand/Xtensa/atomicrmw-expand.ll
new file mode 100644
index 0000000000000..647f187ed7f3a
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/Xtensa/atomicrmw-expand.ll
@@ -0,0 +1,2643 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=xtensa-- -passes=atomic-expand %s | FileCheck %s
+
+define i8 @atomicrmw_xchg_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xchg_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_exchange_1(ptr [[A]], i8 [[B]], i32 0)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xchg_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xchg_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_exchange_1(ptr [[A]], i8 [[B]], i32 2)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xchg_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xchg_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_exchange_1(ptr [[A]], i8 [[B]], i32 3)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xchg_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xchg_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_exchange_1(ptr [[A]], i8 [[B]], i32 4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xchg_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xchg_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_exchange_1(ptr [[A]], i8 [[B]], i32 5)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_add_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_add_1(ptr [[A]], i8 [[B]], i32 0)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_add_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_add_1(ptr [[A]], i8 [[B]], i32 2)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_add_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_add_1(ptr [[A]], i8 [[B]], i32 3)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_add_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_add_1(ptr [[A]], i8 [[B]], i32 4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_add_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_add_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_add_1(ptr [[A]], i8 [[B]], i32 5)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_sub_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_sub_1(ptr [[A]], i8 [[B]], i32 0)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_sub_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_sub_1(ptr [[A]], i8 [[B]], i32 2)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_sub_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_sub_1(ptr [[A]], i8 [[B]], i32 3)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_sub_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_sub_1(ptr [[A]], i8 [[B]], i32 4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_sub_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_sub_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_sub_1(ptr [[A]], i8 [[B]], i32 5)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_and_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_and_1(ptr [[A]], i8 [[B]], i32 0)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_and_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_and_1(ptr [[A]], i8 [[B]], i32 2)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_and_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_and_1(ptr [[A]], i8 [[B]], i32 3)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_and_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_and_1(ptr [[A]], i8 [[B]], i32 4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_and_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_and_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_and_1(ptr [[A]], i8 [[B]], i32 5)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_nand_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_nand_1(ptr [[A]], i8 [[B]], i32 0)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_nand_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_nand_1(ptr [[A]], i8 [[B]], i32 2)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_nand_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_nand_1(ptr [[A]], i8 [[B]], i32 3)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_nand_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_nand_1(ptr [[A]], i8 [[B]], i32 4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_nand_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_nand_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_nand_1(ptr [[A]], i8 [[B]], i32 5)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_or_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_or_1(ptr [[A]], i8 [[B]], i32 0)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_or_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_or_1(ptr [[A]], i8 [[B]], i32 2)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_or_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_or_1(ptr [[A]], i8 [[B]], i32 3)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_or_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_or_1(ptr [[A]], i8 [[B]], i32 4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_or_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_or_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_or_1(ptr [[A]], i8 [[B]], i32 5)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xor_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_xor_1(ptr [[A]], i8 [[B]], i32 0)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xor_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_xor_1(ptr [[A]], i8 [[B]], i32 2)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xor_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_xor_1(ptr [[A]], i8 [[B]], i32 3)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xor_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_xor_1(ptr [[A]], i8 [[B]], i32 4)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_xor_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_xor_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @__atomic_fetch_xor_1(ptr [[A]], i8 [[B]], i32 5)
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_max_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_max_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_max_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_max_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_max_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_max_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_min_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_min_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_min_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_min_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_min_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_min_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umax_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umax_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umax_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umax_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umax_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umax_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_monotonic(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umin_i8_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i8 %b monotonic
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_acquire(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umin_i8_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i8 %b acquire
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_release(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umin_i8_release(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i8 %b release
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_acq_rel(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umin_i8_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i8 %b acq_rel
+  ret i8 %res
+}
+
+define i8 @atomicrmw_umin_i8_seq_cst(ptr %a, i8 %b) nounwind {
+; CHECK-LABEL: define i8 @atomicrmw_umin_i8_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i8, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[A]], align 1
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i8 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i8 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i8 [[LOADED]], i8 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i8 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_1(ptr [[A]], ptr [[TMP1]], i8 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i8, i1 } poison, i8 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i8, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i8, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i8, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i8 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i8 %b seq_cst
+  ret i8 %res
+}
+
+define i16 @atomicrmw_xchg_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xchg_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_exchange_2(ptr [[A]], i16 [[B]], i32 0)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xchg_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xchg_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_exchange_2(ptr [[A]], i16 [[B]], i32 2)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xchg_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xchg_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_exchange_2(ptr [[A]], i16 [[B]], i32 3)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xchg_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xchg_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_exchange_2(ptr [[A]], i16 [[B]], i32 4)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xchg_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xchg_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_exchange_2(ptr [[A]], i16 [[B]], i32 5)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_add_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_add_2(ptr [[A]], i16 [[B]], i32 0)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_add_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_add_2(ptr [[A]], i16 [[B]], i32 2)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_add_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_add_2(ptr [[A]], i16 [[B]], i32 3)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_add_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_add_2(ptr [[A]], i16 [[B]], i32 4)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_add_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_add_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_add_2(ptr [[A]], i16 [[B]], i32 5)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_sub_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_sub_2(ptr [[A]], i16 [[B]], i32 0)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_sub_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_sub_2(ptr [[A]], i16 [[B]], i32 2)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_sub_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_sub_2(ptr [[A]], i16 [[B]], i32 3)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_sub_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_sub_2(ptr [[A]], i16 [[B]], i32 4)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_sub_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_sub_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_sub_2(ptr [[A]], i16 [[B]], i32 5)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_and_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_and_2(ptr [[A]], i16 [[B]], i32 0)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_and_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_and_2(ptr [[A]], i16 [[B]], i32 2)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_and_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_and_2(ptr [[A]], i16 [[B]], i32 3)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_and_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_and_2(ptr [[A]], i16 [[B]], i32 4)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_and_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_and_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_and_2(ptr [[A]], i16 [[B]], i32 5)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_nand_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_nand_2(ptr [[A]], i16 [[B]], i32 0)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_nand_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_nand_2(ptr [[A]], i16 [[B]], i32 2)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_nand_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_nand_2(ptr [[A]], i16 [[B]], i32 3)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_nand_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_nand_2(ptr [[A]], i16 [[B]], i32 4)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_nand_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_nand_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_nand_2(ptr [[A]], i16 [[B]], i32 5)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw nand ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_or_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_or_2(ptr [[A]], i16 [[B]], i32 0)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_or_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_or_2(ptr [[A]], i16 [[B]], i32 2)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_or_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_or_2(ptr [[A]], i16 [[B]], i32 3)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_or_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_or_2(ptr [[A]], i16 [[B]], i32 4)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_or_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_or_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_or_2(ptr [[A]], i16 [[B]], i32 5)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xor_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_xor_2(ptr [[A]], i16 [[B]], i32 0)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xor_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_xor_2(ptr [[A]], i16 [[B]], i32 2)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xor_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_xor_2(ptr [[A]], i16 [[B]], i32 3)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xor_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_xor_2(ptr [[A]], i16 [[B]], i32 4)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_xor_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_xor_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @__atomic_fetch_xor_2(ptr [[A]], i16 [[B]], i32 5)
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_max_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_max_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_max_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_max_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_max_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_max_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_min_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_min_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_min_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_min_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_min_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_min_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umax_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umax_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umax_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umax_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umax_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umax_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_monotonic(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umin_i16_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i16 %b monotonic
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_acquire(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umin_i16_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i16 %b acquire
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_release(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umin_i16_release(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i16 %b release
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_acq_rel(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umin_i16_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i16 %b acq_rel
+  ret i16 %res
+}
+
+define i16 @atomicrmw_umin_i16_seq_cst(ptr %a, i16 %b) nounwind {
+; CHECK-LABEL: define i16 @atomicrmw_umin_i16_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i16, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr [[A]], align 2
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i16 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i16 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i16 [[LOADED]], i16 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i16 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_2(ptr [[A]], ptr [[TMP1]], i16 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i16, i1 } poison, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i16, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i16, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i16 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i16 %b seq_cst
+  ret i16 %res
+}
+
+define i32 @atomicrmw_xchg_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xchg_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_exchange_4(ptr [[A]], i32 [[B]], i32 0)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xchg_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xchg_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_exchange_4(ptr [[A]], i32 [[B]], i32 2)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xchg_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xchg_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_exchange_4(ptr [[A]], i32 [[B]], i32 3)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xchg_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xchg_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_exchange_4(ptr [[A]], i32 [[B]], i32 4)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xchg_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xchg_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_exchange_4(ptr [[A]], i32 [[B]], i32 5)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xchg ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_add_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_add_4(ptr [[A]], i32 [[B]], i32 0)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_add_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_add_4(ptr [[A]], i32 [[B]], i32 2)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_add_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_add_4(ptr [[A]], i32 [[B]], i32 3)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_add_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_add_4(ptr [[A]], i32 [[B]], i32 4)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_add_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_add_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_add_4(ptr [[A]], i32 [[B]], i32 5)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw add ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_sub_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_sub_4(ptr [[A]], i32 [[B]], i32 0)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_sub_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_sub_4(ptr [[A]], i32 [[B]], i32 2)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_sub_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_sub_4(ptr [[A]], i32 [[B]], i32 3)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_sub_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_sub_4(ptr [[A]], i32 [[B]], i32 4)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_sub_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_sub_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_sub_4(ptr [[A]], i32 [[B]], i32 5)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw sub ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_and_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_and_4(ptr [[A]], i32 [[B]], i32 0)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_and_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_and_4(ptr [[A]], i32 [[B]], i32 2)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_and_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_and_4(ptr [[A]], i32 [[B]], i32 3)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_and_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_and_4(ptr [[A]], i32 [[B]], i32 4)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_and_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_and_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_and_4(ptr [[A]], i32 [[B]], i32 5)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw and ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+;define i32 @atomicrmw_nand_i32_monotonic(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b monotonic
+;  ret i32 %res
+;}
+;define i32 @atomicrmw_nand_i32_acquire(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b acquire
+;  ret i32 %res
+;}
+;define i32 @atomicrmw_nand_i32_release(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b release
+;  ret i32 %res
+;}
+;define i32 @atomicrmw_nand_i32_acq_rel(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b acq_rel
+;  ret i32 %res
+;}
+;define i32 @atomicrmw_nand_i32_seq_cst(ptr %a, i32 %b) nounwind {
+;  %res = atomicrmw nand ptr %a, i32 %b seq_cst
+;  ret i32 %res
+;}
+
+define i32 @atomicrmw_or_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_or_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_or_4(ptr [[A]], i32 [[B]], i32 0)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_or_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_or_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_or_4(ptr [[A]], i32 [[B]], i32 2)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_or_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_or_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_or_4(ptr [[A]], i32 [[B]], i32 3)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_or_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_or_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_or_4(ptr [[A]], i32 [[B]], i32 4)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_or_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_or_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_or_4(ptr [[A]], i32 [[B]], i32 5)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw or ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xor_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_xor_4(ptr [[A]], i32 [[B]], i32 0)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xor_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_xor_4(ptr [[A]], i32 [[B]], i32 2)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xor_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_xor_4(ptr [[A]], i32 [[B]], i32 3)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xor_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_xor_4(ptr [[A]], i32 [[B]], i32 4)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_xor_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_xor_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @__atomic_fetch_xor_4(ptr [[A]], i32 [[B]], i32 5)
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+  %res = atomicrmw xor ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_max_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_max_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_max_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_max_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_max_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_max_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw max ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_min_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_min_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_min_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_min_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_min_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_min_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp sle i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw min ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umax_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umax_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umax_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umax_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umax_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umax_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umax ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_monotonic(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umin_i32_monotonic(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 0, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i32 %b monotonic
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_acquire(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umin_i32_acquire(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 2, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i32 %b acquire
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_release(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umin_i32_release(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 3, i32 0)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i32 %b release
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_acq_rel(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umin_i32_acq_rel(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 4, i32 2)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i32 %b acq_rel
+  ret i32 %res
+}
+
+define i32 @atomicrmw_umin_i32_seq_cst(ptr %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @atomicrmw_umin_i32_seq_cst(
+; CHECK-SAME: ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    br label %[[ATOMICRMW_START:.*]]
+; CHECK:       [[ATOMICRMW_START]]:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], %[[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[LOADED]], [[B]]
+; CHECK-NEXT:    [[NEW:%.*]] = select i1 [[TMP3]], i32 [[LOADED]], i32 [[B]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
+; CHECK-NEXT:    store i32 [[LOADED]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(ptr [[A]], ptr [[TMP1]], i32 [[NEW]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = insertvalue { i32, i1 } poison, i32 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertvalue { i32, i1 } [[TMP6]], i1 [[TMP4]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label %[[ATOMICRMW_END:.*]], label %[[ATOMICRMW_START]]
+; CHECK:       [[ATOMICRMW_END]]:
+; CHECK-NEXT:    ret i32 [[NEWLOADED]]
+;
+  %res = atomicrmw umin ptr %a, i32 %b seq_cst
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/AtomicExpand/Xtensa/lit.local.cfg b/llvm/test/Transforms/AtomicExpand/Xtensa/lit.local.cfg
new file mode 100644
index 0000000000000..e81bfa773f36a
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/Xtensa/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "Xtensa" in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack.ll b/llvm/test/Transforms/Attributor/heap_to_stack.ll
index 67192902b52df..d54f713e7bbfe 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack.ll
@@ -27,7 +27,7 @@ declare i32 @no_return_call() noreturn
 
 declare void @free(ptr nocapture) allockind("free")
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
 
 ;.
 ; CHECK: @G = internal global ptr undef, align 4
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll b/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll
index 0be9434829cce..9a6e0680bb44d 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll
@@ -32,7 +32,7 @@ declare i32 @no_return_call() noreturn
 
 declare void @free(ptr nocapture)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
 
 ;.
 ; CHECK: @G = internal global ptr undef, align 4
diff --git a/llvm/test/Transforms/Attributor/liveness.ll b/llvm/test/Transforms/Attributor/liveness.ll
index 874eff661f053..c112d995148d5 100644
--- a/llvm/test/Transforms/Attributor/liveness.ll
+++ b/llvm/test/Transforms/Attributor/liveness.ll
@@ -2589,7 +2589,7 @@ define void @bad_gep() {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[N1:%.*]] = alloca i8, i32 0, align 1
 ; TUNIT-NEXT:    [[M2:%.*]] = alloca i8, i32 0, align 1
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1, ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR18:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR18:[0-9]+]]
 ; TUNIT-NEXT:    br label [[EXIT:%.*]]
 ; TUNIT:       while.body:
 ; TUNIT-NEXT:    unreachable
@@ -2598,7 +2598,7 @@ define void @bad_gep() {
 ; TUNIT:       if.end:
 ; TUNIT-NEXT:    unreachable
 ; TUNIT:       exit:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1, ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR18]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR18]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
@@ -2607,7 +2607,7 @@ define void @bad_gep() {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[N1:%.*]] = alloca i8, i32 0, align 1
 ; CGSCC-NEXT:    [[M2:%.*]] = alloca i8, i32 0, align 1
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1, ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR21:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR21:[0-9]+]]
 ; CGSCC-NEXT:    br label [[EXIT:%.*]]
 ; CGSCC:       while.body:
 ; CGSCC-NEXT:    unreachable
@@ -2616,13 +2616,13 @@ define void @bad_gep() {
 ; CGSCC:       if.end:
 ; CGSCC-NEXT:    unreachable
 ; CGSCC:       exit:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1, ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR21]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull captures(none) dereferenceable(1) [[N1]]) #[[ATTR21]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
   %n = alloca i8
   %m = alloca i8
-  call void @llvm.lifetime.start.p0(i64 1, ptr %n)
+  call void @llvm.lifetime.start.p0(ptr %n)
   br label %exit
 
 while.body:
@@ -2640,7 +2640,7 @@ if.end:
   br i1 %cmp, label %exit, label %while.body
 
 exit:
-  call void @llvm.lifetime.end.p0(i64 1, ptr %n)
+  call void @llvm.lifetime.end.p0(ptr %n)
   ret void
 }
 
@@ -2679,8 +2679,8 @@ b2:
 declare i1 @bad_gep_helper1(ptr, ptr, ptr)
 declare void @bad_gep_helper2(i8)
 
-declare void @llvm.lifetime.start.p0(i64 %0, ptr %1)
-declare void @llvm.lifetime.end.p0(i64 %0, ptr %1)
+declare void @llvm.lifetime.start.p0(ptr %1)
+declare void @llvm.lifetime.end.p0(ptr %1)
 ;.
 ; TUNIT: attributes #[[ATTR0]] = { nofree noreturn nosync nounwind }
 ; TUNIT: attributes #[[ATTR1:[0-9]+]] = { memory(none) }
diff --git a/llvm/test/Transforms/Attributor/noalias.ll b/llvm/test/Transforms/Attributor/noalias.ll
index 46d9f77660253..b7c295ae4179c 100644
--- a/llvm/test/Transforms/Attributor/noalias.ll
+++ b/llvm/test/Transforms/Attributor/noalias.ll
@@ -577,31 +577,31 @@ define internal fastcc double @strtox(ptr %s, ptr %p, i32 %prec) unnamed_addr {
 ; TUNIT-SAME: (ptr [[S:%.*]]) unnamed_addr {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[F:%.*]] = alloca [[STRUCT__IO_FILE:%.*]], align 8
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 144, ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]]) #[[ATTR13:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]]) #[[ATTR13:[0-9]+]]
 ; TUNIT-NEXT:    [[CALL:%.*]] = call i32 @sh_fromstring(ptr noundef nonnull align 8 dereferenceable(240) [[F]], ptr [[S]])
 ; TUNIT-NEXT:    call void @__shlim(ptr noundef nonnull align 8 dereferenceable(240) [[F]], i64 noundef 0)
 ; TUNIT-NEXT:    [[CALL1:%.*]] = call double @__floatscan(ptr noundef nonnull align 8 dereferenceable(240) [[F]], i32 noundef 1, i32 noundef 1)
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 144, ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]])
 ; TUNIT-NEXT:    ret double [[CALL1]]
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@strtox
 ; CGSCC-SAME: (ptr [[S:%.*]]) unnamed_addr {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[F:%.*]] = alloca [[STRUCT__IO_FILE:%.*]], align 8
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 144, ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]]) #[[ATTR14:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]]) #[[ATTR14:[0-9]+]]
 ; CGSCC-NEXT:    [[CALL:%.*]] = call i32 @sh_fromstring(ptr noundef nonnull align 8 dereferenceable(240) [[F]], ptr [[S]])
 ; CGSCC-NEXT:    call void @__shlim(ptr noundef nonnull align 8 dereferenceable(240) [[F]], i64 noundef 0)
 ; CGSCC-NEXT:    [[CALL1:%.*]] = call double @__floatscan(ptr noundef nonnull align 8 dereferenceable(240) [[F]], i32 noundef 1, i32 noundef 1)
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 144, ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 8 captures(none) dereferenceable(240) [[F]])
 ; CGSCC-NEXT:    ret double [[CALL1]]
 ;
 entry:
   %f = alloca %struct._IO_FILE, align 8
-  call void @llvm.lifetime.start.p0(i64 144, ptr nonnull %f)
+  call void @llvm.lifetime.start.p0(ptr nonnull %f)
   %call = call i32 @sh_fromstring(ptr nonnull %f, ptr %s)
   call void @__shlim(ptr nonnull %f, i64 0)
   %call1 = call double @__floatscan(ptr nonnull %f, i32 %prec, i32 1)
-  call void @llvm.lifetime.end.p0(i64 144, ptr nonnull %f)
+  call void @llvm.lifetime.end.p0(ptr nonnull %f)
 
   ret double %call1
 }
@@ -620,7 +620,7 @@ entry:
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 ; Function Attrs: optsize
 declare dso_local i32 @sh_fromstring(...) local_unnamed_addr
@@ -632,7 +632,7 @@ declare dso_local void @__shlim(ptr, i64) local_unnamed_addr
 declare dso_local double @__floatscan(ptr, i32, i32) local_unnamed_addr
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ; Test 15
 ; propagate noalias to some callsite arguments that there is no possibly reachable capture before it
diff --git a/llvm/test/Transforms/Attributor/openmp_parallel.ll b/llvm/test/Transforms/Attributor/openmp_parallel.ll
index d7b194dd64071..54da16c58afc7 100644
--- a/llvm/test/Transforms/Attributor/openmp_parallel.ll
+++ b/llvm/test/Transforms/Attributor/openmp_parallel.ll
@@ -68,13 +68,13 @@ define internal void @.omp_outlined.(ptr noalias nocapture readonly %.global_tid
 ; TUNIT-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    br label [[OMP_PRECOND_THEN:%.*]]
 ; TUNIT:       omp.precond.then:
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]]) #[[ATTR3:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]]) #[[ATTR3:[0-9]+]]
 ; TUNIT-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]]) #[[ATTR3]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]]) #[[ATTR3]]
 ; TUNIT-NEXT:    store i32 197, ptr [[DOTOMP_UB]], align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]]) #[[ATTR3]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]]) #[[ATTR3]]
 ; TUNIT-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]]) #[[ATTR3]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]]) #[[ATTR3]]
 ; TUNIT-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 ; TUNIT-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
 ; TUNIT-NEXT:    call void @__kmpc_for_static_init_4(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 noundef 34, ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_IS_LAST]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_LB]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_UB]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_STRIDE]], i32 noundef 1, i32 noundef 1)
@@ -103,10 +103,10 @@ define internal void @.omp_outlined.(ptr noalias nocapture readonly %.global_tid
 ; TUNIT-NEXT:    br i1 [[CMP5]], label [[OMP_INNER_FOR_BODY]], label [[OMP_LOOP_EXIT]]
 ; TUNIT:       omp.loop.exit:
 ; TUNIT-NEXT:    call void @__kmpc_for_static_fini(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]])
 ; TUNIT-NEXT:    br label [[OMP_PRECOND_END:%.*]]
 ; TUNIT:       omp.precond.end:
 ; TUNIT-NEXT:    ret void
@@ -124,13 +124,13 @@ define internal void @.omp_outlined.(ptr noalias nocapture readonly %.global_tid
 ; CGSCC-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], 1
 ; CGSCC-NEXT:    br i1 [[CMP]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
 ; CGSCC:       omp.precond.then:
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]]) #[[ATTR3:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]]) #[[ATTR3:[0-9]+]]
 ; CGSCC-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]]) #[[ATTR3]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]]) #[[ATTR3]]
 ; CGSCC-NEXT:    store i32 [[SUB2]], ptr [[DOTOMP_UB]], align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]]) #[[ATTR3]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]]) #[[ATTR3]]
 ; CGSCC-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]]) #[[ATTR3]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]]) #[[ATTR3]]
 ; CGSCC-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 ; CGSCC-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
 ; CGSCC-NEXT:    call void @__kmpc_for_static_init_4(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP1]], i32 noundef 34, ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_IS_LAST]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_LB]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_UB]], ptr noundef nonnull align 4 dereferenceable(4) [[DOTOMP_STRIDE]], i32 noundef 1, i32 noundef 1)
@@ -159,10 +159,10 @@ define internal void @.omp_outlined.(ptr noalias nocapture readonly %.global_tid
 ; CGSCC-NEXT:    br i1 [[CMP5]], label [[OMP_INNER_FOR_BODY]], label [[OMP_LOOP_EXIT]]
 ; CGSCC:       omp.loop.exit:
 ; CGSCC-NEXT:    call void @__kmpc_for_static_fini(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP1]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_IS_LAST]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_STRIDE]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_UB]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[DOTOMP_LB]])
 ; CGSCC-NEXT:    br label [[OMP_PRECOND_END]]
 ; CGSCC:       omp.precond.end:
 ; CGSCC-NEXT:    ret void
@@ -178,13 +178,13 @@ entry:
   br i1 %cmp, label %omp.precond.then, label %omp.precond.end
 
 omp.precond.then:                                 ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %.omp.lb) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %.omp.lb) #3
   store i32 0, ptr %.omp.lb, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %.omp.ub) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %.omp.ub) #3
   store i32 %sub2, ptr %.omp.ub, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %.omp.stride) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %.omp.stride) #3
   store i32 1, ptr %.omp.stride, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %.omp.is_last) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %.omp.is_last) #3
   store i32 0, ptr %.omp.is_last, align 4
   %1 = load i32, ptr %.global_tid., align 4
   call void @__kmpc_for_static_init_4(ptr nonnull @1, i32 %1, i32 34, ptr nonnull %.omp.is_last, ptr nonnull %.omp.lb, ptr nonnull %.omp.ub, ptr nonnull %.omp.stride, i32 1, i32 1) #3
@@ -216,10 +216,10 @@ omp.inner.for.body:                               ; preds = %omp.inner.for.body,
 
 omp.loop.exit:                                    ; preds = %omp.inner.for.body, %omp.precond.then
   call void @__kmpc_for_static_fini(ptr nonnull @1, i32 %1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %.omp.is_last) #3
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %.omp.stride) #3
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %.omp.ub) #3
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %.omp.lb) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %.omp.is_last) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %.omp.stride) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %.omp.ub) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %.omp.lb) #3
   br label %omp.precond.end
 
 omp.precond.end:                                  ; preds = %omp.loop.exit, %entry
@@ -227,10 +227,10 @@ omp.precond.end:                                  ; preds = %omp.loop.exit, %ent
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 declare dso_local void @__kmpc_for_static_init_4(ptr, i32, i32, ptr, ptr, ptr, ptr, i32, i32) local_unnamed_addr
 
diff --git a/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll b/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll
index 472ed30367884..eb7d78f51018f 100644
--- a/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll
+++ b/llvm/test/Transforms/Attributor/reduced/register_benchmark_test.ll
@@ -201,7 +201,7 @@ declare dso_local void @_ZNSt6vectorIN12_GLOBAL__N_18TestCaseESaIS1_EED2Ev(ptr)
 declare dso_local void @_Z11BM_functionRN9benchmark5StateE(ptr dereferenceable(144)) #0
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #5
+declare void @llvm.lifetime.start.p0(ptr nocapture) #5
 
 ; Function Attrs: alwaysinline uwtable
 declare dso_local { i64, ptr } @_ZN9benchmark5State5beginEv(ptr) #6 align 2
@@ -216,7 +216,7 @@ declare dso_local zeroext i1 @_ZNK9benchmark5State13StateIteratorneERKS1_(ptr, p
 declare dso_local void @_ZNK9benchmark5State13StateIteratordeEv(ptr) #7 align 2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #5
+declare void @llvm.lifetime.end.p0(ptr nocapture) #5
 
 ; Function Attrs: alwaysinline nounwind uwtable
 declare dso_local dereferenceable(16) ptr @_ZN9benchmark5State13StateIteratorppEv(ptr) #7 align 2
diff --git a/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll b/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll
index fa942c99fb36f..82bed0f27c046 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-pointer-info.ll
@@ -116,7 +116,7 @@ define void @local_alloca_simplifiable_1(ptr noalias sret(%struct.S) align 4 %ag
 ; TUNIT-SAME: (ptr noalias nofree writeonly sret([[STRUCT_S:%.*]]) align 4 captures(none) dereferenceable_or_null(24) [[AGG_RESULT:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[S:%.*]] = alloca [[STRUCT_S]], align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 24, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR17:[0-9]+]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR17:[0-9]+]]
 ; TUNIT-NEXT:    [[F1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 3
 ; TUNIT-NEXT:    [[F2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 4
 ; TUNIT-NEXT:    [[F3:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 5
@@ -136,7 +136,7 @@ define void @local_alloca_simplifiable_1(ptr noalias sret(%struct.S) align 4 %ag
 ; TUNIT-NEXT:    store i32 4, ptr [[I212]], align 4, !tbaa [[TBAA13:![0-9]+]]
 ; TUNIT-NEXT:    [[I316:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 2
 ; TUNIT-NEXT:    store i32 4, ptr [[I316]], align 4, !tbaa [[TBAA14:![0-9]+]]
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 24, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
@@ -144,7 +144,7 @@ define void @local_alloca_simplifiable_1(ptr noalias sret(%struct.S) align 4 %ag
 ; CGSCC-SAME: (ptr noalias nofree noundef nonnull writeonly sret([[STRUCT_S:%.*]]) align 4 captures(none) dereferenceable(24) [[AGG_RESULT:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[S:%.*]] = alloca [[STRUCT_S]], align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 24, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR20:[0-9]+]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR20:[0-9]+]]
 ; CGSCC-NEXT:    [[F1:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 3
 ; CGSCC-NEXT:    store float 0x3FF19999A0000000, ptr [[F1]], align 4, !tbaa [[TBAA7:![0-9]+]]
 ; CGSCC-NEXT:    [[F2:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[S]], i64 0, i32 4
@@ -185,12 +185,12 @@ define void @local_alloca_simplifiable_1(ptr noalias sret(%struct.S) align 4 %ag
 ; CGSCC-NEXT:    [[ADD15:%.*]] = add nsw i32 [[I10]], [[I11]]
 ; CGSCC-NEXT:    [[I316:%.*]] = getelementptr inbounds [[STRUCT_S]], ptr [[AGG_RESULT]], i64 0, i32 2
 ; CGSCC-NEXT:    store i32 [[ADD15]], ptr [[I316]], align 4, !tbaa [[TBAA14]]
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 24, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(24) [[S]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
   %s = alloca %struct.S, align 4
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %s)
+  call void @llvm.lifetime.start.p0(ptr nonnull %s)
   %f1 = getelementptr inbounds %struct.S, ptr %s, i64 0, i32 3
   store float 0x3FF19999A0000000, ptr %f1, align 4, !tbaa !7
   %f2 = getelementptr inbounds %struct.S, ptr %s, i64 0, i32 4
@@ -231,13 +231,13 @@ entry:
   %add15 = add nsw i32 %i10, %i11
   %i316 = getelementptr inbounds %struct.S, ptr %agg.result, i64 0, i32 2
   store i32 %add15, ptr %i316, align 4, !tbaa !14
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %s)
+  call void @llvm.lifetime.end.p0(ptr nonnull %s)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ;    void local_alloca_simplifiable_2(void) {
 ;      char Bytes[1024];
@@ -260,7 +260,7 @@ define void @local_alloca_simplifiable_2() {
 ; TUNIT-SAME: () #[[ATTR3:[0-9]+]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1024, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR17]]
 ; TUNIT-NEXT:    br label [[FOR_COND:%.*]]
 ; TUNIT:       for.cond:
 ; TUNIT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
@@ -326,7 +326,7 @@ define void @local_alloca_simplifiable_2() {
 ; TUNIT-NEXT:    [[INDVARS_IV_NEXT13]] = add nuw nsw i64 [[INDVARS_IV12]], 1
 ; TUNIT-NEXT:    br label [[FOR_COND28]], !llvm.loop [[LOOP20:![0-9]+]]
 ; TUNIT:       for.end38:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1024, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret void
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn
@@ -334,7 +334,7 @@ define void @local_alloca_simplifiable_2() {
 ; CGSCC-SAME: () #[[ATTR3:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[BYTES:%.*]] = alloca [1024 x i8], align 16
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 1024, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR20]]
 ; CGSCC-NEXT:    br label [[FOR_COND:%.*]]
 ; CGSCC:       for.cond:
 ; CGSCC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ]
@@ -406,12 +406,12 @@ define void @local_alloca_simplifiable_2() {
 ; CGSCC-NEXT:    [[INDVARS_IV_NEXT13]] = add nuw nsw i64 [[INDVARS_IV12]], 1
 ; CGSCC-NEXT:    br label [[FOR_COND28]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CGSCC:       for.end38:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 1024, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(1024) [[BYTES]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret void
 ;
 entry:
   %Bytes = alloca [1024 x i8], align 16
-  call void @llvm.lifetime.start.p0(i64 1024, ptr nonnull %Bytes)
+  call void @llvm.lifetime.start.p0(ptr nonnull %Bytes)
   br label %for.cond
 
 for.cond:                                         ; preds = %for.inc, %entry
@@ -503,7 +503,7 @@ for.inc36:                                        ; preds = %for.body31
   br label %for.cond28, !llvm.loop !23
 
 for.end38:                                        ; preds = %for.cond.cleanup30
-  call void @llvm.lifetime.end.p0(i64 1024, ptr nonnull %Bytes)
+  call void @llvm.lifetime.end.p0(ptr nonnull %Bytes)
   ret void
 }
 
@@ -558,7 +558,7 @@ define i32 @multi_obj_simplifiable_1(i32 %cnd) {
 ; TUNIT-SAME: (i32 [[CND:%.*]]) #[[ATTR3]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[L:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
 ; TUNIT-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CND]], 0
 ; TUNIT-NEXT:    br i1 [[TOBOOL_NOT]], label [[COND_FALSE:%.*]], label [[COND_TRUE:%.*]]
 ; TUNIT:       cond.true:
@@ -566,7 +566,7 @@ define i32 @multi_obj_simplifiable_1(i32 %cnd) {
 ; TUNIT:       cond.false:
 ; TUNIT-NEXT:    br label [[COND_END]]
 ; TUNIT:       cond.end:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret i32 5
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
@@ -574,7 +574,7 @@ define i32 @multi_obj_simplifiable_1(i32 %cnd) {
 ; CGSCC-SAME: (i32 [[CND:%.*]]) #[[ATTR5:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[L:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
 ; CGSCC-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CND]], 0
 ; CGSCC-NEXT:    br i1 [[TOBOOL_NOT]], label [[COND_FALSE:%.*]], label [[COND_TRUE:%.*]]
 ; CGSCC:       cond.true:
@@ -582,12 +582,12 @@ define i32 @multi_obj_simplifiable_1(i32 %cnd) {
 ; CGSCC:       cond.false:
 ; CGSCC-NEXT:    br label [[COND_END]]
 ; CGSCC:       cond.end:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret i32 5
 ;
 entry:
   %L = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %L)
+  call void @llvm.lifetime.start.p0(ptr nonnull %L)
   store i32 5, ptr @GI1, align 4, !tbaa !3
   store i32 5, ptr %L, align 4, !tbaa !3
   %tobool.not = icmp eq i32 %cnd, 0
@@ -602,7 +602,7 @@ cond.false:                                       ; preds = %entry
 cond.end:                                         ; preds = %cond.false, %cond.true
   %cond = phi ptr [ @GI1, %cond.true ], [ %L, %cond.false ]
   %i1 = load i32, ptr %cond, align 4, !tbaa !3
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %L)
+  call void @llvm.lifetime.end.p0(ptr nonnull %L)
   ret i32 %i1
 }
 
@@ -620,7 +620,7 @@ define i32 @multi_obj_simplifiable_2(i32 %cnd) {
 ; TUNIT-SAME: (i32 [[CND:%.*]]) #[[ATTR3]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[L:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
 ; TUNIT-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CND]], 0
 ; TUNIT-NEXT:    br i1 [[TOBOOL_NOT]], label [[COND_FALSE:%.*]], label [[COND_TRUE:%.*]]
 ; TUNIT:       cond.true:
@@ -628,7 +628,7 @@ define i32 @multi_obj_simplifiable_2(i32 %cnd) {
 ; TUNIT:       cond.false:
 ; TUNIT-NEXT:    br label [[COND_END]]
 ; TUNIT:       cond.end:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret i32 5
 ;
 ; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
@@ -636,7 +636,7 @@ define i32 @multi_obj_simplifiable_2(i32 %cnd) {
 ; CGSCC-SAME: (i32 [[CND:%.*]]) #[[ATTR5]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[L:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
 ; CGSCC-NEXT:    [[TOBOOL_NOT:%.*]] = icmp eq i32 [[CND]], 0
 ; CGSCC-NEXT:    br i1 [[TOBOOL_NOT]], label [[COND_FALSE:%.*]], label [[COND_TRUE:%.*]]
 ; CGSCC:       cond.true:
@@ -644,12 +644,12 @@ define i32 @multi_obj_simplifiable_2(i32 %cnd) {
 ; CGSCC:       cond.false:
 ; CGSCC-NEXT:    br label [[COND_END]]
 ; CGSCC:       cond.end:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[L]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret i32 5
 ;
 entry:
   %L = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %L)
+  call void @llvm.lifetime.start.p0(ptr nonnull %L)
   %tobool.not = icmp eq i32 %cnd, 0
   br i1 %tobool.not, label %cond.false, label %cond.true
 
@@ -663,7 +663,7 @@ cond.end:                                         ; preds = %cond.false, %cond.t
   %cond = phi ptr [ @GI2, %cond.true ], [ %L, %cond.false ]
   store i32 5, ptr %cond, align 4, !tbaa !3
   %l = load i32, ptr %cond, align 4, !tbaa !3
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %L)
+  call void @llvm.lifetime.end.p0(ptr nonnull %L)
   ret i32 %l
 }
 
@@ -1528,8 +1528,8 @@ define i32 @local_alloca_not_simplifiable_1() {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[X:%.*]] = alloca i32, align 4
 ; TUNIT-NEXT:    [[Y:%.*]] = alloca i32, align 4
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]]) #[[ATTR17]]
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]]) #[[ATTR17]]
 ; TUNIT-NEXT:    store i32 1, ptr [[Y]], align 4, !tbaa [[TBAA3]]
 ; TUNIT-NEXT:    store i32 1, ptr [[X]], align 4, !tbaa [[TBAA3]]
 ; TUNIT-NEXT:    call void @escape(ptr noundef nonnull align 4 dereferenceable(4) [[X]])
@@ -1540,16 +1540,16 @@ define i32 @local_alloca_not_simplifiable_1() {
 ; TUNIT-NEXT:    [[I4:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA3]]
 ; TUNIT-NEXT:    [[ADD:%.*]] = add nsw i32 [[I3]], [[I4]]
 ; TUNIT-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[COND]]
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]])
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]])
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]])
 ; TUNIT-NEXT:    ret i32 [[ADD1]]
 ;
 ; CGSCC-LABEL: define {{[^@]+}}@local_alloca_not_simplifiable_1() {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[X:%.*]] = alloca i32, align 4
 ; CGSCC-NEXT:    [[Y:%.*]] = alloca i32, align 4
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]]) #[[ATTR20]]
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]]) #[[ATTR20]]
 ; CGSCC-NEXT:    store i32 1, ptr [[Y]], align 4, !tbaa [[TBAA3]]
 ; CGSCC-NEXT:    store i32 1, ptr [[X]], align 4, !tbaa [[TBAA3]]
 ; CGSCC-NEXT:    call void @escape(ptr noundef nonnull align 4 dereferenceable(4) [[X]])
@@ -1560,15 +1560,15 @@ define i32 @local_alloca_not_simplifiable_1() {
 ; CGSCC-NEXT:    [[I4:%.*]] = load i32, ptr [[Y]], align 4, !tbaa [[TBAA3]]
 ; CGSCC-NEXT:    [[ADD:%.*]] = add nsw i32 [[I3]], [[I4]]
 ; CGSCC-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[COND]]
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]])
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[Y]])
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[X]])
 ; CGSCC-NEXT:    ret i32 [[ADD1]]
 ;
 entry:
   %X = alloca i32, align 4
   %Y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %X)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %Y)
+  call void @llvm.lifetime.start.p0(ptr nonnull %X)
+  call void @llvm.lifetime.start.p0(ptr nonnull %Y)
   store i32 1, ptr %Y, align 4, !tbaa !3
   store i32 1, ptr %X, align 4, !tbaa !3
   call void @escape(ptr nonnull %X)
@@ -1579,8 +1579,8 @@ entry:
   %i4 = load i32, ptr %Y, align 4, !tbaa !3
   %add = add nsw i32 %i3, %i4
   %add1 = add nsw i32 %add, %cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %Y)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %X)
+  call void @llvm.lifetime.end.p0(ptr nonnull %Y)
+  call void @llvm.lifetime.end.p0(ptr nonnull %X)
   ret i32 %add1
 }
 
@@ -2755,7 +2755,7 @@ define hidden void @no_propagation_of_unknown_index_access(ptr %in, ptr %out, i3
 ; TUNIT-SAME: (ptr nofree readonly captures(none) [[IN:%.*]], ptr nofree writeonly captures(none) [[OUT:%.*]], i32 [[IDX:%.*]]) #[[ATTR1]] {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[BUF:%.*]] = alloca [128 x i32], align 16
-; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 512, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR17]]
 ; TUNIT-NEXT:    br label [[FOR_COND:%.*]]
 ; TUNIT:       for.cond:
 ; TUNIT-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -2776,7 +2776,7 @@ define hidden void @no_propagation_of_unknown_index_access(ptr %in, ptr %out, i3
 ; TUNIT-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[I3_0]], 128
 ; TUNIT-NEXT:    br i1 [[CMP5]], label [[FOR_BODY7]], label [[FOR_COND_CLEANUP6:%.*]]
 ; TUNIT:       for.cond.cleanup6:
-; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 512, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR17]]
+; TUNIT-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR17]]
 ; TUNIT-NEXT:    ret void
 ; TUNIT:       for.body7:
 ; TUNIT-NEXT:    [[IDXPROM8:%.*]] = sext i32 [[I3_0]] to i64
@@ -2797,7 +2797,7 @@ define hidden void @no_propagation_of_unknown_index_access(ptr %in, ptr %out, i3
 ; CGSCC-SAME: (ptr nofree readonly captures(none) [[IN:%.*]], ptr nofree writeonly captures(none) [[OUT:%.*]], i32 [[IDX:%.*]]) #[[ATTR13:[0-9]+]] {
 ; CGSCC-NEXT:  entry:
 ; CGSCC-NEXT:    [[BUF:%.*]] = alloca [128 x i32], align 16
-; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 512, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.start.p0(ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR20]]
 ; CGSCC-NEXT:    br label [[FOR_COND:%.*]]
 ; CGSCC:       for.cond:
 ; CGSCC-NEXT:    [[I_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY:%.*]] ]
@@ -2818,7 +2818,7 @@ define hidden void @no_propagation_of_unknown_index_access(ptr %in, ptr %out, i3
 ; CGSCC-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[I3_0]], 128
 ; CGSCC-NEXT:    br i1 [[CMP5]], label [[FOR_BODY7]], label [[FOR_COND_CLEANUP6:%.*]]
 ; CGSCC:       for.cond.cleanup6:
-; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 512, ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR20]]
+; CGSCC-NEXT:    call void @llvm.lifetime.end.p0(ptr noalias nofree noundef nonnull align 16 captures(none) dereferenceable(512) [[BUF]]) #[[ATTR20]]
 ; CGSCC-NEXT:    ret void
 ; CGSCC:       for.body7:
 ; CGSCC-NEXT:    [[IDXPROM8:%.*]] = sext i32 [[I3_0]] to i64
@@ -2836,7 +2836,7 @@ define hidden void @no_propagation_of_unknown_index_access(ptr %in, ptr %out, i3
 ;
 entry:
   %buf = alloca [128 x i32], align 16
-  call void @llvm.lifetime.start.p0(i64 512, ptr %buf) #2
+  call void @llvm.lifetime.start.p0(ptr %buf) #2
   br label %for.cond
 
 for.cond:                                         ; preds = %for.body, %entry
@@ -2862,7 +2862,7 @@ for.cond4:                                        ; preds = %for.body7, %for.con
   br i1 %cmp5, label %for.body7, label %for.cond.cleanup6
 
 for.cond.cleanup6:                                ; preds = %for.cond4
-  call void @llvm.lifetime.end.p0(i64 512, ptr %buf) #2
+  call void @llvm.lifetime.end.p0(ptr %buf) #2
   ret void
 
 for.body7:                                        ; preds = %for.cond4
diff --git a/llvm/test/Transforms/CodeExtractor/LoopExtractor_alloca.ll b/llvm/test/Transforms/CodeExtractor/LoopExtractor_alloca.ll
index b932a7dc0bf9f..09abf1f3cd85b 100644
--- a/llvm/test/Transforms/CodeExtractor/LoopExtractor_alloca.ll
+++ b/llvm/test/Transforms/CodeExtractor/LoopExtractor_alloca.ll
@@ -30,16 +30,16 @@ entry:
   br label %loop1
 
 loop1:
-  call void @llvm.lifetime.start.p0(i64 4, ptr %v1)
+  call void @llvm.lifetime.start.p0(ptr %v1)
   %r1 = call i32 @foo(ptr %v1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %v1)
+  call void @llvm.lifetime.end.p0(ptr %v1)
   %cmp1 = icmp ne i32 %r1, 0
   br i1 %cmp1, label %loop1, label %loop2
 
 loop2:
-  call void @llvm.lifetime.start.p0(i64 4, ptr %v2)
+  call void @llvm.lifetime.start.p0(ptr %v2)
   %r2 = call i32 @foo(ptr %v2)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %v2)
+  call void @llvm.lifetime.end.p0(ptr %v2)
   %cmp2 = icmp ne i32 %r2, 0
   br i1 %cmp2, label %loop2, label %exit
 
@@ -49,6 +49,6 @@ exit:
 
 declare i32 @foo(ptr)
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca.ll
index 9cdc37a2864ef..a24bb744e90c7 100644
--- a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca.ll
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca.ll
@@ -20,11 +20,11 @@ bb:
   br i1 %tmp4, label %bb6, label %bb5
 
 bb5:                                              ; preds = %bb
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp) #2
   store i32 %tmp3, ptr %tmp, align 4, !tbaa !2
   store i32 %tmp3, ptr @g, align 4, !tbaa !2
   call void @bar(ptr nonnull %tmp) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp) #2
   br label %bb6
 
 bb6:                                              ; preds = %bb5, %bb
@@ -32,14 +32,14 @@ bb6:                                              ; preds = %bb5, %bb
   ret i32 %tmp7
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare void @bar(ptr) local_unnamed_addr #2
 declare void @bar2(ptr, ptr) local_unnamed_addr #1
 
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define i32 @caller(i32 %arg) local_unnamed_addr #0 {
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca2.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca2.ll
index f4a37e7708726..22c0baf4b47ae 100644
--- a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca2.ll
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca2.ll
@@ -18,10 +18,10 @@ bb:
   br i1 %tmp4, label %bb6, label %bb5
 
 bb5:                                              ; preds = %bb
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp) #2
   store i32 %tmp3, ptr @g, align 4, !tbaa !2
   call void @bar(ptr nonnull %tmp) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp) #2
   br label %bb6
 
 bb6:                                              ; preds = %bb5, %bb
@@ -30,14 +30,14 @@ bb6:                                              ; preds = %bb5, %bb
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare void @bar(ptr) local_unnamed_addr #2
 declare void @bar2(ptr, ptr) local_unnamed_addr #1
 
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define i32 @caller(i32 %arg) local_unnamed_addr #0 {
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll
index bd51910aaf1c8..8b9c5dd619ba3 100644
--- a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll
@@ -7,7 +7,7 @@
 @g = external local_unnamed_addr global i32, align 4
 
 ; CHECK-LABEL: define{{.*}}@caller(
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %tmp.i)
+; CHECK: call void @llvm.lifetime.start.p0(ptr %tmp.i)
 ; CHECK-NEXT: call void @callee_unknown_use1.{{.*}}(ptr %tmp.i
 
 define i32 @callee_unknown_use1(i32 %arg) local_unnamed_addr #0 {
@@ -21,11 +21,11 @@ bb:
   br i1 %tmp4, label %bb6, label %bb5
 
 bb5:                                              ; preds = %bb
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp) #2
   store i32 %tmp3, ptr @g, align 4, !tbaa !2
   %tmp11 = bitcast ptr %tmp to ptr
   call void @bar(ptr nonnull %tmp11) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp) #2
   br label %bb6
 
 bb6:                                              ; preds = %bb5, %bb
@@ -36,14 +36,14 @@ bb6:                                              ; preds = %bb5, %bb
 
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare void @bar(ptr) local_unnamed_addr #2
 declare void @bar2(ptr, ptr) local_unnamed_addr #1
 
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define i32 @caller(i32 %arg) local_unnamed_addr #0 {
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll
index 54782c505e8f2..10be1c8f95990 100644
--- a/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll
@@ -18,11 +18,11 @@ bb:
   br i1 %tmp4, label %bb6, label %bb5
 
 bb5:                                              ; preds = %bb
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp) #2
   store i32 %tmp3, ptr %tmp, align 4, !tbaa !2
   store i32 %tmp3, ptr @g, align 4, !tbaa !2
   call void @bar(ptr nonnull %tmp) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp) #2
   br label %bb6
 
 bb6:                                              ; preds = %bb5, %bb
@@ -32,14 +32,14 @@ bb6:                                              ; preds = %bb5, %bb
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare void @bar(ptr) local_unnamed_addr #2
 declare void @bar2(ptr, ptr) local_unnamed_addr #1
 
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define i32 @caller(i32 %arg) local_unnamed_addr #0 {
diff --git a/llvm/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll b/llvm/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll
index bdf9e233dc30f..5e0ce20645265 100644
--- a/llvm/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll
+++ b/llvm/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll
@@ -26,10 +26,10 @@ bb5:                                              ; preds = %bb4, %bb1, %bb
 ; CHECK-LABEL: bb:
 ; CHECK-NEXT:  [[CALL26LOC:%.*]] = alloca ptr
 ; CHECK-LABEL: codeRepl.i:
-; CHECK-NEXT:   call void @llvm.lifetime.start.p0(i64 -1, ptr [[CALL26LOC]])
+; CHECK-NEXT:   call void @llvm.lifetime.start.p0(ptr [[CALL26LOC]])
 ; CHECK-NEXT:   call void @bar.1.bb1(ptr [[CALL26LOC]])
 ; CHECK-NEXT:   %call26.reload.i = load ptr, ptr [[CALL26LOC]]
-; CHECK-NEXT:   call void @llvm.lifetime.end.p0(i64 -1, ptr [[CALL26LOC]])
+; CHECK-NEXT:   call void @llvm.lifetime.end.p0(ptr [[CALL26LOC]])
 define ptr @dummy_caller(i32 %arg) {
 bb:
   %tmp = tail call ptr @bar(i32 %arg)
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink.ll b/llvm/test/Transforms/CodeExtractor/live_shrink.ll
index f5debc5035859..43cc248fa874a 100644
--- a/llvm/test/Transforms/CodeExtractor/live_shrink.ll
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink.ll
@@ -8,7 +8,7 @@
 define void @_Z3foov() local_unnamed_addr  {
 bb:
   %tmp = alloca %class.A, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp)
   %tmp2 = load i32, ptr @cond, align 4, !tbaa !2
   %tmp3 = icmp eq i32 %tmp2, 0
   br i1 %tmp3, label %bb4, label %bb5
@@ -18,17 +18,17 @@ bb4:                                              ; preds = %bb
   br label %bb5
 
 bb5:                                              ; preds = %bb4, %bb
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp)
   ret void
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @_ZN1A7memfuncEv(ptr) local_unnamed_addr
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ; Function Attrs: uwtable
 define void @_Z3goov() local_unnamed_addr  {
@@ -49,8 +49,8 @@ bb:
 ; CHECK-LABEL: define internal void @_Z3foov.1.
 ; CHECK: newFuncRoot:
 ; CHECK-NEXT:  %tmp = alloca %class.A
-; CHECK-NEXT:  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp)
-; CHECK:  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp)
+; CHECK-NEXT:  call void @llvm.lifetime.start.p0(ptr nonnull %tmp)
+; CHECK:  call void @llvm.lifetime.end.p0(ptr nonnull %tmp)
 ; CHECK-NEXT:  br label %bb5.exitStub
 
 
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink_gep.ll b/llvm/test/Transforms/CodeExtractor/live_shrink_gep.ll
index e9d5fb6556faf..ef815adc6387e 100644
--- a/llvm/test/Transforms/CodeExtractor/live_shrink_gep.ll
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink_gep.ll
@@ -9,7 +9,7 @@
 define void @_Z3foov() local_unnamed_addr  {
 bb:
   %tmp = alloca %class.A, align 1
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp)
   %tmp2 = load i32, ptr @cond, align 4, !tbaa !2
   %tmp3 = icmp eq i32 %tmp2, 0
   br i1 %tmp3, label %bb4, label %bb5
@@ -19,17 +19,17 @@ bb4:                                              ; preds = %bb
   br label %bb5
 
 bb5:                                              ; preds = %bb4, %bb
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp)
   ret void
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @_ZN1A7memfuncEv(ptr) local_unnamed_addr
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ; Function Attrs: uwtable
 define void @_Z3goov() local_unnamed_addr  {
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink_hoist.ll b/llvm/test/Transforms/CodeExtractor/live_shrink_hoist.ll
index 6f63bca00c633..70748542fd0ee 100644
--- a/llvm/test/Transforms/CodeExtractor/live_shrink_hoist.ll
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink_hoist.ll
@@ -9,7 +9,7 @@
 define void @_Z3foov() local_unnamed_addr  {
 bb:
   %tmp = alloca %class.A, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp)
   %tmp2 = load i32, ptr @cond, align 4, !tbaa !2
   %tmp3 = icmp eq i32 %tmp2, 0
   br i1 %tmp3, label %bb4, label %bb9
@@ -29,17 +29,17 @@ bb8:                                              ; preds = %bb4
   br label %bb9
 
 bb9:                                              ; preds = %bb8, %bb4, %bb
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp)
   ret void
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @_ZN1A7memfuncEv(ptr) local_unnamed_addr
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ; Function Attrs: uwtable
 define void @_Z3goov() local_unnamed_addr  {
@@ -50,7 +50,7 @@ bb:
 
 ; CHECK-LABEL: define internal void @_Z3foov.1.
 ; CHECK: bb9:
-; CHECK: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp)
+; CHECK: call void @llvm.lifetime.end.p0(ptr nonnull %tmp)
 ; CHECK:  br label %.exitStub
 
 
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink_multiple.ll b/llvm/test/Transforms/CodeExtractor/live_shrink_multiple.ll
index 2512ac96b46ce..1d0af23e74c7f 100644
--- a/llvm/test/Transforms/CodeExtractor/live_shrink_multiple.ll
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink_multiple.ll
@@ -8,8 +8,8 @@ define void @_Z3foov() local_unnamed_addr  {
 bb:
   %tmp = alloca %class.A, align 4
   %tmp1 = alloca %class.A, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp1)
   %tmp4 = load i32, ptr @cond, align 4, !tbaa !2
   %tmp5 = icmp eq i32 %tmp4, 0
   br i1 %tmp5, label %bb6, label %bb7
@@ -19,18 +19,18 @@ bb6:                                              ; preds = %bb
   br label %bb7
 
 bb7:                                              ; preds = %bb6, %bb
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp)
   ret void
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @_ZN1A7memfuncEv(ptr) local_unnamed_addr
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ; Function Attrs: uwtable
 define void @_Z3goov() local_unnamed_addr  {
diff --git a/llvm/test/Transforms/CodeExtractor/live_shrink_unsafe.ll b/llvm/test/Transforms/CodeExtractor/live_shrink_unsafe.ll
index 7942418e8fb2e..c5bd6265f81dd 100644
--- a/llvm/test/Transforms/CodeExtractor/live_shrink_unsafe.ll
+++ b/llvm/test/Transforms/CodeExtractor/live_shrink_unsafe.ll
@@ -14,8 +14,8 @@ define void @_Z3foo_unknown_mem_accessv() local_unnamed_addr  {
 bb:
   %tmp = alloca %class.A, align 4
   %tmp1 = alloca %class.A, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp1)
   %tmp4 = load ptr, ptr @condptr, align 8, !tbaa !2
   %tmp5 = load i32, ptr %tmp4, align 4, !tbaa !6
   %tmp6 = icmp eq i32 %tmp5, 0
@@ -26,20 +26,20 @@ bb7:                                              ; preds = %bb
   br label %bb8
 
 bb8:                                              ; preds = %bb7, %bb
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp)
   ret void
 }
 
 declare void @_Z3barv() local_unnamed_addr
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 declare void @_ZN1A7memfuncEv(ptr) local_unnamed_addr
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @_Z3foo_unknown_calli(i32 %arg) local_unnamed_addr {
 bb:
   %tmp = alloca %class.A, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp)
   tail call void @_Z3barv()
   %tmp2 = icmp eq i32 %arg, 0
   br i1 %tmp2, label %bb3, label %bb4
@@ -49,7 +49,7 @@ bb3:                                              ; preds = %bb
   br label %bb4
 
 bb4:                                              ; preds = %bb3, %bb
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp)
   ret void
 }
 
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll
index 3f113e6ea1632..d2b79ab1151b7 100644
--- a/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/tailcall-dup.ll
@@ -4,8 +4,8 @@ target triple = "armv8m.main-none-eabi"
 
 declare ptr @f0()
 declare ptr @f1()
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 
 define ptr @tail_dup() {
 ; CHECK-LABEL: tail_dup
@@ -15,7 +15,7 @@ define ptr @tail_dup() {
 ; CHECK-NEXT: ret ptr
 bb0:
   %a = alloca i32
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %a) nounwind
+  call void @llvm.lifetime.start.p0(ptr %a) nounwind
   %tmp0 = tail call ptr @f0()
   br label %return
 bb1:
@@ -23,7 +23,7 @@ bb1:
   br label %return
 return:
   %retval = phi ptr [ %tmp0, %bb0 ], [ %tmp1, %bb1 ]
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %a) nounwind
+  call void @llvm.lifetime.end.p0(ptr %a) nounwind
   ret ptr %retval
 }
 
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll b/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll
index dd47d5eb6cc46..f72756db8cad3 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/tailcall-assume-xbb.ll
@@ -14,7 +14,7 @@
 define ptr @foo(i64 %size, i64 %v1, i64 %v2) {
 entry:
   %a = alloca i8
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %a) nounwind
+  call void @llvm.lifetime.start.p0(ptr %a) nounwind
   %cmp1 = icmp ult i64 %size, 1025
   br i1 %cmp1, label %if.end, label %case1
 
@@ -42,12 +42,12 @@ exit1:
 
 exit2:
   %retval2 = phi ptr [ %ret1, %case1 ], [ %retval1, %exit1 ]
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %a) nounwind
+  call void @llvm.lifetime.end.p0(ptr %a) nounwind
   ret ptr %retval2
 }
 
 declare void @llvm.assume(i1)
 declare ptr @qux()
 declare ptr @bar()
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-intrinsics.ll b/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-intrinsics.ll
index 6bf268b1c162c..6997c9f11a808 100644
--- a/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-intrinsics.ll
+++ b/llvm/test/Transforms/ConstantHoisting/AArch64/const-hoist-intrinsics.ll
@@ -65,25 +65,25 @@ define void @test_free_intrinsics(i64 %x) {
 ; CHECK-LABEL: @test_free_intrinsics(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[PTR:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 100000000032, ptr [[PTR]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 100000000064, ptr [[PTR]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 100000000128, ptr [[PTR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PTR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PTR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PTR]])
 ; CHECK-NEXT:    [[I:%.*]] = call ptr @llvm.invariant.start.p0(i64 100000000256, ptr [[PTR]])
 ; CHECK-NEXT:    call void @llvm.invariant.end.p0(ptr [[I]], i64 100000000256, ptr [[PTR]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %ptr = alloca i8
-  call void @llvm.lifetime.start.p0(i64 100000000032, ptr %ptr)
-  call void @llvm.lifetime.start.p0(i64 100000000064, ptr %ptr)
-  call void @llvm.lifetime.end.p0(i64 100000000128, ptr %ptr)
+  call void @llvm.lifetime.start.p0(ptr %ptr)
+  call void @llvm.lifetime.start.p0(ptr %ptr)
+  call void @llvm.lifetime.end.p0(ptr %ptr)
   %i = call ptr @llvm.invariant.start.p0(i64 100000000256, ptr %ptr)
   call void @llvm.invariant.end.p0(ptr %i, i64 100000000256, ptr %ptr)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 
 declare ptr @llvm.invariant.start.p0(i64, ptr nocapture)
 declare void @llvm.invariant.end.p0(ptr, i64, ptr nocapture)
diff --git a/llvm/test/Transforms/Coroutines/coro-alloca-06.ll b/llvm/test/Transforms/Coroutines/coro-alloca-06.ll
index 89149ceba4c14..bf75196047aff 100644
--- a/llvm/test/Transforms/Coroutines/coro-alloca-06.ll
+++ b/llvm/test/Transforms/Coroutines/coro-alloca-06.ll
@@ -17,11 +17,11 @@ entry:
 tricky:
   %2 = call ptr @await_suspend()
   store ptr %2, ptr %0, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %1)
+  call void @llvm.lifetime.start.p0(ptr %1)
   store ptr %0, ptr %1, align 8
   %3 = load ptr, ptr %1, align 8
   %4 = load ptr, ptr %3, align 8
-  call void @llvm.lifetime.end.p0(i64 8, ptr %1)
+  call void @llvm.lifetime.end.p0(ptr %1)
   br label %finish
 
 finish:
@@ -49,9 +49,9 @@ suspend:
 
 ; CHECK:         [[TMP2:%.*]] = call ptr @await_suspend()
 ; CHECK-NEXT:    store ptr [[TMP2]], ptr [[TMP0]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP1]])
 ; CHECK-NEXT:    store ptr [[TMP0]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[TMP1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP1]])
 ;
 
 declare ptr @llvm.coro.free(token, ptr)
@@ -65,8 +65,8 @@ declare i1 @llvm.coro.alloc(token)
 declare ptr @llvm.coro.begin(token, ptr)
 declare i1 @llvm.coro.end(ptr, i1, token)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare ptr @await_suspend()
 declare void @print(ptr nocapture)
diff --git a/llvm/test/Transforms/Coroutines/coro-alloca-07.ll b/llvm/test/Transforms/Coroutines/coro-alloca-07.ll
index 3b0acdd794af4..8bfb8cfabbd27 100644
--- a/llvm/test/Transforms/Coroutines/coro-alloca-07.ll
+++ b/llvm/test/Transforms/Coroutines/coro-alloca-07.ll
@@ -13,11 +13,11 @@ entry:
   br i1 %n, label %flag_true, label %flag_false
 
 flag_true:
-  call void @llvm.lifetime.start.p0(i64 8, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
   br label %merge
 
 flag_false:
-  call void @llvm.lifetime.start.p0(i64 8, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
   br label %merge
 
 merge:
@@ -51,7 +51,7 @@ declare i1 @llvm.coro.alloc(token)
 declare ptr @llvm.coro.begin(token, ptr)
 declare i1 @llvm.coro.end(ptr, i1, token)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @print(ptr)
 declare noalias ptr @malloc(i32)
diff --git a/llvm/test/Transforms/Coroutines/coro-alloca-08.ll b/llvm/test/Transforms/Coroutines/coro-alloca-08.ll
index 5a14a0eb98869..80be62ac64c8c 100644
--- a/llvm/test/Transforms/Coroutines/coro-alloca-08.ll
+++ b/llvm/test/Transforms/Coroutines/coro-alloca-08.ll
@@ -18,9 +18,9 @@ entry:
   %alloc = call ptr @malloc(i64 16) #3
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
 
-  call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   call void @consume.i8.array(ptr %testval)
-  call void @llvm.lifetime.end.p0(i64 100, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
 
   %save = call token @llvm.coro.save(ptr null)
   %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
@@ -53,9 +53,9 @@ entry:
 await.ready:
   %StrayCoroSave = call token @llvm.coro.save(ptr null)
 
-  call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   call void @consume.i8.array(ptr %testval)
-  call void @llvm.lifetime.end.p0(i64 100, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
 
   br label %exit
 exit:
@@ -76,5 +76,5 @@ declare ptr @llvm.coro.frame() #5
 declare i8 @llvm.coro.suspend(token, i1) #3
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-alloca-09.ll b/llvm/test/Transforms/Coroutines/coro-alloca-09.ll
deleted file mode 100644
index 5c60c5be46206..0000000000000
--- a/llvm/test/Transforms/Coroutines/coro-alloca-09.ll
+++ /dev/null
@@ -1,57 +0,0 @@
-; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
-
-%"struct.std::coroutine_handle" = type { ptr }
-%"struct.std::coroutine_handle.0" = type { %"struct.std::coroutine_handle" }
-%"struct.lean_future<int>::Awaiter" = type { i32, %"struct.std::coroutine_handle.0" }
-
-declare ptr @malloc(i64)
-
-%i8.array = type { [100 x i8] }
-declare void @consume.i8(ptr)
-
-; The testval lives across suspend point so that it should be put on the frame.
-; However, part of testval has lifetime marker which indicates the part
-; wouldn't live across suspend point.
-; This test whether or not %testval would be put on the frame by ignoring the
-; partial lifetime markers.
-define void @foo(ptr %to_store) presplitcoroutine {
-entry:
-  %testval = alloca %i8.array
-  %subrange = getelementptr inbounds %i8.array, ptr %testval, i64 0, i32 0, i64 50
-  %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
-  %alloc = call ptr @malloc(i64 16) #3
-  %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
-
-  call void @llvm.lifetime.start.p0(i64 50, ptr %subrange)
-  call void @consume.i8(ptr %subrange)
-  call void @llvm.lifetime.end.p0(i64 50, ptr  %subrange)
-  store ptr %testval, ptr %to_store
-
-  %save = call token @llvm.coro.save(ptr null)
-  %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
-  switch i8 %suspend, label %exit [
-    i8 0, label %await.ready
-    i8 1, label %exit
-  ]
-await.ready:
-  %StrayCoroSave = call token @llvm.coro.save(ptr null)
-  br label %exit
-exit:
-  call i1 @llvm.coro.end(ptr null, i1 false, token none)
-  ret void
-}
-
-; Verify that for both foo and bar, testval isn't put on the frame.
-; CHECK: %foo.Frame = type { ptr, ptr, %i8.array, i1 }
-
-declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr)
-declare i1 @llvm.coro.alloc(token) #3
-declare i64 @llvm.coro.size.i64() #5
-declare ptr @llvm.coro.begin(token, ptr writeonly) #3
-declare token @llvm.coro.save(ptr) #3
-declare ptr @llvm.coro.frame() #5
-declare i8 @llvm.coro.suspend(token, i1) #3
-declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
-declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-alloca-loop-carried-address.ll b/llvm/test/Transforms/Coroutines/coro-alloca-loop-carried-address.ll
index f828b226698fb..8b8dbacbfc5c7 100644
--- a/llvm/test/Transforms/Coroutines/coro-alloca-loop-carried-address.ll
+++ b/llvm/test/Transforms/Coroutines/coro-alloca-loop-carried-address.ll
@@ -50,7 +50,7 @@ entry:
   br label %loop
 
 loop:
-  call void @llvm.lifetime.start(i64 8, ptr %stackvar0)
+  call void @llvm.lifetime.start(ptr %stackvar0)
 
   store i64 1234, ptr %stackvar0
 
@@ -58,7 +58,7 @@ loop:
   ; %stackvar1 and rely on it staying the same across suspension.
   call void @bar()
 
-  call void @llvm.lifetime.end(i64 8, ptr %stackvar0)
+  call void @llvm.lifetime.end(ptr %stackvar0)
 
   %save = call token @llvm.coro.save(ptr null)
   %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
@@ -81,5 +81,5 @@ declare ptr @llvm.coro.begin(token, ptr writeonly)
 declare token @llvm.coro.save(ptr)
 declare i8 @llvm.coro.suspend(token, i1)
 declare i1 @llvm.coro.end(ptr, i1, token)
-declare void @llvm.lifetime.start(i64, ptr nocapture)
-declare void @llvm.lifetime.end(i64, ptr nocapture)
+declare void @llvm.lifetime.start(ptr nocapture)
+declare void @llvm.lifetime.end(ptr nocapture)
diff --git a/llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-infinite-loop-bug.ll b/llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-infinite-loop-bug.ll
index 07b3bd8fa94ac..d662638d2dd9a 100644
--- a/llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-infinite-loop-bug.ll
+++ b/llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-infinite-loop-bug.ll
@@ -49,7 +49,7 @@ entry:
   %id = call token @llvm.coro.id.async(i32 128, i32 16, i32 0,
           ptr @my_async_function_fp)
   %hdl = call ptr @llvm.coro.begin(token %id, ptr null)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %escaped_addr)
+  call void @llvm.lifetime.start.p0(ptr %escaped_addr)
   call void @escape(ptr %escaped_addr)
   br label %callblock
 
@@ -80,6 +80,6 @@ declare void @llvm.coro.async.context.dealloc(ptr)
 declare swiftcc void @asyncSuspend(ptr)
 declare ptr @llvm.coro.async.resume()
 declare void @llvm.coro.async.size.replace(ptr, ptr)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
+declare void @llvm.lifetime.end.p0(ptr nocapture) #0
 attributes #0 = { argmemonly nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-start-bug.ll b/llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-start-bug.ll
index 40101595092b0..49c4207887340 100644
--- a/llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-start-bug.ll
+++ b/llvm/test/Transforms/Coroutines/coro-async-addr-lifetime-start-bug.ll
@@ -43,7 +43,7 @@ entry:
   %id = call token @llvm.coro.id.async(i32 128, i32 16, i32 0,
           ptr @my_async_function_fp)
   %hdl = call ptr @llvm.coro.begin(token %id, ptr null)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %escaped_addr)
+  call void @llvm.lifetime.start.p0(ptr %escaped_addr)
   br label %callblock
 
 
@@ -81,7 +81,7 @@ loop:
   br label %callblock
 
 loop_exit:
-  call void @llvm.lifetime.end.p0(i64 4, ptr %escaped_addr)
+  call void @llvm.lifetime.end.p0(ptr %escaped_addr)
   call i1 (ptr, i1, ...) @llvm.coro.end.async(ptr %hdl, i1 false)
   unreachable
 }
@@ -104,6 +104,6 @@ declare void @llvm.coro.async.context.dealloc(ptr)
 declare swiftcc void @asyncSuspend(ptr)
 declare ptr @llvm.coro.async.resume()
 declare void @llvm.coro.async.size.replace(ptr, ptr)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
+declare void @llvm.lifetime.end.p0(ptr nocapture) #0
 attributes #0 = { argmemonly nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/Coroutines/coro-async.ll b/llvm/test/Transforms/Coroutines/coro-async.ll
index e5d2e6c2c42c6..331d6a60bed6b 100644
--- a/llvm/test/Transforms/Coroutines/coro-async.ll
+++ b/llvm/test/Transforms/Coroutines/coro-async.ll
@@ -496,6 +496,35 @@ entry:
 ; CHECK: call void @use(ptr null)
 ; CHECK: ret
 
+@simpleFuncTu = global <{i32, i32}> <{
+  i32 trunc (i64 sub (i64 ptrtoint (ptr @simpleFunc to i64),
+             i64 ptrtoint (ptr @simpleFuncTu to i64)) to i32), i32 16 }>
+
+define swifttailcc void @simpleFunc(ptr swiftasync %0) presplitcoroutine {
+entry:
+  %1 = alloca ptr, align 8
+  %2 = call token @llvm.coro.id.async(i32 16, i32 16, i32 0, ptr @simpleFuncTu)
+  %3 = call ptr @llvm.coro.begin(token %2, ptr null)
+  store ptr %0, ptr %1, align 8
+  %4 = load ptr, ptr %1, align 8
+  %5 = getelementptr inbounds <{ ptr, ptr }>, ptr %4, i32 0, i32 1
+  %6 = load ptr, ptr %5, align 8
+  %7 = load ptr, ptr %1, align 8
+  %8 = call i1 (ptr, i1, ...) @llvm.coro.end.async(ptr %3, i1 false, ptr @simpleFunc.0, ptr %6, ptr %7)
+  unreachable
+}
+
+; CHECK-LABEL: define swifttailcc void @simpleFunc(ptr swiftasync %0) {
+; CHECK-NOT: define
+; CHECK:  [[RESUME:%.*]] = load ptr
+; CHECK:  musttail call swifttailcc void [[RESUME]]
+
+define internal swifttailcc void @simpleFunc.0(ptr %0, ptr %1) alwaysinline {
+entry:
+  musttail call swifttailcc void %0(ptr swiftasync %1)
+  ret void
+}
+
 declare { ptr, ptr, ptr, ptr } @llvm.coro.suspend.async.sl_p0i8p0i8p0i8p0i8s(i32, ptr, ptr, ...)
 declare ptr @llvm.coro.prepare.async(ptr)
 declare token @llvm.coro.id.async(i32, i32, i32, ptr)
diff --git a/llvm/test/Transforms/Coroutines/coro-byval-param.ll b/llvm/test/Transforms/Coroutines/coro-byval-param.ll
index 47059182d8ed6..38ab5ac481cd9 100644
--- a/llvm/test/Transforms/Coroutines/coro-byval-param.ll
+++ b/llvm/test/Transforms/Coroutines/coro-byval-param.ll
@@ -19,7 +19,7 @@ coro.alloc:                                       ; preds = %entry
 coro.init:                                        ; preds = %coro.alloc, %entry
   %3 = phi ptr [ null, %entry ], [ %call, %coro.alloc ]
   %4 = call ptr @llvm.coro.begin(token %0, ptr %3) #10
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %__promise) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %__promise) #2
   %call2 = call ptr @_ZN4task12promise_type17get_return_objectEv(ptr nonnull dereferenceable(1) %__promise)
   call void @initial_suspend(ptr nonnull dereferenceable(1) %__promise)
   %5 = call token @llvm.coro.save(ptr null)
@@ -31,9 +31,9 @@ coro.init:                                        ; preds = %coro.alloc, %entry
   ]
 
 init.ready:                                       ; preds = %coro.init
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %a2) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %a2) #2
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a2, ptr align 8 %a1, i64 24, i1 false)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %a2) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %a2) #2
   call void @_ZN4task12promise_type13final_suspendEv(ptr nonnull dereferenceable(1) %__promise) #2
   %7 = call token @llvm.coro.save(ptr null)
   call fastcc void @_ZNSt12experimental13coroutines_v116coroutine_handleIN4task12promise_typeEE12from_addressEPv(ptr %4) #2
@@ -42,7 +42,7 @@ init.ready:                                       ; preds = %coro.init
   br i1 %switch, label %cleanup33, label %coro.ret
 
 cleanup33:                                        ; preds = %init.ready, %coro.init
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %__promise) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %__promise) #2
   %9 = call ptr @llvm.coro.free(token %0, ptr %4)
   %.not = icmp eq ptr %9, null
   br i1 %.not, label %coro.ret, label %coro.free
@@ -75,7 +75,7 @@ declare i64 @llvm.coro.size.i64() #4
 declare ptr @llvm.coro.begin(token, ptr writeonly) #2
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #5
+declare void @llvm.lifetime.start.p0(ptr nocapture) #5
 
 ; Function Attrs: argmemonly nofree nounwind willreturn
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #6
@@ -93,7 +93,7 @@ declare token @llvm.coro.save(ptr) #2
 declare hidden fastcc void @_ZNSt12experimental13coroutines_v116coroutine_handleIN4task12promise_typeEE12from_addressEPv(ptr) unnamed_addr #7 align 2
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #5
+declare void @llvm.lifetime.end.p0(ptr nocapture) #5
 
 ; Function Attrs: nounwind
 declare i8 @llvm.coro.suspend(token, i1) #2
diff --git a/llvm/test/Transforms/Coroutines/coro-elide-musttail.ll b/llvm/test/Transforms/Coroutines/coro-elide-musttail.ll
index 6c6e5a69af91d..d369a2177b7cb 100644
--- a/llvm/test/Transforms/Coroutines/coro-elide-musttail.ll
+++ b/llvm/test/Transforms/Coroutines/coro-elide-musttail.ll
@@ -48,7 +48,7 @@ entry:
 }
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
 
 ; Function Attrs: argmemonly nounwind readonly
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #1
diff --git a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-00.ll b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-00.ll
index c9700c8a09961..bf08d6ff0b205 100644
--- a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-00.ll
+++ b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-00.ll
@@ -16,33 +16,33 @@ entry:
   br i1 %cond, label %then, label %else
 
 then:
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %data)
+  call void @llvm.lifetime.start.p0(ptr nonnull %data)
   call void @consume(ptr %data)
   %suspend.value = call i8 @llvm.coro.suspend(token none, i1 false)
   switch i8 %suspend.value, label %coro.ret [i8 0, label %resume
                                              i8 1, label %cleanup1]
 
 resume:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %data)
+  call void @llvm.lifetime.end.p0(ptr nonnull %data)
   br label %cleanup1
 
 cleanup1:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %data)
+  call void @llvm.lifetime.end.p0(ptr nonnull %data)
   br label %cleanup
 
 else:
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %data2)
+  call void @llvm.lifetime.start.p0(ptr nonnull %data2)
   call void @consume(ptr %data2)
   %suspend.value2 = call i8 @llvm.coro.suspend(token none, i1 false)
   switch i8 %suspend.value2, label %coro.ret [i8 0, label %resume2
                                               i8 1, label %cleanup2]
 
 resume2:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %data2)
+  call void @llvm.lifetime.end.p0(ptr nonnull %data2)
   br label %cleanup2
 
 cleanup2:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %data2)
+  call void @llvm.lifetime.end.p0(ptr nonnull %data2)
   br label %cleanup
 
 cleanup:
@@ -72,5 +72,5 @@ declare noalias ptr @malloc(i32)
 declare double @print(double)
 declare void @free(ptr)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-01.ll b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-01.ll
index 584caa356e9b8..78c6f0cacd695 100644
--- a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-01.ll
+++ b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-01.ll
@@ -17,10 +17,10 @@ entry:
   br label %init.ready
 init.ready:
   %1 = call noalias nonnull ptr @llvm.coro.begin(token %0, ptr null)
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %__promise)
+  call void @llvm.lifetime.start.p0(ptr nonnull %__promise)
   br i1 %cond, label %if.then, label %if.else
 if.then:
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call void @consume(ptr nonnull %a)
   %save = call token @llvm.coro.save(ptr null)
   %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
@@ -29,10 +29,10 @@ if.then:
     i8 1, label %cleanup1
   ]
 await.ready:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   br label %cleanup1
 if.else:
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %b)
+  call void @llvm.lifetime.start.p0(ptr nonnull %b)
   call void @consume(ptr nonnull %b)
   %save2 = call token @llvm.coro.save(ptr null)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
@@ -41,13 +41,13 @@ if.else:
     i8 1, label %cleanup2
   ]
 await2.ready:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   br label %cleanup2
 cleanup1:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   br label %cleanup
 cleanup2:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   br label %cleanup
 cleanup:
   call ptr @llvm.coro.free(token %0, ptr %1)
@@ -69,5 +69,5 @@ declare ptr @llvm.coro.frame() #5
 declare i8 @llvm.coro.suspend(token, i1) #3
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-02.ll b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-02.ll
index f916ebb915982..82657318d7785 100644
--- a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-02.ll
+++ b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-02.ll
@@ -19,10 +19,10 @@ entry:
   br label %init.ready
 init.ready:
   %1 = call noalias nonnull ptr @llvm.coro.begin(token %0, ptr null)
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %__promise)
+  call void @llvm.lifetime.start.p0(ptr nonnull %__promise)
   br i1 %cond, label %if.then, label %if.else
 if.then:
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call void @consume(ptr nonnull %a)
   %save = call token @llvm.coro.save(ptr null)
   %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
@@ -31,10 +31,10 @@ if.then:
     i8 1, label %cleanup1
   ]
 await.ready:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   br label %cleanup1
 if.else:
-  call void @llvm.lifetime.start.p0(i64 300, ptr nonnull %b)
+  call void @llvm.lifetime.start.p0(ptr nonnull %b)
   call void @consume.2(ptr nonnull %b)
   %save2 = call token @llvm.coro.save(ptr null)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
@@ -43,13 +43,13 @@ if.else:
     i8 1, label %cleanup2
   ]
 await2.ready:
-  call void @llvm.lifetime.end.p0(i64 300, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   br label %cleanup2
 cleanup1:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   br label %cleanup
 cleanup2:
-  call void @llvm.lifetime.end.p0(i64 300, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   br label %cleanup
 cleanup:
   call ptr @llvm.coro.free(token %0, ptr %1)
@@ -70,5 +70,5 @@ declare ptr @llvm.coro.frame() #5
 declare i8 @llvm.coro.suspend(token, i1) #3
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-03.ll b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-03.ll
index 525df873a0f0c..66d41372cd9e7 100644
--- a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-03.ll
+++ b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-03.ll
@@ -16,33 +16,33 @@ entry:
   br i1 %cond, label %then, label %else
 
 then:
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %data)
+  call void @llvm.lifetime.start.p0(ptr nonnull %data)
   call void @consume(ptr %data)
   %suspend.value = call i8 @llvm.coro.suspend(token none, i1 false)
   switch i8 %suspend.value, label %coro.ret [i8 0, label %resume
                                              i8 1, label %cleanup1]
 
 resume:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %data)
+  call void @llvm.lifetime.end.p0(ptr nonnull %data)
   br label %cleanup1
 
 cleanup1:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %data)
+  call void @llvm.lifetime.end.p0(ptr nonnull %data)
   br label %cleanup
 
 else:
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %data2)
+  call void @llvm.lifetime.start.p0(ptr nonnull %data2)
   call void @consume(ptr %data2)
   %suspend.value2 = call i8 @llvm.coro.suspend(token none, i1 false)
   switch i8 %suspend.value2, label %coro.ret [i8 0, label %resume2
                                               i8 1, label %cleanup2]
 
 resume2:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %data2)
+  call void @llvm.lifetime.end.p0(ptr nonnull %data2)
   br label %cleanup2
 
 cleanup2:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %data2)
+  call void @llvm.lifetime.end.p0(ptr nonnull %data2)
   br label %cleanup
 
 cleanup:
@@ -72,5 +72,5 @@ declare noalias ptr @malloc(i32)
 declare double @print(double)
 declare void @free(ptr)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-04.ll b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-04.ll
index 27e0c47cb9042..6ff31e566283b 100644
--- a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-04.ll
+++ b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-04.ll
@@ -19,10 +19,10 @@ entry:
   br label %init.ready
 init.ready:
   %1 = call noalias nonnull ptr @llvm.coro.begin(token %0, ptr null)
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %__promise)
+  call void @llvm.lifetime.start.p0(ptr nonnull %__promise)
   br i1 %cond, label %if.then, label %if.else
 if.then:
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call void @consume(ptr nonnull %a)
   %save = call token @llvm.coro.save(ptr null)
   %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
@@ -31,10 +31,10 @@ if.then:
     i8 1, label %cleanup1
   ]
 await.ready:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   br label %cleanup1
 if.else:
-  call void @llvm.lifetime.start.p0(i64 300, ptr nonnull %b)
+  call void @llvm.lifetime.start.p0(ptr nonnull %b)
   call void @consume.2(ptr nonnull %b)
   %save2 = call token @llvm.coro.save(ptr null)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
@@ -43,13 +43,13 @@ if.else:
     i8 1, label %cleanup2
   ]
 await2.ready:
-  call void @llvm.lifetime.end.p0(i64 300, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   br label %cleanup2
 cleanup1:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   br label %cleanup
 cleanup2:
-  call void @llvm.lifetime.end.p0(i64 300, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   br label %cleanup
 cleanup:
   call ptr @llvm.coro.free(token %0, ptr %1)
@@ -70,5 +70,5 @@ declare ptr @llvm.coro.frame() #5
 declare i8 @llvm.coro.suspend(token, i1) #3
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-05.ll b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-05.ll
index 6d93eeaa3211e..c3da8e872dc07 100644
--- a/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-05.ll
+++ b/llvm/test/Transforms/Coroutines/coro-frame-reuse-alloca-05.ll
@@ -19,10 +19,10 @@ entry:
   br label %init.ready
 init.ready:
   %1 = call noalias nonnull ptr @llvm.coro.begin(token %0, ptr null)
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %__promise)
+  call void @llvm.lifetime.start.p0(ptr nonnull %__promise)
   br i1 %cond, label %if.then, label %if.else
 if.then:
-  call void @llvm.lifetime.start.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call void @consume(ptr nonnull %a)
   %save = call token @llvm.coro.save(ptr null)
   %suspend = call i8 @llvm.coro.suspend(token %save, i1 false)
@@ -31,10 +31,10 @@ if.then:
     i8 1, label %cleanup1
   ]
 await.ready:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   br label %cleanup1
 if.else:
-  call void @llvm.lifetime.start.p0(i64 400, ptr nonnull %b)
+  call void @llvm.lifetime.start.p0(ptr nonnull %b)
   call void @consume.2(ptr nonnull %b)
   %save2 = call token @llvm.coro.save(ptr null)
   %suspend2 = call i8 @llvm.coro.suspend(token %save2, i1 false)
@@ -43,13 +43,13 @@ if.else:
     i8 1, label %cleanup2
   ]
 await2.ready:
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   br label %cleanup2
 cleanup1:
-  call void @llvm.lifetime.end.p0(i64 500, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   br label %cleanup
 cleanup2:
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   br label %cleanup
 cleanup:
   call ptr @llvm.coro.free(token %0, ptr %1)
@@ -70,5 +70,5 @@ declare ptr @llvm.coro.frame() #5
 declare i8 @llvm.coro.suspend(token, i1) #3
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll b/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll
index 8d0e7729d4a4f..df2ed7e4bcead 100644
--- a/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll
+++ b/llvm/test/Transforms/Coroutines/coro-lifetime-end.ll
@@ -31,7 +31,7 @@ entry:
   %alloc = call ptr @malloc(i64 16) #3
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
 
-  call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   call void @consume.i8.array(ptr %testval)
 
   %save = call token @llvm.coro.save(ptr null)
@@ -68,7 +68,7 @@ entry:
   %alloc = call ptr @malloc(i64 16) #3
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
 
-  call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   call void @consume.i8.array(ptr %testval)
 
   %save = call token @llvm.coro.save(ptr null)
@@ -81,7 +81,7 @@ await.ready:
   br label %exit
 exit:
   call i1 @llvm.coro.end(ptr null, i1 false, token none)
-  call void @llvm.lifetime.end.p0(i64 100, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
   ret void
 }
 
@@ -107,7 +107,7 @@ entry:
   %alloc = call ptr @malloc(i64 16) #3
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
 
-  call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   call void @consume.i8.array(ptr %testval)
 
   %0 = load i8, ptr @testbool, align 1
@@ -115,7 +115,7 @@ entry:
   br i1 %tobool, label %if.then, label %if.end
 
 if.then:
-  call void @llvm.lifetime.end.p0(i64 100, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
   br label %if.end
 
 if.end:
@@ -138,5 +138,5 @@ declare ptr @llvm.coro.begin(token, ptr writeonly) #3
 declare ptr @llvm.coro.frame() #5
 declare i8 @llvm.coro.suspend(token, i1) #3
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-only-destroy-when-complete.ll b/llvm/test/Transforms/Coroutines/coro-only-destroy-when-complete.ll
index 3f0899a02178e..c3d0fb1a18dd2 100644
--- a/llvm/test/Transforms/Coroutines/coro-only-destroy-when-complete.ll
+++ b/llvm/test/Transforms/Coroutines/coro-only-destroy-when-complete.ll
@@ -19,7 +19,7 @@ coro.alloc:                                       ; preds = %entry
 init.suspend:                                     ; preds = %entry, %coro.alloc
   %3 = phi ptr [ null, %entry ], [ %call, %coro.alloc ]
   %4 = call ptr @llvm.coro.begin(token %0, ptr %3) #12
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %__promise) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %__promise) #3
   store ptr null, ptr %__promise, align 8
   %5 = call token @llvm.coro.save(ptr null)
   %6 = call i8 @llvm.coro.suspend(token %5, i1 false)
@@ -80,7 +80,7 @@ cleanup3:
   br label %cleanup62
 
 cleanup62:                                        ; preds = %await2.suspend, %await.suspend, %init.suspend, %final.suspend
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %__promise) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %__promise) #3
   %18 = call ptr @llvm.coro.free(token %0, ptr %4)
   %.not = icmp eq ptr %18, null
   br i1 %.not, label %coro.ret, label %coro.free
@@ -99,9 +99,9 @@ declare i1 @llvm.coro.alloc(token) #3
 declare dso_local noundef nonnull ptr @_Znwm(i64 noundef) local_unnamed_addr #4
 declare i64 @llvm.coro.size.i64() #5
 declare ptr @llvm.coro.begin(token, ptr writeonly) #3
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #6
+declare void @llvm.lifetime.start.p0(ptr nocapture) #6
 declare token @llvm.coro.save(ptr) #7
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #6
+declare void @llvm.lifetime.end.p0(ptr nocapture) #6
 declare i8 @llvm.coro.suspend(token, i1) #3
 declare ptr @_Z5Innerv() local_unnamed_addr
 declare dso_local void @_ZdlPv(ptr noundef) local_unnamed_addr #8
diff --git a/llvm/test/Transforms/Coroutines/coro-retcon-unreachable.ll b/llvm/test/Transforms/Coroutines/coro-retcon-unreachable.ll
index 8ed03849fb63b..31839aa2a2b5f 100644
--- a/llvm/test/Transforms/Coroutines/coro-retcon-unreachable.ll
+++ b/llvm/test/Transforms/Coroutines/coro-retcon-unreachable.ll
@@ -37,9 +37,9 @@ declare token @llvm.coro.id.retcon.once(i32, i32, ptr, ptr, ptr, ptr) #5
 declare ptr @llvm.coro.begin(token, ptr writeonly) #5
 declare token @llvm.coro.alloca.alloc.i64(i64, i32) #5
 declare ptr @llvm.coro.alloca.get(token) #5
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #6
+declare void @llvm.lifetime.start.p0(ptr nocapture) #6
 declare i1 @llvm.coro.suspend.retcon.i1(...) #5
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #6
+declare void @llvm.lifetime.end.p0(ptr nocapture) #6
 declare void @llvm.coro.alloca.free(token) #5
 declare i1 @llvm.coro.end(ptr, i1, token) #5
 
diff --git a/llvm/test/Transforms/Coroutines/coro-split-02.ll b/llvm/test/Transforms/Coroutines/coro-split-02.ll
index 31e8e8155035c..c487ab1e42ff6 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-02.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-02.ll
@@ -27,10 +27,10 @@ entry:
 await.ready:
   %StrayCoroSave = call token @llvm.coro.save(ptr null)
   %val = load i32, ptr %ref.tmp7
-  call void @llvm.lifetime.start.p0(i64 4, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   %test = load i32, ptr %testval
   call void @print(i32 %test)
-  call void @llvm.lifetime.end.p0(i64 4, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
   call void @print(i32 %val)
   br label %exit
 exit:
@@ -42,10 +42,10 @@ exit:
 ; CHECK:         %testval = alloca i32
 ; CHECK-NOT:     call token @llvm.coro.save(ptr null)
 ; CHECK:         %val = load i32, ptr %ref.tmp7
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %testval)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %testval)
 ; CHECK-NEXT:    %test = load i32, ptr %testval
 ; CHECK-NEXT:    call void @print(i32 %test)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %testval)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %testval)
 ; CHECK-NEXT:    call void @print(i32 %val)
 ; CHECK-NEXT:    ret void
 
@@ -61,5 +61,5 @@ declare i8 @llvm.coro.suspend(token, i1) #3
 declare void @"\01??3@YAXPEAX@Z"(ptr) local_unnamed_addr #10
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-split-dbg.ll b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
index 184d4a564ab72..9a9e3c3f2bf0e 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
@@ -43,7 +43,7 @@ coro_Suspend:                                     ; preds = %for.cond, %if.then,
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
 
 ; Function Attrs: argmemonly nounwind readonly
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr) #5
@@ -54,7 +54,7 @@ declare i64 @llvm.coro.size.i64() #1
 declare ptr @llvm.coro.begin(token, ptr writeonly) #7
 declare token @llvm.coro.save(ptr) #7
 declare i8 @llvm.coro.suspend(token, i1) #7
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #5
 declare void @free(ptr nocapture) local_unnamed_addr #6
 declare i1 @llvm.coro.end(ptr, i1, token) #7
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll
index e2ed205f2c2f4..e661932bf020e 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll
@@ -33,8 +33,8 @@ declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8)
 declare i8 @llvm.coro.suspend(token, i1)
 declare void @llvm.instrprof.increment(ptr, i64, i32, i32)
 declare void @llvm.instrprof.value.profile(ptr, i64, i64, i32, i32)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ; Function Attrs: noinline nounwind presplitcoroutine uwtable
 define ptr @f(i32 %0) presplitcoroutine align 32 {
@@ -56,11 +56,11 @@ define ptr @f(i32 %0) presplitcoroutine align 32 {
 12:                                               ; preds = %8, %1
   %13 = phi ptr [ null, %1 ], [ %11, %8 ]
   %14 = call ptr @llvm.coro.begin(token %6, ptr %13) #28
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %3) #9
+  call void @llvm.lifetime.start.p0(ptr nonnull %3) #9
   store ptr null, ptr %3, align 16
   %15 = getelementptr inbounds {ptr, i64}, ptr %3, i64 0, i32 1
   store i64 0, ptr %15, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %4) #9
+  call void @llvm.lifetime.start.p0(ptr nonnull %4) #9
   store ptr %3, ptr %4, align 8
   %16 = call token @llvm.coro.save(ptr null)
   call void @await_suspend(ptr noundef nonnull align 1 dereferenceable(1) %4, ptr %14) #9
@@ -71,7 +71,7 @@ define ptr @f(i32 %0) presplitcoroutine align 32 {
   ]
 
 18:                                               ; preds = %12
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %4) #9
+  call void @llvm.lifetime.end.p0(ptr nonnull %4) #9
   %19 = icmp slt i32 0, %0
   br i1 %19, label %20, label %36
 
@@ -79,12 +79,12 @@ define ptr @f(i32 %0) presplitcoroutine align 32 {
   br label %22
 
 21:                                               ; preds = %12
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %4) #9
+  call void @llvm.lifetime.end.p0(ptr nonnull %4) #9
   br label %54
 
 22:                                               ; preds = %20, %31
   %23 = phi i32 [ 0, %20 ], [ %32, %31 ]
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %5) #9
+  call void @llvm.lifetime.start.p0(ptr nonnull %5) #9
   %24 = call ptr @other_coro()
   store ptr %3, ptr %5, align 8
   %25 = getelementptr inbounds { ptr, ptr }, ptr %5, i64 0, i32 1
@@ -98,13 +98,13 @@ define ptr @f(i32 %0) presplitcoroutine align 32 {
   ]
 
 31:                                               ; preds = %22
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %5) #9
+  call void @llvm.lifetime.end.p0(ptr nonnull %5) #9
   %32 = add nuw nsw i32 %23, 1
   %33 = icmp slt i32 %32, %0
   br i1 %33, label %22, label %35, !llvm.loop !0
 
 34:                                               ; preds = %22
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %5) #9
+  call void @llvm.lifetime.end.p0(ptr nonnull %5) #9
   br label %54
 
 35:                                               ; preds = %31
@@ -142,11 +142,11 @@ define ptr @f(i32 %0) presplitcoroutine align 32 {
   br label %54
 
 53:                                               ; preds = %47
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %2) #9
+  call void @llvm.lifetime.start.p0(ptr nonnull %2) #9
   unreachable
 
 54:                                               ; preds = %52, %34, %21
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %3) #9
+  call void @llvm.lifetime.end.p0(ptr nonnull %3) #9
   %55 = call ptr @llvm.coro.free(token %6, ptr %14)
   %56 = icmp eq ptr %55, null
   br i1 %56, label %61, label %57
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll
index 7c1a13fd83cec..b2561751e6377 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail5.ll
@@ -9,7 +9,7 @@ entry:
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %alloc.var = alloca i8
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.start.p0(ptr %alloc.var)
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
 
   %save = call token @llvm.coro.save(ptr null)
@@ -29,7 +29,7 @@ await.suspend:
   ]
 await.ready:
   call void @consume(ptr %alloc.var)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.end.p0(ptr %alloc.var)
   br label %exit
 exit:
   call i1 @llvm.coro.end(ptr null, i1 false, token none)
@@ -53,8 +53,8 @@ declare i1 @llvm.coro.end(ptr, i1, token) #2
 declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1
 declare ptr @malloc(i64)
 declare void @consume(ptr)
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll
index e05169a729168..99174ff283120 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail6.ll
@@ -13,7 +13,7 @@ entry:
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %alloc.var = alloca i64
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.start.p0(ptr %alloc.var)
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
 
   %save = call token @llvm.coro.save(ptr null)
@@ -33,7 +33,7 @@ await.suspend:
   ]
 await.ready:
   call void @consume(ptr %alloc.var)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.end.p0(ptr %alloc.var)
   br label %exit
 exit:
   call i1 @llvm.coro.end(ptr null, i1 false, token none)
@@ -51,7 +51,7 @@ entry:
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %alloc.var = alloca i64
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.start.p0(ptr %alloc.var)
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
 
   %save = call token @llvm.coro.save(ptr null)
@@ -71,7 +71,7 @@ await.suspend:
   ]
 await.ready:
   call void @consume(ptr %alloc.var)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.end.p0(ptr %alloc.var)
   br label %exit
 
 cleanup:
@@ -106,8 +106,8 @@ declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1
 declare ptr @malloc(i64)
 declare void @delete(ptr nonnull) #2
 declare void @consume(ptr)
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll b/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll
index 8ceb0dda94f6a..91f8543dffe93 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-musttail7.ll
@@ -11,7 +11,7 @@ entry:
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %alloc.var = alloca i64
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.start.p0(ptr %alloc.var)
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
 
   %save = call token @llvm.coro.save(ptr null)
@@ -36,7 +36,7 @@ await.suspend:
   ]
 await.ready:
   call void @consume(ptr %alloc.var)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.end.p0(ptr %alloc.var)
   br label %exit
 exit:
   %result = phi i64 [0, %entry], [0, %entry], [%foo, %await.suspend], [%foo, %await.suspend], [%foo, %await.ready]
@@ -57,7 +57,7 @@ entry:
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %alloc.var = alloca i64
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.start.p0(ptr %alloc.var)
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
 
   %save = call token @llvm.coro.save(ptr null)
@@ -77,7 +77,7 @@ await.suspend:
   ]
 await.ready:
   call void @consume(ptr %alloc.var)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %alloc.var)
+  call void @llvm.lifetime.end.p0(ptr %alloc.var)
   br label %exit
 
 cleanup:
@@ -114,8 +114,8 @@ declare ptr @llvm.coro.subfn.addr(ptr nocapture readonly, i8) #1
 declare ptr @malloc(i64)
 declare void @delete(ptr nonnull) #2
 declare void @consume(ptr)
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare ptr @await_suspend_function(ptr %awaiter, ptr %hdl)
 
 attributes #0 = { presplitcoroutine }
diff --git a/llvm/test/Transforms/Coroutines/coro-split-no-lifetime.ll b/llvm/test/Transforms/Coroutines/coro-split-no-lifetime.ll
index 157807dc05363..12d65647c8b01 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-no-lifetime.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-no-lifetime.ll
@@ -12,11 +12,11 @@ entry:
   br i1 %n, label %flag_true, label %flag_false
 
 flag_true:
-  call void @llvm.lifetime.start.p0(i64 8, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
   br label %merge
 
 flag_false:
-  call void @llvm.lifetime.start.p0(i64 8, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
   br label %merge
 
 merge:
@@ -27,8 +27,8 @@ merge:
                                   i8 1, label %cleanup]
 resume:
   call void @print(ptr %phi)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %x)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %y)
   br label %cleanup
 
 cleanup:
@@ -54,8 +54,8 @@ declare i1 @llvm.coro.alloc(token)
 declare ptr @llvm.coro.begin(token, ptr)
 declare i1 @llvm.coro.end(ptr, i1, token)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @print(ptr)
 declare noalias ptr @malloc(i32)
diff --git a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-01.ll b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-01.ll
index 1d0cf94c1a979..a5a2bcf2ecb81 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-01.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-01.ll
@@ -17,7 +17,7 @@ entry:
   %ref.tmp7 = alloca %"struct.lean_future<int>::Awaiter", align 8
   %testval = alloca i32
   ; lifetime of %testval starts here, but not used until await.ready.
-  call void @llvm.lifetime.start.p0(i64 4, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
@@ -33,7 +33,7 @@ await.ready:
   %val = load i32, ptr %ref.tmp7
   %test = load i32, ptr %testval
   call void @print(i32 %test)
-  call void @llvm.lifetime.end.p0(i64 4, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
   call void @print(i32 %val)
   br label %exit
 exit:
@@ -43,11 +43,11 @@ exit:
 
 ; CHECK-LABEL: @a.resume(
 ; CHECK:         %testval = alloca i32, align 4
-; CHECK:         call void @llvm.lifetime.start.p0(i64 4, ptr %testval)
+; CHECK:         call void @llvm.lifetime.start.p0(ptr %testval)
 ; CHECK-NEXT:    %val = load i32, ptr %ref.tmp7
 ; CHECK-NEXT:    %test = load i32, ptr %testval
 ; CHECK-NEXT:    call void @print(i32 %test)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %testval)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %testval)
 ; CHECK-NEXT:    call void @print(i32 %val)
 ; CHECK-NEXT:    ret void
 
@@ -56,7 +56,7 @@ entry:
   %ref.tmp7 = alloca %"struct.lean_future<int>::Awaiter", align 8
   %testval = alloca i32
   ; lifetime of %testval starts here, but not used until await.ready.
-  call void @llvm.lifetime.start.p0(i64 4, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
@@ -72,7 +72,7 @@ await.ready:
   %val = load i32, ptr %ref.tmp7
   %test = load i32, ptr %testval
   call void @print(i32 %test)
-  call void @llvm.lifetime.end.p0(i64 4, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
   call void @print(i32 %val)
   br label %exit
 exit:
@@ -92,5 +92,5 @@ declare i8 @llvm.coro.suspend(token, i1) #3
 declare void @"\01??3@YAXPEAX@Z"(ptr) local_unnamed_addr #10
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-02.ll b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-02.ll
index 38a2a33efe051..abc91c3b11c6b 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-02.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-02.ll
@@ -15,7 +15,7 @@ entry:
   %ref.tmp7 = alloca %"struct.lean_future<int>::Awaiter", align 8
   %testval = alloca i32
   ; lifetime of %testval starts here, but not used until await.ready.
-  call void @llvm.lifetime.start.p0(i64 4, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
@@ -44,7 +44,7 @@ await.ready:
 after.await:
   %test1 = load i32, ptr %testval
   call void @print(i32 %test1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
   br label %exit
 
 exit:
@@ -54,7 +54,7 @@ exit:
 
 ; CHECK-LABEL: @a.resume(
 ; CHECK:    %[[VAL:testval.+]] = getelementptr inbounds %a.Frame
-; CHECK-NOT:     call void @llvm.lifetime.start.p0(i64 4, ptr %{{.*}})
+; CHECK-NOT:     call void @llvm.lifetime.start.p0(ptr %{{.*}})
 ; CHECK:         %test = load i32, ptr %[[VAL]]
 
 declare token @llvm.coro.id(i32, ptr readnone, ptr nocapture readonly, ptr)
@@ -69,5 +69,5 @@ declare i8 @llvm.coro.suspend(token, i1) #3
 declare void @"\01??3@YAXPEAX@Z"(ptr) local_unnamed_addr #10
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-03.ll b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-03.ll
index de377a6a38b94..efd1adfc54b53 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-03.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-03.ll
@@ -17,7 +17,7 @@ entry:
   %ref.tmp7 = alloca %"struct.lean_future<int>::Awaiter", align 8
   %testval = alloca %i8.array
   ; lifetime of %testval starts here, but not used until await.ready.
-  call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
@@ -32,7 +32,7 @@ await.ready:
   %StrayCoroSave = call token @llvm.coro.save(ptr null)
   %val = load i32, ptr %ref.tmp7
   call void @consume.i8.array(ptr %testval)
-  call void @llvm.lifetime.end.p0(i64 100, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
   call void @print(i32 %val)
   br label %exit
 exit:
@@ -41,10 +41,10 @@ exit:
 }
 ; CHECK-LABEL: @a.gep.resume(
 ; CHECK:         %testval = alloca %i8.array
-; CHECK:         call void @llvm.lifetime.start.p0(i64 100, ptr %testval)
+; CHECK:         call void @llvm.lifetime.start.p0(ptr %testval)
 ; CHECK-NEXT:    %val = load i32, ptr %ref.tmp7
 ; CHECK-NEXT:    call void @consume.i8.array(ptr %testval)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 100, ptr %testval)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %testval)
 ; CHECK-NEXT:    call void @print(i32 %val)
 ; CHECK-NEXT:    ret void
 
@@ -60,5 +60,5 @@ declare i8 @llvm.coro.suspend(token, i1) #3
 declare void @"\01??3@YAXPEAX@Z"(ptr) local_unnamed_addr #10
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-04.ll b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-04.ll
index 821045583092d..af5aa8ade0b65 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-04.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-sink-lifetime-04.ll
@@ -15,7 +15,7 @@ entry:
   %ref.tmp7 = alloca %"struct.lean_future<int>::Awaiter", align 8
   %testval = alloca i8
   ; lifetime of %testval starts here, but not used until await.ready.
-  call void @llvm.lifetime.start.p0(i64 1, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
   %alloc = call ptr @malloc(i64 16) #3
   %vFrame = call noalias nonnull ptr @llvm.coro.begin(token %id, ptr %alloc)
@@ -31,7 +31,7 @@ await.ready:
   %val = load i32, ptr %ref.tmp7
   %test = load i8, ptr %testval
   call void @consume.i8(i8 %test)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %testval)
+  call void @llvm.lifetime.end.p0(ptr %testval)
   call void @print(i32 %val)
   br label %exit
 exit:
@@ -41,11 +41,11 @@ exit:
 
 ; CHECK-LABEL: @a.resume(
 ; CHECK:         %testval = alloca i8, align 1
-; CHECK:         call void @llvm.lifetime.start.p0(i64 1, ptr %testval)
+; CHECK:         call void @llvm.lifetime.start.p0(ptr %testval)
 ; CHECK-NEXT:    %val = load i32, ptr %ref.tmp7
 ; CHECK-NEXT:    %test = load i8, ptr %testval
 ; CHECK-NEXT:    call void @consume.i8(i8 %test)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr %testval)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %testval)
 ; CHECK-NEXT:    call void @print(i32 %val)
 ; CHECK-NEXT:    ret void
 
@@ -62,5 +62,5 @@ declare i8 @llvm.coro.suspend(token, i1) #3
 declare void @"\01??3@YAXPEAX@Z"(ptr) local_unnamed_addr #10
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1, token) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Coroutines/coro-transform-must-elide.ll b/llvm/test/Transforms/Coroutines/coro-transform-must-elide.ll
index d2c4f57478b52..4eec7edad8b0f 100644
--- a/llvm/test/Transforms/Coroutines/coro-transform-must-elide.ll
+++ b/llvm/test/Transforms/Coroutines/coro-transform-must-elide.ll
@@ -61,11 +61,11 @@ entry:
   ret ptr %task
   ; CHECK: %[[TASK:.+]] = alloca %struct.Task, align 8
   ; CHECK-NEXT: %[[FRAME:.+]] = alloca [32 x i8], align 8
-  ; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr %[[TASK]])
+  ; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr %[[TASK]])
   ; CHECK-NEXT: %[[ID:.+]] = call token @llvm.coro.id(i32 0, ptr null, ptr @callee, ptr @callee.resumers)
   ; CHECK-NEXT: %[[HDL:.+]] = call ptr @llvm.coro.begin(token %[[ID]], ptr null)
   ; CHECK-NEXT: store ptr %[[HDL]], ptr %[[TASK]], align 8
-  ; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr %[[TASK]])
+  ; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr %[[TASK]])
   ; CHECK-NEXT: ret ptr %[[TASK]]
 }
 
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/alloca.ll b/llvm/test/Transforms/CorrelatedValuePropagation/alloca.ll
index 9e47bd283e3b5..390e96ecbb9f2 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/alloca.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/alloca.ll
@@ -12,14 +12,14 @@ target triple = "x86_64-unknown-linux-gnu"
 @.str = private unnamed_addr constant [8 x i8] c"a = %l\0A\00", align 1
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @hoo(ptr)
 
 declare i32 @printf(ptr nocapture readonly, ...)
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @goo(i32 %N, ptr %b) {
 entry:
@@ -32,12 +32,12 @@ for.cond:                                         ; preds = %for.body, %entry
   br i1 %cmp, label %for.body, label %for.end
 
 for.body:                                         ; preds = %for.cond
-  call void @llvm.lifetime.start.p0(i64 8, ptr %a.i)
+  call void @llvm.lifetime.start.p0(ptr %a.i)
   call void @hoo(ptr %a.i)
   call void @hoo(ptr %b)
   %tmp1 = load volatile i64, ptr %a.i, align 8
   %call.i = call i32 (ptr, ...) @printf(ptr @.str, i64 %tmp1)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %a.i)
+  call void @llvm.lifetime.end.p0(ptr %a.i)
   %inc = add nsw i32 %i.0, 1
   br label %for.cond
 
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/trunc.ll b/llvm/test/Transforms/CorrelatedValuePropagation/trunc.ll
index 9b6604298840d..42a89ab0dbc03 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/trunc.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/trunc.ll
@@ -106,3 +106,43 @@ define i1 @overdefined_range_negative(i8 %A, i8 %B) {
   %trunc = trunc i8 %xor to i1
   ret i1 %trunc
 }
+
+define i1 @trunc_nuw_infere_false_for_icmp_ne_1(i8 %x)  {
+; CHECK-LABEL: define i1 @trunc_nuw_infere_false_for_icmp_ne_1(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ne i8 [[X]], 1
+; CHECK-NEXT:    br i1 [[ICMP]], label %[[IFTRUE:.*]], label %[[IFFALSE:.*]]
+; CHECK:       [[IFTRUE]]:
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw i8 [[X]] to i1
+; CHECK-NEXT:    ret i1 false
+; CHECK:       [[IFFALSE]]:
+; CHECK-NEXT:    ret i1 true
+;
+  %icmp = icmp ne i8 %x, 1
+  br i1 %icmp, label %iftrue, label %iffalse
+iftrue:
+  %trunc = trunc nuw i8 %x to i1
+  ret i1 %trunc
+iffalse:
+  ret i1 true
+}
+
+define i1 @neg_trunc_do_not_infere_false_for_icmp_ne_1(i8 %x)  {
+; CHECK-LABEL: define i1 @neg_trunc_do_not_infere_false_for_icmp_ne_1(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ne i8 [[X]], 1
+; CHECK-NEXT:    br i1 [[ICMP]], label %[[IFTRUE:.*]], label %[[IFFALSE:.*]]
+; CHECK:       [[IFTRUE]]:
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[X]] to i1
+; CHECK-NEXT:    ret i1 [[TRUNC]]
+; CHECK:       [[IFFALSE]]:
+; CHECK-NEXT:    ret i1 true
+;
+  %icmp = icmp ne i8 %x, 1
+  br i1 %icmp, label %iftrue, label %iffalse
+iftrue:
+  %trunc = trunc i8 %x to i1
+  ret i1 %trunc
+iffalse:
+  ret i1 true
+}
diff --git a/llvm/test/Transforms/DCE/basic.ll b/llvm/test/Transforms/DCE/basic.ll
index 1a3b12e8cfc92..28772f0bac37b 100644
--- a/llvm/test/Transforms/DCE/basic.ll
+++ b/llvm/test/Transforms/DCE/basic.ll
@@ -10,8 +10,8 @@ define void @test() {
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 
 ; CHECK-LABEL: @test_lifetime_alloca
 define i32 @test_lifetime_alloca() {
@@ -21,8 +21,8 @@ define i32 @test_lifetime_alloca() {
 ; CHECK-NOT: llvm.lifetime.start
 ; CHECK-NOT: llvm.lifetime.end
   %i = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %i)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %i)
+  call void @llvm.lifetime.start.p0(ptr %i)
+  call void @llvm.lifetime.end.p0(ptr %i)
   ret i32 0
 }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/batchaa-caching-new-pointers.ll b/llvm/test/Transforms/DeadStoreElimination/batchaa-caching-new-pointers.ll
index ee9bd6912e2ae..4ec69bcfc9be6 100644
--- a/llvm/test/Transforms/DeadStoreElimination/batchaa-caching-new-pointers.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/batchaa-caching-new-pointers.ll
@@ -12,17 +12,17 @@ define ptr @foo(ptr noundef %ptr) {
 ; CHECK-LABEL: define ptr @foo(
 ; CHECK-SAME: ptr noundef [[PTR:%.*]]) {
 ; CHECK-NEXT:    [[STRUCT_ALLOCA:%.*]] = alloca [[STRUCT_TYPE:%.*]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 56, ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6:[0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6:[0-9]+]]
 ; CHECK-NEXT:    [[STRUCT_BYTE_8:%.*]] = getelementptr inbounds i8, ptr [[STRUCT_ALLOCA]], i64 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[STRUCT_BYTE_8]], i64 4
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr noundef nonnull align 4 [[TMP1]], i8 42, i64 4, i1 false)
 ; CHECK-NEXT:    store i32 43, ptr [[STRUCT_BYTE_8]], align 4
 ; CHECK-NEXT:    [[RET:%.*]] = load ptr, ptr [[STRUCT_BYTE_8]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 56, ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
   %struct.alloca = alloca %struct.type, align 8
-  call void @llvm.lifetime.start.p0(i64 56, ptr nonnull %struct.alloca) nounwind
+  call void @llvm.lifetime.start.p0(ptr nonnull %struct.alloca) nounwind
   %struct.byte.8 = getelementptr inbounds i8, ptr %struct.alloca, i64 8
   ; Set %struct.alloca[8, 16) to 42.
   call void @llvm.memset.p0.i64(ptr noundef nonnull align 4 %struct.byte.8, i8 42, i64 8, i1 false)
@@ -33,7 +33,7 @@ define ptr @foo(ptr noundef %ptr) {
   store i32 44, ptr %struct.byte.4, align 4
   ; Return %struct.alloca[8, 16).
   %ret = load ptr, ptr %struct.byte.8
-  call void @llvm.lifetime.end.p0(i64 56, ptr nonnull %struct.alloca) nounwind
+  call void @llvm.lifetime.end.p0(ptr nonnull %struct.alloca) nounwind
   ret ptr %ret
 }
 
@@ -44,7 +44,7 @@ define ptr @foo(ptr noundef %ptr) {
 define ptr @foo_with_removable_malloc() {
 ; CHECK-LABEL: define ptr @foo_with_removable_malloc() {
 ; CHECK-NEXT:    [[STRUCT_ALLOCA:%.*]] = alloca [[STRUCT_TYPE:%.*]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 56, ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
 ; CHECK-NEXT:    [[STRUCT_BYTE_4:%.*]] = getelementptr inbounds i8, ptr [[STRUCT_ALLOCA]], i64 4
 ; CHECK-NEXT:    [[STRUCT_BYTE_8:%.*]] = getelementptr inbounds i8, ptr [[STRUCT_ALLOCA]], i64 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[STRUCT_BYTE_8]], i64 4
@@ -53,11 +53,11 @@ define ptr @foo_with_removable_malloc() {
 ; CHECK-NEXT:    [[RET:%.*]] = load ptr, ptr [[STRUCT_BYTE_8]], align 8
 ; CHECK-NEXT:    call void @readnone(ptr [[STRUCT_BYTE_4]])
 ; CHECK-NEXT:    call void @readnone(ptr [[STRUCT_BYTE_8]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 56, ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
   %struct.alloca = alloca %struct.type, align 8
-  call void @llvm.lifetime.start.p0(i64 56, ptr nonnull %struct.alloca) nounwind
+  call void @llvm.lifetime.start.p0(ptr nonnull %struct.alloca) nounwind
   %struct.byte.4 = getelementptr inbounds i8, ptr %struct.alloca, i64 4
   %struct.byte.8 = getelementptr inbounds i8, ptr %struct.alloca, i64 8
 
@@ -79,7 +79,7 @@ define ptr @foo_with_removable_malloc() {
   %ret = load ptr, ptr %struct.byte.8
   call void @readnone(ptr %struct.byte.4);
   call void @readnone(ptr %struct.byte.8);
-  call void @llvm.lifetime.end.p0(i64 56, ptr nonnull %struct.alloca) nounwind
+  call void @llvm.lifetime.end.p0(ptr nonnull %struct.alloca) nounwind
   ret ptr %ret
 }
 
@@ -87,7 +87,7 @@ define ptr @foo_with_removable_malloc_free() {
 ; CHECK-LABEL: define ptr @foo_with_removable_malloc_free() {
 ; CHECK-NEXT:    [[STRUCT_ALLOCA:%.*]] = alloca [[STRUCT_TYPE:%.*]], align 8
 ; CHECK-NEXT:    [[M1:%.*]] = tail call ptr @malloc(i64 4)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 56, ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
 ; CHECK-NEXT:    [[STRUCT_BYTE_4:%.*]] = getelementptr inbounds i8, ptr [[STRUCT_ALLOCA]], i64 4
 ; CHECK-NEXT:    [[STRUCT_BYTE_8:%.*]] = getelementptr inbounds i8, ptr [[STRUCT_ALLOCA]], i64 8
 ; CHECK-NEXT:    [[M2:%.*]] = tail call ptr @malloc(i64 4)
@@ -99,12 +99,12 @@ define ptr @foo_with_removable_malloc_free() {
 ; CHECK-NEXT:    [[RET:%.*]] = load ptr, ptr [[STRUCT_BYTE_8]], align 8
 ; CHECK-NEXT:    call void @readnone(ptr [[STRUCT_BYTE_4]])
 ; CHECK-NEXT:    call void @readnone(ptr [[STRUCT_BYTE_8]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 56, ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
   %struct.alloca = alloca %struct.type, align 8
   %m1 = tail call ptr @malloc(i64 4)
-  call void @llvm.lifetime.start.p0(i64 56, ptr nonnull %struct.alloca) nounwind
+  call void @llvm.lifetime.start.p0(ptr nonnull %struct.alloca) nounwind
   %struct.byte.4 = getelementptr inbounds i8, ptr %struct.alloca, i64 4
   %struct.byte.8 = getelementptr inbounds i8, ptr %struct.alloca, i64 8
 
@@ -126,14 +126,14 @@ define ptr @foo_with_removable_malloc_free() {
   %ret = load ptr, ptr %struct.byte.8
   call void @readnone(ptr %struct.byte.4);
   call void @readnone(ptr %struct.byte.8);
-  call void @llvm.lifetime.end.p0(i64 56, ptr nonnull %struct.alloca) nounwind
+  call void @llvm.lifetime.end.p0(ptr nonnull %struct.alloca) nounwind
   ret ptr %ret
 }
 
 define ptr @foo_with_malloc_to_calloc() {
 ; CHECK-LABEL: define ptr @foo_with_malloc_to_calloc() {
 ; CHECK-NEXT:    [[STRUCT_ALLOCA:%.*]] = alloca [[STRUCT_TYPE:%.*]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 56, ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
 ; CHECK-NEXT:    [[STRUCT_BYTE_8:%.*]] = getelementptr inbounds i8, ptr [[STRUCT_ALLOCA]], i64 8
 ; CHECK-NEXT:    [[STRUCT_BYTE_4:%.*]] = getelementptr inbounds i8, ptr [[STRUCT_ALLOCA]], i64 4
 ; CHECK-NEXT:    [[CALLOC1:%.*]] = call ptr @calloc(i64 1, i64 4)
@@ -144,13 +144,13 @@ define ptr @foo_with_malloc_to_calloc() {
 ; CHECK-NEXT:    [[RET:%.*]] = load ptr, ptr [[STRUCT_BYTE_8]], align 8
 ; CHECK-NEXT:    call void @readnone(ptr [[STRUCT_BYTE_4]])
 ; CHECK-NEXT:    call void @readnone(ptr [[STRUCT_BYTE_8]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 56, ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[STRUCT_ALLOCA]]) #[[ATTR6]]
 ; CHECK-NEXT:    call void @use(ptr [[CALLOC1]])
 ; CHECK-NEXT:    call void @use(ptr [[CALLOC]])
 ; CHECK-NEXT:    ret ptr [[RET]]
 ;
   %struct.alloca = alloca %struct.type, align 8
-  call void @llvm.lifetime.start.p0(i64 56, ptr nonnull %struct.alloca) nounwind
+  call void @llvm.lifetime.start.p0(ptr nonnull %struct.alloca) nounwind
   %struct.byte.8 = getelementptr inbounds i8, ptr %struct.alloca, i64 8
   %struct.byte.4 = getelementptr inbounds i8, ptr %struct.alloca, i64 4
 
@@ -172,15 +172,15 @@ define ptr @foo_with_malloc_to_calloc() {
   %ret = load ptr, ptr %struct.byte.8
   call void @readnone(ptr %struct.byte.4);
   call void @readnone(ptr %struct.byte.8);
-  call void @llvm.lifetime.end.p0(i64 56, ptr nonnull %struct.alloca) nounwind
+  call void @llvm.lifetime.end.p0(ptr nonnull %struct.alloca) nounwind
   call void @use(ptr %m1)
   call void @use(ptr %m2)
   ret ptr %ret
 }
 
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare noalias ptr @malloc(i64) willreturn allockind("alloc,uninitialized") "alloc-family"="malloc"
 declare void @readnone(ptr) readnone nounwind
diff --git a/llvm/test/Transforms/DeadStoreElimination/captures-before-load.ll b/llvm/test/Transforms/DeadStoreElimination/captures-before-load.ll
index 7d827fa2f6996..56c84c7ed5c77 100644
--- a/llvm/test/Transforms/DeadStoreElimination/captures-before-load.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/captures-before-load.ll
@@ -865,7 +865,7 @@ exit:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #1
 
 declare void @use.i64(i64)
@@ -883,7 +883,7 @@ define i64 @test_a_not_captured_at_all(ptr %ptr, ptr %ptr.2, i1 %c) {
 ; CHECK-NEXT:    call void @use.i64(i64 [[LV_2]])
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    call void @clobber()
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[A]], i8 0, i64 8, i1 false)
 ; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[A]], align 4
@@ -902,7 +902,7 @@ then:
   br label %exit
 
 exit:
-  call void @llvm.lifetime.start.p0(i64 8, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   store i64 99, ptr %a
   call void @clobber()
   call void @llvm.memset.p0.i64(ptr %a, i8 0, i64 8, i1 false)
@@ -1112,7 +1112,7 @@ else:
 
 declare void @capture_and_clobber_multiple(ptr, ptr)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define i64 @earliest_escape_ptrtoint(ptr %p.1) {
 ; CHECK-LABEL: @earliest_escape_ptrtoint(
@@ -1122,7 +1122,7 @@ define i64 @earliest_escape_ptrtoint(ptr %p.1) {
 ; CHECK-NEXT:    [[LV_1:%.*]] = load ptr, ptr [[P_1:%.*]], align 8
 ; CHECK-NEXT:    [[LV_2:%.*]] = load i64, ptr [[LV_1]], align 4
 ; CHECK-NEXT:    store ptr [[A_1]], ptr [[P_1]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[A_2]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_2]])
 ; CHECK-NEXT:    ret i64 [[LV_2]]
 ;
 entry:
@@ -1134,7 +1134,7 @@ entry:
   store ptr %a.1, ptr %p.1, align 8
   %int = ptrtoint ptr %a.2 to i64
   store i64 %int , ptr %a.2, align 8
-  call void @llvm.lifetime.end.p0(i64 8, ptr %a.2)
+  call void @llvm.lifetime.end.p0(ptr %a.2)
   ret i64 %lv.2
 }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/dominate.ll b/llvm/test/Transforms/DeadStoreElimination/dominate.ll
index 262d16e45849c..7e3ddb30ffe64 100644
--- a/llvm/test/Transforms/DeadStoreElimination/dominate.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/dominate.ll
@@ -8,12 +8,12 @@ bb1:
   br label %bb3
 
 bb2:
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %memtmp3.i)
+  call void @llvm.lifetime.end.p0(ptr %memtmp3.i)
   br label %bb3
 
 bb3:
   call void @bar()
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %memtmp3.i)
+  call void @llvm.lifetime.end.p0(ptr %memtmp3.i)
   br label %bb4
 
 bb4:
@@ -21,4 +21,4 @@ bb4:
 
 }
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
diff --git a/llvm/test/Transforms/DeadStoreElimination/libcalls.ll b/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
index 27ad6390e5ddb..8225e143cba99 100644
--- a/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/libcalls.ll
@@ -56,14 +56,14 @@ define void @test3(ptr %src) {
 define void @test_strcat_with_lifetime(ptr %src) {
 ; CHECK-LABEL: @test_strcat_with_lifetime(
 ; CHECK-NEXT:    [[B:%.*]] = alloca [16 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[B]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[B]])
 ; CHECK-NEXT:    ret void
 ;
   %B = alloca [16 x i8]
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %B)
+  call void @llvm.lifetime.start.p0(ptr nonnull %B)
   %call = call ptr @strcat(ptr %B, ptr %src)
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %B)
+  call void @llvm.lifetime.end.p0(ptr nonnull %B)
   ret void
 }
 
@@ -344,61 +344,61 @@ entry:
 define void @dse_strcpy(ptr nocapture readonly %src) {
 ; CHECK-LABEL: @dse_strcpy(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [256 x i8], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 256, ptr nonnull [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 256, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [256 x i8], align 16
-  call void @llvm.lifetime.start.p0(i64 256, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call ptr @strcpy(ptr nonnull %a, ptr nonnull dereferenceable(1) %src)
-  call void @llvm.lifetime.end.p0(i64 256, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret void
 }
 
 define void @dse_strncpy(ptr nocapture readonly %src) {
 ; CHECK-LABEL: @dse_strncpy(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [256 x i8], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 256, ptr nonnull [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 256, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [256 x i8], align 16
-  call void @llvm.lifetime.start.p0(i64 256, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call ptr @strncpy(ptr nonnull %a, ptr nonnull dereferenceable(1) %src, i64 6)
-  call void @llvm.lifetime.end.p0(i64 256, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret void
 }
 
 define void @dse_strcat(ptr nocapture readonly %src) {
 ; CHECK-LABEL: @dse_strcat(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [256 x i8], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 256, ptr nonnull [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 256, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [256 x i8], align 16
-  call void @llvm.lifetime.start.p0(i64 256, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call ptr @strcat(ptr nonnull %a, ptr nonnull dereferenceable(1) %src)
-  call void @llvm.lifetime.end.p0(i64 256, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret void
 }
 
 define void @dse_strncat(ptr nocapture readonly %src) {
 ; CHECK-LABEL: @dse_strncat(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [256 x i8], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 256, ptr nonnull [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 256, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [256 x i8], align 16
-  call void @llvm.lifetime.start.p0(i64 256, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call ptr @strncat(ptr nonnull %a, ptr nonnull dereferenceable(1) %src, i64 6)
-  call void @llvm.lifetime.end.p0(i64 256, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1) nounwind
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/lifetime.ll b/llvm/test/Transforms/DeadStoreElimination/lifetime.ll
index f2a372eafb0b7..3d74c846032fc 100644
--- a/llvm/test/Transforms/DeadStoreElimination/lifetime.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/lifetime.ll
@@ -3,20 +3,20 @@
 
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 declare void @llvm.memset.p0.i8(ptr nocapture, i8, i8, i1) nounwind
 
 define void @test1() {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca i8
 
   store i8 0, ptr %A  ;; Written to by memset
-  call void @llvm.lifetime.end.p0(i64 1, ptr %A)
+  call void @llvm.lifetime.end.p0(ptr %A)
 
   call void @llvm.memset.p0.i8(ptr %A, i8 0, i8 -1, i1 false)
 
@@ -26,14 +26,14 @@ define void @test1() {
 define void @test2(ptr %P) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    [[Q:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[Q]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[Q]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[Q]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[Q]])
 ; CHECK-NEXT:    ret void
 ;
   %Q = alloca i32
-  call void @llvm.lifetime.start.p0(i64 4, ptr %Q)
+  call void @llvm.lifetime.start.p0(ptr %Q)
   store i32 0, ptr %Q  ;; This store is dead.
-  call void @llvm.lifetime.end.p0(i64 4, ptr %Q)
+  call void @llvm.lifetime.end.p0(ptr %Q)
   ret void
 }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/memcpy-lifetimes.ll b/llvm/test/Transforms/DeadStoreElimination/memcpy-lifetimes.ll
index 7dd8e4153e6e8..264e816c24e81 100644
--- a/llvm/test/Transforms/DeadStoreElimination/memcpy-lifetimes.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/memcpy-lifetimes.ll
@@ -15,7 +15,7 @@ define ptr @alloc_tree() {
 ; CHECK-LABEL: @alloc_tree(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[FVAL:%.*]] = alloca [4 x ptr], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[FVAL]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[FVAL]])
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call dereferenceable_or_null(192) ptr @malloc(i64 192)
 ; CHECK-NEXT:    [[CALL3:%.*]] = tail call ptr @alloc(ptr [[CALL]])
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x ptr], ptr [[FVAL]], i64 0, i64 3
@@ -29,12 +29,12 @@ define ptr @alloc_tree() {
 ; CHECK-NEXT:    [[CALL3_3:%.*]] = tail call ptr @alloc(ptr [[CALL]])
 ; CHECK-NEXT:    store ptr [[CALL3_3]], ptr [[FVAL]], align 16
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(32) [[CALL]], ptr nonnull align 16 dereferenceable(32) [[FVAL]], i64 32, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[FVAL]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[FVAL]])
 ; CHECK-NEXT:    ret ptr [[CALL]]
 ;
 entry:
   %fval = alloca [4 x ptr], align 16
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %fval) #7
+  call void @llvm.lifetime.start.p0(ptr nonnull %fval) #7
   %call = tail call dereferenceable_or_null(192) ptr @malloc(i64 192) #8
   %call3 = tail call ptr @alloc(ptr %call)
   %arrayidx = getelementptr inbounds [4 x ptr], ptr %fval, i64 0, i64 3
@@ -48,11 +48,11 @@ entry:
   %call3.3 = tail call ptr @alloc(ptr %call)
   store ptr %call3.3, ptr %fval, align 16
   call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(32) %call, ptr nonnull align 16 dereferenceable(32) %fval, i64 32, i1 false)
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %fval) #7
+  call void @llvm.lifetime.end.p0(ptr nonnull %fval) #7
   ret ptr %call
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 declare noalias ptr @malloc(i64)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
diff --git a/llvm/test/Transforms/DeadStoreElimination/multiblock-loop-carried-dependence.ll b/llvm/test/Transforms/DeadStoreElimination/multiblock-loop-carried-dependence.ll
index f3f5cb1e4c981..112e9f4489905 100644
--- a/llvm/test/Transforms/DeadStoreElimination/multiblock-loop-carried-dependence.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/multiblock-loop-carried-dependence.ll
@@ -103,7 +103,7 @@ define void @test.2() {
 ; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i64 [[IV_2_NEXT]], 100
 ; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_2]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 400, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -136,11 +136,11 @@ loop.2:
   br i1 %c.2, label %loop.2, label %exit
 
 exit:
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %A) #5
+  call void @llvm.lifetime.end.p0(ptr nonnull %A) #5
   ret void
 }
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ; Make sure `store i32 10, ptr %ptr.2` in %cond.store is not removed. The
 ; stored value may be read by `%use = load i32, ptr %ptr.1` in a future
@@ -171,7 +171,7 @@ define void@test.3() {
 ; CHECK-NEXT:    [[DEPTH_1_BE]] = phi i32 [ [[SUB]], [[COND_READ]] ], [ [[INC]], [[COND_STORE]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER]]
 ; CHECK:       cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 48, ptr nonnull [[NODESTACK]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[NODESTACK]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -203,7 +203,7 @@ loop.latch:
   br label %loop.header
 
 cleanup:                                          ; preds = %while.body, %while.end, %entry
-  call void @llvm.lifetime.end.p0(i64 48, ptr nonnull %nodeStack) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %nodeStack) #3
   ret void
 }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/multiblock-malloc-free.ll b/llvm/test/Transforms/DeadStoreElimination/multiblock-malloc-free.ll
index d32d562eedc44..8ecc7939c549c 100644
--- a/llvm/test/Transforms/DeadStoreElimination/multiblock-malloc-free.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/multiblock-malloc-free.ll
@@ -4,8 +4,8 @@
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 declare void @unknown_func()
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
 declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i32, i1) nounwind
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/nounwind-invoke.ll b/llvm/test/Transforms/DeadStoreElimination/nounwind-invoke.ll
index 3712bec25ee94..72932280de3f0 100644
--- a/llvm/test/Transforms/DeadStoreElimination/nounwind-invoke.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/nounwind-invoke.ll
@@ -9,7 +9,7 @@ define void @test_nounwind_invoke() personality ptr @__gxx_personality_v0 {
 ; CHECK-NEXT:    invoke void @foo(ptr [[TMP]])
 ; CHECK-NEXT:    to label [[BB1:%.*]] unwind label [[BB2:%.*]]
 ; CHECK:       bb1:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[TMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[ABCTMP1:%.*]] = landingpad { ptr, i32 }
@@ -26,7 +26,7 @@ bb:
   to label %bb1 unwind label %bb2
 
 bb1:                                              ; preds = %bb
-  call void @llvm.lifetime.end.p0(i64 4, ptr %tmp)
+  call void @llvm.lifetime.end.p0(ptr %tmp)
   ret void
 
 bb2:                                              ; preds = %bb
@@ -36,7 +36,7 @@ bb2:                                              ; preds = %bb
 }
 
 ; Function Attrs: argmemonly nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #0
+declare void @llvm.lifetime.end.p0(ptr nocapture) #0
 ; Function Attrs: argmemonly nounwind willreturn
 declare void @foo(ptr) #1
 declare i32 @__gxx_personality_v0(...)
diff --git a/llvm/test/Transforms/DeadStoreElimination/simple.ll b/llvm/test/Transforms/DeadStoreElimination/simple.ll
index 6c04e15edc374..9d28395a4ccd0 100644
--- a/llvm/test/Transforms/DeadStoreElimination/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/simple.ll
@@ -697,26 +697,26 @@ define void @test39_atomic(ptr %P, ptr %Q, ptr %R) {
 declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1)
 declare void @llvm.memmove.element.unordered.atomic.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i32)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 define void @test40(ptr noalias %Pp, ptr noalias %Q)  {
 ; CHECK-LABEL: @test40(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    [[PC:%.*]] = load ptr, ptr [[PP:%.*]], align 8
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 4 [[A]], ptr align 4 [[Q:%.*]], i64 4, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[PC]], ptr nonnull align 4 [[A]], i64 4, i1 true)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %A = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %A)
+  call void @llvm.lifetime.start.p0(ptr nonnull %A)
   %Pc = load ptr, ptr %Pp, align 8
   call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 4 %A, ptr align 4 %Q, i64 4, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %Pc, ptr nonnull align 4 %A, i64 4, i1 true)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %A)
+  call void @llvm.lifetime.end.p0(ptr nonnull %A)
   ret void
 }
 
diff --git a/llvm/test/Transforms/DeadStoreElimination/trivial-dse-calls.ll b/llvm/test/Transforms/DeadStoreElimination/trivial-dse-calls.ll
index df2feb087e397..0970ed35b2f2f 100644
--- a/llvm/test/Transforms/DeadStoreElimination/trivial-dse-calls.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/trivial-dse-calls.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=dse -S < %s | FileCheck %s
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @unknown()
 declare void @f(ptr)
@@ -23,14 +23,14 @@ define void @test_dead() {
 define void @test_lifetime() {
 ; CHECK-LABEL: @test_lifetime(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @f(ptr writeonly nocapture %a) argmemonly nounwind willreturn
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -39,18 +39,18 @@ define void @test_lifetime() {
 define void @test_lifetime2() {
 ; CHECK-LABEL: @test_lifetime2(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    call void @unknown()
 ; CHECK-NEXT:    call void @unknown()
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @unknown()
   call void @f(ptr writeonly nocapture %a) argmemonly nounwind willreturn
   call void @unknown()
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
diff --git a/llvm/test/Transforms/EarlyCSE/memoryssa.ll b/llvm/test/Transforms/EarlyCSE/memoryssa.ll
index ba4cce406a12e..f7f7ba31aef9b 100644
--- a/llvm/test/Transforms/EarlyCSE/memoryssa.ll
+++ b/llvm/test/Transforms/EarlyCSE/memoryssa.ll
@@ -146,12 +146,12 @@ define void @test_writeback_lifetimes() {
 ; CHECK-NOMEMSSA-LABEL: @test_writeback_lifetimes(
 ; CHECK-NOMEMSSA-NEXT:  entry:
 ; CHECK-NOMEMSSA-NEXT:    [[P:%.*]] = alloca i64, align 8
-; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[P]])
+; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NOMEMSSA-NEXT:    [[Q:%.*]] = getelementptr i32, ptr [[P]], i64 1
 ; CHECK-NOMEMSSA-NEXT:    [[PV:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NOMEMSSA-NEXT:    [[QV:%.*]] = load i32, ptr [[Q]], align 4
-; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[P]])
-; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[P]])
+; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.end.p0(ptr [[P]])
+; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NOMEMSSA-NEXT:    store i32 [[PV]], ptr [[P]], align 4
 ; CHECK-NOMEMSSA-NEXT:    store i32 [[QV]], ptr [[Q]], align 4
 ; CHECK-NOMEMSSA-NEXT:    ret void
@@ -159,24 +159,24 @@ define void @test_writeback_lifetimes() {
 ; CHECK-LABEL: @test_writeback_lifetimes(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NEXT:    [[Q:%.*]] = getelementptr i32, ptr [[P]], i64 1
 ; CHECK-NEXT:    [[PV:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    [[QV:%.*]] = load i32, ptr [[Q]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[P]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NEXT:    store i32 [[PV]], ptr [[P]], align 4
 ; CHECK-NEXT:    store i32 [[QV]], ptr [[Q]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %p = alloca i64
-  call void @llvm.lifetime.start.p0(i64 8, ptr %p)
+  call void @llvm.lifetime.start.p0(ptr %p)
   %q = getelementptr i32, ptr %p, i64 1
   %pv = load i32, ptr %p
   %qv = load i32, ptr %q
-  call void @llvm.lifetime.end.p0(i64 8, ptr %p)
-  call void @llvm.lifetime.start.p0(i64 8, ptr %p)
+  call void @llvm.lifetime.end.p0(ptr %p)
+  call void @llvm.lifetime.start.p0(ptr %p)
   store i32 %pv, ptr %p
   store i32 %qv, ptr %q
   ret void
@@ -188,11 +188,11 @@ define void @test_writeback_lifetimes_multi_arg(ptr %q) {
 ; CHECK-NOMEMSSA-LABEL: @test_writeback_lifetimes_multi_arg(
 ; CHECK-NOMEMSSA-NEXT:  entry:
 ; CHECK-NOMEMSSA-NEXT:    [[P:%.*]] = alloca i64, align 8
-; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[P]])
+; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NOMEMSSA-NEXT:    [[PV:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NOMEMSSA-NEXT:    [[QV:%.*]] = load i32, ptr [[Q:%.*]], align 4
-; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[P]])
-; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[P]])
+; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.end.p0(ptr [[P]])
+; CHECK-NOMEMSSA-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NOMEMSSA-NEXT:    store i32 [[PV]], ptr [[P]], align 4
 ; CHECK-NOMEMSSA-NEXT:    store i32 [[QV]], ptr [[Q]], align 4
 ; CHECK-NOMEMSSA-NEXT:    ret void
@@ -200,25 +200,25 @@ define void @test_writeback_lifetimes_multi_arg(ptr %q) {
 ; CHECK-LABEL: @test_writeback_lifetimes_multi_arg(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NEXT:    [[PV:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    [[QV:%.*]] = load i32, ptr [[Q:%.*]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[P]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NEXT:    store i32 [[PV]], ptr [[P]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %p = alloca i64
-  call void @llvm.lifetime.start.p0(i64 8, ptr %p)
+  call void @llvm.lifetime.start.p0(ptr %p)
   %pv = load i32, ptr %p
   %qv = load i32, ptr %q
-  call void @llvm.lifetime.end.p0(i64 8, ptr %p)
-  call void @llvm.lifetime.start.p0(i64 8, ptr %p)
+  call void @llvm.lifetime.end.p0(ptr %p)
+  call void @llvm.lifetime.start.p0(ptr %p)
   store i32 %pv, ptr %p
   store i32 %qv, ptr %q
   ret void
 }
 
-declare void @llvm.lifetime.end.p0(i64, ptr)
-declare void @llvm.lifetime.start.p0(i64, ptr)
+declare void @llvm.lifetime.end.p0(ptr)
+declare void @llvm.lifetime.start.p0(ptr)
diff --git a/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll
index f7e21cd586e60..736b07276cebf 100644
--- a/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll
+++ b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll
@@ -25,11 +25,11 @@ define void @defn_simple(...) {
 ; OPT-LABEL: define {{[^@]+}}@defn_simple(...) {
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    %va_start = alloca ptr, align 4
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr %va_start)
 ; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
 ; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
 ; OPT-NEXT:    call void @defn_simple.valist(ptr %0)
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr %va_start)
 ; OPT-NEXT:    ret void
 ;
 ; ABI-LABEL: define {{[^@]+}}@defn_simple(ptr %varargs) {
@@ -50,11 +50,11 @@ define private void @defn_private_simple(...) {
 ; OPT-LABEL: define {{[^@]+}}@defn_private_simple(...) {
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    %va_start = alloca ptr, align 4
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr %va_start)
 ; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
 ; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
 ; OPT-NEXT:    call void @defn_private_simple.valist(ptr %0)
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr %va_start)
 ; OPT-NEXT:    ret void
 ;
 ; ABI-LABEL: define {{[^@]+}}@defn_private_simple(ptr %varargs) {
@@ -75,11 +75,11 @@ define internal void @defn_internal_simple(...) {
 ; OPT-LABEL: define {{[^@]+}}@defn_internal_simple(...) {
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    %va_start = alloca ptr, align 4
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr %va_start)
 ; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
 ; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
 ; OPT-NEXT:    call void @defn_internal_simple.valist(ptr %0)
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr %va_start)
 ; OPT-NEXT:    ret void
 ;
 ; ABI-LABEL: define {{[^@]+}}@defn_internal_simple(ptr %varargs) {
@@ -211,11 +211,11 @@ define external void @defn_external_simple(...) {
 ; OPT-LABEL: define {{[^@]+}}@defn_external_simple(...) {
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    %va_start = alloca ptr, align 4
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr %va_start)
 ; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
 ; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
 ; OPT-NEXT:    call void @defn_external_simple.valist(ptr %0)
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr %va_start)
 ; OPT-NEXT:    ret void
 ;
 ; ABI-LABEL: define {{[^@]+}}@defn_external_simple(ptr %varargs) {
diff --git a/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll
index 96cc826aef4ec..e21b72dbc4d2d 100644
--- a/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll
+++ b/llvm/test/Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll
@@ -10,11 +10,11 @@ define i32 @variadic_int_double_get_firstz(...) {
 ; OPT-LABEL: define {{[^@]+}}@variadic_int_double_get_firstz(...) {
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    %va_start = alloca ptr, align 4
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr %va_start)
 ; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
 ; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
 ; OPT-NEXT:    %1 = call i32 @variadic_int_double_get_firstz.valist(ptr %0)
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr %va_start)
 ; OPT-NEXT:    ret i32 %1
 ;
 ; ABI-LABEL: define {{[^@]+}}@variadic_int_double_get_firstz(ptr %varargs) {
@@ -61,11 +61,11 @@ define double @variadic_int_double_get_secondz(...) {
 ; OPT-LABEL: define {{[^@]+}}@variadic_int_double_get_secondz(...) {
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    %va_start = alloca ptr, align 4
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr %va_start)
 ; OPT-NEXT:    call void @llvm.va_start.p0(ptr %va_start)
 ; OPT-NEXT:    %0 = load ptr, ptr %va_start, align 4
 ; OPT-NEXT:    %1 = call double @variadic_int_double_get_secondz.valist(ptr %0)
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr %va_start)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr %va_start)
 ; OPT-NEXT:    ret double %1
 ;
 ; ABI-LABEL: define {{[^@]+}}@variadic_int_double_get_secondz(ptr %varargs) {
@@ -115,13 +115,13 @@ entry:
 ; CHECK-LABEL: @variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %variadic_can_get_firstIidEEbT_T0_.vararg, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store double %y, ptr %1, align 4
 ; CHECK-NEXT:    %call = call i32 @variadic_int_double_get_firstz.valist(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %cmp.i = icmp eq i32 %call, %x
 ; CHECK-NEXT:    ret i1 %cmp.i
 ; CHECK-NEXT:  }
@@ -130,26 +130,26 @@ define zeroext i1 @variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) {
 ; OPT-LABEL: define {{[^@]+}}@variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) {
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    %vararg_buffer = alloca %variadic_can_get_firstIidEEbT_T0_.vararg, align 16
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; OPT-NEXT:    %0 = getelementptr inbounds nuw %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; OPT-NEXT:    store i32 %x, ptr %0, align 4
 ; OPT-NEXT:    %1 = getelementptr inbounds nuw %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2
 ; OPT-NEXT:    store double %y, ptr %1, align 8
 ; OPT-NEXT:    %call = call i32 @variadic_int_double_get_firstz.valist(ptr %vararg_buffer)
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; OPT-NEXT:    %cmp.i = icmp eq i32 %call, %x
 ; OPT-NEXT:    ret i1 %cmp.i
 ;
 ; ABI-LABEL: define {{[^@]+}}@variadic_can_get_firstIidEEbT_T0_(i32 %x, double %y) {
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    %vararg_buffer = alloca %variadic_can_get_firstIidEEbT_T0_.vararg, align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; ABI-NEXT:    %0 = getelementptr inbounds nuw %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; ABI-NEXT:    store i32 %x, ptr %0, align 4
 ; ABI-NEXT:    %1 = getelementptr inbounds nuw %variadic_can_get_firstIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2
 ; ABI-NEXT:    store double %y, ptr %1, align 8
 ; ABI-NEXT:    %call = call i32 @variadic_int_double_get_firstz(ptr %vararg_buffer)
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; ABI-NEXT:    %cmp.i = icmp eq i32 %call, %x
 ; ABI-NEXT:    ret i1 %cmp.i
 ;
@@ -162,13 +162,13 @@ entry:
 ; CHECK-LABEL: @variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %vararg_buffer = alloca %variadic_can_get_secondIidEEbT_T0_.vararg, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %0 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; CHECK-NEXT:    store i32 %x, ptr %0, align 4
 ; CHECK-NEXT:    %1 = getelementptr inbounds %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 1
 ; CHECK-NEXT:    store double %y, ptr %1, align 4
 ; CHECK-NEXT:    %call = call double @variadic_int_double_get_secondz.valist(ptr %vararg_buffer)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr %vararg_buffer)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; CHECK-NEXT:    %cmp.i = fcmp oeq double %call, %y
 ; CHECK-NEXT:    ret i1 %cmp.i
 ; CHECK-NEXT:  }
@@ -177,26 +177,26 @@ define zeroext i1 @variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) {
 ; OPT-LABEL: define {{[^@]+}}@variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) {
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    %vararg_buffer = alloca %variadic_can_get_secondIidEEbT_T0_.vararg, align 16
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; OPT-NEXT:    %0 = getelementptr inbounds nuw %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; OPT-NEXT:    store i32 %x, ptr %0, align 4
 ; OPT-NEXT:    %1 = getelementptr inbounds nuw %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2
 ; OPT-NEXT:    store double %y, ptr %1, align 8
 ; OPT-NEXT:    %call = call double @variadic_int_double_get_secondz.valist(ptr %vararg_buffer)
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; OPT-NEXT:    %cmp.i = fcmp oeq double %call, %y
 ; OPT-NEXT:    ret i1 %cmp.i
 ;
 ; ABI-LABEL: define {{[^@]+}}@variadic_can_get_secondIidEEbT_T0_(i32 %x, double %y) {
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    %vararg_buffer = alloca %variadic_can_get_secondIidEEbT_T0_.vararg, align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr %vararg_buffer)
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr %vararg_buffer)
 ; ABI-NEXT:    %0 = getelementptr inbounds nuw %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 0
 ; ABI-NEXT:    store i32 %x, ptr %0, align 4
 ; ABI-NEXT:    %1 = getelementptr inbounds nuw %variadic_can_get_secondIidEEbT_T0_.vararg, ptr %vararg_buffer, i32 0, i32 2
 ; ABI-NEXT:    store double %y, ptr %1, align 8
 ; ABI-NEXT:    %call = call double @variadic_int_double_get_secondz(ptr %vararg_buffer)
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr %vararg_buffer)
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr %vararg_buffer)
 ; ABI-NEXT:    %cmp.i = fcmp oeq double %call, %y
 ; ABI-NEXT:    ret i1 %cmp.i
 ;
diff --git a/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll b/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll
index b661f7f3e495f..0f178c7555c2f 100644
--- a/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll
+++ b/llvm/test/Transforms/ExpandVariadics/indirect-calls.ll
@@ -19,11 +19,11 @@ define hidden void @fptr_single_i32(i32 noundef %x) {
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[FPTR_SINGLE_I32_VARARG:%.*]], align 16
 ; ABI-NEXT:    [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[FPTR_SINGLE_I32_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP1]], align 4
 ; ABI-NEXT:    call void [[TMP0]](ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -45,11 +45,11 @@ define hidden void @fptr_libcS(ptr noundef byval(%struct.libcS) align 8 %x) {
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[FPTR_LIBCS_VARARG:%.*]], align 16
 ; ABI-NEXT:    [[TMP0:%.*]] = load volatile ptr, ptr @vararg_ptr, align 4
 ; ABI-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[INDIRECTALLOCA]], ptr [[X:%.*]], i64 24, i1 false)
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[FPTR_LIBCS_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4
 ; ABI-NEXT:    call void [[TMP0]](ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/ExpandVariadics/intrinsics.ll b/llvm/test/Transforms/ExpandVariadics/intrinsics.ll
index 1782c92295744..52ce80eb4b7b6 100644
--- a/llvm/test/Transforms/ExpandVariadics/intrinsics.ll
+++ b/llvm/test/Transforms/ExpandVariadics/intrinsics.ll
@@ -3,13 +3,13 @@
 ; RUN: opt -mtriple=wasm32-unknown-unknown -S --passes=expand-variadics --expand-variadics-override=lowering < %s | FileCheck %s -check-prefixes=CHECK,ABI
 ; REQUIRES: webassembly-registered-target
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @llvm.va_copy.p0(ptr, ptr)
 
 declare void @valist(ptr noundef)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @llvm.va_start.p0(ptr)
 
@@ -20,31 +20,31 @@ define void @start_once(...) {
 ; OPT-LABEL: @start_once(
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    [[VA_START:%.*]] = alloca ptr, align 4
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VA_START]])
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VA_START]])
 ; OPT-NEXT:    call void @llvm.va_start.p0(ptr [[VA_START]])
 ; OPT-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VA_START]], align 4
 ; OPT-NEXT:    call void @start_once.valist(ptr [[TMP0]])
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VA_START]])
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VA_START]])
 ; OPT-NEXT:    ret void
 ;
 ; ABI-LABEL: @start_once(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[S:%.*]] = alloca ptr, align 4
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[S]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S]])
 ; ABI-NEXT:    store ptr [[VARARGS:%.*]], ptr [[S]], align 4
 ; ABI-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S]], align 4
 ; ABI-NEXT:    call void @valist(ptr noundef [[TMP0]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[S]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S]])
 ; ABI-NEXT:    ret void
 ;
 entry:
   %s = alloca ptr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s)
+  call void @llvm.lifetime.start.p0(ptr nonnull %s)
   call void @llvm.va_start.p0(ptr nonnull %s)
   %0 = load ptr, ptr %s, align 4
   call void @valist(ptr noundef %0)
   call void @llvm.va_end.p0(ptr %s)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s)
+  call void @llvm.lifetime.end.p0(ptr nonnull %s)
   ret void
 }
 
@@ -53,34 +53,34 @@ define void @start_twice(...) {
 ; OPT-LABEL: @start_twice(
 ; OPT-NEXT:  entry:
 ; OPT-NEXT:    [[VA_START:%.*]] = alloca ptr, align 4
-; OPT-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VA_START]])
+; OPT-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VA_START]])
 ; OPT-NEXT:    call void @llvm.va_start.p0(ptr [[VA_START]])
 ; OPT-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[VA_START]], align 4
 ; OPT-NEXT:    call void @start_twice.valist(ptr [[TMP0]])
-; OPT-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VA_START]])
+; OPT-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VA_START]])
 ; OPT-NEXT:    ret void
 ;
 ; ABI-LABEL: @start_twice(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[S0:%.*]] = alloca ptr, align 4
 ; ABI-NEXT:    [[S1:%.*]] = alloca ptr, align 4
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[S0]])
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[S1]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S0]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S1]])
 ; ABI-NEXT:    store ptr [[VARARGS:%.*]], ptr [[S0]], align 4
 ; ABI-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S0]], align 4
 ; ABI-NEXT:    call void @valist(ptr noundef [[TMP0]])
 ; ABI-NEXT:    store ptr [[VARARGS]], ptr [[S1]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[S1]], align 4
 ; ABI-NEXT:    call void @valist(ptr noundef [[TMP1]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[S1]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[S0]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S1]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S0]])
 ; ABI-NEXT:    ret void
 ;
 entry:
   %s0 = alloca ptr, align 4
   %s1 = alloca ptr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s0)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %s1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %s0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %s1)
   call void @llvm.va_start.p0(ptr nonnull %s0)
   %0 = load ptr, ptr %s0, align 4
   call void @valist(ptr noundef %0)
@@ -89,8 +89,8 @@ entry:
   %1 = load ptr, ptr %s1, align 4
   call void @valist(ptr noundef %1)
   call void @llvm.va_end.p0(ptr %s1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %s0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %s1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %s0)
   ret void
 }
 
@@ -100,21 +100,21 @@ define void @copy(ptr noundef %va) {
 ; CHECK-NEXT:    [[VA_ADDR:%.*]] = alloca ptr, align 4
 ; CHECK-NEXT:    [[CP:%.*]] = alloca ptr, align 4
 ; CHECK-NEXT:    store ptr [[VA:%.*]], ptr [[VA_ADDR]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[CP]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[CP]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr [[CP]], ptr [[VA_ADDR]], i32 4, i1 false)
 ; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[CP]], align 4
 ; CHECK-NEXT:    call void @valist(ptr noundef [[TMP0]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[CP]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[CP]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %va.addr = alloca ptr, align 4
   %cp = alloca ptr, align 4
   store ptr %va, ptr %va.addr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %cp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %cp)
   call void @llvm.va_copy.p0(ptr nonnull %cp, ptr nonnull %va.addr)
   %0 = load ptr, ptr %cp, align 4
   call void @valist(ptr noundef %0)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %cp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %cp)
   ret void
 }
diff --git a/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll b/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll
index a9f27f730ce91..83b33b93e2cd1 100644
--- a/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll
+++ b/llvm/test/Transforms/ExpandVariadics/pass-byval-byref.ll
@@ -16,11 +16,11 @@ define void @pass_byval(ptr byval(i32) %b) {
 ; ABI-LABEL: @pass_byval(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_BYVAL_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[TMP0]], ptr [[B:%.*]], i64 4, i1 false)
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -41,13 +41,13 @@ define void @i32_libcS_byval(i32 %x, ptr noundef byval(%struct.libcS) align 8 %y
 ; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[I32_LIBCS_BYVAL_VARARG:%.*]], align 16
 ; ABI-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[INDIRECTALLOCA]], ptr [[Y:%.*]], i64 24, i1 false)
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[I32_LIBCS_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[I32_LIBCS_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
 ; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -66,13 +66,13 @@ define void @libcS_i32_byval(ptr byval(%struct.libcS) align 8 %x, i32 %y) {
 ; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[LIBCS_I32_BYVAL_VARARG:%.*]], align 16
 ; ABI-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[INDIRECTALLOCA]], ptr [[X:%.*]], i64 24, i1 false)
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[LIBCS_I32_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[LIBCS_I32_BYVAL_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
 ; ABI-NEXT:    store i32 [[Y:%.*]], ptr [[TMP1]], align 4
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -90,11 +90,11 @@ define void @pass_byref(ptr byref(i32) %b) {
 ; ABI-LABEL: @pass_byref(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_BYREF_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store ptr [[B:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -113,13 +113,13 @@ define void @i32_libcS_byref(i32 %x, ptr noundef byref(%struct.libcS) align 8 %y
 ; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[I32_LIBCS_BYREF_VARARG:%.*]], align 16
 ; ABI-NEXT:    store ptr [[Y:%.*]], ptr [[INDIRECTALLOCA]], align 4
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[I32_LIBCS_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[I32_LIBCS_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
 ; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -138,13 +138,13 @@ define void @libcS_i32_byref(ptr byref(%struct.libcS) align 8 %x, i32 %y) {
 ; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[LIBCS_I32_BYREF_VARARG:%.*]], align 16
 ; ABI-NEXT:    store ptr [[X:%.*]], ptr [[INDIRECTALLOCA]], align 4
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[LIBCS_I32_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[LIBCS_I32_BYREF_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
 ; ABI-NEXT:    store i32 [[Y:%.*]], ptr [[TMP1]], align 4
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll b/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll
index 67cb2690e51b5..46e19040d86b5 100644
--- a/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll
+++ b/llvm/test/Transforms/ExpandVariadics/pass-indirect.ll
@@ -19,13 +19,13 @@ define void @i32_libcS(i32 %x, %struct.libcS %y) {
 ; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[I32_LIBCS_VARARG:%.*]], align 16
 ; ABI-NEXT:    store [[STRUCT_LIBCS]] [[Y:%.*]], ptr [[INDIRECTALLOCA]], align 8
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[I32_LIBCS_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[I32_LIBCS_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
 ; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP1]], align 4
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -44,13 +44,13 @@ define void @libcS_i32(%struct.libcS %x, i32 %y) {
 ; ABI-NEXT:    [[INDIRECTALLOCA:%.*]] = alloca [[STRUCT_LIBCS:%.*]], align 8
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[LIBCS_I32_VARARG:%.*]], align 16
 ; ABI-NEXT:    store [[STRUCT_LIBCS]] [[X:%.*]], ptr [[INDIRECTALLOCA]], align 8
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[LIBCS_I32_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store ptr [[INDIRECTALLOCA]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[LIBCS_I32_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
 ; ABI-NEXT:    store i32 [[Y:%.*]], ptr [[TMP1]], align 4
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/ExpandVariadics/pass-integers.ll b/llvm/test/Transforms/ExpandVariadics/pass-integers.ll
index 7a0c004fff2f7..cf527246673e8 100644
--- a/llvm/test/Transforms/ExpandVariadics/pass-integers.ll
+++ b/llvm/test/Transforms/ExpandVariadics/pass-integers.ll
@@ -17,9 +17,9 @@ define void @pass_nothing()  {
 ; ABI-LABEL: @pass_nothing(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_NOTHING_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -36,11 +36,11 @@ define void @pass_s1(i8 %x) {
 ; ABI-LABEL: @pass_s1(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S1_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_S1_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i8 [[X:%.*]], ptr [[TMP0]], align 1
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -57,11 +57,11 @@ define void @pass_s2(i16 %x) {
 ; ABI-LABEL: @pass_s2(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S2_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_S2_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i16 [[X:%.*]], ptr [[TMP0]], align 2
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -78,11 +78,11 @@ define void @pass_s3(i32 %x) {
 ; ABI-LABEL: @pass_s3(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S3_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_S3_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -99,11 +99,11 @@ define void @pass_s4(i64 %x) {
 ; ABI-LABEL: @pass_s4(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S4_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_S4_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i64 [[X:%.*]], ptr [[TMP0]], align 8
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -120,11 +120,11 @@ define void @pass_s5(<4 x i32> %x) {
 ; ABI-LABEL: @pass_s5(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_S5_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_S5_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store <4 x i32> [[X:%.*]], ptr [[TMP0]], align 16
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -141,13 +141,13 @@ define void @pass_int_s1(i32 %i, i8 %x) {
 ; ABI-LABEL: @pass_int_s1(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S1_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 5, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_INT_S1_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[PASS_INT_S1_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
 ; ABI-NEXT:    store i8 [[X:%.*]], ptr [[TMP1]], align 1
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 5, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -164,13 +164,13 @@ define void @pass_int_s2(i32 %i, i16 %x) {
 ; ABI-LABEL: @pass_int_s2(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S2_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 6, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_INT_S2_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[PASS_INT_S2_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
 ; ABI-NEXT:    store i16 [[X:%.*]], ptr [[TMP1]], align 2
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 6, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -187,13 +187,13 @@ define void @pass_int_s3(i32 %i, i32 %x) {
 ; ABI-LABEL: @pass_int_s3(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S3_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_INT_S3_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[PASS_INT_S3_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
 ; ABI-NEXT:    store i32 [[X:%.*]], ptr [[TMP1]], align 4
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -210,13 +210,13 @@ define void @pass_int_s4(i32 %i, i64 %x) {
 ; ABI-LABEL: @pass_int_s4(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S4_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_INT_S4_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[PASS_INT_S4_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2
 ; ABI-NEXT:    store i64 [[X:%.*]], ptr [[TMP1]], align 8
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -233,13 +233,13 @@ define void @pass_int_s5(i32 %i, <4 x i32> %x) {
 ; ABI-LABEL: @pass_int_s5(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_INT_S5_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_INT_S5_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[PASS_INT_S5_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2
 ; ABI-NEXT:    store <4 x i32> [[X:%.*]], ptr [[TMP1]], align 16
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -256,7 +256,7 @@ define void @pass_asc(i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32> %x5) {
 ; ABI-LABEL: @pass_asc(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_ASC_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 48, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i8 [[X1:%.*]], ptr [[TMP0]], align 1
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 2
@@ -268,7 +268,7 @@ define void @pass_asc(i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32> %x5) {
 ; ABI-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [[PASS_ASC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 8
 ; ABI-NEXT:    store <4 x i32> [[X5:%.*]], ptr [[TMP4]], align 16
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 48, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -285,7 +285,7 @@ define void @pass_dsc(<4 x i32> %x0, i64 %x1, i32 %x2, i16 %x3, i8 %x4) {
 ; ABI-LABEL: @pass_dsc(
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_DSC_VARARG:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 33, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store <4 x i32> [[X0:%.*]], ptr [[TMP0]], align 16
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
@@ -297,7 +297,7 @@ define void @pass_dsc(<4 x i32> %x0, i64 %x1, i32 %x2, i16 %x3, i8 %x4) {
 ; ABI-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [[PASS_DSC_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 5
 ; ABI-NEXT:    store i8 [[X4:%.*]], ptr [[TMP4]], align 1
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 33, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    ret void
 ;
 entry:
@@ -316,7 +316,7 @@ define void @pass_multiple(i32 %i, i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32>
 ; ABI-NEXT:  entry:
 ; ABI-NEXT:    [[VARARG_BUFFER:%.*]] = alloca [[PASS_MULTIPLE_VARARG:%.*]], align 16
 ; ABI-NEXT:    [[VARARG_BUFFER1:%.*]] = alloca [[PASS_MULTIPLE_VARARG_0:%.*]], align 16
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER]])
 ; ABI-NEXT:    [[TMP0:%.*]] = getelementptr inbounds nuw [[PASS_MULTIPLE_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[I:%.*]], ptr [[TMP0]], align 4
 ; ABI-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[PASS_MULTIPLE_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 1
@@ -324,8 +324,8 @@ define void @pass_multiple(i32 %i, i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32>
 ; ABI-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[PASS_MULTIPLE_VARARG]], ptr [[VARARG_BUFFER]], i32 0, i32 3
 ; ABI-NEXT:    store i64 [[X4:%.*]], ptr [[TMP2]], align 8
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[VARARG_BUFFER]])
-; ABI-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[VARARG_BUFFER1]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER]])
+; ABI-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VARARG_BUFFER1]])
 ; ABI-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 0
 ; ABI-NEXT:    store i32 [[I]], ptr [[TMP3]], align 4
 ; ABI-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 1
@@ -335,7 +335,7 @@ define void @pass_multiple(i32 %i, i8 %x1, i16 %x2, i32 %x3, i64 %x4, <4 x i32>
 ; ABI-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw [[PASS_MULTIPLE_VARARG_0]], ptr [[VARARG_BUFFER1]], i32 0, i32 5
 ; ABI-NEXT:    store <4 x i32> [[X5:%.*]], ptr [[TMP6]], align 16
 ; ABI-NEXT:    call void @sink(ptr [[VARARG_BUFFER1]])
-; ABI-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[VARARG_BUFFER1]])
+; ABI-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VARARG_BUFFER1]])
 ; ABI-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/FunctionAttrs/mismatched-signature-invalidation.ll b/llvm/test/Transforms/FunctionAttrs/mismatched-signature-invalidation.ll
new file mode 100644
index 0000000000000..c42535756e8f1
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/mismatched-signature-invalidation.ll
@@ -0,0 +1,31 @@
+; RUN: opt -disable-output -passes="function(print<memoryssa>),cgscc(function-attrs),function(print<memoryssa>)" < %s 2>&1 | FileCheck %s
+
+@g = external global i16
+
+define i16 @fn() {
+  %v = load i16, ptr @g
+  ret i16 %v
+}
+
+declare void @fn2(i16)
+
+; CHECK-LABEL: MemorySSA for function: test
+; CHECK: 1 = MemoryDef(3)
+; CHECK-NEXT: %call = call i16 @fn(i32 0)
+
+; CHECK-LABEL: MemorySSA for function: test
+; CHECK: MemoryUse(2)
+; CHECK-NEXT: %call = call i16 @fn(i32 0)
+
+define void @test() {
+entry:
+  br label %loop
+
+loop:
+  %call = call i16 @fn(i32 0) ; intentional signature mismatch
+  call void @fn2(i16 %call)
+  br i1 false, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/nocapture.ll b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
index 9d6acc410de75..26b5dc2dc7760 100644
--- a/llvm/test/Transforms/FunctionAttrs/nocapture.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nocapture.ll
@@ -1082,6 +1082,65 @@ define i64 @captures_not_ret_only(ptr %p) {
   ret i64 %int
 }
 
+@gi = global i64 0
+
+;; Unlike ptrtoint, ptrtoaddr only captures the address
+define i64 @captures_ptrtoaddr_stored(ptr %p) {
+; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write, argmem: none, inaccessiblemem: none)
+; FNATTRS-LABEL: define noundef i64 @captures_ptrtoaddr_stored
+; FNATTRS-SAME: (ptr captures(address) [[P:%.*]]) #[[ATTR1]] {
+; FNATTRS-NEXT:    [[INT:%.*]] = ptrtoaddr ptr [[P]] to i64
+; FNATTRS-NEXT:    store i64 [[INT]], ptr @gi, align 8
+; FNATTRS-NEXT:    ret i64 0
+;
+; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(write)
+; ATTRIBUTOR-LABEL: define i64 @captures_ptrtoaddr_stored
+; ATTRIBUTOR-SAME: (ptr nofree writeonly [[P:%.*]]) #[[ATTR1]] {
+; ATTRIBUTOR-NEXT:    [[INT:%.*]] = ptrtoaddr ptr [[P]] to i64
+; ATTRIBUTOR-NEXT:    store i64 [[INT]], ptr @gi, align 8
+; ATTRIBUTOR-NEXT:    ret i64 0
+;
+  %int = ptrtoaddr ptr %p to i64
+  store i64 %int, ptr @gi, align 8
+  ret i64 0
+}
+
+;; Note: ptrtoaddr is a location-independent capture, so we don't get captures(ret: address) here.
+define i64 @captures_ptrtoaddr_ret(ptr %p) {
+; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; FNATTRS-LABEL: define i64 @captures_ptrtoaddr_ret
+; FNATTRS-SAME: (ptr captures(address) [[P:%.*]]) #[[ATTR0]] {
+; FNATTRS-NEXT:    [[INT:%.*]] = ptrtoaddr ptr [[P]] to i64
+; FNATTRS-NEXT:    ret i64 [[INT]]
+;
+; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; ATTRIBUTOR-LABEL: define i64 @captures_ptrtoaddr_ret
+; ATTRIBUTOR-SAME: (ptr nofree readnone [[P:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-NEXT:    [[INT:%.*]] = ptrtoaddr ptr [[P]] to i64
+; ATTRIBUTOR-NEXT:    ret i64 [[INT]]
+;
+  %int = ptrtoaddr ptr %p to i64
+  ret i64 %int
+}
+
+;; Note: ptrtoaddr is a location-independent capture, so we don't get captures(none) here.
+define i64 @captures_ptrtoaddr_ignored(ptr %p) {
+; FNATTRS: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; FNATTRS-LABEL: define noundef i64 @captures_ptrtoaddr_ignored
+; FNATTRS-SAME: (ptr captures(address) [[P:%.*]]) #[[ATTR0]] {
+; FNATTRS-NEXT:    [[INT:%.*]] = ptrtoaddr ptr [[P]] to i64
+; FNATTRS-NEXT:    ret i64 0
+;
+; ATTRIBUTOR: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; ATTRIBUTOR-LABEL: define i64 @captures_ptrtoaddr_ignored
+; ATTRIBUTOR-SAME: (ptr nofree readnone [[P:%.*]]) #[[ATTR0]] {
+; ATTRIBUTOR-NEXT:    [[INT:%.*]] = ptrtoaddr ptr [[P]] to i64
+; ATTRIBUTOR-NEXT:    ret i64 0
+;
+  %int = ptrtoaddr ptr %p to i64
+  ret i64 0
+}
+
 define void @captures_read_provenance(ptr %p) {
 ; FNATTRS-LABEL: define void @captures_read_provenance
 ; FNATTRS-SAME: (ptr captures(address, read_provenance) [[P:%.*]]) {
diff --git a/llvm/test/Transforms/FunctionSpecialization/ssa-copy.ll b/llvm/test/Transforms/FunctionSpecialization/ssa-copy.ll
index aaafe294480b2..11696cb76c8bb 100644
--- a/llvm/test/Transforms/FunctionSpecialization/ssa-copy.ll
+++ b/llvm/test/Transforms/FunctionSpecialization/ssa-copy.ll
@@ -53,17 +53,17 @@ exit4:
 ; PREDINF-NEXT:    br label %[[BLOCK1:.*]]
 ; PREDINF:       [[BLOCK1]]:
 ; PREDINF-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
-; PREDINF:         [[CMP_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[CMP]])
-; PREDINF:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; PREDINF:         [[X_4:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; PREDINF:         [[CMP_0:%.*]] = bitcast i1 [[CMP]] to i1
+; PREDINF:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; PREDINF:         [[X_4:%.*]] = bitcast i32 [[X]] to i32
 ; PREDINF-NEXT:    br i1 [[CMP]], label %[[BLOCK2:.*]], label %[[EXIT1:.*]]
 ; PREDINF:       [[BLOCK2]]:
-; PREDINF:         [[CMP_0_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[CMP_0]])
-; PREDINF:         [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X_0]])
-; PREDINF:         [[X_0_3:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X_0]])
+; PREDINF:         [[CMP_0_1:%.*]] = bitcast i1 [[CMP_0]] to i1
+; PREDINF:         [[X_0_1:%.*]] = bitcast i32 [[X_0]] to i32
+; PREDINF:         [[X_0_3:%.*]] = bitcast i32 [[X_0]] to i32
 ; PREDINF-NEXT:    br i1 [[CMP_0]], label %[[BLOCK3:.*]], label %[[EXIT2:.*]]
 ; PREDINF:       [[BLOCK3]]:
-; PREDINF:         [[X_0_1_2:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X_0_1]])
+; PREDINF:         [[X_0_1_2:%.*]] = bitcast i32 [[X_0_1]] to i32
 ; PREDINF-NEXT:    br i1 [[CMP_0_1]], label %[[EXIT4:.*]], label %[[EXIT3:.*]]
 ; PREDINF:       [[EXIT1]]:
 ; PREDINF-NEXT:    ret i32 [[X_4]]
diff --git a/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll b/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll
index a38d3e50a6121..1e789b0613f4f 100644
--- a/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll
+++ b/llvm/test/Transforms/GVN/PRE/phi-translate-2.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=gvn -S | FileCheck %s
+; RUN: opt < %s -passes=gvn -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -S | FileCheck %s --check-prefixes=CHECK,MSSA
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 @a = common global [100 x i64] zeroinitializer, align 16
@@ -50,32 +51,56 @@ if.end:                                           ; preds = %if.then, %entry
 }
 
 define void @test2(i64 %i) {
-; CHECK-LABEL: @test2(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [100 x i64], ptr @a, i64 0, i64 [[I:%.*]]
-; CHECK-NEXT:    [[T0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [100 x i64], ptr @b, i64 0, i64 [[I]]
-; CHECK-NEXT:    [[T1:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8
-; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i64 [[T1]], [[T0]]
-; CHECK-NEXT:    store i64 [[MUL]], ptr @g1, align 8
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[MUL]], 3
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i64 (...) @goo()
-; CHECK-NEXT:    store i64 [[CALL]], ptr @g2, align 8
-; CHECK-NEXT:    [[T2_PRE:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @a, i64 24), align 8
-; CHECK-NEXT:    [[T3_PRE:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @b, i64 24), align 8
-; CHECK-NEXT:    [[DOTPRE:%.*]] = mul nsw i64 [[T3_PRE]], [[T2_PRE]]
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[MUL5_PRE_PHI:%.*]] = phi i64 [ [[DOTPRE]], [[IF_THEN]] ], [ [[MUL]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[T3:%.*]] = phi i64 [ [[T3_PRE]], [[IF_THEN]] ], [ [[T1]], [[ENTRY]] ]
-; CHECK-NEXT:    [[T2:%.*]] = phi i64 [ [[T2_PRE]], [[IF_THEN]] ], [ [[T0]], [[ENTRY]] ]
-; CHECK-NEXT:    [[I_ADDR_0:%.*]] = phi i64 [ 3, [[IF_THEN]] ], [ [[I]], [[ENTRY]] ]
-; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [100 x i64], ptr @a, i64 0, i64 [[I_ADDR_0]]
-; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [100 x i64], ptr @b, i64 0, i64 [[I_ADDR_0]]
-; CHECK-NEXT:    store i64 [[MUL5_PRE_PHI]], ptr @g3, align 8
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test2(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [100 x i64], ptr @a, i64 0, i64 [[I:%.*]]
+; MDEP-NEXT:    [[T0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; MDEP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [100 x i64], ptr @b, i64 0, i64 [[I]]
+; MDEP-NEXT:    [[T1:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8
+; MDEP-NEXT:    [[MUL:%.*]] = mul nsw i64 [[T1]], [[T0]]
+; MDEP-NEXT:    store i64 [[MUL]], ptr @g1, align 8
+; MDEP-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[MUL]], 3
+; MDEP-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; MDEP:       if.then:
+; MDEP-NEXT:    [[CALL:%.*]] = tail call i64 (...) @goo()
+; MDEP-NEXT:    store i64 [[CALL]], ptr @g2, align 8
+; MDEP-NEXT:    [[T2_PRE:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @a, i64 24), align 8
+; MDEP-NEXT:    [[T3_PRE:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @b, i64 24), align 8
+; MDEP-NEXT:    [[DOTPRE:%.*]] = mul nsw i64 [[T3_PRE]], [[T2_PRE]]
+; MDEP-NEXT:    br label [[IF_END]]
+; MDEP:       if.end:
+; MDEP-NEXT:    [[MUL5_PRE_PHI:%.*]] = phi i64 [ [[DOTPRE]], [[IF_THEN]] ], [ [[MUL]], [[ENTRY:%.*]] ]
+; MDEP-NEXT:    [[T3:%.*]] = phi i64 [ [[T3_PRE]], [[IF_THEN]] ], [ [[T1]], [[ENTRY]] ]
+; MDEP-NEXT:    [[T2:%.*]] = phi i64 [ [[T2_PRE]], [[IF_THEN]] ], [ [[T0]], [[ENTRY]] ]
+; MDEP-NEXT:    [[I_ADDR_0:%.*]] = phi i64 [ 3, [[IF_THEN]] ], [ [[I]], [[ENTRY]] ]
+; MDEP-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [100 x i64], ptr @a, i64 0, i64 [[I_ADDR_0]]
+; MDEP-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [100 x i64], ptr @b, i64 0, i64 [[I_ADDR_0]]
+; MDEP-NEXT:    store i64 [[MUL5_PRE_PHI]], ptr @g3, align 8
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test2(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [100 x i64], ptr @a, i64 0, i64 [[I:%.*]]
+; MSSA-NEXT:    [[T0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
+; MSSA-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [100 x i64], ptr @b, i64 0, i64 [[I]]
+; MSSA-NEXT:    [[T1:%.*]] = load i64, ptr [[ARRAYIDX1]], align 8
+; MSSA-NEXT:    [[MUL:%.*]] = mul nsw i64 [[T1]], [[T0]]
+; MSSA-NEXT:    store i64 [[MUL]], ptr @g1, align 8
+; MSSA-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[MUL]], 3
+; MSSA-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; MSSA:       if.then:
+; MSSA-NEXT:    [[CALL:%.*]] = tail call i64 (...) @goo()
+; MSSA-NEXT:    store i64 [[CALL]], ptr @g2, align 8
+; MSSA-NEXT:    br label [[IF_END]]
+; MSSA:       if.end:
+; MSSA-NEXT:    [[I_ADDR_0:%.*]] = phi i64 [ 3, [[IF_THEN]] ], [ [[I]], [[ENTRY:%.*]] ]
+; MSSA-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [100 x i64], ptr @a, i64 0, i64 [[I_ADDR_0]]
+; MSSA-NEXT:    [[T2:%.*]] = load i64, ptr [[ARRAYIDX3]], align 8
+; MSSA-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [100 x i64], ptr @b, i64 0, i64 [[I_ADDR_0]]
+; MSSA-NEXT:    [[T3:%.*]] = load i64, ptr [[ARRAYIDX4]], align 8
+; MSSA-NEXT:    [[MUL5:%.*]] = mul nsw i64 [[T3]], [[T2]]
+; MSSA-NEXT:    store i64 [[MUL5]], ptr @g3, align 8
+; MSSA-NEXT:    ret void
 ;
 entry:
   %arrayidx = getelementptr inbounds [100 x i64], ptr @a, i64 0, i64 %i
@@ -252,29 +277,50 @@ if.end3:                                          ; preds = %if.then2, %if.else,
 ; available in if.then. Check that we correctly phi-translate to the phi that
 ; the load has been replaced with.
 define void @test6(ptr %ptr, i1 %arg) {
-; CHECK-LABEL: @test6(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ARRAYIDX1_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 1
-; CHECK-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[ARRAYIDX1_PHI_TRANS_INSERT]], align 4
-; CHECK-NEXT:    br label [[WHILE:%.*]]
-; CHECK:       while:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP2:%.*]], [[IF_END:%.*]] ]
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 1, [[ENTRY]] ], [ [[I_NEXT:%.*]], [[IF_END]] ]
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[I]]
-; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[I_NEXT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END]]
-; CHECK:       if.then:
-; CHECK-NEXT:    store i32 [[TMP1]], ptr [[ARRAYIDX1]], align 4
-; CHECK-NEXT:    store i32 [[TMP0]], ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[TMP2]] = phi i32 [ [[TMP0]], [[IF_THEN]] ], [ [[TMP1]], [[WHILE]] ]
-; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[WHILE_END:%.*]], label [[WHILE]]
-; CHECK:       while.end:
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test6(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[ARRAYIDX1_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 1
+; MDEP-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[ARRAYIDX1_PHI_TRANS_INSERT]], align 4
+; MDEP-NEXT:    br label [[WHILE:%.*]]
+; MDEP:       while:
+; MDEP-NEXT:    [[TMP0:%.*]] = phi i32 [ [[DOTPRE]], [[ENTRY:%.*]] ], [ [[TMP2:%.*]], [[IF_END:%.*]] ]
+; MDEP-NEXT:    [[I:%.*]] = phi i64 [ 1, [[ENTRY]] ], [ [[I_NEXT:%.*]], [[IF_END]] ]
+; MDEP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[I]]
+; MDEP-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; MDEP-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[I_NEXT]]
+; MDEP-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; MDEP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], [[TMP1]]
+; MDEP-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END]]
+; MDEP:       if.then:
+; MDEP-NEXT:    store i32 [[TMP1]], ptr [[ARRAYIDX1]], align 4
+; MDEP-NEXT:    store i32 [[TMP0]], ptr [[ARRAYIDX2]], align 4
+; MDEP-NEXT:    br label [[IF_END]]
+; MDEP:       if.end:
+; MDEP-NEXT:    [[TMP2]] = phi i32 [ [[TMP0]], [[IF_THEN]] ], [ [[TMP1]], [[WHILE]] ]
+; MDEP-NEXT:    br i1 [[ARG:%.*]], label [[WHILE_END:%.*]], label [[WHILE]]
+; MDEP:       while.end:
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test6(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    br label [[WHILE:%.*]]
+; MSSA:       while:
+; MSSA-NEXT:    [[I:%.*]] = phi i64 [ 1, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ]
+; MSSA-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[I]]
+; MSSA-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; MSSA-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; MSSA-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[I_NEXT]]
+; MSSA-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; MSSA-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], [[TMP1]]
+; MSSA-NEXT:    br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END]]
+; MSSA:       if.then:
+; MSSA-NEXT:    store i32 [[TMP1]], ptr [[ARRAYIDX1]], align 4
+; MSSA-NEXT:    store i32 [[TMP0]], ptr [[ARRAYIDX2]], align 4
+; MSSA-NEXT:    br label [[IF_END]]
+; MSSA:       if.end:
+; MSSA-NEXT:    br i1 [[ARG:%.*]], label [[WHILE_END:%.*]], label [[WHILE]]
+; MSSA:       while.end:
+; MSSA-NEXT:    ret void
 ;
 entry:
   br label %while
@@ -304,24 +350,40 @@ while.end:
 ; Load from arrayidx2 is partially redundant, check that address translation can
 ; fold sext + trunc across phi node together.
 define i32 @test7(ptr noalias %ptr1, ptr noalias %ptr2, i32 %i, i1 %cond) {
-; CHECK-LABEL: @test7(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[ENTRY_IF_END_CRIT_EDGE:%.*]]
-; CHECK:       entry.if.end_crit_edge:
-; CHECK-NEXT:    [[RES_PRE:%.*]] = load i32, ptr [[PTR1:%.*]], align 4
-; CHECK-NEXT:    br label [[IF_END:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[PTR1]], i32 [[I:%.*]]
-; CHECK-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    store i32 [[TMP]], ptr [[PTR2:%.*]], align 4
-; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[I]] to i64
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[RES_PRE]], [[ENTRY_IF_END_CRIT_EDGE]] ], [ [[TMP]], [[IF_THEN]] ]
-; CHECK-NEXT:    [[IDX:%.*]] = phi i64 [ 0, [[ENTRY_IF_END_CRIT_EDGE]] ], [ [[IDX_EXT]], [[IF_THEN]] ]
-; CHECK-NEXT:    [[IDX_TRUNC:%.*]] = trunc i64 [[IDX]] to i32
-; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[PTR1]], i32 [[IDX_TRUNC]]
-; CHECK-NEXT:    ret i32 [[RES]]
+; MDEP-LABEL: @test7(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[ENTRY_IF_END_CRIT_EDGE:%.*]]
+; MDEP:       entry.if.end_crit_edge:
+; MDEP-NEXT:    [[RES_PRE:%.*]] = load i32, ptr [[PTR1:%.*]], align 4
+; MDEP-NEXT:    br label [[IF_END:%.*]]
+; MDEP:       if.then:
+; MDEP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[PTR1]], i32 [[I:%.*]]
+; MDEP-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; MDEP-NEXT:    store i32 [[TMP]], ptr [[PTR2:%.*]], align 4
+; MDEP-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[I]] to i64
+; MDEP-NEXT:    br label [[IF_END]]
+; MDEP:       if.end:
+; MDEP-NEXT:    [[RES:%.*]] = phi i32 [ [[RES_PRE]], [[ENTRY_IF_END_CRIT_EDGE]] ], [ [[TMP]], [[IF_THEN]] ]
+; MDEP-NEXT:    [[IDX:%.*]] = phi i64 [ 0, [[ENTRY_IF_END_CRIT_EDGE]] ], [ [[IDX_EXT]], [[IF_THEN]] ]
+; MDEP-NEXT:    [[IDX_TRUNC:%.*]] = trunc i64 [[IDX]] to i32
+; MDEP-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[PTR1]], i32 [[IDX_TRUNC]]
+; MDEP-NEXT:    ret i32 [[RES]]
+;
+; MSSA-LABEL: @test7(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; MSSA:       if.then:
+; MSSA-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[PTR1:%.*]], i32 [[I:%.*]]
+; MSSA-NEXT:    [[TMP:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; MSSA-NEXT:    store i32 [[TMP]], ptr [[PTR2:%.*]], align 4
+; MSSA-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[I]] to i64
+; MSSA-NEXT:    br label [[IF_END]]
+; MSSA:       if.end:
+; MSSA-NEXT:    [[IDX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IDX_EXT]], [[IF_THEN]] ]
+; MSSA-NEXT:    [[IDX_TRUNC:%.*]] = trunc i64 [[IDX]] to i32
+; MSSA-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[PTR1]], i32 [[IDX_TRUNC]]
+; MSSA-NEXT:    [[RES:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; MSSA-NEXT:    ret i32 [[RES]]
 ;
 entry:
   br i1 %cond, label %if.then, label %if.end
diff --git a/llvm/test/Transforms/GVN/PRE/phi-translate-add.ll b/llvm/test/Transforms/GVN/PRE/phi-translate-add.ll
index ea43307649572..cb05a8ed384ce 100644
--- a/llvm/test/Transforms/GVN/PRE/phi-translate-add.ll
+++ b/llvm/test/Transforms/GVN/PRE/phi-translate-add.ll
@@ -1,21 +1,35 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=gvn -gvn-add-phi-translation=true  -S < %s | FileCheck %s --check-prefix=ADD-TRANS-ON
-; RUN: opt -passes=gvn -gvn-add-phi-translation=false -S < %s | FileCheck %s --check-prefix=ADD-TRANS-OFF
+; RUN: opt -passes=gvn -gvn-add-phi-translation=true  -S < %s | FileCheck %s --check-prefix=ADD-TRANS-ON --check-prefixes=CHECK,PT-ON-MDEP
+; RUN: opt -passes='gvn<memoryssa>' -gvn-add-phi-translation=true  -S < %s | FileCheck %s --check-prefix=ADD-TRANS-ON --check-prefixes=CHECK,PT-ON-MSSA
+; RUN: opt -passes=gvn -gvn-add-phi-translation=false -S < %s | FileCheck %s --check-prefix=ADD-TRANS-OFF --check-prefixes=CHECK,PT-OFF-MDEP
+; RUN: opt -passes='gvn<memoryssa>' -gvn-add-phi-translation=false -S < %s | FileCheck %s --check-prefix=ADD-TRANS-OFF --check-prefixes=CHECK,PT-OFF-MSSA
 
 ; Test that phi translation is able to hoist a load whose address
 ; depends on an add also being hoisted.
 define double @phi_translation_hoists_add(ptr %a, i64 %idx) {
-; ADD-TRANS-ON-LABEL: @phi_translation_hoists_add(
-; ADD-TRANS-ON-NEXT:  entry:
-; ADD-TRANS-ON-NEXT:    [[ADD_PHI_TRANS_INSERT:%.*]] = add nuw nsw i64 [[IDX:%.*]], 1
-; ADD-TRANS-ON-NEXT:    [[GEP_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[ADD_PHI_TRANS_INSERT]]
-; ADD-TRANS-ON-NEXT:    [[LOAD_PRE:%.*]] = load double, ptr [[GEP_PHI_TRANS_INSERT]], align 8
-; ADD-TRANS-ON-NEXT:    br label [[FOR_BODY:%.*]]
-; ADD-TRANS-ON:       for.body:
-; ADD-TRANS-ON-NEXT:    [[CMP:%.*]] = fcmp ole double [[LOAD_PRE]], 1.000000e+00
-; ADD-TRANS-ON-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[FOR_BODY]]
-; ADD-TRANS-ON:       exit:
-; ADD-TRANS-ON-NEXT:    ret double [[LOAD_PRE]]
+; PT-ON-MDEP-LABEL: @phi_translation_hoists_add(
+; PT-ON-MDEP-NEXT:  entry:
+; PT-ON-MDEP-NEXT:    [[ADD_PHI_TRANS_INSERT:%.*]] = add nuw nsw i64 [[IDX:%.*]], 1
+; PT-ON-MDEP-NEXT:    [[GEP_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[ADD_PHI_TRANS_INSERT]]
+; PT-ON-MDEP-NEXT:    [[LOAD_PRE:%.*]] = load double, ptr [[GEP_PHI_TRANS_INSERT]], align 8
+; PT-ON-MDEP-NEXT:    br label [[FOR_BODY:%.*]]
+; PT-ON-MDEP:       for.body:
+; PT-ON-MDEP-NEXT:    [[CMP:%.*]] = fcmp ole double [[LOAD_PRE]], 1.000000e+00
+; PT-ON-MDEP-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[FOR_BODY]]
+; PT-ON-MDEP:       exit:
+; PT-ON-MDEP-NEXT:    ret double [[LOAD_PRE]]
+;
+; PT-ON-MSSA-LABEL: @phi_translation_hoists_add(
+; PT-ON-MSSA-NEXT:  entry:
+; PT-ON-MSSA-NEXT:    br label [[FOR_BODY:%.*]]
+; PT-ON-MSSA:       for.body:
+; PT-ON-MSSA-NEXT:    [[ADD:%.*]] = add nuw nsw i64 [[IDX:%.*]], 1
+; PT-ON-MSSA-NEXT:    [[GEP:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i64 [[ADD]]
+; PT-ON-MSSA-NEXT:    [[LOAD:%.*]] = load double, ptr [[GEP]], align 8
+; PT-ON-MSSA-NEXT:    [[CMP:%.*]] = fcmp ole double [[LOAD]], 1.000000e+00
+; PT-ON-MSSA-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[FOR_BODY]]
+; PT-ON-MSSA:       exit:
+; PT-ON-MSSA-NEXT:    ret double [[LOAD]]
 ;
 ; ADD-TRANS-OFF-LABEL: @phi_translation_hoists_add(
 ; ADD-TRANS-OFF-NEXT:  entry:
@@ -42,3 +56,8 @@ for.body:                                         ; preds = %for.body, %entry
 exit:
   ret double %load
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; ADD-TRANS-ON: {{.*}}
+; CHECK: {{.*}}
+; PT-OFF-MDEP: {{.*}}
+; PT-OFF-MSSA: {{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/phi-translate.ll b/llvm/test/Transforms/GVN/PRE/phi-translate.ll
index 713f012583403..084c449879b16 100644
--- a/llvm/test/Transforms/GVN/PRE/phi-translate.ll
+++ b/llvm/test/Transforms/GVN/PRE/phi-translate.ll
@@ -1,23 +1,53 @@
-; RUN: opt -passes=gvn -S < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt -passes='gvn<memoryssa>' -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
 target datalayout = "e-p:64:64:64"
 
-; CHECK-LABEL: @foo(
-; CHECK: entry.end_crit_edge:
-; CHECK: %[[INDEX:[a-z0-9.]+]] = sext i32 %x to i64{{$}}
-; CHECK: %[[ADDRESS:[a-z0-9.]+]] = getelementptr [100 x i32], ptr @G, i64 0, i64 %[[INDEX]]{{$}}
-; CHECK:   %n.pre = load i32, ptr %[[ADDRESS]], align 4, !dbg [[N_LOC:![0-9]+]]
-; CHECK: br label %end
-; CHECK: then:
-; CHECK:   store i32 %z
-; CHECK: end:
-; CHECK:   %n = phi i32 [ %n.pre, %entry.end_crit_edge ], [ %z, %then ], !dbg [[N_LOC]]
-; CHECK:   ret i32 %n
 
-; CHECK: [[N_LOC]] = !DILocation(line: 47, column: 1, scope: !{{.*}})
 
 @G = external global [100 x i32]
 define i32 @foo(i32 %x, i32 %z) !dbg !6 {
+; MDEP-LABEL: define i32 @foo(
+; MDEP-SAME: i32 [[X:%.*]], i32 [[Z:%.*]]) !dbg [[DBG5:![0-9]+]] {
+; MDEP-NEXT:  [[ENTRY:.*:]]
+; MDEP-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[Z]], 0, !dbg [[DBG8:![0-9]+]]
+; MDEP-NEXT:    br i1 [[TOBOOL]], label %[[ENTRY_END_CRIT_EDGE:.*]], label %[[THEN:.*]], !dbg [[DBG8]]
+; MDEP:       [[ENTRY_END_CRIT_EDGE]]:
+; MDEP-NEXT:    [[J_PHI_TRANS_INSERT:%.*]] = sext i32 [[X]] to i64
+; MDEP-NEXT:    [[Q_PHI_TRANS_INSERT:%.*]] = getelementptr [100 x i32], ptr @G, i64 0, i64 [[J_PHI_TRANS_INSERT]]
+; MDEP-NEXT:    [[N_PRE:%.*]] = load i32, ptr [[Q_PHI_TRANS_INSERT]], align 4, !dbg [[DBG9:![0-9]+]]
+; MDEP-NEXT:    br label %[[END:.*]], !dbg [[DBG8]]
+; MDEP:       [[THEN]]:
+; MDEP-NEXT:    [[I:%.*]] = sext i32 [[X]] to i64, !dbg [[DBG10:![0-9]+]]
+; MDEP-NEXT:    [[P:%.*]] = getelementptr [100 x i32], ptr @G, i64 0, i64 [[I]], !dbg [[DBG10]]
+; MDEP-NEXT:    store i32 [[Z]], ptr [[P]], align 4, !dbg [[DBG10]]
+; MDEP-NEXT:    br label %[[END]], !dbg [[DBG10]]
+; MDEP:       [[END]]:
+; MDEP-NEXT:    [[J_PRE_PHI:%.*]] = phi i64 [ [[J_PHI_TRANS_INSERT]], %[[ENTRY_END_CRIT_EDGE]] ], [ [[I]], %[[THEN]] ], !dbg [[DBG11:![0-9]+]]
+; MDEP-NEXT:    [[N:%.*]] = phi i32 [ [[N_PRE]], %[[ENTRY_END_CRIT_EDGE]] ], [ [[Z]], %[[THEN]] ], !dbg [[DBG9]]
+; MDEP-NEXT:    [[Q:%.*]] = getelementptr [100 x i32], ptr @G, i64 0, i64 [[J_PRE_PHI]], !dbg [[DBG12:![0-9]+]]
+; MDEP-NEXT:    ret i32 [[N]], !dbg [[DBG9]]
+;
+; MSSA-LABEL: define i32 @foo(
+; MSSA-SAME: i32 [[X:%.*]], i32 [[Z:%.*]]) !dbg [[DBG5:![0-9]+]] {
+; MSSA-NEXT:  [[ENTRY:.*:]]
+; MSSA-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[Z]], 0, !dbg [[DBG8:![0-9]+]]
+; MSSA-NEXT:    br i1 [[TOBOOL]], label %[[ENTRY_END_CRIT_EDGE:.*]], label %[[THEN:.*]], !dbg [[DBG8]]
+; MSSA:       [[ENTRY_END_CRIT_EDGE]]:
+; MSSA-NEXT:    [[DOTPRE:%.*]] = sext i32 [[X]] to i64, !dbg [[DBG9:![0-9]+]]
+; MSSA-NEXT:    br label %[[END:.*]], !dbg [[DBG8]]
+; MSSA:       [[THEN]]:
+; MSSA-NEXT:    [[I:%.*]] = sext i32 [[X]] to i64, !dbg [[DBG10:![0-9]+]]
+; MSSA-NEXT:    [[P:%.*]] = getelementptr [100 x i32], ptr @G, i64 0, i64 [[I]], !dbg [[DBG10]]
+; MSSA-NEXT:    store i32 [[Z]], ptr [[P]], align 4, !dbg [[DBG10]]
+; MSSA-NEXT:    br label %[[END]], !dbg [[DBG10]]
+; MSSA:       [[END]]:
+; MSSA-NEXT:    [[J_PRE_PHI:%.*]] = phi i64 [ [[DOTPRE]], %[[ENTRY_END_CRIT_EDGE]] ], [ [[I]], %[[THEN]] ], !dbg [[DBG9]]
+; MSSA-NEXT:    [[Q:%.*]] = getelementptr [100 x i32], ptr @G, i64 0, i64 [[J_PRE_PHI]], !dbg [[DBG11:![0-9]+]]
+; MSSA-NEXT:    [[N:%.*]] = load i32, ptr [[Q]], align 4, !dbg [[DBG12:![0-9]+]]
+; MSSA-NEXT:    ret i32 [[N]], !dbg [[DBG12]]
+;
 entry:
   %tobool = icmp eq i32 %z, 0, !dbg !7
   br i1 %tobool, label %end, label %then, !dbg !7
@@ -51,6 +81,31 @@ end:
 !10 = !DILocation(line: 46, column: 1, scope: !6)
 !11 = !DILocation(line: 47, column: 1, scope: !6)
 !12 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang",
-                             file: !5,
-                             isOptimized: true, flags: "-O2",
-                             splitDebugFilename: "abc.debug", emissionKind: 2)
+  file: !5,
+  isOptimized: true, flags: "-O2",
+  splitDebugFilename: "abc.debug", emissionKind: 2)
+;.
+; MDEP: [[META3:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C99, file: [[META4:![0-9]+]], producer: "clang", isOptimized: true, flags: "-O2", runtimeVersion: 0, splitDebugFilename: "abc.debug", emissionKind: LineTablesOnly)
+; MDEP: [[META4]] = !DIFile(filename: "{{.*}}a.cc", directory: {{.*}})
+; MDEP: [[DBG5]] = distinct !DISubprogram(name: "foo", scope: [[META4]], file: [[META4]], line: 42, type: [[META6:![0-9]+]], scopeLine: 43, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: [[META3]], retainedNodes: [[META7:![0-9]+]])
+; MDEP: [[META6]] = !DISubroutineType(types: [[META7]])
+; MDEP: [[META7]] = !{}
+; MDEP: [[DBG8]] = !DILocation(line: 43, column: 1, scope: [[DBG5]])
+; MDEP: [[DBG9]] = !DILocation(line: 47, column: 1, scope: [[DBG5]])
+; MDEP: [[DBG10]] = !DILocation(line: 44, column: 1, scope: [[DBG5]])
+; MDEP: [[DBG11]] = !DILocation(line: 45, column: 1, scope: [[DBG5]])
+; MDEP: [[DBG12]] = !DILocation(line: 46, column: 1, scope: [[DBG5]])
+;.
+; MSSA: [[META3:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C99, file: [[META4:![0-9]+]], producer: "clang", isOptimized: true, flags: "-O2", runtimeVersion: 0, splitDebugFilename: "abc.debug", emissionKind: LineTablesOnly)
+; MSSA: [[META4]] = !DIFile(filename: "{{.*}}a.cc", directory: {{.*}})
+; MSSA: [[DBG5]] = distinct !DISubprogram(name: "foo", scope: [[META4]], file: [[META4]], line: 42, type: [[META6:![0-9]+]], scopeLine: 43, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: [[META3]], retainedNodes: [[META7:![0-9]+]])
+; MSSA: [[META6]] = !DISubroutineType(types: [[META7]])
+; MSSA: [[META7]] = !{}
+; MSSA: [[DBG8]] = !DILocation(line: 43, column: 1, scope: [[DBG5]])
+; MSSA: [[DBG9]] = !DILocation(line: 45, column: 1, scope: [[DBG5]])
+; MSSA: [[DBG10]] = !DILocation(line: 44, column: 1, scope: [[DBG5]])
+; MSSA: [[DBG11]] = !DILocation(line: 46, column: 1, scope: [[DBG5]])
+; MSSA: [[DBG12]] = !DILocation(line: 47, column: 1, scope: [[DBG5]])
+;.
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-aliasning-path.ll b/llvm/test/Transforms/GVN/PRE/pre-aliasning-path.ll
index 9ca3e1b53f06c..60611a032ded5 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-aliasning-path.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-aliasning-path.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -enable-load-pre -enable-pre -passes=gvn -S < %s | FileCheck %s
+; RUN: opt -enable-load-pre -enable-pre -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt -enable-load-pre -enable-pre -passes='gvn<memoryssa>' -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
 declare void @side_effect_0() nofree
 
@@ -102,25 +103,45 @@ exit:
 }
 
 define i32 @test_03(ptr %p) {
-; CHECK-LABEL: @test_03(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X_PRE:%.*]] = load i32, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[X_PRE]], 100
-; CHECK-NEXT:    br i1 [[COND]], label [[HOT_PATH:%.*]], label [[COLD_PATH:%.*]]
-; CHECK:       hot_path:
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       cold_path:
-; CHECK-NEXT:    call void @no_side_effect()
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[X_PRE]]
-; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret i32 [[X_PRE]]
+; MDEP-LABEL: @test_03(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[X_PRE:%.*]] = load i32, ptr [[P:%.*]], align 4
+; MDEP-NEXT:    br label [[LOOP:%.*]]
+; MDEP:       loop:
+; MDEP-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; MDEP-NEXT:    [[COND:%.*]] = icmp ult i32 [[X_PRE]], 100
+; MDEP-NEXT:    br i1 [[COND]], label [[HOT_PATH:%.*]], label [[COLD_PATH:%.*]]
+; MDEP:       hot_path:
+; MDEP-NEXT:    br label [[BACKEDGE]]
+; MDEP:       cold_path:
+; MDEP-NEXT:    call void @no_side_effect()
+; MDEP-NEXT:    br label [[BACKEDGE]]
+; MDEP:       backedge:
+; MDEP-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[X_PRE]]
+; MDEP-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], 1000
+; MDEP-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; MDEP:       exit:
+; MDEP-NEXT:    ret i32 [[X_PRE]]
+;
+; MSSA-LABEL: @test_03(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    br label [[LOOP:%.*]]
+; MSSA:       loop:
+; MSSA-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
+; MSSA-NEXT:    [[X:%.*]] = load i32, ptr [[P:%.*]], align 4
+; MSSA-NEXT:    [[COND:%.*]] = icmp ult i32 [[X]], 100
+; MSSA-NEXT:    br i1 [[COND]], label [[HOT_PATH:%.*]], label [[COLD_PATH:%.*]]
+; MSSA:       hot_path:
+; MSSA-NEXT:    br label [[BACKEDGE]]
+; MSSA:       cold_path:
+; MSSA-NEXT:    call void @no_side_effect()
+; MSSA-NEXT:    br label [[BACKEDGE]]
+; MSSA:       backedge:
+; MSSA-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[X]]
+; MSSA-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], 1000
+; MSSA-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[EXIT:%.*]]
+; MSSA:       exit:
+; MSSA-NEXT:    ret i32 [[X]]
 ;
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll b/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll
index f099ddcdbd7f5..9bf64962ecb1f 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-basic-add.ll
@@ -1,33 +1,53 @@
-; RUN: opt < %s -passes=gvn -enable-pre -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=gvn -enable-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -enable-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA
 ; RUN: opt < %s -passes="gvn<pre>" -enable-pre=false -S | FileCheck %s
 
 @H = common global i32 0		; <ptr> [#uses=2]
 @G = common global i32 0		; <ptr> [#uses=1]
 
 define i32 @test() nounwind {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr @H, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (...) @foo() #[[ATTR0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[BB:.*]], label %[[ENTRY_BB1_CRIT_EDGE:.*]]
+; CHECK:       [[ENTRY_BB1_CRIT_EDGE]]:
+; CHECK-NEXT:    [[DOTPRE:%.*]] = add i32 [[TMP0]], 42
+; CHECK-NEXT:    br label %[[BB1:.*]]
+; CHECK:       [[BB]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP0]], 42
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @G, align 4
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[DOTPRE_PHI:%.*]] = phi i32 [ [[DOTPRE]], %[[ENTRY_BB1_CRIT_EDGE]] ], [ [[TMP3]], %[[BB]] ]
+; CHECK-NEXT:    store i32 [[DOTPRE_PHI]], ptr @H, align 4
+; CHECK-NEXT:    ret i32 0
+;
 entry:
-	%0 = load i32, ptr @H, align 4		; <i32> [#uses=2]
-	%1 = call i32 (...) @foo() nounwind		; <i32> [#uses=1]
-	%2 = icmp ne i32 %1, 0		; <i1> [#uses=1]
-	br i1 %2, label %bb, label %bb1
+  %0 = load i32, ptr @H, align 4		; <i32> [#uses=2]
+  %1 = call i32 (...) @foo() nounwind		; <i32> [#uses=1]
+  %2 = icmp ne i32 %1, 0		; <i1> [#uses=1]
+  br i1 %2, label %bb, label %bb1
 
 bb:		; preds = %entry
-	%3 = add i32 %0, 42		; <i32> [#uses=1]
-; CHECK: %.pre = add i32 %0, 42
-	store i32 %3, ptr @G, align 4
-	br label %bb1
+  %3 = add i32 %0, 42		; <i32> [#uses=1]
+  store i32 %3, ptr @G, align 4
+  br label %bb1
 
 bb1:		; preds = %bb, %entry
-	%4 = add i32 %0, 42		; <i32> [#uses=1]
-	store i32 %4, ptr @H, align 4
-	br label %return
+  %4 = add i32 %0, 42		; <i32> [#uses=1]
+  store i32 %4, ptr @H, align 4
+  br label %return
 
-; CHECK: %.pre-phi = phi i32 [ %.pre, %entry.bb1_crit_edge ], [ %3, %bb ]
-; CHECK-NEXT: store i32 %.pre-phi, ptr @H, align 4
-; CHECK-NEXT: ret i32 0
 
 return:		; preds = %bb1
-	ret i32 0
+  ret i32 0
 }
 
 declare i32 @foo(...)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; MDEP: {{.*}}
+; MSSA: {{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-jt-add.ll b/llvm/test/Transforms/GVN/PRE/pre-jt-add.ll
index 95f8f3ff31892..f62d06dbf0f84 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-jt-add.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-jt-add.ll
@@ -1,16 +1,33 @@
-; RUN: opt < %s -passes=gvn,jump-threading -enable-pre -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=gvn,jump-threading -enable-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>',jump-threading -enable-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA
 
 @H = common global i32 0
 @G = common global i32 0
 
 define i32 @test(i1 %cond, i32 %v) nounwind {
-; CHECK-LABEL: @test
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[BB:.*]], label %[[MERGE:.*]]
+; CHECK:       [[BB]]:
+; CHECK-NEXT:    store i32 -1, ptr @G, align 4
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 [[V]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[ADD_2]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[ACTION:.*]], label %[[RETURN:.*]]
+; CHECK:       [[ACTION]]:
+; CHECK-NEXT:    store i32 [[ADD_2]], ptr @H, align 4
+; CHECK-NEXT:    br label %[[RETURN]]
+; CHECK:       [[RETURN]]:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, %[[MERGE]] ], [ 1, %[[ACTION]] ]
+; CHECK-NEXT:    ret i32 [[P]]
+;
 entry:
   br i1 %cond, label %bb, label %bb1
 
 bb:
-; CHECK: store
-; CHECK-NOT: br label %return
   %add.1 = add nuw nsw i32 %v, -1
   store i32 %add.1, ptr @G, align 4
   br label %merge
@@ -24,8 +41,6 @@ merge:
   br i1 %cmp, label %action, label %return
 
 action:
-; CHECK: store
-; CHECK-NEXT: br label %return
   store i32 %add.2, ptr @H, align 4
   br label %return
 
@@ -34,3 +49,6 @@ return:
   ret i32 %p
 }
 
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; MDEP: {{.*}}
+; MSSA: {{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load-dbg.ll b/llvm/test/Transforms/GVN/PRE/pre-load-dbg.ll
index 8c020fd036193..f961f23d6b10e 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-load-dbg.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-load-dbg.ll
@@ -1,4 +1,6 @@
-; RUN: opt < %s -passes=gvn -gvn-max-num-insns=22 -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=gvn -gvn-max-num-insns=22 -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -gvn-max-num-insns=22 -S | FileCheck %s --check-prefixes=CHECK,MSSA
 
 ; Debug information should not impact gvn. The following two functions have same
 ; code except debug information. They should generate same optimized
@@ -11,13 +13,80 @@
 @h = global %struct.a zeroinitializer, align 1
 
 define void @withdbg() {
-; CHECK-LABEL: @withdbg
-; CHECK:         [[PRE_PRE1:%.*]] = load i16, ptr @f, align 1
-; CHECK-NEXT:    [[PRE_PRE2:%.*]] = load ptr, ptr @m, align 1
-; CHECK-NEXT:    br i1 true, label %[[BLOCK1:.*]], label %[[BLOCK2:.*]]
-; CHECK:       [[BLOCK1]]:
-; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[PRE_PRE1]] to i32
-; CHECK-NEXT:    store i32 [[CONV]], ptr [[PRE_PRE2]], align 1
+; MDEP-LABEL: define void @withdbg() {
+; MDEP-NEXT:  [[ENTRY:.*:]]
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_I:%.*]] = alloca i16, align 1
+; MDEP-NEXT:    [[TMP11_PRE:%.*]] = load i16, ptr @f, align 1
+; MDEP-NEXT:    [[TMP12_PRE:%.*]] = load ptr, ptr @m, align 1
+; MDEP-NEXT:    br i1 true, label %[[LOR_END:.*]], label %[[LOR_RHS:.*]]
+; MDEP:       [[LOR_RHS]]:
+; MDEP-NEXT:      #dbg_declare(ptr undef, [[META4:![0-9]+]], !DIExpression(), [[META14:![0-9]+]])
+; MDEP-NEXT:      #dbg_declare(ptr undef, [[META10:![0-9]+]], !DIExpression(), [[META14]])
+; MDEP-NEXT:      #dbg_declare(ptr undef, [[META11:![0-9]+]], !DIExpression(), [[META14]])
+; MDEP-NEXT:      #dbg_declare(ptr undef, [[META12:![0-9]+]], !DIExpression(), [[META14]])
+; MDEP-NEXT:      #dbg_declare(ptr undef, [[META13:![0-9]+]], !DIExpression(), [[META14]])
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_1_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_1_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_2_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_2_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_3_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_3_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_4_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_4_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_5_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_5_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_6_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_6_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_7_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_7_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_8_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_8_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    br label %[[LOR_END]]
+; MDEP:       [[LOR_END]]:
+; MDEP-NEXT:    [[CONV_I_I6:%.*]] = sext i16 [[TMP11_PRE]] to i32
+; MDEP-NEXT:    store i32 [[CONV_I_I6]], ptr [[TMP12_PRE]], align 1
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: define void @withdbg() {
+; MSSA-NEXT:  [[ENTRY:.*:]]
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_I:%.*]] = alloca i16, align 1
+; MSSA-NEXT:    br i1 true, label %[[LOR_END:.*]], label %[[LOR_RHS:.*]]
+; MSSA:       [[LOR_RHS]]:
+; MSSA-NEXT:      #dbg_declare(ptr undef, [[META4:![0-9]+]], !DIExpression(), [[META14:![0-9]+]])
+; MSSA-NEXT:      #dbg_declare(ptr undef, [[META10:![0-9]+]], !DIExpression(), [[META14]])
+; MSSA-NEXT:      #dbg_declare(ptr undef, [[META11:![0-9]+]], !DIExpression(), [[META14]])
+; MSSA-NEXT:      #dbg_declare(ptr undef, [[META12:![0-9]+]], !DIExpression(), [[META14]])
+; MSSA-NEXT:      #dbg_declare(ptr undef, [[META13:![0-9]+]], !DIExpression(), [[META14]])
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_1_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_1_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_2_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_2_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_3_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_3_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_4_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_4_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_5_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_5_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_6_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_6_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_7_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_7_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_8_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_8_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[FVALUE:%.*]] = load i16, ptr @f, align 1
+; MSSA-NEXT:    [[MVALUE:%.*]] = load ptr, ptr @m, align 1
+; MSSA-NEXT:    br label %[[LOR_END]]
+; MSSA:       [[LOR_END]]:
+; MSSA-NEXT:    [[TMP11:%.*]] = load i16, ptr @f, align 1
+; MSSA-NEXT:    [[CONV_I_I6:%.*]] = sext i16 [[TMP11]] to i32
+; MSSA-NEXT:    [[TMP12:%.*]] = load ptr, ptr @m, align 1
+; MSSA-NEXT:    store i32 [[CONV_I_I6]], ptr [[TMP12]], align 1
+; MSSA-NEXT:    ret void
+;
 
 entry:
   %agg.tmp.ensured.sroa.0.i = alloca i16, align 1
@@ -61,13 +130,70 @@ lor.end:                                          ; preds = %lor.rhs, %entry
 }
 
 define void @lessdbg() {
-; CHECK-LABEL: @lessdbg
-; CHECK:         [[PRE_PRE1:%.*]] = load i16, ptr @f, align 1
-; CHECK-NEXT:    [[PRE_PRE2:%.*]] = load ptr, ptr @m, align 1
-; CHECK-NEXT:    br i1 true, label %[[BLOCK1:.*]], label %[[BLOCK2:.*]]
-; CHECK:       [[BLOCK1]]:
-; CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[PRE_PRE1]] to i32
-; CHECK-NEXT:    store i32 [[CONV]], ptr [[PRE_PRE2]], align 1
+; MDEP-LABEL: define void @lessdbg() {
+; MDEP-NEXT:  [[ENTRY:.*:]]
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_I:%.*]] = alloca i16, align 1
+; MDEP-NEXT:    [[TMP11_PRE:%.*]] = load i16, ptr @f, align 1
+; MDEP-NEXT:    [[TMP12_PRE:%.*]] = load ptr, ptr @m, align 1
+; MDEP-NEXT:    br i1 true, label %[[LOR_END:.*]], label %[[LOR_RHS:.*]]
+; MDEP:       [[LOR_RHS]]:
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_1_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_1_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_2_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_2_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_3_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_3_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_4_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_4_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_5_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_5_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_6_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_6_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_7_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_7_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_8_I:%.*]] = load volatile i16, ptr @h, align 1
+; MDEP-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_8_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MDEP-NEXT:    br label %[[LOR_END]]
+; MDEP:       [[LOR_END]]:
+; MDEP-NEXT:    [[CONV_I_I6:%.*]] = sext i16 [[TMP11_PRE]] to i32
+; MDEP-NEXT:    store i32 [[CONV_I_I6]], ptr [[TMP12_PRE]], align 1
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: define void @lessdbg() {
+; MSSA-NEXT:  [[ENTRY:.*:]]
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_I:%.*]] = alloca i16, align 1
+; MSSA-NEXT:    br i1 true, label %[[LOR_END:.*]], label %[[LOR_RHS:.*]]
+; MSSA:       [[LOR_RHS]]:
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_1_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_1_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_2_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_2_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_3_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_3_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_4_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_4_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_5_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_5_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_6_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_6_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_7_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_7_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_8_I:%.*]] = load volatile i16, ptr @h, align 1
+; MSSA-NEXT:    store i16 [[AGG_TMP_ENSURED_SROA_0_0_COPYLOAD_8_I]], ptr [[AGG_TMP_ENSURED_SROA_0_I]], align 1
+; MSSA-NEXT:    [[FVALUE:%.*]] = load i16, ptr @f, align 1
+; MSSA-NEXT:    [[MVALUE:%.*]] = load ptr, ptr @m, align 1
+; MSSA-NEXT:    br label %[[LOR_END]]
+; MSSA:       [[LOR_END]]:
+; MSSA-NEXT:    [[TMP11:%.*]] = load i16, ptr @f, align 1
+; MSSA-NEXT:    [[CONV_I_I6:%.*]] = sext i16 [[TMP11]] to i32
+; MSSA-NEXT:    [[TMP12:%.*]] = load ptr, ptr @m, align 1
+; MSSA-NEXT:    store i32 [[CONV_I_I6]], ptr [[TMP12]], align 1
+; MSSA-NEXT:    ret void
+;
 
 entry:
   %agg.tmp.ensured.sroa.0.i = alloca i16, align 1
@@ -126,3 +252,34 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 !48 = !DILocalVariable(name: "v", scope: !41, file: !1, line: 15, type: !5)
 !49 = !DILocalVariable(name: "d", scope: !41, file: !1, line: 15, type: !5)
 !50 = !DILocalVariable(name: "u", scope: !41, file: !1, line: 16, type: !5)
+;.
+; MDEP: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C11, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+; MDEP: [[META1]] = !DIFile(filename: "{{.*}}bbi-78272.c", directory: {{.*}})
+; MDEP: [[META4]] = !DILocalVariable(name: "t", scope: [[META5:![0-9]+]], file: [[META1]], line: 15, type: [[META8:![0-9]+]])
+; MDEP: [[META5]] = distinct !DISubprogram(name: "x", scope: [[META1]], file: [[META1]], line: 14, type: [[META6:![0-9]+]], scopeLine: 14, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META9:![0-9]+]])
+; MDEP: [[META6]] = !DISubroutineType(types: [[META7:![0-9]+]])
+; MDEP: [[META7]] = !{[[META8]]}
+; MDEP: [[META8]] = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+; MDEP: [[META9]] = !{[[META4]], [[META10]], [[META11]], [[META12]], [[META13]]}
+; MDEP: [[META10]] = !DILocalVariable(name: "c", scope: [[META5]], file: [[META1]], line: 15, type: [[META8]])
+; MDEP: [[META11]] = !DILocalVariable(name: "v", scope: [[META5]], file: [[META1]], line: 15, type: [[META8]])
+; MDEP: [[META12]] = !DILocalVariable(name: "d", scope: [[META5]], file: [[META1]], line: 15, type: [[META8]])
+; MDEP: [[META13]] = !DILocalVariable(name: "u", scope: [[META5]], file: [[META1]], line: 16, type: [[META8]])
+; MDEP: [[META14]] = !DILocation(line: 15, column: 7, scope: [[META5]])
+;.
+; MSSA: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C11, file: [[META1:![0-9]+]], producer: "{{.*}}clang version {{.*}}", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+; MSSA: [[META1]] = !DIFile(filename: "{{.*}}bbi-78272.c", directory: {{.*}})
+; MSSA: [[META4]] = !DILocalVariable(name: "t", scope: [[META5:![0-9]+]], file: [[META1]], line: 15, type: [[META8:![0-9]+]])
+; MSSA: [[META5]] = distinct !DISubprogram(name: "x", scope: [[META1]], file: [[META1]], line: 14, type: [[META6:![0-9]+]], scopeLine: 14, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META9:![0-9]+]])
+; MSSA: [[META6]] = !DISubroutineType(types: [[META7:![0-9]+]])
+; MSSA: [[META7]] = !{[[META8]]}
+; MSSA: [[META8]] = !DIBasicType(name: "int", size: 16, encoding: DW_ATE_signed)
+; MSSA: [[META9]] = !{[[META4]], [[META10]], [[META11]], [[META12]], [[META13]]}
+; MSSA: [[META10]] = !DILocalVariable(name: "c", scope: [[META5]], file: [[META1]], line: 15, type: [[META8]])
+; MSSA: [[META11]] = !DILocalVariable(name: "v", scope: [[META5]], file: [[META1]], line: 15, type: [[META8]])
+; MSSA: [[META12]] = !DILocalVariable(name: "d", scope: [[META5]], file: [[META1]], line: 15, type: [[META8]])
+; MSSA: [[META13]] = !DILocalVariable(name: "u", scope: [[META5]], file: [[META1]], line: 16, type: [[META8]])
+; MSSA: [[META14]] = !DILocation(line: 15, column: 7, scope: [[META5]])
+;.
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll b/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll
index 1ca907df35215..ca1852f49bf31 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-load-guards.ll
@@ -1,4 +1,6 @@
-; RUN: opt < %s -passes=gvn -enable-load-pre -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=gvn -enable-load-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -enable-load-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 declare void @llvm.experimental.guard(i1, ...)
@@ -8,20 +10,33 @@ declare void @llvm.experimental.guard(i1, ...)
 ; the element in this case and deoptimize otherwise. If we hoist the load to a
 ; place above the guard, it will may lead to out-of-bound array access.
 define i32 @test_motivation(ptr %p, ptr %q, i1 %C, i32 %index, i32 %len) {
-; CHECK-LABEL: @test_motivation(
+; CHECK-LABEL: define i32 @test_motivation(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C:%.*]], i32 [[INDEX:%.*]], i32 [[LEN:%.*]]) {
+; CHECK-NEXT:  [[BLOCK1:.*:]]
+; CHECK-NEXT:    [[EL1:%.*]] = getelementptr inbounds i32, ptr [[Q]], i32 [[INDEX]]
+; CHECK-NEXT:    [[EL2:%.*]] = getelementptr inbounds i32, ptr [[P]], i32 [[INDEX]]
+; CHECK-NEXT:    br i1 [[C]], label %[[BLOCK2:.*]], label %[[BLOCK3:.*]]
+; CHECK:       [[BLOCK2]]:
+; CHECK-NEXT:    br label %[[BLOCK4:.*]]
+; CHECK:       [[BLOCK3]]:
+; CHECK-NEXT:    store i32 0, ptr [[EL1]], align 4
+; CHECK-NEXT:    br label %[[BLOCK4]]
+; CHECK:       [[BLOCK4]]:
+; CHECK-NEXT:    [[P2:%.*]] = phi ptr [ [[EL2]], %[[BLOCK3]] ], [ [[EL1]], %[[BLOCK2]] ]
+; CHECK-NEXT:    [[COND1:%.*]] = icmp sge i32 [[INDEX]], 0
+; CHECK-NEXT:    [[COND2:%.*]] = icmp slt i32 [[INDEX]], [[LEN]]
+; CHECK-NEXT:    [[IN_BOUNDS:%.*]] = and i1 [[COND1]], [[COND2]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[IN_BOUNDS]]) [ "deopt"() ]
+; CHECK-NEXT:    [[PRE:%.*]] = load i32, ptr [[P2]], align 4
+; CHECK-NEXT:    ret i32 [[PRE]]
+;
 block1:
   %el1 = getelementptr inbounds i32, ptr %q, i32 %index
   %el2 = getelementptr inbounds i32, ptr %p, i32 %index
-	br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
 
-; CHECK:        block2:
-; CHECK-NEXT:     br
-; CHECK-NOT:      load
-; CHECK-NOT:      sge
-; CHECK-NOT:      slt
-; CHECK-NOT:      and
   br label %block4
 
 block3:
@@ -30,13 +45,6 @@ block3:
 
 block4:
 
-; CHECK:        block4:
-; CHECK:          %cond1 = icmp sge i32 %index, 0
-; CHECK-NEXT:     %cond2 = icmp slt i32 %index, %len
-; CHECK-NEXT:     %in.bounds = and i1 %cond1, %cond2
-; CHECK:          call void (i1, ...) @llvm.experimental.guard(i1 %in.bounds)
-; CHECK-NEXT:     %PRE = load i32, ptr %P2
-; CHECK:          ret i32 %PRE
 
   %P2 = phi ptr [%el2, %block3], [%el1, %block2]
   %cond1 = icmp sge i32 %index, 0
@@ -49,17 +57,28 @@ block4:
 
 ; Guard in load's block that is above the load should prohibit the PRE.
 define i32 @test_guard_01(ptr %p, ptr %q, i1 %C, i1 %G) {
-; CHECK-LABEL: @test_guard_01(
+; CHECK-LABEL: define i32 @test_guard_01(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C:%.*]], i1 [[G:%.*]]) {
+; CHECK-NEXT:  [[BLOCK1:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[BLOCK2:.*]], label %[[BLOCK3:.*]]
+; CHECK:       [[BLOCK2]]:
+; CHECK-NEXT:    br label %[[BLOCK4:.*]]
+; CHECK:       [[BLOCK3]]:
+; CHECK-NEXT:    store i32 0, ptr [[P]], align 4
+; CHECK-NEXT:    br label %[[BLOCK4]]
+; CHECK:       [[BLOCK4]]:
+; CHECK-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], %[[BLOCK3]] ], [ [[Q]], %[[BLOCK2]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[G]]) [ "deopt"() ]
+; CHECK-NEXT:    [[PRE:%.*]] = load i32, ptr [[P2]], align 4
+; CHECK-NEXT:    ret i32 [[PRE]]
+;
 block1:
-	br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
 
-; CHECK:        block2:
-; CHECK-NEXT:     br
-; CHECK-NOT:      load
 
- br label %block4
+  br label %block4
 
 block3:
   store i32 0, ptr %p
@@ -67,10 +86,6 @@ block3:
 
 block4:
 
-; CHECK:        block4:
-; CHECK:          call void (i1, ...) @llvm.experimental.guard(i1 %G)
-; CHECK-NEXT:     load
-; CHECK:          ret i32
 
   %P2 = phi ptr [%p, %block3], [%q, %block2]
   call void (i1, ...) @llvm.experimental.guard(i1 %G) [ "deopt"() ]
@@ -80,16 +95,44 @@ block4:
 
 ; Guard in load's block that is below the load should not prohibit the PRE.
 define i32 @test_guard_02(ptr %p, ptr %q, i1 %C, i1 %G) {
-; CHECK-LABEL: @test_guard_02(
+; MDEP-LABEL: define i32 @test_guard_02(
+; MDEP-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C:%.*]], i1 [[G:%.*]]) {
+; MDEP-NEXT:  [[BLOCK1:.*:]]
+; MDEP-NEXT:    br i1 [[C]], label %[[BLOCK2:.*]], label %[[BLOCK3:.*]]
+; MDEP:       [[BLOCK2]]:
+; MDEP-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[Q]], align 4
+; MDEP-NEXT:    br label %[[BLOCK4:.*]]
+; MDEP:       [[BLOCK3]]:
+; MDEP-NEXT:    store i32 0, ptr [[P]], align 4
+; MDEP-NEXT:    br label %[[BLOCK4]]
+; MDEP:       [[BLOCK4]]:
+; MDEP-NEXT:    [[PRE:%.*]] = phi i32 [ 0, %[[BLOCK3]] ], [ [[PRE_PRE]], %[[BLOCK2]] ]
+; MDEP-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], %[[BLOCK3]] ], [ [[Q]], %[[BLOCK2]] ]
+; MDEP-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[G]]) [ "deopt"() ]
+; MDEP-NEXT:    ret i32 [[PRE]]
+;
+; MSSA-LABEL: define i32 @test_guard_02(
+; MSSA-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C:%.*]], i1 [[G:%.*]]) {
+; MSSA-NEXT:  [[BLOCK1:.*:]]
+; MSSA-NEXT:    br i1 [[C]], label %[[BLOCK2:.*]], label %[[BLOCK3:.*]]
+; MSSA:       [[BLOCK2]]:
+; MSSA-NEXT:    br label %[[BLOCK4:.*]]
+; MSSA:       [[BLOCK3]]:
+; MSSA-NEXT:    store i32 0, ptr [[P]], align 4
+; MSSA-NEXT:    br label %[[BLOCK4]]
+; MSSA:       [[BLOCK4]]:
+; MSSA-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], %[[BLOCK3]] ], [ [[Q]], %[[BLOCK2]] ]
+; MSSA-NEXT:    [[PRE:%.*]] = load i32, ptr [[P2]], align 4
+; MSSA-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[G]]) [ "deopt"() ]
+; MSSA-NEXT:    ret i32 [[PRE]]
+;
 block1:
-	br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
 
-; CHECK:        block2:
-; CHECK-NEXT:     load i32, ptr %q
 
- br label %block4
+  br label %block4
 
 block3:
   store i32 0, ptr %p
@@ -97,12 +140,6 @@ block3:
 
 block4:
 
-; CHECK:        block4:
-; CHECK-NEXT:     phi i32 [
-; CHECK-NEXT:     phi ptr [
-; CHECK-NEXT:     call void (i1, ...) @llvm.experimental.guard(i1 %G)
-; CHECK-NOT:      load
-; CHECK:          ret i32
 
   %P2 = phi ptr [%p, %block3], [%q, %block2]
   %PRE = load i32, ptr %P2
@@ -112,17 +149,28 @@ block4:
 
 ; Guard above the load's block should prevent PRE from hoisting through it.
 define i32 @test_guard_03(ptr %p, ptr %q, i1 %C, i1 %G) {
-; CHECK-LABEL: @test_guard_03(
+; CHECK-LABEL: define i32 @test_guard_03(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C:%.*]], i1 [[G:%.*]]) {
+; CHECK-NEXT:  [[BLOCK1:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[BLOCK2:.*]], label %[[BLOCK3:.*]]
+; CHECK:       [[BLOCK2]]:
+; CHECK-NEXT:    br label %[[BLOCK4:.*]]
+; CHECK:       [[BLOCK3]]:
+; CHECK-NEXT:    store i32 0, ptr [[P]], align 4
+; CHECK-NEXT:    br label %[[BLOCK4]]
+; CHECK:       [[BLOCK4]]:
+; CHECK-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], %[[BLOCK3]] ], [ [[Q]], %[[BLOCK2]] ]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[G]]) [ "deopt"() ]
+; CHECK-NEXT:    [[PRE:%.*]] = load i32, ptr [[P2]], align 4
+; CHECK-NEXT:    ret i32 [[PRE]]
+;
 block1:
-	br i1 %C, label %block2, label %block3
+  br i1 %C, label %block2, label %block3
 
 block2:
 
-; CHECK:        block2:
-; CHECK-NEXT:     br
-; CHECK-NOT:      load
 
- br label %block4
+  br label %block4
 
 block3:
   store i32 0, ptr %p
@@ -130,11 +178,6 @@ block3:
 
 block4:
 
-; CHECK:        block4:
-; CHECK-NEXT:     phi ptr
-; CHECK-NEXT:     call void (i1, ...) @llvm.experimental.guard(i1 %G)
-; CHECK-NEXT:     load
-; CHECK-NEXT:     ret i32
 
   %P2 = phi ptr [%p, %block3], [%q, %block2]
   call void (i1, ...) @llvm.experimental.guard(i1 %G) [ "deopt"() ]
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load-implicit-cf-updates.ll b/llvm/test/Transforms/GVN/PRE/pre-load-implicit-cf-updates.ll
index 0585781e7985f..17fbc0e38ec66 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-load-implicit-cf-updates.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-load-implicit-cf-updates.ll
@@ -1,4 +1,6 @@
-; RUN: opt -S -passes=gvn -enable-load-pre < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=gvn -enable-load-pre < %s | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt -S -passes='gvn<memoryssa>' -enable-load-pre < %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -9,18 +11,28 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 declare i32 @foo(i32 %arg) #0
 
 define hidden void @test_01(i32 %x, i32 %y) {
-
 ; c2 only throws if c1 throws, so it can be safely removed and then PRE can
 ; hoist the load out of loop.
-
-; CHECK-LABEL: @test_01
-; CHECK:       entry:
-; CHECK-NEXT:    %c1 = call i32 @foo(i32 %x)
-; CHECK-NEXT:    %val.pre = load i32, ptr null, align 8
-; CHECK-NEXT:    br label %loop
-; CHECK:       loop:
-; CHECK-NEXT:    %c3 = call i32 @foo(i32 %val.pre)
-; CHECK-NEXT:    br label %loop
+; MDEP-LABEL: define hidden void @test_01(
+; MDEP-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; MDEP-NEXT:  [[ENTRY:.*:]]
+; MDEP-NEXT:    [[C1:%.*]] = call i32 @foo(i32 [[X]])
+; MDEP-NEXT:    [[VAL_PRE:%.*]] = load i32, ptr null, align 8
+; MDEP-NEXT:    br label %[[LOOP:.*]]
+; MDEP:       [[LOOP]]:
+; MDEP-NEXT:    [[C3:%.*]] = call i32 @foo(i32 [[VAL_PRE]])
+; MDEP-NEXT:    br label %[[LOOP]]
+;
+; MSSA-LABEL: define hidden void @test_01(
+; MSSA-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; MSSA-NEXT:  [[ENTRY:.*:]]
+; MSSA-NEXT:    [[C1:%.*]] = call i32 @foo(i32 [[X]])
+; MSSA-NEXT:    br label %[[LOOP:.*]]
+; MSSA:       [[LOOP]]:
+; MSSA-NEXT:    [[VAL:%.*]] = load i32, ptr null, align 8
+; MSSA-NEXT:    [[C3:%.*]] = call i32 @foo(i32 [[VAL]])
+; MSSA-NEXT:    br label %[[LOOP]]
+;
 
 entry:
   %c1 = call i32 @foo(i32 %x)
@@ -34,18 +46,18 @@ loop:
 }
 
 define hidden void @test_02(i32 %x, i32 %y) {
-
 ; PRE is not allowed because c2 may throw.
-
-; CHECK-LABEL: @test_02
-; CHECK:       entry:
-; CHECK-NEXT:    %c1 = call i32 @foo(i32 %x)
-; CHECK-NEXT:    br label %loop
-; CHECK:       loop:
-; CHECK-NEXT:    %c2 = call i32 @foo(i32 %y)
-; CHECK-NEXT:    %val = load i32, ptr null, align 8
-; CHECK-NEXT:    %c3 = call i32 @foo(i32 %val)
-; CHECK-NEXT:    br label %loop
+; CHECK-LABEL: define hidden void @test_02(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[C1:%.*]] = call i32 @foo(i32 [[X]])
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[C2:%.*]] = call i32 @foo(i32 [[Y]])
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr null, align 8
+; CHECK-NEXT:    [[C3:%.*]] = call i32 @foo(i32 [[VAL]])
+; CHECK-NEXT:    br label %[[LOOP]]
+;
 
 entry:
   %c1 = call i32 @foo(i32 %x)
@@ -59,19 +71,31 @@ loop:
 }
 
 define hidden void @test_03(i32 %x, i32 %y) {
-
 ; PRE of load is allowed because c2 only throws if c1 throws. c3 should
 ; not be eliminated. c4 is eliminated because it only throws if c3 throws.
-
-; CHECK-LABEL: @test_03
-; CHECK:       entry:
-; CHECK-NEXT:    %c1 = call i32 @foo(i32 %x)
-; CHECK-NEXT:    %val.pre = load i32, ptr null, align 8
-; CHECK-NEXT:    br label %loop
-; CHECK:       loop:
-; CHECK-NEXT:    %c3 = call i32 @foo(i32 %y)
-; CHECK-NEXT:    %c5 = call i32 @foo(i32 %val.pre)
-; CHECK-NEXT:    br label %loop
+; MDEP-LABEL: define hidden void @test_03(
+; MDEP-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; MDEP-NEXT:  [[ENTRY:.*:]]
+; MDEP-NEXT:    [[C1:%.*]] = call i32 @foo(i32 [[X]])
+; MDEP-NEXT:    [[VAL_PRE:%.*]] = load i32, ptr null, align 8
+; MDEP-NEXT:    br label %[[LOOP:.*]]
+; MDEP:       [[LOOP]]:
+; MDEP-NEXT:    [[C3:%.*]] = call i32 @foo(i32 [[Y]])
+; MDEP-NEXT:    [[C5:%.*]] = call i32 @foo(i32 [[VAL_PRE]])
+; MDEP-NEXT:    br label %[[LOOP]]
+;
+; MSSA-LABEL: define hidden void @test_03(
+; MSSA-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; MSSA-NEXT:  [[ENTRY:.*:]]
+; MSSA-NEXT:    [[C1:%.*]] = call i32 @foo(i32 [[X]])
+; MSSA-NEXT:    br label %[[LOOP:.*]]
+; MSSA:       [[LOOP]]:
+; MSSA-NEXT:    [[VAL:%.*]] = load i32, ptr null, align 8
+; MSSA-NEXT:    [[C3:%.*]] = call i32 @foo(i32 [[Y]])
+; MSSA-NEXT:    [[VAL2:%.*]] = load i32, ptr null, align 8
+; MSSA-NEXT:    [[C5:%.*]] = call i32 @foo(i32 [[VAL]])
+; MSSA-NEXT:    br label %[[LOOP]]
+;
 
 entry:
   %c1 = call i32 @foo(i32 %x)
@@ -88,18 +112,18 @@ loop:
 }
 
 define hidden void @test_04(i32 %x, i32 %y) {
-
 ; PRE is not allowed even after we remove c2 because now c3 prevents us from it.
-
-; CHECK-LABEL: @test_04
-; CHECK:       entry:
-; CHECK-NEXT:    %c1 = call i32 @foo(i32 %x)
-; CHECK-NEXT:    br label %loop
-; CHECK:       loop:
-; CHECK-NEXT:    %c3 = call i32 @foo(i32 %y)
-; CHECK-NEXT:    %val = load i32, ptr null, align 8
-; CHECK-NEXT:    %c5 = call i32 @foo(i32 %val)
-; CHECK-NEXT:    br label %loop
+; CHECK-LABEL: define hidden void @test_04(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[C1:%.*]] = call i32 @foo(i32 [[X]])
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[C3:%.*]] = call i32 @foo(i32 [[Y]])
+; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr null, align 8
+; CHECK-NEXT:    [[C5:%.*]] = call i32 @foo(i32 [[VAL]])
+; CHECK-NEXT:    br label %[[LOOP]]
+;
 
 entry:
   %c1 = call i32 @foo(i32 %x)
diff --git a/llvm/test/Transforms/GVN/PRE/pre-load.ll b/llvm/test/Transforms/GVN/PRE/pre-load.ll
index bbd20bccdc166..5a07f9f7aa6de 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-load.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-load.ll
@@ -1,21 +1,34 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=gvn -enable-load-pre -S | FileCheck %s
+; RUN: opt < %s -passes=gvn -enable-load-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -enable-load-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes="gvn<load-pre>" -enable-load-pre=false -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 
 define i32 @test1(ptr %p, i1 %C) {
-; CHECK-LABEL: @test1(
-; CHECK-NEXT:  block1:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
-; CHECK:       block2:
-; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    br label [[BLOCK4:%.*]]
-; CHECK:       block3:
-; CHECK-NEXT:    store i32 0, ptr [[P]], align 4
-; CHECK-NEXT:    br label [[BLOCK4]]
-; CHECK:       block4:
-; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
-; CHECK-NEXT:    ret i32 [[PRE]]
+; MDEP-LABEL: @test1(
+; MDEP-NEXT:  block1:
+; MDEP-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MDEP:       block2:
+; MDEP-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P:%.*]], align 4
+; MDEP-NEXT:    br label [[BLOCK4:%.*]]
+; MDEP:       block3:
+; MDEP-NEXT:    store i32 0, ptr [[P]], align 4
+; MDEP-NEXT:    br label [[BLOCK4]]
+; MDEP:       block4:
+; MDEP-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; MDEP-NEXT:    ret i32 [[PRE]]
+;
+; MSSA-LABEL: @test1(
+; MSSA-NEXT:  block1:
+; MSSA-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MSSA:       block2:
+; MSSA-NEXT:    br label [[BLOCK4:%.*]]
+; MSSA:       block3:
+; MSSA-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; MSSA-NEXT:    br label [[BLOCK4]]
+; MSSA:       block4:
+; MSSA-NEXT:    [[PRE:%.*]] = load i32, ptr [[P]], align 4
+; MSSA-NEXT:    ret i32 [[PRE]]
 ;
 block1:
   br i1 %C, label %block2, label %block3
@@ -34,19 +47,32 @@ block4:
 
 ; This is a simple phi translation case.
 define i32 @test2(ptr %p, ptr %q, i1 %C) {
-; CHECK-LABEL: @test2(
-; CHECK-NEXT:  block1:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
-; CHECK:       block2:
-; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[Q:%.*]], align 4
-; CHECK-NEXT:    br label [[BLOCK4:%.*]]
-; CHECK:       block3:
-; CHECK-NEXT:    store i32 0, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    br label [[BLOCK4]]
-; CHECK:       block4:
-; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
-; CHECK-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
-; CHECK-NEXT:    ret i32 [[PRE]]
+; MDEP-LABEL: @test2(
+; MDEP-NEXT:  block1:
+; MDEP-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MDEP:       block2:
+; MDEP-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[Q:%.*]], align 4
+; MDEP-NEXT:    br label [[BLOCK4:%.*]]
+; MDEP:       block3:
+; MDEP-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; MDEP-NEXT:    br label [[BLOCK4]]
+; MDEP:       block4:
+; MDEP-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; MDEP-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; MDEP-NEXT:    ret i32 [[PRE]]
+;
+; MSSA-LABEL: @test2(
+; MSSA-NEXT:  block1:
+; MSSA-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MSSA:       block2:
+; MSSA-NEXT:    br label [[BLOCK4:%.*]]
+; MSSA:       block3:
+; MSSA-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; MSSA-NEXT:    br label [[BLOCK4]]
+; MSSA:       block4:
+; MSSA-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q:%.*]], [[BLOCK2]] ]
+; MSSA-NEXT:    [[PRE:%.*]] = load i32, ptr [[P2]], align 4
+; MSSA-NEXT:    ret i32 [[PRE]]
 ;
 block1:
   br i1 %C, label %block2, label %block3
@@ -66,23 +92,40 @@ block4:
 
 ; This is a PRE case that requires phi translation through a GEP.
 define i32 @test3(ptr %p, ptr %q, ptr %Hack, i1 %C) {
-; CHECK-LABEL: @test3(
-; CHECK-NEXT:  block1:
-; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[Q:%.*]], i32 1
-; CHECK-NEXT:    store ptr [[B]], ptr [[HACK:%.*]], align 8
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
-; CHECK:       block2:
-; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[B]], align 4
-; CHECK-NEXT:    br label [[BLOCK4:%.*]]
-; CHECK:       block3:
-; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
-; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
-; CHECK-NEXT:    br label [[BLOCK4]]
-; CHECK:       block4:
-; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
-; CHECK-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P2]], i32 1
-; CHECK-NEXT:    ret i32 [[PRE]]
+; MDEP-LABEL: @test3(
+; MDEP-NEXT:  block1:
+; MDEP-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[Q:%.*]], i32 1
+; MDEP-NEXT:    store ptr [[B]], ptr [[HACK:%.*]], align 8
+; MDEP-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MDEP:       block2:
+; MDEP-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[B]], align 4
+; MDEP-NEXT:    br label [[BLOCK4:%.*]]
+; MDEP:       block3:
+; MDEP-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
+; MDEP-NEXT:    store i32 0, ptr [[A]], align 4
+; MDEP-NEXT:    br label [[BLOCK4]]
+; MDEP:       block4:
+; MDEP-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; MDEP-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; MDEP-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P2]], i32 1
+; MDEP-NEXT:    ret i32 [[PRE]]
+;
+; MSSA-LABEL: @test3(
+; MSSA-NEXT:  block1:
+; MSSA-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[Q:%.*]], i32 1
+; MSSA-NEXT:    store ptr [[B]], ptr [[HACK:%.*]], align 8
+; MSSA-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MSSA:       block2:
+; MSSA-NEXT:    br label [[BLOCK4:%.*]]
+; MSSA:       block3:
+; MSSA-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
+; MSSA-NEXT:    store i32 0, ptr [[A]], align 4
+; MSSA-NEXT:    br label [[BLOCK4]]
+; MSSA:       block4:
+; MSSA-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; MSSA-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P2]], i32 1
+; MSSA-NEXT:    [[PRE:%.*]] = load i32, ptr [[P3]], align 4
+; MSSA-NEXT:    ret i32 [[PRE]]
 ;
 block1:
   %B = getelementptr i32, ptr %q, i32 1
@@ -107,24 +150,41 @@ block4:
 ;; Here the loaded address is available, but the computation is in 'block3'
 ;; which does not dominate 'block2'.
 define i32 @test4(ptr %p, ptr %q, ptr %Hack, i1 %C) {
-; CHECK-LABEL: @test4(
-; CHECK-NEXT:  block1:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
-; CHECK:       block2:
-; CHECK-NEXT:    [[P3_PHI_TRANS_INSERT:%.*]] = getelementptr i32, ptr [[Q:%.*]], i32 1
-; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P3_PHI_TRANS_INSERT]], align 4
-; CHECK-NEXT:    br label [[BLOCK4:%.*]]
-; CHECK:       block3:
-; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[Q]], i32 1
-; CHECK-NEXT:    store ptr [[B]], ptr [[HACK:%.*]], align 8
-; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
-; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
-; CHECK-NEXT:    br label [[BLOCK4]]
-; CHECK:       block4:
-; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
-; CHECK-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P2]], i32 1
-; CHECK-NEXT:    ret i32 [[PRE]]
+; MDEP-LABEL: @test4(
+; MDEP-NEXT:  block1:
+; MDEP-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MDEP:       block2:
+; MDEP-NEXT:    [[P3_PHI_TRANS_INSERT:%.*]] = getelementptr i32, ptr [[Q:%.*]], i32 1
+; MDEP-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P3_PHI_TRANS_INSERT]], align 4
+; MDEP-NEXT:    br label [[BLOCK4:%.*]]
+; MDEP:       block3:
+; MDEP-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[Q]], i32 1
+; MDEP-NEXT:    store ptr [[B]], ptr [[HACK:%.*]], align 8
+; MDEP-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
+; MDEP-NEXT:    store i32 0, ptr [[A]], align 4
+; MDEP-NEXT:    br label [[BLOCK4]]
+; MDEP:       block4:
+; MDEP-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; MDEP-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; MDEP-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P2]], i32 1
+; MDEP-NEXT:    ret i32 [[PRE]]
+;
+; MSSA-LABEL: @test4(
+; MSSA-NEXT:  block1:
+; MSSA-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MSSA:       block2:
+; MSSA-NEXT:    br label [[BLOCK4:%.*]]
+; MSSA:       block3:
+; MSSA-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[Q:%.*]], i32 1
+; MSSA-NEXT:    store ptr [[B]], ptr [[HACK:%.*]], align 8
+; MSSA-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
+; MSSA-NEXT:    store i32 0, ptr [[A]], align 4
+; MSSA-NEXT:    br label [[BLOCK4]]
+; MSSA:       block4:
+; MSSA-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; MSSA-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P2]], i32 1
+; MSSA-NEXT:    [[PRE:%.*]] = load i32, ptr [[P3]], align 4
+; MSSA-NEXT:    ret i32 [[PRE]]
 ;
 block1:
   br i1 %C, label %block2, label %block3
@@ -149,24 +209,41 @@ block4:
 
 ; Same as test4, with a nuw flag on the GEP.
 define i32 @test4_nuw(ptr %p, ptr %q, ptr %Hack, i1 %C) {
-; CHECK-LABEL: @test4_nuw(
-; CHECK-NEXT:  block1:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
-; CHECK:       block2:
-; CHECK-NEXT:    [[P3_PHI_TRANS_INSERT:%.*]] = getelementptr nuw i32, ptr [[Q:%.*]], i32 1
-; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P3_PHI_TRANS_INSERT]], align 4
-; CHECK-NEXT:    br label [[BLOCK4:%.*]]
-; CHECK:       block3:
-; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[Q]], i32 1
-; CHECK-NEXT:    store ptr [[B]], ptr [[HACK:%.*]], align 8
-; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
-; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
-; CHECK-NEXT:    br label [[BLOCK4]]
-; CHECK:       block4:
-; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
-; CHECK-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr nuw i32, ptr [[P2]], i32 1
-; CHECK-NEXT:    ret i32 [[PRE]]
+; MDEP-LABEL: @test4_nuw(
+; MDEP-NEXT:  block1:
+; MDEP-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MDEP:       block2:
+; MDEP-NEXT:    [[P3_PHI_TRANS_INSERT:%.*]] = getelementptr nuw i32, ptr [[Q:%.*]], i32 1
+; MDEP-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P3_PHI_TRANS_INSERT]], align 4
+; MDEP-NEXT:    br label [[BLOCK4:%.*]]
+; MDEP:       block3:
+; MDEP-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[Q]], i32 1
+; MDEP-NEXT:    store ptr [[B]], ptr [[HACK:%.*]], align 8
+; MDEP-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
+; MDEP-NEXT:    store i32 0, ptr [[A]], align 4
+; MDEP-NEXT:    br label [[BLOCK4]]
+; MDEP:       block4:
+; MDEP-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; MDEP-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; MDEP-NEXT:    [[P3:%.*]] = getelementptr nuw i32, ptr [[P2]], i32 1
+; MDEP-NEXT:    ret i32 [[PRE]]
+;
+; MSSA-LABEL: @test4_nuw(
+; MSSA-NEXT:  block1:
+; MSSA-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MSSA:       block2:
+; MSSA-NEXT:    br label [[BLOCK4:%.*]]
+; MSSA:       block3:
+; MSSA-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[Q:%.*]], i32 1
+; MSSA-NEXT:    store ptr [[B]], ptr [[HACK:%.*]], align 8
+; MSSA-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
+; MSSA-NEXT:    store i32 0, ptr [[A]], align 4
+; MSSA-NEXT:    br label [[BLOCK4]]
+; MSSA:       block4:
+; MSSA-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; MSSA-NEXT:    [[P3:%.*]] = getelementptr nuw i32, ptr [[P2]], i32 1
+; MSSA-NEXT:    [[PRE:%.*]] = load i32, ptr [[P3]], align 4
+; MSSA-NEXT:    ret i32 [[PRE]]
 ;
 block1:
   br i1 %C, label %block2, label %block3
@@ -196,28 +273,50 @@ block4:
 ;}
 
 define void @test5(i32 %N, ptr nocapture %G) nounwind ssp {
-; CHECK-LABEL: @test5(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
-; CHECK-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
-; CHECK:       bb.nph:
-; CHECK-NEXT:    [[TMP:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[DOTPRE:%.*]] = load double, ptr [[G:%.*]], align 8
-; CHECK-NEXT:    br label [[BB:%.*]]
-; CHECK:       bb:
-; CHECK-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP3:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[TMP6]] = add i64 [[INDVAR]], 1
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP6]]
-; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, ptr [[G]], i64 [[INDVAR]]
-; CHECK-NEXT:    [[TMP3]] = load double, ptr [[SCEVGEP]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd double [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    store double [[TMP4]], ptr [[SCEVGEP7]], align 8
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP6]], [[TMP]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
-; CHECK:       return:
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test5(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; MDEP-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; MDEP-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MDEP:       bb.nph:
+; MDEP-NEXT:    [[TMP:%.*]] = zext i32 [[TMP0]] to i64
+; MDEP-NEXT:    [[DOTPRE:%.*]] = load double, ptr [[G:%.*]], align 8
+; MDEP-NEXT:    br label [[BB:%.*]]
+; MDEP:       bb:
+; MDEP-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP3:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[TMP6]] = add i64 [[INDVAR]], 1
+; MDEP-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP6]]
+; MDEP-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, ptr [[G]], i64 [[INDVAR]]
+; MDEP-NEXT:    [[TMP3]] = load double, ptr [[SCEVGEP]], align 8
+; MDEP-NEXT:    [[TMP4:%.*]] = fadd double [[TMP2]], [[TMP3]]
+; MDEP-NEXT:    store double [[TMP4]], ptr [[SCEVGEP7]], align 8
+; MDEP-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP6]], [[TMP]]
+; MDEP-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MDEP:       return:
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test5(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; MSSA-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; MSSA-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MSSA:       bb.nph:
+; MSSA-NEXT:    [[TMP:%.*]] = zext i32 [[TMP0]] to i64
+; MSSA-NEXT:    br label [[BB:%.*]]
+; MSSA:       bb:
+; MSSA-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
+; MSSA-NEXT:    [[TMP6]] = add i64 [[INDVAR]], 1
+; MSSA-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G:%.*]], i64 [[TMP6]]
+; MSSA-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, ptr [[G]], i64 [[INDVAR]]
+; MSSA-NEXT:    [[TMP2:%.*]] = load double, ptr [[SCEVGEP7]], align 8
+; MSSA-NEXT:    [[TMP3:%.*]] = load double, ptr [[SCEVGEP]], align 8
+; MSSA-NEXT:    [[TMP4:%.*]] = fadd double [[TMP2]], [[TMP3]]
+; MSSA-NEXT:    store double [[TMP4]], ptr [[SCEVGEP7]], align 8
+; MSSA-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP6]], [[TMP]]
+; MSSA-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MSSA:       return:
+; MSSA-NEXT:    ret void
 ;
 entry:
   %0 = add i32 %N, -1
@@ -254,28 +353,50 @@ return:
 ;}
 
 define void @test6(i32 %N, ptr nocapture %G) nounwind ssp {
-; CHECK-LABEL: @test6(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
-; CHECK-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
-; CHECK:       bb.nph:
-; CHECK-NEXT:    [[TMP:%.*]] = zext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[DOTPRE:%.*]] = load double, ptr [[G:%.*]], align 8
-; CHECK-NEXT:    br label [[BB:%.*]]
-; CHECK:       bb:
-; CHECK-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP4:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[TMP6]] = add i64 [[INDVAR]], 1
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP6]]
-; CHECK-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, ptr [[G]], i64 [[INDVAR]]
-; CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr [[SCEVGEP]], align 8
-; CHECK-NEXT:    [[TMP4]] = fadd double [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    store double [[TMP4]], ptr [[SCEVGEP]], align 8
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP6]], [[TMP]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
-; CHECK:       return:
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test6(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; MDEP-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; MDEP-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MDEP:       bb.nph:
+; MDEP-NEXT:    [[TMP:%.*]] = zext i32 [[TMP0]] to i64
+; MDEP-NEXT:    [[DOTPRE:%.*]] = load double, ptr [[G:%.*]], align 8
+; MDEP-NEXT:    br label [[BB:%.*]]
+; MDEP:       bb:
+; MDEP-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP4:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[TMP6]] = add i64 [[INDVAR]], 1
+; MDEP-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP6]]
+; MDEP-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, ptr [[G]], i64 [[INDVAR]]
+; MDEP-NEXT:    [[TMP3:%.*]] = load double, ptr [[SCEVGEP]], align 8
+; MDEP-NEXT:    [[TMP4]] = fadd double [[TMP2]], [[TMP3]]
+; MDEP-NEXT:    store double [[TMP4]], ptr [[SCEVGEP]], align 8
+; MDEP-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP6]], [[TMP]]
+; MDEP-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MDEP:       return:
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test6(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; MSSA-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 0
+; MSSA-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MSSA:       bb.nph:
+; MSSA-NEXT:    [[TMP:%.*]] = zext i32 [[TMP0]] to i64
+; MSSA-NEXT:    br label [[BB:%.*]]
+; MSSA:       bb:
+; MSSA-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
+; MSSA-NEXT:    [[TMP6]] = add i64 [[INDVAR]], 1
+; MSSA-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G:%.*]], i64 [[TMP6]]
+; MSSA-NEXT:    [[SCEVGEP7:%.*]] = getelementptr double, ptr [[G]], i64 [[INDVAR]]
+; MSSA-NEXT:    [[TMP2:%.*]] = load double, ptr [[SCEVGEP7]], align 8
+; MSSA-NEXT:    [[TMP3:%.*]] = load double, ptr [[SCEVGEP]], align 8
+; MSSA-NEXT:    [[TMP4:%.*]] = fadd double [[TMP2]], [[TMP3]]
+; MSSA-NEXT:    store double [[TMP4]], ptr [[SCEVGEP]], align 8
+; MSSA-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP6]], [[TMP]]
+; MSSA-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MSSA:       return:
+; MSSA-NEXT:    ret void
 ;
 entry:
   %0 = add i32 %N, -1
@@ -314,31 +435,57 @@ return:
 
 ; This requires phi translation of the adds.
 define void @test7(i32 %N, ptr nocapture %G) nounwind ssp {
-; CHECK-LABEL: @test7(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[G:%.*]], i64 1
-; CHECK-NEXT:    store double 1.000000e+00, ptr [[TMP0]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[N:%.*]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP1]], 1
-; CHECK-NEXT:    br i1 [[TMP2]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
-; CHECK:       bb.nph:
-; CHECK-NEXT:    [[TMP:%.*]] = sext i32 [[TMP1]] to i64
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP]], -1
-; CHECK-NEXT:    br label [[BB:%.*]]
-; CHECK:       bb:
-; CHECK-NEXT:    [[TMP3:%.*]] = phi double [ 1.000000e+00, [[BB_NPH]] ], [ [[TMP5:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP9:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDVAR]], 2
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP9]] = add i64 [[INDVAR]], 1
-; CHECK-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP4:%.*]] = load double, ptr [[SCEVGEP]], align 8
-; CHECK-NEXT:    [[TMP5]] = fadd double [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    store double [[TMP5]], ptr [[SCEVGEP]], align 8
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP9]], [[TMP7]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
-; CHECK:       return:
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test7(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[G:%.*]], i64 1
+; MDEP-NEXT:    store double 1.000000e+00, ptr [[TMP0]], align 8
+; MDEP-NEXT:    [[TMP1:%.*]] = add i32 [[N:%.*]], -1
+; MDEP-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP1]], 1
+; MDEP-NEXT:    br i1 [[TMP2]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MDEP:       bb.nph:
+; MDEP-NEXT:    [[TMP:%.*]] = sext i32 [[TMP1]] to i64
+; MDEP-NEXT:    [[TMP7:%.*]] = add i64 [[TMP]], -1
+; MDEP-NEXT:    br label [[BB:%.*]]
+; MDEP:       bb:
+; MDEP-NEXT:    [[TMP3:%.*]] = phi double [ 1.000000e+00, [[BB_NPH]] ], [ [[TMP5:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP9:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[TMP8:%.*]] = add i64 [[INDVAR]], 2
+; MDEP-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP8]]
+; MDEP-NEXT:    [[TMP9]] = add i64 [[INDVAR]], 1
+; MDEP-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP9]]
+; MDEP-NEXT:    [[TMP4:%.*]] = load double, ptr [[SCEVGEP]], align 8
+; MDEP-NEXT:    [[TMP5]] = fadd double [[TMP3]], [[TMP4]]
+; MDEP-NEXT:    store double [[TMP5]], ptr [[SCEVGEP]], align 8
+; MDEP-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP9]], [[TMP7]]
+; MDEP-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MDEP:       return:
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test7(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[G:%.*]], i64 1
+; MSSA-NEXT:    store double 1.000000e+00, ptr [[TMP0]], align 8
+; MSSA-NEXT:    [[TMP1:%.*]] = add i32 [[N:%.*]], -1
+; MSSA-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[TMP1]], 1
+; MSSA-NEXT:    br i1 [[TMP2]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MSSA:       bb.nph:
+; MSSA-NEXT:    [[TMP:%.*]] = sext i32 [[TMP1]] to i64
+; MSSA-NEXT:    [[TMP7:%.*]] = add i64 [[TMP]], -1
+; MSSA-NEXT:    br label [[BB:%.*]]
+; MSSA:       bb:
+; MSSA-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP9:%.*]], [[BB]] ]
+; MSSA-NEXT:    [[TMP8:%.*]] = add i64 [[INDVAR]], 2
+; MSSA-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP8]]
+; MSSA-NEXT:    [[TMP9]] = add i64 [[INDVAR]], 1
+; MSSA-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP9]]
+; MSSA-NEXT:    [[TMP3:%.*]] = load double, ptr [[SCEVGEP10]], align 8
+; MSSA-NEXT:    [[TMP4:%.*]] = load double, ptr [[SCEVGEP]], align 8
+; MSSA-NEXT:    [[TMP5:%.*]] = fadd double [[TMP3]], [[TMP4]]
+; MSSA-NEXT:    store double [[TMP5]], ptr [[SCEVGEP]], align 8
+; MSSA-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP9]], [[TMP7]]
+; MSSA-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MSSA:       return:
+; MSSA-NEXT:    ret void
 ;
 entry:
   %0 = getelementptr inbounds double, ptr %G, i64 1
@@ -374,22 +521,37 @@ return:
 ;; Here the loaded address isn't available in 'block2' at all, requiring a new
 ;; GEP to be inserted into it.
 define i32 @test8(ptr %p, ptr %q, ptr %Hack, i1 %C) {
-; CHECK-LABEL: @test8(
-; CHECK-NEXT:  block1:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
-; CHECK:       block2:
-; CHECK-NEXT:    [[P3_PHI_TRANS_INSERT:%.*]] = getelementptr i32, ptr [[Q:%.*]], i32 1
-; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P3_PHI_TRANS_INSERT]], align 4
-; CHECK-NEXT:    br label [[BLOCK4:%.*]]
-; CHECK:       block3:
-; CHECK-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
-; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
-; CHECK-NEXT:    br label [[BLOCK4]]
-; CHECK:       block4:
-; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
-; CHECK-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
-; CHECK-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P2]], i32 1
-; CHECK-NEXT:    ret i32 [[PRE]]
+; MDEP-LABEL: @test8(
+; MDEP-NEXT:  block1:
+; MDEP-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MDEP:       block2:
+; MDEP-NEXT:    [[P3_PHI_TRANS_INSERT:%.*]] = getelementptr i32, ptr [[Q:%.*]], i32 1
+; MDEP-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P3_PHI_TRANS_INSERT]], align 4
+; MDEP-NEXT:    br label [[BLOCK4:%.*]]
+; MDEP:       block3:
+; MDEP-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
+; MDEP-NEXT:    store i32 0, ptr [[A]], align 4
+; MDEP-NEXT:    br label [[BLOCK4]]
+; MDEP:       block4:
+; MDEP-NEXT:    [[PRE:%.*]] = phi i32 [ 0, [[BLOCK3]] ], [ [[PRE_PRE]], [[BLOCK2]] ]
+; MDEP-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q]], [[BLOCK2]] ]
+; MDEP-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P2]], i32 1
+; MDEP-NEXT:    ret i32 [[PRE]]
+;
+; MSSA-LABEL: @test8(
+; MSSA-NEXT:  block1:
+; MSSA-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MSSA:       block2:
+; MSSA-NEXT:    br label [[BLOCK4:%.*]]
+; MSSA:       block3:
+; MSSA-NEXT:    [[A:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 1
+; MSSA-NEXT:    store i32 0, ptr [[A]], align 4
+; MSSA-NEXT:    br label [[BLOCK4]]
+; MSSA:       block4:
+; MSSA-NEXT:    [[P2:%.*]] = phi ptr [ [[P]], [[BLOCK3]] ], [ [[Q:%.*]], [[BLOCK2]] ]
+; MSSA-NEXT:    [[P3:%.*]] = getelementptr i32, ptr [[P2]], i32 1
+; MSSA-NEXT:    [[PRE:%.*]] = load i32, ptr [[P3]], align 4
+; MSSA-NEXT:    ret i32 [[PRE]]
 ;
 block1:
   br i1 %C, label %block2, label %block3
@@ -417,31 +579,55 @@ block4:
 
 ; This requires phi translation of the adds.
 define void @test9(i32 %N, ptr nocapture %G) nounwind ssp {
-; CHECK-LABEL: @test9(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 1
-; CHECK-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
-; CHECK:       bb.nph:
-; CHECK-NEXT:    [[TMP:%.*]] = sext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP]], -1
-; CHECK-NEXT:    [[SCEVGEP10_PHI_TRANS_INSERT:%.*]] = getelementptr double, ptr [[G:%.*]], i64 1
-; CHECK-NEXT:    [[DOTPRE:%.*]] = load double, ptr [[SCEVGEP10_PHI_TRANS_INSERT]], align 8
-; CHECK-NEXT:    br label [[BB:%.*]]
-; CHECK:       bb:
-; CHECK-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP4:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP9:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[INDVAR]], 2
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP9]] = add i64 [[INDVAR]], 1
-; CHECK-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP3:%.*]] = load double, ptr [[SCEVGEP]], align 8
-; CHECK-NEXT:    [[TMP4]] = fadd double [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    store double [[TMP4]], ptr [[SCEVGEP]], align 8
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP9]], [[TMP7]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
-; CHECK:       return:
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test9(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; MDEP-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 1
+; MDEP-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MDEP:       bb.nph:
+; MDEP-NEXT:    [[TMP:%.*]] = sext i32 [[TMP0]] to i64
+; MDEP-NEXT:    [[TMP7:%.*]] = add i64 [[TMP]], -1
+; MDEP-NEXT:    [[SCEVGEP10_PHI_TRANS_INSERT:%.*]] = getelementptr double, ptr [[G:%.*]], i64 1
+; MDEP-NEXT:    [[DOTPRE:%.*]] = load double, ptr [[SCEVGEP10_PHI_TRANS_INSERT]], align 8
+; MDEP-NEXT:    br label [[BB:%.*]]
+; MDEP:       bb:
+; MDEP-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP4:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP9:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[TMP8:%.*]] = add i64 [[INDVAR]], 2
+; MDEP-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP8]]
+; MDEP-NEXT:    [[TMP9]] = add i64 [[INDVAR]], 1
+; MDEP-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP9]]
+; MDEP-NEXT:    [[TMP3:%.*]] = load double, ptr [[SCEVGEP]], align 8
+; MDEP-NEXT:    [[TMP4]] = fadd double [[TMP2]], [[TMP3]]
+; MDEP-NEXT:    store double [[TMP4]], ptr [[SCEVGEP]], align 8
+; MDEP-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP9]], [[TMP7]]
+; MDEP-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MDEP:       return:
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test9(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; MSSA-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 1
+; MSSA-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MSSA:       bb.nph:
+; MSSA-NEXT:    [[TMP:%.*]] = sext i32 [[TMP0]] to i64
+; MSSA-NEXT:    [[TMP7:%.*]] = add i64 [[TMP]], -1
+; MSSA-NEXT:    br label [[BB:%.*]]
+; MSSA:       bb:
+; MSSA-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP9:%.*]], [[BB]] ]
+; MSSA-NEXT:    [[TMP8:%.*]] = add i64 [[INDVAR]], 2
+; MSSA-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G:%.*]], i64 [[TMP8]]
+; MSSA-NEXT:    [[TMP9]] = add i64 [[INDVAR]], 1
+; MSSA-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP9]]
+; MSSA-NEXT:    [[TMP2:%.*]] = load double, ptr [[SCEVGEP10]], align 8
+; MSSA-NEXT:    [[TMP3:%.*]] = load double, ptr [[SCEVGEP]], align 8
+; MSSA-NEXT:    [[TMP4:%.*]] = fadd double [[TMP2]], [[TMP3]]
+; MSSA-NEXT:    store double [[TMP4]], ptr [[SCEVGEP]], align 8
+; MSSA-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP9]], [[TMP7]]
+; MSSA-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MSSA:       return:
+; MSSA-NEXT:    ret void
 ;
 entry:
   add i32 0, 0
@@ -482,35 +668,62 @@ return:
 
 ; PR5501
 define void @test10(i32 %N, ptr nocapture %G) nounwind ssp {
-; CHECK-LABEL: @test10(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 1
-; CHECK-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
-; CHECK:       bb.nph:
-; CHECK-NEXT:    [[TMP:%.*]] = sext i32 [[TMP0]] to i64
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[TMP]], -1
-; CHECK-NEXT:    [[SCEVGEP12_PHI_TRANS_INSERT:%.*]] = getelementptr double, ptr [[G:%.*]], i64 1
-; CHECK-NEXT:    [[DOTPRE:%.*]] = load double, ptr [[SCEVGEP12_PHI_TRANS_INSERT]], align 8
-; CHECK-NEXT:    [[DOTPRE1:%.*]] = load double, ptr [[G]], align 8
-; CHECK-NEXT:    br label [[BB:%.*]]
-; CHECK:       bb:
-; CHECK-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE1]], [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP4:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP11:%.*]], [[BB]] ]
-; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[INDVAR]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDVAR]], 2
-; CHECK-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP11]] = add i64 [[INDVAR]], 1
-; CHECK-NEXT:    [[SCEVGEP12:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP11]]
-; CHECK-NEXT:    [[TMP4]] = load double, ptr [[SCEVGEP10]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd double [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6]] = fadd double [[TMP5]], [[TMP2]]
-; CHECK-NEXT:    store double [[TMP6]], ptr [[SCEVGEP12]], align 8
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP11]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
-; CHECK:       return:
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test10(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; MDEP-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 1
+; MDEP-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MDEP:       bb.nph:
+; MDEP-NEXT:    [[TMP:%.*]] = sext i32 [[TMP0]] to i64
+; MDEP-NEXT:    [[TMP8:%.*]] = add i64 [[TMP]], -1
+; MDEP-NEXT:    [[SCEVGEP12_PHI_TRANS_INSERT:%.*]] = getelementptr double, ptr [[G:%.*]], i64 1
+; MDEP-NEXT:    [[DOTPRE:%.*]] = load double, ptr [[SCEVGEP12_PHI_TRANS_INSERT]], align 8
+; MDEP-NEXT:    [[DOTPRE1:%.*]] = load double, ptr [[G]], align 8
+; MDEP-NEXT:    br label [[BB:%.*]]
+; MDEP:       bb:
+; MDEP-NEXT:    [[TMP2:%.*]] = phi double [ [[DOTPRE1]], [[BB_NPH]] ], [ [[TMP6:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[TMP3:%.*]] = phi double [ [[DOTPRE]], [[BB_NPH]] ], [ [[TMP4:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP11:%.*]], [[BB]] ]
+; MDEP-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G]], i64 [[INDVAR]]
+; MDEP-NEXT:    [[TMP9:%.*]] = add i64 [[INDVAR]], 2
+; MDEP-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP9]]
+; MDEP-NEXT:    [[TMP11]] = add i64 [[INDVAR]], 1
+; MDEP-NEXT:    [[SCEVGEP12:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP11]]
+; MDEP-NEXT:    [[TMP4]] = load double, ptr [[SCEVGEP10]], align 8
+; MDEP-NEXT:    [[TMP5:%.*]] = fadd double [[TMP3]], [[TMP4]]
+; MDEP-NEXT:    [[TMP6]] = fadd double [[TMP5]], [[TMP2]]
+; MDEP-NEXT:    store double [[TMP6]], ptr [[SCEVGEP12]], align 8
+; MDEP-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP11]], [[TMP8]]
+; MDEP-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MDEP:       return:
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test10(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; MSSA-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[TMP0]], 1
+; MSSA-NEXT:    br i1 [[TMP1]], label [[BB_NPH:%.*]], label [[RETURN:%.*]]
+; MSSA:       bb.nph:
+; MSSA-NEXT:    [[TMP:%.*]] = sext i32 [[TMP0]] to i64
+; MSSA-NEXT:    [[TMP8:%.*]] = add i64 [[TMP]], -1
+; MSSA-NEXT:    br label [[BB:%.*]]
+; MSSA:       bb:
+; MSSA-NEXT:    [[INDVAR:%.*]] = phi i64 [ 0, [[BB_NPH]] ], [ [[TMP11:%.*]], [[BB]] ]
+; MSSA-NEXT:    [[SCEVGEP:%.*]] = getelementptr double, ptr [[G:%.*]], i64 [[INDVAR]]
+; MSSA-NEXT:    [[TMP9:%.*]] = add i64 [[INDVAR]], 2
+; MSSA-NEXT:    [[SCEVGEP10:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP9]]
+; MSSA-NEXT:    [[TMP11]] = add i64 [[INDVAR]], 1
+; MSSA-NEXT:    [[SCEVGEP12:%.*]] = getelementptr double, ptr [[G]], i64 [[TMP11]]
+; MSSA-NEXT:    [[TMP2:%.*]] = load double, ptr [[SCEVGEP12]], align 8
+; MSSA-NEXT:    [[TMP3:%.*]] = load double, ptr [[SCEVGEP10]], align 8
+; MSSA-NEXT:    [[TMP4:%.*]] = fadd double [[TMP2]], [[TMP3]]
+; MSSA-NEXT:    [[TMP5:%.*]] = load double, ptr [[SCEVGEP]], align 8
+; MSSA-NEXT:    [[TMP6:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; MSSA-NEXT:    store double [[TMP6]], ptr [[SCEVGEP12]], align 8
+; MSSA-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP11]], [[TMP8]]
+; MSSA-NEXT:    br i1 [[EXITCOND]], label [[RETURN]], label [[BB]]
+; MSSA:       return:
+; MSSA-NEXT:    ret void
 ;
 entry:
   %0 = add i32 %N, -1
@@ -547,24 +760,40 @@ return:
 
 ; Test critical edge splitting.
 define i32 @test11(ptr %p, i1 %C, i32 %N) {
-; CHECK-LABEL: @test11(
-; CHECK-NEXT:  block1:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
-; CHECK:       block2:
-; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[N:%.*]], 1
-; CHECK-NEXT:    br i1 [[COND]], label [[BLOCK2_BLOCK4_CRIT_EDGE:%.*]], label [[BLOCK5:%.*]]
-; CHECK:       block2.block4_crit_edge:
-; CHECK-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    br label [[BLOCK4:%.*]]
-; CHECK:       block3:
-; CHECK-NEXT:    store i32 0, ptr [[P]], align 4
-; CHECK-NEXT:    br label [[BLOCK4]]
-; CHECK:       block4:
-; CHECK-NEXT:    [[PRE:%.*]] = phi i32 [ [[PRE_PRE]], [[BLOCK2_BLOCK4_CRIT_EDGE]] ], [ 0, [[BLOCK3]] ]
-; CHECK-NEXT:    br label [[BLOCK5]]
-; CHECK:       block5:
-; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ 0, [[BLOCK2]] ], [ [[PRE]], [[BLOCK4]] ]
-; CHECK-NEXT:    ret i32 [[RET]]
+; MDEP-LABEL: @test11(
+; MDEP-NEXT:  block1:
+; MDEP-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MDEP:       block2:
+; MDEP-NEXT:    [[COND:%.*]] = icmp sgt i32 [[N:%.*]], 1
+; MDEP-NEXT:    br i1 [[COND]], label [[BLOCK2_BLOCK4_CRIT_EDGE:%.*]], label [[BLOCK5:%.*]]
+; MDEP:       block2.block4_crit_edge:
+; MDEP-NEXT:    [[PRE_PRE:%.*]] = load i32, ptr [[P:%.*]], align 4
+; MDEP-NEXT:    br label [[BLOCK4:%.*]]
+; MDEP:       block3:
+; MDEP-NEXT:    store i32 0, ptr [[P]], align 4
+; MDEP-NEXT:    br label [[BLOCK4]]
+; MDEP:       block4:
+; MDEP-NEXT:    [[PRE:%.*]] = phi i32 [ [[PRE_PRE]], [[BLOCK2_BLOCK4_CRIT_EDGE]] ], [ 0, [[BLOCK3]] ]
+; MDEP-NEXT:    br label [[BLOCK5]]
+; MDEP:       block5:
+; MDEP-NEXT:    [[RET:%.*]] = phi i32 [ 0, [[BLOCK2]] ], [ [[PRE]], [[BLOCK4]] ]
+; MDEP-NEXT:    ret i32 [[RET]]
+;
+; MSSA-LABEL: @test11(
+; MSSA-NEXT:  block1:
+; MSSA-NEXT:    br i1 [[C:%.*]], label [[BLOCK2:%.*]], label [[BLOCK3:%.*]]
+; MSSA:       block2:
+; MSSA-NEXT:    [[COND:%.*]] = icmp sgt i32 [[N:%.*]], 1
+; MSSA-NEXT:    br i1 [[COND]], label [[BLOCK4:%.*]], label [[BLOCK5:%.*]]
+; MSSA:       block3:
+; MSSA-NEXT:    store i32 0, ptr [[P:%.*]], align 4
+; MSSA-NEXT:    br label [[BLOCK4]]
+; MSSA:       block4:
+; MSSA-NEXT:    [[PRE:%.*]] = load i32, ptr [[P]], align 4
+; MSSA-NEXT:    br label [[BLOCK5]]
+; MSSA:       block5:
+; MSSA-NEXT:    [[RET:%.*]] = phi i32 [ 0, [[BLOCK2]] ], [ [[PRE]], [[BLOCK4]] ]
+; MSSA-NEXT:    ret i32 [[RET]]
 ;
 block1:
   br i1 %C, label %block2, label %block3
@@ -726,17 +955,30 @@ follow_2:
 ; Since it is OK to speculate, PRE is allowed.
 
 define i32 @test15(ptr noalias nocapture readonly dereferenceable(8) align 4 %x, ptr noalias nocapture %r, i32 %a) nofree nosync {
-; CHECK-LABEL: @test15(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
-; CHECK-NEXT:    [[VV_PRE:%.*]] = load i32, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    store i32 [[VV_PRE]], ptr [[R:%.*]], align 4
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    call void @f()
-; CHECK-NEXT:    ret i32 [[VV_PRE]]
+; MDEP-LABEL: @test15(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
+; MDEP-NEXT:    [[VV_PRE:%.*]] = load i32, ptr [[X:%.*]], align 4
+; MDEP-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; MDEP:       if.then:
+; MDEP-NEXT:    store i32 [[VV_PRE]], ptr [[R:%.*]], align 4
+; MDEP-NEXT:    br label [[IF_END]]
+; MDEP:       if.end:
+; MDEP-NEXT:    call void @f()
+; MDEP-NEXT:    ret i32 [[VV_PRE]]
+;
+; MSSA-LABEL: @test15(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
+; MSSA-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; MSSA:       if.then:
+; MSSA-NEXT:    [[UU:%.*]] = load i32, ptr [[X:%.*]], align 4
+; MSSA-NEXT:    store i32 [[UU]], ptr [[R:%.*]], align 4
+; MSSA-NEXT:    br label [[IF_END]]
+; MSSA:       if.end:
+; MSSA-NEXT:    call void @f()
+; MSSA-NEXT:    [[VV:%.*]] = load i32, ptr [[X]], align 4
+; MSSA-NEXT:    ret i32 [[VV]]
 ;
 
 entry:
@@ -763,17 +1005,30 @@ if.end:
 ; Since it is OK to speculate, PRE is allowed.
 
 define i32 @test16(ptr noalias nocapture readonly dereferenceable(8) align 4 %x, ptr noalias nocapture %r, i32 %a) nofree nosync {
-; CHECK-LABEL: @test16(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
-; CHECK-NEXT:    [[VV_PRE:%.*]] = load i32, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    store i32 [[VV_PRE]], ptr [[R:%.*]], align 4
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    call void @f()
-; CHECK-NEXT:    ret i32 [[VV_PRE]]
+; MDEP-LABEL: @test16(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
+; MDEP-NEXT:    [[VV_PRE:%.*]] = load i32, ptr [[X:%.*]], align 4
+; MDEP-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; MDEP:       if.then:
+; MDEP-NEXT:    store i32 [[VV_PRE]], ptr [[R:%.*]], align 4
+; MDEP-NEXT:    br label [[IF_END]]
+; MDEP:       if.end:
+; MDEP-NEXT:    call void @f()
+; MDEP-NEXT:    ret i32 [[VV_PRE]]
+;
+; MSSA-LABEL: @test16(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[A:%.*]], 0
+; MSSA-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; MSSA:       if.then:
+; MSSA-NEXT:    [[UU:%.*]] = load i32, ptr [[X:%.*]], align 4
+; MSSA-NEXT:    store i32 [[UU]], ptr [[R:%.*]], align 4
+; MSSA-NEXT:    br label [[IF_END]]
+; MSSA:       if.end:
+; MSSA-NEXT:    call void @f()
+; MSSA-NEXT:    [[VV:%.*]] = load i32, ptr [[X]], align 4
+; MSSA-NEXT:    ret i32 [[VV]]
 ;
 
 entry:
@@ -808,36 +1063,67 @@ declare i1 @bar()
 ; We can move all loads into predecessors.
 
 define void @test17(ptr %p1, ptr %p2, ptr %p3, ptr %p4)
-; CHECK-LABEL: @test17(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[V1:%.*]] = load i64, ptr [[P1:%.*]], align 8
-; CHECK-NEXT:    [[COND1:%.*]] = icmp sgt i64 [[V1]], 200
-; CHECK-NEXT:    br i1 [[COND1]], label [[BB200:%.*]], label [[BB1:%.*]]
-; CHECK:       bb1:
-; CHECK-NEXT:    [[COND2:%.*]] = icmp sgt i64 [[V1]], 100
-; CHECK-NEXT:    br i1 [[COND2]], label [[BB100:%.*]], label [[BB2:%.*]]
-; CHECK:       bb2:
-; CHECK-NEXT:    [[V2:%.*]] = add nsw i64 [[V1]], 1
-; CHECK-NEXT:    store i64 [[V2]], ptr [[P1]], align 8
-; CHECK-NEXT:    br label [[BB3:%.*]]
-; CHECK:       bb3:
-; CHECK-NEXT:    [[V3:%.*]] = phi i64 [ [[V3_PRE:%.*]], [[BB200]] ], [ [[V3_PRE1:%.*]], [[BB100]] ], [ [[V2]], [[BB2]] ]
-; CHECK-NEXT:    store i64 [[V3]], ptr [[P2:%.*]], align 8
-; CHECK-NEXT:    ret void
-; CHECK:       bb100:
-; CHECK-NEXT:    [[COND3:%.*]] = call i1 @foo()
-; CHECK-NEXT:    [[V3_PRE1]] = load i64, ptr [[P1]], align 8
-; CHECK-NEXT:    br i1 [[COND3]], label [[BB3]], label [[BB101:%.*]]
-; CHECK:       bb101:
-; CHECK-NEXT:    store i64 [[V3_PRE1]], ptr [[P3:%.*]], align 8
-; CHECK-NEXT:    ret void
-; CHECK:       bb200:
-; CHECK-NEXT:    [[COND4:%.*]] = call i1 @bar()
-; CHECK-NEXT:    [[V3_PRE]] = load i64, ptr [[P1]], align 8
-; CHECK-NEXT:    br i1 [[COND4]], label [[BB3]], label [[BB201:%.*]]
-; CHECK:       bb201:
-; CHECK-NEXT:    store i64 [[V3_PRE]], ptr [[P4:%.*]], align 8
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test17(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[V1:%.*]] = load i64, ptr [[P1:%.*]], align 8
+; MDEP-NEXT:    [[COND1:%.*]] = icmp sgt i64 [[V1]], 200
+; MDEP-NEXT:    br i1 [[COND1]], label [[BB200:%.*]], label [[BB1:%.*]]
+; MDEP:       bb1:
+; MDEP-NEXT:    [[COND2:%.*]] = icmp sgt i64 [[V1]], 100
+; MDEP-NEXT:    br i1 [[COND2]], label [[BB100:%.*]], label [[BB2:%.*]]
+; MDEP:       bb2:
+; MDEP-NEXT:    [[V2:%.*]] = add nsw i64 [[V1]], 1
+; MDEP-NEXT:    store i64 [[V2]], ptr [[P1]], align 8
+; MDEP-NEXT:    br label [[BB3:%.*]]
+; MDEP:       bb3:
+; MDEP-NEXT:    [[V3:%.*]] = phi i64 [ [[V3_PRE:%.*]], [[BB200]] ], [ [[V3_PRE1:%.*]], [[BB100]] ], [ [[V2]], [[BB2]] ]
+; MDEP-NEXT:    store i64 [[V3]], ptr [[P2:%.*]], align 8
+; MDEP-NEXT:    ret void
+; MDEP:       bb100:
+; MDEP-NEXT:    [[COND3:%.*]] = call i1 @foo()
+; MDEP-NEXT:    [[V3_PRE1]] = load i64, ptr [[P1]], align 8
+; MDEP-NEXT:    br i1 [[COND3]], label [[BB3]], label [[BB101:%.*]]
+; MDEP:       bb101:
+; MDEP-NEXT:    store i64 [[V3_PRE1]], ptr [[P3:%.*]], align 8
+; MDEP-NEXT:    ret void
+; MDEP:       bb200:
+; MDEP-NEXT:    [[COND4:%.*]] = call i1 @bar()
+; MDEP-NEXT:    [[V3_PRE]] = load i64, ptr [[P1]], align 8
+; MDEP-NEXT:    br i1 [[COND4]], label [[BB3]], label [[BB201:%.*]]
+; MDEP:       bb201:
+; MDEP-NEXT:    store i64 [[V3_PRE]], ptr [[P4:%.*]], align 8
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test17(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[V1:%.*]] = load i64, ptr [[P1:%.*]], align 8
+; MSSA-NEXT:    [[COND1:%.*]] = icmp sgt i64 [[V1]], 200
+; MSSA-NEXT:    br i1 [[COND1]], label [[BB200:%.*]], label [[BB1:%.*]]
+; MSSA:       bb1:
+; MSSA-NEXT:    [[COND2:%.*]] = icmp sgt i64 [[V1]], 100
+; MSSA-NEXT:    br i1 [[COND2]], label [[BB100:%.*]], label [[BB2:%.*]]
+; MSSA:       bb2:
+; MSSA-NEXT:    [[V2:%.*]] = add nsw i64 [[V1]], 1
+; MSSA-NEXT:    store i64 [[V2]], ptr [[P1]], align 8
+; MSSA-NEXT:    br label [[BB3:%.*]]
+; MSSA:       bb3:
+; MSSA-NEXT:    [[V3:%.*]] = load i64, ptr [[P1]], align 8
+; MSSA-NEXT:    store i64 [[V3]], ptr [[P2:%.*]], align 8
+; MSSA-NEXT:    ret void
+; MSSA:       bb100:
+; MSSA-NEXT:    [[COND3:%.*]] = call i1 @foo()
+; MSSA-NEXT:    br i1 [[COND3]], label [[BB3]], label [[BB101:%.*]]
+; MSSA:       bb101:
+; MSSA-NEXT:    [[V4:%.*]] = load i64, ptr [[P1]], align 8
+; MSSA-NEXT:    store i64 [[V4]], ptr [[P3:%.*]], align 8
+; MSSA-NEXT:    ret void
+; MSSA:       bb200:
+; MSSA-NEXT:    [[COND4:%.*]] = call i1 @bar()
+; MSSA-NEXT:    br i1 [[COND4]], label [[BB3]], label [[BB201:%.*]]
+; MSSA:       bb201:
+; MSSA-NEXT:    [[V5:%.*]] = load i64, ptr [[P1]], align 8
+; MSSA-NEXT:    store i64 [[V5]], ptr [[P4:%.*]], align 8
+; MSSA-NEXT:    ret void
 ;
 {
 entry:
@@ -882,18 +1168,31 @@ bb201:
 ; So ValuesPerBlock[%if.then] should not be replaced when the load instruction
 ; is moved to %entry.
 define void @test18(i1 %cond, ptr %p1, ptr %p2) {
-; CHECK-LABEL: @test18(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[V2_PRE:%.*]] = load i16, ptr [[P1:%.*]], align 2
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[DEC:%.*]] = add i16 [[V2_PRE]], -1
-; CHECK-NEXT:    store i16 [[DEC]], ptr [[P1]], align 2
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[V2:%.*]] = phi i16 [ [[DEC]], [[IF_THEN]] ], [ [[V2_PRE]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    store i16 [[V2]], ptr [[P2:%.*]], align 2
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test18(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[V2_PRE:%.*]] = load i16, ptr [[P1:%.*]], align 2
+; MDEP-NEXT:    br i1 [[COND:%.*]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; MDEP:       if.then:
+; MDEP-NEXT:    [[DEC:%.*]] = add i16 [[V2_PRE]], -1
+; MDEP-NEXT:    store i16 [[DEC]], ptr [[P1]], align 2
+; MDEP-NEXT:    br label [[IF_END]]
+; MDEP:       if.end:
+; MDEP-NEXT:    [[V2:%.*]] = phi i16 [ [[DEC]], [[IF_THEN]] ], [ [[V2_PRE]], [[ENTRY:%.*]] ]
+; MDEP-NEXT:    store i16 [[V2]], ptr [[P2:%.*]], align 2
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test18(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    br i1 [[COND:%.*]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; MSSA:       if.then:
+; MSSA-NEXT:    [[V1:%.*]] = load i16, ptr [[P1:%.*]], align 2
+; MSSA-NEXT:    [[DEC:%.*]] = add i16 [[V1]], -1
+; MSSA-NEXT:    store i16 [[DEC]], ptr [[P1]], align 2
+; MSSA-NEXT:    br label [[IF_END]]
+; MSSA:       if.end:
+; MSSA-NEXT:    [[V2:%.*]] = load i16, ptr [[P1]], align 2
+; MSSA-NEXT:    store i16 [[V2]], ptr [[P2:%.*]], align 2
+; MSSA-NEXT:    ret void
 ;
 entry:
   br i1 %cond, label %if.end, label %if.then
@@ -912,32 +1211,56 @@ if.end:
 
 ; PRE of load instructions should not cross exception handling instructions.
 define void @test19(i1 %cond, ptr %p1, ptr %p2)
-; CHECK-LABEL: @test19(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    [[V2:%.*]] = load i64, ptr [[P2:%.*]], align 8
-; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[V2]], 1
-; CHECK-NEXT:    store i64 [[ADD]], ptr [[P1:%.*]], align 8
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       else:
-; CHECK-NEXT:    invoke void @f()
-; CHECK-NEXT:            to label [[ELSE_END_CRIT_EDGE:%.*]] unwind label [[LPAD:%.*]]
-; CHECK:       else.end_crit_edge:
-; CHECK-NEXT:    [[V1_PRE:%.*]] = load i64, ptr [[P1]], align 8
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[V1:%.*]] = phi i64 [ [[V1_PRE]], [[ELSE_END_CRIT_EDGE]] ], [ [[ADD]], [[THEN]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[V1]], 100
-; CHECK-NEXT:    store i64 [[AND]], ptr [[P2]], align 8
-; CHECK-NEXT:    ret void
-; CHECK:       lpad:
-; CHECK-NEXT:    [[LP:%.*]] = landingpad { ptr, i32 }
-; CHECK-NEXT:            cleanup
-; CHECK-NEXT:    [[V3:%.*]] = load i64, ptr [[P1]], align 8
-; CHECK-NEXT:    [[OR:%.*]] = or i64 [[V3]], 200
-; CHECK-NEXT:    store i64 [[OR]], ptr [[P1]], align 8
-; CHECK-NEXT:    resume { ptr, i32 } [[LP]]
+; MDEP-LABEL: @test19(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; MDEP:       then:
+; MDEP-NEXT:    [[V2:%.*]] = load i64, ptr [[P2:%.*]], align 8
+; MDEP-NEXT:    [[ADD:%.*]] = add i64 [[V2]], 1
+; MDEP-NEXT:    store i64 [[ADD]], ptr [[P1:%.*]], align 8
+; MDEP-NEXT:    br label [[END:%.*]]
+; MDEP:       else:
+; MDEP-NEXT:    invoke void @f()
+; MDEP-NEXT:            to label [[ELSE_END_CRIT_EDGE:%.*]] unwind label [[LPAD:%.*]]
+; MDEP:       else.end_crit_edge:
+; MDEP-NEXT:    [[V1_PRE:%.*]] = load i64, ptr [[P1]], align 8
+; MDEP-NEXT:    br label [[END]]
+; MDEP:       end:
+; MDEP-NEXT:    [[V1:%.*]] = phi i64 [ [[V1_PRE]], [[ELSE_END_CRIT_EDGE]] ], [ [[ADD]], [[THEN]] ]
+; MDEP-NEXT:    [[AND:%.*]] = and i64 [[V1]], 100
+; MDEP-NEXT:    store i64 [[AND]], ptr [[P2]], align 8
+; MDEP-NEXT:    ret void
+; MDEP:       lpad:
+; MDEP-NEXT:    [[LP:%.*]] = landingpad { ptr, i32 }
+; MDEP-NEXT:            cleanup
+; MDEP-NEXT:    [[V3:%.*]] = load i64, ptr [[P1]], align 8
+; MDEP-NEXT:    [[OR:%.*]] = or i64 [[V3]], 200
+; MDEP-NEXT:    store i64 [[OR]], ptr [[P1]], align 8
+; MDEP-NEXT:    resume { ptr, i32 } [[LP]]
+;
+; MSSA-LABEL: @test19(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; MSSA:       then:
+; MSSA-NEXT:    [[V2:%.*]] = load i64, ptr [[P2:%.*]], align 8
+; MSSA-NEXT:    [[ADD:%.*]] = add i64 [[V2]], 1
+; MSSA-NEXT:    store i64 [[ADD]], ptr [[P1:%.*]], align 8
+; MSSA-NEXT:    br label [[END:%.*]]
+; MSSA:       else:
+; MSSA-NEXT:    invoke void @f()
+; MSSA-NEXT:            to label [[END]] unwind label [[LPAD:%.*]]
+; MSSA:       end:
+; MSSA-NEXT:    [[V1:%.*]] = load i64, ptr [[P1]], align 8
+; MSSA-NEXT:    [[AND:%.*]] = and i64 [[V1]], 100
+; MSSA-NEXT:    store i64 [[AND]], ptr [[P2]], align 8
+; MSSA-NEXT:    ret void
+; MSSA:       lpad:
+; MSSA-NEXT:    [[LP:%.*]] = landingpad { ptr, i32 }
+; MSSA-NEXT:            cleanup
+; MSSA-NEXT:    [[V3:%.*]] = load i64, ptr [[P1]], align 8
+; MSSA-NEXT:    [[OR:%.*]] = or i64 [[V3]], 200
+; MSSA-NEXT:    store i64 [[OR]], ptr [[P1]], align 8
+; MSSA-NEXT:    resume { ptr, i32 } [[LP]]
 ;
   personality ptr @__CxxFrameHandler3 {
 entry:
@@ -1050,29 +1373,50 @@ if.end:
 ; Call to function @maybethrow may cause exception, so the load of %v3 can't
 ; be hoisted to block %if.else.
 define void @test22(i1 %cond, ptr %p1, ptr %p2) {
-; CHECK-LABEL: @test22(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[V1:%.*]] = load i64, ptr [[P1:%.*]], align 8
-; CHECK-NEXT:    [[DEC:%.*]] = add i64 [[V1]], -1
-; CHECK-NEXT:    store i64 [[DEC]], ptr [[P1]], align 8
-; CHECK-NEXT:    br label [[IF_END:%.*]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[V2:%.*]] = phi i64 [ [[V2_PRE:%.*]], [[IF_ELSE_IF_END_CRIT_EDGE:%.*]] ], [ [[DEC]], [[IF_THEN]] ]
-; CHECK-NEXT:    store i64 [[V2]], ptr [[P2:%.*]], align 8
-; CHECK-NEXT:    ret void
-; CHECK:       if.else:
-; CHECK-NEXT:    [[COND2:%.*]] = call i1 @foo()
-; CHECK-NEXT:    br i1 [[COND2]], label [[IF_ELSE_IF_END_CRIT_EDGE]], label [[EXIT:%.*]]
-; CHECK:       if.else.if.end_crit_edge:
-; CHECK-NEXT:    [[V2_PRE]] = load i64, ptr [[P1]], align 8
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[_:%.*]] = call i1 @maybethrow()
-; CHECK-NEXT:    [[V3:%.*]] = load i64, ptr [[P1]], align 8
-; CHECK-NEXT:    store i64 [[V3]], ptr [[P2]], align 8
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test22(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; MDEP:       if.then:
+; MDEP-NEXT:    [[V1:%.*]] = load i64, ptr [[P1:%.*]], align 8
+; MDEP-NEXT:    [[DEC:%.*]] = add i64 [[V1]], -1
+; MDEP-NEXT:    store i64 [[DEC]], ptr [[P1]], align 8
+; MDEP-NEXT:    br label [[IF_END:%.*]]
+; MDEP:       if.end:
+; MDEP-NEXT:    [[V2:%.*]] = phi i64 [ [[V2_PRE:%.*]], [[IF_ELSE_IF_END_CRIT_EDGE:%.*]] ], [ [[DEC]], [[IF_THEN]] ]
+; MDEP-NEXT:    store i64 [[V2]], ptr [[P2:%.*]], align 8
+; MDEP-NEXT:    ret void
+; MDEP:       if.else:
+; MDEP-NEXT:    [[COND2:%.*]] = call i1 @foo()
+; MDEP-NEXT:    br i1 [[COND2]], label [[IF_ELSE_IF_END_CRIT_EDGE]], label [[EXIT:%.*]]
+; MDEP:       if.else.if.end_crit_edge:
+; MDEP-NEXT:    [[V2_PRE]] = load i64, ptr [[P1]], align 8
+; MDEP-NEXT:    br label [[IF_END]]
+; MDEP:       exit:
+; MDEP-NEXT:    [[_:%.*]] = call i1 @maybethrow()
+; MDEP-NEXT:    [[V3:%.*]] = load i64, ptr [[P1]], align 8
+; MDEP-NEXT:    store i64 [[V3]], ptr [[P2]], align 8
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test22(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; MSSA:       if.then:
+; MSSA-NEXT:    [[V1:%.*]] = load i64, ptr [[P1:%.*]], align 8
+; MSSA-NEXT:    [[DEC:%.*]] = add i64 [[V1]], -1
+; MSSA-NEXT:    store i64 [[DEC]], ptr [[P1]], align 8
+; MSSA-NEXT:    br label [[IF_END:%.*]]
+; MSSA:       if.end:
+; MSSA-NEXT:    [[V2:%.*]] = load i64, ptr [[P1]], align 8
+; MSSA-NEXT:    store i64 [[V2]], ptr [[P2:%.*]], align 8
+; MSSA-NEXT:    ret void
+; MSSA:       if.else:
+; MSSA-NEXT:    [[COND2:%.*]] = call i1 @foo()
+; MSSA-NEXT:    br i1 [[COND2]], label [[IF_END]], label [[EXIT:%.*]]
+; MSSA:       exit:
+; MSSA-NEXT:    [[_:%.*]] = call i1 @maybethrow()
+; MSSA-NEXT:    [[V3:%.*]] = load i64, ptr [[P1]], align 8
+; MSSA-NEXT:    store i64 [[V3]], ptr [[P2]], align 8
+; MSSA-NEXT:    ret void
 ;
 entry:
   br i1 %cond, label %if.then, label %if.else
@@ -1106,21 +1450,38 @@ declare void @maybethrow() readnone
 ; also be replaced by ValuesPerBlock(BB, NewLoad). So we'll not use the deleted
 ; OldLoad in later PHI instruction.
 define void @test23(i1 %cond1, i1 %cond2) {
-; CHECK-LABEL: @test23(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[G:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    [[VAL1_PRE:%.*]] = load i64, ptr @B, align 8
-; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[THEN:%.*]], label [[WRONG:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[STORE:%.*]], label [[EXIT:%.*]]
-; CHECK:       store:
-; CHECK-NEXT:    store i64 [[VAL1_PRE]], ptr @B, align 8
-; CHECK-NEXT:    br label [[WRONG]]
-; CHECK:       wrong:
-; CHECK-NEXT:    store i64 [[VAL1_PRE]], ptr [[G]], align 8
-; CHECK-NEXT:    ret void
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; MDEP-LABEL: @test23(
+; MDEP-NEXT:  entry:
+; MDEP-NEXT:    [[G:%.*]] = alloca i64, align 8
+; MDEP-NEXT:    [[VAL1_PRE:%.*]] = load i64, ptr @B, align 8
+; MDEP-NEXT:    br i1 [[COND2:%.*]], label [[THEN:%.*]], label [[WRONG:%.*]]
+; MDEP:       then:
+; MDEP-NEXT:    br i1 [[COND1:%.*]], label [[STORE:%.*]], label [[EXIT:%.*]]
+; MDEP:       store:
+; MDEP-NEXT:    store i64 [[VAL1_PRE]], ptr @B, align 8
+; MDEP-NEXT:    br label [[WRONG]]
+; MDEP:       wrong:
+; MDEP-NEXT:    store i64 [[VAL1_PRE]], ptr [[G]], align 8
+; MDEP-NEXT:    ret void
+; MDEP:       exit:
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: @test23(
+; MSSA-NEXT:  entry:
+; MSSA-NEXT:    [[G:%.*]] = alloca i64, align 8
+; MSSA-NEXT:    br i1 [[COND2:%.*]], label [[THEN:%.*]], label [[WRONG:%.*]]
+; MSSA:       then:
+; MSSA-NEXT:    [[VAL2:%.*]] = load i64, ptr @B, align 8
+; MSSA-NEXT:    br i1 [[COND1:%.*]], label [[STORE:%.*]], label [[EXIT:%.*]]
+; MSSA:       store:
+; MSSA-NEXT:    store i64 [[VAL2]], ptr @B, align 8
+; MSSA-NEXT:    br label [[WRONG]]
+; MSSA:       wrong:
+; MSSA-NEXT:    [[VAL1:%.*]] = load i64, ptr @B, align 8
+; MSSA-NEXT:    store i64 [[VAL1]], ptr [[G]], align 8
+; MSSA-NEXT:    ret void
+; MSSA:       exit:
+; MSSA-NEXT:    ret void
 ;
 entry:
   %G = alloca i64, align 8
diff --git a/llvm/test/Transforms/GVN/PRE/pre-loop-load-new-pm.ll b/llvm/test/Transforms/GVN/PRE/pre-loop-load-new-pm.ll
index e16c21e516eb5..4cd2e47b3c31f 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-loop-load-new-pm.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-loop-load-new-pm.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -aa-pipeline=basic-aa -enable-load-pre -enable-pre -passes=gvn -S < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -enable-load-pre -enable-pre -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt -aa-pipeline=basic-aa -enable-load-pre -enable-pre -passes='gvn<memoryssa>' -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
 declare void @side_effect()
 declare i1 @side_effect_cond()
@@ -216,7 +217,7 @@ define i32 @test_load_on_exiting_cold_path_02(ptr %p) gc "statepoint-example" pe
 ; CHECK-NEXT:    br label [[BACKEDGE]]
 ; CHECK:       cold_path:
 ; CHECK-NEXT:    invoke void @side_effect()
-; CHECK-NEXT:    to label [[BACKEDGE]] unwind label [[COLD_EXIT:%.*]]
+; CHECK-NEXT:            to label [[BACKEDGE]] unwind label [[COLD_EXIT:%.*]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[X]]
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], 1000
@@ -225,7 +226,7 @@ define i32 @test_load_on_exiting_cold_path_02(ptr %p) gc "statepoint-example" pe
 ; CHECK-NEXT:    ret i32 [[X]]
 ; CHECK:       cold_exit:
 ; CHECK-NEXT:    [[LANDING_PAD:%.*]] = landingpad token
-; CHECK-NEXT:    cleanup
+; CHECK-NEXT:            cleanup
 ; CHECK-NEXT:    ret i32 -1
 ;
 entry:
@@ -447,7 +448,7 @@ define i32 @test_inner_loop(ptr %p, i1 %arg) {
 ; CHECK-NEXT:    br label [[INNER_LOOP:%.*]]
 ; CHECK:       inner_loop:
 ; CHECK-NEXT:    call void @side_effect()
-; CHECK-NEXT:    br i1 %arg, label [[INNER_LOOP]], label [[BACKEDGE]]
+; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[INNER_LOOP]], label [[BACKEDGE]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], [[X]]
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp ult i32 [[IV_NEXT]], 1000
@@ -633,3 +634,6 @@ exit:
 cold_exit:
   ret i32 -1
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; MDEP: {{.*}}
+; MSSA: {{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll b/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll
index 2009c29931455..22c628bb35464 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-no-cost-phi.ll
@@ -1,4 +1,6 @@
-; RUN: opt < %s -passes=gvn -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=gvn -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -S | FileCheck %s --check-prefixes=CHECK,MSSA
 ; This testcase tests insertion of no-cost phis.  That is,
 ; when the value is already available in every predecessor,
 ; and we just need to insert a phi node to merge the available values.
@@ -8,6 +10,22 @@
 
 
 define i32 @mai(i32 %foo, i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @mai(
+; CHECK-SAME: i32 [[FOO:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[FOO]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[BB1:.*]], label %[[BB2:.*]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw i32 [[A]], [[B]]
+; CHECK-NEXT:    store i32 [[TMP2]], ptr @c, align 4
+; CHECK-NEXT:    br label %[[MERGEBLOCK:.*]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i32 [[A]], [[B]]
+; CHECK-NEXT:    store i32 [[TMP3]], ptr @d, align 4
+; CHECK-NEXT:    br label %[[MERGEBLOCK]]
+; CHECK:       [[MERGEBLOCK]]:
+; CHECK-NEXT:    [[DOTPRE_PHI:%.*]] = phi i32 [ [[TMP3]], %[[BB2]] ], [ [[TMP2]], %[[BB1]] ]
+; CHECK-NEXT:    ret i32 [[DOTPRE_PHI]]
+;
   %1 = icmp ne i32 %foo, 0
   br i1 %1, label %bb1, label %bb2
 
@@ -22,10 +40,11 @@ bb2:
   br label %mergeblock
 
 mergeblock:
-; CHECK: pre-phi = phi i32 [ %3, %bb2 ], [ %2, %bb1 ]
-; CHECK-NEXT: ret i32 %.pre-phi
   %4 = add nsw i32 %a, %b
   ret i32 %4
 }
 
 
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; MDEP: {{.*}}
+; MSSA: {{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll b/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll
index d17c459f1cff8..32f149b881d72 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-poison-add.ll
@@ -1,52 +1,77 @@
-; RUN: opt < %s -passes=gvn -enable-pre -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=gvn -enable-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -enable-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA
 
 @H = common global i32 0
 @G = common global i32 0
 
 define i32 @test1(i1 %cond, i32 %v) nounwind {
-; CHECK-LABEL: @test1
+; CHECK-LABEL: define i32 @test1(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[BB:.*]], label %[[BB1:.*]]
+; CHECK:       [[BB]]:
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 [[V]], 42
+; CHECK-NEXT:    store i32 [[ADD_1]], ptr @G, align 4
+; CHECK-NEXT:    br label %[[RETURN:.*]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[DOTPRE:%.*]] = add i32 [[V]], 42
+; CHECK-NEXT:    br label %[[RETURN]]
+; CHECK:       [[RETURN]]:
+; CHECK-NEXT:    [[ADD_2_PRE_PHI:%.*]] = phi i32 [ [[DOTPRE]], %[[BB1]] ], [ [[ADD_1]], %[[BB]] ]
+; CHECK-NEXT:    store i32 [[ADD_2_PRE_PHI]], ptr @H, align 4
+; CHECK-NEXT:    ret i32 0
+;
 entry:
-    br i1 %cond, label %bb, label %bb1
+  br i1 %cond, label %bb, label %bb1
 
 bb:
-    %add.1 = add nuw nsw i32 %v, 42
-; CHECK: %add.1 = add i32 %v, 42
-    store i32 %add.1, ptr @G, align 4
-    br label %return
+  %add.1 = add nuw nsw i32 %v, 42
+  store i32 %add.1, ptr @G, align 4
+  br label %return
 
 bb1:
-; CHECK: %.pre = add i32 %v, 42
-    br label %return
+  br label %return
 
 return:
-; CHECK: %add.2.pre-phi = phi i32 [ %.pre, %bb1 ], [ %add.1, %bb ]
-; CHECK-NEXT: store i32 %add.2.pre-phi, ptr @H, align 4
-; CHECK-NEXT: ret i32 0
-    %add.2 = add i32 %v, 42
-    store i32 %add.2, ptr @H, align 4
-    ret i32 0
+  %add.2 = add i32 %v, 42
+  store i32 %add.2, ptr @H, align 4
+  ret i32 0
 }
 
 define i32 @test2(i1 %cond, i32 %v) nounwind {
-; CHECK-LABEL: @test2
+; CHECK-LABEL: define i32 @test2(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[BB:.*]], label %[[BB1:.*]]
+; CHECK:       [[BB]]:
+; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 [[V]], 42
+; CHECK-NEXT:    store i32 [[ADD_1]], ptr @G, align 4
+; CHECK-NEXT:    br label %[[RETURN:.*]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[DOTPRE:%.*]] = add nuw nsw i32 [[V]], 42
+; CHECK-NEXT:    br label %[[RETURN]]
+; CHECK:       [[RETURN]]:
+; CHECK-NEXT:    [[ADD_2_PRE_PHI:%.*]] = phi i32 [ [[DOTPRE]], %[[BB1]] ], [ [[ADD_1]], %[[BB]] ]
+; CHECK-NEXT:    store i32 [[ADD_2_PRE_PHI]], ptr @H, align 4
+; CHECK-NEXT:    ret i32 0
+;
 entry:
-    br i1 %cond, label %bb, label %bb1
+  br i1 %cond, label %bb, label %bb1
 
 bb:
-    %add.1 = add i32 %v, 42
-; CHECK: %add.1 = add i32 %v, 42
-    store i32 %add.1, ptr @G, align 4
-    br label %return
+  %add.1 = add i32 %v, 42
+  store i32 %add.1, ptr @G, align 4
+  br label %return
 
 bb1:
-; CHECK: %.pre = add nuw nsw i32 %v, 42
-    br label %return
+  br label %return
 
 return:
-; CHECK: %add.2.pre-phi = phi i32 [ %.pre, %bb1 ], [ %add.1, %bb ]
-; CHECK-NEXT: store i32 %add.2.pre-phi, ptr @H, align 4
-; CHECK-NEXT: ret i32 0
-    %add.2 = add nuw nsw i32 %v, 42
-    store i32 %add.2, ptr @H, align 4
-    ret i32 0
+  %add.2 = add nuw nsw i32 %v, 42
+  store i32 %add.2, ptr @H, align 4
+  ret i32 0
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; MDEP: {{.*}}
+; MSSA: {{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/pre-single-pred.ll b/llvm/test/Transforms/GVN/PRE/pre-single-pred.ll
index 7342925a0c3be..74bc6bcaecf67 100644
--- a/llvm/test/Transforms/GVN/PRE/pre-single-pred.ll
+++ b/llvm/test/Transforms/GVN/PRE/pre-single-pred.ll
@@ -1,4 +1,6 @@
-; RUN: opt < %s -passes=gvn -enable-load-pre -S | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=gvn -enable-load-pre -S | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt < %s -passes='gvn<memoryssa>' -enable-load-pre -S | FileCheck %s --check-prefixes=CHECK,MSSA
 ; RUN: opt < %s -passes="gvn<load-pre>" -enable-load-pre=false -S | FileCheck %s
 ; This testcase assumed we'll PRE the load into %for.cond, but we don't actually
 ; verify that doing so is safe.  If there didn't _happen_ to be a load in
@@ -12,35 +14,85 @@
 
 @p = external global i32
 define i32 @f(i32 %n) nounwind {
+; MDEP-LABEL: define i32 @f(
+; MDEP-SAME: i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; MDEP-NEXT:  [[ENTRY:.*]]:
+; MDEP-NEXT:    br label %[[FOR_COND:.*]]
+; MDEP:       [[FOR_COND]]:
+; MDEP-NEXT:    [[I_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDVAR_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; MDEP-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N]]
+; MDEP-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_COND_FOR_END_CRIT_EDGE:.*]]
+; MDEP:       [[FOR_COND_FOR_END_CRIT_EDGE]]:
+; MDEP-NEXT:    [[TMP9_PRE:%.*]] = load i32, ptr @p, align 4
+; MDEP-NEXT:    br label %[[FOR_END:.*]]
+; MDEP:       [[FOR_BODY]]:
+; MDEP-NEXT:    [[TMP3:%.*]] = load i32, ptr @p, align 4
+; MDEP-NEXT:    [[DEC:%.*]] = add i32 [[TMP3]], -1
+; MDEP-NEXT:    store i32 [[DEC]], ptr @p, align 4
+; MDEP-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[DEC]], 0
+; MDEP-NEXT:    br i1 [[CMP6]], label %[[FOR_BODY_FOR_END_CRIT_EDGE:.*]], label %[[FOR_INC]]
+; MDEP:       [[FOR_BODY_FOR_END_CRIT_EDGE]]:
+; MDEP-NEXT:    br label %[[FOR_END]]
+; MDEP:       [[FOR_INC]]:
+; MDEP-NEXT:    [[INDVAR_NEXT]] = add i32 [[I_0]], 1
+; MDEP-NEXT:    br label %[[FOR_COND]]
+; MDEP:       [[FOR_END]]:
+; MDEP-NEXT:    [[TMP9:%.*]] = phi i32 [ [[DEC]], %[[FOR_BODY_FOR_END_CRIT_EDGE]] ], [ [[TMP9_PRE]], %[[FOR_COND_FOR_END_CRIT_EDGE]] ]
+; MDEP-NEXT:    ret i32 [[TMP9]]
+;
+; MSSA-LABEL: define i32 @f(
+; MSSA-SAME: i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; MSSA-NEXT:  [[ENTRY:.*]]:
+; MSSA-NEXT:    br label %[[FOR_COND:.*]]
+; MSSA:       [[FOR_COND]]:
+; MSSA-NEXT:    [[I_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INDVAR_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; MSSA-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_0]], [[N]]
+; MSSA-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_COND_FOR_END_CRIT_EDGE:.*]]
+; MSSA:       [[FOR_COND_FOR_END_CRIT_EDGE]]:
+; MSSA-NEXT:    br label %[[FOR_END:.*]]
+; MSSA:       [[FOR_BODY]]:
+; MSSA-NEXT:    [[TMP3:%.*]] = load i32, ptr @p, align 4
+; MSSA-NEXT:    [[DEC:%.*]] = add i32 [[TMP3]], -1
+; MSSA-NEXT:    store i32 [[DEC]], ptr @p, align 4
+; MSSA-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[DEC]], 0
+; MSSA-NEXT:    br i1 [[CMP6]], label %[[FOR_BODY_FOR_END_CRIT_EDGE:.*]], label %[[FOR_INC]]
+; MSSA:       [[FOR_BODY_FOR_END_CRIT_EDGE]]:
+; MSSA-NEXT:    br label %[[FOR_END]]
+; MSSA:       [[FOR_INC]]:
+; MSSA-NEXT:    [[INDVAR_NEXT]] = add i32 [[I_0]], 1
+; MSSA-NEXT:    br label %[[FOR_COND]]
+; MSSA:       [[FOR_END]]:
+; MSSA-NEXT:    [[TMP9:%.*]] = load i32, ptr @p, align 4
+; MSSA-NEXT:    ret i32 [[TMP9]]
+;
 entry:
-	br label %for.cond
+  br label %for.cond
 
 for.cond:		; preds = %for.inc, %entry
-	%i.0 = phi i32 [ 0, %entry ], [ %indvar.next, %for.inc ]		; <i32> [#uses=2]
-	%cmp = icmp slt i32 %i.0, %n		; <i1> [#uses=1]
-	br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
+  %i.0 = phi i32 [ 0, %entry ], [ %indvar.next, %for.inc ]		; <i32> [#uses=2]
+  %cmp = icmp slt i32 %i.0, %n		; <i1> [#uses=1]
+  br i1 %cmp, label %for.body, label %for.cond.for.end_crit_edge
 
 for.cond.for.end_crit_edge:		; preds = %for.cond
-	br label %for.end
+  br label %for.end
 
-; CHECK: for.body:
-; CHECK-NEXT: %tmp3 = load i32, ptr @p
 for.body:		; preds = %for.cond
-	%tmp3 = load i32, ptr @p		; <i32> [#uses=1]
-	%dec = add i32 %tmp3, -1		; <i32> [#uses=2]
-	store i32 %dec, ptr @p
-	%cmp6 = icmp slt i32 %dec, 0		; <i1> [#uses=1]
-	br i1 %cmp6, label %for.body.for.end_crit_edge, label %for.inc
+  %tmp3 = load i32, ptr @p		; <i32> [#uses=1]
+  %dec = add i32 %tmp3, -1		; <i32> [#uses=2]
+  store i32 %dec, ptr @p
+  %cmp6 = icmp slt i32 %dec, 0		; <i1> [#uses=1]
+  br i1 %cmp6, label %for.body.for.end_crit_edge, label %for.inc
 
-; CHECK: for.body.for.end_crit_edge:
 for.body.for.end_crit_edge:		; preds = %for.body
-	br label %for.end
+  br label %for.end
 
 for.inc:		; preds = %for.body
-	%indvar.next = add i32 %i.0, 1		; <i32> [#uses=1]
-	br label %for.cond
+  %indvar.next = add i32 %i.0, 1		; <i32> [#uses=1]
+  br label %for.cond
 
 for.end:		; preds = %for.body.for.end_crit_edge, %for.cond.for.end_crit_edge
-	%tmp9 = load i32, ptr @p		; <i32> [#uses=1]
-	ret i32 %tmp9
+  %tmp9 = load i32, ptr @p		; <i32> [#uses=1]
+  ret i32 %tmp9
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll b/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll
index 3df63beefea68..abbb17f11f436 100644
--- a/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll
+++ b/llvm/test/Transforms/GVN/PRE/preserve-tbaa.ll
@@ -1,13 +1,45 @@
-; RUN: opt -passes=gvn -S < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=gvn -S < %s | FileCheck %s --check-prefixes=CHECK,MDEP
+; RUN: opt -passes='gvn<memoryssa>' -S < %s | FileCheck %s --check-prefixes=CHECK,MSSA
 
 target datalayout = "e-p:64:64:64"
 
 ; GVN should preserve the TBAA tag on loads when doing PRE.
 
-; CHECK-LABEL: @test(
-; CHECK: %tmp33.pre = load i16, ptr %P, align 2, !tbaa !0
-; CHECK: br label %for.body
 define void @test(ptr %P, ptr %Q, i1 %arg) nounwind {
+; MDEP-LABEL: define void @test(
+; MDEP-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; MDEP-NEXT:  [[ENTRY:.*:]]
+; MDEP-NEXT:    br i1 [[ARG]], label %[[BB_NPH:.*]], label %[[FOR_END:.*]]
+; MDEP:       [[BB_NPH]]:
+; MDEP-NEXT:    [[TMP33_PRE:%.*]] = load i16, ptr [[P]], align 2, !tbaa [[TBAA0:![0-9]+]]
+; MDEP-NEXT:    br label %[[FOR_BODY:.*]]
+; MDEP:       [[FOR_BODY]]:
+; MDEP-NEXT:    [[TMP33:%.*]] = phi i16 [ 0, %[[FOR_BODY]] ], [ [[TMP33_PRE]], %[[BB_NPH]] ]
+; MDEP-NEXT:    store i16 [[TMP33]], ptr [[Q]], align 2
+; MDEP-NEXT:    store i16 0, ptr [[P]], align 2, !tbaa [[TBAA0]]
+; MDEP-NEXT:    br i1 false, label %[[FOR_BODY_FOR_END_CRIT_EDGE:.*]], label %[[FOR_BODY]]
+; MDEP:       [[FOR_BODY_FOR_END_CRIT_EDGE]]:
+; MDEP-NEXT:    br label %[[FOR_END]]
+; MDEP:       [[FOR_END]]:
+; MDEP-NEXT:    ret void
+;
+; MSSA-LABEL: define void @test(
+; MSSA-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; MSSA-NEXT:  [[ENTRY:.*:]]
+; MSSA-NEXT:    br i1 [[ARG]], label %[[BB_NPH:.*]], label %[[FOR_END:.*]]
+; MSSA:       [[BB_NPH]]:
+; MSSA-NEXT:    br label %[[FOR_BODY:.*]]
+; MSSA:       [[FOR_BODY]]:
+; MSSA-NEXT:    [[TMP33:%.*]] = load i16, ptr [[P]], align 2, !tbaa [[TBAA0:![0-9]+]]
+; MSSA-NEXT:    store i16 [[TMP33]], ptr [[Q]], align 2
+; MSSA-NEXT:    store i16 0, ptr [[P]], align 2, !tbaa [[TBAA0]]
+; MSSA-NEXT:    br i1 false, label %[[FOR_BODY_FOR_END_CRIT_EDGE:.*]], label %[[FOR_BODY]]
+; MSSA:       [[FOR_BODY_FOR_END_CRIT_EDGE]]:
+; MSSA-NEXT:    br label %[[FOR_END]]
+; MSSA:       [[FOR_END]]:
+; MSSA-NEXT:    ret void
+;
 entry:
   br i1 %arg, label %bb.nph, label %for.end
 
@@ -29,3 +61,16 @@ for.end:                                          ; preds = %for.body, %entry
 !1 = !{!"omnipotent char", !2}
 !2 = !{!"Simple C/C++ TBAA"}
 !3 = !{!"short", !1}
+;.
+; MDEP: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; MDEP: [[META1]] = !{!"short", [[META2:![0-9]+]]}
+; MDEP: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]]}
+; MDEP: [[META3]] = !{!"Simple C/C++ TBAA"}
+;.
+; MSSA: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; MSSA: [[META1]] = !{!"short", [[META2:![0-9]+]]}
+; MSSA: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]]}
+; MSSA: [[META3]] = !{!"Simple C/C++ TBAA"}
+;.
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/GVN/cond_br.ll b/llvm/test/Transforms/GVN/cond_br.ll
index 19166d17a8320..fb84b626c7455 100644
--- a/llvm/test/Transforms/GVN/cond_br.ll
+++ b/llvm/test/Transforms/GVN/cond_br.ll
@@ -53,3 +53,22 @@ if.end:                                           ; preds = %if.else, %if.then
 }
 
 declare void @bar(i32)
+
+define void @indirectbr_could_not_split() {
+; CHECK-LABEL: define void @indirectbr_could_not_split() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[IBR:.*]], label %[[EXIT:.*]]
+; CHECK:       [[IBR]]:
+; CHECK-NEXT:    indirectbr ptr null, [label %[[EXIT]], label %exit]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 false, label %ibr, label %exit
+
+ibr:
+  indirectbr ptr null, [label %exit, label %exit]
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/GVN/cond_br2.ll b/llvm/test/Transforms/GVN/cond_br2.ll
index 420246713882e..ff80328ae6b48 100644
--- a/llvm/test/Transforms/GVN/cond_br2.ll
+++ b/llvm/test/Transforms/GVN/cond_br2.ll
@@ -17,7 +17,7 @@ define void @_Z4testv() #0 personality ptr @__gxx_personality_v0 {
 
 entry:
   %sv = alloca %"class.llvm::SmallVector", align 16
-  call void @llvm.lifetime.start.p0(i64 64, ptr %sv) #1
+  call void @llvm.lifetime.start.p0(ptr %sv) #1
   %FirstEl.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", ptr %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 3
   store ptr %FirstEl.i.i.i.i.i.i, ptr %sv, align 16, !tbaa !4
   %EndX.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", ptr %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 1
@@ -86,7 +86,7 @@ if.then.i.i.i20:                                  ; preds = %invoke.cont3
   br label %_ZN4llvm11SmallVectorIiLj8EED1Ev.exit21
 
 _ZN4llvm11SmallVectorIiLj8EED1Ev.exit21:          ; preds = %invoke.cont3, %if.then.i.i.i20
-  call void @llvm.lifetime.end.p0(i64 64, ptr %sv) #1
+  call void @llvm.lifetime.end.p0(ptr %sv) #1
   ret void
 
 lpad:                                             ; preds = %if.end.i14, %if.end.i, %invoke.cont2
@@ -105,14 +105,14 @@ eh.resume:                                        ; preds = %if.then.i.i.i, %lpa
 }
 
 ; Function Attrs: nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare i32 @__gxx_personality_v0(...)
 
 declare void @_Z1gRN4llvm11SmallVectorIiLj8EEE(ptr) #2
 
 ; Function Attrs: nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 declare void @_ZN4llvm15SmallVectorBase8grow_podEmm(ptr, i64, i64) #2
 
diff --git a/llvm/test/Transforms/GVN/lifetime-simple.ll b/llvm/test/Transforms/GVN/lifetime-simple.ll
index 89ca127a47fda..bd35052680cef 100644
--- a/llvm/test/Transforms/GVN/lifetime-simple.ll
+++ b/llvm/test/Transforms/GVN/lifetime-simple.ll
@@ -6,18 +6,18 @@ define i8 @test() nounwind {
 ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[P:%.*]] = alloca [32 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NEXT:    store i8 1, ptr [[P]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[P]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[P]], align 1
 ; CHECK-NEXT:    ret i8 [[TMP0]]
 ;
 entry:
   %P = alloca [32 x i8]
-  call void @llvm.lifetime.start.p0(i64 32, ptr %P)
+  call void @llvm.lifetime.start.p0(ptr %P)
   %0 = load i8, ptr %P
   store i8 1, ptr %P
-  call void @llvm.lifetime.end.p0(i64 32, ptr %P)
+  call void @llvm.lifetime.end.p0(ptr %P)
   %1 = load i8, ptr %P
   ret i8 %1
 }
@@ -28,17 +28,17 @@ define void @assume_eq_arg(ptr %arg) {
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[ALLOCA]], [[ARG]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ALLOCA]])
 ; CHECK-NEXT:    store volatile i32 0, ptr [[ALLOCA]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ALLOCA]])
 ; CHECK-NEXT:    ret void
 ;
   %alloca = alloca i32
   %cmp = icmp eq ptr %alloca, %arg
   call void @llvm.assume(i1 %cmp)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %alloca)
+  call void @llvm.lifetime.start.p0(ptr %alloca)
   store volatile i32 0, ptr %alloca
-  call void @llvm.lifetime.end.p0(i64 4, ptr %alloca)
+  call void @llvm.lifetime.end.p0(ptr %alloca)
   ret void
 }
 
@@ -47,17 +47,17 @@ define void @assume_eq_null() {
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(1)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr addrspace(1) [[ALLOCA]], null
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p1(i64 4, ptr addrspace(1) [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p1(ptr addrspace(1) [[ALLOCA]])
 ; CHECK-NEXT:    store volatile i32 0, ptr addrspace(1) null, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p1(i64 4, ptr addrspace(1) [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p1(ptr addrspace(1) [[ALLOCA]])
 ; CHECK-NEXT:    ret void
 ;
   %alloca = alloca i32, addrspace(1)
   %cmp = icmp eq ptr addrspace(1) %alloca, null
   call void @llvm.assume(i1 %cmp)
-  call void @llvm.lifetime.start.p1(i64 4, ptr addrspace(1) %alloca)
+  call void @llvm.lifetime.start.p1(ptr addrspace(1) %alloca)
   store volatile i32 0, ptr addrspace(1) %alloca
-  call void @llvm.lifetime.end.p1(i64 4, ptr addrspace(1) %alloca)
+  call void @llvm.lifetime.end.p1(ptr addrspace(1) %alloca)
   ret void
 }
 
@@ -67,9 +67,9 @@ define void @dom_eq_null() {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr addrspace(1) [[ALLOCA]], null
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[IF:.*]], label %[[ELSE:.*]]
 ; CHECK:       [[IF]]:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p1(i64 4, ptr addrspace(1) [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p1(ptr addrspace(1) [[ALLOCA]])
 ; CHECK-NEXT:    store volatile i32 0, ptr addrspace(1) null, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p1(i64 4, ptr addrspace(1) [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p1(ptr addrspace(1) [[ALLOCA]])
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    ret void
@@ -79,14 +79,14 @@ define void @dom_eq_null() {
   br i1 %cmp, label %if, label %else
 
 if:
-  call void @llvm.lifetime.start.p1(i64 4, ptr addrspace(1) %alloca)
+  call void @llvm.lifetime.start.p1(ptr addrspace(1) %alloca)
   store volatile i32 0, ptr addrspace(1) %alloca
-  call void @llvm.lifetime.end.p1(i64 4, ptr addrspace(1) %alloca)
+  call void @llvm.lifetime.end.p1(ptr addrspace(1) %alloca)
   ret void
 
 else:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 %S, ptr nocapture %P) readonly
-declare void @llvm.lifetime.end.p0(i64 %S, ptr nocapture %P)
+declare void @llvm.lifetime.start.p0(ptr nocapture %P) readonly
+declare void @llvm.lifetime.end.p0(ptr nocapture %P)
diff --git a/llvm/test/Transforms/GVN/opt-remarks.ll b/llvm/test/Transforms/GVN/opt-remarks.ll
index 87cd54dd566b9..a5c3cb5c55925 100644
--- a/llvm/test/Transforms/GVN/opt-remarks.ll
+++ b/llvm/test/Transforms/GVN/opt-remarks.ll
@@ -109,9 +109,9 @@ entry:
 
 define i8 @lifetime_end(i8 %val) {
   %p = alloca [32 x i8]
-  call void @llvm.lifetime.start.p0(i64 32, ptr %p)
+  call void @llvm.lifetime.start.p0(ptr %p)
   store i8 %val, ptr %p
-  call void @llvm.lifetime.end.p0(i64 32, ptr %p)
+  call void @llvm.lifetime.end.p0(ptr %p)
   %1 = load i8, ptr %p
   ret i8 %1
 }
diff --git a/llvm/test/Transforms/GVN/vscale.ll b/llvm/test/Transforms/GVN/vscale.ll
index 5d6c55990a855..b358df54750d8 100644
--- a/llvm/test/Transforms/GVN/vscale.ll
+++ b/llvm/test/Transforms/GVN/vscale.ll
@@ -696,7 +696,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
 ; MDEP-LABEL: @bigexample(
 ; MDEP-NEXT:  entry:
 ; MDEP-NEXT:    [[REF_TMP:%.*]] = alloca { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }, align 16
-; MDEP-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr nonnull [[REF_TMP]])
+; MDEP-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[REF_TMP]])
 ; MDEP-NEXT:    [[A_ELT:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A:%.*]], 0
 ; MDEP-NEXT:    store <vscale x 4 x i32> [[A_ELT]], ptr [[REF_TMP]], align 16
 ; MDEP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
@@ -720,13 +720,13 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
 ; MDEP-NEXT:    [[TMP9:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP7]], <vscale x 16 x i8> [[TMP8]], 2
 ; MDEP-NEXT:    [[TMP10:%.*]] = bitcast <vscale x 4 x i32> [[A_ELT6]] to <vscale x 16 x i8>
 ; MDEP-NEXT:    [[TMP11:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP9]], <vscale x 16 x i8> [[TMP10]], 3
-; MDEP-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr nonnull [[REF_TMP]])
+; MDEP-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[REF_TMP]])
 ; MDEP-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP11]]
 ;
 ; MSSA-LABEL: @bigexample(
 ; MSSA-NEXT:  entry:
 ; MSSA-NEXT:    [[REF_TMP:%.*]] = alloca { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }, align 16
-; MSSA-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr nonnull [[REF_TMP]])
+; MSSA-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[REF_TMP]])
 ; MSSA-NEXT:    [[A_ELT:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A:%.*]], 0
 ; MSSA-NEXT:    store <vscale x 4 x i32> [[A_ELT]], ptr [[REF_TMP]], align 16
 ; MSSA-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
@@ -750,12 +750,12 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
 ; MSSA-NEXT:    [[TMP6:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP5]], <vscale x 16 x i8> [[DOTUNPACK10]], 2
 ; MSSA-NEXT:    [[DOTUNPACK12:%.*]] = load <vscale x 16 x i8>, ptr [[REF_TMP_REPACK5]], align 16
 ; MSSA-NEXT:    [[TMP7:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP6]], <vscale x 16 x i8> [[DOTUNPACK12]], 3
-; MSSA-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr nonnull [[REF_TMP]])
+; MSSA-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[REF_TMP]])
 ; MSSA-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP7]]
 ;
 entry:
   %ref.tmp = alloca { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }, align 16
-  call void @llvm.lifetime.start.p0(i64 -1, ptr nonnull %ref.tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %ref.tmp)
   %a.elt = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %a, 0
   store <vscale x 4 x i32> %a.elt, ptr %ref.tmp, align 16
   %0 = call i64 @llvm.vscale.i64()
@@ -790,7 +790,7 @@ entry:
   %.elt11 = getelementptr inbounds i8, ptr %ref.tmp, i64 %14
   %.unpack12 = load <vscale x 16 x i8>, ptr %.elt11, align 16
   %15 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %12, <vscale x 16 x i8> %.unpack12, 3
-  call void @llvm.lifetime.end.p0(i64 -1, ptr nonnull %ref.tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %ref.tmp)
   ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %15
 }
 
diff --git a/llvm/test/Transforms/GVNHoist/pr29034.ll b/llvm/test/Transforms/GVNHoist/pr29034.ll
index f5378eacac3c6..a5294c52a87bd 100644
--- a/llvm/test/Transforms/GVNHoist/pr29034.ll
+++ b/llvm/test/Transforms/GVNHoist/pr29034.ll
@@ -37,7 +37,7 @@
 define void @music_task(ptr nocapture readnone %p) local_unnamed_addr {
 entry:
   %mapi = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %mapi)
+  call void @llvm.lifetime.start.p0(ptr %mapi)
   store ptr null, ptr %mapi, align 8, !tbaa !1
   %call = call i32 @music_decoder_init(ptr nonnull %mapi)
   br label %while.cond
@@ -99,7 +99,7 @@ while.cond2.backedge:                             ; preds = %sw.default, %sw.bb1
   br label %while.cond2
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 declare i32 @music_decoder_init(ptr)
 declare i32 @music_play_api(ptr, i32, i32, i32, ptr)
 declare i32 @printf(ptr nocapture readonly, ...)
diff --git a/llvm/test/Transforms/GVNSink/lifetime.ll b/llvm/test/Transforms/GVNSink/lifetime.ll
index 1a8a69bb0986e..f8731e54bb846 100644
--- a/llvm/test/Transforms/GVNSink/lifetime.ll
+++ b/llvm/test/Transforms/GVNSink/lifetime.ll
@@ -9,34 +9,34 @@ define void @test_cant_sink(i1 %c) {
 ; CHECK-SAME: i1 [[C:%.*]]) {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[B:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B]])
 ; CHECK-NEXT:    br i1 [[C]], label %[[IF:.*]], label %[[ELSE:.*]]
 ; CHECK:       [[IF]]:
 ; CHECK-NEXT:    store i64 1, ptr [[A]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    br label %[[JOIN:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    store i64 1, ptr [[B]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B]])
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i8
   %b = alloca i8
-  call void @llvm.lifetime.start(i64 1, ptr %a)
-  call void @llvm.lifetime.start(i64 1, ptr %b)
+  call void @llvm.lifetime.start(ptr %a)
+  call void @llvm.lifetime.start(ptr %b)
   br i1 %c, label %if, label %else
 
 if:
   store i64 1, ptr %a
-  call void @llvm.lifetime.end(i64 1, ptr %a)
+  call void @llvm.lifetime.end(ptr %a)
   br label %join
 
 else:
   store i64 1, ptr %b
-  call void @llvm.lifetime.end(i64 1, ptr %b)
+  call void @llvm.lifetime.end(ptr %b)
   br label %join
 
 join:
@@ -47,7 +47,7 @@ define void @test_can_sink(i1 %c) {
 ; CHECK-LABEL: define void @test_can_sink(
 ; CHECK-SAME: i1 [[C:%.*]]) {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    br i1 [[C]], label %[[IF:.*]], label %[[ELSE:.*]]
 ; CHECK:       [[IF]]:
 ; CHECK-NEXT:    br label %[[JOIN:.*]]
@@ -55,21 +55,21 @@ define void @test_can_sink(i1 %c) {
 ; CHECK-NEXT:    br label %[[JOIN]]
 ; CHECK:       [[JOIN]]:
 ; CHECK-NEXT:    store i64 1, ptr [[A]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i8
-  call void @llvm.lifetime.start(i64 1, ptr %a)
+  call void @llvm.lifetime.start(ptr %a)
   br i1 %c, label %if, label %else
 
 if:
   store i64 1, ptr %a
-  call void @llvm.lifetime.end(i64 1, ptr %a)
+  call void @llvm.lifetime.end(ptr %a)
   br label %join
 
 else:
   store i64 1, ptr %a
-  call void @llvm.lifetime.end(i64 1, ptr %a)
+  call void @llvm.lifetime.end(ptr %a)
   br label %join
 
 join:
diff --git a/llvm/test/Transforms/GlobalOpt/dead-store-status.ll b/llvm/test/Transforms/GlobalOpt/dead-store-status.ll
index 9a8fbb8d65f0e..7cb3a9626ec05 100644
--- a/llvm/test/Transforms/GlobalOpt/dead-store-status.ll
+++ b/llvm/test/Transforms/GlobalOpt/dead-store-status.ll
@@ -24,17 +24,17 @@ entry:
 define i16 @bar() local_unnamed_addr #1 {
 entry:
   %local2 = alloca [1 x i16], align 1
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %local2)
+  call void @llvm.lifetime.start.p0(ptr nonnull %local2)
   store ptr %local2, ptr @global, align 1
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %local2)
+  call void @llvm.lifetime.end.p0(ptr nonnull %local2)
   ret i16 undef
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 attributes #0 = { nofree noinline norecurse nounwind writeonly }
 attributes #1 = { noinline nounwind writeonly }
diff --git a/llvm/test/Transforms/GlobalOpt/stored-once-addrspacecast.ll b/llvm/test/Transforms/GlobalOpt/stored-once-addrspacecast.ll
new file mode 100644
index 0000000000000..35678598032dc
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/stored-once-addrspacecast.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
+; RUN: opt -passes=globalopt < %s -S | FileCheck %s
+
+; Test that we do not fold away addresscasts when optimizing once-stored
+; globals, as these may be runtime operations.
+
+@g1 = internal global ptr null
+@g2 = addrspace(1) global i32 0
+
+;.
+; CHECK: @g1 = internal unnamed_addr global ptr null
+; CHECK: @g2 = addrspace(1) global i32 0
+; CHECK: @g4 = local_unnamed_addr addrspace(1) global i32 0
+;.
+define i64 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr addrspace(1) @g2 to ptr
+; CHECK-NEXT:    store ptr [[TMP1]], ptr @g1, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr @g1, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
+; CHECK-NEXT:    ret i64 [[TMP3]]
+;
+  %l1 = addrspacecast ptr addrspace(1) @g2 to ptr
+  store ptr %l1, ptr @g1, align 8
+  %l2 = load ptr, ptr @g1, align 8
+  %l3 = load i64, ptr %l2, align 8
+  ret i64 %l3
+}
+
+@g3 = internal global ptr null
+@g4 = addrspace(1) global i32 0
+
+define i64 @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr addrspacecast (ptr addrspace(1) @g4 to ptr), align 8
+; CHECK-NEXT:    ret i64 [[TMP1]]
+;
+  store ptr addrspacecast (ptr addrspace(1) @g4 to ptr), ptr @g3, align 8
+  %l1 = load ptr, ptr @g3, align 8
+  %l2 = load i64, ptr %l1, align 8
+  ret i64 %l2
+}
diff --git a/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll
index e5bab0c02222a..28782d5f802d0 100644
--- a/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll
+++ b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll
@@ -1,8 +1,8 @@
 ; RUN: opt -S -passes=hotcoldsplit -hotcoldsplit-threshold=0 < %s 2>&1 | FileCheck %s
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @use(ptr)
 
@@ -18,17 +18,17 @@ entry:
 normalPath:
   ; These two uses of stack slots are non-overlapping. Based on this alone,
   ; the stack slots could be merged.
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local1)
+  call void @llvm.lifetime.start.p0(ptr %local1)
   call void @use(ptr %local1)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local1)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local2)
+  call void @llvm.lifetime.end.p0(ptr %local1)
+  call void @llvm.lifetime.start.p0(ptr %local2)
   call void @use(ptr %local2)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local2)
+  call void @llvm.lifetime.end.p0(ptr %local2)
   ret void
 
 ; CHECK-LABEL: codeRepl:
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 -1, ptr %local1)
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 -1, ptr %local2)
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr %local1)
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr %local2)
 ; CHECK-NEXT: call i1 @foo.cold.1(ptr %local1, ptr %local2)
 ; CHECK-NEXT: br i1
 
@@ -36,19 +36,19 @@ outlinedPath:
   ; These two uses of stack slots are overlapping. This should prevent
   ; merging of stack slots. CodeExtractor must replicate the effects of
   ; these markers in the caller to inhibit stack coloring.
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local1)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local2)
+  call void @llvm.lifetime.start.p0(ptr %local1)
+  call void @llvm.lifetime.start.p0(ptr %local2)
   call void @cold_use2(ptr %local1, ptr %local2)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local1)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local2)
+  call void @llvm.lifetime.end.p0(ptr %local1)
+  call void @llvm.lifetime.end.p0(ptr %local2)
   br i1 undef, label %outlinedPath2, label %outlinedPathExit
 
 outlinedPath2:
   ; These extra lifetime markers are used to test that we emit only one
   ; pair of guard markers in the caller per memory object.
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local2)
+  call void @llvm.lifetime.start.p0(ptr %local2)
   call void @use(ptr %local2)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local2)
+  call void @llvm.lifetime.end.p0(ptr %local2)
   ret void
 
 outlinedPathExit:
diff --git a/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll
index e42db78c195bc..da7a9b8d7531b 100644
--- a/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll
+++ b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=hotcoldsplit -hotcoldsplit-threshold=0 < %s 2>&1 | FileCheck %s
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @cold_use(ptr) cold
 
@@ -40,13 +40,13 @@ define void @only_lifetime_start_is_cold(i1 %arg) {
 ; CHECK-NEXT:    [[LOCAL1:%.*]] = alloca i256, align 8
 ; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[CODEREPL:%.*]], label [[NO_EXTRACT1:%.*]]
 ; CHECK:       codeRepl:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[LOCAL1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[LOCAL1]])
 ; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @only_lifetime_start_is_cold.cold.1(ptr [[LOCAL1]], i1 [[ARG]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[NO_EXTRACT1]], label [[EXIT:%.*]]
 ; CHECK:       no-extract1:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[LOCAL1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[LOCAL1]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -55,7 +55,7 @@ entry:
 
 extract1:
   ; lt.start
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local1)
+  call void @llvm.lifetime.start.p0(ptr %local1)
   call void @cold_use(ptr %local1)
   br i1 %arg, label %extract2, label %no-extract1
 
@@ -67,7 +67,7 @@ no-extract1:
 
 exit:
   ; lt.end
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local1)
+  call void @llvm.lifetime.end.p0(ptr %local1)
   ret void
 }
 
@@ -96,10 +96,10 @@ define void @only_lifetime_end_is_cold(i1 %arg) {
 ; CHECK-LABEL: @only_lifetime_end_is_cold(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LOCAL1:%.*]] = alloca i256, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[LOCAL1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[LOCAL1]])
 ; CHECK-NEXT:    br i1 [[ARG:%.*]], label [[NO_EXTRACT1:%.*]], label [[CODEREPL:%.*]]
 ; CHECK:       no-extract1:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[LOCAL1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[LOCAL1]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       codeRepl:
 ; CHECK-NEXT:    call void @only_lifetime_end_is_cold.cold.1(ptr [[LOCAL1]]) #[[ATTR3]]
@@ -110,18 +110,18 @@ define void @only_lifetime_end_is_cold(i1 %arg) {
 entry:
   ; lt.start
   %local1 = alloca i256
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local1)
+  call void @llvm.lifetime.start.p0(ptr %local1)
   br i1 %arg, label %no-extract1, label %extract1
 
 no-extract1:
   ; lt.end
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local1)
+  call void @llvm.lifetime.end.p0(ptr %local1)
   br label %exit
 
 extract1:
   ; lt.end
   call void @cold_use(ptr %local1)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local1)
+  call void @llvm.lifetime.end.p0(ptr %local1)
   br label %exit
 
 exit:
@@ -134,7 +134,7 @@ define void @do_not_lift_lifetime_end(i1 %arg) {
 ; CHECK-LABEL: @do_not_lift_lifetime_end(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LOCAL1:%.*]] = alloca i256, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[LOCAL1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[LOCAL1]])
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
 ; CHECK-NEXT:    call void @use(ptr [[LOCAL1]])
@@ -148,7 +148,7 @@ define void @do_not_lift_lifetime_end(i1 %arg) {
 entry:
   ; lt.start
   %local1 = alloca i256
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local1)
+  call void @llvm.lifetime.start.p0(ptr %local1)
   br label %header
 
 header:
@@ -167,7 +167,7 @@ extract2:
 
 extract3:
   ; lt.end
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local1)
+  call void @llvm.lifetime.end.p0(ptr %local1)
   br label %exit
 
 exit:
diff --git a/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-3.ll b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-3.ll
index 26faaa326141f..b453c61b9aa88 100644
--- a/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-3.ll
+++ b/llvm/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-3.ll
@@ -3,9 +3,9 @@
 %type1 = type opaque
 %type2 = type opaque
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @use(ptr, ptr)
 
@@ -23,16 +23,16 @@ normalPath:
   ret void
 
 ; CHECK-LABEL: codeRepl:
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 -1, ptr %local1)
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 -1, ptr %local2)
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr %local1)
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr %local2)
 ; CHECK-NEXT: call void @foo.cold.1(ptr %local1, ptr %local2
 
 outlinedPath:
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local1)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %local2)
+  call void @llvm.lifetime.start.p0(ptr %local1)
+  call void @llvm.lifetime.start.p0(ptr %local2)
   call void @use2(ptr %local1, ptr %local2)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local1)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %local2)
+  call void @llvm.lifetime.end.p0(ptr %local1)
+  call void @llvm.lifetime.end.p0(ptr %local2)
   br label %outlinedPathExit
 
 outlinedPathExit:
diff --git a/llvm/test/Transforms/HotColdSplit/sink-multiple-bitcasts-of-allocas-pr42451.ll b/llvm/test/Transforms/HotColdSplit/sink-multiple-bitcasts-of-allocas-pr42451.ll
index df7cb3c0874b0..80249fc64fb96 100644
--- a/llvm/test/Transforms/HotColdSplit/sink-multiple-bitcasts-of-allocas-pr42451.ll
+++ b/llvm/test/Transforms/HotColdSplit/sink-multiple-bitcasts-of-allocas-pr42451.ll
@@ -6,8 +6,8 @@ target triple = "x86_64-apple-macosx10.14.0"
 @c = common global i32 0, align 4
 @h = common global i32 0, align 4
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
+declare void @llvm.lifetime.end.p0(ptr nocapture) #0
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #1
 
 declare ptr @m()
@@ -27,15 +27,15 @@ bb:
 
 bb3:                                              ; preds = %bb
   %i4 = call ptr @m()
-  call void @llvm.lifetime.start.p0(i64 20, ptr %.sroa.4.i)
-  call void @llvm.lifetime.start.p0(i64 6, ptr %.sroa.5.i)
+  call void @llvm.lifetime.start.p0(ptr %.sroa.4.i)
+  call void @llvm.lifetime.start.p0(ptr %.sroa.5.i)
   call void @llvm.memset.p0.i64(ptr align 2 %.sroa.4.i, i8 0, i64 20, i1 false)
   call void @llvm.memset.p0.i64(ptr align 8 %.sroa.5.i, i8 0, i64 6, i1 false)
   %i5 = load i32, ptr @c, align 4, !tbaa !4
   %i6 = trunc i32 %i5 to i16
-  call void @llvm.lifetime.end.p0(i64 20, ptr %.sroa.4.i)
-  call void @llvm.lifetime.end.p0(i64 6, ptr %.sroa.5.i)
-  call void @llvm.lifetime.start.p0(i64 6, ptr %.sroa.5.i)
+  call void @llvm.lifetime.end.p0(ptr %.sroa.4.i)
+  call void @llvm.lifetime.end.p0(ptr %.sroa.5.i)
+  call void @llvm.lifetime.start.p0(ptr %.sroa.5.i)
   call void @llvm.memset.p0.i64(ptr align 1 %.sroa.5.i, i8 3, i64 6, i1 false)
   br label %bb7
 
@@ -47,7 +47,7 @@ bb7:                                              ; preds = %bb7, %bb3
   br i1 %i10, label %bb7, label %l.exit
 
 l.exit:                                           ; preds = %bb7
-  call void @llvm.lifetime.end.p0(i64 6, ptr %.sroa.5.i)
+  call void @llvm.lifetime.end.p0(ptr %.sroa.5.i)
   br label %bb11
 
 bb11:                                             ; preds = %l.exit, %bb
diff --git a/llvm/test/Transforms/IRNormalizer/regression-convergence-tokens.ll b/llvm/test/Transforms/IRNormalizer/regression-convergence-tokens.ll
index 88eff971b9576..0c2db4a2862b2 100644
--- a/llvm/test/Transforms/IRNormalizer/regression-convergence-tokens.ll
+++ b/llvm/test/Transforms/IRNormalizer/regression-convergence-tokens.ll
@@ -7,9 +7,9 @@ define i32 @nested(i32 %src) #0 {
 ; CHECK-SAME: i32 [[A0:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[BB15160:.*:]]
 ; CHECK-NEXT:    [[T1:%.*]] = call token @llvm.experimental.convergence.entry()
-; CHECK-NEXT:    %"vl77672llvm.experimental.convergence.anchor()" = call token @llvm.experimental.convergence.anchor()
-; CHECK-NEXT:    %"op68297(vl77672)" = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[A0]]) [ "convergencectrl"(token %"vl77672llvm.experimental.convergence.anchor()") ]
-; CHECK-NEXT:    ret i32 %"op68297(vl77672)"
+; CHECK-NEXT:    %"vl14659llvm.experimental.convergence.anchor()" = call token @llvm.experimental.convergence.anchor()
+; CHECK-NEXT:    %"op15516(vl14659)" = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[A0]]) [ "convergencectrl"(token %"vl14659llvm.experimental.convergence.anchor()") ]
+; CHECK-NEXT:    ret i32 %"op15516(vl14659)"
 ;
   %t1 = call token @llvm.experimental.convergence.entry()
   %t2 = call token @llvm.experimental.convergence.anchor()
diff --git a/llvm/test/Transforms/IRNormalizer/regression-infinite-loop.ll b/llvm/test/Transforms/IRNormalizer/regression-infinite-loop.ll
index 35ac0fd8c329a..b9be105bcda48 100644
--- a/llvm/test/Transforms/IRNormalizer/regression-infinite-loop.ll
+++ b/llvm/test/Transforms/IRNormalizer/regression-infinite-loop.ll
@@ -8,18 +8,18 @@ define void @test(ptr, i32) {
 ; CHECK-NEXT:    %"vl72693([[A1]], 1)" = add i32 [[A1]], 1
 ; CHECK-NEXT:    br label %[[BB16110:.*]]
 ; CHECK:       [[BB16110]]:
-; CHECK-NEXT:    %"op10912(op18080, vl72693)" = phi i32 [ %"op18080(op10412, op17645)", %[[BB16110]] ], [ %"vl72693([[A1]], 1)", %[[BB76951]] ]
-; CHECK-NEXT:    %"op10912(op17645, vl72693)" = phi i32 [ %"op17645(op10912)70", %[[BB16110]] ], [ %"vl72693([[A1]], 1)", %[[BB76951]] ]
-; CHECK-NEXT:    %"op15084(op10912)" = mul i32 %"op10912(op18080, vl72693)", undef
-; CHECK-NEXT:    %"op16562(op15084)" = xor i32 -1, %"op15084(op10912)"
-; CHECK-NEXT:    %"op44627(op10912, op16562)" = add i32 %"op10912(op18080, vl72693)", %"op16562(op15084)"
-; CHECK-NEXT:    %"op17645(op10912)" = add i32 -1, %"op10912(op17645, vl72693)"
-; CHECK-NEXT:    %"op18080(op17645, op44627)" = add i32 %"op17645(op10912)", %"op44627(op10912, op16562)"
-; CHECK-NEXT:    %"op17720(op15084, op18080)" = mul i32 %"op15084(op10912)", %"op18080(op17645, op44627)"
-; CHECK-NEXT:    %"op16562(op17720)" = xor i32 -1, %"op17720(op15084, op18080)"
-; CHECK-NEXT:    %"op17430(op16562, op18080)" = add i32 %"op16562(op17720)", %"op18080(op17645, op44627)"
+; CHECK-NEXT:    %"op81283(op18080, vl72693)" = phi i32 [ %"op18080(op10412, op18131)", %[[BB16110]] ], [ %"vl72693([[A1]], 1)", %[[BB76951]] ]
+; CHECK-NEXT:    %"op81283(op18131, vl72693)" = phi i32 [ %"op18131(op81283)70", %[[BB16110]] ], [ %"vl72693([[A1]], 1)", %[[BB76951]] ]
+; CHECK-NEXT:    %"op13219(op81283)" = mul i32 %"op81283(op18080, vl72693)", undef
+; CHECK-NEXT:    %"op16562(op13219)" = xor i32 -1, %"op13219(op81283)"
+; CHECK-NEXT:    %"op12556(op16562, op81283)" = add i32 %"op16562(op13219)", %"op81283(op18080, vl72693)"
+; CHECK-NEXT:    %"op18131(op81283)" = add i32 -1, %"op81283(op18131, vl72693)"
+; CHECK-NEXT:    %"op18080(op12556, op18131)" = add i32 %"op12556(op16562, op81283)", %"op18131(op81283)"
+; CHECK-NEXT:    %"op17720(op13219, op18080)" = mul i32 %"op13219(op81283)", %"op18080(op12556, op18131)"
+; CHECK-NEXT:    %"op16562(op17720)" = xor i32 -1, %"op17720(op13219, op18080)"
+; CHECK-NEXT:    %"op17430(op16562, op18080)" = add i32 %"op16562(op17720)", %"op18080(op12556, op18131)"
 ; CHECK-NEXT:    %"op10412(op17430)" = add i32 %"op17430(op16562, op18080)", undef
-; CHECK-NEXT:    %"op17720(op10412, op17720)" = mul i32 %"op10412(op17430)", %"op17720(op15084, op18080)"
+; CHECK-NEXT:    %"op17720(op10412, op17720)" = mul i32 %"op10412(op17430)", %"op17720(op13219, op18080)"
 ; CHECK-NEXT:    %"op16562(op17720)1" = xor i32 -1, %"op17720(op10412, op17720)"
 ; CHECK-NEXT:    %"op17430(op10412, op16562)" = add i32 %"op10412(op17430)", %"op16562(op17720)1"
 ; CHECK-NEXT:    %"op10412(op17430)2" = add i32 %"op17430(op10412, op16562)", undef
@@ -45,11 +45,11 @@ define void @test(ptr, i32) {
 ; CHECK-NEXT:    %"op17720(op10412, op17720)21" = mul i32 %"op10412(op17430)20", %"op17720(op10412, op17720)17"
 ; CHECK-NEXT:    %"op16562(op17720)22" = xor i32 -1, %"op17720(op10412, op17720)21"
 ; CHECK-NEXT:    %"op17430(op10412, op16562)23" = add i32 %"op10412(op17430)20", %"op16562(op17720)22"
-; CHECK-NEXT:    %"op17645(op10912)24" = add i32 -9, %"op10912(op17645, vl72693)"
-; CHECK-NEXT:    %"op18080(op17430, op17645)" = add i32 %"op17430(op10412, op16562)23", %"op17645(op10912)24"
-; CHECK-NEXT:    %"op17720(op17720, op18080)" = mul i32 %"op17720(op10412, op17720)21", %"op18080(op17430, op17645)"
+; CHECK-NEXT:    %"op18131(op81283)24" = add i32 -9, %"op81283(op18131, vl72693)"
+; CHECK-NEXT:    %"op18080(op17430, op18131)" = add i32 %"op17430(op10412, op16562)23", %"op18131(op81283)24"
+; CHECK-NEXT:    %"op17720(op17720, op18080)" = mul i32 %"op17720(op10412, op17720)21", %"op18080(op17430, op18131)"
 ; CHECK-NEXT:    %"op16562(op17720)25" = xor i32 -1, %"op17720(op17720, op18080)"
-; CHECK-NEXT:    %"op17430(op16562, op18080)26" = add i32 %"op16562(op17720)25", %"op18080(op17430, op17645)"
+; CHECK-NEXT:    %"op17430(op16562, op18080)26" = add i32 %"op16562(op17720)25", %"op18080(op17430, op18131)"
 ; CHECK-NEXT:    %"op10412(op17430)27" = add i32 %"op17430(op16562, op18080)26", undef
 ; CHECK-NEXT:    %"op17720(op10412, op17720)28" = mul i32 %"op10412(op17430)27", %"op17720(op17720, op18080)"
 ; CHECK-NEXT:    %"op16562(op17720)29" = xor i32 -1, %"op17720(op10412, op17720)28"
@@ -66,11 +66,11 @@ define void @test(ptr, i32) {
 ; CHECK-NEXT:    %"op17720(op10412, op17720)40" = mul i32 %"op10412(op17430)39", %"op17720(op10412, op17720)36"
 ; CHECK-NEXT:    %"op16562(op17720)41" = xor i32 -1, %"op17720(op10412, op17720)40"
 ; CHECK-NEXT:    %"op17430(op10412, op16562)42" = add i32 %"op10412(op17430)39", %"op16562(op17720)41"
-; CHECK-NEXT:    %"op17645(op10912)43" = add i32 -14, %"op10912(op17645, vl72693)"
-; CHECK-NEXT:    %"op18080(op17430, op17645)44" = add i32 %"op17430(op10412, op16562)42", %"op17645(op10912)43"
-; CHECK-NEXT:    %"op17720(op17720, op18080)45" = mul i32 %"op17720(op10412, op17720)40", %"op18080(op17430, op17645)44"
+; CHECK-NEXT:    %"op18131(op81283)43" = add i32 -14, %"op81283(op18131, vl72693)"
+; CHECK-NEXT:    %"op18080(op17430, op18131)44" = add i32 %"op17430(op10412, op16562)42", %"op18131(op81283)43"
+; CHECK-NEXT:    %"op17720(op17720, op18080)45" = mul i32 %"op17720(op10412, op17720)40", %"op18080(op17430, op18131)44"
 ; CHECK-NEXT:    %"op16562(op17720)46" = xor i32 -1, %"op17720(op17720, op18080)45"
-; CHECK-NEXT:    %"op17430(op16562, op18080)47" = add i32 %"op16562(op17720)46", %"op18080(op17430, op17645)44"
+; CHECK-NEXT:    %"op17430(op16562, op18080)47" = add i32 %"op16562(op17720)46", %"op18080(op17430, op18131)44"
 ; CHECK-NEXT:    %"op10412(op17430)48" = add i32 %"op17430(op16562, op18080)47", undef
 ; CHECK-NEXT:    %"op17720(op10412, op17720)49" = mul i32 %"op10412(op17430)48", %"op17720(op17720, op18080)45"
 ; CHECK-NEXT:    %"op16562(op17720)50" = xor i32 -1, %"op17720(op10412, op17720)49"
@@ -93,9 +93,9 @@ define void @test(ptr, i32) {
 ; CHECK-NEXT:    %"op17430(op10412, op16562)67" = add i32 %"op10412(op17430)64", %"op16562(op17720)66"
 ; CHECK-NEXT:    %"op10412(op17430)68" = add i32 %"op17430(op10412, op16562)67", undef
 ; CHECK-NEXT:    %"op10412(op10412)69" = add i32 %"op10412(op17430)68", undef
-; CHECK-NEXT:    %"op17645(op10912)70" = add i32 -21, %"op10912(op17645, vl72693)"
-; CHECK-NEXT:    %"op18080(op10412, op17645)" = add i32 %"op10412(op10412)69", %"op17645(op10912)70"
-; CHECK-NEXT:    store i32 %"op18080(op10412, op17645)", ptr [[A0]], align 4
+; CHECK-NEXT:    %"op18131(op81283)70" = add i32 -21, %"op81283(op18131, vl72693)"
+; CHECK-NEXT:    %"op18080(op10412, op18131)" = add i32 %"op10412(op10412)69", %"op18131(op81283)70"
+; CHECK-NEXT:    store i32 %"op18080(op10412, op18131)", ptr [[A0]], align 4
 ; CHECK-NEXT:    br label %[[BB16110]]
 ;
 bb:
diff --git a/llvm/test/Transforms/IRNormalizer/reordering-basic.ll b/llvm/test/Transforms/IRNormalizer/reordering-basic.ll
index fd09ce016addb..06e67e0feb7e4 100644
--- a/llvm/test/Transforms/IRNormalizer/reordering-basic.ll
+++ b/llvm/test/Transforms/IRNormalizer/reordering-basic.ll
@@ -28,16 +28,16 @@ define double @baz(double %x) {
 ; CHECK-SAME: double [[A0:%.*]]) {
 ; CHECK-NEXT:  [[BB76951:.*:]]
 ; CHECK-NEXT:    [[IFCOND:%.*]] = fcmp one double [[A0]], 0.000000e+00
-; CHECK-NEXT:    br i1 [[IFCOND]], label %[[BB91455:.*]], label %[[BB914551:.*]]
-; CHECK:       [[BB91455]]:
-; CHECK-NEXT:    %"vl15001bir()" = call double @bir()
+; CHECK-NEXT:    br i1 [[IFCOND]], label %[[BB47054:.*]], label %[[BB470541:.*]]
+; CHECK:       [[BB47054]]:
+; CHECK-NEXT:    %"vl16994bir()" = call double @bir()
 ; CHECK-NEXT:    br label %[[BB17254:.*]]
-; CHECK:       [[BB914551]]:
-; CHECK-NEXT:    %"vl69719bar()" = call double @bar()
+; CHECK:       [[BB470541]]:
+; CHECK-NEXT:    %"vl88592bar()" = call double @bar()
 ; CHECK-NEXT:    br label %[[BB17254]]
 ; CHECK:       [[BB17254]]:
-; CHECK-NEXT:    %"op19734(vl15001, vl69719)" = phi double [ %"vl15001bir()", %[[BB91455]] ], [ %"vl69719bar()", %[[BB914551]] ]
-; CHECK-NEXT:    ret double %"op19734(vl15001, vl69719)"
+; CHECK-NEXT:    %"op16411(vl16994, vl88592)" = phi double [ %"vl16994bir()", %[[BB47054]] ], [ %"vl88592bar()", %[[BB470541]] ]
+; CHECK-NEXT:    ret double %"op16411(vl16994, vl88592)"
 ;
 entry:
   %ifcond = fcmp one double %x, 0.000000e+00
diff --git a/llvm/test/Transforms/IRNormalizer/reordering.ll b/llvm/test/Transforms/IRNormalizer/reordering.ll
index 64abe8eb56ce1..a3dbcb5494875 100644
--- a/llvm/test/Transforms/IRNormalizer/reordering.ll
+++ b/llvm/test/Transforms/IRNormalizer/reordering.ll
@@ -23,7 +23,7 @@ declare void @effecting()
 ; Place dead instruction(s) before the terminator
 define void @call_effecting() {
 ; CHECK-LABEL: define void @call_effecting() {
-; CHECK-NEXT:  bb15160:
+; CHECK-NEXT:  bb14885:
 ; CHECK-NEXT:    call void @effecting()
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 0, 1
 ; CHECK-NEXT:    ret void
@@ -51,7 +51,7 @@ exit:
 
 define void @dont_move_above_alloca() {
 ; CHECK-LABEL: define void @dont_move_above_alloca() {
-; CHECK-NEXT:  bb15160:
+; CHECK-NEXT:  bb14885:
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @effecting()
 ; CHECK-NEXT:    ret void
@@ -65,7 +65,7 @@ declare void @effecting1()
 
 define void @dont_reorder_effecting() {
 ; CHECK-LABEL: define void @dont_reorder_effecting() {
-; CHECK-NEXT:  bb10075:
+; CHECK-NEXT:  bb45003:
 ; CHECK-NEXT:    call void @effecting()
 ; CHECK-NEXT:    call void @effecting1()
 ; CHECK-NEXT:    ret void
@@ -79,7 +79,7 @@ declare void @effecting2(i32)
 
 define void @dont_reorder_effecting1() {
 ; CHECK-LABEL: define void @dont_reorder_effecting1() {
-; CHECK-NEXT:  bb10075:
+; CHECK-NEXT:  bb45003:
 ; CHECK-NEXT:    [[ONE:%.*]] = add i32 1, 1
 ; CHECK-NEXT:    call void @effecting2(i32 [[ONE]])
 ; CHECK-NEXT:    [[TWO:%.*]] = add i32 2, 2
diff --git a/llvm/test/Transforms/IROutliner/alloca-addrspace-1.ll b/llvm/test/Transforms/IROutliner/alloca-addrspace-1.ll
index a096e6d38e4ff..73db71b6bafc9 100644
--- a/llvm/test/Transforms/IROutliner/alloca-addrspace-1.ll
+++ b/llvm/test/Transforms/IROutliner/alloca-addrspace-1.ll
@@ -20,14 +20,14 @@ declare i32 @llvm.foo(i32, i32)
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I3_LOC:%.*]] = alloca i32, align 4, addrspace(5)
 ; CHECK-NEXT:    [[I1_LOC:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 -1, ptr addrspace(5) [[I1_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[I1_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 0, i32 1, ptr addrspace(5) [[I1_LOC]])
 ; CHECK-NEXT:    [[I1_RELOAD:%.*]] = load i32, ptr addrspace(5) [[I1_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 -1, ptr addrspace(5) [[I1_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 -1, ptr addrspace(5) [[I3_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[I1_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[I3_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 [[I1_RELOAD]], i32 0, ptr addrspace(5) [[I3_LOC]])
 ; CHECK-NEXT:    [[I3_RELOAD:%.*]] = load i32, ptr addrspace(5) [[I3_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 -1, ptr addrspace(5) [[I3_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[I3_LOC]])
 ; CHECK-NEXT:    [[I4:%.*]] = tail call i32 @llvm.foo(i32 [[I3_RELOAD]], i32 0)
 ; CHECK-NEXT:    ret i32 0
 ;
diff --git a/llvm/test/Transforms/IROutliner/alloca-addrspace.ll b/llvm/test/Transforms/IROutliner/alloca-addrspace.ll
index e8701500b502d..ed7644426ffc0 100644
--- a/llvm/test/Transforms/IROutliner/alloca-addrspace.ll
+++ b/llvm/test/Transforms/IROutliner/alloca-addrspace.ll
@@ -18,10 +18,10 @@ declare i32 @func(i32, i32)
 ; CHECK-LABEL: define {{[^@]+}}@outlineable() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[I1_LOC:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 -1, ptr addrspace(5) [[I1_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[I1_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 0, i32 1, ptr addrspace(5) [[I1_LOC]], i32 0)
 ; CHECK-NEXT:    [[I1_RELOAD:%.*]] = load i32, ptr addrspace(5) [[I1_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 -1, ptr addrspace(5) [[I1_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[I1_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 [[I1_RELOAD]], i32 0, ptr addrspace(5) null, i32 -1)
 ; CHECK-NEXT:    ret i32 0
 ;
diff --git a/llvm/test/Transforms/IROutliner/different-intrinsics.ll b/llvm/test/Transforms/IROutliner/different-intrinsics.ll
index 5fb22c3101d82..f0e43bb533fdc 100644
--- a/llvm/test/Transforms/IROutliner/different-intrinsics.ll
+++ b/llvm/test/Transforms/IROutliner/different-intrinsics.ll
@@ -31,18 +31,18 @@ entry:
 ; CHECK-NEXT:    [[B_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[A_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(ptr [[S:%.*]], ptr [[D:%.*]], ptr [[A_LOC]], ptr [[B_LOC]])
 ; CHECK-NEXT:    [[A_RELOAD:%.*]] = load i8, ptr [[A_LOC]], align 1
 ; CHECK-NEXT:    [[B_RELOAD:%.*]] = load i8, ptr [[B_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[D]], ptr [[S]], i64 [[LEN:%.*]], i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i8 [[A_RELOAD]], i8 [[B_RELOAD]], ptr [[S]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 ;
@@ -51,18 +51,18 @@ entry:
 ; CHECK-NEXT:    [[B_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[A_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(ptr [[S:%.*]], ptr [[D:%.*]], ptr [[A_LOC]], ptr [[B_LOC]])
 ; CHECK-NEXT:    [[A_RELOAD:%.*]] = load i8, ptr [[A_LOC]], align 1
 ; CHECK-NEXT:    [[B_RELOAD:%.*]] = load i8, ptr [[B_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[D]], ptr [[S]], i64 [[LEN:%.*]], i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i8 [[A_RELOAD]], i8 [[B_RELOAD]], ptr [[S]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 ;
diff --git a/llvm/test/Transforms/IROutliner/different-order-phi-merges.ll b/llvm/test/Transforms/IROutliner/different-order-phi-merges.ll
index baf27edfca3d9..6730d1bda0f82 100644
--- a/llvm/test/Transforms/IROutliner/different-order-phi-merges.ll
+++ b/llvm/test/Transforms/IROutliner/different-order-phi-merges.ll
@@ -46,10 +46,10 @@ bb5:
 ; CHECK-LABEL: @f1(
 ; CHECK-NEXT:  bb1:
 ; CHECK-NEXT:    [[PHINODE_CE_LOC:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PHINODE_CE_LOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @outlined_ir_func_0(ptr [[PHINODE_CE_LOC]], i32 0)
 ; CHECK-NEXT:    [[PHINODE_CE_RELOAD:%.*]] = load i32, ptr [[PHINODE_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PHINODE_CE_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[BB5:%.*]], label [[BB1_AFTER_OUTLINE:%.*]]
 ; CHECK:       bb1_after_outline:
 ; CHECK-NEXT:    ret void
@@ -61,10 +61,10 @@ bb5:
 ; CHECK-LABEL: @f2(
 ; CHECK-NEXT:  bb1:
 ; CHECK-NEXT:    [[PHINODE_CE_LOC:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PHINODE_CE_LOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @outlined_ir_func_0(ptr [[PHINODE_CE_LOC]], i32 1)
 ; CHECK-NEXT:    [[PHINODE_CE_RELOAD:%.*]] = load i32, ptr [[PHINODE_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PHINODE_CE_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[BB5:%.*]], label [[BB1_AFTER_OUTLINE:%.*]]
 ; CHECK:       bb1_after_outline:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IROutliner/duplicate-merging-phis.ll b/llvm/test/Transforms/IROutliner/duplicate-merging-phis.ll
index 534efc38e3971..53d52f51f7ad2 100644
--- a/llvm/test/Transforms/IROutliner/duplicate-merging-phis.ll
+++ b/llvm/test/Transforms/IROutliner/duplicate-merging-phis.ll
@@ -49,13 +49,13 @@ bb5:
 ; CHECK-NEXT:  bb1:
 ; CHECK-NEXT:    [[PHINODE1_CE_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[PHINODE_CE_LOC:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[PHINODE1_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PHINODE1_CE_LOC]])
 ; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @outlined_ir_func_0(ptr [[PHINODE_CE_LOC]], ptr [[PHINODE1_CE_LOC]])
 ; CHECK-NEXT:    [[PHINODE_CE_RELOAD:%.*]] = load i32, ptr [[PHINODE_CE_LOC]], align 4
 ; CHECK-NEXT:    [[PHINODE1_CE_RELOAD:%.*]] = load i32, ptr [[PHINODE1_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[PHINODE1_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PHINODE1_CE_LOC]])
 ; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[BB5:%.*]], label [[BB1_AFTER_OUTLINE:%.*]]
 ; CHECK:       bb1_after_outline:
 ; CHECK-NEXT:    ret void
@@ -69,13 +69,13 @@ bb5:
 ; CHECK-NEXT:  bb1:
 ; CHECK-NEXT:    [[PHINODE1_CE_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[PHINODE_CE_LOC:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[PHINODE1_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PHINODE1_CE_LOC]])
 ; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @outlined_ir_func_0(ptr [[PHINODE_CE_LOC]], ptr [[PHINODE1_CE_LOC]])
 ; CHECK-NEXT:    [[PHINODE_CE_RELOAD:%.*]] = load i32, ptr [[PHINODE_CE_LOC]], align 4
 ; CHECK-NEXT:    [[PHINODE1_CE_RELOAD:%.*]] = load i32, ptr [[PHINODE1_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[PHINODE1_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PHINODE1_CE_LOC]])
 ; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[BB5:%.*]], label [[BB1_AFTER_OUTLINE:%.*]]
 ; CHECK:       bb1_after_outline:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IROutliner/exit-block-phi-node-value-attribution.ll b/llvm/test/Transforms/IROutliner/exit-block-phi-node-value-attribution.ll
index 3d3dbffcc243a..04ec9284ff5b6 100644
--- a/llvm/test/Transforms/IROutliner/exit-block-phi-node-value-attribution.ll
+++ b/llvm/test/Transforms/IROutliner/exit-block-phi-node-value-attribution.ll
@@ -42,19 +42,19 @@ bb5:
 ; CHECK-NEXT:  bb1:
 ; CHECK-NEXT:    [[PHINODE_CE_LOC1:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[PHINODE_CE_LOC:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PHINODE_CE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[PHINODE_CE_LOC]], i32 0)
 ; CHECK-NEXT:    [[PHINODE_CE_RELOAD:%.*]] = load i32, ptr [[PHINODE_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[PHINODE_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PHINODE_CE_LOC]])
 ; CHECK-NEXT:    br label [[BB5:%.*]]
 ; CHECK:       placeholder:
 ; CHECK-NEXT:    [[A:%.*]] = sub i32 5, 4
 ; CHECK-NEXT:    br label [[BB5]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[PHINODE_CE_LOC1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PHINODE_CE_LOC1]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[PHINODE_CE_LOC1]], i32 1)
 ; CHECK-NEXT:    [[PHINODE_CE_RELOAD2:%.*]] = load i32, ptr [[PHINODE_CE_LOC1]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[PHINODE_CE_LOC1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PHINODE_CE_LOC1]])
 ; CHECK-NEXT:    br label [[BB5]]
 ; CHECK:       placeholder1:
 ; CHECK-NEXT:    [[B:%.*]] = add i32 5, 4
diff --git a/llvm/test/Transforms/IROutliner/exit-phi-nodes-incoming-value-constant-argument.ll b/llvm/test/Transforms/IROutliner/exit-phi-nodes-incoming-value-constant-argument.ll
index cd60f936a9ed5..0e82217708d14 100644
--- a/llvm/test/Transforms/IROutliner/exit-phi-nodes-incoming-value-constant-argument.ll
+++ b/llvm/test/Transforms/IROutliner/exit-phi-nodes-incoming-value-constant-argument.ll
@@ -72,10 +72,10 @@ bb5:
 ; CHECK-NEXT:    br label [[BB5:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[A:%.*]] = sub i32 [[TMP0:%.*]], [[TMP1:%.*]]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[F_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[F_CE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 [[TMP0]], i32 [[TMP1]], i32 1, ptr [[F_CE_LOC]], i32 0)
 ; CHECK-NEXT:    [[F_CE_RELOAD:%.*]] = load i32, ptr [[F_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[F_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[F_CE_LOC]])
 ; CHECK-NEXT:    br label [[BB5]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[E:%.*]] = add i32 [[TMP0]], [[TMP1]]
diff --git a/llvm/test/Transforms/IROutliner/extraction.ll b/llvm/test/Transforms/IROutliner/extraction.ll
index 1eca4ea2f135e..77f904dd70861 100644
--- a/llvm/test/Transforms/IROutliner/extraction.ll
+++ b/llvm/test/Transforms/IROutliner/extraction.ll
@@ -59,13 +59,13 @@ define void @extract_outs1() #0 {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[OUTPUT:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[ADD_LOC]], ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; CHECK-NEXT:    [[DOTRELOAD:%.*]] = load i32, ptr [[DOTLOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[OUTPUT]], align 4
 ; CHECK-NEXT:    call void @outlined_ir_func_2(i32 [[DOTRELOAD]], i32 [[ADD_RELOAD]], ptr [[RESULT]])
 ; CHECK-NEXT:    ret void
@@ -99,13 +99,13 @@ define void @extract_outs2() #0 {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[OUTPUT:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[ADD_LOC]], ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; CHECK-NEXT:    [[DOTRELOAD:%.*]] = load i32, ptr [[DOTLOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_2(i32 [[DOTRELOAD]], i32 [[ADD_RELOAD]], ptr [[RESULT]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/IROutliner/gvn-output-set-overload.ll b/llvm/test/Transforms/IROutliner/gvn-output-set-overload.ll
index 1184b4a3d64fd..54f013c5652e6 100644
--- a/llvm/test/Transforms/IROutliner/gvn-output-set-overload.ll
+++ b/llvm/test/Transforms/IROutliner/gvn-output-set-overload.ll
@@ -44,10 +44,10 @@ next:
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTCE_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], ptr [[DOTCE_LOC]], ptr null, i32 0)
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FIRST:%.*]], label [[NEXT:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[DOTCE_RELOAD]], [[ENTRY:%.*]] ]
@@ -61,13 +61,13 @@ next:
 ; CHECK-NEXT:    [[E_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[C_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[C_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[C_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], ptr [[C_LOC]], ptr [[E_LOC]], i32 1)
 ; CHECK-NEXT:    [[C_RELOAD:%.*]] = load i32, ptr [[C_LOC]], align 4
 ; CHECK-NEXT:    [[E_RELOAD:%.*]] = load i32, ptr [[E_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[C_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[C_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FIRST:%.*]], label [[NEXT:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IROutliner/gvn-phi-debug.ll b/llvm/test/Transforms/IROutliner/gvn-phi-debug.ll
index 951466c359403..0c899dcd0339f 100644
--- a/llvm/test/Transforms/IROutliner/gvn-phi-debug.ll
+++ b/llvm/test/Transforms/IROutliner/gvn-phi-debug.ll
@@ -8,10 +8,10 @@ define i32 @r() {
 ; CHECK-LABEL: define i32 @r() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTLOC:%.*]] = alloca ptr, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @outlined_ir_func_0(ptr [[DOTLOC]], i32 0)
 ; CHECK-NEXT:    [[DOTRELOAD:%.*]] = load ptr, ptr [[DOTLOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_END8:%.*]], label [[ENTRY_AFTER_OUTLINE:%.*]]
 ; CHECK:       entry_after_outline:
 ; CHECK-NEXT:    [[CALL7:%.*]] = call i32 [[DOTRELOAD]]()
@@ -91,10 +91,10 @@ define i32 @w() !dbg !8 {
 ; CHECK-SAME: ) !dbg [[DBG8:![0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RETVAL_1_CE_LOC:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RETVAL_1_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RETVAL_1_CE_LOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @outlined_ir_func_0(ptr [[RETVAL_1_CE_LOC]], i32 1), !dbg [[DBG11:![0-9]+]]
 ; CHECK-NEXT:    [[RETVAL_1_CE_RELOAD:%.*]] = load i32, ptr [[RETVAL_1_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RETVAL_1_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RETVAL_1_CE_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[CLEANUP10:%.*]], label [[ENTRY_AFTER_OUTLINE:%.*]]
 ; CHECK:       entry_after_outline:
 ; CHECK-NEXT:    [[CALL8:%.*]] = call i32 @llvm.bswap.i32(i32 0)
diff --git a/llvm/test/Transforms/IROutliner/illegal-assumes.ll b/llvm/test/Transforms/IROutliner/illegal-assumes.ll
index d863fe7a29903..c0c4e1aa42e34 100644
--- a/llvm/test/Transforms/IROutliner/illegal-assumes.ll
+++ b/llvm/test/Transforms/IROutliner/illegal-assumes.ll
@@ -12,10 +12,10 @@ define void @outline_assumes() {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[D:%.*]] = alloca i1, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DL_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_4(i1 true, ptr [[D]], ptr [[DL_LOC]])
 ; CHECK-NEXT:    [[DL_RELOAD:%.*]] = load i1, ptr [[DL_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DL_LOC]])
 ; CHECK-NEXT:    [[SPLIT_INST:%.*]] = sub i1 [[DL_RELOAD]], [[DL_RELOAD]]
 ; CHECK-NEXT:    call void @outlined_ir_func_1(ptr [[A]], ptr [[B]], ptr [[C]])
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[DL_RELOAD]])
@@ -48,10 +48,10 @@ define void @outline_assumes2() {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[D:%.*]] = alloca i1, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DL_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_4(i1 false, ptr [[D]], ptr [[DL_LOC]])
 ; CHECK-NEXT:    [[DL_RELOAD:%.*]] = load i1, ptr [[DL_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DL_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(ptr [[A]], ptr [[B]], ptr [[C]])
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[DL_RELOAD]])
 ; CHECK-NEXT:    call void @outlined_ir_func_2(ptr [[A]], ptr [[B]], ptr [[C]])
@@ -82,10 +82,10 @@ define void @outline_assumes3() {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[D:%.*]] = alloca i1, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DL_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i1 true, ptr [[D]], ptr [[A]], ptr [[B]], ptr [[C]], ptr [[DL_LOC]])
 ; CHECK-NEXT:    [[DL_RELOAD:%.*]] = load i1, ptr [[DL_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DL_LOC]])
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[DL_RELOAD]])
 ; CHECK-NEXT:    call void @outlined_ir_func_3(ptr [[A]])
 ; CHECK-NEXT:    ret void
@@ -115,10 +115,10 @@ define void @outline_assumes4() {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[C:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[D:%.*]] = alloca i1, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DL_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i1 false, ptr [[D]], ptr [[A]], ptr [[B]], ptr [[C]], ptr [[DL_LOC]])
 ; CHECK-NEXT:    [[DL_RELOAD:%.*]] = load i1, ptr [[DL_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DL_LOC]])
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[DL_RELOAD]])
 ; CHECK-NEXT:    call void @outlined_ir_func_3(ptr [[A]])
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IROutliner/illegal-memcpy.ll b/llvm/test/Transforms/IROutliner/illegal-memcpy.ll
index 20e009aa32721..523fd23230461 100644
--- a/llvm/test/Transforms/IROutliner/illegal-memcpy.ll
+++ b/llvm/test/Transforms/IROutliner/illegal-memcpy.ll
@@ -12,18 +12,18 @@ define i8 @function1(ptr noalias %s, ptr noalias %d, i64 %len) {
 ; CHECK-NEXT:    [[B_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[A_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(ptr [[S:%.*]], ptr [[D:%.*]], ptr [[A_LOC]], ptr [[B_LOC]])
 ; CHECK-NEXT:    [[A_RELOAD:%.*]] = load i8, ptr [[A_LOC]], align 1
 ; CHECK-NEXT:    [[B_RELOAD:%.*]] = load i8, ptr [[B_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[D]], ptr [[S]], i64 [[LEN:%.*]], i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i8 [[A_RELOAD]], i8 [[B_RELOAD]], ptr [[S]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 entry:
@@ -41,18 +41,18 @@ define i8 @function2(ptr noalias %s, ptr noalias %d, i64 %len) {
 ; CHECK-NEXT:    [[B_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[A_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(ptr [[S:%.*]], ptr [[D:%.*]], ptr [[A_LOC]], ptr [[B_LOC]])
 ; CHECK-NEXT:    [[A_RELOAD:%.*]] = load i8, ptr [[A_LOC]], align 1
 ; CHECK-NEXT:    [[B_RELOAD:%.*]] = load i8, ptr [[B_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[D]], ptr [[S]], i64 [[LEN:%.*]], i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i8 [[A_RELOAD]], i8 [[B_RELOAD]], ptr [[S]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 entry:
diff --git a/llvm/test/Transforms/IROutliner/illegal-memmove.ll b/llvm/test/Transforms/IROutliner/illegal-memmove.ll
index 06480c8adb653..7482405b9d04d 100644
--- a/llvm/test/Transforms/IROutliner/illegal-memmove.ll
+++ b/llvm/test/Transforms/IROutliner/illegal-memmove.ll
@@ -12,18 +12,18 @@ define i8 @function1(ptr noalias %s, ptr noalias %d, i64 %len) {
 ; CHECK-NEXT:    [[B_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[A_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(ptr [[S:%.*]], ptr [[D:%.*]], ptr [[A_LOC]], ptr [[B_LOC]])
 ; CHECK-NEXT:    [[A_RELOAD:%.*]] = load i8, ptr [[A_LOC]], align 1
 ; CHECK-NEXT:    [[B_RELOAD:%.*]] = load i8, ptr [[B_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[D]], ptr [[S]], i64 [[LEN:%.*]], i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i8 [[A_RELOAD]], i8 [[B_RELOAD]], ptr [[S]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 entry:
@@ -41,18 +41,18 @@ define i8 @function2(ptr noalias %s, ptr noalias %d, i64 %len) {
 ; CHECK-NEXT:    [[B_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[A_LOC:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(ptr [[S:%.*]], ptr [[D:%.*]], ptr [[A_LOC]], ptr [[B_LOC]])
 ; CHECK-NEXT:    [[A_RELOAD:%.*]] = load i8, ptr [[A_LOC]], align 1
 ; CHECK-NEXT:    [[B_RELOAD:%.*]] = load i8, ptr [[B_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B_LOC]])
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr [[D]], ptr [[S]], i64 [[LEN:%.*]], i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i8 [[A_RELOAD]], i8 [[B_RELOAD]], ptr [[S]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 entry:
diff --git a/llvm/test/Transforms/IROutliner/illegal-vaarg.ll b/llvm/test/Transforms/IROutliner/illegal-vaarg.ll
index 38dfd25e039e6..15f9aa2f5932a 100644
--- a/llvm/test/Transforms/IROutliner/illegal-vaarg.ll
+++ b/llvm/test/Transforms/IROutliner/illegal-vaarg.ll
@@ -21,10 +21,10 @@ define i32 @func1(i32 %a, double %b, ptr %v, ...) nounwind {
 ; CHECK-NEXT:    [[TMP0:%.*]] = va_arg ptr [[AP]], i32
 ; CHECK-NEXT:    call void @llvm.va_copy.p0(ptr [[V:%.*]], ptr [[AP]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[AP]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[TMP0]], ptr [[C]], ptr [[TMP_LOC]])
 ; CHECK-NEXT:    [[TMP_RELOAD:%.*]] = load i32, ptr [[TMP_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[TMP_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP_LOC]])
 ; CHECK-NEXT:    ret i32 [[TMP_RELOAD]]
 ;
 entry:
@@ -56,10 +56,10 @@ define i32 @func2(i32 %a, double %b, ptr %v, ...) nounwind {
 ; CHECK-NEXT:    [[TMP0:%.*]] = va_arg ptr [[AP]], i32
 ; CHECK-NEXT:    call void @llvm.va_copy.p0(ptr [[V:%.*]], ptr [[AP]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[AP]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[TMP0]], ptr [[C]], ptr [[TMP_LOC]])
 ; CHECK-NEXT:    [[TMP_RELOAD:%.*]] = load i32, ptr [[TMP_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[TMP_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP_LOC]])
 ; CHECK-NEXT:    ret i32 [[TMP_RELOAD]]
 ;
 entry:
diff --git a/llvm/test/Transforms/IROutliner/mismatched-phi-exits-not-in-first-outlined.ll b/llvm/test/Transforms/IROutliner/mismatched-phi-exits-not-in-first-outlined.ll
index 24ad86f8e9c48..f9d49993b692d 100644
--- a/llvm/test/Transforms/IROutliner/mismatched-phi-exits-not-in-first-outlined.ll
+++ b/llvm/test/Transforms/IROutliner/mismatched-phi-exits-not-in-first-outlined.ll
@@ -47,10 +47,10 @@ first:
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTCE_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[TMP0]], ptr [[DOTCE_LOC]], i32 0)
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br label [[FIRST:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[DOTCE_RELOAD]], [[ENTRY:%.*]] ]
diff --git a/llvm/test/Transforms/IROutliner/mismatched-phi-exits.ll b/llvm/test/Transforms/IROutliner/mismatched-phi-exits.ll
index 6b50e99e2cf25..7191c80534f1b 100644
--- a/llvm/test/Transforms/IROutliner/mismatched-phi-exits.ll
+++ b/llvm/test/Transforms/IROutliner/mismatched-phi-exits.ll
@@ -38,10 +38,10 @@ first:
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTCE_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[TMP0]], ptr [[DOTCE_LOC]], i32 0)
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br label [[FIRST:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[DOTCE_RELOAD]], [[ENTRY:%.*]] ]
diff --git a/llvm/test/Transforms/IROutliner/mismatched-phi-outputs-ordering.ll b/llvm/test/Transforms/IROutliner/mismatched-phi-outputs-ordering.ll
index ab1836f7a4d28..9085e7e507726 100644
--- a/llvm/test/Transforms/IROutliner/mismatched-phi-outputs-ordering.ll
+++ b/llvm/test/Transforms/IROutliner/mismatched-phi-outputs-ordering.ll
@@ -48,16 +48,16 @@ next:
 ; CHECK-NEXT:    [[D_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[E_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[E_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[D_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[D_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], ptr [[E_LOC]], ptr [[D_LOC]], ptr [[DOTCE_LOC]], i32 0)
 ; CHECK-NEXT:    [[E_RELOAD:%.*]] = load i32, ptr [[E_LOC]], align 4
 ; CHECK-NEXT:    [[D_RELOAD:%.*]] = load i32, ptr [[D_LOC]], align 4
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[E_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[D_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[D_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FIRST:%.*]], label [[NEXT:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[DOTCE_RELOAD]], [[ENTRY:%.*]] ]
@@ -72,13 +72,13 @@ next:
 ; CHECK-NEXT:    [[D_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[E_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[E_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[D_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[D_LOC]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], ptr [[E_LOC]], ptr [[D_LOC]], ptr null, i32 1)
 ; CHECK-NEXT:    [[E_RELOAD:%.*]] = load i32, ptr [[E_LOC]], align 4
 ; CHECK-NEXT:    [[D_RELOAD:%.*]] = load i32, ptr [[D_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[E_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[D_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[D_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FIRST:%.*]], label [[NEXT:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IROutliner/must-capture-all-phi-nodes-begin.ll b/llvm/test/Transforms/IROutliner/must-capture-all-phi-nodes-begin.ll
index 32973ea3530dd..3229f4255c31d 100644
--- a/llvm/test/Transforms/IROutliner/must-capture-all-phi-nodes-begin.ll
+++ b/llvm/test/Transforms/IROutliner/must-capture-all-phi-nodes-begin.ll
@@ -57,10 +57,10 @@ first:
 ; CHECK:       test1:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[E_RELOAD:%.*]], [[TEST1]] ], [ [[Y]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[E_RELOAD]], [[TEST1]] ], [ [[Y]], [[ENTRY]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], i32 [[C]], ptr [[E_LOC]])
 ; CHECK-NEXT:    [[E_RELOAD]] = load i32, ptr [[E_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[TEST1]], label [[FIRST:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    ret void
@@ -78,10 +78,10 @@ first:
 ; CHECK:       test1:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[E_RELOAD:%.*]], [[TEST1]] ], [ [[Y]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[Y]], [[ENTRY]] ], [ [[E_RELOAD]], [[TEST1]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], i32 [[C]], ptr [[E_LOC]])
 ; CHECK-NEXT:    [[E_RELOAD]] = load i32, ptr [[E_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[TEST1]], label [[FIRST:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IROutliner/no-external-block-entries.ll b/llvm/test/Transforms/IROutliner/no-external-block-entries.ll
index 44260097e4415..fb2c5e9701f68 100644
--- a/llvm/test/Transforms/IROutliner/no-external-block-entries.ll
+++ b/llvm/test/Transforms/IROutliner/no-external-block-entries.ll
@@ -35,10 +35,10 @@ block_6:
 ; CHECK-LABEL: @fn1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B_CE_LOC:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_CE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[B_CE_LOC]], i32 0)
 ; CHECK-NEXT:    [[B_CE_RELOAD:%.*]] = load i32, ptr [[B_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B_CE_LOC]])
 ; CHECK-NEXT:    br label [[BLOCK_3:%.*]]
 ; CHECK:       block_3:
 ; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[B_CE_RELOAD]], [[ENTRY:%.*]] ]
diff --git a/llvm/test/Transforms/IROutliner/one-external-incoming-block-phi-node.ll b/llvm/test/Transforms/IROutliner/one-external-incoming-block-phi-node.ll
index 77e3a827453aa..96272745df70b 100644
--- a/llvm/test/Transforms/IROutliner/one-external-incoming-block-phi-node.ll
+++ b/llvm/test/Transforms/IROutliner/one-external-incoming-block-phi-node.ll
@@ -33,10 +33,10 @@ block_6:
 ; CHECK-LABEL: @fn1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B_CE_LOC:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_CE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[B_CE_LOC]], i32 0)
 ; CHECK-NEXT:    [[B_CE_RELOAD:%.*]] = load i32, ptr [[B_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B_CE_LOC]])
 ; CHECK-NEXT:    br label [[BLOCK_3:%.*]]
 ; CHECK:       block_3:
 ; CHECK-NEXT:    [[B:%.*]] = phi i32 [ [[B_CE_RELOAD]], [[ENTRY:%.*]] ]
diff --git a/llvm/test/Transforms/IROutliner/outline-memcpy.ll b/llvm/test/Transforms/IROutliner/outline-memcpy.ll
index 0cf4f34d23212..83fd5f6af925c 100644
--- a/llvm/test/Transforms/IROutliner/outline-memcpy.ll
+++ b/llvm/test/Transforms/IROutliner/outline-memcpy.ll
@@ -27,20 +27,20 @@ entry:
 ; CHECK-LABEL: @function1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[S:%.*]], ptr [[D:%.*]], i64 [[LEN:%.*]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 ;
 ; CHECK-LABEL: @function2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[S:%.*]], ptr [[D:%.*]], i64 [[LEN:%.*]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 ;
diff --git a/llvm/test/Transforms/IROutliner/outline-memmove.ll b/llvm/test/Transforms/IROutliner/outline-memmove.ll
index cf7924433d5bb..c512cd4a22e8f 100644
--- a/llvm/test/Transforms/IROutliner/outline-memmove.ll
+++ b/llvm/test/Transforms/IROutliner/outline-memmove.ll
@@ -27,20 +27,20 @@ entry:
 ; CHECK-LABEL: @function1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[S:%.*]], ptr [[D:%.*]], i64 [[LEN:%.*]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 ;
 ; CHECK-LABEL: @function2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RET_LOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[S:%.*]], ptr [[D:%.*]], i64 [[LEN:%.*]], ptr [[RET_LOC]])
 ; CHECK-NEXT:    [[RET_RELOAD:%.*]] = load i8, ptr [[RET_LOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[RET_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RET_LOC]])
 ; CHECK-NEXT:    ret i8 [[RET_RELOAD]]
 ;
 ;
diff --git a/llvm/test/Transforms/IROutliner/outline-vaarg-intrinsic.ll b/llvm/test/Transforms/IROutliner/outline-vaarg-intrinsic.ll
index 2d526086fae49..6a9cbca7dfddb 100644
--- a/llvm/test/Transforms/IROutliner/outline-vaarg-intrinsic.ll
+++ b/llvm/test/Transforms/IROutliner/outline-vaarg-intrinsic.ll
@@ -53,10 +53,10 @@ entry:
 ; CHECK-NEXT:    store double [[B]], ptr [[B_ADDR]], align 8
 ; CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[AP]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = va_arg ptr [[AP]], i32
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[V]], ptr [[AP]], i32 [[TMP0]], ptr [[C]], ptr [[TMP_LOC]])
 ; CHECK-NEXT:    [[TMP_RELOAD:%.*]] = load i32, ptr [[TMP_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[TMP_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP_LOC]])
 ; CHECK-NEXT:    ret i32 [[TMP_RELOAD]]
 ;
 ;
@@ -72,10 +72,10 @@ entry:
 ; CHECK-NEXT:    store double [[B]], ptr [[B_ADDR]], align 8
 ; CHECK-NEXT:    call void @llvm.va_start.p0(ptr [[AP]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = va_arg ptr [[AP]], i32
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[TMP_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[V]], ptr [[AP]], i32 [[TMP0]], ptr [[C]], ptr [[TMP_LOC]])
 ; CHECK-NEXT:    [[TMP_RELOAD:%.*]] = load i32, ptr [[TMP_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[TMP_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP_LOC]])
 ; CHECK-NEXT:    ret i32 [[TMP_RELOAD]]
 ;
 ;
diff --git a/llvm/test/Transforms/IROutliner/outlining-bitcasts.ll b/llvm/test/Transforms/IROutliner/outlining-bitcasts.ll
index 31f1d12bd3603..a8153a4fda446 100644
--- a/llvm/test/Transforms/IROutliner/outlining-bitcasts.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-bitcasts.ll
@@ -8,8 +8,8 @@
 ; Additionally, we check that the newly added bitcast instruction is excluded in
 ; further extractions.
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @outline_bitcast_base() {
 entry:
@@ -38,11 +38,11 @@ entry:
   %al = load i32, ptr %a
   %bl = load i32, ptr %b
   %cl = load i32, ptr %c
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %d)
+  call void @llvm.lifetime.start.p0(ptr %d)
   %am = load i32, ptr %b
   %bm = load i32, ptr %a
   %cm = load i32, ptr %c
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %d)
+  call void @llvm.lifetime.end.p0(ptr %d)
   ret void
 }
 
@@ -61,8 +61,8 @@ entry:
   %am = add i32 %a, %b
   %bm = add i32 %b, %a
   %cm = add i32 %b, %c
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %d)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %d)
+  call void @llvm.lifetime.start.p0(ptr %d)
+  call void @llvm.lifetime.end.p0(ptr %d)
   ret void
 }
 
@@ -114,13 +114,13 @@ entry:
 ; CHECK-LABEL: @outlined_ir_func_1(
 ; CHECK-NEXT:  newFuncRoot:
 ; CHECK-NEXT:    [[D:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[D]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[D]])
 ; CHECK-NEXT:    br label [[ENTRY_TO_OUTLINE:%.*]]
 ; CHECK:       entry_to_outline:
 ; CHECK-NEXT:    [[AL:%.*]] = add i32 [[TMP0:%.*]], [[TMP1:%.*]]
 ; CHECK-NEXT:    [[BL:%.*]] = add i32 [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[CL:%.*]] = add i32 [[TMP1]], [[TMP2:%.*]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[D]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[D]])
 ; CHECK-NEXT:    br label [[ENTRY_AFTER_OUTLINE_EXITSTUB:%.*]]
 ; CHECK:       entry_after_outline.exitStub:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IROutliner/outlining-branches-phi-nodes.ll b/llvm/test/Transforms/IROutliner/outlining-branches-phi-nodes.ll
index 28c23e34771ac..bb6bf8f2e1454 100644
--- a/llvm/test/Transforms/IROutliner/outlining-branches-phi-nodes.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-branches-phi-nodes.ll
@@ -100,10 +100,10 @@ block_6:
 ; CHECK-NEXT:    [[B2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br label [[BLOCK_2]]
 ; CHECK:       block_2:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DIFF_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DIFF_CE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[RESULT]], ptr [[DIFF_CE_LOC]])
 ; CHECK-NEXT:    [[DIFF_CE_RELOAD:%.*]] = load i32, ptr [[DIFF_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DIFF_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DIFF_CE_LOC]])
 ; CHECK-NEXT:    br label [[BLOCK_6:%.*]]
 ; CHECK: dummy:
 ; CHECK-NEXT:  ret void
@@ -127,10 +127,10 @@ block_6:
 ; CHECK-NEXT:    [[B2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br label [[BLOCK_2]]
 ; CHECK:       block_2:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DIFF_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DIFF_CE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[RESULT]], ptr [[DIFF_CE_LOC]])
 ; CHECK-NEXT:    [[DIFF_CE_RELOAD:%.*]] = load i32, ptr [[DIFF_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DIFF_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DIFF_CE_LOC]])
 ; CHECK-NEXT:    br label [[BLOCK_6:%.*]]
 ; CHECK: dummy:
 ; CHECK-NEXT:  ret void
diff --git a/llvm/test/Transforms/IROutliner/outlining-cost-model.ll b/llvm/test/Transforms/IROutliner/outlining-cost-model.ll
index 81bf4f09a70d7..bb3163a7941fe 100644
--- a/llvm/test/Transforms/IROutliner/outlining-cost-model.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-cost-model.ll
@@ -104,13 +104,13 @@ define void @function3() #0 {
 ; NOCOST-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; NOCOST-NEXT:    [[OUTPUT:%.*]] = alloca i32, align 4
 ; NOCOST-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
-; NOCOST-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; NOCOST-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; NOCOST-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; NOCOST-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; NOCOST-NEXT:    call void @outlined_ir_func_1(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[ADD_LOC]], ptr [[DOTLOC]])
 ; NOCOST-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; NOCOST-NEXT:    [[DOTRELOAD:%.*]] = load i32, ptr [[DOTLOC]], align 4
-; NOCOST-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; NOCOST-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; NOCOST-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; NOCOST-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; NOCOST-NEXT:    [[TMP0:%.*]] = load i32, ptr [[OUTPUT]], align 4
 ; NOCOST-NEXT:    call void @outlined_ir_func_2(i32 [[DOTRELOAD]], i32 [[ADD_RELOAD]], ptr [[RESULT]])
 ; NOCOST-NEXT:    ret void
@@ -159,13 +159,13 @@ define void @function4() #0 {
 ; NOCOST-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; NOCOST-NEXT:    [[OUTPUT:%.*]] = alloca i32, align 4
 ; NOCOST-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
-; NOCOST-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; NOCOST-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; NOCOST-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; NOCOST-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; NOCOST-NEXT:    call void @outlined_ir_func_1(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[ADD_LOC]], ptr [[DOTLOC]])
 ; NOCOST-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; NOCOST-NEXT:    [[DOTRELOAD:%.*]] = load i32, ptr [[DOTLOC]], align 4
-; NOCOST-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; NOCOST-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; NOCOST-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; NOCOST-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; NOCOST-NEXT:    call void @outlined_ir_func_2(i32 [[DOTRELOAD]], i32 [[ADD_RELOAD]], ptr [[RESULT]])
 ; NOCOST-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/IROutliner/outlining-different-output-blocks.ll b/llvm/test/Transforms/IROutliner/outlining-different-output-blocks.ll
index 2e1fae352cd44..64e87fcf7fb75 100644
--- a/llvm/test/Transforms/IROutliner/outlining-different-output-blocks.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-different-output-blocks.ll
@@ -14,13 +14,13 @@ define void @outline_outputs1() #0 {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[OUTPUT:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[ADD_LOC]], ptr [[DOTLOC]], i32 0)
 ; CHECK-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; CHECK-NEXT:    [[DOTRELOAD:%.*]] = load i32, ptr [[DOTLOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[OUTPUT]], align 4
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[DOTRELOAD]], i32 [[ADD_RELOAD]], ptr [[RESULT]])
 ; CHECK-NEXT:    ret void
@@ -53,13 +53,13 @@ define void @outline_outputs2() #0 {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[OUTPUT:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[SUB_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[SUB_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[SUB_LOC]], ptr [[DOTLOC]], i32 1)
 ; CHECK-NEXT:    [[SUB_RELOAD:%.*]] = load i32, ptr [[SUB_LOC]], align 4
 ; CHECK-NEXT:    [[DOTRELOAD:%.*]] = load i32, ptr [[DOTLOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[SUB_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[SUB_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[DOTRELOAD]], i32 [[SUB_RELOAD]], ptr [[RESULT]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/IROutliner/outlining-exits-to-phi-node.ll b/llvm/test/Transforms/IROutliner/outlining-exits-to-phi-node.ll
index cb5d5057182a4..d901955caadfb 100644
--- a/llvm/test/Transforms/IROutliner/outlining-exits-to-phi-node.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-exits-to-phi-node.ll
@@ -43,10 +43,10 @@ first:
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTCE_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[TMP0]], ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br label [[FIRST:%.*]]
 ; CHECK: dummy:
 ; CHECK-NEXT:  ret void
@@ -59,10 +59,10 @@ first:
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTCE_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[TMP0]], ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br label [[FIRST:%.*]]
 ; CHECK: dummy:
 ; CHECK-NEXT:  ret void
diff --git a/llvm/test/Transforms/IROutliner/outlining-multiple-exits-diff-outputs.ll b/llvm/test/Transforms/IROutliner/outlining-multiple-exits-diff-outputs.ll
index 463e097cd2f29..9dbfa9ebd7e37 100644
--- a/llvm/test/Transforms/IROutliner/outlining-multiple-exits-diff-outputs.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-multiple-exits-diff-outputs.ll
@@ -103,19 +103,19 @@ block_7:
 ; CHECK-NEXT:    [[B2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br label [[BLOCK_2]]
 ; CHECK:       block_2:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[AVAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[BVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[AVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[BVAL_LOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[RESULT]], ptr [[A2VAL_LOC]], ptr [[B2VAL_LOC]], ptr [[AVAL_LOC]], ptr [[BVAL_LOC]], i32 0)
 ; CHECK-NEXT:    [[A2VAL_RELOAD:%.*]] = load i32, ptr [[A2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[B2VAL_RELOAD:%.*]] = load i32, ptr [[B2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[AVAL_RELOAD:%.*]] = load i32, ptr [[AVAL_LOC]], align 4
 ; CHECK-NEXT:    [[BVAL_RELOAD:%.*]] = load i32, ptr [[BVAL_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[AVAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[BVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[AVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[BVAL_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[BLOCK_6:%.*]], label [[BLOCK_7:%.*]]
 ; CHECK:       block_6:
 ; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[AVAL_RELOAD]], [[BVAL_RELOAD]]
@@ -143,19 +143,19 @@ block_7:
 ; CHECK-NEXT:    [[B2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br label [[BLOCK_2]]
 ; CHECK:       block_2:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[MUL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[MUL_LOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[RESULT]], ptr [[A2VAL_LOC]], ptr [[B2VAL_LOC]], ptr [[ADD_LOC]], ptr [[MUL_LOC]], i32 1)
 ; CHECK-NEXT:    [[A2VAL_RELOAD:%.*]] = load i32, ptr [[A2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[B2VAL_RELOAD:%.*]] = load i32, ptr [[B2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; CHECK-NEXT:    [[MUL_RELOAD:%.*]] = load i32, ptr [[MUL_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[MUL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[MUL_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[BLOCK_7:%.*]], label [[BLOCK_6:%.*]]
 ; CHECK:       block_6:
 ; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[A2VAL_RELOAD]], [[B2VAL_RELOAD]]
diff --git a/llvm/test/Transforms/IROutliner/outlining-multiple-exits-one-output-set.ll b/llvm/test/Transforms/IROutliner/outlining-multiple-exits-one-output-set.ll
index 5293647ad6c4c..f7897358a55bc 100644
--- a/llvm/test/Transforms/IROutliner/outlining-multiple-exits-one-output-set.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-multiple-exits-one-output-set.ll
@@ -124,19 +124,19 @@ block_7:
 ; CHECK-NEXT:    [[B2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br label [[BLOCK_2]]
 ; CHECK:       block_2:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[MUL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[MUL_LOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i1 @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[RESULT]], ptr [[A2VAL_LOC]], ptr [[B2VAL_LOC]], ptr [[ADD_LOC]], ptr [[MUL_LOC]], i32 0)
 ; CHECK-NEXT:    [[A2VAL_RELOAD:%.*]] = load i32, ptr [[A2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[B2VAL_RELOAD:%.*]] = load i32, ptr [[B2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; CHECK-NEXT:    [[MUL_RELOAD:%.*]] = load i32, ptr [[MUL_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[MUL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[MUL_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[BLOCK_7:%.*]], label [[BLOCK_6:%.*]]
 ; CHECK:       block_6:
 ; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[A2VAL_RELOAD]], [[B2VAL_RELOAD]]
diff --git a/llvm/test/Transforms/IROutliner/outlining-multiple-exits.ll b/llvm/test/Transforms/IROutliner/outlining-multiple-exits.ll
index 663e6d865b64a..1de13ebc3bc3a 100644
--- a/llvm/test/Transforms/IROutliner/outlining-multiple-exits.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-multiple-exits.ll
@@ -104,19 +104,19 @@ block_7:
 ; CHECK-NEXT:    [[B2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br label [[BLOCK_2]]
 ; CHECK:       block_2:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[AVAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[BVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[AVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[BVAL_LOC]])
 ; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[RESULT]], ptr [[A2VAL_LOC]], ptr [[B2VAL_LOC]], ptr [[AVAL_LOC]], ptr [[BVAL_LOC]])
 ; CHECK-NEXT:    [[A2VAL_RELOAD:%.*]] = load i32, ptr [[A2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[B2VAL_RELOAD:%.*]] = load i32, ptr [[B2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[AVAL_RELOAD:%.*]] = load i32, ptr [[AVAL_LOC]], align 4
 ; CHECK-NEXT:    [[BVAL_RELOAD:%.*]] = load i32, ptr [[BVAL_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[AVAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[BVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[AVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[BVAL_LOC]])
 ; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[BLOCK_6:%.*]], label [[BLOCK_7:%.*]]
 ; CHECK:       block_6:
 ; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[AVAL_RELOAD]], [[BVAL_RELOAD]]
@@ -144,19 +144,19 @@ block_7:
 ; CHECK-NEXT:    [[B2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br label [[BLOCK_2]]
 ; CHECK:       block_2:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[AVAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[BVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[AVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[BVAL_LOC]])
 ; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[RESULT]], ptr [[A2VAL_LOC]], ptr [[B2VAL_LOC]], ptr [[AVAL_LOC]], ptr [[BVAL_LOC]])
 ; CHECK-NEXT:    [[A2VAL_RELOAD:%.*]] = load i32, ptr [[A2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[B2VAL_RELOAD:%.*]] = load i32, ptr [[B2VAL_LOC]], align 4
 ; CHECK-NEXT:    [[AVAL_RELOAD:%.*]] = load i32, ptr [[AVAL_LOC]], align 4
 ; CHECK-NEXT:    [[BVAL_RELOAD:%.*]] = load i32, ptr [[BVAL_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B2VAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[AVAL_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[BVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B2VAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[AVAL_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[BVAL_LOC]])
 ; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[BLOCK_7:%.*]], label [[BLOCK_6:%.*]]
 ; CHECK:       block_6:
 ; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[A2VAL_RELOAD]], [[B2VAL_RELOAD]]
diff --git a/llvm/test/Transforms/IROutliner/outlining-remapped-outputs.ll b/llvm/test/Transforms/IROutliner/outlining-remapped-outputs.ll
index 6d0b15394f330..77f17c3162131 100644
--- a/llvm/test/Transforms/IROutliner/outlining-remapped-outputs.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-remapped-outputs.ll
@@ -17,24 +17,24 @@ define void @outline_outputs1() #0 {
 ; CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[OUTPUT2:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[RESULT2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 2, ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[ADD_LOC]], ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; CHECK-NEXT:    [[DOTRELOAD:%.*]] = load i32, ptr [[DOTLOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[OUTPUT]], align 4
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[DOTRELOAD]], i32 [[ADD_RELOAD]], ptr [[RESULT]])
 ; CHECK-NEXT:    br label [[NEXT:%.*]]
 ; CHECK:       next:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD2_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC2]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD2_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC2]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 [[ADD_RELOAD]], ptr [[OUTPUT]], ptr [[RESULT]], ptr [[OUTPUT2]], ptr [[ADD2_LOC]], ptr [[DOTLOC2]])
 ; CHECK-NEXT:    [[ADD2_RELOAD:%.*]] = load i32, ptr [[ADD2_LOC]], align 4
 ; CHECK-NEXT:    [[DOTRELOAD3:%.*]] = load i32, ptr [[DOTLOC2]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD2_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC2]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD2_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC2]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[DOTRELOAD3]], i32 [[ADD2_RELOAD]], ptr [[RESULT2]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/IROutliner/outlining-same-output-blocks.ll b/llvm/test/Transforms/IROutliner/outlining-same-output-blocks.ll
index 380c53d1e77e3..cc4f6ef6d7e07 100644
--- a/llvm/test/Transforms/IROutliner/outlining-same-output-blocks.ll
+++ b/llvm/test/Transforms/IROutliner/outlining-same-output-blocks.ll
@@ -14,13 +14,13 @@ define void @outline_outputs1() #0 {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[OUTPUT:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[ADD_LOC]], ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; CHECK-NEXT:    [[DOTRELOAD:%.*]] = load i32, ptr [[DOTLOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[OUTPUT]], align 4
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[DOTRELOAD]], i32 [[ADD_RELOAD]], ptr [[RESULT]])
 ; CHECK-NEXT:    ret void
@@ -52,13 +52,13 @@ define void @outline_outputs2() #0 {
 ; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[OUTPUT:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[RESULT:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[A]], ptr [[B]], ptr [[OUTPUT]], ptr [[ADD_LOC]], ptr [[DOTLOC]])
 ; CHECK-NEXT:    [[ADD_RELOAD:%.*]] = load i32, ptr [[ADD_LOC]], align 4
 ; CHECK-NEXT:    [[DOTRELOAD:%.*]] = load i32, ptr [[DOTLOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ADD_LOC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ADD_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTLOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_1(i32 [[DOTRELOAD]], i32 [[ADD_RELOAD]], ptr [[RESULT]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/IROutliner/phi-node-exit-path-order.ll b/llvm/test/Transforms/IROutliner/phi-node-exit-path-order.ll
index 4bbe6e7ea9918..15d313a41bf7c 100644
--- a/llvm/test/Transforms/IROutliner/phi-node-exit-path-order.ll
+++ b/llvm/test/Transforms/IROutliner/phi-node-exit-path-order.ll
@@ -58,10 +58,10 @@ bb5:
 ; CHECK-NEXT:    br label [[BB5:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[A:%.*]] = add i32 [[TMP0:%.*]], [[TMP1:%.*]]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[F_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[F_CE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 [[TMP0]], i32 [[TMP1]], ptr [[F_CE_LOC]], i32 0)
 ; CHECK-NEXT:    [[F_CE_RELOAD:%.*]] = load i32, ptr [[F_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[F_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[F_CE_LOC]])
 ; CHECK-NEXT:    br label [[BB5]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[E:%.*]] = sub i32 [[TMP0]], [[TMP1]]
@@ -77,10 +77,10 @@ bb5:
 ; CHECK-NEXT:    br label [[BB5:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[A:%.*]] = sub i32 [[TMP0:%.*]], [[TMP1:%.*]]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[F_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[F_CE_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(i32 [[TMP0]], i32 [[TMP1]], ptr [[F_CE_LOC]], i32 1)
 ; CHECK-NEXT:    [[F_CE_RELOAD:%.*]] = load i32, ptr [[F_CE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[F_CE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[F_CE_LOC]])
 ; CHECK-NEXT:    br label [[BB5]]
 ; CHECK:       bb4:
 ; CHECK-NEXT:    [[E:%.*]] = add i32 [[TMP0]], [[TMP1]]
diff --git a/llvm/test/Transforms/IROutliner/phi-nodes-output-overload.ll b/llvm/test/Transforms/IROutliner/phi-nodes-output-overload.ll
index 9631bfac7d1c0..9e443abaaff50 100644
--- a/llvm/test/Transforms/IROutliner/phi-nodes-output-overload.ll
+++ b/llvm/test/Transforms/IROutliner/phi-nodes-output-overload.ll
@@ -43,10 +43,10 @@ next:
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTCE_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], ptr [[DOTCE_LOC]], i32 0)
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FIRST:%.*]], label [[NEXT:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[DOTCE_RELOAD]], [[ENTRY:%.*]] ]
@@ -59,10 +59,10 @@ next:
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[DOTCE_LOC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], ptr [[DOTCE_LOC]], i32 1)
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[FIRST:%.*]], label [[NEXT:%.*]]
 ; CHECK:       first:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/IROutliner/phi-nodes-parent-block-referential.ll b/llvm/test/Transforms/IROutliner/phi-nodes-parent-block-referential.ll
index 608abfa95333c..02930d77be3a8 100644
--- a/llvm/test/Transforms/IROutliner/phi-nodes-parent-block-referential.ll
+++ b/llvm/test/Transforms/IROutliner/phi-nodes-parent-block-referential.ll
@@ -53,10 +53,10 @@ first:
 ; CHECK:       test1:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[E_RELOAD:%.*]], [[TEST1]] ], [ [[Y]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[E_RELOAD]], [[TEST1]] ], [ [[Y]], [[ENTRY]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[TMP0]], i32 [[C]], ptr [[E_LOC]])
 ; CHECK-NEXT:    [[E_RELOAD]] = load i32, ptr [[E_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = sub i32 [[C]], [[C]]
 ; CHECK-NEXT:    br i1 true, label [[FIRST:%.*]], label [[TEST1]]
 ; CHECK:       first:
@@ -75,10 +75,10 @@ first:
 ; CHECK:       test1:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[E_RELOAD:%.*]], [[TEST1]] ], [ [[Y]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = phi i32 [ [[Y]], [[ENTRY]] ], [ [[E_RELOAD]], [[TEST1]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    call void @outlined_ir_func_0(ptr [[TMP0]], i32 [[C]], ptr [[E_LOC]])
 ; CHECK-NEXT:    [[E_RELOAD]] = load i32, ptr [[E_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[E_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[E_LOC]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[C]], [[C]]
 ; CHECK-NEXT:    br i1 true, label [[FIRST:%.*]], label [[TEST1]]
 ; CHECK:       first:
diff --git a/llvm/test/Transforms/IROutliner/region-inputs-in-phi-nodes.ll b/llvm/test/Transforms/IROutliner/region-inputs-in-phi-nodes.ll
index f46035a883832..25b1e8ee29f3c 100644
--- a/llvm/test/Transforms/IROutliner/region-inputs-in-phi-nodes.ll
+++ b/llvm/test/Transforms/IROutliner/region-inputs-in-phi-nodes.ll
@@ -53,10 +53,10 @@ next:
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[Z:%.*]] = add i32 [[C]], [[C]]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], i32 [[C]], ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[FIRST:%.*]], label [[NEXT:%.*]]
 ; CHECK: dummy:
 ; CHECK-NEXT:  ret void
@@ -73,10 +73,10 @@ next:
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[Z:%.*]] = mul i32 [[C]], [[C]]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @outlined_ir_func_0(ptr [[TMP0]], i32 [[C]], ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    [[DOTCE_RELOAD:%.*]] = load i32, ptr [[DOTCE_LOC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[DOTCE_LOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DOTCE_LOC]])
 ; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[FIRST:%.*]], label [[NEXT:%.*]]
 ; CHECK: dummy:
 ; CHECK-NEXT:  ret void
diff --git a/llvm/test/Transforms/IndVarSimplify/exit_value_test2.ll b/llvm/test/Transforms/IndVarSimplify/exit_value_test2.ll
index 66c7222ce6d4e..697c81603318d 100644
--- a/llvm/test/Transforms/IndVarSimplify/exit_value_test2.ll
+++ b/llvm/test/Transforms/IndVarSimplify/exit_value_test2.ll
@@ -6,14 +6,14 @@
 ; udiv will be introduced by expand and the cost will be high.
 
 declare void @_Z3mixRjj(ptr dereferenceable(4), i32)
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define i32 @_Z3fooPKcjj(ptr nocapture readonly %s, i32 %len, i32 %c) {
 ; CHECK-LABEL: @_Z3fooPKcjj(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    store i32 -1640531527, ptr [[A]], align 4
 ; CHECK-NEXT:    [[CMP8:%.*]] = icmp ugt i32 [[LEN:%.*]], 11
 ; CHECK-NEXT:    br i1 [[CMP8]], label [[WHILE_BODY_LR_PH:%.*]], label [[WHILE_END:%.*]]
@@ -40,12 +40,12 @@ define i32 @_Z3fooPKcjj(ptr nocapture readonly %s, i32 %len, i32 %c) {
 ; CHECK-NEXT:    [[KEYLEN_0_LCSSA:%.*]] = phi i32 [ [[SUB_LCSSA]], [[WHILE_COND_WHILE_END_CRIT_EDGE]] ], [ [[LEN]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    call void @_Z3mixRjj(ptr dereferenceable(4) [[A]], i32 [[KEYLEN_0_LCSSA]])
 ; CHECK-NEXT:    [[T4:%.*]] = load i32, ptr [[A]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret i32 [[T4]]
 ;
 entry:
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   store i32 -1640531527, ptr %a, align 4
   %cmp8 = icmp ugt i32 %len, 11
   br i1 %cmp8, label %while.body.lr.ph, label %while.end
@@ -76,7 +76,7 @@ while.end:                                        ; preds = %while.cond.while.en
   %keylen.0.lcssa = phi i32 [ %sub.lcssa, %while.cond.while.end_crit_edge ], [ %len, %entry ]
   call void @_Z3mixRjj(ptr dereferenceable(4) %a, i32 %keylen.0.lcssa)
   %t4 = load i32, ptr %a, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret i32 %t4
 }
 
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/lifetime.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/lifetime.ll
index 053d073d06309..4c04e6d02e7be 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/lifetime.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/lifetime.ll
@@ -4,17 +4,17 @@
 define i32 @lifetime_flat_pointer() {
 ; CHECK-LABEL: define i32 @lifetime_flat_pointer() {
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4, addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p5(i64 4, ptr addrspace(5) [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p5(ptr addrspace(5) [[ALLOCA]])
 ; CHECK-NEXT:    store i32 1, ptr addrspace(5) [[ALLOCA]], align 4
 ; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr addrspace(5) [[ALLOCA]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p5(i64 4, ptr addrspace(5) [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p5(ptr addrspace(5) [[ALLOCA]])
 ; CHECK-NEXT:    ret i32 [[RET]]
 ;
   %alloca = alloca i32, align 4, addrspace(5)
   %flat = addrspacecast ptr addrspace(5) %alloca to ptr
-  call void @llvm.lifetime.start(i64 4, ptr addrspace(5) %alloca)
+  call void @llvm.lifetime.start(ptr addrspace(5) %alloca)
   store i32 1, ptr %flat, align 4
   %ret = load i32, ptr %flat, align 4
-  call void @llvm.lifetime.end(i64 4, ptr addrspace(5) %alloca)
+  call void @llvm.lifetime.end(ptr addrspace(5) %alloca)
   ret i32 %ret
 }
diff --git a/llvm/test/Transforms/InferAddressSpaces/NVPTX/lifetime.ll b/llvm/test/Transforms/InferAddressSpaces/NVPTX/lifetime.ll
index 31e914a64de4e..1a214162d19f7 100644
--- a/llvm/test/Transforms/InferAddressSpaces/NVPTX/lifetime.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/NVPTX/lifetime.ll
@@ -7,20 +7,20 @@ define i32 @lifetime_flat_pointer() {
 ; CHECK-LABEL: define i32 @lifetime_flat_pointer() {
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = addrspacecast ptr [[ALLOCA]] to ptr addrspace(5)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ALLOCA]])
 ; CHECK-NEXT:    store i32 1, ptr addrspace(5) [[TMP1]], align 4
 ; CHECK-NEXT:    [[RET:%.*]] = load i32, ptr addrspace(5) [[TMP1]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ALLOCA]])
 ; CHECK-NEXT:    ret i32 [[RET]]
 ;
   %alloca = alloca i32, align 4
   %1 = addrspacecast ptr %alloca to ptr addrspace(5)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %alloca)
+  call void @llvm.lifetime.start.p0(ptr %alloca)
   store i32 1, ptr addrspace(5) %1, align 4
   %ret = load i32, ptr addrspace(5) %1, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %alloca)
+  call void @llvm.lifetime.end.p0(ptr %alloca)
   ret i32 %ret
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Transforms/InferAlignment/propagate-from-other-load-stores.ll b/llvm/test/Transforms/InferAlignment/propagate-from-other-load-stores.ll
new file mode 100644
index 0000000000000..3fc7c59a512a5
--- /dev/null
+++ b/llvm/test/Transforms/InferAlignment/propagate-from-other-load-stores.ll
@@ -0,0 +1,194 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=infer-alignment -S | FileCheck %s
+%struct.S1 = type { %struct.float3, %struct.float3, i32, i32 }
+%struct.float3 = type { float, float, float }
+
+
+; ------------------------------------------------------------------------------
+; Test that we can propagate the align 16 to the load and store that are set to align 4
+; ------------------------------------------------------------------------------
+
+define void @prop_align(ptr %v, ptr %vout) {
+; CHECK-LABEL: define void @prop_align(
+; CHECK-SAME: ptr [[V:%.*]], ptr [[VOUT:%.*]]) {
+; CHECK-NEXT:    [[DOTUNPACK_UNPACK:%.*]] = load float, ptr [[V]], align 16
+; CHECK-NEXT:    [[DOTUNPACK_ELT7:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 4
+; CHECK-NEXT:    [[DOTUNPACK_UNPACK8:%.*]] = load float, ptr [[DOTUNPACK_ELT7]], align 4
+; CHECK-NEXT:    [[DOTUNPACK_ELT9:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 8
+; CHECK-NEXT:    [[DOTUNPACK_UNPACK10:%.*]] = load float, ptr [[DOTUNPACK_ELT9]], align 8
+; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 12
+; CHECK-NEXT:    [[DOTUNPACK2_UNPACK:%.*]] = load float, ptr [[DOTELT1]], align 4
+; CHECK-NEXT:    [[DOTUNPACK2_ELT12:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 16
+; CHECK-NEXT:    [[DOTUNPACK2_UNPACK13:%.*]] = load float, ptr [[DOTUNPACK2_ELT12]], align 16
+; CHECK-NEXT:    [[DOTUNPACK2_ELT14:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 20
+; CHECK-NEXT:    [[DOTUNPACK2_UNPACK15:%.*]] = load float, ptr [[DOTUNPACK2_ELT14]], align 4
+; CHECK-NEXT:    [[DOTELT3:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 24
+; CHECK-NEXT:    [[DOTUNPACK4:%.*]] = load i32, ptr [[DOTELT3]], align 8
+; CHECK-NEXT:    [[DOTELT5:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 28
+; CHECK-NEXT:    [[DOTUNPACK6:%.*]] = load i32, ptr [[DOTELT5]], align 4
+; CHECK-NEXT:    store float [[DOTUNPACK_UNPACK]], ptr [[VOUT]], align 16
+; CHECK-NEXT:    [[VOUT_REPACK23:%.*]] = getelementptr inbounds nuw i8, ptr [[VOUT]], i64 4
+; CHECK-NEXT:    store float [[DOTUNPACK_UNPACK8]], ptr [[VOUT_REPACK23]], align 4
+; CHECK-NEXT:    [[VOUT_REPACK25:%.*]] = getelementptr inbounds nuw i8, ptr [[VOUT]], i64 8
+; CHECK-NEXT:    store float [[DOTUNPACK_UNPACK10]], ptr [[VOUT_REPACK25]], align 8
+; CHECK-NEXT:    [[VOUT_REPACK17:%.*]] = getelementptr inbounds nuw i8, ptr [[VOUT]], i64 12
+; CHECK-NEXT:    store float [[DOTUNPACK2_UNPACK]], ptr [[VOUT_REPACK17]], align 4
+; CHECK-NEXT:    [[VOUT_REPACK17_REPACK27:%.*]] = getelementptr inbounds nuw i8, ptr [[VOUT]], i64 16
+; CHECK-NEXT:    store float [[DOTUNPACK2_UNPACK13]], ptr [[VOUT_REPACK17_REPACK27]], align 16
+; CHECK-NEXT:    [[VOUT_REPACK17_REPACK29:%.*]] = getelementptr inbounds nuw i8, ptr [[VOUT]], i64 20
+; CHECK-NEXT:    store float [[DOTUNPACK2_UNPACK15]], ptr [[VOUT_REPACK17_REPACK29]], align 4
+; CHECK-NEXT:    [[VOUT_REPACK19:%.*]] = getelementptr inbounds nuw i8, ptr [[VOUT]], i64 24
+; CHECK-NEXT:    store i32 [[DOTUNPACK4]], ptr [[VOUT_REPACK19]], align 8
+; CHECK-NEXT:    [[VOUT_REPACK21:%.*]] = getelementptr inbounds nuw i8, ptr [[VOUT]], i64 28
+; CHECK-NEXT:    store i32 [[DOTUNPACK6]], ptr [[VOUT_REPACK21]], align 4
+; CHECK-NEXT:    ret void
+;
+  %.unpack.unpack = load float, ptr %v, align 16
+  %.unpack.elt7 = getelementptr inbounds nuw i8, ptr %v, i64 4
+  %.unpack.unpack8 = load float, ptr %.unpack.elt7, align 4
+  %.unpack.elt9 = getelementptr inbounds nuw i8, ptr %v, i64 8
+  %.unpack.unpack10 = load float, ptr %.unpack.elt9, align 8
+  %.elt1 = getelementptr inbounds nuw i8, ptr %v, i64 12
+  %.unpack2.unpack = load float, ptr %.elt1, align 4
+  %.unpack2.elt12 = getelementptr inbounds nuw i8, ptr %v, i64 16
+  %.unpack2.unpack13 = load float, ptr %.unpack2.elt12, align 4
+  %.unpack2.elt14 = getelementptr inbounds nuw i8, ptr %v, i64 20
+  %.unpack2.unpack15 = load float, ptr %.unpack2.elt14, align 4
+  %.elt3 = getelementptr inbounds nuw i8, ptr %v, i64 24
+  %.unpack4 = load i32, ptr %.elt3, align 8
+  %.elt5 = getelementptr inbounds nuw i8, ptr %v, i64 28
+  %.unpack6 = load i32, ptr %.elt5, align 4
+  store float %.unpack.unpack, ptr %vout, align 16
+  %vout.repack23 = getelementptr inbounds nuw i8, ptr %vout, i64 4
+  store float %.unpack.unpack8, ptr %vout.repack23, align 4
+  %vout.repack25 = getelementptr inbounds nuw i8, ptr %vout, i64 8
+  store float %.unpack.unpack10, ptr %vout.repack25, align 8
+  %vout.repack17 = getelementptr inbounds nuw i8, ptr %vout, i64 12
+  store float %.unpack2.unpack, ptr %vout.repack17, align 4
+  %vout.repack17.repack27 = getelementptr inbounds nuw i8, ptr %vout, i64 16
+  store float %.unpack2.unpack13, ptr %vout.repack17.repack27, align 4
+  %vout.repack17.repack29 = getelementptr inbounds nuw i8, ptr %vout, i64 20
+  store float %.unpack2.unpack15, ptr %vout.repack17.repack29, align 4
+  %vout.repack19 = getelementptr inbounds nuw i8, ptr %vout, i64 24
+  store i32 %.unpack4, ptr %vout.repack19, align 8
+  %vout.repack21 = getelementptr inbounds nuw i8, ptr %vout, i64 28
+  store i32 %.unpack6, ptr %vout.repack21, align 4
+  ret void
+}
+
+; ------------------------------------------------------------------------------
+; Test that alignment is not propagated from a source that does not dominate the destination
+; ------------------------------------------------------------------------------
+
+define void @no_prop_align(ptr %v, ptr %vout, i1 %cond) {
+; CHECK-LABEL: define void @no_prop_align(
+; CHECK-SAME: ptr [[V:%.*]], ptr [[VOUT:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    br i1 [[COND]], label %[[BRANCH1:.*]], label %[[BRANCH2:.*]]
+; CHECK:       [[BRANCH1]]:
+; CHECK-NEXT:    [[DOTUNPACK_UNPACK:%.*]] = load float, ptr [[V]], align 16
+; CHECK-NEXT:    [[DOTUNPACK_ELT7:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 4
+; CHECK-NEXT:    [[DOTUNPACK_UNPACK8:%.*]] = load float, ptr [[DOTUNPACK_ELT7]], align 4
+; CHECK-NEXT:    [[DOTUNPACK_ELT9:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 8
+; CHECK-NEXT:    [[DOTUNPACK_UNPACK10:%.*]] = load float, ptr [[DOTUNPACK_ELT9]], align 8
+; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 12
+; CHECK-NEXT:    [[DOTUNPACK2_UNPACK:%.*]] = load float, ptr [[DOTELT1]], align 4
+; CHECK-NEXT:    br label %[[END:.*]]
+; CHECK:       [[BRANCH2]]:
+; CHECK-NEXT:    [[DOTUNPACK2_ELT12:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 16
+; CHECK-NEXT:    [[DOTUNPACK2_UNPACK13:%.*]] = load float, ptr [[DOTUNPACK2_ELT12]], align 4
+; CHECK-NEXT:    [[DOTUNPACK2_ELT14:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 20
+; CHECK-NEXT:    [[DOTUNPACK2_UNPACK15:%.*]] = load float, ptr [[DOTUNPACK2_ELT14]], align 4
+; CHECK-NEXT:    [[DOTELT3:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 24
+; CHECK-NEXT:    [[DOTUNPACK4:%.*]] = load i32, ptr [[DOTELT3]], align 8
+; CHECK-NEXT:    [[DOTELT5:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 28
+; CHECK-NEXT:    [[DOTUNPACK6:%.*]] = load i32, ptr [[DOTELT5]], align 4
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    ret void
+;
+  br i1 %cond, label %branch1, label %branch2
+
+branch1:
+  %.unpack.unpack = load float, ptr %v, align 16
+  %.unpack.elt7 = getelementptr inbounds nuw i8, ptr %v, i64 4
+  %.unpack.unpack8 = load float, ptr %.unpack.elt7, align 4
+  %.unpack.elt9 = getelementptr inbounds nuw i8, ptr %v, i64 8
+  %.unpack.unpack10 = load float, ptr %.unpack.elt9, align 8
+  %.elt1 = getelementptr inbounds nuw i8, ptr %v, i64 12
+  %.unpack2.unpack = load float, ptr %.elt1, align 4
+  br label %end
+
+branch2:
+  %.unpack2.elt12 = getelementptr inbounds nuw i8, ptr %v, i64 16
+  %.unpack2.unpack13 = load float, ptr %.unpack2.elt12, align 4
+  %.unpack2.elt14 = getelementptr inbounds nuw i8, ptr %v, i64 20
+  %.unpack2.unpack15 = load float, ptr %.unpack2.elt14, align 4
+  %.elt3 = getelementptr inbounds nuw i8, ptr %v, i64 24
+  %.unpack4 = load i32, ptr %.elt3, align 8
+  %.elt5 = getelementptr inbounds nuw i8, ptr %v, i64 28
+  %.unpack6 = load i32, ptr %.elt5, align 4
+  br label %end
+
+end:
+  ret void
+}
+
+; ------------------------------------------------------------------------------
+; Test that we can propagate to/from negative offset GEPs
+; ------------------------------------------------------------------------------
+
+define void @prop_align_negative_offset(ptr %v) {
+; CHECK-LABEL: define void @prop_align_negative_offset(
+; CHECK-SAME: ptr [[V:%.*]]) {
+; CHECK-NEXT:    [[LOADALIGNED:%.*]] = load float, ptr [[V]], align 16
+; CHECK-NEXT:    [[GEPNEGATIVE:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 -16
+; CHECK-NEXT:    [[LOADUNALIGNED:%.*]] = load float, ptr [[GEPNEGATIVE]], align 16
+; CHECK-NEXT:    ret void
+;
+  %loadAligned= load float, ptr %v, align 16
+  %gepNegative = getelementptr inbounds nuw i8, ptr %v, i64 -16
+  %loadUnaligned = load float, ptr %gepNegative, align 4
+  ret void
+}
+
+define void @prop_align_negative_offset_2(ptr %v) {
+; CHECK-LABEL: define void @prop_align_negative_offset_2(
+; CHECK-SAME: ptr [[V:%.*]]) {
+; CHECK-NEXT:    [[GEPNEGATIVE:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 -16
+; CHECK-NEXT:    [[LOADALIGNED:%.*]] = load float, ptr [[GEPNEGATIVE]], align 16
+; CHECK-NEXT:    [[LOADUNALIGNED:%.*]] = load float, ptr [[V]], align 16
+; CHECK-NEXT:    ret void
+;
+  %gepNegative = getelementptr inbounds nuw i8, ptr %v, i64 -16
+  %loadAligned = load float, ptr %gepNegative, align 16
+  %loadUnaligned= load float, ptr %v, align 4
+  ret void
+}
+
+define void @prop_align_negative_offset_3(ptr %v) {
+; CHECK-LABEL: define void @prop_align_negative_offset_3(
+; CHECK-SAME: ptr [[V:%.*]]) {
+; CHECK-NEXT:    [[LOADALIGNED:%.*]] = load float, ptr [[V]], align 16
+; CHECK-NEXT:    [[GEPNEGATIVE:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 -8
+; CHECK-NEXT:    [[LOADUNALIGNED:%.*]] = load float, ptr [[GEPNEGATIVE]], align 8
+; CHECK-NEXT:    ret void
+;
+  %loadAligned= load float, ptr %v, align 16
+  %gepNegative = getelementptr inbounds nuw i8, ptr %v, i64 -8
+  %loadUnaligned = load float, ptr %gepNegative, align 4
+  ret void
+}
+
+define void @prop_align_negative_offset_4(ptr %v) {
+; CHECK-LABEL: define void @prop_align_negative_offset_4(
+; CHECK-SAME: ptr [[V:%.*]]) {
+; CHECK-NEXT:    [[LOADALIGNED:%.*]] = load float, ptr [[V]], align 16
+; CHECK-NEXT:    [[GEPNEGATIVE:%.*]] = getelementptr inbounds nuw i8, ptr [[V]], i64 -20
+; CHECK-NEXT:    [[LOADUNALIGNED:%.*]] = load float, ptr [[GEPNEGATIVE]], align 4
+; CHECK-NEXT:    ret void
+;
+  %loadAligned= load float, ptr %v, align 16
+  %gepNegative = getelementptr inbounds nuw i8, ptr %v, i64 -20
+  %loadUnaligned = load float, ptr %gepNegative, align 4
+  ret void
+}
diff --git a/llvm/test/Transforms/Inline/AArch64/sve-alloca-merge.ll b/llvm/test/Transforms/Inline/AArch64/sve-alloca-merge.ll
index c1375cbf3493d..54daf7c3f4cc9 100644
--- a/llvm/test/Transforms/Inline/AArch64/sve-alloca-merge.ll
+++ b/llvm/test/Transforms/Inline/AArch64/sve-alloca-merge.ll
@@ -14,8 +14,8 @@ entry:
 
 define i64 @foo() {
 ; CHECK-LABEL: @foo(
-; CHECK: call void @llvm.lifetime.start.p0(i64 -1, ptr %{{.*}})
-; CHECK: call void @llvm.lifetime.end.p0(i64 -1, ptr %{{.*}})
+; CHECK: call void @llvm.lifetime.start.p0(ptr %{{.*}})
+; CHECK: call void @llvm.lifetime.end.p0(ptr %{{.*}})
 entry:
   %a = alloca <vscale x 2 x i64>, align 16
   store <vscale x 2 x i64> zeroinitializer, ptr %a, align 16
diff --git a/llvm/test/Transforms/Inline/AMDGPU/load-intrinsics.ll b/llvm/test/Transforms/Inline/AMDGPU/load-intrinsics.ll
new file mode 100644
index 0000000000000..46f53d8f82cfd
--- /dev/null
+++ b/llvm/test/Transforms/Inline/AMDGPU/load-intrinsics.ll
@@ -0,0 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals all --version 5
+; RUN: opt -mtriple=amdgcn --passes=inline --enable-noalias-to-md-conversion -S %s | FileCheck --check-prefix=OPT %s
+
+; This test tests if the load intrinsic gets correct memory(argmem: read) attribute and
+; the call instruction is assigned correct !alias.scope metadata post inlining
+
+define void @caller(ptr addrspace(3) %addr_f, ptr addrspace(1) %use_f) {
+; OPT-LABEL: define void @caller(
+; OPT-SAME: ptr addrspace(3) [[ADDR_F:%.*]], ptr addrspace(1) [[USE_F:%.*]]) {
+; OPT-NEXT:  [[ENTRY:.*:]]
+; OPT-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
+; OPT-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META3:![0-9]+]])
+; OPT-NEXT:    [[GEP_I:%.*]] = getelementptr i64, ptr addrspace(3) [[ADDR_F]], i32 4
+; OPT-NEXT:    [[VAL_I:%.*]] = call <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32(ptr addrspace(3) [[GEP_I]]), !alias.scope [[META0]], !noalias [[META3]]
+; OPT-NEXT:    store <2 x i32> [[VAL_I]], ptr addrspace(1) [[USE_F]], align 8, !alias.scope [[META3]], !noalias [[META0]]
+; OPT-NEXT:    ret void
+;
+entry:
+  call void @callee(ptr addrspace(3) %addr_f, ptr addrspace(1) %use_f)
+  ret void
+}
+
+define void @callee(ptr addrspace(3) noalias %addr, ptr addrspace(1) noalias %use) {
+; OPT-LABEL: define void @callee(
+; OPT-SAME: ptr addrspace(3) noalias [[ADDR:%.*]], ptr addrspace(1) noalias [[USE:%.*]]) {
+; OPT-NEXT:  [[ENTRY:.*:]]
+; OPT-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr addrspace(3) [[ADDR]], i32 4
+; OPT-NEXT:    [[VAL:%.*]] = call <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32(ptr addrspace(3) [[GEP]])
+; OPT-NEXT:    store <2 x i32> [[VAL]], ptr addrspace(1) [[USE]], align 8
+; OPT-NEXT:    ret void
+;
+entry:
+  %gep = getelementptr i64, ptr addrspace(3) %addr, i32 4
+  %val = call <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32.p3(ptr addrspace(3) %gep)
+  store <2 x i32> %val, ptr addrspace(1) %use
+  ret void
+}
+;.
+; Check Function Attribute on decl
+; OPT: declare <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32(ptr addrspace(3) captures(none)) #[[ATTR0:[0-9]+]]
+declare <2 x i32> @llvm.amdgcn.ds.read.tr4.b64.v2i32(ptr addrspace(3))
+; OPT: attributes #[[ATTR0]] = { convergent nocallback nofree nounwind willreturn memory(argmem: read) }
+; OPT: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite) }
+;.
+; OPT: [[META0]] = !{[[META1:![0-9]+]]}
+; OPT: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]], !"callee: %addr"}
+; OPT: [[META2]] = distinct !{[[META2]], !"callee"}
+; OPT: [[META3]] = !{[[META4:![0-9]+]]}
+; OPT: [[META4]] = distinct !{[[META4]], [[META2]], !"callee: %use"}
+;.
diff --git a/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll b/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll
index 0cd61c8a232d9..c5842a1cfb5fd 100644
--- a/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll
+++ b/llvm/test/Transforms/Inline/ML/bounds-checks-rewards.ll
@@ -13,20 +13,20 @@
 ; RUN: %python %S/../../../../lib/Analysis/models/saved-model-to-tflite.py %t_savedmodel %t
 ;
 ; When the bounds are very wide ("no bounds"), all inlinings happen.
-; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%t -training-log=%t1 -enable-ml-inliner=development -ml-advisor-size-increase-threshold=10.0 -S < %s | FileCheck %s --check-prefixes=NOBOUNDS-OUT,CHECK
+; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-model-under-training=%t -training-log=%t1 -enable-ml-inliner=development -ml-advisor-size-increase-threshold=10.0 -S < %s | FileCheck %s --check-prefixes=NOBOUNDS-OUT,CHECK
 ; RUN: %python %S/../../../../lib/Analysis/models/log_reader.py %t1 | FileCheck %s --check-prefix=NOBOUNDS
 ;
 ; When the bounds are very restrictive, the first inlining happens but it's
 ; considered as "bad" (since it trips over the bounds) and its reward is a
 ; penalty. However, the mandatory inlining, which is considered next, happens.
 ; No other inlinings happend.
-; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%t -training-log=%t2 -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.0 -S < %s | FileCheck %s --check-prefixes=BOUNDS-OUT,CHECK
+; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-model-under-training=%t -training-log=%t2 -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.0 -S < %s | FileCheck %s --check-prefixes=BOUNDS-OUT,CHECK
 ; RUN: %python %S/../../../../lib/Analysis/models/log_reader.py %t2 | FileCheck %s --check-prefix=BOUNDS
 ;
 ; With more restrictive bounds, the first inlining happens and is OK. The
 ; mandatory inlining happens next, and it trips over the bounds, which then
 ; forces no further inlinings.
-; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-model-under-training=%t -training-log=%t3 -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.1 -S < %s | FileCheck %s --check-prefixes=RELAXED-BOUNDS-OUT,CHECK
+; RUN: opt -passes=scc-oz-module-inliner -ml-inliner-model-under-training=%t -training-log=%t3 -enable-ml-inliner=development -ml-advisor-size-increase-threshold=1.2 -S < %s | FileCheck %s --check-prefixes=RELAXED-BOUNDS-OUT,CHECK
 ; RUN: %python %S/../../../../lib/Analysis/models/log_reader.py %t3 | FileCheck %s --check-prefix=RELAXED-BOUNDS
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -52,15 +52,16 @@ define i64 @top() {
 }
 attributes #0 = { alwaysinline }
 ; NOBOUNDS: observation: 0
-; NOBOUNDS: delta_size: 6
-; RELAXED-BOUNDS: delta_size: 6
-; BOUNDS: delta_size: 2147483647
+; NOBOUNDS: inlining_decision: 1
+; RELAXED-BOUNDS: inlining_decision: 1
+; BOUNDS: inlining_decision: 1
 ; NOBOUNDS: observation: 1
-; BOUNDS-NOT: observation
-; RELAXED-BOUNDS-NOT: observation
-; NOBOUNDS: delta_size: -11
+; BOUNDS-NOT: observation: 1
+; RELAXED-BOUNDS: observation: 1
+; NOBOUNDS: inlining_decision: 1
 ; NOBOUNDS: observation: 2
-; NOBOUNDS: delta_size: 4
+; NOBOUNDS: inlining_decision
+; RELAXED-BOUNDS-NOT: observation: 2
 
 ; CHECK-LABEL: @top
 ; must_be_inlined must always be inlined, so we won't find a call to it in @top()
diff --git a/llvm/test/Transforms/Inline/ML/development-training-log.ll b/llvm/test/Transforms/Inline/ML/development-training-log.ll
index b53b6bc95ef11..c8b5198bb8387 100644
--- a/llvm/test/Transforms/Inline/ML/development-training-log.ll
+++ b/llvm/test/Transforms/Inline/ML/development-training-log.ll
@@ -5,12 +5,6 @@
 ; RUN: %python %S/../../../../lib/Analysis/models/gen-inline-oz-test-model.py %t_savedmodel
 ; RUN: %python %S/../../../../lib/Analysis/models/saved-model-to-tflite.py %t_savedmodel %t
 ;
-; RUN: opt -enable-ml-inliner=development -passes=scc-oz-module-inliner -training-log=%t1 -ml-inliner-model-under-training=%t -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -S < %s
-; RUN: %python %S/../../../../lib/Analysis/models/log_reader.py %t1 | FileCheck %s
-; RUN: opt -enable-ml-inliner=development -passes=scc-oz-module-inliner -training-log=%t2 -ml-inliner-model-under-training=%t -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -ml-inliner-output-spec-override=%S/Inputs/test_output_spec.json -S < %s
-; RUN: %python %S/../../../../lib/Analysis/models/log_reader.py %t2 | FileCheck %s --check-prefixes=EXTRA-OUTPUTS,CHECK
-; RUN: opt -enable-ml-inliner=development -passes=scc-oz-module-inliner -training-log=%t3 -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -S < %s
-; RUN: %python %S/../../../../lib/Analysis/models/log_reader.py %t3 | FileCheck %s
 ; RUN: opt -enable-ml-inliner=development -passes=scc-oz-module-inliner -training-log=%t4 -ml-inliner-model-under-training=%t -S < %s
 ; RUN: %python %S/../../../../lib/Analysis/models/log_reader.py %t4 | FileCheck %s --check-prefix=NOREWARD
 ; RUN: opt -enable-ml-inliner=development -passes=scc-oz-module-inliner -training-log=%t5 -S < %s
diff --git a/llvm/test/Transforms/Inline/ML/size-estimator-default.ll b/llvm/test/Transforms/Inline/ML/size-estimator-default.ll
deleted file mode 100644
index c9e5978ccfc39..0000000000000
--- a/llvm/test/Transforms/Inline/ML/size-estimator-default.ll
+++ /dev/null
@@ -1,4 +0,0 @@
-; REQUIRES: !have_tflite
-; RUN: opt -passes='print<inliner-size-estimator>' -S < %S/Inputs/size-estimator.ll 2>&1 | FileCheck %s
-
-; CHECK: [InlineSizeEstimatorAnalysis] size estimate for branches: None
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/ML/size-estimator-training.ll b/llvm/test/Transforms/Inline/ML/size-estimator-training.ll
deleted file mode 100644
index 462cc6459e333..0000000000000
--- a/llvm/test/Transforms/Inline/ML/size-estimator-training.ll
+++ /dev/null
@@ -1,6 +0,0 @@
-; REQUIRES: have_tflite
-; RUN: opt -passes='print<inliner-size-estimator>' -S < %S/Inputs/size-estimator.ll 2>&1 | FileCheck %s --check-prefix=DEFAULT
-; RUN: opt -passes='print<inliner-size-estimator>' -ml-inliner-ir2native-model=%S/../../../../unittests/Analysis/Inputs/ir2native_x86_64_model -S < %S/Inputs/size-estimator.ll 2>&1 | FileCheck %s
-
-; DEFAULT: [InlineSizeEstimatorAnalysis] size estimate for branches: None
-; CHECK: [InlineSizeEstimatorAnalysis] size estimate for branches: 28
\ No newline at end of file
diff --git a/llvm/test/Transforms/Inline/ML/state-tracking-coro.ll b/llvm/test/Transforms/Inline/ML/state-tracking-coro.ll
index bbce9e43b10c9..c3f6dd700b451 100644
--- a/llvm/test/Transforms/Inline/ML/state-tracking-coro.ll
+++ b/llvm/test/Transforms/Inline/ML/state-tracking-coro.ll
@@ -31,10 +31,10 @@ entry:
 await.ready:
   %StrayCoroSave = call token @llvm.coro.save(ptr null)
   %val = load i32, ptr %ref.tmp7
-  call void @llvm.lifetime.start.p0(i64 4, ptr %testval)
+  call void @llvm.lifetime.start.p0(ptr %testval)
   %test = load i32, ptr %testval
   call void @print(i32 %test)
-  call void @llvm.lifetime.end.p0(i64 4, ptr  %testval)
+  call void @llvm.lifetime.end.p0(ptr  %testval)
   call void @print(i32 %val)
   br label %exit
 exit:
@@ -54,5 +54,5 @@ declare i8 @llvm.coro.suspend(token, i1) #3
 declare void @"\01??3@YAXPEAX@Z"(ptr) local_unnamed_addr #10
 declare ptr @llvm.coro.free(token, ptr nocapture readonly) #2
 declare i1 @llvm.coro.end(ptr, i1) #3
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #4
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
diff --git a/llvm/test/Transforms/Inline/SystemZ/inline-target-attr.ll b/llvm/test/Transforms/Inline/SystemZ/inline-target-attr.ll
index b5c4f42655bb4..71b463b2d2b00 100644
--- a/llvm/test/Transforms/Inline/SystemZ/inline-target-attr.ll
+++ b/llvm/test/Transforms/Inline/SystemZ/inline-target-attr.ll
@@ -12,28 +12,28 @@ entry:
 
 declare i32 @baz(...) #0
 
-define i32 @bar() #1 {
+define i32 @features_subset() #1 {
 entry:
   %call = call i32 @foo()
   ret i32 %call
-; CHECK-LABEL: bar
-; CHECK: call i32 @foo()
+; CHECK-LABEL: features_subset
+; CHECK: call i32 (...) @baz()
 }
 
-define i32 @qux() #0 {
+define i32 @features_equal() #0 {
 entry:
   %call = call i32 @foo()
   ret i32 %call
-; CHECK-LABEL: qux
+; CHECK-LABEL: features_equal
 ; CHECK: call i32 (...) @baz()
 }
 
-define i32 @quux() #2 {
+define i32 @features_different() #2 {
 entry:
-  %call = call i32 @bar()
+  %call = call i32 @foo()
   ret i32 %call
-; CHECK-LABEL: quux
-; CHECK: call i32 @bar()
+; CHECK-LABEL: features_different
+; CHECK: call i32 @foo()
 }
 
 
diff --git a/llvm/test/Transforms/Inline/X86/call-abi-compatibility.ll b/llvm/test/Transforms/Inline/X86/call-abi-compatibility.ll
index 6f582cab2f145..9d0981cd63756 100644
--- a/llvm/test/Transforms/Inline/X86/call-abi-compatibility.ll
+++ b/llvm/test/Transforms/Inline/X86/call-abi-compatibility.ll
@@ -34,8 +34,7 @@ define i64 @callee_not_avx(<4 x i64> %arg) noinline {
   ret i64 %v
 }
 
-; This call also shouldn't be inlined, as we don't know whether callee_unknown
-; is ABI compatible or not.
+; This call also shouldn't be inlined, as caller_not_avx2 is not ABI compatible.
 define void @caller_avx2() "target-features"="+avx" {
 ; CHECK-LABEL: define {{[^@]+}}@caller_avx2
 ; CHECK-SAME: () #[[ATTR0]] {
@@ -55,6 +54,29 @@ define internal void @caller_not_avx2() {
   ret void
 }
 
+; Should be inlined, as caller_avx7 is ABI compatible. The fact that we don't
+; know anything about callee_unknown doesn't matter, as it is the caller that
+; determines the ABI as far as target features are concerned.
+define void @caller_avx6() "target-features"="+avx" {
+; CHECK-LABEL: define {{[^@]+}}@caller_avx6
+; CHECK-SAME: () #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @callee_unknown(<4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+; CHECK-NEXT:    ret void
+;
+  call void @caller_avx7()
+  ret void
+}
+
+define void @caller_avx7() "target-features"="+avx" {
+; CHECK-LABEL: define {{[^@]+}}@caller_avx7
+; CHECK-SAME: () #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @callee_unknown(<4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+; CHECK-NEXT:    ret void
+;
+  call i64 @callee_unknown(<4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+  ret void
+}
+
 declare i64 @callee_unknown(<4 x i64>)
 
 ; This call should get inlined, because we assume that intrinsics are always
@@ -62,20 +84,24 @@ declare i64 @callee_unknown(<4 x i64>)
 define void @caller_avx3() "target-features"="+avx" {
 ; CHECK-LABEL: define {{[^@]+}}@caller_avx3
 ; CHECK-SAME: () #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.some_intrinsic(<4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+; CHECK-NEXT:    [[V_I:%.*]] = load <4 x i64>, ptr @g, align 32
+; CHECK-NEXT:    [[V2_I:%.*]] = call <4 x i64> @llvm.abs.v4i64(<4 x i64> [[V_I]], i1 false)
+; CHECK-NEXT:    store <4 x i64> [[V2_I]], ptr @g, align 32
 ; CHECK-NEXT:    ret void
 ;
   call void @caller_not_avx3()
   ret void
 }
 
+@g = external global <4 x i64>
+
 define internal void @caller_not_avx3() {
-  call i64 @llvm.some_intrinsic(<4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+  %v = load <4 x i64>, ptr @g
+  %v2 = call <4 x i64> @llvm.abs(<4 x i64> %v, i1 false)
+  store <4 x i64> %v2, ptr @g
   ret void
 }
 
-declare i64 @llvm.some_intrinsic(<4 x i64>)
-
 ; This call should get inlined, because only simple types are involved.
 define void @caller_avx4() "target-features"="+avx" {
 ; CHECK-LABEL: define {{[^@]+}}@caller_avx4
diff --git a/llvm/test/Transforms/Inline/access-attributes-prop.ll b/llvm/test/Transforms/Inline/access-attributes-prop.ll
index 28fa44ecc5b13..fed2590d36669 100644
--- a/llvm/test/Transforms/Inline/access-attributes-prop.ll
+++ b/llvm/test/Transforms/Inline/access-attributes-prop.ll
@@ -410,9 +410,9 @@ define void @prop_fn_decl_fail_alloca(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@prop_fn_decl_fail_alloca
 ; CHECK-SAME: (ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[A_I:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @bar2(ptr [[P]], ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @bar1(ptr [[P]])
 ; CHECK-NEXT:    ret void
 ;
@@ -425,9 +425,9 @@ define void @prop_cb_def_wr_fail_alloca(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@prop_cb_def_wr_fail_alloca
 ; CHECK-SAME: (ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[A_I:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @bar2(ptr [[P]], ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @bar1(ptr [[P]])
 ; CHECK-NEXT:    ret void
 ;
@@ -440,10 +440,10 @@ define void @prop_fn_decl_partially_okay_alloca(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@prop_fn_decl_partially_okay_alloca
 ; CHECK-SAME: (ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[A_I:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @bar1(ptr [[P]])
 ; CHECK-NEXT:    call void @bar2(ptr [[P]], ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @bar1(ptr [[P]])
 ; CHECK-NEXT:    ret void
 ;
@@ -456,10 +456,10 @@ define void @prop_cb_def_wr_partially_okay_alloca(ptr %p) {
 ; CHECK-LABEL: define {{[^@]+}}@prop_cb_def_wr_partially_okay_alloca
 ; CHECK-SAME: (ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[A_I:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @bar1(ptr [[P]])
 ; CHECK-NEXT:    call void @bar2(ptr [[P]], ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @bar1(ptr [[P]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/Inline/byval-align.ll b/llvm/test/Transforms/Inline/byval-align.ll
index 0b135aa570a72..a23f3640abb13 100644
--- a/llvm/test/Transforms/Inline/byval-align.ll
+++ b/llvm/test/Transforms/Inline/byval-align.ll
@@ -28,13 +28,13 @@ define void @byval_caller(ptr nocapture align 64 %a, ptr %b) #0 {
 ; CHECK-SAME: (ptr align 64 captures(none) [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A1:%.*]] = alloca float, align 128
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A1]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 128 [[A1]], ptr align 128 [[A]], i64 4, i1 false)
 ; CHECK-NEXT:    [[LOAD_I:%.*]] = load float, ptr [[A1]], align 4
 ; CHECK-NEXT:    [[B_IDX_I:%.*]] = getelementptr inbounds float, ptr [[B]], i64 8
 ; CHECK-NEXT:    [[ADD_I:%.*]] = fadd float [[LOAD_I]], 2.000000e+00
 ; CHECK-NEXT:    store float [[ADD_I]], ptr [[B_IDX_I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A1]])
 ; CHECK-NEXT:    [[CALLER_LOAD:%.*]] = load float, ptr [[B]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 7
 ; CHECK-NEXT:    store float [[CALLER_LOAD]], ptr [[ARRAYIDX]], align 4
diff --git a/llvm/test/Transforms/Inline/byval-tail-call.ll b/llvm/test/Transforms/Inline/byval-tail-call.ll
index 808104c591791..f8fd4a646aa1d 100644
--- a/llvm/test/Transforms/Inline/byval-tail-call.ll
+++ b/llvm/test/Transforms/Inline/byval-tail-call.ll
@@ -22,11 +22,11 @@ define void @bar(ptr byval(i32) %x) {
 define void @foo(ptr %x) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:    [[X1:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X:%.*]], align 1
 ; CHECK-NEXT:    store i32 [[TMP2]], ptr [[X1]], align 4
 ; CHECK-NEXT:    call void @ext(ptr nonnull [[X1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X1]])
 ; CHECK-NEXT:    ret void
 ;
   call void @bar(ptr byval(i32) %x)
@@ -42,12 +42,12 @@ define internal void @qux(ptr byval(i32) %x) {
 define void @frob(ptr %x) {
 ; CHECK-LABEL: @frob(
 ; CHECK-NEXT:    [[X1:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X:%.*]], align 1
 ; CHECK-NEXT:    store i32 [[TMP2]], ptr [[X1]], align 4
 ; CHECK-NEXT:    call void @ext(ptr nonnull [[X1]])
 ; CHECK-NEXT:    tail call void @ext(ptr null)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X1]])
 ; CHECK-NEXT:    ret void
 ;
   tail call void @qux(ptr byval(i32) %x)
@@ -71,11 +71,11 @@ define void @bar2(ptr byval(i32) %x) {
 define void @foobar(ptr %x) {
 ; CHECK-LABEL: @foobar(
 ; CHECK-NEXT:    [[X1:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[X:%.*]], align 1
 ; CHECK-NEXT:    store i32 [[TMP2]], ptr [[X1]], align 4
 ; CHECK-NEXT:    tail call void @ext2(ptr nonnull byval(i32) [[X1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X1]])
 ; CHECK-NEXT:    ret void
 ;
   tail call void @bar2(ptr byval(i32) %x)
@@ -85,9 +85,9 @@ define void @foobar(ptr %x) {
 define void @barfoo() {
 ; CHECK-LABEL: @barfoo(
 ; CHECK-NEXT:    [[X1:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X1]])
 ; CHECK-NEXT:    tail call void @ext2(ptr nonnull byval(i32) [[X1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X1]])
 ; CHECK-NEXT:    ret void
 ;
   %x = alloca i32
diff --git a/llvm/test/Transforms/Inline/byval-with-non-alloca-addrspace.ll b/llvm/test/Transforms/Inline/byval-with-non-alloca-addrspace.ll
index 1d1cb459d53b6..e79ac66423d12 100644
--- a/llvm/test/Transforms/Inline/byval-with-non-alloca-addrspace.ll
+++ b/llvm/test/Transforms/Inline/byval-with-non-alloca-addrspace.ll
@@ -26,11 +26,11 @@ define i64 @foo(ptr %arg) {
 ; CHECK-LABEL: define i64 @foo(
 ; CHECK-SAME: ptr [[ARG:%.*]]) {
 ; CHECK-NEXT:    [[ARG1:%.*]] = alloca [[STRUCT:%.*]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[ARG1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ARG1]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[ARG1]], ptr align 8 [[ARG]], i64 16, i1 false)
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT]], ptr [[ARG1]], i64 0, i32 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[ARG1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ARG1]])
 ; CHECK-NEXT:    ret i64 0
 ;
   %1 = call i64 @bar(ptr byval(%struct) align 8 %arg)
diff --git a/llvm/test/Transforms/Inline/byval.ll b/llvm/test/Transforms/Inline/byval.ll
index b4a19c55c20a0..c945d7fe1a01a 100644
--- a/llvm/test/Transforms/Inline/byval.ll
+++ b/llvm/test/Transforms/Inline/byval.ll
@@ -35,12 +35,12 @@ define i32 @test1() nounwind  {
 ; CHECK-NEXT:    store i32 1, ptr [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
 ; CHECK-NEXT:    store i64 2, ptr [[TMP4]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr [[S1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[S1]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[S1]], ptr [[S]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP1_I:%.*]] = load i32, ptr [[S1]], align 4
 ; CHECK-NEXT:    [[TMP2_I:%.*]] = add i32 [[TMP1_I]], 1
 ; CHECK-NEXT:    store i32 [[TMP2_I]], ptr [[S1]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr [[S1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[S1]])
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -104,10 +104,10 @@ define void @test3() nounwind  {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[S1:%.*]] = alloca [[STRUCT_SS:%.*]], align 64
 ; CHECK-NEXT:    [[S:%.*]] = alloca [[STRUCT_SS]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr [[S1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[S1]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 64 [[S1]], ptr align 64 [[S]], i64 12, i1 false)
 ; CHECK-NEXT:    call void @g3(ptr align 64 [[S1]]) #[[ATTR0]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr [[S1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[S1]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -157,12 +157,12 @@ define i32 @test5() {
 ; CHECK-LABEL: define i32 @test5() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[B:%.*]] = alloca [[STRUCT_S0:%.*]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[B]], ptr align 4 @b, i64 4, i1 false)
 ; CHECK-NEXT:    store i32 0, ptr @b, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[B]], align 4
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr @a, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr @a, align 4
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
diff --git a/llvm/test/Transforms/Inline/callbr.ll b/llvm/test/Transforms/Inline/callbr.ll
index 160770051c368..57e92bb2a6774 100644
--- a/llvm/test/Transforms/Inline/callbr.ll
+++ b/llvm/test/Transforms/Inline/callbr.ll
@@ -10,8 +10,8 @@ define dso_local i32 @main() {
 ; CHECK-NEXT:    [[I1_I:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I1_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I1_I]])
 ; CHECK-NEXT:    store i32 0, ptr [[I1_I]], align 4
 ; CHECK-NEXT:    [[I2_I:%.*]] = load i32, ptr [[I1_I]], align 4
 ; CHECK-NEXT:    callbr void asm sideeffect "", "r,!i,!i,~{dirflag},~{fpsr},~{flags}"(i32 [[I2_I]])
@@ -27,8 +27,8 @@ define dso_local i32 @main() {
 ; CHECK-NEXT:    br label [[T32_EXIT]]
 ; CHECK:       t32.exit:
 ; CHECK-NEXT:    [[I7_I:%.*]] = load i32, ptr [[I_I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I1_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I1_I]])
 ; CHECK-NEXT:    ret i32 [[I7_I]]
 ;
 bb:
diff --git a/llvm/test/Transforms/Inline/devirtualize-4.ll b/llvm/test/Transforms/Inline/devirtualize-4.ll
index d29360f73b473..f96b5a9cdcab4 100644
--- a/llvm/test/Transforms/Inline/devirtualize-4.ll
+++ b/llvm/test/Transforms/Inline/devirtualize-4.ll
@@ -48,14 +48,14 @@
 define dso_local void @_Z4Testv() local_unnamed_addr {
 entry:
   %o = alloca %class.Impl, align 8
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %o)
+  call void @llvm.lifetime.start.p0(ptr nonnull %o)
   call void @_ZN4ImplC2Ev(ptr nonnull %o)
   call fastcc void @_ZL11IndirectRunR9Interface(ptr nonnull dereferenceable(8) %o)
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %o)
+  call void @llvm.lifetime.end.p0(ptr nonnull %o)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 define linkonce_odr dso_local void @_ZN4ImplC2Ev(ptr %this) unnamed_addr align 2 {
 entry:
@@ -74,7 +74,7 @@ entry:
   ret void
 }
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define linkonce_odr dso_local void @_ZN9InterfaceC2Ev(ptr %this) unnamed_addr align 2 {
 entry:
@@ -85,10 +85,10 @@ entry:
 define linkonce_odr dso_local void @_ZN4Impl3RunEv(ptr %this) unnamed_addr align 2 {
 entry:
   %ref.tmp = alloca ptr, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %ref.tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %ref.tmp)
   store ptr %this, ptr %ref.tmp, align 8
   call void @_Z13DoNotOptimizeIP4ImplEvRKT_(ptr nonnull dereferenceable(8) %ref.tmp)
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %ref.tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %ref.tmp)
   ret void
 }
 
@@ -160,10 +160,10 @@ memptr.end:                                       ; preds = %memptr.nonvirtual,
 define i32 @_Z2g1v() {
 entry:
   %a = alloca %struct.A, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call void @_ZN1AC1Ev(ptr nonnull %a)
   %call = call i32 @_Z1fP1AMS_FivE(ptr nonnull %a, i64 1, i64 0)
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret i32 %call
 }
 
@@ -176,10 +176,10 @@ entry:
 define i32 @_Z2g2v() {
 entry:
   %a = alloca %struct.A, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   call void @_ZN1AC1Ev(ptr nonnull %a)
   %call = call i32 @_Z1fP1AMS_FivE(ptr nonnull %a, i64 9, i64 0)
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret i32 %call
 }
 
diff --git a/llvm/test/Transforms/Inline/dynamic-alloca-simplified-large.ll b/llvm/test/Transforms/Inline/dynamic-alloca-simplified-large.ll
index 9b293d39c85f9..aad192a84fe24 100644
--- a/llvm/test/Transforms/Inline/dynamic-alloca-simplified-large.ll
+++ b/llvm/test/Transforms/Inline/dynamic-alloca-simplified-large.ll
@@ -54,9 +54,9 @@ define void @caller2_below_threshold(ptr %p1, i1 %b) {
 ; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[SPLIT:%.*]]
 ; CHECK:       split:
 ; CHECK-NEXT:    [[SAVEDSTACK:%.*]] = call ptr @llvm.stacksave.p0()
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 60000, ptr [[VLA_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[VLA_I]])
 ; CHECK-NEXT:    call void @extern_call(ptr nonnull [[VLA_I]]) #[[ATTR3]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 60000, ptr [[VLA_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[VLA_I]])
 ; CHECK-NEXT:    call void @llvm.stackrestore.p0(ptr [[SAVEDSTACK]])
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
diff --git a/llvm/test/Transforms/Inline/inline-all-viable-calls.ll b/llvm/test/Transforms/Inline/inline-all-viable-calls.ll
new file mode 100644
index 0000000000000..a06ec1acd4ef3
--- /dev/null
+++ b/llvm/test/Transforms/Inline/inline-all-viable-calls.ll
@@ -0,0 +1,114 @@
+; RUN: opt -passes=inline -inline-threshold=0 -inline-all-viable-calls -S < %s | FileCheck %s
+
+; Check that viable calls that are beyond the cost threshold are still inlined.
+define i32 @callee_simple(i32 %x) {
+  %1 = add i32 %x, 1
+  %2 = mul i32 %1, 2
+  %3 = sub i32 %2, 1
+  %4 = add i32 %3, 3
+  %5 = mul i32 %4, 2
+  %6 = sub i32 %5, 2
+  %7 = add i32 %6, 1
+  ret i32 %7
+}
+
+; Check that user decisions are respected.
+define i32 @callee_alwaysinline(i32 %x) alwaysinline {
+  %sub = sub i32 %x, 3
+  ret i32 %sub
+}
+
+define i32 @callee_noinline(i32 %x) noinline {
+  %div = sdiv i32 %x, 2
+  ret i32 %div
+}
+
+define i32 @callee_optnone(i32 %x) optnone noinline {
+  %rem = srem i32 %x, 2
+  ret i32 %rem
+}
+
+define i32 @caller(i32 %a) {
+; CHECK-LABEL: define i32 @caller(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[A]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i32 [[TMP8]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i32 [[TMP5]], 2
+; CHECK-NEXT:    [[ADD_I:%.*]] = add i32 [[TMP6]], 1
+; CHECK-NEXT:    [[SUB_I:%.*]] = sub i32 [[ADD_I]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @callee_noinline(i32 [[SUB_I]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @callee_optnone(i32 [[TMP1]])
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[SUM]]
+;
+  %1 = call i32 @callee_simple(i32 %a)
+  %2 = call i32 @callee_alwaysinline(i32 %1)
+  %3 = call i32 @callee_noinline(i32 %2)
+  %4 = call i32 @callee_optnone(i32 %3)
+  %sum = add i32 %4, %3
+  ret i32 %sum
+}
+
+; Check that non-viable calls are not inlined
+
+; Test recursive function is not inlined
+define i32 @recursive(i32 %n) {
+entry:
+  %cmp = icmp eq i32 %n, 0
+  br i1 %cmp, label %base, label %recurse
+
+base:
+  ret i32 0
+
+recurse:
+  %dec = sub i32 %n, 1
+  %rec = call i32 @recursive(i32 %dec)
+  %add = add i32 %rec, 1
+  ret i32 %add
+}
+
+define i32 @call_recursive(i32 %x) {
+; CHECK-LABEL: define i32 @call_recursive(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call i32 @recursive(i32 [[X]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %r = call i32 @recursive(i32 %x)
+  ret i32 %r
+}
+
+; Test indirectbr prevents inlining
+define void @has_indirectbr(ptr %ptr, i32 %cond) {
+entry:
+  switch i32 %cond, label %default [
+  i32 0, label %target0
+  i32 1, label %target1
+  ]
+
+target0:
+  br label %end
+
+target1:
+  br label %end
+
+default:
+  br label %end
+
+end:
+  indirectbr ptr %ptr, [label %target0, label %target1]
+  ret void
+}
+
+define void @call_indirectbr(ptr %p, i32 %c) {
+; CHECK-LABEL: define void @call_indirectbr(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:    call void @has_indirectbr(ptr [[P]], i32 [[C]])
+; CHECK-NEXT:    ret void
+;
+  call void @has_indirectbr(ptr %p, i32 %c)
+  ret void
+}
+
diff --git a/llvm/test/Transforms/Inline/inline-deferred-instsimplify.ll b/llvm/test/Transforms/Inline/inline-deferred-instsimplify.ll
index c74351b300399..02f5774d05076 100644
--- a/llvm/test/Transforms/Inline/inline-deferred-instsimplify.ll
+++ b/llvm/test/Transforms/Inline/inline-deferred-instsimplify.ll
@@ -52,7 +52,7 @@ return:                                           ; preds = %check_pointers_are_
 define i32 @main() {
 ; CHECK-LABEL: define i32 @main() {
 ; CHECK-NEXT:    [[G_VAR:%.*]] = alloca [[STRUCT_A:%.*]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr [[G_VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[G_VAR]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[G_VAR]], ptr align 8 @g_var, i64 20, i1 false)
 ; CHECK-NEXT:    [[VAL_I:%.*]] = load i32, ptr [[G_VAR]], align 8
 ; CHECK-NEXT:    [[DOTNOT_I:%.*]] = icmp eq i32 [[VAL_I]], 0
@@ -68,7 +68,7 @@ define i32 @main() {
 ; CHECK-NEXT:    call void @abort()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       callee.exit:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr [[G_VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[G_VAR]])
 ; CHECK-NEXT:    ret i32 0
 ;
   call void @callee(ptr noundef byval(%struct.a) align 8 @g_var)
diff --git a/llvm/test/Transforms/Inline/inline-tail.ll b/llvm/test/Transforms/Inline/inline-tail.ll
index 0bfd0565eef57..b2bf3bbdf26b3 100644
--- a/llvm/test/Transforms/Inline/inline-tail.ll
+++ b/llvm/test/Transforms/Inline/inline-tail.ll
@@ -64,7 +64,7 @@ define void @test_byval_a(ptr byval(i32) %p) {
 ; CHECK-LABEL: define void @test_byval_a
 ; CHECK-SAME: (ptr byval(i32) [[P:%.*]]) {
 ; CHECK-NEXT:    [[P1:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[P1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P1]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[P1]], ptr [[P]], i64 4, i1 false)
 ; CHECK-NEXT:    musttail call void @test_byval_c(ptr byval(i32) [[P1]])
 ; CHECK-NEXT:    ret void
@@ -87,7 +87,7 @@ define void @test_dynalloca_a(ptr byval(i32) %p, i32 %n) {
 ; CHECK-SAME: (ptr byval(i32) [[P:%.*]], i32 [[N:%.*]]) {
 ; CHECK-NEXT:    [[P1:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[SAVEDSTACK:%.*]] = call ptr @llvm.stacksave.p0()
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[P1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P1]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[P1]], ptr [[P]], i64 4, i1 false)
 ; CHECK-NEXT:    [[BUF_I:%.*]] = alloca i8, i32 [[N]], align 1
 ; CHECK-NEXT:    call void @escape(ptr [[BUF_I]])
diff --git a/llvm/test/Transforms/Inline/inlined-mustprogress-loop-metadata.ll b/llvm/test/Transforms/Inline/inlined-mustprogress-loop-metadata.ll
index 4e13ff4e90075..4ac4675720385 100644
--- a/llvm/test/Transforms/Inline/inlined-mustprogress-loop-metadata.ll
+++ b/llvm/test/Transforms/Inline/inlined-mustprogress-loop-metadata.ll
@@ -312,9 +312,9 @@ define void @caller_multiple(i32 %a, i32 %b) #1 {
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[I]], align 4
 ; CHECK-NEXT:    br label %[[FOR_COND1]]
 ; CHECK:       [[FOR_END4]]:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A_ADDR_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[B_ADDR_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_ADDR_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_ADDR_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I_I]])
 ; CHECK-NEXT:    store i32 0, ptr [[A_ADDR_I]], align 4
 ; CHECK-NEXT:    store i32 5, ptr [[B_ADDR_I]], align 4
 ; CHECK-NEXT:    br label %[[FOR_COND_I:.*]]
@@ -526,9 +526,9 @@ define void @caller_nested(i32 %a, i32 %b) #1 {
 ; CHECK-NEXT:    store i32 [[INC14]], ptr [[I9]], align 4
 ; CHECK-NEXT:    br label %[[FOR_COND10]]
 ; CHECK:       [[FOR_END15]]:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A_ADDR_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[B_ADDR_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_ADDR_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B_ADDR_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I_I]])
 ; CHECK-NEXT:    store i32 0, ptr [[A_ADDR_I]], align 4
 ; CHECK-NEXT:    store i32 5, ptr [[B_ADDR_I]], align 4
 ; CHECK-NEXT:    br label %[[FOR_COND_I:.*]]
diff --git a/llvm/test/Transforms/Inline/lifetime-no-datalayout.ll b/llvm/test/Transforms/Inline/lifetime-no-datalayout.ll
index 7438ef3b4e744..074550b41f263 100644
--- a/llvm/test/Transforms/Inline/lifetime-no-datalayout.ll
+++ b/llvm/test/Transforms/Inline/lifetime-no-datalayout.ll
@@ -18,9 +18,9 @@ define void @helper() {
 define void @test() {
 ; CHECK-LABEL: define void @test() {
 ; CHECK-NEXT:    [[A_I:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @use(ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I]])
 ; CHECK-NEXT:    ret void
 ;
   call void @helper()
diff --git a/llvm/test/Transforms/Inline/lifetime.ll b/llvm/test/Transforms/Inline/lifetime.ll
index 3ef5019303b48..06b911d20257e 100644
--- a/llvm/test/Transforms/Inline/lifetime.ll
+++ b/llvm/test/Transforms/Inline/lifetime.ll
@@ -2,21 +2,21 @@
 ; RUN: opt -passes=inline -S < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 
 define void @helper_both_markers() {
 ; CHECK-LABEL: define void @helper_both_markers() {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i8
   ; Size in llvm.lifetime.start / llvm.lifetime.end differs from
   ; allocation size. We should use the former.
-  call void @llvm.lifetime.start.p0(i64 2, ptr %a)
-  call void @llvm.lifetime.end.p0(i64 2, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -24,10 +24,10 @@ define void @test_both_markers() {
 ; CHECK-LABEL: define void @test_both_markers() {
 ; CHECK-NEXT:    [[A_I1:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[A_I:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr [[A_I1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr [[A_I1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I1]])
 ; CHECK-NEXT:    ret void
 ;
   call void @helper_both_markers()
@@ -54,12 +54,12 @@ define void @test_no_marker() {
 ; CHECK-LABEL: define void @test_no_marker() {
 ; CHECK-NEXT:    [[A_I1:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[A_I:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @use(ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[A_I1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I1]])
 ; CHECK-NEXT:    call void @use(ptr [[A_I1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[A_I1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I1]])
 ; CHECK-NEXT:    ret void
 ;
   call void @helper_no_markers()
@@ -70,13 +70,13 @@ define void @test_no_marker() {
 define void @helper_two_casts() {
 ; CHECK-LABEL: define void @helper_two_casts() {
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -84,10 +84,10 @@ define void @test_two_casts() {
 ; CHECK-LABEL: define void @test_two_casts() {
 ; CHECK-NEXT:    [[A_I1:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[A_I:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A_I1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A_I1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I1]])
 ; CHECK-NEXT:    ret void
 ;
   call void @helper_two_casts()
@@ -109,9 +109,9 @@ define void @helper_arrays_alloca() {
 define void @test_arrays_alloca() {
 ; CHECK-LABEL: define void @test_arrays_alloca() {
 ; CHECK-NEXT:    [[A_I:%.*]] = alloca [10 x i32], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 40, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_I]])
 ; CHECK-NEXT:    call void @use(ptr [[A_I]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 40, ptr [[A_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A_I]])
 ; CHECK-NEXT:    ret void
 ;
   call void @helper_arrays_alloca()
diff --git a/llvm/test/Transforms/Inline/no-inline-incompatible-gc.ll b/llvm/test/Transforms/Inline/no-inline-incompatible-gc.ll
index 531801df7cc46..2bded9ce4f916 100644
--- a/llvm/test/Transforms/Inline/no-inline-incompatible-gc.ll
+++ b/llvm/test/Transforms/Inline/no-inline-incompatible-gc.ll
@@ -16,12 +16,12 @@
 define i32 @caller_no_gc() {
 ; CHECK-LABEL: define i32 @caller_no_gc() gc "example" {
 ; CHECK-NEXT:    [[ROOT_I:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[ROOT_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ROOT_I]])
 ; CHECK-NEXT:    call void @llvm.gcroot(ptr [[ROOT_I]], ptr null)
 ; CHECK-NEXT:    [[OBJ_I:%.*]] = call ptr @h()
 ; CHECK-NEXT:    store ptr [[OBJ_I]], ptr [[ROOT_I]], align 8
 ; CHECK-NEXT:    [[LENGTH_I:%.*]] = load i32, ptr [[OBJ_I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[ROOT_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ROOT_I]])
 ; CHECK-NEXT:    ret i32 [[LENGTH_I]]
 ;
   %x = call i32 @callee_with_gc()
@@ -32,12 +32,12 @@ define i32 @caller_no_gc() {
 define i32 @caller_same_gc() gc "example" {
 ; CHECK-LABEL: define i32 @caller_same_gc() gc "example" {
 ; CHECK-NEXT:    [[ROOT_I:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[ROOT_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ROOT_I]])
 ; CHECK-NEXT:    call void @llvm.gcroot(ptr [[ROOT_I]], ptr null)
 ; CHECK-NEXT:    [[OBJ_I:%.*]] = call ptr @h()
 ; CHECK-NEXT:    store ptr [[OBJ_I]], ptr [[ROOT_I]], align 8
 ; CHECK-NEXT:    [[LENGTH_I:%.*]] = load i32, ptr [[OBJ_I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[ROOT_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ROOT_I]])
 ; CHECK-NEXT:    ret i32 [[LENGTH_I]]
 ;
   %x = call i32 @callee_with_gc()
@@ -97,12 +97,12 @@ define i32 @callee_with_other_gc() gc "other-example" {
 define i32 @caller_inline_first_caller() {
 ; CHECK-LABEL: define i32 @caller_inline_first_caller() gc "example" {
 ; CHECK-NEXT:    [[ROOT_I:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[ROOT_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ROOT_I]])
 ; CHECK-NEXT:    call void @llvm.gcroot(ptr [[ROOT_I]], ptr null)
 ; CHECK-NEXT:    [[OBJ_I:%.*]] = call ptr @h()
 ; CHECK-NEXT:    store ptr [[OBJ_I]], ptr [[ROOT_I]], align 8
 ; CHECK-NEXT:    [[LENGTH_I:%.*]] = load i32, ptr [[OBJ_I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[ROOT_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ROOT_I]])
 ; CHECK-NEXT:    [[Y:%.*]] = call i32 @callee_with_other_gc()
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[LENGTH_I]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
@@ -118,12 +118,12 @@ define i32 @caller_inline_first_caller() {
 define i32 @caller_inline_second_caller() gc "example" {
 ; CHECK-LABEL: define i32 @caller_inline_second_caller() gc "example" {
 ; CHECK-NEXT:    [[ROOT_I:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[ROOT_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ROOT_I]])
 ; CHECK-NEXT:    call void @llvm.gcroot(ptr [[ROOT_I]], ptr null)
 ; CHECK-NEXT:    [[OBJ_I:%.*]] = call ptr @h()
 ; CHECK-NEXT:    store ptr [[OBJ_I]], ptr [[ROOT_I]], align 8
 ; CHECK-NEXT:    [[LENGTH_I:%.*]] = load i32, ptr [[OBJ_I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[ROOT_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ROOT_I]])
 ; CHECK-NEXT:    [[Y:%.*]] = call i32 @callee_with_other_gc()
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[LENGTH_I]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[ADD]]
diff --git a/llvm/test/Transforms/Inline/noalias-calls-always.ll b/llvm/test/Transforms/Inline/noalias-calls-always.ll
index a80cd12b26b65..18a65b9876412 100644
--- a/llvm/test/Transforms/Inline/noalias-calls-always.ll
+++ b/llvm/test/Transforms/Inline/noalias-calls-always.ll
@@ -33,13 +33,13 @@ define void @foo(ptr nocapture %a, ptr nocapture readonly %c, ptr nocapture %b)
 ; CHECK-NEXT:    [[L_I:%.*]] = alloca i8, i32 512, align 1
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META3:![0-9]+]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 512, ptr [[L_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[L_I]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A:%.*]], ptr align 16 [[B:%.*]], i64 16, i1 false), !noalias [[META3]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[B]], ptr readonly align 16 [[C:%.*]], i64 16, i1 false), !noalias [[META0]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A]], ptr readonly align 16 [[C]], i64 16, i1 false), !alias.scope [[META5:![0-9]+]]
 ; CHECK-NEXT:    call void @hey(), !noalias [[META5]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[L_I]], ptr readonly align 16 [[C]], i64 16, i1 false), !noalias [[META0]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 512, ptr [[L_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[L_I]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -74,13 +74,13 @@ define void @foo_cs(ptr nocapture %a, ptr nocapture readonly %c, ptr nocapture %
 ; CHECK-NEXT:    [[L_I:%.*]] = alloca i8, i32 512, align 1
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META6:![0-9]+]])
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META9:![0-9]+]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 512, ptr [[L_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[L_I]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A:%.*]], ptr align 16 [[B:%.*]], i64 16, i1 false), !noalias [[META9]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[B]], ptr readonly align 16 [[C:%.*]], i64 16, i1 false), !noalias [[META6]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A]], ptr readonly align 16 [[C]], i64 16, i1 false), !alias.scope [[META11:![0-9]+]]
 ; CHECK-NEXT:    call void @hey(), !noalias [[META11]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[L_I]], ptr readonly align 16 [[C]], i64 16, i1 false), !noalias [[META6]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 512, ptr [[L_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[L_I]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/Inline/noalias-calls.ll b/llvm/test/Transforms/Inline/noalias-calls.ll
index fdbad60b0e1cd..4673daefa4db2 100644
--- a/llvm/test/Transforms/Inline/noalias-calls.ll
+++ b/llvm/test/Transforms/Inline/noalias-calls.ll
@@ -36,13 +36,13 @@ define void @foo(ptr nocapture %a, ptr nocapture readonly %c, ptr nocapture %b)
 ; CHECK-NEXT:    [[L_I:%.*]] = alloca i8, i32 512, align 1
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META3:![0-9]+]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 512, ptr [[L_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[L_I]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A]], ptr align 16 [[B]], i64 16, i1 false), !noalias [[META3]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[B]], ptr readonly align 16 [[C]], i64 16, i1 false), !noalias [[META0]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A]], ptr readonly align 16 [[C]], i64 16, i1 false), !alias.scope [[META5:![0-9]+]]
 ; CHECK-NEXT:    call void @hey(), !noalias [[META5]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[L_I]], ptr readonly align 16 [[C]], i64 16, i1 false), !noalias [[META0]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 512, ptr [[L_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[L_I]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -79,13 +79,13 @@ define void @foo_cs(ptr nocapture %a, ptr nocapture readonly %c, ptr nocapture %
 ; CHECK-NEXT:    [[L_I:%.*]] = alloca i8, i32 512, align 1
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META6:![0-9]+]])
 ; CHECK-NEXT:    call void @llvm.experimental.noalias.scope.decl(metadata [[META9:![0-9]+]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 512, ptr [[L_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[L_I]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A]], ptr align 16 [[B]], i64 16, i1 false), !noalias [[META9]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[B]], ptr readonly align 16 [[C]], i64 16, i1 false), !noalias [[META6]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[A]], ptr readonly align 16 [[C]], i64 16, i1 false), !alias.scope [[META11:![0-9]+]]
 ; CHECK-NEXT:    call void @hey(), !noalias [[META11]]
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 16 [[L_I]], ptr readonly align 16 [[C]], i64 16, i1 false), !noalias [[META6]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 512, ptr [[L_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[L_I]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/Inline/optimization-remarks-passed-deleted-callee-yaml.ll b/llvm/test/Transforms/Inline/optimization-remarks-passed-deleted-callee-yaml.ll
new file mode 100644
index 0000000000000..d3947137b4d80
--- /dev/null
+++ b/llvm/test/Transforms/Inline/optimization-remarks-passed-deleted-callee-yaml.ll
@@ -0,0 +1,97 @@
+; RUN: opt < %s -S -passes=inline -pass-remarks-output=%t -pass-remarks=inline \
+; RUN:    -pass-remarks-missed=inline -pass-remarks-analysis=inline \
+; RUN:    -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefixes=YAML,YAML-NO-ANNOTATE %s
+
+; RUN: opt < %s -S -passes=inline -pass-remarks-output=%t -pass-remarks=inline \
+; RUN:    -pass-remarks-missed=inline -pass-remarks-analysis=inline \
+; RUN:    -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefixes=YAML,YAML-NO-ANNOTATE %s
+
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-output=%t -pass-remarks=inline \
+; RUN:    -pass-remarks-missed=inline -pass-remarks-analysis=inline \
+; RUN:    -annotate-inline-phase=false \
+; RUN:    -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefixes=YAML,YAML-NO-ANNOTATE %s
+
+; RUN: opt < %s -S -passes=inliner-wrapper -pass-remarks-output=%t -pass-remarks=inline \
+; RUN:    -pass-remarks-missed=inline -pass-remarks-analysis=inline \
+; RUN:    -annotate-inline-phase \
+; RUN:    -pass-remarks-with-hotness 2>&1 | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefixes=YAML,YAML-ANNOTATE %s
+
+; Same as optimization-remarks-passed-yaml.ll, but here foo has internal linkage
+; instead of external linkage. This exercises a different hook for reporting the inlining decision,
+; because foo is deleted after it is inlined (due to internal linkage).
+; Make sure that the Remark correctly emits the DebugLoc for foo before it is deleted.
+
+; CHECK: remark: /tmp/s.c:4:10: 'foo' inlined into 'bar' with (cost={{[0-9\-]+}}, threshold={{[0-9]+}}) at callsite bar:1:10; (hotness: 30)
+
+; YAML:      --- !Passed
+; YAML-NO-ANNOTATE-NEXT: Pass:            inline
+; YAML-ANNOTATE-NEXT: Pass:            main-always-inline
+; YAML-NEXT: Name:            Inlined
+; YAML-NEXT: DebugLoc:        { File: '/tmp/s.c', Line: 4, Column: 10 }
+; YAML-NEXT: Function:        bar
+; YAML-NEXT: Hotness:         30
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String: ''''
+; YAML-NEXT:   - Callee: foo
+; YAML-NEXT:     DebugLoc:        { File: '/tmp/s.c', Line: 1, Column: 0 }
+; YAML-NEXT:   - String: ''' inlined into '''
+; YAML-NEXT:   - Caller: bar
+; YAML-NEXT:     DebugLoc:        { File: '/tmp/s.c', Line: 3, Column: 0 }
+; YAML-NEXT:   - String: ''''
+; YAML-NEXT:   - String: ' with '
+; YAML-NEXT:   - String: '(cost='
+; YAML-NEXT:   - Cost: '{{[0-9\-]+}}'
+; YAML-NEXT:   - String: ', threshold='
+; YAML-NEXT:   - Threshold: '{{[0-9]+}}'
+; YAML-NEXT:   - String: ')'
+; YAML-NEXT:   - String:          ' at callsite '
+; YAML-NEXT:   - String:          bar
+; YAML-NEXT:   - String:          ':'
+; YAML-NEXT:   - Line:            '1'
+; YAML-NEXT:   - String:          ':'
+; YAML-NEXT:   - Column:          '10'
+; YAML-NEXT:   - String:          ';'
+; YAML-NEXT: ...
+
+; ModuleID = '/tmp/s.c'
+source_filename = "/tmp/s.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+; Function Attrs: nounwind ssp uwtable
+define internal i32 @foo() #0 !dbg !7 {
+entry:
+  ret i32 1, !dbg !9
+}
+
+; Function Attrs: nounwind ssp uwtable
+define i32 @bar() #0 !dbg !10 !prof !13 {
+entry:
+  %call = call i32 @foo(), !dbg !11
+  ret i32 %call, !dbg !12
+}
+
+attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="core2" "target-features"="+cx16,+fxsr,+mmx,+sse,+sse2,+sse3,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "/tmp/s.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang version 4.0.0 (trunk 282540) (llvm/trunk 282542)"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 1, column: 13, scope: !7)
+!10 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !0, retainedNodes: !2)
+!11 = !DILocation(line: 4, column: 10, scope: !10)
+!12 = !DILocation(line: 4, column: 3, scope: !10)
+!13 = !{!"function_entry_count", i64 30}
diff --git a/llvm/test/Transforms/InstCombine/2025-08-06-shufflevector-bitcast-vector-of-pointers.ll b/llvm/test/Transforms/InstCombine/2025-08-06-shufflevector-bitcast-vector-of-pointers.ll
new file mode 100644
index 0000000000000..e778d921d5b4c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/2025-08-06-shufflevector-bitcast-vector-of-pointers.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+; Make sure that we don't crash when optimizing shufflevector of <N x ptr> with <1 x i32> mask followed by bitcast of <1 x ptr> to ptr
+
+define ptr @test(<3 x ptr> %vptr) {
+; CHECK-LABEL: define ptr @test(
+; CHECK-SAME: <3 x ptr> [[VPTR:%.*]]) {
+; CHECK-NEXT:    [[SV_EXTRACT:%.*]] = extractelement <3 x ptr> [[VPTR]], i64 0
+; CHECK-NEXT:    ret ptr [[SV_EXTRACT]]
+;
+  %SV = shufflevector <3 x ptr> %vptr, <3 x ptr> zeroinitializer, <1 x i32> zeroinitializer
+  %BC = bitcast <1 x ptr> %SV to ptr
+  ret ptr %BC
+}
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sme-intrinsic-opts-counting-elems.ll b/llvm/test/Transforms/InstCombine/AArch64/sme-intrinsic-opts-counting-elems.ll
new file mode 100644
index 0000000000000..f213c0b53f6ef
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/sme-intrinsic-opts-counting-elems.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mattr=+sme -S -passes=instcombine < %s | FileCheck %s
+; RUN: opt -mattr=+sme -S -passes=instcombine -force-streaming-compatible < %s | FileCheck %s
+; RUN: opt -mattr=+sme -S -passes=instcombine -force-streaming < %s | FileCheck %s -check-prefix=CHECK-STREAMING
+
+target triple = "aarch64-unknown-linux-gnu"
+
+define i64 @cntsb() {
+; CHECK-LABEL: @cntsb(
+; CHECK-NEXT:    [[OUT:%.*]] = call i64 @llvm.aarch64.sme.cntsb()
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+; CHECK-STREAMING-LABEL: @cntsb(
+; CHECK-STREAMING-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-STREAMING-NEXT:    [[OUT:%.*]] = shl nuw i64 [[TMP1]], 4
+; CHECK-STREAMING-NEXT:    ret i64 [[OUT]]
+;
+  %out = call i64 @llvm.aarch64.sme.cntsb()
+  ret i64 %out
+}
+
+define i64 @cntsh() {
+; CHECK-LABEL: @cntsh(
+; CHECK-NEXT:    [[OUT:%.*]] = call i64 @llvm.aarch64.sme.cntsh()
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+; CHECK-STREAMING-LABEL: @cntsh(
+; CHECK-STREAMING-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-STREAMING-NEXT:    [[OUT:%.*]] = shl nuw i64 [[TMP1]], 3
+; CHECK-STREAMING-NEXT:    ret i64 [[OUT]]
+;
+  %out = call i64 @llvm.aarch64.sme.cntsh()
+  ret i64 %out
+}
+
+define i64 @cntsw() {
+; CHECK-LABEL: @cntsw(
+; CHECK-NEXT:    [[OUT:%.*]] = call i64 @llvm.aarch64.sme.cntsw()
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+; CHECK-STREAMING-LABEL: @cntsw(
+; CHECK-STREAMING-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-STREAMING-NEXT:    [[OUT:%.*]] = shl nuw i64 [[TMP1]], 2
+; CHECK-STREAMING-NEXT:    ret i64 [[OUT]]
+;
+  %out = call i64 @llvm.aarch64.sme.cntsw()
+  ret i64 %out
+}
+
+define i64 @cntsd() {
+; CHECK-LABEL: @cntsd(
+; CHECK-NEXT:    [[OUT:%.*]] = call i64 @llvm.aarch64.sme.cntsd()
+; CHECK-NEXT:    ret i64 [[OUT]]
+;
+; CHECK-STREAMING-LABEL: @cntsd(
+; CHECK-STREAMING-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-STREAMING-NEXT:    [[OUT:%.*]] = shl nuw i64 [[TMP1]], 1
+; CHECK-STREAMING-NEXT:    ret i64 [[OUT]]
+;
+  %out = call i64 @llvm.aarch64.sme.cntsd()
+  ret i64 %out
+}
+
+declare i64 @llvm.aarch64.sve.cntsb()
+declare i64 @llvm.aarch64.sve.cntsh()
+declare i64 @llvm.aarch64.sve.cntsw()
+declare i64 @llvm.aarch64.sve.cntsd()
+
diff --git a/llvm/test/Transforms/InstCombine/and-or-icmps.ll b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
index 42e5020748129..553c7ac5af0e9 100644
--- a/llvm/test/Transforms/InstCombine/and-or-icmps.ll
+++ b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
@@ -3491,3 +3491,198 @@ define i1 @and_icmp_eq_with_binary_range_operands(i8 range(i8 0, 2) %x, i8 range
   %ret = and i1 %icmp1, %icmp2
   ret i1 %ret
 }
+
+define i1 @or_icmp_eq_and_pow2(i32 %x) {
+; CHECK-LABEL: @or_icmp_eq_and_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -127
+; CHECK-NEXT:    [[RET:%.*]] = icmp ult i32 [[TMP1]], 33
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %icmp1 = icmp eq i32 %x, 127
+  %and = and i32 %x, -32
+  %icmp2 = icmp eq i32 %and, 128
+  %ret = or i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @or_icmp_eq_and_pow2_mask_equal_to_icmp(i32 %x) {
+; CHECK-LABEL: @or_icmp_eq_and_pow2_mask_equal_to_icmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -63
+; CHECK-NEXT:    [[RET:%.*]] = icmp ult i32 [[TMP1]], 65
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %icmp1 = icmp eq i32 %x, 63
+  %and = and i32 %x, -64
+  %icmp2 = icmp eq i32 %and, 64
+  %ret = or i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @and_icmp_ne_and_pow2(i32 %x) {
+; CHECK-LABEL: @and_icmp_ne_and_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -160
+; CHECK-NEXT:    [[RET:%.*]] = icmp ult i32 [[TMP1]], -33
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %icmp1 = icmp ne i32 %x, 127
+  %and = and i32 %x, -32
+  %icmp2 = icmp ne i32 %and, 128
+  %ret = and i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @and_icmp_ne_and_pow2_mask_equal_to_icmp(i32 %x) {
+; CHECK-LABEL: @and_icmp_ne_and_pow2_mask_equal_to_icmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -128
+; CHECK-NEXT:    [[RET:%.*]] = icmp ult i32 [[TMP1]], -65
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %icmp1 = icmp ne i32 %x, 63
+  %and = and i32 %x, -64
+  %icmp2 = icmp ne i32 %and, 64
+  %ret = and i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @or_icmp_eq_and_pow2_commute(i32 %x) {
+; CHECK-LABEL: @or_icmp_eq_and_pow2_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -127
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 [[TMP1]], 33
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %and = and i32 %x, -32
+  %icmp1 = icmp eq i32 %and, 128
+  %icmp2 = icmp eq i32 %x, 127
+  %ret = or i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @and_icmp_ne_and_pow2_commute(i32 %x) {
+; CHECK-LABEL: @and_icmp_ne_and_pow2_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -160
+; CHECK-NEXT:    [[RET:%.*]] = icmp ult i32 [[TMP1]], -33
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %and = and i32 %x, -32
+  %icmp1 = icmp ne i32 %and, 128
+  %icmp2 = icmp ne i32 %x, 127
+  %ret = and i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @neg_or_icmp_eq_and_pow2_multi_use(i32 %x) {
+; CHECK-LABEL: @neg_or_icmp_eq_and_pow2_multi_use(
+; CHECK-NEXT:    [[ICMP1:%.*]] = icmp eq i32 [[X:%.*]], 127
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], -32
+; CHECK-NEXT:    call void @use32(i32 [[AND]])
+; CHECK-NEXT:    [[ICMP2:%.*]] = icmp eq i32 [[AND]], 128
+; CHECK-NEXT:    [[RET:%.*]] = or i1 [[ICMP1]], [[ICMP2]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %icmp1 = icmp eq i32 %x, 127
+  %and = and i32 %x, -32
+  call void @use32(i32 %and)
+  %icmp2 = icmp eq i32 %and, 128
+  %ret = or i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @neg_and_icmp_eq_and_pow2(i32 %x) {
+; CHECK-LABEL: @neg_and_icmp_eq_and_pow2(
+; CHECK-NEXT:    ret i1 false
+;
+  %icmp1 = icmp eq i32 %x, 127
+  %and = and i32 %x, -32
+  %icmp2 = icmp eq i32 %and, 128
+  %ret = and i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @neg_or_icmp_eq_and_non_pow2_mask(i32 %x) {
+; CHECK-LABEL: @neg_or_icmp_eq_and_non_pow2_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 127
+; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[X]], -33
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 128
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    ret i1 [[TMP4]]
+;
+  %icmp1 = icmp eq i32 %x, 127
+  %and = and i32 %x, -33
+  %icmp2 = icmp eq i32 %and, 128
+  %ret = or i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @neg_and_icmp_ne_and_non_pow2_icmp(i32 %x) {
+; CHECK-LABEL: @neg_and_icmp_ne_and_non_pow2_icmp(
+; CHECK-NEXT:    [[ICMP1:%.*]] = icmp ne i32 [[X:%.*]], 127
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], -33
+; CHECK-NEXT:    [[ICMP2:%.*]] = icmp ne i32 [[AND]], 128
+; CHECK-NEXT:    [[RET:%.*]] = and i1 [[ICMP1]], [[ICMP2]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %icmp1 = icmp ne i32 %x, 127
+  %and = and i32 %x, -33
+  %icmp2 = icmp ne i32 %and, 128
+  %ret = and i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @neg_or_icmp_eq_and_const_less_than_mask(i32 %x) {
+; CHECK-LABEL: @neg_or_icmp_eq_and_const_less_than_mask(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 15
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %icmp1 = icmp eq i32 %x, 15
+  %and = and i32 %x, -32
+  %icmp2 = icmp eq i32 %and, 16
+  %ret = or i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @neg_and_icmp_ne_and_pow2_disjoint(i32 %x) {
+; CHECK-LABEL: @neg_and_icmp_ne_and_pow2_disjoint(
+; CHECK-NEXT:    [[ICMP1:%.*]] = icmp ne i32 [[X:%.*]], 126
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], -32
+; CHECK-NEXT:    [[ICMP2:%.*]] = icmp ne i32 [[AND]], 128
+; CHECK-NEXT:    [[RET:%.*]] = and i1 [[ICMP1]], [[ICMP2]]
+; CHECK-NEXT:    ret i1 [[RET]]
+;
+  %icmp1 = icmp ne i32 %x, 126
+  %and = and i32 %x, -32
+  %icmp2 = icmp ne i32 %and, 128
+  %ret = and i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @neg_or_icmp_eq_double_and_pow2(i32 %x) {
+; CHECK-LABEL: @neg_or_icmp_eq_double_and_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -16
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 64
+; CHECK-NEXT:    [[AND2:%.*]] = and i32 [[X]], -32
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[AND2]], 128
+; CHECK-NEXT:    [[TMP5:%.*]] = or i1 [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    ret i1 [[TMP5]]
+;
+  %and1 = and i32 %x, -16
+  %icmp1 = icmp eq i32 %and1, 64
+  %and2 = and i32 %x, -32
+  %icmp2 = icmp eq i32 %and2, 128
+  %ret = or i1 %icmp1, %icmp2
+  ret i1 %ret
+}
+
+define i1 @neg_select_icmp_eq_and_pow2(i32 %x) {
+; CHECK-LABEL: @neg_select_icmp_eq_and_pow2(
+; CHECK-NEXT:    [[ICMP1:%.*]] = icmp sgt i32 [[X:%.*]], 127
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], -32
+; CHECK-NEXT:    [[ICMP2:%.*]] = icmp eq i32 [[AND]], 128
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 [[ICMP1]], [[ICMP2]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %icmp1 = icmp sgt i32 %x, 127
+  %and = and i32 %x, -32
+  %icmp2 = icmp eq i32 %and, 128
+  %1 = select i1 %icmp1, i1 %icmp2, i1 false
+  ret i1 %1
+}
diff --git a/llvm/test/Transforms/InstCombine/assume_inevitable.ll b/llvm/test/Transforms/InstCombine/assume_inevitable.ll
index 2643c9b525cb5..5f27ff1e609ba 100644
--- a/llvm/test/Transforms/InstCombine/assume_inevitable.ll
+++ b/llvm/test/Transforms/InstCombine/assume_inevitable.ll
@@ -35,10 +35,10 @@ entry:
   %dummy_eq = icmp ugt i32 %loadres, 42
   tail call void @llvm.assume(i1 %dummy_eq)
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %dummy)
+  call void @llvm.lifetime.start.p0(ptr %dummy)
   %i = call ptr @llvm.invariant.start.p0(i64 1, ptr %dummy)
   call void @llvm.invariant.end.p0(ptr %i, i64 1, ptr %dummy)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %dummy)
+  call void @llvm.lifetime.end.p0(ptr %dummy)
 
   %m_a = call ptr @llvm.ptr.annotation.p0(ptr %m, ptr @.str, ptr @.str1, i32 2, ptr null)
   %objsz = call i64 @llvm.objectsize.i64.p0(ptr %c, i1 false)
@@ -61,8 +61,8 @@ declare i64 @llvm.objectsize.i64.p0(ptr, i1)
 declare i32 @llvm.annotation.i32(i32, ptr, ptr, i32)
 declare ptr @llvm.ptr.annotation.p0(ptr, ptr, ptr, i32, ptr)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare ptr @llvm.invariant.start.p0(i64, ptr nocapture)
 declare void @llvm.invariant.end.p0(ptr, i64, ptr nocapture)
diff --git a/llvm/test/Transforms/InstCombine/builtin-object-size-custom-dl.ll b/llvm/test/Transforms/InstCombine/builtin-object-size-custom-dl.ll
index fe8b321114e47..93c4ae68c3a2f 100644
--- a/llvm/test/Transforms/InstCombine/builtin-object-size-custom-dl.ll
+++ b/llvm/test/Transforms/InstCombine/builtin-object-size-custom-dl.ll
@@ -15,16 +15,16 @@ entry:
 define i32 @objsize2_custom_idx() #0 {
 entry:
   %var = alloca %struct.V, align 4
-  call void @llvm.lifetime.start.p0(i64 28, ptr %var) #3
+  call void @llvm.lifetime.start.p0(ptr %var) #3
   %arrayidx = getelementptr inbounds [10 x i8], ptr %var, i64 0, i64 1
   %0 = call i64 @llvm.objectsize.i64.p0(ptr %arrayidx, i1 false, i1 false, i1 false)
   %conv = trunc i64 %0 to i32
-  call void @llvm.lifetime.end.p0(i64 28, ptr %var) #3
+  call void @llvm.lifetime.end.p0(ptr %var) #3
   ret i32 %conv
 ; CHECK: ret i32 27
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 declare ptr @malloc(i64)
 declare i64 @llvm.objectsize.i64.p0(ptr, i1, i1, i1)
diff --git a/llvm/test/Transforms/InstCombine/builtin-object-size-offset.ll b/llvm/test/Transforms/InstCombine/builtin-object-size-offset.ll
index b8919a76b122a..051466f95ab18 100644
--- a/llvm/test/Transforms/InstCombine/builtin-object-size-offset.ll
+++ b/llvm/test/Transforms/InstCombine/builtin-object-size-offset.ll
@@ -25,24 +25,24 @@ define i32 @foo1(i32 %N) {
 entry:
   %Big = alloca [20 x i8], align 16
   %Small = alloca [10 x i8], align 1
-  call void @llvm.lifetime.start.p0(i64 20, ptr %Big)
-  call void @llvm.lifetime.start.p0(i64 10, ptr %Small)
+  call void @llvm.lifetime.start.p0(ptr %Big)
+  call void @llvm.lifetime.start.p0(ptr %Small)
   %tobool = icmp ne i32 %N, 0
   %add.ptr = getelementptr inbounds [20 x i8], ptr %Big, i64 0, i64 10
   %cond = select i1 %tobool, ptr %add.ptr, ptr %Small
   %0 = call i64 @llvm.objectsize.i64.p0(ptr %cond, i1 false)
   %conv = trunc i64 %0 to i32
-  call void @llvm.lifetime.end.p0(i64 10, ptr %Small)
-  call void @llvm.lifetime.end.p0(i64 20, ptr %Big)
+  call void @llvm.lifetime.end.p0(ptr %Small)
+  call void @llvm.lifetime.end.p0(ptr %Big)
   ret i32 %conv
 ; CHECK: ret i32 10 
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare i64 @llvm.objectsize.i64.p0(ptr, i1)
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @foo() {
 entry:
diff --git a/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll b/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll
index 533451f26666a..3a7b76002b123 100644
--- a/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll
+++ b/llvm/test/Transforms/InstCombine/builtin-object-size-ptr.ll
@@ -18,11 +18,11 @@ define i32 @foo() #0 {
 ; CHECK-NEXT:    ret i32 27
 ;
   %var = alloca %struct.V, align 4
-  call void @llvm.lifetime.start.p0(i64 28, ptr %var) #3
+  call void @llvm.lifetime.start.p0(ptr %var) #3
   %arrayidx = getelementptr inbounds [10 x i8], ptr %var, i64 0, i64 1
   %t1 = call i64 @llvm.objectsize.i64.p0(ptr %arrayidx, i1 false)
   %conv = trunc i64 %t1 to i32
-  call void @llvm.lifetime.end.p0(i64 28, ptr %var) #3
+  call void @llvm.lifetime.end.p0(ptr %var) #3
   ret i32 %conv
 }
 
@@ -63,9 +63,9 @@ define ptr @minimal_invariant_start_use(i8 %x) {
   ret ptr %i
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare i64 @llvm.objectsize.i64.p0(ptr, i1) #2
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #0
 declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture) #0
 declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr nocapture) #0
diff --git a/llvm/test/Transforms/InstCombine/compare-alloca.ll b/llvm/test/Transforms/InstCombine/compare-alloca.ll
index a27cd70fe8bab..55d92b79a9ed6 100644
--- a/llvm/test/Transforms/InstCombine/compare-alloca.ll
+++ b/llvm/test/Transforms/InstCombine/compare-alloca.ll
@@ -86,18 +86,18 @@ define i1 @alloca_argument_compare_escaped_through_store(ptr %arg, ptr %ptr) {
   ret i1 %cmp
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 define i1 @alloca_argument_compare_benign_instrs(ptr %arg) {
 ; CHECK-LABEL: @alloca_argument_compare_benign_instrs(
 ; CHECK-NEXT:    ret i1 false
 ;
   %alloc = alloca i8
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloc)
+  call void @llvm.lifetime.start.p0(ptr %alloc)
   %cmp = icmp eq ptr %arg, %alloc
   %x = load i8, ptr %arg
   store i8 %x, ptr %alloc
-  call void @llvm.lifetime.end.p0(i64 1, ptr %alloc)
+  call void @llvm.lifetime.end.p0(ptr %alloc)
   ret i1 %cmp
 }
 
diff --git a/llvm/test/Transforms/InstCombine/deadcode.ll b/llvm/test/Transforms/InstCombine/deadcode.ll
index f3e1ba6787ad2..4dcdbb9618808 100644
--- a/llvm/test/Transforms/InstCombine/deadcode.ll
+++ b/llvm/test/Transforms/InstCombine/deadcode.ll
@@ -22,13 +22,13 @@ define ptr @test2(i32 %width) {
 
 declare ptr @llvm.stacksave()
 
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 
 define void @test3() {
   %a = alloca i32
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %a)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
diff --git a/llvm/test/Transforms/InstCombine/fold-bin-operand.ll b/llvm/test/Transforms/InstCombine/fold-bin-operand.ll
index f28262b2a77e0..79671215dba61 100644
--- a/llvm/test/Transforms/InstCombine/fold-bin-operand.ll
+++ b/llvm/test/Transforms/InstCombine/fold-bin-operand.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
-target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
+target datalayout = "E-p:64:64:64-p1:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
 define i1 @f(i1 %x) {
 ; CHECK-LABEL: @f(
@@ -30,6 +30,226 @@ define i32 @g(i32 %x) {
   ret i32 %b
 }
 
+define i1 @inttoptr_add_ptrtoint_used_by_single_icmp(ptr %src, ptr %p2) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp(
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 10
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  %c = icmp eq ptr %p, %p2
+  ret i1 %c
+}
+
+define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_operands_swapped(ptr %src, ptr %p2) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_operands_swapped(
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 10
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 10, %i
+  %p = inttoptr i64 %a to ptr
+  %c = icmp eq ptr %p, %p2
+  ret i1 %c
+}
+
+define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_constant_offset(ptr %src, i64 %off, ptr %p2) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_constant_offset(
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFF:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, %off
+  %p = inttoptr i64 %a to ptr
+  %c = icmp eq ptr %p, %p2
+  ret i1 %c
+}
+
+define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_constant_offset_operands_swapped(ptr %src, i64 %off, ptr %p2) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_constant_offset_operands_swapped(
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFF:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %off, %i
+  %p = inttoptr i64 %a to ptr
+  %c = icmp eq ptr %p, %p2
+  ret i1 %c
+}
+
+define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_different_src_address_spaces(ptr addrspace(1) %src, ptr %p2) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_different_src_address_spaces(
+; CHECK-NEXT:    [[I:%.*]] = ptrtoint ptr addrspace(1) [[SRC:%.*]] to i64
+; CHECK-NEXT:    [[A:%.*]] = add i64 [[I]], 10
+; CHECK-NEXT:    [[P:%.*]] = inttoptr i64 [[A]] to ptr
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr [[P2:%.*]], [[P]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = ptrtoint ptr addrspace(1) %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  %c = icmp eq ptr %p, %p2
+  ret i1 %c
+}
+
+define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_different_dst_address_spaces(ptr %src, ptr addrspace(1) %p2) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_different_dst_address_spaces(
+; CHECK-NEXT:    [[I:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64
+; CHECK-NEXT:    [[A:%.*]] = add i64 [[I]], 10
+; CHECK-NEXT:    [[P:%.*]] = inttoptr i64 [[A]] to ptr addrspace(1)
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr addrspace(1) [[P2:%.*]], [[P]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr addrspace(1)
+  %c = icmp eq ptr addrspace(1) %p, %p2
+  ret i1 %c
+}
+
+define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_int_type_does_not_match_ptr_ty(ptr %src, ptr %p2) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_int_type_does_not_match_ptr_ty(
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64
+; CHECK-NEXT:    [[I:%.*]] = trunc i64 [[TMP1]] to i8
+; CHECK-NEXT:    [[A:%.*]] = add i8 [[I]], 10
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[A]] to i64
+; CHECK-NEXT:    [[P:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr [[P2:%.*]], [[P]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = ptrtoint ptr %src to i8
+  %a = add i8 %i, 10
+  %p = inttoptr i8 %a to ptr
+  %c = icmp eq ptr %p, %p2
+  ret i1 %c
+}
+
+define i1 @inttoptr_add_multiple_users_ptrtoint_used_by_single_icmp(ptr %src, ptr %p2) {
+; CHECK-LABEL: @inttoptr_add_multiple_users_ptrtoint_used_by_single_icmp(
+; CHECK-NEXT:    [[I:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64
+; CHECK-NEXT:    [[A:%.*]] = add i64 [[I]], 10
+; CHECK-NEXT:    [[P:%.*]] = inttoptr i64 [[A]] to ptr
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr [[P2:%.*]], [[P]]
+; CHECK-NEXT:    call void @bar(i64 [[A]])
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  %c = icmp eq ptr %p, %p2
+  call void @bar(i64 %a)
+  ret i1 %c
+}
+
+define i1 @multiple_inttoptr_add_ptrtoint_used_by_single_icmp(ptr %src) {
+; CHECK-LABEL: @multiple_inttoptr_add_ptrtoint_used_by_single_icmp(
+; CHECK-NEXT:    ret i1 false
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  %a.2 = add i64 %i, 11
+  %p.2 = inttoptr i64 %a.2 to ptr
+  %c = icmp eq ptr %p, %p.2
+  ret i1 %c
+}
+
+define i1 @multiple_inttoptr_add_ptrtoint_used_by_single_icmp_non_constant_offset(ptr %src, i64 %off.1) {
+; CHECK-LABEL: @multiple_inttoptr_add_ptrtoint_used_by_single_icmp_non_constant_offset(
+; CHECK-NEXT:    ret i1 false
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  %a.2 = add i64 %i, 11
+  %p.2 = inttoptr i64 %a.2 to ptr
+  %c = icmp eq ptr %p, %p.2
+  ret i1 %c
+}
+
+define i1 @multiple_inttoptr_add_ptrtoint_used_by_single_icmp_multiple_non_constant_offset(ptr %src, i64 %off.1, i64 %off.2) {
+; CHECK-LABEL: @multiple_inttoptr_add_ptrtoint_used_by_single_icmp_multiple_non_constant_offset(
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[OFF_1:%.*]], [[OFF_2:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, %off.1
+  %p = inttoptr i64 %a to ptr
+  %a.2 = add i64 %i, %off.2
+  %p.2 = inttoptr i64 %a.2 to ptr
+  %c = icmp eq ptr %p, %p.2
+  ret i1 %c
+}
+
+define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_in_different_bb(i1 %bc, ptr %src, ptr %p2) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_in_different_bb(
+; CHECK-NEXT:    br i1 [[BC:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 10
+; CHECK-NEXT:    [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  br i1 %bc, label %then, label %else
+
+then:
+  %c = icmp eq ptr %p, %p2
+  ret i1 %c
+
+else:
+  ret i1 false
+}
+
+define i1 @inttoptr_add_ptrtoint_used_by_multiple_icmps(ptr %src, ptr %p2, ptr %p3) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_multiple_icmps(
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 10
+; CHECK-NEXT:    [[C_1:%.*]] = icmp eq ptr [[P2]], [[P:%.*]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp eq ptr [[P2]], [[P3:%.*]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  %c.1 = icmp eq ptr %p, %p2
+  %c.2 = icmp eq ptr %p, %p3
+  %xor = xor i1 %c.1, %c.2
+  ret i1 %xor
+}
+
+declare void @foo(ptr)
+declare void @bar(i64)
+
+define i1 @inttoptr_add_ptrtoint_used_by_multiple_icmps_and_other_user(ptr %src, ptr %p2, ptr %p3) {
+; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_multiple_icmps_and_other_user(
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 10
+; CHECK-NEXT:    [[P:%.*]] = inttoptr i64 [[TMP2]] to ptr
+; CHECK-NEXT:    [[C_1:%.*]] = icmp eq ptr [[P2:%.*]], [[P]]
+; CHECK-NEXT:    [[C_2:%.*]] = icmp eq ptr [[P3:%.*]], [[P]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i1 [[C_1]], [[C_2]]
+; CHECK-NEXT:    call void @foo(ptr [[P]])
+; CHECK-NEXT:    ret i1 [[XOR]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  %c.1 = icmp eq ptr %p, %p2
+  %c.2 = icmp eq ptr %p, %p3
+  %xor = xor i1 %c.1, %c.2
+  call void @foo(ptr %p)
+  ret i1 %xor
+}
+
 define i32 @h(i1 %A, i32 %B) {
 ; CHECK-LABEL: @h(
 ; CHECK-NEXT:  EntryBlock:
diff --git a/llvm/test/Transforms/InstCombine/freeze-fp-ops.ll b/llvm/test/Transforms/InstCombine/freeze-fp-ops.ll
index d1a36dcef4d65..08f81e6db9bb5 100644
--- a/llvm/test/Transforms/InstCombine/freeze-fp-ops.ll
+++ b/llvm/test/Transforms/InstCombine/freeze-fp-ops.ll
@@ -1,12 +1,13 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -S -passes=instcombine < %s | FileCheck %s
 
 ; Test that floating point operations are not treated as sources of
 ; poison.
 
 define float @freeze_fneg(float %arg) {
-; CHECK-LABEL: @freeze_fneg(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_fneg(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = fneg float [[ARG_FR]]
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -16,9 +17,10 @@ define float @freeze_fneg(float %arg) {
 }
 
 define float @freeze_fadd(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_fadd(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = fadd float [[ARG0_FR]], [[ARG1:%.*]]
+; CHECK-LABEL: define float @freeze_fadd(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = fadd float [[ARG0_FR]], [[ARG1]]
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = fadd float %arg0, %arg1
@@ -27,9 +29,10 @@ define float @freeze_fadd(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_fsub(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_fsub(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = fsub float [[ARG0_FR]], [[ARG1:%.*]]
+; CHECK-LABEL: define float @freeze_fsub(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = fsub float [[ARG0_FR]], [[ARG1]]
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = fsub float %arg0, %arg1
@@ -38,9 +41,10 @@ define float @freeze_fsub(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_fmul(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_fmul(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = fmul float [[ARG0_FR]], [[ARG1:%.*]]
+; CHECK-LABEL: define float @freeze_fmul(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = fmul float [[ARG0_FR]], [[ARG1]]
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = fmul float %arg0, %arg1
@@ -49,9 +53,10 @@ define float @freeze_fmul(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_fdiv(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_fdiv(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = fdiv float [[ARG0_FR]], [[ARG1:%.*]]
+; CHECK-LABEL: define float @freeze_fdiv(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = fdiv float [[ARG0_FR]], [[ARG1]]
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = fdiv float %arg0, %arg1
@@ -60,9 +65,10 @@ define float @freeze_fdiv(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_frem(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_frem(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = fdiv float [[ARG0_FR]], [[ARG1:%.*]]
+; CHECK-LABEL: define float @freeze_frem(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = fdiv float [[ARG0_FR]], [[ARG1]]
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = fdiv float %arg0, %arg1
@@ -71,8 +77,9 @@ define float @freeze_frem(float %arg0, float noundef %arg1) {
 }
 
 define double @fpext(float %arg) {
-; CHECK-LABEL: @fpext(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define double @fpext(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = fpext float [[ARG_FR]] to double
 ; CHECK-NEXT:    ret double [[OP]]
 ;
@@ -82,8 +89,9 @@ define double @fpext(float %arg) {
 }
 
 define float @fptrunc(double %arg) {
-; CHECK-LABEL: @fptrunc(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze double [[ARG:%.*]]
+; CHECK-LABEL: define float @fptrunc(
+; CHECK-SAME: double [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze double [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = fptrunc double [[ARG_FR]] to float
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -93,8 +101,9 @@ define float @fptrunc(double %arg) {
 }
 
 define float @sitofp(i32 %arg) {
-; CHECK-LABEL: @sitofp(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define float @sitofp(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = sitofp i32 [[ARG_FR]] to float
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -104,8 +113,9 @@ define float @sitofp(i32 %arg) {
 }
 
 define float @uitofp(i32 %arg) {
-; CHECK-LABEL: @uitofp(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define float @uitofp(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = uitofp i32 [[ARG_FR]] to float
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -115,9 +125,10 @@ define float @uitofp(i32 %arg) {
 }
 
 define float @freeze_fma(float %arg0, float noundef %arg1, float noundef %arg2) {
-; CHECK-LABEL: @freeze_fma(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.fma.f32(float [[ARG0_FR]], float noundef [[ARG1:%.*]], float noundef [[ARG2:%.*]])
+; CHECK-LABEL: define float @freeze_fma(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]], float noundef [[ARG2:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.fma.f32(float [[ARG0_FR]], float noundef [[ARG1]], float noundef [[ARG2]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.fma.f32(float %arg0, float noundef %arg1, float noundef %arg2)
@@ -126,9 +137,10 @@ define float @freeze_fma(float %arg0, float noundef %arg1, float noundef %arg2)
 }
 
 define float @freeze_fmuladd(float %arg0, float noundef %arg1, float noundef %arg2) {
-; CHECK-LABEL: @freeze_fmuladd(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.fmuladd.f32(float [[ARG0_FR]], float noundef [[ARG1:%.*]], float noundef [[ARG2:%.*]])
+; CHECK-LABEL: define float @freeze_fmuladd(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]], float noundef [[ARG2:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.fmuladd.f32(float [[ARG0_FR]], float noundef [[ARG1]], float noundef [[ARG2]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.fmuladd.f32(float %arg0, float noundef %arg1, float noundef %arg2)
@@ -137,8 +149,9 @@ define float @freeze_fmuladd(float %arg0, float noundef %arg1, float noundef %ar
 }
 
 define float @freeze_sqrt(float %arg) {
-; CHECK-LABEL: @freeze_sqrt(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_sqrt(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.sqrt.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -148,9 +161,11 @@ define float @freeze_sqrt(float %arg) {
 }
 
 define float @freeze_powi(float %arg0, i32 %arg1) {
-; CHECK-LABEL: @freeze_powi(
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0:%.*]], i32 [[ARG1:%.*]])
-; CHECK-NEXT:    [[FREEZE:%.*]] = freeze float [[OP]]
+; CHECK-LABEL: define float @freeze_powi(
+; CHECK-SAME: float [[ARG0:%.*]], i32 [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[ARG1_FR:%.*]] = freeze i32 [[ARG1]]
+; CHECK-NEXT:    [[FREEZE:%.*]] = call float @llvm.powi.f32.i32(float [[ARG0_FR]], i32 [[ARG1_FR]])
 ; CHECK-NEXT:    ret float [[FREEZE]]
 ;
   %op = call float @llvm.powi.f32.i32(float %arg0, i32 %arg1)
@@ -159,8 +174,9 @@ define float @freeze_powi(float %arg0, i32 %arg1) {
 }
 
 define float @freeze_sin(float %arg) {
-; CHECK-LABEL: @freeze_sin(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_sin(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.sin.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -170,8 +186,9 @@ define float @freeze_sin(float %arg) {
 }
 
 define float @freeze_cos(float %arg) {
-; CHECK-LABEL: @freeze_cos(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_cos(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.cos.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -181,9 +198,10 @@ define float @freeze_cos(float %arg) {
 }
 
 define float @freeze_pow(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_pow(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.pow.f32(float [[ARG0_FR]], float noundef [[ARG1:%.*]])
+; CHECK-LABEL: define float @freeze_pow(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.pow.f32(float [[ARG0_FR]], float noundef [[ARG1]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.pow.f32(float %arg0, float noundef %arg1)
@@ -192,8 +210,9 @@ define float @freeze_pow(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_log(float %arg) {
-; CHECK-LABEL: @freeze_log(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_log(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.log.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -203,8 +222,9 @@ define float @freeze_log(float %arg) {
 }
 
 define float @freeze_log10(float %arg) {
-; CHECK-LABEL: @freeze_log10(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_log10(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.log10.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -214,8 +234,9 @@ define float @freeze_log10(float %arg) {
 }
 
 define float @freeze_log2(float %arg) {
-; CHECK-LABEL: @freeze_log2(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_log2(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.log2.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -225,8 +246,9 @@ define float @freeze_log2(float %arg) {
 }
 
 define float @freeze_exp(float %arg) {
-; CHECK-LABEL: @freeze_exp(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_exp(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.exp.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -236,8 +258,9 @@ define float @freeze_exp(float %arg) {
 }
 
 define float @freeze_exp2(float %arg) {
-; CHECK-LABEL: @freeze_exp2(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_exp2(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.exp2.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -247,8 +270,9 @@ define float @freeze_exp2(float %arg) {
 }
 
 define float @freeze_exp10(float %arg) {
-; CHECK-LABEL: @freeze_exp10(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_exp10(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.exp10.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -258,8 +282,9 @@ define float @freeze_exp10(float %arg) {
 }
 
 define float @freeze_fabs(float %arg) {
-; CHECK-LABEL: @freeze_fabs(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_fabs(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.fabs.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -269,9 +294,10 @@ define float @freeze_fabs(float %arg) {
 }
 
 define float @freeze_copysign(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_copysign(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.copysign.f32(float [[ARG0_FR]], float noundef [[ARG1:%.*]])
+; CHECK-LABEL: define float @freeze_copysign(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.copysign.f32(float [[ARG0_FR]], float noundef [[ARG1]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.copysign.f32(float %arg0, float noundef %arg1)
@@ -280,8 +306,9 @@ define float @freeze_copysign(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_floor(float %arg) {
-; CHECK-LABEL: @freeze_floor(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_floor(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.floor.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -291,8 +318,9 @@ define float @freeze_floor(float %arg) {
 }
 
 define float @freeze_ceil(float %arg) {
-; CHECK-LABEL: @freeze_ceil(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_ceil(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.ceil.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -302,8 +330,9 @@ define float @freeze_ceil(float %arg) {
 }
 
 define float @freeze_trunc(float %arg) {
-; CHECK-LABEL: @freeze_trunc(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_trunc(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.trunc.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -313,8 +342,9 @@ define float @freeze_trunc(float %arg) {
 }
 
 define float @freeze_rint(float %arg) {
-; CHECK-LABEL: @freeze_rint(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_rint(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.rint.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -324,8 +354,9 @@ define float @freeze_rint(float %arg) {
 }
 
 define float @freeze_nearbyint(float %arg) {
-; CHECK-LABEL: @freeze_nearbyint(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_nearbyint(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.nearbyint.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -335,8 +366,9 @@ define float @freeze_nearbyint(float %arg) {
 }
 
 define float @freeze_round(float %arg) {
-; CHECK-LABEL: @freeze_round(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_round(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.round.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -346,8 +378,9 @@ define float @freeze_round(float %arg) {
 }
 
 define float @freeze_roundeven(float %arg) {
-; CHECK-LABEL: @freeze_roundeven(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_roundeven(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.roundeven.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -357,8 +390,9 @@ define float @freeze_roundeven(float %arg) {
 }
 
 define float @freeze_canonicalize(float %arg) {
-; CHECK-LABEL: @freeze_canonicalize(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_canonicalize(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.canonicalize.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -368,8 +402,9 @@ define float @freeze_canonicalize(float %arg) {
 }
 
 define float @freeze_arithmetic_fence(float %arg) {
-; CHECK-LABEL: @freeze_arithmetic_fence(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @freeze_arithmetic_fence(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.arithmetic.fence.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -379,8 +414,9 @@ define float @freeze_arithmetic_fence(float %arg) {
 }
 
 define i32 @freeze_lround(float %arg) {
-; CHECK-LABEL: @freeze_lround(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define i32 @freeze_lround(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call i32 @llvm.lround.i32.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret i32 [[OP]]
 ;
@@ -390,8 +426,9 @@ define i32 @freeze_lround(float %arg) {
 }
 
 define i32 @freeze_llround(float %arg) {
-; CHECK-LABEL: @freeze_llround(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define i32 @freeze_llround(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call i32 @llvm.llround.i32.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret i32 [[OP]]
 ;
@@ -401,8 +438,9 @@ define i32 @freeze_llround(float %arg) {
 }
 
 define i32 @freeze_lrint(float %arg) {
-; CHECK-LABEL: @freeze_lrint(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define i32 @freeze_lrint(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call i32 @llvm.lrint.i32.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret i32 [[OP]]
 ;
@@ -412,8 +450,9 @@ define i32 @freeze_lrint(float %arg) {
 }
 
 define i32 @freeze_llrint(float %arg) {
-; CHECK-LABEL: @freeze_llrint(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define i32 @freeze_llrint(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[OP:%.*]] = call i32 @llvm.llrint.i32.f32(float [[ARG_FR]])
 ; CHECK-NEXT:    ret i32 [[OP]]
 ;
@@ -423,8 +462,9 @@ define i32 @freeze_llrint(float %arg) {
 }
 
 define i32 @freeze_noundef_lround(float %arg) {
-; CHECK-LABEL: @freeze_noundef_lround(
-; CHECK-NEXT:    [[OP:%.*]] = call noundef i32 @llvm.lround.i32.f32(float [[ARG:%.*]])
+; CHECK-LABEL: define i32 @freeze_noundef_lround(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[OP:%.*]] = call noundef i32 @llvm.lround.i32.f32(float [[ARG]])
 ; CHECK-NEXT:    ret i32 [[OP]]
 ;
   %op = call noundef i32 @llvm.lround.i32.f32(float %arg)
@@ -433,8 +473,9 @@ define i32 @freeze_noundef_lround(float %arg) {
 }
 
 define i32 @freeze_noundef_llround(float %arg) {
-; CHECK-LABEL: @freeze_noundef_llround(
-; CHECK-NEXT:    [[OP:%.*]] = call noundef i32 @llvm.llround.i32.f32(float [[ARG:%.*]])
+; CHECK-LABEL: define i32 @freeze_noundef_llround(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[OP:%.*]] = call noundef i32 @llvm.llround.i32.f32(float [[ARG]])
 ; CHECK-NEXT:    ret i32 [[OP]]
 ;
   %op = call noundef i32 @llvm.llround.i32.f32(float %arg)
@@ -443,8 +484,9 @@ define i32 @freeze_noundef_llround(float %arg) {
 }
 
 define i32 @freeze_noundef_lrint(float %arg) {
-; CHECK-LABEL: @freeze_noundef_lrint(
-; CHECK-NEXT:    [[OP:%.*]] = call noundef i32 @llvm.lrint.i32.f32(float [[ARG:%.*]])
+; CHECK-LABEL: define i32 @freeze_noundef_lrint(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[OP:%.*]] = call noundef i32 @llvm.lrint.i32.f32(float [[ARG]])
 ; CHECK-NEXT:    ret i32 [[OP]]
 ;
   %op = call noundef i32 @llvm.lrint.i32.f32(float %arg)
@@ -453,8 +495,9 @@ define i32 @freeze_noundef_lrint(float %arg) {
 }
 
 define i32 @freeze_noundef_llrint(float %arg) {
-; CHECK-LABEL: @freeze_noundef_llrint(
-; CHECK-NEXT:    [[OP:%.*]] = call noundef i32 @llvm.llrint.i32.f32(float [[ARG:%.*]])
+; CHECK-LABEL: define i32 @freeze_noundef_llrint(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[OP:%.*]] = call noundef i32 @llvm.llrint.i32.f32(float [[ARG]])
 ; CHECK-NEXT:    ret i32 [[OP]]
 ;
   %op = call noundef i32 @llvm.llrint.i32.f32(float %arg)
@@ -463,9 +506,10 @@ define i32 @freeze_noundef_llrint(float %arg) {
 }
 
 define float @freeze_minnum(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_minnum(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.minnum.f32(float [[ARG0_FR]], float noundef [[ARG1:%.*]])
+; CHECK-LABEL: define float @freeze_minnum(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.minnum.f32(float [[ARG0_FR]], float noundef [[ARG1]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.minnum.f32(float %arg0, float noundef %arg1)
@@ -474,9 +518,10 @@ define float @freeze_minnum(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_maxnum(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_maxnum(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.maxnum.f32(float [[ARG0_FR]], float noundef [[ARG1:%.*]])
+; CHECK-LABEL: define float @freeze_maxnum(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.maxnum.f32(float [[ARG0_FR]], float noundef [[ARG1]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.maxnum.f32(float %arg0, float noundef %arg1)
@@ -485,9 +530,10 @@ define float @freeze_maxnum(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_minimum(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_minimum(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.minimum.f32(float [[ARG0_FR]], float noundef [[ARG1:%.*]])
+; CHECK-LABEL: define float @freeze_minimum(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.minimum.f32(float [[ARG0_FR]], float noundef [[ARG1]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.minimum.f32(float %arg0, float noundef %arg1)
@@ -496,9 +542,10 @@ define float @freeze_minimum(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_maximum(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_maximum(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.maximum.f32(float [[ARG0_FR]], float noundef [[ARG1:%.*]])
+; CHECK-LABEL: define float @freeze_maximum(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.maximum.f32(float [[ARG0_FR]], float noundef [[ARG1]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.maximum.f32(float %arg0, float noundef %arg1)
@@ -507,10 +554,11 @@ define float @freeze_maximum(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_minimumnum(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_minimumnum(
-; CHECK-NEXT:    [[FREEZE:%.*]] = freeze float [[OP:%.*]]
-; CHECK-NEXT:    [[OP1:%.*]] = call float @llvm.minimumnum.f32(float [[FREEZE]], float noundef [[ARG1:%.*]])
-; CHECK-NEXT:    ret float [[OP1]]
+; CHECK-LABEL: define float @freeze_minimumnum(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.minimumnum.f32(float [[ARG0_FR]], float noundef [[ARG1]])
+; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.minimumnum.f32(float %arg0, float noundef %arg1)
   %freeze = freeze float %op
@@ -518,10 +566,11 @@ define float @freeze_minimumnum(float %arg0, float noundef %arg1) {
 }
 
 define float @freeze_maximumnum(float %arg0, float noundef %arg1) {
-; CHECK-LABEL: @freeze_maximumnum(
-; CHECK-NEXT:    [[FREEZE:%.*]] = freeze float [[OP:%.*]]
-; CHECK-NEXT:    [[OP1:%.*]] = call float @llvm.maximumnum.f32(float [[FREEZE]], float noundef [[ARG1:%.*]])
-; CHECK-NEXT:    ret float [[OP1]]
+; CHECK-LABEL: define float @freeze_maximumnum(
+; CHECK-SAME: float [[ARG0:%.*]], float noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.maximumnum.f32(float [[ARG0_FR]], float noundef [[ARG1]])
+; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.maximumnum.f32(float %arg0, float noundef %arg1)
   %freeze = freeze float %op
@@ -529,8 +578,9 @@ define float @freeze_maximumnum(float %arg0, float noundef %arg1) {
 }
 
 define i1 @freeze_isfpclass(float %arg0) {
-; CHECK-LABEL: @freeze_isfpclass(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
+; CHECK-LABEL: define i1 @freeze_isfpclass(
+; CHECK-SAME: float [[ARG0:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
 ; CHECK-NEXT:    [[OP:%.*]] = call i1 @llvm.is.fpclass.f32(float [[ARG0_FR]], i32 27)
 ; CHECK-NEXT:    ret i1 [[OP]]
 ;
@@ -540,8 +590,9 @@ define i1 @freeze_isfpclass(float %arg0) {
 }
 
 define float @freeze_fptrunc_round(double %arg0) {
-; CHECK-LABEL: @freeze_fptrunc_round(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze double [[ARG0:%.*]]
+; CHECK-LABEL: define float @freeze_fptrunc_round(
+; CHECK-SAME: double [[ARG0:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze double [[ARG0]]
 ; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.fptrunc.round.f32.f64(double [[ARG0_FR]], metadata !"round.downward")
 ; CHECK-NEXT:    ret float [[OP]]
 ;
@@ -551,9 +602,10 @@ define float @freeze_fptrunc_round(double %arg0) {
 }
 
 define float @freeze_ldexp(float %arg0, i32 noundef %arg1) {
-; CHECK-LABEL: @freeze_ldexp(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
-; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.ldexp.f32.i32(float [[ARG0_FR]], i32 [[ARG1:%.*]])
+; CHECK-LABEL: define float @freeze_ldexp(
+; CHECK-SAME: float [[ARG0:%.*]], i32 noundef [[ARG1:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
+; CHECK-NEXT:    [[OP:%.*]] = call float @llvm.ldexp.f32.i32(float [[ARG0_FR]], i32 [[ARG1]])
 ; CHECK-NEXT:    ret float [[OP]]
 ;
   %op = call float @llvm.ldexp.f32.i32(float %arg0, i32 %arg1)
@@ -562,8 +614,9 @@ define float @freeze_ldexp(float %arg0, i32 noundef %arg1) {
 }
 
 define { float, i32 } @freeze_frexp(float %arg0) {
-; CHECK-LABEL: @freeze_frexp(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
+; CHECK-LABEL: define { float, i32 } @freeze_frexp(
+; CHECK-SAME: float [[ARG0:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
 ; CHECK-NEXT:    [[OP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[ARG0_FR]])
 ; CHECK-NEXT:    ret { float, i32 } [[OP]]
 ;
@@ -573,8 +626,9 @@ define { float, i32 } @freeze_frexp(float %arg0) {
 }
 
 define float @freeze_frexp_0(float %arg0) {
-; CHECK-LABEL: @freeze_frexp_0(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
+; CHECK-LABEL: define float @freeze_frexp_0(
+; CHECK-SAME: float [[ARG0:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
 ; CHECK-NEXT:    [[OP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[ARG0_FR]])
 ; CHECK-NEXT:    [[EXTRACT_0:%.*]] = extractvalue { float, i32 } [[OP]], 0
 ; CHECK-NEXT:    ret float [[EXTRACT_0]]
@@ -586,8 +640,9 @@ define float @freeze_frexp_0(float %arg0) {
 }
 
 define i32 @freeze_frexp_1(float %arg0) {
-; CHECK-LABEL: @freeze_frexp_1(
-; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0:%.*]]
+; CHECK-LABEL: define i32 @freeze_frexp_1(
+; CHECK-SAME: float [[ARG0:%.*]]) {
+; CHECK-NEXT:    [[ARG0_FR:%.*]] = freeze float [[ARG0]]
 ; CHECK-NEXT:    [[OP:%.*]] = call { float, i32 } @llvm.frexp.f32.i32(float [[ARG0_FR]])
 ; CHECK-NEXT:    [[EXTRACT_1:%.*]] = extractvalue { float, i32 } [[OP]], 1
 ; CHECK-NEXT:    ret i32 [[EXTRACT_1]]
diff --git a/llvm/test/Transforms/InstCombine/freeze-integer-intrinsics.ll b/llvm/test/Transforms/InstCombine/freeze-integer-intrinsics.ll
index c4a590e1a12bf..3cab09c2876bf 100644
--- a/llvm/test/Transforms/InstCombine/freeze-integer-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/freeze-integer-intrinsics.ll
@@ -426,6 +426,32 @@ define i1 @widenable_condition() {
   ret i1 %freeze
 }
 
+define i32 @freeze_scmp(i32 %a0) {
+; CHECK-LABEL: @freeze_scmp(
+; CHECK-NEXT:    [[A0_FR:%.*]] = freeze i32 [[A0:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call i32 @llvm.scmp.i32.i32(i32 2, i32 [[A0_FR]])
+; CHECK-NEXT:    [[Z:%.*]] = call i32 @llvm.scmp.i32.i32(i32 0, i32 [[X]])
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %x = call i32 @llvm.scmp.i32(i32 2, i32 %a0)
+  %y = freeze i32 %x
+  %z = call i32 @llvm.scmp.i32(i32 0, i32 %y)
+  ret i32 %z
+}
+
+define i32 @freeze_ucmp(i32 %a0) {
+; CHECK-LABEL: @freeze_ucmp(
+; CHECK-NEXT:    [[A0_FR:%.*]] = freeze i32 [[A0:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call i32 @llvm.ucmp.i32.i32(i32 2, i32 [[A0_FR]])
+; CHECK-NEXT:    [[Z:%.*]] = call i32 @llvm.ucmp.i32.i32(i32 [[X]], i32 1)
+; CHECK-NEXT:    ret i32 [[Z]]
+;
+  %x = call i32 @llvm.ucmp.i32(i32 2, i32 %a0)
+  %y = freeze i32 %x
+  %z = call i32 @llvm.ucmp.i32(i32 %y, i32 1)
+  ret i32 %z
+}
+
 declare i32 @llvm.ctlz.i32(i32, i1 immarg)
 declare i32 @llvm.cttz.i32(i32, i1 immarg)
 declare i32 @llvm.abs.i32(i32, i1 immarg)
diff --git a/llvm/test/Transforms/InstCombine/freeze.ll b/llvm/test/Transforms/InstCombine/freeze.ll
index 3fedead2feab8..b29421a655fa8 100644
--- a/llvm/test/Transforms/InstCombine/freeze.ll
+++ b/llvm/test/Transforms/InstCombine/freeze.ll
@@ -1,9 +1,10 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 define i32 @fold(i32 %x) {
-; CHECK-LABEL: @fold(
-; CHECK-NEXT:    [[Y:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-LABEL: define i32 @fold(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[Y:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[Y]]
 ;
   %y = freeze i32 %x
@@ -12,7 +13,7 @@ define i32 @fold(i32 %x) {
 }
 
 define i32 @make_const() {
-; CHECK-LABEL: @make_const(
+; CHECK-LABEL: define i32 @make_const() {
 ; CHECK-NEXT:    ret i32 10
 ;
   %x = freeze i32 10
@@ -20,7 +21,8 @@ define i32 @make_const() {
 }
 
 define i32 @and_freeze_undef(i32 %x) {
-; CHECK-LABEL: @and_freeze_undef(
+; CHECK-LABEL: define i32 @and_freeze_undef(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i32 0
 ;
   %f = freeze i32 undef
@@ -32,7 +34,8 @@ declare void @use_i32(i32)
 declare void @use_p32(ptr)
 
 define i32 @and_freeze_undef_multipleuses(i32 %x) {
-; CHECK-LABEL: @and_freeze_undef_multipleuses(
+; CHECK-LABEL: define i32 @and_freeze_undef_multipleuses(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    call void @use_i32(i32 0)
 ; CHECK-NEXT:    ret i32 0
 ;
@@ -43,7 +46,8 @@ define i32 @and_freeze_undef_multipleuses(i32 %x) {
 }
 
 define i32 @or_freeze_undef(i32 %x) {
-; CHECK-LABEL: @or_freeze_undef(
+; CHECK-LABEL: define i32 @or_freeze_undef(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i32 -1
 ;
   %f = freeze i32 undef
@@ -52,9 +56,10 @@ define i32 @or_freeze_undef(i32 %x) {
 }
 
 define i32 @or_freeze_undef_multipleuses(i32 %x) {
-; CHECK-LABEL: @or_freeze_undef_multipleuses(
+; CHECK-LABEL: define i32 @or_freeze_undef_multipleuses(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    call void @use_i32(i32 0)
-; CHECK-NEXT:    ret i32 [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[X]]
 ;
   %f = freeze i32 undef
   %res = or i32 %x, %f
@@ -65,8 +70,9 @@ define i32 @or_freeze_undef_multipleuses(i32 %x) {
 declare void @use_i32_i1(i32, i1)
 
 define void @or_select_multipleuses(i32 %x, i1 %y) {
-; CHECK-LABEL: @or_select_multipleuses(
-; CHECK-NEXT:    call void @use_i32_i1(i32 32, i1 [[Y:%.*]])
+; CHECK-LABEL: define void @or_select_multipleuses(
+; CHECK-SAME: i32 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    call void @use_i32_i1(i32 32, i1 [[Y]])
 ; CHECK-NEXT:    ret void
 ;
   %f = freeze i1 undef
@@ -77,8 +83,9 @@ define void @or_select_multipleuses(i32 %x, i1 %y) {
 }
 
 define void @or_select_multipleuses_logical(i32 %x, i1 %y) {
-; CHECK-LABEL: @or_select_multipleuses_logical(
-; CHECK-NEXT:    call void @use_i32_i1(i32 32, i1 [[Y:%.*]])
+; CHECK-LABEL: define void @or_select_multipleuses_logical(
+; CHECK-SAME: i32 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    call void @use_i32_i1(i32 32, i1 [[Y]])
 ; CHECK-NEXT:    ret void
 ;
   %f = freeze i1 undef
@@ -89,7 +96,7 @@ define void @or_select_multipleuses_logical(i32 %x, i1 %y) {
 }
 
 define <3 x i4> @partial_undef_vec() {
-; CHECK-LABEL: @partial_undef_vec(
+; CHECK-LABEL: define <3 x i4> @partial_undef_vec() {
 ; CHECK-NEXT:    ret <3 x i4> splat (i4 1)
 ;
   %f = freeze <3 x i4> <i4 poison, i4 1, i4 undef>
@@ -100,9 +107,10 @@ define <3 x i4> @partial_undef_vec() {
 
 define i32 @early_freeze_test1(i32 %x, i32 %y) {
 ; CHECK-LABEL: @early_freeze_test1(
-; CHECK-NEXT:    [[V1:%.*]] = add i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[V1_FR:%.*]] = freeze i32 [[V1]]
-; CHECK-NEXT:    [[V2:%.*]] = shl i32 [[V1_FR]], 1
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y:%.*]]
+; CHECK-NEXT:    [[V1:%.*]] = add i32 [[X_FR]], [[Y_FR]]
+; CHECK-NEXT:    [[V2:%.*]] = shl i32 [[V1]], 1
 ; CHECK-NEXT:    [[V3:%.*]] = and i32 [[V2]], 2
 ; CHECK-NEXT:    ret i32 [[V3]]
 ;
@@ -114,8 +122,9 @@ define i32 @early_freeze_test1(i32 %x, i32 %y) {
 }
 
 define i1 @early_freeze_test2(ptr %ptr) {
-; CHECK-LABEL: @early_freeze_test2(
-; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[PTR:%.*]], align 4
+; CHECK-LABEL: define i1 @early_freeze_test2(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[V1_FR:%.*]] = freeze i32 [[V1]]
 ; CHECK-NEXT:    [[V2:%.*]] = and i32 [[V1_FR]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[V2]], 0
@@ -129,8 +138,9 @@ define i1 @early_freeze_test2(ptr %ptr) {
 }
 
 define i32 @early_freeze_test3(i32 %v1) {
-; CHECK-LABEL: @early_freeze_test3(
-; CHECK-NEXT:    [[V1_FR:%.*]] = freeze i32 [[V1:%.*]]
+; CHECK-LABEL: define i32 @early_freeze_test3(
+; CHECK-SAME: i32 [[V1:%.*]]) {
+; CHECK-NEXT:    [[V1_FR:%.*]] = freeze i32 [[V1]]
 ; CHECK-NEXT:    [[V2:%.*]] = shl i32 [[V1_FR]], 1
 ; CHECK-NEXT:    [[V4:%.*]] = add i32 [[V2]], 3
 ; CHECK-NEXT:    ret i32 [[V4]]
@@ -144,9 +154,9 @@ define i32 @early_freeze_test3(i32 %v1) {
 
 define i32 @early_freeze_test4(i32 %v1) {
 ; CHECK-LABEL: @early_freeze_test4(
-; CHECK-NEXT:    [[V2_FR:%.*]] = freeze i32 [[V2:%.*]]
-; CHECK-NEXT:    [[V3:%.*]] = mul i32 [[V2_FR]], [[V2_FR]]
-; CHECK-NEXT:    ret i32 [[V3]]
+; CHECK-NEXT:    [[V1_FR:%.*]] = freeze i32 [[V1:%.*]]
+; CHECK-NEXT:    [[V2:%.*]] = mul i32 [[V1_FR]], [[V1_FR]]
+; CHECK-NEXT:    ret i32 [[V2]]
 ;
   %v2 = mul i32 %v1, %v1
   %v2.fr = freeze i32 %v2
@@ -156,8 +166,9 @@ define i32 @early_freeze_test4(i32 %v1) {
 ; If replace all dominated uses of v to freeze(v).
 
 define void @freeze_dominated_uses_test1(i32 %v) {
-; CHECK-LABEL: @freeze_dominated_uses_test1(
-; CHECK-NEXT:    [[V_FR:%.*]] = freeze i32 [[V:%.*]]
+; CHECK-LABEL: define void @freeze_dominated_uses_test1(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[V_FR:%.*]] = freeze i32 [[V]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[V_FR]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[V_FR]])
 ; CHECK-NEXT:    ret void
@@ -169,22 +180,23 @@ define void @freeze_dominated_uses_test1(i32 %v) {
 }
 
 define void @freeze_dominated_uses_test2(i32 %v) {
-; CHECK-LABEL: @freeze_dominated_uses_test2(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @freeze_dominated_uses_test2(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[V_FR:%.*]] = freeze i32 [[V:%.*]]
+; CHECK-NEXT:    [[V_FR:%.*]] = freeze i32 [[V]]
 ; CHECK-NEXT:    call void @use_p32(ptr nonnull [[A]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[V_FR]])
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[V_FR]], 0
-; CHECK-NEXT:    br i1 [[COND]], label [[BB0:%.*]], label [[BB1:%.*]]
-; CHECK:       bb0:
+; CHECK-NEXT:    br i1 [[COND]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK:       [[BB0]]:
 ; CHECK-NEXT:    call void @use_i32(i32 [[V_FR]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[V_FR]])
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       bb1:
+; CHECK-NEXT:    br label %[[END:.*]]
+; CHECK:       [[BB1]]:
 ; CHECK-NEXT:    call void @use_i32(i32 [[V_FR]])
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -211,18 +223,19 @@ end:
 ; If there is a duplicate freeze, it will be removed.
 
 define void @freeze_dominated_uses_test3(i32 %v, i1 %cond) {
-; CHECK-LABEL: @freeze_dominated_uses_test3(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[V_FR1:%.*]] = freeze i32 [[V:%.*]]
+; CHECK-LABEL: define void @freeze_dominated_uses_test3(
+; CHECK-SAME: i32 [[V:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[V_FR1:%.*]] = freeze i32 [[V]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[V_FR1]])
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
-; CHECK:       bb0:
+; CHECK-NEXT:    br i1 [[COND]], label %[[BB0:.*]], label %[[BB1:.*]]
+; CHECK:       [[BB0]]:
 ; CHECK-NEXT:    call void @use_i32(i32 [[V_FR1]])
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       bb1:
+; CHECK-NEXT:    br label %[[END:.*]]
+; CHECK:       [[BB1]]:
 ; CHECK-NEXT:    call void @use_i32(i32 [[V_FR1]])
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -246,28 +259,29 @@ end:
 declare i32 @__CxxFrameHandler3(...)
 
 define void @freeze_dominated_uses_catchswitch(i1 %c, i32 %x) personality ptr @__CxxFrameHandler3 {
-; CHECK-LABEL: @freeze_dominated_uses_catchswitch(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK:       if.then:
+; CHECK-LABEL: define void @freeze_dominated_uses_catchswitch(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[X:%.*]]) personality ptr @__CxxFrameHandler3 {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
 ; CHECK-NEXT:    invoke void @use_i32(i32 0)
-; CHECK-NEXT:            to label [[CLEANUP:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
-; CHECK:       if.else:
+; CHECK-NEXT:            to label %[[CLEANUP:.*]] unwind label %[[CATCH_DISPATCH:.*]]
+; CHECK:       [[IF_ELSE]]:
 ; CHECK-NEXT:    invoke void @use_i32(i32 1)
-; CHECK-NEXT:            to label [[CLEANUP]] unwind label [[CATCH_DISPATCH]]
-; CHECK:       catch.dispatch:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[IF_THEN]] ], [ [[X:%.*]], [[IF_ELSE]] ]
-; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label [[CATCH:%.*]], label %catch2] unwind to caller
-; CHECK:       catch:
+; CHECK-NEXT:            to label %[[CLEANUP]] unwind label %[[CATCH_DISPATCH]]
+; CHECK:       [[CATCH_DISPATCH]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, %[[IF_THEN]] ], [ [[X]], %[[IF_ELSE]] ]
+; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label %[[CATCH:.*]], label %catch2] unwind to caller
+; CHECK:       [[CATCH]]:
 ; CHECK-NEXT:    [[CP:%.*]] = catchpad within [[CS]] [ptr null, i32 64, ptr null]
 ; CHECK-NEXT:    [[PHI_FREEZE:%.*]] = freeze i32 [[PHI]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[PHI_FREEZE]]) [ "funclet"(token [[CP]]) ]
 ; CHECK-NEXT:    unreachable
-; CHECK:       catch2:
+; CHECK:       [[CATCH2:.*:]]
 ; CHECK-NEXT:    [[CP2:%.*]] = catchpad within [[CS]] [ptr null, i32 64, ptr null]
 ; CHECK-NEXT:    call void @use_i32(i32 [[PHI]]) [ "funclet"(token [[CP2]]) ]
 ; CHECK-NEXT:    unreachable
-; CHECK:       cleanup:
+; CHECK:       [[CLEANUP]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -303,16 +317,17 @@ cleanup:
 declare i32 @get_i32()
 
 define i32 @freeze_use_in_different_branches(i1 %c) {
-; CHECK-LABEL: @freeze_use_in_different_branches(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define i32 @freeze_use_in_different_branches(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[X:%.*]] = call i32 @get_i32()
 ; CHECK-NEXT:    [[FR:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    call void @use_i32(i32 0)
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
-; CHECK:       if:
+; CHECK-NEXT:    br i1 [[C]], label %[[IF:.*]], label %[[ELSE:.*]]
+; CHECK:       [[IF]]:
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
 ; CHECK-NEXT:    ret i32 0
-; CHECK:       else:
+; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
 ; CHECK-NEXT:    ret i32 1
@@ -334,16 +349,17 @@ else:
 }
 
 define i32 @freeze_phi_use(i1 %c) {
-; CHECK-LABEL: @freeze_phi_use(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define i32 @freeze_phi_use(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[X:%.*]] = call i32 @get_i32()
 ; CHECK-NEXT:    [[FR:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    call void @use_i32(i32 0)
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[JOIN:%.*]]
-; CHECK:       if:
-; CHECK-NEXT:    br label [[JOIN]]
-; CHECK:       join:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[FR]], [[IF]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br i1 [[C]], label %[[IF:.*]], label %[[JOIN:.*]]
+; CHECK:       [[IF]]:
+; CHECK-NEXT:    br label %[[JOIN]]
+; CHECK:       [[JOIN]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[FR]], %[[IF]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
 ; CHECK-NEXT:    ret i32 [[PHI]]
 ;
@@ -363,14 +379,15 @@ join:
 }
 
 define i32 @freeze_phi_followed_by_phi(i1 %c, i32 %y, i32 %z) {
-; CHECK-LABEL: @freeze_phi_followed_by_phi(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[JOIN:%.*]]
-; CHECK:       if:
-; CHECK-NEXT:    br label [[JOIN]]
-; CHECK:       join:
-; CHECK-NEXT:    [[X:%.*]] = phi i32 [ [[Y:%.*]], [[IF]] ], [ [[Z:%.*]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[Z]], [[IF]] ], [ [[Y]], [[ENTRY]] ]
+; CHECK-LABEL: define i32 @freeze_phi_followed_by_phi(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[C]], label %[[IF:.*]], label %[[JOIN:.*]]
+; CHECK:       [[IF]]:
+; CHECK-NEXT:    br label %[[JOIN]]
+; CHECK:       [[JOIN]]:
+; CHECK-NEXT:    [[X:%.*]] = phi i32 [ [[Y]], %[[IF]] ], [ [[Z]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[Z]], %[[IF]] ], [ [[Y]], %[[ENTRY]] ]
 ; CHECK-NEXT:    [[FR:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
@@ -392,17 +409,18 @@ join:
 }
 
 define i32 @freeze_invoke_use_in_phi(i1 %c) personality ptr undef {
-; CHECK-LABEL: @freeze_invoke_use_in_phi(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define i32 @freeze_invoke_use_in_phi(
+; CHECK-SAME: i1 [[C:%.*]]) personality ptr undef {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[X:%.*]] = invoke i32 @get_i32()
-; CHECK-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[INVOKE_UNWIND:%.*]]
-; CHECK:       invoke.cont:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ 0, [[INVOKE_CONT]] ]
+; CHECK-NEXT:            to label %[[INVOKE_CONT:.*]] unwind label %[[INVOKE_UNWIND:.*]]
+; CHECK:       [[INVOKE_CONT]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], %[[ENTRY]] ], [ 0, %[[INVOKE_CONT]] ]
 ; CHECK-NEXT:    [[FR:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[PHI]])
-; CHECK-NEXT:    br label [[INVOKE_CONT]]
-; CHECK:       invoke.unwind:
+; CHECK-NEXT:    br label %[[INVOKE_CONT]]
+; CHECK:       [[INVOKE_UNWIND]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = landingpad i8
 ; CHECK-NEXT:            cleanup
 ; CHECK-NEXT:    unreachable
@@ -424,18 +442,19 @@ invoke.unwind:
 }
 
 define i32 @freeze_invoke_use_after_phi(i1 %c) personality ptr undef {
-; CHECK-LABEL: @freeze_invoke_use_after_phi(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define i32 @freeze_invoke_use_after_phi(
+; CHECK-SAME: i1 [[C:%.*]]) personality ptr undef {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[X:%.*]] = invoke i32 @get_i32()
-; CHECK-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[INVOKE_UNWIND:%.*]]
-; CHECK:       invoke.cont:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ 0, [[INVOKE_CONT]] ]
+; CHECK-NEXT:            to label %[[INVOKE_CONT:.*]] unwind label %[[INVOKE_UNWIND:.*]]
+; CHECK:       [[INVOKE_CONT]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], %[[ENTRY]] ], [ 0, %[[INVOKE_CONT]] ]
 ; CHECK-NEXT:    [[FR:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[PHI]])
-; CHECK-NEXT:    br label [[INVOKE_CONT]]
-; CHECK:       invoke.unwind:
+; CHECK-NEXT:    br label %[[INVOKE_CONT]]
+; CHECK:       [[INVOKE_UNWIND]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = landingpad i8
 ; CHECK-NEXT:            cleanup
 ; CHECK-NEXT:    unreachable
@@ -458,17 +477,18 @@ invoke.unwind:
 }
 
 define i32 @freeze_callbr_use_after_phi(i1 %c) {
-; CHECK-LABEL: @freeze_callbr_use_after_phi(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define i32 @freeze_callbr_use_after_phi(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[X:%.*]] = callbr i32 asm sideeffect "", "=r"() #[[ATTR1:[0-9]+]]
-; CHECK-NEXT:            to label [[CALLBR_CONT:%.*]] []
-; CHECK:       callbr.cont:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], [[ENTRY:%.*]] ], [ 0, [[CALLBR_CONT]] ]
+; CHECK-NEXT:            to label %[[CALLBR_CONT:.*]] []
+; CHECK:       [[CALLBR_CONT]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[X]], %[[ENTRY]] ], [ 0, %[[CALLBR_CONT]] ]
 ; CHECK-NEXT:    call void @use_i32(i32 [[X]])
 ; CHECK-NEXT:    [[FR:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[FR]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[PHI]])
-; CHECK-NEXT:    br label [[CALLBR_CONT]]
+; CHECK-NEXT:    br label %[[CALLBR_CONT]]
 ;
 entry:
   %x = callbr i32 asm sideeffect "", "=r"()
@@ -484,8 +504,9 @@ callbr.cont:
 }
 
 define i1 @combine_and_after_freezing_uses(i32 %x) {
-; CHECK-LABEL: @combine_and_after_freezing_uses(
-; CHECK-NEXT:    [[X_FR:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-LABEL: define i1 @combine_and_after_freezing_uses(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X_FR]], 15
 ; CHECK-NEXT:    [[AND:%.*]] = icmp eq i32 [[TMP1]], 15
 ; CHECK-NEXT:    ret i1 [[AND]]
@@ -501,12 +522,13 @@ define i1 @combine_and_after_freezing_uses(i32 %x) {
 
 declare i1 @mock_use(i64, i64)
 define i1 @fully_propagate_freeze(i32 %0, i32 noundef %1) {
-; CHECK-LABEL: @fully_propagate_freeze(
-; CHECK-NEXT:    [[DOTFR:%.*]] = freeze i32 [[TMP0:%.*]]
+; CHECK-LABEL: define i1 @fully_propagate_freeze(
+; CHECK-SAME: i32 [[TMP0:%.*]], i32 noundef [[TMP1:%.*]]) {
+; CHECK-NEXT:    [[DOTFR:%.*]] = freeze i32 [[TMP0]]
 ; CHECK-NEXT:    [[DR:%.*]] = lshr i32 [[DOTFR]], 2
 ; CHECK-NEXT:    [[IDX1:%.*]] = zext nneg i32 [[DR]] to i64
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[DR]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[ADD]], [[TMP1:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[ADD]], [[TMP1]]
 ; CHECK-NEXT:    [[IDX2:%.*]] = zext nneg i32 [[DR]] to i64
 ; CHECK-NEXT:    [[V:%.*]] = call i1 @mock_use(i64 [[IDX1]], i64 [[IDX2]])
 ; CHECK-NEXT:    [[RET:%.*]] = and i1 [[V]], [[CMP]]
@@ -524,8 +546,9 @@ define i1 @fully_propagate_freeze(i32 %0, i32 noundef %1) {
 }
 
 define i32 @propagate_drop_flags_add(i32 %arg) {
-; CHECK-LABEL: @propagate_drop_flags_add(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_flags_add(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = add i32 [[ARG_FR]], 2
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
@@ -535,8 +558,9 @@ define i32 @propagate_drop_flags_add(i32 %arg) {
 }
 
 define i32 @propagate_drop_flags_add_foldaway(i32 noundef %arg) {
-; CHECK-LABEL: @propagate_drop_flags_add_foldaway(
-; CHECK-NEXT:    [[V1:%.*]] = add i32 [[ARG:%.*]], 2
+; CHECK-LABEL: define i32 @propagate_drop_flags_add_foldaway(
+; CHECK-SAME: i32 noundef [[ARG:%.*]]) {
+; CHECK-NEXT:    [[V1:%.*]] = add i32 [[ARG]], 2
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
   %v1 = add nsw nuw i32 %arg, 2
@@ -545,8 +569,9 @@ define i32 @propagate_drop_flags_add_foldaway(i32 noundef %arg) {
 }
 
 define i32 @propagate_drop_flags_sub(i32 %arg) {
-; CHECK-LABEL: @propagate_drop_flags_sub(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_flags_sub(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = add i32 [[ARG_FR]], -2
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
@@ -556,8 +581,9 @@ define i32 @propagate_drop_flags_sub(i32 %arg) {
 }
 
 define i32 @propagate_drop_flags_mul(i32 %arg) {
-; CHECK-LABEL: @propagate_drop_flags_mul(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_flags_mul(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = shl i32 [[ARG_FR]], 1
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
@@ -567,8 +593,9 @@ define i32 @propagate_drop_flags_mul(i32 %arg) {
 }
 
 define i32 @propagate_drop_flags_udiv(i32 %arg) {
-; CHECK-LABEL: @propagate_drop_flags_udiv(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_flags_udiv(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[V11:%.*]] = lshr i32 [[ARG_FR]], 1
 ; CHECK-NEXT:    ret i32 [[V11]]
 ;
@@ -578,8 +605,9 @@ define i32 @propagate_drop_flags_udiv(i32 %arg) {
 }
 
 define i32 @propagate_drop_flags_sdiv(i32 %arg) {
-; CHECK-LABEL: @propagate_drop_flags_sdiv(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_flags_sdiv(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = ashr i32 [[ARG_FR]], 1
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
@@ -589,8 +617,9 @@ define i32 @propagate_drop_flags_sdiv(i32 %arg) {
 }
 
 define i32 @propagate_drop_shl1(i32 %arg) {
-; CHECK-LABEL: @propagate_drop_shl1(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_shl1(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = shl i32 [[ARG_FR]], 2
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
@@ -600,8 +629,9 @@ define i32 @propagate_drop_shl1(i32 %arg) {
 }
 
 define i32 @propagate_drop_shl2(i32 %arg, i32 %unknown) {
-; CHECK-LABEL: @propagate_drop_shl2(
-; CHECK-NEXT:    [[V1:%.*]] = shl nuw nsw i32 [[ARG:%.*]], [[UNKNOWN:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_shl2(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[UNKNOWN:%.*]]) {
+; CHECK-NEXT:    [[V1:%.*]] = shl nuw nsw i32 [[ARG]], [[UNKNOWN]]
 ; CHECK-NEXT:    [[V1_FR:%.*]] = freeze i32 [[V1]]
 ; CHECK-NEXT:    ret i32 [[V1_FR]]
 ;
@@ -611,8 +641,9 @@ define i32 @propagate_drop_shl2(i32 %arg, i32 %unknown) {
 }
 
 define i32 @propagate_drop_ashr1(i32 %arg) {
-; CHECK-LABEL: @propagate_drop_ashr1(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_ashr1(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = ashr i32 [[ARG_FR]], 2
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
@@ -622,8 +653,9 @@ define i32 @propagate_drop_ashr1(i32 %arg) {
 }
 
 define i32 @propagate_drop_ashr2(i32 %arg, i32 %unknown) {
-; CHECK-LABEL: @propagate_drop_ashr2(
-; CHECK-NEXT:    [[V1:%.*]] = ashr exact i32 [[ARG:%.*]], [[UNKNOWN:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_ashr2(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[UNKNOWN:%.*]]) {
+; CHECK-NEXT:    [[V1:%.*]] = ashr exact i32 [[ARG]], [[UNKNOWN]]
 ; CHECK-NEXT:    [[V1_FR:%.*]] = freeze i32 [[V1]]
 ; CHECK-NEXT:    ret i32 [[V1_FR]]
 ;
@@ -633,8 +665,9 @@ define i32 @propagate_drop_ashr2(i32 %arg, i32 %unknown) {
 }
 
 define i32 @propagate_drop_lshr1(i32 %arg) {
-; CHECK-LABEL: @propagate_drop_lshr1(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_lshr1(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = lshr i32 [[ARG_FR]], 2
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
@@ -644,8 +677,9 @@ define i32 @propagate_drop_lshr1(i32 %arg) {
 }
 
 define i32 @propagate_drop_lshr2(i32 %arg, i32 %unknown) {
-; CHECK-LABEL: @propagate_drop_lshr2(
-; CHECK-NEXT:    [[V1:%.*]] = lshr exact i32 [[ARG:%.*]], [[UNKNOWN:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_lshr2(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[UNKNOWN:%.*]]) {
+; CHECK-NEXT:    [[V1:%.*]] = lshr exact i32 [[ARG]], [[UNKNOWN]]
 ; CHECK-NEXT:    [[V1_FR:%.*]] = freeze i32 [[V1]]
 ; CHECK-NEXT:    ret i32 [[V1_FR]]
 ;
@@ -655,8 +689,9 @@ define i32 @propagate_drop_lshr2(i32 %arg, i32 %unknown) {
 }
 
 define ptr @propagate_drop_gep1(ptr %arg) {
-; CHECK-LABEL: @propagate_drop_gep1(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze ptr [[ARG:%.*]]
+; CHECK-LABEL: define ptr @propagate_drop_gep1(
+; CHECK-SAME: ptr [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze ptr [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = getelementptr i8, ptr [[ARG_FR]], i64 16
 ; CHECK-NEXT:    ret ptr [[V1]]
 ;
@@ -666,8 +701,9 @@ define ptr @propagate_drop_gep1(ptr %arg) {
 }
 
 define float @propagate_drop_fneg(float %arg) {
-; CHECK-LABEL: @propagate_drop_fneg(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @propagate_drop_fneg(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = fneg float [[ARG_FR]]
 ; CHECK-NEXT:    ret float [[V1]]
 ;
@@ -678,8 +714,9 @@ define float @propagate_drop_fneg(float %arg) {
 
 
 define float @propagate_drop_fadd(float %arg) {
-; CHECK-LABEL: @propagate_drop_fadd(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @propagate_drop_fadd(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = fadd float [[ARG_FR]], 2.000000e+00
 ; CHECK-NEXT:    ret float [[V1]]
 ;
@@ -689,8 +726,9 @@ define float @propagate_drop_fadd(float %arg) {
 }
 
 define float @propagate_drop_fsub(float %arg) {
-; CHECK-LABEL: @propagate_drop_fsub(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @propagate_drop_fsub(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = fadd float [[ARG_FR]], -2.000000e+00
 ; CHECK-NEXT:    ret float [[V1]]
 ;
@@ -700,8 +738,9 @@ define float @propagate_drop_fsub(float %arg) {
 }
 
 define float @propagate_drop_fmul(float %arg) {
-; CHECK-LABEL: @propagate_drop_fmul(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @propagate_drop_fmul(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = fmul float [[ARG_FR]], 2.000000e+00
 ; CHECK-NEXT:    ret float [[V1]]
 ;
@@ -711,8 +750,9 @@ define float @propagate_drop_fmul(float %arg) {
 }
 
 define float @propagate_drop_fdiv(float %arg) {
-; CHECK-LABEL: @propagate_drop_fdiv(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @propagate_drop_fdiv(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = fmul float [[ARG_FR]], 5.000000e-01
 ; CHECK-NEXT:    ret float [[V1]]
 ;
@@ -722,8 +762,9 @@ define float @propagate_drop_fdiv(float %arg) {
 }
 
 define float @propagate_drop_frem(float %arg) {
-; CHECK-LABEL: @propagate_drop_frem(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define float @propagate_drop_frem(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = frem float [[ARG_FR]], 2.000000e+00
 ; CHECK-NEXT:    ret float [[V1]]
 ;
@@ -733,8 +774,9 @@ define float @propagate_drop_frem(float %arg) {
 }
 
 define i1 @propagate_drop_fcmp(float %arg) {
-; CHECK-LABEL: @propagate_drop_fcmp(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG:%.*]]
+; CHECK-LABEL: define i1 @propagate_drop_fcmp(
+; CHECK-SAME: float [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze float [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = fcmp une float [[ARG_FR]], 2.000000e+00
 ; CHECK-NEXT:    ret i1 [[V1]]
 ;
@@ -744,8 +786,9 @@ define i1 @propagate_drop_fcmp(float %arg) {
 }
 
 define float @propagate_drop_fmath_select(i1 %arg) {
-; CHECK-LABEL: @propagate_drop_fmath_select(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i1 [[ARG:%.*]]
+; CHECK-LABEL: define float @propagate_drop_fmath_select(
+; CHECK-SAME: i1 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i1 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = select i1 [[ARG_FR]], float 1.000000e+00, float -1.000000e+00
 ; CHECK-NEXT:    ret float [[V1]]
 ;
@@ -755,15 +798,16 @@ define float @propagate_drop_fmath_select(i1 %arg) {
 }
 
 define void @fold_phi_noop(i32 noundef %init, i32 %n) {
-; CHECK-LABEL: @fold_phi_noop(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-LABEL: define void @fold_phi_noop(
+; CHECK-SAME: i32 noundef [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -781,16 +825,17 @@ exit:                                             ; preds = %loop
 }
 
 define void @fold_phi_through(i32 %init, i32 %n) {
-; CHECK-LABEL: @fold_phi_through(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[INIT:%.*]]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-LABEL: define void @fold_phi_through(
+; CHECK-SAME: i32 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[INIT]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -808,16 +853,17 @@ exit:                                             ; preds = %loop
 }
 
 define void @fold_phi_drop_flags(i32 %init, i32 %n) {
-; CHECK-LABEL: @fold_phi_drop_flags(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INIT_FR:%.*]] = freeze i32 [[INIT:%.*]]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT_FR]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-LABEL: define void @fold_phi_drop_flags(
+; CHECK-SAME: i32 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[INIT_FR:%.*]] = freeze i32 [[INIT]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT_FR]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -835,16 +881,17 @@ exit:                                             ; preds = %loop
 }
 
 define void @fold_phi_non_add(i32 %init, i32 %n) {
-; CHECK-LABEL: @fold_phi_non_add(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[INIT:%.*]]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-LABEL: define void @fold_phi_non_add(
+; CHECK-SAME: i32 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i32 [[INIT]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[TMP0]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = shl i32 [[I]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -862,16 +909,17 @@ exit:                                             ; preds = %loop
 }
 
 define void @fold_phi_gep(ptr %init, ptr %end) {
-; CHECK-LABEL: @fold_phi_gep(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = freeze ptr [[INIT:%.*]]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[TMP0]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-LABEL: define void @fold_phi_gep(
+; CHECK-SAME: ptr [[INIT:%.*]], ptr [[END:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze ptr [[INIT]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[TMP0]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = getelementptr i8, ptr [[I]], i64 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT]], [[END:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT]], [[END]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -889,18 +937,18 @@ exit:                                             ; preds = %loop
 }
 
 ; The recurrence for the GEP offset can't produce poison so the freeze should
-; be pushed through to the ptr, but this is not currently supported.
+; be pushed through to the ptr.
 define void @fold_phi_gep_phi_offset(ptr %init, ptr %end, i64 noundef %n) {
 ; CHECK-LABEL: @fold_phi_gep_phi_offset(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze ptr [[INIT:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[TMP0]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFF_NEXT]] = shl i64 [[OFF]], 3
-; CHECK-NEXT:    [[I_NEXT:%.*]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
-; CHECK-NEXT:    [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
+; CHECK-NEXT:    [[I_NEXT]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -921,19 +969,19 @@ exit:                                             ; preds = %loop
   ret void
 }
 
-; Offset is still guaranteed not to be poison, so the freeze could be moved
-; here if we strip inbounds from the GEP, but this is not currently supported.
+; Offset is still guaranteed not to be poison, so the freeze can be moved
+; here if we strip inbounds from the GEP.
 define void @fold_phi_gep_inbounds_phi_offset(ptr %init, ptr %end, i64 noundef %n) {
 ; CHECK-LABEL: @fold_phi_gep_inbounds_phi_offset(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze ptr [[INIT:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[TMP0]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFF_NEXT]] = shl i64 [[OFF]], 3
-; CHECK-NEXT:    [[I_NEXT:%.*]] = getelementptr inbounds i8, ptr [[I]], i64 [[OFF_NEXT]]
-; CHECK-NEXT:    [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
+; CHECK-NEXT:    [[I_NEXT]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -954,18 +1002,19 @@ exit:                                             ; preds = %loop
   ret void
 }
 
-; GEP can produce poison, check freeze isn't moved.
-define void @cant_fold_phi_gep_phi_offset(ptr %init, ptr %end, i64 %n) {
-; CHECK-LABEL: @cant_fold_phi_gep_phi_offset(
+; Same as previous, but also requires freezing %n.
+define void @fold_phi_gep_phi_offset_multiple(ptr %init, ptr %end, i64 %n) {
+; CHECK-LABEL: @fold_phi_gep_phi_offset_multiple(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze ptr [[INIT:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[N:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[I_NEXT_FR:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFF:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi ptr [ [[TMP0]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFF:%.*]] = phi i64 [ [[TMP1]], [[ENTRY]] ], [ [[OFF_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFF_NEXT]] = shl i64 [[OFF]], 3
-; CHECK-NEXT:    [[I_NEXT:%.*]] = getelementptr inbounds i8, ptr [[I]], i64 [[OFF_NEXT]]
-; CHECK-NEXT:    [[I_NEXT_FR]] = freeze ptr [[I_NEXT]]
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT_FR]], [[END:%.*]]
+; CHECK-NEXT:    [[I_NEXT]] = getelementptr i8, ptr [[I]], i64 [[OFF_NEXT]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[I_NEXT]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
@@ -987,17 +1036,18 @@ exit:                                             ; preds = %loop
 }
 
 define void @fold_phi_multiple_insts(i32 %init, i32 %n) {
-; CHECK-LABEL: @fold_phi_multiple_insts(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INIT_FR:%.*]] = freeze i32 [[INIT:%.*]]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT_FR]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-LABEL: define void @fold_phi_multiple_insts(
+; CHECK-SAME: i32 [[INIT:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[INIT_FR:%.*]] = freeze i32 [[INIT]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT_FR]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_SQ:%.*]] = mul i32 [[I]], [[I]]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I_SQ]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -1016,20 +1066,21 @@ exit:                                             ; preds = %loop
 }
 
 define void @fold_phi_multiple_back_edges(i32 %init, i32 %n, i32 %m) {
-; CHECK-LABEL: @fold_phi_multiple_back_edges(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INIT_FR:%.*]] = freeze i32 [[INIT:%.*]]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT_FR]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ], [ [[I_NEXT2:%.*]], [[LOOP_LATCH2:%.*]] ]
+; CHECK-LABEL: define void @fold_phi_multiple_back_edges(
+; CHECK-SAME: i32 [[INIT:%.*]], i32 [[N:%.*]], i32 [[M:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[INIT_FR:%.*]] = freeze i32 [[INIT]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT_FR]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ], [ [[I_NEXT2:%.*]], %[[LOOP_LATCH2:.*]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[LOOP_LATCH2]]
-; CHECK:       loop.latch2:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[LOOP_LATCH2]]
+; CHECK:       [[LOOP_LATCH2]]:
 ; CHECK-NEXT:    [[I_NEXT2]] = add i32 [[I]], 2
-; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[I_NEXT]], [[M:%.*]]
-; CHECK-NEXT:    br i1 [[COND2]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    [[COND2:%.*]] = icmp eq i32 [[I_NEXT]], [[M]]
+; CHECK-NEXT:    br i1 [[COND2]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -1052,18 +1103,19 @@ exit:                                             ; preds = %loop
 }
 
 define void @fold_phi_multiple_start_values(i1 %c, i32 %init, i32 %init2, i32 %n) {
-; CHECK-LABEL: @fold_phi_multiple_start_values(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF:%.*]], label [[LOOP:%.*]]
-; CHECK:       if:
-; CHECK-NEXT:    br label [[LOOP]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT:%.*]], [[ENTRY:%.*]] ], [ [[INIT2:%.*]], [[IF]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-LABEL: define void @fold_phi_multiple_start_values(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[INIT:%.*]], i32 [[INIT2:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[C]], label %[[IF:.*]], label %[[LOOP:.*]]
+; CHECK:       [[IF]]:
+; CHECK-NEXT:    br label %[[LOOP]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[INIT2]], %[[IF]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_FR:%.*]] = freeze i32 [[I]]
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I_FR]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -1084,21 +1136,22 @@ exit:
 }
 
 define void @fold_phi_invoke_start_value(i32 %n) personality ptr undef {
-; CHECK-LABEL: @fold_phi_invoke_start_value(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @fold_phi_invoke_start_value(
+; CHECK-SAME: i32 [[N:%.*]]) personality ptr undef {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[INIT:%.*]] = invoke i32 @get_i32()
-; CHECK-NEXT:            to label [[LOOP:%.*]] unwind label [[UNWIND:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:            to label %[[LOOP:.*]] unwind label %[[UNWIND:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_FR:%.*]] = freeze i32 [[I]]
 ; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I_FR]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       unwind:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[UNWIND]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = landingpad i8
 ; CHECK-NEXT:            cleanup
 ; CHECK-NEXT:    unreachable
-; CHECK:       exit:
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -1121,20 +1174,21 @@ exit:
 }
 
 define void @fold_phi_invoke_noundef_start_value(i32 %n) personality ptr undef {
-; CHECK-LABEL: @fold_phi_invoke_noundef_start_value(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define void @fold_phi_invoke_noundef_start_value(
+; CHECK-SAME: i32 [[N:%.*]]) personality ptr undef {
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[INIT:%.*]] = invoke noundef i32 @get_i32()
-; CHECK-NEXT:            to label [[LOOP:%.*]] unwind label [[UNWIND:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT]], [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:            to label %[[LOOP:.*]] unwind label %[[UNWIND:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[INIT]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       unwind:
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[UNWIND]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = landingpad i8
 ; CHECK-NEXT:            cleanup
 ; CHECK-NEXT:    unreachable
-; CHECK:       exit:
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -1157,8 +1211,9 @@ exit:
 }
 
 define ptr @freeze_load_noundef(ptr %ptr) {
-; CHECK-LABEL: @freeze_load_noundef(
-; CHECK-NEXT:    [[P:%.*]] = load ptr, ptr [[PTR:%.*]], align 8, !noundef [[META0:![0-9]+]]
+; CHECK-LABEL: define ptr @freeze_load_noundef(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[P:%.*]] = load ptr, ptr [[PTR]], align 8, !noundef [[META0:![0-9]+]]
 ; CHECK-NEXT:    ret ptr [[P]]
 ;
   %p = load ptr, ptr %ptr, !noundef !0
@@ -1167,8 +1222,9 @@ define ptr @freeze_load_noundef(ptr %ptr) {
 }
 
 define ptr @freeze_load_dereferenceable(ptr %ptr) {
-; CHECK-LABEL: @freeze_load_dereferenceable(
-; CHECK-NEXT:    [[P:%.*]] = load ptr, ptr [[PTR:%.*]], align 8, !dereferenceable [[META1:![0-9]+]]
+; CHECK-LABEL: define ptr @freeze_load_dereferenceable(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[P:%.*]] = load ptr, ptr [[PTR]], align 8, !dereferenceable [[META1:![0-9]+]]
 ; CHECK-NEXT:    ret ptr [[P]]
 ;
   %p = load ptr, ptr %ptr, !dereferenceable !1
@@ -1177,8 +1233,9 @@ define ptr @freeze_load_dereferenceable(ptr %ptr) {
 }
 
 define ptr @freeze_load_dereferenceable_or_null(ptr %ptr) {
-; CHECK-LABEL: @freeze_load_dereferenceable_or_null(
-; CHECK-NEXT:    [[P:%.*]] = load ptr, ptr [[PTR:%.*]], align 8, !dereferenceable_or_null [[META1]]
+; CHECK-LABEL: define ptr @freeze_load_dereferenceable_or_null(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[P:%.*]] = load ptr, ptr [[PTR]], align 8, !dereferenceable_or_null [[META1]]
 ; CHECK-NEXT:    ret ptr [[P]]
 ;
   %p = load ptr, ptr %ptr, !dereferenceable_or_null !1
@@ -1187,8 +1244,9 @@ define ptr @freeze_load_dereferenceable_or_null(ptr %ptr) {
 }
 
 define i32 @freeze_load_with_range(ptr %ptr) {
-; CHECK-LABEL: @freeze_load_with_range(
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[PTR:%.*]], align 4, !range [[RNG2:![0-9]+]]
+; CHECK-LABEL: define i32 @freeze_load_with_range(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[PTR]], align 4, !range [[RNG2:![0-9]+]]
 ; CHECK-NEXT:    [[X_FR:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[X_FR]]
 ;
@@ -1200,7 +1258,7 @@ define i32 @freeze_load_with_range(ptr %ptr) {
 declare i32 @foo.i32()
 
 define i32 @freeze_call_with_range() {
-; CHECK-LABEL: @freeze_call_with_range(
+; CHECK-LABEL: define i32 @freeze_call_with_range() {
 ; CHECK-NEXT:    [[X:%.*]] = call i32 @foo.i32(), !range [[RNG2]]
 ; CHECK-NEXT:    [[X_FR:%.*]] = freeze i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[X_FR]]
@@ -1213,8 +1271,9 @@ define i32 @freeze_call_with_range() {
 declare i32 @llvm.ctpop.i32(i32)
 
 define i32 @freeze_ctpop(i32 %x) {
-; CHECK-LABEL: @freeze_ctpop(
-; CHECK-NEXT:    [[Y:%.*]] = lshr i32 2047, [[X:%.*]]
+; CHECK-LABEL: define i32 @freeze_ctpop(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[Y:%.*]] = lshr i32 2047, [[X]]
 ; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
 ; CHECK-NEXT:    [[CTPOP:%.*]] = call range(i32 0, 33) i32 @llvm.ctpop.i32(i32 [[Y_FR]])
 ; CHECK-NEXT:    ret i32 [[CTPOP]]
@@ -1226,8 +1285,9 @@ define i32 @freeze_ctpop(i32 %x) {
 }
 
 define i32 @freeze_zext_nneg(i8 %x) {
-; CHECK-LABEL: @freeze_zext_nneg(
-; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X:%.*]]
+; CHECK-LABEL: define i32 @freeze_zext_nneg(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X]]
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[X_FR]] to i32
 ; CHECK-NEXT:    ret i32 [[ZEXT]]
 ;
@@ -1237,8 +1297,9 @@ define i32 @freeze_zext_nneg(i8 %x) {
 }
 
 define float @freeze_uitofp_nneg(i8 %x) {
-; CHECK-LABEL: @freeze_uitofp_nneg(
-; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X:%.*]]
+; CHECK-LABEL: define float @freeze_uitofp_nneg(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X]]
 ; CHECK-NEXT:    [[UITOFP:%.*]] = uitofp i8 [[X_FR]] to float
 ; CHECK-NEXT:    ret float [[UITOFP]]
 ;
@@ -1248,8 +1309,9 @@ define float @freeze_uitofp_nneg(i8 %x) {
 }
 
 define i32 @propagate_drop_flags_or(i32 %arg) {
-; CHECK-LABEL: @propagate_drop_flags_or(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_flags_or(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = or i32 [[ARG_FR]], 2
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
@@ -1259,8 +1321,9 @@ define i32 @propagate_drop_flags_or(i32 %arg) {
 }
 
 define i32 @propagate_drop_flags_trunc(i64 %arg) {
-; CHECK-LABEL: @propagate_drop_flags_trunc(
-; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i64 [[ARG:%.*]]
+; CHECK-LABEL: define i32 @propagate_drop_flags_trunc(
+; CHECK-SAME: i64 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i64 [[ARG]]
 ; CHECK-NEXT:    [[V1:%.*]] = trunc i64 [[ARG_FR]] to i32
 ; CHECK-NEXT:    ret i32 [[V1]]
 ;
@@ -1270,8 +1333,9 @@ define i32 @propagate_drop_flags_trunc(i64 %arg) {
 }
 
 define ptr @propagate_drop_flags_gep_nusw(ptr %p) {
-; CHECK-LABEL: @propagate_drop_flags_gep_nusw(
-; CHECK-NEXT:    [[P_FR:%.*]] = freeze ptr [[P:%.*]]
+; CHECK-LABEL: define ptr @propagate_drop_flags_gep_nusw(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[P_FR:%.*]] = freeze ptr [[P]]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[P_FR]], i64 1
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
@@ -1281,8 +1345,9 @@ define ptr @propagate_drop_flags_gep_nusw(ptr %p) {
 }
 
 define ptr @propagate_drop_flags_gep_nuw(ptr %p) {
-; CHECK-LABEL: @propagate_drop_flags_gep_nuw(
-; CHECK-NEXT:    [[P_FR:%.*]] = freeze ptr [[P:%.*]]
+; CHECK-LABEL: define ptr @propagate_drop_flags_gep_nuw(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[P_FR:%.*]] = freeze ptr [[P]]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[P_FR]], i64 1
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
@@ -1292,8 +1357,9 @@ define ptr @propagate_drop_flags_gep_nuw(ptr %p) {
 }
 
 define i1 @propagate_drop_flags_icmp(i32 %a, i32 %b) {
-; CHECK-LABEL: @propagate_drop_flags_icmp(
-; CHECK-NEXT:    [[A_FR:%.*]] = freeze i32 [[A:%.*]]
+; CHECK-LABEL: define i1 @propagate_drop_flags_icmp(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[A_FR:%.*]] = freeze i32 [[A]]
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ult i32 [[A_FR]], 3
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
@@ -1305,8 +1371,9 @@ define i1 @propagate_drop_flags_icmp(i32 %a, i32 %b) {
 declare i32 @llvm.umax.i32(i32 %a, i32 %b)
 
 define i32 @freeze_call_with_range_attr(i32 %a) {
-; CHECK-LABEL: @freeze_call_with_range_attr(
-; CHECK-NEXT:    [[Y:%.*]] = lshr i32 2047, [[A:%.*]]
+; CHECK-LABEL: define i32 @freeze_call_with_range_attr(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[Y:%.*]] = lshr i32 2047, [[A]]
 ; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
 ; CHECK-NEXT:    [[X:%.*]] = call i32 @llvm.umax.i32(i32 [[Y_FR]], i32 50)
 ; CHECK-NEXT:    ret i32 [[X]]
@@ -1320,9 +1387,10 @@ define i32 @freeze_call_with_range_attr(i32 %a) {
 declare ptr @llvm.ptrmask.p0.i64(ptr, i64)
 
 define ptr @freeze_ptrmask_align(ptr %p, i64 noundef %m) {
-; CHECK-LABEL: @freeze_ptrmask_align(
-; CHECK-NEXT:    [[P_FR:%.*]] = freeze ptr [[P:%.*]]
-; CHECK-NEXT:    [[MASK:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[P_FR]], i64 [[M:%.*]])
+; CHECK-LABEL: define ptr @freeze_ptrmask_align(
+; CHECK-SAME: ptr [[P:%.*]], i64 noundef [[M:%.*]]) {
+; CHECK-NEXT:    [[P_FR:%.*]] = freeze ptr [[P]]
+; CHECK-NEXT:    [[MASK:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[P_FR]], i64 [[M]])
 ; CHECK-NEXT:    ret ptr [[MASK]]
 ;
   %mask = call align(4) ptr @llvm.ptrmask.p0.i64(ptr %p, i64 %m)
@@ -1331,9 +1399,10 @@ define ptr @freeze_ptrmask_align(ptr %p, i64 noundef %m) {
 }
 
 define ptr @freeze_ptrmask_nonnull(ptr %p, i64 noundef %m) {
-; CHECK-LABEL: @freeze_ptrmask_nonnull(
-; CHECK-NEXT:    [[P_FR:%.*]] = freeze ptr [[P:%.*]]
-; CHECK-NEXT:    [[MASK:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[P_FR]], i64 [[M:%.*]])
+; CHECK-LABEL: define ptr @freeze_ptrmask_nonnull(
+; CHECK-SAME: ptr [[P:%.*]], i64 noundef [[M:%.*]]) {
+; CHECK-NEXT:    [[P_FR:%.*]] = freeze ptr [[P]]
+; CHECK-NEXT:    [[MASK:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[P_FR]], i64 [[M]])
 ; CHECK-NEXT:    ret ptr [[MASK]]
 ;
   %mask = call nonnull ptr @llvm.ptrmask.p0.i64(ptr %p, i64 %m)
@@ -1345,9 +1414,6 @@ define ptr @freeze_ptrmask_nonnull(ptr %p, i64 noundef %m) {
 !1 = !{i64 4}
 !2 = !{i32 0, i32 100}
 ;.
-; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-; CHECK: attributes #[[ATTR1]] = { nounwind }
-;.
 ; CHECK: [[META0]] = !{}
 ; CHECK: [[META1]] = !{i64 4}
 ; CHECK: [[RNG2]] = !{i32 0, i32 100}
diff --git a/llvm/test/Transforms/InstCombine/icmp-add.ll b/llvm/test/Transforms/InstCombine/icmp-add.ll
index 1a41c1f3e1045..8449c7c5ea935 100644
--- a/llvm/test/Transforms/InstCombine/icmp-add.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-add.ll
@@ -3209,13 +3209,11 @@ entry:
   ret i1 %cmp
 }
 
-; TODO: should be canonicalized to (x - 4) u> 2
 define i1 @zext_range_check_ugt(i8 %x) {
 ; CHECK-LABEL: @zext_range_check_ugt(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[X:%.*]] to i32
-; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i32 [[CONV]], -7
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP0]], -3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i8 [[X:%.*]], -7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[TMP0]], -3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
 entry:
@@ -3225,13 +3223,11 @@ entry:
   ret i1 %cmp
 }
 
-; TODO: should be canonicalized to (x - 4) u> 2
 define i1 @zext_range_check_ult_alter(i8 %x) {
 ; CHECK-LABEL: @zext_range_check_ult_alter(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[X:%.*]] to i32
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV]], -7
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD]], -3
+; CHECK-NEXT:    [[TMP0:%.*]] = add i8 [[X:%.*]], -7
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[TMP0]], -3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
 entry:
@@ -3241,6 +3237,19 @@ entry:
   ret i1 %cmp
 }
 
+define i1 @zext_range_check_ult_alter2(i8 %x) {
+; CHECK-LABEL: @zext_range_check_ult_alter2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[X:%.*]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+entry:
+  %conv = zext i8 %x to i32
+  %add = add i32 %conv, -4
+  %cmp = icmp ult i32 %add, 253
+  ret i1 %cmp
+}
+
 define i1 @zext_range_check_mergable(i8 %x) {
 ; CHECK-LABEL: @zext_range_check_mergable(
 ; CHECK-NEXT:    [[COND:%.*]] = icmp slt i8 [[X:%.*]], 7
@@ -3286,17 +3295,148 @@ entry:
   ret i1 %cmp
 }
 
-define i1 @zext_range_check_ult_range_check_failure(i8 %x) {
-; CHECK-LABEL: @zext_range_check_ult_range_check_failure(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[X:%.*]] to i32
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CONV]], -4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD]], 253
-; CHECK-NEXT:    ret i1 [[CMP]]
+; PR 152851
+
+define i1 @val_is_aligend_const_pow2(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_pow2(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[NUM:%.*]], 4095
+; CHECK-NEXT:    [[_0:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[_0]]
 ;
-entry:
-  %conv = zext i8 %x to i32
-  %add = add i32 %conv, -4
-  %cmp = icmp ult i32 %add, 253
-  ret i1 %cmp
+  %num.biased = add i32 %num, 4095
+  %num.masked = and i32 %num.biased, -4096
+  %_0 = icmp eq i32 %num.masked, %num
+  ret i1 %_0
+}
+
+define i1 @val_is_aligend_const_pow2_add_commute(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_pow2_add_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[NUM:%.*]], 4095
+; CHECK-NEXT:    [[_0:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32  4095, %num
+  %num.masked = and i32 %num.biased, -4096
+  %_0 = icmp eq i32 %num.masked, %num
+  ret i1 %_0
+}
+
+define i1 @val_is_aligend_const_pow2_and_commute(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_pow2_and_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[NUM:%.*]], 4095
+; CHECK-NEXT:    [[_0:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32 %num, 4095
+  %num.masked = and i32 -4096, %num.biased
+  %_0 = icmp eq i32 %num.masked, %num
+  ret i1 %_0
+}
+
+define i1 @val_is_aligend_const_pow2_icm_commute(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_pow2_icm_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[NUM:%.*]], 4095
+; CHECK-NEXT:    [[_0:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32 %num, 4095
+  %num.masked = and i32 %num.biased, -4096
+  %_0 = icmp eq i32 %num, %num.masked
+  ret i1 %_0
+}
+
+; Should not work for non-power-of-two cases
+define i1 @val_is_aligend_const_non_pow2(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_non_pow2(
+; CHECK-NEXT:    [[NUM_BIASED:%.*]] = add i32 [[NUM:%.*]], 6
+; CHECK-NEXT:    [[NUM_MASKED:%.*]] = and i32 [[NUM_BIASED]], -7
+; CHECK-NEXT:    [[_0:%.*]] = icmp eq i32 [[NUM_MASKED]], [[NUM]]
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32 %num, 6
+  %num.masked = and i32 %num.biased, -7
+  %_0 = icmp eq i32 %num.masked, %num
+  ret i1 %_0
+}
+
+define i1 @val_is_aligend_const_pow2_multiuse(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_pow2_multiuse(
+; CHECK-NEXT:    [[NUM_BIASED:%.*]] = add i32 [[NUM:%.*]], 4095
+; CHECK-NEXT:    [[NUM_MASKED:%.*]] = and i32 [[NUM_BIASED]], -4096
+; CHECK-NEXT:    call void @use(i32 [[NUM_MASKED]])
+; CHECK-NEXT:    [[_0:%.*]] = icmp eq i32 [[NUM_MASKED]], [[NUM]]
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32 %num, 4095
+  %num.masked = and i32 %num.biased, -4096
+  call void @use(i32 %num.masked)
+  %_0 = icmp eq i32 %num.masked, %num
+  ret i1 %_0
+}
+
+; Applies since number of instructions do not change
+define i1 @val_is_aligend_const_pow2_multiuse1(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_pow2_multiuse1(
+; CHECK-NEXT:    [[NUM_BIASED:%.*]] = add i32 [[NUM:%.*]], 4095
+; CHECK-NEXT:    call void @use(i32 [[NUM_BIASED]])
+; CHECK-NEXT:    [[NUM_MASKED:%.*]] = and i32 [[NUM_BIASED]], -4096
+; CHECK-NEXT:    [[_0:%.*]] = icmp eq i32 [[NUM_MASKED]], [[NUM]]
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32 %num, 4095
+  call void @use(i32 %num.biased)
+  %num.masked = and i32 %num.biased, -4096
+  %_0 = icmp eq i32 %num.masked, %num
+  ret i1 %_0
+}
+
+define i1 @val_is_aligend_const_pow2_ne(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_pow2_ne(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[NUM:%.*]], 4095
+; CHECK-NEXT:    [[_0:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32 %num, 4095
+  %num.masked = and i32 %num.biased, -4096
+  %_0 = icmp ne i32 %num.masked, %num
+  ret i1 %_0
+}
+
+define i1 @val_is_aligend_const_mismatch(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_mismatch(
+; CHECK-NEXT:    [[NUM_BIASED:%.*]] = add i32 [[NUM:%.*]], 4095
+; CHECK-NEXT:    [[NUM_MASKED:%.*]] = and i32 [[NUM_BIASED]], -4095
+; CHECK-NEXT:    [[_0:%.*]] = icmp ne i32 [[NUM_MASKED]], [[NUM]]
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32 %num, 4095
+  %num.masked = and i32 %num.biased, -4095
+  %_0 = icmp ne i32 %num.masked, %num
+  ret i1 %_0
+}
+
+define i1 @val_is_aligend_const_mismatch1(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_const_mismatch1(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[NUM:%.*]], -4096
+; CHECK-NEXT:    [[NUM_MASKED:%.*]] = add i32 [[TMP1]], 4096
+; CHECK-NEXT:    [[_0:%.*]] = icmp ne i32 [[NUM_MASKED]], [[NUM]]
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32 %num, 4096
+  %num.masked = and i32 %num.biased, -4096
+  %_0 = icmp ne i32 %num.masked, %num
+  ret i1 %_0
+}
+
+define i1 @val_is_aligend_pred_mismatch(i32 %num) {
+; CHECK-LABEL: @val_is_aligend_pred_mismatch(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[NUM:%.*]], -4096
+; CHECK-NEXT:    [[NUM_MASKED:%.*]] = add i32 [[TMP1]], 4096
+; CHECK-NEXT:    [[_0:%.*]] = icmp sge i32 [[NUM_MASKED]], [[NUM]]
+; CHECK-NEXT:    ret i1 [[_0]]
+;
+  %num.biased = add i32 %num, 4096
+  %num.masked = and i32 %num.biased, -4096
+  %_0 = icmp sge i32 %num.masked, %num
+  ret i1 %_0
 }
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index a090f9c4d2614..0faa7da482ef2 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-p1:16:16:16-p2:32:32:32-p3:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
@@ -11,8 +11,9 @@ declare void @use_i32(i32)
 declare void @use_i64(i64)
 
 define i32 @test1(i32 %X) {
-; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr i32 [[X:%.*]], 31
+; CHECK-LABEL: define i32 @test1(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr i32 [[X]], 31
 ; CHECK-NEXT:    ret i32 [[X_LOBIT]]
 ;
   %a = icmp slt i32 %X, 0
@@ -21,8 +22,9 @@ define i32 @test1(i32 %X) {
 }
 
 define <2 x i32> @test1vec(<2 x i32> %X) {
-; CHECK-LABEL: @test1vec(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr <2 x i32> [[X:%.*]], splat (i32 31)
+; CHECK-LABEL: define <2 x i32> @test1vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr <2 x i32> [[X]], splat (i32 31)
 ; CHECK-NEXT:    ret <2 x i32> [[X_LOBIT]]
 ;
   %a = icmp slt <2 x i32> %X, zeroinitializer
@@ -31,8 +33,9 @@ define <2 x i32> @test1vec(<2 x i32> %X) {
 }
 
 define i32 @test2(i32 %X) {
-; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[A:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-LABEL: define i32 @test2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i32 [[X]], -1
 ; CHECK-NEXT:    [[B:%.*]] = zext i1 [[A]] to i32
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
@@ -42,8 +45,9 @@ define i32 @test2(i32 %X) {
 }
 
 define <2 x i32> @test2vec(<2 x i32> %X) {
-; CHECK-LABEL: @test2vec(
-; CHECK-NEXT:    [[A:%.*]] = icmp sgt <2 x i32> [[X:%.*]], splat (i32 -1)
+; CHECK-LABEL: define <2 x i32> @test2vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt <2 x i32> [[X]], splat (i32 -1)
 ; CHECK-NEXT:    [[B:%.*]] = zext <2 x i1> [[A]] to <2 x i32>
 ; CHECK-NEXT:    ret <2 x i32> [[B]]
 ;
@@ -53,8 +57,9 @@ define <2 x i32> @test2vec(<2 x i32> %X) {
 }
 
 define i32 @test3(i32 %X) {
-; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-LABEL: define i32 @test3(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X]], 31
 ; CHECK-NEXT:    ret i32 [[X_LOBIT]]
 ;
   %a = icmp slt i32 %X, 0
@@ -63,8 +68,9 @@ define i32 @test3(i32 %X) {
 }
 
 define i32 @test4(i32 %X) {
-; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[A:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-LABEL: define i32 @test4(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = icmp sgt i32 [[X]], -1
 ; CHECK-NEXT:    [[B:%.*]] = sext i1 [[A]] to i32
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
@@ -75,28 +81,31 @@ define i32 @test4(i32 %X) {
 
 ; PR4837
 define <2 x i1> @test5_eq(<2 x i64> %x) {
-; CHECK-LABEL: @test5_eq(
+; CHECK-LABEL: define <2 x i1> @test5_eq(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> undef
 ;
   %V = icmp eq <2 x i64> %x, undef
   ret <2 x i1> %V
 }
 define <2 x i1> @test5_ne(<2 x i64> %x) {
-; CHECK-LABEL: @test5_ne(
+; CHECK-LABEL: define <2 x i1> @test5_ne(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> undef
 ;
   %V = icmp ne <2 x i64> %x, undef
   ret <2 x i1> %V
 }
 define <2 x i1> @test5_ugt(<2 x i64> %x) {
-; CHECK-LABEL: @test5_ugt(
+; CHECK-LABEL: define <2 x i1> @test5_ugt(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %V = icmp ugt <2 x i64> %x, undef
   ret <2 x i1> %V
 }
 define <2 x i1> @test5_zero() {
-; CHECK-LABEL: @test5_zero(
+; CHECK-LABEL: define <2 x i1> @test5_zero() {
 ; CHECK-NEXT:    ret <2 x i1> undef
 ;
   %V = icmp eq <2 x i64> zeroinitializer, undef
@@ -104,9 +113,10 @@ define <2 x i1> @test5_zero() {
 }
 
 define i32 @test6(i32 %a, i32 %b) {
-; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[A:%.*]], 0
-; CHECK-NEXT:    [[F:%.*]] = select i1 [[ISNEG]], i32 [[B:%.*]], i32 0
+; CHECK-LABEL: define i32 @test6(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    [[F:%.*]] = select i1 [[ISNEG]], i32 [[B]], i32 0
 ; CHECK-NEXT:    ret i32 [[F]]
 ;
   %c = icmp sle i32 %a, -1
@@ -118,8 +128,9 @@ define i32 @test6(i32 %a, i32 %b) {
 
 
 define i1 @test7(i32 %x) {
-; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @test7(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %a = add i32 %x, -1
@@ -128,8 +139,9 @@ define i1 @test7(i32 %x) {
 }
 
 define <2 x i1> @test7_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test7_vec(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @test7_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ne <2 x i32> [[X]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[B]]
 ;
   %a = add <2 x i32> %x, <i32 -1, i32 -1>
@@ -138,7 +150,8 @@ define <2 x i1> @test7_vec(<2 x i32> %x) {
 }
 
 define i1 @test8(i32 %x) {
-; CHECK-LABEL: @test8(
+; CHECK-LABEL: define i1 @test8(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %a = add i32 %x, -1
@@ -147,7 +160,8 @@ define i1 @test8(i32 %x) {
 }
 
 define <2 x i1> @test8_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test8_vec(
+; CHECK-LABEL: define <2 x i1> @test8_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %a = add <2 x i32> %x, <i32 -1, i32 -1>
@@ -156,8 +170,9 @@ define <2 x i1> @test8_vec(<2 x i32> %x) {
 }
 
 define i1 @test9(i32 %x) {
-; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[B:%.*]] = icmp ugt i32 [[X:%.*]], 1
+; CHECK-LABEL: define i1 @test9(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i32 [[X]], 1
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %a = add i32 %x, -2
@@ -166,8 +181,9 @@ define i1 @test9(i32 %x) {
 }
 
 define <2 x i1> @test9_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test9_vec(
-; CHECK-NEXT:    [[B:%.*]] = icmp ugt <2 x i32> [[X:%.*]], splat (i32 1)
+; CHECK-LABEL: define <2 x i1> @test9_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt <2 x i32> [[X]], splat (i32 1)
 ; CHECK-NEXT:    ret <2 x i1> [[B]]
 ;
   %a = add <2 x i32> %x, <i32 -2, i32 -2>
@@ -176,8 +192,9 @@ define <2 x i1> @test9_vec(<2 x i32> %x) {
 }
 
 define i1 @test9b(i32 %x) {
-; CHECK-LABEL: @test9b(
-; CHECK-NEXT:    [[B:%.*]] = icmp ult i32 [[X:%.*]], 2
+; CHECK-LABEL: define i1 @test9b(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i32 [[X]], 2
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %a = add i32 %x, -2
@@ -186,8 +203,9 @@ define i1 @test9b(i32 %x) {
 }
 
 define <2 x i1> @test9b_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test9b_vec(
-; CHECK-NEXT:    [[B:%.*]] = icmp ult <2 x i32> [[X:%.*]], splat (i32 2)
+; CHECK-LABEL: define <2 x i1> @test9b_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ult <2 x i32> [[X]], splat (i32 2)
 ; CHECK-NEXT:    ret <2 x i1> [[B]]
 ;
   %a = add <2 x i32> %x, <i32 -2, i32 -2>
@@ -196,8 +214,9 @@ define <2 x i1> @test9b_vec(<2 x i32> %x) {
 }
 
 define i1 @test10(i32 %x) {
-; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], -2147483648
+; CHECK-LABEL: define i1 @test10(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X]], -2147483648
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %a = add i32 %x, -1
@@ -206,8 +225,9 @@ define i1 @test10(i32 %x) {
 }
 
 define <2 x i1> @test10_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test10_vec(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne <2 x i32> [[X:%.*]], splat (i32 -2147483648)
+; CHECK-LABEL: define <2 x i1> @test10_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ne <2 x i32> [[X]], splat (i32 -2147483648)
 ; CHECK-NEXT:    ret <2 x i1> [[B]]
 ;
   %a = add <2 x i32> %x, <i32 -1, i32 -1>
@@ -216,8 +236,9 @@ define <2 x i1> @test10_vec(<2 x i32> %x) {
 }
 
 define i1 @test10b(i32 %x) {
-; CHECK-LABEL: @test10b(
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[X:%.*]], -2147483648
+; CHECK-LABEL: define i1 @test10b(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[X]], -2147483648
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %a = add i32 %x, -1
@@ -226,8 +247,9 @@ define i1 @test10b(i32 %x) {
 }
 
 define <2 x i1> @test10b_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test10b_vec(
-; CHECK-NEXT:    [[B:%.*]] = icmp eq <2 x i32> [[X:%.*]], splat (i32 -2147483648)
+; CHECK-LABEL: define <2 x i1> @test10b_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp eq <2 x i32> [[X]], splat (i32 -2147483648)
 ; CHECK-NEXT:    ret <2 x i1> [[B]]
 ;
   %a = add <2 x i32> %x, <i32 -1, i32 -1>
@@ -236,7 +258,8 @@ define <2 x i1> @test10b_vec(<2 x i32> %x) {
 }
 
 define i1 @test11(i32 %x) {
-; CHECK-LABEL: @test11(
+; CHECK-LABEL: define i1 @test11(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 true
 ;
   %a = add nsw i32 %x, 8
@@ -245,7 +268,8 @@ define i1 @test11(i32 %x) {
 }
 
 define <2 x i1> @test11_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test11_vec(
+; CHECK-LABEL: define <2 x i1> @test11_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> splat (i1 true)
 ;
   %a = add nsw <2 x i32> %x, <i32 8, i32 8>
@@ -255,8 +279,9 @@ define <2 x i1> @test11_vec(<2 x i32> %x) {
 
 ; PR6195
 define i1 @test12(i1 %A) {
-; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[NOT_A:%.*]] = xor i1 [[A:%.*]], true
+; CHECK-LABEL: define i1 @test12(
+; CHECK-SAME: i1 [[A:%.*]]) {
+; CHECK-NEXT:    [[NOT_A:%.*]] = xor i1 [[A]], true
 ; CHECK-NEXT:    ret i1 [[NOT_A]]
 ;
   %S = select i1 %A, i64 -4294967295, i64 8589934591
@@ -266,7 +291,8 @@ define i1 @test12(i1 %A) {
 
 ; PR6481
 define i1 @test13(i8 %X) {
-; CHECK-LABEL: @test13(
+; CHECK-LABEL: define i1 @test13(
+; CHECK-SAME: i8 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %cmp = icmp slt i8 undef, %X
@@ -274,7 +300,8 @@ define i1 @test13(i8 %X) {
 }
 
 define i1 @test14(i8 %X) {
-; CHECK-LABEL: @test14(
+; CHECK-LABEL: define i1 @test14(
+; CHECK-SAME: i8 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %cmp = icmp slt i8 undef, -128
@@ -282,7 +309,7 @@ define i1 @test14(i8 %X) {
 }
 
 define i1 @test15() {
-; CHECK-LABEL: @test15(
+; CHECK-LABEL: define i1 @test15() {
 ; CHECK-NEXT:    ret i1 undef
 ;
   %cmp = icmp eq i8 undef, -128
@@ -290,7 +317,7 @@ define i1 @test15() {
 }
 
 define i1 @test16() {
-; CHECK-LABEL: @test16(
+; CHECK-LABEL: define i1 @test16() {
 ; CHECK-NEXT:    ret i1 undef
 ;
   %cmp = icmp ne i8 undef, -128
@@ -298,8 +325,9 @@ define i1 @test16() {
 }
 
 define i1 @test17(i32 %x) {
-; CHECK-LABEL: @test17(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @test17(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X]], 3
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %shl = shl i32 1, %x
@@ -309,8 +337,9 @@ define i1 @test17(i32 %x) {
 }
 
 define <2 x i1> @test17vec(<2 x i32> %x) {
-; CHECK-LABEL: @test17vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i32> [[X:%.*]], splat (i32 3)
+; CHECK-LABEL: define <2 x i1> @test17vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i32> [[X]], splat (i32 3)
 ; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -320,8 +349,9 @@ define <2 x i1> @test17vec(<2 x i32> %x) {
 }
 
 define i1 @test17a(i32 %x) {
-; CHECK-LABEL: @test17a(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], 2
+; CHECK-LABEL: define i1 @test17a(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X]], 2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %x
@@ -331,8 +361,9 @@ define i1 @test17a(i32 %x) {
 }
 
 define <2 x i1> @test17a_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test17a_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], splat (i32 2)
+; CHECK-LABEL: define <2 x i1> @test17a_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X]], splat (i32 2)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -342,8 +373,9 @@ define <2 x i1> @test17a_vec(<2 x i32> %x) {
 }
 
 define i1 @test18_eq(i32 %x) {
-; CHECK-LABEL: @test18_eq(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @test18_eq(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[X]], 3
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %sh = lshr i32 8, %x
@@ -353,8 +385,9 @@ define i1 @test18_eq(i32 %x) {
 }
 
 define <2 x i1> @test18_eq_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test18_eq_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i32> [[X:%.*]], splat (i32 3)
+; CHECK-LABEL: define <2 x i1> @test18_eq_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i32> [[X]], splat (i32 3)
 ; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
 ;
   %sh = lshr <2 x i32> <i32 8, i32 8>, %x
@@ -364,8 +397,9 @@ define <2 x i1> @test18_eq_vec(<2 x i32> %x) {
 }
 
 define i1 @test18_ne(i32 %x) {
-; CHECK-LABEL: @test18_ne(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @test18_ne(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X]], 3
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %sh = lshr i32 8, %x
@@ -375,8 +409,9 @@ define i1 @test18_ne(i32 %x) {
 }
 
 define <2 x i1> @test18_ne_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test18_ne_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[X:%.*]], splat (i32 3)
+; CHECK-LABEL: define <2 x i1> @test18_ne_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[X]], splat (i32 3)
 ; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
 ;
   %sh = lshr <2 x i32> <i32 8, i32 8>, %x
@@ -386,8 +421,9 @@ define <2 x i1> @test18_ne_vec(<2 x i32> %x) {
 }
 
 define i1 @test19(i32 %x) {
-; CHECK-LABEL: @test19(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @test19(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X]], 3
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %shl = shl i32 1, %x
@@ -397,8 +433,9 @@ define i1 @test19(i32 %x) {
 }
 
 define <2 x i1> @test19vec(<2 x i32> %x) {
-; CHECK-LABEL: @test19vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[X:%.*]], splat (i32 3)
+; CHECK-LABEL: define <2 x i1> @test19vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[X]], splat (i32 3)
 ; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -408,8 +445,9 @@ define <2 x i1> @test19vec(<2 x i32> %x) {
 }
 
 define <2 x i1> @cmp_and_signbit_vec(<2 x i3> %x) {
-; CHECK-LABEL: @cmp_and_signbit_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i3> [[X:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @cmp_and_signbit_vec(
+; CHECK-SAME: <2 x i3> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i3> [[X]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %and = and <2 x i3> %x, <i3 4, i3 4>
@@ -418,8 +456,9 @@ define <2 x i1> @cmp_and_signbit_vec(<2 x i3> %x) {
 }
 
 define i1 @test20(i32 %x) {
-; CHECK-LABEL: @test20(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @test20(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X]], 3
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %shl = shl i32 1, %x
@@ -429,8 +468,9 @@ define i1 @test20(i32 %x) {
 }
 
 define <2 x i1> @test20vec(<2 x i32> %x) {
-; CHECK-LABEL: @test20vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[X:%.*]], splat (i32 3)
+; CHECK-LABEL: define <2 x i1> @test20vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i32> [[X]], splat (i32 3)
 ; CHECK-NEXT:    ret <2 x i1> [[TMP1]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -440,8 +480,9 @@ define <2 x i1> @test20vec(<2 x i32> %x) {
 }
 
 define i1 @test20a(i32 %x) {
-; CHECK-LABEL: @test20a(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @test20a(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %x
@@ -451,8 +492,9 @@ define i1 @test20a(i32 %x) {
 }
 
 define <2 x i1> @test20a_vec(<2 x i32> %x) {
-; CHECK-LABEL: @test20a_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X:%.*]], splat (i32 3)
+; CHECK-LABEL: define <2 x i1> @test20a_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X]], splat (i32 3)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -462,8 +504,9 @@ define <2 x i1> @test20a_vec(<2 x i32> %x) {
 }
 
 define i1 @test21(i8 %x, i8 %y) {
-; CHECK-LABEL: @test21(
-; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @test21(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 [[X]], 3
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = or i8 %x, 1
@@ -472,8 +515,9 @@ define i1 @test21(i8 %x, i8 %y) {
 }
 
 define i1 @test22(i8 %x, i8 %y) {
-; CHECK-LABEL: @test22(
-; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 [[X:%.*]], 4
+; CHECK-LABEL: define i1 @test22(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 [[X]], 4
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = or i8 %x, 1
@@ -483,8 +527,9 @@ define i1 @test22(i8 %x, i8 %y) {
 
 ; PR2740
 define i1 @test23(i32 %x) {
-; CHECK-LABEL: @test23(
-; CHECK-NEXT:    [[I4:%.*]] = icmp sgt i32 [[X:%.*]], 1328634634
+; CHECK-LABEL: define i1 @test23(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[I4:%.*]] = icmp sgt i32 [[X]], 1328634634
 ; CHECK-NEXT:    ret i1 [[I4]]
 ;
   %i3 = sdiv i32 %x, -1328634635
@@ -493,8 +538,9 @@ define i1 @test23(i32 %x) {
 }
 
 define <2 x i1> @test23vec(<2 x i32> %x) {
-; CHECK-LABEL: @test23vec(
-; CHECK-NEXT:    [[I4:%.*]] = icmp sgt <2 x i32> [[X:%.*]], splat (i32 1328634634)
+; CHECK-LABEL: define <2 x i1> @test23vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[I4:%.*]] = icmp sgt <2 x i32> [[X]], splat (i32 1328634634)
 ; CHECK-NEXT:    ret <2 x i1> [[I4]]
 ;
   %i3 = sdiv <2 x i32> %x, <i32 -1328634635, i32 -1328634635>
@@ -505,8 +551,9 @@ define <2 x i1> @test23vec(<2 x i32> %x) {
 ; Note: offs can be negative, LLVM used to make an incorrect assumption that
 ; unsigned overflow does not happen during offset computation
 define i1 @test24_neg_offs(ptr %p, i64 %offs) {
-; CHECK-LABEL: @test24_neg_offs(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[OFFS:%.*]], -2
+; CHECK-LABEL: define i1 @test24_neg_offs(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[OFFS:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[OFFS]], -2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %p1 = getelementptr inbounds i32, ptr %p, i64 %offs
@@ -519,8 +566,9 @@ define i1 @test24_neg_offs(ptr %p, i64 %offs) {
 
 ; X - Z > Y - Z -> X > Y if there is no overflow.
 define i1 @test27(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test27(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test27(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nsw i32 %x, %z
@@ -530,10 +578,11 @@ define i1 @test27(i32 %x, i32 %y, i32 %z) {
 }
 
 define i1 @test27_extra_uses(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test27_extra_uses(
-; CHECK-NEXT:    [[LHS:%.*]] = sub nsw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-LABEL: define i1 @test27_extra_uses(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[LHS:%.*]] = sub nsw i32 [[X]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[LHS]])
-; CHECK-NEXT:    [[RHS:%.*]] = sub nsw i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[RHS:%.*]] = sub nsw i32 [[Y]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[RHS]])
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -548,8 +597,9 @@ define i1 @test27_extra_uses(i32 %x, i32 %y, i32 %z) {
 
 ; X - Z > Y - Z -> X > Y if there is no overflow.
 define i1 @test28(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test28(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test28(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nuw i32 %x, %z
@@ -559,10 +609,11 @@ define i1 @test28(i32 %x, i32 %y, i32 %z) {
 }
 
 define i1 @test28_extra_uses(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test28_extra_uses(
-; CHECK-NEXT:    [[LHS:%.*]] = sub nuw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-LABEL: define i1 @test28_extra_uses(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[LHS:%.*]] = sub nuw i32 [[X]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[LHS]])
-; CHECK-NEXT:    [[RHS:%.*]] = sub nuw i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[RHS:%.*]] = sub nuw i32 [[Y]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[RHS]])
 ; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -578,9 +629,10 @@ define i1 @test28_extra_uses(i32 %x, i32 %y, i32 %z) {
 ; PR36969 - https://bugs.llvm.org/show_bug.cgi?id=36969
 
 define i1 @ugt_sub(i32 %xsrc, i32 %y) {
-; CHECK-LABEL: @ugt_sub(
-; CHECK-NEXT:    [[X:%.*]] = udiv i32 [[XSRC:%.*]], 42
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[Y:%.*]], [[X]]
+; CHECK-LABEL: define i1 @ugt_sub(
+; CHECK-SAME: i32 [[XSRC:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = udiv i32 [[XSRC]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[Y]], [[X]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %x = udiv i32 %xsrc, 42 ; thwart complexity-based canonicalization
@@ -592,9 +644,10 @@ define i1 @ugt_sub(i32 %xsrc, i32 %y) {
 ; Swap operands and predicate. Try a vector type to verify that works too.
 
 define <2 x i1> @ult_sub(<2 x i8> %xsrc, <2 x i8> %y) {
-; CHECK-LABEL: @ult_sub(
-; CHECK-NEXT:    [[X:%.*]] = udiv <2 x i8> [[XSRC:%.*]], <i8 42, i8 -42>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> [[X]], [[Y:%.*]]
+; CHECK-LABEL: define <2 x i1> @ult_sub(
+; CHECK-SAME: <2 x i8> [[XSRC:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = udiv <2 x i8> [[XSRC]], <i8 42, i8 -42>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> [[X]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %x = udiv <2 x i8> %xsrc, <i8 42, i8 -42> ; thwart complexity-based canonicalization
@@ -605,8 +658,9 @@ define <2 x i1> @ult_sub(<2 x i8> %xsrc, <2 x i8> %y) {
 
 ; X - Y > X -> 0 > Y if there is no overflow.
 define i1 @test33(i32 %x, i32 %y) {
-; CHECK-LABEL: @test33(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-LABEL: define i1 @test33(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nsw i32 %x, %y
@@ -616,7 +670,8 @@ define i1 @test33(i32 %x, i32 %y) {
 
 ; X - Y > X -> 0 > Y if there is no overflow.
 define i1 @test34(i32 %x, i32 %y) {
-; CHECK-LABEL: @test34(
+; CHECK-LABEL: define i1 @test34(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %lhs = sub nuw i32 %x, %y
@@ -626,8 +681,9 @@ define i1 @test34(i32 %x, i32 %y) {
 
 ; X > X - Y -> Y > 0 if there is no overflow.
 define i1 @test35(i32 %x, i32 %y) {
-; CHECK-LABEL: @test35(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y:%.*]], 0
+; CHECK-LABEL: define i1 @test35(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %rhs = sub nsw i32 %x, %y
@@ -637,8 +693,9 @@ define i1 @test35(i32 %x, i32 %y) {
 
 ; X > X - Y -> Y > 0 if there is no overflow.
 define i1 @test36(i32 %x, i32 %y) {
-; CHECK-LABEL: @test36(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y:%.*]], 0
+; CHECK-LABEL: define i1 @test36(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %rhs = sub nuw i32 %x, %y
@@ -648,8 +705,9 @@ define i1 @test36(i32 %x, i32 %y) {
 
 ; X - Y > X - Z -> Z > Y if there is no overflow.
 define i1 @test37(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test37(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Z:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test37(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Z]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nsw i32 %x, %y
@@ -659,10 +717,11 @@ define i1 @test37(i32 %x, i32 %y, i32 %z) {
 }
 
 define i1 @test37_extra_uses(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test37_extra_uses(
-; CHECK-NEXT:    [[LHS:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test37_extra_uses(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[LHS:%.*]] = sub nsw i32 [[X]], [[Y]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[LHS]])
-; CHECK-NEXT:    [[RHS:%.*]] = sub nsw i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = sub nsw i32 [[X]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[RHS]])
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Z]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -678,8 +737,9 @@ define i1 @test37_extra_uses(i32 %x, i32 %y, i32 %z) {
 ; TODO: Min/max pattern should not prevent the fold.
 
 define i32 @neg_max_s32(i32 %x, i32 %y) {
-; CHECK-LABEL: @neg_max_s32(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.smin.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-LABEL: define i32 @neg_max_s32(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.smin.i32(i32 [[X]], i32 [[Y]])
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %nx = sub nsw i32 0, %x
@@ -691,8 +751,9 @@ define i32 @neg_max_s32(i32 %x, i32 %y) {
 }
 
 define <4 x i32> @neg_max_v4s32(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: @neg_max_v4s32(
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]])
+; CHECK-LABEL: define <4 x i32> @neg_max_v4s32(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.smin.v4i32(<4 x i32> [[X]], <4 x i32> [[Y]])
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
   %nx = sub nsw <4 x i32> zeroinitializer, %x
@@ -705,8 +766,9 @@ define <4 x i32> @neg_max_v4s32(<4 x i32> %x, <4 x i32> %y) {
 
 ; X - Y > X - Z -> Z > Y if there is no overflow.
 define i1 @test38(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test38(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[Z:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test38(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[Z]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nuw i32 %x, %y
@@ -716,10 +778,11 @@ define i1 @test38(i32 %x, i32 %y, i32 %z) {
 }
 
 define i1 @test38_extra_uses(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test38_extra_uses(
-; CHECK-NEXT:    [[LHS:%.*]] = sub nuw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test38_extra_uses(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[LHS:%.*]] = sub nuw i32 [[X]], [[Y]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[LHS]])
-; CHECK-NEXT:    [[RHS:%.*]] = sub nuw i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[RHS:%.*]] = sub nuw i32 [[X]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[RHS]])
 ; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[Z]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -733,8 +796,9 @@ define i1 @test38_extra_uses(i32 %x, i32 %y, i32 %z) {
 }
 
 define i1 @shr_exact(i132 %x) {
-; CHECK-LABEL: @shr_exact(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i132 [[X:%.*]], 32
+; CHECK-LABEL: define i1 @shr_exact(
+; CHECK-SAME: i132 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i132 [[X]], 32
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sh = ashr exact i132 %x, 4
@@ -743,8 +807,9 @@ define i1 @shr_exact(i132 %x) {
 }
 
 define <2 x i1> @shr_exact_vec(<2 x i132> %x) {
-; CHECK-LABEL: @shr_exact_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i132> [[X:%.*]], splat (i132 32)
+; CHECK-LABEL: define <2 x i1> @shr_exact_vec(
+; CHECK-SAME: <2 x i132> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i132> [[X]], splat (i132 32)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %sh = lshr exact <2 x i132> %x, <i132 4, i132 4>
@@ -754,7 +819,8 @@ define <2 x i1> @shr_exact_vec(<2 x i132> %x) {
 
 ; PR9343 #3
 define i1 @test41(i32 %X, i32 %Y) {
-; CHECK-LABEL: @test41(
+; CHECK-LABEL: define i1 @test41(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
 ; CHECK-NEXT:    ret i1 true
 ;
   %A = urem i32 %X, %Y
@@ -763,8 +829,9 @@ define i1 @test41(i32 %X, i32 %Y) {
 }
 
 define i1 @test42(i32 %X, i32 %Y) {
-; CHECK-LABEL: @test42(
-; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 [[Y:%.*]], -1
+; CHECK-LABEL: define i1 @test42(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 [[Y]], -1
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = srem i32 %X, %Y
@@ -773,8 +840,9 @@ define i1 @test42(i32 %X, i32 %Y) {
 }
 
 define i1 @test43(i32 %X, i32 %Y) {
-; CHECK-LABEL: @test43(
-; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-LABEL: define i1 @test43(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[Y]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = srem i32 %X, %Y
@@ -783,8 +851,9 @@ define i1 @test43(i32 %X, i32 %Y) {
 }
 
 define i1 @test44(i32 %X, i32 %Y) {
-; CHECK-LABEL: @test44(
-; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 [[Y:%.*]], -1
+; CHECK-LABEL: define i1 @test44(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 [[Y]], -1
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = srem i32 %X, %Y
@@ -793,8 +862,9 @@ define i1 @test44(i32 %X, i32 %Y) {
 }
 
 define i1 @test45(i32 %X, i32 %Y) {
-; CHECK-LABEL: @test45(
-; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-LABEL: define i1 @test45(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[Y]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = srem i32 %X, %Y
@@ -804,8 +874,9 @@ define i1 @test45(i32 %X, i32 %Y) {
 
 ; PR9343 #4
 define i1 @test46(i32 %X, i32 %Y, i32 %Z) {
-; CHECK-LABEL: @test46(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test46(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = ashr exact i32 %X, %Z
@@ -815,10 +886,11 @@ define i1 @test46(i32 %X, i32 %Y, i32 %Z) {
 }
 
 define i1 @test46_multiuse1(i32 %X, i32 %Y, i32 %Z) {
-; CHECK-LABEL: @test46_multiuse1(
-; CHECK-NEXT:    [[A:%.*]] = ashr exact i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-LABEL: define i1 @test46_multiuse1(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = ashr exact i32 [[X]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[A]])
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = ashr exact i32 %X, %Z
@@ -829,10 +901,11 @@ define i1 @test46_multiuse1(i32 %X, i32 %Y, i32 %Z) {
 }
 
 define i1 @test46_multiuse2(i32 %X, i32 %Y, i32 %Z) {
-; CHECK-LABEL: @test46_multiuse2(
-; CHECK-NEXT:    [[B:%.*]] = ashr exact i32 [[Y:%.*]], [[Z:%.*]]
+; CHECK-LABEL: define i1 @test46_multiuse2(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = ashr exact i32 [[Y]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[B]])
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = ashr exact i32 %X, %Z
@@ -843,10 +916,11 @@ define i1 @test46_multiuse2(i32 %X, i32 %Y, i32 %Z) {
 }
 
 define i1 @test46_multiuse3(i32 %X, i32 %Y, i32 %Z) {
-; CHECK-LABEL: @test46_multiuse3(
-; CHECK-NEXT:    [[A:%.*]] = ashr exact i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-LABEL: define i1 @test46_multiuse3(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = ashr exact i32 [[X]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[A]])
-; CHECK-NEXT:    [[B:%.*]] = ashr exact i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    [[B:%.*]] = ashr exact i32 [[Y]], [[Z]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[B]])
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[A]], [[B]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -861,8 +935,9 @@ define i1 @test46_multiuse3(i32 %X, i32 %Y, i32 %Z) {
 
 ; PR9343 #5
 define i1 @test47(i32 %X, i32 %Y, i32 %Z) {
-; CHECK-LABEL: @test47(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test47(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = ashr exact i32 %X, %Z
@@ -873,8 +948,9 @@ define i1 @test47(i32 %X, i32 %Y, i32 %Z) {
 
 ; PR9343 #8
 define i1 @test48(i32 %X, i32 %Y, i32 %Z) {
-; CHECK-LABEL: @test48(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test48(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = sdiv exact i32 %X, %Z
@@ -886,9 +962,10 @@ define i1 @test48(i32 %X, i32 %Y, i32 %Z) {
 ; The above transform only works for equality predicates.
 
 define i1 @PR32949(i32 %X, i32 %Y, i32 %Z) {
-; CHECK-LABEL: @PR32949(
-; CHECK-NEXT:    [[A:%.*]] = sdiv exact i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    [[B:%.*]] = sdiv exact i32 [[Y:%.*]], [[Z]]
+; CHECK-LABEL: define i1 @PR32949(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = sdiv exact i32 [[X]], [[Z]]
+; CHECK-NEXT:    [[B:%.*]] = sdiv exact i32 [[Y]], [[Z]]
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], [[B]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -899,8 +976,9 @@ define i1 @PR32949(i32 %X, i32 %Y, i32 %Z) {
 }
 
 define i1 @test_sdiv_pos_slt(i32 %x, i32 %y) {
-; CHECK-LABEL: @test_sdiv_pos_slt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test_sdiv_pos_slt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %divx = sdiv exact i32 %x, 40
@@ -910,8 +988,9 @@ define i1 @test_sdiv_pos_slt(i32 %x, i32 %y) {
 }
 
 define i1 @test_sdiv_pos_sle(i32 %x, i32 %y) {
-; CHECK-LABEL: @test_sdiv_pos_sle(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test_sdiv_pos_sle(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %divx = sdiv exact i32 %x, 40
@@ -921,8 +1000,9 @@ define i1 @test_sdiv_pos_sle(i32 %x, i32 %y) {
 }
 
 define i1 @test_sdiv_pos_sgt(i32 %x, i32 %y) {
-; CHECK-LABEL: @test_sdiv_pos_sgt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test_sdiv_pos_sgt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %divx = sdiv exact i32 %x, 40
@@ -932,8 +1012,9 @@ define i1 @test_sdiv_pos_sgt(i32 %x, i32 %y) {
 }
 
 define i1 @test_sdiv_pos_sge(i32 %x, i32 %y) {
-; CHECK-LABEL: @test_sdiv_pos_sge(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test_sdiv_pos_sge(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %divx = sdiv exact i32 %x, 40
@@ -943,8 +1024,9 @@ define i1 @test_sdiv_pos_sge(i32 %x, i32 %y) {
 }
 
 define i1 @test_sdiv_pos_ult(i32 %x, i32 %y) {
-; CHECK-LABEL: @test_sdiv_pos_ult(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test_sdiv_pos_ult(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %divx = sdiv exact i32 %x, 40
@@ -954,8 +1036,9 @@ define i1 @test_sdiv_pos_ult(i32 %x, i32 %y) {
 }
 
 define i1 @test_sdiv_pos_ule(i32 %x, i32 %y) {
-; CHECK-LABEL: @test_sdiv_pos_ule(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test_sdiv_pos_ule(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %divx = sdiv exact i32 %x, 40
@@ -965,8 +1048,9 @@ define i1 @test_sdiv_pos_ule(i32 %x, i32 %y) {
 }
 
 define i1 @test_sdiv_pos_ugt(i32 %x, i32 %y) {
-; CHECK-LABEL: @test_sdiv_pos_ugt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test_sdiv_pos_ugt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %divx = sdiv exact i32 %x, 40
@@ -976,8 +1060,9 @@ define i1 @test_sdiv_pos_ugt(i32 %x, i32 %y) {
 }
 
 define i1 @test_sdiv_pos_uge(i32 %x, i32 %y) {
-; CHECK-LABEL: @test_sdiv_pos_uge(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test_sdiv_pos_uge(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %divx = sdiv exact i32 %x, 40
@@ -987,9 +1072,10 @@ define i1 @test_sdiv_pos_uge(i32 %x, i32 %y) {
 }
 
 define i1 @test_sdiv_neg_slt(i32 %x, i32 %y) {
-; CHECK-LABEL: @test_sdiv_neg_slt(
-; CHECK-NEXT:    [[DIVX:%.*]] = sdiv exact i32 [[X:%.*]], -40
-; CHECK-NEXT:    [[DIVY:%.*]] = sdiv exact i32 [[Y:%.*]], -40
+; CHECK-LABEL: define i1 @test_sdiv_neg_slt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[DIVX:%.*]] = sdiv exact i32 [[X]], -40
+; CHECK-NEXT:    [[DIVY:%.*]] = sdiv exact i32 [[Y]], -40
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[DIVX]], [[DIVY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1001,8 +1087,9 @@ define i1 @test_sdiv_neg_slt(i32 %x, i32 %y) {
 
 ; PR8469
 define <2 x i1> @test49(<2 x i32> %i3) {
-; CHECK-LABEL: @test49(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define <2 x i1> @test49(
+; CHECK-SAME: <2 x i32> [[I3:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    ret <2 x i1> splat (i1 true)
 ;
 entry:
@@ -1013,7 +1100,8 @@ entry:
 
 ; PR9343 #7
 define i1 @test50(i16 %X, i32 %Y) {
-; CHECK-LABEL: @test50(
+; CHECK-LABEL: define i1 @test50(
+; CHECK-SAME: i16 [[X:%.*]], i32 [[Y:%.*]]) {
 ; CHECK-NEXT:    ret i1 true
 ;
   %A = zext i16 %X to i32
@@ -1023,9 +1111,10 @@ define i1 @test50(i16 %X, i32 %Y) {
 }
 
 define i1 @test51(i32 %X, i32 %Y) {
-; CHECK-LABEL: @test51(
-; CHECK-NEXT:    [[A:%.*]] = and i32 [[X:%.*]], -2147483648
-; CHECK-NEXT:    [[B:%.*]] = srem i32 [[A]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @test51(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[X]], -2147483648
+; CHECK-NEXT:    [[B:%.*]] = srem i32 [[A]], [[Y]]
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[B]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -1036,8 +1125,9 @@ define i1 @test51(i32 %X, i32 %Y) {
 }
 
 define i1 @test52(i32 %x1) {
-; CHECK-LABEL: @test52(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X1:%.*]], 16711935
+; CHECK-LABEL: define i1 @test52(
+; CHECK-SAME: i32 [[X1:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X1]], 16711935
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[TMP1]], 4980863
 ; CHECK-NEXT:    ret i1 [[A]]
 ;
@@ -1052,8 +1142,9 @@ define i1 @test52(i32 %x1) {
 }
 
 define i1 @test52_logical(i32 %x1) {
-; CHECK-LABEL: @test52_logical(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X1:%.*]], 16711935
+; CHECK-LABEL: define i1 @test52_logical(
+; CHECK-SAME: i32 [[X1:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X1]], 16711935
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq i32 [[TMP1]], 4980863
 ; CHECK-NEXT:    ret i1 [[A]]
 ;
@@ -1068,8 +1159,9 @@ define i1 @test52_logical(i32 %x1) {
 }
 
 define i1 @test52b(i128 %x1) {
-; CHECK-LABEL: @test52b(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[X1:%.*]], 16711935
+; CHECK-LABEL: define i1 @test52b(
+; CHECK-SAME: i128 [[X1:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[X1]], 16711935
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq i128 [[TMP1]], 4980863
 ; CHECK-NEXT:    ret i1 [[A]]
 ;
@@ -1084,8 +1176,9 @@ define i1 @test52b(i128 %x1) {
 }
 
 define i1 @test52b_logical(i128 %x1) {
-; CHECK-LABEL: @test52b_logical(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[X1:%.*]], 16711935
+; CHECK-LABEL: define i1 @test52b_logical(
+; CHECK-SAME: i128 [[X1:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i128 [[X1]], 16711935
 ; CHECK-NEXT:    [[A:%.*]] = icmp eq i128 [[TMP1]], 4980863
 ; CHECK-NEXT:    ret i1 [[A]]
 ;
@@ -1101,9 +1194,10 @@ define i1 @test52b_logical(i128 %x1) {
 
 ; PR9838
 define i1 @test53(i32 %a, i32 %b) {
-; CHECK-LABEL: @test53(
-; CHECK-NEXT:    [[X:%.*]] = sdiv exact i32 [[A:%.*]], 30
-; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[B:%.*]], 30
+; CHECK-LABEL: define i1 @test53(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = sdiv exact i32 [[A]], 30
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[B]], 30
 ; CHECK-NEXT:    [[Z:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
@@ -1114,8 +1208,9 @@ define i1 @test53(i32 %a, i32 %b) {
 }
 
 define i1 @test54(i8 %a) {
-; CHECK-LABEL: @test54(
-; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[A:%.*]], -64
+; CHECK-LABEL: define i1 @test54(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = icmp slt i8 [[A]], -64
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %ext = zext i8 %a to i32
@@ -1125,8 +1220,9 @@ define i1 @test54(i8 %a) {
 }
 
 define i1 @test55(i32 %a) {
-; CHECK-LABEL: @test55(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], -123
+; CHECK-LABEL: define i1 @test55(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], -123
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = sub i32 0, %a
@@ -1135,8 +1231,9 @@ define i1 @test55(i32 %a) {
 }
 
 define <2 x i1> @test55vec(<2 x i32> %a) {
-; CHECK-LABEL: @test55vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[A:%.*]], splat (i32 -123)
+; CHECK-LABEL: define <2 x i1> @test55vec(
+; CHECK-SAME: <2 x i32> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[A]], splat (i32 -123)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %sub = sub <2 x i32> zeroinitializer, %a
@@ -1145,8 +1242,9 @@ define <2 x i1> @test55vec(<2 x i32> %a) {
 }
 
 define i1 @test56(i32 %a) {
-; CHECK-LABEL: @test56(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], -113
+; CHECK-LABEL: define i1 @test56(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], -113
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = sub i32 10, %a
@@ -1155,8 +1253,9 @@ define i1 @test56(i32 %a) {
 }
 
 define <2 x i1> @test56vec(<2 x i32> %a) {
-; CHECK-LABEL: @test56vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[A:%.*]], splat (i32 -113)
+; CHECK-LABEL: define <2 x i1> @test56vec(
+; CHECK-SAME: <2 x i32> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[A]], splat (i32 -113)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %sub = sub <2 x i32> <i32 10, i32 10>, %a
@@ -1166,8 +1265,9 @@ define <2 x i1> @test56vec(<2 x i32> %a) {
 
 ; PR10267 Don't make icmps more expensive when no other inst is subsumed.
 define i1 @test57(i32 %a) {
-; CHECK-LABEL: @test57(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], -2
+; CHECK-LABEL: define i1 @test57(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A]], -2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
 ; CHECK-NEXT:    call void @use_i32(i32 [[AND]])
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -1180,8 +1280,9 @@ define i1 @test57(i32 %a) {
 
 ; rdar://problem/10482509
 define zeroext i1 @cmpabs1(i64 %val) {
-; CHECK-LABEL: @cmpabs1(
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[VAL:%.*]], 0
+; CHECK-LABEL: define zeroext i1 @cmpabs1(
+; CHECK-SAME: i64 [[VAL:%.*]]) {
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[VAL]], 0
 ; CHECK-NEXT:    ret i1 [[TOBOOL]]
 ;
   %sub = sub nsw i64 0, %val
@@ -1192,8 +1293,9 @@ define zeroext i1 @cmpabs1(i64 %val) {
 }
 
 define zeroext i1 @cmpabs2(i64 %val) {
-; CHECK-LABEL: @cmpabs2(
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[VAL:%.*]], 0
+; CHECK-LABEL: define zeroext i1 @cmpabs2(
+; CHECK-SAME: i64 [[VAL:%.*]]) {
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[VAL]], 0
 ; CHECK-NEXT:    ret i1 [[TOBOOL]]
 ;
   %sub = sub nsw i64 0, %val
@@ -1204,8 +1306,9 @@ define zeroext i1 @cmpabs2(i64 %val) {
 }
 
 define i1 @abs_intrin_eq_zero(i8 %x) {
-; CHECK-LABEL: @abs_intrin_eq_zero(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @abs_intrin_eq_zero(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %abs = call i8 @llvm.abs.i8(i8 %x, i1 false)
@@ -1214,8 +1317,9 @@ define i1 @abs_intrin_eq_zero(i8 %x) {
 }
 
 define i1 @abs_intrin_ne_zero(i8 %x) {
-; CHECK-LABEL: @abs_intrin_ne_zero(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @abs_intrin_ne_zero(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %abs = call i8 @llvm.abs.i8(i8 %x, i1 false)
@@ -1224,7 +1328,7 @@ define i1 @abs_intrin_ne_zero(i8 %x) {
 }
 
 define void @test58() {
-; CHECK-LABEL: @test58(
+; CHECK-LABEL: define void @test58() {
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 36029346783166592)
 ; CHECK-NEXT:    ret void
 ;
@@ -1236,8 +1340,9 @@ declare i32 @test58_d(i64)
 
 ; Negative test: GEP inbounds may cross sign boundary.
 define i1 @test62(ptr %a) {
-; CHECK-LABEL: @test62(
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[A:%.*]], i64 1
+; CHECK-LABEL: define i1 @test62(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 1
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 10
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt ptr [[ARRAYIDX1]], [[ARRAYIDX2]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -1249,8 +1354,9 @@ define i1 @test62(ptr %a) {
 }
 
 define i1 @test62_as1(ptr addrspace(1) %a) {
-; CHECK-LABEL: @test62_as1(
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A:%.*]], i16 1
+; CHECK-LABEL: define i1 @test62_as1(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]]) {
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i16 1
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i16 10
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt ptr addrspace(1) [[ARRAYIDX1]], [[ARRAYIDX2]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -1262,9 +1368,10 @@ define i1 @test62_as1(ptr addrspace(1) %a) {
 }
 
 define i1 @low_mask_eq_zext(i8 %a, i32 %b) {
-; CHECK-LABEL: @low_mask_eq_zext(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A:%.*]], [[TMP1]]
+; CHECK-LABEL: define i1 @low_mask_eq_zext(
+; CHECK-SAME: i8 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B]] to i8
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %z = zext i8 %a to i32
@@ -1274,9 +1381,10 @@ define i1 @low_mask_eq_zext(i8 %a, i32 %b) {
 }
 
 define i1 @low_mask_eq_zext_commute(i8 %a, i32 %b) {
-; CHECK-LABEL: @low_mask_eq_zext_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A:%.*]], [[TMP1]]
+; CHECK-LABEL: define i1 @low_mask_eq_zext_commute(
+; CHECK-SAME: i8 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B]] to i8
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %t = and i32 %b, 255
@@ -1288,9 +1396,10 @@ define i1 @low_mask_eq_zext_commute(i8 %a, i32 %b) {
 ; negative test
 
 define i1 @wrong_low_mask_eq_zext(i8 %a, i32 %b) {
-; CHECK-LABEL: @wrong_low_mask_eq_zext(
-; CHECK-NEXT:    [[T:%.*]] = and i32 [[B:%.*]], 127
-; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-LABEL: define i1 @wrong_low_mask_eq_zext(
+; CHECK-SAME: i8 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = and i32 [[B]], 127
+; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[A]] to i32
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[T]], [[Z]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -1303,9 +1412,10 @@ define i1 @wrong_low_mask_eq_zext(i8 %a, i32 %b) {
 ; negative test
 
 define i1 @wrong_low_mask_eq_zext2(i8 %a, i32 %b) {
-; CHECK-LABEL: @wrong_low_mask_eq_zext2(
-; CHECK-NEXT:    [[T:%.*]] = and i32 [[B:%.*]], 254
-; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-LABEL: define i1 @wrong_low_mask_eq_zext2(
+; CHECK-SAME: i8 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = and i32 [[B]], 254
+; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[A]] to i32
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[T]], [[Z]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -1316,11 +1426,12 @@ define i1 @wrong_low_mask_eq_zext2(i8 %a, i32 %b) {
 }
 
 define i1 @low_mask_eq_zext_use1(i8 %a, i32 %b) {
-; CHECK-LABEL: @low_mask_eq_zext_use1(
-; CHECK-NEXT:    [[T:%.*]] = and i32 [[B:%.*]], 255
+; CHECK-LABEL: define i1 @low_mask_eq_zext_use1(
+; CHECK-SAME: i8 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = and i32 [[B]], 255
 ; CHECK-NEXT:    call void @use_i32(i32 [[T]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B]] to i8
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %t = and i32 %b, 255
@@ -1331,10 +1442,11 @@ define i1 @low_mask_eq_zext_use1(i8 %a, i32 %b) {
 }
 
 define i1 @low_mask_eq_zext_use2(i8 %a, i32 %b) {
-; CHECK-LABEL: @low_mask_eq_zext_use2(
-; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-LABEL: define i1 @low_mask_eq_zext_use2(
+; CHECK-SAME: i8 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[A]] to i32
 ; CHECK-NEXT:    call void @use_i32(i32 [[Z]])
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B]] to i8
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -1346,10 +1458,11 @@ define i1 @low_mask_eq_zext_use2(i8 %a, i32 %b) {
 }
 
 define i1 @low_mask_eq_zext_use3(i8 %a, i32 %b) {
-; CHECK-LABEL: @low_mask_eq_zext_use3(
-; CHECK-NEXT:    [[T:%.*]] = and i32 [[B:%.*]], 255
+; CHECK-LABEL: define i1 @low_mask_eq_zext_use3(
+; CHECK-SAME: i8 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = and i32 [[B]], 255
 ; CHECK-NEXT:    call void @use_i32(i32 [[T]])
-; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = zext i8 [[A]] to i32
 ; CHECK-NEXT:    call void @use_i32(i32 [[Z]])
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[T]], [[Z]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -1363,9 +1476,10 @@ define i1 @low_mask_eq_zext_use3(i8 %a, i32 %b) {
 }
 
 define <2 x i1> @low_mask_eq_zext_vec_splat(<2 x i8> %a, <2 x i32> %b) {
-; CHECK-LABEL: @low_mask_eq_zext_vec_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[B:%.*]] to <2 x i8>
-; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i8> [[A:%.*]], [[TMP1]]
+; CHECK-LABEL: define <2 x i1> @low_mask_eq_zext_vec_splat(
+; CHECK-SAME: <2 x i8> [[A:%.*]], <2 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[B]] to <2 x i8>
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i8> [[A]], [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %t = and <2 x i32> %b, <i32 255, i32 255>
@@ -1375,7 +1489,8 @@ define <2 x i1> @low_mask_eq_zext_vec_splat(<2 x i8> %a, <2 x i32> %b) {
 }
 
 define i1 @test65(i64 %A, i64 %B) {
-; CHECK-LABEL: @test65(
+; CHECK-LABEL: define i1 @test65(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
 ; CHECK-NEXT:    ret i1 true
 ;
   %s1 = add i64 %A, %B
@@ -1385,7 +1500,8 @@ define i1 @test65(i64 %A, i64 %B) {
 }
 
 define i1 @test66(i64 %A, i64 %B) {
-; CHECK-LABEL: @test66(
+; CHECK-LABEL: define i1 @test66(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
 ; CHECK-NEXT:    ret i1 true
 ;
   %s1 = add i64 %A, %B
@@ -1395,8 +1511,9 @@ define i1 @test66(i64 %A, i64 %B) {
 }
 
 define i1 @test67(i32 %x) {
-; CHECK-LABEL: @test67(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 96
+; CHECK-LABEL: define i1 @test67(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], 96
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1406,8 +1523,9 @@ define i1 @test67(i32 %x) {
 }
 
 define i1 @test67inverse(i32 %x) {
-; CHECK-LABEL: @test67inverse(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 96
+; CHECK-LABEL: define i1 @test67inverse(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], 96
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1420,8 +1538,9 @@ define i1 @test67inverse(i32 %x) {
 ; This test only checks the last of those (icmp ugt -> icmp ne).
 
 define <2 x i1> @test67vec(<2 x i32> %x) {
-; CHECK-LABEL: @test67vec(
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 96)
+; CHECK-LABEL: define <2 x i1> @test67vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X]], splat (i32 96)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[AND]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1431,8 +1550,9 @@ define <2 x i1> @test67vec(<2 x i32> %x) {
 }
 
 define <2 x i1> @test67vec2(<2 x i32> %x) {
-; CHECK-LABEL: @test67vec2(
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 96)
+; CHECK-LABEL: define <2 x i1> @test67vec2(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X]], splat (i32 96)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[AND]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1442,8 +1562,9 @@ define <2 x i1> @test67vec2(<2 x i32> %x) {
 }
 
 define <2 x i1> @test67vecinverse(<2 x i32> %x) {
-; CHECK-LABEL: @test67vecinverse(
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 96)
+; CHECK-LABEL: define <2 x i1> @test67vecinverse(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X]], splat (i32 96)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[AND]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1453,8 +1574,9 @@ define <2 x i1> @test67vecinverse(<2 x i32> %x) {
 }
 
 define i1 @test68(i32 %x) {
-; CHECK-LABEL: @test68(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 127
+; CHECK-LABEL: define i1 @test68(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], 127
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp samesign ugt i32 [[AND]], 30
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1465,8 +1587,9 @@ define i1 @test68(i32 %x) {
 
 ; PR15940
 define i1 @test70(i32 %X) {
-; CHECK-LABEL: @test70(
-; CHECK-NEXT:    [[A:%.*]] = srem i32 5, [[X:%.*]]
+; CHECK-LABEL: define i1 @test70(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = srem i32 5, [[X]]
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[A]], 2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -1477,8 +1600,9 @@ define i1 @test70(i32 %X) {
 }
 
 define <2 x i1> @test70vec(<2 x i32> %X) {
-; CHECK-LABEL: @test70vec(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[X:%.*]], splat (i32 2)
+; CHECK-LABEL: define <2 x i1> @test70vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[X]], splat (i32 2)
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %B = add <2 x i32> %X, <i32 2, i32 2>
@@ -1487,8 +1611,9 @@ define <2 x i1> @test70vec(<2 x i32> %X) {
 }
 
 define i1 @icmp_sext16trunc(i32 %x) {
-; CHECK-LABEL: @icmp_sext16trunc(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-LABEL: define i1 @icmp_sext16trunc(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X]] to i16
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[TMP1]], 36
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1499,8 +1624,9 @@ define i1 @icmp_sext16trunc(i32 %x) {
 }
 
 define i1 @icmp_sext8trunc(i32 %x) {
-; CHECK-LABEL: @icmp_sext8trunc(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-LABEL: define i1 @icmp_sext8trunc(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X]] to i8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 36
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1512,8 +1638,9 @@ define i1 @icmp_sext8trunc(i32 %x) {
 
 ; Vectors should fold the same way.
 define <2 x i1> @icmp_sext8trunc_vec(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_sext8trunc_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[X:%.*]] to <2 x i8>
+; CHECK-LABEL: define <2 x i1> @icmp_sext8trunc_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[X]] to <2 x i8>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[TMP1]], splat (i8 36)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1524,8 +1651,9 @@ define <2 x i1> @icmp_sext8trunc_vec(<2 x i32> %x) {
 }
 
 define i1 @icmp_shl16(i32 %x) {
-; CHECK-LABEL: @icmp_shl16(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-LABEL: define i1 @icmp_shl16(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X]] to i16
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[TMP1]], 36
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1537,8 +1665,9 @@ define i1 @icmp_shl16(i32 %x) {
 ; D25952: Don't create illegal types like i15 in InstCombine
 
 define i1 @icmp_shl17(i32 %x) {
-; CHECK-LABEL: @icmp_shl17(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], 17
+; CHECK-LABEL: define i1 @icmp_shl17(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X]], 17
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[SHL]], 2359296
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1548,8 +1677,9 @@ define i1 @icmp_shl17(i32 %x) {
 }
 
 define <2 x i1> @icmp_shl16_vec(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_shl16_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[X:%.*]] to <2 x i16>
+; CHECK-LABEL: define <2 x i1> @icmp_shl16_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[X]] to <2 x i16>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i16> [[TMP1]], splat (i16 36)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1559,8 +1689,9 @@ define <2 x i1> @icmp_shl16_vec(<2 x i32> %x) {
 }
 
 define i1 @icmp_shl24(i32 %x) {
-; CHECK-LABEL: @icmp_shl24(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-LABEL: define i1 @icmp_shl24(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X]] to i8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 36
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1570,8 +1701,9 @@ define i1 @icmp_shl24(i32 %x) {
 }
 
 define i1 @icmp_shl_eq(i32 %x) {
-; CHECK-LABEL: @icmp_shl_eq(
-; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 [[X:%.*]], 134217727
+; CHECK-LABEL: define i1 @icmp_shl_eq(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 [[X]], 134217727
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MUL_MASK]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1581,8 +1713,9 @@ define i1 @icmp_shl_eq(i32 %x) {
 }
 
 define <2 x i1> @icmp_shl_eq_vec(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_shl_eq_vec(
-; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 134217727)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_eq_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> [[X]], splat (i32 134217727)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[MUL_MASK]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1592,8 +1725,9 @@ define <2 x i1> @icmp_shl_eq_vec(<2 x i32> %x) {
 }
 
 define i1 @icmp_shl_nsw_ne(i32 %x) {
-; CHECK-LABEL: @icmp_shl_nsw_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @icmp_shl_nsw_ne(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = shl nsw i32 %x, 7
@@ -1602,8 +1736,9 @@ define i1 @icmp_shl_nsw_ne(i32 %x) {
 }
 
 define <2 x i1> @icmp_shl_nsw_ne_vec(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_shl_nsw_ne_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @icmp_shl_nsw_ne_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %mul = shl nsw <2 x i32> %x, <i32 7, i32 7>
@@ -1612,8 +1747,9 @@ define <2 x i1> @icmp_shl_nsw_ne_vec(<2 x i32> %x) {
 }
 
 define i1 @icmp_shl_ne(i32 %x) {
-; CHECK-LABEL: @icmp_shl_ne(
-; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 [[X:%.*]], 33554431
+; CHECK-LABEL: define i1 @icmp_shl_ne(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 [[X]], 33554431
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MUL_MASK]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1623,8 +1759,9 @@ define i1 @icmp_shl_ne(i32 %x) {
 }
 
 define <2 x i1> @icmp_shl_ne_vec(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_shl_ne_vec(
-; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 33554431)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_ne_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> [[X]], splat (i32 33554431)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[MUL_MASK]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1634,8 +1771,9 @@ define <2 x i1> @icmp_shl_ne_vec(<2 x i32> %x) {
 }
 
 define <2 x i1> @icmp_shl_nuw_ne_vec(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_shl_nuw_ne_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], splat (i32 2)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_nuw_ne_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X]], splat (i32 2)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl nuw <2 x i32> %x, <i32 7, i32 7>
@@ -1646,8 +1784,9 @@ define <2 x i1> @icmp_shl_nuw_ne_vec(<2 x i32> %x) {
 ; If the (mul x, C) preserved the sign and this is sign test,
 ; compare the LHS operand instead
 define i1 @icmp_mul_nsw(i32 %x) {
-; CHECK-LABEL: @icmp_mul_nsw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @icmp_mul_nsw(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, 12
@@ -1656,8 +1795,9 @@ define i1 @icmp_mul_nsw(i32 %x) {
 }
 
 define i1 @icmp_mul_nsw1(i32 %x) {
-; CHECK-LABEL: @icmp_mul_nsw1(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @icmp_mul_nsw1(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, 12
@@ -1666,8 +1806,9 @@ define i1 @icmp_mul_nsw1(i32 %x) {
 }
 
 define i1 @icmp_mul_nsw_neg(i32 %x) {
-; CHECK-LABEL: @icmp_mul_nsw_neg(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
+; CHECK-LABEL: define i1 @icmp_mul_nsw_neg(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, -12
@@ -1676,8 +1817,9 @@ define i1 @icmp_mul_nsw_neg(i32 %x) {
 }
 
 define i1 @icmp_mul_nsw_neg1(i32 %x) {
-; CHECK-LABEL: @icmp_mul_nsw_neg1(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @icmp_mul_nsw_neg1(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, -12
@@ -1686,8 +1828,9 @@ define i1 @icmp_mul_nsw_neg1(i32 %x) {
 }
 
 define <2 x i1> @icmp_mul_nsw_neg1_vec(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_mul_nsw_neg1_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @icmp_mul_nsw_neg1_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %mul = mul nsw <2 x i32> %x, <i32 -12, i32 -12>
@@ -1696,7 +1839,8 @@ define <2 x i1> @icmp_mul_nsw_neg1_vec(<2 x i32> %x) {
 }
 
 define i1 @icmp_mul_nsw_0(i32 %x) {
-; CHECK-LABEL: @icmp_mul_nsw_0(
+; CHECK-LABEL: define i1 @icmp_mul_nsw_0(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %mul = mul nsw i32 %x, 0
@@ -1705,8 +1849,9 @@ define i1 @icmp_mul_nsw_0(i32 %x) {
 }
 
 define i1 @icmp_mul(i32 %x) {
-; CHECK-LABEL: @icmp_mul(
-; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X:%.*]], -12
+; CHECK-LABEL: define i1 @icmp_mul(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X]], -12
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MUL]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1717,8 +1862,9 @@ define i1 @icmp_mul(i32 %x) {
 
 ; Checks for icmp (eq|ne) (mul x, C), 0
 define i1 @icmp_mul_neq0(i32 %x) {
-; CHECK-LABEL: @icmp_mul_neq0(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @icmp_mul_neq0(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, -12
@@ -1727,8 +1873,9 @@ define i1 @icmp_mul_neq0(i32 %x) {
 }
 
 define <2 x i1> @icmp_mul_neq0_vec(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_mul_neq0_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @icmp_mul_neq0_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %mul = mul nsw <2 x i32> %x, <i32 -12, i32 -12>
@@ -1737,8 +1884,9 @@ define <2 x i1> @icmp_mul_neq0_vec(<2 x i32> %x) {
 }
 
 define i1 @icmp_mul_eq0(i32 %x) {
-; CHECK-LABEL: @icmp_mul_eq0(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @icmp_mul_eq0(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, 12
@@ -1747,7 +1895,8 @@ define i1 @icmp_mul_eq0(i32 %x) {
 }
 
 define i1 @icmp_mul0_eq0(i32 %x) {
-; CHECK-LABEL: @icmp_mul0_eq0(
+; CHECK-LABEL: define i1 @icmp_mul0_eq0(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 true
 ;
   %mul = mul i32 %x, 0
@@ -1756,7 +1905,8 @@ define i1 @icmp_mul0_eq0(i32 %x) {
 }
 
 define i1 @icmp_mul0_ne0(i32 %x) {
-; CHECK-LABEL: @icmp_mul0_ne0(
+; CHECK-LABEL: define i1 @icmp_mul0_ne0(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %mul = mul i32 %x, 0
@@ -1765,9 +1915,10 @@ define i1 @icmp_mul0_ne0(i32 %x) {
 }
 
 define i1 @icmp_add20_eq_add57(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_add20_eq_add57(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[Y:%.*]], 37
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[TMP1]]
+; CHECK-LABEL: define i1 @icmp_add20_eq_add57(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[Y]], 37
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %1 = add i32 %x, 20
@@ -1777,9 +1928,10 @@ define i1 @icmp_add20_eq_add57(i32 %x, i32 %y) {
 }
 
 define <2 x i1> @icmp_add20_eq_add57_splat(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_add20_eq_add57_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[Y:%.*]], splat (i32 37)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X:%.*]], [[TMP1]]
+; CHECK-LABEL: define <2 x i1> @icmp_add20_eq_add57_splat(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[Y]], splat (i32 37)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add <2 x i32> %x, <i32 20, i32 20>
@@ -1789,9 +1941,10 @@ define <2 x i1> @icmp_add20_eq_add57_splat(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define <2 x i1> @icmp_add20_eq_add57_poison(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_add20_eq_add57_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[Y:%.*]], splat (i32 37)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X:%.*]], [[TMP1]]
+; CHECK-LABEL: define <2 x i1> @icmp_add20_eq_add57_poison(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[Y]], splat (i32 37)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add <2 x i32> %x, <i32 20, i32 20>
@@ -1801,9 +1954,10 @@ define <2 x i1> @icmp_add20_eq_add57_poison(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define <2 x i1> @icmp_add20_eq_add57_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_add20_eq_add57_vec_nonsplat(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[Y:%.*]], <i32 37, i32 39>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X:%.*]], [[TMP1]]
+; CHECK-LABEL: define <2 x i1> @icmp_add20_eq_add57_vec_nonsplat(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[Y]], <i32 37, i32 39>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add <2 x i32> %x, <i32 20, i32 19>
@@ -1813,9 +1967,10 @@ define <2 x i1> @icmp_add20_eq_add57_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define i1 @icmp_sub57_ne_sub20(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_sub57_ne_sub20(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -37
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_sub57_ne_sub20(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], -37
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %1 = add i32 %x, -57
@@ -1825,9 +1980,10 @@ define i1 @icmp_sub57_ne_sub20(i32 %x, i32 %y) {
 }
 
 define <2 x i1> @icmp_sub57_ne_sub20_splat(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_sub57_ne_sub20_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], splat (i32 -37)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], [[Y:%.*]]
+; CHECK-LABEL: define <2 x i1> @icmp_sub57_ne_sub20_splat(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X]], splat (i32 -37)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add <2 x i32> %x, <i32 -57, i32 -57>
@@ -1837,9 +1993,10 @@ define <2 x i1> @icmp_sub57_ne_sub20_splat(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define <2 x i1> @icmp_sub57_ne_sub20_vec_poison(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_sub57_ne_sub20_vec_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], splat (i32 -37)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], [[Y:%.*]]
+; CHECK-LABEL: define <2 x i1> @icmp_sub57_ne_sub20_vec_poison(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X]], splat (i32 -37)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add <2 x i32> %x, <i32 -57, i32 poison>
@@ -1849,9 +2006,10 @@ define <2 x i1> @icmp_sub57_ne_sub20_vec_poison(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define <2 x i1> @icmp_sub57_ne_sub20_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_sub57_ne_sub20_vec_nonsplat(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[Y:%.*]], splat (i32 37)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], [[TMP1]]
+; CHECK-LABEL: define <2 x i1> @icmp_sub57_ne_sub20_vec_nonsplat(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[Y]], splat (i32 37)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add <2 x i32> %x, <i32 -57, i32 -58>
@@ -1861,8 +2019,9 @@ define <2 x i1> @icmp_sub57_ne_sub20_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define i1 @icmp_sub1_sge(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_sub1_sge(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_sub1_sge(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = add nsw i32 %x, -1
@@ -1871,8 +2030,9 @@ define i1 @icmp_sub1_sge(i32 %x, i32 %y) {
 }
 
 define i1 @icmp_add1_sgt(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_add1_sgt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_add1_sgt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %x, 1
@@ -1881,8 +2041,9 @@ define i1 @icmp_add1_sgt(i32 %x, i32 %y) {
 }
 
 define i1 @icmp_sub1_slt(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_sub1_slt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_sub1_slt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = add nsw i32 %x, -1
@@ -1891,8 +2052,9 @@ define i1 @icmp_sub1_slt(i32 %x, i32 %y) {
 }
 
 define i1 @icmp_add1_sle(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_add1_sle(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_add1_sle(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %x, 1
@@ -1901,9 +2063,10 @@ define i1 @icmp_add1_sle(i32 %x, i32 %y) {
 }
 
 define i1 @icmp_slt_offset_with_common_divisor(i64 %x, i64 %y) {
-; CHECK-LABEL: @icmp_slt_offset_with_common_divisor(
-; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
-; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-LABEL: define i1 @icmp_slt_offset_with_common_divisor(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y]], 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[SHLX]], [[SHLY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1915,9 +2078,10 @@ define i1 @icmp_slt_offset_with_common_divisor(i64 %x, i64 %y) {
 }
 
 define i1 @icmp_slt_offset_with_smaller_common_divisor(i64 %x, i64 %y) {
-; CHECK-LABEL: @icmp_slt_offset_with_smaller_common_divisor(
-; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
-; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-LABEL: define i1 @icmp_slt_offset_with_smaller_common_divisor(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y]], 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i64 [[SHLX]], [[SHLY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1929,9 +2093,10 @@ define i1 @icmp_slt_offset_with_smaller_common_divisor(i64 %x, i64 %y) {
 }
 
 define i1 @icmp_sle_offset_with_common_divisor(i64 %x, i64 %y) {
-; CHECK-LABEL: @icmp_sle_offset_with_common_divisor(
-; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
-; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-LABEL: define i1 @icmp_sle_offset_with_common_divisor(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y]], 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SHLX]], [[SHLY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1943,9 +2108,10 @@ define i1 @icmp_sle_offset_with_common_divisor(i64 %x, i64 %y) {
 }
 
 define i1 @icmp_ule_offset_with_common_divisor(i64 %x, i64 %y) {
-; CHECK-LABEL: @icmp_ule_offset_with_common_divisor(
-; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
-; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-LABEL: define i1 @icmp_ule_offset_with_common_divisor(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y]], 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[SHLX]], [[SHLY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1958,9 +2124,10 @@ define i1 @icmp_ule_offset_with_common_divisor(i64 %x, i64 %y) {
 
 ; TODO: Handle non-power-of-2 divisors
 define i1 @icmp_ule_offset_with_common_non_pow2_divisor(i64 %x, i64 %y) {
-; CHECK-LABEL: @icmp_ule_offset_with_common_non_pow2_divisor(
-; CHECK-NEXT:    [[MULX:%.*]] = mul nuw i64 [[X:%.*]], 7
-; CHECK-NEXT:    [[MULY:%.*]] = mul nuw i64 [[Y:%.*]], 7
+; CHECK-LABEL: define i1 @icmp_ule_offset_with_common_non_pow2_divisor(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[MULX:%.*]] = mul nuw i64 [[X]], 7
+; CHECK-NEXT:    [[MULY:%.*]] = mul nuw i64 [[Y]], 7
 ; CHECK-NEXT:    [[MULX_OFFSET:%.*]] = add nuw i64 [[MULX]], 7
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i64 [[MULX_OFFSET]], [[MULY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -1973,9 +2140,10 @@ define i1 @icmp_ule_offset_with_common_non_pow2_divisor(i64 %x, i64 %y) {
 }
 
 define i1 @neg_icmp_slt_offset_without_common_divisor(i64 %x, i64 %y) {
-; CHECK-LABEL: @neg_icmp_slt_offset_without_common_divisor(
-; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
-; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-LABEL: define i1 @neg_icmp_slt_offset_without_common_divisor(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y]], 4
 ; CHECK-NEXT:    [[SHLX_OFFSET:%.*]] = add nsw i64 [[SHLX]], -32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SHLX_OFFSET]], [[SHLY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -1988,9 +2156,10 @@ define i1 @neg_icmp_slt_offset_without_common_divisor(i64 %x, i64 %y) {
 }
 
 define i1 @neg_icmp_slt_offset_with_wrong_sign(i64 %x, i64 %y) {
-; CHECK-LABEL: @neg_icmp_slt_offset_with_wrong_sign(
-; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X:%.*]], 4
-; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y:%.*]], 4
+; CHECK-LABEL: define i1 @neg_icmp_slt_offset_with_wrong_sign(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[SHLX:%.*]] = shl i64 [[X]], 4
+; CHECK-NEXT:    [[SHLY:%.*]] = shl i64 [[Y]], 4
 ; CHECK-NEXT:    [[SHLX_OFFSET:%.*]] = add nsw i64 [[SHLX]], 16
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[SHLX_OFFSET]], [[SHLY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -2003,9 +2172,10 @@ define i1 @neg_icmp_slt_offset_with_wrong_sign(i64 %x, i64 %y) {
 }
 
 define i1 @icmp_add20_sge_add57(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_add20_sge_add57(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[Y:%.*]], 37
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], [[TMP1]]
+; CHECK-LABEL: define i1 @icmp_add20_sge_add57(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[Y]], 37
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %1 = add nsw i32 %x, 20
@@ -2015,9 +2185,10 @@ define i1 @icmp_add20_sge_add57(i32 %x, i32 %y) {
 }
 
 define <2 x i1> @icmp_add20_sge_add57_splat(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_add20_sge_add57_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[Y:%.*]], splat (i32 37)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[X:%.*]], [[TMP1]]
+; CHECK-LABEL: define <2 x i1> @icmp_add20_sge_add57_splat(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[Y]], splat (i32 37)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add nsw <2 x i32> %x, <i32 20, i32 20>
@@ -2027,9 +2198,10 @@ define <2 x i1> @icmp_add20_sge_add57_splat(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define <2 x i1> @icmp_add20_sge_add57_poison(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_add20_sge_add57_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[Y:%.*]], splat (i32 37)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[X:%.*]], [[TMP1]]
+; CHECK-LABEL: define <2 x i1> @icmp_add20_sge_add57_poison(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[Y]], splat (i32 37)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add nsw <2 x i32> %x, <i32 20, i32 20>
@@ -2039,9 +2211,10 @@ define <2 x i1> @icmp_add20_sge_add57_poison(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define <2 x i1> @icmp_add20_sge_add57_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_add20_sge_add57_vec_nonsplat(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[X:%.*]], <i32 20, i32 19>
-; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <2 x i32> [[Y:%.*]], <i32 57, i32 58>
+; CHECK-LABEL: define <2 x i1> @icmp_add20_sge_add57_vec_nonsplat(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[X]], <i32 20, i32 19>
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <2 x i32> [[Y]], <i32 57, i32 58>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2052,9 +2225,10 @@ define <2 x i1> @icmp_add20_sge_add57_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define i1 @icmp_sub57_sge_sub20(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_sub57_sge_sub20(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[X:%.*]], -37
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[TMP1]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_sub57_sge_sub20(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[X]], -37
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %1 = add nsw i32 %x, -57
@@ -2064,9 +2238,10 @@ define i1 @icmp_sub57_sge_sub20(i32 %x, i32 %y) {
 }
 
 define <2 x i1> @icmp_sub57_sge_sub20_splat(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_sub57_sge_sub20_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[X:%.*]], splat (i32 -37)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[TMP1]], [[Y:%.*]]
+; CHECK-LABEL: define <2 x i1> @icmp_sub57_sge_sub20_splat(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[X]], splat (i32 -37)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add nsw <2 x i32> %x, <i32 -57, i32 -57>
@@ -2076,9 +2251,10 @@ define <2 x i1> @icmp_sub57_sge_sub20_splat(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define <2 x i1> @icmp_sub57_sge_sub20_vec_poison(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_sub57_sge_sub20_vec_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[X:%.*]], splat (i32 -37)
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[TMP1]], [[Y:%.*]]
+; CHECK-LABEL: define <2 x i1> @icmp_sub57_sge_sub20_vec_poison(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[X]], splat (i32 -37)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %1 = add nsw <2 x i32> %x, <i32 -57, i32 poison>
@@ -2088,9 +2264,10 @@ define <2 x i1> @icmp_sub57_sge_sub20_vec_poison(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define <2 x i1> @icmp_sub57_sge_sub20_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_sub57_sge_sub20_vec_nonsplat(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[X:%.*]], <i32 -57, i32 -58>
-; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <2 x i32> [[Y:%.*]], <i32 -20, i32 -21>
+; CHECK-LABEL: define <2 x i1> @icmp_sub57_sge_sub20_vec_nonsplat(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <2 x i32> [[X]], <i32 -57, i32 -58>
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <2 x i32> [[Y]], <i32 -20, i32 -21>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge <2 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2101,9 +2278,10 @@ define <2 x i1> @icmp_sub57_sge_sub20_vec_nonsplat(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define i1 @icmp_and_shl_neg_ne_0(i32 %A, i32 %B) {
-; CHECK-LABEL: @icmp_and_shl_neg_ne_0(
-; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[B:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A:%.*]]
+; CHECK-LABEL: define i1 @icmp_and_shl_neg_ne_0(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2115,9 +2293,10 @@ define i1 @icmp_and_shl_neg_ne_0(i32 %A, i32 %B) {
 }
 
 define i1 @icmp_and_shl_neg_ne_0_shl2_no_flags(i32 %A, i32 %B) {
-; CHECK-LABEL: @icmp_and_shl_neg_ne_0_shl2_no_flags(
-; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[A:%.*]], -1
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 2, [[B:%.*]]
+; CHECK-LABEL: define i1 @icmp_and_shl_neg_ne_0_shl2_no_flags(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[NEG:%.*]] = xor i32 [[A]], -1
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 2, [[B]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHL]], [[NEG]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -2130,9 +2309,10 @@ define i1 @icmp_and_shl_neg_ne_0_shl2_no_flags(i32 %A, i32 %B) {
 }
 
 define i1 @icmp_and_shl_neg_ne_0_shl2_nuw(i32 %A, i32 %B) {
-; CHECK-LABEL: @icmp_and_shl_neg_ne_0_shl2_nuw(
-; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 2, [[B:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A:%.*]]
+; CHECK-LABEL: define i1 @icmp_and_shl_neg_ne_0_shl2_nuw(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 2, [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2144,9 +2324,10 @@ define i1 @icmp_and_shl_neg_ne_0_shl2_nuw(i32 %A, i32 %B) {
 }
 
 define i1 @icmp_and_shl_neg_ne_0_shl2_nsw(i32 %A, i32 %B) {
-; CHECK-LABEL: @icmp_and_shl_neg_ne_0_shl2_nsw(
-; CHECK-NEXT:    [[SHL:%.*]] = shl nsw i32 2, [[B:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A:%.*]]
+; CHECK-LABEL: define i1 @icmp_and_shl_neg_ne_0_shl2_nsw(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl nsw i32 2, [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2158,9 +2339,10 @@ define i1 @icmp_and_shl_neg_ne_0_shl2_nsw(i32 %A, i32 %B) {
 }
 
 define i1 @icmp_and_shl_neg_eq_0(i32 %A, i32 %B) {
-; CHECK-LABEL: @icmp_and_shl_neg_eq_0(
-; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[B:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A:%.*]]
+; CHECK-LABEL: define i1 @icmp_and_shl_neg_eq_0(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i32 1, [[B]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2172,8 +2354,9 @@ define i1 @icmp_and_shl_neg_eq_0(i32 %A, i32 %B) {
 }
 
 define i1 @icmp_add_and_shr_ne_0(i32 %X) {
-; CHECK-LABEL: @icmp_add_and_shr_ne_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 240
+; CHECK-LABEL: define i1 @icmp_add_and_shr_ne_0(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], 240
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP1]], 224
 ; CHECK-NEXT:    ret i1 [[TOBOOL]]
 ;
@@ -2185,8 +2368,9 @@ define i1 @icmp_add_and_shr_ne_0(i32 %X) {
 }
 
 define <2 x i1> @icmp_add_and_shr_ne_0_vec(<2 x i32> %X) {
-; CHECK-LABEL: @icmp_add_and_shr_ne_0_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 240)
+; CHECK-LABEL: define <2 x i1> @icmp_add_and_shr_ne_0_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], splat (i32 240)
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne <2 x i32> [[TMP1]], splat (i32 224)
 ; CHECK-NEXT:    ret <2 x i1> [[TOBOOL]]
 ;
@@ -2199,8 +2383,9 @@ define <2 x i1> @icmp_add_and_shr_ne_0_vec(<2 x i32> %X) {
 
 ; Variation of the above with an extra use of the shift
 define i1 @icmp_and_shr_multiuse(i32 %X) {
-; CHECK-LABEL: @icmp_and_shr_multiuse(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 240
+; CHECK-LABEL: define i1 @icmp_and_shr_multiuse(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], 240
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP1]], 224
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[X]], 496
 ; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp ne i32 [[TMP2]], 432
@@ -2217,8 +2402,9 @@ define i1 @icmp_and_shr_multiuse(i32 %X) {
 }
 
 define i1 @icmp_and_shr_multiuse_logical(i32 %X) {
-; CHECK-LABEL: @icmp_and_shr_multiuse_logical(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 240
+; CHECK-LABEL: define i1 @icmp_and_shr_multiuse_logical(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], 240
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP1]], 224
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[X]], 496
 ; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp ne i32 [[TMP2]], 432
@@ -2236,8 +2422,9 @@ define i1 @icmp_and_shr_multiuse_logical(i32 %X) {
 
 ; Variation of the above with an ashr
 define i1 @icmp_and_ashr_multiuse(i32 %X) {
-; CHECK-LABEL: @icmp_and_ashr_multiuse(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 240
+; CHECK-LABEL: define i1 @icmp_and_ashr_multiuse(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], 240
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP1]], 224
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[X]], 496
 ; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp ne i32 [[TMP2]], 432
@@ -2254,8 +2441,9 @@ define i1 @icmp_and_ashr_multiuse(i32 %X) {
 }
 
 define i1 @icmp_and_ashr_multiuse_logical(i32 %X) {
-; CHECK-LABEL: @icmp_and_ashr_multiuse_logical(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 240
+; CHECK-LABEL: define i1 @icmp_and_ashr_multiuse_logical(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], 240
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[TMP1]], 224
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[X]], 496
 ; CHECK-NEXT:    [[TOBOOL2:%.*]] = icmp ne i32 [[TMP2]], 432
@@ -2272,8 +2460,9 @@ define i1 @icmp_and_ashr_multiuse_logical(i32 %X) {
 }
 
 define i1 @icmp_lshr_and_overshift(i8 %X) {
-; CHECK-LABEL: @icmp_lshr_and_overshift(
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ugt i8 [[X:%.*]], 31
+; CHECK-LABEL: define i1 @icmp_lshr_and_overshift(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ugt i8 [[X]], 31
 ; CHECK-NEXT:    ret i1 [[TOBOOL]]
 ;
   %shr = lshr i8 %X, 5
@@ -2284,8 +2473,9 @@ define i1 @icmp_lshr_and_overshift(i8 %X) {
 
 ; We shouldn't simplify this because the and uses bits that are shifted in.
 define i1 @icmp_ashr_and_overshift(i8 %X) {
-; CHECK-LABEL: @icmp_ashr_and_overshift(
-; CHECK-NEXT:    [[SHR:%.*]] = ashr i8 [[X:%.*]], 5
+; CHECK-LABEL: define i1 @icmp_ashr_and_overshift(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i8 [[X]], 5
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[SHR]], 15
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i8 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[TOBOOL]]
@@ -2297,8 +2487,9 @@ define i1 @icmp_ashr_and_overshift(i8 %X) {
 }
 
 define i1 @icmp_and_ashr_neg_and_legal(i8 %x) {
-; CHECK-LABEL: @icmp_and_ashr_neg_and_legal(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 32
+; CHECK-LABEL: define i1 @icmp_and_ashr_neg_and_legal(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X]], 32
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %ashr = ashr i8 %x, 4
@@ -2309,8 +2500,9 @@ define i1 @icmp_and_ashr_neg_and_legal(i8 %x) {
 
 ; Negative test.
 define i1 @icmp_and_ashr_mixed_and_shiftout(i8 %x) {
-; CHECK-LABEL: @icmp_and_ashr_mixed_and_shiftout(
-; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[X:%.*]], 4
+; CHECK-LABEL: define i1 @icmp_and_ashr_mixed_and_shiftout(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[X]], 4
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[ASHR]], 31
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp samesign ugt i8 [[AND]], 8
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -2322,8 +2514,9 @@ define i1 @icmp_and_ashr_mixed_and_shiftout(i8 %x) {
 }
 
 define i1 @icmp_and_ashr_neg_cmp_slt_legal(i8 %x) {
-; CHECK-LABEL: @icmp_and_ashr_neg_cmp_slt_legal(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -32
+; CHECK-LABEL: define i1 @icmp_and_ashr_neg_cmp_slt_legal(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X]], -32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], -64
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2335,8 +2528,9 @@ define i1 @icmp_and_ashr_neg_cmp_slt_legal(i8 %x) {
 
 ; Negative test.
 define i1 @icmp_and_ashr_neg_cmp_slt_shiftout(i8 %x) {
-; CHECK-LABEL: @icmp_and_ashr_neg_cmp_slt_shiftout(
-; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[X:%.*]], 4
+; CHECK-LABEL: define i1 @icmp_and_ashr_neg_cmp_slt_shiftout(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i8 [[X]], 4
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[ASHR]], -2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[AND]], -68
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -2348,8 +2542,9 @@ define i1 @icmp_and_ashr_neg_cmp_slt_shiftout(i8 %x) {
 }
 
 define i1 @icmp_and_ashr_neg_cmp_eq_legal(i8 %x) {
-; CHECK-LABEL: @icmp_and_ashr_neg_cmp_eq_legal(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X:%.*]], -32
+; CHECK-LABEL: define i1 @icmp_and_ashr_neg_cmp_eq_legal(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[X]], -32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP1]], -64
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2360,7 +2555,8 @@ define i1 @icmp_and_ashr_neg_cmp_eq_legal(i8 %x) {
 }
 
 define i1 @icmp_and_ashr_neg_cmp_eq_shiftout(i8 %x) {
-; CHECK-LABEL: @icmp_and_ashr_neg_cmp_eq_shiftout(
+; CHECK-LABEL: define i1 @icmp_and_ashr_neg_cmp_eq_shiftout(
+; CHECK-SAME: i8 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %ashr = ashr i8 %x, 4
@@ -2370,7 +2566,8 @@ define i1 @icmp_and_ashr_neg_cmp_eq_shiftout(i8 %x) {
 }
 
 define i1 @icmp_and_ashr_neg_cmp_ne_shiftout(i8 %x) {
-; CHECK-LABEL: @icmp_and_ashr_neg_cmp_ne_shiftout(
+; CHECK-LABEL: define i1 @icmp_and_ashr_neg_cmp_ne_shiftout(
+; CHECK-SAME: i8 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 true
 ;
   %ashr = ashr i8 %x, 4
@@ -2380,8 +2577,9 @@ define i1 @icmp_and_ashr_neg_cmp_ne_shiftout(i8 %x) {
 }
 
 define i1 @icmp_shl_1_V_ult_32(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ult_32(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V:%.*]], 5
+; CHECK-LABEL: define i1 @icmp_shl_1_V_ult_32(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V]], 5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2390,8 +2588,9 @@ define i1 @icmp_shl_1_V_ult_32(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_ult_32_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ult_32_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V:%.*]], splat (i32 5)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_ult_32_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V]], splat (i32 5)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2400,8 +2599,9 @@ define <2 x i1> @icmp_shl_1_V_ult_32_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_eq_32(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_eq_32(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V:%.*]], 5
+; CHECK-LABEL: define i1 @icmp_shl_1_V_eq_32(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V]], 5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2410,8 +2610,9 @@ define i1 @icmp_shl_1_V_eq_32(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_eq_32_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_eq_32_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V:%.*]], splat (i32 5)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_eq_32_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V]], splat (i32 5)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2420,8 +2621,9 @@ define <2 x i1> @icmp_shl_1_V_eq_32_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_ult_30(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ult_30(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V:%.*]], 5
+; CHECK-LABEL: define i1 @icmp_shl_1_V_ult_30(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V]], 5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2430,8 +2632,9 @@ define i1 @icmp_shl_1_V_ult_30(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_ult_30_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ult_30_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V:%.*]], splat (i32 5)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_ult_30_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V]], splat (i32 5)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2440,8 +2643,9 @@ define <2 x i1> @icmp_shl_1_V_ult_30_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_ugt_30(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ugt_30(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[V:%.*]], 4
+; CHECK-LABEL: define i1 @icmp_shl_1_V_ugt_30(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[V]], 4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2450,8 +2654,9 @@ define i1 @icmp_shl_1_V_ugt_30(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_ugt_30_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ugt_30_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[V:%.*]], splat (i32 4)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_ugt_30_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[V]], splat (i32 4)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2460,8 +2665,9 @@ define <2 x i1> @icmp_shl_1_V_ugt_30_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_ule_30(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ule_30(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V:%.*]], 5
+; CHECK-LABEL: define i1 @icmp_shl_1_V_ule_30(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V]], 5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2470,8 +2676,9 @@ define i1 @icmp_shl_1_V_ule_30(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_ule_30_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ule_30_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V:%.*]], splat (i32 5)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_ule_30_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V]], splat (i32 5)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2480,8 +2687,9 @@ define <2 x i1> @icmp_shl_1_V_ule_30_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_uge_30(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_uge_30(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[V:%.*]], 4
+; CHECK-LABEL: define i1 @icmp_shl_1_V_uge_30(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[V]], 4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2490,8 +2698,9 @@ define i1 @icmp_shl_1_V_uge_30(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_uge_30_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_uge_30_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[V:%.*]], splat (i32 4)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_uge_30_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[V]], splat (i32 4)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2500,8 +2709,9 @@ define <2 x i1> @icmp_shl_1_V_uge_30_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_uge_2147483648(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_uge_2147483648(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V:%.*]], 31
+; CHECK-LABEL: define i1 @icmp_shl_1_V_uge_2147483648(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V]], 31
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2510,8 +2720,9 @@ define i1 @icmp_shl_1_V_uge_2147483648(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_uge_2147483648_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_uge_2147483648_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V:%.*]], splat (i32 31)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_uge_2147483648_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V]], splat (i32 31)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2520,8 +2731,9 @@ define <2 x i1> @icmp_shl_1_V_uge_2147483648_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_ult_2147483648(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ult_2147483648(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[V:%.*]], 31
+; CHECK-LABEL: define i1 @icmp_shl_1_V_ult_2147483648(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[V]], 31
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2530,8 +2742,9 @@ define i1 @icmp_shl_1_V_ult_2147483648(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_ult_2147483648_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_ult_2147483648_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[V:%.*]], splat (i32 31)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_ult_2147483648_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[V]], splat (i32 31)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2540,8 +2753,9 @@ define <2 x i1> @icmp_shl_1_V_ult_2147483648_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_sle_0(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_sle_0(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V:%.*]], 31
+; CHECK-LABEL: define i1 @icmp_shl_1_V_sle_0(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V]], 31
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2550,8 +2764,9 @@ define i1 @icmp_shl_1_V_sle_0(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_sle_0_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_sle_0_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V:%.*]], splat (i32 31)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_sle_0_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V]], splat (i32 31)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2560,8 +2775,9 @@ define <2 x i1> @icmp_shl_1_V_sle_0_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_sle_negative(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_sle_negative(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V:%.*]], 31
+; CHECK-LABEL: define i1 @icmp_shl_1_V_sle_negative(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V]], 31
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2570,8 +2786,9 @@ define i1 @icmp_shl_1_V_sle_negative(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_sle_0_negative(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_sle_0_negative(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V:%.*]], splat (i32 31)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_sle_0_negative(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V]], splat (i32 31)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2580,8 +2797,9 @@ define <2 x i1> @icmp_shl_1_V_sle_0_negative(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_sgt_0(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_sgt_0(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[V:%.*]], 31
+; CHECK-LABEL: define i1 @icmp_shl_1_V_sgt_0(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[V]], 31
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2590,8 +2808,9 @@ define i1 @icmp_shl_1_V_sgt_0(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_sgt_0_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_sgt_0_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[V:%.*]], splat (i32 31)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_sgt_0_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[V]], splat (i32 31)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2600,8 +2819,9 @@ define <2 x i1> @icmp_shl_1_V_sgt_0_vec(<2 x i32> %V) {
 }
 
 define i1 @icmp_shl_1_V_sgt_negative(i32 %V) {
-; CHECK-LABEL: @icmp_shl_1_V_sgt_negative(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[V:%.*]], 31
+; CHECK-LABEL: define i1 @icmp_shl_1_V_sgt_negative(
+; CHECK-SAME: i32 [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[V]], 31
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -2610,8 +2830,9 @@ define i1 @icmp_shl_1_V_sgt_negative(i32 %V) {
 }
 
 define <2 x i1> @icmp_shl_1_V_sgt_negative_vec(<2 x i32> %V) {
-; CHECK-LABEL: @icmp_shl_1_V_sgt_negative_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[V:%.*]], splat (i32 31)
+; CHECK-LABEL: define <2 x i1> @icmp_shl_1_V_sgt_negative_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[V]], splat (i32 31)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -2620,9 +2841,10 @@ define <2 x i1> @icmp_shl_1_V_sgt_negative_vec(<2 x i32> %V) {
 }
 
 define i1 @or_icmp_eq_B_0_icmp_ult_A_B(i64 %a, i64 %b) {
-; CHECK-LABEL: @or_icmp_eq_B_0_icmp_ult_A_B(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B:%.*]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[TMP1]], [[A:%.*]]
+; CHECK-LABEL: define i1 @or_icmp_eq_B_0_icmp_ult_A_B(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[TMP1]], [[A]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %1 = icmp eq i64 %b, 0
@@ -2632,9 +2854,10 @@ define i1 @or_icmp_eq_B_0_icmp_ult_A_B(i64 %a, i64 %b) {
 }
 
 define i1 @or_icmp_eq_B_0_icmp_ult_A_B_logical(i64 %a, i64 %b) {
-; CHECK-LABEL: @or_icmp_eq_B_0_icmp_ult_A_B_logical(
-; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[A:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[B:%.*]], -1
+; CHECK-LABEL: define i1 @or_icmp_eq_B_0_icmp_ult_A_B_logical(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[B]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp uge i64 [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -2645,9 +2868,10 @@ define i1 @or_icmp_eq_B_0_icmp_ult_A_B_logical(i64 %a, i64 %b) {
 }
 
 define <2 x i1> @or_icmp_eq_B_0_icmp_ult_A_B_uniform(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @or_icmp_eq_B_0_icmp_ult_A_B_uniform(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i64> [[B:%.*]], splat (i64 -1)
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge <2 x i64> [[TMP1]], [[A:%.*]]
+; CHECK-LABEL: define <2 x i1> @or_icmp_eq_B_0_icmp_ult_A_B_uniform(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i64> [[B]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge <2 x i64> [[TMP1]], [[A]]
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
   %1 = icmp eq <2 x i64> %b, zeroinitializer
@@ -2657,9 +2881,10 @@ define <2 x i1> @or_icmp_eq_B_0_icmp_ult_A_B_uniform(<2 x i64> %a, <2 x i64> %b)
 }
 
 define <2 x i1> @or_icmp_eq_B_0_icmp_ult_A_B_poison(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @or_icmp_eq_B_0_icmp_ult_A_B_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i64> [[B:%.*]], splat (i64 -1)
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge <2 x i64> [[TMP1]], [[A:%.*]]
+; CHECK-LABEL: define <2 x i1> @or_icmp_eq_B_0_icmp_ult_A_B_poison(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i64> [[B]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge <2 x i64> [[TMP1]], [[A]]
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
   %1 = icmp eq <2 x i64> %b, <i64 0, i64 poison>
@@ -2669,8 +2894,9 @@ define <2 x i1> @or_icmp_eq_B_0_icmp_ult_A_B_poison(<2 x i64> %a, <2 x i64> %b)
 }
 
 define i1 @or_icmp_ne_A_0_icmp_ne_B_0(i64 %a, i64 %b) {
-; CHECK-LABEL: @or_icmp_ne_A_0_icmp_ne_B_0(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i64 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i1 @or_icmp_ne_A_0_icmp_ne_B_0(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = or i64 [[A]], [[B]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i64 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -2681,9 +2907,10 @@ define i1 @or_icmp_ne_A_0_icmp_ne_B_0(i64 %a, i64 %b) {
 }
 
 define i1 @or_icmp_ne_A_0_icmp_ne_B_0_logical(i64 %a, i64 %b) {
-; CHECK-LABEL: @or_icmp_ne_A_0_icmp_ne_B_0_logical(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[A:%.*]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i64 [[B:%.*]], 0
+; CHECK-LABEL: define i1 @or_icmp_ne_A_0_icmp_ne_B_0_logical(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[A]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i64 [[B]], 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP1]], i1 true, i1 [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -2694,8 +2921,9 @@ define i1 @or_icmp_ne_A_0_icmp_ne_B_0_logical(i64 %a, i64 %b) {
 }
 
 define <2 x i1> @or_icmp_ne_A_0_icmp_ne_B_0_uniform(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @or_icmp_ne_A_0_icmp_ne_B_0_uniform(
-; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define <2 x i1> @or_icmp_ne_A_0_icmp_ne_B_0_uniform(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i64> [[A]], [[B]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
@@ -2706,8 +2934,9 @@ define <2 x i1> @or_icmp_ne_A_0_icmp_ne_B_0_uniform(<2 x i64> %a, <2 x i64> %b)
 }
 
 define <2 x i1> @or_icmp_ne_A_0_icmp_ne_B_0_poison(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @or_icmp_ne_A_0_icmp_ne_B_0_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define <2 x i1> @or_icmp_ne_A_0_icmp_ne_B_0_poison(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i64> [[A]], [[B]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i64> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
@@ -2718,9 +2947,10 @@ define <2 x i1> @or_icmp_ne_A_0_icmp_ne_B_0_poison(<2 x i64> %a, <2 x i64> %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B:%.*]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[TMP1]], [[A:%.*]]
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[TMP1]], [[A]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %1 = icmp ne i64 %b, 0
@@ -2730,9 +2960,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted1(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_commuted1(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B:%.*]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[TMP1]], [[A:%.*]]
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted1(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[TMP1]], [[A]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %1 = icmp uge i64 %a, %b
@@ -2742,9 +2973,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted1(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted2(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_commuted2(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B:%.*]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[TMP1]], [[A:%.*]]
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted2(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[TMP1]], [[A]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %1 = icmp ne i64 %b, 0
@@ -2754,9 +2986,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted2(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted1_logical(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_commuted1_logical(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B:%.*]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[TMP1]], [[A:%.*]]
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted1_logical(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i64 [[TMP1]], [[A]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %1 = icmp uge i64 %a, %b
@@ -2766,9 +2999,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted1_logical(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted2_logical(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_commuted2_logical(
-; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[A:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[B:%.*]], -1
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted2_logical(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[B]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -2779,11 +3013,12 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_commuted2_logical(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_extra_use1(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_extra_use1(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B:%.*]], 0
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_extra_use1(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B]], 0
 ; CHECK-NEXT:    call void @use_i1(i1 [[TMP1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[B]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[A:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[A]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
   %1 = icmp ne i64 %b, 0
@@ -2794,8 +3029,9 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_extra_use1(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_extra_use2(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_extra_use2(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i64 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_extra_use2(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i64 [[A]], [[B]]
 ; CHECK-NEXT:    call void @use_i1(i1 [[TMP1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[B]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[A]]
@@ -2809,10 +3045,11 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_extra_use2(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_extra_use3(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_extra_use3(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B:%.*]], 0
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_extra_use3(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B]], 0
 ; CHECK-NEXT:    call void @use_i1(i1 [[TMP1]])
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[A]], [[B]]
 ; CHECK-NEXT:    call void @use_i1(i1 [[TMP2]])
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
@@ -2826,9 +3063,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_extra_use3(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_pred1(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_wrong_pred1(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[B:%.*]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[A:%.*]], [[B]]
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_pred1(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[B]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[A]], [[B]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -2839,9 +3077,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_pred1(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_pred2(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_wrong_pred2(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B:%.*]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[A:%.*]], [[B]]
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_pred2(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[A]], [[B]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -2852,9 +3091,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_pred2(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_op1(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_wrong_op1(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B:%.*]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[A:%.*]], [[B]]
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_op1(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[A]], [[B]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -2865,9 +3105,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_op1(i64 %a, i64 %b) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_op2(i64 %a, i64 %b, i64 %c) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_wrong_op2(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B:%.*]], 0
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[A:%.*]], [[C:%.*]]
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_op2(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i64 [[B]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[A]], [[C]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -2878,9 +3119,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_wrong_op2(i64 %a, i64 %b, i64 %c) {
 }
 
 define i1 @and_icmp_ne_B_0_icmp_uge_A_B_logical(i64 %a, i64 %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_logical(
-; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[A:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[B:%.*]], -1
+; CHECK-LABEL: define i1 @and_icmp_ne_B_0_icmp_uge_A_B_logical(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[B]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
 ;
@@ -2891,9 +3133,10 @@ define i1 @and_icmp_ne_B_0_icmp_uge_A_B_logical(i64 %a, i64 %b) {
 }
 
 define <2 x i1> @and_icmp_ne_B_0_icmp_uge_A_B_uniform(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_uniform(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i64> [[B:%.*]], splat (i64 -1)
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <2 x i64> [[TMP1]], [[A:%.*]]
+; CHECK-LABEL: define <2 x i1> @and_icmp_ne_B_0_icmp_uge_A_B_uniform(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i64> [[B]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <2 x i64> [[TMP1]], [[A]]
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
   %1 = icmp ne <2 x i64> %b, zeroinitializer
@@ -2903,9 +3146,10 @@ define <2 x i1> @and_icmp_ne_B_0_icmp_uge_A_B_uniform(<2 x i64> %a, <2 x i64> %b
 }
 
 define <2 x i1> @and_icmp_ne_B_0_icmp_uge_A_B_poison(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @and_icmp_ne_B_0_icmp_uge_A_B_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i64> [[B:%.*]], splat (i64 -1)
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <2 x i64> [[TMP1]], [[A:%.*]]
+; CHECK-LABEL: define <2 x i1> @and_icmp_ne_B_0_icmp_uge_A_B_poison(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i64> [[B]], splat (i64 -1)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <2 x i64> [[TMP1]], [[A]]
 ; CHECK-NEXT:    ret <2 x i1> [[TMP2]]
 ;
   %1 = icmp ne <2 x i64> %b, <i64 0, i64 poison>
@@ -2915,8 +3159,9 @@ define <2 x i1> @and_icmp_ne_B_0_icmp_uge_A_B_poison(<2 x i64> %a, <2 x i64> %b)
 }
 
 define i1 @icmp_add_ult_2(i32 %X) {
-; CHECK-LABEL: @icmp_add_ult_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-LABEL: define i1 @icmp_add_ult_2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], -2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 14
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2926,8 +3171,9 @@ define i1 @icmp_add_ult_2(i32 %X) {
 }
 
 define <2 x i1> @icmp_add_X_-14_ult_2_vec(<2 x i32> %X) {
-; CHECK-LABEL: @icmp_add_X_-14_ult_2_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 -2)
+; CHECK-LABEL: define <2 x i1> @icmp_add_X_-14_ult_2_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], splat (i32 -2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], splat (i32 14)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2937,8 +3183,9 @@ define <2 x i1> @icmp_add_X_-14_ult_2_vec(<2 x i32> %X) {
 }
 
 define i1 @icmp_sub_3_X_ult_2(i32 %X) {
-; CHECK-LABEL: @icmp_sub_3_X_ult_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-LABEL: define i1 @icmp_sub_3_X_ult_2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], -2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2948,8 +3195,9 @@ define i1 @icmp_sub_3_X_ult_2(i32 %X) {
 }
 
 define <2 x i1> @icmp_sub_3_X_ult_2_vec(<2 x i32> %X) {
-; CHECK-LABEL: @icmp_sub_3_X_ult_2_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 -2)
+; CHECK-LABEL: define <2 x i1> @icmp_sub_3_X_ult_2_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], splat (i32 -2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], splat (i32 2)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2959,8 +3207,9 @@ define <2 x i1> @icmp_sub_3_X_ult_2_vec(<2 x i32> %X) {
 }
 
 define i1 @icmp_add_X_-14_uge_2(i32 %X) {
-; CHECK-LABEL: @icmp_add_X_-14_uge_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-LABEL: define i1 @icmp_add_X_-14_uge_2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], -2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], 14
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2970,8 +3219,9 @@ define i1 @icmp_add_X_-14_uge_2(i32 %X) {
 }
 
 define <2 x i1> @icmp_add_X_-14_uge_2_vec(<2 x i32> %X) {
-; CHECK-LABEL: @icmp_add_X_-14_uge_2_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 -2)
+; CHECK-LABEL: define <2 x i1> @icmp_add_X_-14_uge_2_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], splat (i32 -2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], splat (i32 14)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2981,8 +3231,9 @@ define <2 x i1> @icmp_add_X_-14_uge_2_vec(<2 x i32> %X) {
 }
 
 define i1 @icmp_sub_3_X_uge_2(i32 %X) {
-; CHECK-LABEL: @icmp_sub_3_X_uge_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-LABEL: define i1 @icmp_sub_3_X_uge_2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], -2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], 2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2992,8 +3243,9 @@ define i1 @icmp_sub_3_X_uge_2(i32 %X) {
 }
 
 define <2 x i1> @icmp_sub_3_X_uge_2_vec(<2 x i32> %X) {
-; CHECK-LABEL: @icmp_sub_3_X_uge_2_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 -2)
+; CHECK-LABEL: define <2 x i1> @icmp_sub_3_X_uge_2_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], splat (i32 -2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], splat (i32 2)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -3003,8 +3255,9 @@ define <2 x i1> @icmp_sub_3_X_uge_2_vec(<2 x i32> %X) {
 }
 
 define i1 @icmp_and_X_-16_eq-16(i32 %X) {
-; CHECK-LABEL: @icmp_and_X_-16_eq-16(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], -17
+; CHECK-LABEL: define i1 @icmp_and_X_-16_eq-16(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X]], -17
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %and = and i32 %X, -16
@@ -3013,8 +3266,9 @@ define i1 @icmp_and_X_-16_eq-16(i32 %X) {
 }
 
 define <2 x i1> @icmp_and_X_-16_eq-16_vec(<2 x i32> %X) {
-; CHECK-LABEL: @icmp_and_X_-16_eq-16_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], splat (i32 -17)
+; CHECK-LABEL: define <2 x i1> @icmp_and_X_-16_eq-16_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X]], splat (i32 -17)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %and = and <2 x i32> %X, <i32 -16, i32 -16>
@@ -3023,8 +3277,9 @@ define <2 x i1> @icmp_and_X_-16_eq-16_vec(<2 x i32> %X) {
 }
 
 define i1 @icmp_and_X_-16_ne-16(i32 %X) {
-; CHECK-LABEL: @icmp_and_X_-16_ne-16(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], -16
+; CHECK-LABEL: define i1 @icmp_and_X_-16_ne-16(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], -16
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %and = and i32 %X, -16
@@ -3033,8 +3288,9 @@ define i1 @icmp_and_X_-16_ne-16(i32 %X) {
 }
 
 define <2 x i1> @icmp_and_X_-16_ne-16_vec(<2 x i32> %X) {
-; CHECK-LABEL: @icmp_and_X_-16_ne-16_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X:%.*]], splat (i32 -16)
+; CHECK-LABEL: define <2 x i1> @icmp_and_X_-16_ne-16_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X]], splat (i32 -16)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %and = and <2 x i32> %X, <i32 -16, i32 -16>
@@ -3046,8 +3302,9 @@ define <2 x i1> @icmp_and_X_-16_ne-16_vec(<2 x i32> %X) {
 ; X | C == C --> X <=u C (when C+1 is PowerOf2).
 
 define i1 @or1_eq1(i32 %x) {
-; CHECK-LABEL: @or1_eq1(
-; CHECK-NEXT:    [[T1:%.*]] = icmp ult i32 [[X:%.*]], 2
+; CHECK-LABEL: define i1 @or1_eq1(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[T1:%.*]] = icmp ult i32 [[X]], 2
 ; CHECK-NEXT:    ret i1 [[T1]]
 ;
   %t0 = or i32 %x, 1
@@ -3056,8 +3313,9 @@ define i1 @or1_eq1(i32 %x) {
 }
 
 define <2 x i1> @or1_eq1_vec(<2 x i32> %x) {
-; CHECK-LABEL: @or1_eq1_vec(
-; CHECK-NEXT:    [[T1:%.*]] = icmp ult <2 x i32> [[X:%.*]], splat (i32 2)
+; CHECK-LABEL: define <2 x i1> @or1_eq1_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[T1:%.*]] = icmp ult <2 x i32> [[X]], splat (i32 2)
 ; CHECK-NEXT:    ret <2 x i1> [[T1]]
 ;
   %t0 = or <2 x i32> %x, splat (i32 1)
@@ -3066,8 +3324,9 @@ define <2 x i1> @or1_eq1_vec(<2 x i32> %x) {
 }
 
 define <2 x i1> @or_eq_vec_nonsplat(<2 x i32> %x) {
-; CHECK-LABEL: @or_eq_vec_nonsplat(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 -2, i32 -3>
+; CHECK-LABEL: define <2 x i1> @or_eq_vec_nonsplat(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], <i32 -2, i32 -3>
 ; CHECK-NEXT:    [[T1:%.*]] = icmp eq <2 x i32> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[T1]]
 ;
@@ -3077,16 +3336,17 @@ define <2 x i1> @or_eq_vec_nonsplat(<2 x i32> %x) {
 }
 
 define void @or_eq_vec_multiple_nonsplat(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z, ptr %ptr0, ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: @or_eq_vec_multiple_nonsplat(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 -2, i32 -3>
+; CHECK-LABEL: define void @or_eq_vec_multiple_nonsplat(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[Z:%.*]], ptr [[PTR0:%.*]], ptr [[PTR1:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], <i32 -2, i32 -3>
 ; CHECK-NEXT:    [[CMP0:%.*]] = icmp eq <2 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    store <2 x i1> [[CMP0]], ptr [[PTR0:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[Y:%.*]], <i32 -2, i32 -3>
+; CHECK-NEXT:    store <2 x i1> [[CMP0]], ptr [[PTR0]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[Y]], <i32 -2, i32 -3>
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq <2 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    store <2 x i1> [[CMP1]], ptr [[PTR1:%.*]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i32> [[Z:%.*]], <i32 -2, i32 -3>
+; CHECK-NEXT:    store <2 x i1> [[CMP1]], ptr [[PTR1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i32> [[Z]], <i32 -2, i32 -3>
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq <2 x i32> [[TMP3]], zeroinitializer
-; CHECK-NEXT:    store <2 x i1> [[CMP2]], ptr [[PTR2:%.*]], align 1
+; CHECK-NEXT:    store <2 x i1> [[CMP2]], ptr [[PTR2]], align 1
 ; CHECK-NEXT:    ret void
 ;
   %t0 = or <2 x i32> %x, <i32 1, i32 2>
@@ -3105,13 +3365,14 @@ define void @or_eq_vec_multiple_nonsplat(<2 x i32> %x, <2 x i32> %y, <2 x i32> %
 
 ; Make sure use count of 1 doesn't matter
 define i1 @or1_eq1_multiple(i32 %x, i32 %y, i32 %z, ptr %ptr0, ptr %ptr1) {
-; CHECK-LABEL: @or1_eq1_multiple(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[X:%.*]], 2
-; CHECK-NEXT:    store i1 [[CMP1]], ptr [[PTR:%.*]], align 1
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Y:%.*]], 2
-; CHECK-NEXT:    store i1 [[CMP2]], ptr [[PTR1:%.*]], align 1
-; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[Z:%.*]], 2
-; CHECK-NEXT:    ret i1 [[CMP3]]
+; CHECK-LABEL: define i1 @or1_eq1_multiple(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]], ptr [[PTR0:%.*]], ptr [[PTR1:%.*]]) {
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp ult i32 [[X]], 2
+; CHECK-NEXT:    store i1 [[CMP0]], ptr [[PTR0]], align 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i32 [[Y]], 2
+; CHECK-NEXT:    store i1 [[CMP1]], ptr [[PTR1]], align 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i32 [[Z]], 2
+; CHECK-NEXT:    ret i1 [[CMP2]]
 ;
   %t0 = or i32 %x, 1
   %cmp0 = icmp eq i32 %t0, 1
@@ -3127,12 +3388,13 @@ define i1 @or1_eq1_multiple(i32 %x, i32 %y, i32 %z, ptr %ptr0, ptr %ptr1) {
 }
 
 define <2 x i1> @or1_eq1_multiple_vec(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z, ptr %ptr0, ptr %ptr1) {
-; CHECK-LABEL: @or1_eq1_multiple_vec(
-; CHECK-NEXT:    [[CMP0:%.*]] = icmp ult <2 x i32> [[X:%.*]], splat (i32 2)
-; CHECK-NEXT:    store <2 x i1> [[CMP0]], ptr [[PTR0:%.*]], align 1
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult <2 x i32> [[Y:%.*]], splat (i32 2)
-; CHECK-NEXT:    store <2 x i1> [[CMP1]], ptr [[PTR1:%.*]], align 1
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult <2 x i32> [[Z:%.*]], splat (i32 2)
+; CHECK-LABEL: define <2 x i1> @or1_eq1_multiple_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]], <2 x i32> [[Z:%.*]], ptr [[PTR0:%.*]], ptr [[PTR1:%.*]]) {
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp ult <2 x i32> [[X]], splat (i32 2)
+; CHECK-NEXT:    store <2 x i1> [[CMP0]], ptr [[PTR0]], align 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult <2 x i32> [[Y]], splat (i32 2)
+; CHECK-NEXT:    store <2 x i1> [[CMP1]], ptr [[PTR1]], align 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult <2 x i32> [[Z]], splat (i32 2)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP2]]
 ;
   %t0 = or <2 x i32> %x, splat (i32 1)
@@ -3151,8 +3413,9 @@ define <2 x i1> @or1_eq1_multiple_vec(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z,
 ; X | C == C --> X <=u C (when C+1 is PowerOf2).
 
 define <2 x i1> @or3_eq3_vec(<2 x i8> %x) {
-; CHECK-LABEL: @or3_eq3_vec(
-; CHECK-NEXT:    [[T1:%.*]] = icmp ult <2 x i8> [[X:%.*]], splat (i8 4)
+; CHECK-LABEL: define <2 x i1> @or3_eq3_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[T1:%.*]] = icmp ult <2 x i8> [[X]], splat (i8 4)
 ; CHECK-NEXT:    ret <2 x i1> [[T1]]
 ;
   %t0 = or <2 x i8> %x, <i8 3, i8 3>
@@ -3163,8 +3426,9 @@ define <2 x i1> @or3_eq3_vec(<2 x i8> %x) {
 ; X | C != C --> X >u C (when C+1 is PowerOf2).
 
 define i1 @or7_ne7(i32 %x) {
-; CHECK-LABEL: @or7_ne7(
-; CHECK-NEXT:    [[T1:%.*]] = icmp ugt i32 [[X:%.*]], 7
+; CHECK-LABEL: define i1 @or7_ne7(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[T1:%.*]] = icmp ugt i32 [[X]], 7
 ; CHECK-NEXT:    ret i1 [[T1]]
 ;
   %t0 = or i32 %x, 7
@@ -3175,8 +3439,9 @@ define i1 @or7_ne7(i32 %x) {
 ; X | C != C --> X >u C (when C+1 is PowerOf2).
 
 define <2 x i1> @or63_ne63_vec(<2 x i8> %x) {
-; CHECK-LABEL: @or63_ne63_vec(
-; CHECK-NEXT:    [[T1:%.*]] = icmp ugt <2 x i8> [[X:%.*]], splat (i8 63)
+; CHECK-LABEL: define <2 x i1> @or63_ne63_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[T1:%.*]] = icmp ugt <2 x i8> [[X]], splat (i8 63)
 ; CHECK-NEXT:    ret <2 x i1> [[T1]]
 ;
   %t0 = or <2 x i8> %x, <i8 63, i8 63>
@@ -3188,8 +3453,9 @@ define <2 x i1> @or63_ne63_vec(<2 x i8> %x) {
 ; X | C == C --> (X & ~C) == 0
 
 define i1 @orC_eqC(i32 %x) {
-; CHECK-LABEL: @orC_eqC(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -43
+; CHECK-LABEL: define i1 @orC_eqC(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], -43
 ; CHECK-NEXT:    [[T1:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[T1]]
 ;
@@ -3201,8 +3467,9 @@ define i1 @orC_eqC(i32 %x) {
 ; X | C == C --> (X & ~C) == 0
 
 define <2 x i1> @orC_eqC_vec(<2 x i8> %x) {
-; CHECK-LABEL: @orC_eqC_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], splat (i8 -44)
+; CHECK-LABEL: define <2 x i1> @orC_eqC_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X]], splat (i8 -44)
 ; CHECK-NEXT:    [[T1:%.*]] = icmp eq <2 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[T1]]
 ;
@@ -3214,8 +3481,9 @@ define <2 x i1> @orC_eqC_vec(<2 x i8> %x) {
 ; X | C != C --> (X & ~C) != 0
 
 define i1 @orC_neC(i32 %x) {
-; CHECK-LABEL: @orC_neC(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 41
+; CHECK-LABEL: define i1 @orC_neC(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], 41
 ; CHECK-NEXT:    [[T1:%.*]] = icmp ne i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[T1]]
 ;
@@ -3227,8 +3495,9 @@ define i1 @orC_neC(i32 %x) {
 ; X | C != C --> (X & ~C) != 0
 
 define <2 x i1> @orC_neC_vec(<2 x i8> %x) {
-; CHECK-LABEL: @orC_neC_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], splat (i8 42)
+; CHECK-LABEL: define <2 x i1> @orC_neC_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X]], splat (i8 42)
 ; CHECK-NEXT:    [[T1:%.*]] = icmp ne <2 x i8> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[T1]]
 ;
@@ -3238,8 +3507,9 @@ define <2 x i1> @orC_neC_vec(<2 x i8> %x) {
 }
 
 define i1 @shrink_constant(i32 %X) {
-; CHECK-LABEL: @shrink_constant(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], -12
+; CHECK-LABEL: define i1 @shrink_constant(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], -12
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[XOR]], 4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -3249,8 +3519,9 @@ define i1 @shrink_constant(i32 %X) {
 }
 
 define <2 x i1> @shrink_constant_vec(<2 x i32> %X) {
-; CHECK-LABEL: @shrink_constant_vec(
-; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[X:%.*]], splat (i32 -12)
+; CHECK-LABEL: define <2 x i1> @shrink_constant_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[X]], splat (i32 -12)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[XOR]], splat (i32 4)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -3261,8 +3532,9 @@ define <2 x i1> @shrink_constant_vec(<2 x i32> %X) {
 
 ; This test requires 3 different transforms to get to the result.
 define i1 @icmp_sub_-1_X_ult_4(i32 %X) {
-; CHECK-LABEL: @icmp_sub_-1_X_ult_4(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], -5
+; CHECK-LABEL: define i1 @icmp_sub_-1_X_ult_4(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X]], -5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = sub i32 -1, %X
@@ -3271,8 +3543,9 @@ define i1 @icmp_sub_-1_X_ult_4(i32 %X) {
 }
 
 define <2 x i1> @icmp_xor_neg4_X_ult_4_vec(<2 x i32> %X) {
-; CHECK-LABEL: @icmp_xor_neg4_X_ult_4_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], splat (i32 -5)
+; CHECK-LABEL: define <2 x i1> @icmp_xor_neg4_X_ult_4_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X]], splat (i32 -5)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %xor = xor <2 x i32> %X, <i32 -4, i32 -4>
@@ -3281,8 +3554,9 @@ define <2 x i1> @icmp_xor_neg4_X_ult_4_vec(<2 x i32> %X) {
 }
 
 define i1 @icmp_sub_-1_X_uge_4(i32 %X) {
-; CHECK-LABEL: @icmp_sub_-1_X_uge_4(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], -4
+; CHECK-LABEL: define i1 @icmp_sub_-1_X_uge_4(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], -4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = sub i32 -1, %X
@@ -3291,8 +3565,9 @@ define i1 @icmp_sub_-1_X_uge_4(i32 %X) {
 }
 
 define <2 x i1> @icmp_xor_neg4_X_uge_4_vec(<2 x i32> %X) {
-; CHECK-LABEL: @icmp_xor_neg4_X_uge_4_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X:%.*]], splat (i32 -4)
+; CHECK-LABEL: define <2 x i1> @icmp_xor_neg4_X_uge_4_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X]], splat (i32 -4)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %xor = xor <2 x i32> %X, <i32 -4, i32 -4>
@@ -3301,8 +3576,9 @@ define <2 x i1> @icmp_xor_neg4_X_uge_4_vec(<2 x i32> %X) {
 }
 
 define <2 x i1> @xor_ult(<2 x i8> %x) {
-; CHECK-LABEL: @xor_ult(
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[X:%.*]], splat (i8 3)
+; CHECK-LABEL: define <2 x i1> @xor_ult(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[X]], splat (i8 3)
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
   %xor = xor <2 x i8> %x, <i8 -4, i8 -4>
@@ -3311,9 +3587,10 @@ define <2 x i1> @xor_ult(<2 x i8> %x) {
 }
 
 define i1 @xor_ult_extra_use(i8 %x, ptr %p) {
-; CHECK-LABEL: @xor_ult_extra_use(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[X:%.*]], -32
-; CHECK-NEXT:    store i8 [[XOR]], ptr [[P:%.*]], align 1
+; CHECK-LABEL: define i1 @xor_ult_extra_use(
+; CHECK-SAME: i8 [[X:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[X]], -32
+; CHECK-NEXT:    store i8 [[XOR]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], 31
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -3324,8 +3601,9 @@ define i1 @xor_ult_extra_use(i8 %x, ptr %p) {
 }
 
 define <2 x i1> @xor_ugt(<2 x i8> %x) {
-; CHECK-LABEL: @xor_ugt(
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[X:%.*]], splat (i8 7)
+; CHECK-LABEL: define <2 x i1> @xor_ugt(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = icmp ugt <2 x i8> [[X]], splat (i8 7)
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
   %xor = xor <2 x i8> %x, <i8 7, i8 7>
@@ -3334,9 +3612,10 @@ define <2 x i1> @xor_ugt(<2 x i8> %x) {
 }
 
 define i1 @xor_ugt_extra_use(i8 %x, ptr %p) {
-; CHECK-LABEL: @xor_ugt_extra_use(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[X:%.*]], 63
-; CHECK-NEXT:    store i8 [[XOR]], ptr [[P:%.*]], align 1
+; CHECK-LABEL: define i1 @xor_ugt_extra_use(
+; CHECK-SAME: i8 [[X:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i8 [[X]], 63
+; CHECK-NEXT:    store i8 [[XOR]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[X]], 63
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -3347,8 +3626,9 @@ define i1 @xor_ugt_extra_use(i8 %x, ptr %p) {
 }
 
 define i1 @icmp_lshr_lshr_eq(i32 %a, i32 %b) {
-; CHECK-LABEL: @icmp_lshr_lshr_eq(
-; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i1 @icmp_lshr_lshr_eq(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 [[A]], [[B]]
 ; CHECK-NEXT:    [[Z:%.*]] = icmp ult i32 [[Z_UNSHIFTED]], 1073741824
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
@@ -3359,8 +3639,9 @@ define i1 @icmp_lshr_lshr_eq(i32 %a, i32 %b) {
 }
 
 define i1 @icmp_ashr_ashr_ne(i32 %a, i32 %b) {
-; CHECK-LABEL: @icmp_ashr_ashr_ne(
-; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i1 @icmp_ashr_ashr_ne(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 [[A]], [[B]]
 ; CHECK-NEXT:    [[Z:%.*]] = icmp ugt i32 [[Z_UNSHIFTED]], 255
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
@@ -3371,8 +3652,9 @@ define i1 @icmp_ashr_ashr_ne(i32 %a, i32 %b) {
 }
 
 define i1 @icmp_neg_cst_slt(i32 %a) {
-; CHECK-LABEL: @icmp_neg_cst_slt(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], 10
+; CHECK-LABEL: define i1 @icmp_neg_cst_slt(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A]], 10
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %1 = sub nsw i32 0, %a
@@ -3381,10 +3663,11 @@ define i1 @icmp_neg_cst_slt(i32 %a) {
 }
 
 define i1 @icmp_and_or_lshr(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_and_or_lshr(
-; CHECK-NEXT:    [[SHF1:%.*]] = shl nuw i32 1, [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_and_or_lshr(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[SHF1:%.*]] = shl nuw i32 1, [[Y]]
 ; CHECK-NEXT:    [[OR2:%.*]] = or i32 [[SHF1]], 1
-; CHECK-NEXT:    [[AND3:%.*]] = and i32 [[X:%.*]], [[OR2]]
+; CHECK-NEXT:    [[AND3:%.*]] = and i32 [[X]], [[OR2]]
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne i32 [[AND3]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
@@ -3396,10 +3679,11 @@ define i1 @icmp_and_or_lshr(i32 %x, i32 %y) {
 }
 
 define i1 @icmp_and_or_lshr_samesign(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_and_or_lshr_samesign(
-; CHECK-NEXT:    [[SHF1:%.*]] = shl nuw i32 1, [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_and_or_lshr_samesign(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[SHF1:%.*]] = shl nuw i32 1, [[Y]]
 ; CHECK-NEXT:    [[OR2:%.*]] = or i32 [[SHF1]], 1
-; CHECK-NEXT:    [[AND3:%.*]] = and i32 [[X:%.*]], [[OR2]]
+; CHECK-NEXT:    [[AND3:%.*]] = and i32 [[X]], [[OR2]]
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne i32 [[AND3]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
@@ -3411,8 +3695,9 @@ define i1 @icmp_and_or_lshr_samesign(i32 %x, i32 %y) {
 }
 
 define <2 x i1> @icmp_and_or_lshr_vec(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_and_or_lshr_vec(
-; CHECK-NEXT:    [[SHF:%.*]] = lshr <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define <2 x i1> @icmp_and_or_lshr_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[SHF:%.*]] = lshr <2 x i32> [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[SHF]], [[X]]
 ; CHECK-NEXT:    [[RET:%.*]] = trunc <2 x i32> [[OR]] to <2 x i1>
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
@@ -3425,9 +3710,10 @@ define <2 x i1> @icmp_and_or_lshr_vec(<2 x i32> %x, <2 x i32> %y) {
 }
 
 define <2 x i1> @icmp_and_or_lshr_vec_commute(<2 x i32> %xp, <2 x i32> %y) {
-; CHECK-LABEL: @icmp_and_or_lshr_vec_commute(
-; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP:%.*]], splat (i32 42)
-; CHECK-NEXT:    [[SHF:%.*]] = lshr <2 x i32> [[X]], [[Y:%.*]]
+; CHECK-LABEL: define <2 x i1> @icmp_and_or_lshr_vec_commute(
+; CHECK-SAME: <2 x i32> [[XP:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP]], splat (i32 42)
+; CHECK-NEXT:    [[SHF:%.*]] = lshr <2 x i32> [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[X]], [[SHF]]
 ; CHECK-NEXT:    [[RET:%.*]] = trunc <2 x i32> [[OR]] to <2 x i1>
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
@@ -3441,8 +3727,9 @@ define <2 x i1> @icmp_and_or_lshr_vec_commute(<2 x i32> %xp, <2 x i32> %y) {
 }
 
 define i1 @icmp_and_or_lshr_cst(i32 %x) {
-; CHECK-LABEL: @icmp_and_or_lshr_cst(
-; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @icmp_and_or_lshr_cst(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[X]], 3
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne i32 [[AND1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
@@ -3454,8 +3741,9 @@ define i1 @icmp_and_or_lshr_cst(i32 %x) {
 }
 
 define <2 x i1> @icmp_and_or_lshr_cst_vec(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_and_or_lshr_cst_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 3)
+; CHECK-LABEL: define <2 x i1> @icmp_and_or_lshr_cst_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], splat (i32 3)
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
 ;
@@ -3467,8 +3755,9 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec(<2 x i32> %x) {
 }
 
 define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_nonuniform(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 3, i32 5>
+; CHECK-LABEL: define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], <i32 3, i32 5>
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
 ;
@@ -3480,8 +3769,9 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform(<2 x i32> %x) {
 }
 
 define <2 x i1> @icmp_and_or_lshr_cst_vec_poison(<2 x i32> %x) {
-; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 3, i32 poison>
+; CHECK-LABEL: define <2 x i1> @icmp_and_or_lshr_cst_vec_poison(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], <i32 3, i32 poison>
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
 ;
@@ -3493,8 +3783,9 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec_poison(<2 x i32> %x) {
 }
 
 define <2 x i1> @icmp_and_or_lshr_cst_vec_commute(<2 x i32> %xp) {
-; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_commute(
-; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP:%.*]], splat (i32 42)
+; CHECK-LABEL: define <2 x i1> @icmp_and_or_lshr_cst_vec_commute(
+; CHECK-SAME: <2 x i32> [[XP:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP]], splat (i32 42)
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], splat (i32 3)
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
@@ -3508,8 +3799,9 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec_commute(<2 x i32> %xp) {
 }
 
 define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform_commute(<2 x i32> %xp) {
-; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_nonuniform_commute(
-; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP:%.*]], splat (i32 42)
+; CHECK-LABEL: define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform_commute(
+; CHECK-SAME: <2 x i32> [[XP:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP]], splat (i32 42)
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], <i32 3, i32 5>
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
@@ -3523,8 +3815,9 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec_nonuniform_commute(<2 x i32> %xp) {
 }
 
 define <2 x i1> @icmp_and_or_lshr_cst_vec_poison_commute(<2 x i32> %xp) {
-; CHECK-LABEL: @icmp_and_or_lshr_cst_vec_poison_commute(
-; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP:%.*]], splat (i32 42)
+; CHECK-LABEL: define <2 x i1> @icmp_and_or_lshr_cst_vec_poison_commute(
+; CHECK-SAME: <2 x i32> [[XP:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = srem <2 x i32> [[XP]], splat (i32 42)
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], <i32 3, i32 poison>
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RET]]
@@ -3538,8 +3831,9 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec_poison_commute(<2 x i32> %xp) {
 }
 
 define i1 @shl_ap1_zero_ap2_non_zero_2(i32 %a) {
-; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_2(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 29
+; CHECK-LABEL: define i1 @shl_ap1_zero_ap2_non_zero_2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A]], 29
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 4, %a
@@ -3548,8 +3842,9 @@ define i1 @shl_ap1_zero_ap2_non_zero_2(i32 %a) {
 }
 
 define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec(<2 x i32> %a) {
-; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_2_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[A:%.*]], splat (i32 29)
+; CHECK-LABEL: define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec(
+; CHECK-SAME: <2 x i32> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[A]], splat (i32 29)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 4, i32 4>, %a
@@ -3558,8 +3853,9 @@ define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec(<2 x i32> %a) {
 }
 
 define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec_nonuniform(<2 x i32> %a) {
-; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_2_vec_nonuniform(
-; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i32> <i32 4, i32 5>, [[A:%.*]]
+; CHECK-LABEL: define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec_nonuniform(
+; CHECK-SAME: <2 x i32> [[A:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i32> <i32 4, i32 5>, [[A]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[SHL]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -3569,8 +3865,9 @@ define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec_nonuniform(<2 x i32> %a) {
 }
 
 define i1 @shl_ap1_zero_ap2_non_zero_4(i32 %a) {
-; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_4(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 30
+; CHECK-LABEL: define i1 @shl_ap1_zero_ap2_non_zero_4(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A]], 30
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 -2, %a
@@ -3579,8 +3876,9 @@ define i1 @shl_ap1_zero_ap2_non_zero_4(i32 %a) {
 }
 
 define i1 @shl_ap1_non_zero_ap2_non_zero_both_positive(i32 %a) {
-; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_both_positive(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-LABEL: define i1 @shl_ap1_non_zero_ap2_non_zero_both_positive(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 50, %a
@@ -3589,8 +3887,9 @@ define i1 @shl_ap1_non_zero_ap2_non_zero_both_positive(i32 %a) {
 }
 
 define i1 @shl_ap1_non_zero_ap2_non_zero_both_negative(i32 %a) {
-; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_both_negative(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
+; CHECK-LABEL: define i1 @shl_ap1_non_zero_ap2_non_zero_both_negative(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 -50, %a
@@ -3599,7 +3898,8 @@ define i1 @shl_ap1_non_zero_ap2_non_zero_both_negative(i32 %a) {
 }
 
 define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_1(i32 %a) {
-; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_ap1_1(
+; CHECK-LABEL: define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_1(
+; CHECK-SAME: i32 [[A:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %shl = shl i32 50, %a
@@ -3608,8 +3908,9 @@ define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_1(i32 %a) {
 }
 
 define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_2(i32 %a) {
-; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_ap1_2(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 1
+; CHECK-LABEL: define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_2(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 25, %a
@@ -3618,7 +3919,8 @@ define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_2(i32 %a) {
 }
 
 define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_3(i32 %a) {
-; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_ap1_3(
+; CHECK-LABEL: define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_3(
+; CHECK-SAME: i32 [[A:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %shl = shl i32 26, %a
@@ -3627,8 +3929,9 @@ define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_3(i32 %a) {
 }
 
 define i1 @icmp_sgt_zero_add_nsw(i32 %a) {
-; CHECK-LABEL: @icmp_sgt_zero_add_nsw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], -1
+; CHECK-LABEL: define i1 @icmp_sgt_zero_add_nsw(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %a, 1
@@ -3637,8 +3940,9 @@ define i1 @icmp_sgt_zero_add_nsw(i32 %a) {
 }
 
 define i1 @icmp_sge_zero_add_nsw(i32 %a) {
-; CHECK-LABEL: @icmp_sge_zero_add_nsw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], -2
+; CHECK-LABEL: define i1 @icmp_sge_zero_add_nsw(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A]], -2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %a, 1
@@ -3647,8 +3951,9 @@ define i1 @icmp_sge_zero_add_nsw(i32 %a) {
 }
 
 define i1 @icmp_sle_zero_add_nsw(i32 %a) {
-; CHECK-LABEL: @icmp_sle_zero_add_nsw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A:%.*]], 0
+; CHECK-LABEL: define i1 @icmp_sle_zero_add_nsw(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %a, 1
@@ -3657,8 +3962,9 @@ define i1 @icmp_sle_zero_add_nsw(i32 %a) {
 }
 
 define zeroext i1 @icmp_cmpxchg_strong(ptr %sc, i32 %old_val, i32 %new_val) {
-; CHECK-LABEL: @icmp_cmpxchg_strong(
-; CHECK-NEXT:    [[XCHG:%.*]] = cmpxchg ptr [[SC:%.*]], i32 [[OLD_VAL:%.*]], i32 [[NEW_VAL:%.*]] seq_cst seq_cst, align 4
+; CHECK-LABEL: define zeroext i1 @icmp_cmpxchg_strong(
+; CHECK-SAME: ptr [[SC:%.*]], i32 [[OLD_VAL:%.*]], i32 [[NEW_VAL:%.*]]) {
+; CHECK-NEXT:    [[XCHG:%.*]] = cmpxchg ptr [[SC]], i32 [[OLD_VAL]], i32 [[NEW_VAL]] seq_cst seq_cst, align 4
 ; CHECK-NEXT:    [[ICMP:%.*]] = extractvalue { i32, i1 } [[XCHG]], 1
 ; CHECK-NEXT:    ret i1 [[ICMP]]
 ;
@@ -3669,8 +3975,9 @@ define zeroext i1 @icmp_cmpxchg_strong(ptr %sc, i32 %old_val, i32 %new_val) {
 }
 
 define i1 @f1(i64 %a, i64 %b) {
-; CHECK-LABEL: @f1(
-; CHECK-NEXT:    [[V:%.*]] = icmp sge i64 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i1 @f1(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp sge i64 [[A]], [[B]]
 ; CHECK-NEXT:    ret i1 [[V]]
 ;
   %t = sub nsw i64 %a, %b
@@ -3679,8 +3986,9 @@ define i1 @f1(i64 %a, i64 %b) {
 }
 
 define <2 x i1> @f1_vec(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @f1_vec(
-; CHECK-NEXT:    [[V:%.*]] = icmp sge <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define <2 x i1> @f1_vec(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp sge <2 x i64> [[A]], [[B]]
 ; CHECK-NEXT:    ret <2 x i1> [[V]]
 ;
   %t = sub nsw <2 x i64> %a, %b
@@ -3689,8 +3997,9 @@ define <2 x i1> @f1_vec(<2 x i64> %a, <2 x i64> %b) {
 }
 
 define i1 @f2(i64 %a, i64 %b) {
-; CHECK-LABEL: @f2(
-; CHECK-NEXT:    [[V:%.*]] = icmp sgt i64 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i1 @f2(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp sgt i64 [[A]], [[B]]
 ; CHECK-NEXT:    ret i1 [[V]]
 ;
   %t = sub nsw i64 %a, %b
@@ -3699,8 +4008,9 @@ define i1 @f2(i64 %a, i64 %b) {
 }
 
 define <2 x i1> @f2_vec(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @f2_vec(
-; CHECK-NEXT:    [[V:%.*]] = icmp sgt <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define <2 x i1> @f2_vec(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp sgt <2 x i64> [[A]], [[B]]
 ; CHECK-NEXT:    ret <2 x i1> [[V]]
 ;
   %t = sub nsw <2 x i64> %a, %b
@@ -3709,8 +4019,9 @@ define <2 x i1> @f2_vec(<2 x i64> %a, <2 x i64> %b) {
 }
 
 define i1 @f3(i64 %a, i64 %b) {
-; CHECK-LABEL: @f3(
-; CHECK-NEXT:    [[V:%.*]] = icmp slt i64 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i1 @f3(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp slt i64 [[A]], [[B]]
 ; CHECK-NEXT:    ret i1 [[V]]
 ;
   %t = sub nsw i64 %a, %b
@@ -3719,8 +4030,9 @@ define i1 @f3(i64 %a, i64 %b) {
 }
 
 define <2 x i1> @f3_vec(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @f3_vec(
-; CHECK-NEXT:    [[V:%.*]] = icmp slt <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define <2 x i1> @f3_vec(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp slt <2 x i64> [[A]], [[B]]
 ; CHECK-NEXT:    ret <2 x i1> [[V]]
 ;
   %t = sub nsw <2 x i64> %a, %b
@@ -3729,8 +4041,9 @@ define <2 x i1> @f3_vec(<2 x i64> %a, <2 x i64> %b) {
 }
 
 define i1 @f4(i64 %a, i64 %b) {
-; CHECK-LABEL: @f4(
-; CHECK-NEXT:    [[V:%.*]] = icmp sle i64 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i1 @f4(
+; CHECK-SAME: i64 [[A:%.*]], i64 [[B:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp sle i64 [[A]], [[B]]
 ; CHECK-NEXT:    ret i1 [[V]]
 ;
   %t = sub nsw i64 %a, %b
@@ -3739,8 +4052,9 @@ define i1 @f4(i64 %a, i64 %b) {
 }
 
 define <2 x i1> @f4_vec(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: @f4_vec(
-; CHECK-NEXT:    [[V:%.*]] = icmp sle <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define <2 x i1> @f4_vec(
+; CHECK-SAME: <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp sle <2 x i64> [[A]], [[B]]
 ; CHECK-NEXT:    ret <2 x i1> [[V]]
 ;
   %t = sub nsw <2 x i64> %a, %b
@@ -3749,9 +4063,10 @@ define <2 x i1> @f4_vec(<2 x i64> %a, <2 x i64> %b) {
 }
 
 define i32 @f5(i8 %a, i8 %b) {
-; CHECK-LABEL: @f5(
-; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[A:%.*]] to i32
-; CHECK-NEXT:    [[CONV3:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-LABEL: define i32 @f5(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[A]] to i32
+; CHECK-NEXT:    [[CONV3:%.*]] = zext i8 [[B]] to i32
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[CONV]], [[CONV3]]
 ; CHECK-NEXT:    [[SUB7_SUB:%.*]] = call i32 @llvm.abs.i32(i32 [[SUB]], i1 true)
 ; CHECK-NEXT:    ret i32 [[SUB7_SUB]]
@@ -3766,8 +4081,9 @@ define i32 @f5(i8 %a, i8 %b) {
 }
 
 define i32 @f6(i32 %a, i32 %b) {
-; CHECK-LABEL: @f6(
-; CHECK-NEXT:    [[CMP_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i32 @f6(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP_UNSHIFTED:%.*]] = xor i32 [[A]], [[B]]
 ; CHECK-NEXT:    [[CMP_MASK:%.*]] = and i32 [[CMP_UNSHIFTED]], 255
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CMP_MASK]], 0
 ; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i32 10000, i32 0
@@ -3783,8 +4099,9 @@ define i32 @f6(i32 %a, i32 %b) {
 }
 
 define i32 @f7(i32 %a, i32 %b) {
-; CHECK-LABEL: @f7(
-; CHECK-NEXT:    [[CMP_NOT_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define i32 @f7(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[CMP_NOT_UNSHIFTED:%.*]] = xor i32 [[A]], [[B]]
 ; CHECK-NEXT:    [[CMP_NOT_MASK:%.*]] = and i32 [[CMP_NOT_UNSHIFTED]], 511
 ; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[CMP_NOT_MASK]], 0
 ; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP_NOT]], i32 0, i32 10000
@@ -3798,8 +4115,9 @@ define i32 @f7(i32 %a, i32 %b) {
 }
 
 define i1 @f8(i32 %val, i32 %lim) {
-; CHECK-LABEL: @f8(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[LIM:%.*]], 0
+; CHECK-LABEL: define i1 @f8(
+; CHECK-SAME: i32 [[VAL:%.*]], i32 [[LIM:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[LIM]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %lim.sub = add i32 %lim, -1
@@ -3809,8 +4127,9 @@ define i1 @f8(i32 %val, i32 %lim) {
 }
 
 define i1 @f9(i32 %val, i32 %lim) {
-; CHECK-LABEL: @f9(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[LIM:%.*]], 0
+; CHECK-LABEL: define i1 @f9(
+; CHECK-SAME: i32 [[VAL:%.*]], i32 [[LIM:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[LIM]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %lim.sub = sub i32 %lim, 1
@@ -3820,11 +4139,12 @@ define i1 @f9(i32 %val, i32 %lim) {
 }
 
 define i1 @f10(i16 %p) {
-; CHECK-LABEL: @f10(
+; CHECK-LABEL: define i1 @f10(
+; CHECK-SAME: i16 [[P:%.*]]) {
 ; CHECK-NEXT:    [[EXT1:%.*]] = zext i8 ptrtoint (ptr @f10 to i8) to i16
 ; CHECK-NEXT:    [[EXT2:%.*]] = zext i8 ptrtoint (ptr @f10 to i8) to i16
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw i16 [[EXT1]], [[EXT2]]
-; CHECK-NEXT:    [[CMP580:%.*]] = icmp ule i16 [[MUL]], [[P:%.*]]
+; CHECK-NEXT:    [[CMP580:%.*]] = icmp ule i16 [[MUL]], [[P]]
 ; CHECK-NEXT:    ret i1 [[CMP580]]
 ;
   %ext1 = zext i8 ptrtoint (ptr @f10 to i8) to i16
@@ -3839,9 +4159,10 @@ define i1 @f10(i16 %p) {
 ; equivalent to one another.
 
 define i1 @cmp_sgt_rhs_dec(float %x, i32 %i) {
-; CHECK-LABEL: @cmp_sgt_rhs_dec(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[I:%.*]], [[CONV]]
+; CHECK-LABEL: define i1 @cmp_sgt_rhs_dec(
+; CHECK-SAME: float [[X:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[I]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -3851,9 +4172,10 @@ define i1 @cmp_sgt_rhs_dec(float %x, i32 %i) {
 }
 
 define i1 @cmp_sle_rhs_dec(float %x, i32 %i) {
-; CHECK-LABEL: @cmp_sle_rhs_dec(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I:%.*]], [[CONV]]
+; CHECK-LABEL: define i1 @cmp_sle_rhs_dec(
+; CHECK-SAME: float [[X:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[I]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -3863,9 +4185,10 @@ define i1 @cmp_sle_rhs_dec(float %x, i32 %i) {
 }
 
 define i1 @cmp_sge_rhs_inc(float %x, i32 %i) {
-; CHECK-LABEL: @cmp_sge_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I:%.*]], [[CONV]]
+; CHECK-LABEL: define i1 @cmp_sge_rhs_inc(
+; CHECK-SAME: float [[X:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -3875,9 +4198,10 @@ define i1 @cmp_sge_rhs_inc(float %x, i32 %i) {
 }
 
 define i1 @cmp_slt_rhs_inc(float %x, i32 %i) {
-; CHECK-LABEL: @cmp_slt_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[I:%.*]], [[CONV]]
+; CHECK-LABEL: define i1 @cmp_slt_rhs_inc(
+; CHECK-SAME: float [[X:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[I]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -3887,9 +4211,10 @@ define i1 @cmp_slt_rhs_inc(float %x, i32 %i) {
 }
 
 define i1 @PR26407(i32 %x, i32 %y) {
-; CHECK-LABEL: @PR26407(
-; CHECK-NEXT:    [[ADDX:%.*]] = add i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    [[ADDY:%.*]] = add i32 [[Y:%.*]], 2147483647
+; CHECK-LABEL: define i1 @PR26407(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[ADDX:%.*]] = add i32 [[X]], 2147483647
+; CHECK-NEXT:    [[ADDY:%.*]] = add i32 [[Y]], 2147483647
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[ADDX]], [[ADDY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -3900,8 +4225,9 @@ define i1 @PR26407(i32 %x, i32 %y) {
 }
 
 define i1 @cmp_inverse_mask_bits_set_eq(i32 %x) {
-; CHECK-LABEL: @cmp_inverse_mask_bits_set_eq(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -43
+; CHECK-LABEL: define i1 @cmp_inverse_mask_bits_set_eq(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], -43
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], -43
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -3911,8 +4237,9 @@ define i1 @cmp_inverse_mask_bits_set_eq(i32 %x) {
 }
 
 define <2 x i1> @cmp_inverse_mask_bits_set_eq_vec(<2 x i32> %x) {
-; CHECK-LABEL: @cmp_inverse_mask_bits_set_eq_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], splat (i32 -43)
+; CHECK-LABEL: define <2 x i1> @cmp_inverse_mask_bits_set_eq_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X]], splat (i32 -43)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], splat (i32 -43)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -3922,8 +4249,9 @@ define <2 x i1> @cmp_inverse_mask_bits_set_eq_vec(<2 x i32> %x) {
 }
 
 define i1 @cmp_inverse_mask_bits_set_ne(i32 %x) {
-; CHECK-LABEL: @cmp_inverse_mask_bits_set_ne(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -43
+; CHECK-LABEL: define i1 @cmp_inverse_mask_bits_set_ne(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], -43
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], -43
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -3935,8 +4263,9 @@ define i1 @cmp_inverse_mask_bits_set_ne(i32 %x) {
 ; When canonicalizing to 'gt/lt', make sure the constant is correct.
 
 define i1 @PR27792(i128 %a) {
-; CHECK-LABEL: @PR27792(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i128 [[A:%.*]], -1
+; CHECK-LABEL: define i1 @PR27792(
+; CHECK-SAME: i128 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i128 [[A]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %cmp = icmp sge i128 %a, 0
@@ -3944,8 +4273,9 @@ define i1 @PR27792(i128 %a) {
 }
 
 define i1 @PR27792_2(i128 %a) {
-; CHECK-LABEL: @PR27792_2(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i128 [[A:%.*]], 0
+; CHECK-LABEL: define i1 @PR27792_2(
+; CHECK-SAME: i128 [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i128 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %b = icmp uge i128 %a, 1
@@ -3953,8 +4283,9 @@ define i1 @PR27792_2(i128 %a) {
 }
 
 define i1 @ugtMaxSignedVal(i8 %a) {
-; CHECK-LABEL: @ugtMaxSignedVal(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A:%.*]], 0
+; CHECK-LABEL: define i1 @ugtMaxSignedVal(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %cmp = icmp ugt i8 %a, 127
@@ -3962,8 +4293,9 @@ define i1 @ugtMaxSignedVal(i8 %a) {
 }
 
 define <2 x i1> @ugtMaxSignedValVec(<2 x i8> %a) {
-; CHECK-LABEL: @ugtMaxSignedValVec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @ugtMaxSignedValVec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %cmp = icmp ugt <2 x i8> %a, <i8 127, i8 127>
@@ -3971,8 +4303,9 @@ define <2 x i1> @ugtMaxSignedValVec(<2 x i8> %a) {
 }
 
 define i1 @ugtKnownBits(i8 %a) {
-; CHECK-LABEL: @ugtKnownBits(
-; CHECK-NEXT:    [[B:%.*]] = and i8 [[A:%.*]], 17
+; CHECK-LABEL: define i1 @ugtKnownBits(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[A]], 17
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[B]], 17
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -3982,8 +4315,9 @@ define i1 @ugtKnownBits(i8 %a) {
 }
 
 define <2 x i1> @ugtKnownBitsVec(<2 x i8> %a) {
-; CHECK-LABEL: @ugtKnownBitsVec(
-; CHECK-NEXT:    [[B:%.*]] = and <2 x i8> [[A:%.*]], splat (i8 17)
+; CHECK-LABEL: define <2 x i1> @ugtKnownBitsVec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i8> [[A]], splat (i8 17)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[B]], splat (i8 17)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -3993,9 +4327,10 @@ define <2 x i1> @ugtKnownBitsVec(<2 x i8> %a) {
 }
 
 define i1 @or_ptrtoint_mismatch(ptr %p, ptr %q) {
-; CHECK-LABEL: @or_ptrtoint_mismatch(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[P:%.*]], null
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[Q:%.*]], null
+; CHECK-LABEL: define i1 @or_ptrtoint_mismatch(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq ptr [[P]], null
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq ptr [[Q]], null
 ; CHECK-NEXT:    [[B:%.*]] = and i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
@@ -4008,8 +4343,9 @@ define i1 @or_ptrtoint_mismatch(ptr %p, ptr %q) {
 }
 
 define i1 @icmp_add1_ugt(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_add1_ugt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_add1_ugt(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nuw i32 %x, 1
@@ -4018,8 +4354,9 @@ define i1 @icmp_add1_ugt(i32 %x, i32 %y) {
 }
 
 define i1 @icmp_add1_ule(i32 %x, i32 %y) {
-; CHECK-LABEL: @icmp_add1_ule(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @icmp_add1_ule(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nuw i32 %x, 1
@@ -4028,9 +4365,10 @@ define i1 @icmp_add1_ule(i32 %x, i32 %y) {
 }
 
 define i1 @cmp_uge_rhs_inc(float %x, i32 %i) {
-; CHECK-LABEL: @cmp_uge_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[I:%.*]], [[CONV]]
+; CHECK-LABEL: define i1 @cmp_uge_rhs_inc(
+; CHECK-SAME: float [[X:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[I]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -4040,9 +4378,10 @@ define i1 @cmp_uge_rhs_inc(float %x, i32 %i) {
 }
 
 define i1 @cmp_ult_rhs_inc(float %x, i32 %i) {
-; CHECK-LABEL: @cmp_ult_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[I:%.*]], [[CONV]]
+; CHECK-LABEL: define i1 @cmp_ult_rhs_inc(
+; CHECK-SAME: float [[X:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[I]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -4052,9 +4391,10 @@ define i1 @cmp_ult_rhs_inc(float %x, i32 %i) {
 }
 
 define i1 @cmp_sge_lhs_inc(i32 %x, i32 %y) {
-; CHECK-LABEL: @cmp_sge_lhs_inc(
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[INC]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @cmp_sge_lhs_inc(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[X]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[INC]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %inc = add nsw i32 %x, 1
@@ -4063,9 +4403,10 @@ define i1 @cmp_sge_lhs_inc(i32 %x, i32 %y) {
 }
 
 define i1 @cmp_uge_lhs_inc(i32 %x, i32 %y) {
-; CHECK-LABEL: @cmp_uge_lhs_inc(
-; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[INC]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @cmp_uge_lhs_inc(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[X]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[INC]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %inc = add nuw i32 %x, 1
@@ -4074,9 +4415,10 @@ define i1 @cmp_uge_lhs_inc(i32 %x, i32 %y) {
 }
 
 define i1 @cmp_sgt_lhs_dec(i32 %x, i32 %y) {
-; CHECK-LABEL: @cmp_sgt_lhs_dec(
-; CHECK-NEXT:    [[DEC:%.*]] = add nsw i32 [[X:%.*]], -1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[DEC]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @cmp_sgt_lhs_dec(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[DEC:%.*]] = add nsw i32 [[X]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[DEC]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %dec = sub nsw i32 %x, 1
@@ -4085,9 +4427,10 @@ define i1 @cmp_sgt_lhs_dec(i32 %x, i32 %y) {
 }
 
 define i1 @cmp_ugt_lhs_dec(i32 %x, i32 %y) {
-; CHECK-LABEL: @cmp_ugt_lhs_dec(
-; CHECK-NEXT:    [[DEC:%.*]] = add i32 [[X:%.*]], -1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[DEC]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @cmp_ugt_lhs_dec(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[DEC:%.*]] = add i32 [[X]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[DEC]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %dec = sub nuw i32 %x, 1
@@ -4096,9 +4439,10 @@ define i1 @cmp_ugt_lhs_dec(i32 %x, i32 %y) {
 }
 
 define i1 @cmp_sle_rhs_inc(float %x, i32 %y) {
-; CHECK-LABEL: @cmp_sle_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[Y:%.*]], 1
+; CHECK-LABEL: define i1 @cmp_sle_rhs_inc(
+; CHECK-SAME: float [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[Y]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[INC]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -4109,9 +4453,10 @@ define i1 @cmp_sle_rhs_inc(float %x, i32 %y) {
 }
 
 define i1 @cmp_ule_rhs_inc(float %x, i32 %y) {
-; CHECK-LABEL: @cmp_ule_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[Y:%.*]], 1
+; CHECK-LABEL: define i1 @cmp_ule_rhs_inc(
+; CHECK-SAME: float [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[Y]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[INC]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -4122,9 +4467,10 @@ define i1 @cmp_ule_rhs_inc(float %x, i32 %y) {
 }
 
 define i1 @cmp_slt_rhs_dec(float %x, i32 %y) {
-; CHECK-LABEL: @cmp_slt_rhs_dec(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[DEC:%.*]] = add nsw i32 [[Y:%.*]], -1
+; CHECK-LABEL: define i1 @cmp_slt_rhs_dec(
+; CHECK-SAME: float [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[DEC:%.*]] = add nsw i32 [[Y]], -1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[DEC]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -4135,9 +4481,10 @@ define i1 @cmp_slt_rhs_dec(float %x, i32 %y) {
 }
 
 define i1 @cmp_ult_rhs_dec(float %x, i32 %y) {
-; CHECK-LABEL: @cmp_ult_rhs_dec(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
-; CHECK-NEXT:    [[DEC:%.*]] = add i32 [[Y:%.*]], -1
+; CHECK-LABEL: define i1 @cmp_ult_rhs_dec(
+; CHECK-SAME: float [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X]] to i32
+; CHECK-NEXT:    [[DEC:%.*]] = add i32 [[Y]], -1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[DEC]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -4148,8 +4495,9 @@ define i1 @cmp_ult_rhs_dec(float %x, i32 %y) {
 }
 
 define i1 @eq_add_constants(i32 %x, i32 %y) {
-; CHECK-LABEL: @eq_add_constants(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @eq_add_constants(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = add i32 %x, 5
@@ -4161,8 +4509,9 @@ define i1 @eq_add_constants(i32 %x, i32 %y) {
 declare i32 @llvm.bswap.i32(i32)
 
 define i1 @bswap_ne(i32 %x, i32 %y) {
-; CHECK-LABEL: @bswap_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @bswap_ne(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %swapx = call i32 @llvm.bswap.i32(i32 %x)
@@ -4174,8 +4523,9 @@ define i1 @bswap_ne(i32 %x, i32 %y) {
 declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
 
 define <8 x i1> @bswap_vec_eq(<8 x i16> %x, <8 x i16> %y) {
-; CHECK-LABEL: @bswap_vec_eq(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i16> [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define <8 x i1> @bswap_vec_eq(
+; CHECK-SAME: <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i16> [[X]], [[Y]]
 ; CHECK-NEXT:    ret <8 x i1> [[CMP]]
 ;
   %swapx = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %x)
@@ -4187,8 +4537,9 @@ define <8 x i1> @bswap_vec_eq(<8 x i16> %x, <8 x i16> %y) {
 declare i64 @llvm.bitreverse.i64(i64)
 
 define i1 @bitreverse_eq(i64 %x, i64 %y) {
-; CHECK-LABEL: @bitreverse_eq(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i1 @bitreverse_eq(
+; CHECK-SAME: i64 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %revx = call i64 @llvm.bitreverse.i64(i64 %x)
@@ -4200,8 +4551,9 @@ define i1 @bitreverse_eq(i64 %x, i64 %y) {
 declare <8 x i16> @llvm.bitreverse.v8i16(<8 x i16>)
 
 define <8 x i1> @bitreverse_vec_ne(<8 x i16> %x, <8 x i16> %y) {
-; CHECK-LABEL: @bitreverse_vec_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i16> [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define <8 x i1> @bitreverse_vec_ne(
+; CHECK-SAME: <8 x i16> [[X:%.*]], <8 x i16> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i16> [[X]], [[Y]]
 ; CHECK-NEXT:    ret <8 x i1> [[CMP]]
 ;
   %revx = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %x)
@@ -4213,9 +4565,10 @@ define <8 x i1> @bitreverse_vec_ne(<8 x i16> %x, <8 x i16> %y) {
 ; These perform a comparison of a value known to be between 4 and 5 with a value between 5 and 7.
 ; They should all simplify to equality compares.
 define i1 @knownbits1(i8 %a, i8 %b) {
-; CHECK-LABEL: @knownbits1(
-; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], 1
-; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-LABEL: define i1 @knownbits1(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A]], 1
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = or disjoint i8 [[B1]], 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A1]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -4229,9 +4582,10 @@ define i1 @knownbits1(i8 %a, i8 %b) {
 }
 
 define i1 @knownbits2(i8 %a, i8 %b) {
-; CHECK-LABEL: @knownbits2(
-; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], 1
-; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-LABEL: define i1 @knownbits2(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A]], 1
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = or disjoint i8 [[B1]], 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[A1]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -4245,9 +4599,10 @@ define i1 @knownbits2(i8 %a, i8 %b) {
 }
 
 define i1 @knownbits3(i8 %a, i8 %b) {
-; CHECK-LABEL: @knownbits3(
-; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], 1
-; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-LABEL: define i1 @knownbits3(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A]], 1
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = or disjoint i8 [[B1]], 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[TMP1]], [[A1]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -4261,9 +4616,10 @@ define i1 @knownbits3(i8 %a, i8 %b) {
 }
 
 define <2 x i1> @knownbits4(<2 x i8> %a, <2 x i8> %b) {
-; CHECK-LABEL: @knownbits4(
-; CHECK-NEXT:    [[A1:%.*]] = and <2 x i8> [[A:%.*]], splat (i8 1)
-; CHECK-NEXT:    [[B1:%.*]] = and <2 x i8> [[B:%.*]], splat (i8 2)
+; CHECK-LABEL: define <2 x i1> @knownbits4(
+; CHECK-SAME: <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = and <2 x i8> [[A]], splat (i8 1)
+; CHECK-NEXT:    [[B1:%.*]] = and <2 x i8> [[B]], splat (i8 2)
 ; CHECK-NEXT:    [[TMP1:%.*]] = or disjoint <2 x i8> [[B1]], splat (i8 1)
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i8> [[TMP1]], [[A1]]
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
@@ -4279,9 +4635,10 @@ define <2 x i1> @knownbits4(<2 x i8> %a, <2 x i8> %b) {
 ; These are the signed versions of the above. One value is less than or equal to 5, but maybe negative.
 ; The other is known to be a value 5-7. These should simplify to equality comparisons.
 define i1 @knownbits5(i8 %a, i8 %b) {
-; CHECK-LABEL: @knownbits5(
-; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], -127
-; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-LABEL: define i1 @knownbits5(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A]], -127
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = or disjoint i8 [[B1]], 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A1]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -4295,9 +4652,10 @@ define i1 @knownbits5(i8 %a, i8 %b) {
 }
 
 define i1 @knownbits6(i8 %a, i8 %b) {
-; CHECK-LABEL: @knownbits6(
-; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], -127
-; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-LABEL: define i1 @knownbits6(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A]], -127
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = or disjoint i8 [[B1]], 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[A1]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -4311,9 +4669,10 @@ define i1 @knownbits6(i8 %a, i8 %b) {
 }
 
 define <2 x i1> @knownbits7(<2 x i8> %a, <2 x i8> %b) {
-; CHECK-LABEL: @knownbits7(
-; CHECK-NEXT:    [[A1:%.*]] = and <2 x i8> [[A:%.*]], splat (i8 -127)
-; CHECK-NEXT:    [[B1:%.*]] = and <2 x i8> [[B:%.*]], splat (i8 2)
+; CHECK-LABEL: define <2 x i1> @knownbits7(
+; CHECK-SAME: <2 x i8> [[A:%.*]], <2 x i8> [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = and <2 x i8> [[A]], splat (i8 -127)
+; CHECK-NEXT:    [[B1:%.*]] = and <2 x i8> [[B]], splat (i8 2)
 ; CHECK-NEXT:    [[TMP1:%.*]] = or disjoint <2 x i8> [[B1]], splat (i8 1)
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i8> [[TMP1]], [[A1]]
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
@@ -4327,9 +4686,10 @@ define <2 x i1> @knownbits7(<2 x i8> %a, <2 x i8> %b) {
 }
 
 define i1 @knownbits8(i8 %a, i8 %b) {
-; CHECK-LABEL: @knownbits8(
-; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A:%.*]], -127
-; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B:%.*]], 2
+; CHECK-LABEL: define i1 @knownbits8(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = and i8 [[A]], -127
+; CHECK-NEXT:    [[B1:%.*]] = and i8 [[B]], 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = or disjoint i8 [[B1]], 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[TMP1]], [[A1]]
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -4344,8 +4704,9 @@ define i1 @knownbits8(i8 %a, i8 %b) {
 
 ; Make sure InstCombine doesn't try too hard to simplify the icmp and break the abs idiom
 define i32 @abs_preserve(i32 %x) {
-; CHECK-LABEL: @abs_preserve(
-; CHECK-NEXT:    [[A:%.*]] = shl nsw i32 [[X:%.*]], 1
+; CHECK-LABEL: define i32 @abs_preserve(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = shl nsw i32 [[X]], 1
 ; CHECK-NEXT:    [[ABS:%.*]] = call i32 @llvm.abs.i32(i32 [[A]], i1 false)
 ; CHECK-NEXT:    ret i32 [[ABS]]
 ;
@@ -4360,8 +4721,9 @@ define i32 @abs_preserve(i32 %x) {
 
 declare void @llvm.assume(i1)
 define i1 @PR35794(ptr %a) {
-; CHECK-LABEL: @PR35794(
-; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq ptr [[A:%.*]], null
+; CHECK-LABEL: define i1 @PR35794(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq ptr [[A]], null
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
 ; CHECK-NEXT:    ret i1 true
 ;
@@ -4373,8 +4735,9 @@ define i1 @PR35794(ptr %a) {
 
 ; Don't crash by assuming the compared values are integers.
 define <2 x i1> @PR36583(<2 x ptr>)  {
-; CHECK-LABEL: @PR36583(
-; CHECK-NEXT:    [[RES:%.*]] = icmp eq <2 x ptr> [[TMP0:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @PR36583(
+; CHECK-SAME: <2 x ptr> [[TMP0:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq <2 x ptr> [[TMP0]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RES]]
 ;
   %cast = ptrtoint <2 x ptr> %0 to <2 x i64>
@@ -4384,8 +4747,9 @@ define <2 x i1> @PR36583(<2 x ptr>)  {
 
 ; fold (icmp pred (sub (0, X)) C1) for vec type
 define <2 x i32> @Op1Negated_Vec(<2 x i32> %x) {
-; CHECK-LABEL: @Op1Negated_Vec(
-; CHECK-NEXT:    [[COND:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[X:%.*]], i1 true)
+; CHECK-LABEL: define <2 x i32> @Op1Negated_Vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call <2 x i32> @llvm.abs.v2i32(<2 x i32> [[X]], i1 true)
 ; CHECK-NEXT:    ret <2 x i32> [[COND]]
 ;
   %sub = sub nsw <2 x i32> zeroinitializer, %x
@@ -4395,8 +4759,9 @@ define <2 x i32> @Op1Negated_Vec(<2 x i32> %x) {
 }
 
 define i1 @signbit_bitcast_fpext(float %x) {
-; CHECK-LABEL: @signbit_bitcast_fpext(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-LABEL: define i1 @signbit_bitcast_fpext(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[R:%.*]] = icmp slt i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -4407,8 +4772,9 @@ define i1 @signbit_bitcast_fpext(float %x) {
 }
 
 define <2 x i1> @signbit_bitcast_fpext_vec(<2 x half> %x) {
-; CHECK-LABEL: @signbit_bitcast_fpext_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x half> [[X:%.*]] to <2 x i16>
+; CHECK-LABEL: define <2 x i1> @signbit_bitcast_fpext_vec(
+; CHECK-SAME: <2 x half> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x half> [[X]] to <2 x i16>
 ; CHECK-NEXT:    [[R:%.*]] = icmp slt <2 x i16> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
@@ -4419,8 +4785,9 @@ define <2 x i1> @signbit_bitcast_fpext_vec(<2 x half> %x) {
 }
 
 define i1 @signbit_bitcast_fptrunc(float %x) {
-; CHECK-LABEL: @signbit_bitcast_fptrunc(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-LABEL: define i1 @signbit_bitcast_fptrunc(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[TMP1]], -1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -4431,8 +4798,9 @@ define i1 @signbit_bitcast_fptrunc(float %x) {
 }
 
 define <2 x i1> @signbit_bitcast_fptrunc_vec(<2 x double> %x) {
-; CHECK-LABEL: @signbit_bitcast_fptrunc_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x double> [[X:%.*]] to <2 x i64>
+; CHECK-LABEL: define <2 x i1> @signbit_bitcast_fptrunc_vec(
+; CHECK-SAME: <2 x double> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x double> [[X]] to <2 x i64>
 ; CHECK-NEXT:    [[R:%.*]] = icmp sgt <2 x i64> [[TMP1]], splat (i64 -1)
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
@@ -4443,8 +4811,9 @@ define <2 x i1> @signbit_bitcast_fptrunc_vec(<2 x double> %x) {
 }
 
 define i1 @signbit_bitcast_fpext_wrong_cmp(float %x) {
-; CHECK-LABEL: @signbit_bitcast_fpext_wrong_cmp(
-; CHECK-NEXT:    [[F:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-LABEL: define i1 @signbit_bitcast_fpext_wrong_cmp(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[F:%.*]] = fpext float [[X]] to double
 ; CHECK-NEXT:    [[B:%.*]] = bitcast double [[F]] to i64
 ; CHECK-NEXT:    [[R:%.*]] = icmp slt i64 [[B]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -4456,8 +4825,9 @@ define i1 @signbit_bitcast_fpext_wrong_cmp(float %x) {
 }
 
 define <4 x i1> @signbit_bitcast_fpext_vec_wrong_bitcast(<2 x half> %x) {
-; CHECK-LABEL: @signbit_bitcast_fpext_vec_wrong_bitcast(
-; CHECK-NEXT:    [[F:%.*]] = fpext <2 x half> [[X:%.*]] to <2 x float>
+; CHECK-LABEL: define <4 x i1> @signbit_bitcast_fpext_vec_wrong_bitcast(
+; CHECK-SAME: <2 x half> [[X:%.*]]) {
+; CHECK-NEXT:    [[F:%.*]] = fpext <2 x half> [[X]] to <2 x float>
 ; CHECK-NEXT:    [[B:%.*]] = bitcast <2 x float> [[F]] to <4 x i16>
 ; CHECK-NEXT:    [[R:%.*]] = icmp sgt <4 x i16> [[B]], splat (i16 -1)
 ; CHECK-NEXT:    ret <4 x i1> [[R]]
@@ -4469,8 +4839,9 @@ define <4 x i1> @signbit_bitcast_fpext_vec_wrong_bitcast(<2 x half> %x) {
 }
 
 define i1 @signbit_bitcast_fpext_extra_use(float %x, ptr %p) {
-; CHECK-LABEL: @signbit_bitcast_fpext_extra_use(
-; CHECK-NEXT:    [[F:%.*]] = fpext float [[X:%.*]] to double
+; CHECK-LABEL: define i1 @signbit_bitcast_fpext_extra_use(
+; CHECK-SAME: float [[X:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[F:%.*]] = fpext float [[X]] to double
 ; CHECK-NEXT:    [[B:%.*]] = bitcast double [[F]] to i64
 ; CHECK-NEXT:    call void @use_i64(i64 [[B]])
 ; CHECK-NEXT:    [[R:%.*]] = icmp slt i64 [[B]], 0
@@ -4484,8 +4855,9 @@ define i1 @signbit_bitcast_fpext_extra_use(float %x, ptr %p) {
 }
 
 define i1 @signbit_bitcast_fpext_ppc_fp128(float %x) {
-; CHECK-LABEL: @signbit_bitcast_fpext_ppc_fp128(
-; CHECK-NEXT:    [[S2:%.*]] = fpext float [[X:%.*]] to ppc_fp128
+; CHECK-LABEL: define i1 @signbit_bitcast_fpext_ppc_fp128(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[S2:%.*]] = fpext float [[X]] to ppc_fp128
 ; CHECK-NEXT:    [[S3:%.*]] = bitcast ppc_fp128 [[S2]] to i128
 ; CHECK-NEXT:    [[S4:%.*]] = icmp slt i128 [[S3]], 0
 ; CHECK-NEXT:    ret i1 [[S4]]
@@ -4497,8 +4869,9 @@ define i1 @signbit_bitcast_fpext_ppc_fp128(float %x) {
 }
 
 define i1 @signbit_bitcast_fptrunc_ppc_fp128(ppc_fp128 %x) {
-; CHECK-LABEL: @signbit_bitcast_fptrunc_ppc_fp128(
-; CHECK-NEXT:    [[S2:%.*]] = fptrunc ppc_fp128 [[X:%.*]] to float
+; CHECK-LABEL: define i1 @signbit_bitcast_fptrunc_ppc_fp128(
+; CHECK-SAME: ppc_fp128 [[X:%.*]]) {
+; CHECK-NEXT:    [[S2:%.*]] = fptrunc ppc_fp128 [[X]] to float
 ; CHECK-NEXT:    [[S3:%.*]] = bitcast float [[S2]] to i32
 ; CHECK-NEXT:    [[S4:%.*]] = icmp slt i32 [[S3]], 0
 ; CHECK-NEXT:    ret i1 [[S4]]
@@ -4512,9 +4885,10 @@ define i1 @signbit_bitcast_fptrunc_ppc_fp128(ppc_fp128 %x) {
 @x = external dso_local local_unnamed_addr global i32, align 4
 @y = external dso_local local_unnamed_addr global i32, align 4
 define i1 @pr47997(i32 %arg) {
-; CHECK-LABEL: @pr47997(
-; CHECK-NEXT:  bb:
-; CHECK-NEXT:    [[I:%.*]] = add nsw i32 [[ARG:%.*]], -1
+; CHECK-LABEL: define i1 @pr47997(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[I:%.*]] = add nsw i32 [[ARG]], -1
 ; CHECK-NEXT:    store i32 [[I]], ptr @x, align 4
 ; CHECK-NEXT:    [[I1:%.*]] = sub nsw i32 1, [[ARG]]
 ; CHECK-NEXT:    store i32 [[I1]], ptr @y, align 4
@@ -4533,7 +4907,8 @@ bb:
 ; PR50944
 
 define i1 @thread_cmp_over_select_with_poison_trueval(i1 %b) {
-; CHECK-LABEL: @thread_cmp_over_select_with_poison_trueval(
+; CHECK-LABEL: define i1 @thread_cmp_over_select_with_poison_trueval(
+; CHECK-SAME: i1 [[B:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %s = select i1 %b, i32 poison, i32 0
@@ -4542,7 +4917,8 @@ define i1 @thread_cmp_over_select_with_poison_trueval(i1 %b) {
 }
 
 define i1 @thread_cmp_over_select_with_poison_falseval(i1 %b) {
-; CHECK-LABEL: @thread_cmp_over_select_with_poison_falseval(
+; CHECK-LABEL: define i1 @thread_cmp_over_select_with_poison_falseval(
+; CHECK-SAME: i1 [[B:%.*]]) {
 ; CHECK-NEXT:    ret i1 true
 ;
   %s = select i1 %b, i32 1, i32 poison
@@ -4551,8 +4927,9 @@ define i1 @thread_cmp_over_select_with_poison_falseval(i1 %b) {
 }
 
 define i1 @signbit_true_logic(i8 %x) {
-; CHECK-LABEL: @signbit_true_logic(
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[X:%.*]], 0
+; CHECK-LABEL: define i1 @signbit_true_logic(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[X]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %dec = add i8 %x, -1
@@ -4563,8 +4940,9 @@ define i1 @signbit_true_logic(i8 %x) {
 }
 
 define <2 x i1> @signbit_false_logic(<2 x i5> %x) {
-; CHECK-LABEL: @signbit_false_logic(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne <2 x i5> [[X:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @signbit_false_logic(
+; CHECK-SAME: <2 x i5> [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = icmp ne <2 x i5> [[X]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
   %dec = add <2 x i5> %x,  <i5 -1, i5 poison>
@@ -4577,8 +4955,9 @@ define <2 x i1> @signbit_false_logic(<2 x i5> %x) {
 ; Confirm that complexity canonicalization works for commuted pattern.
 
 define i1 @signbit_true_logic_uses_commute(i64 %x) {
-; CHECK-LABEL: @signbit_true_logic_uses_commute(
-; CHECK-NEXT:    [[DEC:%.*]] = add i64 [[X:%.*]], -1
+; CHECK-LABEL: define i1 @signbit_true_logic_uses_commute(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[DEC:%.*]] = add i64 [[X]], -1
 ; CHECK-NEXT:    call void @use_i64(i64 [[DEC]])
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i64 [[X]], -1
 ; CHECK-NEXT:    call void @use_i64(i64 [[NOT]])
@@ -4598,8 +4977,9 @@ define i1 @signbit_true_logic_uses_commute(i64 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ult_1_2(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ult_1_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], 4
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ult_1_2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], 4
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[TMP1]], 8
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4610,8 +4990,9 @@ define i1 @redundant_sign_bit_count_ult_1_2(i32 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ult_1_30(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ult_1_30(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], 1073741824
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ult_1_30(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], 1073741824
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[TMP1]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4622,8 +5003,9 @@ define i1 @redundant_sign_bit_count_ult_1_30(i32 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ult_31_2(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ult_31_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], 4
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ult_31_2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], 4
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[TMP1]], 8
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4634,8 +5016,9 @@ define i1 @redundant_sign_bit_count_ult_31_2(i32 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ult_31_30(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ult_31_30(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], 1073741824
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ult_31_30(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], 1073741824
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[TMP1]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4646,8 +5029,9 @@ define i1 @redundant_sign_bit_count_ult_31_30(i32 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ult_31_30_extra_use_ashr(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ult_31_30_extra_use_ashr(
-; CHECK-NEXT:    [[Y:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ult_31_30_extra_use_ashr(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[Y:%.*]] = ashr i32 [[X]], 31
 ; CHECK-NEXT:    call void @use_i32(i32 [[Y]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], 1073741824
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[TMP1]], -1
@@ -4661,8 +5045,9 @@ define i1 @redundant_sign_bit_count_ult_31_30_extra_use_ashr(i32 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ult_31_30_extra_use_xor(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ult_31_30_extra_use_xor(
-; CHECK-NEXT:    [[Y:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ult_31_30_extra_use_xor(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[Y:%.*]] = ashr i32 [[X]], 31
 ; CHECK-NEXT:    [[Z:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[Z]])
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[Z]], 1073741824
@@ -4676,9 +5061,10 @@ define i1 @redundant_sign_bit_count_ult_31_30_extra_use_xor(i32 %x) {
 }
 
 define i1 @not_redundant_sign_bit_count_ult(i32 %w, i32 %x) {
-; CHECK-LABEL: @not_redundant_sign_bit_count_ult(
-; CHECK-NEXT:    [[Y:%.*]] = ashr i32 [[X:%.*]], 31
-; CHECK-NEXT:    [[Z:%.*]] = xor i32 [[Y]], [[W:%.*]]
+; CHECK-LABEL: define i1 @not_redundant_sign_bit_count_ult(
+; CHECK-SAME: i32 [[W:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[Y:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT:    [[Z:%.*]] = xor i32 [[Y]], [[W]]
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[Z]], 1073741824
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4689,8 +5075,9 @@ define i1 @not_redundant_sign_bit_count_ult(i32 %w, i32 %x) {
 }
 
 define i1 @wrong_shift_opcode_i8(i8 %x) {
-; CHECK-LABEL: @wrong_shift_opcode_i8(
-; CHECK-NEXT:    [[Y:%.*]] = lshr i8 [[X:%.*]], 5
+; CHECK-LABEL: define i1 @wrong_shift_opcode_i8(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[Y:%.*]] = lshr i8 [[X]], 5
 ; CHECK-NEXT:    [[Z:%.*]] = xor i8 [[Y]], [[X]]
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[Z]], 2
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -4702,8 +5089,9 @@ define i1 @wrong_shift_opcode_i8(i8 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ult_31_30_commute(i32 %xsrc) {
-; CHECK-LABEL: @redundant_sign_bit_count_ult_31_30_commute(
-; CHECK-NEXT:    [[X:%.*]] = mul i32 [[XSRC:%.*]], 13
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ult_31_30_commute(
+; CHECK-SAME: i32 [[XSRC:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = mul i32 [[XSRC]], 13
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], 1073741824
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[TMP1]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -4716,8 +5104,9 @@ define i1 @redundant_sign_bit_count_ult_31_30_commute(i32 %xsrc) {
 }
 
 define i1 @redundant_sign_bit_count_i8(i8 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_i8(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X:%.*]], 2
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_i8(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[X]], 2
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 [[TMP1]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4728,8 +5117,9 @@ define i1 @redundant_sign_bit_count_i8(i8 %x) {
 }
 
 define <2 x i1> @redundant_sign_bit_count_ult_31_30_vector(<2 x i32> %xsrc) {
-; CHECK-LABEL: @redundant_sign_bit_count_ult_31_30_vector(
-; CHECK-NEXT:    [[X:%.*]] = mul <2 x i32> [[XSRC:%.*]], splat (i32 13)
+; CHECK-LABEL: define <2 x i1> @redundant_sign_bit_count_ult_31_30_vector(
+; CHECK-SAME: <2 x i32> [[XSRC:%.*]]) {
+; CHECK-NEXT:    [[X:%.*]] = mul <2 x i32> [[XSRC]], splat (i32 13)
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[X]], splat (i32 1073741824)
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt <2 x i32> [[TMP1]], splat (i32 -1)
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
@@ -4742,8 +5132,9 @@ define <2 x i1> @redundant_sign_bit_count_ult_31_30_vector(<2 x i32> %xsrc) {
 }
 
 define i1 @redundant_sign_bit_count_ugt_1_2(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ugt_1_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -4
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ugt_1_2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], -4
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[TMP1]], -8
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4754,8 +5145,9 @@ define i1 @redundant_sign_bit_count_ugt_1_2(i32 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ugt_1_30(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ugt_1_30(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], 1073741824
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ugt_1_30(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], 1073741824
 ; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4766,8 +5158,9 @@ define i1 @redundant_sign_bit_count_ugt_1_30(i32 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ugt_31_2(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ugt_31_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -4
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ugt_31_2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], -4
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[TMP1]], -8
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4778,8 +5171,9 @@ define i1 @redundant_sign_bit_count_ugt_31_2(i32 %x) {
 }
 
 define i1 @redundant_sign_bit_count_ugt_31_30(i32 %x) {
-; CHECK-LABEL: @redundant_sign_bit_count_ugt_31_30(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], 1073741824
+; CHECK-LABEL: define i1 @redundant_sign_bit_count_ugt_31_30(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X]], 1073741824
 ; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -4790,10 +5184,11 @@ define i1 @redundant_sign_bit_count_ugt_31_30(i32 %x) {
 }
 
 define i1 @zext_bool_and_eq0(i1 %x, i8 %y) {
-; CHECK-LABEL: @zext_bool_and_eq0(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[Y:%.*]], 1
+; CHECK-LABEL: define i1 @zext_bool_and_eq0(
+; CHECK-SAME: i1 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[Y]], 1
 ; CHECK-NEXT:    [[R1:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[NOT_X:%.*]] = xor i1 [[X:%.*]], true
+; CHECK-NEXT:    [[NOT_X:%.*]] = xor i1 [[X]], true
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[NOT_X]], i1 true, i1 [[R1]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -4804,10 +5199,11 @@ define i1 @zext_bool_and_eq0(i1 %x, i8 %y) {
 }
 
 define <2 x i1> @zext_bool_and_eq0_commute(<2 x i1> %x, <2 x i8> %p) {
-; CHECK-LABEL: @zext_bool_and_eq0_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[P:%.*]], splat (i8 1)
+; CHECK-LABEL: define <2 x i1> @zext_bool_and_eq0_commute(
+; CHECK-SAME: <2 x i1> [[X:%.*]], <2 x i8> [[P:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[P]], splat (i8 1)
 ; CHECK-NEXT:    [[R1:%.*]] = icmp eq <2 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[NOT_X:%.*]] = xor <2 x i1> [[X:%.*]], splat (i1 true)
+; CHECK-NEXT:    [[NOT_X:%.*]] = xor <2 x i1> [[X]], splat (i1 true)
 ; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[NOT_X]], <2 x i1> splat (i1 true), <2 x i1> [[R1]]
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
@@ -4819,10 +5215,11 @@ define <2 x i1> @zext_bool_and_eq0_commute(<2 x i1> %x, <2 x i8> %p) {
 }
 
 define i1 @zext_bool_and_ne0(i1 %x, i8 %y) {
-; CHECK-LABEL: @zext_bool_and_ne0(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[Y:%.*]], 1
+; CHECK-LABEL: define i1 @zext_bool_and_ne0(
+; CHECK-SAME: i1 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[Y]], 1
 ; CHECK-NEXT:    [[R1:%.*]] = icmp ne i8 [[TMP1]], 0
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[X:%.*]], i1 [[R1]], i1 false
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[X]], i1 [[R1]], i1 false
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %zx = zext i1 %x to i8
@@ -4832,10 +5229,11 @@ define i1 @zext_bool_and_ne0(i1 %x, i8 %y) {
 }
 
 define i1 @zext_bool_and_ne1(i1 %x, i8 %y) {
-; CHECK-LABEL: @zext_bool_and_ne1(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[Y:%.*]], 1
+; CHECK-LABEL: define i1 @zext_bool_and_ne1(
+; CHECK-SAME: i1 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[Y]], 1
 ; CHECK-NEXT:    [[R1:%.*]] = icmp eq i8 [[TMP1]], 0
-; CHECK-NEXT:    [[NOT_X:%.*]] = xor i1 [[X:%.*]], true
+; CHECK-NEXT:    [[NOT_X:%.*]] = xor i1 [[X]], true
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[NOT_X]], i1 true, i1 [[R1]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -4846,9 +5244,10 @@ define i1 @zext_bool_and_ne1(i1 %x, i8 %y) {
 }
 
 define <2 x i1> @zext_bool_and_eq1(<2 x i1> %x, <2 x i8> %y) {
-; CHECK-LABEL: @zext_bool_and_eq1(
-; CHECK-NEXT:    [[R1:%.*]] = trunc <2 x i8> [[Y:%.*]] to <2 x i1>
-; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[X:%.*]], <2 x i1> [[R1]], <2 x i1> zeroinitializer
+; CHECK-LABEL: define <2 x i1> @zext_bool_and_eq1(
+; CHECK-SAME: <2 x i1> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[R1:%.*]] = trunc <2 x i8> [[Y]] to <2 x i1>
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[X]], <2 x i1> [[R1]], <2 x i1> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
   %zx = zext <2 x i1> %x to <2 x i8>
@@ -4860,9 +5259,10 @@ define <2 x i1> @zext_bool_and_eq1(<2 x i1> %x, <2 x i8> %y) {
 ; negative test - wrong logic op
 
 define i1 @zext_bool_or_eq0(i1 %x, i8 %y) {
-; CHECK-LABEL: @zext_bool_or_eq0(
-; CHECK-NEXT:    [[ZX:%.*]] = zext i1 [[X:%.*]] to i8
-; CHECK-NEXT:    [[A:%.*]] = or i8 [[Y:%.*]], [[ZX]]
+; CHECK-LABEL: define i1 @zext_bool_or_eq0(
+; CHECK-SAME: i1 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[ZX:%.*]] = zext i1 [[X]] to i8
+; CHECK-NEXT:    [[A:%.*]] = or i8 [[Y]], [[ZX]]
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -4875,10 +5275,11 @@ define i1 @zext_bool_or_eq0(i1 %x, i8 %y) {
 ; negative test - extra use
 
 define i1 @zext_bool_and_eq0_use(i1 %x, i64 %y) {
-; CHECK-LABEL: @zext_bool_and_eq0_use(
-; CHECK-NEXT:    [[ZX:%.*]] = zext i1 [[X:%.*]] to i64
+; CHECK-LABEL: define i1 @zext_bool_and_eq0_use(
+; CHECK-SAME: i1 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[ZX:%.*]] = zext i1 [[X]] to i64
 ; CHECK-NEXT:    call void @use_i64(i64 [[ZX]])
-; CHECK-NEXT:    [[A:%.*]] = and i64 [[Y:%.*]], [[ZX]]
+; CHECK-NEXT:    [[A:%.*]] = and i64 [[Y]], [[ZX]]
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i64 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -4892,9 +5293,10 @@ define i1 @zext_bool_and_eq0_use(i1 %x, i64 %y) {
 ; negative test - extra use
 
 define i1 @zext_bool_and_ne0_use(i1 %x, i64 %y) {
-; CHECK-LABEL: @zext_bool_and_ne0_use(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[Y:%.*]], 1
-; CHECK-NEXT:    [[A:%.*]] = select i1 [[X:%.*]], i64 [[TMP1]], i64 0
+; CHECK-LABEL: define i1 @zext_bool_and_ne0_use(
+; CHECK-SAME: i1 [[X:%.*]], i64 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[Y]], 1
+; CHECK-NEXT:    [[A:%.*]] = select i1 [[X]], i64 [[TMP1]], i64 0
 ; CHECK-NEXT:    call void @use_i64(i64 [[A]])
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i64 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -4909,9 +5311,10 @@ define i1 @zext_bool_and_ne0_use(i1 %x, i64 %y) {
 ; negative test - must zext from i1
 
 define i1 @zext_notbool_and_ne0(i2 %x, i8 %y) {
-; CHECK-LABEL: @zext_notbool_and_ne0(
-; CHECK-NEXT:    [[ZX:%.*]] = zext i2 [[X:%.*]] to i8
-; CHECK-NEXT:    [[A:%.*]] = and i8 [[Y:%.*]], [[ZX]]
+; CHECK-LABEL: define i1 @zext_notbool_and_ne0(
+; CHECK-SAME: i2 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[ZX:%.*]] = zext i2 [[X]] to i8
+; CHECK-NEXT:    [[A:%.*]] = and i8 [[Y]], [[ZX]]
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -4924,8 +5327,9 @@ define i1 @zext_notbool_and_ne0(i2 %x, i8 %y) {
 ; fold icmp(X | OrC, C) --> icmp(X, 0)
 
 define i1 @or_positive_sgt_zero(i8 %a) {
-; CHECK-LABEL: @or_positive_sgt_zero(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A:%.*]], -1
+; CHECK-LABEL: define i1 @or_positive_sgt_zero(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %b = or i8 %a, 24
@@ -4934,8 +5338,9 @@ define i1 @or_positive_sgt_zero(i8 %a) {
 }
 
 define <2 x i1> @or_postive_sgt_zero_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_postive_sgt_zero_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[A:%.*]], splat (i8 -1)
+; CHECK-LABEL: define <2 x i1> @or_postive_sgt_zero_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[A]], splat (i8 -1)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
 
@@ -4945,7 +5350,8 @@ define <2 x i1> @or_postive_sgt_zero_vec(<2 x i8> %a) {
 }
 
 define <2 x i1> @or_poison_vec_sgt_zero_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_poison_vec_sgt_zero_vec(
+; CHECK-LABEL: define <2 x i1> @or_poison_vec_sgt_zero_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> poison
 ;
 
@@ -4955,8 +5361,9 @@ define <2 x i1> @or_poison_vec_sgt_zero_vec(<2 x i8> %a) {
 }
 
 define i1 @or_positive_sge_zero(i8 %a) {
-; CHECK-LABEL: @or_positive_sge_zero(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A:%.*]], -1
+; CHECK-LABEL: define i1 @or_positive_sge_zero(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %b = or i8 %a, 24
@@ -4965,8 +5372,9 @@ define i1 @or_positive_sge_zero(i8 %a) {
 }
 
 define <2 x i1> @or_postive_sge_zero_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_postive_sge_zero_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[A:%.*]], splat (i8 -1)
+; CHECK-LABEL: define <2 x i1> @or_postive_sge_zero_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[A]], splat (i8 -1)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
 
@@ -4976,7 +5384,8 @@ define <2 x i1> @or_postive_sge_zero_vec(<2 x i8> %a) {
 }
 
 define <2 x i1> @or_poison_vec_sge_zero_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_poison_vec_sge_zero_vec(
+; CHECK-LABEL: define <2 x i1> @or_poison_vec_sge_zero_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> poison
 ;
 
@@ -4986,8 +5395,9 @@ define <2 x i1> @or_poison_vec_sge_zero_vec(<2 x i8> %a) {
 }
 
 define i1 @or_positive_sge_postive(i8 %a) {
-; CHECK-LABEL: @or_positive_sge_postive(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A:%.*]], -1
+; CHECK-LABEL: define i1 @or_positive_sge_postive(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %b = or i8 %a, 24
@@ -4996,8 +5406,9 @@ define i1 @or_positive_sge_postive(i8 %a) {
 }
 
 define <2 x i1> @or_postive_sge_positive_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_postive_sge_positive_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[A:%.*]], splat (i8 -1)
+; CHECK-LABEL: define <2 x i1> @or_postive_sge_positive_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[A]], splat (i8 -1)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
 
@@ -5007,7 +5418,8 @@ define <2 x i1> @or_postive_sge_positive_vec(<2 x i8> %a) {
 }
 
 define <2 x i1> @or_poison_vec_sge_positive_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_poison_vec_sge_positive_vec(
+; CHECK-LABEL: define <2 x i1> @or_poison_vec_sge_positive_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> poison
 ;
 
@@ -5017,8 +5429,9 @@ define <2 x i1> @or_poison_vec_sge_positive_vec(<2 x i8> %a) {
 }
 
 define i1 @or_positive_sle_zero(i8 %a) {
-; CHECK-LABEL: @or_positive_sle_zero(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A:%.*]], 0
+; CHECK-LABEL: define i1 @or_positive_sle_zero(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %b = or i8 %a, 24
@@ -5027,8 +5440,9 @@ define i1 @or_positive_sle_zero(i8 %a) {
 }
 
 define <2 x i1> @or_postive_sle_zero_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_postive_sle_zero_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @or_postive_sle_zero_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
 
@@ -5038,7 +5452,8 @@ define <2 x i1> @or_postive_sle_zero_vec(<2 x i8> %a) {
 }
 
 define <2 x i1> @or_poison_vec_sle_zero_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_poison_vec_sle_zero_vec(
+; CHECK-LABEL: define <2 x i1> @or_poison_vec_sle_zero_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> poison
 ;
 
@@ -5048,8 +5463,9 @@ define <2 x i1> @or_poison_vec_sle_zero_vec(<2 x i8> %a) {
 }
 
 define i1 @or_positive_slt_zero(i8 %a) {
-; CHECK-LABEL: @or_positive_slt_zero(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A:%.*]], 0
+; CHECK-LABEL: define i1 @or_positive_slt_zero(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %b = or i8 %a, 24
@@ -5058,8 +5474,9 @@ define i1 @or_positive_slt_zero(i8 %a) {
 }
 
 define <2 x i1> @or_postive_slt_zero_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_postive_slt_zero_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @or_postive_slt_zero_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
 
@@ -5069,7 +5486,8 @@ define <2 x i1> @or_postive_slt_zero_vec(<2 x i8> %a) {
 }
 
 define <2 x i1> @or_poison_vec_slt_zero_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_poison_vec_slt_zero_vec(
+; CHECK-LABEL: define <2 x i1> @or_poison_vec_slt_zero_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> poison
 ;
 
@@ -5079,8 +5497,9 @@ define <2 x i1> @or_poison_vec_slt_zero_vec(<2 x i8> %a) {
 }
 
 define i1 @or_positive_slt_postive(i8 %a) {
-; CHECK-LABEL: @or_positive_slt_postive(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A:%.*]], 0
+; CHECK-LABEL: define i1 @or_positive_slt_postive(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %b = or i8 %a, 24
@@ -5089,8 +5508,9 @@ define i1 @or_positive_slt_postive(i8 %a) {
 }
 
 define <2 x i1> @or_postive_slt_positive_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_postive_slt_positive_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @or_postive_slt_positive_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
 
@@ -5100,7 +5520,8 @@ define <2 x i1> @or_postive_slt_positive_vec(<2 x i8> %a) {
 }
 
 define <2 x i1> @or_poison_vec_slt_positive_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_poison_vec_slt_positive_vec(
+; CHECK-LABEL: define <2 x i1> @or_poison_vec_slt_positive_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> poison
 ;
 
@@ -5112,8 +5533,9 @@ define <2 x i1> @or_poison_vec_slt_positive_vec(<2 x i8> %a) {
 ; negative tests for icmp(X | OrC, C) --> icmp(X, 0)
 
 define i1 @or_positive_sgt_neg(i8 %a) {
-; CHECK-LABEL: @or_positive_sgt_neg(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A:%.*]], -1
+; CHECK-LABEL: define i1 @or_positive_sgt_neg(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %b = or i8 %a, 24
@@ -5122,8 +5544,9 @@ define i1 @or_positive_sgt_neg(i8 %a) {
 }
 
 define <2 x i1> @or_postive_sgt_neg_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_postive_sgt_neg_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[A:%.*]], splat (i8 -1)
+; CHECK-LABEL: define <2 x i1> @or_postive_sgt_neg_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[A]], splat (i8 -1)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
 
@@ -5133,8 +5556,9 @@ define <2 x i1> @or_postive_sgt_neg_vec(<2 x i8> %a) {
 }
 
 define i1 @mul_or_positive_sge_neg(i8 %a) {
-; CHECK-LABEL: @mul_or_positive_sge_neg(
-; CHECK-NEXT:    [[B:%.*]] = or i8 [[A:%.*]], 24
+; CHECK-LABEL: define i1 @mul_or_positive_sge_neg(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or i8 [[A]], 24
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[B]], -2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5144,8 +5568,9 @@ define i1 @mul_or_positive_sge_neg(i8 %a) {
 }
 
 define <2 x i1> @or_postive_sge_neg_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_postive_sge_neg_vec(
-; CHECK-NEXT:    [[B:%.*]] = or <2 x i8> [[A:%.*]], splat (i8 24)
+; CHECK-LABEL: define <2 x i1> @or_postive_sge_neg_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or <2 x i8> [[A]], splat (i8 24)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[B]], splat (i8 -2)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -5156,8 +5581,9 @@ define <2 x i1> @or_postive_sge_neg_vec(<2 x i8> %a) {
 }
 
 define i1 @mul_or_small_sge_large(i8 %a) {
-; CHECK-LABEL: @mul_or_small_sge_large(
-; CHECK-NEXT:    [[B:%.*]] = or i8 [[A:%.*]], 24
+; CHECK-LABEL: define i1 @mul_or_small_sge_large(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or i8 [[A]], 24
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[B]], 24
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5167,8 +5593,9 @@ define i1 @mul_or_small_sge_large(i8 %a) {
 }
 
 define <2 x i1> @or_small_sge_large_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_small_sge_large_vec(
-; CHECK-NEXT:    [[B:%.*]] = or <2 x i8> [[A:%.*]], splat (i8 24)
+; CHECK-LABEL: define <2 x i1> @or_small_sge_large_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or <2 x i8> [[A]], splat (i8 24)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i8> [[B]], splat (i8 24)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -5179,8 +5606,9 @@ define <2 x i1> @or_small_sge_large_vec(<2 x i8> %a) {
 }
 
 define i1 @or_positive_sle_neg(i8 %a) {
-; CHECK-LABEL: @or_positive_sle_neg(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A:%.*]], 0
+; CHECK-LABEL: define i1 @or_positive_sle_neg(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %b = or i8 %a, 24
@@ -5189,8 +5617,9 @@ define i1 @or_positive_sle_neg(i8 %a) {
 }
 
 define <2 x i1> @or_sle_neg_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_sle_neg_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i1> @or_sle_neg_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
 
@@ -5200,8 +5629,9 @@ define <2 x i1> @or_sle_neg_vec(<2 x i8> %a) {
 }
 
 define i1 @or_positive_slt_neg(i8 %a) {
-; CHECK-LABEL: @or_positive_slt_neg(
-; CHECK-NEXT:    [[B:%.*]] = or i8 [[A:%.*]], 24
+; CHECK-LABEL: define i1 @or_positive_slt_neg(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or i8 [[A]], 24
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[B]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5211,8 +5641,9 @@ define i1 @or_positive_slt_neg(i8 %a) {
 }
 
 define <2 x i1> @or_postive_slt_neg_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_postive_slt_neg_vec(
-; CHECK-NEXT:    [[B:%.*]] = or <2 x i8> [[A:%.*]], splat (i8 24)
+; CHECK-LABEL: define <2 x i1> @or_postive_slt_neg_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or <2 x i8> [[A]], splat (i8 24)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[B]], splat (i8 -1)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -5223,8 +5654,9 @@ define <2 x i1> @or_postive_slt_neg_vec(<2 x i8> %a) {
 }
 
 define i1 @or_small_slt_large(i8 %a) {
-; CHECK-LABEL: @or_small_slt_large(
-; CHECK-NEXT:    [[B:%.*]] = or i8 [[A:%.*]], 24
+; CHECK-LABEL: define i1 @or_small_slt_large(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or i8 [[A]], 24
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[B]], 25
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5234,8 +5666,9 @@ define i1 @or_small_slt_large(i8 %a) {
 }
 
 define <2 x i1> @or_small_slt_large_vec(<2 x i8> %a) {
-; CHECK-LABEL: @or_small_slt_large_vec(
-; CHECK-NEXT:    [[B:%.*]] = or <2 x i8> [[A:%.*]], splat (i8 24)
+; CHECK-LABEL: define <2 x i1> @or_small_slt_large_vec(
+; CHECK-SAME: <2 x i8> [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or <2 x i8> [[A]], splat (i8 24)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[B]], splat (i8 25)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -5246,8 +5679,9 @@ define <2 x i1> @or_small_slt_large_vec(<2 x i8> %a) {
 }
 
 define i1 @or_positive_sgt_zero_multi_use(i8 %a) {
-; CHECK-LABEL: @or_positive_sgt_zero_multi_use(
-; CHECK-NEXT:    [[B:%.*]] = or i8 [[A:%.*]], 24
+; CHECK-LABEL: define i1 @or_positive_sgt_zero_multi_use(
+; CHECK-SAME: i8 [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or i8 [[A]], 24
 ; CHECK-NEXT:    call void @use_i8(i8 [[B]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[A]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -5260,9 +5694,10 @@ define i1 @or_positive_sgt_zero_multi_use(i8 %a) {
 
 
 define i1 @disjoint_or_sgt_1(i8 %a, i8 %b) {
-; CHECK-LABEL: @disjoint_or_sgt_1(
-; CHECK-NEXT:    [[B1:%.*]] = add nsw i8 [[B:%.*]], 2
-; CHECK-NEXT:    [[ICMP_:%.*]] = icmp sge i8 [[A:%.*]], [[B1]]
+; CHECK-LABEL: define i1 @disjoint_or_sgt_1(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[B1:%.*]] = add nsw i8 [[B]], 2
+; CHECK-NEXT:    [[ICMP_:%.*]] = icmp sge i8 [[A]], [[B1]]
 ; CHECK-NEXT:    ret i1 [[ICMP_]]
 ;
   %a1 = or disjoint i8 %a, 1
@@ -5272,9 +5707,10 @@ define i1 @disjoint_or_sgt_1(i8 %a, i8 %b) {
 }
 
 define i1 @disjoint_or_sgt_2(i8 %a, i8 %b) {
-; CHECK-LABEL: @disjoint_or_sgt_2(
-; CHECK-NEXT:    [[A1:%.*]] = or disjoint i8 [[A:%.*]], 2
-; CHECK-NEXT:    [[B1:%.*]] = add i8 [[B:%.*]], 1
+; CHECK-LABEL: define i1 @disjoint_or_sgt_2(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = or disjoint i8 [[A]], 2
+; CHECK-NEXT:    [[B1:%.*]] = add i8 [[B]], 1
 ; CHECK-NEXT:    [[ICMP_:%.*]] = icmp sgt i8 [[A1]], [[B1]]
 ; CHECK-NEXT:    ret i1 [[ICMP_]]
 ;
@@ -5285,9 +5721,10 @@ define i1 @disjoint_or_sgt_2(i8 %a, i8 %b) {
 }
 
 define i1 @disjoint_or_sgt_3(i8 %a, i8 %b) {
-; CHECK-LABEL: @disjoint_or_sgt_3(
-; CHECK-NEXT:    [[A1:%.*]] = or disjoint i8 [[A:%.*]], 2
-; CHECK-NEXT:    [[B1:%.*]] = add nuw i8 [[B:%.*]], 1
+; CHECK-LABEL: define i1 @disjoint_or_sgt_3(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = or disjoint i8 [[A]], 2
+; CHECK-NEXT:    [[B1:%.*]] = add nuw i8 [[B]], 1
 ; CHECK-NEXT:    [[ICMP_:%.*]] = icmp sgt i8 [[A1]], [[B1]]
 ; CHECK-NEXT:    ret i1 [[ICMP_]]
 ;
@@ -5298,9 +5735,10 @@ define i1 @disjoint_or_sgt_3(i8 %a, i8 %b) {
 }
 
 define i1 @disjoint_or_ugt_1(i8 %a, i8 %b) {
-; CHECK-LABEL: @disjoint_or_ugt_1(
-; CHECK-NEXT:    [[B1:%.*]] = add nsw i8 [[B:%.*]], 2
-; CHECK-NEXT:    [[ICMP_:%.*]] = icmp uge i8 [[A:%.*]], [[B1]]
+; CHECK-LABEL: define i1 @disjoint_or_ugt_1(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[B1:%.*]] = add nsw i8 [[B]], 2
+; CHECK-NEXT:    [[ICMP_:%.*]] = icmp uge i8 [[A]], [[B1]]
 ; CHECK-NEXT:    ret i1 [[ICMP_]]
 ;
   %a1 = or disjoint i8 %a, 1
@@ -5310,9 +5748,10 @@ define i1 @disjoint_or_ugt_1(i8 %a, i8 %b) {
 }
 
 define i1 @disjoint_or_ugt_2(i8 %a, i8 %b) {
-; CHECK-LABEL: @disjoint_or_ugt_2(
-; CHECK-NEXT:    [[A1:%.*]] = or disjoint i8 [[A:%.*]], 2
-; CHECK-NEXT:    [[B1:%.*]] = add i8 [[B:%.*]], 1
+; CHECK-LABEL: define i1 @disjoint_or_ugt_2(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = or disjoint i8 [[A]], 2
+; CHECK-NEXT:    [[B1:%.*]] = add i8 [[B]], 1
 ; CHECK-NEXT:    [[ICMP_:%.*]] = icmp ugt i8 [[A1]], [[B1]]
 ; CHECK-NEXT:    ret i1 [[ICMP_]]
 ;
@@ -5323,9 +5762,10 @@ define i1 @disjoint_or_ugt_2(i8 %a, i8 %b) {
 }
 
 define i1 @disjoint_or_ugt_3(i8 %a, i8 %b) {
-; CHECK-LABEL: @disjoint_or_ugt_3(
-; CHECK-NEXT:    [[A1:%.*]] = or disjoint i8 [[A:%.*]], 2
-; CHECK-NEXT:    [[B1:%.*]] = add nuw i8 [[B:%.*]], 1
+; CHECK-LABEL: define i1 @disjoint_or_ugt_3(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[A1:%.*]] = or disjoint i8 [[A]], 2
+; CHECK-NEXT:    [[B1:%.*]] = add nuw i8 [[B]], 1
 ; CHECK-NEXT:    [[ICMP_:%.*]] = icmp ugt i8 [[A1]], [[B1]]
 ; CHECK-NEXT:    ret i1 [[ICMP_]]
 ;
@@ -5336,10 +5776,11 @@ define i1 @disjoint_or_ugt_3(i8 %a, i8 %b) {
 }
 
 define i1 @deduce_nuw_flag_1(i8 %a, i8 %b) {
-; CHECK-LABEL: @deduce_nuw_flag_1(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i8 [[B:%.*]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP0]], [[A:%.*]]
+; CHECK-LABEL: define i1 @deduce_nuw_flag_1(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i8 [[B]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP0]], [[A]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
 entry:
@@ -5350,10 +5791,11 @@ entry:
 }
 
 define i1 @deduce_nuw_flag_2(i8 %a, i8 %b) {
-; CHECK-LABEL: @deduce_nuw_flag_2(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i8 [[B:%.*]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A:%.*]], [[TMP0]]
+; CHECK-LABEL: define i1 @deduce_nuw_flag_2(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i8 [[B]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], [[TMP0]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
 entry:
@@ -5364,10 +5806,11 @@ entry:
 }
 
 define i1 @dont_deduce_nuw_flag_1(i8 %a, i8 %b) {
-; CHECK-LABEL: @dont_deduce_nuw_flag_1(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i8 [[B:%.*]], -1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP0]], [[A:%.*]]
+; CHECK-LABEL: define i1 @dont_deduce_nuw_flag_1(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i8 [[B]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[TMP0]], [[A]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
 entry:
@@ -5378,10 +5821,11 @@ entry:
 }
 
 define i1 @dont_deduce_nuw_flag_2(i8 %a, i8 %b) {
-; CHECK-LABEL: @dont_deduce_nuw_flag_2(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = add i8 [[B:%.*]], -1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A:%.*]], [[TMP0]]
+; CHECK-LABEL: define i1 @dont_deduce_nuw_flag_2(
+; CHECK-SAME: i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i8 [[B]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], [[TMP0]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
 entry:
@@ -5392,11 +5836,11 @@ entry:
 }
 
 define i1 @icmp_freeze_sext(i16 %x, i16 %y) {
-; CHECK-LABEL: @icmp_freeze_sext(
-; CHECK-NEXT:    [[CMP1:%.*]] = icmp uge i16 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[CMP1_FR:%.*]] = freeze i1 [[CMP1]]
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i16 [[Y]], 0
-; CHECK-NEXT:    [[CMP2:%.*]] = or i1 [[TMP1]], [[CMP1_FR]]
+; CHECK-LABEL: define i1 @icmp_freeze_sext(
+; CHECK-SAME: i16 [[X:%.*]], i16 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i16 [[Y]]
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i16 [[X]]
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp uge i16 [[X_FR]], [[Y_FR]]
 ; CHECK-NEXT:    ret i1 [[CMP2]]
 ;
   %cmp1 = icmp uge i16 %x, %y
@@ -5407,8 +5851,9 @@ define i1 @icmp_freeze_sext(i16 %x, i16 %y) {
 }
 
 define i1 @test_icmp_shl(i64 %x) {
-; CHECK-LABEL: @test_icmp_shl(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-LABEL: define i1 @test_icmp_shl(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X]] to i32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP1]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5418,8 +5863,9 @@ define i1 @test_icmp_shl(i64 %x) {
 }
 
 define i1 @test_icmp_shl_multiuse(i64 %x) {
-; CHECK-LABEL: @test_icmp_shl_multiuse(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[X:%.*]], 32
+; CHECK-LABEL: define i1 @test_icmp_shl_multiuse(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[X]], 32
 ; CHECK-NEXT:    call void @use_i64(i64 [[SHL]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[SHL]], 8589934593
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -5431,8 +5877,9 @@ define i1 @test_icmp_shl_multiuse(i64 %x) {
 }
 
 define i1 @test_icmp_shl_illegal_length(i64 %x) {
-; CHECK-LABEL: @test_icmp_shl_illegal_length(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[X:%.*]], 31
+; CHECK-LABEL: define i1 @test_icmp_shl_illegal_length(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[X]], 31
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[SHL]], 8589934593
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5442,8 +5889,9 @@ define i1 @test_icmp_shl_illegal_length(i64 %x) {
 }
 
 define i1 @test_icmp_shl_invalid_rhsc(i64 %x) {
-; CHECK-LABEL: @test_icmp_shl_invalid_rhsc(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[X:%.*]], 32
+; CHECK-LABEL: define i1 @test_icmp_shl_invalid_rhsc(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[X]], 32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[SHL]], 8589934595
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5453,8 +5901,9 @@ define i1 @test_icmp_shl_invalid_rhsc(i64 %x) {
 }
 
 define i1 @test_icmp_shl_nuw(i64 %x) {
-; CHECK-LABEL: @test_icmp_shl_nuw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @test_icmp_shl_nuw(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[X]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl nuw i64 %x, 32
@@ -5463,8 +5912,9 @@ define i1 @test_icmp_shl_nuw(i64 %x) {
 }
 
 define i1 @test_icmp_shl_nuw_i31(i31 %x) {
-; CHECK-LABEL: @test_icmp_shl_nuw_i31(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i31 [[X:%.*]], 250
+; CHECK-LABEL: define i1 @test_icmp_shl_nuw_i31(
+; CHECK-SAME: i31 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i31 [[X]], 250
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl nuw i31 %x, 23
@@ -5473,8 +5923,9 @@ define i1 @test_icmp_shl_nuw_i31(i31 %x) {
 }
 
 define i1 @test_icmp_shl_nsw(i64 %x) {
-; CHECK-LABEL: @test_icmp_shl_nsw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[X:%.*]], 3
+; CHECK-LABEL: define i1 @test_icmp_shl_nsw(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[X]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl nsw i64 %x, 32
@@ -5483,8 +5934,9 @@ define i1 @test_icmp_shl_nsw(i64 %x) {
 }
 
 define i1 @test_icmp_shl_nsw_i31(i31 %x) {
-; CHECK-LABEL: @test_icmp_shl_nsw_i31(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i31 [[X:%.*]] to i8
+; CHECK-LABEL: define i1 @test_icmp_shl_nsw_i31(
+; CHECK-SAME: i31 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i31 [[X]] to i8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[TMP1]], -6
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5494,8 +5946,9 @@ define i1 @test_icmp_shl_nsw_i31(i31 %x) {
 }
 
 define <2 x i1> @test_icmp_shl_vec(<2 x i64> %x) {
-; CHECK-LABEL: @test_icmp_shl_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[X:%.*]] to <2 x i32>
+; CHECK-LABEL: define <2 x i1> @test_icmp_shl_vec(
+; CHECK-SAME: <2 x i64> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[X]] to <2 x i32>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[TMP1]], splat (i32 3)
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -5505,7 +5958,8 @@ define <2 x i1> @test_icmp_shl_vec(<2 x i64> %x) {
 }
 
 define i1 @test_icmp_shl_eq(i64 %x) {
-; CHECK-LABEL: @test_icmp_shl_eq(
+; CHECK-LABEL: define i1 @test_icmp_shl_eq(
+; CHECK-SAME: i64 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %shl = shl i64 %x, 32
@@ -5514,8 +5968,9 @@ define i1 @test_icmp_shl_eq(i64 %x) {
 }
 
 define i1 @test_icmp_shl_sgt(i64 %x) {
-; CHECK-LABEL: @test_icmp_shl_sgt(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; CHECK-LABEL: define i1 @test_icmp_shl_sgt(
+; CHECK-SAME: i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X]] to i32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5525,8 +5980,9 @@ define i1 @test_icmp_shl_sgt(i64 %x) {
 }
 
 define i1 @pr94897(i32 range(i32 -2147483648, 0) %x) {
-; CHECK-LABEL: @pr94897(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp samesign ugt i32 [[X:%.*]], -3
+; CHECK-LABEL: define i1 @pr94897(
+; CHECK-SAME: i32 range(i32 -2147483648, 0) [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp samesign ugt i32 [[X]], -3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl nsw i32 %x, 24
@@ -5535,11 +5991,12 @@ define i1 @pr94897(i32 range(i32 -2147483648, 0) %x) {
 }
 
 define i1 @icmp_and_inv_pow2_ne_0(i32 %A, i32 %B) {
-; CHECK-LABEL: @icmp_and_inv_pow2_ne_0(
-; CHECK-NEXT:    [[POPCNT:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctpop.i32(i32 [[A:%.*]])
+; CHECK-LABEL: define i1 @icmp_and_inv_pow2_ne_0(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[POPCNT:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctpop.i32(i32 [[A]])
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[POPCNT]], 1
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[COND]])
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[A]], [[B]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -5554,11 +6011,12 @@ define i1 @icmp_and_inv_pow2_ne_0(i32 %A, i32 %B) {
 }
 
 define i1 @icmp_and_inv_pow2_or_zero_ne_0(i32 %A, i32 %B) {
-; CHECK-LABEL: @icmp_and_inv_pow2_or_zero_ne_0(
-; CHECK-NEXT:    [[POPCNT:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctpop.i32(i32 [[A:%.*]])
+; CHECK-LABEL: define i1 @icmp_and_inv_pow2_or_zero_ne_0(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[POPCNT:%.*]] = tail call range(i32 0, 33) i32 @llvm.ctpop.i32(i32 [[A]])
 ; CHECK-NEXT:    [[COND:%.*]] = icmp samesign ult i32 [[POPCNT]], 2
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[COND]])
-; CHECK-NEXT:    [[INV:%.*]] = xor i32 [[B:%.*]], -1
+; CHECK-NEXT:    [[INV:%.*]] = xor i32 [[B]], -1
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A]], [[INV]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -5574,9 +6032,10 @@ define i1 @icmp_and_inv_pow2_or_zero_ne_0(i32 %A, i32 %B) {
 }
 
 define i1 @icmp_samesign_logical_and(i32 %In) {
-; CHECK-LABEL: @icmp_samesign_logical_and(
-; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32 [[IN:%.*]], 1
-; CHECK-NEXT:    ret i1 [[C2]]
+; CHECK-LABEL: define i1 @icmp_samesign_logical_and(
+; CHECK-SAME: i32 [[IN:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp eq i32 [[IN]], 1
+; CHECK-NEXT:    ret i1 [[V]]
 ;
   %c1 = icmp samesign sgt i32 %In, -1
   %c2 = icmp samesign eq i32 %In, 1
@@ -5585,8 +6044,9 @@ define i1 @icmp_samesign_logical_and(i32 %In) {
 }
 
 define i1 @icmp_samesign_logical_or(i32 %In) {
-; CHECK-LABEL: @icmp_samesign_logical_or(
-; CHECK-NEXT:    [[V:%.*]] = icmp ne i32 [[IN:%.*]], 1
+; CHECK-LABEL: define i1 @icmp_samesign_logical_or(
+; CHECK-SAME: i32 [[IN:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = icmp ne i32 [[IN]], 1
 ; CHECK-NEXT:    ret i1 [[V]]
 ;
   %c1 = icmp samesign slt i32 %In, 0
diff --git a/llvm/test/Transforms/InstCombine/inttoptr-add-phi.ll b/llvm/test/Transforms/InstCombine/inttoptr-add-phi.ll
new file mode 100644
index 0000000000000..acceead23a5e5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/inttoptr-add-phi.ll
@@ -0,0 +1,69 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=instcombine -S %s | FileCheck %s
+
+define i64 @inttoptr_used_by_phi_with_ptrtoint(i1 %c, ptr %src, ptr %p2) {
+; CHECK-LABEL: define i64 @inttoptr_used_by_phi_with_ptrtoint(
+; CHECK-SAME: i1 [[C:%.*]], ptr [[SRC:%.*]], ptr [[P2:%.*]]) {
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[P:%.*]] = getelementptr i8, ptr [[SRC]], i64 10
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i64 [ [[TMP1]], %[[THEN]] ], [ 0, %[[ELSE]] ]
+; CHECK-NEXT:    ret i64 [[PHI]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  br i1 %c, label %then, label %else
+
+then:
+  br label %exit
+
+else:
+  br label %exit
+
+exit:
+  %phi = phi ptr [ %p, %then ], [ null, %else ]
+  %i.2 = ptrtoint ptr %phi to i64
+  ret i64 %i.2
+}
+
+declare void @foo(ptr)
+
+define i64 @inttoptr_used_by_phi_with_other_users(i1 %c, ptr %src, ptr %p2) {
+; CHECK-LABEL: define i64 @inttoptr_used_by_phi_with_other_users(
+; CHECK-SAME: i1 [[C:%.*]], ptr [[SRC:%.*]], ptr [[P2:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = ptrtoint ptr [[SRC]] to i64
+; CHECK-NEXT:    [[A:%.*]] = add i64 [[I]], 10
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i64 [ [[A]], %[[THEN]] ], [ 0, %[[ELSE]] ]
+; CHECK-NEXT:    [[P:%.*]] = inttoptr i64 [[A]] to ptr
+; CHECK-NEXT:    call void @foo(ptr [[P]])
+; CHECK-NEXT:    ret i64 [[PHI]]
+;
+  %i = ptrtoint ptr %src to i64
+  %a = add i64 %i, 10
+  %p = inttoptr i64 %a to ptr
+  br i1 %c, label %then, label %else
+
+then:
+  br label %exit
+
+else:
+  br label %exit
+
+exit:
+  %phi = phi ptr [ %p, %then ], [ null, %else ]
+  call void @foo(ptr %p)
+  %i.2 = ptrtoint ptr %phi to i64
+  ret i64 %i.2
+}
diff --git a/llvm/test/Transforms/InstCombine/invariant-metadata-propagation.ll b/llvm/test/Transforms/InstCombine/invariant-metadata-propagation.ll
new file mode 100644
index 0000000000000..acc5e7ca8d2b4
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/invariant-metadata-propagation.ll
@@ -0,0 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S < %s -passes=instcombine | FileCheck %s
+
+%struct.double2 = type { double, double }
+%struct.double1 = type { double }
+
+define %struct.double2 @func1(ptr addrspace(1) %a) {
+; CHECK-LABEL: define %struct.double2 @func1(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]]) {
+; CHECK-NEXT:    [[DOTUNPACK:%.*]] = load double, ptr addrspace(1) [[A]], align 16, !invariant.load [[META0:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[STRUCT_DOUBLE2:%.*]] poison, double [[DOTUNPACK]], 0
+; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds nuw i8, ptr addrspace(1) [[A]], i64 8
+; CHECK-NEXT:    [[DOTUNPACK2:%.*]] = load double, ptr addrspace(1) [[DOTELT1]], align 8, !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue [[STRUCT_DOUBLE2]] [[TMP1]], double [[DOTUNPACK2]], 1
+; CHECK-NEXT:    ret [[STRUCT_DOUBLE2]] [[TMP2]]
+;
+  %1 = load %struct.double2, ptr addrspace(1) %a, align 16, !invariant.load !1
+  ret %struct.double2 %1
+}
+
+define %struct.double2 @func2(ptr %a) {
+; CHECK-LABEL: define %struct.double2 @func2(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    [[DOTUNPACK:%.*]] = load double, ptr [[A]], align 16, !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[STRUCT_DOUBLE2:%.*]] poison, double [[DOTUNPACK]], 0
+; CHECK-NEXT:    [[DOTELT1:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
+; CHECK-NEXT:    [[DOTUNPACK2:%.*]] = load double, ptr [[DOTELT1]], align 8, !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue [[STRUCT_DOUBLE2]] [[TMP1]], double [[DOTUNPACK2]], 1
+; CHECK-NEXT:    ret [[STRUCT_DOUBLE2]] [[TMP2]]
+;
+  %1 = load %struct.double2, ptr %a, align 16, !invariant.load !1
+  ret %struct.double2 %1
+}
+
+define %struct.double1 @func3(ptr %a) {
+; CHECK-LABEL: define %struct.double1 @func3(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    [[DOTUNPACK:%.*]] = load double, ptr [[A]], align 16, !invariant.load [[META0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue [[STRUCT_DOUBLE1:%.*]] poison, double [[DOTUNPACK]], 0
+; CHECK-NEXT:    ret [[STRUCT_DOUBLE1]] [[TMP1]]
+;
+  %1 = load %struct.double1, ptr %a, align 16, !invariant.load !1
+  ret %struct.double1 %1
+}
+
+!1 = !{}
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index 9a9fec694ff0e..f8c97d86a9230 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1244,6 +1244,48 @@ define i1 @extract_value_smul_fail(i8 %xx, i8 %yy) {
   ret i1 %r
 }
 
+define i8 @known_self_mul_bit_0_set(i8 noundef %x) {
+; CHECK-LABEL: @known_self_mul_bit_0_set(
+; CHECK-NEXT:    ret i8 0
+;
+  %bit_0_set = or i8 %x, 1
+  %self_mul = mul i8 %bit_0_set, %bit_0_set
+  %r = and i8 %self_mul, 4
+  ret i8 %r
+}
+
+define i8 @known_self_mul_bit_0_unset(i8 noundef %x) {
+; CHECK-LABEL: @known_self_mul_bit_0_unset(
+; CHECK-NEXT:    ret i8 0
+;
+  %bit_0_unset = and i8 %x, -2
+  %self_mul = mul i8 %bit_0_unset, %bit_0_unset
+  %r = and i8 %self_mul, 8
+  ret i8 %r
+}
+
+define i8 @known_self_mul_bit_1_set_bit_0_unset(i8 noundef %x) {
+; CHECK-LABEL: @known_self_mul_bit_1_set_bit_0_unset(
+; CHECK-NEXT:    ret i8 0
+;
+  %lower_2_unset = and i8 %x, -4
+  %bit_1_set_bit_0_unset = or disjoint i8 %lower_2_unset, 2
+  %self_mul = mul i8 %bit_1_set_bit_0_unset, %bit_1_set_bit_0_unset
+  %r = and i8 %self_mul, 24
+  ret i8 %r
+}
+
+define i4 @known_self_mul_bit_1_set_bit_0_unset_i4(i4 noundef %x) {
+; CHECK-LABEL: @known_self_mul_bit_1_set_bit_0_unset_i4(
+; CHECK-NEXT:    ret i4 0
+;
+  %lower_2_unset = and i4 %x, -4
+  %bit_1_set_bit_0_unset = or disjoint i4 %lower_2_unset, 2
+  %self_mul = mul i4 %bit_1_set_bit_0_unset, %bit_1_set_bit_0_unset
+  %r = and i4 %self_mul, 24
+  ret i4 %r
+}
+
 define i8 @known_reduce_or(<2 x i8> %xx) {
 ; CHECK-LABEL: @known_reduce_or(
 ; CHECK-NEXT:    ret i8 1
diff --git a/llvm/test/Transforms/InstCombine/lifetime-no-null-opt.ll b/llvm/test/Transforms/InstCombine/lifetime-no-null-opt.ll
index 422e1796b2bed..597bd2c80c059 100644
--- a/llvm/test/Transforms/InstCombine/lifetime-no-null-opt.ll
+++ b/llvm/test/Transforms/InstCombine/lifetime-no-null-opt.ll
@@ -2,8 +2,8 @@
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 declare void @llvm.dbg.declare(metadata, metadata, metadata)
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @foo(ptr nocapture, ptr nocapture)
 
 define void @bar(i1 %flag) #0 !dbg !4 {
@@ -20,11 +20,11 @@ define void @bar(i1 %flag) #0 !dbg !4 {
 ; CHECK-NEXT:      #dbg_declare(ptr [[TEXT]], [[META16:![0-9]+]], !DIExpression(), [[META24:![0-9]+]])
 ; CHECK-NEXT:    br label [[FIN:%.*]]
 ; CHECK:       else:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[TEXT]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[BUFF]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TEXT]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BUFF]])
 ; CHECK-NEXT:    call void @foo(ptr nonnull [[BUFF]], ptr nonnull [[TEXT]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[BUFF]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[TEXT]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BUFF]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TEXT]])
 ; CHECK-NEXT:    br label [[FIN]]
 ; CHECK:       fin:
 ; CHECK-NEXT:    ret void
@@ -35,31 +35,31 @@ entry:
   br i1 %flag, label %if, label %else
 
 if:
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %buff)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %buff)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %buff)
+  call void @llvm.lifetime.end.p0(ptr %buff)
+  call void @llvm.lifetime.end.p0(ptr %text)
   br label %bb2
 
 bb2:
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %buff)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %buff)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %buff)
+  call void @llvm.lifetime.end.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %buff)
   br label %bb3
 
 bb3:
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
   call void @llvm.dbg.declare(metadata ptr %text, metadata !14, metadata !25), !dbg !26
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   br label %fin
 
 else:
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %buff)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %buff)
   call void @foo(ptr %buff, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %buff)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %buff)
+  call void @llvm.lifetime.end.p0(ptr %text)
   br  label %fin
 
 fin:
diff --git a/llvm/test/Transforms/InstCombine/lifetime-sanitizer.ll b/llvm/test/Transforms/InstCombine/lifetime-sanitizer.ll
index e379b32b45734..fd45fe2dd27d7 100644
--- a/llvm/test/Transforms/InstCombine/lifetime-sanitizer.ll
+++ b/llvm/test/Transforms/InstCombine/lifetime-sanitizer.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @foo(ptr nocapture)
 
 define void @asan() sanitize_address {
@@ -9,8 +9,8 @@ entry:
   ; CHECK-LABEL: @asan(
   %text = alloca i8, align 1
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   ; CHECK: call void @llvm.lifetime.start
   ; CHECK-NEXT: call void @llvm.lifetime.end
 
@@ -24,8 +24,8 @@ entry:
   ; CHECK-LABEL: @hwasan(
   %text = alloca i8, align 1
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   ; CHECK: call void @llvm.lifetime.start
   ; CHECK-NEXT: call void @llvm.lifetime.end
 
@@ -39,8 +39,8 @@ entry:
   ; CHECK-LABEL: @msan(
   %text = alloca i8, align 1
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   ; CHECK: call void @llvm.lifetime.start
   ; CHECK-NEXT: call void @llvm.lifetime.end
 
@@ -54,8 +54,8 @@ entry:
   ; CHECK-LABEL: @no_asan(
   %text = alloca i8, align 1
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   ; CHECK-NO: call void @llvm.lifetime
 
   call void @foo(ptr %text) ; Keep alloca alive
diff --git a/llvm/test/Transforms/InstCombine/lifetime.ll b/llvm/test/Transforms/InstCombine/lifetime.ll
index b94c9694d8f07..6313dba6c35ef 100644
--- a/llvm/test/Transforms/InstCombine/lifetime.ll
+++ b/llvm/test/Transforms/InstCombine/lifetime.ll
@@ -2,8 +2,8 @@
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 declare void @llvm.dbg.declare(metadata, metadata, metadata)
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @foo(ptr nocapture, ptr nocapture)
 
 define void @bar(i1 %flag) !dbg !4 {
@@ -20,11 +20,11 @@ define void @bar(i1 %flag) !dbg !4 {
 ; CHECK-NEXT:      #dbg_declare(ptr [[TEXT]], [[META16:![0-9]+]], !DIExpression(), [[META24:![0-9]+]])
 ; CHECK-NEXT:    br label [[FIN:%.*]]
 ; CHECK:       else:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[TEXT]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[BUFF]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TEXT]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[BUFF]])
 ; CHECK-NEXT:    call void @foo(ptr nonnull [[BUFF]], ptr nonnull [[TEXT]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[BUFF]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[TEXT]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[BUFF]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TEXT]])
 ; CHECK-NEXT:    br label [[FIN]]
 ; CHECK:       fin:
 ; CHECK-NEXT:    ret void
@@ -35,31 +35,31 @@ entry:
   br i1 %flag, label %if, label %else
 
 if:
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %buff)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %buff)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %buff)
+  call void @llvm.lifetime.end.p0(ptr %buff)
+  call void @llvm.lifetime.end.p0(ptr %text)
   br label %bb2
 
 bb2:
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %buff)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %buff)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %buff)
+  call void @llvm.lifetime.end.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %buff)
   br label %bb3
 
 bb3:
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
   call void @llvm.dbg.declare(metadata ptr %text, metadata !14, metadata !25), !dbg !26
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   br label %fin
 
 else:
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.start.p0(i64 1, ptr %buff)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %buff)
   call void @foo(ptr %buff, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %buff)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %buff)
+  call void @llvm.lifetime.end.p0(ptr %text)
   br  label %fin
 
 fin:
diff --git a/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll b/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll
index 0072153f7ca4c..7aa5eed2e88aa 100644
--- a/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll
+++ b/llvm/test/Transforms/InstCombine/lower-dbg-declare.ll
@@ -26,7 +26,7 @@ entry:
   %retval = alloca i32, align 4
   %d1 = alloca i32, align 4
   store i32 0, ptr %retval, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %d1) #4, !dbg !17
+  call void @llvm.lifetime.start.p0(ptr %d1) #4, !dbg !17
 ; CHECK: #dbg_value(i32 42, [[METADATA_IDX1:![0-9]+]], !DIExpression(),
 ; CHECK-NEXT: store
   call void @llvm.dbg.declare(metadata ptr %d1, metadata !16, metadata !DIExpression()), !dbg !17
@@ -48,11 +48,11 @@ while.body:                                       ; preds = %while.cond
   br label %while.cond, !dbg !22, !llvm.loop !24
 
 while.end:                                        ; preds = %while.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %d1) #4, !dbg !25
+  call void @llvm.lifetime.end.p0(ptr %d1) #4, !dbg !25
   ret i32 0, !dbg !26
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @llvm.dbg.declare(metadata, metadata, metadata)
 
@@ -64,7 +64,7 @@ define internal void @_ZL6escapeRi(ptr dereferenceable(4) %c) #3 !dbg !34 {
   ret void
 }
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!8, !9, !10}
diff --git a/llvm/test/Transforms/InstCombine/malloc-free.ll b/llvm/test/Transforms/InstCombine/malloc-free.ll
index d8a1c07a55429..5cff5d6dc10c2 100644
--- a/llvm/test/Transforms/InstCombine/malloc-free.ll
+++ b/llvm/test/Transforms/InstCombine/malloc-free.ll
@@ -97,8 +97,8 @@ define i1 @foo() {
   ret i1 %z
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 declare i64 @llvm.objectsize.i64(ptr, i1)
 declare void @llvm.memcpy.p0.p0.i32(ptr nocapture, ptr nocapture, i32, i1) nounwind
 declare void @llvm.memmove.p0.p0.i32(ptr nocapture, ptr nocapture, i32, i1) nounwind
diff --git a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
index 9c9ba83425845..64091a90752bf 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
@@ -178,14 +178,14 @@ define void @test4() {
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
 define void @test5() {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:    call void @baz(ptr nonnull byval(i8) @G)
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca %T
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %A)
+  call void @llvm.lifetime.start.p0(ptr %A)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %A, ptr align 4 @G, i64 124, i1 false)
   call void @baz(ptr byval(i8) %A)
   ret void
@@ -308,7 +308,7 @@ define float @test11(i64 %i) {
 
 entry:
   %a = alloca [4 x float], align 4
-  call void @llvm.lifetime.start.p0(i64 16, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memcpy.p0.p1.i64(ptr align 4 %a, ptr addrspace(1) align 4 @I, i64 16, i1 false)
   %g = getelementptr inbounds [4 x float], ptr %a, i64 0, i64 %i
   %r = load float, ptr %g, align 4
@@ -320,7 +320,7 @@ define float @test11_volatile(i64 %i) {
 ; CHECK-LABEL: @test11_volatile(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = alloca [4 x float], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p1.i64(ptr align 4 [[A]], ptr addrspace(1) align 4 @I, i64 16, i1 true)
 ; CHECK-NEXT:    [[G:%.*]] = getelementptr inbounds [4 x float], ptr [[A]], i64 0, i64 [[I:%.*]]
 ; CHECK-NEXT:    [[R:%.*]] = load float, ptr [[G]], align 4
@@ -329,7 +329,7 @@ define float @test11_volatile(i64 %i) {
 
 entry:
   %a = alloca [4 x float], align 4
-  call void @llvm.lifetime.start.p0(i64 16, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memcpy.p0.p1.i64(ptr align 4 %a, ptr addrspace(1) align 4 @I, i64 16, i1 true)
   %g = getelementptr inbounds [4 x float], ptr %a, i64 0, i64 %i
   %r = load float, ptr %g, align 4
diff --git a/llvm/test/Transforms/InstCombine/nsw.ll b/llvm/test/Transforms/InstCombine/nsw.ll
index b00f2e58add78..4615ac0ec1ffb 100644
--- a/llvm/test/Transforms/InstCombine/nsw.ll
+++ b/llvm/test/Transforms/InstCombine/nsw.ll
@@ -255,9 +255,10 @@ define i32 @sub_sub1_nsw_nsw(i32 %a, i32 %b, i32 %c) {
 
 define i8 @neg_nsw_freeze(i8 %a1, i8 %a2) {
 ; CHECK-LABEL: @neg_nsw_freeze(
-; CHECK-NEXT:    [[A_NEG:%.*]] = sub nsw i8 [[A2:%.*]], [[A1:%.*]]
-; CHECK-NEXT:    [[FR_NEG:%.*]] = freeze i8 [[A_NEG]]
-; CHECK-NEXT:    ret i8 [[FR_NEG]]
+; CHECK-NEXT:    [[A1_FR:%.*]] = freeze i8 [[A1:%.*]]
+; CHECK-NEXT:    [[A2_FR:%.*]] = freeze i8 [[A2:%.*]]
+; CHECK-NEXT:    [[A_NEG:%.*]] = sub i8 [[A2_FR]], [[A1_FR]]
+; CHECK-NEXT:    ret i8 [[A_NEG]]
 ;
   %a = sub nsw i8 %a1, %a2
   %fr = freeze i8 %a
diff --git a/llvm/test/Transforms/InstCombine/recurrence-binary-intrinsic.ll b/llvm/test/Transforms/InstCombine/recurrence-binary-intrinsic.ll
index 86e586ef0a16c..a4e247efc4d23 100644
--- a/llvm/test/Transforms/InstCombine/recurrence-binary-intrinsic.ll
+++ b/llvm/test/Transforms/InstCombine/recurrence-binary-intrinsic.ll
@@ -236,12 +236,11 @@ define float @simple_recurrence_intrinsic_maximumnum(i32 %n, float %a, float %b)
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[FMAX_ACC:%.*]] = phi float [ [[FMAX:%.*]], %[[LOOP]] ], [ [[A]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[FMAX]] = call nnan float @llvm.maximumnum.f32(float [[FMAX_ACC]], float [[B]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw i32 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FMAX:%.*]] = call nnan float @llvm.maximumnum.f32(float [[A]], float [[B]])
 ; CHECK-NEXT:    ret float [[FMAX]]
 ;
 entry:
@@ -265,12 +264,11 @@ define float @simple_recurrence_intrinsic_minimumnum(i32 %n, float %a, float %b)
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[FMIN_ACC:%.*]] = phi float [ [[FMIN:%.*]], %[[LOOP]] ], [ [[A]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[FMIN]] = call nnan float @llvm.minimumnum.f32(float [[FMIN_ACC]], float [[B]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw i32 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
 ; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[FMIN:%.*]] = call nnan float @llvm.minimumnum.f32(float [[A]], float [[B]])
 ; CHECK-NEXT:    ret float [[FMIN]]
 ;
 entry:
@@ -296,7 +294,7 @@ define i8 @simple_recurrence_intrinsic_multiuse_phi(i8 %n, i8 %a, i8 %b) {
 ; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    [[UMAX_ACC:%.*]] = phi i8 [ [[UMAX:%.*]], %[[LOOP]] ], [ [[A]], %[[ENTRY]] ]
 ; CHECK-NEXT:    call void @use(i8 [[UMAX_ACC]])
-; CHECK-NEXT:    [[UMAX]] = call i8 @llvm.umax.i8(i8 [[UMAX_ACC]], i8 [[B]])
+; CHECK-NEXT:    [[UMAX]] = call i8 @llvm.umax.i8(i8 [[A]], i8 [[B]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw i8 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
diff --git a/llvm/test/Transforms/InstCombine/repack-ints-thru-zext.ll b/llvm/test/Transforms/InstCombine/repack-ints-thru-zext.ll
new file mode 100644
index 0000000000000..c90f08b7322ac
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/repack-ints-thru-zext.ll
@@ -0,0 +1,242 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=instcombine %s -S | FileCheck %s
+
+declare void @use.i32(i32)
+declare void @use.i64(i64)
+
+define i64 @full_shl(i32 %x) {
+; CHECK-LABEL: define i64 @full_shl(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_ZEXT:%.*]] = zext i32 [[X]] to i64
+; CHECK-NEXT:    [[LO_SHL:%.*]] = shl nuw nsw i64 [[X_ZEXT]], 24
+; CHECK-NEXT:    ret i64 [[LO_SHL]]
+;
+  %lo = and i32 %x, u0xffff
+  %lo.zext = zext nneg i32 %lo to i64
+  %lo.shl = shl nuw nsw i64 %lo.zext, 24
+
+  %hi = lshr i32 %x, 16
+  %hi.zext = zext nneg i32 %hi to i64
+  %hi.shl = shl nuw nsw i64 %hi.zext, 40
+
+  %res = or disjoint i64 %lo.shl, %hi.shl
+  ret i64 %res
+}
+
+define <2 x i64> @full_shl_vec(<2 x i32> %v) {
+; CHECK-LABEL: define <2 x i64> @full_shl_vec(
+; CHECK-SAME: <2 x i32> [[V:%.*]]) {
+; CHECK-NEXT:    [[V_ZEXT:%.*]] = zext <2 x i32> [[V]] to <2 x i64>
+; CHECK-NEXT:    [[LO_SHL:%.*]] = shl nuw nsw <2 x i64> [[V_ZEXT]], splat (i64 24)
+; CHECK-NEXT:    ret <2 x i64> [[LO_SHL]]
+;
+  %lo = and <2 x i32> %v, splat(i32 u0xffff)
+  %lo.zext = zext nneg <2 x i32> %lo to <2 x i64>
+  %lo.shl = shl nuw nsw <2 x i64> %lo.zext, splat(i64 24)
+
+  %hi = lshr <2 x i32> %v, splat(i32 16)
+  %hi.zext = zext nneg <2 x i32> %hi to <2 x i64>
+  %hi.shl = shl nuw nsw <2 x i64> %hi.zext, splat(i64 40)
+
+  %res = or disjoint <2 x i64> %lo.shl, %hi.shl
+  ret <2 x i64> %res
+}
+
+; u0xaabbccdd = -1430532899
+define i64 @partial_shl(i32 %x) {
+; CHECK-LABEL: define i64 @partial_shl(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_MASK:%.*]] = and i32 [[X]], -1430532899
+; CHECK-NEXT:    [[X_ZEXT:%.*]] = zext i32 [[X_MASK]] to i64
+; CHECK-NEXT:    [[LO_SHL:%.*]] = shl nuw nsw i64 [[X_ZEXT]], 24
+; CHECK-NEXT:    ret i64 [[LO_SHL]]
+;
+  %lo = and i32 %x, u0xccdd
+  %lo.zext = zext nneg i32 %lo to i64
+  %lo.shl = shl nuw nsw i64 %lo.zext, 24
+
+  %hi = lshr i32 %x, 16
+  %hi.mask = and i32 %hi, u0xaabb
+  %hi.zext = zext nneg i32 %hi.mask to i64
+  %hi.shl = shl nuw nsw i64 %hi.zext, 40
+
+  %res = or disjoint i64 %lo.shl, %hi.shl
+  ret i64 %res
+}
+
+define i64 @shl_multi_use_shl(i32 %x) {
+; CHECK-LABEL: define i64 @shl_multi_use_shl(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X]], 24
+; CHECK-NEXT:    [[LO_SHL:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    call void @use.i64(i64 [[LO_SHL]])
+; CHECK-NEXT:    [[HI:%.*]] = lshr i32 [[X]], 16
+; CHECK-NEXT:    [[HI_ZEXT:%.*]] = zext nneg i32 [[HI]] to i64
+; CHECK-NEXT:    [[HI_SHL:%.*]] = shl nuw nsw i64 [[HI_ZEXT]], 40
+; CHECK-NEXT:    [[RES:%.*]] = or disjoint i64 [[HI_SHL]], [[LO_SHL]]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %lo = and i32 %x, u0x00ff
+  %lo.zext = zext nneg i32 %lo to i64
+  %lo.shl = shl nuw nsw i64 %lo.zext, 24
+  call void @use.i64(i64 %lo.shl)
+
+  %hi = lshr i32 %x, 16
+  %hi.zext = zext nneg i32 %hi to i64
+  %hi.shl = shl nuw nsw i64 %hi.zext, 40
+
+  %res = or disjoint i64 %lo.shl, %hi.shl
+  ret i64 %res
+}
+
+define i64 @shl_multi_use_zext(i32 %x) {
+; CHECK-LABEL: define i64 @shl_multi_use_zext(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[LO:%.*]] = and i32 [[X]], 255
+; CHECK-NEXT:    [[LO_ZEXT:%.*]] = zext nneg i32 [[LO]] to i64
+; CHECK-NEXT:    call void @use.i64(i64 [[LO_ZEXT]])
+; CHECK-NEXT:    [[LO_SHL:%.*]] = shl nuw nsw i64 [[LO_ZEXT]], 24
+; CHECK-NEXT:    [[HI:%.*]] = lshr i32 [[X]], 16
+; CHECK-NEXT:    [[HI_ZEXT:%.*]] = zext nneg i32 [[HI]] to i64
+; CHECK-NEXT:    [[HI_SHL:%.*]] = shl nuw nsw i64 [[HI_ZEXT]], 40
+; CHECK-NEXT:    [[RES:%.*]] = or disjoint i64 [[LO_SHL]], [[HI_SHL]]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %lo = and i32 %x, u0x00ff
+  %lo.zext = zext nneg i32 %lo to i64
+  call void @use.i64(i64 %lo.zext)
+  %lo.shl = shl nuw nsw i64 %lo.zext, 24
+
+  %hi = lshr i32 %x, 16
+  %hi.zext = zext nneg i32 %hi to i64
+  %hi.shl = shl nuw nsw i64 %hi.zext, 40
+
+  %res = or disjoint i64 %lo.shl, %hi.shl
+  ret i64 %res
+}
+
+define i64 @shl_multi_use_lshr(i32 %x) {
+; CHECK-LABEL: define i64 @shl_multi_use_lshr(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X]], 24
+; CHECK-NEXT:    [[LO_SHL:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[HI:%.*]] = lshr i32 [[X]], 16
+; CHECK-NEXT:    call void @use.i32(i32 [[HI]])
+; CHECK-NEXT:    [[HI_ZEXT:%.*]] = zext nneg i32 [[HI]] to i64
+; CHECK-NEXT:    [[HI_SHL:%.*]] = shl nuw nsw i64 [[HI_ZEXT]], 40
+; CHECK-NEXT:    [[RES:%.*]] = or disjoint i64 [[HI_SHL]], [[LO_SHL]]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %lo = and i32 %x, u0x00ff
+  %lo.zext = zext nneg i32 %lo to i64
+  %lo.shl = shl nuw nsw i64 %lo.zext, 24
+
+  %hi = lshr i32 %x, 16
+  call void @use.i32(i32 %hi)
+  %hi.zext = zext nneg i32 %hi to i64
+  %hi.shl = shl nuw nsw i64 %hi.zext, 40
+
+  %res = or disjoint i64 %lo.shl, %hi.shl
+  ret i64 %res
+}
+
+define i64 @shl_non_disjoint(i32 %x) {
+; CHECK-LABEL: define i64 @shl_non_disjoint(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[LO:%.*]] = and i32 [[X]], 16711680
+; CHECK-NEXT:    [[LO_ZEXT:%.*]] = zext nneg i32 [[LO]] to i64
+; CHECK-NEXT:    [[LO_SHL:%.*]] = shl nuw nsw i64 [[LO_ZEXT]], 24
+; CHECK-NEXT:    [[HI:%.*]] = lshr i32 [[X]], 16
+; CHECK-NEXT:    call void @use.i32(i32 [[HI]])
+; CHECK-NEXT:    [[HI_ZEXT:%.*]] = zext nneg i32 [[HI]] to i64
+; CHECK-NEXT:    [[HI_SHL:%.*]] = shl nuw nsw i64 [[HI_ZEXT]], 40
+; CHECK-NEXT:    [[RES:%.*]] = or i64 [[LO_SHL]], [[HI_SHL]]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %lo = and i32 %x, u0x00ff0000
+  %lo.zext = zext nneg i32 %lo to i64
+  %lo.shl = shl nuw nsw i64 %lo.zext, 24
+
+  %hi = lshr i32 %x, 16
+  call void @use.i32(i32 %hi)
+  %hi.zext = zext nneg i32 %hi to i64
+  %hi.shl = shl nuw nsw i64 %hi.zext, 40
+
+  %res = or i64 %lo.shl, %hi.shl
+  ret i64 %res
+}
+
+define i64 @combine(i32 %lower, i32 %upper) {
+; CHECK-LABEL: define i64 @combine(
+; CHECK-SAME: i32 [[LOWER:%.*]], i32 [[UPPER:%.*]]) {
+; CHECK-NEXT:    [[BASE:%.*]] = zext i32 [[LOWER]] to i64
+; CHECK-NEXT:    [[UPPER_ZEXT:%.*]] = zext i32 [[UPPER]] to i64
+; CHECK-NEXT:    [[S_0:%.*]] = shl nuw i64 [[UPPER_ZEXT]], 32
+; CHECK-NEXT:    [[O_3:%.*]] = or disjoint i64 [[S_0]], [[BASE]]
+; CHECK-NEXT:    ret i64 [[O_3]]
+;
+  %base = zext i32 %lower to i64
+
+  %u.0 = and i32 %upper, u0xff
+  %z.0 = zext i32 %u.0 to i64
+  %s.0 = shl i64 %z.0, 32
+  %o.0 = or i64 %base, %s.0
+
+  %r.1 = lshr i32 %upper, 8
+  %u.1 = and i32 %r.1, u0xff
+  %z.1 = zext i32 %u.1 to i64
+  %s.1 = shl i64 %z.1, 40
+  %o.1 = or i64 %o.0, %s.1
+
+  %r.2 = lshr i32 %upper, 16
+  %u.2 = and i32 %r.2, u0xff
+  %z.2 = zext i32 %u.2 to i64
+  %s.2 = shl i64 %z.2, 48
+  %o.2 = or i64 %o.1, %s.2
+
+  %r.3 = lshr i32 %upper, 24
+  %u.3 = and i32 %r.3, u0xff
+  %z.3 = zext i32 %u.3 to i64
+  %s.3 = shl i64 %z.3, 56
+  %o.3 = or i64 %o.2, %s.3
+
+  ret i64 %o.3
+}
+
+define i64 @combine_2(i32 %lower, i32 %upper) {
+; CHECK-LABEL: define i64 @combine_2(
+; CHECK-SAME: i32 [[LOWER:%.*]], i32 [[UPPER:%.*]]) {
+; CHECK-NEXT:    [[BASE:%.*]] = zext i32 [[LOWER]] to i64
+; CHECK-NEXT:    [[S_03:%.*]] = zext i32 [[UPPER]] to i64
+; CHECK-NEXT:    [[O:%.*]] = shl nuw i64 [[S_03]], 32
+; CHECK-NEXT:    [[RES:%.*]] = or disjoint i64 [[O]], [[BASE]]
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %base = zext i32 %lower to i64
+
+  %u.0 = and i32 %upper, u0xff
+  %z.0 = zext i32 %u.0 to i64
+  %s.0 = shl i64 %z.0, 32
+
+  %r.1 = lshr i32 %upper, 8
+  %u.1 = and i32 %r.1, u0xff
+  %z.1 = zext i32 %u.1 to i64
+  %s.1 = shl i64 %z.1, 40
+  %o.1 = or i64 %s.0, %s.1
+
+  %r.2 = lshr i32 %upper, 16
+  %u.2 = and i32 %r.2, u0xff
+  %z.2 = zext i32 %u.2 to i64
+  %s.2 = shl i64 %z.2, 48
+
+  %r.3 = lshr i32 %upper, 24
+  %u.3 = and i32 %r.3, u0xff
+  %z.3 = zext i32 %u.3 to i64
+  %s.3 = shl i64 %z.3, 56
+  %o.3 = or i64 %s.2, %s.3
+
+  %o = or i64 %o.1, %o.3
+  %res = or i64 %o, %base
+
+  ret i64 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/select.ll b/llvm/test/Transforms/InstCombine/select.ll
index 1f9ee83536016..db117aa28cad6 100644
--- a/llvm/test/Transforms/InstCombine/select.ll
+++ b/llvm/test/Transforms/InstCombine/select.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 ; PR1822
@@ -6,8 +6,9 @@
 target datalayout = "e-p:64:64-p1:16:16-p2:32:32:32-p3:64:64:64"
 
 define i1 @test5(i1 %C) {
-; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-LABEL: define i1 @test5(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C]], true
 ; CHECK-NEXT:    ret i1 [[NOT_C]]
 ;
   %V = select i1 %C, i1 false, i1 true
@@ -15,8 +16,9 @@ define i1 @test5(i1 %C) {
 }
 
 define i32 @test6(i1 %C) {
-; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[V:%.*]] = zext i1 [[C:%.*]] to i32
+; CHECK-LABEL: define i32 @test6(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = zext i1 [[C]] to i32
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
   %V = select i1 %C, i32 1, i32 0
@@ -24,8 +26,9 @@ define i32 @test6(i1 %C) {
 }
 
 define i1 @trueval_is_true(i1 %C, i1 %X) {
-; CHECK-LABEL: @trueval_is_true(
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[X:%.*]]
+; CHECK-LABEL: define i1 @trueval_is_true(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i1 true, i1 [[X]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %R = select i1 %C, i1 true, i1 %X
@@ -33,8 +36,9 @@ define i1 @trueval_is_true(i1 %C, i1 %X) {
 }
 
 define <2 x i1> @trueval_is_true_vec(<2 x i1> %C, <2 x i1> %X) {
-; CHECK-LABEL: @trueval_is_true_vec(
-; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C:%.*]], <2 x i1> splat (i1 true), <2 x i1> [[X:%.*]]
+; CHECK-LABEL: define <2 x i1> @trueval_is_true_vec(
+; CHECK-SAME: <2 x i1> [[C:%.*]], <2 x i1> [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C]], <2 x i1> splat (i1 true), <2 x i1> [[X]]
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
   %R = select <2 x i1> %C, <2 x i1> <i1 true, i1 true>, <2 x i1> %X
@@ -42,8 +46,9 @@ define <2 x i1> @trueval_is_true_vec(<2 x i1> %C, <2 x i1> %X) {
 }
 
 define <2 x i1> @trueval_is_true_vec_poison_elt(<2 x i1> %C, <2 x i1> %X) {
-; CHECK-LABEL: @trueval_is_true_vec_poison_elt(
-; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C:%.*]], <2 x i1> <i1 poison, i1 true>, <2 x i1> [[X:%.*]]
+; CHECK-LABEL: define <2 x i1> @trueval_is_true_vec_poison_elt(
+; CHECK-SAME: <2 x i1> [[C:%.*]], <2 x i1> [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C]], <2 x i1> <i1 poison, i1 true>, <2 x i1> [[X]]
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
   %R = select <2 x i1> %C, <2 x i1> <i1 poison, i1 true>, <2 x i1> %X
@@ -51,8 +56,9 @@ define <2 x i1> @trueval_is_true_vec_poison_elt(<2 x i1> %C, <2 x i1> %X) {
 }
 
 define i1 @test8(i1 %C, i1 %X) {
-; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], i1 [[X:%.*]], i1 false
+; CHECK-LABEL: define i1 @test8(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i1 [[X]], i1 false
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %R = select i1 %C, i1 %X, i1 false
@@ -60,8 +66,9 @@ define i1 @test8(i1 %C, i1 %X) {
 }
 
 define <2 x i1> @test8vec(<2 x i1> %C, <2 x i1> %X) {
-; CHECK-LABEL: @test8vec(
-; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C:%.*]], <2 x i1> [[X:%.*]], <2 x i1> zeroinitializer
+; CHECK-LABEL: define <2 x i1> @test8vec(
+; CHECK-SAME: <2 x i1> [[C:%.*]], <2 x i1> [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C]], <2 x i1> [[X]], <2 x i1> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
   %R = select <2 x i1> %C, <2 x i1> %X, <2 x i1> <i1 false, i1 false>
@@ -69,8 +76,9 @@ define <2 x i1> @test8vec(<2 x i1> %C, <2 x i1> %X) {
 }
 
 define <vscale x 2 x i1> @test8vvec(<vscale x 2 x i1> %C, <vscale x 2 x i1> %X) {
-; CHECK-LABEL: @test8vvec(
-; CHECK-NEXT:    [[R:%.*]] = select <vscale x 2 x i1> [[C:%.*]], <vscale x 2 x i1> [[X:%.*]], <vscale x 2 x i1> zeroinitializer
+; CHECK-LABEL: define <vscale x 2 x i1> @test8vvec(
+; CHECK-SAME: <vscale x 2 x i1> [[C:%.*]], <vscale x 2 x i1> [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = select <vscale x 2 x i1> [[C]], <vscale x 2 x i1> [[X]], <vscale x 2 x i1> zeroinitializer
 ; CHECK-NEXT:    ret <vscale x 2 x i1> [[R]]
 ;
   %R = select <vscale x 2 x i1> %C, <vscale x 2 x i1> %X, <vscale x 2 x i1> zeroinitializer
@@ -78,9 +86,10 @@ define <vscale x 2 x i1> @test8vvec(<vscale x 2 x i1> %C, <vscale x 2 x i1> %X)
 }
 
 define i1 @test9(i1 %C, i1 %X) {
-; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[NOT_C]], i1 [[X:%.*]], i1 false
+; CHECK-LABEL: define i1 @test9(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[X:%.*]]) {
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[NOT_C]], i1 [[X]], i1 false
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %R = select i1 %C, i1 false, i1 %X
@@ -88,9 +97,10 @@ define i1 @test9(i1 %C, i1 %X) {
 }
 
 define <2 x i1> @test9vec(<2 x i1> %C, <2 x i1> %X) {
-; CHECK-LABEL: @test9vec(
-; CHECK-NEXT:    [[NOT_C:%.*]] = xor <2 x i1> [[C:%.*]], splat (i1 true)
-; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[NOT_C]], <2 x i1> [[X:%.*]], <2 x i1> zeroinitializer
+; CHECK-LABEL: define <2 x i1> @test9vec(
+; CHECK-SAME: <2 x i1> [[C:%.*]], <2 x i1> [[X:%.*]]) {
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor <2 x i1> [[C]], splat (i1 true)
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[NOT_C]], <2 x i1> [[X]], <2 x i1> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
   %R = select <2 x i1> %C, <2 x i1> <i1 false, i1 false>, <2 x i1> %X
@@ -98,9 +108,10 @@ define <2 x i1> @test9vec(<2 x i1> %C, <2 x i1> %X) {
 }
 
 define <vscale x 2 x i1> @test9vvec(<vscale x 2 x i1> %C, <vscale x 2 x i1> %X) {
-; CHECK-LABEL: @test9vvec(
-; CHECK-NEXT:    [[NOT_C:%.*]] = xor <vscale x 2 x i1> [[C:%.*]], splat (i1 true)
-; CHECK-NEXT:    [[R:%.*]] = select <vscale x 2 x i1> [[NOT_C]], <vscale x 2 x i1> [[X:%.*]], <vscale x 2 x i1> zeroinitializer
+; CHECK-LABEL: define <vscale x 2 x i1> @test9vvec(
+; CHECK-SAME: <vscale x 2 x i1> [[C:%.*]], <vscale x 2 x i1> [[X:%.*]]) {
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor <vscale x 2 x i1> [[C]], splat (i1 true)
+; CHECK-NEXT:    [[R:%.*]] = select <vscale x 2 x i1> [[NOT_C]], <vscale x 2 x i1> [[X]], <vscale x 2 x i1> zeroinitializer
 ; CHECK-NEXT:    ret <vscale x 2 x i1> [[R]]
 ;
   %R = select <vscale x 2 x i1> %C, <vscale x 2 x i1> zeroinitializer, <vscale x 2 x i1> %X
@@ -108,9 +119,10 @@ define <vscale x 2 x i1> @test9vvec(<vscale x 2 x i1> %C, <vscale x 2 x i1> %X)
 }
 
 define i1 @test10(i1 %C, i1 %X) {
-; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[NOT_C]], i1 true, i1 [[X:%.*]]
+; CHECK-LABEL: define i1 @test10(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[X:%.*]]) {
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C]], true
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[NOT_C]], i1 true, i1 [[X]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %R = select i1 %C, i1 %X, i1 true
@@ -118,9 +130,10 @@ define i1 @test10(i1 %C, i1 %X) {
 }
 
 define <2 x i1> @test10vec(<2 x i1> %C, <2 x i1> %X) {
-; CHECK-LABEL: @test10vec(
-; CHECK-NEXT:    [[NOT_C:%.*]] = xor <2 x i1> [[C:%.*]], splat (i1 true)
-; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[NOT_C]], <2 x i1> splat (i1 true), <2 x i1> [[X:%.*]]
+; CHECK-LABEL: define <2 x i1> @test10vec(
+; CHECK-SAME: <2 x i1> [[C:%.*]], <2 x i1> [[X:%.*]]) {
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor <2 x i1> [[C]], splat (i1 true)
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[NOT_C]], <2 x i1> splat (i1 true), <2 x i1> [[X]]
 ; CHECK-NEXT:    ret <2 x i1> [[R]]
 ;
   %R = select <2 x i1> %C, <2 x i1> %X, <2 x i1> <i1 true, i1 true>
@@ -128,8 +141,9 @@ define <2 x i1> @test10vec(<2 x i1> %C, <2 x i1> %X) {
 }
 
 define i1 @test23(i1 %a, i1 %b) {
-; CHECK-LABEL: @test23(
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[A:%.*]], i1 [[B:%.*]], i1 false
+; CHECK-LABEL: define i1 @test23(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i1 [[B]], i1 false
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c = select i1 %a, i1 %b, i1 %a
@@ -137,8 +151,9 @@ define i1 @test23(i1 %a, i1 %b) {
 }
 
 define <2 x i1> @test23vec(<2 x i1> %a, <2 x i1> %b) {
-; CHECK-LABEL: @test23vec(
-; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]], <2 x i1> zeroinitializer
+; CHECK-LABEL: define <2 x i1> @test23vec(
+; CHECK-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A]], <2 x i1> [[B]], <2 x i1> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %c = select <2 x i1> %a, <2 x i1> %b, <2 x i1> %a
@@ -146,8 +161,9 @@ define <2 x i1> @test23vec(<2 x i1> %a, <2 x i1> %b) {
 }
 
 define i1 @test24(i1 %a, i1 %b) {
-; CHECK-LABEL: @test24(
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[B:%.*]]
+; CHECK-LABEL: define i1 @test24(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[A]], i1 true, i1 [[B]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %c = select i1 %a, i1 %a, i1 %b
@@ -155,8 +171,9 @@ define i1 @test24(i1 %a, i1 %b) {
 }
 
 define <2 x i1> @test24vec(<2 x i1> %a, <2 x i1> %b) {
-; CHECK-LABEL: @test24vec(
-; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A:%.*]], <2 x i1> splat (i1 true), <2 x i1> [[B:%.*]]
+; CHECK-LABEL: define <2 x i1> @test24vec(
+; CHECK-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[A]], <2 x i1> splat (i1 true), <2 x i1> [[B]]
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %c = select <2 x i1> %a, <2 x i1> %a, <2 x i1> %b
@@ -164,9 +181,10 @@ define <2 x i1> @test24vec(<2 x i1> %a, <2 x i1> %b) {
 }
 
 define i1 @test62(i1 %A, i1 %B) {
-; CHECK-LABEL: @test62(
-; CHECK-NEXT:    [[NOT_A:%.*]] = xor i1 [[A:%.*]], true
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[NOT_A]], i1 [[B:%.*]], i1 false
+; CHECK-LABEL: define i1 @test62(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B:%.*]]) {
+; CHECK-NEXT:    [[NOT_A:%.*]] = xor i1 [[A]], true
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[NOT_A]], i1 [[B]], i1 false
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %not = xor i1 %A, true
@@ -175,9 +193,10 @@ define i1 @test62(i1 %A, i1 %B) {
 }
 
 define <2 x i1> @test62vec(<2 x i1> %A, <2 x i1> %B) {
-; CHECK-LABEL: @test62vec(
-; CHECK-NEXT:    [[NOT_A:%.*]] = xor <2 x i1> [[A:%.*]], splat (i1 true)
-; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[NOT_A]], <2 x i1> [[B:%.*]], <2 x i1> zeroinitializer
+; CHECK-LABEL: define <2 x i1> @test62vec(
+; CHECK-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]]) {
+; CHECK-NEXT:    [[NOT_A:%.*]] = xor <2 x i1> [[A]], splat (i1 true)
+; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[NOT_A]], <2 x i1> [[B]], <2 x i1> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %not = xor <2 x i1> %A, <i1 true, i1 true>
@@ -186,9 +205,10 @@ define <2 x i1> @test62vec(<2 x i1> %A, <2 x i1> %B) {
 }
 
 define i1 @test63(i1 %A, i1 %B) {
-; CHECK-LABEL: @test63(
-; CHECK-NEXT:    [[NOT_A:%.*]] = xor i1 [[A:%.*]], true
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[NOT_A]], i1 true, i1 [[B:%.*]]
+; CHECK-LABEL: define i1 @test63(
+; CHECK-SAME: i1 [[A:%.*]], i1 [[B:%.*]]) {
+; CHECK-NEXT:    [[NOT_A:%.*]] = xor i1 [[A]], true
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[NOT_A]], i1 true, i1 [[B]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %not = xor i1 %A, true
@@ -197,9 +217,10 @@ define i1 @test63(i1 %A, i1 %B) {
 }
 
 define <2 x i1> @test63vec(<2 x i1> %A, <2 x i1> %B) {
-; CHECK-LABEL: @test63vec(
-; CHECK-NEXT:    [[NOT_A:%.*]] = xor <2 x i1> [[A:%.*]], splat (i1 true)
-; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[NOT_A]], <2 x i1> splat (i1 true), <2 x i1> [[B:%.*]]
+; CHECK-LABEL: define <2 x i1> @test63vec(
+; CHECK-SAME: <2 x i1> [[A:%.*]], <2 x i1> [[B:%.*]]) {
+; CHECK-NEXT:    [[NOT_A:%.*]] = xor <2 x i1> [[A]], splat (i1 true)
+; CHECK-NEXT:    [[C:%.*]] = select <2 x i1> [[NOT_A]], <2 x i1> splat (i1 true), <2 x i1> [[B]]
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %not = xor <2 x i1> %A, <i1 true, i1 true>
@@ -208,8 +229,9 @@ define <2 x i1> @test63vec(<2 x i1> %A, <2 x i1> %B) {
 }
 
 define i32 @test11(i32 %a) {
-; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[A:%.*]], 0
+; CHECK-LABEL: define i32 @test11(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[A]], 0
 ; CHECK-NEXT:    [[R:%.*]] = zext i1 [[C]] to i32
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -219,9 +241,10 @@ define i32 @test11(i32 %a) {
 }
 
 define i32 @test12(i1 %cond, i32 %a) {
-; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[B:%.*]] = zext i1 [[COND:%.*]] to i32
-; CHECK-NEXT:    [[C:%.*]] = or i32 [[A:%.*]], [[B]]
+; CHECK-LABEL: define i32 @test12(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[COND]] to i32
+; CHECK-NEXT:    [[C:%.*]] = or i32 [[A]], [[B]]
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
   %b = or i32 %a, 1
@@ -230,9 +253,10 @@ define i32 @test12(i1 %cond, i32 %a) {
 }
 
 define <2 x i32> @test12vec(<2 x i1> %cond, <2 x i32> %a) {
-; CHECK-LABEL: @test12vec(
-; CHECK-NEXT:    [[B:%.*]] = zext <2 x i1> [[COND:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[C:%.*]] = or <2 x i32> [[A:%.*]], [[B]]
+; CHECK-LABEL: define <2 x i32> @test12vec(
+; CHECK-SAME: <2 x i1> [[COND:%.*]], <2 x i32> [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = zext <2 x i1> [[COND]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = or <2 x i32> [[A]], [[B]]
 ; CHECK-NEXT:    ret <2 x i32> [[C]]
 ;
   %b = or <2 x i32> %a, <i32 1, i32 1>
@@ -241,9 +265,10 @@ define <2 x i32> @test12vec(<2 x i1> %cond, <2 x i32> %a) {
 }
 
 define i32 @test12a(i1 %cond, i32 %a) {
-; CHECK-LABEL: @test12a(
-; CHECK-NEXT:    [[B:%.*]] = zext i1 [[COND:%.*]] to i32
-; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[A:%.*]], [[B]]
+; CHECK-LABEL: define i32 @test12a(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[COND]] to i32
+; CHECK-NEXT:    [[C:%.*]] = ashr i32 [[A]], [[B]]
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
   %b = ashr i32 %a, 1
@@ -252,9 +277,10 @@ define i32 @test12a(i1 %cond, i32 %a) {
 }
 
 define <2 x i32> @test12avec(<2 x i1> %cond, <2 x i32> %a) {
-; CHECK-LABEL: @test12avec(
-; CHECK-NEXT:    [[B:%.*]] = zext <2 x i1> [[COND:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[C:%.*]] = ashr <2 x i32> [[A:%.*]], [[B]]
+; CHECK-LABEL: define <2 x i32> @test12avec(
+; CHECK-SAME: <2 x i1> [[COND:%.*]], <2 x i32> [[A:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = zext <2 x i1> [[COND]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = ashr <2 x i32> [[A]], [[B]]
 ; CHECK-NEXT:    ret <2 x i32> [[C]]
 ;
   %b = ashr <2 x i32> %a, <i32 1, i32 1>
@@ -263,10 +289,11 @@ define <2 x i32> @test12avec(<2 x i1> %cond, <2 x i32> %a) {
 }
 
 define i32 @test12b(i1 %cond, i32 %a) {
-; CHECK-LABEL: @test12b(
-; CHECK-NEXT:    [[NOT_COND:%.*]] = xor i1 [[COND:%.*]], true
+; CHECK-LABEL: define i32 @test12b(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[NOT_COND:%.*]] = xor i1 [[COND]], true
 ; CHECK-NEXT:    [[B:%.*]] = zext i1 [[NOT_COND]] to i32
-; CHECK-NEXT:    [[D:%.*]] = ashr i32 [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = ashr i32 [[A]], [[B]]
 ; CHECK-NEXT:    ret i32 [[D]]
 ;
   %b = ashr i32 %a, 1
@@ -275,10 +302,11 @@ define i32 @test12b(i1 %cond, i32 %a) {
 }
 
 define <2 x i32> @test12bvec(<2 x i1> %cond, <2 x i32> %a) {
-; CHECK-LABEL: @test12bvec(
-; CHECK-NEXT:    [[NOT_COND:%.*]] = xor <2 x i1> [[COND:%.*]], splat (i1 true)
+; CHECK-LABEL: define <2 x i32> @test12bvec(
+; CHECK-SAME: <2 x i1> [[COND:%.*]], <2 x i32> [[A:%.*]]) {
+; CHECK-NEXT:    [[NOT_COND:%.*]] = xor <2 x i1> [[COND]], splat (i1 true)
 ; CHECK-NEXT:    [[B:%.*]] = zext <2 x i1> [[NOT_COND]] to <2 x i32>
-; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[A]], [[B]]
 ; CHECK-NEXT:    ret <2 x i32> [[D]]
 ;
   %b = ashr <2 x i32> %a, <i32 1, i32 1>
@@ -287,8 +315,9 @@ define <2 x i32> @test12bvec(<2 x i1> %cond, <2 x i32> %a) {
 }
 
 define i32 @test13(i32 %a, i32 %b) {
-; CHECK-LABEL: @test13(
-; CHECK-NEXT:    ret i32 [[B:%.*]]
+; CHECK-LABEL: define i32 @test13(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret i32 [[B]]
 ;
   %C = icmp eq i32 %a, %b
   %V = select i1 %C, i32 %a, i32 %b
@@ -296,8 +325,9 @@ define i32 @test13(i32 %a, i32 %b) {
 }
 
 define i32 @test13a(i32 %a, i32 %b) {
-; CHECK-LABEL: @test13a(
-; CHECK-NEXT:    ret i32 [[A:%.*]]
+; CHECK-LABEL: define i32 @test13a(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret i32 [[A]]
 ;
   %C = icmp ne i32 %a, %b
   %V = select i1 %C, i32 %a, i32 %b
@@ -305,8 +335,9 @@ define i32 @test13a(i32 %a, i32 %b) {
 }
 
 define i32 @test13b(i32 %a, i32 %b) {
-; CHECK-LABEL: @test13b(
-; CHECK-NEXT:    ret i32 [[A:%.*]]
+; CHECK-LABEL: define i32 @test13b(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    ret i32 [[A]]
 ;
   %C = icmp eq i32 %a, %b
   %V = select i1 %C, i32 %b, i32 %a
@@ -314,9 +345,10 @@ define i32 @test13b(i32 %a, i32 %b) {
 }
 
 define i1 @test14a(i1 %C, i32 %X) {
-; CHECK-LABEL: @test14a(
-; CHECK-NEXT:    [[R1:%.*]] = icmp slt i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-LABEL: define i1 @test14a(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[R1:%.*]] = icmp slt i32 [[X]], 1
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C]], true
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[NOT_C]], i1 true, i1 [[R1]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -327,9 +359,10 @@ define i1 @test14a(i1 %C, i32 %X) {
 }
 
 define i1 @test14b(i1 %C, i32 %X) {
-; CHECK-LABEL: @test14b(
-; CHECK-NEXT:    [[R1:%.*]] = icmp slt i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[R1]]
+; CHECK-LABEL: define i1 @test14b(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[R1:%.*]] = icmp slt i32 [[X]], 1
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i1 true, i1 [[R1]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %V = select i1 %C, i32 0, i32 %X
@@ -339,8 +372,9 @@ define i1 @test14b(i1 %C, i32 %X) {
 }
 
 define i32 @test16(i1 %C, ptr %P) {
-; CHECK-LABEL: @test16(
-; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-LABEL: define i32 @test16(
+; CHECK-SAME: i1 [[C:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
   %P2 = select i1 %C, ptr %P, ptr null
@@ -350,8 +384,9 @@ define i32 @test16(i1 %C, ptr %P) {
 
 ;; It may be legal to load from a null address in a non-zero address space
 define i32 @test16_neg(i1 %C, ptr addrspace(1) %P) {
-; CHECK-LABEL: @test16_neg(
-; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C:%.*]], ptr addrspace(1) [[P:%.*]], ptr addrspace(1) null
+; CHECK-LABEL: define i32 @test16_neg(
+; CHECK-SAME: i1 [[C:%.*]], ptr addrspace(1) [[P:%.*]]) {
+; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C]], ptr addrspace(1) [[P]], ptr addrspace(1) null
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr addrspace(1) [[P2]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -361,8 +396,9 @@ define i32 @test16_neg(i1 %C, ptr addrspace(1) %P) {
 }
 
 define i32 @test16_neg2(i1 %C, ptr addrspace(1) %P) {
-; CHECK-LABEL: @test16_neg2(
-; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C:%.*]], ptr addrspace(1) null, ptr addrspace(1) [[P:%.*]]
+; CHECK-LABEL: define i32 @test16_neg2(
+; CHECK-SAME: i1 [[C:%.*]], ptr addrspace(1) [[P:%.*]]) {
+; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C]], ptr addrspace(1) null, ptr addrspace(1) [[P]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr addrspace(1) [[P2]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -373,8 +409,9 @@ define i32 @test16_neg2(i1 %C, ptr addrspace(1) %P) {
 
 ;; It may be legal to load from a null address with null pointer valid attribute.
 define i32 @test16_no_null_opt(i1 %C, ptr %P) #0 {
-; CHECK-LABEL: @test16_no_null_opt(
-; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C:%.*]], ptr [[P:%.*]], ptr null
+; CHECK-LABEL: define i32 @test16_no_null_opt(
+; CHECK-SAME: i1 [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C]], ptr [[P]], ptr null
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P2]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -384,8 +421,9 @@ define i32 @test16_no_null_opt(i1 %C, ptr %P) #0 {
 }
 
 define i32 @test16_no_null_opt_2(i1 %C, ptr %P) #0 {
-; CHECK-LABEL: @test16_no_null_opt_2(
-; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C:%.*]], ptr null, ptr [[P:%.*]]
+; CHECK-LABEL: define i32 @test16_no_null_opt_2(
+; CHECK-SAME: i1 [[C:%.*]], ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[P2:%.*]] = select i1 [[C]], ptr null, ptr [[P]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P2]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -397,9 +435,10 @@ define i32 @test16_no_null_opt_2(i1 %C, ptr %P) #0 {
 attributes #0 = { null_pointer_is_valid }
 
 define i1 @test17(ptr %X, i1 %C) {
-; CHECK-LABEL: @test17(
-; CHECK-NEXT:    [[RV1:%.*]] = icmp eq ptr [[X:%.*]], null
-; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-LABEL: define i1 @test17(
+; CHECK-SAME: ptr [[X:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[RV1:%.*]] = icmp eq ptr [[X]], null
+; CHECK-NEXT:    [[NOT_C:%.*]] = xor i1 [[C]], true
 ; CHECK-NEXT:    [[RV:%.*]] = select i1 [[NOT_C]], i1 true, i1 [[RV1]]
 ; CHECK-NEXT:    ret i1 [[RV]]
 ;
@@ -409,8 +448,9 @@ define i1 @test17(ptr %X, i1 %C) {
 }
 
 define i32 @test18(i32 %X, i32 %Y, i1 %C) {
-; CHECK-LABEL: @test18(
-; CHECK-NEXT:    [[V:%.*]] = sdiv i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @test18(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[V:%.*]] = sdiv i32 [[Y]], [[X]]
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
   %R = select i1 %C, i32 %X, i32 0
@@ -419,8 +459,9 @@ define i32 @test18(i32 %X, i32 %Y, i1 %C) {
 }
 
 define i32 @test19(i32 %x) {
-; CHECK-LABEL: @test19(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-LABEL: define i32 @test19(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X]], 31
 ; CHECK-NEXT:    ret i32 [[X_LOBIT]]
 ;
   %t = icmp ugt i32 %x, 2147483647
@@ -429,8 +470,9 @@ define i32 @test19(i32 %x) {
 }
 
 define i32 @test20(i32 %x) {
-; CHECK-LABEL: @test20(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-LABEL: define i32 @test20(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X]], 31
 ; CHECK-NEXT:    ret i32 [[X_LOBIT]]
 ;
   %t = icmp slt i32 %x, 0
@@ -439,8 +481,9 @@ define i32 @test20(i32 %x) {
 }
 
 define i64 @test21(i32 %x) {
-; CHECK-LABEL: @test21(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-LABEL: define i64 @test21(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X]], 31
 ; CHECK-NEXT:    [[RETVAL:%.*]] = sext i32 [[X_LOBIT]] to i64
 ; CHECK-NEXT:    ret i64 [[RETVAL]]
 ;
@@ -450,8 +493,9 @@ define i64 @test21(i32 %x) {
 }
 
 define i16 @test22(i32 %x) {
-; CHECK-LABEL: @test22(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
+; CHECK-LABEL: define i16 @test22(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X]], 31
 ; CHECK-NEXT:    [[RETVAL:%.*]] = trunc nsw i32 [[X_LOBIT]] to i16
 ; CHECK-NEXT:    ret i16 [[RETVAL]]
 ;
@@ -461,13 +505,14 @@ define i16 @test22(i32 %x) {
 }
 
 define i32 @test25(i1 %c)  {
-; CHECK-LABEL: @test25(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
-; CHECK:       jump:
-; CHECK-NEXT:    br label [[RET]]
-; CHECK:       ret:
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ 10, [[JUMP]] ], [ 20, [[ENTRY:%.*]] ]
+; CHECK-LABEL: define i32 @test25(
+; CHECK-SAME: i1 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[C]], label %[[JUMP:.*]], label %[[RET:.*]]
+; CHECK:       [[JUMP]]:
+; CHECK-NEXT:    br label %[[RET]]
+; CHECK:       [[RET]]:
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ 10, %[[JUMP]] ], [ 20, %[[ENTRY]] ]
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
 entry:
@@ -481,13 +526,14 @@ ret:
 }
 
 define i32 @test26(i1 %cond)  {
-; CHECK-LABEL: @test26(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
-; CHECK:       jump:
-; CHECK-NEXT:    br label [[RET]]
-; CHECK:       ret:
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ 20, [[ENTRY:%.*]] ], [ 10, [[JUMP]] ]
+; CHECK-LABEL: define i32 @test26(
+; CHECK-SAME: i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[JUMP:.*]], label %[[RET:.*]]
+; CHECK:       [[JUMP]]:
+; CHECK-NEXT:    br label %[[RET]]
+; CHECK:       [[RET]]:
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ 20, %[[ENTRY]] ], [ 10, %[[JUMP]] ]
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
 entry:
@@ -502,13 +548,14 @@ ret:
 }
 
 define i32 @test26_logical(i1 %cond)  {
-; CHECK-LABEL: @test26_logical(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
-; CHECK:       jump:
-; CHECK-NEXT:    br label [[RET]]
-; CHECK:       ret:
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ 20, [[ENTRY:%.*]] ], [ 10, [[JUMP]] ]
+; CHECK-LABEL: define i32 @test26_logical(
+; CHECK-SAME: i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[JUMP:.*]], label %[[RET:.*]]
+; CHECK:       [[JUMP]]:
+; CHECK-NEXT:    br label %[[RET]]
+; CHECK:       [[RET]]:
+; CHECK-NEXT:    [[B:%.*]] = phi i32 [ 20, %[[ENTRY]] ], [ 10, %[[JUMP]] ]
 ; CHECK-NEXT:    ret i32 [[B]]
 ;
 entry:
@@ -523,13 +570,14 @@ ret:
 }
 
 define i32 @test27(i1 %c, i32 %A, i32 %B)  {
-; CHECK-LABEL: @test27(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
-; CHECK:       jump:
-; CHECK-NEXT:    br label [[RET]]
-; CHECK:       ret:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[A:%.*]], [[JUMP]] ], [ [[B:%.*]], [[ENTRY:%.*]] ]
+; CHECK-LABEL: define i32 @test27(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[C]], label %[[JUMP:.*]], label %[[RET:.*]]
+; CHECK:       [[JUMP]]:
+; CHECK-NEXT:    br label %[[RET]]
+; CHECK:       [[RET]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[A]], %[[JUMP]] ], [ [[B]], %[[ENTRY]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -543,13 +591,14 @@ ret:
 }
 
 define i32 @test28(i1 %cond, i32 %A, i32 %B)  {
-; CHECK-LABEL: @test28(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
-; CHECK:       jump:
-; CHECK-NEXT:    br label [[RET]]
-; CHECK:       ret:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[A:%.*]], [[JUMP]] ], [ [[B:%.*]], [[ENTRY:%.*]] ]
+; CHECK-LABEL: define i32 @test28(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[JUMP:.*]], label %[[RET:.*]]
+; CHECK:       [[JUMP]]:
+; CHECK-NEXT:    br label %[[RET]]
+; CHECK:       [[RET]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[A]], %[[JUMP]] ], [ [[B]], %[[ENTRY]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -564,15 +613,16 @@ ret:
 }
 
 define i32 @test29(i1 %cond, i32 %A, i32 %B)  {
-; CHECK-LABEL: @test29(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[JUMP:%.*]], label [[RET:%.*]]
-; CHECK:       jump:
-; CHECK-NEXT:    br label [[RET]]
-; CHECK:       ret:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[A:%.*]], [[JUMP]] ], [ [[B:%.*]], [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[NEXT:%.*]]
-; CHECK:       next:
+; CHECK-LABEL: define i32 @test29(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[JUMP:.*]], label %[[RET:.*]]
+; CHECK:       [[JUMP]]:
+; CHECK-NEXT:    br label %[[RET]]
+; CHECK:       [[RET]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[A]], %[[JUMP]] ], [ [[B]], %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[NEXT:.*]]
+; CHECK:       [[NEXT]]:
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -591,8 +641,9 @@ next:
 
 ; SMAX(SMAX(x, y), x) -> SMAX(x, y)
 define i32 @test30(i32 %x, i32 %y) {
-; CHECK-LABEL: @test30(
-; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-LABEL: define i32 @test30(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smax.i32(i32 [[X]], i32 [[Y]])
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
   %cmp = icmp sgt i32 %x, %y
@@ -604,8 +655,9 @@ define i32 @test30(i32 %x, i32 %y) {
 
 ; UMAX(UMAX(x, y), x) -> UMAX(x, y)
 define i32 @test31(i32 %x, i32 %y) {
-; CHECK-LABEL: @test31(
-; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.umax.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-LABEL: define i32 @test31(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.umax.i32(i32 [[X]], i32 [[Y]])
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
   %cmp = icmp ugt i32 %x, %y
@@ -617,8 +669,9 @@ define i32 @test31(i32 %x, i32 %y) {
 
 ; SMIN(SMIN(x, y), x) -> SMIN(x, y)
 define i32 @test32(i32 %x, i32 %y) {
-; CHECK-LABEL: @test32(
-; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smin.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-LABEL: define i32 @test32(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = call i32 @llvm.smin.i32(i32 [[X]], i32 [[Y]])
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
   %cmp = icmp sgt i32 %x, %y
@@ -630,8 +683,9 @@ define i32 @test32(i32 %x, i32 %y) {
 
 ; MAX(MIN(x, y), x) -> x
 define i32 @test33(i32 %x, i32 %y) {
-; CHECK-LABEL: @test33(
-; CHECK-NEXT:    ret i32 [[X:%.*]]
+; CHECK-LABEL: define i32 @test33(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    ret i32 [[X]]
 ;
   %cmp = icmp sgt i32 %x, %y
   %cond = select i1 %cmp, i32 %y, i32 %x
@@ -642,8 +696,9 @@ define i32 @test33(i32 %x, i32 %y) {
 
 ; MIN(MAX(x, y), x) -> x
 define i32 @test34(i32 %x, i32 %y) {
-; CHECK-LABEL: @test34(
-; CHECK-NEXT:    ret i32 [[X:%.*]]
+; CHECK-LABEL: define i32 @test34(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    ret i32 [[X]]
 ;
   %cmp = icmp sgt i32 %x, %y
   %cond = select i1 %cmp, i32 %x, i32 %y
@@ -653,7 +708,8 @@ define i32 @test34(i32 %x, i32 %y) {
 }
 
 define i1 @test38(i1 %cond) {
-; CHECK-LABEL: @test38(
+; CHECK-LABEL: define i1 @test38(
+; CHECK-SAME: i1 [[COND:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %zero = alloca i32
@@ -664,7 +720,8 @@ define i1 @test38(i1 %cond) {
 }
 
 define i1 @test39(i1 %cond, double %x) {
-; CHECK-LABEL: @test39(
+; CHECK-LABEL: define i1 @test39(
+; CHECK-SAME: i1 [[COND:%.*]], double [[X:%.*]]) {
 ; CHECK-NEXT:    ret i1 true
 ;
   %s = select i1 %cond, double %x, double 0x7FF0000000000000   ; RHS = +infty
@@ -673,7 +730,8 @@ define i1 @test39(i1 %cond, double %x) {
 }
 
 define i1 @test40(i1 %cond) {
-; CHECK-LABEL: @test40(
+; CHECK-LABEL: define i1 @test40(
+; CHECK-SAME: i1 [[COND:%.*]]) {
 ; CHECK-NEXT:    ret i1 false
 ;
   %a = alloca i32
@@ -685,8 +743,9 @@ define i1 @test40(i1 %cond) {
 }
 
 define i32 @test41(i1 %cond, i32 %x, i32 %y) {
-; CHECK-LABEL: @test41(
-; CHECK-NEXT:    [[R:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @test41(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %z = and i32 %x, %y
@@ -696,10 +755,11 @@ define i32 @test41(i1 %cond, i32 %x, i32 %y) {
 }
 
 define i32 @test42(i32 %x, i32 %y) {
-; CHECK-LABEL: @test42(
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-LABEL: define i32 @test42(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[X]], 0
 ; CHECK-NEXT:    [[B:%.*]] = sext i1 [[COND]] to i32
-; CHECK-NEXT:    [[C:%.*]] = add i32 [[Y:%.*]], [[B]]
+; CHECK-NEXT:    [[C:%.*]] = add i32 [[Y]], [[B]]
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
   %b = add i32 %y, -1
@@ -709,10 +769,11 @@ define i32 @test42(i32 %x, i32 %y) {
 }
 
 define <2 x i32> @test42vec(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @test42vec(
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i32> @test42vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq <2 x i32> [[X]], zeroinitializer
 ; CHECK-NEXT:    [[B:%.*]] = sext <2 x i1> [[COND]] to <2 x i32>
-; CHECK-NEXT:    [[C:%.*]] = add <2 x i32> [[Y:%.*]], [[B]]
+; CHECK-NEXT:    [[C:%.*]] = add <2 x i32> [[Y]], [[B]]
 ; CHECK-NEXT:    ret <2 x i32> [[C]]
 ;
   %b = add <2 x i32> %y, <i32 -1, i32 -1>
@@ -726,8 +787,9 @@ define <2 x i32> @test42vec(<2 x i32> %x, <2 x i32> %y) {
 ; This select instruction can't be eliminated because trying to do so would
 ; change the number of vector elements. This used to assert.
 define i48 @test51(<3 x i1> %icmp, <3 x i16> %t) {
-; CHECK-LABEL: @test51(
-; CHECK-NEXT:    [[SELECT:%.*]] = select <3 x i1> [[ICMP:%.*]], <3 x i16> zeroinitializer, <3 x i16> [[T:%.*]]
+; CHECK-LABEL: define i48 @test51(
+; CHECK-SAME: <3 x i1> [[ICMP:%.*]], <3 x i16> [[T:%.*]]) {
+; CHECK-NEXT:    [[SELECT:%.*]] = select <3 x i1> [[ICMP]], <3 x i16> zeroinitializer, <3 x i16> [[T]]
 ; CHECK-NEXT:    [[T2:%.*]] = bitcast <3 x i16> [[SELECT]] to i48
 ; CHECK-NEXT:    ret i48 [[T2]]
 ;
@@ -737,9 +799,10 @@ define i48 @test51(<3 x i1> %icmp, <3 x i16> %t) {
 }
 
 define <vscale x 4 x float> @bitcast_select_bitcast(<vscale x 4 x i1> %icmp, <vscale x 4 x i32> %a, <vscale x 4 x float> %b) {
-; CHECK-LABEL: @bitcast_select_bitcast(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[A:%.*]] to <vscale x 4 x float>
-; CHECK-NEXT:    [[BC2:%.*]] = select <vscale x 4 x i1> [[ICMP:%.*]], <vscale x 4 x float> [[B:%.*]], <vscale x 4 x float> [[TMP1]]
+; CHECK-LABEL: define <vscale x 4 x float> @bitcast_select_bitcast(
+; CHECK-SAME: <vscale x 4 x i1> [[ICMP:%.*]], <vscale x 4 x i32> [[A:%.*]], <vscale x 4 x float> [[B:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <vscale x 4 x i32> [[A]] to <vscale x 4 x float>
+; CHECK-NEXT:    [[BC2:%.*]] = select <vscale x 4 x i1> [[ICMP]], <vscale x 4 x float> [[B]], <vscale x 4 x float> [[TMP1]]
 ; CHECK-NEXT:    ret <vscale x 4 x float> [[BC2]]
 ;
   %bc1 = bitcast <vscale x 4 x float> %b to <vscale x 4 x i32>
@@ -749,10 +812,11 @@ define <vscale x 4 x float> @bitcast_select_bitcast(<vscale x 4 x i1> %icmp, <vs
 }
 
 define void @select_oneuse_bitcast(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x i32> %c, <vscale x 4 x i32> %d, ptr %ptr1) {
-; CHECK-LABEL: @select_oneuse_bitcast(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <vscale x 4 x i32> [[C:%.*]], [[D:%.*]]
-; CHECK-NEXT:    [[SEL1_V:%.*]] = select <vscale x 4 x i1> [[CMP]], <vscale x 4 x float> [[A:%.*]], <vscale x 4 x float> [[B:%.*]]
-; CHECK-NEXT:    store <vscale x 4 x float> [[SEL1_V]], ptr [[PTR1:%.*]], align 16
+; CHECK-LABEL: define void @select_oneuse_bitcast(
+; CHECK-SAME: <vscale x 4 x float> [[A:%.*]], <vscale x 4 x float> [[B:%.*]], <vscale x 4 x i32> [[C:%.*]], <vscale x 4 x i32> [[D:%.*]], ptr [[PTR1:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <vscale x 4 x i32> [[C]], [[D]]
+; CHECK-NEXT:    [[SEL1_V:%.*]] = select <vscale x 4 x i1> [[CMP]], <vscale x 4 x float> [[A]], <vscale x 4 x float> [[B]]
+; CHECK-NEXT:    store <vscale x 4 x float> [[SEL1_V]], ptr [[PTR1]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %cmp = icmp ult <vscale x 4 x i32> %c, %d
@@ -767,12 +831,13 @@ define void @select_oneuse_bitcast(<vscale x 4 x float> %a, <vscale x 4 x float>
 ; Hoisting the selects allows later pattern matching to see that these are min/max ops.
 
 define void @min_max_bitcast(<4 x float> %a, <4 x float> %b, ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: @min_max_bitcast(
-; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define void @min_max_bitcast(
+; CHECK-SAME: <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], ptr [[PTR1:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x float> [[A]], [[B]]
 ; CHECK-NEXT:    [[SEL1_V:%.*]] = select <4 x i1> [[CMP]], <4 x float> [[A]], <4 x float> [[B]]
 ; CHECK-NEXT:    [[SEL2_V:%.*]] = select <4 x i1> [[CMP]], <4 x float> [[B]], <4 x float> [[A]]
-; CHECK-NEXT:    store <4 x float> [[SEL1_V]], ptr [[PTR1:%.*]], align 16
-; CHECK-NEXT:    store <4 x float> [[SEL2_V]], ptr [[PTR2:%.*]], align 16
+; CHECK-NEXT:    store <4 x float> [[SEL1_V]], ptr [[PTR1]], align 16
+; CHECK-NEXT:    store <4 x float> [[SEL2_V]], ptr [[PTR2]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %cmp = fcmp olt <4 x float> %a, %b
@@ -786,12 +851,13 @@ define void @min_max_bitcast(<4 x float> %a, <4 x float> %b, ptr %ptr1, ptr %ptr
 }
 
 define void @min_max_bitcast1(<vscale x 4 x float> %a, <vscale x 4 x float> %b, ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: @min_max_bitcast1(
-; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <vscale x 4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-LABEL: define void @min_max_bitcast1(
+; CHECK-SAME: <vscale x 4 x float> [[A:%.*]], <vscale x 4 x float> [[B:%.*]], ptr [[PTR1:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <vscale x 4 x float> [[A]], [[B]]
 ; CHECK-NEXT:    [[SEL1_V:%.*]] = select <vscale x 4 x i1> [[CMP]], <vscale x 4 x float> [[A]], <vscale x 4 x float> [[B]]
 ; CHECK-NEXT:    [[SEL2_V:%.*]] = select <vscale x 4 x i1> [[CMP]], <vscale x 4 x float> [[B]], <vscale x 4 x float> [[A]]
-; CHECK-NEXT:    store <vscale x 4 x float> [[SEL1_V]], ptr [[PTR1:%.*]], align 16
-; CHECK-NEXT:    store <vscale x 4 x float> [[SEL2_V]], ptr [[PTR2:%.*]], align 16
+; CHECK-NEXT:    store <vscale x 4 x float> [[SEL1_V]], ptr [[PTR1]], align 16
+; CHECK-NEXT:    store <vscale x 4 x float> [[SEL2_V]], ptr [[PTR2]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %cmp = fcmp olt <vscale x 4 x float> %a, %b
@@ -807,14 +873,15 @@ define void @min_max_bitcast1(<vscale x 4 x float> %a, <vscale x 4 x float> %b,
 ; To avoid potential backend problems, we don't do the same transform for other casts.
 
 define void @truncs_before_selects(<4 x float> %f1, <4 x float> %f2, <4 x i64> %a, <4 x i64> %b, ptr %ptr1, ptr %ptr2) {
-; CHECK-LABEL: @truncs_before_selects(
-; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x float> [[F1:%.*]], [[F2:%.*]]
-; CHECK-NEXT:    [[BC1:%.*]] = trunc <4 x i64> [[A:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[BC2:%.*]] = trunc <4 x i64> [[B:%.*]] to <4 x i32>
+; CHECK-LABEL: define void @truncs_before_selects(
+; CHECK-SAME: <4 x float> [[F1:%.*]], <4 x float> [[F2:%.*]], <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], ptr [[PTR1:%.*]], ptr [[PTR2:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt <4 x float> [[F1]], [[F2]]
+; CHECK-NEXT:    [[BC1:%.*]] = trunc <4 x i64> [[A]] to <4 x i32>
+; CHECK-NEXT:    [[BC2:%.*]] = trunc <4 x i64> [[B]] to <4 x i32>
 ; CHECK-NEXT:    [[SEL1:%.*]] = select <4 x i1> [[CMP]], <4 x i32> [[BC1]], <4 x i32> [[BC2]]
 ; CHECK-NEXT:    [[SEL2:%.*]] = select <4 x i1> [[CMP]], <4 x i32> [[BC2]], <4 x i32> [[BC1]]
-; CHECK-NEXT:    store <4 x i32> [[SEL1]], ptr [[PTR1:%.*]], align 16
-; CHECK-NEXT:    store <4 x i32> [[SEL2]], ptr [[PTR2:%.*]], align 16
+; CHECK-NEXT:    store <4 x i32> [[SEL1]], ptr [[PTR1]], align 16
+; CHECK-NEXT:    store <4 x i32> [[SEL2]], ptr [[PTR2]], align 16
 ; CHECK-NEXT:    ret void
 ;
   %cmp = fcmp olt <4 x float> %f1, %f2
@@ -830,8 +897,9 @@ define void @truncs_before_selects(<4 x float> %f1, <4 x float> %f2, <4 x i64> %
 ; PR8575
 
 define i32 @test52(i32 %n, i32 %m) {
-; CHECK-LABEL: @test52(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N:%.*]], [[M:%.*]]
+; CHECK-LABEL: define i32 @test52(
+; CHECK-SAME: i32 [[N:%.*]], i32 [[M:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[N]], [[M]]
 ; CHECK-NEXT:    [[STOREMERGE:%.*]] = select i1 [[CMP]], i32 1, i32 6
 ; CHECK-NEXT:    ret i32 [[STOREMERGE]]
 ;
@@ -845,8 +913,9 @@ define i32 @test52(i32 %n, i32 %m) {
 ; PR9454
 
 define i32 @test53(i32 %x) {
-; CHECK-LABEL: @test53(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -3
+; CHECK-LABEL: define i32 @test53(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X]], -3
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 2, i32 1
 ; CHECK-NEXT:    ret i32 [[SEL]]
@@ -858,8 +927,9 @@ define i32 @test53(i32 %x) {
 }
 
 define i32 @test54(i32 %X, i32 %Y) {
-; CHECK-LABEL: @test54(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], 0
+; CHECK-LABEL: define i32 @test54(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X]], 0
 ; CHECK-NEXT:    [[C:%.*]] = zext i1 [[B]] to i32
 ; CHECK-NEXT:    ret i32 [[C]]
 ;
@@ -870,8 +940,9 @@ define i32 @test54(i32 %X, i32 %Y) {
 }
 
 define i1 @test55(i1 %X, i32 %Y, i32 %Z) {
-; CHECK-LABEL: @test55(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[Y:%.*]], 0
+; CHECK-LABEL: define i1 @test55(
+; CHECK-SAME: i1 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[Y]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = ashr exact i32 %Y, %Z
@@ -881,8 +952,9 @@ define i1 @test55(i1 %X, i32 %Y, i32 %Z) {
 }
 
 define i32 @test56(i16 %x) {
-; CHECK-LABEL: @test56(
-; CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[X:%.*]] to i32
+; CHECK-LABEL: define i32 @test56(
+; CHECK-SAME: i16 [[X:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[X]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
 ;
   %tobool = icmp eq i16 %x, 0
@@ -892,9 +964,10 @@ define i32 @test56(i16 %x) {
 }
 
 define i32 @test57(i32 %x, i32 %y) {
-; CHECK-LABEL: @test57(
-; CHECK-NEXT:    [[Y:%.*]] = freeze i32 [[Y1:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y]]
+; CHECK-LABEL: define i32 @test57(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y_FR]]
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %and = and i32 %x, %y
@@ -904,8 +977,9 @@ define i32 @test57(i32 %x, i32 %y) {
 }
 
 define i32 @test58(i16 %x) {
-; CHECK-LABEL: @test58(
-; CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[X:%.*]] to i32
+; CHECK-LABEL: define i32 @test58(
+; CHECK-SAME: i16 [[X:%.*]]) {
+; CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[X]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
 ;
   %tobool = icmp ne i16 %x, 1
@@ -915,8 +989,9 @@ define i32 @test58(i16 %x) {
 }
 
 define i32 @test59(i32 %x, i32 %y) {
-; CHECK-LABEL: @test59(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @test59(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %and = and i32 %x, %y
@@ -926,9 +1001,10 @@ define i32 @test59(i32 %x, i32 %y) {
 }
 
 define i1 @test60(i32 %x, ptr %y) {
-; CHECK-LABEL: @test60(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[LOAD:%.*]] = load i1, ptr [[Y:%.*]], align 1
+; CHECK-LABEL: define i1 @test60(
+; CHECK-SAME: i32 [[X:%.*]], ptr [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    [[LOAD:%.*]] = load i1, ptr [[Y]], align 1
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[X]], 1
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i1 [[LOAD]], i1 [[CMP1]]
 ; CHECK-NEXT:    ret i1 [[SEL]]
@@ -942,7 +1018,8 @@ define i1 @test60(i32 %x, ptr %y) {
 
 @glbl = constant i32 10
 define i32 @test61(ptr %ptr) {
-; CHECK-LABEL: @test61(
+; CHECK-LABEL: define i32 @test61(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
 ; CHECK-NEXT:    ret i32 10
 ;
   %A = load i32, ptr %ptr
@@ -953,19 +1030,20 @@ define i32 @test61(ptr %ptr) {
 
 ; PR14131
 define void @test64(i32 %p, i16 %b, i1 %c1) noreturn {
-; CHECK-LABEL: @test64(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C1:%.*]], label [[LOR_RHS:%.*]], label [[LOR_END:%.*]]
-; CHECK:       lor.rhs:
-; CHECK-NEXT:    br label [[LOR_END]]
-; CHECK:       lor.end:
-; CHECK-NEXT:    br i1 poison, label [[COND_END17:%.*]], label [[COND_FALSE16:%.*]]
-; CHECK:       cond.false16:
-; CHECK-NEXT:    br label [[COND_END17]]
-; CHECK:       cond.end17:
-; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
-; CHECK:       while.body:
-; CHECK-NEXT:    br label [[WHILE_BODY]]
+; CHECK-LABEL: define void @test64(
+; CHECK-SAME: i32 [[P:%.*]], i16 [[B:%.*]], i1 [[C1:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[C1]], label %[[LOR_RHS:.*]], label %[[LOR_END:.*]]
+; CHECK:       [[LOR_RHS]]:
+; CHECK-NEXT:    br label %[[LOR_END]]
+; CHECK:       [[LOR_END]]:
+; CHECK-NEXT:    br i1 poison, label %[[COND_END17:.*]], label %[[COND_FALSE16:.*]]
+; CHECK:       [[COND_FALSE16]]:
+; CHECK-NEXT:    br label %[[COND_END17]]
+; CHECK:       [[COND_END17]]:
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    br label %[[WHILE_BODY]]
 ;
 entry:
   %p.addr.0.insert.mask = and i32 %p, -65536
@@ -1003,9 +1081,10 @@ while.body:
 ; select is trivially dereferenceable but may have a lower alignment than the
 ; load does.
 define i32 @test76(i1 %flag, ptr %x) {
-; CHECK-LABEL: @test76(
-; CHECK-NEXT:    store i32 0, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG:%.*]], ptr @under_aligned, ptr [[X]]
+; CHECK-LABEL: define i32 @test76(
+; CHECK-SAME: i1 [[FLAG:%.*]], ptr [[X:%.*]]) {
+; CHECK-NEXT:    store i32 0, ptr [[X]], align 4
+; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG]], ptr @under_aligned, ptr [[X]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -1022,11 +1101,12 @@ declare void @scribble_on_i32(ptr)
 ; load does.
 
 define i32 @test77(i1 %flag, ptr %x) {
-; CHECK-LABEL: @test77(
+; CHECK-LABEL: define i32 @test77(
+; CHECK-SAME: i1 [[FLAG:%.*]], ptr [[X:%.*]]) {
 ; CHECK-NEXT:    [[UNDER_ALIGNED:%.*]] = alloca i32, align 1
 ; CHECK-NEXT:    call void @scribble_on_i32(ptr nonnull [[UNDER_ALIGNED]])
-; CHECK-NEXT:    store i32 0, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG:%.*]], ptr [[UNDER_ALIGNED]], ptr [[X]]
+; CHECK-NEXT:    store i32 0, ptr [[X]], align 4
+; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG]], ptr [[UNDER_ALIGNED]], ptr [[X]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -1041,14 +1121,15 @@ define i32 @test77(i1 %flag, ptr %x) {
 define i32 @test78(i1 %flag, ptr %x, ptr %y, ptr %z) {
 ; Test that we can speculate the loads around the select even when we can't
 ; fold the load completely away.
-; CHECK-LABEL: @test78(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    store i32 0, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    store i32 0, ptr [[Y:%.*]], align 4
-; CHECK-NEXT:    store i32 42, ptr [[Z:%.*]], align 4
+; CHECK-LABEL: define i32 @test78(
+; CHECK-SAME: i1 [[FLAG:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    store i32 0, ptr [[X]], align 4
+; CHECK-NEXT:    store i32 0, ptr [[Y]], align 4
+; CHECK-NEXT:    store i32 42, ptr [[Z]], align 4
 ; CHECK-NEXT:    [[X_VAL:%.*]] = load i32, ptr [[X]], align 4
 ; CHECK-NEXT:    [[Y_VAL:%.*]] = load i32, ptr [[Y]], align 4
-; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], i32 [[X_VAL]], i32 [[Y_VAL]]
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG]], i32 [[X_VAL]], i32 [[Y_VAL]]
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
 entry:
@@ -1064,10 +1145,11 @@ entry:
 ; Test that we can speculate the loads around the select even when we can't
 ; fold the load completely away.
 define i32 @test78_deref(i1 %flag, ptr dereferenceable(4) align 4 %x, ptr dereferenceable(4) align 4 %y, ptr %z) nofree nosync {
-; CHECK-LABEL: @test78_deref(
-; CHECK-NEXT:    [[X_VAL:%.*]] = load i32, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    [[Y_VAL:%.*]] = load i32, ptr [[Y:%.*]], align 4
-; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], i32 [[X_VAL]], i32 [[Y_VAL]]
+; CHECK-LABEL: define i32 @test78_deref(
+; CHECK-SAME: i1 [[FLAG:%.*]], ptr align 4 dereferenceable(4) [[X:%.*]], ptr align 4 dereferenceable(4) [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-NEXT:    [[X_VAL:%.*]] = load i32, ptr [[X]], align 4
+; CHECK-NEXT:    [[Y_VAL:%.*]] = load i32, ptr [[Y]], align 4
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG]], i32 [[X_VAL]], i32 [[Y_VAL]]
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
   %p = select i1 %flag, ptr %x, ptr %y
@@ -1078,11 +1160,12 @@ define i32 @test78_deref(i1 %flag, ptr dereferenceable(4) align 4 %x, ptr derefe
 ; The same as @test78 but we can't speculate the load because it can trap
 ; if under-aligned.
 define i32 @test78_neg(i1 %flag, ptr %x, ptr %y, ptr %z) {
-; CHECK-LABEL: @test78_neg(
-; CHECK-NEXT:    store i32 0, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    store i32 0, ptr [[Y:%.*]], align 4
-; CHECK-NEXT:    store i32 42, ptr [[Z:%.*]], align 4
-; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG:%.*]], ptr [[X]], ptr [[Y]]
+; CHECK-LABEL: define i32 @test78_neg(
+; CHECK-SAME: i1 [[FLAG:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) {
+; CHECK-NEXT:    store i32 0, ptr [[X]], align 4
+; CHECK-NEXT:    store i32 0, ptr [[Y]], align 4
+; CHECK-NEXT:    store i32 42, ptr [[Z]], align 4
+; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG]], ptr [[X]], ptr [[Y]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 16
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -1098,8 +1181,9 @@ define i32 @test78_neg(i1 %flag, ptr %x, ptr %y, ptr %z) {
 ; The same as @test78_deref but we can't speculate the load because
 ; one of the arguments is not sufficiently dereferenceable.
 define i32 @test78_deref_neg(i1 %flag, ptr dereferenceable(2) %x, ptr dereferenceable(4) %y, ptr %z) nofree nosync {
-; CHECK-LABEL: @test78_deref_neg(
-; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]
+; CHECK-LABEL: define i32 @test78_deref_neg(
+; CHECK-SAME: i1 [[FLAG:%.*]], ptr dereferenceable(2) [[X:%.*]], ptr dereferenceable(4) [[Y:%.*]], ptr [[Z:%.*]]) #[[ATTR2]] {
+; CHECK-NEXT:    [[P:%.*]] = select i1 [[FLAG]], ptr [[X]], ptr [[Y]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[P]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -1111,13 +1195,14 @@ define i32 @test78_deref_neg(i1 %flag, ptr dereferenceable(2) %x, ptr dereferenc
 ; Test that we can speculate the loads around the select even when we can't
 ; fold the load completely away.
 define float @test79(i1 %flag, ptr %x, ptr %y, ptr %z) {
-; CHECK-LABEL: @test79(
-; CHECK-NEXT:    store i32 0, ptr [[X:%.*]], align 4
-; CHECK-NEXT:    store i32 0, ptr [[Y:%.*]], align 4
-; CHECK-NEXT:    store i32 42, ptr [[Z:%.*]], align 4
+; CHECK-LABEL: define float @test79(
+; CHECK-SAME: i1 [[FLAG:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]]) {
+; CHECK-NEXT:    store i32 0, ptr [[X]], align 4
+; CHECK-NEXT:    store i32 0, ptr [[Y]], align 4
+; CHECK-NEXT:    store i32 42, ptr [[Z]], align 4
 ; CHECK-NEXT:    [[X_VAL:%.*]] = load float, ptr [[X]], align 4
 ; CHECK-NEXT:    [[Y_VAL:%.*]] = load float, ptr [[Y]], align 4
-; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], float [[X_VAL]], float [[Y_VAL]]
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG]], float [[X_VAL]], float [[Y_VAL]]
 ; CHECK-NEXT:    ret float [[V]]
 ;
   store i32 0, ptr %x
@@ -1132,7 +1217,8 @@ define float @test79(i1 %flag, ptr %x, ptr %y, ptr %z) {
 ; Test that when we speculate the loads around the select they fold throug
 ; load->load folding and load->store folding.
 define i32 @test80(i1 %flag) {
-; CHECK-LABEL: @test80(
+; CHECK-LABEL: define i32 @test80(
+; CHECK-SAME: i1 [[FLAG:%.*]]) {
 ; CHECK-NEXT:    [[X:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @scribble_on_i32(ptr nonnull [[X]])
@@ -1155,7 +1241,8 @@ define i32 @test80(i1 %flag) {
 ; Test that we can speculate the load around the select even though they use
 ; differently typed pointers.
 define float @test81(i1 %flag) {
-; CHECK-LABEL: @test81(
+; CHECK-LABEL: define float @test81(
+; CHECK-SAME: i1 [[FLAG:%.*]]) {
 ; CHECK-NEXT:    [[X:%.*]] = alloca float, align 4
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @scribble_on_i32(ptr nonnull [[X]])
@@ -1179,7 +1266,8 @@ define float @test81(i1 %flag) {
 ; Test that we can speculate the load around the select even though they use
 ; differently typed pointers.
 define i32 @test82(i1 %flag) {
-; CHECK-LABEL: @test82(
+; CHECK-LABEL: define i32 @test82(
+; CHECK-SAME: i1 [[FLAG:%.*]]) {
 ; CHECK-NEXT:    [[X:%.*]] = alloca float, align 4
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @scribble_on_i32(ptr nonnull [[X]])
@@ -1206,7 +1294,8 @@ declare void @scribble_on_i128(ptr)
 ; Test that we can speculate the load around the select even though they use
 ; differently typed pointers and requires inttoptr casts.
 define ptr @test83(i1 %flag) {
-; CHECK-LABEL: @test83(
+; CHECK-LABEL: define ptr @test83(
+; CHECK-SAME: i1 [[FLAG:%.*]]) {
 ; CHECK-NEXT:    [[X:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i64, align 8
 ; CHECK-NEXT:    call void @scribble_on_i64(ptr nonnull [[X]])
@@ -1230,7 +1319,8 @@ define ptr @test83(i1 %flag) {
 ; Test that we can speculate the load around the select even though they use
 ; differently typed pointers and requires a ptrtoint cast.
 define i64 @test84(i1 %flag) {
-; CHECK-LABEL: @test84(
+; CHECK-LABEL: define i64 @test84(
+; CHECK-SAME: i1 [[FLAG:%.*]]) {
 ; CHECK-NEXT:    [[X:%.*]] = alloca ptr, align 8
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i64, align 8
 ; CHECK-NEXT:    call void @scribble_on_i64(ptr nonnull [[X]])
@@ -1255,7 +1345,8 @@ define i64 @test84(i1 %flag) {
 ; pointer doesn't load all of the stored integer bits. We could fix this, but it
 ; would require endianness checks and other nastiness.
 define ptr @test85(i1 %flag) {
-; CHECK-LABEL: @test85(
+; CHECK-LABEL: define ptr @test85(
+; CHECK-SAME: i1 [[FLAG:%.*]]) {
 ; CHECK-NEXT:    [[X:%.*]] = alloca [2 x ptr], align 8
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i128, align 8
 ; CHECK-NEXT:    call void @scribble_on_i128(ptr nonnull [[X]])
@@ -1264,7 +1355,7 @@ define ptr @test85(i1 %flag) {
 ; CHECK-NEXT:    store i128 [[T]], ptr [[Y]], align 4
 ; CHECK-NEXT:    [[X_VAL:%.*]] = load ptr, ptr [[X]], align 8
 ; CHECK-NEXT:    [[Y_VAL:%.*]] = load ptr, ptr [[Y]], align 8
-; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], ptr [[X_VAL]], ptr [[Y_VAL]]
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG]], ptr [[X_VAL]], ptr [[Y_VAL]]
 ; CHECK-NEXT:    ret ptr [[V]]
 ;
   %x = alloca [2 x ptr]
@@ -1282,7 +1373,8 @@ define ptr @test85(i1 %flag) {
 ; is larger than the pointer size. The store of the pointer doesn't store to all
 ; the bits of the integer.
 define i128 @test86(i1 %flag) {
-; CHECK-LABEL: @test86(
+; CHECK-LABEL: define i128 @test86(
+; CHECK-SAME: i1 [[FLAG:%.*]]) {
 ; CHECK-NEXT:    [[X:%.*]] = alloca [2 x ptr], align 8
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i128, align 8
 ; CHECK-NEXT:    call void @scribble_on_i128(ptr nonnull [[X]])
@@ -1291,7 +1383,7 @@ define i128 @test86(i1 %flag) {
 ; CHECK-NEXT:    store ptr [[T]], ptr [[Y]], align 8
 ; CHECK-NEXT:    [[X_VAL:%.*]] = load i128, ptr [[X]], align 4
 ; CHECK-NEXT:    [[Y_VAL:%.*]] = load i128, ptr [[Y]], align 4
-; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG:%.*]], i128 [[X_VAL]], i128 [[Y_VAL]]
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[FLAG]], i128 [[X_VAL]], i128 [[Y_VAL]]
 ; CHECK-NEXT:    ret i128 [[V]]
 ;
   %x = alloca [2 x ptr]
@@ -1306,10 +1398,11 @@ define i128 @test86(i1 %flag) {
 }
 
 define i32 @test_select_select0(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
-; CHECK-LABEL: @test_select_select0(
-; CHECK-NEXT:    [[C0_NOT:%.*]] = icmp slt i32 [[A:%.*]], [[V1:%.*]]
-; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0_NOT]], i32 [[R1:%.*]], i32 [[R0:%.*]]
-; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A]], [[V2:%.*]]
+; CHECK-LABEL: define i32 @test_select_select0(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[R0:%.*]], i32 [[R1:%.*]], i32 [[V1:%.*]], i32 [[V2:%.*]]) {
+; CHECK-NEXT:    [[C0_NOT:%.*]] = icmp slt i32 [[A]], [[V1]]
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0_NOT]], i32 [[R1]], i32 [[R0]]
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A]], [[V2]]
 ; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], i32 [[S0]], i32 [[R1]]
 ; CHECK-NEXT:    ret i32 [[S1]]
 ;
@@ -1321,10 +1414,11 @@ define i32 @test_select_select0(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
 }
 
 define i32 @test_select_select1(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
-; CHECK-LABEL: @test_select_select1(
-; CHECK-NEXT:    [[C0_NOT:%.*]] = icmp slt i32 [[A:%.*]], [[V1:%.*]]
-; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0_NOT]], i32 [[R1:%.*]], i32 [[R0:%.*]]
-; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A]], [[V2:%.*]]
+; CHECK-LABEL: define i32 @test_select_select1(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[R0:%.*]], i32 [[R1:%.*]], i32 [[V1:%.*]], i32 [[V2:%.*]]) {
+; CHECK-NEXT:    [[C0_NOT:%.*]] = icmp slt i32 [[A]], [[V1]]
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C0_NOT]], i32 [[R1]], i32 [[R0]]
+; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A]], [[V2]]
 ; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], i32 [[R0]], i32 [[S0]]
 ; CHECK-NEXT:    ret i32 [[S1]]
 ;
@@ -1336,8 +1430,9 @@ define i32 @test_select_select1(i32 %a, i32 %r0, i32 %r1, i32 %v1, i32 %v2) {
 }
 
 define i32 @PR23757(i32 %x) {
-; CHECK-LABEL: @PR23757(
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 1
+; CHECK-LABEL: define i32 @PR23757(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], 1
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
   %cmp = icmp eq i32 %x, 2147483647
@@ -1347,7 +1442,8 @@ define i32 @PR23757(i32 %x) {
 }
 
 define i32 @PR23757_swapped(i32 %x) {
-; CHECK-LABEL: @PR23757_swapped(
+; CHECK-LABEL: define i32 @PR23757_swapped(
+; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i32 -2147483648
 ;
   %cmp = icmp eq i32 %x, 2147483647
@@ -1357,9 +1453,10 @@ define i32 @PR23757_swapped(i32 %x) {
 }
 
 define i32 @PR23757_ne(i32 %x, ptr %p) {
-; CHECK-LABEL: @PR23757_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    store i1 [[CMP]], ptr [[P:%.*]], align 1
+; CHECK-LABEL: define i32 @PR23757_ne(
+; CHECK-SAME: i32 [[X:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X]], 2147483647
+; CHECK-NEXT:    store i1 [[CMP]], ptr [[P]], align 1
 ; CHECK-NEXT:    ret i32 -2147483648
 ;
   %cmp = icmp ne i32 %x, 2147483647
@@ -1370,9 +1467,10 @@ define i32 @PR23757_ne(i32 %x, ptr %p) {
 }
 
 define i32 @PR23757_ne_swapped(i32 %x, ptr %p) {
-; CHECK-LABEL: @PR23757_ne_swapped(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 2147483647
-; CHECK-NEXT:    store i1 [[CMP]], ptr [[P:%.*]], align 1
+; CHECK-LABEL: define i32 @PR23757_ne_swapped(
+; CHECK-SAME: i32 [[X:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X]], 2147483647
+; CHECK-NEXT:    store i1 [[CMP]], ptr [[P]], align 1
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], 1
 ; CHECK-NEXT:    ret i32 [[ADD]]
 ;
@@ -1386,8 +1484,9 @@ define i32 @PR23757_ne_swapped(i32 %x, ptr %p) {
 ; max(max(~a, -1), -1) --> ~min(a, 0)
 
 define i32 @PR27137(i32 %a) {
-; CHECK-LABEL: @PR27137(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.smin.i32(i32 [[A:%.*]], i32 0)
+; CHECK-LABEL: define i32 @PR27137(
+; CHECK-SAME: i32 [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.smin.i32(i32 [[A]], i32 0)
 ; CHECK-NEXT:    [[S1:%.*]] = xor i32 [[TMP1]], -1
 ; CHECK-NEXT:    ret i32 [[S1]]
 ;
@@ -1401,8 +1500,9 @@ define i32 @PR27137(i32 %a) {
 
 ; ub-safe negation pattern
 define i32 @PR27817(i32 %x) {
-; CHECK-LABEL: @PR27817(
-; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-LABEL: define i32 @PR27817(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[X]]
 ; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %cmp = icmp eq i32 %x, -2147483648
@@ -1412,8 +1512,9 @@ define i32 @PR27817(i32 %x) {
 }
 
 define i32 @PR27817_nsw(i32 %x) {
-; CHECK-LABEL: @PR27817_nsw(
-; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-LABEL: define i32 @PR27817_nsw(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[X]]
 ; CHECK-NEXT:    ret i32 [[SUB]]
 ;
   %cmp = icmp eq i32 %x, -2147483648
@@ -1423,8 +1524,9 @@ define i32 @PR27817_nsw(i32 %x) {
 }
 
 define <2 x i32> @PR27817_nsw_vec(<2 x i32> %x) {
-; CHECK-LABEL: @PR27817_nsw_vec(
-; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i32> zeroinitializer, [[X:%.*]]
+; CHECK-LABEL: define <2 x i32> @PR27817_nsw_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i32> zeroinitializer, [[X]]
 ; CHECK-NEXT:    ret <2 x i32> [[SUB]]
 ;
   %cmp = icmp eq <2 x i32> %x, <i32 -2147483648, i32 -2147483648>
@@ -1434,8 +1536,9 @@ define <2 x i32> @PR27817_nsw_vec(<2 x i32> %x) {
 }
 
 define i32 @select_icmp_slt0_xor(i32 %x) {
-; CHECK-LABEL: @select_icmp_slt0_xor(
-; CHECK-NEXT:    [[X_XOR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-LABEL: define i32 @select_icmp_slt0_xor(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_XOR:%.*]] = or i32 [[X]], -2147483648
 ; CHECK-NEXT:    ret i32 [[X_XOR]]
 ;
   %cmp = icmp slt i32 %x, zeroinitializer
@@ -1445,8 +1548,9 @@ define i32 @select_icmp_slt0_xor(i32 %x) {
 }
 
 define <2 x i32> @select_icmp_slt0_xor_vec(<2 x i32> %x) {
-; CHECK-LABEL: @select_icmp_slt0_xor_vec(
-; CHECK-NEXT:    [[X_XOR:%.*]] = or <2 x i32> [[X:%.*]], splat (i32 -2147483648)
+; CHECK-LABEL: define <2 x i32> @select_icmp_slt0_xor_vec(
+; CHECK-SAME: <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[X_XOR:%.*]] = or <2 x i32> [[X]], splat (i32 -2147483648)
 ; CHECK-NEXT:    ret <2 x i32> [[X_XOR]]
 ;
   %cmp = icmp slt <2 x i32> %x, zeroinitializer
@@ -1456,8 +1560,9 @@ define <2 x i32> @select_icmp_slt0_xor_vec(<2 x i32> %x) {
 }
 
 define <4 x i32> @canonicalize_to_shuffle(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-LABEL: @canonicalize_to_shuffle(
-; CHECK-NEXT:    [[SEL:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-LABEL: define <4 x i32> @canonicalize_to_shuffle(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
 ; CHECK-NEXT:    ret <4 x i32> [[SEL]]
 ;
   %sel = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> %a, <4 x i32> %b
@@ -1469,8 +1574,9 @@ define <4 x i32> @canonicalize_to_shuffle(<4 x i32> %a, <4 x i32> %b) {
 ; https://bugs.llvm.org/show_bug.cgi?id=32486
 
 define <4 x i32> @undef_elts_in_condition(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-LABEL: @undef_elts_in_condition(
-; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> <i1 true, i1 undef, i1 false, i1 undef>, <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]
+; CHECK-LABEL: define <4 x i32> @undef_elts_in_condition(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> <i1 true, i1 undef, i1 false, i1 undef>, <4 x i32> [[A]], <4 x i32> [[B]]
 ; CHECK-NEXT:    ret <4 x i32> [[SEL]]
 ;
   %sel = select <4 x i1> <i1 true, i1 undef, i1 false, i1 undef>, <4 x i32> %a, <4 x i32> %b
@@ -1482,8 +1588,9 @@ define <4 x i32> @undef_elts_in_condition(<4 x i32> %a, <4 x i32> %b) {
 @g = global i32 0
 
 define <4 x i32> @cannot_canonicalize_to_shuffle1(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-LABEL: @cannot_canonicalize_to_shuffle1(
-; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> bitcast (i4 ptrtoint (ptr @g to i4) to <4 x i1>), <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]
+; CHECK-LABEL: define <4 x i32> @cannot_canonicalize_to_shuffle1(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> bitcast (i4 ptrtoint (ptr @g to i4) to <4 x i1>), <4 x i32> [[A]], <4 x i32> [[B]]
 ; CHECK-NEXT:    ret <4 x i32> [[SEL]]
 ;
   %sel = select <4 x i1> bitcast (i4 ptrtoint (ptr @g to i4) to <4 x i1>), <4 x i32> %a, <4 x i32> %b
@@ -1491,8 +1598,9 @@ define <4 x i32> @cannot_canonicalize_to_shuffle1(<4 x i32> %a, <4 x i32> %b) {
 }
 
 define <4 x i32> @cannot_canonicalize_to_shuffle2(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-LABEL: @cannot_canonicalize_to_shuffle2(
-; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> <i1 true, i1 undef, i1 false, i1 ptrtoint (ptr @g to i1)>, <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]
+; CHECK-LABEL: define <4 x i32> @cannot_canonicalize_to_shuffle2(
+; CHECK-SAME: <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select <4 x i1> <i1 true, i1 undef, i1 false, i1 ptrtoint (ptr @g to i1)>, <4 x i32> [[A]], <4 x i32> [[B]]
 ; CHECK-NEXT:    ret <4 x i32> [[SEL]]
 ;
   %sel = select <4 x i1> <i1 true, i1 undef, i1 false, i1 ptrtoint (ptr @g to i1)>, <4 x i32> %a, <4 x i32> %b
@@ -1502,9 +1610,10 @@ define <4 x i32> @cannot_canonicalize_to_shuffle2(<4 x i32> %a, <4 x i32> %b) {
 declare void @llvm.assume(i1)
 
 define i8 @assume_cond_true(i1 %cond, i8 %x, i8 %y) {
-; CHECK-LABEL: @assume_cond_true(
-; CHECK-NEXT:    call void @llvm.assume(i1 [[COND:%.*]])
-; CHECK-NEXT:    ret i8 [[X:%.*]]
+; CHECK-LABEL: define i8 @assume_cond_true(
+; CHECK-SAME: i1 [[COND:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    call void @llvm.assume(i1 [[COND]])
+; CHECK-NEXT:    ret i8 [[X]]
 ;
   call void @llvm.assume(i1 %cond)
   %sel = select i1 %cond, i8 %x, i8 %y
@@ -1514,10 +1623,11 @@ define i8 @assume_cond_true(i1 %cond, i8 %x, i8 %y) {
 ; computeKnownBitsFromAssume() understands the 'not' of an assumed condition.
 
 define i8 @assume_cond_false(i1 %cond, i8 %x, i8 %y) {
-; CHECK-LABEL: @assume_cond_false(
-; CHECK-NEXT:    [[NOTCOND:%.*]] = xor i1 [[COND:%.*]], true
+; CHECK-LABEL: define i8 @assume_cond_false(
+; CHECK-SAME: i1 [[COND:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[NOTCOND:%.*]] = xor i1 [[COND]], true
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[NOTCOND]])
-; CHECK-NEXT:    ret i8 [[Y:%.*]]
+; CHECK-NEXT:    ret i8 [[Y]]
 ;
   %notcond = xor i1 %cond, true
   call void @llvm.assume(i1 %notcond)
@@ -1527,9 +1637,10 @@ define i8 @assume_cond_false(i1 %cond, i8 %x, i8 %y) {
 
 ; Test case to make sure we don't consider an all ones float values for converting the select into a sext.
 define <4 x float> @PR33721(<4 x float> %w) {
-; CHECK-LABEL: @PR33721(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fcmp ole <4 x float> [[W:%.*]], zeroinitializer
+; CHECK-LABEL: define <4 x float> @PR33721(
+; CHECK-SAME: <4 x float> [[W:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = fcmp ole <4 x float> [[W]], zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = select <4 x i1> [[TMP0]], <4 x float> splat (float 0xFFFFFFFFE0000000), <4 x float> zeroinitializer
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
@@ -1541,9 +1652,10 @@ entry:
 
 ; select(C, binop(select(C, X, Y), W), Z) -> select(C, binop(X, W), Z)
 define i8 @test87(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @test87(
-; CHECK-NEXT:    [[B:%.*]] = add i8 [[X:%.*]], [[W:%.*]]
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND:%.*]], i8 [[B]], i8 [[Z:%.*]]
+; CHECK-LABEL: define i8 @test87(
+; CHECK-SAME: i1 [[COND:%.*]], i8 [[W:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = add i8 [[X]], [[W]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND]], i8 [[B]], i8 [[Z]]
 ; CHECK-NEXT:    ret i8 [[C]]
 ;
   %a = select i1 %cond, i8 %x, i8 %y
@@ -1554,9 +1666,10 @@ define i8 @test87(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
 
 ; select(C, binop(select(C, X, Y), W), Z) -> select(C, Z, binop(Y, W))
 define i8 @test88(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @test88(
-; CHECK-NEXT:    [[B:%.*]] = sub i8 [[Y:%.*]], [[W:%.*]]
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND:%.*]], i8 [[Z:%.*]], i8 [[B]]
+; CHECK-LABEL: define i8 @test88(
+; CHECK-SAME: i1 [[COND:%.*]], i8 [[W:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = sub i8 [[Y]], [[W]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND]], i8 [[Z]], i8 [[B]]
 ; CHECK-NEXT:    ret i8 [[C]]
 ;
   %a = select i1 %cond, i8 %x, i8 %y
@@ -1567,9 +1680,10 @@ define i8 @test88(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
 
 ; select(C, Z, binop(W, select(C, X, Y))) -> select(C, binop(X, W), Z)
 define i8 @test89(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @test89(
-; CHECK-NEXT:    [[B:%.*]] = and i8 [[W:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND:%.*]], i8 [[B]], i8 [[Z:%.*]]
+; CHECK-LABEL: define i8 @test89(
+; CHECK-SAME: i1 [[COND:%.*]], i8 [[W:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[W]], [[X]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND]], i8 [[B]], i8 [[Z]]
 ; CHECK-NEXT:    ret i8 [[C]]
 ;
   %a = select i1 %cond, i8 %x, i8 %y
@@ -1580,9 +1694,10 @@ define i8 @test89(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
 
 ; select(C, Z, binop(W, select(C, X, Y))) -> select(C, Z, binop(W, Y))
 define i8 @test90(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @test90(
-; CHECK-NEXT:    [[B:%.*]] = or i8 [[W:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND:%.*]], i8 [[Z:%.*]], i8 [[B]]
+; CHECK-LABEL: define i8 @test90(
+; CHECK-SAME: i1 [[COND:%.*]], i8 [[W:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[B:%.*]] = or i8 [[W]], [[Y]]
+; CHECK-NEXT:    [[C:%.*]] = select i1 [[COND]], i8 [[Z]], i8 [[B]]
 ; CHECK-NEXT:    ret i8 [[C]]
 ;
   %a = select i1 %cond, i8 %x, i8 %y
@@ -1592,8 +1707,9 @@ define i8 @test90(i1 %cond, i8 %w, i8 %x, i8 %y, i8 %z) {
 }
 
 define i32 @test_shl_zext_bool(i1 %t) {
-; CHECK-LABEL: @test_shl_zext_bool(
-; CHECK-NEXT:    [[R:%.*]] = select i1 [[T:%.*]], i32 4, i32 0
+; CHECK-LABEL: define i32 @test_shl_zext_bool(
+; CHECK-SAME: i1 [[T:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[T]], i32 4, i32 0
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %r = select i1 %t, i32 4, i32 0
@@ -1601,8 +1717,9 @@ define i32 @test_shl_zext_bool(i1 %t) {
 }
 
 define <2 x i32> @test_shl_zext_bool_splat(<2 x i1> %t) {
-; CHECK-LABEL: @test_shl_zext_bool_splat(
-; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T:%.*]], <2 x i32> splat (i32 8), <2 x i32> zeroinitializer
+; CHECK-LABEL: define <2 x i32> @test_shl_zext_bool_splat(
+; CHECK-SAME: <2 x i1> [[T:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T]], <2 x i32> splat (i32 8), <2 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i32> [[R]]
 ;
   %r = select <2 x i1> %t, <2 x i32> <i32 8, i32 8>, <2 x i32> zeroinitializer
@@ -1610,8 +1727,9 @@ define <2 x i32> @test_shl_zext_bool_splat(<2 x i1> %t) {
 }
 
 define <2 x i32> @test_shl_zext_bool_vec(<2 x i1> %t) {
-; CHECK-LABEL: @test_shl_zext_bool_vec(
-; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T:%.*]], <2 x i32> <i32 4, i32 8>, <2 x i32> zeroinitializer
+; CHECK-LABEL: define <2 x i32> @test_shl_zext_bool_vec(
+; CHECK-SAME: <2 x i1> [[T:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[T]], <2 x i32> <i32 4, i32 8>, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i32> [[R]]
 ;
   %r = select <2 x i1> %t, <2 x i32> <i32 4, i32 8>, <2 x i32> zeroinitializer
@@ -1619,8 +1737,9 @@ define <2 x i32> @test_shl_zext_bool_vec(<2 x i1> %t) {
 }
 
 define float @copysign1(float %x) {
-; CHECK-LABEL: @copysign1(
-; CHECK-NEXT:    [[R:%.*]] = call float @llvm.copysign.f32(float 1.000000e+00, float [[X:%.*]])
+; CHECK-LABEL: define float @copysign1(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call float @llvm.copysign.f32(float 1.000000e+00, float [[X]])
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %i = bitcast float %x to i32
@@ -1630,8 +1749,9 @@ define float @copysign1(float %x) {
 }
 
 define float @copysign1_fmf(float %x) {
-; CHECK-LABEL: @copysign1_fmf(
-; CHECK-NEXT:    [[R:%.*]] = call float @llvm.copysign.f32(float 1.000000e+00, float [[X:%.*]])
+; CHECK-LABEL: define float @copysign1_fmf(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call float @llvm.copysign.f32(float 1.000000e+00, float [[X]])
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %i = bitcast float %x to i32
@@ -1641,8 +1761,9 @@ define float @copysign1_fmf(float %x) {
 }
 
 define <2 x float> @copysign2(<2 x float> %x) {
-; CHECK-LABEL: @copysign2(
-; CHECK-NEXT:    [[TMP1:%.*]] = fneg <2 x float> [[X:%.*]]
+; CHECK-LABEL: define <2 x float> @copysign2(
+; CHECK-SAME: <2 x float> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg <2 x float> [[X]]
 ; CHECK-NEXT:    [[R:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> splat (float 4.200000e+01), <2 x float> [[TMP1]])
 ; CHECK-NEXT:    ret <2 x float> [[R]]
 ;
@@ -1653,8 +1774,9 @@ define <2 x float> @copysign2(<2 x float> %x) {
 }
 
 define float @copysign3(float %x) {
-; CHECK-LABEL: @copysign3(
-; CHECK-NEXT:    [[TMP1:%.*]] = fneg float [[X:%.*]]
+; CHECK-LABEL: define float @copysign3(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg float [[X]]
 ; CHECK-NEXT:    [[R:%.*]] = call float @llvm.copysign.f32(float 4.300000e+01, float [[TMP1]])
 ; CHECK-NEXT:    ret float [[R]]
 ;
@@ -1665,8 +1787,9 @@ define float @copysign3(float %x) {
 }
 
 define <2 x float> @copysign_vec_poison(<2 x float> %x) {
-; CHECK-LABEL: @copysign_vec_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = fneg <2 x float> [[X:%.*]]
+; CHECK-LABEL: define <2 x float> @copysign_vec_poison(
+; CHECK-SAME: <2 x float> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = fneg <2 x float> [[X]]
 ; CHECK-NEXT:    [[R:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> splat (float 4.200000e+01), <2 x float> [[TMP1]])
 ; CHECK-NEXT:    ret <2 x float> [[R]]
 ;
@@ -1677,8 +1800,9 @@ define <2 x float> @copysign_vec_poison(<2 x float> %x) {
 }
 
 define <2 x float> @copysign_vec_poison1(<2 x float> %x) {
-; CHECK-LABEL: @copysign_vec_poison1(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> splat (float 4.200000e+01), <2 x float> [[X:%.*]])
+; CHECK-LABEL: define <2 x float> @copysign_vec_poison1(
+; CHECK-SAME: <2 x float> [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> splat (float 4.200000e+01), <2 x float> [[X]])
 ; CHECK-NEXT:    ret <2 x float> [[R]]
 ;
   %i = bitcast <2 x float> %x to <2 x i32>
@@ -1688,8 +1812,9 @@ define <2 x float> @copysign_vec_poison1(<2 x float> %x) {
 }
 
 define <2 x float> @copysign_vec_poison3(<2 x float> %x) {
-; CHECK-LABEL: @copysign_vec_poison3(
-; CHECK-NEXT:    [[R:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> splat (float 4.200000e+01), <2 x float> [[X:%.*]])
+; CHECK-LABEL: define <2 x float> @copysign_vec_poison3(
+; CHECK-SAME: <2 x float> [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = call <2 x float> @llvm.copysign.v2f32(<2 x float> splat (float 4.200000e+01), <2 x float> [[X]])
 ; CHECK-NEXT:    ret <2 x float> [[R]]
 ;
   %i = bitcast <2 x float> %x to <2 x i32>
@@ -1703,8 +1828,9 @@ declare void @use1(i1)
 ; Negative test
 
 define float @copysign_extra_use(float %x) {
-; CHECK-LABEL: @copysign_extra_use(
-; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-LABEL: define float @copysign_extra_use(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[ISNEG:%.*]] = icmp slt i32 [[I]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[ISNEG]])
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISNEG]], float -4.400000e+01, float 4.400000e+01
@@ -1720,8 +1846,9 @@ define float @copysign_extra_use(float %x) {
 ; Negative test
 
 define float @copysign_type_mismatch(double %x) {
-; CHECK-LABEL: @copysign_type_mismatch(
-; CHECK-NEXT:    [[I:%.*]] = bitcast double [[X:%.*]] to i64
+; CHECK-LABEL: define float @copysign_type_mismatch(
+; CHECK-SAME: double [[X:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = bitcast double [[X]] to i64
 ; CHECK-NEXT:    [[ISPOS:%.*]] = icmp sgt i64 [[I]], -1
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISPOS]], float 1.000000e+00, float -1.000000e+00
 ; CHECK-NEXT:    ret float [[R]]
@@ -1735,8 +1862,9 @@ define float @copysign_type_mismatch(double %x) {
 ; Negative test
 
 define <2 x float> @copysign_type_mismatch2(<2 x float> %x) {
-; CHECK-LABEL: @copysign_type_mismatch2(
-; CHECK-NEXT:    [[I:%.*]] = bitcast <2 x float> [[X:%.*]] to i64
+; CHECK-LABEL: define <2 x float> @copysign_type_mismatch2(
+; CHECK-SAME: <2 x float> [[X:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = bitcast <2 x float> [[X]] to i64
 ; CHECK-NEXT:    [[ISPOS:%.*]] = icmp sgt i64 [[I]], -1
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISPOS]], <2 x float> splat (float 1.000000e+00), <2 x float> splat (float -1.000000e+00)
 ; CHECK-NEXT:    ret <2 x float> [[R]]
@@ -1750,8 +1878,9 @@ define <2 x float> @copysign_type_mismatch2(<2 x float> %x) {
 ; Negative test
 
 define float @copysign_wrong_cmp(float %x) {
-; CHECK-LABEL: @copysign_wrong_cmp(
-; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-LABEL: define float @copysign_wrong_cmp(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[ISPOS:%.*]] = icmp sgt i32 [[I]], 0
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISPOS]], float 1.000000e+00, float -1.000000e+00
 ; CHECK-NEXT:    ret float [[R]]
@@ -1765,8 +1894,9 @@ define float @copysign_wrong_cmp(float %x) {
 ; Negative test
 
 define float @copysign_wrong_const(float %x) {
-; CHECK-LABEL: @copysign_wrong_const(
-; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X:%.*]] to i32
+; CHECK-LABEL: define float @copysign_wrong_const(
+; CHECK-SAME: float [[X:%.*]]) {
+; CHECK-NEXT:    [[I:%.*]] = bitcast float [[X]] to i32
 ; CHECK-NEXT:    [[ISPOS:%.*]] = icmp sgt i32 [[I]], -1
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[ISPOS]], float 2.000000e+00, float -1.000000e+00
 ; CHECK-NEXT:    ret float [[R]]
@@ -1779,15 +1909,16 @@ define float @copysign_wrong_const(float %x) {
 
 ; TODO: we can replace select with a Phi.
 define i32 @select_dominating_cond(i1 %cond, i32 %x, i32 %y) {
-; CHECK-LABEL: @select_dominating_cond(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[Y:%.*]], [[IF_FALSE]] ], [ [[X:%.*]], [[IF_TRUE]] ]
+; CHECK-LABEL: define i32 @select_dominating_cond(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[Y]], %[[IF_FALSE]] ], [ [[X]], %[[IF_TRUE]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -1805,15 +1936,16 @@ merge:
 }
 
 define i32 @select_dominating_inverted(i1 %cond, i32 %x, i32 %y) {
-; CHECK-LABEL: @select_dominating_inverted(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X:%.*]], [[IF_FALSE]] ], [ [[Y:%.*]], [[IF_TRUE]] ]
+; CHECK-LABEL: define i32 @select_dominating_inverted(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_FALSE:.*]], label %[[IF_TRUE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X]], %[[IF_FALSE]] ], [ [[Y]], %[[IF_TRUE]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -1833,23 +1965,24 @@ merge:
 
 ; More complex CFG: the block with select has multiple predecessors.
 define i32 @select_dominating_cond_multiple_preds(i1 %cond, i1 %cond2, i1 %cond3, i32 %x, i32 %y) {
-; CHECK-LABEL: @select_dominating_cond_multiple_preds(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[IF_TRUE_1:%.*]], label [[IF_TRUE_2:%.*]]
-; CHECK:       if.true.1:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.true.2:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[IF_FALSE_1:%.*]], label [[EXIT:%.*]]
-; CHECK:       if.false.1:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[Y:%.*]], [[IF_FALSE_1]] ], [ [[X:%.*]], [[IF_TRUE_2]] ], [ [[X]], [[IF_TRUE_1]] ]
+; CHECK-LABEL: define i32 @select_dominating_cond_multiple_preds(
+; CHECK-SAME: i1 [[COND:%.*]], i1 [[COND2:%.*]], i1 [[COND3:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br i1 [[COND2]], label %[[IF_TRUE_1:.*]], label %[[IF_TRUE_2:.*]]
+; CHECK:       [[IF_TRUE_1]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_TRUE_2]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br i1 [[COND3]], label %[[IF_FALSE_1:.*]], label %[[EXIT:.*]]
+; CHECK:       [[IF_FALSE_1]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[Y]], %[[IF_FALSE_1]] ], [ [[X]], %[[IF_TRUE_2]] ], [ [[X]], %[[IF_TRUE_1]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
-; CHECK:       exit:
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -1880,23 +2013,24 @@ exit:
 
 ; More complex CFG for inverted case: the block with select has multiple predecessors.
 define i32 @select_dominating_cond_inverted_multiple_preds(i1 %cond, i1 %cond2, i1 %cond3, i32 %x, i32 %y) {
-; CHECK-LABEL: @select_dominating_cond_inverted_multiple_preds(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[IF_TRUE_1:%.*]], label [[IF_TRUE_2:%.*]]
-; CHECK:       if.true.1:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.true.2:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[IF_FALSE_1:%.*]], label [[EXIT:%.*]]
-; CHECK:       if.false.1:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X:%.*]], [[IF_FALSE_1]] ], [ [[Y:%.*]], [[IF_TRUE_2]] ], [ [[Y]], [[IF_TRUE_1]] ]
+; CHECK-LABEL: define i32 @select_dominating_cond_inverted_multiple_preds(
+; CHECK-SAME: i1 [[COND:%.*]], i1 [[COND2:%.*]], i1 [[COND3:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_FALSE:.*]], label %[[IF_TRUE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br i1 [[COND2]], label %[[IF_TRUE_1:.*]], label %[[IF_TRUE_2:.*]]
+; CHECK:       [[IF_TRUE_1]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_TRUE_2]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br i1 [[COND3]], label %[[IF_FALSE_1:.*]], label %[[EXIT:.*]]
+; CHECK:       [[IF_FALSE_1]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X]], %[[IF_FALSE_1]] ], [ [[Y]], %[[IF_TRUE_2]] ], [ [[Y]], %[[IF_TRUE_1]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
-; CHECK:       exit:
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -1928,25 +2062,26 @@ exit:
 
 ; More complex CFG for inverted case: the block with select has multiple predecessors that can duplicate.
 define i32 @select_dominating_cond_inverted_multiple_duplicating_preds(i1 %cond, i32 %cond2, i1 %cond3, i32 %x, i32 %y) {
-; CHECK-LABEL: @select_dominating_cond_inverted_multiple_duplicating_preds(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    switch i32 [[COND2:%.*]], label [[SWITCH_CASE_1:%.*]] [
-; CHECK-NEXT:      i32 1, label [[MERGE:%.*]]
-; CHECK-NEXT:      i32 2, label [[MERGE]]
-; CHECK-NEXT:      i32 3, label [[MERGE]]
+; CHECK-LABEL: define i32 @select_dominating_cond_inverted_multiple_duplicating_preds(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[COND2:%.*]], i1 [[COND3:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_FALSE:.*]], label %[[IF_TRUE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    switch i32 [[COND2]], label %[[SWITCH_CASE_1:.*]] [
+; CHECK-NEXT:      i32 1, label %[[MERGE:.*]]
+; CHECK-NEXT:      i32 2, label %[[MERGE]]
+; CHECK-NEXT:      i32 3, label %[[MERGE]]
 ; CHECK-NEXT:    ]
-; CHECK:       switch.case.1:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[IF_FALSE_1:%.*]], label [[EXIT:%.*]]
-; CHECK:       if.false.1:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X:%.*]], [[IF_FALSE_1]] ], [ [[Y:%.*]], [[SWITCH_CASE_1]] ], [ [[Y]], [[IF_TRUE]] ], [ [[Y]], [[IF_TRUE]] ], [ [[Y]], [[IF_TRUE]] ]
+; CHECK:       [[SWITCH_CASE_1]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br i1 [[COND3]], label %[[IF_FALSE_1:.*]], label %[[EXIT:.*]]
+; CHECK:       [[IF_FALSE_1]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X]], %[[IF_FALSE_1]] ], [ [[Y]], %[[SWITCH_CASE_1]] ], [ [[Y]], %[[IF_TRUE]] ], [ [[Y]], %[[IF_TRUE]] ], [ [[Y]], %[[IF_TRUE]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
-; CHECK:       exit:
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -1979,17 +2114,18 @@ exit:
 
 ; Negative test: currently we take condition from IDom, but might be willing to expand it in the future.
 define i32 @select_not_imm_dominating_cond_neg(i1 %cond, i32 %x, i32 %y) {
-; CHECK-LABEL: @select_not_imm_dominating_cond_neg(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-LABEL: define i32 @select_not_imm_dominating_cond_neg(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -2011,27 +2147,28 @@ exit:
 
 ; Shows how we can leverage dominance to eliminate duplicating selects.
 define i32 @select_dominance_chain(i1 %cond, i32 %x, i32 %y) {
-; CHECK-LABEL: @select_dominance_chain(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE_1:%.*]], label [[IF_FALSE_1:%.*]]
-; CHECK:       if.true.1:
-; CHECK-NEXT:    br label [[MERGE_1:%.*]]
-; CHECK:       if.false.1:
-; CHECK-NEXT:    br label [[MERGE_1]]
-; CHECK:       merge.1:
-; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE_2:%.*]], label [[IF_FALSE_2:%.*]]
-; CHECK:       if.true.2:
-; CHECK-NEXT:    br label [[MERGE_2:%.*]]
-; CHECK:       if.false.2:
-; CHECK-NEXT:    br label [[MERGE_2]]
-; CHECK:       merge.2:
-; CHECK-NEXT:    br i1 [[COND]], label [[IF_TRUE_3:%.*]], label [[IF_FALSE_3:%.*]]
-; CHECK:       if.true.3:
-; CHECK-NEXT:    br label [[MERGE_3:%.*]]
-; CHECK:       if.false.3:
-; CHECK-NEXT:    br label [[MERGE_3]]
-; CHECK:       merge.3:
-; CHECK-NEXT:    [[S_1:%.*]] = phi i32 [ [[Y:%.*]], [[IF_FALSE_3]] ], [ [[X:%.*]], [[IF_TRUE_3]] ]
+; CHECK-LABEL: define i32 @select_dominance_chain(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE_1:.*]], label %[[IF_FALSE_1:.*]]
+; CHECK:       [[IF_TRUE_1]]:
+; CHECK-NEXT:    br label %[[MERGE_1:.*]]
+; CHECK:       [[IF_FALSE_1]]:
+; CHECK-NEXT:    br label %[[MERGE_1]]
+; CHECK:       [[MERGE_1]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE_2:.*]], label %[[IF_FALSE_2:.*]]
+; CHECK:       [[IF_TRUE_2]]:
+; CHECK-NEXT:    br label %[[MERGE_2:.*]]
+; CHECK:       [[IF_FALSE_2]]:
+; CHECK-NEXT:    br label %[[MERGE_2]]
+; CHECK:       [[MERGE_2]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE_3:.*]], label %[[IF_FALSE_3:.*]]
+; CHECK:       [[IF_TRUE_3]]:
+; CHECK-NEXT:    br label %[[MERGE_3:.*]]
+; CHECK:       [[IF_FALSE_3]]:
+; CHECK-NEXT:    br label %[[MERGE_3]]
+; CHECK:       [[MERGE_3]]:
+; CHECK-NEXT:    [[S_1:%.*]] = phi i32 [ [[Y]], %[[IF_FALSE_3]] ], [ [[X]], %[[IF_TRUE_3]] ]
 ; CHECK-NEXT:    [[SUM_2:%.*]] = mul i32 [[S_1]], 3
 ; CHECK-NEXT:    ret i32 [[SUM_2]]
 ;
@@ -2074,15 +2211,16 @@ merge.3:
 ; TODO: We can replace select with a Phi and then sink a and b to respective
 ; branches.
 define i32 @select_dominating_cond_and_sink(i1 %cond, i32 %x, i32 %y) {
-; CHECK-LABEL: @select_dominating_cond_and_sink(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[B:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @select_dominating_cond_and_sink(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[A:%.*]] = add i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 [[A]], i32 [[B]]
 ; CHECK-NEXT:    ret i32 [[S]]
@@ -2104,11 +2242,12 @@ merge:
 }
 
 define i32 @select_dominating_cond_same_labels(i1 %cond) {
-; CHECK-LABEL: @select_dominating_cond_same_labels(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[EXIT]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[RESULT:%.*]] = select i1 [[COND:%.*]], i32 123, i32 456
+; CHECK-LABEL: define i32 @select_dominating_cond_same_labels(
+; CHECK-SAME: i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RESULT:%.*]] = select i1 [[COND]], i32 123, i32 456
 ; CHECK-NEXT:    ret i32 [[RESULT]]
 ;
 entry:
@@ -2119,15 +2258,16 @@ exit:
 }
 
 define i32 @select_phi_same_condition(i1 %cond, i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_phi_same_condition(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X:%.*]], [[IF_TRUE]] ], [ [[Z:%.*]], [[IF_FALSE]] ]
+; CHECK-LABEL: define i32 @select_phi_same_condition(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X]], %[[IF_TRUE]] ], [ [[Z]], %[[IF_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -2148,17 +2288,18 @@ merge:
 
 ; TODO: Replace with phi[a, c] and sink them to respective branches.
 define i32 @select_phi_same_condition_sink(i1 %cond, i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @select_phi_same_condition_sink(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    [[B:%.*]] = mul i32 [[X:%.*]], [[Z:%.*]]
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[IF_TRUE]] ], [ [[B]], [[IF_FALSE]] ]
-; CHECK-NEXT:    [[A:%.*]] = add i32 [[X]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @select_phi_same_condition_sink(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    [[B:%.*]] = mul i32 [[X]], [[Z]]
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, %[[IF_TRUE]] ], [ [[B]], %[[IF_FALSE]] ]
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 [[A]], i32 [[PHI]]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
@@ -2183,14 +2324,15 @@ declare i32 @__gxx_personality_v0(...)
 declare i1 @foo()
 
 define i32 @test_invoke_neg(i32 %x, i32 %y) nounwind uwtable ssp personality ptr @__gxx_personality_v0 {
-; CHECK-LABEL: @test_invoke_neg(
-; CHECK-NEXT:  entry:
+; CHECK-LABEL: define i32 @test_invoke_neg(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR4:[0-9]+]] personality ptr @__gxx_personality_v0 {
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[COND:%.*]] = invoke i1 @foo()
-; CHECK-NEXT:            to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]]
-; CHECK:       invoke.cont:
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT:            to label %[[INVOKE_CONT:.*]] unwind label %[[LPAD:.*]]
+; CHECK:       [[INVOKE_CONT]]:
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 [[X]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
-; CHECK:       lpad:
+; CHECK:       [[LPAD]]:
 ; CHECK-NEXT:    [[LP:%.*]] = landingpad { i1, i32 }
 ; CHECK-NEXT:            filter [0 x i1] zeroinitializer
 ; CHECK-NEXT:    unreachable
@@ -2212,19 +2354,20 @@ lpad:
 declare i32 @bar()
 
 define i32 @test_invoke_2_neg(i1 %cond, i32 %x, i32 %y) nounwind uwtable ssp personality ptr @__gxx_personality_v0 {
-; CHECK-LABEL: @test_invoke_2_neg(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
+; CHECK-LABEL: define i32 @test_invoke_2_neg(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR4]] personality ptr @__gxx_personality_v0 {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
 ; CHECK-NEXT:    [[RESULT:%.*]] = invoke i32 @bar()
-; CHECK-NEXT:            to label [[MERGE]] unwind label [[LPAD:%.*]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[IF_TRUE]] ], [ [[RESULT]], [[IF_FALSE]] ]
+; CHECK-NEXT:            to label %[[MERGE]] unwind label %[[LPAD:.*]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ 0, %[[IF_TRUE]] ], [ [[RESULT]], %[[IF_FALSE]] ]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 1, i32 [[PHI]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
-; CHECK:       lpad:
+; CHECK:       [[LPAD]]:
 ; CHECK-NEXT:    [[LP:%.*]] = landingpad { i1, i32 }
 ; CHECK-NEXT:            filter [0 x i1] zeroinitializer
 ; CHECK-NEXT:    unreachable
@@ -2251,20 +2394,21 @@ lpad:
 }
 
 define i32 @select_phi_same_condition_switch(i1 %cond, i32 %x, i32 %y) {
-; CHECK-LABEL: @select_phi_same_condition_switch(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    switch i32 [[X:%.*]], label [[EXIT:%.*]] [
-; CHECK-NEXT:      i32 1, label [[MERGE:%.*]]
-; CHECK-NEXT:      i32 2, label [[MERGE]]
+; CHECK-LABEL: define i32 @select_phi_same_condition_switch(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    switch i32 [[X]], label %[[EXIT:.*]] [
+; CHECK-NEXT:      i32 1, label %[[MERGE:.*]]
+; CHECK-NEXT:      i32 2, label %[[MERGE]]
 ; CHECK-NEXT:    ]
-; CHECK:       exit:
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 0
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X]], [[IF_TRUE]] ], [ [[X]], [[IF_TRUE]] ], [ [[Y:%.*]], [[IF_FALSE]] ]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ [[X]], %[[IF_TRUE]] ], [ [[X]], %[[IF_TRUE]] ], [ [[Y]], %[[IF_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
 entry:
@@ -2289,21 +2433,22 @@ merge:
 }
 
 define i32 @transit_different_values_through_phi(i1 %cond, i1 %cond2) {
-; CHECK-LABEL: @transit_different_values_through_phi(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[IF_TRUE_1:%.*]], label [[IF_TRUE_2:%.*]]
-; CHECK:       if.true.1:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.true.2:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[S:%.*]] = phi i32 [ 1, [[IF_TRUE_1]] ], [ 2, [[IF_TRUE_2]] ], [ 3, [[IF_FALSE]] ]
+; CHECK-LABEL: define i32 @transit_different_values_through_phi(
+; CHECK-SAME: i1 [[COND:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br i1 [[COND2]], label %[[IF_TRUE_1:.*]], label %[[IF_TRUE_2:.*]]
+; CHECK:       [[IF_TRUE_1]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_TRUE_2]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[S:%.*]] = phi i32 [ 1, %[[IF_TRUE_1]] ], [ 2, %[[IF_TRUE_2]] ], [ 3, %[[IF_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[S]]
-; CHECK:       exit:
+; CHECK:       [[EXIT:.*:]]
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
@@ -2331,16 +2476,17 @@ exit:
 }
 
 define i32 @select_phi_degenerate(i1 %cond, i1 %cond2) {
-; CHECK-LABEL: @select_phi_degenerate(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[LOOP:%.*]], label [[EXIT:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[SELECT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_INC:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[IV_INC]] = add i32 [[SELECT]], 1
-; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[LOOP]], label [[EXIT2:%.*]]
-; CHECK:       exit:
+; CHECK-LABEL: define i32 @select_phi_degenerate(
+; CHECK-SAME: i1 [[COND:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP:.*]], label %[[EXIT:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_INC]] = add i32 [[IV]], 1
+; CHECK-NEXT:    br i1 [[COND2]], label %[[LOOP]], label %[[EXIT2:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 0
-; CHECK:       exit2:
+; CHECK:       [[EXIT2]]:
 ; CHECK-NEXT:    ret i32 [[IV_INC]]
 ;
 entry:
@@ -2360,17 +2506,18 @@ exit2:
 }
 
 define i32 @test_select_into_phi_not_idom(i1 %cond, i32 %A, i32 %B)  {
-; CHECK-LABEL: @test_select_into_phi_not_idom(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret i32 [[A:%.*]]
+; CHECK-LABEL: define i32 @test_select_into_phi_not_idom(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[A]]
 ;
 entry:
   br i1 %cond, label %if.true, label %if.false
@@ -2391,17 +2538,18 @@ exit:
 }
 
 define i32 @test_select_into_phi_not_idom_2(i1 %cond, i32 %A, i32 %B)  {
-; CHECK-LABEL: @test_select_into_phi_not_idom_2(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret i32 [[B:%.*]]
+; CHECK-LABEL: define i32 @test_select_into_phi_not_idom_2(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret i32 [[B]]
 ;
 entry:
   br i1 %cond, label %if.true, label %if.false
@@ -2422,17 +2570,18 @@ exit:
 }
 
 define i32 @test_select_into_phi_not_idom_inverted(i1 %cond, i32 %A, i32 %B)  {
-; CHECK-LABEL: @test_select_into_phi_not_idom_inverted(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[A:%.*]], [[IF_TRUE]] ], [ [[B:%.*]], [[IF_FALSE]] ]
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-LABEL: define i32 @test_select_into_phi_not_idom_inverted(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_FALSE:.*]], label %[[IF_TRUE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[A]], %[[IF_TRUE]] ], [ [[B]], %[[IF_FALSE]] ]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
 entry:
@@ -2455,17 +2604,18 @@ exit:
 }
 
 define i32 @test_select_into_phi_not_idom_inverted_2(i1 %cond, i32 %A, i32 %B)  {
-; CHECK-LABEL: @test_select_into_phi_not_idom_inverted_2(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_FALSE:%.*]], label [[IF_TRUE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[A:%.*]], [[IF_TRUE]] ], [ [[B:%.*]], [[IF_FALSE]] ]
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-LABEL: define i32 @test_select_into_phi_not_idom_inverted_2(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_FALSE:.*]], label %[[IF_TRUE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[A]], %[[IF_TRUE]] ], [ [[B]], %[[IF_FALSE]] ]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
 entry:
@@ -2488,18 +2638,19 @@ exit:
 }
 
 define i32 @test_select_into_phi_not_idom_no_dom_input_1(i1 %cond, i32 %A, i32 %B, ptr %p)  {
-; CHECK-LABEL: @test_select_into_phi_not_idom_no_dom_input_1(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[C]], [[IF_TRUE]] ], [ [[A:%.*]], [[IF_FALSE]] ]
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-LABEL: define i32 @test_select_into_phi_not_idom_no_dom_input_1(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[C]], %[[IF_TRUE]] ], [ [[A]], %[[IF_FALSE]] ]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
 entry:
@@ -2522,18 +2673,19 @@ exit:
 }
 
 define i32 @test_select_into_phi_not_idom_no_dom_input_2(i1 %cond, i32 %A, i32 %B, ptr %p)  {
-; CHECK-LABEL: @test_select_into_phi_not_idom_no_dom_input_2(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
-; CHECK:       if.true:
-; CHECK-NEXT:    br label [[MERGE:%.*]]
-; CHECK:       if.false:
-; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[P:%.*]], align 4
-; CHECK-NEXT:    br label [[MERGE]]
-; CHECK:       merge:
-; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[B:%.*]], [[IF_TRUE]] ], [ [[C]], [[IF_FALSE]] ]
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-LABEL: define i32 @test_select_into_phi_not_idom_no_dom_input_2(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF_TRUE:.*]], label %[[IF_FALSE:.*]]
+; CHECK:       [[IF_TRUE]]:
+; CHECK-NEXT:    br label %[[MERGE:.*]]
+; CHECK:       [[IF_FALSE]]:
+; CHECK-NEXT:    [[C:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    br label %[[MERGE]]
+; CHECK:       [[MERGE]]:
+; CHECK-NEXT:    [[SEL:%.*]] = phi i32 [ [[B]], %[[IF_TRUE]] ], [ [[C]], %[[IF_FALSE]] ]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
 entry:
@@ -2560,8 +2712,9 @@ exit:
 ; https://lists.llvm.org/pipermail/llvm-dev/2016-October/106182.html
 ; https://reviews.llvm.org/D83360
 define i32 @false_undef(i1 %cond, i32 %x) {
-; CHECK-LABEL: @false_undef(
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], i32 [[X:%.*]], i32 undef
+; CHECK-LABEL: define i32 @false_undef(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 [[X]], i32 undef
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
   %s = select i1 %cond, i32 %x, i32 undef
@@ -2569,8 +2722,9 @@ define i32 @false_undef(i1 %cond, i32 %x) {
 }
 
 define i32 @true_undef(i1 %cond, i32 %x) {
-; CHECK-LABEL: @true_undef(
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], i32 undef, i32 [[X:%.*]]
+; CHECK-LABEL: define i32 @true_undef(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], i32 undef, i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
   %s = select i1 %cond, i32 undef, i32 %x
@@ -2578,8 +2732,9 @@ define i32 @true_undef(i1 %cond, i32 %x) {
 }
 
 define <2 x i32> @false_undef_vec(i1 %cond, <2 x i32> %x) {
-; CHECK-LABEL: @false_undef_vec(
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], <2 x i32> [[X:%.*]], <2 x i32> undef
+; CHECK-LABEL: define <2 x i32> @false_undef_vec(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], <2 x i32> [[X]], <2 x i32> undef
 ; CHECK-NEXT:    ret <2 x i32> [[S]]
 ;
   %s = select i1 %cond, <2 x i32> %x, <2 x i32> undef
@@ -2587,8 +2742,9 @@ define <2 x i32> @false_undef_vec(i1 %cond, <2 x i32> %x) {
 }
 
 define <2 x i32> @true_undef_vec(i1 %cond, <2 x i32> %x) {
-; CHECK-LABEL: @true_undef_vec(
-; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND:%.*]], <2 x i32> undef, <2 x i32> [[X:%.*]]
+; CHECK-LABEL: define <2 x i32> @true_undef_vec(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = select i1 [[COND]], <2 x i32> undef, <2 x i32> [[X]]
 ; CHECK-NEXT:    ret <2 x i32> [[S]]
 ;
   %s = select i1 %cond, <2 x i32> undef, <2 x i32> %x
@@ -2596,8 +2752,9 @@ define <2 x i32> @true_undef_vec(i1 %cond, <2 x i32> %x) {
 }
 
 define i8 @cond_freeze(i8 %x, i8 %y) {
-; CHECK-LABEL: @cond_freeze(
-; CHECK-NEXT:    ret i8 [[Y:%.*]]
+; CHECK-LABEL: define i8 @cond_freeze(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    ret i8 [[Y]]
 ;
   %cond.fr = freeze i1 undef
   %s = select i1 %cond.fr, i8 %x, i8 %y
@@ -2605,7 +2762,8 @@ define i8 @cond_freeze(i8 %x, i8 %y) {
 }
 
 define i8 @cond_freeze_constant_false_val(i8 %x) {
-; CHECK-LABEL: @cond_freeze_constant_false_val(
+; CHECK-LABEL: define i8 @cond_freeze_constant_false_val(
+; CHECK-SAME: i8 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i8 1
 ;
   %cond.fr = freeze i1 undef
@@ -2614,7 +2772,8 @@ define i8 @cond_freeze_constant_false_val(i8 %x) {
 }
 
 define i8 @cond_freeze_constant_true_val(i8 %x) {
-; CHECK-LABEL: @cond_freeze_constant_true_val(
+; CHECK-LABEL: define i8 @cond_freeze_constant_true_val(
+; CHECK-SAME: i8 [[X:%.*]]) {
 ; CHECK-NEXT:    ret i8 1
 ;
   %cond.fr = freeze i1 undef
@@ -2623,7 +2782,7 @@ define i8 @cond_freeze_constant_true_val(i8 %x) {
 }
 
 define i8 @cond_freeze_both_arms_constant() {
-; CHECK-LABEL: @cond_freeze_both_arms_constant(
+; CHECK-LABEL: define i8 @cond_freeze_both_arms_constant() {
 ; CHECK-NEXT:    ret i8 42
 ;
   %cond.fr = freeze i1 undef
@@ -2632,7 +2791,8 @@ define i8 @cond_freeze_both_arms_constant() {
 }
 
 define <2 x i8> @cond_freeze_constant_true_val_vec(<2 x i8> %x) {
-; CHECK-LABEL: @cond_freeze_constant_true_val_vec(
+; CHECK-LABEL: define <2 x i8> @cond_freeze_constant_true_val_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i8> <i8 1, i8 2>
 ;
   %cond.fr = freeze <2 x i1> <i1 undef, i1 undef>
@@ -2641,7 +2801,8 @@ define <2 x i8> @cond_freeze_constant_true_val_vec(<2 x i8> %x) {
 }
 
 define <2 x i8> @partial_cond_freeze_constant_true_val_vec(<2 x i8> %x) {
-; CHECK-LABEL: @partial_cond_freeze_constant_true_val_vec(
+; CHECK-LABEL: define <2 x i8> @partial_cond_freeze_constant_true_val_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i8> <i8 1, i8 2>
 ;
   %cond.fr = freeze <2 x i1> <i1 true, i1 undef>
@@ -2650,8 +2811,9 @@ define <2 x i8> @partial_cond_freeze_constant_true_val_vec(<2 x i8> %x) {
 }
 
 define <2 x i8> @partial_cond_freeze_constant_false_val_vec(<2 x i8> %x) {
-; CHECK-LABEL: @partial_cond_freeze_constant_false_val_vec(
-; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
+; CHECK-LABEL: define <2 x i8> @partial_cond_freeze_constant_false_val_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    ret <2 x i8> [[X]]
 ;
   %cond.fr = freeze <2 x i1> <i1 true, i1 undef>
   %s = select <2 x i1> %cond.fr, <2 x i8> %x, <2 x i8> <i8 1, i8 2>
@@ -2659,7 +2821,7 @@ define <2 x i8> @partial_cond_freeze_constant_false_val_vec(<2 x i8> %x) {
 }
 
 define <2 x i8> @partial_cond_freeze_both_arms_constant_vec() {
-; CHECK-LABEL: @partial_cond_freeze_both_arms_constant_vec(
+; CHECK-LABEL: define <2 x i8> @partial_cond_freeze_both_arms_constant_vec() {
 ; CHECK-NEXT:    ret <2 x i8> <i8 42, i8 43>
 ;
   %cond.fr = freeze <2 x i1> <i1 false, i1 undef>
@@ -2670,8 +2832,9 @@ define <2 x i8> @partial_cond_freeze_both_arms_constant_vec() {
 declare void @foo2(i8, i8)
 
 define void @cond_freeze_multipleuses(i8 %x, i8 %y) {
-; CHECK-LABEL: @cond_freeze_multipleuses(
-; CHECK-NEXT:    call void @foo2(i8 [[Y:%.*]], i8 [[X:%.*]])
+; CHECK-LABEL: define void @cond_freeze_multipleuses(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    call void @foo2(i8 [[Y]], i8 [[X]])
 ; CHECK-NEXT:    ret void
 ;
   %cond.fr = freeze i1 undef
@@ -2682,8 +2845,10 @@ define void @cond_freeze_multipleuses(i8 %x, i8 %y) {
 }
 
 define i32 @select_freeze_icmp_eq(i32 %x, i32 %y) {
-; CHECK-LABEL: @select_freeze_icmp_eq(
-; CHECK-NEXT:    ret i32 [[Y:%.*]]
+; CHECK-LABEL: define i32 @select_freeze_icmp_eq(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    ret i32 [[Y_FR]]
 ;
   %c = icmp eq i32 %x, %y
   %c.fr = freeze i1 %c
@@ -2692,8 +2857,10 @@ define i32 @select_freeze_icmp_eq(i32 %x, i32 %y) {
 }
 
 define i32 @select_freeze_icmp_ne(i32 %x, i32 %y) {
-; CHECK-LABEL: @select_freeze_icmp_ne(
-; CHECK-NEXT:    ret i32 [[X:%.*]]
+; CHECK-LABEL: define i32 @select_freeze_icmp_ne(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i32 [[X]]
+; CHECK-NEXT:    ret i32 [[X_FR]]
 ;
   %c = icmp ne i32 %x, %y
   %c.fr = freeze i1 %c
@@ -2702,10 +2869,11 @@ define i32 @select_freeze_icmp_ne(i32 %x, i32 %y) {
 }
 
 define i32 @select_freeze_icmp_else(i32 %x, i32 %y) {
-; CHECK-LABEL: @select_freeze_icmp_else(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[C_FR:%.*]] = freeze i1 [[C]]
-; CHECK-NEXT:    [[V:%.*]] = select i1 [[C_FR]], i32 [[X]], i32 [[Y]]
+; CHECK-LABEL: define i32 @select_freeze_icmp_else(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i32 [[X]]
+; CHECK-NEXT:    [[V:%.*]] = call i32 @llvm.umin.i32(i32 [[X_FR]], i32 [[Y_FR]])
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
   %c = icmp ult i32 %x, %y
@@ -2717,10 +2885,11 @@ define i32 @select_freeze_icmp_else(i32 %x, i32 %y) {
 declare void @use_i1_i32(i1, i32)
 
 define void @select_freeze_icmp_multuses(i32 %x, i32 %y) {
-; CHECK-LABEL: @select_freeze_icmp_multuses(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[C_FR:%.*]] = freeze i1 [[C]]
-; CHECK-NEXT:    [[V:%.*]] = select i1 [[C_FR]], i32 [[X]], i32 [[Y]]
+; CHECK-LABEL: define void @select_freeze_icmp_multuses(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    [[V:%.*]] = freeze i32 [[X]]
+; CHECK-NEXT:    [[C_FR:%.*]] = icmp ne i32 [[V]], [[Y_FR]]
 ; CHECK-NEXT:    call void @use_i1_i32(i1 [[C_FR]], i32 [[V]])
 ; CHECK-NEXT:    ret void
 ;
@@ -2732,8 +2901,9 @@ define void @select_freeze_icmp_multuses(i32 %x, i32 %y) {
 }
 
 define i32 @pr47322_more_poisonous_replacement(i32 %arg) {
-; CHECK-LABEL: @pr47322_more_poisonous_replacement(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[ARG:%.*]], 0
+; CHECK-LABEL: define i32 @pr47322_more_poisonous_replacement(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[ARG]], 0
 ; CHECK-NEXT:    [[TRAILING:%.*]] = call range(i32 0, 33) i32 @llvm.cttz.i32(i32 [[ARG]], i1 true)
 ; CHECK-NEXT:    [[SHIFTED:%.*]] = lshr exact i32 [[ARG]], [[TRAILING]]
 ; CHECK-NEXT:    [[R1_SROA_0_1:%.*]] = select i1 [[CMP]], i32 0, i32 [[SHIFTED]]
@@ -2747,9 +2917,10 @@ define i32 @pr47322_more_poisonous_replacement(i32 %arg) {
 }
 
 define i8 @select_replacement_add_eq(i8 %x, i8 %y) {
-; CHECK-LABEL: @select_replacement_add_eq(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 2, i8 [[Y:%.*]]
+; CHECK-LABEL: define i8 @select_replacement_add_eq(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 2, i8 [[Y]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %cmp = icmp eq i8 %x, 1
@@ -2759,9 +2930,10 @@ define i8 @select_replacement_add_eq(i8 %x, i8 %y) {
 }
 
 define <2 x i8> @select_replacement_add_eq_vec(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: @select_replacement_add_eq_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], splat (i8 1)
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> splat (i8 2), <2 x i8> [[Y:%.*]]
+; CHECK-LABEL: define <2 x i8> @select_replacement_add_eq_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X]], splat (i8 1)
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> splat (i8 2), <2 x i8> [[Y]]
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %cmp = icmp eq <2 x i8> %x, <i8 1, i8 1>
@@ -2771,9 +2943,10 @@ define <2 x i8> @select_replacement_add_eq_vec(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i8> @select_replacement_add_eq_vec_nonuniform(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: @select_replacement_add_eq_vec_nonuniform(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 1, i8 2>
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> <i8 4, i8 6>, <2 x i8> [[Y:%.*]]
+; CHECK-LABEL: define <2 x i8> @select_replacement_add_eq_vec_nonuniform(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X]], <i8 1, i8 2>
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> <i8 4, i8 6>, <2 x i8> [[Y]]
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %cmp = icmp eq <2 x i8> %x, <i8 1, i8 2>
@@ -2783,9 +2956,10 @@ define <2 x i8> @select_replacement_add_eq_vec_nonuniform(<2 x i8> %x, <2 x i8>
 }
 
 define <2 x i8> @select_replacement_add_eq_vec_poison(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: @select_replacement_add_eq_vec_poison(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 1, i8 poison>
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> <i8 2, i8 poison>, <2 x i8> [[Y:%.*]]
+; CHECK-LABEL: define <2 x i8> @select_replacement_add_eq_vec_poison(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X]], <i8 1, i8 poison>
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> <i8 2, i8 poison>, <2 x i8> [[Y]]
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %cmp = icmp eq <2 x i8> %x, <i8 1, i8 poison>
@@ -2795,10 +2969,11 @@ define <2 x i8> @select_replacement_add_eq_vec_poison(<2 x i8> %x, <2 x i8> %y)
 }
 
 define <2 x i8> @select_replacement_add_eq_vec_undef(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: @select_replacement_add_eq_vec_undef(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 1, i8 undef>
+; CHECK-LABEL: define <2 x i8> @select_replacement_add_eq_vec_undef(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X]], <i8 1, i8 undef>
 ; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i8> [[X]], splat (i8 1)
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Y]]
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %cmp = icmp eq <2 x i8> %x, <i8 1, i8 undef>
@@ -2808,9 +2983,10 @@ define <2 x i8> @select_replacement_add_eq_vec_undef(<2 x i8> %x, <2 x i8> %y) {
 }
 
 define <2 x i8> @select_replacement_add_eq_vec_undef_okay(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: @select_replacement_add_eq_vec_undef_okay(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], splat (i8 1)
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> <i8 2, i8 undef>, <2 x i8> [[Y:%.*]]
+; CHECK-LABEL: define <2 x i8> @select_replacement_add_eq_vec_undef_okay(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X]], splat (i8 1)
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> <i8 2, i8 undef>, <2 x i8> [[Y]]
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %cmp = icmp eq <2 x i8> %x, <i8 1, i8 1>
@@ -2821,10 +2997,11 @@ define <2 x i8> @select_replacement_add_eq_vec_undef_okay(<2 x i8> %x, <2 x i8>
 
 
 define <2 x i8> @select_replacement_add_eq_vec_undef_okay_todo(<2 x i8> %x, <2 x i8> %y) {
-; CHECK-LABEL: @select_replacement_add_eq_vec_undef_okay_todo(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], <i8 1, i8 undef>
+; CHECK-LABEL: define <2 x i8> @select_replacement_add_eq_vec_undef_okay_todo(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X]], <i8 1, i8 undef>
 ; CHECK-NEXT:    [[ADD:%.*]] = add <2 x i8> [[X]], <i8 1, i8 undef>
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Y:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> [[ADD]], <2 x i8> [[Y]]
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %cmp = icmp eq <2 x i8> %x, <i8 1, i8 undef>
@@ -2834,9 +3011,10 @@ define <2 x i8> @select_replacement_add_eq_vec_undef_okay_todo(<2 x i8> %x, <2 x
 }
 
 define <2 x i8> @select_replacement_xor_eq_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
-; CHECK-LABEL: @select_replacement_xor_eq_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> zeroinitializer, <2 x i8> [[Z:%.*]]
+; CHECK-LABEL: define <2 x i8> @select_replacement_xor_eq_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <2 x i8> [[Z:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[X]], [[Y]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i8> zeroinitializer, <2 x i8> [[Z]]
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %cmp = icmp eq <2 x i8> %x, %y
@@ -2847,10 +3025,11 @@ define <2 x i8> @select_replacement_xor_eq_vec(<2 x i8> %x, <2 x i8> %y, <2 x i8
 
 
 define i8 @select_replacement_add_ne(i8 %x, i8 %y) {
-; CHECK-LABEL: @select_replacement_add_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[X:%.*]], 1
+; CHECK-LABEL: define i8 @select_replacement_add_ne(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[X]], 1
 ; CHECK-NEXT:    call void @use(i1 [[CMP]])
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[Y:%.*]], i8 2
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[Y]], i8 2
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %cmp = icmp ne i8 %x, 1
@@ -2861,9 +3040,10 @@ define i8 @select_replacement_add_ne(i8 %x, i8 %y) {
 }
 
 define i8 @select_replacement_add_nuw(i8 %x, i8 %y) {
-; CHECK-LABEL: @select_replacement_add_nuw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 2, i8 [[Y:%.*]]
+; CHECK-LABEL: define i8 @select_replacement_add_nuw(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], 1
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 2, i8 [[Y]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %cmp = icmp eq i8 %x, 1
@@ -2873,9 +3053,10 @@ define i8 @select_replacement_add_nuw(i8 %x, i8 %y) {
 }
 
 define i8 @select_replacement_sub_noundef(i8 %x, i8 noundef %y, i8 %z) {
-; CHECK-LABEL: @select_replacement_sub_noundef(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z:%.*]]
+; CHECK-LABEL: define i8 @select_replacement_sub_noundef(
+; CHECK-SAME: i8 [[X:%.*]], i8 noundef [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %cmp = icmp eq i8 %x, %y
@@ -2885,10 +3066,11 @@ define i8 @select_replacement_sub_noundef(i8 %x, i8 noundef %y, i8 %z) {
 }
 
 define i8 @select_replacement_sub_noundef_but_may_be_poison(i8 %x, i8 noundef %yy, i8 %z) {
-; CHECK-LABEL: @select_replacement_sub_noundef_but_may_be_poison(
-; CHECK-NEXT:    [[Y:%.*]] = shl nuw i8 [[YY:%.*]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], [[Y]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z:%.*]]
+; CHECK-LABEL: define i8 @select_replacement_sub_noundef_but_may_be_poison(
+; CHECK-SAME: i8 [[X:%.*]], i8 noundef [[YY:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[Y:%.*]] = shl nuw i8 [[YY]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %y = shl nuw i8 %yy, 1
@@ -2900,9 +3082,10 @@ define i8 @select_replacement_sub_noundef_but_may_be_poison(i8 %x, i8 noundef %y
 
 ; TODO: The transform is also safe without noundef.
 define i8 @select_replacement_sub(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @select_replacement_sub(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z:%.*]]
+; CHECK-LABEL: define i8 @select_replacement_sub(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[Z]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %cmp = icmp eq i8 %x, %y
@@ -2913,12 +3096,13 @@ define i8 @select_replacement_sub(i8 %x, i8 %y, i8 %z) {
 
 ; FIXME: This is safe to fold.
 define i8 @select_replacement_shift_noundef(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @select_replacement_shift_noundef(
-; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i8 [[X:%.*]], 1
+; CHECK-LABEL: define i8 @select_replacement_shift_noundef(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i8 [[X]], 1
 ; CHECK-NEXT:    call void @use_i8(i8 noundef [[SHR]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[SHR]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[SHR]], [[Y]]
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[Y]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[SHL]], i8 [[Z:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[SHL]], i8 [[Z]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %shr = lshr exact i8 %x, 1
@@ -2931,11 +3115,12 @@ define i8 @select_replacement_shift_noundef(i8 %x, i8 %y, i8 %z) {
 
 ; TODO: The transform is also safe without noundef.
 define i8 @select_replacement_shift(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @select_replacement_shift(
-; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i8 [[X:%.*]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[SHR]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @select_replacement_shift(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[SHR:%.*]] = lshr exact i8 [[X]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[SHR]], [[Y]]
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[Y]], 1
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[SHL]], i8 [[Z:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[SHL]], i8 [[Z]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %shr = lshr exact i8 %x, 1
@@ -2946,9 +3131,10 @@ define i8 @select_replacement_shift(i8 %x, i8 %y, i8 %z) {
 }
 
 define i8 @select_replacement_loop(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @select_replacement_loop(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[X]], i8 [[Z:%.*]]
+; CHECK-LABEL: define i8 @select_replacement_loop(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], [[Y]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 [[X]], i8 [[Z]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %cmp = icmp eq i8 %x, %y
@@ -2957,8 +3143,9 @@ define i8 @select_replacement_loop(i8 %x, i8 %y, i8 %z) {
 }
 
 define i32 @select_replacement_loop2(i32 %arg, i32 %arg2) {
-; CHECK-LABEL: @select_replacement_loop2(
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[ARG:%.*]], [[ARG2:%.*]]
+; CHECK-LABEL: define i32 @select_replacement_loop2(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG2:%.*]]) {
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[ARG]], [[ARG2]]
 ; CHECK-NEXT:    ret i32 [[DIV]]
 ;
   %div = udiv i32 %arg, %arg2
@@ -2969,8 +3156,9 @@ define i32 @select_replacement_loop2(i32 %arg, i32 %arg2) {
 }
 
 define i8 @select_replacement_loop3(i32 noundef %x) {
-; CHECK-LABEL: @select_replacement_loop3(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-LABEL: define i8 @select_replacement_loop3(
+; CHECK-SAME: i32 noundef [[X:%.*]]) {
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[X]] to i8
 ; CHECK-NEXT:    [[REV:%.*]] = call i8 @llvm.bitreverse.i8(i8 [[TRUNC]])
 ; CHECK-NEXT:    [[EXT:%.*]] = zext i8 [[REV]] to i32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], [[EXT]]
@@ -2986,10 +3174,11 @@ define i8 @select_replacement_loop3(i32 noundef %x) {
 }
 
 define i16 @select_replacement_loop4(i16 noundef %p_12) {
-; CHECK-LABEL: @select_replacement_loop4(
-; CHECK-NEXT:    [[P_12:%.*]] = call i16 @llvm.umin.i16(i16 [[P_13:%.*]], i16 2)
-; CHECK-NEXT:    [[AND1:%.*]] = and i16 [[P_12]], 1
-; CHECK-NEXT:    ret i16 [[AND1]]
+; CHECK-LABEL: define i16 @select_replacement_loop4(
+; CHECK-SAME: i16 noundef [[P_12:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.umin.i16(i16 [[P_12]], i16 2)
+; CHECK-NEXT:    [[AND3:%.*]] = and i16 [[TMP1]], 1
+; CHECK-NEXT:    ret i16 [[AND3]]
 ;
   %cmp1 = icmp ult i16 %p_12, 2
   %and1 = and i16 %p_12, 1
@@ -3000,8 +3189,9 @@ define i16 @select_replacement_loop4(i16 noundef %p_12) {
 }
 
 define ptr @select_replacement_gep_inbounds(ptr %base, i64 %offset) {
-; CHECK-LABEL: @select_replacement_gep_inbounds(
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[BASE:%.*]], i64 [[OFFSET:%.*]]
+; CHECK-LABEL: define ptr @select_replacement_gep_inbounds(
+; CHECK-SAME: ptr [[BASE:%.*]], i64 [[OFFSET:%.*]]) {
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[BASE]], i64 [[OFFSET]]
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
   %cmp = icmp eq i64 %offset, 0
@@ -3011,8 +3201,9 @@ define ptr @select_replacement_gep_inbounds(ptr %base, i64 %offset) {
 }
 
 define i8 @replace_false_op_eq_shl_or_disjoint(i8 %x) {
-; CHECK-LABEL: @replace_false_op_eq_shl_or_disjoint(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[X:%.*]], 3
+; CHECK-LABEL: define i8 @replace_false_op_eq_shl_or_disjoint(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[SHL:%.*]] = shl i8 [[X]], 3
 ; CHECK-NEXT:    [[OR:%.*]] = or i8 [[X]], [[SHL]]
 ; CHECK-NEXT:    ret i8 [[OR]]
 ;
@@ -3024,8 +3215,9 @@ define i8 @replace_false_op_eq_shl_or_disjoint(i8 %x) {
 }
 
 define i8 @select_or_disjoint_eq(i8 %x, i8 %y) {
-; CHECK-LABEL: @select_or_disjoint_eq(
-; CHECK-NEXT:    [[OR:%.*]] = or i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @select_or_disjoint_eq(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[OR:%.*]] = or i8 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[OR]]
 ;
   %cmp = icmp eq i8 %x, %y
@@ -3035,7 +3227,8 @@ define i8 @select_or_disjoint_eq(i8 %x, i8 %y) {
 }
 
 define <2 x i1> @partial_true_undef_condval(<2 x i1> %x) {
-; CHECK-LABEL: @partial_true_undef_condval(
+; CHECK-LABEL: define <2 x i1> @partial_true_undef_condval(
+; CHECK-SAME: <2 x i1> [[X:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 poison>
 ;
   %r = select <2 x i1> <i1 true, i1 poison>, <2 x i1> <i1 true, i1 poison>, <2 x i1> %x
@@ -3043,7 +3236,8 @@ define <2 x i1> @partial_true_undef_condval(<2 x i1> %x) {
 }
 
 define <2 x i1> @partial_false_undef_condval(<2 x i1> %x) {
-; CHECK-LABEL: @partial_false_undef_condval(
+; CHECK-LABEL: define <2 x i1> @partial_false_undef_condval(
+; CHECK-SAME: <2 x i1> [[X:%.*]]) {
 ; CHECK-NEXT:    ret <2 x i1> <i1 false, i1 poison>
 ;
   %r = select <2 x i1> <i1 false, i1 poison>, <2 x i1> %x, <2 x i1> <i1 false, i1 poison>
@@ -3052,9 +3246,10 @@ define <2 x i1> @partial_false_undef_condval(<2 x i1> %x) {
 
 ; select (x == 0), 0, x * y --> freeze(y) * x
 define i32 @mul_select_eq_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: @mul_select_eq_zero(
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y:%.*]]
-; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X:%.*]], [[Y_FR]]
+; CHECK-LABEL: define i32 @mul_select_eq_zero(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X]], [[Y_FR]]
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %c = icmp eq i32 %x, 0
@@ -3065,9 +3260,10 @@ define i32 @mul_select_eq_zero(i32 %x, i32 %y) {
 
 ; select (y == 0), 0, x * y --> freeze(x) * y
 define i32 @mul_select_eq_zero_commute(i32 %x, i32 %y) {
-; CHECK-LABEL: @mul_select_eq_zero_commute(
-; CHECK-NEXT:    [[X_FR:%.*]] = freeze i32 [[X:%.*]]
-; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X_FR]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @mul_select_eq_zero_commute(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i32 [[X]]
+; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X_FR]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %c = icmp eq i32 %y, 0
@@ -3078,9 +3274,10 @@ define i32 @mul_select_eq_zero_commute(i32 %x, i32 %y) {
 
 ; Check that mul's flags preserved during the transformation.
 define i32 @mul_select_eq_zero_copy_flags(i32 %x, i32 %y) {
-; CHECK-LABEL: @mul_select_eq_zero_copy_flags(
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y:%.*]]
-; CHECK-NEXT:    [[M:%.*]] = mul nuw nsw i32 [[X:%.*]], [[Y_FR]]
+; CHECK-LABEL: define i32 @mul_select_eq_zero_copy_flags(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    [[M:%.*]] = mul nuw nsw i32 [[X]], [[Y_FR]]
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %c = icmp eq i32 %x, 0
@@ -3092,9 +3289,10 @@ define i32 @mul_select_eq_zero_copy_flags(i32 %x, i32 %y) {
 ; Check that the transformation could be applied after condition's inversion.
 ; select (x != 0), x * y, 0 --> freeze(y) * x
 define i32 @mul_select_ne_zero(i32 %x, i32 %y) {
-; CHECK-LABEL: @mul_select_ne_zero(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y:%.*]]
+; CHECK-LABEL: define i32 @mul_select_ne_zero(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[X]], 0
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
 ; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X]], [[Y_FR]]
 ; CHECK-NEXT:    call void @use(i1 [[C]])
 ; CHECK-NEXT:    ret i32 [[M]]
@@ -3110,9 +3308,10 @@ define i32 @mul_select_ne_zero(i32 %x, i32 %y) {
 ; an expression could be folded into mul as if there was a 0 instead of undef.
 ; select (x == 0), undef, x * y --> freeze(y) * x
 define i32 @mul_select_eq_zero_sel_undef(i32 %x, i32 %y) {
-; CHECK-LABEL: @mul_select_eq_zero_sel_undef(
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y:%.*]]
-; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X:%.*]], [[Y_FR]]
+; CHECK-LABEL: define i32 @mul_select_eq_zero_sel_undef(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X]], [[Y_FR]]
 ; CHECK-NEXT:    ret i32 [[M]]
 ;
   %c = icmp eq i32 %x, 0
@@ -3124,9 +3323,10 @@ define i32 @mul_select_eq_zero_sel_undef(i32 %x, i32 %y) {
 ; Check that the transformation is applied disregard to a number
 ; of expression's users.
 define i32 @mul_select_eq_zero_multiple_users(i32 %x, i32 %y) {
-; CHECK-LABEL: @mul_select_eq_zero_multiple_users(
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y:%.*]]
-; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X:%.*]], [[Y_FR]]
+; CHECK-LABEL: define i32 @mul_select_eq_zero_multiple_users(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i32 [[Y]]
+; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X]], [[Y_FR]]
 ; CHECK-NEXT:    call void @use_i32(i32 [[M]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[M]])
 ; CHECK-NEXT:    call void @use_i32(i32 [[M]])
@@ -3144,9 +3344,10 @@ define i32 @mul_select_eq_zero_multiple_users(i32 %x, i32 %y) {
 ; Negative test: select's condition is unrelated to multiplied values,
 ; so the transformation should not be applied.
 define i32 @mul_select_eq_zero_unrelated_condition(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @mul_select_eq_zero_unrelated_condition(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[Z:%.*]], 0
-; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @mul_select_eq_zero_unrelated_condition(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[Z]], 0
+; CHECK-NEXT:    [[M:%.*]] = mul i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 0, i32 [[M]]
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
@@ -3158,9 +3359,10 @@ define i32 @mul_select_eq_zero_unrelated_condition(i32 %x, i32 %y, i32 %z) {
 
 ; select (<k x elt> x == 0), <k x elt> 0, <k x elt> x * y --> freeze(y) * x
 define <4 x i32> @mul_select_eq_zero_vector(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: @mul_select_eq_zero_vector(
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze <4 x i32> [[Y:%.*]]
-; CHECK-NEXT:    [[M:%.*]] = mul <4 x i32> [[X:%.*]], [[Y_FR]]
+; CHECK-LABEL: define <4 x i32> @mul_select_eq_zero_vector(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze <4 x i32> [[Y]]
+; CHECK-NEXT:    [[M:%.*]] = mul <4 x i32> [[X]], [[Y_FR]]
 ; CHECK-NEXT:    ret <4 x i32> [[M]]
 ;
   %c = icmp eq <4 x i32> %x, zeroinitializer
@@ -3173,9 +3375,10 @@ define <4 x i32> @mul_select_eq_zero_vector(<4 x i32> %x, <4 x i32> %y) {
 ; is a vector consisting of zeros and poisons.
 ; select (<k x elt> x == {0, poison, ...}), <k x elt> 0, <k x elt> x * y --> freeze(y) * x
 define <2 x i32> @mul_select_eq_poison_vector(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @mul_select_eq_poison_vector(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 0, i32 poison>
-; CHECK-NEXT:    [[M:%.*]] = mul <2 x i32> [[X]], [[Y:%.*]]
+; CHECK-LABEL: define <2 x i32> @mul_select_eq_poison_vector(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[X]], <i32 0, i32 poison>
+; CHECK-NEXT:    [[M:%.*]] = mul <2 x i32> [[X]], [[Y]]
 ; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C]], <2 x i32> <i32 0, i32 42>, <2 x i32> [[M]]
 ; CHECK-NEXT:    ret <2 x i32> [[R]]
 ;
@@ -3189,9 +3392,10 @@ define <2 x i32> @mul_select_eq_poison_vector(<2 x i32> %x, <2 x i32> %y) {
 ; is a vector consisting of zeros and poisons.
 ; select (<k x elt> x == 0), <k x elt> {0, poison, ...}, <k x elt> x * y --> freeze(y) * x
 define <2 x i32> @mul_select_eq_zero_sel_poison_vector(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @mul_select_eq_zero_sel_poison_vector(
-; CHECK-NEXT:    [[Y_FR:%.*]] = freeze <2 x i32> [[Y:%.*]]
-; CHECK-NEXT:    [[M:%.*]] = mul <2 x i32> [[X:%.*]], [[Y_FR]]
+; CHECK-LABEL: define <2 x i32> @mul_select_eq_zero_sel_poison_vector(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze <2 x i32> [[Y]]
+; CHECK-NEXT:    [[M:%.*]] = mul <2 x i32> [[X]], [[Y_FR]]
 ; CHECK-NEXT:    ret <2 x i32> [[M]]
 ;
   %c = icmp eq <2 x i32> %x, zeroinitializer
@@ -3203,9 +3407,10 @@ define <2 x i32> @mul_select_eq_zero_sel_poison_vector(<2 x i32> %x, <2 x i32> %
 ; Negative test: select should not be folded into mul because
 ; condition's operand and select's operand do not merge into zero vector.
 define <2 x i32> @mul_select_eq_poison_vector_not_merging_to_zero(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: @mul_select_eq_poison_vector_not_merging_to_zero(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 0, i32 poison>
-; CHECK-NEXT:    [[M:%.*]] = mul <2 x i32> [[X]], [[Y:%.*]]
+; CHECK-LABEL: define <2 x i32> @mul_select_eq_poison_vector_not_merging_to_zero(
+; CHECK-SAME: <2 x i32> [[X:%.*]], <2 x i32> [[Y:%.*]]) {
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[X]], <i32 0, i32 poison>
+; CHECK-NEXT:    [[M:%.*]] = mul <2 x i32> [[X]], [[Y]]
 ; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[C]], <2 x i32> <i32 1, i32 0>, <2 x i32> [[M]]
 ; CHECK-NEXT:    ret <2 x i32> [[R]]
 ;
@@ -3216,8 +3421,9 @@ define <2 x i32> @mul_select_eq_poison_vector_not_merging_to_zero(<2 x i32> %x,
 }
 
 define i8 @ne0_is_all_ones(i8 %x) {
-; CHECK-LABEL: @ne0_is_all_ones(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-LABEL: define i8 @ne0_is_all_ones(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X]], 0
 ; CHECK-NEXT:    [[R:%.*]] = sext i1 [[TMP1]] to i8
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
@@ -3228,8 +3434,9 @@ define i8 @ne0_is_all_ones(i8 %x) {
 }
 
 define i8 @ne0_is_all_ones_use1(i8 %x) {
-; CHECK-LABEL: @ne0_is_all_ones_use1(
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @ne0_is_all_ones_use1(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    call void @use_i8(i8 [[NEGX]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i8 [[X]], 0
 ; CHECK-NEXT:    [[R:%.*]] = sext i1 [[TMP1]] to i8
@@ -3245,8 +3452,9 @@ define i8 @ne0_is_all_ones_use1(i8 %x) {
 ; negative test
 
 define i8 @ne0_is_all_ones_use2(i8 %x) {
-; CHECK-LABEL: @ne0_is_all_ones_use2(
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @ne0_is_all_ones_use2(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[UGT1:%.*]] = icmp ugt i8 [[X]], 1
 ; CHECK-NEXT:    call void @use(i1 [[UGT1]])
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[UGT1]], i8 -1, i8 [[NEGX]]
@@ -3262,8 +3470,9 @@ define i8 @ne0_is_all_ones_use2(i8 %x) {
 ; negative test
 
 define i8 @ne0_is_all_ones_wrong_pred(i8 %x) {
-; CHECK-LABEL: @ne0_is_all_ones_wrong_pred(
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @ne0_is_all_ones_wrong_pred(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[UGT1:%.*]] = icmp sgt i8 [[X]], 2
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[UGT1]], i8 -1, i8 [[NEGX]]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -3277,8 +3486,9 @@ define i8 @ne0_is_all_ones_wrong_pred(i8 %x) {
 ; negative test
 
 define i8 @ne0_is_all_ones_wrong_cmp(i8 %x) {
-; CHECK-LABEL: @ne0_is_all_ones_wrong_cmp(
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @ne0_is_all_ones_wrong_cmp(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[UGT1:%.*]] = icmp ugt i8 [[X]], 2
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[UGT1]], i8 -1, i8 [[NEGX]]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -3292,8 +3502,9 @@ define i8 @ne0_is_all_ones_wrong_cmp(i8 %x) {
 ; negative test
 
 define i8 @ne0_is_all_ones_wrong_sel(i8 %x) {
-; CHECK-LABEL: @ne0_is_all_ones_wrong_sel(
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @ne0_is_all_ones_wrong_sel(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[UGT1:%.*]] = icmp ugt i8 [[X]], 2
 ; CHECK-NEXT:    [[R:%.*]] = select i1 [[UGT1]], i8 1, i8 [[NEGX]]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -3305,8 +3516,9 @@ define i8 @ne0_is_all_ones_wrong_sel(i8 %x) {
 }
 
 define <2 x i8> @ne0_is_all_ones_swap_vec(<2 x i8> %x) {
-; CHECK-LABEL: @ne0_is_all_ones_swap_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i8> @ne0_is_all_ones_swap_vec(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i8> [[X]], zeroinitializer
 ; CHECK-NEXT:    [[R:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i8>
 ; CHECK-NEXT:    ret <2 x i8> [[R]]
 ;
@@ -3317,8 +3529,9 @@ define <2 x i8> @ne0_is_all_ones_swap_vec(<2 x i8> %x) {
 }
 
 define <2 x i8> @ne0_is_all_ones_swap_vec_poison(<2 x i8> %x) {
-; CHECK-LABEL: @ne0_is_all_ones_swap_vec_poison(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i8> [[X:%.*]], zeroinitializer
+; CHECK-LABEL: define <2 x i8> @ne0_is_all_ones_swap_vec_poison(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <2 x i8> [[X]], zeroinitializer
 ; CHECK-NEXT:    [[R:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i8>
 ; CHECK-NEXT:    ret <2 x i8> [[R]]
 ;
@@ -3329,8 +3542,9 @@ define <2 x i8> @ne0_is_all_ones_swap_vec_poison(<2 x i8> %x) {
 }
 
 define i64 @udiv_of_select_constexpr(i1 %c, i64 %x) {
-; CHECK-LABEL: @udiv_of_select_constexpr(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i64 [[X:%.*]], i64 ptrtoint (ptr @glbl to i64)
+; CHECK-LABEL: define i64 @udiv_of_select_constexpr(
+; CHECK-SAME: i1 [[C:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C]], i64 [[X]], i64 ptrtoint (ptr @glbl to i64)
 ; CHECK-NEXT:    [[OP:%.*]] = udiv i64 [[SEL]], 3
 ; CHECK-NEXT:    ret i64 [[OP]]
 ;
@@ -3340,8 +3554,9 @@ define i64 @udiv_of_select_constexpr(i1 %c, i64 %x) {
 }
 
 define i64 @udiv_of_select_constexpr_commuted(i1 %c, i64 %x) {
-; CHECK-LABEL: @udiv_of_select_constexpr_commuted(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C:%.*]], i64 ptrtoint (ptr @glbl to i64), i64 [[X:%.*]]
+; CHECK-LABEL: define i64 @udiv_of_select_constexpr_commuted(
+; CHECK-SAME: i1 [[C:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[C]], i64 ptrtoint (ptr @glbl to i64), i64 [[X]]
 ; CHECK-NEXT:    [[OP:%.*]] = udiv i64 [[SEL]], 3
 ; CHECK-NEXT:    ret i64 [[OP]]
 ;
@@ -3356,8 +3571,9 @@ declare void @use_i32(i32)
 declare i32 @llvm.cttz.i32(i32, i1 immarg)
 
 define i32 @select_cond_zext_cond(i1 %cond, i32 %b) {
-; CHECK-LABEL: @select_cond_zext_cond(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 1, i32 [[B:%.*]]
+; CHECK-LABEL: define i32 @select_cond_zext_cond(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 1, i32 [[B]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %zext = zext i1 %cond to i32
@@ -3366,8 +3582,9 @@ define i32 @select_cond_zext_cond(i1 %cond, i32 %b) {
 }
 
 define <2 x i32> @select_cond_zext_cond_vec(<2 x i1> %cond, <2 x i32> %b) {
-; CHECK-LABEL: @select_cond_zext_cond_vec(
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND:%.*]], <2 x i32> splat (i32 1), <2 x i32> [[B:%.*]]
+; CHECK-LABEL: define <2 x i32> @select_cond_zext_cond_vec(
+; CHECK-SAME: <2 x i1> [[COND:%.*]], <2 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND]], <2 x i32> splat (i32 1), <2 x i32> [[B]]
 ; CHECK-NEXT:    ret <2 x i32> [[SEL]]
 ;
   %zext = zext <2 x i1> %cond to <2 x i32>
@@ -3376,8 +3593,9 @@ define <2 x i32> @select_cond_zext_cond_vec(<2 x i1> %cond, <2 x i32> %b) {
 }
 
 define i32 @select_cond_sext_cond(i1 %cond, i32 %b) {
-; CHECK-LABEL: @select_cond_sext_cond(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 -1, i32 [[B:%.*]]
+; CHECK-LABEL: define i32 @select_cond_sext_cond(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 -1, i32 [[B]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %sext = sext i1 %cond to i32
@@ -3386,8 +3604,9 @@ define i32 @select_cond_sext_cond(i1 %cond, i32 %b) {
 }
 
 define <2 x i32> @select_cond_sext_cond_vec(<2 x i1> %cond, <2 x i32> %b) {
-; CHECK-LABEL: @select_cond_sext_cond_vec(
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND:%.*]], <2 x i32> splat (i32 -1), <2 x i32> [[B:%.*]]
+; CHECK-LABEL: define <2 x i32> @select_cond_sext_cond_vec(
+; CHECK-SAME: <2 x i1> [[COND:%.*]], <2 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND]], <2 x i32> splat (i32 -1), <2 x i32> [[B]]
 ; CHECK-NEXT:    ret <2 x i32> [[SEL]]
 ;
   %sext = sext <2 x i1> %cond to <2 x i32>
@@ -3396,8 +3615,9 @@ define <2 x i32> @select_cond_sext_cond_vec(<2 x i1> %cond, <2 x i32> %b) {
 }
 
 define i32 @select_cond_val_zext_cond(i1 %cond, i32 %b) {
-; CHECK-LABEL: @select_cond_val_zext_cond(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 [[B:%.*]], i32 0
+; CHECK-LABEL: define i32 @select_cond_val_zext_cond(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 [[B]], i32 0
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %zext = zext i1 %cond to i32
@@ -3406,8 +3626,9 @@ define i32 @select_cond_val_zext_cond(i1 %cond, i32 %b) {
 }
 
 define <2 x i32> @select_cond_val_zext_cond_vec(<2 x i1> %cond, <2 x i32> %b) {
-; CHECK-LABEL: @select_cond_val_zext_cond_vec(
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND:%.*]], <2 x i32> [[B:%.*]], <2 x i32> zeroinitializer
+; CHECK-LABEL: define <2 x i32> @select_cond_val_zext_cond_vec(
+; CHECK-SAME: <2 x i1> [[COND:%.*]], <2 x i32> [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND]], <2 x i32> [[B]], <2 x i32> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i32> [[SEL]]
 ;
   %zext = zext <2 x i1> %cond to <2 x i32>
@@ -3416,8 +3637,9 @@ define <2 x i32> @select_cond_val_zext_cond_vec(<2 x i1> %cond, <2 x i32> %b) {
 }
 
 define i32 @select_cond_val_sext_cond(i1 %cond, i32 %b) {
-; CHECK-LABEL: @select_cond_val_sext_cond(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 [[B:%.*]], i32 0
+; CHECK-LABEL: define i32 @select_cond_val_sext_cond(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 [[B]], i32 0
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %sext = sext i1 %cond to i32
@@ -3426,8 +3648,9 @@ define i32 @select_cond_val_sext_cond(i1 %cond, i32 %b) {
 }
 
 define i32 @select_cond_zext_not_cond_val(i1 %cond, i32 %b) {
-; CHECK-LABEL: @select_cond_zext_not_cond_val(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 0, i32 [[B:%.*]]
+; CHECK-LABEL: define i32 @select_cond_zext_not_cond_val(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 0, i32 [[B]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %not_cond = xor i1 %cond, true
@@ -3437,8 +3660,9 @@ define i32 @select_cond_zext_not_cond_val(i1 %cond, i32 %b) {
 }
 
 define i32 @select_cond_sext_not_cond_val(i1 %cond, i32 %b) {
-; CHECK-LABEL: @select_cond_sext_not_cond_val(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 0, i32 [[B:%.*]]
+; CHECK-LABEL: define i32 @select_cond_sext_not_cond_val(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 0, i32 [[B]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %not_cond = xor i1 %cond, true
@@ -3449,8 +3673,9 @@ define i32 @select_cond_sext_not_cond_val(i1 %cond, i32 %b) {
 
 
 define i32 @select_cond_val_zext_not_cond(i1 %cond, i32 %b) {
-; CHECK-LABEL: @select_cond_val_zext_not_cond(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 [[B:%.*]], i32 1
+; CHECK-LABEL: define i32 @select_cond_val_zext_not_cond(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 [[B]], i32 1
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %not_cond = xor i1 %cond, true
@@ -3460,8 +3685,9 @@ define i32 @select_cond_val_zext_not_cond(i1 %cond, i32 %b) {
 }
 
 define i32 @select_cond_val_sext_not_cond(i1 %cond, i32 %b) {
-; CHECK-LABEL: @select_cond_val_sext_not_cond(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 [[B:%.*]], i32 -1
+; CHECK-LABEL: define i32 @select_cond_val_sext_not_cond(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], i32 [[B]], i32 -1
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %not_cond = xor i1 %cond, true
@@ -3471,7 +3697,8 @@ define i32 @select_cond_val_sext_not_cond(i1 %cond, i32 %b) {
 }
 
 define i32 @select_cond_not_cond_cond1(i1 %cond) {
-; CHECK-LABEL: @select_cond_not_cond_cond1(
+; CHECK-LABEL: define i32 @select_cond_not_cond_cond1(
+; CHECK-SAME: i1 [[COND:%.*]]) {
 ; CHECK-NEXT:    ret i32 0
 ;
   %z = zext i1 %cond to i32
@@ -3482,7 +3709,8 @@ define i32 @select_cond_not_cond_cond1(i1 %cond) {
 }
 
 define i32 @select_cond_not_cond_cond2(i1 %cond) {
-; CHECK-LABEL: @select_cond_not_cond_cond2(
+; CHECK-LABEL: define i32 @select_cond_not_cond_cond2(
+; CHECK-SAME: i1 [[COND:%.*]]) {
 ; CHECK-NEXT:    ret i32 0
 ;
   %z = sext i1 %cond to i32
@@ -3495,9 +3723,10 @@ define i32 @select_cond_not_cond_cond2(i1 %cond) {
 ; This previously crashed due to Constant::getUniqueInteger not handling
 ; scalable vector splat ConstantExprs.
 define <vscale x 2 x i32> @and_constant_select_svec(<vscale x 2 x i32> %x, <vscale x 2 x i1> %cond) {
-; CHECK-LABEL: @and_constant_select_svec(
-; CHECK-NEXT:    [[A:%.*]] = and <vscale x 2 x i32> [[X:%.*]], splat (i32 1)
-; CHECK-NEXT:    [[B:%.*]] = select <vscale x 2 x i1> [[COND:%.*]], <vscale x 2 x i32> [[A]], <vscale x 2 x i32> [[X]]
+; CHECK-LABEL: define <vscale x 2 x i32> @and_constant_select_svec(
+; CHECK-SAME: <vscale x 2 x i32> [[X:%.*]], <vscale x 2 x i1> [[COND:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = and <vscale x 2 x i32> [[X]], splat (i32 1)
+; CHECK-NEXT:    [[B:%.*]] = select <vscale x 2 x i1> [[COND]], <vscale x 2 x i32> [[A]], <vscale x 2 x i32> [[X]]
 ; CHECK-NEXT:    ret <vscale x 2 x i32> [[B]]
 ;
   %a = and <vscale x 2 x i32> %x, splat (i32 1)
@@ -3506,8 +3735,9 @@ define <vscale x 2 x i32> @and_constant_select_svec(<vscale x 2 x i32> %x, <vsca
 }
 
 define <vscale x 2 x i32> @scalable_sign_bits(<vscale x 2 x i8> %x) {
-; CHECK-LABEL: @scalable_sign_bits(
-; CHECK-NEXT:    [[A:%.*]] = sext <vscale x 2 x i8> [[X:%.*]] to <vscale x 2 x i32>
+; CHECK-LABEL: define <vscale x 2 x i32> @scalable_sign_bits(
+; CHECK-SAME: <vscale x 2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = sext <vscale x 2 x i8> [[X]] to <vscale x 2 x i32>
 ; CHECK-NEXT:    [[B:%.*]] = shl nsw <vscale x 2 x i32> [[A]], splat (i32 16)
 ; CHECK-NEXT:    ret <vscale x 2 x i32> [[B]]
 ;
@@ -3517,8 +3747,9 @@ define <vscale x 2 x i32> @scalable_sign_bits(<vscale x 2 x i8> %x) {
 }
 
 define <vscale x 2 x i1> @scalable_non_zero(<vscale x 2 x i32> %x) {
-; CHECK-LABEL: @scalable_non_zero(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <vscale x 2 x i32> [[X:%.*]], splat (i32 56)
+; CHECK-LABEL: define <vscale x 2 x i1> @scalable_non_zero(
+; CHECK-SAME: <vscale x 2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <vscale x 2 x i32> [[X]], splat (i32 56)
 ; CHECK-NEXT:    ret <vscale x 2 x i1> [[CMP]]
 ;
   %a = or <vscale x 2 x i32> %x, splat (i32 1)
@@ -3528,8 +3759,9 @@ define <vscale x 2 x i1> @scalable_non_zero(<vscale x 2 x i32> %x) {
 }
 
 define i32 @clamp_umin(i32 %x) {
-; CHECK-LABEL: @clamp_umin(
-; CHECK-NEXT:    [[SEL:%.*]] = call i32 @llvm.umax.i32(i32 [[X:%.*]], i32 1)
+; CHECK-LABEL: define i32 @clamp_umin(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = call i32 @llvm.umax.i32(i32 [[X]], i32 1)
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
   %cmp = icmp eq i32 %x, 0
@@ -3538,8 +3770,9 @@ define i32 @clamp_umin(i32 %x) {
 }
 
 define i32 @clamp_umin_use(i32 %x) {
-; CHECK-LABEL: @clamp_umin_use(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-LABEL: define i32 @clamp_umin_use(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[CMP]])
 ; CHECK-NEXT:    [[SEL:%.*]] = call i32 @llvm.umax.i32(i32 [[X]], i32 1)
 ; CHECK-NEXT:    ret i32 [[SEL]]
@@ -3553,8 +3786,9 @@ define i32 @clamp_umin_use(i32 %x) {
 ; negative test - wrong cmp constant
 
 define i32 @not_clamp_umin1(i32 %x) {
-; CHECK-LABEL: @not_clamp_umin1(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 2
+; CHECK-LABEL: define i32 @not_clamp_umin1(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 2
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 1, i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
@@ -3566,8 +3800,9 @@ define i32 @not_clamp_umin1(i32 %x) {
 ; negative test - wrong select constant
 
 define i32 @not_clamp_umin2(i32 %x) {
-; CHECK-LABEL: @not_clamp_umin2(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-LABEL: define i32 @not_clamp_umin2(
+; CHECK-SAME: i32 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i32 -1, i32 [[X]]
 ; CHECK-NEXT:    ret i32 [[SEL]]
 ;
@@ -3577,8 +3812,9 @@ define i32 @not_clamp_umin2(i32 %x) {
 }
 
 define <2 x i8> @clamp_umaxval(<2 x i8> %x) {
-; CHECK-LABEL: @clamp_umaxval(
-; CHECK-NEXT:    [[SEL:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> splat (i8 -2))
+; CHECK-LABEL: define <2 x i8> @clamp_umaxval(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = call <2 x i8> @llvm.umin.v2i8(<2 x i8> [[X]], <2 x i8> splat (i8 -2))
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %cmp = icmp eq <2 x i8> %x, <i8 255, i8 255>
@@ -3589,8 +3825,9 @@ define <2 x i8> @clamp_umaxval(<2 x i8> %x) {
 ; negative test - wrong cmp constant
 
 define i8 @not_clamp_umax1(i8 %x) {
-; CHECK-LABEL: @not_clamp_umax1(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], -3
+; CHECK-LABEL: define i8 @not_clamp_umax1(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], -3
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 -2, i8 [[X]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
@@ -3602,8 +3839,9 @@ define i8 @not_clamp_umax1(i8 %x) {
 ; negative test - wrong select constant
 
 define i8 @not_clamp_umax2(i8 %x) {
-; CHECK-LABEL: @not_clamp_umax2(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], -1
+; CHECK-LABEL: define i8 @not_clamp_umax2(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], -1
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 1, i8 [[X]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
@@ -3613,8 +3851,9 @@ define i8 @not_clamp_umax2(i8 %x) {
 }
 
 define i8 @clamp_smin(i8 %x) {
-; CHECK-LABEL: @clamp_smin(
-; CHECK-NEXT:    [[SEL:%.*]] = call i8 @llvm.smax.i8(i8 [[X:%.*]], i8 -127)
+; CHECK-LABEL: define i8 @clamp_smin(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 -127)
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
   %cmp = icmp eq i8 %x, -128
@@ -3623,8 +3862,9 @@ define i8 @clamp_smin(i8 %x) {
 }
 
 define i8 @clamp_smin_use(i8 %x) {
-; CHECK-LABEL: @clamp_smin_use(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], -128
+; CHECK-LABEL: define i8 @clamp_smin_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], -128
 ; CHECK-NEXT:    call void @use1(i1 [[CMP]])
 ; CHECK-NEXT:    [[SEL:%.*]] = call i8 @llvm.smax.i8(i8 [[X]], i8 -127)
 ; CHECK-NEXT:    ret i8 [[SEL]]
@@ -3638,8 +3878,9 @@ define i8 @clamp_smin_use(i8 %x) {
 ; negative test - wrong cmp constant
 
 define i8 @not_clamp_smin1(i8 %x) {
-; CHECK-LABEL: @not_clamp_smin1(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 127
+; CHECK-LABEL: define i8 @not_clamp_smin1(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], 127
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 -127, i8 [[X]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
@@ -3651,8 +3892,9 @@ define i8 @not_clamp_smin1(i8 %x) {
 ; negative test - wrong select constant
 
 define i8 @not_clamp_smin2(i8 %x) {
-; CHECK-LABEL: @not_clamp_smin2(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], -128
+; CHECK-LABEL: define i8 @not_clamp_smin2(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], -128
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 -1, i8 [[X]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
@@ -3662,8 +3904,9 @@ define i8 @not_clamp_smin2(i8 %x) {
 }
 
 define <2 x i8> @clamp_smaxval(<2 x i8> %x) {
-; CHECK-LABEL: @clamp_smaxval(
-; CHECK-NEXT:    [[SEL:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X:%.*]], <2 x i8> splat (i8 126))
+; CHECK-LABEL: define <2 x i8> @clamp_smaxval(
+; CHECK-SAME: <2 x i8> [[X:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = call <2 x i8> @llvm.smin.v2i8(<2 x i8> [[X]], <2 x i8> splat (i8 126))
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %cmp = icmp eq <2 x i8> %x, <i8 127, i8 127>
@@ -3674,8 +3917,9 @@ define <2 x i8> @clamp_smaxval(<2 x i8> %x) {
 ; negative test - wrong cmp constant
 
 define i8 @not_clamp_smax1(i8 %x) {
-; CHECK-LABEL: @not_clamp_smax1(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], -128
+; CHECK-LABEL: define i8 @not_clamp_smax1(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], -128
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 126, i8 [[X]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
@@ -3687,8 +3931,9 @@ define i8 @not_clamp_smax1(i8 %x) {
 ; negative test - wrong select constant
 
 define i8 @not_clamp_smax2(i8 %x) {
-; CHECK-LABEL: @not_clamp_smax2(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], 127
+; CHECK-LABEL: define i8 @not_clamp_smax2(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], 127
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i8 125, i8 [[X]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
 ;
@@ -3699,8 +3944,9 @@ define i8 @not_clamp_smax2(i8 %x) {
 
 ; Used to infinite loop.
 define i32 @pr61361(i32 %arg) {
-; CHECK-LABEL: @pr61361(
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[ARG:%.*]], 0
+; CHECK-LABEL: define i32 @pr61361(
+; CHECK-SAME: i32 [[ARG:%.*]]) {
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[ARG]], 0
 ; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[CMP2]], i32 16777215, i32 0
 ; CHECK-NEXT:    ret i32 [[SEL2]]
 ;
@@ -3713,18 +3959,18 @@ define i32 @pr61361(i32 %arg) {
 }
 
 define i32 @pr62088() {
-; CHECK-LABEL: @pr62088(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[NOT2:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ -2, [[LOOP]] ]
-; CHECK-NEXT:    [[H_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ 1, [[LOOP]] ]
+; CHECK-LABEL: define i32 @pr62088() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[NOT2:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ -2, %[[LOOP]] ]
+; CHECK-NEXT:    [[H_0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ 1, %[[LOOP]] ]
 ; CHECK-NEXT:    [[XOR:%.*]] = or disjoint i32 [[H_0]], [[NOT2]]
 ; CHECK-NEXT:    [[SUB5:%.*]] = sub i32 -1824888657, [[XOR]]
 ; CHECK-NEXT:    [[XOR6:%.*]] = xor i32 [[SUB5]], -1260914025
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[XOR6]], 824855120
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret i32 [[H_0]]
 ;
 entry:
@@ -3759,9 +4005,10 @@ exit:
 ; X&Y==C?X|Y:X^Y, X&Y==C?X^Y:X|Y
 ; TODO: X&Y==0 could imply no_common_bit to TrueValue
 define i32 @src_and_eq_0_or_xor(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_and_eq_0_or_xor(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_and_eq_0_or_xor(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
@@ -3779,9 +4026,10 @@ entry:
 
 ; TODO: X&Y==0 could imply no_common_bit to TrueValue
 define i32 @src_and_eq_0_xor_or(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_and_eq_0_xor_or(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_and_eq_0_xor_or(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
@@ -3799,9 +4047,10 @@ entry:
 
 ; TODO: X&Y==-1 could imply all_common_bit to TrueValue
 define i32 @src_and_eq_neg1_or_xor(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_and_eq_neg1_or_xor(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_and_eq_neg1_or_xor(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -1
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 -1, i32 [[XOR]]
@@ -3818,9 +4067,10 @@ entry:
 
 ; TODO: X&Y==-1 could imply all_common_bit to TrueValue
 define i32 @src_and_eq_neg1_xor_or(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_and_eq_neg1_xor_or(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_and_eq_neg1_xor_or(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], -1
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 0, i32 [[OR]]
@@ -3836,10 +4086,11 @@ entry:
 }
 
 define i32 @src_and_eq_C_or_xororC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_and_eq_C_or_xororC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_and_eq_C_or_xororC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], [[C]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[XOR]], [[C]]
@@ -3857,10 +4108,11 @@ entry:
 }
 
 define i32 @src_and_eq_C_or_xorxorC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_and_eq_C_or_xorxorC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_and_eq_C_or_xorxorC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], [[C]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[XOR]], [[C]]
@@ -3878,10 +4130,11 @@ entry:
 }
 
 define i32 @src_and_eq_C_xor_OrAndNotC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_and_eq_C_xor_OrAndNotC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_and_eq_C_xor_OrAndNotC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], [[C]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[C]], -1
@@ -3901,10 +4154,11 @@ entry:
 }
 
 define <2 x i32> @src_and_eq_C_xor_OrAndNotC_vec_poison(<2 x i32> %0, <2 x i32> %1, <2 x i32> %2) {
-; CHECK-LABEL: @src_and_eq_C_xor_OrAndNotC_vec_poison(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[TMP1:%.*]], [[TMP0:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[AND]], [[TMP2:%.*]]
+; CHECK-LABEL: define <2 x i32> @src_and_eq_C_xor_OrAndNotC_vec_poison(
+; CHECK-SAME: <2 x i32> [[TMP0:%.*]], <2 x i32> [[TMP1:%.*]], <2 x i32> [[TMP2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[AND]], [[TMP2]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor <2 x i32> [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[OR:%.*]] = or <2 x i32> [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[NOT:%.*]] = xor <2 x i32> [[TMP2]], <i32 -1, i32 poison>
@@ -3924,10 +4178,11 @@ entry:
 }
 
 define i32 @src_and_eq_C_xor_orxorC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_and_eq_C_xor_orxorC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_and_eq_C_xor_orxorC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], [[C]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[OR]], [[C]]
@@ -3948,9 +4203,10 @@ entry:
 ; X|Y==C?X&Y:X^Y, X|Y==C?X^Y:X&Y
 ; TODO: X|Y==0 could imply no_common_bit to TrueValue
 define i32 @src_or_eq_0_and_xor(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_or_eq_0_and_xor(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_or_eq_0_and_xor(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    ret i32 [[XOR]]
 ;
 entry:
@@ -3964,9 +4220,10 @@ entry:
 
 ; TODO: X|Y==0 could imply no_common_bit to TrueValue
 define i32 @src_or_eq_0_xor_and(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_or_eq_0_xor_and(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_or_eq_0_xor_and(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
 entry:
@@ -3979,9 +4236,10 @@ entry:
 }
 
 define i32 @src_or_eq_neg1_and_xor(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_or_eq_neg1_and_xor(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_or_eq_neg1_and_xor(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = xor i32 [[X]], [[Y]]
@@ -4000,9 +4258,10 @@ entry:
 }
 
 define i32 @src_or_eq_neg1_xor_and(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_or_eq_neg1_xor_and(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_or_eq_neg1_xor_and(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], -1
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
@@ -4021,10 +4280,11 @@ entry:
 }
 
 define i32 @src_or_eq_C_and_xorC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_or_eq_C_and_xorC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_or_eq_C_and_xorC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], [[C]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[XOR]], [[C]]
@@ -4042,10 +4302,11 @@ entry:
 }
 
 define i32 @src_or_eq_C_and_andnotxorC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_or_eq_C_and_andnotxorC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_or_eq_C_and_andnotxorC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], [[C]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[TMP0]], -1
@@ -4065,10 +4326,11 @@ entry:
 }
 
 define i32 @src_or_eq_C_xor_xorandC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_or_eq_C_xor_xorandC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_or_eq_C_xor_xorandC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], [[C]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[AND]], [[C]]
@@ -4086,10 +4348,11 @@ entry:
 }
 
 define i32 @src_or_eq_C_xor_andnotandC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_or_eq_C_xor_andnotandC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_or_eq_C_xor_andnotandC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[OR]], [[C]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[AND]], -1
@@ -4111,9 +4374,10 @@ entry:
 ; https://alive2.llvm.org/ce/z/c6oXi4
 ; X^Y==C?X&Y:X|Y, X^Y==C?X|Y:X&Y
 define i32 @src_xor_eq_neg1_and(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_xor_eq_neg1_and(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_xor_eq_neg1_and(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], -1
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
@@ -4133,9 +4397,10 @@ entry:
 
 ; TODO: X^Y==-1 could imply no_common_bit to TrueValue
 define i32 @src_xor_eq_neg1_or(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_xor_eq_neg1_or(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i32 @src_xor_eq_neg1_or(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], -1
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 [[OR]], i32 -1
@@ -4150,10 +4415,11 @@ entry:
 }
 
 define i32 @src_xor_eq_C_and_xororC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_xor_eq_C_and_xororC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_xor_eq_C_and_xororC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[C]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[OR]], [[C]]
@@ -4171,10 +4437,11 @@ entry:
 }
 
 define i32 @src_xor_eq_C_and_andornotC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_xor_eq_C_and_andornotC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_xor_eq_C_and_andornotC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[C]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[NOT:%.*]] = xor i32 [[C]], -1
@@ -4194,10 +4461,11 @@ entry:
 }
 
 define i32 @src_xor_eq_C_or_xorandC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_xor_eq_C_or_xorandC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_xor_eq_C_or_xorandC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[C]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[XOR1:%.*]] = xor i32 [[AND]], [[C]]
@@ -4215,10 +4483,11 @@ entry:
 }
 
 define i32 @src_xor_eq_C_or_orandC(i32 %x, i32 %y, i32 %c) {
-; CHECK-LABEL: @src_xor_eq_C_or_orandC(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[C:%.*]]
+; CHECK-LABEL: define i32 @src_xor_eq_C_or_orandC(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i32 [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[XOR]], [[C]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[Y]], [[X]]
 ; CHECK-NEXT:    [[OR1:%.*]] = or i32 [[AND]], [[C]]
@@ -4238,8 +4507,9 @@ entry:
 ; Select icmp and/or/xor
 ; NO TRANSFORMED - select condition is compare with not 0
 define i32 @src_select_and_min_positive_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_and_min_positive_int(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_and_min_positive_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND0:%.*]] = icmp eq i32 [[AND]], 1
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
@@ -4255,8 +4525,9 @@ define i32 @src_select_and_min_positive_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_and_max_positive_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_and_max_positive_int(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_and_max_positive_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND0:%.*]] = icmp eq i32 [[AND]], 2147483647
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
@@ -4272,8 +4543,9 @@ define i32 @src_select_and_max_positive_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_and_min_negative_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_and_min_negative_int(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_and_min_negative_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND0:%.*]] = icmp eq i32 [[AND]], -2147483648
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
@@ -4289,8 +4561,9 @@ define i32 @src_select_and_min_negative_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_or_min_positive_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_or_min_positive_int(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_or_min_positive_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR0:%.*]] = icmp eq i32 [[OR]], 1
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
@@ -4306,8 +4579,9 @@ define i32 @src_select_or_min_positive_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_or_max_positive_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_or_max_positive_int(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_or_max_positive_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR0:%.*]] = icmp eq i32 [[OR]], 2147483647
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
@@ -4323,8 +4597,9 @@ define i32 @src_select_or_max_positive_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_or_min_negative_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_or_min_negative_int(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_or_min_negative_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR0:%.*]] = icmp eq i32 [[OR]], -2147483648
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
@@ -4340,8 +4615,9 @@ define i32 @src_select_or_min_negative_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_or_max_negative_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_or_max_negative_int(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_or_max_negative_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR0:%.*]] = icmp eq i32 [[OR]], -1
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
@@ -4357,8 +4633,9 @@ define i32 @src_select_or_max_negative_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_xor_min_positive_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_xor_min_positive_int(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_xor_min_positive_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[XOR]], 1
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
@@ -4374,8 +4651,9 @@ define i32 @src_select_xor_min_positive_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_xor_max_positive_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_xor_max_positive_int(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_xor_max_positive_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[XOR]], 2147483647
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
@@ -4391,8 +4669,9 @@ define i32 @src_select_xor_max_positive_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_xor_min_negative_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_xor_min_negative_int(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_xor_min_negative_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[XOR]], -2147483648
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
@@ -4408,8 +4687,9 @@ define i32 @src_select_xor_min_negative_int(i32 %x, i32 %y) {
 }
 
 define i32 @src_select_xor_max_negative_int(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_xor_max_negative_int(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_xor_max_negative_int(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[XOR]], -1
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
@@ -4428,8 +4708,9 @@ define i32 @src_select_xor_max_negative_int(i32 %x, i32 %y) {
 ; https://alive2.llvm.org/ce/z/BVgrJ-
 ; NO TRANSFORMED - not supported
 define i32 @src_no_trans_select_and_eq0_and_or(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_and_eq0_and_or(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_and_eq0_and_or(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND0:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[AND0]], i32 0, i32 [[OR]]
@@ -4443,8 +4724,9 @@ define i32 @src_no_trans_select_and_eq0_and_or(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_and_eq0_and_xor(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_and_eq0_and_xor(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_and_eq0_and_xor(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND0:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[AND0]], i32 0, i32 [[XOR]]
@@ -4458,8 +4740,9 @@ define i32 @src_no_trans_select_and_eq0_and_xor(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_and_eq0_or_and(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_and_eq0_or_and(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_and_eq0_or_and(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND0:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[AND0]], i32 [[OR]], i32 [[AND]]
@@ -4473,8 +4756,9 @@ define i32 @src_no_trans_select_and_eq0_or_and(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_and_eq0_xor_and(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_and_eq0_xor_and(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_and_eq0_xor_and(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND0:%.*]] = icmp eq i32 [[AND]], 0
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[AND0]], i32 [[XOR]], i32 [[AND]]
@@ -4488,8 +4772,9 @@ define i32 @src_no_trans_select_and_eq0_xor_and(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_or_eq0_or_and(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_or_eq0_or_and(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_or_eq0_or_and(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[AND]]
 ;
   %or = or i32 %x, %y
@@ -4500,8 +4785,9 @@ define i32 @src_no_trans_select_or_eq0_or_and(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_or_eq0_or_xor(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_or_eq0_or_xor(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_or_eq0_or_xor(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[XOR]]
 ;
   %or = or i32 %x, %y
@@ -4512,8 +4798,9 @@ define i32 @src_no_trans_select_or_eq0_or_xor(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_or_eq0_and_or(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_or_eq0_and_or(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_or_eq0_and_or(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %or = or i32 %x, %y
@@ -4524,8 +4811,9 @@ define i32 @src_no_trans_select_or_eq0_and_or(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_or_eq0_xor_or(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_or_eq0_xor_or(
-; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_or_eq0_xor_or(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[OR]]
 ;
   %or = or i32 %x, %y
@@ -4536,8 +4824,9 @@ define i32 @src_no_trans_select_or_eq0_xor_or(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_and_ne0_xor_or(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_and_ne0_xor_or(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_and_ne0_xor_or(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[XOR]]
 ;
   %or = or i32 %x, %y
@@ -4548,8 +4837,9 @@ define i32 @src_no_trans_select_and_ne0_xor_or(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_xor_eq0_xor_and(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_xor_eq0_xor_and(
-; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_xor_eq0_xor_and(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[XOR0]], i32 0, i32 [[AND]]
 ; CHECK-NEXT:    ret i32 [[COND]]
@@ -4562,8 +4852,9 @@ define i32 @src_no_trans_select_xor_eq0_xor_and(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_xor_eq0_xor_or(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_xor_eq0_xor_or(
-; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_xor_eq0_xor_or(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[XOR0]], i32 0, i32 [[OR]]
 ; CHECK-NEXT:    ret i32 [[COND]]
@@ -4576,8 +4867,9 @@ define i32 @src_no_trans_select_xor_eq0_xor_or(i32 %x, i32 %y) {
 }
 
 define i32 @src_no_trans_select_xor_eq0_and_xor(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_xor_eq0_and_xor(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_xor_eq0_and_xor(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[XOR0]], i32 [[AND]], i32 [[XOR]]
@@ -4592,8 +4884,9 @@ define i32 @src_no_trans_select_xor_eq0_and_xor(i32 %x, i32 %y) {
 
 ; https://alive2.llvm.org/ce/z/SBe8ei
 define i32 @src_no_trans_select_xor_eq0_or_xor(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_no_trans_select_xor_eq0_or_xor(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_no_trans_select_xor_eq0_or_xor(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[OR:%.*]] = or i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[XOR0]], i32 [[OR]], i32 [[XOR]]
@@ -4609,8 +4902,9 @@ define i32 @src_no_trans_select_xor_eq0_or_xor(i32 %x, i32 %y) {
 ; (X == C) ? X : Y -> (X == C) ? C : Y
 ; Fixed #77553
 define i32 @src_select_xxory_eq0_xorxy_y(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_select_xxory_eq0_xorxy_y(
-; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_select_xxory_eq0_xorxy_y(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR0:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[COND:%.*]] = select i1 [[XOR0]], i32 0, i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[COND]]
 ;
@@ -4621,9 +4915,10 @@ define i32 @src_select_xxory_eq0_xorxy_y(i32 %x, i32 %y) {
 }
 
 define i32 @sequence_select_with_same_cond_false(i1 %c1, i1 %c2){
-; CHECK-LABEL: @sequence_select_with_same_cond_false(
-; CHECK-NEXT:    [[S2:%.*]] = select i1 [[C2:%.*]], i32 666, i32 45
-; CHECK-NEXT:    [[S3:%.*]] = select i1 [[C1:%.*]], i32 789, i32 [[S2]]
+; CHECK-LABEL: define i32 @sequence_select_with_same_cond_false(
+; CHECK-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT:    [[S2:%.*]] = select i1 [[C2]], i32 666, i32 45
+; CHECK-NEXT:    [[S3:%.*]] = select i1 [[C1]], i32 789, i32 [[S2]]
 ; CHECK-NEXT:    ret i32 [[S3]]
 ;
   %s1 = select i1 %c1, i32 23, i32 45
@@ -4633,9 +4928,10 @@ define i32 @sequence_select_with_same_cond_false(i1 %c1, i1 %c2){
 }
 
 define i32 @sequence_select_with_same_cond_true(i1 %c1, i1 %c2){
-; CHECK-LABEL: @sequence_select_with_same_cond_true(
-; CHECK-NEXT:    [[S2:%.*]] = select i1 [[C2:%.*]], i32 45, i32 666
-; CHECK-NEXT:    [[S3:%.*]] = select i1 [[C1:%.*]], i32 [[S2]], i32 789
+; CHECK-LABEL: define i32 @sequence_select_with_same_cond_true(
+; CHECK-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT:    [[S2:%.*]] = select i1 [[C2]], i32 45, i32 666
+; CHECK-NEXT:    [[S3:%.*]] = select i1 [[C1]], i32 [[S2]], i32 789
 ; CHECK-NEXT:    ret i32 [[S3]]
 ;
   %s1 = select i1 %c1, i32 45, i32 23
@@ -4645,9 +4941,10 @@ define i32 @sequence_select_with_same_cond_true(i1 %c1, i1 %c2){
 }
 
 define double @sequence_select_with_same_cond_double(double %a, i1 %c1, i1 %c2, double %r1, double %r2){
-; CHECK-LABEL: @sequence_select_with_same_cond_double(
-; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1:%.*]], double 1.000000e+00, double 0.000000e+00
-; CHECK-NEXT:    [[S2:%.*]] = select i1 [[C2:%.*]], double [[S1]], double 2.000000e+00
+; CHECK-LABEL: define double @sequence_select_with_same_cond_double(
+; CHECK-SAME: double [[A:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]], double [[R1:%.*]], double [[R2:%.*]]) {
+; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], double 1.000000e+00, double 0.000000e+00
+; CHECK-NEXT:    [[S2:%.*]] = select i1 [[C2]], double [[S1]], double 2.000000e+00
 ; CHECK-NEXT:    [[S3:%.*]] = select i1 [[C1]], double [[S2]], double 3.000000e+00
 ; CHECK-NEXT:    ret double [[S3]]
 ;
@@ -4660,10 +4957,11 @@ define double @sequence_select_with_same_cond_double(double %a, i1 %c1, i1 %c2,
 declare void @use32(i32)
 
 define i32 @sequence_select_with_same_cond_extra_use(i1 %c1, i1 %c2){
-; CHECK-LABEL: @sequence_select_with_same_cond_extra_use(
-; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1:%.*]], i32 23, i32 45
+; CHECK-LABEL: define i32 @sequence_select_with_same_cond_extra_use(
+; CHECK-SAME: i1 [[C1:%.*]], i1 [[C2:%.*]]) {
+; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], i32 23, i32 45
 ; CHECK-NEXT:    call void @use32(i32 [[S1]])
-; CHECK-NEXT:    [[S2:%.*]] = select i1 [[C2:%.*]], i32 666, i32 [[S1]]
+; CHECK-NEXT:    [[S2:%.*]] = select i1 [[C2]], i32 666, i32 [[S1]]
 ; CHECK-NEXT:    [[S3:%.*]] = select i1 [[C1]], i32 789, i32 [[S2]]
 ; CHECK-NEXT:    ret i32 [[S3]]
 ;
@@ -4675,9 +4973,10 @@ define i32 @sequence_select_with_same_cond_extra_use(i1 %c1, i1 %c2){
 }
 
 define i8 @test_replace_freeze_multiuse(i1 %x, i8 %y) {
-; CHECK-LABEL: @test_replace_freeze_multiuse(
-; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[X:%.*]] to i8
-; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i8 [[EXT]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @test_replace_freeze_multiuse(
+; CHECK-SAME: i1 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[X]] to i8
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i8 [[EXT]], [[Y]]
 ; CHECK-NEXT:    [[SHL_FR:%.*]] = freeze i8 [[SHL]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X]], i8 0, i8 [[SHL_FR]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[SHL_FR]], [[SEL]]
@@ -4692,9 +4991,10 @@ define i8 @test_replace_freeze_multiuse(i1 %x, i8 %y) {
 }
 
 define i8 @test_replace_freeze_oneuse(i1 %x, i8 %y) {
-; CHECK-LABEL: @test_replace_freeze_oneuse(
-; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[X:%.*]] to i8
-; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i8 [[EXT]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @test_replace_freeze_oneuse(
+; CHECK-SAME: i1 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[X]] to i8
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw i8 [[EXT]], [[Y]]
 ; CHECK-NEXT:    [[SHL_FR:%.*]] = freeze i8 [[SHL]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[X]], i8 0, i8 [[SHL_FR]]
 ; CHECK-NEXT:    ret i8 [[SEL]]
@@ -4707,8 +5007,9 @@ define i8 @test_replace_freeze_oneuse(i1 %x, i8 %y) {
 }
 
 define i8 @select_knownbits_simplify(i8 noundef %x)  {
-; CHECK-LABEL: @select_knownbits_simplify(
-; CHECK-NEXT:    [[X_LO:%.*]] = and i8 [[X:%.*]], 1
+; CHECK-LABEL: define i8 @select_knownbits_simplify(
+; CHECK-SAME: i8 noundef [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LO:%.*]] = and i8 [[X]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X_LO]], 0
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], -2
 ; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i8 [[AND]], i8 0
@@ -4722,8 +5023,9 @@ define i8 @select_knownbits_simplify(i8 noundef %x)  {
 }
 
 define i8 @select_knownbits_simplify_nested(i8 noundef %x)  {
-; CHECK-LABEL: @select_knownbits_simplify_nested(
-; CHECK-NEXT:    [[X_LO:%.*]] = and i8 [[X:%.*]], 1
+; CHECK-LABEL: define i8 @select_knownbits_simplify_nested(
+; CHECK-SAME: i8 noundef [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LO:%.*]] = and i8 [[X]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X_LO]], 0
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], -2
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i8 [[AND]], [[AND]]
@@ -4739,8 +5041,9 @@ define i8 @select_knownbits_simplify_nested(i8 noundef %x)  {
 }
 
 define i8 @select_knownbits_simplify_missing_noundef(i8 %x)  {
-; CHECK-LABEL: @select_knownbits_simplify_missing_noundef(
-; CHECK-NEXT:    [[X_LO:%.*]] = and i8 [[X:%.*]], 1
+; CHECK-LABEL: define i8 @select_knownbits_simplify_missing_noundef(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[X_LO:%.*]] = and i8 [[X]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X_LO]], 0
 ; CHECK-NEXT:    [[AND:%.*]] = and i8 [[X]], -2
 ; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i8 [[AND]], i8 0
@@ -4757,8 +5060,9 @@ define i8 @select_knownbits_simplify_missing_noundef(i8 %x)  {
 
 ; Make sure we don't replace %ptr with @g_ext, which may cause the load to trigger UB.
 define i32 @pr99436(ptr align 4 dereferenceable(4) %ptr) {
-; CHECK-LABEL: @pr99436(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[PTR:%.*]], @g_ext
+; CHECK-LABEL: define i32 @pr99436(
+; CHECK-SAME: ptr align 4 dereferenceable(4) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[PTR]], @g_ext
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[RET:%.*]] = select i1 [[CMP]], i32 [[VAL]], i32 0
 ; CHECK-NEXT:    ret i32 [[RET]]
@@ -4770,9 +5074,10 @@ define i32 @pr99436(ptr align 4 dereferenceable(4) %ptr) {
 }
 
 define i8 @sel_trunc_simplify(i1 %c, i8 %x, i16 %y) {
-; CHECK-LABEL: @sel_trunc_simplify(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i16 [[Y:%.*]] to i8
-; CHECK-NEXT:    [[TRUNC:%.*]] = select i1 [[C:%.*]], i8 [[X:%.*]], i8 [[TMP1]]
+; CHECK-LABEL: define i8 @sel_trunc_simplify(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i16 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i16 [[Y]] to i8
+; CHECK-NEXT:    [[TRUNC:%.*]] = select i1 [[C]], i8 [[X]], i8 [[TMP1]]
 ; CHECK-NEXT:    ret i8 [[TRUNC]]
 ;
   %x.ext = zext i8 %x to i16
@@ -4782,10 +5087,11 @@ define i8 @sel_trunc_simplify(i1 %c, i8 %x, i16 %y) {
 }
 
 define i32 @sel_umin_simplify(i1 %c, i32 %x, i16 %y) {
-; CHECK-LABEL: @sel_umin_simplify(
-; CHECK-NEXT:    [[ARG2_EXT:%.*]] = zext i16 [[ARG2:%.*]] to i32
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.umin.i32(i32 [[X:%.*]], i32 [[ARG2_EXT]])
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C:%.*]], i32 [[TMP1]], i32 0
+; CHECK-LABEL: define i32 @sel_umin_simplify(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[X:%.*]], i16 [[Y:%.*]]) {
+; CHECK-NEXT:    [[Y_EXT:%.*]] = zext i16 [[Y]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.umin.i32(i32 [[X]], i32 [[Y_EXT]])
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C]], i32 [[TMP1]], i32 0
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %sel = select i1 %c, i32 %x, i32 0
@@ -4795,9 +5101,10 @@ define i32 @sel_umin_simplify(i1 %c, i32 %x, i16 %y) {
 }
 
 define i32 @sel_extractvalue_simplify(i1 %c, { i32, i32 } %agg1, i32 %x, i32 %y) {
-; CHECK-LABEL: @sel_extractvalue_simplify(
-; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i32 } [[AGG1:%.*]], 1
-; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C:%.*]], i32 [[TMP1]], i32 [[Y:%.*]]
+; CHECK-LABEL: define i32 @sel_extractvalue_simplify(
+; CHECK-SAME: i1 [[C:%.*]], { i32, i32 } [[AGG1:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = extractvalue { i32, i32 } [[AGG1]], 1
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[C]], i32 [[TMP1]], i32 [[Y]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %agg2.0 = insertvalue { i32, i32 } poison, i32 %x, 0
@@ -4808,9 +5115,10 @@ define i32 @sel_extractvalue_simplify(i1 %c, { i32, i32 } %agg1, i32 %x, i32 %y)
 }
 
 define i1 @replace_select_cond_true(i1 %cond, i32 %v1, i32 %v2, i32 %v3) {
-; CHECK-LABEL: @replace_select_cond_true(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[SEL:%.*]], [[V2:%.*]]
-; CHECK-NEXT:    [[AND:%.*]] = select i1 [[COND:%.*]], i1 [[CMP]], i1 false
+; CHECK-LABEL: define i1 @replace_select_cond_true(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[V1:%.*]], i32 [[V2:%.*]], i32 [[V3:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V1]], [[V2]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[COND]], i1 [[CMP]], i1 false
 ; CHECK-NEXT:    ret i1 [[AND]]
 ;
   %sel = select i1 %cond, i32 %v1, i32 %v3
@@ -4820,9 +5128,10 @@ define i1 @replace_select_cond_true(i1 %cond, i32 %v1, i32 %v2, i32 %v3) {
 }
 
 define i1 @replace_select_cond_false(i1 %cond, i32 %v1, i32 %v2, i32 %v3) {
-; CHECK-LABEL: @replace_select_cond_false(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[SEL:%.*]], [[V2:%.*]]
-; CHECK-NEXT:    [[OR:%.*]] = select i1 [[COND:%.*]], i1 true, i1 [[CMP]]
+; CHECK-LABEL: define i1 @replace_select_cond_false(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[V1:%.*]], i32 [[V2:%.*]], i32 [[V3:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V3]], [[V2]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[COND]], i1 true, i1 [[CMP]]
 ; CHECK-NEXT:    ret i1 [[OR]]
 ;
   %sel = select i1 %cond, i32 %v1, i32 %v3
@@ -4832,9 +5141,10 @@ define i1 @replace_select_cond_false(i1 %cond, i32 %v1, i32 %v2, i32 %v3) {
 }
 
 define i32 @replace_and_cond(i1 %cond1, i1 %cond2) {
-; CHECK-LABEL: @replace_and_cond(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[AND:%.*]], i32 3, i32 2
-; CHECK-NEXT:    [[MUX:%.*]] = select i1 [[COND1:%.*]], i32 [[SEL]], i32 1
+; CHECK-LABEL: define i32 @replace_and_cond(
+; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND2]], i32 3, i32 2
+; CHECK-NEXT:    [[MUX:%.*]] = select i1 [[COND1]], i32 [[SEL]], i32 1
 ; CHECK-NEXT:    ret i32 [[MUX]]
 ;
   %and = and i1 %cond1, %cond2
@@ -4844,9 +5154,10 @@ define i32 @replace_and_cond(i1 %cond1, i1 %cond2) {
 }
 
 define <2 x i32> @replace_and_cond_vec(<2 x i1> %cond1, <2 x i1> %cond2) {
-; CHECK-LABEL: @replace_and_cond_vec(
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND2:%.*]], <2 x i32> splat (i32 3), <2 x i32> splat (i32 2)
-; CHECK-NEXT:    [[MUX:%.*]] = select <2 x i1> [[COND1:%.*]], <2 x i32> [[SEL]], <2 x i32> splat (i32 1)
+; CHECK-LABEL: define <2 x i32> @replace_and_cond_vec(
+; CHECK-SAME: <2 x i1> [[COND1:%.*]], <2 x i1> [[COND2:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[COND2]], <2 x i32> splat (i32 3), <2 x i32> splat (i32 2)
+; CHECK-NEXT:    [[MUX:%.*]] = select <2 x i1> [[COND1]], <2 x i32> [[SEL]], <2 x i32> splat (i32 1)
 ; CHECK-NEXT:    ret <2 x i32> [[MUX]]
 ;
   %and = and <2 x i1> %cond1, %cond2
@@ -4857,8 +5168,9 @@ define <2 x i32> @replace_and_cond_vec(<2 x i1> %cond1, <2 x i1> %cond2) {
 
 ; TODO: We can still replace the use of %and with %cond2
 define i32 @replace_and_cond_multiuse1(i1 %cond1, i1 %cond2) {
-; CHECK-LABEL: @replace_and_cond_multiuse1(
-; CHECK-NEXT:    [[AND:%.*]] = and i1 [[COND1:%.*]], [[COND2:%.*]]
+; CHECK-LABEL: define i32 @replace_and_cond_multiuse1(
+; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[COND1]], [[COND2]]
 ; CHECK-NEXT:    call void @use(i1 [[AND]])
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[AND]], i32 3, i32 2
 ; CHECK-NEXT:    [[MUX:%.*]] = select i1 [[COND1]], i32 [[SEL]], i32 1
@@ -4872,8 +5184,9 @@ define i32 @replace_and_cond_multiuse1(i1 %cond1, i1 %cond2) {
 }
 
 define i32 @replace_and_cond_multiuse2(i1 %cond1, i1 %cond2) {
-; CHECK-LABEL: @replace_and_cond_multiuse2(
-; CHECK-NEXT:    [[AND:%.*]] = and i1 [[COND1:%.*]], [[COND2:%.*]]
+; CHECK-LABEL: define i32 @replace_and_cond_multiuse2(
+; CHECK-SAME: i1 [[COND1:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and i1 [[COND1]], [[COND2]]
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[AND]], i32 3, i32 2
 ; CHECK-NEXT:    call void @use32(i32 [[SEL]])
 ; CHECK-NEXT:    [[MUX:%.*]] = select i1 [[COND1]], i32 [[SEL]], i32 1
@@ -4887,8 +5200,9 @@ define i32 @replace_and_cond_multiuse2(i1 %cond1, i1 %cond2) {
 }
 
 define i32 @src_simplify_2x_at_once_and(i32 %x, i32 %y) {
-; CHECK-LABEL: @src_simplify_2x_at_once_and(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i32 @src_simplify_2x_at_once_and(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i32 [[XOR]]
 ;
   %and = and i32 %x, %y
@@ -4900,8 +5214,9 @@ define i32 @src_simplify_2x_at_once_and(i32 %x, i32 %y) {
 }
 
 define void @select_freeze_poison_parameter(ptr noundef %addr.src, ptr %addr.tgt, i1 %cond) {
-; CHECK-LABEL: @select_freeze_poison_parameter(
-; CHECK-NEXT:    store ptr [[ADDR_SRC:%.*]], ptr [[ADDR_TGT:%.*]], align 8
+; CHECK-LABEL: define void @select_freeze_poison_parameter(
+; CHECK-SAME: ptr noundef [[ADDR_SRC:%.*]], ptr [[ADDR_TGT:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    store ptr [[ADDR_SRC]], ptr [[ADDR_TGT]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %freeze = freeze ptr poison
@@ -4911,9 +5226,10 @@ define void @select_freeze_poison_parameter(ptr noundef %addr.src, ptr %addr.tgt
 }
 
 define i8 @select_freeze_poison_different_parameters(i8 noundef %x, i8 noundef %y, i1 %cond1, i1 %cond2) {
-; CHECK-LABEL: @select_freeze_poison_different_parameters(
-; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[COND1:%.*]], i8 [[X:%.*]], i8 0
-; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[COND2:%.*]], i8 [[Y:%.*]], i8 0
+; CHECK-LABEL: define i8 @select_freeze_poison_different_parameters(
+; CHECK-SAME: i8 noundef [[X:%.*]], i8 noundef [[Y:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:    [[SEL1:%.*]] = select i1 [[COND1]], i8 [[X]], i8 0
+; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[COND2]], i8 [[Y]], i8 0
 ; CHECK-NEXT:    [[CONJ:%.*]] = and i8 [[SEL1]], [[SEL2]]
 ; CHECK-NEXT:    ret i8 [[CONJ]]
 ;
@@ -4925,7 +5241,8 @@ define i8 @select_freeze_poison_different_parameters(i8 noundef %x, i8 noundef %
 }
 
 define i8 @select_or_freeze_poison_parameter(i8 noundef %x, i1 %cond1) {
-; CHECK-LABEL: @select_or_freeze_poison_parameter(
+; CHECK-LABEL: define i8 @select_or_freeze_poison_parameter(
+; CHECK-SAME: i8 noundef [[X:%.*]], i1 [[COND1:%.*]]) {
 ; CHECK-NEXT:    ret i8 -1
 ;
   %freeze = freeze i8 poison
@@ -4936,8 +5253,9 @@ define i8 @select_or_freeze_poison_parameter(i8 noundef %x, i1 %cond1) {
 }
 
 define i8 @selects_freeze_poison_parameter(i8 noundef %x, i1 %cond1, i1 %cond2) {
-; CHECK-LABEL: @selects_freeze_poison_parameter(
-; CHECK-NEXT:    ret i8 [[X:%.*]]
+; CHECK-LABEL: define i8 @selects_freeze_poison_parameter(
+; CHECK-SAME: i8 noundef [[X:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:    ret i8 [[X]]
 ;
   %freeze = freeze i8 poison
   %sel1 = select i1 %cond1, i8 %x, i8 %freeze
@@ -4949,8 +5267,9 @@ define i8 @selects_freeze_poison_parameter(i8 noundef %x, i1 %cond1, i1 %cond2)
 @glb = global ptr null
 
 define void @select_freeze_poison_global(ptr %addr.tgt, i1 %cond) {
-; CHECK-LABEL: @select_freeze_poison_global(
-; CHECK-NEXT:    store ptr @glb, ptr [[ADDR_TGT:%.*]], align 8
+; CHECK-LABEL: define void @select_freeze_poison_global(
+; CHECK-SAME: ptr [[ADDR_TGT:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    store ptr @glb, ptr [[ADDR_TGT]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %freeze = freeze ptr poison
@@ -4960,8 +5279,9 @@ define void @select_freeze_poison_global(ptr %addr.tgt, i1 %cond) {
 }
 
 define void @select_freeze_poison_constant(ptr %addr.tgt, i1 %cond) {
-; CHECK-LABEL: @select_freeze_poison_constant(
-; CHECK-NEXT:    store i32 72, ptr [[ADDR_TGT:%.*]], align 4
+; CHECK-LABEL: define void @select_freeze_poison_constant(
+; CHECK-SAME: ptr [[ADDR_TGT:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:    store i32 72, ptr [[ADDR_TGT]], align 4
 ; CHECK-NEXT:    ret void
 ;
   %freeze = freeze i32 poison
@@ -4971,8 +5291,9 @@ define void @select_freeze_poison_constant(ptr %addr.tgt, i1 %cond) {
 }
 
 define <2 x i8> @select_freeze_poison_mask_vector(i1 %cond, <2 x i8> noundef %y) {
-; CHECK-LABEL: @select_freeze_poison_mask_vector(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], <2 x i8> [[Y:%.*]], <2 x i8> zeroinitializer
+; CHECK-LABEL: define <2 x i8> @select_freeze_poison_mask_vector(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x i8> noundef [[Y:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], <2 x i8> [[Y]], <2 x i8> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %freeze = freeze <2 x i8> <i8 0, i8 poison>
@@ -4981,10 +5302,11 @@ define <2 x i8> @select_freeze_poison_mask_vector(i1 %cond, <2 x i8> noundef %y)
 }
 
 define <2 x i8> @selects_freeze_poison_mask_vector(<2 x i8> noundef %x, i1 %cond1, i1 %cond2) {
-; CHECK-LABEL: @selects_freeze_poison_mask_vector(
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[COND1:%.*]], i1 [[COND2:%.*]], i1 false
-; CHECK-NEXT:    [[X:%.*]] = select i1 [[TMP1]], <2 x i8> [[X1:%.*]], <2 x i8> zeroinitializer
-; CHECK-NEXT:    ret <2 x i8> [[X]]
+; CHECK-LABEL: define <2 x i8> @selects_freeze_poison_mask_vector(
+; CHECK-SAME: <2 x i8> noundef [[X:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[COND1]], i1 [[COND2]], i1 false
+; CHECK-NEXT:    [[CONJ:%.*]] = select i1 [[TMP1]], <2 x i8> [[X]], <2 x i8> zeroinitializer
+; CHECK-NEXT:    ret <2 x i8> [[CONJ]]
 ;
   %freeze = freeze <2 x i8> <i8 0, i8 poison>
   %sel1 = select i1 %cond1, <2 x i8> %x, <2 x i8> %freeze
@@ -4994,8 +5316,9 @@ define <2 x i8> @selects_freeze_poison_mask_vector(<2 x i8> noundef %x, i1 %cond
 }
 
 define <2 x i8> @select_freeze_poison_splat_vector(i1 %cond, <2 x i8> noundef %y) {
-; CHECK-LABEL: @select_freeze_poison_splat_vector(
-; CHECK-NEXT:    ret <2 x i8> [[Y:%.*]]
+; CHECK-LABEL: define <2 x i8> @select_freeze_poison_splat_vector(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x i8> noundef [[Y:%.*]]) {
+; CHECK-NEXT:    ret <2 x i8> [[Y]]
 ;
   %freeze = freeze <2 x i8> <i8 poison, i8 poison>
   %sel = select i1 %cond, <2 x i8> %y, <2 x i8> %freeze
@@ -5003,8 +5326,9 @@ define <2 x i8> @select_freeze_poison_splat_vector(i1 %cond, <2 x i8> noundef %y
 }
 
 define <2 x i8> @selects_freeze_poison_splat_vector(<2 x i8> noundef %x, i1 %cond1, i1 %cond2) {
-; CHECK-LABEL: @selects_freeze_poison_splat_vector(
-; CHECK-NEXT:    ret <2 x i8> [[X:%.*]]
+; CHECK-LABEL: define <2 x i8> @selects_freeze_poison_splat_vector(
+; CHECK-SAME: <2 x i8> noundef [[X:%.*]], i1 [[COND1:%.*]], i1 [[COND2:%.*]]) {
+; CHECK-NEXT:    ret <2 x i8> [[X]]
 ;
   %freeze = freeze <2 x i8> <i8 poison, i8 poison>
   %sel1 = select i1 %cond1, <2 x i8> %x, <2 x i8> %freeze
@@ -5014,8 +5338,9 @@ define <2 x i8> @selects_freeze_poison_splat_vector(<2 x i8> noundef %x, i1 %con
 }
 
 define <2 x i8> @select_freeze_constant_vector(i1 %cond, <2 x i8> noundef %y) {
-; CHECK-LABEL: @select_freeze_constant_vector(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], <2 x i8> [[Y:%.*]], <2 x i8> zeroinitializer
+; CHECK-LABEL: define <2 x i8> @select_freeze_constant_vector(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x i8> noundef [[Y:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], <2 x i8> [[Y]], <2 x i8> zeroinitializer
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %freeze = freeze <2 x i8> <i8 0, i8 0>
@@ -5024,8 +5349,9 @@ define <2 x i8> @select_freeze_constant_vector(i1 %cond, <2 x i8> noundef %y) {
 }
 
 define <2 x i8> @select_freeze_constant_expression_vector_add(i1 %cond, <2 x i8> noundef %y) {
-; CHECK-LABEL: @select_freeze_constant_expression_vector_add(
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], <2 x i8> [[Y:%.*]], <2 x i8> splat (i8 3)
+; CHECK-LABEL: define <2 x i8> @select_freeze_constant_expression_vector_add(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x i8> noundef [[Y:%.*]]) {
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], <2 x i8> [[Y]], <2 x i8> splat (i8 3)
 ; CHECK-NEXT:    ret <2 x i8> [[SEL]]
 ;
   %freeze = freeze <2 x i8> <i8 poison, i8 add (i8 1, i8 2)>
@@ -5037,9 +5363,10 @@ define <2 x i8> @select_freeze_constant_expression_vector_add(i1 %cond, <2 x i8>
 @glb.struct.1 = global %struct.1 {i32 1, i32 2}
 
 define <2 x ptr> @select_freeze_constant_expression_vector_gep(i1 %cond, <2 x ptr> noundef %y) {
-; CHECK-LABEL: @select_freeze_constant_expression_vector_gep(
+; CHECK-LABEL: define <2 x ptr> @select_freeze_constant_expression_vector_gep(
+; CHECK-SAME: i1 [[COND:%.*]], <2 x ptr> noundef [[Y:%.*]]) {
 ; CHECK-NEXT:    [[FREEZE:%.*]] = freeze <2 x ptr> <ptr poison, ptr getelementptr inbounds nuw (i8, ptr @glb.struct.1, i64 800)>
-; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], <2 x ptr> [[Y:%.*]], <2 x ptr> [[FREEZE]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND]], <2 x ptr> [[Y]], <2 x ptr> [[FREEZE]]
 ; CHECK-NEXT:    ret <2 x ptr> [[SEL]]
 ;
   %freeze = freeze <2 x ptr> <ptr poison, ptr getelementptr inbounds (%struct.1, ptr @glb.struct.1, i64 100)>
@@ -5048,24 +5375,25 @@ define <2 x ptr> @select_freeze_constant_expression_vector_gep(i1 %cond, <2 x pt
 }
 
 define void @no_fold_masked_min_loop(ptr nocapture readonly %vals, ptr nocapture readonly %masks, ptr nocapture %out, i64 %n) {
-; CHECK-LABEL: @no_fold_masked_min_loop(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXT_INDEX:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[ACC:%.*]] = phi i8 [ -1, [[ENTRY]] ], [ [[RES:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[VAL_PTR:%.*]] = getelementptr inbounds i8, ptr [[VALS:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[MASK_PTR:%.*]] = getelementptr inbounds i8, ptr [[MASKS:%.*]], i64 [[INDEX]]
+; CHECK-LABEL: define void @no_fold_masked_min_loop(
+; CHECK-SAME: ptr readonly captures(none) [[VALS:%.*]], ptr readonly captures(none) [[MASKS:%.*]], ptr captures(none) [[OUT:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT_INDEX:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[ACC:%.*]] = phi i8 [ -1, %[[ENTRY]] ], [ [[RES:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[VAL_PTR:%.*]] = getelementptr inbounds i8, ptr [[VALS]], i64 [[INDEX]]
+; CHECK-NEXT:    [[MASK_PTR:%.*]] = getelementptr inbounds i8, ptr [[MASKS]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[VAL:%.*]] = load i8, ptr [[VAL_PTR]], align 1
 ; CHECK-NEXT:    [[MASK:%.*]] = load i8, ptr [[MASK_PTR]], align 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i8 [[MASK]], 0
 ; CHECK-NEXT:    [[MASKED_VAL:%.*]] = select i1 [[COND]], i8 [[VAL]], i8 -1
 ; CHECK-NEXT:    [[RES]] = call i8 @llvm.umin.i8(i8 [[ACC]], i8 [[MASKED_VAL]])
 ; CHECK-NEXT:    [[NEXT_INDEX]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXT_INDEX]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]]
-; CHECK:       exit:
-; CHECK-NEXT:    store i8 [[RES]], ptr [[OUT:%.*]], align 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXT_INDEX]], [[N]]
+; CHECK-NEXT:    br i1 [[DONE]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    store i8 [[RES]], ptr [[OUT]], align 1
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/InstCombine/sink_sideeffecting_instruction.ll b/llvm/test/Transforms/InstCombine/sink_sideeffecting_instruction.ll
index ccb96012ae3cc..bd43daa4aed3b 100644
--- a/llvm/test/Transforms/InstCombine/sink_sideeffecting_instruction.ll
+++ b/llvm/test/Transforms/InstCombine/sink_sideeffecting_instruction.ll
@@ -49,12 +49,12 @@ define i32 @test() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[VAR:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[VAR1:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    [[VAR3:%.*]] = call i32 @foo(ptr nonnull writeonly [[VAR]])
 ; CHECK-NEXT:    [[VAR4:%.*]] = icmp eq i32 [[VAR3]], 0
 ; CHECK-NEXT:    br i1 [[VAR4]], label [[BB5:%.*]], label [[BB14:%.*]]
 ; CHECK:       bb5:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[VAR1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VAR1]])
 ; CHECK-NEXT:    [[VAR8:%.*]] = load i32, ptr [[VAR]], align 4
 ; CHECK-NEXT:    [[VAR9:%.*]] = icmp eq i32 [[VAR8]], 0
 ; CHECK-NEXT:    [[VAR7:%.*]] = call i32 @foo(ptr nonnull writeonly [[VAR1]])
@@ -66,23 +66,23 @@ define i32 @test() {
 ; CHECK-NEXT:    br label [[BB12]]
 ; CHECK:       bb12:
 ; CHECK-NEXT:    [[VAR13:%.*]] = phi i32 [ [[VAR11]], [[BB10]] ], [ [[VAR7]], [[BB_CRIT_EDGE]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[VAR1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VAR1]])
 ; CHECK-NEXT:    br label [[BB14]]
 ; CHECK:       bb14:
 ; CHECK-NEXT:    [[VAR15:%.*]] = phi i32 [ [[VAR13]], [[BB12]] ], [ 0, [[BB:%.*]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    ret i32 [[VAR15]]
 ;
 bb:
   %var = alloca i32, align 4
   %var1 = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %var) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %var) #4
   %var3 = call i32 @foo(ptr nonnull writeonly %var)
   %var4 = icmp eq i32 %var3, 0
   br i1 %var4, label %bb5, label %bb14
 
 bb5:                                              ; preds = %bb
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %var1) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %var1) #4
   %var8 = load i32, ptr %var, align 4
   %var9 = icmp eq i32 %var8, 0
   %var7 = call i32 @foo(ptr nonnull writeonly %var1)
@@ -97,12 +97,12 @@ bb_crit_edge:
 
 bb12:                                             ; preds = %bb10, %bb5
   %var13 = phi i32 [ %var11, %bb10 ], [ %var7, %bb_crit_edge ]
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %var1) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %var1) #4
   br label %bb14
 
 bb14:                                             ; preds = %bb12, %bb
   %var15 = phi i32 [ %var13, %bb12 ], [ 0, %bb ]
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %var)
+  call void @llvm.lifetime.end.p0(ptr nonnull %var)
   ret i32 %var15
 }
 
@@ -325,18 +325,18 @@ define i32 @sink_lifetime1(i1 %c) {
 ; CHECK-LABEL: @sink_lifetime1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[VAR:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    [[VAR3:%.*]] = call i32 @unknown(ptr nonnull [[VAR]]) #[[ATTR1]]
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[EARLY_RETURN:%.*]], label [[USE_BLOCK:%.*]]
 ; CHECK:       early_return:
 ; CHECK-NEXT:    ret i32 0
 ; CHECK:       use_block:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    ret i32 [[VAR3]]
 ;
 entry:
   %var = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %var)
+  call void @llvm.lifetime.start.p0(ptr %var)
   %var3 = call i32 @unknown(ptr %var) argmemonly nounwind willreturn
   br i1 %c, label %early_return, label %use_block
 
@@ -344,7 +344,7 @@ early_return:
   ret i32 0
 
 use_block:
-  call void @llvm.lifetime.end.p0(i64 4, ptr %var)
+  call void @llvm.lifetime.end.p0(ptr %var)
   ret i32 %var3
 }
 
@@ -352,25 +352,25 @@ define i32 @sink_lifetime2(i1 %c) {
 ; CHECK-LABEL: @sink_lifetime2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[VAR:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    [[VAR3:%.*]] = call i32 @unknown(ptr nonnull [[VAR]]) #[[ATTR1]]
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[MERGE:%.*]], label [[USE_BLOCK:%.*]]
 ; CHECK:       merge:
 ; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[VAR3]], [[USE_BLOCK]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    ret i32 [[RET]]
 ; CHECK:       use_block:
 ; CHECK-NEXT:    br label [[MERGE]]
 ;
 entry:
   %var = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %var)
+  call void @llvm.lifetime.start.p0(ptr %var)
   %var3 = call i32 @unknown(ptr %var) argmemonly nounwind willreturn
   br i1 %c, label %merge, label %use_block
 
 merge:
   %ret = phi i32 [0, %entry], [%var3, %use_block]
-  call void @llvm.lifetime.end.p0(i64 4, ptr %var)
+  call void @llvm.lifetime.end.p0(ptr %var)
   ret i32 %ret
 
 use_block:
@@ -390,8 +390,8 @@ define i32 @sink_lifetime3(i1 %c) {
 ;
 entry:
   %var = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %var)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %var)
+  call void @llvm.lifetime.start.p0(ptr %var)
+  call void @llvm.lifetime.end.p0(ptr %var)
   ; If unknown accesses %var, that's UB
   %var3 = call i32 @unknown(ptr %var) argmemonly nounwind willreturn
   br i1 %c, label %early_return, label %use_block
@@ -407,9 +407,9 @@ define i32 @sink_lifetime4a(i1 %c) {
 ; CHECK-LABEL: @sink_lifetime4a(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[VAR:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    [[VAR3:%.*]] = call i32 @unknown(ptr nonnull [[VAR]]) #[[ATTR1]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[EARLY_RETURN:%.*]], label [[USE_BLOCK:%.*]]
 ; CHECK:       early_return:
 ; CHECK-NEXT:    ret i32 0
@@ -418,9 +418,9 @@ define i32 @sink_lifetime4a(i1 %c) {
 ;
 entry:
   %var = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %var)
+  call void @llvm.lifetime.start.p0(ptr %var)
   %var3 = call i32 @unknown(ptr %var) argmemonly nounwind willreturn
-  call void @llvm.lifetime.end.p0(i64 4, ptr %var)
+  call void @llvm.lifetime.end.p0(ptr %var)
   br i1 %c, label %early_return, label %use_block
 
 early_return:
@@ -436,9 +436,9 @@ define i32 @sink_lifetime4b(i1 %c) {
 ; CHECK-LABEL: @sink_lifetime4b(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[VAR:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    [[VAR3:%.*]] = call i32 @unknown(ptr nonnull writeonly [[VAR]]) #[[ATTR1]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[VAR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VAR]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[EARLY_RETURN:%.*]], label [[USE_BLOCK:%.*]]
 ; CHECK:       early_return:
 ; CHECK-NEXT:    ret i32 0
@@ -447,9 +447,9 @@ define i32 @sink_lifetime4b(i1 %c) {
 ;
 entry:
   %var = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %var)
+  call void @llvm.lifetime.start.p0(ptr %var)
   %var3 = call i32 @unknown(ptr writeonly %var) argmemonly nounwind willreturn
-  call void @llvm.lifetime.end.p0(i64 4, ptr %var)
+  call void @llvm.lifetime.end.p0(ptr %var)
   br i1 %c, label %early_return, label %use_block
 
 early_return:
@@ -486,6 +486,6 @@ use_block:
 
 
 declare i32 @bar()
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
diff --git a/llvm/test/Transforms/InstCombine/strlen-1.ll b/llvm/test/Transforms/InstCombine/strlen-1.ll
index 4616767997512..27f9e4ace3a7b 100644
--- a/llvm/test/Transforms/InstCombine/strlen-1.ll
+++ b/llvm/test/Transforms/InstCombine/strlen-1.ll
@@ -116,6 +116,25 @@ define i32 @test_simplify10_inbounds(i32 %x) {
   ret i32 %hello_l
 }
 
+define i32 @test_simplify10_inbounds_i8gep(i32 %x) {
+; CHECK-LABEL: @test_simplify10_inbounds_i8gep(
+; CHECK-NEXT:    [[HELLO_L:%.*]] = sub i32 5, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds i8, ptr @hello, i32 %x
+  %hello_l = call i32 @strlen(ptr %hello_p)
+  ret i32 %hello_l
+}
+
+define i32 @test_simplify10_inbounds_i8gep_const() {
+; CHECK-LABEL: @test_simplify10_inbounds_i8gep_const(
+; CHECK-NEXT:    ret i32 3
+;
+  %hello_p = getelementptr inbounds i8, ptr @hello, i32 2
+  %hello_l = call i32 @strlen(ptr %hello_p)
+  ret i32 %hello_l
+}
+
 define i32 @test_simplify10_no_inbounds(i32 %x) {
 ; CHECK-LABEL: @test_simplify10_no_inbounds(
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = sub i32 5, [[X:%.*]]
@@ -126,6 +145,16 @@ define i32 @test_simplify10_no_inbounds(i32 %x) {
   ret i32 %hello_l
 }
 
+define i32 @test_simplify10_no_inbounds_i8gep(i32 %x) {
+; CHECK-LABEL: @test_simplify10_no_inbounds_i8gep(
+; CHECK-NEXT:    [[HELLO_L:%.*]] = sub i32 5, [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[HELLO_L]]
+;
+  %hello_p = getelementptr i8, ptr @hello, i32 %x
+  %hello_l = call i32 @strlen(ptr %hello_p)
+  ret i32 %hello_l
+}
+
 ; strlen(@null_hello_mid + (x & 7)) --> 9 - (x & 7)
 
 define i32 @test_simplify11(i32 %x) {
diff --git a/llvm/test/Transforms/InstCombine/sub-of-negatible-inseltpoison.ll b/llvm/test/Transforms/InstCombine/sub-of-negatible-inseltpoison.ll
index 3bbb9b931e433..109669e91d22f 100644
--- a/llvm/test/Transforms/InstCombine/sub-of-negatible-inseltpoison.ll
+++ b/llvm/test/Transforms/InstCombine/sub-of-negatible-inseltpoison.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 declare void @use4(i4)
@@ -8,8 +8,9 @@ declare i1 @use32gen1(i32)
 
 ; Constant can be freely negated.
 define i8 @t0(i8 %x) {
-; CHECK-LABEL: @t0(
-; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @t0(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    ret i8 [[T0]]
 ;
   %t0 = sub i8 %x, -42
@@ -18,10 +19,11 @@ define i8 @t0(i8 %x) {
 
 ; Negation can be negated for free
 define i8 @t1(i8 %x, i8 %y) {
-; CHECK-LABEL: @t1(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @t1(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sub i8 0, %y
@@ -32,9 +34,10 @@ define i8 @t1(i8 %x, i8 %y) {
 
 ; Shift-left can be negated if all uses can be updated
 define i8 @t2(i8 %x, i8 %y) {
-; CHECK-LABEL: @t2(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl i8 42, [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @t2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl i8 42, [[Y]]
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = shl i8 -42, %y
@@ -42,10 +45,11 @@ define i8 @t2(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @n2(i8 %x, i8 %y) {
-; CHECK-LABEL: @n2(
-; CHECK-NEXT:    [[T0:%.*]] = shl i8 -42, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -42, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = shl i8 -42, %y
@@ -54,11 +58,12 @@ define i8 @n2(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @t3(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @t3(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @t3(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = shl i8 [[Z]], [[Y:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = shl i8 [[Z]], [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -68,12 +73,13 @@ define i8 @t3(i8 %x, i8 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @n3(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n3(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @n3(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -86,9 +92,10 @@ define i8 @n3(i8 %x, i8 %y, i8 %z) {
 
 ; Select can be negated if all it's operands can be negated and all the users of select can be updated
 define i8 @t4(i8 %x, i1 %y) {
-; CHECK-LABEL: @t4(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = select i1 [[Y:%.*]], i8 42, i8 -44
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @t4(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = select i1 [[Y]], i8 42, i8 -44
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = select i1 %y, i8 -42, i8 44
@@ -96,10 +103,11 @@ define i8 @t4(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @n4(i8 %x, i1 %y) {
-; CHECK-LABEL: @n4(
-; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y:%.*]], i8 -42, i8 44
+; CHECK-LABEL: define i8 @n4(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y]], i8 -42, i8 44
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = select i1 %y, i8 -42, i8 44
@@ -108,9 +116,10 @@ define i8 @n4(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @n5(i8 %x, i1 %y, i8 %z) {
-; CHECK-LABEL: @n5(
-; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y:%.*]], i8 -42, i8 [[Z:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @n5(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y]], i8 -42, i8 [[Z]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = select i1 %y, i8 -42, i8 %z
@@ -118,11 +127,12 @@ define i8 @n5(i8 %x, i1 %y, i8 %z) {
   ret i8 %t1
 }
 define i8 @t6(i8 %x, i1 %y, i8 %z) {
-; CHECK-LABEL: @t6(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @t6(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = select i1 [[Y:%.*]], i8 42, i8 [[Z]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = select i1 [[Y]], i8 42, i8 [[Z]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -132,10 +142,11 @@ define i8 @t6(i8 %x, i1 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @t7(i8 %x, i1 %y, i8 %z) {
-; CHECK-LABEL: @t7(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl nsw i8 -1, [[Z:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = select i1 [[Y:%.*]], i8 0, i8 [[T0_NEG]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @t7(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl nsw i8 -1, [[Z]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = select i1 [[Y]], i8 0, i8 [[T0_NEG]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 1, %z
@@ -144,11 +155,12 @@ define i8 @t7(i8 %x, i1 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @n8(i8 %x, i1 %y, i8 %z) {
-; CHECK-LABEL: @n8(
-; CHECK-NEXT:    [[T0:%.*]] = shl nuw i8 1, [[Z:%.*]]
+; CHECK-LABEL: define i8 @n8(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl nuw i8 1, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = select i1 [[Y:%.*]], i8 0, i8 [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[Y]], i8 0, i8 [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 1, %z
@@ -161,8 +173,9 @@ define i8 @n8(i8 %x, i1 %y, i8 %z) {
 ; Subtraction can be negated by swapping its operands.
 ; x - (y - z) -> x - y + z -> x + (z - y)
 define i8 @t9(i8 %x, i8 %y) {
-; CHECK-LABEL: @t9(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @t9(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i8 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T0_NEG]]
 ;
   %t0 = sub i8 %y, %x
@@ -171,8 +184,9 @@ define i8 @t9(i8 %x, i8 %y) {
 }
 
 define i8 @n10(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n10(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i8 @n10(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[Y]], [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
@@ -184,8 +198,9 @@ define i8 @n10(i8 %x, i8 %y, i8 %z) {
 }
 
 define i8 @neg_of_sub_from_constant(i8 %x) {
-; CHECK-LABEL: @neg_of_sub_from_constant(
-; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X:%.*]], -42
+; CHECK-LABEL: define i8 @neg_of_sub_from_constant(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X]], -42
 ; CHECK-NEXT:    ret i8 [[S_NEG]]
 ;
   %s = sub i8 42, %x
@@ -194,8 +209,9 @@ define i8 @neg_of_sub_from_constant(i8 %x) {
 }
 
 define i8 @neg_of_sub_from_constant_multi_use(i8 %x) {
-; CHECK-LABEL: @neg_of_sub_from_constant_multi_use(
-; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X:%.*]], -42
+; CHECK-LABEL: define i8 @neg_of_sub_from_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X]], -42
 ; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
 ; CHECK-NEXT:    ret i8 [[S_NEG]]
@@ -207,8 +223,9 @@ define i8 @neg_of_sub_from_constant_multi_use(i8 %x) {
 }
 
 define i8 @sub_from_constant_of_sub_from_constant(i8 %x) {
-; CHECK-LABEL: @sub_from_constant_of_sub_from_constant(
-; CHECK-NEXT:    [[R:%.*]] = add i8 [[X:%.*]], -31
+; CHECK-LABEL: define i8 @sub_from_constant_of_sub_from_constant(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[X]], -31
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = sub i8 42, %x
@@ -217,8 +234,9 @@ define i8 @sub_from_constant_of_sub_from_constant(i8 %x) {
 }
 
 define i8 @sub_from_constant_of_sub_from_constant_multi_use(i8 %x) {
-; CHECK-LABEL: @sub_from_constant_of_sub_from_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X:%.*]]
+; CHECK-LABEL: define i8 @sub_from_constant_of_sub_from_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
 ; CHECK-NEXT:    [[R:%.*]] = add i8 [[X]], -31
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -230,9 +248,10 @@ define i8 @sub_from_constant_of_sub_from_constant_multi_use(i8 %x) {
 }
 
 define i8 @sub_from_variable_of_sub_from_constant(i8 %x, i8 %y) {
-; CHECK-LABEL: @sub_from_variable_of_sub_from_constant(
-; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X:%.*]], -42
-; CHECK-NEXT:    [[R:%.*]] = add i8 [[S_NEG]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @sub_from_variable_of_sub_from_constant(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X]], -42
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[S_NEG]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = sub i8 42, %x
@@ -241,10 +260,11 @@ define i8 @sub_from_variable_of_sub_from_constant(i8 %x, i8 %y) {
 }
 
 define i8 @sub_from_variable_of_sub_from_constant_multi_use(i8 %x, i8 %y) {
-; CHECK-LABEL: @sub_from_variable_of_sub_from_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X:%.*]]
+; CHECK-LABEL: define i8 @sub_from_variable_of_sub_from_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
-; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y:%.*]], [[S]]
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y]], [[S]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = sub i8 42, %x
@@ -256,13 +276,14 @@ define i8 @sub_from_variable_of_sub_from_constant_multi_use(i8 %x, i8 %y) {
 ; Addition can be negated if both operands can be negated
 ; x - (y + z) -> x - y - z -> x + ((-y) + (-z)))
 define i8 @t12(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @t12(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @t12(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y]], [[Z]]
-; CHECK-NEXT:    [[T3:%.*]] = add i8 [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -274,11 +295,12 @@ define i8 @t12(i8 %x, i8 %y, i8 %z) {
   ret i8 %t3
 }
 define i8 @n13(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n13(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n13(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = sub i8 [[Y]], [[Z:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = sub i8 [[Y]], [[Z]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %y
@@ -288,15 +310,16 @@ define i8 @n13(i8 %x, i8 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @n14(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n14(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n14(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y]], [[Z]]
 ; CHECK-NEXT:    [[T2:%.*]] = sub i8 0, [[TMP1]]
 ; CHECK-NEXT:    call void @use8(i8 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = add i8 [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -310,8 +333,9 @@ define i8 @n14(i8 %x, i8 %y, i8 %z) {
 }
 
 define i8 @neg_of_add_with_constant(i8 %x) {
-; CHECK-LABEL: @neg_of_add_with_constant(
-; CHECK-NEXT:    [[R:%.*]] = sub i8 -42, [[X:%.*]]
+; CHECK-LABEL: define i8 @neg_of_add_with_constant(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = sub i8 -42, [[X]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = add i8 %x, 42
@@ -320,8 +344,9 @@ define i8 @neg_of_add_with_constant(i8 %x) {
 }
 
 define i8 @neg_of_add_with_constant_multi_use(i8 %x) {
-; CHECK-LABEL: @neg_of_add_with_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @neg_of_add_with_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
 ; CHECK-NEXT:    [[R:%.*]] = sub i8 -42, [[X]]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -333,8 +358,9 @@ define i8 @neg_of_add_with_constant_multi_use(i8 %x) {
 }
 
 define i8 @sub_from_constant_of_add_with_constant(i8 %x) {
-; CHECK-LABEL: @sub_from_constant_of_add_with_constant(
-; CHECK-NEXT:    [[R:%.*]] = sub i8 -31, [[X:%.*]]
+; CHECK-LABEL: define i8 @sub_from_constant_of_add_with_constant(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = sub i8 -31, [[X]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = add i8 %x, 42
@@ -343,8 +369,9 @@ define i8 @sub_from_constant_of_add_with_constant(i8 %x) {
 }
 
 define i8 @sub_from_constant_of_add_with_constant_multi_use(i8 %x) {
-; CHECK-LABEL: @sub_from_constant_of_add_with_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @sub_from_constant_of_add_with_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
 ; CHECK-NEXT:    [[R:%.*]] = sub i8 -31, [[X]]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -356,9 +383,10 @@ define i8 @sub_from_constant_of_add_with_constant_multi_use(i8 %x) {
 }
 
 define i8 @sub_from_variable_of_add_with_constant(i8 %x, i8 %y) {
-; CHECK-LABEL: @sub_from_variable_of_add_with_constant(
-; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], 42
-; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y:%.*]], [[S]]
+; CHECK-LABEL: define i8 @sub_from_variable_of_add_with_constant(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X]], 42
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y]], [[S]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = add i8 %x, 42
@@ -367,10 +395,11 @@ define i8 @sub_from_variable_of_add_with_constant(i8 %x, i8 %y) {
 }
 
 define i8 @sub_from_variable_of_add_with_constant_multi_use(i8 %x, i8 %y) {
-; CHECK-LABEL: @sub_from_variable_of_add_with_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @sub_from_variable_of_add_with_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
-; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y:%.*]], [[S]]
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y]], [[S]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = add i8 %x, 42
@@ -382,11 +411,12 @@ define i8 @sub_from_variable_of_add_with_constant_multi_use(i8 %x, i8 %y) {
 ; Multiplication can be negated if either one of operands can be negated
 ; x - (y * z) -> x + ((-y) * z) or  x + ((-z) * y)
 define i8 @t15(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @t15(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @t15(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = mul i8 [[Y]], [[Z:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = mul i8 [[Y]], [[Z]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %y
@@ -396,12 +426,13 @@ define i8 @t15(i8 %x, i8 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @n16(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n16(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n16(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = mul i8 [[Z:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = mul i8 [[Z]], [[T0]]
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %y
@@ -414,15 +445,16 @@ define i8 @n16(i8 %x, i8 %y, i8 %z) {
 
 ; Phi can be negated if all incoming values can be negated
 define i8 @t16(i1 %c, i8 %x) {
-; CHECK-LABEL: @t16(
-; CHECK-NEXT:  begin:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       else:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[Z_NEG:%.*]] = phi i8 [ [[X:%.*]], [[THEN]] ], [ 42, [[ELSE]] ]
+; CHECK-LABEL: define i8 @t16(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]]) {
+; CHECK-NEXT:  [[BEGIN:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[END:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    [[Z_NEG:%.*]] = phi i8 [ [[X]], %[[THEN]] ], [ 42, %[[ELSE]] ]
 ; CHECK-NEXT:    ret i8 [[Z_NEG]]
 ;
 begin:
@@ -438,16 +470,17 @@ end:
   ret i8 %n
 }
 define i8 @n17(i1 %c, i8 %x) {
-; CHECK-LABEL: @n17(
-; CHECK-NEXT:  begin:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    [[Y:%.*]] = sub i8 0, [[X:%.*]]
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       else:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[Z:%.*]] = phi i8 [ [[Y]], [[THEN]] ], [ -42, [[ELSE]] ]
+; CHECK-LABEL: define i8 @n17(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]]) {
+; CHECK-NEXT:  [[BEGIN:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[Y:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    br label %[[END:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    [[Z:%.*]] = phi i8 [ [[Y]], %[[THEN]] ], [ -42, %[[ELSE]] ]
 ; CHECK-NEXT:    call void @use8(i8 [[Z]])
 ; CHECK-NEXT:    [[N:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    ret i8 [[N]]
@@ -466,16 +499,17 @@ end:
   ret i8 %n
 }
 define i8 @n19(i1 %c, i8 %x, i8 %y) {
-; CHECK-LABEL: @n19(
-; CHECK-NEXT:  begin:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    [[Z:%.*]] = sub i8 0, [[X:%.*]]
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       else:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[R:%.*]] = phi i8 [ [[Z]], [[THEN]] ], [ [[Y:%.*]], [[ELSE]] ]
+; CHECK-LABEL: define i8 @n19(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:  [[BEGIN:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[Z:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    br label %[[END:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    [[R:%.*]] = phi i8 [ [[Z]], %[[THEN]] ], [ [[Y]], %[[ELSE]] ]
 ; CHECK-NEXT:    [[N:%.*]] = sub i8 0, [[R]]
 ; CHECK-NEXT:    ret i8 [[N]]
 ;
@@ -492,21 +526,22 @@ end:
   ret i8 %n
 }
 define void @phi_with_duplicate_incoming_basic_blocks(i32 %x, i32 %y, i1 %should_lookup, i32 %z) {
-; CHECK-LABEL: @phi_with_duplicate_incoming_basic_blocks(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X_INC_NEG:%.*]] = xor i32 [[X:%.*]], -1
-; CHECK-NEXT:    br i1 [[SHOULD_LOOKUP:%.*]], label [[LOOKUP:%.*]], label [[LOOP:%.*]]
-; CHECK:       lookup:
-; CHECK-NEXT:    [[TO_LOOKUP:%.*]] = phi i32 [ [[Y:%.*]], [[ENTRY:%.*]] ], [ [[METAVAL_NEG:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    switch i32 [[TO_LOOKUP]], label [[END:%.*]] [
-; CHECK-NEXT:      i32 0, label [[LOOP]]
-; CHECK-NEXT:      i32 42, label [[LOOP]]
+; CHECK-LABEL: define void @phi_with_duplicate_incoming_basic_blocks(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i1 [[SHOULD_LOOKUP:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[X_INC_NEG:%.*]] = xor i32 [[X]], -1
+; CHECK-NEXT:    br i1 [[SHOULD_LOOKUP]], label %[[LOOKUP:.*]], label %[[LOOP:.*]]
+; CHECK:       [[LOOKUP]]:
+; CHECK-NEXT:    [[TO_LOOKUP:%.*]] = phi i32 [ [[Y]], %[[ENTRY]] ], [ [[METAVAL_NEG:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    switch i32 [[TO_LOOKUP]], label %[[END:.*]] [
+; CHECK-NEXT:      i32 0, label %[[LOOP]]
+; CHECK-NEXT:      i32 42, label %[[LOOP]]
 ; CHECK-NEXT:    ]
-; CHECK:       loop:
-; CHECK-NEXT:    [[METAVAL_NEG]] = phi i32 [ [[X_INC_NEG]], [[LOOKUP]] ], [ [[X_INC_NEG]], [[LOOKUP]] ], [ -84, [[ENTRY]] ]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[METAVAL_NEG]] = phi i32 [ [[X_INC_NEG]], %[[LOOKUP]] ], [ [[X_INC_NEG]], %[[LOOKUP]] ], [ -84, %[[ENTRY]] ]
 ; CHECK-NEXT:    [[REPEAT:%.*]] = call i1 @use32gen1(i32 [[METAVAL_NEG]])
-; CHECK-NEXT:    br i1 [[REPEAT]], label [[LOOKUP]], label [[END]]
-; CHECK:       end:
+; CHECK-NEXT:    br i1 [[REPEAT]], label %[[LOOKUP]], label %[[END]]
+; CHECK:       [[END]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -532,10 +567,11 @@ end:
 
 ; truncation can be negated if it's operand can be negated
 define i8 @t20(i8 %x, i16 %y) {
-; CHECK-LABEL: @t20(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl i16 42, [[Y:%.*]]
+; CHECK-LABEL: define i8 @t20(
+; CHECK-SAME: i8 [[X:%.*]], i16 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl i16 42, [[Y]]
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = trunc i16 [[T0_NEG]] to i8
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X:%.*]], [[T1_NEG]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X]], [[T1_NEG]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i16 -42, %y
@@ -544,11 +580,12 @@ define i8 @t20(i8 %x, i16 %y) {
   ret i8 %t2
 }
 define i8 @n21(i8 %x, i16 %y) {
-; CHECK-LABEL: @n21(
-; CHECK-NEXT:    [[T0:%.*]] = shl i16 -42, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n21(
+; CHECK-SAME: i8 [[X:%.*]], i16 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i16 -42, [[Y]]
 ; CHECK-NEXT:    [[T1:%.*]] = trunc i16 [[T0]] to i8
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i16 -42, %y
@@ -559,8 +596,9 @@ define i8 @n21(i8 %x, i16 %y) {
 }
 
 define i4 @negate_xor(i4 %x) {
-; CHECK-LABEL: @negate_xor(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], -6
+; CHECK-LABEL: define i4 @negate_xor(
+; CHECK-SAME: i4 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X]], -6
 ; CHECK-NEXT:    [[O_NEG:%.*]] = add i4 [[TMP1]], 1
 ; CHECK-NEXT:    ret i4 [[O_NEG]]
 ;
@@ -570,8 +608,9 @@ define i4 @negate_xor(i4 %x) {
 }
 
 define <2 x i4> @negate_xor_vec(<2 x i4> %x) {
-; CHECK-LABEL: @negate_xor_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i4> [[X:%.*]], <i4 -6, i4 5>
+; CHECK-LABEL: define <2 x i4> @negate_xor_vec(
+; CHECK-SAME: <2 x i4> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i4> [[X]], <i4 -6, i4 5>
 ; CHECK-NEXT:    [[O_NEG:%.*]] = add <2 x i4> [[TMP1]], splat (i4 1)
 ; CHECK-NEXT:    ret <2 x i4> [[O_NEG]]
 ;
@@ -581,8 +620,9 @@ define <2 x i4> @negate_xor_vec(<2 x i4> %x) {
 }
 
 define i8 @negate_xor_use(i8 %x) {
-; CHECK-LABEL: @negate_xor_use(
-; CHECK-NEXT:    [[O:%.*]] = xor i8 [[X:%.*]], 5
+; CHECK-LABEL: define i8 @negate_xor_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[O:%.*]] = xor i8 [[X]], 5
 ; CHECK-NEXT:    call void @use8(i8 [[O]])
 ; CHECK-NEXT:    [[R:%.*]] = sub i8 0, [[O]]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -594,10 +634,11 @@ define i8 @negate_xor_use(i8 %x) {
 }
 
 define i4 @negate_shl_xor(i4 %x, i4 %y) {
-; CHECK-LABEL: @negate_shl_xor(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], -6
+; CHECK-LABEL: define i4 @negate_shl_xor(
+; CHECK-SAME: i4 [[X:%.*]], i4 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X]], -6
 ; CHECK-NEXT:    [[O_NEG:%.*]] = add i4 [[TMP1]], 1
-; CHECK-NEXT:    [[S_NEG:%.*]] = shl i4 [[O_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[S_NEG:%.*]] = shl i4 [[O_NEG]], [[Y]]
 ; CHECK-NEXT:    ret i4 [[S_NEG]]
 ;
   %o = xor i4 %x, 5
@@ -607,11 +648,12 @@ define i4 @negate_shl_xor(i4 %x, i4 %y) {
 }
 
 define i8 @negate_shl_not_uses(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_shl_not_uses(
-; CHECK-NEXT:    [[O_NEG:%.*]] = add i8 [[X:%.*]], 1
+; CHECK-LABEL: define i8 @negate_shl_not_uses(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[O_NEG:%.*]] = add i8 [[X]], 1
 ; CHECK-NEXT:    [[O:%.*]] = xor i8 [[X]], -1
 ; CHECK-NEXT:    call void @use8(i8 [[O]])
-; CHECK-NEXT:    [[S_NEG:%.*]] = shl i8 [[O_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[S_NEG:%.*]] = shl i8 [[O_NEG]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[S_NEG]]
 ;
   %o = xor i8 %x, -1
@@ -622,11 +664,12 @@ define i8 @negate_shl_not_uses(i8 %x, i8 %y) {
 }
 
 define <2 x i4> @negate_mul_not_uses_vec(<2 x i4> %x, <2 x i4> %y) {
-; CHECK-LABEL: @negate_mul_not_uses_vec(
-; CHECK-NEXT:    [[O_NEG:%.*]] = add <2 x i4> [[X:%.*]], splat (i4 1)
+; CHECK-LABEL: define <2 x i4> @negate_mul_not_uses_vec(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]]) {
+; CHECK-NEXT:    [[O_NEG:%.*]] = add <2 x i4> [[X]], splat (i4 1)
 ; CHECK-NEXT:    [[O:%.*]] = xor <2 x i4> [[X]], splat (i4 -1)
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[O]])
-; CHECK-NEXT:    [[S_NEG:%.*]] = mul <2 x i4> [[O_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[S_NEG:%.*]] = mul <2 x i4> [[O_NEG]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i4> [[S_NEG]]
 ;
   %o = xor <2 x i4> %x, <i4 -1, i4 -1>
@@ -638,9 +681,10 @@ define <2 x i4> @negate_mul_not_uses_vec(<2 x i4> %x, <2 x i4> %y) {
 
 ; signed division can be negated if divisor can be negated and is not 1/-1
 define i8 @negate_sdiv(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_sdiv(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sdiv i8 [[Y:%.*]], -42
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_sdiv(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = sdiv i8 [[Y]], -42
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sdiv i8 %y, 42
@@ -648,10 +692,11 @@ define i8 @negate_sdiv(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_sdiv_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_sdiv_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sdiv i8 [[Y:%.*]], 42
+; CHECK-LABEL: define i8 @negate_sdiv_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sdiv i8 [[Y]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sdiv i8 %y, 42
@@ -660,8 +705,9 @@ define i8 @negate_sdiv_extrause(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_sdiv_extrause2(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_sdiv_extrause2(
-; CHECK-NEXT:    [[T0:%.*]] = sdiv i8 [[Y:%.*]], 42
+; CHECK-LABEL: define i8 @negate_sdiv_extrause2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sdiv i8 [[Y]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = sub nsw i8 0, [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
@@ -674,9 +720,10 @@ define i8 @negate_sdiv_extrause2(i8 %x, i8 %y) {
 
 ; Right-shift sign bit smear is negatible.
 define i8 @negate_ashr(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_ashr(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = lshr i8 [[Y:%.*]], 7
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_ashr(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = lshr i8 [[Y]], 7
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = ashr i8 %y, 7
@@ -684,9 +731,10 @@ define i8 @negate_ashr(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_lshr(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_lshr(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = ashr i8 [[Y:%.*]], 7
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_lshr(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = ashr i8 [[Y]], 7
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = lshr i8 %y, 7
@@ -694,10 +742,11 @@ define i8 @negate_lshr(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_ashr_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_ashr_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = ashr i8 [[Y:%.*]], 7
+; CHECK-LABEL: define i8 @negate_ashr_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = ashr i8 [[Y]], 7
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = ashr i8 %y, 7
@@ -706,10 +755,11 @@ define i8 @negate_ashr_extrause(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_lshr_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_lshr_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = lshr i8 [[Y:%.*]], 7
+; CHECK-LABEL: define i8 @negate_lshr_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = lshr i8 [[Y]], 7
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = lshr i8 %y, 7
@@ -718,9 +768,10 @@ define i8 @negate_lshr_extrause(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_ashr_wrongshift(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_ashr_wrongshift(
-; CHECK-NEXT:    [[T0:%.*]] = ashr i8 [[Y:%.*]], 6
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_ashr_wrongshift(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = ashr i8 [[Y]], 6
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = ashr i8 %y, 6
@@ -728,9 +779,10 @@ define i8 @negate_ashr_wrongshift(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_lshr_wrongshift(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_lshr_wrongshift(
-; CHECK-NEXT:    [[T0:%.*]] = lshr i8 [[Y:%.*]], 6
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_lshr_wrongshift(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = lshr i8 [[Y]], 6
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = lshr i8 %y, 6
@@ -740,9 +792,10 @@ define i8 @negate_lshr_wrongshift(i8 %x, i8 %y) {
 
 ; *ext of i1 is always negatible
 define i8 @negate_sext(i8 %x, i1 %y) {
-; CHECK-LABEL: @negate_sext(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = zext i1 [[Y:%.*]] to i8
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X:%.*]], [[T0_NEG]]
+; CHECK-LABEL: define i8 @negate_sext(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = zext i1 [[Y]] to i8
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X]], [[T0_NEG]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sext i1 %y to i8
@@ -750,9 +803,10 @@ define i8 @negate_sext(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @negate_zext(i8 %x, i1 %y) {
-; CHECK-LABEL: @negate_zext(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sext i1 [[Y:%.*]] to i8
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X:%.*]], [[T0_NEG]]
+; CHECK-LABEL: define i8 @negate_zext(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = sext i1 [[Y]] to i8
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X]], [[T0_NEG]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = zext i1 %y to i8
@@ -760,10 +814,11 @@ define i8 @negate_zext(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @negate_sext_extrause(i8 %x, i1 %y) {
-; CHECK-LABEL: @negate_sext_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sext i1 [[Y:%.*]] to i8
+; CHECK-LABEL: define i8 @negate_sext_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sext i1 [[Y]] to i8
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sext i1 %y to i8
@@ -772,10 +827,11 @@ define i8 @negate_sext_extrause(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @negate_zext_extrause(i8 %x, i1 %y) {
-; CHECK-LABEL: @negate_zext_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = zext i1 [[Y:%.*]] to i8
+; CHECK-LABEL: define i8 @negate_zext_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = zext i1 [[Y]] to i8
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = zext i1 %y to i8
@@ -784,9 +840,10 @@ define i8 @negate_zext_extrause(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @negate_sext_wrongwidth(i8 %x, i2 %y) {
-; CHECK-LABEL: @negate_sext_wrongwidth(
-; CHECK-NEXT:    [[T0:%.*]] = sext i2 [[Y:%.*]] to i8
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_sext_wrongwidth(
+; CHECK-SAME: i8 [[X:%.*]], i2 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sext i2 [[Y]] to i8
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sext i2 %y to i8
@@ -794,9 +851,10 @@ define i8 @negate_sext_wrongwidth(i8 %x, i2 %y) {
   ret i8 %t1
 }
 define i8 @negate_zext_wrongwidth(i8 %x, i2 %y) {
-; CHECK-LABEL: @negate_zext_wrongwidth(
-; CHECK-NEXT:    [[T0:%.*]] = zext i2 [[Y:%.*]] to i8
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_zext_wrongwidth(
+; CHECK-SAME: i8 [[X:%.*]], i2 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = zext i2 [[Y]] to i8
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = zext i2 %y to i8
@@ -805,10 +863,11 @@ define i8 @negate_zext_wrongwidth(i8 %x, i2 %y) {
 }
 
 define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y) {
-; CHECK-LABEL: @negate_shufflevector_oneinput_reverse(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_shufflevector_oneinput_reverse(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X]]
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i4> [[T2]]
 ;
   %t0 = shl <2 x i4> <i4 -6, i4 5>, %x
@@ -817,10 +876,11 @@ define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y)
   ret <2 x i4> %t2
 }
 define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(<2 x i4> %x, <2 x i4> %y) {
-; CHECK-LABEL: @negate_shufflevector_oneinput_second_lane_is_undef(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X]]
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> poison, <2 x i32> <i32 0, i32 poison>
-; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i4> [[T2]]
 ;
   %t0 = shl <2 x i4> <i4 -6, i4 5>, %x
@@ -829,11 +889,12 @@ define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(<2 x i4> %x,
   ret <2 x i4> %t2
 }
 define <2 x i4> @negate_shufflevector_twoinputs(<2 x i4> %x, <2 x i4> %y, <2 x i4> %z) {
-; CHECK-LABEL: @negate_shufflevector_twoinputs(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = add <2 x i4> [[Y:%.*]], <i4 poison, i4 1>
+; CHECK-LABEL: define <2 x i4> @negate_shufflevector_twoinputs(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]], <2 x i4> [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = add <2 x i4> [[Y]], <i4 poison, i4 1>
 ; CHECK-NEXT:    [[T2_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> [[T1_NEG]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[T3:%.*]] = add <2 x i4> [[T2_NEG]], [[Z:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = add <2 x i4> [[T2_NEG]], [[Z]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t0 = shl <2 x i4> <i4 -6, i4 5>, %x
@@ -843,11 +904,12 @@ define <2 x i4> @negate_shufflevector_twoinputs(<2 x i4> %x, <2 x i4> %y, <2 x i
   ret <2 x i4> %t3
 }
 define <2 x i4> @negate_shufflevector_oneinput_extrause(<2 x i4> %x, <2 x i4> %y) {
-; CHECK-LABEL: @negate_shufflevector_oneinput_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i4> <i4 -6, i4 5>, [[X:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_shufflevector_oneinput_extrause(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i4> <i4 -6, i4 5>, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x i4> [[T0]], <2 x i4> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub <2 x i4> [[Y:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub <2 x i4> [[Y]], [[T1]]
 ; CHECK-NEXT:    ret <2 x i4> [[T2]]
 ;
   %t0 = shl <2 x i4> <i4 -6, i4 5>, %x
@@ -860,15 +922,16 @@ define <2 x i4> @negate_shufflevector_oneinput_extrause(<2 x i4> %x, <2 x i4> %y
 ; zext of non-negative can be negated
 ; sext of non-positive can be negated
 define i16 @negation_of_zeroext_of_nonnegative(i8 %x) {
-; CHECK-LABEL: @negation_of_zeroext_of_nonnegative(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_zeroext_of_nonnegative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i8 [[T0]], -1
-; CHECK-NEXT:    br i1 [[T1]], label [[NONNEG_BB:%.*]], label [[NEG_BB:%.*]]
-; CHECK:       nonneg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NONNEG_BB:.*]], label %[[NEG_BB:.*]]
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = zext nneg i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       neg_bb:
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -884,15 +947,16 @@ neg_bb:
   ret i16 0
 }
 define i16 @negation_of_zeroext_of_positive(i8 %x) {
-; CHECK-LABEL: @negation_of_zeroext_of_positive(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_zeroext_of_positive(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i8 [[T0]], 0
-; CHECK-NEXT:    br i1 [[T1]], label [[NONNEG_BB:%.*]], label [[NEG_BB:%.*]]
-; CHECK:       nonneg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NONNEG_BB:.*]], label %[[NEG_BB:.*]]
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = zext nneg i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       neg_bb:
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -908,15 +972,16 @@ neg_bb:
   ret i16 0
 }
 define i16 @negation_of_signext_of_negative(i8 %x) {
-; CHECK-LABEL: @negation_of_signext_of_negative(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_signext_of_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp slt i8 [[T0]], 0
-; CHECK-NEXT:    br i1 [[T1]], label [[NEG_BB:%.*]], label [[NONNEG_BB:%.*]]
-; CHECK:       neg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NEG_BB:.*]], label %[[NONNEG_BB:.*]]
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = sext i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       nonneg_bb:
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -932,15 +997,16 @@ nonneg_bb:
   ret i16 0
 }
 define i16 @negation_of_signext_of_nonpositive(i8 %x) {
-; CHECK-LABEL: @negation_of_signext_of_nonpositive(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_signext_of_nonpositive(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp slt i8 [[T0]], 1
-; CHECK-NEXT:    br i1 [[T1]], label [[NEG_BB:%.*]], label [[NONNEG_BB:%.*]]
-; CHECK:       neg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NEG_BB:.*]], label %[[NONNEG_BB:.*]]
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = sext i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       nonneg_bb:
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -956,15 +1022,16 @@ nonneg_bb:
   ret i16 0
 }
 define i16 @negation_of_signext_of_nonnegative__wrong_cast(i8 %x) {
-; CHECK-LABEL: @negation_of_signext_of_nonnegative__wrong_cast(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_signext_of_nonnegative__wrong_cast(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i8 [[T0]], -1
-; CHECK-NEXT:    br i1 [[T1]], label [[NONNEG_BB:%.*]], label [[NEG_BB:%.*]]
-; CHECK:       nonneg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NONNEG_BB:.*]], label %[[NEG_BB:.*]]
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = zext nneg i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       neg_bb:
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -980,15 +1047,16 @@ neg_bb:
   ret i16 0
 }
 define i16 @negation_of_zeroext_of_negative_wrongcast(i8 %x) {
-; CHECK-LABEL: @negation_of_zeroext_of_negative_wrongcast(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_zeroext_of_negative_wrongcast(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp slt i8 [[T0]], 0
-; CHECK-NEXT:    br i1 [[T1]], label [[NEG_BB:%.*]], label [[NONNEG_BB:%.*]]
-; CHECK:       neg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NEG_BB:.*]], label %[[NONNEG_BB:.*]]
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = zext i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       nonneg_bb:
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -1006,10 +1074,11 @@ nonneg_bb:
 
 ; 'or' of 1 and operand with no lowest bit set is 'inc'
 define i8 @negation_of_increment_via_or_with_no_common_bits_set(i8 %x, i8 %y) {
-; CHECK-LABEL: @negation_of_increment_via_or_with_no_common_bits_set(
-; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y:%.*]], 1
+; CHECK-LABEL: define i8 @negation_of_increment_via_or_with_no_common_bits_set(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y]], 1
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = xor i8 [[T0]], -1
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X:%.*]], [[T1_NEG]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X]], [[T1_NEG]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 %y, 1
@@ -1018,11 +1087,12 @@ define i8 @negation_of_increment_via_or_with_no_common_bits_set(i8 %x, i8 %y) {
   ret i8 %t2
 }
 define i8 @negation_of_increment_via_or_with_no_common_bits_set_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negation_of_increment_via_or_with_no_common_bits_set_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y:%.*]], 1
+; CHECK-LABEL: define i8 @negation_of_increment_via_or_with_no_common_bits_set_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y]], 1
 ; CHECK-NEXT:    [[T1:%.*]] = or disjoint i8 [[T0]], 1
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 %y, 1
@@ -1032,10 +1102,11 @@ define i8 @negation_of_increment_via_or_with_no_common_bits_set_extrause(i8 %x,
   ret i8 %t2
 }
 define i8 @negation_of_increment_via_or_common_bits_set(i8 %x, i8 %y) {
-; CHECK-LABEL: @negation_of_increment_via_or_common_bits_set(
-; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y:%.*]], 1
+; CHECK-LABEL: define i8 @negation_of_increment_via_or_common_bits_set(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y]], 1
 ; CHECK-NEXT:    [[T1:%.*]] = or i8 [[T0]], 3
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 %y, 1
@@ -1046,12 +1117,13 @@ define i8 @negation_of_increment_via_or_common_bits_set(i8 %x, i8 %y) {
 
 ; 'or' of operands with no common bits set is 'add'
 define i8 @add_via_or_with_no_common_bits_set(i8 %x, i8 %y) {
-; CHECK-LABEL: @add_via_or_with_no_common_bits_set(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @add_via_or_with_no_common_bits_set(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = shl i8 [[Y]], 2
 ; CHECK-NEXT:    [[T2_NEG:%.*]] = add i8 [[T1_NEG]], -3
-; CHECK-NEXT:    [[T3:%.*]] = add i8 [[T2_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[T2_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -1062,12 +1134,13 @@ define i8 @add_via_or_with_no_common_bits_set(i8 %x, i8 %y) {
   ret i8 %t3
 }
 define i8 @add_via_or_with_common_bit_maybe_set(i8 %x, i8 %y) {
-; CHECK-LABEL: @add_via_or_with_common_bit_maybe_set(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @add_via_or_with_common_bit_maybe_set(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 2
 ; CHECK-NEXT:    [[T2:%.*]] = or i8 [[T1]], 4
-; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[X]], [[T2]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -1078,13 +1151,14 @@ define i8 @add_via_or_with_common_bit_maybe_set(i8 %x, i8 %y) {
   ret i8 %t3
 }
 define i8 @add_via_or_with_no_common_bits_set_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @add_via_or_with_no_common_bits_set_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @add_via_or_with_no_common_bits_set_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 2
 ; CHECK-NEXT:    [[T2:%.*]] = or disjoint i8 [[T1]], 3
 ; CHECK-NEXT:    call void @use8(i8 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[X]], [[T2]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -1098,11 +1172,12 @@ define i8 @add_via_or_with_no_common_bits_set_extrause(i8 %x, i8 %y) {
 
 ; `extractelement` is negatible if source operand is negatible.
 define i4 @negate_extractelement(<2 x i4> %x, i32 %y, i4 %z) {
-; CHECK-LABEL: @negate_extractelement(
-; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[X:%.*]]
+; CHECK-LABEL: define i4 @negate_extractelement(
+; CHECK-SAME: <2 x i4> [[X:%.*]], i32 [[Y:%.*]], i4 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[X]]
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = extractelement <2 x i4> [[X]], i32 [[Y:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i4 [[T1_NEG]], [[Z:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = extractelement <2 x i4> [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = add i4 [[T1_NEG]], [[Z]]
 ; CHECK-NEXT:    ret i4 [[T2]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %x
@@ -1112,12 +1187,13 @@ define i4 @negate_extractelement(<2 x i4> %x, i32 %y, i4 %z) {
   ret i4 %t2
 }
 define i4 @negate_extractelement_extrause(<2 x i4> %x, i32 %y, i4 %z) {
-; CHECK-LABEL: @negate_extractelement_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[X:%.*]]
+; CHECK-LABEL: define i4 @negate_extractelement_extrause(
+; CHECK-SAME: <2 x i4> [[X:%.*]], i32 [[Y:%.*]], i4 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[X]]
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = extractelement <2 x i4> [[T0]], i32 [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = extractelement <2 x i4> [[T0]], i32 [[Y]]
 ; CHECK-NEXT:    call void @use4(i4 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i4 [[Z:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i4 [[Z]], [[T1]]
 ; CHECK-NEXT:    ret i4 [[T2]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %x
@@ -1130,9 +1206,10 @@ define i4 @negate_extractelement_extrause(<2 x i4> %x, i32 %y, i4 %z) {
 
 ; `insertelement` is negatible if both source vector and element-to-be-inserted are negatible.
 define <2 x i4> @negate_insertelement(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b) {
-; CHECK-LABEL: @negate_insertelement(
-; CHECK-NEXT:    [[T2_NEG:%.*]] = insertelement <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = add <2 x i4> [[T2_NEG]], [[B:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_insertelement(
+; CHECK-SAME: <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[T2_NEG:%.*]] = insertelement <2 x i4> [[SRC]], i4 [[A]], i32 [[X]]
+; CHECK-NEXT:    [[T3:%.*]] = add <2 x i4> [[T2_NEG]], [[B]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %src
@@ -1142,12 +1219,13 @@ define <2 x i4> @negate_insertelement(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b)
   ret <2 x i4> %t3
 }
 define <2 x i4> @negate_insertelement_extrause(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b) {
-; CHECK-LABEL: @negate_insertelement_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[SRC:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = sub i4 0, [[A:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[T0]], i4 [[T1]], i32 [[X:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_insertelement_extrause(
+; CHECK-SAME: <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[SRC]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i4 0, [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[T0]], i4 [[T1]], i32 [[X]]
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B]], [[T2]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %src
@@ -1158,10 +1236,11 @@ define <2 x i4> @negate_insertelement_extrause(<2 x i4> %src, i4 %a, i32 %x, <2
   ret <2 x i4> %t3
 }
 define <2 x i4> @negate_insertelement_nonnegatible_base(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b) {
-; CHECK-LABEL: @negate_insertelement_nonnegatible_base(
-; CHECK-NEXT:    [[T1:%.*]] = sub i4 0, [[A:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[SRC:%.*]], i4 [[T1]], i32 [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B:%.*]], [[T2]]
+; CHECK-LABEL: define <2 x i4> @negate_insertelement_nonnegatible_base(
+; CHECK-SAME: <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[T1:%.*]] = sub i4 0, [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[SRC]], i4 [[T1]], i32 [[X]]
+; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B]], [[T2]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t1 = sub i4 zeroinitializer, %a
@@ -1170,10 +1249,11 @@ define <2 x i4> @negate_insertelement_nonnegatible_base(<2 x i4> %src, i4 %a, i3
   ret <2 x i4> %t3
 }
 define <2 x i4> @negate_insertelement_nonnegatible_insert(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b) {
-; CHECK-LABEL: @negate_insertelement_nonnegatible_insert(
-; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[SRC:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[T0]], i4 [[A:%.*]], i32 [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B:%.*]], [[T2]]
+; CHECK-LABEL: define <2 x i4> @negate_insertelement_nonnegatible_insert(
+; CHECK-SAME: <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[SRC]]
+; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[T0]], i4 [[A]], i32 [[X]]
+; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B]], [[T2]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %src
@@ -1184,11 +1264,12 @@ define <2 x i4> @negate_insertelement_nonnegatible_insert(<2 x i4> %src, i4 %a,
 
 ; left-shift by constant can always be negated
 define i8 @negate_left_shift_by_constant_prefer_keeping_shl(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @negate_left_shift_by_constant_prefer_keeping_shl(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @negate_left_shift_by_constant_prefer_keeping_shl(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = shl i8 [[Z]], 4
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -1198,12 +1279,13 @@ define i8 @negate_left_shift_by_constant_prefer_keeping_shl(i8 %x, i8 %y, i8 %z)
   ret i8 %t2
 }
 define i8 @negate_left_shift_by_constant_prefer_keeping_shl_extrause(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @negate_left_shift_by_constant_prefer_keeping_shl_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @negate_left_shift_by_constant_prefer_keeping_shl_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 4
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -1214,11 +1296,12 @@ define i8 @negate_left_shift_by_constant_prefer_keeping_shl_extrause(i8 %x, i8 %
   ret i8 %t2
 }
 define i8 @negate_left_shift_by_constant(i8 %x, i8 %y, i8 %z, i8 %k) {
-; CHECK-LABEL: @negate_left_shift_by_constant(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[K:%.*]], [[Z:%.*]]
+; CHECK-LABEL: define i8 @negate_left_shift_by_constant(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]], i8 [[K:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[K]], [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 4
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 %k, %z
@@ -1228,12 +1311,13 @@ define i8 @negate_left_shift_by_constant(i8 %x, i8 %y, i8 %z, i8 %k) {
   ret i8 %t2
 }
 define i8 @negate_left_shift_by_constant_extrause(i8 %x, i8 %y, i8 %z, i8 %k) {
-; CHECK-LABEL: @negate_left_shift_by_constant_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[K:%.*]], [[Z:%.*]]
+; CHECK-LABEL: define i8 @negate_left_shift_by_constant_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]], i8 [[K:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[K]], [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 4
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 %k, %z
@@ -1246,8 +1330,9 @@ define i8 @negate_left_shift_by_constant_extrause(i8 %x, i8 %y, i8 %z, i8 %k) {
 
 ; `add` with single negatible operand is still negatible
 define i8 @negate_add_with_single_negatible_operand(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_add_with_single_negatible_operand(
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 -42, [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_add_with_single_negatible_operand(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 -42, [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = add i8 %x, 42
@@ -1256,9 +1341,10 @@ define i8 @negate_add_with_single_negatible_operand(i8 %x, i8 %y) {
 }
 ; do so even if we are two levels deep
 define i8 @negate_add_with_single_negatible_operand_depth2(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_add_with_single_negatible_operand_depth2(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i8 -21, [[X:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = mul i8 [[T0_NEG]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @negate_add_with_single_negatible_operand_depth2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i8 -21, [[X]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = mul i8 [[T0_NEG]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T1_NEG]]
 ;
   %t0 = add i8 %x, 21
@@ -1268,8 +1354,9 @@ define i8 @negate_add_with_single_negatible_operand_depth2(i8 %x, i8 %y) {
 }
 
 define i8 @negate_add_with_single_negatible_operand_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_add_with_single_negatible_operand_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @negate_add_with_single_negatible_operand_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = sub i8 -42, [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
@@ -1281,9 +1368,10 @@ define i8 @negate_add_with_single_negatible_operand_extrause(i8 %x, i8 %y) {
 }
 ; But don't do this if that means just sinking the negation.
 define i8 @negate_add_with_single_negatible_operand_non_negation(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_add_with_single_negatible_operand_non_negation(
-; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], 42
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[Y:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_add_with_single_negatible_operand_non_negation(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X]], 42
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[Y]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = add i8 %x, 42
@@ -1293,11 +1381,12 @@ define i8 @negate_add_with_single_negatible_operand_non_negation(i8 %x, i8 %y) {
 
 ; abs/nabs can be negated
 define i8 @negate_abs(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_abs(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_abs(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T2:%.*]] = call i8 @llvm.abs.i8(i8 [[X]], i1 false)
-; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[Y:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[Y]], [[T2]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %x
@@ -1308,11 +1397,12 @@ define i8 @negate_abs(i8 %x, i8 %y) {
   ret i8 %t3
 }
 define i8 @negate_nabs(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_nabs(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_nabs(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.abs.i8(i8 [[X]], i1 false)
-; CHECK-NEXT:    [[T3:%.*]] = add i8 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[Y]], [[TMP1]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %x
@@ -1326,11 +1416,12 @@ define i8 @negate_nabs(i8 %x, i8 %y) {
 ; And in general, if hands of select are known to be negation of each other,
 ; we can negate the select
 define i8 @negate_select_of_op_vs_negated_op(i8 %x, i8 %y, i1 %c) {
-; CHECK-LABEL: @negate_select_of_op_vs_negated_op(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_select_of_op_vs_negated_op(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C:%.*]], i8 [[X]], i8 [[T0]], !prof [[PROF0:![0-9]+]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C]], i8 [[X]], i8 [[T0]], !prof [[PROF0:![0-9]+]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %x
@@ -1340,9 +1431,10 @@ define i8 @negate_select_of_op_vs_negated_op(i8 %x, i8 %y, i1 %c) {
   ret i8 %t2
 }
 define i8 @dont_negate_ordinary_select(i8 %x, i8 %y, i8 %z, i1 %c) {
-; CHECK-LABEL: @dont_negate_ordinary_select(
-; CHECK-NEXT:    [[T0:%.*]] = select i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[Z:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @dont_negate_ordinary_select(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[C]], i8 [[X]], i8 [[Y]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[Z]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = select i1 %c, i8 %x, i8 %y
@@ -1352,10 +1444,12 @@ define i8 @dont_negate_ordinary_select(i8 %x, i8 %y, i8 %z, i1 %c) {
 
 ; Freeze is transparent as far as negation is concerned
 define i4 @negate_freeze(i4 %x, i4 %y, i4 %z) {
-; CHECK-LABEL: @negate_freeze(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i4 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = freeze i4 [[T0_NEG]]
-; CHECK-NEXT:    [[T2:%.*]] = add i4 [[T1_NEG]], [[Z:%.*]]
+; CHECK-LABEL: define i4 @negate_freeze(
+; CHECK-SAME: i4 [[X:%.*]], i4 [[Y:%.*]], i4 [[Z:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i4 [[X]]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i4 [[Y]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = sub i4 [[Y_FR]], [[X_FR]]
+; CHECK-NEXT:    [[T2:%.*]] = add i4 [[T1_NEG]], [[Z]]
 ; CHECK-NEXT:    ret i4 [[T2]]
 ;
   %t0 = sub i4 %x, %y
@@ -1364,11 +1458,13 @@ define i4 @negate_freeze(i4 %x, i4 %y, i4 %z) {
   ret i4 %t2
 }
 define i4 @negate_freeze_extrause(i4 %x, i4 %y, i4 %z) {
-; CHECK-LABEL: @negate_freeze_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub i4 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = freeze i4 [[T0]]
+; CHECK-LABEL: define i4 @negate_freeze_extrause(
+; CHECK-SAME: i4 [[X:%.*]], i4 [[Y:%.*]], i4 [[Z:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i4 [[X]]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i4 [[Y]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i4 [[X_FR]], [[Y_FR]]
 ; CHECK-NEXT:    call void @use4(i4 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i4 [[Z:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i4 [[Z]], [[T1]]
 ; CHECK-NEXT:    ret i4 [[T2]]
 ;
   %t0 = sub i4 %x, %y
@@ -1384,11 +1480,11 @@ define i4 @negate_freeze_extrause(i4 %x, i4 %y, i4 %z) {
 ; constants to RHS ourselves, since that is helpful sometimes.
 ; This used to cause an endless combine loop.
 define void @noncanonical_mul_with_constant_as_first_operand() {
-; CHECK-LABEL: @noncanonical_mul_with_constant_as_first_operand(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[IF_END:%.*]]
-; CHECK:       if.end:
-; CHECK-NEXT:    br label [[IF_END]]
+; CHECK-LABEL: define void @noncanonical_mul_with_constant_as_first_operand() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[IF_END:.*]]
+; CHECK:       [[IF_END]]:
+; CHECK-NEXT:    br label %[[IF_END]]
 ;
 entry:
   br label %if.end
@@ -1402,5 +1498,7 @@ if.end:
   br label %if.end
 }
 
-; CHECK: !0 = !{!"branch_weights", i32 40, i32 1}
 !0 = !{!"branch_weights", i32 40, i32 1}
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 40, i32 1}
+;.
diff --git a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
index 871cf37976d89..aad006de0e361 100644
--- a/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
+++ b/llvm/test/Transforms/InstCombine/sub-of-negatible.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 declare void @use4(i4)
@@ -8,8 +8,9 @@ declare i1 @use32gen1(i32)
 
 ; Constant can be freely negated.
 define i8 @t0(i8 %x) {
-; CHECK-LABEL: @t0(
-; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @t0(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    ret i8 [[T0]]
 ;
   %t0 = sub i8 %x, -42
@@ -18,10 +19,11 @@ define i8 @t0(i8 %x) {
 
 ; Negation can be negated for free
 define i8 @t1(i8 %x, i8 %y) {
-; CHECK-LABEL: @t1(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @t1(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X:%.*]], [[Y]]
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sub i8 0, %y
@@ -32,9 +34,10 @@ define i8 @t1(i8 %x, i8 %y) {
 
 ; Shift-left can be negated if all uses can be updated
 define i8 @t2(i8 %x, i8 %y) {
-; CHECK-LABEL: @t2(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl i8 42, [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @t2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl i8 42, [[Y]]
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = shl i8 -42, %y
@@ -42,10 +45,11 @@ define i8 @t2(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @n2(i8 %x, i8 %y) {
-; CHECK-LABEL: @n2(
-; CHECK-NEXT:    [[T0:%.*]] = shl i8 -42, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 -42, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = shl i8 -42, %y
@@ -54,11 +58,12 @@ define i8 @n2(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @t3(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @t3(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @t3(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = shl i8 [[Z]], [[Y:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = shl i8 [[Z]], [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -68,12 +73,13 @@ define i8 @t3(i8 %x, i8 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @n3(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n3(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @n3(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -86,9 +92,10 @@ define i8 @n3(i8 %x, i8 %y, i8 %z) {
 
 ; Select can be negated if all it's operands can be negated and all the users of select can be updated
 define i8 @t4(i8 %x, i1 %y) {
-; CHECK-LABEL: @t4(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = select i1 [[Y:%.*]], i8 42, i8 -44
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @t4(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = select i1 [[Y]], i8 42, i8 -44
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = select i1 %y, i8 -42, i8 44
@@ -97,8 +104,9 @@ define i8 @t4(i8 %x, i1 %y) {
 }
 
 define i8 @select_of_constants_multi_use(i1 %b) {
-; CHECK-LABEL: @select_of_constants_multi_use(
-; CHECK-NEXT:    [[S_NEG:%.*]] = select i1 [[B:%.*]], i8 -42, i8 2
+; CHECK-LABEL: define i8 @select_of_constants_multi_use(
+; CHECK-SAME: i1 [[B:%.*]]) {
+; CHECK-NEXT:    [[S_NEG:%.*]] = select i1 [[B]], i8 -42, i8 2
 ; CHECK-NEXT:    [[S:%.*]] = select i1 [[B]], i8 42, i8 -2
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
 ; CHECK-NEXT:    ret i8 [[S_NEG]]
@@ -110,7 +118,8 @@ define i8 @select_of_constants_multi_use(i1 %b) {
 }
 
 define i32 @PR52261(i1 %b) {
-; CHECK-LABEL: @PR52261(
+; CHECK-LABEL: define i32 @PR52261(
+; CHECK-SAME: i1 [[B:%.*]]) {
 ; CHECK-NEXT:    ret i32 2
 ;
   %s = select i1 %b, i32 2, i32 -2
@@ -120,10 +129,11 @@ define i32 @PR52261(i1 %b) {
 }
 
 define i8 @n4(i8 %x, i1 %y) {
-; CHECK-LABEL: @n4(
-; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y:%.*]], i8 -42, i8 44
+; CHECK-LABEL: define i8 @n4(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y]], i8 -42, i8 44
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = select i1 %y, i8 -42, i8 44
@@ -132,9 +142,10 @@ define i8 @n4(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @n5(i8 %x, i1 %y, i8 %z) {
-; CHECK-LABEL: @n5(
-; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y:%.*]], i8 -42, i8 [[Z:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @n5(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[Y]], i8 -42, i8 [[Z]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = select i1 %y, i8 -42, i8 %z
@@ -142,11 +153,12 @@ define i8 @n5(i8 %x, i1 %y, i8 %z) {
   ret i8 %t1
 }
 define i8 @t6(i8 %x, i1 %y, i8 %z) {
-; CHECK-LABEL: @t6(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @t6(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = select i1 [[Y:%.*]], i8 42, i8 [[Z]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = select i1 [[Y]], i8 42, i8 [[Z]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -156,10 +168,11 @@ define i8 @t6(i8 %x, i1 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @t7(i8 %x, i1 %y, i8 %z) {
-; CHECK-LABEL: @t7(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl nsw i8 -1, [[Z:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = select i1 [[Y:%.*]], i8 0, i8 [[T0_NEG]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @t7(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl nsw i8 -1, [[Z]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = select i1 [[Y]], i8 0, i8 [[T0_NEG]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 1, %z
@@ -168,11 +181,12 @@ define i8 @t7(i8 %x, i1 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @n8(i8 %x, i1 %y, i8 %z) {
-; CHECK-LABEL: @n8(
-; CHECK-NEXT:    [[T0:%.*]] = shl nuw i8 1, [[Z:%.*]]
+; CHECK-LABEL: define i8 @n8(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl nuw i8 1, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = select i1 [[Y:%.*]], i8 0, i8 [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T1:%.*]] = select i1 [[Y]], i8 0, i8 [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 1, %z
@@ -185,8 +199,9 @@ define i8 @n8(i8 %x, i1 %y, i8 %z) {
 ; Subtraction can be negated by swapping its operands.
 ; x - (y - z) -> x - y + z -> x + (z - y)
 define i8 @t9(i8 %x, i8 %y) {
-; CHECK-LABEL: @t9(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @t9(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i8 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T0_NEG]]
 ;
   %t0 = sub i8 %y, %x
@@ -195,8 +210,9 @@ define i8 @t9(i8 %x, i8 %y) {
 }
 
 define i8 @n10(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n10(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-LABEL: define i8 @n10(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[Y]], [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
@@ -208,8 +224,9 @@ define i8 @n10(i8 %x, i8 %y, i8 %z) {
 }
 
 define i8 @neg_of_sub_from_constant(i8 %x) {
-; CHECK-LABEL: @neg_of_sub_from_constant(
-; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X:%.*]], -42
+; CHECK-LABEL: define i8 @neg_of_sub_from_constant(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X]], -42
 ; CHECK-NEXT:    ret i8 [[S_NEG]]
 ;
   %s = sub i8 42, %x
@@ -218,8 +235,9 @@ define i8 @neg_of_sub_from_constant(i8 %x) {
 }
 
 define i8 @neg_of_sub_from_constant_multi_use(i8 %x) {
-; CHECK-LABEL: @neg_of_sub_from_constant_multi_use(
-; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X:%.*]], -42
+; CHECK-LABEL: define i8 @neg_of_sub_from_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X]], -42
 ; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
 ; CHECK-NEXT:    ret i8 [[S_NEG]]
@@ -231,8 +249,9 @@ define i8 @neg_of_sub_from_constant_multi_use(i8 %x) {
 }
 
 define i8 @sub_from_constant_of_sub_from_constant(i8 %x) {
-; CHECK-LABEL: @sub_from_constant_of_sub_from_constant(
-; CHECK-NEXT:    [[R:%.*]] = add i8 [[X:%.*]], -31
+; CHECK-LABEL: define i8 @sub_from_constant_of_sub_from_constant(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[X]], -31
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = sub i8 42, %x
@@ -241,8 +260,9 @@ define i8 @sub_from_constant_of_sub_from_constant(i8 %x) {
 }
 
 define i8 @sub_from_constant_of_sub_from_constant_multi_use(i8 %x) {
-; CHECK-LABEL: @sub_from_constant_of_sub_from_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X:%.*]]
+; CHECK-LABEL: define i8 @sub_from_constant_of_sub_from_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
 ; CHECK-NEXT:    [[R:%.*]] = add i8 [[X]], -31
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -254,9 +274,10 @@ define i8 @sub_from_constant_of_sub_from_constant_multi_use(i8 %x) {
 }
 
 define i8 @sub_from_variable_of_sub_from_constant(i8 %x, i8 %y) {
-; CHECK-LABEL: @sub_from_variable_of_sub_from_constant(
-; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X:%.*]], -42
-; CHECK-NEXT:    [[R:%.*]] = add i8 [[S_NEG]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @sub_from_variable_of_sub_from_constant(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[S_NEG:%.*]] = add i8 [[X]], -42
+; CHECK-NEXT:    [[R:%.*]] = add i8 [[S_NEG]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = sub i8 42, %x
@@ -265,10 +286,11 @@ define i8 @sub_from_variable_of_sub_from_constant(i8 %x, i8 %y) {
 }
 
 define i8 @sub_from_variable_of_sub_from_constant_multi_use(i8 %x, i8 %y) {
-; CHECK-LABEL: @sub_from_variable_of_sub_from_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X:%.*]]
+; CHECK-LABEL: define i8 @sub_from_variable_of_sub_from_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
-; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y:%.*]], [[S]]
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y]], [[S]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = sub i8 42, %x
@@ -280,13 +302,14 @@ define i8 @sub_from_variable_of_sub_from_constant_multi_use(i8 %x, i8 %y) {
 ; Addition can be negated if both operands can be negated
 ; x - (y + z) -> x - y - z -> x + ((-y) + (-z)))
 define i8 @t12(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @t12(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @t12(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y]], [[Z]]
-; CHECK-NEXT:    [[T3:%.*]] = add i8 [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -298,11 +321,12 @@ define i8 @t12(i8 %x, i8 %y, i8 %z) {
   ret i8 %t3
 }
 define i8 @n13(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n13(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n13(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = sub i8 [[Y]], [[Z:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = sub i8 [[Y]], [[Z]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %y
@@ -312,15 +336,16 @@ define i8 @n13(i8 %x, i8 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @n14(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n14(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n14(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[Y]], [[Z]]
 ; CHECK-NEXT:    [[T2:%.*]] = sub i8 0, [[TMP1]]
 ; CHECK-NEXT:    call void @use8(i8 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = add i8 [[X:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[X]], [[TMP1]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -334,8 +359,9 @@ define i8 @n14(i8 %x, i8 %y, i8 %z) {
 }
 
 define i8 @neg_of_add_with_constant(i8 %x) {
-; CHECK-LABEL: @neg_of_add_with_constant(
-; CHECK-NEXT:    [[R:%.*]] = sub i8 -42, [[X:%.*]]
+; CHECK-LABEL: define i8 @neg_of_add_with_constant(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = sub i8 -42, [[X]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = add i8 %x, 42
@@ -344,8 +370,9 @@ define i8 @neg_of_add_with_constant(i8 %x) {
 }
 
 define i8 @neg_of_add_with_constant_multi_use(i8 %x) {
-; CHECK-LABEL: @neg_of_add_with_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @neg_of_add_with_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
 ; CHECK-NEXT:    [[R:%.*]] = sub i8 -42, [[X]]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -357,8 +384,9 @@ define i8 @neg_of_add_with_constant_multi_use(i8 %x) {
 }
 
 define i8 @sub_from_constant_of_add_with_constant(i8 %x) {
-; CHECK-LABEL: @sub_from_constant_of_add_with_constant(
-; CHECK-NEXT:    [[R:%.*]] = sub i8 -31, [[X:%.*]]
+; CHECK-LABEL: define i8 @sub_from_constant_of_add_with_constant(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[R:%.*]] = sub i8 -31, [[X]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = add i8 %x, 42
@@ -367,8 +395,9 @@ define i8 @sub_from_constant_of_add_with_constant(i8 %x) {
 }
 
 define i8 @sub_from_constant_of_add_with_constant_multi_use(i8 %x) {
-; CHECK-LABEL: @sub_from_constant_of_add_with_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @sub_from_constant_of_add_with_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
 ; CHECK-NEXT:    [[R:%.*]] = sub i8 -31, [[X]]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -380,9 +409,10 @@ define i8 @sub_from_constant_of_add_with_constant_multi_use(i8 %x) {
 }
 
 define i8 @sub_from_variable_of_add_with_constant(i8 %x, i8 %y) {
-; CHECK-LABEL: @sub_from_variable_of_add_with_constant(
-; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], 42
-; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y:%.*]], [[S]]
+; CHECK-LABEL: define i8 @sub_from_variable_of_add_with_constant(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X]], 42
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y]], [[S]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = add i8 %x, 42
@@ -391,10 +421,11 @@ define i8 @sub_from_variable_of_add_with_constant(i8 %x, i8 %y) {
 }
 
 define i8 @sub_from_variable_of_add_with_constant_multi_use(i8 %x, i8 %y) {
-; CHECK-LABEL: @sub_from_variable_of_add_with_constant_multi_use(
-; CHECK-NEXT:    [[S:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @sub_from_variable_of_add_with_constant_multi_use(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[S]])
-; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y:%.*]], [[S]]
+; CHECK-NEXT:    [[R:%.*]] = sub i8 [[Y]], [[S]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %s = add i8 %x, 42
@@ -406,11 +437,12 @@ define i8 @sub_from_variable_of_add_with_constant_multi_use(i8 %x, i8 %y) {
 ; Multiplication can be negated if either one of operands can be negated
 ; x - (y * z) -> x + ((-y) * z) or  x + ((-z) * y)
 define i8 @t15(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @t15(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @t15(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = mul i8 [[Y]], [[Z:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = mul i8 [[Y]], [[Z]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %y
@@ -420,12 +452,13 @@ define i8 @t15(i8 %x, i8 %y, i8 %z) {
   ret i8 %t2
 }
 define i8 @n16(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @n16(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n16(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = mul i8 [[Z:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = mul i8 [[Z]], [[T0]]
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %y
@@ -438,15 +471,16 @@ define i8 @n16(i8 %x, i8 %y, i8 %z) {
 
 ; Phi can be negated if all incoming values can be negated
 define i8 @t16(i1 %c, i8 %x) {
-; CHECK-LABEL: @t16(
-; CHECK-NEXT:  begin:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       else:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[Z_NEG:%.*]] = phi i8 [ [[X:%.*]], [[THEN]] ], [ 42, [[ELSE]] ]
+; CHECK-LABEL: define i8 @t16(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]]) {
+; CHECK-NEXT:  [[BEGIN:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    br label %[[END:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    [[Z_NEG:%.*]] = phi i8 [ [[X]], %[[THEN]] ], [ 42, %[[ELSE]] ]
 ; CHECK-NEXT:    ret i8 [[Z_NEG]]
 ;
 begin:
@@ -462,16 +496,17 @@ end:
   ret i8 %n
 }
 define i8 @n17(i1 %c, i8 %x) {
-; CHECK-LABEL: @n17(
-; CHECK-NEXT:  begin:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    [[Y:%.*]] = sub i8 0, [[X:%.*]]
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       else:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[Z:%.*]] = phi i8 [ [[Y]], [[THEN]] ], [ -42, [[ELSE]] ]
+; CHECK-LABEL: define i8 @n17(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]]) {
+; CHECK-NEXT:  [[BEGIN:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[Y:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    br label %[[END:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    [[Z:%.*]] = phi i8 [ [[Y]], %[[THEN]] ], [ -42, %[[ELSE]] ]
 ; CHECK-NEXT:    call void @use8(i8 [[Z]])
 ; CHECK-NEXT:    [[N:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    ret i8 [[N]]
@@ -490,16 +525,17 @@ end:
   ret i8 %n
 }
 define i8 @n19(i1 %c, i8 %x, i8 %y) {
-; CHECK-LABEL: @n19(
-; CHECK-NEXT:  begin:
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    [[Z:%.*]] = sub i8 0, [[X:%.*]]
-; CHECK-NEXT:    br label [[END:%.*]]
-; CHECK:       else:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[R:%.*]] = phi i8 [ [[Z]], [[THEN]] ], [ [[Y:%.*]], [[ELSE]] ]
+; CHECK-LABEL: define i8 @n19(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:  [[BEGIN:.*:]]
+; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[Z:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    br label %[[END:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    [[R:%.*]] = phi i8 [ [[Z]], %[[THEN]] ], [ [[Y]], %[[ELSE]] ]
 ; CHECK-NEXT:    [[N:%.*]] = sub i8 0, [[R]]
 ; CHECK-NEXT:    ret i8 [[N]]
 ;
@@ -516,21 +552,22 @@ end:
   ret i8 %n
 }
 define void @phi_with_duplicate_incoming_basic_blocks(i32 %x, i32 %y, i1 %should_lookup, i32 %z) {
-; CHECK-LABEL: @phi_with_duplicate_incoming_basic_blocks(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X_INC_NEG:%.*]] = xor i32 [[X:%.*]], -1
-; CHECK-NEXT:    br i1 [[SHOULD_LOOKUP:%.*]], label [[LOOKUP:%.*]], label [[LOOP:%.*]]
-; CHECK:       lookup:
-; CHECK-NEXT:    [[TO_LOOKUP:%.*]] = phi i32 [ [[Y:%.*]], [[ENTRY:%.*]] ], [ [[METAVAL_NEG:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    switch i32 [[TO_LOOKUP]], label [[END:%.*]] [
-; CHECK-NEXT:      i32 0, label [[LOOP]]
-; CHECK-NEXT:      i32 42, label [[LOOP]]
+; CHECK-LABEL: define void @phi_with_duplicate_incoming_basic_blocks(
+; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], i1 [[SHOULD_LOOKUP:%.*]], i32 [[Z:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[X_INC_NEG:%.*]] = xor i32 [[X]], -1
+; CHECK-NEXT:    br i1 [[SHOULD_LOOKUP]], label %[[LOOKUP:.*]], label %[[LOOP:.*]]
+; CHECK:       [[LOOKUP]]:
+; CHECK-NEXT:    [[TO_LOOKUP:%.*]] = phi i32 [ [[Y]], %[[ENTRY]] ], [ [[METAVAL_NEG:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    switch i32 [[TO_LOOKUP]], label %[[END:.*]] [
+; CHECK-NEXT:      i32 0, label %[[LOOP]]
+; CHECK-NEXT:      i32 42, label %[[LOOP]]
 ; CHECK-NEXT:    ]
-; CHECK:       loop:
-; CHECK-NEXT:    [[METAVAL_NEG]] = phi i32 [ [[X_INC_NEG]], [[LOOKUP]] ], [ [[X_INC_NEG]], [[LOOKUP]] ], [ -84, [[ENTRY]] ]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[METAVAL_NEG]] = phi i32 [ [[X_INC_NEG]], %[[LOOKUP]] ], [ [[X_INC_NEG]], %[[LOOKUP]] ], [ -84, %[[ENTRY]] ]
 ; CHECK-NEXT:    [[REPEAT:%.*]] = call i1 @use32gen1(i32 [[METAVAL_NEG]])
-; CHECK-NEXT:    br i1 [[REPEAT]], label [[LOOKUP]], label [[END]]
-; CHECK:       end:
+; CHECK-NEXT:    br i1 [[REPEAT]], label %[[LOOKUP]], label %[[END]]
+; CHECK:       [[END]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -556,10 +593,11 @@ end:
 
 ; truncation can be negated if it's operand can be negated
 define i8 @t20(i8 %x, i16 %y) {
-; CHECK-LABEL: @t20(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl i16 42, [[Y:%.*]]
+; CHECK-LABEL: define i8 @t20(
+; CHECK-SAME: i8 [[X:%.*]], i16 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl i16 42, [[Y]]
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = trunc i16 [[T0_NEG]] to i8
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X:%.*]], [[T1_NEG]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X]], [[T1_NEG]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i16 -42, %y
@@ -568,11 +606,12 @@ define i8 @t20(i8 %x, i16 %y) {
   ret i8 %t2
 }
 define i8 @n21(i8 %x, i16 %y) {
-; CHECK-LABEL: @n21(
-; CHECK-NEXT:    [[T0:%.*]] = shl i16 -42, [[Y:%.*]]
+; CHECK-LABEL: define i8 @n21(
+; CHECK-SAME: i8 [[X:%.*]], i16 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i16 -42, [[Y]]
 ; CHECK-NEXT:    [[T1:%.*]] = trunc i16 [[T0]] to i8
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i16 -42, %y
@@ -583,8 +622,9 @@ define i8 @n21(i8 %x, i16 %y) {
 }
 
 define i4 @negate_xor(i4 %x) {
-; CHECK-LABEL: @negate_xor(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], -6
+; CHECK-LABEL: define i4 @negate_xor(
+; CHECK-SAME: i4 [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X]], -6
 ; CHECK-NEXT:    [[O_NEG:%.*]] = add i4 [[TMP1]], 1
 ; CHECK-NEXT:    ret i4 [[O_NEG]]
 ;
@@ -594,8 +634,9 @@ define i4 @negate_xor(i4 %x) {
 }
 
 define <2 x i4> @negate_xor_vec(<2 x i4> %x) {
-; CHECK-LABEL: @negate_xor_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i4> [[X:%.*]], <i4 -6, i4 5>
+; CHECK-LABEL: define <2 x i4> @negate_xor_vec(
+; CHECK-SAME: <2 x i4> [[X:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i4> [[X]], <i4 -6, i4 5>
 ; CHECK-NEXT:    [[O_NEG:%.*]] = add <2 x i4> [[TMP1]], splat (i4 1)
 ; CHECK-NEXT:    ret <2 x i4> [[O_NEG]]
 ;
@@ -605,8 +646,9 @@ define <2 x i4> @negate_xor_vec(<2 x i4> %x) {
 }
 
 define i8 @negate_xor_use(i8 %x) {
-; CHECK-LABEL: @negate_xor_use(
-; CHECK-NEXT:    [[O:%.*]] = xor i8 [[X:%.*]], 5
+; CHECK-LABEL: define i8 @negate_xor_use(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[O:%.*]] = xor i8 [[X]], 5
 ; CHECK-NEXT:    call void @use8(i8 [[O]])
 ; CHECK-NEXT:    [[R:%.*]] = sub i8 0, [[O]]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -618,10 +660,11 @@ define i8 @negate_xor_use(i8 %x) {
 }
 
 define i4 @negate_shl_xor(i4 %x, i4 %y) {
-; CHECK-LABEL: @negate_shl_xor(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X:%.*]], -6
+; CHECK-LABEL: define i4 @negate_shl_xor(
+; CHECK-SAME: i4 [[X:%.*]], i4 [[Y:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i4 [[X]], -6
 ; CHECK-NEXT:    [[O_NEG:%.*]] = add i4 [[TMP1]], 1
-; CHECK-NEXT:    [[S_NEG:%.*]] = shl i4 [[O_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[S_NEG:%.*]] = shl i4 [[O_NEG]], [[Y]]
 ; CHECK-NEXT:    ret i4 [[S_NEG]]
 ;
   %o = xor i4 %x, 5
@@ -631,11 +674,12 @@ define i4 @negate_shl_xor(i4 %x, i4 %y) {
 }
 
 define i8 @negate_shl_not_uses(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_shl_not_uses(
-; CHECK-NEXT:    [[O_NEG:%.*]] = add i8 [[X:%.*]], 1
+; CHECK-LABEL: define i8 @negate_shl_not_uses(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[O_NEG:%.*]] = add i8 [[X]], 1
 ; CHECK-NEXT:    [[O:%.*]] = xor i8 [[X]], -1
 ; CHECK-NEXT:    call void @use8(i8 [[O]])
-; CHECK-NEXT:    [[S_NEG:%.*]] = shl i8 [[O_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[S_NEG:%.*]] = shl i8 [[O_NEG]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[S_NEG]]
 ;
   %o = xor i8 %x, -1
@@ -646,11 +690,12 @@ define i8 @negate_shl_not_uses(i8 %x, i8 %y) {
 }
 
 define <2 x i4> @negate_mul_not_uses_vec(<2 x i4> %x, <2 x i4> %y) {
-; CHECK-LABEL: @negate_mul_not_uses_vec(
-; CHECK-NEXT:    [[O_NEG:%.*]] = add <2 x i4> [[X:%.*]], splat (i4 1)
+; CHECK-LABEL: define <2 x i4> @negate_mul_not_uses_vec(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]]) {
+; CHECK-NEXT:    [[O_NEG:%.*]] = add <2 x i4> [[X]], splat (i4 1)
 ; CHECK-NEXT:    [[O:%.*]] = xor <2 x i4> [[X]], splat (i4 -1)
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[O]])
-; CHECK-NEXT:    [[S_NEG:%.*]] = mul <2 x i4> [[O_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[S_NEG:%.*]] = mul <2 x i4> [[O_NEG]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i4> [[S_NEG]]
 ;
   %o = xor <2 x i4> %x, <i4 -1, i4 -1>
@@ -662,9 +707,10 @@ define <2 x i4> @negate_mul_not_uses_vec(<2 x i4> %x, <2 x i4> %y) {
 
 ; signed division can be negated if divisor can be negated and is not 1/-1
 define i8 @negate_sdiv(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_sdiv(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sdiv i8 [[Y:%.*]], -42
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_sdiv(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = sdiv i8 [[Y]], -42
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sdiv i8 %y, 42
@@ -672,10 +718,11 @@ define i8 @negate_sdiv(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_sdiv_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_sdiv_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sdiv i8 [[Y:%.*]], 42
+; CHECK-LABEL: define i8 @negate_sdiv_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sdiv i8 [[Y]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sdiv i8 %y, 42
@@ -684,8 +731,9 @@ define i8 @negate_sdiv_extrause(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_sdiv_extrause2(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_sdiv_extrause2(
-; CHECK-NEXT:    [[T0:%.*]] = sdiv i8 [[Y:%.*]], 42
+; CHECK-LABEL: define i8 @negate_sdiv_extrause2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sdiv i8 [[Y]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = sub nsw i8 0, [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
@@ -698,9 +746,10 @@ define i8 @negate_sdiv_extrause2(i8 %x, i8 %y) {
 
 ; Right-shift sign bit smear is negatible.
 define i8 @negate_ashr(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_ashr(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = lshr i8 [[Y:%.*]], 7
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_ashr(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = lshr i8 [[Y]], 7
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = ashr i8 %y, 7
@@ -708,9 +757,10 @@ define i8 @negate_ashr(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_lshr(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_lshr(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = ashr i8 [[Y:%.*]], 7
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_lshr(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = ashr i8 [[Y]], 7
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[T0_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = lshr i8 %y, 7
@@ -718,10 +768,11 @@ define i8 @negate_lshr(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_ashr_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_ashr_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = ashr i8 [[Y:%.*]], 7
+; CHECK-LABEL: define i8 @negate_ashr_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = ashr i8 [[Y]], 7
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = ashr i8 %y, 7
@@ -730,10 +781,11 @@ define i8 @negate_ashr_extrause(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_lshr_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_lshr_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = lshr i8 [[Y:%.*]], 7
+; CHECK-LABEL: define i8 @negate_lshr_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = lshr i8 [[Y]], 7
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = lshr i8 %y, 7
@@ -742,9 +794,10 @@ define i8 @negate_lshr_extrause(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_ashr_wrongshift(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_ashr_wrongshift(
-; CHECK-NEXT:    [[T0:%.*]] = ashr i8 [[Y:%.*]], 6
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_ashr_wrongshift(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = ashr i8 [[Y]], 6
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = ashr i8 %y, 6
@@ -752,9 +805,10 @@ define i8 @negate_ashr_wrongshift(i8 %x, i8 %y) {
   ret i8 %t1
 }
 define i8 @negate_lshr_wrongshift(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_lshr_wrongshift(
-; CHECK-NEXT:    [[T0:%.*]] = lshr i8 [[Y:%.*]], 6
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_lshr_wrongshift(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = lshr i8 [[Y]], 6
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = lshr i8 %y, 6
@@ -764,9 +818,10 @@ define i8 @negate_lshr_wrongshift(i8 %x, i8 %y) {
 
 ; *ext of i1 is always negatible
 define i8 @negate_sext(i8 %x, i1 %y) {
-; CHECK-LABEL: @negate_sext(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = zext i1 [[Y:%.*]] to i8
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X:%.*]], [[T0_NEG]]
+; CHECK-LABEL: define i8 @negate_sext(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = zext i1 [[Y]] to i8
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X]], [[T0_NEG]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sext i1 %y to i8
@@ -774,9 +829,10 @@ define i8 @negate_sext(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @negate_zext(i8 %x, i1 %y) {
-; CHECK-LABEL: @negate_zext(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sext i1 [[Y:%.*]] to i8
-; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X:%.*]], [[T0_NEG]]
+; CHECK-LABEL: define i8 @negate_zext(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = sext i1 [[Y]] to i8
+; CHECK-NEXT:    [[T1:%.*]] = add i8 [[X]], [[T0_NEG]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = zext i1 %y to i8
@@ -784,10 +840,11 @@ define i8 @negate_zext(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @negate_sext_extrause(i8 %x, i1 %y) {
-; CHECK-LABEL: @negate_sext_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sext i1 [[Y:%.*]] to i8
+; CHECK-LABEL: define i8 @negate_sext_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sext i1 [[Y]] to i8
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sext i1 %y to i8
@@ -796,10 +853,11 @@ define i8 @negate_sext_extrause(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @negate_zext_extrause(i8 %x, i1 %y) {
-; CHECK-LABEL: @negate_zext_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = zext i1 [[Y:%.*]] to i8
+; CHECK-LABEL: define i8 @negate_zext_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i1 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = zext i1 [[Y]] to i8
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = zext i1 %y to i8
@@ -808,9 +866,10 @@ define i8 @negate_zext_extrause(i8 %x, i1 %y) {
   ret i8 %t1
 }
 define i8 @negate_sext_wrongwidth(i8 %x, i2 %y) {
-; CHECK-LABEL: @negate_sext_wrongwidth(
-; CHECK-NEXT:    [[T0:%.*]] = sext i2 [[Y:%.*]] to i8
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_sext_wrongwidth(
+; CHECK-SAME: i8 [[X:%.*]], i2 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sext i2 [[Y]] to i8
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = sext i2 %y to i8
@@ -818,9 +877,10 @@ define i8 @negate_sext_wrongwidth(i8 %x, i2 %y) {
   ret i8 %t1
 }
 define i8 @negate_zext_wrongwidth(i8 %x, i2 %y) {
-; CHECK-LABEL: @negate_zext_wrongwidth(
-; CHECK-NEXT:    [[T0:%.*]] = zext i2 [[Y:%.*]] to i8
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_zext_wrongwidth(
+; CHECK-SAME: i8 [[X:%.*]], i2 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = zext i2 [[Y]] to i8
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[X]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = zext i2 %y to i8
@@ -829,10 +889,11 @@ define i8 @negate_zext_wrongwidth(i8 %x, i2 %y) {
 }
 
 define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y) {
-; CHECK-LABEL: @negate_shufflevector_oneinput_reverse(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_shufflevector_oneinput_reverse(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X]]
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T1_NEG]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i4> [[T2]]
 ;
   %t0 = shl <2 x i4> <i4 -6, i4 5>, %x
@@ -841,10 +902,11 @@ define <2 x i4> @negate_shufflevector_oneinput_reverse(<2 x i4> %x, <2 x i4> %y)
   ret <2 x i4> %t2
 }
 define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(<2 x i4> %x, <2 x i4> %y) {
-; CHECK-LABEL: @negate_shufflevector_oneinput_second_lane_is_undef(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X]]
 ; CHECK-NEXT:    [[T11_NEG:%.*]] = insertelement <2 x i4> [[T0_NEG]], i4 undef, i64 1
-; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T11_NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <2 x i4> [[T11_NEG]], [[Y]]
 ; CHECK-NEXT:    ret <2 x i4> [[T2]]
 ;
   %t0 = shl <2 x i4> <i4 -6, i4 5>, %x
@@ -853,11 +915,12 @@ define <2 x i4> @negate_shufflevector_oneinput_second_lane_is_undef(<2 x i4> %x,
   ret <2 x i4> %t2
 }
 define <2 x i4> @negate_shufflevector_twoinputs(<2 x i4> %x, <2 x i4> %y, <2 x i4> %z) {
-; CHECK-LABEL: @negate_shufflevector_twoinputs(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = add <2 x i4> [[Y:%.*]], <i4 poison, i4 1>
+; CHECK-LABEL: define <2 x i4> @negate_shufflevector_twoinputs(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]], <2 x i4> [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = shl <2 x i4> <i4 6, i4 -5>, [[X]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = add <2 x i4> [[Y]], <i4 poison, i4 1>
 ; CHECK-NEXT:    [[T2_NEG:%.*]] = shufflevector <2 x i4> [[T0_NEG]], <2 x i4> [[T1_NEG]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT:    [[T3:%.*]] = add <2 x i4> [[T2_NEG]], [[Z:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = add <2 x i4> [[T2_NEG]], [[Z]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t0 = shl <2 x i4> <i4 -6, i4 5>, %x
@@ -867,11 +930,12 @@ define <2 x i4> @negate_shufflevector_twoinputs(<2 x i4> %x, <2 x i4> %y, <2 x i
   ret <2 x i4> %t3
 }
 define <2 x i4> @negate_shufflevector_oneinput_extrause(<2 x i4> %x, <2 x i4> %y) {
-; CHECK-LABEL: @negate_shufflevector_oneinput_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i4> <i4 -6, i4 5>, [[X:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_shufflevector_oneinput_extrause(
+; CHECK-SAME: <2 x i4> [[X:%.*]], <2 x i4> [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl <2 x i4> <i4 -6, i4 5>, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = shufflevector <2 x i4> [[T0]], <2 x i4> poison, <2 x i32> <i32 1, i32 0>
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub <2 x i4> [[Y:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub <2 x i4> [[Y]], [[T1]]
 ; CHECK-NEXT:    ret <2 x i4> [[T2]]
 ;
   %t0 = shl <2 x i4> <i4 -6, i4 5>, %x
@@ -884,15 +948,16 @@ define <2 x i4> @negate_shufflevector_oneinput_extrause(<2 x i4> %x, <2 x i4> %y
 ; zext of non-negative can be negated
 ; sext of non-positive can be negated
 define i16 @negation_of_zeroext_of_nonnegative(i8 %x) {
-; CHECK-LABEL: @negation_of_zeroext_of_nonnegative(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_zeroext_of_nonnegative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i8 [[T0]], -1
-; CHECK-NEXT:    br i1 [[T1]], label [[NONNEG_BB:%.*]], label [[NEG_BB:%.*]]
-; CHECK:       nonneg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NONNEG_BB:.*]], label %[[NEG_BB:.*]]
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = zext nneg i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       neg_bb:
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -908,15 +973,16 @@ neg_bb:
   ret i16 0
 }
 define i16 @negation_of_zeroext_of_positive(i8 %x) {
-; CHECK-LABEL: @negation_of_zeroext_of_positive(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_zeroext_of_positive(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i8 [[T0]], 0
-; CHECK-NEXT:    br i1 [[T1]], label [[NONNEG_BB:%.*]], label [[NEG_BB:%.*]]
-; CHECK:       nonneg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NONNEG_BB:.*]], label %[[NEG_BB:.*]]
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = zext nneg i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       neg_bb:
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -932,15 +998,16 @@ neg_bb:
   ret i16 0
 }
 define i16 @negation_of_signext_of_negative(i8 %x) {
-; CHECK-LABEL: @negation_of_signext_of_negative(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_signext_of_negative(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp slt i8 [[T0]], 0
-; CHECK-NEXT:    br i1 [[T1]], label [[NEG_BB:%.*]], label [[NONNEG_BB:%.*]]
-; CHECK:       neg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NEG_BB:.*]], label %[[NONNEG_BB:.*]]
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = sext i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       nonneg_bb:
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -956,15 +1023,16 @@ nonneg_bb:
   ret i16 0
 }
 define i16 @negation_of_signext_of_nonpositive(i8 %x) {
-; CHECK-LABEL: @negation_of_signext_of_nonpositive(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_signext_of_nonpositive(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp slt i8 [[T0]], 1
-; CHECK-NEXT:    br i1 [[T1]], label [[NEG_BB:%.*]], label [[NONNEG_BB:%.*]]
-; CHECK:       neg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NEG_BB:.*]], label %[[NONNEG_BB:.*]]
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = sext i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       nonneg_bb:
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -980,15 +1048,16 @@ nonneg_bb:
   ret i16 0
 }
 define i16 @negation_of_signext_of_nonnegative__wrong_cast(i8 %x) {
-; CHECK-LABEL: @negation_of_signext_of_nonnegative__wrong_cast(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_signext_of_nonnegative__wrong_cast(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp sgt i8 [[T0]], -1
-; CHECK-NEXT:    br i1 [[T1]], label [[NONNEG_BB:%.*]], label [[NEG_BB:%.*]]
-; CHECK:       nonneg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NONNEG_BB:.*]], label %[[NEG_BB:.*]]
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = zext nneg i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       neg_bb:
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -1004,15 +1073,16 @@ neg_bb:
   ret i16 0
 }
 define i16 @negation_of_zeroext_of_negative_wrongcast(i8 %x) {
-; CHECK-LABEL: @negation_of_zeroext_of_negative_wrongcast(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i16 @negation_of_zeroext_of_negative_wrongcast(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    [[T1:%.*]] = icmp slt i8 [[T0]], 0
-; CHECK-NEXT:    br i1 [[T1]], label [[NEG_BB:%.*]], label [[NONNEG_BB:%.*]]
-; CHECK:       neg_bb:
+; CHECK-NEXT:    br i1 [[T1]], label %[[NEG_BB:.*]], label %[[NONNEG_BB:.*]]
+; CHECK:       [[NEG_BB]]:
 ; CHECK-NEXT:    [[T2:%.*]] = zext i8 [[T0]] to i16
 ; CHECK-NEXT:    [[T3:%.*]] = sub nsw i16 0, [[T2]]
 ; CHECK-NEXT:    ret i16 [[T3]]
-; CHECK:       nonneg_bb:
+; CHECK:       [[NONNEG_BB]]:
 ; CHECK-NEXT:    ret i16 0
 ;
   %t0 = sub i8 0, %x
@@ -1030,10 +1100,11 @@ nonneg_bb:
 
 ; 'or' of 1 and operand with no lowest bit set is 'inc'
 define i8 @negation_of_increment_via_or_with_no_common_bits_set(i8 %x, i8 %y) {
-; CHECK-LABEL: @negation_of_increment_via_or_with_no_common_bits_set(
-; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y:%.*]], 1
+; CHECK-LABEL: define i8 @negation_of_increment_via_or_with_no_common_bits_set(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y]], 1
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = xor i8 [[T0]], -1
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X:%.*]], [[T1_NEG]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X]], [[T1_NEG]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 %y, 1
@@ -1042,11 +1113,12 @@ define i8 @negation_of_increment_via_or_with_no_common_bits_set(i8 %x, i8 %y) {
   ret i8 %t2
 }
 define i8 @negation_of_increment_via_or_with_no_common_bits_set_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negation_of_increment_via_or_with_no_common_bits_set_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y:%.*]], 1
+; CHECK-LABEL: define i8 @negation_of_increment_via_or_with_no_common_bits_set_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y]], 1
 ; CHECK-NEXT:    [[T1:%.*]] = or disjoint i8 [[T0]], 1
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 %y, 1
@@ -1056,10 +1128,11 @@ define i8 @negation_of_increment_via_or_with_no_common_bits_set_extrause(i8 %x,
   ret i8 %t2
 }
 define i8 @negation_of_increment_via_or_common_bits_set(i8 %x, i8 %y) {
-; CHECK-LABEL: @negation_of_increment_via_or_common_bits_set(
-; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y:%.*]], 1
+; CHECK-LABEL: define i8 @negation_of_increment_via_or_common_bits_set(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = shl i8 [[Y]], 1
 ; CHECK-NEXT:    [[T1:%.*]] = or i8 [[T0]], 3
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = shl i8 %y, 1
@@ -1069,9 +1142,10 @@ define i8 @negation_of_increment_via_or_common_bits_set(i8 %x, i8 %y) {
 }
 
 define i8 @negation_of_increment_via_or_disjoint(i8 %x, i8 %y) {
-; CHECK-LABEL: @negation_of_increment_via_or_disjoint(
-; CHECK-NEXT:    [[T1_NEG:%.*]] = xor i8 [[Y:%.*]], -1
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X:%.*]], [[T1_NEG]]
+; CHECK-LABEL: define i8 @negation_of_increment_via_or_disjoint(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T1_NEG:%.*]] = xor i8 [[Y]], -1
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[X]], [[T1_NEG]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t1 = or disjoint i8 %y, 1
@@ -1081,12 +1155,13 @@ define i8 @negation_of_increment_via_or_disjoint(i8 %x, i8 %y) {
 
 ; 'or' of operands with no common bits set is 'add'
 define i8 @add_via_or_with_no_common_bits_set(i8 %x, i8 %y) {
-; CHECK-LABEL: @add_via_or_with_no_common_bits_set(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @add_via_or_with_no_common_bits_set(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = shl i8 [[Y]], 2
 ; CHECK-NEXT:    [[T2_NEG:%.*]] = add i8 [[T1_NEG]], -3
-; CHECK-NEXT:    [[T3:%.*]] = add i8 [[T2_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[T2_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -1097,12 +1172,13 @@ define i8 @add_via_or_with_no_common_bits_set(i8 %x, i8 %y) {
   ret i8 %t3
 }
 define i8 @add_via_or_with_common_bit_maybe_set(i8 %x, i8 %y) {
-; CHECK-LABEL: @add_via_or_with_common_bit_maybe_set(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @add_via_or_with_common_bit_maybe_set(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 2
 ; CHECK-NEXT:    [[T2:%.*]] = or i8 [[T1]], 4
-; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[X]], [[T2]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -1113,13 +1189,14 @@ define i8 @add_via_or_with_common_bit_maybe_set(i8 %x, i8 %y) {
   ret i8 %t3
 }
 define i8 @add_via_or_with_no_common_bits_set_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @add_via_or_with_no_common_bits_set_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y:%.*]]
+; CHECK-LABEL: define i8 @add_via_or_with_no_common_bits_set_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Y]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 2
 ; CHECK-NEXT:    [[T2:%.*]] = or disjoint i8 [[T1]], 3
 ; CHECK-NEXT:    call void @use8(i8 [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[X]], [[T2]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %y
@@ -1133,11 +1210,12 @@ define i8 @add_via_or_with_no_common_bits_set_extrause(i8 %x, i8 %y) {
 
 ; `extractelement` is negatible if source operand is negatible.
 define i4 @negate_extractelement(<2 x i4> %x, i32 %y, i4 %z) {
-; CHECK-LABEL: @negate_extractelement(
-; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[X:%.*]]
+; CHECK-LABEL: define i4 @negate_extractelement(
+; CHECK-SAME: <2 x i4> [[X:%.*]], i32 [[Y:%.*]], i4 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[X]]
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[T0]])
-; CHECK-NEXT:    [[T1_NEG:%.*]] = extractelement <2 x i4> [[X]], i32 [[Y:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i4 [[T1_NEG]], [[Z:%.*]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = extractelement <2 x i4> [[X]], i32 [[Y]]
+; CHECK-NEXT:    [[T2:%.*]] = add i4 [[T1_NEG]], [[Z]]
 ; CHECK-NEXT:    ret i4 [[T2]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %x
@@ -1147,12 +1225,13 @@ define i4 @negate_extractelement(<2 x i4> %x, i32 %y, i4 %z) {
   ret i4 %t2
 }
 define i4 @negate_extractelement_extrause(<2 x i4> %x, i32 %y, i4 %z) {
-; CHECK-LABEL: @negate_extractelement_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[X:%.*]]
+; CHECK-LABEL: define i4 @negate_extractelement_extrause(
+; CHECK-SAME: <2 x i4> [[X:%.*]], i32 [[Y:%.*]], i4 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[X]]
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[T0]])
-; CHECK-NEXT:    [[T1:%.*]] = extractelement <2 x i4> [[T0]], i32 [[Y:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = extractelement <2 x i4> [[T0]], i32 [[Y]]
 ; CHECK-NEXT:    call void @use4(i4 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i4 [[Z:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i4 [[Z]], [[T1]]
 ; CHECK-NEXT:    ret i4 [[T2]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %x
@@ -1165,9 +1244,10 @@ define i4 @negate_extractelement_extrause(<2 x i4> %x, i32 %y, i4 %z) {
 
 ; `insertelement` is negatible if both source vector and element-to-be-inserted are negatible.
 define <2 x i4> @negate_insertelement(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b) {
-; CHECK-LABEL: @negate_insertelement(
-; CHECK-NEXT:    [[T2_NEG:%.*]] = insertelement <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = add <2 x i4> [[T2_NEG]], [[B:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_insertelement(
+; CHECK-SAME: <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[T2_NEG:%.*]] = insertelement <2 x i4> [[SRC]], i4 [[A]], i32 [[X]]
+; CHECK-NEXT:    [[T3:%.*]] = add <2 x i4> [[T2_NEG]], [[B]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %src
@@ -1177,12 +1257,13 @@ define <2 x i4> @negate_insertelement(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b)
   ret <2 x i4> %t3
 }
 define <2 x i4> @negate_insertelement_extrause(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b) {
-; CHECK-LABEL: @negate_insertelement_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[SRC:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = sub i4 0, [[A:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[T0]], i4 [[T1]], i32 [[X:%.*]]
+; CHECK-LABEL: define <2 x i4> @negate_insertelement_extrause(
+; CHECK-SAME: <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[SRC]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i4 0, [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[T0]], i4 [[T1]], i32 [[X]]
 ; CHECK-NEXT:    call void @use_v2i4(<2 x i4> [[T2]])
-; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B]], [[T2]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %src
@@ -1193,10 +1274,11 @@ define <2 x i4> @negate_insertelement_extrause(<2 x i4> %src, i4 %a, i32 %x, <2
   ret <2 x i4> %t3
 }
 define <2 x i4> @negate_insertelement_nonnegatible_base(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b) {
-; CHECK-LABEL: @negate_insertelement_nonnegatible_base(
-; CHECK-NEXT:    [[T1:%.*]] = sub i4 0, [[A:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[SRC:%.*]], i4 [[T1]], i32 [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B:%.*]], [[T2]]
+; CHECK-LABEL: define <2 x i4> @negate_insertelement_nonnegatible_base(
+; CHECK-SAME: <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[T1:%.*]] = sub i4 0, [[A]]
+; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[SRC]], i4 [[T1]], i32 [[X]]
+; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B]], [[T2]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t1 = sub i4 zeroinitializer, %a
@@ -1205,10 +1287,11 @@ define <2 x i4> @negate_insertelement_nonnegatible_base(<2 x i4> %src, i4 %a, i3
   ret <2 x i4> %t3
 }
 define <2 x i4> @negate_insertelement_nonnegatible_insert(<2 x i4> %src, i4 %a, i32 %x, <2 x i4> %b) {
-; CHECK-LABEL: @negate_insertelement_nonnegatible_insert(
-; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[SRC:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[T0]], i4 [[A:%.*]], i32 [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B:%.*]], [[T2]]
+; CHECK-LABEL: define <2 x i4> @negate_insertelement_nonnegatible_insert(
+; CHECK-SAME: <2 x i4> [[SRC:%.*]], i4 [[A:%.*]], i32 [[X:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub <2 x i4> zeroinitializer, [[SRC]]
+; CHECK-NEXT:    [[T2:%.*]] = insertelement <2 x i4> [[T0]], i4 [[A]], i32 [[X]]
+; CHECK-NEXT:    [[T3:%.*]] = sub <2 x i4> [[B]], [[T2]]
 ; CHECK-NEXT:    ret <2 x i4> [[T3]]
 ;
   %t0 = sub <2 x i4> zeroinitializer, %src
@@ -1219,11 +1302,12 @@ define <2 x i4> @negate_insertelement_nonnegatible_insert(<2 x i4> %src, i4 %a,
 
 ; left-shift by constant can always be negated
 define i8 @negate_left_shift_by_constant_prefer_keeping_shl(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @negate_left_shift_by_constant_prefer_keeping_shl(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @negate_left_shift_by_constant_prefer_keeping_shl(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1_NEG:%.*]] = shl i8 [[Z]], 4
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[T1_NEG]], [[X]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -1233,12 +1317,13 @@ define i8 @negate_left_shift_by_constant_prefer_keeping_shl(i8 %x, i8 %y, i8 %z)
   ret i8 %t2
 }
 define i8 @negate_left_shift_by_constant_prefer_keeping_shl_extrause(i8 %x, i8 %y, i8 %z) {
-; CHECK-LABEL: @negate_left_shift_by_constant_prefer_keeping_shl_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z:%.*]]
+; CHECK-LABEL: define i8 @negate_left_shift_by_constant_prefer_keeping_shl_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 4
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %z
@@ -1249,11 +1334,12 @@ define i8 @negate_left_shift_by_constant_prefer_keeping_shl_extrause(i8 %x, i8 %
   ret i8 %t2
 }
 define i8 @negate_left_shift_by_constant(i8 %x, i8 %y, i8 %z, i8 %k) {
-; CHECK-LABEL: @negate_left_shift_by_constant(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[K:%.*]], [[Z:%.*]]
+; CHECK-LABEL: define i8 @negate_left_shift_by_constant(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]], i8 [[K:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[K]], [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 4
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 %k, %z
@@ -1263,12 +1349,13 @@ define i8 @negate_left_shift_by_constant(i8 %x, i8 %y, i8 %z, i8 %k) {
   ret i8 %t2
 }
 define i8 @negate_left_shift_by_constant_extrause(i8 %x, i8 %y, i8 %z, i8 %k) {
-; CHECK-LABEL: @negate_left_shift_by_constant_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[K:%.*]], [[Z:%.*]]
+; CHECK-LABEL: define i8 @negate_left_shift_by_constant_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]], i8 [[K:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 [[K]], [[Z]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = shl i8 [[T0]], 4
 ; CHECK-NEXT:    call void @use8(i8 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i8 [[X]], [[T1]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 %k, %z
@@ -1281,8 +1368,9 @@ define i8 @negate_left_shift_by_constant_extrause(i8 %x, i8 %y, i8 %z, i8 %k) {
 
 ; `add` with single negatible operand is still negatible
 define i8 @negate_add_with_single_negatible_operand(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_add_with_single_negatible_operand(
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 -42, [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_add_with_single_negatible_operand(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 -42, [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = add i8 %x, 42
@@ -1291,9 +1379,10 @@ define i8 @negate_add_with_single_negatible_operand(i8 %x, i8 %y) {
 }
 ; do so even if we are two levels deep
 define i8 @negate_add_with_single_negatible_operand_depth2(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_add_with_single_negatible_operand_depth2(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i8 -21, [[X:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = mul i8 [[T0_NEG]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @negate_add_with_single_negatible_operand_depth2(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i8 -21, [[X]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = mul i8 [[T0_NEG]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T1_NEG]]
 ;
   %t0 = add i8 %x, 21
@@ -1303,8 +1392,9 @@ define i8 @negate_add_with_single_negatible_operand_depth2(i8 %x, i8 %y) {
 }
 
 define i8 @negate_add_with_single_negatible_operand_extrause(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_add_with_single_negatible_operand_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], 42
+; CHECK-LABEL: define i8 @negate_add_with_single_negatible_operand_extrause(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X]], 42
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T1:%.*]] = sub i8 -42, [[X]]
 ; CHECK-NEXT:    ret i8 [[T1]]
@@ -1316,9 +1406,10 @@ define i8 @negate_add_with_single_negatible_operand_extrause(i8 %x, i8 %y) {
 }
 ; But don't do this if that means just sinking the negation.
 define i8 @negate_add_with_single_negatible_operand_non_negation(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_add_with_single_negatible_operand_non_negation(
-; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X:%.*]], 42
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[Y:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @negate_add_with_single_negatible_operand_non_negation(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = add i8 [[X]], 42
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[Y]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = add i8 %x, 42
@@ -1328,11 +1419,12 @@ define i8 @negate_add_with_single_negatible_operand_non_negation(i8 %x, i8 %y) {
 
 ; abs/nabs can be negated
 define i8 @negate_abs(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_abs(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_abs(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[T2:%.*]] = call i8 @llvm.abs.i8(i8 [[X]], i1 false)
-; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[Y:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = sub i8 [[Y]], [[T2]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %x
@@ -1343,11 +1435,12 @@ define i8 @negate_abs(i8 %x, i8 %y) {
   ret i8 %t3
 }
 define i8 @negate_nabs(i8 %x, i8 %y) {
-; CHECK-LABEL: @negate_nabs(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_nabs(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.abs.i8(i8 [[X]], i1 false)
-; CHECK-NEXT:    [[T3:%.*]] = add i8 [[Y:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[T3:%.*]] = add i8 [[Y]], [[TMP1]]
 ; CHECK-NEXT:    ret i8 [[T3]]
 ;
   %t0 = sub i8 0, %x
@@ -1361,11 +1454,12 @@ define i8 @negate_nabs(i8 %x, i8 %y) {
 ; And in general, if hands of select are known to be negation of each other,
 ; we can negate the select
 define i8 @negate_select_of_op_vs_negated_op(i8 %x, i8 %y, i1 %c) {
-; CHECK-LABEL: @negate_select_of_op_vs_negated_op(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
+; CHECK-LABEL: define i8 @negate_select_of_op_vs_negated_op(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
 ; CHECK-NEXT:    call void @use8(i8 [[T0]])
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C:%.*]], i8 [[X]], i8 [[T0]], !prof [[PROF0:![0-9]+]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C]], i8 [[X]], i8 [[T0]], !prof [[PROF0:![0-9]+]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub i8 0, %x
@@ -1376,10 +1470,11 @@ define i8 @negate_select_of_op_vs_negated_op(i8 %x, i8 %y, i1 %c) {
 }
 
 define i8 @negate_select_of_op_vs_negated_op_nsw(i8 %x, i8 %y, i1 %c) {
-; CHECK-LABEL: @negate_select_of_op_vs_negated_op_nsw(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C:%.*]], i8 [[X]], i8 [[T0]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @negate_select_of_op_vs_negated_op_nsw(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C]], i8 [[X]], i8 [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub nsw i8 0, %x
@@ -1389,10 +1484,11 @@ define i8 @negate_select_of_op_vs_negated_op_nsw(i8 %x, i8 %y, i1 %c) {
 }
 
 define i8 @negate_select_of_op_vs_negated_op_nsw_commuted(i8 %x, i8 %y, i1 %c) {
-; CHECK-LABEL: @negate_select_of_op_vs_negated_op_nsw_commuted(
-; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X:%.*]]
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C:%.*]], i8 [[T0]], i8 [[X]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @negate_select_of_op_vs_negated_op_nsw_commuted(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = sub i8 0, [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C]], i8 [[T0]], i8 [[X]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Y]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %t0 = sub nsw i8 0, %x
@@ -1402,11 +1498,12 @@ define i8 @negate_select_of_op_vs_negated_op_nsw_commuted(i8 %x, i8 %y, i1 %c) {
 }
 
 define i8 @negate_select_of_op_vs_negated_op_nsw_xyyx(i8 %x, i8 %y, i8 %z, i1 %c) {
-; CHECK-LABEL: @negate_select_of_op_vs_negated_op_nsw_xyyx(
-; CHECK-NEXT:    [[SUB1:%.*]] = sub i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define i8 @negate_select_of_op_vs_negated_op_nsw_xyyx(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[SUB1:%.*]] = sub i8 [[X]], [[Y]]
 ; CHECK-NEXT:    [[SUB2:%.*]] = sub i8 [[Y]], [[X]]
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C:%.*]], i8 [[SUB2]], i8 [[SUB1]]
-; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Z:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[C]], i8 [[SUB2]], i8 [[SUB1]]
+; CHECK-NEXT:    [[T2:%.*]] = add i8 [[TMP1]], [[Z]]
 ; CHECK-NEXT:    ret i8 [[T2]]
 ;
   %sub1 = sub nsw i8 %x, %y
@@ -1417,9 +1514,10 @@ define i8 @negate_select_of_op_vs_negated_op_nsw_xyyx(i8 %x, i8 %y, i8 %z, i1 %c
 }
 
 define i8 @dont_negate_ordinary_select(i8 %x, i8 %y, i8 %z, i1 %c) {
-; CHECK-LABEL: @dont_negate_ordinary_select(
-; CHECK-NEXT:    [[T0:%.*]] = select i1 [[C:%.*]], i8 [[X:%.*]], i8 [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[Z:%.*]], [[T0]]
+; CHECK-LABEL: define i8 @dont_negate_ordinary_select(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]], i1 [[C:%.*]]) {
+; CHECK-NEXT:    [[T0:%.*]] = select i1 [[C]], i8 [[X]], i8 [[Y]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i8 [[Z]], [[T0]]
 ; CHECK-NEXT:    ret i8 [[T1]]
 ;
   %t0 = select i1 %c, i8 %x, i8 %y
@@ -1428,9 +1526,10 @@ define i8 @dont_negate_ordinary_select(i8 %x, i8 %y, i8 %z, i1 %c) {
 }
 
 define <2 x i32> @negate_select_of_negation_poison(<2 x i1> %c, <2 x i32> %x) {
-; CHECK-LABEL: @negate_select_of_negation_poison(
-; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i32> <i32 0, i32 poison>, [[X:%.*]]
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[C:%.*]], <2 x i32> [[NEG]], <2 x i32> [[X]]
+; CHECK-LABEL: define <2 x i32> @negate_select_of_negation_poison(
+; CHECK-SAME: <2 x i1> [[C:%.*]], <2 x i32> [[X:%.*]]) {
+; CHECK-NEXT:    [[NEG:%.*]] = sub <2 x i32> <i32 0, i32 poison>, [[X]]
+; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[C]], <2 x i32> [[NEG]], <2 x i32> [[X]]
 ; CHECK-NEXT:    [[NEG2:%.*]] = sub <2 x i32> zeroinitializer, [[SEL]]
 ; CHECK-NEXT:    ret <2 x i32> [[NEG2]]
 ;
@@ -1442,10 +1541,12 @@ define <2 x i32> @negate_select_of_negation_poison(<2 x i1> %c, <2 x i32> %x) {
 
 ; Freeze is transparent as far as negation is concerned
 define i4 @negate_freeze(i4 %x, i4 %y, i4 %z) {
-; CHECK-LABEL: @negate_freeze(
-; CHECK-NEXT:    [[T0_NEG:%.*]] = sub i4 [[Y:%.*]], [[X:%.*]]
-; CHECK-NEXT:    [[T1_NEG:%.*]] = freeze i4 [[T0_NEG]]
-; CHECK-NEXT:    [[T2:%.*]] = add i4 [[T1_NEG]], [[Z:%.*]]
+; CHECK-LABEL: define i4 @negate_freeze(
+; CHECK-SAME: i4 [[X:%.*]], i4 [[Y:%.*]], i4 [[Z:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i4 [[X]]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i4 [[Y]]
+; CHECK-NEXT:    [[T1_NEG:%.*]] = sub i4 [[Y_FR]], [[X_FR]]
+; CHECK-NEXT:    [[T2:%.*]] = add i4 [[T1_NEG]], [[Z]]
 ; CHECK-NEXT:    ret i4 [[T2]]
 ;
   %t0 = sub i4 %x, %y
@@ -1454,11 +1555,13 @@ define i4 @negate_freeze(i4 %x, i4 %y, i4 %z) {
   ret i4 %t2
 }
 define i4 @negate_freeze_extrause(i4 %x, i4 %y, i4 %z) {
-; CHECK-LABEL: @negate_freeze_extrause(
-; CHECK-NEXT:    [[T0:%.*]] = sub i4 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = freeze i4 [[T0]]
+; CHECK-LABEL: define i4 @negate_freeze_extrause(
+; CHECK-SAME: i4 [[X:%.*]], i4 [[Y:%.*]], i4 [[Z:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i4 [[X]]
+; CHECK-NEXT:    [[Y_FR:%.*]] = freeze i4 [[Y]]
+; CHECK-NEXT:    [[T1:%.*]] = sub i4 [[X_FR]], [[Y_FR]]
 ; CHECK-NEXT:    call void @use4(i4 [[T1]])
-; CHECK-NEXT:    [[T2:%.*]] = sub i4 [[Z:%.*]], [[T1]]
+; CHECK-NEXT:    [[T2:%.*]] = sub i4 [[Z]], [[T1]]
 ; CHECK-NEXT:    ret i4 [[T2]]
 ;
   %t0 = sub i4 %x, %y
@@ -1474,11 +1577,11 @@ define i4 @negate_freeze_extrause(i4 %x, i4 %y, i4 %z) {
 ; constants to RHS ourselves, since that is helpful sometimes.
 ; This used to cause an endless combine loop.
 define void @noncanonical_mul_with_constant_as_first_operand() {
-; CHECK-LABEL: @noncanonical_mul_with_constant_as_first_operand(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[IF_END:%.*]]
-; CHECK:       if.end:
-; CHECK-NEXT:    br label [[IF_END]]
+; CHECK-LABEL: define void @noncanonical_mul_with_constant_as_first_operand() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[IF_END:.*]]
+; CHECK:       [[IF_END]]:
+; CHECK-NEXT:    br label %[[IF_END]]
 ;
 entry:
   br label %if.end
@@ -1499,9 +1602,10 @@ if.end:
 @g = external hidden global [1 x [1 x double]]
 
 define <1 x i64> @PR56601(<1 x i64> %x, <1 x i64> %y) {
-; CHECK-LABEL: @PR56601(
-; CHECK-NEXT:    [[M1:%.*]] = mul nsw <1 x i64> [[X:%.*]], splat (i64 42)
-; CHECK-NEXT:    [[M2:%.*]] = mul nsw <1 x i64> [[Y:%.*]], splat (i64 12)
+; CHECK-LABEL: define <1 x i64> @PR56601(
+; CHECK-SAME: <1 x i64> [[X:%.*]], <1 x i64> [[Y:%.*]]) {
+; CHECK-NEXT:    [[M1:%.*]] = mul nsw <1 x i64> [[X]], splat (i64 42)
+; CHECK-NEXT:    [[M2:%.*]] = mul nsw <1 x i64> [[Y]], splat (i64 12)
 ; CHECK-NEXT:    [[A1:%.*]] = add <1 x i64> [[M1]], <i64 add (i64 ptrtoint (ptr @g to i64), i64 -4)>
 ; CHECK-NEXT:    [[A2:%.*]] = add <1 x i64> [[M2]], <i64 add (i64 ptrtoint (ptr @g to i64), i64 -3)>
 ; CHECK-NEXT:    [[R:%.*]] = sub <1 x i64> [[A1]], [[A2]]
@@ -1515,5 +1619,7 @@ define <1 x i64> @PR56601(<1 x i64> %x, <1 x i64> %y) {
   ret <1 x i64> %r
 }
 
-; CHECK: !0 = !{!"branch_weights", i32 40, i32 1}
 !0 = !{!"branch_weights", i32 40, i32 1}
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 40, i32 1}
+;.
diff --git a/llvm/test/Transforms/InstCombine/trivial-dse-calls.ll b/llvm/test/Transforms/InstCombine/trivial-dse-calls.ll
index 128edffc1ffb0..758071abdbe76 100644
--- a/llvm/test/Transforms/InstCombine/trivial-dse-calls.ll
+++ b/llvm/test/Transforms/InstCombine/trivial-dse-calls.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes=instcombine -S < %s | FileCheck %s
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @unknown()
 declare void @f(ptr)
@@ -25,9 +25,9 @@ define void @test_lifetime() {
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @f(ptr writeonly nocapture %a) argmemonly nounwind willreturn
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -40,11 +40,11 @@ define void @test_lifetime2() {
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @unknown()
   call void @f(ptr writeonly nocapture %a) argmemonly nounwind willreturn
   call void @unknown()
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
diff --git a/llvm/test/Transforms/InstCombine/unreachable-alloca-lifetime-markers.ll b/llvm/test/Transforms/InstCombine/unreachable-alloca-lifetime-markers.ll
index ab744c6213e4b..9c64bfb36303d 100644
--- a/llvm/test/Transforms/InstCombine/unreachable-alloca-lifetime-markers.ll
+++ b/llvm/test/Transforms/InstCombine/unreachable-alloca-lifetime-markers.ll
@@ -12,7 +12,7 @@ define void @pr150338(ptr %arg) {
   %a = alloca i32
   store ptr %a, ptr %arg
   store i1 true, ptr poison
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -33,7 +33,7 @@ entry:
 
 bb1:
   %phi1 = phi ptr [ null, %entry ], [ %phi2, %bb2 ]
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   br label %bb2
 
 bb2:
@@ -45,7 +45,7 @@ define void @lifetime_poison() {
 ; CHECK-LABEL: define void @lifetime_poison() {
 ; CHECK-NEXT:    ret void
 ;
-  call void @llvm.lifetime.start.p0(i64 4, ptr poison)
-  call void @llvm.lifetime.end.p0(i64 4, ptr poison)
+  call void @llvm.lifetime.start.p0(ptr poison)
+  call void @llvm.lifetime.end.p0(ptr poison)
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/urem-via-cmp-select.ll b/llvm/test/Transforms/InstCombine/urem-via-cmp-select.ll
index 02be67a2ca250..81e35daff6ec6 100644
--- a/llvm/test/Transforms/InstCombine/urem-via-cmp-select.ll
+++ b/llvm/test/Transforms/InstCombine/urem-via-cmp-select.ll
@@ -1,11 +1,12 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=instcombine -S | FileCheck %s
 
 ; https://alive2.llvm.org/ce/z/5eCiWi
 define i8 @urem_assume(i8 %x, i8 %n) {
-; CHECK-LABEL: @urem_assume(
-; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X_FR]], [[N:%.*]]
+; CHECK-LABEL: define i8 @urem_assume(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[N:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X_FR]], [[N]]
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X_FR]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[ADD]], [[N]]
@@ -21,9 +22,10 @@ define i8 @urem_assume(i8 %x, i8 %n) {
 
 ; https://alive2.llvm.org/ce/z/MGgtYN
 define i8 @urem_assume_without_nuw(i8 %x, i8 %n) {
-; CHECK-LABEL: @urem_assume_without_nuw(
-; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X:%.*]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X_FR]], [[N:%.*]]
+; CHECK-LABEL: define i8 @urem_assume_without_nuw(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[N:%.*]]) {
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X_FR]], [[N]]
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X_FR]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[ADD]], [[N]]
@@ -39,8 +41,9 @@ define i8 @urem_assume_without_nuw(i8 %x, i8 %n) {
 
 ; Negative test: The assume is false
 define i8 @urem_assume_eq(i8 %x, i8 %n) {
-; CHECK-LABEL: @urem_assume_eq(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X:%.*]], [[N:%.*]]
+; CHECK-LABEL: define i8 @urem_assume_eq(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[N:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[X]], [[N]]
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X]], 1
 ; CHECK-NEXT:    [[OUT:%.*]] = urem i8 [[ADD]], [[N]]
@@ -55,9 +58,10 @@ define i8 @urem_assume_eq(i8 %x, i8 %n) {
 
 ; Negative test: The assume is false
 define i8 @urem_assume_ne(i8 %x, i8 %n) {
-; CHECK-LABEL: @urem_assume_ne(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[X:%.*]], [[N:%.*]]
+; CHECK-LABEL: define i8 @urem_assume_ne(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[N:%.*]]) {
+; CHECK-NEXT:  [[START:.*:]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8 [[X]], [[N]]
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X]], 1
 ; CHECK-NEXT:    [[OUT:%.*]] = urem i8 [[ADD]], [[N]]
@@ -73,8 +77,9 @@ start:
 
 ; Negative test: The add constant is not 1
 define i8 @urem_assume_with_unexpected_const(i8 %x, i8 %n) {
-; CHECK-LABEL: @urem_assume_with_unexpected_const(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X:%.*]], [[N:%.*]]
+; CHECK-LABEL: define i8 @urem_assume_with_unexpected_const(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[N:%.*]]) {
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[X]], [[N]]
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X]], 2
 ; CHECK-NEXT:    [[OUT:%.*]] = urem i8 [[ADD]], [[N]]
@@ -89,11 +94,13 @@ define i8 @urem_assume_with_unexpected_const(i8 %x, i8 %n) {
 
 ; https://alive2.llvm.org/ce/z/gNhZ2x
 define i8 @urem_without_assume(i8 %arg, i8 %arg2) {
-; CHECK-LABEL: @urem_without_assume(
-; CHECK-NEXT:    [[X:%.*]] = urem i8 [[ARG:%.*]], [[ARG2:%.*]]
-; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X]]
+; CHECK-LABEL: define i8 @urem_without_assume(
+; CHECK-SAME: i8 [[ARG:%.*]], i8 [[ARG2:%.*]]) {
+; CHECK-NEXT:    [[ARG2_FR:%.*]] = freeze i8 [[ARG2]]
+; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i8 [[ARG]]
+; CHECK-NEXT:    [[X_FR:%.*]] = urem i8 [[ARG_FR]], [[ARG2_FR]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X_FR]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[ADD]], [[ARG2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8 [[ADD]], [[ARG2_FR]]
 ; CHECK-NEXT:    [[OUT:%.*]] = select i1 [[TMP1]], i8 0, i8 [[ADD]]
 ; CHECK-NEXT:    ret i8 [[OUT]]
 ;
@@ -105,17 +112,18 @@ define i8 @urem_without_assume(i8 %arg, i8 %arg2) {
 
 ; https://alive2.llvm.org/ce/z/eHkgRa
 define i8 @urem_with_dominating_condition(i8 %x, i8 %n) {
-; CHECK-LABEL: @urem_with_dominating_condition(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X:%.*]]
-; CHECK-NEXT:    [[COND:%.*]] = icmp ult i8 [[X_FR]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[DOTBB0:%.*]], label [[DOTBB1:%.*]]
-; CHECK:       .bb0:
+; CHECK-LABEL: define i8 @urem_with_dominating_condition(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[N:%.*]]) {
+; CHECK-NEXT:  [[START:.*:]]
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i8 [[X_FR]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], [[DOTBB0:label %.*]], [[DOTBB1:label %.*]]
+; CHECK:       [[_BB0:.*:]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X_FR]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i8 [[ADD]], [[N]]
 ; CHECK-NEXT:    [[OUT:%.*]] = select i1 [[TMP0]], i8 0, i8 [[ADD]]
 ; CHECK-NEXT:    ret i8 [[OUT]]
-; CHECK:       .bb1:
+; CHECK:       [[_BB1:.*:]]
 ; CHECK-NEXT:    ret i8 0
 ;
 start:
@@ -131,17 +139,18 @@ start:
 
 ; Revert the dominating condition and target branch at the same time.
 define i8 @urem_with_dominating_condition_false(i8 %x, i8 %n) {
-; CHECK-LABEL: @urem_with_dominating_condition_false(
-; CHECK-NEXT:  start:
-; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X:%.*]]
-; CHECK-NEXT:    [[COND_NOT:%.*]] = icmp ult i8 [[X_FR]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND_NOT]], label [[DOTBB0:%.*]], label [[DOTBB1:%.*]]
-; CHECK:       .bb0:
+; CHECK-LABEL: define i8 @urem_with_dominating_condition_false(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[N:%.*]]) {
+; CHECK-NEXT:  [[START:.*:]]
+; CHECK-NEXT:    [[X_FR:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT:    [[COND_NOT:%.*]] = icmp ult i8 [[X_FR]], [[N]]
+; CHECK-NEXT:    br i1 [[COND_NOT]], [[DOTBB0:label %.*]], [[DOTBB1:label %.*]]
+; CHECK:       [[_BB0:.*:]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X_FR]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq i8 [[ADD]], [[N]]
 ; CHECK-NEXT:    [[OUT:%.*]] = select i1 [[TMP0]], i8 0, i8 [[ADD]]
 ; CHECK-NEXT:    ret i8 [[OUT]]
-; CHECK:       .bb1:
+; CHECK:       [[_BB1:.*:]]
 ; CHECK-NEXT:    ret i8 0
 ;
 start:
@@ -157,14 +166,15 @@ start:
 
 ; Negative test
 define noundef i8 @urem_with_opposite_condition(i8 %x, i8 %n) {
-; CHECK-LABEL: @urem_with_opposite_condition(
-; CHECK-NEXT:    [[COND:%.*]] = icmp ult i8 [[X:%.*]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[COND]], label [[DOTBB1:%.*]], label [[DOTBB0:%.*]]
-; CHECK:       .bb0:
+; CHECK-LABEL: define noundef i8 @urem_with_opposite_condition(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[N:%.*]]) {
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i8 [[X]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], [[DOTBB1:label %.*]], [[DOTBB0:label %.*]]
+; CHECK:       [[_BB0:.*:]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[X]], 1
 ; CHECK-NEXT:    [[OUT:%.*]] = urem i8 [[ADD]], [[N]]
 ; CHECK-NEXT:    ret i8 [[OUT]]
-; CHECK:       .bb1:
+; CHECK:       [[_BB1:.*:]]
 ; CHECK-NEXT:    ret i8 0
 ;
   %cond = icmp ult i8 %x, %n
diff --git a/llvm/test/Transforms/InstCombine/vararg.ll b/llvm/test/Transforms/InstCombine/vararg.ll
index eb24256cfa9bc..93d230d22d8c8 100644
--- a/llvm/test/Transforms/InstCombine/vararg.ll
+++ b/llvm/test/Transforms/InstCombine/vararg.ll
@@ -12,14 +12,14 @@ define void @func(ptr nocapture readnone %fmt, ...) {
 entry:
   %va0 = alloca %struct.__va_list, align 8
   %va1 = alloca %struct.__va_list, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %va0)
+  call void @llvm.lifetime.start.p0(ptr %va0)
   call void @llvm.va_start(ptr %va0)
-  call void @llvm.lifetime.start.p0(i64 32, ptr %va1)
+  call void @llvm.lifetime.start.p0(ptr %va1)
   call void @llvm.va_copy(ptr %va1, ptr %va0)
   call void @llvm.va_end(ptr %va1)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %va1)
+  call void @llvm.lifetime.end.p0(ptr %va1)
   call void @llvm.va_end(ptr %va0)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %va0)
+  call void @llvm.lifetime.end.p0(ptr %va0)
   ret void
 }
 
@@ -31,28 +31,28 @@ define void @func_destroy_copy_src(ptr nocapture readnone %fmt, ...) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[VA0:%.*]] = alloca [[STRUCT___VA_LIST:%.*]], align 8
 ; CHECK-NEXT:    [[VA1:%.*]] = alloca [[STRUCT___VA_LIST]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VA0]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[VA1]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VA0]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[VA1]])
 ; CHECK-NEXT:    call void @llvm.va_start.p0(ptr nonnull [[VA0]])
 ; CHECK-NEXT:    call void @llvm.va_copy.p0(ptr nonnull [[VA1]], ptr nonnull [[VA0]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA0]])
 ; CHECK-NEXT:    call void @callee(ptr nonnull [[VA1]])
 ; CHECK-NEXT:    call void @llvm.va_end.p0(ptr [[VA1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VA1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[VA0]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VA1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[VA0]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %va0 = alloca %struct.__va_list, align 8
   %va1 = alloca %struct.__va_list, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %va0)
-  call void @llvm.lifetime.start.p0(i64 32, ptr %va1)
+  call void @llvm.lifetime.start.p0(ptr %va0)
+  call void @llvm.lifetime.start.p0(ptr %va1)
   call void @llvm.va_start(ptr %va0)
   call void @llvm.va_copy(ptr %va1, ptr %va0)
   call void @llvm.va_end(ptr %va0)
   call void @callee(ptr %va1)
   call void @llvm.va_end(ptr %va1)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %va1)
-  call void @llvm.lifetime.end.p0(i64 32, ptr %va0)
+  call void @llvm.lifetime.end.p0(ptr %va1)
+  call void @llvm.lifetime.end.p0(ptr %va0)
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/wcslen-1.ll b/llvm/test/Transforms/InstCombine/wcslen-1.ll
index 8a2b66d7bd8d3..ffd62564f1dde 100644
--- a/llvm/test/Transforms/InstCombine/wcslen-1.ll
+++ b/llvm/test/Transforms/InstCombine/wcslen-1.ll
@@ -118,6 +118,63 @@ define i64 @test_simplify10(i32 %x) {
   ret i64 %hello_l
 }
 
+define i64 @test_simplify10_gepi32(i64 %x) {
+; CHECK-LABEL: @test_simplify10_gepi32(
+; CHECK-NEXT:    [[HELLO_L:%.*]] = sub i64 5, [[X:%.*]]
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds i32, ptr @hello, i64 %x
+  %hello_l = call i64 @wcslen(ptr %hello_p)
+  ret i64 %hello_l
+}
+
+define i64 @test_simplify10_gepi64(i32 %x) {
+; CHECK-LABEL: @test_simplify10_gepi64(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds i64, ptr @hello, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds i64, ptr @hello, i32 %x
+  %hello_l = call i64 @wcslen(ptr %hello_p)
+  ret i64 %hello_l
+}
+
+define i64 @test_simplify10_gepi16(i64 %x) {
+; CHECK-LABEL: @test_simplify10_gepi16(
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds i16, ptr @hello, i64 [[X:%.*]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds i16, ptr @hello, i64 %x
+  %hello_l = call i64 @wcslen(ptr %hello_p)
+  ret i64 %hello_l
+}
+
+define i64 @test_simplify10_gepi8(i64 %x) {
+; CHECK-LABEL: @test_simplify10_gepi8(
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds i8, ptr @hello, i64 [[TMP1:%.*]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %hello_p = getelementptr inbounds i8, ptr @hello, i64 %x
+  %hello_l = call i64 @wcslen(ptr %hello_p)
+  ret i64 %hello_l
+}
+
+define i64 @test_simplify10_gepi8mul4(i64 %x) {
+; CHECK-LABEL: @test_simplify10_gepi8mul4(
+; CHECK-NEXT:    [[Y:%.*]] = shl i64 [[X:%.*]], 2
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds i8, ptr @hello, i64 [[Y]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %y = mul i64 %x, 4
+  %hello_p = getelementptr inbounds i8, ptr @hello, i64 %y
+  %hello_l = call i64 @wcslen(ptr %hello_p)
+  ret i64 %hello_l
+}
+
 ; wcslen(@null_hello_mid + (x & 7)) --> 9 - (x & 7)
 
 define i64 @test_simplify11(i32 %x) {
@@ -133,6 +190,33 @@ define i64 @test_simplify11(i32 %x) {
   ret i64 %hello_l
 }
 
+define i64 @test_simplify11_gepi32(i32 %x) {
+; CHECK-LABEL: @test_simplify11_gepi32(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 7
+; CHECK-NEXT:    [[NARROW:%.*]] = sub nuw nsw i32 9, [[AND]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = zext nneg i32 [[NARROW]] to i64
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %and = and i32 %x, 7
+  %hello_p = getelementptr inbounds i32, ptr @null_hello_mid, i32 %and
+  %hello_l = call i64 @wcslen(ptr %hello_p)
+  ret i64 %hello_l
+}
+
+define i64 @test_simplify11_gepi8(i32 %x) {
+; CHECK-LABEL: @test_simplify11_gepi8(
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 7
+; CHECK-NEXT:    [[TMP1:%.*]] = zext nneg i32 [[AND]] to i64
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds nuw i8, ptr @null_hello_mid, i64 [[TMP1]]
+; CHECK-NEXT:    [[HELLO_L:%.*]] = call i64 @wcslen(ptr nonnull [[HELLO_P]])
+; CHECK-NEXT:    ret i64 [[HELLO_L]]
+;
+  %and = and i32 %x, 7
+  %hello_p = getelementptr inbounds i8, ptr @null_hello_mid, i32 %and
+  %hello_l = call i64 @wcslen(ptr %hello_p)
+  ret i64 %hello_l
+}
+
 ; Check cases that shouldn't be simplified.
 
 define i64 @test_no_simplify1() {
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
new file mode 100644
index 0000000000000..a3d87439d74cf
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-add.ll
@@ -0,0 +1,876 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instsimplify -march=nvptx64 --mcpu=sm_86 --mattr=+ptx72 -S | FileCheck %s
+
+; Check constant-folding for NVVM add intrinsics with different rounding modes
+
+;###############################################################
+;#                    Add(1.25, -2.0)                          #
+;###############################################################
+; Tests addition of two normal numbers (1.25 and -2.0) where the result
+; is exactly representable. All rounding modes produce the same result.
+
+define double @test_1_25_minus_2_rm_d() {
+; CHECK-LABEL: define double @test_1_25_minus_2_rm_d() {
+; CHECK-NEXT:    ret double -7.500000e-01
+;
+  %res = call double @llvm.nvvm.add.rm.d(double 1.25, double -2.0)
+  ret double %res
+}
+
+define double @test_1_25_minus_2_rn_d() {
+; CHECK-LABEL: define double @test_1_25_minus_2_rn_d() {
+; CHECK-NEXT:    ret double -7.500000e-01
+;
+  %res = call double @llvm.nvvm.add.rn.d(double 1.25, double -2.0)
+  ret double %res
+}
+
+define double @test_1_25_minus_2_rp_d() {
+; CHECK-LABEL: define double @test_1_25_minus_2_rp_d() {
+; CHECK-NEXT:    ret double -7.500000e-01
+;
+  %res = call double @llvm.nvvm.add.rp.d(double 1.25, double -2.0)
+  ret double %res
+}
+
+define double @test_1_25_minus_2_rz_d() {
+; CHECK-LABEL: define double @test_1_25_minus_2_rz_d() {
+; CHECK-NEXT:    ret double -7.500000e-01
+;
+  %res = call double @llvm.nvvm.add.rz.d(double 1.25, double -2.0)
+  ret double %res
+}
+
+define float @test_1_25_minus_2_rm_f() {
+; CHECK-LABEL: define float @test_1_25_minus_2_rm_f() {
+; CHECK-NEXT:    ret float -7.500000e-01
+;
+  %res = call float @llvm.nvvm.add.rm.f(float 1.25, float -2.0)
+  ret float %res
+}
+
+define float @test_1_25_minus_2_rn_f() {
+; CHECK-LABEL: define float @test_1_25_minus_2_rn_f() {
+; CHECK-NEXT:    ret float -7.500000e-01
+;
+  %res = call float @llvm.nvvm.add.rn.f(float 1.25, float -2.0)
+  ret float %res
+}
+
+define float @test_1_25_minus_2_rp_f() {
+; CHECK-LABEL: define float @test_1_25_minus_2_rp_f() {
+; CHECK-NEXT:    ret float -7.500000e-01
+;
+  %res = call float @llvm.nvvm.add.rp.f(float 1.25, float -2.0)
+  ret float %res
+}
+
+define float @test_1_25_minus_2_rz_f() {
+; CHECK-LABEL: define float @test_1_25_minus_2_rz_f() {
+; CHECK-NEXT:    ret float -7.500000e-01
+;
+  %res = call float @llvm.nvvm.add.rz.f(float 1.25, float -2.0)
+  ret float %res
+}
+
+define float @test_1_25_minus_2_rm_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_minus_2_rm_ftz_f() {
+; CHECK-NEXT:    ret float -7.500000e-01
+;
+  %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.25, float -2.0)
+  ret float %res
+}
+
+define float @test_1_25_minus_2_rn_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_minus_2_rn_ftz_f() {
+; CHECK-NEXT:    ret float -7.500000e-01
+;
+  %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.25, float -2.0)
+  ret float %res
+}
+
+define float @test_1_25_minus_2_rp_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_minus_2_rp_ftz_f() {
+; CHECK-NEXT:    ret float -7.500000e-01
+;
+  %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.25, float -2.0)
+  ret float %res
+}
+
+define float @test_1_25_minus_2_rz_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_minus_2_rz_ftz_f() {
+; CHECK-NEXT:    ret float -7.500000e-01
+;
+  %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.25, float -2.0)
+  ret float %res
+}
+
+;###############################################################
+;#                          Add(0.0, NaN)                      #
+;###############################################################
+; Tests addition of a zero with NaN.
+; The result is always NaN and the operation is not constant-folded.
+
+define double @test_zero_plus_nan_rm_d() {
+; CHECK-LABEL: define double @test_zero_plus_nan_rm_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.add.rm.d(double 0.000000e+00, double 0x7FF4444400000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.add.rm.d(double 0.0, double 0x7ff4444400000000)
+  ret double %res
+}
+
+define double @test_zero_plus_nan_rn_d() {
+; CHECK-LABEL: define double @test_zero_plus_nan_rn_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.add.rn.d(double 0.000000e+00, double 0x7FF4444400000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.add.rn.d(double 0.0, double 0x7ff4444400000000)
+  ret double %res
+}
+
+define double @test_zero_plus_nan_rp_d() {
+; CHECK-LABEL: define double @test_zero_plus_nan_rp_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.add.rp.d(double 0.000000e+00, double 0x7FF4444400000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.add.rp.d(double 0.0, double 0x7ff4444400000000)
+  ret double %res
+}
+
+define double @test_zero_plus_nan_rz_d() {
+; CHECK-LABEL: define double @test_zero_plus_nan_rz_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.add.rz.d(double 0.000000e+00, double 0x7FF4444400000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.add.rz.d(double 0.0, double 0x7ff4444400000000)
+  ret double %res
+}
+
+define float @test_zero_plus_nan_rm_f() {
+; CHECK-LABEL: define float @test_zero_plus_nan_rm_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.add.rm.f(float 0.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.add.rm.f(float 0.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_zero_plus_nan_rn_f() {
+; CHECK-LABEL: define float @test_zero_plus_nan_rn_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.add.rn.f(float 0.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.add.rn.f(float 0.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_zero_plus_nan_rp_f() {
+; CHECK-LABEL: define float @test_zero_plus_nan_rp_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.add.rp.f(float 0.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.add.rp.f(float 0.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_zero_plus_nan_rz_f() {
+; CHECK-LABEL: define float @test_zero_plus_nan_rz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.add.rz.f(float 0.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.add.rz.f(float 0.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_zero_plus_nan_rm_ftz_f() {
+; CHECK-LABEL: define float @test_zero_plus_nan_rm_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.add.rm.ftz.f(float 0.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.add.rm.ftz.f(float 0.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_zero_plus_nan_rn_ftz_f() {
+; CHECK-LABEL: define float @test_zero_plus_nan_rn_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.add.rn.ftz.f(float 0.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.add.rn.ftz.f(float 0.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_zero_plus_nan_rp_ftz_f() {
+; CHECK-LABEL: define float @test_zero_plus_nan_rp_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.add.rp.ftz.f(float 0.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.add.rp.ftz.f(float 0.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_zero_plus_nan_rz_ftz_f() {
+; CHECK-LABEL: define float @test_zero_plus_nan_rz_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.add.rz.ftz.f(float 0.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.add.rz.ftz.f(float 0.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+;###############################################################
+;#                Add(Subnormal, Subnormal) -> Normal          #
+;###############################################################
+; Tests addition of two positive subnormal numbers (2^-127)
+; - Without FTZ: The result is the sum of the subnormals (2^-126) - a normal number
+; - With FTZ: The inputs are flushed to zero, so the result is zero (despite the output being normal)
+
+define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
+; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rm_d() {
+; CHECK-NEXT:    ret double 0x3810000000000000
+;
+  %res = call double @llvm.nvvm.add.rm.d(double 0x3800000000000000, double 0x3800000000000000)
+  ret double %res
+}
+
+define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
+; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rn_d() {
+; CHECK-NEXT:    ret double 0x3810000000000000
+;
+  %res = call double @llvm.nvvm.add.rn.d(double 0x3800000000000000, double 0x3800000000000000)
+  ret double %res
+}
+
+define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
+; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rp_d() {
+; CHECK-NEXT:    ret double 0x3810000000000000
+;
+  %res = call double @llvm.nvvm.add.rp.d(double 0x3800000000000000, double 0x3800000000000000)
+  ret double %res
+}
+
+define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
+; CHECK-LABEL: define double @test_subnorm_plus_subnorm_to_normal_rz_d() {
+; CHECK-NEXT:    ret double 0x3810000000000000
+;
+  %res = call double @llvm.nvvm.add.rz.d(double 0x3800000000000000, double 0x3800000000000000)
+  ret double %res
+}
+
+define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
+; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.add.rm.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
+; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.add.rn.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
+; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.add.rp.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
+; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.add.rz.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
+; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rm.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
+; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rn.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
+; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rp.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
+; CHECK-LABEL: define float @test_subnorm_plus_subnorm_to_normal_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rz.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                  Add(Normal, -Subnormal) -> Subnormal       #
+;###############################################################
+; Tests addition of 2^-126 (the smallest normal number) and -(2^127).
+; - Without FTZ: The result is correctly computed as a subnormal (2^127)
+; - With FTZ: The result is flushed to zero.
+; This verifies that the output is also flushed to zero, as we'd end up
+; with 2^-126 if we only flushed the inputs.
+
+define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
+; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rm_d() {
+; CHECK-NEXT:    ret double 0x3800000000000000
+;
+  %res = call double @llvm.nvvm.add.rm.d(double 0x3810000000000000, double 0xB800000000000000)
+  ret double %res
+}
+
+define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
+; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rn_d() {
+; CHECK-NEXT:    ret double 0x3800000000000000
+;
+  %res = call double @llvm.nvvm.add.rn.d(double 0x3810000000000000, double 0xB800000000000000)
+  ret double %res
+}
+
+define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
+; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rp_d() {
+; CHECK-NEXT:    ret double 0x3800000000000000
+;
+  %res = call double @llvm.nvvm.add.rp.d(double 0x3810000000000000, double 0xB800000000000000)
+  ret double %res
+}
+
+define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
+; CHECK-LABEL: define double @test_normal_minus_subnorm_to_subnorm_rz_d() {
+; CHECK-NEXT:    ret double 0x3800000000000000
+;
+  %res = call double @llvm.nvvm.add.rz.d(double 0x3810000000000000, double 0xB800000000000000)
+  ret double %res
+}
+
+define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
+; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.add.rm.f(float 0x3810000000000000, float 0xB800000000000000)
+  ret float %res
+}
+
+define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
+; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.add.rn.f(float 0x3810000000000000, float 0xB800000000000000)
+  ret float %res
+}
+
+define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
+; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.add.rp.f(float 0x3810000000000000, float 0xB800000000000000)
+  ret float %res
+}
+
+define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
+; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.add.rz.f(float 0x3810000000000000, float 0xB800000000000000)
+  ret float %res
+}
+
+define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.add.rm.ftz.f(float 0x3810000000000000, float 0xB800000000000000)
+  ret float %res
+}
+
+define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.add.rn.ftz.f(float 0x3810000000000000, float 0xB800000000000000)
+  ret float %res
+}
+
+define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rp.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_normal_minus_subnorm_to_subnorm_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rz.ftz.f(float 0x3800000000000000, float 0x3800000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Add(1.0, 2^(-25))                        #
+;###############################################################
+; Tests addition of 1.0 and 2^(-25) where the exact result falls between
+; 1.0 and 1.0 + 2^(-23):
+; - RN, RZ, RM: Return 1.0 (rounding toward nearest/zero/down)
+; - RP: Returns 1.0 + 2^(-23) (rounding up)
+
+define float @test_1_plus_ulp_rm_f() {
+; CHECK-LABEL: define float @test_1_plus_ulp_rm_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rm.f(float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_1_plus_ulp_rn_f() {
+; CHECK-LABEL: define float @test_1_plus_ulp_rn_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rn.f(float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_1_plus_ulp_rp_f() {
+; CHECK-LABEL: define float @test_1_plus_ulp_rp_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.add.rp.f(float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_1_plus_ulp_rz_f() {
+; CHECK-LABEL: define float @test_1_plus_ulp_rz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rz.f(float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_1_plus_ulp_rm_ftz_f() {
+; CHECK-LABEL: define float @test_1_plus_ulp_rm_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_1_plus_ulp_rn_ftz_f() {
+; CHECK-LABEL: define float @test_1_plus_ulp_rn_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_1_plus_ulp_rp_ftz_f() {
+; CHECK-LABEL: define float @test_1_plus_ulp_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_1_plus_ulp_rz_ftz_f() {
+; CHECK-LABEL: define float @test_1_plus_ulp_rz_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Add(1.0, 2^(-54))                        #
+;###############################################################
+; Tests addition of 1.0 and 2^(-54) where the exact result falls between
+; 1.0 and 1.0 + 2^(-52):
+; - RN, RZ, RM: Return 1.0 (rounding to nearest/zero/down)
+; - RP: Returns 1.0 + 2^(-52) (rounding up)
+
+define double @test_1_plus_ulp_rm_d() {
+; CHECK-LABEL: define double @test_1_plus_ulp_rm_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rm.d(double 1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_1_plus_ulp_rn_d() {
+; CHECK-LABEL: define double @test_1_plus_ulp_rn_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rn.d(double 1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_1_plus_ulp_rp_d() {
+; CHECK-LABEL: define double @test_1_plus_ulp_rp_d() {
+; CHECK-NEXT:    ret double 0x3FF0000000000001
+;
+  %res = call double @llvm.nvvm.add.rp.d(double 1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_1_plus_ulp_rz_d() {
+; CHECK-LABEL: define double @test_1_plus_ulp_rz_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rz.d(double 1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+;###############################################################
+;#                       Add(-1.0, 2^(-25))                    #
+;###############################################################
+; Tests addition of -1.0 and 2^(-25)) where the exact result falls between
+; -1.0 and -1.0 + 2^(-23):
+; - RN, RM: Returns -1.0 (rounding toward nearest/down)
+; - RZ, RP: Return -1.0 + 2^(-23) (rounding toward zero/up)
+
+define float @test_neg_1_plus_ulp_rm_f() {
+; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rm.f(float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_plus_ulp_rn_f() {
+; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rn.f(float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_plus_ulp_rp_f() {
+; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.add.rp.f(float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_plus_ulp_rz_f() {
+; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.add.rz.f(float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_plus_ulp_rm_ftz_f() {
+; CHECK-LABEL: define float @test_neg_1_plus_ulp_rm_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rm.ftz.f(float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_plus_ulp_rn_ftz_f() {
+; CHECK-LABEL: define float @test_neg_1_plus_ulp_rn_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rn.ftz.f(float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_plus_ulp_rp_ftz_f() {
+; CHECK-LABEL: define float @test_neg_1_plus_ulp_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.add.rp.ftz.f(float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_plus_ulp_rz_ftz_f() {
+; CHECK-LABEL: define float @test_neg_1_plus_ulp_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.add.rz.ftz.f(float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                       Add(-1.0, 2^(-54))                    #
+;###############################################################
+; Tests addition of -1.0 and 2^(-54) where the exact result falls between
+; -1.0 and -1.0 + 2^(-52):
+; - RN, RM: Return -1.0 (rounding toward nearest/down)
+; - RZ, RP: Return -1.0 + 2^(-52) (rounding toward zero/up)
+
+define double @test_neg_1_plus_ulp_rm_d() {
+; CHECK-LABEL: define double @test_neg_1_plus_ulp_rm_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rm.d(double -1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_neg_1_plus_ulp_rn_d() {
+; CHECK-LABEL: define double @test_neg_1_plus_ulp_rn_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rn.d(double -1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_neg_1_plus_ulp_rp_d() {
+; CHECK-LABEL: define double @test_neg_1_plus_ulp_rp_d() {
+; CHECK-NEXT:    ret double 0xBFEFFFFFFFFFFFFF
+;
+  %res = call double @llvm.nvvm.add.rp.d(double -1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_neg_1_plus_ulp_rz_d() {
+; CHECK-LABEL: define double @test_neg_1_plus_ulp_rz_d() {
+; CHECK-NEXT:    ret double 0xBFEFFFFFFFFFFFFF
+;
+  %res = call double @llvm.nvvm.add.rz.d(double -1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+;###############################################################
+;#                    Add(1.0, -2^(-25))                       #
+;###############################################################
+; Tests addition of 1.0 and -2^(-25) where the exact result falls between
+; 1.0 and 1.0 - 2^(-23):
+; - RN, RP: Return 1.0 (rounding toward nearest/up)
+; - RZ, RM: Return 1.0 - 2^(-23) (rounding toward zero/down)
+
+define float @test_1_minus_ulp_rm_f() {
+; CHECK-LABEL: define float @test_1_minus_ulp_rm_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.add.rm.f(float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_1_minus_ulp_rn_f() {
+; CHECK-LABEL: define float @test_1_minus_ulp_rn_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rn.f(float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_1_minus_ulp_rp_f() {
+; CHECK-LABEL: define float @test_1_minus_ulp_rp_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rp.f(float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_1_minus_ulp_rz_f() {
+; CHECK-LABEL: define float @test_1_minus_ulp_rz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.add.rz.f(float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_1_minus_ulp_rm_ftz_f() {
+; CHECK-LABEL: define float @test_1_minus_ulp_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.add.rm.ftz.f(float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_1_minus_ulp_rn_ftz_f() {
+; CHECK-LABEL: define float @test_1_minus_ulp_rn_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rn.ftz.f(float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_1_minus_ulp_rp_ftz_f() {
+; CHECK-LABEL: define float @test_1_minus_ulp_rp_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rp.ftz.f(float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_1_minus_ulp_rz_ftz_f() {
+; CHECK-LABEL: define float @test_1_minus_ulp_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.add.rz.ftz.f(float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Add(1.0, -2^(-54))                       #
+;###############################################################
+; Tests addition of 1.0 and -2^(-54) where the exact result falls between
+; 1.0 and 1.0 - 2^(-52):
+; - RN, RP: Return 1.0 (rounding toward nearest/up)
+; - RZ, RM: Return 1.0 - 2^(-52) (rounding toward zero/down)
+
+define double @test_1_minus_ulp_rm_d() {
+; CHECK-LABEL: define double @test_1_minus_ulp_rm_d() {
+; CHECK-NEXT:    ret double 0x3FEFFFFFFFFFFFFF
+;
+  %res = call double @llvm.nvvm.add.rm.d(double 1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_1_minus_ulp_rn_d() {
+; CHECK-LABEL: define double @test_1_minus_ulp_rn_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rn.d(double 1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_1_minus_ulp_rp_d() {
+; CHECK-LABEL: define double @test_1_minus_ulp_rp_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rp.d(double 1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_1_minus_ulp_rz_d() {
+; CHECK-LABEL: define double @test_1_minus_ulp_rz_d() {
+; CHECK-NEXT:    ret double 0x3FEFFFFFFFFFFFFF
+;
+  %res = call double @llvm.nvvm.add.rz.d(double 1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+;###############################################################
+;#                    Add(-1.0, -2^(-25))                      #
+;###############################################################
+; Tests addition of -1.0 and -2^(-25) where the exact result falls between
+; -1.0 and -1.0 - 2^(-23):
+; - RN, RZ, RP: Return -1.0 (rounding to nearest/zero/up)
+; - RM: Return -1.0 - 2^(-23) (rounding down)
+
+define float @test_neg_1_minus_ulp_rm_f() {
+; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.add.rm.f(float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_minus_ulp_rn_f() {
+; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rn.f(float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_minus_ulp_rp_f() {
+; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rp.f(float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_minus_ulp_rz_f() {
+; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rz.f(float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_minus_ulp_rm_ftz_f() {
+; CHECK-LABEL: define float @test_neg_1_minus_ulp_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.add.rm.ftz.f(float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_minus_ulp_rn_ftz_f() {
+; CHECK-LABEL: define float @test_neg_1_minus_ulp_rn_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rn.ftz.f(float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_minus_ulp_rp_ftz_f() {
+; CHECK-LABEL: define float @test_neg_1_minus_ulp_rp_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rp.ftz.f(float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_neg_1_minus_ulp_rz_ftz_f() {
+; CHECK-LABEL: define float @test_neg_1_minus_ulp_rz_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.add.rz.ftz.f(float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Add(-1.0, -2^(-54))                      #
+;###############################################################
+; Tests addition of -1.0 and -2^(-54) where the exact result falls between
+; -1.0 and -1.0 - 2^(-52):
+; - RN, RZ, RP: Return -1.0 (rounding to nearest/zero/up)
+; - RM: Return -1.0 - 2^(-52) (rounding down)
+
+define double @test_neg_1_minus_ulp_rm_d() {
+; CHECK-LABEL: define double @test_neg_1_minus_ulp_rm_d() {
+; CHECK-NEXT:    ret double 0xBFF0000000000001
+;
+  %res = call double @llvm.nvvm.add.rm.d(double -1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_neg_1_minus_ulp_rn_d() {
+; CHECK-LABEL: define double @test_neg_1_minus_ulp_rn_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rn.d(double -1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_neg_1_minus_ulp_rp_d() {
+; CHECK-LABEL: define double @test_neg_1_minus_ulp_rp_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rp.d(double -1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_neg_1_minus_ulp_rz_d() {
+; CHECK-LABEL: define double @test_neg_1_minus_ulp_rz_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.add.rz.d(double -1.0, double 0xBC90000000000000)
+  ret double %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-div.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-div.ll
new file mode 100644
index 0000000000000..fab674cd9069b
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-div.ll
@@ -0,0 +1,880 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instsimplify -march=nvptx64 --mcpu=sm_86 --mattr=+ptx72 -S | FileCheck %s
+
+; Check constant-folding for NVVM divide intrinsics with different rounding modes
+
+;###############################################################
+;#                    Div(1.25, 2.0)                           #
+;###############################################################
+; Tests division of two normal numbers (1.25 by 2.0) where the result
+; is exactly representable. All rounding modes should produce the same result.
+
+define double @test_1_25_div_2_rm_d() {
+; CHECK-LABEL: define double @test_1_25_div_2_rm_d() {
+; CHECK-NEXT:    ret double 6.250000e-01
+;
+  %res = call double @llvm.nvvm.div.rm.d(double 1.25, double 2.0)
+  ret double %res
+}
+
+define double @test_1_25_div_2_rn_d() {
+; CHECK-LABEL: define double @test_1_25_div_2_rn_d() {
+; CHECK-NEXT:    ret double 6.250000e-01
+;
+  %res = call double @llvm.nvvm.div.rn.d(double 1.25, double 2.0)
+  ret double %res
+}
+
+define double @test_1_25_div_2_rp_d() {
+; CHECK-LABEL: define double @test_1_25_div_2_rp_d() {
+; CHECK-NEXT:    ret double 6.250000e-01
+;
+  %res = call double @llvm.nvvm.div.rp.d(double 1.25, double 2.0)
+  ret double %res
+}
+
+define double @test_1_25_div_2_rz_d() {
+; CHECK-LABEL: define double @test_1_25_div_2_rz_d() {
+; CHECK-NEXT:    ret double 6.250000e-01
+;
+  %res = call double @llvm.nvvm.div.rz.d(double 1.25, double 2.0)
+  ret double %res
+}
+
+define float @test_1_25_div_2_rm_f() {
+; CHECK-LABEL: define float @test_1_25_div_2_rm_f() {
+; CHECK-NEXT:    ret float 6.250000e-01
+;
+  %res = call float @llvm.nvvm.div.rm.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_div_2_rn_f() {
+; CHECK-LABEL: define float @test_1_25_div_2_rn_f() {
+; CHECK-NEXT:    ret float 6.250000e-01
+;
+  %res = call float @llvm.nvvm.div.rn.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_div_2_rp_f() {
+; CHECK-LABEL: define float @test_1_25_div_2_rp_f() {
+; CHECK-NEXT:    ret float 6.250000e-01
+;
+  %res = call float @llvm.nvvm.div.rp.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_div_2_rz_f() {
+; CHECK-LABEL: define float @test_1_25_div_2_rz_f() {
+; CHECK-NEXT:    ret float 6.250000e-01
+;
+  %res = call float @llvm.nvvm.div.rz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_div_2_rm_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_div_2_rm_ftz_f() {
+; CHECK-NEXT:    ret float 6.250000e-01
+;
+  %res = call float @llvm.nvvm.div.rm.ftz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_div_2_rn_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_div_2_rn_ftz_f() {
+; CHECK-NEXT:    ret float 6.250000e-01
+;
+  %res = call float @llvm.nvvm.div.rn.ftz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_div_2_rp_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_div_2_rp_ftz_f() {
+; CHECK-NEXT:    ret float 6.250000e-01
+;
+  %res = call float @llvm.nvvm.div.rp.ftz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_div_2_rz_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_div_2_rz_ftz_f() {
+; CHECK-NEXT:    ret float 6.250000e-01
+;
+  %res = call float @llvm.nvvm.div.rz.ftz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+;###############################################################
+;#                    Div(Subnormal, 1.0)                      #
+;###############################################################
+; Tests division of a subnormal number by 1.0 to verify FTZ behavior.
+; For float, we use 2^-149 (smallest subnormal).
+; For double, we use 2^-1074 (smallest subnormal).
+; Without FTZ, the result should be the subnormal number.
+; With FTZ, the result should be 0.0.
+
+define double @test_subnorm_div_1_rm_d() {
+; CHECK-LABEL: define double @test_subnorm_div_1_rm_d() {
+; CHECK-NEXT:    ret double 4.940660e-324
+;
+  %res = call double @llvm.nvvm.div.rm.d(double 0x0000000000000001, double 1.0)
+  ret double %res
+}
+
+define double @test_subnorm_div_1_rn_d() {
+; CHECK-LABEL: define double @test_subnorm_div_1_rn_d() {
+; CHECK-NEXT:    ret double 4.940660e-324
+;
+  %res = call double @llvm.nvvm.div.rn.d(double 0x0000000000000001, double 1.0)
+  ret double %res
+}
+
+define double @test_subnorm_div_1_rp_d() {
+; CHECK-LABEL: define double @test_subnorm_div_1_rp_d() {
+; CHECK-NEXT:    ret double 4.940660e-324
+;
+  %res = call double @llvm.nvvm.div.rp.d(double 0x0000000000000001, double 1.0)
+  ret double %res
+}
+
+define double @test_subnorm_div_1_rz_d() {
+; CHECK-LABEL: define double @test_subnorm_div_1_rz_d() {
+; CHECK-NEXT:    ret double 4.940660e-324
+;
+  %res = call double @llvm.nvvm.div.rz.d(double 0x0000000000000001, double 1.0)
+  ret double %res
+}
+
+define float @test_subnorm_div_1_rm_f() {
+; CHECK-LABEL: define float @test_subnorm_div_1_rm_f() {
+; CHECK-NEXT:    ret float 0x36A0000000000000
+;
+  %res = call float @llvm.nvvm.div.rm.f(float 0x36A0000000000000, float 1.0)
+  ret float %res
+}
+
+define float @test_subnorm_div_1_rn_f() {
+; CHECK-LABEL: define float @test_subnorm_div_1_rn_f() {
+; CHECK-NEXT:    ret float 0x36A0000000000000
+;
+  %res = call float @llvm.nvvm.div.rn.f(float 0x36A0000000000000, float 1.0)
+  ret float %res
+}
+
+define float @test_subnorm_div_1_rp_f() {
+; CHECK-LABEL: define float @test_subnorm_div_1_rp_f() {
+; CHECK-NEXT:    ret float 0x36A0000000000000
+;
+  %res = call float @llvm.nvvm.div.rp.f(float 0x36A0000000000000, float 1.0)
+  ret float %res
+}
+
+define float @test_subnorm_div_1_rz_f() {
+; CHECK-LABEL: define float @test_subnorm_div_1_rz_f() {
+; CHECK-NEXT:    ret float 0x36A0000000000000
+;
+  %res = call float @llvm.nvvm.div.rz.f(float 0x36A0000000000000, float 1.0)
+  ret float %res
+}
+
+define float @test_subnorm_div_1_rm_ftz_f() {
+; CHECK-LABEL: define float @test_subnorm_div_1_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rm.ftz.f(float 0x36A0000000000000, float 1.0)
+  ret float %res
+}
+
+define float @test_subnorm_div_1_rn_ftz_f() {
+; CHECK-LABEL: define float @test_subnorm_div_1_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rn.ftz.f(float 0x36A0000000000000, float 1.0)
+  ret float %res
+}
+
+define float @test_subnorm_div_1_rp_ftz_f() {
+; CHECK-LABEL: define float @test_subnorm_div_1_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rp.ftz.f(float 0x36A0000000000000, float 1.0)
+  ret float %res
+}
+
+define float @test_subnorm_div_1_rz_ftz_f() {
+; CHECK-LABEL: define float @test_subnorm_div_1_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rz.ftz.f(float 0x36A0000000000000, float 1.0)
+  ret float %res
+}
+
+;###############################################################
+;#                    Div(Normal, Normal) -> Subnormal         #
+;###############################################################
+; Tests division of two normal numbers that produces a subnormal result.
+; We divide the smallest normal float (2^-126 or 2^-1022 for doubles) by 2 to get 2^-127 (or 2^-1023),
+; which is subnormal. This tests the transition from normal to subnormal numbers.
+
+define double @test_normal_div_normal_to_subnorm_rm_d() {
+; CHECK-LABEL: define double @test_normal_div_normal_to_subnorm_rm_d() {
+; CHECK-NEXT:    ret double 0x8000000000000
+;
+  %res = call double @llvm.nvvm.div.rm.d(double 0x10000000000000, double 2.0)
+  ret double %res
+}
+
+define double @test_normal_div_normal_to_subnorm_rn_d() {
+; CHECK-LABEL: define double @test_normal_div_normal_to_subnorm_rn_d() {
+; CHECK-NEXT:    ret double 0x8000000000000
+;
+  %res = call double @llvm.nvvm.div.rn.d(double 0x10000000000000, double 2.0)
+  ret double %res
+}
+
+define double @test_normal_div_normal_to_subnorm_rp_d() {
+; CHECK-LABEL: define double @test_normal_div_normal_to_subnorm_rp_d() {
+; CHECK-NEXT:    ret double 0x8000000000000
+;
+  %res = call double @llvm.nvvm.div.rp.d(double 0x10000000000000, double 2.0)
+  ret double %res
+}
+
+define double @test_normal_div_normal_to_subnorm_rz_d() {
+; CHECK-LABEL: define double @test_normal_div_normal_to_subnorm_rz_d() {
+; CHECK-NEXT:    ret double 0x8000000000000
+;
+  %res = call double @llvm.nvvm.div.rz.d(double 0x10000000000000, double 2.0)
+  ret double %res
+}
+
+define float @test_normal_div_normal_to_subnorm_rm_f() {
+; CHECK-LABEL: define float @test_normal_div_normal_to_subnorm_rm_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.div.rm.f(float 0x3810000000000000, float 2.0)
+  ret float %res
+}
+
+define float @test_normal_div_normal_to_subnorm_rn_f() {
+; CHECK-LABEL: define float @test_normal_div_normal_to_subnorm_rn_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.div.rn.f(float 0x3810000000000000, float 2.0)
+  ret float %res
+}
+
+define float @test_normal_div_normal_to_subnorm_rp_f() {
+; CHECK-LABEL: define float @test_normal_div_normal_to_subnorm_rp_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.div.rp.f(float 0x3810000000000000, float 2.0)
+  ret float %res
+}
+
+define float @test_normal_div_normal_to_subnorm_rz_f() {
+; CHECK-LABEL: define float @test_normal_div_normal_to_subnorm_rz_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.div.rz.f(float 0x3810000000000000, float 2.0)
+  ret float %res
+}
+
+define float @test_normal_div_normal_to_subnorm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_normal_div_normal_to_subnorm_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rm.ftz.f(float 0x3810000000000000, float 2.0)
+  ret float %res
+}
+
+define float @test_normal_div_normal_to_subnorm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_normal_div_normal_to_subnorm_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rn.ftz.f(float 0x3810000000000000, float 2.0)
+  ret float %res
+}
+
+define float @test_normal_div_normal_to_subnorm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_normal_div_normal_to_subnorm_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rp.ftz.f(float 0x3810000000000000, float 2.0)
+  ret float %res
+}
+
+define float @test_normal_div_normal_to_subnorm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_normal_div_normal_to_subnorm_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rz.ftz.f(float 0x3810000000000000, float 2.0)
+  ret float %res
+}
+
+;###############################################################
+;#              Div( 4/3 + epsilon , 4/3 - epsilon)            #
+;###############################################################
+; Tests division of numbers just above and just below 4/3.
+; The result falls between 1.0 and 1.0 + 2^-23
+; - RZ, RM round to 1.0 (rounding towards zero/down)
+; - RN, RP rounds to 1.0 + 2^-23 (rounding towards nearest/up)
+
+define float @test_div_just_above_1_rm_f() {
+; CHECK-LABEL: define float @test_div_just_above_1_rm_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rm.f(float 0x3FF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_above_1_rn_f() {
+; CHECK-LABEL: define float @test_div_just_above_1_rn_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.div.rn.f(float 0x3FF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_above_1_rp_f() {
+; CHECK-LABEL: define float @test_div_just_above_1_rp_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.div.rp.f(float 0x3FF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_above_1_rz_f() {
+; CHECK-LABEL: define float @test_div_just_above_1_rz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rz.f(float 0x3FF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_above_1_rm_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_above_1_rm_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rm.ftz.f(float 0x3FF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_above_1_rn_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_above_1_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.div.rn.ftz.f(float 0x3FF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_above_1_rp_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_above_1_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.div.rp.ftz.f(float 0x3FF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_above_1_rz_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_above_1_rz_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rz.ftz.f(float 0x3FF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+;###############################################################
+;#               Div(4/3 + epsilon / 4/3 - epsilon)            #
+;###############################################################
+; Tests division of numbers just above and just below 4/3.
+; The result falls between 1.0 and 1.0 + 2^-52
+; - RZ, RM round to 1.0 (rounding towards zero/down)
+; - RN, RP rounds to 1.0 + 2^-23 (rounding towards nearest/up)
+
+define double @test_div_just_above_1_rm_d() {
+; CHECK-LABEL: define double @test_div_just_above_1_rm_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.div.rm.d(double 0x3FF5555555555555, double 0x3FF5555555555554)
+  ret double %res
+}
+
+define double @test_div_just_above_1_rn_d() {
+; CHECK-LABEL: define double @test_div_just_above_1_rn_d() {
+; CHECK-NEXT:    ret double 0x3FF0000000000001
+;
+  %res = call double @llvm.nvvm.div.rn.d(double 0x3FF5555555555555, double 0x3FF5555555555554)
+  ret double %res
+}
+
+define double @test_div_just_above_1_rp_d() {
+; CHECK-LABEL: define double @test_div_just_above_1_rp_d() {
+; CHECK-NEXT:    ret double 0x3FF0000000000001
+;
+  %res = call double @llvm.nvvm.div.rp.d(double 0x3FF5555555555555, double 0x3FF5555555555554)
+  ret double %res
+}
+
+define double @test_div_just_above_1_rz_d() {
+; CHECK-LABEL: define double @test_div_just_above_1_rz_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.div.rz.d(double 0x3FF5555555555555, double 0x3FF5555555555554)
+  ret double %res
+}
+
+;###############################################################
+;#            Div( -(4/3 + epsilon),  4/3 - epsilon  )         #
+;###############################################################
+; Tests division of numbers just below -4/3 and just below 4/3.
+; The result falls between -1.0 and -1.0 - 2^-52
+; - RZ, RP round to -1.0 (rounding towards zero/up)
+; - RN, RM rounds to -1.0 - 2^-52 (rounding towards nearest/down)
+
+define float @test_div_just_below_negative_1_rm_f() {
+; CHECK-LABEL: define float @test_div_just_below_negative_1_rm_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.div.rm.f(float 0xBFF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_below_negative_1_rn_f() {
+; CHECK-LABEL: define float @test_div_just_below_negative_1_rn_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.div.rn.f(float 0xBFF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_below_negative_1_rp_f() {
+; CHECK-LABEL: define float @test_div_just_below_negative_1_rp_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rp.f(float 0xBFF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_below_negative_1_rz_f() {
+; CHECK-LABEL: define float @test_div_just_below_negative_1_rz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rz.f(float 0xBFF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_below_negative_1_rm_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_below_negative_1_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.div.rm.ftz.f(float 0xBFF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_below_negative_1_rn_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_below_negative_1_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.div.rn.ftz.f(float 0xBFF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_below_negative_1_rp_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_below_negative_1_rp_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rp.ftz.f(float 0xBFF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+define float @test_div_just_below_negative_1_rz_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_below_negative_1_rz_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.div.rz.ftz.f(float 0xBFF5555560000000, float 0x3FF5555540000000)
+  ret float %res
+}
+
+;###############################################################
+;#            Div( -(4/3 + epsilon),  4/3 - epsilon  )         #
+;###############################################################
+; Tests division of numbers just below -4/3 and just below 4/3.
+; The result falls between -1.0 and -1.0 - 2^-52
+; - RZ, RP round to -1.0 (rounding towards zero/up)
+; - RN, RM rounds to -1.0 - 2^-52 (rounding towards nearest/down)
+
+define double @test_div_just_below_negative_1_rm_d() {
+; CHECK-LABEL: define double @test_div_just_below_negative_1_rm_d() {
+; CHECK-NEXT:    ret double 0xBFF0000000000001
+;
+  %res = call double @llvm.nvvm.div.rm.d(double 0xBFF5555555555555, double 0x3FF5555555555554)
+  ret double %res
+}
+
+define double @test_div_just_below_negative_1_rn_d() {
+; CHECK-LABEL: define double @test_div_just_below_negative_1_rn_d() {
+; CHECK-NEXT:    ret double 0xBFF0000000000001
+;
+  %res = call double @llvm.nvvm.div.rn.d(double 0xBFF5555555555555, double 0x3FF5555555555554)
+  ret double %res
+}
+
+define double @test_div_just_below_negative_1_rp_d() {
+; CHECK-LABEL: define double @test_div_just_below_negative_1_rp_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.div.rp.d(double 0xBFF5555555555555, double 0x3FF5555555555554)
+  ret double %res
+}
+
+define double @test_div_just_below_negative_1_rz_d() {
+; CHECK-LABEL: define double @test_div_just_below_negative_1_rz_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.div.rz.d(double 0xBFF5555555555555, double 0x3FF5555555555554)
+  ret double %res
+}
+
+;###############################################################
+;#                    Div(~4/3 , ~4/3 + epsilon)               #
+;###############################################################
+; Tests division of ~4/3 by a value just over 4/3
+; The exact result falls between 1.0 - 2^23 ( = 0x3FEFFFFFC0000000)
+; and 1.0 - (2^23 + 2^24)  ( = 0x3FEFFFFFA0000000).
+; - RN, RZ, RM round to 1.0 - 2^-23 - 2^-24 (rounding towards nearest/zero/down)
+; - RP rounds to 1.0 - 2^-23 (rounding up)
+
+define float @test_div_just_below_1_rm_f() {
+; CHECK-LABEL: define float @test_div_just_below_1_rm_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rm.f(float 0x3FF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_below_1_rn_f() {
+; CHECK-LABEL: define float @test_div_just_below_1_rn_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rn.f(float 0x3FF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_below_1_rp_f() {
+; CHECK-LABEL: define float @test_div_just_below_1_rp_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFC0000000
+;
+  %res = call float @llvm.nvvm.div.rp.f(float 0x3FF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_below_1_rz_f() {
+; CHECK-LABEL: define float @test_div_just_below_1_rz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rz.f(float 0x3FF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_below_1_rm_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_below_1_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rm.ftz.f(float 0x3FF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_below_1_rn_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_below_1_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rn.ftz.f(float 0x3FF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_below_1_rp_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_below_1_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFC0000000
+;
+  %res = call float @llvm.nvvm.div.rp.ftz.f(float 0x3FF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_below_1_rz_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_below_1_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rz.ftz.f(float 0x3FF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Div(~4/3 , ~4/3 + epsilon)               #
+;###############################################################
+; Tests division of ~4/3 by a value just over 4/3
+; The exact result falls between 1.0 - 2^-51 - 2^-52  ( = 0x3FEFFFFFFFFFFFFA)
+; and 1.0 - 2^-51 - 2^-53  ( = 0x3FEFFFFFFFFFFFFB).
+; - RN, RZ, RM round to 1.0 - 2^-51 - 2^-52 (rounding towards nearest/zero/down)
+; - RP rounds to 1.0 - 2^-51 - 2^-53 (rounding up)
+
+define double @test_div_just_below_1_rm_d() {
+; CHECK-LABEL: define double @test_div_just_below_1_rm_d() {
+; CHECK-NEXT:    ret double 0x3FEFFFFFFFFFFFFA
+;
+  %res = call double @llvm.nvvm.div.rm.d(double 0x3FF5555555555554, double 0x3FF5555555555558)
+  ret double %res
+}
+
+define double @test_div_just_below_1_rn_d() {
+; CHECK-LABEL: define double @test_div_just_below_1_rn_d() {
+; CHECK-NEXT:    ret double 0x3FEFFFFFFFFFFFFA
+;
+  %res = call double @llvm.nvvm.div.rn.d(double 0x3FF5555555555554, double 0x3FF5555555555558)
+  ret double %res
+}
+
+define double @test_div_just_below_1_rp_d() {
+; CHECK-LABEL: define double @test_div_just_below_1_rp_d() {
+; CHECK-NEXT:    ret double 0x3FEFFFFFFFFFFFFB
+;
+  %res = call double @llvm.nvvm.div.rp.d(double 0x3FF5555555555554, double 0x3FF5555555555558)
+  ret double %res
+}
+
+define double @test_div_just_below_1_rz_d() {
+; CHECK-LABEL: define double @test_div_just_below_1_rz_d() {
+; CHECK-NEXT:    ret double 0x3FEFFFFFFFFFFFFA
+;
+  %res = call double @llvm.nvvm.div.rz.d(double 0x3FF5555555555554, double 0x3FF5555555555558)
+  ret double %res
+}
+
+;###############################################################
+;#                   Div(-4/3, ~4/3 + epsilon)                 #
+;###############################################################
+; Tests division of ~4/3 by a value just over 4/3
+; The exact result falls between 1.0 - 2^23 ( = 0x3FEFFFFFC0000000)
+; and 1.0 - (2^23 + 2^24)  ( = 0x3FEFFFFFA0000000).
+; - RN, RZ, RP round to -1.0 + 2^-23 + + 2^-24 (rounding towards nearest/zero/up)
+; - RM rounds to -1.0 + 2^-23 (rounding up)
+
+define float @test_div_just_above_negative_1_rm_f() {
+; CHECK-LABEL: define float @test_div_just_above_negative_1_rm_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFC0000000
+;
+  %res = call float @llvm.nvvm.div.rm.f(float 0xBFF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_above_negative_1_rn_f() {
+; CHECK-LABEL: define float @test_div_just_above_negative_1_rn_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rn.f(float 0xBFF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_above_negative_1_rp_f() {
+; CHECK-LABEL: define float @test_div_just_above_negative_1_rp_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rp.f(float 0xBFF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_above_negative_1_rz_f() {
+; CHECK-LABEL: define float @test_div_just_above_negative_1_rz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rz.f(float 0xBFF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_above_negative_1_rm_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_above_negative_1_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFC0000000
+;
+  %res = call float @llvm.nvvm.div.rm.ftz.f(float 0xBFF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_above_negative_1_rn_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_above_negative_1_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rn.ftz.f(float 0xBFF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_above_negative_1_rp_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_above_negative_1_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rp.ftz.f(float 0xBFF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+define float @test_div_just_above_negative_1_rz_ftz_f() {
+; CHECK-LABEL: define float @test_div_just_above_negative_1_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFA0000000
+;
+  %res = call float @llvm.nvvm.div.rz.ftz.f(float 0xBFF5555540000000, float 0x3FF5555580000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Div(~4/3 , ~4/3 + epsilon)               #
+;###############################################################
+; Tests division of ~4/3 by a value just over 4/3
+; The exact result falls between -1.0 + 2^-51 + 2^-52  ( = 0x3FEFFFFFFFFFFFFA)
+; and -1.0 + 2^-51 + 2^-53  ( = 0x3FEFFFFFFFFFFFFB).
+; - RN, RZ, RP round to -1.0 + 2^-51 + 2^-52 (rounding towards nearest/zero/up)
+; - RM rounds to -1.0 + 2^-51 + 2^-53 (rounding down)
+
+define double @test_div_just_above_negative_1_rm_d() {
+; CHECK-LABEL: define double @test_div_just_above_negative_1_rm_d() {
+; CHECK-NEXT:    ret double 0xBFEFFFFFFFFFFFFB
+;
+  %res = call double @llvm.nvvm.div.rm.d(double 0xBFF5555555555554, double 0x3FF5555555555558)
+  ret double %res
+}
+
+define double @test_div_just_above_negative_1_rn_d() {
+; CHECK-LABEL: define double @test_div_just_above_negative_1_rn_d() {
+; CHECK-NEXT:    ret double 0xBFEFFFFFFFFFFFFA
+;
+  %res = call double @llvm.nvvm.div.rn.d(double 0xBFF5555555555554, double 0x3FF5555555555558)
+  ret double %res
+}
+
+define double @test_div_just_above_negative_1_rp_d() {
+; CHECK-LABEL: define double @test_div_just_above_negative_1_rp_d() {
+; CHECK-NEXT:    ret double 0xBFEFFFFFFFFFFFFA
+;
+  %res = call double @llvm.nvvm.div.rp.d(double 0xBFF5555555555554, double 0x3FF5555555555558)
+  ret double %res
+}
+
+define double @test_div_just_above_negative_1_rz_d() {
+; CHECK-LABEL: define double @test_div_just_above_negative_1_rz_d() {
+; CHECK-NEXT:    ret double 0xBFEFFFFFFFFFFFFA
+;
+  %res = call double @llvm.nvvm.div.rz.d(double 0xBFF5555555555554, double 0x3FF5555555555558)
+  ret double %res
+}
+
+;###############################################################
+;#                    Div(NaN, NaN)                           #
+;###############################################################
+; Tests division of NaN by NaN to verify that constant folding is not performed
+; when the result would be NaN.
+
+define float @test_nan_div_nan_rm_f() {
+; CHECK-LABEL: define float @test_nan_div_nan_rm_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.div.rm.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.div.rm.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+  ret float %res
+}
+
+define float @test_nan_div_nan_rn_f() {
+; CHECK-LABEL: define float @test_nan_div_nan_rn_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.div.rn.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.div.rn.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+  ret float %res
+}
+
+define float @test_nan_div_nan_rp_f() {
+; CHECK-LABEL: define float @test_nan_div_nan_rp_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.div.rp.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.div.rp.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+  ret float %res
+}
+
+define float @test_nan_div_nan_rz_f() {
+; CHECK-LABEL: define float @test_nan_div_nan_rz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.div.rz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.div.rz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+  ret float %res
+}
+
+define float @test_nan_div_nan_rm_ftz_f() {
+; CHECK-LABEL: define float @test_nan_div_nan_rm_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.div.rm.ftz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.div.rm.ftz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+  ret float %res
+}
+
+define float @test_nan_div_nan_rn_ftz_f() {
+; CHECK-LABEL: define float @test_nan_div_nan_rn_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.div.rn.ftz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.div.rn.ftz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+  ret float %res
+}
+
+define float @test_nan_div_nan_rp_ftz_f() {
+; CHECK-LABEL: define float @test_nan_div_nan_rp_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.div.rp.ftz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.div.rp.ftz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+  ret float %res
+}
+
+define float @test_nan_div_nan_rz_ftz_f() {
+; CHECK-LABEL: define float @test_nan_div_nan_rz_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.div.rz.ftz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.div.rz.ftz.f(float 0x7FFFFFFF00000000, float 0x7FFFFFFF00000000)
+  ret float %res
+}
+
+define double @test_nan_div_nan_rm_d() {
+; CHECK-LABEL: define double @test_nan_div_nan_rm_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.div.rm.d(double 0x7FFFFFFFFFFFFFFF, double 0x7FFFFFFFFFFFFFFF)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.div.rm.d(double 0x7FFFFFFFFFFFFFFF, double 0x7FFFFFFFFFFFFFFF)
+  ret double %res
+}
+
+define double @test_nan_div_nan_rn_d() {
+; CHECK-LABEL: define double @test_nan_div_nan_rn_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.div.rn.d(double 0x7FFFFFFFFFFFFFFF, double 0x7FFFFFFFFFFFFFFF)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.div.rn.d(double 0x7FFFFFFFFFFFFFFF, double 0x7FFFFFFFFFFFFFFF)
+  ret double %res
+}
+
+define double @test_nan_div_nan_rp_d() {
+; CHECK-LABEL: define double @test_nan_div_nan_rp_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.div.rp.d(double 0x7FFFFFFFFFFFFFFF, double 0x7FFFFFFFFFFFFFFF)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.div.rp.d(double 0x7FFFFFFFFFFFFFFF, double 0x7FFFFFFFFFFFFFFF)
+  ret double %res
+}
+
+define double @test_nan_div_nan_rz_d() {
+; CHECK-LABEL: define double @test_nan_div_nan_rz_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.div.rz.d(double 0x7FFFFFFFFFFFFFFF, double 0x7FFFFFFFFFFFFFFF)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.div.rz.d(double 0x7FFFFFFFFFFFFFFF, double 0x7FFFFFFFFFFFFFFF)
+  ret double %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-fma.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-fma.ll
new file mode 100644
index 0000000000000..d52ff1ce66440
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-fma.ll
@@ -0,0 +1,874 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instsimplify -march=nvptx64 --mcpu=sm_86 --mattr=+ptx72 -S | FileCheck %s
+
+; Check constant-folding for NVVM FMA intrinsics with different rounding modes
+
+;###############################################################
+;#                    FMA(2.0, 3.0, 4.0)                       #
+;###############################################################
+; Tests FMA with regular numbers that produce a precise result
+
+define double @test_fma_2_3_4_rm_d() {
+; CHECK-LABEL: define double @test_fma_2_3_4_rm_d() {
+; CHECK-NEXT:    ret double 1.000000e+01
+;
+  %res = call double @llvm.nvvm.fma.rm.d(double 2.0, double 3.0, double 4.0)
+  ret double %res
+}
+
+define double @test_fma_2_3_4_rn_d() {
+; CHECK-LABEL: define double @test_fma_2_3_4_rn_d() {
+; CHECK-NEXT:    ret double 1.000000e+01
+;
+  %res = call double @llvm.nvvm.fma.rn.d(double 2.0, double 3.0, double 4.0)
+  ret double %res
+}
+
+define double @test_fma_2_3_4_rp_d() {
+; CHECK-LABEL: define double @test_fma_2_3_4_rp_d() {
+; CHECK-NEXT:    ret double 1.000000e+01
+;
+  %res = call double @llvm.nvvm.fma.rp.d(double 2.0, double 3.0, double 4.0)
+  ret double %res
+}
+
+define double @test_fma_2_3_4_rz_d() {
+; CHECK-LABEL: define double @test_fma_2_3_4_rz_d() {
+; CHECK-NEXT:    ret double 1.000000e+01
+;
+  %res = call double @llvm.nvvm.fma.rz.d(double 2.0, double 3.0, double 4.0)
+  ret double %res
+}
+
+define float @test_fma_2_3_4_rm_f() {
+; CHECK-LABEL: define float @test_fma_2_3_4_rm_f() {
+; CHECK-NEXT:    ret float 1.000000e+01
+;
+  %res = call float @llvm.nvvm.fma.rm.f(float 2.0, float 3.0, float 4.0)
+  ret float %res
+}
+
+define float @test_fma_2_3_4_rn_f() {
+; CHECK-LABEL: define float @test_fma_2_3_4_rn_f() {
+; CHECK-NEXT:    ret float 1.000000e+01
+;
+  %res = call float @llvm.nvvm.fma.rn.f(float 2.0, float 3.0, float 4.0)
+  ret float %res
+}
+
+define float @test_fma_2_3_4_rp_f() {
+; CHECK-LABEL: define float @test_fma_2_3_4_rp_f() {
+; CHECK-NEXT:    ret float 1.000000e+01
+;
+  %res = call float @llvm.nvvm.fma.rp.f(float 2.0, float 3.0, float 4.0)
+  ret float %res
+}
+
+define float @test_fma_2_3_4_rz_f() {
+; CHECK-LABEL: define float @test_fma_2_3_4_rz_f() {
+; CHECK-NEXT:    ret float 1.000000e+01
+;
+  %res = call float @llvm.nvvm.fma.rz.f(float 2.0, float 3.0, float 4.0)
+  ret float %res
+}
+
+define float @test_fma_2_3_4_rm_ftz_f() {
+; CHECK-LABEL: define float @test_fma_2_3_4_rm_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+01
+;
+  %res = call float @llvm.nvvm.fma.rm.ftz.f(float 2.0, float 3.0, float 4.0)
+  ret float %res
+}
+
+define float @test_fma_2_3_4_rn_ftz_f() {
+; CHECK-LABEL: define float @test_fma_2_3_4_rn_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+01
+;
+  %res = call float @llvm.nvvm.fma.rn.ftz.f(float 2.0, float 3.0, float 4.0)
+  ret float %res
+}
+
+define float @test_fma_2_3_4_rp_ftz_f() {
+; CHECK-LABEL: define float @test_fma_2_3_4_rp_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+01
+;
+  %res = call float @llvm.nvvm.fma.rp.ftz.f(float 2.0, float 3.0, float 4.0)
+  ret float %res
+}
+
+define float @test_fma_2_3_4_rz_ftz_f() {
+; CHECK-LABEL: define float @test_fma_2_3_4_rz_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+01
+;
+  %res = call float @llvm.nvvm.fma.rz.ftz.f(float 2.0, float 3.0, float 4.0)
+  ret float %res
+}
+
+;###############################################################
+;#                    FMA(Subnormal, 2.0, 0.0) -> Normal       #
+;###############################################################
+; Tests FMA with subnormal multiplier that produces a normal result
+; For float: 2^-127 * 2.0 = 2^-126 (smallest normal)
+; For double: 2^-1023 * 2.0 = 2^-1022 (smallest normal)
+; The FTZ variants should return 0.0, as they flush their input to zero.
+
+define double @test_fma_subnorm_to_norm_rm_d() {
+; CHECK-LABEL: define double @test_fma_subnorm_to_norm_rm_d() {
+; CHECK-NEXT:    ret double 0x10000000000000
+;
+  %res = call double @llvm.nvvm.fma.rm.d(double 0x0008000000000000, double 2.0, double 0.0)
+  ret double %res
+}
+
+define double @test_fma_subnorm_to_norm_rn_d() {
+; CHECK-LABEL: define double @test_fma_subnorm_to_norm_rn_d() {
+; CHECK-NEXT:    ret double 0x10000000000000
+;
+  %res = call double @llvm.nvvm.fma.rn.d(double 0x0008000000000000, double 2.0, double 0.0)
+  ret double %res
+}
+
+define double @test_fma_subnorm_to_norm_rp_d() {
+; CHECK-LABEL: define double @test_fma_subnorm_to_norm_rp_d() {
+; CHECK-NEXT:    ret double 0x10000000000000
+;
+  %res = call double @llvm.nvvm.fma.rp.d(double 0x0008000000000000, double 2.0, double 0.0)
+  ret double %res
+}
+
+define double @test_fma_subnorm_to_norm_rz_d() {
+; CHECK-LABEL: define double @test_fma_subnorm_to_norm_rz_d() {
+; CHECK-NEXT:    ret double 0x10000000000000
+;
+  %res = call double @llvm.nvvm.fma.rz.d(double 0x0008000000000000, double 2.0, double 0.0)
+  ret double %res
+}
+
+define float @test_fma_subnorm_to_norm_rm_f() {
+; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rm_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.fma.rm.f(float 0x3800000000000000, float 2.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_subnorm_to_norm_rn_f() {
+; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rn_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.fma.rn.f(float 0x3800000000000000, float 2.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_subnorm_to_norm_rp_f() {
+; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rp_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.fma.rp.f(float 0x3800000000000000, float 2.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_subnorm_to_norm_rz_f() {
+; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rz_f() {
+; CHECK-NEXT:    ret float 0x3810000000000000
+;
+  %res = call float @llvm.nvvm.fma.rz.f(float 0x3800000000000000, float 2.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_subnorm_to_norm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rm.ftz.f(float 0x3800000000000000, float 2.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_subnorm_to_norm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.ftz.f(float 0x3800000000000000, float 2.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_subnorm_to_norm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rp.ftz.f(float 0x3800000000000000, float 2.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_subnorm_to_norm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_fma_subnorm_to_norm_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rz.ftz.f(float 0x3800000000000000, float 2.0, float 0.0)
+  ret float %res
+}
+
+;###############################################################
+;#                    FMA(Normal, 0.5, 0.0) -> Subnormal       #
+;###############################################################
+; Tests FMA with normal inputs that produce a subnormal result
+; For float: 2^-126 * 0.5 = 2^-127 (subnormal)
+; For double: 2^-1022 * 0.5 = 2^-1023 (subnormal)
+; With FTZ mode, the subnormal output should be flushed to zero.
+
+define double @test_fma_norm_to_subnorm_rm_d() {
+; CHECK-LABEL: define double @test_fma_norm_to_subnorm_rm_d() {
+; CHECK-NEXT:    ret double 0x8000000000000
+;
+  %res = call double @llvm.nvvm.fma.rm.d(double 0x10000000000000, double 0.5, double 0.0)
+  ret double %res
+}
+
+define double @test_fma_norm_to_subnorm_rn_d() {
+; CHECK-LABEL: define double @test_fma_norm_to_subnorm_rn_d() {
+; CHECK-NEXT:    ret double 0x8000000000000
+;
+  %res = call double @llvm.nvvm.fma.rn.d(double 0x10000000000000, double 0.5, double 0.0)
+  ret double %res
+}
+
+define double @test_fma_norm_to_subnorm_rp_d() {
+; CHECK-LABEL: define double @test_fma_norm_to_subnorm_rp_d() {
+; CHECK-NEXT:    ret double 0x8000000000000
+;
+  %res = call double @llvm.nvvm.fma.rp.d(double 0x10000000000000, double 0.5, double 0.0)
+  ret double %res
+}
+
+define double @test_fma_norm_to_subnorm_rz_d() {
+; CHECK-LABEL: define double @test_fma_norm_to_subnorm_rz_d() {
+; CHECK-NEXT:    ret double 0x8000000000000
+;
+  %res = call double @llvm.nvvm.fma.rz.d(double 0x10000000000000, double 0.5, double 0.0)
+  ret double %res
+}
+
+define float @test_fma_norm_to_subnorm_rm_f() {
+; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rm_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.fma.rm.f(float 0x3810000000000000, float 0.5, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_norm_to_subnorm_rn_f() {
+; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rn_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.fma.rn.f(float 0x3810000000000000, float 0.5, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_norm_to_subnorm_rp_f() {
+; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rp_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.fma.rp.f(float 0x3810000000000000, float 0.5, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_norm_to_subnorm_rz_f() {
+; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rz_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.fma.rz.f(float 0x3810000000000000, float 0.5, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_norm_to_subnorm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rm.ftz.f(float 0x3810000000000000, float 0.5, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_norm_to_subnorm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.ftz.f(float 0x3810000000000000, float 0.5, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_norm_to_subnorm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rp.ftz.f(float 0x3810000000000000, float 0.5, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_norm_to_subnorm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_fma_norm_to_subnorm_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rz.ftz.f(float 0x3810000000000000, float 0.5, float 0.0)
+  ret float %res
+}
+
+;###############################################################
+;#                    FMA(NaN, 1.0, 0.0)                       #
+;###############################################################
+; Tests FMA with NaN input to verify that the instruction is preserved
+; since the result would be NaN. The instruction should not be folded.
+
+define double @test_fma_nan_rm_d() {
+; CHECK-LABEL: define double @test_fma_nan_rm_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.fma.rm.d(double 0x7FF8000000000000, double 1.000000e+00, double 0.000000e+00)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.fma.rm.d(double 0x7FF8000000000000, double 1.0, double 0.0)
+  ret double %res
+}
+
+define double @test_fma_nan_rn_d() {
+; CHECK-LABEL: define double @test_fma_nan_rn_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.fma.rn.d(double 0x7FF8000000000000, double 1.000000e+00, double 0.000000e+00)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.fma.rn.d(double 0x7FF8000000000000, double 1.0, double 0.0)
+  ret double %res
+}
+
+define double @test_fma_nan_rp_d() {
+; CHECK-LABEL: define double @test_fma_nan_rp_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.fma.rp.d(double 0x7FF8000000000000, double 1.000000e+00, double 0.000000e+00)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.fma.rp.d(double 0x7FF8000000000000, double 1.0, double 0.0)
+  ret double %res
+}
+
+define double @test_fma_nan_rz_d() {
+; CHECK-LABEL: define double @test_fma_nan_rz_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.fma.rz.d(double 0x7FF8000000000000, double 1.000000e+00, double 0.000000e+00)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.fma.rz.d(double 0x7FF8000000000000, double 1.0, double 0.0)
+  ret double %res
+}
+
+define float @test_fma_nan_rm_f() {
+; CHECK-LABEL: define float @test_fma_nan_rm_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.fma.rm.f(float 0x7FFC000000000000, float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.fma.rm.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_nan_rn_f() {
+; CHECK-LABEL: define float @test_fma_nan_rn_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.fma.rn.f(float 0x7FFC000000000000, float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.fma.rn.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_nan_rp_f() {
+; CHECK-LABEL: define float @test_fma_nan_rp_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.fma.rp.f(float 0x7FFC000000000000, float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.fma.rp.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_nan_rz_f() {
+; CHECK-LABEL: define float @test_fma_nan_rz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.fma.rz.f(float 0x7FFC000000000000, float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.fma.rz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_nan_rm_ftz_f() {
+; CHECK-LABEL: define float @test_fma_nan_rm_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.fma.rm.ftz.f(float 0x7FFC000000000000, float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.fma.rm.ftz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_nan_rn_ftz_f() {
+; CHECK-LABEL: define float @test_fma_nan_rn_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.fma.rn.ftz.f(float 0x7FFC000000000000, float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.fma.rn.ftz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_nan_rp_ftz_f() {
+; CHECK-LABEL: define float @test_fma_nan_rp_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.fma.rp.ftz.f(float 0x7FFC000000000000, float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.fma.rp.ftz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+  ret float %res
+}
+
+define float @test_fma_nan_rz_ftz_f() {
+; CHECK-LABEL: define float @test_fma_nan_rz_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.fma.rz.ftz.f(float 0x7FFC000000000000, float 1.000000e+00, float 0.000000e+00)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.fma.rz.ftz.f(float 0x7FFC000000000000, float 1.0, float 0.0)
+  ret float %res
+}
+
+;###############################################################
+;#                    FMA(1.0, 1.0, 2^(-25))                   #
+;###############################################################
+; Tests FMA with 1.0 and 2^(-25) where different rounding modes produce different results.
+; The exact result falls between 1.0 and 1.0 + 2^(-23).
+; RM, RN, and RZ return 1.0, while RP returns 1.0 + 2^(-23) (0x3F800001).
+
+define float @test_fma_1_plus_ulp_rm_f() {
+; CHECK-LABEL: define float @test_fma_1_plus_ulp_rm_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rm.f(float 1.0, float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_plus_ulp_rn_f() {
+; CHECK-LABEL: define float @test_fma_1_plus_ulp_rn_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.f(float 1.0, float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_plus_ulp_rp_f() {
+; CHECK-LABEL: define float @test_fma_1_plus_ulp_rp_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.fma.rp.f(float 1.0, float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_plus_ulp_rz_f() {
+; CHECK-LABEL: define float @test_fma_1_plus_ulp_rz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_plus_ulp_rm_ftz_f() {
+; CHECK-LABEL: define float @test_fma_1_plus_ulp_rm_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rm.ftz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_plus_ulp_rn_ftz_f() {
+; CHECK-LABEL: define float @test_fma_1_plus_ulp_rn_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.ftz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_plus_ulp_rp_ftz_f() {
+; CHECK-LABEL: define float @test_fma_1_plus_ulp_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.fma.rp.ftz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_plus_ulp_rz_ftz_f() {
+; CHECK-LABEL: define float @test_fma_1_plus_ulp_rz_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rz.ftz.f(float 1.0, float 1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    FMA(1.0, 1.0, 2^(-54))                   #
+;###############################################################
+; Tests FMA with 1.0 and 2^(-54) where different rounding modes produce different results.
+; The exact result falls between 1.0 and 1.0 + 2^(-52).
+; - RN, RZ, RM: Return 1.0 (rounding to nearest/zero/down)
+; - RP: Returns 1.0 + 2^(-52) (rounding up)
+
+define double @test_fma_1_plus_ulp_rm_d() {
+; CHECK-LABEL: define double @test_fma_1_plus_ulp_rm_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rm.d(double 1.0, double 1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_fma_1_plus_ulp_rn_d() {
+; CHECK-LABEL: define double @test_fma_1_plus_ulp_rn_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rn.d(double 1.0, double 1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_fma_1_plus_ulp_rp_d() {
+; CHECK-LABEL: define double @test_fma_1_plus_ulp_rp_d() {
+; CHECK-NEXT:    ret double 0x3FF0000000000001
+;
+  %res = call double @llvm.nvvm.fma.rp.d(double 1.0, double 1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_fma_1_plus_ulp_rz_d() {
+; CHECK-LABEL: define double @test_fma_1_plus_ulp_rz_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rz.d(double 1.0, double 1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+;###############################################################
+;#                    FMA(1.0, -1.0, 2^(-25))                  #
+;###############################################################
+; Tests FMA with -1.0 and 2^(-25) where different rounding modes produce different results.
+; The exact result falls between -1.0 and -1.0 + 2^(-23).
+; - RN, RM: Returns -1.0 (rounding toward nearest/down)
+; - RZ, RP: Return -1.0 + 2^(-23) (rounding toward zero/up)
+
+define float @test_fma_neg_1_plus_ulp_rm_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rm_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rm.f(float 1.0, float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_plus_ulp_rn_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rn_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.f(float 1.0, float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_plus_ulp_rp_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rp_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.fma.rp.f(float 1.0, float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_plus_ulp_rz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.fma.rz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_plus_ulp_rm_ftz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rm_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rm.ftz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_plus_ulp_rn_ftz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rn_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.ftz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_plus_ulp_rp_ftz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.fma.rp.ftz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_plus_ulp_rz_ftz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_plus_ulp_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.fma.rz.ftz.f(float 1.0, float -1.0, float 0x3E60000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    FMA(1.0, -1.0, 2^(-54))                  #
+;###############################################################
+; Tests FMA with -1.0 and 2^(-54) where different rounding modes produce different results.
+; The exact result falls between -1.0 and -1.0 + 2^(-52).
+; - RN, RM: Return -1.0 (rounding toward nearest/down)
+; - RZ, RP: Return -1.0 + 2^(-52) (rounding toward zero/up)
+
+define double @test_fma_neg_1_plus_ulp_rm_d() {
+; CHECK-LABEL: define double @test_fma_neg_1_plus_ulp_rm_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rm.d(double 1.0, double -1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_fma_neg_1_plus_ulp_rn_d() {
+; CHECK-LABEL: define double @test_fma_neg_1_plus_ulp_rn_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rn.d(double 1.0, double -1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_fma_neg_1_plus_ulp_rp_d() {
+; CHECK-LABEL: define double @test_fma_neg_1_plus_ulp_rp_d() {
+; CHECK-NEXT:    ret double 0xBFEFFFFFFFFFFFFF
+;
+  %res = call double @llvm.nvvm.fma.rp.d(double 1.0, double -1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+define double @test_fma_neg_1_plus_ulp_rz_d() {
+; CHECK-LABEL: define double @test_fma_neg_1_plus_ulp_rz_d() {
+; CHECK-NEXT:    ret double 0xBFEFFFFFFFFFFFFF
+;
+  %res = call double @llvm.nvvm.fma.rz.d(double 1.0, double -1.0, double 0x3C90000000000000)
+  ret double %res
+}
+
+;###############################################################
+;#                    FMA(1.0, 1.0, -2^(-25))                  #
+;###############################################################
+; Tests FMA with 1.0 and -2^(-25) where different rounding modes produce different results.
+; The exact result falls between 1.0 and 1.0 - 2^(-23).
+; - RN, RP: Return 1.0 (rounding toward nearest/up)
+; - RZ, RM: Return 1.0 - 2^(-23) (rounding toward zero/down)
+
+define float @test_fma_1_minus_ulp_rm_f() {
+; CHECK-LABEL: define float @test_fma_1_minus_ulp_rm_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.fma.rm.f(float 1.0, float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_minus_ulp_rn_f() {
+; CHECK-LABEL: define float @test_fma_1_minus_ulp_rn_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.f(float 1.0, float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_minus_ulp_rp_f() {
+; CHECK-LABEL: define float @test_fma_1_minus_ulp_rp_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rp.f(float 1.0, float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_minus_ulp_rz_f() {
+; CHECK-LABEL: define float @test_fma_1_minus_ulp_rz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.fma.rz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_minus_ulp_rm_ftz_f() {
+; CHECK-LABEL: define float @test_fma_1_minus_ulp_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.fma.rm.ftz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_minus_ulp_rn_ftz_f() {
+; CHECK-LABEL: define float @test_fma_1_minus_ulp_rn_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.ftz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_minus_ulp_rp_ftz_f() {
+; CHECK-LABEL: define float @test_fma_1_minus_ulp_rp_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rp.ftz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_1_minus_ulp_rz_ftz_f() {
+; CHECK-LABEL: define float @test_fma_1_minus_ulp_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FEFFFFFE0000000
+;
+  %res = call float @llvm.nvvm.fma.rz.ftz.f(float 1.0, float 1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    FMA(1.0, 1.0, -2^(-54))                  #
+;###############################################################
+; Tests FMA with 1.0 and -2^(-54) where different rounding modes produce different results.
+; The exact result falls between 1.0 and 1.0 - 2^(-52).
+; - RN, RP: Return 1.0 (rounding toward nearest/up)
+; - RZ, RM: Return 1.0 - 2^(-52) (rounding toward zero/down)
+
+define double @test_fma_1_minus_ulp_rm_d() {
+; CHECK-LABEL: define double @test_fma_1_minus_ulp_rm_d() {
+; CHECK-NEXT:    ret double 0x3FEFFFFFFFFFFFFF
+;
+  %res = call double @llvm.nvvm.fma.rm.d(double 1.0, double 1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_fma_1_minus_ulp_rn_d() {
+; CHECK-LABEL: define double @test_fma_1_minus_ulp_rn_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rn.d(double 1.0, double 1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_fma_1_minus_ulp_rp_d() {
+; CHECK-LABEL: define double @test_fma_1_minus_ulp_rp_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rp.d(double 1.0, double 1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_fma_1_minus_ulp_rz_d() {
+; CHECK-LABEL: define double @test_fma_1_minus_ulp_rz_d() {
+; CHECK-NEXT:    ret double 0x3FEFFFFFFFFFFFFF
+;
+  %res = call double @llvm.nvvm.fma.rz.d(double 1.0, double 1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+;###############################################################
+;#                    FMA(1.0, -1.0, -2^(-25))                 #
+;###############################################################
+; Tests FMA with -1.0 and -2^(-25) where different rounding modes produce different results.
+; The exact result falls between -1.0 and -1.0 - 2^(-23).
+; - RN, RZ, RP: Return -1.0 (rounding to nearest/zero/up)
+; - RM: Return -1.0 - 2^(-23) (rounding down)
+
+define float @test_fma_neg_1_minus_ulp_rm_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rm_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.fma.rm.f(float 1.0, float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_minus_ulp_rn_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rn_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.f(float 1.0, float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_minus_ulp_rp_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rp_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rp.f(float 1.0, float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_minus_ulp_rz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_minus_ulp_rm_ftz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.fma.rm.ftz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_minus_ulp_rn_ftz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rn_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rn.ftz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_minus_ulp_rp_ftz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rp_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rp.ftz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+define float @test_fma_neg_1_minus_ulp_rz_ftz_f() {
+; CHECK-LABEL: define float @test_fma_neg_1_minus_ulp_rz_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.fma.rz.ftz.f(float 1.0, float -1.0, float 0xBE60000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    FMA(1.0, -1.0, -2^(-54))                 #
+;###############################################################
+; Tests FMA with -1.0 and -2^(-54) where different rounding modes produce different results.
+; The exact result falls between -1.0 and -1.0 - 2^(-52).
+; - RN, RZ, RP: Return -1.0 (rounding to nearest/zero/up)
+; - RM: Return -1.0 - 2^(-52) (rounding down)
+
+define double @test_fma_neg_1_minus_ulp_rm_d() {
+; CHECK-LABEL: define double @test_fma_neg_1_minus_ulp_rm_d() {
+; CHECK-NEXT:    ret double 0xBFF0000000000001
+;
+  %res = call double @llvm.nvvm.fma.rm.d(double 1.0, double -1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_fma_neg_1_minus_ulp_rn_d() {
+; CHECK-LABEL: define double @test_fma_neg_1_minus_ulp_rn_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rn.d(double 1.0, double -1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_fma_neg_1_minus_ulp_rp_d() {
+; CHECK-LABEL: define double @test_fma_neg_1_minus_ulp_rp_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rp.d(double 1.0, double -1.0, double 0xBC90000000000000)
+  ret double %res
+}
+
+define double @test_fma_neg_1_minus_ulp_rz_d() {
+; CHECK-LABEL: define double @test_fma_neg_1_minus_ulp_rz_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.fma.rz.d(double 1.0, double -1.0, double 0xBC90000000000000)
+  ret double %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/const-fold-nvvm-mul.ll b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-mul.ll
new file mode 100644
index 0000000000000..12391e8bf0631
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/const-fold-nvvm-mul.ll
@@ -0,0 +1,994 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instsimplify -march=nvptx64 --mcpu=sm_86 --mattr=+ptx72 -S | FileCheck %s
+
+; Check constant-folding for NVVM multiply intrinsics with different rounding modes
+
+;###############################################################
+;#                    Mul(1.25, 2.0)                           #
+;###############################################################
+; Tests multiplication of two normal numbers (1.25 and 2.0) where the result
+; is exactly representable. All rounding modes should produce the same result.
+
+define double @test_1_25_times_2_rm_d() {
+; CHECK-LABEL: define double @test_1_25_times_2_rm_d() {
+; CHECK-NEXT:    ret double 2.500000e+00
+;
+  %res = call double @llvm.nvvm.mul.rm.d(double 1.25, double 2.0)
+  ret double %res
+}
+
+define double @test_1_25_times_2_rn_d() {
+; CHECK-LABEL: define double @test_1_25_times_2_rn_d() {
+; CHECK-NEXT:    ret double 2.500000e+00
+;
+  %res = call double @llvm.nvvm.mul.rn.d(double 1.25, double 2.0)
+  ret double %res
+}
+
+define double @test_1_25_times_2_rp_d() {
+; CHECK-LABEL: define double @test_1_25_times_2_rp_d() {
+; CHECK-NEXT:    ret double 2.500000e+00
+;
+  %res = call double @llvm.nvvm.mul.rp.d(double 1.25, double 2.0)
+  ret double %res
+}
+
+define double @test_1_25_times_2_rz_d() {
+; CHECK-LABEL: define double @test_1_25_times_2_rz_d() {
+; CHECK-NEXT:    ret double 2.500000e+00
+;
+  %res = call double @llvm.nvvm.mul.rz.d(double 1.25, double 2.0)
+  ret double %res
+}
+
+define float @test_1_25_times_2_rm_f() {
+; CHECK-LABEL: define float @test_1_25_times_2_rm_f() {
+; CHECK-NEXT:    ret float 2.500000e+00
+;
+  %res = call float @llvm.nvvm.mul.rm.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_times_2_rn_f() {
+; CHECK-LABEL: define float @test_1_25_times_2_rn_f() {
+; CHECK-NEXT:    ret float 2.500000e+00
+;
+  %res = call float @llvm.nvvm.mul.rn.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_times_2_rp_f() {
+; CHECK-LABEL: define float @test_1_25_times_2_rp_f() {
+; CHECK-NEXT:    ret float 2.500000e+00
+;
+  %res = call float @llvm.nvvm.mul.rp.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_times_2_rz_f() {
+; CHECK-LABEL: define float @test_1_25_times_2_rz_f() {
+; CHECK-NEXT:    ret float 2.500000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_times_2_rm_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_times_2_rm_ftz_f() {
+; CHECK-NEXT:    ret float 2.500000e+00
+;
+  %res = call float @llvm.nvvm.mul.rm.ftz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_times_2_rn_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_times_2_rn_ftz_f() {
+; CHECK-NEXT:    ret float 2.500000e+00
+;
+  %res = call float @llvm.nvvm.mul.rn.ftz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_times_2_rp_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_times_2_rp_ftz_f() {
+; CHECK-NEXT:    ret float 2.500000e+00
+;
+  %res = call float @llvm.nvvm.mul.rp.ftz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+define float @test_1_25_times_2_rz_ftz_f() {
+; CHECK-LABEL: define float @test_1_25_times_2_rz_ftz_f() {
+; CHECK-NEXT:    ret float 2.500000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.ftz.f(float 1.25, float 2.0)
+  ret float %res
+}
+
+;###############################################################
+;#                    Mul(1.0, Subnormal)                      #
+;###############################################################
+; Tests multiplication of 1.0 by a subnormal number to verify FTZ behavior.
+; For float, we use 2^-149 (smallest subnormal float).
+; For double, we use 2^-1074 (smallest subnormal double).
+; Without FTZ, the result should be the subnormal number.
+; With FTZ, the result should be 0.0.
+
+define double @test_1_times_subnorm_rm_d() {
+; CHECK-LABEL: define double @test_1_times_subnorm_rm_d() {
+; CHECK-NEXT:    ret double 4.940660e-324
+;
+  %res = call double @llvm.nvvm.mul.rm.d(double 1.0, double 0x0000000000000001)
+  ret double %res
+}
+
+define double @test_1_times_subnorm_rn_d() {
+; CHECK-LABEL: define double @test_1_times_subnorm_rn_d() {
+; CHECK-NEXT:    ret double 4.940660e-324
+;
+  %res = call double @llvm.nvvm.mul.rn.d(double 1.0, double 0x0000000000000001)
+  ret double %res
+}
+
+define double @test_1_times_subnorm_rp_d() {
+; CHECK-LABEL: define double @test_1_times_subnorm_rp_d() {
+; CHECK-NEXT:    ret double 4.940660e-324
+;
+  %res = call double @llvm.nvvm.mul.rp.d(double 1.0, double 0x0000000000000001)
+  ret double %res
+}
+
+define double @test_1_times_subnorm_rz_d() {
+; CHECK-LABEL: define double @test_1_times_subnorm_rz_d() {
+; CHECK-NEXT:    ret double 4.940660e-324
+;
+  %res = call double @llvm.nvvm.mul.rz.d(double 1.0, double 0x0000000000000001)
+  ret double %res
+}
+
+define float @test_1_times_subnorm_rm_f() {
+; CHECK-LABEL: define float @test_1_times_subnorm_rm_f() {
+; CHECK-NEXT:    ret float 0x36A0000000000000
+;
+  %res = call float @llvm.nvvm.mul.rm.f(float 1.0, float 0x36A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_subnorm_rn_f() {
+; CHECK-LABEL: define float @test_1_times_subnorm_rn_f() {
+; CHECK-NEXT:    ret float 0x36A0000000000000
+;
+  %res = call float @llvm.nvvm.mul.rn.f(float 1.0, float 0x36A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_subnorm_rp_f() {
+; CHECK-LABEL: define float @test_1_times_subnorm_rp_f() {
+; CHECK-NEXT:    ret float 0x36A0000000000000
+;
+  %res = call float @llvm.nvvm.mul.rp.f(float 1.0, float 0x36A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_subnorm_rz_f() {
+; CHECK-LABEL: define float @test_1_times_subnorm_rz_f() {
+; CHECK-NEXT:    ret float 0x36A0000000000000
+;
+  %res = call float @llvm.nvvm.mul.rz.f(float 1.0, float 0x36A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_subnorm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_1_times_subnorm_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rm.ftz.f(float 1.0, float 0x36A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_subnorm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_1_times_subnorm_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rn.ftz.f(float 1.0, float 0x36A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_subnorm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_1_times_subnorm_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rp.ftz.f(float 1.0, float 0x36A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_subnorm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_1_times_subnorm_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.ftz.f(float 1.0, float 0x36A0000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Mul(1.0, -Subnormal)                     #
+;###############################################################
+; Tests multiplication of 1.0 by a negative subnormal number to verify FTZ behavior.
+; For float, we use -2^-149 (negative smallest subnormal).
+; For double, we use -2^-1074 (negative smallest subnormal).
+; Without FTZ, the result should be the negative subnormal number.
+; With FTZ, the result should be -0.0.
+
+define double @test_1_times_neg_subnorm_rm_d() {
+; CHECK-LABEL: define double @test_1_times_neg_subnorm_rm_d() {
+; CHECK-NEXT:    ret double -4.940660e-324
+;
+  %res = call double @llvm.nvvm.mul.rm.d(double 1.0, double 0x8000000000000001)
+  ret double %res
+}
+
+define double @test_1_times_neg_subnorm_rn_d() {
+; CHECK-LABEL: define double @test_1_times_neg_subnorm_rn_d() {
+; CHECK-NEXT:    ret double -4.940660e-324
+;
+  %res = call double @llvm.nvvm.mul.rn.d(double 1.0, double 0x8000000000000001)
+  ret double %res
+}
+
+define double @test_1_times_neg_subnorm_rp_d() {
+; CHECK-LABEL: define double @test_1_times_neg_subnorm_rp_d() {
+; CHECK-NEXT:    ret double -4.940660e-324
+;
+  %res = call double @llvm.nvvm.mul.rp.d(double 1.0, double 0x8000000000000001)
+  ret double %res
+}
+
+define double @test_1_times_neg_subnorm_rz_d() {
+; CHECK-LABEL: define double @test_1_times_neg_subnorm_rz_d() {
+; CHECK-NEXT:    ret double -4.940660e-324
+;
+  %res = call double @llvm.nvvm.mul.rz.d(double 1.0, double 0x8000000000000001)
+  ret double %res
+}
+
+define float @test_1_times_neg_subnorm_rm_f() {
+; CHECK-LABEL: define float @test_1_times_neg_subnorm_rm_f() {
+; CHECK-NEXT:    ret float 0xB6A0000000000000
+;
+  %res = call float @llvm.nvvm.mul.rm.f(float 1.0, float 0xB6A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_neg_subnorm_rn_f() {
+; CHECK-LABEL: define float @test_1_times_neg_subnorm_rn_f() {
+; CHECK-NEXT:    ret float 0xB6A0000000000000
+;
+  %res = call float @llvm.nvvm.mul.rn.f(float 1.0, float 0xB6A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_neg_subnorm_rp_f() {
+; CHECK-LABEL: define float @test_1_times_neg_subnorm_rp_f() {
+; CHECK-NEXT:    ret float 0xB6A0000000000000
+;
+  %res = call float @llvm.nvvm.mul.rp.f(float 1.0, float 0xB6A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_neg_subnorm_rz_f() {
+; CHECK-LABEL: define float @test_1_times_neg_subnorm_rz_f() {
+; CHECK-NEXT:    ret float 0xB6A0000000000000
+;
+  %res = call float @llvm.nvvm.mul.rz.f(float 1.0, float 0xB6A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_neg_subnorm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_1_times_neg_subnorm_rm_ftz_f() {
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rm.ftz.f(float 1.0, float 0xB6A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_neg_subnorm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_1_times_neg_subnorm_rn_ftz_f() {
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rn.ftz.f(float 1.0, float 0xB6A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_neg_subnorm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_1_times_neg_subnorm_rp_ftz_f() {
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rp.ftz.f(float 1.0, float 0xB6A0000000000000)
+  ret float %res
+}
+
+define float @test_1_times_neg_subnorm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_1_times_neg_subnorm_rz_ftz_f() {
+; CHECK-NEXT:    ret float -0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.ftz.f(float 1.0, float 0xB6A0000000000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Mul(Normal, Normal) -> Subnormal         #
+;###############################################################
+; Tests multiplication of two normal numbers that produces a subnormal result.
+; We multiply the smallest normal float (2^-126 = 0x3810000000000000) by 0.5 to get 2^-127,
+; which is subnormal. This tests the transition from normal to subnormal numbers.
+; For double precision, we just use the same float value since there is no FTZ variant.
+
+define double @test_normal_times_normal_to_subnorm_rm_d() {
+; CHECK-LABEL: define double @test_normal_times_normal_to_subnorm_rm_d() {
+; CHECK-NEXT:    ret double 0x3800000000000000
+;
+  %res = call double @llvm.nvvm.mul.rm.d(double 0x3810000000000000, double 0.5)
+  ret double %res
+}
+
+define double @test_normal_times_normal_to_subnorm_rn_d() {
+; CHECK-LABEL: define double @test_normal_times_normal_to_subnorm_rn_d() {
+; CHECK-NEXT:    ret double 0x3800000000000000
+;
+  %res = call double @llvm.nvvm.mul.rn.d(double 0x3810000000000000, double 0.5)
+  ret double %res
+}
+
+define double @test_normal_times_normal_to_subnorm_rp_d() {
+; CHECK-LABEL: define double @test_normal_times_normal_to_subnorm_rp_d() {
+; CHECK-NEXT:    ret double 0x3800000000000000
+;
+  %res = call double @llvm.nvvm.mul.rp.d(double 0x3810000000000000, double 0.5)
+  ret double %res
+}
+
+define double @test_normal_times_normal_to_subnorm_rz_d() {
+; CHECK-LABEL: define double @test_normal_times_normal_to_subnorm_rz_d() {
+; CHECK-NEXT:    ret double 0x3800000000000000
+;
+  %res = call double @llvm.nvvm.mul.rz.d(double 0x3810000000000000, double 0.5)
+  ret double %res
+}
+
+define float @test_normal_times_normal_to_subnorm_rm_f() {
+; CHECK-LABEL: define float @test_normal_times_normal_to_subnorm_rm_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.mul.rm.f(float 0x3810000000000000, float 0.5)
+  ret float %res
+}
+
+define float @test_normal_times_normal_to_subnorm_rn_f() {
+; CHECK-LABEL: define float @test_normal_times_normal_to_subnorm_rn_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.mul.rn.f(float 0x3810000000000000, float 0.5)
+  ret float %res
+}
+
+define float @test_normal_times_normal_to_subnorm_rp_f() {
+; CHECK-LABEL: define float @test_normal_times_normal_to_subnorm_rp_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.mul.rp.f(float 0x3810000000000000, float 0.5)
+  ret float %res
+}
+
+define float @test_normal_times_normal_to_subnorm_rz_f() {
+; CHECK-LABEL: define float @test_normal_times_normal_to_subnorm_rz_f() {
+; CHECK-NEXT:    ret float 0x3800000000000000
+;
+  %res = call float @llvm.nvvm.mul.rz.f(float 0x3810000000000000, float 0.5)
+  ret float %res
+}
+
+define float @test_normal_times_normal_to_subnorm_rm_ftz_f() {
+; CHECK-LABEL: define float @test_normal_times_normal_to_subnorm_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rm.ftz.f(float 0x3810000000000000, float 0.5)
+  ret float %res
+}
+
+define float @test_normal_times_normal_to_subnorm_rn_ftz_f() {
+; CHECK-LABEL: define float @test_normal_times_normal_to_subnorm_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rn.ftz.f(float 0x3810000000000000, float 0.5)
+  ret float %res
+}
+
+define float @test_normal_times_normal_to_subnorm_rp_ftz_f() {
+; CHECK-LABEL: define float @test_normal_times_normal_to_subnorm_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rp.ftz.f(float 0x3810000000000000, float 0.5)
+  ret float %res
+}
+
+define float @test_normal_times_normal_to_subnorm_rz_ftz_f() {
+; CHECK-LABEL: define float @test_normal_times_normal_to_subnorm_rz_ftz_f() {
+; CHECK-NEXT:    ret float 0.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.ftz.f(float 0x3810000000000000, float 0.5)
+  ret float %res
+}
+
+;###############################################################
+;#                    Mul(2.0, NaN)                            #
+;###############################################################
+; Tests multiplication with NaN to verify that we do not fold these,
+; as host and device NaNs may be different.
+
+define double @test_2_times_nan_rm_d() {
+; CHECK-LABEL: define double @test_2_times_nan_rm_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.mul.rm.d(double 2.000000e+00, double 0x7FF4444400000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.mul.rm.d(double 2.0, double 0x7FF4444400000000)
+  ret double %res
+}
+
+define double @test_2_times_nan_rn_d() {
+; CHECK-LABEL: define double @test_2_times_nan_rn_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.mul.rn.d(double 2.000000e+00, double 0x7FF4444400000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.mul.rn.d(double 2.0, double 0x7FF4444400000000)
+  ret double %res
+}
+
+define double @test_2_times_nan_rp_d() {
+; CHECK-LABEL: define double @test_2_times_nan_rp_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.mul.rp.d(double 2.000000e+00, double 0x7FF4444400000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.mul.rp.d(double 2.0, double 0x7FF4444400000000)
+  ret double %res
+}
+
+define double @test_2_times_nan_rz_d() {
+; CHECK-LABEL: define double @test_2_times_nan_rz_d() {
+; CHECK-NEXT:    [[RES:%.*]] = call double @llvm.nvvm.mul.rz.d(double 2.000000e+00, double 0x7FF4444400000000)
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %res = call double @llvm.nvvm.mul.rz.d(double 2.0, double 0x7FF4444400000000)
+  ret double %res
+}
+
+define float @test_2_times_nan_rm_f() {
+; CHECK-LABEL: define float @test_2_times_nan_rm_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.mul.rm.f(float 2.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.mul.rm.f(float 2.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_2_times_nan_rn_f() {
+; CHECK-LABEL: define float @test_2_times_nan_rn_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.mul.rn.f(float 2.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.mul.rn.f(float 2.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_2_times_nan_rp_f() {
+; CHECK-LABEL: define float @test_2_times_nan_rp_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.mul.rp.f(float 2.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.mul.rp.f(float 2.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_2_times_nan_rz_f() {
+; CHECK-LABEL: define float @test_2_times_nan_rz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.mul.rz.f(float 2.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.mul.rz.f(float 2.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_2_times_nan_rm_ftz_f() {
+; CHECK-LABEL: define float @test_2_times_nan_rm_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.mul.rm.ftz.f(float 2.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.mul.rm.ftz.f(float 2.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_2_times_nan_rn_ftz_f() {
+; CHECK-LABEL: define float @test_2_times_nan_rn_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.mul.rn.ftz.f(float 2.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.mul.rn.ftz.f(float 2.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_2_times_nan_rp_ftz_f() {
+; CHECK-LABEL: define float @test_2_times_nan_rp_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.mul.rp.ftz.f(float 2.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.mul.rp.ftz.f(float 2.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+define float @test_2_times_nan_rz_ftz_f() {
+; CHECK-LABEL: define float @test_2_times_nan_rz_ftz_f() {
+; CHECK-NEXT:    [[RES:%.*]] = call float @llvm.nvvm.mul.rz.ftz.f(float 2.000000e+00, float 0x7FFF444400000000)
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %res = call float @llvm.nvvm.mul.rz.ftz.f(float 2.0, float 0x7FFF444400000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Mul(0.75, 4/3 + epsilon)                 #
+;###############################################################
+; Tests multiplication of 0.75 (3/4) by a value slightly above 4/3,
+; where different rounding modes produce different results.
+; The exact result would be 1.0, but since 4/3 cannot be exactly encoded
+; as a float, the calculated result falls between 1.0 and 1.0 + 2^-23.
+; - RN, RZ, RM round to 1.0 (rounding to nearest/zero/down)
+; - RP rounds to 1.0 + 2^-23 (rounding up)
+
+define float @test_mul_just_above_1_rm_f() {
+; CHECK-LABEL: define float @test_mul_just_above_1_rm_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rm.f(float 0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_above_1_rn_f() {
+; CHECK-LABEL: define float @test_mul_just_above_1_rn_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rn.f(float 0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_above_1_rp_f() {
+; CHECK-LABEL: define float @test_mul_just_above_1_rp_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rp.f(float 0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_above_1_rz_f() {
+; CHECK-LABEL: define float @test_mul_just_above_1_rz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.f(float 0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_above_1_rm_ftz_f() {
+; CHECK-LABEL: define float @test_mul_just_above_1_rm_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rm.ftz.f(float 0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_above_1_rn_ftz_f() {
+; CHECK-LABEL: define float @test_mul_just_above_1_rn_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rn.ftz.f(float 0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_above_1_rp_ftz_f() {
+; CHECK-LABEL: define float @test_mul_just_above_1_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rp.ftz.f(float 0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_above_1_rz_ftz_f() {
+; CHECK-LABEL: define float @test_mul_just_above_1_rz_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.ftz.f(float 0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Mul(0.75, 4/3 + epsilon)                 #
+;###############################################################
+; Tests multiplication of 0.75 by a value slightly above 4/3,
+; where different rounding modes produce different results.
+; The exact result would be 1.0, but since 4/3 cannot be exactly encoded
+; as a double, the calculated result falls between 1.0 and 1.0 + 2^-52.
+; - RN, RZ, RM round to 1.0 (rounding to nearest/zero/down)
+; - RP rounds to 1.0 + 2^-52 (rounding up)
+
+define double @test_mul_just_above_1_rm_d() {
+; CHECK-LABEL: define double @test_mul_just_above_1_rm_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rm.d(double 0.75, double 0x3FF5555555555556)
+  ret double %res
+}
+
+define double @test_mul_just_above_1_rn_d() {
+; CHECK-LABEL: define double @test_mul_just_above_1_rn_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rn.d(double 0.75, double 0x3FF5555555555556)
+  ret double %res
+}
+
+define double @test_mul_just_above_1_rp_d() {
+; CHECK-LABEL: define double @test_mul_just_above_1_rp_d() {
+; CHECK-NEXT:    ret double 0x3FF0000000000001
+;
+  %res = call double @llvm.nvvm.mul.rp.d(double 0.75, double 0x3FF5555555555556)
+  ret double %res
+}
+
+define double @test_mul_just_above_1_rz_d() {
+; CHECK-LABEL: define double @test_mul_just_above_1_rz_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rz.d(double 0.75, double 0x3FF5555555555556)
+  ret double %res
+}
+
+;###############################################################
+;#                    Mul(-0.75, 4/3 + epsilon)                #
+;###############################################################
+; Tests multiplication of -0.75 by a value slightly above 4/3,
+; where different rounding modes produce different results.
+; The exact result would be -1.0, but since 4/3 cannot be exactly encoded
+; as a double, the calculated result falls between -1.0 and -1.0 - 2^-23.
+; - RN, RZ, RP round to -1.0 (rounding to nearest/zero/up)
+; - RM rounds to -1.0 - 2^-23 (rounding down)
+
+define float @test_mul_just_below_negative_1_rm_f() {
+; CHECK-LABEL: define float @test_mul_just_below_negative_1_rm_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rm.f(float -0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_below_negative_1_rn_f() {
+; CHECK-LABEL: define float @test_mul_just_below_negative_1_rn_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rn.f(float -0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_below_negative_1_rp_f() {
+; CHECK-LABEL: define float @test_mul_just_below_negative_1_rp_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rp.f(float -0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_below_negative_1_rz_f() {
+; CHECK-LABEL: define float @test_mul_just_below_negative_1_rz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.f(float -0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_below_negative_1_rm_ftz_f() {
+; CHECK-LABEL: define float @test_mul_just_below_negative_1_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rm.ftz.f(float -0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_below_negative_1_rn_ftz_f() {
+; CHECK-LABEL: define float @test_mul_just_below_negative_1_rn_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rn.ftz.f(float -0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_below_negative_1_rp_ftz_f() {
+; CHECK-LABEL: define float @test_mul_just_below_negative_1_rp_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rp.ftz.f(float -0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+define float @test_mul_just_below_negative_1_rz_ftz_f() {
+; CHECK-LABEL: define float @test_mul_just_below_negative_1_rz_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.ftz.f(float -0.75, float 0x3FF5555560000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Mul(-0.75, 4/3 + epsilon)                #
+;###############################################################
+; Tests multiplication of -0.75 by a value slightly above 4/3,
+; where different rounding modes produce different results.
+; The exact result would be -1.0, but since 4/3 cannot be exactly encoded
+; as a double, the calculated result falls between -1.0 and -1.0 - 2^-52.
+; - RN, RZ, RP round to -1.0 (rounding to nearest/zero/up)
+; - RM rounds to -1.0 - 2^-52 (rounding down)
+
+define double @test_mul_just_below_negative_1_rm_d() {
+; CHECK-LABEL: define double @test_mul_just_below_negative_1_rm_d() {
+; CHECK-NEXT:    ret double 0xBFF0000000000001
+;
+  %res = call double @llvm.nvvm.mul.rm.d(double -0.75, double 0x3FF5555555555556)
+  ret double %res
+}
+
+define double @test_mul_just_below_negative_1_rn_d() {
+; CHECK-LABEL: define double @test_mul_just_below_negative_1_rn_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rn.d(double -0.75, double 0x3FF5555555555556)
+  ret double %res
+}
+
+define double @test_mul_just_below_negative_1_rp_d() {
+; CHECK-LABEL: define double @test_mul_just_below_negative_1_rp_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rp.d(double -0.75, double 0x3FF5555555555556)
+  ret double %res
+}
+
+define double @test_mul_just_below_negative_1_rz_d() {
+; CHECK-LABEL: define double @test_mul_just_below_negative_1_rz_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rz.d(double -0.75, double 0x3FF5555555555556)
+  ret double %res
+}
+
+;###############################################################
+;#                   Mul(0.625, 1.6 + epsilon)                 #
+;###############################################################
+; Tests multiplication of 5/8 * ~8/5 with different rounding modes.
+; Multiply 0.625 (5/8) by a value very slightly above 8/5 = 1.6 + epsilon.
+; The exact result is between 1.0 and 1.0 + 2^-23
+; - RN, RP round to 1.0 + 2^-23 (rounding towards nearest/up)
+; - RZ, RM round to 1.0 (rounding towards zero/down)
+
+define float @test_mul_slightly_more_above_1_rm_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_above_1_rm_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rm.f(float 0x3FE4000000000000, float 0x3FF99999C0000000)
+  ret float %res
+}
+
+define float @test_mul_slightly_more_above_1_rn_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_above_1_rn_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rn.f(float 0x3FE4000000000000, float 0x3FF99999C0000000)
+  ret float %res
+}
+
+define float @test_mul_slightly_more_above_1_rp_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_above_1_rp_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rp.f(float 0x3FE4000000000000, float 0x3FF99999C0000000 )
+  ret float %res
+}
+
+define float @test_mul_slightly_more_above_1_rz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_above_1_rz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.f(float 0x3FE4000000000000, float 0x3FF99999C0000000 )
+  ret float %res
+}
+
+define float @test_mul_slightly_more_above_1_rm_ftz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_above_1_rm_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rm.ftz.f(float 0x3FE4000000000000, float 0x3FF99999C0000000 )
+  ret float %res
+}
+
+define float @test_mul_slightly_more_above_1_rn_ftz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_above_1_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rn.ftz.f(float 0x3FE4000000000000, float 0x3FF99999C0000000 )
+  ret float %res
+}
+
+define float @test_mul_slightly_more_above_1_rp_ftz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_above_1_rp_ftz_f() {
+; CHECK-NEXT:    ret float 0x3FF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rp.ftz.f(float 0x3FE4000000000000, float 0x3FF99999C0000000 )
+  ret float %res
+}
+
+define float @test_mul_slightly_more_above_1_rz_ftz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_above_1_rz_ftz_f() {
+; CHECK-NEXT:    ret float 1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.ftz.f(float 0x3FE4000000000000, float 0x3FF99999C0000000 )
+  ret float %res
+}
+
+;###############################################################
+;#                    Mul(0.625, 1.6 + epsilon)                #
+;###############################################################
+; Tests multiplication of 5/8 * ~8/5  with different rounding modes.
+; Multiply 0.625 (5/8) by a value very slightly above 8/5 = 1.6 + epsilon.
+; The exact result is between 1.0 and 1.0 + 2^-52
+; - RN, RP round to 1.0 + 2^-52 (rounding towards nearest/up)
+; - RZ, RM round to 1.0 (rounding towards zero/down)
+
+define double @test_mul_slightly_more_above_1_rm_d() {
+; CHECK-LABEL: define double @test_mul_slightly_more_above_1_rm_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rm.d(double 0x3FE4000000000000, double 0x3FF999999999999B)
+  ret double %res
+}
+
+define double @test_mul_slightly_more_above_1_rn_d() {
+; CHECK-LABEL: define double @test_mul_slightly_more_above_1_rn_d() {
+; CHECK-NEXT:    ret double 0x3FF0000000000001
+;
+  %res = call double @llvm.nvvm.mul.rn.d(double 0x3FE4000000000000, double 0x3FF999999999999B)
+  ret double %res
+}
+
+define double @test_mul_slightly_more_above_1_rp_d() {
+; CHECK-LABEL: define double @test_mul_slightly_more_above_1_rp_d() {
+; CHECK-NEXT:    ret double 0x3FF0000000000001
+;
+  %res = call double @llvm.nvvm.mul.rp.d(double 0x3FE4000000000000, double 0x3FF999999999999B)
+  ret double %res
+}
+
+define double @test_mul_slightly_more_above_1_rz_d() {
+; CHECK-LABEL: define double @test_mul_slightly_more_above_1_rz_d() {
+; CHECK-NEXT:    ret double 1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rz.d(double 0x3FE4000000000000, double 0x3FF999999999999B)
+  ret double %res
+}
+
+;###############################################################
+;#                    Mul(0.625, -(1.6 + epsilon)              #
+;###############################################################
+; Tests multiplication of 5/8 * ~-8/5 with different rounding modes.
+; Multiply 0.625 (5/8) by a value very slightly below -8/5 = -(1.6 + epsilon).
+; The exact result is between -1.0 and -1.0 + 2^-23
+; - RN, RM round to -1.0 - 2^-23 (rounding towards nearest/down)
+; - RZ, RP round to -1.0 (rounding towards zero/up)
+
+define float @test_mul_slightly_more_below_negative_1_rm_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_below_negative_1_rm_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rm.f(float 0x3FE4000000000000, float 0xBFF99999C0000000)
+  ret float %res
+}
+
+define float @test_mul_slightly_more_below_negative_1_rn_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_below_negative_1_rn_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rn.f(float 0x3FE4000000000000, float 0xBFF99999C0000000)
+  ret float %res
+}
+
+define float @test_mul_slightly_more_below_negative_1_rp_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_below_negative_1_rp_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rp.f(float 0x3FE4000000000000, float 0xBFF99999C0000000)
+  ret float %res
+}
+
+define float @test_mul_slightly_more_below_negative_1_rz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_below_negative_1_rz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.f(float 0x3FE4000000000000, float 0xBFF99999C0000000)
+  ret float %res
+}
+
+define float @test_mul_slightly_more_below_negative_1_rm_ftz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_below_negative_1_rm_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rm.ftz.f(float 0x3FE4000000000000, float 0xBFF99999C0000000)
+  ret float %res
+}
+
+define float @test_mul_slightly_more_below_negative_1_rn_ftz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_below_negative_1_rn_ftz_f() {
+; CHECK-NEXT:    ret float 0xBFF0000020000000
+;
+  %res = call float @llvm.nvvm.mul.rn.ftz.f(float 0x3FE4000000000000, float 0xBFF99999C0000000)
+  ret float %res
+}
+
+define float @test_mul_slightly_more_below_negative_1_rp_ftz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_below_negative_1_rp_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rp.ftz.f(float 0x3FE4000000000000, float 0xBFF99999C0000000)
+  ret float %res
+}
+
+define float @test_mul_slightly_more_below_negative_1_rz_ftz_f() {
+; CHECK-LABEL: define float @test_mul_slightly_more_below_negative_1_rz_ftz_f() {
+; CHECK-NEXT:    ret float -1.000000e+00
+;
+  %res = call float @llvm.nvvm.mul.rz.ftz.f(float 0x3FE4000000000000, float 0xBFF99999C0000000)
+  ret float %res
+}
+
+;###############################################################
+;#                    Mul(0.625, -(1.6 + epsilon)              #
+;###############################################################
+; Tests multiplication of 5/8 * ~-8/5 with different rounding modes.
+; Multiply 0.625 (5/8) by a value very slightly below -8/5 = -(1.6 + epsilon).
+; The exact result is between -1.0 and -1.0 + 2^-52
+; - RN, RM round to -1.0 - 2^-52 (rounding towards nearest/down)
+; - RZ, RP round to -1.0 (rounding towards zero/up)
+
+define double @test_mul_slightly_more_below_negative_1_rm_d() {
+; CHECK-LABEL: define double @test_mul_slightly_more_below_negative_1_rm_d() {
+; CHECK-NEXT:    ret double 0xBFF0000000000001
+;
+  %res = call double @llvm.nvvm.mul.rm.d(double 0x3FE4000000000000, double 0xBFF999999999999B)
+  ret double %res
+}
+
+define double @test_mul_slightly_more_below_negative_1_rn_d() {
+; CHECK-LABEL: define double @test_mul_slightly_more_below_negative_1_rn_d() {
+; CHECK-NEXT:    ret double 0xBFF0000000000001
+;
+  %res = call double @llvm.nvvm.mul.rn.d(double 0x3FE4000000000000, double 0xBFF999999999999B)
+  ret double %res
+}
+
+define double @test_mul_slightly_more_below_negative_1_rp_d() {
+; CHECK-LABEL: define double @test_mul_slightly_more_below_negative_1_rp_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rp.d(double 0x3FE4000000000000, double 0xBFF999999999999B)
+  ret double %res
+}
+
+define double @test_mul_slightly_more_below_negative_1_rz_d() {
+; CHECK-LABEL: define double @test_mul_slightly_more_below_negative_1_rz_d() {
+; CHECK-NEXT:    ret double -1.000000e+00
+;
+  %res = call double @llvm.nvvm.mul.rz.d(double 0x3FE4000000000000, double 0xBFF999999999999B)
+  ret double %res
+}
diff --git a/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll
index ab4448b460bfc..820fff433e9e0 100644
--- a/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll
+++ b/llvm/test/Transforms/InstSimplify/floating-point-arithmetic.ll
@@ -213,7 +213,7 @@ define double @fmul_nnan_ninf_nneg_n0.0_commute(i127 %x) {
 
 define float @fmul_ninf_nnan_mul_zero_nsz(float nofpclass(inf nan) %f) {
 ; CHECK-LABEL: @fmul_ninf_nnan_mul_zero_nsz(
-; CHECK-NEXT:     ret float 0.000000e+00
+; CHECK-NEXT:    ret float 0.000000e+00
 ;
   %r = fmul nsz float %f, 0.0
   ret float %r
@@ -221,7 +221,7 @@ define float @fmul_ninf_nnan_mul_zero_nsz(float nofpclass(inf nan) %f) {
 
 define float @fmul_ninf_nnan_mul_nzero_nsz(float nofpclass(inf nan) %f) {
 ; CHECK-LABEL: @fmul_ninf_nnan_mul_nzero_nsz(
-; CHECK-NEXT:     ret float 0.000000e+00
+; CHECK-NEXT:    ret float 0.000000e+00
 ;
   %r = fmul nsz float %f, -0.0
   ret float %r
@@ -1255,3 +1255,20 @@ define i1 @fptrunc_round_unknown_positive(double %unknown) {
   %cmp = fcmp nnan oge float %op, 0.0
   ret i1 %cmp
 }
+
+define half @fabs_select_fabs(half noundef %x) {
+; CHECK-LABEL: @fabs_select_fabs(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ABS1:%.*]] = call half @llvm.fabs.f16(half [[X:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt half [[ABS1]], 0xH0000
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], half [[X]], half 0xH0000
+; CHECK-NEXT:    [[ABS2:%.*]] = call half @llvm.fabs.f16(half [[SEL]])
+; CHECK-NEXT:    ret half [[ABS2]]
+;
+entry:
+  %abs1 = call half @llvm.fabs.f16(half %x)
+  %cmp = fcmp ogt half %abs1, 0xH0000
+  %sel = select i1 %cmp, half %x, half 0xH0000
+  %abs2 = call half @llvm.fabs.f16(half %sel)
+  ret half %abs2
+}
diff --git a/llvm/test/Transforms/JumpTableToSwitch/basic.ll b/llvm/test/Transforms/JumpTableToSwitch/basic.ll
index 321f837077ab6..e3c032b9d4bf7 100644
--- a/llvm/test/Transforms/JumpTableToSwitch/basic.ll
+++ b/llvm/test/Transforms/JumpTableToSwitch/basic.ll
@@ -1,14 +1,39 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals all --version 4
 ; RUN: opt < %s -passes=jump-table-to-switch -verify-dom-info -S | FileCheck %s
 ; RUN: opt < %s -passes=jump-table-to-switch -jump-table-to-switch-size-threshold=0 -verify-dom-info -S | FileCheck %s --check-prefix=THRESHOLD-0
 
 @func_array = constant [2 x ptr] [ptr @func0, ptr @func1]
 
-define i32 @func0() {
+;.
+; CHECK: @func_array = constant [2 x ptr] [ptr @func0, ptr @func1]
+; CHECK: @void_func_array = constant [2 x ptr] [ptr @void_func0, ptr @void_func1]
+; CHECK: @func_array_addrspace_42 = addrspace(42) constant [2 x ptr addrspace(42)] [ptr addrspace(42) @func0_addrspace_42, ptr addrspace(42) @func1_addrspace_42]
+;.
+; THRESHOLD-0: @func_array = constant [2 x ptr] [ptr @func0, ptr @func1]
+; THRESHOLD-0: @void_func_array = constant [2 x ptr] [ptr @void_func0, ptr @void_func1]
+; THRESHOLD-0: @func_array_addrspace_42 = addrspace(42) constant [2 x ptr addrspace(42)] [ptr addrspace(42) @func0_addrspace_42, ptr addrspace(42) @func1_addrspace_42]
+;.
+define i32 @func0() !guid !0 {
+; CHECK-LABEL: define i32 @func0(
+; CHECK-SAME: ) !guid [[META0:![0-9]+]] {
+; CHECK-NEXT:    ret i32 1
+;
+; THRESHOLD-0-LABEL: define i32 @func0(
+; THRESHOLD-0-SAME: ) !guid [[META0:![0-9]+]] {
+; THRESHOLD-0-NEXT:    ret i32 1
+;
   ret i32 1
 }
 
-define i32 @func1() {
+define i32 @func1() !guid !1 {
+; CHECK-LABEL: define i32 @func1(
+; CHECK-SAME: ) !guid [[META1:![0-9]+]] {
+; CHECK-NEXT:    ret i32 2
+;
+; THRESHOLD-0-LABEL: define i32 @func1(
+; THRESHOLD-0-SAME: ) !guid [[META1:![0-9]+]] {
+; THRESHOLD-0-NEXT:    ret i32 2
+;
   ret i32 2
 }
 
@@ -20,7 +45,7 @@ define i32 @function_with_jump_table(i32 %index) {
 ; CHECK-NEXT:    switch i32 [[INDEX]], label [[DEFAULT_SWITCH_CASE_UNREACHABLE:%.*]] [
 ; CHECK-NEXT:      i32 0, label [[CALL_0:%.*]]
 ; CHECK-NEXT:      i32 1, label [[CALL_1:%.*]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    ], !prof [[PROF2:![0-9]+]]
 ; CHECK:       default.switch.case.unreachable:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       call.0:
@@ -37,12 +62,12 @@ define i32 @function_with_jump_table(i32 %index) {
 ; THRESHOLD-0-SAME: i32 [[INDEX:%.*]]) {
 ; THRESHOLD-0-NEXT:    [[GEP:%.*]] = getelementptr inbounds [2 x ptr], ptr @func_array, i32 0, i32 [[INDEX]]
 ; THRESHOLD-0-NEXT:    [[FUNC_PTR:%.*]] = load ptr, ptr [[GEP]], align 8
-; THRESHOLD-0-NEXT:    [[RESULT:%.*]] = call i32 [[FUNC_PTR]]()
+; THRESHOLD-0-NEXT:    [[RESULT:%.*]] = call i32 [[FUNC_PTR]](), !prof [[PROF2:![0-9]+]]
 ; THRESHOLD-0-NEXT:    ret i32 [[RESULT]]
 ;
   %gep = getelementptr inbounds [2 x ptr], ptr @func_array, i32 0, i32 %index
   %func_ptr = load ptr, ptr %gep
-  %result = call i32 %func_ptr()
+  %result = call i32 %func_ptr(), !prof !2
   ret i32 %result
 }
 
@@ -54,7 +79,7 @@ define i32 @basic_block_splitted_twice(i32 %index) {
 ; CHECK-NEXT:    switch i32 [[INDEX]], label [[DEFAULT_SWITCH_CASE_UNREACHABLE:%.*]] [
 ; CHECK-NEXT:      i32 0, label [[CALL_0:%.*]]
 ; CHECK-NEXT:      i32 1, label [[CALL_1:%.*]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    ], !prof [[PROF3:![0-9]+]]
 ; CHECK:       default.switch.case.unreachable:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       call.0:
@@ -70,7 +95,7 @@ define i32 @basic_block_splitted_twice(i32 %index) {
 ; CHECK-NEXT:    switch i32 [[INDEX]], label [[DEFAULT_SWITCH_CASE_UNREACHABLE1:%.*]] [
 ; CHECK-NEXT:      i32 0, label [[CALL_02:%.*]]
 ; CHECK-NEXT:      i32 1, label [[CALL_13:%.*]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    ], !prof [[PROF3]]
 ; CHECK:       default.switch.case.unreachable1:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       call.02:
@@ -106,10 +131,22 @@ define i32 @basic_block_splitted_twice(i32 %index) {
 }
 
 define void @void_func0() {
+; CHECK-LABEL: define void @void_func0() {
+; CHECK-NEXT:    ret void
+;
+; THRESHOLD-0-LABEL: define void @void_func0() {
+; THRESHOLD-0-NEXT:    ret void
+;
   ret void
 }
 
 define void @void_func1() {
+; CHECK-LABEL: define void @void_func1() {
+; CHECK-NEXT:    ret void
+;
+; THRESHOLD-0-LABEL: define void @void_func1() {
+; THRESHOLD-0-NEXT:    ret void
+;
   ret void
 }
 
@@ -123,7 +160,7 @@ define void @void_function_with_jump_table(i32 %index) {
 ; CHECK-NEXT:    switch i32 [[INDEX]], label [[DEFAULT_SWITCH_CASE_UNREACHABLE:%.*]] [
 ; CHECK-NEXT:      i32 0, label [[CALL_0:%.*]]
 ; CHECK-NEXT:      i32 1, label [[CALL_1:%.*]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    ], !prof [[PROF3]]
 ; CHECK:       default.switch.case.unreachable:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       call.0:
@@ -156,7 +193,7 @@ define void @void_function_with_jump_table_and_call_site_attr(i32 %index) {
 ; CHECK-NEXT:    switch i32 [[INDEX]], label [[DEFAULT_SWITCH_CASE_UNREACHABLE:%.*]] [
 ; CHECK-NEXT:      i32 0, label [[CALL_0:%.*]]
 ; CHECK-NEXT:      i32 1, label [[CALL_1:%.*]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    ], !prof [[PROF3]]
 ; CHECK:       default.switch.case.unreachable:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       call.0:
@@ -183,10 +220,22 @@ define void @void_function_with_jump_table_and_call_site_attr(i32 %index) {
 
 
 define i32 @func0_addrspace_42() addrspace(42) {
+; CHECK-LABEL: define i32 @func0_addrspace_42() addrspace(42) {
+; CHECK-NEXT:    ret i32 1
+;
+; THRESHOLD-0-LABEL: define i32 @func0_addrspace_42() addrspace(42) {
+; THRESHOLD-0-NEXT:    ret i32 1
+;
   ret i32 1
 }
 
 define i32 @func1_addrspace_42() addrspace(42) {
+; CHECK-LABEL: define i32 @func1_addrspace_42() addrspace(42) {
+; CHECK-NEXT:    ret i32 2
+;
+; THRESHOLD-0-LABEL: define i32 @func1_addrspace_42() addrspace(42) {
+; THRESHOLD-0-NEXT:    ret i32 2
+;
   ret i32 2
 }
 
@@ -200,7 +249,7 @@ define i32 @function_with_jump_table_addrspace_42(i32 %index) addrspace(42) {
 ; CHECK-NEXT:    switch i32 [[INDEX]], label [[DEFAULT_SWITCH_CASE_UNREACHABLE:%.*]] [
 ; CHECK-NEXT:      i32 0, label [[CALL_0:%.*]]
 ; CHECK-NEXT:      i32 1, label [[CALL_1:%.*]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    ], !prof [[PROF3]]
 ; CHECK:       default.switch.case.unreachable:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       call.0:
@@ -226,3 +275,20 @@ define i32 @function_with_jump_table_addrspace_42(i32 %index) addrspace(42) {
   ret i32 %result
 }
 
+!0 = !{i64 5678}
+!1 = !{i64 5555}
+!2 = !{!"VP", i32 0, i64 25, i64 5678, i64 20, i64 5555, i64 5}
+;.
+; CHECK: attributes #[[ATTR0]] = { nounwind }
+;.
+; THRESHOLD-0: attributes #[[ATTR0]] = { nounwind }
+;.
+; CHECK: [[META0]] = !{i64 5678}
+; CHECK: [[META1]] = !{i64 5555}
+; CHECK: [[PROF2]] = !{!"branch_weights", i32 0, i32 20, i32 5}
+; CHECK: [[PROF3]] = !{!"unknown"}
+;.
+; THRESHOLD-0: [[META0]] = !{i64 5678}
+; THRESHOLD-0: [[META1]] = !{i64 5555}
+; THRESHOLD-0: [[PROF2]] = !{!"VP", i32 0, i64 25, i64 5678, i64 20, i64 5555, i64 5}
+;.
diff --git a/llvm/test/Transforms/LICM/PR116813-memoryssa-outdated.ll b/llvm/test/Transforms/LICM/PR116813-memoryssa-outdated.ll
index 562701420f806..a040c3cc6947c 100644
--- a/llvm/test/Transforms/LICM/PR116813-memoryssa-outdated.ll
+++ b/llvm/test/Transforms/LICM/PR116813-memoryssa-outdated.ll
@@ -18,7 +18,7 @@ define i32 @foo(i1 %arg, ptr %arg1) {
 ; CHECK:       [[BB1]]:
 ; CHECK-NEXT:    [[UNSWITCHED_SELECT_US:%.*]] = phi ptr [ [[ARG1]], %[[BB0]] ]
 ; CHECK-NEXT:    [[I3_US:%.*]] = call i32 [[UNSWITCHED_SELECT_US]]()
-; CHECK-NEXT:    br i1 true, label %[[LOOP_US]], label %[[RET_SPLIT_US:.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br i1 true, label %[[LOOP_US]], label %[[RET_SPLIT_US:.*]]
 ; CHECK:       [[RET_SPLIT_US]]:
 ; CHECK-NEXT:    [[I3_LCSSA_US:%.*]] = phi i32 [ [[I3_US]], %[[BB1]] ]
 ; CHECK-NEXT:    br label %[[RET:.*]]
diff --git a/llvm/test/Transforms/LICM/dropped-tbaa.ll b/llvm/test/Transforms/LICM/dropped-tbaa.ll
index 11083b4e4f3fb..92839f1d19968 100644
--- a/llvm/test/Transforms/LICM/dropped-tbaa.ll
+++ b/llvm/test/Transforms/LICM/dropped-tbaa.ll
@@ -24,7 +24,7 @@ define void @foo(ptr %data, ptr %addend) #0 {
 ; CHECK-NEXT:    [[CONV_I:%.*]] = sitofp i32 [[TMP2]] to double
 entry:
   %i = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.start.p0(ptr %i) #2
   store i32 0, ptr %i, align 4, !tbaa !1
   br i1 true, label %for.body.lr.ph, label %for.cond.cleanup
 
@@ -35,7 +35,7 @@ for.cond.for.cond.cleanup_crit_edge:              ; preds = %for.inc
   br label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond.for.cond.cleanup_crit_edge, %entry
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.end.p0(ptr %i) #2
   br label %for.end
 
 for.body:                                         ; preds = %for.body.lr.ph, %for.inc
@@ -67,8 +67,8 @@ for.end:                                          ; preds = %for.cond.cleanup
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #0
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
+declare void @llvm.lifetime.end.p0(ptr nocapture) #0
 
 attributes #0 = { argmemonly nounwind }
 
diff --git a/llvm/test/Transforms/LICM/hoist-binop.ll b/llvm/test/Transforms/LICM/hoist-binop.ll
index 1b1347776fb9e..724f45979f6ba 100644
--- a/llvm/test/Transforms/LICM/hoist-binop.ll
+++ b/llvm/test/Transforms/LICM/hoist-binop.ll
@@ -22,6 +22,31 @@ loop:
   br label %loop
 }
 
+; Don't hoist ADD if the op has more than one use.
+define void @add_two_uses(i64 %c1, i64 %c2) {
+; CHECK-LABEL: @add_two_uses(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add i64 [[INDEX]], [[C1:%.*]]
+; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[STEP_ADD]], [[C2:%.*]]
+; CHECK-NEXT:    call void @use(i64 [[INDEX_NEXT]])
+; CHECK-NEXT:    br label [[LOOP]]
+;
+entry:
+  br label %loop
+
+loop:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
+  %step.add = add i64 %index, %c1
+  call void @use(i64 %step.add)
+  %index.next = add i64 %step.add, %c2
+  call void @use(i64 %index.next)
+  br label %loop
+}
+
 ; Hoist MUL and remove old op if unused.
 define void @mul_one_use(i64 %c1, i64 %c2) {
 ; CHECK-LABEL: @mul_one_use(
@@ -51,8 +76,6 @@ define void @add_nuw(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nuw i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add nuw i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -62,7 +85,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add nuw i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = add nuw i64 %step.add, %c2
   br label %loop
 }
@@ -76,8 +98,6 @@ define void @add_nuw_comm(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nuw i64 [[C1]], [[INDEX]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add nuw i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -87,7 +107,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add nuw i64 %c1, %index
-  call void @use(i64 %step.add)
   %index.next = add nuw i64 %step.add, %c2
   br label %loop
 }
@@ -101,8 +120,6 @@ define void @add_nuw_comm2(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nuw i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add nuw i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -112,7 +129,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add nuw i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = add nuw i64 %c2, %step.add
   br label %loop
 }
@@ -126,8 +142,6 @@ define void @add_nuw_comm3(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nuw i64 [[C1]], [[INDEX]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add nuw i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -137,7 +151,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add nuw i64 %c1, %index
-  call void @use(i64 %step.add)
   %index.next = add nuw i64 %c2, %step.add
   br label %loop
 }
@@ -152,8 +165,6 @@ define void @add_nuw_twobinops(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nuw i64 [[C1]], [[INDEX]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add nuw i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -163,135 +174,134 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add nuw i64 %c1, %index
-  call void @use(i64 %step.add)
   %c2.plus.2 = add nuw i64 %c2, 2
   %index.next = add nuw i64 %step.add, %c2.plus.2
   br label %loop
 }
 
 ; Hoist MUL and drop NUW even if both ops have it.
-define void @mul_nuw(i64 %c1, i64 %c2) {
+define void @mul_nuw(<2 x i64> %c1, <2 x i64> %c2) {
 ; CHECK-LABEL: @mul_nuw(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul i64 [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul <2 x i64> [[C1:%.*]], [[C2:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
-; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi <2 x i64> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw <2 x i64> [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(<2 x i64> [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul <2 x i64> [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 entry:
   br label %loop
 
 loop:
-  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
-  %step.add = mul nuw i64 %index, %c1
-  call void @use(i64 %step.add)
-  %index.next = mul nuw i64 %step.add, %c2
+  %index = phi <2 x i64> [ zeroinitializer, %entry ], [ %index.next, %loop ]
+  %step.add = mul nuw <2 x i64> %index, %c1
+  call void @use(<2 x i64> %step.add)
+  %index.next = mul nuw <2 x i64> %step.add, %c2
   br label %loop
 }
 
 ; Hoist MUL and drop NUW even if both ops have it.
 ; Version where operands are commuted.
-define void @mul_nuw_comm(i64 %c1, i64 %c2) {
+define void @mul_nuw_comm(<2 x i64> %c1, <2 x i64> %c2) {
 ; CHECK-LABEL: @mul_nuw_comm(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul i64 [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul <2 x i64> [[C1:%.*]], [[C2:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw i64 [[C1]], [[INDEX]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
-; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi <2 x i64> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw <2 x i64> [[C1]], [[INDEX]]
+; CHECK-NEXT:    call void @use(<2 x i64> [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul <2 x i64> [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 entry:
   br label %loop
 
 loop:
-  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
-  %step.add = mul nuw i64 %c1, %index
-  call void @use(i64 %step.add)
-  %index.next = mul nuw i64 %step.add, %c2
+  %index = phi <2 x i64> [ zeroinitializer, %entry ], [ %index.next, %loop ]
+  %step.add = mul nuw <2 x i64> %c1, %index
+  call void @use(<2 x i64> %step.add)
+  %index.next = mul nuw <2 x i64> %step.add, %c2
   br label %loop
 }
 
 ; Hoist MUL and drop NUW even if both ops have it.
 ; Another version where operands are commuted.
-define void @mul_nuw_comm2(i64 %c1, i64 %c2) {
+define void @mul_nuw_comm2(<2 x i64> %c1, <2 x i64> %c2) {
 ; CHECK-LABEL: @mul_nuw_comm2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul i64 [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul <2 x i64> [[C1:%.*]], [[C2:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
-; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi <2 x i64> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw <2 x i64> [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(<2 x i64> [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul <2 x i64> [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 entry:
   br label %loop
 
 loop:
-  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
-  %step.add = mul nuw i64 %index, %c1
-  call void @use(i64 %step.add)
-  %index.next = mul nuw i64 %c2, %step.add
+  %index = phi <2 x i64> [ zeroinitializer, %entry ], [ %index.next, %loop ]
+  %step.add = mul nuw <2 x i64> %index, %c1
+  call void @use(<2 x i64> %step.add)
+  %index.next = mul nuw <2 x i64> %c2, %step.add
   br label %loop
 }
 
 ; Hoist MUL and drop NUW even if both ops have it.
 ; Another version where operands are commuted.
-define void @mul_nuw_comm3(i64 %c1, i64 %c2) {
+define void @mul_nuw_comm3(<2 x i64> %c1, <2 x i64> %c2) {
 ; CHECK-LABEL: @mul_nuw_comm3(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul i64 [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul <2 x i64> [[C1:%.*]], [[C2:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw i64 [[C1]], [[INDEX]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
-; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi <2 x i64> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw <2 x i64> [[C1]], [[INDEX]]
+; CHECK-NEXT:    call void @use(<2 x i64> [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul <2 x i64> [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 entry:
   br label %loop
 
 loop:
-  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
-  %step.add = mul nuw i64 %c1, %index
-  call void @use(i64 %step.add)
-  %index.next = mul nuw i64 %c2, %step.add
+  %index = phi <2 x i64> [ zeroinitializer, %entry ], [ %index.next, %loop ]
+  %step.add = mul nuw <2 x i64> %c1, %index
+  call void @use(<2 x i64> %step.add)
+  %index.next = mul nuw <2 x i64> %c2, %step.add
   br label %loop
 }
 
 ; Hoist MUL and drop NUW even if both ops have it.
 ; A version where the LHS and RHS of the outer BinOp are BinOps.
-define void @mul_nuw_twobinops(i64 %c1, i64 %c2) {
+define void @mul_nuw_twobinops(<2 x i64> %c1, <2 x i64> %c2) {
 ; CHECK-LABEL: @mul_nuw_twobinops(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[C2_PLUS_2:%.*]] = add nuw i64 [[C2:%.*]], 2
-; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul i64 [[C1:%.*]], [[C2_PLUS_2]]
+; CHECK-NEXT:    [[C2_PLUS_2:%.*]] = add nuw <2 x i64> [[C2:%.*]], splat (i64 2)
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul <2 x i64> [[C1:%.*]], [[C2_PLUS_2]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw i64 [[C1]], [[INDEX]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
-; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi <2 x i64> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nuw <2 x i64> [[C1]], [[INDEX]]
+; CHECK-NEXT:    call void @use(<2 x i64> [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul <2 x i64> [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 entry:
   br label %loop
 
 loop:
-  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
-  %step.add = mul nuw i64 %c1, %index
-  call void @use(i64 %step.add)
-  %c2.plus.2 = add nuw i64 %c2, 2
-  %index.next = mul nuw i64 %step.add, %c2.plus.2
+  %index = phi <2 x i64> [ zeroinitializer, %entry ], [ %index.next, %loop ]
+  %step.add = mul nuw <2 x i64> %c1, %index
+  call void @use(<2 x i64> %step.add)
+  %c2.plus.2 = add nuw <2 x i64> %c2, <i64 2, i64 2>
+  %index.next = mul nuw <2 x i64> %step.add, %c2.plus.2
   br label %loop
 }
 
@@ -303,8 +313,6 @@ define void @add_no_nuw(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -314,7 +322,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = add nuw i64 %step.add, %c2
   br label %loop
 }
@@ -327,8 +334,6 @@ define void @add_no_nsw(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -338,7 +343,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = add nsw i64 %step.add, %c2
   br label %loop
 }
@@ -351,8 +355,6 @@ define void @add_no_nsw_2(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nsw i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -362,7 +364,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add nsw i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = add nsw i64 %step.add, %c2
   br label %loop
 }
@@ -375,8 +376,6 @@ define void @add_nuw_nsw(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nuw nsw i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add nuw nsw i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -386,7 +385,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add nuw nsw i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = add nuw nsw i64 %step.add, %c2
   br label %loop
 }
@@ -398,8 +396,6 @@ define void @add_both_nsw_first_nuw(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nuw nsw i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -409,7 +405,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add nuw nsw i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = add nsw i64 %step.add, %c2
   br label %loop
 }
@@ -421,8 +416,6 @@ define void @add_both_nsw_second_nuw(i64 %c1, i64 %c2) {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = add nsw i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = add i64 [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -432,33 +425,32 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add nsw i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = add nuw nsw i64 %step.add, %c2
   br label %loop
 }
 
 ;
 ; Hoist MUL and drop NSW even if both ops have it.
-define void @mul_no_nsw_2(i64 %c1, i64 %c2) {
+define void @mul_no_nsw_2(<2 x i64> %c1, <2 x i64> %c2) {
 ; CHECK-LABEL: @mul_no_nsw_2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul i64 [[C1:%.*]], [[C2:%.*]]
+; CHECK-NEXT:    [[INVARIANT_OP:%.*]] = mul <2 x i64> [[C1:%.*]], [[C2:%.*]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nsw i64 [[INDEX]], [[C1]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
-; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul i64 [[INDEX]], [[INVARIANT_OP]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi <2 x i64> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[INDEX_NEXT_REASS:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = mul nsw <2 x i64> [[INDEX]], [[C1]]
+; CHECK-NEXT:    call void @use(<2 x i64> [[STEP_ADD]])
+; CHECK-NEXT:    [[INDEX_NEXT_REASS]] = mul <2 x i64> [[INDEX]], [[INVARIANT_OP]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 entry:
   br label %loop
 
 loop:
-  %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
-  %step.add = mul nsw i64 %index, %c1
-  call void @use(i64 %step.add)
-  %index.next = mul nsw i64 %step.add, %c2
+  %index = phi <2 x i64> [ zeroinitializer, %entry ], [ %index.next, %loop ]
+  %step.add = mul nsw <2 x i64> %index, %c1
+  call void @use(<2 x i64> %step.add)
+  %index.next = mul nsw <2 x i64> %step.add, %c2
   br label %loop
 }
 
@@ -470,7 +462,6 @@ define void @diff_ops(i64 %c1, i64 %c2) {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[STEP_ADD:%.*]] = add i64 [[INDEX]], [[C1:%.*]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = mul i64 [[STEP_ADD]], [[C2:%.*]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -480,7 +471,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = add i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = mul i64 %step.add, %c2
   br label %loop
 }
@@ -493,7 +483,6 @@ define void @noassoc_ops(i64 %c1, i64 %c2) {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[STEP_ADD:%.*]] = sub i64 [[INDEX]], [[C1:%.*]]
-; CHECK-NEXT:    call void @use(i64 [[STEP_ADD]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = sub i64 [[STEP_ADD]], [[C2:%.*]]
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
@@ -503,7 +492,6 @@ entry:
 loop:
   %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
   %step.add = sub i64 %index, %c1
-  call void @use(i64 %step.add)
   %index.next = sub i64 %step.add, %c2
   br label %loop
 }
diff --git a/llvm/test/Transforms/LICM/hoist-phi-metadata.ll b/llvm/test/Transforms/LICM/hoist-phi-metadata.ll
new file mode 100644
index 0000000000000..6f64bf7d7c875
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-phi-metadata.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 2
+; Test that hoisting conditional branches copies the debug and profiling info
+; metadata from the branch being hoisted.
+; RUN: opt -S -passes=licm -licm-control-flow-hoisting=1 %s -o - | FileCheck %s
+
+define void @triangle_phi(i32 %x, ptr %p) {
+; CHECK-LABEL: define void @triangle_phi
+; CHECK-SAME: (i32 [[X:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[X]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_LICM:%.*]], label [[THEN_LICM:%.*]], !prof [[PROF2:![0-9]+]]
+; CHECK:       if.licm:
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT:    br label [[THEN_LICM]]
+; CHECK:       then.licm:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[ADD]], [[IF_LICM]] ], [ [[X]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    store i32 [[PHI]], ptr [[P]], align 4
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[PHI]], 0
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF:%.*]], label [[THEN:%.*]], !dbg [[DBG3:![0-9]+]], !prof [[PROF2]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[THEN]]
+; CHECK:       then:
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP]], label [[END:%.*]], !dbg [[DBG7:![0-9]+]], !prof [[PROF8:![0-9]+]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop, !dbg !5
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %then, !dbg !6, !prof !8
+if:
+  %add = add i32 %x, 1
+  br label %then
+
+then:
+  %phi = phi i32 [ %add, %if ], [ %x, %loop ]
+  store i32 %phi, ptr %p
+  %cmp2 = icmp ne i32 %phi, 0
+  br i1 %cmp2, label %loop, label %end, !dbg !7, !prof !9
+
+end:
+  ret void
+}
+
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1)
+!1 = !DIFile(filename: "t", directory: "/")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "triangle_phi", linkageName: "triangle_phi", scope: !1, file: !1, line: 1, unit: !0)
+!5 = !DILocation(line: 1, column: 22, scope: !4)
+!6 = !DILocation(line: 2, column: 22, scope: !4)
+!7 = !DILocation(line: 3, column: 22, scope: !4)
+!8 = !{!"branch_weights", i32 5, i32 7}
+!9 = !{!"branch_weights", i32 13, i32 11}
+;.
+; CHECK: [[META0:![0-9]+]] = !{i32 7, !"Dwarf Version", i32 5}
+; CHECK: [[META1:![0-9]+]] = !{i32 2, !"Debug Info Version", i32 3}
+; CHECK: [[PROF2]] = !{!"branch_weights", i32 5, i32 7}
+; CHECK: [[DBG3]] = !DILocation(line: 2, column: 22, scope: [[META4:![0-9]+]])
+; CHECK: [[META4]] = distinct !DISubprogram(name: "triangle_phi", linkageName: "triangle_phi", scope: [[META5:![0-9]+]], file: [[META5]], line: 1, spFlags: DISPFlagDefinition, unit: [[META6:![0-9]+]])
+; CHECK: [[META5]] = !DIFile(filename: "{{.*}}t", directory: {{.*}})
+; CHECK: [[META6]] = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: [[META5]], isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug)
+; CHECK: [[DBG7]] = !DILocation(line: 3, column: 22, scope: [[META4]])
+; CHECK: [[PROF8]] = !{!"branch_weights", i32 13, i32 11}
+;.
diff --git a/llvm/test/Transforms/LICM/hoist-profdata.ll b/llvm/test/Transforms/LICM/hoist-profdata.ll
new file mode 100644
index 0000000000000..18fa1b9f92e8a
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-profdata.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 2
+; Test that hoisting conditional branches copies the debug and profiling info
+; metadata from the branch being hoisted.
+; RUN: opt -S -passes=licm %s -o - | FileCheck %s
+
+declare i32 @foo()
+
+; to_hoist should get hoisted, and that should not result
+; in a loss of profiling info
+define i32 @hoist_select(i1 %cond, i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: define i32 @hoist_select
+; CHECK-SAME: (i1 [[COND:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TO_HOIST:%.*]] = select i1 [[COND]], i32 [[A]], i32 [[B]], !prof [[PROF0:![0-9]+]]
+; CHECK-NEXT:    br label [[L0:%.*]]
+; CHECK:       L0:
+; CHECK-NEXT:    [[G:%.*]] = call i32 @foo()
+; CHECK-NEXT:    [[SUM:%.*]] = add i32 [[G]], [[TO_HOIST]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[SUM]], 0
+; CHECK-NEXT:    br i1 [[C]], label [[L0]], label [[EXIT:%.*]], !prof [[PROF1:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[SUM_LCSSA:%.*]] = phi i32 [ [[SUM]], [[L0]] ]
+; CHECK-NEXT:    ret i32 [[SUM_LCSSA]]
+;
+entry:
+  br label %L0
+L0:
+  %g = call i32 @foo()
+  %to_hoist = select i1 %cond, i32 %a, i32 %b, !prof !0
+  %sum = add i32 %g, %to_hoist
+  %c = icmp eq i32 %sum, 0
+  br i1 %c, label %L0, label %exit, !prof !1
+
+exit:
+  ret i32 %sum
+}
+
+!0 = !{!"branch_weights", i32 2, i32 5}
+!1 = !{!"branch_weights", i32 101, i32 189}
+;.
+; CHECK: attributes #[[ATTR0]] = { nounwind }
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 2, i32 5}
+; CHECK: [[PROF1]] = !{!"branch_weights", i32 101, i32 189}
+;.
diff --git a/llvm/test/Transforms/LICM/hoist-readonly-memory-def.ll b/llvm/test/Transforms/LICM/hoist-readonly-memory-def.ll
new file mode 100644
index 0000000000000..887a0d05264f5
--- /dev/null
+++ b/llvm/test/Transforms/LICM/hoist-readonly-memory-def.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes="function(print<memoryssa>),cgscc(function-attrs),function(loop-mssa(licm))" < %s 2>/dev/null | FileCheck %s
+
+@g = external global i16
+
+define i16 @fn() {
+; CHECK-LABEL: define i16 @fn(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[V:%.*]] = load i16, ptr @g, align 2
+; CHECK-NEXT:    ret i16 [[V]]
+;
+  %v = load i16, ptr @g
+  ret i16 %v
+}
+
+declare void @fn2(i16)
+
+define void @test() {
+; CHECK-LABEL: define void @test() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[CALL:%.*]] = call i16 @fn(i32 0)
+; CHECK-NEXT:    call void @fn2(i16 [[CALL]])
+; CHECK-NEXT:    br i1 false, label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %call = call i16 @fn(i32 0) ; intentional signature mismatch
+  call void @fn2(i16 %call)
+  br i1 false, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LICM/hoisting-preheader-debugloc.ll b/llvm/test/Transforms/LICM/hoisting-preheader-debugloc.ll
index 61f0eb19a9bd1..0aa56d2ce431e 100644
--- a/llvm/test/Transforms/LICM/hoisting-preheader-debugloc.ll
+++ b/llvm/test/Transforms/LICM/hoisting-preheader-debugloc.ll
@@ -17,7 +17,7 @@ declare i16 @e(i32)
 define i16 @g() !dbg !13 {
 entry:
   %l_284 = alloca [2 x [3 x [6 x i32]]], align 16
-  call void @llvm.lifetime.start.p0(i64 144, ptr nonnull %l_284), !dbg !24
+  call void @llvm.lifetime.start.p0(ptr nonnull %l_284), !dbg !24
   call void @llvm.dbg.declare(metadata ptr %l_284, metadata !17, metadata !DIExpression()), !dbg !25
   %0 = load i16, ptr @a, align 2, !dbg !26, !tbaa !29
   %cmp11 = icmp sgt i16 %0, -1, !dbg !33
@@ -51,15 +51,15 @@ for.body.cleanup_crit_edge:                       ; preds = %for.body
   br label %cleanup, !dbg !38
 
 cleanup:                                          ; preds = %for.body.cleanup_crit_edge, %for.cond.cleanup_crit_edge, %entry
-  call void @llvm.lifetime.end.p0(i64 144, ptr nonnull %l_284), !dbg !51
+  call void @llvm.lifetime.end.p0(ptr nonnull %l_284), !dbg !51
   ret i16 1, !dbg !51
 }
 
 ; Function Attrs: argmemonly nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: argmemonly nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind readnone speculatable willreturn
 declare void @llvm.dbg.value(metadata, metadata, metadata) #0
diff --git a/llvm/test/Transforms/LICM/licm-coroutine.ll b/llvm/test/Transforms/LICM/licm-coroutine.ll
new file mode 100644
index 0000000000000..a4765acfb93f8
--- /dev/null
+++ b/llvm/test/Transforms/LICM/licm-coroutine.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=licm -S | FileCheck %s
+
+; %fca.0 and %fca.1 should not be hoisted out of the loop because the ramp
+; function and resume function have different stack frames, so %pointer1 and
+; %pointer2 have different values before and after @llvm.coro.suspend.
+
+define ptr @f(i32 %n) presplitcoroutine {
+; CHECK-LABEL: define ptr @f(
+; CHECK-SAME: i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[POINTER1:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[POINTER2:%.*]] = alloca ptr, align 8
+; CHECK-NEXT:    [[ID:%.*]] = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
+; CHECK-NEXT:    [[SIZE:%.*]] = call i32 @llvm.coro.size.i32()
+; CHECK-NEXT:    [[ALLOC:%.*]] = call ptr @malloc(i32 [[SIZE]])
+; CHECK-NEXT:    [[HDL:%.*]] = call noalias ptr @llvm.coro.begin(token [[ID]], ptr [[ALLOC]])
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[N_VAL:%.*]] = phi i32 [ [[N]], %[[ENTRY]] ], [ [[INC:%.*]], %[[RESUME:.*]] ]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[N_VAL]], 1
+; CHECK-NEXT:    call void @print(i32 [[N_VAL]])
+; CHECK-NEXT:    [[TMP0:%.*]] = call i8 @llvm.coro.suspend(token none, i1 false)
+; CHECK-NEXT:    switch i8 [[TMP0]], label %[[SUSPEND_LOOPEXIT:.*]] [
+; CHECK-NEXT:      i8 0, label %[[RESUME]]
+; CHECK-NEXT:      i8 1, label %[[CLEANUP:.*]]
+; CHECK-NEXT:    ]
+; CHECK:       [[RESUME]]:
+; CHECK-NEXT:    [[FCA_0:%.*]] = insertvalue [2 x ptr] poison, ptr [[POINTER1]], 0
+; CHECK-NEXT:    [[FCA_1:%.*]] = insertvalue [2 x ptr] [[FCA_0]], ptr [[POINTER2]], 1
+; CHECK-NEXT:    call void @foo([2 x ptr] [[FCA_1]])
+; CHECK-NEXT:    br label %[[LOOP]]
+; CHECK:       [[CLEANUP]]:
+; CHECK-NEXT:    [[MEM:%.*]] = call ptr @llvm.coro.free(token [[ID]], ptr [[HDL]])
+; CHECK-NEXT:    call void @free(ptr [[MEM]])
+; CHECK-NEXT:    br label %[[SUSPEND:.*]]
+; CHECK:       [[SUSPEND_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[SUSPEND]]
+; CHECK:       [[SUSPEND]]:
+; CHECK-NEXT:    [[UNUSED:%.*]] = call i1 @llvm.coro.end(ptr [[HDL]], i1 false, token none)
+; CHECK-NEXT:    ret ptr [[HDL]]
+;
+entry:
+  %pointer1 = alloca ptr
+  %pointer2 = alloca ptr
+  %id = call token @llvm.coro.id(i32 0, ptr null, ptr null, ptr null)
+  %size = call i32 @llvm.coro.size.i32()
+  %alloc = call ptr @malloc(i32 %size)
+  %hdl = call noalias ptr @llvm.coro.begin(token %id, ptr %alloc)
+  br label %loop
+
+loop:
+  %n.val = phi i32 [ %n, %entry ], [ %inc, %resume ]
+  %inc = add nsw i32 %n.val, 1
+  call void @print(i32 %n.val)
+  %0 = call i8 @llvm.coro.suspend(token none, i1 false)
+  switch i8 %0, label %suspend [i8 0, label %resume
+  i8 1, label %cleanup]
+
+resume:
+  %fca.0 = insertvalue [2 x ptr] poison, ptr %pointer1, 0
+  %fca.1 = insertvalue [2 x ptr] %fca.0, ptr %pointer2, 1
+  call void @foo([2 x ptr] %fca.1)
+  br label %loop
+
+cleanup:
+  %mem = call ptr @llvm.coro.free(token %id, ptr %hdl)
+  call void @free(ptr %mem)
+  br label %suspend
+suspend:
+  %unused = call i1 @llvm.coro.end(ptr %hdl, i1 false, token none)
+  ret ptr %hdl
+}
+
+declare void @free(ptr)
+declare ptr @malloc(i32)
+declare void @print(i32)
+declare void @foo([2 x ptr])
diff --git a/llvm/test/Transforms/LICM/loopsink-pr38462.ll b/llvm/test/Transforms/LICM/loopsink-pr38462.ll
index 51eee1fbbde2b..8b2ff104cbb69 100644
--- a/llvm/test/Transforms/LICM/loopsink-pr38462.ll
+++ b/llvm/test/Transforms/LICM/loopsink-pr38462.ll
@@ -37,7 +37,7 @@ __except:
   catchret from %1 to label %__except3
 
 __except3:
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %s)
+  call void @llvm.lifetime.start.p0(ptr nonnull %s)
   %call.i = call zeroext i1 @g(ptr nonnull %s)
   br i1 %call.i, label %if.then.i, label %exit
 
@@ -46,7 +46,7 @@ if.then.i:
   br label %exit
 
 exit:
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %s)
+  call void @llvm.lifetime.end.p0(ptr nonnull %s)
   br label %__try.cont
 
 __try.cont:
@@ -58,8 +58,8 @@ __try.cont:
 declare i32 @__C_specific_handler(...)
 declare i32 @f()
 declare zeroext i1 @g(ptr)
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 !1 = !{!"function_entry_count", i64 1}
 
diff --git a/llvm/test/Transforms/LICM/sink-foldable.ll b/llvm/test/Transforms/LICM/sink-foldable.ll
index d1cf3de5301b2..59dea58a6adee 100644
--- a/llvm/test/Transforms/LICM/sink-foldable.ll
+++ b/llvm/test/Transforms/LICM/sink-foldable.ll
@@ -97,7 +97,7 @@ define ptr @test2(i32 %j, ptr readonly %P, ptr readnone %Q) {
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds ptr, ptr [[ADD_PTR]], i64 [[IDX2_EXT]]
 ; CHECK-NEXT:    [[L1:%.*]] = load ptr, ptr [[ARRAYIDX2]], align 8
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt ptr [[L1]], [[Q]]
-; CHECK-NEXT:    [[ADD_REASS]] = add i32 [[I_ADDR]], 2
+; CHECK-NEXT:    [[ADD_REASS]] = add nsw i32 [[ADD_I]], 1
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOPEXIT2:%.*]], label [[FOR_COND]]
 ; CHECK:       loopexit0:
 ; CHECK-NEXT:    [[P0:%.*]] = phi ptr [ null, [[FOR_COND]] ]
diff --git a/llvm/test/Transforms/LICM/update-scev-after-hoist.ll b/llvm/test/Transforms/LICM/update-scev-after-hoist.ll
index e303d04ce3191..8f9045350813a 100644
--- a/llvm/test/Transforms/LICM/update-scev-after-hoist.ll
+++ b/llvm/test/Transforms/LICM/update-scev-after-hoist.ll
@@ -4,7 +4,7 @@
 define i16 @main() {
 ; SCEV-EXPR-LABEL: 'main'
 ; SCEV-EXPR-NEXT:  Classifying expressions for: @main
-; SCEV-EXPR-NEXT:    %mul = phi i16 [ 1, %entry ], [ %mul.n.3.reass, %loop ]
+; SCEV-EXPR-NEXT:    %mul = phi i16 [ 1, %entry ], [ %mul.n.3, %loop ]
 ; SCEV-EXPR-NEXT:    --> %mul U: [0,-15) S: [-32768,32753) Exits: 4096 LoopDispositions: { %loop: Variant }
 ; SCEV-EXPR-NEXT:    %div = phi i16 [ 32767, %entry ], [ %div.n.3, %loop ]
 ; SCEV-EXPR-NEXT:    --> %div U: [-2048,-32768) S: [-2048,-32768) Exits: 7 LoopDispositions: { %loop: Variant }
@@ -16,7 +16,7 @@ define i16 @main() {
 ; SCEV-EXPR-NEXT:    --> %div.n.1 U: [-8192,8192) S: [-8192,8192) Exits: 1 LoopDispositions: { %loop: Variant }
 ; SCEV-EXPR-NEXT:    %div.n.2 = sdiv i16 %div.n.1, 2
 ; SCEV-EXPR-NEXT:    --> %div.n.2 U: [-4096,4096) S: [-4096,4096) Exits: 0 LoopDispositions: { %loop: Variant }
-; SCEV-EXPR-NEXT:    %mul.n.3.reass = mul i16 %mul, 16
+; SCEV-EXPR-NEXT:    %mul.n.3 = mul i16 %mul.n.reass.reass, 2
 ; SCEV-EXPR-NEXT:    --> (16 * %mul) U: [0,-15) S: [-32768,32753) Exits: 0 LoopDispositions: { %loop: Variant }
 ; SCEV-EXPR-NEXT:    %div.n.3 = sdiv i16 %div.n.2, 2
 ; SCEV-EXPR-NEXT:    --> %div.n.3 U: [-2048,2048) S: [-2048,2048) Exits: 0 LoopDispositions: { %loop: Variant }
diff --git a/llvm/test/Transforms/LoopDistribute/cross-partition-access.ll b/llvm/test/Transforms/LoopDistribute/cross-partition-access.ll
new file mode 100644
index 0000000000000..f9c1b365a6b2b
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/cross-partition-access.ll
@@ -0,0 +1,160 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -aa-pipeline=basic-aa -passes=loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -S %s | FileCheck %s
+
+; Test emit safety guards for ptr access of %a and %c on a cross parition load which
+; has a corresponding store to the same address. This ensures that if %a and %c
+; overlap in some scenarios, that we execute the original loop for safety reasons.
+
+define dso_local void @_Z13distribution3PiS_S_S_i(ptr nocapture noundef %a, ptr nocapture noundef readonly %b, ptr nocapture noundef %c, ptr nocapture noundef writeonly %d, i64 noundef signext %len) {
+; CHECK-LABEL: define dso_local void @_Z13distribution3PiS_S_S_i(
+; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], ptr noundef readonly captures(none) [[B:%.*]], ptr noundef captures(none) [[C:%.*]], ptr noundef writeonly captures(none) [[D:%.*]], i64 noundef signext [[LEN:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[LEN]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label %[[END:.*]], label %[[FOR_BODY_LVER_CHECK:.*]]
+; CHECK:       [[FOR_BODY_LVER_CHECK]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[LEN]], 2
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[C]], i64 -4
+; CHECK-NEXT:    [[SCEVGEP5:%.*]] = getelementptr i8, ptr [[C]], i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[D]], i64 [[TMP0]]
+; CHECK-NEXT:    [[SCEVGEP4:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[A]], [[SCEVGEP5]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND1]], [[BOUND0]]
+; CHECK-NEXT:    [[BOUND07:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT:    [[BOUND18:%.*]] = icmp ult ptr [[D]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[FOUND_CONFLICT9:%.*]] = and i1 [[BOUND07]], [[BOUND18]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT9]]
+; CHECK-NEXT:    [[BOUND010:%.*]] = icmp ult ptr [[A]], [[SCEVGEP4]]
+; CHECK-NEXT:    [[BOUND111:%.*]] = icmp ult ptr [[B]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[FOUND_CONFLICT12:%.*]] = and i1 [[BOUND010]], [[BOUND111]]
+; CHECK-NEXT:    [[CONFLICT_RDX13:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT12]]
+; CHECK-NEXT:    [[BOUND014:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP3]]
+; CHECK-NEXT:    [[BOUND115:%.*]] = icmp ult ptr [[D]], [[SCEVGEP5]]
+; CHECK-NEXT:    [[FOUND_CONFLICT16:%.*]] = and i1 [[BOUND014]], [[BOUND115]]
+; CHECK-NEXT:    [[CONFLICT_RDX17:%.*]] = or i1 [[CONFLICT_RDX13]], [[FOUND_CONFLICT16]]
+; CHECK-NEXT:    [[BOUND018:%.*]] = icmp ult ptr [[D]], [[SCEVGEP4]]
+; CHECK-NEXT:    [[BOUND119:%.*]] = icmp ult ptr [[B]], [[SCEVGEP3]]
+; CHECK-NEXT:    [[FOUND_CONFLICT20:%.*]] = and i1 [[BOUND018]], [[BOUND119]]
+; CHECK-NEXT:    [[CONFLICT_RDX21:%.*]] = or i1 [[CONFLICT_RDX17]], [[FOUND_CONFLICT20]]
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX21]], label %[[FOR_BODY_PH_LVER_ORIG:.*]], label %[[FOR_BODY_PH_LDIST1:.*]]
+; CHECK:       [[FOR_BODY_PH_LVER_ORIG]]:
+; CHECK-NEXT:    br label %[[FOR_BODY_LVER_ORIG:.*]]
+; CHECK:       [[FOR_BODY_LVER_ORIG]]:
+; CHECK-NEXT:    [[IDXPROM_LVER_ORIG:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH_LVER_ORIG]] ], [ [[I6_LVER_ORIG:%.*]], %[[FOR_BODY_LVER_ORIG]] ]
+; CHECK-NEXT:    [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IDXPROM_LVER_ORIG]]
+; CHECK-NEXT:    [[I2_LVER_ORIG:%.*]] = load i32, ptr [[ARRAYIDX_LVER_ORIG]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT:    [[ADD4_LVER_ORIG:%.*]] = add nsw i32 [[I2_LVER_ORIG]], 1
+; CHECK-NEXT:    [[ARRAYIDX8_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM_LVER_ORIG]]
+; CHECK-NEXT:    store i32 [[ADD4_LVER_ORIG]], ptr [[ARRAYIDX8_LVER_ORIG]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT:    [[I3_LVER_ORIG:%.*]] = getelementptr i32, ptr [[C]], i64 [[IDXPROM_LVER_ORIG]]
+; CHECK-NEXT:    [[ARRAYIDX17_LVER_ORIG:%.*]] = getelementptr i8, ptr [[I3_LVER_ORIG]], i64 -4
+; CHECK-NEXT:    [[I4_LVER_ORIG:%.*]] = load i32, ptr [[ARRAYIDX17_LVER_ORIG]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT:    [[SUB18_LVER_ORIG:%.*]] = sub nsw i32 [[ADD4_LVER_ORIG]], [[I4_LVER_ORIG]]
+; CHECK-NEXT:    store i32 [[SUB18_LVER_ORIG]], ptr [[I3_LVER_ORIG]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT:    [[I5_LVER_ORIG:%.*]] = load i32, ptr [[ARRAYIDX8_LVER_ORIG]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT:    [[ADD27_LVER_ORIG:%.*]] = add nsw i32 [[I5_LVER_ORIG]], 2
+; CHECK-NEXT:    [[ARRAYIDX31_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[IDXPROM_LVER_ORIG]]
+; CHECK-NEXT:    store i32 [[ADD27_LVER_ORIG]], ptr [[ARRAYIDX31_LVER_ORIG]], align 4, !tbaa [[TBAA0]]
+; CHECK-NEXT:    [[I6_LVER_ORIG]] = add i64 [[IDXPROM_LVER_ORIG]], 1
+; CHECK-NEXT:    [[CMP1_NOT_LVER_ORIG:%.*]] = icmp eq i64 [[I6_LVER_ORIG]], [[LEN]]
+; CHECK-NEXT:    br i1 [[CMP1_NOT_LVER_ORIG]], label %[[END_LOOPEXIT_LOOPEXIT:.*]], label %[[FOR_BODY_LVER_ORIG]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[FOR_BODY_PH_LDIST1]]:
+; CHECK-NEXT:    br label %[[FOR_BODY_LDIST1:.*]]
+; CHECK:       [[FOR_BODY_LDIST1]]:
+; CHECK-NEXT:    [[IDXPROM_LDIST1:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH_LDIST1]] ], [ [[I6_LDIST1:%.*]], %[[FOR_BODY_LDIST1]] ]
+; CHECK-NEXT:    [[ARRAYIDX_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IDXPROM_LDIST1]]
+; CHECK-NEXT:    [[I2_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META7:![0-9]+]]
+; CHECK-NEXT:    [[ADD4_LDIST1:%.*]] = add nsw i32 [[I2_LDIST1]], 1
+; CHECK-NEXT:    [[ARRAYIDX8_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM_LDIST1]]
+; CHECK-NEXT:    store i32 [[ADD4_LDIST1]], ptr [[ARRAYIDX8_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META10:![0-9]+]], !noalias [[META12:![0-9]+]]
+; CHECK-NEXT:    [[I3_LDIST1:%.*]] = getelementptr i32, ptr [[C]], i64 [[IDXPROM_LDIST1]]
+; CHECK-NEXT:    [[ARRAYIDX17_LDIST1:%.*]] = getelementptr i8, ptr [[I3_LDIST1]], i64 -4
+; CHECK-NEXT:    [[I4_LDIST1:%.*]] = load i32, ptr [[ARRAYIDX17_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META15:![0-9]+]], !noalias [[META16:![0-9]+]]
+; CHECK-NEXT:    [[SUB18_LDIST1:%.*]] = sub nsw i32 [[ADD4_LDIST1]], [[I4_LDIST1]]
+; CHECK-NEXT:    store i32 [[SUB18_LDIST1]], ptr [[I3_LDIST1]], align 4, !tbaa [[TBAA0]], !alias.scope [[META15]], !noalias [[META16]]
+; CHECK-NEXT:    [[I6_LDIST1]] = add i64 [[IDXPROM_LDIST1]], 1
+; CHECK-NEXT:    [[CMP1_NOT_LDIST1:%.*]] = icmp eq i64 [[I6_LDIST1]], [[LEN]]
+; CHECK-NEXT:    br i1 [[CMP1_NOT_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       [[FOR_BODY_PH]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[IDXPROM:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH]] ], [ [[I6:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[I5:%.*]] = load i32, ptr [[ARRAYIDX8]], align 4, !tbaa [[TBAA0]], !alias.scope [[META10]], !noalias [[META12]]
+; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[I5]], 2
+; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[IDXPROM]]
+; CHECK-NEXT:    store i32 [[ADD27]], ptr [[ARRAYIDX31]], align 4, !tbaa [[TBAA0]], !alias.scope [[META16]], !noalias [[META7]]
+; CHECK-NEXT:    [[I6]] = add i64 [[IDXPROM]], 1
+; CHECK-NEXT:    [[CMP1_NOT:%.*]] = icmp eq i64 [[I6]], [[LEN]]
+; CHECK-NEXT:    br i1 [[CMP1_NOT]], label %[[END_LOOPEXIT_LOOPEXIT20:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP17]]
+; CHECK:       [[END_LOOPEXIT_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[END_LOOPEXIT:.*]]
+; CHECK:       [[END_LOOPEXIT_LOOPEXIT20]]:
+; CHECK-NEXT:    br label %[[END_LOOPEXIT]]
+; CHECK:       [[END_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[END]]
+; CHECK:       [[END]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp = icmp sgt i64 %len, 0
+  br i1 %cmp, label %end, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.preheader
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, ptr %b, i64 %indvars.iv
+  %i2 = load i32, ptr %arrayidx, align 4, !tbaa !0
+  %add4 = add nsw i32 %i2, 1
+  %arrayidx8 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+  store i32 %add4, ptr %arrayidx8, align 4, !tbaa !0
+  %i3 = getelementptr i32, ptr %c, i64 %indvars.iv
+  %arrayidx17 = getelementptr i8, ptr %i3, i64 -4
+  %i4 = load i32, ptr %arrayidx17, align 4, !tbaa !0
+  %sub18 = sub nsw i32 %add4, %i4
+  store i32 %sub18, ptr %i3, align 4, !tbaa !0
+  %i5 = load i32, ptr %arrayidx8, align 4, !tbaa !0
+  %add27 = add nsw i32 %i5, 2
+  %arrayidx31 = getelementptr inbounds i32, ptr %d, i64 %indvars.iv
+  store i32 %add27, ptr %arrayidx31, align 4, !tbaa !0
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %cmp1.not = icmp eq i64 %indvars.iv.next, %len
+  br i1 %cmp1.not, label %end.loopexit, label %for.body, !llvm.loop !4
+
+end.loopexit:                                     ; preds = %for.body
+  br label %end
+
+end:                                              ; preds = %end.loopexit, %entry
+  ret void
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"int", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C++ TBAA"}
+!4 = distinct !{!4, !5}
+!5 = !{!"llvm.loop.mustprogress"}
+
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; CHECK: [[META5]] = !{!"llvm.loop.mustprogress"}
+; CHECK: [[META6]] = !{!"llvm.loop.isdistributed", i32 1}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META9:![0-9]+]]}
+; CHECK: [[META9]] = distinct !{[[META9]], !"LVerDomain"}
+; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
+; CHECK: [[META11]] = distinct !{[[META11]], [[META9]]}
+; CHECK: [[META12]] = !{[[META13:![0-9]+]], [[META14:![0-9]+]], [[META8]]}
+; CHECK: [[META13]] = distinct !{[[META13]], [[META9]]}
+; CHECK: [[META14]] = distinct !{[[META14]], [[META9]]}
+; CHECK: [[META15]] = !{[[META13]]}
+; CHECK: [[META16]] = !{[[META14]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]}
+;.
diff --git a/llvm/test/Transforms/LoopDistribute/followup.ll b/llvm/test/Transforms/LoopDistribute/followup.ll
index 55307bdf24991..ae418ad9e6b42 100644
--- a/llvm/test/Transforms/LoopDistribute/followup.ll
+++ b/llvm/test/Transforms/LoopDistribute/followup.ll
@@ -58,29 +58,29 @@ define void @f(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) {
 ; CHECK:       [[FOR_BODY_LDIST1]]:
 ; CHECK-NEXT:    [[IND_LDIST1:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH_LDIST1]] ], [ [[ADD_LDIST1:%.*]], %[[FOR_BODY_LDIST1]] ]
 ; CHECK-NEXT:    [[ARRAYIDXA_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IND_LDIST1]]
-; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META3:![0-9]+]], !noalias [[META6:![0-9]+]]
+; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDXB_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IND_LDIST1]]
-; CHECK-NEXT:    [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4, !alias.scope [[META10:![0-9]+]]
+; CHECK-NEXT:    [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4, !alias.scope [[META11:![0-9]+]]
 ; CHECK-NEXT:    [[MULA_LDIST1:%.*]] = mul i32 [[LOADB_LDIST1]], [[LOADA_LDIST1]]
 ; CHECK-NEXT:    [[ADD_LDIST1]] = add nuw nsw i64 [[IND_LDIST1]], 1
 ; CHECK-NEXT:    [[ARRAYIDXA_PLUS_4_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD_LDIST1]]
-; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META3]], !noalias [[META6]]
+; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META4]], !noalias [[META7]]
 ; CHECK-NEXT:    [[EXITCOND_LDIST1:%.*]] = icmp eq i64 [[ADD_LDIST1]], 20
-; CHECK-NEXT:    br i1 [[EXITCOND_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_LDIST1]], label %[[FOR_BODY_PH:.*]], label %[[FOR_BODY_LDIST1]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       [[FOR_BODY_PH]]:
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[IND:%.*]] = phi i64 [ 0, %[[FOR_BODY_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ADD]] = add nuw nsw i64 [[IND]], 1
 ; CHECK-NEXT:    [[ARRAYIDXD:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[IND]]
-; CHECK-NEXT:    [[LOADD:%.*]] = load i32, ptr [[ARRAYIDXD]], align 4, !alias.scope [[META14:![0-9]+]]
+; CHECK-NEXT:    [[LOADD:%.*]] = load i32, ptr [[ARRAYIDXD]], align 4, !alias.scope [[META15:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDXE:%.*]] = getelementptr inbounds i32, ptr [[E]], i64 [[IND]]
-; CHECK-NEXT:    [[LOADE:%.*]] = load i32, ptr [[ARRAYIDXE]], align 4, !alias.scope [[META15:![0-9]+]]
+; CHECK-NEXT:    [[LOADE:%.*]] = load i32, ptr [[ARRAYIDXE]], align 4, !alias.scope [[META16:![0-9]+]]
 ; CHECK-NEXT:    [[MULC:%.*]] = mul i32 [[LOADD]], [[LOADE]]
 ; CHECK-NEXT:    [[ARRAYIDXC:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IND]]
-; CHECK-NEXT:    store i32 [[MULC]], ptr [[ARRAYIDXC]], align 4, !alias.scope [[META16:![0-9]+]], !noalias [[META10]]
+; CHECK-NEXT:    store i32 [[MULC]], ptr [[ARRAYIDXC]], align 4, !alias.scope [[META17:![0-9]+]], !noalias [[META11]]
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[ADD]], 20
-; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT16:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END_LOOPEXIT16:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       [[FOR_END_LOOPEXIT]]:
 ; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[FOR_END_LOOPEXIT16]]:
@@ -131,23 +131,24 @@ for.end:
 !4 = !{!"llvm.loop.distribute.followup_sequential", !{!"FollowupSequential", i32 8}}
 !5 = !{!"llvm.loop.distribute.followup_fallback", !{!"FollowupFallback"}}
 ;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"FollowupAll"}
 ; CHECK: [[META2]] = !{!"FollowupFallback"}
-; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
-; CHECK: [[META4]] = distinct !{[[META4]], [[META5:![0-9]+]]}
-; CHECK: [[META5]] = distinct !{[[META5]], !"LVerDomain"}
-; CHECK: [[META6]] = !{[[META7:![0-9]+]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
-; CHECK: [[META7]] = distinct !{[[META7]], [[META5]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META5]]}
-; CHECK: [[META9]] = distinct !{[[META9]], [[META5]]}
-; CHECK: [[META10]] = !{[[META11:![0-9]+]]}
-; CHECK: [[META11]] = distinct !{[[META11]], [[META5]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META13:![0-9]+]]}
-; CHECK: [[META13]] = !{!"FollowupSequential", i32 8}
-; CHECK: [[META14]] = !{[[META8]]}
+; CHECK: [[META3]] = !{!"llvm.loop.isdistributed", i32 1}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]], [[META9:![0-9]+]], [[META10:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
+; CHECK: [[META9]] = distinct !{[[META9]], [[META6]]}
+; CHECK: [[META10]] = distinct !{[[META10]], [[META6]]}
+; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], [[META6]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]], [[META14:![0-9]+]]}
+; CHECK: [[META14]] = !{!"FollowupSequential", i32 8}
 ; CHECK: [[META15]] = !{[[META9]]}
-; CHECK: [[META16]] = !{[[META7]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META18:![0-9]+]]}
-; CHECK: [[META18]] = !{!"FollowupCoincident", i1 false}
+; CHECK: [[META16]] = !{[[META10]]}
+; CHECK: [[META17]] = !{[[META8]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META19:![0-9]+]]}
+; CHECK: [[META19]] = !{!"FollowupCoincident", i1 false}
 ;.
diff --git a/llvm/test/Transforms/LoopDistribute/laa-invalidation.ll b/llvm/test/Transforms/LoopDistribute/laa-invalidation.ll
index ee42860cd250e..62c5627ac2d38 100644
--- a/llvm/test/Transforms/LoopDistribute/laa-invalidation.ll
+++ b/llvm/test/Transforms/LoopDistribute/laa-invalidation.ll
@@ -27,13 +27,13 @@ define void @test_pr50940(ptr %A, ptr %B) {
 ; CHECK-NEXT:    store i16 1, ptr [[B]], align 1
 ; CHECK-NEXT:    [[IV_NEXT_LVER_ORIG]] = add nuw nsw i16 [[IV_LVER_ORIG]], 1
 ; CHECK-NEXT:    [[C_1_LVER_ORIG:%.*]] = icmp ult i16 [[IV_LVER_ORIG]], 38
-; CHECK-NEXT:    br i1 [[C_1_LVER_ORIG]], label [[INNER_LVER_ORIG]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK-NEXT:    br i1 [[C_1_LVER_ORIG]], label [[INNER_LVER_ORIG]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       inner.ph3.ldist1:
 ; CHECK-NEXT:    br label [[INNER_LDIST1:%.*]]
 ; CHECK:       inner.ldist1:
 ; CHECK-NEXT:    [[IV_LDIST1:%.*]] = phi i16 [ 0, [[INNER_PH3_LDIST1]] ], [ [[IV_NEXT_LDIST1:%.*]], [[INNER_LDIST1]] ]
-; CHECK-NEXT:    [[L_LDIST1:%.*]] = load <2 x i16>, ptr [[UGLYGEP]], align 1, !alias.scope !0, !noalias !3
-; CHECK-NEXT:    store i16 0, ptr [[GEP_A_3]], align 1, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    [[L_LDIST1:%.*]] = load <2 x i16>, ptr [[UGLYGEP]], align 1, !alias.scope [[META2:![0-9]+]], !noalias [[META5:![0-9]+]]
+; CHECK-NEXT:    store i16 0, ptr [[GEP_A_3]], align 1, !alias.scope [[META2]], !noalias [[META5]]
 ; CHECK-NEXT:    [[IV_NEXT_LDIST1]] = add nuw nsw i16 [[IV_LDIST1]], 1
 ; CHECK-NEXT:    [[C_1_LDIST1:%.*]] = icmp ult i16 [[IV_LDIST1]], 38
 ; CHECK-NEXT:    br i1 [[C_1_LDIST1]], label [[INNER_LDIST1]], label [[INNER_PH3:%.*]]
@@ -41,7 +41,7 @@ define void @test_pr50940(ptr %A, ptr %B) {
 ; CHECK-NEXT:    br label [[INNER:%.*]]
 ; CHECK:       inner:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[INNER_PH3]] ], [ [[IV_NEXT:%.*]], [[INNER]] ]
-; CHECK-NEXT:    store i16 1, ptr [[B]], align 1, !alias.scope !3
+; CHECK-NEXT:    store i16 1, ptr [[B]], align 1, !alias.scope [[META5]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i16 [[IV]], 1
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp ult i16 [[IV]], 38
 ; CHECK-NEXT:    br i1 [[C_1]], label [[INNER]], label [[EXIT_LOOPEXIT4:%.*]]
diff --git a/llvm/test/Transforms/LoopDistribute/no-reprocess.ll b/llvm/test/Transforms/LoopDistribute/no-reprocess.ll
new file mode 100644
index 0000000000000..2dbc77aa756f6
--- /dev/null
+++ b/llvm/test/Transforms/LoopDistribute/no-reprocess.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -aa-pipeline=basic-aa -passes=loop-distribute -enable-loop-distribute -verify-loop-info -verify-dom-info -debug-only=loop-distribute --disable-output -S < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Test if a loop already distributed will not reprocess because of metadata
+; information marking it as processed.
+
+; CHECK: LDist: Distributed loop guarded for reprocessing
+; CHECK: LDist: Skipping;
+
+define dso_local void @_Z13distribution3PiS_S_S_i(ptr noundef captures(none) %a, ptr noundef readonly captures(none) %b, ptr noundef captures(none) %c, ptr noundef writeonly captures(none) %d, i64 noundef signext %len) {
+entry:
+  %cmp = icmp sgt i64 %len, 0
+  br i1 %cmp, label %end, label %for.body.lver.check
+
+for.body.lver.check:                              ; preds = %entry
+  %0 = shl i64 %len, 2
+  %scevgep = getelementptr i8, ptr %a, i64 %0
+  %scevgep1 = getelementptr i8, ptr %c, i64 -4
+  %scevgep2 = getelementptr i8, ptr %c, i64 %0
+  %scevgep3 = getelementptr i8, ptr %d, i64 %0
+  %scevgep4 = getelementptr i8, ptr %b, i64 %0
+  %bound0 = icmp ult ptr %a, %scevgep2
+  %bound1 = icmp ult ptr %scevgep1, %scevgep
+  %found.conflict = and i1 %bound0, %bound1
+  %bound05 = icmp ult ptr %a, %scevgep3
+  %bound16 = icmp ult ptr %d, %scevgep
+  %found.conflict7 = and i1 %bound05, %bound16
+  %conflict.rdx = or i1 %found.conflict, %found.conflict7
+  %bound08 = icmp ult ptr %a, %scevgep4
+  %bound19 = icmp ult ptr %b, %scevgep
+  %found.conflict10 = and i1 %bound08, %bound19
+  %conflict.rdx11 = or i1 %conflict.rdx, %found.conflict10
+  %bound012 = icmp ult ptr %scevgep1, %scevgep3
+  %bound113 = icmp ult ptr %d, %scevgep2
+  %found.conflict14 = and i1 %bound012, %bound113
+  %conflict.rdx15 = or i1 %conflict.rdx11, %found.conflict14
+  %bound016 = icmp ult ptr %d, %scevgep4
+  %bound117 = icmp ult ptr %b, %scevgep3
+  %found.conflict18 = and i1 %bound016, %bound117
+  %conflict.rdx19 = or i1 %conflict.rdx15, %found.conflict18
+  br i1 %conflict.rdx19, label %for.body.ph.lver.orig, label %for.body.ph.ldist1
+
+for.body.ph.lver.orig:                            ; preds = %for.body.lver.check
+  br label %for.body.lver.orig
+
+for.body.lver.orig:                               ; preds = %for.body.lver.orig, %for.body.ph.lver.orig
+  %indvars.iv.lver.orig = phi i64 [ 0, %for.body.ph.lver.orig ], [ %indvars.iv.next.lver.orig, %for.body.lver.orig ]
+  %arrayidx.lver.orig = getelementptr inbounds i32, ptr %b, i64 %indvars.iv.lver.orig
+  %i2.lver.orig = load i32, ptr %arrayidx.lver.orig, align 4, !tbaa !0
+  %add4.lver.orig = add nsw i32 %i2.lver.orig, 1
+  %arrayidx8.lver.orig = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.lver.orig
+  store i32 %add4.lver.orig, ptr %arrayidx8.lver.orig, align 4, !tbaa !0
+  %i3.lver.orig = getelementptr i32, ptr %c, i64 %indvars.iv.lver.orig
+  %arrayidx17.lver.orig = getelementptr i8, ptr %i3.lver.orig, i64 -4
+  %i4.lver.orig = load i32, ptr %arrayidx17.lver.orig, align 4, !tbaa !0
+  %sub18.lver.orig = sub nsw i32 %add4.lver.orig, %i4.lver.orig
+  store i32 %sub18.lver.orig, ptr %i3.lver.orig, align 4, !tbaa !0
+  %i5.lver.orig = load i32, ptr %arrayidx8.lver.orig, align 4, !tbaa !0
+  %add27.lver.orig = add nsw i32 %i5.lver.orig, 2
+  %arrayidx31.lver.orig = getelementptr inbounds i32, ptr %d, i64 %indvars.iv.lver.orig
+  store i32 %add27.lver.orig, ptr %arrayidx31.lver.orig, align 4, !tbaa !0
+  %indvars.iv.next.lver.orig = add i64 %indvars.iv.lver.orig, 1
+  %cmp1.not.lver.orig = icmp eq i64 %indvars.iv.next.lver.orig, %len
+  br i1 %cmp1.not.lver.orig, label %end.loopexit.loopexit, label %for.body.lver.orig, !llvm.loop !4
+
+for.body.ph.ldist1:                               ; preds = %for.body.lver.check
+  br label %for.body.ldist1
+
+for.body.ldist1:                                  ; preds = %for.body.ldist1, %for.body.ph.ldist1
+  %indvars.iv.ldist1 = phi i64 [ 0, %for.body.ph.ldist1 ], [ %indvars.iv.next.ldist1, %for.body.ldist1 ]
+  %arrayidx.ldist1 = getelementptr inbounds i32, ptr %b, i64 %indvars.iv.ldist1
+  %i2.ldist1 = load i32, ptr %arrayidx.ldist1, align 4, !tbaa !0, !alias.scope !7
+  %add4.ldist1 = add nsw i32 %i2.ldist1, 1
+  %arrayidx8.ldist1 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv.ldist1
+  store i32 %add4.ldist1, ptr %arrayidx8.ldist1, align 4, !tbaa !0, !alias.scope !10, !noalias !12
+  %i3.ldist1 = getelementptr i32, ptr %c, i64 %indvars.iv.ldist1
+  %arrayidx17.ldist1 = getelementptr i8, ptr %i3.ldist1, i64 -4
+  %i4.ldist1 = load i32, ptr %arrayidx17.ldist1, align 4, !tbaa !0, !alias.scope !15, !noalias !16
+  %sub18.ldist1 = sub nsw i32 %add4.ldist1, %i4.ldist1
+  store i32 %sub18.ldist1, ptr %i3.ldist1, align 4, !tbaa !0, !alias.scope !15, !noalias !16
+  %indvars.iv.next.ldist1 = add i64 %indvars.iv.ldist1, 1
+  %cmp1.not.ldist1 = icmp eq i64 %indvars.iv.next.ldist1, %len
+  br i1 %cmp1.not.ldist1, label %for.body.ph, label %for.body.ldist1, !llvm.loop !17
+
+for.body.ph:                                      ; preds = %for.body.ldist1
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %for.body.ph
+  %indvars.iv = phi i64 [ 0, %for.body.ph ], [ %indvars.iv.next, %for.body ]
+  %arrayidx8 = getelementptr inbounds i32, ptr %a, i64 %indvars.iv
+  %i5 = load i32, ptr %arrayidx8, align 4, !tbaa !0, !alias.scope !10, !noalias !12
+  %add27 = add nsw i32 %i5, 2
+  %arrayidx31 = getelementptr inbounds i32, ptr %d, i64 %indvars.iv
+  store i32 %add27, ptr %arrayidx31, align 4, !tbaa !0, !alias.scope !16, !noalias !7
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %cmp1.not = icmp eq i64 %indvars.iv.next, %len
+  br i1 %cmp1.not, label %end.loopexit.loopexit20, label %for.body, !llvm.loop !17
+
+end.loopexit.loopexit:                            ; preds = %for.body.lver.orig
+  br label %end.loopexit
+
+end.loopexit.loopexit20:                          ; preds = %for.body
+  br label %end.loopexit
+
+end.loopexit:                                     ; preds = %end.loopexit.loopexit20, %end.loopexit.loopexit
+  br label %end
+
+end:                                              ; preds = %end.loopexit, %entry
+  ret void
+}
+
+!0 = !{!1, !1, i64 0}
+!1 = !{!"int", !2, i64 0}
+!2 = !{!"omnipotent char", !3, i64 0}
+!3 = !{!"Simple C++ TBAA"}
+!4 = distinct !{!4, !5, !6}
+!5 = !{!"llvm.loop.mustprogress"}
+!6 = !{!"llvm.loop.isdistributed", i32 1}
+!7 = !{!8}
+!8 = distinct !{!8, !9}
+!9 = distinct !{!9, !"LVerDomain"}
+!10 = !{!11}
+!11 = distinct !{!11, !9}
+!12 = !{!13, !14, !8}
+!13 = distinct !{!13, !9}
+!14 = distinct !{!14, !9}
+!15 = !{!13}
+!16 = !{!14}
+!17 = distinct !{!17, !5}
diff --git a/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll b/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll
index 98b89abfeafda..dd23a1121ebcb 100644
--- a/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll
+++ b/llvm/test/Transforms/LoopDistribute/scev-inserted-runtime-check.ll
@@ -36,7 +36,7 @@ define void @f(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %d, p
 ; CHECK-NEXT:    [[ARRAYIDXC_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[MUL_EXT_LVER_ORIG]]
 ; CHECK-NEXT:    store i32 [[MULC_LVER_ORIG]], ptr [[ARRAYIDXC_LVER_ORIG]], align 4
 ; CHECK-NEXT:    [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[ADD_LVER_ORIG]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]]
+; CHECK-NEXT:    br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       for.body.ph.ldist1:
 ; CHECK-NEXT:    br label [[FOR_BODY_LDIST1:%.*]]
 ; CHECK:       for.body.ldist1:
@@ -45,14 +45,14 @@ define void @f(ptr noalias %a, ptr noalias %b, ptr noalias %c, ptr noalias %d, p
 ; CHECK-NEXT:    [[MUL_LDIST1:%.*]] = mul i32 [[IND1_LDIST1]], 2
 ; CHECK-NEXT:    [[MUL_EXT_LDIST1:%.*]] = zext i32 [[MUL_LDIST1]] to i64
 ; CHECK-NEXT:    [[ARRAYIDXA_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_EXT_LDIST1]]
-; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META2:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDXB_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[MUL_EXT_LDIST1]]
 ; CHECK-NEXT:    [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4
 ; CHECK-NEXT:    [[MULA_LDIST1:%.*]] = mul i32 [[LOADB_LDIST1]], [[LOADA_LDIST1]]
 ; CHECK-NEXT:    [[ADD_LDIST1]] = add nuw nsw i64 [[IND_LDIST1]], 1
 ; CHECK-NEXT:    [[INC1_LDIST1]] = add i32 [[IND1_LDIST1]], 1
 ; CHECK-NEXT:    [[ARRAYIDXA_PLUS_4_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD_LDIST1]]
-; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META3:![0-9]+]]
+; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META5:![0-9]+]]
 ; CHECK-NEXT:    [[EXITCOND_LDIST1:%.*]] = icmp eq i64 [[ADD_LDIST1]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_LDIST1]], label [[FOR_BODY_PH:%.*]], label [[FOR_BODY_LDIST1]]
 ; CHECK:       for.body.ph:
@@ -161,7 +161,7 @@ define void @f_with_offset(ptr noalias %b, ptr noalias %c, ptr noalias %d, ptr n
 ; CHECK-NEXT:    [[ARRAYIDXC_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[MUL_EXT_LVER_ORIG]]
 ; CHECK-NEXT:    store i32 [[MULC_LVER_ORIG]], ptr [[ARRAYIDXC_LVER_ORIG]], align 4
 ; CHECK-NEXT:    [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[ADD_LVER_ORIG]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]]
+; CHECK-NEXT:    br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       for.body.ph.ldist1:
 ; CHECK-NEXT:    br label [[FOR_BODY_LDIST1:%.*]]
 ; CHECK:       for.body.ldist1:
@@ -170,14 +170,14 @@ define void @f_with_offset(ptr noalias %b, ptr noalias %c, ptr noalias %d, ptr n
 ; CHECK-NEXT:    [[MUL_LDIST1:%.*]] = mul i32 [[IND1_LDIST1]], 2
 ; CHECK-NEXT:    [[MUL_EXT_LDIST1:%.*]] = zext i32 [[MUL_LDIST1]] to i64
 ; CHECK-NEXT:    [[ARRAYIDXA_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_EXT_LDIST1]]
-; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META5:![0-9]+]]
+; CHECK-NEXT:    [[LOADA_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXA_LDIST1]], align 4, !alias.scope [[META8:![0-9]+]]
 ; CHECK-NEXT:    [[ARRAYIDXB_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[MUL_EXT_LDIST1]]
 ; CHECK-NEXT:    [[LOADB_LDIST1:%.*]] = load i32, ptr [[ARRAYIDXB_LDIST1]], align 4
 ; CHECK-NEXT:    [[MULA_LDIST1:%.*]] = mul i32 [[LOADB_LDIST1]], [[LOADA_LDIST1]]
 ; CHECK-NEXT:    [[ADD_LDIST1]] = add nuw nsw i64 [[IND_LDIST1]], 1
 ; CHECK-NEXT:    [[INC1_LDIST1]] = add i32 [[IND1_LDIST1]], 1
 ; CHECK-NEXT:    [[ARRAYIDXA_PLUS_4_LDIST1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[ADD_LDIST1]]
-; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META8:![0-9]+]]
+; CHECK-NEXT:    store i32 [[MULA_LDIST1]], ptr [[ARRAYIDXA_PLUS_4_LDIST1]], align 4, !alias.scope [[META11:![0-9]+]]
 ; CHECK-NEXT:    [[EXITCOND_LDIST1:%.*]] = icmp eq i64 [[ADD_LDIST1]], [[N]]
 ; CHECK-NEXT:    br i1 [[EXITCOND_LDIST1]], label [[FOR_BODY_PH:%.*]], label [[FOR_BODY_LDIST1]]
 ; CHECK:       for.body.ph:
diff --git a/llvm/test/Transforms/LoopIdiom/expand-scev-expand-simplifications.ll b/llvm/test/Transforms/LoopIdiom/expand-scev-expand-simplifications.ll
new file mode 100644
index 0000000000000..9a59e5a8ccabb
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/expand-scev-expand-simplifications.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-idiom -S %s | FileCheck %s
+
+target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32"
+
+define void @test_simplify_scev_during_expansion_flags(i64 %start) {
+; CHECK-LABEL: define void @test_simplify_scev_during_expansion_flags(
+; CHECK-SAME: i64 [[START:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[START_NEG:%.*]] = sub i64 0, [[START]]
+; CHECK-NEXT:    [[START_MUL:%.*]] = ashr exact i64 [[START_NEG]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[START_MUL]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 404, [[TMP0]]
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 null, i8 0, i64 [[TMP1]], i1 false)
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[START_MUL]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 100
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %start.neg = sub i64 0, %start
+  %start.mul = ashr exact i64 %start.neg, 2
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %start.mul, %entry ], [ %iv.next, %loop ]
+  %ptr.iv = phi ptr [ null, %entry ], [ %ptr.iv.next, %loop ]
+  store i32 0, ptr %ptr.iv, align 4
+  %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 4
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopIdiom/introduce-memset-in-outerloop.ll b/llvm/test/Transforms/LoopIdiom/introduce-memset-in-outerloop.ll
new file mode 100644
index 0000000000000..bf78ebae3dc3d
--- /dev/null
+++ b/llvm/test/Transforms/LoopIdiom/introduce-memset-in-outerloop.ll
@@ -0,0 +1,47 @@
+; RUN: opt <%s -p "print<scalar-evolution>" -disable-output 2>&1 | FileCheck %s
+; RUN: opt <%s -p "loop(loop-idiom),print<scalar-evolution>" -disable-output 2>&1 | FileCheck %s
+
+; CHECK: backedge-taken count is i64 1
+
+; IR corresponds to the following C test:
+; extern char a[];
+; void foo() {
+;   for (long c = 0; c < 6; c += -2078836808675943215)
+;     for (long d; d < 6; d++)
+;       a[c + d] = 0;
+; }
+
+@a = external global [0 x i8]
+
+define void @foo()  {
+entry:
+  br label %outerL
+
+outerL:                              ; preds = %entry, %outerLatch
+  %e = phi i64 [ poison, %entry ], [ %lcssa, %outerLatch ]
+  %c = phi i64 [ 0, %entry ], [ %c.next, %outerLatch ]
+  %e.cmp = icmp slt i64 %e, 6
+  br i1 %e.cmp, label %innerL, label %outerLatch
+
+innerL:                                          ; preds = %outerL, %innerL
+  %d = phi i64 [ %d.next, %innerL ], [ %e, %outerL ]
+  %add = add nsw i64 %d, %c
+  %arrayidx = getelementptr inbounds [0 x i8], ptr @a, i64 0, i64 %add
+  store i8 0, ptr %arrayidx
+  %d.next = add nsw i64 %d, 1
+  %d.cmp = icmp slt i64 %d, 5
+  br i1 %d.cmp, label %innerL, label %outerLatch, !llvm.loop !0
+
+outerLatch:                                         ; preds = %innerL, %outerL
+  %lcssa = phi i64 [ %e, %outerL ], [ %d.next, %innerL ]
+  %c.next = add nsw i64 %c, -2078836808675943215
+  %c.cmp = icmp slt i64 %c, 2078836808675943221
+  br i1 %c.cmp, label %outerL, label %exit, !llvm.loop !1
+
+exit:                                         ; preds = %outerLatch
+  ret void
+}
+
+!0 = distinct !{!0, !2}
+!1 = distinct !{!1, !2}
+!2 = !{!"llvm.loop.mustprogress"}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
index c7a0de22b200b..970643ac7bc7d 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/lifetime-use.ll
@@ -6,7 +6,7 @@ define void @test(ptr %p, i64 %idx) {
 ; CHECK-SAME: ptr [[P:%.*]], i64 [[IDX:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca [4 x [4 x i32]], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ALLOCA]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = shl i64 [[IDX]], 6
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 48
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP1]]
@@ -31,12 +31,12 @@ define void @test(ptr %p, i64 %idx) {
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[LSR_IV_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ALLOCA]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %alloca = alloca [4 x [4 x i32]], align 16
-  call void @llvm.lifetime.start.p0(i64 64, ptr %alloca)
+  call void @llvm.lifetime.start.p0(ptr %alloca)
   br label %loop
 
 loop:
@@ -54,6 +54,6 @@ loop:
   br i1 %exitcond.not, label %exit, label %loop
 
 exit:
-  call void @llvm.lifetime.end.p0(i64 64, ptr %alloca)
+  call void @llvm.lifetime.end.p0(ptr %alloca)
   ret void
 }
diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-comp-time.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-comp-time.ll
index 3ec4fea7b9671..f8c5e8244df4a 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/lsr-comp-time.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-comp-time.ll
@@ -5,10 +5,10 @@
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind readnone uwtable
 define dso_local i32 @foo(i32 %arg, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, i32 %arg6) local_unnamed_addr #3 {
@@ -83,31 +83,31 @@ bb:
   %tmp16 = alloca [100 x [100 x i32]], align 16
   %tmp17 = alloca [100 x [100 x i32]], align 16
   %tmp18 = alloca [100 x [100 x i32]], align 16
-  call void @llvm.lifetime.start.p0(i64 400, ptr nonnull %tmp) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp, i8 0, i64 400, i1 false)
-  call void @llvm.lifetime.start.p0(i64 400, ptr nonnull %tmp7) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp7) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp7, i8 0, i64 400, i1 false)
-  call void @llvm.lifetime.start.p0(i64 400, ptr nonnull %tmp8) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp8) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp8, i8 0, i64 400, i1 false)
-  call void @llvm.lifetime.start.p0(i64 40000, ptr nonnull %tmp9) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp9) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp9, i8 0, i64 40000, i1 false)
-  call void @llvm.lifetime.start.p0(i64 400, ptr nonnull %tmp10) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp10) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp10, i8 0, i64 400, i1 false)
-  call void @llvm.lifetime.start.p0(i64 40000, ptr nonnull %tmp11) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp11) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp11, i8 0, i64 40000, i1 false)
-  call void @llvm.lifetime.start.p0(i64 400, ptr nonnull %tmp12) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp12) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp12, i8 0, i64 400, i1 false)
-  call void @llvm.lifetime.start.p0(i64 400, ptr nonnull %tmp13) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp13) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp13, i8 0, i64 400, i1 false)
-  call void @llvm.lifetime.start.p0(i64 40000, ptr nonnull %tmp14) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp14) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp14, i8 0, i64 40000, i1 false)
-  call void @llvm.lifetime.start.p0(i64 400, ptr nonnull %tmp15) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp15) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp15, i8 0, i64 400, i1 false)
-  call void @llvm.lifetime.start.p0(i64 40000, ptr nonnull %tmp16) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp16) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp16, i8 0, i64 40000, i1 false)
-  call void @llvm.lifetime.start.p0(i64 40000, ptr nonnull %tmp17) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp17) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp17, i8 0, i64 40000, i1 false)
-  call void @llvm.lifetime.start.p0(i64 40000, ptr nonnull %tmp18) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %tmp18) #4
   call void @llvm.memset.p0.i64(ptr nonnull align 16 %tmp18, i8 0, i64 40000, i1 false)
   %tmp32 = getelementptr inbounds [100 x i32], ptr %tmp8, i64 0, i64 3
   br label %bb33
@@ -1300,19 +1300,19 @@ bb1051:                                           ; preds = %bb1007
   %tmp1063 = sub i32 %tmp1062, %tmp960
   %tmp1064 = add i32 %tmp1063, %tmp1004
   %tmp1065 = sub i32 %tmp1064, %tmp1048
-  call void @llvm.lifetime.end.p0(i64 40000, ptr nonnull %tmp18) #4
-  call void @llvm.lifetime.end.p0(i64 40000, ptr nonnull %tmp17) #4
-  call void @llvm.lifetime.end.p0(i64 40000, ptr nonnull %tmp16) #4
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %tmp15) #4
-  call void @llvm.lifetime.end.p0(i64 40000, ptr nonnull %tmp14) #4
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %tmp13) #4
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %tmp12) #4
-  call void @llvm.lifetime.end.p0(i64 40000, ptr nonnull %tmp11) #4
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %tmp10) #4
-  call void @llvm.lifetime.end.p0(i64 40000, ptr nonnull %tmp9) #4
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %tmp8) #4
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %tmp7) #4
-  call void @llvm.lifetime.end.p0(i64 400, ptr nonnull %tmp) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp18) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp17) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp16) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp15) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp14) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp13) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp12) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp11) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp10) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp9) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp8) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp7) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %tmp) #4
   ret i32 %tmp1065
 }
 
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll b/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll
index 0b78beea54aa9..a62e45d27a5c5 100644
--- a/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/apple-unrolling.ll
@@ -165,6 +165,204 @@ exit:
   ret void
 }
 
+define void @load_op_store_loop(ptr %src, ptr %dst, i64 %N, i64 %scale, float %k) {
+; APPLE-LABEL: define void @load_op_store_loop(
+; APPLE-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[SCALE:%.*]], float [[K:%.*]]) #[[ATTR0]] {
+; APPLE-NEXT:  [[ENTRY:.*]]:
+; APPLE-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
+; APPLE-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 1
+; APPLE-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; APPLE-NEXT:    br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]]
+; APPLE:       [[ENTRY_NEW]]:
+; APPLE-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
+; APPLE-NEXT:    br label %[[LOOP:.*]]
+; APPLE:       [[LOOP]]:
+; APPLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
+; APPLE-NEXT:    [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
+; APPLE-NEXT:    [[SCALED_IV:%.*]] = mul nuw nsw i64 [[IV]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV]]
+; APPLE-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; APPLE-NEXT:    [[O:%.*]] = fadd float [[L]], [[K]]
+; APPLE-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
+; APPLE-NEXT:    store float [[O]], ptr [[GEP_DST]], align 4
+; APPLE-NEXT:    [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
+; APPLE-NEXT:    [[SCALED_IV_1:%.*]] = mul nuw nsw i64 [[IV_NEXT]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_1]]
+; APPLE-NEXT:    [[L_1:%.*]] = load float, ptr [[GEP_SRC_1]], align 4
+; APPLE-NEXT:    [[O_1:%.*]] = fadd float [[L_1]], [[K]]
+; APPLE-NEXT:    [[GEP_DST_1:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT]]
+; APPLE-NEXT:    store float [[O_1]], ptr [[GEP_DST_1]], align 4
+; APPLE-NEXT:    [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
+; APPLE-NEXT:    [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; APPLE-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; APPLE-NEXT:    br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]]
+; APPLE:       [[EXIT_UNR_LCSSA_LOOPEXIT]]:
+; APPLE-NEXT:    [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
+; APPLE-NEXT:    br label %[[EXIT_UNR_LCSSA]]
+; APPLE:       [[EXIT_UNR_LCSSA]]:
+; APPLE-NEXT:    [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ]
+; APPLE-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; APPLE-NEXT:    br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]]
+; APPLE:       [[LOOP_EPIL_PREHEADER]]:
+; APPLE-NEXT:    br label %[[LOOP_EPIL:.*]]
+; APPLE:       [[LOOP_EPIL]]:
+; APPLE-NEXT:    [[SCALED_IV_EPIL:%.*]] = mul nuw nsw i64 [[IV_UNR]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC_EPIL:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_EPIL]]
+; APPLE-NEXT:    [[L_EPIL:%.*]] = load float, ptr [[GEP_SRC_EPIL]], align 4
+; APPLE-NEXT:    [[O_EPIL:%.*]] = fadd float [[L_EPIL]], [[K]]
+; APPLE-NEXT:    [[GEP_DST_EPIL:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_UNR]]
+; APPLE-NEXT:    store float [[O_EPIL]], ptr [[GEP_DST_EPIL]], align 4
+; APPLE-NEXT:    br label %[[EXIT]]
+; APPLE:       [[EXIT]]:
+; APPLE-NEXT:    ret void
+;
+; OTHER-LABEL: define void @load_op_store_loop(
+; OTHER-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[SCALE:%.*]], float [[K:%.*]]) #[[ATTR0]] {
+; OTHER-NEXT:  [[ENTRY:.*]]:
+; OTHER-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
+; OTHER-NEXT:    [[XTRAITER:%.*]] = and i64 [[N]], 1
+; OTHER-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[TMP0]], 1
+; OTHER-NEXT:    br i1 [[TMP1]], label %[[EXIT_UNR_LCSSA:.*]], label %[[ENTRY_NEW:.*]]
+; OTHER:       [[ENTRY_NEW]]:
+; OTHER-NEXT:    [[UNROLL_ITER:%.*]] = sub i64 [[N]], [[XTRAITER]]
+; OTHER-NEXT:    br label %[[LOOP:.*]]
+; OTHER:       [[LOOP]]:
+; OTHER-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_1:%.*]], %[[LOOP]] ]
+; OTHER-NEXT:    [[NITER:%.*]] = phi i64 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_1:%.*]], %[[LOOP]] ]
+; OTHER-NEXT:    [[SCALED_IV:%.*]] = mul nuw nsw i64 [[IV]], [[SCALE]]
+; OTHER-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV]]
+; OTHER-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; OTHER-NEXT:    [[O:%.*]] = fadd float [[L]], [[K]]
+; OTHER-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
+; OTHER-NEXT:    store float [[O]], ptr [[GEP_DST]], align 4
+; OTHER-NEXT:    [[IV_NEXT:%.*]] = add nuw nsw i64 [[IV]], 1
+; OTHER-NEXT:    [[SCALED_IV_1:%.*]] = mul nuw nsw i64 [[IV_NEXT]], [[SCALE]]
+; OTHER-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_1]]
+; OTHER-NEXT:    [[L_1:%.*]] = load float, ptr [[GEP_SRC_1]], align 4
+; OTHER-NEXT:    [[O_1:%.*]] = fadd float [[L_1]], [[K]]
+; OTHER-NEXT:    [[GEP_DST_1:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_NEXT]]
+; OTHER-NEXT:    store float [[O_1]], ptr [[GEP_DST_1]], align 4
+; OTHER-NEXT:    [[IV_NEXT_1]] = add nuw nsw i64 [[IV]], 2
+; OTHER-NEXT:    [[NITER_NEXT_1]] = add i64 [[NITER]], 2
+; OTHER-NEXT:    [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NEXT_1]], [[UNROLL_ITER]]
+; OTHER-NEXT:    br i1 [[NITER_NCMP_1]], label %[[EXIT_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP]]
+; OTHER:       [[EXIT_UNR_LCSSA_LOOPEXIT]]:
+; OTHER-NEXT:    [[IV_UNR_PH:%.*]] = phi i64 [ [[IV_NEXT_1]], %[[LOOP]] ]
+; OTHER-NEXT:    br label %[[EXIT_UNR_LCSSA]]
+; OTHER:       [[EXIT_UNR_LCSSA]]:
+; OTHER-NEXT:    [[IV_UNR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_UNR_PH]], %[[EXIT_UNR_LCSSA_LOOPEXIT]] ]
+; OTHER-NEXT:    [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; OTHER-NEXT:    br i1 [[LCMP_MOD]], label %[[LOOP_EPIL_PREHEADER:.*]], label %[[EXIT:.*]]
+; OTHER:       [[LOOP_EPIL_PREHEADER]]:
+; OTHER-NEXT:    br label %[[LOOP_EPIL:.*]]
+; OTHER:       [[LOOP_EPIL]]:
+; OTHER-NEXT:    [[SCALED_IV_EPIL:%.*]] = mul nuw nsw i64 [[IV_UNR]], [[SCALE]]
+; OTHER-NEXT:    [[GEP_SRC_EPIL:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV_EPIL]]
+; OTHER-NEXT:    [[L_EPIL:%.*]] = load float, ptr [[GEP_SRC_EPIL]], align 4
+; OTHER-NEXT:    [[O_EPIL:%.*]] = fadd float [[L_EPIL]], [[K]]
+; OTHER-NEXT:    [[GEP_DST_EPIL:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV_UNR]]
+; OTHER-NEXT:    store float [[O_EPIL]], ptr [[GEP_DST_EPIL]], align 4
+; OTHER-NEXT:    br label %[[EXIT]]
+; OTHER:       [[EXIT]]:
+; OTHER-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %scaled.iv = mul nuw nsw i64 %iv, %scale
+  %gep.src = getelementptr inbounds float, ptr %src, i64 %scaled.iv
+  %l = load float, ptr %gep.src, align 4
+  %o = fadd float %l, %k
+  %gep.dst = getelementptr inbounds float, ptr %dst, i64 %iv
+  store float %o, ptr %gep.dst, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @load_op_store_loop_multiblock(ptr %src, ptr %dst, i64 %N, i64 %scale, float %k) {
+; APPLE-LABEL: define void @load_op_store_loop_multiblock(
+; APPLE-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[SCALE:%.*]], float [[K:%.*]]) #[[ATTR0]] {
+; APPLE-NEXT:  [[ENTRY:.*]]:
+; APPLE-NEXT:    br label %[[LOOP:.*]]
+; APPLE:       [[LOOP]]:
+; APPLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOPCONT:.*]] ]
+; APPLE-NEXT:    [[SCALED_IV:%.*]] = mul nuw nsw i64 [[IV]], [[SCALE]]
+; APPLE-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV]]
+; APPLE-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; APPLE-NEXT:    [[AND:%.*]] = and i64 [[IV]], 1
+; APPLE-NEXT:    [[ODD:%.*]] = icmp eq i64 [[AND]], 1
+; APPLE-NEXT:    br i1 [[ODD]], label %[[LOOPODD:.*]], label %[[LOOPCONT]]
+; APPLE:       [[LOOPCONT]]:
+; APPLE-NEXT:    [[D:%.*]] = phi float [ [[L2:%.*]], %[[LOOPODD]] ], [ [[L]], %[[LOOP]] ]
+; APPLE-NEXT:    [[O:%.*]] = fadd float [[D]], [[K]]
+; APPLE-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
+; APPLE-NEXT:    store float [[O]], ptr [[GEP_DST]], align 4
+; APPLE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; APPLE-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; APPLE-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; APPLE:       [[LOOPODD]]:
+; APPLE-NEXT:    [[L2]] = fneg float [[L]]
+; APPLE-NEXT:    br label %[[LOOPCONT]]
+; APPLE:       [[EXIT]]:
+; APPLE-NEXT:    ret void
+;
+; OTHER-LABEL: define void @load_op_store_loop_multiblock(
+; OTHER-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]], i64 [[N:%.*]], i64 [[SCALE:%.*]], float [[K:%.*]]) #[[ATTR0]] {
+; OTHER-NEXT:  [[ENTRY:.*]]:
+; OTHER-NEXT:    br label %[[LOOP:.*]]
+; OTHER:       [[LOOP]]:
+; OTHER-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOPCONT:.*]] ]
+; OTHER-NEXT:    [[SCALED_IV:%.*]] = mul nuw nsw i64 [[IV]], [[SCALE]]
+; OTHER-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[SCALED_IV]]
+; OTHER-NEXT:    [[L:%.*]] = load float, ptr [[GEP_SRC]], align 4
+; OTHER-NEXT:    [[AND:%.*]] = and i64 [[IV]], 1
+; OTHER-NEXT:    [[ODD:%.*]] = icmp eq i64 [[AND]], 1
+; OTHER-NEXT:    br i1 [[ODD]], label %[[LOOPODD:.*]], label %[[LOOPCONT]]
+; OTHER:       [[LOOPCONT]]:
+; OTHER-NEXT:    [[D:%.*]] = phi float [ [[L2:%.*]], %[[LOOPODD]] ], [ [[L]], %[[LOOP]] ]
+; OTHER-NEXT:    [[O:%.*]] = fadd float [[D]], [[K]]
+; OTHER-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
+; OTHER-NEXT:    store float [[O]], ptr [[GEP_DST]], align 4
+; OTHER-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; OTHER-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; OTHER-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; OTHER:       [[LOOPODD]]:
+; OTHER-NEXT:    [[L2]] = fneg float [[L]]
+; OTHER-NEXT:    br label %[[LOOPCONT]]
+; OTHER:       [[EXIT]]:
+; OTHER-NEXT:    ret void
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loopcont ]
+  %scaled.iv = mul nuw nsw i64 %iv, %scale
+  %gep.src = getelementptr inbounds float, ptr %src, i64 %scaled.iv
+  %l1 = load float, ptr %gep.src, align 4
+  %and = and i64 %iv, 1
+  %odd = icmp eq i64 %and, 1
+  br i1 %odd, label %loopodd, label %loopcont
+loopcont:
+  %d = phi float [ %l2, %loopodd ], [ %l1, %loop]
+  %o = fadd float %d, %k
+  %gep.dst = getelementptr inbounds float, ptr %dst, i64 %iv
+  store float %o, ptr %gep.dst, align 4
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop
+loopodd:
+  %l2 = fneg float %l1
+  br label %loopcont
+exit:
+  ret void
+}
+
 @A = external constant [9 x i8], align 1
 @B = external constant [8 x i32], align 4
 @C = external constant [8 x i32], align 4
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-phi-analysis-iv-not-triggered.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-phi-analysis-iv-not-triggered.ll
new file mode 100644
index 0000000000000..747175a089a07
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-phi-analysis-iv-not-triggered.ll
@@ -0,0 +1,150 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -S -passes=loop-unroll -enable-peeling-for-iv | FileCheck %s
+; RUN: opt < %s -S -passes=loop-unroll-full -enable-peeling-for-iv | FileCheck %s
+
+; Check that unnecessary peeling doesn't occur if for a comparison instruction
+; between two instructions. The original code is as below. Both i and j are
+; inductions, but the comparison i < j is not an induction.
+;
+; val = 42;
+; for (i=0,j=100; i<10000; i+=2,j+=1) {
+;   a[i] = val;
+;   val = i < j;
+; }
+;
+define void @dont_peel_cmp_ind_ind(ptr %a) {
+; CHECK-LABEL: define void @dont_peel_cmp_ind_ind(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 100, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[VAL:%.*]] = phi i32 [ 42, %[[ENTRY]] ], [ [[VAL_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i32 [[I]]
+; CHECK-NEXT:    store i32 10, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[VAL_NEXT_CMP:%.*]] = icmp slt i32 [[I]], [[J]]
+; CHECK-NEXT:    [[VAL_NEXT]] = zext i1 [[VAL_NEXT_CMP]] to i32
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 2
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I_NEXT]], 10000
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp slt i32 [[I_NEXT]], 10000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %for.body ]
+  %j = phi i32 [ 100, %entry ] , [ %j.next, %for.body ]
+  %val = phi i32 [ 42, %entry ], [ %val.next, %for.body ]
+  %arrayidx = getelementptr inbounds nuw i32, ptr %a, i32 %i
+  store i32 10, ptr %arrayidx, align 4
+  %val.next.cmp = icmp slt i32 %i, %j
+  %val.next = zext i1 %val.next.cmp to i32
+  %i.next = add i32 %i, 2
+  %j.next = add i32 %j, 1
+  %cmp = icmp ne i32 %i.next, 10000
+  %exitcond = icmp slt i32 %i.next, 10000
+  br i1 %exitcond, label %for.body, label %exit
+
+exit:
+  ret void
+}
+
+
+; Check that unnecessary peeling doesn't occur if for a bitwise instructions
+; between IVs. The original code is as below. The variable i is an induction,
+; but vals (val0 through val4) are not.
+;
+; val0 = 42;
+; val1 = 42;
+; val2 = 42;
+; val3 = 42;
+; val4 = 42;
+; for (i=0,j=100; i<10000; i+=2,j+=1) {
+;   a[i] = val0;
+;   b[i] = val1;
+;   c[i] = val2;
+;   d[i] = val3;
+;   e[i] = val4;
+;   val0 = i & j;
+;   val1 = i | j;
+;   val2 = i ^ j;
+;   val3 = i >> j;
+;   val4 = i << j;
+; }
+;
+define void @dont_peel_bitwise_op_iv_iv(ptr %a, ptr %b, ptr %c, ptr %d, ptr %e) {
+; CHECK-LABEL: define void @dont_peel_bitwise_op_iv_iv(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 100, %[[ENTRY]] ], [ [[J_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[VAL0:%.*]] = phi i32 [ 42, %[[ENTRY]] ], [ [[VAL0_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[VAL1:%.*]] = phi i32 [ 42, %[[ENTRY]] ], [ [[VAL1_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[VAL2:%.*]] = phi i32 [ 42, %[[ENTRY]] ], [ [[VAL2_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = phi i32 [ 42, %[[ENTRY]] ], [ [[VAL3_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[VAL4:%.*]] = phi i32 [ 42, %[[ENTRY]] ], [ [[VAL4_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IDX_0:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i32 [[I]]
+; CHECK-NEXT:    [[IDX_1:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i32 [[I]]
+; CHECK-NEXT:    [[IDX_2:%.*]] = getelementptr inbounds nuw i32, ptr [[C]], i32 [[I]]
+; CHECK-NEXT:    [[IDX_3:%.*]] = getelementptr inbounds nuw i32, ptr [[D]], i32 [[I]]
+; CHECK-NEXT:    [[IDX_4:%.*]] = getelementptr inbounds nuw i32, ptr [[E]], i32 [[I]]
+; CHECK-NEXT:    store i32 [[VAL0]], ptr [[IDX_0]], align 4
+; CHECK-NEXT:    store i32 [[VAL1]], ptr [[IDX_1]], align 4
+; CHECK-NEXT:    store i32 [[VAL2]], ptr [[IDX_2]], align 4
+; CHECK-NEXT:    store i32 [[VAL3]], ptr [[IDX_3]], align 4
+; CHECK-NEXT:    store i32 [[VAL4]], ptr [[IDX_4]], align 4
+; CHECK-NEXT:    [[VAL0_NEXT]] = and i32 [[I]], [[J]]
+; CHECK-NEXT:    [[VAL1_NEXT]] = or i32 [[I]], [[J]]
+; CHECK-NEXT:    [[VAL2_NEXT]] = xor i32 [[I]], [[J]]
+; CHECK-NEXT:    [[VAL3_NEXT]] = shl i32 [[I]], [[J]]
+; CHECK-NEXT:    [[VAL4_NEXT]] = lshr i32 [[I]], [[J]]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 2
+; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[I_NEXT]], 10000
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp slt i32 [[I_NEXT]], 10000
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_BODY]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %for.body ]
+  %j = phi i32 [ 100, %entry ] , [ %j.next, %for.body ]
+  %val0 = phi i32 [ 42, %entry ], [ %val0.next, %for.body ]
+  %val1 = phi i32 [ 42, %entry ], [ %val1.next, %for.body ]
+  %val2 = phi i32 [ 42, %entry ], [ %val2.next, %for.body ]
+  %val3 = phi i32 [ 42, %entry ], [ %val3.next, %for.body ]
+  %val4 = phi i32 [ 42, %entry ], [ %val4.next, %for.body ]
+  %idx.0 = getelementptr inbounds nuw i32, ptr %a, i32 %i
+  %idx.1 = getelementptr inbounds nuw i32, ptr %b, i32 %i
+  %idx.2 = getelementptr inbounds nuw i32, ptr %c, i32 %i
+  %idx.3 = getelementptr inbounds nuw i32, ptr %d, i32 %i
+  %idx.4 = getelementptr inbounds nuw i32, ptr %e, i32 %i
+  store i32 %val0, ptr %idx.0, align 4
+  store i32 %val1, ptr %idx.1, align 4
+  store i32 %val2, ptr %idx.2, align 4
+  store i32 %val3, ptr %idx.3, align 4
+  store i32 %val4, ptr %idx.4, align 4
+  %val0.next = and i32 %i, %j
+  %val1.next = or i32 %i, %j
+  %val2.next = xor i32 %i, %j
+  %val3.next = shl i32 %i, %j
+  %val4.next = lshr i32 %i, %j
+  %i.next = add i32 %i, 2
+  %j.next = add i32 %j, 1
+  %cmp = icmp ne i32 %i.next, 10000
+  %exitcond = icmp slt i32 %i.next, 10000
+  br i1 %exitcond, label %for.body, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/peel-loop-phi-analysis-iv.ll b/llvm/test/Transforms/LoopUnroll/peel-loop-phi-analysis-iv.ll
new file mode 100644
index 0000000000000..4e16bafc65f03
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/peel-loop-phi-analysis-iv.ll
@@ -0,0 +1,247 @@
+; RUN: opt -passes=loop-unroll -enable-peeling-for-iv -disable-output \
+; RUN:     -pass-remarks-output=- %s | FileCheck %s
+; RUN: opt -passes=loop-unroll-full -enable-peeling-for-iv -disable-output \
+; RUN:     -pass-remarks-output=- %s | FileCheck %s
+
+; void g(int);
+declare void @g(i32)
+
+; Check that phi analysis can handle a binary operator with an addition or a
+; subtraction on loop-invariants or IVs. In the following case, the phis for x,
+; y, a, and b become IVs by peeling.
+;
+;
+; void g(int);
+; void binary() {
+;   int x = 0;
+;   int y = 0;
+;   int a = 42;
+;   int b = 314;
+;   for(int i = 0; i <100000; ++i) {
+;     g(x);
+;     g(b);
+;     x = y;
+;     y = a + 1;
+;     a = i - 2;
+;     b = i + a;
+;   }
+; }
+;
+;
+; Consider the calls to g:
+;
+;                |   i |    g(x) |     g(b) |   x |   y |   a |     b |
+; ---------------|-----|---------|----------|-----|-----|-----|-------|
+;  1st iteration |   0 |    g(0) |   g(314) |   0 |  43 |  -2 |    -2 |
+;  2nd iteration |   1 |    g(0) |    g(-2) |  43 |  -1 |  -1 |     0 |
+;  3rd iteration |   2 |   g(43) |     g(0) |  -1 |   0 |   0 |     2 |
+;  4th iteration |   3 |   g(-1) |     g(2) |   0 |   1 |   1 |     4 |
+;  5th iteration |   4 |    g(0) |     g(4) |   1 |   2 |   2 |     6 |
+; i-th iteration |   i |  g(i-5) | g(2*i-4) | i-4 | i-3 | i-2 | 2*i-4 |
+;
+; After the 4th iteration, the arguments to g become IVs, so peeling 3 times
+; converts all PHIs into IVs.
+;
+
+; CHECK:      --- !Passed
+; CHECK-NEXT: Pass:            loop-unroll
+; CHECK-NEXT: Name:            Peeled
+; CHECK-NEXT: Function:        binary_induction
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          ' peeled loop by '
+; CHECK-NEXT:   - PeelCount:       '3'
+; CHECK-NEXT:   - String:          ' iterations'
+; CHECK-NEXT: ...
+define void @binary_induction() {
+entry:
+  br label %for.body
+
+exit:
+  ret void
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %for.body ]
+  %x = phi i32 [ 0, %entry ], [ %y, %for.body ]
+  %y = phi i32 [ 0, %entry ], [ %y.next, %for.body ]
+  %a = phi i32 [ 42, %entry ], [ %a.next, %for.body ]
+  %b = phi i32 [ 314, %entry ], [ %b.next, %for.body ]
+  tail call void @g(i32 %x)
+  tail call void @g(i32 %b)
+  %i.next = add i32 %i, 1
+  %y.next = add i32 %a, 1
+  %a.next = sub i32 %i, 2
+  %b.next = add i32 %i, %a
+  %cmp = icmp ne i32 %i.next, 100000
+  br i1 %cmp, label %for.body, label %exit
+}
+
+; Check that peeling works fine in the following case. This is based on TSVC
+; s291, where peeling 1 time makes the variable im an IV so we can vectorize
+; the loop.
+;
+; #define N 100
+; void f(float * restrict a, float * restrict b) {
+;   int im = N - 1;
+;   for (int i = 0; i < N; i++) {
+;     a[i] = b[i] + b[im];
+;     im = i;
+;   }
+; }
+;
+
+; CHECK:      --- !Passed
+; CHECK-NEXT: Pass:            loop-unroll
+; CHECK-NEXT: Name:            Peeled
+; CHECK-NEXT: Function:        tsvc_s291
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          ' peeled loop by '
+; CHECK-NEXT:   - PeelCount:       '1'
+; CHECK-NEXT:   - String:          ' iterations'
+; CHECK-NEXT: ...
+define void @tsvc_s291(ptr noalias %a, ptr noalias %b) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [0, %entry], [ %i.next, %for.body ]
+  %im = phi i32 [ 99, %entry ], [ %i, %for.body ]
+  %a.idx = getelementptr inbounds float, ptr %a, i32 %i
+  %b.idx.0 = getelementptr inbounds float, ptr %b, i32 %i
+  %b.idx.1 = getelementptr inbounds float, ptr %b, i32 %im
+  %lhs = load float, ptr %b.idx.0, align 4
+  %rhs = load float, ptr %b.idx.1, align 4
+  %sum = fadd float %lhs, %rhs
+  store float %sum, ptr %a.idx, align 4
+  %i.next = add i32 %i, 1
+  %cmp = icmp ne i32 %i.next, 100
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  ret void
+}
+
+; Check that the unnecessary peeling occurs in the following case. The cause is
+; that the analyzer determines a casted IV as a non-IV.
+;
+; for (unsigned int i=0; i<10000; i++)
+;   a[(unsigned long)j] = 10;
+;
+
+; CHECK:      --- !Passed
+; CHECK-NEXT: Pass:            loop-unroll
+; CHECK-NEXT: Name:            Peeled
+; CHECK-NEXT: Function:        induction_undesirable_peel1
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          ' peeled loop by '
+; CHECK-NEXT:   - PeelCount:       '1'
+; CHECK-NEXT:   - String:          ' iterations'
+; CHECK-NEXT: ...
+define void @induction_undesirable_peel1(ptr %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %conv = phi i64 [ 0, %entry ], [ %conv.next, %for.body ]
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds nuw i32, ptr %a, i64 %conv
+  store i32 10, ptr %arrayidx, align 4
+  %iv.next = add nsw nuw i32 %iv, 1
+  %conv.next = zext i32 %iv.next to i64
+  %cmp = icmp ugt i64 10000, %conv.next
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  ret void
+}
+
+; Check that the unnecessary peeling occurs in the following case. The analyzer
+; cannot detect that the difference between the initial value of %i and %j is
+; equal to the step value of the %i.
+;
+; int j = 0;
+; for (int i=1; i<N; i++) {
+;   a[j] = 10;
+;   j = i;
+; }
+
+; CHECK:      --- !Passed
+; CHECK-NEXT: Pass:            loop-unroll
+; CHECK-NEXT: Name:            Peeled
+; CHECK-NEXT: Function:        induction_undesirable_peel2
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          ' peeled loop by '
+; CHECK-NEXT:   - PeelCount:       '1'
+; CHECK-NEXT:   - String:          ' iterations'
+; CHECK-NEXT: ...
+define void @induction_undesirable_peel2(ptr %a) {
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 1, %entry ], [ %i.next, %for.body ]
+  %j = phi i32 [ 0, %entry ], [ %i, %for.body ]
+  %arrayidx = getelementptr i32, ptr %a, i32 %j
+  store i32 10, ptr %arrayidx, align 4
+  %i.next = add i32 %i, 1
+  %cmp = icmp slt i32 %i, 10000
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  ret void
+}
+
+; Check that phi analysis can handle a binary operator with an addition or a
+; subtraction on loop-invariants or IVs. In the following case, the phis for x,
+; y, a, and b become IVs by peeling.
+;
+;
+; void g(int);
+; void binary() {
+;   int x = 2;
+;   for(int i = 0; i <100000; ++i) {
+;     g(x);
+;     tmp = i - 2;
+;     x = i - tmp;
+;   }
+; }
+;
+;
+; Consider the calls to g:
+;
+;                | i | g(x) | tmp | x |
+; ---------------|---|------|-----|---|
+;  1st iteration | 0 | g(2) |  -2 | 2 |
+;  2nd iteration | 1 | g(2) |  -1 | 2 |
+;  3rd iteration | 2 | g(2) |   0 | 2 |
+;  4th iteration | 3 | g(2) |   1 | 2 |
+;
+; In this case, the value of x is always 2. However, the analyzer recognizes
+; the expression "i - tmp" as an IV.
+;
+
+; CHECK:      --- !Passed
+; CHECK-NEXT: Pass:            loop-unroll
+; CHECK-NEXT: Name:            Peeled
+; CHECK-NEXT: Function:        induction_undesirable_peel3
+; CHECK-NEXT: Args:
+; CHECK-NEXT:   - String:          ' peeled loop by '
+; CHECK-NEXT:   - PeelCount:       '1'
+; CHECK-NEXT:   - String:          ' iterations'
+; CHECK-NEXT: ...
+define void @induction_undesirable_peel3() {
+entry:
+  br label %for.body
+
+exit:
+  ret void
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %for.body ]
+  %x = phi i32 [ 0, %entry ], [ %x.next, %for.body ]
+  tail call void @g(i32 %x)
+  %tmp = sub i32 %i, 2
+  %x.next = sub i32 %i, %tmp
+  %i.next = add i32 %i, 1
+  %cmp = icmp ne i32 %i.next, 100000
+  br i1 %cmp, label %for.body, label %exit
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
index b7706da364289..f099c22333c3e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/call-costs.ll
@@ -72,7 +72,7 @@ exit:
 define void @powi_call(ptr %P) {
 ; CHECK-LABEL: define void @powi_call(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -84,10 +84,9 @@ define void @powi_call(ptr %P) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds double, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load double, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[POWI:%.*]] = tail call double @llvm.powi.f64.i32(double [[L]], i32 3)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll b/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
index 027a88de14153..7dcddbf630d1f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll
@@ -1,5 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br" --filter "^.*:" --version 5
 ; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v1 -force-vector-interleave=1 -S < %s |  FileCheck %s -check-prefix=CHECK-V1-IC1
+; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v1 -force-vector-interleave=1 \
+; RUN:   -epilogue-vectorization-force-VF=4 -S < %s |  FileCheck %s -check-prefix=CHECK-V1-IC1-FORCE-EPI4
 ; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v2 -force-vector-interleave=1 -S < %s |  FileCheck %s -check-prefix=CHECK-V2-IC1
 ; RUN: opt -passes="print<block-freq>,loop-vectorize" -mcpu=neoverse-v2 -force-vector-interleave=4 -S < %s |  FileCheck %s -check-prefix=CHECK-V2-IC4
 
@@ -10,15 +12,15 @@ target triple = "aarch64-unknown-linux-gnu"
 
 ; We expect the branch weight computations after vectorisation to use
 ; vscale=2 for neoverse-v1 and vscale=1 for neoverse-v2.
-define void @_Z3foov(i64 %n) {
-; CHECK-V1-IC1-LABEL: define void @_Z3foov(
+define void @foo_i32(i64 %n) {
+; CHECK-V1-IC1-LABEL: define void @foo_i32(
 ; CHECK-V1-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-V1-IC1:  [[ENTRY:.*:]]
 ; CHECK-V1-IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
 ; CHECK-V1-IC1:  [[VECTOR_PH]]:
 ; CHECK-V1-IC1:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-V1-IC1:  [[VECTOR_BODY]]:
-; CHECK-V1-IC1:    br i1 [[TMP10:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK-V1-IC1:    br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP1:![0-9]+]]
 ; CHECK-V1-IC1:  [[MIDDLE_BLOCK]]:
 ; CHECK-V1-IC1:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]], !prof [[PROF4:![0-9]+]]
 ; CHECK-V1-IC1:  [[SCALAR_PH]]:
@@ -27,32 +29,58 @@ define void @_Z3foov(i64 %n) {
 ; CHECK-V1-IC1:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF5:![0-9]+]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-V1-IC1:  [[FOR_COND_CLEANUP]]:
 ;
-; CHECK-V2-IC1-LABEL: define void @_Z3foov(
+; CHECK-V1-IC1-FORCE-EPI4-LABEL: define void @foo_i32(
+; CHECK-V1-IC1-FORCE-EPI4-SAME: i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-V1-IC1-FORCE-EPI4:  [[ITER_CHECK:.*:]]
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VECTOR_PH]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br label %[[VECTOR_BODY:.*]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VECTOR_BODY]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF0]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[MIDDLE_BLOCK]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF4:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_PH]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF7:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br label %[[FOR_BODY:.*]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[FOR_BODY]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[FOR_COND_CLEANUP]]:
+;
+; CHECK-V2-IC1-LABEL: define void @foo_i32(
 ; CHECK-V2-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-V2-IC1:  [[ENTRY:.*:]]
 ; CHECK-V2-IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0:![0-9]+]]
 ; CHECK-V2-IC1:  [[VECTOR_PH]]:
 ; CHECK-V2-IC1:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-V2-IC1:  [[VECTOR_BODY]]:
-; CHECK-V2-IC1:    br i1 [[TMP4:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-V2-IC1:    br i1 [[TMP2:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK-V2-IC1:  [[MIDDLE_BLOCK]]:
 ; CHECK-V2-IC1:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[SCALAR_PH]], !prof [[PROF5:![0-9]+]]
 ; CHECK-V2-IC1:  [[SCALAR_PH]]:
 ; CHECK-V2-IC1:    br label %[[FOR_BODY:.*]]
 ; CHECK-V2-IC1:  [[FOR_BODY]]:
-; CHECK-V2-IC1:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF5:![0-9]+]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-V2-IC1:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF6:![0-9]+]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-V2-IC1:  [[FOR_COND_CLEANUP]]:
 ;
-; CHECK-V2-IC4-LABEL: define void @_Z3foov(
+; CHECK-V2-IC4-LABEL: define void @foo_i32(
 ; CHECK-V2-IC4-SAME: i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-V2-IC4:  [[VEC_EPILOG_VECTOR_BODY1:.*:]]
+; CHECK-V2-IC4:  [[ITER_CHECK:.*:]]
 ; CHECK-V2-IC4:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0:![0-9]+]]
 ; CHECK-V2-IC4:  [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
 ; CHECK-V2-IC4:    br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
 ; CHECK-V2-IC4:  [[VECTOR_PH]]:
 ; CHECK-V2-IC4:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-V2-IC4:  [[VECTOR_BODY]]:
-; CHECK-V2-IC4:    br i1 [[TMP10:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-V2-IC4:    br i1 [[TMP8:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF1:![0-9]+]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK-V2-IC4:  [[MIDDLE_BLOCK]]:
 ; CHECK-V2-IC4:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF5:![0-9]+]]
 ; CHECK-V2-IC4:  [[VEC_EPILOG_ITER_CHECK]]:
@@ -60,7 +88,7 @@ define void @_Z3foov(i64 %n) {
 ; CHECK-V2-IC4:  [[VEC_EPILOG_PH]]:
 ; CHECK-V2-IC4:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK-V2-IC4:  [[VEC_EPILOG_VECTOR_BODY]]:
-; CHECK-V2-IC4:    br i1 [[TMP15:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-V2-IC4:    br i1 [[TMP11:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-V2-IC4:  [[VEC_EPILOG_MIDDLE_BLOCK]]:
 ; CHECK-V2-IC4:    br i1 [[CMP_N10:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF8:![0-9]+]]
 ; CHECK-V2-IC4:  [[VEC_EPILOG_SCALAR_PH]]:
@@ -86,6 +114,128 @@ for.cond.cleanup:                                 ; preds = %for.body
   ret void
 }
 
+define void @foo_i8(i64 %n) {
+; CHECK-V1-IC1-LABEL: define void @foo_i8(
+; CHECK-V1-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-V1-IC1:  [[ITER_CHECK:.*:]]
+; CHECK-V1-IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0]]
+; CHECK-V1-IC1:  [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-V1-IC1:    br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; CHECK-V1-IC1:  [[VECTOR_PH]]:
+; CHECK-V1-IC1:    br label %[[VECTOR_BODY:.*]]
+; CHECK-V1-IC1:  [[VECTOR_BODY]]:
+; CHECK-V1-IC1:    br i1 [[TMP8:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF7:![0-9]+]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-V1-IC1:  [[MIDDLE_BLOCK]]:
+; CHECK-V1-IC1:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF7]]
+; CHECK-V1-IC1:  [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-V1-IC1:    br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF9:![0-9]+]]
+; CHECK-V1-IC1:  [[VEC_EPILOG_PH]]:
+; CHECK-V1-IC1:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-V1-IC1:  [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-V1-IC1:    br i1 [[TMP15:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-V1-IC1:  [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-V1-IC1:    br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF11:![0-9]+]]
+; CHECK-V1-IC1:  [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-V1-IC1:    br label %[[FOR_BODY:.*]]
+; CHECK-V1-IC1:  [[FOR_BODY]]:
+; CHECK-V1-IC1:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF5]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-V1-IC1:  [[FOR_COND_CLEANUP]]:
+;
+; CHECK-V1-IC1-FORCE-EPI4-LABEL: define void @foo_i8(
+; CHECK-V1-IC1-FORCE-EPI4-SAME: i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-V1-IC1-FORCE-EPI4:  [[ITER_CHECK:.*:]]
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VECTOR_PH]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br label %[[VECTOR_BODY:.*]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VECTOR_BODY]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[TMP6:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF10:![0-9]+]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[MIDDLE_BLOCK]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF10]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF12:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_PH]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF7]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br label %[[FOR_BODY:.*]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[FOR_BODY]]:
+; CHECK-V1-IC1-FORCE-EPI4:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF8]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-V1-IC1-FORCE-EPI4:  [[FOR_COND_CLEANUP]]:
+;
+; CHECK-V2-IC1-LABEL: define void @foo_i8(
+; CHECK-V2-IC1-SAME: i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-V2-IC1:  [[ITER_CHECK:.*:]]
+; CHECK-V2-IC1:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0]]
+; CHECK-V2-IC1:  [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-V2-IC1:    br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; CHECK-V2-IC1:  [[VECTOR_PH]]:
+; CHECK-V2-IC1:    br label %[[VECTOR_BODY:.*]]
+; CHECK-V2-IC1:  [[VECTOR_BODY]]:
+; CHECK-V2-IC1:    br i1 [[TMP4:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF8:![0-9]+]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-V2-IC1:  [[MIDDLE_BLOCK]]:
+; CHECK-V2-IC1:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF10:![0-9]+]]
+; CHECK-V2-IC1:  [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-V2-IC1:    br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
+; CHECK-V2-IC1:  [[VEC_EPILOG_PH]]:
+; CHECK-V2-IC1:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-V2-IC1:  [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-V2-IC1:    br i1 [[TMP11:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-V2-IC1:  [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-V2-IC1:    br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF5]]
+; CHECK-V2-IC1:  [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-V2-IC1:    br label %[[FOR_BODY:.*]]
+; CHECK-V2-IC1:  [[FOR_BODY]]:
+; CHECK-V2-IC1:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF6]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-V2-IC1:  [[FOR_COND_CLEANUP]]:
+;
+; CHECK-V2-IC4-LABEL: define void @foo_i8(
+; CHECK-V2-IC4-SAME: i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-V2-IC4:  [[ITER_CHECK:.*:]]
+; CHECK-V2-IC4:    br i1 [[MIN_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF0]]
+; CHECK-V2-IC4:  [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-V2-IC4:    br i1 [[MIN_ITERS_CHECK1:%.*]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF0]]
+; CHECK-V2-IC4:  [[VECTOR_PH]]:
+; CHECK-V2-IC4:    br label %[[VECTOR_BODY:.*]]
+; CHECK-V2-IC4:  [[VECTOR_BODY]]:
+; CHECK-V2-IC4:    br i1 [[TMP8:%.*]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF5]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-V2-IC4:  [[MIDDLE_BLOCK]]:
+; CHECK-V2-IC4:    br i1 [[CMP_N:%.*]], label %[[FOR_COND_CLEANUP:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF1]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-V2-IC4:    br i1 [[MIN_EPILOG_ITERS_CHECK:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF12:![0-9]+]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_PH]]:
+; CHECK-V2-IC4:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-V2-IC4:    br i1 [[TMP11:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-V2-IC4:    br i1 [[CMP_N10:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF14:![0-9]+]]
+; CHECK-V2-IC4:  [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-V2-IC4:    br label %[[FOR_BODY:.*]]
+; CHECK-V2-IC4:  [[FOR_BODY]]:
+; CHECK-V2-IC4:    br i1 [[EXITCOND:%.*]], label %[[FOR_COND_CLEANUP]], label %[[FOR_BODY]], !prof [[PROF9]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-V2-IC4:  [[FOR_COND_CLEANUP]]:
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [1024 x i8], ptr @b, i64 0, i64 %iv
+  %load = load i8, ptr %arrayidx, align 1
+  %arrayidx2 = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %iv
+  store i8 %load, ptr %arrayidx2, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond = icmp eq i64 %iv.next, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !prof !0
+
+for.cond.cleanup:
+  ret void
+}
+
 !0 = !{!"branch_weights", i32 1, i32 1023}
 ;.
 ; CHECK-V1-IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
@@ -95,14 +245,43 @@ for.cond.cleanup:                                 ; preds = %for.body
 ; CHECK-V1-IC1: [[PROF4]] = !{!"branch_weights", i32 1, i32 7}
 ; CHECK-V1-IC1: [[PROF5]] = !{!"branch_weights", i32 0, i32 0}
 ; CHECK-V1-IC1: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META2]]}
+; CHECK-V1-IC1: [[PROF7]] = !{!"branch_weights", i32 1, i32 31}
+; CHECK-V1-IC1: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META3]]}
+; CHECK-V1-IC1: [[PROF9]] = !{!"branch_weights", i32 16, i32 16}
+; CHECK-V1-IC1: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META3]]}
+; CHECK-V1-IC1: [[PROF11]] = !{!"branch_weights", i32 1, i32 15}
+; CHECK-V1-IC1: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]], [[META2]]}
+;.
+; CHECK-V1-IC1-FORCE-EPI4: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
+; CHECK-V1-IC1-FORCE-EPI4: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK-V1-IC1-FORCE-EPI4: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-V1-IC1-FORCE-EPI4: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-V1-IC1-FORCE-EPI4: [[PROF4]] = !{!"branch_weights", i32 1, i32 7}
+; CHECK-V1-IC1-FORCE-EPI4: [[PROF5]] = !{!"branch_weights", i32 4, i32 4}
+; CHECK-V1-IC1-FORCE-EPI4: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META3]]}
+; CHECK-V1-IC1-FORCE-EPI4: [[PROF7]] = !{!"branch_weights", i32 1, i32 3}
+; CHECK-V1-IC1-FORCE-EPI4: [[PROF8]] = !{!"branch_weights", i32 0, i32 0}
+; CHECK-V1-IC1-FORCE-EPI4: [[LOOP9]] = distinct !{[[LOOP9]], [[META3]], [[META2]]}
+; CHECK-V1-IC1-FORCE-EPI4: [[PROF10]] = !{!"branch_weights", i32 1, i32 31}
+; CHECK-V1-IC1-FORCE-EPI4: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META3]]}
+; CHECK-V1-IC1-FORCE-EPI4: [[PROF12]] = !{!"branch_weights", i32 4, i32 28}
+; CHECK-V1-IC1-FORCE-EPI4: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META3]]}
+; CHECK-V1-IC1-FORCE-EPI4: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]], [[META2]]}
 ;.
 ; CHECK-V2-IC1: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
 ; CHECK-V2-IC1: [[PROF1]] = !{!"branch_weights", i32 1, i32 255}
 ; CHECK-V2-IC1: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
 ; CHECK-V2-IC1: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK-V2-IC1: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK-V2-IC1: [[PROF5]] = !{!"branch_weights", i32 0, i32 0}
-; CHECK-V2-IC1: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]], [[META3]]}
+; CHECK-V2-IC1: [[PROF5]] = !{!"branch_weights", i32 1, i32 3}
+; CHECK-V2-IC1: [[PROF6]] = !{!"branch_weights", i32 0, i32 0}
+; CHECK-V2-IC1: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]], [[META3]]}
+; CHECK-V2-IC1: [[PROF8]] = !{!"branch_weights", i32 1, i32 63}
+; CHECK-V2-IC1: [[LOOP9]] = distinct !{[[LOOP9]], [[META3]], [[META4]]}
+; CHECK-V2-IC1: [[PROF10]] = !{!"branch_weights", i32 1, i32 15}
+; CHECK-V2-IC1: [[PROF11]] = !{!"branch_weights", i32 4, i32 12}
+; CHECK-V2-IC1: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]], [[META4]]}
+; CHECK-V2-IC1: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]], [[META3]]}
 ;.
 ; CHECK-V2-IC4: [[PROF0]] = !{!"branch_weights", i32 1, i32 127}
 ; CHECK-V2-IC4: [[PROF1]] = !{!"branch_weights", i32 1, i32 63}
@@ -110,9 +289,14 @@ for.cond.cleanup:                                 ; preds = %for.body
 ; CHECK-V2-IC4: [[META3]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK-V2-IC4: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK-V2-IC4: [[PROF5]] = !{!"branch_weights", i32 1, i32 15}
-; CHECK-V2-IC4: [[PROF6]] = !{!"branch_weights", i32 4, i32 0}
+; CHECK-V2-IC4: [[PROF6]] = !{!"branch_weights", i32 4, i32 12}
 ; CHECK-V2-IC4: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]], [[META4]]}
 ; CHECK-V2-IC4: [[PROF8]] = !{!"branch_weights", i32 1, i32 3}
 ; CHECK-V2-IC4: [[PROF9]] = !{!"branch_weights", i32 0, i32 0}
 ; CHECK-V2-IC4: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]], [[META3]]}
+; CHECK-V2-IC4: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]], [[META4]]}
+; CHECK-V2-IC4: [[PROF12]] = !{!"branch_weights", i32 8, i32 56}
+; CHECK-V2-IC4: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]], [[META4]]}
+; CHECK-V2-IC4: [[PROF14]] = !{!"branch_weights", i32 1, i32 7}
+; CHECK-V2-IC4: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]], [[META3]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
index 795de3d978e74..626242667e203 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
@@ -9,20 +9,13 @@ define void @clamped_tc_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range(1,1
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 8, [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 8)
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[VAL]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 8 x i64> [[TMP8]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 1, [[TMP6]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP12]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP1]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -34,19 +27,17 @@ define void @clamped_tc_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range(1,1
 ; CHECK-NEXT:    [[TMP11:%.*]] = lshr <vscale x 8 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = trunc <vscale x 8 x i64> [[TMP11]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP14]], ptr [[NEXT_GEP]], i32 1, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 8)
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[DST]], [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[P_OUT_TAIL_09:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P_OUT_TAIL_09:%.*]] = phi ptr [ [[DST]], [[SCALAR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 3
 ; CHECK-NEXT:    [[SHR3:%.*]] = lshr i64 [[VAL]], [[TMP19]]
 ; CHECK-NEXT:    [[CONV4:%.*]] = trunc i64 [[SHR3]] to i8
@@ -92,20 +83,13 @@ define void @clamped_tc_max_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[WIDE_TRIP_COUNT]], [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[VAL]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 8 x i64> [[TMP8]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP7]]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 1, [[TMP6]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP12]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP1]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -117,19 +101,17 @@ define void @clamped_tc_max_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range
 ; CHECK-NEXT:    [[TMP11:%.*]] = lshr <vscale x 8 x i64> [[BROADCAST_SPLAT]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = trunc <vscale x 8 x i64> [[TMP11]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP14]], ptr [[NEXT_GEP]], i32 1, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]])
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[DST]], [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[P_OUT_TAIL_09:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P_OUT_TAIL_09:%.*]] = phi ptr [ [[DST]], [[SCALAR_PH]] ], [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 3
 ; CHECK-NEXT:    [[SHR3:%.*]] = lshr i64 [[VAL]], [[TMP19]]
 ; CHECK-NEXT:    [[CONV4:%.*]] = trunc i64 [[SHR3]] to i8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 0232d88347d0a..c914aa3582e5c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -1,48 +1,29 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
-; RUN: opt -p loop-vectorize -S %s | FileCheck --check-prefix=DEFAULT %s
-; RUN: opt -p loop-vectorize  -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s | FileCheck --check-prefix=PRED %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
+; RUN: opt -p loop-vectorize -S %s | FileCheck --check-prefixes=COMMON,DEFAULT %s
+; RUN: opt -p loop-vectorize  -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S %s | FileCheck --check-prefixes=COMMON,PRED %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-macosx14.0.0"
 
 define void @invar_cond_gep_store(ptr %dst, i32 %0) {
-; DEFAULT-LABEL: define void @invar_cond_gep_store(
-; DEFAULT-SAME: ptr [[DST:%.*]], i32 [[TMP0:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
-; DEFAULT-NEXT:    br label %[[LOOP_HEADER:.*]]
-; DEFAULT:       [[LOOP_HEADER]]:
-; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; DEFAULT-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; DEFAULT-NEXT:    [[CMP9:%.*]] = icmp eq i32 [[TMP0]], 0
-; DEFAULT-NEXT:    br i1 [[CMP9]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
-; DEFAULT:       [[THEN]]:
-; DEFAULT-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]]
-; DEFAULT-NEXT:    store i32 1, ptr [[GEP]], align 4
-; DEFAULT-NEXT:    br label %[[LOOP_LATCH]]
-; DEFAULT:       [[LOOP_LATCH]]:
-; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 100
-; DEFAULT-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret void
-;
-; PRED-LABEL: define void @invar_cond_gep_store(
-; PRED-SAME: ptr [[DST:%.*]], i32 [[TMP0:%.*]]) {
-; PRED-NEXT:  [[ENTRY:.*]]:
-; PRED-NEXT:    br label %[[LOOP_HEADER:.*]]
-; PRED:       [[LOOP_HEADER]]:
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; PRED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; PRED-NEXT:    [[CMP9:%.*]] = icmp eq i32 [[TMP0]], 0
-; PRED-NEXT:    br i1 [[CMP9]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
-; PRED:       [[THEN]]:
-; PRED-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]]
-; PRED-NEXT:    store i32 1, ptr [[GEP]], align 4
-; PRED-NEXT:    br label %[[LOOP_LATCH]]
-; PRED:       [[LOOP_LATCH]]:
-; PRED-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 100
-; PRED-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
-; PRED:       [[EXIT]]:
-; PRED-NEXT:    ret void
+; COMMON-LABEL: define void @invar_cond_gep_store(
+; COMMON-SAME: ptr [[DST:%.*]], i32 [[TMP0:%.*]]) {
+; COMMON-NEXT:  [[ENTRY:.*]]:
+; COMMON-NEXT:    br label %[[LOOP_HEADER:.*]]
+; COMMON:       [[LOOP_HEADER]]:
+; COMMON-NEXT:    [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COMMON-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; COMMON-NEXT:    [[CMP9:%.*]] = icmp eq i32 [[TMP0]], 0
+; COMMON-NEXT:    br i1 [[CMP9]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; COMMON:       [[THEN]]:
+; COMMON-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[DST]], i64 [[IV_NEXT]]
+; COMMON-NEXT:    store i32 1, ptr [[GEP]], align 4
+; COMMON-NEXT:    br label %[[LOOP_LATCH]]
+; COMMON:       [[LOOP_LATCH]]:
+; COMMON-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 100
+; COMMON-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COMMON:       [[EXIT]]:
+; COMMON-NEXT:    ret void
 ;
 entry:
   br label %loop.header
@@ -71,7 +52,7 @@ declare double @llvm.fabs.f64(double) #0
 define void @loop_dependent_cond(ptr %src, ptr noalias %dst, i64 %N) {
 ; DEFAULT-LABEL: define void @loop_dependent_cond(
 ; DEFAULT-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -118,26 +99,8 @@ define void @loop_dependent_cond(ptr %src, ptr noalias %dst, i64 %N) {
 ; DEFAULT-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; DEFAULT-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; DEFAULT-NEXT:    br label %[[LOOP_HEADER:.*]]
-; DEFAULT:       [[LOOP_HEADER]]:
-; DEFAULT-NEXT:    [[IV175:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT176:%.*]], %[[LOOP_LATCH:.*]] ]
-; DEFAULT-NEXT:    [[ARRAYIDX114:%.*]] = getelementptr double, ptr [[SRC]], i64 [[IV175]]
-; DEFAULT-NEXT:    [[TMP16:%.*]] = load double, ptr [[ARRAYIDX114]], align 8
-; DEFAULT-NEXT:    [[TMP17:%.*]] = tail call double @llvm.fabs.f64(double [[TMP16]])
-; DEFAULT-NEXT:    [[CMP115:%.*]] = fcmp ogt double [[TMP17]], 1.000000e+00
-; DEFAULT-NEXT:    br i1 [[CMP115]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
-; DEFAULT:       [[THEN]]:
-; DEFAULT-NEXT:    store i32 0, ptr [[DST]], align 4
-; DEFAULT-NEXT:    br label %[[LOOP_LATCH]]
-; DEFAULT:       [[LOOP_LATCH]]:
-; DEFAULT-NEXT:    [[IV_NEXT176]] = add i64 [[IV175]], 1
-; DEFAULT-NEXT:    [[EXITCOND180_NOT:%.*]] = icmp eq i64 [[IV175]], [[N]]
-; DEFAULT-NEXT:    br i1 [[EXITCOND180_NOT]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret void
 ;
 ; PRED-LABEL: define void @loop_dependent_cond(
 ; PRED-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) {
@@ -185,53 +148,29 @@ exit:
 }
 
 define void @invar_cond_chain_1(ptr %I, ptr noalias %src, i1 %c) {
-; DEFAULT-LABEL: define void @invar_cond_chain_1(
-; DEFAULT-SAME: ptr [[I:%.*]], ptr noalias [[SRC:%.*]], i1 [[C:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
-; DEFAULT-NEXT:    br label %[[LOOP_HEADER:.*]]
-; DEFAULT:       [[LOOP_HEADER]]:
-; DEFAULT-NEXT:    [[__BEGIN3_011973:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[INCDEC_PTR329:%.*]], %[[LOOP_LATCH:.*]] ]
-; DEFAULT-NEXT:    [[TMP28:%.*]] = load i32, ptr [[__BEGIN3_011973]], align 4
-; DEFAULT-NEXT:    br i1 true, label %[[ELSE_1:.*]], label %[[IF:.*]]
-; DEFAULT:       [[IF]]:
-; DEFAULT-NEXT:    br label %[[SPLIT:.*]]
-; DEFAULT:       [[ELSE_1]]:
-; DEFAULT-NEXT:    br i1 [[C]], label %[[ELSE_2:.*]], label %[[LOOP_LATCH]]
-; DEFAULT:       [[ELSE_2]]:
-; DEFAULT-NEXT:    br label %[[SPLIT]]
-; DEFAULT:       [[SPLIT]]:
-; DEFAULT-NEXT:    store i32 [[TMP28]], ptr [[I]], align 4
-; DEFAULT-NEXT:    br label %[[LOOP_LATCH]]
-; DEFAULT:       [[LOOP_LATCH]]:
-; DEFAULT-NEXT:    [[INCDEC_PTR329]] = getelementptr inbounds i8, ptr [[__BEGIN3_011973]], i64 4
-; DEFAULT-NEXT:    [[CMP311_NOT:%.*]] = icmp eq ptr [[__BEGIN3_011973]], [[I]]
-; DEFAULT-NEXT:    br i1 [[CMP311_NOT]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret void
-;
-; PRED-LABEL: define void @invar_cond_chain_1(
-; PRED-SAME: ptr [[I:%.*]], ptr noalias [[SRC:%.*]], i1 [[C:%.*]]) {
-; PRED-NEXT:  [[ENTRY:.*]]:
-; PRED-NEXT:    br label %[[LOOP_HEADER:.*]]
-; PRED:       [[LOOP_HEADER]]:
-; PRED-NEXT:    [[__BEGIN3_011973:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[INCDEC_PTR329:%.*]], %[[LOOP_LATCH:.*]] ]
-; PRED-NEXT:    [[TMP0:%.*]] = load i32, ptr [[__BEGIN3_011973]], align 4
-; PRED-NEXT:    br i1 true, label %[[ELSE_1:.*]], label %[[IF:.*]]
-; PRED:       [[IF]]:
-; PRED-NEXT:    br label %[[SPLIT:.*]]
-; PRED:       [[ELSE_1]]:
-; PRED-NEXT:    br i1 [[C]], label %[[ELSE_2:.*]], label %[[LOOP_LATCH]]
-; PRED:       [[ELSE_2]]:
-; PRED-NEXT:    br label %[[SPLIT]]
-; PRED:       [[SPLIT]]:
-; PRED-NEXT:    store i32 [[TMP0]], ptr [[I]], align 4
-; PRED-NEXT:    br label %[[LOOP_LATCH]]
-; PRED:       [[LOOP_LATCH]]:
-; PRED-NEXT:    [[INCDEC_PTR329]] = getelementptr inbounds i8, ptr [[__BEGIN3_011973]], i64 4
-; PRED-NEXT:    [[CMP311_NOT:%.*]] = icmp eq ptr [[__BEGIN3_011973]], [[I]]
-; PRED-NEXT:    br i1 [[CMP311_NOT]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
-; PRED:       [[EXIT]]:
-; PRED-NEXT:    ret void
+; COMMON-LABEL: define void @invar_cond_chain_1(
+; COMMON-SAME: ptr [[I:%.*]], ptr noalias [[SRC:%.*]], i1 [[C:%.*]]) {
+; COMMON-NEXT:  [[ENTRY:.*]]:
+; COMMON-NEXT:    br label %[[LOOP_HEADER:.*]]
+; COMMON:       [[LOOP_HEADER]]:
+; COMMON-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COMMON-NEXT:    [[L:%.*]] = load i32, ptr [[PTR_IV]], align 4
+; COMMON-NEXT:    br i1 true, label %[[ELSE_1:.*]], label %[[IF:.*]]
+; COMMON:       [[IF]]:
+; COMMON-NEXT:    br label %[[SPLIT:.*]]
+; COMMON:       [[ELSE_1]]:
+; COMMON-NEXT:    br i1 [[C]], label %[[ELSE_2:.*]], label %[[LOOP_LATCH]]
+; COMMON:       [[ELSE_2]]:
+; COMMON-NEXT:    br label %[[SPLIT]]
+; COMMON:       [[SPLIT]]:
+; COMMON-NEXT:    store i32 [[L]], ptr [[I]], align 4
+; COMMON-NEXT:    br label %[[LOOP_LATCH]]
+; COMMON:       [[LOOP_LATCH]]:
+; COMMON-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 4
+; COMMON-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV]], [[I]]
+; COMMON-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COMMON:       [[EXIT]]:
+; COMMON-NEXT:    ret void
 ;
 entry:
   br label %loop.header
@@ -264,45 +203,25 @@ exit:
 }
 
 define void @invar_cond_chain_2(ptr %I, ptr noalias %src, ptr noalias %dst, i32 %a) {
-; DEFAULT-LABEL: define void @invar_cond_chain_2(
-; DEFAULT-SAME: ptr [[I:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[A:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
-; DEFAULT-NEXT:    br label %[[LOOP_HEADER:.*]]
-; DEFAULT:       [[LOOP_HEADER]]:
-; DEFAULT-NEXT:    [[__BEGIN3_01197:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[INCDEC_PTR329:%.*]], %[[LOOP_LATCH:.*]] ]
-; DEFAULT-NEXT:    [[CMP315_NOT:%.*]] = icmp sgt i32 [[A]], 0
-; DEFAULT-NEXT:    br i1 [[CMP315_NOT]], label %[[LOOP_LATCH]], label %[[IF:.*]]
-; DEFAULT:       [[IF]]:
-; DEFAULT-NEXT:    br label %[[ELSE:.*]]
-; DEFAULT:       [[ELSE]]:
-; DEFAULT-NEXT:    store i32 0, ptr [[DST]], align 4
-; DEFAULT-NEXT:    br label %[[LOOP_LATCH]]
-; DEFAULT:       [[LOOP_LATCH]]:
-; DEFAULT-NEXT:    [[INCDEC_PTR329]] = getelementptr inbounds i8, ptr [[__BEGIN3_01197]], i64 4
-; DEFAULT-NEXT:    [[CMP311_NOT:%.*]] = icmp eq ptr [[__BEGIN3_01197]], [[I]]
-; DEFAULT-NEXT:    br i1 [[CMP311_NOT]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret void
-;
-; PRED-LABEL: define void @invar_cond_chain_2(
-; PRED-SAME: ptr [[I:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[A:%.*]]) {
-; PRED-NEXT:  [[ENTRY:.*]]:
-; PRED-NEXT:    br label %[[LOOP_HEADER:.*]]
-; PRED:       [[LOOP_HEADER]]:
-; PRED-NEXT:    [[__BEGIN3_01197:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[INCDEC_PTR329:%.*]], %[[LOOP_LATCH:.*]] ]
-; PRED-NEXT:    [[CMP315_NOT:%.*]] = icmp sgt i32 [[A]], 0
-; PRED-NEXT:    br i1 [[CMP315_NOT]], label %[[LOOP_LATCH]], label %[[IF:.*]]
-; PRED:       [[IF]]:
-; PRED-NEXT:    br label %[[ELSE:.*]]
-; PRED:       [[ELSE]]:
-; PRED-NEXT:    store i32 0, ptr [[DST]], align 4
-; PRED-NEXT:    br label %[[LOOP_LATCH]]
-; PRED:       [[LOOP_LATCH]]:
-; PRED-NEXT:    [[INCDEC_PTR329]] = getelementptr inbounds i8, ptr [[__BEGIN3_01197]], i64 4
-; PRED-NEXT:    [[CMP311_NOT:%.*]] = icmp eq ptr [[__BEGIN3_01197]], [[I]]
-; PRED-NEXT:    br i1 [[CMP311_NOT]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
-; PRED:       [[EXIT]]:
-; PRED-NEXT:    ret void
+; COMMON-LABEL: define void @invar_cond_chain_2(
+; COMMON-SAME: ptr [[I:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[A:%.*]]) {
+; COMMON-NEXT:  [[ENTRY:.*]]:
+; COMMON-NEXT:    br label %[[LOOP_HEADER:.*]]
+; COMMON:       [[LOOP_HEADER]]:
+; COMMON-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COMMON-NEXT:    [[CMP315_NOT:%.*]] = icmp sgt i32 [[A]], 0
+; COMMON-NEXT:    br i1 [[CMP315_NOT]], label %[[LOOP_LATCH]], label %[[IF:.*]]
+; COMMON:       [[IF]]:
+; COMMON-NEXT:    br label %[[ELSE:.*]]
+; COMMON:       [[ELSE]]:
+; COMMON-NEXT:    store i32 0, ptr [[DST]], align 4
+; COMMON-NEXT:    br label %[[LOOP_LATCH]]
+; COMMON:       [[LOOP_LATCH]]:
+; COMMON-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds i8, ptr [[PTR_IV]], i64 4
+; COMMON-NEXT:    [[CMP311_NOT:%.*]] = icmp eq ptr [[PTR_IV]], [[I]]
+; COMMON-NEXT:    br i1 [[CMP311_NOT]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COMMON:       [[EXIT]]:
+; COMMON-NEXT:    ret void
 ;
 entry:
   br label %loop.header
@@ -331,7 +250,7 @@ exit:
 define void @latch_branch_cost(ptr %dst) {
 ; DEFAULT-LABEL: define void @latch_branch_cost(
 ; DEFAULT-SAME: ptr [[DST:%.*]]) {
-; DEFAULT-NEXT:  [[ITER_CHECK:.*]]:
+; DEFAULT-NEXT:  [[ITER_CHECK:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; DEFAULT:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
 ; DEFAULT-NEXT:    br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -347,7 +266,7 @@ define void @latch_branch_cost(ptr %dst) {
 ; DEFAULT-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
 ; DEFAULT-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
-; DEFAULT-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; DEFAULT-NEXT:    br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; DEFAULT:       [[VEC_EPILOG_ITER_CHECK]]:
 ; DEFAULT-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; DEFAULT:       [[VEC_EPILOG_PH]]:
@@ -361,23 +280,12 @@ define void @latch_branch_cost(ptr %dst) {
 ; DEFAULT-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100
 ; DEFAULT-NEXT:    br i1 [[TMP10]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; DEFAULT:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; DEFAULT-NEXT:    br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; DEFAULT-NEXT:    br i1 true, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; DEFAULT:       [[VEC_EPILOG_SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 100, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; DEFAULT-NEXT:    br label %[[LOOP:.*]]
-; DEFAULT:       [[LOOP]]:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDVARS_IV]]
-; DEFAULT-NEXT:    store i8 0, ptr [[ARRAYIDX]], align 1
-; DEFAULT-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
-; DEFAULT-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 100
-; DEFAULT-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret void
 ;
 ; PRED-LABEL: define void @latch_branch_cost(
 ; PRED-SAME: ptr [[DST:%.*]]) {
-; PRED-NEXT:  [[ENTRY:.*]]:
+; PRED-NEXT:  [[ENTRY:.*:]]
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -454,19 +362,8 @@ define void @latch_branch_cost(ptr %dst) {
 ; PRED-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 104
 ; PRED-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
-; PRED-NEXT:    br label %[[EXIT:.*]]
+; PRED-NEXT:    br [[EXIT:label %.*]]
 ; PRED:       [[SCALAR_PH]]:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; PRED-NEXT:    br label %[[LOOP:.*]]
-; PRED:       [[LOOP]]:
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; PRED-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV]]
-; PRED-NEXT:    store i8 0, ptr [[GEP]], align 1
-; PRED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; PRED-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; PRED-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; PRED:       [[EXIT]]:
-; PRED-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -486,7 +383,7 @@ exit:
 define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, ptr %E, i64 %N) "target-features"="+sve" {
 ; DEFAULT-LABEL: define i32 @header_mask_and_invariant_compare(
 ; DEFAULT-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 60
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -578,29 +475,8 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
 ; DEFAULT-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; DEFAULT-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
-; DEFAULT-NEXT:    br label %[[LOOP_HEADER:.*]]
-; DEFAULT:       [[LOOP_HEADER]]:
-; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; DEFAULT-NEXT:    [[L_A:%.*]] = load i32, ptr [[A]], align 4
-; DEFAULT-NEXT:    [[L_B:%.*]] = load i32, ptr [[B]], align 4
-; DEFAULT-NEXT:    [[OR:%.*]] = or i32 [[L_B]], [[L_A]]
-; DEFAULT-NEXT:    [[L_C:%.*]] = load i32, ptr [[C]], align 4
-; DEFAULT-NEXT:    [[C_0:%.*]] = icmp ugt i32 [[L_C]], [[OR]]
-; DEFAULT-NEXT:    br i1 [[C_0]], label %[[IF_THEN:.*]], label %[[LOOP_LATCH]]
-; DEFAULT:       [[IF_THEN]]:
-; DEFAULT-NEXT:    [[GEP_D:%.*]] = getelementptr i32, ptr [[D]], i64 [[IV]]
-; DEFAULT-NEXT:    store i32 [[OR]], ptr [[E]], align 4
-; DEFAULT-NEXT:    store i32 0, ptr [[GEP_D]], align 4
-; DEFAULT-NEXT:    br label %[[LOOP_LATCH]]
-; DEFAULT:       [[LOOP_LATCH]]:
-; DEFAULT-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; DEFAULT-NEXT:    [[C_1:%.*]] = icmp eq i64 [[IV]], [[N]]
-; DEFAULT-NEXT:    br i1 [[C_1]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP21:![0-9]+]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret i32 0
 ;
 ; PRED-LABEL: define i32 @header_mask_and_invariant_compare(
 ; PRED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[D:%.*]], ptr [[E:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
@@ -656,7 +532,7 @@ exit:
 define void @multiple_exit_conditions(ptr %src, ptr noalias %dst) #1 {
 ; DEFAULT-LABEL: define void @multiple_exit_conditions(
 ; DEFAULT-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR2:[0-9]+]] {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[DST]], i64 2048
@@ -677,39 +553,16 @@ define void @multiple_exit_conditions(ptr %src, ptr noalias %dst) #1 {
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[SCALAR_PH]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[DST]], %[[ENTRY]] ]
-; DEFAULT-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 512, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; DEFAULT-NEXT:    br label %[[LOOP:.*]]
-; DEFAULT:       [[LOOP]]:
-; DEFAULT-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT:    [[L:%.*]] = load i16, ptr [[SRC]], align 2
-; DEFAULT-NEXT:    [[O:%.*]] = or i16 [[L]], 1
-; DEFAULT-NEXT:    [[CONV:%.*]] = uitofp i16 [[O]] to double
-; DEFAULT-NEXT:    store double [[CONV]], ptr [[PTR_IV]], align 8
-; DEFAULT-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; DEFAULT-NEXT:    [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 8
-; DEFAULT-NEXT:    [[IV_CLAMP:%.*]] = and i64 [[IV]], 4294967294
-; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_CLAMP]], 512
-; DEFAULT-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret void
 ;
 ; PRED-LABEL: define void @multiple_exit_conditions(
 ; PRED-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR2:[0-9]+]] {
-; PRED-NEXT:  [[ENTRY:.*]]:
+; PRED-NEXT:  [[ENTRY:.*:]]
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; PRED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 257, [[TMP2]]
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; PRED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; PRED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 2
+; PRED-NEXT:    [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 1
 ; PRED-NEXT:    [[TMP8:%.*]] = sub i64 257, [[TMP7]]
 ; PRED-NEXT:    [[TMP9:%.*]] = icmp ugt i64 257, [[TMP7]]
 ; PRED-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
@@ -726,31 +579,14 @@ define void @multiple_exit_conditions(ptr %src, ptr noalias %dst) #1 {
 ; PRED-NEXT:    [[TMP13:%.*]] = or <vscale x 2 x i16> [[BROADCAST_SPLAT]], splat (i16 1)
 ; PRED-NEXT:    [[TMP14:%.*]] = uitofp <vscale x 2 x i16> [[TMP13]] to <vscale x 2 x double>
 ; PRED-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP14]], ptr [[NEXT_GEP]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
+; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP10]])
-; PRED-NEXT:    [[TMP16:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; PRED-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 2 x i1> [[TMP16]], i32 0
-; PRED-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; PRED-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP15]], true
+; PRED-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
-; PRED-NEXT:    br label %[[EXIT:.*]]
+; PRED-NEXT:    br [[EXIT:label %.*]]
 ; PRED:       [[SCALAR_PH]]:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ]
-; PRED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; PRED-NEXT:    br label %[[LOOP:.*]]
-; PRED:       [[LOOP]]:
-; PRED-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; PRED-NEXT:    [[L:%.*]] = load i16, ptr [[SRC]], align 2
-; PRED-NEXT:    [[O:%.*]] = or i16 [[L]], 1
-; PRED-NEXT:    [[CONV:%.*]] = uitofp i16 [[O]] to double
-; PRED-NEXT:    store double [[CONV]], ptr [[PTR_IV]], align 8
-; PRED-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; PRED-NEXT:    [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 8
-; PRED-NEXT:    [[IV_CLAMP:%.*]] = and i64 [[IV]], 4294967294
-; PRED-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_CLAMP]], 512
-; PRED-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; PRED:       [[EXIT]]:
-; PRED-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -773,209 +609,65 @@ exit:
 }
 
 define void @low_trip_count_fold_tail_scalarized_store(ptr %dst) {
-; DEFAULT-LABEL: define void @low_trip_count_fold_tail_scalarized_store(
-; DEFAULT-SAME: ptr [[DST:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
-; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; DEFAULT:       [[VECTOR_PH]]:
-; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
-; DEFAULT:       [[VECTOR_BODY]]:
-; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE14:.*]] ]
-; DEFAULT-NEXT:    [[VEC_IND:%.*]] = phi <8 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE14]] ]
-; DEFAULT-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDEX]] to i8
-; DEFAULT-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i8> [[VEC_IND]], splat (i8 6)
-; DEFAULT-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
-; DEFAULT-NEXT:    br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; DEFAULT:       [[PRED_STORE_IF]]:
-; DEFAULT-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
-; DEFAULT-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
-; DEFAULT-NEXT:    [[TMP5:%.*]] = add i8 [[TMP0]], 0
-; DEFAULT-NEXT:    store i8 [[TMP5]], ptr [[TMP4]], align 1
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE]]
-; DEFAULT:       [[PRED_STORE_CONTINUE]]:
-; DEFAULT-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
-; DEFAULT-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; DEFAULT:       [[PRED_STORE_IF1]]:
-; DEFAULT-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
-; DEFAULT-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
-; DEFAULT-NEXT:    [[TMP9:%.*]] = add i8 [[TMP0]], 1
-; DEFAULT-NEXT:    store i8 [[TMP9]], ptr [[TMP8]], align 1
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
-; DEFAULT:       [[PRED_STORE_CONTINUE2]]:
-; DEFAULT-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
-; DEFAULT-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; DEFAULT:       [[PRED_STORE_IF3]]:
-; DEFAULT-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 2
-; DEFAULT-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]
-; DEFAULT-NEXT:    [[TMP13:%.*]] = add i8 [[TMP0]], 2
-; DEFAULT-NEXT:    store i8 [[TMP13]], ptr [[TMP12]], align 1
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
-; DEFAULT:       [[PRED_STORE_CONTINUE4]]:
-; DEFAULT-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
-; DEFAULT-NEXT:    br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; DEFAULT:       [[PRED_STORE_IF5]]:
-; DEFAULT-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 3
-; DEFAULT-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP15]]
-; DEFAULT-NEXT:    [[TMP17:%.*]] = add i8 [[TMP0]], 3
-; DEFAULT-NEXT:    store i8 [[TMP17]], ptr [[TMP16]], align 1
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
-; DEFAULT:       [[PRED_STORE_CONTINUE6]]:
-; DEFAULT-NEXT:    [[TMP18:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
-; DEFAULT-NEXT:    br i1 [[TMP18]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; DEFAULT:       [[PRED_STORE_IF7]]:
-; DEFAULT-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX]], 4
-; DEFAULT-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP19]]
-; DEFAULT-NEXT:    [[TMP21:%.*]] = add i8 [[TMP0]], 4
-; DEFAULT-NEXT:    store i8 [[TMP21]], ptr [[TMP20]], align 1
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
-; DEFAULT:       [[PRED_STORE_CONTINUE8]]:
-; DEFAULT-NEXT:    [[TMP22:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
-; DEFAULT-NEXT:    br i1 [[TMP22]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; DEFAULT:       [[PRED_STORE_IF9]]:
-; DEFAULT-NEXT:    [[TMP23:%.*]] = add i64 [[INDEX]], 5
-; DEFAULT-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP23]]
-; DEFAULT-NEXT:    [[TMP25:%.*]] = add i8 [[TMP0]], 5
-; DEFAULT-NEXT:    store i8 [[TMP25]], ptr [[TMP24]], align 1
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
-; DEFAULT:       [[PRED_STORE_CONTINUE10]]:
-; DEFAULT-NEXT:    [[TMP26:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
-; DEFAULT-NEXT:    br i1 [[TMP26]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; DEFAULT:       [[PRED_STORE_IF11]]:
-; DEFAULT-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX]], 6
-; DEFAULT-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP27]]
-; DEFAULT-NEXT:    [[TMP29:%.*]] = add i8 [[TMP0]], 6
-; DEFAULT-NEXT:    store i8 [[TMP29]], ptr [[TMP28]], align 1
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE12]]
-; DEFAULT:       [[PRED_STORE_CONTINUE12]]:
-; DEFAULT-NEXT:    [[TMP30:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
-; DEFAULT-NEXT:    br i1 [[TMP30]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14]]
-; DEFAULT:       [[PRED_STORE_IF13]]:
-; DEFAULT-NEXT:    [[TMP31:%.*]] = add i64 [[INDEX]], 7
-; DEFAULT-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP31]]
-; DEFAULT-NEXT:    [[TMP33:%.*]] = add i8 [[TMP0]], 7
-; DEFAULT-NEXT:    store i8 [[TMP33]], ptr [[TMP32]], align 1
-; DEFAULT-NEXT:    br label %[[PRED_STORE_CONTINUE14]]
-; DEFAULT:       [[PRED_STORE_CONTINUE14]]:
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <8 x i8> [[VEC_IND]], splat (i8 8)
-; DEFAULT-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
-; DEFAULT:       [[MIDDLE_BLOCK]]:
-; DEFAULT-NEXT:    br label %[[EXIT:.*]]
-; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; DEFAULT-NEXT:    br label %[[LOOP:.*]]
-; DEFAULT:       [[LOOP]]:
-; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; DEFAULT-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i8
-; DEFAULT-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV]]
-; DEFAULT-NEXT:    store i8 [[IV_TRUNC]], ptr [[GEP]], align 1
-; DEFAULT-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 7
-; DEFAULT-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret void
-;
-; PRED-LABEL: define void @low_trip_count_fold_tail_scalarized_store(
-; PRED-SAME: ptr [[DST:%.*]]) {
-; PRED-NEXT:  [[ENTRY:.*]]:
-; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; PRED:       [[VECTOR_PH]]:
-; PRED-NEXT:    br label %[[VECTOR_BODY:.*]]
-; PRED:       [[VECTOR_BODY]]:
-; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE14:.*]] ]
-; PRED-NEXT:    [[VEC_IND:%.*]] = phi <8 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE14]] ]
-; PRED-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDEX]] to i8
-; PRED-NEXT:    [[TMP1:%.*]] = icmp ule <8 x i8> [[VEC_IND]], splat (i8 6)
-; PRED-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
-; PRED-NEXT:    br i1 [[TMP2]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
-; PRED:       [[PRED_STORE_IF]]:
-; PRED-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
-; PRED-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP3]]
-; PRED-NEXT:    [[TMP5:%.*]] = add i8 [[TMP0]], 0
-; PRED-NEXT:    store i8 [[TMP5]], ptr [[TMP4]], align 1
-; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE]]
-; PRED:       [[PRED_STORE_CONTINUE]]:
-; PRED-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
-; PRED-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
-; PRED:       [[PRED_STORE_IF1]]:
-; PRED-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
-; PRED-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP7]]
-; PRED-NEXT:    [[TMP9:%.*]] = add i8 [[TMP0]], 1
-; PRED-NEXT:    store i8 [[TMP9]], ptr [[TMP8]], align 1
-; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
-; PRED:       [[PRED_STORE_CONTINUE2]]:
-; PRED-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
-; PRED-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
-; PRED:       [[PRED_STORE_IF3]]:
-; PRED-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 2
-; PRED-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP11]]
-; PRED-NEXT:    [[TMP13:%.*]] = add i8 [[TMP0]], 2
-; PRED-NEXT:    store i8 [[TMP13]], ptr [[TMP12]], align 1
-; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
-; PRED:       [[PRED_STORE_CONTINUE4]]:
-; PRED-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
-; PRED-NEXT:    br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
-; PRED:       [[PRED_STORE_IF5]]:
-; PRED-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 3
-; PRED-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP15]]
-; PRED-NEXT:    [[TMP17:%.*]] = add i8 [[TMP0]], 3
-; PRED-NEXT:    store i8 [[TMP17]], ptr [[TMP16]], align 1
-; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
-; PRED:       [[PRED_STORE_CONTINUE6]]:
-; PRED-NEXT:    [[TMP18:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
-; PRED-NEXT:    br i1 [[TMP18]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
-; PRED:       [[PRED_STORE_IF7]]:
-; PRED-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX]], 4
-; PRED-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP19]]
-; PRED-NEXT:    [[TMP21:%.*]] = add i8 [[TMP0]], 4
-; PRED-NEXT:    store i8 [[TMP21]], ptr [[TMP20]], align 1
-; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
-; PRED:       [[PRED_STORE_CONTINUE8]]:
-; PRED-NEXT:    [[TMP22:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
-; PRED-NEXT:    br i1 [[TMP22]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
-; PRED:       [[PRED_STORE_IF9]]:
-; PRED-NEXT:    [[TMP23:%.*]] = add i64 [[INDEX]], 5
-; PRED-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP23]]
-; PRED-NEXT:    [[TMP25:%.*]] = add i8 [[TMP0]], 5
-; PRED-NEXT:    store i8 [[TMP25]], ptr [[TMP24]], align 1
-; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
-; PRED:       [[PRED_STORE_CONTINUE10]]:
-; PRED-NEXT:    [[TMP26:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
-; PRED-NEXT:    br i1 [[TMP26]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
-; PRED:       [[PRED_STORE_IF11]]:
-; PRED-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX]], 6
-; PRED-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP27]]
-; PRED-NEXT:    [[TMP29:%.*]] = add i8 [[TMP0]], 6
-; PRED-NEXT:    store i8 [[TMP29]], ptr [[TMP28]], align 1
-; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE12]]
-; PRED:       [[PRED_STORE_CONTINUE12]]:
-; PRED-NEXT:    [[TMP30:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
-; PRED-NEXT:    br i1 [[TMP30]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14]]
-; PRED:       [[PRED_STORE_IF13]]:
-; PRED-NEXT:    [[TMP31:%.*]] = add i64 [[INDEX]], 7
-; PRED-NEXT:    [[TMP32:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP31]]
-; PRED-NEXT:    [[TMP33:%.*]] = add i8 [[TMP0]], 7
-; PRED-NEXT:    store i8 [[TMP33]], ptr [[TMP32]], align 1
-; PRED-NEXT:    br label %[[PRED_STORE_CONTINUE14]]
-; PRED:       [[PRED_STORE_CONTINUE14]]:
-; PRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; PRED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i8> [[VEC_IND]], splat (i8 8)
-; PRED-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; PRED:       [[MIDDLE_BLOCK]]:
-; PRED-NEXT:    br label %[[EXIT:.*]]
-; PRED:       [[SCALAR_PH]]:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; PRED-NEXT:    br label %[[LOOP:.*]]
-; PRED:       [[LOOP]]:
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; PRED-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i8
-; PRED-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV]]
-; PRED-NEXT:    store i8 [[IV_TRUNC]], ptr [[GEP]], align 1
-; PRED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; PRED-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 7
-; PRED-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; PRED:       [[EXIT]]:
-; PRED-NEXT:    ret void
+; COMMON-LABEL: define void @low_trip_count_fold_tail_scalarized_store(
+; COMMON-SAME: ptr [[DST:%.*]]) {
+; COMMON-NEXT:  [[ENTRY:.*:]]
+; COMMON-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; COMMON:       [[VECTOR_PH]]:
+; COMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
+; COMMON:       [[VECTOR_BODY]]:
+; COMMON-NEXT:    br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; COMMON:       [[PRED_STORE_IF]]:
+; COMMON-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 0
+; COMMON-NEXT:    store i8 0, ptr [[TMP0]], align 1
+; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; COMMON:       [[PRED_STORE_CONTINUE]]:
+; COMMON-NEXT:    br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; COMMON:       [[PRED_STORE_IF1]]:
+; COMMON-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 1
+; COMMON-NEXT:    store i8 1, ptr [[TMP1]], align 1
+; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
+; COMMON:       [[PRED_STORE_CONTINUE2]]:
+; COMMON-NEXT:    br i1 true, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; COMMON:       [[PRED_STORE_IF3]]:
+; COMMON-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[DST]], i64 2
+; COMMON-NEXT:    store i8 2, ptr [[TMP2]], align 1
+; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
+; COMMON:       [[PRED_STORE_CONTINUE4]]:
+; COMMON-NEXT:    br i1 true, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; COMMON:       [[PRED_STORE_IF5]]:
+; COMMON-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 3
+; COMMON-NEXT:    store i8 3, ptr [[TMP3]], align 1
+; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
+; COMMON:       [[PRED_STORE_CONTINUE6]]:
+; COMMON-NEXT:    br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; COMMON:       [[PRED_STORE_IF7]]:
+; COMMON-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 4
+; COMMON-NEXT:    store i8 4, ptr [[TMP4]], align 1
+; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
+; COMMON:       [[PRED_STORE_CONTINUE8]]:
+; COMMON-NEXT:    br i1 true, label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; COMMON:       [[PRED_STORE_IF9]]:
+; COMMON-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DST]], i64 5
+; COMMON-NEXT:    store i8 5, ptr [[TMP5]], align 1
+; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
+; COMMON:       [[PRED_STORE_CONTINUE10]]:
+; COMMON-NEXT:    br i1 true, label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
+; COMMON:       [[PRED_STORE_IF11]]:
+; COMMON-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[DST]], i64 6
+; COMMON-NEXT:    store i8 6, ptr [[TMP6]], align 1
+; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE12]]
+; COMMON:       [[PRED_STORE_CONTINUE12]]:
+; COMMON-NEXT:    br i1 false, label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
+; COMMON:       [[PRED_STORE_IF13]]:
+; COMMON-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], i64 7
+; COMMON-NEXT:    store i8 7, ptr [[TMP7]], align 1
+; COMMON-NEXT:    br label %[[PRED_STORE_CONTINUE14]]
+; COMMON:       [[PRED_STORE_CONTINUE14]]:
+; COMMON-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
+; COMMON:       [[MIDDLE_BLOCK]]:
+; COMMON-NEXT:    br [[EXIT:label %.*]]
+; COMMON:       [[SCALAR_PH]]:
 ;
 entry:
   br label %loop
@@ -996,7 +688,7 @@ exit:
 define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias %src.2, ptr noalias %src.3, ptr noalias %src.4, ptr noalias %dst, i64 %N) #2 {
 ; DEFAULT-LABEL: define void @test_conditional_interleave_group(
 ; DEFAULT-SAME: ptr noalias [[SRC_1:%.*]], ptr noalias [[SRC_2:%.*]], ptr noalias [[SRC_3:%.*]], ptr noalias [[SRC_4:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR3:[0-9]+]] {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
@@ -1174,44 +866,15 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
 ; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
 ; DEFAULT-NEXT:    [[TMP80:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[TMP80]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[TMP80]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; DEFAULT-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; DEFAULT-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; DEFAULT-NEXT:    br label %[[LOOP_HEADER:.*]]
-; DEFAULT:       [[LOOP_HEADER]]:
-; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; DEFAULT-NEXT:    [[TMP81:%.*]] = load float, ptr [[SRC_1]], align 4
-; DEFAULT-NEXT:    [[TMP82:%.*]] = load float, ptr [[SRC_2]], align 4
-; DEFAULT-NEXT:    [[MUL8_I_US:%.*]] = fmul float [[TMP82]], 0.000000e+00
-; DEFAULT-NEXT:    [[TMP83:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP81]], float 0.000000e+00, float [[MUL8_I_US]])
-; DEFAULT-NEXT:    [[TMP84:%.*]] = load float, ptr [[SRC_3]], align 4
-; DEFAULT-NEXT:    [[TMP85:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP84]], float 0.000000e+00, float [[TMP83]])
-; DEFAULT-NEXT:    [[TMP86:%.*]] = load float, ptr [[SRC_3]], align 4
-; DEFAULT-NEXT:    [[C:%.*]] = fcmp ogt float [[TMP85]], [[TMP86]]
-; DEFAULT-NEXT:    br i1 [[C]], label %[[IF_THEN:.*]], label %[[LOOP_LATCH]]
-; DEFAULT:       [[IF_THEN]]:
-; DEFAULT-NEXT:    [[DST_0:%.*]] = getelementptr { [4 x float] }, ptr [[DST]], i64 [[IV]]
-; DEFAULT-NEXT:    store float 0.000000e+00, ptr [[DST_0]], align 4
-; DEFAULT-NEXT:    [[DST_1:%.*]] = getelementptr i8, ptr [[DST_0]], i64 4
-; DEFAULT-NEXT:    store float 0.000000e+00, ptr [[DST_1]], align 4
-; DEFAULT-NEXT:    [[DST_2:%.*]] = getelementptr i8, ptr [[DST_0]], i64 8
-; DEFAULT-NEXT:    store float 0.000000e+00, ptr [[DST_2]], align 4
-; DEFAULT-NEXT:    [[DST_3:%.*]] = getelementptr i8, ptr [[DST_0]], i64 16
-; DEFAULT-NEXT:    store float 0.000000e+00, ptr [[DST_0]], align 4
-; DEFAULT-NEXT:    br label %[[LOOP_LATCH]]
-; DEFAULT:       [[LOOP_LATCH]]:
-; DEFAULT-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; DEFAULT-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP27:![0-9]+]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret void
 ;
 ; PRED-LABEL: define void @test_conditional_interleave_group(
 ; PRED-SAME: ptr noalias [[SRC_1:%.*]], ptr noalias [[SRC_2:%.*]], ptr noalias [[SRC_3:%.*]], ptr noalias [[SRC_4:%.*]], ptr noalias [[DST:%.*]], i64 [[N:%.*]]) #[[ATTR3:[0-9]+]] {
-; PRED-NEXT:  [[ENTRY:.*]]:
+; PRED-NEXT:  [[ENTRY:.*:]]
 ; PRED-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; PRED:       [[VECTOR_SCEVCHECK]]:
@@ -1242,9 +905,6 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
 ; PRED-NEXT:    [[TMP14:%.*]] = or i1 [[TMP13]], [[TMP12]]
 ; PRED-NEXT:    br i1 [[TMP14]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 7
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP15:%.*]] = sub i64 [[TMP0]], 8
 ; PRED-NEXT:    [[TMP16:%.*]] = icmp ugt i64 [[TMP0]], 8
 ; PRED-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i64 [[TMP15]], i64 0
@@ -1394,42 +1054,13 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
 ; PRED:       [[PRED_STORE_CONTINUE27]]:
 ; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 [[INDEX]], i64 [[TMP17]])
-; PRED-NEXT:    [[TMP84:%.*]] = xor <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; PRED-NEXT:    [[TMP84:%.*]] = extractelement <8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP85:%.*]] = xor i1 [[TMP84]], true
 ; PRED-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
-; PRED-NEXT:    [[TMP85:%.*]] = extractelement <8 x i1> [[TMP84]], i32 0
-; PRED-NEXT:    br i1 [[TMP85]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; PRED-NEXT:    br i1 [[TMP85]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
-; PRED-NEXT:    br label %[[EXIT:.*]]
+; PRED-NEXT:    br [[EXIT:label %.*]]
 ; PRED:       [[SCALAR_PH]]:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
-; PRED-NEXT:    br label %[[LOOP_HEADER:.*]]
-; PRED:       [[LOOP_HEADER]]:
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; PRED-NEXT:    [[TMP86:%.*]] = load float, ptr [[SRC_1]], align 4
-; PRED-NEXT:    [[TMP87:%.*]] = load float, ptr [[SRC_2]], align 4
-; PRED-NEXT:    [[MUL8_I_US:%.*]] = fmul float [[TMP87]], 0.000000e+00
-; PRED-NEXT:    [[TMP88:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP86]], float 0.000000e+00, float [[MUL8_I_US]])
-; PRED-NEXT:    [[TMP89:%.*]] = load float, ptr [[SRC_3]], align 4
-; PRED-NEXT:    [[TMP90:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP89]], float 0.000000e+00, float [[TMP88]])
-; PRED-NEXT:    [[TMP91:%.*]] = load float, ptr [[SRC_3]], align 4
-; PRED-NEXT:    [[C:%.*]] = fcmp ogt float [[TMP90]], [[TMP91]]
-; PRED-NEXT:    br i1 [[C]], label %[[IF_THEN:.*]], label %[[LOOP_LATCH]]
-; PRED:       [[IF_THEN]]:
-; PRED-NEXT:    [[DST_0:%.*]] = getelementptr { [4 x float] }, ptr [[DST]], i64 [[IV]]
-; PRED-NEXT:    store float 0.000000e+00, ptr [[DST_0]], align 4
-; PRED-NEXT:    [[DST_1:%.*]] = getelementptr i8, ptr [[DST_0]], i64 4
-; PRED-NEXT:    store float 0.000000e+00, ptr [[DST_1]], align 4
-; PRED-NEXT:    [[DST_2:%.*]] = getelementptr i8, ptr [[DST_0]], i64 8
-; PRED-NEXT:    store float 0.000000e+00, ptr [[DST_2]], align 4
-; PRED-NEXT:    [[DST_3:%.*]] = getelementptr i8, ptr [[DST_0]], i64 16
-; PRED-NEXT:    store float 0.000000e+00, ptr [[DST_0]], align 4
-; PRED-NEXT:    br label %[[LOOP_LATCH]]
-; PRED:       [[LOOP_LATCH]]:
-; PRED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; PRED-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; PRED-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
-; PRED:       [[EXIT]]:
-; PRED-NEXT:    ret void
 ;
 entry:
   br label %loop.header
@@ -1469,7 +1100,7 @@ exit:
 define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) {
 ; DEFAULT-LABEL: define void @redundant_branch_and_tail_folding(
 ; DEFAULT-SAME: ptr [[DST:%.*]], i1 [[C:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -1484,29 +1115,14 @@ define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) {
 ; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
 ; DEFAULT-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; DEFAULT-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[SCALAR_PH]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; DEFAULT-NEXT:    br label %[[LOOP_HEADER:.*]]
-; DEFAULT:       [[LOOP_HEADER]]:
-; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; DEFAULT-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
-; DEFAULT:       [[THEN]]:
-; DEFAULT-NEXT:    br label %[[LOOP_LATCH]]
-; DEFAULT:       [[LOOP_LATCH]]:
-; DEFAULT-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; DEFAULT-NEXT:    [[T:%.*]] = trunc nuw nsw i64 [[IV_NEXT]] to i32
-; DEFAULT-NEXT:    store i32 [[T]], ptr [[DST]], align 4
-; DEFAULT-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 21
-; DEFAULT-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP29:![0-9]+]]
-; DEFAULT:       [[EXIT]]:
-; DEFAULT-NEXT:    ret void
 ;
 ; PRED-LABEL: define void @redundant_branch_and_tail_folding(
 ; PRED-SAME: ptr [[DST:%.*]], i1 [[C:%.*]]) {
-; PRED-NEXT:  [[ENTRY:.*]]:
+; PRED-NEXT:  [[ENTRY:.*:]]
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -1547,25 +1163,10 @@ define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) {
 ; PRED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; PRED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
 ; PRED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
-; PRED-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; PRED-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
-; PRED-NEXT:    br label %[[EXIT:.*]]
+; PRED-NEXT:    br [[EXIT:label %.*]]
 ; PRED:       [[SCALAR_PH]]:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; PRED-NEXT:    br label %[[LOOP_HEADER:.*]]
-; PRED:       [[LOOP_HEADER]]:
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; PRED-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
-; PRED:       [[THEN]]:
-; PRED-NEXT:    br label %[[LOOP_LATCH]]
-; PRED:       [[LOOP_LATCH]]:
-; PRED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; PRED-NEXT:    [[T:%.*]] = trunc nuw nsw i64 [[IV_NEXT]] to i32
-; PRED-NEXT:    store i32 [[T]], ptr [[DST]], align 4
-; PRED-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 21
-; PRED-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
-; PRED:       [[EXIT]]:
-; PRED-NEXT:    ret void
 ;
 entry:
   br label %loop.header
@@ -1588,8 +1189,212 @@ exit:
   ret void
 }
 
-; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+define double @test_cond_intrinsics(ptr %A, i1 %c) {
+; COMMON-LABEL: define double @test_cond_intrinsics(
+; COMMON-SAME: ptr [[A:%.*]], i1 [[C:%.*]]) {
+; COMMON-NEXT:  [[ENTRY:.*]]:
+; COMMON-NEXT:    br label %[[LOOP_HEADER:.*]]
+; COMMON:       [[LOOP_HEADER]]:
+; COMMON-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; COMMON-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
+; COMMON:       [[THEN]]:
+; COMMON-NEXT:    [[GEP:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
+; COMMON-NEXT:    [[L:%.*]] = load double, ptr [[GEP]], align 8
+; COMMON-NEXT:    [[POW:%.*]] = tail call double @llvm.pow.f64(double 2.000000e+00, double [[L]])
+; COMMON-NEXT:    store double [[POW]], ptr [[GEP]], align 8
+; COMMON-NEXT:    br label %[[LOOP_LATCH]]
+; COMMON:       [[LOOP_LATCH]]:
+; COMMON-NEXT:    [[P:%.*]] = phi double [ 0.000000e+00, %[[LOOP_HEADER]] ], [ [[POW]], %[[THEN]] ]
+; COMMON-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; COMMON-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 1024
+; COMMON-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
+; COMMON:       [[EXIT]]:
+; COMMON-NEXT:    [[P_LCSSA:%.*]] = phi double [ [[P]], %[[LOOP_LATCH]] ]
+; COMMON-NEXT:    ret double [[P_LCSSA]]
+;
+entry:
+  br label %loop.header
+
+loop.header:                                 ; preds = %loop.latch, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  br i1 %c, label %then, label %loop.latch
+
+then:
+  %gep = getelementptr inbounds double, ptr %A, i64 %iv
+  %l = load double, ptr %gep, align 8
+  %pow = tail call double @llvm.pow.f64(double 2.0, double %l)
+  store double %pow, ptr %gep
+  br label %loop.latch
+
+loop.latch:
+  %p = phi double [ 0.000000e+00, %loop.header ], [ %pow, %then ]
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, 1024
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret double %p
+}
+
+define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
+; DEFAULT-LABEL: define void @pred_udiv_select_cost(
+; DEFAULT-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]], i8 [[Y:%.*]]) #[[ATTR2]] {
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
+; DEFAULT-NEXT:    [[B3:%.*]] = ptrtoint ptr [[B]] to i64
+; DEFAULT-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; DEFAULT-NEXT:    [[C1:%.*]] = ptrtoint ptr [[C]] to i64
+; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
+; DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
+; DEFAULT-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP2]])
+; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
+; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; DEFAULT:       [[VECTOR_MEMCHECK]]:
+; DEFAULT-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; DEFAULT-NEXT:    [[TMP6:%.*]] = sub i64 [[C1]], [[A2]]
+; DEFAULT-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
+; DEFAULT-NEXT:    [[TMP7:%.*]] = sub i64 [[C1]], [[B3]]
+; DEFAULT-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP7]], [[TMP5]]
+; DEFAULT-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; DEFAULT-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; DEFAULT:       [[VECTOR_PH]]:
+; DEFAULT-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; DEFAULT-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
+; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP9]]
+; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[Y]], i64 0
+; DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
+; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
+; DEFAULT:       [[VECTOR_BODY]]:
+; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEFAULT-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
+; DEFAULT-NEXT:    [[TMP11:%.*]] = uitofp <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x float>
+; DEFAULT-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; DEFAULT-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
+; DEFAULT-NEXT:    [[TMP13:%.*]] = icmp eq <vscale x 4 x i8> [[WIDE_LOAD5]], zeroinitializer
+; DEFAULT-NEXT:    [[TMP14:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD]], splat (i8 1)
+; DEFAULT-NEXT:    [[TMP15:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i8> splat (i8 1), <vscale x 4 x i8> [[BROADCAST_SPLAT]]
+; DEFAULT-NEXT:    [[TMP16:%.*]] = udiv <vscale x 4 x i8> [[TMP14]], [[TMP15]]
+; DEFAULT-NEXT:    [[TMP17:%.*]] = icmp ugt <vscale x 4 x i8> [[TMP16]], splat (i8 1)
+; DEFAULT-NEXT:    [[TMP18:%.*]] = select <vscale x 4 x i1> [[TMP17]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> splat (i32 255)
+; DEFAULT-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP18]]
+; DEFAULT-NEXT:    [[TMP19:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; DEFAULT-NEXT:    [[TMP20:%.*]] = sub <vscale x 4 x i32> [[PREDPHI]], [[TMP19]]
+; DEFAULT-NEXT:    [[TMP21:%.*]] = sitofp <vscale x 4 x i32> [[TMP20]] to <vscale x 4 x float>
+; DEFAULT-NEXT:    [[TMP22:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP21]], <vscale x 4 x float> splat (float 3.000000e+00), <vscale x 4 x float> [[TMP11]])
+; DEFAULT-NEXT:    [[TMP23:%.*]] = fptoui <vscale x 4 x float> [[TMP22]] to <vscale x 4 x i8>
+; DEFAULT-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
+; DEFAULT-NEXT:    store <vscale x 4 x i8> [[TMP23]], ptr [[TMP24]], align 1
+; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; DEFAULT-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; DEFAULT-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; DEFAULT:       [[MIDDLE_BLOCK]]:
+; DEFAULT-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; DEFAULT-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; DEFAULT:       [[SCALAR_PH]]:
+;
+; PRED-LABEL: define void @pred_udiv_select_cost(
+; PRED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], i64 [[N:%.*]], i8 [[Y:%.*]]) #[[ATTR2]] {
+; PRED-NEXT:  [[ENTRY:.*:]]
+; PRED-NEXT:    [[B3:%.*]] = ptrtoint ptr [[B]] to i64
+; PRED-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; PRED-NEXT:    [[C1:%.*]] = ptrtoint ptr [[C]] to i64
+; PRED-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
+; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; PRED:       [[VECTOR_MEMCHECK]]:
+; PRED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; PRED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
+; PRED-NEXT:    [[TMP3:%.*]] = sub i64 [[C1]], [[A2]]
+; PRED-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
+; PRED-NEXT:    [[TMP4:%.*]] = sub i64 [[C1]], [[B3]]
+; PRED-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP4]], [[TMP2]]
+; PRED-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; PRED-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; PRED:       [[VECTOR_PH]]:
+; PRED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; PRED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
+; PRED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; PRED-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 4
+; PRED-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP0]], [[TMP8]]
+; PRED-NEXT:    [[TMP10:%.*]] = icmp ugt i64 [[TMP0]], [[TMP8]]
+; PRED-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 0
+; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP0]])
+; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[Y]], i64 0
+; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
+; PRED-NEXT:    br label %[[VECTOR_BODY:.*]]
+; PRED:       [[VECTOR_BODY]]:
+; PRED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; PRED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP12]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; PRED-NEXT:    [[TMP13:%.*]] = uitofp <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x float>
+; PRED-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; PRED-NEXT:    [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP14]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; PRED-NEXT:    [[TMP15:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]], zeroinitializer
+; PRED-NEXT:    [[TMP16:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP15]], <vscale x 16 x i1> zeroinitializer
+; PRED-NEXT:    [[TMP17:%.*]] = xor <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], splat (i8 1)
+; PRED-NEXT:    [[TMP18:%.*]] = select <vscale x 16 x i1> [[TMP16]], <vscale x 16 x i8> [[BROADCAST_SPLAT]], <vscale x 16 x i8> splat (i8 1)
+; PRED-NEXT:    [[TMP19:%.*]] = udiv <vscale x 16 x i8> [[TMP17]], [[TMP18]]
+; PRED-NEXT:    [[TMP20:%.*]] = icmp ugt <vscale x 16 x i8> [[TMP19]], splat (i8 1)
+; PRED-NEXT:    [[TMP21:%.*]] = select <vscale x 16 x i1> [[TMP20]], <vscale x 16 x i32> zeroinitializer, <vscale x 16 x i32> splat (i32 255)
+; PRED-NEXT:    [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP16]], <vscale x 16 x i32> [[TMP21]], <vscale x 16 x i32> zeroinitializer
+; PRED-NEXT:    [[TMP22:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; PRED-NEXT:    [[TMP23:%.*]] = sub <vscale x 16 x i32> [[PREDPHI]], [[TMP22]]
+; PRED-NEXT:    [[TMP24:%.*]] = sitofp <vscale x 16 x i32> [[TMP23]] to <vscale x 16 x float>
+; PRED-NEXT:    [[TMP25:%.*]] = call <vscale x 16 x float> @llvm.fmuladd.nxv16f32(<vscale x 16 x float> [[TMP24]], <vscale x 16 x float> splat (float 3.000000e+00), <vscale x 16 x float> [[TMP13]])
+; PRED-NEXT:    [[TMP26:%.*]] = fptoui <vscale x 16 x float> [[TMP25]] to <vscale x 16 x i8>
+; PRED-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[C]], i64 [[INDEX]]
+; PRED-NEXT:    call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP26]], ptr [[TMP27]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP11]])
+; PRED-NEXT:    [[TMP28:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP29:%.*]] = xor i1 [[TMP28]], true
+; PRED-NEXT:    br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; PRED:       [[MIDDLE_BLOCK]]:
+; PRED-NEXT:    br [[EXIT:label %.*]]
+; PRED:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop.header
+
+loop.header:                                    ; preds = %loop.latch, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep.A = getelementptr i8, ptr %A, i64 %iv
+  %l.1 = load i8, ptr %gep.A, align 1
+  %l.1.to.float = uitofp i8 %l.1 to float
+  %gep.B = getelementptr i8, ptr %B, i64 %iv
+  %l.2 = load i8, ptr %gep.B, align 1
+  %c = icmp eq i8 %l.2, 0
+  br i1 %c, label %loop.latch, label %else
+
+else:
+  %x = xor i8 %l.1, 1
+  %ud = udiv i8 %x, %y
+  %c.sel = icmp ugt i8 %ud, 1
+  %sel = select i1 %c.sel, i32 0, i32 255
+  br label %loop.latch
+
+loop.latch:
+  %p = phi i32 [ %sel, %else ], [ 0, %loop.header ]
+  %l.1.ext = zext i8 %l.1 to i32
+  %sub = sub i32 %p, %l.1.ext
+  %sub.to.float = sitofp i32 %sub to float
+  %muladd = tail call float @llvm.fmuladd.f32(float %sub.to.float, float 3.000000e+00, float %l.1.to.float)
+  %muladd.conv = fptoui float %muladd to i8
+  %gep.C = getelementptr i8, ptr %C, i64 %iv
+  store i8 %muladd.conv, ptr %gep.C, align 1
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, %n
+  br i1 %ec, label %exit, label %loop.header
+
+exit:
+  ret void
+}
+
 declare float @llvm.fmuladd.f32(float, float, float) #1
+declare double @llvm.pow.f64(double, double)
 
 attributes #1 = { "target-cpu"="neoverse-512tvb" }
 attributes #2 = { vscale_range(2,2) "target-cpu"="neoverse-512tvb" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
index 1ad1e42678c5a..bfebbdad5af0b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
@@ -7,8 +7,8 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK-SAME: ptr [[DST:%.*]], i32 [[X:%.*]], i64 [[M:%.*]], i64 [[CONV6:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CONV61:%.*]] = zext i32 [[X]] to i64
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
@@ -24,8 +24,6 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP9]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = sdiv i64 [[M]], [[CONV6]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP18]] to i32
 ; CHECK-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP18]], [[CONV61]]
@@ -39,11 +37,11 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK-NEXT:    [[TMP32:%.*]] = sext i32 [[TMP30]] to i64
 ; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP32]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP38:%.*]] = mul nuw i64 [[TMP37]], 2
+; CHECK-NEXT:    [[TMP38:%.*]] = shl nuw i64 [[TMP37]], 1
 ; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr double, ptr [[TMP34]], i64 [[TMP38]]
 ; CHECK-NEXT:    store <vscale x 2 x double> zeroinitializer, ptr [[TMP34]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x double> zeroinitializer, ptr [[TMP39]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -110,14 +108,8 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
@@ -127,8 +119,7 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
 ; CHECK-NEXT:    [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    [[TMP17:%.*]] = mul <vscale x 2 x i64> [[TMP15]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP17]]
-; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 1, [[TMP9]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP20]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
@@ -149,12 +140,12 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
 ; CHECK-NEXT:    [[TMP33:%.*]] = sext i32 [[TMP32]] to i64
 ; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP33]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> zeroinitializer, ptr [[TMP34]], i32 8, <vscale x 2 x i1> [[TMP23]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP14]])
-; CHECK-NEXT:    [[TMP36:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP36:%.*]] = xor i1 [[TMP35]], true
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <vscale x 2 x i1> [[TMP36]], i32 0
-; CHECK-NEXT:    br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -232,14 +223,8 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP7]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[TMP0]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[TMP0]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
@@ -249,8 +234,7 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    [[TMP17:%.*]] = mul <vscale x 2 x i64> [[TMP15]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP17]]
-; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 1, [[TMP9]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP20]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP6]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
@@ -274,12 +258,12 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK-NEXT:    [[TMP37:%.*]] = ashr i64 [[TMP36]], 32
 ; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP37]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP23]], ptr [[TMP38]], i32 4, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP14]])
-; CHECK-NEXT:    [[TMP47:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP40:%.*]] = xor i1 [[TMP39]], true
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT4]]
-; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <vscale x 2 x i1> [[TMP47]], i32 0
-; CHECK-NEXT:    br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP40]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll
index 221d944e1bc2d..0cb46e18c5367 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll
@@ -8,7 +8,7 @@ target triple = "aarch64-unknown-linux"
 define void @check_widen_intrinsic_with_nnan(ptr noalias %dst.0, ptr noalias %dst.1, ptr noalias %src.1, ptr %src.2) {
 ; CHECK-LABEL: define void @check_widen_intrinsic_with_nnan(
 ; CHECK-SAME: ptr noalias [[DST_0:%.*]], ptr noalias [[DST_1:%.*]], ptr noalias [[SRC_1:%.*]], ptr [[SRC_2:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -70,10 +70,9 @@ define void @check_widen_intrinsic_with_nnan(ptr noalias %dst.0, ptr noalias %ds
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds double, ptr [[SRC_1]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_1:%.*]] = load double, ptr [[GEP_SRC_1]], align 8
 ; CHECK-NEXT:    [[ABS:%.*]] = tail call nnan double @llvm.fabs.f64(double [[L_1]])
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll
index ab008e76cccbd..7363f86cdab7a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll
@@ -10,23 +10,21 @@ define void @f1(ptr %A) #0 {
 ; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 4 x i32> splat (i32 1), ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    store <vscale x 4 x i32> splat (i32 1), ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
index 37b5f5dc16958..b83d3af3a0d65 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
@@ -333,3 +333,185 @@ for.body:
 exit:
   ret void
 }
+
+; TODO: Choose smaller VF * UF for main loop, so we do not create a dead vector loop.
+define void @small_trip_count_loop(ptr %arg, ptr %arg2) {
+; CHECK-LABEL: @small_trip_count_loop(
+; CHECK-NEXT:  iter.check:
+; CHECK-NEXT:    [[ARG3:%.*]] = ptrtoint ptr [[ARG:%.*]] to i64
+; CHECK-NEXT:    [[ARG21:%.*]] = ptrtoint ptr [[ARG2:%.*]] to i64
+; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; CHECK:       vector.memcheck:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[ARG21]], [[ARG3]]
+; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 64
+; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK:       vector.main.loop.iter.check:
+; CHECK-NEXT:    br i1 true, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 16
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 32
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 48
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[ARG]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <16 x i8>, ptr [[TMP1]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = add <16 x i8> [[WIDE_LOAD]], splat (i8 10)
+; CHECK-NEXT:    [[TMP5:%.*]] = add <16 x i8> [[WIDE_LOAD4]], splat (i8 10)
+; CHECK-NEXT:    [[TMP6:%.*]] = add <16 x i8> [[WIDE_LOAD5]], splat (i8 10)
+; CHECK-NEXT:    [[TMP7:%.*]] = add <16 x i8> [[WIDE_LOAD6]], splat (i8 10)
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 16
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 32
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 48
+; CHECK-NEXT:    store <16 x i8> [[TMP4]], ptr [[ARG2]], align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP5]], ptr [[TMP8]], align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP6]], ptr [[TMP9]], align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP7]], ptr [[TMP10]], align 1
+; CHECK-NEXT:    br label [[MIDDLE_BLOCK:%.*]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK:       vec.epilog.iter.check:
+; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 0, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK:       vec.epilog.vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <16 x i8>, ptr [[TMP11]], align 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add <16 x i8> [[WIDE_LOAD7]], splat (i8 10)
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[INDEX]]
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
+; CHECK-NEXT:    br i1 [[TMP14]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       vec.epilog.middle.block:
+; CHECK-NEXT:    br i1 false, label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       vec.epilog.scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 16, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i32 [[IV]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load i8, ptr [[GEP_A]], align 1
+; CHECK-NEXT:    [[SELECT:%.*]] = add i8 [[LOAD]], 10
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i32 [[IV]]
+; CHECK-NEXT:    store i8 [[SELECT]], ptr [[GEP_B]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 20
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.A = getelementptr inbounds i8, ptr %arg, i32 %iv
+  %load = load i8, ptr %gep.A, align 1
+  %select = add i8 %load, 10
+  %gep.B = getelementptr inbounds i8, ptr %arg2, i32 %iv
+  store i8 %select, ptr %gep.B, align 1
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv, 20
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; TODO: Should not try to create dead main vector loop.
+define void @trip_count_based_on_ptrtoint(i64 %x) "target-cpu"="apple-m1" {
+; CHECK-LABEL: @trip_count_based_on_ptrtoint(
+; CHECK-NEXT:  iter.check:
+; CHECK-NEXT:    [[PTR_START:%.*]] = inttoptr i64 [[X:%.*]] to ptr
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[X]], 40
+; CHECK-NEXT:    [[PTR_END:%.*]] = inttoptr i64 [[ADD]] to ptr
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[ADD]], [[X]]
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; CHECK:       vector.scevcheck:
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[ADD]] to i2
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[X]] to i2
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i2 [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i2 [[TMP5]] to i64
+; CHECK-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[IDENT_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
+; CHECK:       vector.main.loop.iter.check:
+; CHECK-NEXT:    br i1 true, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 16
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[INDEX]], 4
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i32 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i32 8
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[NEXT_GEP]], i32 12
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP8]], align 4
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP9]], align 4
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
+; CHECK:       vec.epilog.iter.check:
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[N_VEC]], 4
+; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[TMP12]]
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP2]], [[N_VEC]]
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 4
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
+; CHECK:       vec.epilog.ph:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[N_MOD_VF1:%.*]] = urem i64 [[TMP2]], 4
+; CHECK-NEXT:    [[N_VEC2:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF1]]
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[N_VEC2]], 4
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[TMP13]]
+; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
+; CHECK:       vec.epilog.vector.body:
+; CHECK-NEXT:    [[INDEX3:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT5:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX3]], 4
+; CHECK-NEXT:    [[NEXT_GEP4:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    store <4 x i32> zeroinitializer, ptr [[NEXT_GEP4]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT5]] = add nuw i64 [[INDEX3]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT5]], [[N_VEC2]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       vec.epilog.middle.block:
+; CHECK-NEXT:    [[CMP_N6:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC2]]
+; CHECK-NEXT:    br i1 [[CMP_N6]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       vec.epilog.scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP14]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[PTR_START]], [[VECTOR_SCEVCHECK]] ], [ [[PTR_START]], [[ITER_CHECK:%.*]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    store i32 0, ptr [[IV]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr i8, ptr [[IV]], i64 4
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq ptr [[IV]], [[PTR_END]]
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %ptr.start = inttoptr i64 %x to ptr
+  %add = add i64 %x, 40
+  %ptr.end = inttoptr i64 %add to ptr
+  br label %loop
+
+loop:
+  %iv = phi ptr [ %ptr.start, %entry ], [ %iv.next, %loop ]
+  store i32 0, ptr %iv, align 4
+  %iv.next = getelementptr i8, ptr %iv, i64 4
+  %ec = icmp eq ptr %iv, %ptr.end
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll
index d9710328d6048..f3542f63a4273 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll
@@ -4,73 +4,102 @@
 define double @fp128_fmuladd_reduction(ptr %start0, ptr %start1, ptr %end0, ptr %end1, double %x, i64 %n) {
 ; CHECK-LABEL: define double @fp128_fmuladd_reduction(
 ; CHECK-SAME: ptr [[START0:%.*]], ptr [[START1:%.*]], ptr [[END0:%.*]], ptr [[END1:%.*]], double [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ITER_CHECK:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[N_VEC]], 16
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[N_VEC]], 8
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP2]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi double [ [[X]], %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 16
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 16
-; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 32
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 48
-; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START0]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP4]]
-; CHECK-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP5]]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi double [ [[X]], %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr fp128, ptr [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr fp128, ptr [[TMP1]], i32 4
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr fp128, ptr [[TMP1]], i32 6
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x fp128>, ptr [[TMP1]], align 16
+; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <2 x fp128>, ptr [[TMP24]], align 16
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <2 x fp128>, ptr [[TMP4]], align 16
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <2 x fp128>, ptr [[TMP5]], align 16
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr double, ptr [[TMP3]], i32 2
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr double, ptr [[TMP3]], i32 4
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr double, ptr [[TMP3]], i32 6
+; CHECK-NEXT:    [[WIDE_LOAD6:%.*]] = load <2 x double>, ptr [[TMP3]], align 16
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <2 x double>, ptr [[TMP28]], align 16
+; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <2 x double>, ptr [[TMP35]], align 16
+; CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <2 x double>, ptr [[TMP36]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD]] to <2 x double>
+; CHECK-NEXT:    [[TMP11:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD3]] to <2 x double>
+; CHECK-NEXT:    [[TMP12:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD4]] to <2 x double>
+; CHECK-NEXT:    [[TMP13:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD5]] to <2 x double>
+; CHECK-NEXT:    [[TMP14:%.*]] = fmul <2 x double> [[TMP10]], [[WIDE_LOAD6]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul <2 x double> [[TMP11]], [[WIDE_LOAD7]]
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul <2 x double> [[TMP12]], [[WIDE_LOAD8]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fmul <2 x double> [[TMP13]], [[WIDE_LOAD9]]
+; CHECK-NEXT:    [[TMP18:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> [[TMP14]])
+; CHECK-NEXT:    [[TMP19:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double [[TMP18]], <2 x double> [[TMP15]])
+; CHECK-NEXT:    [[TMP20:%.*]] = call double @llvm.vector.reduce.fadd.v2f64(double [[TMP19]], <2 x double> [[TMP16]])
+; CHECK-NEXT:    [[TMP21]] = call double @llvm.vector.reduce.fadd.v2f64(double [[TMP20]], <2 x double> [[TMP17]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[N_VEC]], 16
 ; CHECK-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP6]]
-; CHECK-NEXT:    [[OFFSET_IDX4:%.*]] = mul i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX4]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = add i64 [[OFFSET_IDX4]], 16
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[OFFSET_IDX4]], 24
-; CHECK-NEXT:    [[NEXT_GEP5:%.*]] = getelementptr i8, ptr [[START1]], i64 [[OFFSET_IDX4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[N_VEC]], 8
 ; CHECK-NEXT:    [[NEXT_GEP6:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP7]]
+; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2
+; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
+; CHECK:       [[VEC_EPILOG_PH]]:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP21]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[X]], %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[N_MOD_VF10:%.*]] = urem i64 [[N]], 2
+; CHECK-NEXT:    [[N_VEC11:%.*]] = sub i64 [[N]], [[N_MOD_VF10]]
+; CHECK-NEXT:    [[TMP25:%.*]] = mul i64 [[N_VEC11]], 16
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[START0]], i64 [[TMP25]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[N_VEC11]], 8
 ; CHECK-NEXT:    [[NEXT_GEP7:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP8]]
+; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT19:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI13:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[VEC_EPILOG_PH]] ], [ [[TMP33:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX12]], 16
+; CHECK-NEXT:    [[NEXT_GEP14:%.*]] = getelementptr i8, ptr [[START0]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[INDEX12]], 8
 ; CHECK-NEXT:    [[NEXT_GEP8:%.*]] = getelementptr i8, ptr [[START1]], i64 [[TMP9]]
-; CHECK-NEXT:    [[TMP10:%.*]] = load fp128, ptr [[NEXT_GEP]], align 16
-; CHECK-NEXT:    [[TMP11:%.*]] = load fp128, ptr [[NEXT_GEP1]], align 16
-; CHECK-NEXT:    [[TMP12:%.*]] = load fp128, ptr [[NEXT_GEP2]], align 16
-; CHECK-NEXT:    [[TMP13:%.*]] = load fp128, ptr [[NEXT_GEP3]], align 16
-; CHECK-NEXT:    [[TMP14:%.*]] = load double, ptr [[NEXT_GEP5]], align 16
-; CHECK-NEXT:    [[TMP15:%.*]] = load double, ptr [[NEXT_GEP6]], align 16
-; CHECK-NEXT:    [[TMP16:%.*]] = load double, ptr [[NEXT_GEP7]], align 16
-; CHECK-NEXT:    [[TMP17:%.*]] = load double, ptr [[NEXT_GEP8]], align 16
-; CHECK-NEXT:    [[TMP18:%.*]] = fptrunc fp128 [[TMP10]] to double
-; CHECK-NEXT:    [[TMP19:%.*]] = fptrunc fp128 [[TMP11]] to double
-; CHECK-NEXT:    [[TMP20:%.*]] = fptrunc fp128 [[TMP12]] to double
-; CHECK-NEXT:    [[TMP21:%.*]] = fptrunc fp128 [[TMP13]] to double
-; CHECK-NEXT:    [[TMP22:%.*]] = fmul double [[TMP18]], [[TMP14]]
-; CHECK-NEXT:    [[TMP23:%.*]] = fmul double [[TMP19]], [[TMP15]]
-; CHECK-NEXT:    [[TMP24:%.*]] = fmul double [[TMP20]], [[TMP16]]
-; CHECK-NEXT:    [[TMP25:%.*]] = fmul double [[TMP21]], [[TMP17]]
-; CHECK-NEXT:    [[TMP26:%.*]] = fadd double [[VEC_PHI]], [[TMP22]]
-; CHECK-NEXT:    [[TMP27:%.*]] = fadd double [[TMP26]], [[TMP23]]
-; CHECK-NEXT:    [[TMP28:%.*]] = fadd double [[TMP27]], [[TMP24]]
-; CHECK-NEXT:    [[TMP29]] = fadd double [[TMP28]], [[TMP25]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[START0]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL9:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[START1]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL10:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ [[TMP29]], %[[MIDDLE_BLOCK]] ], [ [[X]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[WIDE_LOAD17:%.*]] = load <2 x fp128>, ptr [[NEXT_GEP14]], align 16
+; CHECK-NEXT:    [[WIDE_LOAD18:%.*]] = load <2 x double>, ptr [[NEXT_GEP8]], align 16
+; CHECK-NEXT:    [[TMP31:%.*]] = fptrunc <2 x fp128> [[WIDE_LOAD17]] to <2 x double>
+; CHECK-NEXT:    [[TMP32:%.*]] = fmul <2 x double> [[TMP31]], [[WIDE_LOAD18]]
+; CHECK-NEXT:    [[TMP33]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI13]], <2 x double> [[TMP32]])
+; CHECK-NEXT:    [[INDEX_NEXT19]] = add nuw i64 [[INDEX12]], 2
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT19]], [[N_VEC11]]
+; CHECK-NEXT:    br i1 [[TMP34]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N20:%.*]] = icmp eq i64 [[N]], [[N_VEC11]]
+; CHECK-NEXT:    br i1 [[CMP_N20]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP26]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[NEXT_GEP3]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START0]], %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL21:%.*]] = phi ptr [ [[NEXT_GEP7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[NEXT_GEP6]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START1]], %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL22:%.*]] = phi i64 [ [[N_VEC11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX23:%.*]] = phi double [ [[TMP33]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP21]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[X]], %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[PTR0:%.*]] = phi ptr [ [[PTR0_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ [[PTR1_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL9]], %[[SCALAR_PH]] ]
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL10]], %[[SCALAR_PH]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi double [ [[RED_NEXT:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR0:%.*]] = phi ptr [ [[PTR0_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR1:%.*]] = phi ptr [ [[PTR1_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL21]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL22]], %[[VEC_EPILOG_SCALAR_PH]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi double [ [[RED_NEXT:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX23]], %[[VEC_EPILOG_SCALAR_PH]] ]
 ; CHECK-NEXT:    [[PTR0_NEXT]] = getelementptr i8, ptr [[PTR0]], i64 16
 ; CHECK-NEXT:    [[PTR1_NEXT]] = getelementptr i8, ptr [[PTR1]], i64 8
 ; CHECK-NEXT:    [[LOAD0:%.*]] = load fp128, ptr [[PTR0]], align 16
@@ -79,9 +108,9 @@ define double @fp128_fmuladd_reduction(ptr %start0, ptr %start1, ptr %end0, ptr
 ; CHECK-NEXT:    [[RED_NEXT]] = tail call double @llvm.fmuladd.f64(double [[TRUNC]], double [[LOAD1]], double [[RED]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[CMP1_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[CMP1_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP1_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[LCSSA:%.*]] = phi double [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP29]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[LCSSA:%.*]] = phi double [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[TMP33]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret double [[LCSSA]]
 ;
 entry:
@@ -110,5 +139,6 @@ exit:
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
index fff99f1498ae7..30109973b91aa 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
@@ -64,19 +64,17 @@ define i32 @test_phi_iterator_invalidation(ptr %A, ptr noalias %B) {
 ; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP26]], ptr [[TMP28]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX_NEXT]], i64 1002)
-; CHECK-NEXT:    [[TMP30:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP30:%.*]] = xor i1 [[TMP29]], true
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[TMP30]], i32 0
-; CHECK-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[SEXT:%.*]] = sext i16 [[SCALAR_RECUR]] to i32
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV_NEXT]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
index 5de9d0e3fc93f..3475b951e54f5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
@@ -75,8 +75,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP6]], true
 ; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
 ; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
index ea44fc35e1484..ab4ec848ad4c5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
@@ -75,8 +75,7 @@ define float @fminnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.minnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP6]], true
 ; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
 ; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index c9cef142e484c..ad184bec2ac7b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -6,29 +6,24 @@ target triple = "arm64-apple-macosx14.0.0"
 
 define double @test_reduction_costs() {
 ; CHECK-LABEL: define double @test_reduction_costs() {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP0:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI]], <2 x double> splat (double 3.000000e+00))
 ; CHECK-NEXT:    [[TMP1]] = call double @llvm.vector.reduce.fadd.v2f64(double [[VEC_PHI1]], <2 x double> splat (double 9.000000e+00))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_1:.*]]
 ; CHECK:       [[LOOP_1]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_1]] ]
-; CHECK-NEXT:    [[R_1:%.*]] = phi double [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[R_1_NEXT:%.*]], %[[LOOP_1]] ]
-; CHECK-NEXT:    [[R_2:%.*]] = phi double [ [[BC_MERGE_RDX2]], %[[SCALAR_PH]] ], [ [[R_2_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[R_1:%.*]] = phi double [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[R_1_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[R_2:%.*]] = phi double [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[R_2_NEXT:%.*]], %[[LOOP_1]] ]
 ; CHECK-NEXT:    [[R_1_NEXT]] = fadd double [[R_1]], 3.000000e+00
 ; CHECK-NEXT:    [[R_2_NEXT]] = fadd double [[R_2]], 9.000000e+00
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll
index 596a2eddfc96a..4ddf51b9bdd58 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll
@@ -72,7 +72,7 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur
 ; SVE:       for.body.preheader:
 ; SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[SIZE]] to i64
 ; SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SVE:       vector.ph:
@@ -80,8 +80,6 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur
 ; SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; SVE-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; SVE-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SVE:       vector.body:
 ; SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -92,7 +90,7 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur
 ; SVE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DATA:%.*]], <vscale x 2 x i64> [[TMP7]]
 ; SVE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> [[TMP8]], i32 8, <vscale x 2 x i1> splat (i1 true), <vscale x 2 x double> poison)
 ; SVE-NEXT:    [[TMP9]] = fadd <vscale x 2 x double> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; SVE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; SVE-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SVE:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index 8b354d91909b1..137e07336fd50 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -13,7 +13,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
+; DEFAULT-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; DEFAULT:       [[VECTOR_MEMCHECK]]:
@@ -28,8 +28,6 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 16
 ; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP10]]
 ; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; DEFAULT-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
 ; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
 ; DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
 ; DEFAULT-NEXT:    [[TMP13:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
@@ -38,7 +36,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; DEFAULT-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
 ; DEFAULT-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 8
+; DEFAULT-NEXT:    [[TMP24:%.*]] = shl nuw i64 [[TMP23]], 3
 ; DEFAULT-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP20]], i64 [[TMP24]]
 ; DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP20]], align 1
 ; DEFAULT-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP25]], align 1
@@ -56,11 +54,11 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[TMP37:%.*]] = trunc <vscale x 8 x i16> [[TMP35]] to <vscale x 8 x i8>
 ; DEFAULT-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
 ; DEFAULT-NEXT:    [[TMP41:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP42:%.*]] = mul nuw i64 [[TMP41]], 8
+; DEFAULT-NEXT:    [[TMP42:%.*]] = shl nuw i64 [[TMP41]], 3
 ; DEFAULT-NEXT:    [[TMP43:%.*]] = getelementptr i8, ptr [[TMP38]], i64 [[TMP42]]
 ; DEFAULT-NEXT:    store <vscale x 8 x i8> [[TMP36]], ptr [[TMP38]], align 1
 ; DEFAULT-NEXT:    store <vscale x 8 x i8> [[TMP37]], ptr [[TMP43]], align 1
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; DEFAULT-NEXT:    [[TMP44:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[TMP44]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
@@ -103,16 +101,10 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
-; PRED-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP5]], 1
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP8]]
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; PRED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 16
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[X]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PRED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
+; PRED-NEXT:    [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 4
 ; PRED-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP0]], [[TMP12]]
 ; PRED-NEXT:    [[TMP14:%.*]] = icmp ugt i64 [[TMP0]], [[TMP12]]
 ; PRED-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[TMP13]], i64 0
@@ -132,11 +124,11 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; PRED-NEXT:    [[TMP23:%.*]] = trunc <vscale x 16 x i16> [[TMP21]] to <vscale x 16 x i8>
 ; PRED-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
 ; PRED-NEXT:    call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP23]], ptr [[TMP26]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]]
+; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP15]])
-; PRED-NEXT:    [[TMP28:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; PRED-NEXT:    [[TMP29:%.*]] = extractelement <vscale x 16 x i1> [[TMP28]], i32 0
-; PRED-NEXT:    br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; PRED-NEXT:    [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP27:%.*]] = xor i1 [[TMP25]], true
+; PRED-NEXT:    br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
 ; PRED-NEXT:    br label %[[EXIT:.*]]
 ; PRED:       [[SCALAR_PH]]:
@@ -270,9 +262,6 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED-NEXT:    [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]]
 ; PRED-NEXT:    br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 1
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP0]], 2
 ; PRED-NEXT:    [[TMP14:%.*]] = icmp ugt i64 [[TMP0]], 2
 ; PRED-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[TMP13]], i64 0
@@ -304,9 +293,9 @@ define void @iv_trunc(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED:       [[PRED_STORE_CONTINUE2]]:
 ; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP15]])
-; PRED-NEXT:    [[TMP24:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; PRED-NEXT:    [[TMP24:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP25:%.*]] = xor i1 [[TMP24]], true
 ; PRED-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
-; PRED-NEXT:    [[TMP25:%.*]] = extractelement <2 x i1> [[TMP24]], i32 0
 ; PRED-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
 ; PRED-NEXT:    br label %[[EXIT:.*]]
@@ -441,9 +430,6 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED-NEXT:    [[TMP13:%.*]] = or i1 [[TMP9]], [[TMP12]]
 ; PRED-NEXT:    br i1 [[TMP13]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP14:%.*]] = sub i64 [[TMP0]], 4
 ; PRED-NEXT:    [[TMP15:%.*]] = icmp ugt i64 [[TMP0]], 4
 ; PRED-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i64 [[TMP14]], i64 0
@@ -496,9 +482,9 @@ define void @trunc_ivs_and_store(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED:       [[PRED_STORE_CONTINUE7]]:
 ; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 [[TMP16]])
-; PRED-NEXT:    [[TMP35:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; PRED-NEXT:    [[TMP35:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP36:%.*]] = xor i1 [[TMP35]], true
 ; PRED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
-; PRED-NEXT:    [[TMP36:%.*]] = extractelement <4 x i1> [[TMP35]], i32 0
 ; PRED-NEXT:    br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
 ; PRED-NEXT:    br label %[[EXIT:.*]]
@@ -635,9 +621,6 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED-NEXT:    [[TMP12:%.*]] = or i1 [[TMP8]], [[TMP11]]
 ; PRED-NEXT:    br i1 [[TMP12]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP0]], 4
 ; PRED-NEXT:    [[TMP14:%.*]] = icmp ugt i64 [[TMP0]], 4
 ; PRED-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[TMP13]], i64 0
@@ -690,9 +673,9 @@ define void @ivs_trunc_and_ext(i32 %x, ptr %dst, i64 %N) #0 {
 ; PRED:       [[PRED_STORE_CONTINUE6]]:
 ; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX]], i64 [[TMP15]])
-; PRED-NEXT:    [[TMP34:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; PRED-NEXT:    [[TMP34:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP35:%.*]] = xor i1 [[TMP34]], true
 ; PRED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
-; PRED-NEXT:    [[TMP35:%.*]] = extractelement <4 x i1> [[TMP34]], i32 0
 ; PRED-NEXT:    br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
 ; PRED-NEXT:    br label %[[EXIT:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
index aa2ec2de14c29..fed57c919c6c8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs.ll
@@ -171,14 +171,11 @@ define i64 @int_and_pointer_iv(ptr %start, i32 %N) {
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i64> [[TMP5]], i32 2
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[START]], [[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[RECUR_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], [[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[RECUR_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[PTR_IV]], align 4
 ; CHECK-NEXT:    [[RECUR_NEXT]] = zext i32 [[L]] to i64
 ; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
index 649be65e8e671..c23695dc5dbe3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
@@ -11,7 +11,7 @@ define i64 @vector_loop_with_remaining_iterations(ptr %src, ptr noalias %dst, i3
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ITER_CHECK:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 17, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
@@ -43,7 +43,7 @@ define i64 @vector_loop_with_remaining_iterations(ptr %src, ptr noalias %dst, i3
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 1
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i64 1, [[TMP15]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; CHECK:       [[VEC_EPILOG_PH]]:
@@ -55,8 +55,6 @@ define i64 @vector_loop_with_remaining_iterations(ptr %src, ptr noalias %dst, i3
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 17, [[TMP19]]
-; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <vscale x 2 x i64> zeroinitializer, i64 [[BC_MERGE_RDX]], i32 0
@@ -67,8 +65,7 @@ define i64 @vector_loop_with_remaining_iterations(ptr %src, ptr noalias %dst, i3
 ; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP26:%.*]] = mul <vscale x 2 x i64> [[TMP25]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT4]], [[TMP26]]
-; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 1, [[TMP21]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP27]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP17]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
@@ -84,7 +81,7 @@ define i64 @vector_loop_with_remaining_iterations(ptr %src, ptr noalias %dst, i3
 ; CHECK-NEXT:    store <vscale x 2 x i8> zeroinitializer, ptr [[TMP32]], align 1
 ; CHECK-NEXT:    [[TMP34:%.*]] = zext <vscale x 2 x i32> [[TMP31]] to <vscale x 2 x i64>
 ; CHECK-NEXT:    [[TMP35]] = or <vscale x 2 x i64> [[VEC_PHI8]], [[TMP34]]
-; CHECK-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], [[TMP21]]
+; CHECK-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], [[TMP17]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -149,7 +146,7 @@ define i64 @main_vector_loop_fixed_with_no_remaining_iterations(ptr %src, ptr no
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ITER_CHECK:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 17, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
@@ -181,7 +178,7 @@ define i64 @main_vector_loop_fixed_with_no_remaining_iterations(ptr %src, ptr no
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 1
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i64 1, [[TMP15]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; CHECK:       [[VEC_EPILOG_PH]]:
@@ -193,8 +190,6 @@ define i64 @main_vector_loop_fixed_with_no_remaining_iterations(ptr %src, ptr no
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i64 [[TMP17]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 17, [[TMP19]]
-; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <vscale x 2 x i64> zeroinitializer, i64 [[BC_MERGE_RDX]], i32 0
@@ -205,8 +200,7 @@ define i64 @main_vector_loop_fixed_with_no_remaining_iterations(ptr %src, ptr no
 ; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP38:%.*]] = mul <vscale x 2 x i64> [[TMP25]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> [[BROADCAST_SPLAT4]], [[TMP38]]
-; CHECK-NEXT:    [[TMP39:%.*]] = mul i64 1, [[TMP21]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP39]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP17]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
@@ -222,7 +216,7 @@ define i64 @main_vector_loop_fixed_with_no_remaining_iterations(ptr %src, ptr no
 ; CHECK-NEXT:    store <vscale x 2 x i8> zeroinitializer, ptr [[TMP32]], align 1
 ; CHECK-NEXT:    [[TMP34:%.*]] = zext <vscale x 2 x i32> [[TMP31]] to <vscale x 2 x i64>
 ; CHECK-NEXT:    [[TMP35]] = or <vscale x 2 x i64> [[VEC_PHI8]], [[TMP34]]
-; CHECK-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], [[TMP21]]
+; CHECK-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], [[TMP17]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT6]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
new file mode 100644
index 0000000000000..67826e5cb9f96
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
+; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+
+target triple = "arm64-apple-macosx15.0.0"
+
+define void @interleave_groups_separated_by_offset(ptr %A, i64 %offset) {
+; CHECK-LABEL: define void @interleave_groups_separated_by_offset(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[OFFSET:%.*]]) {
+; CHECK-NEXT:  [[ITER_CHECK:.*:]]
+; CHECK-NEXT:    [[A_OFFSET:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET]]
+; CHECK-NEXT:    br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 1999
+; CHECK-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[A]], i64 1
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[A]], i64 2000
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET]], 1999
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[A]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    [[BOUND04:%.*]] = icmp ult ptr [[A]], [[SCEVGEP3]]
+; CHECK-NEXT:    [[BOUND15:%.*]] = icmp ult ptr [[A_OFFSET]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
+; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
+; CHECK-NEXT:    [[BOUND07:%.*]] = icmp ult ptr [[SCEVGEP1]], [[SCEVGEP3]]
+; CHECK-NEXT:    [[BOUND18:%.*]] = icmp ult ptr [[A_OFFSET]], [[SCEVGEP2]]
+; CHECK-NEXT:    [[FOUND_CONFLICT9:%.*]] = and i1 [[BOUND07]], [[BOUND18]]
+; CHECK-NEXT:    [[CONFLICT_RDX10:%.*]] = or i1 [[CONFLICT_RDX]], [[FOUND_CONFLICT9]]
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX10]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-NEXT:    br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 32
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    store <32 x i8> zeroinitializer, ptr [[NEXT_GEP]], align 1
+; CHECK-NEXT:    store <32 x i8> zeroinitializer, ptr [[NEXT_GEP11]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 992
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[A]], i64 1984
+; CHECK-NEXT:    [[IND_END17:%.*]] = getelementptr i8, ptr [[A_OFFSET]], i64 1984
+; CHECK-NEXT:    br i1 true, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
+; CHECK:       [[VEC_EPILOG_PH]]:
+; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 992, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[A]], i64 1984
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[A_OFFSET]], i64 1984
+; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX12:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT14:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX12]], 2
+; CHECK-NEXT:    [[NEXT_GEP13:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    store <16 x i8> zeroinitializer, ptr [[NEXT_GEP13]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT14]] = add nuw i64 [[INDEX12]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT14]], 992
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
+;
+entry:
+  %A.offset = getelementptr i8, ptr %A, i64 %offset
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %ptr.iv = phi ptr [ %A, %entry ], [ %ptr.iv.next, %loop ]
+  %ptr.iv.offset = phi ptr [ %A.offset, %entry ], [ %ptr.iv.offset.next, %loop ]
+  %ptr.iv.1 = getelementptr i8, ptr %ptr.iv, i64 1
+  store i8 0, ptr %ptr.iv, align 1
+  %ptr.iv.offset.next = getelementptr i8, ptr %ptr.iv.offset, i64 2
+  %1 = load i8, ptr %ptr.iv.offset, align 1
+  %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 2
+  store i8 0, ptr %ptr.iv.1, align 1
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 1000
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
index f0693475c5958..7ff4609f8ec4b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
@@ -137,7 +137,7 @@ define i32 @interleave_integer_reduction(ptr %src, i64 %N) {
 ; INTERLEAVE-4-VLA-LABEL: @interleave_integer_reduction(
 ; INTERLEAVE-4-VLA-NEXT:  entry:
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-4-VLA-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; INTERLEAVE-4-VLA-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; INTERLEAVE-4-VLA-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; INTERLEAVE-4-VLA-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; INTERLEAVE-4-VLA:       vector.ph:
@@ -145,8 +145,6 @@ define i32 @interleave_integer_reduction(ptr %src, i64 %N) {
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; INTERLEAVE-4-VLA-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; INTERLEAVE-4-VLA-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; INTERLEAVE-4-VLA-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-4-VLA-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; INTERLEAVE-4-VLA-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; INTERLEAVE-4-VLA:       vector.body:
 ; INTERLEAVE-4-VLA-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -156,10 +154,10 @@ define i32 @interleave_integer_reduction(ptr %src, i64 %N) {
 ; INTERLEAVE-4-VLA-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[INDEX]]
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-4-VLA-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
+; INTERLEAVE-4-VLA-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 2
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 [[TMP8]]
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-4-VLA-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
+; INTERLEAVE-4-VLA-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 3
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 [[TMP11]]
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 12
@@ -172,7 +170,7 @@ define i32 @interleave_integer_reduction(ptr %src, i64 %N) {
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP17]] = add <vscale x 4 x i32> [[VEC_PHI1]], [[WIDE_LOAD4]]
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP18]] = add <vscale x 4 x i32> [[VEC_PHI2]], [[WIDE_LOAD5]]
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP19]] = add <vscale x 4 x i32> [[VEC_PHI3]], [[WIDE_LOAD6]]
-; INTERLEAVE-4-VLA-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; INTERLEAVE-4-VLA-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; INTERLEAVE-4-VLA-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; INTERLEAVE-4-VLA-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; INTERLEAVE-4-VLA:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
index d45dbcc8b1663..0327334e220f2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
@@ -8,7 +8,7 @@ target triple = "arm64-apple-macosx14.0.0"
 define i32 @test_invariant_replicate_region(i32 %x, i1 %c) {
 ; CHECK-LABEL: define i32 @test_invariant_replicate_region(
 ; CHECK-SAME: i32 [[X:%.*]], i1 [[C:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
@@ -52,10 +52,9 @@ define i32 @test_invariant_replicate_region(i32 %x, i1 %c) {
 ; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
 ; CHECK:       [[THEN]]:
 ; CHECK-NEXT:    [[REM_1:%.*]] = urem i32 10, [[X]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
index 5f5d326cb4bba..1f486fba069c9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
@@ -49,7 +49,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-VS1-NEXT:    [[TMP3:%.*]] = sub i64 20, [[TMP2]]
 ; CHECK-VS1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
+; CHECK-VS1-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 3
 ; CHECK-VS1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
 ; CHECK-VS1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; CHECK-VS1:       [[VECTOR_SCEVCHECK]]:
@@ -64,7 +64,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    br i1 [[TMP13]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK-VS1:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
 ; CHECK-VS1-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 16
+; CHECK-VS1-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 4
 ; CHECK-VS1-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], [[TMP15]]
 ; CHECK-VS1-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VS1:       [[VECTOR_PH]]:
@@ -72,8 +72,6 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 16
 ; CHECK-VS1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP17]]
 ; CHECK-VS1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; CHECK-VS1-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 16
 ; CHECK-VS1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-VS1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
 ; CHECK-VS1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -84,7 +82,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP22]], align 1
 ; CHECK-VS1-NEXT:    [[TMP24:%.*]] = add <vscale x 16 x i8> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; CHECK-VS1-NEXT:    store <vscale x 16 x i8> [[TMP24]], ptr [[TMP22]], align 1
-; CHECK-VS1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; CHECK-VS1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]]
 ; CHECK-VS1-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VS1-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-VS1:       [[MIDDLE_BLOCK]]:
@@ -94,7 +92,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    [[IND_END4:%.*]] = add i64 [[TMP0]], [[N_VEC]]
 ; CHECK-VS1-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
 ; CHECK-VS1-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 8
+; CHECK-VS1-NEXT:    [[TMP27:%.*]] = shl nuw i64 [[TMP26]], 3
 ; CHECK-VS1-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP27]]
 ; CHECK-VS1-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; CHECK-VS1:       [[VEC_EPILOG_PH]]:
@@ -103,8 +101,6 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 8
 ; CHECK-VS1-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP3]], [[TMP29]]
 ; CHECK-VS1-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF2]]
-; CHECK-VS1-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 8
 ; CHECK-VS1-NEXT:    [[TMP39:%.*]] = add i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-VS1-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-VS1-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
@@ -116,7 +112,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x i8>, ptr [[TMP33]], align 1
 ; CHECK-VS1-NEXT:    [[TMP35:%.*]] = add <vscale x 8 x i8> [[WIDE_LOAD6]], [[BROADCAST_SPLAT8]]
 ; CHECK-VS1-NEXT:    store <vscale x 8 x i8> [[TMP35]], ptr [[TMP33]], align 1
-; CHECK-VS1-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX5]], [[TMP31]]
+; CHECK-VS1-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX5]], [[TMP29]]
 ; CHECK-VS1-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC3]]
 ; CHECK-VS1-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK-VS1:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
@@ -153,7 +149,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-VS2-NEXT:    [[TMP3:%.*]] = sub i64 20, [[TMP2]]
 ; CHECK-VS2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; CHECK-VS2-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-VS2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
 ; CHECK-VS2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; CHECK-VS2:       [[VECTOR_SCEVCHECK]]:
@@ -168,7 +164,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    br i1 [[TMP13]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK-VS2:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
 ; CHECK-VS2-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 8
+; CHECK-VS2-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 3
 ; CHECK-VS2-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], [[TMP15]]
 ; CHECK-VS2-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VS2:       [[VECTOR_PH]]:
@@ -176,8 +172,6 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 8
 ; CHECK-VS2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP17]]
 ; CHECK-VS2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; CHECK-VS2-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 8
 ; CHECK-VS2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-VS2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-VS2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -188,7 +182,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP22]], align 1
 ; CHECK-VS2-NEXT:    [[TMP24:%.*]] = add <vscale x 8 x i8> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
 ; CHECK-VS2-NEXT:    store <vscale x 8 x i8> [[TMP24]], ptr [[TMP22]], align 1
-; CHECK-VS2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; CHECK-VS2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]]
 ; CHECK-VS2-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VS2-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-VS2:       [[MIDDLE_BLOCK]]:
@@ -198,7 +192,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    [[IND_END4:%.*]] = add i64 [[TMP0]], [[N_VEC]]
 ; CHECK-VS2-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
 ; CHECK-VS2-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 4
+; CHECK-VS2-NEXT:    [[TMP27:%.*]] = shl nuw i64 [[TMP26]], 2
 ; CHECK-VS2-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP27]]
 ; CHECK-VS2-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; CHECK-VS2:       [[VEC_EPILOG_PH]]:
@@ -207,8 +201,6 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 4
 ; CHECK-VS2-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP3]], [[TMP29]]
 ; CHECK-VS2-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF2]]
-; CHECK-VS2-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 4
 ; CHECK-VS2-NEXT:    [[TMP39:%.*]] = add i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-VS2-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-VS2-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
@@ -220,7 +212,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 4 x i8>, ptr [[TMP33]], align 1
 ; CHECK-VS2-NEXT:    [[TMP35:%.*]] = add <vscale x 4 x i8> [[WIDE_LOAD6]], [[BROADCAST_SPLAT8]]
 ; CHECK-VS2-NEXT:    store <vscale x 4 x i8> [[TMP35]], ptr [[TMP33]], align 1
-; CHECK-VS2-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX5]], [[TMP31]]
+; CHECK-VS2-NEXT:    [[INDEX_NEXT9]] = add nuw i64 [[INDEX5]], [[TMP29]]
 ; CHECK-VS2-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC3]]
 ; CHECK-VS2-NEXT:    br i1 [[TMP36]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK-VS2:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
@@ -416,12 +408,6 @@ define void @overflow_indvar_known_false(ptr nocapture noundef %p, i32 noundef %
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP1]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
@@ -434,11 +420,11 @@ define void @overflow_indvar_known_false(ptr nocapture noundef %p, i32 noundef %
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr [[TMP13]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
 ; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP15]], ptr [[TMP13]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[TMP1]])
-; CHECK-NEXT:    [[TMP16:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[TMP16]], i32 0
-; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP31:%.*]] = xor i1 [[TMP30]], true
+; CHECK-NEXT:    br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[WHILE_END_LOOPEXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -488,7 +474,7 @@ while.end:
 define i32 @tc4(ptr noundef readonly captures(none) %tmp) vscale_range(1,16) {
 ; CHECK-LABEL: define i32 @tc4(
 ; CHECK-SAME: ptr noundef readonly captures(none) [[TMP:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -504,12 +490,10 @@ define i32 @tc4(ptr noundef readonly captures(none) %tmp) vscale_range(1,16) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_0179:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_0179:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[TMP]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ADD]] = add i32 [[SUM_0179]], [[TMP5]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
index 078f98f54525b..aed1c3d9fcc4c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
@@ -87,8 +87,8 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
 ; TFCOMMON:       pred.store.continue2:
 ; TFCOMMON-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
 ; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP3]])
-; TFCOMMON-NEXT:    [[TMP18:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFCOMMON-NEXT:    [[TMP17:%.*]] = extractelement <2 x i1> [[TMP18]], i32 0
+; TFCOMMON-NEXT:    [[TMP15:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFCOMMON-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP15]], true
 ; TFCOMMON-NEXT:    br i1 [[TMP17]], label [[END:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
 ; TFCOMMON:       end:
 ; TFCOMMON-NEXT:    ret void
@@ -153,9 +153,9 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX]], 2
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[INDEX]], i64 [[TMP3]])
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT10]] = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i64(i64 [[TMP27]], i64 [[TMP3]])
-; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = xor <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFA_INTERLEAVE-NEXT:    [[TMP30:%.*]] = extractelement <2 x i1> [[TMP28]], i32 0
-; TFA_INTERLEAVE-NEXT:    br i1 [[TMP30]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; TFA_INTERLEAVE-NEXT:    [[TMP26:%.*]] = extractelement <2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = xor i1 [[TMP26]], true
+; TFA_INTERLEAVE-NEXT:    br i1 [[TMP28]], label [[END:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; TFA_INTERLEAVE:       end:
 ; TFA_INTERLEAVE-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 6029095bbe7b1..e5697e121e9e9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -18,8 +18,6 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -28,7 +26,7 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i1> splat (i1 true))
 ; TFNONE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFNONE-NEXT:    store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; TFNONE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TFNONE-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; TFNONE:       [[MIDDLE_BLOCK]]:
@@ -66,8 +64,8 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFCOMMON-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFCOMMON-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFCOMMON-NEXT:    [[TMP8:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFCOMMON-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP8]], i32 0
+; TFCOMMON-NEXT:    [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFCOMMON-NEXT:    [[TMP9:%.*]] = xor i1 [[TMP8]], true
 ; TFCOMMON-NEXT:    br i1 [[TMP9]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; TFCOMMON:       [[FOR_COND_CLEANUP]]:
 ; TFCOMMON-NEXT:    ret void
@@ -78,7 +76,7 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -88,7 +86,7 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
@@ -96,18 +94,18 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP12:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP15]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP11]], ptr [[TMP13]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr [[TMP16]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP18]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025)
-; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFA_INTERLEAVE-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 2 x i1> [[TMP20]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP21:%.*]] = xor i1 [[TMP20]], true
 ; TFA_INTERLEAVE-NEXT:    br i1 [[TMP21]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; TFA_INTERLEAVE:       [[FOR_COND_CLEANUP]]:
 ; TFA_INTERLEAVE-NEXT:    ret void
@@ -141,8 +139,6 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -153,7 +149,7 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i64> [[TMP8]], <vscale x 2 x i64> zeroinitializer
 ; TFNONE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
 ; TFNONE-NEXT:    store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP10]], align 8
-; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; TFNONE-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TFNONE-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; TFNONE:       [[MIDDLE_BLOCK]]:
@@ -168,10 +164,10 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP12]], 50
 ; TFNONE-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END]]
 ; TFNONE:       [[IF_THEN]]:
-; TFNONE-NEXT:    [[TMP13:%.*]] = call i64 @foo(i64 [[TMP12]]) #[[ATTR4]]
+; TFNONE-NEXT:    [[TMP9:%.*]] = call i64 @foo(i64 [[TMP12]]) #[[ATTR4]]
 ; TFNONE-NEXT:    br label %[[IF_END]]
 ; TFNONE:       [[IF_END]]:
-; TFNONE-NEXT:    [[TMP14:%.*]] = phi i64 [ [[TMP13]], %[[IF_THEN]] ], [ 0, %[[FOR_BODY]] ]
+; TFNONE-NEXT:    [[TMP14:%.*]] = phi i64 [ [[TMP9]], %[[IF_THEN]] ], [ 0, %[[FOR_BODY]] ]
 ; TFNONE-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; TFNONE-NEXT:    store i64 [[TMP14]], ptr [[ARRAYIDX1]], align 8
 ; TFNONE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -200,8 +196,8 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFCOMMON-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP9]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFCOMMON-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFCOMMON-NEXT:    [[TMP10:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFCOMMON-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 2 x i1> [[TMP10]], i32 0
+; TFCOMMON-NEXT:    [[TMP10:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFCOMMON-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
 ; TFCOMMON-NEXT:    br i1 [[TMP11]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; TFCOMMON:       [[FOR_COND_CLEANUP]]:
 ; TFCOMMON-NEXT:    ret void
@@ -212,7 +208,7 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -222,7 +218,7 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
@@ -236,18 +232,18 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[PREDPHI4:%.*]] = select <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> zeroinitializer
 ; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i64 [[TMP19]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP17]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr [[TMP20]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP23:%.*]] = add i64 [[INDEX_NEXT]], [[TMP22]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP23]], i64 1025)
-; TFA_INTERLEAVE-NEXT:    [[TMP24:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = extractelement <vscale x 2 x i1> [[TMP24]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP24:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = xor i1 [[TMP24]], true
 ; TFA_INTERLEAVE-NEXT:    br i1 [[TMP25]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; TFA_INTERLEAVE:       [[FOR_COND_CLEANUP]]:
 ; TFA_INTERLEAVE-NEXT:    ret void
@@ -292,8 +288,6 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -306,7 +300,7 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP9]]
 ; TFNONE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
 ; TFNONE-NEXT:    store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP11]], align 8
-; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; TFNONE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TFNONE-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; TFNONE:       [[MIDDLE_BLOCK]]:
@@ -359,8 +353,8 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFCOMMON-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP12]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFCOMMON-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFCOMMON-NEXT:    [[TMP13:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFCOMMON-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 2 x i1> [[TMP13]], i32 0
+; TFCOMMON-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFCOMMON-NEXT:    [[TMP14:%.*]] = xor i1 [[TMP13]], true
 ; TFCOMMON-NEXT:    br i1 [[TMP14]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; TFCOMMON:       [[FOR_COND_CLEANUP]]:
 ; TFCOMMON-NEXT:    ret void
@@ -371,7 +365,7 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -381,7 +375,7 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
@@ -401,18 +395,18 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[PREDPHI4:%.*]] = select <vscale x 2 x i1> [[TMP20]], <vscale x 2 x i64> [[TMP22]], <vscale x 2 x i64> [[TMP18]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[TMP23]], i64 [[TMP25]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP23]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr [[TMP26]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = shl nuw i64 [[TMP27]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP29:%.*]] = add i64 [[INDEX_NEXT]], [[TMP28]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP29]], i64 1025)
-; TFA_INTERLEAVE-NEXT:    [[TMP30:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFA_INTERLEAVE-NEXT:    [[TMP31:%.*]] = extractelement <vscale x 2 x i1> [[TMP30]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP30:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP31:%.*]] = xor i1 [[TMP30]], true
 ; TFA_INTERLEAVE-NEXT:    br i1 [[TMP31]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; TFA_INTERLEAVE:       [[FOR_COND_CLEANUP]]:
 ; TFA_INTERLEAVE-NEXT:    ret void
@@ -460,8 +454,6 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -470,7 +462,7 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64> [[WIDE_LOAD]])
 ; TFNONE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFNONE-NEXT:    store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; TFNONE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TFNONE-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; TFNONE:       [[MIDDLE_BLOCK]]:
@@ -515,8 +507,6 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFFALLBACK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFFALLBACK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFFALLBACK-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; TFFALLBACK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFFALLBACK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFFALLBACK:       [[VECTOR_BODY]]:
 ; TFFALLBACK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -525,7 +515,7 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFFALLBACK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64> [[WIDE_LOAD]])
 ; TFFALLBACK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFFALLBACK-NEXT:    store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; TFFALLBACK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TFFALLBACK-NEXT:    br i1 [[TMP9]], label %[[FOR_BODY:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; TFFALLBACK:       [[FOR_BODY]]:
@@ -589,8 +579,6 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -599,7 +587,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @foo_vector_nomask(<vscale x 2 x i64> [[WIDE_LOAD]])
 ; TFNONE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFNONE-NEXT:    store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; TFNONE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TFNONE-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; TFNONE:       [[MIDDLE_BLOCK]]:
@@ -637,8 +625,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFALWAYS-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFALWAYS-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFALWAYS-NEXT:    [[TMP8:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFALWAYS-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP8]], i32 0
+; TFALWAYS-NEXT:    [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFALWAYS-NEXT:    [[TMP9:%.*]] = xor i1 [[TMP8]], true
 ; TFALWAYS-NEXT:    br i1 [[TMP9]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; TFALWAYS:       [[FOR_COND_CLEANUP]]:
 ; TFALWAYS-NEXT:    ret void
@@ -660,8 +648,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFFALLBACK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP6]], ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFFALLBACK-NEXT:    [[TMP8:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFFALLBACK-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[TMP8]], i32 0
+; TFFALLBACK-NEXT:    [[TMP8:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFFALLBACK-NEXT:    [[TMP9:%.*]] = xor i1 [[TMP8]], true
 ; TFFALLBACK-NEXT:    br i1 [[TMP9]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; TFFALLBACK:       [[FOR_COND_CLEANUP]]:
 ; TFFALLBACK-NEXT:    ret void
@@ -672,7 +660,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -682,7 +670,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
@@ -690,18 +678,18 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP12:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP15]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP11]], ptr [[TMP13]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr [[TMP16]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP18]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025)
-; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFA_INTERLEAVE-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 2 x i1> [[TMP20]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP21:%.*]] = xor i1 [[TMP20]], true
 ; TFA_INTERLEAVE-NEXT:    br i1 [[TMP21]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; TFA_INTERLEAVE:       [[FOR_COND_CLEANUP]]:
 ; TFA_INTERLEAVE-NEXT:    ret void
@@ -737,8 +725,6 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
 ; TFNONE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -753,7 +739,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFNONE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFNONE-NEXT:    store <vscale x 2 x i64> [[TMP9]], ptr [[TMP10]], align 8
 ; TFNONE-NEXT:    [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP7]])
-; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; TFNONE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TFNONE-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; TFNONE:       [[MIDDLE_BLOCK]]:
@@ -803,8 +789,8 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFALWAYS-NEXT:    [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP10]])
 ; TFALWAYS-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFALWAYS-NEXT:    [[TMP12:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFALWAYS-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP12]], i32 0
+; TFALWAYS-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFALWAYS-NEXT:    [[TMP13:%.*]] = xor i1 [[TMP12]], true
 ; TFALWAYS-NEXT:    br i1 [[TMP13]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; TFALWAYS:       [[FOR_COND_CLEANUP]]:
 ; TFALWAYS-NEXT:    ret double [[TMP11]]
@@ -833,8 +819,8 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFFALLBACK-NEXT:    [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP10]])
 ; TFFALLBACK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; TFFALLBACK-NEXT:    [[TMP12:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFFALLBACK-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP12]], i32 0
+; TFFALLBACK-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFFALLBACK-NEXT:    [[TMP13:%.*]] = xor i1 [[TMP12]], true
 ; TFFALLBACK-NEXT:    br i1 [[TMP13]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; TFFALLBACK:       [[FOR_COND_CLEANUP]]:
 ; TFFALLBACK-NEXT:    ret double [[TMP11]]
@@ -845,7 +831,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
@@ -858,7 +844,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-NEXT:    [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr double, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
@@ -870,7 +856,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-NEXT:    [[TMP16:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i64 [[TMP19]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr [[TMP17]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP16]], ptr [[TMP20]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
@@ -880,12 +866,12 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-NEXT:    [[TMP24]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP22]], <vscale x 2 x double> [[TMP23]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP26:%.*]] = shl nuw i64 [[TMP25]], 1
 ; TFA_INTERLEAVE-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX_NEXT]], [[TMP26]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP27]], i64 1025)
-; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; TFA_INTERLEAVE-NEXT:    [[TMP29:%.*]] = extractelement <vscale x 2 x i1> [[TMP28]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; TFA_INTERLEAVE-NEXT:    [[TMP29:%.*]] = xor i1 [[TMP28]], true
 ; TFA_INTERLEAVE-NEXT:    br i1 [[TMP29]], label %[[FOR_COND_CLEANUP:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; TFA_INTERLEAVE:       [[FOR_COND_CLEANUP]]:
 ; TFA_INTERLEAVE-NEXT:    ret double [[TMP24]]
@@ -917,32 +903,23 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
 ; TFNONE-SAME: ptr noalias [[P2:%.*]], ptr noalias [[P:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
 ; TFNONE-NEXT:  [[ENTRY:.*]]:
 ; TFNONE-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
-; TFNONE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; TFNONE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
+; TFNONE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
 ; TFNONE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TFNONE:       [[VECTOR_PH]]:
-; TFNONE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
-; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
+; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; TFNONE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; TFNONE-NEXT:    [[TMP7:%.*]] = load double, ptr [[P2]], align 8
-; TFNONE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[TMP7]], i64 0
-; TFNONE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
-; TFNONE-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x double> @exp_masked_scalable(<vscale x 2 x double> [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true))
-; TFNONE-NEXT:    [[TMP9:%.*]] = fcmp ogt <vscale x 2 x double> [[TMP8]], zeroinitializer
-; TFNONE-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP9]], <vscale x 2 x double> zeroinitializer, <vscale x 2 x double> splat (double 1.000000e+00)
-; TFNONE-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; TFNONE-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP11]], 2
-; TFNONE-NEXT:    [[TMP13:%.*]] = sub i32 [[TMP12]], 1
-; TFNONE-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 2 x double> [[PREDPHI]], i32 [[TMP13]]
+; TFNONE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 0
+; TFNONE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; TFNONE-NEXT:    [[TMP2:%.*]] = call <2 x double> @exp_fixed(<2 x double> [[BROADCAST_SPLAT]])
+; TFNONE-NEXT:    [[TMP3:%.*]] = fcmp ogt <2 x double> [[TMP2]], zeroinitializer
+; TFNONE-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x double> zeroinitializer, <2 x double> splat (double 1.000000e+00)
+; TFNONE-NEXT:    [[TMP14:%.*]] = extractelement <2 x double> [[PREDPHI]], i32 1
 ; TFNONE-NEXT:    store double [[TMP14]], ptr [[P]], align 8
-; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; TFNONE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; TFNONE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TFNONE-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; TFNONE:       [[MIDDLE_BLOCK]]:
@@ -996,18 +973,16 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[TMP0]]
 ; TFA_INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFA_INTERLEAVE:       [[VECTOR_BODY]]:
-; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[TMP19:.*]] ]
-; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[TMP19]] ]
-; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT6:%.*]], %[[TMP19]] ]
+; TFA_INTERLEAVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[TMP18:.*]] ]
+; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[TMP18]] ]
+; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi i1 [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT6:%.*]], %[[TMP18]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = load double, ptr [[P2]], align 8
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7:[0-9]+]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = tail call double @llvm.exp.f64(double [[TMP4]]) #[[ATTR7]]
-; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = fcmp ogt double [[TMP5]], 0.000000e+00
-; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = fcmp ogt double [[TMP6]], 0.000000e+00
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = xor i1 [[TMP7]], true
-; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = xor i1 [[TMP8]], true
-; TFA_INTERLEAVE-NEXT:    [[TMP11:%.*]] = select i1 [[ACTIVE_LANE_MASK]], i1 [[TMP9]], i1 false
-; TFA_INTERLEAVE-NEXT:    [[TMP12:%.*]] = select i1 [[ACTIVE_LANE_MASK2]], i1 [[TMP10]], i1 false
+; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = fcmp ule double [[TMP5]], 0.000000e+00
+; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = fcmp ule double [[TMP6]], 0.000000e+00
+; TFA_INTERLEAVE-NEXT:    [[TMP11:%.*]] = select i1 [[ACTIVE_LANE_MASK]], i1 [[TMP7]], i1 false
+; TFA_INTERLEAVE-NEXT:    [[TMP12:%.*]] = select i1 [[ACTIVE_LANE_MASK2]], i1 [[TMP8]], i1 false
 ; TFA_INTERLEAVE-NEXT:    [[PREDPHI:%.*]] = select i1 [[TMP11]], double 1.000000e+00, double 0.000000e+00
 ; TFA_INTERLEAVE-NEXT:    [[PREDPHI3:%.*]] = select i1 [[TMP12]], double 1.000000e+00, double 0.000000e+00
 ; TFA_INTERLEAVE-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[ACTIVE_LANE_MASK2]], double [[PREDPHI3]], double [[PREDPHI]]
@@ -1016,11 +991,11 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = xor i1 [[TMP13]], true
 ; TFA_INTERLEAVE-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP14]], true
 ; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = or i1 [[TMP15]], [[TMP16]]
-; TFA_INTERLEAVE-NEXT:    br i1 [[TMP17]], label %[[BB18:.*]], label %[[TMP19]]
-; TFA_INTERLEAVE:       [[BB18]]:
+; TFA_INTERLEAVE-NEXT:    br i1 [[TMP17]], label %[[BB16:.*]], label %[[TMP18]]
+; TFA_INTERLEAVE:       [[BB16]]:
 ; TFA_INTERLEAVE-NEXT:    store double [[SPEC_SELECT]], ptr [[P]], align 8
-; TFA_INTERLEAVE-NEXT:    br label %[[TMP19]]
-; TFA_INTERLEAVE:       [[TMP19]]:
+; TFA_INTERLEAVE-NEXT:    br label %[[TMP18]]
+; TFA_INTERLEAVE:       [[TMP18]]:
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = add i64 [[INDEX]], 1
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = icmp ult i64 [[INDEX]], [[TMP3]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll b/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
index 1159a641f5ce5..2ca117c33dbb2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/mul-simplification.ll
@@ -45,7 +45,7 @@ exit:
 
 define i32 @add_reduction_select_operand_constant_but_non_uniform() {
 ; CHECK-LABEL: define i32 @add_reduction_select_operand_constant_but_non_uniform() {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -63,12 +63,10 @@ define i32 @add_reduction_select_operand_constant_but_non_uniform() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 42, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD2_REASS:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[ADD2_REASS:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ 42, %[[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[ADD2_REASS]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[RDX_NEXT]] = add i32 0, [[RDX]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD2_REASS]], 64
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
index 1471896f99329..67e6902b5d32a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
@@ -17,7 +17,7 @@ target triple = "aarch64-unknown-linux-gnu"
 define void @always_vectorize(ptr %p, i32 %x) {
 ; DEFAULT-LABEL: define void @always_vectorize(
 ; DEFAULT-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
@@ -31,10 +31,9 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; DEFAULT:       [[FOR_BODY]]:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[INDVARS_IV]]
 ; DEFAULT-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; DEFAULT-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[X]]
@@ -47,7 +46,7 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ;
 ; OPTSIZE-LABEL: define void @always_vectorize(
 ; OPTSIZE-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
-; OPTSIZE-NEXT:  [[ENTRY:.*]]:
+; OPTSIZE-NEXT:  [[ENTRY:.*:]]
 ; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; OPTSIZE:       [[VECTOR_PH]]:
 ; OPTSIZE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
@@ -61,10 +60,9 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; OPTSIZE:       [[SCALAR_PH]]:
-; OPTSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; OPTSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; OPTSIZE:       [[FOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; OPTSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[INDVARS_IV]]
 ; OPTSIZE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; OPTSIZE-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[X]]
@@ -77,7 +75,7 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ;
 ; MINSIZE-LABEL: define void @always_vectorize(
 ; MINSIZE-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
-; MINSIZE-NEXT:  [[ENTRY:.*]]:
+; MINSIZE-NEXT:  [[ENTRY:.*:]]
 ; MINSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; MINSIZE:       [[VECTOR_PH]]:
 ; MINSIZE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
@@ -91,10 +89,9 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ; MINSIZE:       [[MIDDLE_BLOCK]]:
 ; MINSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; MINSIZE:       [[SCALAR_PH]]:
-; MINSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; MINSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; MINSIZE:       [[FOR_BODY]]:
-; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; MINSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[INDVARS_IV]]
 ; MINSIZE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; MINSIZE-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[X]]
@@ -221,7 +218,7 @@ for.cond.cleanup:
 define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n) {
 ; DEFAULT-LABEL: define void @tail_predicate_without_optsize(
 ; DEFAULT-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[A]], i64 0
@@ -394,10 +391,9 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; DEFAULT:       [[FOR_BODY]]:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; DEFAULT-NEXT:    [[TMP72:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i8
 ; DEFAULT-NEXT:    [[MUL:%.*]] = mul i8 [[A]], [[TMP72]]
 ; DEFAULT-NEXT:    [[SHR:%.*]] = lshr i8 [[TMP72]], 1
@@ -490,19 +486,13 @@ for.cond.cleanup:
 define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n) #0 {
 ; DEFAULT-LABEL: define void @sve_tail_predicate_without_minsize(
 ; DEFAULT-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; DEFAULT-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; DEFAULT-NEXT:    [[N_RND_UP:%.*]] = add i64 15, [[TMP2]]
-; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; DEFAULT-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; DEFAULT-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
+; DEFAULT-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
 ; DEFAULT-NEXT:    [[TMP7:%.*]] = sub i64 15, [[TMP6]]
 ; DEFAULT-NEXT:    [[TMP8:%.*]] = icmp ugt i64 15, [[TMP6]]
 ; DEFAULT-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -516,9 +506,8 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; DEFAULT-NEXT:    [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
 ; DEFAULT-NEXT:    [[TMP11:%.*]] = mul <vscale x 16 x i8> [[TMP10]], splat (i8 1)
 ; DEFAULT-NEXT:    [[INDUCTION:%.*]] = add <vscale x 16 x i8> zeroinitializer, [[TMP11]]
-; DEFAULT-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP4]] to i8
-; DEFAULT-NEXT:    [[TMP13:%.*]] = mul i8 1, [[TMP12]]
-; DEFAULT-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP13]], i64 0
+; DEFAULT-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP1]] to i8
+; DEFAULT-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP12]], i64 0
 ; DEFAULT-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[DOTSPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
 ; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; DEFAULT:       [[VECTOR_BODY]]:
@@ -534,19 +523,18 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; DEFAULT-NEXT:    [[TMP21:%.*]] = add <vscale x 16 x i8> [[TMP18]], [[TMP20]]
 ; DEFAULT-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
 ; DEFAULT-NEXT:    call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP21]], ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; DEFAULT-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; DEFAULT-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; DEFAULT-NEXT:    [[TMP24:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; DEFAULT-NEXT:    [[TMP24:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; DEFAULT-NEXT:    [[TMP23:%.*]] = xor i1 [[TMP24]], true
 ; DEFAULT-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[DOTSPLAT]]
-; DEFAULT-NEXT:    [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[TMP24]], i32 0
-; DEFAULT-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; DEFAULT-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; DEFAULT:       [[FOR_BODY]]:
-; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; DEFAULT-NEXT:    [[TMP26:%.*]] = trunc nuw nsw i64 [[IV]] to i8
 ; DEFAULT-NEXT:    [[MUL:%.*]] = mul i8 [[A]], [[TMP26]]
 ; DEFAULT-NEXT:    [[SHR:%.*]] = lshr i8 [[TMP26]], 1
@@ -565,19 +553,13 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ;
 ; OPTSIZE-LABEL: define void @sve_tail_predicate_without_minsize(
 ; OPTSIZE-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; OPTSIZE-NEXT:  [[ENTRY:.*]]:
+; OPTSIZE-NEXT:  [[ENTRY:.*:]]
 ; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; OPTSIZE:       [[VECTOR_PH]]:
 ; OPTSIZE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; OPTSIZE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; OPTSIZE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; OPTSIZE-NEXT:    [[N_RND_UP:%.*]] = add i64 15, [[TMP2]]
-; OPTSIZE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; OPTSIZE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; OPTSIZE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; OPTSIZE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; OPTSIZE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; OPTSIZE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
+; OPTSIZE-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
 ; OPTSIZE-NEXT:    [[TMP7:%.*]] = sub i64 15, [[TMP6]]
 ; OPTSIZE-NEXT:    [[TMP8:%.*]] = icmp ugt i64 15, [[TMP6]]
 ; OPTSIZE-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -591,9 +573,8 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; OPTSIZE-NEXT:    [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
 ; OPTSIZE-NEXT:    [[TMP11:%.*]] = mul <vscale x 16 x i8> [[TMP10]], splat (i8 1)
 ; OPTSIZE-NEXT:    [[INDUCTION:%.*]] = add <vscale x 16 x i8> zeroinitializer, [[TMP11]]
-; OPTSIZE-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP4]] to i8
-; OPTSIZE-NEXT:    [[TMP13:%.*]] = mul i8 1, [[TMP12]]
-; OPTSIZE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP13]], i64 0
+; OPTSIZE-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP1]] to i8
+; OPTSIZE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP12]], i64 0
 ; OPTSIZE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[DOTSPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
 ; OPTSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; OPTSIZE:       [[VECTOR_BODY]]:
@@ -609,19 +590,18 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; OPTSIZE-NEXT:    [[TMP21:%.*]] = add <vscale x 16 x i8> [[TMP18]], [[TMP20]]
 ; OPTSIZE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
 ; OPTSIZE-NEXT:    call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP21]], ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; OPTSIZE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; OPTSIZE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; OPTSIZE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; OPTSIZE-NEXT:    [[TMP24:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; OPTSIZE-NEXT:    [[TMP24:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; OPTSIZE-NEXT:    [[TMP23:%.*]] = xor i1 [[TMP24]], true
 ; OPTSIZE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[DOTSPLAT]]
-; OPTSIZE-NEXT:    [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[TMP24]], i32 0
-; OPTSIZE-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; OPTSIZE-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; OPTSIZE:       [[SCALAR_PH]]:
-; OPTSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; OPTSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; OPTSIZE:       [[FOR_BODY]]:
-; OPTSIZE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; OPTSIZE-NEXT:    [[TMP26:%.*]] = trunc nuw nsw i64 [[IV]] to i8
 ; OPTSIZE-NEXT:    [[MUL:%.*]] = mul i8 [[A]], [[TMP26]]
 ; OPTSIZE-NEXT:    [[SHR:%.*]] = lshr i8 [[TMP26]], 1
@@ -640,19 +620,13 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ;
 ; MINSIZE-LABEL: define void @sve_tail_predicate_without_minsize(
 ; MINSIZE-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; MINSIZE-NEXT:  [[ENTRY:.*]]:
+; MINSIZE-NEXT:  [[ENTRY:.*:]]
 ; MINSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; MINSIZE:       [[VECTOR_PH]]:
 ; MINSIZE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; MINSIZE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; MINSIZE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; MINSIZE-NEXT:    [[N_RND_UP:%.*]] = add i64 15, [[TMP2]]
-; MINSIZE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; MINSIZE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; MINSIZE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; MINSIZE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; MINSIZE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; MINSIZE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
+; MINSIZE-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
 ; MINSIZE-NEXT:    [[TMP7:%.*]] = sub i64 15, [[TMP6]]
 ; MINSIZE-NEXT:    [[TMP8:%.*]] = icmp ugt i64 15, [[TMP6]]
 ; MINSIZE-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -666,9 +640,8 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; MINSIZE-NEXT:    [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.stepvector.nxv16i8()
 ; MINSIZE-NEXT:    [[TMP11:%.*]] = mul <vscale x 16 x i8> [[TMP10]], splat (i8 1)
 ; MINSIZE-NEXT:    [[INDUCTION:%.*]] = add <vscale x 16 x i8> zeroinitializer, [[TMP11]]
-; MINSIZE-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP4]] to i8
-; MINSIZE-NEXT:    [[TMP13:%.*]] = mul i8 1, [[TMP12]]
-; MINSIZE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP13]], i64 0
+; MINSIZE-NEXT:    [[TMP12:%.*]] = trunc i64 [[TMP1]] to i8
+; MINSIZE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[TMP12]], i64 0
 ; MINSIZE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[DOTSPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
 ; MINSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; MINSIZE:       [[VECTOR_BODY]]:
@@ -684,19 +657,18 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; MINSIZE-NEXT:    [[TMP21:%.*]] = add <vscale x 16 x i8> [[TMP18]], [[TMP20]]
 ; MINSIZE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]]
 ; MINSIZE-NEXT:    call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP21]], ptr [[TMP22]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; MINSIZE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; MINSIZE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; MINSIZE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; MINSIZE-NEXT:    [[TMP24:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; MINSIZE-NEXT:    [[TMP24:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; MINSIZE-NEXT:    [[TMP23:%.*]] = xor i1 [[TMP24]], true
 ; MINSIZE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i8> [[VEC_IND]], [[DOTSPLAT]]
-; MINSIZE-NEXT:    [[TMP25:%.*]] = extractelement <vscale x 16 x i1> [[TMP24]], i32 0
-; MINSIZE-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; MINSIZE-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; MINSIZE:       [[MIDDLE_BLOCK]]:
 ; MINSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; MINSIZE:       [[SCALAR_PH]]:
-; MINSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; MINSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; MINSIZE:       [[FOR_BODY]]:
-; MINSIZE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; MINSIZE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; MINSIZE-NEXT:    [[TMP26:%.*]] = trunc nuw nsw i64 [[IV]] to i8
 ; MINSIZE-NEXT:    [[MUL:%.*]] = mul i8 [[A]], [[TMP26]]
 ; MINSIZE-NEXT:    [[SHR:%.*]] = lshr i8 [[TMP26]], 1
@@ -743,7 +715,7 @@ for.cond.cleanup:
 ; FIXME: We currently vectorize with minsize as the trunc cost is incorrect
 define void @dont_vectorize_with_minsize() {
 ; DEFAULT-LABEL: define void @dont_vectorize_with_minsize() {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -775,10 +747,9 @@ define void @dont_vectorize_with_minsize() {
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; DEFAULT:       [[FOR_BODY]]:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; DEFAULT-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; DEFAULT-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -797,7 +768,7 @@ define void @dont_vectorize_with_minsize() {
 ;
 ; OPTSIZE-LABEL: define void @dont_vectorize_with_minsize(
 ; OPTSIZE-SAME: ) #[[ATTR0]] {
-; OPTSIZE-NEXT:  [[ENTRY:.*]]:
+; OPTSIZE-NEXT:  [[ENTRY:.*:]]
 ; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; OPTSIZE:       [[VECTOR_PH]]:
 ; OPTSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -819,10 +790,9 @@ define void @dont_vectorize_with_minsize() {
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; OPTSIZE:       [[SCALAR_PH]]:
-; OPTSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; OPTSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; OPTSIZE:       [[FOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; OPTSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; OPTSIZE-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; OPTSIZE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -841,7 +811,7 @@ define void @dont_vectorize_with_minsize() {
 ;
 ; MINSIZE-LABEL: define void @dont_vectorize_with_minsize(
 ; MINSIZE-SAME: ) #[[ATTR0]] {
-; MINSIZE-NEXT:  [[ENTRY:.*]]:
+; MINSIZE-NEXT:  [[ENTRY:.*:]]
 ; MINSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; MINSIZE:       [[VECTOR_PH]]:
 ; MINSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -863,10 +833,9 @@ define void @dont_vectorize_with_minsize() {
 ; MINSIZE:       [[MIDDLE_BLOCK]]:
 ; MINSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; MINSIZE:       [[SCALAR_PH]]:
-; MINSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; MINSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; MINSIZE:       [[FOR_BODY]]:
-; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; MINSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; MINSIZE-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; MINSIZE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -913,7 +882,7 @@ for.cond.cleanup:
 ; FIXME: We currently use width 2 as the load/store cost is incorrect.
 define void @vectorization_forced_minsize_reduce_width() {
 ; DEFAULT-LABEL: define void @vectorization_forced_minsize_reduce_width() {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -945,10 +914,9 @@ define void @vectorization_forced_minsize_reduce_width() {
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; DEFAULT:       [[FOR_BODY]]:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; DEFAULT-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; DEFAULT-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -967,7 +935,7 @@ define void @vectorization_forced_minsize_reduce_width() {
 ;
 ; OPTSIZE-LABEL: define void @vectorization_forced_minsize_reduce_width(
 ; OPTSIZE-SAME: ) #[[ATTR0]] {
-; OPTSIZE-NEXT:  [[ENTRY:.*]]:
+; OPTSIZE-NEXT:  [[ENTRY:.*:]]
 ; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; OPTSIZE:       [[VECTOR_PH]]:
 ; OPTSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -989,10 +957,9 @@ define void @vectorization_forced_minsize_reduce_width() {
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; OPTSIZE:       [[SCALAR_PH]]:
-; OPTSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; OPTSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; OPTSIZE:       [[FOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; OPTSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; OPTSIZE-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; OPTSIZE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -1011,7 +978,7 @@ define void @vectorization_forced_minsize_reduce_width() {
 ;
 ; MINSIZE-LABEL: define void @vectorization_forced_minsize_reduce_width(
 ; MINSIZE-SAME: ) #[[ATTR0]] {
-; MINSIZE-NEXT:  [[ENTRY:.*]]:
+; MINSIZE-NEXT:  [[ENTRY:.*:]]
 ; MINSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; MINSIZE:       [[VECTOR_PH]]:
 ; MINSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -1033,10 +1000,9 @@ define void @vectorization_forced_minsize_reduce_width() {
 ; MINSIZE:       [[MIDDLE_BLOCK]]:
 ; MINSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; MINSIZE:       [[SCALAR_PH]]:
-; MINSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; MINSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; MINSIZE:       [[FOR_BODY]]:
-; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; MINSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; MINSIZE-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; MINSIZE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
index 2cec6ca498e49..50df6fcd3cdca 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
@@ -10,7 +10,7 @@ define void @foo() {
 ; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -18,36 +18,33 @@ define void @foo() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 1)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP7]]
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP5]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <vscale x 4 x i64> [[TMP4]], splat (i64 1)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP5]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_LATCH]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP9]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP6]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
 ; CHECK-NEXT:    br label [[INNER_LOOP1:%.*]]
 ; CHECK:       inner_loop1:
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP12:%.*]], [[INNER_LOOP1]] ]
-; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 4 x float> [ [[WIDE_MASKED_GATHER]], [[VECTOR_BODY]] ], [ [[TMP11:%.*]], [[INNER_LOOP1]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, <vscale x 4 x i64> [[VEC_PHI]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
-; CHECK-NEXT:    [[TMP11]] = fmul <vscale x 4 x float> [[VEC_PHI2]], [[WIDE_MASKED_GATHER3]]
-; CHECK-NEXT:    [[TMP12]] = add nuw nsw <vscale x 4 x i64> [[VEC_PHI]], splat (i64 1)
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq <vscale x 4 x i64> [[TMP12]], splat (i64 512)
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 4 x i1> [[TMP13]], i32 0
-; CHECK-NEXT:    br i1 [[TMP14]], label [[VECTOR_LATCH]], label [[INNER_LOOP1]]
+; CHECK-NEXT:    [[TMP7:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP11:%.*]], [[INNER_LOOP1]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <vscale x 4 x float> [ [[WIDE_MASKED_GATHER]], [[VECTOR_BODY]] ], [ [[TMP10:%.*]], [[INNER_LOOP1]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, <vscale x 4 x i64> [[TMP7]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP9]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison)
+; CHECK-NEXT:    [[TMP10]] = fmul <vscale x 4 x float> [[TMP8]], [[WIDE_MASKED_GATHER2]]
+; CHECK-NEXT:    [[TMP11]] = add nuw nsw <vscale x 4 x i64> [[TMP7]], splat (i64 1)
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq <vscale x 4 x i64> [[TMP11]], splat (i64 512)
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[TMP12]], i32 0
+; CHECK-NEXT:    br i1 [[TMP13]], label [[VECTOR_LATCH]], label [[INNER_LOOP1]]
 ; CHECK:       vector.latch:
-; CHECK-NEXT:    [[VEC_PHI4:%.*]] = phi <vscale x 4 x float> [ [[TMP11]], [[INNER_LOOP1]] ]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[VEC_PHI4]], <vscale x 4 x ptr> [[TMP9]], i32 4, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi <vscale x 4 x float> [ [[TMP10]], [[INNER_LOOP1]] ]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[TMP14]], <vscale x 4 x ptr> [[TMP6]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
index 787d63cdb421e..330c22d736809 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
@@ -53,7 +53,7 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
@@ -61,8 +61,6 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -80,7 +78,7 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP17:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
 ; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
 ; CHECK-SVE-NEXT:    [[TMP19]] = sub <vscale x 4 x i32> [[TMP17]], [[TMP18]]
-; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-SVE:       middle.block:
@@ -96,7 +94,7 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
@@ -104,8 +102,6 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -124,7 +120,7 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP15]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = sub <vscale x 8 x i32> zeroinitializer, [[TMP17]]
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP18]])
-; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-SVE-MAXBW:       middle.block:
@@ -209,7 +205,7 @@ define i32 @chained_partial_reduce_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
@@ -217,8 +213,6 @@ define i32 @chained_partial_reduce_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -236,7 +230,7 @@ define i32 @chained_partial_reduce_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP17:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
 ; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
 ; CHECK-SVE-NEXT:    [[TMP19]] = add <vscale x 4 x i32> [[TMP17]], [[TMP18]]
-; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-SVE:       middle.block:
@@ -252,7 +246,7 @@ define i32 @chained_partial_reduce_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
@@ -260,8 +254,6 @@ define i32 @chained_partial_reduce_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -279,7 +271,7 @@ define i32 @chained_partial_reduce_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP16]])
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP15]]
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP17]])
-; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-SVE-MAXBW:       middle.block:
@@ -365,7 +357,7 @@ define i32 @chained_partial_reduce_sub_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
@@ -373,8 +365,6 @@ define i32 @chained_partial_reduce_sub_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -392,7 +382,7 @@ define i32 @chained_partial_reduce_sub_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP17:%.*]] = sub <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
 ; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
 ; CHECK-SVE-NEXT:    [[TMP19]] = add <vscale x 4 x i32> [[TMP17]], [[TMP18]]
-; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-SVE:       middle.block:
@@ -408,7 +398,7 @@ define i32 @chained_partial_reduce_sub_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
@@ -416,8 +406,6 @@ define i32 @chained_partial_reduce_sub_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -436,7 +424,7 @@ define i32 @chained_partial_reduce_sub_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP17]])
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP15]]
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP18]])
-; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-SVE-MAXBW:       middle.block:
@@ -525,7 +513,7 @@ define i32 @chained_partial_reduce_sub_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
@@ -533,8 +521,6 @@ define i32 @chained_partial_reduce_sub_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -552,7 +538,7 @@ define i32 @chained_partial_reduce_sub_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP17:%.*]] = sub <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
 ; CHECK-SVE-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP15]]
 ; CHECK-SVE-NEXT:    [[TMP19]] = sub <vscale x 4 x i32> [[TMP17]], [[TMP18]]
-; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-SVE:       middle.block:
@@ -568,7 +554,7 @@ define i32 @chained_partial_reduce_sub_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
@@ -576,8 +562,6 @@ define i32 @chained_partial_reduce_sub_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -597,7 +581,7 @@ define i32 @chained_partial_reduce_sub_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP15]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = sub <vscale x 8 x i32> zeroinitializer, [[TMP18]]
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP19]])
-; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-SVE-MAXBW:       middle.block:
@@ -687,7 +671,7 @@ define i32 @chained_partial_reduce_add_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
@@ -695,8 +679,6 @@ define i32 @chained_partial_reduce_add_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -716,7 +698,7 @@ define i32 @chained_partial_reduce_add_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[TMP19:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[TMP18]]
 ; CHECK-SVE-NEXT:    [[TMP20:%.*]] = mul nsw <vscale x 4 x i32> [[TMP14]], [[TMP15]]
 ; CHECK-SVE-NEXT:    [[TMP21]] = add <vscale x 4 x i32> [[TMP19]], [[TMP20]]
-; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-SVE:       middle.block:
@@ -732,7 +714,7 @@ define i32 @chained_partial_reduce_add_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
@@ -740,8 +722,6 @@ define i32 @chained_partial_reduce_add_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -761,7 +741,7 @@ define i32 @chained_partial_reduce_add_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP17]])
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 8 x i32> [[TMP14]], [[TMP15]]
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE4]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE3]], <vscale x 8 x i32> [[TMP18]])
-; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-SVE-MAXBW:       middle.block:
@@ -855,7 +835,7 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
@@ -863,8 +843,6 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -884,7 +862,7 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[TMP19:%.*]] = add <vscale x 4 x i32> [[TMP17]], [[TMP18]]
 ; CHECK-SVE-NEXT:    [[TMP20:%.*]] = mul nsw <vscale x 4 x i32> [[TMP14]], [[TMP15]]
 ; CHECK-SVE-NEXT:    [[TMP21]] = sub <vscale x 4 x i32> [[TMP19]], [[TMP20]]
-; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-SVE:       middle.block:
@@ -900,7 +878,7 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
@@ -908,8 +886,6 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -931,7 +907,7 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = mul nsw <vscale x 8 x i32> [[TMP14]], [[TMP15]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP20:%.*]] = sub <vscale x 8 x i32> zeroinitializer, [[TMP19]]
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE4]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE3]], <vscale x 8 x i32> [[TMP20]])
-; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-SVE-MAXBW:       middle.block:
@@ -1021,7 +997,7 @@ define i32 @chained_partial_reduce_madd_extadd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
@@ -1029,8 +1005,6 @@ define i32 @chained_partial_reduce_madd_extadd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1047,7 +1021,7 @@ define i32 @chained_partial_reduce_madd_extadd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-SVE-NEXT:    [[TMP17:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP16]]
 ; CHECK-SVE-NEXT:    [[TMP18]] = add <vscale x 4 x i32> [[TMP17]], [[TMP15]]
-; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-SVE:       middle.block:
@@ -1063,7 +1037,7 @@ define i32 @chained_partial_reduce_madd_extadd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
@@ -1071,8 +1045,6 @@ define i32 @chained_partial_reduce_madd_extadd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1089,7 +1061,7 @@ define i32 @chained_partial_reduce_madd_extadd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP16]])
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP15]])
-; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-SVE-MAXBW:       middle.block:
@@ -1169,7 +1141,7 @@ define i32 @chained_partial_reduce_extadd_extadd(ptr %a, ptr %b, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
@@ -1177,8 +1149,6 @@ define i32 @chained_partial_reduce_extadd_extadd(ptr %a, ptr %b, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1191,7 +1161,7 @@ define i32 @chained_partial_reduce_extadd_extadd(ptr %a, ptr %b, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[TMP12:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; CHECK-SVE-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP11]]
 ; CHECK-SVE-NEXT:    [[TMP14]] = add <vscale x 4 x i32> [[TMP13]], [[TMP12]]
-; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-SVE:       middle.block:
@@ -1207,7 +1177,7 @@ define i32 @chained_partial_reduce_extadd_extadd(ptr %a, ptr %b, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
@@ -1215,8 +1185,6 @@ define i32 @chained_partial_reduce_extadd_extadd(ptr %a, ptr %b, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1229,7 +1197,7 @@ define i32 @chained_partial_reduce_extadd_extadd(ptr %a, ptr %b, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP12:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP11]])
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE2]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP12]])
-; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-SVE-MAXBW:       middle.block:
@@ -1310,7 +1278,7 @@ define i32 @chained_partial_reduce_extadd_madd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
@@ -1318,8 +1286,6 @@ define i32 @chained_partial_reduce_extadd_madd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1336,7 +1302,7 @@ define i32 @chained_partial_reduce_extadd_madd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[TMP16:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP15]]
 ; CHECK-SVE-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 4 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-SVE-NEXT:    [[TMP18]] = add <vscale x 4 x i32> [[TMP16]], [[TMP17]]
-; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK-SVE:       middle.block:
@@ -1352,7 +1318,7 @@ define i32 @chained_partial_reduce_extadd_madd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
@@ -1360,8 +1326,6 @@ define i32 @chained_partial_reduce_extadd_madd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1378,7 +1342,7 @@ define i32 @chained_partial_reduce_extadd_madd(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE:%.*]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP15]])
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP16:%.*]] = mul nsw <vscale x 8 x i32> [[TMP13]], [[TMP14]]
 ; CHECK-SVE-MAXBW-NEXT:    [[PARTIAL_REDUCE3]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]], <vscale x 8 x i32> [[TMP16]])
-; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-SVE-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK-SVE-MAXBW:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
index be8cfa2fb64b7..9d4a969b571e7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
@@ -27,6 +27,8 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-NEXT:    br label [[FOR_EXIT:%.*]]
+; CHECK:       scalar.ph:
 ;
 entry:
   br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
index d01effdeacfc1..82ea025700c37 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
@@ -10,15 +10,13 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -26,7 +24,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 3
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
@@ -34,7 +32,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[TMP12:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 3
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP13]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP17]], align 1
@@ -44,7 +42,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[TMP21:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP12]]
 ; CHECK-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP20]])
 ; CHECK-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP21]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -58,15 +56,13 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NOI8MM-NEXT:  entry:
 ; CHECK-NOI8MM-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-NOI8MM-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NOI8MM-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-NOI8MM:       vector.ph:
 ; CHECK-NOI8MM-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NOI8MM-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NOI8MM-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NOI8MM-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NOI8MM-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NOI8MM-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-NOI8MM:       vector.body:
 ; CHECK-NOI8MM-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -74,7 +70,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NOI8MM-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
+; CHECK-NOI8MM-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 3
 ; CHECK-NOI8MM-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP6]], align 1
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
@@ -82,7 +78,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP12:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
 ; CHECK-NOI8MM-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
+; CHECK-NOI8MM-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 3
 ; CHECK-NOI8MM-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP13]], align 1
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP17]], align 1
@@ -92,7 +88,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP21:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP12]]
 ; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP20]])
 ; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP21]])
-; CHECK-NOI8MM-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NOI8MM-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NOI8MM-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-NOI8MM:       middle.block:
@@ -129,15 +125,13 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -145,7 +139,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 3
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
@@ -153,7 +147,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[TMP12:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 3
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP13]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP17]], align 1
@@ -163,7 +157,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[TMP21:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP12]]
 ; CHECK-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP20]])
 ; CHECK-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP21]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
@@ -177,15 +171,13 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NOI8MM-NEXT:  entry:
 ; CHECK-NOI8MM-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-NOI8MM-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NOI8MM-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-NOI8MM:       vector.ph:
 ; CHECK-NOI8MM-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NOI8MM-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NOI8MM-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NOI8MM-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NOI8MM-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NOI8MM-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-NOI8MM:       vector.body:
 ; CHECK-NOI8MM-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -193,7 +185,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NOI8MM-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
+; CHECK-NOI8MM-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 3
 ; CHECK-NOI8MM-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP6]], align 1
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
@@ -201,7 +193,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP12:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i32>
 ; CHECK-NOI8MM-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
+; CHECK-NOI8MM-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 3
 ; CHECK-NOI8MM-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP13]], align 1
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP17]], align 1
@@ -211,7 +203,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP21:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP12]]
 ; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP20]])
 ; CHECK-NOI8MM-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP21]])
-; CHECK-NOI8MM-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NOI8MM-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NOI8MM-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NOI8MM-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-NOI8MM:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
index 25a4ab8e0a943..cc3203cdff46f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
@@ -493,6 +493,8 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = extractelement <16 x i32> [[TMP9]], i32 15
+; CHECK-INTERLEAVE1-NEXT:    br label [[FOR_EXIT:%.*]]
 ; CHECK-INTERLEAVE1:       scalar.ph:
 ;
 ; CHECK-INTERLEAVED-LABEL: define i32 @not_dotp_not_loop_carried(
@@ -517,6 +519,8 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = extractelement <16 x i32> [[TMP9]], i32 15
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_EXIT:%.*]]
 ; CHECK-INTERLEAVED:       scalar.ph:
 ;
 ; CHECK-MAXBW-LABEL: define i32 @not_dotp_not_loop_carried(
@@ -541,6 +545,8 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) {
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = extractelement <16 x i32> [[TMP9]], i32 15
+; CHECK-MAXBW-NEXT:    br label [[FOR_EXIT:%.*]]
 ; CHECK-MAXBW:       scalar.ph:
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index 195101f650faa..938d58bf5e1e3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -11,15 +11,13 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY]] ]
@@ -32,7 +30,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = mul <vscale x 4 x i32> [[TMP18]], [[TMP9]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP14]] = add <vscale x 4 x i32> [[TMP13]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -45,15 +43,13 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY]] ]
@@ -61,7 +57,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX1]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP14]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP14]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP20]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP20]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
@@ -69,7 +65,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX1]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = shl nuw i64 [[TMP26]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP28]], i64 [[TMP27]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i8>, ptr [[TMP28]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
@@ -79,7 +75,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = mul <vscale x 4 x i32> [[TMP29]], [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23]] = add <vscale x 4 x i32> [[TMP30]], [[VEC_PHI]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24]] = add <vscale x 4 x i32> [[TMP22]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT1]] = add nuw i64 [[INDEX1]], [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT1]], [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -93,15 +89,13 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -114,7 +108,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD4]] to <vscale x 8 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = mul <vscale x 8 x i32> [[TMP20]], [[TMP13]]
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE5]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP22]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -150,15 +144,13 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-INTERLEAVE1-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP7]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP7]], 1
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP9]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP18]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -173,7 +165,7 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD2]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP13]], [[TMP11]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15]] = add <vscale x 2 x i64> [[TMP14]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -186,15 +178,13 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-INTERLEAVED-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP7]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP7]], 2
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP16]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP28]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -205,14 +195,14 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP12]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[NEXT_GEP]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i8>, ptr [[TMP13]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD3]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i64 [[TMP18]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i8>, ptr [[NEXT_GEP1]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i8>, ptr [[TMP19]], align 1
@@ -222,7 +212,7 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP21]], [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24]] = add <vscale x 2 x i64> [[TMP22]], [[VEC_PHI]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25]] = add <vscale x 2 x i64> [[TMP23]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -236,15 +226,13 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-MAXBW-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -259,7 +247,7 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i64>
 ; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = mul nuw nsw <vscale x 8 x i64> [[TMP13]], [[TMP11]]
 ; CHECK-MAXBW-NEXT:    [[TMP15]] = add <vscale x 8 x i64> [[TMP14]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -297,15 +285,13 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-INTERLEAVE1-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP7]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP7]], 1
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP14]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP10]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP20]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP10]], 2
@@ -324,7 +310,7 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD3]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP15]], [[TMP13]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17]] = add <vscale x 2 x i64> [[TMP16]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -337,15 +323,13 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-INTERLEAVED-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP7]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP7]], 2
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP11]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP12]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP15]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP15]], 2
@@ -360,14 +344,14 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-INTERLEAVED-NEXT:    [[OFFSET_IDX2:%.*]] = mul i64 [[INDEX]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX2]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i64 [[TMP14]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i16>, ptr [[NEXT_GEP]], align 2
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i16>, ptr [[TMP30]], align 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD4]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = getelementptr i16, ptr [[NEXT_GEP3]], i64 [[TMP20]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i16>, ptr [[NEXT_GEP3]], align 2
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 2 x i16>, ptr [[TMP21]], align 2
@@ -377,7 +361,7 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP23]], [[TMP17]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26]] = add <vscale x 2 x i64> [[TMP24]], [[VEC_PHI]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27]] = add <vscale x 2 x i64> [[TMP25]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -391,15 +375,13 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-MAXBW-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul i64 [[N_VEC]], 2
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = mul i64 [[N_VEC]], 2
@@ -418,7 +400,7 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = zext <vscale x 4 x i16> [[WIDE_LOAD3]] to <vscale x 4 x i64>
 ; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = mul nuw nsw <vscale x 4 x i64> [[TMP15]], [[TMP13]]
 ; CHECK-MAXBW-NEXT:    [[TMP17]] = add <vscale x 4 x i64> [[TMP16]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -812,15 +794,13 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
@@ -838,7 +818,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16]] = mul <vscale x 8 x i32> [[TMP15]], [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> [[VECTOR_RECUR]], <vscale x 8 x i32> [[TMP16]], i32 -1)
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = add <vscale x 8 x i32> [[TMP16]], [[TMP17]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -858,21 +838,19 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
@@ -880,7 +858,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 3
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP20]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP17]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP21]], align 1
@@ -890,7 +868,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = mul <vscale x 8 x i32> [[TMP23]], [[TMP16]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> [[TMP24]], <vscale x 8 x i32> [[TMP25]], i32 -1)
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = add <vscale x 8 x i32> [[TMP25]], [[TMP26]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -910,15 +888,13 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
@@ -936,7 +912,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP25]] = mul <vscale x 8 x i32> [[TMP23]], [[TMP16]]
 ; CHECK-MAXBW-NEXT:    [[TMP26:%.*]] = call <vscale x 8 x i32> @llvm.vector.splice.nxv8i32(<vscale x 8 x i32> [[VECTOR_RECUR]], <vscale x 8 x i32> [[TMP25]], i32 -1)
 ; CHECK-MAXBW-NEXT:    [[TMP27:%.*]] = add <vscale x 8 x i32> [[TMP25]], [[TMP26]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -979,15 +955,13 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -999,7 +973,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = mul <vscale x 8 x i32> [[TMP15]], [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = add <vscale x 8 x i32> [[TMP16]], [[TMP15]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -1019,33 +993,31 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 3
 ; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP19]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP20]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = mul <vscale x 8 x i32> [[TMP22]], [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = add <vscale x 8 x i32> [[TMP30]], [[TMP22]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -1065,15 +1037,13 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1085,7 +1055,7 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = mul <vscale x 8 x i32> [[TMP19]], [[TMP14]]
 ; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = add <vscale x 8 x i32> [[TMP20]], [[TMP19]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -1128,7 +1098,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: i32 [[NUM_OUT:%.*]], i64 [[NUM_IN:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP13]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP13]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_IN]], [[TMP15]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
@@ -1136,8 +1106,6 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP16]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[NUM_IN]], [[TMP18]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[NUM_IN]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP20]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1180,7 +1148,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP39:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD10]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP40:%.*]] = mul nsw <vscale x 4 x i32> [[TMP37]], [[TMP39]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP41]] = add <vscale x 4 x i32> [[TMP40]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP26]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -1196,7 +1164,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: i32 [[NUM_OUT:%.*]], i64 [[NUM_IN:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP13]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP13]], 3
 ; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_IN]], [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
@@ -1204,8 +1172,6 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP16]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[NUM_IN]], [[TMP18]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[NUM_IN]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP34:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP40:%.*]] = mul nuw i64 [[TMP34]], 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1229,14 +1195,14 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP56:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP56]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = shl nuw i64 [[TMP56]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[TMP20]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP1]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 4 x i8>, ptr [[TMP21]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP66:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD8]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = shl nuw i64 [[TMP25]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 [[TMP26]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 4 x i8>, ptr [[TMP2]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i8>, ptr [[TMP72]], align 1
@@ -1247,14 +1213,14 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP50]] = add <vscale x 4 x i32> [[TMP30]], [[VEC_PHI6]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP33]] = add <vscale x 4 x i32> [[TMP31]], [[VEC_PHI7]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP35:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP36:%.*]] = mul nuw i64 [[TMP35]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP36:%.*]] = shl nuw i64 [[TMP35]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 [[TMP36]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD11:%.*]] = load <vscale x 4 x i8>, ptr [[TMP4]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD12:%.*]] = load <vscale x 4 x i8>, ptr [[TMP37]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP38:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD11]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP39:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD12]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP41:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP42:%.*]] = mul nuw i64 [[TMP41]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP42:%.*]] = shl nuw i64 [[TMP41]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 [[TMP42]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD13:%.*]] = load <vscale x 4 x i8>, ptr [[TMP5]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD14:%.*]] = load <vscale x 4 x i8>, ptr [[TMP43]], align 1
@@ -1265,14 +1231,14 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP48]] = add <vscale x 4 x i32> [[TMP46]], [[VEC_PHI4]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP49]] = add <vscale x 4 x i32> [[TMP47]], [[VEC_PHI5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP51:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP52:%.*]] = mul nuw i64 [[TMP51]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP52:%.*]] = shl nuw i64 [[TMP51]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP52]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD15:%.*]] = load <vscale x 4 x i8>, ptr [[TMP7]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD16:%.*]] = load <vscale x 4 x i8>, ptr [[TMP53]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP54:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD15]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP55:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD16]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP57:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP58:%.*]] = mul nuw i64 [[TMP57]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP58:%.*]] = shl nuw i64 [[TMP57]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 [[TMP58]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD17:%.*]] = load <vscale x 4 x i8>, ptr [[TMP8]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD18:%.*]] = load <vscale x 4 x i8>, ptr [[TMP59]], align 1
@@ -1283,14 +1249,14 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP64]] = add <vscale x 4 x i32> [[TMP62]], [[VEC_PHI2]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP65]] = add <vscale x 4 x i32> [[TMP63]], [[VEC_PHI3]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP67:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP68:%.*]] = mul nuw i64 [[TMP67]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP68:%.*]] = shl nuw i64 [[TMP67]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP69:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 [[TMP68]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD19:%.*]] = load <vscale x 4 x i8>, ptr [[TMP10]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD20:%.*]] = load <vscale x 4 x i8>, ptr [[TMP69]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP70:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD19]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP71:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD20]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP73:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP74:%.*]] = mul nuw i64 [[TMP73]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP74:%.*]] = shl nuw i64 [[TMP73]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 [[TMP74]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD21:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD22:%.*]] = load <vscale x 4 x i8>, ptr [[TMP75]], align 1
@@ -1300,7 +1266,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP79:%.*]] = mul nsw <vscale x 4 x i32> [[TMP71]], [[TMP77]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP80]] = add <vscale x 4 x i32> [[TMP78]], [[VEC_PHI]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP81]] = add <vscale x 4 x i32> [[TMP79]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP40]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -1320,7 +1286,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: i32 [[NUM_OUT:%.*]], i64 [[NUM_IN:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_IN]], [[TMP1]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
@@ -1328,8 +1294,6 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[NUM_IN]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[NUM_IN]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1372,7 +1336,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP71:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD26]] to <vscale x 8 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP73:%.*]] = mul nsw <vscale x 8 x i32> [[TMP65]], [[TMP71]]
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE16]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI4]], <vscale x 8 x i32> [[TMP73]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP74:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP74]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -1445,16 +1409,10 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
-; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-INTERLEAVE1-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -1475,8 +1433,8 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP19]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-INTERLEAVE1-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[TMP20]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP21:%.*]] = xor i1 [[TMP20]], true
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
@@ -1488,16 +1446,10 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
-; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-INTERLEAVED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
-; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -1518,8 +1470,8 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP18]], <vscale x 4 x i32> [[VEC_PHI]]
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[TMP20]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = xor i1 [[TMP20]], true
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
@@ -1531,16 +1483,10 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
-; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-MAXBW-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -1561,8 +1507,8 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP18]])
 ; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 16 x i1> [[TMP19]], i32 0
+; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = xor i1 [[TMP19]], true
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
@@ -1596,15 +1542,13 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP11]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1617,7 +1561,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = mul <vscale x 4 x i32> [[TMP12]], [[TMP9]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP14]] = add <vscale x 4 x i32> [[TMP13]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -1634,15 +1578,13 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP7]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP14]], 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1650,7 +1592,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP15]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP15]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP3]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
@@ -1658,7 +1600,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = shl nuw i64 [[TMP16]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP17]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i8>, ptr [[TMP8]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
@@ -1668,7 +1610,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = mul <vscale x 4 x i32> [[TMP20]], [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23]] = add <vscale x 4 x i32> [[TMP21]], [[VEC_PHI]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24]] = add <vscale x 4 x i32> [[TMP22]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -1686,15 +1628,13 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1707,7 +1647,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD4]] to <vscale x 8 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = mul <vscale x 8 x i32> [[TMP20]], [[TMP13]]
 ; CHECK-MAXBW-NEXT:    [[TMP24]] = add <vscale x 8 x i32> [[TMP22]], [[VEC_PHI1]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -1748,15 +1688,13 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 41, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 41, [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1770,7 +1708,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD1]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP13]], [[TMP9]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP14]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -1783,7 +1721,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 41, [[TMP1]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
@@ -1791,8 +1729,6 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 41, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 41, [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1800,7 +1736,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP7]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 2 x i8>, ptr [[TMP11]], align 1
@@ -1809,7 +1745,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = add nuw nsw i64 [[INDEX]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP14]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 [[TMP18]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i8>, ptr [[TMP15]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i8>, ptr [[TMP19]], align 1
@@ -1819,7 +1755,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP21]], [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP22]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP23]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -1833,7 +1769,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 41, [[TMP1]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
@@ -1841,8 +1777,6 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 41, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 41, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1853,14 +1787,14 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = add nuw nsw i64 [[INDEX]], 1
 ; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP10]]
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
-; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i64>
-; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = mul nuw nsw <vscale x 8 x i64> [[TMP12]], [[TMP9]]
-; CHECK-MAXBW-NEXT:    [[TMP14]] = add <vscale x 8 x i64> [[VEC_PHI]], [[TMP15]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i64>
+; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = mul nuw nsw <vscale x 8 x i64> [[TMP8]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[TMP14]] = add <vscale x 8 x i64> [[VEC_PHI]], [[TMP13]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
-; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP14]])
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vector.reduce.add.nxv8i64(<vscale x 8 x i64> [[TMP14]])
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 41, [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-MAXBW:       scalar.ph:
@@ -2142,7 +2076,7 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW:       for.ph:
 ; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = zext i16 [[B]] to i64
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
@@ -2150,8 +2084,6 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[EXT_B]], i64 0
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -2163,7 +2095,7 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i64>
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = mul nuw nsw <vscale x 4 x i64> [[TMP9]], [[BROADCAST_SPLAT]]
 ; CHECK-MAXBW-NEXT:    [[TMP11]] = add <vscale x 4 x i64> [[TMP10]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -2277,7 +2209,7 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW:       for.ph:
 ; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = zext i16 [[B]] to i64
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
@@ -2285,8 +2217,6 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[EXT_B]], i64 0
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -2298,7 +2228,7 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i64>
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = mul nuw nsw <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP9]]
 ; CHECK-MAXBW-NEXT:    [[TMP11]] = add <vscale x 4 x i64> [[TMP10]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -2341,7 +2271,7 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVE1:       for.body.preheader:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
@@ -2349,8 +2279,6 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
@@ -2367,7 +2295,7 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD2]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP16]], [[TMP14]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18]] = add <vscale x 2 x i64> [[TMP17]], [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -2384,7 +2312,7 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVED:       for.body.preheader:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
@@ -2392,8 +2320,6 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
@@ -2406,14 +2332,14 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[NEXT_GEP]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i8>, ptr [[TMP16]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD3]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 [[TMP21]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i8>, ptr [[NEXT_GEP2]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i8>, ptr [[TMP22]], align 1
@@ -2423,7 +2349,7 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = mul nuw nsw <vscale x 2 x i64> [[TMP24]], [[TMP18]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27]] = add <vscale x 2 x i64> [[TMP25]], [[VEC_PHI]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP28]] = add <vscale x 2 x i64> [[TMP26]], [[VEC_PHI1]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -2441,7 +2367,7 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-MAXBW:       for.body.preheader:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
@@ -2449,8 +2375,6 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
@@ -2467,7 +2391,7 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD2]] to <vscale x 8 x i64>
 ; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = mul nuw nsw <vscale x 8 x i64> [[TMP16]], [[TMP14]]
 ; CHECK-MAXBW-NEXT:    [[TMP20]] = add <vscale x 8 x i64> [[TMP17]], [[VEC_PHI]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
index a46340c16bc03..89b5689845612 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
@@ -11,15 +11,13 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -33,7 +31,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = mul <vscale x 4 x i32> [[TMP12]], [[TMP9]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = sub <vscale x 4 x i32> zeroinitializer, [[TMP13]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP15]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP14]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
@@ -46,15 +44,13 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -62,7 +58,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP7]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP7]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
@@ -70,7 +66,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD2]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = shl nuw i64 [[TMP16]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP17]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i8>, ptr [[TMP14]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
@@ -82,7 +78,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = sub <vscale x 4 x i32> zeroinitializer, [[TMP22]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP23]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26]] = add <vscale x 4 x i32> [[VEC_PHI1]], [[TMP24]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
@@ -96,15 +92,13 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -118,7 +112,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i32> [[TMP12]], [[TMP9]]
 ; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = sub <vscale x 8 x i32> zeroinitializer, [[TMP13]]
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP14]])
-; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
index d2c03d14995e4..c3a7f2c477db8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll
@@ -61,15 +61,13 @@ define i32 @zext_add_reduc_i8_i32_sve(ptr %a) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -78,7 +76,7 @@ define i32 @zext_add_reduc_i8_i32_sve(ptr %a) #0 {
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -247,15 +245,13 @@ define i64 @zext_add_reduc_i8_i64(ptr %a) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -264,7 +260,7 @@ define i64 @zext_add_reduc_i8_i64(ptr %a) #0 {
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i64>
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv16i64(<vscale x 2 x i64> [[VEC_PHI]], <vscale x 16 x i64> [[TMP9]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -345,15 +341,13 @@ define i64 @zext_add_reduc_i16_i64(ptr %a) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -362,7 +356,7 @@ define i64 @zext_add_reduc_i16_i64(ptr %a) #0 {
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP7]], align 2
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x i64>
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i64> @llvm.experimental.vector.partial.reduce.add.nxv2i64.nxv8i64(<vscale x 2 x i64> [[VEC_PHI]], <vscale x 8 x i64> [[TMP9]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -443,15 +437,13 @@ define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(ptr %a) #1 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR2:[0-9]+]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -460,7 +452,7 @@ define i32 @zext_add_reduc_i8_i32_has_neon_dotprod(ptr %a) #1 {
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -496,12 +488,6 @@ define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-INTERLEAVE1-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
@@ -513,10 +499,10 @@ define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[VEC_PHI]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[VEC_PHI]]
-; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; CHECK-INTERLEAVE1-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP11]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = xor i1 [[TMP11]], true
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
@@ -530,12 +516,6 @@ define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-INTERLEAVED-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025)
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
@@ -547,10 +527,10 @@ define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = zext <vscale x 4 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[VEC_PHI]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[VEC_PHI]]
-; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; CHECK-INTERLEAVED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[TMP11]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = xor i1 [[TMP11]], true
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
@@ -564,12 +544,6 @@ define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-MAXBW-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 1025)
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
@@ -581,10 +555,10 @@ define i32 @zext_add_reduc_i8_i32_predicated(ptr %a) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; CHECK-MAXBW-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 16 x i1> [[TMP10]], i32 0
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
@@ -745,15 +719,13 @@ define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(ptr %a) #1 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR2]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -762,7 +734,7 @@ define i32 @zext_sub_reduc_i8_i32_has_neon_dotprod(ptr %a) #1 {
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP7]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
 ; CHECK-MAXBW-NEXT:    [[TMP10]] = sub <vscale x 8 x i32> [[VEC_PHI]], [[TMP9]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -843,15 +815,13 @@ define i32 @sext_add_reduc_i8_i32(ptr %a) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -860,7 +830,7 @@ define i32 @sext_add_reduc_i8_i32(ptr %a) #0 {
 ; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = sext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -962,7 +932,7 @@ define i32 @add_of_zext_outside_loop(i32 %a, ptr noalias %b, i8 %c, i32 %d) #0 {
 ; CHECK-MAXBW-NEXT:    [[CONV1:%.*]] = zext i8 [[C]] to i32
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = sub i32 1024, [[D]]
 ; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = mul nuw i32 [[TMP1]], 16
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], [[TMP2]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
@@ -970,8 +940,6 @@ define i32 @add_of_zext_outside_loop(i32 %a, ptr noalias %b, i8 %c, i32 %d) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 16
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul nuw i32 [[TMP5]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = add i32 [[D]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = insertelement <vscale x 16 x i32> zeroinitializer, i32 [[A]], i32 0
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV1]], i64 0
@@ -984,7 +952,7 @@ define i32 @add_of_zext_outside_loop(i32 %a, ptr noalias %b, i8 %c, i32 %d) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[OFFSET_IDX]]
 ; CHECK-MAXBW-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP9]], align 1
 ; CHECK-MAXBW-NEXT:    [[TMP11]] = add <vscale x 16 x i32> [[VEC_PHI]], [[BROADCAST_SPLAT]]
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
@@ -1085,7 +1053,7 @@ define i32 @add_of_loop_invariant_zext(i32 %a, ptr %b, i8 %c, i32 %d) #0 {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = sub i32 1024, [[D]]
 ; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = mul nuw i32 [[TMP1]], 16
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0]], [[TMP2]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
@@ -1093,8 +1061,6 @@ define i32 @add_of_loop_invariant_zext(i32 %a, ptr %b, i8 %c, i32 %d) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 16
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP0]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul nuw i32 [[TMP5]], 16
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[C]], i64 0
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = add i32 [[D]], [[N_VEC]]
@@ -1108,7 +1074,7 @@ define i32 @add_of_loop_invariant_zext(i32 %a, ptr %b, i8 %c, i32 %d) #0 {
 ; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B]], i32 [[OFFSET_IDX]]
 ; CHECK-MAXBW-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP10]], align 1
 ; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.experimental.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr151664-cost-hoisted-vector-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr151664-cost-hoisted-vector-scalable.ll
index 8495deea31e8b..5b0696e8118d3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr151664-cost-hoisted-vector-scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr151664-cost-hoisted-vector-scalable.ll
@@ -1,47 +1,28 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 5
-; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -S \
-; RUN:   -debug-only=loop-vectorize %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64 -mattr=+sve -S %s | FileCheck %s
 
-; FIXME: Hoisted vector code should be costed with scalable cost.
-; In this example, `<vscale x 4 x float> @llvm.minimumnum` has an invalid cost,
-; and hence should not be produced by LoopVectorize.
-
-; CHECK: LV: Found an estimated cost of Invalid for VF vscale x 4 For instruction:   %res = tail call float @llvm.minimumnum.f32(float %arg, float 0.000000e+00)
 define void @cost_hoisted_vector_code(ptr %p, float %arg) {
 ; CHECK-LABEL: define void @cost_hoisted_vector_code(
 ; CHECK-SAME: ptr [[P:%.*]], float [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 -1, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 -1, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[ARG]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = add i64 1, [[N_VEC]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x float> @llvm.minimumnum.nxv4f32(<vscale x 4 x float> [[BROADCAST_SPLAT]], <vscale x 4 x float> zeroinitializer)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[ARG]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.minimumnum.v4f32(<4 x float> [[BROADCAST_SPLAT]], <4 x float> zeroinitializer)
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[INDEX:%.*]] = add i64 1, [[INDEX1]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr float, ptr [[P]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[TMP10]]
-; CHECK-NEXT:    store <vscale x 4 x float> [[TMP7]], ptr [[TMP8]], align 4
-; CHECK-NEXT:    store <vscale x 4 x float> [[TMP7]], ptr [[TMP11]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX1]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr float, ptr [[P]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr float, ptr [[TMP1]], i32 4
+; CHECK-NEXT:    store <4 x float> [[TMP0]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    store <4 x float> [[TMP0]], ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], -8
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 -1, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
index 6d5bbde36642a..94172dea8ee6a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
@@ -9,15 +9,13 @@ define void @test_invar_gep(ptr %dst) #0 {
 ; CHECK-LABEL: @test_invar_gep(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -36,7 +34,7 @@ define void @test_invar_gep(ptr %dst) #0 {
 ; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 [[TMP16]], 1
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 4 x i64> [[TMP9]], i32 [[TMP17]]
 ; CHECK-NEXT:    store i64 [[TMP18]], ptr [[TMP14:%.*]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
index d9a3a71141540..f365bfbc3f3e5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr73894.ll
@@ -12,9 +12,6 @@ define i32 @pr70988(ptr %src, i32 %n) {
 ; CHECK-NEXT:    [[UMAX:%.*]] = zext i32 [[TMP2]] to i64
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], 1
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 2
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = icmp ult i64 0, [[UMAX]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = icmp ult i64 1, [[UMAX]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -55,12 +52,10 @@ define i32 @pr70988(ptr %src, i32 %n) {
 ; CHECK-NEXT:    [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP17]], i32 [[TMP18]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDUC:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDUC_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[MAX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TMP24:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDUC:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDUC_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[MAX:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[TMP24:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[SRC]], i64 [[INDUC]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index 08d35f71e7cc3..8b2da8c4a7047 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -51,7 +51,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; VSCALEFORTUNING2-NEXT:  [[ENTRY:.*]]:
 ; VSCALEFORTUNING2-NEXT:    [[TMP0:%.*]] = add i64 [[Y]], 1
 ; VSCALEFORTUNING2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; VSCALEFORTUNING2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; VSCALEFORTUNING2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VSCALEFORTUNING2:       [[VECTOR_PH]]:
@@ -59,8 +59,6 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; VSCALEFORTUNING2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; VSCALEFORTUNING2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; VSCALEFORTUNING2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; VSCALEFORTUNING2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
@@ -121,7 +119,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; VSCALEFORTUNING2-NEXT:    [[TMP46:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER8]], [[VEC_PHI5]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP47]] = or <vscale x 4 x i32> [[TMP45]], [[WIDE_MASKED_GATHER9]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP48]] = or <vscale x 4 x i32> [[TMP46]], [[WIDE_MASKED_GATHER10]]
-; VSCALEFORTUNING2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; VSCALEFORTUNING2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; VSCALEFORTUNING2-NEXT:    br i1 [[TMP49]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; VSCALEFORTUNING2:       [[MIDDLE_BLOCK]]:
@@ -176,24 +174,18 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ;
 ; PRED-LABEL: define i32 @chained_recurrences(
 ; PRED-SAME: i32 [[X:%.*]], i64 [[Y:%.*]], ptr [[SRC_1:%.*]], i32 [[Z:%.*]], ptr [[SRC_2:%.*]]) #[[ATTR0:[0-9]+]] {
-; PRED-NEXT:  [[ENTRY:.*]]:
+; PRED-NEXT:  [[ENTRY:.*:]]
 ; PRED-NEXT:    [[TMP0:%.*]] = add i64 [[Y]], 1
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; PRED-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP2]], 1
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP3]]
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]]
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; PRED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; PRED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
+; PRED-NEXT:    [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 2
 ; PRED-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP0]], [[TMP7]]
 ; PRED-NEXT:    [[TMP9:%.*]] = icmp ugt i64 [[TMP0]], [[TMP7]]
 ; PRED-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
@@ -218,7 +210,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; PRED-NEXT:    [[VECTOR_RECUR_INIT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP27]]
 ; PRED-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; PRED:       [[VECTOR_BODY]]:
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[IV_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; PRED-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; PRED-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT6:%.*]], %[[VECTOR_BODY]] ]
 ; PRED-NEXT:    [[VECTOR_RECUR4:%.*]] = phi <vscale x 4 x i32> [ [[VECTOR_RECUR_INIT3]], %[[VECTOR_PH]] ], [ [[TMP29:%.*]], %[[VECTOR_BODY]] ]
@@ -241,25 +233,21 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; PRED-NEXT:    [[TMP39:%.*]] = or <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], [[VEC_PHI]]
 ; PRED-NEXT:    [[TMP40:%.*]] = or <vscale x 4 x i32> [[TMP39]], [[WIDE_MASKED_GATHER7]]
 ; PRED-NEXT:    [[TMP41]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP40]], <vscale x 4 x i32> [[VEC_PHI]]
-; PRED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], [[TMP5]]
+; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[IV]], [[TMP2]]
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[IV]], i64 [[TMP10]])
-; PRED-NEXT:    [[TMP42:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; PRED-NEXT:    [[TMP43:%.*]] = extractelement <vscale x 4 x i1> [[TMP42]], i32 0
-; PRED-NEXT:    br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; PRED-NEXT:    [[TMP43:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP42:%.*]] = xor i1 [[TMP43]], true
+; PRED-NEXT:    br i1 [[TMP42]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
 ; PRED-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[TMP41]])
 ; PRED-NEXT:    br label %[[EXIT:.*]]
 ; PRED:       [[SCALAR_PH]]:
-; PRED-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; PRED-NEXT:    [[SCALAR_RECUR_INIT8:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; PRED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; PRED-NEXT:    br label %[[LOOP:.*]]
 ; PRED:       [[LOOP]]:
-; PRED-NEXT:    [[TMP45:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
-; PRED-NEXT:    [[SCALAR_RECUR10:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT8]], %[[SCALAR_PH]] ], [ [[TMP45]], %[[LOOP]] ]
-; PRED-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], %[[LOOP]] ]
-; PRED-NEXT:    [[SUM_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_2:%.*]], %[[LOOP]] ]
+; PRED-NEXT:    [[TMP45:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[TMP53:%.*]], %[[LOOP]] ]
+; PRED-NEXT:    [[SCALAR_RECUR10:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[TMP45]], %[[LOOP]] ]
+; PRED-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], %[[LOOP]] ]
+; PRED-NEXT:    [[SUM_RED:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[RED_2:%.*]], %[[LOOP]] ]
 ; PRED-NEXT:    [[TMP52:%.*]] = add i64 [[Y]], 1
 ; PRED-NEXT:    [[GEP_1:%.*]] = getelementptr i32, ptr [[SRC_1]], i64 [[TMP52]]
 ; PRED-NEXT:    [[TMP53]] = load i32, ptr [[GEP_1]], align 4
@@ -335,7 +323,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:  [[ENTRY:.*]]:
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; DEFAULT-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
@@ -343,8 +331,6 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; DEFAULT-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
 ; DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
 ; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -354,7 +340,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ]
 ; DEFAULT-NEXT:    [[TMP13:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
 ; DEFAULT-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 4
+; DEFAULT-NEXT:    [[TMP17:%.*]] = shl nuw i64 [[TMP16]], 2
 ; DEFAULT-NEXT:    [[TMP18:%.*]] = getelementptr i16, ptr [[TMP13]], i64 [[TMP17]]
 ; DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP13]], align 2
 ; DEFAULT-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
@@ -362,7 +348,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[TMP20:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD2]], [[BROADCAST_SPLAT]]
 ; DEFAULT-NEXT:    [[TMP21]] = or <vscale x 4 x i16> [[TMP19]], [[VEC_PHI]]
 ; DEFAULT-NEXT:    [[TMP22]] = or <vscale x 4 x i16> [[TMP20]], [[VEC_PHI1]]
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; DEFAULT-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
@@ -393,7 +379,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; VSCALEFORTUNING2-NEXT:  [[ENTRY:.*]]:
 ; VSCALEFORTUNING2-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; VSCALEFORTUNING2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; VSCALEFORTUNING2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; VSCALEFORTUNING2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VSCALEFORTUNING2:       [[VECTOR_PH]]:
@@ -401,8 +387,6 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; VSCALEFORTUNING2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; VSCALEFORTUNING2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; VSCALEFORTUNING2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; VSCALEFORTUNING2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
 ; VSCALEFORTUNING2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -412,7 +396,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; VSCALEFORTUNING2-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i16> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP16:%.*]], %[[VECTOR_BODY]] ]
 ; VSCALEFORTUNING2-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
+; VSCALEFORTUNING2-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 2
 ; VSCALEFORTUNING2-NEXT:    [[TMP12:%.*]] = getelementptr i16, ptr [[TMP8]], i64 [[TMP11]]
 ; VSCALEFORTUNING2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP8]], align 2
 ; VSCALEFORTUNING2-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i16>, ptr [[TMP12]], align 2
@@ -420,7 +404,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; VSCALEFORTUNING2-NEXT:    [[TMP14:%.*]] = udiv <vscale x 4 x i16> [[WIDE_LOAD2]], [[BROADCAST_SPLAT]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP15]] = or <vscale x 4 x i16> [[TMP13]], [[VEC_PHI]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP16]] = or <vscale x 4 x i16> [[TMP14]], [[VEC_PHI1]]
-; VSCALEFORTUNING2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; VSCALEFORTUNING2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; VSCALEFORTUNING2-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; VSCALEFORTUNING2:       [[MIDDLE_BLOCK]]:
@@ -448,20 +432,14 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ;
 ; PRED-LABEL: define i16 @reduce_udiv(
 ; PRED-SAME: ptr [[SRC:%.*]], i16 [[X:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; PRED-NEXT:  [[ENTRY:.*]]:
+; PRED-NEXT:  [[ENTRY:.*:]]
 ; PRED-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
-; PRED-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP2]], 1
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP5]]
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]]
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; PRED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; PRED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
+; PRED-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 3
 ; PRED-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP0]], [[TMP9]]
 ; PRED-NEXT:    [[TMP11:%.*]] = icmp ugt i64 [[TMP0]], [[TMP9]]
 ; PRED-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
@@ -478,21 +456,19 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; PRED-NEXT:    [[TMP20:%.*]] = udiv <vscale x 8 x i16> [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]]
 ; PRED-NEXT:    [[TMP21:%.*]] = or <vscale x 8 x i16> [[TMP20]], [[VEC_PHI]]
 ; PRED-NEXT:    [[TMP16]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x i16> [[TMP21]], <vscale x 8 x i16> [[VEC_PHI]]
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]]
+; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP12]])
-; PRED-NEXT:    [[TMP17:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; PRED-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 8 x i1> [[TMP17]], i32 0
-; PRED-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; PRED-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP15]], true
+; PRED-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; PRED:       [[MIDDLE_BLOCK]]:
 ; PRED-NEXT:    [[TMP19:%.*]] = call i16 @llvm.vector.reduce.or.nxv8i16(<vscale x 8 x i16> [[TMP16]])
 ; PRED-NEXT:    br label %[[EXIT:.*]]
 ; PRED:       [[SCALAR_PH]]:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; PRED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; PRED-NEXT:    br label %[[LOOP:.*]]
 ; PRED:       [[LOOP]]:
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; PRED-NEXT:    [[RED:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; PRED-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; PRED-NEXT:    [[RED:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
 ; PRED-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
 ; PRED-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
 ; PRED-NEXT:    [[DIV:%.*]] = udiv i16 [[L]], [[X]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
index eb3d724d224aa..2521ece2eea06 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
@@ -15,7 +15,7 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 [[TMP0]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[SMAX]], [[IDX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -23,16 +23,13 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i32 [[IDX]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[IDX]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[DOTSPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i32> [[TMP8]], splat (i32 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i32> [[DOTSPLAT]], [[TMP10]]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i32 1, [[TMP7]]
-; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP13]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP5]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 2 x i32> [[DOTSPLATINSERT1]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -44,7 +41,7 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x double>, ptr [[TMP16]], align 8
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr i64, ptr [[B:%.*]], i32 [[OFFSET_IDX]]
 ; CHECK-NEXT:    store <vscale x 2 x double> [[WIDE_LOAD]], ptr [[TMP18]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i32> [[VEC_IND]], [[DOTSPLAT2]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll
index a7ec749ff6b9f..92b2a44967cb8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll
@@ -6,7 +6,7 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture
 ; CHECK-LABEL: @cond_fadd(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -14,8 +14,6 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -28,7 +26,7 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture
 ; CHECK-NEXT:    [[TMP12:%.*]] = select fast <vscale x 4 x i1> [[TMP9]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> zeroinitializer
 ; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP14]] = fadd fast float [[VEC_PHI]], [[TMP13]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -90,7 +88,7 @@ define float @cond_cmp_sel(ptr noalias %a, ptr noalias %cond, i64 %N) {
 ; CHECK-LABEL: @cond_cmp_sel(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -98,8 +96,6 @@ define float @cond_cmp_sel(ptr noalias %a, ptr noalias %cond, i64 %N) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -113,7 +109,7 @@ define float @cond_cmp_sel(ptr noalias %a, ptr noalias %cond, i64 %N) {
 ; CHECK-NEXT:    [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt float [[TMP13]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT]] = select fast i1 [[RDX_MINMAX_CMP]], float [[TMP13]], float [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
index a60d35d407fb0..9583c7f174333 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
@@ -34,7 +34,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
@@ -42,44 +42,42 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP7]] = fadd <vscale x 8 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP4]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP5]] = fadd <vscale x 8 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[TMP7]])
+; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[TMP5]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD]] = fadd float [[TMP10]], [[SUM_07]]
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD]] = fadd float [[TMP8]], [[SUM_07]]
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_strict
 ; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
@@ -87,36 +85,34 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP7]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[WIDE_LOAD]])
-; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP4]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP5]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[WIDE_LOAD]])
+; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
 ; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-ORDERED-NEXT:    [[ADD]] = fadd float [[TMP9]], [[SUM_07]]
+; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-NEXT:    [[ADD]] = fadd float [[TMP7]], [[SUM_07]]
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fadd_strict
@@ -126,49 +122,41 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = sub i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = icmp ugt i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP4]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       vector.body:
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP10]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[WIDE_MASKED_LOAD]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP12]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP11]])
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 8 x i1> [[TMP13]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP7]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[WIDE_MASKED_LOAD]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP9]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP8]])
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP10]], true
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
-; CHECK-ORDERED-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       for.body:
-; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD]] = fadd float [[TMP15]], [[SUM_07]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD]] = fadd float [[TMP12]], [[SUM_07]]
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    ret float [[ADD_LCSSA]]
 ;
 
@@ -214,7 +202,7 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
@@ -222,65 +210,63 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP8]]
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP11]]
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP14]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP12]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP15]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP16]] = fadd <vscale x 8 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-UNORDERED-NEXT:    [[TMP17]] = fadd <vscale x 8 x float> [[WIDE_LOAD4]], [[VEC_PHI1]]
-; CHECK-UNORDERED-NEXT:    [[TMP18]] = fadd <vscale x 8 x float> [[WIDE_LOAD5]], [[VEC_PHI2]]
-; CHECK-UNORDERED-NEXT:    [[TMP19]] = fadd <vscale x 8 x float> [[WIDE_LOAD6]], [[VEC_PHI3]]
-; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 3
+; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP9]]
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP12]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP4]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP14]] = fadd <vscale x 8 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-UNORDERED-NEXT:    [[TMP15]] = fadd <vscale x 8 x float> [[WIDE_LOAD4]], [[VEC_PHI1]]
+; CHECK-UNORDERED-NEXT:    [[TMP16]] = fadd <vscale x 8 x float> [[WIDE_LOAD5]], [[VEC_PHI2]]
+; CHECK-UNORDERED-NEXT:    [[TMP17]] = fadd <vscale x 8 x float> [[WIDE_LOAD6]], [[VEC_PHI3]]
+; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd <vscale x 8 x float> [[TMP17]], [[TMP16]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX7:%.*]] = fadd <vscale x 8 x float> [[TMP18]], [[BIN_RDX]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX8:%.*]] = fadd <vscale x 8 x float> [[TMP19]], [[BIN_RDX7]]
-; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX8]])
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd <vscale x 8 x float> [[TMP15]], [[TMP14]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX7:%.*]] = fadd <vscale x 8 x float> [[TMP16]], [[BIN_RDX]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX8:%.*]] = fadd <vscale x 8 x float> [[TMP17]], [[BIN_RDX7]]
+; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX8]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP21]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD]] = fadd float [[TMP22]], [[SUM_07]]
+; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD]] = fadd float [[TMP20]], [[SUM_07]]
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP21]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_strict_unroll
 ; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
@@ -288,51 +274,49 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP8]]
-; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 16
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP11]]
-; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 24
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP14]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP12]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP15]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[WIDE_LOAD]])
-; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP16]], <vscale x 8 x float> [[WIDE_LOAD1]])
-; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP17]], <vscale x 8 x float> [[WIDE_LOAD2]])
-; CHECK-ORDERED-NEXT:    [[TMP19]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP18]], <vscale x 8 x float> [[WIDE_LOAD3]])
-; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 3
+; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 4
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP9]]
+; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 24
+; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP12]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP4]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[WIDE_LOAD]])
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP14]], <vscale x 8 x float> [[WIDE_LOAD1]])
+; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP15]], <vscale x 8 x float> [[WIDE_LOAD2]])
+; CHECK-ORDERED-NEXT:    [[TMP17]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP16]], <vscale x 8 x float> [[WIDE_LOAD3]])
+; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
 ; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-ORDERED-NEXT:    [[ADD]] = fadd float [[TMP21]], [[SUM_07]]
+; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-NEXT:    [[ADD]] = fadd float [[TMP19]], [[SUM_07]]
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    ret float [[ADD_LCSSA]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fadd_strict_unroll
@@ -342,26 +326,20 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP11]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 16
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP13]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP15]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 5
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = sub i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = icmp ugt i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP4]], i64 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 3
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP8]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 4
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP10]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 24
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP12]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_PART_NEXT]], i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_PART_NEXT1]], i64 [[N]])
@@ -373,63 +351,61 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT12:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT13:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT14:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP33:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP30:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 3
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP15]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP18]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP18]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP21]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP24]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP16]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP19]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP22]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP25]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[WIDE_MASKED_LOAD]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP26]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP27]], <vscale x 8 x float> [[TMP28]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP29]], <vscale x 8 x float> [[TMP30]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP33]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP31]], <vscale x 8 x float> [[TMP32]])
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP21]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP13]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP16]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP19]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP22]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[WIDE_MASKED_LOAD]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP23]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP24]], <vscale x 8 x float> [[TMP25]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP26]], <vscale x 8 x float> [[TMP27]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP30]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP28]], <vscale x 8 x float> [[TMP29]])
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = shl nuw i64 [[TMP31]], 3
+; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = add i64 [[INDEX]], [[TMP32]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = mul nuw i64 [[TMP34]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = shl nuw i64 [[TMP34]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = add i64 [[INDEX]], [[TMP35]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = mul nuw i64 [[TMP37]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = mul nuw i64 [[TMP37]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = add i64 [[INDEX]], [[TMP38]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = mul nuw i64 [[TMP40]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = add i64 [[INDEX]], [[TMP41]]
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP36]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP39]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP42]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP44:%.*]] = extractelement <vscale x 8 x i1> [[TMP43]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP44]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP33]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP36]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP39]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = xor i1 [[TMP40]], true
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP41]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
-; CHECK-ORDERED-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       for.body:
-; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD]] = fadd float [[TMP45]], [[SUM_07]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD]] = fadd float [[TMP42]], [[SUM_07]]
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP33]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP30]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    ret float [[ADD_LCSSA]]
 ;
 
@@ -492,7 +468,7 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
 ; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
@@ -500,54 +476,52 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 2
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A2]], i32 0
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A1]], i32 0
+; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = mul i64 [[N_VEC]], 2
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A2]], i32 0
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A1]], i32 0
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ [[TMP10]], [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x float> [ [[TMP11]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ [[TMP8]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x float> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OFFSET_IDX]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP12]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
 ; CHECK-UNORDERED-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
-; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
-; CHECK-UNORDERED-NEXT:    [[TMP15]] = fadd <vscale x 4 x float> [[TMP13]], [[VEC_PHI1]]
-; CHECK-UNORDERED-NEXT:    [[TMP16]] = fadd <vscale x 4 x float> [[TMP14]], [[VEC_PHI]]
-; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
-; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-UNORDERED-NEXT:    [[TMP13]] = fadd <vscale x 4 x float> [[TMP11]], [[VEC_PHI1]]
+; CHECK-UNORDERED-NEXT:    [[TMP14]] = fadd <vscale x 4 x float> [[TMP12]], [[VEC_PHI]]
+; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP16]])
-; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP15]])
+; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP14]])
+; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP13]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY:%.*]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[ADD_PHI1:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ADD_PHI2:%.*]] = phi float [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDXB1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD1]] = fadd float [[TMP20]], [[ADD_PHI2]]
+; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD1]] = fadd float [[TMP18]], [[ADD_PHI2]]
 ; CHECK-UNORDERED-NEXT:    [[OR:%.*]] = or disjoint i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDXB2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OR]]
-; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD2]] = fadd float [[TMP21]], [[ADD_PHI1]]
+; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD2]] = fadd float [[TMP19]], [[ADD_PHI1]]
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
-; CHECK-UNORDERED-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    store float [[ADD1_LCSSA]], ptr [[A]], align 4
 ; CHECK-UNORDERED-NEXT:    store float [[ADD2_LCSSA]], ptr [[ARRAYIDXA]], align 4
 ; CHECK-UNORDERED-NEXT:    ret void
@@ -562,7 +536,7 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
 ; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
@@ -570,50 +544,48 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 2
+; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = mul i64 [[N_VEC]], 2
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OFFSET_IDX]]
-; CHECK-ORDERED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-ORDERED-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x float>, ptr [[TMP8]], align 4
 ; CHECK-ORDERED-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_VEC]])
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
-; CHECK-ORDERED-NEXT:    [[TMP13]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP12]])
-; CHECK-ORDERED-NEXT:    [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP11]])
-; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-ORDERED-NEXT:    [[TMP11]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP10]])
+; CHECK-ORDERED-NEXT:    [[TMP12]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP9]])
+; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP13]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY:%.*]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP14]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
-; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ [[A2]], [[ENTRY:%.*]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ [[A1]], [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[ADD_PHI1:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ADD_PHI2:%.*]] = phi float [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDXB1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
-; CHECK-ORDERED-NEXT:    [[ADD1]] = fadd float [[TMP16]], [[ADD_PHI2]]
+; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
+; CHECK-ORDERED-NEXT:    [[ADD1]] = fadd float [[TMP14]], [[ADD_PHI2]]
 ; CHECK-ORDERED-NEXT:    [[OR:%.*]] = or disjoint i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDXB2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OR]]
-; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
-; CHECK-ORDERED-NEXT:    [[ADD2]] = fadd float [[TMP17]], [[ADD_PHI1]]
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
+; CHECK-ORDERED-NEXT:    [[ADD2]] = fadd float [[TMP15]], [[ADD_PHI1]]
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]
-; CHECK-ORDERED-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    store float [[ADD1_LCSSA]], ptr [[A]], align 4
 ; CHECK-ORDERED-NEXT:    store float [[ADD2_LCSSA]], ptr [[ARRAYIDXA]], align 4
 ; CHECK-ORDERED-NEXT:    ret void
@@ -631,64 +603,55 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP4]], 1
-; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP2]], [[TMP5]]
-; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = icmp ugt i64 [[TMP2]], [[TMP9]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
+; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP2]], [[TMP6]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[TMP2]], [[TMP6]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[TMP2]])
 ; CHECK-ORDERED-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       vector.body:
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[A2]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI1:%.*]] = phi float [ [[A1]], [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OFFSET_IDX]]
 ; CHECK-ORDERED-TF-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP13]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP10]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x float> poison)
 ; CHECK-ORDERED-TF-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> [[WIDE_MASKED_VEC]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP15]], <vscale x 4 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP17]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP16]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP19]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP18]])
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]]
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP12]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[TMP20]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } [[STRIDED_VEC]], 1
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP12]], <vscale x 4 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP13]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP11]], <vscale x 4 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP16]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], <vscale x 4 x float> [[TMP15]])
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP17]], true
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[A2]], [[ENTRY:%.*]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX2:%.*]] = phi float [ [[A1]], [[ENTRY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY]] ]
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       for.body:
-; CHECK-ORDERED-TF-NEXT:    [[ADD_PHI1:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[ADD_PHI2:%.*]] = phi float [ [[BC_MERGE_RDX2]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD_PHI1:%.*]] = phi float [ [[A2]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD_PHI2:%.*]] = phi float [ [[A1]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDXB1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD1]] = fadd float [[TMP22]], [[ADD_PHI2]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDXB1]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD1]] = fadd float [[TMP19]], [[ADD_PHI2]]
 ; CHECK-ORDERED-TF-NEXT:    [[OR:%.*]] = or disjoint i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDXB2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[OR]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD2]] = fadd float [[TMP23]], [[ADD_PHI1]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDXB2]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD2]] = fadd float [[TMP20]], [[ADD_PHI1]]
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 2
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
-; CHECK-ORDERED-TF-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD1_LCSSA:%.*]] = phi float [ [[ADD1]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[ADD2_LCSSA:%.*]] = phi float [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    store float [[ADD1_LCSSA]], ptr [[A]], align 4
 ; CHECK-ORDERED-TF-NEXT:    store float [[ADD2_LCSSA]], ptr [[ARRAYIDXA]], align 4
 ; CHECK-ORDERED-TF-NEXT:    ret void
@@ -762,7 +725,7 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
 ; CHECK-UNORDERED:       for.body.preheader:
 ; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
@@ -770,43 +733,41 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = fadd <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-UNORDERED-NEXT:    [[TMP10]] = fadd <vscale x 4 x float> [[VEC_PHI]], [[TMP9]]
-; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = fadd <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-UNORDERED-NEXT:    [[TMP8]] = fadd <vscale x 4 x float> [[VEC_PHI]], [[TMP7]]
+; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP10]])
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP8]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[FOR_BODY_PREHEADER]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-UNORDERED-NEXT:    [[RES_014:%.*]] = phi float [ [[RDX:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD:%.*]] = fadd float [[TMP13]], [[TMP14]]
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD:%.*]] = fadd float [[TMP11]], [[TMP12]]
 ; CHECK-UNORDERED-NEXT:    [[RDX]] = fadd float [[RES_014]], [[ADD]]
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK-UNORDERED:       for.end.loopexit:
-; CHECK-UNORDERED-NEXT:    [[RDX_LCSSA:%.*]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[RDX_LCSSA:%.*]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_END]]
 ; CHECK-UNORDERED:       for.end:
 ; CHECK-UNORDERED-NEXT:    [[RES:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[RDX_LCSSA]], [[FOR_END_LOOPEXIT]] ]
@@ -821,7 +782,7 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
 ; CHECK-ORDERED:       for.body.preheader:
 ; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
@@ -829,42 +790,40 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = fadd <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; CHECK-ORDERED-NEXT:    [[TMP10]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP9]])
-; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP5]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = fadd <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
+; CHECK-ORDERED-NEXT:    [[TMP8]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP7]])
+; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END_LOOPEXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
 ; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[FOR_BODY_PREHEADER]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
 ; CHECK-ORDERED-NEXT:    [[RES_014:%.*]] = phi float [ [[RDX:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
-; CHECK-ORDERED-NEXT:    [[ADD:%.*]] = fadd float [[TMP12]], [[TMP13]]
+; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-ORDERED-NEXT:    [[ADD:%.*]] = fadd float [[TMP10]], [[TMP11]]
 ; CHECK-ORDERED-NEXT:    [[RDX]] = fadd float [[RES_014]], [[ADD]]
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK-ORDERED:       for.end.loopexit:
-; CHECK-ORDERED-NEXT:    [[RDX_LCSSA:%.*]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[TMP10]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[RDX_LCSSA:%.*]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_END]]
 ; CHECK-ORDERED:       for.end:
 ; CHECK-ORDERED-NEXT:    [[RES:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[RDX_LCSSA]], [[FOR_END_LOOPEXIT]] ]
@@ -882,55 +841,47 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP2]], 1
-; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP3]]
-; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = sub i64 [[N]], [[TMP7]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = icmp ugt i64 [[N]], [[TMP7]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
+; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], [[TMP4]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = icmp ugt i64 [[N]], [[TMP4]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = select i1 [[TMP6]], i64 [[TMP5]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       vector.body:
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = fadd <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP13]], <vscale x 4 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP15]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP14]])
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]]
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP10]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[TMP16]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP8]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP9]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = fadd <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[TMP10]], <vscale x 4 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP12]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP11]])
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP2]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP7]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END_LOOPEXIT:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
-; CHECK-ORDERED-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       for.body:
-; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-ORDERED-TF-NEXT:    [[RES_014:%.*]] = phi float [ [[RDX:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-ORDERED-TF-NEXT:    [[RES_014:%.*]] = phi float [ [[RDX:%.*]], [[FOR_BODY]] ], [ 0.000000e+00, [[SCALAR_PH]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[ADD:%.*]] = fadd float [[TMP18]], [[TMP19]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX4]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[ADD:%.*]] = fadd float [[TMP15]], [[TMP16]]
 ; CHECK-ORDERED-TF-NEXT:    [[RDX]] = fadd float [[RES_014]], [[ADD]]
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end.loopexit:
-; CHECK-ORDERED-TF-NEXT:    [[RDX_LCSSA:%.*]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[RDX_LCSSA:%.*]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END]]
 ; CHECK-ORDERED-TF:       for.end:
 ; CHECK-ORDERED-TF-NEXT:    [[RES:%.*]] = phi float [ 0.000000e+00, [[ENTRY:%.*]] ], [ [[RDX_LCSSA]], [[FOR_END_LOOPEXIT]] ]
@@ -994,7 +945,7 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
@@ -1002,56 +953,54 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float 1.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = fcmp une <vscale x 4 x float> [[WIDE_LOAD]], zeroinitializer
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP8]], i32 4, <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> poison)
-; CHECK-UNORDERED-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> splat (float 3.000000e+00)
-; CHECK-UNORDERED-NEXT:    [[TMP9]] = fadd <vscale x 4 x float> [[VEC_PHI]], [[PREDPHI]]
-; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float 1.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP4]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = fcmp une <vscale x 4 x float> [[WIDE_LOAD]], zeroinitializer
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP6]], i32 4, <vscale x 4 x i1> [[TMP5]], <vscale x 4 x float> poison)
+; CHECK-UNORDERED-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP5]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> splat (float 3.000000e+00)
+; CHECK-UNORDERED-NEXT:    [[TMP7]] = fadd <vscale x 4 x float> [[VEC_PHI]], [[PREDPHI]]
+; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP9]])
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP7]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-UNORDERED-NEXT:    [[RES:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[FADD:%.*]], [[FOR_INC]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-UNORDERED-NEXT:    [[TOBOOL:%.*]] = fcmp une float [[TMP12]], 0.000000e+00
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-UNORDERED-NEXT:    [[TOBOOL:%.*]] = fcmp une float [[TMP10]], 0.000000e+00
 ; CHECK-UNORDERED-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; CHECK-UNORDERED:       if.then:
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_INC]]
 ; CHECK-UNORDERED:       for.inc:
-; CHECK-UNORDERED-NEXT:    [[PHI:%.*]] = phi float [ [[TMP13]], [[IF_THEN]] ], [ 3.000000e+00, [[FOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[PHI:%.*]] = phi float [ [[TMP11]], [[IF_THEN]] ], [ 3.000000e+00, [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[FADD]] = fadd float [[RES]], [[PHI]]
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[RDX:%.*]] = phi float [ [[FADD]], [[FOR_INC]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[RDX:%.*]] = phi float [ [[FADD]], [[FOR_INC]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    ret float [[RDX]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_conditional
 ; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
@@ -1059,48 +1008,46 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = fcmp une <vscale x 4 x float> [[WIDE_LOAD]], zeroinitializer
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP8]], i32 4, <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> poison)
-; CHECK-ORDERED-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> splat (float 3.000000e+00)
-; CHECK-ORDERED-NEXT:    [[TMP9]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[PREDPHI]])
-; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP4]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = fcmp une <vscale x 4 x float> [[WIDE_LOAD]], zeroinitializer
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP6]], i32 4, <vscale x 4 x i1> [[TMP5]], <vscale x 4 x float> poison)
+; CHECK-ORDERED-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP5]], <vscale x 4 x float> [[WIDE_MASKED_LOAD]], <vscale x 4 x float> splat (float 3.000000e+00)
+; CHECK-ORDERED-NEXT:    [[TMP7]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[PREDPHI]])
+; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
 ; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-ORDERED-NEXT:    [[RES:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[FADD:%.*]], [[FOR_INC]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-ORDERED-NEXT:    [[TOBOOL:%.*]] = fcmp une float [[TMP11]], 0.000000e+00
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-NEXT:    [[TOBOOL:%.*]] = fcmp une float [[TMP9]], 0.000000e+00
 ; CHECK-ORDERED-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; CHECK-ORDERED:       if.then:
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-ORDERED-NEXT:    br label [[FOR_INC]]
 ; CHECK-ORDERED:       for.inc:
-; CHECK-ORDERED-NEXT:    [[PHI:%.*]] = phi float [ [[TMP12]], [[IF_THEN]] ], [ 3.000000e+00, [[FOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[PHI:%.*]] = phi float [ [[TMP10]], [[IF_THEN]] ], [ 3.000000e+00, [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[FADD]] = fadd float [[RES]], [[PHI]]
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[RDX:%.*]] = phi float [ [[FADD]], [[FOR_INC]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[RDX:%.*]] = phi float [ [[FADD]], [[FOR_INC]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    ret float [[RDX]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fadd_conditional
@@ -1110,62 +1057,54 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = sub i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = icmp ugt i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP4]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       vector.body:
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP10]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = fcmp une <vscale x 4 x float> [[WIDE_MASKED_LOAD]], zeroinitializer
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> zeroinitializer
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP13]], i32 4, <vscale x 4 x i1> [[TMP12]], <vscale x 4 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP12]], <vscale x 4 x float> [[WIDE_MASKED_LOAD1]], <vscale x 4 x float> splat (float 3.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[PREDPHI]], <vscale x 4 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP15]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP14]])
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[TMP16]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP7]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = fcmp une <vscale x 4 x float> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i1> [[TMP8]], <vscale x 4 x i1> zeroinitializer
+; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = getelementptr float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP10]], i32 4, <vscale x 4 x i1> [[TMP9]], <vscale x 4 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x float> [[WIDE_MASKED_LOAD1]], <vscale x 4 x float> splat (float 3.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> [[PREDPHI]], <vscale x 4 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP12]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP11]])
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
-; CHECK-ORDERED-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 1.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       for.body:
-; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-ORDERED-TF-NEXT:    [[RES:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[FADD:%.*]], [[FOR_INC]] ]
+; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-ORDERED-TF-NEXT:    [[RES:%.*]] = phi float [ 1.000000e+00, [[SCALAR_PH]] ], [ [[FADD:%.*]], [[FOR_INC]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[TOBOOL:%.*]] = fcmp une float [[TMP18]], 0.000000e+00
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[TOBOOL:%.*]] = fcmp une float [[TMP15]], 0.000000e+00
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; CHECK-ORDERED-TF:       if.then:
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_INC]]
 ; CHECK-ORDERED-TF:       for.inc:
-; CHECK-ORDERED-TF-NEXT:    [[PHI:%.*]] = phi float [ [[TMP19]], [[IF_THEN]] ], [ 3.000000e+00, [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[PHI:%.*]] = phi float [ [[TMP16]], [[IF_THEN]] ], [ 3.000000e+00, [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[FADD]] = fadd float [[RES]], [[PHI]]
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[RDX:%.*]] = phi float [ [[FADD]], [[FOR_INC]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[RDX:%.*]] = phi float [ [[FADD]], [[FOR_INC]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    ret float [[RDX]]
 ;
 
@@ -1226,7 +1165,7 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b,
 ; CHECK-UNORDERED-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
@@ -1234,43 +1173,41 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b,
 ; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float -0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = fadd <vscale x 8 x float> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP8]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP9]] = fadd <vscale x 8 x float> [[TMP7]], [[WIDE_LOAD1]]
-; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float -0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP4]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = fadd <vscale x 8 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP7]] = fadd <vscale x 8 x float> [[TMP5]], [[WIDE_LOAD1]]
+; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[TMP9]])
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[TMP7]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ -0.000000e+00, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ -0.000000e+00, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[SUM:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD3:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD:%.*]] = fadd float [[SUM]], [[TMP12]]
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD:%.*]] = fadd float [[SUM]], [[TMP10]]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-UNORDERED-NEXT:    [[ADD3]] = fadd float [[ADD]], [[TMP13]]
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-UNORDERED-NEXT:    [[ADD3]] = fadd float [[ADD]], [[TMP11]]
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[RDX:%.*]] = phi float [ [[ADD3]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[RDX:%.*]] = phi float [ [[ADD3]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    ret float [[RDX]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fadd_multiple
@@ -1363,7 +1300,7 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
@@ -1371,81 +1308,79 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP8]]
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP11]]
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP14]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP12]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP15]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP18]]
-; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP21]]
-; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP24]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP19]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 8 x float>, ptr [[TMP25]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP26]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD]], <vscale x 8 x float> [[WIDE_LOAD7]], <vscale x 8 x float> [[VEC_PHI]])
-; CHECK-UNORDERED-NEXT:    [[TMP27]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD4]], <vscale x 8 x float> [[WIDE_LOAD8]], <vscale x 8 x float> [[VEC_PHI1]])
-; CHECK-UNORDERED-NEXT:    [[TMP28]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD5]], <vscale x 8 x float> [[WIDE_LOAD9]], <vscale x 8 x float> [[VEC_PHI2]])
-; CHECK-UNORDERED-NEXT:    [[TMP29]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD6]], <vscale x 8 x float> [[WIDE_LOAD10]], <vscale x 8 x float> [[VEC_PHI3]])
-; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-UNORDERED-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 3
+; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP9]]
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP12]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP4]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 3
+; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP16]]
+; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP19]]
+; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP22]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 8 x float>, ptr [[TMP20]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 8 x float>, ptr [[TMP23]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP24]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD]], <vscale x 8 x float> [[WIDE_LOAD7]], <vscale x 8 x float> [[VEC_PHI]])
+; CHECK-UNORDERED-NEXT:    [[TMP25]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD4]], <vscale x 8 x float> [[WIDE_LOAD8]], <vscale x 8 x float> [[VEC_PHI1]])
+; CHECK-UNORDERED-NEXT:    [[TMP26]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD5]], <vscale x 8 x float> [[WIDE_LOAD9]], <vscale x 8 x float> [[VEC_PHI2]])
+; CHECK-UNORDERED-NEXT:    [[TMP27]] = call <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD6]], <vscale x 8 x float> [[WIDE_LOAD10]], <vscale x 8 x float> [[VEC_PHI3]])
+; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-UNORDERED-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd <vscale x 8 x float> [[TMP27]], [[TMP26]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX11:%.*]] = fadd <vscale x 8 x float> [[TMP28]], [[BIN_RDX]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX12:%.*]] = fadd <vscale x 8 x float> [[TMP29]], [[BIN_RDX11]]
-; CHECK-UNORDERED-NEXT:    [[TMP31:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX12]])
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd <vscale x 8 x float> [[TMP25]], [[TMP24]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX11:%.*]] = fadd <vscale x 8 x float> [[TMP26]], [[BIN_RDX]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX12:%.*]] = fadd <vscale x 8 x float> [[TMP27]], [[BIN_RDX11]]
+; CHECK-UNORDERED-NEXT:    [[TMP29:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX12]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP31]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP32:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP30:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP33:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-UNORDERED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP32]], float [[TMP33]], float [[SUM_07]])
+; CHECK-UNORDERED-NEXT:    [[TMP31:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-UNORDERED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP30]], float [[TMP31]], float [[SUM_07]])
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP31]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fmuladd_strict
 ; CHECK-ORDERED-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
@@ -1453,71 +1388,69 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP33:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP8]]
-; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 16
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP11]]
-; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 24
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP14]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP12]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP15]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 8
-; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP18]]
-; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 16
-; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP21]]
-; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 24
-; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP24]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP19]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP25]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
-; CHECK-ORDERED-NEXT:    [[TMP28:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
-; CHECK-ORDERED-NEXT:    [[TMP29:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
-; CHECK-ORDERED-NEXT:    [[TMP30:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP26]])
-; CHECK-ORDERED-NEXT:    [[TMP31:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP30]], <vscale x 8 x float> [[TMP27]])
-; CHECK-ORDERED-NEXT:    [[TMP32:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP31]], <vscale x 8 x float> [[TMP28]])
-; CHECK-ORDERED-NEXT:    [[TMP33]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP32]], <vscale x 8 x float> [[TMP29]])
-; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-ORDERED-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP34]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP31:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 3
+; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 4
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP9]]
+; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 24
+; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP12]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP4]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 3
+; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP16]]
+; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 4
+; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP19]]
+; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 24
+; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP22]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP20]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP23]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
+; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
+; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
+; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = fmul <vscale x 8 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
+; CHECK-ORDERED-NEXT:    [[TMP28:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP24]])
+; CHECK-ORDERED-NEXT:    [[TMP29:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP28]], <vscale x 8 x float> [[TMP25]])
+; CHECK-ORDERED-NEXT:    [[TMP30:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP29]], <vscale x 8 x float> [[TMP26]])
+; CHECK-ORDERED-NEXT:    [[TMP31]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP30]], <vscale x 8 x float> [[TMP27]])
+; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-ORDERED-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
 ; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP33]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP31]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP35:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP33:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP36:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-ORDERED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP35]], float [[TMP36]], float [[SUM_07]])
+; CHECK-ORDERED-NEXT:    [[TMP34:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP33]], float [[TMP34]], float [[SUM_07]])
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP33]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP31]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fmuladd_strict
@@ -1527,26 +1460,20 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP11]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 16
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP13]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP15]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 5
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = sub i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = icmp ugt i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP4]], i64 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 3
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP8]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 4
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP10]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 24
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP12]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_PART_NEXT]], i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_PART_NEXT1]], i64 [[N]])
@@ -1558,83 +1485,81 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT17:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP44:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 3
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP15]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP18]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP18]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP21]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP24]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP16]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP19]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP22]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP25]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP21]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP13]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP16]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP19]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP22]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 3
+; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[TMP25]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[TMP28]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = shl nuw i64 [[TMP27]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[TMP28]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[TMP31]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = mul nuw i64 [[TMP33]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[TMP34]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP26]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP29]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP32]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP35]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD12]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD13]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], [[WIDE_MASKED_LOAD14]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], [[WIDE_MASKED_LOAD15]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[TMP36]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP40]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP37]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP41]], <vscale x 8 x float> [[TMP42]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP44:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[TMP38]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP43]], <vscale x 8 x float> [[TMP44]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[TMP39]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP47]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP45]], <vscale x 8 x float> [[TMP46]])
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[TMP31]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP23]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP26]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP29]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP32]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD12]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD13]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], [[WIDE_MASKED_LOAD14]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = fmul <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], [[WIDE_MASKED_LOAD15]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[TMP33]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP37]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP34]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP38]], <vscale x 8 x float> [[TMP39]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[TMP35]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP40]], <vscale x 8 x float> [[TMP41]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = select <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[TMP36]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP44]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP42]], <vscale x 8 x float> [[TMP43]])
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = shl nuw i64 [[TMP45]], 3
+; CHECK-ORDERED-TF-NEXT:    [[TMP47:%.*]] = add i64 [[INDEX]], [[TMP46]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP48:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP49:%.*]] = mul nuw i64 [[TMP48]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP49:%.*]] = shl nuw i64 [[TMP48]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP50:%.*]] = add i64 [[INDEX]], [[TMP49]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = mul nuw i64 [[TMP51]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = mul nuw i64 [[TMP51]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP53:%.*]] = add i64 [[INDEX]], [[TMP52]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = mul nuw i64 [[TMP54]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = add i64 [[INDEX]], [[TMP55]]
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP50]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP53]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP56]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP58:%.*]] = extractelement <vscale x 8 x i1> [[TMP57]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP58]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP47]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP50]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP53]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = xor i1 [[TMP54]], true
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP55]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
-; CHECK-ORDERED-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       for.body:
-; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP59:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP60:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP59]], float [[TMP60]], float [[SUM_07]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP56]], float [[TMP57]], float [[SUM_07]])
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP47]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP44]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 
@@ -1685,7 +1610,7 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
@@ -1693,81 +1618,79 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[VECTOR_BODY]] ]
-; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP8]]
-; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP11]]
-; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP14]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP12]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP15]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 8
-; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP18]]
-; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 16
-; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP21]]
-; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 24
-; CHECK-UNORDERED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP24]]
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP19]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
-; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 8 x float>, ptr [[TMP25]], align 4
-; CHECK-UNORDERED-NEXT:    [[TMP26]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD]], <vscale x 8 x float> [[WIDE_LOAD7]], <vscale x 8 x float> [[VEC_PHI]])
-; CHECK-UNORDERED-NEXT:    [[TMP27]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD4]], <vscale x 8 x float> [[WIDE_LOAD8]], <vscale x 8 x float> [[VEC_PHI1]])
-; CHECK-UNORDERED-NEXT:    [[TMP28]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD5]], <vscale x 8 x float> [[WIDE_LOAD9]], <vscale x 8 x float> [[VEC_PHI2]])
-; CHECK-UNORDERED-NEXT:    [[TMP29]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD6]], <vscale x 8 x float> [[WIDE_LOAD10]], <vscale x 8 x float> [[VEC_PHI3]])
-; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-UNORDERED-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-UNORDERED-NEXT:    br i1 [[TMP30]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x float> [ insertelement (<vscale x 8 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP25:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 8 x float> [ splat (float -0.000000e+00), [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[VECTOR_BODY]] ]
+; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 3
+; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP9]]
+; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP12]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP4]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 3
+; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP16]]
+; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP19]]
+; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP22]]
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 8 x float>, ptr [[TMP20]], align 4
+; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 8 x float>, ptr [[TMP23]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP24]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD]], <vscale x 8 x float> [[WIDE_LOAD7]], <vscale x 8 x float> [[VEC_PHI]])
+; CHECK-UNORDERED-NEXT:    [[TMP25]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD4]], <vscale x 8 x float> [[WIDE_LOAD8]], <vscale x 8 x float> [[VEC_PHI1]])
+; CHECK-UNORDERED-NEXT:    [[TMP26]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD5]], <vscale x 8 x float> [[WIDE_LOAD9]], <vscale x 8 x float> [[VEC_PHI2]])
+; CHECK-UNORDERED-NEXT:    [[TMP27]] = call nnan <vscale x 8 x float> @llvm.fmuladd.nxv8f32(<vscale x 8 x float> [[WIDE_LOAD6]], <vscale x 8 x float> [[WIDE_LOAD10]], <vscale x 8 x float> [[VEC_PHI3]])
+; CHECK-UNORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-UNORDERED-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-UNORDERED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-UNORDERED:       middle.block:
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd nnan <vscale x 8 x float> [[TMP27]], [[TMP26]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX11:%.*]] = fadd nnan <vscale x 8 x float> [[TMP28]], [[BIN_RDX]]
-; CHECK-UNORDERED-NEXT:    [[BIN_RDX12:%.*]] = fadd nnan <vscale x 8 x float> [[TMP29]], [[BIN_RDX11]]
-; CHECK-UNORDERED-NEXT:    [[TMP31:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX12]])
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX:%.*]] = fadd nnan <vscale x 8 x float> [[TMP25]], [[TMP24]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX11:%.*]] = fadd nnan <vscale x 8 x float> [[TMP26]], [[BIN_RDX]]
+; CHECK-UNORDERED-NEXT:    [[BIN_RDX12:%.*]] = fadd nnan <vscale x 8 x float> [[TMP27]], [[BIN_RDX11]]
+; CHECK-UNORDERED-NEXT:    [[TMP29:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float -0.000000e+00, <vscale x 8 x float> [[BIN_RDX12]])
 ; CHECK-UNORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-UNORDERED:       scalar.ph:
 ; CHECK-UNORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP31]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-UNORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-UNORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-UNORDERED:       for.body:
 ; CHECK-UNORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP32:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-UNORDERED-NEXT:    [[TMP30:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-UNORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-UNORDERED-NEXT:    [[TMP33:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-UNORDERED-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP32]], float [[TMP33]], float [[SUM_07]])
+; CHECK-UNORDERED-NEXT:    [[TMP31:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-UNORDERED-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP30]], float [[TMP31]], float [[SUM_07]])
 ; CHECK-UNORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-UNORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK-UNORDERED:       for.end:
-; CHECK-UNORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP31]], [[MIDDLE_BLOCK]] ]
+; CHECK-UNORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ]
 ; CHECK-UNORDERED-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 ; CHECK-ORDERED-LABEL: define float @fmuladd_strict_fmf
 ; CHECK-ORDERED-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
@@ -1775,71 +1698,69 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP33:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP8]]
-; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 16
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP11]]
-; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 24
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP14]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP6]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP9]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP12]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP15]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 8
-; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP18]]
-; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 16
-; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP21]]
-; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 24
-; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP24]]
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP16]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP19]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP22]], align 4
-; CHECK-ORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP25]], align 4
-; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
-; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
-; CHECK-ORDERED-NEXT:    [[TMP28:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
-; CHECK-ORDERED-NEXT:    [[TMP29:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
-; CHECK-ORDERED-NEXT:    [[TMP30:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP26]])
-; CHECK-ORDERED-NEXT:    [[TMP31:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP30]], <vscale x 8 x float> [[TMP27]])
-; CHECK-ORDERED-NEXT:    [[TMP32:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP31]], <vscale x 8 x float> [[TMP28]])
-; CHECK-ORDERED-NEXT:    [[TMP33]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP32]], <vscale x 8 x float> [[TMP29]])
-; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-ORDERED-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-ORDERED-NEXT:    br i1 [[TMP34]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-ORDERED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP31:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 3
+; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP6]]
+; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 4
+; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP9]]
+; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 24
+; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 [[TMP12]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP4]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x float>, ptr [[TMP13]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 3
+; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP16]]
+; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 4
+; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP19]]
+; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 24
+; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[TMP22]]
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP14]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP17]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 8 x float>, ptr [[TMP20]], align 4
+; CHECK-ORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP23]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD]], [[WIDE_LOAD4]]
+; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD1]], [[WIDE_LOAD5]]
+; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
+; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
+; CHECK-ORDERED-NEXT:    [[TMP28:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP24]])
+; CHECK-ORDERED-NEXT:    [[TMP29:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP28]], <vscale x 8 x float> [[TMP25]])
+; CHECK-ORDERED-NEXT:    [[TMP30:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP29]], <vscale x 8 x float> [[TMP26]])
+; CHECK-ORDERED-NEXT:    [[TMP31]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP30]], <vscale x 8 x float> [[TMP27]])
+; CHECK-ORDERED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-ORDERED-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-ORDERED-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-ORDERED:       middle.block:
 ; CHECK-ORDERED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK-ORDERED:       scalar.ph:
 ; CHECK-ORDERED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP33]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-ORDERED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP31]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED:       for.body:
 ; CHECK-ORDERED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP35:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-NEXT:    [[TMP33:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-ORDERED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-NEXT:    [[TMP36:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-ORDERED-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP35]], float [[TMP36]], float [[SUM_07]])
+; CHECK-ORDERED-NEXT:    [[TMP34:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP33]], float [[TMP34]], float [[SUM_07]])
 ; CHECK-ORDERED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-ORDERED:       for.end:
-; CHECK-ORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP33]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP31]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 ; CHECK-ORDERED-TF-LABEL: define float @fmuladd_strict_fmf
@@ -1849,26 +1770,20 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
-; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 32
-; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
-; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP11]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 16
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP13]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP15]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 5
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = sub i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = icmp ugt i64 [[N]], [[TMP3]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP4]], i64 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 3
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP8]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 4
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP10]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 24
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP12]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_PART_NEXT]], i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY4:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_PART_NEXT1]], i64 [[N]])
@@ -1880,83 +1795,81 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK6:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY3]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK7:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY4]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT17:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK8:%.*]] = phi <vscale x 8 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT18:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP47:%.*]], [[VECTOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP44:%.*]], [[VECTOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 3
+; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP15]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP18]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP18]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP21]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP24]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP16]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP19]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP22]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP25]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i64 [[TMP21]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP13]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP16]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP19]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD11:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP22]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 3
+; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[TMP25]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 8
-; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[TMP28]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = shl nuw i64 [[TMP27]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[TMP28]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 16
-; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[TMP31]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = mul nuw i64 [[TMP33]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[TMP34]]
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP26]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP29]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP32]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP35]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
-; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD12]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD13]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], [[WIDE_MASKED_LOAD14]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], [[WIDE_MASKED_LOAD15]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[TMP36]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP40]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP37]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP41]], <vscale x 8 x float> [[TMP42]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP44:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[TMP38]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP43]], <vscale x 8 x float> [[TMP44]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[TMP39]], <vscale x 8 x float> splat (float -0.000000e+00)
-; CHECK-ORDERED-TF-NEXT:    [[TMP47]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP45]], <vscale x 8 x float> [[TMP46]])
-; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[TMP31]]
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP23]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP26]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP29]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP32]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> poison)
+; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD12]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD9]], [[WIDE_MASKED_LOAD13]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD10]], [[WIDE_MASKED_LOAD14]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = fmul nnan <vscale x 8 x float> [[WIDE_MASKED_LOAD11]], [[WIDE_MASKED_LOAD15]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> [[TMP33]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], <vscale x 8 x float> [[TMP37]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> [[TMP34]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP38]], <vscale x 8 x float> [[TMP39]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 8 x float> [[TMP35]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP40]], <vscale x 8 x float> [[TMP41]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = select nnan <vscale x 8 x i1> [[ACTIVE_LANE_MASK8]], <vscale x 8 x float> [[TMP36]], <vscale x 8 x float> splat (float -0.000000e+00)
+; CHECK-ORDERED-TF-NEXT:    [[TMP44]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP42]], <vscale x 8 x float> [[TMP43]])
+; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-ORDERED-TF-NEXT:    [[TMP45:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-ORDERED-TF-NEXT:    [[TMP46:%.*]] = shl nuw i64 [[TMP45]], 3
+; CHECK-ORDERED-TF-NEXT:    [[TMP47:%.*]] = add i64 [[INDEX]], [[TMP46]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP48:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP49:%.*]] = mul nuw i64 [[TMP48]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP49:%.*]] = shl nuw i64 [[TMP48]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP50:%.*]] = add i64 [[INDEX]], [[TMP49]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = mul nuw i64 [[TMP51]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = mul nuw i64 [[TMP51]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP53:%.*]] = add i64 [[INDEX]], [[TMP52]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = mul nuw i64 [[TMP54]], 24
-; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = add i64 [[INDEX]], [[TMP55]]
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP50]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP53]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP56]], i64 [[TMP9]])
-; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = xor <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-ORDERED-TF-NEXT:    [[TMP58:%.*]] = extractelement <vscale x 8 x i1> [[TMP57]], i32 0
-; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP58]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP47]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT17]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP50]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT18]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP53]], i64 [[TMP6]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = extractelement <vscale x 8 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = xor i1 [[TMP54]], true
+; CHECK-ORDERED-TF-NEXT:    br i1 [[TMP55]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK-ORDERED-TF:       middle.block:
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-ORDERED-TF:       scalar.ph:
-; CHECK-ORDERED-TF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-ORDERED-TF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ORDERED-TF-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-ORDERED-TF:       for.body:
-; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-ORDERED-TF-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP59:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-ORDERED-TF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
-; CHECK-ORDERED-TF-NEXT:    [[TMP60:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
-; CHECK-ORDERED-TF-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP59]], float [[TMP60]], float [[SUM_07]])
+; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-ORDERED-TF-NEXT:    [[MULADD]] = tail call nnan float @llvm.fmuladd.f32(float [[TMP56]], float [[TMP57]], float [[SUM_07]])
 ; CHECK-ORDERED-TF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-ORDERED-TF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK-ORDERED-TF:       for.end:
-; CHECK-ORDERED-TF-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP47]], [[MIDDLE_BLOCK]] ]
+; CHECK-ORDERED-TF-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP44]], [[MIDDLE_BLOCK]] ]
 ; CHECK-ORDERED-TF-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
index eaf856946f3a1..b7016ff4abf88 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
@@ -14,7 +14,7 @@ define i64 @same_exit_block_pre_inc_use1() #1 {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -22,8 +22,6 @@ define i64 @same_exit_block_pre_inc_use1() #1 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 3, [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -34,7 +32,7 @@ define i64 @same_exit_block_pre_inc_use1() #1 {
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP13]], align 1
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]]
-; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1(<vscale x 16 x i1> [[TMP16]])
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = or i1 [[TMP17]], [[TMP18]]
@@ -130,10 +128,9 @@ define i64 @same_exit_block_pre_inc_use4() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP8]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i64, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i64 [[INDEX]], [[LD1]]
@@ -204,10 +201,9 @@ define i64 @loop_contains_safe_call() #1 {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP9]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load float, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[SQRT:%.*]] = tail call fast float @llvm.sqrt.f32(float [[LD1]])
@@ -256,7 +252,7 @@ define i64 @loop_contains_safe_div() #1 {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 2
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP12]])
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -264,8 +260,6 @@ define i64 @loop_contains_safe_div() #1 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
 ; CHECK-NEXT:    [[INDEX1:%.*]] = sub i64 64, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -275,7 +269,7 @@ define i64 @loop_contains_safe_div() #1 {
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP1]], align 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = udiv <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 20000)
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp ne <vscale x 4 x i32> [[TMP13]], splat (i32 1)
-; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX2]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX2]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP15]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[INDEX1]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]]
@@ -371,10 +365,9 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP11]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[LD1]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
index 070f6580edc8e..46493e0bcd417 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
@@ -14,7 +14,7 @@ define i64 @same_exit_block_pre_inc_use1() #0 {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 64
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 6
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 510, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -22,8 +22,6 @@ define i64 @same_exit_block_pre_inc_use1() #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 64
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 510, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 510, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 64
 ; CHECK-NEXT:    [[INDEX_NEXT:%.*]] = add i64 3, [[N_VEC]]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       vector.body:
@@ -31,10 +29,10 @@ define i64 @same_exit_block_pre_inc_use1() #0 {
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 4
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP11]], 32
+; CHECK-NEXT:    [[TMP27:%.*]] = shl nuw i64 [[TMP11]], 5
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP27]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 48
@@ -45,10 +43,10 @@ define i64 @same_exit_block_pre_inc_use1() #0 {
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 16 x i8>, ptr [[TMP28]], align 1
 ; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 16
+; CHECK-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 4
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP29]], i64 [[TMP19]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 32
+; CHECK-NEXT:    [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 5
 ; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i8, ptr [[TMP29]], i64 [[TMP22]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 48
@@ -61,7 +59,7 @@ define i64 @same_exit_block_pre_inc_use1() #0 {
 ; CHECK-NEXT:    [[TMP30:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], [[WIDE_LOAD6]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]]
 ; CHECK-NEXT:    [[TMP59:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD4]], [[WIDE_LOAD8]]
-; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP34:%.*]] = or <vscale x 16 x i1> [[TMP32]], [[TMP30]]
 ; CHECK-NEXT:    [[TMP37:%.*]] = or <vscale x 16 x i1> [[TMP34]], [[TMP31]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = or <vscale x 16 x i1> [[TMP37]], [[TMP59]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 51efbe96f83b8..f6de370874d12 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -10,12 +10,12 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; DEFAULT-SAME: ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
 ; DEFAULT-NEXT:  iter.check:
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; DEFAULT-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP1]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; DEFAULT:       vector.main.loop.iter.check:
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
+; DEFAULT-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 5
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP3]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DEFAULT:       vector.ph:
@@ -23,18 +23,16 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; DEFAULT-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 101, [[TMP5]]
 ; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
-; DEFAULT-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 32
 ; DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; DEFAULT:       vector.body:
 ; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
 ; DEFAULT-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 16
+; DEFAULT-NEXT:    [[TMP23:%.*]] = shl nuw i64 [[TMP22]], 4
 ; DEFAULT-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP23]]
 ; DEFAULT-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP9]], align 1
 ; DEFAULT-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP24]], align 1
-; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; DEFAULT-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; DEFAULT-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; DEFAULT-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DEFAULT:       middle.block:
@@ -43,7 +41,7 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; DEFAULT:       vec.epilog.iter.check:
 ; DEFAULT-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 101, [[N_VEC]]
 ; DEFAULT-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 8
+; DEFAULT-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 3
 ; DEFAULT-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP13]]
 ; DEFAULT-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; DEFAULT:       vec.epilog.ph:
@@ -52,14 +50,12 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; DEFAULT-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 8
 ; DEFAULT-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 101, [[TMP15]]
 ; DEFAULT-NEXT:    [[N_VEC3:%.*]] = sub i64 101, [[N_MOD_VF2]]
-; DEFAULT-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 8
 ; DEFAULT-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; DEFAULT:       vec.epilog.vector.body:
 ; DEFAULT-NEXT:    [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
 ; DEFAULT-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX5]]
 ; DEFAULT-NEXT:    store <vscale x 8 x i8> zeroinitializer, ptr [[TMP19]], align 1
-; DEFAULT-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX5]], [[TMP17]]
+; DEFAULT-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX5]], [[TMP15]]
 ; DEFAULT-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
 ; DEFAULT-NEXT:    br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; DEFAULT:       vec.epilog.middle.block:
@@ -85,14 +81,8 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; PRED:       vector.ph:
 ; PRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; PRED-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 101, [[TMP4]]
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; PRED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; PRED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
+; PRED-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 4
 ; PRED-NEXT:    [[TMP9:%.*]] = sub i64 101, [[TMP8]]
 ; PRED-NEXT:    [[TMP10:%.*]] = icmp ugt i64 101, [[TMP8]]
 ; PRED-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 0
@@ -103,18 +93,17 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; PRED-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
 ; PRED-NEXT:    call void @llvm.masked.store.nxv16i8.p0(<vscale x 16 x i8> zeroinitializer, ptr [[TMP13]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX]], i64 [[TMP11]])
-; PRED-NEXT:    [[TMP15:%.*]] = xor <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; PRED-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 16 x i1> [[TMP15]], i32 0
-; PRED-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; PRED-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP12:%.*]] = xor i1 [[TMP14]], true
+; PRED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; PRED:       middle.block:
 ; PRED-NEXT:    br label [[EXIT:%.*]]
 ; PRED:       scalar.ph:
-; PRED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; PRED-NEXT:    br label [[LOOP:%.*]]
 ; PRED:       loop:
-; PRED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; PRED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; PRED-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV]]
 ; PRED-NEXT:    store i8 0, ptr [[GEP]], align 1
 ; PRED-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
@@ -227,12 +216,6 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
 ; PRED:       vector.ph:
 ; PRED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
 ; PRED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP10]], 2
-; PRED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 1000, [[TMP2]]
-; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; PRED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[X]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1000)
@@ -248,10 +231,10 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
 ; PRED-NEXT:    [[TMP9:%.*]] = and <vscale x 2 x i8> [[TMP8]], [[TMP11]]
 ; PRED-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
 ; PRED-NEXT:    call void @llvm.masked.store.nxv2i8.p0(<vscale x 2 x i8> [[TMP9]], ptr [[TMP5]], i32 1, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
-; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; PRED-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1000)
-; PRED-NEXT:    [[TMP12:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; PRED-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 2 x i1> [[TMP12]], i32 0
+; PRED-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; PRED-NEXT:    [[TMP13:%.*]] = xor i1 [[TMP12]], true
 ; PRED-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; PRED:       middle.block:
 ; PRED-NEXT:    br label [[EXIT:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
index c721493243734..bdbbfdfa97427 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/struct-return-cost.ll
@@ -82,8 +82,8 @@ define void @struct_return_replicate(ptr noalias %in, ptr noalias writeonly %out
 ; CHECK:  [[ENTRY:.*:]]
 ; CHECK:  [[VECTOR_PH:.*:]]
 ; CHECK:  [[VECTOR_BODY:.*:]]
-; CHECK:    [[TMP4:%.*]] = tail call { half, half } @foo(half [[TMP3:%.*]]) #[[ATTR3:[0-9]+]]
-; CHECK:    [[TMP6:%.*]] = tail call { half, half } @foo(half [[TMP5:%.*]]) #[[ATTR3]]
+; CHECK:    [[TMP2:%.*]] = tail call { half, half } @foo(half [[TMP1:%.*]]) #[[ATTR3:[0-9]+]]
+; CHECK:    [[TMP4:%.*]] = tail call { half, half } @foo(half [[TMP3:%.*]]) #[[ATTR3]]
 ; CHECK:  [[MIDDLE_BLOCK:.*:]]
 ; CHECK:  [[SCALAR_PH:.*:]]
 ; CHECK:  [[FOR_BODY:.*:]]
@@ -149,10 +149,9 @@ exit:
 define void @struct_return_scalable(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) #2 {
 ; CHECK-LABEL: define void @struct_return_scalable(
 ; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK:  [[ENTRY:.*:]]
-; CHECK:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK:  [[VECTOR_PH:.*:]]
 ; CHECK:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK:  [[VECTOR_PH1:.*:]]
 ; CHECK:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK:  [[VECTOR_BODY:.*:]]
 ; CHECK:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
index 67f50832603b7..495f9c07dae16 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll
@@ -10,11 +10,10 @@ define void @cond_inv_load_i32i32i16(ptr noalias nocapture %a, ptr noalias nocap
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[INV:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -69,11 +68,10 @@ define void @cond_inv_load_f64f64f64(ptr noalias nocapture %a, ptr noalias nocap
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[INV:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -126,11 +124,10 @@ define void @invariant_load_cond(ptr noalias nocapture %a, ptr nocapture readonl
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[B:%.*]], i64 168
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[TMP5]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
index e555785afe109..793813e55409e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
@@ -11,7 +11,7 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -19,8 +19,6 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -28,7 +26,7 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
 ; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi i64 [ -1, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 1
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP10]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i64>, ptr [[TMP15]], align 8
@@ -36,7 +34,7 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
 ; CHECK-NEXT:    [[TMP17]] = and i64 [[VEC_PHI]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vector.reduce.and.nxv2i64(<vscale x 2 x i64> [[WIDE_LOAD3]])
 ; CHECK-NEXT:    [[TMP19]] = and i64 [[VEC_PHI2]], [[TMP18]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP21]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
index af9c39e42961d..44a8eba84a1d0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
@@ -11,7 +11,7 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -19,8 +19,6 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -28,13 +26,13 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 1
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP10]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i64>, ptr [[TMP15]], align 8
 ; CHECK-NEXT:    [[TMP16]] = add <vscale x 2 x i64> [[WIDE_LOAD]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP17]] = add <vscale x 2 x i64> [[WIDE_LOAD3]], [[VEC_PHI2]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
index fca29cd8da7d6..4c4c9e57b4ffb 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
@@ -11,7 +11,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -19,21 +19,19 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP17]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[WIDE_LOAD]])
 ; CHECK-NEXT:    [[TMP19]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[TMP18]], <vscale x 4 x float> [[WIDE_LOAD2]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
index 0f407cd565457..04b0075e5a5b2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
@@ -22,12 +22,12 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-LABEL: @main_vf_vscale_x_16(
 ; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 5
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 1024, [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -35,18 +35,16 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 32
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 16
+; CHECK-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 4
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 [[TMP18]]
 ; CHECK-NEXT:    store <vscale x 16 x i8> splat (i8 1), ptr [[TMP14]], align 1
 ; CHECK-NEXT:    store <vscale x 16 x i8> splat (i8 1), ptr [[TMP19]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -55,7 +53,7 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 8
+; CHECK-NEXT:    [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 3
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP22]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
@@ -64,14 +62,12 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 8
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 1024, [[TMP24]]
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 1024, [[N_MOD_VF2]]
-; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 8
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX5]]
 ; CHECK-NEXT:    store <vscale x 8 x i8> splat (i8 1), ptr [[TMP28]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX5]], [[TMP26]]
+; CHECK-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX5]], [[TMP24]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[TMP30]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
@@ -87,7 +83,7 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-VF8-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK-VF8:       vector.main.loop.iter.check:
 ; CHECK-VF8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
+; CHECK-VF8-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5
 ; CHECK-VF8-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-VF8-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-VF8:       vector.ph:
@@ -95,18 +91,16 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-VF8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 16
+; CHECK-VF8-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 4
 ; CHECK-VF8-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-VF8-NEXT:    store <vscale x 16 x i8> splat (i8 1), ptr [[TMP12]], align 1
 ; CHECK-VF8-NEXT:    store <vscale x 16 x i8> splat (i8 1), ptr [[TMP17]], align 1
-; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-VF8-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF8-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-VF8:       middle.block:
@@ -152,25 +146,23 @@ define void @main_vf_vscale_x_2_no_epi_iteration(ptr %A) #0 vscale_range(8, 8) {
 ; CHECK-LABEL: @main_vf_vscale_x_2_no_epi_iteration(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP8]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP6]], align 1
 ; CHECK-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP9]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
@@ -186,7 +178,7 @@ define void @main_vf_vscale_x_2_no_epi_iteration(ptr %A) #0 vscale_range(8, 8) {
 ; CHECK-VF8-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK-VF8:       vector.main.loop.iter.check:
 ; CHECK-VF8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-VF8-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-VF8-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-VF8-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-VF8:       vector.ph:
@@ -194,18 +186,16 @@ define void @main_vf_vscale_x_2_no_epi_iteration(ptr %A) #0 vscale_range(8, 8) {
 ; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-VF8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
+; CHECK-VF8-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 1
 ; CHECK-VF8-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP8]]
 ; CHECK-VF8-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP6]], align 1
 ; CHECK-VF8-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP9]], align 1
-; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-VF8-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF8-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK-VF8:       middle.block:
@@ -268,7 +258,7 @@ define void @main_vf_vscale_x_2(ptr %A, i64 %n) #0 vscale_range(8, 8) {
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -276,18 +266,16 @@ define void @main_vf_vscale_x_2(ptr %A, i64 %n) #0 vscale_range(8, 8) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 2
+; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 1
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP12]], align 1
 ; CHECK-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP17]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       middle.block:
@@ -323,7 +311,7 @@ define void @main_vf_vscale_x_2(ptr %A, i64 %n) #0 vscale_range(8, 8) {
 ; CHECK-VF8-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK-VF8:       vector.main.loop.iter.check:
 ; CHECK-VF8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-VF8-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-VF8-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-VF8-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-VF8:       vector.ph:
@@ -331,18 +319,16 @@ define void @main_vf_vscale_x_2(ptr %A, i64 %n) #0 vscale_range(8, 8) {
 ; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-VF8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 2
+; CHECK-VF8-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 1
 ; CHECK-VF8-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-VF8-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP12]], align 1
 ; CHECK-VF8-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP17]], align 1
-; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-VF8-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF8-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK-VF8:       middle.block:
@@ -393,12 +379,12 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-LABEL: @test_pr57912_pointer_induction(
 ; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 10000, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 5
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 10000, [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -406,18 +392,16 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 32
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 16
+; CHECK-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 4
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP18]]
 ; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP14]], align 1
 ; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP19]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
@@ -427,7 +411,7 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-NEXT:    [[IND_END4:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 10000, [[N_VEC]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 8
+; CHECK-NEXT:    [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 3
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP22]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
@@ -436,15 +420,13 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 8
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 10000, [[TMP24]]
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 10000, [[N_MOD_VF2]]
-; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC3]]
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[START]], i64 [[INDEX7]]
 ; CHECK-NEXT:    store <vscale x 8 x i8> zeroinitializer, ptr [[TMP28]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT8]] = add nuw i64 [[INDEX7]], [[TMP26]]
+; CHECK-NEXT:    [[INDEX_NEXT8]] = add nuw i64 [[INDEX7]], [[TMP24]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[TMP30]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
@@ -461,7 +443,7 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-VF8-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK-VF8:       vector.main.loop.iter.check:
 ; CHECK-VF8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
+; CHECK-VF8-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 5
 ; CHECK-VF8-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 10000, [[TMP1]]
 ; CHECK-VF8-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-VF8:       vector.ph:
@@ -469,18 +451,16 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-VF8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP3]]
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]]
-; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 16
+; CHECK-VF8-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 4
 ; CHECK-VF8-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-VF8-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP12]], align 1
 ; CHECK-VF8-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP17]], align 1
-; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-VF8-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF8-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-VF8:       middle.block:
@@ -533,11 +513,11 @@ define void @trip_count_vscale(ptr noalias %a, ptr noalias %b) vscale_range(1, 1
 ; CHECK-NEXT:    [[V:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[N:%.*]] = mul nuw nsw i64 [[V]], 1033
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -545,31 +525,29 @@ define void @trip_count_vscale(ptr noalias %a, ptr noalias %b) vscale_range(1, 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x float>, ptr [[TMP17]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP21]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP13]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP19]], ptr [[TMP22]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       middle.block:
@@ -578,7 +556,7 @@ define void @trip_count_vscale(ptr noalias %a, ptr noalias %b) vscale_range(1, 1
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 2
+; CHECK-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP25]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
@@ -587,8 +565,6 @@ define void @trip_count_vscale(ptr noalias %a, ptr noalias %b) vscale_range(1, 1
 ; CHECK-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 2
 ; CHECK-NEXT:    [[N_MOD_VF5:%.*]] = urem i64 [[N]], [[TMP27]]
 ; CHECK-NEXT:    [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
-; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 2
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT10:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
@@ -598,7 +574,7 @@ define void @trip_count_vscale(ptr noalias %a, ptr noalias %b) vscale_range(1, 1
 ; CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 2 x float>, ptr [[TMP32]], align 4
 ; CHECK-NEXT:    [[TMP34:%.*]] = fmul <vscale x 2 x float> [[WIDE_LOAD8]], [[WIDE_LOAD9]]
 ; CHECK-NEXT:    store <vscale x 2 x float> [[TMP34]], ptr [[TMP32]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT10]] = add nuw i64 [[INDEX7]], [[TMP29]]
+; CHECK-NEXT:    [[INDEX_NEXT10]] = add nuw i64 [[INDEX7]], [[TMP27]]
 ; CHECK-NEXT:    [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT10]], [[N_VEC6]]
 ; CHECK-NEXT:    br i1 [[TMP35]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
@@ -614,38 +590,36 @@ define void @trip_count_vscale(ptr noalias %a, ptr noalias %b) vscale_range(1, 1
 ; CHECK-VF8-NEXT:    [[V:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-VF8-NEXT:    [[N:%.*]] = mul nuw nsw i64 [[V]], 1033
 ; CHECK-VF8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-VF8-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-VF8-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-VF8:       vector.ph:
 ; CHECK-VF8-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-VF8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
+; CHECK-VF8-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2
 ; CHECK-VF8-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw float, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-VF8-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
 ; CHECK-VF8-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP10]], align 4
 ; CHECK-VF8-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 4
+; CHECK-VF8-NEXT:    [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 2
 ; CHECK-VF8-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw float, ptr [[TMP11]], i64 [[TMP14]]
 ; CHECK-VF8-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
 ; CHECK-VF8-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP15]], align 4
 ; CHECK-VF8-NEXT:    [[TMP16:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-VF8-NEXT:    [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-VF8-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
+; CHECK-VF8-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 2
 ; CHECK-VF8-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw float, ptr [[TMP11]], i64 [[TMP19]]
 ; CHECK-VF8-NEXT:    store <vscale x 4 x float> [[TMP16]], ptr [[TMP11]], align 4
 ; CHECK-VF8-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP20]], align 4
-; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-VF8-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF8-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK-VF8:       middle.block:
@@ -685,11 +659,11 @@ define void @trip_count_vscale_no_epilogue_iterations(ptr noalias %a, ptr noalia
 ; CHECK-NEXT:    [[V:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[N:%.*]] = mul nuw nsw i64 [[V]], 1024
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -697,31 +671,29 @@ define void @trip_count_vscale_no_epilogue_iterations(ptr noalias %a, ptr noalia
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x float>, ptr [[TMP17]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP21]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP13]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP19]], ptr [[TMP22]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       middle.block:
@@ -730,7 +702,7 @@ define void @trip_count_vscale_no_epilogue_iterations(ptr noalias %a, ptr noalia
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 2
+; CHECK-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP25]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
@@ -739,8 +711,6 @@ define void @trip_count_vscale_no_epilogue_iterations(ptr noalias %a, ptr noalia
 ; CHECK-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 2
 ; CHECK-NEXT:    [[N_MOD_VF5:%.*]] = urem i64 [[N]], [[TMP27]]
 ; CHECK-NEXT:    [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
-; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 2
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT10:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
@@ -750,7 +720,7 @@ define void @trip_count_vscale_no_epilogue_iterations(ptr noalias %a, ptr noalia
 ; CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 2 x float>, ptr [[TMP32]], align 4
 ; CHECK-NEXT:    [[TMP34:%.*]] = fmul <vscale x 2 x float> [[WIDE_LOAD8]], [[WIDE_LOAD9]]
 ; CHECK-NEXT:    store <vscale x 2 x float> [[TMP34]], ptr [[TMP32]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT10]] = add nuw i64 [[INDEX7]], [[TMP29]]
+; CHECK-NEXT:    [[INDEX_NEXT10]] = add nuw i64 [[INDEX7]], [[TMP27]]
 ; CHECK-NEXT:    [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT10]], [[N_VEC6]]
 ; CHECK-NEXT:    br i1 [[TMP35]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
@@ -766,38 +736,36 @@ define void @trip_count_vscale_no_epilogue_iterations(ptr noalias %a, ptr noalia
 ; CHECK-VF8-NEXT:    [[V:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-VF8-NEXT:    [[N:%.*]] = mul nuw nsw i64 [[V]], 1024
 ; CHECK-VF8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-VF8-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-VF8-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-VF8:       vector.ph:
 ; CHECK-VF8-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-VF8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
+; CHECK-VF8-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2
 ; CHECK-VF8-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw float, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-VF8-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
 ; CHECK-VF8-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP10]], align 4
 ; CHECK-VF8-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 4
+; CHECK-VF8-NEXT:    [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 2
 ; CHECK-VF8-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw float, ptr [[TMP11]], i64 [[TMP14]]
 ; CHECK-VF8-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
 ; CHECK-VF8-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP15]], align 4
 ; CHECK-VF8-NEXT:    [[TMP16:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-VF8-NEXT:    [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-VF8-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
+; CHECK-VF8-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 2
 ; CHECK-VF8-NEXT:    [[TMP20:%.*]] = getelementptr inbounds nuw float, ptr [[TMP11]], i64 [[TMP19]]
 ; CHECK-VF8-NEXT:    store <vscale x 4 x float> [[TMP16]], ptr [[TMP11]], align 4
 ; CHECK-VF8-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP20]], align 4
-; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-VF8-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF8-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK-VF8:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll
index 20bc0af648458..76a7536501bd6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fixed-width-inorder-core.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -mtriple=aarch64-none-elf -mcpu=cortex-a510 -mattr=+sve -passes=loop-vectorize -S | FileCheck %s --check-prefix=CHECK-CA510
 ; RUN: opt < %s -mtriple=aarch64-none-elf -mcpu=cortex-a520 -mattr=+sve -passes=loop-vectorize -S | FileCheck %s --check-prefix=CHECK-CA520
+; RUN: opt < %s -mtriple=aarch64-none-elf -mcpu=cortex-a320 -mattr=+sve -passes=loop-vectorize -S | FileCheck %s --check-prefix=CHECK-CA320
 
 define void @sve_add(ptr  %dst, ptr  %a, ptr  %b, i64 %n) {
 ; CHECK-CA510-LABEL: define void @sve_add(
@@ -131,6 +132,70 @@ define void @sve_add(ptr  %dst, ptr  %a, ptr  %b, i64 %n) {
 ; CHECK-CA520:       [[FOR_COND_CLEANUP]]:
 ; CHECK-CA520-NEXT:    ret void
 ;
+; CHECK-CA320-LABEL: define void @sve_add(
+; CHECK-CA320-SAME: ptr [[DST:%.*]], ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-CA320-NEXT:  [[ENTRY:.*:]]
+; CHECK-CA320-NEXT:    [[B3:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-CA320-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-CA320-NEXT:    [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64
+; CHECK-CA320-NEXT:    [[CMP9_NOT:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-CA320-NEXT:    br i1 [[CMP9_NOT]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK-CA320:       [[FOR_BODY_PREHEADER]]:
+; CHECK-CA320-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-CA320-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-CA320:       [[VECTOR_MEMCHECK]]:
+; CHECK-CA320-NEXT:    [[TMP0:%.*]] = sub i64 [[DST1]], [[A2]]
+; CHECK-CA320-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 32
+; CHECK-CA320-NEXT:    [[TMP1:%.*]] = sub i64 [[DST1]], [[B3]]
+; CHECK-CA320-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP1]], 32
+; CHECK-CA320-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
+; CHECK-CA320-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK-CA320:       [[VECTOR_PH]]:
+; CHECK-CA320-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; CHECK-CA320-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-CA320-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK-CA320:       [[VECTOR_BODY]]:
+; CHECK-CA320-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-CA320-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
+; CHECK-CA320-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[TMP2]], i32 4
+; CHECK-CA320-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-CA320-NEXT:    [[WIDE_LOAD5:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-CA320-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
+; CHECK-CA320-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[TMP4]], i32 4
+; CHECK-CA320-NEXT:    [[WIDE_LOAD6:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
+; CHECK-CA320-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x float>, ptr [[TMP5]], align 4
+; CHECK-CA320-NEXT:    [[TMP6:%.*]] = fadd fast <4 x float> [[WIDE_LOAD6]], [[WIDE_LOAD]]
+; CHECK-CA320-NEXT:    [[TMP7:%.*]] = fadd fast <4 x float> [[WIDE_LOAD7]], [[WIDE_LOAD5]]
+; CHECK-CA320-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[DST]], i64 [[INDEX]]
+; CHECK-CA320-NEXT:    [[TMP9:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i32 4
+; CHECK-CA320-NEXT:    store <4 x float> [[TMP6]], ptr [[TMP8]], align 4
+; CHECK-CA320-NEXT:    store <4 x float> [[TMP7]], ptr [[TMP9]], align 4
+; CHECK-CA320-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-CA320-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-CA320-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-CA320:       [[MIDDLE_BLOCK]]:
+; CHECK-CA320-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-CA320-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-CA320:       [[SCALAR_PH]]:
+; CHECK-CA320-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-CA320-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK-CA320:       [[FOR_BODY]]:
+; CHECK-CA320-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-CA320-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-CA320-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-CA320-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-CA320-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-CA320-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP12]], [[TMP11]]
+; CHECK-CA320-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw float, ptr [[DST]], i64 [[INDVARS_IV]]
+; CHECK-CA320-NEXT:    store float [[ADD]], ptr [[ARRAYIDX4]], align 4
+; CHECK-CA320-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-CA320-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-CA320-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-CA320:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-CA320-NEXT:    br label %[[FOR_COND_CLEANUP]]
+; CHECK-CA320:       [[FOR_COND_CLEANUP]]:
+; CHECK-CA320-NEXT:    ret void
+;
 entry:
   %cmp9.not = icmp eq i64 %n, 0
   br i1 %cmp9.not, label %for.cond.cleanup, label %for.body
@@ -160,3 +225,8 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup.lo
 ; CHECK-CA520: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK-CA520: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
 ;.
+; CHECK-CA320: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK-CA320: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-CA320: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-CA320: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
index 24f93f04d6eed..95153a4a59f1a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
@@ -16,7 +16,7 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
@@ -31,14 +31,12 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP7]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds half, ptr [[S]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 3
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds half, ptr [[TMP11]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP11]], align 2
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x half>, ptr [[TMP15]], align 2
@@ -46,11 +44,11 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read
 ; CHECK-NEXT:    [[TMP17:%.*]] = fneg <vscale x 8 x half> [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds half, ptr [[D]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 8
+; CHECK-NEXT:    [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 3
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds half, ptr [[TMP18]], i64 [[TMP21]]
 ; CHECK-NEXT:    store <vscale x 8 x half> [[TMP16]], ptr [[TMP18]], align 2
 ; CHECK-NEXT:    store <vscale x 8 x half> [[TMP17]], ptr [[TMP22]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
index 663cf4173cc91..d336f5f70755e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
@@ -10,11 +10,10 @@ define void @gather_nxv4i32_ind64(ptr noalias nocapture readonly %a, ptr noalias
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -62,11 +61,10 @@ define void @scatter_nxv4i32_ind32(ptr noalias nocapture %a, ptr noalias nocaptu
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -113,11 +111,10 @@ define void @scatter_inv_nxv4i32(ptr noalias nocapture %inv, ptr noalias nocaptu
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[INV:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -166,11 +163,10 @@ define void @gather_inv_nxv4i32(ptr noalias nocapture %a, ptr noalias nocapture
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[INV:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -228,14 +224,12 @@ define void @gather_nxv4i32_ind64_stride2(ptr noalias nocapture %a, ptr noalias
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[N_VEC]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP7]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[N_VEC1:%.*]] = sub i64 [[N]], [[TMP6]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 3
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[DOTIDX1:%.*]] = shl i64 [[INDEX]], 3
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 [[DOTIDX1]]
-; CHECK-NEXT:    [[DOTIDX3:%.*]] = shl nuw nsw i64 [[TMP3]], 5
+; CHECK-NEXT:    [[DOTIDX3:%.*]] = shl nuw nsw i64 [[TMP2]], 5
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[DOTIDX3]]
 ; CHECK-NEXT:    [[DOTIDX4:%.*]] = shl i64 [[INDEX]], 3
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[TMP11]], i64 [[DOTIDX4]]
@@ -251,7 +245,7 @@ define void @gather_nxv4i32_ind64_stride2(ptr noalias nocapture %a, ptr noalias
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP12]], i64 [[DOTIDX]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[WIDE_MASKED_GATHER]], ptr [[TMP12]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[WIDE_MASKED_GATHER2]], ptr [[TMP14]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC1]]
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll
index fefb5af826755..731272675940e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll
@@ -12,16 +12,14 @@ define void @induction_i7(ptr %dst) #0 {
 ; CHECK-LABEL: define void @induction_i7(
 ; CHECK-SAME: ptr [[DST:%.*]])
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP40:%.*]] = mul nuw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP40]], 2
-; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i7
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP40]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT_:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP40]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i7
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = trunc <vscale x 2 x i64> [[DOTSPLAT_]] to <vscale x 2 x i7>
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i8> @llvm.stepvector.nxv2i8()
 ; CHECK-NEXT:    [[TMP7:%.*]] = trunc <vscale x 2 x i8> [[TMP6]] to <vscale x 2 x i7> 
@@ -38,7 +36,7 @@ define void @induction_i7(ptr %dst) #0 {
 ; CHECK-NEXT:    [[TMP23:%.*]] = zext <vscale x 2 x i7> [[TMP19]] to <vscale x 2 x i64>
 ; CHECK-NEXT:    [[TMP24:%.*]] = zext <vscale x 2 x i7> [[TMP20]] to <vscale x 2 x i64>
 ; CHECK-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 2
+; CHECK-NEXT:    [[TMP27:%.*]] = shl nuw i64 [[TMP26]], 1
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i64 [[TMP27]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP23]], ptr [[TMP21]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP24]], ptr [[TMP28]], align 8
@@ -76,16 +74,14 @@ define void @induction_i3_zext(ptr %dst) #0 {
 ; CHECK-LABEL: define void @induction_i3_zext(
 ; CHECK-SAME: ptr [[DST:%.*]])
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP40:%.*]] = mul nuw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP40]], 2
-; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i3
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP40]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT_:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP40]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i3
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = trunc <vscale x 2 x i64> [[DOTSPLAT_]] to <vscale x 2 x i3>
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i8> @llvm.stepvector.nxv2i8()
 ; CHECK-NEXT:    [[TMP7:%.*]] = trunc <vscale x 2 x i8> [[TMP6]] to <vscale x 2 x i3>
@@ -100,7 +96,7 @@ define void @induction_i3_zext(ptr %dst) #0 {
 ; CHECK-NEXT:    [[TMP20:%.*]] = zext <vscale x 2 x i3> [[STEP_ADD]] to <vscale x 2 x i64>
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 2
+; CHECK-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i64 [[TMP25]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP19]], ptr [[TMP21]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP20]], ptr [[TMP26]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
index 4f0637fd8db2f..95836f81918e2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
@@ -23,10 +23,8 @@ define void @cond_ind64(ptr noalias nocapture %a, ptr noalias nocapture readonly
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -37,7 +35,7 @@ define void @cond_ind64(ptr noalias nocapture %a, ptr noalias nocapture readonly
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP10]], i32 4, <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> poison)
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr [[TMP11]], i32 4, <vscale x 4 x i1> [[TMP9]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 47ce05da78e68..fd0bc0b6c20ef 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -437,8 +437,6 @@ define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 512, [[TMP1]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = shl nuw nsw i64 [[N_VEC]], 1
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -452,7 +450,7 @@ define void @even_load_static_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[TMP7:%.*]] = and i64 [[INDEX]], 9223372036854775804
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP7]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP6]], ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
@@ -508,8 +506,6 @@ define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP6]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP4]], [[TMP9]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = shl i64 [[N_VEC]], 1
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -523,7 +519,7 @@ define void @even_load_dynamic_tc(ptr noalias nocapture readonly %A, ptr noalias
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i64 [[INDEX]], 9223372036854775804
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i32, ptr [[B:%.*]], i64 [[TMP15]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP14]], ptr [[TMP16]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       middle.block:
@@ -803,12 +799,10 @@ define void @PR27626_0(ptr %p, i32 %z, i64 %n) #1 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -822,7 +816,7 @@ define void @PR27626_0(ptr %p, i32 %z, i64 %n) #1 {
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP15]], <vscale x 4 x ptr> [[TMP13]], i32 4, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
@@ -875,10 +869,8 @@ define i32 @PR27626_1(ptr %p, i64 %n) #1 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -896,7 +888,7 @@ define i32 @PR27626_1(ptr %p, i64 %n) #1 {
 ; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC1]])
 ; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
 ; CHECK-NEXT:    [[TMP17]] = add <vscale x 4 x i32> [[TMP16]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
@@ -952,12 +944,10 @@ define void @PR27626_2(ptr %p, i64 %n, i32 %z) #1 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -971,7 +961,7 @@ define void @PR27626_2(ptr %p, i64 %n, i32 %z) #1 {
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP15]], <vscale x 4 x ptr> [[TMP14]], i32 4, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
@@ -1025,10 +1015,8 @@ define i32 @PR27626_3(ptr %p, i64 %n, i32 %z) #1 {
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -1047,7 +1035,7 @@ define i32 @PR27626_3(ptr %p, i64 %n, i32 %z) #1 {
 ; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC1]])
 ; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
 ; CHECK-NEXT:    [[TMP18]] = add <vscale x 4 x i32> [[TMP17]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
@@ -1103,11 +1091,10 @@ define void @PR27626_4(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) #1 {
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP5]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP7]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNOT]]
 ; CHECK-NEXT:    [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 1
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -1180,11 +1167,10 @@ define void @PR27626_5(ptr %a, i32 %x, i32 %y, i32 %z, i64 %n) #1 {
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP5]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP8]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNOT]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[N_VEC]], 1
 ; CHECK-NEXT:    [[IND_END:%.*]] = or disjoint i64 [[TMP11]], 3
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X:%.*]], i64 0
@@ -1273,11 +1259,10 @@ define void @PR34743(ptr %a, ptr %b, i64 %n) #1 {
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP8]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP1]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw nsw i64 [[TMP9]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP10]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP1]], [[DOTNOT]]
 ; CHECK-NEXT:    [[IND_END:%.*]] = shl i64 [[N_VEC]], 1
 ; CHECK-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP15:%.*]] = shl <vscale x 4 x i64> [[TMP14]], splat (i64 1)
@@ -1372,10 +1357,8 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -1402,7 +1385,7 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
 ; CHECK-NEXT:    [[TMP17:%.*]] = shl <vscale x 4 x i32> [[TMP9]], [[TMP13]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP10]], i64 8
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP17]], <vscale x 4 x ptr> [[TMP25]], i32 4, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
@@ -1466,8 +1449,6 @@ define void @interleave_deinterleave(ptr writeonly noalias %dst, ptr readonly %a
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1492,7 +1473,7 @@ define void @interleave_deinterleave(ptr writeonly noalias %dst, ptr readonly %a
 ; CHECK-NEXT:    [[TMP24:%.*]] = ashr <vscale x 4 x i32> [[TMP12]], [[TMP19]]
 ; CHECK-NEXT:    [[INTERLEAVED_VEC13:%.*]] = call <vscale x 16 x i32> @llvm.vector.interleave4.nxv16i32(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[TMP23]], <vscale x 4 x i32> [[TMP24]])
 ; CHECK-NEXT:    store <vscale x 16 x i32> [[INTERLEAVED_VEC13]], ptr [[TMP21]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
index f152dd308cb69..32235860dd9e2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
@@ -30,41 +30,39 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.ph:
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP2]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.body:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl i32 [[INDEX]], 1
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr [[TMP10]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i8> poison)
+; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl i32 [[INDEX]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr [[TMP7]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i8> poison)
 ; SCALAR_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[WIDE_MASKED_VEC]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[TMP12]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP8]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP14]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP13]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP16]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP15]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP9]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP5]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP11]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP10]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP13]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP12]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SCALAR_TAIL_FOLDING:       middle.block:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -80,42 +78,42 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPA:%.*]] = sub i32 1024, [[TMP3]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPB:%.*]] = icmp ult i32 [[TMP2]], 64
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = select i1 [[TMPB]], i32 [[TMPA]], i32 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = sub i32 1024, [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP2]], 64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PREDICATED_TAIL_FOLDING:       vector.body:
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> zeroinitializer
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl i32 [[INDEX]], 1
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr [[TMP10]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = shl i32 [[INDEX]], 1
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> [[TMP9]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr [[TMP12]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i8> poison)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[WIDE_MASKED_VEC]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[TMP12]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP8]] to i64
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP14]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP13]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP16]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP15]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = sext i32 [[TMP10]] to i64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP16]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP15]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP18]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> [[TMP9]])
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP17]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP4]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP17]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP19]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
 ; PREDICATED_TAIL_FOLDING:       middle.block:
 ; PREDICATED_TAIL_FOLDING-NEXT:    br label [[FOR_END:%.*]]
 ; PREDICATED_TAIL_FOLDING:       scalar.ph:
@@ -176,34 +174,32 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.ph:
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP2]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.body:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP8]]
-; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 1), <vscale x 16 x ptr> [[TMP9]], i32 1, <vscale x 16 x i1> splat (i1 true))
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP12]]
-; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> [[TMP13]], i32 1, <vscale x 16 x i1> [[TMP10]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = zext nneg <vscale x 16 x i32> [[TMP4]] to <vscale x 16 x i64>
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP5]]
+; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 1), <vscale x 16 x ptr> [[TMP6]], i32 1, <vscale x 16 x i1> splat (i1 true))
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = or disjoint <vscale x 16 x i32> [[TMP4]], splat (i32 1)
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP9]]
+; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> [[TMP10]], i32 1, <vscale x 16 x i1> [[TMP7]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; SCALAR_TAIL_FOLDING:       middle.block:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -219,35 +215,35 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPA:%.*]] = sub i32 1024, [[TMP3]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPB:%.*]] = icmp ult i32 [[TMP2]], 64
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = select i1 [[TMPB]], i32 [[TMPA]], i32 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = sub i32 1024, [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP2]], 64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PREDICATED_TAIL_FOLDING:       vector.body:
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = zext nneg <vscale x 16 x i32> [[TMP6]] to <vscale x 16 x i64>
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP7]]
-; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 1), <vscale x 16 x ptr> [[TMP8]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> zeroinitializer
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = or disjoint <vscale x 16 x i32> [[TMP6]], splat (i32 1)
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = zext nneg <vscale x 16 x i32> [[TMP11]] to <vscale x 16 x i64>
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP12]]
-; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> [[TMP13]], i32 1, <vscale x 16 x i1> [[TMP10]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP9]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 1), <vscale x 16 x ptr> [[TMP10]], i32 1, <vscale x 16 x i1> [[ACTIVE_LANE_MASK]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 1)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP13]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP14]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> [[TMP15]], i32 1, <vscale x 16 x i1> [[TMP12]])
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP4]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP14]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP16]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
 ; PREDICATED_TAIL_FOLDING:       middle.block:
 ; PREDICATED_TAIL_FOLDING-NEXT:    br label [[FOR_END:%.*]]
 ; PREDICATED_TAIL_FOLDING:       scalar.ph:
@@ -304,37 +300,35 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.ph:
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV3]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP2]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.body:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = zext nneg <vscale x 16 x i32> [[TMP7]] to <vscale x 16 x i64>
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP9]]
-; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 1), <vscale x 16 x ptr> [[TMP10]], i32 1, <vscale x 16 x i1> [[TMP8]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = or disjoint <vscale x 16 x i32> [[TMP7]], splat (i32 1)
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = zext nneg <vscale x 16 x i32> [[TMP12]] to <vscale x 16 x i64>
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP13]]
-; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> [[TMP14]], i32 1, <vscale x 16 x i1> [[TMP11]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = zext nneg <vscale x 16 x i32> [[TMP4]] to <vscale x 16 x i64>
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP6]]
+; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 1), <vscale x 16 x ptr> [[TMP7]], i32 1, <vscale x 16 x i1> [[TMP5]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = or disjoint <vscale x 16 x i32> [[TMP4]], splat (i32 1)
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = zext nneg <vscale x 16 x i32> [[TMP9]] to <vscale x 16 x i64>
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP10]]
+; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> [[TMP11]], i32 1, <vscale x 16 x i1> [[TMP8]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; SCALAR_TAIL_FOLDING:       middle.block:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -351,39 +345,39 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPA:%.*]] = sub i32 1024, [[TMP3]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPB:%.*]] = icmp ult i32 [[TMP2]], 64
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = select i1 [[TMPB]], i32 [[TMPA]], i32 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = sub i32 1024, [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP2]], 64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV3]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PREDICATED_TAIL_FOLDING:       vector.body:
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> zeroinitializer
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = zext nneg <vscale x 16 x i32> [[TMP6]] to <vscale x 16 x i64>
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP9]]
-; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 1), <vscale x 16 x ptr> [[TMP10]], i32 1, <vscale x 16 x i1> [[TMP8]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i1> zeroinitializer
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = or disjoint <vscale x 16 x i32> [[TMP6]], splat (i32 1)
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = zext nneg <vscale x 16 x i32> [[TMP13]] to <vscale x 16 x i64>
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP14]]
-; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> [[TMP15]], i32 1, <vscale x 16 x i1> [[TMP12]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl nuw nsw <vscale x 16 x i32> [[VEC_IND]], splat (i32 1)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = zext nneg <vscale x 16 x i32> [[TMP8]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP11]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 1), <vscale x 16 x ptr> [[TMP12]], i32 1, <vscale x 16 x i1> [[TMP10]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP13]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = or disjoint <vscale x 16 x i32> [[TMP8]], splat (i32 1)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = zext nneg <vscale x 16 x i32> [[TMP15]] to <vscale x 16 x i64>
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[Q]], <vscale x 16 x i64> [[TMP16]]
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> splat (i8 2), <vscale x 16 x ptr> [[TMP17]], i32 1, <vscale x 16 x i1> [[TMP14]])
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP4]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP16]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP6:![0-9]+]]
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP18]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP6:![0-9]+]]
 ; PREDICATED_TAIL_FOLDING:       middle.block:
 ; PREDICATED_TAIL_FOLDING-NEXT:    br label [[FOR_END:%.*]]
 ; PREDICATED_TAIL_FOLDING:       scalar.ph:
@@ -452,45 +446,43 @@ define dso_local void @masked_strided_factor4(ptr noalias nocapture readonly %p,
 ; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.ph:
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP5]], i64 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP2]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.body:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl i32 [[INDEX]], 2
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP10]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
+; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl i32 [[INDEX]], 2
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP7]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
 ; SCALAR_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave4.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[TMP12]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP15]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP17]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = sext i32 [[TMP8]] to i64
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP19]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]], <vscale x 16 x i8> [[TMP17]], <vscale x 16 x i8> [[TMP18]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP20]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
-; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP9]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP12]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP14]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = sext i32 [[TMP5]] to i64
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP16]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]], <vscale x 16 x i8> [[TMP15]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; SCALAR_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP17]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
+; SCALAR_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; SCALAR_TAIL_FOLDING:       middle.block:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -506,46 +498,46 @@ define dso_local void @masked_strided_factor4(ptr noalias nocapture readonly %p,
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPA:%.*]] = sub i32 1024, [[TMP3]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPB:%.*]] = icmp ult i32 [[TMP2]], 64
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = select i1 [[TMPB]], i32 [[TMPA]], i32 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = sub i32 1024, [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = icmp ult i32 [[TMP2]], 64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_TAIL_FOLDING-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PREDICATED_TAIL_FOLDING:       vector.body:
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP7:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> zeroinitializer
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = shl i32 [[INDEX]], 2
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP8]] to i64
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP9]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP10]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
+; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP8:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP9:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP10:%.*]] = shl i32 [[INDEX]], 2
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP11]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> [[TMP9]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP12]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave4.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[TMP12]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP15]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP17:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]])
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP18:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP17]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = sext i32 [[TMP8]] to i64
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP19]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]], <vscale x 16 x i8> [[TMP17]], <vscale x 16 x i8> [[TMP18]])
-; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i1> [[TMP7]])
-; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP20]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP19:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP20:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP19]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP10]] to i64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP21]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP17]], <vscale x 16 x i8> [[TMP18]], <vscale x 16 x i8> [[TMP19]], <vscale x 16 x i8> [[TMP20]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> [[TMP9]])
+; PREDICATED_TAIL_FOLDING-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP22]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP1]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP4]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP6]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
-; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP21]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP8:![0-9]+]]
+; PREDICATED_TAIL_FOLDING-NEXT:    br i1 [[TMP23]], label [[VECTOR_BODY]], label [[MIDDLE_BLOCK:%.*]], !llvm.loop [[LOOP8:![0-9]+]]
 ; PREDICATED_TAIL_FOLDING:       middle.block:
 ; PREDICATED_TAIL_FOLDING-NEXT:    br label [[FOR_END:%.*]]
 ; PREDICATED_TAIL_FOLDING:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll
index f0675a4affa5d..7dc6a6aa890f9 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll
@@ -7,7 +7,7 @@ define void @inv_store_i16(ptr noalias %dst, ptr noalias readonly %src, i64 %N)
 ; CHECK-LABEL: @inv_store_i16(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -15,8 +15,6 @@ define void @inv_store_i16(ptr noalias %dst, ptr noalias readonly %src, i64 %N)
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -27,7 +25,7 @@ define void @inv_store_i16(ptr noalias %dst, ptr noalias readonly %src, i64 %N)
 ; CHECK-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD]], i32 [[TMP11]]
 ; CHECK-NEXT:    store i16 [[TMP12]], ptr [[DST:%.*]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -56,7 +54,7 @@ define void @cond_inv_store_i32(ptr noalias %dst, ptr noalias readonly %src, i64
 ; CHECK-LABEL: @cond_inv_store_i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -64,8 +62,6 @@ define void @cond_inv_store_i32(ptr noalias %dst, ptr noalias readonly %src, i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -75,7 +71,7 @@ define void @cond_inv_store_i32(ptr noalias %dst, ptr noalias readonly %src, i64
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x ptr> [[BROADCAST_SPLAT]], i32 4, <vscale x 4 x i1> [[TMP9]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll
index 2b4aad1db794c..9312306ce519a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll
@@ -11,7 +11,7 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -19,8 +19,6 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP7]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[N_VEC]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 8
@@ -31,11 +29,11 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[START_2]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP34:%.*]] = mul nuw i64 [[TMP33]], 2
+; CHECK-NEXT:    [[TMP34:%.*]] = shl nuw i64 [[TMP33]], 1
 ; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr i64, ptr [[TMP30]], i64 [[TMP34]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[TMP30]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[TMP35]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
index ce7b78ecc2486..e9643c974eb8f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
@@ -1,81 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
 ; RUN: opt -passes=loop-vectorize -S < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
 define void @trip7_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
-; CHECK-LABEL: @trip7_i64(
-; CHECK:         = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    = mul nuw i64
-; CHECK:         [[VSCALE:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[VF:%.*]] = mul nuw i64 [[VSCALE]], 2
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK:         [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ {{%.*}}, %vector.ph ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %vector.body ]
-; CHECK:         {{%.*}} = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr {{%.*}}, i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; CHECK:         {{%.*}} = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr {{%.*}}, i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
-; CHECK:         call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> {{%.*}}, ptr {{%.*}}, i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[VF]]
+; CHECK-LABEL: define void @trip7_i64(
+; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 7)
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP5]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nsw <vscale x 2 x i64> [[WIDE_MASKED_LOAD]], splat (i64 1)
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
+; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <vscale x 2 x i64> [[WIDE_MASKED_LOAD1]], [[TMP6]]
+; CHECK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP8]], ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 7)
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NOT:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[COND:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NOT]], i32 0
-; CHECK-NEXT:    br i1 [[COND]], label %middle.block, label %vector.body
+; CHECK-NEXT:    [[EXTRACT_FIRST_LANE_MASK:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[COND:%.*]] = xor i1 [[EXTRACT_FIRST_LANE_MASK]], true
+; CHECK-NEXT:    br i1 [[COND]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
 ;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                         ; preds = %entry, %for.body
-  %i.06 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
-  %arrayidx = getelementptr inbounds i64, ptr %src, i64 %i.06
-  %0 = load i64, ptr %arrayidx, align 8
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds i64, ptr %src, i64 %iv
+  %0 = load i64, ptr %gep.src, align 8
   %mul = shl nsw i64 %0, 1
-  %arrayidx1 = getelementptr inbounds i64, ptr %dst, i64 %i.06
-  %1 = load i64, ptr %arrayidx1, align 8
+  %gep.dst = getelementptr inbounds i64, ptr %dst, i64 %iv
+  %1 = load i64, ptr %gep.dst, align 8
   %add = add nsw i64 %1, %mul
-  store i64 %add, ptr %arrayidx1, align 8
-  %inc = add nuw nsw i64 %i.06, 1
-  %exitcond.not = icmp eq i64 %inc, 7
-  br i1 %exitcond.not, label %for.end, label %for.body
+  store i64 %add, ptr %gep.dst, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 7
+  br i1 %ec, label %exit, label %loop
 
-for.end:                                          ; preds = %for.body
+exit:
   ret void
 }
 
 define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
-; CHECK-LABEL: @trip5_i8(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
-; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[I_08]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-LABEL: define void @trip5_i8(
+; CHECK-SAME: ptr noalias noundef captures(none) [[DST:%.*]], ptr noalias noundef readonly captures(none) [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[GEP_SRC]], align 1
 ; CHECK-NEXT:    [[MUL:%.*]] = shl i8 [[TMP0]], 1
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[I_08]]
-; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr [[GEP_DST]], align 1
 ; CHECK-NEXT:    [[ADD:%.*]] = add i8 [[MUL]], [[TMP1]]
-; CHECK-NEXT:    store i8 [[ADD]], ptr [[ARRAYIDX1]], align 1
-; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_08]], 1
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 5
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]
-; CHECK:       for.end:
+; CHECK-NEXT:    store i8 [[ADD]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 5
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:                                         ; preds = %entry, %for.body
-  %i.08 = phi i64 [ 0, %entry ], [ %inc, %for.body ]
-  %arrayidx = getelementptr inbounds i8, ptr %src, i64 %i.08
-  %0 = load i8, ptr %arrayidx, align 1
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %gep.src = getelementptr inbounds i8, ptr %src, i64 %iv
+  %0 = load i8, ptr %gep.src, align 1
   %mul = shl i8 %0, 1
-  %arrayidx1 = getelementptr inbounds i8, ptr %dst, i64 %i.08
-  %1 = load i8, ptr %arrayidx1, align 1
+  %gep.dst = getelementptr inbounds i8, ptr %dst, i64 %iv
+  %1 = load i8, ptr %gep.dst, align 1
   %add = add i8 %mul, %1
-  store i8 %add, ptr %arrayidx1, align 1
-  %inc = add nuw nsw i64 %i.08, 1
-  %exitcond.not = icmp eq i64 %inc, 5
-  br i1 %exitcond.not, label %for.end, label %for.body
+  store i8 %add, ptr %gep.dst, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 5
+  br i1 %ec, label %exit, label %loop
 
-for.end:                                          ; preds = %for.body
+exit:
   ret void
 }
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
index 993c0486ab1a0..ed49dc5a7573f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
@@ -16,7 +16,7 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[N:%.*]], i32 999)
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
@@ -33,24 +33,22 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP10]]
-; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP11]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
+; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw i64 [[TMP22]], 2
 ; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP23]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP19]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP24]], align 4
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 4
+; CHECK-NEXT:    [[TMP29:%.*]] = shl nuw i64 [[TMP28]], 2
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP29]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP25]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD3]], ptr [[TMP30]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -88,7 +86,7 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[N:%.*]], i32 999)
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
@@ -105,24 +103,22 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP10]]
-; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP11]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
+; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw i64 [[TMP22]], 2
 ; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP23]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP19]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP24]], align 4
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 4
+; CHECK-NEXT:    [[TMP29:%.*]] = shl nuw i64 [[TMP28]], 2
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP29]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP25]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD3]], ptr [[TMP30]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
index 893ebef0d84df..3c667ccbdedfc 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
@@ -14,7 +14,7 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
 ; CHECK-NEXT:    [[DST_21:%.*]] = ptrtoint ptr [[DST_2:%.*]] to i64
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 20, i64 [[TMP1]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
@@ -46,20 +46,18 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP16]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP49:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP50:%.*]] = mul nuw i64 [[TMP49]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i64, ptr [[SRC_1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i64, ptr [[SRC_2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 2
+; CHECK-NEXT:    [[TMP29:%.*]] = shl nuw i64 [[TMP28]], 1
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i64, ptr [[TMP27]], i64 [[TMP29]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP27]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD12:%.*]] = load <vscale x 2 x i64>, ptr [[TMP30]], align 8
 ; CHECK-NEXT:    [[TMP32:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP33:%.*]] = mul nuw i64 [[TMP32]], 2
+; CHECK-NEXT:    [[TMP33:%.*]] = shl nuw i64 [[TMP32]], 1
 ; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i64, ptr [[TMP31]], i64 [[TMP33]]
 ; CHECK-NEXT:    [[WIDE_LOAD13:%.*]] = load <vscale x 2 x i64>, ptr [[TMP31]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD14:%.*]] = load <vscale x 2 x i64>, ptr [[TMP34]], align 8
@@ -68,16 +66,16 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
 ; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr i64, ptr [[DST_1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr i64, ptr [[DST_2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP42:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP43:%.*]] = mul nuw i64 [[TMP42]], 2
+; CHECK-NEXT:    [[TMP43:%.*]] = shl nuw i64 [[TMP42]], 1
 ; CHECK-NEXT:    [[TMP44:%.*]] = getelementptr i64, ptr [[TMP41]], i64 [[TMP43]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP35]], ptr [[TMP41]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP36]], ptr [[TMP44]], align 8
 ; CHECK-NEXT:    [[TMP46:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP47:%.*]] = mul nuw i64 [[TMP46]], 2
+; CHECK-NEXT:    [[TMP47:%.*]] = shl nuw i64 [[TMP46]], 1
 ; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr i64, ptr [[TMP45]], i64 [[TMP47]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP35]], ptr [[TMP45]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP36]], ptr [[TMP48]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP50]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP51]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
index 1cda568c9845d..ad9eefd8ee6a1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
@@ -46,16 +46,10 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -70,9 +64,9 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
 ; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP14]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[TMP15]], i32 0
-; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i1 [[TMP15]], true
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll
index fb0447bace360..9b725c3304549 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll
@@ -10,12 +10,6 @@ define void @trip1025_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapt
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -28,11 +22,11 @@ define void @trip1025_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapt
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP11]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <vscale x 2 x i64> [[WIDE_MASKED_LOAD1]], [[TMP10]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP13]], ptr [[TMP11]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
-; CHECK-NEXT:    [[TMP14:%.*]] = xor <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 2 x i1> [[TMP14]], i32 0
-; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <vscale x 2 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i1 [[TMP9]], true
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll
index f4982e602bcdc..821b9fbbda78f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll
@@ -12,16 +12,10 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -37,19 +31,17 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[TMP14]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[TMP13]], <vscale x 4 x i32> [[VEC_PHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP17:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[TMP17]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP17]], true
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ [[RED_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ [[RED_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[VAL:%.*]] = load i32, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[RED_NEXT]] = add i32 [[RED]], [[VAL]]
@@ -65,16 +57,10 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
 ; CHECK-IN-LOOP-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-IN-LOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-IN-LOOP:       vector.ph:
-; CHECK-IN-LOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-IN-LOOP-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-IN-LOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
-; CHECK-IN-LOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-IN-LOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-IN-LOOP-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-IN-LOOP-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -91,18 +77,16 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
 ; CHECK-IN-LOOP-NEXT:    [[TMP15]] = add i32 [[VEC_PHI]], [[TMP14]]
 ; CHECK-IN-LOOP-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP17]]
 ; CHECK-IN-LOOP-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-IN-LOOP-NEXT:    [[TMP18:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-IN-LOOP-NEXT:    [[TMP19:%.*]] = extractelement <vscale x 4 x i1> [[TMP18]], i32 0
+; CHECK-IN-LOOP-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-IN-LOOP-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP18]], true
 ; CHECK-IN-LOOP-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-IN-LOOP:       middle.block:
 ; CHECK-IN-LOOP-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK-IN-LOOP:       scalar.ph:
-; CHECK-IN-LOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-IN-LOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-IN-LOOP-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK-IN-LOOP:       while.body:
-; CHECK-IN-LOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-IN-LOOP-NEXT:    [[RED:%.*]] = phi i32 [ [[RED_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-IN-LOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-IN-LOOP-NEXT:    [[RED:%.*]] = phi i32 [ [[RED_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-IN-LOOP-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
 ; CHECK-IN-LOOP-NEXT:    [[VAL:%.*]] = load i32, ptr [[GEP]], align 4
 ; CHECK-IN-LOOP-NEXT:    [[RED_NEXT]] = add i32 [[RED]], [[VAL]]
@@ -136,16 +120,10 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -161,18 +139,16 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP13]])
 ; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP17:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[TMP17]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP17]], true
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi float [ [[RED_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi float [ [[RED_NEXT:%.*]], [[WHILE_BODY]] ], [ 0.000000e+00, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr float, ptr [[PTR]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[VAL:%.*]] = load float, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[RED_NEXT]] = fadd float [[RED]], [[VAL]]
@@ -188,16 +164,10 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
 ; CHECK-IN-LOOP-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-IN-LOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-IN-LOOP:       vector.ph:
-; CHECK-IN-LOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-IN-LOOP-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-IN-LOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
-; CHECK-IN-LOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-IN-LOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-IN-LOOP-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-IN-LOOP-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -213,18 +183,16 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
 ; CHECK-IN-LOOP-NEXT:    [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[TMP13]])
 ; CHECK-IN-LOOP-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]]
 ; CHECK-IN-LOOP-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-IN-LOOP-NEXT:    [[TMP17:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-IN-LOOP-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[TMP17]], i32 0
+; CHECK-IN-LOOP-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-IN-LOOP-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP17]], true
 ; CHECK-IN-LOOP-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-IN-LOOP:       middle.block:
 ; CHECK-IN-LOOP-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK-IN-LOOP:       scalar.ph:
-; CHECK-IN-LOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-IN-LOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-IN-LOOP-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK-IN-LOOP:       while.body:
-; CHECK-IN-LOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-IN-LOOP-NEXT:    [[RED:%.*]] = phi float [ [[RED_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-IN-LOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-IN-LOOP-NEXT:    [[RED:%.*]] = phi float [ [[RED_NEXT:%.*]], [[WHILE_BODY]] ], [ 0.000000e+00, [[SCALAR_PH]] ]
 ; CHECK-IN-LOOP-NEXT:    [[GEP:%.*]] = getelementptr float, ptr [[PTR]], i64 [[INDEX]]
 ; CHECK-IN-LOOP-NEXT:    [[VAL:%.*]] = load float, ptr [[GEP]], align 4
 ; CHECK-IN-LOOP-NEXT:    [[RED_NEXT]] = fadd float [[RED]], [[VAL]]
@@ -257,17 +225,11 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N:%.*]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
@@ -287,19 +249,17 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
 ; CHECK-NEXT:    [[TMP20]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x i32> [[VEC_PHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP22]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP23:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <vscale x 4 x i1> [[TMP23]], i32 0
-; CHECK-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP16]], true
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP20]])
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 7, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ 7, [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP26]], 5
@@ -322,17 +282,11 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
 ; CHECK-IN-LOOP-NEXT:  entry:
 ; CHECK-IN-LOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-IN-LOOP:       vector.ph:
-; CHECK-IN-LOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-IN-LOOP-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-IN-LOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]]
-; CHECK-IN-LOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-IN-LOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-IN-LOOP-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; CHECK-IN-LOOP-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
+; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
+; CHECK-IN-LOOP-NEXT:    [[TMP7:%.*]] = sub i64 [[N:%.*]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-IN-LOOP-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
@@ -352,18 +306,16 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
 ; CHECK-IN-LOOP-NEXT:    [[TMP19]] = xor i32 [[VEC_PHI]], [[TMP18]]
 ; CHECK-IN-LOOP-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP21]]
 ; CHECK-IN-LOOP-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-IN-LOOP-NEXT:    [[TMP22:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-IN-LOOP-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 4 x i1> [[TMP22]], i32 0
+; CHECK-IN-LOOP-NEXT:    [[TMP22:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-IN-LOOP-NEXT:    [[TMP23:%.*]] = xor i1 [[TMP22]], true
 ; CHECK-IN-LOOP-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK-IN-LOOP:       middle.block:
 ; CHECK-IN-LOOP-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK-IN-LOOP:       scalar.ph:
-; CHECK-IN-LOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-IN-LOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 7, [[ENTRY]] ]
 ; CHECK-IN-LOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-IN-LOOP:       for.body:
-; CHECK-IN-LOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-IN-LOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ]
+; CHECK-IN-LOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-IN-LOOP-NEXT:    [[RDX:%.*]] = phi i32 [ 7, [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ]
 ; CHECK-IN-LOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[IV]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP24:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-IN-LOOP-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP24]], 5
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
index 4ec7d4d873200..90224caa68cd4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
@@ -10,24 +10,18 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP61:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP62:%.*]] = mul nuw i64 [[TMP61]], 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP26:%.*]] = shl nuw i64 [[TMP25]], 2
 ; CHECK-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 8
+; CHECK-NEXT:    [[TMP28:%.*]] = shl nuw i64 [[TMP27]], 3
 ; CHECK-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP28]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP30:%.*]] = mul nuw i64 [[TMP29]], 12
@@ -47,10 +41,10 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK9:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT13:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX6]]
 ; CHECK-NEXT:    [[TMP52:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP53:%.*]] = mul nuw i64 [[TMP52]], 4
+; CHECK-NEXT:    [[TMP53:%.*]] = shl nuw i64 [[TMP52]], 2
 ; CHECK-NEXT:    [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP53]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP56:%.*]] = mul nuw i64 [[TMP55]], 8
+; CHECK-NEXT:    [[TMP56:%.*]] = shl nuw i64 [[TMP55]], 3
 ; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP56]]
 ; CHECK-NEXT:    [[TMP58:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP59:%.*]] = mul nuw i64 [[TMP58]], 12
@@ -61,10 +55,10 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP60]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK9]])
 ; CHECK-NEXT:    [[INDEX_NEXT10]] = add i64 [[INDEX6]], [[TMP62]]
 ; CHECK-NEXT:    [[TMP63:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP64:%.*]] = mul nuw i64 [[TMP63]], 4
+; CHECK-NEXT:    [[TMP64:%.*]] = shl nuw i64 [[TMP63]], 2
 ; CHECK-NEXT:    [[TMP65:%.*]] = add i64 [[INDEX6]], [[TMP64]]
 ; CHECK-NEXT:    [[TMP66:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP67:%.*]] = mul nuw i64 [[TMP66]], 8
+; CHECK-NEXT:    [[TMP67:%.*]] = shl nuw i64 [[TMP66]], 3
 ; CHECK-NEXT:    [[TMP68:%.*]] = add i64 [[INDEX6]], [[TMP67]]
 ; CHECK-NEXT:    [[TMP69:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP70:%.*]] = mul nuw i64 [[TMP69]], 12
@@ -73,9 +67,9 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP68]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT13]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP71]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP72:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP76:%.*]] = extractelement <vscale x 4 x i1> [[TMP72]], i32 0
-; CHECK-NEXT:    br i1 [[TMP76]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP36:%.*]] = xor i1 [[TMP35]], true
+; CHECK-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
@@ -101,24 +95,18 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP83:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP84:%.*]] = mul nuw i64 [[TMP83]], 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP26:%.*]] = shl nuw i64 [[TMP25]], 2
 ; CHECK-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 8
+; CHECK-NEXT:    [[TMP28:%.*]] = shl nuw i64 [[TMP27]], 3
 ; CHECK-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP28]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP30:%.*]] = mul nuw i64 [[TMP29]], 12
@@ -138,10 +126,10 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK9:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY5]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT16:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr i32, ptr [[COND_PTR:%.*]], i64 [[INDEX6]]
 ; CHECK-NEXT:    [[TMP52:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP53:%.*]] = mul nuw i64 [[TMP52]], 4
+; CHECK-NEXT:    [[TMP53:%.*]] = shl nuw i64 [[TMP52]], 2
 ; CHECK-NEXT:    [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP53]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP56:%.*]] = mul nuw i64 [[TMP55]], 8
+; CHECK-NEXT:    [[TMP56:%.*]] = shl nuw i64 [[TMP55]], 3
 ; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP56]]
 ; CHECK-NEXT:    [[TMP58:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP59:%.*]] = mul nuw i64 [[TMP58]], 12
@@ -160,10 +148,10 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    [[TMP72:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK9]], <vscale x 4 x i1> [[TMP64]], <vscale x 4 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX6]]
 ; CHECK-NEXT:    [[TMP74:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP75:%.*]] = mul nuw i64 [[TMP74]], 4
+; CHECK-NEXT:    [[TMP75:%.*]] = shl nuw i64 [[TMP74]], 2
 ; CHECK-NEXT:    [[TMP76:%.*]] = getelementptr i32, ptr [[TMP65]], i64 [[TMP75]]
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP78:%.*]] = mul nuw i64 [[TMP77]], 8
+; CHECK-NEXT:    [[TMP78:%.*]] = shl nuw i64 [[TMP77]], 3
 ; CHECK-NEXT:    [[TMP79:%.*]] = getelementptr i32, ptr [[TMP65]], i64 [[TMP78]]
 ; CHECK-NEXT:    [[TMP80:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP81:%.*]] = mul nuw i64 [[TMP80]], 12
@@ -172,12 +160,12 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP76]], i32 4, <vscale x 4 x i1> [[TMP70]])
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP79]], i32 4, <vscale x 4 x i1> [[TMP71]])
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP82]], i32 4, <vscale x 4 x i1> [[TMP72]])
-; CHECK-NEXT:    [[INDEX_NEXT13]] = add i64 [[INDEX6]], [[TMP84]]
+; CHECK-NEXT:    [[INDEX_NEXT13]] = add i64 [[INDEX6]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP85:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP86:%.*]] = mul nuw i64 [[TMP85]], 4
+; CHECK-NEXT:    [[TMP86:%.*]] = shl nuw i64 [[TMP85]], 2
 ; CHECK-NEXT:    [[TMP87:%.*]] = add i64 [[INDEX6]], [[TMP86]]
 ; CHECK-NEXT:    [[TMP88:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP89:%.*]] = mul nuw i64 [[TMP88]], 8
+; CHECK-NEXT:    [[TMP89:%.*]] = shl nuw i64 [[TMP88]], 3
 ; CHECK-NEXT:    [[TMP90:%.*]] = add i64 [[INDEX6]], [[TMP89]]
 ; CHECK-NEXT:    [[TMP91:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP92:%.*]] = mul nuw i64 [[TMP91]], 12
@@ -186,9 +174,9 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT15]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP90]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP93]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP94:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <vscale x 4 x i1> [[TMP94]], i32 0
-; CHECK-NEXT:    br i1 [[TMP98]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP66:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP67:%.*]] = xor i1 [[TMP66]], true
+; CHECK-NEXT:    br i1 [[TMP67]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
index 672523edf3d4f..4cd7dac62e79f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
@@ -12,14 +12,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -32,11 +26,11 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP13:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 4 x i1> [[TMP13]], i32 0
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i1 [[TMP13]], true
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
@@ -63,9 +57,6 @@ define void @simple_memset_v4i32(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], 3
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[UMAX]], 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i64 [[UMAX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i64 [[TMP0]], i64 0
@@ -80,9 +71,9 @@ define void @simple_memset_v4i32(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP4]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
 ; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], 4
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX1]], i64 [[TMP2]])
-; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
@@ -111,14 +102,8 @@ define void @simple_memcpy(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -131,11 +116,11 @@ define void @simple_memcpy(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[DST:%.*]], i64 [[INDEX1]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], ptr [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[TMP15]], i32 0
-; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i1 [[TMP12]], true
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
@@ -169,14 +154,8 @@ define void @copy_stride4(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP4]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP2]], [[TMP5]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP8]], 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp ugt i64 [[TMP2]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
@@ -184,7 +163,7 @@ define void @copy_stride4(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NEXT:    [[TMP13:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP15:%.*]] = mul <vscale x 4 x i64> [[TMP13]], splat (i64 4)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP15]]
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 4, [[TMP7]]
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 4, [[TMP4]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP18]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -196,11 +175,11 @@ define void @copy_stride4(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP19]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i32, ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x ptr> [[TMP20]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP7]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP4]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP12]])
-; CHECK-NEXT:    [[TMP21:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP22:%.*]] = xor i1 [[TMP21]], true
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <vscale x 4 x i1> [[TMP21]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
@@ -232,14 +211,8 @@ define void @simple_gather_scatter(ptr noalias %dst, ptr noalias %src, ptr noali
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -254,10 +227,10 @@ define void @simple_gather_scatter(ptr noalias %dst, ptr noalias %src, ptr noali
 ; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP13]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[DST:%.*]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x ptr> [[TMP14]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP15]], true
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
@@ -292,15 +265,9 @@ define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) #
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N:%.*]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
@@ -313,11 +280,11 @@ define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) #
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP14:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[TMP14]], i32 0
-; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
@@ -351,15 +318,9 @@ define void @cond_uniform_load(ptr noalias %dst, ptr noalias readonly %src, ptr
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N:%.*]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
@@ -377,11 +338,11 @@ define void @cond_uniform_load(ptr noalias %dst, ptr noalias readonly %src, ptr
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX1]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[PREDPHI]], ptr [[TMP16]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP18:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <vscale x 4 x i1> [[TMP18]], i32 0
-; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP17]], true
+; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
@@ -423,15 +384,9 @@ define void @uniform_store(ptr noalias %dst, ptr noalias readonly %src, i64 %n)
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N:%.*]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]])
@@ -444,11 +399,11 @@ define void @uniform_store(ptr noalias %dst, ptr noalias readonly %src, i64 %n)
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x ptr> [[BROADCAST_SPLAT]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP13:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 4 x i1> [[TMP13]], i32 0
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = xor i1 [[TMP13]], true
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
@@ -479,14 +434,8 @@ define void @simple_fdiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -501,11 +450,11 @@ define void @simple_fdiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 4 x float> @llvm.masked.load.nxv4f32.p0(ptr [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x float> poison)
 ; CHECK-NEXT:    [[TMP15:%.*]] = fdiv <vscale x 4 x float> [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD2]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float> [[TMP15]], ptr [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP16:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x i1> [[TMP16]], i32 0
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = xor i1 [[TMP13]], true
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
@@ -539,14 +488,8 @@ define void @simple_idiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -562,11 +505,11 @@ define void @simple_idiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NEXT:    [[TMP15:%.*]] = select <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD2]], <vscale x 4 x i32> splat (i32 1)
 ; CHECK-NEXT:    [[TMP16:%.*]] = udiv <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[TMP15]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP16]], ptr [[TMP12]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP1]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; CHECK-NEXT:    [[TMP17:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 4 x i1> [[TMP17]], i32 0
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP14]], true
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
@@ -594,15 +537,13 @@ define void @simple_memset_trip1024(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-LABEL: @simple_memset_trip1024(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -610,7 +551,7 @@ define void @simple_memset_trip1024(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
index 33fa3607730da..9989209f8c575 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
@@ -19,13 +19,11 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 3
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP4]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -98,13 +96,11 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP4]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 3
 ; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP7]], 4
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP9]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll
index 352f4fe3dae21..8dafa3453dcf5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll
@@ -10,13 +10,11 @@ define void @vscale_mul_4(ptr noalias noundef readonly captures(none) %a, ptr no
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[A]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[B]], align 4
 ; CHECK-NEXT:    [[TMP10:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
@@ -65,27 +63,25 @@ define  void @vscale_mul_8(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[MUL1:%.*]] = shl nuw nsw i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[A]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 2
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[B]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[TMP21]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[B]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP22]], align 4
@@ -133,7 +129,7 @@ define void @vscale_mul_12(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[TMP0]], 12
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[MUL1]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -141,8 +137,6 @@ define void @vscale_mul_12(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -152,7 +146,7 @@ define void @vscale_mul_12(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP11]], ptr [[TMP9]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -203,7 +197,7 @@ define void @vscale_mul_31(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[TMP0]], 31
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[MUL1]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -211,31 +205,29 @@ define void @vscale_mul_31(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 2
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 2
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP20]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP12]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP21]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -286,7 +278,7 @@ define void @vscale_mul_64(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[TMP0]], 64
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[MUL1]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -294,31 +286,29 @@ define void @vscale_mul_64(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = shl nuw i64 [[TMP9]], 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = shl nuw i64 [[TMP14]], 2
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP20:%.*]] = shl nuw i64 [[TMP19]], 2
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP20]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP12]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP21]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -371,7 +361,7 @@ define void @trip_count_with_overflow(ptr noalias noundef readonly captures(none
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -379,31 +369,29 @@ define void @trip_count_with_overflow(ptr noalias noundef readonly captures(none
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP17]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP21]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP13]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP19]], ptr [[TMP22]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -455,36 +443,34 @@ define void @trip_count_too_big_for_element_count(ptr noalias noundef readonly c
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 32
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[TMP8]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP12]], align 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP17]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 2
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP21]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP13]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP19]], ptr [[TMP22]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll
index e214e82e782a6..0c6a490ddf4ba 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-extractvalue.ll
@@ -8,7 +8,7 @@ define void @widen_extractvalue(ptr %dst, {i64, i64} %sv) #0 {
 ; CHECK-SAME: ptr [[DST:%.*]], { i64, i64 } [[SV:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1000, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -16,8 +16,6 @@ define void @widen_extractvalue(ptr %dst, {i64, i64} %sv) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1000, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 1000, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 2
 ; CHECK-NEXT:    [[EXTRACT0:%.*]] = extractvalue { i64, i64 } [[SV]], 0
 ; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[EXTRACT0]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
@@ -30,7 +28,7 @@ define void @widen_extractvalue(ptr %dst, {i64, i64} %sv) #0 {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[DST]], i32 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index 9929f35d47dac..ef111caafbf0e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -18,7 +18,7 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK-LABEL: @pointer_induction_used_as_vector(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -26,8 +26,6 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[IND_END2:%.*]] = getelementptr i8, ptr [[START_2:%.*]], i64 [[N_VEC]]
@@ -35,20 +33,18 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 1, [[TMP6]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul <vscale x 2 x i64> [[TMP13]], splat (i64 1)
-; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP15]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP7]]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START_1]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 1
-; CHECK-NEXT:    store <vscale x 2 x ptr> [[TMP16]], ptr [[NEXT_GEP]], align 8
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 2 x ptr> [[VECTOR_GEP]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP18]], align 1
-; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 2 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT:    store <vscale x 2 x i8> [[TMP20]], ptr [[TMP18]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, <vscale x 2 x ptr> [[VECTOR_GEP]], i64 1
+; CHECK-NEXT:    store <vscale x 2 x ptr> [[TMP9]], ptr [[NEXT_GEP]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <vscale x 2 x ptr> [[VECTOR_GEP]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP10]], align 1
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i8> [[WIDE_LOAD]], splat (i8 1)
+; CHECK-NEXT:    store <vscale x 2 x i8> [[TMP12]], ptr [[TMP10]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -104,7 +100,7 @@ define void @pointer_induction(ptr noalias %start, i64 %N) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N:%.*]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -112,23 +108,19 @@ define void @pointer_induction(ptr noalias %start, i64 %N) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 1, [[TMP6]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul <vscale x 2 x i64> [[TMP12]], splat (i64 1)
-; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP14]]
+; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP12]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 2 x ptr> [[VECTOR_GEP]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP15]], align 1
 ; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 2 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; CHECK-NEXT:    store <vscale x 2 x i8> [[TMP17]], ptr [[TMP15]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX2]], [[TMP6]]
-; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP10]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX2]], [[TMP4]]
+; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll
index 6947884efb699..11eef23f99f8c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll
@@ -22,11 +22,10 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -8
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = shl i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[TMP26]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -134,11 +133,10 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -8
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 3
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP6]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[N_VEC]], 2
@@ -225,11 +223,10 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp samesign ult i64 [[SMAX]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -2
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 1
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP6]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[SMAX]], [[DOTNOT]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[N_VEC]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[N_VEC]], 3
@@ -239,7 +236,6 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP5]], 3
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = shl <vscale x 2 x i64> [[TMP9]], splat (i64 2)
 ; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP10]]
@@ -250,6 +246,7 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 {
 ; CHECK-NEXT:    [[TMP12]] = add <vscale x 2 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
 ; CHECK-NEXT:    store <vscale x 2 x ptr> [[VECTOR_GEP]], ptr [[NEXT_GEP]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP5]], 3
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
@@ -313,7 +310,6 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr %
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[PTR:%.*]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl <vscale x 2 x i64> [[TMP4]], splat (i64 1)
 ; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP5]]
@@ -321,6 +317,7 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr %
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <vscale x 2 x ptr> [[VECTOR_GEP]], i64 0
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv2i16.p0(<vscale x 2 x i16> zeroinitializer, ptr [[TMP7]], i32 2, <vscale x 2 x i1> [[TMP6]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw nsw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP3]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index 5848d317ff145..c54511e957ef8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -22,8 +22,6 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -32,7 +30,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    [[TMP14:%.*]] = zext <vscale x 4 x i32> [[WIDE_LOAD1]] to <vscale x 4 x i64>
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 4 x i64> [[TMP14]]
 ; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv4p0.i32(<vscale x 4 x ptr> [[TMP15]], i32 1, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -49,8 +47,6 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP25]]
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
-; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP17:%.*]] = shl nuw i64 [[TMP16]], 1
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[FOR_BODY]] ]
@@ -59,7 +55,7 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    [[TMP19:%.*]] = zext <vscale x 2 x i32> [[WIDE_LOAD5]] to <vscale x 2 x i64>
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[BUCKETS]], <vscale x 2 x i64> [[TMP19]]
 ; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.nxv2p0.i32(<vscale x 2 x ptr> [[TMP20]], i32 1, <vscale x 2 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP17]]
+; CHECK-NEXT:    [[INDEX_NEXT6]] = add nuw i64 [[INDEX4]], [[TMP25]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]]
 ; CHECK-NEXT:    br i1 [[TMP21]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       vec.epilog.middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
index abee8b9340caa..baf050c7facee 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
@@ -70,10 +70,8 @@ define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %othe
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP4]], 2
 ; NORMAL_DEP_LIMIT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NORMAL_DEP_LIMIT-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NORMAL_DEP_LIMIT-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NORMAL_DEP_LIMIT-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; NORMAL_DEP_LIMIT-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP6]] to i32
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP8]] to i32
 ; NORMAL_DEP_LIMIT-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0
 ; NORMAL_DEP_LIMIT-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NORMAL_DEP_LIMIT-NEXT:    br label [[FOR_BODY:%.*]]
@@ -91,7 +89,7 @@ define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %othe
 ; NORMAL_DEP_LIMIT-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4, !alias.scope [[META7:![0-9]+]], !noalias [[META0]]
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP15:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD10]], [[VEC_IND]]
 ; NORMAL_DEP_LIMIT-NEXT:    store <vscale x 4 x i32> [[TMP15]], ptr [[TMP14]], align 4, !alias.scope [[META7]], !noalias [[META0]]
-; NORMAL_DEP_LIMIT-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP6]]
+; NORMAL_DEP_LIMIT-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NORMAL_DEP_LIMIT-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
 ; NORMAL_DEP_LIMIT-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
index 9257e45c809e9..3b19e9ee1a5a3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt.ll
@@ -36,11 +36,10 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -86,11 +85,10 @@ define void @simple_histogram_inc_param(ptr noalias %buckets, ptr readonly %indi
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -136,11 +134,10 @@ define void @simple_histogram_sub(ptr noalias %buckets, ptr readonly %indices, i
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -186,11 +183,10 @@ define void @conditional_histogram(ptr noalias %buckets, ptr readonly %indices,
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
@@ -248,11 +244,10 @@ define void @histogram_8bit(ptr noalias %buckets, ptr readonly %indices, i64 %N)
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP9]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -385,11 +380,10 @@ define void @simple_histogram_user_interleave(ptr noalias %buckets, ptr readonly
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -8
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 3
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -446,11 +440,10 @@ define void @histogram_array_3op_gep(i64 noundef %N) #0 {
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -500,11 +493,10 @@ define void @histogram_array_4op_gep_nonzero_const_idx(i64 noundef %N, ptr reado
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -611,11 +603,10 @@ define void @simple_histogram_rtdepcheck(ptr noalias %buckets, ptr %array, ptr %
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], [[TMP4]]
 ; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP6]], -4
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP8]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; CHECK-NEXT:    [[TMP11:%.*]] = trunc nuw nsw i64 [[TMP8]] to i32
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
@@ -714,11 +705,10 @@ define void @simple_histogram_64b(ptr noalias %buckets, ptr readonly %indices, i
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -2
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNEG]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 1
+; CHECK-NEXT:    [[DOTNOT:%.*]] = sub nsw i64 0, [[TMP4]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], [[DOTNOT]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
index 3b04df3fec5e0..e450fe7b54d43 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll
@@ -319,10 +319,9 @@ define void @test_v4_v4m(ptr noalias %a, ptr readonly %b) #3 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR1:[0-9]+]]
@@ -372,10 +371,9 @@ define void @test_v2_v4m(ptr noalias %a, ptr readonly %b) #3 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR2:[0-9]+]]
@@ -425,10 +423,9 @@ define void @test_v2_v4(ptr noalias %a, ptr readonly %b) #3 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR3:[0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/tail-fold-uniform-memops.ll b/llvm/test/Transforms/LoopVectorize/AArch64/tail-fold-uniform-memops.ll
index e55e3222bc5b0..150b79c448005 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/tail-fold-uniform-memops.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/tail-fold-uniform-memops.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
 ; RUN: opt -passes=loop-vectorize -scalable-vectorization=off -force-vector-width=4 -prefer-predicate-over-epilogue=predicate-dont-vectorize -S < %s | FileCheck %s
 
 ; NOTE: These tests aren't really target-specific, but it's convenient to target AArch64
@@ -8,26 +9,33 @@ target triple = "aarch64-linux-gnu"
 ; The original loop had an unconditional uniform load. Let's make sure
 ; we don't artificially create new predicated blocks for the load.
 define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) #0 {
-; CHECK-LABEL: @uniform_load(
-; CHECK:       vector.ph:
-; CHECK:         [[N_MINUS_VF:%.*]] = sub i64 %n, [[VSCALE_X_VF:.*]]
-; CHECK:         [[CMP:%.*]] = icmp ugt i64 %n, [[VSCALE_X_VF]]
-; CHECK:         [[N2:%.*]] = select i1 [[CMP]], i64 [[N_MINUS_VF]], i64 0
-; CHECK:         [[INIT_ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 %n)
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[IDX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[IDX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[INIT_ACTIVE_LANE_MASK]], %vector.ph ], [ [[NEXT_ACTIVE_LANE_MASK:%.*]], %vector.body ]
-; CHECK-NEXT:    [[LOAD_VAL:%.*]] = load i32, ptr %src, align 4
-; CHECK-NOT:     load i32, ptr %src, align 4
+; CHECK-LABEL: define void @uniform_load(
+; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias readonly [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MINUS_VF:%.*]] = sub i64 [[N]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[N]], 4
+; CHECK-NEXT:    [[N2:%.*]] = select i1 [[CMP]], i64 [[N_MINUS_VF]], i64 0
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[IDX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[NEXT_ACTIVE_LANE_MASK:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[LOAD_VAL:%.*]] = load i32, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> poison, i32 [[LOAD_VAL]], i64 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr %dst, i64 [[IDX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IDX]]
 ; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[TMP5]], ptr [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
-; CHECK-NEXT:    [[IDX_NEXT]] = add i64 [[IDX]], 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[IDX]], 4
 ; CHECK-NEXT:    [[NEXT_ACTIVE_LANE_MASK]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[IDX]], i64 [[N2]])
-; CHECK-NEXT:    [[NOT_ACTIVE_LANE_MASK:%.*]] = xor <4 x i1> [[NEXT_ACTIVE_LANE_MASK]], splat (i1 true)
-; CHECK-NEXT:    [[FIRST_LANE_SET:%.*]] = extractelement <4 x i1> [[NOT_ACTIVE_LANE_MASK]], i32 0
-; CHECK-NEXT:    br i1 [[FIRST_LANE_SET]], label %middle.block, label %vector.body
+; CHECK-NEXT:    [[EXTRACT_FIRST_LANE_MASK:%.*]] = extractelement <4 x i1> [[NEXT_ACTIVE_LANE_MASK]], i32 0
+; CHECK-NEXT:    [[FIRST_LANE_SET:%.*]] = xor i1 [[EXTRACT_FIRST_LANE_MASK]], true
+; CHECK-NEXT:    br i1 [[FIRST_LANE_SET]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[FOR_END:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
 
 entry:
   br label %for.body
@@ -49,19 +57,39 @@ for.end:                                          ; preds = %for.body, %entry
 ; do need to perform conditional loads and so we end up using a gather instead.
 ; However, we at least ensure the mask is the overlap of the loop predicate
 ; and the original condition.
-define void @cond_uniform_load(ptr nocapture %dst, ptr nocapture readonly %src, ptr nocapture readonly %cond, i64 %n) #0 {
-; CHECK-LABEL: @cond_uniform_load(
-; CHECK:       vector.ph:
-; CHECK:         [[INIT_ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 %n)
-; CHECK:         [[TMP1:%.*]] = insertelement <4 x ptr> poison, ptr %src, i64 0
+define void @cond_uniform_load(ptr noalias nocapture %dst, ptr nocapture readonly %src, ptr nocapture readonly %cond, i64 %n) #0 {
+; CHECK-LABEL: define void @cond_uniform_load(
+; CHECK-SAME: ptr noalias captures(none) [[DST:%.*]], ptr readonly captures(none) [[SRC:%.*]], ptr readonly captures(none) [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[N]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i64 [[N]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = select i1 [[TMP2]], i64 [[TMP5]], i64 0
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 [[N]])
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x ptr> poison, ptr [[SRC]], i64 0
 ; CHECK-NEXT:    [[SRC_SPLAT:%.*]] = shufflevector <4 x ptr> [[TMP1]], <4 x ptr> poison, <4 x i32> zeroinitializer
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[IDX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[IDX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[INIT_ACTIVE_LANE_MASK]], %vector.ph ], [ [[NEXT_ACTIVE_LANE_MASK:%.*]], %vector.body ]
-; CHECK:         [[COND_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr {{%.*}}, i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX6:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX6]]
+; CHECK-NEXT:    [[COND_LOAD:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[TMP6]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i32> poison)
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i32> [[COND_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[MASK:%.*]] = select <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
-; CHECK-NEXT:    call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[SRC_SPLAT]], i32 4, <4 x i1> [[MASK]], <4 x i32> poison)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[SRC_SPLAT]], i32 4, <4 x i1> [[MASK]], <4 x i32> poison)
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[MASK]], <4 x i32> [[WIDE_MASKED_GATHER]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX6]]
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[PREDPHI]], ptr [[TMP7]], i32 4, <4 x i1> [[ACTIVE_LANE_MASK]])
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX6]], 4
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 [[INDEX6]], i64 [[TMP3]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = xor i1 [[TMP8]], true
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[FOR_END:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
 entry:
   br label %for.body
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
index a11896a288000..24d23d81b1f92 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
@@ -14,7 +14,7 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; NONE-NEXT:  entry:
 ; NONE-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; NONE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NONE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NONE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NONE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
 ; NONE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NONE:       vector.ph:
@@ -22,8 +22,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; NONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP3]]
 ; NONE-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
-; NONE-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NONE-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NONE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; NONE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NONE-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -31,7 +29,7 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; NONE-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; NONE-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]
 ; NONE-NEXT:    store <vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP5]], align 4
-; NONE-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP8]]
+; NONE-NEXT:    [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP3]]
 ; NONE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
 ; NONE-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NONE:       middle.block:
@@ -61,8 +59,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]]
 ; DATA-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; DATA-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; DATA-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; DATA-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; DATA-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; DATA-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; DATA-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -71,16 +67,15 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[UMAX]])
 ; DATA-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]
 ; DATA-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP10]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; DATA-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP13]]
+; DATA-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP5]]
 ; DATA-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
 ; DATA-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DATA:       middle.block:
 ; DATA-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; DATA:       scalar.ph:
-; DATA-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; DATA-NEXT:    br label [[WHILE_BODY:%.*]]
 ; DATA:       while.body:
-; DATA-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; DATA-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; DATA-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
 ; DATA-NEXT:    store i32 [[VAL]], ptr [[GEP]], align 4
 ; DATA-NEXT:    [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
@@ -100,8 +95,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_NO_LANEMASK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]]
 ; DATA_NO_LANEMASK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; DATA_NO_LANEMASK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; DATA_NO_LANEMASK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_NO_LANEMASK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; DATA_NO_LANEMASK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[UMAX]], 1
 ; DATA_NO_LANEMASK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; DATA_NO_LANEMASK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
@@ -118,16 +111,15 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_NO_LANEMASK-NEXT:    [[TMP12:%.*]] = icmp ule <vscale x 4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
 ; DATA_NO_LANEMASK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]
 ; DATA_NO_LANEMASK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT5]], ptr [[TMP13]], i32 4, <vscale x 4 x i1> [[TMP12]])
-; DATA_NO_LANEMASK-NEXT:    [[INDEX_NEXT6]] = add i64 [[INDEX1]], [[TMP16]]
+; DATA_NO_LANEMASK-NEXT:    [[INDEX_NEXT6]] = add i64 [[INDEX1]], [[TMP5]]
 ; DATA_NO_LANEMASK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC]]
 ; DATA_NO_LANEMASK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DATA_NO_LANEMASK:       middle.block:
 ; DATA_NO_LANEMASK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; DATA_NO_LANEMASK:       scalar.ph:
-; DATA_NO_LANEMASK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; DATA_NO_LANEMASK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; DATA_NO_LANEMASK:       while.body:
-; DATA_NO_LANEMASK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; DATA_NO_LANEMASK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; DATA_NO_LANEMASK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
 ; DATA_NO_LANEMASK-NEXT:    store i32 [[VAL]], ptr [[GEP]], align 4
 ; DATA_NO_LANEMASK-NEXT:    [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
@@ -143,12 +135,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_AND_CONTROL:       vector.ph:
 ; DATA_AND_CONTROL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; DATA_AND_CONTROL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; DATA_AND_CONTROL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP5]], 1
-; DATA_AND_CONTROL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]]
-; DATA_AND_CONTROL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; DATA_AND_CONTROL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; DATA_AND_CONTROL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; DATA_AND_CONTROL-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; DATA_AND_CONTROL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; DATA_AND_CONTROL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -158,18 +144,17 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_AND_CONTROL-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; DATA_AND_CONTROL-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]
 ; DATA_AND_CONTROL-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP10]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; DATA_AND_CONTROL-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP13]]
+; DATA_AND_CONTROL-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP5]]
 ; DATA_AND_CONTROL-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]])
-; DATA_AND_CONTROL-NEXT:    [[TMP14:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; DATA_AND_CONTROL-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[TMP14]], i32 0
-; DATA_AND_CONTROL-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; DATA_AND_CONTROL-NEXT:    [[TMP6:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; DATA_AND_CONTROL-NEXT:    [[TMP7:%.*]] = xor i1 [[TMP6]], true
+; DATA_AND_CONTROL-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DATA_AND_CONTROL:       middle.block:
 ; DATA_AND_CONTROL-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; DATA_AND_CONTROL:       scalar.ph:
-; DATA_AND_CONTROL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; DATA_AND_CONTROL-NEXT:    br label [[WHILE_BODY:%.*]]
 ; DATA_AND_CONTROL:       while.body:
-; DATA_AND_CONTROL-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; DATA_AND_CONTROL-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; DATA_AND_CONTROL-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
 ; DATA_AND_CONTROL-NEXT:    store i32 [[VAL]], ptr [[GEP]], align 4
 ; DATA_AND_CONTROL-NEXT:    [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
@@ -185,14 +170,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_AND_CONTROL_NO_RT_CHECK:       vector.ph:
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 4
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -205,18 +184,17 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX1]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP11]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP14]]
+; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]])
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP15:%.*]] = xor <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], splat (i1 true)
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP16:%.*]] = extractelement <vscale x 4 x i1> [[TMP15]], i32 0
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 4 x i1> [[ACTIVE_LANE_MASK_NEXT]], i32 0
+; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP12:%.*]] = xor i1 [[TMP15]], true
+; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; DATA_AND_CONTROL_NO_RT_CHECK:       middle.block:
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; DATA_AND_CONTROL_NO_RT_CHECK:       scalar.ph:
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; DATA_AND_CONTROL_NO_RT_CHECK:       while.body:
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR]], i64 [[INDEX]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    store i32 [[VAL]], ptr [[GEP]], align 4
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
index d0ea8288ea126..a4173cb2781c1 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
@@ -8,7 +8,7 @@ target triple = "arm64-apple-macosx"
 define void @load_store_interleave_group_tc_2(ptr noalias %data) {
 ; VF2-LABEL: define void @load_store_interleave_group_tc_2(
 ; VF2-SAME: ptr noalias [[DATA:%.*]]) {
-; VF2-NEXT:  [[ENTRY:.*]]:
+; VF2-NEXT:  [[ENTRY:.*:]]
 ; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF2:       [[VECTOR_PH]]:
 ; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -23,10 +23,9 @@ define void @load_store_interleave_group_tc_2(ptr noalias %data) {
 ; VF2:       [[MIDDLE_BLOCK]]:
 ; VF2-NEXT:    br label %[[EXIT:.*]]
 ; VF2:       [[SCALAR_PH]]:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF2-NEXT:    br label %[[LOOP:.*]]
 ; VF2:       [[LOOP]]:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF2-NEXT:    [[MUL_2:%.*]] = shl nsw i64 [[IV]], 1
 ; VF2-NEXT:    [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[MUL_2]]
 ; VF2-NEXT:    [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
@@ -43,19 +42,14 @@ define void @load_store_interleave_group_tc_2(ptr noalias %data) {
 ;
 ; VF4-LABEL: define void @load_store_interleave_group_tc_2(
 ; VF4-SAME: ptr noalias [[DATA:%.*]]) {
-; VF4-NEXT:  [[ENTRY:.*]]:
+; VF4-NEXT:  [[ENTRY:.*:]]
 ; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF4:       [[VECTOR_PH]]:
 ; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; VF4:       [[VECTOR_BODY]]:
-; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE6:.*]] ]
-; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE6]] ]
-; VF4-NEXT:    [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 1)
-; VF4-NEXT:    [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
-; VF4-NEXT:    br i1 [[TMP1]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; VF4-NEXT:    br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; VF4:       [[PRED_STORE_IF]]:
-; VF4-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 0
-; VF4-NEXT:    [[TMP3:%.*]] = shl nsw i64 [[TMP2]], 1
+; VF4-NEXT:    [[TMP3:%.*]] = shl nsw i64 0, 1
 ; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP3]]
 ; VF4-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 8
 ; VF4-NEXT:    store i64 [[TMP5]], ptr [[TMP4]], align 8
@@ -65,11 +59,9 @@ define void @load_store_interleave_group_tc_2(ptr noalias %data) {
 ; VF4-NEXT:    store i64 [[TMP8]], ptr [[TMP7]], align 8
 ; VF4-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; VF4:       [[PRED_STORE_CONTINUE]]:
-; VF4-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
-; VF4-NEXT:    br i1 [[TMP9]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
+; VF4-NEXT:    br i1 true, label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2:.*]]
 ; VF4:       [[PRED_STORE_IF1]]:
-; VF4-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 1
-; VF4-NEXT:    [[TMP11:%.*]] = shl nsw i64 [[TMP10]], 1
+; VF4-NEXT:    [[TMP11:%.*]] = shl nsw i64 1, 1
 ; VF4-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP11]]
 ; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP12]], align 8
 ; VF4-NEXT:    store i64 [[TMP13]], ptr [[TMP12]], align 8
@@ -79,11 +71,9 @@ define void @load_store_interleave_group_tc_2(ptr noalias %data) {
 ; VF4-NEXT:    store i64 [[TMP16]], ptr [[TMP15]], align 8
 ; VF4-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
 ; VF4:       [[PRED_STORE_CONTINUE2]]:
-; VF4-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
-; VF4-NEXT:    br i1 [[TMP17]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
+; VF4-NEXT:    br i1 false, label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4:.*]]
 ; VF4:       [[PRED_STORE_IF3]]:
-; VF4-NEXT:    [[TMP18:%.*]] = add i64 [[INDEX]], 2
-; VF4-NEXT:    [[TMP19:%.*]] = shl nsw i64 [[TMP18]], 1
+; VF4-NEXT:    [[TMP19:%.*]] = shl nsw i64 2, 1
 ; VF4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP19]]
 ; VF4-NEXT:    [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8
 ; VF4-NEXT:    store i64 [[TMP21]], ptr [[TMP20]], align 8
@@ -93,11 +83,9 @@ define void @load_store_interleave_group_tc_2(ptr noalias %data) {
 ; VF4-NEXT:    store i64 [[TMP24]], ptr [[TMP23]], align 8
 ; VF4-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
 ; VF4:       [[PRED_STORE_CONTINUE4]]:
-; VF4-NEXT:    [[TMP25:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
-; VF4-NEXT:    br i1 [[TMP25]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6]]
+; VF4-NEXT:    br i1 false, label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
 ; VF4:       [[PRED_STORE_IF5]]:
-; VF4-NEXT:    [[TMP26:%.*]] = add i64 [[INDEX]], 3
-; VF4-NEXT:    [[TMP27:%.*]] = shl nsw i64 [[TMP26]], 1
+; VF4-NEXT:    [[TMP27:%.*]] = shl nsw i64 3, 1
 ; VF4-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP27]]
 ; VF4-NEXT:    [[TMP29:%.*]] = load i64, ptr [[TMP28]], align 8
 ; VF4-NEXT:    store i64 [[TMP29]], ptr [[TMP28]], align 8
@@ -107,16 +95,13 @@ define void @load_store_interleave_group_tc_2(ptr noalias %data) {
 ; VF4-NEXT:    store i64 [[TMP32]], ptr [[TMP31]], align 8
 ; VF4-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
 ; VF4:       [[PRED_STORE_CONTINUE6]]:
-; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
-; VF4-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF4-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; VF4:       [[MIDDLE_BLOCK]]:
 ; VF4-NEXT:    br label %[[EXIT:.*]]
 ; VF4:       [[SCALAR_PH]]:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF4-NEXT:    br label %[[LOOP:.*]]
 ; VF4:       [[LOOP]]:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF4-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF4-NEXT:    [[MUL_2:%.*]] = shl nsw i64 [[IV]], 1
 ; VF4-NEXT:    [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[MUL_2]]
 ; VF4-NEXT:    [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
@@ -127,7 +112,7 @@ define void @load_store_interleave_group_tc_2(ptr noalias %data) {
 ; VF4-NEXT:    store i64 [[L_1]], ptr [[DATA_1]], align 8
 ; VF4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; VF4-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 2
-; VF4-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; VF4-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
 ; VF4:       [[EXIT]]:
 ; VF4-NEXT:    ret void
 ;
@@ -227,7 +212,7 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
 ;
 ; VF2-LABEL: define void @test_complex_add_float_tc_4(
 ; VF2-SAME: ptr [[RES:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
-; VF2-NEXT:  [[ENTRY:.*]]:
+; VF2-NEXT:  [[ENTRY:.*:]]
 ; VF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF2:       [[VECTOR_PH]]:
 ; VF2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -253,10 +238,9 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
 ; VF2:       [[MIDDLE_BLOCK]]:
 ; VF2-NEXT:    br label %[[EXIT:.*]]
 ; VF2:       [[SCALAR_PH]]:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF2-NEXT:    br label %[[LOOP:.*]]
 ; VF2:       [[LOOP]]:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF2-NEXT:    [[GEP_A_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[IV]]
 ; VF2-NEXT:    [[GEP_B_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[IV]]
 ; VF2-NEXT:    [[L_A_0:%.*]] = load float, ptr [[GEP_A_0]], align 4
@@ -279,7 +263,7 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
 ;
 ; VF4-LABEL: define void @test_complex_add_float_tc_4(
 ; VF4-SAME: ptr [[RES:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
-; VF4-NEXT:  [[ENTRY:.*]]:
+; VF4-NEXT:  [[ENTRY:.*:]]
 ; VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF4:       [[VECTOR_PH]]:
 ; VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -299,10 +283,9 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
 ; VF4:       [[MIDDLE_BLOCK]]:
 ; VF4-NEXT:    br label %[[EXIT:.*]]
 ; VF4:       [[SCALAR_PH]]:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF4-NEXT:    br label %[[LOOP:.*]]
 ; VF4:       [[LOOP]]:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF4-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF4-NEXT:    [[GEP_A_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[A]], i64 [[IV]]
 ; VF4-NEXT:    [[GEP_B_0:%.*]] = getelementptr inbounds nuw { float, float }, ptr [[B]], i64 [[IV]]
 ; VF4-NEXT:    [[L_A_0:%.*]] = load float, ptr [[GEP_A_0]], align 4
@@ -319,7 +302,7 @@ define void @test_complex_add_float_tc_4(ptr %res, ptr noalias %A, ptr noalias %
 ; VF4-NEXT:    store float [[ADD_1]], ptr [[GEP_RES_1]], align 4
 ; VF4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; VF4-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 4
-; VF4-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF4-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; VF4:       [[EXIT]]:
 ; VF4-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
new file mode 100644
index 0000000000000..7533636f9d41c
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-scalable.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 5
+; RUN: opt -p loop-vectorize -force-vector-interleave=1 -S -mcpu=neoverse-512tvb %s | FileCheck --check-prefixes=CHECK %s
+
+target triple = "aarch64-unknown-linux"
+
+define void @load_store_interleave_group(ptr noalias %data) {
+; CHECK-LABEL: define void @load_store_interleave_group(
+; CHECK-SAME: ptr noalias [[DATA:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP5]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[TMP0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]])
+; CHECK-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 100, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %mul.2 = shl nsw i64 %iv, 1
+  %data.0 = getelementptr inbounds i64, ptr %data, i64 %mul.2
+  %l.0 = load i64, ptr %data.0, align 8
+  store i64 %l.0, ptr %data.0, align 8
+  %add.1 = or disjoint i64 %mul.2, 1
+  %data.1 = getelementptr inbounds i64, ptr %data, i64 %add.1
+  %l.1 = load i64, ptr %data.1, align 8
+  store i64 %l.1, ptr %data.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 100
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @test_2xi64_unary_op_load_interleave_group(ptr noalias %data, ptr noalias %factor) {
+; CHECK-LABEL: define void @test_2xi64_unary_op_load_interleave_group(
+; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1111, [[TMP5]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1111, [[TMP3]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1111, [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds double, ptr [[DATA]], i64 [[TMP0]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x double>, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> [[WIDE_VEC]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = fneg <vscale x 2 x double> [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fneg <vscale x 2 x double> [[TMP7]]
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x double> [[TMP8]], <vscale x 2 x double> [[TMP9]])
+; CHECK-NEXT:    store <vscale x 4 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1111, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %1 = shl nsw i64 %iv, 1
+  %data.0 = getelementptr inbounds double, ptr %data, i64 %1
+  %l.0 = load double, ptr %data.0, align 8
+  %neg.0 = fneg double %l.0
+  store double %neg.0, ptr %data.0, align 8
+  %3 = or disjoint i64 %1, 1
+  %data.1 = getelementptr inbounds double, ptr %data, i64 %3
+  %l.1 = load double, ptr %data.1, align 8
+  %neg.1 = fneg double %l.1
+  store double %neg.1, ptr %data.1, align 8
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 1111
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll
index 4df02a78a4809..ee5f1929f41e4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-unroll.ll
@@ -7,7 +7,7 @@ target triple = "arm64-apple-macosx"
 define void @load_store_interleave_group(ptr noalias %data) {
 ; CHECK-LABEL: define void @load_store_interleave_group(
 ; CHECK-SAME: ptr noalias [[DATA:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -28,10 +28,9 @@ define void @load_store_interleave_group(ptr noalias %data) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[MUL_2:%.*]] = shl nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[DATA_0:%.*]] = getelementptr inbounds i64, ptr [[DATA]], i64 [[MUL_2]]
 ; CHECK-NEXT:    [[L_0:%.*]] = load i64, ptr [[DATA_0]], align 8
@@ -70,7 +69,7 @@ exit:
 define void @test_2xi64_with_wide_load(ptr noalias %data, ptr noalias %factor) {
 ; CHECK-LABEL: define void @test_2xi64_with_wide_load(
 ; CHECK-SAME: ptr noalias [[DATA:%.*]], ptr noalias [[FACTOR:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -107,10 +106,9 @@ define void @test_2xi64_with_wide_load(ptr noalias %data, ptr noalias %factor) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[FACTOR]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_FACTOR:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[TMP13:%.*]] = shl nsw i64 [[IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll
index a2cbf6f9c5a08..0ada7d0f22573 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-insertelt.ll
@@ -50,10 +50,9 @@ define void @test0(ptr noalias %M3, ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_INC1286_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[IF_THEN1165_US:%.*]]
 ; CHECK:       if.then1165.us:
-; CHECK-NEXT:    [[INDVARS_IV1783:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT1784:%.*]], [[IF_THEN1165_US]] ]
+; CHECK-NEXT:    [[INDVARS_IV1783:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT1784:%.*]], [[IF_THEN1165_US]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[INDVARS_IV1783]]
 ; CHECK-NEXT:    [[L_A:%.*]] = load i16, ptr [[GEP_A]], align 2
 ; CHECK-NEXT:    [[CONV1177_US:%.*]] = zext i16 [[L_A]] to i32
@@ -143,10 +142,9 @@ define void @test1(ptr noalias %M3, ptr noalias %A, ptr noalias %B, ptr noalias
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_INC1286_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[IF_THEN1165_US:%.*]]
 ; CHECK:       if.then1165.us:
-; CHECK-NEXT:    [[INDVARS_IV1783:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT1784:%.*]], [[IF_THEN1165_US]] ]
+; CHECK-NEXT:    [[INDVARS_IV1783:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT1784:%.*]], [[IF_THEN1165_US]] ]
 ; CHECK-NEXT:    [[FPTR:%.*]] = load i32, ptr [[C]], align 4
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[INDVARS_IV1783]]
 ; CHECK-NEXT:    [[L_A:%.*]] = load i16, ptr [[GEP_A]], align 2
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
index f6f2e39594dd8..64fc573e660bd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
@@ -1310,7 +1310,8 @@ define void @fma_f64(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; LIBMVEC-SVE-LABEL: define void @fma_f64
 ; LIBMVEC-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; LIBMVEC-SVE:    [[TMP2:%.*]] = call <2 x double> @llvm.fma.v2f64(<2 x double> [[WIDE_LOAD:%.*]], <2 x double> [[WIDE_LOAD]], <2 x double> [[WIDE_LOAD]])
+; LIBMVEC-SVE:    [[TMP8:%.*]] = call <vscale x 2 x double> @llvm.fma.nxv2f64(<vscale x 2 x double> [[WIDE_LOAD:%.*]], <vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD]])
+; LIBMVEC-SVE:    [[CALL:%.*]] = tail call double @llvm.fma.f64(double [[IN:%.*]], double [[IN]], double [[IN]])
 ;
 ; SLEEF-NEON-LABEL: define void @fma_f64
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
@@ -1357,7 +1358,8 @@ define void @fma_f32(ptr noalias %in.ptr, ptr %out.ptr) {
 ;
 ; LIBMVEC-SVE-LABEL: define void @fma_f32
 ; LIBMVEC-SVE-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
-; LIBMVEC-SVE:    [[TMP2:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[WIDE_LOAD:%.*]], <4 x float> [[WIDE_LOAD]], <4 x float> [[WIDE_LOAD]])
+; LIBMVEC-SVE:    [[TMP8:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD:%.*]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD]])
+; LIBMVEC-SVE:    [[CALL:%.*]] = tail call float @llvm.fma.f32(float [[IN:%.*]], float [[IN]], float [[IN]])
 ;
 ; SLEEF-NEON-LABEL: define void @fma_f32
 ; SLEEF-NEON-SAME: (ptr noalias [[IN_PTR:%.*]], ptr [[OUT_PTR:%.*]]) #[[ATTR1]] {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-backedge-elimination-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-backedge-elimination-epilogue.ll
index a431fdd3178bd..21928ce715007 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-backedge-elimination-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-loop-backedge-elimination-epilogue.ll
@@ -34,18 +34,15 @@ define void @test_remove_vector_loop_region_epilogue(ptr %dst, i1 %c)  {
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; CHECK:       [[VEC_EPILOG_PH]]:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TC]], 8
-; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TC]], [[N_MOD_VF2]]
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
 ; CHECK:       [[VEC_EPILOG_VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DST]], i64 [[VEC_EPILOG_RESUME_VAL]]
 ; CHECK-NEXT:    store <8 x i8> zeroinitializer, ptr [[TMP5]], align 4
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]]
 ; CHECK:       [[VEC_EPILOG_MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N4:%.*]] = icmp eq i64 [[TC]], [[N_VEC3]]
-; CHECK-NEXT:    br i1 [[CMP_N4]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-NEXT:    br i1 true, label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
 ; CHECK:       [[VEC_EPILOG_SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TC]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
index bba92933c56e1..d4e5dea3d4aab 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
@@ -72,8 +72,6 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-NEXT: No successors
 ; CHECK-NEXT: }
 ; CHECK: VPlan 'Final VPlan for VF={8,16},UF={1}' {
-; CHECK-NEXT: Live-in ir<[[EP_VFxUF:.+]]> = VF * UF
-; CHECK-NEXT: Live-in ir<[[EP_VEC_TC:.+]]> = vector-trip-count
 ; CHECK-NEXT: Live-in ir<1024> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<entry>:
@@ -83,7 +81,7 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<vector.ph>:
-; CHECK-NEXT:   EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
+; CHECK-NEXT:  EMIT vp<[[RDX_START:%.+]]> = reduction-start-vector ir<0>, ir<0>, ir<4>
 ; CHECK-NEXT: Successor(s): vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.body:
@@ -103,8 +101,7 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
 ; CHECK-NEXT:   EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<%accum>, ir<%add>
-; CHECK-NEXT:   EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, ir<1024>
-; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
+; CHECK-NEXT:   EMIT branch-on-cond ir<true>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<exit>:
@@ -114,6 +111,7 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-NEXT: ir-bb<scalar.ph>:
 ; CHECK-NEXT:   EMIT-SCALAR vp<[[EP_RESUME:%.+]]> = phi [ ir<1024>, middle.block ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:   EMIT-SCALAR vp<[[EP_MERGE:%.+]]> = phi [ vp<[[RED_RESULT]]>, middle.block ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<%6> = resume-for-epilogue vp<%vec.epilog.resume.val>
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.body>:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
index c1d4317a18333..9edd6ce53ec5d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
@@ -13,8 +13,6 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #1 {
 ; WIDE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; WIDE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; WIDE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; WIDE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; WIDE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; WIDE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; WIDE:       vector.body:
 ; WIDE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -24,7 +22,7 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #1 {
 ; WIDE-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x float> @foo_vector(<vscale x 4 x float> [[TMP5]], <vscale x 4 x i1> splat (i1 true))
 ; WIDE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
 ; WIDE-NEXT:    store <vscale x 4 x float> [[TMP6]], ptr [[TMP7]], align 4
-; WIDE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; WIDE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; WIDE-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; WIDE-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; WIDE:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
index 66bb80bbe21aa..d134dc50ecac8 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
@@ -27,10 +27,9 @@ define void @test_stride1_4i32(ptr readonly %data, ptr noalias nocapture %dst, i
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], 1
 ; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i32 [[ADD5]]
@@ -215,10 +214,9 @@ define void @test_stride3_4i32(ptr readonly %data, ptr noalias nocapture %dst, i
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], 3
 ; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i32 [[ADD5]]
@@ -277,10 +275,9 @@ define void @test_stride4_4i32(ptr readonly %data, ptr noalias nocapture %dst, i
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_023:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[I_023]], 4
 ; CHECK-NEXT:    [[ADD5:%.*]] = add nuw nsw i32 [[MUL]], 2
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, ptr [[DATA]], i32 [[ADD5]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-known-trip-count.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-known-trip-count.ll
index f4102ff8c402b..fe3504bc4b679 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-known-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-known-trip-count.ll
@@ -239,8 +239,8 @@ for.body:                                         ; preds = %entry, %for.body
 define dso_local i32 @predicated_test(i32 noundef %0, ptr %glob) #0 {
   %2 = alloca [101 x i32], align 4
   %3 = alloca [21 x i32], align 4
-  call void @llvm.lifetime.start.p0(i64 404, ptr nonnull %2)
-  call void @llvm.lifetime.start.p0(i64 84, ptr nonnull %3)
+  call void @llvm.lifetime.start.p0(ptr nonnull %2)
+  call void @llvm.lifetime.start.p0(ptr nonnull %3)
   %4 = icmp sgt i32 %0, 0
   br i1 %4, label %5, label %159
 
@@ -433,8 +433,8 @@ define dso_local i32 @predicated_test(i32 noundef %0, ptr %glob) #0 {
   br label %159
 
 159:                                              ; preds = %158, %1
-  call void @llvm.lifetime.end.p0(i64 84, ptr nonnull %3)
-  call void @llvm.lifetime.end.p0(i64 404, ptr nonnull %2)
+  call void @llvm.lifetime.end.p0(ptr nonnull %3)
+  call void @llvm.lifetime.end.p0(ptr nonnull %2)
   ret i32 0
 }
 
@@ -472,7 +472,7 @@ while.end:                                        ; preds = %while.end.loopexit,
 }
 
 
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 
 attributes #0 = { "target-features"="+mve.fp" }
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll
index 83cb3250fe87b..1b9bdb036cbd2 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reduction-types.ll
@@ -36,12 +36,10 @@ define i32 @mla_i32(ptr noalias nocapture readonly %A, ptr noalias nocapture rea
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[RES_010:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[RES_010:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[I_011]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[TMP12]] to i32
@@ -116,12 +114,10 @@ define i32 @mla_i8(ptr noalias nocapture readonly %A, ptr noalias nocapture read
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[RES_010:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_011:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[RES_010:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[I_011]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[TMP12]] to i32
@@ -191,12 +187,10 @@ define i32 @add_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP7]], [[R_07]]
@@ -256,12 +250,10 @@ define i32 @mul_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[TMP4]])
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 1, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 1, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = mul nsw i32 [[TMP7]], [[R_07]]
@@ -321,12 +313,10 @@ define i32 @and_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP4]])
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ -1, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ -1, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = and i32 [[TMP7]], [[R_07]]
@@ -386,12 +376,10 @@ define i32 @or_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP4]])
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = or i32 [[TMP7]], [[R_07]]
@@ -451,12 +439,10 @@ define i32 @xor_i32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP4]])
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = xor i32 [[TMP7]], [[R_07]]
@@ -516,12 +502,10 @@ define float @fadd_f32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP4]])
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[R_07:%.*]] = phi float [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[R_07:%.*]] = phi float [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0.000000e+00, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = fadd fast float [[TMP7]], [[R_07]]
@@ -581,12 +565,10 @@ define float @fmul_f32(ptr nocapture readonly %x, i32 %n) #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[TMP4]])
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 1.000000e+00, [[FOR_BODY_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[R_07:%.*]] = phi float [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[R_07:%.*]] = phi float [ [[ADD:%.*]], [[FOR_BODY]] ], [ 1.000000e+00, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X]], i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = fmul fast float [[TMP7]], [[R_07]]
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
index 0f4d40f202759..dcf4bee728b29 100644
--- a/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/ARM/optsize_minsize.ll
@@ -17,7 +17,7 @@ target triple = "armv7a-none-eabi"
 define void @always_vectorize(ptr %p, i32 %x) {
 ; DEFAULT-LABEL: define void @always_vectorize(
 ; DEFAULT-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
@@ -31,10 +31,9 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; DEFAULT:       [[FOR_BODY]]:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[INDVARS_IV]]
 ; DEFAULT-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; DEFAULT-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[X]]
@@ -47,7 +46,7 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ;
 ; OPTSIZE-LABEL: define void @always_vectorize(
 ; OPTSIZE-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
-; OPTSIZE-NEXT:  [[ENTRY:.*]]:
+; OPTSIZE-NEXT:  [[ENTRY:.*:]]
 ; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; OPTSIZE:       [[VECTOR_PH]]:
 ; OPTSIZE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
@@ -61,10 +60,9 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; OPTSIZE:       [[SCALAR_PH]]:
-; OPTSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; OPTSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; OPTSIZE:       [[FOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; OPTSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[INDVARS_IV]]
 ; OPTSIZE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; OPTSIZE-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[X]]
@@ -77,7 +75,7 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ;
 ; MINSIZE-LABEL: define void @always_vectorize(
 ; MINSIZE-SAME: ptr [[P:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
-; MINSIZE-NEXT:  [[ENTRY:.*]]:
+; MINSIZE-NEXT:  [[ENTRY:.*:]]
 ; MINSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; MINSIZE:       [[VECTOR_PH]]:
 ; MINSIZE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i64 0
@@ -91,10 +89,9 @@ define void @always_vectorize(ptr %p, i32 %x) {
 ; MINSIZE:       [[MIDDLE_BLOCK]]:
 ; MINSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; MINSIZE:       [[SCALAR_PH]]:
-; MINSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; MINSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; MINSIZE:       [[FOR_BODY]]:
-; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; MINSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P]], i64 [[INDVARS_IV]]
 ; MINSIZE-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; MINSIZE-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[X]]
@@ -217,7 +214,7 @@ for.cond.cleanup:
 define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n) {
 ; DEFAULT-LABEL: define void @tail_predicate_without_optsize(
 ; DEFAULT-SAME: ptr [[P:%.*]], i8 [[A:%.*]], i8 [[B:%.*]], i8 [[C:%.*]], i32 [[N:%.*]]) {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[A]], i64 0
@@ -390,10 +387,9 @@ define void @tail_predicate_without_optsize(ptr %p, i8 %a, i8 %b, i8 %c, i32 %n)
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; DEFAULT:       [[FOR_BODY]]:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; DEFAULT-NEXT:    [[TMP72:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i8
 ; DEFAULT-NEXT:    [[MUL:%.*]] = mul i8 [[A]], [[TMP72]]
 ; DEFAULT-NEXT:    [[SHR:%.*]] = lshr i8 [[TMP72]], 1
@@ -485,7 +481,7 @@ for.cond.cleanup:
 ; we don't account for the addressing mode difference.
 define void @dont_vectorize_with_minsize() {
 ; DEFAULT-LABEL: define void @dont_vectorize_with_minsize() {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -507,10 +503,9 @@ define void @dont_vectorize_with_minsize() {
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; DEFAULT:       [[FOR_BODY]]:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; DEFAULT-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; DEFAULT-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -529,7 +524,7 @@ define void @dont_vectorize_with_minsize() {
 ;
 ; OPTSIZE-LABEL: define void @dont_vectorize_with_minsize(
 ; OPTSIZE-SAME: ) #[[ATTR0]] {
-; OPTSIZE-NEXT:  [[ENTRY:.*]]:
+; OPTSIZE-NEXT:  [[ENTRY:.*:]]
 ; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; OPTSIZE:       [[VECTOR_PH]]:
 ; OPTSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -551,10 +546,9 @@ define void @dont_vectorize_with_minsize() {
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; OPTSIZE:       [[SCALAR_PH]]:
-; OPTSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; OPTSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; OPTSIZE:       [[FOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; OPTSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; OPTSIZE-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; OPTSIZE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -573,7 +567,7 @@ define void @dont_vectorize_with_minsize() {
 ;
 ; MINSIZE-LABEL: define void @dont_vectorize_with_minsize(
 ; MINSIZE-SAME: ) #[[ATTR0]] {
-; MINSIZE-NEXT:  [[ENTRY:.*]]:
+; MINSIZE-NEXT:  [[ENTRY:.*:]]
 ; MINSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; MINSIZE:       [[VECTOR_PH]]:
 ; MINSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -595,10 +589,9 @@ define void @dont_vectorize_with_minsize() {
 ; MINSIZE:       [[MIDDLE_BLOCK]]:
 ; MINSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; MINSIZE:       [[SCALAR_PH]]:
-; MINSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; MINSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; MINSIZE:       [[FOR_BODY]]:
-; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; MINSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; MINSIZE-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; MINSIZE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -645,7 +638,7 @@ for.cond.cleanup:
 ; FIXME: We currently use width 2 as the load/store cost is incorrect.
 define void @vectorization_forced() {
 ; DEFAULT-LABEL: define void @vectorization_forced() {
-; DEFAULT-NEXT:  [[ENTRY:.*]]:
+; DEFAULT-NEXT:  [[ENTRY:.*:]]
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -667,10 +660,9 @@ define void @vectorization_forced() {
 ; DEFAULT:       [[MIDDLE_BLOCK]]:
 ; DEFAULT-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; DEFAULT:       [[SCALAR_PH]]:
-; DEFAULT-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; DEFAULT-NEXT:    br label %[[FOR_BODY:.*]]
 ; DEFAULT:       [[FOR_BODY]]:
-; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; DEFAULT-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; DEFAULT-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; DEFAULT-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; DEFAULT-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -689,7 +681,7 @@ define void @vectorization_forced() {
 ;
 ; OPTSIZE-LABEL: define void @vectorization_forced(
 ; OPTSIZE-SAME: ) #[[ATTR0]] {
-; OPTSIZE-NEXT:  [[ENTRY:.*]]:
+; OPTSIZE-NEXT:  [[ENTRY:.*:]]
 ; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; OPTSIZE:       [[VECTOR_PH]]:
 ; OPTSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -711,10 +703,9 @@ define void @vectorization_forced() {
 ; OPTSIZE:       [[MIDDLE_BLOCK]]:
 ; OPTSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; OPTSIZE:       [[SCALAR_PH]]:
-; OPTSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; OPTSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; OPTSIZE:       [[FOR_BODY]]:
-; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; OPTSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; OPTSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; OPTSIZE-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; OPTSIZE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
@@ -733,7 +724,7 @@ define void @vectorization_forced() {
 ;
 ; MINSIZE-LABEL: define void @vectorization_forced(
 ; MINSIZE-SAME: ) #[[ATTR0]] {
-; MINSIZE-NEXT:  [[ENTRY:.*]]:
+; MINSIZE-NEXT:  [[ENTRY:.*:]]
 ; MINSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; MINSIZE:       [[VECTOR_PH]]:
 ; MINSIZE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -755,10 +746,9 @@ define void @vectorization_forced() {
 ; MINSIZE:       [[MIDDLE_BLOCK]]:
 ; MINSIZE-NEXT:    br label %[[FOR_COND_CLEANUP:.*]]
 ; MINSIZE:       [[SCALAR_PH]]:
-; MINSIZE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; MINSIZE-NEXT:    br label %[[FOR_BODY:.*]]
 ; MINSIZE:       [[FOR_BODY]]:
-; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; MINSIZE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; MINSIZE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
 ; MINSIZE-NEXT:    [[BVAL:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; MINSIZE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw [1000 x i32], ptr @C, i64 0, i64 [[INDVARS_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll b/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll
index 1ac556a2fda39..26bab4dfeaf4d 100644
--- a/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll
+++ b/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll
@@ -11,14 +11,14 @@ target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i
 
 @g0 = external dso_local local_unnamed_addr global ptr, align 4
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #0
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
+declare void @llvm.lifetime.end.p0(ptr nocapture) #0
 
 ; Function Attrs: nounwind
 define hidden fastcc void @f0(ptr nocapture %a0, i32 %a1, i32 %a2, i32 %a3, i32 %a4, i8 zeroext %a5) unnamed_addr #1 {
 b0:
   %v0 = alloca [4 x [9 x i16]], align 8
-  call void @llvm.lifetime.start.p0(i64 72, ptr nonnull %v0) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %v0) #2
   %v2 = add i32 %a1, -2
   %v3 = add i32 %a3, -9
   %v4 = icmp ugt i32 %v2, %v3
@@ -147,7 +147,7 @@ b1:                                               ; preds = %b1, %b0
   br i1 %v120, label %b2, label %b1
 
 b2:                                               ; preds = %b1, %b0
-  call void @llvm.lifetime.end.p0(i64 72, ptr nonnull %v0) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %v0) #2
   ret void
 }
 
diff --git a/llvm/test/Transforms/LoopVectorize/LoongArch/defaults.ll b/llvm/test/Transforms/LoopVectorize/LoongArch/defaults.ll
index 17eeafa574ae8..8072a3d973137 100644
--- a/llvm/test/Transforms/LoopVectorize/LoongArch/defaults.ll
+++ b/llvm/test/Transforms/LoopVectorize/LoongArch/defaults.ll
@@ -35,10 +35,9 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[ELEM]], [[V]]
diff --git a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
index 1bacb57700df8..6b72f20d51b84 100644
--- a/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
+++ b/llvm/test/Transforms/LoopVectorize/PowerPC/vectorize-only-for-real.ll
@@ -10,11 +10,11 @@ define zeroext i32 @test() #0 {
 entry:
   %a = alloca [1600 x i32], align 4
   %c = alloca [1600 x i32], align 4
-  call void @llvm.lifetime.start(i64 6400, ptr %a) #3
+  call void @llvm.lifetime.start(ptr %a) #3
   br label %for.body
 
 for.cond.cleanup:                                 ; preds = %for.body
-  call void @llvm.lifetime.start(i64 6400, ptr %c) #3
+  call void @llvm.lifetime.start(ptr %c) #3
   %call = call signext i32 @bar(ptr %a, ptr %c) #3
   br label %for.body6
 
@@ -28,8 +28,8 @@ for.body:                                         ; preds = %for.body, %entry
   br i1 %exitcond27, label %for.cond.cleanup, label %for.body
 
 for.cond.cleanup5:                                ; preds = %for.body6
-  call void @llvm.lifetime.end(i64 6400, ptr nonnull %c) #3
-  call void @llvm.lifetime.end(i64 6400, ptr %a) #3
+  call void @llvm.lifetime.end(ptr nonnull %c) #3
+  call void @llvm.lifetime.end(ptr %a) #3
   ret i32 %add
 
 for.body6:                                        ; preds = %for.body6, %for.cond.cleanup
@@ -44,10 +44,10 @@ for.body6:                                        ; preds = %for.body6, %for.con
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start(ptr nocapture) #1
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end(ptr nocapture) #1
 
 declare signext i32 @bar(ptr, ptr) #2
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
index 162440aacbada..6d498bc06cffe 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
 ; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+v -S | FileCheck %s -check-prefix=NO-ZVFBFMIN
+; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+v -S -prefer-predicate-over-epilogue=scalar-epilogue | FileCheck %s -check-prefix=NO-ZVFBFMIN
 ; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+v -S -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue | FileCheck %s -check-prefix=NO-ZVFBFMIN
 ; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfbfmin -S | FileCheck %s -check-prefix=ZVFBFMIN
 
@@ -24,38 +25,31 @@ define void @fadd(ptr noalias %a, ptr noalias %b, i64 %n) {
 ;
 ; ZVFBFMIN-LABEL: define void @fadd(
 ; ZVFBFMIN-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; ZVFBFMIN-NEXT:  [[ENTRY:.*]]:
-; ZVFBFMIN-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
-; ZVFBFMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP8]]
-; ZVFBFMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; ZVFBFMIN-NEXT:  [[ENTRY:.*:]]
+; ZVFBFMIN-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; ZVFBFMIN:       [[VECTOR_PH]]:
-; ZVFBFMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
-; ZVFBFMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
-; ZVFBFMIN-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; ZVFBFMIN-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP12]], 8
 ; ZVFBFMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFBFMIN:       [[VECTOR_BODY]]:
-; ZVFBFMIN-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFBFMIN-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFBFMIN-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFBFMIN-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; ZVFBFMIN-NEXT:    [[TMP1:%.*]] = getelementptr bfloat, ptr [[A]], i64 [[TMP0]]
 ; ZVFBFMIN-NEXT:    [[TMP2:%.*]] = getelementptr bfloat, ptr [[B]], i64 [[TMP0]]
-; ZVFBFMIN-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x bfloat>, ptr [[TMP1]], align 2
-; ZVFBFMIN-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x bfloat>, ptr [[TMP2]], align 2
+; ZVFBFMIN-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 2 [[TMP1]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; ZVFBFMIN-NEXT:    [[WIDE_LOAD1:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
 ; ZVFBFMIN-NEXT:    [[TMP11:%.*]] = fadd <vscale x 8 x bfloat> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; ZVFBFMIN-NEXT:    store <vscale x 8 x bfloat> [[TMP11]], ptr [[TMP1]], align 2
-; ZVFBFMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], [[TMP5]]
-; ZVFBFMIN-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; ZVFBFMIN-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; ZVFBFMIN-NEXT:    call void @llvm.vp.store.nxv8bf16.p0(<vscale x 8 x bfloat> [[TMP11]], ptr align 2 [[TMP1]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; ZVFBFMIN-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
+; ZVFBFMIN-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP13]], [[TMP0]]
+; ZVFBFMIN-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; ZVFBFMIN-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; ZVFBFMIN-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; ZVFBFMIN:       [[MIDDLE_BLOCK]]:
-; ZVFBFMIN-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; ZVFBFMIN-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; ZVFBFMIN-NEXT:    br label %[[EXIT:.*]]
 ; ZVFBFMIN:       [[SCALAR_PH]]:
-; ZVFBFMIN-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; ZVFBFMIN-NEXT:    br label %[[LOOP:.*]]
 ; ZVFBFMIN:       [[LOOP]]:
-; ZVFBFMIN-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; ZVFBFMIN-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; ZVFBFMIN-NEXT:    [[A_GEP:%.*]] = getelementptr bfloat, ptr [[A]], i64 [[I]]
 ; ZVFBFMIN-NEXT:    [[B_GEP:%.*]] = getelementptr bfloat, ptr [[B]], i64 [[I]]
 ; ZVFBFMIN-NEXT:    [[X:%.*]] = load bfloat, ptr [[A_GEP]], align 2
@@ -64,7 +58,7 @@ define void @fadd(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; ZVFBFMIN-NEXT:    store bfloat [[Z]], ptr [[A_GEP]], align 2
 ; ZVFBFMIN-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
 ; ZVFBFMIN-NEXT:    [[DONE:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
-; ZVFBFMIN-NEXT:    br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; ZVFBFMIN-NEXT:    br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; ZVFBFMIN:       [[EXIT]]:
 ; ZVFBFMIN-NEXT:    ret void
 ;
@@ -136,42 +130,35 @@ define void @vfwmaccbf16.vv(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64
 ;
 ; ZVFBFMIN-LABEL: define void @vfwmaccbf16.vv(
 ; ZVFBFMIN-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; ZVFBFMIN-NEXT:  [[ENTRY:.*]]:
-; ZVFBFMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; ZVFBFMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; ZVFBFMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; ZVFBFMIN-NEXT:  [[ENTRY:.*:]]
+; ZVFBFMIN-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; ZVFBFMIN:       [[VECTOR_PH]]:
-; ZVFBFMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; ZVFBFMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; ZVFBFMIN-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; ZVFBFMIN-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVFBFMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFBFMIN:       [[VECTOR_BODY]]:
-; ZVFBFMIN-NEXT:    [[TMP6:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFBFMIN-NEXT:    [[TMP6:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFBFMIN-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFBFMIN-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; ZVFBFMIN-NEXT:    [[TMP7:%.*]] = getelementptr bfloat, ptr [[A]], i64 [[TMP6]]
 ; ZVFBFMIN-NEXT:    [[TMP8:%.*]] = getelementptr bfloat, ptr [[B]], i64 [[TMP6]]
 ; ZVFBFMIN-NEXT:    [[TMP9:%.*]] = getelementptr float, ptr [[C]], i64 [[TMP6]]
-; ZVFBFMIN-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x bfloat>, ptr [[TMP7]], align 2
-; ZVFBFMIN-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x bfloat>, ptr [[TMP8]], align 2
-; ZVFBFMIN-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP9]], align 4
+; ZVFBFMIN-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x bfloat> @llvm.vp.load.nxv4bf16.p0(ptr align 2 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; ZVFBFMIN-NEXT:    [[WIDE_LOAD1:%.*]] = call <vscale x 4 x bfloat> @llvm.vp.load.nxv4bf16.p0(ptr align 2 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; ZVFBFMIN-NEXT:    [[WIDE_LOAD2:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
 ; ZVFBFMIN-NEXT:    [[TMP13:%.*]] = fpext <vscale x 4 x bfloat> [[WIDE_LOAD]] to <vscale x 4 x float>
 ; ZVFBFMIN-NEXT:    [[TMP14:%.*]] = fpext <vscale x 4 x bfloat> [[WIDE_LOAD1]] to <vscale x 4 x float>
 ; ZVFBFMIN-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[TMP13]], <vscale x 4 x float> [[TMP14]], <vscale x 4 x float> [[WIDE_LOAD2]])
-; ZVFBFMIN-NEXT:    store <vscale x 4 x float> [[TMP15]], ptr [[TMP9]], align 4
-; ZVFBFMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP6]], [[TMP5]]
-; ZVFBFMIN-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; ZVFBFMIN-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; ZVFBFMIN-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[TMP15]], ptr align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; ZVFBFMIN-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; ZVFBFMIN-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP12]], [[TMP6]]
+; ZVFBFMIN-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]
+; ZVFBFMIN-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; ZVFBFMIN-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; ZVFBFMIN:       [[MIDDLE_BLOCK]]:
-; ZVFBFMIN-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; ZVFBFMIN-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; ZVFBFMIN-NEXT:    br label %[[EXIT:.*]]
 ; ZVFBFMIN:       [[SCALAR_PH]]:
-; ZVFBFMIN-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; ZVFBFMIN-NEXT:    br label %[[LOOP:.*]]
 ; ZVFBFMIN:       [[LOOP]]:
-; ZVFBFMIN-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; ZVFBFMIN-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; ZVFBFMIN-NEXT:    [[A_GEP:%.*]] = getelementptr bfloat, ptr [[A]], i64 [[I]]
 ; ZVFBFMIN-NEXT:    [[B_GEP:%.*]] = getelementptr bfloat, ptr [[B]], i64 [[I]]
 ; ZVFBFMIN-NEXT:    [[C_GEP:%.*]] = getelementptr float, ptr [[C]], i64 [[I]]
@@ -184,7 +171,7 @@ define void @vfwmaccbf16.vv(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64
 ; ZVFBFMIN-NEXT:    store float [[FMULADD]], ptr [[C_GEP]], align 4
 ; ZVFBFMIN-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
 ; ZVFBFMIN-NEXT:    [[DONE:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
-; ZVFBFMIN-NEXT:    br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; ZVFBFMIN-NEXT:    br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; ZVFBFMIN:       [[EXIT]]:
 ; ZVFBFMIN-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
index 75ae6df5fcd37..9f7ac7ac1771f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
@@ -62,50 +62,10 @@ define i32 @any_of_reduction_used_in_blend_with_multiple_phis(ptr %src, i64 %N,
 ; CHECK-LABEL: define i32 @any_of_reduction_used_in_blend_with_multiple_phis(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]], i1 [[C_0:%.*]], i1 [[C_1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C_1]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C_0]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = xor <vscale x 2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP7:%.*]] = xor <vscale x 2 x i1> [[BROADCAST_SPLAT2]], splat (i1 true)
-; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[TMP7]], <vscale x 2 x i1> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[SRC]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
-; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x ptr> @llvm.masked.gather.nxv2p0.nxv2p0(<vscale x 2 x ptr> [[BROADCAST_SPLAT4]], i32 8, <vscale x 2 x i1> [[TMP8]], <vscale x 2 x ptr> poison)
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <vscale x 2 x ptr> [[WIDE_MASKED_GATHER]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = or <vscale x 2 x i1> [[VEC_PHI]], [[TMP9]]
-; CHECK-NEXT:    [[PREDPHI]] = select <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv2i1(<vscale x 2 x i1> [[PREDPHI]])
-; CHECK-NEXT:    [[TMP13:%.*]] = freeze i1 [[TMP12]]
-; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP13]], i32 0, i32 0
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[ANY_OF_RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ANY_OF_RED_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[ANY_OF_RED:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ANY_OF_RED_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    br i1 [[C_0]], label %[[X_1:.*]], label %[[ELSE_1:.*]]
 ; CHECK:       [[ELSE_1]]:
 ; CHECK-NEXT:    br i1 [[C_1]], label %[[X_1]], label %[[ELSE_2:.*]]
@@ -121,9 +81,9 @@ define i32 @any_of_reduction_used_in_blend_with_multiple_phis(ptr %src, i64 %N,
 ; CHECK-NEXT:    [[ANY_OF_RED_NEXT]] = phi i32 [ [[P]], %[[X_1]] ], [ [[SEL]], %[[ELSE_2]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[ANY_OF_RED_NEXT]], %[[LOOP_LATCH]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[ANY_OF_RED_NEXT]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
 entry:
@@ -159,9 +119,3 @@ exit:
 }
 
 attributes #0 = { "target-cpu"="sifive-p670" }
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
index aad9128a240de..757c77fef98c8 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
@@ -7,49 +7,40 @@ target triple = "riscv64-unknown-linux-gnu"
 define void @block_with_dead_inst_1(ptr %src, i64 %N) #0 {
 ; CHECK-LABEL: define void @block_with_dead_inst_1(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -3
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP11]], splat (i64 3)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP13]]
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP10]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP17]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP20]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP14]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ 1, %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
@@ -62,7 +53,7 @@ define void @block_with_dead_inst_1(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[TMP25]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP25]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -95,46 +86,37 @@ exit:
 define void @block_with_dead_inst_2(ptr %src) #0 {
 ; CHECK-LABEL: define void @block_with_dead_inst_2(
 ; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
-; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 3)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP7]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 8 x i64> [[TMP5]], splat (i64 3)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP8]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 333, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP7]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP9]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 333
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
@@ -147,7 +129,7 @@ define void @block_with_dead_inst_2(ptr %src) #0 {
 ; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -180,46 +162,37 @@ exit:
 define void @multiple_blocks_with_dead_insts_3(ptr %src) #0 {
 ; CHECK-LABEL: define void @multiple_blocks_with_dead_insts_3(
 ; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
-; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 3)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP7]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 8 x i64> [[TMP5]], splat (i64 3)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP8]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 333, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP7]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP9]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 333
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
@@ -235,7 +208,7 @@ define void @multiple_blocks_with_dead_insts_3(ptr %src) #0 {
 ; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -272,49 +245,40 @@ exit:
 define void @multiple_blocks_with_dead_insts_4(ptr %src, i64 %N) #0 {
 ; CHECK-LABEL: define void @multiple_blocks_with_dead_insts_4(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -3
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP11]], splat (i64 3)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP13]]
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP10]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP17]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP20]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP14]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ 1, %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
@@ -332,7 +296,7 @@ define void @multiple_blocks_with_dead_insts_4(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -372,46 +336,37 @@ exit:
 define void @multiple_blocks_with_dead_inst_multiple_successors_5(ptr %src) #0 {
 ; CHECK-LABEL: define void @multiple_blocks_with_dead_inst_multiple_successors_5(
 ; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
-; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 3)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP7]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
+; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 8 x i64> [[TMP5]], splat (i64 3)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP8]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i16.nxv4p0(<vscale x 4 x i16> zeroinitializer, <vscale x 4 x ptr> [[TMP17]], i32 2, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 333, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 3, [[TMP7]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP9]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 333
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ 1, %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
@@ -431,7 +386,7 @@ define void @multiple_blocks_with_dead_inst_multiple_successors_5(ptr %src) #0 {
 ; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -474,49 +429,56 @@ exit:
 define void @multiple_blocks_with_dead_inst_multiple_successors_6(ptr %src, i1 %ic, i64 %N) #0 {
 ; CHECK-LABEL: define void @multiple_blocks_with_dead_inst_multiple_successors_6(
 ; CHECK-SAME: ptr [[SRC:%.*]], i1 [[IC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -3
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i1> poison, i1 [[IC]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i1> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i1> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = xor <vscale x 8 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP11]], splat (i64 3)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP13]]
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP10]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP27:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP27]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP27]] to i64
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 3, [[TMP12]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP16]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = call <vscale x 8 x i32> @llvm.stepvector.nxv8i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult <vscale x 8 x i32> [[TMP14]], [[BROADCAST_SPLAT4]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i16, ptr [[SRC]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP20]], i32 2, <vscale x 8 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 8 x i16> @llvm.vp.gather.nxv8i16.nxv8p0(<vscale x 8 x ptr> align 2 [[TMP20]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP27]])
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq <vscale x 8 x i16> [[WIDE_MASKED_GATHER]], zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = select <vscale x 8 x i1> [[TMP15]], <vscale x 8 x i1> [[TMP17]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = select <vscale x 8 x i1> [[TMP18]], <vscale x 8 x i1> [[TMP8]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP28:%.*]] = xor <vscale x 8 x i1> [[TMP17]], splat (i1 true)
+; CHECK-NEXT:    [[TMP21:%.*]] = select <vscale x 8 x i1> [[TMP15]], <vscale x 8 x i1> [[TMP28]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = or <vscale x 8 x i1> [[TMP19]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <vscale x 8 x i1> [[TMP18]], <vscale x 8 x i1> [[BROADCAST_SPLAT]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = or <vscale x 8 x i1> [[TMP22]], [[TMP23]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP20]], <vscale x 8 x i1> [[TMP24]], i32 [[TMP27]])
+; CHECK-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP27]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP25]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP25]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i16 [ 1, %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    [[XOR]] = xor i16 0, 0
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
@@ -536,7 +498,7 @@ define void @multiple_blocks_with_dead_inst_multiple_successors_6(ptr %src, i1 %
 ; CHECK-NEXT:    store i16 [[XOR]], ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -579,39 +541,36 @@ exit:
 define void @empty_block_with_phi_1(ptr %src, i64 %N) #0 {
 ; CHECK-LABEL: define void @empty_block_with_phi_1(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP13]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 8 x i32> @llvm.stepvector.nxv8i32()
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult <vscale x 8 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP9]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP10]], align 2
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq <vscale x 8 x i16> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP12]], <vscale x 8 x i16> splat (i16 99), <vscale x 8 x i16> [[WIDE_LOAD]]
-; CHECK-NEXT:    store <vscale x 8 x i16> [[PREDPHI]], ptr [[TMP10]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP9]], [[TMP5]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ne <vscale x 8 x i16> [[VP_OP_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x i1> [[TMP14]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP15]], <vscale x 8 x i16> [[VP_OP_LOAD]], <vscale x 8 x i16> splat (i16 99)
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i16.p0(<vscale x 8 x i16> [[PREDPHI]], ptr align 2 [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP11]], [[TMP9]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[XOR1315:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i32 [ 1, %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    [[XOR]] = xor i32 0, 0
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
@@ -624,7 +583,7 @@ define void @empty_block_with_phi_1(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    store i16 [[P]], ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -657,39 +616,36 @@ exit:
 define void @empty_block_with_phi_2(ptr %src, i64 %N) #0 {
 ; CHECK-LABEL: define void @empty_block_with_phi_2(
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP13]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 8 x i32> @llvm.stepvector.nxv8i32()
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult <vscale x 8 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP9]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP10]], align 2
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 2 [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq <vscale x 8 x i16> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP12]], <vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> splat (i16 99)
-; CHECK-NEXT:    store <vscale x 8 x i16> [[PREDPHI]], ptr [[TMP10]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP9]], [[TMP5]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    [[TMP14:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x i1> [[TMP12]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 8 x i1> [[TMP14]], <vscale x 8 x i16> [[WIDE_LOAD]], <vscale x 8 x i16> splat (i16 99)
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i16.p0(<vscale x 8 x i16> [[PREDPHI]], ptr align 2 [[TMP10]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP11]], [[TMP9]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
-; CHECK-NEXT:    [[XOR1315:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[XOR1315:%.*]] = phi i32 [ 1, %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[LOOP_LATCH]] ]
 ; CHECK-NEXT:    [[XOR]] = xor i32 0, 0
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
@@ -702,7 +658,7 @@ define void @empty_block_with_phi_2(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    store i16 [[P]], ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[TMP18]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -743,11 +699,7 @@ define void @dead_load_in_block(ptr %dst, ptr %src, i8 %N, i64 %x) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[UMIN7]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umax.i64(i64 40, i64 [[TMP5]])
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP6]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[N_EXT]], i64 1)
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N_EXT]], [[UMIN]]
@@ -770,34 +722,31 @@ define void @dead_load_in_block(ptr %dst, ptr %src, i8 %N, i64 %x) #0 {
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP15]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 2
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
-; CHECK-NEXT:    [[TMP18:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT:    [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP18]], splat (i64 3)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP20]]
-; CHECK-NEXT:    [[TMP23:%.*]] = mul i64 3, [[TMP17]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP23]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP24:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP25:%.*]] = mul <vscale x 4 x i64> [[TMP24]], splat (i64 3)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP25]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 2 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> zeroinitializer, <vscale x 2 x ptr> [[TMP24]], i32 4, <vscale x 2 x i1> splat (i1 true)), !alias.scope [[META18:![0-9]+]], !noalias [[META21:![0-9]+]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP3]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP18:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP18]] to i64
+; CHECK-NEXT:    [[TMP23:%.*]] = mul i64 3, [[TMP17]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP23]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> zeroinitializer, <vscale x 4 x ptr> align 4 [[TMP21]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP18]]), !alias.scope [[META19:![0-9]+]], !noalias [[META22:![0-9]+]]
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP18]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP22]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP22]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
@@ -813,7 +762,7 @@ define void @dead_load_in_block(ptr %dst, ptr %src, i8 %N, i64 %x) #0 {
 ; CHECK-NEXT:    store i32 0, ptr [[GEP_DST]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 3
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[IV]], [[N_EXT]]
-; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -845,30 +794,31 @@ exit:
 
 attributes #0 = { "target-features"="+64bit,+v" }
 ;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]}
-; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]}
-; CHECK: [[META18]] = !{[[META19:![0-9]+]]}
-; CHECK: [[META19]] = distinct !{[[META19]], [[META20:![0-9]+]]}
-; CHECK: [[META20]] = distinct !{[[META20]], !"LVerDomain"}
-; CHECK: [[META21]] = !{[[META22:![0-9]+]], [[META23:![0-9]+]]}
-; CHECK: [[META22]] = distinct !{[[META22]], [[META20]]}
-; CHECK: [[META23]] = distinct !{[[META23]], [[META20]]}
-; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META1]], [[META2]]}
-; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META1]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META1]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META1]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]], [[META1]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]], [[META1]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]], [[META1]]}
+; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]], [[META1]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META3]], [[META1]]}
+; CHECK: [[META19]] = !{[[META20:![0-9]+]]}
+; CHECK: [[META20]] = distinct !{[[META20]], [[META21:![0-9]+]]}
+; CHECK: [[META21]] = distinct !{[[META21]], !"LVerDomain"}
+; CHECK: [[META22]] = !{[[META23:![0-9]+]], [[META24:![0-9]+]]}
+; CHECK: [[META23]] = distinct !{[[META23]], [[META21]]}
+; CHECK: [[META24]] = distinct !{[[META24]], [[META21]]}
+; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index ab8875bc2a825..29d901f084bdb 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -8,7 +8,7 @@ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
 define void @dead_load(ptr %p, i16 %start) {
 ; CHECK-LABEL: define void @dead_load(
 ; CHECK-SAME: ptr [[P:%.*]], i16 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[START_EXT:%.*]] = sext i16 [[START]] to i64
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[START_EXT]], i64 111)
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[SMAX]], [[START_EXT]]
@@ -18,52 +18,43 @@ define void @dead_load(ptr %p, i16 %start) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = udiv i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[UMIN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP5]], [[TMP7]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP5]], [[TMP9]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP5]], [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 8
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[N_VEC]], 3
-; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[START_EXT]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[START_EXT]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP17:%.*]] = mul <vscale x 8 x i64> [[TMP15]], splat (i64 3)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[DOTSPLAT]], [[TMP17]]
-; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 3, [[TMP14]]
-; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP20]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP5]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP16]] to i64
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 3, [[TMP19]]
+; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP20]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT1]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i16, ptr [[P]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP21]], i32 2, <vscale x 8 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP21]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP16]])
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP16]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP22]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP22]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT2]]
-; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[START_EXT]], %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[START_EXT]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    store i16 0, ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 3
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[IV]], 111
-; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -93,8 +84,8 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 8, i32 [[TMP1]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 6, i32 [[TMP1]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 252, [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
@@ -111,13 +102,11 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 252, [[TMP6]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 4
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i32 [[N_VEC]], 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i32> [[TMP9]], splat (i32 4)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP11]]
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 4, [[TMP8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 4, [[TMP4]]
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP14]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -126,11 +115,11 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP15:%.*]] = sext <vscale x 4 x i32> [[VEC_IND]] to <vscale x 4 x i64>
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[DST]], <vscale x 4 x i64> [[TMP15]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i8.nxv4p0(<vscale x 4 x i8> zeroinitializer, <vscale x 4 x ptr> [[TMP16]], i32 1, <vscale x 4 x i1> splat (i1 true)), !alias.scope [[META4:![0-9]+]], !noalias [[META7:![0-9]+]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i8.nxv4p0(<vscale x 4 x i8> zeroinitializer, <vscale x 4 x ptr> [[TMP16]], i32 1, <vscale x 4 x i1> splat (i1 true)), !alias.scope [[META5:![0-9]+]], !noalias [[META8:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -145,7 +134,7 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    store i8 0, ptr [[GEP_DST]], align 1
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[IV]], 1001
-; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[R:%.*]] = phi i8 [ [[L]], %[[LOOP]] ]
 ; CHECK-NEXT:    ret i8 [[R]]
@@ -181,7 +170,7 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 770, [[UMAX3]]
 ; CHECK-NEXT:    [[SMAX4:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP1]], i32 0)
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i32 [[SMAX4]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP2]], 24
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP2]], 19
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 1
@@ -210,48 +199,48 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[B]], i32 [[INDEX]]
 ; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
-; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11:![0-9]+]], !noalias [[META14:![0-9]+]]
+; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META12:![0-9]+]], !noalias [[META15:![0-9]+]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
 ; CHECK:       [[PRED_STORE_IF5]]:
-; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
+; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META12]], !noalias [[META15]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
 ; CHECK:       [[PRED_STORE_CONTINUE6]]:
 ; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
 ; CHECK:       [[PRED_STORE_IF7]]:
-; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
+; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META12]], !noalias [[META15]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
 ; CHECK:       [[PRED_STORE_CONTINUE8]]:
 ; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
 ; CHECK:       [[PRED_STORE_IF9]]:
-; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
+; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META12]], !noalias [[META15]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
 ; CHECK:       [[PRED_STORE_CONTINUE10]]:
 ; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
 ; CHECK:       [[PRED_STORE_IF11]]:
-; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
+; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META12]], !noalias [[META15]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE12]]
 ; CHECK:       [[PRED_STORE_CONTINUE12]]:
 ; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14:.*]]
 ; CHECK:       [[PRED_STORE_IF13]]:
-; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
+; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META12]], !noalias [[META15]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE14]]
 ; CHECK:       [[PRED_STORE_CONTINUE14]]:
 ; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF15:.*]], label %[[PRED_STORE_CONTINUE16:.*]]
 ; CHECK:       [[PRED_STORE_IF15]]:
-; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
+; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META12]], !noalias [[META15]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE16]]
 ; CHECK:       [[PRED_STORE_CONTINUE16]]:
 ; CHECK-NEXT:    br i1 [[C]], label %[[PRED_STORE_IF17:.*]], label %[[PRED_STORE_CONTINUE18]]
 ; CHECK:       [[PRED_STORE_IF17]]:
-; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META11]], !noalias [[META14]]
+; CHECK-NEXT:    store i1 false, ptr [[A]], align 1, !alias.scope [[META12]], !noalias [[META15]]
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE18]]
 ; CHECK:       [[PRED_STORE_CONTINUE18]]:
-; CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr [[TMP11]], i32 4, <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META14]]
+; CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr [[TMP11]], i32 4, <8 x i1> [[BROADCAST_SPLAT]]), !alias.scope [[META15]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -273,7 +262,7 @@ define i32 @cost_of_exit_branch_and_cond_insts(ptr %a, ptr %b, i1 %c, i16 %x) #0
 ; CHECK-NEXT:    [[EC:%.*]] = icmp slt i32 [[IV]], [[SUB]]
 ; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP_LATCH]], label %[[EXIT:.*]]
 ; CHECK:       [[LOOP_LATCH]]:
-; CHECK-NEXT:    br label %[[LOOP_HEADER]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br label %[[LOOP_HEADER]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    br label %[[RETURN:.*]]
 ; CHECK:       [[RETURN]]:
@@ -314,46 +303,47 @@ return:
 define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
 ; CHECK-LABEL: define void @test_phi_in_latch_redundant(
 ; CHECK-SAME: ptr [[DST:%.*]], i32 [[A:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 37, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 37, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 37, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[A]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[N_VEC]], 9
-; CHECK-NEXT:    [[TMP10:%.*]] = xor <vscale x 2 x i32> [[BROADCAST_SPLAT]], splat (i32 -1)
-; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 9)
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP8]]
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 9, [[TMP5]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP9]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[A]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP19:%.*]] = xor <vscale x 4 x i32> [[BROADCAST_SPLAT]], splat (i32 -1)
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 9)
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP7]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 2 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i32.nxv2p0(<vscale x 2 x i32> [[TMP10]], <vscale x 2 x ptr> [[TMP11]], i32 4, <vscale x 2 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 37, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 9, [[TMP5]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult <vscale x 4 x i32> [[TMP11]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = select <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = select <vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> zeroinitializer, <vscale x 4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = or <vscale x 4 x i1> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[TMP19]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[TMP16]], <vscale x 4 x i1> [[TMP15]], i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP17]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP17]]
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 37
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 37, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 false, label %[[LOOP_LATCH]], label %[[THEN:.*]]
 ; CHECK:       [[THEN]]:
 ; CHECK-NEXT:    [[NOT_A:%.*]] = xor i32 [[A]], -1
@@ -364,7 +354,7 @@ define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
 ; CHECK-NEXT:    store i32 [[P]], ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 9
 ; CHECK-NEXT:    [[EC:%.*]] = icmp slt i64 [[IV]], 322
-; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -395,54 +385,52 @@ exit:
 define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %src, ptr noalias %dst) #0 {
 ; CHECK-LABEL: define void @gather_interleave_group_with_dead_insert_pos(
 ; CHECK-SAME: i64 [[N:%.*]], ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 0)
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nuw i64 [[SMAX]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[N_VEC]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i64> [[TMP9]], splat (i64 2)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP11]]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 2, [[TMP8]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i8>, ptr [[TMP15]], align 1
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave2.nxv8i8(<vscale x 8 x i8> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP2]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP10]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 2, [[TMP16]]
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP13:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ult <vscale x 4 x i32> [[TMP13]], [[BROADCAST_SPLAT2]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[EVL_BASED_IV]], 2
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[TMP14]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x i8> @llvm.masked.load.nxv8i8.p0(ptr [[TMP22]], i32 1, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x i8> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i8>, <vscale x 4 x i8> } @llvm.vector.deinterleave2.nxv8i8(<vscale x 8 x i8> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT:    [[TMP23:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 4 x i8>, <vscale x 4 x i8> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[TMP18:%.*]] = zext <vscale x 4 x i8> [[TMP17]] to <vscale x 4 x i32>
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i32, ptr [[DST]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x ptr> [[TMP19]], i32 4, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP18]], <vscale x 4 x ptr> align 4 [[TMP19]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP20]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP20]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_DEAD:%.*]] = load i8, ptr [[GEP_SRC_0]], align 1
 ; CHECK-NEXT:    [[IV_1:%.*]] = add i64 [[IV]], 1
@@ -453,7 +441,7 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
 ; CHECK-NEXT:    store i32 [[EXT]], ptr [[GEP_DST]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
 ; CHECK-NEXT:    [[EC:%.*]] = icmp slt i64 [[IV]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -481,26 +469,27 @@ exit:
 attributes #0 = { "target-features"="+64bit,+v" }
 
 ;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
-; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
-; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
-; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]]}
-; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
-; CHECK: [[META12]] = distinct !{[[META12]], [[META13:![0-9]+]]}
-; CHECK: [[META13]] = distinct !{[[META13]], !"LVerDomain"}
-; CHECK: [[META14]] = !{[[META15:![0-9]+]]}
-; CHECK: [[META15]] = distinct !{[[META15]], [[META13]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]]}
-; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]}
-; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META2]], [[META1]]}
-; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META1]], [[META2]]}
-; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META2]], [[META1]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
+; CHECK: [[META5]] = !{[[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], [[META7:![0-9]+]]}
+; CHECK: [[META7]] = distinct !{[[META7]], !"LVerDomain"}
+; CHECK: [[META8]] = !{[[META9:![0-9]+]]}
+; CHECK: [[META9]] = distinct !{[[META9]], [[META7]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META3]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
+; CHECK: [[META12]] = !{[[META13:![0-9]+]]}
+; CHECK: [[META13]] = distinct !{[[META13]], [[META14:![0-9]+]]}
+; CHECK: [[META14]] = distinct !{[[META14]], !"LVerDomain"}
+; CHECK: [[META15]] = !{[[META16:![0-9]+]]}
+; CHECK: [[META16]] = distinct !{[[META16]], [[META14]]}
+; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META1]], [[META3]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]]}
+; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META3]], [[META1]]}
+; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META3]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll
index db3215a6d2d3d..e985f1140d3fe 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll
@@ -13,44 +13,37 @@ target triple = "riscv64"
 define void @vector_add(ptr noalias nocapture %a, i64 %v) {
 ; CHECK-LABEL: @vector_add(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP9]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[ELEM]], [[V]]
 ; CHECK-NEXT:    store i64 [[ADD]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -74,44 +67,37 @@ for.end:
 define i64 @vector_add_reduce(ptr noalias nocapture %a) {
 ; CHECK-LABEL: @vector_add_reduce(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[TMP9]] = add <vscale x 2 x i64> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[VEC_PHI]], [[VP_OP_LOAD]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[VEC_PHI]], i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP9]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[SUM_NEXT]] = add i64 [[SUM]], [[ELEM]]
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i64 [ [[SUM_NEXT]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[SUM_NEXT_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index f02e5de8950b4..3af328fb6568e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -10,44 +10,37 @@ target triple = "riscv64"
 define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @vector_udiv(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP9]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[DIVREM:%.*]] = udiv i64 [[ELEM]], [[V]]
 ; CHECK-NEXT:    store i64 [[DIVREM]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -61,23 +54,18 @@ define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 4
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[TMP4:%.*]] = udiv <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
 ; FIXED-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; FIXED-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP1]], align 8
-; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; FIXED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[FOR_END:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[FOR_BODY:%.*]]
 ; FIXED:       for.body:
-; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; FIXED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; FIXED-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; FIXED-NEXT:    [[DIVREM:%.*]] = udiv i64 [[ELEM]], [[V]]
@@ -108,44 +96,37 @@ for.end:
 define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @vector_sdiv(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP9]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[DIVREM:%.*]] = sdiv i64 [[ELEM]], [[V]]
 ; CHECK-NEXT:    store i64 [[DIVREM]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -159,23 +140,18 @@ define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 4
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[TMP4:%.*]] = sdiv <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
 ; FIXED-NEXT:    [[TMP5:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; FIXED-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP1]], align 8
-; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; FIXED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[FOR_END:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[FOR_BODY:%.*]]
 ; FIXED:       for.body:
-; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; FIXED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; FIXED-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; FIXED-NEXT:    [[DIVREM:%.*]] = sdiv i64 [[ELEM]], [[V]]
@@ -206,44 +182,37 @@ for.end:
 define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @vector_urem(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = urem <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP9]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[DIVREM:%.*]] = urem i64 [[ELEM]], [[V]]
 ; CHECK-NEXT:    store i64 [[DIVREM]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -257,23 +226,18 @@ define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 4
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[TMP4:%.*]] = urem <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
 ; FIXED-NEXT:    [[TMP5:%.*]] = urem <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; FIXED-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP1]], align 8
-; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; FIXED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[FOR_END:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[FOR_BODY:%.*]]
 ; FIXED:       for.body:
-; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; FIXED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; FIXED-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; FIXED-NEXT:    [[DIVREM:%.*]] = urem i64 [[ELEM]], [[V]]
@@ -304,44 +268,37 @@ for.end:
 define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @vector_srem(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = srem <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP9]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP9]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP8]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[DIVREM:%.*]] = srem i64 [[ELEM]], [[V]]
 ; CHECK-NEXT:    store i64 [[DIVREM]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -355,23 +312,18 @@ define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 4
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[TMP4:%.*]] = srem <4 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
 ; FIXED-NEXT:    [[TMP5:%.*]] = srem <4 x i64> [[WIDE_LOAD1]], [[BROADCAST_SPLAT]]
-; FIXED-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP1]], align 8
-; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP1]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; FIXED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[FOR_END:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[FOR_BODY:%.*]]
 ; FIXED:       for.body:
-; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; FIXED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; FIXED-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; FIXED-NEXT:    [[DIVREM:%.*]] = srem i64 [[ELEM]], [[V]]
@@ -402,40 +354,38 @@ for.end:
 define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @predicated_udiv(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i64> splat (i64 1)
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP12]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult <vscale x 2 x i32> [[TMP7]], [[BROADCAST_SPLAT2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP16:%.*]] = select <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[TMP16]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i64> splat (i64 1)
 ; CHECK-NEXT:    [[TMP11:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], [[TMP10]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP8]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP16]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i64 [[V]], 0
@@ -448,7 +398,7 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:    store i64 [[PHI]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -464,25 +414,19 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 4
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
-; FIXED-NEXT:    [[TMP7:%.*]] = udiv <4 x i64> [[WIDE_LOAD]], [[TMP5]]
+; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
 ; FIXED-NEXT:    [[TMP8:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], [[TMP5]]
-; FIXED-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP7]], <4 x i64> [[WIDE_LOAD]]
 ; FIXED-NEXT:    [[PREDPHI2:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP8]], <4 x i64> [[WIDE_LOAD1]]
-; FIXED-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP2]], align 8
-; FIXED-NEXT:    store <4 x i64> [[PREDPHI2]], ptr [[TMP4]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; FIXED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[FOR_END:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[FOR_BODY:%.*]]
 ; FIXED:       for.body:
-; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; FIXED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; FIXED-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; FIXED-NEXT:    [[C:%.*]] = icmp ne i64 [[V]], 0
@@ -525,40 +469,38 @@ for.end:
 define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @predicated_sdiv(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i64> splat (i64 1)
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP12]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult <vscale x 2 x i32> [[TMP7]], [[BROADCAST_SPLAT2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP16:%.*]] = select <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = select <vscale x 2 x i1> [[TMP16]], <vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x i64> splat (i64 1)
 ; CHECK-NEXT:    [[TMP11:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], [[TMP10]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP8]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP16]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i64 [[V]], 0
@@ -571,7 +513,7 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:    store i64 [[PHI]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -587,25 +529,19 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; FIXED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 4
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
-; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
-; FIXED-NEXT:    [[TMP7:%.*]] = sdiv <4 x i64> [[WIDE_LOAD]], [[TMP5]]
+; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
 ; FIXED-NEXT:    [[TMP8:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], [[TMP5]]
-; FIXED-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP7]], <4 x i64> [[WIDE_LOAD]]
 ; FIXED-NEXT:    [[PREDPHI2:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP8]], <4 x i64> [[WIDE_LOAD1]]
-; FIXED-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP2]], align 8
-; FIXED-NEXT:    store <4 x i64> [[PREDPHI2]], ptr [[TMP4]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    store <4 x i64> [[PREDPHI2]], ptr [[TMP2]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; FIXED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[FOR_END:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[FOR_BODY:%.*]]
 ; FIXED:       for.body:
-; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; FIXED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; FIXED-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; FIXED-NEXT:    [[C:%.*]] = icmp ne i64 [[V]], 0
@@ -648,37 +584,35 @@ for.end:
 define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-LABEL: @predicated_udiv_by_constant(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP14]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)
 ; CHECK-NEXT:    [[TMP10:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 27)
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP9]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[WIDE_LOAD]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP9]], <vscale x 2 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[TMP10]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i64 [[ELEM]], 42
@@ -691,7 +625,7 @@ define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-NEXT:    store i64 [[PHI]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -703,27 +637,20 @@ define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 4
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
 ; FIXED-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD1]], splat (i64 42)
-; FIXED-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[WIDE_LOAD]], splat (i64 27)
 ; FIXED-NEXT:    [[TMP7:%.*]] = udiv <4 x i64> [[WIDE_LOAD1]], splat (i64 27)
-; FIXED-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP6]], <4 x i64> [[WIDE_LOAD]]
 ; FIXED-NEXT:    [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> [[TMP7]], <4 x i64> [[WIDE_LOAD1]]
-; FIXED-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP1]], align 8
-; FIXED-NEXT:    store <4 x i64> [[PREDPHI2]], ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    store <4 x i64> [[PREDPHI2]], ptr [[TMP1]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; FIXED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[FOR_END:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[FOR_BODY:%.*]]
 ; FIXED:       for.body:
-; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; FIXED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; FIXED-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; FIXED-NEXT:    [[C:%.*]] = icmp ne i64 [[ELEM]], 42
@@ -766,37 +693,35 @@ for.end:
 define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-LABEL: @predicated_sdiv_by_constant(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP14]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 42)
 ; CHECK-NEXT:    [[TMP10:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 27)
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP9]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[WIDE_LOAD]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[PREDPHI]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i1> [[TMP9]], <vscale x 2 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i64> [[WIDE_LOAD]], <vscale x 2 x i64> [[TMP10]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP12]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i64 [[ELEM]], 42
@@ -809,7 +734,7 @@ define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-NEXT:    store i64 [[PHI]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -821,27 +746,20 @@ define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 4
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD]], splat (i64 42)
+; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
 ; FIXED-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i64> [[WIDE_LOAD1]], splat (i64 42)
-; FIXED-NEXT:    [[TMP6:%.*]] = sdiv <4 x i64> [[WIDE_LOAD]], splat (i64 27)
 ; FIXED-NEXT:    [[TMP7:%.*]] = sdiv <4 x i64> [[WIDE_LOAD1]], splat (i64 27)
-; FIXED-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[TMP6]], <4 x i64> [[WIDE_LOAD]]
 ; FIXED-NEXT:    [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x i64> [[TMP7]], <4 x i64> [[WIDE_LOAD1]]
-; FIXED-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP1]], align 8
-; FIXED-NEXT:    store <4 x i64> [[PREDPHI2]], ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    store <4 x i64> [[PREDPHI2]], ptr [[TMP1]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; FIXED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[FOR_END:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[FOR_BODY:%.*]]
 ; FIXED:       for.body:
-; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; FIXED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; FIXED-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; FIXED-NEXT:    [[C:%.*]] = icmp ne i64 [[ELEM]], 42
@@ -884,38 +802,36 @@ for.end:
 define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-LABEL: @predicated_sdiv_by_minus_one(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP12]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp ult <vscale x 16 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP7]], align 1
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP12]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 -128)
-; CHECK-NEXT:    [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i8> splat (i8 -1), <vscale x 16 x i8> splat (i8 1)
+; CHECK-NEXT:    [[TMP16:%.*]] = select <vscale x 16 x i1> [[TMP15]], <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP16]], <vscale x 16 x i8> splat (i8 -1), <vscale x 16 x i8> splat (i8 1)
 ; CHECK-NEXT:    [[TMP11:%.*]] = sdiv <vscale x 16 x i8> [[WIDE_LOAD]], [[TMP10]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP9]], <vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[WIDE_LOAD]]
-; CHECK-NEXT:    store <vscale x 16 x i8> [[PREDPHI]], ptr [[TMP7]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 16 x i1> [[TMP16]], <vscale x 16 x i8> [[TMP11]], <vscale x 16 x i8> [[WIDE_LOAD]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[PREDPHI]], ptr align 1 [[TMP7]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i8 [[ELEM]], -128
@@ -928,7 +844,7 @@ define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-NEXT:    store i8 [[PHI]], ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -940,29 +856,21 @@ define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; FIXED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 32
-; FIXED-NEXT:    [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP1]], align 1
-; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP3]], align 1
-; FIXED-NEXT:    [[TMP4:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD]], splat (i8 -128)
+; FIXED-NEXT:    [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP1]], align 1
 ; FIXED-NEXT:    [[TMP5:%.*]] = icmp ne <32 x i8> [[WIDE_LOAD1]], splat (i8 -128)
-; FIXED-NEXT:    [[TMP6:%.*]] = select <32 x i1> [[TMP4]], <32 x i8> splat (i8 -1), <32 x i8> splat (i8 1)
 ; FIXED-NEXT:    [[TMP7:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> splat (i8 -1), <32 x i8> splat (i8 1)
-; FIXED-NEXT:    [[TMP8:%.*]] = sdiv <32 x i8> [[WIDE_LOAD]], [[TMP6]]
 ; FIXED-NEXT:    [[TMP9:%.*]] = sdiv <32 x i8> [[WIDE_LOAD1]], [[TMP7]]
-; FIXED-NEXT:    [[PREDPHI:%.*]] = select <32 x i1> [[TMP4]], <32 x i8> [[TMP8]], <32 x i8> [[WIDE_LOAD]]
 ; FIXED-NEXT:    [[PREDPHI2:%.*]] = select <32 x i1> [[TMP5]], <32 x i8> [[TMP9]], <32 x i8> [[WIDE_LOAD1]]
-; FIXED-NEXT:    store <32 x i8> [[PREDPHI]], ptr [[TMP1]], align 1
-; FIXED-NEXT:    store <32 x i8> [[PREDPHI2]], ptr [[TMP3]], align 1
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; FIXED-NEXT:    store <32 x i8> [[PREDPHI2]], ptr [[TMP1]], align 1
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
 ; FIXED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[FOR_END:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[FOR_BODY:%.*]]
 ; FIXED:       for.body:
-; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; FIXED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; FIXED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
 ; FIXED-NEXT:    [[ELEM:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; FIXED-NEXT:    [[C:%.*]] = icmp ne i8 [[ELEM]], -128
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
index 5f13089ff17fd..be6f32a6f4ea2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/evl-compatible-loops.ll
@@ -10,14 +10,6 @@ define void @test_wide_integer_induction(ptr noalias %a, i64 %N) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i64> [[TMP9]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP10]]
@@ -28,8 +20,7 @@ define void @test_wide_integer_induction(ptr noalias %a, i64 %N) {
 ; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[ENTRY]] ], [ [[AVL_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
 ; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VEC_IND]], ptr align 8 [[TMP14]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
@@ -42,10 +33,9 @@ define void @test_wide_integer_induction(ptr noalias %a, i64 %N) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY1:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV1]]
 ; CHECK-NEXT:    store i64 [[IV1]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
@@ -74,16 +64,40 @@ define void @test_wide_ptr_induction(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; CHECK-LABEL: define void @test_wide_ptr_induction(
 ; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[B]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <vscale x 2 x i64> [[TMP5]], splat (i64 8)
+; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 2 x i64> [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2p0.p0(<vscale x 2 x ptr> [[VECTOR_GEP]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 8, [[TMP10]]
+; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[ADDR:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[VECTOR_BODY]] ], [ [[B]], [[VECTOR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ADDR:%.*]] = phi ptr [ [[INCDEC_PTR:%.*]], [[FOR_BODY]] ], [ [[B]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds i8, ptr [[ADDR]], i64 8
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    store ptr [[ADDR]], ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw nsw i64 [[EVL_BASED_IV]], 1
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ;
@@ -109,4 +123,6 @@ for.cond.cleanup:
 ; CHECK: [[META2]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
 ; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/f16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/f16.ll
index effaf57a51e3c..94386888ad4f3 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/f16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/f16.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
 ; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+v -S | FileCheck %s -check-prefix=NO-ZVFHMIN
+; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+v -S -prefer-predicate-over-epilogue=scalar-epilogue | FileCheck %s -check-prefix=NO-ZVFHMIN
 ; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+v -S -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue | FileCheck %s -check-prefix=NO-ZVFHMIN
 ; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfhmin -S | FileCheck %s -check-prefix=ZVFHMIN
 
@@ -24,38 +25,31 @@ define void @fadd(ptr noalias %a, ptr noalias %b, i64 %n) {
 ;
 ; ZVFHMIN-LABEL: define void @fadd(
 ; ZVFHMIN-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; ZVFHMIN-NEXT:  [[ENTRY:.*]]:
-; ZVFHMIN-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
-; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP8]]
-; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; ZVFHMIN-NEXT:  [[ENTRY:.*:]]
+; ZVFHMIN-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
-; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
-; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
-; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP12]], 8
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
-; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFHMIN-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFHMIN-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; ZVFHMIN-NEXT:    [[TMP1:%.*]] = getelementptr half, ptr [[A]], i64 [[INDEX]]
 ; ZVFHMIN-NEXT:    [[TMP2:%.*]] = getelementptr half, ptr [[B]], i64 [[INDEX]]
-; ZVFHMIN-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP1]], align 2
-; ZVFHMIN-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x half>, ptr [[TMP2]], align 2
+; ZVFHMIN-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP1]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; ZVFHMIN-NEXT:    [[WIDE_LOAD1:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
 ; ZVFHMIN-NEXT:    [[TMP11:%.*]] = fadd <vscale x 8 x half> [[WIDE_LOAD]], [[WIDE_LOAD1]]
-; ZVFHMIN-NEXT:    store <vscale x 8 x half> [[TMP11]], ptr [[TMP1]], align 2
-; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; ZVFHMIN-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; ZVFHMIN-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; ZVFHMIN-NEXT:    call void @llvm.vp.store.nxv8f16.p0(<vscale x 8 x half> [[TMP11]], ptr align 2 [[TMP1]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; ZVFHMIN-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP6]] to i64
+; ZVFHMIN-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP13]], [[INDEX]]
+; ZVFHMIN-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; ZVFHMIN-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; ZVFHMIN-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; ZVFHMIN:       [[MIDDLE_BLOCK]]:
-; ZVFHMIN-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; ZVFHMIN-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; ZVFHMIN-NEXT:    br label %[[EXIT:.*]]
 ; ZVFHMIN:       [[SCALAR_PH]]:
-; ZVFHMIN-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; ZVFHMIN-NEXT:    br label %[[LOOP:.*]]
 ; ZVFHMIN:       [[LOOP]]:
-; ZVFHMIN-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; ZVFHMIN-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
 ; ZVFHMIN-NEXT:    [[A_GEP:%.*]] = getelementptr half, ptr [[A]], i64 [[I]]
 ; ZVFHMIN-NEXT:    [[B_GEP:%.*]] = getelementptr half, ptr [[B]], i64 [[I]]
 ; ZVFHMIN-NEXT:    [[X:%.*]] = load half, ptr [[A_GEP]], align 2
@@ -64,7 +58,7 @@ define void @fadd(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; ZVFHMIN-NEXT:    store half [[Z]], ptr [[A_GEP]], align 2
 ; ZVFHMIN-NEXT:    [[I_NEXT]] = add i64 [[I]], 1
 ; ZVFHMIN-NEXT:    [[DONE:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
-; ZVFHMIN-NEXT:    br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; ZVFHMIN-NEXT:    br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; ZVFHMIN:       [[EXIT]]:
 ; ZVFHMIN-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll b/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
index ce58ae11f3c01..7eb3d7fc5a36d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
@@ -15,7 +15,6 @@ define i64 @pr97452_scalable_vf1_for(ptr %src, ptr noalias %dst) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 23, [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 23, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
 ; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 [[TMP4]]
@@ -28,9 +27,9 @@ define i64 @pr97452_scalable_vf1_for(ptr %src, ptr noalias %dst) #0 {
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> [[VECTOR_RECUR]], <vscale x 1 x i64> [[WIDE_LOAD]], i32 -1)
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 1 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 [[TMP11]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
index c9ba2af92df7e..8b6db703d8274 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
@@ -11,8 +11,8 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP8]], 4
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 15, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
@@ -31,8 +31,6 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP18]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP19]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -43,7 +41,7 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[TMP17:%.*]] = call <vscale x 4 x float> @llvm.minimumnum.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD5]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP10]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -74,8 +72,8 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 15, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
@@ -94,8 +92,6 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -106,7 +102,7 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[TMP17:%.*]] = call <vscale x 4 x float> @llvm.minimumnum.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD5]])
 ; ZVFHMIN-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
 ; ZVFHMIN-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP18]], align 4
-; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; ZVFHMIN-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; ZVFHMIN-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; ZVFHMIN:       [[MIDDLE_BLOCK]]:
@@ -160,8 +156,8 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP8]], 4
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 15, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
@@ -180,8 +176,6 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP18]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP19]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -192,7 +186,7 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[TMP17:%.*]] = call <vscale x 4 x float> @llvm.maximumnum.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD5]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP10]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -223,8 +217,8 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 15, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
@@ -243,8 +237,6 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -255,7 +247,7 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[TMP17:%.*]] = call <vscale x 4 x float> @llvm.maximumnum.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD5]])
 ; ZVFHMIN-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw [4096 x float], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
 ; ZVFHMIN-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP18]], align 4
-; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; ZVFHMIN-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; ZVFHMIN-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; ZVFHMIN:       [[MIDDLE_BLOCK]]:
@@ -309,8 +301,8 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP8]], 2
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 15, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
@@ -329,8 +321,6 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP9]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP18]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP19]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -341,7 +331,7 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[TMP17:%.*]] = call <vscale x 2 x double> @llvm.minimumnum.nxv2f64(<vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD5]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 2 x double> [[TMP17]], ptr [[TMP10]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -372,8 +362,8 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 15, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
@@ -392,8 +382,6 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -404,7 +392,7 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[TMP17:%.*]] = call <vscale x 2 x double> @llvm.minimumnum.nxv2f64(<vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD5]])
 ; ZVFHMIN-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
 ; ZVFHMIN-NEXT:    store <vscale x 2 x double> [[TMP17]], ptr [[TMP18]], align 8
-; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; ZVFHMIN-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; ZVFHMIN-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; ZVFHMIN:       [[MIDDLE_BLOCK]]:
@@ -458,8 +446,8 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP8]], 2
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP8]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 15, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
@@ -478,8 +466,6 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP9]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP18]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP19]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -490,7 +476,7 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[TMP17:%.*]] = call <vscale x 2 x double> @llvm.maximumnum.nxv2f64(<vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD5]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 2 x double> [[TMP17]], ptr [[TMP10]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -521,8 +507,8 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 15, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
@@ -541,8 +527,6 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -553,7 +537,7 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[TMP17:%.*]] = call <vscale x 2 x double> @llvm.maximumnum.nxv2f64(<vscale x 2 x double> [[WIDE_LOAD]], <vscale x 2 x double> [[WIDE_LOAD5]])
 ; ZVFHMIN-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw [4096 x double], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
 ; ZVFHMIN-NEXT:    store <vscale x 2 x double> [[TMP17]], ptr [[TMP18]], align 8
-; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; ZVFHMIN-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; ZVFHMIN-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; ZVFHMIN:       [[MIDDLE_BLOCK]]:
@@ -606,11 +590,7 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT23:%.*]] = ptrtoint ptr [[INPUT2]] to i64
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP6]], 8
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
@@ -623,30 +603,27 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 4096, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[INPUT1]], i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP2]], align 2
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[INPUT2]], i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x half>, ptr [[TMP4]], align 2
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP4]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
 ; CHECK-NEXT:    [[TMP17:%.*]] = call <vscale x 8 x half> @llvm.minimumnum.nxv8f16(<vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[WIDE_LOAD5]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 8 x half> [[TMP17]], ptr [[TMP7]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
-; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8f16.p0(<vscale x 8 x half> [[TMP17]], ptr align 2 [[TMP7]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP20]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -659,7 +636,7 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    store half [[OUT]], ptr [[ARRAYIDX4]], align 2
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -669,11 +646,7 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT23:%.*]] = ptrtoint ptr [[INPUT2]] to i64
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
-; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
-; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
-; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; ZVFHMIN-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
 ; ZVFHMIN-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
@@ -686,30 +659,27 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; ZVFHMIN-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
-; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
-; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
-; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
-; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFHMIN-NEXT:    [[AVL:%.*]] = phi i64 [ 4096, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFHMIN-NEXT:    [[TMP19:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; ZVFHMIN-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[INPUT1]], i64 0, i64 [[INDEX]]
-; ZVFHMIN-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP13]], align 2
+; ZVFHMIN-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP13]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP19]])
 ; ZVFHMIN-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[INPUT2]], i64 0, i64 [[INDEX]]
-; ZVFHMIN-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x half>, ptr [[TMP15]], align 2
+; ZVFHMIN-NEXT:    [[WIDE_LOAD5:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP15]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP19]])
 ; ZVFHMIN-NEXT:    [[TMP17:%.*]] = call <vscale x 8 x half> @llvm.minimumnum.nxv8f16(<vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[WIDE_LOAD5]])
 ; ZVFHMIN-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
-; ZVFHMIN-NEXT:    store <vscale x 8 x half> [[TMP17]], ptr [[TMP18]], align 2
-; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
-; ZVFHMIN-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; ZVFHMIN-NEXT:    call void @llvm.vp.store.nxv8f16.p0(<vscale x 8 x half> [[TMP17]], ptr align 2 [[TMP18]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP19]])
+; ZVFHMIN-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP19]] to i64
+; ZVFHMIN-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; ZVFHMIN-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; ZVFHMIN-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 4096
 ; ZVFHMIN-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; ZVFHMIN:       [[MIDDLE_BLOCK]]:
-; ZVFHMIN-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, [[N_VEC]]
-; ZVFHMIN-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; ZVFHMIN-NEXT:    br label %[[EXIT:.*]]
 ; ZVFHMIN:       [[SCALAR_PH]]:
-; ZVFHMIN-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; ZVFHMIN-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; ZVFHMIN-NEXT:    br label %[[FOR_BODY:.*]]
 ; ZVFHMIN:       [[FOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -722,7 +692,7 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    store half [[OUT]], ptr [[ARRAYIDX4]], align 2
 ; ZVFHMIN-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; ZVFHMIN-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
-; ZVFHMIN-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; ZVFHMIN-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; ZVFHMIN:       [[EXIT]]:
 ; ZVFHMIN-NEXT:    ret void
 ;
@@ -755,11 +725,7 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT23:%.*]] = ptrtoint ptr [[INPUT2]] to i64
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP6]], 8
-; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
@@ -772,30 +738,27 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 4096, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[INPUT1]], i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP2]], align 2
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP2]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[INPUT2]], i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x half>, ptr [[TMP4]], align 2
+; CHECK-NEXT:    [[WIDE_LOAD5:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP4]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
 ; CHECK-NEXT:    [[TMP17:%.*]] = call <vscale x 8 x half> @llvm.maximumnum.nxv8f16(<vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[WIDE_LOAD5]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 8 x half> [[TMP17]], ptr [[TMP7]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
-; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8f16.p0(<vscale x 8 x half> [[TMP17]], ptr align 2 [[TMP7]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP20]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP20]]
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 4096
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -808,7 +771,7 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    store half [[OUT]], ptr [[ARRAYIDX4]], align 2
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 4096
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -818,11 +781,7 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT23:%.*]] = ptrtoint ptr [[INPUT2]] to i64
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
-; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
-; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
-; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; ZVFHMIN-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
 ; ZVFHMIN-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
 ; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
@@ -835,30 +794,27 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; ZVFHMIN-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
-; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
-; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
-; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
-; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
-; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFHMIN-NEXT:    [[AVL:%.*]] = phi i64 [ 4096, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; ZVFHMIN-NEXT:    [[TMP19:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; ZVFHMIN-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[INPUT1]], i64 0, i64 [[INDEX]]
-; ZVFHMIN-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP13]], align 2
+; ZVFHMIN-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP13]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP19]])
 ; ZVFHMIN-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[INPUT2]], i64 0, i64 [[INDEX]]
-; ZVFHMIN-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x half>, ptr [[TMP15]], align 2
+; ZVFHMIN-NEXT:    [[WIDE_LOAD5:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 2 [[TMP15]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP19]])
 ; ZVFHMIN-NEXT:    [[TMP17:%.*]] = call <vscale x 8 x half> @llvm.maximumnum.nxv8f16(<vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[WIDE_LOAD5]])
 ; ZVFHMIN-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw [4096 x half], ptr [[OUTPUT]], i64 0, i64 [[INDEX]]
-; ZVFHMIN-NEXT:    store <vscale x 8 x half> [[TMP17]], ptr [[TMP18]], align 2
-; ZVFHMIN-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
-; ZVFHMIN-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; ZVFHMIN-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; ZVFHMIN-NEXT:    call void @llvm.vp.store.nxv8f16.p0(<vscale x 8 x half> [[TMP17]], ptr align 2 [[TMP18]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP19]])
+; ZVFHMIN-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP19]] to i64
+; ZVFHMIN-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; ZVFHMIN-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; ZVFHMIN-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 4096
+; ZVFHMIN-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; ZVFHMIN:       [[MIDDLE_BLOCK]]:
-; ZVFHMIN-NEXT:    [[CMP_N:%.*]] = icmp eq i64 4096, [[N_VEC]]
-; ZVFHMIN-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; ZVFHMIN-NEXT:    br label %[[EXIT:.*]]
 ; ZVFHMIN:       [[SCALAR_PH]]:
-; ZVFHMIN-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; ZVFHMIN-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
 ; ZVFHMIN-NEXT:    br label %[[FOR_BODY:.*]]
 ; ZVFHMIN:       [[FOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
@@ -871,7 +827,7 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    store half [[OUT]], ptr [[ARRAYIDX4]], align 2
 ; ZVFHMIN-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; ZVFHMIN-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 4096
-; ZVFHMIN-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; ZVFHMIN-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; ZVFHMIN:       [[EXIT]]:
 ; ZVFHMIN-NEXT:    ret void
 ;
@@ -907,10 +863,11 @@ declare half @llvm.maximumnum.f16(half, half)
 ; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
 ; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META11:![0-9]+]], [[META2]]}
+; CHECK: [[META11]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]]}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]], [[META11]], [[META2]]}
+; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]]}
 ;.
 ; ZVFHMIN: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; ZVFHMIN: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
@@ -922,8 +879,9 @@ declare half @llvm.maximumnum.f16(half, half)
 ; ZVFHMIN: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]}
 ; ZVFHMIN: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
 ; ZVFHMIN: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
-; ZVFHMIN: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
-; ZVFHMIN: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
-; ZVFHMIN: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]}
-; ZVFHMIN: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]]}
+; ZVFHMIN: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META11:![0-9]+]], [[META2]]}
+; ZVFHMIN: [[META11]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; ZVFHMIN: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]]}
+; ZVFHMIN: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]], [[META11]], [[META2]]}
+; ZVFHMIN: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll b/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
index 1ea70b6d342f8..7819ae27d7238 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/force-vect-msg.ll
@@ -3,8 +3,8 @@
 
 ; CHECK: LV: Loop hints: force=enabled
 ; CHECK: LV: Scalar loop costs: 4.
-; ChosenFactor.Cost is 4, but the real cost will be divided by the width, which is 2.
-; CHECK: Cost for VF 2: 4 (Estimated cost per lane: 2.0)
+; ChosenFactor.Cost is 9, but the real cost will be divided by the width, which is 2.2.
+; CHECK: Cost for VF vscale x 2: 9
 ; Regardless of force vectorization or not, this loop will eventually be vectorized because of the cost model.
 ; Therefore, the following message does not need to be printed even if vectorization is explicitly forced in the metadata.
 ; CHECK-NOT: LV: Vectorization seems to be not beneficial, but was forced by a user.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
index e6825faf3f8d5..18dc20495117e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
@@ -20,11 +20,7 @@ define void @skip_free_iv_truncate(i16 %x, ptr %A) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = udiv i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[UMIN21]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.umax.i64(i64 128, i64 [[TMP7]])
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP5]], [[TMP8]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP31:%.*]] = shl nsw i64 [[X_I64]], 1
 ; CHECK-NEXT:    [[SCEVGEP9:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP31]]
@@ -57,42 +53,34 @@ define void @skip_free_iv_truncate(i16 %x, ptr %A) #0 {
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT19]]
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP45:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP46:%.*]] = mul nuw i64 [[TMP45]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP5]], [[TMP46]]
-; CHECK-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], i64 [[TMP46]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP5]], [[TMP48]]
-; CHECK-NEXT:    [[TMP51:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP52:%.*]] = mul nuw i64 [[TMP51]], 8
-; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[N_VEC]], 3
-; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[X_I64]], [[TMP49]]
-; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
-; CHECK-NEXT:    [[TMP50:%.*]] = mul i32 [[DOTCAST]], 3
-; CHECK-NEXT:    [[IND_END22:%.*]] = add i32 [[X_I32]], [[TMP50]]
 ; CHECK-NEXT:    [[TMP53:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[X_I64]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP55:%.*]] = mul <vscale x 8 x i64> [[TMP53]], splat (i64 3)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> [[DOTSPLAT]], [[TMP55]]
-; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 3, [[TMP52]]
-; CHECK-NEXT:    [[DOTSPLATINSERT24:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP58]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT25:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT24]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP5]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP27:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[TMP28:%.*]] = zext i32 [[TMP27]] to i64
+; CHECK-NEXT:    [[TMP58:%.*]] = mul i64 3, [[TMP28]]
+; CHECK-NEXT:    [[DOTSPLATINSERT24:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP58]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT25:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT24]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP59:%.*]] = getelementptr i16, ptr [[A]], <vscale x 8 x i64> [[VEC_IND]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> [[TMP59]], i32 2, <vscale x 8 x i1> splat (i1 true)), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP52]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv8i16.nxv8p0(<vscale x 8 x i16> zeroinitializer, <vscale x 8 x ptr> align 2 [[TMP59]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP27]]), !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT:    [[TMP47:%.*]] = zext i32 [[TMP27]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP47]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP47]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT25]]
-; CHECK-NEXT:    [[TMP60:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP60]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP48:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], %[[MIDDLE_BLOCK]] ], [ [[X_I64]], %[[ENTRY]] ], [ [[X_I64]], %[[VECTOR_MEMCHECK]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i32 [ [[IND_END22]], %[[MIDDLE_BLOCK]] ], [ [[X_I32]], %[[ENTRY]] ], [ [[X_I32]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[X_I64]], %[[ENTRY]] ], [ [[X_I64]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL13:%.*]] = phi i32 [ [[X_I32]], %[[ENTRY]] ], [ [[X_I32]], %[[VECTOR_MEMCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -107,7 +95,7 @@ define void @skip_free_iv_truncate(i16 %x, ptr %A) #0 {
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 3
 ; CHECK-NEXT:    [[TMP64]] = trunc i64 [[IV_NEXT]] to i32
 ; CHECK-NEXT:    [[C:%.*]] = icmp slt i64 [[IV]], 99
-; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT:.*]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -144,8 +132,9 @@ attributes #0 = { "target-features"="+64bit,+v,+zvl256b" }
 ; CHECK: [[META3]] = !{[[META4:![0-9]+]], [[META5:![0-9]+]]}
 ; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
 ; CHECK: [[META5]] = distinct !{[[META5]], [[META2]]}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]], [[META8:![0-9]+]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META7:![0-9]+]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
 ; CHECK: [[META7]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META8]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META7]]}
+; CHECK: [[META8]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[META9]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META7]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index 6e2434aefce9d..4fc9b781cb20b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -mtriple riscv64-linux-gnu -mattr=+v,+d -passes=loop-vectorize < %s -S -o - | FileCheck %s -check-prefix=OUTLOOP
-; RUN: opt -mtriple riscv64-linux-gnu -mattr=+v,+d -passes=loop-vectorize -prefer-inloop-reductions < %s -S -o - | FileCheck %s -check-prefix=INLOOP
+; RUN: opt -mtriple riscv64-linux-gnu -mattr=+v,+d -prefer-predicate-over-epilogue=scalar-epilogue -passes=loop-vectorize < %s -S -o - | FileCheck %s -check-prefix=OUTLOOP
+; RUN: opt -mtriple riscv64-linux-gnu -mattr=+v,+d -prefer-predicate-over-epilogue=scalar-epilogue -passes=loop-vectorize -prefer-inloop-reductions < %s -S -o - | FileCheck %s -check-prefix=INLOOP
 ; RUN: opt -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -mtriple=riscv64 -mattr=+v -S < %s 2>&1 | FileCheck --check-prefix=IF-EVL-OUTLOOP %s
 ; RUN: opt -passes=loop-vectorize -prefer-inloop-reductions -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -mtriple=riscv64 -mattr=+v -S < %s 2>&1 | FileCheck --check-prefix=IF-EVL-INLOOP %s
 
@@ -15,7 +15,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; OUTLOOP-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; OUTLOOP:       for.body.preheader:
 ; OUTLOOP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; OUTLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
+; OUTLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
 ; OUTLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[TMP1]]
 ; OUTLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; OUTLOOP:       vector.ph:
@@ -23,8 +23,6 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; OUTLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
 ; OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]]
 ; OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; OUTLOOP-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
 ; OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; OUTLOOP:       vector.body:
 ; OUTLOOP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -33,7 +31,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; OUTLOOP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP7]], align 2
 ; OUTLOOP-NEXT:    [[TMP9:%.*]] = sext <vscale x 4 x i16> [[WIDE_LOAD]] to <vscale x 4 x i32>
 ; OUTLOOP-NEXT:    [[TMP10]] = add <vscale x 4 x i32> [[VEC_PHI]], [[TMP9]]
-; OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
 ; OUTLOOP-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; OUTLOOP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; OUTLOOP:       middle.block:
@@ -67,7 +65,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; INLOOP-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; INLOOP:       for.body.preheader:
 ; INLOOP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 8
+; INLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 3
 ; INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[TMP1]]
 ; INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; INLOOP:       vector.ph:
@@ -75,8 +73,6 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 8
 ; INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]]
 ; INLOOP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; INLOOP-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 8
 ; INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; INLOOP:       vector.body:
 ; INLOOP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -86,7 +82,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; INLOOP-NEXT:    [[TMP9:%.*]] = sext <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x i32>
 ; INLOOP-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP9]])
 ; INLOOP-NEXT:    [[TMP11]] = add i32 [[VEC_PHI]], [[TMP10]]
-; INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
 ; INLOOP-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; INLOOP-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; INLOOP:       middle.block:
@@ -120,14 +116,6 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; IF-EVL-OUTLOOP:       for.body.preheader:
 ; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = sub i32 [[TMP1]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], [[TMP2]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.body:
 ; IF-EVL-OUTLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -147,12 +135,10 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; IF-EVL-OUTLOOP:       scalar.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[I_08]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; IF-EVL-OUTLOOP-NEXT:    [[CONV:%.*]] = sext i16 [[TMP13]] to i32
@@ -174,14 +160,6 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; IF-EVL-INLOOP:       for.body.preheader:
 ; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
-; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 8
-; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = sub i32 [[TMP1]], 1
-; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], [[TMP2]]
-; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-INLOOP-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 8
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -200,12 +178,10 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; IF-EVL-INLOOP:       middle.block:
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; IF-EVL-INLOOP:       scalar.ph:
-; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ]
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; IF-EVL-INLOOP-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; IF-EVL-INLOOP-NEXT:    [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; IF-EVL-INLOOP-NEXT:    [[R_07:%.*]] = phi i32 [ [[ADD:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[I_08]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP13:%.*]] = load i16, ptr [[ARRAYIDX]], align 2
 ; IF-EVL-INLOOP-NEXT:    [[CONV:%.*]] = sext i16 [[TMP13]] to i32
@@ -244,7 +220,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; OUTLOOP-LABEL: @smin(
 ; OUTLOOP-NEXT:  entry:
 ; OUTLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; OUTLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; OUTLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; OUTLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; OUTLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; OUTLOOP:       vector.ph:
@@ -252,8 +228,6 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; OUTLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; OUTLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -264,7 +238,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; OUTLOOP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; OUTLOOP-NEXT:    [[TMP9:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
 ; OUTLOOP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
-; OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; OUTLOOP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; OUTLOOP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; OUTLOOP:       middle.block:
@@ -292,7 +266,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; INLOOP-LABEL: @smin(
 ; INLOOP-NEXT:  entry:
 ; INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; INLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; INLOOP:       vector.ph:
@@ -300,8 +274,6 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; INLOOP:       vector.body:
 ; INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -310,7 +282,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; INLOOP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; INLOOP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; INLOOP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smin.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
-; INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; INLOOP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; INLOOP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; INLOOP:       middle.block:
@@ -338,21 +310,13 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
 ; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.body:
 ; IF-EVL-OUTLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -368,12 +332,10 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP15]])
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-OUTLOOP:       scalar.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-OUTLOOP-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP19]], [[RDX]]
@@ -389,19 +351,11 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
-; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -415,12 +369,10 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP:       middle.block:
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-INLOOP:       scalar.ph:
-; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-INLOOP-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP16]], [[RDX]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index e226eeac9e95f..dc963f1bf264f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -6,41 +6,40 @@
 define void @load_store_factor2_i32(ptr %p) {
 ; CHECK-LABEL: @load_store_factor2_i32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult <vscale x 4 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP14:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP14]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> [[TMP13]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr [[TMP15]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x i32> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_MASKED_VEC]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP8]], splat (i32 1)
 ; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 2)
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP11]])
-; CHECK-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> [[TMP13]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP15]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK1]])
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -53,7 +52,7 @@ define void @load_store_factor2_i32(ptr %p) {
 ; CHECK-NEXT:    store i32 [[Y1]], ptr [[Q1]], align 4
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -80,10 +79,9 @@ define void @load_store_factor2_i32(ptr %p) {
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -102,41 +100,40 @@ define void @load_store_factor2_i32(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor2_i32(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
-; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP7]], align 4
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp ult <vscale x 4 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP14:%.*]] = shl i64 [[INDEX]], 1
+; SCALABLE-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP14]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> [[TMP13]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr [[TMP15]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x i32> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_MASKED_VEC]])
 ; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP8]], splat (i32 1)
 ; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 2)
 ; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.vector.interleave2.nxv8i32(<vscale x 4 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP11]])
-; SCALABLE-NEXT:    store <vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> [[TMP13]])
+; SCALABLE-NEXT:    call void @llvm.masked.store.nxv8i32.p0(<vscale x 8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP15]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK1]])
+; SCALABLE-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; SCALABLE-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -149,7 +146,7 @@ define void @load_store_factor2_i32(ptr %p) {
 ; SCALABLE-NEXT:    store i32 [[Y1]], ptr [[Q1]], align 4
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -180,41 +177,40 @@ exit:
 define void @load_store_factor2_i64(ptr %p) {
 ; CHECK-LABEL: @load_store_factor2_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP8]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr [[TMP14]], i32 8, <vscale x 4 x i1> [[INTERLEAVED_MASK]], <vscale x 4 x i64> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT:    [[TMP20:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 1)
 ; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 2)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP11]])
-; CHECK-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP13]], <vscale x 2 x i64> [[TMP11]])
+; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv4i64.p0(<vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP14]], i32 8, <vscale x 4 x i1> [[INTERLEAVED_MASK1]])
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -227,7 +223,7 @@ define void @load_store_factor2_i64(ptr %p) {
 ; CHECK-NEXT:    store i64 [[Y1]], ptr [[Q1]], align 8
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -254,10 +250,9 @@ define void @load_store_factor2_i64(ptr %p) {
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -276,41 +271,40 @@ define void @load_store_factor2_i64(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor2_i64(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
-; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP7]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; SCALABLE-NEXT:    [[TMP10:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP8:%.*]] = shl i64 [[INDEX]], 1
+; SCALABLE-NEXT:    [[TMP14:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP8]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr [[TMP14]], i32 8, <vscale x 4 x i1> [[INTERLEAVED_MASK]], <vscale x 4 x i64> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_MASKED_VEC]])
+; SCALABLE-NEXT:    [[TMP20:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
 ; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[TMP8]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i64> [[TMP20]], splat (i64 1)
 ; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 2)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP11]])
-; SCALABLE-NEXT:    store <vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.vector.interleave2.nxv4i64(<vscale x 2 x i64> [[TMP13]], <vscale x 2 x i64> [[TMP11]])
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]])
+; SCALABLE-NEXT:    call void @llvm.masked.store.nxv4i64.p0(<vscale x 4 x i64> [[INTERLEAVED_VEC]], ptr [[TMP14]], i32 8, <vscale x 4 x i1> [[INTERLEAVED_MASK1]])
+; SCALABLE-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; SCALABLE-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -323,7 +317,7 @@ define void @load_store_factor2_i64(ptr %p) {
 ; SCALABLE-NEXT:    store i64 [[Y1]], ptr [[Q1]], align 8
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -354,24 +348,22 @@ exit:
 define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-LABEL: @load_store_factor3_i32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp ult <vscale x 4 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP16]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 12 x i1> @llvm.vector.interleave3.nxv12i1(<vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[TMP14]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.masked.load.nxv12i32.p0(ptr [[TMP17]], i32 4, <vscale x 12 x i1> [[INTERLEAVED_MASK]], <vscale x 12 x i32> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_MASKED_VEC]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
@@ -379,18 +371,19 @@ define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 3)
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> [[TMP13]])
-; CHECK-NEXT:    store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 12 x i1> @llvm.vector.interleave3.nxv12i1(<vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[TMP14]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv12i32.p0(<vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP17]], i32 4, <vscale x 12 x i1> [[INTERLEAVED_MASK1]])
+; CHECK-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP19]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -408,7 +401,7 @@ define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-NEXT:    store i32 [[Y2]], ptr [[Q2]], align 4
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -439,10 +432,9 @@ define void @load_store_factor3_i32(ptr %p) {
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -466,24 +458,22 @@ define void @load_store_factor3_i32(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor3_i32(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
-; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP1]], align 4
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp ult <vscale x 4 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP16:%.*]] = mul i64 [[INDEX]], 3
+; SCALABLE-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP16]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 12 x i1> @llvm.vector.interleave3.nxv12i1(<vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[TMP14]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.masked.load.nxv12i32.p0(ptr [[TMP17]], i32 4, <vscale x 12 x i1> [[INTERLEAVED_MASK]], <vscale x 12 x i32> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_MASKED_VEC]])
 ; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
@@ -491,18 +481,19 @@ define void @load_store_factor3_i32(ptr %p) {
 ; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP9]], splat (i32 2)
 ; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP10]], splat (i32 3)
 ; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[TMP12]], <vscale x 4 x i32> [[TMP13]])
-; SCALABLE-NEXT:    store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 4
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 12 x i1> @llvm.vector.interleave3.nxv12i1(<vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i1> [[TMP14]])
+; SCALABLE-NEXT:    call void @llvm.masked.store.nxv12i32.p0(<vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP17]], i32 4, <vscale x 12 x i1> [[INTERLEAVED_MASK1]])
+; SCALABLE-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP19]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
+; SCALABLE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -520,7 +511,7 @@ define void @load_store_factor3_i32(ptr %p) {
 ; SCALABLE-NEXT:    store i32 [[Y2]], ptr [[Q2]], align 4
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -557,43 +548,42 @@ exit:
 define void @load_store_factor3_i64(ptr %p) {
 ; CHECK-LABEL: @load_store_factor3_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 6 x i64>, ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave3.nxv6i64(<vscale x 6 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP8]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 6 x i1> @llvm.vector.interleave3.nxv6i1(<vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP11]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.masked.load.nxv6i64.p0(ptr [[TMP14]], i32 8, <vscale x 6 x i1> [[INTERLEAVED_MASK]], <vscale x 6 x i64> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave3.nxv6i64(<vscale x 6 x i64> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT:    [[TMP23:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 2
-; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT:    [[TMP25:%.*]] = add <vscale x 2 x i64> [[TMP23]], splat (i64 1)
 ; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 2)
 ; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 3)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave3.nxv6i64(<vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP13]])
-; CHECK-NEXT:    store <vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave3.nxv6i64(<vscale x 2 x i64> [[TMP25]], <vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP13]])
+; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 6 x i1> @llvm.vector.interleave3.nxv6i1(<vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP11]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv6i64.p0(<vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP14]], i32 8, <vscale x 6 x i1> [[INTERLEAVED_MASK1]])
+; CHECK-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP19]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
+; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -611,7 +601,7 @@ define void @load_store_factor3_i64(ptr %p) {
 ; CHECK-NEXT:    store i64 [[Y2]], ptr [[Q2]], align 8
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -642,10 +632,9 @@ define void @load_store_factor3_i64(ptr %p) {
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -669,43 +658,42 @@ define void @load_store_factor3_i64(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor3_i64(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
-; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 6 x i64>, ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave3.nxv6i64(<vscale x 6 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; SCALABLE-NEXT:    [[TMP11:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP8:%.*]] = mul i64 [[INDEX]], 3
+; SCALABLE-NEXT:    [[TMP14:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP8]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 6 x i1> @llvm.vector.interleave3.nxv6i1(<vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP11]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.masked.load.nxv6i64.p0(ptr [[TMP14]], i32 8, <vscale x 6 x i1> [[INTERLEAVED_MASK]], <vscale x 6 x i64> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave3.nxv6i64(<vscale x 6 x i64> [[WIDE_MASKED_VEC]])
+; SCALABLE-NEXT:    [[TMP23:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
 ; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 2
-; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 2 x i64> [[TMP8]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP25:%.*]] = add <vscale x 2 x i64> [[TMP23]], splat (i64 1)
 ; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 2 x i64> [[TMP9]], splat (i64 2)
 ; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 3)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave3.nxv6i64(<vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP13]])
-; SCALABLE-NEXT:    store <vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave3.nxv6i64(<vscale x 2 x i64> [[TMP25]], <vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP13]])
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 6 x i1> @llvm.vector.interleave3.nxv6i1(<vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP11]], <vscale x 2 x i1> [[TMP11]])
+; SCALABLE-NEXT:    call void @llvm.masked.store.nxv6i64.p0(<vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP14]], i32 8, <vscale x 6 x i1> [[INTERLEAVED_MASK1]])
+; SCALABLE-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP19]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
+; SCALABLE-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 3
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -723,7 +711,7 @@ define void @load_store_factor3_i64(ptr %p) {
 ; SCALABLE-NEXT:    store i64 [[Y2]], ptr [[Q2]], align 8
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -760,45 +748,44 @@ exit:
 define void @load_store_factor4(ptr %p) {
 ; CHECK-LABEL: @load_store_factor4(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave4.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP8]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64.p0(ptr [[TMP9]], i32 8, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x i64> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave4.nxv8i64(<vscale x 8 x i64> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT:    [[TMP24:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 2
 ; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 3
-; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 1)
+; CHECK-NEXT:    [[TMP26:%.*]] = add <vscale x 2 x i64> [[TMP24]], splat (i64 1)
 ; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP11]], splat (i64 2)
 ; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 2 x i64> [[TMP12]], splat (i64 3)
 ; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 2 x i64> [[TMP13]], splat (i64 4)
-; CHECK-NEXT:    [[INTERLEAVED_VEC4:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave4.nxv8i64(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> [[TMP17]])
-; CHECK-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC4]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave4.nxv8i64(<vscale x 2 x i64> [[TMP26]], <vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> [[TMP17]])
+; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv8i64.p0(<vscale x 8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP9]], i32 8, <vscale x 8 x i1> [[INTERLEAVED_MASK1]])
+; CHECK-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP22]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP22]]
+; CHECK-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 4
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -821,7 +808,7 @@ define void @load_store_factor4(ptr %p) {
 ; CHECK-NEXT:    store i64 [[Y3]], ptr [[Q3]], align 8
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -854,10 +841,9 @@ define void @load_store_factor4(ptr %p) {
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 4
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -886,45 +872,44 @@ define void @load_store_factor4(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor4(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 4
-; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[TMP7]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave4.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; SCALABLE-NEXT:    [[TMP10:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP8:%.*]] = mul i64 [[INDEX]], 4
+; SCALABLE-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP8]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64.p0(ptr [[TMP9]], i32 8, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x i64> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave4.nxv8i64(<vscale x 8 x i64> [[WIDE_MASKED_VEC]])
+; SCALABLE-NEXT:    [[TMP24:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
 ; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
 ; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 2
 ; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 3
-; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 2 x i64> [[TMP10]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP26:%.*]] = add <vscale x 2 x i64> [[TMP24]], splat (i64 1)
 ; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 2 x i64> [[TMP11]], splat (i64 2)
 ; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 2 x i64> [[TMP12]], splat (i64 3)
 ; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 2 x i64> [[TMP13]], splat (i64 4)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC4:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave4.nxv8i64(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> [[TMP17]])
-; SCALABLE-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC4]], ptr [[TMP7]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave4.nxv8i64(<vscale x 2 x i64> [[TMP26]], <vscale x 2 x i64> [[TMP15]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> [[TMP17]])
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]], <vscale x 2 x i1> [[TMP10]])
+; SCALABLE-NEXT:    call void @llvm.masked.store.nxv8i64.p0(<vscale x 8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP9]], i32 8, <vscale x 8 x i1> [[INTERLEAVED_MASK1]])
+; SCALABLE-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP22]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP22]]
+; SCALABLE-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 4
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -947,7 +932,7 @@ define void @load_store_factor4(ptr %p) {
 ; SCALABLE-NEXT:    store i64 [[Y3]], ptr [[Q3]], align 8
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -990,44 +975,46 @@ exit:
 define void @load_store_factor5(ptr %p) {
 ; CHECK-LABEL: @load_store_factor5(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 5
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 5 x i64>, ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave5.nxv5i64(<vscale x 5 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
-; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
-; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
-; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
-; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
-; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 5 x i64> @llvm.vector.interleave5.nxv5i64(<vscale x 1 x i64> [[TMP10]], <vscale x 1 x i64> [[TMP11]], <vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]])
-; CHECK-NEXT:    store <vscale x 5 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = call <vscale x 1 x i32> @llvm.stepvector.nxv1i32()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <vscale x 1 x i32> [[TMP18]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 5 x i1> @llvm.vector.interleave5.nxv5i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 5 x i64> @llvm.masked.load.nxv5i64.p0(ptr [[TMP19]], i32 8, <vscale x 5 x i1> [[INTERLEAVED_MASK]], <vscale x 5 x i64> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave5.nxv5i64(<vscale x 5 x i64> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 2)
+; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 3)
+; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 4)
+; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 5)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 5 x i64> @llvm.vector.interleave5.nxv5i64(<vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]])
+; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 5 x i1> @llvm.vector.interleave5.nxv5i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv5i64.p0(<vscale x 5 x i64> [[INTERLEAVED_VEC]], ptr [[TMP19]], i32 8, <vscale x 5 x i1> [[INTERLEAVED_MASK1]])
+; CHECK-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP25]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP25]]
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 5
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1055,7 +1042,7 @@ define void @load_store_factor5(ptr %p) {
 ; CHECK-NEXT:    store i64 [[Y4]], ptr [[Q4]], align 8
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -1092,10 +1079,9 @@ define void @load_store_factor5(ptr %p) {
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 5
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1129,44 +1115,46 @@ define void @load_store_factor5(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor5(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 5
-; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 5 x i64>, ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave5.nxv5i64(<vscale x 5 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
-; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
-; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
-; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
-; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 5 x i64> @llvm.vector.interleave5.nxv5i64(<vscale x 1 x i64> [[TMP10]], <vscale x 1 x i64> [[TMP11]], <vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]])
-; SCALABLE-NEXT:    store <vscale x 5 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; SCALABLE-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP18:%.*]] = call <vscale x 1 x i32> @llvm.stepvector.nxv1i32()
+; SCALABLE-NEXT:    [[TMP5:%.*]] = icmp ult <vscale x 1 x i32> [[TMP18]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 5
+; SCALABLE-NEXT:    [[TMP19:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 5 x i1> @llvm.vector.interleave5.nxv5i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 5 x i64> @llvm.masked.load.nxv5i64.p0(ptr [[TMP19]], i32 8, <vscale x 5 x i1> [[INTERLEAVED_MASK]], <vscale x 5 x i64> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave5.nxv5i64(<vscale x 5 x i64> [[WIDE_MASKED_VEC]])
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 4)
+; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 5)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 5 x i64> @llvm.vector.interleave5.nxv5i64(<vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]])
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 5 x i1> @llvm.vector.interleave5.nxv5i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; SCALABLE-NEXT:    call void @llvm.masked.store.nxv5i64.p0(<vscale x 5 x i64> [[INTERLEAVED_VEC]], ptr [[TMP19]], i32 8, <vscale x 5 x i1> [[INTERLEAVED_MASK1]])
+; SCALABLE-NEXT:    [[TMP25:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP25]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP25]]
+; SCALABLE-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 5
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1194,7 +1182,7 @@ define void @load_store_factor5(ptr %p) {
 ; SCALABLE-NEXT:    store i64 [[Y4]], ptr [[Q4]], align 8
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -1243,46 +1231,48 @@ exit:
 define void @load_store_factor6(ptr %p) {
 ; CHECK-LABEL: @load_store_factor6(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 6
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 6 x i64>, ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave6.nxv6i64(<vscale x 6 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
-; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
-; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
-; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
-; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
-; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
-; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 6)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave6.nxv6i64(<vscale x 1 x i64> [[TMP11]], <vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]])
-; CHECK-NEXT:    store <vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP20:%.*]] = call <vscale x 1 x i32> @llvm.stepvector.nxv1i32()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <vscale x 1 x i32> [[TMP20]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 6 x i1> @llvm.vector.interleave6.nxv6i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.masked.load.nxv6i64.p0(ptr [[TMP21]], i32 8, <vscale x 6 x i1> [[INTERLEAVED_MASK]], <vscale x 6 x i64> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave6.nxv6i64(<vscale x 6 x i64> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 2)
+; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 3)
+; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 4)
+; CHECK-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 5)
+; CHECK-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 6)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave6.nxv6i64(<vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]], <vscale x 1 x i64> [[TMP19]])
+; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 6 x i1> @llvm.vector.interleave6.nxv6i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv6i64.p0(<vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP21]], i32 8, <vscale x 6 x i1> [[INTERLEAVED_MASK1]])
+; CHECK-NEXT:    [[TMP28:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP28]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 6
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1315,7 +1305,7 @@ define void @load_store_factor6(ptr %p) {
 ; CHECK-NEXT:    store i64 [[Y5]], ptr [[Q5]], align 8
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -1355,10 +1345,9 @@ define void @load_store_factor6(ptr %p) {
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 6
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1397,46 +1386,48 @@ define void @load_store_factor6(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor6(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 6
-; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 6 x i64>, ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave6.nxv6i64(<vscale x 6 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
-; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
-; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
-; SCALABLE-NEXT:    [[TMP11:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
-; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
-; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 6)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave6.nxv6i64(<vscale x 1 x i64> [[TMP11]], <vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]])
-; SCALABLE-NEXT:    store <vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; SCALABLE-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP20:%.*]] = call <vscale x 1 x i32> @llvm.stepvector.nxv1i32()
+; SCALABLE-NEXT:    [[TMP5:%.*]] = icmp ult <vscale x 1 x i32> [[TMP20]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 6
+; SCALABLE-NEXT:    [[TMP21:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 6 x i1> @llvm.vector.interleave6.nxv6i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.masked.load.nxv6i64.p0(ptr [[TMP21]], i32 8, <vscale x 6 x i1> [[INTERLEAVED_MASK]], <vscale x 6 x i64> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave6.nxv6i64(<vscale x 6 x i64> [[WIDE_MASKED_VEC]])
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 4)
+; SCALABLE-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 5)
+; SCALABLE-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 6)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 6 x i64> @llvm.vector.interleave6.nxv6i64(<vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]], <vscale x 1 x i64> [[TMP19]])
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 6 x i1> @llvm.vector.interleave6.nxv6i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; SCALABLE-NEXT:    call void @llvm.masked.store.nxv6i64.p0(<vscale x 6 x i64> [[INTERLEAVED_VEC]], ptr [[TMP21]], i32 8, <vscale x 6 x i1> [[INTERLEAVED_MASK1]])
+; SCALABLE-NEXT:    [[TMP28:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP28]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP28]]
+; SCALABLE-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 6
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1469,7 +1460,7 @@ define void @load_store_factor6(ptr %p) {
 ; SCALABLE-NEXT:    store i64 [[Y5]], ptr [[Q5]], align 8
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -1524,48 +1515,50 @@ exit:
 define void @load_store_factor7(ptr %p) {
 ; CHECK-LABEL: @load_store_factor7(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 7
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 7 x i64>, ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave7.nxv7i64(<vscale x 7 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
-; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
-; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
-; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
-; CHECK-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
-; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
-; CHECK-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
-; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
-; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
-; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 6)
-; CHECK-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 7)
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 7 x i64> @llvm.vector.interleave7.nxv7i64(<vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]])
-; CHECK-NEXT:    store <vscale x 7 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP22:%.*]] = call <vscale x 1 x i32> @llvm.stepvector.nxv1i32()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <vscale x 1 x i32> [[TMP22]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 7 x i1> @llvm.vector.interleave7.nxv7i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 7 x i64> @llvm.masked.load.nxv7i64.p0(ptr [[TMP23]], i32 8, <vscale x 7 x i1> [[INTERLEAVED_MASK]], <vscale x 7 x i64> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave7.nxv7i64(<vscale x 7 x i64> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
+; CHECK-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 2)
+; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 3)
+; CHECK-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 4)
+; CHECK-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 5)
+; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 6)
+; CHECK-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 7)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 7 x i64> @llvm.vector.interleave7.nxv7i64(<vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]], <vscale x 1 x i64> [[TMP19]], <vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP21]])
+; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 7 x i1> @llvm.vector.interleave7.nxv7i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv7i64.p0(<vscale x 7 x i64> [[INTERLEAVED_VEC]], ptr [[TMP23]], i32 8, <vscale x 7 x i1> [[INTERLEAVED_MASK1]])
+; CHECK-NEXT:    [[TMP31:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP31]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP31]]
+; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 7
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1603,7 +1596,7 @@ define void @load_store_factor7(ptr %p) {
 ; CHECK-NEXT:    store i64 [[Y6]], ptr [[Q6]], align 8
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -1647,10 +1640,9 @@ define void @load_store_factor7(ptr %p) {
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 7
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1694,48 +1686,50 @@ define void @load_store_factor7(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor7(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 7
-; SCALABLE-NEXT:    [[TMP1:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP0]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 7 x i64>, ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave7.nxv7i64(<vscale x 7 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP6:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
-; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
-; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
-; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
-; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
-; SCALABLE-NEXT:    [[TMP12:%.*]] = add <vscale x 1 x i64> [[TMP5]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP13:%.*]] = add <vscale x 1 x i64> [[TMP6]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP14:%.*]] = add <vscale x 1 x i64> [[TMP7]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 4)
-; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 5)
-; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 6)
-; SCALABLE-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 7)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 7 x i64> @llvm.vector.interleave7.nxv7i64(<vscale x 1 x i64> [[TMP12]], <vscale x 1 x i64> [[TMP13]], <vscale x 1 x i64> [[TMP14]], <vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]])
-; SCALABLE-NEXT:    store <vscale x 7 x i64> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; SCALABLE-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP22:%.*]] = call <vscale x 1 x i32> @llvm.stepvector.nxv1i32()
+; SCALABLE-NEXT:    [[TMP5:%.*]] = icmp ult <vscale x 1 x i32> [[TMP22]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = mul i64 [[INDEX]], 7
+; SCALABLE-NEXT:    [[TMP23:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 7 x i1> @llvm.vector.interleave7.nxv7i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 7 x i64> @llvm.masked.load.nxv7i64.p0(ptr [[TMP23]], i32 8, <vscale x 7 x i1> [[INTERLEAVED_MASK]], <vscale x 7 x i64> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave7.nxv7i64(<vscale x 7 x i64> [[WIDE_MASKED_VEC]])
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; SCALABLE-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
+; SCALABLE-NEXT:    [[TMP15:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 4)
+; SCALABLE-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 5)
+; SCALABLE-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 6)
+; SCALABLE-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 7)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 7 x i64> @llvm.vector.interleave7.nxv7i64(<vscale x 1 x i64> [[TMP15]], <vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]], <vscale x 1 x i64> [[TMP19]], <vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP21]])
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 7 x i1> @llvm.vector.interleave7.nxv7i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; SCALABLE-NEXT:    call void @llvm.masked.store.nxv7i64.p0(<vscale x 7 x i64> [[INTERLEAVED_VEC]], ptr [[TMP23]], i32 8, <vscale x 7 x i1> [[INTERLEAVED_MASK1]])
+; SCALABLE-NEXT:    [[TMP31:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP31]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP31]]
+; SCALABLE-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = mul i64 [[I]], 7
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1773,7 +1767,7 @@ define void @load_store_factor7(ptr %p) {
 ; SCALABLE-NEXT:    store i64 [[Y6]], ptr [[Q6]], align 8
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP18:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -1834,50 +1828,52 @@ exit:
 define void @load_store_factor8(ptr %p) {
 ; CHECK-LABEL: @load_store_factor8(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP3]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[TMP4]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave8.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
-; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
-; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
-; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
-; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
-; CHECK-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
-; CHECK-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
-; CHECK-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 7
-; CHECK-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 1)
-; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 2)
-; CHECK-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 3)
-; CHECK-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 4)
-; CHECK-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 5)
-; CHECK-NEXT:    [[TMP24:%.*]] = add <vscale x 1 x i64> [[TMP16]], splat (i64 6)
-; CHECK-NEXT:    [[TMP25:%.*]] = add <vscale x 1 x i64> [[TMP17]], splat (i64 7)
-; CHECK-NEXT:    [[TMP26:%.*]] = add <vscale x 1 x i64> [[TMP18]], splat (i64 8)
-; CHECK-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave8.nxv8i64(<vscale x 1 x i64> [[TMP19]], <vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP23]], <vscale x 1 x i64> [[TMP24]], <vscale x 1 x i64> [[TMP25]], <vscale x 1 x i64> [[TMP26]])
-; CHECK-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC12]], ptr [[TMP4]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i32> @llvm.stepvector.nxv1i32()
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ult <vscale x 1 x i32> [[TMP4]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave8.nxv8i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64.p0(ptr [[TMP24]], i32 8, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x i64> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave8.nxv8i64(<vscale x 8 x i64> [[WIDE_MASKED_VEC]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; CHECK-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
+; CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 7
+; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 1)
+; CHECK-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 2)
+; CHECK-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 3)
+; CHECK-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 4)
+; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 5)
+; CHECK-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 6)
+; CHECK-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 7)
+; CHECK-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 8)
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave8.nxv8i64(<vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]], <vscale x 1 x i64> [[TMP19]], <vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP23]])
+; CHECK-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave8.nxv8i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; CHECK-NEXT:    call void @llvm.masked.store.nxv8i64.p0(<vscale x 8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP24]], i32 8, <vscale x 8 x i1> [[INTERLEAVED_MASK1]])
+; CHECK-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP34]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP34]]
+; CHECK-NEXT:    [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -1920,7 +1916,7 @@ define void @load_store_factor8(ptr %p) {
 ; CHECK-NEXT:    store i64 [[Y7]], ptr [[Q7]], align 8
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -1965,10 +1961,9 @@ define void @load_store_factor8(ptr %p) {
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -2017,50 +2012,52 @@ define void @load_store_factor8(ptr %p) {
 ;
 ; SCALABLE-LABEL: @load_store_factor8(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP0]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP3:%.*]] = shl i64 [[INDEX]], 3
-; SCALABLE-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP3]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i64>, ptr [[TMP4]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave8.nxv8i64(<vscale x 8 x i64> [[WIDE_VEC]])
-; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
-; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
-; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
-; SCALABLE-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
-; SCALABLE-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
-; SCALABLE-NEXT:    [[TMP16:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
-; SCALABLE-NEXT:    [[TMP17:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
-; SCALABLE-NEXT:    [[TMP18:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 7
-; SCALABLE-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 2)
-; SCALABLE-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 3)
-; SCALABLE-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 4)
-; SCALABLE-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 5)
-; SCALABLE-NEXT:    [[TMP24:%.*]] = add <vscale x 1 x i64> [[TMP16]], splat (i64 6)
-; SCALABLE-NEXT:    [[TMP25:%.*]] = add <vscale x 1 x i64> [[TMP17]], splat (i64 7)
-; SCALABLE-NEXT:    [[TMP26:%.*]] = add <vscale x 1 x i64> [[TMP18]], splat (i64 8)
-; SCALABLE-NEXT:    [[INTERLEAVED_VEC12:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave8.nxv8i64(<vscale x 1 x i64> [[TMP19]], <vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP23]], <vscale x 1 x i64> [[TMP24]], <vscale x 1 x i64> [[TMP25]], <vscale x 1 x i64> [[TMP26]])
-; SCALABLE-NEXT:    store <vscale x 8 x i64> [[INTERLEAVED_VEC12]], ptr [[TMP4]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; SCALABLE-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 1, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 1 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i32> poison, <vscale x 1 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP4:%.*]] = call <vscale x 1 x i32> @llvm.stepvector.nxv1i32()
+; SCALABLE-NEXT:    [[TMP5:%.*]] = icmp ult <vscale x 1 x i32> [[TMP4]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 3
+; SCALABLE-NEXT:    [[TMP24:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave8.nxv8i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x i64> @llvm.masked.load.nxv8i64.p0(ptr [[TMP24]], i32 8, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x i64> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } @llvm.vector.deinterleave8.nxv8i64(<vscale x 8 x i64> [[WIDE_MASKED_VEC]])
+; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 0
+; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 1
+; SCALABLE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 2
+; SCALABLE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 3
+; SCALABLE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 4
+; SCALABLE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 5
+; SCALABLE-NEXT:    [[TMP14:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 6
+; SCALABLE-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64> } [[STRIDED_VEC]], 7
+; SCALABLE-NEXT:    [[TMP16:%.*]] = add <vscale x 1 x i64> [[TMP8]], splat (i64 1)
+; SCALABLE-NEXT:    [[TMP17:%.*]] = add <vscale x 1 x i64> [[TMP9]], splat (i64 2)
+; SCALABLE-NEXT:    [[TMP18:%.*]] = add <vscale x 1 x i64> [[TMP10]], splat (i64 3)
+; SCALABLE-NEXT:    [[TMP19:%.*]] = add <vscale x 1 x i64> [[TMP11]], splat (i64 4)
+; SCALABLE-NEXT:    [[TMP20:%.*]] = add <vscale x 1 x i64> [[TMP12]], splat (i64 5)
+; SCALABLE-NEXT:    [[TMP21:%.*]] = add <vscale x 1 x i64> [[TMP13]], splat (i64 6)
+; SCALABLE-NEXT:    [[TMP22:%.*]] = add <vscale x 1 x i64> [[TMP14]], splat (i64 7)
+; SCALABLE-NEXT:    [[TMP23:%.*]] = add <vscale x 1 x i64> [[TMP15]], splat (i64 8)
+; SCALABLE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 8 x i64> @llvm.vector.interleave8.nxv8i64(<vscale x 1 x i64> [[TMP16]], <vscale x 1 x i64> [[TMP17]], <vscale x 1 x i64> [[TMP18]], <vscale x 1 x i64> [[TMP19]], <vscale x 1 x i64> [[TMP20]], <vscale x 1 x i64> [[TMP21]], <vscale x 1 x i64> [[TMP22]], <vscale x 1 x i64> [[TMP23]])
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK1:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave8.nxv8i1(<vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]], <vscale x 1 x i1> [[TMP5]])
+; SCALABLE-NEXT:    call void @llvm.masked.store.nxv8i64.p0(<vscale x 8 x i64> [[INTERLEAVED_VEC]], ptr [[TMP24]], i32 8, <vscale x 8 x i1> [[INTERLEAVED_MASK1]])
+; SCALABLE-NEXT:    [[TMP34:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP34]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP34]]
+; SCALABLE-NEXT:    [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 3
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -2103,7 +2100,7 @@ define void @load_store_factor8(ptr %p) {
 ; SCALABLE-NEXT:    store i64 [[Y7]], ptr [[Q7]], align 8
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -2170,40 +2167,38 @@ exit:
 define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; CHECK-LABEL: @combine_load_factor2_i32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult <vscale x 4 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP13]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> [[TMP12]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr [[TMP15]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x i32> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_MASKED_VEC]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP10]], ptr [[TMP11]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP10]], ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -2215,7 +2210,7 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    store i32 [[RES]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -2226,33 +2221,23 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 8
-; FIXED-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX]], 1
-; FIXED-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP0]], 1
-; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP1]]
-; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP2]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP3]], align 4
-; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
-; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; FIXED-NEXT:    [[TMP2:%.*]] = shl i64 [[INDEX]], 1
+; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP2]]
 ; FIXED-NEXT:    [[WIDE_VEC2:%.*]] = load <16 x i32>, ptr [[TMP4]], align 4
 ; FIXED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
 ; FIXED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <16 x i32> [[WIDE_VEC2]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
-; FIXED-NEXT:    [[TMP5:%.*]] = add <8 x i32> [[STRIDED_VEC]], [[STRIDED_VEC1]]
 ; FIXED-NEXT:    [[TMP6:%.*]] = add <8 x i32> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
 ; FIXED-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[TMP7]], i32 8
-; FIXED-NEXT:    store <8 x i32> [[TMP5]], ptr [[TMP7]], align 4
-; FIXED-NEXT:    store <8 x i32> [[TMP6]], ptr [[TMP9]], align 4
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; FIXED-NEXT:    store <8 x i32> [[TMP6]], ptr [[TMP7]], align 4
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; FIXED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -2270,40 +2255,38 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ;
 ; SCALABLE-LABEL: @combine_load_factor2_i32(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
-; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP6]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP7]], align 4
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; SCALABLE-NEXT:    [[TMP12:%.*]] = icmp ult <vscale x 4 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP13:%.*]] = shl i64 [[INDEX]], 1
+; SCALABLE-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[TMP13]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> [[TMP12]], <vscale x 4 x i1> [[TMP12]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 8 x i32> @llvm.masked.load.nxv8i32.p0(ptr [[TMP15]], i32 4, <vscale x 8 x i1> [[INTERLEAVED_MASK]], <vscale x 8 x i32> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_MASKED_VEC]])
 ; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
 ; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[TMP9]]
 ; SCALABLE-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[Q:%.*]], i64 [[INDEX]]
-; SCALABLE-NEXT:    store <vscale x 4 x i32> [[TMP10]], ptr [[TMP11]], align 4
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; SCALABLE-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP10]], ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; SCALABLE-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; SCALABLE-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -2315,7 +2298,7 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    store i32 [[RES]], ptr [[DST]], align 4
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -2347,40 +2330,38 @@ exit:
 define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; CHECK-LABEL: @combine_load_factor2_i64(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP7]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP13:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP13]]
+; CHECK-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> [[TMP12]], <vscale x 2 x i1> [[TMP12]])
+; CHECK-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr [[TMP15]], i32 8, <vscale x 4 x i1> [[INTERLEAVED_MASK]], <vscale x 4 x i64> poison)
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_MASKED_VEC]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[TMP8]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP10]], ptr [[TMP11]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP10]], ptr align 8 [[TMP11]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -2392,7 +2373,7 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; CHECK-NEXT:    store i64 [[RES]], ptr [[DST]], align 8
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -2403,33 +2384,23 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; FIXED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; FIXED:       vector.body:
 ; FIXED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; FIXED-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 4
-; FIXED-NEXT:    [[TMP1:%.*]] = shl i64 [[INDEX]], 1
-; FIXED-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP0]], 1
-; FIXED-NEXT:    [[TMP3:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]]
-; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP2]]
-; FIXED-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP3]], align 8
-; FIXED-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; FIXED-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; FIXED-NEXT:    [[TMP2:%.*]] = shl i64 [[INDEX]], 1
+; FIXED-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP2]]
 ; FIXED-NEXT:    [[WIDE_VEC2:%.*]] = load <8 x i64>, ptr [[TMP4]], align 8
 ; FIXED-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <8 x i64> [[WIDE_VEC2]], <8 x i64> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; FIXED-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <8 x i64> [[WIDE_VEC2]], <8 x i64> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; FIXED-NEXT:    [[TMP5:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]]
 ; FIXED-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[STRIDED_VEC3]], [[STRIDED_VEC4]]
 ; FIXED-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[INDEX]]
-; FIXED-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[TMP7]], i32 4
-; FIXED-NEXT:    store <4 x i64> [[TMP5]], ptr [[TMP7]], align 8
-; FIXED-NEXT:    store <4 x i64> [[TMP6]], ptr [[TMP9]], align 8
-; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FIXED-NEXT:    store <4 x i64> [[TMP6]], ptr [[TMP7]], align 8
+; FIXED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; FIXED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; FIXED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; FIXED:       middle.block:
 ; FIXED-NEXT:    br label [[EXIT:%.*]]
 ; FIXED:       scalar.ph:
-; FIXED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; FIXED-NEXT:    br label [[LOOP:%.*]]
 ; FIXED:       loop:
-; FIXED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; FIXED-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; FIXED-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; FIXED-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; FIXED-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -2447,40 +2418,38 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ;
 ; SCALABLE-LABEL: @combine_load_factor2_i64(
 ; SCALABLE-NEXT:  entry:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; SCALABLE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = shl i64 [[INDEX]], 1
-; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]]
-; SCALABLE-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 4 x i64>, ptr [[TMP7]], align 8
-; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_VEC]])
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP7]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
+; SCALABLE-NEXT:    [[TMP12:%.*]] = icmp ult <vscale x 2 x i32> [[TMP6]], [[BROADCAST_SPLAT]]
+; SCALABLE-NEXT:    [[TMP13:%.*]] = shl i64 [[INDEX]], 1
+; SCALABLE-NEXT:    [[TMP15:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP13]]
+; SCALABLE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> [[TMP12]], <vscale x 2 x i1> [[TMP12]])
+; SCALABLE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 4 x i64> @llvm.masked.load.nxv4i64.p0(ptr [[TMP15]], i32 8, <vscale x 4 x i1> [[INTERLEAVED_MASK]], <vscale x 4 x i64> poison)
+; SCALABLE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave2.nxv4i64(<vscale x 4 x i64> [[WIDE_MASKED_VEC]])
 ; SCALABLE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 0
 ; SCALABLE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[STRIDED_VEC]], 1
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = add <vscale x 2 x i64> [[TMP8]], [[TMP9]]
 ; SCALABLE-NEXT:    [[TMP11:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[INDEX]]
-; SCALABLE-NEXT:    store <vscale x 2 x i64> [[TMP10]], ptr [[TMP11]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; SCALABLE-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP10]], ptr align 8 [[TMP11]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; SCALABLE-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; SCALABLE-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SCALABLE-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; SCALABLE:       middle.block:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; SCALABLE-NEXT:    br label [[EXIT:%.*]]
 ; SCALABLE:       scalar.ph:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; SCALABLE-NEXT:    br label [[LOOP:%.*]]
 ; SCALABLE:       loop:
-; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; SCALABLE-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; SCALABLE-NEXT:    [[OFFSET0:%.*]] = shl i64 [[I]], 1
 ; SCALABLE-NEXT:    [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]]
 ; SCALABLE-NEXT:    [[X0:%.*]] = load i64, ptr [[Q0]], align 8
@@ -2492,7 +2461,7 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-NEXT:    store i64 [[RES]], ptr [[DST]], align 8
 ; SCALABLE-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; SCALABLE-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP24:![0-9]+]]
 ; SCALABLE:       exit:
 ; SCALABLE-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
index d6f16bfcba1af..ee91f75fc9706 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-cost.ll
@@ -7,22 +7,31 @@
 ; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+optimized-nf6-segment-load-store -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=OPT-NF6
 ; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+optimized-nf7-segment-load-store -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=OPT-NF7
 ; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+optimized-nf8-segment-load-store -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=OPT-NF8
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,-optimized-nf2-segment-load-store -scalable-vectorization=off -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FIXED-NO-OPT
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -scalable-vectorization=off -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FIXED-OPT-NF2
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+optimized-nf3-segment-load-store -scalable-vectorization=off -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FIXED-OPT-NF3
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+optimized-nf4-segment-load-store -scalable-vectorization=off -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FIXED-OPT-NF4
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+optimized-nf5-segment-load-store -scalable-vectorization=off -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FIXED-OPT-NF5
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+optimized-nf6-segment-load-store -scalable-vectorization=off -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FIXED-OPT-NF6
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+optimized-nf7-segment-load-store -scalable-vectorization=off -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FIXED-OPT-NF7
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v,+optimized-nf8-segment-load-store -scalable-vectorization=off -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s --check-prefix=FIXED-OPT-NF8
 
 %i8.2 = type {i8, i8}
 define void @i8_factor_2(ptr %data, i64 %n) {
 entry:
   br label %for.body
+; FIXED-OPT-NF2-LABEL: Checking a loop in 'i8_factor_2'
+; FIXED-OPT-NF2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-OPT-NF2: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; FIXED-OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; FIXED-OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
 ; OPT-NF2-LABEL: Checking a loop in 'i8_factor_2'
-; OPT-NF2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; OPT-NF2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; OPT-NF2: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; OPT-NF2: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; OPT-NF2: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; OPT-NF2: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; OPT-NF2: Cost of 4 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; OPT-NF2: Cost of 8 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
 ; OPT-NF2: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
 ; OPT-NF2: Cost of 3 for VF vscale x 1: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
 ; OPT-NF2: Cost of 3 for VF vscale x 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
@@ -33,17 +42,18 @@ entry:
 ; OPT-NF2: Cost of 4 for VF vscale x 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
 ; OPT-NF2: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
 ; OPT-NF2: Cost of 8 for VF vscale x 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_2'
+; FIXED-NO-OPT: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
 ; NO-OPT-LABEL: Checking a loop in 'i8_factor_2'
-; NO-OPT: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; NO-OPT: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; NO-OPT: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; NO-OPT: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; NO-OPT: Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; NO-OPT: Cost of 16 for VF 8: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; NO-OPT: Cost of 32 for VF 16: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
-; NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
-; NO-OPT: Cost of 64 for VF 32: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
 ; NO-OPT: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
 ; NO-OPT: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 2 at <badref>, ir<%p0>
 ; NO-OPT: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 2 at %l0, ir<%p0>
@@ -76,28 +86,50 @@ for.end:
 define void @i8_factor_3(ptr %data, i64 %n) {
 entry:
   br label %for.body
+; FIXED-OPT-NF3-LABEL: Checking a loop in 'i8_factor_3'
+; FIXED-OPT-NF3: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-OPT-NF3: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; FIXED-OPT-NF3: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-OPT-NF3: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; FIXED-OPT-NF3: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-OPT-NF3: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; FIXED-OPT-NF3: Cost of 7 for VF 16: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-OPT-NF3: Cost of 7 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; FIXED-OPT-NF3: Cost of 14 for VF 32: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-OPT-NF3: Cost of 14 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
 ; OPT-NF3-LABEL: Checking a loop in 'i8_factor_3'
-; OPT-NF3: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; OPT-NF3: Cost of 4 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; OPT-NF3: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; OPT-NF3: Cost of 4 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; OPT-NF3: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; OPT-NF3: Cost of 5 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; OPT-NF3: Cost of 7 for VF 16: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; OPT-NF3: Cost of 7 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; OPT-NF3: Cost of 14 for VF 32: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; OPT-NF3: Cost of 14 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; OPT-NF3: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; OPT-NF3: Cost of 4 for VF vscale x 1: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; OPT-NF3: Cost of 4 for VF vscale x 2: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; OPT-NF3: Cost of 4 for VF vscale x 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; OPT-NF3: Cost of 5 for VF vscale x 4: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; OPT-NF3: Cost of 5 for VF vscale x 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; OPT-NF3: Cost of 7 for VF vscale x 8: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; OPT-NF3: Cost of 7 for VF vscale x 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; OPT-NF3: Cost of 14 for VF vscale x 16: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; OPT-NF3: Cost of 14 for VF vscale x 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_3'
+; FIXED-NO-OPT: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
 ; NO-OPT-LABEL: Checking a loop in 'i8_factor_3'
-; NO-OPT: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; NO-OPT: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; NO-OPT: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; NO-OPT: Cost of 12 for VF 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; NO-OPT: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; NO-OPT: Cost of 24 for VF 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; NO-OPT: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; NO-OPT: Cost of 48 for VF 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
-; NO-OPT: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
-; NO-OPT: Cost of 96 for VF 32: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; NO-OPT: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; NO-OPT: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; NO-OPT: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; NO-OPT: Cost of 12 for VF vscale x 2: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; NO-OPT: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; NO-OPT: Cost of 24 for VF vscale x 4: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; NO-OPT: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; NO-OPT: Cost of 48 for VF vscale x 8: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
+; NO-OPT: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3 at %l0, ir<%p0>
+; NO-OPT: Cost of 96 for VF vscale x 16: INTERLEAVE-GROUP with factor 3 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 0
@@ -124,28 +156,50 @@ for.end:
 define void @i8_factor_4(ptr %data, i64 %n) {
 entry:
   br label %for.body
+; FIXED-OPT-NF4-LABEL: Checking a loop in 'i8_factor_4'
+; FIXED-OPT-NF4: Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-OPT-NF4: Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; FIXED-OPT-NF4: Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-OPT-NF4: Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; FIXED-OPT-NF4: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-OPT-NF4: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; FIXED-OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; FIXED-OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
 ; OPT-NF4-LABEL: Checking a loop in 'i8_factor_4'
-; OPT-NF4: Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; OPT-NF4: Cost of 5 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; OPT-NF4: Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; OPT-NF4: Cost of 5 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; OPT-NF4: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; OPT-NF4: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; OPT-NF4: Cost of 8 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; OPT-NF4: Cost of 16 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; OPT-NF4: Cost of 5 for VF vscale x 1: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; OPT-NF4: Cost of 5 for VF vscale x 1: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; OPT-NF4: Cost of 5 for VF vscale x 2: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; OPT-NF4: Cost of 5 for VF vscale x 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; OPT-NF4: Cost of 6 for VF vscale x 4: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; OPT-NF4: Cost of 6 for VF vscale x 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; OPT-NF4: Cost of 8 for VF vscale x 8: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; OPT-NF4: Cost of 8 for VF vscale x 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; OPT-NF4: Cost of 16 for VF vscale x 16: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; OPT-NF4: Cost of 16 for VF vscale x 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_4'
+; FIXED-NO-OPT: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
 ; NO-OPT-LABEL: Checking a loop in 'i8_factor_4'
-; NO-OPT: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; NO-OPT: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; NO-OPT: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; NO-OPT: Cost of 16 for VF 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; NO-OPT: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; NO-OPT: Cost of 32 for VF 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; NO-OPT: Cost of 64 for VF 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
-; NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
-; NO-OPT: Cost of 128 for VF 32: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; NO-OPT: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; NO-OPT: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; NO-OPT: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; NO-OPT: Cost of 16 for VF vscale x 2: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; NO-OPT: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; NO-OPT: Cost of 32 for VF vscale x 4: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; NO-OPT: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; NO-OPT: Cost of 64 for VF vscale x 8: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
+; NO-OPT: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4 at %l0, ir<%p0>
+; NO-OPT: Cost of 128 for VF vscale x 16: INTERLEAVE-GROUP with factor 4 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.4, ptr %data, i64 %i, i32 0
@@ -176,24 +230,42 @@ for.end:
 define void @i8_factor_5(ptr %data, i64 %n) {
 entry:
   br label %for.body
+; FIXED-OPT-NF5-LABEL: Checking a loop in 'i8_factor_5'
+; FIXED-OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; FIXED-OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; FIXED-OPT-NF5: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; FIXED-OPT-NF5: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; FIXED-OPT-NF5: Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; FIXED-OPT-NF5: Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; FIXED-OPT-NF5: Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; FIXED-OPT-NF5: Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
 ; OPT-NF5-LABEL: Checking a loop in 'i8_factor_5'
-; OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; OPT-NF5: Cost of 6 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
-; OPT-NF5: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; OPT-NF5: Cost of 7 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
-; OPT-NF5: Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; OPT-NF5: Cost of 9 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
-; OPT-NF5: Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; OPT-NF5: Cost of 13 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; OPT-NF5: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; OPT-NF5: Cost of 6 for VF vscale x 1: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; OPT-NF5: Cost of 7 for VF vscale x 2: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; OPT-NF5: Cost of 7 for VF vscale x 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; OPT-NF5: Cost of 9 for VF vscale x 4: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; OPT-NF5: Cost of 9 for VF vscale x 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; OPT-NF5: Cost of 13 for VF vscale x 8: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; OPT-NF5: Cost of 13 for VF vscale x 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_5'
+; FIXED-NO-OPT: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
 ; NO-OPT-LABEL: Checking a loop in 'i8_factor_5'
-; NO-OPT: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; NO-OPT: Cost of 10 for VF 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
-; NO-OPT: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; NO-OPT: Cost of 20 for VF 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
-; NO-OPT: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; NO-OPT: Cost of 40 for VF 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
-; NO-OPT: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
-; NO-OPT: Cost of 80 for VF 16: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; NO-OPT: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; NO-OPT: Cost of 10 for VF vscale x 1: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; NO-OPT: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; NO-OPT: Cost of 20 for VF vscale x 2: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; NO-OPT: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; NO-OPT: Cost of 40 for VF vscale x 4: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
+; NO-OPT: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5 at %l0, ir<%p0>
+; NO-OPT: Cost of 80 for VF vscale x 8: INTERLEAVE-GROUP with factor 5 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.5, ptr %data, i64 %i, i32 0
@@ -228,24 +300,42 @@ for.end:
 define void @i8_factor_6(ptr %data, i64 %n) {
 entry:
   br label %for.body
+; FIXED-OPT-NF6-LABEL: Checking a loop in 'i8_factor_6'
+; FIXED-OPT-NF6: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; FIXED-OPT-NF6: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; FIXED-OPT-NF6: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; FIXED-OPT-NF6: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; FIXED-OPT-NF6: Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; FIXED-OPT-NF6: Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; FIXED-OPT-NF6: Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; FIXED-OPT-NF6: Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
 ; OPT-NF6-LABEL: Checking a loop in 'i8_factor_6'
-; OPT-NF6: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; OPT-NF6: Cost of 7 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
-; OPT-NF6: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; OPT-NF6: Cost of 8 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
-; OPT-NF6: Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; OPT-NF6: Cost of 10 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
-; OPT-NF6: Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; OPT-NF6: Cost of 14 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; OPT-NF6: Cost of 7 for VF vscale x 1: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; OPT-NF6: Cost of 7 for VF vscale x 1: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; OPT-NF6: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; OPT-NF6: Cost of 8 for VF vscale x 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; OPT-NF6: Cost of 10 for VF vscale x 4: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; OPT-NF6: Cost of 10 for VF vscale x 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; OPT-NF6: Cost of 14 for VF vscale x 8: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; OPT-NF6: Cost of 14 for VF vscale x 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_6'
+; FIXED-NO-OPT: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
 ; NO-OPT-LABEL: Checking a loop in 'i8_factor_6'
-; NO-OPT: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; NO-OPT: Cost of 12 for VF 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
-; NO-OPT: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; NO-OPT: Cost of 24 for VF 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
-; NO-OPT: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; NO-OPT: Cost of 48 for VF 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
-; NO-OPT: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
-; NO-OPT: Cost of 96 for VF 16: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; NO-OPT: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; NO-OPT: Cost of 12 for VF vscale x 1: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; NO-OPT: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; NO-OPT: Cost of 24 for VF vscale x 2: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; NO-OPT: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; NO-OPT: Cost of 48 for VF vscale x 4: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
+; NO-OPT: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6 at %l0, ir<%p0>
+; NO-OPT: Cost of 96 for VF vscale x 8: INTERLEAVE-GROUP with factor 6 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.6, ptr %data, i64 %i, i32 0
@@ -284,24 +374,42 @@ for.end:
 define void @i8_factor_7(ptr %data, i64 %n) {
 entry:
   br label %for.body
+; FIXED-OPT-NF7-LABEL: Checking a loop in 'i8_factor_7'
+; FIXED-OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; FIXED-OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; FIXED-OPT-NF7: Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; FIXED-OPT-NF7: Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; FIXED-OPT-NF7: Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; FIXED-OPT-NF7: Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; FIXED-OPT-NF7: Cost of 15 for VF 16: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; FIXED-OPT-NF7: Cost of 15 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
 ; OPT-NF7-LABEL: Checking a loop in 'i8_factor_7'
-; OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; OPT-NF7: Cost of 8 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
-; OPT-NF7: Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; OPT-NF7: Cost of 9 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
-; OPT-NF7: Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; OPT-NF7: Cost of 11 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
-; OPT-NF7: Cost of 15 for VF 16: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; OPT-NF7: Cost of 15 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; OPT-NF7: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; OPT-NF7: Cost of 8 for VF vscale x 1: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; OPT-NF7: Cost of 9 for VF vscale x 2: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; OPT-NF7: Cost of 9 for VF vscale x 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; OPT-NF7: Cost of 11 for VF vscale x 4: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; OPT-NF7: Cost of 11 for VF vscale x 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; OPT-NF7: Cost of 15 for VF vscale x 8: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; OPT-NF7: Cost of 15 for VF vscale x 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_7'
+; FIXED-NO-OPT: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
 ; NO-OPT-LABEL: Checking a loop in 'i8_factor_7'
-; NO-OPT: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; NO-OPT: Cost of 14 for VF 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
-; NO-OPT: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; NO-OPT: Cost of 28 for VF 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
-; NO-OPT: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; NO-OPT: Cost of 56 for VF 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
-; NO-OPT: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
-; NO-OPT: Cost of 112 for VF 16: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; NO-OPT: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; NO-OPT: Cost of 14 for VF vscale x 1: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; NO-OPT: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; NO-OPT: Cost of 28 for VF vscale x 2: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; NO-OPT: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; NO-OPT: Cost of 56 for VF vscale x 4: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
+; NO-OPT: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7 at %l0, ir<%p0>
+; NO-OPT: Cost of 112 for VF vscale x 8: INTERLEAVE-GROUP with factor 7 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.7, ptr %data, i64 %i, i32 0
@@ -344,24 +452,42 @@ for.end:
 define void @i8_factor_8(ptr %data, i64 %n) {
 entry:
   br label %for.body
+; FIXED-OPT-NF8-LABEL: Checking a loop in 'i8_factor_8'
+; FIXED-OPT-NF8: Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; FIXED-OPT-NF8: Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; FIXED-OPT-NF8: Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; FIXED-OPT-NF8: Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; FIXED-OPT-NF8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; FIXED-OPT-NF8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; FIXED-OPT-NF8: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; FIXED-OPT-NF8: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
 ; OPT-NF8-LABEL: Checking a loop in 'i8_factor_8'
-; OPT-NF8: Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; OPT-NF8: Cost of 9 for VF 2: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
-; OPT-NF8: Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; OPT-NF8: Cost of 10 for VF 4: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
-; OPT-NF8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; OPT-NF8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
-; OPT-NF8: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; OPT-NF8: Cost of 16 for VF 16: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; OPT-NF8: Cost of 9 for VF vscale x 1: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; OPT-NF8: Cost of 9 for VF vscale x 1: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; OPT-NF8: Cost of 10 for VF vscale x 2: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; OPT-NF8: Cost of 10 for VF vscale x 2: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; OPT-NF8: Cost of 12 for VF vscale x 4: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; OPT-NF8: Cost of 12 for VF vscale x 4: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; OPT-NF8: Cost of 16 for VF vscale x 8: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; OPT-NF8: Cost of 16 for VF vscale x 8: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; FIXED-NO-OPT-LABEL: Checking a loop in 'i8_factor_8'
+; FIXED-NO-OPT: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; FIXED-NO-OPT: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; FIXED-NO-OPT: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
 ; NO-OPT-LABEL: Checking a loop in 'i8_factor_8'
-; NO-OPT: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; NO-OPT: Cost of 16 for VF 2: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
-; NO-OPT: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; NO-OPT: Cost of 32 for VF 4: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
-; NO-OPT: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; NO-OPT: Cost of 64 for VF 8: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
-; NO-OPT: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
-; NO-OPT: Cost of 128 for VF 16: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; NO-OPT: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; NO-OPT: Cost of 16 for VF vscale x 1: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; NO-OPT: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; NO-OPT: Cost of 32 for VF vscale x 2: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; NO-OPT: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; NO-OPT: Cost of 64 for VF vscale x 4: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
+; NO-OPT: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8 at %l0, ir<%p0>
+; NO-OPT: Cost of 128 for VF vscale x 8: INTERLEAVE-GROUP with factor 8 at <badref>, ir<%p0>
 for.body:
   %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
   %p0 = getelementptr inbounds %i8.8, ptr %data, i64 %i, i32 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
index 976ce77d2ba29..48e8a1dac348b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
@@ -18,37 +18,35 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = shl nuw i32 [[TMP3]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP4]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP2]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_EPILOGUE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_EPILOGUE:       vector.body:
 ; SCALAR_EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = shl i32 [[INDEX]], 1
-; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
-; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP8]]
-; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]])
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr [[TMP9]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl i32 [[INDEX]], 1
+; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
+; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]]
+; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr [[TMP7]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i8> poison)
 ; SCALAR_EPILOGUE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[WIDE_MASKED_VEC]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
-; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
-; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP7]] to i64
-; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP13]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP12]]
-; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP15]])
-; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]])
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP14]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
-; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP9]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP5]] to i64
+; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP11]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP10]]
+; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP13]])
+; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP12]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
+; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]]
 ; SCALAR_EPILOGUE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SCALAR_EPILOGUE:       middle.block:
 ; SCALAR_EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -65,39 +63,37 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_DATA-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
 ; PREDICATED_DATA-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
 ; PREDICATED_DATA-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; PREDICATED_DATA-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_DATA-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; PREDICATED_DATA-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP3]], i64 0
+; PREDICATED_DATA-NEXT:    [[TMP2:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
 ; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_DATA-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PREDICATED_DATA:       vector.body:
 ; PREDICATED_DATA-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_DATA-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_DATA-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; PREDICATED_DATA-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 1024)
-; PREDICATED_DATA-NEXT:    [[TMP5:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; PREDICATED_DATA-NEXT:    [[TMP6:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i1> zeroinitializer
-; PREDICATED_DATA-NEXT:    [[TMP7:%.*]] = shl i32 [[INDEX]], 1
-; PREDICATED_DATA-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
-; PREDICATED_DATA-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP8]]
-; PREDICATED_DATA-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]])
-; PREDICATED_DATA-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr [[TMP9]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i8> poison)
+; PREDICATED_DATA-NEXT:    [[TMP3:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_DATA-NEXT:    [[TMP4:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP3]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_DATA-NEXT:    [[TMP5:%.*]] = shl i32 [[INDEX]], 1
+; PREDICATED_DATA-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
+; PREDICATED_DATA-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]]
+; PREDICATED_DATA-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; PREDICATED_DATA-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.masked.load.nxv32i8.p0(ptr [[TMP7]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK]], <vscale x 32 x i8> poison)
 ; PREDICATED_DATA-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave2.nxv32i8(<vscale x 32 x i8> [[WIDE_MASKED_VEC]])
-; PREDICATED_DATA-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
-; PREDICATED_DATA-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
-; PREDICATED_DATA-NEXT:    [[TMP12:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]])
-; PREDICATED_DATA-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP7]] to i64
-; PREDICATED_DATA-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP13]]
-; PREDICATED_DATA-NEXT:    [[TMP15:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP12]]
-; PREDICATED_DATA-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP15]])
-; PREDICATED_DATA-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]])
-; PREDICATED_DATA-NEXT:    call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP14]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
-; PREDICATED_DATA-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; PREDICATED_DATA-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; PREDICATED_DATA-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; PREDICATED_DATA-NEXT:    [[TMP10:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP9]])
+; PREDICATED_DATA-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP5]] to i64
+; PREDICATED_DATA-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP11]]
+; PREDICATED_DATA-NEXT:    [[TMP13:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP10]]
+; PREDICATED_DATA-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 32 x i8> @llvm.vector.interleave2.nxv32i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP13]])
+; PREDICATED_DATA-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; PREDICATED_DATA-NEXT:    call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP12]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK3]])
+; PREDICATED_DATA-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
 ; PREDICATED_DATA-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; PREDICATED_DATA-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; PREDICATED_DATA-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; PREDICATED_DATA-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREDICATED_DATA-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; PREDICATED_DATA:       middle.block:
 ; PREDICATED_DATA-NEXT:    br label [[FOR_END:%.*]]
 ; PREDICATED_DATA:       scalar.ph:
@@ -119,8 +115,6 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; PREDICATED_DATA-WITH-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
-; PREDICATED_DATA-WITH-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP2:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP3:%.*]] = icmp ult <vscale x 16 x i32> [[TMP2]], [[BROADCAST_SPLAT4]]
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP4:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
@@ -142,7 +136,7 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_DATA-WITH-EVL-NEXT:    call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> [[INTERLEAVED_VEC]], ptr [[TMP13]], i32 1, <vscale x 32 x i1> [[INTERLEAVED_MASK5]])
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[EVL_BASED_IV]]
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
-; PREDICATED_DATA-WITH-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_DATA-WITH-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[INDEX_EVL_NEXT]], 1024
 ; PREDICATED_DATA-WITH-EVL-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; PREDICATED_DATA-WITH-EVL:       middle.block:
@@ -197,41 +191,39 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = shl nuw i32 [[TMP1]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP2]]
 ; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = shl nuw i32 [[TMP3]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP4]], i64 0
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP2]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_EPILOGUE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALAR_EPILOGUE:       vector.body:
 ; SCALAR_EPILOGUE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = shl i32 [[INDEX]], 2
-; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
-; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP8]]
-; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]])
-; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP9]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
+; SCALAR_EPILOGUE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl i32 [[INDEX]], 2
+; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
+; SCALAR_EPILOGUE-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]]
+; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; SCALAR_EPILOGUE-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP7]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
 ; SCALAR_EPILOGUE-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave4.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
-; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
-; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
-; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
+; SCALAR_EPILOGUE-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; SCALAR_EPILOGUE-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; SCALAR_EPILOGUE-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
+; SCALAR_EPILOGUE-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
+; SCALAR_EPILOGUE-NEXT:    [[TMP12:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP9]])
+; SCALAR_EPILOGUE-NEXT:    [[TMP13:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP12]]
 ; SCALAR_EPILOGUE-NEXT:    [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]])
 ; SCALAR_EPILOGUE-NEXT:    [[TMP15:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP14]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP13]])
-; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP16]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP7]] to i64
-; SCALAR_EPILOGUE-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP18]]
-; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP14]], <vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]], <vscale x 16 x i8> [[TMP17]])
-; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]])
-; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP19]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
-; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP4]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP16:%.*]] = sext i32 [[TMP5]] to i64
+; SCALAR_EPILOGUE-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP16]]
+; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]], <vscale x 16 x i8> [[TMP15]])
+; SCALAR_EPILOGUE-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; SCALAR_EPILOGUE-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP17]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
+; SCALAR_EPILOGUE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP2]]
 ; SCALAR_EPILOGUE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; SCALAR_EPILOGUE-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALAR_EPILOGUE-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SCALAR_EPILOGUE-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; SCALAR_EPILOGUE:       middle.block:
 ; SCALAR_EPILOGUE-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -248,43 +240,41 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_DATA-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
 ; PREDICATED_DATA-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
 ; PREDICATED_DATA-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; PREDICATED_DATA-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_DATA-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; PREDICATED_DATA-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
-; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP3]], i64 0
+; PREDICATED_DATA-NEXT:    [[TMP2:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
+; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
 ; PREDICATED_DATA-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_DATA-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; PREDICATED_DATA:       vector.body:
 ; PREDICATED_DATA-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; PREDICATED_DATA-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; PREDICATED_DATA-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 16 x i32> [ [[TMP2]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; PREDICATED_DATA-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 1024)
-; PREDICATED_DATA-NEXT:    [[TMP5:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; PREDICATED_DATA-NEXT:    [[TMP6:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i1> zeroinitializer
-; PREDICATED_DATA-NEXT:    [[TMP7:%.*]] = shl i32 [[INDEX]], 2
-; PREDICATED_DATA-NEXT:    [[TMP8:%.*]] = sext i32 [[TMP7]] to i64
-; PREDICATED_DATA-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP8]]
-; PREDICATED_DATA-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]])
-; PREDICATED_DATA-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP9]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
+; PREDICATED_DATA-NEXT:    [[TMP3:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; PREDICATED_DATA-NEXT:    [[TMP4:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP3]], <vscale x 16 x i1> zeroinitializer
+; PREDICATED_DATA-NEXT:    [[TMP5:%.*]] = shl i32 [[INDEX]], 2
+; PREDICATED_DATA-NEXT:    [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
+; PREDICATED_DATA-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP6]]
+; PREDICATED_DATA-NEXT:    [[INTERLEAVED_MASK:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; PREDICATED_DATA-NEXT:    [[WIDE_MASKED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.masked.load.nxv64i8.p0(ptr [[TMP7]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK]], <vscale x 64 x i8> poison)
 ; PREDICATED_DATA-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave4.nxv64i8(<vscale x 64 x i8> [[WIDE_MASKED_VEC]])
-; PREDICATED_DATA-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
-; PREDICATED_DATA-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
-; PREDICATED_DATA-NEXT:    [[TMP12:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
-; PREDICATED_DATA-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
+; PREDICATED_DATA-NEXT:    [[TMP8:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 0
+; PREDICATED_DATA-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 1
+; PREDICATED_DATA-NEXT:    [[TMP10:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 2
+; PREDICATED_DATA-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[STRIDED_VEC]], 3
+; PREDICATED_DATA-NEXT:    [[TMP12:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP9]])
+; PREDICATED_DATA-NEXT:    [[TMP13:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP12]]
 ; PREDICATED_DATA-NEXT:    [[TMP14:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]])
 ; PREDICATED_DATA-NEXT:    [[TMP15:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP14]]
-; PREDICATED_DATA-NEXT:    [[TMP16:%.*]] = call <vscale x 16 x i8> @llvm.smax.nxv16i8(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP13]])
-; PREDICATED_DATA-NEXT:    [[TMP17:%.*]] = sub <vscale x 16 x i8> zeroinitializer, [[TMP16]]
-; PREDICATED_DATA-NEXT:    [[TMP18:%.*]] = sext i32 [[TMP7]] to i64
-; PREDICATED_DATA-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP18]]
-; PREDICATED_DATA-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP14]], <vscale x 16 x i8> [[TMP15]], <vscale x 16 x i8> [[TMP16]], <vscale x 16 x i8> [[TMP17]])
-; PREDICATED_DATA-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i1> [[TMP6]])
-; PREDICATED_DATA-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP19]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
-; PREDICATED_DATA-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
+; PREDICATED_DATA-NEXT:    [[TMP16:%.*]] = sext i32 [[TMP5]] to i64
+; PREDICATED_DATA-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[Q]], i64 [[TMP16]]
+; PREDICATED_DATA-NEXT:    [[INTERLEAVED_VEC:%.*]] = call <vscale x 64 x i8> @llvm.vector.interleave4.nxv64i8(<vscale x 16 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP13]], <vscale x 16 x i8> [[TMP14]], <vscale x 16 x i8> [[TMP15]])
+; PREDICATED_DATA-NEXT:    [[INTERLEAVED_MASK3:%.*]] = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i1> [[TMP4]])
+; PREDICATED_DATA-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP17]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK3]])
+; PREDICATED_DATA-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP1]]
 ; PREDICATED_DATA-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; PREDICATED_DATA-NEXT:    [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
-; PREDICATED_DATA-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; PREDICATED_DATA-NEXT:    [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; PREDICATED_DATA-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; PREDICATED_DATA:       middle.block:
 ; PREDICATED_DATA-NEXT:    br label [[FOR_END:%.*]]
 ; PREDICATED_DATA:       scalar.ph:
@@ -306,8 +296,6 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 16, i1 true)
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
-; PREDICATED_DATA-WITH-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[TMP1]], i64 0
-; PREDICATED_DATA-WITH-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP2:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP3:%.*]] = icmp ult <vscale x 16 x i32> [[TMP2]], [[BROADCAST_SPLAT4]]
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP4:%.*]] = icmp ugt <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
@@ -333,7 +321,7 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_DATA-WITH-EVL-NEXT:    call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> [[INTERLEAVED_VEC]], ptr [[TMP18]], i32 1, <vscale x 64 x i1> [[INTERLEAVED_MASK5]])
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP1]], [[EVL_BASED_IV]]
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP1]]
-; PREDICATED_DATA-WITH-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; PREDICATED_DATA-WITH-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 16 x i32> [[VEC_IND]], [[BROADCAST_SPLAT4]]
 ; PREDICATED_DATA-WITH-EVL-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_EVL_NEXT]], 1024
 ; PREDICATED_DATA-WITH-EVL-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; PREDICATED_DATA-WITH-EVL:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll
index 93e0f9038361d..3ea37fcc7b654 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll
@@ -15,7 +15,6 @@ define void @load_store(ptr %p) {
 ; LMUL1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; LMUL1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
 ; LMUL1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; LMUL1-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; LMUL1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; LMUL1:       vector.body:
 ; LMUL1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -23,9 +22,9 @@ define void @load_store(ptr %p) {
 ; LMUL1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 1 x i64>, ptr [[TMP3]], align 8
 ; LMUL1-NEXT:    [[TMP5:%.*]] = add <vscale x 1 x i64> [[WIDE_LOAD]], splat (i64 1)
 ; LMUL1-NEXT:    store <vscale x 1 x i64> [[TMP5]], ptr [[TMP3]], align 8
-; LMUL1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; LMUL1-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; LMUL1-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; LMUL1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; LMUL1-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; LMUL1-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; LMUL1:       middle.block:
 ; LMUL1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
 ; LMUL1-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -46,124 +45,103 @@ define void @load_store(ptr %p) {
 ;
 ; LMUL2-LABEL: @load_store(
 ; LMUL2-NEXT:  entry:
-; LMUL2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; LMUL2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; LMUL2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; LMUL2-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; LMUL2:       vector.ph:
-; LMUL2-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL2-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; LMUL2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; LMUL2-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; LMUL2-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL2-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; LMUL2-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; LMUL2:       vector.body:
 ; LMUL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; LMUL2-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; LMUL2-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; LMUL2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[INDEX]]
-; LMUL2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP5]], align 8
+; LMUL2-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP6]])
 ; LMUL2-NEXT:    [[TMP7:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD]], splat (i64 1)
-; LMUL2-NEXT:    store <vscale x 2 x i64> [[TMP7]], ptr [[TMP5]], align 8
-; LMUL2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
-; LMUL2-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; LMUL2-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; LMUL2-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP7]], ptr align 8 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP6]])
+; LMUL2-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP6]] to i64
+; LMUL2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; LMUL2-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; LMUL2-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; LMUL2-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; LMUL2:       middle.block:
-; LMUL2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; LMUL2-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; LMUL2-NEXT:    br label [[FOR_END:%.*]]
 ; LMUL2:       scalar.ph:
-; LMUL2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; LMUL2-NEXT:    br label [[FOR_BODY:%.*]]
 ; LMUL2:       for.body:
-; LMUL2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; LMUL2-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; LMUL2-NEXT:    [[Q:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[IV]]
 ; LMUL2-NEXT:    [[V:%.*]] = load i64, ptr [[Q]], align 8
 ; LMUL2-NEXT:    [[W:%.*]] = add i64 [[V]], 1
 ; LMUL2-NEXT:    store i64 [[W]], ptr [[Q]], align 8
 ; LMUL2-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; LMUL2-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; LMUL2-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; LMUL2-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; LMUL2:       for.end:
 ; LMUL2-NEXT:    ret void
 ;
 ; LMUL4-LABEL: @load_store(
 ; LMUL4-NEXT:  entry:
-; LMUL4-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL4-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; LMUL4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; LMUL4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; LMUL4-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; LMUL4:       vector.ph:
-; LMUL4-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL4-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; LMUL4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; LMUL4-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; LMUL4-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL4-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; LMUL4-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; LMUL4:       vector.body:
 ; LMUL4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; LMUL4-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; LMUL4-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; LMUL4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[INDEX]]
-; LMUL4-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i64>, ptr [[TMP5]], align 8
+; LMUL4-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i64> @llvm.vp.load.nxv4i64.p0(ptr align 8 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
 ; LMUL4-NEXT:    [[TMP7:%.*]] = add <vscale x 4 x i64> [[WIDE_LOAD]], splat (i64 1)
-; LMUL4-NEXT:    store <vscale x 4 x i64> [[TMP7]], ptr [[TMP5]], align 8
-; LMUL4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
-; LMUL4-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; LMUL4-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; LMUL4-NEXT:    call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[TMP7]], ptr align 8 [[TMP5]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; LMUL4-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP6]] to i64
+; LMUL4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; LMUL4-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; LMUL4-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; LMUL4-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; LMUL4:       middle.block:
-; LMUL4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; LMUL4-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; LMUL4-NEXT:    br label [[FOR_END:%.*]]
 ; LMUL4:       scalar.ph:
-; LMUL4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; LMUL4-NEXT:    br label [[FOR_BODY:%.*]]
 ; LMUL4:       for.body:
-; LMUL4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; LMUL4-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; LMUL4-NEXT:    [[Q:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[IV]]
 ; LMUL4-NEXT:    [[V:%.*]] = load i64, ptr [[Q]], align 8
 ; LMUL4-NEXT:    [[W:%.*]] = add i64 [[V]], 1
 ; LMUL4-NEXT:    store i64 [[W]], ptr [[Q]], align 8
 ; LMUL4-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; LMUL4-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; LMUL4-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; LMUL4-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; LMUL4:       for.end:
 ; LMUL4-NEXT:    ret void
 ;
 ; LMUL8-LABEL: @load_store(
 ; LMUL8-NEXT:  entry:
-; LMUL8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; LMUL8-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; LMUL8-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; LMUL8-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; LMUL8:       vector.ph:
-; LMUL8-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; LMUL8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; LMUL8-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; LMUL8-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL8-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; LMUL8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; LMUL8:       vector.body:
 ; LMUL8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; LMUL8-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; LMUL8-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; LMUL8-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i64 [[INDEX]]
-; LMUL8-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i64>, ptr [[TMP5]], align 8
+; LMUL8-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x i64> @llvm.vp.load.nxv8i64.p0(ptr align 8 [[TMP5]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
 ; LMUL8-NEXT:    [[TMP7:%.*]] = add <vscale x 8 x i64> [[WIDE_LOAD]], splat (i64 1)
-; LMUL8-NEXT:    store <vscale x 8 x i64> [[TMP7]], ptr [[TMP5]], align 8
-; LMUL8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
-; LMUL8-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; LMUL8-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; LMUL8-NEXT:    call void @llvm.vp.store.nxv8i64.p0(<vscale x 8 x i64> [[TMP7]], ptr align 8 [[TMP5]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP6]])
+; LMUL8-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP6]] to i64
+; LMUL8-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; LMUL8-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; LMUL8-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; LMUL8-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; LMUL8:       middle.block:
-; LMUL8-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; LMUL8-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; LMUL8-NEXT:    br label [[FOR_END:%.*]]
 ; LMUL8:       scalar.ph:
-; LMUL8-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; LMUL8-NEXT:    br label [[FOR_BODY:%.*]]
 ; LMUL8:       for.body:
-; LMUL8-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; LMUL8-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; LMUL8-NEXT:    [[Q:%.*]] = getelementptr inbounds i64, ptr [[P]], i64 [[IV]]
 ; LMUL8-NEXT:    [[V:%.*]] = load i64, ptr [[Q]], align 8
 ; LMUL8-NEXT:    [[W:%.*]] = add i64 [[V]], 1
 ; LMUL8-NEXT:    store i64 [[W]], ptr [[Q]], align 8
 ; LMUL8-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; LMUL8-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; LMUL8-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; LMUL8-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; LMUL8:       for.end:
 ; LMUL8-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
index 0a872578f70b5..d081487cadc5c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
@@ -123,14 +123,6 @@ define void @trip8_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 8, [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 8, i32 4, i1 true)
@@ -143,10 +135,9 @@ define void @trip8_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i64 [[I_08]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[MUL:%.*]] = shl i8 [[TMP15]], 1
@@ -187,19 +178,19 @@ define void @trip16_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP1:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = shl <16 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <16 x i8>, ptr [[TMP4:%.*]], align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = add <16 x i8> [[TMP2]], [[WIDE_LOAD1]]
-; CHECK-NEXT:    store <16 x i8> [[TMP5]], ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 16, i32 8, i1 true)
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 8 x i8> @llvm.vp.load.nxv8i8.p0(ptr align 1 [[TMP1:%.*]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP6:%.*]] = shl <vscale x 8 x i8> [[VP_OP_LOAD]], splat (i8 1)
+; CHECK-NEXT:    [[VP_OP_LOAD1:%.*]] = call <vscale x 8 x i8> @llvm.vp.load.nxv8i8.p0(ptr align 1 [[TMP4:%.*]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 8 x i8> [[TMP6]], [[VP_OP_LOAD1]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP11]], ptr align 1 [[TMP4]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP5]])
 ; CHECK-NEXT:    br label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[MUL:%.*]] = shl i8 [[TMP7]], 1
@@ -241,19 +232,19 @@ define void @trip32_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[TMP1:%.*]], align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = shl <32 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <32 x i8>, ptr [[TMP4:%.*]], align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = add <32 x i8> [[TMP2]], [[WIDE_LOAD1]]
-; CHECK-NEXT:    store <32 x i8> [[TMP5]], ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 32, i32 16, i1 true)
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP1:%.*]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP6:%.*]] = shl <vscale x 16 x i8> [[VP_OP_LOAD]], splat (i8 1)
+; CHECK-NEXT:    [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[TMP4:%.*]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP11:%.*]] = add <vscale x 16 x i8> [[TMP6]], [[VP_OP_LOAD1]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP11]], ptr align 1 [[TMP4]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP5]])
 ; CHECK-NEXT:    br label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[MUL:%.*]] = shl i8 [[TMP7]], 1
@@ -294,24 +285,19 @@ define void @trip24_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = shl <8 x i8> [[WIDE_LOAD]], splat (i8 1)
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP4]], align 1
-; CHECK-NEXT:    [[TMP6:%.*]] = add <8 x i8> [[TMP3]], [[WIDE_LOAD1]]
-; CHECK-NEXT:    store <8 x i8> [[TMP6]], ptr [[TMP4]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 24, i32 16, i1 true)
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[SRC:%.*]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP6:%.*]] = shl <vscale x 16 x i8> [[VP_OP_LOAD]], splat (i8 1)
+; CHECK-NEXT:    [[VP_OP_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.vp.load.nxv16i8.p0(ptr align 1 [[DST:%.*]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = add <vscale x 16 x i8> [[TMP6]], [[VP_OP_LOAD1]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv16i8.p0(<vscale x 16 x i8> [[TMP7]], ptr align 1 [[DST]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP5]])
+; CHECK-NEXT:    br label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[I_08]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[MUL:%.*]] = shl i8 [[TMP8]], 1
@@ -321,7 +307,7 @@ define void @trip24_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK-NEXT:    store i8 [[ADD]], ptr [[ARRAYIDX1]], align 1
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_08]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 24
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -360,23 +346,19 @@ define i8 @mul_non_pow_2_low_trip_count(ptr noalias %a) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <16 x i8> [ <i8 2, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, [[VECTOR_PH]] ], [ [[TMP2:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i64> poison, i64 [[INDEX]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT]], <16 x i64> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    [[VEC_IV:%.*]] = add <16 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>
-; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = icmp ule <16 x i64> [[VEC_IV]], splat (i64 9)
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i8> [ <i8 2, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr [[TMP0]], i32 1, <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> poison)
-; CHECK-NEXT:    [[TMP2]] = mul <16 x i8> [[WIDE_MASKED_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP3:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i8> [[TMP2]], <16 x i8> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP0]], align 1
+; CHECK-NEXT:    [[TMP1]] = mul <8 x i8> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8
+; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP4:%.*]] = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> [[TMP3]])
-; CHECK-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> [[TMP1]])
+; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ 2, [[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 8, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ 2, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -386,9 +368,9 @@ define i8 @mul_non_pow_2_low_trip_count(ptr noalias %a) {
 ; CHECK-NEXT:    [[MUL]] = mul i8 [[TMP5]], [[RDX]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 10
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi i8 [ [[MUL]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi i8 [ [[MUL]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    ret i8 [[MUL_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll
index 10ba208390c33..a46d877825f87 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll
@@ -11,48 +11,46 @@ target triple = "riscv64"
 define void @test(ptr noalias nocapture %a, ptr noalias nocapture %b, i32 %v) {
 ; VLENUNK-LABEL: @test(
 ; VLENUNK-NEXT:  entry:
-; VLENUNK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; VLENUNK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; VLENUNK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; VLENUNK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLENUNK:       vector.ph:
-; VLENUNK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; VLENUNK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; VLENUNK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; VLENUNK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; VLENUNK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
 ; VLENUNK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; VLENUNK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; VLENUNK-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 1)
 ; VLENUNK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP8]]
-; VLENUNK-NEXT:    [[TMP11:%.*]] = mul i64 1, [[TMP5]]
-; VLENUNK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
-; VLENUNK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; VLENUNK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VLENUNK:       vector.body:
 ; VLENUNK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VLENUNK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VLENUNK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; VLENUNK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; VLENUNK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
+; VLENUNK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; VLENUNK-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP7]] to i64
+; VLENUNK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP15]], i64 0
+; VLENUNK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; VLENUNK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; VLENUNK-NEXT:    [[TMP11:%.*]] = icmp ult <vscale x 4 x i32> [[TMP10]], [[BROADCAST_SPLAT4]]
 ; VLENUNK-NEXT:    [[TMP13:%.*]] = icmp ult <vscale x 4 x i64> [[VEC_IND]], splat (i64 512)
+; VLENUNK-NEXT:    [[TMP16:%.*]] = select <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i1> zeroinitializer
 ; VLENUNK-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]]
-; VLENUNK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP14]], i32 4, <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> poison)
-; VLENUNK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32> zeroinitializer
+; VLENUNK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> [[TMP13]], i32 [[TMP7]])
+; VLENUNK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; VLENUNK-NEXT:    [[TMP17:%.*]] = add <vscale x 4 x i32> [[PREDPHI]], [[BROADCAST_SPLAT]]
 ; VLENUNK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
-; VLENUNK-NEXT:    store <vscale x 4 x i32> [[TMP17]], ptr [[TMP18]], align 4
-; VLENUNK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; VLENUNK-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP17]], ptr align 4 [[TMP18]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; VLENUNK-NEXT:    [[TMP19:%.*]] = zext i32 [[TMP7]] to i64
+; VLENUNK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP19]], [[INDEX]]
+; VLENUNK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP19]]
 ; VLENUNK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; VLENUNK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; VLENUNK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; VLENUNK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; VLENUNK:       middle.block:
-; VLENUNK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; VLENUNK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; VLENUNK-NEXT:    br label [[FOR_END:%.*]]
 ; VLENUNK:       scalar.ph:
-; VLENUNK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; VLENUNK-NEXT:    br label [[FOR_BODY:%.*]]
 ; VLENUNK:       for.body:
-; VLENUNK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; VLENUNK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
 ; VLENUNK-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[IV]], 512
 ; VLENUNK-NEXT:    br i1 [[ICMP]], label [[DO_LOAD:%.*]], label [[LATCH]]
 ; VLENUNK:       do_load:
@@ -66,7 +64,7 @@ define void @test(ptr noalias nocapture %a, ptr noalias nocapture %b, i32 %v) {
 ; VLENUNK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX2]], align 4
 ; VLENUNK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; VLENUNK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; VLENUNK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; VLENUNK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; VLENUNK:       for.end:
 ; VLENUNK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index 6800a93ce6db9..83f23a48285f5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -16,11 +16,7 @@
 define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture readonly %trigger) local_unnamed_addr #0 {
 ; RV32-LABEL: @foo4(
 ; RV32-NEXT:  entry:
-; RV32-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; RV32-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
-; RV32-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 625, [[TMP2]]
-; RV32-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; RV32-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; RV32:       vector.memcheck:
 ; RV32-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i32 79880
 ; RV32-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TRIGGER:%.*]], i32 39940
@@ -34,42 +30,39 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV32-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
 ; RV32-NEXT:    br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; RV32:       vector.ph:
-; RV32-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
-; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]]
-; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]]
-; RV32-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
-; RV32-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 16
 ; RV32-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; RV32-NEXT:    [[TMP9:%.*]] = mul <vscale x 2 x i64> [[TMP7]], splat (i64 16)
 ; RV32-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP9]]
-; RV32-NEXT:    [[TMP12:%.*]] = mul i64 16, [[TMP6]]
-; RV32-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i64 0
-; RV32-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; RV32-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; RV32:       vector.body:
 ; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; RV32-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RV32-NEXT:    [[AVL:%.*]] = phi i64 [ 625, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RV32-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; RV32-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
+; RV32-NEXT:    [[TMP11:%.*]] = mul i64 16, [[TMP8]]
+; RV32-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
+; RV32-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; RV32-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
-; RV32-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> [[TMP13]], i32 4, <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i32> poison), !alias.scope [[META0:![0-9]+]]
+; RV32-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.vp.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP13]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
 ; RV32-NEXT:    [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
 ; RV32-NEXT:    [[TMP15:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], splat (i64 1)
 ; RV32-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP15]]
-; RV32-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> [[TMP14]], <vscale x 2 x double> poison), !alias.scope [[META3:![0-9]+]]
+; RV32-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.vp.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP16]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
 ; RV32-NEXT:    [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
 ; RV32-NEXT:    [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
 ; RV32-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
-; RV32-NEXT:    call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP18]], <vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> [[TMP14]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
-; RV32-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; RV32-NEXT:    call void @llvm.vp.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP18]], <vscale x 2 x ptr> align 8 [[TMP19]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
+; RV32-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP10]] to i64
+; RV32-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP20]], [[INDEX]]
+; RV32-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP20]]
 ; RV32-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; RV32-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV32-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; RV32-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 625
+; RV32-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; RV32:       middle.block:
-; RV32-NEXT:    [[CMP_N:%.*]] = icmp eq i64 625, [[N_VEC]]
-; RV32-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; RV32-NEXT:    br label [[FOR_END:%.*]]
 ; RV32:       scalar.ph:
-; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; RV32-NEXT:    br label [[FOR_BODY:%.*]]
 ; RV32:       for.body:
 ; RV32-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -89,17 +82,13 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV32:       for.inc:
 ; RV32-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 16
 ; RV32-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 10000
-; RV32-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP11:![0-9]+]]
+; RV32-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP12:![0-9]+]]
 ; RV32:       for.end:
 ; RV32-NEXT:    ret void
 ;
 ; RV64-LABEL: @foo4(
 ; RV64-NEXT:  entry:
-; RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; RV64-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
-; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 625, [[TMP2]]
-; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; RV64-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; RV64:       vector.memcheck:
 ; RV64-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 79880
 ; RV64-NEXT:    [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TRIGGER:%.*]], i64 39940
@@ -113,42 +102,39 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV64-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT5]]
 ; RV64-NEXT:    br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; RV64:       vector.ph:
-; RV64-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
-; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]]
-; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]]
-; RV64-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
-; RV64-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 16
 ; RV64-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; RV64-NEXT:    [[TMP9:%.*]] = mul <vscale x 2 x i64> [[TMP7]], splat (i64 16)
 ; RV64-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP9]]
-; RV64-NEXT:    [[TMP12:%.*]] = mul i64 16, [[TMP6]]
-; RV64-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i64 0
-; RV64-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; RV64-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; RV64:       vector.body:
 ; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; RV64-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RV64-NEXT:    [[AVL:%.*]] = phi i64 [ 625, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; RV64-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; RV64-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP10]] to i64
+; RV64-NEXT:    [[TMP11:%.*]] = mul i64 16, [[TMP8]]
+; RV64-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
+; RV64-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; RV64-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TRIGGER]], <vscale x 2 x i64> [[VEC_IND]]
-; RV64-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> [[TMP13]], i32 4, <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i32> poison), !alias.scope [[META0:![0-9]+]]
+; RV64-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i32> @llvm.vp.gather.nxv2i32.nxv2p0(<vscale x 2 x ptr> align 4 [[TMP13]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
 ; RV64-NEXT:    [[TMP14:%.*]] = icmp slt <vscale x 2 x i32> [[WIDE_MASKED_GATHER]], splat (i32 100)
 ; RV64-NEXT:    [[TMP15:%.*]] = shl nuw nsw <vscale x 2 x i64> [[VEC_IND]], splat (i64 1)
 ; RV64-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, ptr [[B]], <vscale x 2 x i64> [[TMP15]]
-; RV64-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> [[TMP16]], i32 8, <vscale x 2 x i1> [[TMP14]], <vscale x 2 x double> poison), !alias.scope [[META3:![0-9]+]]
+; RV64-NEXT:    [[WIDE_MASKED_GATHER6:%.*]] = call <vscale x 2 x double> @llvm.vp.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP16]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]]
 ; RV64-NEXT:    [[TMP17:%.*]] = sitofp <vscale x 2 x i32> [[WIDE_MASKED_GATHER]] to <vscale x 2 x double>
 ; RV64-NEXT:    [[TMP18:%.*]] = fadd <vscale x 2 x double> [[WIDE_MASKED_GATHER6]], [[TMP17]]
 ; RV64-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, ptr [[A]], <vscale x 2 x i64> [[VEC_IND]]
-; RV64-NEXT:    call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP18]], <vscale x 2 x ptr> [[TMP19]], i32 8, <vscale x 2 x i1> [[TMP14]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
-; RV64-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; RV64-NEXT:    call void @llvm.vp.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[TMP18]], <vscale x 2 x ptr> align 8 [[TMP19]], <vscale x 2 x i1> [[TMP14]], i32 [[TMP10]]), !alias.scope [[META5:![0-9]+]], !noalias [[META7:![0-9]+]]
+; RV64-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP10]] to i64
+; RV64-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP20]], [[INDEX]]
+; RV64-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP20]]
 ; RV64-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; RV64-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV64-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; RV64-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], 625
+; RV64-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; RV64:       middle.block:
-; RV64-NEXT:    [[CMP_N:%.*]] = icmp eq i64 625, [[N_VEC]]
-; RV64-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; RV64-NEXT:    br label [[FOR_END:%.*]]
 ; RV64:       scalar.ph:
-; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; RV64-NEXT:    br label [[FOR_BODY:%.*]]
 ; RV64:       for.body:
 ; RV64-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
@@ -168,7 +154,7 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV64:       for.inc:
 ; RV64-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 16
 ; RV64-NEXT:    [[CMP:%.*]] = icmp ult i64 [[INDVARS_IV_NEXT]], 10000
-; RV64-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP11:![0-9]+]]
+; RV64-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP12:![0-9]+]]
 ; RV64:       for.end:
 ; RV64-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll
index 0afe04e610bce..07a7b7b1fcc1c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/only-compute-cost-for-vplan-vfs.ll
@@ -1,29 +1,36 @@
-; RUN: opt -passes=loop-vectorize \
-; RUN: -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
-; RUN: -mtriple=riscv64 -mattr=+v -S -debug %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -mtriple=riscv64 -mattr=+v -S -debug %s 2>&1 | FileCheck %s
 
 ; REQUIRES: asserts
 
-; Make sure we do not vectorize a loop with a widened pointer induction.
-define void @test_wide_pointer_induction(ptr noalias %a, i64 %N) {
+; For %for.1, we are fine initially, because the previous value %for.1.next dominates the
+; user of %for.1. But for %for.2, we have to sink the user (%for.1.next) past the previous
+; value %for.2.next. This however breaks the condition we have for %for.1. We cannot fix
+; both first order recurrences and cannot vectorize the loop.
+;
+; Make sure we don't compute costs if there are no vector VPlans.
+
 ; CHECK-NOT: LV: Vector loop of width {{.+}} costs:
 ;
-; CHECK: define void @test_wide_pointer_induction(
+; CHECK: define i32 @test(
 ; CHECK-NOT: vector.body
 ;
+define i32 @test(i32 %N) {
 entry:
-  br label %loop
+  br label %for.body
 
-loop:
-  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
-  %iv.ptr = phi ptr [ %a, %entry ], [ %iv.ptr.next, %loop ]
-  %arrayidx = getelementptr inbounds i64, ptr %a, i64 %iv
-  store ptr %iv.ptr, ptr %arrayidx, align 8
-  %iv.next = add nuw nsw i64 %iv, 1
-  %iv.ptr.next = getelementptr i64, ptr %iv.ptr, i32 1
-  %exitcond.not = icmp eq i64 %iv.next, %N
-  br i1 %exitcond.not, label %exit, label %loop
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %iv  = phi i32 [ %inc, %for.body ], [ 10, %entry ]
+  %for.1 = phi i32 [ %for.1.next, %for.body ], [ 20, %entry ]
+  %for.2 = phi i32 [ %for.2.next, %for.body ], [ 11, %entry ]
+  %for.1.next = add nsw i32 %for.2, 1
+  %for.2.next = shl i32 %for.1, 24
+  %inc = add nsw i32 %iv, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond1.for.end_crit_edge, label %for.body
 
-exit:
-  ret void
+for.cond1.for.end_crit_edge:                      ; preds = %for.body
+  %add.lcssa = phi i32 [ %for.1.next, %for.body ]
+  %sext.lcssa = phi i32 [ %for.2.next, %for.body ]
+  %res = add i32 %add.lcssa, %sext.lcssa
+  ret i32 %res
 }
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
index ee6b950f9b911..dff6c793897da 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph:" --version 4
-; RUN: opt -passes=loop-vectorize -mattr=+v -S < %s | FileCheck %s --check-prefixes=CHECK,V
-; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvqdotq -S < %s | FileCheck %s --check-prefixes=CHECK,ZVQDOTQ
-; RUN: opt -passes=loop-vectorize -mattr=+v -scalable-vectorization=off -S < %s | FileCheck %s --check-prefixes=FIXED,FIXED-V
-; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvqdotq -scalable-vectorization=off -S < %s | FileCheck %s --check-prefixes=FIXED,FIXED-ZVQDOTQ
+; RUN: opt -passes=loop-vectorize -mattr=+v -prefer-predicate-over-epilogue=scalar-epilogue -S < %s | FileCheck %s --check-prefixes=CHECK,V
+; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvqdotq -prefer-predicate-over-epilogue=scalar-epilogue -S < %s | FileCheck %s --check-prefixes=CHECK,ZVQDOTQ
+; RUN: opt -passes=loop-vectorize -mattr=+v -scalable-vectorization=off -prefer-predicate-over-epilogue=scalar-epilogue -S < %s | FileCheck %s --check-prefixes=FIXED,FIXED-V
+; RUN: opt -passes=loop-vectorize -mattr=+v,+experimental-zvqdotq -scalable-vectorization=off -prefer-predicate-over-epilogue=scalar-epilogue -S < %s | FileCheck %s --check-prefixes=FIXED,FIXED-ZVQDOTQ
+
+; TODO: Remove -prefer-predicate-over-epilogue=scalar-epilogue when partial reductions with EVL tail folding is supported.
 
 target triple = "riscv64-none-unknown-elf"
 
@@ -11,7 +13,7 @@ define i32 @vqdot(ptr %a, ptr %b) #0 {
 ; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; V-NEXT:  entry:
 ; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; V-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; V:       vector.ph:
@@ -19,8 +21,6 @@ define i32 @vqdot(ptr %a, ptr %b) #0 {
 ; V-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; V-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; V:       vector.body:
 ; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -33,7 +33,7 @@ define i32 @vqdot(ptr %a, ptr %b) #0 {
 ; V-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; V-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
 ; V-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; V-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; V-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; V:       middle.block:
@@ -46,7 +46,7 @@ define i32 @vqdot(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ZVQDOTQ-NEXT:  entry:
 ; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; ZVQDOTQ:       vector.ph:
@@ -54,8 +54,6 @@ define i32 @vqdot(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ZVQDOTQ:       vector.body:
 ; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -68,7 +66,7 @@ define i32 @vqdot(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; ZVQDOTQ-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
 ; ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.experimental.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP12]])
-; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; ZVQDOTQ-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; ZVQDOTQ:       middle.block:
@@ -175,7 +173,7 @@ define i32 @vqdotu(ptr %a, ptr %b) #0 {
 ; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; V-NEXT:  entry:
 ; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; V-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; V:       vector.ph:
@@ -183,8 +181,6 @@ define i32 @vqdotu(ptr %a, ptr %b) #0 {
 ; V-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; V-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; V:       vector.body:
 ; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -197,7 +193,7 @@ define i32 @vqdotu(ptr %a, ptr %b) #0 {
 ; V-NEXT:    [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; V-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
 ; V-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; V-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; V-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; V:       middle.block:
@@ -210,7 +206,7 @@ define i32 @vqdotu(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; ZVQDOTQ-NEXT:  entry:
 ; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; ZVQDOTQ:       vector.ph:
@@ -218,8 +214,6 @@ define i32 @vqdotu(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ZVQDOTQ:       vector.body:
 ; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -232,7 +226,7 @@ define i32 @vqdotu(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-NEXT:    [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; ZVQDOTQ-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
 ; ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.experimental.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP12]])
-; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; ZVQDOTQ-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; ZVQDOTQ:       middle.block:
@@ -339,7 +333,7 @@ define i32 @vqdotsu(ptr %a, ptr %b) #0 {
 ; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; V-NEXT:  entry:
 ; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; V-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; V:       vector.ph:
@@ -347,8 +341,6 @@ define i32 @vqdotsu(ptr %a, ptr %b) #0 {
 ; V-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; V-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; V:       vector.body:
 ; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -361,7 +353,7 @@ define i32 @vqdotsu(ptr %a, ptr %b) #0 {
 ; V-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; V-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
 ; V-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; V-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; V-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; V:       middle.block:
@@ -374,7 +366,7 @@ define i32 @vqdotsu(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; ZVQDOTQ-NEXT:  entry:
 ; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; ZVQDOTQ:       vector.ph:
@@ -382,8 +374,6 @@ define i32 @vqdotsu(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ZVQDOTQ:       vector.body:
 ; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -396,7 +386,7 @@ define i32 @vqdotsu(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-NEXT:    [[TMP11:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; ZVQDOTQ-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
 ; ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.experimental.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP12]])
-; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; ZVQDOTQ-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; ZVQDOTQ:       middle.block:
@@ -502,7 +492,7 @@ define i32 @vqdotsu2(ptr %a, ptr %b) #0 {
 ; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; V-NEXT:  entry:
 ; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; V-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; V:       vector.ph:
@@ -510,8 +500,6 @@ define i32 @vqdotsu2(ptr %a, ptr %b) #0 {
 ; V-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; V-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; V:       vector.body:
 ; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -524,7 +512,7 @@ define i32 @vqdotsu2(ptr %a, ptr %b) #0 {
 ; V-NEXT:    [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; V-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
 ; V-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP12]], [[VEC_PHI]]
-; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; V-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; V-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; V-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; V:       middle.block:
@@ -537,7 +525,7 @@ define i32 @vqdotsu2(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; ZVQDOTQ-NEXT:  entry:
 ; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; ZVQDOTQ:       vector.ph:
@@ -545,8 +533,6 @@ define i32 @vqdotsu2(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ZVQDOTQ:       vector.body:
 ; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -559,7 +545,7 @@ define i32 @vqdotsu2(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-NEXT:    [[TMP11:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD1]] to <vscale x 4 x i32>
 ; ZVQDOTQ-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], [[TMP8]]
 ; ZVQDOTQ-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 1 x i32> @llvm.experimental.vector.partial.reduce.add.nxv1i32.nxv4i32(<vscale x 1 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP12]])
-; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; ZVQDOTQ-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; ZVQDOTQ-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; ZVQDOTQ:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
index b5b62d0704c91..37a0a8b4d7c87 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
@@ -9,17 +9,8 @@ define void @pr87378_vpinstruction_or_drop_poison_generating_flags(ptr %arg, i64
 ; CHECK-LABEL: define void @pr87378_vpinstruction_or_drop_poison_generating_flags(
 ; CHECK-SAME: ptr [[ARG:%.*]], i64 [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1001, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1001, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1001, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[B]], i64 0
@@ -29,39 +20,47 @@ define void @pr87378_vpinstruction_or_drop_poison_generating_flags(ptr %arg, i64
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 8 x i64> [[TMP6]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 8 x i64> zeroinitializer, [[TMP7]]
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP5]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP8]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[DOTSPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 8 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1001, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP25:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[TMP25]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP25]] to i64
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 8 x i32> @llvm.stepvector.nxv8i32()
+; CHECK-NEXT:    [[TMP11:%.*]] = icmp ult <vscale x 8 x i32> [[TMP10]], [[BROADCAST_SPLAT8]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ule <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP28:%.*]] = select <vscale x 8 x i1> [[TMP11]], <vscale x 8 x i1> [[TMP13]], <vscale x 8 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp ule <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
-; CHECK-NEXT:    [[TMP15:%.*]] = select <vscale x 8 x i1> [[TMP13]], <vscale x 8 x i1> [[TMP14]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP15:%.*]] = select <vscale x 8 x i1> [[TMP28]], <vscale x 8 x i1> [[TMP14]], <vscale x 8 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP16:%.*]] = xor <vscale x 8 x i1> [[TMP13]], splat (i1 true)
-; CHECK-NEXT:    [[TMP17:%.*]] = or <vscale x 8 x i1> [[TMP15]], [[TMP16]]
+; CHECK-NEXT:    [[TMP29:%.*]] = select <vscale x 8 x i1> [[TMP11]], <vscale x 8 x i1> [[TMP16]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP17:%.*]] = or <vscale x 8 x i1> [[TMP15]], [[TMP29]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = icmp ule <vscale x 8 x i64> [[VEC_IND]], [[BROADCAST_SPLAT4]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = select <vscale x 8 x i1> [[TMP17]], <vscale x 8 x i1> [[TMP18]], <vscale x 8 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP20:%.*]] = xor <vscale x 8 x i1> [[TMP14]], splat (i1 true)
-; CHECK-NEXT:    [[TMP21:%.*]] = select <vscale x 8 x i1> [[TMP13]], <vscale x 8 x i1> [[TMP20]], <vscale x 8 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = select <vscale x 8 x i1> [[TMP28]], <vscale x 8 x i1> [[TMP20]], <vscale x 8 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP22:%.*]] = or <vscale x 8 x i1> [[TMP19]], [[TMP21]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 8 x i1> [[TMP21]], i32 0
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select i1 [[TMP23]], i64 poison, i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr i16, ptr [[ARG]], i64 [[PREDPHI]]
-; CHECK-NEXT:    call void @llvm.masked.store.nxv8i16.p0(<vscale x 8 x i16> zeroinitializer, ptr [[TMP24]], i32 2, <vscale x 8 x i1> [[TMP22]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i16.p0(<vscale x 8 x i16> zeroinitializer, ptr align 2 [[TMP24]], <vscale x 8 x i1> [[TMP22]], i32 [[TMP25]])
+; CHECK-NEXT:    [[TMP26:%.*]] = zext i32 [[TMP25]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP26]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 8 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1001
+; CHECK-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1001, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp ule i64 [[IV]], [[A]]
 ; CHECK-NEXT:    br i1 [[C_1]], label [[THEN_1:%.*]], label [[ELSE_1:%.*]]
 ; CHECK:       then.1:
@@ -80,7 +79,7 @@ define void @pr87378_vpinstruction_or_drop_poison_generating_flags(ptr %arg, i64
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[ICMP:%.*]] = icmp eq i64 [[IV]], 1000
-; CHECK-NEXT:    br i1 [[ICMP]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[ICMP]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -118,8 +117,9 @@ exit:
   ret void
 }
 ;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
index 01df43618aad0..5820aaee1677a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
@@ -7,14 +7,6 @@ define void @test(ptr %p, i64 %a, i8 %b) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH1:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 9, [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i8> poison, i8 [[B]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[A]], i64 0
@@ -34,13 +26,10 @@ define void @test(ptr %p, i64 %a, i8 %b) {
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_COND]] ]
 ; CHECK-NEXT:    [[AVL:%.*]] = phi i32 [ 9, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[FOR_COND]] ]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 2, i1 true)
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP11]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT5]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 1, [[TMP11]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP20]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP11]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT7]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP19:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult <vscale x 2 x i32> [[TMP19]], [[BROADCAST_SPLAT6]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult <vscale x 2 x i32> [[TMP19]], [[BROADCAST_SPLAT8]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp sge <vscale x 2 x i32> [[VEC_IND]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP15:%.*]] = select <vscale x 2 x i1> [[TMP13]], <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i1> zeroinitializer
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP15]], <vscale x 2 x i32> [[TMP7]], <vscale x 2 x i32> [[TMP8]]
@@ -55,10 +44,9 @@ define void @test(ptr %p, i64 %a, i8 %b) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT1:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_COND1:%.*]]
 ; CHECK:       for.cond:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH1]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH1]] ], [ [[ADD:%.*]], [[FOR_BODY:%.*]] ]
 ; CHECK-NEXT:    [[ADD]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[CMP_SLT:%.*]] = icmp slt i32 [[IV]], 2
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[A]], 48
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll
index 554ce7b81e953..f0a42b9214243 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reductions.ll
@@ -8,47 +8,40 @@
 define i32 @add(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: define i32 @add(
 ; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP7]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP7:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP7]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP8]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ 2, %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP10]], [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[ADD_LCSSA]]
 ;
 entry:
@@ -68,52 +61,167 @@ for.end:                                 ; preds = %for.body, %entry
   ret i32 %add
 }
 
+define i32 @sub(ptr %a, i64 %n) {
+; CHECK-LABEL: define i32 @sub(
+; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 1024, i32 0), %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP0]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sub <vscale x 4 x i32> [[VEC_PHI]], [[VP_OP_LOAD]]
+; CHECK-NEXT:    [[TMP3]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP1]])
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP4]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP3]])
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ 1024, %[[SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[SUB]] = sub i32 [[RDX]], [[X]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SUB_LCSSA:%.*]] = phi i32 [ [[SUB]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[SUB_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ 1024, %entry ], [ %sub, %loop ]
+  %gep = getelementptr i32, ptr %a, i64 %iv
+  %x = load i32, ptr %gep
+  %sub = sub i32 %rdx, %x
+  %iv.next = add i64 %iv, 1
+  %done = icmp eq i64 %iv.next, %n
+  br i1 %done, label %exit, label %loop
+
+exit:
+  ret i32 %sub
+}
+
+define i32 @addsub(ptr %a, ptr %b, i64 %n) {
+; CHECK-LABEL: define i32 @addsub(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP0]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[VP_OP_LOAD]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[VP_OP_LOAD1:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP1]])
+; CHECK-NEXT:    [[TMP9:%.*]] = sub <vscale x 4 x i32> [[TMP2]], [[VP_OP_LOAD1]]
+; CHECK-NEXT:    [[TMP5]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP9]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP1]])
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP6]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP5]])
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[GEP_A]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[RDX]], [[X]]
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    [[Y:%.*]] = load i32, ptr [[GEP_B]], align 4
+; CHECK-NEXT:    [[SUB]] = sub i32 [[ADD]], [[Y]]
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[DONE]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[SUB_LCSSA:%.*]] = phi i32 [ [[SUB]], %[[LOOP]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[SUB_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ 0, %entry ], [ %sub, %loop ]
+  %gep.a = getelementptr i32, ptr %a, i64 %iv
+  %x = load i32, ptr %gep.a
+  %add = add i32 %rdx, %x
+  %gep.b = getelementptr i32, ptr %b, i64 %iv
+  %y = load i32, ptr %gep.b
+  %sub = sub i32 %add, %y
+  %iv.next = add i64 %iv, 1
+  %done = icmp eq i64 %iv.next, %n
+  br i1 %done, label %exit, label %loop
+
+exit:
+  ret i32 %sub
+}
+
+
 ; OR
 
 define i32 @or(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: define i32 @or(
 ; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP7]] = or <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP7:%.*]] = or <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[TMP7]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[TMP8]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[OR:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ 2, %[[SCALAR_PH]] ], [ [[OR:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[OR]] = or i32 [[TMP10]], [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[OR_LCSSA:%.*]] = phi i32 [ [[OR]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[OR_LCSSA]]
 ;
 entry:
@@ -138,47 +246,40 @@ for.end:                                 ; preds = %for.body, %entry
 define i32 @and(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: define i32 @and(
 ; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> splat (i32 -1), i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> splat (i32 -1), i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP7]] = and <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP7:%.*]] = and <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> [[TMP7]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> [[TMP8]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[AND:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ 2, %[[SCALAR_PH]] ], [ [[AND:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[AND]] = and i32 [[TMP10]], [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[AND_LCSSA]]
 ;
 entry:
@@ -203,47 +304,40 @@ for.end:                                 ; preds = %for.body, %entry
 define i32 @xor(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: define i32 @xor(
 ; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ insertelement (<vscale x 4 x i32> zeroinitializer, i32 2, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP7]] = xor <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP7:%.*]] = xor <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP7]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP7]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP8]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ 2, %[[SCALAR_PH]] ], [ [[XOR:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[XOR]] = xor i32 [[TMP10]], [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[XOR_LCSSA:%.*]] = phi i32 [ [[XOR]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[XOR_LCSSA:%.*]] = phi i32 [ [[XOR]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[XOR_LCSSA]]
 ;
 entry:
@@ -268,49 +362,42 @@ for.end:                                 ; preds = %for.body, %entry
 define i32 @smin(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: define i32 @smin(
 ; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 2), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 2), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_010:%.*]] = phi i32 [ 2, %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP11]], [[SUM_010]]
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED]] = select i1 [[CMP_I]], i32 [[TMP11]], i32 [[SUM_010]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi i32 [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi i32 [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[DOTSROA_SPECULATED_LCSSA]]
 ;
 entry:
@@ -336,49 +423,42 @@ for.end:
 define i32 @umax(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-LABEL: define i32 @umax(
 ; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 2), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 2), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp ugt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP8]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_010:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_010:%.*]] = phi i32 [ 2, %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP_I:%.*]] = icmp ugt i32 [[TMP11]], [[SUM_010]]
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED]] = select i1 [[CMP_I]], i32 [[TMP11]], i32 [[SUM_010]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi i32 [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi i32 [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[DOTSROA_SPECULATED_LCSSA]]
 ;
 entry:
@@ -404,47 +484,40 @@ for.end:
 define float @fadd_fast(ptr noalias nocapture readonly %a, i64 %n) {
 ; CHECK-LABEL: define float @fadd_fast(
 ; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP7]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP7]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP7]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float 0.000000e+00, <vscale x 4 x float> [[TMP8]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = fadd fast float [[TMP10]], [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[ADD_LCSSA]]
 ;
 entry:
@@ -467,47 +540,40 @@ for.end:
 define half @fadd_fast_half_zvfh(ptr noalias nocapture readonly %a, i64 %n) "target-features"="+zvfh" {
 ; CHECK-LABEL: define half @fadd_fast_half_zvfh(
 ; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP7]] = fadd fast <vscale x 8 x half> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <vscale x 8 x half> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 8 x half> @llvm.vp.merge.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[TMP7]], <vscale x 8 x half> [[VEC_PHI]], i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = call fast half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> [[TMP7]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call fast half @llvm.vector.reduce.fadd.nxv8f16(half 0xH0000, <vscale x 8 x half> [[TMP8]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi half [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0xH0000, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi half [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi half [ 0xH0000, %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load half, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = fadd fast half [[TMP10]], [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi half [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi half [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret half [[ADD_LCSSA]]
 ;
 entry:
@@ -549,7 +615,7 @@ define half @fadd_fast_half_zvfhmin(ptr noalias nocapture readonly %a, i64 %n) "
 ; CHECK-NEXT:    [[TMP3]] = fadd fast <16 x half> [[WIDE_LOAD2]], [[VEC_PHI1]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x half> [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call fast half @llvm.vector.reduce.fadd.v16f16(half 0xH0000, <16 x half> [[BIN_RDX]])
@@ -567,7 +633,7 @@ define half @fadd_fast_half_zvfhmin(ptr noalias nocapture readonly %a, i64 %n) "
 ; CHECK-NEXT:    [[ADD]] = fadd fast half [[TMP6]], [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi half [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret half [[ADD_LCSSA]]
@@ -611,7 +677,7 @@ define bfloat @fadd_fast_bfloat(ptr noalias nocapture readonly %a, i64 %n) "targ
 ; CHECK-NEXT:    [[TMP3]] = fadd fast <16 x bfloat> [[WIDE_LOAD2]], [[VEC_PHI1]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x bfloat> [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call fast bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR0000, <16 x bfloat> [[BIN_RDX]])
@@ -629,7 +695,7 @@ define bfloat @fadd_fast_bfloat(ptr noalias nocapture readonly %a, i64 %n) "targ
 ; CHECK-NEXT:    [[ADD]] = fadd fast bfloat [[TMP6]], [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi bfloat [ [[ADD]], %[[FOR_BODY]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret bfloat [[ADD_LCSSA]]
@@ -656,49 +722,42 @@ for.end:
 define float @fmin_fast(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-LABEL: define float @fmin_fast(
 ; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR4:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = fcmp olt <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP8]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[TMP11]], [[SUM_07]]
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED]] = select i1 [[CMP_I]], float [[TMP11]], float [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi float [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi float [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[DOTSROA_SPECULATED_LCSSA]]
 ;
 entry:
@@ -722,49 +781,42 @@ for.end:
 define half @fmin_fast_half_zvfhmin(ptr noalias nocapture readonly %a, i64 %n) #1 {
 ; CHECK-LABEL: define half @fmin_fast_half_zvfhmin(
 ; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR5:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = fcmp olt <vscale x 8 x half> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 8 x half> @llvm.vp.merge.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[TMP8]], <vscale x 8 x half> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call half @llvm.vector.reduce.fmin.nxv8f16(<vscale x 8 x half> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call half @llvm.vector.reduce.fmin.nxv8f16(<vscale x 8 x half> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi half [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0xH0000, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi half [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi half [ 0xH0000, %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load half, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp olt half [[TMP11]], [[SUM_07]]
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED]] = select i1 [[CMP_I]], half [[TMP11]], half [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi half [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi half [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret half [[DOTSROA_SPECULATED_LCSSA]]
 ;
 entry:
@@ -788,49 +840,42 @@ for.end:
 define bfloat @fmin_fast_bfloat_zvfbfmin(ptr noalias nocapture readonly %a, i64 %n) #2 {
 ; CHECK-LABEL: define bfloat @fmin_fast_bfloat_zvfbfmin(
 ; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR6:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x bfloat> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x bfloat> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x bfloat>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = fcmp olt <vscale x 8 x bfloat> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x bfloat> [[WIDE_LOAD]], <vscale x 8 x bfloat> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x bfloat> [[WIDE_LOAD]], <vscale x 8 x bfloat> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 8 x bfloat> @llvm.vp.merge.nxv8bf16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call bfloat @llvm.vector.reduce.fmin.nxv8bf16(<vscale x 8 x bfloat> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call bfloat @llvm.vector.reduce.fmin.nxv8bf16(<vscale x 8 x bfloat> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi bfloat [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0xR0000, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi bfloat [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi bfloat [ 0xR0000, %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load bfloat, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp olt bfloat [[TMP11]], [[SUM_07]]
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED]] = select i1 [[CMP_I]], bfloat [[TMP11]], bfloat [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi bfloat [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi bfloat [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret bfloat [[DOTSROA_SPECULATED_LCSSA]]
 ;
 entry:
@@ -856,49 +901,42 @@ for.end:
 define float @fmax_fast(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-LABEL: define float @fmax_fast(
 ; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP8]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp fast ogt float [[TMP11]], [[SUM_07]]
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED]] = select i1 [[CMP_I]], float [[TMP11]], float [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi float [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi float [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[DOTSROA_SPECULATED_LCSSA]]
 ;
 entry:
@@ -922,49 +960,42 @@ for.end:
 define half @fmax_fast_half_zvfhmin(ptr noalias nocapture readonly %a, i64 %n) #1 {
 ; CHECK-LABEL: define half @fmax_fast_half_zvfhmin(
 ; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR5]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt <vscale x 8 x half> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 8 x half> @llvm.vp.merge.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[TMP8]], <vscale x 8 x half> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call fast half @llvm.vector.reduce.fmax.nxv8f16(<vscale x 8 x half> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call fast half @llvm.vector.reduce.fmax.nxv8f16(<vscale x 8 x half> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi half [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0xH0000, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi half [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi half [ 0xH0000, %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load half, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp fast ogt half [[TMP11]], [[SUM_07]]
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED]] = select i1 [[CMP_I]], half [[TMP11]], half [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi half [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi half [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret half [[DOTSROA_SPECULATED_LCSSA]]
 ;
 entry:
@@ -988,49 +1019,42 @@ for.end:
 define bfloat @fmax_fast_bfloat_zvfbfmin(ptr noalias nocapture readonly %a, i64 %n) #2 {
 ; CHECK-LABEL: define bfloat @fmax_fast_bfloat_zvfbfmin(
 ; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR6]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x bfloat> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x bfloat> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x bfloat>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x bfloat> @llvm.vp.load.nxv8bf16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt <vscale x 8 x bfloat> [[WIDE_LOAD]], [[VEC_PHI]]
-; CHECK-NEXT:    [[TMP8]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x bfloat> [[WIDE_LOAD]], <vscale x 8 x bfloat> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = select <vscale x 8 x i1> [[TMP7]], <vscale x 8 x bfloat> [[WIDE_LOAD]], <vscale x 8 x bfloat> [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 8 x bfloat> @llvm.vp.merge.nxv8bf16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x bfloat> [[TMP8]], <vscale x 8 x bfloat> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call fast bfloat @llvm.vector.reduce.fmax.nxv8bf16(<vscale x 8 x bfloat> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call fast bfloat @llvm.vector.reduce.fmax.nxv8bf16(<vscale x 8 x bfloat> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi bfloat [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0xR0000, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi bfloat [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi bfloat [ 0xR0000, %[[SCALAR_PH]] ], [ [[DOTSROA_SPECULATED:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds bfloat, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load bfloat, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP_I:%.*]] = fcmp fast ogt bfloat [[TMP11]], [[SUM_07]]
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED]] = select i1 [[CMP_I]], bfloat [[TMP11]], bfloat [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi bfloat [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[DOTSROA_SPECULATED_LCSSA:%.*]] = phi bfloat [ [[DOTSROA_SPECULATED]], %[[FOR_BODY]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret bfloat [[DOTSROA_SPECULATED_LCSSA]]
 ;
 entry:
@@ -1077,7 +1101,7 @@ define i32 @mul(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-NEXT:    [[TMP3]] = mul <8 x i32> [[WIDE_LOAD2]], [[VEC_PHI1]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = mul <8 x i32> [[TMP3]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[BIN_RDX]])
@@ -1095,7 +1119,7 @@ define i32 @mul(ptr nocapture %a, ptr nocapture readonly %b, i64 %n) {
 ; CHECK-NEXT:    [[MUL]] = mul nsw i32 [[TMP6]], [[SUM_07]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], %[[FOR_BODY]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[MUL_LCSSA]]
@@ -1142,7 +1166,7 @@ define i32 @memory_dependence(ptr noalias nocapture %a, ptr noalias nocapture re
 ; CHECK-NEXT:    [[TMP5]] = mul <8 x i32> [[WIDE_LOAD1]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> [[TMP5]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -1165,7 +1189,7 @@ define i32 @memory_dependence(ptr noalias nocapture %a, ptr noalias nocapture re
 ; CHECK-NEXT:    [[MUL]] = mul nsw i32 [[TMP9]], [[SUM]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[MUL_LCSSA:%.*]] = phi i32 [ [[MUL]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[MUL_LCSSA]]
@@ -1196,41 +1220,34 @@ for.end:
 define float @fmuladd(ptr %a, ptr %b, i64 %n) {
 ; CHECK-LABEL: define float @fmuladd(
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ insertelement (<vscale x 4 x float> splat (float -0.000000e+00), float 0.000000e+00, i32 0), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[TMP8]] = call reassoc <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD1]], <vscale x 4 x float> [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP8:%.*]] = call reassoc <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD1]], <vscale x 4 x float> [[VEC_PHI]])
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 4 x float> @llvm.vp.merge.nxv4f32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> [[TMP8]], <vscale x 4 x float> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
@@ -1238,9 +1255,9 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    [[MULADD]] = tail call reassoc float @llvm.fmuladd.f32(float [[TMP11]], float [[TMP12]], float [[SUM_07]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MULADD_LCSSA]]
 ;
 entry:
@@ -1265,41 +1282,34 @@ for.end:
 define half @fmuladd_f16_zvfh(ptr %a, ptr %b, i64 %n) "target-features"="+zvfh" {
 ; CHECK-LABEL: define half @fmuladd_f16_zvfh(
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ insertelement (<vscale x 8 x half> splat (half 0xH8000), half 0xH0000, i32 0), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 8 x half> [ insertelement (<vscale x 8 x half> splat (half 0xH8000), half 0xH0000, i32 0), %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP6]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds half, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x half>, ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[TMP8]] = call reassoc <vscale x 8 x half> @llvm.fmuladd.nxv8f16(<vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[WIDE_LOAD1]], <vscale x 8 x half> [[VEC_PHI]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = call <vscale x 8 x half> @llvm.vp.load.nxv8f16.p0(ptr align 4 [[TMP7]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP8:%.*]] = call reassoc <vscale x 8 x half> @llvm.fmuladd.nxv8f16(<vscale x 8 x half> [[WIDE_LOAD]], <vscale x 8 x half> [[WIDE_LOAD1]], <vscale x 8 x half> [[VEC_PHI]])
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 8 x half> @llvm.vp.merge.nxv8f16(<vscale x 8 x i1> splat (i1 true), <vscale x 8 x half> [[TMP8]], <vscale x 8 x half> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = call reassoc half @llvm.vector.reduce.fadd.nxv8f16(half 0xH8000, <vscale x 8 x half> [[TMP8]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP16:%.*]] = call reassoc half @llvm.vector.reduce.fadd.nxv8f16(half 0xH8000, <vscale x 8 x half> [[TMP9]])
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi half [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0xH0000, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi half [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi half [ 0xH0000, %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load half, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds half, ptr [[B]], i64 [[IV]]
@@ -1307,9 +1317,9 @@ define half @fmuladd_f16_zvfh(ptr %a, ptr %b, i64 %n) "target-features"="+zvfh"
 ; CHECK-NEXT:    [[MULADD]] = tail call reassoc half @llvm.fmuladd.f16(half [[TMP11]], half [[TMP12]], half [[SUM_07]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
-; CHECK-NEXT:    [[MULADD_LCSSA:%.*]] = phi half [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[MULADD_LCSSA:%.*]] = phi half [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret half [[MULADD_LCSSA]]
 ;
 entry:
@@ -1360,7 +1370,7 @@ define half @fmuladd_f16_zvfhmin(ptr %a, ptr %b, i64 %n) "target-features"="+zvf
 ; CHECK-NEXT:    [[TMP5]] = call reassoc <16 x half> @llvm.fmuladd.v16f16(<16 x half> [[WIDE_LOAD2]], <16 x half> [[WIDE_LOAD4]], <16 x half> [[VEC_PHI1]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd reassoc <16 x half> [[TMP5]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call reassoc half @llvm.vector.reduce.fadd.v16f16(half 0xH8000, <16 x half> [[BIN_RDX]])
@@ -1380,7 +1390,7 @@ define half @fmuladd_f16_zvfhmin(ptr %a, ptr %b, i64 %n) "target-features"="+zvf
 ; CHECK-NEXT:    [[MULADD]] = tail call reassoc half @llvm.fmuladd.f16(half [[TMP8]], half [[TMP9]], half [[SUM_07]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[MULADD_LCSSA:%.*]] = phi half [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret half [[MULADD_LCSSA]]
@@ -1430,7 +1440,7 @@ define bfloat @fmuladd_bf16(ptr %a, ptr %b, i64 %n) "target-features"="+zvfbfmin
 ; CHECK-NEXT:    [[TMP5]] = call reassoc <16 x bfloat> @llvm.fmuladd.v16bf16(<16 x bfloat> [[WIDE_LOAD2]], <16 x bfloat> [[WIDE_LOAD4]], <16 x bfloat> [[VEC_PHI1]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd reassoc <16 x bfloat> [[TMP5]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call reassoc bfloat @llvm.vector.reduce.fadd.v16bf16(bfloat 0xR8000, <16 x bfloat> [[BIN_RDX]])
@@ -1450,7 +1460,7 @@ define bfloat @fmuladd_bf16(ptr %a, ptr %b, i64 %n) "target-features"="+zvfbfmin
 ; CHECK-NEXT:    [[MULADD]] = tail call reassoc bfloat @llvm.fmuladd.bf16(bfloat [[TMP8]], bfloat [[TMP9]], bfloat [[SUM_07]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[MULADD_LCSSA:%.*]] = phi bfloat [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret bfloat [[MULADD_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
index 5a67b54c7a3d5..346f1cbcc7e3d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll
@@ -1,5 +1,9 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfbfmin -debug-only=loop-vectorize,vplan --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfbfmin -prefer-predicate-over-epilogue=scalar-epilogue -debug-only=loop-vectorize,vplan --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s
+
+; TODO: -prefer-predicate-over-epilogue=scalar-epilogue was added to allow
+; unrolling. Calculate register pressure for all VPlans, not just unrolled ones,
+; and remove.
 
 define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) {
 ; CHECK-LABEL: add
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
index d4909fa61b4f5..b25bc485a9ca7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll
@@ -1,6 +1,10 @@
 ; REQUIRES: asserts
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfh -debug-only=loop-vectorize,vplan --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFH
-; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfhmin -debug-only=loop-vectorize,vplan --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFHMIN
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfh -prefer-predicate-over-epilogue=scalar-epilogue -debug-only=loop-vectorize,vplan --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFH
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v,+zvfhmin -prefer-predicate-over-epilogue=scalar-epilogue -debug-only=loop-vectorize,vplan --disable-output -riscv-v-register-bit-width-lmul=1 -S < %s 2>&1 | FileCheck %s --check-prefix=ZVFHMIN
+
+; TODO: -prefer-predicate-over-epilogue=scalar-epilogue was added to allow
+; unrolling. Calculate register pressure for all VPlans, not just unrolled ones,
+; and remove.
 
 define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) {
 ; CHECK-LABEL: add
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
new file mode 100644
index 0000000000000..6bb0d64314d3e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-maxbandwidth.ll
@@ -0,0 +1,33 @@
+; REQUIRES: asserts
+; RUN: opt -passes=loop-vectorize -mtriple riscv64 -mattr=+v -vectorizer-maximize-bandwidth -debug-only=loop-vectorize,vplan -disable-output -S < %s 2>&1 | FileCheck %s --check-prefixes=CHECK-REGS-VP
+
+define i32 @dotp(ptr %a, ptr %b) {
+; CHECK-REGS-VP:      LV(REG): VF = vscale x 16
+; CHECK-REGS-VP-NEXT: LV(REG): Found max usage: 2 item
+; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 6 registers
+; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 24 registers
+; CHECK-REGS-VP-NEXT: LV(REG): Found invariant usage: 1 item
+; CHECK-REGS-VP-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
+; CHECK-REGS-VP:      LV: Selecting VF: vscale x 16.
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
+  %gep.a = getelementptr i8, ptr %a, i64 %iv
+  %load.a = load i8, ptr %gep.a, align 1
+  %ext.a = zext i8 %load.a to i32
+  %gep.b = getelementptr i8, ptr %b, i64 %iv
+  %load.b = load i8, ptr %gep.b, align 1
+  %ext.b = zext i8 %load.b to i32
+  %mul = mul i32 %ext.b, %ext.a
+  %sub = sub i32 0, %mul
+  %add = add i32 %accum, %sub
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 1024
+  br i1 %exitcond.not, label %for.exit, label %for.body
+
+for.exit:                        ; preds = %for.body
+  ret i32 %add
+}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
index 70372826a8256..116ccc9961795 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll
@@ -1,25 +1,29 @@
 ; REQUIRES: asserts
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
 ; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
-; RUN:   -force-vector-width=1 \
+; RUN:   -force-vector-width=1 -prefer-predicate-over-epilogue=scalar-epilogue \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALAR
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
 ; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
-; RUN:   -riscv-v-register-bit-width-lmul=1 \
+; RUN:   -riscv-v-register-bit-width-lmul=1 -prefer-predicate-over-epilogue=scalar-epilogue \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL1
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
 ; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
-; RUN:   -riscv-v-register-bit-width-lmul=2 \
+; RUN:   -riscv-v-register-bit-width-lmul=2 -prefer-predicate-over-epilogue=scalar-epilogue \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL2
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
 ; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
-; RUN:   -riscv-v-register-bit-width-lmul=4 \
+; RUN:   -riscv-v-register-bit-width-lmul=4 -prefer-predicate-over-epilogue=scalar-epilogue \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL4
 ; RUN: opt -passes=loop-vectorize -mtriple riscv64-linux-gnu \
 ; RUN:   -mattr=+v,+d -debug-only=loop-vectorize,vplan --disable-output \
-; RUN:   -riscv-v-register-bit-width-lmul=8 \
+; RUN:   -riscv-v-register-bit-width-lmul=8 -prefer-predicate-over-epilogue=scalar-epilogue \
 ; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-LMUL8
 
+; TODO: -prefer-predicate-over-epilogue=scalar-epilogue was added to allow
+; unrolling. Calculate register pressure for all VPlans, not just unrolled ones,
+; and remove.
+
 define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) {
 ; CHECK-LABEL: add
 ; CHECK-SCALAR:      LV(REG): VF = 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll
index 85163c79072b5..587f69a4a9be2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll
@@ -1,45 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -mtriple=riscv64 -mattr=+v -p loop-vectorize -pass-remarks-analysis=loop-vectorize -S 2>&1 | FileCheck %s
 
 ; CHECK: remark: <unknown>:0:0: the cost-model indicates that interleaving is not beneficial
 define float @s311(float %a_0, float %s311_sum) {
 ; CHECK-LABEL: define float @s311(
 ; CHECK-SAME: float [[A_0:%.*]], float [[S311_SUM:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1200, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1200, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 1200, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[A_0]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[S311_SUM]], %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP6]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[BROADCAST_SPLAT]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i32 [ 1200, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[TMP6]] = call float @llvm.vp.reduce.fadd.nxv4f32(float [[VEC_PHI]], <vscale x 4 x float> [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP9]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_EVL_NEXT]], 1200
 ; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 1200, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[S311_SUM]], %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi float [ [[S311_SUM]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[RED_NEXT]] = fadd float [[A_0]], [[RED]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], 1200
-; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[RED_LCSSA:%.*]] = phi float [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[RED_LCSSA]]
@@ -60,8 +52,9 @@ exit:
   ret float %red.lcssa
 }
 ;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index 0b3dcf80c448b..e49a71eb5b930 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -18,99 +18,83 @@
 define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-LABEL: define void @vector_reverse_i32(
 ; RV64-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; RV64-NEXT:  [[ENTRY:.*]]:
-; RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
-; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV64-NEXT:  [[ENTRY:.*:]]
+; RV64-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV64:       [[VECTOR_PH]]:
-; RV64-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
-; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
-; RV64-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; RV64-NEXT:    [[TMP6:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV64-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64:       [[VECTOR_BODY]]:
-; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[AVL:%.*]] = phi i64 [ 1023, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[TMP19:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; RV64-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
 ; RV64-NEXT:    [[TMP7:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP7]]
-; RV64-NEXT:    [[TMP9:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP24:%.*]] = zext i32 [[TMP19]] to i64
+; RV64-NEXT:    [[TMP9:%.*]] = mul i64 0, [[TMP24]]
+; RV64-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP24]], 1
 ; RV64-NEXT:    [[TMP11:%.*]] = mul i64 -1, [[TMP10]]
-; RV64-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 [[TMP9]]
-; RV64-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 [[TMP11]]
-; RV64-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP13]], align 4
-; RV64-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
+; RV64-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[TMP8]], i64 [[TMP9]]
+; RV64-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i64 [[TMP11]]
+; RV64-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP19]])
+; RV64-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP19]])
 ; RV64-NEXT:    [[TMP14:%.*]] = add <vscale x 4 x i32> [[REVERSE]], splat (i32 1)
 ; RV64-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
-; RV64-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP17:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP19]] to i64
+; RV64-NEXT:    [[TMP25:%.*]] = mul i64 0, [[TMP16]]
+; RV64-NEXT:    [[TMP17:%.*]] = sub i64 [[TMP16]], 1
 ; RV64-NEXT:    [[TMP18:%.*]] = mul i64 -1, [[TMP17]]
-; RV64-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP16]]
-; RV64-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP18]]
-; RV64-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP14]])
-; RV64-NEXT:    store <vscale x 4 x i32> [[REVERSE1]], ptr [[TMP20]], align 4
-; RV64-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; RV64-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV64-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; RV64-NEXT:    [[TMP20:%.*]] = getelementptr i32, ptr [[TMP15]], i64 [[TMP25]]
+; RV64-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[TMP20]], i64 [[TMP18]]
+; RV64-NEXT:    [[VP_REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP19]])
+; RV64-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE1]], ptr align 4 [[TMP21]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP19]])
+; RV64-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP19]] to i64
+; RV64-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP22]], [[INDEX]]
+; RV64-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP22]]
+; RV64-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1023
+; RV64-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; RV64:       [[MIDDLE_BLOCK]]:
-; RV64-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1023, [[N_VEC]]
-; RV64-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; RV64-NEXT:    br [[EXIT:label %.*]]
 ; RV64:       [[SCALAR_PH]]:
-; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
 ; RV64-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV64:       [[FOR_BODY]]:
 ;
 ; RV32-LABEL: define void @vector_reverse_i32(
 ; RV32-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
-; RV32-NEXT:  [[ENTRY:.*]]:
-; RV32-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; RV32-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
-; RV32-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV32-NEXT:  [[ENTRY:.*:]]
+; RV32-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV32:       [[VECTOR_PH]]:
-; RV32-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
-; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
-; RV32-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; RV32-NEXT:    [[TMP6:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV32-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV32:       [[VECTOR_BODY]]:
-; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[AVL:%.*]] = phi i64 [ 1023, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; RV32-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
 ; RV32-NEXT:    [[TMP7:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV32-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP7]]
-; RV32-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP10:%.*]] = mul i32 0, [[TMP9]]
 ; RV32-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP9]], 1
 ; RV32-NEXT:    [[TMP12:%.*]] = mul i32 -1, [[TMP11]]
-; RV32-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i32 [[TMP10]]
-; RV32-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i32 [[TMP12]]
-; RV32-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
-; RV32-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
+; RV32-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP8]], i32 [[TMP10]]
+; RV32-NEXT:    [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i32 [[TMP12]]
+; RV32-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; RV32-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; RV32-NEXT:    [[TMP15:%.*]] = add <vscale x 4 x i32> [[REVERSE]], splat (i32 1)
 ; RV32-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
-; RV32-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP5]] to i32
-; RV32-NEXT:    [[TMP18:%.*]] = mul i32 0, [[TMP17]]
-; RV32-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP17]], 1
+; RV32-NEXT:    [[TMP17:%.*]] = mul i32 0, [[TMP9]]
+; RV32-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP9]], 1
 ; RV32-NEXT:    [[TMP20:%.*]] = mul i32 -1, [[TMP19]]
-; RV32-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 [[TMP18]]
-; RV32-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i32 [[TMP20]]
-; RV32-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP15]])
-; RV32-NEXT:    store <vscale x 4 x i32> [[REVERSE1]], ptr [[TMP22]], align 4
-; RV32-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; RV32-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV32-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; RV32-NEXT:    [[TMP18:%.*]] = getelementptr i32, ptr [[TMP16]], i32 [[TMP17]]
+; RV32-NEXT:    [[TMP22:%.*]] = getelementptr i32, ptr [[TMP18]], i32 [[TMP20]]
+; RV32-NEXT:    [[VP_REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; RV32-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE1]], ptr align 4 [[TMP22]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; RV32-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP9]] to i64
+; RV32-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP23]], [[INDEX]]
+; RV32-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP23]]
+; RV32-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1023
+; RV32-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; RV32:       [[MIDDLE_BLOCK]]:
-; RV32-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1023, [[N_VEC]]
-; RV32-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; RV32-NEXT:    br [[EXIT:label %.*]]
 ; RV32:       [[SCALAR_PH]]:
-; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
 ; RV32-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV32:       [[FOR_BODY]]:
 ;
@@ -118,18 +102,16 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; RV64-UF2-NEXT:  [[ENTRY:.*]]:
 ; RV64-UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; RV64-UF2-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; RV64-UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
 ; RV64-UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV64-UF2:       [[VECTOR_PH]]:
-; RV64-UF2-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; RV64-UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
-; RV64-UF2-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
 ; RV64-UF2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; RV64-UF2-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; RV64-UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; RV64-UF2-NEXT:    [[N_VEC:%.*]] = urem i64 1023, [[TMP6]]
 ; RV64-UF2-NEXT:    [[TMP7:%.*]] = sub i64 1023, [[N_VEC]]
+; RV64-UF2-NEXT:    [[TMP33:%.*]] = sub i64 1023, [[TMP7]]
 ; RV64-UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64-UF2:       [[VECTOR_BODY]]:
 ; RV64-UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -168,13 +150,13 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP21]])
 ; RV64-UF2-NEXT:    store <vscale x 4 x i32> [[REVERSE4]], ptr [[TMP32]], align 4
 ; RV64-UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; RV64-UF2-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV64-UF2-NEXT:    br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; RV64-UF2-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP7]]
+; RV64-UF2-NEXT:    br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; RV64-UF2:       [[MIDDLE_BLOCK]]:
-; RV64-UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1023, [[N_VEC]]
+; RV64-UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1023, [[TMP7]]
 ; RV64-UF2-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
 ; RV64-UF2:       [[SCALAR_PH]]:
-; RV64-UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; RV64-UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
 ; RV64-UF2-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV64-UF2:       [[FOR_BODY]]:
 ;
@@ -206,10 +188,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
 ; RV64:       [[FOR_BODY_PREHEADER]]:
 ; RV64-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
-; RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; RV64-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; RV64:       [[VECTOR_SCEVCHECK]]:
 ; RV64-NEXT:    [[TMP3:%.*]] = add nsw i64 [[TMP0]], -1
 ; RV64-NEXT:    [[TMP4:%.*]] = add i32 [[N]], -1
@@ -231,48 +210,44 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], [[TMP13]]
 ; RV64-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; RV64:       [[VECTOR_PH]]:
-; RV64-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
-; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP16]]
-; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; RV64-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 4
-; RV64-NEXT:    [[TMP19:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
-; RV64-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
-; RV64-NEXT:    [[TMP20:%.*]] = sub i32 [[N]], [[DOTCAST]]
 ; RV64-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64:       [[VECTOR_BODY]]:
-; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[TMP20:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; RV64-NEXT:    [[DOTCAST3:%.*]] = trunc i64 [[INDEX]] to i32
 ; RV64-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[N]], [[DOTCAST3]]
 ; RV64-NEXT:    [[TMP21:%.*]] = add nsw i32 [[OFFSET_IDX]], -1
 ; RV64-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP21]] to i64
 ; RV64-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP22]]
-; RV64-NEXT:    [[TMP24:%.*]] = mul i64 0, [[TMP18]]
-; RV64-NEXT:    [[TMP25:%.*]] = sub i64 [[TMP18]], 1
+; RV64-NEXT:    [[TMP24:%.*]] = zext i32 [[TMP20]] to i64
+; RV64-NEXT:    [[TMP28:%.*]] = mul i64 0, [[TMP24]]
+; RV64-NEXT:    [[TMP25:%.*]] = sub i64 [[TMP24]], 1
 ; RV64-NEXT:    [[TMP26:%.*]] = mul i64 -1, [[TMP25]]
-; RV64-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i64 [[TMP24]]
-; RV64-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP27]], i64 [[TMP26]]
-; RV64-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP28]], align 4
-; RV64-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
+; RV64-NEXT:    [[TMP38:%.*]] = getelementptr i32, ptr [[TMP23]], i64 [[TMP28]]
+; RV64-NEXT:    [[TMP27:%.*]] = getelementptr i32, ptr [[TMP38]], i64 [[TMP26]]
+; RV64-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP27]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP20]])
+; RV64-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP20]])
 ; RV64-NEXT:    [[TMP29:%.*]] = add <vscale x 4 x i32> [[REVERSE]], splat (i32 1)
 ; RV64-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP22]]
-; RV64-NEXT:    [[TMP31:%.*]] = mul i64 0, [[TMP18]]
-; RV64-NEXT:    [[TMP32:%.*]] = sub i64 [[TMP18]], 1
+; RV64-NEXT:    [[TMP39:%.*]] = zext i32 [[TMP20]] to i64
+; RV64-NEXT:    [[TMP31:%.*]] = mul i64 0, [[TMP39]]
+; RV64-NEXT:    [[TMP32:%.*]] = sub i64 [[TMP39]], 1
 ; RV64-NEXT:    [[TMP33:%.*]] = mul i64 -1, [[TMP32]]
-; RV64-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i32, ptr [[TMP30]], i64 [[TMP31]]
-; RV64-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i32, ptr [[TMP34]], i64 [[TMP33]]
-; RV64-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP29]])
-; RV64-NEXT:    store <vscale x 4 x i32> [[REVERSE4]], ptr [[TMP35]], align 4
-; RV64-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
-; RV64-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV64-NEXT:    br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; RV64-NEXT:    [[TMP34:%.*]] = getelementptr i32, ptr [[TMP30]], i64 [[TMP31]]
+; RV64-NEXT:    [[TMP35:%.*]] = getelementptr i32, ptr [[TMP34]], i64 [[TMP33]]
+; RV64-NEXT:    [[VP_REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[TMP29]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP20]])
+; RV64-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE3]], ptr align 4 [[TMP35]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP20]])
+; RV64-NEXT:    [[TMP36:%.*]] = zext i32 [[TMP20]] to i64
+; RV64-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP36]], [[INDEX]]
+; RV64-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP36]]
+; RV64-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP0]]
+; RV64-NEXT:    br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; RV64:       [[MIDDLE_BLOCK]]:
-; RV64-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; RV64-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; RV64-NEXT:    br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
 ; RV64:       [[SCALAR_PH]]:
-; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
-; RV64-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
+; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; RV64-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
 ; RV64-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV64:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
 ; RV64-NEXT:    br label %[[FOR_COND_CLEANUP]]
@@ -289,10 +264,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV32-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
 ; RV32:       [[FOR_BODY_PREHEADER]]:
 ; RV32-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
-; RV32-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; RV32-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; RV32-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; RV32-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; RV32:       [[VECTOR_MEMCHECK]]:
 ; RV32-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
 ; RV32-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
@@ -301,50 +273,42 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV32-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i32 [[TMP6]], [[TMP5]]
 ; RV32-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; RV32:       [[VECTOR_PH]]:
-; RV32-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
-; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP8]]
-; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; RV32-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
-; RV32-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
-; RV32-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
-; RV32-NEXT:    [[TMP12:%.*]] = sub i32 [[N]], [[DOTCAST]]
 ; RV32-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV32:       [[VECTOR_BODY]]:
-; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[TMP16:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; RV32-NEXT:    [[DOTCAST3:%.*]] = trunc i64 [[INDEX]] to i32
 ; RV32-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[N]], [[DOTCAST3]]
 ; RV32-NEXT:    [[TMP13:%.*]] = add nsw i32 [[OFFSET_IDX]], -1
 ; RV32-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
 ; RV32-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP14]]
-; RV32-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP10]] to i32
 ; RV32-NEXT:    [[TMP17:%.*]] = mul i32 0, [[TMP16]]
 ; RV32-NEXT:    [[TMP18:%.*]] = sub i32 [[TMP16]], 1
 ; RV32-NEXT:    [[TMP19:%.*]] = mul i32 -1, [[TMP18]]
-; RV32-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i32 [[TMP17]]
-; RV32-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP20]], i32 [[TMP19]]
-; RV32-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP21]], align 4
-; RV32-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
+; RV32-NEXT:    [[TMP20:%.*]] = getelementptr i32, ptr [[TMP15]], i32 [[TMP17]]
+; RV32-NEXT:    [[TMP28:%.*]] = getelementptr i32, ptr [[TMP20]], i32 [[TMP19]]
+; RV32-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP28]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
+; RV32-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
 ; RV32-NEXT:    [[TMP22:%.*]] = add <vscale x 4 x i32> [[REVERSE]], splat (i32 1)
 ; RV32-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP14]]
-; RV32-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP10]] to i32
-; RV32-NEXT:    [[TMP25:%.*]] = mul i32 0, [[TMP24]]
-; RV32-NEXT:    [[TMP26:%.*]] = sub i32 [[TMP24]], 1
+; RV32-NEXT:    [[TMP21:%.*]] = mul i32 0, [[TMP16]]
+; RV32-NEXT:    [[TMP26:%.*]] = sub i32 [[TMP16]], 1
 ; RV32-NEXT:    [[TMP27:%.*]] = mul i32 -1, [[TMP26]]
-; RV32-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i32 [[TMP25]]
-; RV32-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP28]], i32 [[TMP27]]
-; RV32-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP22]])
-; RV32-NEXT:    store <vscale x 4 x i32> [[REVERSE4]], ptr [[TMP29]], align 4
-; RV32-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
-; RV32-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV32-NEXT:    br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; RV32-NEXT:    [[TMP24:%.*]] = getelementptr i32, ptr [[TMP23]], i32 [[TMP21]]
+; RV32-NEXT:    [[TMP25:%.*]] = getelementptr i32, ptr [[TMP24]], i32 [[TMP27]]
+; RV32-NEXT:    [[VP_REVERSE3:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[TMP22]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
+; RV32-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[VP_REVERSE3]], ptr align 4 [[TMP25]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
+; RV32-NEXT:    [[TMP29:%.*]] = zext i32 [[TMP16]] to i64
+; RV32-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP29]], [[INDEX]]
+; RV32-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP29]]
+; RV32-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP0]]
+; RV32-NEXT:    br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; RV32:       [[MIDDLE_BLOCK]]:
-; RV32-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; RV32-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; RV32-NEXT:    br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
 ; RV32:       [[SCALAR_PH]]:
-; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
-; RV32-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
+; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; RV32-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
 ; RV32-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV32:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
 ; RV32-NEXT:    br label %[[FOR_COND_CLEANUP]]
@@ -362,7 +326,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-UF2:       [[FOR_BODY_PREHEADER]]:
 ; RV64-UF2-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; RV64-UF2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; RV64-UF2-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; RV64-UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; RV64-UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; RV64-UF2:       [[VECTOR_SCEVCHECK]]:
@@ -386,15 +350,13 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-UF2-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], [[TMP13]]
 ; RV64-UF2-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; RV64-UF2:       [[VECTOR_PH]]:
-; RV64-UF2-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
-; RV64-UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP16]]
-; RV64-UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; RV64-UF2-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
 ; RV64-UF2-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 4
 ; RV64-UF2-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 2
+; RV64-UF2-NEXT:    [[N_VEC:%.*]] = urem i64 [[TMP0]], [[TMP19]]
 ; RV64-UF2-NEXT:    [[TMP20:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
-; RV64-UF2-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
+; RV64-UF2-NEXT:    [[TMP48:%.*]] = sub i64 [[TMP0]], [[TMP20]]
+; RV64-UF2-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[TMP20]] to i32
 ; RV64-UF2-NEXT:    [[TMP21:%.*]] = sub i32 [[N]], [[DOTCAST]]
 ; RV64-UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64-UF2:       [[VECTOR_BODY]]:
@@ -436,13 +398,13 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-UF2-NEXT:    [[REVERSE7:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[TMP36]])
 ; RV64-UF2-NEXT:    store <vscale x 4 x i32> [[REVERSE7]], ptr [[TMP47]], align 4
 ; RV64-UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
-; RV64-UF2-NEXT:    [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV64-UF2-NEXT:    br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; RV64-UF2-NEXT:    [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP20]]
+; RV64-UF2-NEXT:    br i1 [[TMP49]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; RV64-UF2:       [[MIDDLE_BLOCK]]:
-; RV64-UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; RV64-UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[TMP20]]
 ; RV64-UF2-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; RV64-UF2:       [[SCALAR_PH]]:
-; RV64-UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; RV64-UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP48]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
 ; RV64-UF2-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
 ; RV64-UF2-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV64-UF2:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
@@ -487,10 +449,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
 ; RV64:       [[FOR_BODY_PREHEADER]]:
 ; RV64-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
-; RV64-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; RV64-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; RV64:       [[VECTOR_SCEVCHECK]]:
 ; RV64-NEXT:    [[TMP3:%.*]] = add nsw i64 [[TMP0]], -1
 ; RV64-NEXT:    [[TMP4:%.*]] = add i32 [[N]], -1
@@ -512,48 +471,44 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], [[TMP13]]
 ; RV64-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; RV64:       [[VECTOR_PH]]:
-; RV64-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
-; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP16]]
-; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; RV64-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 4
-; RV64-NEXT:    [[TMP19:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
-; RV64-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
-; RV64-NEXT:    [[TMP20:%.*]] = sub i32 [[N]], [[DOTCAST]]
 ; RV64-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64:       [[VECTOR_BODY]]:
-; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[TMP20:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; RV64-NEXT:    [[DOTCAST3:%.*]] = trunc i64 [[INDEX]] to i32
 ; RV64-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[N]], [[DOTCAST3]]
 ; RV64-NEXT:    [[TMP21:%.*]] = add nsw i32 [[OFFSET_IDX]], -1
 ; RV64-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP21]] to i64
 ; RV64-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP22]]
-; RV64-NEXT:    [[TMP24:%.*]] = mul i64 0, [[TMP18]]
-; RV64-NEXT:    [[TMP25:%.*]] = sub i64 [[TMP18]], 1
+; RV64-NEXT:    [[TMP24:%.*]] = zext i32 [[TMP20]] to i64
+; RV64-NEXT:    [[TMP28:%.*]] = mul i64 0, [[TMP24]]
+; RV64-NEXT:    [[TMP25:%.*]] = sub i64 [[TMP24]], 1
 ; RV64-NEXT:    [[TMP26:%.*]] = mul i64 -1, [[TMP25]]
-; RV64-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[TMP24]]
-; RV64-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP26]]
-; RV64-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP28]], align 4
-; RV64-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]])
+; RV64-NEXT:    [[TMP38:%.*]] = getelementptr float, ptr [[TMP23]], i64 [[TMP28]]
+; RV64-NEXT:    [[TMP27:%.*]] = getelementptr float, ptr [[TMP38]], i64 [[TMP26]]
+; RV64-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP27]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP20]])
+; RV64-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP20]])
 ; RV64-NEXT:    [[TMP29:%.*]] = fadd <vscale x 4 x float> [[REVERSE]], splat (float 1.000000e+00)
 ; RV64-NEXT:    [[TMP30:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP22]]
-; RV64-NEXT:    [[TMP31:%.*]] = mul i64 0, [[TMP18]]
-; RV64-NEXT:    [[TMP32:%.*]] = sub i64 [[TMP18]], 1
+; RV64-NEXT:    [[TMP39:%.*]] = zext i32 [[TMP20]] to i64
+; RV64-NEXT:    [[TMP31:%.*]] = mul i64 0, [[TMP39]]
+; RV64-NEXT:    [[TMP32:%.*]] = sub i64 [[TMP39]], 1
 ; RV64-NEXT:    [[TMP33:%.*]] = mul i64 -1, [[TMP32]]
-; RV64-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP30]], i64 [[TMP31]]
-; RV64-NEXT:    [[TMP35:%.*]] = getelementptr inbounds float, ptr [[TMP34]], i64 [[TMP33]]
-; RV64-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP29]])
-; RV64-NEXT:    store <vscale x 4 x float> [[REVERSE4]], ptr [[TMP35]], align 4
-; RV64-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]]
-; RV64-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV64-NEXT:    br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; RV64-NEXT:    [[TMP34:%.*]] = getelementptr float, ptr [[TMP30]], i64 [[TMP31]]
+; RV64-NEXT:    [[TMP35:%.*]] = getelementptr float, ptr [[TMP34]], i64 [[TMP33]]
+; RV64-NEXT:    [[VP_REVERSE3:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[TMP29]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP20]])
+; RV64-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_REVERSE3]], ptr align 4 [[TMP35]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP20]])
+; RV64-NEXT:    [[TMP36:%.*]] = zext i32 [[TMP20]] to i64
+; RV64-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP36]], [[INDEX]]
+; RV64-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP36]]
+; RV64-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP0]]
+; RV64-NEXT:    br i1 [[TMP37]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; RV64:       [[MIDDLE_BLOCK]]:
-; RV64-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; RV64-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; RV64-NEXT:    br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
 ; RV64:       [[SCALAR_PH]]:
-; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
-; RV64-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
+; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; RV64-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
 ; RV64-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV64:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
 ; RV64-NEXT:    br label %[[FOR_COND_CLEANUP]]
@@ -570,10 +525,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV32-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY_PREHEADER:.*]], label %[[FOR_COND_CLEANUP:.*]]
 ; RV32:       [[FOR_BODY_PREHEADER]]:
 ; RV32-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
-; RV32-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; RV32-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; RV32-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; RV32-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; RV32:       [[VECTOR_MEMCHECK]]:
 ; RV32-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
 ; RV32-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
@@ -582,50 +534,42 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV32-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i32 [[TMP6]], [[TMP5]]
 ; RV32-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; RV32:       [[VECTOR_PH]]:
-; RV32-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
-; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP8]]
-; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; RV32-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
-; RV32-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
-; RV32-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
-; RV32-NEXT:    [[TMP12:%.*]] = sub i32 [[N]], [[DOTCAST]]
 ; RV32-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV32:       [[VECTOR_BODY]]:
-; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[TMP16:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; RV32-NEXT:    [[DOTCAST3:%.*]] = trunc i64 [[INDEX]] to i32
 ; RV32-NEXT:    [[OFFSET_IDX:%.*]] = sub i32 [[N]], [[DOTCAST3]]
 ; RV32-NEXT:    [[TMP13:%.*]] = add nsw i32 [[OFFSET_IDX]], -1
 ; RV32-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
 ; RV32-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP14]]
-; RV32-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP10]] to i32
 ; RV32-NEXT:    [[TMP17:%.*]] = mul i32 0, [[TMP16]]
 ; RV32-NEXT:    [[TMP18:%.*]] = sub i32 [[TMP16]], 1
 ; RV32-NEXT:    [[TMP19:%.*]] = mul i32 -1, [[TMP18]]
-; RV32-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i32 [[TMP17]]
-; RV32-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i32 [[TMP19]]
-; RV32-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP21]], align 4
-; RV32-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]])
+; RV32-NEXT:    [[TMP20:%.*]] = getelementptr float, ptr [[TMP15]], i32 [[TMP17]]
+; RV32-NEXT:    [[TMP28:%.*]] = getelementptr float, ptr [[TMP20]], i32 [[TMP19]]
+; RV32-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP28]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
+; RV32-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
 ; RV32-NEXT:    [[TMP22:%.*]] = fadd <vscale x 4 x float> [[REVERSE]], splat (float 1.000000e+00)
 ; RV32-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP14]]
-; RV32-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP10]] to i32
-; RV32-NEXT:    [[TMP25:%.*]] = mul i32 0, [[TMP24]]
-; RV32-NEXT:    [[TMP26:%.*]] = sub i32 [[TMP24]], 1
+; RV32-NEXT:    [[TMP21:%.*]] = mul i32 0, [[TMP16]]
+; RV32-NEXT:    [[TMP26:%.*]] = sub i32 [[TMP16]], 1
 ; RV32-NEXT:    [[TMP27:%.*]] = mul i32 -1, [[TMP26]]
-; RV32-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i32 [[TMP25]]
-; RV32-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i32 [[TMP27]]
-; RV32-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP22]])
-; RV32-NEXT:    store <vscale x 4 x float> [[REVERSE4]], ptr [[TMP29]], align 4
-; RV32-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
-; RV32-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV32-NEXT:    br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; RV32-NEXT:    [[TMP24:%.*]] = getelementptr float, ptr [[TMP23]], i32 [[TMP21]]
+; RV32-NEXT:    [[TMP25:%.*]] = getelementptr float, ptr [[TMP24]], i32 [[TMP27]]
+; RV32-NEXT:    [[VP_REVERSE3:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[TMP22]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
+; RV32-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_REVERSE3]], ptr align 4 [[TMP25]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
+; RV32-NEXT:    [[TMP29:%.*]] = zext i32 [[TMP16]] to i64
+; RV32-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP29]], [[INDEX]]
+; RV32-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP29]]
+; RV32-NEXT:    [[TMP30:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP0]]
+; RV32-NEXT:    br i1 [[TMP30]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; RV32:       [[MIDDLE_BLOCK]]:
-; RV32-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; RV32-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; RV32-NEXT:    br label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]]
 ; RV32:       [[SCALAR_PH]]:
-; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
-; RV32-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i32 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
+; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; RV32-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i32 [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
 ; RV32-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV32:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
 ; RV32-NEXT:    br label %[[FOR_COND_CLEANUP]]
@@ -643,7 +587,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-UF2:       [[FOR_BODY_PREHEADER]]:
 ; RV64-UF2-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; RV64-UF2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; RV64-UF2-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; RV64-UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; RV64-UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; RV64-UF2:       [[VECTOR_SCEVCHECK]]:
@@ -667,15 +611,13 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-UF2-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP14]], [[TMP13]]
 ; RV64-UF2-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; RV64-UF2:       [[VECTOR_PH]]:
-; RV64-UF2-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
-; RV64-UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP16]]
-; RV64-UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; RV64-UF2-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
 ; RV64-UF2-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 4
 ; RV64-UF2-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 2
+; RV64-UF2-NEXT:    [[N_VEC:%.*]] = urem i64 [[TMP0]], [[TMP19]]
 ; RV64-UF2-NEXT:    [[TMP20:%.*]] = sub i64 [[TMP0]], [[N_VEC]]
-; RV64-UF2-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
+; RV64-UF2-NEXT:    [[TMP48:%.*]] = sub i64 [[TMP0]], [[TMP20]]
+; RV64-UF2-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[TMP20]] to i32
 ; RV64-UF2-NEXT:    [[TMP21:%.*]] = sub i32 [[N]], [[DOTCAST]]
 ; RV64-UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64-UF2:       [[VECTOR_BODY]]:
@@ -717,13 +659,13 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; RV64-UF2-NEXT:    [[REVERSE7:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP36]])
 ; RV64-UF2-NEXT:    store <vscale x 4 x float> [[REVERSE7]], ptr [[TMP47]], align 4
 ; RV64-UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
-; RV64-UF2-NEXT:    [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV64-UF2-NEXT:    br i1 [[TMP48]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; RV64-UF2-NEXT:    [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP20]]
+; RV64-UF2-NEXT:    br i1 [[TMP49]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; RV64-UF2:       [[MIDDLE_BLOCK]]:
-; RV64-UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; RV64-UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[TMP20]]
 ; RV64-UF2-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
 ; RV64-UF2:       [[SCALAR_PH]]:
-; RV64-UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP20]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
+; RV64-UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP48]], %[[MIDDLE_BLOCK]] ], [ [[TMP0]], %[[FOR_BODY_PREHEADER]] ], [ [[TMP0]], %[[VECTOR_SCEVCHECK]] ], [ [[TMP0]], %[[VECTOR_MEMCHECK]] ]
 ; RV64-UF2-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ [[N]], %[[FOR_BODY_PREHEADER]] ], [ [[N]], %[[VECTOR_SCEVCHECK]] ], [ [[N]], %[[VECTOR_MEMCHECK]] ]
 ; RV64-UF2-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV64-UF2:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
@@ -761,99 +703,83 @@ for.body:                                         ; preds = %for.body.preheader,
 define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) {
 ; RV64-LABEL: define void @vector_reverse_f32_simplify(
 ; RV64-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; RV64-NEXT:  [[ENTRY:.*]]:
-; RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
-; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV64-NEXT:  [[ENTRY:.*:]]
+; RV64-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV64:       [[VECTOR_PH]]:
-; RV64-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
-; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
-; RV64-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; RV64-NEXT:    [[TMP6:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV64-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64:       [[VECTOR_BODY]]:
-; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[AVL:%.*]] = phi i64 [ 1023, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV64-NEXT:    [[TMP19:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; RV64-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
 ; RV64-NEXT:    [[TMP7:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV64-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP7]]
-; RV64-NEXT:    [[TMP9:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP24:%.*]] = zext i32 [[TMP19]] to i64
+; RV64-NEXT:    [[TMP9:%.*]] = mul i64 0, [[TMP24]]
+; RV64-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP24]], 1
 ; RV64-NEXT:    [[TMP11:%.*]] = mul i64 -1, [[TMP10]]
-; RV64-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP8]], i64 [[TMP9]]
-; RV64-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[TMP11]]
-; RV64-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
-; RV64-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]])
+; RV64-NEXT:    [[TMP12:%.*]] = getelementptr float, ptr [[TMP8]], i64 [[TMP9]]
+; RV64-NEXT:    [[TMP13:%.*]] = getelementptr float, ptr [[TMP12]], i64 [[TMP11]]
+; RV64-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP19]])
+; RV64-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP19]])
 ; RV64-NEXT:    [[TMP14:%.*]] = fadd <vscale x 4 x float> [[REVERSE]], splat (float 1.000000e+00)
 ; RV64-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]]
-; RV64-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP5]]
-; RV64-NEXT:    [[TMP17:%.*]] = sub i64 [[TMP5]], 1
+; RV64-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP19]] to i64
+; RV64-NEXT:    [[TMP25:%.*]] = mul i64 0, [[TMP16]]
+; RV64-NEXT:    [[TMP17:%.*]] = sub i64 [[TMP16]], 1
 ; RV64-NEXT:    [[TMP18:%.*]] = mul i64 -1, [[TMP17]]
-; RV64-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[TMP16]]
-; RV64-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 [[TMP18]]
-; RV64-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP14]])
-; RV64-NEXT:    store <vscale x 4 x float> [[REVERSE1]], ptr [[TMP20]], align 4
-; RV64-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; RV64-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV64-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; RV64-NEXT:    [[TMP20:%.*]] = getelementptr float, ptr [[TMP15]], i64 [[TMP25]]
+; RV64-NEXT:    [[TMP21:%.*]] = getelementptr float, ptr [[TMP20]], i64 [[TMP18]]
+; RV64-NEXT:    [[VP_REVERSE1:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP19]])
+; RV64-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_REVERSE1]], ptr align 4 [[TMP21]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP19]])
+; RV64-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP19]] to i64
+; RV64-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP22]], [[INDEX]]
+; RV64-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP22]]
+; RV64-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1023
+; RV64-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; RV64:       [[MIDDLE_BLOCK]]:
-; RV64-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1023, [[N_VEC]]
-; RV64-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; RV64-NEXT:    br [[EXIT:label %.*]]
 ; RV64:       [[SCALAR_PH]]:
-; RV64-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
 ; RV64-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV64:       [[FOR_BODY]]:
 ;
 ; RV32-LABEL: define void @vector_reverse_f32_simplify(
 ; RV32-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
-; RV32-NEXT:  [[ENTRY:.*]]:
-; RV32-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; RV32-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
-; RV32-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; RV32-NEXT:  [[ENTRY:.*:]]
+; RV32-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV32:       [[VECTOR_PH]]:
-; RV32-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
-; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
-; RV32-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; RV32-NEXT:    [[TMP6:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV32-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV32:       [[VECTOR_BODY]]:
-; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[AVL:%.*]] = phi i64 [ 1023, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; RV32-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; RV32-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
 ; RV32-NEXT:    [[TMP7:%.*]] = add nsw i64 [[OFFSET_IDX]], -1
 ; RV32-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP7]]
-; RV32-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP5]] to i32
 ; RV32-NEXT:    [[TMP10:%.*]] = mul i32 0, [[TMP9]]
 ; RV32-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP9]], 1
 ; RV32-NEXT:    [[TMP12:%.*]] = mul i32 -1, [[TMP11]]
-; RV32-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP8]], i32 [[TMP10]]
-; RV32-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP13]], i32 [[TMP12]]
-; RV32-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP14]], align 4
-; RV32-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]])
+; RV32-NEXT:    [[TMP13:%.*]] = getelementptr float, ptr [[TMP8]], i32 [[TMP10]]
+; RV32-NEXT:    [[TMP14:%.*]] = getelementptr float, ptr [[TMP13]], i32 [[TMP12]]
+; RV32-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; RV32-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[VP_OP_LOAD]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
 ; RV32-NEXT:    [[TMP15:%.*]] = fadd <vscale x 4 x float> [[REVERSE]], splat (float 1.000000e+00)
 ; RV32-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]]
-; RV32-NEXT:    [[TMP17:%.*]] = trunc i64 [[TMP5]] to i32
-; RV32-NEXT:    [[TMP18:%.*]] = mul i32 0, [[TMP17]]
-; RV32-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP17]], 1
+; RV32-NEXT:    [[TMP17:%.*]] = mul i32 0, [[TMP9]]
+; RV32-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP9]], 1
 ; RV32-NEXT:    [[TMP20:%.*]] = mul i32 -1, [[TMP19]]
-; RV32-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 [[TMP18]]
-; RV32-NEXT:    [[TMP22:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i32 [[TMP20]]
-; RV32-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP15]])
-; RV32-NEXT:    store <vscale x 4 x float> [[REVERSE1]], ptr [[TMP22]], align 4
-; RV32-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; RV32-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV32-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; RV32-NEXT:    [[TMP18:%.*]] = getelementptr float, ptr [[TMP16]], i32 [[TMP17]]
+; RV32-NEXT:    [[TMP22:%.*]] = getelementptr float, ptr [[TMP18]], i32 [[TMP20]]
+; RV32-NEXT:    [[VP_REVERSE1:%.*]] = call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; RV32-NEXT:    call void @llvm.vp.store.nxv4f32.p0(<vscale x 4 x float> [[VP_REVERSE1]], ptr align 4 [[TMP22]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; RV32-NEXT:    [[TMP23:%.*]] = zext i32 [[TMP9]] to i64
+; RV32-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP23]], [[INDEX]]
+; RV32-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP23]]
+; RV32-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1023
+; RV32-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; RV32:       [[MIDDLE_BLOCK]]:
-; RV32-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1023, [[N_VEC]]
-; RV32-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; RV32-NEXT:    br [[EXIT:label %.*]]
 ; RV32:       [[SCALAR_PH]]:
-; RV32-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
 ; RV32-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV32:       [[FOR_BODY]]:
 ;
@@ -861,18 +787,16 @@ define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
 ; RV64-UF2-NEXT:  [[ENTRY:.*]]:
 ; RV64-UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; RV64-UF2-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; RV64-UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
 ; RV64-UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV64-UF2:       [[VECTOR_PH]]:
-; RV64-UF2-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; RV64-UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
-; RV64-UF2-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
 ; RV64-UF2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; RV64-UF2-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; RV64-UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; RV64-UF2-NEXT:    [[N_VEC:%.*]] = urem i64 1023, [[TMP6]]
 ; RV64-UF2-NEXT:    [[TMP7:%.*]] = sub i64 1023, [[N_VEC]]
+; RV64-UF2-NEXT:    [[TMP33:%.*]] = sub i64 1023, [[TMP7]]
 ; RV64-UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64-UF2:       [[VECTOR_BODY]]:
 ; RV64-UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -911,13 +835,13 @@ define void @vector_reverse_f32_simplify(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x float> @llvm.vector.reverse.nxv4f32(<vscale x 4 x float> [[TMP21]])
 ; RV64-UF2-NEXT:    store <vscale x 4 x float> [[REVERSE4]], ptr [[TMP32]], align 4
 ; RV64-UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; RV64-UF2-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; RV64-UF2-NEXT:    br i1 [[TMP33]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; RV64-UF2-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[TMP7]]
+; RV64-UF2-NEXT:    br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; RV64-UF2:       [[MIDDLE_BLOCK]]:
-; RV64-UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1023, [[N_VEC]]
+; RV64-UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1023, [[TMP7]]
 ; RV64-UF2-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
 ; RV64-UF2:       [[SCALAR_PH]]:
-; RV64-UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
+; RV64-UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP33]], %[[MIDDLE_BLOCK]] ], [ 1023, %[[ENTRY]] ]
 ; RV64-UF2-NEXT:    br label %[[FOR_BODY:.*]]
 ; RV64-UF2:       [[FOR_BODY]]:
 ;
@@ -984,7 +908,7 @@ define void @vector_reverse_irregular_type(ptr noalias %A, ptr noalias %B) {
 ; RV64-NEXT:    store i7 [[TMP28]], ptr [[TMP24]], align 1
 ; RV64-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; RV64-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1020
-; RV64-NEXT:    br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; RV64-NEXT:    br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; RV64:       [[MIDDLE_BLOCK]]:
 ; RV64-NEXT:    br label %[[SCALAR_PH]]
 ; RV64:       [[SCALAR_PH]]:
@@ -1036,7 +960,7 @@ define void @vector_reverse_irregular_type(ptr noalias %A, ptr noalias %B) {
 ; RV32-NEXT:    store i7 [[TMP28]], ptr [[TMP24]], align 1
 ; RV32-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; RV32-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1020
-; RV32-NEXT:    br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; RV32-NEXT:    br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; RV32:       [[MIDDLE_BLOCK]]:
 ; RV32-NEXT:    br label %[[SCALAR_PH]]
 ; RV32:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
index 3370e921b089b..e0436e8eb0022 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
@@ -10,36 +10,29 @@ target triple = "riscv64"
 define void @test(ptr %p) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 200, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 32 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 200
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP10]], align 32
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP_LOAD]], ptr align 32 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
 ; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 200, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; CHECK-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 200
@@ -47,7 +40,7 @@ define void @test(ptr %p) {
 ; CHECK-NEXT:    store i64 [[V]], ptr [[A2]], align 32
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IV]], 199
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -86,14 +79,13 @@ define void @test_may_clobber(ptr %p) {
 ; CHECK-NEXT:    store <4 x i64> [[WIDE_LOAD]], ptr [[TMP4]], align 32
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
-; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; CHECK-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 100
@@ -101,7 +93,7 @@ define void @test_may_clobber(ptr %p) {
 ; CHECK-NEXT:    store i64 [[V]], ptr [[A2]], align 32
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IV]], 199
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -127,36 +119,29 @@ exit:
 define void @trivial_due_max_vscale(ptr %p) {
 ; CHECK-LABEL: @trivial_due_max_vscale(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 200, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 32 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 8192
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP10]], align 32
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP_LOAD]], ptr align 32 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 200, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; CHECK-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 8192
@@ -164,7 +149,7 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; CHECK-NEXT:    store i64 [[V]], ptr [[A2]], align 32
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IV]], 199
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -190,36 +175,29 @@ exit:
 define void @no_high_lmul_or_interleave(ptr %p) {
 ; CHECK-LABEL: @no_high_lmul_or_interleave(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 200, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP7]], align 32
+; CHECK-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 32 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 1024
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP10]], align 32
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[VP_OP_LOAD]], ptr align 32 [[TMP10]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP8]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP8]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 200
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 200, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; CHECK-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 1024
@@ -227,7 +205,7 @@ define void @no_high_lmul_or_interleave(ptr %p) {
 ; CHECK-NEXT:    store i64 [[V]], ptr [[A2]], align 32
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i64 [[IV]], 199
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -277,7 +255,7 @@ define void @safe_load_store_distance_not_pow_of_2(i64 %N) {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 24)
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -291,7 +269,7 @@ define void @safe_load_store_distance_not_pow_of_2(i64 %N) {
 ; CHECK-NEXT:    store i16 0, ptr [[GEP_OFF]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[IV]], [[N]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[LOOP]], label [[EXIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll
index e51f6fa7484c8..a200d8e2d5ded 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll
@@ -8,44 +8,37 @@
 define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @vector_add(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = add <vscale x 2 x i64> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP8]], ptr [[TMP6]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP8]], ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[ELEM]], [[V]]
 ; CHECK-NEXT:    store i64 [[ADD]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -72,44 +65,37 @@ for.end:
 define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) {
 ; CHECK-LABEL: @vector_add_i32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    store <vscale x 4 x i32> [[TMP8]], ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP8]], ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[ELEM]], [[V]]
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -174,44 +160,37 @@ for.end:
 define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
 ; CHECK-LABEL: @indexed_store(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], <vscale x 2 x i64> [[WIDE_LOAD]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x ptr> [[TMP8]], i32 8, <vscale x 2 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], <vscale x 2 x ptr> align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8
 ; CHECK-NEXT:    [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AIDX]]
 ; CHECK-NEXT:    store i64 [[V]], ptr [[AADDR]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -235,40 +214,33 @@ for.end:
 define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
 ; CHECK-LABEL: @indexed_load(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.vp.load.nxv2i64.p0(ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], <vscale x 2 x i64> [[WIDE_LOAD]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> [[TMP8]], i32 8, <vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> poison)
-; CHECK-NEXT:    [[TMP9]] = add <vscale x 2 x i64> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 2 x i64> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT:    [[TMP9]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> splat (i1 true), <vscale x 2 x i64> [[TMP13]], <vscale x 2 x i64> [[VEC_PHI]], i32 [[TMP12]])
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP9]])
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8
 ; CHECK-NEXT:    [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AIDX]]
@@ -276,7 +248,7 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[SUM_NEXT]] = add i64 [[SUM]], [[ELEM]]
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi i64 [ [[SUM_NEXT]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[SUM_NEXT_LCSSA]]
@@ -303,40 +275,33 @@ for.end:
 define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @splat_int(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -358,40 +323,33 @@ for.end:
 define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) {
 ; CHECK-LABEL: @splat_ptr(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 2 x ptr> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv2p0.p0(<vscale x 2 x ptr> [[BROADCAST_SPLAT]], ptr align 8 [[TMP6]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP10]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    store ptr [[V]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
index ed507961ef825..4d1f4ce2aeb39 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
@@ -11,14 +11,6 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -38,10 +30,9 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[ELEM]], [[V]]
@@ -76,14 +67,6 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -103,10 +86,9 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8
 ; CHECK-NEXT:    [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AIDX]]
@@ -139,14 +121,6 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -168,12 +142,10 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP11]])
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[BADDR:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[AIDX:%.*]] = load i64, ptr [[BADDR]], align 8
 ; CHECK-NEXT:    [[AADDR:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AIDX]]
@@ -210,14 +182,6 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -235,10 +199,9 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -267,14 +230,6 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -293,10 +248,9 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    store i64 [[V]], ptr [[B]], align 8
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
@@ -326,7 +280,7 @@ define i64 @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; CHECK-LABEL: @uniform_load(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK:       vector.ph:
@@ -334,8 +288,6 @@ define i64 @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -344,7 +296,7 @@ define i64 @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[IV]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP5]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       middle.block:
@@ -387,14 +339,6 @@ define void @vector_add_trip1024(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP4]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -414,10 +358,9 @@ define void @vector_add_trip1024(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[ELEM:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[ELEM]], [[V]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll
index 5c6febc41da2d..acdfa72e8fd07 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll
@@ -4,51 +4,43 @@
 define i32 @select_icmp(i32 %x, i32 %y, ptr nocapture readonly %c, i64 %n) {
 ; CHECK-LABEL: define i32 @select_icmp(
 ; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr readonly captures(none) [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp sge <vscale x 4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP8]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP7]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP8]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = freeze i1 [[TMP10]]
 ; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP11]], i32 [[Y]], i32 0
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[A:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 [[TMP12]], [[X]]
 ; CHECK-NEXT:    [[COND]] = select i1 [[CMP1]], i32 [[A]], i32 [[Y]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[COND_LCSSA]]
@@ -74,51 +66,43 @@ for.end:
 define i32 @select_fcmp(float %x, i32 %y, ptr nocapture readonly %c, i64 %n) {
 ; CHECK-LABEL: define i32 @select_fcmp(
 ; CHECK-SAME: float [[X:%.*]], i32 [[Y:%.*]], ptr readonly captures(none) [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast uge <vscale x 4 x float> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP8]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP7]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP8]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = freeze i1 [[TMP10]]
 ; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP11]], i32 [[Y]], i32 0
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[A:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[COND:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP1:%.*]] = fcmp fast olt float [[TMP12]], [[X]]
 ; CHECK-NEXT:    [[COND]] = select i1 [[CMP1]], i32 [[A]], i32 [[Y]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[COND_LCSSA:%.*]] = phi i32 [ [[COND]], %[[FOR_BODY]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[COND_LCSSA]]
@@ -144,49 +128,41 @@ for.end:
 define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) {
 ; CHECK-LABEL: define i32 @select_const_i32_from_icmp(
 ; CHECK-SAME: ptr readonly captures(none) [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP21]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 3)
-; CHECK-NEXT:    [[TMP8]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP7]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP21]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP21]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP8]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = freeze i1 [[TMP10]]
 ; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP11]], i32 7, i32 3
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 3, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[TMP18:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP13:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[TMP17:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[TMP18:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi i32 [ 3, %[[SCALAR_PH]] ], [ [[TMP17:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP12]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 3
 ; CHECK-NEXT:    [[TMP17]] = select i1 [[TMP16]], i32 [[TMP13]], i32 7
 ; CHECK-NEXT:    [[TMP18]] = add nuw nsw i64 [[TMP12]], 1
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[TMP18]], [[N]]
-; CHECK-NEXT:    br i1 [[TMP19]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], %[[FOR_BODY]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[DOTLCSSA]]
@@ -212,49 +188,41 @@ exit:                                     ; preds = %for.body
 define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 %n) {
 ; CHECK-LABEL: define i32 @select_i32_from_icmp(
 ; CHECK-SAME: ptr readonly captures(none) [[V:%.*]], i32 [[A:%.*]], i32 [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP21]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 3)
-; CHECK-NEXT:    [[TMP8]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP7]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP21]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP21]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP8]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = freeze i1 [[TMP10]]
 ; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP11]], i32 [[B]], i32 [[A]]
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[A]], %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[TMP18:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP13:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[TMP17:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[TMP18:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi i32 [ [[A]], %[[SCALAR_PH]] ], [ [[TMP17:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP12]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 3
 ; CHECK-NEXT:    [[TMP17]] = select i1 [[TMP16]], i32 [[TMP13]], i32 [[B]]
 ; CHECK-NEXT:    [[TMP18]] = add nuw nsw i64 [[TMP12]], 1
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[TMP18]], [[N]]
-; CHECK-NEXT:    br i1 [[TMP19]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], %[[FOR_BODY]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[DOTLCSSA]]
@@ -280,49 +248,41 @@ exit:                                     ; preds = %for.body
 define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) {
 ; CHECK-LABEL: define i32 @select_const_i32_from_fcmp(
 ; CHECK-SAME: ptr readonly captures(none) [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP21]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast one <vscale x 4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
-; CHECK-NEXT:    [[TMP8]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP7]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    [[TMP8]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP21]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP21]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP8]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = freeze i1 [[TMP10]]
 ; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP11]], i32 1, i32 2
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 2, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[TMP18:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP13:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[TMP17:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[TMP18:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi i32 [ 2, %[[SCALAR_PH]] ], [ [[TMP17:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[TMP12]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = load float, ptr [[TMP14]], align 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = fcmp fast ueq float [[TMP15]], 3.000000e+00
 ; CHECK-NEXT:    [[TMP17]] = select i1 [[TMP16]], i32 [[TMP13]], i32 1
 ; CHECK-NEXT:    [[TMP18]] = add nuw nsw i64 [[TMP12]], 1
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[TMP18]], [[N]]
-; CHECK-NEXT:    br i1 [[TMP19]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP19]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi i32 [ [[TMP17]], %[[FOR_BODY]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[DOTLCSSA]]
@@ -385,46 +345,44 @@ exit:                                     ; preds = %for.body
 define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i64 %n) {
 ; CHECK-LABEL: define i32 @pred_select_const_i32_from_icmp(
 ; CHECK-SAME: ptr noalias readonly captures(none) [[SRC1:%.*]], ptr noalias readonly captures(none) [[SRC2:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP17]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP18:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; CHECK-NEXT:    [[TMP19:%.*]] = icmp ult <vscale x 4 x i32> [[TMP18]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP17]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 35)
+; CHECK-NEXT:    [[TMP20:%.*]] = select <vscale x 4 x i1> [[TMP19]], <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP8]], i32 4, <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> poison)
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP8]], <vscale x 4 x i1> [[TMP7]], i32 [[TMP17]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP10:%.*]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP9]]
-; CHECK-NEXT:    [[PREDPHI]] = select <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i1> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[PREDPHI1:%.*]] = select <vscale x 4 x i1> [[TMP20]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i1> [[VEC_PHI]]
+; CHECK-NEXT:    [[PREDPHI]] = call <vscale x 4 x i1> @llvm.vp.merge.nxv4i1(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i1> [[PREDPHI1]], <vscale x 4 x i1> [[VEC_PHI]], i32 [[TMP17]])
+; CHECK-NEXT:    [[TMP21:%.*]] = zext i32 [[TMP17]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP21]]
+; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[PREDPHI]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = freeze i1 [[TMP12]]
 ; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP13]], i32 1, i32 0
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[FOR_END_LOOPEXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[I_013:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_INC:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT:    [[R_012:%.*]] = phi i32 [ [[R_1:%.*]], %[[FOR_INC]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_013:%.*]] = phi i64 [ [[INC:%.*]], %[[FOR_INC:.*]] ], [ 0, %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[R_012:%.*]] = phi i32 [ [[R_1:%.*]], %[[FOR_INC]] ], [ 0, %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[I_013]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP14]], 35
@@ -439,7 +397,7 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1
 ; CHECK-NEXT:    [[R_1]] = phi i32 [ [[R_012]], %[[FOR_BODY]] ], [ [[SPEC_SELECT]], %[[IF_THEN]] ]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i64 [[I_013]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; CHECK:       [[FOR_END_LOOPEXIT]]:
 ; CHECK-NEXT:    [[R_1_LCSSA:%.*]] = phi i32 [ [[R_1]], %[[FOR_INC]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[R_1_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
index 13a4b166431c8..8c804e5e62f5c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll
@@ -4,28 +4,16 @@
 define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_range(4,1024) {
 ; CHECK-LABEL: @small_trip_count_min_vlen_128(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4
-; CHECK-NEXT:    br label [[MIDDLE_BLOCK:%.*]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP1]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 [[IV]]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP1]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i32 [[IV]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[V]], 1
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[IV]], 3
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[LOOP1]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -49,28 +37,16 @@ exit:
 define void @small_trip_count_min_vlen_32(ptr nocapture %a) nounwind vscale_range(1,1024) {
 ; CHECK-LABEL: @small_trip_count_min_vlen_32(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[TMP1]], align 4
-; CHECK-NEXT:    br label [[MIDDLE_BLOCK:%.*]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP1]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 [[IV]]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP1]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1:%.*]], i32 [[IV]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[V]], 1
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[IV]], 3
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[LOOP1]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 9e492c62a5577..47979d358690a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -6,45 +6,37 @@
 define void @single_constant_stride_int_scaled(ptr %p) {
 ; CHECK-LABEL: @single_constant_stride_int_scaled(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH1:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[TMP5]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP7]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP12]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw nsw <vscale x 4 x i64> [[VEC_IND]], splat (i64 8)
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[P:%.*]], <vscale x 4 x i64> [[TMP14]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP15]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
 ; CHECK-NEXT:    [[TMP16:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP16]], <vscale x 4 x ptr> [[TMP15]], i32 4, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP16]], <vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
 ; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[SCALAR_PH:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH1]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[OFFSET:%.*]] = mul nuw nsw i64 [[I]], 8
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
@@ -52,7 +44,7 @@ define void @single_constant_stride_int_scaled(ptr %p) {
 ; CHECK-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[SCALAR_PH]], label [[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -77,46 +69,38 @@ exit:
 define void @single_constant_stride_int_iv(ptr %p) {
 ; CHECK-LABEL: @single_constant_stride_int_iv(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 64
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 64)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP8]]
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 64, [[TMP5]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 64, [[TMP11]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[P:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP12]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> [[TMP12]], i32 4, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP13]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP7]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP14]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[OFFSET:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[OFFSET_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[OFFSET:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[OFFSET_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[Q0:%.*]] = getelementptr i32, ptr [[P]], i64 [[OFFSET]]
 ; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[Q0]], align 4
 ; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
@@ -124,7 +108,7 @@ define void @single_constant_stride_int_iv(ptr %p) {
 ; CHECK-NEXT:    [[OFFSET_NEXT]] = add nuw nsw i64 [[OFFSET]], 64
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -151,55 +135,42 @@ exit:
 define void @single_constant_stride_ptr_iv(ptr %p) {
 ; CHECK-LABEL: @single_constant_stride_ptr_iv(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 1024, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH1:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[N_VEC]], 8
-; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[TMP18]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 8, [[TMP8]]
+; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[P:%.*]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul <vscale x 4 x i64> [[TMP14]], splat (i64 8)
 ; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i64> [[TMP16]]
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <vscale x 4 x ptr> [[VECTOR_GEP]], i32 0
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <vscale x 8 x i32>, ptr [[TMP17]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32> [[WIDE_VEC]])
-; CHECK-NEXT:    [[TMP19:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[TMP19:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[VECTOR_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
 ; CHECK-NEXT:    [[TMP20:%.*]] = add <vscale x 4 x i32> [[TMP19]], splat (i32 1)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x ptr> [[VECTOR_GEP]], i32 4, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x ptr> align 4 [[VECTOR_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
+; CHECK-NEXT:    [[TMP9:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP9]], [[INDEX]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 8, [[TMP10]]
 ; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    br label [[SCALAR_PH]]
+; CHECK-NEXT:    br label [[SCALAR_PH:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P]], [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[PTR_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH1]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[P]], [[SCALAR_PH1]] ], [ [[PTR_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[X0:%.*]] = load i32, ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[Y0:%.*]] = add i32 [[X0]], 1
 ; CHECK-NEXT:    store i32 [[Y0]], ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i8, ptr [[PTR]], i64 8
 ; CHECK-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; CHECK-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; CHECK-NEXT:    br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[DONE]], label [[SCALAR_PH]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -225,36 +196,29 @@ exit:
 define void @single_stride_int_scaled(ptr %p, i64 %stride) {
 ; NOSTRIDED-LABEL: @single_stride_int_scaled(
 ; NOSTRIDED-NEXT:  entry:
-; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
-; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; NOSTRIDED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
 ; NOSTRIDED:       vector.scevcheck:
 ; NOSTRIDED-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
 ; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NOSTRIDED:       vector.ph:
-; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NOSTRIDED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NOSTRIDED:       vector.body:
 ; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; NOSTRIDED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[INDEX]]
-; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
 ; NOSTRIDED-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; NOSTRIDED-NEXT:    store <vscale x 4 x i32> [[TMP10]], ptr [[TMP8]], align 4
-; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; NOSTRIDED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NOSTRIDED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; NOSTRIDED-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP10]], ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; NOSTRIDED-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
+; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; NOSTRIDED-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; NOSTRIDED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; NOSTRIDED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; NOSTRIDED:       middle.block:
-; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NOSTRIDED-NEXT:    br label [[EXIT:%.*]]
 ; NOSTRIDED:       scalar.ph:
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
 ; NOSTRIDED:       loop:
 ; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -265,7 +229,7 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) {
 ; NOSTRIDED-NEXT:    store i32 [[Y0]], ptr [[Q0]], align 4
 ; NOSTRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; NOSTRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
 ; NOSTRIDED:       exit:
 ; NOSTRIDED-NEXT:    ret void
 ;
@@ -306,37 +270,30 @@ exit:
 define void @single_stride_int_iv(ptr %p, i64 %stride) {
 ; NOSTRIDED-LABEL: @single_stride_int_iv(
 ; NOSTRIDED-NEXT:  entry:
-; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
-; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; NOSTRIDED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
 ; NOSTRIDED:       vector.scevcheck:
 ; NOSTRIDED-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
 ; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NOSTRIDED:       vector.ph:
-; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NOSTRIDED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NOSTRIDED:       vector.body:
 ; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; NOSTRIDED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[INDEX]]
-; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
 ; NOSTRIDED-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; NOSTRIDED-NEXT:    store <vscale x 4 x i32> [[TMP10]], ptr [[TMP8]], align 4
-; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; NOSTRIDED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NOSTRIDED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; NOSTRIDED-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP10]], ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; NOSTRIDED-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
+; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; NOSTRIDED-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; NOSTRIDED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; NOSTRIDED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; NOSTRIDED:       middle.block:
-; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NOSTRIDED-NEXT:    br label [[EXIT:%.*]]
 ; NOSTRIDED:       scalar.ph:
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
 ; NOSTRIDED:       loop:
 ; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -348,7 +305,7 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) {
 ; NOSTRIDED-NEXT:    [[OFFSET_NEXT]] = add nuw nsw i64 [[OFFSET]], [[STRIDE]]
 ; NOSTRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; NOSTRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
 ; NOSTRIDED:       exit:
 ; NOSTRIDED-NEXT:    ret void
 ;
@@ -429,11 +386,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-NEXT:  entry:
 ; NOSTRIDED-NEXT:    [[P3:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; NOSTRIDED-NEXT:    [[P21:%.*]] = ptrtoint ptr [[P2:%.*]] to i64
-; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
-; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; NOSTRIDED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
 ; NOSTRIDED:       vector.scevcheck:
 ; NOSTRIDED-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
 ; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
@@ -445,28 +398,25 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
 ; NOSTRIDED-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NOSTRIDED:       vector.ph:
-; NOSTRIDED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
-; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP8]]
-; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NOSTRIDED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NOSTRIDED:       vector.body:
 ; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[TMP16:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; NOSTRIDED-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
-; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP12]], align 4
+; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
 ; NOSTRIDED-NEXT:    [[TMP14:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 1)
 ; NOSTRIDED-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[P2]], i64 [[INDEX]]
-; NOSTRIDED-NEXT:    store <vscale x 4 x i32> [[TMP14]], ptr [[TMP15]], align 4
-; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
-; NOSTRIDED-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NOSTRIDED-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; NOSTRIDED-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP14]], ptr align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP16]])
+; NOSTRIDED-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP16]] to i64
+; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP13]], [[INDEX]]
+; NOSTRIDED-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; NOSTRIDED-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; NOSTRIDED-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; NOSTRIDED:       middle.block:
-; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NOSTRIDED-NEXT:    br label [[EXIT:%.*]]
 ; NOSTRIDED:       scalar.ph:
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
 ; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
 ; NOSTRIDED:       loop:
 ; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -478,17 +428,13 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-NEXT:    store i32 [[Y0]], ptr [[Q1]], align 4
 ; NOSTRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; NOSTRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
 ; NOSTRIDED:       exit:
 ; NOSTRIDED-NEXT:    ret void
 ;
 ; STRIDED-LABEL: @double_stride_int_scaled(
 ; STRIDED-NEXT:  entry:
-; STRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; STRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 80, i64 [[TMP1]])
-; STRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; STRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; STRIDED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
 ; STRIDED:       vector.scevcheck:
 ; STRIDED-NEXT:    [[TMP24:%.*]] = shl i64 [[STRIDE:%.*]], 2
 ; STRIDED-NEXT:    [[TMP25:%.*]] = mul i64 [[STRIDE]], -4
@@ -537,39 +483,36 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; STRIDED-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; STRIDED:       vector.ph:
-; STRIDED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
-; STRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP9]]
-; STRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; STRIDED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
 ; STRIDED-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; STRIDED-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; STRIDED-NEXT:    [[TMP14:%.*]] = mul <vscale x 4 x i64> [[TMP12]], splat (i64 1)
 ; STRIDED-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP14]]
-; STRIDED-NEXT:    [[TMP17:%.*]] = mul i64 1, [[TMP11]]
-; STRIDED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP17]], i64 0
-; STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; STRIDED:       vector.body:
 ; STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; STRIDED-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; STRIDED-NEXT:    [[TMP43:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; STRIDED-NEXT:    [[TMP44:%.*]] = zext i32 [[TMP43]] to i64
+; STRIDED-NEXT:    [[BROADCAST_SPLATINSERT9:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP44]], i64 0
+; STRIDED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT9]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; STRIDED-NEXT:    [[TMP18:%.*]] = mul nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT1]]
 ; STRIDED-NEXT:    [[TMP19:%.*]] = getelementptr i32, ptr [[P]], <vscale x 4 x i64> [[TMP18]]
-; STRIDED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP19]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison), !alias.scope [[META8:![0-9]+]]
+; STRIDED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP19]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP43]]), !alias.scope [[META9:![0-9]+]]
 ; STRIDED-NEXT:    [[TMP20:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
 ; STRIDED-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[P2]], <vscale x 4 x i64> [[TMP18]]
-; STRIDED-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x ptr> [[TMP21]], i32 4, <vscale x 4 x i1> splat (i1 true)), !alias.scope [[META11:![0-9]+]], !noalias [[META8]]
-; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
+; STRIDED-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x ptr> align 4 [[TMP21]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP43]]), !alias.scope [[META12:![0-9]+]], !noalias [[META9]]
+; STRIDED-NEXT:    [[TMP46:%.*]] = zext i32 [[TMP43]] to i64
+; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP46]], [[INDEX]]
+; STRIDED-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP46]]
 ; STRIDED-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; STRIDED-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; STRIDED-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; STRIDED-NEXT:    [[TMP41:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; STRIDED-NEXT:    br i1 [[TMP41]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; STRIDED:       middle.block:
-; STRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; STRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; STRIDED-NEXT:    br label [[EXIT:%.*]]
 ; STRIDED:       scalar.ph:
-; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ], [ 0, [[VECTOR_MEMCHECK1]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK1]] ]
 ; STRIDED-NEXT:    br label [[LOOP:%.*]]
 ; STRIDED:       loop:
 ; STRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -581,7 +524,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    store i32 [[Y0]], ptr [[Q1]], align 4
 ; STRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; STRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
+; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
 ; STRIDED:       exit:
 ; STRIDED-NEXT:    ret void
 ;
@@ -607,37 +550,30 @@ exit:
 define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-LABEL: @double_stride_int_iv(
 ; NOSTRIDED-NEXT:  entry:
-; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
-; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
+; NOSTRIDED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
 ; NOSTRIDED:       vector.scevcheck:
 ; NOSTRIDED-NEXT:    [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1
 ; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NOSTRIDED:       vector.ph:
-; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
-; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
-; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NOSTRIDED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NOSTRIDED:       vector.body:
 ; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NOSTRIDED-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; NOSTRIDED-NEXT:    [[TMP8:%.*]] = getelementptr i32, ptr [[P:%.*]], i64 [[INDEX]]
-; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
+; NOSTRIDED-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
 ; NOSTRIDED-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 1)
-; NOSTRIDED-NEXT:    store <vscale x 4 x i32> [[TMP10]], ptr [[TMP8]], align 4
-; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; NOSTRIDED-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; NOSTRIDED-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; NOSTRIDED-NEXT:    call void @llvm.vp.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP10]], ptr align 4 [[TMP8]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP7]])
+; NOSTRIDED-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
+; NOSTRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; NOSTRIDED-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; NOSTRIDED-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; NOSTRIDED-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; NOSTRIDED:       middle.block:
-; NOSTRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; NOSTRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NOSTRIDED-NEXT:    br label [[EXIT:%.*]]
 ; NOSTRIDED:       scalar.ph:
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
-; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; NOSTRIDED-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; NOSTRIDED-NEXT:    br label [[LOOP:%.*]]
 ; NOSTRIDED:       loop:
 ; NOSTRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -650,7 +586,7 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-NEXT:    [[OFFSET_NEXT]] = add nuw nsw i64 [[OFFSET]], [[STRIDE]]
 ; NOSTRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; NOSTRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
+; NOSTRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
 ; NOSTRIDED:       exit:
 ; NOSTRIDED-NEXT:    ret void
 ;
@@ -692,7 +628,6 @@ exit:
   ret void
 }
 
-
 define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-LABEL: @double_stride_ptr_iv(
 ; NOSTRIDED-NEXT:  entry:
@@ -714,11 +649,7 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ;
 ; STRIDED-LABEL: @double_stride_ptr_iv(
 ; STRIDED-NEXT:  entry:
-; STRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; STRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
-; STRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
-; STRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; STRIDED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; STRIDED:       vector.memcheck:
 ; STRIDED-NEXT:    [[TMP3:%.*]] = mul i64 [[STRIDE:%.*]], 1023
 ; STRIDED-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[P2:%.*]], i64 [[TMP3]]
@@ -738,46 +669,41 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; STRIDED-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; STRIDED:       vector.ph:
-; STRIDED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
-; STRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP9]]
-; STRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; STRIDED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
-; STRIDED-NEXT:    [[TMP10:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
-; STRIDED-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP10]]
-; STRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
-; STRIDED-NEXT:    [[IND_END7:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP11]]
 ; STRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; STRIDED:       vector.body:
 ; STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[POINTER_PHI11:%.*]] = phi ptr [ [[P2]], [[VECTOR_PH]] ], [ [[PTR_IND12:%.*]], [[VECTOR_BODY]] ]
-; STRIDED-NEXT:    [[TMP17:%.*]] = mul i64 [[STRIDE]], [[TMP13]]
+; STRIDED-NEXT:    [[AVL:%.*]] = phi i64 [ 1024, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[TMP19:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; STRIDED-NEXT:    [[DOTSPLATINSERT9:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
 ; STRIDED-NEXT:    [[DOTSPLAT10:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT9]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; STRIDED-NEXT:    [[TMP18:%.*]] = mul <vscale x 4 x i64> [[TMP19]], [[DOTSPLAT10]]
 ; STRIDED-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI11]], <vscale x 4 x i64> [[TMP18]]
-; STRIDED-NEXT:    [[TMP25:%.*]] = mul i64 [[STRIDE]], [[TMP13]]
 ; STRIDED-NEXT:    [[TMP27:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; STRIDED-NEXT:    [[TMP21:%.*]] = mul <vscale x 4 x i64> [[TMP27]], [[DOTSPLAT10]]
 ; STRIDED-NEXT:    [[VECTOR_GEP7:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i64> [[TMP21]]
-; STRIDED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[VECTOR_GEP7]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison), !alias.scope [[META15:![0-9]+]]
+; STRIDED-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; STRIDED-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[VECTOR_GEP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]]), !alias.scope [[META16:![0-9]+]]
 ; STRIDED-NEXT:    [[TMP30:%.*]] = add <vscale x 4 x i32> [[WIDE_MASKED_GATHER]], splat (i32 1)
-; STRIDED-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP30]], <vscale x 4 x ptr> [[VECTOR_GEP]], i32 4, <vscale x 4 x i1> splat (i1 true)), !alias.scope [[META18:![0-9]+]], !noalias [[META15]]
-; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]]
+; STRIDED-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP30]], <vscale x 4 x ptr> align 4 [[VECTOR_GEP]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]]), !alias.scope [[META19:![0-9]+]], !noalias [[META16]]
+; STRIDED-NEXT:    [[TMP16:%.*]] = zext i32 [[TMP14]] to i64
+; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP16]], [[INDEX]]
+; STRIDED-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]
+; STRIDED-NEXT:    [[TMP20:%.*]] = zext i32 [[TMP14]] to i64
+; STRIDED-NEXT:    [[TMP25:%.*]] = mul i64 [[STRIDE]], [[TMP20]]
 ; STRIDED-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP25]]
+; STRIDED-NEXT:    [[TMP22:%.*]] = zext i32 [[TMP14]] to i64
+; STRIDED-NEXT:    [[TMP17:%.*]] = mul i64 [[STRIDE]], [[TMP22]]
 ; STRIDED-NEXT:    [[PTR_IND12]] = getelementptr i8, ptr [[POINTER_PHI11]], i64 [[TMP17]]
-; STRIDED-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; STRIDED-NEXT:    br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; STRIDED-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; STRIDED-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; STRIDED:       middle.block:
-; STRIDED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
-; STRIDED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; STRIDED-NEXT:    br label [[EXIT:%.*]]
 ; STRIDED:       scalar.ph:
-; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
-; STRIDED-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[P]], [[ENTRY]] ], [ [[P]], [[VECTOR_MEMCHECK]] ]
-; STRIDED-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[IND_END7]], [[MIDDLE_BLOCK]] ], [ [[P2]], [[ENTRY]] ], [ [[P2]], [[VECTOR_MEMCHECK]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[P]], [[ENTRY]] ], [ [[P]], [[VECTOR_MEMCHECK]] ]
+; STRIDED-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[P2]], [[ENTRY]] ], [ [[P2]], [[VECTOR_MEMCHECK]] ]
 ; STRIDED-NEXT:    br label [[LOOP:%.*]]
 ; STRIDED:       loop:
 ; STRIDED-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ]
@@ -790,7 +716,7 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    [[PTR2_NEXT]] = getelementptr inbounds i8, ptr [[PTR2]], i64 [[STRIDE]]
 ; STRIDED-NEXT:    [[NEXTI]] = add i64 [[I]], 1
 ; STRIDED-NEXT:    [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024
-; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
+; STRIDED-NEXT:    br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
 ; STRIDED:       exit:
 ; STRIDED-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
index 38e78321a4f36..e6ed26cfa1109 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-bin-unary-ops-args.ll
@@ -22,14 +22,6 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -68,8 +60,8 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -84,8 +76,6 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -94,7 +84,7 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = and <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -148,14 +138,6 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -194,8 +176,8 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -210,8 +192,6 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -220,7 +200,7 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = or <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -274,14 +254,6 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -320,8 +292,8 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -336,8 +308,6 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -346,7 +316,7 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = xor <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -400,14 +370,6 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -446,8 +408,8 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -462,8 +424,6 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -472,7 +432,7 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = shl <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -526,14 +486,6 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -572,8 +524,8 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -588,8 +540,6 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -598,7 +548,7 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = lshr <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -652,14 +602,6 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -698,8 +640,8 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -714,8 +656,6 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -724,7 +664,7 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = ashr <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -778,14 +718,6 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -824,8 +756,8 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -840,8 +772,6 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -850,7 +780,7 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = add <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -904,14 +834,6 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -950,8 +872,8 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -966,8 +888,6 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -976,7 +896,7 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = sub <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 1)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1030,14 +950,6 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1076,8 +988,8 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1092,8 +1004,6 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1102,7 +1012,7 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = mul <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 3)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1156,14 +1066,6 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1202,8 +1104,8 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1218,8 +1120,6 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1228,7 +1128,7 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = sdiv <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 3)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1282,14 +1182,6 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1328,8 +1220,8 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1344,8 +1236,6 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1354,7 +1244,7 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = udiv <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 3)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1408,14 +1298,6 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1454,8 +1336,8 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1470,8 +1352,6 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1480,7 +1360,7 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = srem <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 3)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1534,14 +1414,6 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
-; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1580,8 +1452,8 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 16
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1596,8 +1468,6 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1606,7 +1476,7 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = urem <vscale x 16 x i8> [[WIDE_LOAD]], splat (i8 3)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 16 x i8> [[TMP10]], ptr [[TMP11]], align 1
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1663,14 +1533,6 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1709,8 +1571,8 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP15]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1726,8 +1588,6 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1736,7 +1596,7 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = fadd fast <vscale x 4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x float> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1791,14 +1651,6 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1837,8 +1689,8 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP15]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1854,8 +1706,6 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1864,7 +1714,7 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = fsub fast <vscale x 4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x float> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1919,14 +1769,6 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1965,8 +1807,8 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP15]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1982,8 +1824,6 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1992,7 +1832,7 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = fmul fast <vscale x 4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x float> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -2047,14 +1887,6 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -2093,8 +1925,8 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP15]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -2110,8 +1942,6 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -2120,7 +1950,7 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = fdiv fast <vscale x 4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x float> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -2228,14 +2058,6 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -2274,8 +2096,8 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[A2:%.*]] = ptrtoint ptr [[A]] to i64
 ; NO-VP-NEXT:    [[B1:%.*]] = ptrtoint ptr [[B]] to i64
-; NO-VP-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP15]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 100, [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -2291,8 +2113,6 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -2301,7 +2121,7 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = fneg fast <vscale x 4 x float> [[WIDE_LOAD]]
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x float> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
index f60474534ff3c..0ce9ef650379d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-call-intrinsics.ll
@@ -27,14 +27,6 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -78,9 +70,9 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP19]])
+; NO-VP-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP12]], 2
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP11]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; NO-VP:       [[VECTOR_MEMCHECK]]:
@@ -99,8 +91,6 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -111,7 +101,7 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.smax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD5]])
 ; NO-VP-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP15]], ptr [[TMP16]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; NO-VP-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -176,14 +166,6 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -227,9 +209,9 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP19]])
+; NO-VP-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP12]], 2
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP11]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; NO-VP:       [[VECTOR_MEMCHECK]]:
@@ -248,8 +230,6 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -260,7 +240,7 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD5]])
 ; NO-VP-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP15]], ptr [[TMP16]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; NO-VP-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -325,14 +305,6 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -376,9 +348,9 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP19]])
+; NO-VP-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP12]], 2
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP11]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; NO-VP:       [[VECTOR_MEMCHECK]]:
@@ -397,8 +369,6 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -409,7 +379,7 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD5]])
 ; NO-VP-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP15]], ptr [[TMP16]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; NO-VP-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -474,14 +444,6 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -525,9 +487,9 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP19]])
+; NO-VP-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP12]], 2
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP11]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; NO-VP:       [[VECTOR_MEMCHECK]]:
@@ -546,8 +508,6 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -558,7 +518,7 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP15:%.*]] = call <vscale x 4 x i32> @llvm.umin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[WIDE_LOAD5]])
 ; NO-VP-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP15]], ptr [[TMP16]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
 ; NO-VP-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -619,14 +579,6 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], [[TMP20]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP23]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP23]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -665,9 +617,9 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP11]], 4
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP15]])
+; NO-VP-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP10]], 2
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP9]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; NO-VP:       [[VECTOR_MEMCHECK]]:
@@ -682,8 +634,6 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -692,7 +642,7 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.ctlz.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i1 true)
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -748,14 +698,6 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP6]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
-; IF-EVL-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP9]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP10]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP9]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -794,9 +736,9 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP11]], 4
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP15]])
+; NO-VP-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP10]], 2
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP9]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; NO-VP:       [[VECTOR_MEMCHECK]]:
@@ -811,8 +753,6 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -821,7 +761,7 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.cttz.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i1 true)
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -877,14 +817,6 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP26]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP26]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -927,9 +859,9 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP11]], 4
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP17]])
+; NO-VP-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP10]], 2
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP9]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; NO-VP:       [[VECTOR_MEMCHECK]]:
@@ -944,8 +876,6 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -956,7 +886,7 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP14:%.*]] = trunc <vscale x 4 x i64> [[TMP13]] to <vscale x 4 x i32>
 ; NO-VP-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP14]], ptr [[TMP15]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1016,14 +946,6 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP26]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP26]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1066,9 +988,9 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP11]], 4
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP17]])
+; NO-VP-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP10]], 2
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP9]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; NO-VP:       [[VECTOR_MEMCHECK]]:
@@ -1083,8 +1005,6 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1095,7 +1015,7 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP14:%.*]] = trunc <vscale x 4 x i64> [[TMP13]] to <vscale x 4 x i32>
 ; NO-VP-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP14]], ptr [[TMP15]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1155,14 +1075,6 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], [[TMP20]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP23]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP23]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1201,8 +1113,8 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP11]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1218,8 +1130,6 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1228,7 +1138,7 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]], i1 true)
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
index ce2b790cdbd4f..7ea462eed42db 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cast-intrinsics.ll
@@ -22,14 +22,6 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
-; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -67,7 +59,7 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP9]], 2
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 1
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 20, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -85,8 +77,6 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -95,7 +85,7 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = sext <vscale x 2 x i32> [[WIDE_LOAD]] to <vscale x 2 x i64>
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP10]], ptr [[TMP11]], align 8, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -150,14 +140,6 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
-; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -195,7 +177,7 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP9]], 2
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 1
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 20, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -213,8 +195,6 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -223,7 +203,7 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = zext <vscale x 2 x i32> [[WIDE_LOAD]] to <vscale x 2 x i64>
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP10]], ptr [[TMP11]], align 8, !alias.scope [[META12:![0-9]+]], !noalias [[META9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -278,14 +258,6 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
-; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -323,7 +295,7 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP9]], 2
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 1
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 20, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -341,8 +313,6 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -351,7 +321,7 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = trunc <vscale x 2 x i64> [[WIDE_LOAD]] to <vscale x 2 x i32>
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 2 x i32> [[TMP10]], ptr [[TMP11]], align 4, !alias.scope [[META19:![0-9]+]], !noalias [[META16]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -406,14 +376,6 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
-; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -450,8 +412,8 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-LABEL: define void @vp_fpext(
 ; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP9]], 2
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 1
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -469,8 +431,6 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -479,7 +439,7 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = fpext <vscale x 2 x float> [[WIDE_LOAD]] to <vscale x 2 x double>
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 2 x double> [[TMP10]], ptr [[TMP11]], align 8, !alias.scope [[META26:![0-9]+]], !noalias [[META23]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -534,14 +494,6 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
-; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -578,8 +530,8 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-LABEL: define void @vp_fptrunc(
 ; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP9]], 2
+; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP7]], 1
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -597,8 +549,6 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -607,7 +557,7 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = fptrunc <vscale x 2 x double> [[WIDE_LOAD]] to <vscale x 2 x float>
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 2 x float> [[TMP10]], ptr [[TMP11]], align 4, !alias.scope [[META33:![0-9]+]], !noalias [[META30]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP6]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP35:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -662,14 +612,6 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
-; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -708,8 +650,8 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP11]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -725,8 +667,6 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -735,7 +675,7 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = sitofp <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x float>
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 4 x float> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -790,14 +730,6 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
-; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -836,8 +768,8 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP11]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -853,8 +785,6 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -863,7 +793,7 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = uitofp <vscale x 4 x i32> [[WIDE_LOAD]] to <vscale x 4 x float>
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 4 x float> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP39:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -918,14 +848,6 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
-; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -964,8 +886,8 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP11]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -981,8 +903,6 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -991,7 +911,7 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = fptosi <vscale x 4 x float> [[WIDE_LOAD]] to <vscale x 4 x i32>
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1046,14 +966,6 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
-; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1092,8 +1004,8 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP11]], 4
+; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -1109,8 +1021,6 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1119,7 +1029,7 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = fptoui <vscale x 4 x float> [[WIDE_LOAD]] to <vscale x 4 x i32>
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP12]], ptr [[TMP13]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1174,14 +1084,6 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
-; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1220,9 +1122,9 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; NO-VP-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP11]], 2
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP15]])
+; NO-VP-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP9:%.*]] = shl nuw i64 [[TMP10]], 1
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP9]])
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; NO-VP:       [[VECTOR_MEMCHECK]]:
@@ -1237,8 +1139,6 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1247,7 +1147,7 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP12:%.*]] = inttoptr <vscale x 2 x i64> [[WIDE_LOAD]] to <vscale x 2 x ptr>
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds ptr, ptr [[A]], i64 [[IV]]
 ; NO-VP-NEXT:    store <vscale x 2 x ptr> [[TMP12]], ptr [[TMP13]], align 8
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP10]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP8]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -1290,17 +1190,9 @@ exit:
 define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-LABEL: define void @vp_ptrtoint(
 ; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:  [[ENTRY:.*:]]
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i64> [[TMP9]], splat (i64 1)
 ; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP10]]
@@ -1311,8 +1203,7 @@ define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
-; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B]], <vscale x 2 x i64> [[VEC_IND]]
 ; IF-EVL-NEXT:    [[TMP15:%.*]] = ptrtoint <vscale x 2 x ptr> [[TMP14]] to <vscale x 2 x i64>
@@ -1327,10 +1218,9 @@ define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br label %[[EXIT:.*]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; IF-EVL-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[GEP]] to i64
 ; IF-EVL-NEXT:    [[GEP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
@@ -1345,7 +1235,7 @@ define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP13]], 2
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP13]], 1
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -1353,13 +1243,10 @@ define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; NO-VP-NEXT:    [[TMP7:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 1)
 ; NO-VP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP7]]
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP5]]
-; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
@@ -1369,7 +1256,7 @@ define void @vp_ptrtoint(ptr %a, ptr %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = ptrtoint <vscale x 2 x ptr> [[TMP9]] to <vscale x 2 x i64>
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP10]], ptr [[TMP11]], align 8
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll
index d02d53b8e1207..fe3a723a9bbea 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll
@@ -23,14 +23,6 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
 ; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.body:
@@ -53,12 +45,10 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP24:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP20]])
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-OUTLOOP:       scalar.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP27:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-OUTLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP27]], 3
@@ -76,14 +66,6 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
-; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
-; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]]
-; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
-; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -104,12 +86,10 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP:       middle.block:
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-INLOOP:       scalar.ph:
-; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP25:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-INLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP25]], 3
@@ -126,7 +106,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; NO-VP-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-OUTLOOP:       vector.ph:
@@ -134,8 +114,6 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       vector.body:
@@ -146,7 +124,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[TMP21:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 3)
 ; NO-VP-OUTLOOP-NEXT:    [[TMP16:%.*]] = select <vscale x 4 x i1> [[TMP21]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; NO-VP-OUTLOOP-NEXT:    [[TMP17]] = add <vscale x 4 x i32> [[TMP16]], [[VEC_PHI]]
-; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP-OUTLOOP:       middle.block:
@@ -176,7 +154,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-INLOOP:       vector.ph:
@@ -184,8 +162,6 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-INLOOP:       vector.body:
 ; NO-VP-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -196,7 +172,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:    [[TMP10:%.*]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; NO-VP-INLOOP-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
 ; NO-VP-INLOOP-NEXT:    [[TMP12]] = add i32 [[VEC_PHI]], [[TMP11]]
-; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP-INLOOP:       middle.block:
@@ -246,14 +222,6 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
 ; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.body:
@@ -281,12 +249,10 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PREDPHI]])
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-OUTLOOP:       scalar.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[FOR_INC]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[FOR_INC]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP28:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-OUTLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP28]], 3
@@ -308,14 +274,6 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
-; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
-; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]]
-; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
-; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -335,12 +293,10 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP:       middle.block:
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-INLOOP:       scalar.ph:
-; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[FOR_INC]] ]
+; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; IF-EVL-INLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[FOR_INC]] ]
 ; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP25:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-INLOOP-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP25]], 3
@@ -361,7 +317,7 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; NO-VP-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-OUTLOOP:       vector.ph:
@@ -369,8 +325,6 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       vector.body:
@@ -381,7 +335,7 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[TMP18:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], splat (i32 3)
 ; NO-VP-OUTLOOP-NEXT:    [[TMP16:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]
 ; NO-VP-OUTLOOP-NEXT:    [[PREDPHI]] = select <vscale x 4 x i1> [[TMP18]], <vscale x 4 x i32> [[TMP16]], <vscale x 4 x i32> [[VEC_PHI]]
-; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; NO-VP-OUTLOOP:       middle.block:
@@ -415,7 +369,7 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-INLOOP:       vector.ph:
@@ -423,8 +377,6 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-INLOOP:       vector.body:
 ; NO-VP-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -435,7 +387,7 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:    [[TMP10:%.*]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; NO-VP-INLOOP-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP10]])
 ; NO-VP-INLOOP-NEXT:    [[TMP12]] = add i32 [[VEC_PHI]], [[TMP11]]
-; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; NO-VP-INLOOP:       middle.block:
@@ -495,14 +447,6 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
 ; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i32> [[TMP10]], splat (i32 1)
@@ -514,8 +458,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = mul i32 1, [[TMP12]]
-; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP13]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV1]]
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
@@ -533,12 +476,10 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-OUTLOOP:       scalar.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP37:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
@@ -557,14 +498,6 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
-; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i32> [[TMP9]], splat (i32 1)
 ; IF-EVL-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP10]]
@@ -575,8 +508,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = mul i32 1, [[TMP11]]
-; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
 ; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-INLOOP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[EVL_BASED_IV1]]
 ; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -593,12 +525,10 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP:       middle.block:
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-INLOOP:       scalar.ph:
-; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-INLOOP-NEXT:    [[RDX1:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-INLOOP-NEXT:    [[RDX1:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ADD1:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP28:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
@@ -616,7 +546,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; NO-VP-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-OUTLOOP:       vector.ph:
@@ -624,15 +554,12 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP14:%.*]] = mul <vscale x 4 x i32> [[TMP12]], splat (i32 1)
 ; NO-VP-OUTLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP14]]
-; NO-VP-OUTLOOP-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP10]] to i32
-; NO-VP-OUTLOOP-NEXT:    [[TMP17:%.*]] = mul i32 1, [[TMP16]]
-; NO-VP-OUTLOOP-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP17]], i64 0
+; NO-VP-OUTLOOP-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP7]] to i32
+; NO-VP-OUTLOOP-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP16]], i64 0
 ; NO-VP-OUTLOOP-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       vector.body:
@@ -644,7 +571,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[TMP27:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_IND]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP22:%.*]] = select <vscale x 4 x i1> [[TMP27]], <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; NO-VP-OUTLOOP-NEXT:    [[TMP23]] = add <vscale x 4 x i32> [[TMP22]], [[VEC_PHI]]
-; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -676,7 +603,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-INLOOP:       vector.ph:
@@ -684,14 +611,11 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-INLOOP-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i32> [[TMP6]], splat (i32 1)
 ; NO-VP-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP8]]
-; NO-VP-INLOOP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i32
-; NO-VP-INLOOP-NEXT:    [[TMP11:%.*]] = mul i32 1, [[TMP10]]
-; NO-VP-INLOOP-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
+; NO-VP-INLOOP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP3]] to i32
+; NO-VP-INLOOP-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP10]], i64 0
 ; NO-VP-INLOOP-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-INLOOP:       vector.body:
@@ -704,7 +628,7 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:    [[TMP16:%.*]] = select <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; NO-VP-INLOOP-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP16]])
 ; NO-VP-INLOOP-NEXT:    [[TMP18]] = add i32 [[VEC_PHI]], [[TMP17]]
-; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
 ; NO-VP-INLOOP-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -757,14 +681,6 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
 ; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP13:%.*]] = mul <vscale x 4 x i32> [[TMP10]], splat (i32 1)
@@ -776,13 +692,10 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[VEC_IND2:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT7:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP14]], i64 0
-; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
-; IF-EVL-OUTLOOP-NEXT:    [[TMP11:%.*]] = mul i32 1, [[TMP14]]
-; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
+; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP14]], i64 0
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP18:%.*]] = icmp ult <vscale x 4 x i32> [[TMP12]], [[BROADCAST_SPLAT4]]
+; IF-EVL-OUTLOOP-NEXT:    [[TMP18:%.*]] = icmp ult <vscale x 4 x i32> [[TMP12]], [[BROADCAST_SPLAT2]]
 ; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-OUTLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[ARRAYIDX]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP14]])
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP21:%.*]] = icmp sle <vscale x 4 x i32> [[VP_OP_LOAD]], [[VEC_IND2]]
@@ -800,12 +713,10 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP24]])
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-OUTLOOP:       scalar.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       for.body:
-; IF-EVL-OUTLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
-; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
+; IF-EVL-OUTLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-OUTLOOP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP38:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
 ; IF-EVL-OUTLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV1]] to i32
@@ -828,14 +739,6 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
-; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
-; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i32> [[TMP9]], splat (i32 1)
 ; IF-EVL-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP10]]
@@ -846,8 +749,7 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
-; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = mul i32 1, [[TMP11]]
-; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP12]], i64 0
+; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
 ; IF-EVL-INLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-INLOOP-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[ARRAYIDX]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -863,12 +765,10 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP:       middle.block:
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL-INLOOP:       scalar.ph:
-; IF-EVL-INLOOP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX1:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-INLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       for.body:
-; IF-EVL-INLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
-; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX1]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-INLOOP-NEXT:    [[IV1:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[MIDDLE_BLOCK:%.*]] ]
+; IF-EVL-INLOOP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[RDX_ADD:%.*]], [[MIDDLE_BLOCK]] ]
 ; IF-EVL-INLOOP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP35:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
 ; IF-EVL-INLOOP-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV1]] to i32
@@ -890,7 +790,7 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; NO-VP-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-OUTLOOP:       vector.ph:
@@ -898,15 +798,12 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP14:%.*]] = mul <vscale x 4 x i32> [[TMP12]], splat (i32 1)
 ; NO-VP-OUTLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP14]]
-; NO-VP-OUTLOOP-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP10]] to i32
-; NO-VP-OUTLOOP-NEXT:    [[TMP17:%.*]] = mul i32 1, [[TMP16]]
-; NO-VP-OUTLOOP-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP17]], i64 0
+; NO-VP-OUTLOOP-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP7]] to i32
+; NO-VP-OUTLOOP-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP16]], i64 0
 ; NO-VP-OUTLOOP-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       vector.body:
@@ -918,7 +815,7 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-NEXT:    [[TMP28:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_MASKED_LOAD]], [[VEC_IND]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP22:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_LOAD]]
 ; NO-VP-OUTLOOP-NEXT:    [[PREDPHI]] = select <vscale x 4 x i1> [[TMP28]], <vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[VEC_PHI]]
-; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]]
+; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP26]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -954,7 +851,7 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-INLOOP:       vector.ph:
@@ -962,14 +859,11 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-INLOOP-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i32> [[TMP6]], splat (i32 1)
 ; NO-VP-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP8]]
-; NO-VP-INLOOP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i32
-; NO-VP-INLOOP-NEXT:    [[TMP11:%.*]] = mul i32 1, [[TMP10]]
-; NO-VP-INLOOP-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
+; NO-VP-INLOOP-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP3]] to i32
+; NO-VP-INLOOP-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP10]], i64 0
 ; NO-VP-INLOOP-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-INLOOP:       vector.body:
@@ -982,7 +876,7 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-NEXT:    [[TMP16:%.*]] = select <vscale x 4 x i1> [[TMP15]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> zeroinitializer
 ; NO-VP-INLOOP-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP16]])
 ; NO-VP-INLOOP-NEXT:    [[TMP18]] = add i32 [[VEC_PHI]], [[TMP17]]
-; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
 ; NO-VP-INLOOP-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
index ae047f5f63476..6b7fd976cbe6c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-div.ll
@@ -10,17 +10,9 @@
 define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL-LABEL: define void @test_sdiv(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
-; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
+; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*:]]
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -42,10 +34,9 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br label %[[EXIT:.*]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[SCALAR_PH]] ]
 ; IF-EVL-NEXT:    [[A_GEP:%.*]] = getelementptr i64, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = load i64, ptr [[A_GEP]], align 8
 ; IF-EVL-NEXT:    [[B_GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]
@@ -63,7 +54,7 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
+; NO-VP-NEXT:    [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -71,8 +62,6 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP13]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -83,7 +72,7 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = sdiv <vscale x 2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
 ; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[C]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], align 8
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -131,17 +120,9 @@ exit:
 define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL-LABEL: define void @test_udiv(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
-; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
+; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*:]]
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -163,10 +144,9 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br label %[[EXIT:.*]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[SCALAR_PH]] ]
 ; IF-EVL-NEXT:    [[A_GEP:%.*]] = getelementptr i64, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = load i64, ptr [[A_GEP]], align 8
 ; IF-EVL-NEXT:    [[B_GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]
@@ -184,7 +164,7 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
+; NO-VP-NEXT:    [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -192,8 +172,6 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP13]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -204,7 +182,7 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = udiv <vscale x 2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
 ; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[C]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], align 8
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -251,17 +229,9 @@ exit:
 define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL-LABEL: define void @test_srem(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
-; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
+; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*:]]
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -283,10 +253,9 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br label %[[EXIT:.*]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[SCALAR_PH]] ]
 ; IF-EVL-NEXT:    [[A_GEP:%.*]] = getelementptr i64, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = load i64, ptr [[A_GEP]], align 8
 ; IF-EVL-NEXT:    [[B_GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]
@@ -304,7 +273,7 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
+; NO-VP-NEXT:    [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -312,8 +281,6 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP13]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -324,7 +291,7 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = srem <vscale x 2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
 ; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[C]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], align 8
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -371,17 +338,9 @@ exit:
 define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL-LABEL: define void @test_urem(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
-; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*]]:
+; IF-EVL-NEXT:  [[LOOP_PREHEADER:.*:]]
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -403,10 +362,9 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br label %[[EXIT:.*]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
 ; IF-EVL-NEXT:    br label %[[LOOP:.*]]
 ; IF-EVL:       [[LOOP]]:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[SCALAR_PH]] ]
 ; IF-EVL-NEXT:    [[A_GEP:%.*]] = getelementptr i64, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP16:%.*]] = load i64, ptr [[A_GEP]], align 8
 ; IF-EVL-NEXT:    [[B_GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]
@@ -424,7 +382,7 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[LOOP_PREHEADER:.*]]:
 ; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
+; NO-VP-NEXT:    [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP12]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -432,8 +390,6 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP13]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; NO-VP:       [[VECTOR_BODY]]:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -444,7 +400,7 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = urem <vscale x 2 x i64> [[WIDE_LOAD]], [[WIDE_LOAD1]]
 ; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr i64, ptr [[C]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP8]], ptr [[TMP9]], align 8
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll
index 987f9460c2172..bf54f669b65f9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll
@@ -12,15 +12,9 @@
 define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-LABEL: define void @first_order_recurrence(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0:[0-9]+]] {
-; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:  [[ENTRY:.*:]]
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP8]] to i32
@@ -49,12 +43,10 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br label %[[FOR_END:.*]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; IF-EVL-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 33, %[[ENTRY]] ]
 ; IF-EVL-NEXT:    br label %[[FOR_BODY:.*]]
 ; IF-EVL:       [[FOR_BODY]]:
-; IF-EVL-NEXT:    [[INDVARS:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], %[[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP24:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[INDVARS:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i32 [ 33, %[[SCALAR_PH]] ], [ [[TMP24:%.*]], %[[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS]]
 ; IF-EVL-NEXT:    [[TMP24]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ADD:%.*]] = add nsw i32 [[FOR1]], [[TMP24]]
@@ -70,7 +62,7 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -78,8 +70,6 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
 ; NO-VP-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4
 ; NO-VP-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
@@ -94,7 +84,7 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP13:%.*]] = add nsw <vscale x 4 x i32> [[TMP12]], [[WIDE_LOAD]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP13]], ptr [[TMP14]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -144,15 +134,9 @@ for.end:
 define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-LABEL: define void @second_order_recurrence(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
-; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:  [[ENTRY:.*:]]
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP32:%.*]] = trunc i64 [[TMP8]] to i32
@@ -187,14 +171,11 @@ define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br label %[[FOR_END:.*]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; IF-EVL-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 33, %[[ENTRY]] ]
-; IF-EVL-NEXT:    [[SCALAR_RECUR_INIT3:%.*]] = phi i32 [ 22, %[[ENTRY]] ]
 ; IF-EVL-NEXT:    br label %[[FOR_BODY:.*]]
 ; IF-EVL:       [[FOR_BODY]]:
-; IF-EVL-NEXT:    [[INDVARS:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], %[[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP31:%.*]], %[[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[FOR2:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT3]], %[[SCALAR_PH]] ], [ [[FOR1]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[INDVARS:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i32 [ 33, %[[SCALAR_PH]] ], [ [[TMP31:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[FOR2:%.*]] = phi i32 [ 22, %[[SCALAR_PH]] ], [ [[FOR1]], %[[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS]]
 ; IF-EVL-NEXT:    [[TMP31]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ADD:%.*]] = add nsw i32 [[FOR1]], [[FOR2]]
@@ -210,7 +191,7 @@ define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -218,8 +199,6 @@ define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
 ; NO-VP-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4
 ; NO-VP-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
@@ -240,7 +219,7 @@ define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP17:%.*]] = add nsw <vscale x 4 x i32> [[TMP15]], [[TMP16]]
 ; NO-VP-NEXT:    [[TMP18:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP17]], ptr [[TMP18]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -297,15 +276,9 @@ for.end:
 define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-LABEL: define void @third_order_recurrence(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
-; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:  [[ENTRY:.*:]]
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP8]] to i32
@@ -347,16 +320,12 @@ define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br label %[[FOR_END:.*]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; IF-EVL-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 33, %[[ENTRY]] ]
-; IF-EVL-NEXT:    [[SCALAR_RECUR_INIT5:%.*]] = phi i32 [ 22, %[[ENTRY]] ]
-; IF-EVL-NEXT:    [[SCALAR_RECUR_INIT6:%.*]] = phi i32 [ 11, %[[ENTRY]] ]
 ; IF-EVL-NEXT:    br label %[[FOR_BODY:.*]]
 ; IF-EVL:       [[FOR_BODY]]:
-; IF-EVL-NEXT:    [[INDVARS:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], %[[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP38:%.*]], %[[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[FOR2:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT5]], %[[SCALAR_PH]] ], [ [[FOR1]], %[[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[FOR3:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT6]], %[[SCALAR_PH]] ], [ [[FOR2]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[INDVARS:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i32 [ 33, %[[SCALAR_PH]] ], [ [[TMP38:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[FOR2:%.*]] = phi i32 [ 22, %[[SCALAR_PH]] ], [ [[FOR1]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[FOR3:%.*]] = phi i32 [ 11, %[[SCALAR_PH]] ], [ [[FOR2]], %[[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[A]], i64 [[INDVARS]]
 ; IF-EVL-NEXT:    [[TMP38]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ADD:%.*]] = add nsw i32 [[FOR2]], [[FOR3]]
@@ -373,7 +342,7 @@ define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -381,8 +350,6 @@ define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
 ; NO-VP-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4
 ; NO-VP-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
@@ -410,7 +377,7 @@ define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP22:%.*]] = add <vscale x 4 x i32> [[TMP21]], [[TMP18]]
 ; NO-VP-NEXT:    [[TMP23:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP22]], ptr [[TMP23]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -478,7 +445,7 @@ define i32 @FOR_reduction(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP9]], 2
 ; IF-EVL-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
@@ -486,8 +453,6 @@ define i32 @FOR_reduction(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
@@ -502,7 +467,7 @@ define i32 @FOR_reduction(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = add nsw <vscale x 4 x i32> [[TMP10]], [[WIDE_LOAD]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS]]
 ; IF-EVL-NEXT:    store <vscale x 4 x i32> [[TMP11]], ptr [[TMP12]], align 4
-; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDVARS]], [[TMP5]]
+; IF-EVL-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDVARS]], [[TMP3]]
 ; IF-EVL-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; IF-EVL-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
@@ -539,7 +504,7 @@ define i32 @FOR_reduction(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -547,8 +512,6 @@ define i32 @FOR_reduction(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
 ; NO-VP-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4
 ; NO-VP-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
@@ -563,7 +526,7 @@ define i32 @FOR_reduction(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP13:%.*]] = add nsw <vscale x 4 x i32> [[TMP12]], [[WIDE_LOAD]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP13]], ptr [[TMP14]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
@@ -618,15 +581,9 @@ for.end:
 define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
 ; IF-EVL-LABEL: define void @first_order_recurrence_indvar(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
-; IF-EVL-NEXT:  [[ENTRY:.*]]:
+; IF-EVL-NEXT:  [[ENTRY:.*:]]
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP18]], 2
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
@@ -646,8 +603,7 @@ define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
 ; IF-EVL-NEXT:    [[PREV_EVL:%.*]] = phi i32 [ [[TMP5]], %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP7]]
-; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP7]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; IF-EVL-NEXT:    [[TMP20]] = add <vscale x 2 x i64> [[VEC_IND]], splat (i64 42)
 ; IF-EVL-NEXT:    [[TMP15:%.*]] = call <vscale x 2 x i64> @llvm.experimental.vp.splice.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[TMP20]], i32 -1, <vscale x 2 x i1> splat (i1 true), i32 [[PREV_EVL]], i32 [[TMP11]])
@@ -662,12 +618,10 @@ define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
 ; IF-EVL:       [[MIDDLE_BLOCK]]:
 ; IF-EVL-NEXT:    br label %[[FOR_END:.*]]
 ; IF-EVL:       [[SCALAR_PH]]:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; IF-EVL-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 33, %[[ENTRY]] ]
 ; IF-EVL-NEXT:    br label %[[FOR_BODY:.*]]
 ; IF-EVL:       [[FOR_BODY]]:
-; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV1_NEXT:%.*]], %[[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TMP14:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV1_NEXT:%.*]], %[[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[FOR1:%.*]] = phi i64 [ 33, %[[SCALAR_PH]] ], [ [[TMP14:%.*]], %[[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP14]] = add i64 [[IV1]], 42
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[IV1]]
 ; IF-EVL-NEXT:    store i64 [[FOR1]], ptr [[ARRAYIDX]], align 8
@@ -681,7 +635,7 @@ define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
@@ -689,13 +643,10 @@ define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP4]], 2
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; NO-VP-NEXT:    [[TMP7:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 1)
 ; NO-VP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP7]]
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul i64 1, [[TMP9]]
-; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP10]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; NO-VP-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
 ; NO-VP-NEXT:    [[TMP16:%.*]] = mul nuw i32 [[TMP14]], 2
@@ -710,7 +661,7 @@ define void @first_order_recurrence_indvar(ptr noalias %A, i64 %TC) {
 ; NO-VP-NEXT:    [[TMP13:%.*]] = call <vscale x 2 x i64> @llvm.vector.splice.nxv2i64(<vscale x 2 x i64> [[VECTOR_RECUR]], <vscale x 2 x i64> [[TMP12]], i32 -1)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i64, ptr [[A]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[TMP13]], ptr [[TMP11]], align 8
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; NO-VP-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
index 2aeb1d0b25b5d..6f723c268914c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-gather-scatter.ll
@@ -12,14 +12,6 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = mul <vscale x 2 x i64> [[TMP9]], splat (i64 1)
 ; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP10]]
@@ -27,11 +19,10 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[ENTRY]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[ENTRY]] ], [ [[AVL_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[ENTRY]] ], [ [[AVL_NEXT:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP11]] to i64
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 1, [[TMP12]]
-; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[INDEX:%.*]], <vscale x 2 x i64> [[VEC_IND]]
 ; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x i64> @llvm.vp.gather.nxv2i64.nxv2p0(<vscale x 2 x ptr> align 8 [[TMP14]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP11]])
@@ -48,10 +39,9 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY1:%.*]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[INDVARS_IV1:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT1:%.*]], [[FOR_BODY1]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[INDEX]], i64 [[INDVARS_IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX3]], align 8
 ; IF-EVL-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[IN]], i64 [[TMP0]]
@@ -67,7 +57,7 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
 ; NO-VP-LABEL: @gather_scatter(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 2
+; NO-VP-NEXT:    [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 1
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP14]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; NO-VP:       vector.ph:
@@ -75,13 +65,10 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; NO-VP-NEXT:    [[TMP7:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 1)
 ; NO-VP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP7]]
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP5]]
-; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -93,7 +80,7 @@ define void @gather_scatter(ptr noalias %in, ptr noalias %out, ptr noalias %inde
 ; NO-VP-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32.nxv2p0(<vscale x 2 x ptr> [[TMP10]], i32 4, <vscale x 2 x i1> splat (i1 true), <vscale x 2 x float> poison)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[OUT:%.*]], <vscale x 2 x i64> [[WIDE_MASKED_GATHER]]
 ; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv2f32.nxv2p0(<vscale x 2 x float> [[WIDE_MASKED_GATHER2]], <vscale x 2 x ptr> [[TMP11]], i32 4, <vscale x 2 x i1> splat (i1 true))
-; NO-VP-NEXT:    [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], [[TMP5]]
+; NO-VP-NEXT:    [[INDVARS_IV_NEXT]] = add nuw i64 [[INDVARS_IV]], [[TMP3]]
 ; NO-VP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll
index 3e23df78e0b68..517055a345406 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll
@@ -14,19 +14,11 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -40,12 +32,10 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ADD]] = add nsw i32 [[TMP18]], [[RDX]]
@@ -59,7 +49,7 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @add(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -67,8 +57,6 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -77,7 +65,7 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[TMP10]] = add i32 [[VEC_PHI]], [[TMP9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -229,19 +217,11 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -255,12 +235,10 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[OR:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[OR:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[OR]] = or i32 [[TMP18]], [[RDX]]
@@ -274,7 +252,7 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @or(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -282,8 +260,6 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -292,7 +268,7 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[TMP10]] = or i32 [[VEC_PHI]], [[TMP9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -337,19 +313,11 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -363,12 +331,10 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[AND]] = and i32 [[TMP18]], [[RDX]]
@@ -382,7 +348,7 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @and(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -390,8 +356,6 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -400,7 +364,7 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[TMP10]] = and i32 [[VEC_PHI]], [[TMP9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -445,19 +409,11 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -471,12 +427,10 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[XOR:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[XOR:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[XOR]] = xor i32 [[TMP18]], [[RDX]]
@@ -490,7 +444,7 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @xor(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -498,8 +452,6 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -508,7 +460,7 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[TMP10]] = xor i32 [[VEC_PHI]], [[TMP9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -553,19 +505,11 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -579,12 +523,10 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP17]], [[RDX]]
@@ -599,7 +541,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @smin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -607,8 +549,6 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -617,7 +557,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smin.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -664,19 +604,11 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -690,12 +622,10 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMAX:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[SMAX:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp sgt i32 [[TMP17]], [[RDX]]
@@ -710,7 +640,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @smax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -718,8 +648,6 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -728,7 +656,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.smax.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -775,19 +703,11 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -801,12 +721,10 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[UMIN:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[UMIN:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp ult i32 [[TMP17]], [[RDX]]
@@ -821,7 +739,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @umin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -829,8 +747,6 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -839,7 +755,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umin.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -886,19 +802,11 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -912,12 +820,10 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[UMAX:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[UMAX:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp ugt i32 [[TMP17]], [[RDX]]
@@ -932,7 +838,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @umax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -940,8 +846,6 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -950,7 +854,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[RDX_MINMAX]] = call i32 @llvm.umax.i32(i32 [[TMP9]], i32 [[VEC_PHI]])
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -997,19 +901,11 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[START:%.*]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -1023,12 +919,10 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ADD]] = fadd reassoc float [[TMP18]], [[RDX]]
@@ -1042,7 +936,7 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; NO-VP-LABEL: @fadd(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1050,8 +944,6 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1060,7 +952,7 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[TMP10]] = fadd reassoc float [[VEC_PHI]], [[TMP9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1212,19 +1104,11 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX_SELECT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -1239,12 +1123,10 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MIN:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START]], [[SCALAR_PH]] ], [ [[MIN:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP:%.*]] = fcmp fast olt float [[TMP17]], [[RDX]]
@@ -1259,7 +1141,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-LABEL: @fmin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1267,8 +1149,6 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1278,7 +1158,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt float [[TMP9]], [[VEC_PHI]]
 ; NO-VP-NEXT:    [[RDX_MINMAX_SELECT]] = select fast i1 [[RDX_MINMAX_CMP]], float [[TMP9]], float [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1325,19 +1205,11 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[START:%.*]], [[VECTOR_PH]] ], [ [[RDX_MINMAX_SELECT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -1352,12 +1224,10 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MAX:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START]], [[SCALAR_PH]] ], [ [[MAX:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP:%.*]] = fcmp fast ogt float [[TMP17]], [[RDX]]
@@ -1372,7 +1242,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-LABEL: @fmax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1380,8 +1250,6 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1391,7 +1259,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt float [[TMP9]], [[VEC_PHI]]
 ; NO-VP-NEXT:    [[RDX_MINMAX_SELECT]] = select fast i1 [[RDX_MINMAX_CMP]], float [[TMP9]], float [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1654,19 +1522,11 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi float [ [[START:%.*]], [[VECTOR_PH]] ], [ [[TMP18:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -1683,12 +1543,10 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
@@ -1704,7 +1562,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; NO-VP-LABEL: @fmuladd(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1712,8 +1570,6 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1725,7 +1581,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; NO-VP-NEXT:    [[TMP11:%.*]] = fmul reassoc <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP11]])
 ; NO-VP-NEXT:    [[TMP13]] = fadd reassoc float [[VEC_PHI]], [[TMP12]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1774,19 +1630,11 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -1803,12 +1651,10 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP20]], i32 [[INV:%.*]], i32 [[START:%.*]]
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ANYOF:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ANYOF:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP21]], 3
@@ -1823,7 +1669,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-LABEL: @anyof_icmp(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1831,8 +1677,6 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1841,7 +1685,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 3)
 ; NO-VP-NEXT:    [[TMP10]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1891,19 +1735,11 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP11]])
@@ -1920,12 +1756,10 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP20]], i32 [[INV:%.*]], i32 [[START:%.*]]
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ANYOF:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ANYOF:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = fcmp fast olt float [[TMP21]], 3.000000e+00
@@ -1940,7 +1774,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-LABEL: @anyof_fcmp(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1948,8 +1782,6 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1958,7 +1790,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = fcmp fast olt <vscale x 4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
 ; NO-VP-NEXT:    [[TMP10]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; NO-VP:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
index 8d987a94d383d..332c16e8eb656 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
@@ -13,18 +13,10 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP11]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -47,10 +39,9 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i64 [[IV]], i32 0
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [2 x i32], ptr [[B]], i64 [[IV]], i32 1
@@ -67,7 +58,7 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; NO-VP-LABEL: @interleave(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -75,8 +66,6 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -88,7 +77,7 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP21:%.*]] = add nsw <vscale x 4 x i32> [[TMP19]], [[TMP18]]
 ; NO-VP-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP21]], ptr [[TMP22]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -132,5 +121,863 @@ for.cond.cleanup:
   ret void
 }
 
+; Interleaved group with gap but without tail gap
+; E.g.
+; int (*a)[4];
+; int rdx = 0;
+; for (int i = 0; i < n; i++) {
+;   rdx += a[i][0];
+;   rdx += a[i][1];
+;   // No access a[i][2]
+;   rdx += a[i][3];
+; }
+;
+define i32 @load_factor_4_with_gap(i64 %n, ptr noalias %a) {
+; IF-EVL-LABEL: @load_factor_4_with_gap(
+; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; IF-EVL-NEXT:    [[TMP3:%.*]] = mul <vscale x 4 x i64> [[TMP2]], splat (i64 1)
+; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP3]]
+; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT:    [[TMP8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[WIDE_MASKED_GATHER1]]
+; IF-EVL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[WIDE_MASKED_GATHER2]]
+; IF-EVL-NEXT:    [[TMP12]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP4]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP4]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP13]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP12]])
+; IF-EVL-NEXT:    br label [[EXIT:%.*]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       for.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
+; IF-EVL-NEXT:    [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; IF-EVL-NEXT:    [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP16]]
+; IF-EVL-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1
+; IF-EVL-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; IF-EVL-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP17]]
+; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 3
+; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; IF-EVL-NEXT:    [[ADD2]] = add nsw i32 [[ADD1]], [[TMP18]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; IF-EVL:       exit:
+; IF-EVL-NEXT:    [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-NEXT:    ret i32 [[ADD2_LCSSA]]
+;
+; NO-VP-LABEL: @load_factor_4_with_gap(
+; NO-VP-NEXT:  entry:
+; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N:%.*]], [[TMP1]]
+; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP:       vector.ph:
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; NO-VP-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; NO-VP-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
+; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP5]]
+; NO-VP-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; NO-VP-NEXT:    [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 1)
+; NO-VP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP7]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NO-VP:       vector.body:
+; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP9]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP9]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER1]]
+; NO-VP-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP12]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER2]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; NO-VP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; NO-VP-NEXT:    br label [[SCALAR_PH]]
+; NO-VP:       scalar.ph:
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; NO-VP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
+; NO-VP:       for.body:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
+; NO-VP-NEXT:    [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT:    [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP16]]
+; NO-VP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1
+; NO-VP-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP17]]
+; NO-VP-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 3
+; NO-VP-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; NO-VP-NEXT:    [[ADD2]] = add nsw i32 [[ADD1]], [[TMP18]]
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; NO-VP:       exit:
+; NO-VP-NEXT:    [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    ret i32 [[ADD2_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %rdx = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+  %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 0
+  %0 = load i32, ptr %arrayidx, align 4
+  %add = add nsw i32 %rdx, %0
+  %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 1
+  %1 = load i32, ptr %arrayidx, align 4
+  %add1 = add nsw i32 %add, %1
+  %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 3
+  %2 = load i32, ptr %arrayidx2, align 4
+  %add2 = add nsw i32 %add1, %2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  ret i32 %add2
+}
+
+; Interleaved group with gap but without tail gap
+; E.g.
+; int (*a)[4];
+; for (int i = 0; i < n; i++) {
+;   a[i][0] = i;
+;   a[i][1] = i;
+;   // No access a[i][2]
+;   a[i][3] = i;
+; }
+;
+define void @store_factor_4_with_gap(i32 %n, ptr noalias %a) {
+; IF-EVL-LABEL: @store_factor_4_with_gap(
+; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], splat (i32 1)
+; IF-EVL-NEXT:    [[INDUCTION1:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP5]]
+; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND2:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i32 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP6]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND2]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND2]], i32 1
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND2]], i32 3
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT5]] = add <vscale x 4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_EVL_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    br label [[EXIT:%.*]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       for.body:
+; IF-EVL-NEXT:    [[TMP15:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 0
+; IF-EVL-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
+; IF-EVL-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 1
+; IF-EVL-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4
+; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 3
+; IF-EVL-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX2]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[TMP15]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; IF-EVL:       exit:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: @store_factor_4_with_gap(
+; NO-VP-NEXT:  entry:
+; NO-VP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP1]]
+; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP:       vector.ph:
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP9:%.*]] = mul nuw i32 [[TMP2]], 4
+; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP9]]
+; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; NO-VP-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i32> [[TMP7]], splat (i32 1)
+; NO-VP-NEXT:    [[INDUCTION1:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP8]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NO-VP:       vector.body:
+; NO-VP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_IND4:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND4]], i32 0
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> [[TMP11]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 1
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> [[TMP12]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 3
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> [[TMP13]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP9]]
+; NO-VP-NEXT:    [[VEC_IND_NEXT5]] = add <vscale x 4 x i32> [[VEC_IND4]], [[BROADCAST_SPLAT3]]
+; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NO-VP:       scalar.ph:
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
+; NO-VP:       for.body:
+; NO-VP-NEXT:    [[TMP15:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 0
+; NO-VP-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 1
+; NO-VP-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4
+; NO-VP-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 3
+; NO-VP-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX2]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[TMP15]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; NO-VP:       exit:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 0
+  store i32 %iv, ptr %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 1
+  store i32 %iv, ptr %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 3
+  store i32 %iv, ptr %arrayidx2, align 4
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+; Interleaved group with tail gap
+; E.g.
+; int (*a)[4];
+; int rdx = 0;
+; for (int i = 0; i < n; i++) {
+;   rdx += a[i][0];
+;   rdx += a[i][1];
+;   rdx += a[i][2];
+;   // No access a[i][3]
+; }
+;
+define i32 @load_factor_4_with_tail_gap(i64 %n, ptr noalias %a) {
+; IF-EVL-LABEL: @load_factor_4_with_tail_gap(
+; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[TMP2:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; IF-EVL-NEXT:    [[TMP3:%.*]] = mul <vscale x 4 x i64> [[TMP2]], splat (i64 1)
+; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP3]]
+; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT:    [[TMP8:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP7]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP8]], [[WIDE_MASKED_GATHER1]]
+; IF-EVL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP4]])
+; IF-EVL-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP9]], [[WIDE_MASKED_GATHER2]]
+; IF-EVL-NEXT:    [[TMP12]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP11]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP4]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP4]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP13]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP13]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP12]])
+; IF-EVL-NEXT:    br label [[EXIT:%.*]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       for.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
+; IF-EVL-NEXT:    [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; IF-EVL-NEXT:    [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP16]]
+; IF-EVL-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1
+; IF-EVL-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; IF-EVL-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP17]]
+; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 2
+; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; IF-EVL-NEXT:    [[ADD2]] = add nsw i32 [[ADD1]], [[TMP18]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; IF-EVL:       exit:
+; IF-EVL-NEXT:    [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-NEXT:    ret i32 [[ADD2_LCSSA]]
+;
+; NO-VP-LABEL: @load_factor_4_with_tail_gap(
+; NO-VP-NEXT:  entry:
+; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[N:%.*]], [[TMP1]]
+; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP:       vector.ph:
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; NO-VP-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
+; NO-VP-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
+; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[TMP5]]
+; NO-VP-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; NO-VP-NEXT:    [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 1)
+; NO-VP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP7]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NO-VP:       vector.body:
+; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP9]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER1:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP9]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER1]]
+; NO-VP-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP12]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP13]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER2]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; NO-VP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP13]])
+; NO-VP-NEXT:    br label [[SCALAR_PH]]
+; NO-VP:       scalar.ph:
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; NO-VP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
+; NO-VP:       for.body:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD2:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
+; NO-VP-NEXT:    [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT:    [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP16]]
+; NO-VP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1
+; NO-VP-NEXT:    [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP17]]
+; NO-VP-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 2
+; NO-VP-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; NO-VP-NEXT:    [[ADD2]] = add nsw i32 [[ADD1]], [[TMP18]]
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; NO-VP:       exit:
+; NO-VP-NEXT:    [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    ret i32 [[ADD2_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %rdx = phi i32 [ 0, %entry ], [ %add2, %for.body ]
+  %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 0
+  %0 = load i32, ptr %arrayidx, align 4
+  %add = add nsw i32 %rdx, %0
+  %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 1
+  %1 = load i32, ptr %arrayidx, align 4
+  %add1 = add nsw i32 %add, %1
+  %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 2
+  %2 = load i32, ptr %arrayidx2, align 4
+  %add2 = add nsw i32 %add1, %2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  ret i32 %add2
+}
+
+; Interleaved group with tail gap
+; E.g.
+; int (*a)[4];
+; int rdx = 0;
+; for (int i = 0; i < n; i++) {
+;   a[i][0] = i;
+;   a[i][1] = i;
+;   a[i][2] = i;
+;   // No access a[i][3]
+; }
+;
+define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) {
+; IF-EVL-LABEL: @store_factor_4_with_tail_gap(
+; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul <vscale x 4 x i32> [[TMP4]], splat (i32 1)
+; IF-EVL-NEXT:    [[INDUCTION1:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP5]]
+; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND2:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i32 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP6]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND2]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND2]], i32 1
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND2]], i32 2
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND2]], <vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i32 [[TMP6]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT5]] = add <vscale x 4 x i32> [[VEC_IND2]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_EVL_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    br label [[EXIT:%.*]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       for.body:
+; IF-EVL-NEXT:    [[TMP15:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 0
+; IF-EVL-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
+; IF-EVL-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 1
+; IF-EVL-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4
+; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 2
+; IF-EVL-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX2]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[TMP15]], 1
+; IF-EVL-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; IF-EVL-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; IF-EVL:       exit:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: @store_factor_4_with_tail_gap(
+; NO-VP-NEXT:  entry:
+; NO-VP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
+; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP1]]
+; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP:       vector.ph:
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP9:%.*]] = mul nuw i32 [[TMP2]], 4
+; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP9]]
+; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; NO-VP-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i32> [[TMP7]], splat (i32 1)
+; NO-VP-NEXT:    [[INDUCTION1:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP8]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NO-VP:       vector.body:
+; NO-VP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_IND4:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION1]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND4]], i32 0
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> [[TMP11]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 1
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> [[TMP12]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 2
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> [[TMP13]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP9]]
+; NO-VP-NEXT:    [[VEC_IND_NEXT5]] = add <vscale x 4 x i32> [[VEC_IND4]], [[BROADCAST_SPLAT3]]
+; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NO-VP:       scalar.ph:
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
+; NO-VP:       for.body:
+; NO-VP-NEXT:    [[TMP15:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 0
+; NO-VP-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 1
+; NO-VP-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4
+; NO-VP-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP15]], i32 2
+; NO-VP-NEXT:    store i32 [[TMP15]], ptr [[ARRAYIDX2]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[TMP15]], 1
+; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; NO-VP:       exit:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 0
+  store i32 %iv, ptr %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 1
+  store i32 %iv, ptr %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 2
+  store i32 %iv, ptr %arrayidx2, align 4
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exitcond.not = icmp eq i32 %iv.next, %n
+  br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define i32 @load_factor_4_reverse(i64 %n, ptr noalias %a) {
+; IF-EVL-LABEL: @load_factor_4_reverse(
+; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1
+; IF-EVL-NEXT:    [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
+; IF-EVL-NEXT:    [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[N]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul <vscale x 4 x i64> [[TMP4]], splat (i64 -1)
+; IF-EVL-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP5]]
+; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP1]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP6:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP6]] to i64
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 -1, [[TMP7]]
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[TMP10:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP9]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[TMP10]], [[WIDE_MASKED_GATHER3]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[TMP13:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER4]]
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3
+; IF-EVL-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP6]])
+; IF-EVL-NEXT:    [[TMP15:%.*]] = add <vscale x 4 x i32> [[TMP13]], [[WIDE_MASKED_GATHER5]]
+; IF-EVL-NEXT:    [[TMP16]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP15]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP6]])
+; IF-EVL-NEXT:    [[TMP17:%.*]] = zext i32 [[TMP6]] to i64
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP17]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP17]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; IF-EVL-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP1]]
+; IF-EVL-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP16]])
+; IF-EVL-NEXT:    br label [[EXIT:%.*]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       for.body:
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[N]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ADD3:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
+; IF-EVL-NEXT:    [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; IF-EVL-NEXT:    [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP20]]
+; IF-EVL-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1
+; IF-EVL-NEXT:    [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; IF-EVL-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP21]]
+; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 2
+; IF-EVL-NEXT:    [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; IF-EVL-NEXT:    [[ADD2:%.*]] = add nsw i32 [[ADD1]], [[TMP22]]
+; IF-EVL-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 3
+; IF-EVL-NEXT:    [[TMP23:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
+; IF-EVL-NEXT:    [[ADD3]] = add nsw i32 [[ADD2]], [[TMP23]]
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; IF-EVL-NEXT:    [[EXITCOND:%.*]] = icmp sgt i64 [[IV_NEXT]], 0
+; IF-EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP14:![0-9]+]]
+; IF-EVL:       exit:
+; IF-EVL-NEXT:    [[ADD3_LCSSA:%.*]] = phi i32 [ [[ADD3]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
+; IF-EVL-NEXT:    ret i32 [[ADD3_LCSSA]]
+;
+; NO-VP-LABEL: @load_factor_4_reverse(
+; NO-VP-NEXT:  entry:
+; NO-VP-NEXT:    [[TMP0:%.*]] = add nsw i64 [[N:%.*]], -1
+; NO-VP-NEXT:    [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 0)
+; NO-VP-NEXT:    [[TMP1:%.*]] = sub i64 [[N]], [[SMIN]]
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
+; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
+; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP:       vector.ph:
+; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
+; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[TMP6:%.*]] = sub i64 [[N]], [[N_VEC]]
+; NO-VP-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[N]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i64> [[TMP7]], splat (i64 -1)
+; NO-VP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> [[BROADCAST_SPLAT]], [[TMP8]]
+; NO-VP-NEXT:    [[TMP9:%.*]] = mul i64 -1, [[TMP5]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NO-VP:       vector.body:
+; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP16:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]], i32 0
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP11:%.*]] = add <vscale x 4 x i32> [[VEC_PHI]], [[WIDE_MASKED_GATHER]]
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP12:%.*]] = add <vscale x 4 x i32> [[TMP11]], [[WIDE_MASKED_GATHER3]]
+; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 2
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER4:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP13]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP14:%.*]] = add <vscale x 4 x i32> [[TMP12]], [[WIDE_MASKED_GATHER4]]
+; NO-VP-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i64> [[VEC_IND]], i32 3
+; NO-VP-NEXT:    [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> [[TMP15]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; NO-VP-NEXT:    [[TMP16]] = add <vscale x 4 x i32> [[TMP14]], [[WIDE_MASKED_GATHER5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT2]]
+; NO-VP-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP16]])
+; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NO-VP:       scalar.ph:
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ [[N]], [[ENTRY:%.*]] ]
+; NO-VP-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
+; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
+; NO-VP:       for.body:
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD3:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 0
+; NO-VP-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT:    [[ADD:%.*]] = add nsw i32 [[RDX]], [[TMP19]]
+; NO-VP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 1
+; NO-VP-NEXT:    [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT:    [[ADD1:%.*]] = add nsw i32 [[ADD]], [[TMP20]]
+; NO-VP-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 2
+; NO-VP-NEXT:    [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; NO-VP-NEXT:    [[ADD2:%.*]] = add nsw i32 [[ADD1]], [[TMP21]]
+; NO-VP-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i64 [[IV]], i32 3
+; NO-VP-NEXT:    [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4
+; NO-VP-NEXT:    [[ADD3]] = add nsw i32 [[ADD2]], [[TMP22]]
+; NO-VP-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
+; NO-VP-NEXT:    [[EXITCOND:%.*]] = icmp sgt i64 [[IV_NEXT]], 0
+; NO-VP-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP13:![0-9]+]]
+; NO-VP:       exit:
+; NO-VP-NEXT:    [[ADD3_LCSSA:%.*]] = phi i32 [ [[ADD3]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
+; NO-VP-NEXT:    ret i32 [[ADD3_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ %n, %entry ], [ %iv.next, %for.body ]
+  %rdx = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 0
+  %0 = load i32, ptr %arrayidx, align 4
+  %add = add nsw i32 %rdx, %0
+  %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 1
+  %1 = load i32, ptr %arrayidx, align 4
+  %add1 = add nsw i32 %add, %1
+  %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 2
+  %2 = load i32, ptr %arrayidx2, align 4
+  %add2 = add nsw i32 %add1, %2
+  %arrayidx3 = getelementptr inbounds [4 x i32], ptr %a, i64 %iv, i32 3
+  %3 = load i32, ptr %arrayidx3, align 4
+  %add3 = add nsw i32 %add2, %3
+  %iv.next = add nsw i64 %iv, -1
+  %exitcond = icmp sgt i64 %iv.next, 0
+  br i1 %exitcond, label %for.body, label %exit
+
+exit:
+  ret i32 %add3
+}
+
+define void @store_factor_4_reverse(i32 %n, ptr noalias %a) {
+; IF-EVL-LABEL: @store_factor_4_reverse(
+; IF-EVL-NEXT:  entry:
+; IF-EVL-NEXT:    [[TMP0:%.*]] = add nsw i32 [[TMP6:%.*]], -1
+; IF-EVL-NEXT:    [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 0)
+; IF-EVL-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP6]], [[SMIN]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL:       vector.ph:
+; IF-EVL-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP6]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i32> [[TMP7]], splat (i32 -1)
+; IF-EVL-NEXT:    [[INDUCTION3:%.*]] = add <vscale x 4 x i32> [[BROADCAST_SPLAT2]], [[TMP8]]
+; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
+; IF-EVL:       vector.body:
+; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[VEC_IND4:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT9:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i32 [ [[TMP1]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i32 -1, [[TMP9]]
+; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP10]], i64 0
+; IF-EVL-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; IF-EVL-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND4]], i32 0
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP13]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 1
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 2
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND4]], i32 3
+; IF-EVL-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND4]], <vscale x 4 x ptr> align 4 [[TMP16]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
+; IF-EVL-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP9]], [[EVL_BASED_IV]]
+; IF-EVL-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP9]]
+; IF-EVL-NEXT:    [[VEC_IND_NEXT9]] = add <vscale x 4 x i32> [[VEC_IND4]], [[BROADCAST_SPLAT6]]
+; IF-EVL-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_EVL_NEXT]], [[TMP1]]
+; IF-EVL-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; IF-EVL:       middle.block:
+; IF-EVL-NEXT:    br label [[EXIT:%.*]]
+; IF-EVL:       scalar.ph:
+; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
+; IF-EVL:       for.body:
+; IF-EVL-NEXT:    [[TMP19:%.*]] = phi i32 [ [[TMP6]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP19]], i32 0
+; IF-EVL-NEXT:    store i32 [[TMP19]], ptr [[ARRAYIDX]], align 4
+; IF-EVL-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP19]], i32 1
+; IF-EVL-NEXT:    store i32 [[TMP19]], ptr [[ARRAYIDX1]], align 4
+; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP19]], i32 2
+; IF-EVL-NEXT:    store i32 [[TMP19]], ptr [[ARRAYIDX2]], align 4
+; IF-EVL-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP19]], i32 3
+; IF-EVL-NEXT:    store i32 [[TMP19]], ptr [[ARRAYIDX3]], align 4
+; IF-EVL-NEXT:    [[IV_NEXT]] = add nsw i32 [[TMP19]], -1
+; IF-EVL-NEXT:    [[EXITCOND:%.*]] = icmp sgt i32 [[IV_NEXT]], 0
+; IF-EVL-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP16:![0-9]+]]
+; IF-EVL:       exit:
+; IF-EVL-NEXT:    ret void
+;
+; NO-VP-LABEL: @store_factor_4_reverse(
+; NO-VP-NEXT:  entry:
+; NO-VP-NEXT:    [[TMP0:%.*]] = add nsw i32 [[TMP10:%.*]], -1
+; NO-VP-NEXT:    [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 0)
+; NO-VP-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP10]], [[SMIN]]
+; NO-VP-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 2
+; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], [[TMP3]]
+; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP:       vector.ph:
+; NO-VP-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
+; NO-VP-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP4]], 4
+; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], [[TMP13]]
+; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
+; NO-VP-NEXT:    [[TMP6:%.*]] = sub i32 [[TMP10]], [[N_VEC]]
+; NO-VP-NEXT:    [[TMP11:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP10]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT:    [[TMP12:%.*]] = mul <vscale x 4 x i32> [[TMP11]], splat (i32 -1)
+; NO-VP-NEXT:    [[INDUCTION5:%.*]] = add <vscale x 4 x i32> [[BROADCAST_SPLAT4]], [[TMP12]]
+; NO-VP-NEXT:    [[TMP14:%.*]] = mul i32 -1, [[TMP13]]
+; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT6:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP14]], i64 0
+; NO-VP-NEXT:    [[BROADCAST_SPLAT7:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT6]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NO-VP:       vector.body:
+; NO-VP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[VEC_IND8:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT9:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-NEXT:    [[TMP15:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], <vscale x 4 x i32> [[VEC_IND8]], i32 0
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND8]], <vscale x 4 x ptr> [[TMP15]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND8]], i32 1
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND8]], <vscale x 4 x ptr> [[TMP16]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND8]], i32 2
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND8]], <vscale x 4 x ptr> [[TMP17]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], <vscale x 4 x i32> [[VEC_IND8]], i32 3
+; NO-VP-NEXT:    call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[VEC_IND8]], <vscale x 4 x ptr> [[TMP18]], i32 4, <vscale x 4 x i1> splat (i1 true))
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP13]]
+; NO-VP-NEXT:    [[VEC_IND_NEXT9]] = add <vscale x 4 x i32> [[VEC_IND8]], [[BROADCAST_SPLAT7]]
+; NO-VP-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; NO-VP:       middle.block:
+; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; NO-VP-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; NO-VP:       scalar.ph:
+; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ [[TMP10]], [[ENTRY:%.*]] ]
+; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
+; NO-VP:       for.body:
+; NO-VP-NEXT:    [[TMP20:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 0
+; NO-VP-NEXT:    store i32 [[TMP20]], ptr [[ARRAYIDX]], align 4
+; NO-VP-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 1
+; NO-VP-NEXT:    store i32 [[TMP20]], ptr [[ARRAYIDX1]], align 4
+; NO-VP-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 2
+; NO-VP-NEXT:    store i32 [[TMP20]], ptr [[ARRAYIDX2]], align 4
+; NO-VP-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], i32 [[TMP20]], i32 3
+; NO-VP-NEXT:    store i32 [[TMP20]], ptr [[ARRAYIDX3]], align 4
+; NO-VP-NEXT:    [[IV_NEXT]] = add nsw i32 [[TMP20]], -1
+; NO-VP-NEXT:    [[EXITCOND:%.*]] = icmp sgt i32 [[IV_NEXT]], 0
+; NO-VP-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
+; NO-VP:       exit:
+; NO-VP-NEXT:    ret void
+;
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ %n, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 0
+  store i32 %iv, ptr %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 1
+  store i32 %iv, ptr %arrayidx1, align 4
+  %arrayidx2 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 2
+  store i32 %iv, ptr %arrayidx2, align 4
+  %arrayidx3 = getelementptr inbounds [4 x i32], ptr %a, i32 %iv, i32 3
+  store i32 %iv, ptr %arrayidx3, align 4
+  %iv.next = add nsw i32 %iv, -1
+  %exitcond = icmp sgt i32 %iv.next, 0
+  br i1 %exitcond, label %for.body, label %exit
+
+exit:
+  ret void
+}
+
 !0 = distinct !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-intermediate-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-intermediate-store.ll
index be6ae1d4a0f6b..5e16468d5dd6c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-intermediate-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-intermediate-store.ll
@@ -31,14 +31,6 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-OUTLOOP-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; IF-EVL-OUTLOOP-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[ENTRY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP10:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.body:
@@ -89,14 +81,6 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-INLOOP-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; IF-EVL-INLOOP-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
-; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP9]], 1
-; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP10]]
-; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP9]]
-; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -136,7 +120,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]], ptr [[ADDR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-OUTLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; NO-VP-OUTLOOP:       vector.memcheck:
@@ -152,8 +136,6 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP5]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-OUTLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP8:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       vector.body:
@@ -162,7 +144,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; NO-VP-OUTLOOP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP10]], align 4, !alias.scope [[META0:![0-9]+]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP12]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; NO-VP-OUTLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; NO-VP-OUTLOOP:       middle.block:
@@ -191,7 +173,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]], ptr [[ADDR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; NO-VP-INLOOP:       vector.memcheck:
@@ -207,8 +189,6 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP5]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-INLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-INLOOP:       vector.body:
 ; NO-VP-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -217,7 +197,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-INLOOP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP9]], align 4, !alias.scope [[META0:![0-9]+]]
 ; NO-VP-INLOOP-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-INLOOP-NEXT:    [[TMP12]] = add i32 [[VEC_PHI]], [[TMP11]]
-; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; NO-VP-INLOOP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
 ; NO-VP-INLOOP-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
 ; NO-VP-INLOOP:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll
index d474a03b90ee9..dd048ba8a43b9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-iv32.ll
@@ -12,18 +12,10 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N:%.*]], [[TMP8]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INDEX_EVL_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[TMP11:%.*]] = phi i32 [ [[N]], [[ENTRY]] ], [ [[AVL_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = phi i32 [ [[N:%.*]], [[ENTRY]] ], [ [[AVL_NEXT:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[TMP11]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i32 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
@@ -36,10 +28,9 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY1:%.*]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY1:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; IF-EVL-NEXT:    [[IV1:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[IV1]]
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[IV1]]
@@ -53,7 +44,7 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
 ; NO-VP-LABEL: @iv32(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP10:%.*]] = shl nuw i32 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP10]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -61,8 +52,6 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
 ; NO-VP-NEXT:    [[TMP11:%.*]] = mul nuw i32 [[TMP1]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP11]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP2]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -70,7 +59,7 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP4]], align 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i32 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP6]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP12]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP11]]
 ; NO-VP-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-known-no-overflow.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-known-no-overflow.ll
index 06c6bfe64dd2f..b40609877f89b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-known-no-overflow.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-known-no-overflow.ll
@@ -16,14 +16,6 @@ define void @trip_count_max_1024(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TC]], i64 1)
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP6]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -41,10 +33,9 @@ define void @trip_count_max_1024(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[P]], i64 [[I]]
 ; CHECK-NEXT:    [[X:%.*]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[Y:%.*]] = add i64 [[X]], 1
@@ -85,14 +76,6 @@ define void @overflow_at_0(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK:       [[LOOP_PREHEADER]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
-; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP6]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -110,10 +93,9 @@ define void @overflow_at_0(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[P]], i64 [[I]]
 ; CHECK-NEXT:    [[X:%.*]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[Y:%.*]] = add i64 [[X]], 1
@@ -154,14 +136,6 @@ define void @no_overflow_at_0(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK:       [[LOOP_PREHEADER]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC_ADD]], [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -179,10 +153,9 @@ define void @no_overflow_at_0(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT_LOOPEXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[P]], i64 [[I]]
 ; CHECK-NEXT:    [[X:%.*]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[Y:%.*]] = add i64 [[X]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll
index 5f407fcca259d..042e54a694d21 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-masked-loadstore.ll
@@ -12,18 +12,10 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[TMP9:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP9]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
@@ -40,10 +32,9 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[I_011:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; IF-EVL-NEXT:    [[I_011:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_INC:%.*]] ], [ 0, [[SCALAR_PH]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I_011]]
 ; IF-EVL-NEXT:    [[TMP23:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP1:%.*]] = icmp ne i32 [[TMP23]], 0
@@ -64,7 +55,7 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; NO-VP-LABEL: @masked_loadstore(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
+; NO-VP-NEXT:    [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP12]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -72,8 +63,6 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[INC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
@@ -84,7 +73,7 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; NO-VP-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP8]], i32 4, <vscale x 4 x i1> [[TMP7]], <vscale x 4 x i32> poison)
 ; NO-VP-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[WIDE_MASKED_LOAD]]
 ; NO-VP-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP9]], ptr [[TMP8]], i32 4, <vscale x 4 x i1> [[TMP7]])
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[INC]]
 ; NO-VP-NEXT:    br i1 [[TMP10]], label [[FOR_INC:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-ordered-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-ordered-reduction.ll
index 59d137011d288..a2b2b5bc05141 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-ordered-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-ordered-reduction.ll
@@ -14,19 +14,11 @@ define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[TMP9:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP9:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP9]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
@@ -39,12 +31,10 @@ define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ADD]] = fadd float [[TMP17]], [[SUM_07]]
@@ -58,7 +48,7 @@ define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
 ; NO-VP-LABEL: @fadd(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N1:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; NO-VP:       vector.ph:
@@ -66,8 +56,6 @@ define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N1]], [[TMP3]]
 ; NO-VP-NEXT:    [[N:%.*]] = sub i64 [[N1]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -75,7 +63,7 @@ define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
 ; NO-VP-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[IV]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[ARRAYIDX]], align 4
 ; NO-VP-NEXT:    [[ADD]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[SUM_07]], <vscale x 4 x float> [[WIDE_LOAD]])
-; NO-VP-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP5]]
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
 ; NO-VP-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; NO-VP-NEXT:    br i1 [[EXITCOND_NOT]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
index 2d5718b24cb30..326e543276be2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
@@ -12,20 +12,12 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
@@ -40,12 +32,10 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ADD]] = add nsw i32 [[TMP18]], [[RDX]]
@@ -59,7 +49,7 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @add(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -67,8 +57,6 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -77,7 +65,7 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
 ; NO-VP-NEXT:    [[TMP10]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -230,20 +218,12 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
@@ -258,12 +238,10 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[OR:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[OR:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[OR]] = or i32 [[TMP18]], [[RDX]]
@@ -277,7 +255,7 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @or(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -285,8 +263,6 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -295,7 +271,7 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
 ; NO-VP-NEXT:    [[TMP10]] = or <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -341,20 +317,12 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> splat (i32 -1), i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
@@ -369,12 +337,10 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.and.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[AND:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[AND]] = and i32 [[TMP18]], [[RDX]]
@@ -388,7 +354,7 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @and(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -396,8 +362,6 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x i32> splat (i32 -1), i32 [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -406,7 +370,7 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
 ; NO-VP-NEXT:    [[TMP10]] = and <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -452,20 +416,12 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
@@ -480,12 +436,10 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[XOR:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[XOR:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[XOR]] = xor i32 [[TMP18]], [[RDX]]
@@ -499,7 +453,7 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @xor(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -507,8 +461,6 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -517,7 +469,7 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP8]], align 4
 ; NO-VP-NEXT:    [[TMP10]] = xor <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -563,21 +515,13 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -593,12 +537,10 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[SMIN:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP19]], [[RDX]]
@@ -613,7 +555,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @smin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -621,8 +563,6 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -633,7 +573,7 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
 ; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -681,21 +621,13 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -711,12 +643,10 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.smax.nxv4i32(<vscale x 4 x i32> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SMAX:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[SMAX:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp sgt i32 [[TMP19]], [[RDX]]
@@ -731,7 +661,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @smax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -739,8 +669,6 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -751,7 +679,7 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = icmp sgt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
 ; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -799,21 +727,13 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -829,12 +749,10 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.umin.nxv4i32(<vscale x 4 x i32> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[UMIN:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[UMIN:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp ult i32 [[TMP19]], [[RDX]]
@@ -849,7 +767,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @umin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -857,8 +775,6 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -869,7 +785,7 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = icmp ult <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
 ; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -917,21 +833,13 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -947,12 +855,10 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vector.reduce.umax.nxv4i32(<vscale x 4 x i32> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[UMAX:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[UMAX:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp ugt i32 [[TMP19]], [[RDX]]
@@ -967,7 +873,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @umax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -975,8 +881,6 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -987,7 +891,7 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = icmp ugt <vscale x 4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
 ; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1035,20 +939,12 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
@@ -1063,12 +959,10 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP14]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ADD]] = fadd reassoc float [[TMP18]], [[RDX]]
@@ -1082,7 +976,7 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; NO-VP-LABEL: @fadd(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1090,8 +984,6 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -1100,7 +992,7 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; NO-VP-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
 ; NO-VP-NEXT:    [[TMP10]] = fadd reassoc <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1253,21 +1145,13 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -1283,12 +1167,10 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32(<vscale x 4 x float> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MIN:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START]], [[SCALAR_PH]] ], [ [[MIN:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP:%.*]] = fcmp fast olt float [[TMP19]], [[RDX]]
@@ -1303,7 +1185,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-LABEL: @fmin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1311,8 +1193,6 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -1323,7 +1203,7 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = fcmp fast olt <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
 ; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1371,21 +1251,13 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ [[BROADCAST_SPLAT]], [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -1401,12 +1273,10 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:    [[TMP18:%.*]] = call fast float @llvm.vector.reduce.fmax.nxv4f32(<vscale x 4 x float> [[TMP15]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MAX:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START]], [[SCALAR_PH]] ], [ [[MAX:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP:%.*]] = fcmp fast ogt float [[TMP19]], [[RDX]]
@@ -1421,7 +1291,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-LABEL: @fmax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1429,8 +1299,6 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -1441,7 +1309,7 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = fcmp fast ogt <vscale x 4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
 ; NO-VP-NEXT:    [[TMP10]] = select <vscale x 4 x i1> [[TMP9]], <vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[VEC_PHI]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1705,20 +1573,12 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x float> [ [[TMP9]], [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]])
@@ -1735,12 +1595,10 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = call reassoc float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, <vscale x 4 x float> [[TMP17]])
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi float [ [[START]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
@@ -1756,7 +1614,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; NO-VP-LABEL: @fmuladd(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1764,8 +1622,6 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -1776,7 +1632,7 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; NO-VP-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP10]], align 4
 ; NO-VP-NEXT:    [[TMP12]] = call reassoc <vscale x 4 x float> @llvm.fmuladd.nxv4f32(<vscale x 4 x float> [[WIDE_LOAD]], <vscale x 4 x float> [[WIDE_LOAD1]], <vscale x 4 x float> [[VEC_PHI]])
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1826,19 +1682,11 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -1855,12 +1703,10 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP19]], i32 [[INV:%.*]], i32 [[START:%.*]]
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ANYOF:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ANYOF:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = icmp slt i32 [[TMP20]], 3
@@ -1875,7 +1721,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-LABEL: @anyof_icmp(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -1883,8 +1729,6 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1893,7 +1737,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 3)
 ; NO-VP-NEXT:    [[TMP10]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -1943,19 +1787,11 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[AVL:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x float> @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP11]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP9]])
@@ -1972,12 +1808,10 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP19]], i32 [[INV:%.*]], i32 [[START:%.*]]
 ; IF-EVL-NEXT:    br label [[FOR_END:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[START]], [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ANYOF:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[RDX:%.*]] = phi i32 [ [[START]], [[SCALAR_PH]] ], [ [[ANYOF:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[CMP_I:%.*]] = fcmp fast olt float [[TMP20]], 3.000000e+00
@@ -1992,7 +1826,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-LABEL: @anyof_fcmp(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -2000,8 +1834,6 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2010,7 +1842,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP7]], align 4
 ; NO-VP-NEXT:    [[TMP9:%.*]] = fcmp fast olt <vscale x 4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
 ; NO-VP-NEXT:    [[TMP10]] = or <vscale x 4 x i1> [[VEC_PHI]], [[TMP9]]
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; NO-VP:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
index e2db28d54ac58..c804fd9ceb7a9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reverse-load-store.ll
@@ -12,14 +12,6 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -53,12 +45,10 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[LOOPEND:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[STARTVAL]], [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[ADD_PHI:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
-; IF-EVL-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[ADD_PHI:%.*]] = phi i64 [ [[STARTVAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[I:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ADD]] = add i64 [[ADD_PHI]], -1
 ; IF-EVL-NEXT:    [[GEPL:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[ADD]]
 ; IF-EVL-NEXT:    [[TMP:%.*]] = load i32, ptr [[GEPL]], align 4
@@ -73,7 +63,7 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; NO-VP-LABEL: @reverse_load_store(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -81,8 +71,6 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = sub i64 [[STARTVAL:%.*]], [[N_VEC]]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
 ; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
@@ -91,22 +79,22 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; NO-VP-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 [[STARTVAL]], [[INDEX]]
 ; NO-VP-NEXT:    [[TMP8:%.*]] = add i64 [[OFFSET_IDX]], -1
 ; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[PTR:%.*]], i64 [[TMP8]]
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul i64 0, [[TMP5]]
-; NO-VP-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT:    [[TMP10:%.*]] = mul i64 0, [[TMP3]]
+; NO-VP-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP3]], 1
 ; NO-VP-NEXT:    [[TMP12:%.*]] = mul i64 -1, [[TMP11]]
 ; NO-VP-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 [[TMP10]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP13]], i64 [[TMP12]]
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
 ; NO-VP-NEXT:    [[REVERSE:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[WIDE_LOAD]])
 ; NO-VP-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[PTR2:%.*]], i64 [[TMP8]]
-; NO-VP-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP5]]
-; NO-VP-NEXT:    [[TMP17:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT:    [[TMP16:%.*]] = mul i64 0, [[TMP3]]
+; NO-VP-NEXT:    [[TMP17:%.*]] = sub i64 [[TMP3]], 1
 ; NO-VP-NEXT:    [[TMP18:%.*]] = mul i64 -1, [[TMP17]]
 ; NO-VP-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 [[TMP16]]
 ; NO-VP-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP18]]
 ; NO-VP-NEXT:    [[REVERSE1:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[REVERSE]])
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[REVERSE1]], ptr [[TMP20]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -154,14 +142,6 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -201,12 +181,10 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[LOOPEND:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[STARTVAL]], [[ENTRY:%.*]] ]
-; IF-EVL-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[ADD_PHI:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_INC:%.*]] ]
-; IF-EVL-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_INC]] ]
+; IF-EVL-NEXT:    [[ADD_PHI:%.*]] = phi i64 [ [[STARTVAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_INC:%.*]] ]
+; IF-EVL-NEXT:    [[I:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_INC]] ]
 ; IF-EVL-NEXT:    [[ADD]] = add i64 [[ADD_PHI]], -1
 ; IF-EVL-NEXT:    [[GEPL:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i32 [[I]]
 ; IF-EVL-NEXT:    [[TMP:%.*]] = load i32, ptr [[GEPL]], align 4
@@ -228,7 +206,7 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; NO-VP-LABEL: @reverse_load_store_masked(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[ENTRY:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -236,8 +214,6 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = sub i64 [[STARTVAL1:%.*]], [[N_VEC]]
 ; NO-VP-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
 ; NO-VP-NEXT:    br label [[FOR_BODY:%.*]]
@@ -250,8 +226,8 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; NO-VP-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP9]], align 4
 ; NO-VP-NEXT:    [[TMP10:%.*]] = icmp slt <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 100)
 ; NO-VP-NEXT:    [[TMP11:%.*]] = getelementptr i32, ptr [[PTR1:%.*]], i64 [[TMP8]]
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul i64 0, [[TMP5]]
-; NO-VP-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT:    [[TMP12:%.*]] = mul i64 0, [[TMP3]]
+; NO-VP-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP3]], 1
 ; NO-VP-NEXT:    [[TMP14:%.*]] = mul i64 -1, [[TMP13]]
 ; NO-VP-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[TMP11]], i64 [[TMP12]]
 ; NO-VP-NEXT:    [[TMP16:%.*]] = getelementptr i32, ptr [[TMP15]], i64 [[TMP14]]
@@ -259,15 +235,15 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; NO-VP-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP16]], i32 4, <vscale x 4 x i1> [[REVERSE]], <vscale x 4 x i32> poison)
 ; NO-VP-NEXT:    [[REVERSE2:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[WIDE_MASKED_LOAD]])
 ; NO-VP-NEXT:    [[TMP17:%.*]] = getelementptr i32, ptr [[PTR2:%.*]], i64 [[TMP8]]
-; NO-VP-NEXT:    [[TMP18:%.*]] = mul i64 0, [[TMP5]]
-; NO-VP-NEXT:    [[TMP19:%.*]] = sub i64 [[TMP5]], 1
+; NO-VP-NEXT:    [[TMP18:%.*]] = mul i64 0, [[TMP3]]
+; NO-VP-NEXT:    [[TMP19:%.*]] = sub i64 [[TMP3]], 1
 ; NO-VP-NEXT:    [[TMP20:%.*]] = mul i64 -1, [[TMP19]]
 ; NO-VP-NEXT:    [[TMP21:%.*]] = getelementptr i32, ptr [[TMP17]], i64 [[TMP18]]
 ; NO-VP-NEXT:    [[TMP22:%.*]] = getelementptr i32, ptr [[TMP21]], i64 [[TMP20]]
 ; NO-VP-NEXT:    [[REVERSE3:%.*]] = call <vscale x 4 x i1> @llvm.vector.reverse.nxv4i1(<vscale x 4 x i1> [[TMP10]])
 ; NO-VP-NEXT:    [[REVERSE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.reverse.nxv4i32(<vscale x 4 x i32> [[REVERSE2]])
 ; NO-VP-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[REVERSE4]], ptr [[TMP22]], i32 4, <vscale x 4 x i1> [[REVERSE3]])
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -334,14 +310,6 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -385,10 +353,9 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[LOOP:%.*]]
 ; IF-EVL:       loop:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 1024, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; IF-EVL-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[X:%.*]] = load i8, ptr [[GEP_A]], align 1
 ; IF-EVL-NEXT:    [[GEP_B:%.*]] = getelementptr i8, ptr [[B]], i8 [[X]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
index 1c78b25e114c4..21e87fdc7586c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
@@ -15,14 +15,6 @@ define void @test(ptr %p) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 200, [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -41,10 +33,9 @@ define void @test(ptr %p) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[LOOP:%.*]]
 ; IF-EVL:       loop:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; IF-EVL-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 8
 ; IF-EVL-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 200
@@ -59,7 +50,7 @@ define void @test(ptr %p) {
 ; NO-VP-LABEL: @test(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; NO-VP:       vector.ph:
@@ -67,8 +58,6 @@ define void @test(ptr %p) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; NO-VP-NEXT:    br label [[LOOP:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -77,7 +66,7 @@ define void @test(ptr %p) {
 ; NO-VP-NEXT:    [[TMP7:%.*]] = add i64 [[IV]], 200
 ; NO-VP-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP8]], align 8
-; NO-VP-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP5]]
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -138,10 +127,9 @@ define void @test_may_clobber1(ptr %p) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[LOOP:%.*]]
 ; IF-EVL:       loop:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; IF-EVL-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; IF-EVL-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 100
@@ -171,10 +159,9 @@ define void @test_may_clobber1(ptr %p) {
 ; NO-VP:       middle.block:
 ; NO-VP-NEXT:    br label [[EXIT:%.*]]
 ; NO-VP:       scalar.ph:
-; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; NO-VP-NEXT:    br label [[LOOP:%.*]]
 ; NO-VP:       loop:
-; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; NO-VP-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; NO-VP-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; NO-VP-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 100
@@ -274,10 +261,9 @@ define void @test_may_clobber3(ptr %p) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[LOOP:%.*]]
 ; IF-EVL:       loop:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; IF-EVL-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; IF-EVL-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 10
@@ -307,10 +293,9 @@ define void @test_may_clobber3(ptr %p) {
 ; NO-VP:       middle.block:
 ; NO-VP-NEXT:    br label [[EXIT:%.*]]
 ; NO-VP:       scalar.ph:
-; NO-VP-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; NO-VP-NEXT:    br label [[LOOP:%.*]]
 ; NO-VP:       loop:
-; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; NO-VP-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; NO-VP-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; NO-VP-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 10
@@ -346,14 +331,6 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 200, [[TMP2]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -372,10 +349,9 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[LOOP:%.*]]
 ; IF-EVL:       loop:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; IF-EVL-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; IF-EVL-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 8192
@@ -390,7 +366,7 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; NO-VP-LABEL: @trivial_due_max_vscale(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; NO-VP:       vector.ph:
@@ -398,8 +374,6 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; NO-VP-NEXT:    br label [[LOOP:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
@@ -408,7 +382,7 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; NO-VP-NEXT:    [[TMP7:%.*]] = add i64 [[IV]], 8192
 ; NO-VP-NEXT:    [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]]
 ; NO-VP-NEXT:    store <vscale x 2 x i64> [[WIDE_LOAD]], ptr [[TMP8]], align 32
-; NO-VP-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP5]]
+; NO-VP-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -454,12 +428,6 @@ define void @no_high_lmul_or_interleave(ptr %p) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP7]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 3002, [[TMP1]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -480,10 +448,9 @@ define void @no_high_lmul_or_interleave(ptr %p) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[EXIT:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[LOOP:%.*]]
 ; IF-EVL:       loop:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; IF-EVL-NEXT:    [[A1:%.*]] = getelementptr i64, ptr [[P]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[V:%.*]] = load i64, ptr [[A1]], align 32
 ; IF-EVL-NEXT:    [[OFFSET:%.*]] = add i64 [[IV]], 1024
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-uniform-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-uniform-store.ll
index 687a2e7bf9312..55c4c4a411837 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-uniform-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-uniform-store.ll
@@ -9,20 +9,12 @@ target triple = "riscv64-unknown-linux-gnu"
 define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
 ; CHECK-LABEL: define void @lshift_significand(
 ; CHECK-SAME: i32 [[N:%.*]], ptr writeonly captures(none) [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp eq i32 [[N]], 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[CMP1_PEEL]], i64 2, i64 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 3, [[SPEC_SELECT]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP7]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -47,10 +39,9 @@ define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[SPEC_SELECT]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[TMP22:%.*]] = sub nuw nsw i64 1, [[IV1]]
 ; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr i64, ptr [[DST]], i64 [[TMP22]]
 ; CHECK-NEXT:    store i64 0, ptr [[ARRAYIDX14]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index 24649729f43ba..7c9cade024285 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -7,41 +7,34 @@ target triple = "riscv64-unknown-linux-gnu"
 define void @test_pr98413_zext_removed(ptr %src, ptr noalias %dst, i64 %x) {
 ; CHECK-LABEL: define void @test_pr98413_zext_removed(
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 97, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 97, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 97, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc <vscale x 8 x i64> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 97, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 8 [[TMP8]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = trunc <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    [[TMP11:%.*]] = and <vscale x 8 x i8> [[TMP6]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT:    store <vscale x 8 x i8> [[TMP11]], ptr [[TMP12]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[TMP5]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP11]], ptr align 1 [[TMP12]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[TMP7]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 97
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 97, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP_SRC]], align 8
 ; CHECK-NEXT:    [[EXT_L:%.*]] = zext i16 [[L]] to i64
@@ -51,7 +44,7 @@ define void @test_pr98413_zext_removed(ptr %src, ptr noalias %dst, i64 %x) {
 ; CHECK-NEXT:    store i8 [[TRUNC_AND]], ptr [[GEP_DST]], align 1
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 96
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -78,41 +71,34 @@ exit:
 define void @test_pr98413_sext_removed(ptr %src, ptr noalias %dst, i64 %x) {
 ; CHECK-LABEL: define void @test_pr98413_sext_removed(
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 97, [[TMP1]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 97, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 97, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc <vscale x 8 x i64> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 97, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 8, i1 true)
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[TMP7]]
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i16>, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = call <vscale x 8 x i16> @llvm.vp.load.nxv8i16.p0(ptr align 8 [[TMP8]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = trunc <vscale x 8 x i16> [[WIDE_LOAD]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    [[TMP11:%.*]] = and <vscale x 8 x i8> [[TMP6]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[TMP7]]
-; CHECK-NEXT:    store <vscale x 8 x i8> [[TMP11]], ptr [[TMP12]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[TMP5]]
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.vp.store.nxv8i8.p0(<vscale x 8 x i8> [[TMP11]], ptr align 1 [[TMP12]], <vscale x 8 x i1> splat (i1 true), i32 [[TMP13]])
+; CHECK-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP14]], [[TMP7]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP14]]
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 97
+; CHECK-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 97, [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i16, ptr [[GEP_SRC]], align 8
 ; CHECK-NEXT:    [[EXT_L:%.*]] = sext i16 [[L]] to i64
@@ -122,7 +108,7 @@ define void @test_pr98413_sext_removed(ptr %src, ptr noalias %dst, i64 %x) {
 ; CHECK-NEXT:    store i8 [[TRUNC_AND]], ptr [[GEP_DST]], align 1
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV]], 96
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -150,17 +136,9 @@ exit:
 define void @truncate_to_i1_used_by_branch(i8 %x, ptr %dst) #0 {
 ; CHECK-LABEL: define void @truncate_to_i1_used_by_branch(
 ; CHECK-SAME: i8 [[X:%.*]], ptr [[DST:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
-; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 [[TMP3]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 9, [[TMP12]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP10]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -172,14 +150,13 @@ define void @truncate_to_i1_used_by_branch(i8 %x, ptr %dst) #0 {
 ; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i32 [[TMP6]], [[EVL_BASED_IV]]
 ; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_EVL_NEXT]], 9
-; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[F_039:%.*]] = phi i8 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[F_039:%.*]] = phi i8 [ 0, %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = or i8 23, [[X]]
 ; CHECK-NEXT:    [[EXTRACT_T:%.*]] = trunc i8 [[TMP4]] to i1
 ; CHECK-NEXT:    br i1 [[EXTRACT_T]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
@@ -256,20 +233,11 @@ exit:
 define void @icmp_only_first_op_truncated(ptr noalias %dst, i32 %x, i64 %N, i64 %v, ptr noalias %src) #1 {
 ; CHECK-LABEL: define void @icmp_only_first_op_truncated(
 ; CHECK-SAME: ptr noalias [[DST:%.*]], i32 [[X:%.*]], i64 [[N:%.*]], i64 [[V:%.*]], ptr noalias [[SRC:%.*]]) #[[ATTR2]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[T:%.*]] = trunc i64 [[N]] to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[V]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[N]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[T]], i64 0
@@ -284,20 +252,22 @@ define void @icmp_only_first_op_truncated(ptr noalias %dst, i32 %x, i64 %N, i64
 ; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> [[BROADCAST_SPLAT4]], i32 8, <vscale x 2 x i1> [[TMP8]], <vscale x 2 x double> poison)
-; CHECK-NEXT:    call void @llvm.masked.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[WIDE_MASKED_GATHER]], <vscale x 2 x ptr> [[BROADCAST_SPLAT6]], i32 8, <vscale x 2 x i1> [[TMP8]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ [[TMP0]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 2 x double> @llvm.vp.gather.nxv2f64.nxv2p0(<vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT4]], <vscale x 2 x i1> [[TMP8]], i32 [[TMP14]])
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv2f64.nxv2p0(<vscale x 2 x double> [[WIDE_MASKED_GATHER]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT6]], <vscale x 2 x i1> [[TMP8]], i32 [[TMP14]])
+; CHECK-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[INDEX_EVL_NEXT]] = add i64 [[TMP11]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[T1:%.*]] = trunc i64 [[N]] to i32
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[T1]], [[T]]
 ; CHECK-NEXT:    br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
@@ -344,15 +314,15 @@ attributes #0 = { "target-features"="+64bit,+v,+zvl256b" }
 attributes #1 = { "target-features"="+64bit,+v" }
 
 ;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META7:![0-9]+]], [[META2]]}
-; CHECK: [[META7]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
-; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META2]], [[META1]]}
+; CHECK: [[META2]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]], [[META1]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]], [[META1]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META1]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]], [[META3]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
index dfdc893570810..413a013e6cf31 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
@@ -8,17 +8,9 @@
 define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) {
 ; CHECK-LABEL: define void @truncate_to_minimal_bitwidths_widen_cast_recipe(
 ; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 9, [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -33,10 +25,9 @@ define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_SRC1:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV1]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i8, ptr [[GEP_SRC1]], align 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP11]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
index 6476373c08401..4eac0b1ad1a15 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
@@ -20,14 +20,6 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP8]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[DSTV]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
index 568aa953de511..e3a93cbf450ae 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
@@ -9,43 +9,36 @@ target triple = "riscv64"
 define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {
 ; SCALABLE-LABEL: define void @uniform_load(
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; SCALABLE-NEXT:  [[ENTRY:.*]]:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SCALABLE-NEXT:  [[ENTRY:.*:]]
+; SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; SCALABLE-NEXT:    [[TMP7:%.*]] = load i64, ptr [[B]], align 8
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP7]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; SCALABLE-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP8]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; SCALABLE-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; SCALABLE-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
+; SCALABLE-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; SCALABLE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1025
+; SCALABLE-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; SCALABLE:       [[MIDDLE_BLOCK]]:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; SCALABLE:       [[SCALAR_PH]]:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; SCALABLE:       [[FOR_BODY]]:
-; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; SCALABLE-NEXT:    [[V:%.*]] = load i64, ptr [[B]], align 8
 ; SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
 ; SCALABLE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; SCALABLE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
-; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; SCALABLE:       [[FOR_END]]:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -85,17 +78,9 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
 ;
 ; TF-SCALABLE-LABEL: define void @uniform_load(
 ; TF-SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; TF-SCALABLE-NEXT:  [[ENTRY:.*]]:
+; TF-SCALABLE-NEXT:  [[ENTRY:.*:]]
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TF-SCALABLE:       [[VECTOR_BODY]]:
 ; TF-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -114,10 +99,9 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
 ; TF-SCALABLE:       [[MIDDLE_BLOCK]]:
 ; TF-SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; TF-SCALABLE:       [[SCALAR_PH]]:
-; TF-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; TF-SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; TF-SCALABLE:       [[FOR_BODY]]:
-; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; TF-SCALABLE-NEXT:    [[V:%.*]] = load i64, ptr [[B]], align 8
 ; TF-SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; TF-SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
@@ -148,7 +132,7 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
@@ -156,8 +140,6 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
 ; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -166,9 +148,9 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; SCALABLE-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP8]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; SCALABLE:       [[MIDDLE_BLOCK]]:
 ; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
 ; SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
@@ -182,7 +164,7 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
 ; SCALABLE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; SCALABLE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
-; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; SCALABLE:       [[FOR_END]]:
 ; SCALABLE-NEXT:    [[V_LCSSA:%.*]] = phi i64 [ [[V]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
 ; SCALABLE-NEXT:    ret i64 [[V_LCSSA]]
@@ -226,7 +208,7 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
 ; TF-SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; TF-SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; TF-SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; TF-SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
@@ -234,8 +216,6 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
 ; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TF-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TF-SCALABLE:       [[VECTOR_BODY]]:
 ; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -244,7 +224,7 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; TF-SCALABLE-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[ARRAYIDX]], align 8
-; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP5]]
+; TF-SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[IV]], [[TMP3]]
 ; TF-SCALABLE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; TF-SCALABLE-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; TF-SCALABLE:       [[MIDDLE_BLOCK]]:
@@ -285,47 +265,45 @@ for.end:
 define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {
 ; SCALABLE-LABEL: define void @conditional_uniform_load(
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SCALABLE-NEXT:  [[ENTRY:.*]]:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SCALABLE-NEXT:  [[ENTRY:.*:]]
+; SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[B]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; SCALABLE-NEXT:    [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 1)
 ; SCALABLE-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP7]]
-; SCALABLE-NEXT:    [[TMP9:%.*]] = mul i64 1, [[TMP5]]
-; SCALABLE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
-; SCALABLE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP17:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP17]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP17]] to i64
+; SCALABLE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; SCALABLE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP18:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
+; SCALABLE-NEXT:    [[TMP11:%.*]] = icmp ult <vscale x 4 x i32> [[TMP18]], [[BROADCAST_SPLAT4]]
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = icmp ugt <vscale x 4 x i64> [[VEC_IND]], splat (i64 10)
-; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.masked.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> [[BROADCAST_SPLAT]], i32 8, <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i64> poison)
-; SCALABLE-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], <vscale x 4 x i64> zeroinitializer
+; SCALABLE-NEXT:    [[TMP13:%.*]] = select <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP10]], <vscale x 4 x i1> zeroinitializer
+; SCALABLE-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i64> @llvm.vp.gather.nxv4i64.nxv4p0(<vscale x 4 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP10]], i32 [[TMP17]])
+; SCALABLE-NEXT:    [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[TMP13]], <vscale x 4 x i64> [[WIDE_MASKED_GATHER]], <vscale x 4 x i64> zeroinitializer
 ; SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; SCALABLE-NEXT:    store <vscale x 4 x i64> [[PREDPHI]], ptr [[TMP12]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; SCALABLE-NEXT:    call void @llvm.vp.store.nxv4i64.p0(<vscale x 4 x i64> [[PREDPHI]], ptr align 8 [[TMP12]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP17]])
+; SCALABLE-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP17]] to i64
+; SCALABLE-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP15]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP15]]
 ; SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
-; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1025
+; SCALABLE-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
 ; SCALABLE:       [[MIDDLE_BLOCK]]:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; SCALABLE:       [[SCALAR_PH]]:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; SCALABLE:       [[FOR_BODY]]:
-; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
 ; SCALABLE-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[IV]], 10
 ; SCALABLE-NEXT:    br i1 [[CMP]], label %[[DO_LOAD:.*]], label %[[LATCH]]
 ; SCALABLE:       [[DO_LOAD]]:
@@ -337,7 +315,7 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
 ; SCALABLE-NEXT:    store i64 [[PHI]], ptr [[ARRAYIDX]], align 8
 ; SCALABLE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; SCALABLE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
-; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; SCALABLE:       [[FOR_END]]:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -391,17 +369,9 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
 ;
 ; TF-SCALABLE-LABEL: define void @conditional_uniform_load(
 ; TF-SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; TF-SCALABLE-NEXT:  [[ENTRY:.*]]:
+; TF-SCALABLE-NEXT:  [[ENTRY:.*:]]
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
-; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[B]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
@@ -416,8 +386,7 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP7]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
-; TF-SCALABLE-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP11]]
-; TF-SCALABLE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0
+; TF-SCALABLE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP11]], i64 0
 ; TF-SCALABLE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[TMP16:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; TF-SCALABLE-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = icmp ult <vscale x 4 x i32> [[TMP16]], [[BROADCAST_SPLAT4]]
@@ -436,10 +405,9 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
 ; TF-SCALABLE:       [[MIDDLE_BLOCK]]:
 ; TF-SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; TF-SCALABLE:       [[SCALAR_PH]]:
-; TF-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; TF-SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; TF-SCALABLE:       [[FOR_BODY]]:
-; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
 ; TF-SCALABLE-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[IV]], 10
 ; TF-SCALABLE-NEXT:    br i1 [[CMP]], label %[[DO_LOAD:.*]], label %[[LATCH]]
 ; TF-SCALABLE:       [[DO_LOAD]]:
@@ -481,43 +449,36 @@ for.end:
 define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %n) {
 ; SCALABLE-LABEL: define void @uniform_load_unaligned(
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SCALABLE-NEXT:  [[ENTRY:.*]]:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SCALABLE-NEXT:  [[ENTRY:.*:]]
+; SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; SCALABLE-NEXT:    [[TMP6:%.*]] = load i64, ptr [[B]], align 1
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP6]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; SCALABLE-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP8]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SCALABLE-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP8]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP10]])
+; SCALABLE-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP10]] to i64
+; SCALABLE-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; SCALABLE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1025
+; SCALABLE-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; SCALABLE:       [[MIDDLE_BLOCK]]:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; SCALABLE:       [[SCALAR_PH]]:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; SCALABLE:       [[FOR_BODY]]:
-; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; SCALABLE-NEXT:    [[V:%.*]] = load i64, ptr [[B]], align 1
 ; SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
 ; SCALABLE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; SCALABLE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
-; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; SCALABLE:       [[FOR_END]]:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -557,17 +518,9 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
 ;
 ; TF-SCALABLE-LABEL: define void @uniform_load_unaligned(
 ; TF-SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; TF-SCALABLE-NEXT:  [[ENTRY:.*]]:
+; TF-SCALABLE-NEXT:  [[ENTRY:.*:]]
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TF-SCALABLE:       [[VECTOR_BODY]]:
 ; TF-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -586,10 +539,9 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
 ; TF-SCALABLE:       [[MIDDLE_BLOCK]]:
 ; TF-SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; TF-SCALABLE:       [[SCALAR_PH]]:
-; TF-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; TF-SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; TF-SCALABLE:       [[FOR_BODY]]:
-; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; TF-SCALABLE-NEXT:    [[V:%.*]] = load i64, ptr [[B]], align 1
 ; TF-SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; TF-SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
@@ -618,43 +570,36 @@ for.end:
 define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
 ; SCALABLE-LABEL: define void @uniform_store(
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SCALABLE-NEXT:  [[ENTRY:.*]]:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SCALABLE-NEXT:  [[ENTRY:.*:]]
+; SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[B]], align 8
 ; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; SCALABLE-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; SCALABLE-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
+; SCALABLE-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP9]] to i64
+; SCALABLE-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; SCALABLE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1025
+; SCALABLE-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; SCALABLE:       [[MIDDLE_BLOCK]]:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; SCALABLE:       [[SCALAR_PH]]:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; SCALABLE:       [[FOR_BODY]]:
-; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[B]], align 8
 ; SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
 ; SCALABLE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; SCALABLE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
-; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; SCALABLE:       [[FOR_END]]:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -694,17 +639,9 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ;
 ; TF-SCALABLE-LABEL: define void @uniform_store(
 ; TF-SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; TF-SCALABLE-NEXT:  [[ENTRY:.*]]:
+; TF-SCALABLE-NEXT:  [[ENTRY:.*:]]
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -723,10 +660,9 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; TF-SCALABLE:       [[MIDDLE_BLOCK]]:
 ; TF-SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; TF-SCALABLE:       [[SCALAR_PH]]:
-; TF-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; TF-SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; TF-SCALABLE:       [[FOR_BODY]]:
-; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; TF-SCALABLE-NEXT:    store i64 [[V]], ptr [[B]], align 8
 ; TF-SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; TF-SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
@@ -755,55 +691,46 @@ for.end:
 define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
 ; SCALABLE-LABEL: define void @uniform_store_of_loop_varying(
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SCALABLE-NEXT:  [[ENTRY:.*]]:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SCALABLE-NEXT:  [[ENTRY:.*:]]
+; SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; SCALABLE-NEXT:    [[TMP13:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 1)
+; SCALABLE-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP13]]
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
-; SCALABLE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[INDEX]], i64 0
+; SCALABLE-NEXT:    [[TMP10:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; SCALABLE-NEXT:    [[TMP8:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
 ; SCALABLE-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
-; SCALABLE-NEXT:    [[TMP7:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP6]]
-; SCALABLE-NEXT:    [[TMP8:%.*]] = mul <vscale x 2 x i64> [[TMP7]], splat (i64 1)
-; SCALABLE-NEXT:    [[TMP9:%.*]] = add <vscale x 2 x i64> [[DOTSPLAT]], [[TMP8]]
-; SCALABLE-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 0
-; SCALABLE-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
-; SCALABLE-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
-; SCALABLE-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 2
-; SCALABLE-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], 1
-; SCALABLE-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 2 x i64> [[TMP9]], i32 [[TMP14]]
-; SCALABLE-NEXT:    store i64 [[TMP15]], ptr [[B]], align 8
+; SCALABLE-NEXT:    call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT1]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
 ; SCALABLE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; SCALABLE-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP16]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; SCALABLE-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP16]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; SCALABLE-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP11]], [[TMP10]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP11]]
+; SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
+; SCALABLE-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1025
+; SCALABLE-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; SCALABLE:       [[MIDDLE_BLOCK]]:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; SCALABLE:       [[SCALAR_PH]]:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; SCALABLE:       [[FOR_BODY]]:
-; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; SCALABLE-NEXT:    store i64 [[IV]], ptr [[B]], align 8
 ; SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
 ; SCALABLE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; SCALABLE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
-; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; SCALABLE:       [[FOR_END]]:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -847,17 +774,9 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ;
 ; TF-SCALABLE-LABEL: define void @uniform_store_of_loop_varying(
 ; TF-SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; TF-SCALABLE-NEXT:  [[ENTRY:.*]]:
+; TF-SCALABLE-NEXT:  [[ENTRY:.*:]]
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
@@ -872,8 +791,7 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ; TF-SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; TF-SCALABLE-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; TF-SCALABLE-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP9]] to i64
-; TF-SCALABLE-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP13]]
-; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP13]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[VEC_IND]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
 ; TF-SCALABLE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
@@ -887,10 +805,9 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ; TF-SCALABLE:       [[MIDDLE_BLOCK]]:
 ; TF-SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; TF-SCALABLE:       [[SCALAR_PH]]:
-; TF-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; TF-SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; TF-SCALABLE:       [[FOR_BODY]]:
-; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; TF-SCALABLE-NEXT:    store i64 [[IV]], ptr [[B]], align 8
 ; TF-SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; TF-SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
@@ -919,18 +836,9 @@ for.end:
 define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
 ; SCALABLE-LABEL: define void @conditional_uniform_store(
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SCALABLE-NEXT:  [[ENTRY:.*]]:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SCALABLE-NEXT:  [[ENTRY:.*:]]
+; SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
@@ -938,29 +846,31 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
 ; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; SCALABLE-NEXT:    [[TMP8:%.*]] = mul <vscale x 2 x i64> [[TMP6]], splat (i64 1)
 ; SCALABLE-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP8]]
-; SCALABLE-NEXT:    [[TMP9:%.*]] = mul i64 1, [[TMP5]]
-; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP9]], i64 0
-; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; SCALABLE-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP7:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; SCALABLE-NEXT:    [[TMP14:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP14]], i64 0
+; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = icmp ugt <vscale x 2 x i64> [[VEC_IND]], splat (i64 10)
-; SCALABLE-NEXT:    call void @llvm.masked.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[BROADCAST_SPLAT1]], <vscale x 2 x ptr> [[BROADCAST_SPLAT2]], i32 8, <vscale x 2 x i1> [[TMP10]])
+; SCALABLE-NEXT:    call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[BROADCAST_SPLAT1]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT2]], <vscale x 2 x i1> [[TMP10]], i32 [[TMP7]])
 ; SCALABLE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; SCALABLE-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT1]], ptr [[TMP12]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; SCALABLE-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT1]], ptr align 8 [[TMP12]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP7]])
+; SCALABLE-NEXT:    [[TMP15:%.*]] = zext i32 [[TMP7]] to i64
+; SCALABLE-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP15]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP15]]
 ; SCALABLE-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; SCALABLE-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; SCALABLE-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1025
+; SCALABLE-NEXT:    br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; SCALABLE:       [[MIDDLE_BLOCK]]:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; SCALABLE:       [[SCALAR_PH]]:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; SCALABLE:       [[FOR_BODY]]:
-; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
 ; SCALABLE-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[IV]], 10
 ; SCALABLE-NEXT:    br i1 [[CMP]], label %[[DO_STORE:.*]], label %[[LATCH]]
 ; SCALABLE:       [[DO_STORE]]:
@@ -971,7 +881,7 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
 ; SCALABLE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; SCALABLE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
-; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; SCALABLE:       [[FOR_END]]:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -1024,17 +934,9 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
 ;
 ; TF-SCALABLE-LABEL: define void @conditional_uniform_store(
 ; TF-SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; TF-SCALABLE-NEXT:  [[ENTRY:.*]]:
+; TF-SCALABLE-NEXT:  [[ENTRY:.*:]]
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
@@ -1049,8 +951,7 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
 ; TF-SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; TF-SCALABLE-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; TF-SCALABLE-NEXT:    [[TMP11:%.*]] = zext i32 [[TMP9]] to i64
-; TF-SCALABLE-NEXT:    [[TMP8:%.*]] = mul i64 1, [[TMP11]]
-; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[TMP10:%.*]] = icmp ugt <vscale x 2 x i64> [[VEC_IND]], splat (i64 10)
 ; TF-SCALABLE-NEXT:    call void @llvm.vp.scatter.nxv2i64.nxv2p0(<vscale x 2 x i64> [[BROADCAST_SPLAT1]], <vscale x 2 x ptr> align 8 [[BROADCAST_SPLAT2]], <vscale x 2 x i1> [[TMP10]], i32 [[TMP9]])
@@ -1065,10 +966,9 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
 ; TF-SCALABLE:       [[MIDDLE_BLOCK]]:
 ; TF-SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; TF-SCALABLE:       [[SCALAR_PH]]:
-; TF-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; TF-SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; TF-SCALABLE:       [[FOR_BODY]]:
-; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
 ; TF-SCALABLE-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[IV]], 10
 ; TF-SCALABLE-NEXT:    br i1 [[CMP]], label %[[DO_STORE:.*]], label %[[LATCH]]
 ; TF-SCALABLE:       [[DO_STORE]]:
@@ -1108,43 +1008,36 @@ for.end:
 define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 %v, i64 %n) {
 ; SCALABLE-LABEL: define void @uniform_store_unaligned(
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SCALABLE-NEXT:  [[ENTRY:.*]]:
-; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
-; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SCALABLE-NEXT:  [[ENTRY:.*:]]
+; SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
-; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
-; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
-; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
-; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[AVL:%.*]] = phi i64 [ 1025, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SCALABLE-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[B]], align 1
 ; SCALABLE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
-; SCALABLE-NEXT:    store <vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8
-; SCALABLE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; SCALABLE-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; SCALABLE-NEXT:    call void @llvm.vp.store.nxv2i64.p0(<vscale x 2 x i64> [[BROADCAST_SPLAT]], ptr align 8 [[TMP7]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP9]])
+; SCALABLE-NEXT:    [[TMP10:%.*]] = zext i32 [[TMP9]] to i64
+; SCALABLE-NEXT:    [[INDEX_EVL_NEXT]] = add nuw i64 [[TMP10]], [[INDEX]]
+; SCALABLE-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP10]]
+; SCALABLE-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_EVL_NEXT]], 1025
+; SCALABLE-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
 ; SCALABLE:       [[MIDDLE_BLOCK]]:
-; SCALABLE-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1025, [[N_VEC]]
-; SCALABLE-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; SCALABLE:       [[SCALAR_PH]]:
-; SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; SCALABLE:       [[FOR_BODY]]:
-; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[B]], align 1
 ; SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
 ; SCALABLE-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; SCALABLE-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 1025
-; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; SCALABLE-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; SCALABLE:       [[FOR_END]]:
 ; SCALABLE-NEXT:    ret void
 ;
@@ -1184,17 +1077,9 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap
 ;
 ; TF-SCALABLE-LABEL: define void @uniform_store_unaligned(
 ; TF-SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; TF-SCALABLE-NEXT:  [[ENTRY:.*]]:
+; TF-SCALABLE-NEXT:  [[ENTRY:.*:]]
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
-; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
-; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -1213,10 +1098,9 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap
 ; TF-SCALABLE:       [[MIDDLE_BLOCK]]:
 ; TF-SCALABLE-NEXT:    br label %[[FOR_END:.*]]
 ; TF-SCALABLE:       [[SCALAR_PH]]:
-; TF-SCALABLE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; TF-SCALABLE-NEXT:    br label %[[FOR_BODY:.*]]
 ; TF-SCALABLE:       [[FOR_BODY]]:
-; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; TF-SCALABLE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; TF-SCALABLE-NEXT:    store i64 [[V]], ptr [[B]], align 1
 ; TF-SCALABLE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; TF-SCALABLE-NEXT:    store i64 [[V]], ptr [[ARRAYIDX]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vector-loop-backedge-elimination-with-evl.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vector-loop-backedge-elimination-with-evl.ll
index 7c1ec9ab6c5fd..0e3507e11c230 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vector-loop-backedge-elimination-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vector-loop-backedge-elimination-with-evl.ll
@@ -5,17 +5,9 @@
 define void @foo(ptr %arg) #0 {
 ; CHECK-LABEL: define void @foo(
 ; CHECK-SAME: ptr [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 3, [[TMP2]]
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 3, i32 2, i1 true)
@@ -24,10 +16,9 @@ define void @foo(ptr %arg) #0 {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr [3 x i64], ptr [[ARG]], i64 0, i64 [[IV]]
 ; CHECK-NEXT:    store i64 0, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
@@ -51,10 +42,60 @@ exit:
   ret void
 }
 
+define i32 @test_remove_iv(i32 %start) #0 {
+; CHECK-LABEL: define i32 @test_remove_iv(
+; CHECK-SAME: i32 [[START:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ [[TMP2]], %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i32 [ 6, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <vscale x 4 x i32> [[VEC_PHI]], splat (i32 3)
+; CHECK-NEXT:    [[TMP5]] = call <vscale x 4 x i32> @llvm.vp.merge.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[VEC_PHI]], i32 [[TMP3]])
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP3]]
+; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32(<vscale x 4 x i32> [[TMP5]])
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ [[START]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED_NEXT]] = xor i32 [[RED]], 3
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 5
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[RED_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %red = phi i32 [ %start , %entry ], [ %red.next, %loop ]
+  %red.next = xor i32 %red, 3
+  %iv.next = add i32 %iv, 1
+  %ec = icmp eq i32 %iv, 5
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret i32 %red.next
+}
+
 attributes #0 = { vscale_range(2,2) }
 
 ;.
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK: [[META2]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META4:![0-9]+]], [[META1]]}
+; CHECK: [[META4]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
index 85116feab6a33..4da482f594350 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
@@ -12,18 +12,10 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
-; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
-; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP8]]
-; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
-; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_EVL_NEXT:%.*]], [[VECTOR_BODY]] ]
-; IF-EVL-NEXT:    [[TMP11:%.*]] = phi i64 [ [[N]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
+; IF-EVL-NEXT:    [[TMP11:%.*]] = phi i64 [ [[N:%.*]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[TMP11]], i32 4, i1 true)
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[EVL_BASED_IV]]
 ; IF-EVL-NEXT:    [[VP_OP_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP14]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP12]])
@@ -40,10 +32,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
@@ -60,7 +51,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-LABEL: @foo(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -68,8 +59,6 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -80,7 +69,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP11:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP11]], ptr [[TMP12]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; NO-VP-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
index bda983983b70a..e4344b9eb37d5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vf-will-not-generate-any-vector-insts.ll
@@ -17,17 +17,20 @@ define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <2 x ptr> poison, ptr [[DST]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <2 x ptr> [[BROADCAST_SPLATINSERT2]], <2 x ptr> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4, !alias.scope [[META0:![0-9]+]]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT4:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT5:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT4]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> [[BROADCAST_SPLAT5]], <2 x ptr> [[BROADCAST_SPLAT3]], i32 4, <2 x i1> splat (i1 true)), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> [[BROADCAST_SPLAT5]], <2 x ptr> [[BROADCAST_SPLAT3]], i32 4, <2 x i1> splat (i1 true)), !alias.scope [[META3]], !noalias [[META0]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[EVL_BASED_IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[AVL:%.*]] = phi i64 [ 100, %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 4, i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[SRC]], align 4, !alias.scope [[META0:![0-9]+]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP6]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT2]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[BROADCAST_SPLAT3]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP5]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP7]], [[EVL_BASED_IV]]
+; CHECK-NEXT:    [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
 ; CHECK-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -41,7 +44,7 @@ define void @vf_will_not_generate_any_vector_insts(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    store i32 [[DOTPRE]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[TMP3]] = add nuw i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[TMP3]], 100
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -65,8 +68,9 @@ exit:
 ; CHECK: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
 ; CHECK: [[META3]] = !{[[META4:![0-9]+]]}
 ; CHECK: [[META4]] = distinct !{[[META4]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]], [[META8:![0-9]+]]}
 ; CHECK: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META6]]}
+; CHECK: [[META7]] = !{!"llvm.loop.isvectorized.tailfoldingstyle", !"evl"}
+; CHECK: [[META8]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META6]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll
index d7c9ce4216c1e..46695221c27db 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-riscv-vector-reverse.ll
@@ -10,8 +10,7 @@
 ; RUN: -disable-output < %s 2>&1 | FileCheck %s
 
 define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocapture noundef readonly %B, i32 noundef signext %n) {
-; CHECK: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' {
-; CHECK-NEXT: Live-in vp<[[VF:%.+]]> = VF
+; CHECK: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF={1}' {
 ; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF
 ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: vp<[[OTC:%.+]]> = original trip-count
@@ -21,41 +20,42 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT: Successor(s): scalar.ph, vector.ph
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.ph:
-; CHECK-NEXT:   vp<[[RESUME_IV_A:%.+]]> = DERIVED-IV ir<%n> + vp<[[VTC]]> * ir<-1>
-; CHECK-NEXT:   vp<[[RESUME_IV_B:%.+]]> = DERIVED-IV ir<%n> + vp<[[VTC]]> * ir<-1>
 ; CHECK-NEXT: Successor(s): vector loop
 ; CHECK-EMPTY:
 ; CHECK-NEXT: <x1> vector loop: {
 ; CHECK-NEXT:   vector.body:
 ; CHECK-NEXT:     EMIT vp<[[INDUCTION:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[INDEX_NEXT:%.+]]>
-; CHECK-NEXT:     vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[INDUCTION]]> * ir<-1>
-; CHECK-NEXT:     vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>, vp<[[VF]]>
+; CHECK-NEXT:     EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI vp<[[EVL_PHI:%.+]]> = phi ir<0>, vp<[[IV_NEXT:%.+]]>
+; CHECK-NEXT:     EMIT-SCALAR vp<[[AVL:%.+]]> = phi [ vp<[[OTC]]>, vector.ph ], [ vp<[[AVL_NEXT:%.+]]>, vector.body ]
+; CHECK-NEXT:     EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
+; CHECK-NEXT:     vp<[[DERIVED_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[EVL_PHI]]> * ir<-1>
+; CHECK-NEXT:     vp<[[SCALAR_STEPS:%.+]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<-1>, vp<[[EVL]]>
 ; CHECK-NEXT:     CLONE ir<[[IDX:%.+]]> = add nsw vp<[[SCALAR_STEPS]]>, ir<-1>
 ; CHECK-NEXT:     CLONE ir<[[IDX_PROM:%.+]]> = zext ir<[[IDX]]>
 ; CHECK-NEXT:     CLONE ir<[[ARRAY_IDX_B:%.+]]> = getelementptr inbounds ir<[[B:%.+]]>, ir<[[IDX_PROM]]>
-; CHECK-NEXT:     vp<[[VEC_END_PTR_B:%.+]]> = vector-end-pointer inbounds ir<[[ARRAY_IDX_B]]>, vp<[[VF]]>
-; CHECK-NEXT:     WIDEN ir<[[VAL_B:%.+]]> = load vp<[[VEC_END_PTR_B]]>
+; CHECK-NEXT:     vp<[[VEC_END_PTR_B:%.+]]> = vector-end-pointer ir<[[ARRAY_IDX_B]]>, vp<[[EVL]]>
+; CHECK-NEXT:     WIDEN ir<[[VAL_B:%.+]]> = vp.load vp<[[VEC_END_PTR_B]]>, vp<[[EVL]]>
 ; CHECK-NEXT:     WIDEN ir<[[ADD_RESULT:%.+]]> = add ir<[[VAL_B]]>, ir<1>
 ; CHECK-NEXT:     CLONE ir<[[ARRAY_IDX_A:%.+]]> = getelementptr inbounds ir<[[A:%.+]]>, ir<[[IDX_PROM]]>
-; CHECK-NEXT:     vp<[[VEC_END_PTR_A:%.+]]> = vector-end-pointer inbounds ir<[[ARRAY_IDX_A]]>, vp<[[VF]]>
-; CHECK-NEXT:     WIDEN store vp<[[VEC_END_PTR_A]]>, ir<[[ADD_RESULT]]>
-; CHECK-NEXT:     EMIT vp<[[INDEX_NEXT]]> = add nuw vp<[[INDUCTION]]>, vp<[[VFxUF]]>
+; CHECK-NEXT:     vp<[[VEC_END_PTR_A:%.+]]> = vector-end-pointer ir<[[ARRAY_IDX_A]]>, vp<[[EVL]]>
+; CHECK-NEXT:     WIDEN vp.store vp<[[VEC_END_PTR_A]]>, ir<[[ADD_RESULT]]>, vp<[[EVL]]>
+; CHECK-NEXT:     EMIT vp<[[IV_NEXT]]> = add vp<[[EVL]]>, vp<[[EVL_PHI]]>
+; CHECK-NEXT:     EMIT vp<[[AVL_NEXT]]> = sub nuw vp<[[AVL]]>, vp<[[EVL]]>
+; CHECK-NEXT:     EMIT vp<[[INDEX_NEXT]]> = add vp<[[INDUCTION]]>, vp<[[VFxUF]]>
 ; CHECK-NEXT:     EMIT branch-on-count vp<[[INDEX_NEXT]]>, vp<[[VTC]]>
 ; CHECK-NEXT:   No successors
 ; CHECK-NEXT: }
 ; CHECK-NEXT: Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[CMP:%.+]]> = icmp eq vp<[[OTC]]>, vp<[[VTC]]>
-; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
-; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>, scalar.ph
+; CHECK-NEXT: Successor(s): ir-bb<for.cond.cleanup>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<for.cond.cleanup>:
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: scalar.ph:
-; CHECK-NEXT:   EMIT-SCALAR vp<%bc.resume.val> = phi [ vp<[[RESUME_IV_A]]>, middle.block ], [ ir<%n>, ir-bb<entry> ]
-; CHECK-NEXT:   EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ vp<[[RESUME_IV_B]]>, middle.block ], [ ir<%n>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<%bc.resume.val> = phi [ ir<%n>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR vp<%bc.resume.val>.1 = phi [ ir<%n>, ir-bb<entry> ]
 ; CHECK-NEXT: Successor(s): ir-bb<for.body>
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
index 082e3266e7c8f..19294714de5d6 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
@@ -6,7 +6,7 @@ target triple = "systemz-unknown-linux-unknown"
 define void @test_scalar_steps_target_instruction_cost(ptr %dst) {
 ; CHECK-LABEL: define void @test_scalar_steps_target_instruction_cost(
 ; CHECK-SAME: ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -39,10 +39,9 @@ define void @test_scalar_steps_target_instruction_cost(ptr %dst) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[IV]], ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs-03.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs-03.ll
index 13c443c4d579f..b4eebcc050e29 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs-03.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/mem-interleaving-costs-03.ll
@@ -12,7 +12,7 @@
 define noundef i32 @fun(i32 %argc, ptr nocapture readnone %argv) {
 entry:
   %l_4774.i = alloca [4 x [2 x i128]], align 8
-  call void @llvm.lifetime.start.p0(i64 128, ptr nonnull %l_4774.i)
+  call void @llvm.lifetime.start.p0(ptr nonnull %l_4774.i)
   br label %for.cond4.preheader.i
 
 for.cond4.preheader.i:                            ; preds = %for.cond4.preheader.i, %entry
@@ -31,13 +31,13 @@ func_1.exit:                                      ; preds = %for.cond4.preheader
   %cmp200.i = icmp ne i128 %0, 0
   %conv202.i = zext i1 %cmp200.i to i64
   %call203.i = tail call i64 @safe_sub_func_int64_t_s_s(i64 noundef %conv202.i, i64 noundef 9139899272418802852)
-  call void @llvm.lifetime.end.p0(i64 128, ptr nonnull %l_4774.i)
+  call void @llvm.lifetime.end.p0(ptr nonnull %l_4774.i)
   br label %for.cond
 
 for.cond:                                         ; preds = %for.cond, %func_1.exit
   br label %for.cond
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare dso_local i64 @safe_sub_func_int64_t_s_s(i64, i64)
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll
index 02a876a3fda67..ab402b1ed311b 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/pr47665.ll
@@ -93,10 +93,9 @@ define void @test(ptr %p, i40 %a) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i40 [[A]], 24
 ; CHECK-NEXT:    [[ASHR:%.*]] = ashr i40 [[SHL]], 28
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i40 [[ASHR]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
index e0fc73f669946..05b8332079ae7 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
@@ -65,12 +65,10 @@ define void @func_21() {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[LV:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[LV:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds [5 x i32], ptr @A, i64 0, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[LV]] = load i32, ptr [[A_PTR]], align 4
 ; CHECK-NEXT:    [[B_PTR:%.*]] = getelementptr inbounds [5 x i32], ptr @B, i64 0, i64 [[INDVARS_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll b/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll
index a91bc656cc7ef..87f81881be32d 100644
--- a/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll
@@ -11,7 +11,7 @@ target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 define void @test_scalar_iv_steps_used_by_replicate_and_first_lane_only_vpinst(ptr noalias %dst, ptr noalias %src.1) {
 ; CHECK-LABEL: define void @test_scalar_iv_steps_used_by_replicate_and_first_lane_only_vpinst(
 ; CHECK-SAME: ptr noalias [[DST:%.*]], ptr noalias [[SRC_1:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -67,10 +67,9 @@ define void @test_scalar_iv_steps_used_by_replicate_and_first_lane_only_vpinst(p
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[MUL_IV:%.*]] = mul nsw i64 [[IV]], 4
 ; CHECK-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds i8, ptr [[SRC_1]], i64 [[MUL_IV]]
 ; CHECK-NEXT:    [[L_1:%.*]] = load i8, ptr [[GEP_SRC_1]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
index e26221eeefd2d..ab8eae6823963 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
 ; REQUIRES: asserts
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=loop-vectorize,instcombine -S -debug-only=loop-vectorize -disable-output -print-after=instcombine 2>&1 | FileCheck %s
 ; RUN: opt < %s -passes=loop-vectorize -force-vector-width=2 -S | FileCheck %s -check-prefix=FORCE
@@ -18,32 +19,119 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK-NOT:   LV: Found uniform instruction: %tmp3 = getelementptr inbounds %data, ptr %d, i64 0, i32 0, i64 %i
 ; CHECK-NOT:   LV: Found uniform instruction: %i = phi i64 [ %i.next, %for.body ], [ 0, %entry ]
 ; CHECK-NOT:   LV: Found uniform instruction: %i.next = add nuw nsw i64 %i, 5
-; CHECK:       define void @PR31671(
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x float> poison, float %x, i64 0
+
+
+%data = type { [32000 x float], [3 x i32], [4 x i8], [32000 x float] }
+
+define void @PR31671(float %x, ptr %d) #0 {
+; CHECK-LABEL: define void @PR31671(
+; CHECK-SAME: float [[X:%.*]], ptr [[D:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x float> poison, float [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x float> [[BROADCAST_SPLATINSERT]], <16 x float> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT:    br label %vector.body
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 5, i64 10, i64 15, i64 20, i64 25, i64 30, i64 35, i64 40, i64 45, i64 50, i64 55, i64 60, i64 65, i64 70, i64 75>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], %vector.body ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i64> [ <i64 0, i64 5, i64 10, i64 15, i64 20, i64 25, i64 30, i64 35, i64 40, i64 45, i64 50, i64 55, i64 60, i64 65, i64 70, i64 75>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 5
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds %data, ptr %d, i64 0, i32 3, i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[DATA:%.*]], ptr [[D]], i64 0, i32 3, i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <80 x float>, ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <80 x float> [[WIDE_VEC]], <80 x float> poison, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul <16 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds %data, ptr %d, i64 0, i32 0, <16 x i64> [[VEC_IND]]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <16 x ptr> [[TMP3]], i64 0
-; CHECK-NEXT:    [[WIDE_VEC1:%.*]] = load <80 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <16 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, <16 x i64> [[VEC_IND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x ptr> [[TMP2]], i64 0
+; CHECK-NEXT:    [[WIDE_VEC1:%.*]] = load <80 x float>, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <80 x float> [[WIDE_VEC1]], <80 x float> poison, <16 x i32> <i32 0, i32 5, i32 10, i32 15, i32 20, i32 25, i32 30, i32 35, i32 40, i32 45, i32 50, i32 55, i32 60, i32 65, i32 70, i32 75>
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd <16 x float> [[STRIDED_VEC2]], [[TMP2]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[TMP5]], <16 x ptr> [[TMP3]], i32 4, <16 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <16 x float> [[STRIDED_VEC2]], [[TMP1]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[TMP4]], <16 x ptr> [[TMP2]], i32 4, <16 x i1> splat (i1 true))
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <16 x i64> [[VEC_IND]], splat (i64 80)
-; CHECK:         br i1 {{.*}}, label %middle.block, label %vector.body
-
-%data = type { [32000 x float], [3 x i32], [4 x i8], [32000 x float] }
-
-define void @PR31671(float %x, ptr %d) #0 {
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 6384
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
+; FORCE-LABEL: define void @PR31671(
+; FORCE-SAME: float [[X:%.*]], ptr [[D:%.*]]) #[[ATTR0:[0-9]+]] {
+; FORCE-NEXT:  [[ENTRY:.*:]]
+; FORCE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; FORCE:       [[VECTOR_PH]]:
+; FORCE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[X]], i64 0
+; FORCE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; FORCE-NEXT:    br label %[[VECTOR_BODY:.*]]
+; FORCE:       [[VECTOR_BODY]]:
+; FORCE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; FORCE-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 5
+; FORCE-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
+; FORCE-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 5
+; FORCE-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 10
+; FORCE-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 15
+; FORCE-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 20
+; FORCE-NEXT:    [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], 25
+; FORCE-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 30
+; FORCE-NEXT:    [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], 35
+; FORCE-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[DATA:%.*]], ptr [[D]], i64 0, i32 3, i64 [[TMP0]]
+; FORCE-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 3, i64 [[TMP2]]
+; FORCE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 3, i64 [[TMP4]]
+; FORCE-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 3, i64 [[TMP6]]
+; FORCE-NEXT:    [[WIDE_VEC:%.*]] = load <10 x float>, ptr [[TMP8]], align 4
+; FORCE-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <10 x float> [[WIDE_VEC]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT:    [[WIDE_VEC1:%.*]] = load <10 x float>, ptr [[TMP9]], align 4
+; FORCE-NEXT:    [[STRIDED_VEC2:%.*]] = shufflevector <10 x float> [[WIDE_VEC1]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT:    [[WIDE_VEC3:%.*]] = load <10 x float>, ptr [[TMP10]], align 4
+; FORCE-NEXT:    [[STRIDED_VEC4:%.*]] = shufflevector <10 x float> [[WIDE_VEC3]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT:    [[WIDE_VEC5:%.*]] = load <10 x float>, ptr [[TMP11]], align 4
+; FORCE-NEXT:    [[STRIDED_VEC6:%.*]] = shufflevector <10 x float> [[WIDE_VEC5]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT:    [[TMP12:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC]]
+; FORCE-NEXT:    [[TMP13:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC2]]
+; FORCE-NEXT:    [[TMP14:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC4]]
+; FORCE-NEXT:    [[TMP15:%.*]] = fmul <2 x float> [[BROADCAST_SPLAT]], [[STRIDED_VEC6]]
+; FORCE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP0]]
+; FORCE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP1]]
+; FORCE-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP2]]
+; FORCE-NEXT:    [[TMP19:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP3]]
+; FORCE-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP4]]
+; FORCE-NEXT:    [[TMP21:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP5]]
+; FORCE-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP6]]
+; FORCE-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [[DATA]], ptr [[D]], i64 0, i32 0, i64 [[TMP7]]
+; FORCE-NEXT:    [[WIDE_VEC7:%.*]] = load <10 x float>, ptr [[TMP16]], align 4
+; FORCE-NEXT:    [[STRIDED_VEC8:%.*]] = shufflevector <10 x float> [[WIDE_VEC7]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT:    [[WIDE_VEC9:%.*]] = load <10 x float>, ptr [[TMP18]], align 4
+; FORCE-NEXT:    [[STRIDED_VEC10:%.*]] = shufflevector <10 x float> [[WIDE_VEC9]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT:    [[WIDE_VEC11:%.*]] = load <10 x float>, ptr [[TMP20]], align 4
+; FORCE-NEXT:    [[STRIDED_VEC12:%.*]] = shufflevector <10 x float> [[WIDE_VEC11]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT:    [[WIDE_VEC13:%.*]] = load <10 x float>, ptr [[TMP22]], align 4
+; FORCE-NEXT:    [[STRIDED_VEC14:%.*]] = shufflevector <10 x float> [[WIDE_VEC13]], <10 x float> poison, <2 x i32> <i32 0, i32 5>
+; FORCE-NEXT:    [[TMP24:%.*]] = fadd <2 x float> [[STRIDED_VEC8]], [[TMP12]]
+; FORCE-NEXT:    [[TMP25:%.*]] = fadd <2 x float> [[STRIDED_VEC10]], [[TMP13]]
+; FORCE-NEXT:    [[TMP26:%.*]] = fadd <2 x float> [[STRIDED_VEC12]], [[TMP14]]
+; FORCE-NEXT:    [[TMP27:%.*]] = fadd <2 x float> [[STRIDED_VEC14]], [[TMP15]]
+; FORCE-NEXT:    [[TMP28:%.*]] = extractelement <2 x float> [[TMP24]], i32 0
+; FORCE-NEXT:    store float [[TMP28]], ptr [[TMP16]], align 4
+; FORCE-NEXT:    [[TMP29:%.*]] = extractelement <2 x float> [[TMP24]], i32 1
+; FORCE-NEXT:    store float [[TMP29]], ptr [[TMP17]], align 4
+; FORCE-NEXT:    [[TMP30:%.*]] = extractelement <2 x float> [[TMP25]], i32 0
+; FORCE-NEXT:    store float [[TMP30]], ptr [[TMP18]], align 4
+; FORCE-NEXT:    [[TMP31:%.*]] = extractelement <2 x float> [[TMP25]], i32 1
+; FORCE-NEXT:    store float [[TMP31]], ptr [[TMP19]], align 4
+; FORCE-NEXT:    [[TMP32:%.*]] = extractelement <2 x float> [[TMP26]], i32 0
+; FORCE-NEXT:    store float [[TMP32]], ptr [[TMP20]], align 4
+; FORCE-NEXT:    [[TMP33:%.*]] = extractelement <2 x float> [[TMP26]], i32 1
+; FORCE-NEXT:    store float [[TMP33]], ptr [[TMP21]], align 4
+; FORCE-NEXT:    [[TMP34:%.*]] = extractelement <2 x float> [[TMP27]], i32 0
+; FORCE-NEXT:    store float [[TMP34]], ptr [[TMP22]], align 4
+; FORCE-NEXT:    [[TMP35:%.*]] = extractelement <2 x float> [[TMP27]], i32 1
+; FORCE-NEXT:    store float [[TMP35]], ptr [[TMP23]], align 4
+; FORCE-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; FORCE-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 6392
+; FORCE-NEXT:    br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; FORCE:       [[MIDDLE_BLOCK]]:
+; FORCE-NEXT:    br label %[[SCALAR_PH]]
+; FORCE:       [[SCALAR_PH]]:
+;
 entry:
   br label %for.body
 
@@ -79,39 +167,56 @@ attributes #0 = { "target-cpu"="knl" }
 ; CHECK:     LV: Found not uniform due to requiring predication:  {{%.*}} = load i32, ptr {{%.*}}, align 1
 ; CHECK:     LV: Found scalar instruction:   {{%.*}} = getelementptr inbounds [3 x i32], ptr @a, i32 0, i32 {{%.*}}
 ;
-; FORCE-LABEL: @PR40816(
-; FORCE-NEXT:  entry:
-; FORCE-NEXT:    br i1 false, label {{%.*}}, label [[VECTOR_PH:%.*]]
-; FORCE:       vector.ph:
-; FORCE-NEXT:    br label [[VECTOR_BODY:%.*]]
-; FORCE:       vector.body:
-; FORCE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
-; FORCE-NEXT:    [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ]
+;
+@a = internal constant [3 x i32] [i32 7, i32 7, i32 0], align 1
+@b = external global i32, align 1
+
+define void @PR40816() #1 {
+; CHECK-LABEL: define void @PR40816(
+; CHECK-SAME: ) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    store i32 [[TMP0]], ptr @b, align 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[TMP0]], 2
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[TMP0]], 1
+; CHECK-NEXT:    br i1 [[CMP2]], label %[[RETURN:.*]], label %[[FOR_BODY]]
+; CHECK:       [[RETURN]]:
+; CHECK-NEXT:    ret void
+;
+; FORCE-LABEL: define void @PR40816(
+; FORCE-SAME: ) #[[ATTR1:[0-9]+]] {
+; FORCE-NEXT:  [[ENTRY:.*:]]
+; FORCE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; FORCE:       [[VECTOR_PH]]:
+; FORCE-NEXT:    br label %[[VECTOR_BODY:.*]]
+; FORCE:       [[VECTOR_BODY]]:
+; FORCE-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE4:.*]] ]
+; FORCE-NEXT:    [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE4]] ]
 ; FORCE-NEXT:    [[TMP2:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 2)
 ; FORCE-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
-; FORCE-NEXT:    br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
-; FORCE:       pred.store.if:
+; FORCE-NEXT:    br i1 [[TMP3]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; FORCE:       [[PRED_STORE_IF]]:
 ; FORCE-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
 ; FORCE-NEXT:    store i32 [[TMP0]], ptr @b, align 1
-; FORCE-NEXT:    br label [[PRED_STORE_CONTINUE]]
-; FORCE:       pred.store.continue:
+; FORCE-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; FORCE:       [[PRED_STORE_CONTINUE]]:
 ; FORCE-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
-; FORCE-NEXT:    br i1 [[TMP10]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4]]
-; FORCE:       pred.store.if1:
+; FORCE-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE4]]
+; FORCE:       [[PRED_STORE_IF1]]:
 ; FORCE-NEXT:    [[TMP1:%.*]] = add i32 [[INDEX]], 1
 ; FORCE-NEXT:    store i32 [[TMP1]], ptr @b, align 1
-; FORCE-NEXT:    br label [[PRED_STORE_CONTINUE4]]
-; FORCE:       pred.store.continue2:
+; FORCE-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
+; FORCE:       [[PRED_STORE_CONTINUE4]]:
 ; FORCE-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; FORCE-NEXT:    [[VEC_IND_NEXT]] = add <2 x i8> [[VEC_IND]], splat (i8 2)
 ; FORCE-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
-; FORCE-NEXT:    br i1 [[TMP15]], label {{%.*}}, label [[VECTOR_BODY]]
+; FORCE-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; FORCE:       [[MIDDLE_BLOCK]]:
+; FORCE-NEXT:    br [[RETURN:label %.*]]
+; FORCE:       [[SCALAR_PH]]:
 ;
-@a = internal constant [3 x i32] [i32 7, i32 7, i32 0], align 1
-@b = external global i32, align 1
-
-define void @PR40816() #1 {
-
 entry:
   br label %for.body
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll b/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll
index c61b1b90f3dfe..da1a5aa3a9f04 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/constant-fold.ll
@@ -24,10 +24,9 @@ define void @f1() {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[BB3:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[BB1:%.*]] ]
 ; CHECK-NEXT:    br label [[BB2:%.*]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    [[C_1_0:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[_TMP9:%.*]], [[BB2]] ]
+; CHECK-NEXT:    [[C_1_0:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[_TMP9:%.*]], [[BB2]] ]
 ; CHECK-NEXT:    [[_TMP1:%.*]] = zext i16 0 to i64
 ; CHECK-NEXT:    [[_TMP2:%.*]] = getelementptr [1 x %rec8], ptr @a, i16 0, i64 [[_TMP1]]
 ; CHECK-NEXT:    [[_TMP6:%.*]] = sext i16 [[C_1_0]] to i64
@@ -71,53 +70,43 @@ define void @redundant_or_1(ptr %dst, i1 %c.0, i1 %c.1) {
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE8:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE8]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 2)
-; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[BROADCAST_SPLAT2]], <4 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 0
 ; CHECK-NEXT:    store i32 0, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP5]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; CHECK:       pred.store.if3:
-; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 1
 ; CHECK-NEXT:    store i32 0, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; CHECK:       pred.store.continue4:
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i1> [[TMP5]], i32 2
 ; CHECK-NEXT:    br i1 [[TMP12]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
 ; CHECK:       pred.store.if5:
-; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP13]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 2
 ; CHECK-NEXT:    store i32 0, ptr [[TMP14]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP5]], i32 3
-; CHECK-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8]]
+; CHECK-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
 ; CHECK:       pred.store.if7:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP16]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 3
 ; CHECK-NEXT:    store i32 0, ptr [[TMP17]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE8]]
 ; CHECK:       pred.store.continue8:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
-; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_LATCH]], label [[THEN_1:%.*]]
 ; CHECK:       then.1:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[IV]], 2
@@ -131,7 +120,7 @@ define void @redundant_or_1(ptr %dst, i1 %c.0, i1 %c.1) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 3
-; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
@@ -174,53 +163,43 @@ define void @redundant_or_2(ptr %dst, i1 %c.0, i1 %c.1) {
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT1]], <4 x i1> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE8:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE8]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 2)
-; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x i1> [[TMP0]], <4 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[BROADCAST_SPLAT2]], <4 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
 ; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i32 0
 ; CHECK-NEXT:    store i32 0, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE]]
 ; CHECK:       pred.store.continue:
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_STORE_IF3:%.*]], label [[PRED_STORE_CONTINUE4:%.*]]
 ; CHECK:       pred.store.if3:
-; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 1
 ; CHECK-NEXT:    store i32 0, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE4]]
 ; CHECK:       pred.store.continue4:
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
 ; CHECK:       pred.store.if5:
-; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 2
 ; CHECK-NEXT:    store i32 0, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
 ; CHECK:       pred.store.continue6:
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
-; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8]]
+; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE8:%.*]]
 ; CHECK:       pred.store.if7:
-; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[TMP15]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 3
 ; CHECK-NEXT:    store i32 0, ptr [[TMP16]], align 4
 ; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE8]]
 ; CHECK:       pred.store.continue8:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
-; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br label [[MIDDLE_BLOCK:%.*]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_LATCH]], label [[THEN_1:%.*]]
 ; CHECK:       then.1:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[IV]], 2
@@ -234,7 +213,7 @@ define void @redundant_or_2(ptr %dst, i1 %c.0, i1 %c.1) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 3
-; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
index 04e0dafba6b86..aecfc668cf293 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
@@ -62,7 +62,7 @@ exit:
 ; Test case for https://github.com/llvm/llvm-project/issues/109528.
 define i64 @second_lshr_operand_zero_via_scev() {
 ; CHECK-LABEL: define i64 @second_lshr_operand_zero_via_scev() {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[EXT_0:%.*]] = sext i8 0 to i32
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -97,12 +97,10 @@ define i64 @second_lshr_operand_zero_via_scev() {
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vector.reduce.or.v2i64(<2 x i64> [[BIN_RDX]])
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOPS:.*]]
 ; CHECK:       [[LOOPS]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOPS]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOPS]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOPS]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOPS]] ]
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i64 [[IV]], 0
 ; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = trunc i64 [[IV]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 85b475c996c79..472aa0b5b716e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -1051,12 +1051,10 @@ define i64 @live_in_known_1_via_scev() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> [[VEC_PHI]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[PH]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 3, [[PH]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_MUL:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ 3, [[SCALAR_PH]] ], [ [[RED_MUL:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[RED_MUL]] = mul nsw i64 [[RED]], [[P_EXT]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
@@ -1103,21 +1101,17 @@ define i64 @cost_loop_invariant_recipes(i1 %x, i64 %y) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl <2 x i64> [[BROADCAST_SPLAT2]], [[TMP1]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x i64> [ splat (i64 1), [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP3]] = mul <2 x i64> [[TMP2]], [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> [[TMP3]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 1, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT_I_I_I:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_MUL:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT_I_I_I:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ 1, [[SCALAR_PH]] ], [ [[RED_MUL:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[NOT_X:%.*]] = xor i1 [[X]], true
 ; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[NOT_X]] to i64
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[Y]], [[EXT]]
@@ -1158,25 +1152,21 @@ define i32 @narrowed_reduction(ptr %a, i1 %cmp) #0 {
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi <16 x i32> [ zeroinitializer, [[VECTOR_PH1]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <16 x i32> [[VEC_PHI1]], splat (i32 1)
 ; CHECK-NEXT:    [[TMP3:%.*]] = or <16 x i32> [[TMP1]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = trunc <16 x i32> [[TMP3]] to <16 x i1>
 ; CHECK-NEXT:    [[TMP7]] = zext <16 x i1> [[TMP5]] to <16 x i32>
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP5]])
 ; CHECK-NEXT:    [[TMP21:%.*]] = zext i1 [[TMP20]] to i32
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INC:%.*]], [[LOOP1]] ]
-; CHECK-NEXT:    [[OR13:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[VEC_EPILOG_PH]] ], [ [[OR:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 1, [[VEC_EPILOG_PH]] ], [ [[INC:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[OR13:%.*]] = phi i32 [ 0, [[VEC_EPILOG_PH]] ], [ [[OR:%.*]], [[LOOP1]] ]
 ; CHECK-NEXT:    [[AND:%.*]] = and i32 [[OR13]], 1
 ; CHECK-NEXT:    [[OR]] = or i32 [[AND]], [[CONV]]
 ; CHECK-NEXT:    [[INC]] = add i32 [[IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/drop-inbounds-flags-for-reverse-vector-pointer.ll b/llvm/test/Transforms/LoopVectorize/X86/drop-inbounds-flags-for-reverse-vector-pointer.ll
index 1249df4af62ad..96d8ba49328fa 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/drop-inbounds-flags-for-reverse-vector-pointer.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/drop-inbounds-flags-for-reverse-vector-pointer.ll
@@ -42,12 +42,10 @@ define i1 @fn(ptr %nno) #0 {
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP12]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 10, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY20:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC35:%.*]] ]
-; CHECK-NEXT:    [[SUM_01:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_1:%.*]], [[FOR_INC35]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 10, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC35:%.*]] ]
+; CHECK-NEXT:    [[SUM_01:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[SUM_1:%.*]], [[FOR_INC35]] ]
 ; CHECK-NEXT:    [[REM4:%.*]] = and i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[CMP21:%.*]] = icmp eq i64 [[REM4]], 0
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds nuw i32, ptr [[NNO]], i64 [[INDVARS_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
index 84e36cbb33552..c04dc7b29a6a9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
@@ -27,19 +27,11 @@ define void @drop_scalar_nuw_nsw(ptr noalias nocapture readonly %input, ptr %out
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP2]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP3]], i32 4, <4 x i1> [[TMP1]], <4 x float> poison), !invariant.load [[META0:![0-9]+]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr float, ptr [[INPUT]], i64 -1
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP0]], i32 4, <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> poison), !invariant.load [[META0:![0-9]+]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> [[WIDE_MASKED_LOAD]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -77,19 +69,11 @@ define void @drop_scalar_gep_nusw(ptr noalias nocapture readonly %input, ptr %ou
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP2]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP3]], i32 4, <4 x i1> [[TMP1]], <4 x float> poison), !invariant.load [[META0]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr nusw float, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr float, ptr [[INPUT]], i64 -1
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP0]], i32 4, <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> poison), !invariant.load [[META0]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> [[WIDE_MASKED_LOAD]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -127,19 +111,11 @@ define void @drop_scalar_gep_nuw(ptr noalias nocapture readonly %input, ptr %out
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP2]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP3]], i32 4, <4 x i1> [[TMP1]], <4 x float> poison), !invariant.load [[META0]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr nuw float, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr float, ptr [[INPUT]], i64 -1
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP0]], i32 4, <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> poison), !invariant.load [[META0]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> [[WIDE_MASKED_LOAD]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -178,19 +154,11 @@ define void @drop_nonpred_scalar_nuw_nsw(ptr noalias nocapture readonly %input,
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = xor <4 x i1> [[TMP2]], splat (i1 true)
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP1]], i32 4, <4 x i1> [[TMP3]], <4 x float> poison), !invariant.load [[META0]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP2]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr float, ptr [[INPUT]], i64 -1
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP0]], i32 4, <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> poison), !invariant.load [[META0]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> [[WIDE_MASKED_LOAD]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -228,20 +196,11 @@ define void @preserve_vector_nuw_nsw(ptr noalias nocapture readonly %input, ptr
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
-; CHECK-NEXT:    [[TMP2:%.*]] = sub nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw nsw <4 x i64> [[TMP2]], splat (i64 2)
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[INPUT]], <4 x i64> [[TMP3]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> [[TMP4]], i32 4, <4 x i1> [[TMP1]], <4 x float> poison), !invariant.load [[META0]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[INPUT]], <4 x i64> <i64 -2, i64 0, i64 2, i64 4>
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> [[TMP0]], i32 4, <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> poison), !invariant.load [[META0]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> [[WIDE_MASKED_GATHER]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -280,22 +239,13 @@ define void @drop_vector_nuw_nsw(ptr noalias nocapture readonly %input, ptr %out
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds ptr, ptr [[PTRS]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = sub <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr float, ptr [[INPUT]], <4 x i64> [[TMP2]]
-; CHECK-NEXT:    store <4 x ptr> [[TMP3]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr float, ptr [[INPUT]], <4 x i64> <i64 -1, i64 0, i64 1, i64 2>
+; CHECK-NEXT:    store <4 x ptr> [[TMP3]], ptr [[PTRS]], align 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x ptr> [[TMP3]], i32 0
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP6]], i32 4, <4 x i1> [[TMP5]], <4 x float> poison), !invariant.load [[META0]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP6]], i32 4, <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> poison), !invariant.load [[META0]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> [[WIDE_MASKED_LOAD]], <4 x float> zeroinitializer
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -337,18 +287,11 @@ define void @drop_nonvector_nuw_nsw_avx1(ptr noalias nocapture readonly %input,
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[PTRS]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP0]], 1
-; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP2]], 1
-; CHECK-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP3]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds ptr, ptr [[PTRS]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 0, 1
+; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 1, 1
+; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 2, 1
+; CHECK-NEXT:    [[TMP9:%.*]] = sub i64 3, 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP6]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP8]]
@@ -358,14 +301,11 @@ define void @drop_nonvector_nuw_nsw_avx1(ptr noalias nocapture readonly %input,
 ; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <4 x ptr> [[TMP15]], ptr [[TMP12]], i32 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x ptr> [[TMP16]], ptr [[TMP13]], i32 3
 ; CHECK-NEXT:    store <4 x ptr> [[TMP17]], ptr [[TMP5]], align 8
-; CHECK-NEXT:    [[TMP19:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP10]], i32 4, <4 x i1> [[TMP19]], <4 x float> poison), !invariant.load [[META0]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[OUTPUT]], i64 [[TMP0]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP10]], i32 4, <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> poison), !invariant.load [[META0]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x float> [[WIDE_MASKED_LOAD]], <4 x float> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[OUTPUT]], i64 0
 ; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP21]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -406,16 +346,8 @@ define void @preserve_nuw_nsw_no_addr(ptr %output) local_unnamed_addr #0 {
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> zeroinitializer, <4 x i64> [[TMP1]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    store <4 x i64> <i64 0, i64 0, i64 1, i64 2>, ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -451,22 +383,10 @@ define void @drop_scalar_exact(ptr noalias nocapture readonly %input, ptr %outpu
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp ne <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = and <4 x i1> [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
-; CHECK-NEXT:    [[TMP5:%.*]] = sdiv i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr float, ptr [[INPUT]], i64 [[TMP5]]
-; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[TMP6]], i32 4, <4 x i1> [[TMP4]], <4 x float> poison), !invariant.load [[META0]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_LOAD]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP8]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[INPUT]], i32 4, <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x float> poison), !invariant.load [[META0]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_LOAD]]
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -519,7 +439,7 @@ define void @drop_zext_nneg(ptr noalias %p, ptr noalias %p1) #0 {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -558,22 +478,11 @@ define void @preserve_vector_exact_no_addr(ptr noalias nocapture readonly %input
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp ne <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = and <4 x i1> [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
-; CHECK-NEXT:    [[TMP5:%.*]] = sdiv exact <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[INPUT]], <4 x i64> [[TMP5]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> [[TMP6]], i32 4, <4 x i1> [[TMP4]], <4 x float> poison), !invariant.load [[META0]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP3]], <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[INPUT]], <4 x i64> <i64 0, i64 0, i64 1, i64 1>
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> [[TMP0]], i32 4, <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x float> poison), !invariant.load [[META0]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x float> zeroinitializer, <4 x float> [[WIDE_MASKED_GATHER]]
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -615,16 +524,8 @@ define void @preserve_exact_no_addr(ptr %output) local_unnamed_addr #0 {
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = sdiv exact <4 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> zeroinitializer, <4 x i64> [[TMP1]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[OUTPUT]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT:    store <4 x i64> <i64 0, i64 0, i64 1, i64 1>, ptr [[OUTPUT]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 entry:
@@ -718,15 +619,11 @@ define void @pr70590_recipe_without_underlying_instr(i64 %n, ptr noalias %dst) {
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], poison
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 0, poison
 ; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP4]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x i8> poison, i8 [[TMP6]], i32 0
@@ -736,8 +633,7 @@ define void @pr70590_recipe_without_underlying_instr(i64 %n, ptr noalias %dst) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
 ; CHECK:       [[PRED_LOAD_IF1]]:
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], poison
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 1, poison
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP11]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = load i8, ptr [[TMP25]], align 1
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i8> [[TMP8]], i8 [[TMP26]], i32 1
@@ -747,8 +643,7 @@ define void @pr70590_recipe_without_underlying_instr(i64 %n, ptr noalias %dst) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
 ; CHECK-NEXT:    br i1 [[TMP7]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
 ; CHECK:       [[PRED_LOAD_IF3]]:
-; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], poison
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 2, poison
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP18]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
 ; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i8> [[TMP29]], i8 [[TMP20]], i32 2
@@ -756,22 +651,18 @@ define void @pr70590_recipe_without_underlying_instr(i64 %n, ptr noalias %dst) {
 ; CHECK:       [[PRED_LOAD_CONTINUE4]]:
 ; CHECK-NEXT:    [[TMP22:%.*]] = phi <4 x i8> [ [[TMP29]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP21]], %[[PRED_LOAD_IF3]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
 ; CHECK:       [[PRED_LOAD_IF5]]:
-; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP30]], poison
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 3, poison
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP12]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = load i8, ptr [[TMP13]], align 1
 ; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x i8> [[TMP22]], i8 [[TMP27]], i32 3
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE6]]
 ; CHECK:       [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = phi <4 x i8> [ [[TMP22]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP28]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i8> zeroinitializer, <4 x i8> [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i8> [[PREDPHI]], ptr [[TMP15]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT:    [[TMP30:%.*]] = phi <4 x i8> [ [[TMP22]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP28]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i8> [[TMP30]], <4 x i8> zeroinitializer
+; CHECK-NEXT:    store <4 x i8> [[PREDPHI]], ptr [[DST]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 
@@ -814,15 +705,11 @@ define void @recipe_without_underlying_instr_lanes_used(i64 %n, ptr noalias %dst
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
-; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[TMP9]], poison
+; CHECK-NEXT:    [[TMP16:%.*]] = add i64 0, poison
 ; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP16]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP23]], align 1
 ; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x i8> poison, i8 [[TMP6]], i32 0
@@ -832,8 +719,7 @@ define void @recipe_without_underlying_instr_lanes_used(i64 %n, ptr noalias %dst
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP3]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
 ; CHECK:       [[PRED_LOAD_IF1]]:
-; CHECK-NEXT:    [[TMP26:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP29:%.*]] = add i64 [[TMP26]], poison
+; CHECK-NEXT:    [[TMP29:%.*]] = add i64 1, poison
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP29]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr [[TMP30]], align 1
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i8> [[TMP25]], i8 [[TMP13]], i32 1
@@ -843,8 +729,7 @@ define void @recipe_without_underlying_instr_lanes_used(i64 %n, ptr noalias %dst
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
 ; CHECK:       [[PRED_LOAD_IF3]]:
-; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], poison
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 2, poison
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP18]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
 ; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i8> [[TMP15]], i8 [[TMP20]], i32 2
@@ -852,25 +737,21 @@ define void @recipe_without_underlying_instr_lanes_used(i64 %n, ptr noalias %dst
 ; CHECK:       [[PRED_LOAD_CONTINUE4]]:
 ; CHECK-NEXT:    [[TMP22:%.*]] = phi <4 x i8> [ [[TMP15]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP21]], %[[PRED_LOAD_IF3]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
 ; CHECK:       [[PRED_LOAD_IF5]]:
-; CHECK-NEXT:    [[TMP31:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP31]], poison
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 3, poison
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP7]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = load i8, ptr [[TMP8]], align 1
 ; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x i8> [[TMP22]], i8 [[TMP27]], i32 3
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE6]]
 ; CHECK:       [[PRED_LOAD_CONTINUE6]]:
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = phi <4 x i8> [ [[TMP22]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP28]], %[[PRED_LOAD_IF5]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i8> zeroinitializer, <4 x i8> [[WIDE_LOAD]]
-; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> zeroinitializer, <4 x i64> poison
+; CHECK-NEXT:    [[TMP26:%.*]] = phi <4 x i8> [ [[TMP22]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP28]], %[[PRED_LOAD_IF5]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i8> [[TMP26]], <4 x i8> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP1]], <4 x i64> poison, <4 x i64> zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[PREDPHI7]], i32 3
 ; CHECK-NEXT:    store i64 [[TMP12]], ptr [[AUX]], align 8
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT:    store <4 x i8> [[PREDPHI]], ptr [[TMP10]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-NEXT:    store <4 x i8> [[PREDPHI]], ptr [[DST]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 
@@ -926,7 +807,7 @@ define void @Bgep_inbounds_unconditionally_due_to_store(ptr noalias %B, ptr read
 ; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ;
 
diff --git a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
index fe2ad661967e6..8b7dd5d5c07bc 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
@@ -503,12 +503,10 @@ define void @test_first_order_recurrence_tried_to_scalarized(ptr %dst, i1 %c, i3
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 4, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i32 [ 4, [[SCALAR_PH]] ], [ [[IV]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 10, [[FOR]]
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds nuw i32, ptr [[DST]], i32 [[IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll b/llvm/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll
index 7fe4c14781e8c..8164c10ac3713 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll
@@ -92,12 +92,10 @@ define double @sumIfVector(ptr nocapture readonly %arr) {
 ; SSE-NEXT:    [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[BIN_RDX]])
 ; SSE-NEXT:    br label [[DONE:%.*]]
 ; SSE:       scalar.ph:
-; SSE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; SSE-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ]
 ; SSE-NEXT:    br label [[LOOP:%.*]]
 ; SSE:       loop:
-; SSE-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
-; SSE-NEXT:    [[TOT:%.*]] = phi double [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
+; SSE-NEXT:    [[I:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
+; SSE-NEXT:    [[TOT:%.*]] = phi double [ 0.000000e+00, [[SCALAR_PH]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
 ; SSE-NEXT:    [[ADDR:%.*]] = getelementptr double, ptr [[ARR]], i32 [[I]]
 ; SSE-NEXT:    [[NEXTVAL:%.*]] = load double, ptr [[ADDR]], align 8
 ; SSE-NEXT:    [[TST:%.*]] = fcmp fast une double [[NEXTVAL]], 4.200000e+01
@@ -157,12 +155,10 @@ define double @sumIfVector(ptr nocapture readonly %arr) {
 ; AVX-NEXT:    [[TMP21:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[BIN_RDX11]])
 ; AVX-NEXT:    br label [[DONE:%.*]]
 ; AVX:       scalar.ph:
-; AVX-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; AVX-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ]
 ; AVX-NEXT:    br label [[LOOP:%.*]]
 ; AVX:       loop:
-; AVX-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
-; AVX-NEXT:    [[TOT:%.*]] = phi double [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
+; AVX-NEXT:    [[I:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
+; AVX-NEXT:    [[TOT:%.*]] = phi double [ 0.000000e+00, [[SCALAR_PH]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
 ; AVX-NEXT:    [[ADDR:%.*]] = getelementptr double, ptr [[ARR]], i32 [[I]]
 ; AVX-NEXT:    [[NEXTVAL:%.*]] = load double, ptr [[ADDR]], align 8
 ; AVX-NEXT:    [[TST:%.*]] = fcmp fast une double [[NEXTVAL]], 4.200000e+01
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
index fcd94f444e8a5..ff2846f235c98 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-costs.ll
@@ -401,13 +401,11 @@ define i16 @iv_and_step_trunc() {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add <2 x i64> [[VEC_IND]], splat (i64 1)
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[TMP0]] to <2 x i16>
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul <2 x i16> [[VEC_IND1]], [[TMP1]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
 ; CHECK-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i16> [[VEC_IND1]], splat (i16 2)
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
@@ -415,12 +413,10 @@ define i16 @iv_and_step_trunc() {
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <2 x i16> [[TMP2]], i32 0
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[REC_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[REC_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[IV]] to i16
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[IV_NEXT]] to i16
@@ -620,10 +616,9 @@ define void @wide_iv_trunc(ptr %dst, i64 %N) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
 ; CHECK-NEXT:    store i32 [[IV_TRUNC]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
index fea027d6803c6..4b4103e9806b9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/induction-step.ll
@@ -111,7 +111,6 @@ define i16 @wide_sub_induction_step_live_in(ptr %dst, i64 %N, i16 %off) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i16> [[TMP10]], i32 3
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -128,7 +127,7 @@ define i16 @wide_sub_induction_step_live_in(ptr %dst, i64 %N, i16 %off) {
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
 ; CHECK-NEXT:    br i1 [[EC]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[SUB_LCSSA:%.*]] = phi i16 [ [[SUB]], [[LOOP]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[SUB_LCSSA:%.*]] = phi i16 [ [[SUB]], [[LOOP]] ], [ [[TMP1]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i16 [[SUB_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
index 6d562be03a9b0..6210a9aa66d50 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/interleave-cost.ll
@@ -7,7 +7,7 @@ target triple = "x86_64-apple-macosx10.15.0"
 define void @test_free_instructions_feeding_geps_for_interleave_groups(ptr noalias %p.invar, ptr noalias %dst.1, ptr noalias %dst.2) {
 ; CHECK-LABEL: define void @test_free_instructions_feeding_geps_for_interleave_groups(
 ; CHECK-SAME: ptr noalias [[P_INVAR:%.*]], ptr noalias [[DST_1:%.*]], ptr noalias [[DST_2:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -39,10 +39,9 @@ define void @test_free_instructions_feeding_geps_for_interleave_groups(ptr noali
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[L_0:%.*]] = load float, ptr [[P_INVAR]], align 4
 ; CHECK-NEXT:    [[IV_MUL:%.*]] = shl i64 [[IV]], 2
 ; CHECK-NEXT:    [[GEP_DST_19:%.*]] = getelementptr float, ptr [[DST_1]], i64 [[IV_MUL]]
@@ -146,7 +145,7 @@ define void @geps_feeding_interleave_groups_with_reuse(ptr %arg, i64 %arg1, ptr
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = shl i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[ARG2]], i64 [[TMP11]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x float>, ptr [[TMP26]], align 4
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x float>, ptr [[TMP26]], align 4, !alias.scope [[META4:![0-9]+]]
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 0, i32 8>
 ; CHECK-NEXT:    [[STRIDED_VEC14:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 1, i32 9>
 ; CHECK-NEXT:    [[STRIDED_VEC15:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 2, i32 10>
@@ -167,10 +166,10 @@ define void @geps_feeding_interleave_groups_with_reuse(ptr %arg, i64 %arg1, ptr
 ; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <2 x float> [[TMP35]], <2 x float> [[TMP37]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[TMP42:%.*]] = shufflevector <4 x float> [[TMP40]], <4 x float> [[TMP41]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP42]], <8 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:    store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP28]], align 4
+; CHECK-NEXT:    store <8 x float> [[INTERLEAVED_VEC]], ptr [[TMP28]], align 4, !alias.scope [[META7:![0-9]+]], !noalias [[META4]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP43]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
@@ -216,7 +215,7 @@ define void @geps_feeding_interleave_groups_with_reuse(ptr %arg, i64 %arg1, ptr
 ; CHECK-NEXT:    store float [[MUL_4]], ptr [[GEP_11]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[ARG1]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -397,12 +396,12 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP51:%.*]] = lshr exact i64 [[OFFSET_IDX]], 1
 ; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr nusw i32, ptr [[B]], i64 [[TMP51]]
-; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP52]], align 4
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, ptr [[TMP52]], align 4, !alias.scope [[META11:![0-9]+]], !noalias [[META14:![0-9]+]]
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
 ; CHECK-NEXT:    [[STRIDED_VEC34:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
 ; CHECK-NEXT:    [[TMP56:%.*]] = getelementptr i32, ptr [[A]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP54:%.*]] = getelementptr i32, ptr [[B]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[TMP54]], i32 4, <4 x i1> splat (i1 true), <4 x i32> poison), !alias.scope [[META6:![0-9]+]], !noalias [[META9:![0-9]+]]
+; CHECK-NEXT:    [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[TMP54]], i32 4, <4 x i1> splat (i1 true), <4 x i32> poison), !alias.scope [[META16:![0-9]+]], !noalias [[META14]]
 ; CHECK-NEXT:    [[TMP58:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP59:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC34]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <4 x i32> [[WIDE_MASKED_GATHER]], <4 x i32> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -410,11 +409,11 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
 ; CHECK-NEXT:    [[TMP62:%.*]] = shufflevector <8 x i32> [[TMP60]], <8 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[TMP63:%.*]] = shufflevector <16 x i32> [[TMP61]], <16 x i32> [[TMP62]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[TMP63]], <32 x i32> poison, <32 x i32> <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 28, i32 1, i32 5, i32 9, i32 13, i32 17, i32 21, i32 25, i32 29, i32 2, i32 6, i32 10, i32 14, i32 18, i32 22, i32 26, i32 30, i32 3, i32 7, i32 11, i32 15, i32 19, i32 23, i32 27, i32 31>
-; CHECK-NEXT:    store <32 x i32> [[INTERLEAVED_VEC]], ptr [[TMP56]], align 4
+; CHECK-NEXT:    store <32 x i32> [[INTERLEAVED_VEC]], ptr [[TMP56]], align 4, !alias.scope [[META14]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 32)
 ; CHECK-NEXT:    [[TMP64:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP64]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP64]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -455,7 +454,7 @@ define void @geps_feeding_interleave_groups_with_reuse2(ptr %A, ptr %B, i64 %N)
 ; CHECK-NEXT:    store i32 0, ptr [[GEP_A_7]], align 4
 ; CHECK-NEXT:    [[IV_NEXT_7]] = add nuw nsw i64 [[IV]], 8
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[N]]
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -507,35 +506,26 @@ exit:
 define void @interleave_store_double_i64(ptr %dst) {
 ; CHECK-LABEL: define void @interleave_store_double_i64(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr { double, i64 }, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[VEC_IND]] to <2 x double>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> zeroinitializer, <2 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    store <4 x double> <double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double 4.940660e-324>, ptr [[DST]], align 8
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr { double, i64 }, ptr [[DST]], i64 [[IV]], i32 1
 ; CHECK-NEXT:    store i64 [[IV]], ptr [[GEP_1]], align 8
 ; CHECK-NEXT:    [[GEP_0:%.*]] = getelementptr { double, i64 }, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP_0]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 1
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -628,35 +618,26 @@ exit:
 define void @interleave_store_i64_double_2(ptr %dst) {
 ; CHECK-LABEL: define void @interleave_store_i64_double_2(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr { i64, double }, ptr [[DST]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[VEC_IND]] to <2 x double>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
-; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    store <4 x double> <double 0.000000e+00, double 0.000000e+00, double 4.940660e-324, double 0.000000e+00>, ptr [[DST]], align 8
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_0:%.*]] = getelementptr { i64, double }, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[IV]], ptr [[GEP_0]], align 8
 ; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr { i64, double }, ptr [[DST]], i64 [[IV]], i32 1
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP_1]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 1
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -687,17 +668,22 @@ attributes #1 = { "min-legal-vector-width"="0" "target-cpu"="cascadelake" }
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
 ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]]}
-; CHECK: [[META6]] = !{[[META7:![0-9]+]]}
-; CHECK: [[META7]] = distinct !{[[META7]], [[META8:![0-9]+]]}
-; CHECK: [[META8]] = distinct !{[[META8]], !"LVerDomain"}
-; CHECK: [[META9]] = !{[[META10:![0-9]+]]}
-; CHECK: [[META10]] = distinct !{[[META10]], [[META8]]}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]], [[META2]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META1]], [[META2]]}
-; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META2]], [[META1]]}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META1]], [[META2]]}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META2]], [[META1]]}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]]}
+; CHECK: [[META5]] = distinct !{[[META5]], [[META6:![0-9]+]]}
+; CHECK: [[META6]] = distinct !{[[META6]], !"LVerDomain"}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META6]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]], [[META2]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]]}
+; CHECK: [[META11]] = !{[[META12:![0-9]+]]}
+; CHECK: [[META12]] = distinct !{[[META12]], [[META13:![0-9]+]]}
+; CHECK: [[META13]] = distinct !{[[META13]], !"LVerDomain"}
+; CHECK: [[META14]] = !{[[META15:![0-9]+]]}
+; CHECK: [[META15]] = distinct !{[[META15]], [[META13]]}
+; CHECK: [[META16]] = !{[[META17:![0-9]+]]}
+; CHECK: [[META17]] = distinct !{[[META17]], [[META13]]}
+; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]}
+; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META1]]}
+; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META2]], [[META1]]}
+; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
index f615e23bcb8b0..452868ddd59c1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/load-deref-pred.ll
@@ -65,12 +65,10 @@ define i32 @test_explicit_pred(i64 %len) {
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX14]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = icmp slt i64 [[IV]], [[LEN]]
 ; CHECK-NEXT:    br i1 [[EARLYCND]], label [[PRED:%.*]], label [[LATCH]]
@@ -216,12 +214,10 @@ define i32 @test_explicit_pred_generic(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -396,12 +392,10 @@ define i32 @test_invariant_address(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP101:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX8]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -667,12 +661,10 @@ define i32 @test_step_narrower_than_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP149:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX38]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -984,12 +976,10 @@ define i32 @test_non_zero_start(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1024, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -1228,12 +1218,10 @@ define i32 @test_non_unit_stride(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP117:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX8]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 2
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -1380,12 +1368,10 @@ define i32 @neg_off_by_many(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -1532,12 +1518,10 @@ define i32 @neg_off_by_one_iteration(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -1684,12 +1668,10 @@ define i32 @neg_off_by_one_byte(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -2005,12 +1987,10 @@ define i32 @test_allocsize(i64 %len, ptr %test_base) nofree nosync {
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -2158,12 +2138,10 @@ define i32 @test_allocsize_array(i64 %len, ptr %test_base) nofree nosync {
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
@@ -2321,12 +2299,10 @@ define i32 @test_allocsize_cond_deref(i1 %allzero, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[PREHEADER]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll b/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
index f26064a4a81db..6a90f03c2f7fd 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/metadata-enable.ll
@@ -1201,10 +1201,9 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; O1VEC2:       middle.block:
 ; O1VEC2-NEXT:    br label [[FOR_END:%.*]]
 ; O1VEC2:       scalar.ph:
-; O1VEC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; O1VEC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; O1VEC2:       for.body:
-; O1VEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; O1VEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; O1VEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
 ; O1VEC2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; O1VEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP10]], [[N]]
@@ -1242,10 +1241,9 @@ define i32 @nopragma(ptr noalias nocapture %a, ptr noalias nocapture readonly %b
 ; OzVEC2:       middle.block:
 ; OzVEC2-NEXT:    br label [[FOR_END:%.*]]
 ; OzVEC2:       scalar.ph:
-; OzVEC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; OzVEC2-NEXT:    br label [[FOR_BODY:%.*]]
 ; OzVEC2:       for.body:
-; OzVEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; OzVEC2-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; OzVEC2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[B]], i64 [[INDVARS_IV]]
 ; OzVEC2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; OzVEC2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP10]], [[N]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
index 07e2df360e249..6d7b8a222c51e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/optsize.ll
@@ -32,10 +32,9 @@ define i32 @foo_optsize() #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP7]], 0
@@ -69,10 +68,9 @@ define i32 @foo_optsize() #0 {
 ; AUTOVF:       middle.block:
 ; AUTOVF-NEXT:    br label [[FOR_END:%.*]]
 ; AUTOVF:       scalar.ph:
-; AUTOVF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; AUTOVF-NEXT:    br label [[FOR_BODY:%.*]]
 ; AUTOVF:       for.body:
-; AUTOVF-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; AUTOVF-NEXT:    [[I_08:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; AUTOVF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
 ; AUTOVF-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; AUTOVF-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP7]], 0
@@ -128,10 +126,9 @@ define i32 @foo_minsize() #1 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP7]], 0
@@ -165,10 +162,9 @@ define i32 @foo_minsize() #1 {
 ; AUTOVF:       middle.block:
 ; AUTOVF-NEXT:    br label [[FOR_END:%.*]]
 ; AUTOVF:       scalar.ph:
-; AUTOVF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; AUTOVF-NEXT:    br label [[FOR_BODY:%.*]]
 ; AUTOVF:       for.body:
-; AUTOVF-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; AUTOVF-NEXT:    [[I_08:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; AUTOVF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32 x i8], ptr @tab, i32 0, i32 [[I_08]]
 ; AUTOVF-NEXT:    [[TMP7:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; AUTOVF-NEXT:    [[CMP1:%.*]] = icmp eq i8 [[TMP7]], 0
@@ -226,10 +222,9 @@ define void @scev4stride1(ptr noalias nocapture %a, ptr noalias nocapture readon
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[I_07]], [[K]]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[MUL]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
@@ -263,10 +258,9 @@ define void @scev4stride1(ptr noalias nocapture %a, ptr noalias nocapture readon
 ; AUTOVF:       middle.block:
 ; AUTOVF-NEXT:    br label [[FOR_END_LOOPEXIT:%.*]]
 ; AUTOVF:       scalar.ph:
-; AUTOVF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
 ; AUTOVF-NEXT:    br label [[FOR_BODY:%.*]]
 ; AUTOVF:       for.body:
-; AUTOVF-NEXT:    [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; AUTOVF-NEXT:    [[I_07:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; AUTOVF-NEXT:    [[MUL:%.*]] = mul nsw i32 [[I_07]], [[K]]
 ; AUTOVF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[MUL]]
 ; AUTOVF-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
@@ -376,10 +370,9 @@ define void @tail_folded_store_avx512(ptr %start, ptr %end) #3 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[START]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], [[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr nusw i8, ptr [[PTR_IV]], i64 -72
 ; CHECK-NEXT:    store ptr null, ptr [[PTR_IV]], align 8
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
@@ -420,10 +413,9 @@ define void @tail_folded_store_avx512(ptr %start, ptr %end) #3 {
 ; AUTOVF:       middle.block:
 ; AUTOVF-NEXT:    br label [[EXIT:%.*]]
 ; AUTOVF:       scalar.ph:
-; AUTOVF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[START]], [[ENTRY:%.*]] ]
 ; AUTOVF-NEXT:    br label [[LOOP:%.*]]
 ; AUTOVF:       loop:
-; AUTOVF-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
+; AUTOVF-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[START]], [[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], [[LOOP]] ]
 ; AUTOVF-NEXT:    [[PTR_IV_NEXT]] = getelementptr nusw i8, ptr [[PTR_IV]], i64 -72
 ; AUTOVF-NEXT:    store ptr null, ptr [[PTR_IV]], align 8
 ; AUTOVF-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll b/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
index 7a59884bdfba9..732983a708c51 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
@@ -67,26 +67,19 @@ define void @foo(i32 %n) {
 ; AVX-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
 ; AVX-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; AVX:       [[VECTOR_BODY]]:
-; AVX-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
-; AVX-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
-; AVX-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [8 x i32], ptr @arr2, i64 0, <8 x i64> [[VEC_IND]]
-; AVX-NEXT:    [[TMP1:%.*]] = trunc <8 x i64> [[VEC_IND]] to <8 x i32>
-; AVX-NEXT:    call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[TMP1]], <8 x ptr> [[TMP0]], i32 4, <8 x i1> splat (i1 true))
-; AVX-NEXT:    [[TMP7:%.*]] = trunc <8 x i64> [[VEC_IND]] to <8 x i32>
-; AVX-NEXT:    [[TMP2:%.*]] = add nsw <8 x i32> [[TMP7]], [[BROADCAST_SPLAT]]
+; AVX-NEXT:    call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, <8 x ptr> getelementptr inbounds ([8 x i32], ptr @arr2, <8 x i64> zeroinitializer, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>), i32 4, <8 x i1> splat (i1 true))
+; AVX-NEXT:    [[TMP2:%.*]] = add nsw <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[BROADCAST_SPLAT]]
 ; AVX-NEXT:    br label %[[FOR_BODY31:.*]]
 ; AVX:       [[FOR_BODY31]]:
 ; AVX-NEXT:    [[VEC_PHI:%.*]] = phi <8 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR_BODY31]] ]
-; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, <8 x i64> [[VEC_PHI]], <8 x i64> [[VEC_IND]]
+; AVX-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [8 x [8 x i32]], ptr @arr, i64 0, <8 x i64> [[VEC_PHI]], <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
 ; AVX-NEXT:    call void @llvm.masked.scatter.v8i32.v8p0(<8 x i32> [[TMP2]], <8 x ptr> [[TMP3]], i32 4, <8 x i1> splat (i1 true))
 ; AVX-NEXT:    [[TMP4]] = add nuw nsw <8 x i64> [[VEC_PHI]], splat (i64 1)
 ; AVX-NEXT:    [[TMP5:%.*]] = icmp eq <8 x i64> [[TMP4]], splat (i64 8)
 ; AVX-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i32 0
-; AVX-NEXT:    br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR_BODY31]]
+; AVX-NEXT:    br i1 [[TMP6]], label %[[VECTOR_LATCH:.*]], label %[[FOR_BODY31]], !llvm.loop [[LOOP0:![0-9]+]]
 ; AVX:       [[VECTOR_LATCH]]:
-; AVX-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; AVX-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
-; AVX-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; AVX:       [[MIDDLE_BLOCK]]:
 ; AVX-NEXT:    br i1 true, [[FOR_END10:label %.*]], label %[[SCALAR_PH]]
 ; AVX:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll b/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
index c1d08e152fc56..9181cce613ad8 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
@@ -6,7 +6,7 @@ target triple = "x86_64"
 define i8 @pr141968(i1 %cond, i8 %v) {
 ; CHECK-LABEL: define i8 @pr141968(
 ; CHECK-SAME: i1 [[COND:%.*]], i8 [[V:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[ZEXT_TRUE:%.*]] = zext i1 true to i16
 ; CHECK-NEXT:    [[SEXT:%.*]] = sext i8 [[V]] to i16
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -107,10 +107,9 @@ define i8 @pr141968(i1 %cond, i8 %v) {
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i8> [[PREDPHI]], i32 15
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    br i1 [[COND]], label %[[LOOP_LATCH]], label %[[COND_FALSE:.*]]
 ; CHECK:       [[COND_FALSE]]:
 ; CHECK-NEXT:    [[SDIV:%.*]] = sdiv i16 [[SEXT]], [[ZEXT_TRUE]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll b/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll
index df2e35d3922d0..204271173da0c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr34438.ll
@@ -24,10 +24,9 @@ define void @small_tc(ptr noalias nocapture %A, ptr noalias nocapture readonly %
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !llvm.access.group [[ACC_GRP0]]
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i64 [[INDVARS_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll b/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
index 639fb86d5ddb3..6fc7ed2efe852 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr35432.ll
@@ -13,9 +13,9 @@ define i32 @main(ptr %ptr) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[S:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[I]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr nonnull [[S]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S]])
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 (ptr, ...) @goo(ptr nonnull [[I]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I]], align 4
 ; CHECK-NEXT:    [[STOREMERGE6:%.*]] = trunc i32 [[TMP0]] to i16
@@ -111,16 +111,16 @@ define i32 @main(ptr %ptr) {
 ; CHECK-NEXT:    br label [[FOR_END12]]
 ; CHECK:       for.end12:
 ; CHECK-NEXT:    [[CALL13:%.*]] = call i32 (ptr, ...) @foo(ptr nonnull [[S]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr nonnull [[S]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[S]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[I]])
 ; CHECK-NEXT:    ret i32 0
 ;
 entry:
   %i = alloca i32, align 4
   %s = alloca i16, align 2
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %i) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %i) #3
   store i32 0, ptr %i, align 4
-  call void @llvm.lifetime.start.p0(i64 2, ptr nonnull %s) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %s) #3
   %call = call i32 (ptr, ...) @goo(ptr nonnull %i) #3
   %0 = load i32, ptr %i, align 4
   %storemerge6 = trunc i32 %0 to i16
@@ -174,17 +174,17 @@ for.cond.for.end12_crit_edge:                     ; preds = %for.inc9
 
 for.end12:                                        ; preds = %for.cond.for.end12_crit_edge, %entry
   %call13 = call i32 (ptr, ...) @foo(ptr nonnull %s) #3
-  call void @llvm.lifetime.end.p0(i64 2, ptr nonnull %s) #3
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %i) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %s) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %i) #3
   ret i32 0
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare i32 @goo(...) local_unnamed_addr #2
 
 declare i32 @foo(...) local_unnamed_addr #2
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll b/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
index c8cf9fb0544a9..0fdcf89deb902 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
@@ -1,46 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
 ; RUN: opt -passes=loop-vectorize -mtriple=x86_64-unknown-linux -force-vector-width=2 -force-vector-interleave=1 -S %s | FileCheck %s
 
 define ptr @test(ptr noalias %src, ptr noalias %dst) {
-; CHECK-LABEL: @test(
-; CHECK-NEXT:  entry:
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE2:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %vector.ph ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE2]] ]
+; CHECK-LABEL: define ptr @test(
+; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP6]], i32 0
 ; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x ptr> [[TMP16]], ptr [[TMP2]], i32 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <2 x i64> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i64> [[VEC_IND]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
-; CHECK-NEXT:    br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
-; CHECK:       pred.load.if:
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK:       [[PRED_LOAD_IF]]:
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
-; CHECK:       pred.load.continue:
-; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ poison, %vector.body ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; CHECK:       [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
-; CHECK-NEXT:    br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2]]
-; CHECK:       pred.load.if1:
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
+; CHECK:       [[PRED_LOAD_IF1]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP11]], i32 1
-; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
-; CHECK:       pred.load.continue2:
-; CHECK-NEXT:    [[TMP13:%.*]] = phi <2 x i32> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> zeroinitializer, <2 x i32> [[TMP13]]
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[TMP0]]
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK:       [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP15]], <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label %vector.body
-; CHECK:       middle.block:
-; CHECK-NEXT:    br label %exit
-; CHECK:       exit:
-; CHECK-NEXT:    [[GEP_LCSSA:%.*]] = phi ptr [ %gep.src, %loop.latch ], [ [[TMP2]], %middle.block ]
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[IV]], 0
+; CHECK-NEXT:    br i1 [[CMP_1]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[M:%.*]] = phi i32 [ [[L]], %[[THEN]] ], [ 0, %[[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT:    store i32 [[M]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[CMP_2:%.*]] = icmp slt i64 [[IV_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[CMP_2]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[GEP_LCSSA:%.*]] = phi ptr [ [[GEP_SRC]], %[[LOOP_LATCH]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret ptr [[GEP_LCSSA]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll
index 08adfdd4793eb..d261827d4e111 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr81872.ll
@@ -41,10 +41,9 @@ define void @test(ptr noundef align 8 dereferenceable_or_null(16) %arr) #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[BB6:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 99, [[BB5:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 99, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], 1
 ; CHECK-NEXT:    [[ICMP17:%.*]] = icmp eq i64 [[AND]], 0
 ; CHECK-NEXT:    br i1 [[ICMP17]], label [[BB18:%.*]], label [[LOOP_LATCH]], !prof [[PROF5:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll b/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll
index 0e83cf374fc30..04cdc759d8124 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/reduction-fastmath.ll
@@ -73,12 +73,10 @@ define float @reduction_sum_float_fastmath(i32 %n, ptr %array) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi float [ [[SUM_INC:%.*]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_INC:%.*]], [[LOOP]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi float [ [[SUM_INC:%.*]], [[LOOP]] ], [ 0.000000e+00, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ADDRESS:%.*]] = getelementptr float, ptr [[ARRAY]], i32 [[IDX]]
 ; CHECK-NEXT:    [[VALUE:%.*]] = load float, ptr [[ADDRESS]], align 4
 ; CHECK-NEXT:    [[SUM_INC]] = fadd fast float [[SUM]], [[VALUE]]
@@ -138,12 +136,10 @@ define float @reduction_sum_float_only_reassoc(i32 %n, ptr %array) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[BIN_RDX]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ -0.000000e+00, [[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi float [ [[SUM_INC:%.*]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_INC:%.*]], [[LOOP]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi float [ [[SUM_INC:%.*]], [[LOOP]] ], [ -0.000000e+00, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ADDRESS:%.*]] = getelementptr float, ptr [[ARRAY]], i32 [[IDX]]
 ; CHECK-NEXT:    [[VALUE:%.*]] = load float, ptr [[ADDRESS]], align 4
 ; CHECK-NEXT:    [[SUM_INC]] = fadd reassoc float [[SUM]], [[VALUE]]
@@ -203,12 +199,10 @@ define float @reduction_sum_float_only_reassoc_and_contract(i32 %n, ptr %array)
 ; CHECK-NEXT:    [[TMP9:%.*]] = call reassoc contract float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[BIN_RDX]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ -0.000000e+00, [[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_INC:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi float [ [[SUM_INC:%.*]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = phi i32 [ [[IDX_INC:%.*]], [[LOOP]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi float [ [[SUM_INC:%.*]], [[LOOP]] ], [ -0.000000e+00, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ADDRESS:%.*]] = getelementptr float, ptr [[ARRAY]], i32 [[IDX]]
 ; CHECK-NEXT:    [[VALUE:%.*]] = load float, ptr [[ADDRESS]], align 4
 ; CHECK-NEXT:    [[SUM_INC]] = fadd reassoc contract float [[SUM]], [[VALUE]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
index 1c6a225f99f61..e2e44b1701633 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
 ; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck %s
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
@@ -215,11 +215,98 @@ loop.latch:
 exit:
   ret void
 }
-;.
-; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
-; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
-; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
-;.
+
+define float @uniform_load_replicating_select(ptr %A, ptr %B, i64 %1) {
+; CHECK-LABEL: define float @uniform_load_replicating_select(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[TMP0:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 5
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 6
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[INDEX]], 7
+; CHECK-NEXT:    [[TMP6:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = fcmp ogt float [[TMP6]], 0.000000e+00
+; CHECK-NEXT:    [[TMP8:%.*]] = fcmp ogt float [[TMP6]], 0.000000e+00
+; CHECK-NEXT:    [[TMP9:%.*]] = fcmp ogt float [[TMP6]], 0.000000e+00
+; CHECK-NEXT:    [[TMP10:%.*]] = fcmp ogt float [[TMP6]], 0.000000e+00
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i1> poison, i1 [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x i1> [[TMP11]], i1 [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i1> [[TMP12]], i1 [[TMP9]], i32 2
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i1> [[TMP13]], i1 [[TMP10]], i32 3
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP19:%.*]] = select i1 [[TMP7]], ptr [[A]], ptr [[TMP15]]
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP8]], ptr [[A]], ptr [[TMP16]]
+; CHECK-NEXT:    [[TMP21:%.*]] = select i1 [[TMP9]], ptr [[A]], ptr [[TMP17]]
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP10]], ptr [[A]], ptr [[TMP18]]
+; CHECK-NEXT:    [[TMP23:%.*]] = select <4 x i1> [[TMP14]], <4 x float> splat (float 1.000000e+01), <4 x float> splat (float 1.000000e+00)
+; CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[TMP19]], align 4
+; CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[TMP20]], align 4
+; CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[TMP21]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[TMP22]], align 4
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x float> poison, float [[TMP24]], i32 0
+; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <4 x float> [[TMP28]], float [[TMP25]], i32 1
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <4 x float> [[TMP29]], float [[TMP26]], i32 2
+; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <4 x float> [[TMP30]], float [[TMP27]], i32 3
+; CHECK-NEXT:    [[TMP32:%.*]] = fdiv <4 x float> splat (float 4.000000e+00), [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = call <4 x float> @llvm.pow.v4f32(<4 x float> [[TMP23]], <4 x float> [[TMP32]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP34]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <4 x float> [[TMP33]], i32 3
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[A]], align 4
+; CHECK-NEXT:    [[C:%.*]] = fcmp ogt float [[L]], 0.000000e+00
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    [[SEL_PTR:%.*]] = select i1 [[C]], ptr [[A]], ptr [[GEP_B]]
+; CHECK-NEXT:    [[BASE:%.*]] = select i1 [[C]], float 1.000000e+01, float 1.000000e+00
+; CHECK-NEXT:    [[L_2:%.*]] = load float, ptr [[SEL_PTR]], align 4
+; CHECK-NEXT:    [[DIV:%.*]] = fdiv float 4.000000e+00, [[L_2]]
+; CHECK-NEXT:    [[POW:%.*]] = tail call float @llvm.pow.f32(float [[BASE]], float [[DIV]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[POW_LCSSA:%.*]] = phi float [ [[POW]], %[[LOOP]] ], [ [[TMP35]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret float [[POW_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.next, %loop ], [ 0, %entry ]
+  %l = load float, ptr %A, align 4
+  %c = fcmp ogt float %l, 0.000000e+00
+  %gep.B = getelementptr inbounds float, ptr %B, i64 %iv
+  %sel.ptr = select i1 %c, ptr %A, ptr %gep.B
+  %base = select i1 %c, float 10.000000e+00, float 1.000000e+00
+  %l.2 = load float, ptr %sel.ptr, align 4
+  %div = fdiv float 4.000000e+00, %l.2
+  %pow = tail call float @llvm.pow.f32(float %base, float %div)
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, %1
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret float %pow
+}
+
+declare float @llvm.pow.f32(float, float)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll b/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
index 0b61f207d041a..28435d4f34ac1 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/replicate-uniform-call.ll
@@ -8,7 +8,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define void @smax_call_uniform(ptr %dst, i64 %x) {
 ; CHECK-LABEL: define void @smax_call_uniform(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i8 -68, -69
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[X]], 0
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -55,10 +55,9 @@ define void @smax_call_uniform(ptr %dst, i64 %x) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[ELSE:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    [[REM1:%.*]] = urem i64 [[MUL]], [[X]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll b/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
index 440f6e1dfeeff..7d3f557245b11 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/scev-checks-unprofitable.ll
@@ -23,9 +23,6 @@ define void @value_defined_in_loop1_used_for_trip_counts(i32 %start, i1 %c, ptr
 ; CHECK:       [[LOOP_3_PREHEADER]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[IV_1_LCSSA2]], 15
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 16
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[IV_1_LCSSA2]], 1
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT]], <16 x i64> poison, <16 x i32> zeroinitializer
@@ -37,7 +34,6 @@ define void @value_defined_in_loop1_used_for_trip_counts(i32 %start, i1 %c, ptr
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT_1_LOOPEXIT1:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[LOOP_3_PREHEADER]] ]
 ; CHECK-NEXT:    br label %[[LOOP_3:.*]]
 ; CHECK:       [[LOOP_2_PREHEADER]]:
 ; CHECK-NEXT:    br label %[[LOOP_2:.*]]
@@ -53,7 +49,7 @@ define void @value_defined_in_loop1_used_for_trip_counts(i32 %start, i1 %c, ptr
 ; CHECK-NEXT:    [[EC_2:%.*]] = icmp ult i64 [[IV_2]], [[IV_1_LCSSA]]
 ; CHECK-NEXT:    br i1 [[EC_2]], label %[[LOOP_2]], label %[[EXIT_1_LOOPEXIT:.*]]
 ; CHECK:       [[LOOP_3]]:
-; CHECK-NEXT:    [[IV_4:%.*]] = phi i64 [ [[IV_4_NEXT:%.*]], %[[LOOP_3]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV_4:%.*]] = phi i64 [ [[IV_4_NEXT:%.*]], %[[LOOP_3]] ], [ 0, %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[GEP_DST_2:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV_4]]
 ; CHECK-NEXT:    store i8 0, ptr [[GEP_DST_2]], align 1
 ; CHECK-NEXT:    [[IV_4_NEXT]] = add i64 [[IV_4]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
index 33b173d3a7004..35f61b2aa838a 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/strided_load_cost.ll
@@ -556,10 +556,9 @@ define void @test(ptr %A, ptr noalias %B) #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[IV_0:%.*]] = add nuw nsw i64 [[IV]], 0
 ; CHECK-NEXT:    [[IV_1:%.*]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[IN0:%.*]] = getelementptr inbounds [1024 x i32], ptr [[A]], i64 0, i64 [[IV_0]]
@@ -676,10 +675,9 @@ define void @test(ptr %A, ptr noalias %B) #0 {
 ; MAX-BW:       middle.block:
 ; MAX-BW-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; MAX-BW:       scalar.ph:
-; MAX-BW-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; MAX-BW-NEXT:    br label [[FOR_BODY:%.*]]
 ; MAX-BW:       for.body:
-; MAX-BW-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; MAX-BW-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; MAX-BW-NEXT:    [[IV_0:%.*]] = add nuw nsw i64 [[IV]], 0
 ; MAX-BW-NEXT:    [[IV_1:%.*]] = add nuw nsw i64 [[IV]], 1
 ; MAX-BW-NEXT:    [[IN0:%.*]] = getelementptr inbounds [1024 x i32], ptr [[A]], i64 0, i64 [[IV_0]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
index 5e35c4ae1f404..b33fbb0830988 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/tail_loop_folding.ll
@@ -30,12 +30,11 @@ define dso_local void @tail_folding_enabled(ptr noalias nocapture %A, ptr noalia
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDVARS_IV]]
@@ -94,12 +93,11 @@ define dso_local void @tail_folding_disabled(ptr noalias nocapture %A, ptr noali
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDVARS_IV]]
@@ -177,12 +175,10 @@ define i32 @reduction_i32(ptr nocapture readonly %A, ptr nocapture readonly %B,
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP11]])
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[SUM_0:%.*]] = phi i32 [ [[SUM_1:%.*]], [[FOR_BODY]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM_0:%.*]] = phi i32 [ [[SUM_1:%.*]], [[FOR_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[ARRAYIDXA:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[ARRAYIDXA]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
index 68b4f202e106f..a491a6233fda2 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/uniform_mem_op.ll
@@ -406,12 +406,10 @@ define i32 @test_count_bits(ptr %test_base) {
 ; CHECK-NEXT:    [[TMP41:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX14]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[BYTE:%.*]] = udiv i64 [[IV]], 8
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE]], i64 [[BYTE]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
index f7eba42edaf57..52f491eed0309 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
@@ -143,10 +143,9 @@ define void @vectorized1(ptr noalias nocapture %A, ptr noalias nocapture readonl
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !llvm.access.group [[ACC_GRP7]]
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
@@ -205,10 +204,9 @@ define void @vectorized2(ptr noalias nocapture %A, ptr noalias nocapture readonl
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !llvm.access.group [[ACC_GRP7]]
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
index 70134fa6bc78d..5ec093c5af6ba 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorization-remarks-missed.ll
@@ -117,6 +117,33 @@
 ; YAML-NEXT: ...
 ; YAML-NEXT: --- !Analysis
 ; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            NonReductionValueUsedOutsideLoop
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 27, Column: 3 }
+; YAML-NEXT: Function:        test_multiple_failures
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: '
+; YAML-NEXT:   - String:          value that could not be identified as reduction is used outside the loop
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Analysis
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            CantVectorizeLibcall
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 29, Column: 11 }
+; YAML-NEXT: Function:        test_multiple_failures
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: '
+; YAML-NEXT:   - String:          call instruction cannot be vectorized
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Analysis
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            NoInductionVariable
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 27, Column: 3 }
+; YAML-NEXT: Function:        test_multiple_failures
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: '
+; YAML-NEXT:   - String:          loop induction variable could not be identified
+; YAML-NEXT: ...
+; YAML-NEXT: --- !Analysis
+; YAML-NEXT: Pass:            loop-vectorize
 ; YAML-NEXT: Name:            UnsupportedUncountableLoop
 ; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 27, Column: 3 }
 ; YAML-NEXT: Function:        test_multiple_failures
@@ -124,6 +151,15 @@
 ; YAML-NEXT:   - String:          'loop not vectorized: '
 ; YAML-NEXT:   - String:          Cannot vectorize uncountable loop
 ; YAML-NEXT: ...
+; YAML-NEXT: --- !Analysis
+; YAML-NEXT: Pass:            loop-vectorize
+; YAML-NEXT: Name:            CantComputeNumberOfIterations
+; YAML-NEXT: DebugLoc:        { File: source.cpp, Line: 27, Column: 3 }
+; YAML-NEXT: Function:        test_multiple_failures
+; YAML-NEXT: Args:
+; YAML-NEXT:   - String:          'loop not vectorized: '
+; YAML-NEXT:   - String:          could not determine number of loop iterations
+; YAML-NEXT: ...
 ; YAML:      --- !Missed
 ; YAML-NEXT: Pass:            loop-vectorize
 ; YAML-NEXT: Name:            MissedDetails
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll
index 59f2925d01fa2..6dea2f6b146da 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll
@@ -40,10 +40,9 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL:       middle.block:
 ; IF-EVL-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; IF-EVL:       scalar.ph:
-; IF-EVL-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       for.body:
-; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; IF-EVL-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; IF-EVL-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; IF-EVL-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
index e9d85c26d26e3..3c8dd94a7a605 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
@@ -76,10 +76,9 @@ define void @test_pr59090(ptr %l_out, ptr noalias %b) #0 {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IV_MUL:%.*]] = mul nuw i64 [[IV]], 6
 ; CHECK-NEXT:    [[L:%.*]] = load i8, ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
 ; CHECK-NEXT:    store i8 [[L]], ptr [[B]], align 1, !llvm.access.group [[ACC_GRP0]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll b/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll
index 05d08a4e3635b..c49d36962796b 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/widened-value-used-as-scalar-and-first-lane.ll
@@ -50,10 +50,9 @@ define void @iv.4_used_as_vector_and_first_lane(ptr %src, ptr noalias %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[G_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[G_SRC]], align 8
 ; CHECK-NEXT:    [[IV_4:%.*]] = add nuw nsw i64 [[IV]], 4
@@ -134,10 +133,9 @@ define void @iv.4_used_as_first_lane(ptr %src, ptr noalias %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[G_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i64, ptr [[G_SRC]], align 8
 ; CHECK-NEXT:    [[IV_4:%.*]] = add nuw nsw i64 [[IV]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
index b0ae40cafbde5..34c6384b63c80 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
@@ -136,12 +136,10 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
 ; SINK-GATHER-NEXT:    [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> [[TMP47]])
 ; SINK-GATHER-NEXT:    br label [[FOR_END:%.*]]
 ; SINK-GATHER:       scalar.ph:
-; SINK-GATHER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; SINK-GATHER-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; SINK-GATHER-NEXT:    br label [[FOR_BODY:%.*]]
 ; SINK-GATHER:       for.body:
-; SINK-GATHER-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; SINK-GATHER-NEXT:    [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[T7:%.*]], [[FOR_INC]] ]
+; SINK-GATHER-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; SINK-GATHER-NEXT:    [[R:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[T7:%.*]], [[FOR_INC]] ]
 ; SINK-GATHER-NEXT:    [[T0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I]]
 ; SINK-GATHER-NEXT:    [[T1:%.*]] = load i32, ptr [[T0]], align 4
 ; SINK-GATHER-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[FOR_INC]]
diff --git a/llvm/test/Transforms/LoopVectorize/branch-weights.ll b/llvm/test/Transforms/LoopVectorize/branch-weights.ll
index 6892709f085f7..7ae06953c5544 100644
--- a/llvm/test/Transforms/LoopVectorize/branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/branch-weights.ll
@@ -4,13 +4,6 @@
 ; RUN: opt < %s -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=4  -enable-epilogue-vectorization \
 ; RUN:   -epilogue-vectorization-force-VF=4 | FileCheck %s --check-prefix=MAINVF4IC2_EPI4
 
-; FIXME: For MAINVF4IC2_EPI4 the branch weights in the terminator of
-; the VEC_EPILOG_ITER_CHECK block should be [4,4] since we process 8
-; scalar iterations in the main loop, leaving the remaining count to
-; be in the range [0,7]. That gives a 4:4 chance of skipping the
-; vector epilogue. I believe the problem lies in
-; EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck
-; where the main loop VF is set to the same value as the epilogue VF.
 define void @f0(i8 %n, i32 %len, ptr %p) !prof !0 {
 ; MAINVF4IC1_EPI4-LABEL: define void @f0(
 ; MAINVF4IC1_EPI4-SAME: i8 [[N:%.*]], i32 [[LEN:%.*]], ptr [[P:%.*]]) !prof [[PROF0:![0-9]+]] {
@@ -145,7 +138,7 @@ exit:
 ; MAINVF4IC2_EPI4: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
 ; MAINVF4IC2_EPI4: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; MAINVF4IC2_EPI4: [[PROF7]] = !{!"branch_weights", i32 1, i32 7}
-; MAINVF4IC2_EPI4: [[PROF8]] = !{!"branch_weights", i32 4, i32 0}
+; MAINVF4IC2_EPI4: [[PROF8]] = !{!"branch_weights", i32 4, i32 4}
 ; MAINVF4IC2_EPI4: [[PROF9]] = !{!"branch_weights", i32 0, i32 0}
 ; MAINVF4IC2_EPI4: [[LOOP10]] = distinct !{[[LOOP10]], [[META5]], [[META6]]}
 ; MAINVF4IC2_EPI4: [[PROF11]] = !{!"branch_weights", i32 1, i32 3}
diff --git a/llvm/test/Transforms/LoopVectorize/cast-induction.ll b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
index 937bc1807d8a8..0b58bb65e2bc5 100644
--- a/llvm/test/Transforms/LoopVectorize/cast-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/cast-induction.ll
@@ -114,10 +114,8 @@ exit:
 
 define void @cast_induction_tail_folding(ptr %A) {
 ; VF4-LABEL: @cast_induction_tail_folding(
-; VF4:       [[INDEX:%.+]] = phi i32 [ 0, %vector.ph ]
-; VF4-NEXT:  [[VEC_IND:%.+]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %vector.ph ]
-; VF4-NEXT:  = icmp ule <4 x i32> [[VEC_IND]], splat (i32 2)
-; VF4-NEXT:  = sext <4 x i32> [[VEC_IND]] to <4 x i64>
+; VF4-LABEL: vector.body:
+; VF4-NEXT:   br i1 true, label %pred.store.if, label %pred.store.continue
 
 ; IC2-LABEL: @cast_induction_tail_folding(
 ; IC2:      [[INDEX:%.+]] = phi i32 [ 0, %vector.ph ]
diff --git a/llvm/test/Transforms/LoopVectorize/constantfolder-infer-correct-gepty.ll b/llvm/test/Transforms/LoopVectorize/constantfolder-infer-correct-gepty.ll
index af528eee503d6..e629560354f2a 100644
--- a/llvm/test/Transforms/LoopVectorize/constantfolder-infer-correct-gepty.ll
+++ b/llvm/test/Transforms/LoopVectorize/constantfolder-infer-correct-gepty.ll
@@ -6,7 +6,7 @@
 define void @test(ptr %data) {
 ; CHECK-LABEL: define void @test(
 ; CHECK-SAME: ptr [[DATA:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -20,10 +20,9 @@ define void @test(ptr %data) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[OR_IV_1:%.*]] = or disjoint i64 [[IV]], 1
 ; CHECK-NEXT:    [[GEP_POSTSCALE:%.*]] = getelementptr [64 x float], ptr @postscale, i64 0, i64 [[OR_IV_1]]
 ; CHECK-NEXT:    [[LOAD_POSTSCALE:%.*]] = load float, ptr [[GEP_POSTSCALE]], align 4, !tbaa [[TBAA0]]
diff --git a/llvm/test/Transforms/LoopVectorize/constantfolder.ll b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
index cfd36bfe36526..9fbd1330de74f 100644
--- a/llvm/test/Transforms/LoopVectorize/constantfolder.ll
+++ b/llvm/test/Transforms/LoopVectorize/constantfolder.ll
@@ -4,7 +4,7 @@
 define void @const_fold_ptradd(ptr %dst, i64 %d) {
 ; CHECK-LABEL: define void @const_fold_ptradd(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -17,10 +17,9 @@ define void @const_fold_ptradd(ptr %dst, i64 %d) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    br label %[[LOOP_LATCH]]
@@ -59,7 +58,7 @@ exit:
 define void @const_fold_inbounds_ptradd(ptr %dst, i64 %d) {
 ; CHECK-LABEL: define void @const_fold_inbounds_ptradd(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -72,10 +71,9 @@ define void @const_fold_inbounds_ptradd(ptr %dst, i64 %d) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    br label %[[LOOP_LATCH]]
@@ -114,7 +112,7 @@ exit:
 define void @const_fold_select(ptr %dst, i64 %d) {
 ; CHECK-LABEL: define void @const_fold_select(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = or i64 [[D]], 1
@@ -128,10 +126,9 @@ define void @const_fold_select(ptr %dst, i64 %d) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    br label %[[LOOP_LATCH]]
@@ -170,7 +167,7 @@ exit:
 define void @const_fold_add_sub_mul_ashr_lshr(ptr %dst, i64 %d) {
 ; CHECK-LABEL: define void @const_fold_add_sub_mul_ashr_lshr(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -183,10 +180,9 @@ define void @const_fold_add_sub_mul_ashr_lshr(ptr %dst, i64 %d) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    br label %[[LOOP_LATCH]]
@@ -233,7 +229,7 @@ exit:
 define void @const_fold_and_or_xor(ptr %dst, i64 %d) {
 ; CHECK-LABEL: define void @const_fold_and_or_xor(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -246,10 +242,9 @@ define void @const_fold_and_or_xor(ptr %dst, i64 %d) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    br label %[[LOOP_LATCH]]
@@ -292,7 +287,7 @@ exit:
 define void @const_fold_cmp_zext(ptr %dst, i64 %d) {
 ; CHECK-LABEL: define void @const_fold_cmp_zext(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -305,10 +300,9 @@ define void @const_fold_cmp_zext(ptr %dst, i64 %d) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    br label %[[LOOP_LATCH]]
@@ -349,7 +343,7 @@ exit:
 define void @const_fold_trunc(ptr %dst, i64 %d) {
 ; CHECK-LABEL: define void @const_fold_trunc(
 ; CHECK-SAME: ptr [[DST:%.*]], i64 [[D:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -362,10 +356,9 @@ define void @const_fold_trunc(ptr %dst, i64 %d) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    br label %[[LOOP_LATCH]]
diff --git a/llvm/test/Transforms/LoopVectorize/create-induction-resume.ll b/llvm/test/Transforms/LoopVectorize/create-induction-resume.ll
index e9c7f75cb3377..fbdc11dd98479 100644
--- a/llvm/test/Transforms/LoopVectorize/create-induction-resume.ll
+++ b/llvm/test/Transforms/LoopVectorize/create-induction-resume.ll
@@ -69,12 +69,10 @@ define void @test(i32 %arg, i32 %L1.limit, i32 %L2.switch, i1 %c, ptr %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[L2_HEADER_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1, [[L2_INNER_HEADER_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ 1, [[L2_INNER_HEADER_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[L2_INNER_HEADER:%.*]]
 ; CHECK:       L2.Inner.header:
-; CHECK-NEXT:    [[L2_ACCUM:%.*]] = phi i32 [ [[L2_ACCUM_NEXT:%.*]], [[L2_INNER_HEADER]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[L2_IV:%.*]] = phi i64 [ [[L2_IV_NEXT:%.*]], [[L2_INNER_HEADER]] ], [ [[BC_RESUME_VAL2]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[L2_ACCUM:%.*]] = phi i32 [ [[L2_ACCUM_NEXT:%.*]], [[L2_INNER_HEADER]] ], [ 1, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[L2_IV:%.*]] = phi i64 [ [[L2_IV_NEXT:%.*]], [[L2_INNER_HEADER]] ], [ 1, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[L2_ACCUM_NEXT]] = sub i32 [[L2_ACCUM]], [[L1_EXIT_VAL]]
 ; CHECK-NEXT:    [[L2_DUMMY_BUT_NEED_IT:%.*]] = sext i32 [[L2_ACCUM_NEXT]] to i64
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[L2_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll b/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
index baa967c485cc1..e3a8ca777ddd5 100644
--- a/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
+++ b/llvm/test/Transforms/LoopVectorize/dbg-outer-loop-vect.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -S -passes=loop-vectorize -enable-vplan-native-path -force-vector-interleave=1 -force-vector-width=4 | FileCheck %s
 
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
@@ -6,60 +6,60 @@ target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
 define void @foo(ptr %h) !dbg !4 {
 ; CHECK-LABEL: define void @foo(
 ; CHECK-SAME: ptr [[H:%.*]]) !dbg [[DBG4:![0-9]+]] {
-; CHECK-NEXT:  entry:
+; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:      #dbg_value(i64 0, [[META11:![0-9]+]], !DIExpression(), [[META20:![0-9]+]])
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]], !dbg [[DBG21:![0-9]+]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]], !dbg [[DBG21]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_COND_CLEANUP32:%.*]] ], !dbg [[DBG222:![0-9]+]]
-; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER1:%.*]], !dbg [[DBG21]]
-; CHECK:       for.cond5.preheader1:
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP4:%.*]], [[FOR_COND5_PREHEADER1]] ], !dbg [[DBG22:![0-9]+]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !dbg [[DBG21:![0-9]+]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]], !dbg [[DBG21]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ], !dbg [[DBG22:![0-9]+]]
+; CHECK-NEXT:    br label %[[FOR_COND5_PREHEADER1:.*]], !dbg [[DBG21]]
+; CHECK:       [[FOR_COND5_PREHEADER1]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP5:%.*]], %[[FOR_COND5_PREHEADER1]] ], !dbg [[DBG23:![0-9]+]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i32, ptr [[H]], <4 x i64> [[VEC_PHI]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true)), !dbg [[DBG23:![0-9]+]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i32, <4 x ptr> [[TMP0]], i64 1, !dbg [[DBG25:![0-9]+]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> splat (i32 1), <4 x ptr> [[TMP1]], i32 4, <4 x i1> splat (i1 true)), !dbg [[DBG23]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, <4 x ptr> [[TMP0]], i64 2, !dbg [[DBG25]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> splat (i32 2), <4 x ptr> [[TMP2]], i32 4, <4 x i1> splat (i1 true)), !dbg [[DBG23]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, <4 x ptr> [[TMP0]], i64 3, !dbg [[DBG25]]
-; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> splat (i32 3), <4 x ptr> [[TMP3]], i32 4, <4 x i1> splat (i1 true)), !dbg [[DBG23]]
-; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1), !dbg [[DBG26:![0-9]+]]
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 5), !dbg [[DBG27:![0-9]+]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0, !dbg [[DBG28:![0-9]+]]
-; CHECK-NEXT:    br i1 [[TMP6]], label [[FOR_COND_CLEANUP32]], label [[FOR_COND5_PREHEADER1]], !dbg [[DBG28]]
-; CHECK:       vector.latch:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4, !dbg [[DBG222]]
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20, !dbg [[DBG21]]
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !dbg [[DBG21]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]], !dbg [[DBG21]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 20, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]], !dbg [[DBG21]]
-; CHECK:       for.cond1.preheader:
-; CHECK-NEXT:    [[I_023:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC13:%.*]], [[FOR_COND_CLEANUP3:%.*]] ]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> zeroinitializer, <4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true)), !dbg [[DBG24:![0-9]+]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, <4 x ptr> [[TMP0]], i64 1, !dbg [[DBG26:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> splat (i32 1), <4 x ptr> [[TMP2]], i32 4, <4 x i1> splat (i1 true)), !dbg [[DBG24]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i32, <4 x ptr> [[TMP0]], i64 2, !dbg [[DBG26]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> splat (i32 2), <4 x ptr> [[TMP3]], i32 4, <4 x i1> splat (i1 true)), !dbg [[DBG24]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, <4 x ptr> [[TMP0]], i64 3, !dbg [[DBG26]]
+; CHECK-NEXT:    call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> splat (i32 3), <4 x ptr> [[TMP4]], i32 4, <4 x i1> splat (i1 true)), !dbg [[DBG24]]
+; CHECK-NEXT:    [[TMP5]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1), !dbg [[DBG27:![0-9]+]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], splat (i64 5), !dbg [[DBG28:![0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0, !dbg [[DBG29:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[VECTOR_LATCH]], label %[[FOR_COND5_PREHEADER1]], !dbg [[DBG29]]
+; CHECK:       [[VECTOR_LATCH]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4, !dbg [[DBG22]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 20, !dbg [[DBG21]]
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG21]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]], !dbg [[DBG21]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 20, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[FOR_COND1_PREHEADER:.*]], !dbg [[DBG21]]
+; CHECK:       [[FOR_COND1_PREHEADER]]:
+; CHECK-NEXT:    [[I_023:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC13:%.*]], %[[FOR_COND_CLEANUP3:.*]] ]
 ; CHECK-NEXT:      #dbg_value(i64 [[I_023]], [[META11]], !DIExpression(), [[META20]])
-; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER:%.*]], !dbg [[DBG28]]
-; CHECK:       for.cond5.preheader:
-; CHECK-NEXT:    [[L_022:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INC10:%.*]], [[FOR_COND5_PREHEADER]] ], !dbg [[DBG22]]
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i32, ptr [[H]], i64 [[L_022]]
-; CHECK-NEXT:    store i32 0, ptr [[TMP10]], align 4, !dbg [[DBG23]]
-; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr i32, ptr [[TMP10]], i64 1, !dbg [[DBG25]]
-; CHECK-NEXT:    store i32 1, ptr [[ARRAYIDX_1]], align 4, !dbg [[DBG23]]
-; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr i32, ptr [[TMP10]], i64 2, !dbg [[DBG25]]
-; CHECK-NEXT:    store i32 2, ptr [[ARRAYIDX_2]], align 4, !dbg [[DBG23]]
-; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr i32, ptr [[TMP10]], i64 3, !dbg [[DBG25]]
-; CHECK-NEXT:    store i32 3, ptr [[ARRAYIDX_3]], align 4, !dbg [[DBG23]]
-; CHECK-NEXT:    [[INC10]] = add nuw nsw i64 [[L_022]], 1, !dbg [[DBG26]]
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC10]], 5, !dbg [[DBG27]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP3]], label [[FOR_COND5_PREHEADER]], !dbg [[DBG28]]
-; CHECK:       for.cond.cleanup3:
-; CHECK-NEXT:    [[INC13]] = add nuw nsw i64 [[I_023]], 1, !dbg [[DBG33:![0-9]+]]
+; CHECK-NEXT:    br label %[[FOR_COND5_PREHEADER:.*]], !dbg [[DBG29]]
+; CHECK:       [[FOR_COND5_PREHEADER]]:
+; CHECK-NEXT:    [[L_022:%.*]] = phi i64 [ 0, %[[FOR_COND1_PREHEADER]] ], [ [[INC10:%.*]], %[[FOR_COND5_PREHEADER]] ], !dbg [[DBG23]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i32, ptr [[H]], i64 [[L_022]]
+; CHECK-NEXT:    store i32 0, ptr [[TMP9]], align 4, !dbg [[DBG24]]
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr i32, ptr [[TMP9]], i64 1, !dbg [[DBG26]]
+; CHECK-NEXT:    store i32 1, ptr [[ARRAYIDX_1]], align 4, !dbg [[DBG24]]
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr i32, ptr [[TMP9]], i64 2, !dbg [[DBG26]]
+; CHECK-NEXT:    store i32 2, ptr [[ARRAYIDX_2]], align 4, !dbg [[DBG24]]
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr i32, ptr [[TMP9]], i64 3, !dbg [[DBG26]]
+; CHECK-NEXT:    store i32 3, ptr [[ARRAYIDX_3]], align 4, !dbg [[DBG24]]
+; CHECK-NEXT:    [[INC10]] = add nuw nsw i64 [[L_022]], 1, !dbg [[DBG27]]
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC10]], 5, !dbg [[DBG28]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP3]], label %[[FOR_COND5_PREHEADER]], !dbg [[DBG29]]
+; CHECK:       [[FOR_COND_CLEANUP3]]:
+; CHECK-NEXT:    [[INC13]] = add nuw nsw i64 [[I_023]], 1, !dbg [[DBG22]]
 ; CHECK-NEXT:      #dbg_value(i64 [[INC13]], [[META11]], !DIExpression(), [[META20]])
 ; CHECK-NEXT:    [[EXITCOND24_NOT:%.*]] = icmp eq i64 [[INC13]], 23, !dbg [[DBG34:![0-9]+]]
-; CHECK-NEXT:    br i1 [[EXITCOND24_NOT]], label [[EXIT]], label [[FOR_COND1_PREHEADER]], !dbg [[DBG21]], !llvm.loop [[LOOP35:![0-9]+]]
-; CHECK:       exit:
+; CHECK-NEXT:    br i1 [[EXITCOND24_NOT]], label %[[EXIT]], label %[[FOR_COND1_PREHEADER]], !dbg [[DBG21]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void, !dbg [[DBG36:![0-9]+]]
 ;
 entry:
@@ -137,7 +137,7 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 !34 = !DILocation(line: 10, column: 5, scope: !12)
 ;.
 ; CHECK: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C11, file: [[META1:![0-9]+]], producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-; CHECK: [[META1]] = !DIFile(filename: "outer-loop-vect.c", directory: {{.*}})
+; CHECK: [[META1]] = !DIFile(filename: "{{.*}}outer-loop-vect.c", directory: {{.*}})
 ; CHECK: [[DBG4]] = distinct !DISubprogram(name: "foo", scope: [[META1]], file: [[META1]], line: 8, type: [[META5:![0-9]+]], scopeLine: 8, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META9:![0-9]+]])
 ; CHECK: [[META5]] = !DISubroutineType(types: [[META6:![0-9]+]])
 ; CHECK: [[META6]] = !{null, [[META7:![0-9]+]]}
@@ -156,19 +156,19 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 ; CHECK: [[META19]] = distinct !DILexicalBlock(scope: [[META15]], file: [[META1]], line: 11, column: 5)
 ; CHECK: [[META20]] = !DILocation(line: 0, scope: [[META12]])
 ; CHECK: [[DBG21]] = !DILocation(line: 10, column: 3, scope: [[META12]])
-; CHECK: [[DBG222]] = !DILocation(line: 10, column: 30, scope: [[META16]])
-; CHECK: [[DBG22]] = !DILocation(line: 10, column: 5, scope: [[META12]])
-; CHECK: [[DBG23]] = !DILocation(line: 13, column: 11, scope: [[META24:![0-9]+]])
-; CHECK: [[META24]] = distinct !DILexicalBlock(scope: [[META18]], file: [[META1]], line: 12, column: 7)
-; CHECK: [[DBG25]] = !DILocation(line: 13, column: 2, scope: [[META24]])
-; CHECK: [[DBG26]] = !DILocation(line: 11, column: 32, scope: [[META19]])
-; CHECK: [[DBG27]] = !DILocation(line: 11, column: 26, scope: [[META19]])
-; CHECK: [[DBG28]] = !DILocation(line: 11, column: 5, scope: [[META15]])
-; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[DBG21]], [[META30:![0-9]+]], [[META31:![0-9]+]], [[META32:![0-9]+]]}
-; CHECK: [[META30]] = !DILocation(line: 13, column: 13, scope: [[META12]])
-; CHECK: [[META31]] = !{!"llvm.loop.isvectorized", i32 1}
-; CHECK: [[META32]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[DBG22]] = !DILocation(line: 10, column: 30, scope: [[META16]])
+; CHECK: [[DBG23]] = !DILocation(line: 10, column: 5, scope: [[META12]])
+; CHECK: [[DBG24]] = !DILocation(line: 13, column: 11, scope: [[META25:![0-9]+]])
+; CHECK: [[META25]] = distinct !DILexicalBlock(scope: [[META18]], file: [[META1]], line: 12, column: 7)
+; CHECK: [[DBG26]] = !DILocation(line: 13, column: 2, scope: [[META25]])
+; CHECK: [[DBG27]] = !DILocation(line: 11, column: 32, scope: [[META19]])
+; CHECK: [[DBG28]] = !DILocation(line: 11, column: 26, scope: [[META19]])
+; CHECK: [[DBG29]] = !DILocation(line: 11, column: 5, scope: [[META15]])
+; CHECK: [[LOOP30]] = distinct !{[[LOOP30]], [[DBG21]], [[META31:![0-9]+]], [[META32:![0-9]+]], [[META33:![0-9]+]]}
+; CHECK: [[META31]] = !DILocation(line: 13, column: 13, scope: [[META12]])
+; CHECK: [[META32]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META33]] = !{!"llvm.loop.unroll.runtime.disable"}
 ; CHECK: [[DBG34]] = !DILocation(line: 10, column: 24, scope: [[META16]])
-; CHECK: [[LOOP35]] = distinct !{[[LOOP35]], [[DBG21]], [[META30]], [[META31]]}
+; CHECK: [[LOOP35]] = distinct !{[[LOOP35]], [[DBG21]], [[META31]], [[META32]]}
 ; CHECK: [[DBG36]] = !DILocation(line: 14, column: 1, scope: [[DBG4]])
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
index 42d45bda9d7d2..50e23fd8d3ef2 100644
--- a/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/dead_instructions.ll
@@ -83,7 +83,7 @@ for.end:
 define void @pr47390(ptr %a) {
 ; CHECK-LABEL: define void @pr47390(
 ; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -95,16 +95,13 @@ define void @pr47390(ptr %a) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ -1, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i32 [ 1, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[PRIMARY:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PRIMARY_ADD:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[USE_PRIMARY:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[PRIMARY]], %[[LOOP]] ]
-; CHECK-NEXT:    [[SECONDARY:%.*]] = phi i32 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[SECONDARY_ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PRIMARY:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[PRIMARY_ADD:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[USE_PRIMARY:%.*]] = phi i32 [ -1, %[[SCALAR_PH]] ], [ [[PRIMARY]], %[[LOOP]] ]
+; CHECK-NEXT:    [[SECONDARY:%.*]] = phi i32 [ 1, %[[SCALAR_PH]] ], [ [[SECONDARY_ADD:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[PRIMARY_ADD]] = add i32 [[PRIMARY]], 1
 ; CHECK-NEXT:    [[SECONDARY_ADD]] = add i32 [[SECONDARY]], 1
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[SECONDARY]]
diff --git a/llvm/test/Transforms/LoopVectorize/debugloc-optimize-vfuf-term.ll b/llvm/test/Transforms/LoopVectorize/debugloc-optimize-vfuf-term.ll
index 9ade6e9a89806..ab9a84dddf921 100644
--- a/llvm/test/Transforms/LoopVectorize/debugloc-optimize-vfuf-term.ll
+++ b/llvm/test/Transforms/LoopVectorize/debugloc-optimize-vfuf-term.ll
@@ -6,7 +6,7 @@
 define i32 @foo(ptr %p) {
 ; CHECK-LABEL: define i32 @foo(
 ; CHECK-SAME: ptr [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -16,10 +16,9 @@ define i32 @foo(ptr %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]], !dbg [[DBG3]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], !dbg [[DBG7:![0-9]+]]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], !dbg [[DBG7]]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[SCALAR_PH]] ], !dbg [[DBG7:![0-9]+]]
 ; CHECK-NEXT:    [[CONV:%.*]] = trunc i64 0 to i8, !dbg [[DBG8:![0-9]+]]
 ; CHECK-NEXT:    store i8 [[CONV]], ptr [[P]], align 1, !dbg [[DBG3]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1, !dbg [[DBG9:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
index 373c8e0b385c2..0316e8e39a6a4 100644
--- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
@@ -6,7 +6,7 @@ declare void @llvm.assume(i1)
 define void @deref_assumption_in_header_constant_trip_count(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -20,8 +20,7 @@ define void @deref_assumption_in_header_constant_trip_count(ptr noalias noundef
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[TMP5]], i64 4), "dereferenceable"(ptr [[TMP5]], i64 4) ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = xor <2 x i1> [[TMP9]], splat (i1 true)
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP10]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -39,8 +38,8 @@ define void @deref_assumption_in_header_constant_trip_count(ptr noalias noundef
 ; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP12]], i32 [[TMP27]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP9]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP17]]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP10]], <2 x i32> [[TMP14]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP30]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -50,10 +49,9 @@ define void @deref_assumption_in_header_constant_trip_count(ptr noalias noundef
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[GEP_A]], i64 4), "dereferenceable"(ptr [[GEP_A]], i64 4) ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
@@ -104,7 +102,7 @@ exit:
 define void @align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -127,10 +125,9 @@ define void @align_deref_assumption_in_header_constant_trip_count_loop_invariant
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
@@ -178,7 +175,7 @@ exit:
 define void @deref_assumption_too_small_in_header_constant_trip_count(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_too_small_in_header_constant_trip_count(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -192,8 +189,7 @@ define void @deref_assumption_too_small_in_header_constant_trip_count(ptr noalia
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[TMP5]], i64 4), "dereferenceable"(ptr [[TMP5]], i64 2) ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = xor <2 x i1> [[TMP9]], splat (i1 true)
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP10]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -211,8 +207,8 @@ define void @deref_assumption_too_small_in_header_constant_trip_count(ptr noalia
 ; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP12]], i32 [[TMP27]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP9]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP17]]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP10]], <2 x i32> [[TMP14]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP30]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -222,10 +218,9 @@ define void @deref_assumption_too_small_in_header_constant_trip_count(ptr noalia
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[GEP_A]], i64 4), "dereferenceable"(ptr [[GEP_A]], i64 2) ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
@@ -276,7 +271,7 @@ exit:
 define void @deref_assumption_in_header_constant_trip_count_align_1(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_align_1(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -290,8 +285,7 @@ define void @deref_assumption_in_header_constant_trip_count_align_1(ptr noalias
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[TMP3]], i64 4) ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = xor <2 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -310,7 +304,7 @@ define void @deref_assumption_in_header_constant_trip_count_align_1(ptr noalias
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP6]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP17]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i32> [[TMP17]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP18]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -320,10 +314,9 @@ define void @deref_assumption_in_header_constant_trip_count_align_1(ptr noalias
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[GEP_A]], i64 4) ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
@@ -374,7 +367,7 @@ exit:
 define void @deref_assumption_in_header_constant_trip_count_align_via_arg_attribute(ptr noalias align 4 %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_align_via_arg_attribute(
 ; CHECK-SAME: ptr noalias align 4 [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -388,8 +381,7 @@ define void @deref_assumption_in_header_constant_trip_count_align_via_arg_attrib
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[TMP3]], i64 4) ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = xor <2 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -408,7 +400,7 @@ define void @deref_assumption_in_header_constant_trip_count_align_via_arg_attrib
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP6]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP17]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i32> [[TMP17]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP18]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -418,10 +410,9 @@ define void @deref_assumption_in_header_constant_trip_count_align_via_arg_attrib
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[GEP_A]], i64 4) ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
@@ -472,7 +463,7 @@ exit:
 define void @deref_assumption_in_header_constant_trip_count_align_not_known(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_align_not_known(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -486,8 +477,7 @@ define void @deref_assumption_in_header_constant_trip_count_align_not_known(ptr
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[TMP3]], i64 4) ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP7:%.*]] = xor <2 x i1> [[TMP6]], splat (i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -506,7 +496,7 @@ define void @deref_assumption_in_header_constant_trip_count_align_not_known(ptr
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP6]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP17]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i32> [[TMP17]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP18]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -516,10 +506,9 @@ define void @deref_assumption_in_header_constant_trip_count_align_not_known(ptr
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[GEP_A]], i64 4) ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
@@ -570,7 +559,7 @@ exit:
 define void @deref_assumption_in_then_constant_trip_count(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_then_constant_trip_count(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -579,8 +568,7 @@ define void @deref_assumption_in_then_constant_trip_count(ptr noalias noundef %a
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[A]], <2 x i64> [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
@@ -599,8 +587,8 @@ define void @deref_assumption_in_then_constant_trip_count(ptr noalias noundef %a
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP23]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[TMP15:%.*]] = phi <2 x i32> [ [[TMP10]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP15]]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x i32> [ [[TMP10]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP26]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -610,10 +598,9 @@ define void @deref_assumption_in_then_constant_trip_count(ptr noalias noundef %a
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
@@ -664,7 +651,7 @@ exit:
 define void @deref_assumption_in_latch_constant_trip_count(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_latch_constant_trip_count(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -674,8 +661,7 @@ define void @deref_assumption_in_latch_constant_trip_count(ptr noalias noundef %
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[A]], <2 x i64> [[VEC_IND]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP5]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -693,8 +679,8 @@ define void @deref_assumption_in_latch_constant_trip_count(ptr noalias noundef %
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP23]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[TMP15:%.*]] = phi <2 x i32> [ [[TMP10]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP15]]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x i32> [ [[TMP10]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP5]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <2 x ptr> [[TMP1]], i32 0
 ; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x ptr> [[TMP1]], i32 0
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[TMP28]], i64 4), "dereferenceable"(ptr [[TMP20]], i64 4) ]
@@ -710,10 +696,9 @@ define void @deref_assumption_in_latch_constant_trip_count(ptr noalias noundef %
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
@@ -781,8 +766,7 @@ define void @deref_assumption_in_header_variable_trip_count(ptr noalias noundef
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[TMP5]], i64 4), "dereferenceable"(ptr [[TMP5]], i64 4) ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP10:%.*]] = xor <2 x i1> [[TMP9]], splat (i1 true)
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i1> [[TMP10]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP8]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -800,8 +784,8 @@ define void @deref_assumption_in_header_variable_trip_count(ptr noalias noundef
 ; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP12]], i32 [[TMP27]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[TMP17:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP9]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP17]]
+; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP10]], <2 x i32> [[TMP14]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP30]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -866,7 +850,7 @@ exit:
 define void @deref_assumption_in_preheader_constant_trip_count_align_1(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_constant_trip_count_align_1(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 4000) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -887,10 +871,9 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_1(ptr noali
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
@@ -940,7 +923,7 @@ exit:
 define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 3999) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -949,8 +932,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -971,7 +953,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP15]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP15]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -980,10 +962,9 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
@@ -1033,7 +1014,7 @@ exit:
 define void @align_and_deref_assumption_in_preheader_constant_trip_count_align_4(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @align_and_deref_assumption_in_preheader_constant_trip_count_align_4(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -1054,10 +1035,9 @@ define void @align_and_deref_assumption_in_preheader_constant_trip_count_align_4
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
@@ -1108,7 +1088,7 @@ exit:
 define void @deref_assumption_in_preheader_constant_trip_count_align_4_known_via_argument_attr(ptr noalias noundef align 4 %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_constant_trip_count_align_4_known_via_argument_attr(
 ; CHECK-SAME: ptr noalias noundef align 4 [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 4000) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -1129,10 +1109,9 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_4_known_via
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
@@ -1182,7 +1161,7 @@ exit:
 define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 4000) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -1191,8 +1170,7 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -1213,7 +1191,7 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP15]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP15]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -1222,10 +1200,9 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
@@ -1275,7 +1252,7 @@ exit:
 define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 3999) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -1284,8 +1261,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -1306,7 +1282,7 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP15]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP15]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -1315,10 +1291,9 @@ define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
@@ -1369,7 +1344,7 @@ exit:
 define void @may_free_align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) {
 ; CHECK-LABEL: define void @may_free_align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(
 ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4) ]
 ; CHECK-NEXT:    call void @may_free()
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1379,8 +1354,7 @@ define void @may_free_align_deref_assumption_in_header_constant_trip_count_loop_
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -1396,8 +1370,8 @@ define void @may_free_align_deref_assumption_in_header_constant_trip_count_loop_
 ; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP12]], i32 [[TMP14]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[TMP11:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP11]]
+; CHECK-NEXT:    [[TMP9:%.*]] = phi <2 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP16]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP9]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -1406,10 +1380,9 @@ define void @may_free_align_deref_assumption_in_header_constant_trip_count_loop_
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
@@ -1459,7 +1432,7 @@ exit:
 define void @may_free_local_ptr_align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(ptr noalias %b, ptr noalias %c) nofree nosync {
 ; CHECK-LABEL: define void @may_free_local_ptr_align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(
 ; CHECK-SAME: ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[A:%.*]] = call ptr @get_ptr()
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4) ]
 ; CHECK-NEXT:    call void @may_free()
@@ -1470,8 +1443,7 @@ define void @may_free_local_ptr_align_deref_assumption_in_header_constant_trip_c
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -1488,7 +1460,7 @@ define void @may_free_local_ptr_align_deref_assumption_in_header_constant_trip_c
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP12:%.*]] = phi <2 x i32> [ [[TMP8]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP12]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP12]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -1497,10 +1469,9 @@ define void @may_free_local_ptr_align_deref_assumption_in_header_constant_trip_c
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4
 ; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
index c8cf2ad8198a9..8417e22fe0d8a 100644
--- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
@@ -180,8 +180,7 @@ define void @deref_assumption_in_preheader_too_small_non_constant_trip_count_acc
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP15]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -201,8 +200,8 @@ define void @deref_assumption_in_preheader_too_small_non_constant_trip_count_acc
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP20:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP15]], <2 x i32> [[TMP20]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -278,8 +277,7 @@ define void @deref_assumption_in_preheader_too_small2_non_constant_trip_count_ac
 ; CHECK-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 1
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <2 x i1> [[TMP4]], splat (i1 true)
+; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP15]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -299,8 +297,8 @@ define void @deref_assumption_in_preheader_too_small2_non_constant_trip_count_ac
 ; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP13]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
-; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP20:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP15]], <2 x i32> [[TMP20]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP6]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2
@@ -458,8 +456,7 @@ define void @deref_assumption_in_preheader_non_constant_trip_count_access_i32_al
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true)
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
@@ -480,7 +477,7 @@ define void @deref_assumption_in_preheader_non_constant_trip_count_access_i32_al
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP15]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP15]], <2 x i32> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
 ; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP16]], align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -540,3 +537,225 @@ loop.latch:
 exit:
   ret void
 }
+
+; The start access is SCEV with non-constant offset because of variable `iv.start`
+; for IV.
+define void @deref_assumption_loop_access_start_variable(i8 %v, ptr noundef %P, i64 range(i64 0, 2000) %N, ptr noalias %b, ptr noalias %c, i64 range(i64 0, 2000) %iv.start) nofree nosync {
+; CHECK-LABEL: define void @deref_assumption_loop_access_start_variable(
+; CHECK-SAME: i8 [[V:%.*]], ptr noundef [[P:%.*]], i64 range(i64 0, 2000) [[N:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 range(i64 0, 2000) [[IV_START:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i8, ptr [[P]], i64 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[IV_START]], [[N]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[MUL:%.*]] = mul i64 [[N]], 4
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[MUL]], 16
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[ADD]]) ]
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i64 [[N]], [[IV_START]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[IV_START]], [[N_VEC]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK:       [[PRED_LOAD_IF]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[TMP7]], align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> poison, i32 [[TMP19]], i32 0
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; CHECK:       [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP9]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
+; CHECK:       [[PRED_LOAD_IF1]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP14]], i32 1
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK:       [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = phi <2 x i32> [ [[TMP10]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP20]], <2 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP17]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[IV_START]], %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
+; CHECK-NEXT:    br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[LOOP_THEN:.*]]
+; CHECK:       [[LOOP_THEN]]:
+; CHECK-NEXT:    [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 1
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[L_A]], %[[LOOP_THEN]] ], [ [[L_B]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; CHECK-NEXT:    store i32 [[MERGE]], ptr [[GEP_C]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TERM_COND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TERM_COND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  %a = getelementptr i8, ptr %P, i64 16
+  %cmp = icmp slt i64 %iv.start, %N
+  call void @llvm.assume(i1 %cmp)
+  %mul = mul i64 %N, 4
+  %add = add i64 %mul, 16
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %P, i64 %add) ]
+  br label %loop
+
+loop:                                             ; preds = %mainloop, %loop.latch
+  %iv = phi i64 [ %iv.next, %loop.latch ], [ %iv.start, %entry ]
+  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+  %l.b = load i32, ptr %gep.b, align 1
+  %c.1 = icmp sge i32 %l.b, 0
+  br i1 %c.1, label %loop.latch, label %loop.then
+
+loop.then:                                        ; preds = %loop
+  %l.a = load i32, ptr %gep.a, align 1
+  br label %loop.latch
+
+loop.latch:                                       ; preds = %loop.then, %loop
+  %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop ]
+  %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+  store i32 %merge, ptr %gep.c, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %term.cond = icmp slt i64 %iv.next, %N
+  br i1 %term.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Same as previous test, but `iv.start` is not known nonnegative.
+define void @deref_assumption_loop_access_start_variable_unknown_range(i8 %v, ptr noundef %P, i64 range(i64 0, 2000) %N, ptr noalias %b, ptr noalias %c, i64 %iv.start) nofree nosync {
+; CHECK-LABEL: define void @deref_assumption_loop_access_start_variable_unknown_range(
+; CHECK-SAME: i8 [[V:%.*]], ptr noundef [[P:%.*]], i64 range(i64 0, 2000) [[N:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[IV_START:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[A:%.*]] = getelementptr i8, ptr [[P]], i64 16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[IV_START]], [[N]]
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[MUL:%.*]] = mul i64 [[N]], 4
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[MUL]], 16
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[ADD]]) ]
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[N]], [[IV_START]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[IV_START]], [[N_VEC]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[IV_START]], [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK:       [[PRED_LOAD_IF]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> poison, i32 [[TMP8]], i32 0
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; CHECK:       [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP9]], %[[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]]
+; CHECK:       [[PRED_LOAD_IF1]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x i32> [[TMP10]], i32 [[TMP14]], i32 1
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK:       [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = phi <2 x i32> [ [[TMP10]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], %[[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP16]], <2 x i32> [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    store <2 x i32> [[PREDPHI]], ptr [[TMP17]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP1]], %[[MIDDLE_BLOCK]] ], [ [[IV_START]], %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; CHECK-NEXT:    [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; CHECK-NEXT:    [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 1
+; CHECK-NEXT:    [[C_1:%.*]] = icmp sge i32 [[L_B]], 0
+; CHECK-NEXT:    br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[LOOP_THEN:.*]]
+; CHECK:       [[LOOP_THEN]]:
+; CHECK-NEXT:    [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 1
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[MERGE:%.*]] = phi i32 [ [[L_A]], %[[LOOP_THEN]] ], [ [[L_B]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; CHECK-NEXT:    store i32 [[MERGE]], ptr [[GEP_C]], align 1
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[TERM_COND:%.*]] = icmp slt i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[TERM_COND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a = getelementptr i8, ptr %P, i64 16
+  %cmp = icmp slt i64 %iv.start, %N
+  call void @llvm.assume(i1 %cmp)
+  %mul = mul i64 %N, 4
+  %add = add i64 %mul, 16
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %P, i64 %add) ]
+  br label %loop
+
+loop:                                             ; preds = %mainloop, %loop.latch
+  %iv = phi i64 [ %iv.next, %loop.latch ], [ %iv.start, %entry ]
+  %gep.a = getelementptr inbounds i32, ptr %a, i64 %iv
+  %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv
+  %l.b = load i32, ptr %gep.b, align 1
+  %c.1 = icmp sge i32 %l.b, 0
+  br i1 %c.1, label %loop.latch, label %loop.then
+
+loop.then:                                        ; preds = %loop
+  %l.a = load i32, ptr %gep.a, align 1
+  br label %loop.latch
+
+loop.latch:                                       ; preds = %loop.then, %loop
+  %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop ]
+  %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv
+  store i32 %merge, ptr %gep.c, align 1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %term.cond = icmp slt i64 %iv.next, %N
+  br i1 %term.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-const-TC.ll b/llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-const-TC.ll
index 4f95bddc4b4c6..dae2cd3cacd03 100644
--- a/llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-const-TC.ll
+++ b/llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-const-TC.ll
@@ -26,10 +26,9 @@ define dso_local void @constTC(ptr noalias nocapture %A) optsize {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[RIV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[RIVPLUS1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RIV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[RIVPLUS1:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[RIV]]
 ; CHECK-NEXT:    store i32 13, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[RIVPLUS1]] = add nuw nsw i32 [[RIV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-divisible-TC.ll b/llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-divisible-TC.ll
index 1936b409bc150..0b8f9b65b27f8 100644
--- a/llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-divisible-TC.ll
+++ b/llvm/test/Transforms/LoopVectorize/dont-fold-tail-for-divisible-TC.ll
@@ -200,10 +200,9 @@ define dso_local void @cannotProveAlignedTC(ptr noalias nocapture %A, i32 %p, i3
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[RIV:%.*]] = phi i32 [ [[RIVPLUS1:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[RIV:%.*]] = phi i32 [ [[RIVPLUS1:%.*]], [[LOOP]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[RIV]]
 ; CHECK-NEXT:    store i32 13, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[RIVPLUS1]] = add nuw nsw i32 [[RIV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-dead-instructions.ll
index 71c2da2681b3d..414773cb00d7a 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-dead-instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-dead-instructions.ll
@@ -6,16 +6,61 @@ define i8 @recurrence_phi_with_same_incoming_values_after_simplifications(i8 %fo
 ; CHECK-LABEL: define i8 @recurrence_phi_with_same_incoming_values_after_simplifications(
 ; CHECK-SAME: i8 [[FOR_START:%.*]], ptr [[DST:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[FOR_START]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLAT]], <4 x i8> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i32 1, [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[OFFSET_IDX]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[OFFSET_IDX]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[OFFSET_IDX]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[OFFSET_IDX]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[OFFSET_IDX]], 5
+; CHECK-NEXT:    [[TMP7:%.*]] = add i32 [[OFFSET_IDX]], 6
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[OFFSET_IDX]], 7
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP7]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[TMP8]]
+; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i8> [[TMP0]], i32 0
+; CHECK-NEXT:    store i8 [[TMP17]], ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i8> [[TMP0]], i32 1
+; CHECK-NEXT:    store i8 [[TMP18]], ptr [[TMP10]], align 1
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <4 x i8> [[TMP0]], i32 2
+; CHECK-NEXT:    store i8 [[TMP19]], ptr [[TMP11]], align 1
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i8> [[TMP0]], i32 3
+; CHECK-NEXT:    store i8 [[TMP20]], ptr [[TMP12]], align 1
+; CHECK-NEXT:    store i8 [[TMP17]], ptr [[TMP13]], align 1
+; CHECK-NEXT:    store i8 [[TMP18]], ptr [[TMP14]], align 1
+; CHECK-NEXT:    store i8 [[TMP19]], ptr [[TMP15]], align 1
+; CHECK-NEXT:    store i8 [[TMP20]], ptr [[TMP16]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], -8
+; CHECK-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ -7, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[FOR_START]], %[[MIDDLE_BLOCK]] ], [ [[FOR_START]], %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[FOR:%.*]] = phi i8 [ [[FOR_START]], %[[ENTRY]] ], [ [[FOR_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i8 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[FOR_NEXT]] = and i8 [[FOR_START]], -1
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i8, ptr [[DST]], i32 [[IV]]
 ; CHECK-NEXT:    store i8 [[FOR]], ptr [[GEP_DST]], align 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 0
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    [[FOR_NEXT_LCSSA:%.*]] = phi i8 [ [[FOR_NEXT]], %[[LOOP]] ]
 ; CHECK-NEXT:    ret i8 [[FOR_NEXT_LCSSA]]
@@ -42,7 +87,7 @@ exit:
 define i32 @sink_after_dead_inst(ptr %A.ptr) {
 ; CHECK-LABEL: define i32 @sink_after_dead_inst(
 ; CHECK-SAME: ptr [[A_PTR:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -61,17 +106,15 @@ define i32 @sink_after_dead_inst(ptr %A.ptr) {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], splat (i16 4)
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
 ; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FOR]], 15
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i1 [[CMP]], true
 ; CHECK-NEXT:    [[VEC_DEAD:%.*]] = and i1 [[C]], true
@@ -82,7 +125,7 @@ define i32 @sink_after_dead_inst(ptr %A.ptr) {
 ; CHECK-NEXT:    [[EXT:%.*]] = zext i1 [[B3]] to i32
 ; CHECK-NEXT:    [[A_GEP:%.*]] = getelementptr i32, ptr [[A_PTR]], i16 [[IV]]
 ; CHECK-NEXT:    store i32 0, ptr [[A_GEP]], align 4
-; CHECK-NEXT:    br i1 [[VEC_DEAD]], label %[[FOR_END]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[VEC_DEAD]], label %[[FOR_END]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    [[FOR_LCSSA:%.*]] = phi i32 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[FOR_LCSSA]]
@@ -142,7 +185,7 @@ define void @sink_dead_inst(ptr %a) {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], splat (i16 4)
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], 40
-; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT1:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
@@ -163,7 +206,7 @@ define void @sink_dead_inst(ptr %a) {
 ; CHECK-NEXT:    [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[A]], i16 [[IV]]
 ; CHECK-NEXT:    store i16 [[USE_REC_1]], ptr [[GEP]], align 2
-; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_END:.*]], label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_END:.*]], label %[[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -205,7 +248,7 @@ define void @unused_recurrence(ptr %a) {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i16> [[STEP_ADD]], splat (i16 4)
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1024
-; CHECK-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i16> [[TMP1]], i32 3
 ; CHECK-NEXT:    br label %[[SCALAR_PH]]
@@ -220,7 +263,7 @@ define void @unused_recurrence(ptr %a) {
 ; CHECK-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
 ; CHECK-NEXT:    [[REC_1_PREV]] = add i16 [[IV_NEXT]], 5
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[IV]], 1000
-; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_END:.*]], label %[[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_END:.*]], label %[[FOR_COND]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-interleave-only.ll
index d95c487178196..c13d3421ba7f0 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-interleave-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-interleave-only.ll
@@ -4,7 +4,7 @@
 define float @for_load_interleave_only(ptr %src) {
 ; CHECK-LABEL: define float @for_load_interleave_only(
 ; CHECK-SAME: ptr [[SRC:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -23,14 +23,11 @@ define float @for_load_interleave_only(ptr %src) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[FOR:%.*]] = phi float [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 1, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[SRC]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi float [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 16
 ; CHECK-NEXT:    [[L]] = load float, ptr [[PTR_IV]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
index 715ea1c51aba6..899c20ab30c6d 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-multiply-recurrences.ll
@@ -121,14 +121,11 @@ define void @test_pr54223_sink_after_insertion_order(ptr noalias %a, ptr noalias
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT5:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi float [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR6:%.*]] = phi float [ [[SCALAR_RECUR_INIT5]], [[SCALAR_PH]] ], [ [[FOR_2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[FOR_1_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR6:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[FOR_2_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[NEG:%.*]] = fneg float [[SCALAR_RECUR6]]
 ; CHECK-NEXT:    [[MULADD:%.*]] = call float @llvm.fmuladd.f32(float [[SCALAR_RECUR]], float [[NEG]], float 0.000000e+00)
 ; CHECK-NEXT:    [[DST_GEP:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll
index 2bafa6c69560f..40587c0c8b68c 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll
@@ -15,7 +15,6 @@ define i64 @pr97452_scalable_vf1_for_live_out(ptr %src) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 23, [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 23, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
 ; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 [[TMP4]]
@@ -26,9 +25,9 @@ define i64 @pr97452_scalable_vf1_for_live_out(ptr %src) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD]] = load <vscale x 1 x i64>, ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> [[VECTOR_RECUR]], <vscale x 1 x i64> [[WIDE_LOAD]], i32 -1)
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-NEXT:    [[TMP10:%.*]] = sub i32 [[TMP9]], 1
@@ -83,7 +82,6 @@ define void @pr97452_scalable_vf1_for_no_live_out(ptr %src, ptr noalias %dst) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 23, [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 23, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
 ; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 [[TMP4]]
@@ -96,9 +94,9 @@ define void @pr97452_scalable_vf1_for_no_live_out(ptr %src, ptr noalias %dst) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> [[VECTOR_RECUR]], <vscale x 1 x i64> [[WIDE_LOAD]], i32 -1)
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 1 x i64> [[TMP7]], ptr [[TMP8]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 [[TMP11]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 3adfcf53e4564..10cbf66c783db 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -1195,12 +1195,10 @@ define i64 @constant_folded_previous_value() {
 ; UNROLL-NO-IC:       middle.block:
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_END:%.*]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[SCALAR_BODY:%.*]]
 ; UNROLL-NO-IC:       scalar.body:
-; UNROLL-NO-IC-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ]
-; UNROLL-NO-IC-NEXT:    [[VAR2:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ]
+; UNROLL-NO-IC-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ]
+; UNROLL-NO-IC-NEXT:    [[VAR2:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ]
 ; UNROLL-NO-IC-NEXT:    [[VAR3]] = add i64 0, 1
 ; UNROLL-NO-IC-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; UNROLL-NO-IC-NEXT:    [[COND:%.*]] = icmp eq i64 [[I_NEXT]], 1000
@@ -1222,12 +1220,10 @@ define i64 @constant_folded_previous_value() {
 ; UNROLL-NO-VF:       middle.block:
 ; UNROLL-NO-VF-NEXT:    br label [[FOR_END:%.*]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[SCALAR_BODY:%.*]]
 ; UNROLL-NO-VF:       scalar.body:
-; UNROLL-NO-VF-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ]
-; UNROLL-NO-VF-NEXT:    [[VAR2:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ]
+; UNROLL-NO-VF-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ]
+; UNROLL-NO-VF-NEXT:    [[VAR2:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ]
 ; UNROLL-NO-VF-NEXT:    [[VAR3]] = add i64 0, 1
 ; UNROLL-NO-VF-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; UNROLL-NO-VF-NEXT:    [[COND:%.*]] = icmp eq i64 [[I_NEXT]], 1000
@@ -1249,12 +1245,10 @@ define i64 @constant_folded_previous_value() {
 ; SINK-AFTER:       middle.block:
 ; SINK-AFTER-NEXT:    br label [[FOR_END:%.*]]
 ; SINK-AFTER:       scalar.ph:
-; SINK-AFTER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; SINK-AFTER-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, [[ENTRY]] ]
 ; SINK-AFTER-NEXT:    br label [[SCALAR_BODY:%.*]]
 ; SINK-AFTER:       scalar.body:
-; SINK-AFTER-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ]
-; SINK-AFTER-NEXT:    [[VAR2:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ]
+; SINK-AFTER-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[SCALAR_BODY]] ]
+; SINK-AFTER-NEXT:    [[VAR2:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[VAR3:%.*]], [[SCALAR_BODY]] ]
 ; SINK-AFTER-NEXT:    [[VAR3]] = add i64 0, 1
 ; SINK-AFTER-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
 ; SINK-AFTER-NEXT:    [[COND:%.*]] = icmp eq i64 [[I_NEXT]], 1000
@@ -2742,17 +2736,14 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; UNROLL-NO-IC-NEXT:    [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
 ; UNROLL-NO-IC-NEXT:    br label [[BB1:%.*]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[Y]], [[BB:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[BB2:%.*]]
 ; UNROLL-NO-IC:       bb1:
 ; UNROLL-NO-IC-NEXT:    [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP51]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-IC-NEXT:    ret i32 [[VAR]]
 ; UNROLL-NO-IC:       bb2:
-; UNROLL-NO-IC-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NO-IC-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
-; UNROLL-NO-IC-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[Y]], [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
 ; UNROLL-NO-IC-NEXT:    [[VAR6]] = add i32 [[VAR5]], [[VAR4]]
 ; UNROLL-NO-IC-NEXT:    [[VAR7]] = udiv i32 219220132, [[VAR3]]
 ; UNROLL-NO-IC-NEXT:    [[VAR8]] = add nsw i32 [[VAR3]], -1
@@ -2805,17 +2796,14 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; UNROLL-NO-VF-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP12]]
 ; UNROLL-NO-VF-NEXT:    br label [[BB1:%.*]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[Y]], [[BB:%.*]] ]
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB]] ]
-; UNROLL-NO-VF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[BB2:%.*]]
 ; UNROLL-NO-VF:       bb1:
 ; UNROLL-NO-VF-NEXT:    [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-VF-NEXT:    ret i32 [[VAR]]
 ; UNROLL-NO-VF:       bb2:
-; UNROLL-NO-VF-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NO-VF-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
-; UNROLL-NO-VF-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[Y]], [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
 ; UNROLL-NO-VF-NEXT:    [[VAR6]] = add i32 [[VAR5]], [[VAR4]]
 ; UNROLL-NO-VF-NEXT:    [[VAR7]] = udiv i32 219220132, [[VAR3]]
 ; UNROLL-NO-VF-NEXT:    [[VAR8]] = add nsw i32 [[VAR3]], -1
@@ -2891,17 +2879,14 @@ define i32 @sink_into_replication_region(i32 %y) {
 ; SINK-AFTER-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP25]])
 ; SINK-AFTER-NEXT:    br label [[BB1:%.*]]
 ; SINK-AFTER:       scalar.ph:
-; SINK-AFTER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[Y]], [[BB:%.*]] ]
-; SINK-AFTER-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB]] ]
-; SINK-AFTER-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ]
 ; SINK-AFTER-NEXT:    br label [[BB2:%.*]]
 ; SINK-AFTER:       bb1:
 ; SINK-AFTER-NEXT:    [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ]
 ; SINK-AFTER-NEXT:    ret i32 [[VAR]]
 ; SINK-AFTER:       bb2:
-; SINK-AFTER-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; SINK-AFTER-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
-; SINK-AFTER-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[Y]], [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
 ; SINK-AFTER-NEXT:    [[VAR6]] = add i32 [[VAR5]], [[VAR4]]
 ; SINK-AFTER-NEXT:    [[VAR7]] = udiv i32 219220132, [[VAR3]]
 ; SINK-AFTER-NEXT:    [[VAR8]] = add nsw i32 [[VAR3]], -1
@@ -3104,19 +3089,15 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; UNROLL-NO-IC-NEXT:    [[TMP75:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
 ; UNROLL-NO-IC-NEXT:    br label [[BB1:%.*]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[Y]], [[BB:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 0, [[BB]] ]
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[BB2:%.*]]
 ; UNROLL-NO-IC:       bb1:
 ; UNROLL-NO-IC-NEXT:    [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP75]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-IC-NEXT:    ret i32 [[VAR]]
 ; UNROLL-NO-IC:       bb2:
-; UNROLL-NO-IC-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NO-IC-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; UNROLL-NO-IC-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
-; UNROLL-NO-IC-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[Y]], [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
+; UNROLL-NO-IC-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
 ; UNROLL-NO-IC-NEXT:    [[G:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[IV]]
 ; UNROLL-NO-IC-NEXT:    [[VAR6]] = add i32 [[VAR5]], [[VAR4]]
 ; UNROLL-NO-IC-NEXT:    [[VAR7]] = udiv i32 219220132, [[VAR3]]
@@ -3185,19 +3166,15 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; UNROLL-NO-VF-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP17]], [[TMP16]]
 ; UNROLL-NO-VF-NEXT:    br label [[BB1:%.*]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[Y]], [[BB:%.*]] ]
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 0, [[BB]] ]
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB]] ]
-; UNROLL-NO-VF-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[BB2:%.*]]
 ; UNROLL-NO-VF:       bb1:
 ; UNROLL-NO-VF-NEXT:    [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
 ; UNROLL-NO-VF-NEXT:    ret i32 [[VAR]]
 ; UNROLL-NO-VF:       bb2:
-; UNROLL-NO-VF-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NO-VF-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; UNROLL-NO-VF-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
-; UNROLL-NO-VF-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[Y]], [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
+; UNROLL-NO-VF-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
 ; UNROLL-NO-VF-NEXT:    [[G:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[IV]]
 ; UNROLL-NO-VF-NEXT:    [[VAR6]] = add i32 [[VAR5]], [[VAR4]]
 ; UNROLL-NO-VF-NEXT:    [[VAR7]] = udiv i32 219220132, [[VAR3]]
@@ -3307,19 +3284,15 @@ define i32 @sink_into_replication_region_multiple(ptr %x, i32 %y) {
 ; SINK-AFTER-NEXT:    [[TMP39:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP37]])
 ; SINK-AFTER-NEXT:    br label [[BB1:%.*]]
 ; SINK-AFTER:       scalar.ph:
-; SINK-AFTER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[Y]], [[BB:%.*]] ]
-; SINK-AFTER-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 0, [[BB]] ]
-; SINK-AFTER-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[BB]] ]
-; SINK-AFTER-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[BB]] ]
 ; SINK-AFTER-NEXT:    br label [[BB2:%.*]]
 ; SINK-AFTER:       bb1:
 ; SINK-AFTER-NEXT:    [[VAR:%.*]] = phi i32 [ [[VAR6:%.*]], [[BB2]] ], [ [[TMP39]], [[MIDDLE_BLOCK]] ]
 ; SINK-AFTER-NEXT:    ret i32 [[VAR]]
 ; SINK-AFTER:       bb2:
-; SINK-AFTER-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; SINK-AFTER-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
-; SINK-AFTER-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ]
-; SINK-AFTER-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[VAR3:%.*]] = phi i32 [ [[VAR8:%.*]], [[BB2]] ], [ [[Y]], [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[VAR4:%.*]] = phi i32 [ [[VAR7:%.*]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
+; SINK-AFTER-NEXT:    [[VAR5:%.*]] = phi i32 [ [[VAR6]], [[BB2]] ], [ 0, [[SCALAR_PH]] ]
 ; SINK-AFTER-NEXT:    [[G:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[IV]]
 ; SINK-AFTER-NEXT:    [[VAR6]] = add i32 [[VAR5]], [[VAR4]]
 ; SINK-AFTER-NEXT:    [[VAR7]] = udiv i32 219220132, [[VAR3]]
@@ -3379,12 +3352,10 @@ define i32 @sink_after_dead_inst(ptr %A.ptr, i32 %n) {
 ; UNROLL-NO-IC-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_END:%.*]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-IC:       loop:
-; UNROLL-NO-IC-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; UNROLL-NO-IC-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ]
+; UNROLL-NO-IC-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; UNROLL-NO-IC-NEXT:    [[FOR:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ]
 ; UNROLL-NO-IC-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FOR]], 15
 ; UNROLL-NO-IC-NEXT:    [[C:%.*]] = icmp eq i1 [[CMP]], true
 ; UNROLL-NO-IC-NEXT:    [[VEC_DEAD:%.*]] = and i1 [[C]], true
@@ -3422,12 +3393,10 @@ define i32 @sink_after_dead_inst(ptr %A.ptr, i32 %n) {
 ; UNROLL-NO-VF:       middle.block:
 ; UNROLL-NO-VF-NEXT:    br label [[FOR_END:%.*]]
 ; UNROLL-NO-VF:       scalar.ph:
-; UNROLL-NO-VF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-VF-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; UNROLL-NO-VF-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-VF:       loop:
-; UNROLL-NO-VF-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; UNROLL-NO-VF-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ]
+; UNROLL-NO-VF-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; UNROLL-NO-VF-NEXT:    [[FOR:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ]
 ; UNROLL-NO-VF-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FOR]], 15
 ; UNROLL-NO-VF-NEXT:    [[C:%.*]] = icmp eq i1 [[CMP]], true
 ; UNROLL-NO-VF-NEXT:    [[VEC_DEAD:%.*]] = and i1 [[C]], true
@@ -3465,12 +3434,10 @@ define i32 @sink_after_dead_inst(ptr %A.ptr, i32 %n) {
 ; SINK-AFTER-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
 ; SINK-AFTER-NEXT:    br label [[FOR_END:%.*]]
 ; SINK-AFTER:       scalar.ph:
-; SINK-AFTER-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ]
-; SINK-AFTER-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; SINK-AFTER-NEXT:    br label [[LOOP:%.*]]
 ; SINK-AFTER:       loop:
-; SINK-AFTER-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; SINK-AFTER-NEXT:    [[FOR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ]
+; SINK-AFTER-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; SINK-AFTER-NEXT:    [[FOR:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[FOR_PREV:%.*]], [[LOOP]] ]
 ; SINK-AFTER-NEXT:    [[CMP:%.*]] = icmp eq i32 [[FOR]], 15
 ; SINK-AFTER-NEXT:    [[C:%.*]] = icmp eq i1 [[CMP]], true
 ; SINK-AFTER-NEXT:    [[VEC_DEAD:%.*]] = and i1 [[C]], true
diff --git a/llvm/test/Transforms/LoopVectorize/float-minmax-instruction-flag.ll b/llvm/test/Transforms/LoopVectorize/float-minmax-instruction-flag.ll
index 2c02f839edffa..5f4214c5d632c 100644
--- a/llvm/test/Transforms/LoopVectorize/float-minmax-instruction-flag.ll
+++ b/llvm/test/Transforms/LoopVectorize/float-minmax-instruction-flag.ll
@@ -68,12 +68,10 @@ define float @minloopattr(ptr nocapture readonly %arg) #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP4]])
 ; CHECK-NEXT:    br label [[OUT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1, [[TOP:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[T]], [[TOP]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[T1:%.*]] = phi i64 [ [[T7:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[T2:%.*]] = phi float [ [[T6:%.*]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[T1:%.*]] = phi i64 [ [[T7:%.*]], [[LOOP]] ], [ 1, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[T2:%.*]] = phi float [ [[T6:%.*]], [[LOOP]] ], [ [[T]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[T3:%.*]] = getelementptr float, ptr [[ARG]], i64 [[T1]]
 ; CHECK-NEXT:    [[T4:%.*]] = load float, ptr [[T3]], align 4
 ; CHECK-NEXT:    [[T5:%.*]] = fcmp olt float [[T2]], [[T4]]
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
index 720ea1f79c36d..3802845e82c82 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
@@ -75,8 +75,7 @@ define float @fmaxnum(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = call <4 x float> @llvm.maxnum.v4f32(<4 x float> [[TMP11]], <4 x float> [[TMP12]])
 ; CHECK-NEXT:    [[TMP13:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[RDX_MINMAX_SELECT]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP15:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP15]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP6]], true
 ; CHECK-NEXT:    [[TMP17:%.*]] = and i1 [[CMP_N]], [[TMP16]]
 ; CHECK-NEXT:    br i1 [[TMP17]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
index 3ef37bc34bb1b..f2d556e2759b7 100644
--- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
@@ -217,8 +217,7 @@ define float @fmaxnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP3]], true
 ; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -284,8 +283,7 @@ define float @fmaxnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP3]], true
 ; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -354,8 +352,7 @@ define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP10:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP8]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 10, [[TMP9]]
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i1 [[TMP6]], true
 ; CHECK-NEXT:    [[TMP14:%.*]] = and i1 [[CMP_N]], [[TMP13]]
 ; CHECK-NEXT:    br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -424,8 +421,7 @@ define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) {
 ; CHECK-NEXT:    [[TMP10:%.*]] = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> [[TMP8]])
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[START]], [[TMP9]]
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP12:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = xor i1 [[TMP6]], true
 ; CHECK-NEXT:    [[TMP14:%.*]] = and i1 [[CMP_N]], [[TMP13]]
 ; CHECK-NEXT:    br i1 [[TMP14]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
index 0f688ab0dfb0b..95c6f2848037e 100644
--- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
@@ -217,8 +217,7 @@ define float @fminnum_1(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP3]], true
 ; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
@@ -284,8 +283,7 @@ define float @fminnum_2(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP3]], i64 [[IV]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP7]])
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
-; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i1 [[TMP3]], true
 ; CHECK-NEXT:    [[TMP12:%.*]] = and i1 [[CMP_N]], [[TMP11]]
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
diff --git a/llvm/test/Transforms/LoopVectorize/forked-pointers.ll b/llvm/test/Transforms/LoopVectorize/forked-pointers.ll
index 677163b51ec64..efd420c11ef06 100644
--- a/llvm/test/Transforms/LoopVectorize/forked-pointers.ll
+++ b/llvm/test/Transforms/LoopVectorize/forked-pointers.ll
@@ -17,21 +17,22 @@ target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 define dso_local void @forked_ptrs_different_base_same_offset(ptr nocapture readonly %Base1, ptr nocapture readonly %Base2, ptr nocapture %Dest, ptr nocapture readonly %Preds) {
 ; CHECK-LABEL: @forked_ptrs_different_base_same_offset(
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE1:%.*]] = freeze ptr [[BASE3:%.*]]
+; CHECK-NEXT:    [[BASE2:%.*]] = freeze ptr [[BASE4:%.*]]
+; CHECK-NEXT:    [[DEST:%.*]] = freeze ptr [[DEST2:%.*]]
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
-; CHECK-NEXT:    [[DEST1:%.*]] = ptrtoint ptr [[DEST:%.*]] to i64
+; CHECK-NEXT:    [[DEST1:%.*]] = ptrtoint ptr [[DEST]] to i64
 ; CHECK-NEXT:    [[PREDS2:%.*]] = ptrtoint ptr [[PREDS:%.*]] to i64
-; CHECK-NEXT:    [[BASE23:%.*]] = ptrtoint ptr [[BASE2:%.*]] to i64
-; CHECK-NEXT:    [[BASE15:%.*]] = ptrtoint ptr [[BASE1:%.*]] to i64
+; CHECK-NEXT:    [[BASE23:%.*]] = ptrtoint ptr [[BASE2]] to i64
+; CHECK-NEXT:    [[BASE15:%.*]] = ptrtoint ptr [[BASE1]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[DEST1]], [[PREDS2]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[DEST1]], [[BASE23]]
-; CHECK-NEXT:    [[DOTFR:%.*]] = freeze i64 [[TMP1]]
-; CHECK-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[DOTFR]], 16
+; CHECK-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP1]], 16
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[DEST1]], [[BASE15]]
-; CHECK-NEXT:    [[DOTFR10:%.*]] = freeze i64 [[TMP2]]
-; CHECK-NEXT:    [[DIFF_CHECK6:%.*]] = icmp ult i64 [[DOTFR10]], 16
+; CHECK-NEXT:    [[DIFF_CHECK6:%.*]] = icmp ult i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[CONFLICT_RDX7:%.*]] = or i1 [[CONFLICT_RDX]], [[DIFF_CHECK6]]
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX7]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/histograms.ll b/llvm/test/Transforms/LoopVectorize/histograms.ll
index 1adc0bf0c9ec0..f0ceae7d5816b 100644
--- a/llvm/test/Transforms/LoopVectorize/histograms.ll
+++ b/llvm/test/Transforms/LoopVectorize/histograms.ll
@@ -1,25 +1,48 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
 ; RUN: opt < %s -passes=loop-vectorize,instcombine -enable-histogram-loop-vectorization -force-vector-width=2 -S | FileCheck %s
 
-;; Currently we don't expect this to vectorize, since the generic cost model returns
-;; invalid for the histogram intrinsic.
 define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %N) {
 ; CHECK-LABEL: define void @simple_histogram(
 ; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], -2
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i32> [[WIDE_LOAD]] to <2 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP3]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x ptr> [[TMP6]], ptr [[TMP5]], i64 1
+; CHECK-NEXT:    call void @llvm.experimental.vector.histogram.add.v2p0.i32(<2 x ptr> [[TMP7]], i32 1, <2 x i1> splat (i1 true))
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[IV_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY1:%.*]] ]
+; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i32, ptr [[INDICES]], i64 [[IV]]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[INDICES]], i64 [[IV1]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[IDXPROM1:%.*]] = zext i32 [[TMP12]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i32, ptr [[BUCKETS]], i64 [[IDXPROM1]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP13]], 1
 ; CHECK-NEXT:    store i32 [[INC]], ptr [[ARRAYIDX2]], align 4
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT:%.*]], label [[FOR_BODY]]
+; CHECK-NEXT:    [[IV_NEXT1]] = add nuw nsw i64 [[IV1]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT1]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_EXIT]], label [[FOR_BODY1]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       for.exit:
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
index b971400c662b3..ade90894ba90a 100644
--- a/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-pred-stores.ll
@@ -69,10 +69,9 @@ define i32 @test(ptr nocapture %f) #0 {
 ; UNROLL-NOSIMPLIFY:       middle.block:
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[FOR_END:%.*]]
 ; UNROLL-NOSIMPLIFY:       scalar.ph:
-; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NOSIMPLIFY:       for.body:
-; UNROLL-NOSIMPLIFY-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[F]], i64 [[INDVARS_IV]]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; UNROLL-NOSIMPLIFY-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[TMP11]], 100
@@ -452,12 +451,10 @@ define void @minimal_bit_widths(i1 %c) {
 ; UNROLL-NOSIMPLIFY:       middle.block:
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[FOR_END:%.*]]
 ; UNROLL-NOSIMPLIFY:       scalar.ph:
-; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ 1000, [[ENTRY]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NOSIMPLIFY:       for.body:
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = phi i64 [ [[TMP9:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP2:%.*]] = phi i64 [ [[TMP7:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = phi i64 [ [[TMP9:%.*]], [[FOR_INC:%.*]] ], [ 0, [[SCALAR_PH]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP2:%.*]] = phi i64 [ [[TMP7:%.*]], [[FOR_INC]] ], [ 1000, [[SCALAR_PH]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr undef, i64 [[TMP1]]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[FOR_INC]]
@@ -580,12 +577,10 @@ define void @minimal_bit_widths_with_aliasing_store(i1 %c, ptr %ptr) {
 ; UNROLL-NOSIMPLIFY:       middle.block:
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[FOR_END:%.*]]
 ; UNROLL-NOSIMPLIFY:       scalar.ph:
-; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NOSIMPLIFY-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i64 [ 1000, [[ENTRY]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NOSIMPLIFY:       for.body:
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = phi i64 [ [[TMP9:%.*]], [[FOR_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; UNROLL-NOSIMPLIFY-NEXT:    [[TMP2:%.*]] = phi i64 [ [[TMP7:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP1:%.*]] = phi i64 [ [[TMP9:%.*]], [[FOR_INC:%.*]] ], [ 0, [[SCALAR_PH]] ]
+; UNROLL-NOSIMPLIFY-NEXT:    [[TMP2:%.*]] = phi i64 [ [[TMP7:%.*]], [[FOR_INC]] ], [ 1000, [[SCALAR_PH]] ]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP1]]
 ; UNROLL-NOSIMPLIFY-NEXT:    [[TMP4:%.*]] = load i8, ptr [[TMP3]], align 1
 ; UNROLL-NOSIMPLIFY-NEXT:    store i8 0, ptr [[TMP3]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/if-reduction.ll b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
index 65330aabf3c9a..ad6a1ec684e32 100644
--- a/llvm/test/Transforms/LoopVectorize/if-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/if-reduction.ll
@@ -1192,14 +1192,13 @@ define float @fcmp_multi(ptr nocapture readonly %a, i32 %n) nounwind readonly {
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
 ; CHECK-NEXT:    [[TMP5:%.*]] = fcmp olt <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
 ; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP6]], <4 x float> [[TMP9]], <4 x float> [[TMP8]]
-; CHECK-NEXT:    [[PREDPHI1:%.*]] = select <4 x i1> [[TMP3]], <4 x float> [[WIDE_LOAD]], <4 x float> [[PREDPHI]]
+; CHECK-NEXT:    [[PREDPHI1:%.*]] = select <4 x i1> [[TMP4]], <4 x float> [[PREDPHI]], <4 x float> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP10]] = fadd fast <4 x float> [[PREDPHI1]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
@@ -1315,14 +1314,13 @@ define float @fcmp_fadd_fsub(ptr nocapture readonly %a, i32 %n) nounwind readonl
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI1:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <4 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
 ; CHECK-NEXT:    [[TMP8:%.*]] = fcmp uge <4 x float> [[WIDE_LOAD]], splat (float 3.000000e+00)
 ; CHECK-NEXT:    [[TMP6:%.*]] = fsub fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <4 x float> [[WIDE_LOAD]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP9]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP6]]
-; CHECK-NEXT:    [[PREDPHI1]] = select <4 x i1> [[TMP3]], <4 x float> [[TMP7]], <4 x float> [[PREDPHI]]
+; CHECK-NEXT:    [[PREDPHI1]] = select <4 x i1> [[TMP4]], <4 x float> [[PREDPHI]], <4 x float> [[TMP7]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/induction-multiple-uses-in-same-instruction.ll b/llvm/test/Transforms/LoopVectorize/induction-multiple-uses-in-same-instruction.ll
index 9cff1cfae0b15..ff7594a5d3a8b 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-multiple-uses-in-same-instruction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-multiple-uses-in-same-instruction.ll
@@ -26,10 +26,9 @@ define void @multiple_iv_uses_in_same_instruction(ptr %ptr) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds [100 x [100 x i32]], ptr [[PTR]], i64 0, i64 [[IV]], i64 [[IV]]
 ; CHECK-NEXT:    [[T:%.*]] = trunc i64 [[IV]] to i32
 ; CHECK-NEXT:    store i32 [[T]], ptr [[GEP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/induction-step.ll b/llvm/test/Transforms/LoopVectorize/induction-step.ll
index 59f6e8b04f54d..0dab78039ea63 100644
--- a/llvm/test/Transforms/LoopVectorize/induction-step.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction-step.ll
@@ -276,7 +276,7 @@ for.end:
 define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
 ; CHECK-LABEL: define void @iv_no_binary_op_in_descriptor(
 ; CHECK-SAME: i1 [[C:%.*]], ptr [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -292,10 +292,9 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT_P:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT_P:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[IV]], ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[IV_NEXT:%.*]] = add i64 [[IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/induction.ll b/llvm/test/Transforms/LoopVectorize/induction.ll
index 77b91ccb913cf..cbf9cc11d9009 100644
--- a/llvm/test/Transforms/LoopVectorize/induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/induction.ll
@@ -2747,12 +2747,10 @@ define i32 @i8_loop() nounwind readnone ssp uwtable {
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[TMP0]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[A_0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[B_0:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[B_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[A_0:%.*]] = phi i32 [ 1, [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[B_0:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[B_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[A_0_AND]] = and i32 [[A_0]], 4
 ; CHECK-NEXT:    [[B_NEXT]] = add i8 [[B_0]], -1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i8 [[B_NEXT]], 0
@@ -2818,12 +2816,10 @@ define i32 @i8_loop() nounwind readnone ssp uwtable {
 ; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[BIN_RDX]])
 ; UNROLL-NO-IC-NEXT:    br label [[EXIT:%.*]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-IC:       loop:
-; UNROLL-NO-IC-NEXT:    [[A_0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ]
-; UNROLL-NO-IC-NEXT:    [[B_0:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[B_NEXT:%.*]], [[LOOP]] ]
+; UNROLL-NO-IC-NEXT:    [[A_0:%.*]] = phi i32 [ 1, [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ]
+; UNROLL-NO-IC-NEXT:    [[B_0:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[B_NEXT:%.*]], [[LOOP]] ]
 ; UNROLL-NO-IC-NEXT:    [[A_0_AND]] = and i32 [[A_0]], 4
 ; UNROLL-NO-IC-NEXT:    [[B_NEXT]] = add i8 [[B_0]], -1
 ; UNROLL-NO-IC-NEXT:    [[EC:%.*]] = icmp eq i8 [[B_NEXT]], 0
@@ -2884,12 +2880,10 @@ define i32 @i16_loop() nounwind readnone ssp uwtable {
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[TMP0]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[A_0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[B_0:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[B_0_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[A_0:%.*]] = phi i32 [ 1, [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[B_0:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[B_0_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[A_0_AND]] = and i32 [[A_0]], 4
 ; CHECK-NEXT:    [[B_0_NEXT]] = add i16 [[B_0]], -1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i16 [[B_0_NEXT]], 0
@@ -2955,12 +2949,10 @@ define i32 @i16_loop() nounwind readnone ssp uwtable {
 ; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.and.v2i32(<2 x i32> [[BIN_RDX]])
 ; UNROLL-NO-IC-NEXT:    br label [[EXIT:%.*]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-IC:       loop:
-; UNROLL-NO-IC-NEXT:    [[A_0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ]
-; UNROLL-NO-IC-NEXT:    [[B_0:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[B_0_NEXT:%.*]], [[LOOP]] ]
+; UNROLL-NO-IC-NEXT:    [[A_0:%.*]] = phi i32 [ 1, [[SCALAR_PH]] ], [ [[A_0_AND:%.*]], [[LOOP]] ]
+; UNROLL-NO-IC-NEXT:    [[B_0:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[B_0_NEXT:%.*]], [[LOOP]] ]
 ; UNROLL-NO-IC-NEXT:    [[A_0_AND]] = and i32 [[A_0]], 4
 ; UNROLL-NO-IC-NEXT:    [[B_0_NEXT]] = add i16 [[B_0]], -1
 ; UNROLL-NO-IC-NEXT:    [[EC:%.*]] = icmp eq i16 [[B_0_NEXT]], 0
@@ -4986,8 +4978,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 -20, i16 -19>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UREM_CONTINUE2]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i32 -20, [[INDEX]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i16> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true)
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
 ; CHECK:       pred.urem.if:
@@ -5006,7 +4997,7 @@ define i32 @PR32419(i32 %a, i16 %b) {
 ; CHECK-NEXT:    br label [[PRED_UREM_CONTINUE2]]
 ; CHECK:       pred.urem.continue2:
 ; CHECK-NEXT:    [[TMP13:%.*]] = phi <2 x i16> [ [[TMP8]], [[PRED_UREM_CONTINUE]] ], [ [[TMP12]], [[PRED_UREM_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i16> zeroinitializer, <2 x i16> [[TMP13]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[TMP13]], <2 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP14:%.*]] = sext <2 x i16> [[PREDPHI]] to <2 x i32>
 ; CHECK-NEXT:    [[TMP15]] = or <2 x i32> [[VEC_PHI]], [[TMP14]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
@@ -5017,12 +5008,10 @@ define i32 @PR32419(i32 %a, i16 %b) {
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[TMP15]])
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ -20, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[A]], [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; CHECK-NEXT:    [[VAR0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR6:%.*]], [[FOR_INC]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ -20, [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[VAR0:%.*]] = phi i32 [ [[A]], [[SCALAR_PH]] ], [ [[VAR6:%.*]], [[FOR_INC]] ]
 ; CHECK-NEXT:    [[VAR1:%.*]] = trunc i32 [[I]] to i16
 ; CHECK-NEXT:    [[VAR2:%.*]] = icmp eq i16 [[VAR1]], 0
 ; CHECK-NEXT:    br i1 [[VAR2]], label [[FOR_INC]], label [[FOR_COND:%.*]]
@@ -5182,10 +5171,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
 ; UNROLL-NO-IC-NEXT:    [[STEP_ADD:%.*]] = add <2 x i16> [[VEC_IND]], splat (i16 2)
 ; UNROLL-NO-IC-NEXT:    [[OFFSET_IDX:%.*]] = add i32 -20, [[INDEX]]
 ; UNROLL-NO-IC-NEXT:    [[TMP1:%.*]] = trunc i32 [[OFFSET_IDX]] to i16
-; UNROLL-NO-IC-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i16> [[VEC_IND]], zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[TMP3:%.*]] = icmp eq <2 x i16> [[STEP_ADD]], zeroinitializer
-; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true)
-; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; UNROLL-NO-IC-NEXT:    [[TMP4:%.*]] = icmp ne <2 x i16> [[VEC_IND]], zeroinitializer
+; UNROLL-NO-IC-NEXT:    [[TMP5:%.*]] = icmp ne <2 x i16> [[STEP_ADD]], zeroinitializer
 ; UNROLL-NO-IC-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0
 ; UNROLL-NO-IC-NEXT:    br i1 [[TMP6]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
 ; UNROLL-NO-IC:       pred.urem.if:
@@ -5222,8 +5209,8 @@ define i32 @PR32419(i32 %a, i16 %b) {
 ; UNROLL-NO-IC-NEXT:    br label [[PRED_UREM_CONTINUE7]]
 ; UNROLL-NO-IC:       pred.urem.continue7:
 ; UNROLL-NO-IC-NEXT:    [[TMP25:%.*]] = phi <2 x i16> [ [[TMP20]], [[PRED_UREM_CONTINUE5]] ], [ [[TMP24]], [[PRED_UREM_IF6]] ]
-; UNROLL-NO-IC-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i16> zeroinitializer, <2 x i16> [[TMP15]]
-; UNROLL-NO-IC-NEXT:    [[PREDPHI8:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> zeroinitializer, <2 x i16> [[TMP25]]
+; UNROLL-NO-IC-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i16> [[TMP15]], <2 x i16> zeroinitializer
+; UNROLL-NO-IC-NEXT:    [[PREDPHI8:%.*]] = select <2 x i1> [[TMP5]], <2 x i16> [[TMP25]], <2 x i16> zeroinitializer
 ; UNROLL-NO-IC-NEXT:    [[TMP26:%.*]] = sext <2 x i16> [[PREDPHI]] to <2 x i32>
 ; UNROLL-NO-IC-NEXT:    [[TMP27:%.*]] = sext <2 x i16> [[PREDPHI8]] to <2 x i32>
 ; UNROLL-NO-IC-NEXT:    [[TMP28]] = or <2 x i32> [[VEC_PHI]], [[TMP26]]
@@ -5237,12 +5224,10 @@ define i32 @PR32419(i32 %a, i16 %b) {
 ; UNROLL-NO-IC-NEXT:    [[TMP31:%.*]] = call i32 @llvm.vector.reduce.or.v2i32(<2 x i32> [[BIN_RDX]])
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_END:%.*]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ -20, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[A]], [[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[FOR_BODY:%.*]]
 ; UNROLL-NO-IC:       for.body:
-; UNROLL-NO-IC-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[VAR0:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[VAR6:%.*]], [[FOR_INC]] ]
+; UNROLL-NO-IC-NEXT:    [[I:%.*]] = phi i32 [ -20, [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; UNROLL-NO-IC-NEXT:    [[VAR0:%.*]] = phi i32 [ [[A]], [[SCALAR_PH]] ], [ [[VAR6:%.*]], [[FOR_INC]] ]
 ; UNROLL-NO-IC-NEXT:    [[VAR1:%.*]] = trunc i32 [[I]] to i16
 ; UNROLL-NO-IC-NEXT:    [[VAR2:%.*]] = icmp eq i16 [[VAR1]], 0
 ; UNROLL-NO-IC-NEXT:    br i1 [[VAR2]], label [[FOR_INC]], label [[FOR_COND:%.*]]
@@ -5833,14 +5818,11 @@ define void @pr52460_first_order_recurrence_truncated_iv(ptr noalias %src, ptr %
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 0, [[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[TRUNC_IV:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[TRUNC_IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_TRUNC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TRUNC_IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[TRUNC_IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RECUR:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_TRUNC:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[LV:%.*]] = load i32, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[LV]], [[RECUR]]
 ; CHECK-NEXT:    [[TRUNC_IV_NEXT]] = add i32 [[TRUNC_IV]], 1
@@ -5955,14 +5937,11 @@ define void @pr52460_first_order_recurrence_truncated_iv(ptr noalias %src, ptr %
 ; UNROLL-NO-IC:       middle.block:
 ; UNROLL-NO-IC-NEXT:    br label [[EXIT:%.*]]
 ; UNROLL-NO-IC:       scalar.ph:
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; UNROLL-NO-IC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 0, [[ENTRY]] ]
-; UNROLL-NO-IC-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; UNROLL-NO-IC-NEXT:    br label [[LOOP:%.*]]
 ; UNROLL-NO-IC:       loop:
-; UNROLL-NO-IC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; UNROLL-NO-IC-NEXT:    [[TRUNC_IV:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[TRUNC_IV_NEXT:%.*]], [[LOOP]] ]
-; UNROLL-NO-IC-NEXT:    [[RECUR:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], [[SCALAR_PH]] ], [ [[IV_TRUNC:%.*]], [[LOOP]] ]
+; UNROLL-NO-IC-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; UNROLL-NO-IC-NEXT:    [[TRUNC_IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[TRUNC_IV_NEXT:%.*]], [[LOOP]] ]
+; UNROLL-NO-IC-NEXT:    [[RECUR:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_TRUNC:%.*]], [[LOOP]] ]
 ; UNROLL-NO-IC-NEXT:    [[LV:%.*]] = load i32, ptr [[SRC]], align 4
 ; UNROLL-NO-IC-NEXT:    [[MUL:%.*]] = mul nsw i32 [[LV]], [[RECUR]]
 ; UNROLL-NO-IC-NEXT:    [[TRUNC_IV_NEXT]] = add i32 [[TRUNC_IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll b/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
index 3330f2bfe6618..acd10a57e0ce0 100644
--- a/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
@@ -20,10 +20,9 @@ define i32 @one_direct_branch(ptr %src) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[SRC_GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]
 ; CHECK-NEXT:    [[LV:%.*]] = load i32, ptr [[SRC_GEP]], align 4
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 25500, [[LV]]
@@ -76,10 +75,9 @@ define i32 @two_direct_branch(ptr %src) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[SRC_GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]
 ; CHECK-NEXT:    [[LV:%.*]] = load i32, ptr [[SRC_GEP]], align 4
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 25500, [[LV]]
@@ -145,10 +143,9 @@ define i32 @cond_branch(i32 %a, ptr %src) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[PREDPHI]], i32 3
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[SRC_GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]
 ; CHECK-NEXT:    [[LV:%.*]] = load i32, ptr [[SRC_GEP]], align 4
 ; CHECK-NEXT:    [[XOR:%.*]] = xor i32 25500, [[LV]]
@@ -210,10 +207,9 @@ define i32 @optimizable_trunc_used_outside() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[VEC_IND]], i32 3
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i32
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT_I_I:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
diff --git a/llvm/test/Transforms/LoopVectorize/interleave-with-i65-induction.ll b/llvm/test/Transforms/LoopVectorize/interleave-with-i65-induction.ll
index 8a4820949af12..3eb16e9a2d78e 100644
--- a/llvm/test/Transforms/LoopVectorize/interleave-with-i65-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleave-with-i65-induction.ll
@@ -5,7 +5,7 @@
 define void @i65_induction_with_negative_step(ptr %dst) {
 ; CHECK-LABEL: define void @i65_induction_with_negative_step(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -34,14 +34,11 @@ define void @i65_induction_with_negative_step(ptr %dst) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i65 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[IV_I65:%.*]] = phi i65 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[IV_I65_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[TRUNC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_I65:%.*]] = phi i65 [ 0, %[[SCALAR_PH]] ], [ [[IV_I65_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[TRUNC:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[TRUNC]] = trunc i65 [[IV_I65]] to i64
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[TRUNC]]
 ; CHECK-NEXT:    store i64 [[FOR]], ptr [[GEP]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-alias.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-alias.ll
deleted file mode 100644
index 7dcae9ea09d3d..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-alias.ll
+++ /dev/null
@@ -1,61 +0,0 @@
-; RUN: opt -S -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true < %s | FileCheck %s
-
-; When merging two stores with interleaved access vectorization, make sure we
-; propagate the alias information from all scalar stores to form the most
-; generic alias info.
-
-target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-target triple = "arm64-apple-ios5.0.0"
-
-%struct.Vec4r = type { double, double, double, double }
-%struct.Vec2r = type { double, double }
-
-define void @foobar(ptr nocapture readonly %p, i32 %i)
-{
-entry:
-  %cp = alloca [20 x %struct.Vec2r], align 8
-  br label %for.body
-
-for.cond.cleanup:                                 ; preds = %for.body
-  call void @g(ptr nonnull %cp) #4
-  ret void
-
-for.body:                                         ; preds = %for.body, %entry
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %x = getelementptr inbounds %struct.Vec4r, ptr %p, i64 %indvars.iv, i32 0
-  %0 = load double, ptr %x, align 8, !tbaa !3
-  %mul = fmul double %0, 2.000000e+00
-  %x4 = getelementptr inbounds [20 x %struct.Vec2r], ptr %cp, i64 0, i64 %indvars.iv, i32 0
-
-; The new store should alias any double rather than one of the fields of Vec2r.
-; CHECK: store <4 x double> {{.*}} !tbaa ![[STORE_TBAA:[0-9]+]]
-; CHECK-DAG: ![[DOUBLE_TBAA:[0-9]+]] = !{!"double", !{{[0-9+]}}, i64 0}
-; CHECK-DAG: ![[STORE_TBAA]] = !{![[DOUBLE_TBAA]], ![[DOUBLE_TBAA]], i64 0}
-  store double %mul, ptr %x4, align 8, !tbaa !8
-  %y = getelementptr inbounds %struct.Vec4r, ptr %p, i64 %indvars.iv, i32 1
-  %1 = load double, ptr %y, align 8, !tbaa !10
-  %mul7 = fmul double %1, 3.000000e+00
-  %y10 = getelementptr inbounds [20 x %struct.Vec2r], ptr %cp, i64 0, i64 %indvars.iv, i32 1
-  store double %mul7, ptr %y10, align 8, !tbaa !11
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv.next, 4
-  br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}
-
-declare void @g(ptr)
-
-!llvm.module.flags = !{!0, !1}
-!llvm.ident = !{!2}
-
-!0 = !{i32 1, !"wchar_size", i32 4}
-!1 = !{i32 7, !"PIC Level", i32 2}
-!2 = !{!"clang version 6.0.0 (trunk 319007) (llvm/trunk 319324)"}
-!3 = !{!4, !5, i64 0}
-!4 = !{!"Vec4r", !5, i64 0, !5, i64 8, !5, i64 16, !5, i64 24}
-!5 = !{!"double", !6, i64 0}
-!6 = !{!"omnipotent char", !7, i64 0}
-!7 = !{!"Simple C/C++ TBAA"}
-!8 = !{!9, !5, i64 0}
-!9 = !{!"Vec2r", !5, i64 0, !5, i64 8}
-!10 = !{!4, !5, i64 8}
-!11 = !{!9, !5, i64 8}
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll
index 651210df823dd..dd6b829fcb5c9 100644
--- a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll
@@ -6,7 +6,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128-Fn32"
 define void @gep_for_first_member_does_not_dominate_insert_point(ptr %str, ptr noalias %dst) {
 ; CHECK-LABEL: define void @gep_for_first_member_does_not_dominate_insert_point(
 ; CHECK-SAME: ptr [[STR:%.*]], ptr noalias [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -28,12 +28,10 @@ define void @gep_for_first_member_does_not_dominate_insert_point(ptr %str, ptr n
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV2:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[OR_1:%.*]] = or disjoint i64 [[IV2]], 1
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i8, ptr [[STR]], i64 [[OR_1]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i8, ptr [[GEP1]], align 1
@@ -88,7 +86,7 @@ define void @test_ig_insert_pos_at_end_of_vpbb(ptr noalias %dst, ptr noalias %sr
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr nusw { i16, i16, i16, i16 }, ptr [[SRC]], i64 [[TMP3]], i32 2
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[TMP4]], i32 -4
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr nusw i8, ptr [[TMP4]], i32 -4
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i16>, ptr [[TMP5]], align 2
 ; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
 ; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i16> [[WIDE_VEC]], <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-gep-nowrap-flags.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-gep-nowrap-flags.ll
new file mode 100644
index 0000000000000..a6ba29ed1ca0e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-gep-nowrap-flags.ll
@@ -0,0 +1,335 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 5
+; RUN: opt -S -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true %s | FileCheck %s
+
+%struct.i32.pair = type { i32, i32 }
+
+define void @nusw_preservation(ptr noalias %A, ptr %B) {
+; CHECK-LABEL: define void @nusw_preservation(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1023, i32 1022, i32 1021, i32 1020>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr nusw [[STRUCT_I32_PAIR:%.*]], ptr [[A]], i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw i32, ptr [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr nusw i32, ptr [[TMP1]], i64 -6
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[REVERSE2:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[REVERSE]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <4 x i32> [[REVERSE2]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr nusw [[STRUCT_I32_PAIR]], ptr [[B]], i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr nusw i32, ptr [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr nusw i32, ptr [[TMP6]], i64 -6
+; CHECK-NEXT:    [[REVERSE3:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[REVERSE4:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1023, %entry ], [ %iv.next, %loop ]
+  %x = getelementptr nusw %struct.i32.pair, ptr %A, i64 %iv, i32 0
+  %load.x = load i32, ptr %x, align 4
+  %trunc = trunc i64 %iv to i32
+  %add = add nsw i32 %load.x, %trunc
+  %y = getelementptr nusw %struct.i32.pair, ptr %A, i64 %iv, i32 1
+  %load.y = load i32, ptr %y, align 4
+  %sub = sub nsw i32 %load.y, %trunc
+  %gep.B.iv.0 = getelementptr nusw %struct.i32.pair, ptr %B, i64 %iv, i32 0
+  store i32 %add, ptr %gep.B.iv.0, align 4
+  %gep.B.iv.1 = getelementptr nusw %struct.i32.pair, ptr %B, i64 %iv, i32 1
+  store i32 %sub, ptr %gep.B.iv.1, align 4
+  %iv.next = add nsw i64 %iv, -1
+  %exit.cond = icmp sgt i64 %iv, 0
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @inbounds_preservation(ptr noalias %A, ptr %B) {
+; CHECK-LABEL: define void @inbounds_preservation(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1023, i32 1022, i32 1021, i32 1020>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds [[STRUCT_I32_PAIR:%.*]], ptr [[A]], i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 -6
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[REVERSE2:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[REVERSE]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <4 x i32> [[REVERSE2]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_I32_PAIR]], ptr [[B]], i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 -6
+; CHECK-NEXT:    [[REVERSE3:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[REVERSE4:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1023, %entry ], [ %iv.next, %loop ]
+  %x = getelementptr inbounds %struct.i32.pair, ptr %A, i64 %iv, i32 0
+  %load.x = load i32, ptr %x, align 4
+  %trunc = trunc i64 %iv to i32
+  %add = add nsw i32 %load.x, %trunc
+  %y = getelementptr inbounds %struct.i32.pair, ptr %A, i64 %iv, i32 1
+  %load.y = load i32, ptr %y, align 4
+  %sub = sub nsw i32 %load.y, %trunc
+  %gep.B.iv.0 = getelementptr inbounds %struct.i32.pair, ptr %B, i64 %iv, i32 0
+  store i32 %add, ptr %gep.B.iv.0, align 4
+  %gep.B.iv.1 = getelementptr inbounds %struct.i32.pair, ptr %B, i64 %iv, i32 1
+  store i32 %sub, ptr %gep.B.iv.1, align 4
+  %iv.next = add nsw i64 %iv, -1
+  %exit.cond = icmp sgt i64 %iv, 0
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @nuw_drop(ptr noalias %A, ptr %B) {
+; CHECK-LABEL: define void @nuw_drop(
+; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 1023, i32 1022, i32 1021, i32 1020>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = sub i64 1023, [[INDEX]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr nuw [[STRUCT_I32_PAIR:%.*]], ptr [[A]], i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i32, ptr [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[TMP1]], i64 -6
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[REVERSE:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[REVERSE2:%.*]] = shufflevector <4 x i32> [[STRIDED_VEC1]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[REVERSE]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <4 x i32> [[REVERSE2]], [[VEC_IND]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr nuw [[STRUCT_I32_PAIR]], ptr [[B]], i64 [[OFFSET_IDX]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i32, ptr [[TMP6]], i64 -6
+; CHECK-NEXT:    [[REVERSE3:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[REVERSE4:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i32> [[REVERSE3]], <4 x i32> [[REVERSE4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT:    store <8 x i32> [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 -4)
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 1023, %entry ], [ %iv.next, %loop ]
+  %x = getelementptr nuw %struct.i32.pair, ptr %A, i64 %iv, i32 0
+  %load.x = load i32, ptr %x, align 4
+  %trunc = trunc i64 %iv to i32
+  %add = add nsw i32 %load.x, %trunc
+  %y = getelementptr nuw %struct.i32.pair, ptr %A, i64 %iv, i32 1
+  %load.y = load i32, ptr %y, align 4
+  %sub = sub nsw i32 %load.y, %trunc
+  %gep.B.iv.0 = getelementptr nuw %struct.i32.pair, ptr %B, i64 %iv, i32 0
+  store i32 %add, ptr %gep.B.iv.0, align 4
+  %gep.B.iv.1 = getelementptr nuw %struct.i32.pair, ptr %B, i64 %iv, i32 1
+  store i32 %sub, ptr %gep.B.iv.1, align 4
+  %iv.next = add nsw i64 %iv, -1
+  %exit.cond = icmp sgt i64 %iv, 0
+  br i1 %exit.cond, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+define void @nusw_preservation_2(ptr %src, ptr noalias %dst) {
+; CHECK-LABEL: define void @nusw_preservation_2(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = or disjoint i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw i8, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr nusw i8, ptr [[TMP1]], i32 -1
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i8> [[STRIDED_VEC1]], [[STRIDED_VEC]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr nusw i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i8> [[TMP3]], ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:                                         ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2.next, %loop ]
+  %or.1 = or disjoint i64 %iv2, 1
+  %gep.src.or.1 = getelementptr nusw i8, ptr %src, i64 %or.1
+  %load.src.1 = load i8, ptr %gep.src.or.1, align 1
+  %gep.src.iv2 = getelementptr nusw i8, ptr %src, i64 %iv2
+  %load.src.2 = load i8, ptr %gep.src.iv2, align 1
+  %add = add i8 %load.src.1, %load.src.2
+  %gep.dst.iv = getelementptr nusw i8, ptr %dst, i64 %iv
+  store i8 %add, ptr %gep.dst.iv, align 1
+  %iv2.next = add i64 %iv2, 2
+  %iv.next = add i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, 100
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @inbounds_preservation_2(ptr %src, ptr noalias %dst) {
+; CHECK-LABEL: define void @inbounds_preservation_2(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = or disjoint i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 -1
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i8> [[STRIDED_VEC1]], [[STRIDED_VEC]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i8> [[TMP3]], ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:                                         ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2.next, %loop ]
+  %or.1 = or disjoint i64 %iv2, 1
+  %gep.src.or.1 = getelementptr inbounds i8, ptr %src, i64 %or.1
+  %load.src.1 = load i8, ptr %gep.src.or.1, align 1
+  %gep.src.iv2 = getelementptr inbounds i8, ptr %src, i64 %iv2
+  %load.src.2 = load i8, ptr %gep.src.iv2, align 1
+  %add = add i8 %load.src.1, %load.src.2
+  %gep.dst.iv = getelementptr inbounds i8, ptr %dst, i64 %iv
+  store i8 %add, ptr %gep.dst.iv, align 1
+  %iv2.next = add i64 %iv2, 2
+  %iv.next = add i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, 100
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+define void @nuw_drop_2(ptr %src, ptr noalias %dst) {
+; CHECK-LABEL: define void @nuw_drop_2(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = or disjoint i64 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nuw i8, ptr [[SRC]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[TMP1]], i32 -1
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i8> [[WIDE_VEC]], <8 x i8> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i8> [[STRIDED_VEC1]], [[STRIDED_VEC]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr nuw i8, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i8> [[TMP3]], ptr [[TMP4]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[EXIT:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
+;
+entry:
+  br label %loop
+
+loop:                                         ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv2 = phi i64 [ 0, %entry ], [ %iv2.next, %loop ]
+  %or.1 = or disjoint i64 %iv2, 1
+  %gep.src.or.1 = getelementptr nuw i8, ptr %src, i64 %or.1
+  %load.src.1 = load i8, ptr %gep.src.or.1, align 1
+  %gep.src.iv2 = getelementptr nuw i8, ptr %src, i64 %iv2
+  %load.src.2 = load i8, ptr %gep.src.iv2, align 1
+  %add = add i8 %load.src.1, %load.src.2
+  %gep.dst.iv = getelementptr nuw i8, ptr %dst, i64 %iv
+  store i8 %add, ptr %gep.dst.iv, align 1
+  %iv2.next = add i64 %iv2, 2
+  %iv.next = add i64 %iv, 1
+  %exit.cond = icmp eq i64 %iv.next, 100
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
new file mode 100644
index 0000000000000..0eb251704f453
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/interleaved-accesses-metadata.ll
@@ -0,0 +1,294 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=1 -enable-interleaved-mem-accesses=true < %s | FileCheck %s
+
+; When merging two stores with interleaved access vectorization, make sure we
+; propagate the alias information from all scalar stores to form the most
+; generic alias info.
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+%struct.Vec4r = type { double, double, double, double }
+%struct.Vec2r = type { double, double }
+
+; The new store for the interleave group should alias any double rather than one of the fields of Vec2r.
+define void @merge_tbaa_interleave_group(ptr nocapture readonly %p, ptr noalias %cp, i32 %i)
+; CHECK-LABEL: define void @merge_tbaa_interleave_group(
+; CHECK-SAME: ptr readonly captures(none) [[P:%.*]], ptr noalias [[CP:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[STRUCT_VEC4R:%.*]], ptr [[P]], i64 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_VEC4R]], ptr [[P]], i64 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = load double, ptr [[TMP2]], align 8, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load double, ptr [[TMP3]], align 8, !tbaa [[TBAA0]]
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x double> poison, double [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x double> [[TMP6]], double [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], splat (double 2.000000e+00)
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [20 x %struct.Vec2r], ptr [[CP]], i64 0, i64 [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_VEC4R]], ptr [[P]], i64 [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_VEC4R]], ptr [[P]], i64 [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP12:%.*]] = load double, ptr [[TMP10]], align 8, !tbaa [[TBAA5:![0-9]+]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load double, ptr [[TMP11]], align 8, !tbaa [[TBAA5]]
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x double> [[TMP14]], double [[TMP13]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul <2 x double> [[TMP15]], splat (double 3.000000e+00)
+; CHECK-NEXT:    [[TMP17:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> [[TMP16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x double> [[TMP17]], <4 x double> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x double> [[INTERLEAVED_VEC]], ptr [[TMP9]], align 8, !tbaa [[TBAA6:![0-9]+]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 4
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[X:%.*]] = getelementptr inbounds [[STRUCT_VEC4R]], ptr [[P]], i64 [[IV]], i32 0
+; CHECK-NEXT:    [[TMP19:%.*]] = load double, ptr [[X]], align 8, !tbaa [[TBAA0]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP19]], 2.000000e+00
+; CHECK-NEXT:    [[X4:%.*]] = getelementptr inbounds [20 x %struct.Vec2r], ptr [[CP]], i64 0, i64 [[IV]], i32 0
+; CHECK-NEXT:    store double [[MUL]], ptr [[X4]], align 8, !tbaa [[TBAA10:![0-9]+]]
+; CHECK-NEXT:    [[Y:%.*]] = getelementptr inbounds [[STRUCT_VEC4R]], ptr [[P]], i64 [[IV]], i32 1
+; CHECK-NEXT:    [[TMP20:%.*]] = load double, ptr [[Y]], align 8, !tbaa [[TBAA5]]
+; CHECK-NEXT:    [[MUL7:%.*]] = fmul double [[TMP20]], 3.000000e+00
+; CHECK-NEXT:    [[Y10:%.*]] = getelementptr inbounds [20 x %struct.Vec2r], ptr [[CP]], i64 0, i64 [[IV]], i32 1
+; CHECK-NEXT:    store double [[MUL7]], ptr [[Y10]], align 8, !tbaa [[TBAA12:![0-9]+]]
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 4
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+{
+entry:
+  br label %loop
+
+loop:                                         ; preds = %loop, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %x = getelementptr inbounds %struct.Vec4r, ptr %p, i64 %iv, i32 0
+  %0 = load double, ptr %x, align 8, !tbaa !3
+  %mul = fmul double %0, 2.000000e+00
+  %x4 = getelementptr inbounds [20 x %struct.Vec2r], ptr %cp, i64 0, i64 %iv, i32 0
+  store double %mul, ptr %x4, align 8, !tbaa !8
+  %y = getelementptr inbounds %struct.Vec4r, ptr %p, i64 %iv, i32 1
+  %1 = load double, ptr %y, align 8, !tbaa !10
+  %mul7 = fmul double %1, 3.000000e+00
+  %y10 = getelementptr inbounds [20 x %struct.Vec2r], ptr %cp, i64 0, i64 %iv, i32 1
+  store double %mul7, ptr %y10, align 8, !tbaa !11
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, 4
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+; Make sure no !tbaa is added to neither the interleave group load nor the store.
+define void @ir_tbaa_different(ptr %base, ptr %end, ptr %src) {
+; CHECK-LABEL: define void @ir_tbaa_different(
+; CHECK-SAME: ptr [[BASE:%.*]], ptr [[END:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[BASE2:%.*]] = ptrtoint ptr [[BASE]] to i64
+; CHECK-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; CHECK-NEXT:    [[BASE3:%.*]] = ptrtoint ptr [[BASE]] to i64
+; CHECK-NEXT:    [[END2:%.*]] = ptrtoint ptr [[END]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[END1]], -8
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[BASE2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[END2]], -8
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[TMP10]], [[BASE3]]
+; CHECK-NEXT:    [[TMP13:%.*]] = lshr i64 [[TMP12]], 3
+; CHECK-NEXT:    [[TMP14:%.*]] = shl nuw i64 [[TMP13]], 3
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], 8
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP15]]
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 4
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ult ptr [[BASE]], [[SCEVGEP3]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
+; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
+; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[SRC]], align 4, !alias.scope [[META14:![0-9]+]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP11]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <4 x float>, ptr [[NEXT_GEP]], align 4, !alias.scope [[META17:![0-9]+]], !noalias [[META14]]
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x float> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x float> [[STRIDED_VEC3]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x float> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 4, !alias.scope [[META17]], !noalias [[META14]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ], [ [[BASE]], %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[L_INVAR:%.*]] = load float, ptr [[SRC]], align 4
+; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 8
+; CHECK-NEXT:    [[L_1:%.*]] = load float, ptr [[PTR_IV]], align 4
+; CHECK-NEXT:    [[MUL_1:%.*]] = fmul float [[L_1]], [[L_INVAR]]
+; CHECK-NEXT:    store float [[MUL_1]], ptr [[PTR_IV]], align 4, !tbaa [[TBAA10]]
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 4
+; CHECK-NEXT:    [[L_2:%.*]] = load float, ptr [[GEP_1]], align 4, !tbaa [[TBAA12]]
+; CHECK-NEXT:    [[MUL_2:%.*]] = fmul float [[L_2]], [[L_INVAR]]
+; CHECK-NEXT:    store float [[MUL_2]], ptr [[GEP_1]], align 4
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %ptr.iv = phi ptr [ %base, %entry ], [ %ptr.iv.next, %loop ]
+  %l.invar = load float, ptr %src
+  %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 8
+  %l.0 = load float, ptr %ptr.iv, align 4
+  %mul.0 = fmul float %l.0, %l.invar
+  store float %mul.0, ptr %ptr.iv, align 4, !tbaa !8
+  %gep.1 = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 4
+  %l.1 = load float, ptr %gep.1, align 4, !tbaa !11
+  %mul.1 = fmul float %l.1, %l.invar
+  store float %mul.1, ptr %gep.1, align 4
+  %ec = icmp eq ptr %ptr.iv.next, %end
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+
+define void @noalias_metadata_from_versioning(ptr %base, ptr %end, ptr %src) {
+; CHECK-LABEL: define void @noalias_metadata_from_versioning(
+; CHECK-SAME: ptr [[BASE:%.*]], ptr [[END:%.*]], ptr [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[BASE2:%.*]] = ptrtoint ptr [[BASE]] to i64
+; CHECK-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[END1]], -8
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[BASE2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 2
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 2
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[TMP4]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[BASE]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <4 x float>, ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[STRIDED_VEC3:%.*]] = shufflevector <4 x float> [[WIDE_VEC]], <4 x float> poison, <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x float> [[STRIDED_VEC]], splat (float 1.000000e+01)
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x float> [[STRIDED_VEC3]], splat (float 1.000000e+01)
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[TMP6]], <2 x float> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 0, i32 2, i32 1, i32 3>
+; CHECK-NEXT:    store <4 x float> [[INTERLEAVED_VEC]], ptr [[NEXT_GEP]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[BASE]], %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 8
+; CHECK-NEXT:    [[L_0:%.*]] = load float, ptr [[PTR_IV]], align 4
+; CHECK-NEXT:    [[MUL_0:%.*]] = fmul float [[L_0]], 1.000000e+01
+; CHECK-NEXT:    store float [[MUL_0]], ptr [[PTR_IV]], align 4
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 4
+; CHECK-NEXT:    [[L_1:%.*]] = load float, ptr [[GEP_1]], align 4
+; CHECK-NEXT:    [[MUL_1:%.*]] = fmul float [[L_1]], 1.000000e+01
+; CHECK-NEXT:    store float [[MUL_1]], ptr [[GEP_1]], align 4
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %ptr.iv = phi ptr [ %base, %entry ], [ %ptr.iv.next, %loop ]
+  %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 8
+  %l.0 = load float, ptr %ptr.iv, align 4
+  %mul.0 = fmul float %l.0, 10.0
+  store float %mul.0, ptr %ptr.iv, align 4
+  %gep.1 = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 4
+  %l.1 = load float, ptr %gep.1, align 4
+  %mul.1 = fmul float %l.1, 10.0
+  store float %mul.1, ptr %gep.1, align 4
+  %ec = icmp eq ptr %ptr.iv.next, %end
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret void
+}
+
+!3 = !{!4, !5, i64 0}
+!4 = !{!"Vec4r", !5, i64 0, !5, i64 8, !5, i64 16, !5, i64 24}
+!5 = !{!"double", !6, i64 0}
+!6 = !{!"omnipotent char", !7, i64 0}
+!7 = !{!"Simple C/C++ TBAA"}
+!8 = !{!9, !5, i64 0}
+!9 = !{!"Vec2r", !5, i64 0, !5, i64 8}
+!10 = !{!4, !5, i64 8}
+!11 = !{!9, !5, i64 8}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META2:![0-9]+]], i64 0}
+; CHECK: [[META1]] = !{!"Vec4r", [[META2]], i64 0, [[META2]], i64 8, [[META2]], i64 16, [[META2]], i64 24}
+; CHECK: [[META2]] = !{!"double", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; CHECK: [[META4]] = !{!"Simple C/C++ TBAA"}
+; CHECK: [[TBAA5]] = !{[[META1]], [[META2]], i64 8}
+; CHECK: [[TBAA6]] = !{[[META2]], [[META2]], i64 0}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META8:![0-9]+]], [[META9:![0-9]+]]}
+; CHECK: [[META8]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META9]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[TBAA10]] = !{[[META11:![0-9]+]], [[META2]], i64 0}
+; CHECK: [[META11]] = !{!"Vec2r", [[META2]], i64 0, [[META2]], i64 8}
+; CHECK: [[TBAA12]] = !{[[META11]], [[META2]], i64 8}
+; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META9]], [[META8]]}
+; CHECK: [[META14]] = !{[[META15:![0-9]+]]}
+; CHECK: [[META15]] = distinct !{[[META15]], [[META16:![0-9]+]]}
+; CHECK: [[META16]] = distinct !{[[META16]], !"LVerDomain"}
+; CHECK: [[META17]] = !{[[META18:![0-9]+]]}
+; CHECK: [[META18]] = distinct !{[[META18]], [[META16]]}
+; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META8]], [[META9]]}
+; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META8]]}
+; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META8]], [[META9]]}
+; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META9]], [[META8]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/intrinsic.ll b/llvm/test/Transforms/LoopVectorize/intrinsic.ll
index 9c910d70807a1..10d83a456d0e2 100644
--- a/llvm/test/Transforms/LoopVectorize/intrinsic.ll
+++ b/llvm/test/Transforms/LoopVectorize/intrinsic.ll
@@ -324,6 +324,56 @@ for.end:                                          ; preds = %for.body, %entry
 
 declare double @llvm.exp2.f64(double)
 
+define void @ldexp_f32i32(i32 %n, ptr %y, ptr %x, i32 %exp) {
+; CHECK-LABEL: @ldexp_f32i32(
+; CHECK: llvm.ldexp.v4f32.v4i32
+; CHECK: ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, ptr %y, i32 %iv
+  %0 = load float, ptr %arrayidx, align 4
+  %call = tail call float @llvm.ldexp.f32.i32(float %0, i32 %exp)
+  %arrayidx2 = getelementptr inbounds float, ptr %x, i32 %iv
+  store float %call, ptr %arrayidx2, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare float @llvm.ldexp.f32.i32(float, i32)
+
+define void @ldexp_f64i32(i32 %n, ptr %y, ptr %x, i32 %exp) {
+; CHECK-LABEL: @ldexp_f64i32(
+; CHECK: llvm.ldexp.v4f64.v4i32
+; CHECK: ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, ptr %y, i32 %iv
+  %0 = load double, ptr %arrayidx, align 8
+  %call = tail call double @llvm.ldexp.f64.i32(double %0, i32 %exp)
+  %arrayidx2 = getelementptr inbounds double, ptr %x, i32 %iv
+  store double %call, ptr %arrayidx2, align 8
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare double @llvm.ldexp.f64.i32(double, i32)
+
 define void @log_f32(i32 %n, ptr %y, ptr %x) {
 ; CHECK-LABEL: @log_f32(
 ; CHECK: llvm.log.v4f32
@@ -976,6 +1026,157 @@ for.end:                                          ; preds = %for.body, %entry
 
 declare double @llvm.roundeven.f64(double)
 
+
+define void @lround_i32f32(i32 %n, ptr %y, ptr %x) {
+; CHECK-LABEL: @lround_i32f32(
+; CHECK: llvm.lround.v4i32.v4f32
+; CHECK: ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, ptr %y, i32 %iv
+  %0 = load float, ptr %arrayidx, align 4
+  %call = tail call i32 @llvm.lround.i32.f32(float %0)
+  %arrayidx2 = getelementptr inbounds i32, ptr %x, i32 %iv
+  store i32 %call, ptr %arrayidx2, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i32 @llvm.lround.i32.f32(float)
+
+define void @lround_i32f64(i32 %n, ptr %y, ptr %x) {
+; CHECK-LABEL: @lround_i32f64(
+; CHECK: llvm.lround.v4i32.v4f64
+; CHECK: ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, ptr %y, i32 %iv
+  %0 = load double, ptr %arrayidx, align 8
+  %call = tail call i32 @llvm.lround.i32.f64(double %0)
+  %arrayidx2 = getelementptr inbounds i32, ptr %x, i32 %iv
+  store i32 %call, ptr %arrayidx2, align 8
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i32 @llvm.lround.i32.f64(double)
+
+define void @lround_i64f32(i32 %n, ptr %y, ptr %x) {
+; CHECK-LABEL: @lround_i64f32(
+; CHECK: llvm.lround.v4i64.v4f32
+; CHECK: ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, ptr %y, i32 %iv
+  %0 = load float, ptr %arrayidx, align 4
+  %call = tail call i64 @llvm.lround.i64.f32(float %0)
+  %arrayidx2 = getelementptr inbounds i64, ptr %x, i32 %iv
+  store i64 %call, ptr %arrayidx2, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i64 @llvm.lround.i64.f32(float)
+
+define void @lround_i64f64(i32 %n, ptr %y, ptr %x) {
+; CHECK-LABEL: @lround_i64f64(
+; CHECK: llvm.lround.v4i64.v4f64
+; CHECK: ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, ptr %y, i32 %iv
+  %0 = load double, ptr %arrayidx, align 8
+  %call = tail call i64 @llvm.lround.i64.f64(double %0)
+  %arrayidx2 = getelementptr inbounds i64, ptr %x, i32 %iv
+  store i64 %call, ptr %arrayidx2, align 8
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i64 @llvm.lround.i64.f64(double)
+
+define void @llround_i64f32(i32 %n, ptr %y, ptr %x) {
+; CHECK-LABEL: @llround_i64f32(
+; CHECK: llvm.llround.v4i64.v4f32
+; CHECK: ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds float, ptr %y, i32 %iv
+  %0 = load float, ptr %arrayidx, align 4
+  %call = tail call i64 @llvm.llround.i64.f32(float %0)
+  %arrayidx2 = getelementptr inbounds i64, ptr %x, i32 %iv
+  store i64 %call, ptr %arrayidx2, align 4
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i64 @llvm.llround.i64.f32(float)
+
+define void @llround_i64f64(i32 %n, ptr %y, ptr %x) {
+; CHECK-LABEL: @llround_i64f64(
+; CHECK: llvm.llround.v4i64.v4f64
+; CHECK: ret void
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds double, ptr %y, i32 %iv
+  %0 = load double, ptr %arrayidx, align 8
+  %call = tail call i64 @llvm.llround.i64.f64(double %0)
+  %arrayidx2 = getelementptr inbounds i64, ptr %x, i32 %iv
+  store i64 %call, ptr %arrayidx2, align 8
+  %iv.next = add i32 %iv, 1
+  %exitcond = icmp eq i32 %iv.next, %n
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  ret void
+}
+
+declare i64 @llvm.llround.i64.f64(double)
+
 define void @fma_f32(i32 %n, ptr %y, ptr %x, ptr %z, ptr %w) {
 ; CHECK-LABEL: @fma_f32(
 ; CHECK: llvm.fma.v4f32
diff --git a/llvm/test/Transforms/LoopVectorize/is_fpclass.ll b/llvm/test/Transforms/LoopVectorize/is_fpclass.ll
index 6eeeace80aa8e..42f6c0532caa4 100644
--- a/llvm/test/Transforms/LoopVectorize/is_fpclass.ll
+++ b/llvm/test/Transforms/LoopVectorize/is_fpclass.ll
@@ -22,10 +22,9 @@ define void @d() {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I7:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[I7:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[I3:%.*]] = load float, ptr null, align 4
 ; CHECK-NEXT:    [[I4:%.*]] = getelementptr float, ptr @d, i64 [[I]]
 ; CHECK-NEXT:    [[I5:%.*]] = tail call i1 @llvm.is.fpclass.f32(float [[I3]], i32 0)
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
index a0068f0f6cab3..615f50124b41d 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
@@ -6,7 +6,7 @@
 define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
 ; IC1VF4-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
 ; IC1VF4-SAME: ptr [[A:%.*]]) {
-; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:  [[ENTRY:.*:]]
 ; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC1VF4:       [[VECTOR_PH]]:
 ; IC1VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -32,12 +32,10 @@ define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
 ; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP6]], i64 331
 ; IC1VF4-NEXT:    br label %[[EXIT:.*]]
 ; IC1VF4:       [[SCALAR_PH]]:
-; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 19999, %[[ENTRY]] ]
-; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ]
 ; IC1VF4-NEXT:    br label %[[LOOP:.*]]
 ; IC1VF4:       [[LOOP]]:
-; IC1VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC1VF4-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i64 [ 19999, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC1VF4-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; IC1VF4-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
 ; IC1VF4-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
@@ -51,7 +49,7 @@ define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
 ;
 ; IC4VF4-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
 ; IC4VF4-SAME: ptr [[A:%.*]]) {
-; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:  [[ENTRY:.*:]]
 ; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC4VF4:       [[VECTOR_PH]]:
 ; IC4VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -104,12 +102,10 @@ define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
 ; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP18]], i64 331
 ; IC4VF4-NEXT:    br label %[[EXIT:.*]]
 ; IC4VF4:       [[SCALAR_PH]]:
-; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 19999, %[[ENTRY]] ]
-; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ]
 ; IC4VF4-NEXT:    br label %[[LOOP:.*]]
 ; IC4VF4:       [[LOOP]]:
-; IC4VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC4VF4-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i64 [ 19999, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC4VF4-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; IC4VF4-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
 ; IC4VF4-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
@@ -123,7 +119,7 @@ define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
 ;
 ; IC4VF1-LABEL: define i64 @select_decreasing_induction_icmp_const_start(
 ; IC4VF1-SAME: ptr [[A:%.*]]) {
-; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:  [[ENTRY:.*:]]
 ; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC4VF1:       [[VECTOR_PH]]:
 ; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -164,12 +160,10 @@ define i64 @select_decreasing_induction_icmp_const_start(ptr %a) {
 ; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[RDX_MINMAX5]], i64 331
 ; IC4VF1-NEXT:    br label %[[EXIT:.*]]
 ; IC4VF1:       [[SCALAR_PH]]:
-; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 19999, %[[ENTRY]] ]
-; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ]
 ; IC4VF1-NEXT:    br label %[[LOOP:.*]]
 ; IC4VF1:       [[LOOP]]:
-; IC4VF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC4VF1-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i64 [ 19999, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC4VF1-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; IC4VF1-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
 ; IC4VF1-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
@@ -204,7 +198,7 @@ exit:                                             ; preds = %loop
 define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
 ; IC1VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
 ; IC1VF4-SAME: i16 noundef [[VAL:%.*]]) {
-; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:  [[ENTRY:.*:]]
 ; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC1VF4:       [[VECTOR_PH]]:
 ; IC1VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[VAL]], i64 0
@@ -234,12 +228,10 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
 ; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP7]], i16 0
 ; IC1VF4-NEXT:    br label %[[EXIT:.*]]
 ; IC1VF4:       [[SCALAR_PH]]:
-; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
-; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; IC1VF4-NEXT:    br label %[[LOOP:.*]]
 ; IC1VF4:       [[LOOP]]:
-; IC1VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC1VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC1VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
 ; IC1VF4-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
 ; IC1VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
@@ -253,7 +245,7 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
 ;
 ; IC4VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
 ; IC4VF4-SAME: i16 noundef [[VAL:%.*]]) {
-; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:  [[ENTRY:.*:]]
 ; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC4VF4:       [[VECTOR_PH]]:
 ; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[VAL]], i64 0
@@ -469,12 +461,10 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
 ; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
 ; IC4VF4-NEXT:    br label %[[EXIT:.*]]
 ; IC4VF4:       [[SCALAR_PH]]:
-; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
-; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; IC4VF4-NEXT:    br label %[[LOOP:.*]]
 ; IC4VF4:       [[LOOP]]:
-; IC4VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC4VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC4VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
 ; IC4VF4-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
 ; IC4VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
@@ -488,7 +478,7 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
 ;
 ; IC4VF1-LABEL: define i16 @select_decreasing_induction_icmp_table_i16(
 ; IC4VF1-SAME: i16 noundef [[VAL:%.*]]) {
-; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:  [[ENTRY:.*:]]
 ; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC4VF1:       [[VECTOR_PH]]:
 ; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -534,12 +524,10 @@ define i16 @select_decreasing_induction_icmp_table_i16(i16 noundef %val) {
 ; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[RDX_MINMAX5]], i16 0
 ; IC4VF1-NEXT:    br label %[[EXIT:.*]]
 ; IC4VF1:       [[SCALAR_PH]]:
-; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
-; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; IC4VF1-NEXT:    br label %[[LOOP:.*]]
 ; IC4VF1:       [[LOOP]]:
-; IC4VF1-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC4VF1-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC4VF1-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
 ; IC4VF1-NEXT:    [[LD_TABLE:%.*]] = load i16, ptr [[GEP_TABLE_IV]], align 1
 ; IC4VF1-NEXT:    [[CMP_TABLE_VAL:%.*]] = icmp ugt i16 [[LD_TABLE]], [[VAL]]
@@ -575,7 +563,7 @@ exit:                                             ; preds = %loop
 define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
 ; IC1VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
 ; IC1VF4-SAME: half noundef [[VAL:%.*]]) {
-; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:  [[ENTRY:.*:]]
 ; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC1VF4:       [[VECTOR_PH]]:
 ; IC1VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x half> poison, half [[VAL]], i64 0
@@ -605,12 +593,10 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
 ; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP7]], i16 0
 ; IC1VF4-NEXT:    br label %[[EXIT:.*]]
 ; IC1VF4:       [[SCALAR_PH]]:
-; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
-; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; IC1VF4-NEXT:    br label %[[LOOP:.*]]
 ; IC1VF4:       [[LOOP]]:
-; IC1VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC1VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC1VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
 ; IC1VF4-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
 ; IC1VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
@@ -624,7 +610,7 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
 ;
 ; IC4VF4-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
 ; IC4VF4-SAME: half noundef [[VAL:%.*]]) {
-; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:  [[ENTRY:.*:]]
 ; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC4VF4:       [[VECTOR_PH]]:
 ; IC4VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x half> poison, half [[VAL]], i64 0
@@ -840,12 +826,10 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
 ; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[TMP116]], i16 0
 ; IC4VF4-NEXT:    br label %[[EXIT:.*]]
 ; IC4VF4:       [[SCALAR_PH]]:
-; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
-; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; IC4VF4-NEXT:    br label %[[LOOP:.*]]
 ; IC4VF4:       [[LOOP]]:
-; IC4VF4-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC4VF4-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC4VF4-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
 ; IC4VF4-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
 ; IC4VF4-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
@@ -859,7 +843,7 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
 ;
 ; IC4VF1-LABEL: define i16 @select_decreasing_induction_icmp_table_half(
 ; IC4VF1-SAME: half noundef [[VAL:%.*]]) {
-; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:  [[ENTRY:.*:]]
 ; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC4VF1:       [[VECTOR_PH]]:
 ; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -905,12 +889,10 @@ define i16 @select_decreasing_induction_icmp_table_half(half noundef %val) {
 ; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i16 [[RDX_MINMAX5]], i16 0
 ; IC4VF1-NEXT:    br label %[[EXIT:.*]]
 ; IC4VF1:       [[SCALAR_PH]]:
-; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 12, %[[ENTRY]] ]
-; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; IC4VF1-NEXT:    br label %[[LOOP:.*]]
 ; IC4VF1:       [[LOOP]]:
-; IC4VF1-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC4VF1-NEXT:    [[RDX:%.*]] = phi i16 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i16 [ 12, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC4VF1-NEXT:    [[GEP_TABLE_IV:%.*]] = getelementptr inbounds [13 x i16], ptr @table, i16 0, i16 [[IV]]
 ; IC4VF1-NEXT:    [[LD_TABLE:%.*]] = load half, ptr [[GEP_TABLE_IV]], align 1
 ; IC4VF1-NEXT:    [[CMP_TABLE_VAL:%.*]] = fcmp ugt half [[LD_TABLE]], [[VAL]]
@@ -947,7 +929,7 @@ exit:                                             ; preds = %loop
 define i64 @select_decreasing_induction_icmp_iv_unsigned(ptr %a) {
 ; IC1VF4-LABEL: define i64 @select_decreasing_induction_icmp_iv_unsigned(
 ; IC1VF4-SAME: ptr [[A:%.*]]) {
-; IC1VF4-NEXT:  [[ENTRY:.*]]:
+; IC1VF4-NEXT:  [[ENTRY:.*:]]
 ; IC1VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC1VF4:       [[VECTOR_PH]]:
 ; IC1VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -973,12 +955,10 @@ define i64 @select_decreasing_induction_icmp_iv_unsigned(ptr %a) {
 ; IC1VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP6]], i64 331
 ; IC1VF4-NEXT:    br label %[[EXIT:.*]]
 ; IC1VF4:       [[SCALAR_PH]]:
-; IC1VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 9223372036854775807, %[[ENTRY]] ]
-; IC1VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ]
 ; IC1VF4-NEXT:    br label %[[LOOP:.*]]
 ; IC1VF4:       [[LOOP]]:
-; IC1VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC1VF4-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[IV:%.*]] = phi i64 [ 9223372036854775807, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC1VF4-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC1VF4-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; IC1VF4-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
 ; IC1VF4-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
@@ -992,7 +972,7 @@ define i64 @select_decreasing_induction_icmp_iv_unsigned(ptr %a) {
 ;
 ; IC4VF4-LABEL: define i64 @select_decreasing_induction_icmp_iv_unsigned(
 ; IC4VF4-SAME: ptr [[A:%.*]]) {
-; IC4VF4-NEXT:  [[ENTRY:.*]]:
+; IC4VF4-NEXT:  [[ENTRY:.*:]]
 ; IC4VF4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC4VF4:       [[VECTOR_PH]]:
 ; IC4VF4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -1045,12 +1025,10 @@ define i64 @select_decreasing_induction_icmp_iv_unsigned(ptr %a) {
 ; IC4VF4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[TMP18]], i64 331
 ; IC4VF4-NEXT:    br label %[[EXIT:.*]]
 ; IC4VF4:       [[SCALAR_PH]]:
-; IC4VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 9223372036854775807, %[[ENTRY]] ]
-; IC4VF4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ]
 ; IC4VF4-NEXT:    br label %[[LOOP:.*]]
 ; IC4VF4:       [[LOOP]]:
-; IC4VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC4VF4-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[IV:%.*]] = phi i64 [ 9223372036854775807, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF4-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC4VF4-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; IC4VF4-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
 ; IC4VF4-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
@@ -1064,7 +1042,7 @@ define i64 @select_decreasing_induction_icmp_iv_unsigned(ptr %a) {
 ;
 ; IC4VF1-LABEL: define i64 @select_decreasing_induction_icmp_iv_unsigned(
 ; IC4VF1-SAME: ptr [[A:%.*]]) {
-; IC4VF1-NEXT:  [[ENTRY:.*]]:
+; IC4VF1-NEXT:  [[ENTRY:.*:]]
 ; IC4VF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC4VF1:       [[VECTOR_PH]]:
 ; IC4VF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -1105,12 +1083,10 @@ define i64 @select_decreasing_induction_icmp_iv_unsigned(ptr %a) {
 ; IC4VF1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i64 [[RDX_MINMAX5]], i64 331
 ; IC4VF1-NEXT:    br label %[[EXIT:.*]]
 ; IC4VF1:       [[SCALAR_PH]]:
-; IC4VF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 9223372036854775807, %[[ENTRY]] ]
-; IC4VF1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ 331, %[[ENTRY]] ]
 ; IC4VF1-NEXT:    br label %[[LOOP:.*]]
 ; IC4VF1:       [[LOOP]]:
-; IC4VF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; IC4VF1-NEXT:    [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[IV:%.*]] = phi i64 [ 9223372036854775807, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4VF1-NEXT:    [[RDX:%.*]] = phi i64 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[LOOP]] ]
 ; IC4VF1-NEXT:    [[GEP_A_IV:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; IC4VF1-NEXT:    [[LD_A:%.*]] = load i64, ptr [[GEP_A_IV]], align 8
 ; IC4VF1-NEXT:    [[CMP_A_3:%.*]] = icmp sgt i64 [[LD_A]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
index 1054482fb80d5..80c5bb359cb4a 100644
--- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll
@@ -240,7 +240,7 @@ exit:                                            ; preds = %for.body, %entry
 define i32 @select_icmp_const_truncated_iv_const_exit(ptr %a) {
 ; CHECK-VF4IC1-LABEL: define i32 @select_icmp_const_truncated_iv_const_exit(
 ; CHECK-VF4IC1-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VF4IC1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4IC1:       [[VECTOR_PH]]:
 ; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -262,12 +262,10 @@ define i32 @select_icmp_const_truncated_iv_const_exit(ptr %a) {
 ; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[TMP6]], i32 331
 ; CHECK-VF4IC1-NEXT:    br label %[[EXIT:.*]]
 ; CHECK-VF4IC1:       [[SCALAR_PH]]:
-; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 331, %[[ENTRY]] ]
 ; CHECK-VF4IC1-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-VF4IC1:       [[FOR_BODY]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-VF4IC1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[TMP7]], 3
@@ -282,7 +280,7 @@ define i32 @select_icmp_const_truncated_iv_const_exit(ptr %a) {
 ;
 ; CHECK-VF4IC4-LABEL: define i32 @select_icmp_const_truncated_iv_const_exit(
 ; CHECK-VF4IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VF4IC4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4IC4:       [[VECTOR_PH]]:
 ; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -325,12 +323,10 @@ define i32 @select_icmp_const_truncated_iv_const_exit(ptr %a) {
 ; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[TMP15]], i32 331
 ; CHECK-VF4IC4-NEXT:    br label %[[EXIT:.*]]
 ; CHECK-VF4IC4:       [[SCALAR_PH]]:
-; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 331, %[[ENTRY]] ]
 ; CHECK-VF4IC4-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-VF4IC4:       [[FOR_BODY]]:
-; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-VF4IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-VF4IC4-NEXT:    [[TMP16:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-VF4IC4-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[TMP16]], 3
@@ -345,7 +341,7 @@ define i32 @select_icmp_const_truncated_iv_const_exit(ptr %a) {
 ;
 ; CHECK-VF1IC4-LABEL: define i32 @select_icmp_const_truncated_iv_const_exit(
 ; CHECK-VF1IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VF1IC4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF1IC4:       [[VECTOR_PH]]:
 ; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -389,12 +385,10 @@ define i32 @select_icmp_const_truncated_iv_const_exit(ptr %a) {
 ; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[RDX_MINMAX5]], i32 331
 ; CHECK-VF1IC4-NEXT:    br label %[[EXIT:.*]]
 ; CHECK-VF1IC4:       [[SCALAR_PH]]:
-; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 331, %[[ENTRY]] ]
 ; CHECK-VF1IC4-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-VF1IC4:       [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-VF1IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-VF1IC4-NEXT:    [[TMP26:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-VF1IC4-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[TMP26]], 3
@@ -431,7 +425,7 @@ exit:                                           ; preds = %for.body
 define i32 @select_fcmp_max_valid_const_ub(ptr %a) {
 ; CHECK-VF4IC1-LABEL: define i32 @select_fcmp_max_valid_const_ub(
 ; CHECK-VF4IC1-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VF4IC1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4IC1:       [[VECTOR_PH]]:
 ; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -453,12 +447,10 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) {
 ; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[TMP6]], i32 -1
 ; CHECK-VF4IC1-NEXT:    br label %[[EXIT:.*]]
 ; CHECK-VF4IC1:       [[SCALAR_PH]]:
-; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ -1, %[[ENTRY]] ]
 ; CHECK-VF4IC1-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-VF4IC1:       [[FOR_BODY]]:
-; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ -1, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-VF4IC1-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; CHECK-VF4IC1-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = fcmp fast olt float [[TMP7]], 0.000000e+00
@@ -473,7 +465,7 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) {
 ;
 ; CHECK-VF4IC4-LABEL: define i32 @select_fcmp_max_valid_const_ub(
 ; CHECK-VF4IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VF4IC4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4IC4:       [[VECTOR_PH]]:
 ; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -516,12 +508,10 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) {
 ; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[TMP15]], i32 -1
 ; CHECK-VF4IC4-NEXT:    br label %[[EXIT:.*]]
 ; CHECK-VF4IC4:       [[SCALAR_PH]]:
-; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ -1, %[[ENTRY]] ]
 ; CHECK-VF4IC4-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-VF4IC4:       [[FOR_BODY]]:
-; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ -1, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-VF4IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; CHECK-VF4IC4-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-VF4IC4-NEXT:    [[CMP:%.*]] = fcmp fast olt float [[TMP16]], 0.000000e+00
@@ -536,7 +526,7 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) {
 ;
 ; CHECK-VF1IC4-LABEL: define i32 @select_fcmp_max_valid_const_ub(
 ; CHECK-VF1IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VF1IC4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF1IC4:       [[VECTOR_PH]]:
 ; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -580,12 +570,10 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) {
 ; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[RDX_MINMAX5]], i32 -1
 ; CHECK-VF1IC4-NEXT:    br label %[[EXIT:.*]]
 ; CHECK-VF1IC4:       [[SCALAR_PH]]:
-; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ -1, %[[ENTRY]] ]
 ; CHECK-VF1IC4-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-VF1IC4:       [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ -1, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-VF1IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
 ; CHECK-VF1IC4-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-VF1IC4-NEXT:    [[CMP:%.*]] = fcmp fast olt float [[TMP26]], 0.000000e+00
@@ -626,7 +614,7 @@ exit:                                            ; preds = %for.body
 define i32 @select_icmp_truncated_unsigned_iv_range(ptr %a) {
 ; CHECK-VF4IC1-LABEL: define i32 @select_icmp_truncated_unsigned_iv_range(
 ; CHECK-VF4IC1-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC1-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC1-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VF4IC1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4IC1:       [[VECTOR_PH]]:
 ; CHECK-VF4IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -649,12 +637,10 @@ define i32 @select_icmp_truncated_unsigned_iv_range(ptr %a) {
 ; CHECK-VF4IC1-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[TMP5]], i32 331
 ; CHECK-VF4IC1-NEXT:    br label %[[EXIT:.*]]
 ; CHECK-VF4IC1:       [[SCALAR_PH]]:
-; CHECK-VF4IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 2147483646, %[[ENTRY]] ]
-; CHECK-VF4IC1-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 331, %[[ENTRY]] ]
 ; CHECK-VF4IC1-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-VF4IC1:       [[FOR_BODY]]:
-; CHECK-VF4IC1-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[IV1:%.*]] = phi i64 [ 2147483646, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC1-NEXT:    [[RDX:%.*]] = phi i32 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-VF4IC1-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV1]]
 ; CHECK-VF4IC1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
 ; CHECK-VF4IC1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP0]], 3
@@ -669,7 +655,7 @@ define i32 @select_icmp_truncated_unsigned_iv_range(ptr %a) {
 ;
 ; CHECK-VF4IC4-LABEL: define i32 @select_icmp_truncated_unsigned_iv_range(
 ; CHECK-VF4IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF4IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF4IC4-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VF4IC4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4IC4:       [[VECTOR_PH]]:
 ; CHECK-VF4IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -713,12 +699,10 @@ define i32 @select_icmp_truncated_unsigned_iv_range(ptr %a) {
 ; CHECK-VF4IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[TMP14]], i32 331
 ; CHECK-VF4IC4-NEXT:    br label %[[EXIT:.*]]
 ; CHECK-VF4IC4:       [[SCALAR_PH]]:
-; CHECK-VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 2147483646, %[[ENTRY]] ]
-; CHECK-VF4IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 331, %[[ENTRY]] ]
 ; CHECK-VF4IC4-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-VF4IC4:       [[FOR_BODY]]:
-; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ 2147483646, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF4IC4-NEXT:    [[RDX:%.*]] = phi i32 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-VF4IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-VF4IC4-NEXT:    [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-VF4IC4-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP15]], 3
@@ -733,7 +717,7 @@ define i32 @select_icmp_truncated_unsigned_iv_range(ptr %a) {
 ;
 ; CHECK-VF1IC4-LABEL: define i32 @select_icmp_truncated_unsigned_iv_range(
 ; CHECK-VF1IC4-SAME: ptr [[A:%.*]]) {
-; CHECK-VF1IC4-NEXT:  [[ENTRY:.*]]:
+; CHECK-VF1IC4-NEXT:  [[ENTRY:.*:]]
 ; CHECK-VF1IC4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF1IC4:       [[VECTOR_PH]]:
 ; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -779,12 +763,10 @@ define i32 @select_icmp_truncated_unsigned_iv_range(ptr %a) {
 ; CHECK-VF1IC4-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[RDX_MINMAX6]], i32 331
 ; CHECK-VF1IC4-NEXT:    br label %[[EXIT:.*]]
 ; CHECK-VF1IC4:       [[SCALAR_PH]]:
-; CHECK-VF1IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 2147483646, %[[ENTRY]] ]
-; CHECK-VF1IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 331, %[[ENTRY]] ]
 ; CHECK-VF1IC4-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK-VF1IC4:       [[FOR_BODY]]:
-; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[IV:%.*]] = phi i64 [ 2147483646, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-VF1IC4-NEXT:    [[RDX:%.*]] = phi i32 [ 331, %[[SCALAR_PH]] ], [ [[SPEC_SELECT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-VF1IC4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-VF1IC4-NEXT:    [[TMP24:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-VF1IC4-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP24]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
index 97d33858bd830..766e7acdfd1c7 100644
--- a/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/iv_outside_user.ll
@@ -91,7 +91,7 @@ for.end:
 
 define i32 @constpre()  {
 ; CHECK-LABEL: define i32 @constpre() {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -103,10 +103,9 @@ define i32 @constpre()  {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[INC_PHI:%.*]] = phi i32 [ 32, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[INC]] = sub nsw i32 [[INC_PHI]], 2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_END]], label %[[FOR_BODY]], {{!llvm.loop ![0-9]+}}
@@ -130,7 +129,7 @@ for.end:
 define ptr @geppre(ptr %ptr) {
 ; CHECK-LABEL: define ptr @geppre(
 ; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[PTR]], i64 512
@@ -144,12 +143,10 @@ define ptr @geppre(ptr %ptr) {
 ; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -16
 ; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[PTR]], %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[INC_PHI:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[PTR_PHI:%.*]] = phi ptr [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[INC_PTR:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[INC_PHI:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[PTR_PHI:%.*]] = phi ptr [ [[PTR]], %[[SCALAR_PH]] ], [ [[INC_PTR:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[INC]] = add nsw i32 [[INC_PHI]], 1
 ; CHECK-NEXT:    [[INC_PTR]] = getelementptr i32, ptr [[PTR_PHI]], i32 4
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[INC]], 32
@@ -399,7 +396,7 @@ BB4:
 define i64 @iv_scalar_steps_and_outside_users(ptr %ptr) {
 ; VEC-LABEL: define i64 @iv_scalar_steps_and_outside_users(
 ; VEC-SAME: ptr [[PTR:%.*]]) {
-; VEC-NEXT:  [[ENTRY:.*]]:
+; VEC-NEXT:  [[ENTRY:.*:]]
 ; VEC-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VEC:       [[VECTOR_PH]]:
 ; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -415,10 +412,9 @@ define i64 @iv_scalar_steps_and_outside_users(ptr %ptr) {
 ; VEC:       [[MIDDLE_BLOCK]]:
 ; VEC-NEXT:    br label %[[EXIT:.*]]
 ; VEC:       [[SCALAR_PH]]:
-; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VEC-NEXT:    br label %[[LOOP:.*]]
 ; VEC:       [[LOOP]]:
-; VEC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VEC-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VEC-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
 ; VEC-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]]
 ; VEC-NEXT:    store i64 [[IV]], ptr [[GEP_PTR]], align 4
@@ -430,7 +426,7 @@ define i64 @iv_scalar_steps_and_outside_users(ptr %ptr) {
 ;
 ; INTERLEAVE-LABEL: define i64 @iv_scalar_steps_and_outside_users(
 ; INTERLEAVE-SAME: ptr [[PTR:%.*]]) {
-; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
+; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
 ; INTERLEAVE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; INTERLEAVE:       [[VECTOR_PH]]:
 ; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -447,10 +443,9 @@ define i64 @iv_scalar_steps_and_outside_users(ptr %ptr) {
 ; INTERLEAVE:       [[MIDDLE_BLOCK]]:
 ; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
 ; INTERLEAVE:       [[SCALAR_PH]]:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
 ; INTERLEAVE:       [[LOOP]]:
-; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; INTERLEAVE-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
 ; INTERLEAVE-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]]
 ; INTERLEAVE-NEXT:    store i64 [[IV]], ptr [[GEP_PTR]], align 4
@@ -481,7 +476,7 @@ exit:
 define i32 @iv_2_dead_in_loop_only_used_outside(ptr %ptr) {
 ; VEC-LABEL: define i32 @iv_2_dead_in_loop_only_used_outside(
 ; VEC-SAME: ptr [[PTR:%.*]]) {
-; VEC-NEXT:  [[ENTRY:.*]]:
+; VEC-NEXT:  [[ENTRY:.*:]]
 ; VEC-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VEC:       [[VECTOR_PH]]:
 ; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -497,12 +492,10 @@ define i32 @iv_2_dead_in_loop_only_used_outside(ptr %ptr) {
 ; VEC:       [[MIDDLE_BLOCK]]:
 ; VEC-NEXT:    br label %[[EXIT:.*]]
 ; VEC:       [[SCALAR_PH]]:
-; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; VEC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; VEC-NEXT:    br label %[[LOOP:.*]]
 ; VEC:       [[LOOP]]:
-; VEC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; VEC-NEXT:    [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
+; VEC-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VEC-NEXT:    [[IV_2:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
 ; VEC-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
 ; VEC-NEXT:    [[IV_2_NEXT]] = add nuw i32 [[IV_2]], 2
 ; VEC-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]]
@@ -515,7 +508,7 @@ define i32 @iv_2_dead_in_loop_only_used_outside(ptr %ptr) {
 ;
 ; INTERLEAVE-LABEL: define i32 @iv_2_dead_in_loop_only_used_outside(
 ; INTERLEAVE-SAME: ptr [[PTR:%.*]]) {
-; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
+; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
 ; INTERLEAVE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; INTERLEAVE:       [[VECTOR_PH]]:
 ; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -532,12 +525,10 @@ define i32 @iv_2_dead_in_loop_only_used_outside(ptr %ptr) {
 ; INTERLEAVE:       [[MIDDLE_BLOCK]]:
 ; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
 ; INTERLEAVE:       [[SCALAR_PH]]:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
 ; INTERLEAVE:       [[LOOP]]:
-; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; INTERLEAVE-NEXT:    [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV_2:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
 ; INTERLEAVE-NEXT:    [[IV_NEXT]] = add nuw i64 [[IV]], 1
 ; INTERLEAVE-NEXT:    [[IV_2_NEXT]] = add nuw i32 [[IV_2]], 2
 ; INTERLEAVE-NEXT:    [[GEP_PTR:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 [[IV]]
@@ -1083,7 +1074,7 @@ exit:
 define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) {
 ; VEC-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification(
 ; VEC-SAME: ptr [[DST:%.*]]) {
-; VEC-NEXT:  [[ENTRY:.*]]:
+; VEC-NEXT:  [[ENTRY:.*:]]
 ; VEC-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
 ; VEC-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
 ; VEC-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1102,10 +1093,9 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) {
 ; VEC:       [[MIDDLE_BLOCK]]:
 ; VEC-NEXT:    br label %[[E_EXIT:.*]]
 ; VEC:       [[SCALAR_PH]]:
-; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; VEC-NEXT:    br label %[[LOOP:.*]]
 ; VEC:       [[LOOP]]:
-; VEC-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VEC-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VEC-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[IV]]
 ; VEC-NEXT:    store i16 0, ptr [[GEP_DST]], align 2
 ; VEC-NEXT:    [[IV_NEXT]] = add i32 [[STEP_2]], [[IV]]
@@ -1117,7 +1107,7 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) {
 ;
 ; INTERLEAVE-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification(
 ; INTERLEAVE-SAME: ptr [[DST:%.*]]) {
-; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
+; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
 ; INTERLEAVE-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
 ; INTERLEAVE-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
 ; INTERLEAVE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1137,10 +1127,9 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification(ptr %dst) {
 ; INTERLEAVE:       [[MIDDLE_BLOCK]]:
 ; INTERLEAVE-NEXT:    br label %[[E_EXIT:.*]]
 ; INTERLEAVE:       [[SCALAR_PH]]:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
 ; INTERLEAVE:       [[LOOP]]:
-; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; INTERLEAVE-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[IV]]
 ; INTERLEAVE-NEXT:    store i16 0, ptr [[GEP_DST]], align 2
 ; INTERLEAVE-NEXT:    [[IV_NEXT]] = add i32 [[STEP_2]], [[IV]]
@@ -1171,7 +1160,7 @@ e.exit:
 define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(ptr %dst) {
 ; VEC-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(
 ; VEC-SAME: ptr [[DST:%.*]]) {
-; VEC-NEXT:  [[ENTRY:.*]]:
+; VEC-NEXT:  [[ENTRY:.*:]]
 ; VEC-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
 ; VEC-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
 ; VEC-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1199,10 +1188,9 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(ptr %dst) {
 ; VEC-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP5]], i32 1
 ; VEC-NEXT:    br label %[[E_EXIT:.*]]
 ; VEC:       [[SCALAR_PH]]:
-; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; VEC-NEXT:    br label %[[LOOP:.*]]
 ; VEC:       [[LOOP]]:
-; VEC-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VEC-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VEC-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[IV]]
 ; VEC-NEXT:    store i16 0, ptr [[GEP_DST]], align 2
 ; VEC-NEXT:    [[INC:%.*]] = add i32 [[IV]], 1
@@ -1215,7 +1203,7 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(ptr %dst) {
 ;
 ; INTERLEAVE-LABEL: define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(
 ; INTERLEAVE-SAME: ptr [[DST:%.*]]) {
-; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
+; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
 ; INTERLEAVE-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
 ; INTERLEAVE-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
 ; INTERLEAVE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1237,10 +1225,9 @@ define i32 @test_iv_uniform_with_outside_use_scev_simplification_2(ptr %dst) {
 ; INTERLEAVE:       [[MIDDLE_BLOCK]]:
 ; INTERLEAVE-NEXT:    br label %[[E_EXIT:.*]]
 ; INTERLEAVE:       [[SCALAR_PH]]:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
 ; INTERLEAVE:       [[LOOP]]:
-; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; INTERLEAVE-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[IV]]
 ; INTERLEAVE-NEXT:    store i16 0, ptr [[GEP_DST]], align 2
 ; INTERLEAVE-NEXT:    [[INC:%.*]] = add i32 [[IV]], 1
@@ -1363,7 +1350,7 @@ exit:
 define i64 @test_iv_increment_incremented(ptr %dst) {
 ; VEC-LABEL: define i64 @test_iv_increment_incremented(
 ; VEC-SAME: ptr [[DST:%.*]]) {
-; VEC-NEXT:  [[ENTRY:.*]]:
+; VEC-NEXT:  [[ENTRY:.*:]]
 ; VEC-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VEC:       [[VECTOR_PH]]:
 ; VEC-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -1378,12 +1365,10 @@ define i64 @test_iv_increment_incremented(ptr %dst) {
 ; VEC:       [[MIDDLE_BLOCK]]:
 ; VEC-NEXT:    br label %[[EXIT:.*]]
 ; VEC:       [[SCALAR_PH]]:
-; VEC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, %[[ENTRY]] ]
-; VEC-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 2, %[[ENTRY]] ]
 ; VEC-NEXT:    br label %[[LOOP:.*]]
 ; VEC:       [[LOOP]]:
-; VEC-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
-; VEC-NEXT:    [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
+; VEC-NEXT:    [[IV_1:%.*]] = phi i64 [ 3, %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
+; VEC-NEXT:    [[IV_2:%.*]] = phi i64 [ 2, %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
 ; VEC-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[DST]], i64 [[IV_1]]
 ; VEC-NEXT:    store i16 1, ptr [[GEP]], align 2
 ; VEC-NEXT:    [[IV_2_NEXT]] = add i64 [[IV_2]], -1
@@ -1396,7 +1381,7 @@ define i64 @test_iv_increment_incremented(ptr %dst) {
 ;
 ; INTERLEAVE-LABEL: define i64 @test_iv_increment_incremented(
 ; INTERLEAVE-SAME: ptr [[DST:%.*]]) {
-; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
+; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
 ; INTERLEAVE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; INTERLEAVE:       [[VECTOR_PH]]:
 ; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -1411,12 +1396,10 @@ define i64 @test_iv_increment_incremented(ptr %dst) {
 ; INTERLEAVE:       [[MIDDLE_BLOCK]]:
 ; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
 ; INTERLEAVE:       [[SCALAR_PH]]:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, %[[ENTRY]] ]
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 2, %[[ENTRY]] ]
 ; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
 ; INTERLEAVE:       [[LOOP]]:
-; INTERLEAVE-NEXT:    [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
-; INTERLEAVE-NEXT:    [[IV_2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV_1:%.*]] = phi i64 [ 3, %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV_2:%.*]] = phi i64 [ 2, %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP]] ]
 ; INTERLEAVE-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[DST]], i64 [[IV_1]]
 ; INTERLEAVE-NEXT:    store i16 1, ptr [[GEP]], align 2
 ; INTERLEAVE-NEXT:    [[IV_2_NEXT]] = add i64 [[IV_2]], -1
diff --git a/llvm/test/Transforms/LoopVectorize/lifetime.ll b/llvm/test/Transforms/LoopVectorize/lifetime.ll
index 3dd41b57d863b..61e8635774d35 100644
--- a/llvm/test/Transforms/LoopVectorize/lifetime.ll
+++ b/llvm/test/Transforms/LoopVectorize/lifetime.ll
@@ -12,23 +12,23 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 define void @test(ptr %d) {
 entry:
   %arr = alloca [1024 x i32], align 16
-  call void @llvm.lifetime.start.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.start.p0(ptr %arr) #1
   br label %for.body
 
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  call void @llvm.lifetime.end.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.end.p0(ptr %arr) #1
   %arrayidx = getelementptr inbounds i32, ptr %d, i64 %indvars.iv
   %0 = load i32, ptr %arrayidx, align 8
   store i32 100, ptr %arrayidx, align 8
-  call void @llvm.lifetime.start.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.start.p0(ptr %arr) #1
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp ne i32 %lftr.wideiv, 128
   br i1 %exitcond, label %for.body, label %for.end
 
 for.end:
-  call void @llvm.lifetime.end.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.end.p0(ptr %arr) #1
   ret void
 }
 
@@ -40,26 +40,26 @@ for.end:
 define void @testbitcast(ptr %d) {
 entry:
   %arr = alloca [1024 x i32], align 16
-  call void @llvm.lifetime.start.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.start.p0(ptr %arr) #1
   br label %for.body
 
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  call void @llvm.lifetime.end.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.end.p0(ptr %arr) #1
   %arrayidx = getelementptr inbounds i32, ptr %d, i64 %indvars.iv
   %0 = load i32, ptr %arrayidx, align 8
   store i32 100, ptr %arrayidx, align 8
-  call void @llvm.lifetime.start.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.start.p0(ptr %arr) #1
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp ne i32 %lftr.wideiv, 128
   br i1 %exitcond, label %for.body, label %for.end
 
 for.end:
-  call void @llvm.lifetime.end.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.end.p0(ptr %arr) #1
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
index 2c7d1bd3a134a..b7b67c263d445 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-align.ll
@@ -50,12 +50,10 @@ define i16 @test_access_size_not_multiple_of_align(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> [[TMP15]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i16 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_T:%.*]] = load i8, ptr [[TEST_ADDR]], align 1
@@ -146,12 +144,10 @@ define i32 @test_access_size_multiple_of_align_but_offset_by_1(i64 %len, ptr %te
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[TMP15]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_T:%.*]] = load i8, ptr [[TEST_ADDR]], align 1
@@ -376,10 +372,9 @@ define void @test_rev_loops_deref_loads(ptr nocapture noundef writeonly %dest) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1023, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1023, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[IV]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[TMP19]], 3
@@ -488,10 +483,9 @@ define void @test_rev_loops_non_deref_loads(ptr nocapture noundef writeonly %des
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1023, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 1023, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    [[OFF:%.*]] = add i64 [[IV]], -1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[OFF]]
 ; CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
@@ -582,12 +576,10 @@ define i16 @test_strided_access(i64 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> [[TMP13]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i16 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i8, ptr [[TEST_BASE]], i64 [[IV]]
 ; CHECK-NEXT:    [[L_T:%.*]] = load i8, ptr [[TEST_ADDR]], align 1
@@ -691,10 +683,9 @@ define void @test_rev_loops_strided_deref_loads(ptr nocapture noundef writeonly
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 511, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 511, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i32], ptr [[LOCAL_CMP]], i64 0, i64 [[IV]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[CMP3_NOT:%.*]] = icmp eq i32 [[TMP21]], 3
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll
index 34c04de22755f..468e6823e9b56 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-neg-off.ll
@@ -54,12 +54,10 @@ define i8 @test_negative_off(i16 %len, ptr %test_base) {
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> [[TMP18]])
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ -1000, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i8 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
-; CHECK-NEXT:    [[ACCUM:%.*]] = phi i8 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ -1000, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-NEXT:    [[ACCUM:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[ACCUM_NEXT:%.*]], [[LATCH]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i16 [[IV]], 1
 ; CHECK-NEXT:    [[TEST_ADDR:%.*]] = getelementptr inbounds i1, ptr [[TEST_BASE]], i16 [[IV]]
 ; CHECK-NEXT:    [[EARLYCND:%.*]] = load i1, ptr [[TEST_ADDR]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/load-of-struct-deref-pred.ll b/llvm/test/Transforms/LoopVectorize/load-of-struct-deref-pred.ll
index f99e883c045d7..c822ded111c62 100644
--- a/llvm/test/Transforms/LoopVectorize/load-of-struct-deref-pred.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-of-struct-deref-pred.ll
@@ -31,10 +31,9 @@ define void @accesses_to_struct_dereferenceable(ptr noalias %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    [[D:%.*]] = load i32, ptr [[GEP_DST]], align 4
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[D]], 0
@@ -98,55 +97,54 @@ define void @accesses_to_struct_may_not_be_dereferenceable_due_to_loop_bound(ptr
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <4 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK:       pred.load.if:
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i32 0
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
-; CHECK-NEXT:    [[TMP8:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
-; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; CHECK:       pred.load.if1:
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4
-; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP11]], i32 1
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
 ; CHECK:       pred.load.continue2:
-; CHECK-NEXT:    [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
-; CHECK-NEXT:    br i1 [[TMP15]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
+; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; CHECK:       pred.load.if3:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4
-; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i32 2
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP17]], i32 2
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
 ; CHECK:       pred.load.continue4:
-; CHECK-NEXT:    [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], [[PRED_LOAD_IF3]] ]
-; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
-; CHECK-NEXT:    br i1 [[TMP21]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK-NEXT:    [[TMP19:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
+; CHECK-NEXT:    br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
 ; CHECK:       pred.load.if5:
-; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP22]]
-; CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4
-; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i32 3
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP23]], i32 3
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
 ; CHECK:       pred.load.continue6:
-; CHECK-NEXT:    [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP25]], [[PRED_LOAD_IF5]] ]
-; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP27]], align 4
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> [[TMP26]]
+; CHECK-NEXT:    [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP26]], align 4
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP25]], <4 x i32> [[WIDE_LOAD7]]
 ; CHECK-NEXT:    store <4 x i32> [[PREDPHI]], ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
-; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
+; CHECK-NEXT:    br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -217,63 +215,61 @@ define void @accesses_to_struct_may_not_be_dereferenceable_access_size(ptr noali
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <4 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i1> [[TMP1]], splat (i1 true)
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i32 0
-; CHECK-NEXT:    br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge <4 x i32> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
+; CHECK-NEXT:    br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
 ; CHECK:       pred.load.if:
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i64> poison, i64 [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr @foo, i64 0, i32 1, i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP5]], i32 0
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE]]
 ; CHECK:       pred.load.continue:
-; CHECK-NEXT:    [[TMP8:%.*]] = phi <4 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i32 1
-; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = phi <4 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
+; CHECK-NEXT:    br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]]
 ; CHECK:       pred.load.if1:
-; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP11]], align 4
-; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i64> [[TMP8]], i64 [[TMP12]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x i64> [[TMP7]], i64 [[TMP11]], i32 1
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE2]]
 ; CHECK:       pred.load.continue2:
-; CHECK-NEXT:    [[TMP14:%.*]] = phi <4 x i64> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i1> [[TMP2]], i32 2
-; CHECK-NEXT:    br i1 [[TMP15]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
+; CHECK-NEXT:    [[TMP13:%.*]] = phi <4 x i64> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
+; CHECK-NEXT:    br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]]
 ; CHECK:       pred.load.if3:
-; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = load i64, ptr [[TMP17]], align 4
-; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP18]], i32 2
+; CHECK-NEXT:    [[TMP15:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP17]], i32 2
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE4]]
 ; CHECK:       pred.load.continue4:
-; CHECK-NEXT:    [[TMP20:%.*]] = phi <4 x i64> [ [[TMP14]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], [[PRED_LOAD_IF3]] ]
-; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP2]], i32 3
-; CHECK-NEXT:    br i1 [[TMP21]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
+; CHECK-NEXT:    [[TMP19:%.*]] = phi <4 x i64> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ]
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
+; CHECK-NEXT:    br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]]
 ; CHECK:       pred.load.if5:
-; CHECK-NEXT:    [[TMP22:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP22]]
-; CHECK-NEXT:    [[TMP24:%.*]] = load i64, ptr [[TMP23]], align 4
-; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <4 x i64> [[TMP20]], i64 [[TMP24]], i32 3
+; CHECK-NEXT:    [[TMP21:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr @foo, i64 0, i32 1, i64 [[TMP21]]
+; CHECK-NEXT:    [[TMP23:%.*]] = load i64, ptr [[TMP22]], align 4
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP23]], i32 3
 ; CHECK-NEXT:    br label [[PRED_LOAD_CONTINUE6]]
 ; CHECK:       pred.load.continue6:
-; CHECK-NEXT:    [[TMP26:%.*]] = phi <4 x i64> [ [[TMP20]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP25]], [[PRED_LOAD_IF5]] ]
-; CHECK-NEXT:    [[TMP27:%.*]] = trunc <4 x i64> [[TMP26]] to <4 x i32>
-; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP28]], align 4
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[WIDE_LOAD7]], <4 x i32> [[TMP27]]
+; CHECK-NEXT:    [[TMP25:%.*]] = phi <4 x i64> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ]
+; CHECK-NEXT:    [[TMP26:%.*]] = trunc <4 x i64> [[TMP25]] to <4 x i32>
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr [[STRUCT_FOO]], ptr @foo, i64 0, i32 0, i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP27]], align 4
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP26]], <4 x i32> [[WIDE_LOAD7]]
 ; CHECK-NEXT:    store <4 x i32> [[PREDPHI]], ptr [[TMP0]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
-; CHECK-NEXT:    br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32000
+; CHECK-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    [[D:%.*]] = load i32, ptr [[GEP_DST]], align 4
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i32 [[D]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/loop-form.ll b/llvm/test/Transforms/LoopVectorize/loop-form.ll
index 10b2e704cb891..1cbfe78cad7de 100644
--- a/llvm/test/Transforms/LoopVectorize/loop-form.ll
+++ b/llvm/test/Transforms/LoopVectorize/loop-form.ll
@@ -81,10 +81,9 @@ define void @bottom_tested(ptr %p, i32 %n) {
 ; TAILFOLD:       middle.block:
 ; TAILFOLD-NEXT:    br label [[IF_END:%.*]]
 ; TAILFOLD:       scalar.ph:
-; TAILFOLD-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; TAILFOLD-NEXT:    br label [[FOR_COND:%.*]]
 ; TAILFOLD:       for.cond:
-; TAILFOLD-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_COND]] ]
+; TAILFOLD-NEXT:    [[I:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_COND]] ]
 ; TAILFOLD-NEXT:    [[IPROM:%.*]] = sext i32 [[I]] to i64
 ; TAILFOLD-NEXT:    [[B:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IPROM]]
 ; TAILFOLD-NEXT:    store i16 0, ptr [[B]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/make-followup-loop-id.ll b/llvm/test/Transforms/LoopVectorize/make-followup-loop-id.ll
index 14a091feb58b5..3190d239e0477 100644
--- a/llvm/test/Transforms/LoopVectorize/make-followup-loop-id.ll
+++ b/llvm/test/Transforms/LoopVectorize/make-followup-loop-id.ll
@@ -15,7 +15,7 @@
 define void @f(ptr noundef captures(none) %a, float noundef %x) {
 ; CHECK-LABEL: define void @f(
 ; CHECK-SAME: ptr noundef captures(none) [[A:%.*]], float noundef [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[X]], i64 0
@@ -68,10 +68,9 @@ define void @f(ptr noundef captures(none) %a, float noundef %x) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT_7:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT_7:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[X]], [[LOAD]]
diff --git a/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll
index c9066f22c5592..03d2e90200eeb 100644
--- a/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/memdep-fold-tail.ll
@@ -71,10 +71,9 @@ define void @maxvf3() {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[J_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[J_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[AJ:%.*]] = getelementptr inbounds [18 x i8], ptr @a, i32 0, i32 [[J]]
 ; CHECK-NEXT:    store i8 69, ptr [[AJ]], align 8
 ; CHECK-NEXT:    [[JP3:%.*]] = add nuw nsw i32 3, [[J]]
diff --git a/llvm/test/Transforms/LoopVectorize/metadata.ll b/llvm/test/Transforms/LoopVectorize/metadata.ll
index ce9c624086331..54779ed55cff8 100644
--- a/llvm/test/Transforms/LoopVectorize/metadata.ll
+++ b/llvm/test/Transforms/LoopVectorize/metadata.ll
@@ -126,7 +126,7 @@ exit:
 define void @widen_call_range(ptr noalias %a, ptr readonly %b) {
 ; CHECK-LABEL: define void @widen_call_range(
 ; CHECK-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -143,10 +143,9 @@ define void @widen_call_range(ptr noalias %a, ptr readonly %b) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4, !tbaa [[TBAA0]], !range [[RNG9:![0-9]+]]
 ; CHECK-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR1:[0-9]+]], !range [[RNG9]]
@@ -160,7 +159,7 @@ define void @widen_call_range(ptr noalias %a, ptr readonly %b) {
 ;
 ; INTERLEAVE-LABEL: define void @widen_call_range(
 ; INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]]) {
-; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
+; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
 ; INTERLEAVE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; INTERLEAVE:       [[VECTOR_PH]]:
 ; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -182,10 +181,9 @@ define void @widen_call_range(ptr noalias %a, ptr readonly %b) {
 ; INTERLEAVE:       [[MIDDLE_BLOCK]]:
 ; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
 ; INTERLEAVE:       [[SCALAR_PH]]:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
 ; INTERLEAVE:       [[LOOP]]:
-; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; INTERLEAVE-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]
 ; INTERLEAVE-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4, !tbaa [[TBAA0]], !range [[RNG9:![0-9]+]]
 ; INTERLEAVE-NEXT:    [[CALL:%.*]] = call i64 @foo(i64 [[LOAD]]) #[[ATTR1:[0-9]+]], !range [[RNG9]]
@@ -218,7 +216,7 @@ exit:
 define void @widen_call_fpmath(ptr noalias %a, ptr readonly %b) {
 ; CHECK-LABEL: define void @widen_call_fpmath(
 ; CHECK-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -235,10 +233,9 @@ define void @widen_call_fpmath(ptr noalias %a, ptr readonly %b) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr double, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load double, ptr [[GEP]], align 8, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    [[CALL:%.*]] = call double @bar(double [[LOAD]]) #[[ATTR2:[0-9]+]], !fpmath [[META3]]
@@ -252,7 +249,7 @@ define void @widen_call_fpmath(ptr noalias %a, ptr readonly %b) {
 ;
 ; INTERLEAVE-LABEL: define void @widen_call_fpmath(
 ; INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]]) {
-; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
+; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
 ; INTERLEAVE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; INTERLEAVE:       [[VECTOR_PH]]:
 ; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -274,10 +271,9 @@ define void @widen_call_fpmath(ptr noalias %a, ptr readonly %b) {
 ; INTERLEAVE:       [[MIDDLE_BLOCK]]:
 ; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
 ; INTERLEAVE:       [[SCALAR_PH]]:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
 ; INTERLEAVE:       [[LOOP]]:
-; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; INTERLEAVE-NEXT:    [[GEP:%.*]] = getelementptr double, ptr [[B]], i64 [[IV]]
 ; INTERLEAVE-NEXT:    [[LOAD:%.*]] = load double, ptr [[GEP]], align 8, !tbaa [[TBAA0]]
 ; INTERLEAVE-NEXT:    [[CALL:%.*]] = call double @bar(double [[LOAD]]) #[[ATTR2:[0-9]+]], !fpmath [[META3]]
@@ -310,7 +306,7 @@ exit:
 define void @widen_intrinsic(ptr noalias %a, ptr readonly %b) {
 ; CHECK-LABEL: define void @widen_intrinsic(
 ; CHECK-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -327,10 +323,9 @@ define void @widen_intrinsic(ptr noalias %a, ptr readonly %b) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[CALL:%.*]] = call i64 @llvm.abs.i64(i64 [[LOAD]], i1 true), !range [[RNG9]]
@@ -344,7 +339,7 @@ define void @widen_intrinsic(ptr noalias %a, ptr readonly %b) {
 ;
 ; INTERLEAVE-LABEL: define void @widen_intrinsic(
 ; INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]]) {
-; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
+; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
 ; INTERLEAVE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; INTERLEAVE:       [[VECTOR_PH]]:
 ; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -366,10 +361,9 @@ define void @widen_intrinsic(ptr noalias %a, ptr readonly %b) {
 ; INTERLEAVE:       [[MIDDLE_BLOCK]]:
 ; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
 ; INTERLEAVE:       [[SCALAR_PH]]:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
 ; INTERLEAVE:       [[LOOP]]:
-; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; INTERLEAVE-NEXT:    [[GEP:%.*]] = getelementptr i64, ptr [[B]], i64 [[IV]]
 ; INTERLEAVE-NEXT:    [[LOAD:%.*]] = load i64, ptr [[GEP]], align 4
 ; INTERLEAVE-NEXT:    [[CALL:%.*]] = call i64 @llvm.abs.i64(i64 [[LOAD]], i1 true), !range [[RNG9]]
@@ -402,7 +396,7 @@ exit:
 define void @widen_intrinsic_fpmath(ptr noalias %a, ptr readonly %b) {
 ; CHECK-LABEL: define void @widen_intrinsic_fpmath(
 ; CHECK-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -419,10 +413,9 @@ define void @widen_intrinsic_fpmath(ptr noalias %a, ptr readonly %b) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr double, ptr [[B]], i64 [[IV]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load double, ptr [[GEP]], align 8, !tbaa [[TBAA0]]
 ; CHECK-NEXT:    [[CALL:%.*]] = call double @llvm.sin.f64(double [[LOAD]]) #[[ATTR2]], !fpmath [[META3]]
@@ -436,7 +429,7 @@ define void @widen_intrinsic_fpmath(ptr noalias %a, ptr readonly %b) {
 ;
 ; INTERLEAVE-LABEL: define void @widen_intrinsic_fpmath(
 ; INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readonly [[B:%.*]]) {
-; INTERLEAVE-NEXT:  [[ENTRY:.*]]:
+; INTERLEAVE-NEXT:  [[ENTRY:.*:]]
 ; INTERLEAVE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; INTERLEAVE:       [[VECTOR_PH]]:
 ; INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -458,10 +451,9 @@ define void @widen_intrinsic_fpmath(ptr noalias %a, ptr readonly %b) {
 ; INTERLEAVE:       [[MIDDLE_BLOCK]]:
 ; INTERLEAVE-NEXT:    br label %[[EXIT:.*]]
 ; INTERLEAVE:       [[SCALAR_PH]]:
-; INTERLEAVE-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; INTERLEAVE-NEXT:    br label %[[LOOP:.*]]
 ; INTERLEAVE:       [[LOOP]]:
-; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; INTERLEAVE-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; INTERLEAVE-NEXT:    [[GEP:%.*]] = getelementptr double, ptr [[B]], i64 [[IV]]
 ; INTERLEAVE-NEXT:    [[LOAD:%.*]] = load double, ptr [[GEP]], align 8, !tbaa [[TBAA0]]
 ; INTERLEAVE-NEXT:    [[CALL:%.*]] = call double @llvm.sin.f64(double [[LOAD]]) #[[ATTR2]], !fpmath [[META3]]
diff --git a/llvm/test/Transforms/LoopVectorize/minimumnum-maximumnum-reductions.ll b/llvm/test/Transforms/LoopVectorize/minimumnum-maximumnum-reductions.ll
index e26fef4f02eef..536dffa920c67 100644
--- a/llvm/test/Transforms/LoopVectorize/minimumnum-maximumnum-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/minimumnum-maximumnum-reductions.ll
@@ -5,7 +5,7 @@
 define float @maximumnum_intrinsic(ptr readonly %x) {
 ; CHECK-LABEL: define float @maximumnum_intrinsic(
 ; CHECK-SAME: ptr readonly [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -27,12 +27,10 @@ define float @maximumnum_intrinsic(ptr readonly %x) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fmax.v2f32(<2 x float> [[RDX_MINMAX]])
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds float, ptr [[X]], i32 [[IV1]]
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP1]], align 4
 ; CHECK-NEXT:    [[RED_NEXT]] = tail call float @llvm.maximumnum.f32(float [[RED]], float [[L]])
@@ -63,7 +61,7 @@ exit:
 define float @maximumnum_intrinsic_fast(ptr readonly %x) {
 ; CHECK-LABEL: define float @maximumnum_intrinsic_fast(
 ; CHECK-SAME: ptr readonly [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -85,12 +83,10 @@ define float @maximumnum_intrinsic_fast(ptr readonly %x) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fmax.v2f32(<2 x float> [[RDX_MINMAX]])
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds float, ptr [[X]], i32 [[IV1]]
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP1]], align 4
 ; CHECK-NEXT:    [[RED_NEXT]] = tail call fast float @llvm.maximumnum.f32(float [[RED]], float [[L]])
@@ -121,7 +117,7 @@ exit:
 define float @minimumnum_intrinsic(ptr readonly %x) {
 ; CHECK-LABEL: define float @minimumnum_intrinsic(
 ; CHECK-SAME: ptr readonly [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -143,12 +139,10 @@ define float @minimumnum_intrinsic(ptr readonly %x) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fmin.v2f32(<2 x float> [[RDX_MINMAX]])
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds float, ptr [[X]], i32 [[IV1]]
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP1]], align 4
 ; CHECK-NEXT:    [[RED_NEXT]] = tail call float @llvm.minimumnum.f32(float [[RED]], float [[L]])
@@ -179,7 +173,7 @@ exit:
 define float @minimumnum_intrinsic_fast(ptr readonly %x) {
 ; CHECK-LABEL: define float @minimumnum_intrinsic_fast(
 ; CHECK-SAME: ptr readonly [[X:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -201,12 +195,10 @@ define float @minimumnum_intrinsic_fast(ptr readonly %x) {
 ; CHECK-NEXT:    [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fmin.v2f32(<2 x float> [[RDX_MINMAX]])
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV1:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi float [ 0.000000e+00, %[[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds float, ptr [[X]], i32 [[IV1]]
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP1]], align 4
 ; CHECK-NEXT:    [[RED_NEXT]] = tail call fast float @llvm.minimumnum.f32(float [[RED]], float [[L]])
diff --git a/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
new file mode 100644
index 0000000000000..83b0856e12fa1
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/nested-loops-scev-expansion.ll
@@ -0,0 +1,409 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=2 -scev-verify-ir -S %s | FileCheck %s
+
+; Make sure SCEV is not queried while the IR is temporarily invalid.
+
+define void @pr49538() {
+; CHECK-LABEL: define void @pr49538() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_0:.*]]
+; CHECK:       [[LOOP_0]]:
+; CHECK-NEXT:    [[IV_0:%.*]] = phi i16 [ -1, %[[ENTRY]] ], [ [[IV_0_NEXT:%.*]], %[[LOOP_0_LATCH:.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[IV_0]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i32 [[TMP0]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[N_VEC]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = add i16 -1, [[DOTCAST]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[LOOP_0_LATCH]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ -1, %[[LOOP_0]] ]
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP_1]] ]
+; CHECK-NEXT:    [[IV_1_NEXT]] = add nsw i16 [[IV_1]], 1
+; CHECK-NEXT:    [[I6:%.*]] = icmp eq i16 [[IV_1_NEXT]], [[IV_0]]
+; CHECK-NEXT:    br i1 [[I6]], label %[[LOOP_0_LATCH]], label %[[LOOP_1]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[LOOP_0_LATCH]]:
+; CHECK-NEXT:    [[I8:%.*]] = phi i16 [ 1, %[[LOOP_1]] ], [ 1, %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[IV_0_NEXT]] = add nsw i16 [[IV_0]], 1
+; CHECK-NEXT:    [[EC_0:%.*]] = icmp eq i16 [[IV_0_NEXT]], [[I8]]
+; CHECK-NEXT:    br i1 [[EC_0]], label %[[EXIT:.*]], label %[[LOOP_0]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.0
+
+loop.0:
+  %iv.0 = phi i16 [ -1, %entry ], [ %iv.0.next, %loop.0.latch ]
+  br label %loop.1
+
+loop.1:
+  %iv.1 = phi i16 [ -1, %loop.0 ], [ %iv.1.next, %loop.1 ]
+  %iv.1.next = add nsw i16 %iv.1, 1
+  %i6 = icmp eq i16 %iv.1.next, %iv.0
+  br i1 %i6, label %loop.0.latch, label %loop.1
+
+loop.0.latch:
+  %i8 = phi i16 [ 1, %loop.1 ]
+  %iv.0.next = add nsw i16 %iv.0, 1
+  %ec.0 = icmp eq i16 %iv.0.next, %i8
+  br i1 %ec.0, label %exit, label %loop.0
+
+exit:
+  ret void
+}
+
+define void @pr49900(i32 %x, ptr %ptr) {
+; CHECK-LABEL: define void @pr49900(
+; CHECK-SAME: i32 [[X:%.*]], ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br label %[[LOOP_0:.*]]
+; CHECK:       [[LOOP_0]]:
+; CHECK-NEXT:    [[EC_0:%.*]] = icmp slt i32 [[X]], 0
+; CHECK-NEXT:    br i1 [[EC_0]], label %[[LOOP_0]], label %[[LOOP_1_PH:.*]]
+; CHECK:       [[LOOP_1_PH]]:
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i32 [ 0, %[[LOOP_1_PH]] ], [ [[IV_3_NEXT_LCSSA:%.*]], %[[LOOP_1_LATCH:.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[IV_1]], 12
+; CHECK-NEXT:    [[SMAX2:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP0]], i32 65537)
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[SMAX2]], -12
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[TMP1]], [[IV_1]]
+; CHECK-NEXT:    [[UMIN3:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP2]], i32 1)
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[UMIN3]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP2]], [[UMIN3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = udiv i32 [[TMP4]], 13
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK5:%.*]] = icmp ult i32 [[TMP6]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK5]], label %[[SCALAR_PH4:.*]], label %[[VECTOR_PH6:.*]]
+; CHECK:       [[VECTOR_PH6]]:
+; CHECK-NEXT:    [[N_MOD_VF7:%.*]] = urem i32 [[TMP6]], 4
+; CHECK-NEXT:    [[N_VEC8:%.*]] = sub i32 [[TMP6]], [[N_MOD_VF7]]
+; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[N_VEC8]], 13
+; CHECK-NEXT:    [[TMP8:%.*]] = add i32 [[IV_1]], [[TMP7]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY9:.*]]
+; CHECK:       [[VECTOR_BODY9]]:
+; CHECK-NEXT:    [[INDEX10:%.*]] = phi i32 [ 0, %[[VECTOR_PH6]] ], [ [[INDEX_NEXT11:%.*]], %[[VECTOR_BODY9]] ]
+; CHECK-NEXT:    [[INDEX_NEXT11]] = add nuw i32 [[INDEX10]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT11]], [[N_VEC8]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK12:.*]], label %[[VECTOR_BODY9]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK12]]:
+; CHECK-NEXT:    [[CMP_N13:%.*]] = icmp eq i32 [[TMP6]], [[N_VEC8]]
+; CHECK-NEXT:    br i1 [[CMP_N13]], label %[[LOOP_3_PH:.*]], label %[[SCALAR_PH4]]
+; CHECK:       [[SCALAR_PH4]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL14:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK12]] ], [ [[IV_1]], %[[LOOP_1]] ]
+; CHECK-NEXT:    br label %[[LOOP_2:.*]]
+; CHECK:       [[LOOP_2]]:
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL14]], %[[SCALAR_PH4]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP_2]] ]
+; CHECK-NEXT:    [[TMP54:%.*]] = add i32 [[IV_2]], 12
+; CHECK-NEXT:    [[IV_2_NEXT]] = add i32 [[IV_2]], 13
+; CHECK-NEXT:    [[EXT:%.*]] = zext i32 [[IV_2_NEXT]] to i64
+; CHECK-NEXT:    [[TMP56:%.*]] = add nuw nsw i64 [[EXT]], 1
+; CHECK-NEXT:    [[C6:%.*]] = icmp sle i32 [[TMP54]], 65536
+; CHECK-NEXT:    br i1 [[C6]], label %[[LOOP_2]], label %[[LOOP_3_PH]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[LOOP_3_PH]]:
+; CHECK-NEXT:    [[IV_2_NEXT_LCSSA:%.*]] = phi i32 [ [[IV_2_NEXT]], %[[LOOP_2]] ], [ [[TMP8]], %[[MIDDLE_BLOCK12]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[IV_1]], 26
+; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[IV_1]], 12
+; CHECK-NEXT:    [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP11]], i32 65537)
+; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[SMAX]], -12
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 [[TMP12]], [[IV_1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP13]], i32 1)
+; CHECK-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], [[UMIN]]
+; CHECK-NEXT:    [[TMP15:%.*]] = udiv i32 [[TMP14]], 13
+; CHECK-NEXT:    [[TMP16:%.*]] = add i32 [[UMIN]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = mul i32 [[TMP16]], 13
+; CHECK-NEXT:    [[TMP18:%.*]] = add i32 [[TMP10]], [[TMP17]]
+; CHECK-NEXT:    [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[TMP18]], i32 65536)
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[UMAX]], -26
+; CHECK-NEXT:    [[TMP20:%.*]] = sub i32 [[TMP19]], [[IV_1]]
+; CHECK-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP20]], [[TMP17]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP21]], i32 1)
+; CHECK-NEXT:    [[TMP22:%.*]] = add i32 [[UMIN1]], 1
+; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP20]], [[UMIN1]]
+; CHECK-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], [[TMP17]]
+; CHECK-NEXT:    [[TMP25:%.*]] = udiv i32 [[TMP24]], 13
+; CHECK-NEXT:    [[TMP26:%.*]] = add i32 [[TMP22]], [[TMP25]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP26]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP26]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP26]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP27:%.*]] = mul i32 [[N_VEC]], 13
+; CHECK-NEXT:    [[TMP28:%.*]] = add i32 [[IV_2_NEXT_LCSSA]], [[TMP27]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP29:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP26]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[LOOP_1_LATCH]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP28]], %[[MIDDLE_BLOCK]] ], [ [[IV_2_NEXT_LCSSA]], %[[LOOP_3_PH]] ]
+; CHECK-NEXT:    br label %[[LOOP_3:.*]]
+; CHECK:       [[LOOP_3]]:
+; CHECK-NEXT:    [[IV_3:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_3_NEXT:%.*]], %[[LOOP_3]] ]
+; CHECK-NEXT:    [[IV_3_NEXT]] = add i32 [[IV_3]], 13
+; CHECK-NEXT:    [[C1:%.*]] = icmp ult i32 [[IV_3_NEXT]], 65536
+; CHECK-NEXT:    br i1 [[C1]], label %[[LOOP_3]], label %[[LOOP_1_LATCH]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[LOOP_1_LATCH]]:
+; CHECK-NEXT:    [[IV_3_NEXT_LCSSA]] = phi i32 [ [[IV_3_NEXT]], %[[LOOP_3]] ], [ [[TMP28]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[IV_1]], 9999
+; CHECK-NEXT:    br i1 [[EC]], label %[[LOOP_1]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.0
+
+loop.0:                                              ; preds = %bb2, %bb
+  %ec.0 = icmp slt i32 %x, 0
+  br i1 %ec.0, label %loop.0, label %loop.1.ph
+
+loop.1.ph:                                              ; preds = %bb2
+  br label %loop.1
+
+loop.1:                                             ; preds = %bb33, %bb5
+  %iv.1 = phi i32 [ 0, %loop.1.ph ], [ %iv.3.next, %loop.1.latch ]
+  br label %loop.2
+
+loop.2:
+  %iv.2 = phi i32 [ %iv.1, %loop.1 ], [ %iv.2.next, %loop.2 ]
+  %tmp54 = add i32 %iv.2, 12
+  %iv.2.next = add i32 %iv.2, 13
+  %ext = zext i32 %iv.2.next to i64
+  %tmp56 = add nuw nsw i64 %ext, 1
+  %C6 = icmp sle i32 %tmp54, 65536
+  br i1 %C6, label %loop.2, label %loop.3.ph
+
+loop.3.ph:
+  br label %loop.3
+
+loop.3:
+  %iv.3 = phi i32 [ %iv.2.next, %loop.3.ph ], [ %iv.3.next, %loop.3 ]
+  %iv.3.next = add i32 %iv.3 , 13
+  %C1 = icmp ult i32 %iv.3.next, 65536
+  br i1 %C1, label %loop.3, label %loop.1.latch
+
+loop.1.latch:
+  %ec = icmp ne i32 %iv.1, 9999
+  br i1 %ec, label %loop.1, label %exit
+
+exit:
+  ret void
+}
+
+define void @pr52024(ptr %dst, i16 %N) {
+; CHECK-LABEL: define void @pr52024(
+; CHECK-SAME: ptr [[DST:%.*]], i16 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP_1:.*]]
+; CHECK:       [[LOOP_1]]:
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i16 [ 1, %[[ENTRY]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP_1_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV_1_NEXT]] = mul i16 [[IV_1]], 3
+; CHECK-NEXT:    [[EXITCOND_1:%.*]] = icmp uge i16 [[IV_1_NEXT]], 99
+; CHECK-NEXT:    br i1 [[EXITCOND_1]], label %[[LOOP_1_LATCH]], label %[[EXIT_LOOPEXIT1:.*]]
+; CHECK:       [[LOOP_1_LATCH]]:
+; CHECK-NEXT:    [[EXITCOND_2:%.*]] = icmp eq i16 [[IV_1_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND_2]], label %[[LOOP_2_PH:.*]], label %[[LOOP_1]]
+; CHECK:       [[LOOP_2_PH]]:
+; CHECK-NEXT:    [[IV_1_LCSSA2:%.*]] = phi i16 [ [[IV_1]], %[[LOOP_1_LATCH]] ]
+; CHECK-NEXT:    [[IV_1_NEXT_LCSSA:%.*]] = phi i16 [ [[IV_1_NEXT]], %[[LOOP_1_LATCH]] ]
+; CHECK-NEXT:    [[IV_1_NEXT_EXT:%.*]] = sext i16 [[IV_1_NEXT_LCSSA]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i16 [[IV_1_LCSSA2]], 3
+; CHECK-NEXT:    br label %[[LOOP_2_HEADER:.*]]
+; CHECK:       [[LOOP_2_HEADER]]:
+; CHECK-NEXT:    [[IV_1_REM:%.*]] = urem i64 100, [[IV_1_NEXT_EXT]]
+; CHECK-NEXT:    [[REM_TRUNC:%.*]] = trunc i64 [[IV_1_REM]] to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 100, [[IV_1_NEXT_EXT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i16
+; CHECK-NEXT:    [[TMP3:%.*]] = mul i16 [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add i16 [[TMP3]], -100
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP4]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i16 24, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <2 x i16> splat (i16 2), [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <2 x i16> poison, i16 [[REM_TRUNC]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT3]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <2 x i16> poison, i16 [[TMP4]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT6:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT5]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP7:%.*]] = mul <2 x i16> <i16 0, i16 1>, [[BROADCAST_SPLAT6]]
+; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i16> zeroinitializer, [[TMP7]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i16> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <2 x i16> [[VEC_IND]], [[TMP6]]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i32 8, [[INDEX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = sub <2 x i16> [[VEC_IND]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sub <2 x i16> [[STEP_ADD]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = zext <2 x i16> [[TMP8]] to <2 x i32>
+; CHECK-NEXT:    [[TMP11:%.*]] = zext <2 x i16> [[TMP9]] to <2 x i32>
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[DST]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 2
+; CHECK-NEXT:    store <2 x i32> [[TMP10]], ptr [[TMP12]], align 4
+; CHECK-NEXT:    store <2 x i32> [[TMP11]], ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i16> [[STEP_ADD]], [[TMP6]]
+; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], 24
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, %[[MIDDLE_BLOCK]] ], [ 8, %[[LOOP_2_HEADER]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL7:%.*]] = phi i16 [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_2_HEADER]] ]
+; CHECK-NEXT:    br label %[[LOOP_3:.*]]
+; CHECK:       [[LOOP_3]]:
+; CHECK-NEXT:    [[IV_3:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_3_NEXT:%.*]], %[[LOOP_3]] ]
+; CHECK-NEXT:    [[SUB_PHI:%.*]] = phi i16 [ [[BC_RESUME_VAL7]], %[[SCALAR_PH]] ], [ [[SUB:%.*]], %[[LOOP_3]] ]
+; CHECK-NEXT:    [[SUB]] = sub i16 [[SUB_PHI]], [[REM_TRUNC]]
+; CHECK-NEXT:    [[SUB_EXT:%.*]] = zext i16 [[SUB]] to i32
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr i32, ptr [[DST]], i32 [[IV_3]]
+; CHECK-NEXT:    store i32 [[SUB_EXT]], ptr [[GEP_DST]], align 4
+; CHECK-NEXT:    [[IV_3_NEXT]] = add nuw nsw i32 [[IV_3]], 1
+; CHECK-NEXT:    [[EXITCOND_3:%.*]] = icmp eq i32 [[IV_3_NEXT]], 34
+; CHECK-NEXT:    br i1 [[EXITCOND_3]], label %[[LOOP_2_LATCH:.*]], label %[[LOOP_3]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[LOOP_2_LATCH]]:
+; CHECK-NEXT:    [[SUB_LCSSA:%.*]] = phi i16 [ [[SUB]], %[[LOOP_3]] ]
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp uge i16 [[SUB_LCSSA]], 200
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP_2_HEADER]]
+; CHECK:       [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[EXIT_LOOPEXIT1]]:
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.1
+
+loop.1:
+  %iv.1 = phi i16 [ 1, %entry ], [ %iv.1.next, %loop.1.latch ]
+  %iv.1.next = mul i16 %iv.1, 3
+  %exitcond.1 = icmp uge i16 %iv.1.next, 99
+  br i1 %exitcond.1, label %loop.1.latch, label %exit
+
+loop.1.latch:
+  %exitcond.2 = icmp eq i16 %iv.1.next, %N
+  br i1 %exitcond.2, label %loop.2.ph, label %loop.1
+
+loop.2.ph:
+  %iv.1.next.lcssa = phi i16 [ %iv.1.next, %loop.1.latch ]
+  %iv.1.next.ext = sext i16 %iv.1.next.lcssa to i64
+  br label %loop.2.header
+
+loop.2.header:
+  %iv.1.rem = urem i64 100, %iv.1.next.ext
+  %rem.trunc = trunc i64 %iv.1.rem to i16
+  br label %loop.3
+
+loop.3:
+  %iv.3 = phi i32 [ 8, %loop.2.header ], [ %iv.3.next, %loop.3 ]
+  %sub.phi = phi i16 [ 0, %loop.2.header ], [ %sub, %loop.3 ]
+  %sub = sub i16 %sub.phi, %rem.trunc
+  %sub.ext = zext i16 %sub to i32
+  %gep.dst = getelementptr i32, ptr %dst, i32 %iv.3
+  store i32 %sub.ext, ptr %gep.dst
+  %iv.3.next= add nuw nsw i32 %iv.3, 1
+  %exitcond.3 = icmp eq i32 %iv.3.next, 34
+  br i1 %exitcond.3, label %loop.2.latch, label %loop.3
+
+loop.2.latch:
+  %sub.lcssa = phi i16 [ %sub, %loop.3 ]
+  %exitcond = icmp uge i16 %sub.lcssa, 200
+  br i1 %exitcond, label %exit, label %loop.2.header
+
+exit:
+  ret void
+}
+
+define void @test_expand_secv_in_entry_before_gep(ptr %dst) {
+; CHECK-LABEL: define void @test_expand_secv_in_entry_before_gep(
+; CHECK-SAME: ptr [[DST:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
+; CHECK:       [[OUTER_HEADER]]:
+; CHECK-NEXT:    [[OUTER_IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[OUTER_IV_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = mul i64 [[OUTER_IV]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[TMP0]], 112
+; CHECK-NEXT:    [[GEP_M:%.*]] = getelementptr [36 x [36 x double]], ptr [[DST]], i64 0, i64 [[OUTER_IV]]
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[OUTER_IV]], [[N_VEC]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 [[OUTER_IV]], [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, ptr [[GEP_M]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds double, ptr [[TMP3]], i32 2
+; CHECK-NEXT:    store <2 x double> zeroinitializer, ptr [[TMP3]], align 8
+; CHECK-NEXT:    store <2 x double> zeroinitializer, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP1]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[OUTER_LATCH]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[OUTER_IV]], %[[OUTER_HEADER]] ]
+; CHECK-NEXT:    br label %[[INNER:.*]]
+; CHECK:       [[INNER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[INNER]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds double, ptr [[GEP_M]], i64 [[IV]]
+; CHECK-NEXT:    store double 0.000000e+00, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 111
+; CHECK-NEXT:    br i1 [[EC]], label %[[OUTER_LATCH]], label %[[INNER]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[OUTER_LATCH]]:
+; CHECK-NEXT:    [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1
+; CHECK-NEXT:    [[C:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C]], label %[[OUTER_HEADER]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer.header
+
+outer.header:
+  %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ]
+  %gep.m = getelementptr [36 x [36 x double]], ptr %dst, i64 0, i64 %outer.iv
+  br label %inner
+
+inner:
+  %iv = phi i64 [ %outer.iv, %outer.header ], [ %iv.next, %inner ]
+  %gep = getelementptr inbounds double, ptr %gep.m, i64 %iv
+  store double 0.00, ptr %gep, align 8
+  %iv.next = add i64 %iv, 1
+  %ec = icmp eq i64 %iv, 111
+  br i1 %ec, label %outer.latch, label %inner
+
+outer.latch:
+  %outer.iv.next = add i64 %outer.iv, 1
+  %c = call i1 @cond()
+  br i1 %c, label %outer.header, label %exit
+
+exit:
+  ret void
+}
+
+declare void @cond()
diff --git a/llvm/test/Transforms/LoopVectorize/no_outside_user.ll b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
index 02f324281bf4d..1eae53559d179 100644
--- a/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
+++ b/llvm/test/Transforms/LoopVectorize/no_outside_user.ll
@@ -169,7 +169,7 @@ define i32 @test3(i32 %N)  {
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP0]], i32 4)
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[SMAX]], [[B_PROMOTED]]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 2
-; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[_LR_PH_I:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], 2
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
@@ -183,12 +183,11 @@ define i32 @test3(i32 %N)  {
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt <2 x i32> [[VEC_IND]], splat (i32 10)
-; CHECK-NEXT:    [[TMP4:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp sle <2 x i32> [[VEC_IND]], splat (i32 10)
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle <2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = select <2 x i1> [[TMP4]], <2 x i1> [[TMP5]], <2 x i1> zeroinitializer
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP6]], <2 x i32> zeroinitializer, <2 x i32> splat (i32 2)
-; CHECK-NEXT:    [[PREDPHI1:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> splat (i32 1), <2 x i32> [[PREDPHI]]
+; CHECK-NEXT:    [[PREDPHI1:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[PREDPHI]], <2 x i32> splat (i32 1)
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
@@ -196,12 +195,12 @@ define i32 @test3(i32 %N)  {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[PREDPHI1]], i32 1
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[CMP_N]], label %[[F1_EXIT_LOOPEXIT:.*]], label %[[SCALAR_PH]]
-; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[F1_EXIT_LOOPEXIT:.*]], label %[[_LR_PH_I]]
+; CHECK:       [[_LR_PH_I]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[B_PROMOTED]], %[[BB]] ]
 ; CHECK-NEXT:    br label %[[DOTLR_PH_I:.*]]
-; CHECK:       [[_LR_PH_I:.*:]]
-; CHECK-NEXT:    [[UNNAMEDTMP8:%.*]] = phi i32 [ [[UNNAMEDTMP18:%.*]], %[[BB16:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK:       [[_LR_PH_I1:.*:]]
+; CHECK-NEXT:    [[UNNAMEDTMP8:%.*]] = phi i32 [ [[UNNAMEDTMP18:%.*]], %[[BB16:.*]] ], [ [[BC_RESUME_VAL]], %[[_LR_PH_I]] ]
 ; CHECK-NEXT:    [[UNNAMEDTMP2:%.*]] = icmp sgt i32 [[UNNAMEDTMP8]], 10
 ; CHECK-NEXT:    br i1 [[UNNAMEDTMP2]], label %[[BB16]], label %[[UNNAMEDBB10:.*]]
 ; CHECK:       [[UNNAMEDBB10]]:
diff --git a/llvm/test/Transforms/LoopVectorize/noalias-scope-decl.ll b/llvm/test/Transforms/LoopVectorize/noalias-scope-decl.ll
index a8e5eb2a19dad..1fe0bf2713dd7 100644
--- a/llvm/test/Transforms/LoopVectorize/noalias-scope-decl.ll
+++ b/llvm/test/Transforms/LoopVectorize/noalias-scope-decl.ll
@@ -1,17 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-vector-interleave=2  -S | FileCheck %s
 
 define void @test1(ptr noalias nocapture %a, ptr noalias nocapture readonly %b) {
+; CHECK-LABEL: define void @test1(
+; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0:![0-9]+]])
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> [[WIDE_LOAD1]], splat (float 1.000000e+00)
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i32 4
+; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[TMP4]], align 4
+; CHECK-NEXT:    store <4 x float> [[TMP3]], ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1600
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = fcmp ogt float [[TMP7]], 1.000000e+02
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0]])
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP7]], 1.000000e+00
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store float [[ADD]], ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV]], 1599
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
-; CHECK-LABEL: @test1
-; CHECK: vector.body:
-; CHECK: @llvm.experimental.noalias.scope.decl
-; CHECK-NOT: @llvm.experimental.noalias.scope.decl
-; CHECK: for.body:
-; CHECK: @llvm.experimental.noalias.scope.decl
-; CHECK-NOT: @llvm.experimental.noalias.scope.decl
-; CHECK: ret void
 
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
@@ -35,6 +69,62 @@ declare void @llvm.experimental.noalias.scope.decl(metadata)
 %struct.data = type { ptr, ptr }
 
 define void @test2(ptr nocapture readonly %d) {
+; CHECK-LABEL: define void @test2(
+; CHECK-SAME: ptr readonly captures(none) [[D:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[B:%.*]] = getelementptr inbounds [[STRUCT_DATA:%.*]], ptr [[D]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[B]], align 8
+; CHECK-NEXT:    [[PTRINT:%.*]] = ptrtoint ptr [[TMP0]] to i64
+; CHECK-NEXT:    [[MASKEDPTR:%.*]] = and i64 [[PTRINT]], 31
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i64 [[MASKEDPTR]], 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[D]], align 8
+; CHECK-NEXT:    [[PTRINT2:%.*]] = ptrtoint ptr [[TMP1]] to i64
+; CHECK-NEXT:    [[MASKEDPTR3:%.*]] = and i64 [[PTRINT2]], 31
+; CHECK-NEXT:    [[MASKCOND4:%.*]] = icmp eq i64 [[MASKEDPTR3]], 0
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[PTRINT2]], [[PTRINT]]
+; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], 32
+; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0]])
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[WIDE_LOAD]], splat (float 1.000000e+00)
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[WIDE_LOAD1]], splat (float 1.000000e+00)
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META7:![0-9]+]])
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 4
+; CHECK-NEXT:    store <4 x float> [[TMP5]], ptr [[TMP7]], align 4
+; CHECK-NEXT:    store <4 x float> [[TMP6]], ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1600
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0]])
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[TMP10]], 1.000000e+00
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META7]])
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store float [[ADD]], ptr [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV]], 1599
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 entry:
   %b = getelementptr inbounds %struct.data, ptr %d, i64 0, i32 1
   %0 = load ptr, ptr %b, align 8
@@ -47,16 +137,6 @@ entry:
   %maskcond4 = icmp eq i64 %maskedptr3, 0
   br label %for.body
 
-; CHECK-LABEL: @test2
-; CHECK: vector.body:
-; CHECK: @llvm.experimental.noalias.scope.decl(metadata [[SCOPE0_LIST:!.*]])
-; CHECK: @llvm.experimental.noalias.scope.decl(metadata [[SCOPE4_LIST:!.*]])
-; CHECK-NOT: @llvm.experimental.noalias.scope.decl
-; CHECK: for.body:
-; CHECK: @llvm.experimental.noalias.scope.decl(metadata [[SCOPE0_LIST]])
-; CHECK: @llvm.experimental.noalias.scope.decl(metadata [[SCOPE4_LIST]])
-; CHECK-NOT: @llvm.experimental.noalias.scope.decl
-; CHECK: ret void
 
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
@@ -76,19 +156,72 @@ for.end:                                          ; preds = %for.body
 }
 
 define void @predicated_noalias_scope_decl(ptr noalias nocapture readonly %a, ptr noalias nocapture %b, i32 %n) {
-
 ; Check that the vector.body still contains a llvm.experimental.noalias.scope.decl
-
-; CHECK-LABEL: @predicated_noalias_scope_decl(
-; CHECK:   vector.body:
-; CHECK:   call void @llvm.experimental.noalias.scope.decl
-; CHECK-NOT: @llvm.experimental.noalias.scope.decl
-; CHECK:   scalar.ph:
-; CHECK-NOT: @llvm.experimental.noalias.scope.decl
-; CHECK:   if.else:
-; CHECK:   call void @llvm.experimental.noalias.scope.decl
-; CHECK-NOT: @llvm.experimental.noalias.scope.decl
-; CHECK: }
+; CHECK-LABEL: define void @predicated_noalias_scope_decl(
+; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i32 [[N:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CMP15:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT:    br i1 [[CMP15]], label %[[FOR_COND_CLEANUP:.*]], label %[[FOR_BODY_PREHEADER:.*]]
+; CHECK:       [[FOR_BODY_PREHEADER]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 8
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult <4 x i64> [[VEC_IND]], splat (i64 495616)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult <4 x i64> [[STEP_ADD]], splat (i64 495616)
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0]])
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP1]], <4 x float> splat (float 2.300000e+01), <4 x float> splat (float 4.200000e+01)
+; CHECK-NEXT:    [[PREDPHI1:%.*]] = select <4 x i1> [[TMP2]], <4 x float> splat (float 2.300000e+01), <4 x float> splat (float 4.200000e+01)
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i32 4
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <4 x float> [[PREDPHI]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[PREDPHI1]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 4
+; CHECK-NEXT:    store <4 x float> [[TMP5]], ptr [[TMP7]], align 4
+; CHECK-NEXT:    store <4 x float> [[TMP6]], ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
+; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-NEXT:    br label %[[FOR_COND_CLEANUP]]
+; CHECK:       [[FOR_COND_CLEANUP]]:
+; CHECK-NEXT:    ret void
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[IF_END5:.*]] ]
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i64 [[INDVARS_IV]], 495616
+; CHECK-NEXT:    br i1 [[CMP1]], label %[[IF_END5]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i64 [[INDVARS_IV]], 991232
+; CHECK-NEXT:    tail call void @llvm.experimental.noalias.scope.decl(metadata [[META0]])
+; CHECK-NEXT:    br label %[[IF_END5]]
+; CHECK:       [[IF_END5]]:
+; CHECK-NEXT:    [[X_0:%.*]] = phi float [ 4.200000e+01, %[[IF_ELSE]] ], [ 2.300000e+01, %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[X_0]], [[TMP10]]
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store float [[MUL]], ptr [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[CMP]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+;
 
 entry:
   %cmp15 = icmp eq i32 %n, 0
@@ -132,8 +265,18 @@ if.end5:                                          ; preds = %for.body, %if.else
 !3 = distinct !{ !3, !2 }
 !4 = !{ !3 }
 
-; CHECK: [[SCOPE0_LIST]] = !{[[SCOPE0:!.*]]}
-; CHECK: [[SCOPE0]] = distinct !{[[SCOPE0]], [[SCOPE0_DOM:!.*]]}
-; CHECK: [[SCOPE0_DOM]] = distinct !{[[SCOPE0_DOM]]}
-; CHECK: [[SCOPE4_LIST]] = !{[[SCOPE4:!.*]]}
-; CHECK: [[SCOPE4]] = distinct !{[[SCOPE4]], [[SCOPE0_DOM]]}
+;.
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]], [[META5:![0-9]+]]}
+; CHECK: [[META4]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META5]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META5]], [[META4]]}
+; CHECK: [[META7]] = !{[[META8:![0-9]+]]}
+; CHECK: [[META8]] = distinct !{[[META8]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]], [[META5]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]], [[META5]]}
+; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META5]], [[META4]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
index f0d026b322e24..a843aeb1ee8a2 100644
--- a/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -253,7 +253,7 @@ define void @pr43371() optsize {
 ;
 ; CHECK-LABEL: define void @pr43371(
 ; CHECK-SAME: ) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -275,12 +275,11 @@ define void @pr43371() optsize {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[FOR_COND_CLEANUP28:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY29:.*]]
 ; CHECK:       [[FOR_COND_CLEANUP28]]:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       [[FOR_BODY29]]:
-; CHECK-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; CHECK-NEXT:    [[I24_0170:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
 ; CHECK-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
 ; CHECK-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
 ; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
@@ -291,7 +290,7 @@ define void @pr43371() optsize {
 ;
 ; PGSO-LABEL: define void @pr43371(
 ; PGSO-SAME: ) #[[ATTR0]] {
-; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:  [[ENTRY:.*:]]
 ; PGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PGSO:       [[VECTOR_PH]]:
 ; PGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -313,12 +312,11 @@ define void @pr43371() optsize {
 ; PGSO:       [[MIDDLE_BLOCK]]:
 ; PGSO-NEXT:    br label %[[FOR_COND_CLEANUP28:.*]]
 ; PGSO:       [[SCALAR_PH]]:
-; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; PGSO-NEXT:    br label %[[FOR_BODY29:.*]]
 ; PGSO:       [[FOR_COND_CLEANUP28]]:
 ; PGSO-NEXT:    unreachable
 ; PGSO:       [[FOR_BODY29]]:
-; PGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; PGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
 ; PGSO-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
 ; PGSO-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
 ; PGSO-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
@@ -329,7 +327,7 @@ define void @pr43371() optsize {
 ;
 ; NPGSO-LABEL: define void @pr43371(
 ; NPGSO-SAME: ) #[[ATTR0]] {
-; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:  [[ENTRY:.*:]]
 ; NPGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NPGSO:       [[VECTOR_PH]]:
 ; NPGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -351,12 +349,11 @@ define void @pr43371() optsize {
 ; NPGSO:       [[MIDDLE_BLOCK]]:
 ; NPGSO-NEXT:    br label %[[FOR_COND_CLEANUP28:.*]]
 ; NPGSO:       [[SCALAR_PH]]:
-; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; NPGSO-NEXT:    br label %[[FOR_BODY29:.*]]
 ; NPGSO:       [[FOR_COND_CLEANUP28]]:
 ; NPGSO-NEXT:    unreachable
 ; NPGSO:       [[FOR_BODY29]]:
-; NPGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; NPGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
 ; NPGSO-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
 ; NPGSO-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
 ; NPGSO-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
@@ -390,7 +387,7 @@ define void @pr43371_pgso() !prof !14 {
 ;
 ; CHECK-LABEL: define void @pr43371_pgso(
 ; CHECK-SAME: ) !prof [[PROF14]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -412,12 +409,11 @@ define void @pr43371_pgso() !prof !14 {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[FOR_COND_CLEANUP28:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY29:.*]]
 ; CHECK:       [[FOR_COND_CLEANUP28]]:
 ; CHECK-NEXT:    unreachable
 ; CHECK:       [[FOR_BODY29]]:
-; CHECK-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; CHECK-NEXT:    [[I24_0170:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
 ; CHECK-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
 ; CHECK-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
 ; CHECK-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
@@ -428,7 +424,7 @@ define void @pr43371_pgso() !prof !14 {
 ;
 ; PGSO-LABEL: define void @pr43371_pgso(
 ; PGSO-SAME: ) !prof [[PROF14]] {
-; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:  [[ENTRY:.*:]]
 ; PGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PGSO:       [[VECTOR_PH]]:
 ; PGSO-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -450,12 +446,11 @@ define void @pr43371_pgso() !prof !14 {
 ; PGSO:       [[MIDDLE_BLOCK]]:
 ; PGSO-NEXT:    br label %[[FOR_COND_CLEANUP28:.*]]
 ; PGSO:       [[SCALAR_PH]]:
-; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; PGSO-NEXT:    br label %[[FOR_BODY29:.*]]
 ; PGSO:       [[FOR_COND_CLEANUP28]]:
 ; PGSO-NEXT:    unreachable
 ; PGSO:       [[FOR_BODY29]]:
-; PGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
+; PGSO-NEXT:    [[I24_0170:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[INC37:%.*]], %[[FOR_BODY29]] ]
 ; PGSO-NEXT:    [[ADD33:%.*]] = add i16 undef, [[I24_0170]]
 ; PGSO-NEXT:    [[IDXPROM34:%.*]] = zext i16 [[ADD33]] to i32
 ; PGSO-NEXT:    [[ARRAYIDX35:%.*]] = getelementptr [2592 x i16], ptr @cm_array, i32 0, i32 [[IDXPROM34]]
@@ -626,6 +621,7 @@ define i32 @pr45526_pgso() !prof !14 {
 ; NPGSO-NEXT:    br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
 ; NPGSO:       [[MIDDLE_BLOCK]]:
 ; NPGSO-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; NPGSO-NEXT:    br label %[[SCALAR_PH]]
 ; NPGSO:       [[SCALAR_PH]]:
 ; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 508, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; NPGSO-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 5, %[[ENTRY]] ]
@@ -661,7 +657,7 @@ exit:
 define void @stride1(ptr noalias %B, i32 %BStride) optsize {
 ; CHECK-LABEL: define void @stride1(
 ; CHECK-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE]], i64 0
@@ -695,10 +691,9 @@ define void @stride1(ptr noalias %B, i32 %BStride) optsize {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[FOR_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
 ; CHECK-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]
 ; CHECK-NEXT:    store i16 42, ptr [[GEPOFB]], align 4
@@ -710,7 +705,7 @@ define void @stride1(ptr noalias %B, i32 %BStride) optsize {
 ;
 ; PGSO-LABEL: define void @stride1(
 ; PGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) #[[ATTR0]] {
-; PGSO-NEXT:  [[ENTRY:.*]]:
+; PGSO-NEXT:  [[ENTRY:.*:]]
 ; PGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PGSO:       [[VECTOR_PH]]:
 ; PGSO-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE]], i64 0
@@ -744,10 +739,9 @@ define void @stride1(ptr noalias %B, i32 %BStride) optsize {
 ; PGSO:       [[MIDDLE_BLOCK]]:
 ; PGSO-NEXT:    br label %[[FOR_END:.*]]
 ; PGSO:       [[SCALAR_PH]]:
-; PGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; PGSO-NEXT:    br label %[[FOR_BODY:.*]]
 ; PGSO:       [[FOR_BODY]]:
-; PGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; PGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
 ; PGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
 ; PGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]
 ; PGSO-NEXT:    store i16 42, ptr [[GEPOFB]], align 4
@@ -759,7 +753,7 @@ define void @stride1(ptr noalias %B, i32 %BStride) optsize {
 ;
 ; NPGSO-LABEL: define void @stride1(
 ; NPGSO-SAME: ptr noalias [[B:%.*]], i32 [[BSTRIDE:%.*]]) #[[ATTR0]] {
-; NPGSO-NEXT:  [[ENTRY:.*]]:
+; NPGSO-NEXT:  [[ENTRY:.*:]]
 ; NPGSO-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NPGSO:       [[VECTOR_PH]]:
 ; NPGSO-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> poison, i32 [[BSTRIDE]], i64 0
@@ -793,10 +787,9 @@ define void @stride1(ptr noalias %B, i32 %BStride) optsize {
 ; NPGSO:       [[MIDDLE_BLOCK]]:
 ; NPGSO-NEXT:    br label %[[FOR_END:.*]]
 ; NPGSO:       [[SCALAR_PH]]:
-; NPGSO-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; NPGSO-NEXT:    br label %[[FOR_BODY:.*]]
 ; NPGSO:       [[FOR_BODY]]:
-; NPGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; NPGSO-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ], [ 0, %[[SCALAR_PH]] ]
 ; NPGSO-NEXT:    [[MULB:%.*]] = mul nsw i32 [[IV]], [[BSTRIDE]]
 ; NPGSO-NEXT:    [[GEPOFB:%.*]] = getelementptr inbounds i16, ptr [[B]], i32 [[MULB]]
 ; NPGSO-NEXT:    store i16 42, ptr [[GEPOFB]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll b/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
index 1cf410c359f06..32b1fc4455d37 100644
--- a/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
@@ -35,7 +35,7 @@ define void @test(ptr %src, i64 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i64> [[TMP2]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP4]], label [[LOOP_2_LATCH4]], label [[LOOP_32]]
-; CHECK:       loop.2.latch4:
+; CHECK:       loop.2.latch3:
 ; CHECK-NEXT:    [[TMP5]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP5]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
index 4fda9d34b9dc1..70ce7a7f33ab4 100644
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
@@ -16,7 +16,7 @@ define void @foo() {
 ; CHECK-LABEL: define void @foo() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -24,13 +24,10 @@ define void @foo() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul <vscale x 4 x i64> [[TMP4]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP6]]
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 1, [[TMP19]]
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP9]], i64 0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -52,7 +49,7 @@ define void @foo() {
 ; CHECK:       vector.latch:
 ; CHECK-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x float> [ [[TMP12]], [[INNER_LOOP1]] ]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[VEC_PHI5]], <vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
index 69931a0774883..d2c53f47a6670 100644
--- a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll
@@ -231,7 +231,6 @@ define void @non_constant_vector_expansion(i32 %0, ptr %call) {
 ; STRIDED:       vector.body:
 ; STRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ null, [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; STRIDED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP1]], 4
 ; STRIDED-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i64 0
 ; STRIDED-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; STRIDED-NEXT:    [[TMP4:%.*]] = mul <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[DOTSPLAT]]
@@ -240,6 +239,7 @@ define void @non_constant_vector_expansion(i32 %0, ptr %call) {
 ; STRIDED-NEXT:    [[TMP6:%.*]] = getelementptr ptr, ptr [[CALL:%.*]], i32 [[OFFSET_IDX]]
 ; STRIDED-NEXT:    store <4 x ptr> [[VECTOR_GEP]], ptr [[TMP6]], align 4
 ; STRIDED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; STRIDED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP1]], 4
 ; STRIDED-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP3]]
 ; STRIDED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 100
 ; STRIDED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/pr154045-dont-fold-extractelement-livein.ll b/llvm/test/Transforms/LoopVectorize/pr154045-dont-fold-extractelement-livein.ll
new file mode 100644
index 0000000000000..39baa27c8a4ff
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/pr154045-dont-fold-extractelement-livein.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=2 -force-widen-divrem-via-safe-divisor=false -S %s | FileCheck %s
+
+; Make sure we don't try to fold a Instruction::ExtractElement ir<0>, ir<0>,
+; since we can't materialize the live-in for the vector operand.
+
+define void @pr154045(ptr %p, i1 %c, i64 %x) {
+; CHECK-LABEL: define void @pr154045(
+; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]], i64 [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i1> poison, i1 [[C]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i1> [[BROADCAST_SPLATINSERT]], <2 x i1> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = xor <2 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[PRED_SREM_IF:.*]], label %[[PRED_SREM_CONTINUE:.*]]
+; CHECK:       [[PRED_SREM_IF]]:
+; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE]]
+; CHECK:       [[PRED_SREM_CONTINUE]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[PRED_SREM_IF1:.*]], label %[[PRED_SREM_CONTINUE2:.*]]
+; CHECK:       [[PRED_SREM_IF1]]:
+; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE2]]
+; CHECK:       [[PRED_SREM_CONTINUE2]]:
+; CHECK-NEXT:    store i32 0, ptr [[P]], align 4
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT:    br i1 [[C]], label %[[LATCH]], label %[[ELSE:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[REM:%.*]] = srem i64 0, [[X]]
+; CHECK-NEXT:    br label %[[LATCH]]
+; CHECK:       [[LATCH]]:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i64 [ [[REM]], %[[ELSE]] ], [ 0, %[[LOOP]] ]
+; CHECK-NEXT:    [[PHI_TRUNC:%.*]] = trunc i64 [[PHI]] to i32
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[PHI_TRUNC]], 0
+; CHECK-NEXT:    store i32 [[SHL]], ptr [[P]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 1
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+  br i1 %c, label %latch, label %else
+
+else:
+  %rem = srem i64 0, %x
+  br label %latch
+
+latch:
+  %phi = phi i64 [ %rem, %else ], [ 0, %loop ]
+  %phi.trunc = trunc i64 %phi to i32
+  %shl = shl i32 %phi.trunc, 0
+  store i32 %shl, ptr %p
+  %iv.next = add i64 %iv, 1
+  %exitcond = icmp eq i64 %iv, 1
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/pr36983-multiple-lcssa.ll b/llvm/test/Transforms/LoopVectorize/pr36983-multiple-lcssa.ll
index 2b21eb21bb947..356a344c15bd1 100644
--- a/llvm/test/Transforms/LoopVectorize/pr36983-multiple-lcssa.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr36983-multiple-lcssa.ll
@@ -5,7 +5,7 @@
 define i16 @duplicate_lcssa(i16 %val) {
 ; CHECK-LABEL: define i16 @duplicate_lcssa(
 ; CHECK-SAME: i16 [[VAL:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -22,12 +22,10 @@ define i16 @duplicate_lcssa(i16 %val) {
 ; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI1:%.*]] = extractelement <4 x i16> [[TMP0]], i32 2
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i16 [ [[VAL]], %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[RES:%.*]] = phi i16 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[IV_NEXT]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i16 [ [[VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = sub nsw i16 [[IV]], 1
 ; CHECK-NEXT:    [[EXIT_COND:%.*]] = icmp ne i16 [[IV_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll b/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
index 315ea12f75510..9824879546c5a 100644
--- a/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr44488-predication.ll
@@ -19,8 +19,7 @@ define i16 @test_true_and_false_branch_equal() {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr @v_38, align 1
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i16> [[BROADCAST_SPLAT]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = xor <2 x i1> [[TMP1]], splat (i1 true)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <2 x i16> [[BROADCAST_SPLAT]], zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[PRED_SREM_IF:%.*]], label [[PRED_SREM_CONTINUE:%.*]]
 ; CHECK:       pred.srem.if:
@@ -37,7 +36,7 @@ define i16 @test_true_and_false_branch_equal() {
 ; CHECK-NEXT:    br label [[PRED_SREM_CONTINUE2]]
 ; CHECK:       pred.srem.continue2:
 ; CHECK-NEXT:    [[TMP10:%.*]] = phi <2 x i16> [ [[TMP6]], [[PRED_SREM_CONTINUE]] ], [ [[TMP9]], [[PRED_SREM_IF1]] ]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i16> splat (i16 5786), <2 x i16> [[TMP10]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i16> [[TMP10]], <2 x i16> splat (i16 5786)
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i16> [[PREDPHI]], i32 1
 ; CHECK-NEXT:    store i16 [[TMP11]], ptr @v_39, align 1
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
@@ -46,10 +45,9 @@ define i16 @test_true_and_false_branch_equal() {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 99, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_07:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC7:%.*]], [[FOR_LATCH:%.*]] ]
+; CHECK-NEXT:    [[I_07:%.*]] = phi i16 [ 99, [[SCALAR_PH]] ], [ [[INC7:%.*]], [[FOR_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[LV:%.*]] = load i16, ptr @v_38, align 1
 ; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i16 [[LV]], 32767
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[COND_END:%.*]], label [[COND_END]]
diff --git a/llvm/test/Transforms/LoopVectorize/pr45525.ll b/llvm/test/Transforms/LoopVectorize/pr45525.ll
index 25a32d9e3e32e..5606a76d85cec 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45525.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45525.ll
@@ -1,16 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
 ; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -S | FileCheck %s
 
 ; Test case for PR45525. Checks that phi's with a single predecessor and a mask are supported.
 
 define void @main(i1 %cond, ptr %arr) {
-; CHECK-LABEL: @main(
-; CHECK-NEXT:  bb.0:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK:         br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK:         [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK:         [[TMP5:%.*]] = mul <4 x i32> [[VEC_IND]], splat (i32 3)
+; CHECK-LABEL: define void @main(
+; CHECK-SAME: i1 [[COND:%.*]], ptr [[ARR:%.*]]) {
+; CHECK-NEXT:  [[BB_0:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[COND]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul <4 x i32> [[VEC_IND]], splat (i32 3)
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i32> splat (i32 7), <4 x i32> [[TMP5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i32 [[INDEX]]
+; CHECK-NEXT:    store <4 x i32> [[PREDPHI]], ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br [[BB_4:label %.*]]
+; CHECK:       [[SCALAR_PH]]:
 ;
 bb.0:
   br label %bb.1
diff --git a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
index c044cc0edf0a3..fbd8475381eef 100644
--- a/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
@@ -59,10 +59,9 @@ define void @pr45679(ptr %A) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[RIV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[RIVPLUS1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RIV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[RIVPLUS1:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[RIV]]
 ; CHECK-NEXT:    store i32 13, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[RIVPLUS1]] = add nuw nsw i32 [[RIV]], 1
@@ -121,10 +120,9 @@ define void @pr45679(ptr %A) {
 ; VF2UF2:       middle.block:
 ; VF2UF2-NEXT:    br label [[EXIT:%.*]]
 ; VF2UF2:       scalar.ph:
-; VF2UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; VF2UF2-NEXT:    br label [[LOOP:%.*]]
 ; VF2UF2:       loop:
-; VF2UF2-NEXT:    [[RIV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[RIVPLUS1:%.*]], [[LOOP]] ]
+; VF2UF2-NEXT:    [[RIV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[RIVPLUS1:%.*]], [[LOOP]] ]
 ; VF2UF2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[RIV]]
 ; VF2UF2-NEXT:    store i32 13, ptr [[ARRAYIDX]], align 1
 ; VF2UF2-NEXT:    [[RIVPLUS1]] = add nuw nsw i32 [[RIV]], 1
@@ -178,10 +176,9 @@ define void @pr45679(ptr %A) {
 ; VF1UF4:       middle.block:
 ; VF1UF4-NEXT:    br label [[EXIT:%.*]]
 ; VF1UF4:       scalar.ph:
-; VF1UF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; VF1UF4-NEXT:    br label [[LOOP:%.*]]
 ; VF1UF4:       loop:
-; VF1UF4-NEXT:    [[RIV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[RIVPLUS1:%.*]], [[LOOP]] ]
+; VF1UF4-NEXT:    [[RIV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[RIVPLUS1:%.*]], [[LOOP]] ]
 ; VF1UF4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[RIV]]
 ; VF1UF4-NEXT:    store i32 13, ptr [[ARRAYIDX]], align 1
 ; VF1UF4-NEXT:    [[RIVPLUS1]] = add nuw nsw i32 [[RIV]], 1
@@ -258,10 +255,9 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[V:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    store i64 [[V]], ptr [[B]], align 8
@@ -325,10 +321,9 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF2UF2:       middle.block:
 ; VF2UF2-NEXT:    br label [[FOR_END:%.*]]
 ; VF2UF2:       scalar.ph:
-; VF2UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; VF2UF2-NEXT:    br label [[FOR_BODY:%.*]]
 ; VF2UF2:       for.body:
-; VF2UF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; VF2UF2-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VF2UF2-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; VF2UF2-NEXT:    [[V:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; VF2UF2-NEXT:    store i64 [[V]], ptr [[B]], align 8
@@ -387,10 +382,9 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
 ; VF1UF4:       middle.block:
 ; VF1UF4-NEXT:    br label [[FOR_END:%.*]]
 ; VF1UF4:       scalar.ph:
-; VF1UF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; VF1UF4-NEXT:    br label [[FOR_BODY:%.*]]
 ; VF1UF4:       for.body:
-; VF1UF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; VF1UF4-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; VF1UF4-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
 ; VF1UF4-NEXT:    [[V:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
 ; VF1UF4-NEXT:    store i64 [[V]], ptr [[B]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll b/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll
index d4a6aed472832..7d0b5d6206383 100644
--- a/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr46525-expander-insertpoint.ll
@@ -33,10 +33,9 @@ define void @test(i16 %x, i64 %y, ptr %ptr) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    store i32 0, ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[V2:%.*]] = trunc i64 [[IV]] to i8
 ; CHECK-NEXT:    [[V3:%.*]] = add i8 [[V2]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll b/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
index 77794dcb9369d..edc32c7364c10 100644
--- a/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
@@ -63,12 +63,10 @@ define dso_local i16 @reverse_interleave_load_fold_mask() optsize {
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> [[TMP26]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 41, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i16 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IVMINUS1:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi i16 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[PREVSUM:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 41, [[SCALAR_PH]] ], [ [[IVMINUS1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[PREVSUM:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IVMINUS1]] = add nsw i16 [[IV]], -1
 ; CHECK-NEXT:    [[GEPA0:%.*]] = getelementptr inbounds [40 x [4 x i16]], ptr @A, i16 0, i16 [[IVMINUS1]], i16 0
 ; CHECK-NEXT:    [[TMP29:%.*]] = load i16, ptr [[GEPA0]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll b/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
index a28bdb8384052..9f811f834818f 100644
--- a/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
@@ -23,8 +23,7 @@ define i32 @test(i32 %a, i1 %c.1, i1 %c.2 ) #0 {
 ; CHECK-NEXT:    [[TMP0:%.*]] = add <2 x i32> [[VEC_PHI]], splat (i32 10)
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i32> [[TMP0]], splat (i32 20)
 ; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP5]], <2 x i32> splat (i32 9), <2 x i32> splat (i32 9)
-; CHECK-NEXT:    [[PREDPHI5:%.*]] = select <2 x i1> [[BROADCAST_SPLAT4]], <2 x i32> [[VEC_IND]], <2 x i32> [[PREDPHI]]
+; CHECK-NEXT:    [[PREDPHI5:%.*]] = select <2 x i1> [[BROADCAST_SPLAT4]], <2 x i32> [[VEC_IND]], <2 x i32> splat (i32 9)
 ; CHECK-NEXT:    [[PREDPHI6:%.*]] = select <2 x i1> [[TMP5]], <2 x i32> [[TMP0]], <2 x i32> [[TMP3]]
 ; CHECK-NEXT:    [[PREDPHI7]] = select <2 x i1> [[BROADCAST_SPLAT4]], <2 x i32> [[VEC_PHI]], <2 x i32> [[PREDPHI6]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
@@ -36,12 +35,10 @@ define i32 @test(i32 %a, i1 %c.1, i1 %c.2 ) #0 {
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i32> [[PREDPHI5]], i32 1
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 6, [[BB:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 35902, [[BB]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
-; CHECK-NEXT:    [[V_2:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[P_2:%.*]], [[LOOP_LATCH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 6, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[V_2:%.*]] = phi i32 [ 35902, [[SCALAR_PH]] ], [ [[P_2:%.*]], [[LOOP_LATCH]] ]
 ; CHECK-NEXT:    br i1 [[C_2]], label [[LOOP_LATCH]], label [[BODY_1:%.*]]
 ; CHECK:       body.1:
 ; CHECK-NEXT:    [[V_2_ADD:%.*]] = add i32 [[V_2]], 10
diff --git a/llvm/test/Transforms/LoopVectorize/pr66616.ll b/llvm/test/Transforms/LoopVectorize/pr66616.ll
index a39fd471080ad..59c64764b9014 100644
--- a/llvm/test/Transforms/LoopVectorize/pr66616.ll
+++ b/llvm/test/Transforms/LoopVectorize/pr66616.ll
@@ -23,10 +23,9 @@ define void @pr66616(ptr %ptr) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
 ; CHECK-NEXT:    br label [[PREHEADER:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_1:%.*]]
 ; CHECK:       loop.1:
-; CHECK-NEXT:    [[IV_1:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[LOOP_1]] ]
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[INC:%.*]], [[LOOP_1]] ]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[ADD3:%.*]] = add i32 [[LOAD]], 1
 ; CHECK-NEXT:    [[INC]] = add i8 [[IV_1]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/predicate-switch.ll
index 724aed888add8..f59d4aa99918f 100644
--- a/llvm/test/Transforms/LoopVectorize/predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicate-switch.ll
@@ -338,21 +338,21 @@ define void @switch_to_header(ptr %start) {
 ; IC1-NEXT:  [[ENTRY:.*]]:
 ; IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; IC1:       [[LOOP_HEADER]]:
-; IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[IF_THEN1:.*]] ]
+; IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[IF_THEN:.*]] ]
 ; IC1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; IC1-NEXT:    switch i64 [[IV]], label %[[LOOP_LATCH:.*]] [
-; IC1-NEXT:      i64 120, label %[[IF_THEN1]]
+; IC1-NEXT:      i64 120, label %[[IF_THEN]]
 ; IC1-NEXT:      i64 100, label %[[LOOP_LATCH]]
 ; IC1-NEXT:    ]
-; IC1:       [[IF_THEN1]]:
+; IC1:       [[IF_THEN]]:
 ; IC1-NEXT:    br label %[[LOOP_HEADER]]
-; IC1:       [[IF_THEN:.*:]]
+; IC1:       [[IF_THEN1:.*:]]
 ; IC1-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[START]], i64 poison
 ; IC1-NEXT:    store i64 42, ptr [[GEP]], align 1
 ; IC1-NEXT:    unreachable
 ; IC1:       [[LOOP_LATCH]]:
 ; IC1-NEXT:    [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; IC1-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[IF_THEN1]]
+; IC1-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[IF_THEN]]
 ; IC1:       [[EXIT]]:
 ; IC1-NEXT:    ret void
 ;
@@ -361,21 +361,21 @@ define void @switch_to_header(ptr %start) {
 ; IC2-NEXT:  [[ENTRY:.*]]:
 ; IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; IC2:       [[LOOP_HEADER]]:
-; IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[IF_THEN1:.*]] ]
+; IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[IF_THEN:.*]] ]
 ; IC2-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; IC2-NEXT:    switch i64 [[IV]], label %[[LOOP_LATCH:.*]] [
-; IC2-NEXT:      i64 120, label %[[IF_THEN1]]
+; IC2-NEXT:      i64 120, label %[[IF_THEN]]
 ; IC2-NEXT:      i64 100, label %[[LOOP_LATCH]]
 ; IC2-NEXT:    ]
-; IC2:       [[IF_THEN1]]:
+; IC2:       [[IF_THEN]]:
 ; IC2-NEXT:    br label %[[LOOP_HEADER]]
-; IC2:       [[IF_THEN:.*:]]
+; IC2:       [[IF_THEN1:.*:]]
 ; IC2-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[START]], i64 poison
 ; IC2-NEXT:    store i64 42, ptr [[GEP]], align 1
 ; IC2-NEXT:    unreachable
 ; IC2:       [[LOOP_LATCH]]:
 ; IC2-NEXT:    [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 100
-; IC2-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[IF_THEN1]]
+; IC2-NEXT:    br i1 [[CMP]], label %[[EXIT:.*]], label %[[IF_THEN]]
 ; IC2:       [[EXIT]]:
 ; IC2-NEXT:    ret void
 ;
@@ -406,7 +406,7 @@ exit:
 define void @switch_all_to_default(ptr %start) {
 ; IC1-LABEL: define void @switch_all_to_default(
 ; IC1-SAME: ptr [[START:%.*]]) {
-; IC1-NEXT:  [[ENTRY:.*]]:
+; IC1-NEXT:  [[ENTRY:.*:]]
 ; IC1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC1:       [[VECTOR_PH]]:
 ; IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -420,10 +420,9 @@ define void @switch_all_to_default(ptr %start) {
 ; IC1:       [[MIDDLE_BLOCK]]:
 ; IC1-NEXT:    br label %[[EXIT:.*]]
 ; IC1:       [[SCALAR_PH]]:
-; IC1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; IC1:       [[LOOP_HEADER]]:
-; IC1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; IC1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; IC1-NEXT:    switch i64 [[IV]], label %[[LOOP_LATCH]] [
 ; IC1-NEXT:      i64 120, label %[[LOOP_LATCH]]
@@ -439,7 +438,7 @@ define void @switch_all_to_default(ptr %start) {
 ;
 ; IC2-LABEL: define void @switch_all_to_default(
 ; IC2-SAME: ptr [[START:%.*]]) {
-; IC2-NEXT:  [[ENTRY:.*]]:
+; IC2-NEXT:  [[ENTRY:.*:]]
 ; IC2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IC2:       [[VECTOR_PH]]:
 ; IC2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -455,10 +454,9 @@ define void @switch_all_to_default(ptr %start) {
 ; IC2:       [[MIDDLE_BLOCK]]:
 ; IC2-NEXT:    br label %[[EXIT:.*]]
 ; IC2:       [[SCALAR_PH]]:
-; IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; IC2:       [[LOOP_HEADER]]:
-; IC2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; IC2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; IC2-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; IC2-NEXT:    switch i64 [[IV]], label %[[LOOP_LATCH]] [
 ; IC2-NEXT:      i64 120, label %[[LOOP_LATCH]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
index ffe118b047064..d93042b45e743 100644
--- a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
@@ -4,7 +4,7 @@
 define void @loop_invariant_store(ptr %p, i64 %a, i8 %b) {
 ; CHECK-LABEL: define void @loop_invariant_store(
 ; CHECK-SAME: ptr [[P:%.*]], i64 [[A:%.*]], i8 [[B:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[B]], i64 0
@@ -60,10 +60,9 @@ define void @loop_invariant_store(ptr %p, i64 %a, i8 %b) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[ADD]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[CMP_SLT:%.*]] = icmp slt i32 [[IV]], 2
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[A]], 48
@@ -114,7 +113,7 @@ exit:                                             ; preds = %loop.latch
 define void @loop_invariant_srem(ptr %p, i64 %a, i8 %b) {
 ; CHECK-LABEL: define void @loop_invariant_srem(
 ; CHECK-SAME: ptr [[P:%.*]], i64 [[A:%.*]], i8 [[B:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[B]], i64 0
@@ -178,10 +177,9 @@ define void @loop_invariant_srem(ptr %p, i64 %a, i8 %b) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i8 [[IV]], 1
 ; CHECK-NEXT:    [[CMP_SLT:%.*]] = icmp slt i8 [[IV]], 2
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[A]], 48
@@ -236,7 +234,7 @@ exit:                                             ; preds = %loop.latch
 define void @loop_invariant_float_store(ptr %p, i32 %a) {
 ; CHECK-LABEL: define void @loop_invariant_float_store(
 ; CHECK-SAME: ptr [[P:%.*]], i32 [[A:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = sitofp i32 [[A]] to float
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -250,10 +248,9 @@ define void @loop_invariant_float_store(ptr %p, i32 %a) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[CMP_SLT:%.*]] = icmp slt i32 [[IV]], 2
 ; CHECK-NEXT:    br i1 [[CMP_SLT]], label %[[COND_FALSE:.*]], label %[[LOOP_LATCH]]
@@ -292,7 +289,7 @@ exit:                                             ; preds = %loop.latch
 define void @test_store_to_invariant_address_needs_mask_due_to_low_trip_count(ptr %dst) {
 ; CHECK-LABEL: define void @test_store_to_invariant_address_needs_mask_due_to_low_trip_count(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -321,10 +318,9 @@ define void @test_store_to_invariant_address_needs_mask_due_to_low_trip_count(pt
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 true, label %[[LOOP_LATCH]], label %[[ELSE:.*]]
 ; CHECK:       [[ELSE]]:
 ; CHECK-NEXT:    br label %[[LOOP_LATCH]]
diff --git a/llvm/test/Transforms/LoopVectorize/preserve-dbg-loc-and-loop-metadata.ll b/llvm/test/Transforms/LoopVectorize/preserve-dbg-loc-and-loop-metadata.ll
index 1bca3866ab9f3..372c703f4cb23 100644
--- a/llvm/test/Transforms/LoopVectorize/preserve-dbg-loc-and-loop-metadata.ll
+++ b/llvm/test/Transforms/LoopVectorize/preserve-dbg-loc-and-loop-metadata.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt < %s -passes=loop-vectorize -force-vector-width=4 -force-widen-divrem-via-safe-divisor=0 -S 2>&1 | FileCheck %s
 ; RUN: opt < %s -passes=debugify,loop-vectorize -force-vector-width=4 -force-widen-divrem-via-safe-divisor=0 -S | FileCheck %s -check-prefix DEBUGLOC
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,9 +10,73 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; Check that the phi to resume the scalar part of the loop
 ; has Debug Location.
 define void @_Z3fooPf(ptr %a) {
+; CHECK-LABEL: define void @_Z3fooPf(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)
+; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[FOR_END:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[P:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[P]], 2.000000e+00
+; CHECK-NEXT:    store float [[MUL]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
+;
 ; DEBUGLOC-LABEL: define void @_Z3fooPf(
-; DEBUGLOC: scalar.ph:
-; DEBUGLOC-NEXT:    %bc.resume.val = phi {{.*}} !dbg ![[RESUMELOC:[0-9]+]]
+; DEBUGLOC-SAME: ptr [[A:%.*]]) !dbg [[DBG5:![0-9]+]] {
+; DEBUGLOC-NEXT:  [[ENTRY:.*:]]
+; DEBUGLOC-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !dbg [[DBG18:![0-9]+]]
+; DEBUGLOC:       [[VECTOR_PH]]:
+; DEBUGLOC-NEXT:    br label %[[VECTOR_BODY:.*]], !dbg [[DBG18]]
+; DEBUGLOC:       [[VECTOR_BODY]]:
+; DEBUGLOC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG19:![0-9]+]]
+; DEBUGLOC-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]], !dbg [[DBG20:![0-9]+]]
+; DEBUGLOC-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4, !dbg [[DBG21:![0-9]+]]
+; DEBUGLOC-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00), !dbg [[DBG22:![0-9]+]]
+; DEBUGLOC-NEXT:    store <4 x float> [[TMP1]], ptr [[TMP0]], align 4, !dbg [[DBG23:![0-9]+]]
+; DEBUGLOC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4, !dbg [[DBG19]]
+; DEBUGLOC-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024, !dbg [[DBG24:![0-9]+]]
+; DEBUGLOC-NEXT:    br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG24]], !llvm.loop [[LOOP25:![0-9]+]]
+; DEBUGLOC:       [[MIDDLE_BLOCK]]:
+; DEBUGLOC-NEXT:    br label %[[FOR_END:.*]], !dbg [[DBG24]]
+; DEBUGLOC:       [[SCALAR_PH]]:
+; DEBUGLOC-NEXT:    br label %[[FOR_BODY:.*]], !dbg [[DBG18]]
+; DEBUGLOC:       [[FOR_BODY]]:
+; DEBUGLOC-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ], !dbg [[DBG19]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[INDVARS_IV]], [[META9:![0-9]+]], !DIExpression(), [[DBG19]])
+; DEBUGLOC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDVARS_IV]], !dbg [[DBG20]]
+; DEBUGLOC-NEXT:      #dbg_value(ptr [[ARRAYIDX]], [[META11:![0-9]+]], !DIExpression(), [[DBG20]])
+; DEBUGLOC-NEXT:    [[P:%.*]] = load float, ptr [[ARRAYIDX]], align 4, !dbg [[DBG21]]
+; DEBUGLOC-NEXT:      #dbg_value(float [[P]], [[META12:![0-9]+]], !DIExpression(), [[DBG21]])
+; DEBUGLOC-NEXT:    [[MUL:%.*]] = fmul float [[P]], 2.000000e+00, !dbg [[DBG22]]
+; DEBUGLOC-NEXT:      #dbg_value(float [[MUL]], [[META14:![0-9]+]], !DIExpression(), [[DBG22]])
+; DEBUGLOC-NEXT:    store float [[MUL]], ptr [[ARRAYIDX]], align 4, !dbg [[DBG23]]
+; DEBUGLOC-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1, !dbg [[DBG28:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[INDVARS_IV_NEXT]], [[META15:![0-9]+]], !DIExpression(), [[DBG28]])
+; DEBUGLOC-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024, !dbg [[DBG29:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(i1 [[EXITCOND]], [[META16:![0-9]+]], !DIExpression(), [[DBG29]])
+; DEBUGLOC-NEXT:    br i1 [[EXITCOND]], label %[[FOR_END]], label %[[FOR_BODY]], !dbg [[DBG24]], !llvm.loop [[LOOP30:![0-9]+]]
+; DEBUGLOC:       [[FOR_END]]:
+; DEBUGLOC-NEXT:    ret void, !dbg [[DBG31:![0-9]+]]
 ;
 entry:
   br label %for.body
@@ -31,13 +96,92 @@ for.end:                                          ; preds = %for.body
 }
 
 define void @widen_ptr_induction_dbg(ptr %start, ptr %end) {
-; DEBUGLOC-LABEL: define void @widen_ptr_induction_dbg(
-; DEBUGLOC: vector.body:
-; DEBUGLOC-NEXT: = phi i64
-; DEBUGLOC-NEXT: = phi ptr {{.+}}, !dbg ![[PTRIVLOC:[0-9]+]]
+; CHECK-LABEL: define void @widen_ptr_induction_dbg(
+; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[START2:%.*]] = ptrtoint ptr [[START]] to i64
+; CHECK-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[END1]], -8
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 8, i64 16, i64 24>
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x ptr> [[VECTOR_GEP]], i32 0
+; CHECK-NEXT:    store <4 x ptr> [[VECTOR_GEP]], ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 32
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = getelementptr inbounds ptr, ptr [[IV]], i64 1
+; CHECK-NEXT:    store ptr [[IV]], ptr [[IV]], align 1
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]]
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
 ;
-; DEBUGLOC: loop:
-; DEBUGLOC-NEXT: = phi ptr {{.+}}, !dbg ![[PTRIVLOC]]
+; DEBUGLOC-LABEL: define void @widen_ptr_induction_dbg(
+; DEBUGLOC-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) !dbg [[DBG32:![0-9]+]] {
+; DEBUGLOC-NEXT:  [[ENTRY:.*]]:
+; DEBUGLOC-NEXT:    [[START2:%.*]] = ptrtoint ptr [[START]] to i64, !dbg [[DBG37:![0-9]+]]
+; DEBUGLOC-NEXT:    [[END1:%.*]] = ptrtoint ptr [[END]] to i64, !dbg [[DBG37]]
+; DEBUGLOC-NEXT:    [[TMP0:%.*]] = add i64 [[END1]], -8, !dbg [[DBG37]]
+; DEBUGLOC-NEXT:    [[TMP1:%.*]] = sub i64 [[TMP0]], [[START2]], !dbg [[DBG37]]
+; DEBUGLOC-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3, !dbg [[DBG37]]
+; DEBUGLOC-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1, !dbg [[DBG37]]
+; DEBUGLOC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4, !dbg [[DBG37]]
+; DEBUGLOC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !dbg [[DBG37]]
+; DEBUGLOC:       [[VECTOR_PH]]:
+; DEBUGLOC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4
+; DEBUGLOC-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
+; DEBUGLOC-NEXT:    [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
+; DEBUGLOC-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]]
+; DEBUGLOC-NEXT:    br label %[[VECTOR_BODY:.*]], !dbg [[DBG37]]
+; DEBUGLOC:       [[VECTOR_BODY]]:
+; DEBUGLOC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; DEBUGLOC-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG38:![0-9]+]]
+; DEBUGLOC-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> <i64 0, i64 8, i64 16, i64 24>, !dbg [[DBG38]]
+; DEBUGLOC-NEXT:    [[TMP6:%.*]] = extractelement <4 x ptr> [[VECTOR_GEP]], i32 0, !dbg [[DBG39:![0-9]+]]
+; DEBUGLOC-NEXT:    store <4 x ptr> [[VECTOR_GEP]], ptr [[TMP6]], align 1, !dbg [[DBG39]]
+; DEBUGLOC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; DEBUGLOC-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 32, !dbg [[DBG38]]
+; DEBUGLOC-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg [[DBG40:![0-9]+]]
+; DEBUGLOC-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG40]], !llvm.loop [[LOOP41:![0-9]+]]
+; DEBUGLOC:       [[MIDDLE_BLOCK]]:
+; DEBUGLOC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]], !dbg [[DBG40]]
+; DEBUGLOC-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !dbg [[DBG40]]
+; DEBUGLOC:       [[SCALAR_PH]]:
+; DEBUGLOC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ], !dbg [[DBG38]]
+; DEBUGLOC-NEXT:    br label %[[LOOP:.*]], !dbg [[DBG37]]
+; DEBUGLOC:       [[LOOP]]:
+; DEBUGLOC-NEXT:    [[IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ], !dbg [[DBG38]]
+; DEBUGLOC-NEXT:      #dbg_value(ptr [[IV]], [[META34:![0-9]+]], !DIExpression(), [[DBG38]])
+; DEBUGLOC-NEXT:    [[IV_NEXT]] = getelementptr inbounds ptr, ptr [[IV]], i64 1, !dbg [[DBG42:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(ptr [[IV_NEXT]], [[META35:![0-9]+]], !DIExpression(), [[DBG42]])
+; DEBUGLOC-NEXT:    store ptr [[IV]], ptr [[IV]], align 1, !dbg [[DBG39]]
+; DEBUGLOC-NEXT:    [[CMP_NOT:%.*]] = icmp eq ptr [[IV_NEXT]], [[END]], !dbg [[DBG43:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(i1 [[CMP_NOT]], [[META36:![0-9]+]], !DIExpression(), [[DBG43]])
+; DEBUGLOC-NEXT:    br i1 [[CMP_NOT]], label %[[EXIT]], label %[[LOOP]], !dbg [[DBG40]], !llvm.loop [[LOOP44:![0-9]+]]
+; DEBUGLOC:       [[EXIT]]:
+; DEBUGLOC-NEXT:    ret void, !dbg [[DBG45:![0-9]+]]
 ;
 entry:
   br label %loop
@@ -54,12 +198,172 @@ exit:
 }
 
 define void @predicated_phi_dbg(i64 %n, ptr %x) {
-; DEBUGLOC-LABEL: define void @predicated_phi_dbg(
-; DEBUGLOC: pred.udiv.continue{{.+}}:
-; DEBUGLOC-NEXT:   = phi <4 x i64> {{.+}}, !dbg [[PREDPHILOC:![0-9]+]]
+; CHECK-LABEL: define void @predicated_phi_dbg(
+; CHECK-SAME: i64 [[N:%.*]], ptr [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1)
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE6:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_UDIV_CONTINUE6]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult <4 x i64> [[VEC_IND]], splat (i64 5)
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
+; CHECK-NEXT:    br i1 [[TMP1]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
+; CHECK:       [[PRED_UDIV_IF]]:
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = udiv i64 [[N]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i32 0
+; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE]]
+; CHECK:       [[PRED_UDIV_CONTINUE]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_UDIV_IF]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2:.*]]
+; CHECK:       [[PRED_UDIV_IF1]]:
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = udiv i64 [[N]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i64> [[TMP5]], i64 [[TMP8]], i32 1
+; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE2]]
+; CHECK:       [[PRED_UDIV_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = phi <4 x i64> [ [[TMP5]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], %[[PRED_UDIV_IF1]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[PRED_UDIV_IF3:.*]], label %[[PRED_UDIV_CONTINUE4:.*]]
+; CHECK:       [[PRED_UDIV_IF3]]:
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = udiv i64 [[N]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP13]], i32 2
+; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE4]]
+; CHECK:       [[PRED_UDIV_CONTINUE4]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = phi <4 x i64> [ [[TMP10]], %[[PRED_UDIV_CONTINUE2]] ], [ [[TMP14]], %[[PRED_UDIV_IF3]] ]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3
+; CHECK-NEXT:    br i1 [[TMP16]], label %[[PRED_UDIV_IF5:.*]], label %[[PRED_UDIV_CONTINUE6]]
+; CHECK:       [[PRED_UDIV_IF5]]:
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP18:%.*]] = udiv i64 [[N]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP18]], i32 3
+; CHECK-NEXT:    br label %[[PRED_UDIV_CONTINUE6]]
+; CHECK:       [[PRED_UDIV_CONTINUE6]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = phi <4 x i64> [ [[TMP15]], %[[PRED_UDIV_CONTINUE4]] ], [ [[TMP19]], %[[PRED_UDIV_IF5]] ]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP20]], <4 x i64> zeroinitializer
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i64, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP21]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
+; CHECK:       [[FOR_BODY]]:
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I]], 5
+; CHECK-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[FOR_INC]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[N]], [[I]]
+; CHECK-NEXT:    br label %[[FOR_INC]]
+; CHECK:       [[FOR_INC]]:
+; CHECK-NEXT:    [[D:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[TMP4]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[IDX:%.*]] = getelementptr i64, ptr [[X]], i64 [[I]]
+; CHECK-NEXT:    store i64 [[D]], ptr [[IDX]], align 8
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[FOR_END]]:
+; CHECK-NEXT:    ret void
 ;
-; DEBUGLOC: for.body:
-; DEBUGLOC:   %tmp4 = udiv i64 %n, %i, !dbg [[PREDPHILOC]]
+; DEBUGLOC-LABEL: define void @predicated_phi_dbg(
+; DEBUGLOC-SAME: i64 [[N:%.*]], ptr [[X:%.*]]) !dbg [[DBG46:![0-9]+]] {
+; DEBUGLOC-NEXT:  [[ENTRY:.*]]:
+; DEBUGLOC-NEXT:    [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1), !dbg [[DBG55:![0-9]+]]
+; DEBUGLOC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[SMAX]], 4, !dbg [[DBG55]]
+; DEBUGLOC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]], !dbg [[DBG55]]
+; DEBUGLOC:       [[VECTOR_PH]]:
+; DEBUGLOC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[SMAX]], 4
+; DEBUGLOC-NEXT:    [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; DEBUGLOC-NEXT:    br label %[[VECTOR_BODY:.*]], !dbg [[DBG55]]
+; DEBUGLOC:       [[VECTOR_BODY]]:
+; DEBUGLOC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE6:.*]] ], !dbg [[DBG56:![0-9]+]]
+; DEBUGLOC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_UDIV_CONTINUE6]] ], !dbg [[DBG56]]
+; DEBUGLOC-NEXT:    [[TMP0:%.*]] = icmp ult <4 x i64> [[VEC_IND]], splat (i64 5), !dbg [[DBG57:![0-9]+]]
+; DEBUGLOC-NEXT:    [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0, !dbg [[DBG57]]
+; DEBUGLOC-NEXT:    br i1 [[TMP1]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]], !dbg [[DBG57]]
+; DEBUGLOC:       [[PRED_UDIV_IF]]:
+; DEBUGLOC-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 0, !dbg [[DBG56]]
+; DEBUGLOC-NEXT:    [[TMP3:%.*]] = udiv i64 [[N]], [[TMP2]], !dbg [[DBG58:![0-9]+]]
+; DEBUGLOC-NEXT:    [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i32 0, !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    br label %[[PRED_UDIV_CONTINUE]], !dbg [[DBG57]]
+; DEBUGLOC:       [[PRED_UDIV_CONTINUE]]:
+; DEBUGLOC-NEXT:    [[TMP5:%.*]] = phi <4 x i64> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_UDIV_IF]] ], !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP0]], i32 1, !dbg [[DBG57]]
+; DEBUGLOC-NEXT:    br i1 [[TMP6]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2:.*]], !dbg [[DBG57]]
+; DEBUGLOC:       [[PRED_UDIV_IF1]]:
+; DEBUGLOC-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], 1, !dbg [[DBG56]]
+; DEBUGLOC-NEXT:    [[TMP8:%.*]] = udiv i64 [[N]], [[TMP7]], !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    [[TMP9:%.*]] = insertelement <4 x i64> [[TMP5]], i64 [[TMP8]], i32 1, !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    br label %[[PRED_UDIV_CONTINUE2]], !dbg [[DBG57]]
+; DEBUGLOC:       [[PRED_UDIV_CONTINUE2]]:
+; DEBUGLOC-NEXT:    [[TMP10:%.*]] = phi <4 x i64> [ [[TMP5]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP9]], %[[PRED_UDIV_IF1]] ], !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    [[TMP11:%.*]] = extractelement <4 x i1> [[TMP0]], i32 2, !dbg [[DBG57]]
+; DEBUGLOC-NEXT:    br i1 [[TMP11]], label %[[PRED_UDIV_IF3:.*]], label %[[PRED_UDIV_CONTINUE4:.*]], !dbg [[DBG57]]
+; DEBUGLOC:       [[PRED_UDIV_IF3]]:
+; DEBUGLOC-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 2, !dbg [[DBG56]]
+; DEBUGLOC-NEXT:    [[TMP13:%.*]] = udiv i64 [[N]], [[TMP12]], !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    [[TMP14:%.*]] = insertelement <4 x i64> [[TMP10]], i64 [[TMP13]], i32 2, !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    br label %[[PRED_UDIV_CONTINUE4]], !dbg [[DBG57]]
+; DEBUGLOC:       [[PRED_UDIV_CONTINUE4]]:
+; DEBUGLOC-NEXT:    [[TMP15:%.*]] = phi <4 x i64> [ [[TMP10]], %[[PRED_UDIV_CONTINUE2]] ], [ [[TMP14]], %[[PRED_UDIV_IF3]] ], !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    [[TMP16:%.*]] = extractelement <4 x i1> [[TMP0]], i32 3, !dbg [[DBG57]]
+; DEBUGLOC-NEXT:    br i1 [[TMP16]], label %[[PRED_UDIV_IF5:.*]], label %[[PRED_UDIV_CONTINUE6]], !dbg [[DBG57]]
+; DEBUGLOC:       [[PRED_UDIV_IF5]]:
+; DEBUGLOC-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 3, !dbg [[DBG56]]
+; DEBUGLOC-NEXT:    [[TMP18:%.*]] = udiv i64 [[N]], [[TMP17]], !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP18]], i32 3, !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    br label %[[PRED_UDIV_CONTINUE6]], !dbg [[DBG57]]
+; DEBUGLOC:       [[PRED_UDIV_CONTINUE6]]:
+; DEBUGLOC-NEXT:    [[TMP20:%.*]] = phi <4 x i64> [ [[TMP15]], %[[PRED_UDIV_CONTINUE4]] ], [ [[TMP19]], %[[PRED_UDIV_IF5]] ], !dbg [[DBG58]]
+; DEBUGLOC-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> [[TMP20]], <4 x i64> zeroinitializer, !dbg [[DBG59:![0-9]+]]
+; DEBUGLOC-NEXT:    [[TMP21:%.*]] = getelementptr i64, ptr [[X]], i64 [[INDEX]], !dbg [[DBG60:![0-9]+]]
+; DEBUGLOC-NEXT:    store <4 x i64> [[PREDPHI]], ptr [[TMP21]], align 8, !dbg [[DBG61:![0-9]+]]
+; DEBUGLOC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4, !dbg [[DBG56]]
+; DEBUGLOC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4), !dbg [[DBG56]]
+; DEBUGLOC-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg [[DBG62:![0-9]+]]
+; DEBUGLOC-NEXT:    br i1 [[TMP22]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG62]], !llvm.loop [[LOOP63:![0-9]+]]
+; DEBUGLOC:       [[MIDDLE_BLOCK]]:
+; DEBUGLOC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[SMAX]], [[N_VEC]], !dbg [[DBG62]]
+; DEBUGLOC-NEXT:    br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]], !dbg [[DBG62]]
+; DEBUGLOC:       [[SCALAR_PH]]:
+; DEBUGLOC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], !dbg [[DBG56]]
+; DEBUGLOC-NEXT:    br label %[[FOR_BODY:.*]], !dbg [[DBG55]]
+; DEBUGLOC:       [[FOR_BODY]]:
+; DEBUGLOC-NEXT:    [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[I_NEXT:%.*]], %[[FOR_INC:.*]] ], !dbg [[DBG56]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[I]], [[META48:![0-9]+]], !DIExpression(), [[DBG56]])
+; DEBUGLOC-NEXT:    [[CMP:%.*]] = icmp ult i64 [[I]], 5, !dbg [[DBG57]]
+; DEBUGLOC-NEXT:      #dbg_value(i1 [[CMP]], [[META49:![0-9]+]], !DIExpression(), [[DBG57]])
+; DEBUGLOC-NEXT:    br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[FOR_INC]], !dbg [[DBG64:![0-9]+]]
+; DEBUGLOC:       [[IF_THEN]]:
+; DEBUGLOC-NEXT:    [[TMP4:%.*]] = udiv i64 [[N]], [[I]], !dbg [[DBG58]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[TMP4]], [[META50:![0-9]+]], !DIExpression(), [[DBG58]])
+; DEBUGLOC-NEXT:    br label %[[FOR_INC]], !dbg [[DBG65:![0-9]+]]
+; DEBUGLOC:       [[FOR_INC]]:
+; DEBUGLOC-NEXT:    [[D:%.*]] = phi i64 [ 0, %[[FOR_BODY]] ], [ [[TMP4]], %[[IF_THEN]] ], !dbg [[DBG59]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[D]], [[META51:![0-9]+]], !DIExpression(), [[DBG59]])
+; DEBUGLOC-NEXT:    [[IDX:%.*]] = getelementptr i64, ptr [[X]], i64 [[I]], !dbg [[DBG60]]
+; DEBUGLOC-NEXT:      #dbg_value(ptr [[IDX]], [[META52:![0-9]+]], !DIExpression(), [[DBG60]])
+; DEBUGLOC-NEXT:    store i64 [[D]], ptr [[IDX]], align 8, !dbg [[DBG61]]
+; DEBUGLOC-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1, !dbg [[DBG66:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[I_NEXT]], [[META53:![0-9]+]], !DIExpression(), [[DBG66]])
+; DEBUGLOC-NEXT:    [[COND:%.*]] = icmp slt i64 [[I_NEXT]], [[N]], !dbg [[DBG67:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(i1 [[COND]], [[META54:![0-9]+]], !DIExpression(), [[DBG67]])
+; DEBUGLOC-NEXT:    br i1 [[COND]], label %[[FOR_BODY]], label %[[FOR_END]], !dbg [[DBG62]], !llvm.loop [[LOOP68:![0-9]+]]
+; DEBUGLOC:       [[FOR_END]]:
+; DEBUGLOC-NEXT:    ret void, !dbg [[DBG69:![0-9]+]]
 ;
 entry:
   br label %for.body
@@ -86,11 +390,96 @@ for.end:
 }
 
 define void @scalar_cast_dbg(ptr nocapture %a, i32 %start, i64 %k) {
-; DEBUGLOC-LABEL: define void @scalar_cast_dbg(
-; DEBUGLOC:   = trunc i64 %index to i32, !dbg [[CASTLOC:![0-9]+]]
+; CHECK-LABEL: define void @scalar_cast_dbg(
+; CHECK-SAME: ptr captures(none) [[A:%.*]], i32 [[START:%.*]], i64 [[K:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[K]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
+; CHECK:       [[VECTOR_SCEVCHECK]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[K]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP0]], 4294967295
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[K]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[K]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDEX]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP5]]
+; CHECK-NEXT:    store <4 x i32> [[VEC_IND]], ptr [[TMP6]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[TRUNC_IV:%.*]] = trunc i64 [[IV]] to i32
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TRUNC_IV]]
+; CHECK-NEXT:    store i32 [[TRUNC_IV]], ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[K]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
 ;
-; DEBUGLOC: loop:
-; DEBUGLOC:   %trunc.iv = trunc i64 %iv to i32, !dbg [[CASTLOC]]
+; DEBUGLOC-LABEL: define void @scalar_cast_dbg(
+; DEBUGLOC-SAME: ptr captures(none) [[A:%.*]], i32 [[START:%.*]], i64 [[K:%.*]]) !dbg [[DBG70:![0-9]+]] {
+; DEBUGLOC-NEXT:  [[ENTRY:.*]]:
+; DEBUGLOC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[K]], 4, !dbg [[DBG77:![0-9]+]]
+; DEBUGLOC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]], !dbg [[DBG77]]
+; DEBUGLOC:       [[VECTOR_SCEVCHECK]]:
+; DEBUGLOC-NEXT:    [[TMP0:%.*]] = add i64 [[K]], -1, !dbg [[DBG77]]
+; DEBUGLOC-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32, !dbg [[DBG77]]
+; DEBUGLOC-NEXT:    [[TMP2:%.*]] = icmp slt i32 [[TMP1]], 0, !dbg [[DBG77]]
+; DEBUGLOC-NEXT:    [[TMP3:%.*]] = icmp ugt i64 [[TMP0]], 4294967295, !dbg [[DBG77]]
+; DEBUGLOC-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]], !dbg [[DBG77]]
+; DEBUGLOC-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !dbg [[DBG78:![0-9]+]]
+; DEBUGLOC:       [[VECTOR_PH]]:
+; DEBUGLOC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[K]], 4
+; DEBUGLOC-NEXT:    [[N_VEC:%.*]] = sub i64 [[K]], [[N_MOD_VF]]
+; DEBUGLOC-NEXT:    br label %[[VECTOR_BODY:.*]], !dbg [[DBG77]]
+; DEBUGLOC:       [[VECTOR_BODY]]:
+; DEBUGLOC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG78]]
+; DEBUGLOC-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG79:![0-9]+]]
+; DEBUGLOC-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDEX]] to i32, !dbg [[DBG79]]
+; DEBUGLOC-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP5]], !dbg [[DBG80:![0-9]+]]
+; DEBUGLOC-NEXT:    store <4 x i32> [[VEC_IND]], ptr [[TMP6]], align 4, !dbg [[DBG81:![0-9]+]]
+; DEBUGLOC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4, !dbg [[DBG78]]
+; DEBUGLOC-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4), !dbg [[DBG79]]
+; DEBUGLOC-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg [[DBG82:![0-9]+]]
+; DEBUGLOC-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG82]], !llvm.loop [[LOOP83:![0-9]+]]
+; DEBUGLOC:       [[MIDDLE_BLOCK]]:
+; DEBUGLOC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[K]], [[N_VEC]], !dbg [[DBG82]]
+; DEBUGLOC-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !dbg [[DBG82]]
+; DEBUGLOC:       [[SCALAR_PH]]:
+; DEBUGLOC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_SCEVCHECK]] ], !dbg [[DBG78]]
+; DEBUGLOC-NEXT:    br label %[[LOOP:.*]], !dbg [[DBG77]]
+; DEBUGLOC:       [[LOOP]]:
+; DEBUGLOC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ], !dbg [[DBG78]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[IV]], [[META72:![0-9]+]], !DIExpression(), [[DBG78]])
+; DEBUGLOC-NEXT:    [[TRUNC_IV:%.*]] = trunc i64 [[IV]] to i32, !dbg [[DBG79]]
+; DEBUGLOC-NEXT:      #dbg_value(i32 [[TRUNC_IV]], [[META73:![0-9]+]], !DIExpression(), [[DBG79]])
+; DEBUGLOC-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TRUNC_IV]], !dbg [[DBG80]]
+; DEBUGLOC-NEXT:      #dbg_value(ptr [[ARRAYIDX]], [[META74:![0-9]+]], !DIExpression(), [[DBG80]])
+; DEBUGLOC-NEXT:    store i32 [[TRUNC_IV]], ptr [[ARRAYIDX]], align 4, !dbg [[DBG81]]
+; DEBUGLOC-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1, !dbg [[DBG84:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[IV_NEXT]], [[META75:![0-9]+]], !DIExpression(), [[DBG84]])
+; DEBUGLOC-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[K]], !dbg [[DBG85:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(i1 [[EXITCOND]], [[META76:![0-9]+]], !DIExpression(), [[DBG85]])
+; DEBUGLOC-NEXT:    br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !dbg [[DBG82]], !llvm.loop [[LOOP86:![0-9]+]]
+; DEBUGLOC:       [[EXIT]]:
+; DEBUGLOC-NEXT:    ret void, !dbg [[DBG87:![0-9]+]]
 ;
 entry:
   br label %loop
@@ -109,11 +498,100 @@ exit:
 }
 
 define void @widen_intrinsic_dbg(i64 %n, ptr %y, ptr %x) {
+; CHECK-LABEL: define void @widen_intrinsic_dbg(
+; CHECK-SAME: i64 [[N:%.*]], ptr [[Y:%.*]], ptr [[X:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[Y2:%.*]] = ptrtoint ptr [[Y]] to i64
+; CHECK-NEXT:    [[X1:%.*]] = ptrtoint ptr [[X]] to i64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[X1]], [[Y2]]
+; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
+; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDEX]]
+; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ]
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[GEP_Y:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 [[IV]]
+; CHECK-NEXT:    [[LOAD:%.*]] = load float, ptr [[GEP_Y]], align 4
+; CHECK-NEXT:    [[CALL:%.*]] = call float @llvm.sqrt.f32(float [[LOAD]])
+; CHECK-NEXT:    [[GEP_X:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[IV]]
+; CHECK-NEXT:    store float [[CALL]], ptr [[GEP_X]], align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
 ; DEBUGLOC-LABEL: define void @widen_intrinsic_dbg(
-; DEBUGLOC: vector.body:
-; DEBUGLOC:   = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %{{.+}}), !dbg ![[INTRINSIC_LOC:[0-9]+]]
-; DEBUGLOC: loop:
-; DEBUGLOC:   = call float @llvm.sqrt.f32(float %{{.+}}), !dbg ![[INTRINSIC_LOC]]
+; DEBUGLOC-SAME: i64 [[N:%.*]], ptr [[Y:%.*]], ptr [[X:%.*]]) !dbg [[DBG88:![0-9]+]] {
+; DEBUGLOC-NEXT:  [[ENTRY:.*]]:
+; DEBUGLOC-NEXT:    [[Y2:%.*]] = ptrtoint ptr [[Y]] to i64, !dbg [[DBG97:![0-9]+]]
+; DEBUGLOC-NEXT:    [[X1:%.*]] = ptrtoint ptr [[X]] to i64, !dbg [[DBG97]]
+; DEBUGLOC-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4, !dbg [[DBG97]]
+; DEBUGLOC-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]], !dbg [[DBG97]]
+; DEBUGLOC:       [[VECTOR_MEMCHECK]]:
+; DEBUGLOC-NEXT:    [[TMP0:%.*]] = sub i64 [[X1]], [[Y2]], !dbg [[DBG97]]
+; DEBUGLOC-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16, !dbg [[DBG97]]
+; DEBUGLOC-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]], !dbg [[DBG98:![0-9]+]]
+; DEBUGLOC:       [[VECTOR_PH]]:
+; DEBUGLOC-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; DEBUGLOC-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; DEBUGLOC-NEXT:    br label %[[VECTOR_BODY:.*]], !dbg [[DBG97]]
+; DEBUGLOC:       [[VECTOR_BODY]]:
+; DEBUGLOC-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ], !dbg [[DBG98]]
+; DEBUGLOC-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 [[INDEX]], !dbg [[DBG99:![0-9]+]]
+; DEBUGLOC-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4, !dbg [[DBG100:![0-9]+]]
+; DEBUGLOC-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD]]), !dbg [[DBG101:![0-9]+]]
+; DEBUGLOC-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[INDEX]], !dbg [[DBG102:![0-9]+]]
+; DEBUGLOC-NEXT:    store <4 x float> [[TMP2]], ptr [[TMP3]], align 4, !dbg [[DBG103:![0-9]+]]
+; DEBUGLOC-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4, !dbg [[DBG98]]
+; DEBUGLOC-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]], !dbg [[DBG104:![0-9]+]]
+; DEBUGLOC-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !dbg [[DBG104]], !llvm.loop [[LOOP105:![0-9]+]]
+; DEBUGLOC:       [[MIDDLE_BLOCK]]:
+; DEBUGLOC-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]], !dbg [[DBG104]]
+; DEBUGLOC-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]], !dbg [[DBG104]]
+; DEBUGLOC:       [[SCALAR_PH]]:
+; DEBUGLOC-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ], [ 0, %[[VECTOR_MEMCHECK]] ], !dbg [[DBG98]]
+; DEBUGLOC-NEXT:    br label %[[LOOP:.*]], !dbg [[DBG97]]
+; DEBUGLOC:       [[LOOP]]:
+; DEBUGLOC-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ], !dbg [[DBG98]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[IV]], [[META90:![0-9]+]], !DIExpression(), [[DBG98]])
+; DEBUGLOC-NEXT:    [[GEP_Y:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 [[IV]], !dbg [[DBG99]]
+; DEBUGLOC-NEXT:      #dbg_value(ptr [[GEP_Y]], [[META91:![0-9]+]], !DIExpression(), [[DBG99]])
+; DEBUGLOC-NEXT:    [[LOAD:%.*]] = load float, ptr [[GEP_Y]], align 4, !dbg [[DBG100]]
+; DEBUGLOC-NEXT:      #dbg_value(float [[LOAD]], [[META92:![0-9]+]], !DIExpression(), [[DBG100]])
+; DEBUGLOC-NEXT:    [[CALL:%.*]] = call float @llvm.sqrt.f32(float [[LOAD]]), !dbg [[DBG101]]
+; DEBUGLOC-NEXT:      #dbg_value(float [[CALL]], [[META93:![0-9]+]], !DIExpression(), [[DBG101]])
+; DEBUGLOC-NEXT:    [[GEP_X:%.*]] = getelementptr inbounds float, ptr [[X]], i64 [[IV]], !dbg [[DBG102]]
+; DEBUGLOC-NEXT:      #dbg_value(ptr [[GEP_X]], [[META94:![0-9]+]], !DIExpression(), [[DBG102]])
+; DEBUGLOC-NEXT:    store float [[CALL]], ptr [[GEP_X]], align 4, !dbg [[DBG103]]
+; DEBUGLOC-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1, !dbg [[DBG106:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(i64 [[IV_NEXT]], [[META95:![0-9]+]], !DIExpression(), [[DBG106]])
+; DEBUGLOC-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]], !dbg [[DBG107:![0-9]+]]
+; DEBUGLOC-NEXT:      #dbg_value(i1 [[EXITCOND]], [[META96:![0-9]+]], !DIExpression(), [[DBG107]])
+; DEBUGLOC-NEXT:    br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !dbg [[DBG104]], !llvm.loop [[LOOP108:![0-9]+]]
+; DEBUGLOC:       [[EXIT]]:
+; DEBUGLOC-NEXT:    ret void, !dbg [[DBG109:![0-9]+]]
 ;
 entry:
   br label %loop
@@ -135,9 +613,126 @@ exit:
 
 !0 = !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.width", i32 4}
-; CHECK-NOT: !{metadata !"llvm.loop.vectorize.width", i32 4}
-; CHECK: !{!"llvm.loop.isvectorized", i32 1}
 
-; DEBUGLOC: ![[RESUMELOC]] = !DILocation(line: 2
-; DEBUGLOC: ![[PTRIVLOC]] = !DILocation(line: 12
-; DEBUGLOC: ![[INTRINSIC_LOC]] = !DILocation(line: 44
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
+; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
+; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META1]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]}
+; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META1]]}
+;.
+; DEBUGLOC: [[META0:![0-9]+]] = distinct !DICompileUnit(language: DW_LANG_C, file: [[META1:![0-9]+]], producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+; DEBUGLOC: [[META1]] = !DIFile(filename: "{{.*}}<stdin>", directory: {{.*}})
+; DEBUGLOC: [[DBG5]] = distinct !DISubprogram(name: "_Z3fooPf", linkageName: "_Z3fooPf", scope: null, file: [[META1]], line: 1, type: [[META6:![0-9]+]], scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META8:![0-9]+]])
+; DEBUGLOC: [[META6]] = !DISubroutineType(types: [[META7:![0-9]+]])
+; DEBUGLOC: [[META7]] = !{}
+; DEBUGLOC: [[META8]] = !{[[META9]], [[META11]], [[META12]], [[META14]], [[META15]], [[META16]]}
+; DEBUGLOC: [[META9]] = !DILocalVariable(name: "1", scope: [[DBG5]], file: [[META1]], line: 2, type: [[META10:![0-9]+]])
+; DEBUGLOC: [[META10]] = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned)
+; DEBUGLOC: [[META11]] = !DILocalVariable(name: "2", scope: [[DBG5]], file: [[META1]], line: 3, type: [[META10]])
+; DEBUGLOC: [[META12]] = !DILocalVariable(name: "3", scope: [[DBG5]], file: [[META1]], line: 4, type: [[META13:![0-9]+]])
+; DEBUGLOC: [[META13]] = !DIBasicType(name: "ty32", size: 32, encoding: DW_ATE_unsigned)
+; DEBUGLOC: [[META14]] = !DILocalVariable(name: "4", scope: [[DBG5]], file: [[META1]], line: 5, type: [[META13]])
+; DEBUGLOC: [[META15]] = !DILocalVariable(name: "5", scope: [[DBG5]], file: [[META1]], line: 7, type: [[META10]])
+; DEBUGLOC: [[META16]] = !DILocalVariable(name: "6", scope: [[DBG5]], file: [[META1]], line: 8, type: [[META17:![0-9]+]])
+; DEBUGLOC: [[META17]] = !DIBasicType(name: "ty8", size: 8, encoding: DW_ATE_unsigned)
+; DEBUGLOC: [[DBG18]] = !DILocation(line: 1, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[DBG19]] = !DILocation(line: 2, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[DBG20]] = !DILocation(line: 3, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[DBG21]] = !DILocation(line: 4, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[DBG22]] = !DILocation(line: 5, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[DBG23]] = !DILocation(line: 6, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[DBG24]] = !DILocation(line: 9, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[LOOP25]] = distinct !{[[LOOP25]], [[META26:![0-9]+]], [[META27:![0-9]+]]}
+; DEBUGLOC: [[META26]] = !{!"llvm.loop.isvectorized", i32 1}
+; DEBUGLOC: [[META27]] = !{!"llvm.loop.unroll.runtime.disable"}
+; DEBUGLOC: [[DBG28]] = !DILocation(line: 7, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[DBG29]] = !DILocation(line: 8, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[LOOP30]] = distinct !{[[LOOP30]], [[META27]], [[META26]]}
+; DEBUGLOC: [[DBG31]] = !DILocation(line: 10, column: 1, scope: [[DBG5]])
+; DEBUGLOC: [[DBG32]] = distinct !DISubprogram(name: "widen_ptr_induction_dbg", linkageName: "widen_ptr_induction_dbg", scope: null, file: [[META1]], line: 11, type: [[META6]], scopeLine: 11, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META33:![0-9]+]])
+; DEBUGLOC: [[META33]] = !{[[META34]], [[META35]], [[META36]]}
+; DEBUGLOC: [[META34]] = !DILocalVariable(name: "7", scope: [[DBG32]], file: [[META1]], line: 12, type: [[META10]])
+; DEBUGLOC: [[META35]] = !DILocalVariable(name: "8", scope: [[DBG32]], file: [[META1]], line: 13, type: [[META10]])
+; DEBUGLOC: [[META36]] = !DILocalVariable(name: "9", scope: [[DBG32]], file: [[META1]], line: 15, type: [[META17]])
+; DEBUGLOC: [[DBG37]] = !DILocation(line: 11, column: 1, scope: [[DBG32]])
+; DEBUGLOC: [[DBG38]] = !DILocation(line: 12, column: 1, scope: [[DBG32]])
+; DEBUGLOC: [[DBG39]] = !DILocation(line: 14, column: 1, scope: [[DBG32]])
+; DEBUGLOC: [[DBG40]] = !DILocation(line: 16, column: 1, scope: [[DBG32]])
+; DEBUGLOC: [[LOOP41]] = distinct !{[[LOOP41]], [[META26]], [[META27]]}
+; DEBUGLOC: [[DBG42]] = !DILocation(line: 13, column: 1, scope: [[DBG32]])
+; DEBUGLOC: [[DBG43]] = !DILocation(line: 15, column: 1, scope: [[DBG32]])
+; DEBUGLOC: [[LOOP44]] = distinct !{[[LOOP44]], [[META27]], [[META26]]}
+; DEBUGLOC: [[DBG45]] = !DILocation(line: 17, column: 1, scope: [[DBG32]])
+; DEBUGLOC: [[DBG46]] = distinct !DISubprogram(name: "predicated_phi_dbg", linkageName: "predicated_phi_dbg", scope: null, file: [[META1]], line: 18, type: [[META6]], scopeLine: 18, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META47:![0-9]+]])
+; DEBUGLOC: [[META47]] = !{[[META48]], [[META49]], [[META50]], [[META51]], [[META52]], [[META53]], [[META54]]}
+; DEBUGLOC: [[META48]] = !DILocalVariable(name: "10", scope: [[DBG46]], file: [[META1]], line: 19, type: [[META10]])
+; DEBUGLOC: [[META49]] = !DILocalVariable(name: "11", scope: [[DBG46]], file: [[META1]], line: 20, type: [[META17]])
+; DEBUGLOC: [[META50]] = !DILocalVariable(name: "12", scope: [[DBG46]], file: [[META1]], line: 22, type: [[META10]])
+; DEBUGLOC: [[META51]] = !DILocalVariable(name: "13", scope: [[DBG46]], file: [[META1]], line: 24, type: [[META10]])
+; DEBUGLOC: [[META52]] = !DILocalVariable(name: "14", scope: [[DBG46]], file: [[META1]], line: 25, type: [[META10]])
+; DEBUGLOC: [[META53]] = !DILocalVariable(name: "15", scope: [[DBG46]], file: [[META1]], line: 27, type: [[META10]])
+; DEBUGLOC: [[META54]] = !DILocalVariable(name: "16", scope: [[DBG46]], file: [[META1]], line: 28, type: [[META17]])
+; DEBUGLOC: [[DBG55]] = !DILocation(line: 18, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG56]] = !DILocation(line: 19, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG57]] = !DILocation(line: 20, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG58]] = !DILocation(line: 22, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG59]] = !DILocation(line: 24, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG60]] = !DILocation(line: 25, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG61]] = !DILocation(line: 26, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG62]] = !DILocation(line: 29, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[LOOP63]] = distinct !{[[LOOP63]], [[META26]], [[META27]]}
+; DEBUGLOC: [[DBG64]] = !DILocation(line: 21, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG65]] = !DILocation(line: 23, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG66]] = !DILocation(line: 27, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG67]] = !DILocation(line: 28, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[LOOP68]] = distinct !{[[LOOP68]], [[META27]], [[META26]]}
+; DEBUGLOC: [[DBG69]] = !DILocation(line: 30, column: 1, scope: [[DBG46]])
+; DEBUGLOC: [[DBG70]] = distinct !DISubprogram(name: "scalar_cast_dbg", linkageName: "scalar_cast_dbg", scope: null, file: [[META1]], line: 31, type: [[META6]], scopeLine: 31, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META71:![0-9]+]])
+; DEBUGLOC: [[META71]] = !{[[META72]], [[META73]], [[META74]], [[META75]], [[META76]]}
+; DEBUGLOC: [[META72]] = !DILocalVariable(name: "17", scope: [[DBG70]], file: [[META1]], line: 32, type: [[META10]])
+; DEBUGLOC: [[META73]] = !DILocalVariable(name: "18", scope: [[DBG70]], file: [[META1]], line: 33, type: [[META13]])
+; DEBUGLOC: [[META74]] = !DILocalVariable(name: "19", scope: [[DBG70]], file: [[META1]], line: 34, type: [[META10]])
+; DEBUGLOC: [[META75]] = !DILocalVariable(name: "20", scope: [[DBG70]], file: [[META1]], line: 36, type: [[META10]])
+; DEBUGLOC: [[META76]] = !DILocalVariable(name: "21", scope: [[DBG70]], file: [[META1]], line: 37, type: [[META17]])
+; DEBUGLOC: [[DBG77]] = !DILocation(line: 31, column: 1, scope: [[DBG70]])
+; DEBUGLOC: [[DBG78]] = !DILocation(line: 32, column: 1, scope: [[DBG70]])
+; DEBUGLOC: [[DBG79]] = !DILocation(line: 33, column: 1, scope: [[DBG70]])
+; DEBUGLOC: [[DBG80]] = !DILocation(line: 34, column: 1, scope: [[DBG70]])
+; DEBUGLOC: [[DBG81]] = !DILocation(line: 35, column: 1, scope: [[DBG70]])
+; DEBUGLOC: [[DBG82]] = !DILocation(line: 38, column: 1, scope: [[DBG70]])
+; DEBUGLOC: [[LOOP83]] = distinct !{[[LOOP83]], [[META26]], [[META27]]}
+; DEBUGLOC: [[DBG84]] = !DILocation(line: 36, column: 1, scope: [[DBG70]])
+; DEBUGLOC: [[DBG85]] = !DILocation(line: 37, column: 1, scope: [[DBG70]])
+; DEBUGLOC: [[LOOP86]] = distinct !{[[LOOP86]], [[META26]]}
+; DEBUGLOC: [[DBG87]] = !DILocation(line: 39, column: 1, scope: [[DBG70]])
+; DEBUGLOC: [[DBG88]] = distinct !DISubprogram(name: "widen_intrinsic_dbg", linkageName: "widen_intrinsic_dbg", scope: null, file: [[META1]], line: 40, type: [[META6]], scopeLine: 40, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: [[META0]], retainedNodes: [[META89:![0-9]+]])
+; DEBUGLOC: [[META89]] = !{[[META90]], [[META91]], [[META92]], [[META93]], [[META94]], [[META95]], [[META96]]}
+; DEBUGLOC: [[META90]] = !DILocalVariable(name: "22", scope: [[DBG88]], file: [[META1]], line: 41, type: [[META10]])
+; DEBUGLOC: [[META91]] = !DILocalVariable(name: "23", scope: [[DBG88]], file: [[META1]], line: 42, type: [[META10]])
+; DEBUGLOC: [[META92]] = !DILocalVariable(name: "24", scope: [[DBG88]], file: [[META1]], line: 43, type: [[META13]])
+; DEBUGLOC: [[META93]] = !DILocalVariable(name: "25", scope: [[DBG88]], file: [[META1]], line: 44, type: [[META13]])
+; DEBUGLOC: [[META94]] = !DILocalVariable(name: "26", scope: [[DBG88]], file: [[META1]], line: 45, type: [[META10]])
+; DEBUGLOC: [[META95]] = !DILocalVariable(name: "27", scope: [[DBG88]], file: [[META1]], line: 47, type: [[META10]])
+; DEBUGLOC: [[META96]] = !DILocalVariable(name: "28", scope: [[DBG88]], file: [[META1]], line: 48, type: [[META17]])
+; DEBUGLOC: [[DBG97]] = !DILocation(line: 40, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[DBG98]] = !DILocation(line: 41, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[DBG99]] = !DILocation(line: 42, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[DBG100]] = !DILocation(line: 43, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[DBG101]] = !DILocation(line: 44, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[DBG102]] = !DILocation(line: 45, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[DBG103]] = !DILocation(line: 46, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[DBG104]] = !DILocation(line: 49, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[LOOP105]] = distinct !{[[LOOP105]], [[META26]], [[META27]]}
+; DEBUGLOC: [[DBG106]] = !DILocation(line: 47, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[DBG107]] = !DILocation(line: 48, column: 1, scope: [[DBG88]])
+; DEBUGLOC: [[LOOP108]] = distinct !{[[LOOP108]], [[META26]]}
+; DEBUGLOC: [[DBG109]] = !DILocation(line: 50, column: 1, scope: [[DBG88]])
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
index 0529d84bcc389..e7f73a82292f0 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=1 -force-vector-width=4 -prefer-inloop-reductions -S | FileCheck %s
+; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=2 -force-vector-width=4 -prefer-inloop-reductions -S | FileCheck %s --check-prefix=CHECK-INTERLEAVED
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
@@ -29,6 +30,37 @@ define i32 @reduction_sum_single(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[TMP2]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_sum_single(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP3]] = add i32 [[VEC_PHI]], [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5]] = add i32 [[VEC_PHI1]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP5]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT:    br label [[DOT_CRIT_EDGE:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVED:       ._crit_edge:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
 entry:
   br label %.lr.ph
 
@@ -82,6 +114,52 @@ define i32 @reduction_sum(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_sum(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[VEC_IND]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = add i32 [[VEC_PHI]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STEP_ADD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = add i32 [[VEC_PHI1]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = add i32 [[TMP5]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = add i32 [[TMP7]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD3]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP13]] = add i32 [[TMP9]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP15]] = add i32 [[TMP11]], [[TMP14]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8)
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP15]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    br label [[DOT_CRIT_EDGE:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-INTERLEAVED:       ._crit_edge:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
 entry:
   br label %.lr.ph
 
@@ -133,6 +211,39 @@ define i32 @reduction_sum_const(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_sum_const(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = add i32 [[VEC_PHI]], [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = add i32 [[VEC_PHI1]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add i32 [[TMP3]], 12
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = add i32 [[TMP5]], 12
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    br label [[DOT_CRIT_EDGE:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-INTERLEAVED:       ._crit_edge:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
 entry:
   br label %.lr.ph
 
@@ -187,6 +298,52 @@ define i32 @reduction_prod(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[PROD_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[PROD_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_prod(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 1, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 1, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[VEC_IND]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i32 [[VEC_PHI]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[STEP_ADD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = mul i32 [[VEC_PHI1]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = mul i32 [[TMP5]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = mul i32 [[TMP7]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD3]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP13]] = mul i32 [[TMP9]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP15]] = mul i32 [[TMP11]], [[TMP14]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8)
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = mul i32 [[TMP15]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    br label [[DOT_CRIT_EDGE:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-INTERLEAVED:       ._crit_edge:
+; CHECK-INTERLEAVED-NEXT:    [[PROD_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[PROD_0_LCSSA]]
+;
 entry:
   br label %.lr.ph
 
@@ -244,6 +401,50 @@ define i32 @reduction_mix(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_mix(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[VEC_IND]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = add i32 [[VEC_PHI]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STEP_ADD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = add i32 [[VEC_PHI1]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD3]], [[WIDE_LOAD]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP8]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP10]] = add i32 [[TMP5]], [[TMP9]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP11]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP13]] = add i32 [[TMP7]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8)
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    br label [[DOT_CRIT_EDGE:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-INTERLEAVED:       ._crit_edge:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
 entry:
   br label %.lr.ph
 
@@ -298,6 +499,45 @@ define i32 @reduction_mul(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_mul(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 19, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 1, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i32 [[VEC_PHI]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = mul i32 [[VEC_PHI1]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD3]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9]] = mul i32 [[TMP5]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11]] = mul i32 [[TMP7]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = mul i32 [[TMP11]], [[TMP9]]
+; CHECK-INTERLEAVED-NEXT:    br label [[DOT_CRIT_EDGE:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-INTERLEAVED:       ._crit_edge:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
 entry:
   br label %.lr.ph
 
@@ -349,6 +589,43 @@ define i32 @start_at_non_zero(ptr nocapture %in, ptr nocapture %coeff, ptr nocap
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @start_at_non_zero(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 120, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[IN:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COEFF:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD3]], [[WIDE_LOAD]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = add i32 [[VEC_PHI]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9]] = add i32 [[VEC_PHI1]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP9]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
 entry:
   br label %for.body
 
@@ -401,6 +678,45 @@ define i32 @reduction_and(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[RESULT_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_and(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ -1, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ -1, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = and i32 [[VEC_PHI]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = and i32 [[VEC_PHI1]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD3]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9]] = and i32 [[TMP5]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11]] = and i32 [[TMP7]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = and i32 [[TMP11]], [[TMP9]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[RESULT_0_LCSSA]]
+;
 entry:
   br label %for.body
 
@@ -452,6 +768,43 @@ define i32 @reduction_or(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[RESULT_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_or(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD3]], [[WIDE_LOAD]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = or i32 [[VEC_PHI]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9]] = or i32 [[VEC_PHI1]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = or i32 [[TMP9]], [[TMP7]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[RESULT_0_LCSSA]]
+;
 entry:
   br label %for.body
 
@@ -503,6 +856,43 @@ define i32 @reduction_xor(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[RESULT_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_xor(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD3]], [[WIDE_LOAD]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = xor i32 [[VEC_PHI]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP5]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9]] = xor i32 [[VEC_PHI1]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = xor i32 [[TMP9]], [[TMP7]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[RESULT_0_LCSSA]]
+;
 entry:
   br label %for.body
 
@@ -553,6 +943,41 @@ define float @reduction_fadd(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi float [ poison, [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[RESULT_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_fadd(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[VEC_PHI]], <4 x float> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[VEC_PHI1]], <4 x float> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP6]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[TMP4]], <4 x float> [[WIDE_LOAD3]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[TMP5]], <4 x float> [[WIDE_LOAD4]])
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = fadd fast float [[TMP7]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi float [ poison, [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret float [[RESULT_0_LCSSA]]
+;
 entry:
   br label %for.body
 
@@ -605,6 +1030,45 @@ define float @reduction_fmul(ptr nocapture %A, ptr nocapture %B) {
 ; CHECK-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi float [ poison, [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[RESULT_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_fmul(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = fmul fast float [[VEC_PHI]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = fmul fast float [[VEC_PHI1]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD3]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9]] = fmul fast float [[TMP5]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11]] = fmul fast float [[TMP7]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = fmul fast float [[TMP11]], [[TMP9]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[RESULT_0_LCSSA:%.*]] = phi float [ poison, [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret float [[RESULT_0_LCSSA]]
+;
 entry:
   br label %for.body
 
@@ -627,7 +1091,7 @@ for.end:                                          ; preds = %for.body, %entry
   ret float %result.0.lcssa
 }
 
-; Sub we can create a reduction, but not inloop
+; We can create an in-loop reduction for sub-reductions with a sub and add-reduction
 define i32 @reduction_sub_lhs(ptr noalias nocapture %A) {
 ; CHECK-LABEL: @reduction_sub_lhs(
 ; CHECK-NEXT:  entry:
@@ -636,30 +1100,61 @@ define i32 @reduction_sub_lhs(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP1:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 3, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT:    [[TMP1]] = sub <4 x i32> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP5]] = sub i32 [[VEC_PHI]], [[TMP1]]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; CHECK:       middle.block:
-; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP1]])
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       scalar.ph:
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP3]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[X_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_sub_lhs(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 3, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI2:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = sub i32 [[VEC_PHI1]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5]] = sub i32 [[VEC_PHI2]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP5]], [[TMP7]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[X_0_LCSSA]]
+;
 entry:
   br label %for.body
 
 for.body:                                         ; preds = %entry, %for.body
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
-  %x.05 = phi i32 [ %sub, %for.body ], [ 0, %entry ]
+  %x.05 = phi i32 [ %sub, %for.body ], [ 3, %entry ]
   %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
   %l0 = load i32, ptr %arrayidx, align 4
   %sub = sub nsw i32 %x.05, %l0
@@ -724,6 +1219,73 @@ define float @reduction_conditional(ptr %A, ptr %B, ptr %C, float %S) {
 ; CHECK-NEXT:    [[SUM_1_LCSSA:%.*]] = phi float [ poison, [[FOR_INC]] ], [ [[TMP13]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[SUM_1_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_conditional(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = insertelement <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[S:%.*]], i64 0
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <4 x float> [ [[TMP0]], [[VECTOR_PH]] ], [ [[PREDPHI6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x float> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PREDPHI9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP3]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD3]], splat (float 1.000000e+00)
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD4]], splat (float 1.000000e+00)
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD2]], splat (float 2.000000e+00)
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = fadd fast <4 x float> [[VEC_PHI1]], [[WIDE_LOAD4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = and <4 x i1> [[TMP5]], [[TMP7]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = and <4 x i1> [[TMP6]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = fadd fast <4 x float> [[VEC_PHI1]], [[WIDE_LOAD2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = and <4 x i1> [[TMP11]], [[TMP9]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = and <4 x i1> [[TMP19]], [[TMP5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = and <4 x i1> [[TMP12]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = and <4 x i1> [[TMP21]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP20]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[PREDPHI5:%.*]] = select <4 x i1> [[TMP15]], <4 x float> [[TMP17]], <4 x float> [[PREDPHI]]
+; CHECK-INTERLEAVED-NEXT:    [[PREDPHI6]] = select <4 x i1> [[TMP5]], <4 x float> [[PREDPHI5]], <4 x float> [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP22]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP14]]
+; CHECK-INTERLEAVED-NEXT:    [[PREDPHI8:%.*]] = select <4 x i1> [[TMP16]], <4 x float> [[TMP18]], <4 x float> [[PREDPHI7]]
+; CHECK-INTERLEAVED-NEXT:    [[PREDPHI9]] = select <4 x i1> [[TMP6]], <4 x float> [[PREDPHI8]], <4 x float> [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[PREDPHI9]], [[PREDPHI6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[IF_THEN:%.*]], label [[FOR_INC:%.*]]
+; CHECK-INTERLEAVED:       if.then:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[IF_THEN8:%.*]], label [[IF_ELSE:%.*]]
+; CHECK-INTERLEAVED:       if.then8:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_INC]]
+; CHECK-INTERLEAVED:       if.else:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[IF_THEN16:%.*]], label [[FOR_INC]]
+; CHECK-INTERLEAVED:       if.then16:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_INC]]
+; CHECK-INTERLEAVED:       for.inc:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP29:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_1_LCSSA:%.*]] = phi float [ poison, [[FOR_INC]] ], [ [[TMP24]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret float [[SUM_1_LCSSA]]
+;
 entry:
   br label %for.body
 
@@ -785,6 +1347,25 @@ define i32 @reduction_sum_multiuse(ptr noalias nocapture %A, ptr noalias nocaptu
 ; CHECK:       end:
 ; CHECK-NEXT:    ret i32 [[L10]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_sum_multiuse(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[SUM_02:%.*]] = phi i32 [ [[L10:%.*]], [[DOTLR_PH]] ], [ 0, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[L2:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-INTERLEAVED-NEXT:    [[L3:%.*]] = load i32, ptr [[L2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[L6:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[L7:%.*]] = add i32 [[SUM_02]], [[L6]]
+; CHECK-INTERLEAVED-NEXT:    [[L8:%.*]] = add i32 [[L7]], [[L3]]
+; CHECK-INTERLEAVED-NEXT:    [[L10]] = add i32 [[L8]], [[SUM_02]]
+; CHECK-INTERLEAVED-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = and i64 [[INDVARS_IV_NEXT]], 4294967295
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP0]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND]], label [[END:%.*]], label [[DOTLR_PH]]
+; CHECK-INTERLEAVED:       end:
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[L10]]
+;
 entry:
   br label %.lr.ph
 
@@ -845,6 +1426,52 @@ define i32 @reduction_predicated(ptr noalias nocapture %A, ptr noalias nocapture
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_predicated(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP15:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[VEC_IND]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = add i32 [[VEC_PHI]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STEP_ADD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = add i32 [[VEC_PHI1]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = add i32 [[TMP5]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = add i32 [[TMP7]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD3]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP13]] = add i32 [[TMP9]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP15]] = add i32 [[TMP11]], [[TMP14]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 8)
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP15]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    br label [[DOT_CRIT_EDGE:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP31:![0-9]+]]
+; CHECK-INTERLEAVED:       ._crit_edge:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ poison, [[DOTLR_PH]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
 entry:
   br label %.lr.ph
 
@@ -896,6 +1523,37 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i8 [ poison, [[DOTLR_PH]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i8 [[SUM_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_add_trunc(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i8> [ <i8 -1, i8 0, i8 0, i8 0>, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i8> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP3]] = add <4 x i8> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP4]] = add <4 x i8> [[VEC_PHI1]], [[WIDE_LOAD2]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <4 x i8> [[TMP4]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br label [[DOT_CRIT_EDGE:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP33:![0-9]+]]
+; CHECK-INTERLEAVED:       ._crit_edge:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i8 [ poison, [[DOTLR_PH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i8 [[SUM_0_LCSSA]]
+;
 entry:
   br label %.lr.ph
 
@@ -945,6 +1603,37 @@ define i8 @reduction_and_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i8 [ poison, [[DOTLR_PH]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i8 [[SUM_0_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_and_trunc(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i8> [ splat (i8 -1), [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i8> [ splat (i8 -1), [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP3]] = and <4 x i8> [[VEC_PHI]], [[WIDE_LOAD]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP4]] = and <4 x i8> [[VEC_PHI1]], [[WIDE_LOAD2]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = and <4 x i8> [[TMP4]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    br label [[DOT_CRIT_EDGE:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK-INTERLEAVED:       .lr.ph:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]], !llvm.loop [[LOOP35:![0-9]+]]
+; CHECK-INTERLEAVED:       ._crit_edge:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i8 [ poison, [[DOTLR_PH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i8 [[SUM_0_LCSSA]]
+;
 entry:
   br label %.lr.ph
 
@@ -1010,6 +1699,57 @@ define float @reduction_fmuladd(ptr %a, ptr %b, i64 %n) {
 ; CHECK-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[TMP4]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[MULADD_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_fmuladd(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8
+; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], -8
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi float [ -0.000000e+00, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = fmul <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = fmul <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7]] = fadd float [[VEC_PHI]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP5]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9]] = fadd float [[VEC_PHI1]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = fadd float [[TMP9]], [[TMP7]]
+; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP11]], float [[TMP12]], float [[SUM_07]])
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP37:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret float [[MULADD_LCSSA]]
+;
 
 entry:
   br label %for.body
@@ -1048,6 +1788,23 @@ define float @reduction_fmuladd_recurrence_first_arg(ptr %a, ptr %b, i64 %n) {
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret float [[MULADD]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_fmuladd_recurrence_first_arg(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[SUM_07]], float [[TMP0]], float [[TMP1]])
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    ret float [[MULADD]]
+;
 
 entry:
   br label %for.body
@@ -1086,6 +1843,23 @@ define float @reduction_fmuladd_recurrence_second_arg(ptr %a, ptr %b, i64 %n) {
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret float [[MULADD]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_fmuladd_recurrence_second_arg(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[SUM_07:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[MULADD:%.*]], [[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds nuw float, ptr [[A:%.*]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw float, ptr [[B:%.*]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float [[SUM_07]], float [[TMP1]])
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N:%.*]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    ret float [[MULADD]]
+;
 
 entry:
   br label %for.body
@@ -1160,6 +1934,69 @@ define float @reduction_fmuladd_blend(ptr %a, ptr %b, i64 %n, i1 %c) {
 ; CHECK-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], [[LATCH]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret float [[SUM_NEXT_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @reduction_fmuladd_blend(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 8
+; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], -8
+; CHECK-INTERLEAVED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLATINSERT]], <i1 true, i1 poison, i1 poison, i1 poison>
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[TMP0]], <4 x i1> poison, <4 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi float [ -0.000000e+00, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP2]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP5]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = fmul <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[TMP6]], <4 x float> splat (float -0.000000e+00)
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP8]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP10]] = fadd float [[VEC_PHI]], [[TMP9]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[TMP7]], <4 x float> splat (float -0.000000e+00)
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP11]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP13]] = fadd float [[VEC_PHI1]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = fadd float [[TMP13]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK-INTERLEAVED:       loop.header:
+; CHECK-INTERLEAVED-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LATCH:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[SUM:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LATCH]] ]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[C]], label [[IF:%.*]], label [[ELSE:%.*]]
+; CHECK-INTERLEAVED:       if:
+; CHECK-INTERLEAVED-NEXT:    br label [[LATCH]]
+; CHECK-INTERLEAVED:       else:
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[MULADD:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP16]], float [[TMP15]], float [[SUM]])
+; CHECK-INTERLEAVED-NEXT:    br label [[LATCH]]
+; CHECK-INTERLEAVED:       latch:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_NEXT]] = phi float [ [[SUM]], [[IF]] ], [ [[MULADD]], [[ELSE]] ]
+; CHECK-INTERLEAVED-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP39:![0-9]+]]
+; CHECK-INTERLEAVED:       exit:
+; CHECK-INTERLEAVED-NEXT:    [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], [[LATCH]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret float [[SUM_NEXT_LCSSA]]
+;
 entry:
   br label %loop.header
 
@@ -1245,6 +2082,69 @@ define i32 @predicated_not_dominates_reduction(ptr nocapture noundef readonly %h
 ; CHECK-NEXT:    [[G_1_LCSSA:%.*]] = phi i32 [ [[G_1]], [[FOR_INC5]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[G_1_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @predicated_not_dominates_reduction(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[I:%.*]], 8
+; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = and i32 [[I]], -8
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ undef, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[H:%.*]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = udiv <4 x i8> [[WIDE_LOAD]], splat (i8 31)
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = udiv <4 x i8> [[WIDE_LOAD2]], splat (i8 31)
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = shl nuw nsw <4 x i8> [[TMP3]], splat (i8 3)
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = shl nuw nsw <4 x i8> [[TMP4]], splat (i8 3)
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = udiv <4 x i8> [[TMP5]], splat (i8 31)
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = udiv <4 x i8> [[TMP6]], splat (i8 31)
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = zext nneg <4 x i8> [[TMP7]] to <4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP9]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11]] = add i32 [[VEC_PHI]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = zext nneg <4 x i8> [[TMP8]] to <4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP12]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP14]] = add i32 [[VEC_PHI1]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP14]], [[TMP11]]
+; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[I]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[FOR_END7:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ undef, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY2:%.*]]
+; CHECK-INTERLEAVED:       for.body2:
+; CHECK-INTERLEAVED-NEXT:    [[A_117:%.*]] = phi i32 [ [[INC6:%.*]], [[FOR_INC5:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVED-NEXT:    [[G_016:%.*]] = phi i32 [ [[G_1:%.*]], [[FOR_INC5]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = sext i32 [[A_117]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[H]], i64 [[TMP16]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TOBOOL3_NOT:%.*]] = icmp eq i8 [[TMP17]], 0
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TOBOOL3_NOT]], label [[FOR_INC5]], label [[IF_THEN:%.*]]
+; CHECK-INTERLEAVED:       if.then:
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = udiv i8 [[TMP17]], 31
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = shl nuw nsw i8 [[TMP18]], 3
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = udiv i8 [[TMP19]], 31
+; CHECK-INTERLEAVED-NEXT:    [[DIV4:%.*]] = zext nneg i8 [[TMP20]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[ADD:%.*]] = add nsw i32 [[G_016]], [[DIV4]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_INC5]]
+; CHECK-INTERLEAVED:       for.inc5:
+; CHECK-INTERLEAVED-NEXT:    [[G_1]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[G_016]], [[FOR_BODY2]] ]
+; CHECK-INTERLEAVED-NEXT:    [[INC6]] = add nuw nsw i32 [[A_117]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC6]], [[I]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END7]], label [[FOR_BODY2]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end7:
+; CHECK-INTERLEAVED-NEXT:    [[G_1_LCSSA:%.*]] = phi i32 [ [[G_1]], [[FOR_INC5]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[G_1_LCSSA]]
+;
 entry:
   br label %for.body2
 
@@ -1333,6 +2233,76 @@ define i32 @predicated_not_dominates_reduction_twoadd(ptr nocapture noundef read
 ; CHECK-NEXT:    [[G_1_LCSSA:%.*]] = phi i32 [ [[G_1]], [[FOR_INC5]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i32 [[G_1_LCSSA]]
 ;
+; CHECK-INTERLEAVED-LABEL: @predicated_not_dominates_reduction_twoadd(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[I:%.*]], 8
+; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = and i32 [[I]], -8
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ undef, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[H:%.*]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = udiv <4 x i8> [[WIDE_LOAD]], splat (i8 31)
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = udiv <4 x i8> [[WIDE_LOAD2]], splat (i8 31)
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = shl nuw nsw <4 x i8> [[TMP3]], splat (i8 3)
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = shl nuw nsw <4 x i8> [[TMP4]], splat (i8 3)
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = udiv <4 x i8> [[TMP5]], splat (i8 31)
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = udiv <4 x i8> [[TMP6]], splat (i8 31)
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = zext nneg <4 x i8> [[TMP7]] to <4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP9]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = add i32 [[VEC_PHI]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = zext nneg <4 x i8> [[TMP8]] to <4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP12]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = add i32 [[VEC_PHI1]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = zext nneg <4 x i8> [[TMP7]] to <4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP17]] = add i32 [[TMP11]], [[TMP16]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = zext nneg <4 x i8> [[TMP8]] to <4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP18]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP20]] = add i32 [[TMP14]], [[TMP19]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP20]], [[TMP17]]
+; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[I]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[FOR_END7:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ undef, [[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY2:%.*]]
+; CHECK-INTERLEAVED:       for.body2:
+; CHECK-INTERLEAVED-NEXT:    [[A_117:%.*]] = phi i32 [ [[INC6:%.*]], [[FOR_INC5:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVED-NEXT:    [[G_016:%.*]] = phi i32 [ [[G_1:%.*]], [[FOR_INC5]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = sext i32 [[A_117]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[H]], i64 [[TMP22]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TOBOOL3_NOT:%.*]] = icmp eq i8 [[TMP23]], 0
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TOBOOL3_NOT]], label [[FOR_INC5]], label [[IF_THEN:%.*]]
+; CHECK-INTERLEAVED:       if.then:
+; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = udiv i8 [[TMP23]], 31
+; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = shl nuw nsw i8 [[TMP24]], 3
+; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = udiv i8 [[TMP25]], 31
+; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = shl nuw nsw i8 [[TMP26]], 1
+; CHECK-INTERLEAVED-NEXT:    [[REASS_ADD:%.*]] = zext nneg i8 [[TMP27]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[ADD:%.*]] = add i32 [[G_016]], [[REASS_ADD]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_INC5]]
+; CHECK-INTERLEAVED:       for.inc5:
+; CHECK-INTERLEAVED-NEXT:    [[G_1]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[G_016]], [[FOR_BODY2]] ]
+; CHECK-INTERLEAVED-NEXT:    [[INC6]] = add nuw nsw i32 [[A_117]], 1
+; CHECK-INTERLEAVED-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC6]], [[I]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_END7]], label [[FOR_BODY2]], !llvm.loop [[LOOP43:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end7:
+; CHECK-INTERLEAVED-NEXT:    [[G_1_LCSSA:%.*]] = phi i32 [ [[G_1]], [[FOR_INC5]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[G_1_LCSSA]]
+;
 entry:
   br label %for.body2
 
@@ -1460,6 +2430,170 @@ define i32 @predicated_or_dominates_reduction(ptr %b) {
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
 ;
+; CHECK-INTERLEAVED-LABEL: @predicated_or_dominates_reduction(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE15:%.*]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ undef, [[VECTOR_PH]] ], [ [[TMP94:%.*]], [[PRED_LOAD_CONTINUE15]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP98:%.*]], [[PRED_LOAD_CONTINUE15]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = or disjoint i32 [[INDEX]], 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = or disjoint i32 [[INDEX]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = or disjoint i32 [[INDEX]], 3
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = or disjoint i32 [[INDEX]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = or disjoint i32 [[INDEX]], 5
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = or disjoint i32 [[INDEX]], 6
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = or disjoint i32 [[INDEX]], 7
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B:%.*]], i64 0, i64 [[TMP7]], i32 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP9]], i32 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP11]], i32 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP2]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP13]], i32 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP15]], i32 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP4]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP17]], i32 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = sext i32 [[TMP5]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP19]], i32 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = sext i32 [[TMP6]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP21]], i32 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = load i32, ptr [[TMP8]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = load i32, ptr [[TMP12]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = load i32, ptr [[TMP14]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = insertelement <4 x i32> poison, i32 [[TMP23]], i64 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP24]], i64 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP25]], i64 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = insertelement <4 x i32> [[TMP29]], i32 [[TMP26]], i64 3
+; CHECK-INTERLEAVED-NEXT:    [[TMP31:%.*]] = load i32, ptr [[TMP16]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP32:%.*]] = load i32, ptr [[TMP18]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP33:%.*]] = load i32, ptr [[TMP20]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP34:%.*]] = load i32, ptr [[TMP22]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP35:%.*]] = insertelement <4 x i32> poison, i32 [[TMP31]], i64 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP36:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP32]], i64 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP37:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP33]], i64 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP38:%.*]] = insertelement <4 x i32> [[TMP37]], i32 [[TMP34]], i64 3
+; CHECK-INTERLEAVED-NEXT:    [[TMP39:%.*]] = icmp eq <4 x i32> [[TMP30]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT:    [[TMP40:%.*]] = icmp eq <4 x i32> [[TMP38]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT:    [[TMP41:%.*]] = extractelement <4 x i1> [[TMP39]], i64 0
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP41]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
+; CHECK-INTERLEAVED:       pred.load.if:
+; CHECK-INTERLEAVED-NEXT:    [[TMP42:%.*]] = sext i32 [[INDEX]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP43:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP42]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP44:%.*]] = load i32, ptr [[TMP43]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP45:%.*]] = insertelement <4 x i32> poison, i32 [[TMP44]], i64 0
+; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE]]
+; CHECK-INTERLEAVED:       pred.load.continue:
+; CHECK-INTERLEAVED-NEXT:    [[TMP46:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP45]], [[PRED_LOAD_IF]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP47:%.*]] = extractelement <4 x i1> [[TMP39]], i64 1
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP47]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]]
+; CHECK-INTERLEAVED:       pred.load.if2:
+; CHECK-INTERLEAVED-NEXT:    [[TMP48:%.*]] = sext i32 [[TMP0]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP49:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP48]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP50:%.*]] = load i32, ptr [[TMP49]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP51:%.*]] = insertelement <4 x i32> [[TMP46]], i32 [[TMP50]], i64 1
+; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE3]]
+; CHECK-INTERLEAVED:       pred.load.continue3:
+; CHECK-INTERLEAVED-NEXT:    [[TMP52:%.*]] = phi <4 x i32> [ [[TMP46]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP51]], [[PRED_LOAD_IF2]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP53:%.*]] = extractelement <4 x i1> [[TMP39]], i64 2
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP53]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
+; CHECK-INTERLEAVED:       pred.load.if4:
+; CHECK-INTERLEAVED-NEXT:    [[TMP54:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP55:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP54]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP56:%.*]] = load i32, ptr [[TMP55]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP57:%.*]] = insertelement <4 x i32> [[TMP52]], i32 [[TMP56]], i64 2
+; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE5]]
+; CHECK-INTERLEAVED:       pred.load.continue5:
+; CHECK-INTERLEAVED-NEXT:    [[TMP58:%.*]] = phi <4 x i32> [ [[TMP52]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP57]], [[PRED_LOAD_IF4]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP59:%.*]] = extractelement <4 x i1> [[TMP39]], i64 3
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP59]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]]
+; CHECK-INTERLEAVED:       pred.load.if6:
+; CHECK-INTERLEAVED-NEXT:    [[TMP60:%.*]] = sext i32 [[TMP2]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP61:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP60]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP62:%.*]] = load i32, ptr [[TMP61]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP63:%.*]] = insertelement <4 x i32> [[TMP58]], i32 [[TMP62]], i64 3
+; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE7]]
+; CHECK-INTERLEAVED:       pred.load.continue7:
+; CHECK-INTERLEAVED-NEXT:    [[TMP64:%.*]] = phi <4 x i32> [ [[TMP58]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP63]], [[PRED_LOAD_IF6]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP65:%.*]] = extractelement <4 x i1> [[TMP40]], i64 0
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP65]], label [[PRED_LOAD_IF8:%.*]], label [[PRED_LOAD_CONTINUE9:%.*]]
+; CHECK-INTERLEAVED:       pred.load.if8:
+; CHECK-INTERLEAVED-NEXT:    [[TMP66:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP67:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP66]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP68:%.*]] = load i32, ptr [[TMP67]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP69:%.*]] = insertelement <4 x i32> poison, i32 [[TMP68]], i64 0
+; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE9]]
+; CHECK-INTERLEAVED:       pred.load.continue9:
+; CHECK-INTERLEAVED-NEXT:    [[TMP70:%.*]] = phi <4 x i32> [ poison, [[PRED_LOAD_CONTINUE7]] ], [ [[TMP69]], [[PRED_LOAD_IF8]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP71:%.*]] = extractelement <4 x i1> [[TMP40]], i64 1
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP71]], label [[PRED_LOAD_IF10:%.*]], label [[PRED_LOAD_CONTINUE11:%.*]]
+; CHECK-INTERLEAVED:       pred.load.if10:
+; CHECK-INTERLEAVED-NEXT:    [[TMP72:%.*]] = sext i32 [[TMP4]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP73:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP72]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP74:%.*]] = load i32, ptr [[TMP73]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP75:%.*]] = insertelement <4 x i32> [[TMP70]], i32 [[TMP74]], i64 1
+; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE11]]
+; CHECK-INTERLEAVED:       pred.load.continue11:
+; CHECK-INTERLEAVED-NEXT:    [[TMP76:%.*]] = phi <4 x i32> [ [[TMP70]], [[PRED_LOAD_CONTINUE9]] ], [ [[TMP75]], [[PRED_LOAD_IF10]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP77:%.*]] = extractelement <4 x i1> [[TMP40]], i64 2
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP77]], label [[PRED_LOAD_IF12:%.*]], label [[PRED_LOAD_CONTINUE13:%.*]]
+; CHECK-INTERLEAVED:       pred.load.if12:
+; CHECK-INTERLEAVED-NEXT:    [[TMP78:%.*]] = sext i32 [[TMP5]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP79:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP78]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP80:%.*]] = load i32, ptr [[TMP79]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP81:%.*]] = insertelement <4 x i32> [[TMP76]], i32 [[TMP80]], i64 2
+; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE13]]
+; CHECK-INTERLEAVED:       pred.load.continue13:
+; CHECK-INTERLEAVED-NEXT:    [[TMP82:%.*]] = phi <4 x i32> [ [[TMP76]], [[PRED_LOAD_CONTINUE11]] ], [ [[TMP81]], [[PRED_LOAD_IF12]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP83:%.*]] = extractelement <4 x i1> [[TMP40]], i64 3
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP83]], label [[PRED_LOAD_IF14:%.*]], label [[PRED_LOAD_CONTINUE15]]
+; CHECK-INTERLEAVED:       pred.load.if14:
+; CHECK-INTERLEAVED-NEXT:    [[TMP84:%.*]] = sext i32 [[TMP6]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TMP85:%.*]] = getelementptr inbounds [0 x %struct.e], ptr [[B]], i64 0, i64 [[TMP84]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP86:%.*]] = load i32, ptr [[TMP85]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP87:%.*]] = insertelement <4 x i32> [[TMP82]], i32 [[TMP86]], i64 3
+; CHECK-INTERLEAVED-NEXT:    br label [[PRED_LOAD_CONTINUE15]]
+; CHECK-INTERLEAVED:       pred.load.continue15:
+; CHECK-INTERLEAVED-NEXT:    [[TMP88:%.*]] = phi <4 x i32> [ [[TMP82]], [[PRED_LOAD_CONTINUE13]] ], [ [[TMP87]], [[PRED_LOAD_IF14]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP89:%.*]] = icmp ne <4 x i32> [[TMP64]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT:    [[TMP90:%.*]] = icmp ne <4 x i32> [[TMP88]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT:    [[NOT_:%.*]] = xor <4 x i1> [[TMP39]], splat (i1 true)
+; CHECK-INTERLEAVED-NEXT:    [[DOTNOT16:%.*]] = select <4 x i1> [[NOT_]], <4 x i1> splat (i1 true), <4 x i1> [[TMP89]]
+; CHECK-INTERLEAVED-NEXT:    [[NOT_18:%.*]] = xor <4 x i1> [[TMP40]], splat (i1 true)
+; CHECK-INTERLEAVED-NEXT:    [[DOTNOT17:%.*]] = select <4 x i1> [[NOT_18]], <4 x i1> splat (i1 true), <4 x i1> [[TMP90]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP91:%.*]] = bitcast <4 x i1> [[DOTNOT16]] to i4
+; CHECK-INTERLEAVED-NEXT:    [[TMP92:%.*]] = call range(i4 0, 5) i4 @llvm.ctpop.i4(i4 [[TMP91]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP93:%.*]] = zext nneg i4 [[TMP92]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[TMP94]] = add i32 [[VEC_PHI]], [[TMP93]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP95:%.*]] = bitcast <4 x i1> [[DOTNOT17]] to i4
+; CHECK-INTERLEAVED-NEXT:    [[TMP96:%.*]] = call range(i4 0, 5) i4 @llvm.ctpop.i4(i4 [[TMP95]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP97:%.*]] = zext nneg i4 [[TMP96]] to i32
+; CHECK-INTERLEAVED-NEXT:    [[TMP98]] = add i32 [[VEC_PHI1]], [[TMP97]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP99:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP99]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP98]], [[TMP94]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.cond.cleanup:
+; CHECK-INTERLEAVED-NEXT:    [[A_1_LCSSA:%.*]] = phi i32 [ poison, [[FOR_INC:%.*]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[A_1_LCSSA]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[LOR_LHS_FALSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK-INTERLEAVED:       lor.lhs.false:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_INC]], label [[IF_THEN]]
+; CHECK-INTERLEAVED:       if.then:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_INC]]
+; CHECK-INTERLEAVED:       for.inc:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP45:![0-9]+]]
+;
 entry:
   br label %for.body
 
@@ -1492,6 +2626,196 @@ for.inc:                                          ; preds = %lor.lhs.false, %if.
   br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
 }
 
+; An in-loop add reduction followed by an in-loop sub reduction becomes an add reduction followed by an add reduction with its input negated
+define i32 @reduction_add_sub(ptr noalias nocapture %A, ptr noalias nocapture %B) {
+; CHECK-LABEL: @reduction_add_sub(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[VEC_PHI]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP6]] = add i32 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[X_0_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: @reduction_add_sub(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = add i32 [[VEC_PHI]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = add i32 [[VEC_PHI1]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11]] = add i32 [[TMP7]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP13]] = add i32 [[TMP9]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP11]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[X_0_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %x.05 = phi i32 [ %sub, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+  %arrayidx.b = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
+  %l0 = load i32, ptr %arrayidx, align 4
+  %l0.b = load i32, ptr %arrayidx.b, align 4
+  %add = add nsw i32 %x.05, %l0
+  %sub = sub nsw i32 %add, %l0.b
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %x.0.lcssa = phi i32 [ %sub, %for.body ]
+  ret i32 %x.0.lcssa
+}
+
+; An in-loop sub reduction followed by an in-loop add reduction becomes an add reduction with its input negated
+define i32 @reduction_sub_add(ptr noalias nocapture %A, ptr noalias nocapture %B) {
+; CHECK-LABEL: @reduction_sub_add(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[VEC_PHI]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD1]])
+; CHECK-NEXT:    [[TMP6]] = add i32 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[FOR_END:%.*]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i32 [[X_0_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: @reduction_sub_add(
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP13:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 16
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = add i32 [[VEC_PHI]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = add i32 [[VEC_PHI1]], [[TMP8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD3]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP11]] = add i32 [[TMP7]], [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD4]])
+; CHECK-INTERLEAVED-NEXT:    [[TMP13]] = add i32 [[TMP9]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP11]]
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_END:%.*]]
+; CHECK-INTERLEAVED:       scalar.ph:
+; CHECK-INTERLEAVED-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       for.body:
+; CHECK-INTERLEAVED-NEXT:    br i1 poison, label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP49:![0-9]+]]
+; CHECK-INTERLEAVED:       for.end:
+; CHECK-INTERLEAVED-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ poison, [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT:    ret i32 [[X_0_LCSSA]]
+;
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %entry, %for.body
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %x.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+  %arrayidx.b = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
+  %l0 = load i32, ptr %arrayidx, align 4
+  %l0.b = load i32, ptr %arrayidx.b, align 4
+  %sub = sub nsw i32 %x.05, %l0
+  %add = add nsw i32 %sub, %l0.b
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, 256
+  br i1 %exitcond, label %for.end, label %for.body
+
+for.end:                                          ; preds = %for.body, %entry
+  %x.0.lcssa = phi i32 [ %add, %for.body ]
+  ret i32 %x.0.lcssa
+}
+
 declare float @llvm.fmuladd.f32(float, float, float)
 
 !6 = distinct !{!6, !7, !8}
diff --git a/llvm/test/Transforms/LoopVectorize/remarks-reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/remarks-reduction-inloop.ll
index 5c52b1ab27784..07643a1c6e83b 100644
--- a/llvm/test/Transforms/LoopVectorize/remarks-reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/remarks-reduction-inloop.ll
@@ -8,7 +8,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 define i32 @reduction_sum(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK-LABEL: define i32 @reduction_sum(
 ; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -25,12 +25,10 @@ define i32 @reduction_sum(ptr noalias nocapture %A, ptr noalias nocapture %B) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[BODY:.*]]
 ; CHECK:       [[BODY]]:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[BODY]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
-; CHECK-NEXT:    [[SUM_TMP:%.*]] = phi i32 [ [[SUM:%.*]], %[[BODY]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[BODY]] ], [ 0, %[[SCALAR_PH]] ]
+; CHECK-NEXT:    [[SUM_TMP:%.*]] = phi i32 [ [[SUM:%.*]], %[[BODY]] ], [ 0, %[[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[GEP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    [[LOAD0:%.*]] = load i32, ptr [[GEP0]], align 4
 ; CHECK-NEXT:    [[SUM]] = add i32 [[SUM_TMP]], [[LOAD0]]
diff --git a/llvm/test/Transforms/LoopVectorize/reverse_induction.ll b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
index cea16c9eb7513..4895f6a83d56d 100644
--- a/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/reverse_induction.ll
@@ -9,7 +9,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 define i32 @reverse_induction_i64(i64 %startval, ptr %ptr) {
 ; CHECK-LABEL: define i32 @reverse_induction_i64(
 ; CHECK-SAME: i64 [[STARTVAL:%.*]], ptr [[PTR:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -38,14 +38,11 @@ define i32 @reverse_induction_i64(i64 %startval, ptr %ptr) {
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
 ; CHECK-NEXT:    br label %[[LOOPEND:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[STARTVAL]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[ADD_I7:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD_I:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[INC4:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[REDUX5:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[INC_REDUX:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ADD_I7:%.*]] = phi i64 [ [[STARTVAL]], %[[SCALAR_PH]] ], [ [[ADD_I:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INC4:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[REDUX5:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INC_REDUX:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ADD_I]] = add i64 [[ADD_I7]], -1
 ; CHECK-NEXT:    [[KIND__I:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i64 [[ADD_I]]
 ; CHECK-NEXT:    [[TMP_I1:%.*]] = load i32, ptr [[KIND__I]], align 4
@@ -80,7 +77,7 @@ loopend:
 define i32 @reverse_induction_i128(i128 %startval, ptr %ptr) {
 ; CHECK-LABEL: define i32 @reverse_induction_i128(
 ; CHECK-SAME: i128 [[STARTVAL:%.*]], ptr [[PTR:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -109,14 +106,11 @@ define i32 @reverse_induction_i128(i128 %startval, ptr %ptr) {
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
 ; CHECK-NEXT:    br label %[[LOOPEND:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i128 [ [[STARTVAL]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[ADD_I7:%.*]] = phi i128 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD_I:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ], [ [[INC4:%.*]], %[[FOR_BODY]] ]
-; CHECK-NEXT:    [[REDUX5:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[INC_REDUX:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[ADD_I7:%.*]] = phi i128 [ [[STARTVAL]], %[[SCALAR_PH]] ], [ [[ADD_I:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[I_06:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INC4:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[REDUX5:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[INC_REDUX:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ADD_I]] = add i128 [[ADD_I7]], -1
 ; CHECK-NEXT:    [[KIND__I:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i128 [[ADD_I]]
 ; CHECK-NEXT:    [[TMP_I1:%.*]] = load i32, ptr [[KIND__I]], align 4
@@ -248,7 +242,7 @@ loopend:
 
 define void @reverse_forward_induction_i64_i8() {
 ; CHECK-LABEL: define void @reverse_forward_induction_i64_i8() {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -277,12 +271,10 @@ define void @reverse_forward_induction_i64_i8() {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[WHILE_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1023, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i8 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
 ; CHECK:       [[WHILE_BODY]]:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
-; CHECK-NEXT:    [[FORWARD_INDUCTION_05:%.*]] = phi i8 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 1023, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[FORWARD_INDUCTION_05:%.*]] = phi i8 [ 0, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[WHILE_BODY]] ]
 ; CHECK-NEXT:    [[INC]] = add i8 [[FORWARD_INDUCTION_05]], 1
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[INC]] to i32
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 [[INDVARS_IV]]
@@ -316,7 +308,7 @@ while.end:
 
 define void @reverse_forward_induction_i64_i8_signed() {
 ; CHECK-LABEL: define void @reverse_forward_induction_i64_i8_signed() {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -345,12 +337,10 @@ define void @reverse_forward_induction_i64_i8_signed() {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[WHILE_END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1023, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL2:%.*]] = phi i8 [ -127, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
 ; CHECK:       [[WHILE_BODY]]:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
-; CHECK-NEXT:    [[FORWARD_INDUCTION_05:%.*]] = phi i8 [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 1023, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[WHILE_BODY]] ]
+; CHECK-NEXT:    [[FORWARD_INDUCTION_05:%.*]] = phi i8 [ -127, %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[WHILE_BODY]] ]
 ; CHECK-NEXT:    [[INC]] = add i8 [[FORWARD_INDUCTION_05]], 1
 ; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[INC]] to i32
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i32], ptr @a, i64 0, i64 [[INDVARS_IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check-readonly.ll b/llvm/test/Transforms/LoopVectorize/runtime-check-readonly.ll
index 26bc36df94103..e6dc2cfd4a120 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check-readonly.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check-readonly.ll
@@ -1,42 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "vector.ph\:" --version 5
 ; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
 define void @add_ints(ptr nocapture %A, ptr nocapture %B, ptr nocapture %C) {
-; CHECK-LABEL: @add_ints(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
-; CHECK-LABEL: vector.memcheck:
-; CHECK-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A:%.*]] to i64
-; CHECK-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B:%.*]] to i64
-; CHECK-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C:%.*]] to i64
+; CHECK-LABEL: define void @add_ints(
+; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]], ptr captures(none) [[C:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, [[SCALAR_PH:label %.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK:       [[VECTOR_MEMCHECK]]:
+; CHECK-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
+; CHECK-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
+; CHECK-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[A1]], [[B2]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 [[A1]], [[C3]]
 ; CHECK-NEXT:    [[DIFF_CHECK4:%.*]] = icmp ult i64 [[TMP1]], 16
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[DIFF_CHECK]], [[DIFF_CHECK4]]
-; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    br label %vector.body
-; CHECK:       vector.body:
+; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], [[SCALAR_PH]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
 ;
 entry:
-  br label %for.body
+  br label %loop
 
-for.body:
-  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %arrayidx = getelementptr inbounds i32, ptr %B, i64 %iv
   %0 = load i32, ptr %arrayidx, align 4
-  %arrayidx2 = getelementptr inbounds i32, ptr %C, i64 %indvars.iv
+  %arrayidx2 = getelementptr inbounds i32, ptr %C, i64 %iv
   %1 = load i32, ptr %arrayidx2, align 4
   %add = add nsw i32 %1, %0
-  %arrayidx4 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+  %arrayidx4 = getelementptr inbounds i32, ptr %A, i64 %iv
   store i32 %add, ptr %arrayidx4, align 4
-  %indvars.iv.next = add i64 %indvars.iv, 1
-  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
-  %exitcond = icmp eq i32 %lftr.wideiv, 200
-  br i1 %exitcond, label %for.end, label %for.body
+  %iv.next = add i64 %iv, 1
+  %iv.trunc = trunc i64 %iv.next to i32
+  %exitcond = icmp eq i32 %iv.trunc, 200
+  br i1 %exitcond, label %exit, label %loop
 
-for.end:
+exit:
   ret void
 }
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
index c5838fe251a8c..8d9ee2a78adcd 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
@@ -26,7 +26,7 @@ define i32 @foo(ptr nocapture %a, ptr nocapture %b, i32 %n) nounwind uwtable ssp
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 16, !dbg [[DBG9]]
 ; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]], !dbg [[DBG9]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644, !dbg [[DBG9]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP0]], 2147483644
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]], !dbg [[DBG9]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], !dbg [[DBG9]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-assume.ll b/llvm/test/Transforms/LoopVectorize/scalable-assume.ll
index 82911640aea8c..ad8cd425cd6c2 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-assume.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-assume.ll
@@ -6,7 +6,7 @@ define void @test1(ptr noalias nocapture %a, ptr noalias nocapture readonly %b)
 ; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1600, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -14,14 +14,12 @@ define void @test1(ptr noalias nocapture %a, ptr noalias nocapture readonly %b)
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1600, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1600, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x float>, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 2 x float>, ptr [[TMP9]], align 4
@@ -35,11 +33,11 @@ define void @test1(ptr noalias nocapture %a, ptr noalias nocapture readonly %b)
 ; CHECK-NEXT:    [[TMP15:%.*]] = fadd <vscale x 2 x float> [[WIDE_LOAD1]], splat (float 1.000000e+00)
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 1
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP18]]
 ; CHECK-NEXT:    store <vscale x 2 x float> [[TMP14]], ptr [[TMP16]], align 4
 ; CHECK-NEXT:    store <vscale x 2 x float> [[TMP15]], ptr [[TMP19]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -82,7 +80,7 @@ define void @test2(ptr %a, ptr noalias %b) {
 ; CHECK-NEXT:    [[PTRINT2:%.*]] = ptrtoint ptr [[B]] to i64
 ; CHECK-NEXT:    [[MASKCOND4:%.*]] = icmp eq i64 [[PTRINT2]], 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1600, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -90,8 +88,6 @@ define void @test2(ptr %a, ptr noalias %b) {
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1600, [[TMP7]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1600, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -99,7 +95,7 @@ define void @test2(ptr %a, ptr noalias %b) {
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
+; CHECK-NEXT:    [[TMP12:%.*]] = shl nuw i64 [[TMP11]], 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP12]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x float>, ptr [[TMP10]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x float>, ptr [[TMP13]], align 4
@@ -109,11 +105,11 @@ define void @test2(ptr %a, ptr noalias %b) {
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND4]])
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = shl nuw i64 [[TMP17]], 1
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP18]]
 ; CHECK-NEXT:    store <vscale x 2 x float> [[TMP14]], ptr [[TMP16]], align 4
 ; CHECK-NEXT:    store <vscale x 2 x float> [[TMP15]], ptr [[TMP19]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
@@ -155,17 +151,17 @@ define void @predicated_assume(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-SAME: ptr noalias readonly captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 2 x i64> [[TMP7]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP8]]
@@ -173,8 +169,6 @@ define void @predicated_assume(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP5]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp ult <vscale x 2 x i64> [[VEC_IND]], splat (i64 495616)
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp ult <vscale x 2 x i64> [[STEP_ADD]], splat (i64 495616)
@@ -182,7 +176,7 @@ define void @predicated_assume(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-NEXT:    [[PREDPHI1:%.*]] = select <vscale x 2 x i1> [[TMP10]], <vscale x 2 x float> splat (float 2.300000e+01), <vscale x 2 x float> splat (float 4.200000e+01)
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 1
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i64 [[TMP13]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x float>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 2 x float>, ptr [[TMP14]], align 4
@@ -190,7 +184,7 @@ define void @predicated_assume(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-NEXT:    [[TMP16:%.*]] = fmul <vscale x 2 x float> [[PREDPHI1]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 2
+; CHECK-NEXT:    [[TMP19:%.*]] = shl nuw i64 [[TMP18]], 1
 ; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP19]]
 ; CHECK-NEXT:    store <vscale x 2 x float> [[TMP15]], ptr [[TMP17]], align 4
 ; CHECK-NEXT:    store <vscale x 2 x float> [[TMP16]], ptr [[TMP20]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
index bfc0a4829aa3f..60da3368b664e 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
@@ -20,7 +20,7 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
 ; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
 ; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_MEMCHECK]]:
@@ -36,8 +36,6 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP11]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = mul nuw i32 [[TMP14]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
@@ -53,7 +51,7 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
 ; CHECK-VF4UF1-NEXT:    [[TMP22:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP20]]
 ; CHECK-VF4UF1-NEXT:    store <vscale x 4 x i32> [[TMP22]], ptr [[TMP21]], align 4
-; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
 ; CHECK-VF4UF1-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
@@ -85,7 +83,7 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
 ; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
 ; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 3
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_MEMCHECK]]:
@@ -101,8 +99,6 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP11]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 8
 ; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = mul nuw i32 [[TMP14]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
@@ -114,7 +110,7 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = add nuw nsw i64 [[INDEX]], 1
 ; CHECK-VF4UF2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP17]]
 ; CHECK-VF4UF2-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = shl nuw i64 [[TMP20]], 2
 ; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i64 [[TMP21]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP18]], align 4
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD3]] = load <vscale x 4 x i32>, ptr [[TMP22]], align 4
@@ -124,11 +120,11 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP26:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD]], [[TMP23]]
 ; CHECK-VF4UF2-NEXT:    [[TMP27:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD3]], [[TMP24]]
 ; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = mul nuw i64 [[TMP29]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = shl nuw i64 [[TMP29]], 2
 ; CHECK-VF4UF2-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP30]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP26]], ptr [[TMP25]], align 4
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP27]], ptr [[TMP31]], align 4
-; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]]
 ; CHECK-VF4UF2-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
@@ -190,7 +186,7 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4
 ; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_PH]]:
@@ -198,8 +194,6 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP8]], 1
@@ -217,7 +211,7 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = select <vscale x 4 x i1> [[TMP14]], <vscale x 4 x i32> [[TMP13]], <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = icmp slt <vscale x 4 x i32> [[VEC_PHI]], [[TMP15]]
 ; CHECK-VF4UF1-NEXT:    [[TMP17]] = select <vscale x 4 x i1> [[TMP16]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP15]]
-; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-VF4UF1-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
@@ -251,7 +245,7 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4
 ; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_PH]]:
@@ -259,8 +253,6 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
-; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP8]], 1
@@ -273,7 +265,7 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ undef, %[[VECTOR_PH]] ], [ [[TMP26:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 2
 ; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 [[TMP13]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP10]], align 4
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD2]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
@@ -289,7 +281,7 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP24:%.*]] = icmp slt <vscale x 4 x i32> [[VEC_PHI1]], [[TMP22]]
 ; CHECK-VF4UF2-NEXT:    [[TMP25]] = select <vscale x 4 x i1> [[TMP23]], <vscale x 4 x i32> [[VEC_PHI]], <vscale x 4 x i32> [[TMP21]]
 ; CHECK-VF4UF2-NEXT:    [[TMP26]] = select <vscale x 4 x i1> [[TMP24]], <vscale x 4 x i32> [[VEC_PHI1]], <vscale x 4 x i32> [[TMP22]]
-; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
 ; CHECK-VF4UF2-NEXT:    [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP27]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
@@ -365,7 +357,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = add i32 [[N]], -1
 ; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_MEMCHECK]]:
@@ -388,8 +380,6 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP12]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = add i64 1, [[N_VEC]]
 ; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x double> poison, double [[CONV1]], i64 0
 ; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x double> [[BROADCAST_SPLATINSERT]], <vscale x 4 x double> poison, <vscale x 4 x i32> zeroinitializer
@@ -411,7 +401,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF1-NEXT:    [[TMP25:%.*]] = fsub fast <vscale x 4 x double> [[TMP22]], [[TMP24]]
 ; CHECK-VF4UF1-NEXT:    [[TMP26:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[OFFSET_IDX]]
 ; CHECK-VF4UF1-NEXT:    store <vscale x 4 x double> [[TMP25]], ptr [[TMP26]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
-; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
 ; CHECK-VF4UF1-NEXT:    [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
@@ -443,7 +433,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = add i32 [[N]], -1
 ; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 3
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_MEMCHECK]]:
@@ -466,8 +456,6 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP12]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
-; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 8
 ; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = add i64 1, [[N_VEC]]
 ; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x double> poison, double [[CONV1]], i64 0
 ; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x double> [[BROADCAST_SPLATINSERT]], <vscale x 4 x double> poison, <vscale x 4 x i32> zeroinitializer
@@ -482,7 +470,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
 ; CHECK-VF4UF2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET_IDX]]
 ; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 2
 ; CHECK-VF4UF2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i16, ptr [[TMP19]], i64 [[TMP22]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP19]], align 2, !alias.scope [[META6:![0-9]+]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD4]] = load <vscale x 4 x i16>, ptr [[TMP23]], align 2, !alias.scope [[META6]]
@@ -498,11 +486,11 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF2-NEXT:    [[TMP33:%.*]] = fsub fast <vscale x 4 x double> [[TMP27]], [[TMP31]]
 ; CHECK-VF4UF2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[OFFSET_IDX]]
 ; CHECK-VF4UF2-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP37:%.*]] = mul nuw i64 [[TMP36]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP37:%.*]] = shl nuw i64 [[TMP36]], 2
 ; CHECK-VF4UF2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds double, ptr [[TMP34]], i64 [[TMP37]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x double> [[TMP32]], ptr [[TMP34]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x double> [[TMP33]], ptr [[TMP38]], align 8, !alias.scope [[META9]], !noalias [[META6]]
-; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]]
 ; CHECK-VF4UF2-NEXT:    [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
@@ -560,7 +548,7 @@ define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF1-LABEL: define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1000, [[TMP1]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_PH]]:
@@ -568,12 +556,10 @@ define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1000, [[TMP3]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 1000, [[N_MOD_VF]]
-; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_BODY]]:
 ; CHECK-VF4UF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
@@ -588,7 +574,7 @@ define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF2-LABEL: define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1000, [[TMP1]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_PH]]:
@@ -596,12 +582,10 @@ define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1000, [[TMP3]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 1000, [[N_MOD_VF]]
-; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_BODY]]:
 ; CHECK-VF4UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
@@ -638,7 +622,7 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; CHECK-VF4UF1-SAME: ptr [[CVAL:%.*]], i32 [[X:%.*]]) {
 ; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 96, [[TMP1]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_PH]]:
@@ -646,22 +630,19 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i32 96, [[TMP3]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i32 96, [[N_MOD_VF]]
-; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
 ; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
 ; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = mul <vscale x 4 x i32> [[TMP6]], splat (i32 1)
 ; CHECK-VF4UF1-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP7]]
-; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul i32 1, [[TMP5]]
-; CHECK-VF4UF1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP8]], i64 0
+; CHECK-VF4UF1-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP3]], i64 0
 ; CHECK-VF4UF1-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_BODY]]:
 ; CHECK-VF4UF1-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-VF4UF1-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[VEC_IND]], [[BROADCAST_SPLAT]]
-; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
 ; CHECK-VF4UF1-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
@@ -686,17 +667,17 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; CHECK-VF4UF2-SAME: ptr [[CVAL:%.*]], i32 [[X:%.*]]) {
 ; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 3
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 96, [[TMP1]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_PH]]:
-; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 8
-; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i32 96, [[TMP3]]
-; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i32 96, [[N_MOD_VF]]
 ; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
+; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], 2
+; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i32 96, [[TMP6]]
+; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i32 96, [[N_MOD_VF]]
 ; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
 ; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
@@ -706,8 +687,6 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; CHECK-VF4UF2:       [[VECTOR_BODY]]:
 ; CHECK-VF4UF2-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-VF4UF2-NEXT:    [[STEP_ADD:%.*]] = phi <vscale x 4 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT1:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP5]], i64 0
-; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4UF2-NEXT:    [[VEC_IND_NEXT:%.*]] = add <vscale x 4 x i32> [[STEP_ADD]], [[BROADCAST_SPLAT2]]
 ; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = add <vscale x 4 x i32> [[VEC_IND_NEXT]], [[BROADCAST_SPLAT]]
 ; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP6]]
@@ -759,7 +738,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF1-NEXT:    [[DOTPRE:%.*]] = load i16, ptr [[A]], align 2
 ; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_MEMCHECK]]:
@@ -778,8 +757,6 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
@@ -797,7 +774,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF1-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP17]], [[TMP16]]
 ; CHECK-VF4UF1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
 ; CHECK-VF4UF1-NEXT:    store <vscale x 4 x i32> [[TMP18]], ptr [[TMP19]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
-; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-VF4UF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-VF4UF1-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
@@ -818,7 +795,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF2-NEXT:    [[DOTPRE:%.*]] = load i16, ptr [[A]], align 2
 ; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_MEMCHECK]]:
@@ -837,8 +814,6 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
 ; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
@@ -850,7 +825,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = add nuw nsw i64 [[INDEX]], 1
 ; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[TMP12]]
 ; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 2
 ; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP13]], align 2, !alias.scope [[META17:![0-9]+]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD3]] = load <vscale x 4 x i16>, ptr [[TMP17]], align 2, !alias.scope [[META17]]
@@ -864,11 +839,11 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP25:%.*]] = mul nsw <vscale x 4 x i32> [[TMP23]], [[TMP21]]
 ; CHECK-VF4UF2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
 ; CHECK-VF4UF2-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = shl nuw i64 [[TMP28]], 2
 ; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP26]], i64 [[TMP29]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP24]], ptr [[TMP26]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP25]], ptr [[TMP30]], align 4, !alias.scope [[META20]], !noalias [[META17]]
-; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]]
+; CHECK-VF4UF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
 ; CHECK-VF4UF2-NEXT:    [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll
index 62649412deb89..359132f7904cc 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll
@@ -15,20 +15,18 @@ define void @add_ind64_unrolled(ptr noalias nocapture %a, ptr noalias nocapture
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP4]], 1
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP5]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP8]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <vscale x 2 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
@@ -97,18 +95,16 @@ define void @add_ind64_unrolled_nxv1i64(ptr noalias nocapture %a, ptr noalias no
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP2]], i64 0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 1 x i64> @llvm.stepvector.nxv1i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 1 x i64> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 1 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 1 x i64> poison, <vscale x 1 x i32> zeroinitializer
 ; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <vscale x 1 x i64> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
@@ -122,7 +118,7 @@ define void @add_ind64_unrolled_nxv1i64(ptr noalias nocapture %a, ptr noalias no
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP14]]
 ; CHECK-NEXT:    store <vscale x 1 x i64> [[TMP11]], ptr [[TMP13]], align 8
 ; CHECK-NEXT:    store <vscale x 1 x i64> [[TMP12]], ptr [[TMP15]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 1 x i64> [[STEP_ADD]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
@@ -185,13 +181,11 @@ define void @add_unique_ind32(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-NEXT:    [[IND_END:%.*]] = shl i32 [[DOTCAST]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; CHECK-NEXT:    [[TMP7:%.*]] = shl <vscale x 4 x i32> [[TMP6]], splat (i32 1)
-; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP5]] to i32
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP3]] to i32
 ; CHECK-NEXT:    [[TMP9:%.*]] = shl i32 [[TMP8]], 1
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[DOTSPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -201,7 +195,7 @@ define void @add_unique_ind32(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x i32> [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[VEC_IND]], ptr [[TMP10]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <vscale x 4 x i32> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
@@ -262,8 +256,6 @@ define void @add_unique_indf32(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = fmul float [[DOTCAST]], 2.000000e+00
 ; CHECK-NEXT:    [[IND_END:%.*]] = fadd float [[TMP4]], 0.000000e+00
@@ -271,7 +263,7 @@ define void @add_unique_indf32(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = uitofp <vscale x 4 x i32> [[TMP7]] to <vscale x 4 x float>
 ; CHECK-NEXT:    [[TMP9:%.*]] = fmul <vscale x 4 x float> [[TMP8]], splat (float 2.000000e+00)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = fadd <vscale x 4 x float> [[TMP9]], zeroinitializer
-; CHECK-NEXT:    [[TMP12:%.*]] = uitofp i64 [[TMP6]] to float
+; CHECK-NEXT:    [[TMP12:%.*]] = uitofp i64 [[TMP3]] to float
 ; CHECK-NEXT:    [[TMP13:%.*]] = fmul float [[TMP12]], 2.000000e+00
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[TMP13]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x float> [[DOTSPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
@@ -281,7 +273,7 @@ define void @add_unique_indf32(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 4 x float> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[VEC_IND]], ptr [[TMP14]], align 4
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = fadd <vscale x 4 x float> [[VEC_IND]], [[DOTSPLAT]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-iv-outside-user.ll b/llvm/test/Transforms/LoopVectorize/scalable-iv-outside-user.ll
index 1ec2993f0014d..cb61fc6e0a046 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-iv-outside-user.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-iv-outside-user.ll
@@ -8,17 +8,17 @@ define i32 @iv_live_out_wide(ptr %dst) {
 ; CHECK-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
 ; CHECK-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 2000, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 2000, [[TMP3]]
-; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 2000, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 2
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP5]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], 2
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 2000, [[TMP6]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 2000, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[STEP_2]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
@@ -28,12 +28,10 @@ define i32 @iv_live_out_wide(ptr %dst) {
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <vscale x 2 x i32> [ [[INDUCTION]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[TMP5]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[STEP_ADD:%.*]] = add <vscale x 2 x i32> [[VEC_IND]], [[BROADCAST_SPLAT2]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = shl nuw i64 [[TMP12]], 1
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 [[TMP13]]
 ; CHECK-NEXT:    store <vscale x 2 x i16> zeroinitializer, ptr [[TMP10]], align 2
 ; CHECK-NEXT:    store <vscale x 2 x i16> zeroinitializer, ptr [[TMP14]], align 2
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll
index 7aac9d1927f76..850d6b299ca23 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll
@@ -10,9 +10,9 @@ define void @test(ptr %d) {
 ; CHECK-SAME: (ptr [[D:%.*]]) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARR:%.*]] = alloca [1024 x i32], align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ARR]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 128, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -20,18 +20,16 @@ define void @test(ptr %d) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4096, ptr [[ARR]])
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 2 x i32> splat (i32 100), ptr [[TMP6]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ARR]])
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDEX]]
+; CHECK-NEXT:    store <vscale x 2 x i32> splat (i32 100), ptr [[TMP4]], align 8
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ARR]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 128, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -40,39 +38,39 @@ define void @test(ptr %d) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4096, ptr [[ARR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ARR]])
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    store i32 100, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ARR]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[LFTR_WIDEIV]], 128
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       for.end:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4096, ptr [[ARR]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ARR]])
 ; CHECK-NEXT:    ret void
 ;
 
 entry:
   %arr = alloca [1024 x i32], align 16
-  call void @llvm.lifetime.start.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.start.p0(ptr %arr) #1
   br label %for.body
 
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
-  call void @llvm.lifetime.end.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.end.p0(ptr %arr) #1
   %arrayidx = getelementptr inbounds i32, ptr %d, i64 %indvars.iv
   %0 = load i32, ptr %arrayidx, align 8
   store i32 100, ptr %arrayidx, align 8
-  call void @llvm.lifetime.start.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.start.p0(ptr %arr) #1
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp ne i32 %lftr.wideiv, 128
   br i1 %exitcond, label %for.body, label %for.end, !llvm.loop !0
 
 for.end:
-  call void @llvm.lifetime.end.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.end.p0(ptr %arr) #1
   ret void
 }
 
@@ -82,7 +80,7 @@ define void @testloopvariant(ptr %d) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARR:%.*]] = alloca [1024 x i32], align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 128, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -90,18 +88,16 @@ define void @testloopvariant(ptr %d) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4096, ptr [[ARR]])
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDEX]]
-; CHECK-NEXT:    store <vscale x 2 x i32> splat (i32 100), ptr [[TMP6]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ARR]])
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDEX]]
+; CHECK-NEXT:    store <vscale x 2 x i32> splat (i32 100), ptr [[TMP4]], align 8
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ARR]])
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 128, [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -110,12 +106,12 @@ define void @testloopvariant(ptr %d) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr [1024 x i32], ptr [[ARR]], i32 0, i64 [[INDVARS_IV]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4096, ptr [[ARR]])
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr [1024 x i32], ptr [[ARR]], i32 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ARR]])
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 8
 ; CHECK-NEXT:    store i32 100, ptr [[ARRAYIDX]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ARR]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i32 [[LFTR_WIDEIV]], 128
@@ -130,11 +126,11 @@ entry:
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %0 = getelementptr [1024 x i32], ptr %arr, i32 0, i64 %indvars.iv
-  call void @llvm.lifetime.end.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.end.p0(ptr %arr) #1
   %arrayidx = getelementptr inbounds i32, ptr %d, i64 %indvars.iv
   %1 = load i32, ptr %arrayidx, align 8
   store i32 100, ptr %arrayidx, align 8
-  call void @llvm.lifetime.start.p0(i64 4096, ptr %arr) #1
+  call void @llvm.lifetime.start.p0(ptr %arr) #1
   %indvars.iv.next = add i64 %indvars.iv, 1
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp ne i32 %lftr.wideiv, 128
@@ -144,9 +140,9 @@ for.end:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 !0 = distinct !{!0, !1}
 !1 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
index f384d3c15ca6e..bdc8734c45f2e 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
@@ -1,91 +1,91 @@
-; RUN: opt -S -passes=loop-vectorize,instcombine -force-vector-interleave=1 -force-vector-width=4 -force-target-supports-scalable-vectors=true -scalable-vectorization=on < %s | FileCheck %s --check-prefix=CHECKUF1
-; RUN: opt -S -passes=loop-vectorize,instcombine -force-vector-interleave=2 -force-vector-width=4 -force-target-supports-scalable-vectors=true -scalable-vectorization=on < %s | FileCheck %s --check-prefix=CHECKUF2
-
-; CHECKUF1: for.body.preheader:
-; CHECKUF1-DAG: %wide.trip.count = zext nneg i32 %N to i64
-; CHECKUF1-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF1-DAG: %[[VSCALEX4:.*]] = shl nuw i64 %[[VSCALE]], 2
-; CHECKUF1-DAG: %min.iters.check = icmp ugt i64 %[[VSCALEX4]], %wide.trip.count
-
-; CHECKUF1: vector.ph:
-; CHECKUF1-DAG:  %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF1-DAG:  %[[VSCALEX4:.*]] = shl nuw i64 %[[VSCALE]], 2
-; CHECKUF1-DAG:  %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX4]]
-; CHECKUF1:      %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf
-; CHECKUF1: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF1: %[[VSCALEX4:.*]] = shl nuw i64 %[[VSCALE]], 2
-
-; CHECKUF1: vector.body:
-; CHECKUF1: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
-; CHECKUF1: %[[IDXB:.*]] = getelementptr inbounds double, ptr %b, i64 %index
-; CHECKUF1: %wide.load = load <vscale x 4 x double>, ptr %[[IDXB]], align 8
-; CHECKUF1: %[[FADD:.*]] = fadd <vscale x 4 x double> %wide.load, splat (double 1.000000e+00)
-; CHECKUF1: %[[IDXA:.*]] = getelementptr inbounds double, ptr %a, i64 %index
-; CHECKUF1: store <vscale x 4 x double> %[[FADD]], ptr %[[IDXA]], align 8
-; CHECKUF1: %index.next = add nuw i64 %index, %[[VSCALEX4]]
-; CHECKUF1: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec
-; CHECKUF1: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !0
-
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -force-target-supports-scalable-vectors=true -scalable-vectorization=on < %s | FileCheck %s --check-prefix=CHECKUF1
+; RUN: opt -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -force-target-supports-scalable-vectors=true -scalable-vectorization=on < %s | FileCheck %s --check-prefix=CHECKUF2
 
 ; For an interleave factor of 2, vscale is scaled by 8 instead of 4 (and thus shifted left by 3 instead of 2).
 ; There is also the increment for the next iteration, e.g. instead of indexing IDXB, it indexes at IDXB + vscale * 4.
-
-; CHECKUF2: for.body.preheader:
-; CHECKUF2-DAG: %wide.trip.count = zext nneg i32 %N to i64
-; CHECKUF2-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF2-DAG: %[[VSCALEX8:.*]] = shl nuw i64 %[[VSCALE]], 3
-; CHECKUF2-DAG: %min.iters.check = icmp ugt i64 %[[VSCALEX8]], %wide.trip.count
-
-; CHECKUF2: vector.ph:
-; CHECKUF2-DAG:  %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF2-DAG:  %[[VSCALEX8:.*]] = shl nuw i64 %[[VSCALE]], 3
-; CHECKUF2-DAG:  %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX8]]
-; CHECKUF2:      %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf
-; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF2: %[[VSCALEX8:.*]] = shl nuw i64 %[[VSCALE]], 3
-
-; CHECKUF2: vector.body:
-; CHECKUF2: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
-; CHECKUF2: %[[IDXB:.*]] = getelementptr inbounds double, ptr %b, i64 %index
-; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF2: %[[VSCALE2:.*]] = shl i64 %[[VSCALE]], 5
-; CHECKUF2: %[[IDXB_NEXT:.*]] = getelementptr inbounds i8, ptr %[[IDXB]], i64 %[[VSCALE2]]
-; CHECKUF2: %wide.load = load <vscale x 4 x double>, ptr %[[IDXB]], align 8
-; CHECKUF2: %wide.load{{[0-9]+}} = load <vscale x 4 x double>, ptr %[[IDXB_NEXT]], align 8
-; CHECKUF2: %[[FADD:.*]] = fadd <vscale x 4 x double> %wide.load, splat (double 1.000000e+00)
-; CHECKUF2: %[[FADD_NEXT:.*]] = fadd <vscale x 4 x double> %wide.load{{[0-9]+}}, splat (double 1.000000e+00)
-; CHECKUF2: %[[IDXA:.*]] = getelementptr inbounds double, ptr %a, i64 %index
-; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF2: %[[VSCALE2:.*]] = shl i64 %[[VSCALE]], 5
-; CHECKUF2: %[[IDXA_NEXT:.*]] = getelementptr inbounds i8, ptr %[[IDXA]], i64 %[[VSCALE2]]
-; CHECKUF2: store <vscale x 4 x double> %[[FADD]], ptr %[[IDXA]], align 8
-; CHECKUF2: store <vscale x 4 x double> %[[FADD_NEXT]], ptr %[[IDXA_NEXT]], align 8
-; CHECKUF2: %index.next = add nuw i64 %index, %[[VSCALEX8]]
-; CHECKUF2: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec
-; CHECKUF2: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !0
-
-define void @loop(i32 %N, ptr nocapture %a, ptr nocapture readonly %b) {
+define void @loop(i64 %N, ptr noalias %a, ptr noalias %b) {
+; CHECKUF1-LABEL: define void @loop(
+; CHECKUF1-SAME: i64 [[N:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECKUF1-NEXT:  [[ENTRY:.*:]]
+; CHECKUF1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF1-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECKUF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECKUF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECKUF1:       [[VECTOR_PH]]:
+; CHECKUF1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
+; CHECKUF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
+; CHECKUF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECKUF1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECKUF1:       [[VECTOR_BODY]]:
+; CHECKUF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECKUF1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[INDEX]]
+; CHECKUF1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x double>, ptr [[TMP7]], align 8
+; CHECKUF1-NEXT:    [[TMP8:%.*]] = fadd <vscale x 4 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECKUF1-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[INDEX]]
+; CHECKUF1-NEXT:    store <vscale x 4 x double> [[TMP8]], ptr [[TMP9]], align 8
+; CHECKUF1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECKUF1-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECKUF1-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECKUF1:       [[MIDDLE_BLOCK]]:
+; CHECKUF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECKUF1-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECKUF1:       [[SCALAR_PH]]:
+;
+; CHECKUF2-LABEL: define void @loop(
+; CHECKUF2-SAME: i64 [[N:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECKUF2-NEXT:  [[ENTRY:.*:]]
+; CHECKUF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 3
+; CHECKUF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECKUF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECKUF2:       [[VECTOR_PH]]:
+; CHECKUF2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
+; CHECKUF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
+; CHECKUF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECKUF2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECKUF2:       [[VECTOR_BODY]]:
+; CHECKUF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECKUF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[INDEX]]
+; CHECKUF2-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP8]], 2
+; CHECKUF2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, ptr [[TMP7]], i64 [[TMP16]]
+; CHECKUF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x double>, ptr [[TMP7]], align 8
+; CHECKUF2-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x double>, ptr [[TMP9]], align 8
+; CHECKUF2-NEXT:    [[TMP10:%.*]] = fadd <vscale x 4 x double> [[WIDE_LOAD]], splat (double 1.000000e+00)
+; CHECKUF2-NEXT:    [[TMP11:%.*]] = fadd <vscale x 4 x double> [[WIDE_LOAD3]], splat (double 1.000000e+00)
+; CHECKUF2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds double, ptr [[A]], i64 [[INDEX]]
+; CHECKUF2-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
+; CHECKUF2-NEXT:    [[TMP17:%.*]] = shl nuw i64 [[TMP13]], 2
+; CHECKUF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double, ptr [[TMP12]], i64 [[TMP17]]
+; CHECKUF2-NEXT:    store <vscale x 4 x double> [[TMP10]], ptr [[TMP12]], align 8
+; CHECKUF2-NEXT:    store <vscale x 4 x double> [[TMP11]], ptr [[TMP14]], align 8
+; CHECKUF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECKUF2-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECKUF2-NEXT:    br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECKUF2:       [[MIDDLE_BLOCK]]:
+; CHECKUF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECKUF2-NEXT:    br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECKUF2:       [[SCALAR_PH]]:
+;
 entry:
-  %cmp7 = icmp sgt i32 %N, 0
-  br i1 %cmp7, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:                               ; preds = %entry
-  %wide.trip.count = zext i32 %N to i64
-  br label %for.body
-
-for.cond.cleanup:                                 ; preds = %for.body, %entry
-  ret void
+  br label %loop
 
-for.body:                                         ; preds = %for.body.preheader, %for.body
-  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
-  %arrayidx = getelementptr inbounds double, ptr %b, i64 %indvars.iv
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %arrayidx = getelementptr inbounds double, ptr %b, i64 %iv
   %0 = load double, ptr %arrayidx, align 8
   %add = fadd double %0, 1.000000e+00
-  %arrayidx2 = getelementptr inbounds double, ptr %a, i64 %indvars.iv
+  %arrayidx2 = getelementptr inbounds double, ptr %a, i64 %iv
   store double %add, ptr %arrayidx2, align 8
-  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
-  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body, !llvm.loop !1
+  %iv.next = add nuw nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %N
+  br i1 %ec, label %exit, label %loop, !llvm.loop !1
+
+exit:
+  ret void
 }
 
 !1 = distinct !{!1, !2}
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-predication.ll b/llvm/test/Transforms/LoopVectorize/scalable-predication.ll
index 8e272debb299d..af5796747e6f4 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-predication.ll
@@ -9,7 +9,7 @@ define void @foo(i32 %val, ptr dereferenceable(1024) %ptr) {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw i64 [[TMP6]], 2
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i64 -257, [[TMP7]]
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -19,22 +19,19 @@ define void @foo(i32 %val, ptr dereferenceable(1024) %ptr) {
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 256, [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 256)
-; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP4]]
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
 ; CHECK:       while.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
index ba337aa52dad4..83f1d30c64321 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
@@ -7,7 +7,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-LABEL: @reduction_add_trunc(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP31:%.*]] = mul nuw i32 [[TMP30]], 16
+; CHECK-NEXT:    [[TMP31:%.*]] = shl nuw i32 [[TMP30]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 256, [[TMP31]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -15,8 +15,6 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 256, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 256, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -26,7 +24,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP15:%.*]] = and <vscale x 8 x i32> [[VEC_PHI1]], splat (i32 255)
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = shl nuw i64 [[TMP10]], 3
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 4
@@ -38,7 +36,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP35:%.*]] = trunc <vscale x 8 x i32> [[TMP29]] to <vscale x 8 x i8>
 ; CHECK-NEXT:    [[TMP34]] = zext <vscale x 8 x i8> [[TMP33]] to <vscale x 8 x i32>
 ; CHECK-NEXT:    [[TMP36]] = zext <vscale x 8 x i8> [[TMP35]] to <vscale x 8 x i32>
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll b/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll
index 4b8ff86774681..ae96fe58caa26 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll
@@ -5,7 +5,7 @@ define void @trunc_minimal_bitwidth(ptr %bptr, ptr noalias %hptr, i32 %val, i64
 ; CHECK-LABEL: @trunc_minimal_bitwidth(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -13,8 +13,6 @@ define void @trunc_minimal_bitwidth(ptr %bptr, ptr noalias %hptr, i32 %val, i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc <vscale x 4 x i32> [[BROADCAST_SPLAT]] to <vscale x 4 x i16>
@@ -23,7 +21,7 @@ define void @trunc_minimal_bitwidth(ptr %bptr, ptr noalias %hptr, i32 %val, i64
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[HPTR:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    store <vscale x 4 x i16> [[TMP4]], ptr [[TMP5]], align 2
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
@@ -65,7 +63,7 @@ define void @trunc_minimal_bitwidths_shufflevector (ptr %p, i32 %arg1, i64 %len)
 ; CHECK-LABEL: @trunc_minimal_bitwidths_shufflevector(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[LEN:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -73,8 +71,6 @@ define void @trunc_minimal_bitwidths_shufflevector (ptr %p, i32 %arg1, i64 %len)
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[LEN]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[LEN]], [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[ARG1:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc <vscale x 4 x i32> [[BROADCAST_SPLAT]] to <vscale x 4 x i8>
@@ -86,7 +82,7 @@ define void @trunc_minimal_bitwidths_shufflevector (ptr %p, i32 %arg1, i64 %len)
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor <vscale x 4 x i8> [[WIDE_LOAD]], [[TMP4]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 4 x i8> [[TMP6]], [[WIDE_LOAD]]
 ; CHECK-NEXT:    store <vscale x 4 x i8> [[TMP7]], ptr [[TMP5]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/scev-during-mutation.ll b/llvm/test/Transforms/LoopVectorize/scev-during-mutation.ll
deleted file mode 100644
index f1307dc7a3481..0000000000000
--- a/llvm/test/Transforms/LoopVectorize/scev-during-mutation.ll
+++ /dev/null
@@ -1,124 +0,0 @@
-; RUN: opt -passes=loop-vectorize -force-vector-width=2 -force-vector-interleave=2 -scev-verify-ir -S %s | FileCheck %s
-
-; Make sure SCEV is not queried while the IR is temporarily invalid. The tests
-; deliberately do not check for details of the vectorized IR, because that's
-; not the focus of the test.
-
-define void @pr49538() {
-; CHECK-LABEL: @pr49538
-; CHECK: vector.body:
-;
-entry:
-  br label %loop.0
-
-loop.0:
-  %iv.0 = phi i16 [ -1, %entry ], [ %iv.0.next, %loop.0.latch ]
-  br label %loop.1
-
-loop.1:
-  %iv.1 = phi i16 [ -1, %loop.0 ], [ %iv.1.next, %loop.1 ]
-  %iv.1.next = add nsw i16 %iv.1, 1
-  %i6 = icmp eq i16 %iv.1.next, %iv.0
-  br i1 %i6, label %loop.0.latch, label %loop.1
-
-loop.0.latch:
-  %i8 = phi i16 [ 1, %loop.1 ]
-  %iv.0.next = add nsw i16 %iv.0, 1
-  %ec.0 = icmp eq i16 %iv.0.next, %i8
-  br i1 %ec.0, label %exit, label %loop.0
-
-exit:
-  ret void
-}
-
-define void @pr49900(i32 %x, ptr %ptr) {
-; CHECK-LABEL: @pr49900
-; CHECK: vector.body{{.*}}:
-; CHECK: vector.body{{.*}}:
-;
-entry:
-  br label %loop.0
-
-loop.0:                                              ; preds = %bb2, %bb
-  %ec.0 = icmp slt i32 %x, 0
-  br i1 %ec.0, label %loop.0, label %loop.1.ph
-
-loop.1.ph:                                              ; preds = %bb2
-  br label %loop.1
-
-loop.1:                                             ; preds = %bb33, %bb5
-  %iv.1 = phi i32 [ 0, %loop.1.ph ], [ %iv.3.next, %loop.1.latch ]
-  br label %loop.2
-
-loop.2:
-  %iv.2 = phi i32 [ %iv.1, %loop.1 ], [ %iv.2.next, %loop.2 ]
-  %tmp54 = add i32 %iv.2, 12
-  %iv.2.next = add i32 %iv.2, 13
-  %ext = zext i32 %iv.2.next to i64
-  %tmp56 = add nuw nsw i64 %ext, 1
-  %C6 = icmp sle i32 %tmp54, 65536
-  br i1 %C6, label %loop.2, label %loop.3.ph
-
-loop.3.ph:
-  br label %loop.3
-
-loop.3:
-  %iv.3 = phi i32 [ %iv.2.next, %loop.3.ph ], [ %iv.3.next, %loop.3 ]
-  %iv.3.next = add i32 %iv.3 , 13
-  %C1 = icmp ult i32 %iv.3.next, 65536
-  br i1 %C1, label %loop.3, label %loop.1.latch
-
-loop.1.latch:
-  %ec = icmp ne i32 %iv.1, 9999
-  br i1 %ec, label %loop.1, label %exit
-
-exit:
-  ret void
-}
-
-; CHECK-LABEL: @pr52024(
-; CHECK: vector.body:
-;
-define void @pr52024(ptr %dst, i16 %N) {
-entry:
-  br label %loop.1
-
-loop.1:
-  %iv.1 = phi i16 [ 1, %entry ], [ %iv.1.next, %loop.1.latch ]
-  %iv.1.next = mul i16 %iv.1, 3
-  %exitcond.1 = icmp uge i16 %iv.1.next, 99
-  br i1 %exitcond.1, label %loop.1.latch, label %exit
-
-loop.1.latch:
-  %exitcond.2 = icmp eq i16 %iv.1.next, %N
-  br i1 %exitcond.2, label %loop.2.ph, label %loop.1
-
-loop.2.ph:
-  %iv.1.next.lcssa = phi i16 [ %iv.1.next, %loop.1.latch ]
-  %iv.1.next.ext = sext i16 %iv.1.next.lcssa to i64
-  br label %loop.2.header
-
-loop.2.header:
-  %iv.1.rem = urem i64 100, %iv.1.next.ext
-  %rem.trunc = trunc i64 %iv.1.rem to i16
-  br label %loop.3
-
-loop.3:
-  %iv.3 = phi i32 [ 8, %loop.2.header ], [ %iv.3.next, %loop.3 ]
-  %sub.phi = phi i16 [ 0, %loop.2.header ], [ %sub, %loop.3 ]
-  %sub = sub i16 %sub.phi, %rem.trunc
-  %sub.ext = zext i16 %sub to i32
-  %gep.dst = getelementptr i32, ptr %dst, i32 %iv.3
-  store i32 %sub.ext, ptr %gep.dst
-  %iv.3.next= add nuw nsw i32 %iv.3, 1
-  %exitcond.3 = icmp eq i32 %iv.3.next, 34
-  br i1 %exitcond.3, label %loop.2.latch, label %loop.3
-
-loop.2.latch:
-  %sub.lcssa = phi i16 [ %sub, %loop.3 ]
-  %exitcond = icmp uge i16 %sub.lcssa, 200
-  br i1 %exitcond, label %exit, label %loop.2.header
-
-exit:
-  ret void
-}
diff --git a/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll b/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
index 70772dcd0cdf6..89f15ea5e9187 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-exit-phi-invalidation.ll
@@ -21,10 +21,9 @@ define void @test_pr63368(i1 %c, ptr %A) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT_1:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_1_HEADER:%.*]]
 ; CHECK:       loop.1.header:
-; CHECK-NEXT:    [[IV_1:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP_1_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_1_NEXT:%.*]], [[LOOP_1_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[A]], align 4
 ; CHECK-NEXT:    br i1 [[C]], label [[LOOP_1_LATCH]], label [[LOOP_1_LATCH]]
 ; CHECK:       loop.1.latch:
@@ -65,10 +64,10 @@ define void @test_pr63368(i1 %c, ptr %A) {
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT_2:%.*]], label [[SCALAR_PH2]]
 ; CHECK:       scalar.ph2:
-; CHECK-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi i8 [ [[TMP9]], [[MIDDLE_BLOCK7]] ], [ 0, [[EXIT_1]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ [[TMP9]], [[MIDDLE_BLOCK7]] ], [ 0, [[EXIT_1]] ], [ 0, [[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label [[LOOP_2:%.*]]
 ; CHECK:       loop.2:
-; CHECK-NEXT:    [[IV_2:%.*]] = phi i8 [ [[BC_RESUME_VAL8]], [[SCALAR_PH2]] ], [ [[IV_2_NEXT:%.*]], [[LOOP_2]] ]
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH2]] ], [ [[IV_2_NEXT:%.*]], [[LOOP_2]] ]
 ; CHECK-NEXT:    [[IV_2_NEXT]] = add i8 [[IV_2]], 1
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr i8, ptr [[A]], i8 [[IV_2_NEXT]]
 ; CHECK-NEXT:    store i8 0, ptr [[GEP_A]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll b/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
index b2acc6470da75..77f2fc587cc9e 100644
--- a/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
+++ b/llvm/test/Transforms/LoopVectorize/scev-predicate-reasoning.ll
@@ -96,17 +96,17 @@ define void @integer_induction_wraps_scev_predicate_known(i32 %x, ptr %call, ptr
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP0]], 4
-; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
-; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = mul <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[DOTSPLAT]]
-; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> [[TMP4]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = mul <4 x i64> <i64 0, i64 1, i64 2, i64 3>, [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> [[TMP3]]
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[INDEX]] to i32
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i32 30, [[DOTCAST]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[OFFSET_IDX]]
-; CHECK-NEXT:    store <4 x ptr> [[VECTOR_GEP]], ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr ptr, ptr [[CALL]], i32 [[OFFSET_IDX]]
+; CHECK-NEXT:    store <4 x ptr> [[VECTOR_GEP]], ptr [[TMP4]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 992
 ; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp.ll b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
index 5e48b1f72b111..e4922d3e4f627 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp.ll
@@ -134,18 +134,14 @@ define i32 @select_const_i32_from_icmp(ptr %v, i64 %n) {
 ; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
 ; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
 ; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP8]], 3
-; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP9]], 3
-; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP10]], 3
-; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP11]], 3
-; CHECK-VF1IC4-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP12]], true
-; CHECK-VF1IC4-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP13]], true
-; CHECK-VF1IC4-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP14]], true
-; CHECK-VF1IC4-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP15]], true
-; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP16]]
-; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP17]]
-; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP18]]
-; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP19]]
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP8]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = icmp ne i32 [[TMP9]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP10]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = icmp ne i32 [[TMP11]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP12]]
+; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP13]]
+; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP14]]
+; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP15]]
 ; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-VF1IC4-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF1IC4-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
@@ -510,18 +506,14 @@ define i32 @select_i32_from_icmp(ptr %v, i32 %a, i32 %b, i64 %n) {
 ; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP5]], align 4
 ; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP6]], align 4
 ; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TMP7]], align 4
-; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP8]], 3
-; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP9]], 3
-; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP10]], 3
-; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = icmp eq i32 [[TMP11]], 3
-; CHECK-VF1IC4-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP12]], true
-; CHECK-VF1IC4-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP13]], true
-; CHECK-VF1IC4-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP14]], true
-; CHECK-VF1IC4-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP15]], true
-; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP16]]
-; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP17]]
-; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP18]]
-; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP19]]
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP8]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = icmp ne i32 [[TMP9]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP10]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = icmp ne i32 [[TMP11]], 3
+; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP12]]
+; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP13]]
+; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP14]]
+; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP15]]
 ; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-VF1IC4-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF1IC4-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
@@ -700,18 +692,14 @@ define i32 @select_const_i32_from_fcmp_fast(ptr %v, i64 %n) {
 ; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load float, ptr [[TMP5]], align 4
 ; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP6]], align 4
 ; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load float, ptr [[TMP7]], align 4
-; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = fcmp fast ueq float [[TMP8]], 3.000000e+00
-; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = fcmp fast ueq float [[TMP9]], 3.000000e+00
-; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = fcmp fast ueq float [[TMP10]], 3.000000e+00
-; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = fcmp fast ueq float [[TMP11]], 3.000000e+00
-; CHECK-VF1IC4-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP12]], true
-; CHECK-VF1IC4-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP13]], true
-; CHECK-VF1IC4-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP14]], true
-; CHECK-VF1IC4-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP15]], true
-; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP16]]
-; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP17]]
-; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP18]]
-; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP19]]
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = fcmp fast one float [[TMP8]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = fcmp fast one float [[TMP9]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = fcmp fast one float [[TMP10]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = fcmp fast one float [[TMP11]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP12]]
+; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP13]]
+; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP14]]
+; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP15]]
 ; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-VF1IC4-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF1IC4-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
@@ -890,18 +878,14 @@ define i32 @select_const_i32_from_fcmp(ptr %v, i64 %n) {
 ; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = load float, ptr [[TMP5]], align 4
 ; CHECK-VF1IC4-NEXT:    [[TMP10:%.*]] = load float, ptr [[TMP6]], align 4
 ; CHECK-VF1IC4-NEXT:    [[TMP11:%.*]] = load float, ptr [[TMP7]], align 4
-; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = fcmp ueq float [[TMP8]], 3.000000e+00
-; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = fcmp ueq float [[TMP9]], 3.000000e+00
-; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = fcmp ueq float [[TMP10]], 3.000000e+00
-; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = fcmp ueq float [[TMP11]], 3.000000e+00
-; CHECK-VF1IC4-NEXT:    [[TMP16:%.*]] = xor i1 [[TMP12]], true
-; CHECK-VF1IC4-NEXT:    [[TMP17:%.*]] = xor i1 [[TMP13]], true
-; CHECK-VF1IC4-NEXT:    [[TMP18:%.*]] = xor i1 [[TMP14]], true
-; CHECK-VF1IC4-NEXT:    [[TMP19:%.*]] = xor i1 [[TMP15]], true
-; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP16]]
-; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP17]]
-; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP18]]
-; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP19]]
+; CHECK-VF1IC4-NEXT:    [[TMP12:%.*]] = fcmp one float [[TMP8]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP13:%.*]] = fcmp one float [[TMP9]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP14:%.*]] = fcmp one float [[TMP10]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP15:%.*]] = fcmp one float [[TMP11]], 3.000000e+00
+; CHECK-VF1IC4-NEXT:    [[TMP20]] = or i1 [[VEC_PHI]], [[TMP12]]
+; CHECK-VF1IC4-NEXT:    [[TMP21]] = or i1 [[VEC_PHI1]], [[TMP13]]
+; CHECK-VF1IC4-NEXT:    [[TMP22]] = or i1 [[VEC_PHI2]], [[TMP14]]
+; CHECK-VF1IC4-NEXT:    [[TMP23]] = or i1 [[VEC_PHI3]], [[TMP15]]
 ; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-VF1IC4-NEXT:    [[TMP24:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF1IC4-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
@@ -1049,8 +1033,7 @@ define i32 @select_i32_from_icmp_same_inputs(i32 %a, i32 %b, i64 %n) {
 ; CHECK-VF1IC4:       [[VECTOR_PH]]:
 ; CHECK-VF1IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
 ; CHECK-VF1IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = icmp eq i32 [[A]], 3
-; CHECK-VF1IC4-NEXT:    [[TMP4:%.*]] = xor i1 [[TMP0]], true
+; CHECK-VF1IC4-NEXT:    [[TMP0:%.*]] = icmp ne i32 [[A]], 3
 ; CHECK-VF1IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-VF1IC4:       [[VECTOR_BODY]]:
 ; CHECK-VF1IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1058,10 +1041,10 @@ define i32 @select_i32_from_icmp_same_inputs(i32 %a, i32 %b, i64 %n) {
 ; CHECK-VF1IC4-NEXT:    [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-VF1IC4-NEXT:    [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-VF1IC4-NEXT:    [[VEC_PHI3:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-VF1IC4-NEXT:    [[TMP5]] = or i1 [[VEC_PHI]], [[TMP4]]
-; CHECK-VF1IC4-NEXT:    [[TMP6]] = or i1 [[VEC_PHI1]], [[TMP4]]
-; CHECK-VF1IC4-NEXT:    [[TMP7]] = or i1 [[VEC_PHI2]], [[TMP4]]
-; CHECK-VF1IC4-NEXT:    [[TMP8]] = or i1 [[VEC_PHI3]], [[TMP4]]
+; CHECK-VF1IC4-NEXT:    [[TMP5]] = or i1 [[VEC_PHI]], [[TMP0]]
+; CHECK-VF1IC4-NEXT:    [[TMP6]] = or i1 [[VEC_PHI1]], [[TMP0]]
+; CHECK-VF1IC4-NEXT:    [[TMP7]] = or i1 [[VEC_PHI2]], [[TMP0]]
+; CHECK-VF1IC4-NEXT:    [[TMP8]] = or i1 [[VEC_PHI3]], [[TMP0]]
 ; CHECK-VF1IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-VF1IC4-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-VF1IC4-NEXT:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/select-neg-cond.ll b/llvm/test/Transforms/LoopVectorize/select-neg-cond.ll
index 285c6742a7f5c..4b080ddaa1193 100644
--- a/llvm/test/Transforms/LoopVectorize/select-neg-cond.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-neg-cond.ll
@@ -4,7 +4,7 @@
 define void @neg_cond(ptr noalias %p, ptr noalias %q) {
 ; CHECK-LABEL: define void @neg_cond(
 ; CHECK-SAME: ptr noalias [[P:%.*]], ptr noalias [[Q:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -21,10 +21,9 @@ define void @neg_cond(ptr noalias %p, ptr noalias %q) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
 ; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[P_GEP]], align 4
 ; CHECK-NEXT:    [[Q_GEP:%.*]] = getelementptr i32, ptr [[Q]], i32 [[IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/select-reduction-start-value-may-be-undef-or-poison.ll b/llvm/test/Transforms/LoopVectorize/select-reduction-start-value-may-be-undef-or-poison.ll
index 37d75ffe2c2f0..30585d07a8e60 100644
--- a/llvm/test/Transforms/LoopVectorize/select-reduction-start-value-may-be-undef-or-poison.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-reduction-start-value-may-be-undef-or-poison.ll
@@ -26,12 +26,10 @@ define i64 @pr62565_incoming_value_known_undef(i64 %a, ptr %src) {
 ; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP5]], i64 [[A]], i64 undef
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ undef, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SELECT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 1, [[SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ undef, [[SCALAR_PH]] ], [ [[SELECT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[L]], 1
@@ -86,12 +84,10 @@ define i64 @pr62565_incoming_value_known_poison(i64 %a, ptr %src) {
 ; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP5]], i64 [[A]], i64 poison
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ poison, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SELECT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 1, [[SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ poison, [[SCALAR_PH]] ], [ [[SELECT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[L]], 1
@@ -146,12 +142,10 @@ define i64 @pr62565_incoming_value_may_be_poison(i64 %a, ptr %src, i64 %start) {
 ; CHECK-NEXT:    [[RDX_SELECT:%.*]] = select i1 [[TMP5]], i64 [[A]], i64 [[START]]
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[START]], [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SELECT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 1, [[SCALAR_PH]] ], [ [[ADD:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i64 [ [[START]], [[SCALAR_PH]] ], [ [[SELECT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[L]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/select-reduction.ll b/llvm/test/Transforms/LoopVectorize/select-reduction.ll
index cfc9bb25a9208..cc2ec68d07687 100644
--- a/llvm/test/Transforms/LoopVectorize/select-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-reduction.ll
@@ -38,12 +38,10 @@ define i32 @test(i64 %N, i32 %x) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP3]])
 ; CHECK-NEXT:    br label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[LOOP_PREHEADER]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[EXTRA_ITER]], [[LOOP_PREHEADER]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[NEXT:%.*]] = phi i32 [ [[SEL:%.*]], [[LOOP]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[NEXT:%.*]] = phi i32 [ [[SEL:%.*]], [[LOOP]] ], [ 0, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ [[EXTRA_ITER]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[SEL_COND:%.*]] = icmp sgt i32 [[NEXT]], 10
 ; CHECK-NEXT:    [[SEL]] = select i1 [[SEL_COND]], i32 [[NEXT]], i32 10
 ; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], -1
@@ -94,12 +92,10 @@ define i32 @pr66895_tail_fold_reduction_exit_inst_gets_simplified(i32 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[VEC_PHI]])
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 12, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 12, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], -1
 ; CHECK-NEXT:    [[RED_NEXT]] = mul i32 [[RED]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll
new file mode 100644
index 0000000000000..ad702595fdcfe
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck --check-prefix=VF4IC2 %s
+; RUN: opt -p loop-vectorize -force-vector-width=8 -force-vector-interleave=1 -S %s | FileCheck --check-prefix=VF8IC1 %s
+
+; Test case from https://github.com/llvm/llvm-project/issues/153946.
+; %shr and thus %early.cond will be poison from %iv == 4 onwards.
+; TODO: Make sure the mask being poison does not propagate across lanes in the
+; OR reduction when computing the early exit condition in the vector loop.
+define noundef i32 @f(i32 noundef %g) {
+; VF4IC2-LABEL: define noundef i32 @f(
+; VF4IC2-SAME: i32 noundef [[G:%.*]]) {
+; VF4IC2-NEXT:  [[ENTRY:.*:]]
+; VF4IC2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF4IC2:       [[VECTOR_PH]]:
+; VF4IC2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[G]], i64 0
+; VF4IC2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; VF4IC2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF4IC2:       [[VECTOR_BODY]]:
+; VF4IC2-NEXT:    [[TMP2:%.*]] = ashr <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 8, i32 16, i32 24>
+; VF4IC2-NEXT:    [[TMP3:%.*]] = ashr <4 x i32> [[BROADCAST_SPLAT]], <i32 32, i32 40, i32 48, i32 56>
+; VF4IC2-NEXT:    [[TMP4:%.*]] = icmp ne <4 x i32> [[TMP2]], zeroinitializer
+; VF4IC2-NEXT:    [[TMP5:%.*]] = icmp ne <4 x i32> [[TMP3]], zeroinitializer
+; VF4IC2-NEXT:    [[TMP6:%.*]] = or <4 x i1> [[TMP4]], [[TMP5]]
+; VF4IC2-NEXT:    [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
+; VF4IC2-NEXT:    br label %[[MIDDLE_SPLIT:.*]]
+; VF4IC2:       [[MIDDLE_SPLIT]]:
+; VF4IC2-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; VF4IC2-NEXT:    br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
+; VF4IC2:       [[MIDDLE_BLOCK]]:
+; VF4IC2-NEXT:    br label %[[RETURN:.*]]
+; VF4IC2:       [[VECTOR_EARLY_EXIT]]:
+; VF4IC2-NEXT:    [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 true)
+; VF4IC2-NEXT:    [[TMP10:%.*]] = add i64 4, [[TMP9]]
+; VF4IC2-NEXT:    [[TMP11:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true)
+; VF4IC2-NEXT:    [[TMP12:%.*]] = add i64 0, [[TMP11]]
+; VF4IC2-NEXT:    [[TMP13:%.*]] = icmp ne i64 [[TMP11]], 4
+; VF4IC2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 [[TMP10]]
+; VF4IC2-NEXT:    [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32
+; VF4IC2-NEXT:    [[TMP16:%.*]] = add i32 0, [[TMP15]]
+; VF4IC2-NEXT:    br label %[[RETURN]]
+; VF4IC2:       [[SCALAR_PH]]:
+; VF4IC2-NEXT:    br label %[[LOOP_HEADER:.*]]
+; VF4IC2:       [[LOOP_HEADER]]:
+; VF4IC2-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF4IC2-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[IV]], 3
+; VF4IC2-NEXT:    [[SHR:%.*]] = ashr i32 [[G]], [[MUL]]
+; VF4IC2-NEXT:    [[EARLY_COND:%.*]] = icmp eq i32 [[SHR]], 0
+; VF4IC2-NEXT:    br i1 [[EARLY_COND]], label %[[LOOP_LATCH]], label %[[RETURN]]
+; VF4IC2:       [[LOOP_LATCH]]:
+; VF4IC2-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; VF4IC2-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 8
+; VF4IC2-NEXT:    br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF4IC2:       [[RETURN]]:
+; VF4IC2-NEXT:    [[RES:%.*]] = phi i32 [ [[SHR]], %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ [[TMP16]], %[[VECTOR_EARLY_EXIT]] ]
+; VF4IC2-NEXT:    ret i32 [[RES]]
+;
+; VF8IC1-LABEL: define noundef i32 @f(
+; VF8IC1-SAME: i32 noundef [[G:%.*]]) {
+; VF8IC1-NEXT:  [[ENTRY:.*:]]
+; VF8IC1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; VF8IC1:       [[VECTOR_PH]]:
+; VF8IC1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[G]], i64 0
+; VF8IC1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
+; VF8IC1-NEXT:    br label %[[VECTOR_BODY:.*]]
+; VF8IC1:       [[VECTOR_BODY]]:
+; VF8IC1-NEXT:    [[TMP1:%.*]] = ashr <8 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 8, i32 16, i32 24, i32 32, i32 40, i32 48, i32 56>
+; VF8IC1-NEXT:    [[TMP2:%.*]] = icmp ne <8 x i32> [[TMP1]], zeroinitializer
+; VF8IC1-NEXT:    [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP2]])
+; VF8IC1-NEXT:    br label %[[MIDDLE_SPLIT:.*]]
+; VF8IC1:       [[MIDDLE_SPLIT]]:
+; VF8IC1-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7
+; VF8IC1-NEXT:    br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
+; VF8IC1:       [[MIDDLE_BLOCK]]:
+; VF8IC1-NEXT:    br label %[[RETURN:.*]]
+; VF8IC1:       [[VECTOR_EARLY_EXIT]]:
+; VF8IC1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP2]], i1 true)
+; VF8IC1-NEXT:    [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32
+; VF8IC1-NEXT:    [[TMP7:%.*]] = add i32 0, [[TMP6]]
+; VF8IC1-NEXT:    br label %[[RETURN]]
+; VF8IC1:       [[SCALAR_PH]]:
+; VF8IC1-NEXT:    br label %[[LOOP_HEADER:.*]]
+; VF8IC1:       [[LOOP_HEADER]]:
+; VF8IC1-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF8IC1-NEXT:    [[MUL:%.*]] = shl nuw nsw i32 [[IV]], 3
+; VF8IC1-NEXT:    [[SHR:%.*]] = ashr i32 [[G]], [[MUL]]
+; VF8IC1-NEXT:    [[EARLY_COND:%.*]] = icmp eq i32 [[SHR]], 0
+; VF8IC1-NEXT:    br i1 [[EARLY_COND]], label %[[LOOP_LATCH]], label %[[RETURN]]
+; VF8IC1:       [[LOOP_LATCH]]:
+; VF8IC1-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; VF8IC1-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 8
+; VF8IC1-NEXT:    br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF8IC1:       [[RETURN]]:
+; VF8IC1-NEXT:    [[RES:%.*]] = phi i32 [ [[SHR]], %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VECTOR_EARLY_EXIT]] ]
+; VF8IC1-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %mul = shl nuw nsw i32 %iv, 3
+  %shr = ashr i32 %g, %mul
+  %early.cond = icmp eq i32 %shr, 0
+  br i1 %early.cond, label %loop.latch, label %return
+
+loop.latch:
+  %iv.next = add nuw nsw i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 8
+  br i1 %ec, label %return, label %loop.header
+
+return:
+  %res = phi i32 [ %shr, %loop.latch ], [ %iv, %loop.header ]
+  ret i32 %res
+}
+
+
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
index f329a18f3eaea..df40ba0d4ac3e 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
@@ -31,10 +31,9 @@ define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_si
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX1]], [[TMP8]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP1:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
index c648bedabc05d..5b9e75a9f7eb0 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
@@ -5,7 +5,7 @@ declare void @init_mem(ptr, i64);
 
 define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
 ; VF4IC4-LABEL: define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
-; VF4IC4-NEXT:  [[ENTRY:.*]]:
+; VF4IC4-NEXT:  [[ENTRY:.*:]]
 ; VF4IC4-NEXT:    [[SRC:%.*]] = alloca [128 x i32], align 4
 ; VF4IC4-NEXT:    call void @init_mem(ptr [[SRC]])
 ; VF4IC4-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -40,10 +40,9 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
 ; VF4IC4:       [[VECTOR_EARLY_EXIT]]:
 ; VF4IC4-NEXT:    br label %[[E1:.*]]
 ; VF4IC4:       [[SCALAR_PH]]:
-; VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF4IC4-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; VF4IC4:       [[LOOP_HEADER]]:
-; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
+; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[INC:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[SCALAR_PH]] ]
 ; VF4IC4-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
 ; VF4IC4-NEXT:    [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
 ; VF4IC4-NEXT:    [[C_1:%.*]] = icmp eq i32 [[L]], 10
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-only.ll
new file mode 100644
index 0000000000000..64cca5196a8d8
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-only.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s | FileCheck %s
+
+
+; FIXME: currently the live-outs are not handled correctly.
+; Test for https://github.com/llvm/llvm-project/issues/154967.
+define i8 @iv_used_in_exit_with_math(i8 noundef %g) {
+; CHECK-LABEL: define i8 @iv_used_in_exit_with_math(
+; CHECK-SAME: i8 noundef [[G:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = trunc i32 [[INDEX]] to i8
+; CHECK-NEXT:    [[TMP0:%.*]] = add i8 [[OFFSET_IDX]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i8 1, [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i8 1, [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP1]], [[G]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[TMP2]], [[G]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i8 [[TMP4]], 0
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_SPLIT]]:
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[RETURN:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT:    br label %[[RETURN]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[S:%.*]] = shl nuw i8 1, [[IV]]
+; CHECK-NEXT:    [[A:%.*]] = and i8 [[S]], [[G]]
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[A]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[RETURN]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i8 [[IV_NEXT]], 4
+; CHECK-NEXT:    br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[RETURN]]:
+; CHECK-NEXT:    [[RES_IV1:%.*]] = phi i8 [ 32, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ 32, %[[MIDDLE_BLOCK]] ], [ [[OFFSET_IDX]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[RES_IV2:%.*]] = phi i8 [ 0, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[OFFSET_IDX]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[RES:%.*]] = add i8 [[RES_IV1]], [[RES_IV2]]
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %s = shl nuw i8 1, %iv
+  %a = and i8 %s, %g
+  %c = icmp eq i8 %a, 0
+  br i1 %c, label %loop.latch, label %return
+
+loop.latch:
+  %iv.next = add nuw nsw i8 %iv, 1
+  %ec = icmp eq i8 %iv.next, 4
+  br i1 %ec, label %return, label %loop.header
+
+return:
+  %res.iv1 = phi i8 [ 32, %loop.latch ], [ %iv, %loop.header ]
+  %res.iv2 = phi i8 [ 0, %loop.latch ], [ %iv, %loop.header ]
+  %res = add i8 %res.iv1, %res.iv2
+  ret i8 %res
+}
+
+define i32 @iv_used_in_exit_with_loads(ptr align 4 dereferenceable(128) %src) {
+; CHECK-LABEL: define i32 @iv_used_in_exit_with_loads(
+; CHECK-SAME: ptr align 4 dereferenceable(128) [[SRC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; CHECK-NEXT:    [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    br i1 [[TMP11]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_SPLIT]]:
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[RETURN:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT:    br label %[[RETURN]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[L]], 0
+; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_LATCH]], label %[[RETURN]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 32
+; CHECK-NEXT:    br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[RETURN]]:
+; CHECK-NEXT:    [[RES_IV1:%.*]] = phi i32 [ 32, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ 32, %[[MIDDLE_BLOCK]] ], [ [[INDEX]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[RES_IV2:%.*]] = phi i32 [ 0, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[INDEX]], %[[VECTOR_EARLY_EXIT]] ]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[RES_IV1]], [[RES_IV2]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+entry:
+  br label %loop.header
+
+loop.header:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %gep = getelementptr inbounds i32, ptr %src, i32 %iv
+  %l = load i32, ptr %gep
+  %c = icmp eq i32 %l, 0
+  br i1 %c, label %loop.latch, label %return
+
+loop.latch:
+  %iv.next = add nuw nsw i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, 32
+  br i1 %ec, label %return, label %loop.header
+
+return:
+  %res.iv1 = phi i32 [ 32, %loop.latch ], [ %iv, %loop.header ]
+  %res.iv2 = phi i32 [ 0, %loop.latch ], [ %iv, %loop.header ]
+  %res = add i32 %res.iv1, %res.iv2
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
index 3f51c72a6d3de..678b171832c36 100644
--- a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll
@@ -40,10 +40,9 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() {
 ; VF4IC4:       vector.early.exit:
 ; VF4IC4-NEXT:    br label [[E1:%.*]]
 ; VF4IC4:       scalar.ph:
-; VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; VF4IC4-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; VF4IC4:       loop.header:
-; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[INC:%.*]], [[LOOP_LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[INC:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[SCALAR_PH]] ]
 ; VF4IC4-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
 ; VF4IC4-NEXT:    [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
 ; VF4IC4-NEXT:    [[C_1:%.*]] = icmp eq i32 [[L]], 10
@@ -149,10 +148,9 @@ define i64 @same_exit_block_pre_inc_use1() {
 ; VF4IC4-NEXT:    [[TMP10:%.*]] = add i64 3, [[TMP9]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
 ; VF4IC4:       scalar.ph:
-; VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; VF4IC4-NEXT:    br label [[LOOP:%.*]]
 ; VF4IC4:       loop:
-; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; VF4IC4-NEXT:    [[GEP_P1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
 ; VF4IC4-NEXT:    [[LD1:%.*]] = load i8, ptr [[GEP_P1]], align 1
 ; VF4IC4-NEXT:    [[GEP_P2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
@@ -247,10 +245,9 @@ define ptr @same_exit_block_pre_inc_use1_ivptr() {
 ; VF4IC4-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP7]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
 ; VF4IC4:       scalar.ph:
-; VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[P1]], [[ENTRY:%.*]] ]
 ; VF4IC4-NEXT:    br label [[LOOP:%.*]]
 ; VF4IC4:       loop:
-; VF4IC4-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; VF4IC4-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[P1]], [[SCALAR_PH]] ]
 ; VF4IC4-NEXT:    [[LD1:%.*]] = load i8, ptr [[PTR]], align 1
 ; VF4IC4-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], 72
 ; VF4IC4-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
@@ -348,10 +345,9 @@ define i64 @same_exit_block_post_inc_use() {
 ; VF4IC4-NEXT:    [[TMP10:%.*]] = add i64 3, [[TMP9]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
 ; VF4IC4:       scalar.ph:
-; VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; VF4IC4-NEXT:    br label [[LOOP:%.*]]
 ; VF4IC4:       loop:
-; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; VF4IC4-NEXT:    [[GEP_P1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
 ; VF4IC4-NEXT:    [[LD1:%.*]] = load i8, ptr [[GEP_P1]], align 1
 ; VF4IC4-NEXT:    [[GEP_P2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
@@ -456,10 +452,9 @@ define i64 @diff_exit_block_pre_inc_use1() {
 ; VF4IC4-NEXT:    [[TMP10:%.*]] = add i64 3, [[TMP9]]
 ; VF4IC4-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; VF4IC4:       scalar.ph:
-; VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; VF4IC4-NEXT:    br label [[LOOP:%.*]]
 ; VF4IC4:       loop:
-; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; VF4IC4-NEXT:    [[GEP_P1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
 ; VF4IC4-NEXT:    [[LD1:%.*]] = load i8, ptr [[GEP_P1]], align 1
 ; VF4IC4-NEXT:    [[GEP_P2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
@@ -571,10 +566,9 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; VF4IC4-NEXT:    [[TMP10:%.*]] = add i64 3, [[TMP9]]
 ; VF4IC4-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; VF4IC4:       scalar.ph:
-; VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; VF4IC4-NEXT:    br label [[LOOP:%.*]]
 ; VF4IC4:       loop:
-; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; VF4IC4-NEXT:    [[GEP_P1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
 ; VF4IC4-NEXT:    [[LD1:%.*]] = load i8, ptr [[GEP_P1]], align 1
 ; VF4IC4-NEXT:    [[GEP_P2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
@@ -823,10 +817,9 @@ define i8 @same_exit_block_use_loaded_value() {
 ; VF4IC4-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i8 [[TMP40]], i8 [[TMP38]]
 ; VF4IC4-NEXT:    br label [[LOOP_END]]
 ; VF4IC4:       scalar.ph:
-; VF4IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; VF4IC4-NEXT:    br label [[LOOP:%.*]]
 ; VF4IC4:       loop:
-; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; VF4IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[SCALAR_PH]] ]
 ; VF4IC4-NEXT:    [[GEP_P1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
 ; VF4IC4-NEXT:    [[LD1:%.*]] = load i8, ptr [[GEP_P1]], align 1
 ; VF4IC4-NEXT:    [[GEP_P2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll b/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll
index 842ff910c89d3..dd3521fd99c87 100644
--- a/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll
+++ b/llvm/test/Transforms/LoopVectorize/single-value-blend-phis.ll
@@ -31,10 +31,9 @@ define void @single_incoming_phi_no_blend_mask(i64 %a, i64 %b) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i16
 ; CHECK-NEXT:    br label [[LOOP_COND:%.*]]
 ; CHECK:       loop.cond:
@@ -104,9 +103,8 @@ define void @single_incoming_phi_with_blend_mask(i64 %a, i64 %b) {
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i16>, ptr [[TMP4]], align 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp sle <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <2 x i1> [[TMP3]], <2 x i1> [[TMP6]], <2 x i1> zeroinitializer
-; CHECK-NEXT:    [[TMP8:%.*]] = xor <2 x i1> [[TMP3]], splat (i1 true)
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP7]], <2 x i16> [[WIDE_LOAD]], <2 x i16> splat (i16 1)
-; CHECK-NEXT:    [[PREDPHI1:%.*]] = select <2 x i1> [[TMP8]], <2 x i16> zeroinitializer, <2 x i16> [[PREDPHI]]
+; CHECK-NEXT:    [[PREDPHI1:%.*]] = select <2 x i1> [[TMP3]], <2 x i16> [[PREDPHI]], <2 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store <2 x i16> [[PREDPHI1]], ptr [[TMP9]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -116,10 +114,9 @@ define void @single_incoming_phi_with_blend_mask(i64 %a, i64 %b) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i16
 ; CHECK-NEXT:    [[CMP_A:%.*]] = icmp ugt i64 [[IV]], [[A]]
 ; CHECK-NEXT:    br i1 [[CMP_A]], label [[LOOP_COND:%.*]], label [[LOOP_LATCH]]
@@ -206,10 +203,9 @@ define void @multiple_incoming_phi_with_blend_mask(i64 %a, ptr noalias %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i16
 ; CHECK-NEXT:    [[IV_TRUNC_2:%.*]] = trunc i64 [[IV]] to i16
 ; CHECK-NEXT:    [[CMP_A:%.*]] = icmp ugt i64 [[IV]], [[A]]
@@ -292,9 +288,8 @@ define void @single_incoming_needs_predication(i64 %a, i64 %b) {
 ; CHECK-NEXT:    [[TMP14:%.*]] = phi <2 x i16> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ]
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp sle <2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = select <2 x i1> [[TMP2]], <2 x i1> [[TMP15]], <2 x i1> zeroinitializer
-; CHECK-NEXT:    [[TMP17:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true)
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP16]], <2 x i16> [[TMP14]], <2 x i16> splat (i16 1)
-; CHECK-NEXT:    [[PREDPHI3:%.*]] = select <2 x i1> [[TMP17]], <2 x i16> zeroinitializer, <2 x i16> [[PREDPHI]]
+; CHECK-NEXT:    [[PREDPHI3:%.*]] = select <2 x i1> [[TMP2]], <2 x i16> [[PREDPHI]], <2 x i16> zeroinitializer
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds [32 x i16], ptr @dst, i16 0, i64 [[INDEX]]
 ; CHECK-NEXT:    store <2 x i16> [[PREDPHI3]], ptr [[TMP18]], align 2
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
@@ -304,10 +299,9 @@ define void @single_incoming_needs_predication(i64 %a, i64 %b) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[IV_TRUNC:%.*]] = trunc i64 [[IV]] to i16
 ; CHECK-NEXT:    [[CMP_A:%.*]] = icmp ugt i64 [[IV]], [[A]]
 ; CHECK-NEXT:    br i1 [[CMP_A]], label [[LOOP_COND:%.*]], label [[LOOP_LATCH]]
@@ -379,10 +373,9 @@ define void @duplicated_incoming_blocks_blend(i32 %x, ptr %ptr) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[ADD_I:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[ADD_I:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[C_0:%.*]] = icmp ugt i32 [[IV]], [[X:%.*]]
 ; CHECK-NEXT:    br i1 [[C_0]], label [[LOOP_LATCH]], label [[LOOP_LATCH]]
 ; CHECK:       loop.latch:
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
index 2c0a6f1b032c2..b3451704ea512 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll
@@ -34,10 +34,9 @@ define i64 @same_exit_block_phi_of_consts() {
 ; CHECK:       vector.early.exit:
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -109,10 +108,9 @@ define i64 @diff_exit_block_phi_of_consts() {
 ; CHECK:       vector.early.exit:
 ; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -292,10 +290,9 @@ define i32 @diff_blocks_invariant_early_exit_cond(ptr %s) {
 ; CHECK:       vector.early.exit:
 ; CHECK-NEXT:    br label [[EARLY_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ -10, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[IND:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IND_NEXT:%.*]], [[FOR_INC:%.*]] ]
+; CHECK-NEXT:    [[IND:%.*]] = phi i32 [ -10, [[SCALAR_PH]] ], [ [[IND_NEXT:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    br i1 [[COND]], label [[FOR_INC]], label [[EARLY_EXIT]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[IND_NEXT]] = add nsw i32 [[IND]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
index 940e3980a01a8..f4b35c779a4b5 100644
--- a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
+++ b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll
@@ -36,10 +36,9 @@ define i64 @same_exit_block_pre_inc_use1() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -116,12 +115,10 @@ define i32 @same_exit_block_pre_inc_use1_iv64_endi32_step2() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i32 9, [[TMP11]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi i32 [ 9, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INDEX2:%.*]] = phi i32 [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX2:%.*]] = phi i32 [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ 9, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -198,14 +195,11 @@ define i32 @same_exit_block_pre_inc_use1_iv128_endi32_step2() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i32 9, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i128 [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL3:%.*]] = phi i32 [ 9, [[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[P1]], [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i128 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INDEX2:%.*]] = phi i32 [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ [[BC_RESUME_VAL3]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP_INC]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i128 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX2:%.*]] = phi i32 [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ 9, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP_INC]] ], [ [[P1]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[PTR]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], 3
 ; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
@@ -280,12 +274,10 @@ define float @same_exit_block_pre_inc_use1_iv64_endf32() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = fadd fast float 9.000000e+00, [[TMP11]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL4:%.*]] = phi float [ 9.000000e+00, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INDEX2:%.*]] = phi float [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX2:%.*]] = phi float [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ 9.000000e+00, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -364,12 +356,10 @@ define ptr @same_exit_block_pre_inc_use1_iv64_endptr() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP20]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL8:%.*]] = phi ptr [ [[P2]], [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INDEX2:%.*]] = phi ptr [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ [[BC_RESUME_VAL8]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX2:%.*]] = phi ptr [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ [[P2]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -443,10 +433,9 @@ define ptr @same_exit_block_pre_inc_use1_ivptr() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP8]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[P1]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[P1]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[PTR]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], 72
 ; CHECK-NEXT:    br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]]
@@ -517,10 +506,9 @@ define i64 @same_exit_block_pre_inc1_use_inv_cond(i1 %cond) {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP11]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -597,10 +585,9 @@ define i64 @same_exit_block_pre_inc_use1_gep_two_indices() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i8], ptr [[P1]], i64 0, i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x i8], ptr [[P2]], i64 0, i64 [[INDEX]]
@@ -675,10 +662,9 @@ define i64 @same_exit_block_pre_inc_use1_alloca_diff_type() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -750,10 +736,9 @@ define i64 @same_exit_block_pre_inc_use2() {
 ; CHECK:       vector.early.exit:
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -828,10 +813,9 @@ define i64 @same_exit_block_pre_inc_use3() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -907,10 +891,9 @@ define i64 @same_exit_block_pre_inc_use4() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP8]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i64, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ult i64 [[INDEX]], [[LD1]]
@@ -981,10 +964,9 @@ define i64 @same_exit_block_post_inc_use() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -1056,10 +1038,9 @@ define ptr @same_exit_block_post_inc_use1_ivptr() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = getelementptr i8, ptr [[P1]], i64 [[TMP9]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[P1]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[PTR:%.*]] = phi ptr [ [[PTR_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[P1]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[PTR]], align 1
 ; CHECK-NEXT:    [[PTR_NEXT]] = getelementptr inbounds i8, ptr [[PTR]], i64 1
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp eq i8 [[LD1]], 72
@@ -1128,10 +1109,9 @@ define i64 @same_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP11]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -1206,10 +1186,9 @@ define i64 @diff_exit_block_pre_inc_use1() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -1288,10 +1267,9 @@ define i64 @diff_exit_block_pre_inc_use2() {
 ; CHECK:       vector.early.exit:
 ; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -1373,10 +1351,9 @@ define i64 @diff_exit_block_pre_inc_use3() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -1456,10 +1433,9 @@ define i64 @diff_exit_block_post_inc_use1() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
@@ -1542,10 +1518,9 @@ define i64 @diff_exit_block_post_inc_use2() {
 ; CHECK-NEXT:    [[TMP21:%.*]] = add i64 3, [[TMP11]]
 ; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
@@ -1630,12 +1605,10 @@ define i64 @diff_exit_block_post_inc_use3(i64 %start) {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 [[START]], [[TMP12]]
 ; CHECK-NEXT:    br label [[LOOP_EARLY_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL5:%.*]] = phi i64 [ [[START]], [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
-; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ [[BC_RESUME_VAL5]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ [[INDEX2_NEXT:%.*]], [[LOOP_INC]] ], [ [[START]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 1
 ; CHECK-NEXT:    [[INDEX2_NEXT]] = add i64 [[INDEX2]], 1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
@@ -1719,10 +1692,9 @@ define i64 @loop_contains_safe_call() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP9]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load float, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[SQRT:%.*]] = tail call fast float @llvm.sqrt.f32(float [[LD1]])
@@ -1794,10 +1766,9 @@ define i64 @loop_contains_safe_div() {
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP9]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[DIV:%.*]] = udiv i32 [[LD1]], 20000
@@ -1870,10 +1841,9 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align(
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP11]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[LD1]], 1
@@ -2077,10 +2047,9 @@ define i64 @same_exit_block_pre_inc_use1_deref_ptrs(ptr dereferenceable(1024) %p
 ; CHECK-NEXT:    [[EARLY_EXIT_VALUE:%.*]] = add i64 3, [[TMP10]]
 ; CHECK-NEXT:    br label [[LOOP_END]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 3, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 3, [[SCALAR_PH]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
diff --git a/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll b/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll
index bf86cbd601f44..ad5bfb8d9e965 100644
--- a/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll
@@ -43,12 +43,10 @@ define void @pr75298_store_reduction_value_in_folded_loop(i64 %iv.start) optsize
 ; CHECK-NEXT:    store i32 [[TMP6]], ptr @a, align 4
 ; CHECK-NEXT:    br label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IV_START]], [[PH]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[PH]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[IV_START]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr @c, align 4
 ; CHECK-NEXT:    [[RED_NEXT]] = xor i32 [[RED]], [[L]]
 ; CHECK-NEXT:    store i32 [[RED_NEXT]], ptr @a, align 4
diff --git a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
index eefa3da97a4bc..76ea53572292a 100644
--- a/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/strict-fadd-interleave-only.ll
@@ -25,12 +25,10 @@ define float @pr70988() {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[RDX_NEXT]] = fadd contract float [[RDX]], 1.000000e+00
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw nsw i32 [[INDEX]], 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[INDEX_NEXT]], 1021
@@ -60,12 +58,10 @@ define float @pr70988() {
 ; CHECK-ALM:       middle.block:
 ; CHECK-ALM-NEXT:    br label [[EXIT:%.*]]
 ; CHECK-ALM:       scalar.ph:
-; CHECK-ALM-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; CHECK-ALM-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ALM-NEXT:    br label [[LOOP:%.*]]
 ; CHECK-ALM:       loop:
-; CHECK-ALM-NEXT:    [[INDEX:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
-; CHECK-ALM-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-ALM-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-ALM-NEXT:    [[RDX:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-ALM-NEXT:    [[RDX_NEXT]] = fadd contract float [[RDX]], 1.000000e+00
 ; CHECK-ALM-NEXT:    [[INDEX_NEXT]] = add nuw nsw i32 [[INDEX]], 1
 ; CHECK-ALM-NEXT:    [[COND:%.*]] = icmp ult i32 [[INDEX_NEXT]], 1021
@@ -129,12 +125,10 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NARROW:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[NARROW:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RDX:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[NARROW]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i32 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
@@ -181,12 +175,10 @@ define float @pr72720reduction_using_active_lane_mask(ptr %src) {
 ; CHECK-ALM:       middle.block:
 ; CHECK-ALM-NEXT:    br label [[EXIT:%.*]]
 ; CHECK-ALM:       scalar.ph:
-; CHECK-ALM-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; CHECK-ALM-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ALM-NEXT:    br label [[LOOP:%.*]]
 ; CHECK-ALM:       loop:
-; CHECK-ALM-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NARROW:%.*]], [[LOOP]] ]
-; CHECK-ALM-NEXT:    [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], [[LOOP]] ]
+; CHECK-ALM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[NARROW:%.*]], [[LOOP]] ]
+; CHECK-ALM-NEXT:    [[RDX:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-ALM-NEXT:    [[NARROW]] = add nuw nsw i32 [[IV]], 1
 ; CHECK-ALM-NEXT:    [[GEP:%.*]] = getelementptr float, ptr [[SRC]], i32 [[IV]]
 ; CHECK-ALM-NEXT:    [[L:%.*]] = load float, ptr [[GEP]], align 4
@@ -239,12 +231,10 @@ define float @fadd_reduction_with_live_in(float %inc) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[SUM:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[INC]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 1000
@@ -275,12 +265,10 @@ define float @fadd_reduction_with_live_in(float %inc) {
 ; CHECK-ALM:       middle.block:
 ; CHECK-ALM-NEXT:    br label [[EXIT:%.*]]
 ; CHECK-ALM:       scalar.ph:
-; CHECK-ALM-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
-; CHECK-ALM-NEXT:    [[BC_MERGE_RDX:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ]
 ; CHECK-ALM-NEXT:    br label [[LOOP:%.*]]
 ; CHECK-ALM:       loop:
-; CHECK-ALM-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-ALM-NEXT:    [[SUM:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP]] ]
+; CHECK-ALM-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-ALM-NEXT:    [[SUM:%.*]] = phi float [ 0.000000e+00, [[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-ALM-NEXT:    [[SUM_NEXT]] = fadd float [[SUM]], [[INC]]
 ; CHECK-ALM-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-ALM-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV]], 1000
diff --git a/llvm/test/Transforms/LoopVectorize/strided-accesses-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/strided-accesses-interleave-only.ll
index 971921a9c1d60..8e47f19efa24f 100644
--- a/llvm/test/Transforms/LoopVectorize/strided-accesses-interleave-only.ll
+++ b/llvm/test/Transforms/LoopVectorize/strided-accesses-interleave-only.ll
@@ -23,10 +23,9 @@ define void @test_variable_stride(ptr %dst, i32 %scale) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IDX:%.*]] = mul i32 [[IV]], [[SCALE]]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i16, ptr [[DST]], i32 [[IDX]]
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[GEP]], align 2
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll
index 3cf8b3f4bf2b7..7bb54e4c7902d 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll
@@ -4,7 +4,7 @@
 define i32 @test(ptr %vf1, i64 %n) {
 ; CHECK-LABEL: define i32 @test(
 ; CHECK-SAME: ptr [[VF1:%.*]], i64 [[N:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -55,10 +55,9 @@ define i32 @test(ptr %vf1, i64 %n) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[FOR_BODY:.*]]
 ; CHECK:       [[FOR_BODY]]:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP18:%.*]] = alloca i8, i64 [[N]], align 16
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds ptr, ptr [[VF1]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store ptr [[TMP18]], ptr [[ARRAYIDX]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-optimize-vector-induction-width.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-optimize-vector-induction-width.ll
index efc2b8d819431..1bb0a3075a8c0 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-optimize-vector-induction-width.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-optimize-vector-induction-width.ll
@@ -4,7 +4,7 @@
 define void @canonical_small_tc_i8(ptr nocapture noundef writeonly %p) {
 ; CHECK-LABEL: define void @canonical_small_tc_i8(
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -35,10 +35,9 @@ define void @canonical_small_tc_i8(ptr nocapture noundef writeonly %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_IV:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    store i16 1, ptr [[P_IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -65,7 +64,7 @@ end:
 define void @canonical_upper_limit_i8(ptr nocapture noundef writeonly %p) {
 ; CHECK-LABEL: define void @canonical_upper_limit_i8(
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -96,10 +95,9 @@ define void @canonical_upper_limit_i8(ptr nocapture noundef writeonly %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_IV:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    store i16 1, ptr [[P_IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -126,7 +124,7 @@ end:
 define void @canonical_lower_limit_i16(ptr nocapture noundef writeonly %p) {
 ; CHECK-LABEL: define void @canonical_lower_limit_i16(
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -157,10 +155,9 @@ define void @canonical_lower_limit_i16(ptr nocapture noundef writeonly %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_IV:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    store i16 1, ptr [[P_IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -187,7 +184,7 @@ end:
 define void @canonical_upper_limit_i16(ptr nocapture noundef writeonly %p) {
 ; CHECK-LABEL: define void @canonical_upper_limit_i16(
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -218,10 +215,9 @@ define void @canonical_upper_limit_i16(ptr nocapture noundef writeonly %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_IV:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    store i16 1, ptr [[P_IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -248,7 +244,7 @@ end:
 define void @canonical_lower_limit_i32(ptr nocapture noundef writeonly %p) {
 ; CHECK-LABEL: define void @canonical_lower_limit_i32(
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -279,10 +275,9 @@ define void @canonical_lower_limit_i32(ptr nocapture noundef writeonly %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_IV:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    store i16 1, ptr [[P_IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -309,7 +304,7 @@ end:
 define void @canonical_upper_limit_i32(ptr nocapture noundef writeonly %p) {
 ; CHECK-LABEL: define void @canonical_upper_limit_i32(
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -340,10 +335,9 @@ define void @canonical_upper_limit_i32(ptr nocapture noundef writeonly %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_IV:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    store i16 1, ptr [[P_IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -370,7 +364,7 @@ end:
 define void @canonical_lower_limit_i64(ptr nocapture noundef writeonly %p) {
 ; CHECK-LABEL: define void @canonical_lower_limit_i64(
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -401,10 +395,9 @@ define void @canonical_lower_limit_i64(ptr nocapture noundef writeonly %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_IV:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    store i16 1, ptr [[P_IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -431,7 +424,7 @@ end:
 define void @canonical_upper_limit_i64(ptr nocapture noundef writeonly %p) {
 ; CHECK-LABEL: define void @canonical_upper_limit_i64(
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -462,10 +455,9 @@ define void @canonical_upper_limit_i64(ptr nocapture noundef writeonly %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_IV:%.*]] = getelementptr inbounds i16, ptr [[P]], i64 [[IV]]
 ; CHECK-NEXT:    store i16 1, ptr [[P_IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
@@ -492,7 +484,7 @@ end:
 define void @canonical_lower_limit_i128(ptr nocapture noundef writeonly %p) {
 ; CHECK-LABEL: define void @canonical_lower_limit_i128(
 ; CHECK-SAME: ptr noundef writeonly captures(none) [[P:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -523,10 +515,9 @@ define void @canonical_lower_limit_i128(ptr nocapture noundef writeonly %p) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[END:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i256 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i256 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i256 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_IV:%.*]] = getelementptr inbounds i16, ptr [[P]], i256 [[IV]]
 ; CHECK-NEXT:    store i16 1, ptr [[P_IV]], align 2
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i256 [[IV]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-switch.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-switch.ll
index 222c1eeb6e443..dd1d779b92a3b 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-switch.ll
@@ -5,7 +5,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128-Fn32"
 define void @tail_fold_switch(ptr %dst, i32 %0) {
 ; CHECK-LABEL: define void @tail_fold_switch(
 ; CHECK-SAME: ptr [[DST:%.*]], i32 [[TMP0:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i64 0
@@ -56,10 +56,9 @@ define void @tail_fold_switch(ptr %dst, i32 %0) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    switch i32 [[TMP0]], label %[[LOOP_LATCH]] [
 ; CHECK-NEXT:      i32 0, label %[[LOOP_LATCH]]
 ; CHECK-NEXT:      i32 1, label %[[IF_THEN:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index 13d5be1b94d15..d12efe86ddb1f 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -55,12 +55,11 @@ define void @VF1-VPlanExe(ptr %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[DST_PTR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 0, ptr [[DST_PTR]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
@@ -135,12 +134,11 @@ define void @VF1-VPWidenCanonicalIVRecipeExe(ptr %ptr1) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[PTR1]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    ret void
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[ADDR:%.*]] = phi ptr [ [[PTR:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[ADDR:%.*]] = phi ptr [ [[PTR:%.*]], [[FOR_BODY]] ], [ [[PTR1]], [[SCALAR_PH]] ]
 ; CHECK-NEXT:    store double 0.000000e+00, ptr [[ADDR]], align 8
 ; CHECK-NEXT:    [[PTR]] = getelementptr inbounds double, ptr [[ADDR]], i64 1
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq ptr [[PTR]], [[PTR2]]
diff --git a/llvm/test/Transforms/LoopVectorize/trunc-extended-icmps.ll b/llvm/test/Transforms/LoopVectorize/trunc-extended-icmps.ll
index a687ecc33af54..9e94768fc2cbc 100644
--- a/llvm/test/Transforms/LoopVectorize/trunc-extended-icmps.ll
+++ b/llvm/test/Transforms/LoopVectorize/trunc-extended-icmps.ll
@@ -145,10 +145,9 @@ define void @ext_cmp(ptr %src.1, ptr %src.2, ptr noalias %dst) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds i16, ptr [[SRC_1]], i64 [[IV]]
 ; CHECK-NEXT:    [[I2:%.*]] = load i16, ptr [[GEP_SRC_1]], align 2
 ; CHECK-NEXT:    [[I3:%.*]] = sext i16 [[I2]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/trunc-loads-p16.ll b/llvm/test/Transforms/LoopVectorize/trunc-loads-p16.ll
index 66dc785d95f46..ac9b03567aab8 100644
--- a/llvm/test/Transforms/LoopVectorize/trunc-loads-p16.ll
+++ b/llvm/test/Transforms/LoopVectorize/trunc-loads-p16.ll
@@ -28,10 +28,9 @@ define void @pr77468(ptr noalias %src, ptr noalias %dst, i1 %x) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr i32, ptr [[SRC]], i16 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 1
 ; CHECK-NEXT:    [[X_EXT:%.*]] = zext i1 [[X]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/trunc-shifts.ll b/llvm/test/Transforms/LoopVectorize/trunc-shifts.ll
index 10e9ae80beb7b..9e710763be131 100644
--- a/llvm/test/Transforms/LoopVectorize/trunc-shifts.ll
+++ b/llvm/test/Transforms/LoopVectorize/trunc-shifts.ll
@@ -26,10 +26,9 @@ define void @test_pr47927_lshr_const_shift_ops(ptr %dst, i32 %f) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[L:%.*]] = lshr i32 [[F]], 18
 ; CHECK-NEXT:    [[L_T:%.*]] = trunc i32 [[L]] to i8
 ; CHECK-NEXT:    [[IV_EXT:%.*]] = zext i8 [[IV]] to i64
@@ -84,10 +83,9 @@ define void @test_shl_const_shift_ops(ptr %dst, i32 %f) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[L:%.*]] = shl i32 [[F]], 18
 ; CHECK-NEXT:    [[L_T:%.*]] = trunc i32 [[L]] to i8
 ; CHECK-NEXT:    [[IV_EXT:%.*]] = zext i8 [[IV]] to i64
@@ -142,10 +140,9 @@ define void @test_ashr_const_shift_ops(ptr %dst, i32 %f) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[L:%.*]] = ashr i32 [[F]], 18
 ; CHECK-NEXT:    [[L_T:%.*]] = trunc i32 [[L]] to i8
 ; CHECK-NEXT:    [[IV_EXT:%.*]] = zext i8 [[IV]] to i64
@@ -200,10 +197,9 @@ define void @test_shl_const_shifted_op(ptr %dst, i32 %f) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IV_EXT:%.*]] = zext i8 [[IV]] to i64
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST]], i64 [[IV_EXT]]
 ; CHECK-NEXT:    [[LV:%.*]] = load i8, ptr [[GEP]], align 1
@@ -263,10 +259,9 @@ define void @test_lshr_by_18(ptr %A) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IV_EXT:%.*]] = zext i8 [[IV]] to i64
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV_EXT]]
 ; CHECK-NEXT:    [[LV:%.*]] = load i8, ptr [[GEP]], align 1
@@ -325,10 +320,9 @@ define void @test_lshr_by_4(ptr %A) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[IV_EXT:%.*]] = zext i8 [[IV]] to i64
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV_EXT]]
 ; CHECK-NEXT:    [[LV:%.*]] = load i8, ptr [[GEP]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/uitofp-preserve-nneg.ll b/llvm/test/Transforms/LoopVectorize/uitofp-preserve-nneg.ll
index c67817556c168..c92dc1a641c66 100644
--- a/llvm/test/Transforms/LoopVectorize/uitofp-preserve-nneg.ll
+++ b/llvm/test/Transforms/LoopVectorize/uitofp-preserve-nneg.ll
@@ -24,10 +24,9 @@ define void @uitofp_preserve_nneg(ptr %result, i32 %size, float %y) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[FOR_EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[FOR_BODY_PREHEADER4]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER4]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[CONV:%.*]] = uitofp nneg i32 [[TMP4]] to float
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul float [[CONV]], [[Y]]
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = zext nneg i32 [[TMP4]] to i64
diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
index 85cf925669feb..dfc0ca5b9bbf0 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll
@@ -6,7 +6,7 @@
 define void @blend_uniform_iv_trunc(i1 %c) {
 ; CHECK-LABEL: define void @blend_uniform_iv_trunc(
 ; CHECK-SAME: i1 [[C:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -22,10 +22,9 @@ define void @blend_uniform_iv_trunc(i1 %c) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[IV_TRUNC_2:%.*]] = trunc i64 [[IV]] to i16
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_NEXT:.*]], label %[[LOOP_LATCH]]
 ; CHECK:       [[LOOP_NEXT]]:
@@ -66,7 +65,7 @@ exit:                                             ; preds = %loop.latch
 define void @blend_uniform_iv(i1 %c) {
 ; CHECK-LABEL: define void @blend_uniform_iv(
 ; CHECK-SAME: i1 [[C:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -81,10 +80,9 @@ define void @blend_uniform_iv(i1 %c) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_NEXT:.*]], label %[[LOOP_LATCH]]
 ; CHECK:       [[LOOP_NEXT]]:
 ; CHECK-NEXT:    br label %[[LOOP_LATCH]]
@@ -124,7 +122,7 @@ exit:                                             ; preds = %loop.latch
 define void @blend_chain_iv(i1 %c) {
 ; CHECK-LABEL: define void @blend_chain_iv(
 ; CHECK-SAME: i1 [[C:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
@@ -156,10 +154,9 @@ define void @blend_chain_iv(i1 %c) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_NEXT:.*]], label %[[LOOP_LATCH]]
 ; CHECK:       [[LOOP_NEXT]]:
 ; CHECK-NEXT:    br i1 [[C]], label %[[LOOP_NEXT_2:.*]], label %[[LOOP_NEXT_3:.*]]
@@ -211,52 +208,41 @@ exit:                                             ; preds = %loop.latch
 define void @redundant_branch_and_blends_without_mask(ptr %A) {
 ; CHECK-LABEL: define void @redundant_branch_and_blends_without_mask(
 ; CHECK-SAME: ptr [[A:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE12:.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE12]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[INDEX]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 3
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 1)
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 3
 ; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP5]], i32 0
 ; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x ptr> [[TMP35]], ptr [[TMP6]], i32 1
 ; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <4 x ptr> [[TMP36]], ptr [[TMP7]], i32 2
 ; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <4 x ptr> [[TMP37]], ptr [[TMP8]], i32 3
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK-NEXT:    br i1 true, label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
 ; CHECK:       [[PRED_LOAD_IF]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[TMP5]], align 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP10]], i32 0
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
 ; CHECK:       [[PRED_LOAD_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP12:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP11]], %[[PRED_LOAD_IF]] ]
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
-; CHECK-NEXT:    br i1 [[TMP13]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK-NEXT:    br i1 true, label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
 ; CHECK:       [[PRED_LOAD_IF1]]:
 ; CHECK-NEXT:    [[TMP14:%.*]] = load i32, ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP14]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
 ; CHECK:       [[PRED_LOAD_CONTINUE2]]:
 ; CHECK-NEXT:    [[TMP16:%.*]] = phi <4 x i32> [ [[TMP12]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP15]], %[[PRED_LOAD_IF1]] ]
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
-; CHECK-NEXT:    br i1 [[TMP17]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK-NEXT:    br i1 false, label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
 ; CHECK:       [[PRED_LOAD_IF3]]:
 ; CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[TMP7]], align 4
 ; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP18]], i32 2
 ; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE4]]
 ; CHECK:       [[PRED_LOAD_CONTINUE4]]:
 ; CHECK-NEXT:    [[TMP20:%.*]] = phi <4 x i32> [ [[TMP16]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP19]], %[[PRED_LOAD_IF3]] ]
-; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
-; CHECK-NEXT:    br i1 [[TMP21]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
+; CHECK-NEXT:    br i1 false, label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6:.*]]
 ; CHECK:       [[PRED_LOAD_IF5]]:
 ; CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP22]], i32 3
@@ -265,44 +251,37 @@ define void @redundant_branch_and_blends_without_mask(ptr %A) {
 ; CHECK-NEXT:    [[TMP24:%.*]] = phi <4 x i32> [ [[TMP20]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], %[[PRED_LOAD_IF5]] ]
 ; CHECK-NEXT:    [[TMP25:%.*]] = add <4 x i32> [[TMP24]], splat (i32 10)
 ; CHECK-NEXT:    [[TMP26:%.*]] = add <4 x i32> [[TMP24]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
-; CHECK-NEXT:    br i1 [[TMP27]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK-NEXT:    br i1 true, label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
 ; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x i32> [[TMP26]], i32 0
 ; CHECK-NEXT:    store i32 [[TMP28]], ptr [[TMP5]], align 4
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
-; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
-; CHECK-NEXT:    br i1 [[TMP29]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; CHECK-NEXT:    br i1 true, label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
 ; CHECK:       [[PRED_STORE_IF7]]:
 ; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <4 x i32> [[TMP26]], i32 1
 ; CHECK-NEXT:    store i32 [[TMP30]], ptr [[TMP6]], align 4
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
 ; CHECK:       [[PRED_STORE_CONTINUE8]]:
-; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
-; CHECK-NEXT:    br i1 [[TMP31]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
+; CHECK-NEXT:    br i1 false, label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
 ; CHECK:       [[PRED_STORE_IF9]]:
 ; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x i32> [[TMP26]], i32 2
 ; CHECK-NEXT:    store i32 [[TMP32]], ptr [[TMP7]], align 4
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
 ; CHECK:       [[PRED_STORE_CONTINUE10]]:
-; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
-; CHECK-NEXT:    br i1 [[TMP33]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12]]
+; CHECK-NEXT:    br i1 false, label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12:.*]]
 ; CHECK:       [[PRED_STORE_IF11]]:
 ; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x i32> [[TMP26]], i32 3
 ; CHECK-NEXT:    store i32 [[TMP34]], ptr [[TMP8]], align 4
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE12]]
 ; CHECK:       [[PRED_STORE_CONTINUE12]]:
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
-; CHECK-NEXT:    br i1 true, label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br label %[[MIDDLE_BLOCK:.*]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; CHECK:       [[LOOP_HEADER]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; CHECK-NEXT:    [[GEP_IV:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
 ; CHECK-NEXT:    [[L:%.*]] = load i32, ptr [[GEP_IV]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[L]], 10
@@ -314,7 +293,7 @@ define void @redundant_branch_and_blends_without_mask(ptr %A) {
 ; CHECK-NEXT:    store i32 [[RES]], ptr [[GEP_IV]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 1
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -350,6 +329,5 @@ exit:
 ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
 ; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
 ; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]}
-; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
-; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META2]], [[META1]]}
 ;.
diff --git a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
index 0541c9d922402..ef1acc0349a61 100644
--- a/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
+++ b/llvm/test/Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
@@ -25,10 +25,9 @@ define void @test_not_first_lane_only_constant(ptr %A, ptr noalias %B)  {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[IV]]
 ; CHECK-NEXT:    br i1 false, label [[LOOP_LATCH]], label [[ELSE_1:%.*]]
 ; CHECK:       else.1:
@@ -101,10 +100,9 @@ define void @test_not_first_lane_only_wide_compare(ptr %A, ptr noalias %B, i16 %
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[IV]]
 ; CHECK-NEXT:    [[L_0:%.*]] = load i16, ptr [[GEP_A]], align 2
 ; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i16 [[L_0]], [[X]]
@@ -183,10 +181,9 @@ define void @test_not_first_lane_only_wide_compare_incoming_order_swapped(ptr %A
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i16 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    [[GEP_A:%.*]] = getelementptr inbounds i16, ptr [[A]], i16 [[IV]]
 ; CHECK-NEXT:    [[L_0:%.*]] = load i16, ptr [[GEP_A]], align 2
 ; CHECK-NEXT:    [[C_0:%.*]] = icmp ult i16 [[L_0]], [[X]]
diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
index e1185207813e2..f1e68d47848a6 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
@@ -10,7 +10,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosync nofree {
 ; VF8UF1-LABEL: define i8 @test_early_exit_max_tc_less_than_16(
 ; VF8UF1-SAME: ptr dereferenceable(16) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
-; VF8UF1-NEXT:  [[ENTRY:.*]]:
+; VF8UF1-NEXT:  [[ENTRY:.*:]]
 ; VF8UF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF8UF1:       [[VECTOR_PH]]:
 ; VF8UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -31,10 +31,9 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn
 ; VF8UF1:       [[VECTOR_EARLY_EXIT]]:
 ; VF8UF1-NEXT:    br label %[[EXIT]]
 ; VF8UF1:       [[SCALAR_PH]]:
-; VF8UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF8UF1-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; VF8UF1:       [[LOOP_HEADER]]:
-; VF8UF1-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF8UF1-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; VF8UF1-NEXT:    [[P_SRC1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]]
 ; VF8UF1-NEXT:    [[L:%.*]] = load i8, ptr [[P_SRC1]], align 1
 ; VF8UF1-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 0
@@ -49,7 +48,7 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn
 ;
 ; VF8UF2-LABEL: define i8 @test_early_exit_max_tc_less_than_16(
 ; VF8UF2-SAME: ptr dereferenceable(16) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
-; VF8UF2-NEXT:  [[ENTRY:.*]]:
+; VF8UF2-NEXT:  [[ENTRY:.*:]]
 ; VF8UF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF8UF2:       [[VECTOR_PH]]:
 ; VF8UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -69,10 +68,9 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn
 ; VF8UF2:       [[VECTOR_EARLY_EXIT]]:
 ; VF8UF2-NEXT:    br label %[[EXIT]]
 ; VF8UF2:       [[SCALAR_PH]]:
-; VF8UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF8UF2-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; VF8UF2:       [[LOOP_HEADER]]:
-; VF8UF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF8UF2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; VF8UF2-NEXT:    [[P_SRC:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
 ; VF8UF2-NEXT:    [[L:%.*]] = load i8, ptr [[P_SRC]], align 1
 ; VF8UF2-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 0
@@ -87,7 +85,7 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn
 ;
 ; VF16UF1-LABEL: define i8 @test_early_exit_max_tc_less_than_16(
 ; VF16UF1-SAME: ptr dereferenceable(16) [[A:%.*]]) #[[ATTR0:[0-9]+]] {
-; VF16UF1-NEXT:  [[ENTRY:.*]]:
+; VF16UF1-NEXT:  [[ENTRY:.*:]]
 ; VF16UF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF16UF1:       [[VECTOR_PH]]:
 ; VF16UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -103,10 +101,9 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn
 ; VF16UF1:       [[VECTOR_EARLY_EXIT]]:
 ; VF16UF1-NEXT:    br label %[[EXIT]]
 ; VF16UF1:       [[SCALAR_PH]]:
-; VF16UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF16UF1-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; VF16UF1:       [[LOOP_HEADER]]:
-; VF16UF1-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF16UF1-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; VF16UF1-NEXT:    [[P_SRC1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]]
 ; VF16UF1-NEXT:    [[L:%.*]] = load i8, ptr [[P_SRC1]], align 1
 ; VF16UF1-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 0
@@ -142,7 +139,7 @@ exit:
 define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr dereferenceable(16) %A) nosync nofree {
 ; VF8UF1-LABEL: define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(
 ; VF8UF1-SAME: ptr dereferenceable(16) [[A:%.*]]) #[[ATTR0]] {
-; VF8UF1-NEXT:  [[ENTRY:.*]]:
+; VF8UF1-NEXT:  [[ENTRY:.*:]]
 ; VF8UF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF8UF1:       [[VECTOR_PH]]:
 ; VF8UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -165,10 +162,9 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
 ; VF8UF1-NEXT:    [[TMP8:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
 ; VF8UF1-NEXT:    br label %[[EXIT]]
 ; VF8UF1:       [[SCALAR_PH]]:
-; VF8UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF8UF1-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; VF8UF1:       [[LOOP_HEADER]]:
-; VF8UF1-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF8UF1-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; VF8UF1-NEXT:    [[P_SRC1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]]
 ; VF8UF1-NEXT:    [[L:%.*]] = load i8, ptr [[P_SRC1]], align 1
 ; VF8UF1-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 0
@@ -183,7 +179,7 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
 ;
 ; VF8UF2-LABEL: define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(
 ; VF8UF2-SAME: ptr dereferenceable(16) [[A:%.*]]) #[[ATTR0]] {
-; VF8UF2-NEXT:  [[ENTRY:.*]]:
+; VF8UF2-NEXT:  [[ENTRY:.*:]]
 ; VF8UF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF8UF2:       [[VECTOR_PH]]:
 ; VF8UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -210,10 +206,9 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
 ; VF8UF2-NEXT:    [[TMP12:%.*]] = add i64 0, [[TMP11]]
 ; VF8UF2-NEXT:    br label %[[EXIT]]
 ; VF8UF2:       [[SCALAR_PH]]:
-; VF8UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF8UF2-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; VF8UF2:       [[LOOP_HEADER]]:
-; VF8UF2-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF8UF2-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; VF8UF2-NEXT:    [[P_SRC1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]]
 ; VF8UF2-NEXT:    [[L:%.*]] = load i8, ptr [[P_SRC1]], align 1
 ; VF8UF2-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 0
@@ -228,7 +223,7 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
 ;
 ; VF16UF1-LABEL: define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(
 ; VF16UF1-SAME: ptr dereferenceable(16) [[A:%.*]]) #[[ATTR0]] {
-; VF16UF1-NEXT:  [[ENTRY:.*]]:
+; VF16UF1-NEXT:  [[ENTRY:.*:]]
 ; VF16UF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF16UF1:       [[VECTOR_PH]]:
 ; VF16UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -246,10 +241,9 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer
 ; VF16UF1-NEXT:    [[TMP5:%.*]] = add i64 0, [[FIRST_ACTIVE_LANE]]
 ; VF16UF1-NEXT:    br label %[[EXIT]]
 ; VF16UF1:       [[SCALAR_PH]]:
-; VF16UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF16UF1-NEXT:    br label %[[LOOP_HEADER:.*]]
 ; VF16UF1:       [[LOOP_HEADER]]:
-; VF16UF1-NEXT:    [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; VF16UF1-NEXT:    [[IV1:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
 ; VF16UF1-NEXT:    [[P_SRC1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]]
 ; VF16UF1-NEXT:    [[L:%.*]] = load i8, ptr [[P_SRC1]], align 1
 ; VF16UF1-NEXT:    [[C:%.*]] = icmp eq i8 [[L]], 0
diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-outside-iv-users.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-outside-iv-users.ll
index 5f1cee887fdaa..ada59e90b8811 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-outside-iv-users.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-outside-iv-users.ll
@@ -6,7 +6,7 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 define i64 @remove_loop_region_int_iv_used_outside(ptr %dst) {
 ; CHECK-LABEL: define i64 @remove_loop_region_int_iv_used_outside(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -18,10 +18,9 @@ define i64 @remove_loop_region_int_iv_used_outside(ptr %dst) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr ptr, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store ptr null, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
@@ -50,7 +49,7 @@ exit:
 define i64 @remove_loop_region_int_iv_inc_used_outside(ptr %dst) {
 ; CHECK-LABEL: define i64 @remove_loop_region_int_iv_inc_used_outside(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -62,10 +61,9 @@ define i64 @remove_loop_region_int_iv_inc_used_outside(ptr %dst) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr ptr, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store ptr null, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
@@ -94,7 +92,7 @@ exit:
 define ptr @remove_loop_region_ptr_iv_used_outside(ptr %dst) {
 ; CHECK-LABEL: define ptr @remove_loop_region_ptr_iv_used_outside(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 128
@@ -108,12 +106,10 @@ define ptr @remove_loop_region_ptr_iv_used_outside(ptr %dst) {
 ; CHECK-NEXT:    [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP0]], i64 -8
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[INT_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[INT_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[DST]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[INT_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INT_IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    store ptr null, ptr [[PTR_IV]], align 8
 ; CHECK-NEXT:    [[INT_IV_NEXT]] = add i64 [[INT_IV]], 1
 ; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 8
@@ -143,7 +139,7 @@ exit:
 define ptr @remove_loop_region_ptr_iv_inc_used_outside(ptr %dst) {
 ; CHECK-LABEL: define ptr @remove_loop_region_ptr_iv_inc_used_outside(
 ; CHECK-SAME: ptr [[DST:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[DST]], i64 128
@@ -156,12 +152,10 @@ define ptr @remove_loop_region_ptr_iv_inc_used_outside(ptr %dst) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[INT_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], %[[SCALAR_PH]] ], [ [[INT_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[DST]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[INT_IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[INT_IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    store ptr null, ptr [[PTR_IV]], align 8
 ; CHECK-NEXT:    [[INT_IV_NEXT]] = add i64 [[INT_IV]], 1
 ; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 8
diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
index 59c76aefbb90f..27fa57928aa96 100644
--- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
+++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination.ll
@@ -148,13 +148,10 @@ exit:
 define void @remove_loop_region_with_replicate_recipe(ptr %dst, i64 range(i64 5, 10) %N) {
 ; VF8UF1-LABEL: define void @remove_loop_region_with_replicate_recipe(
 ; VF8UF1-SAME: ptr [[DST:%.*]], i64 range(i64 5, 10) [[N:%.*]]) {
-; VF8UF1-NEXT:  [[ENTRY:.*]]:
+; VF8UF1-NEXT:  [[ENTRY:.*:]]
 ; VF8UF1-NEXT:    [[TMP0:%.*]] = add nsw i64 [[N]], -2
 ; VF8UF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF8UF1:       [[VECTOR_PH]]:
-; VF8UF1-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 7
-; VF8UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
-; VF8UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; VF8UF1-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
 ; VF8UF1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; VF8UF1-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
@@ -221,10 +218,9 @@ define void @remove_loop_region_with_replicate_recipe(ptr %dst, i64 range(i64 5,
 ; VF8UF1:       [[MIDDLE_BLOCK]]:
 ; VF8UF1-NEXT:    br label %[[EXIT:.*]]
 ; VF8UF1:       [[SCALAR_PH]]:
-; VF8UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 2, %[[ENTRY]] ]
 ; VF8UF1-NEXT:    br label %[[LOOP:.*]]
 ; VF8UF1:       [[LOOP]]:
-; VF8UF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF8UF1-NEXT:    [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF8UF1-NEXT:    [[GEP_DST:%.*]] = getelementptr i16, ptr [[DST]], i64 [[IV]]
 ; VF8UF1-NEXT:    store i16 0, ptr [[GEP_DST]], align 2
 ; VF8UF1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
@@ -235,13 +231,10 @@ define void @remove_loop_region_with_replicate_recipe(ptr %dst, i64 range(i64 5,
 ;
 ; VF8UF2-LABEL: define void @remove_loop_region_with_replicate_recipe(
 ; VF8UF2-SAME: ptr [[DST:%.*]], i64 range(i64 5, 10) [[N:%.*]]) {
-; VF8UF2-NEXT:  [[ENTRY:.*]]:
+; VF8UF2-NEXT:  [[ENTRY:.*:]]
 ; VF8UF2-NEXT:    [[TMP0:%.*]] = add nsw i64 [[N]], -2
 ; VF8UF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF8UF2:       [[VECTOR_PH]]:
-; VF8UF2-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 15
-; VF8UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 16
-; VF8UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; VF8UF2-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
 ; VF8UF2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; VF8UF2-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT1]], <8 x i64> poison, <8 x i32> zeroinitializer
@@ -365,10 +358,9 @@ define void @remove_loop_region_with_replicate_recipe(ptr %dst, i64 range(i64 5,
 ; VF8UF2:       [[MIDDLE_BLOCK]]:
 ; VF8UF2-NEXT:    br label %[[EXIT:.*]]
 ; VF8UF2:       [[SCALAR_PH]]:
-; VF8UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 2, %[[ENTRY]] ]
 ; VF8UF2-NEXT:    br label %[[LOOP:.*]]
 ; VF8UF2:       [[LOOP]]:
-; VF8UF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF8UF2-NEXT:    [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF8UF2-NEXT:    [[GEP_DST:%.*]] = getelementptr i16, ptr [[DST]], i64 [[IV]]
 ; VF8UF2-NEXT:    store i16 0, ptr [[GEP_DST]], align 2
 ; VF8UF2-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
@@ -379,13 +371,10 @@ define void @remove_loop_region_with_replicate_recipe(ptr %dst, i64 range(i64 5,
 ;
 ; VF16UF1-LABEL: define void @remove_loop_region_with_replicate_recipe(
 ; VF16UF1-SAME: ptr [[DST:%.*]], i64 range(i64 5, 10) [[N:%.*]]) {
-; VF16UF1-NEXT:  [[ENTRY:.*]]:
+; VF16UF1-NEXT:  [[ENTRY:.*:]]
 ; VF16UF1-NEXT:    [[TMP0:%.*]] = add nsw i64 [[N]], -2
 ; VF16UF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF16UF1:       [[VECTOR_PH]]:
-; VF16UF1-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 15
-; VF16UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 16
-; VF16UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; VF16UF1-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
 ; VF16UF1-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <16 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; VF16UF1-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT1]], <16 x i64> poison, <16 x i32> zeroinitializer
@@ -508,10 +497,9 @@ define void @remove_loop_region_with_replicate_recipe(ptr %dst, i64 range(i64 5,
 ; VF16UF1:       [[MIDDLE_BLOCK]]:
 ; VF16UF1-NEXT:    br label %[[EXIT:.*]]
 ; VF16UF1:       [[SCALAR_PH]]:
-; VF16UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 2, %[[ENTRY]] ]
 ; VF16UF1-NEXT:    br label %[[LOOP:.*]]
 ; VF16UF1:       [[LOOP]]:
-; VF16UF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF16UF1-NEXT:    [[IV:%.*]] = phi i64 [ 2, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF16UF1-NEXT:    [[GEP_DST:%.*]] = getelementptr i16, ptr [[DST]], i64 [[IV]]
 ; VF16UF1-NEXT:    store i16 0, ptr [[GEP_DST]], align 2
 ; VF16UF1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
@@ -692,7 +680,7 @@ declare void @llvm.assume(i1)
 define void @scev_expand_step(i64 %x, ptr %dst) {
 ; VF8UF1-LABEL: define void @scev_expand_step(
 ; VF8UF1-SAME: i64 [[X:%.*]], ptr [[DST:%.*]]) {
-; VF8UF1-NEXT:  [[ENTRY:.*]]:
+; VF8UF1-NEXT:  [[ENTRY:.*:]]
 ; VF8UF1-NEXT:    [[C:%.*]] = icmp eq i64 [[X]], 65536
 ; VF8UF1-NEXT:    call void @llvm.assume(i1 [[C]])
 ; VF8UF1-NEXT:    [[FR:%.*]] = freeze i64 [[X]]
@@ -701,9 +689,6 @@ define void @scev_expand_step(i64 %x, ptr %dst) {
 ; VF8UF1-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
 ; VF8UF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF8UF1:       [[VECTOR_PH]]:
-; VF8UF1-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], 7
-; VF8UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 8
-; VF8UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; VF8UF1-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
 ; VF8UF1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; VF8UF1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
@@ -794,10 +779,9 @@ define void @scev_expand_step(i64 %x, ptr %dst) {
 ; VF8UF1:       [[MIDDLE_BLOCK]]:
 ; VF8UF1-NEXT:    br label %[[EXIT:.*]]
 ; VF8UF1:       [[SCALAR_PH]]:
-; VF8UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF8UF1-NEXT:    br label %[[LOOP:.*]]
 ; VF8UF1:       [[LOOP]]:
-; VF8UF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF8UF1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF8UF1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], [[STEP]]
 ; VF8UF1-NEXT:    [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV_NEXT]]
 ; VF8UF1-NEXT:    store i8 0, ptr [[GEP_DST]], align 1
@@ -808,7 +792,7 @@ define void @scev_expand_step(i64 %x, ptr %dst) {
 ;
 ; VF8UF2-LABEL: define void @scev_expand_step(
 ; VF8UF2-SAME: i64 [[X:%.*]], ptr [[DST:%.*]]) {
-; VF8UF2-NEXT:  [[ENTRY:.*]]:
+; VF8UF2-NEXT:  [[ENTRY:.*:]]
 ; VF8UF2-NEXT:    [[C:%.*]] = icmp eq i64 [[X]], 65536
 ; VF8UF2-NEXT:    call void @llvm.assume(i1 [[C]])
 ; VF8UF2-NEXT:    [[FR:%.*]] = freeze i64 [[X]]
@@ -817,9 +801,6 @@ define void @scev_expand_step(i64 %x, ptr %dst) {
 ; VF8UF2-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
 ; VF8UF2-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF8UF2:       [[VECTOR_PH]]:
-; VF8UF2-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], 15
-; VF8UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 16
-; VF8UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; VF8UF2-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
 ; VF8UF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; VF8UF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
@@ -991,10 +972,9 @@ define void @scev_expand_step(i64 %x, ptr %dst) {
 ; VF8UF2:       [[MIDDLE_BLOCK]]:
 ; VF8UF2-NEXT:    br label %[[EXIT:.*]]
 ; VF8UF2:       [[SCALAR_PH]]:
-; VF8UF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF8UF2-NEXT:    br label %[[LOOP:.*]]
 ; VF8UF2:       [[LOOP]]:
-; VF8UF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF8UF2-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF8UF2-NEXT:    [[IV_NEXT]] = add i64 [[IV]], [[STEP]]
 ; VF8UF2-NEXT:    [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV_NEXT]]
 ; VF8UF2-NEXT:    store i8 0, ptr [[GEP_DST]], align 1
@@ -1005,7 +985,7 @@ define void @scev_expand_step(i64 %x, ptr %dst) {
 ;
 ; VF16UF1-LABEL: define void @scev_expand_step(
 ; VF16UF1-SAME: i64 [[X:%.*]], ptr [[DST:%.*]]) {
-; VF16UF1-NEXT:  [[ENTRY:.*]]:
+; VF16UF1-NEXT:  [[ENTRY:.*:]]
 ; VF16UF1-NEXT:    [[C:%.*]] = icmp eq i64 [[X]], 65536
 ; VF16UF1-NEXT:    call void @llvm.assume(i1 [[C]])
 ; VF16UF1-NEXT:    [[FR:%.*]] = freeze i64 [[X]]
@@ -1014,9 +994,6 @@ define void @scev_expand_step(i64 %x, ptr %dst) {
 ; VF16UF1-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
 ; VF16UF1-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VF16UF1:       [[VECTOR_PH]]:
-; VF16UF1-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], 15
-; VF16UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 16
-; VF16UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; VF16UF1-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP1]], 1
 ; VF16UF1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; VF16UF1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i64> [[BROADCAST_SPLATINSERT]], <16 x i64> poison, <16 x i32> zeroinitializer
@@ -1187,10 +1164,9 @@ define void @scev_expand_step(i64 %x, ptr %dst) {
 ; VF16UF1:       [[MIDDLE_BLOCK]]:
 ; VF16UF1-NEXT:    br label %[[EXIT:.*]]
 ; VF16UF1:       [[SCALAR_PH]]:
-; VF16UF1-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ]
 ; VF16UF1-NEXT:    br label %[[LOOP:.*]]
 ; VF16UF1:       [[LOOP]]:
-; VF16UF1-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; VF16UF1-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; VF16UF1-NEXT:    [[IV_NEXT]] = add i64 [[IV]], [[STEP]]
 ; VF16UF1-NEXT:    [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i64 [[IV_NEXT]]
 ; VF16UF1-NEXT:    store i8 0, ptr [[GEP_DST]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll
index d3f7794d2b2d4..199bd5b31c70d 100644
--- a/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll
@@ -37,7 +37,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-LABEL: @foo(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP8]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
@@ -45,8 +45,6 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP1]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP14]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -57,7 +55,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-NEXT:    [[TMP16:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
 ; NO-VP-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP16]], ptr [[TMP9]], align 4
-; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]]
+; NO-VP-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]]
 ; NO-VP-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       middle.block:
@@ -90,7 +88,6 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-DEF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; NO-VP-DEF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
 ; NO-VP-DEF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; NO-VP-DEF-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; NO-VP-DEF-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-DEF:       vector.body:
 ; NO-VP-DEF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -101,7 +98,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-DEF-NEXT:    [[TMP8:%.*]] = add nsw <vscale x 1 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
 ; NO-VP-DEF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-DEF-NEXT:    store <vscale x 1 x i32> [[TMP8]], ptr [[TMP9]], align 4
-; NO-VP-DEF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; NO-VP-DEF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
 ; NO-VP-DEF-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; NO-VP-DEF-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP-DEF:       middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
index 128594ca983b0..cae5c4af13792 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
@@ -89,15 +89,14 @@ define void @iv_expand(ptr %p, i64 %n) {
 ; CHECK-NEXT: Successor(s): middle.block
 ; CHECK:      VPlan 'Final VPlan for VF={8},UF={1}'
 ; CHECK:      ir-bb<vector.ph>:
-; CHECK-NEXT:     IR   %n.mod.vf = urem i64 %n, 8
-; CHECK-NEXT:     IR   %n.vec = sub i64 %n, %n.mod.vf
+; CHECK-NEXT:     EMIT vp<%n.mod.vf> = urem ir<%n>, ir<8>
+; CHECK-NEXT:     EMIT vp<%n.vec> = sub ir<%n>, vp<%n.mod.vf>
 ; CHECK-NEXT:     EMIT vp<[[STEP_VECTOR:%.+]]> = step-vector
 ; CHECK-NEXT:     EMIT vp<[[BROADCAST_0:%.+]]> = broadcast ir<0>
 ; CHECK-NEXT:     EMIT vp<[[BROADCAST_1:%.+]]> = broadcast ir<1>
 ; CHECK-NEXT:     EMIT vp<[[MUL:%.+]]> = mul vp<[[STEP_VECTOR]]>, vp<[[BROADCAST_1]]>
 ; CHECK-NEXT:     EMIT vp<[[INDUCTION:%.+]]> = add vp<[[BROADCAST_0]]>, vp<[[MUL]]>
-; CHECK-NEXT:     EMIT vp<[[INC:%.+]]> = mul ir<1>, ir<8>
-; CHECK-NEXT:     EMIT vp<[[BROADCAST_INC:%.+]]> = broadcast vp<[[INC]]>
+; CHECK-NEXT:     EMIT vp<[[BROADCAST_INC:%.+]]> = broadcast ir<8>
 ; CHECK-NEXT: Successor(s): vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.body:
@@ -109,7 +108,7 @@ define void @iv_expand(ptr %p, i64 %n) {
 ; CHECK-NEXT:   WIDEN store ir<%q>, ir<%y>
 ; CHECK-NEXT:   EMIT vp<%index.next> = add nuw vp<[[SCALAR_PHI]]>, ir<8>
 ; CHECK-NEXT:   EMIT vp<%vec.ind.next> = add ir<%iv>, vp<[[BROADCAST_INC]]>
-; CHECK-NEXT:   EMIT branch-on-count vp<%index.next>, ir<%n.vec>
+; CHECK-NEXT:   EMIT branch-on-count vp<%index.next>, vp<%n.vec>
 ; CHECK-NEXT: Successor(s): middle.block, vector.body
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
index 46f91259f5edc..32aadcfb164e6 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-predicate-switch.ll
@@ -3,18 +3,15 @@
 
 define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
 ; CHECK:      VPlan 'Final VPlan for VF={2},UF={1}' {
-; CHECK-NEXT: Live-in ir<[[VF:.+]]> = VF
-; CHECK-NEXT: Live-in ir<[[VFxUF:.+]]> = VF * UF
-; CHECK-NEXT: Live-in ir<[[VTC:%.+]]> = vector-trip-count
 ; CHECK-NEXT: ir<%0> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<entry>:
 ; CHECK-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<vector.ph>:
-; CHECK-NEXT:   IR %n.mod.vf = urem i64 %0, 2
-; CHECK-NEXT:   IR %n.vec = sub i64 %0, %n.mod.vf
-; CHECK-NEXT:   vp<[[END:%.+]]> = DERIVED-IV ir<%start> + ir<%n.vec> * ir<1>
+; CHECK-NEXT:   EMIT vp<%n.mod.vf> = urem ir<%0>, ir<2>
+; CHECK-NEXT:   EMIT vp<[[VTC:%.+]]> = sub ir<%0>, vp<%n.mod.vf>
+; CHECK-NEXT:   vp<[[END:%.+]]> = DERIVED-IV ir<%start> + vp<[[VTC]]> * ir<1>
 ; CHECK-NEXT: Successor(s): vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.body:
@@ -77,12 +74,12 @@ define void @switch4_default_common_dest_with_case(ptr %start, ptr %end) {
 ; CHECK-NEXT: Successor(s): default.2
 ; CHECK-EMPTY:
 ; CHECK-NEXT: default.2:
-; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<[[VFxUF]]>
-; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, ir<[[VTC]]>
+; CHECK-NEXT:   EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, ir<2>
+; CHECK-NEXT:   EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]>
 ; CHECK-NEXT: Successor(s): middle.block, vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq ir<%0>, ir<[[VTC]]>
+; CHECK-NEXT:   EMIT vp<[[MIDDLE_CMP:%.+]]> = icmp eq ir<%0>, vp<[[VTC]]>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[MIDDLE_CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
index d85638733211c..c65d9368d595f 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-before-execute.ll
@@ -62,18 +62,17 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ;
 ; CHECK: Executing best plan with VF=8, UF=2
 ; CHECK-NEXT: VPlan 'Final VPlan for VF={8},UF={2}' {
-; CHECK-NEXT: Live-in ir<[[VTC:%.+]]> = vector-trip-count
-; CHECK-NEXT: ir<%and> = original trip-count
+; CHECK-NEXT: Live-in ir<%and> = original trip-count
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<entry>:
 ; CHECK-NEXT:   IR %and = and i64 %N, 15
 ; CHECK-NEXT:  Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<vector.ph>:
-; CHECK-NEXT:  IR   %n.mod.vf = urem i64 %and, 16
-; CHECK-NEXT:  IR   %n.vec = sub i64 %and, %n.mod.vf
-; CHECK-NEXT:  vp<[[END1:%.+]]> = DERIVED-IV ir<%and> + ir<[[VTC]]> * ir<-1>
-; CHECK-NEXT:  vp<[[END2:%.+]]> = DERIVED-IV ir<%A> + ir<[[VTC]]> * ir<1>
+; CHECK-NEXT:  EMIT vp<%n.mod.vf> = urem ir<%and>, ir<16>
+; CHECK-NEXT:  EMIT vp<[[VTC:%.+]]> = sub ir<%and>, vp<%n.mod.vf>
+; CHECK-NEXT:  vp<[[END1:%.+]]> = DERIVED-IV ir<%and> + vp<[[VTC]]> * ir<-1>
+; CHECK-NEXT:  vp<[[END2:%.+]]> = DERIVED-IV ir<%A> + vp<[[VTC]]> * ir<1>
 ; CHECK-NEXT: Successor(s): vector.body
 ; CHECK-EMPTY:
 ; CHECK-NEXT: vector.body:
@@ -88,7 +87,7 @@ define void @test_tc_less_than_16(ptr %A, i64 %N) {
 ; CHECK-NEXT: Successor(s): middle.block
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
-; CHECK-NEXT:   EMIT vp<[[C:%.+]]> = icmp eq ir<%and>, ir<[[VTC]]>
+; CHECK-NEXT:   EMIT vp<[[C:%.+]]> = icmp eq ir<%and>, vp<[[VTC]]>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[C]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
index 6804817c402bd..20676f3702294 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
@@ -13,14 +13,14 @@ define void @foo(i64 %n) {
 ; CHECK-NEXT: Successor(s): outer.header
 ; CHECK-EMPTY:
 ; CHECK-NEXT: outer.header:
-; CHECK-NEXT:   WIDEN-PHI ir<%outer.iv> = phi [ ir<%outer.iv.next>, outer.latch ], [ ir<0>, ir-bb<entry> ]
+; CHECK-NEXT:   EMIT-SCALAR ir<%outer.iv> = phi [ ir<%outer.iv.next>, outer.latch ], [ ir<0>, ir-bb<entry> ]
 ; CHECK-NEXT:   EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv>
 ; CHECK-NEXT:   EMIT store ir<%outer.iv>, ir<%gep.1>
 ; CHECK-NEXT:   EMIT ir<%add> = add ir<%outer.iv>, ir<%n>
 ; CHECK-NEXT: Successor(s): inner
 ; CHECK-EMPTY:
 ; CHECK-NEXT: inner:
-; CHECK-NEXT:   WIDEN-PHI ir<%inner.iv> = phi [ ir<%inner.iv.next>, inner ], [ ir<0>, outer.header ]
+; CHECK-NEXT:   EMIT-SCALAR ir<%inner.iv> = phi [ ir<%inner.iv.next>, inner ], [ ir<0>, outer.header ]
 ; CHECK-NEXT:   EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv>
 ; CHECK-NEXT:   EMIT store ir<%add>, ir<%gep.2>
 ; CHECK-NEXT:   EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1>
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll b/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
index 2d07b33eaced4..f36b26944cac7 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-widen-select-instruction.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph\:" --version 5
 ; RUN: opt -passes=loop-vectorize -force-vector-width=4 -enable-vplan-native-path -S %s | FileCheck %s
 
 ; Test that VPlan native path is able to widen select instruction in the
@@ -10,24 +11,38 @@
 ;   variables.
 
 define void @loop_invariant_select(ptr noalias nocapture %out, i1 %select, double %a, double %b) {
-; CHECK-LABEL: @loop_invariant_select(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i64 0
+; CHECK-LABEL: define void @loop_invariant_select(
+; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], i1 [[SELECT:%.*]], double [[A:%.*]], double [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT:%.*]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    br label [[FOR2_HEADER3:%.*]]
-; CHECK:       for2.header3:
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP2:%.*]], [[FOR2_HEADER3]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[SELECT:%.*]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    br label %[[FOR2_HEADER3:.*]]
+; CHECK:       [[FOR2_HEADER3]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP2:%.*]], %[[FOR2_HEADER3]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[SELECT]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP1]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP2]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq <4 x i64> [[TMP2]], splat (i64 10000)
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]
+; CHECK:       [[VECTOR_LATCH]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
 entry:
   br label %for1.header
 
@@ -55,25 +70,39 @@ exit:
 }
 
 define void @outer_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) {
-; CHECK-LABEL: @outer_loop_dependant_select(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i64 0
+; CHECK-LABEL: define void @outer_loop_dependant_select(
+; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], double [[A:%.*]], double [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT:%.*]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    br label [[FOR2_HEADER3:%.*]]
-; CHECK:       for2.header3:
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP3:%.*]], [[FOR2_HEADER3]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    br label %[[FOR2_HEADER3:.*]]
+; CHECK:       [[FOR2_HEADER3]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[FOR2_HEADER3]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i64> [[VEC_IND]] to <4 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP2]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP3]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP3]], splat (i64 10000)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]
+; CHECK:       [[VECTOR_LATCH]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
 entry:
   br label %for1.header
 
@@ -102,25 +131,39 @@ exit:
 }
 
 define void @inner_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) {
-; CHECK-LABEL: @inner_loop_dependant_select(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i64 0
+; CHECK-LABEL: define void @inner_loop_dependant_select(
+; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], double [[A:%.*]], double [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT:%.*]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    br label [[FOR2_HEADER3:%.*]]
-; CHECK:       for2.header3:
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP3:%.*]], [[FOR2_HEADER3]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    br label %[[FOR2_HEADER3:.*]]
+; CHECK:       [[FOR2_HEADER3]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP3:%.*]], %[[FOR2_HEADER3]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <4 x i64> [[VEC_PHI]] to <4 x i1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP2]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP3]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq <4 x i64> [[TMP3]], splat (i64 10000)
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i1> [[TMP5]], i32 0
+; CHECK-NEXT:    br i1 [[TMP6]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]
+; CHECK:       [[VECTOR_LATCH]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
 entry:
   br label %for1.header
 
@@ -149,26 +192,40 @@ exit:
 }
 
 define void @outer_and_inner_loop_dependant_select(ptr noalias nocapture %out, double %a, double %b) {
-; CHECK-LABEL: @outer_and_inner_loop_dependant_select(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A:%.*]], i64 0
+; CHECK-LABEL: define void @outer_and_inner_loop_dependant_select(
+; CHECK-SAME: ptr noalias captures(none) [[OUT:%.*]], double [[A:%.*]], double [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x double> poison, double [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <4 x double> poison, double [[B]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT3:%.*]] = shufflevector <4 x double> [[BROADCAST_SPLATINSERT2]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR1_LATCH4:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR1_LATCH4]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT:%.*]], <4 x i64> [[VEC_IND]]
-; CHECK-NEXT:    br label [[FOR2_HEADER3:%.*]]
-; CHECK:       for2.header3:
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP4:%.*]], [[FOR2_HEADER3]] ]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds double, ptr [[OUT]], <4 x i64> [[VEC_IND]]
+; CHECK-NEXT:    br label %[[FOR2_HEADER3:.*]]
+; CHECK:       [[FOR2_HEADER3]]:
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i64> [ zeroinitializer, %[[VECTOR_BODY]] ], [ [[TMP4:%.*]], %[[FOR2_HEADER3]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw <4 x i64> [[VEC_IND]], [[VEC_PHI]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i1>
 ; CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x double> [[BROADCAST_SPLAT]], <4 x double> [[BROADCAST_SPLAT3]]
 ; CHECK-NEXT:    call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP3]], <4 x ptr> [[TMP0]], i32 8, <4 x i1> splat (i1 true))
+; CHECK-NEXT:    [[TMP4]] = add nuw nsw <4 x i64> [[VEC_PHI]], splat (i64 1)
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <4 x i64> [[TMP4]], splat (i64 10000)
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP6]], i32 0
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[VECTOR_LATCH]], label %[[FOR2_HEADER3]]
+; CHECK:       [[VECTOR_LATCH]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br i1 true, [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+;
 entry:
   br label %for1.header
 
diff --git a/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll b/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll
index 5d0d391e5b99d..19cf1069f05b0 100644
--- a/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll
+++ b/llvm/test/Transforms/LoopVectorize/widen-gep-all-indices-invariant.ll
@@ -22,10 +22,9 @@ define void @pr63340(ptr %A, ptr %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
 ; CHECK:       loop.header:
-; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP_LATCH]]
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[F_0_I:%.*]] = phi ptr [ [[A]], [[LOOP_HEADER]] ]
@@ -80,10 +79,9 @@ define void @wide_gep_index_invariant(ptr noalias %dst, ptr noalias %src, i64 %n
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[SRC]], align 8
 ; CHECK-NEXT:    [[GEP_L:%.*]] = getelementptr float, ptr [[L]], i64 [[N]]
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr ptr, ptr [[DST]], i64 [[IV]]
@@ -134,10 +132,9 @@ define void @wide_gep_multiple_indices_some_invariant(ptr noalias %dst, ptr noal
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[L:%.*]] = load ptr, ptr [[SRC]], align 8
 ; CHECK-NEXT:    [[GEP_L:%.*]] = getelementptr [10 x float], ptr [[L]], i32 [[X]], i64 [[IV]]
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr ptr, ptr [[DST]], i64 [[IV]]
diff --git a/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
index c23d2b38659f1..1cc2e871925b8 100644
--- a/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
+++ b/llvm/test/Transforms/LoopVectorize/widen-intrinsic.ll
@@ -5,7 +5,7 @@
 define void @powi_only_first_lane_used_of_second_arg(ptr %p, i32 %pow) {
 ; CHECK-LABEL: define void @powi_only_first_lane_used_of_second_arg(
 ; CHECK-SAME: ptr [[P:%.*]], i32 [[POW:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -21,10 +21,9 @@ define void @powi_only_first_lane_used_of_second_arg(ptr %p, i32 %pow) {
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    br label %[[EXIT:.*]]
 ; CHECK:       [[SCALAR_PH]]:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[P_GEP:%.*]] = getelementptr float, ptr [[P]], i32 [[IV]]
 ; CHECK-NEXT:    [[X:%.*]] = load float, ptr [[P_GEP]], align 4
 ; CHECK-NEXT:    [[Y:%.*]] = call float @llvm.powi.f32.i32(float [[X]], i32 [[POW]])
diff --git a/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
index 80d6d361cf0d5..8a639166d41e4 100644
--- a/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
+++ b/llvm/test/Transforms/LoopVersioning/noalias-version-twice.ll
@@ -42,8 +42,8 @@ for.body:                                         ; preds = %for.body, %entry
 
   %arrayidxA = getelementptr inbounds i32, ptr %a, i64 %ind
 
-; CHECK: %loadA.ldist1 = {{.*}} !noalias !25
-; A noalias C: !25 -> { 17(15), 18(15), 19(15), 26(24) }
+; CHECK: %loadA.ldist1 = {{.*}} !alias.scope !24, !noalias !27
+; A noalias C: !33 -> { 19(17), 20(17), 21(17), 28(26) }
 ;                       ^^^^^^
   %loadA = load i32, ptr %arrayidxA, align 4
 
@@ -60,15 +60,15 @@ for.body:                                         ; preds = %for.body, %entry
 
   %arrayidxD = getelementptr inbounds i32, ptr %d, i64 %ind
 
-; CHECK: %loadD = {{.*}} !alias.scope !31
-; D's scope: !31 -> { 18(15), 32(33) }
+; CHECK: %loadD = {{.*}} !alias.scope !33
+; D's scope: !33 -> { 20(17), 34(35) }
 ;                             ^^^^^^
   %loadD = load i32, ptr %arrayidxD, align 4
 
   %arrayidxE = getelementptr inbounds i32, ptr %e, i64 %ind
 
-; CHECK: %loadE = {{.*}} !alias.scope !34
-; E's scope: !34 -> { 19(15), 35(33) }
+; CHECK: %loadE = {{.*}} !alias.scope !36
+; E's scope: !36 -> { 21(17), 37(33) }
 ;                             ^^^^^^
   %loadE = load i32, ptr %arrayidxE, align 4
 
@@ -76,8 +76,8 @@ for.body:                                         ; preds = %for.body, %entry
 
   %arrayidxC = getelementptr inbounds i32, ptr %c, i64 %ind
 
-; CHECK: store i32 %mulC, {{.*}} !alias.scope !36, !noalias !38
-; C's scope: !36 -> { 17(15), 37(33) }
+; CHECK: store i32 %mulC, {{.*}} !alias.scope !38, !noalias !40
+; C's scope: !38 -> { 19(17), 39(35)
 ;                     ^^^^^^
 ; C noalias D and E: !38 -> { 21(15), 32(33), 35(33) }
 ;                                     ^^^^^^  ^^^^^^
@@ -92,15 +92,19 @@ for.end:                                          ; preds = %for.body
 
 ; Domain for the second loop versioning for the top loop after
 ; distribution.
-; CHECK: !15 = distinct !{!15, !"LVerDomain"}
-; CHECK: !17 = distinct !{!17, !15}
-; CHECK: !25 = !{!17, !18, !19, !26}
-; CHECK: !31 = !{!18, !32}
-; CHECK: !32 = distinct !{!32, !33}
+; CHECK: !17 = distinct !{!17, !"LVerDomain"}
+; CHECK: !19 = distinct !{!19, !17}
+; CHECK: !20 = distinct !{!20, !17}
+; CHECK: !21 = distinct !{!21, !17}
+; CHECK: !27 = !{!19, !20, !21, !28}
+; CHECK: !28 = distinct !{!28, !26}
+; CHECK: !33 = !{!20, !34}
+; CHECK: !34 = distinct !{!34, !35}
 ; Domain for the second loop versioning for the bottom loop after
 ; distribution.
-; CHECK: !33 = distinct !{!33, !"LVerDomain"}
-; CHECK: !34 = !{!19, !35}
-; CHECK: !35 = distinct !{!35, !33}
-; CHECK: !36 = !{!17, !37}
-; CHECK: !38 = !{!21, !32, !35}
+; CHECK: !35 = distinct !{!35, !"LVerDomain"}
+; CHECK: !36 = !{!21, !37}
+; CHECK: !37 = distinct !{!37, !35}
+; CHECK: !38 = !{!19, !39}
+; CHECK: !39 = distinct !{!39, !35}
+; CHECK: !40 = !{!23, !34, !37}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-lifetime-ends.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-lifetime-ends.ll
index 7cc8458b56dbc..612c96c8eacd6 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-lifetime-ends.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/multiply-fused-lifetime-ends.ll
@@ -73,7 +73,7 @@ define void @lifetime_for_first_arg_before_multiply(ptr noalias %B, ptr noalias
 ; CHECK-NEXT:    store <2 x double> [[TMP13]], ptr [[TMP26]], align 8
 ; CHECK-NEXT:    [[VEC_GEP28:%.*]] = getelementptr double, ptr [[TMP26]], i64 2
 ; CHECK-NEXT:    store <2 x double> [[TMP25]], ptr [[VEC_GEP28]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -81,7 +81,7 @@ entry:
   call void @init(ptr %A)
   %a = load <4 x double>, ptr %A, align 8
   %b = load <4 x double>, ptr %B, align 8
-  call void @llvm.lifetime.end(i64 -1, ptr %A)
+  call void @llvm.lifetime.end(ptr %A)
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   store <4 x double> %c, ptr %C, align 8
   ret void
@@ -154,7 +154,7 @@ define void @lifetime_for_second_arg_before_multiply(ptr noalias %A, ptr noalias
 ; CHECK-NEXT:    store <2 x double> [[TMP13]], ptr [[TMP26]], align 8
 ; CHECK-NEXT:    [[VEC_GEP28:%.*]] = getelementptr double, ptr [[TMP26]], i64 2
 ; CHECK-NEXT:    store <2 x double> [[TMP25]], ptr [[VEC_GEP28]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -162,7 +162,7 @@ entry:
   call void @init(ptr %B)
   %a = load <4 x double>, ptr %A, align 8
   %b = load <4 x double>, ptr %B, align 8
-  call void @llvm.lifetime.end(i64 -1, ptr %B)
+  call void @llvm.lifetime.end(ptr %B)
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   store <4 x double> %c, ptr %C, align 8
   ret void
@@ -236,7 +236,7 @@ define void @lifetime_for_first_arg_before_multiply_load_from_offset(ptr noalias
 ; CHECK-NEXT:    store <2 x double> [[TMP13]], ptr [[TMP26]], align 8
 ; CHECK-NEXT:    [[VEC_GEP28:%.*]] = getelementptr double, ptr [[TMP26]], i64 2
 ; CHECK-NEXT:    store <2 x double> [[TMP25]], ptr [[VEC_GEP28]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -245,7 +245,7 @@ entry:
   %gep.8 = getelementptr i8, ptr %A, i64 8
   %a = load <4 x double>, ptr %gep.8, align 8
   %b = load <4 x double>, ptr %B, align 8
-  call void @llvm.lifetime.end(i64 -1, ptr %A)
+  call void @llvm.lifetime.end(ptr %A)
   %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2)
   store <4 x double> %c, ptr %C, align 8
   ret void
@@ -332,7 +332,7 @@ entry:
   br i1 %c.0, label %then, label %exit
 
 then:
-  call void @llvm.lifetime.end(i64 -1, ptr %A)
+  call void @llvm.lifetime.end(ptr %A)
   br label %exit
 
 exit:
@@ -422,7 +422,7 @@ entry:
   br i1 %c.0, label %then, label %exit
 
 then:
-  call void @llvm.lifetime.end(i64 -1, ptr %B)
+  call void @llvm.lifetime.end(ptr %B)
   br label %exit
 
 exit:
@@ -442,8 +442,8 @@ define void @multiple_unrelated_lifetimes(ptr noalias %C, i1 %c.0) {
 ; CHECK-NEXT:    call void @init(ptr [[B]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[EXIT:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ALLOC_1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[ALLOC_2]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ALLOC_1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[ALLOC_2]])
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr double, ptr [[A]], i64 0
@@ -522,10 +522,10 @@ entry:
   br i1 %c.0, label %then, label %exit
 
 then:
-  call void @llvm.lifetime.end(i64 -1, ptr %B)
-  call void @llvm.lifetime.end(i64 -1, ptr %alloc.1)
-  call void @llvm.lifetime.end(i64 -1, ptr %A)
-  call void @llvm.lifetime.end(i64 -1, ptr %alloc.2)
+  call void @llvm.lifetime.end(ptr %B)
+  call void @llvm.lifetime.end(ptr %alloc.1)
+  call void @llvm.lifetime.end(ptr %A)
+  call void @llvm.lifetime.end(ptr %alloc.2)
   br label %exit
 
 exit:
@@ -607,8 +607,8 @@ define void @lifetimes_for_args_in_different_blocks(ptr noalias %C, i1 %c.0) {
 ; CHECK-NEXT:    store <2 x double> [[TMP25]], ptr [[VEC_GEP28]], align 8
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -626,8 +626,8 @@ then:
   br label %exit
 
 exit:
-  call void @llvm.lifetime.end(i64 -1, ptr %A)
-  call void @llvm.lifetime.end(i64 -1, ptr %B)
+  call void @llvm.lifetime.end(ptr %A)
+  call void @llvm.lifetime.end(ptr %B)
   ret void
 }
 
@@ -640,8 +640,8 @@ define void @lifetimes_for_args_in_different_blocks2(ptr noalias %C, i1 %c.0) {
 ; CHECK-NEXT:    call void @init(ptr [[B]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[THEN:%.*]], label [[EXIT:%.*]]
 ; CHECK:       then:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[B]])
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr double, ptr [[A]], i64 0
@@ -716,8 +716,8 @@ entry:
   br i1 %c.0, label %then, label %exit
 
 then:
-  call void @llvm.lifetime.end(i64 -1, ptr %A)
-  call void @llvm.lifetime.end(i64 -1, ptr %B)
+  call void @llvm.lifetime.end(ptr %A)
+  call void @llvm.lifetime.end(ptr %B)
   br label %exit
 
 exit:
@@ -809,7 +809,7 @@ entry:
   call void @init(ptr %A)
   call void @init(ptr %B)
   %a = load <4 x double>, ptr %A, align 8
-  call void @llvm.lifetime.end(i64 -1, ptr %A)
+  call void @llvm.lifetime.end(ptr %A)
   br i1 %c.0, label %then, label %exit
 
 then:
@@ -819,7 +819,7 @@ then:
   br label %exit
 
 exit:
-  call void @llvm.lifetime.end(i64 -1, ptr %B)
+  call void @llvm.lifetime.end(ptr %B)
   ret void
 }
 
@@ -904,7 +904,7 @@ entry:
   call void @init(ptr %A)
   call void @init(ptr %B)
   %b = load <4 x double>, ptr %B, align 8
-  call void @llvm.lifetime.end(i64 -1, ptr %B)
+  call void @llvm.lifetime.end(ptr %B)
   br i1 %c.0, label %then, label %exit
 
 then:
@@ -914,11 +914,11 @@ then:
   br label %exit
 
 exit:
-  call void @llvm.lifetime.end(i64 -1, ptr %A)
+  call void @llvm.lifetime.end(ptr %A)
   ret void
 }
 
 declare void @init(ptr)
-declare void @llvm.lifetime.end(i64, ptr)
+declare void @llvm.lifetime.end(ptr)
 
 declare <4 x double> @llvm.matrix.multiply(<4 x double>, <4 x double>, i32, i32, i32)
diff --git a/llvm/test/Transforms/Mem2Reg/alloca_addrspace.ll b/llvm/test/Transforms/Mem2Reg/alloca_addrspace.ll
index 87ff922275ce3..f7e8427410ab8 100644
--- a/llvm/test/Transforms/Mem2Reg/alloca_addrspace.ll
+++ b/llvm/test/Transforms/Mem2Reg/alloca_addrspace.ll
@@ -10,6 +10,6 @@ define amdgpu_kernel void @addressspace_alloca() {
 ; CHECK-NEXT:    ret void
 ;
   %alloca = alloca i8, align 8, addrspace(5)
-  call void @llvm.lifetime.start(i64 2, ptr addrspace(5) %alloca)
+  call void @llvm.lifetime.start(ptr addrspace(5) %alloca)
   ret void
 }
diff --git a/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll
index d4bc09731b940..a876319281b17 100644
--- a/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll
+++ b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll
@@ -2,8 +2,8 @@
 ; RUN: opt -passes=mem2reg -S -o - < %s | FileCheck %s
 
 declare void @llvm.assume(i1)
-declare void @llvm.lifetime.start.p0(i64 %size, ptr nocapture %ptr)
-declare void @llvm.lifetime.end.p0(i64 %size, ptr nocapture %ptr)
+declare void @llvm.lifetime.start.p0(ptr nocapture %ptr)
+declare void @llvm.lifetime.end.p0(ptr nocapture %ptr)
 
 define void @positive_assume_uses(ptr %arg) {
 ; CHECK-LABEL: @positive_assume_uses(
@@ -54,10 +54,10 @@ define void @positive_gep_assume_uses() {
 ;
   %A = alloca {i8, i16}
   %B = getelementptr {i8, i16}, ptr %A, i32 0, i32 0
-  call void @llvm.lifetime.start.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.start.p0(ptr %A)
   call void @llvm.assume(i1 true) ["align"(ptr %B, i64 8), "align"(ptr %B, i64 16)]
   store {i8, i16} zeroinitializer, ptr %A
-  call void @llvm.lifetime.end.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.end.p0(ptr %A)
   call void @llvm.assume(i1 true) ["nonnull"(ptr %B), "align"(ptr %B, i64 2)]
   ret void
 }
@@ -70,10 +70,10 @@ define void @positive_mixed_assume_uses() {
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca i8
-  call void @llvm.lifetime.start.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.start.p0(ptr %A)
   call void @llvm.assume(i1 true) ["nonnull"(ptr %A), "align"(ptr %A, i64 8), "align"(ptr %A, i64 16)]
   store i8 1, ptr %A
-  call void @llvm.lifetime.end.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.end.p0(ptr %A)
   call void @llvm.assume(i1 true) ["nonnull"(ptr %A), "align"(ptr %A, i64 2), "nonnull"(ptr %A)]
   call void @llvm.assume(i1 true) ["nonnull"(ptr %A), "align"(ptr %A, i64 2), "nonnull"(ptr %A)]
   ret void
diff --git a/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll b/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll
index bcc96937ec408..510fb2b8638e0 100644
--- a/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll
+++ b/llvm/test/Transforms/Mem2Reg/ignore-lifetime.ll
@@ -1,15 +1,15 @@
 ; RUN: opt -passes=mem2reg -S -o - < %s | FileCheck %s
 
-declare void @llvm.lifetime.start.p0(i64 %size, ptr nocapture %ptr)
-declare void @llvm.lifetime.end.p0(i64 %size, ptr nocapture %ptr)
+declare void @llvm.lifetime.start.p0(ptr nocapture %ptr)
+declare void @llvm.lifetime.end.p0(ptr nocapture %ptr)
 
 define void @test1() {
 ; CHECK: test1
 ; CHECK-NOT: alloca
   %A = alloca i32
-  call void @llvm.lifetime.start.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.start.p0(ptr %A)
   store i32 1, ptr %A
-  call void @llvm.lifetime.end.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.end.p0(ptr %A)
   ret void
 }
 
@@ -17,8 +17,8 @@ define void @test2() {
 ; CHECK: test2
 ; CHECK-NOT: alloca
   %A = alloca {i8, i16}
-  call void @llvm.lifetime.start.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.start.p0(ptr %A)
   store {i8, i16} zeroinitializer, ptr %A
-  call void @llvm.lifetime.end.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.end.p0(ptr %A)
   ret void
 }
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll b/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll
index 601498e36a7a3..a0c0e9f5ba78a 100644
--- a/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll
+++ b/llvm/test/Transforms/MemCpyOpt/callslot_badaa.ll
@@ -5,7 +5,7 @@ declare void @use(ptr)
 ; Make sure callslot optimization merges alias.scope metadata correctly when it merges instructions.
 ; Merging here naively generates:
 ;  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 1, i1 false), !alias.scope !3
-;  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %src), !noalias !0
+;  call void @llvm.lifetime.end.p0(ptr nonnull %src), !noalias !0
 ;   ...
 ;  !0 = !{!1}
 ;  !1 = distinct !{!1, !2, !"callee1: %a"}
@@ -20,18 +20,18 @@ define i8 @test(i8 %input) {
   %src = alloca i8
 ; NOTE: we're matching the full line and looking for the lack of !alias.scope here
 ; CHECK:   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %src, i64 1, i1 false)
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %src), !noalias !3
+  call void @llvm.lifetime.start.p0(ptr nonnull %src), !noalias !3
   store i8 %input, ptr %src
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %src, i64 1, i1 false), !alias.scope !0
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %src), !noalias !3
+  call void @llvm.lifetime.end.p0(ptr nonnull %src), !noalias !3
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp, i64 1, i1 false), !alias.scope !3
   %ret_value = load i8, ptr %dst
   call void @use(ptr %src)
   ret i8 %ret_value
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
 
 !0 = !{!1}
diff --git a/llvm/test/Transforms/MemCpyOpt/capturing-func.ll b/llvm/test/Transforms/MemCpyOpt/capturing-func.ll
index 47c435809f92b..c08f60acd8f3a 100644
--- a/llvm/test/Transforms/MemCpyOpt/capturing-func.ll
+++ b/llvm/test/Transforms/MemCpyOpt/capturing-func.ll
@@ -5,8 +5,8 @@ target datalayout = "e"
 
 declare void @foo(ptr)
 declare void @llvm.memcpy.p0.p0.i32(ptr nocapture, ptr nocapture, i32, i1) nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 ; Check that the transformation isn't applied if the called function can
 ; capture the pointer argument (i.e. the nocapture attribute isn't present)
@@ -51,18 +51,18 @@ define void @test_lifetime_end() {
 ; CHECK-LABEL: define {{[^@]+}}@test_lifetime_end() {
 ; CHECK-NEXT:    [[PTR1:%.*]] = alloca i8, align 1
 ; CHECK-NEXT:    [[PTR2:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[PTR2]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PTR2]])
 ; CHECK-NEXT:    call void @foo(ptr [[PTR1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr [[PTR2]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PTR2]])
 ; CHECK-NEXT:    call void @foo(ptr [[PTR1]])
 ; CHECK-NEXT:    ret void
 ;
   %ptr1 = alloca i8
   %ptr2 = alloca i8
-  call void @llvm.lifetime.start.p0(i64 1, ptr %ptr2)
+  call void @llvm.lifetime.start.p0(ptr %ptr2)
   call void @foo(ptr %ptr2)
   call void @llvm.memcpy.p0.p0.i32(ptr %ptr1, ptr %ptr2, i32 1, i1 false)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %ptr2)
+  call void @llvm.lifetime.end.p0(ptr %ptr2)
   call void @foo(ptr %ptr1)
   ret void
 }
diff --git a/llvm/test/Transforms/MemCpyOpt/lifetime-missing.ll b/llvm/test/Transforms/MemCpyOpt/lifetime-missing.ll
index 0626f09702f7e..06d9434e8c5c8 100644
--- a/llvm/test/Transforms/MemCpyOpt/lifetime-missing.ll
+++ b/llvm/test/Transforms/MemCpyOpt/lifetime-missing.ll
@@ -7,7 +7,7 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
 target triple = "x86_64-grtev4-linux-gnu"
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
 
 define void @test() {
@@ -26,7 +26,7 @@ entry:
   %agg.tmp.sroa.14 = alloca [20 x i8], align 4
   %agg.tmp.sroa.14.128.sroa_idx = getelementptr i8, ptr %agg.tmp.sroa.14, i64 4
   call void @llvm.memset.p0.i64(ptr %agg.tmp.sroa.14.128.sroa_idx, i8 0, i64 1, i1 false)
-  call void @llvm.lifetime.start.p0(i64 20, ptr %agg.tmp3.sroa.35)
+  call void @llvm.lifetime.start.p0(ptr %agg.tmp3.sroa.35)
   call void @llvm.memcpy.p0.p0.i64(ptr %agg.tmp3.sroa.35, ptr %agg.tmp.sroa.14, i64 20, i1 false)
   %agg.tmp3.sroa.35.128.sroa_idx = getelementptr i8, ptr %agg.tmp3.sroa.35, i64 4
   call void @llvm.memcpy.p0.p0.i64(ptr inttoptr (i64 4 to ptr), ptr %agg.tmp3.sroa.35.128.sroa_idx, i64 1, i1 false)
diff --git a/llvm/test/Transforms/MemCpyOpt/lifetime.ll b/llvm/test/Transforms/MemCpyOpt/lifetime.ll
index e9fc06b1e1da9..4eab12a4edda9 100644
--- a/llvm/test/Transforms/MemCpyOpt/lifetime.ll
+++ b/llvm/test/Transforms/MemCpyOpt/lifetime.ll
@@ -5,46 +5,46 @@
 ; @llvm.lifetime.start and @llvm.memcpy.
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1)
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @call_slot(ptr nocapture dereferenceable(16) %arg1) {
 ; CHECK-LABEL: @call_slot(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca [8 x i8], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[TMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[TMP]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP]], i64 7
 ; CHECK-NEXT:    store i8 0, ptr [[TMP10]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[TMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[TMP]])
 ; CHECK-NEXT:    ret void
 ;
 bb:
   %tmp = alloca [8 x i8], align 8
-  call void @llvm.lifetime.start.p0(i64 16, ptr %tmp)
+  call void @llvm.lifetime.start.p0(ptr %tmp)
   %tmp10 = getelementptr inbounds i8, ptr %tmp, i64 7
   store i8 0, ptr %tmp10, align 1
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %arg1, ptr align 8 %tmp, i64 16, i1 false)
-  call void @llvm.lifetime.end.p0(i64 16, ptr %tmp)
+  call void @llvm.lifetime.end.p0(ptr %tmp)
   ret void
 }
 
 define void @memcpy_memcpy_across_lifetime(ptr noalias %p1, ptr noalias %p2, ptr noalias %p3) {
 ; CHECK-LABEL: @memcpy_memcpy_across_lifetime(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [16 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[A]], ptr [[P1:%.*]], i64 16, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P1]], ptr [[P2:%.*]], i64 16, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P2]], ptr [[A]], i64 16, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[P3:%.*]], ptr [[P2]], i64 16, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [16 x i8]
-  call void @llvm.lifetime.start.p0(i64 16, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memcpy.p0.p0.i64(ptr %a, ptr %p1, i64 16, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr %p1, ptr %p2, i64 16, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr %p2, ptr %a, i64 16, i1 false)
-  call void @llvm.lifetime.end.p0(i64 16, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   call void @llvm.memcpy.p0.p0.i64(ptr %p3, ptr %p2, i64 16, i1 false)
   ret void
 }
@@ -55,18 +55,18 @@ define i32 @call_slot_move_lifetime_start() {
 ; CHECK-LABEL: @call_slot_move_lifetime_start(
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DST:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DST]])
 ; CHECK-NEXT:    call void @call(ptr [[DST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DST]])
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[DST]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
   %tmp = alloca i32
   %dst = alloca i32
   call void @call(ptr %tmp)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %dst)
+  call void @llvm.lifetime.start.p0(ptr %dst)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dst, ptr align 4 %tmp, i64 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %dst)
+  call void @llvm.lifetime.end.p0(ptr %dst)
   %v = load i32, ptr %dst
   ret i32 %v
 }
@@ -76,20 +76,20 @@ define i32 @call_slot_two_lifetime_starts() {
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DST:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @call(ptr [[TMP]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DST]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DST]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DST]], ptr align 4 [[TMP]], i64 4, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DST]])
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[DST]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
   %tmp = alloca i32
   %dst = alloca i32
   call void @call(ptr %tmp)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %dst)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %dst)
+  call void @llvm.lifetime.start.p0(ptr %dst)
+  call void @llvm.lifetime.start.p0(ptr %dst)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dst, ptr align 4 %tmp, i64 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %dst)
+  call void @llvm.lifetime.end.p0(ptr %dst)
   %v = load i32, ptr %dst
   ret i32 %v
 }
@@ -100,9 +100,9 @@ define i32 @call_slot_clobber_before_lifetime_start() {
 ; CHECK-NEXT:    [[DST:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @call(ptr [[TMP]])
 ; CHECK-NEXT:    store i32 0, ptr [[DST]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[DST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[DST]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DST]], ptr align 4 [[TMP]], i64 4, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[DST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[DST]])
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[DST]], align 4
 ; CHECK-NEXT:    ret i32 [[V]]
 ;
@@ -110,9 +110,9 @@ define i32 @call_slot_clobber_before_lifetime_start() {
   %dst = alloca i32
   call void @call(ptr %tmp)
   store i32 0, ptr %dst
-  call void @llvm.lifetime.start.p0(i64 4, ptr %dst)
+  call void @llvm.lifetime.start.p0(ptr %dst)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dst, ptr align 4 %tmp, i64 4, i1 false)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %dst)
+  call void @llvm.lifetime.end.p0(ptr %dst)
   %v = load i32, ptr %dst
   ret i32 %v
 }
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-byval-forwarding-clobbers.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-byval-forwarding-clobbers.ll
index 383040c6c89e2..e1b32cdf5ab45 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy-byval-forwarding-clobbers.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-byval-forwarding-clobbers.ll
@@ -7,8 +7,8 @@ declare i1 @check(ptr readonly byval(i64) align 8) readonly argmemonly
 
 declare void @clobber(ptr) argmemonly
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
 
 ; %a.2's lifetime ends before the call to @check. We must remove the call to
@@ -25,11 +25,11 @@ define i1 @alloca_forwarding_lifetime_end_clobber() {
 entry:
   %a.1 = alloca i64, align 8
   %a.2 = alloca i64, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %a.2)
+  call void @llvm.lifetime.start.p0(ptr %a.2)
   call void @init(ptr sret(i64) align 8 %a.2)
   store i8 0, ptr %a.2
   call void @llvm.memcpy.p0.p0.i64(ptr %a.1, ptr %a.2, i64 8, i1 false)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %a.2)
+  call void @llvm.lifetime.end.p0(ptr %a.2)
   ;call void @clobber(ptr %a.2)
   %call = call i1 @check(ptr byval(i64) align 8 %a.1)
   ret i1 %call
@@ -42,7 +42,7 @@ define i1 @alloca_forwarding_call_clobber() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_1:%.*]] = alloca i64, align 8
 ; CHECK-NEXT:    [[A_2:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[A_2]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_2]])
 ; CHECK-NEXT:    call void @init(ptr sret(i64) align 8 [[A_2]])
 ; CHECK-NEXT:    store i8 0, ptr [[A_2]], align 1
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[A_1]], ptr [[A_2]], i64 8, i1 false)
@@ -53,7 +53,7 @@ define i1 @alloca_forwarding_call_clobber() {
 entry:
   %a.1 = alloca i64, align 8
   %a.2 = alloca i64, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %a.2)
+  call void @llvm.lifetime.start.p0(ptr %a.2)
   call void @init(ptr sret(i64) align 8 %a.2)
   store i8 0, ptr %a.2
   call void @llvm.memcpy.p0.p0.i64(ptr %a.1, ptr %a.2, i64 8, i1 false)
@@ -67,7 +67,7 @@ define i1 @alloca_forwarding_call_clobber_after() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_1:%.*]] = alloca i64, align 8
 ; CHECK-NEXT:    [[A_2:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[A_2]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A_2]])
 ; CHECK-NEXT:    call void @init(ptr sret(i64) align 8 [[A_2]])
 ; CHECK-NEXT:    store i8 0, ptr [[A_2]], align 1
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[A_1]], ptr [[A_2]], i64 8, i1 false)
@@ -78,7 +78,7 @@ define i1 @alloca_forwarding_call_clobber_after() {
 entry:
   %a.1 = alloca i64, align 8
   %a.2 = alloca i64, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %a.2)
+  call void @llvm.lifetime.start.p0(ptr %a.2)
   call void @init(ptr sret(i64) align 8 %a.2)
   store i8 0, ptr %a.2
   call void @llvm.memcpy.p0.p0.i64(ptr %a.1, ptr %a.2, i64 8, i1 false)
@@ -102,7 +102,7 @@ entry:
   %a.1 = alloca i64, align 8
   %a.2 = alloca i64, align 8
   %a.3 = alloca i64, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %a.2)
+  call void @llvm.lifetime.start.p0(ptr %a.2)
   call void @init(ptr sret(i64) align 8 %a.2)
   store i8 0, ptr %a.2
   call void @llvm.memcpy.p0.p0.i64(ptr %a.1, ptr %a.2, i64 8, i1 false)
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-gep-modification.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-gep-modification.ll
index ba6faf3578c50..5e81c0d05b2c4 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy-gep-modification.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-gep-modification.ll
@@ -3,8 +3,8 @@
 
 %struct.MaskedType = type { i8, i8 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #0
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
+declare void @llvm.lifetime.end.p0(ptr nocapture) #0
 declare void @MaskedFunction1(ptr, ptr addrspace(1))
 declare void @MaskedFunction2(ptr, ptr)
 
@@ -13,11 +13,11 @@ define i8 @test_gep_not_modified(ptr %in0, ptr %in1) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[FUNCALLOC:%.*]] = alloca [[STRUCT_MASKEDTYPE:%.*]], align 4
 ; CHECK-NEXT:    [[PTRALLOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[PTRALLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PTRALLOC]])
 ; CHECK-NEXT:    [[ADDRSPACECAST:%.*]] = addrspacecast ptr [[PTRALLOC]] to ptr addrspace(1)
 ; CHECK-NEXT:    call void @MaskedFunction1(ptr [[IN1:%.*]], ptr addrspace(1) [[ADDRSPACECAST]])
 ; CHECK-NEXT:    [[LOAD1:%.*]] = load i8, ptr [[PTRALLOC]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[PTRALLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PTRALLOC]])
 ; CHECK-NEXT:    [[GETELEMPTR1:%.*]] = getelementptr inbounds [[STRUCT_MASKEDTYPE]], ptr [[FUNCALLOC]], i32 0, i32 1
 ; CHECK-NEXT:    store i8 [[LOAD1]], ptr [[GETELEMPTR1]], align 1
 ; CHECK-NEXT:    ret i8 0
@@ -25,11 +25,11 @@ define i8 @test_gep_not_modified(ptr %in0, ptr %in1) {
 entry:
   %funcAlloc = alloca %struct.MaskedType, align 4
   %ptrAlloc = alloca i8, align 1
-  call void @llvm.lifetime.start.p0(i64 4, ptr %ptrAlloc) #0
+  call void @llvm.lifetime.start.p0(ptr %ptrAlloc) #0
   %addrspaceCast = addrspacecast ptr %ptrAlloc to ptr addrspace(1)
   call void @MaskedFunction1(ptr %in1, ptr addrspace(1) %addrspaceCast)
   %load1 = load i8, ptr %ptrAlloc, align 1
-  call void @llvm.lifetime.end.p0(i64 4, ptr %ptrAlloc) #0
+  call void @llvm.lifetime.end.p0(ptr %ptrAlloc) #0
   %getElemPtr1 = getelementptr inbounds %struct.MaskedType, ptr %funcAlloc, i32 0, i32 1
   store i8 %load1, ptr %getElemPtr1, align 1
   ret i8 0
@@ -40,19 +40,19 @@ define i8 @test_gep_modified(ptr %in0, ptr %in1) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[FUNCALLOC:%.*]] = alloca [[STRUCT_MASKEDTYPE:%.*]], align 4
 ; CHECK-NEXT:    [[PTRALLOC:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[PTRALLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[PTRALLOC]])
 ; CHECK-NEXT:    [[GETELEMPTR1:%.*]] = getelementptr inbounds [[STRUCT_MASKEDTYPE]], ptr [[FUNCALLOC]], i32 0, i32 1
 ; CHECK-NEXT:    call void @MaskedFunction2(ptr [[IN1:%.*]], ptr [[GETELEMPTR1]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[PTRALLOC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[PTRALLOC]])
 ; CHECK-NEXT:    ret i8 0
 ;
 entry:
   %funcAlloc = alloca %struct.MaskedType, align 4
   %ptrAlloc = alloca i8, align 1
-  call void @llvm.lifetime.start.p0(i64 4, ptr %ptrAlloc) #0
+  call void @llvm.lifetime.start.p0(ptr %ptrAlloc) #0
   call void @MaskedFunction2(ptr %in1, ptr %ptrAlloc)
   %load1 = load i8, ptr %ptrAlloc, align 1
-  call void @llvm.lifetime.end.p0(i64 4, ptr %ptrAlloc) #0
+  call void @llvm.lifetime.end.p0(ptr %ptrAlloc) #0
   %getElemPtr1 = getelementptr inbounds %struct.MaskedType, ptr %funcAlloc, i32 0, i32 1
   store i8 %load1, ptr %getElemPtr1, align 1
   ret i8 0
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
index 1771fe6c31f9b..7a7f8e109cbd0 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
@@ -12,10 +12,10 @@ define void @foo(ptr noalias nocapture sret([8 x i64]) dereferenceable(64) %sret
 ;
 entry-block:
   %a = alloca [8 x i64], align 8
-  call void @llvm.lifetime.start.p0(i64 64, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memset.p0.i64(ptr align 8 %a, i8 0, i64 64, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %sret, ptr align 8 %a, i64 64, i1 false)
-  call void @llvm.lifetime.end.p0(i64 64, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 
 }
@@ -24,28 +24,28 @@ define void @bar(ptr noalias nocapture sret([8 x i64]) dereferenceable(64) %sret
 ; CHECK-LABEL: @bar(
 ; CHECK-NEXT:  entry-block:
 ; CHECK-NEXT:    [[A:%.*]] = alloca [8 x i64], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 64, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(64) [[A]], i8 0, i64 64, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(64) [[SRET:%.*]], i8 0, i64 64, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(32) [[A]], i8 42, i64 32, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(64) [[OUT:%.*]], ptr noundef nonnull align 8 dereferenceable(64) [[A]], i64 64, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 64, ptr nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[A]])
 ; CHECK-NEXT:    ret void
 ;
 entry-block:
   %a = alloca [8 x i64], align 8
-  call void @llvm.lifetime.start.p0(i64 64, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memset.p0.i64(ptr align 8 %a, i8 0, i64 64, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %sret, ptr align 8 %a, i64 64, i1 false)
   call void @llvm.memset.p0.i64(ptr align 8 %a, i8 42, i64 32, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %out, ptr align 8 %a, i64 64, i1 false)
-  call void @llvm.lifetime.end.p0(i64 64, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1) nounwind
 declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll
index 84253dca16bae..6e28811460d55 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-undef.ll
@@ -29,11 +29,11 @@ define i32 @test1(ptr nocapture %foobie) nounwind noinline ssp uwtable {
 define void @test2(ptr sret(i8) noalias nocapture %out) nounwind noinline ssp uwtable {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    [[IN:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[IN]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[IN]])
 ; CHECK-NEXT:    ret void
 ;
   %in = alloca i64
-  call void @llvm.lifetime.start.p0(i64 8, ptr %in)
+  call void @llvm.lifetime.start.p0(ptr %in)
   call void @llvm.memcpy.p0.p0.i64(ptr %out, ptr %in, i64 8, i1 false)
   ret void
 }
@@ -42,12 +42,12 @@ define void @test2(ptr sret(i8) noalias nocapture %out) nounwind noinline ssp uw
 define void @test_lifetime_may_alias(ptr %src, ptr %dst) {
 ; CHECK-LABEL: @test_lifetime_may_alias(
 ; CHECK-NEXT:    [[LIFETIME:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[LIFETIME]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[LIFETIME]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[DST:%.*]], ptr [[SRC:%.*]], i64 8, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %lifetime = alloca i64
-  call void @llvm.lifetime.start.p0(i64 8, ptr %lifetime)
+  call void @llvm.lifetime.start.p0(ptr %lifetime)
   call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 8, i1 false)
   ret void
 }
@@ -56,12 +56,12 @@ define void @test_lifetime_may_alias(ptr %src, ptr %dst) {
 define void @test_lifetime_partial_alias_1(ptr noalias %dst) {
 ; CHECK-LABEL: @test_lifetime_partial_alias_1(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [16 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[A]], i64 8
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [16 x i8]
-  call void @llvm.lifetime.start.p0(i64 16, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   %gep = getelementptr i8, ptr %a, i64 8
   call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %gep, i64 8, i1 false)
   ret void
@@ -71,12 +71,12 @@ define void @test_lifetime_partial_alias_1(ptr noalias %dst) {
 define void @test_lifetime_partial_alias_2(ptr noalias %dst) {
 ; CHECK-LABEL: @test_lifetime_partial_alias_2(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [16 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[A]], i64 8
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [16 x i8]
-  call void @llvm.lifetime.start.p0(i64 16, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   %gep = getelementptr i8, ptr %a, i64 8
   call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %gep, i64 16, i1 false)
   ret void
@@ -84,4 +84,4 @@ define void @test_lifetime_partial_alias_2(ptr noalias %dst) {
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
diff --git a/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll
index 343f95181a8e5..2575d58dcac72 100644
--- a/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memset-memcpy-oversized.ll
@@ -23,17 +23,17 @@ define void @test_alloca(ptr %result) {
 define void @test_alloca_with_lifetimes(ptr %result) {
 ; CHECK-LABEL: @test_alloca_with_lifetimes(
 ; CHECK-NEXT:    [[A:%.*]] = alloca [[T:%.*]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 [[A]], i8 0, i64 12, i1 false)
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr [[RESULT:%.*]], i8 0, i64 12, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca %T, align 8
-  call void @llvm.lifetime.start.p0(i64 16, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   call void @llvm.memset.p0.i64(ptr align 8 %a, i8 0, i64 12, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr %result, ptr align 8 %a, i64 16, i1 false)
-  call void @llvm.lifetime.end.p0(i64 16, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -201,5 +201,5 @@ declare void @free(ptr)
 declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1)
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture readonly, i64, i1)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Transforms/MemCpyOpt/pr29105.ll b/llvm/test/Transforms/MemCpyOpt/pr29105.ll
index d47bddd4597fb..f4538b92e292c 100644
--- a/llvm/test/Transforms/MemCpyOpt/pr29105.ll
+++ b/llvm/test/Transforms/MemCpyOpt/pr29105.ll
@@ -7,34 +7,34 @@ define void @baz() unnamed_addr #0 {
 ; CHECK-LABEL: @baz(
 ; CHECK-NEXT:  entry-block:
 ; CHECK-NEXT:    [[TMP2:%.*]] = alloca [[FOO:%.*]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 16384, ptr nonnull [[TMP2]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP2]])
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr noundef nonnull align 8 dereferenceable(16384) [[TMP2]], i8 0, i64 16384, i1 false)
 ; CHECK-NEXT:    call void @bar(ptr noalias nonnull captures(none) dereferenceable(16384) [[TMP2]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 16384, ptr nonnull [[TMP2]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
 entry-block:
   %x.sroa.0 = alloca [2048 x i64], align 8
   %tmp0 = alloca [2048 x i64], align 8
   %tmp2 = alloca %Foo, align 8
-  call void @llvm.lifetime.start.p0(i64 16384, ptr %x.sroa.0)
-  call void @llvm.lifetime.start.p0(i64 16384, ptr %tmp0)
+  call void @llvm.lifetime.start.p0(ptr %x.sroa.0)
+  call void @llvm.lifetime.start.p0(ptr %tmp0)
   call void @llvm.memset.p0.i64(ptr align 8 %tmp0, i8 0, i64 16384, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %x.sroa.0, ptr align 8 %tmp0, i64 16384, i1 false)
-  call void @llvm.lifetime.end.p0(i64 16384, ptr %tmp0)
-  call void @llvm.lifetime.start.p0(i64 16384, ptr %tmp2)
+  call void @llvm.lifetime.end.p0(ptr %tmp0)
+  call void @llvm.lifetime.start.p0(ptr %tmp2)
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp2, ptr align 8 %x.sroa.0, i64 16384, i1 false)
   call void @bar(ptr noalias nocapture nonnull dereferenceable(16384) %tmp2)
-  call void @llvm.lifetime.end.p0(i64 16384, ptr %tmp2)
-  call void @llvm.lifetime.end.p0(i64 16384, ptr %x.sroa.0)
+  call void @llvm.lifetime.end.p0(ptr %tmp2)
+  call void @llvm.lifetime.end.p0(ptr %x.sroa.0)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1) #1
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 declare void @bar(ptr noalias nocapture readonly dereferenceable(16384)) unnamed_addr #0
 
diff --git a/llvm/test/Transforms/MemCpyOpt/preserve-memssa.ll b/llvm/test/Transforms/MemCpyOpt/preserve-memssa.ll
index ff36bf0315311..e1a6c3f00c05d 100644
--- a/llvm/test/Transforms/MemCpyOpt/preserve-memssa.ll
+++ b/llvm/test/Transforms/MemCpyOpt/preserve-memssa.ll
@@ -78,7 +78,7 @@ define void @test5(ptr %ptr) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[EARLY_DATA:%.*]] = alloca [128 x i8], align 8
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca [[T:%.*]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[EARLY_DATA]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[EARLY_DATA]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[PTR:%.*]], align 8
 ; CHECK-NEXT:    call fastcc void @decompose(ptr [[TMP]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr [[EARLY_DATA]], ptr [[TMP]], i64 32, i1 false)
@@ -87,7 +87,7 @@ define void @test5(ptr %ptr) {
 entry:
   %early_data = alloca [128 x i8], align 8
   %tmp = alloca %t, align 8
-  call void @llvm.lifetime.start.p0(i64 32, ptr %early_data)
+  call void @llvm.lifetime.start.p0(ptr %early_data)
   %0 = load i32, ptr %ptr, align 8
   call fastcc void @decompose(ptr %tmp)
   call void @llvm.memcpy.p0.p0.i64(ptr %early_data, ptr %tmp, i64 32, i1 false)
@@ -131,7 +131,7 @@ define void @test8(ptr noalias %src, ptr %dst) {
 declare void @clobber()
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
 
 ; Function Attrs: argmemonly nounwind willreturn
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #0
diff --git a/llvm/test/Transforms/MemCpyOpt/stack-move.ll b/llvm/test/Transforms/MemCpyOpt/stack-move.ll
index 31e255b83eb9e..940e30ec46881 100644
--- a/llvm/test/Transforms/MemCpyOpt/stack-move.ll
+++ b/llvm/test/Transforms/MemCpyOpt/stack-move.ll
@@ -9,12 +9,12 @@ declare void @llvm.memcpy.p2.p1.i64(ptr addrspace(2) noalias nocapture writeonly
 declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg)
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.start.p1(i64, ptr addrspace(1) nocapture)
-declare void @llvm.lifetime.end.p1(i64, ptr addrspace(1) nocapture)
-declare void @llvm.lifetime.start.p2(i64, ptr addrspace(2) nocapture)
-declare void @llvm.lifetime.end.p2(i64, ptr addrspace(2) nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
+declare void @llvm.lifetime.start.p1(ptr addrspace(1) nocapture)
+declare void @llvm.lifetime.end.p1(ptr addrspace(1) nocapture)
+declare void @llvm.lifetime.start.p2(ptr addrspace(2) nocapture)
+declare void @llvm.lifetime.end.p2(ptr addrspace(2) nocapture)
 
 declare i32 @use_nocapture(ptr nocapture)
 declare i32 @use_maycapture(ptr noundef)
@@ -31,8 +31,8 @@ define void @basic_memcpy() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
@@ -40,8 +40,8 @@ define void @basic_memcpy() {
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
 
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -74,8 +74,8 @@ define void @basic_memmove() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
@@ -83,8 +83,8 @@ define void @basic_memmove() {
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
 
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -99,8 +99,8 @@ define void @load_store() {
 ;
   %src = alloca i32, align 4
   %dest = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store i32 42, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
@@ -108,8 +108,8 @@ define void @load_store() {
   store i32 %src.val, ptr %dest
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -125,8 +125,8 @@ define void @load_store_scalable(<vscale x 4 x i32> %x) {
 ;
   %src = alloca <vscale x 4 x i32>
   %dest = alloca <vscale x 4 x i32>
-  call void @llvm.lifetime.start.p0(i64 -1, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store <vscale x 4 x i32> %x, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
@@ -135,8 +135,8 @@ define void @load_store_scalable(<vscale x 4 x i32> %x) {
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -151,16 +151,16 @@ define void @align_up() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 8
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -177,21 +177,21 @@ define void @remove_extra_lifetime_intrinsics() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
   %3 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -230,8 +230,8 @@ define void @alias_no_mod() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   %dest.alias = getelementptr %struct.Foo, ptr %dest, i32 0, i32 0
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
@@ -240,8 +240,8 @@ define void @alias_no_mod() {
 
   %src.alias = getelementptr %struct.Foo, ptr %src, i32 0, i32 0
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -265,16 +265,16 @@ define void @remove_scoped_noalias() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src), !alias.scope !2
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
   %2 = call i32 @use_nocapture(ptr nocapture %dest), !noalias !2
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -289,16 +289,16 @@ define void @remove_alloca_metadata() {
 ;
   %src = alloca %struct.Foo, align 4, !annotation !3
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src), !alias.scope !2
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest), !noalias !2
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -314,16 +314,16 @@ define void @noalias_on_lifetime() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src), !alias.scope !2
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src), !alias.scope !2
+  call void @llvm.lifetime.end.p0(ptr nocapture %src), !alias.scope !2
   %2 = call i32 @use_nocapture(ptr nocapture %dest), !noalias !2
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest), !noalias !2
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest), !noalias !2
   ret void
 }
 
@@ -338,16 +338,16 @@ define void @src_ref_dest_ref_after_copy() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   %1 = call i32 @use_readonly(ptr nocapture %src)
   %2 = call i32 @use_readonly(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -362,16 +362,16 @@ define void @src_mod_dest_mod_after_copy() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   %1 = call i32 @use_writeonly(ptr nocapture %src)
   %2 = call i32 @use_writeonly(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -384,10 +384,10 @@ define void @avoid_memory_use_last_user_crash() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
   %v = load i32, ptr %dest
   ret void
 }
@@ -409,14 +409,14 @@ define void @terminator_lastuse() personality i32 0 {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
-  call void @llvm.lifetime.end.p0(i64 12, ptr %src)
+  call void @llvm.lifetime.end.p0(ptr %src)
   %rv = invoke i32 @use_nocapture(ptr %dest)
   to label %suc unwind label %unw
 unw:
@@ -441,8 +441,8 @@ define void @multi_bb_memcpy(i1 %b) {
 ;
   %src = alloca i32, align 4
   %dest = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store i32 42, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
   br label %bb0
@@ -453,8 +453,8 @@ bb0:
 
 bb1:
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -471,8 +471,8 @@ define void @multi_bb_load_store(i1 %b) {
 ;
   %src = alloca i32, align 4
   %dest = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store i32 42, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
@@ -482,8 +482,8 @@ define void @multi_bb_load_store(i1 %b) {
 
 bb0:
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -494,8 +494,8 @@ define void @multi_bb_separated_load_store(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store i32 42, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    [[SRC_VAL:%.*]] = load i32, ptr [[SRC]], align 4
@@ -505,14 +505,14 @@ define void @multi_bb_separated_load_store(i1 %b) {
 ; CHECK-NEXT:    br label [[BB1:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i32, align 4
   %dest = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store i32 42, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
@@ -525,8 +525,8 @@ bb0:
 
 bb1:
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -548,8 +548,8 @@ define void @multi_bb_simple_br(i1 %b) {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
@@ -564,8 +564,8 @@ bb1:
   br label %bb2
 
 bb2:
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -598,7 +598,7 @@ bb1:
   br label %bb2
 
 bb2:
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
   %1 = call i32 @use_nocapture(ptr noundef nocapture %dest)
 
@@ -620,7 +620,7 @@ define void @multi_bb_dom_test1(i1 %b) {
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 40, i32 50, i32 60 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
@@ -641,7 +641,7 @@ bb1:
   br label %bb2
 
 bb2:
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false); 1
   %1 = call i32 @use_nocapture(ptr noundef nocapture %dest)
 
@@ -671,7 +671,7 @@ define void @multi_bb_pdom_test0(i1 %b) {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false); 1
   br i1 %b, label %bb0, label %bb1
@@ -686,7 +686,7 @@ bb1:
 
 bb2:
   %3 = call i32 @use_nocapture(ptr noundef nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 
   uselistorder ptr %dest, {  2, 3, 0, 1, 4, 5 }
@@ -711,7 +711,7 @@ define void @multi_bb_pdom_test1(i1 %b) {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false); 1
   br i1 %b, label %bb0, label %bb1
@@ -747,7 +747,7 @@ define void @multi_bb_pdom_test2(i1 %b) {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false); 1
   %1 = call i32 @use_nocapture(ptr noundef nocapture %dest)
@@ -784,8 +784,8 @@ entry:
   %nlt1 = icmp slt i32 %n, 1
   %src = alloca %struct.Foo, align 8
   %dest = alloca %struct.Foo, align 8
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 0, i32 1, i32 42 }, ptr %src
   br i1 %nlt1, label %loop_exit, label %loop_body
 
@@ -816,8 +816,8 @@ define void @multi_bb_unreachable_modref(i1 %b0) {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   br i1 %b0, label %bb0, label %exit
@@ -828,8 +828,8 @@ exit:
 
 bb0:
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -850,8 +850,8 @@ define void @multi_bb_non_dominated(i1 %b0, i1 %b1) {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   br i1 %b0, label %bb0, label %bb1
@@ -865,8 +865,8 @@ bb1:
   br label %bb2
 
 bb2:
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -878,30 +878,30 @@ define void @memcpy_is_def() {
 ; CHECK-LABEL: define void @memcpy_is_def() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[SRC]], ptr align 4 [[DEST]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
   %2 = call i32 @use_nocapture(ptr noundef nocapture %dest)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %src, ptr align 4 %dest, i64 12, i1 false)
   %3 = call i32 @use_nocapture(ptr noundef nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -912,30 +912,30 @@ define void @memset_is_def() {
 ; CHECK-LABEL: define void @memset_is_def() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 4 [[SRC]], i8 42, i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
   %2 = call i32 @use_nocapture(ptr noundef nocapture %dest)
   call void @llvm.memset.p0.i64(ptr align 4 %src, i8 42, i64 12, i1 false)
   %3 = call i32 @use_nocapture(ptr noundef nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -946,8 +946,8 @@ define void @store_is_def() {
 ; CHECK-LABEL: define void @store_is_def() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store i32 42, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[SRC]], align 4
@@ -955,14 +955,14 @@ define void @store_is_def() {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    store i32 64, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i32, align 4
   %dest = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store i32 42, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   %2 = load i32, ptr %src
@@ -970,8 +970,8 @@ define void @store_is_def() {
   %3 = call i32 @use_nocapture(ptr noundef nocapture %dest)
   store i32 64, ptr %src
   %4 = call i32 @use_nocapture(ptr noundef nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -982,8 +982,8 @@ define void @multi_bb_dataflow(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
@@ -995,14 +995,14 @@ define void @multi_bb_dataflow(i1 %b) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
@@ -1017,8 +1017,8 @@ bb1:
   br label %bb2
 
 bb2:
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1031,26 +1031,26 @@ define void @incomplete_memcpy() {
 ; CHECK-LABEL: define void @incomplete_memcpy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 11, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 11, i1 false)
   %2 = call i32 @use_nocapture(ptr noundef nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1060,28 +1060,28 @@ define void @incomplete_store() {
 ; CHECK-LABEL: define void @incomplete_store() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[SRC]], align 4
 ; CHECK-NEXT:    store i32 [[TMP2]], ptr [[DEST]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   %2 = load i32, ptr %src
   store i32 %2, ptr %dest
   %3 = call i32 @use_nocapture(ptr noundef nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1091,28 +1091,28 @@ define void @dynamically_sized_alloca(i64 %i) {
 ; CHECK-SAME: (i64 [[I:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i8, i64 [[I]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i8, i64 [[I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO:%.*]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i8, i64 %i, align 4
   %dest = alloca i8, i64 %i, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1122,28 +1122,28 @@ define void @inalloca() {
 ; CHECK-LABEL: define void @inalloca() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca inalloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca inalloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1153,28 +1153,28 @@ define void @dynamically_sized_memcpy(i64 %size) {
 ; CHECK-SAME: (i64 [[SIZE:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 [[SIZE]], i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 %size, i1 false)
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
   ret void
 }
 
@@ -1183,28 +1183,28 @@ define void @mismatched_alloca_size() {
 ; CHECK-LABEL: define void @mismatched_alloca_size() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i8, i64 24, align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i8, i64 12, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 24, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO:%.*]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 24, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i8, i64 24, align 4
   %dest = alloca i8, i64 12, align 4
-  call void @llvm.lifetime.start.p0(i64 24, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1213,28 +1213,28 @@ define void @mismatched_alloca_addrspace() {
 ; CHECK-LABEL: define void @mismatched_alloca_addrspace() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca i8, i64 24, align 4, addrspace(1)
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca i8, i64 12, align 4, addrspace(2)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p1(i64 24, ptr addrspace(1) captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p2(i64 12, ptr addrspace(2) captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p1(ptr addrspace(1) captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p2(ptr addrspace(2) captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO:%.*]] { i32 10, i32 20, i32 30 }, ptr addrspace(1) [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr addrspace(1) captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p2.p1.i64(ptr addrspace(2) align 4 [[DEST]], ptr addrspace(1) align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p1(i64 24, ptr addrspace(1) captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p1(ptr addrspace(1) captures(none) [[SRC]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr addrspace(2) captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p2(i64 12, ptr addrspace(2) captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p2(ptr addrspace(2) captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca i8, i64 24, align 4, addrspace(1)
   %dest = alloca i8, i64 12, align 4, addrspace(2)
-  call void @llvm.lifetime.start.p1(i64 24, ptr addrspace(1) nocapture %src)
-  call void @llvm.lifetime.start.p2(i64 12, ptr addrspace(2) nocapture %dest)
+  call void @llvm.lifetime.start.p1(ptr addrspace(1) nocapture %src)
+  call void @llvm.lifetime.start.p2(ptr addrspace(2) nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr addrspace(1) %src
   %1 = call i32 @use_nocapture(ptr addrspace(1) nocapture %src)
 
   call void @llvm.memcpy.p2.p1.i64(ptr addrspace(2) align 4 %dest, ptr addrspace(1) align 4 %src, i64 12, i1 false)
 
-  call void @llvm.lifetime.end.p1(i64 24, ptr addrspace(1) nocapture %src)
+  call void @llvm.lifetime.end.p1(ptr addrspace(1) nocapture %src)
   %2 = call i32 @use_nocapture(ptr addrspace(2) nocapture %dest)
-  call void @llvm.lifetime.end.p2(i64 12, ptr addrspace(2) nocapture %dest)
+  call void @llvm.lifetime.end.p2(ptr addrspace(2) nocapture %dest)
   ret void
 }
 
@@ -1243,28 +1243,28 @@ define void @volatile_memcpy() {
 ; CHECK-LABEL: define void @volatile_memcpy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 true)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 true)
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1273,28 +1273,28 @@ define void @dest_captured() {
 ; CHECK-LABEL: define void @dest_captured() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_maycapture(ptr [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
 
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
   %2 = call i32 @use_maycapture(ptr %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1303,28 +1303,28 @@ define void @src_captured() {
 ; CHECK-LABEL: define void @src_captured() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_maycapture(ptr [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_maycapture(ptr %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1334,30 +1334,30 @@ define void @mod_ref_before_copy() {
 ; CHECK-LABEL: define void @mod_ref_before_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[R:%.*]] = call i32 @use_readonly(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %r = call i32 @use_readonly(ptr nocapture %dest)
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1366,30 +1366,30 @@ define void @mod_dest_before_copy() {
 ; CHECK-LABEL: define void @mod_dest_before_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    store i32 13, ptr [[DEST]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   store i32 13, ptr %dest
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1397,22 +1397,22 @@ define void @mod_src_before_store_after_load() {
 ; CHECK-LABEL: define void @mod_src_before_store_after_load() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    store i32 13, ptr [[DEST]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 13, i32 13, i32 13 }, ptr [[SRC]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   store i32 13, ptr %dest
   %1 = call i32 @use_nocapture(ptr nocapture %src)
@@ -1421,9 +1421,9 @@ define void @mod_src_before_store_after_load() {
   store %struct.Foo { i32 13, i32 13, i32 13 }, ptr %src
   store %struct.Foo %src.val, ptr %dest
 
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1433,28 +1433,28 @@ define void @src_mod_dest_ref_after_copy() {
 ; CHECK-LABEL: define void @src_mod_dest_ref_after_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 13, i32 13, i32 13 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   store %struct.Foo { i32 13, i32 13, i32 13 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1464,28 +1464,28 @@ define void @src_ref_dest_mod_after_copy() {
 ; CHECK-LABEL: define void @src_ref_dest_mod_after_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 13, i32 13, i32 13 }, ptr [[DEST]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   store %struct.Foo { i32 13, i32 13, i32 13 }, ptr %dest
   %1 = call i32 @use_nocapture(ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1494,22 +1494,22 @@ define void @dest_alias_mod_before_copy() {
 ; CHECK-LABEL: define void @dest_alias_mod_before_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[DEST_ALIAS:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr [[DEST]], i64 0, i32 1
 ; CHECK-NEXT:    store i32 13, ptr [[DEST_ALIAS]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %dest.alias = getelementptr inbounds %struct.Foo, ptr %dest, i64 0, i32 1
   store i32 13, ptr %dest.alias
@@ -1518,8 +1518,8 @@ define void @dest_alias_mod_before_copy() {
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1528,22 +1528,22 @@ define void @alias_src_ref_dest_mod_after_copy() {
 ; CHECK-LABEL: define void @alias_src_ref_dest_mod_after_copy() {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
 ; CHECK-NEXT:    [[DEST_ALIAS:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr [[DEST]], i64 0, i32 1
 ; CHECK-NEXT:    store i32 13, ptr [[DEST_ALIAS]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @use_nocapture(ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
@@ -1552,8 +1552,8 @@ define void @alias_src_ref_dest_mod_after_copy() {
   %dest.alias = getelementptr inbounds %struct.Foo, ptr %dest, i64 0, i32 1
   store i32 13, ptr %dest.alias
   %2 = call i32 @use_nocapture(ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1564,8 +1564,8 @@ define void @multi_bb_dataflow_conflict(i1 %b) {
 ; CHECK-SAME: (i1 [[B:%.*]]) {
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 10, i32 20, i32 30 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[SRC]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC]], i64 12, i1 false)
@@ -1578,14 +1578,14 @@ define void @multi_bb_dataflow_conflict(i1 %b) {
 ; CHECK-NEXT:    br label [[BB2]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @use_nocapture(ptr noundef captures(none) [[DEST]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    ret void
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr noundef nocapture %src)
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
@@ -1601,8 +1601,8 @@ bb1:
 
 bb2:
   %4 = call i32 @use_nocapture(ptr noundef nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
@@ -1614,8 +1614,8 @@ define void @multi_bb_loop_dest_mod_before_copy(i32 %n) {
 ; CHECK-NEXT:    [[NLT1:%.*]] = icmp slt i32 [[N]], 1
 ; CHECK-NEXT:    [[SRC:%.*]] = alloca [[STRUCT_FOO:%.*]], align 8
 ; CHECK-NEXT:    [[DEST:%.*]] = alloca [[STRUCT_FOO]], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[SRC]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 12, ptr captures(none) [[DEST]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[SRC]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr captures(none) [[DEST]])
 ; CHECK-NEXT:    store [[STRUCT_FOO]] { i32 0, i32 1, i32 42 }, ptr [[SRC]], align 4
 ; CHECK-NEXT:    br i1 [[NLT1]], label [[LOOP_EXIT:%.*]], label [[LOOP_BODY:%.*]]
 ; CHECK:       loop_body:
@@ -1632,8 +1632,8 @@ entry:
   %nlt1 = icmp slt i32 %n, 1
   %src = alloca %struct.Foo, align 8
   %dest = alloca %struct.Foo, align 8
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 0, i32 1, i32 42 }, ptr %src
   br i1 %nlt1, label %loop_exit, label %loop_body
 
@@ -1660,17 +1660,17 @@ define void @partial_lifetime() {
 ;
   %src = alloca %struct.Foo, align 4
   %dest = alloca %struct.Foo, align 4
-  call void @llvm.lifetime.start.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.start.p0(i64 3, ptr nocapture %dest)
+  call void @llvm.lifetime.start.p0(ptr nocapture %src)
+  call void @llvm.lifetime.start.p0(ptr nocapture %dest)
   store %struct.Foo { i32 10, i32 20, i32 30 }, ptr %src
   %1 = call i32 @use_nocapture(ptr nocapture %src)
 
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src, i64 12, i1 false)
 
-  call void @llvm.lifetime.end.p0(i64 3, ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
   %2 = call i32 @use_nocapture(ptr nocapture %dest)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %src)
-  call void @llvm.lifetime.end.p0(i64 12, ptr nocapture %dest)
+  call void @llvm.lifetime.end.p0(ptr nocapture %src)
+  call void @llvm.lifetime.end.p0(ptr nocapture %dest)
   ret void
 }
 
diff --git a/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll b/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
index 179d6e6dcfdf5..e2f500753bbdc 100644
--- a/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
+++ b/llvm/test/Transforms/MergeICmps/X86/int64-and-ptr.ll
@@ -35,10 +35,10 @@ if.end5:                                          ; preds = %if.then, %entry
   ret i1 %rez.0
 }
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="none" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/MoveAutoInit/clobber.ll b/llvm/test/Transforms/MoveAutoInit/clobber.ll
index 08ffb13ad4b99..f52034dce217d 100644
--- a/llvm/test/Transforms/MoveAutoInit/clobber.ll
+++ b/llvm/test/Transforms/MoveAutoInit/clobber.ll
@@ -10,9 +10,9 @@ define i32 @foo(i32 noundef %0, i32 noundef %1, i32 noundef %2) #0 {
 ; CHECK-NEXT:    [[TMP4:%.*]] = alloca [100 x i8], align 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = alloca [2 x i8], align 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [100 x i8], ptr [[TMP4]], i64 0, i64 0
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 100, ptr nonnull [[TMP4]]) #[[ATTR3:[0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP4]]) #[[ATTR3:[0-9]+]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [2 x i8], ptr [[TMP5]], i64 0, i64 0
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 2, ptr nonnull [[TMP5]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP5]]) #[[ATTR3]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [2 x i8], ptr [[TMP5]], i64 0, i64 1
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP1:%.*]], 0
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[TMP15:%.*]], label [[TMP10:%.*]]
@@ -38,19 +38,19 @@ define i32 @foo(i32 noundef %0, i32 noundef %1, i32 noundef %2) #0 {
 ; CHECK-NEXT:    br label [[TMP22]]
 ; CHECK:       22:
 ; CHECK-NEXT:    [[TMP23:%.*]] = phi i32 [ [[TMP14]], [[TMP10]] ], [ [[TMP21]], [[TMP17]] ], [ 0, [[TMP15]] ]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 2, ptr nonnull [[TMP5]]) #[[ATTR3]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 100, ptr nonnull [[TMP4]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP5]]) #[[ATTR3]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP4]]) #[[ATTR3]]
 ; CHECK-NEXT:    ret i32 [[TMP23]]
 ;
 
   %4 = alloca [100 x i8], align 16
   %5 = alloca [2 x i8], align 1
   %6 = getelementptr inbounds [100 x i8], ptr %4, i64 0, i64 0
-  call void @llvm.lifetime.start.p0(i64 100, ptr nonnull %4) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %4) #3
   ; This memset must move.
   call void @llvm.memset.p0.i64(ptr noundef nonnull align 16 dereferenceable(100) %6, i8 -86, i64 100, i1 false), !annotation !0
   %7 = getelementptr inbounds [2 x i8], ptr %5, i64 0, i64 0
-  call void @llvm.lifetime.start.p0(i64 2, ptr nonnull %5) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %5) #3
   ; This store must move.
   store i8 -86, ptr %7, align 1, !annotation !0
   %8 = getelementptr inbounds [2 x i8], ptr %5, i64 0, i64 1
@@ -81,16 +81,16 @@ define i32 @foo(i32 noundef %0, i32 noundef %1, i32 noundef %2) #0 {
 
 22:
   %23 = phi i32 [ %14, %10 ], [ %21, %17 ], [ 0, %15 ]
-  call void @llvm.lifetime.end.p0(i64 2, ptr nonnull %5) #3
-  call void @llvm.lifetime.end.p0(i64 100, ptr nonnull %4) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %5) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %4) #3
   ret i32 %23
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) #2
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { mustprogress nofree nosync nounwind readnone uwtable willreturn }
 attributes #1 = { argmemonly mustprogress nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/NewGVN/coercion-different-ptr.ll b/llvm/test/Transforms/NewGVN/coercion-different-ptr.ll
index dfd6d7d9f53a5..979aa69d46363 100644
--- a/llvm/test/Transforms/NewGVN/coercion-different-ptr.ll
+++ b/llvm/test/Transforms/NewGVN/coercion-different-ptr.ll
@@ -11,7 +11,7 @@ define void @foo(ptr %arg) {
 ; CHECK-SAME: ptr [[ARG:%.*]]) {
 ; CHECK-NEXT:  [[BB:.*:]]
 ; CHECK-NEXT:    [[ALLOCA:%.*]] = alloca i8, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[ALLOCA]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[ALLOCA]])
 ; CHECK-NEXT:    [[LOAD:%.*]] = load ptr, ptr [[ARG]], align 8
 ; CHECK-NEXT:    [[LOAD1:%.*]] = load ptr, ptr [[LOAD]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = call ptr [[LOAD1]](ptr [[ALLOCA]])
@@ -19,14 +19,14 @@ define void @foo(ptr %arg) {
 ;
 bb:
   %alloca = alloca i8, align 16
-  call void @llvm.lifetime.start.p0(i64 1, ptr %alloca)
+  call void @llvm.lifetime.start.p0(ptr %alloca)
   %load = load ptr, ptr %arg, align 8
   %load1 = load ptr, ptr %load, align 8
   %call = call ptr %load1(ptr %alloca)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none)) #0
+declare void @llvm.lifetime.start.p0(ptr captures(none)) #0
 
 declare ptr @malloc(i64)
 
diff --git a/llvm/test/Transforms/NewGVN/cond_br2-xfail.ll b/llvm/test/Transforms/NewGVN/cond_br2-xfail.ll
index 017f608aaa63c..8b2d662e484fc 100644
--- a/llvm/test/Transforms/NewGVN/cond_br2-xfail.ll
+++ b/llvm/test/Transforms/NewGVN/cond_br2-xfail.ll
@@ -18,7 +18,7 @@ define void @_Z4testv() #0 personality ptr @__gxx_personality_v0 {
 
 entry:
   %sv = alloca %"class.llvm::SmallVector", align 16
-  call void @llvm.lifetime.start.p0(i64 64, ptr %sv) #1
+  call void @llvm.lifetime.start.p0(ptr %sv) #1
   %FirstEl.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", ptr %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 3
   store ptr %FirstEl.i.i.i.i.i.i, ptr %sv, align 16, !tbaa !4
   %EndX.i.i.i.i.i.i = getelementptr inbounds %"class.llvm::SmallVector", ptr %sv, i64 0, i32 0, i32 0, i32 0, i32 0, i32 1
@@ -87,7 +87,7 @@ if.then.i.i.i20:                                  ; preds = %invoke.cont3
   br label %_ZN4llvm11SmallVectorIiLj8EED1Ev.exit21
 
 _ZN4llvm11SmallVectorIiLj8EED1Ev.exit21:          ; preds = %invoke.cont3, %if.then.i.i.i20
-  call void @llvm.lifetime.end.p0(i64 64, ptr %sv) #1
+  call void @llvm.lifetime.end.p0(ptr %sv) #1
   ret void
 
 lpad:                                             ; preds = %if.end.i14, %if.end.i, %invoke.cont2
@@ -106,14 +106,14 @@ eh.resume:                                        ; preds = %if.then.i.i.i, %lpa
 }
 
 ; Function Attrs: nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare i32 @__gxx_personality_v0(...)
 
 declare void @_Z1gRN4llvm11SmallVectorIiLj8EEE(ptr) #2
 
 ; Function Attrs: nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 declare void @_ZN4llvm15SmallVectorBase8grow_podEmm(ptr, i64, i64) #2
 
diff --git a/llvm/test/Transforms/NewGVN/lifetime-simple.ll b/llvm/test/Transforms/NewGVN/lifetime-simple.ll
index 0a7bd339bfabe..7fe66499dd811 100644
--- a/llvm/test/Transforms/NewGVN/lifetime-simple.ll
+++ b/llvm/test/Transforms/NewGVN/lifetime-simple.ll
@@ -9,21 +9,21 @@ define i8 @test() nounwind {
 ; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[P:%.*]] = alloca [32 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[P]])
 ; CHECK-NEXT:    store i8 1, ptr [[P]], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr [[P]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[P]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i8, ptr [[P]], align 1
 ; CHECK-NEXT:    ret i8 [[TMP0]]
 ;
 entry:
   %P = alloca [32 x i8]
-  call void @llvm.lifetime.start.p0(i64 32, ptr %P)
+  call void @llvm.lifetime.start.p0(ptr %P)
   %0 = load i8, ptr %P
   store i8 1, ptr %P
-  call void @llvm.lifetime.end.p0(i64 32, ptr %P)
+  call void @llvm.lifetime.end.p0(ptr %P)
   %1 = load i8, ptr %P
   ret i8 %1
 }
 
-declare void @llvm.lifetime.start.p0(i64 %S, ptr nocapture %P) readonly
-declare void @llvm.lifetime.end.p0(i64 %S, ptr nocapture %P)
+declare void @llvm.lifetime.start.p0(ptr nocapture %P) readonly
+declare void @llvm.lifetime.end.p0(ptr nocapture %P)
diff --git a/llvm/test/Transforms/NewGVN/verify-memoryphi.ll b/llvm/test/Transforms/NewGVN/verify-memoryphi.ll
index a19a2a6b7e4f3..15bb1cbe92db4 100644
--- a/llvm/test/Transforms/NewGVN/verify-memoryphi.ll
+++ b/llvm/test/Transforms/NewGVN/verify-memoryphi.ll
@@ -5,7 +5,7 @@
 ; REQUIRES: asserts
 
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 define void @tinkywinky() {
 ; CHECK-LABEL: define void @tinkywinky() {
@@ -20,11 +20,11 @@ define void @tinkywinky() {
 ;
 entry:
   %a = alloca i8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   br i1 false, label %body, label %end
 
 body:
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   br label %end
 
 end:
diff --git a/llvm/test/Transforms/NewGVN/vscale.ll b/llvm/test/Transforms/NewGVN/vscale.ll
index 702117213f6eb..64e22e1dda6a3 100644
--- a/llvm/test/Transforms/NewGVN/vscale.ll
+++ b/llvm/test/Transforms/NewGVN/vscale.ll
@@ -579,7 +579,7 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
 ; CHECK-LABEL: @bigexample(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[REF_TMP:%.*]] = alloca { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }, align 16
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr nonnull [[REF_TMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[REF_TMP]])
 ; CHECK-NEXT:    [[A_ELT:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[A:%.*]], 0
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[A_ELT]], ptr [[REF_TMP]], align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
@@ -603,12 +603,12 @@ define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP9]], <vscale x 16 x i8> [[DOTUNPACK10]], 2
 ; CHECK-NEXT:    [[DOTUNPACK12:%.*]] = load <vscale x 16 x i8>, ptr [[REF_TMP_REPACK5]], align 16
 ; CHECK-NEXT:    [[TMP15:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP12]], <vscale x 16 x i8> [[DOTUNPACK12]], 3
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 -1, ptr nonnull [[REF_TMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[REF_TMP]])
 ; CHECK-NEXT:    ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP15]]
 ;
 entry:
   %ref.tmp = alloca { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> }, align 16
-  call void @llvm.lifetime.start.p0(i64 -1, ptr nonnull %ref.tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %ref.tmp)
   %a.elt = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %a, 0
   store <vscale x 4 x i32> %a.elt, ptr %ref.tmp, align 16
   %0 = call i64 @llvm.vscale.i64()
@@ -643,7 +643,7 @@ entry:
   %.elt11 = getelementptr inbounds i8, ptr %ref.tmp, i64 %14
   %.unpack12 = load <vscale x 16 x i8>, ptr %.elt11, align 16
   %15 = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %12, <vscale x 16 x i8> %.unpack12, 3
-  call void @llvm.lifetime.end.p0(i64 -1, ptr nonnull %ref.tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %ref.tmp)
   ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %15
 }
 
diff --git a/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll b/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll
index 180fd0a561716..694deb37873a0 100644
--- a/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll
+++ b/llvm/test/Transforms/ObjCARC/inlined-autorelease-return-value.ll
@@ -7,8 +7,8 @@ declare ptr @llvm.objc.autoreleaseReturnValue(ptr)
 declare ptr @llvm.objc.retainAutoreleasedReturnValue(ptr)
 declare ptr @llvm.objc.unsafeClaimAutoreleasedReturnValue(ptr)
 declare void @opaque()
-declare void @llvm.lifetime.start(i64, ptr nocapture)
-declare void @llvm.lifetime.end(i64, ptr nocapture)
+declare void @llvm.lifetime.start(ptr nocapture)
+declare void @llvm.lifetime.end(ptr nocapture)
 
 ; CHECK-LABEL: define ptr @elide_with_retainRV(
 ; CHECK-NEXT:  entry:
@@ -81,16 +81,16 @@ entry:
 ; CHECK-LABEL: define ptr @elide_with_retainRV_splitByLifetime(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %x = alloca ptr
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr %x)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr %x)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT:    ret ptr %x
 define ptr @elide_with_retainRV_splitByLifetime() nounwind {
 entry:
   ; Cleanup should skip over lifetime intrinsics.
   %x = alloca ptr
-  call void @llvm.lifetime.start(i64 8, ptr %x)
+  call void @llvm.lifetime.start(ptr %x)
   %b = call ptr @llvm.objc.autoreleaseReturnValue(ptr %x) nounwind
-  call void @llvm.lifetime.end(i64 8, ptr %x)
+  call void @llvm.lifetime.end(ptr %x)
   %d = call ptr @llvm.objc.retainAutoreleasedReturnValue(ptr %b) nounwind
   ret ptr %d
 }
@@ -221,17 +221,17 @@ entry:
 ; CHECK-LABEL: define ptr @elide_with_claimRV_splitByLifetime(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    %x = alloca ptr
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr %x)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr %x)
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr %x)
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr %x)
 ; CHECK-NEXT:    tail call void @llvm.objc.release(ptr %x)
 ; CHECK-NEXT:    ret ptr %x
 define ptr @elide_with_claimRV_splitByLifetime() nounwind {
 entry:
   ; Cleanup should skip over lifetime intrinsics.
   %x = alloca ptr
-  call void @llvm.lifetime.start(i64 8, ptr %x)
+  call void @llvm.lifetime.start(ptr %x)
   %b = call ptr @llvm.objc.autoreleaseReturnValue(ptr %x) nounwind
-  call void @llvm.lifetime.end(i64 8, ptr %x)
+  call void @llvm.lifetime.end(ptr %x)
   %d = call ptr @llvm.objc.unsafeClaimAutoreleasedReturnValue(ptr %b) nounwind
   ret ptr %d
 }
diff --git a/llvm/test/Transforms/ObjCARC/post-inlining.ll b/llvm/test/Transforms/ObjCARC/post-inlining.ll
index c15e0896ae0ab..b184bea55077a 100644
--- a/llvm/test/Transforms/ObjCARC/post-inlining.ll
+++ b/llvm/test/Transforms/ObjCARC/post-inlining.ll
@@ -65,22 +65,22 @@ entry:
 
 ; 2) Lifetime markers.
 
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 
 ; CHECK-LABEL: define ptr @testLifetime(
 ; CHECK: entry:
 ; CHECK-NEXT: %obj = alloca i8
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr %obj)
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr %obj)
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr %obj)
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr %obj)
 ; CHECK-NEXT: ret ptr %call.i
 ; CHECK-NEXT: }
 define ptr @testLifetime(ptr %call.i) {
 entry:
   %obj = alloca i8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %obj)
+  call void @llvm.lifetime.start.p0(ptr %obj)
   %0 = tail call ptr @llvm.objc.autoreleaseReturnValue(ptr %call.i) nounwind
-  call void @llvm.lifetime.end.p0(i64 8, ptr %obj)
+  call void @llvm.lifetime.end.p0(ptr %obj)
   %1 = tail call ptr @llvm.objc.retainAutoreleasedReturnValue(ptr %call.i) nounwind
   ret ptr %call.i
 }
diff --git a/llvm/test/Transforms/ObjCARC/related-check.ll b/llvm/test/Transforms/ObjCARC/related-check.ll
index 7c56b2df5a5a9..045c001af4016 100644
--- a/llvm/test/Transforms/ObjCARC/related-check.ll
+++ b/llvm/test/Transforms/ObjCARC/related-check.ll
@@ -52,9 +52,9 @@ for.cond.cleanup:                                 ; preds = %for.cond.cleanup.lo
 
 for.body:                                         ; preds = %for.body.lr.ph, %if.end19
   %i.032 = phi i32 [ 1, %for.body.lr.ph ], [ %inc, %if.end19 ]
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %persistent) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %persistent) #4
   store i32 0, ptr %persistent, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %personalized) #4
+  call void @llvm.lifetime.start.p0(ptr nonnull %personalized) #4
   store i32 0, ptr %personalized, align 4
   %call = call zeroext i1 @lookupType(ptr noundef nonnull %persistent, ptr noundef nonnull %personalized) #8, !clang.arc.no_objc_arc_exceptions !15
   br i1 %call, label %if.then, label %if.end19
@@ -110,18 +110,18 @@ if.end18:                                         ; preds = %if.else, %if.then13
   br label %if.end19
 
 if.end19:                                         ; preds = %if.end18, %for.body
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %personalized) #4
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %persistent) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %personalized) #4
+  call void @llvm.lifetime.end.p0(ptr nonnull %persistent) #4
   %inc = add nuw nsw i32 %i.032, 1
   %exitcond.not = icmp eq i32 %inc, %argc
   br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body
 }
 
 ; Function Attrs: argmemonly mustprogress nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly mustprogress nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 ; Function Attrs: inaccessiblememonly mustprogress nocallback nofree nosync nounwind willreturn
 declare void @llvm.objc.clang.arc.noop.use(...) #5
diff --git a/llvm/test/Transforms/OpenMP/custom_state_machines_remarks.ll b/llvm/test/Transforms/OpenMP/custom_state_machines_remarks.ll
index ad41639511e99..60969ecb683bf 100644
--- a/llvm/test/Transforms/OpenMP/custom_state_machines_remarks.ll
+++ b/llvm/test/Transforms/OpenMP/custom_state_machines_remarks.ll
@@ -72,10 +72,10 @@ common.ret:                                       ; preds = %entry, %user_code.e
 user_code.entry:                                  ; preds = %entry
   %1 = call i32 @__kmpc_global_thread_num(ptr nonnull @3) #3
   call void @unknown() #6, !dbg !20
-  call void @llvm.lifetime.start.p0(i64 0, ptr nonnull %captured_vars_addrs.i.i) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i.i) #3
   %2 = call i32 @__kmpc_global_thread_num(ptr noundef nonnull @13) #3
   call void @__kmpc_parallel_51(ptr noundef nonnull @13, i32 %2, i32 noundef 1, i32 noundef -1, i32 noundef -1, ptr noundef @__omp_outlined__2, ptr noundef @__omp_outlined__2_wrapper, ptr noundef nonnull %captured_vars_addrs.i.i, i64 noundef 0) #3, !dbg !23
-  call void @llvm.lifetime.end.p0(i64 0, ptr nonnull %captured_vars_addrs.i.i) #3, !dbg !26
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i.i) #3, !dbg !26
   call void @unknown() #6, !dbg !27
   call void @__kmpc_target_deinit() #3, !dbg !28
   br label %common.ret
@@ -116,18 +116,18 @@ common.ret:                                       ; preds = %entry, %user_code.e
 
 user_code.entry:                                  ; preds = %entry
   %1 = call i32 @__kmpc_global_thread_num(ptr nonnull @9) #3
-  call void @llvm.lifetime.start.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i2.i) #3
   %2 = call i32 @__kmpc_global_thread_num(ptr noundef nonnull @13) #3
   call void @__kmpc_parallel_51(ptr noundef nonnull @13, i32 %2, i32 noundef 1, i32 noundef -1, i32 noundef -1, ptr noundef @__omp_outlined__2, ptr noundef @__omp_outlined__2_wrapper, ptr noundef nonnull %captured_vars_addrs.i2.i, i64 noundef 0) #3, !dbg !35
-  call void @llvm.lifetime.end.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !39
-  call void @llvm.lifetime.start.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !39
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i2.i) #3
   %3 = call i32 @__kmpc_global_thread_num(ptr noundef nonnull @13) #3
   call void @__kmpc_parallel_51(ptr noundef nonnull @13, i32 %3, i32 noundef 1, i32 noundef -1, i32 noundef -1, ptr noundef @__omp_outlined__2, ptr noundef @__omp_outlined__2_wrapper, ptr noundef nonnull %captured_vars_addrs.i2.i, i64 noundef 0) #3, !dbg !40
-  call void @llvm.lifetime.end.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !42
-  call void @llvm.lifetime.start.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !42
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i2.i) #3
   %4 = call i32 @__kmpc_global_thread_num(ptr noundef nonnull @13) #3
   call void @__kmpc_parallel_51(ptr noundef nonnull @13, i32 %4, i32 noundef 1, i32 noundef -1, i32 noundef -1, ptr noundef @__omp_outlined__2, ptr noundef @__omp_outlined__2_wrapper, ptr noundef nonnull %captured_vars_addrs.i2.i, i64 noundef 0) #3, !dbg !43
-  call void @llvm.lifetime.end.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !45
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !45
   call void @no_openmp()
   call void @no_parallelism()
   call void @__kmpc_target_deinit() #3, !dbg !46
@@ -155,10 +155,10 @@ declare void @__kmpc_get_shared_variables(ptr) local_unnamed_addr
 declare void @__kmpc_parallel_51(ptr, i32, i32, i32, i32, ptr, ptr, ptr, i64) local_unnamed_addr
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #5
+declare void @llvm.lifetime.start.p0(ptr nocapture) #5
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #5
+declare void @llvm.lifetime.end.p0(ptr nocapture) #5
 
 declare void @no_openmp() #7
 declare void @no_parallelism() #8
diff --git a/llvm/test/Transforms/OpenMP/nested_parallelism.ll b/llvm/test/Transforms/OpenMP/nested_parallelism.ll
index 412e5ea5211e4..5d9646537481d 100644
--- a/llvm/test/Transforms/OpenMP/nested_parallelism.ll
+++ b/llvm/test/Transforms/OpenMP/nested_parallelism.ll
@@ -52,7 +52,7 @@ define weak_odr protected ptx_kernel void @__omp_offloading_10302_bd7e0_main_l13
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    ret void
 ; CHECK:       user_code.entry:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = tail call i32 @__kmpc_get_hardware_thread_id_in_block() #[[ATTR2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i32 [[TMP2]], 0
@@ -66,7 +66,7 @@ define weak_odr protected ptx_kernel void @__omp_offloading_10302_bd7e0_main_l13
 ; CHECK-NEXT:    [[TMP4:%.*]] = addrspacecast ptr [[CAPTURED_VARS_ADDRS_I]] to ptr addrspace(5)
 ; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(3) @i_shared to ptr), ptr addrspace(5) [[TMP4]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__, ptr nonnull @__omp_outlined___wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I]], i64 1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    call void @__kmpc_target_deinit()
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ;
@@ -80,7 +80,7 @@ common.ret:                                       ; preds = %entry, %_Z3fooi.int
   ret void
 
 user_code.entry:                                  ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %captured_vars_addrs.i)
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i)
   %1 = tail call i32 @__kmpc_global_thread_num(ptr nonnull @1) #6
   %2 = tail call i32 @__kmpc_get_hardware_thread_id_in_block() #6
   %3 = icmp eq i32 %2, 0
@@ -95,7 +95,7 @@ _Z3fooi.internalized.exit:                        ; preds = %user_code.entry, %r
   tail call void @__kmpc_barrier_simple_spmd(ptr nonnull @1, i32 %2)
   store ptr addrspacecast (ptr addrspace(3) @i_shared to ptr), ptr %captured_vars_addrs.i, align 8
   call void @__kmpc_parallel_51(ptr nonnull @1, i32 %1, i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__, ptr nonnull @__omp_outlined___wrapper, ptr nonnull %captured_vars_addrs.i, i64 1) #6
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %captured_vars_addrs.i)
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i)
   call void @__kmpc_target_deinit() #6
   br label %common.ret
 }
@@ -139,13 +139,13 @@ define weak_odr protected ptx_kernel void @__omp_offloading_10302_bd7e0_main_l16
 ; CHECK-NEXT:    ret void
 ; CHECK:       user_code.entry:
 ; CHECK-NEXT:    [[I_ADDR_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[I:%.*]] to i32
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
 ; CHECK-NEXT:    store i32 [[I_ADDR_SROA_0_0_EXTRACT_TRUNC]], ptr addrspace(3) @i.i_shared, align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = addrspacecast ptr [[CAPTURED_VARS_ADDRS_I]] to ptr addrspace(5)
 ; CHECK-NEXT:    store ptr addrspacecast (ptr addrspace(3) @i.i_shared to ptr), ptr addrspace(5) [[TMP2]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I]], i64 1)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    call void @__kmpc_target_deinit()
 ; CHECK-NEXT:    br label [[COMMON_RET]]
 ;
@@ -160,12 +160,12 @@ common.ret:                                       ; preds = %entry, %user_code.e
 
 user_code.entry:                                  ; preds = %entry
   %i.addr.sroa.0.0.extract.trunc = trunc i64 %i to i32
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %captured_vars_addrs.i)
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i)
   %1 = tail call i32 @__kmpc_global_thread_num(ptr nonnull @1) #6
   store i32 %i.addr.sroa.0.0.extract.trunc, ptr addrspacecast (ptr addrspace(3) @i.i_shared to ptr), align 16
   store ptr addrspacecast (ptr addrspace(3) @i.i_shared to ptr), ptr %captured_vars_addrs.i, align 8
   call void @__kmpc_parallel_51(ptr nonnull @1, i32 %1, i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull %captured_vars_addrs.i, i64 1) #6
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %captured_vars_addrs.i)
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i)
   call void @__kmpc_target_deinit() #6
   br label %common.ret
 }
@@ -201,7 +201,7 @@ define internal void @__omp_outlined__(ptr noalias nocapture readnone %.global_t
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CAPTURED_VARS_ADDRS_I:%.*]] = alloca [1 x ptr], align 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[I:%.*]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = tail call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[I_I:%.*]] = tail call align 16 dereferenceable_or_null(4) ptr @__kmpc_alloc_shared(i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    store i32 [[TMP0]], ptr [[I_I]], align 16
@@ -209,20 +209,20 @@ define internal void @__omp_outlined__(ptr noalias nocapture readnone %.global_t
 ; CHECK-NEXT:    store ptr [[I_I]], ptr addrspace(5) [[TMP2]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP1]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I]], i64 1)
 ; CHECK-NEXT:    call void @__kmpc_free_shared(ptr [[I_I]], i64 4) #[[ATTR2]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[CAPTURED_VARS_ADDRS_I]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %captured_vars_addrs.i = alloca [1 x ptr], align 8
   %0 = load i32, ptr %i, align 4
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %captured_vars_addrs.i)
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i)
   %1 = tail call i32 @__kmpc_global_thread_num(ptr nonnull @1) #6
   %i.i = tail call align 16 dereferenceable_or_null(4) ptr @__kmpc_alloc_shared(i64 4) #6
   store i32 %0, ptr %i.i, align 16
   store ptr %i.i, ptr %captured_vars_addrs.i, align 8
   call void @__kmpc_parallel_51(ptr nonnull @1, i32 %1, i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull %captured_vars_addrs.i, i64 1) #6
   call void @__kmpc_free_shared(ptr %i.i, i64 4) #6
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %captured_vars_addrs.i)
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i)
   ret void
 }
 
@@ -236,7 +236,7 @@ define internal void @__omp_outlined___wrapper(i16 zeroext %0, i32 %1) #5 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr addrspace(5) [[TMP5]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[TMP2]], align 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[CAPTURED_VARS_ADDRS_I_I]])
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @__kmpc_global_thread_num(ptr nonnull @[[GLOB1]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[I_I_I:%.*]] = call align 16 dereferenceable_or_null(4) ptr @__kmpc_alloc_shared(i64 4) #[[ATTR2]]
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr [[I_I_I]], align 16
@@ -244,7 +244,7 @@ define internal void @__omp_outlined___wrapper(i16 zeroext %0, i32 %1) #5 {
 ; CHECK-NEXT:    store ptr [[I_I_I]], ptr addrspace(5) [[TMP7]], align 8
 ; CHECK-NEXT:    call void @__kmpc_parallel_51(ptr nonnull @[[GLOB1]], i32 [[TMP6]], i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull [[CAPTURED_VARS_ADDRS_I_I]], i64 1)
 ; CHECK-NEXT:    call void @__kmpc_free_shared(ptr [[I_I_I]], i64 4) #[[ATTR2]]
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[CAPTURED_VARS_ADDRS_I_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[CAPTURED_VARS_ADDRS_I_I]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -254,14 +254,14 @@ entry:
   %2 = load ptr, ptr %global_args, align 8
   %3 = load ptr, ptr %2, align 8
   %4 = load i32, ptr %3, align 4
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %captured_vars_addrs.i.i)
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i.i)
   %5 = call i32 @__kmpc_global_thread_num(ptr nonnull @1) #6
   %i.i.i = call align 16 dereferenceable_or_null(4) ptr @__kmpc_alloc_shared(i64 4) #6
   store i32 %4, ptr %i.i.i, align 16
   store ptr %i.i.i, ptr %captured_vars_addrs.i.i, align 8
   call void @__kmpc_parallel_51(ptr nonnull @1, i32 %5, i32 1, i32 -1, i32 -1, ptr nonnull @__omp_outlined__1, ptr nonnull @__omp_outlined__1_wrapper, ptr nonnull %captured_vars_addrs.i.i, i64 1) #6
   call void @__kmpc_free_shared(ptr %i.i.i, i64 4) #6
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %captured_vars_addrs.i.i)
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i.i)
   ret void
 }
 
@@ -316,9 +316,9 @@ declare i32 @__kmpc_get_hardware_thread_id_in_block() local_unnamed_addr
 
 declare void @__kmpc_barrier_simple_spmd(ptr, i32) local_unnamed_addr #10
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #11
+declare void @llvm.lifetime.start.p0(ptr nocapture) #11
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #11
+declare void @llvm.lifetime.end.p0(ptr nocapture) #11
 
 
 !omp_offload.info = !{!0, !1}
diff --git a/llvm/test/Transforms/OpenMP/parallel_deletion.ll b/llvm/test/Transforms/OpenMP/parallel_deletion.ll
index 3e16d96eb482b..67970c41f765e 100644
--- a/llvm/test/Transforms/OpenMP/parallel_deletion.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_deletion.ll
@@ -282,46 +282,46 @@ define void @delete_parallel_2() {
 ; CHECK-LABEL: define {{[^@]+}}@delete_parallel_2() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull align 4 dereferenceable(4) [[A]]) #[[ATTR18:[0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull align 4 dereferenceable(4) [[A]]) #[[ATTR18:[0-9]+]]
 ; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
 ; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..3, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
 ; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..4, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
 ; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..5, ptr nofree noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
 ; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..6, ptr noundef nonnull align 4 captures(none) dereferenceable(4) [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[A]])
 ; CHECK-NEXT:    ret void
 ;
 ; CHECK1-LABEL: define {{[^@]+}}@delete_parallel_2() {
 ; CHECK1-NEXT:  entry:
 ; CHECK1-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull align 4 dereferenceable(4) [[A]]) #[[ATTR0]]
+; CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull align 4 dereferenceable(4) [[A]]) #[[ATTR0]]
 ; CHECK1-NEXT:    store i32 0, ptr [[A]], align 4
 ; CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef @.omp_outlined..3, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
 ; CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef @.omp_outlined..4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
 ; CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef @.omp_outlined..5, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
 ; CHECK1-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef @.omp_outlined..6, ptr nocapture noundef nonnull align 4 dereferenceable(4) [[A]])
-; CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[A]])
+; CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[A]])
 ; CHECK1-NEXT:    ret void
 ; CHECK2-LABEL: define {{[^@]+}}@delete_parallel_2() {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK2-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull align 4 dereferenceable(4) [[A]]) #[[ATTR0]]
+; CHECK2-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull align 4 dereferenceable(4) [[A]]) #[[ATTR0]]
 ; CHECK2-NEXT:    store i32 0, ptr [[A]], align 4
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef @.omp_outlined..3, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef @.omp_outlined..4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef @.omp_outlined..5, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef @.omp_outlined..6, ptr nocapture noundef nonnull align 4 dereferenceable(4) [[A]])
-; CHECK2-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[A]])
+; CHECK2-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[A]])
 ; CHECK2-NEXT:    ret void
 entry:
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   store i32 0, ptr %a, align 4
   call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr nonnull @0, i32 1, ptr @.omp_outlined..3, ptr nonnull %a)
   call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr nonnull @0, i32 1, ptr @.omp_outlined..4, ptr nonnull %a)
   call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr nonnull @0, i32 1, ptr @.omp_outlined..5, ptr nonnull %a)
   call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr nonnull @0, i32 1, ptr @.omp_outlined..6, ptr nonnull %a)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret void
 }
 
@@ -445,7 +445,7 @@ omp_if.end:                                       ; preds = %entry, %omp_if.then
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare i32 @omp_get_thread_num() inaccessiblememonly nofree nosync nounwind readonly
 
@@ -531,7 +531,7 @@ define internal void @.omp_outlined..6(ptr noalias %.global_tid., ptr noalias %.
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A1:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull align 4 [[A1]]) #[[ATTR20:[0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull align 4 [[A1]]) #[[ATTR20:[0-9]+]]
 ; CHECK-NEXT:    store i32 1, ptr [[A1]], align 4
 ; CHECK-NEXT:    store ptr [[A1]], ptr [[DOTOMP_REDUCTION_RED_LIST]], align 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
@@ -552,7 +552,7 @@ define internal void @.omp_outlined..6(ptr noalias %.global_tid., ptr noalias %.
 ; CHECK-NEXT:    [[TMP8:%.*]] = atomicrmw add ptr [[A]], i32 [[TMP7]] monotonic, align 4
 ; CHECK-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
 ; CHECK:       .omp.reduction.default:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[A1]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[A1]])
 ; CHECK-NEXT:    ret void
 ;
 ; CHECK1-LABEL: define {{[^@]+}}@.omp_outlined..6
@@ -560,7 +560,7 @@ define internal void @.omp_outlined..6(ptr noalias %.global_tid., ptr noalias %.
 ; CHECK1-NEXT:  entry:
 ; CHECK1-NEXT:    [[A1:%.*]] = alloca i32, align 4
 ; CHECK1-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
-; CHECK1-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull align 4 [[A1]]) #[[ATTR0]]
+; CHECK1-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull align 4 [[A1]]) #[[ATTR0]]
 ; CHECK1-NEXT:    store i32 1, ptr [[A1]], align 4
 ; CHECK1-NEXT:    store ptr [[A1]], ptr [[DOTOMP_REDUCTION_RED_LIST]], align 8
 ; CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
@@ -581,14 +581,14 @@ define internal void @.omp_outlined..6(ptr noalias %.global_tid., ptr noalias %.
 ; CHECK1-NEXT:    [[TMP8:%.*]] = atomicrmw add ptr [[A]], i32 [[TMP7]] monotonic, align 4
 ; CHECK1-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
 ; CHECK1:       .omp.reduction.default:
-; CHECK1-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[A1]])
+; CHECK1-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[A1]])
 ; CHECK1-NEXT:    ret void
 ; CHECK2-LABEL: define {{[^@]+}}@.omp_outlined..6
 ; CHECK2-SAME: (ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture noundef nonnull align 4 dereferenceable(4) [[A:%.*]]) {
 ; CHECK2-NEXT:  entry:
 ; CHECK2-NEXT:    [[A1:%.*]] = alloca i32, align 4
 ; CHECK2-NEXT:    [[DOTOMP_REDUCTION_RED_LIST:%.*]] = alloca [1 x ptr], align 8
-; CHECK2-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull align 4 [[A1]]) #[[ATTR0]]
+; CHECK2-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull align 4 [[A1]]) #[[ATTR0]]
 ; CHECK2-NEXT:    store i32 1, ptr [[A1]], align 4
 ; CHECK2-NEXT:    store ptr [[A1]], ptr [[DOTOMP_REDUCTION_RED_LIST]], align 8
 ; CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
@@ -609,12 +609,12 @@ define internal void @.omp_outlined..6(ptr noalias %.global_tid., ptr noalias %.
 ; CHECK2-NEXT:    [[TMP8:%.*]] = atomicrmw add ptr [[A]], i32 [[TMP7]] monotonic, align 4
 ; CHECK2-NEXT:    br label [[DOTOMP_REDUCTION_DEFAULT]]
 ; CHECK2:       .omp.reduction.default:
-; CHECK2-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[A1]])
+; CHECK2-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[A1]])
 ; CHECK2-NEXT:    ret void
 entry:
   %a1 = alloca i32, align 4
   %.omp.reduction.red_list = alloca [1 x ptr], align 8
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %a1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a1)
   store i32 1, ptr %a1, align 4
   store ptr %a1, ptr %.omp.reduction.red_list, align 8
   %tmp2 = load i32, ptr %.global_tid., align 4
@@ -638,7 +638,7 @@ entry:
   br label %.omp.reduction.default
 
 .omp.reduction.default:                           ; preds = %.omp.reduction.case2, %.omp.reduction.case1, %entry
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %a1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a1)
   ret void
 }
 
@@ -696,7 +696,7 @@ declare i32 @__kmpc_reduce_nowait(ptr, i32, i32, i64, ptr, ptr, ptr)
 
 declare void @__kmpc_end_reduce_nowait(ptr, i32, ptr)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare !callback !2 void @__kmpc_fork_call(ptr, i32, ptr, ...)
 
diff --git a/llvm/test/Transforms/OpenMP/parallel_region_merging.ll b/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
index d587b9a9fdc54..83452e72b56b9 100644
--- a/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
@@ -433,11 +433,11 @@ entry:
   %b = alloca i32, align 4
   store i32 %a, ptr %a.addr, align 4
   call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr nonnull @1, i32 1, ptr @.omp_outlined..14, ptr nonnull %a.addr)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %b)
+  call void @llvm.lifetime.start.p0(ptr nonnull %b)
   %0 = ptrtoint ptr %b to i64
   %1 = trunc i64 %0 to i32
   store i32 %1, ptr %b, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr nonnull @1, i32 1, ptr @.omp_outlined..15, ptr nonnull %a.addr)
   ret void
 }
@@ -449,9 +449,9 @@ entry:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define internal void @.omp_outlined..15(ptr noalias nocapture readnone %.global_tid., ptr noalias nocapture readnone %.bound_tid., ptr nocapture nonnull readonly align 4 dereferenceable(4) %a)  {
 entry:
@@ -466,12 +466,12 @@ entry:
   %b = alloca i32, align 4
   store i32 %a, ptr %a.addr, align 4
   call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr nonnull @1, i32 1, ptr @.omp_outlined..16, ptr nonnull %a.addr)
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %b)
+  call void @llvm.lifetime.start.p0(ptr nonnull %b)
   %0 = load i32, ptr %a.addr, align 4
   %add = add nsw i32 %0, 1
   store i32 %add, ptr %b, align 4
   call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr nonnull @1, i32 2, ptr @.omp_outlined..17, ptr nonnull %a.addr, ptr nonnull %b)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %b)
+  call void @llvm.lifetime.end.p0(ptr nonnull %b)
   ret void
 }
 
@@ -1184,11 +1184,11 @@ entry:
 ; CHECK:       omp_region.body:
 ; CHECK-NEXT:    br label [[SEQ_PAR_MERGED:%.*]]
 ; CHECK:       seq.par.merged:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[B]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
 ; CHECK-NEXT:    store i32 [[TMP5]], ptr [[B]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    br label [[OMP_PAR_MERGED_SPLIT:%.*]]
 ; CHECK:       omp.par.merged.split:
 ; CHECK-NEXT:    br label [[OMP_REGION_BODY_SPLIT:%.*]]
@@ -1216,7 +1216,7 @@ entry:
 ; CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 ; CHECK-NEXT:    br label [[OMP_PARALLEL:%.*]]
 ; CHECK:       omp_parallel:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B]])
 ; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 2, ptr @merge_seq_par_use..omp_par, ptr [[A_ADDR]], ptr [[B]])
 ; CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
 ; CHECK:       omp.par.outlined.exit:
@@ -1224,7 +1224,7 @@ entry:
 ; CHECK:       omp.par.exit.split:
 ; CHECK-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK:       entry.split.split:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    ret void
 ; CHECK-LABEL: define {{[^@]+}}@merge_seq_par_use..omp_par
 ; CHECK-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[A_ADDR:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
@@ -2155,11 +2155,11 @@ entry:
 ; CHECK:       omp_region.body:
 ; CHECK-NEXT:    br label [[SEQ_PAR_MERGED:%.*]]
 ; CHECK:       seq.par.merged:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[B]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
 ; CHECK-NEXT:    store i32 [[TMP5]], ptr [[B]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    br label [[OMP_PAR_MERGED_SPLIT:%.*]]
 ; CHECK:       omp.par.merged.split:
 ; CHECK-NEXT:    br label [[OMP_REGION_BODY_SPLIT:%.*]]
@@ -2187,7 +2187,7 @@ entry:
 ; CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 ; CHECK-NEXT:    br label [[OMP_PARALLEL:%.*]]
 ; CHECK:       omp_parallel:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B]])
 ; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 2, ptr @merge_seq_par_use..omp_par, ptr [[A_ADDR]], ptr [[B]])
 ; CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
 ; CHECK:       omp.par.outlined.exit:
@@ -2195,7 +2195,7 @@ entry:
 ; CHECK:       omp.par.exit.split:
 ; CHECK-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK:       entry.split.split:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    ret void
 ; CHECK-LABEL: define {{[^@]+}}@merge_seq_par_use..omp_par
 ; CHECK-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[A_ADDR:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
@@ -3126,11 +3126,11 @@ entry:
 ; CHECK:       omp_region.body:
 ; CHECK-NEXT:    br label [[SEQ_PAR_MERGED:%.*]]
 ; CHECK:       seq.par.merged:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[B]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
 ; CHECK-NEXT:    store i32 [[TMP5]], ptr [[B]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    br label [[OMP_PAR_MERGED_SPLIT:%.*]]
 ; CHECK:       omp.par.merged.split:
 ; CHECK-NEXT:    br label [[OMP_REGION_BODY_SPLIT:%.*]]
@@ -3158,7 +3158,7 @@ entry:
 ; CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 ; CHECK-NEXT:    br label [[OMP_PARALLEL:%.*]]
 ; CHECK:       omp_parallel:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B]])
 ; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 2, ptr @merge_seq_par_use..omp_par, ptr [[A_ADDR]], ptr [[B]])
 ; CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
 ; CHECK:       omp.par.outlined.exit:
@@ -3166,7 +3166,7 @@ entry:
 ; CHECK:       omp.par.exit.split:
 ; CHECK-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK:       entry.split.split:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    ret void
 ; CHECK-LABEL: define {{[^@]+}}@merge_seq_par_use..omp_par
 ; CHECK-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[A_ADDR:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
@@ -4097,11 +4097,11 @@ entry:
 ; CHECK:       omp_region.body:
 ; CHECK-NEXT:    br label [[SEQ_PAR_MERGED:%.*]]
 ; CHECK:       seq.par.merged:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[B]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
 ; CHECK-NEXT:    store i32 [[TMP5]], ptr [[B]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    br label [[OMP_PAR_MERGED_SPLIT:%.*]]
 ; CHECK:       omp.par.merged.split:
 ; CHECK-NEXT:    br label [[OMP_REGION_BODY_SPLIT:%.*]]
@@ -4129,7 +4129,7 @@ entry:
 ; CHECK-NEXT:    [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1]])
 ; CHECK-NEXT:    br label [[OMP_PARALLEL:%.*]]
 ; CHECK:       omp_parallel:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B]])
 ; CHECK-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB1]], i32 2, ptr @merge_seq_par_use..omp_par, ptr [[A_ADDR]], ptr [[B]])
 ; CHECK-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
 ; CHECK:       omp.par.outlined.exit:
@@ -4137,7 +4137,7 @@ entry:
 ; CHECK:       omp.par.exit.split:
 ; CHECK-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK:       entry.split.split:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK-NEXT:    ret void
 ; CHECK-LABEL: define {{[^@]+}}@merge_seq_par_use..omp_par
 ; CHECK-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[A_ADDR:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
@@ -5148,11 +5148,11 @@ entry:
 ; CHECK2:       omp_region.body:
 ; CHECK2-NEXT:    br label [[SEQ_PAR_MERGED:%.*]]
 ; CHECK2:       seq.par.merged:
-; CHECK2-NEXT:    call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK2-NEXT:    call void @llvm.lifetime.start.p0(ptr noundef nonnull [[B]])
 ; CHECK2-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[B]] to i64
 ; CHECK2-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP4]] to i32
 ; CHECK2-NEXT:    store i32 [[TMP5]], ptr [[B]], align 4
-; CHECK2-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK2-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_MERGED_SPLIT:%.*]]
 ; CHECK2:       omp.par.merged.split:
 ; CHECK2-NEXT:    br label [[OMP_REGION_BODY_SPLIT:%.*]]
@@ -5197,13 +5197,13 @@ entry:
 ; CHECK2-NEXT:    store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
 ; CHECK2-NEXT:    [[GEP_B:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
 ; CHECK2-NEXT:    store ptr [[B]], ptr [[GEP_B]], align 8
-; CHECK2-NEXT:    call void @llvm.lifetime.start.p0(i64 -1, ptr [[B]])
+; CHECK2-NEXT:    call void @llvm.lifetime.start.p0(ptr [[B]])
 ; CHECK2-NEXT:    call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq_par_use..omp_par, ptr [[STRUCTARG]])
 ; CHECK2-NEXT:    br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
 ; CHECK2:       omp.par.exit:
 ; CHECK2-NEXT:    br label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK2:       entry.split.split:
-; CHECK2-NEXT:    call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK2-NEXT:    call void @llvm.lifetime.end.p0(ptr noundef nonnull [[B]])
 ; CHECK2-NEXT:    ret void
 ;
 ;
diff --git a/llvm/test/Transforms/OpenMP/spmdization.ll b/llvm/test/Transforms/OpenMP/spmdization.ll
index e91f1608f066f..0272c41d9d1fc 100644
--- a/llvm/test/Transforms/OpenMP/spmdization.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization.ll
@@ -2922,7 +2922,7 @@ declare void @unknown() #7
 declare void @unknowni32p(ptr) #7
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none)) #8
+declare void @llvm.lifetime.start.p0(ptr captures(none)) #8
 
 define weak i32 @__kmpc_target_init(ptr %0, ptr %1) {
 ; AMDGPU-LABEL: define {{[^@]+}}@__kmpc_target_init
@@ -2958,7 +2958,7 @@ declare void @__kmpc_get_shared_variables(ptr)
 declare void @__kmpc_parallel_51(ptr, i32, i32, i32, i32, ptr, ptr, ptr, i64) #9
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none)) #8
+declare void @llvm.lifetime.end.p0(ptr captures(none)) #8
 
 ; Function Attrs: convergent
 declare void @spmd_amenable() #6
diff --git a/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll b/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll
index 70c0d046dad4e..5a7d097142965 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_constant_prop.ll
@@ -61,7 +61,7 @@ common.ret:                                       ; preds = %user_code.entry, %e
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p5(i64 immarg, ptr addrspace(5) captures(none)) #1
+declare void @llvm.lifetime.start.p5(ptr addrspace(5) captures(none)) #1
 
 ; Function Attrs: alwaysinline mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 define internal void @__omp_outlined__(ptr noalias captures(none) %.global_tid., ptr noalias captures(none) %.bound_tid., ptr nonnull align 4 captures(none) %ng, ptr nonnull align 8 captures(none) %aa) #2 {
@@ -72,7 +72,7 @@ entry:
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p5(i64 immarg, ptr addrspace(5) captures(none)) #1
+declare void @llvm.lifetime.end.p5(ptr addrspace(5) captures(none)) #1
 
 ; Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
 define internal void @__omp_outlined___wrapper(i16 zeroext %0, i32 noundef %1) #3 {
diff --git a/llvm/test/Transforms/OpenMP/spmdization_indirect.ll b/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
index 3c3e1d78a00c1..d1e006a704441 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
@@ -1017,7 +1017,7 @@ declare void @unknown() #5
 declare void @unknowni32p(ptr) #5
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none)) #6
+declare void @llvm.lifetime.start.p0(ptr captures(none)) #6
 
 define weak i32 @__kmpc_target_init(ptr %0, ptr %1) {
 ; AMDGPU-LABEL: define {{[^@]+}}@__kmpc_target_init
@@ -1037,7 +1037,7 @@ declare void @__kmpc_get_shared_variables(ptr)
 declare void @__kmpc_parallel_51(ptr, i32, i32, i32, i32, ptr, ptr, ptr, i64) #7
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none)) #6
+declare void @llvm.lifetime.end.p0(ptr captures(none)) #6
 
 ; Function Attrs: convergent
 declare void @spmd_amenable() #4
diff --git a/llvm/test/Transforms/OpenMP/spmdization_remarks.ll b/llvm/test/Transforms/OpenMP/spmdization_remarks.ll
index ef36937bc5734..f30e827694a34 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_remarks.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_remarks.ll
@@ -75,10 +75,10 @@ common.ret:                                       ; preds = %entry, %user_code.e
 user_code.entry:                                  ; preds = %entry
   %1 = call i32 @__kmpc_global_thread_num(ptr nonnull @3) #3
   call void @unknown() #6, !dbg !20
-  call void @llvm.lifetime.start.p0(i64 0, ptr nonnull %captured_vars_addrs.i.i) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i.i) #3
   %2 = call i32 @__kmpc_global_thread_num(ptr noundef nonnull @13) #3
   call void @__kmpc_parallel_51(ptr noundef nonnull @13, i32 %2, i32 noundef 1, i32 noundef -1, i32 noundef -1, ptr noundef @__omp_outlined__2, ptr noundef @__omp_outlined__2_wrapper, ptr noundef nonnull %captured_vars_addrs.i.i, i64 noundef 0) #3, !dbg !23
-  call void @llvm.lifetime.end.p0(i64 0, ptr nonnull %captured_vars_addrs.i.i) #3, !dbg !26
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i.i) #3, !dbg !26
   call void @unknown() #6, !dbg !27
   call void @__kmpc_target_deinit() #3, !dbg !28
   br label %common.ret
@@ -119,18 +119,18 @@ common.ret:                                       ; preds = %entry, %user_code.e
 
 user_code.entry:                                  ; preds = %entry
   %1 = call i32 @__kmpc_global_thread_num(ptr nonnull @9) #3
-  call void @llvm.lifetime.start.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i2.i) #3
   %2 = call i32 @__kmpc_global_thread_num(ptr noundef nonnull @13) #3
   call void @__kmpc_parallel_51(ptr noundef nonnull @13, i32 %2, i32 noundef 1, i32 noundef -1, i32 noundef -1, ptr noundef @__omp_outlined__2, ptr noundef @__omp_outlined__2_wrapper, ptr noundef nonnull %captured_vars_addrs.i2.i, i64 noundef 0) #3, !dbg !35
-  call void @llvm.lifetime.end.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !39
-  call void @llvm.lifetime.start.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !39
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i2.i) #3
   %3 = call i32 @__kmpc_global_thread_num(ptr noundef nonnull @13) #3
   call void @__kmpc_parallel_51(ptr noundef nonnull @13, i32 %3, i32 noundef 1, i32 noundef -1, i32 noundef -1, ptr noundef @__omp_outlined__2, ptr noundef @__omp_outlined__2_wrapper, ptr noundef nonnull %captured_vars_addrs.i2.i, i64 noundef 0) #3, !dbg !40
-  call void @llvm.lifetime.end.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !42
-  call void @llvm.lifetime.start.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !42
+  call void @llvm.lifetime.start.p0(ptr nonnull %captured_vars_addrs.i2.i) #3
   %4 = call i32 @__kmpc_global_thread_num(ptr noundef nonnull @13) #3
   call void @__kmpc_parallel_51(ptr noundef nonnull @13, i32 %4, i32 noundef 1, i32 noundef -1, i32 noundef -1, ptr noundef @__omp_outlined__2, ptr noundef @__omp_outlined__2_wrapper, ptr noundef nonnull %captured_vars_addrs.i2.i, i64 noundef 0) #3, !dbg !43
-  call void @llvm.lifetime.end.p0(i64 0, ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !45
+  call void @llvm.lifetime.end.p0(ptr nonnull %captured_vars_addrs.i2.i) #3, !dbg !45
   call void @spmd_amenable()
   call void @__kmpc_target_deinit() #3, !dbg !46
   br label %common.ret
@@ -157,10 +157,10 @@ declare void @__kmpc_get_shared_variables(ptr) local_unnamed_addr
 declare void @__kmpc_parallel_51(ptr, i32, i32, i32, i32, ptr, ptr, ptr, i64) local_unnamed_addr
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #5
+declare void @llvm.lifetime.start.p0(ptr nocapture) #5
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #5
+declare void @llvm.lifetime.end.p0(ptr nocapture) #5
 
 declare void @spmd_amenable() #7
 
diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll
index 46f9a2bde7a23..f0a1574c5f209 100644
--- a/llvm/test/Transforms/PGOProfile/chr.ll
+++ b/llvm/test/Transforms/PGOProfile/chr.ll
@@ -1295,6 +1295,7 @@ define i32 @test_chr_14(ptr %i, ptr %j, i32 %sum0, i1 %pred, i32 %z) !prof !14 {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[Z_FR:%.*]] = freeze i32 [[Z:%.*]]
 ; CHECK-NEXT:    [[I0:%.*]] = load i32, ptr [[I:%.*]], align 4
+; CHECK-NEXT:    [[I0_FR:%.*]] = freeze i32 [[I0]]
 ; CHECK-NEXT:    [[V1_NOT:%.*]] = icmp eq i32 [[Z_FR]], 1
 ; CHECK-NEXT:    br i1 [[V1_NOT]], label [[BB1:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof [[PROF15]]
 ; CHECK:       entry.split.nonchr:
@@ -1307,27 +1308,26 @@ define i32 @test_chr_14(ptr %i, ptr %j, i32 %sum0, i1 %pred, i32 %z) !prof !14 {
 ; CHECK-NEXT:    br label [[BB1]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[J0:%.*]] = load i32, ptr [[J:%.*]], align 4
-; CHECK-NEXT:    [[V6:%.*]] = and i32 [[I0]], 2
-; CHECK-NEXT:    [[V4:%.*]] = icmp ne i32 [[V6]], [[J0]]
+; CHECK-NEXT:    [[J0_FR:%.*]] = freeze i32 [[J0]]
+; CHECK-NEXT:    [[V6:%.*]] = and i32 [[I0_FR]], 2
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ne i32 [[V6]], [[J0_FR]]
 ; CHECK-NEXT:    [[V8:%.*]] = add i32 [[SUM0:%.*]], 43
-; CHECK-NEXT:    [[V5:%.*]] = icmp ne i32 [[I0]], [[J0]]
-; CHECK-NEXT:    [[TMP0:%.*]] = freeze i1 [[V4]]
-; CHECK-NEXT:    [[TMP1:%.*]] = freeze i1 [[V5]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ne i32 [[I0_FR]], [[J0_FR]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP0]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[BB1_SPLIT:%.*]], label [[BB1_SPLIT_NONCHR:%.*]], !prof [[PROF15]]
 ; CHECK:       bb1.split:
 ; CHECK-NEXT:    call void @foo()
-; CHECK-NEXT:    [[V9:%.*]] = and i32 [[I0]], 4
+; CHECK-NEXT:    [[V9:%.*]] = and i32 [[I0_FR]], 4
 ; CHECK-NEXT:    [[V10:%.*]] = icmp eq i32 [[V9]], 0
 ; CHECK-NEXT:    br i1 [[V10]], label [[BB3:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb1.split.nonchr:
-; CHECK-NEXT:    [[V5_NONCHR:%.*]] = icmp eq i32 [[I0]], [[J0]]
+; CHECK-NEXT:    [[V5_NONCHR:%.*]] = icmp eq i32 [[I0_FR]], [[J0_FR]]
 ; CHECK-NEXT:    [[SUM3_NONCHR:%.*]] = select i1 [[V5_NONCHR]], i32 [[SUM0]], i32 [[V8]], !prof [[PROF16]]
 ; CHECK-NEXT:    call void @foo()
-; CHECK-NEXT:    [[V9_NONCHR:%.*]] = and i32 [[I0]], 4
+; CHECK-NEXT:    [[V9_NONCHR:%.*]] = and i32 [[I0_FR]], 4
 ; CHECK-NEXT:    [[V10_NONCHR:%.*]] = icmp eq i32 [[V9_NONCHR]], 0
 ; CHECK-NEXT:    br i1 [[V10_NONCHR]], label [[BB3]], label [[BB2_NONCHR:%.*]]
 ; CHECK:       bb2.nonchr:
@@ -1335,7 +1335,7 @@ define i32 @test_chr_14(ptr %i, ptr %j, i32 %sum0, i1 %pred, i32 %z) !prof !14 {
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
 ; CHECK-NEXT:    [[TMP3:%.*]] = phi i32 [ [[V8]], [[BB2]] ], [ [[V8]], [[BB1_SPLIT]] ], [ [[SUM3_NONCHR]], [[BB2_NONCHR]] ], [ [[SUM3_NONCHR]], [[BB1_SPLIT_NONCHR]] ]
-; CHECK-NEXT:    [[V11:%.*]] = add i32 [[I0]], [[TMP3]]
+; CHECK-NEXT:    [[V11:%.*]] = add i32 [[I0_FR]], [[TMP3]]
 ; CHECK-NEXT:    ret i32 [[V11]]
 ;
 entry:
@@ -1932,12 +1932,12 @@ define i32 @test_chr_21(i64 %i, i64 %k, i64 %j) "instcombine-no-verify-fixpoint"
 ; CHECK-LABEL: @test_chr_21(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I_FR:%.*]] = freeze i64 [[I:%.*]]
-; CHECK-NEXT:    [[CMP0:%.*]] = icmp ne i64 [[J:%.*]], [[K:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = freeze i1 [[CMP0]]
+; CHECK-NEXT:    [[J:%.*]] = freeze i64 [[J1:%.*]]
+; CHECK-NEXT:    [[K:%.*]] = freeze i64 [[K1:%.*]]
+; CHECK-NEXT:    [[CMP0:%.*]] = icmp ne i64 [[J]], [[K]]
 ; CHECK-NEXT:    [[CMP3:%.*]] = icmp ne i64 [[J]], [[I_FR]]
 ; CHECK-NEXT:    [[CMP_I:%.*]] = icmp ne i64 [[I_FR]], 86
-; CHECK-NEXT:    [[TMP1:%.*]] = freeze i1 [[CMP3]]
-; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[CMP0]], [[CMP3]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[CMP_I]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[BB1:%.*]], label [[ENTRY_SPLIT_NONCHR:%.*]], !prof [[PROF15]]
 ; CHECK:       bb1:
@@ -1963,7 +1963,7 @@ define i32 @test_chr_21(i64 %i, i64 %k, i64 %j) "instcombine-no-verify-fixpoint"
 ; CHECK-NEXT:    call void @foo()
 ; CHECK-NEXT:    br label [[BB10:%.*]]
 ; CHECK:       entry.split.nonchr:
-; CHECK-NEXT:    br i1 [[TMP0]], label [[BB1_NONCHR:%.*]], label [[BB10]], !prof [[PROF18]]
+; CHECK-NEXT:    br i1 [[CMP0]], label [[BB1_NONCHR:%.*]], label [[BB10]], !prof [[PROF18]]
 ; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[CMP2_NONCHR:%.*]] = icmp eq i64 [[I_FR]], 2
 ; CHECK-NEXT:    br i1 [[CMP2_NONCHR]], label [[BB3_NONCHR:%.*]], label [[BB2_NONCHR:%.*]], !prof [[PROF16]]
diff --git a/llvm/test/Transforms/PGOProfile/consecutive-zeros.ll b/llvm/test/Transforms/PGOProfile/consecutive-zeros.ll
index a7337d04531a1..014f95fd8bec6 100644
--- a/llvm/test/Transforms/PGOProfile/consecutive-zeros.ll
+++ b/llvm/test/Transforms/PGOProfile/consecutive-zeros.ll
@@ -48,11 +48,11 @@ for.end6:
   ret void
 }
 
-declare void @llvm.lifetime.start(i64, ptr nocapture)
+declare void @llvm.lifetime.start(ptr nocapture)
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1)
 
 declare i32 @memcmp(ptr, ptr, i64)
 declare i32 @bcmp(ptr, ptr, i64)
 
-declare void @llvm.lifetime.end(i64, ptr nocapture)
+declare void @llvm.lifetime.end(ptr nocapture)
diff --git a/llvm/test/Transforms/PGOProfile/entry_alloca.ll b/llvm/test/Transforms/PGOProfile/entry_alloca.ll
index 580f055921970..c791e1d9bb6ce 100644
--- a/llvm/test/Transforms/PGOProfile/entry_alloca.ll
+++ b/llvm/test/Transforms/PGOProfile/entry_alloca.ll
@@ -19,8 +19,8 @@
 define dso_local double @foo() {
   %1 = alloca %struct.A, align 4
   %2 = alloca %struct.B, align 8
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %1)
-  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %2)
+  call void @llvm.lifetime.start.p0(ptr nonnull %1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %2)
   call void @bar(ptr noundef nonnull %1, ptr noundef nonnull %2)
   %3 = load i32, ptr %1, align 4
   %4 = icmp sgt i32 %3, 0
@@ -48,8 +48,8 @@ define dso_local double @foo() {
 
 21:
   %22 = phi double [ 0.000000e+00, %0 ], [ %18, %9 ]
-  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %2)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %2)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
   ret double %22
 }
 
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll b/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll
index 3ef185a28cc5f..9122454a1f50c 100644
--- a/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll
+++ b/llvm/test/Transforms/PGOProfile/memop_size_annotation.ll
@@ -56,11 +56,11 @@ for.end6:
   ret void
 }
 
-declare void @llvm.lifetime.start(i64, ptr nocapture)
+declare void @llvm.lifetime.start(ptr nocapture)
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1)
 
 declare i32 @memcmp(ptr, ptr, i64)
 declare i32 @bcmp(ptr, ptr, i64)
 
-declare void @llvm.lifetime.end(i64, ptr nocapture)
+declare void @llvm.lifetime.end(ptr nocapture)
diff --git a/llvm/test/Transforms/PGOProfile/memop_size_opt.ll b/llvm/test/Transforms/PGOProfile/memop_size_opt.ll
index c4f749be747db..f63989abe16a7 100644
--- a/llvm/test/Transforms/PGOProfile/memop_size_opt.ll
+++ b/llvm/test/Transforms/PGOProfile/memop_size_opt.ll
@@ -181,14 +181,14 @@ for.end6:
 !30 = !{!"VP", i32 1, i64 556, i64 0, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22}
 !31 = !{!"VP", i32 1, i64 556, i64 0, i64 99, i64 2, i64 88, i64 3, i64 77, i64 9, i64 72, i64 4, i64 66, i64 5, i64 55, i64 6, i64 44, i64 7, i64 33, i64 8, i64 22}
 
-declare void @llvm.lifetime.start(i64, ptr nocapture)
+declare void @llvm.lifetime.start(ptr nocapture)
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1)
 
 declare i32 @memcmp(ptr, ptr, i64)
 declare i32 @bcmp(ptr, ptr, i64)
 
-declare void @llvm.lifetime.end(i64, ptr nocapture)
+declare void @llvm.lifetime.end(ptr nocapture)
 
 ; YAML:      --- !Passed
 ; YAML-NEXT: Pass:            pgo-memop-opt
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch-correct.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch-correct.ll
index f187d41028fc1..0a1efdade760f 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-branch-correct.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch-correct.ll
@@ -23,10 +23,10 @@ define i32 @bar() #0 {
 entry:
   %rando = alloca i32, align 4
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %rando) #4
+  call void @llvm.lifetime.start.p0(ptr %rando) #4
   %call = call i32 (...) @buzz()
   store i32 %call, ptr %rando, align 4, !tbaa !3
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x) #4
+  call void @llvm.lifetime.start.p0(ptr %x) #4
   store i32 0, ptr %x, align 4, !tbaa !3
   %0 = load i32, ptr %rando, align 4, !tbaa !3
   %rem = srem i32 %0, 200000
@@ -52,13 +52,13 @@ if.else:                                          ; preds = %entry
 
 if.end:                                           ; preds = %if.else, %if.then
   %2 = load i32, ptr %x, align 4, !tbaa !3
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x) #4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %rando) #4
+  call void @llvm.lifetime.end.p0(ptr %x) #4
+  call void @llvm.lifetime.end.p0(ptr %rando) #4
   ret i32 %2
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare i32 @buzz(...) #2
 
@@ -70,7 +70,7 @@ declare i32 @baz(i32) #2
 declare i32 @foo(i32) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch-overflow.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch-overflow.ll
index 146ad44b38918..68b233edda524 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-branch-overflow.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch-overflow.ll
@@ -20,10 +20,10 @@ define i32 @bar() #0 !dbg !6 {
 entry:
   %rando = alloca i32, align 4
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %rando) #4, !dbg !9
+  call void @llvm.lifetime.start.p0(ptr %rando) #4, !dbg !9
   %call = call i32 (...) @buzz(), !dbg !9
   store i32 %call, ptr %rando, align 4, !dbg !9, !tbaa !10
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x) #4, !dbg !14
+  call void @llvm.lifetime.start.p0(ptr %x) #4, !dbg !14
   store i32 0, ptr %x, align 4, !dbg !14, !tbaa !10
   %0 = load i32, ptr %rando, align 4, !dbg !15, !tbaa !10
   %rem = srem i32 %0, 200000, !dbg !15
@@ -49,13 +49,13 @@ if.else:                                          ; preds = %entry
 
 if.end:                                           ; preds = %if.else, %if.then
   %2 = load i32, ptr %x, align 4, !dbg !19, !tbaa !10
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x) #4, !dbg !20
-  call void @llvm.lifetime.end.p0(i64 4, ptr %rando) #4, !dbg !20
+  call void @llvm.lifetime.end.p0(ptr %x) #4, !dbg !20
+  call void @llvm.lifetime.end.p0(ptr %rando) #4, !dbg !20
   ret i32 %2, !dbg !19
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare i32 @buzz(...) #2
 
@@ -67,7 +67,7 @@ declare i32 @baz(i32) #2
 declare i32 @foo(i32) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll
index 18677b720f36b..2f188f5efd6fc 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch-stripped.ll
@@ -33,10 +33,10 @@ define i32 @bar() #0 {
 entry:
   %rando = alloca i32, align 4
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %rando) #4
+  call void @llvm.lifetime.start.p0(ptr %rando) #4
   %call = call i32 (...) @buzz()
   store i32 %call, ptr %rando, align 4, !tbaa !3
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x) #4
+  call void @llvm.lifetime.start.p0(ptr %x) #4
   store i32 0, ptr %x, align 4, !tbaa !3
   %0 = load i32, ptr %rando, align 4, !tbaa !3
   %rem = srem i32 %0, 200000
@@ -62,13 +62,13 @@ if.else:                                          ; preds = %entry
 
 if.end:                                           ; preds = %if.else, %if.then
   %2 = load i32, ptr %x, align 4, !tbaa !3
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x) #4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %rando) #4
+  call void @llvm.lifetime.end.p0(ptr %x) #4
+  call void @llvm.lifetime.end.p0(ptr %rando) #4
   ret i32 %2
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare i32 @buzz(...) #2
 
@@ -80,7 +80,7 @@ declare i32 @baz(i32) #2
 declare i32 @foo(i32) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch-unpredictable.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch-unpredictable.ll
index 1e7e8c1ab418a..4add781d52071 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-branch-unpredictable.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch-unpredictable.ll
@@ -21,10 +21,10 @@ define i32 @bar() #0 {
 entry:
   %rando = alloca i32, align 4
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %rando) #3
+  call void @llvm.lifetime.start.p0(ptr %rando) #3
   %call = call i32 (...) @buzz()
   store i32 %call, ptr %rando, align 4, !tbaa !2
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x) #3
+  call void @llvm.lifetime.start.p0(ptr %x) #3
   store i32 0, ptr %x, align 4, !tbaa !2
   %0 = load i32, ptr %rando, align 4, !tbaa !2
   %rem = srem i32 %0, 200000
@@ -49,13 +49,13 @@ if.else:                                          ; preds = %entry
 
 if.end:                                           ; preds = %if.else, %if.then
   %2 = load i32, ptr %x, align 4, !tbaa !2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x) #3
-  call void @llvm.lifetime.end.p0(i64 4, ptr %rando) #3
+  call void @llvm.lifetime.end.p0(ptr %x) #3
+  call void @llvm.lifetime.end.p0(ptr %rando) #3
   ret i32 %2
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare i32 @buzz(...) #2
 
@@ -64,7 +64,7 @@ declare i32 @baz(i32) #2
 declare i32 @foo(i32) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-branch.ll b/llvm/test/Transforms/PGOProfile/misexpect-branch.ll
index 10c37182d4c5c..5a7731bd2442d 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-branch.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-branch.ll
@@ -37,10 +37,10 @@ define i32 @bar() #0 !dbg !6 {
 entry:
   %rando = alloca i32, align 4
   %x = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %rando) #4, !dbg !9
+  call void @llvm.lifetime.start.p0(ptr %rando) #4, !dbg !9
   %call = call i32 (...) @buzz(), !dbg !9
   store i32 %call, ptr %rando, align 4, !dbg !9, !tbaa !10
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x) #4, !dbg !14
+  call void @llvm.lifetime.start.p0(ptr %x) #4, !dbg !14
   store i32 0, ptr %x, align 4, !dbg !14, !tbaa !10
   %0 = load i32, ptr %rando, align 4, !dbg !15, !tbaa !10
   %rem = srem i32 %0, 200000, !dbg !15
@@ -66,13 +66,13 @@ if.else:                                          ; preds = %entry
 
 if.end:                                           ; preds = %if.else, %if.then
   %2 = load i32, ptr %x, align 4, !dbg !19, !tbaa !10
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x) #4, !dbg !20
-  call void @llvm.lifetime.end.p0(i64 4, ptr %rando) #4, !dbg !20
+  call void @llvm.lifetime.end.p0(ptr %x) #4, !dbg !20
+  call void @llvm.lifetime.end.p0(ptr %rando) #4, !dbg !20
   ret i32 %2, !dbg !19
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare i32 @buzz(...) #2
 
@@ -84,7 +84,7 @@ declare i32 @baz(i32) #2
 declare i32 @foo(i32) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-features"="+cx8,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nounwind willreturn }
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll b/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll
index 6e21c083b91aa..859ba7223a17b 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-switch-default.ll
@@ -40,7 +40,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define dso_local void @init_arry() #0 {
 entry:
   %i = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #6
+  call void @llvm.lifetime.start.p0(ptr %i) #6
   store i32 0, ptr %i, align 4, !tbaa !4
   br label %for.cond
 
@@ -65,12 +65,12 @@ for.inc:                                          ; preds = %for.body
   br label %for.cond
 
 for.end:                                          ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #6
+  call void @llvm.lifetime.end.p0(ptr %i) #6
   ret void
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind readnone speculatable willreturn
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
@@ -79,7 +79,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
 declare dso_local i32 @rand() #3
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define dso_local i32 @main() #0 {
@@ -90,9 +90,9 @@ entry:
   %condition = alloca i32, align 4
   store i32 0, ptr %retval, align 4
   call void @init_arry()
-  call void @llvm.lifetime.start.p0(i64 4, ptr %val) #6
+  call void @llvm.lifetime.start.p0(ptr %val) #6
   store i32 0, ptr %val, align 4, !tbaa !4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %j) #6
+  call void @llvm.lifetime.start.p0(ptr %j) #6
   store i32 0, ptr %j, align 4, !tbaa !4
   br label %for.cond
 
@@ -102,7 +102,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.end
 
 for.body:                                         ; preds = %for.cond
-  call void @llvm.lifetime.start.p0(i64 4, ptr %condition) #6
+  call void @llvm.lifetime.start.p0(ptr %condition) #6
   %call = call i32 @rand() #6
   %rem = srem i32 %call, 5
   store i32 %rem, ptr %condition, align 4, !tbaa !4
@@ -138,7 +138,7 @@ sw.default:                                       ; preds = %for.body
   unreachable
 
 sw.epilog:                                        ; preds = %sw.bb3, %sw.bb2, %sw.bb
-  call void @llvm.lifetime.end.p0(i64 4, ptr %condition) #6
+  call void @llvm.lifetime.end.p0(ptr %condition) #6
   br label %for.inc
 
 for.inc:                                          ; preds = %sw.epilog
@@ -148,8 +148,8 @@ for.inc:                                          ; preds = %sw.epilog
   br label %for.cond
 
 for.end:                                          ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %j) #6
-  call void @llvm.lifetime.end.p0(i64 4, ptr %val) #6
+  call void @llvm.lifetime.end.p0(ptr %j) #6
+  call void @llvm.lifetime.end.p0(ptr %val) #6
   ret i32 0
 }
 
diff --git a/llvm/test/Transforms/PGOProfile/misexpect-switch.ll b/llvm/test/Transforms/PGOProfile/misexpect-switch.ll
index ebecee178b820..242d5b8cc112e 100644
--- a/llvm/test/Transforms/PGOProfile/misexpect-switch.ll
+++ b/llvm/test/Transforms/PGOProfile/misexpect-switch.ll
@@ -43,7 +43,7 @@ target triple = "x86_64-unknown-linux-gnu"
 define dso_local void @init_arry() #0 !dbg !21 {
 entry:
   %i = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #6, !dbg !26
+  call void @llvm.lifetime.start.p0(ptr %i) #6, !dbg !26
   call void @llvm.dbg.declare(metadata ptr %i, metadata !25, metadata !DIExpression()), !dbg !27
   store i32 0, ptr %i, align 4, !dbg !28, !tbaa !30
   br label %for.cond, !dbg !34
@@ -69,12 +69,12 @@ for.inc:                                          ; preds = %for.body
   br label %for.cond, !dbg !47, !llvm.loop !48
 
 for.end:                                          ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #6, !dbg !50
+  call void @llvm.lifetime.end.p0(ptr %i) #6, !dbg !50
   ret void, !dbg !50
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind readnone speculatable willreturn
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
@@ -83,7 +83,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
 declare dso_local i32 @rand() #3
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define dso_local i32 @main() #0 !dbg !51 {
@@ -94,10 +94,10 @@ entry:
   %condition = alloca i32, align 4
   store i32 0, ptr %retval, align 4
   call void @init_arry(), !dbg !62
-  call void @llvm.lifetime.start.p0(i64 4, ptr %val) #6, !dbg !63
+  call void @llvm.lifetime.start.p0(ptr %val) #6, !dbg !63
   call void @llvm.dbg.declare(metadata ptr %val, metadata !55, metadata !DIExpression()), !dbg !64
   store i32 0, ptr %val, align 4, !dbg !64, !tbaa !30
-  call void @llvm.lifetime.start.p0(i64 4, ptr %j) #6, !dbg !65
+  call void @llvm.lifetime.start.p0(ptr %j) #6, !dbg !65
   call void @llvm.dbg.declare(metadata ptr %j, metadata !56, metadata !DIExpression()), !dbg !66
   store i32 0, ptr %j, align 4, !dbg !67, !tbaa !30
   br label %for.cond, !dbg !68
@@ -108,7 +108,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.end, !dbg !71
 
 for.body:                                         ; preds = %for.cond
-  call void @llvm.lifetime.start.p0(i64 4, ptr %condition) #6, !dbg !72
+  call void @llvm.lifetime.start.p0(ptr %condition) #6, !dbg !72
   call void @llvm.dbg.declare(metadata ptr %condition, metadata !57, metadata !DIExpression()), !dbg !73
   %call = call i32 @rand() #6, !dbg !74
   %rem = srem i32 %call, 5, !dbg !75
@@ -145,7 +145,7 @@ sw.default:                                       ; preds = %for.body
   unreachable, !dbg !87
 
 sw.epilog:                                        ; preds = %sw.bb3, %sw.bb2, %sw.bb
-  call void @llvm.lifetime.end.p0(i64 4, ptr %condition) #6, !dbg !88
+  call void @llvm.lifetime.end.p0(ptr %condition) #6, !dbg !88
   br label %for.inc, !dbg !89
 
 for.inc:                                          ; preds = %sw.epilog
@@ -155,8 +155,8 @@ for.inc:                                          ; preds = %sw.epilog
   br label %for.cond, !dbg !91, !llvm.loop !92
 
 for.end:                                          ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %j) #6, !dbg !94
-  call void @llvm.lifetime.end.p0(i64 4, ptr %val) #6, !dbg !94
+  call void @llvm.lifetime.end.p0(ptr %j) #6, !dbg !94
+  call void @llvm.lifetime.end.p0(ptr %val) #6, !dbg !94
   ret i32 0, !dbg !95
 }
 
diff --git a/llvm/test/Transforms/PartialInlining/switch_stmt.ll b/llvm/test/Transforms/PartialInlining/switch_stmt.ll
index 3f43369b050a6..d99fbba4682f9 100644
--- a/llvm/test/Transforms/PartialInlining/switch_stmt.ll
+++ b/llvm/test/Transforms/PartialInlining/switch_stmt.ll
@@ -62,7 +62,7 @@ define dso_local signext i32 @caller(i32 signext %c) !prof !30 {
 ; CHECK-LABEL: @caller(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RC_I:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[RC_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[RC_I]])
 ; CHECK-NEXT:    store i32 0, ptr [[RC_I]], align 4
 ; CHECK-NEXT:    switch i32 [[C:%.*]], label [[SW_DEFAULT_I:%.*]] [
 ; CHECK-NEXT:    i32 0, label [[CODEREPL_I:%.*]]
@@ -83,7 +83,7 @@ define dso_local signext i32 @caller(i32 signext %c) !prof !30 {
 ; CHECK-NEXT:    br label [[CALLEE_1_EXIT]]
 ; CHECK:       callee.1.exit:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[RC_I]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[RC_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[RC_I]])
 ;
 entry:
   %0 = call signext i32 @callee(i32 signext %c, i32 signext %c)
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/block_scaling_decompr_8bit.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/block_scaling_decompr_8bit.ll
index 05674b9efc39d..7175816963ed1 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/block_scaling_decompr_8bit.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/block_scaling_decompr_8bit.ll
@@ -94,7 +94,7 @@ define dso_local noundef i32 @_Z33block_scaling_decompr_8bitjPK27compressed_data
 ; CHECK-NEXT:    [[DST_ADDR_1]] = getelementptr inbounds nuw i8, ptr [[DST_ADDR_052]], i64 48
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT58]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP4]]
 ; CHECK:       [[FOR_END]]:
 ; CHECK-NEXT:    ret i32 0
 ;
@@ -801,8 +801,6 @@ attributes #2 = { nocallback nofree nosync nounwind willreturn memory(none) }
 !4 = distinct !{!4, !5}
 !5 = !{!"llvm.loop.mustprogress"}
 ;.
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]}
 ; CHECK: [[META5]] = !{!"llvm.loop.mustprogress"}
-; CHECK: [[META6]] = !{!"llvm.loop.unswitch.nontrivial.disable"}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
 ;.
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
index b4b12da3244b2..141503d344fe9 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
@@ -68,9 +68,9 @@ entry:
   store ptr %array, ptr %array.addr, align 8
   store i32 %count, ptr %count.addr, align 4
   store i32 %n, ptr %n.addr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %sum) #3
+  call void @llvm.lifetime.start.p0(ptr %sum) #3
   store i32 0, ptr %sum, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #3
+  call void @llvm.lifetime.start.p0(ptr %i) #3
   store i32 0, ptr %i, align 4
   br label %for.cond
 
@@ -81,7 +81,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #3
+  call void @llvm.lifetime.end.p0(ptr %i) #3
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -113,7 +113,7 @@ for.inc:                                          ; preds = %if.end
 
 for.end:                                          ; preds = %for.cond.cleanup
   %9 = load i32, ptr %sum, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %sum)
+  call void @llvm.lifetime.end.p0(ptr %sum)
   ret i32 %9
 }
 
@@ -184,9 +184,9 @@ entry:
   %1 = getelementptr inbounds { ptr, i64 }, ptr %s, i32 0, i32 1
   store i64 %s.coerce1, ptr %1, align 8
   store i64 %n, ptr %n.addr, align 8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %ret) #7
+  call void @llvm.lifetime.start.p0(ptr %ret) #7
   store i32 0, ptr %ret, align 4
-  call void @llvm.lifetime.start.p0(i64 8, ptr %i) #7
+  call void @llvm.lifetime.start.p0(ptr %i) #7
   store i64 0, ptr %i, align 8
   br label %for.cond
 
@@ -197,7 +197,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 8, ptr %i) #7
+  call void @llvm.lifetime.end.p0(ptr %i) #7
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -217,7 +217,7 @@ for.inc:                                          ; preds = %for.body
 
 for.end:                                          ; preds = %for.cond.cleanup
   %8 = load i32, ptr %ret, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %ret)
+  call void @llvm.lifetime.end.p0(ptr %ret)
   ret i32 %8
 }
 
@@ -283,11 +283,11 @@ entry:
   ret i64 %0
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @llvm.trap()
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 ;.
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
index f583a616dd375..e74bf592e1525 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
@@ -89,7 +89,7 @@ entry:
   %i = alloca i32, align 4
   store ptr %X, ptr %X.addr, align 8
   store ptr %Y, ptr %Y.addr, align 8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.start.p0(ptr %i) #2
   store i32 0, ptr %i, align 4
   br label %for.cond
 
@@ -99,7 +99,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.end.p0(ptr %i) #2
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -237,6 +237,6 @@ exit:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
index bb6f3e719bb14..afe7d7498fc1d 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/interleave_vec.ll
@@ -320,30 +320,17 @@ define void @same_op3_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <12 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <12 x float>, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <12 x float> [[WIDE_VEC]], <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT:    [[STRIDED_VEC12:%.*]] = shufflevector <12 x float> [[WIDE_VEC]], <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT:    [[STRIDED_VEC13:%.*]] = shufflevector <12 x float> [[WIDE_VEC]], <12 x float> poison, <4 x i32> <i32 2, i32 5, i32 8, i32 11>
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[WIDE_VEC14:%.*]] = load <12 x float>, ptr [[TMP3]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC15:%.*]] = shufflevector <12 x float> [[WIDE_VEC14]], <12 x float> poison, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
-; CHECK-NEXT:    [[STRIDED_VEC16:%.*]] = shufflevector <12 x float> [[WIDE_VEC14]], <12 x float> poison, <4 x i32> <i32 1, i32 4, i32 7, i32 10>
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast <4 x float> [[STRIDED_VEC15]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[STRIDED_VEC12]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP6:%.*]] = fadd fast <4 x float> [[STRIDED_VEC16]], [[TMP5]]
-; CHECK-NEXT:    [[TMP7:%.*]] = fmul fast <4 x float> [[STRIDED_VEC13]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <12 x float> [[WIDE_VEC14]], <12 x float> poison, <8 x i32> <i32 2, i32 5, i32 8, i32 11, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP11:%.*]] = fadd fast <8 x float> [[TMP9]], [[TMP10]]
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP8]], <8 x float> [[TMP11]], <12 x i32> <i32 0, i32 4, i32 8, i32 1, i32 5, i32 9, i32 2, i32 6, i32 10, i32 3, i32 7, i32 11>
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast <12 x float> [[WIDE_VEC]], [[TMP2]]
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = fadd fast <12 x float> [[WIDE_VEC14]], [[TMP4]]
 ; CHECK-NEXT:    store <12 x float> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 384
@@ -939,53 +926,17 @@ define void @same_op8_splat(ptr noalias noundef %a, ptr noundef %b, ptr noundef
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[C]], align 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x float> [[BROADCAST_SPLATINSERT]], <2 x float> poison, <16 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 3
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x float>, ptr [[TMP5]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT:    [[STRIDED_VEC12:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT:    [[STRIDED_VEC13:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT:    [[STRIDED_VEC14:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT:    [[STRIDED_VEC15:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT:    [[STRIDED_VEC16:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT:    [[STRIDED_VEC17:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT:    [[STRIDED_VEC18:%.*]] = shufflevector <16 x float> [[WIDE_VEC]], <16 x float> poison, <2 x i32> <i32 7, i32 15>
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[WIDE_VEC19:%.*]] = load <16 x float>, ptr [[TMP6]], align 4
-; CHECK-NEXT:    [[STRIDED_VEC20:%.*]] = shufflevector <16 x float> [[WIDE_VEC19]], <16 x float> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT:    [[STRIDED_VEC21:%.*]] = shufflevector <16 x float> [[WIDE_VEC19]], <16 x float> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT:    [[STRIDED_VEC22:%.*]] = shufflevector <16 x float> [[WIDE_VEC19]], <16 x float> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT:    [[STRIDED_VEC23:%.*]] = shufflevector <16 x float> [[WIDE_VEC19]], <16 x float> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT:    [[STRIDED_VEC24:%.*]] = shufflevector <16 x float> [[WIDE_VEC19]], <16 x float> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT:    [[STRIDED_VEC25:%.*]] = shufflevector <16 x float> [[WIDE_VEC19]], <16 x float> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT:    [[STRIDED_VEC26:%.*]] = shufflevector <16 x float> [[WIDE_VEC19]], <16 x float> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT:    [[STRIDED_VEC27:%.*]] = shufflevector <16 x float> [[WIDE_VEC19]], <16 x float> poison, <2 x i32> <i32 7, i32 15>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <2 x float> [[STRIDED_VEC20]], <2 x float> [[STRIDED_VEC21]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <2 x float> [[STRIDED_VEC]], <2 x float> [[STRIDED_VEC12]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast <4 x float> [[TMP8]], [[TMP1]]
-; CHECK-NEXT:    [[TMP10:%.*]] = fadd fast <4 x float> [[TMP7]], [[TMP9]]
-; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x float> [[STRIDED_VEC22]], <2 x float> [[STRIDED_VEC23]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <2 x float> [[STRIDED_VEC13]], <2 x float> [[STRIDED_VEC14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP13:%.*]] = fmul fast <4 x float> [[TMP12]], [[TMP2]]
-; CHECK-NEXT:    [[TMP14:%.*]] = fadd fast <4 x float> [[TMP11]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = shufflevector <2 x float> [[STRIDED_VEC24]], <2 x float> [[STRIDED_VEC25]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x float> [[STRIDED_VEC15]], <2 x float> [[STRIDED_VEC16]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP17:%.*]] = fmul fast <4 x float> [[TMP16]], [[TMP3]]
-; CHECK-NEXT:    [[TMP18:%.*]] = fadd fast <4 x float> [[TMP15]], [[TMP17]]
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <2 x float> [[STRIDED_VEC26]], <2 x float> [[STRIDED_VEC27]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <2 x float> [[STRIDED_VEC17]], <2 x float> [[STRIDED_VEC18]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP21:%.*]] = fmul fast <4 x float> [[TMP20]], [[TMP4]]
-; CHECK-NEXT:    [[TMP22:%.*]] = fadd fast <4 x float> [[TMP19]], [[TMP21]]
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <4 x float> [[TMP10]], <4 x float> [[TMP14]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <4 x float> [[TMP18]], <4 x float> [[TMP22]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x float> [[TMP23]], <8 x float> [[TMP24]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast <16 x float> [[WIDE_VEC]], [[TMP1]]
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = fadd fast <16 x float> [[WIDE_VEC19]], [[TMP4]]
 ; CHECK-NEXT:    store <16 x float> [[INTERLEAVED_VEC]], ptr [[TMP6]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 144
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
index 089511d5b2e57..967980689ad1b 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
@@ -408,7 +408,7 @@ entry:
   store i32 %i, ptr %i.addr, align 4
   store ptr %A, ptr %A.addr, align 8
   store ptr %B, ptr %B.addr, align 8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %j) #3
+  call void @llvm.lifetime.start.p0(ptr %j) #3
   store i32 0, ptr %j, align 4
   br label %for.cond
 
@@ -419,11 +419,11 @@ for.cond:                                         ; preds = %for.inc12, %entry
 
 for.cond.cleanup:                                 ; preds = %for.cond
   store i32 2, ptr %cleanup.dest.slot, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %j) #3
+  call void @llvm.lifetime.end.p0(ptr %j) #3
   br label %for.end14
 
 for.body:                                         ; preds = %for.cond
-  call void @llvm.lifetime.start.p0(i64 4, ptr %k) #3
+  call void @llvm.lifetime.start.p0(ptr %k) #3
   store i32 0, ptr %k, align 4
   br label %for.cond1
 
@@ -435,7 +435,7 @@ for.cond1:                                        ; preds = %for.inc, %for.body
 
 for.cond.cleanup3:                                ; preds = %for.cond1
   store i32 5, ptr %cleanup.dest.slot, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %k) #3
+  call void @llvm.lifetime.end.p0(ptr %k) #3
   br label %for.end
 
 for.body4:                                        ; preds = %for.cond1
@@ -501,13 +501,13 @@ for.end14:                                        ; preds = %for.cond.cleanup
 }
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: inaccessiblememonly nofree nosync nounwind willreturn
 declare void @llvm.assume(i1 noundef) #2
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind ssp uwtable mustprogress
 
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
index c7098d2ce96ce..e8709a50524f8 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
@@ -108,13 +108,13 @@ entry:
   store ptr %samples, ptr %samples.addr, align 8
   store double %Y, ptr %Y.addr, align 8
   store double %Z, ptr %Z.addr, align 8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2
-  call void @llvm.lifetime.start.p0(i64 4, ptr %block) #2
-  call void @llvm.lifetime.start.p0(i64 8, ptr %rngVal) #2
-  call void @llvm.lifetime.start.p0(i64 8, ptr %callValue) #2
-  call void @llvm.lifetime.start.p0(i64 8, ptr %v0) #2
+  call void @llvm.lifetime.start.p0(ptr %i) #2
+  call void @llvm.lifetime.start.p0(ptr %block) #2
+  call void @llvm.lifetime.start.p0(ptr %rngVal) #2
+  call void @llvm.lifetime.start.p0(ptr %callValue) #2
+  call void @llvm.lifetime.start.p0(ptr %v0) #2
   store double 0.000000e+00, ptr %v0, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %v1) #2
+  call void @llvm.lifetime.start.p0(ptr %v1) #2
   store double 0.000000e+00, ptr %v1, align 8
   store i32 0, ptr %i, align 4
   br label %for.cond
@@ -169,12 +169,12 @@ for.end:                                          ; preds = %for.cond
   %15 = load double, ptr %v0, align 8
   %16 = load double, ptr %v1, align 8
   %add5 = fadd fast double %15, %16
-  call void @llvm.lifetime.end.p0(i64 8, ptr %v1) #2
-  call void @llvm.lifetime.end.p0(i64 8, ptr %v0) #2
-  call void @llvm.lifetime.end.p0(i64 8, ptr %callValue) #2
-  call void @llvm.lifetime.end.p0(i64 8, ptr %rngVal) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %block) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.end.p0(ptr %v1) #2
+  call void @llvm.lifetime.end.p0(ptr %v0) #2
+  call void @llvm.lifetime.end.p0(ptr %callValue) #2
+  call void @llvm.lifetime.end.p0(ptr %rngVal) #2
+  call void @llvm.lifetime.end.p0(ptr %block) #2
+  call void @llvm.lifetime.end.p0(ptr %i) #2
   ret double %add5
 }
 
@@ -305,13 +305,13 @@ entry:
   store ptr %samples, ptr %samples.addr, align 8
   store double %Y, ptr %Y.addr, align 8
   store double %Z, ptr %Z.addr, align 8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %block) #4
-  call void @llvm.lifetime.start.p0(i64 8, ptr %rngVal) #4
-  call void @llvm.lifetime.start.p0(i64 8, ptr %callValue) #4
-  call void @llvm.lifetime.start.p0(i64 8, ptr %v0) #4
+  call void @llvm.lifetime.start.p0(ptr %i) #4
+  call void @llvm.lifetime.start.p0(ptr %block) #4
+  call void @llvm.lifetime.start.p0(ptr %rngVal) #4
+  call void @llvm.lifetime.start.p0(ptr %callValue) #4
+  call void @llvm.lifetime.start.p0(ptr %v0) #4
   store double 0.000000e+00, ptr %v0, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %v1) #4
+  call void @llvm.lifetime.start.p0(ptr %v1) #4
   store double 0.000000e+00, ptr %v1, align 8
   store i32 0, ptr %block, align 4
   br label %for.cond
@@ -389,19 +389,19 @@ for.end10:                                        ; preds = %for.cond
   %21 = load double, ptr %v0, align 8
   %22 = load double, ptr %v1, align 8
   %add11 = fadd fast double %21, %22
-  call void @llvm.lifetime.end.p0(i64 8, ptr %v1) #4
-  call void @llvm.lifetime.end.p0(i64 8, ptr %v0) #4
-  call void @llvm.lifetime.end.p0(i64 8, ptr %callValue) #4
-  call void @llvm.lifetime.end.p0(i64 8, ptr %rngVal) #4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %block) #4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #4
+  call void @llvm.lifetime.end.p0(ptr %v1) #4
+  call void @llvm.lifetime.end.p0(ptr %v0) #4
+  call void @llvm.lifetime.end.p0(ptr %callValue) #4
+  call void @llvm.lifetime.end.p0(ptr %rngVal) #4
+  call void @llvm.lifetime.end.p0(ptr %block) #4
+  call void @llvm.lifetime.end.p0(ptr %i) #4
   ret double %add11
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 declare void @resample(i32 noundef, ptr noundef)
 declare double @llvm.exp2.f64(double)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 ;.
 ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
 ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll
index d55559d632019..258ef63503bec 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/quant_4x4.ll
@@ -525,9 +525,9 @@ entry:
   store ptr %dct, ptr %dct.addr, align 8
   store ptr %mf, ptr %mf.addr, align 8
   store ptr %bias, ptr %bias.addr, align 8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %nz) #2
+  call void @llvm.lifetime.start.p0(ptr %nz) #2
   store i32 0, ptr %nz, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.start.p0(ptr %i) #2
   store i32 0, ptr %i, align 4
   br label %for.cond
 
@@ -537,7 +537,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.end.p0(ptr %i) #2
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -636,13 +636,13 @@ for.end:                                          ; preds = %for.cond.cleanup
   %lnot = xor i1 %tobool, true
   %lnot34 = xor i1 %lnot, true
   %lnot.ext = zext i1 %lnot34 to i32
-  call void @llvm.lifetime.end.p0(i64 4, ptr %nz) #2
+  call void @llvm.lifetime.end.p0(ptr %nz) #2
   ret i32 %lnot.ext
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/reduce_muladd.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/reduce_muladd.ll
new file mode 100644
index 0000000000000..ca78ff756ef55
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/reduce_muladd.ll
@@ -0,0 +1,92 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S -O3 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64"
+
+; This function (a 16x reduction of a[i] * b[i]) should be vectorized successfully.
+
+define dso_local nofpclass(nan inf) float @vmlaq(ptr noundef %0, ptr noundef %1) #0 {
+; CHECK-LABEL: define dso_local nofpclass(nan inf) float @vmlaq
+; CHECK-SAME: (ptr noundef readonly captures(none) [[TMP0:%.*]], ptr noundef readonly captures(none) [[TMP1:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x float>, ptr [[TMP0]], align 4, !tbaa [[TBAA4:![0-9]+]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x float>, ptr [[TMP1]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <16 x float> [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call fast float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> [[TMP5]])
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %3 = alloca ptr, align 8
+  %4 = alloca ptr, align 8
+  %5 = alloca float, align 4
+  %6 = alloca i32, align 4
+  store ptr %0, ptr %3, align 8, !tbaa !4
+  store ptr %1, ptr %4, align 8, !tbaa !4
+  call void @llvm.lifetime.start.p0(ptr %5) #2
+  store float 0.000000e+00, ptr %5, align 4, !tbaa !9
+  call void @llvm.lifetime.start.p0(ptr %6) #2
+  store i32 0, ptr %6, align 4, !tbaa !11
+  br label %7
+
+7:                                                ; preds = %25, %2
+  %8 = load i32, ptr %6, align 4, !tbaa !11
+  %9 = icmp slt i32 %8, 16
+  br i1 %9, label %11, label %10
+
+10:                                               ; preds = %7
+  call void @llvm.lifetime.end.p0(ptr %6) #2
+  br label %28
+
+11:                                               ; preds = %7
+  %12 = load ptr, ptr %3, align 8, !tbaa !4
+  %13 = load i32, ptr %6, align 4, !tbaa !11
+  %14 = sext i32 %13 to i64
+  %15 = getelementptr inbounds float, ptr %12, i64 %14
+  %16 = load float, ptr %15, align 4, !tbaa !9
+  %17 = load ptr, ptr %4, align 8, !tbaa !4
+  %18 = load i32, ptr %6, align 4, !tbaa !11
+  %19 = sext i32 %18 to i64
+  %20 = getelementptr inbounds float, ptr %17, i64 %19
+  %21 = load float, ptr %20, align 4, !tbaa !9
+  %22 = fmul fast float %16, %21
+  %23 = load float, ptr %5, align 4, !tbaa !9
+  %24 = fadd fast float %23, %22
+  store float %24, ptr %5, align 4, !tbaa !9
+  br label %25
+
+25:                                               ; preds = %11
+  %26 = load i32, ptr %6, align 4, !tbaa !11
+  %27 = add nsw i32 %26, 1
+  store i32 %27, ptr %6, align 4, !tbaa !11
+  br label %7, !llvm.loop !13
+
+28:                                               ; preds = %10
+  %29 = load float, ptr %5, align 4, !tbaa !9
+  call void @llvm.lifetime.end.p0(ptr %5) #2
+  ret float %29
+}
+
+declare void @llvm.lifetime.start.p0(ptr captures(none)) #1
+declare void @llvm.lifetime.end.p0(ptr captures(none)) #1
+
+attributes #0 = { nounwind uwtable "approx-func-fp-math"="true" "frame-pointer"="non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" "unsafe-fp-math"="true" }
+attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+attributes #2 = { nounwind }
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"uwtable", i32 2}
+!2 = !{i32 7, !"frame-pointer", i32 1}
+!3 = !{!"clang version 22.0.0git"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"p1 float", !6, i64 0}
+!6 = !{!"any pointer", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C/C++ TBAA"}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"float", !7, i64 0}
+!11 = !{!12, !12, i64 0}
+!12 = !{!"int", !7, i64 0}
+!13 = distinct !{!13, !14}
+!14 = !{!"llvm.loop.mustprogress"}
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/reduce_submuladd.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/reduce_submuladd.ll
new file mode 100644
index 0000000000000..fb958b822503a
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/reduce_submuladd.ll
@@ -0,0 +1,274 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -S -O3 < %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64"
+
+; This function (a more complex reduction of (a[i] - b[i]) * itself) should be vectorized successfully.
+
+define dso_local noundef nofpclass(nan inf) float @_Z4testPKfS0_ii(ptr noundef %0, ptr noundef %1, i32 noundef %2, i32 noundef %3) #0 {
+; CHECK-LABEL: define dso_local noundef nofpclass(nan inf) float @_Z4testPKfS0_ii
+; CHECK-SAME: (ptr noundef readonly captures(none) [[TMP0:%.*]], ptr noundef readonly captures(none) [[TMP1:%.*]], i32 noundef [[TMP2:%.*]], i32 noundef [[TMP3:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  .preheader.i:
+; CHECK-NEXT:    [[TMP4:%.*]] = sext i32 [[TMP3]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = sext i32 [[TMP2]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = load <20 x float>, ptr [[TMP0]], align 4, !tbaa [[TBAA4:![0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load <20 x float>, ptr [[TMP1]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fsub fast <20 x float> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast <20 x float> [[TMP8]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 80
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[TMP10]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 80
+; CHECK-NEXT:    [[TMP13:%.*]] = load float, ptr [[TMP12]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fsub fast float [[TMP11]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul fast float [[TMP14]], [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 [[TMP4]]
+; CHECK-NEXT:    [[OP_RDX:%.*]] = tail call fast float @llvm.vector.reduce.fadd.v20f32(float [[TMP15]], <20 x float> [[TMP9]])
+; CHECK-NEXT:    [[TMP18:%.*]] = load <20 x float>, ptr [[TMP16]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP19:%.*]] = load <20 x float>, ptr [[TMP17]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP20:%.*]] = fsub fast <20 x float> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = fmul fast <20 x float> [[TMP20]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP16]], i64 80
+; CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[TMP22]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], i64 80
+; CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[TMP24]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP26:%.*]] = fsub fast float [[TMP23]], [[TMP25]]
+; CHECK-NEXT:    [[TMP27:%.*]] = fmul fast float [[TMP26]], [[TMP26]]
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP4]]
+; CHECK-NEXT:    [[OP_RDX_1:%.*]] = tail call fast float @llvm.vector.reduce.fadd.v20f32(float [[TMP27]], <20 x float> [[TMP21]])
+; CHECK-NEXT:    [[OP_RDX3_1:%.*]] = fadd fast float [[OP_RDX_1]], [[OP_RDX]]
+; CHECK-NEXT:    [[TMP30:%.*]] = load <20 x float>, ptr [[TMP28]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP31:%.*]] = load <20 x float>, ptr [[TMP29]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP32:%.*]] = fsub fast <20 x float> [[TMP30]], [[TMP31]]
+; CHECK-NEXT:    [[TMP33:%.*]] = fmul fast <20 x float> [[TMP32]], [[TMP32]]
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP28]], i64 80
+; CHECK-NEXT:    [[TMP35:%.*]] = load float, ptr [[TMP34]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP29]], i64 80
+; CHECK-NEXT:    [[TMP37:%.*]] = load float, ptr [[TMP36]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP38:%.*]] = fsub fast float [[TMP35]], [[TMP37]]
+; CHECK-NEXT:    [[TMP39:%.*]] = fmul fast float [[TMP38]], [[TMP38]]
+; CHECK-NEXT:    [[TMP40:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr inbounds float, ptr [[TMP29]], i64 [[TMP4]]
+; CHECK-NEXT:    [[OP_RDX_2:%.*]] = tail call fast float @llvm.vector.reduce.fadd.v20f32(float [[TMP39]], <20 x float> [[TMP33]])
+; CHECK-NEXT:    [[OP_RDX3_2:%.*]] = fadd fast float [[OP_RDX_2]], [[OP_RDX3_1]]
+; CHECK-NEXT:    [[TMP42:%.*]] = load <20 x float>, ptr [[TMP40]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP43:%.*]] = load <20 x float>, ptr [[TMP41]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP44:%.*]] = fsub fast <20 x float> [[TMP42]], [[TMP43]]
+; CHECK-NEXT:    [[TMP45:%.*]] = fmul fast <20 x float> [[TMP44]], [[TMP44]]
+; CHECK-NEXT:    [[TMP46:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP40]], i64 80
+; CHECK-NEXT:    [[TMP47:%.*]] = load float, ptr [[TMP46]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP41]], i64 80
+; CHECK-NEXT:    [[TMP49:%.*]] = load float, ptr [[TMP48]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP50:%.*]] = fsub fast float [[TMP47]], [[TMP49]]
+; CHECK-NEXT:    [[TMP51:%.*]] = fmul fast float [[TMP50]], [[TMP50]]
+; CHECK-NEXT:    [[TMP52:%.*]] = getelementptr inbounds float, ptr [[TMP40]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP53:%.*]] = getelementptr inbounds float, ptr [[TMP41]], i64 [[TMP4]]
+; CHECK-NEXT:    [[OP_RDX_3:%.*]] = tail call fast float @llvm.vector.reduce.fadd.v20f32(float [[TMP51]], <20 x float> [[TMP45]])
+; CHECK-NEXT:    [[OP_RDX3_3:%.*]] = fadd fast float [[OP_RDX_3]], [[OP_RDX3_2]]
+; CHECK-NEXT:    [[TMP54:%.*]] = load <20 x float>, ptr [[TMP52]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP55:%.*]] = load <20 x float>, ptr [[TMP53]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP56:%.*]] = fsub fast <20 x float> [[TMP54]], [[TMP55]]
+; CHECK-NEXT:    [[TMP57:%.*]] = fmul fast <20 x float> [[TMP56]], [[TMP56]]
+; CHECK-NEXT:    [[TMP58:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP52]], i64 80
+; CHECK-NEXT:    [[TMP59:%.*]] = load float, ptr [[TMP58]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP53]], i64 80
+; CHECK-NEXT:    [[TMP61:%.*]] = load float, ptr [[TMP60]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP62:%.*]] = fsub fast float [[TMP59]], [[TMP61]]
+; CHECK-NEXT:    [[TMP63:%.*]] = fmul fast float [[TMP62]], [[TMP62]]
+; CHECK-NEXT:    [[TMP64:%.*]] = getelementptr inbounds float, ptr [[TMP52]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr inbounds float, ptr [[TMP53]], i64 [[TMP4]]
+; CHECK-NEXT:    [[OP_RDX_4:%.*]] = tail call fast float @llvm.vector.reduce.fadd.v20f32(float [[TMP63]], <20 x float> [[TMP57]])
+; CHECK-NEXT:    [[OP_RDX3_4:%.*]] = fadd fast float [[OP_RDX_4]], [[OP_RDX3_3]]
+; CHECK-NEXT:    [[TMP66:%.*]] = load <20 x float>, ptr [[TMP64]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP67:%.*]] = load <20 x float>, ptr [[TMP65]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP68:%.*]] = fsub fast <20 x float> [[TMP66]], [[TMP67]]
+; CHECK-NEXT:    [[TMP69:%.*]] = fmul fast <20 x float> [[TMP68]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP64]], i64 80
+; CHECK-NEXT:    [[TMP71:%.*]] = load float, ptr [[TMP70]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP72:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP65]], i64 80
+; CHECK-NEXT:    [[TMP73:%.*]] = load float, ptr [[TMP72]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP74:%.*]] = fsub fast float [[TMP71]], [[TMP73]]
+; CHECK-NEXT:    [[TMP75:%.*]] = fmul fast float [[TMP74]], [[TMP74]]
+; CHECK-NEXT:    [[TMP76:%.*]] = getelementptr inbounds float, ptr [[TMP64]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP77:%.*]] = getelementptr inbounds float, ptr [[TMP65]], i64 [[TMP4]]
+; CHECK-NEXT:    [[OP_RDX_5:%.*]] = tail call fast float @llvm.vector.reduce.fadd.v20f32(float [[TMP75]], <20 x float> [[TMP69]])
+; CHECK-NEXT:    [[OP_RDX3_5:%.*]] = fadd fast float [[OP_RDX_5]], [[OP_RDX3_4]]
+; CHECK-NEXT:    [[TMP78:%.*]] = load <20 x float>, ptr [[TMP76]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP79:%.*]] = load <20 x float>, ptr [[TMP77]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP80:%.*]] = fsub fast <20 x float> [[TMP78]], [[TMP79]]
+; CHECK-NEXT:    [[TMP81:%.*]] = fmul fast <20 x float> [[TMP80]], [[TMP80]]
+; CHECK-NEXT:    [[TMP82:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP76]], i64 80
+; CHECK-NEXT:    [[TMP83:%.*]] = load float, ptr [[TMP82]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP84:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP77]], i64 80
+; CHECK-NEXT:    [[TMP85:%.*]] = load float, ptr [[TMP84]], align 4, !tbaa [[TBAA4]]
+; CHECK-NEXT:    [[TMP86:%.*]] = fsub fast float [[TMP83]], [[TMP85]]
+; CHECK-NEXT:    [[TMP87:%.*]] = fmul fast float [[TMP86]], [[TMP86]]
+; CHECK-NEXT:    [[OP_RDX_6:%.*]] = tail call fast float @llvm.vector.reduce.fadd.v20f32(float [[TMP87]], <20 x float> [[TMP81]])
+; CHECK-NEXT:    [[OP_RDX3_6:%.*]] = fadd fast float [[OP_RDX_6]], [[OP_RDX3_5]]
+; CHECK-NEXT:    ret float [[OP_RDX3_6]]
+;
+  %5 = alloca ptr, align 8
+  %6 = alloca ptr, align 8
+  %7 = alloca i32, align 4
+  %8 = alloca i32, align 4
+  store ptr %0, ptr %5, align 8, !tbaa !4
+  store ptr %1, ptr %6, align 8, !tbaa !4
+  store i32 %2, ptr %7, align 4, !tbaa !9
+  store i32 %3, ptr %8, align 4, !tbaa !9
+  %9 = load ptr, ptr %5, align 8, !tbaa !4
+  %10 = load ptr, ptr %6, align 8, !tbaa !4
+  %11 = load i32, ptr %7, align 4, !tbaa !9
+  %12 = load i32, ptr %8, align 4, !tbaa !9
+  %13 = call fast noundef nofpclass(nan inf) float @_ZL6reduceILi7EEfPKfS1_ii(ptr noundef %9, ptr noundef %10, i32 noundef %11, i32 noundef %12)
+  ret float %13
+}
+
+define internal noundef nofpclass(nan inf) float @_ZL6reduceILi7EEfPKfS1_ii(ptr noundef %0, ptr noundef %1, i32 noundef %2, i32 noundef %3) #1 {
+  %5 = alloca ptr, align 8
+  %6 = alloca ptr, align 8
+  %7 = alloca i32, align 4
+  %8 = alloca i32, align 4
+  %9 = alloca i32, align 4
+  %10 = alloca i32, align 4
+  %11 = alloca i32, align 4
+  %12 = alloca float, align 4
+  %13 = alloca i32, align 4
+  %14 = alloca i32, align 4
+  %15 = alloca float, align 4
+  %16 = alloca i32, align 4
+  %17 = alloca float, align 4
+  store ptr %0, ptr %5, align 8, !tbaa !4
+  store ptr %1, ptr %6, align 8, !tbaa !4
+  store i32 %2, ptr %7, align 4, !tbaa !9
+  store i32 %3, ptr %8, align 4, !tbaa !9
+  call void @llvm.lifetime.start.p0(ptr %9) #3
+  store i32 3, ptr %9, align 4, !tbaa !9
+  call void @llvm.lifetime.start.p0(ptr %10) #3
+  store i32 3, ptr %10, align 4, !tbaa !9
+  call void @llvm.lifetime.start.p0(ptr %11) #3
+  store i32 7, ptr %11, align 4, !tbaa !9
+  call void @llvm.lifetime.start.p0(ptr %12) #3
+  store float 0.000000e+00, ptr %12, align 4, !tbaa !11
+  call void @llvm.lifetime.start.p0(ptr %13) #3
+  store i32 0, ptr %13, align 4, !tbaa !9
+  br label %18
+
+18:                                               ; preds = %59, %4
+  %19 = load i32, ptr %13, align 4, !tbaa !9
+  %20 = icmp slt i32 %19, 7
+  br i1 %20, label %22, label %21
+
+21:                                               ; preds = %18
+  store i32 2, ptr %14, align 4
+  call void @llvm.lifetime.end.p0(ptr %13) #3
+  br label %62
+
+22:                                               ; preds = %18
+  call void @llvm.lifetime.start.p0(ptr %15) #3
+  store float 0.000000e+00, ptr %15, align 4, !tbaa !11
+  call void @llvm.lifetime.start.p0(ptr %16) #3
+  store i32 0, ptr %16, align 4, !tbaa !9
+  br label %23
+
+23:                                               ; preds = %44, %22
+  %24 = load i32, ptr %16, align 4, !tbaa !9
+  %25 = icmp slt i32 %24, 21
+  br i1 %25, label %27, label %26
+
+26:                                               ; preds = %23
+  store i32 5, ptr %14, align 4
+  call void @llvm.lifetime.end.p0(ptr %16) #3
+  br label %47
+
+27:                                               ; preds = %23
+  call void @llvm.lifetime.start.p0(ptr %17) #3
+  %28 = load ptr, ptr %5, align 8, !tbaa !4
+  %29 = load i32, ptr %16, align 4, !tbaa !9
+  %30 = sext i32 %29 to i64
+  %31 = getelementptr inbounds float, ptr %28, i64 %30
+  %32 = load float, ptr %31, align 4, !tbaa !11
+  %33 = load ptr, ptr %6, align 8, !tbaa !4
+  %34 = load i32, ptr %16, align 4, !tbaa !9
+  %35 = sext i32 %34 to i64
+  %36 = getelementptr inbounds float, ptr %33, i64 %35
+  %37 = load float, ptr %36, align 4, !tbaa !11
+  %38 = fsub fast float %32, %37
+  store float %38, ptr %17, align 4, !tbaa !11
+  %39 = load float, ptr %17, align 4, !tbaa !11
+  %40 = load float, ptr %17, align 4, !tbaa !11
+  %41 = fmul fast float %39, %40
+  %42 = load float, ptr %15, align 4, !tbaa !11
+  %43 = fadd fast float %42, %41
+  store float %43, ptr %15, align 4, !tbaa !11
+  call void @llvm.lifetime.end.p0(ptr %17) #3
+  br label %44
+
+44:                                               ; preds = %27
+  %45 = load i32, ptr %16, align 4, !tbaa !9
+  %46 = add nsw i32 %45, 1
+  store i32 %46, ptr %16, align 4, !tbaa !9
+  br label %23, !llvm.loop !13
+
+47:                                               ; preds = %26
+  %48 = load i32, ptr %7, align 4, !tbaa !9
+  %49 = load ptr, ptr %5, align 8, !tbaa !4
+  %50 = sext i32 %48 to i64
+  %51 = getelementptr inbounds float, ptr %49, i64 %50
+  store ptr %51, ptr %5, align 8, !tbaa !4
+  %52 = load i32, ptr %8, align 4, !tbaa !9
+  %53 = load ptr, ptr %6, align 8, !tbaa !4
+  %54 = sext i32 %52 to i64
+  %55 = getelementptr inbounds float, ptr %53, i64 %54
+  store ptr %55, ptr %6, align 8, !tbaa !4
+  %56 = load float, ptr %15, align 4, !tbaa !11
+  %57 = load float, ptr %12, align 4, !tbaa !11
+  %58 = fadd fast float %57, %56
+  store float %58, ptr %12, align 4, !tbaa !11
+  call void @llvm.lifetime.end.p0(ptr %15) #3
+  br label %59
+
+59:                                               ; preds = %47
+  %60 = load i32, ptr %13, align 4, !tbaa !9
+  %61 = add nsw i32 %60, 1
+  store i32 %61, ptr %13, align 4, !tbaa !9
+  br label %18, !llvm.loop !15
+
+62:                                               ; preds = %21
+  %63 = load float, ptr %12, align 4, !tbaa !11
+  store i32 1, ptr %14, align 4
+  call void @llvm.lifetime.end.p0(ptr %12) #3
+  call void @llvm.lifetime.end.p0(ptr %11) #3
+  call void @llvm.lifetime.end.p0(ptr %10) #3
+  call void @llvm.lifetime.end.p0(ptr %9) #3
+  ret float %63
+}
+
+declare void @llvm.lifetime.start.p0(ptr captures(none)) #2
+declare void @llvm.lifetime.end.p0(ptr captures(none)) #2
+
+attributes #0 = { mustprogress uwtable "approx-func-fp-math"="true" "frame-pointer"="non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" "unsafe-fp-math"="true" }
+attributes #1 = { inlinehint mustprogress nounwind uwtable "approx-func-fp-math"="true" "frame-pointer"="non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+fp-armv8,+neon,+v8a,-fmv" "unsafe-fp-math"="true" }
+attributes #2 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0, !1, !2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"uwtable", i32 2}
+!2 = !{i32 7, !"frame-pointer", i32 1}
+!3 = !{!"clang version 22.0.0git"}
+!4 = !{!5, !5, i64 0}
+!5 = !{!"p1 float", !6, i64 0}
+!6 = !{!"any pointer", !7, i64 0}
+!7 = !{!"omnipotent char", !8, i64 0}
+!8 = !{!"Simple C++ TBAA"}
+!9 = !{!10, !10, i64 0}
+!10 = !{!"int", !7, i64 0}
+!11 = !{!12, !12, i64 0}
+!12 = !{!"float", !7, i64 0}
+!13 = distinct !{!13, !14}
+!14 = !{!"llvm.loop.mustprogress"}
+!15 = distinct !{!15, !14}
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll
index a2019836098e8..d19242fdd2e13 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/slpordering.ll
@@ -136,14 +136,14 @@ entry:
   store i32 %ip1, ptr %ip1.addr, align 4, !tbaa !8
   store ptr %p2, ptr %p2.addr, align 8, !tbaa !4
   store i32 %ip2, ptr %ip2.addr, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 64, ptr %emp) #2
-  call void @llvm.lifetime.start.p0(i64 4, ptr %r0) #2
-  call void @llvm.lifetime.start.p0(i64 4, ptr %r1) #2
-  call void @llvm.lifetime.start.p0(i64 4, ptr %r2) #2
-  call void @llvm.lifetime.start.p0(i64 4, ptr %r3) #2
-  call void @llvm.lifetime.start.p0(i64 4, ptr %sum) #2
+  call void @llvm.lifetime.start.p0(ptr %emp) #2
+  call void @llvm.lifetime.start.p0(ptr %r0) #2
+  call void @llvm.lifetime.start.p0(ptr %r1) #2
+  call void @llvm.lifetime.start.p0(ptr %r2) #2
+  call void @llvm.lifetime.start.p0(ptr %r3) #2
+  call void @llvm.lifetime.start.p0(ptr %sum) #2
   store i32 0, ptr %sum, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.start.p0(ptr %i) #2
   store i32 0, ptr %i, align 4, !tbaa !8
   br label %for.cond
 
@@ -153,7 +153,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.end.p0(ptr %i) #2
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -241,22 +241,22 @@ for.body:                                         ; preds = %for.cond
   %shl42 = shl i32 %sub41, 16
   %rdd43 = add nsw i32 %sub36, %shl42
   store i32 %rdd43, ptr %r3, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %e0) #2
+  call void @llvm.lifetime.start.p0(ptr %e0) #2
   %33 = load i32, ptr %r0, align 4, !tbaa !8
   %34 = load i32, ptr %r1, align 4, !tbaa !8
   %rdd44 = add i32 %33, %34
   store i32 %rdd44, ptr %e0, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %e1) #2
+  call void @llvm.lifetime.start.p0(ptr %e1) #2
   %35 = load i32, ptr %r0, align 4, !tbaa !8
   %36 = load i32, ptr %r1, align 4, !tbaa !8
   %sub45 = sub i32 %35, %36
   store i32 %sub45, ptr %e1, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %e2) #2
+  call void @llvm.lifetime.start.p0(ptr %e2) #2
   %37 = load i32, ptr %r2, align 4, !tbaa !8
   %38 = load i32, ptr %r3, align 4, !tbaa !8
   %rdd46 = add i32 %37, %38
   store i32 %rdd46, ptr %e2, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %e3) #2
+  call void @llvm.lifetime.start.p0(ptr %e3) #2
   %39 = load i32, ptr %r2, align 4, !tbaa !8
   %40 = load i32, ptr %r3, align 4, !tbaa !8
   %sub47 = sub i32 %39, %40
@@ -293,10 +293,10 @@ for.body:                                         ; preds = %for.cond
   %rrrayidx61 = getelementptr inbounds [4 x [4 x i32]], ptr %emp, i64 0, i64 %idxprom60
   %rrrayidx62 = getelementptr inbounds [4 x i32], ptr %rrrayidx61, i64 0, i64 3
   store i32 %sub59, ptr %rrrayidx62, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 4, ptr %e3) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %e2) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %e1) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %e0) #2
+  call void @llvm.lifetime.end.p0(ptr %e3) #2
+  call void @llvm.lifetime.end.p0(ptr %e2) #2
+  call void @llvm.lifetime.end.p0(ptr %e1) #2
+  call void @llvm.lifetime.end.p0(ptr %e0) #2
   br label %for.inc
 
 for.inc:                                          ; preds = %for.body
@@ -316,7 +316,7 @@ for.inc:                                          ; preds = %for.body
   br label %for.cond, !llvm.loop !11
 
 for.end:                                          ; preds = %for.cond.cleanup
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i65) #2
+  call void @llvm.lifetime.start.p0(ptr %i65) #2
   store i32 0, ptr %i65, align 4, !tbaa !8
   br label %for.cond66
 
@@ -326,11 +326,11 @@ for.cond66:                                       ; preds = %for.inc114, %for.en
   br i1 %cmp67, label %for.body70, label %for.cond.cleanup69
 
 for.cond.cleanup69:                               ; preds = %for.cond66
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i65) #2
+  call void @llvm.lifetime.end.p0(ptr %i65) #2
   br label %for.end116
 
 for.body70:                                       ; preds = %for.cond66
-  call void @llvm.lifetime.start.p0(i64 4, ptr %e071) #2
+  call void @llvm.lifetime.start.p0(ptr %e071) #2
   %rrrayidx72 = getelementptr inbounds [4 x [4 x i32]], ptr %emp, i64 0, i64 0
   %59 = load i32, ptr %i65, align 4, !tbaa !8
   %idxprom73 = sext i32 %59 to i64
@@ -343,7 +343,7 @@ for.body70:                                       ; preds = %for.cond66
   %62 = load i32, ptr %rrrayidx77, align 4, !tbaa !8
   %rdd78 = add i32 %60, %62
   store i32 %rdd78, ptr %e071, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %e179) #2
+  call void @llvm.lifetime.start.p0(ptr %e179) #2
   %rrrayidx80 = getelementptr inbounds [4 x [4 x i32]], ptr %emp, i64 0, i64 0
   %63 = load i32, ptr %i65, align 4, !tbaa !8
   %idxprom81 = sext i32 %63 to i64
@@ -356,7 +356,7 @@ for.body70:                                       ; preds = %for.cond66
   %66 = load i32, ptr %rrrayidx85, align 4, !tbaa !8
   %sub86 = sub i32 %64, %66
   store i32 %sub86, ptr %e179, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %e287) #2
+  call void @llvm.lifetime.start.p0(ptr %e287) #2
   %rrrayidx88 = getelementptr inbounds [4 x [4 x i32]], ptr %emp, i64 0, i64 2
   %67 = load i32, ptr %i65, align 4, !tbaa !8
   %idxprom89 = sext i32 %67 to i64
@@ -369,7 +369,7 @@ for.body70:                                       ; preds = %for.cond66
   %70 = load i32, ptr %rrrayidx93, align 4, !tbaa !8
   %rdd94 = add i32 %68, %70
   store i32 %rdd94, ptr %e287, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %e395) #2
+  call void @llvm.lifetime.start.p0(ptr %e395) #2
   %rrrayidx96 = getelementptr inbounds [4 x [4 x i32]], ptr %emp, i64 0, i64 2
   %71 = load i32, ptr %i65, align 4, !tbaa !8
   %idxprom97 = sext i32 %71 to i64
@@ -398,10 +398,10 @@ for.body70:                                       ; preds = %for.cond66
   %82 = load i32, ptr %e395, align 4, !tbaa !8
   %sub106 = sub nsw i32 %81, %82
   store i32 %sub106, ptr %r3, align 4, !tbaa !8
-  call void @llvm.lifetime.end.p0(i64 4, ptr %e395) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %e287) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %e179) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %e071) #2
+  call void @llvm.lifetime.end.p0(ptr %e395) #2
+  call void @llvm.lifetime.end.p0(ptr %e287) #2
+  call void @llvm.lifetime.end.p0(ptr %e179) #2
+  call void @llvm.lifetime.end.p0(ptr %e071) #2
   %83 = load i32, ptr %r0, align 4, !tbaa !8
   %call = call i32 @twoabs(i32 noundef %83)
   %84 = load i32, ptr %r1, align 4, !tbaa !8
@@ -432,20 +432,20 @@ for.end116:                                       ; preds = %for.cond.cleanup69
   %shr = lshr i32 %90, 16
   %rdd119 = add i32 %conv118, %shr
   %shr120 = lshr i32 %rdd119, 1
-  call void @llvm.lifetime.end.p0(i64 4, ptr %sum) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %r3) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %r2) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %r1) #2
-  call void @llvm.lifetime.end.p0(i64 4, ptr %r0) #2
-  call void @llvm.lifetime.end.p0(i64 64, ptr %emp) #2
+  call void @llvm.lifetime.end.p0(ptr %sum) #2
+  call void @llvm.lifetime.end.p0(ptr %r3) #2
+  call void @llvm.lifetime.end.p0(ptr %r2) #2
+  call void @llvm.lifetime.end.p0(ptr %r1) #2
+  call void @llvm.lifetime.end.p0(ptr %r0) #2
+  call void @llvm.lifetime.end.p0(ptr %emp) #2
   ret i32 %shr120
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define internal i32 @twoabs(i32 noundef %r) #0 {
@@ -453,7 +453,7 @@ entry:
   %r.addr = alloca i32, align 4
   %s = alloca i32, align 4
   store i32 %r, ptr %r.addr, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %s) #2
+  call void @llvm.lifetime.start.p0(ptr %s) #2
   %0 = load i32, ptr %r.addr, align 4, !tbaa !8
   %shr = lshr i32 %0, 15
   %rnd = and i32 %shr, 65537
@@ -464,7 +464,7 @@ entry:
   %rdd = add i32 %1, %2
   %3 = load i32, ptr %s, align 4, !tbaa !8
   %xor = xor i32 %rdd, %3
-  call void @llvm.lifetime.end.p0(i64 4, ptr %s) #2
+  call void @llvm.lifetime.end.p0(ptr %s) #2
   ret i32 %xor
 }
 
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll
new file mode 100644
index 0000000000000..0ef21adff74a6
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -O3 -S %s | FileCheck %s
+
+target triple = "arm64-apple-macosx15.0.0"
+
+define i64 @std_find_i16_constant_offset_with_assumptions(ptr %first.coerce, i16 noundef signext %s) nofree nosync {
+; CHECK-LABEL: define i64 @std_find_i16_constant_offset_with_assumptions(
+; CHECK-SAME: ptr [[FIRST_COERCE:%.*]], i16 noundef signext [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[FIRST_COERCE]], i64 2) ]
+; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[FIRST_COERCE]], i64 256) ]
+; CHECK-NEXT:    [[COERCE_VAL_IP:%.*]] = getelementptr i8, ptr [[FIRST_COERCE]], i64 256
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1
+; CHECK-NEXT:    [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[FIRST_COERCE]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[NEXT_GEP]], align 2
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <8 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i1> [[TMP0]] to i8
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i8 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
+; CHECK-NEXT:    [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_SPLIT]]:
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[RETURN:.*]]
+; CHECK:       [[VECTOR_EARLY_EXIT]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP0]], i1 true)
+; CHECK-NEXT:    [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP6]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[FIRST_COERCE]], i64 [[TMP7]]
+; CHECK-NEXT:    br label %[[RETURN]]
+; CHECK:       [[RETURN]]:
+; CHECK-NEXT:    [[__FIRST_ADDR_0_LCSSA_I_I_PH:%.*]] = phi ptr [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ], [ [[COERCE_VAL_IP]], %[[MIDDLE_SPLIT]] ]
+; CHECK-NEXT:    [[DOTPRE:%.*]] = ptrtoint ptr [[__FIRST_ADDR_0_LCSSA_I_I_PH]] to i64
+; CHECK-NEXT:    ret i64 [[DOTPRE]]
+;
+entry:
+  %first = alloca { ptr }, align 8
+  %s.addr = alloca i16, align 2
+  store ptr %first.coerce, ptr %first, align 8
+  store i16 %s, ptr %s.addr, align 2
+  %0 = load ptr, ptr %first, align 8
+  call void @llvm.assume(i1 true) [ "align"(ptr %0, i64 2) ]
+  call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %0, i64 256) ]
+  %start.ptr = load ptr, ptr %first, align 8
+  %1 = load i64, ptr %first, align 8
+  %coerce.val.pi.i = add i64 %1, 256
+  %coerce.val.ip = inttoptr i64 %coerce.val.pi.i to ptr
+  %cmp.not6.i.i = icmp eq ptr %start.ptr, %coerce.val.ip
+  br i1 %cmp.not6.i.i, label %return, label %loop.ph
+
+loop.ph:
+  %2 = load i16, ptr %s.addr, align 2
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %start.ptr, %loop.ph ], [ %ptr.iv.next, %loop.latch ]
+  %3 = load i16, ptr %ptr.iv, align 2
+  %cmp2.i.i = icmp eq i16 %3, %2
+  br i1 %cmp2.i.i, label %return, label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 2
+  %cmp.not.i.i = icmp eq ptr %ptr.iv.next, %coerce.val.ip
+  br i1 %cmp.not.i.i, label %return, label %loop.header
+
+return:
+  %merge = phi ptr [ %start.ptr, %entry ], [ %coerce.val.ip, %loop.latch ], [ %ptr.iv, %loop.header ]
+  %res = ptrtoint ptr %merge to i64
+  ret i64 %res
+}
+
+define i64 @std_find_i16_constant_offset_no_assumptions(ptr %first.coerce, i16 noundef signext %s) nofree nosync {
+; CHECK-LABEL: define i64 @std_find_i16_constant_offset_no_assumptions(
+; CHECK-SAME: ptr [[FIRST_COERCE:%.*]], i16 noundef signext [[S:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[COERCE_VAL_IP:%.*]] = getelementptr i8, ptr [[FIRST_COERCE]], i64 256
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[PTR_IV:%.*]] = phi ptr [ [[FIRST_COERCE]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, ptr [[PTR_IV]], align 2
+; CHECK-NEXT:    [[CMP2_I_I:%.*]] = icmp eq i16 [[TMP1]], [[S]]
+; CHECK-NEXT:    br i1 [[CMP2_I_I]], label %[[RETURN:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 2
+; CHECK-NEXT:    [[CMP_NOT_I_I:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[COERCE_VAL_IP]]
+; CHECK-NEXT:    br i1 [[CMP_NOT_I_I]], label %[[RETURN]], label %[[LOOP_HEADER]]
+; CHECK:       [[RETURN]]:
+; CHECK-NEXT:    [[MERGE_PH:%.*]] = phi ptr [ [[COERCE_VAL_IP]], %[[LOOP_LATCH]] ], [ [[PTR_IV]], %[[LOOP_HEADER]] ]
+; CHECK-NEXT:    [[DOTPRE:%.*]] = ptrtoint ptr [[MERGE_PH]] to i64
+; CHECK-NEXT:    ret i64 [[DOTPRE]]
+;
+entry:
+  %first = alloca { ptr }, align 8
+  %s.addr = alloca i16, align 2
+  store ptr %first.coerce, ptr %first, align 8
+  store i16 %s, ptr %s.addr, align 2
+  %0 = load ptr, ptr %first, align 8
+  %start.ptr = load ptr, ptr %first, align 8
+  %1 = load i64, ptr %first, align 8
+  %coerce.val.pi.i = add i64 %1, 256
+  %coerce.val.ip = inttoptr i64 %coerce.val.pi.i to ptr
+  %cmp.not6.i.i = icmp eq ptr %start.ptr, %coerce.val.ip
+  br i1 %cmp.not6.i.i, label %return, label %loop.ph
+
+loop.ph:
+  %2 = load i16, ptr %s.addr, align 2
+  br label %loop.header
+
+loop.header:
+  %ptr.iv = phi ptr [ %start.ptr, %loop.ph ], [ %ptr.iv.next, %loop.latch ]
+  %3 = load i16, ptr %ptr.iv, align 2
+  %cmp2.i.i = icmp eq i16 %3, %2
+  br i1 %cmp2.i.i, label %return, label %loop.latch
+
+loop.latch:
+  %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 2
+  %cmp.not.i.i = icmp eq ptr %ptr.iv.next, %coerce.val.ip
+  br i1 %cmp.not.i.i, label %return, label %loop.header
+
+return:
+  %merge = phi ptr [ %start.ptr, %entry ], [ %coerce.val.ip, %loop.latch ], [ %ptr.iv, %loop.header ]
+  %res = ptrtoint ptr %merge to i64
+  ret i64 %res
+}
+
+declare void @llvm.assume(i1 noundef)
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+;.
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll
index f40afbda1246a..cd7ca1e22b499 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll
@@ -16,7 +16,7 @@ define void @interleave_deinterleave(ptr noalias %dst, ptr %a, ptr %b) {
 ; CHECK-LABEL: @interleave_deinterleave(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
@@ -24,8 +24,6 @@ define void @interleave_deinterleave(ptr noalias %dst, ptr %a, ptr %b) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -47,7 +45,7 @@ define void @interleave_deinterleave(ptr noalias %dst, ptr %a, ptr %b) {
 ; CHECK-NEXT:    [[TMP23:%.*]] = shl <vscale x 4 x i32> [[TMP11]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP24:%.*]] = ashr <vscale x 4 x i32> [[TMP12]], [[TMP19]]
 ; CHECK-NEXT:    call void @llvm.aarch64.sve.st4.nxv4i32(<vscale x 4 x i32> [[TMP20]], <vscale x 4 x i32> [[TMP22]], <vscale x 4 x i32> [[TMP23]], <vscale x 4 x i32> [[TMP24]], <vscale x 4 x i1> splat (i1 true), ptr [[TMP21]])
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/udotabd.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/udotabd.ll
index 3496520c232aa..0967736b6740a 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/udotabd.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/udotabd.ll
@@ -379,9 +379,9 @@ entry:
   store i32 %s_p1, ptr %s_p1.addr, align 4, !tbaa !9
   store ptr %p2, ptr %p2.addr, align 8, !tbaa !4
   store i32 %s_p2, ptr %s_p2.addr, align 4, !tbaa !9
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i_sum) #3
+  call void @llvm.lifetime.start.p0(ptr %i_sum) #3
   store i32 0, ptr %i_sum, align 4, !tbaa !9
-  call void @llvm.lifetime.start.p0(i64 4, ptr %y) #3
+  call void @llvm.lifetime.start.p0(ptr %y) #3
   store i32 0, ptr %y, align 4, !tbaa !9
   br label %for.cond
 
@@ -392,11 +392,11 @@ for.cond:                                         ; preds = %for.inc10, %entry
 
 for.cond.cleanup:                                 ; preds = %for.cond
   store i32 2, ptr %cleanup.dest.slot, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %y) #3
+  call void @llvm.lifetime.end.p0(ptr %y) #3
   br label %for.end12
 
 for.body:                                         ; preds = %for.cond
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x) #3
+  call void @llvm.lifetime.start.p0(ptr %x) #3
   store i32 0, ptr %x, align 4, !tbaa !9
   br label %for.cond1
 
@@ -407,7 +407,7 @@ for.cond1:                                        ; preds = %for.inc, %for.body
 
 for.cond.cleanup3:                                ; preds = %for.cond1
   store i32 5, ptr %cleanup.dest.slot, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x) #3
+  call void @llvm.lifetime.end.p0(ptr %x) #3
   br label %for.end
 
 for.body4:                                        ; preds = %for.cond1
@@ -458,18 +458,18 @@ for.inc10:                                        ; preds = %for.end
 for.end12:                                        ; preds = %for.cond.cleanup
   %16 = load i32, ptr %i_sum, align 4, !tbaa !9
   store i32 1, ptr %cleanup.dest.slot, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i_sum) #3
+  call void @llvm.lifetime.end.p0(ptr %i_sum) #3
   ret i32 %16
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr) #1
+declare void @llvm.lifetime.start.p0(ptr) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
 declare i32 @llvm.abs.i32(i32, i1 immarg) #2
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr) #1
+declare void @llvm.lifetime.end.p0(ptr) #1
 
 attributes #0 = { nounwind uwtable vscale_range(1,16) "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+bf16,+bti,+ccidx,+complxnum,+crc,+dit,+dotprod,+ete,+flagm,+fp-armv8,+fp16fml,+fullfp16,+i8mm,+jsconv,+lse,+mte,+neon,+pauth,+perfmon,+predres,+rand,+ras,+rcpc,+rdm,+sb,+spe,+ssbs,+sve,+sve-bitperm,+sve2,+trbe,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8a,+v9a,-fmv" }
 attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_add_q7.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_add_q7.ll
index 76d9d14b7d14f..0023dea2149b5 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_add_q7.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_add_q7.ll
@@ -43,7 +43,7 @@ entry:
   store ptr %pSrcB, ptr %pSrcB.addr, align 4
   store ptr %pDst, ptr %pDst.addr, align 4
   store i32 %blockSize, ptr %blockSize.addr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %blkCnt)
+  call void @llvm.lifetime.start.p0(ptr %blkCnt)
   %0 = load i32, ptr %blockSize.addr, align 4
   store i32 %0, ptr %blkCnt, align 4
   br label %while.cond
@@ -78,7 +78,7 @@ while.body:                                       ; preds = %while.cond
   br label %while.cond
 
 while.end:                                        ; preds = %while.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %blkCnt)
+  call void @llvm.lifetime.end.p0(ptr %blkCnt)
   ret void
 }
 
@@ -102,13 +102,13 @@ land.lhs.true:                                    ; preds = %entry
   br i1 %cmp1, label %if.then, label %if.end10
 
 if.then:                                          ; preds = %land.lhs.true
-  call void @llvm.lifetime.start.p0(i64 4, ptr %max)
+  call void @llvm.lifetime.start.p0(ptr %max)
   %2 = load i32, ptr %sat.addr, align 4
   %sub = sub i32 %2, 1
   %shl = shl i32 1, %sub
   %sub2 = sub i32 %shl, 1
   store i32 %sub2, ptr %max, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %min)
+  call void @llvm.lifetime.start.p0(ptr %min)
   %3 = load i32, ptr %max, align 4
   %sub3 = sub nsw i32 -1, %3
   store i32 %sub3, ptr %min, align 4
@@ -143,8 +143,8 @@ if.end8:                                          ; preds = %if.end
   br label %cleanup
 
 cleanup:                                          ; preds = %if.end8, %if.then7, %if.then5
-  call void @llvm.lifetime.end.p0(i64 4, ptr %min)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %max)
+  call void @llvm.lifetime.end.p0(ptr %min)
+  call void @llvm.lifetime.end.p0(ptr %max)
   %cleanup.dest = load i32, ptr %cleanup.dest.slot, align 4
   switch i32 %cleanup.dest, label %unreachable [
   i32 0, label %cleanup.cont
@@ -167,8 +167,8 @@ unreachable:                                      ; preds = %cleanup
   unreachable
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp,-aes,-bf16,-cdecp0,-cdecp1,-cdecp2,-cdecp3,-cdecp4,-cdecp5,-cdecp6,-cdecp7,-crc,-crypto,-d32,-dotprod,-fp-armv8,-fp-armv8sp,-fp16fml,-hwdiv-arm,-i8mm,-neon,-sb,-sha2,-vfp3,-vfp3sp,-vfp4,-vfp4sp" "unsafe-fp-math"="true" }
 attributes #1 = { alwaysinline nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp,-aes,-bf16,-cdecp0,-cdecp1,-cdecp2,-cdecp3,-cdecp4,-cdecp5,-cdecp6,-cdecp7,-crc,-crypto,-d32,-dotprod,-fp-armv8,-fp-armv8sp,-fp16fml,-hwdiv-arm,-i8mm,-neon,-sb,-sha2,-vfp3,-vfp3sp,-vfp4,-vfp4sp" "unsafe-fp-math"="true" }
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_fill_q7.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_fill_q7.ll
index 2ab6f2bdca8d6..436f848a94366 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_fill_q7.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_fill_q7.ll
@@ -59,8 +59,8 @@ entry:
   store i8 %value, ptr %value.addr, align 1, !tbaa !3
   store ptr %pDst, ptr %pDst.addr, align 4, !tbaa !6
   store i32 %blockSize, ptr %blockSize.addr, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %blkCnt) #3
-  call void @llvm.lifetime.start.p0(i64 4, ptr %packedValue) #3
+  call void @llvm.lifetime.start.p0(ptr %blkCnt) #3
+  call void @llvm.lifetime.start.p0(ptr %packedValue) #3
   %0 = load i8, ptr %value.addr, align 1, !tbaa !3
   %conv = sext i8 %0 to i32
   %shl = shl i32 %conv, 0
@@ -122,13 +122,13 @@ while.body16:                                     ; preds = %while.cond13
   br label %while.cond13, !llvm.loop !12
 
 while.end18:                                      ; preds = %while.cond13
-  call void @llvm.lifetime.end.p0(i64 4, ptr %packedValue) #3
-  call void @llvm.lifetime.end.p0(i64 4, ptr %blkCnt) #3
+  call void @llvm.lifetime.end.p0(ptr %packedValue) #3
+  call void @llvm.lifetime.end.p0(ptr %blkCnt) #3
   ret void
 }
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: alwaysinline nounwind
 define internal void @write_q7x4_ia(ptr %pQ7, i32 %value) #2 {
@@ -138,7 +138,7 @@ entry:
   %val = alloca i32, align 4
   store ptr %pQ7, ptr %pQ7.addr, align 4, !tbaa !6
   store i32 %value, ptr %value.addr, align 4, !tbaa !8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %val) #3
+  call void @llvm.lifetime.start.p0(ptr %val) #3
   %0 = load i32, ptr %value.addr, align 4, !tbaa !8
   store i32 %0, ptr %val, align 4, !tbaa !8
   %1 = load i32, ptr %val, align 4, !tbaa !8
@@ -175,12 +175,12 @@ entry:
   %14 = load ptr, ptr %13, align 4, !tbaa !6
   %add.ptr = getelementptr inbounds i8, ptr %14, i32 4
   store ptr %add.ptr, ptr %13, align 4, !tbaa !6
-  call void @llvm.lifetime.end.p0(i64 4, ptr %val) #3
+  call void @llvm.lifetime.end.p0(ptr %val) #3
   ret void
 }
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m0plus" "target-features"="+armv6-m,+strict-align,+thumb-mode,-aes,-bf16,-cdecp0,-cdecp1,-cdecp2,-cdecp3,-cdecp4,-cdecp5,-cdecp6,-cdecp7,-crc,-crypto,-dotprod,-dsp,-fp16fml,-fullfp16,-hwdiv,-hwdiv-arm,-i8mm,-lob,-mve,-mve.fp,-ras,-sb,-sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
index b932a69575cfe..6862d8b33faa1 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
@@ -60,9 +60,9 @@ entry:
   store ptr %pSrc, ptr %pSrc.addr, align 4
   store i32 %blockSize, ptr %blockSize.addr, align 4
   store ptr %pResult, ptr %pResult.addr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %blkCnt) #3
-  call void @llvm.lifetime.start.p0(i64 16, ptr %vecSrc) #3
-  call void @llvm.lifetime.start.p0(i64 4, ptr %sum) #3
+  call void @llvm.lifetime.start.p0(ptr %blkCnt) #3
+  call void @llvm.lifetime.start.p0(ptr %vecSrc) #3
+  call void @llvm.lifetime.start.p0(ptr %sum) #3
   store i32 0, ptr %sum, align 4
   %0 = load i32, ptr %blockSize.addr, align 4
   %shr = lshr i32 %0, 4
@@ -123,15 +123,15 @@ while.end5:                                       ; preds = %while.cond1
   %conv6 = trunc i32 %div to i8
   %18 = load ptr, ptr %pResult.addr, align 4
   store i8 %conv6, ptr %18, align 1
-  call void @llvm.lifetime.end.p0(i64 4, ptr %sum) #3
-  call void @llvm.lifetime.end.p0(i64 16, ptr %vecSrc) #3
-  call void @llvm.lifetime.end.p0(i64 4, ptr %blkCnt) #3
+  call void @llvm.lifetime.end.p0(ptr %sum) #3
+  call void @llvm.lifetime.end.p0(ptr %vecSrc) #3
+  call void @llvm.lifetime.end.p0(ptr %blkCnt) #3
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 declare i32 @llvm.arm.mve.addv.v16i8(<16 x i8>, i32) #2
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind "approx-func-fp-math"="true" "frame-pointer"="all" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp,-aes,-bf16,-cdecp0,-cdecp1,-cdecp2,-cdecp3,-cdecp4,-cdecp5,-cdecp6,-cdecp7,-crc,-crypto,-d32,-dotprod,-fp-armv8,-fp-armv8sp,-fp16fml,-hwdiv-arm,-i8mm,-neon,-pacbti,-sb,-sha2,-vfp3,-vfp3sp,-vfp4,-vfp4sp" "unsafe-fp-math"="true" }
 attributes #1 = { argmemonly nocallback nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
index 9d613b8fe456d..42fdafbf3bec5 100644
--- a/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
+++ b/llvm/test/Transforms/PhaseOrdering/ARM/arm_mult_q15.ll
@@ -88,7 +88,7 @@ entry:
   store ptr %pSrcB, ptr %pSrcB.addr, align 4
   store ptr %pDst, ptr %pDst.addr, align 4
   store i32 %blockSize, ptr %blockSize.addr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %blkCnt) #3
+  call void @llvm.lifetime.start.p0(ptr %blkCnt) #3
   %0 = load i32, ptr %blockSize.addr, align 4
   store i32 %0, ptr %blkCnt, align 4
   br label %while.cond
@@ -123,11 +123,11 @@ while.body:                                       ; preds = %while.cond
   br label %while.cond
 
 while.end:                                        ; preds = %while.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %blkCnt) #3
+  call void @llvm.lifetime.end.p0(ptr %blkCnt) #3
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 define internal i32 @__SSAT(i32 %val, i32 %sat) #2 {
 entry:
@@ -149,13 +149,13 @@ land.lhs.true:                                    ; preds = %entry
   br i1 %cmp1, label %if.then, label %if.end10
 
 if.then:                                          ; preds = %land.lhs.true
-  call void @llvm.lifetime.start.p0(i64 4, ptr %max) #3
+  call void @llvm.lifetime.start.p0(ptr %max) #3
   %2 = load i32, ptr %sat.addr, align 4
   %sub = sub i32 %2, 1
   %shl = shl i32 1, %sub
   %sub2 = sub i32 %shl, 1
   store i32 %sub2, ptr %max, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %min) #3
+  call void @llvm.lifetime.start.p0(ptr %min) #3
   %3 = load i32, ptr %max, align 4
   %sub3 = sub nsw i32 -1, %3
   store i32 %sub3, ptr %min, align 4
@@ -190,8 +190,8 @@ if.end8:                                          ; preds = %if.end
   br label %cleanup
 
 cleanup:                                          ; preds = %if.end8, %if.then7, %if.then5
-  call void @llvm.lifetime.end.p0(i64 4, ptr %min) #3
-  call void @llvm.lifetime.end.p0(i64 4, ptr %max) #3
+  call void @llvm.lifetime.end.p0(ptr %min) #3
+  call void @llvm.lifetime.end.p0(ptr %max) #3
   %cleanup.dest = load i32, ptr %cleanup.dest.slot, align 4
   switch i32 %cleanup.dest, label %unreachable [
   i32 0, label %cleanup.cont
@@ -214,7 +214,7 @@ unreachable:                                      ; preds = %cleanup
   unreachable
 }
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+ras,+strict-align,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp,-aes,-bf16,-cdecp0,-cdecp1,-cdecp2,-cdecp3,-cdecp4,-cdecp5,-cdecp6,-cdecp7,-crc,-crypto,-d32,-dotprod,-fp-armv8,-fp-armv8sp,-fp16fml,-hwdiv-arm,-i8mm,-neon,-sb,-sha2,-vfp3,-vfp3sp,-vfp4,-vfp4sp" "unsafe-fp-math"="true" }
 attributes #1 = { argmemonly nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/SROA-after-final-loop-unrolling-2.ll b/llvm/test/Transforms/PhaseOrdering/X86/SROA-after-final-loop-unrolling-2.ll
index 5178e9f1c8e42..7fe3f33430234 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/SROA-after-final-loop-unrolling-2.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/SROA-after-final-loop-unrolling-2.ll
@@ -60,9 +60,9 @@ bb:
   %i5 = alloca ptr, align 8
   store i32 %arg, ptr %i, align 4, !tbaa !5
   store ptr %arg1, ptr %i2, align 8, !tbaa !9
-  call void @llvm.lifetime.start.p0(i64 8, ptr %i3) #3
+  call void @llvm.lifetime.start.p0(ptr %i3) #3
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %i3, ptr align 4 @global, i64 8, i1 false)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i4) #3
+  call void @llvm.lifetime.start.p0(ptr %i4) #3
   store i32 0, ptr %i4, align 4, !tbaa !5
   br label %bb6
 
@@ -75,11 +75,11 @@ bb6:                                              ; preds = %bb22, %bb
   br i1 %i11, label %bb13, label %bb12
 
 bb12:                                             ; preds = %bb6
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i4) #3
+  call void @llvm.lifetime.end.p0(ptr %i4) #3
   br label %bb25
 
 bb13:                                             ; preds = %bb6
-  call void @llvm.lifetime.start.p0(i64 8, ptr %i5) #3
+  call void @llvm.lifetime.start.p0(ptr %i5) #3
   %i14 = load i32, ptr %i4, align 4, !tbaa !5
   %i15 = srem i32 %i14, 2
   %i16 = sext i32 %i15 to i64
@@ -90,7 +90,7 @@ bb13:                                             ; preds = %bb6
   %i20 = load i32, ptr %i19, align 4, !tbaa !5
   %i21 = mul nsw i32 %i20, %i18
   store i32 %i21, ptr %i19, align 4, !tbaa !5
-  call void @llvm.lifetime.end.p0(i64 8, ptr %i5) #3
+  call void @llvm.lifetime.end.p0(ptr %i5) #3
   br label %bb22
 
 bb22:                                             ; preds = %bb13
@@ -102,12 +102,12 @@ bb22:                                             ; preds = %bb13
 bb25:                                             ; preds = %bb12
   %i26 = load ptr, ptr %i2, align 8, !tbaa !9
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %i26, ptr align 4 %i3, i64 8, i1 false), !tbaa.struct !13
-  call void @llvm.lifetime.end.p0(i64 8, ptr %i3) #3
+  call void @llvm.lifetime.end.p0(ptr %i3) #3
   ret void
 }
 
 ; Function Attrs: argmemonly nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: argmemonly nocallback nofree nounwind willreturn
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #2
@@ -126,7 +126,7 @@ bb:
 }
 
 ; Function Attrs: argmemonly nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: mustprogress nounwind uwtable
 define linkonce_odr dso_local noundef nonnull align 4 dereferenceable(4) ptr @widget(ptr noundef nonnull align 4 dereferenceable(8) %arg, i64 noundef %arg1) #0 comdat($_ZNSt14__array_traitsIiLm2EE6_S_refERA2_Kim) align 2 {
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/SROA-after-loop-unrolling.ll b/llvm/test/Transforms/PhaseOrdering/X86/SROA-after-loop-unrolling.ll
index c6dc7b374b00f..51f2a364a2c72 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/SROA-after-loop-unrolling.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/SROA-after-loop-unrolling.ll
@@ -50,14 +50,14 @@ entry:
   %__end15 = alloca ptr
   %elt11 = alloca ptr
   store i32 %cnt, ptr %cnt.addr
-  call void @llvm.lifetime.start.p0(i64 24, ptr %arr)
-  call void @llvm.lifetime.start.p0(i64 8, ptr %__range1)
+  call void @llvm.lifetime.start.p0(ptr %arr)
+  call void @llvm.lifetime.start.p0(ptr %__range1)
   store ptr %arr, ptr %__range1
-  call void @llvm.lifetime.start.p0(i64 8, ptr %__begin1)
+  call void @llvm.lifetime.start.p0(ptr %__begin1)
   %0 = load ptr, ptr %__range1
   %call = call ptr @_ZNSt5arrayIiLm6EE5beginEv(ptr %0)
   store ptr %call, ptr %__begin1
-  call void @llvm.lifetime.start.p0(i64 8, ptr %__end1)
+  call void @llvm.lifetime.start.p0(ptr %__end1)
   %1 = load ptr, ptr %__range1
   %call1 = call ptr @_ZNSt5arrayIiLm6EE3endEv(ptr %1)
   store ptr %call1, ptr %__end1
@@ -70,13 +70,13 @@ for.cond:
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:
-  call void @llvm.lifetime.end.p0(i64 8, ptr %__end1)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %__begin1)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %__range1)
+  call void @llvm.lifetime.end.p0(ptr %__end1)
+  call void @llvm.lifetime.end.p0(ptr %__begin1)
+  call void @llvm.lifetime.end.p0(ptr %__range1)
   br label %for.end
 
 for.body:
-  call void @llvm.lifetime.start.p0(i64 8, ptr %elt)
+  call void @llvm.lifetime.start.p0(ptr %elt)
   %4 = load ptr, ptr %__begin1
   store ptr %4, ptr %elt
   %5 = load i32, ptr %cnt.addr
@@ -84,7 +84,7 @@ for.body:
   store i32 %inc, ptr %cnt.addr
   %6 = load ptr, ptr %elt
   store i32 %inc, ptr %6
-  call void @llvm.lifetime.end.p0(i64 8, ptr %elt)
+  call void @llvm.lifetime.end.p0(ptr %elt)
   br label %for.inc
 
 for.inc:
@@ -94,13 +94,13 @@ for.inc:
   br label %for.cond
 
 for.end:
-  call void @llvm.lifetime.start.p0(i64 8, ptr %__range12)
+  call void @llvm.lifetime.start.p0(ptr %__range12)
   store ptr %arr, ptr %__range12
-  call void @llvm.lifetime.start.p0(i64 8, ptr %__begin13)
+  call void @llvm.lifetime.start.p0(ptr %__begin13)
   %8 = load ptr, ptr %__range12
   %call4 = call ptr @_ZNSt5arrayIiLm6EE5beginEv(ptr %8)
   store ptr %call4, ptr %__begin13
-  call void @llvm.lifetime.start.p0(i64 8, ptr %__end15)
+  call void @llvm.lifetime.start.p0(ptr %__end15)
   %9 = load ptr, ptr %__range12
   %call6 = call ptr @_ZNSt5arrayIiLm6EE3endEv(ptr %9)
   store ptr %call6, ptr %__end15
@@ -113,19 +113,19 @@ for.cond7:
   br i1 %cmp8, label %for.body10, label %for.cond.cleanup9
 
 for.cond.cleanup9:
-  call void @llvm.lifetime.end.p0(i64 8, ptr %__end15)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %__begin13)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %__range12)
+  call void @llvm.lifetime.end.p0(ptr %__end15)
+  call void @llvm.lifetime.end.p0(ptr %__begin13)
+  call void @llvm.lifetime.end.p0(ptr %__range12)
   br label %for.end14
 
 for.body10:
-  call void @llvm.lifetime.start.p0(i64 8, ptr %elt11)
+  call void @llvm.lifetime.start.p0(ptr %elt11)
   %12 = load ptr, ptr %__begin13
   store ptr %12, ptr %elt11
   %13 = load ptr, ptr %elt11
   %14 = load i32, ptr %13
   call void @_Z3usei(i32 %14)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %elt11)
+  call void @llvm.lifetime.end.p0(ptr %elt11)
   br label %for.inc12
 
 for.inc12:
@@ -135,11 +135,11 @@ for.inc12:
   br label %for.cond7
 
 for.end14:
-  call void @llvm.lifetime.end.p0(i64 24, ptr %arr)
+  call void @llvm.lifetime.end.p0(ptr %arr)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 define linkonce_odr dso_local ptr @_ZNSt5arrayIiLm6EE5beginEv(ptr %this) {
 entry:
@@ -160,7 +160,7 @@ entry:
   ret ptr %add.ptr
 }
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare dso_local void @_Z3usei(i32)
 
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll b/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll
index 798df4cd4ff54..63f8250b5f3de 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/hadd.ll
@@ -404,13 +404,13 @@ define <16 x i16> @add_v16i16_FEuCBA98765432u0(<16 x i16> %a, <16 x i16> %b) {
 ; SSE4-LABEL: @add_v16i16_FEuCBA98765432u0(
 ; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 poison, i32 11, i32 12, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP10]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 26, i32 29, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 11, i32 12, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP10]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 26, i32 29, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP6:%.*]] = add <16 x i16> [[TMP4]], [[TMP5]]
-; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 poison, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 poison, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP9:%.*]] = add <16 x i16> [[TMP7]], [[TMP8]]
-; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP9]], <16 x i16> [[TMP6]], <16 x i32> <i32 3, i32 2, i32 poison, i32 1, i32 0, i32 27, i32 26, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
+; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP9]], <16 x i16> [[TMP6]], <16 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
 ; SSE4-NEXT:    ret <16 x i16> [[RESULT]]
 ;
 ; AVX2-LABEL: @add_v16i16_FEuCBA98765432u0(
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll b/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll
index dfad53411aa55..00453e701ee51 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/hoist-load-of-baseptr.ll
@@ -143,7 +143,7 @@ entry:
   %j = alloca i64, align 8
   store ptr %data, ptr %data.addr, align 8, !tbaa !3
   store i64 %numElems, ptr %numElems.addr, align 8, !tbaa !7
-  call void @llvm.lifetime.start.p0(i64 8, ptr %i)
+  call void @llvm.lifetime.start.p0(ptr %i)
   store i64 0, ptr %i, align 8, !tbaa !7
   br label %for.cond
 
@@ -154,11 +154,11 @@ for.cond:
 
 for.cond.cleanup:
   store i32 2, ptr %cleanup.dest.slot, align 4
-  call void @llvm.lifetime.end.p0(i64 8, ptr %i)
+  call void @llvm.lifetime.end.p0(ptr %i)
   br label %for.end8
 
 for.body:
-  call void @llvm.lifetime.start.p0(i64 8, ptr %j)
+  call void @llvm.lifetime.start.p0(ptr %j)
   store i64 0, ptr %j, align 8, !tbaa !7
   br label %for.cond1
 
@@ -170,7 +170,7 @@ for.cond1:
 
 for.cond.cleanup3:
   store i32 5, ptr %cleanup.dest.slot, align 4
-  call void @llvm.lifetime.end.p0(i64 8, ptr %j)
+  call void @llvm.lifetime.end.p0(ptr %j)
   br label %for.end
 
 for.body4:
@@ -201,7 +201,7 @@ for.end8:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 define linkonce_odr dso_local noundef nonnull align 4 dereferenceable(4) ptr @_ZNSt6vectorIiSaIiEEixEm(ptr noundef nonnull align 8 dereferenceable(24) %this, i64 noundef %__n) comdat align 2 {
 entry:
@@ -217,7 +217,7 @@ entry:
   ret ptr %add.ptr
 }
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{i32 7, !"uwtable", i32 2}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll b/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll
index fd160b7c57024..bbfe844400b0c 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/hsub.ll
@@ -398,13 +398,13 @@ define <16 x i16> @sub_v16i16_FEuCBA98765432u0(<16 x i16> %a, <16 x i16> %b) {
 ; SSE4-LABEL: @sub_v16i16_FEuCBA98765432u0(
 ; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP10:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 25, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 poison, i32 10, i32 12, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP10]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 poison, i32 27, i32 29, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <16 x i16> [[TMP10]], <16 x i16> [[A]], <16 x i32> <i32 0, i32 poison, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 27, i32 29, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP6:%.*]] = sub <16 x i16> [[TMP4]], [[TMP5]]
-; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP7:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 14, i32 24, i32 poison, i32 28, i32 30, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 15, i32 25, i32 poison, i32 29, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; SSE4-NEXT:    [[TMP9:%.*]] = sub <16 x i16> [[TMP7]], [[TMP8]]
-; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP9]], <16 x i16> [[TMP6]], <16 x i32> <i32 3, i32 2, i32 poison, i32 1, i32 0, i32 27, i32 26, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
+; SSE4-NEXT:    [[RESULT:%.*]] = shufflevector <16 x i16> [[TMP9]], <16 x i16> [[TMP6]], <16 x i32> <i32 4, i32 3, i32 poison, i32 1, i32 0, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 poison, i32 16>
 ; SSE4-NEXT:    ret <16 x i16> [[RESULT]]
 ;
 ; AVX2-LABEL: @sub_v16i16_FEuCBA98765432u0(
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/loop-vectorizer-noalias.ll b/llvm/test/Transforms/PhaseOrdering/X86/loop-vectorizer-noalias.ll
index 1a4af5ab3cb4b..3c54ed9bb37ba 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/loop-vectorizer-noalias.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/loop-vectorizer-noalias.ll
@@ -63,10 +63,10 @@ for.end:                                          ; preds = %for.cond.cleanup
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87"}
 ;.
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/masked-memory-ops-with-cf.ll b/llvm/test/Transforms/PhaseOrdering/X86/masked-memory-ops-with-cf.ll
index 405a26de3d6af..c649f29effeda 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/masked-memory-ops-with-cf.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/masked-memory-ops-with-cf.ll
@@ -13,7 +13,7 @@ define void @basic(i1 %cond, ptr %b, ptr %p, ptr %q) {
 ; CHECK-NEXT:    [[TMP5:%.*]] = call <1 x i64> @llvm.masked.load.v1i64.p0(ptr [[B:%.*]], i32 8, <1 x i1> [[TMP0]], <1 x i64> poison)
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast <1 x i64> [[TMP5]] to i64
 ; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i16 [[TMP2]] to <1 x i16>
-; CHECK-NEXT:    call void @llvm.masked.store.v1i16.p0(<1 x i16> [[TMP7]], ptr [[B]], i32 2, <1 x i1> [[TMP0]])
+; CHECK-NEXT:    call void @llvm.masked.store.v1i16.p0(<1 x i16> [[TMP7]], ptr [[B]], i32 8, <1 x i1> [[TMP0]])
 ; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32 [[TMP4]] to <1 x i32>
 ; CHECK-NEXT:    call void @llvm.masked.store.v1i32.p0(<1 x i32> [[TMP8]], ptr [[P]], i32 4, <1 x i1> [[TMP0]])
 ; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i64 [[TMP6]] to <1 x i64>
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/pr61061.ll b/llvm/test/Transforms/PhaseOrdering/X86/pr61061.ll
index 362708bacc20f..0c58705d7f757 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/pr61061.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/pr61061.ll
@@ -13,7 +13,7 @@ define <2 x i64> @PR61061(<2 x i64> noundef %vect) {
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
   %ptr = alloca <2 x i64>, align 16
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %ptr)
+  call void @llvm.lifetime.start.p0(ptr nonnull %ptr)
   %bc0 = bitcast <2 x i64> %vect to <16 x i8>
   %bc1 = bitcast <2 x i64> %vect to <16 x i8>
   %bc2 = bitcast <2 x i64> %vect to <16 x i8>
@@ -62,8 +62,8 @@ define <2 x i64> @PR61061(<2 x i64> noundef %vect) {
   store i8 %elt2, ptr %ptr14, align 2
   store i8 %elt3, ptr %ptr15, align 1
   %base = load <2 x i64>, ptr %ptr, align 16
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %ptr)
+  call void @llvm.lifetime.end.p0(ptr nonnull %ptr)
   ret <2 x i64> %base
 }
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
index be7f4c2a941a0..cb378465e30ec 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/preserve-access-group.ll
@@ -79,9 +79,9 @@ entry:
   store ptr %face_cell, ptr %face_cell.addr, align 8, !tbaa !10
   store ptr %x, ptr %x.addr, align 8, !tbaa !10
   store ptr %y, ptr %y.addr, align 8, !tbaa !10
-  call void @llvm.lifetime.start.p0(i64 4, ptr %il) #3
-  call void @llvm.lifetime.start.p0(i64 4, ptr %ir) #3
-  call void @llvm.lifetime.start.p0(i64 4, ptr %iface) #3
+  call void @llvm.lifetime.start.p0(ptr %il) #3
+  call void @llvm.lifetime.start.p0(ptr %ir) #3
+  call void @llvm.lifetime.start.p0(ptr %iface) #3
   store i32 0, ptr %iface, align 4, !tbaa !6
   br label %for.cond
 
@@ -92,7 +92,7 @@ for.cond:
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:
-  call void @llvm.lifetime.end.p0(i64 4, ptr %iface) #3, !llvm.access.group !12
+  call void @llvm.lifetime.end.p0(ptr %iface) #3, !llvm.access.group !12
   br label %for.end
 
 for.body:
@@ -134,12 +134,12 @@ for.inc:
   br label %for.cond, !llvm.loop !15
 
 for.end:
-  call void @llvm.lifetime.end.p0(i64 4, ptr %ir) #3
-  call void @llvm.lifetime.end.p0(i64 4, ptr %il) #3
+  call void @llvm.lifetime.end.p0(ptr %ir) #3
+  call void @llvm.lifetime.end.p0(ptr %il) #3
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 define linkonce_odr noundef nonnull align 8 dereferenceable(8) ptr @max(ptr noundef nonnull align 8 dereferenceable(8) %__a, ptr noundef nonnull align 8 dereferenceable(8) %__b) #2 {
 entry:
@@ -170,7 +170,7 @@ return:
   ret ptr %6
 }
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { mustprogress "target-cpu" = "skylake-avx512" }
 attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vdiv-nounroll.ll b/llvm/test/Transforms/PhaseOrdering/X86/vdiv-nounroll.ll
index f60bc26c5111c..69a46b26decb2 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vdiv-nounroll.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vdiv-nounroll.ll
@@ -39,7 +39,7 @@ entry:
   %i = alloca i32, align 4
   store ptr %a, ptr %a.addr, align 8, !tbaa !3
   store float %b, ptr %b.addr, align 4, !tbaa !7
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.start.p0(ptr %i) #2
   store i32 0, ptr %i, align 4, !tbaa !9
   br label %for.cond
 
@@ -49,7 +49,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2
+  call void @llvm.lifetime.end.p0(ptr %i) #2
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -73,8 +73,8 @@ for.end:                                          ; preds = %for.cond.cleanup
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind ssp uwtable "frame-pointer"="all" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" "unsafe-fp-math"="true" }
 attributes #1 = { argmemonly nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll b/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
index 85f6fceb5bdbe..86bdd125e5e57 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll
@@ -11,13 +11,13 @@ $getAt = comdat any
 
 define dso_local noundef <4 x float> @ConvertVectors_ByRef(ptr noundef nonnull align 16 dereferenceable(16) %0) #0 {
 ; SSE-LABEL: @ConvertVectors_ByRef(
-; SSE-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[TMP0:%.*]], align 16
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; SSE-NEXT:    [[TMP2:%.*]] = load <3 x float>, ptr [[TMP0:%.*]], align 16
+; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
 ; SSE-NEXT:    ret <4 x float> [[TMP3]]
 ;
 ; AVX-LABEL: @ConvertVectors_ByRef(
-; AVX-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[TMP0:%.*]], align 16
-; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; AVX-NEXT:    [[TMP2:%.*]] = load <3 x float>, ptr [[TMP0:%.*]], align 16
+; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
 ; AVX-NEXT:    ret <4 x float> [[TMP3]]
 ;
   %2 = alloca ptr, align 8
@@ -75,7 +75,7 @@ entry:
   %ref.tmp7 = alloca %union.ElementWiseAccess, align 16
   %ref.tmp12 = alloca %union.ElementWiseAccess, align 16
   store ptr %V, ptr %V.addr, align 8
-  call void @llvm.lifetime.start.p0(i64 16, ptr %ref.tmp) #4
+  call void @llvm.lifetime.start.p0(ptr %ref.tmp) #4
   %0 = load ptr, ptr %V.addr, align 8
   %call = call { double, double } @castToElementWiseAccess_ByVal(ptr noundef nonnull align 16 dereferenceable(16) %0)
   %coerce.dive = getelementptr inbounds %union.ElementWiseAccess, ptr %ref.tmp, i32 0, i32 0
@@ -87,7 +87,7 @@ entry:
   store double %4, ptr %3, align 8
   %call1 = call noundef float @ElementWiseAccess5getAt(ptr noundef nonnull align 16 dereferenceable(16) %ref.tmp, i32 noundef 0)
   %vecinit = insertelement <4 x float> undef, float %call1, i32 0
-  call void @llvm.lifetime.start.p0(i64 16, ptr %ref.tmp2) #4
+  call void @llvm.lifetime.start.p0(ptr %ref.tmp2) #4
   %5 = load ptr, ptr %V.addr, align 8
   %call3 = call { double, double } @castToElementWiseAccess_ByVal(ptr noundef nonnull align 16 dereferenceable(16) %5)
   %coerce.dive4 = getelementptr inbounds %union.ElementWiseAccess, ptr %ref.tmp2, i32 0, i32 0
@@ -99,7 +99,7 @@ entry:
   store double %9, ptr %8, align 8
   %call5 = call noundef float @ElementWiseAccess5getAt(ptr noundef nonnull align 16 dereferenceable(16) %ref.tmp2, i32 noundef 1)
   %vecinit6 = insertelement <4 x float> %vecinit, float %call5, i32 1
-  call void @llvm.lifetime.start.p0(i64 16, ptr %ref.tmp7) #4
+  call void @llvm.lifetime.start.p0(ptr %ref.tmp7) #4
   %10 = load ptr, ptr %V.addr, align 8
   %call8 = call { double, double } @castToElementWiseAccess_ByVal(ptr noundef nonnull align 16 dereferenceable(16) %10)
   %coerce.dive9 = getelementptr inbounds %union.ElementWiseAccess, ptr %ref.tmp7, i32 0, i32 0
@@ -111,7 +111,7 @@ entry:
   store double %14, ptr %13, align 8
   %call10 = call noundef float @ElementWiseAccess5getAt(ptr noundef nonnull align 16 dereferenceable(16) %ref.tmp7, i32 noundef 2)
   %vecinit11 = insertelement <4 x float> %vecinit6, float %call10, i32 2
-  call void @llvm.lifetime.start.p0(i64 16, ptr %ref.tmp12) #4
+  call void @llvm.lifetime.start.p0(ptr %ref.tmp12) #4
   %15 = load ptr, ptr %V.addr, align 8
   %call13 = call { double, double } @castToElementWiseAccess_ByVal(ptr noundef nonnull align 16 dereferenceable(16) %15)
   %coerce.dive14 = getelementptr inbounds %union.ElementWiseAccess, ptr %ref.tmp12, i32 0, i32 0
@@ -125,10 +125,10 @@ entry:
   %vecinit16 = insertelement <4 x float> %vecinit11, float %call15, i32 3
   store <4 x float> %vecinit16, ptr %.compoundliteral, align 16
   %20 = load <4 x float>, ptr %.compoundliteral, align 16
-  call void @llvm.lifetime.end.p0(i64 16, ptr %ref.tmp12) #4
-  call void @llvm.lifetime.end.p0(i64 16, ptr %ref.tmp7) #4
-  call void @llvm.lifetime.end.p0(i64 16, ptr %ref.tmp2) #4
-  call void @llvm.lifetime.end.p0(i64 16, ptr %ref.tmp) #4
+  call void @llvm.lifetime.end.p0(ptr %ref.tmp12) #4
+  call void @llvm.lifetime.end.p0(ptr %ref.tmp7) #4
+  call void @llvm.lifetime.end.p0(ptr %ref.tmp2) #4
+  call void @llvm.lifetime.end.p0(ptr %ref.tmp) #4
   ret <4 x float> %20
 }
 
@@ -144,8 +144,8 @@ entry:
   ret { double, double } %1
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #3
 
 define internal noundef nonnull align 16 dereferenceable(16) ptr @castToElementWiseAccess_ByRef(ptr noundef nonnull align 16 dereferenceable(16) %0) #1 {
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll
index ee51e467c5b8e..f6e5b0b29700b 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-logical.ll
@@ -86,11 +86,11 @@ return:
 define float @test_merge_anyof_v4sf(<4 x float> %t) {
 ; CHECK-LABEL: @test_merge_anyof_v4sf(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = fcmp olt <4 x float> [[T:%.*]], zeroinitializer
+; CHECK-NEXT:    [[T:%.*]] = freeze <4 x float> [[T1:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = fcmp olt <4 x float> [[T]], zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = fcmp ogt <4 x float> [[T]], splat (float 1.000000e+00)
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> [[TMP0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP4:%.*]] = freeze <8 x i1> [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
 ; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i8 [[TMP5]], 0
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[T]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = fadd <4 x float> [[T]], [[SHIFT]]
@@ -400,11 +400,11 @@ return:
 define float @test_merge_anyof_v4si(<4 x i32> %t) {
 ; CHECK-LABEL: @test_merge_anyof_v4si(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt <4 x i32> [[T:%.*]], splat (i32 1)
+; CHECK-NEXT:    [[T:%.*]] = freeze <4 x i32> [[T1:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt <4 x i32> [[T]], splat (i32 1)
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[T]], splat (i32 255)
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> [[TMP0]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    [[TMP4:%.*]] = freeze <8 x i1> [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i1> [[TMP4]] to i8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <8 x i1> [[TMP3]] to i8
 ; CHECK-NEXT:    [[DOTNOT:%.*]] = icmp eq i8 [[TMP5]], 0
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i32> [[T]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> [[T]], [[SHIFT]]
diff --git a/llvm/test/Transforms/PhaseOrdering/assume-explosion.ll b/llvm/test/Transforms/PhaseOrdering/assume-explosion.ll
index 4d0f039299e76..dd9ead4ffde39 100644
--- a/llvm/test/Transforms/PhaseOrdering/assume-explosion.ll
+++ b/llvm/test/Transforms/PhaseOrdering/assume-explosion.ll
@@ -130,8 +130,8 @@ for.end34:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { nounwind ssp uwtable "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+cx8,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "tune-cpu"="generic" }
 attributes #1 = { argmemonly nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/PhaseOrdering/deletion-of-loops-that-became-side-effect-free.ll b/llvm/test/Transforms/PhaseOrdering/deletion-of-loops-that-became-side-effect-free.ll
index 641f216302146..df78f3d8fd54a 100644
--- a/llvm/test/Transforms/PhaseOrdering/deletion-of-loops-that-became-side-effect-free.ll
+++ b/llvm/test/Transforms/PhaseOrdering/deletion-of-loops-that-became-side-effect-free.ll
@@ -96,35 +96,10 @@ while.end:
 }
 
 define dso_local zeroext i1 @is_not_empty_variant3(ptr %p) {
-; O3-LABEL: @is_not_empty_variant3(
-; O3-NEXT:  entry:
-; O3-NEXT:    [[TOBOOL_NOT4_I:%.*]] = icmp ne ptr [[P:%.*]], null
-; O3-NEXT:    ret i1 [[TOBOOL_NOT4_I]]
-;
-; O2-LABEL: @is_not_empty_variant3(
-; O2-NEXT:  entry:
-; O2-NEXT:    [[TOBOOL_NOT4_I:%.*]] = icmp ne ptr [[P:%.*]], null
-; O2-NEXT:    ret i1 [[TOBOOL_NOT4_I]]
-;
-; O1-LABEL: @is_not_empty_variant3(
-; O1-NEXT:  entry:
-; O1-NEXT:    [[TOBOOL_NOT4_I:%.*]] = icmp eq ptr [[P:%.*]], null
-; O1-NEXT:    br i1 [[TOBOOL_NOT4_I]], label [[COUNT_NODES_VARIANT3_EXIT:%.*]], label [[WHILE_BODY_I:%.*]]
-; O1:       while.body.i:
-; O1-NEXT:    [[SIZE_06_I:%.*]] = phi i64 [ [[INC_I:%.*]], [[WHILE_BODY_I]] ], [ 0, [[ENTRY:%.*]] ]
-; O1-NEXT:    [[P_ADDR_05_I:%.*]] = phi ptr [ [[TMP0:%.*]], [[WHILE_BODY_I]] ], [ [[P]], [[ENTRY]] ]
-; O1-NEXT:    [[CMP_I:%.*]] = icmp ne i64 [[SIZE_06_I]], -1
-; O1-NEXT:    call void @llvm.assume(i1 [[CMP_I]])
-; O1-NEXT:    [[TMP0]] = load ptr, ptr [[P_ADDR_05_I]], align 8
-; O1-NEXT:    [[INC_I]] = add i64 [[SIZE_06_I]], 1
-; O1-NEXT:    [[TOBOOL_NOT_I:%.*]] = icmp eq ptr [[TMP0]], null
-; O1-NEXT:    br i1 [[TOBOOL_NOT_I]], label [[COUNT_NODES_VARIANT3_EXIT_LOOPEXIT:%.*]], label [[WHILE_BODY_I]], !llvm.loop [[LOOP0:![0-9]+]]
-; O1:       count_nodes_variant3.exit.loopexit:
-; O1-NEXT:    [[PHI_CMP:%.*]] = icmp ne i64 [[INC_I]], 0
-; O1-NEXT:    br label [[COUNT_NODES_VARIANT3_EXIT]]
-; O1:       count_nodes_variant3.exit:
-; O1-NEXT:    [[SIZE_0_LCSSA_I:%.*]] = phi i1 [ false, [[ENTRY]] ], [ [[PHI_CMP]], [[COUNT_NODES_VARIANT3_EXIT_LOOPEXIT]] ]
-; O1-NEXT:    ret i1 [[SIZE_0_LCSSA_I]]
+; ALL-LABEL: @is_not_empty_variant3(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL_NOT4_I:%.*]] = icmp ne ptr [[P:%.*]], null
+; ALL-NEXT:    ret i1 [[TOBOOL_NOT4_I]]
 ;
 entry:
   %p.addr = alloca ptr, align 8
@@ -182,3 +157,7 @@ declare void @llvm.assume(i1 noundef)
 !1 = !{!"llvm.loop.mustprogress"}
 !2 = distinct !{!2, !1}
 !3 = distinct !{!3, !1}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; O1: {{.*}}
+; O2: {{.*}}
+; O3: {{.*}}
diff --git a/llvm/test/Transforms/PhaseOrdering/dse-ephemeral-value-captures.ll b/llvm/test/Transforms/PhaseOrdering/dse-ephemeral-value-captures.ll
index 1297dbe795834..9a6cad4945742 100644
--- a/llvm/test/Transforms/PhaseOrdering/dse-ephemeral-value-captures.ll
+++ b/llvm/test/Transforms/PhaseOrdering/dse-ephemeral-value-captures.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -passes='function(dse),cgscc(inline),function(sroa,gvn,sccp)' -S %s | FileCheck %s
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare void @llvm.assume(i1 noundef)
 
@@ -35,10 +35,10 @@ define i32 @test() {
 ;
 entry:
   %a = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   store i32 1, ptr %a, align 4
   %res = call i1 @check_cond(ptr %a)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   call void @llvm.assume(i1 %res)
   ret i32 0
 }
diff --git a/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll b/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll
index ae98fe602f3fc..c6d1cbd421436 100644
--- a/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll
+++ b/llvm/test/Transforms/PhaseOrdering/gvn-replacement-vs-hoist.ll
@@ -35,7 +35,7 @@ entry:
   %i = alloca i32, align 4
   store ptr %a, ptr %a.addr, align 8
   store i32 %beam, ptr %beam.addr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i)
+  call void @llvm.lifetime.start.p0(ptr %i)
   store i32 0, ptr %i, align 4
   br label %for.cond
 
@@ -45,7 +45,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i)
+  call void @llvm.lifetime.end.p0(ptr %i)
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -85,6 +85,6 @@ for.end:                                          ; preds = %for.cond.cleanup
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
diff --git a/llvm/test/Transforms/PhaseOrdering/instcombine-sroa-inttoptr.ll b/llvm/test/Transforms/PhaseOrdering/instcombine-sroa-inttoptr.ll
index cc20233a5fefc..84cbad3cf7449 100644
--- a/llvm/test/Transforms/PhaseOrdering/instcombine-sroa-inttoptr.ll
+++ b/llvm/test/Transforms/PhaseOrdering/instcombine-sroa-inttoptr.ll
@@ -73,23 +73,23 @@ bb:
   %i = alloca %0, align 8
   %i1 = alloca %0, align 8
   %i2 = alloca %0, align 8
-  call void @llvm.lifetime.start.p0(i64 24, ptr %i)
+  call void @llvm.lifetime.start.p0(ptr %i)
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %i1, ptr align 8 %arg, i64 24, i1 false)
   call void @_Z3gen1S(ptr sret(%0) align 8 %i, ptr byval(%0) align 8 %i1)
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %i2, ptr align 8 %i, i64 24, i1 false)
   call void @_Z7escape01S(ptr byval(%0) align 8 %i2)
   %i9 = load ptr, ptr %i, align 8
-  call void @llvm.lifetime.end.p0(i64 24, ptr %i)
+  call void @llvm.lifetime.end.p0(ptr %i)
   ret ptr %i9
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
 
 declare dso_local void @_Z7escape01S(ptr byval(%0) align 8)
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define dso_local ptr @_Z3bar1S(ptr byval(%0) align 8 %arg) {
 ; CHECK-LABEL: @_Z3bar1S(
@@ -112,7 +112,7 @@ define dso_local ptr @_Z3bar1S(ptr byval(%0) align 8 %arg) {
 bb:
   %i = alloca %0, align 8
   %i1 = alloca %0, align 8
-  call void @llvm.lifetime.start.p0(i64 24, ptr %i)
+  call void @llvm.lifetime.start.p0(ptr %i)
   call void @llvm.memcpy.p0.p0.i64(ptr align 8 %i1, ptr align 8 %arg, i64 24, i1 false)
   call void @_Z3gen1S(ptr sret(%0) align 8 %i, ptr byval(%0) align 8 %i1)
   %i5 = call i32 @_Z4condv()
@@ -133,7 +133,7 @@ bb10:
 
 bb13:
   %i15 = load ptr, ptr %i, align 8
-  call void @llvm.lifetime.end.p0(i64 24, ptr %i)
+  call void @llvm.lifetime.end.p0(ptr %i)
   ret ptr %i15
 }
 
diff --git a/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll b/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll
index 1239b18c07017..c5dbc42f4be0d 100644
--- a/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll
+++ b/llvm/test/Transforms/PhaseOrdering/lifetime-sanitizer.ll
@@ -7,8 +7,8 @@
 ; RUN: opt < %s -passes='default<O2>' -S | FileCheck %s --check-prefixes=CHECK,OPT
 ; RUN: opt < %s -passes="default<O3>" -S | FileCheck %s --check-prefixes=CHECK,OPT
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @foo(ptr nocapture)
 
 define void @asan() sanitize_address {
@@ -16,8 +16,8 @@ entry:
   ; CHECK-LABEL: @asan(
   %text = alloca i8, align 1
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   ; CHECK: call void @llvm.lifetime.start
   ; CHECK-NEXT: call void @llvm.lifetime.end
 
@@ -31,8 +31,8 @@ entry:
   ; CHECK-LABEL: @hwasan(
   %text = alloca i8, align 1
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   ; CHECK: call void @llvm.lifetime.start
   ; CHECK-NEXT: call void @llvm.lifetime.end
 
@@ -46,8 +46,8 @@ entry:
   ; CHECK-LABEL: @msan(
   %text = alloca i8, align 1
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   ; CHECK: call void @llvm.lifetime.start
   ; CHECK-NEXT: call void @llvm.lifetime.end
 
@@ -61,8 +61,8 @@ entry:
   ; CHECK-LABEL: @no_asan(
   %text = alloca i8, align 1
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %text)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %text)
+  call void @llvm.lifetime.start.p0(ptr %text)
+  call void @llvm.lifetime.end.p0(ptr %text)
   ; OPT-NOT:    call void @llvm.lifetime
   ; NOOPT:      call void @llvm.lifetime.start
   ; NOOPT-NEXT: call void @llvm.lifetime.end
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
index 45f18dd567396..ae0e59169d3e5 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-access-checks.ll
@@ -52,7 +52,7 @@ entry:
   %elems.coerce.fca.1.extract = extractvalue [2 x i64] %elems.coerce, 1
   %elems.coerce.fca.1.gep = getelementptr inbounds [2 x i64], ptr %elems, i64 0, i64 1
   store i64 %elems.coerce.fca.1.extract, ptr %elems.coerce.fca.1.gep, align 8
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %__begin1) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %__begin1) #6
   %0 = load ptr, ptr %elems, align 8
   %__size_.i.i = getelementptr inbounds %"class.std::__1::span", ptr %elems, i64 0, i32 1
   %1 = load i64, ptr %__size_.i.i, align 8
@@ -66,7 +66,7 @@ entry:
   br i1 %cmp.not.i.i.i.i, label %error, label %check.2
 
 check.2:
-  call void @llvm.lifetime.start.p0(i64 24, ptr nonnull %__end1) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %__end1) #6
   %l4 = load ptr, ptr %elems, align 8
   %__size_.i.i4 = getelementptr inbounds %"class.std::__1::span", ptr %elems, i64 0, i32 1
   %l5 = load i64, ptr %__size_.i.i4, align 8
@@ -90,8 +90,8 @@ for.cond:
   br i1 %cmp.i, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %__end1)
-  call void @llvm.lifetime.end.p0(i64 24, ptr nonnull %__begin1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %__end1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %__begin1)
   ret void
 
 for.body:                                         ; preds = %for.cond
@@ -115,11 +115,11 @@ for.latch:
 
 declare void @error()
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare void @use(ptr noundef nonnull align 4 dereferenceable(4))
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 
 ; -------------------------------------------------------------------------
@@ -160,11 +160,11 @@ entry:
   %count = alloca i64, align 8
   %i = alloca i64, align 8
   store ptr %vec, ptr %vec.addr, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %count)
+  call void @llvm.lifetime.start.p0(ptr %count)
   %0 = load ptr, ptr %vec.addr, align 8
   %call = call noundef i64 @alloc(ptr noundef nonnull align 8 dereferenceable(24) %0)
   store i64 %call, ptr %count, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %i)
+  call void @llvm.lifetime.start.p0(ptr %i)
   store i64 0, ptr %i, align 8
   br label %for.cond
 
@@ -175,7 +175,7 @@ for.cond:
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:
-  call void @llvm.lifetime.end.p0(i64 8, ptr %i)
+  call void @llvm.lifetime.end.p0(ptr %i)
   br label %for.end
 
 for.body:
@@ -194,7 +194,7 @@ for.inc:
   br label %for.cond
 
 for.end:
-  call void @llvm.lifetime.end.p0(i64 8, ptr %count) #5
+  call void @llvm.lifetime.end.p0(ptr %count) #5
   ret void
 }
 
@@ -299,11 +299,11 @@ entry:
   %count = alloca i64, align 8
   %i = alloca i64, align 8
   store ptr %vec, ptr %vec.addr, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %count)
+  call void @llvm.lifetime.start.p0(ptr %count)
   %0 = load ptr, ptr %vec.addr, align 8
   %call = call noundef i64 @alloc(ptr noundef nonnull align 8 dereferenceable(24) %0)
   store i64 %call, ptr %count, align 8
-  call void @llvm.lifetime.start.p0(i64 8, ptr %i)
+  call void @llvm.lifetime.start.p0(ptr %i)
   store i64 0, ptr %i, align 8
   br label %for.cond
 
@@ -314,7 +314,7 @@ for.cond:
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:
-  call void @llvm.lifetime.end.p0(i64 8, ptr %i)
+  call void @llvm.lifetime.end.p0(ptr %i)
   br label %for.end
 
 for.body:
@@ -333,7 +333,7 @@ for.inc:
   br label %for.cond
 
 for.end:
-  call void @llvm.lifetime.end.p0(i64 8, ptr %count)
+  call void @llvm.lifetime.end.p0(ptr %count)
   ret void
 }
 
@@ -376,7 +376,7 @@ entry:
   %k = alloca i32, align 4
   store ptr %arr, ptr %arr.addr, align 8
   store i32 %len, ptr %len.addr, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #3
+  call void @llvm.lifetime.start.p0(ptr %i) #3
   store i32 1, ptr %i, align 4
   br label %for.cond
 
@@ -388,11 +388,11 @@ for.cond:                                         ; preds = %for.inc5, %entry
 
 for.cond.cleanup:                                 ; preds = %for.cond
   store i32 2, ptr %cleanup.dest.slot, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #3
+  call void @llvm.lifetime.end.p0(ptr %i) #3
   br label %for.end6
 
 for.body:                                         ; preds = %for.cond
-  call void @llvm.lifetime.start.p0(i64 4, ptr %k) #3
+  call void @llvm.lifetime.start.p0(ptr %k) #3
   %2 = load i32, ptr %i, align 4
   store i32 %2, ptr %k, align 4
   br label %for.cond1
@@ -404,7 +404,7 @@ for.cond1:                                        ; preds = %for.inc, %for.body
 
 for.cond.cleanup3:                                ; preds = %for.cond1
   store i32 5, ptr %cleanup.dest.slot, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr %k) #3
+  call void @llvm.lifetime.end.p0(ptr %k) #3
   br label %for.end
 
 for.body4:                                        ; preds = %for.cond1
diff --git a/llvm/test/Transforms/PhaseOrdering/loop-rotation-vs-common-code-hoisting.ll b/llvm/test/Transforms/PhaseOrdering/loop-rotation-vs-common-code-hoisting.ll
index c6b5e5f3ccae0..5ff57eae3a773 100644
--- a/llvm/test/Transforms/PhaseOrdering/loop-rotation-vs-common-code-hoisting.ll
+++ b/llvm/test/Transforms/PhaseOrdering/loop-rotation-vs-common-code-hoisting.ll
@@ -38,8 +38,8 @@ declare void @f0()
 declare void @f1()
 declare void @f2()
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @_Z4loopi(i32 %width) {
 ; HOIST-LABEL: @_Z4loopi(
@@ -100,7 +100,7 @@ if.then:
   br label %return
 
 if.end:
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i)
+  call void @llvm.lifetime.start.p0(ptr %i)
   store i32 0, ptr %i, align 4
   br label %for.cond
 
@@ -112,7 +112,7 @@ for.cond:
   br i1 %cmp1, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i)
+  call void @llvm.lifetime.end.p0(ptr %i)
   br label %for.end
 
 for.body:
diff --git a/llvm/test/Transforms/PhaseOrdering/vector-select.ll b/llvm/test/Transforms/PhaseOrdering/vector-select.ll
index 1bdd135c4cb21..c228723ee958f 100644
--- a/llvm/test/Transforms/PhaseOrdering/vector-select.ll
+++ b/llvm/test/Transforms/PhaseOrdering/vector-select.ll
@@ -19,9 +19,9 @@ define <3 x float> @PR52631(<3 x float> %a, <3 x float> %b, <3 x i32> %c) {
   store <4 x float> %extractVec1, ptr %b.addr, align 16
   %extractVec3 = shufflevector <3 x i32> %c, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
   store <4 x i32> %extractVec3, ptr %c.addr, align 16
-  call void @llvm.lifetime.start.p0(i64 16, ptr %zero) #2
+  call void @llvm.lifetime.start.p0(ptr %zero) #2
   store <4 x i32> <i32 0, i32 0, i32 0, i32 undef>, ptr %zero, align 16
-  call void @llvm.lifetime.start.p0(i64 16, ptr %mask) #2
+  call void @llvm.lifetime.start.p0(ptr %mask) #2
   %loadVec4 = load <4 x i32>, ptr %zero, align 16
   %extractVec6 = shufflevector <4 x i32> %loadVec4, <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
   %loadVec48 = load <4 x i32>, ptr %c.addr, align 16
@@ -30,7 +30,7 @@ define <3 x float> @PR52631(<3 x float> %a, <3 x float> %b, <3 x i32> %c) {
   %sext = sext <3 x i1> %cmp to <3 x i32>
   %extractVec10 = shufflevector <3 x i32> %sext, <3 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
   store <4 x i32> %extractVec10, ptr %mask, align 16
-  call void @llvm.lifetime.start.p0(i64 16, ptr %res) #2
+  call void @llvm.lifetime.start.p0(ptr %res) #2
   %loadVec413 = load <4 x i32>, ptr %mask, align 16
   %extractVec14 = shufflevector <4 x i32> %loadVec413, <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
   %loadVec416 = load <4 x float>, ptr %b.addr, align 16
@@ -51,9 +51,9 @@ define <3 x float> @PR52631(<3 x float> %a, <3 x float> %b, <3 x i32> %c) {
   %extractVec32 = shufflevector <4 x i32> %loadVec431, <4 x i32> poison, <3 x i32> <i32 0, i32 1, i32 2>
   %or = or <3 x i32> %and29, %extractVec32
   %astype33 = bitcast <3 x i32> %or to <3 x float>
-  call void @llvm.lifetime.end.p0(i64 16, ptr %res) #2
-  call void @llvm.lifetime.end.p0(i64 16, ptr %mask) #2
-  call void @llvm.lifetime.end.p0(i64 16, ptr %zero) #2
+  call void @llvm.lifetime.end.p0(ptr %res) #2
+  call void @llvm.lifetime.end.p0(ptr %mask) #2
+  call void @llvm.lifetime.end.p0(ptr %zero) #2
   ret <3 x float> %astype33
 }
 
@@ -112,5 +112,5 @@ for.end:
   ret <4 x i32> %min.addr.0
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/expand-log.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/expand-log.ll
new file mode 100644
index 0000000000000..3a1d1324d8e45
--- /dev/null
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/expand-log.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=pre-isel-intrinsic-lowering -S < %s | FileCheck %s
+target triple = "aarch64"
+
+define <vscale x 4 x float> @scalable_vec_log(<vscale x 4 x float> %input) {
+; CHECK-LABEL: define <vscale x 4 x float> @scalable_vec_log(
+; CHECK-SAME: <vscale x 4 x float> [[INPUT:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
+; CHECK-NEXT:    br label %[[BB3:.*]]
+; CHECK:       [[BB3]]:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP9:%.*]], %[[BB3]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = phi <vscale x 4 x float> [ [[INPUT]], [[TMP0]] ], [ [[TMP8:%.*]], %[[BB3]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <vscale x 4 x float> [[TMP5]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = call float @llvm.log.f32(float [[TMP6]])
+; CHECK-NEXT:    [[TMP8]] = insertelement <vscale x 4 x float> [[TMP5]], float [[TMP7]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP9]] = add i64 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[TMP9]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[BB11:.*]], label %[[BB3]]
+; CHECK:       [[BB11]]:
+; CHECK-NEXT:    ret <vscale x 4 x float> [[TMP8]]
+;
+  %output = call <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float> %input)
+  ret <vscale x 4 x float> %output
+}
+
+define <4 x float> @fixed_vec_log(<4 x float> %input) {
+; CHECK-LABEL: define <4 x float> @fixed_vec_log(
+; CHECK-SAME: <4 x float> [[INPUT:%.*]]) {
+; CHECK-NEXT:    [[OUTPUT:%.*]] = call <4 x float> @llvm.log.v4f32(<4 x float> [[INPUT]])
+; CHECK-NEXT:    ret <4 x float> [[OUTPUT]]
+;
+  %output = call <4 x float> @llvm.log.v4f32(<4 x float> %input)
+  ret <4 x float> %output
+}
+
+declare <4 x float> @llvm.log.v4f32(<4 x float>) #0
+declare <vscale x 4 x float> @llvm.log.nxv4f32(<vscale x 4 x float>) #0
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
diff --git a/llvm/test/Transforms/SCCP/assume.ll b/llvm/test/Transforms/SCCP/assume.ll
index 8146d58d9a897..9beee934bb509 100644
--- a/llvm/test/Transforms/SCCP/assume.ll
+++ b/llvm/test/Transforms/SCCP/assume.ll
@@ -69,3 +69,53 @@ define void @nonnull(ptr %v) {
   call void @use(i1 %c4)
   ret void
 }
+
+define void @trunc_nuw(i8 %v) {
+; CHECK-LABEL: @trunc_nuw(
+; CHECK-NEXT:    [[A:%.*]] = trunc nuw i8 [[V:%.*]] to i1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+;
+  %a = trunc nuw i8 %v to i1
+  call void @llvm.assume(i1 %a)
+  %c1 = icmp eq i8 %v, 0
+  call void @use(i1 %c1)
+  %c2 = icmp ne i8 %v, 0
+  call void @use(i1 %c2)
+  %c3 = icmp eq i8 0, %v
+  call void @use(i1 %c3)
+  %c4 = icmp ne i8 0, %v
+  call void @use(i1 %c4)
+  ret void
+}
+
+define void @neg_trunc(i8 %v) {
+; CHECK-LABEL: @neg_trunc(
+; CHECK-NEXT:    [[A:%.*]] = trunc i8 [[V:%.*]] to i1
+; CHECK-NEXT:    call void @llvm.assume(i1 [[A]])
+; CHECK-NEXT:    [[C1:%.*]] = icmp eq i8 [[V]], 0
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp ne i8 [[V]], 0
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    [[C3:%.*]] = icmp eq i8 0, [[V]]
+; CHECK-NEXT:    call void @use(i1 [[C3]])
+; CHECK-NEXT:    [[C4:%.*]] = icmp ne i8 0, [[V]]
+; CHECK-NEXT:    call void @use(i1 [[C4]])
+; CHECK-NEXT:    ret void
+;
+  %a = trunc i8 %v to i1
+  call void @llvm.assume(i1 %a)
+  %c1 = icmp eq i8 %v, 0
+  call void @use(i1 %c1)
+  %c2 = icmp ne i8 %v, 0
+  call void @use(i1 %c2)
+  %c3 = icmp eq i8 0, %v
+  call void @use(i1 %c3)
+  %c4 = icmp ne i8 0, %v
+  call void @use(i1 %c4)
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/conditions-ranges.ll b/llvm/test/Transforms/SCCP/conditions-ranges.ll
index bb3764160f724..a3cf23bd3ae9a 100644
--- a/llvm/test/Transforms/SCCP/conditions-ranges.ll
+++ b/llvm/test/Transforms/SCCP/conditions-ranges.ll
@@ -1436,6 +1436,47 @@ if.end:
   ret i32 0
 }
 
+define void @trunc_nuw_i1_dominating_icmp_ne_0(i8 %x) {
+; CHECK-LABEL: @trunc_nuw_i1_dominating_icmp_ne_0(
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[ICMP]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       bb2:
+; CHECK-NEXT:    ret void
+;
+  %icmp = icmp ne i8 %x, 0
+  br i1 %icmp, label %bb1, label %bb2
+bb1:
+  %c1 = trunc nuw i8 %x to i1
+  call void @use(i1 %c1)
+  ret void
+bb2:
+  ret void
+}
+
+define void @neg_trunc_i1_dominating_icmp_ne_0(i8 %x) {
+; CHECK-LABEL: @neg_trunc_i1_dominating_icmp_ne_0(
+; CHECK-NEXT:    [[ICMP:%.*]] = icmp ne i8 [[X:%.*]], 0
+; CHECK-NEXT:    br i1 [[ICMP]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[C1:%.*]] = trunc i8 [[X]] to i1
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    ret void
+; CHECK:       bb2:
+; CHECK-NEXT:    ret void
+;
+  %icmp = icmp ne i8 %x, 0
+  br i1 %icmp, label %bb1, label %bb2
+bb1:
+  %c1 = trunc i8 %x to i1
+  call void @use(i1 %c1)
+  ret void
+bb2:
+  ret void
+}
+
 define i1 @ptr_icmp_data_layout() {
 ; CHECK-LABEL: @ptr_icmp_data_layout(
 ; CHECK-NEXT:    ret i1 false
@@ -1444,3 +1485,65 @@ define i1 @ptr_icmp_data_layout() {
   %cmp = icmp eq ptr %a.end, @A
   ret i1 %cmp
 }
+
+define void @trunc_nuw_1_dominating_icmp_ne_0(i8 %x) {
+; CHECK-LABEL: @trunc_nuw_1_dominating_icmp_ne_0(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc nuw i8 [[X:%.*]] to i1
+; CHECK-NEXT:    br i1 [[TRUNC]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    ret void
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    ret void
+;
+  %trunc = trunc nuw i8 %x to i1
+  br i1 %trunc, label %bb1, label %bb2
+bb1:
+  %c1 = icmp ne i8 %x , 0
+  call void @use(i1 %c1)
+  %c2 = icmp ne i8 0, %x
+  call void @use(i1 %c2)
+  ret void
+bb2:
+  %c3 = icmp ne i8 %x , 0
+  call void @use(i1 %c3)
+  %c4 = icmp ne i8 0, %x
+  call void @use(i1 %c4)
+  ret void
+}
+
+define void @neg_trunc_1_dominating_icmp_ne_0(i8 %x) {
+; CHECK-LABEL: @neg_trunc_1_dominating_icmp_ne_0(
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[X:%.*]] to i1
+; CHECK-NEXT:    br i1 [[TRUNC]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[C1:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    call void @use(i1 [[C1]])
+; CHECK-NEXT:    [[C2:%.*]] = icmp ne i8 0, [[X]]
+; CHECK-NEXT:    call void @use(i1 [[C2]])
+; CHECK-NEXT:    ret void
+; CHECK:       bb2:
+; CHECK-NEXT:    [[C3:%.*]] = icmp ne i8 [[X]], 0
+; CHECK-NEXT:    call void @use(i1 [[C3]])
+; CHECK-NEXT:    [[C4:%.*]] = icmp ne i8 0, [[X]]
+; CHECK-NEXT:    call void @use(i1 [[C4]])
+; CHECK-NEXT:    ret void
+;
+  %trunc = trunc i8 %x to i1
+  br i1 %trunc, label %bb1, label %bb2
+bb1:
+  %c1 = icmp ne i8 %x , 0
+  call void @use(i1 %c1)
+  %c2 = icmp ne i8 0, %x
+  call void @use(i1 %c2)
+  ret void
+bb2:
+  %c3 = icmp ne i8 %x , 0
+  call void @use(i1 %c3)
+  %c4 = icmp ne i8 0, %x
+  call void @use(i1 %c4)
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/issue59661-missing-predicate-info-for-ssa-copy.ll b/llvm/test/Transforms/SCCP/issue59661-missing-predicate-info-for-ssa-copy.ll
index 564fe95d22196..c3695a31d18f6 100644
--- a/llvm/test/Transforms/SCCP/issue59661-missing-predicate-info-for-ssa-copy.ll
+++ b/llvm/test/Transforms/SCCP/issue59661-missing-predicate-info-for-ssa-copy.ll
@@ -1,16 +1,25 @@
-; REQUIRES: asserts
-; XFAIL: *
-; RUN: opt -S -passes=ipsccp < %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=ipsccp < %s | FileCheck %s
 
 ; https://github.com/llvm/llvm-project/issues/59661
 
 define i32 @bar() {
+; CHECK-LABEL: define i32 @bar() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @foo()
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %call = call i32 @foo()
   ret i32 0
 }
 
 define internal i32 @foo() {
+; CHECK-LABEL: define internal i32 @foo() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[ARST:%.*]] = call ptr @llvm.ssa.copy.p0(ptr @foo)
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %arst = call ptr @llvm.ssa.copy.p0(ptr @foo)
   ret i32 0
diff --git a/llvm/test/Transforms/SCCP/lifetime.ll b/llvm/test/Transforms/SCCP/lifetime.ll
new file mode 100644
index 0000000000000..2956157725c5f
--- /dev/null
+++ b/llvm/test/Transforms/SCCP/lifetime.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=ipsccp < %s | FileCheck %s
+
+define void @test() {
+; CHECK-LABEL: define void @test() {
+; CHECK-NEXT:    [[A:%.*]] = alloca [4 x i16], align 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq ptr [[A]], inttoptr (i64 -1 to ptr)
+; CHECK-NEXT:    br i1 [[CMP]], label %[[IF:.*]], label %[[EXIT:.*]]
+; CHECK:       [[IF]]:
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+  %a = alloca [4 x i16]
+  %cmp = icmp eq ptr %a, inttoptr (i64 -1 to ptr)
+  br i1 %cmp, label %if, label %exit
+
+if:
+  call void @llvm.lifetime.start.p0(ptr %a)
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/SCCP/pr27712.ll b/llvm/test/Transforms/SCCP/pr27712.ll
index d6b32afccd454..04779dc308701 100644
--- a/llvm/test/Transforms/SCCP/pr27712.ll
+++ b/llvm/test/Transforms/SCCP/pr27712.ll
@@ -1,8 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
 ; RUN: opt -passes=sccp -S < %s | FileCheck %s
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
 
 define i32 @main() {
+; CHECK-LABEL: define range(i32 33554432, 0) i32 @main() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LBL_1154:.*]]
+; CHECK:       [[LBL_1154]]:
+; CHECK-NEXT:    [[B0_0:%.*]] = phi i32 [ -119, %[[ENTRY]] ], [ 0, %[[LBL_1154]] ]
+; CHECK-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[B0_0]], 0
+; CHECK-NEXT:    [[SHL_OP:%.*]] = shl nuw nsw i32 33554432, [[B0_0]]
+; CHECK-NEXT:    [[CMP1445:%.*]] = icmp ult i32 [[SHL_OP]], 33554432
+; CHECK-NEXT:    [[CMP14:%.*]] = or i1 [[CMP11]], [[CMP1445]]
+; CHECK-NEXT:    br i1 [[CMP14]], label %[[LBL_1154]], label %[[IF_END19:.*]]
+; CHECK:       [[IF_END19]]:
+; CHECK-NEXT:    br i1 [[CMP11]], label %[[IF_THEN22:.*]], label %[[CLEANUP26:.*]]
+; CHECK:       [[IF_THEN22]]:
+; CHECK-NEXT:    tail call void @abort()
+; CHECK-NEXT:    unreachable
+; CHECK:       [[CLEANUP26]]:
+; CHECK-NEXT:    ret i32 [[SHL_OP]]
+;
 entry:
   br label %lbl_1154
 
@@ -24,7 +41,5 @@ if.then22:
 cleanup26:
   ret i32 %shl.op
 }
-; CHECK-LABEL: define i32 @main(
-; CHECK-NOT: ret i32 undef
 
 declare void @abort()
diff --git a/llvm/test/Transforms/SCCP/sccptest.ll b/llvm/test/Transforms/SCCP/sccptest.ll
index 299645435bf86..7a37b7236b84d 100644
--- a/llvm/test/Transforms/SCCP/sccptest.ll
+++ b/llvm/test/Transforms/SCCP/sccptest.ll
@@ -43,7 +43,7 @@ define i32 @test2(i32 %i0, i32 %j0) {
 ; CHECK:       BB4:
 ; CHECK-NEXT:    ret i32 1
 ; CHECK:       BB5:
-; CHECK-NEXT:    [[K3]] = add i32 [[K2]], 1
+; CHECK-NEXT:    [[K3]] = add nsw i32 [[K2]], 1
 ; CHECK-NEXT:    br label [[BB7]]
 ; CHECK:       BB7:
 ; CHECK-NEXT:    br label [[BB2]]
diff --git a/llvm/test/Transforms/SCCP/undef-resolve.ll b/llvm/test/Transforms/SCCP/undef-resolve.ll
index 8bb2baa82dce3..7984a52713787 100644
--- a/llvm/test/Transforms/SCCP/undef-resolve.ll
+++ b/llvm/test/Transforms/SCCP/undef-resolve.ll
@@ -33,7 +33,7 @@ define i32 @test2() nounwind readnone ssp {
 ; CHECK:       control.outer.outer.split.us:
 ; CHECK-NEXT:    br label [[CONTROL_OUTER_US:%.*]]
 ; CHECK:       control.outer.us:
-; CHECK-NEXT:    [[A_0_PH_US:%.*]] = phi i32 [ [[SWITCHCOND_0_US:%.*]], [[BB3_US:%.*]] ], [ 4, [[CONTROL_OUTER_OUTER_SPLIT_US]] ]
+; CHECK-NEXT:    [[A_0_PH_US:%.*]] = phi i32 [ 3, [[BB3_US:%.*]] ], [ 4, [[CONTROL_OUTER_OUTER_SPLIT_US]] ]
 ; CHECK-NEXT:    [[SWITCHCOND_0_PH_US:%.*]] = phi i32 [ [[A_0_PH_US]], [[BB3_US]] ], [ [[SWITCHCOND_0_PH_PH]], [[CONTROL_OUTER_OUTER_SPLIT_US]] ]
 ; CHECK-NEXT:    br label [[CONTROL_US:%.*]]
 ; CHECK:       bb3.us:
@@ -41,7 +41,7 @@ define i32 @test2() nounwind readnone ssp {
 ; CHECK:       bb0.us:
 ; CHECK-NEXT:    br label [[CONTROL_US]]
 ; CHECK:       control.us:
-; CHECK-NEXT:    [[SWITCHCOND_0_US]] = phi i32 [ [[A_0_PH_US]], [[BB0_US:%.*]] ], [ [[SWITCHCOND_0_PH_US]], [[CONTROL_OUTER_US]] ]
+; CHECK-NEXT:    [[SWITCHCOND_0_US:%.*]] = phi i32 [ [[A_0_PH_US]], [[BB0_US:%.*]] ], [ [[SWITCHCOND_0_PH_US]], [[CONTROL_OUTER_US]] ]
 ; CHECK-NEXT:    switch i32 [[SWITCHCOND_0_US]], label [[CONTROL_OUTER_LOOPEXIT_US_LCSSA_US:%.*]] [
 ; CHECK-NEXT:      i32 0, label [[BB0_US]]
 ; CHECK-NEXT:      i32 1, label [[BB1_US_LCSSA_US:%.*]]
@@ -55,7 +55,7 @@ define i32 @test2() nounwind readnone ssp {
 ; CHECK:       bb4.us-lcssa.us:
 ; CHECK-NEXT:    br label [[BB4:%.*]]
 ; CHECK:       control.outer:
-; CHECK-NEXT:    [[A_0_PH:%.*]] = phi i32 [ [[NEXTID17:%.*]], [[BB3:%.*]] ], [ 4, [[CONTROL_OUTER_OUTER_CONTROL_OUTER_OUTER_SPLIT_CRIT_EDGE]] ]
+; CHECK-NEXT:    [[A_0_PH:%.*]] = phi i32 [ 1, [[BB3:%.*]] ], [ 4, [[CONTROL_OUTER_OUTER_CONTROL_OUTER_OUTER_SPLIT_CRIT_EDGE]] ]
 ; CHECK-NEXT:    [[SWITCHCOND_0_PH:%.*]] = phi i32 [ 0, [[BB3]] ], [ [[SWITCHCOND_0_PH_PH]], [[CONTROL_OUTER_OUTER_CONTROL_OUTER_OUTER_SPLIT_CRIT_EDGE]] ]
 ; CHECK-NEXT:    br label [[CONTROL:%.*]]
 ; CHECK:       control:
@@ -74,7 +74,6 @@ define i32 @test2() nounwind readnone ssp {
 ; CHECK-NEXT:    [[I_0_PH_PH_BE]] = phi i32 [ 1, [[BB4]] ], [ 0, [[CONTROL_OUTER_LOOPEXIT]] ]
 ; CHECK-NEXT:    br label [[CONTROL_OUTER_OUTER]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[NEXTID17]] = add i32 [[SWITCHCOND_0]], -2
 ; CHECK-NEXT:    br label [[CONTROL_OUTER]]
 ; CHECK:       bb0:
 ; CHECK-NEXT:    br label [[CONTROL]]
diff --git a/llvm/test/Transforms/SCCP/widening.ll b/llvm/test/Transforms/SCCP/widening.ll
index 8c661bee8f599..408038cab5e12 100644
--- a/llvm/test/Transforms/SCCP/widening.ll
+++ b/llvm/test/Transforms/SCCP/widening.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt %s -passes=sccp -S | FileCheck --check-prefix=SCCP %s
-; RUN: opt %s -passes=ipsccp -S | FileCheck --check-prefix=IPSCCP %s
+; RUN: opt %s -passes=sccp -S | FileCheck --check-prefixes=CHECK,SCCP %s
+; RUN: opt %s -passes=ipsccp -S | FileCheck --check-prefixes=CHECK,IPSCCP %s
 
 ; Test different widening scenarios.
 
@@ -8,31 +8,18 @@ declare void @use(i1)
 declare i1 @cond()
 
 define void @test_2_incoming_constants(i32 %x) {
-; SCCP-LABEL: @test_2_incoming_constants(
-; SCCP-NEXT:  entry:
-; SCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[EXIT:%.*]]
-; SCCP:       bb1:
-; SCCP-NEXT:    br label [[EXIT]]
-; SCCP:       exit:
-; SCCP-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB1]] ]
-; SCCP-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
-; SCCP-NEXT:    call void @use(i1 true)
-; SCCP-NEXT:    call void @use(i1 false)
-; SCCP-NEXT:    ret void
-;
-; IPSCCP-LABEL: @test_2_incoming_constants(
-; IPSCCP-NEXT:  entry:
-; IPSCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[EXIT:%.*]]
-; IPSCCP:       bb1:
-; IPSCCP-NEXT:    br label [[EXIT]]
-; IPSCCP:       exit:
-; IPSCCP-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB1]] ]
-; IPSCCP-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
-; IPSCCP-NEXT:    call void @use(i1 true)
-; IPSCCP-NEXT:    call void @use(i1 false)
-; IPSCCP-NEXT:    ret void
+; CHECK-LABEL: @test_2_incoming_constants(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB1]] ]
+; CHECK-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    ret void
 ;
 entry:
   %c.1 = call i1 @cond()
@@ -52,37 +39,21 @@ exit:
 }
 
 define void @test_3_incoming_constants(i32 %x) {
-; SCCP-LABEL: @test_3_incoming_constants(
-; SCCP-NEXT:  entry:
-; SCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[EXIT:%.*]]
-; SCCP:       bb1:
-; SCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_2]], label [[BB2:%.*]], label [[EXIT]]
-; SCCP:       bb2:
-; SCCP-NEXT:    br label [[EXIT]]
-; SCCP:       exit:
-; SCCP-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB1]] ], [ 2, [[BB2]] ]
-; SCCP-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
-; SCCP-NEXT:    call void @use(i1 true)
-; SCCP-NEXT:    call void @use(i1 false)
-; SCCP-NEXT:    ret void
-;
-; IPSCCP-LABEL: @test_3_incoming_constants(
-; IPSCCP-NEXT:  entry:
-; IPSCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[EXIT:%.*]]
-; IPSCCP:       bb1:
-; IPSCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_2]], label [[BB2:%.*]], label [[EXIT]]
-; IPSCCP:       bb2:
-; IPSCCP-NEXT:    br label [[EXIT]]
-; IPSCCP:       exit:
-; IPSCCP-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB1]] ], [ 2, [[BB2]] ]
-; IPSCCP-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
-; IPSCCP-NEXT:    call void @use(i1 true)
-; IPSCCP-NEXT:    call void @use(i1 false)
-; IPSCCP-NEXT:    ret void
+; CHECK-LABEL: @test_3_incoming_constants(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[BB2:%.*]], label [[EXIT]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB1]] ], [ 2, [[BB2]] ]
+; CHECK-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    ret void
 ;
 entry:
   %c.1 = call i1 @cond()
@@ -106,49 +77,27 @@ exit:
 }
 
 define void @test_5_incoming_constants(i32 %x) {
-; SCCP-LABEL: @test_5_incoming_constants(
-; SCCP-NEXT:  entry:
-; SCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[EXIT:%.*]]
-; SCCP:       bb1:
-; SCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_2]], label [[BB2:%.*]], label [[EXIT]]
-; SCCP:       bb2:
-; SCCP-NEXT:    [[C_3:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_3]], label [[BB3:%.*]], label [[EXIT]]
-; SCCP:       bb3:
-; SCCP-NEXT:    [[C_4:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_4]], label [[BB4:%.*]], label [[EXIT]]
-; SCCP:       bb4:
-; SCCP-NEXT:    br label [[EXIT]]
-; SCCP:       exit:
-; SCCP-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB1]] ], [ 2, [[BB2]] ], [ 3, [[BB3]] ], [ 4, [[BB4]] ]
-; SCCP-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
-; SCCP-NEXT:    call void @use(i1 true)
-; SCCP-NEXT:    call void @use(i1 false)
-; SCCP-NEXT:    ret void
-;
-; IPSCCP-LABEL: @test_5_incoming_constants(
-; IPSCCP-NEXT:  entry:
-; IPSCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[EXIT:%.*]]
-; IPSCCP:       bb1:
-; IPSCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_2]], label [[BB2:%.*]], label [[EXIT]]
-; IPSCCP:       bb2:
-; IPSCCP-NEXT:    [[C_3:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_3]], label [[BB3:%.*]], label [[EXIT]]
-; IPSCCP:       bb3:
-; IPSCCP-NEXT:    [[C_4:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_4]], label [[BB4:%.*]], label [[EXIT]]
-; IPSCCP:       bb4:
-; IPSCCP-NEXT:    br label [[EXIT]]
-; IPSCCP:       exit:
-; IPSCCP-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB1]] ], [ 2, [[BB2]] ], [ 3, [[BB3]] ], [ 4, [[BB4]] ]
-; IPSCCP-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
-; IPSCCP-NEXT:    call void @use(i1 true)
-; IPSCCP-NEXT:    call void @use(i1 false)
-; IPSCCP-NEXT:    ret void
+; CHECK-LABEL: @test_5_incoming_constants(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[EXIT:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[BB2:%.*]], label [[EXIT]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[C_3:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_3]], label [[BB3:%.*]], label [[EXIT]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[C_4:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_4]], label [[BB4:%.*]], label [[EXIT]]
+; CHECK:       bb4:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 1, [[BB1]] ], [ 2, [[BB2]] ], [ 3, [[BB3]] ], [ 4, [[BB4]] ]
+; CHECK-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    ret void
 ;
 entry:
   %c.1 = call i1 @cond()
@@ -182,47 +131,26 @@ exit:
 ; For the rotated_loop_* test cases %p and %a are extended on each iteration.
 
 define void @rotated_loop_2(i32 %x) {
-; SCCP-LABEL: @rotated_loop_2(
-; SCCP-NEXT:  entry:
-; SCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_1]], label [[EXIT:%.*]], label [[BB1:%.*]]
-; SCCP:       bb1:
-; SCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_2]], label [[EXIT]], label [[BB2:%.*]]
-; SCCP:       bb2:
-; SCCP-NEXT:    [[C_3:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_3]], label [[BB3:%.*]], label [[EXIT]]
-; SCCP:       bb3:
-; SCCP-NEXT:    br label [[EXIT]]
-; SCCP:       exit:
-; SCCP-NEXT:    [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 3, [[BB1]] ], [ 2, [[BB2]] ], [ 5, [[BB3]] ]
-; SCCP-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
-; SCCP-NEXT:    call void @use(i1 true)
-; SCCP-NEXT:    call void @use(i1 false)
-; SCCP-NEXT:    br label [[EXIT_1:%.*]]
-; SCCP:       exit.1:
-; SCCP-NEXT:    ret void
-;
-; IPSCCP-LABEL: @rotated_loop_2(
-; IPSCCP-NEXT:  entry:
-; IPSCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[EXIT:%.*]], label [[BB1:%.*]]
-; IPSCCP:       bb1:
-; IPSCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_2]], label [[EXIT]], label [[BB2:%.*]]
-; IPSCCP:       bb2:
-; IPSCCP-NEXT:    [[C_3:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_3]], label [[BB3:%.*]], label [[EXIT]]
-; IPSCCP:       bb3:
-; IPSCCP-NEXT:    br label [[EXIT]]
-; IPSCCP:       exit:
-; IPSCCP-NEXT:    [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 3, [[BB1]] ], [ 2, [[BB2]] ], [ 5, [[BB3]] ]
-; IPSCCP-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
-; IPSCCP-NEXT:    call void @use(i1 true)
-; IPSCCP-NEXT:    call void @use(i1 false)
-; IPSCCP-NEXT:    br label [[EXIT_1:%.*]]
-; IPSCCP:       exit.1:
-; IPSCCP-NEXT:    ret void
+; CHECK-LABEL: @rotated_loop_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[EXIT:%.*]], label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[EXIT]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[C_3:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_3]], label [[BB3:%.*]], label [[EXIT]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 3, [[BB1]] ], [ 2, [[BB2]] ], [ 5, [[BB3]] ]
+; CHECK-NEXT:    [[A:%.*]] = add nuw nsw i32 [[P]], 1
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    call void @use(i1 false)
+; CHECK-NEXT:    br label [[EXIT_1:%.*]]
+; CHECK:       exit.1:
+; CHECK-NEXT:    ret void
 ;
 entry:
   %c.1 = call i1 @cond()
@@ -254,53 +182,29 @@ exit.1:
 }
 
 define void @rotated_loop_3(i32 %x) {
-; SCCP-LABEL: @rotated_loop_3(
-; SCCP-NEXT:  entry:
-; SCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_1]], label [[EXIT:%.*]], label [[BB1:%.*]]
-; SCCP:       bb1:
-; SCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_2]], label [[EXIT]], label [[BB2:%.*]]
-; SCCP:       bb2:
-; SCCP-NEXT:    [[C_3:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_3]], label [[BB3:%.*]], label [[EXIT]]
-; SCCP:       bb3:
-; SCCP-NEXT:    br label [[EXIT]]
-; SCCP:       exit:
-; SCCP-NEXT:    [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 3, [[BB1]] ], [ 2, [[BB2]] ], [ 5, [[BB3]] ], [ [[A:%.*]], [[EXIT]] ]
-; SCCP-NEXT:    [[A]] = add i32 [[P]], 1
-; SCCP-NEXT:    [[T_1:%.*]] = icmp ult i32 [[A]], 20
-; SCCP-NEXT:    call void @use(i1 [[T_1]])
-; SCCP-NEXT:    [[F_1:%.*]] = icmp ugt i32 [[A]], 10
-; SCCP-NEXT:    call void @use(i1 [[F_1]])
-; SCCP-NEXT:    [[C_4:%.*]] = icmp ult i32 [[A]], 3
-; SCCP-NEXT:    br i1 [[C_4]], label [[EXIT]], label [[EXIT_1:%.*]]
-; SCCP:       exit.1:
-; SCCP-NEXT:    ret void
-;
-; IPSCCP-LABEL: @rotated_loop_3(
-; IPSCCP-NEXT:  entry:
-; IPSCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[EXIT:%.*]], label [[BB1:%.*]]
-; IPSCCP:       bb1:
-; IPSCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_2]], label [[EXIT]], label [[BB2:%.*]]
-; IPSCCP:       bb2:
-; IPSCCP-NEXT:    [[C_3:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_3]], label [[BB3:%.*]], label [[EXIT]]
-; IPSCCP:       bb3:
-; IPSCCP-NEXT:    br label [[EXIT]]
-; IPSCCP:       exit:
-; IPSCCP-NEXT:    [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 3, [[BB1]] ], [ 2, [[BB2]] ], [ 5, [[BB3]] ], [ [[A:%.*]], [[EXIT]] ]
-; IPSCCP-NEXT:    [[A]] = add i32 [[P]], 1
-; IPSCCP-NEXT:    [[T_1:%.*]] = icmp ult i32 [[A]], 20
-; IPSCCP-NEXT:    call void @use(i1 [[T_1]])
-; IPSCCP-NEXT:    [[F_1:%.*]] = icmp ugt i32 [[A]], 10
-; IPSCCP-NEXT:    call void @use(i1 [[F_1]])
-; IPSCCP-NEXT:    [[C_4:%.*]] = icmp ult i32 [[A]], 3
-; IPSCCP-NEXT:    br i1 [[C_4]], label [[EXIT]], label [[EXIT_1:%.*]]
-; IPSCCP:       exit.1:
-; IPSCCP-NEXT:    ret void
+; CHECK-LABEL: @rotated_loop_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[EXIT:%.*]], label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[EXIT]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[C_3:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_3]], label [[BB3:%.*]], label [[EXIT]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 1, [[ENTRY:%.*]] ], [ 3, [[BB1]] ], [ 2, [[BB2]] ], [ 5, [[BB3]] ], [ [[A:%.*]], [[EXIT]] ]
+; CHECK-NEXT:    [[A]] = add i32 [[P]], 1
+; CHECK-NEXT:    [[T_1:%.*]] = icmp ult i32 [[A]], 20
+; CHECK-NEXT:    call void @use(i1 [[T_1]])
+; CHECK-NEXT:    [[F_1:%.*]] = icmp ugt i32 [[A]], 10
+; CHECK-NEXT:    call void @use(i1 [[F_1]])
+; CHECK-NEXT:    [[C_4:%.*]] = icmp ult i32 [[A]], 3
+; CHECK-NEXT:    br i1 [[C_4]], label [[EXIT]], label [[EXIT_1:%.*]]
+; CHECK:       exit.1:
+; CHECK-NEXT:    ret void
 ;
 entry:
   %c.1 = call i1 @cond()
@@ -334,34 +238,19 @@ exit.1:
 ; For the loop_with_header_* tests, %iv and %a change on each iteration, but we
 ; can use the range imposed by the condition %c.1 when widening.
 define void @loop_with_header_1(i32 %x) {
-; SCCP-LABEL: @loop_with_header_1(
-; SCCP-NEXT:  entry:
-; SCCP-NEXT:    br label [[LOOP_HEADER:%.*]]
-; SCCP:       loop.header:
-; SCCP-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_BODY:%.*]] ]
-; SCCP-NEXT:    [[C_1:%.*]] = icmp slt i32 [[IV]], 2
-; SCCP-NEXT:    br i1 [[C_1]], label [[LOOP_BODY]], label [[EXIT:%.*]]
-; SCCP:       loop.body:
-; SCCP-NEXT:    [[T_1:%.*]] = icmp slt i32 [[IV]], 2
-; SCCP-NEXT:    call void @use(i1 [[T_1]])
-; SCCP-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
-; SCCP-NEXT:    br label [[LOOP_HEADER]]
-; SCCP:       exit:
-; SCCP-NEXT:    ret void
-;
-; IPSCCP-LABEL: @loop_with_header_1(
-; IPSCCP-NEXT:  entry:
-; IPSCCP-NEXT:    br label [[LOOP_HEADER:%.*]]
-; IPSCCP:       loop.header:
-; IPSCCP-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_BODY:%.*]] ]
-; IPSCCP-NEXT:    [[C_1:%.*]] = icmp slt i32 [[IV]], 2
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[LOOP_BODY]], label [[EXIT:%.*]]
-; IPSCCP:       loop.body:
-; IPSCCP-NEXT:    call void @use(i1 true)
-; IPSCCP-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; IPSCCP-NEXT:    br label [[LOOP_HEADER]]
-; IPSCCP:       exit:
-; IPSCCP-NEXT:    ret void
+; CHECK-LABEL: @loop_with_header_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_BODY:%.*]] ]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[IV]], 2
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_BODY]], label [[EXIT:%.*]]
+; CHECK:       loop.body:
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop.header
@@ -382,34 +271,19 @@ exit:
 }
 
 define void @loop_with_header_2(i32 %x) {
-; SCCP-LABEL: @loop_with_header_2(
-; SCCP-NEXT:  entry:
-; SCCP-NEXT:    br label [[LOOP_HEADER:%.*]]
-; SCCP:       loop.header:
-; SCCP-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_BODY:%.*]] ]
-; SCCP-NEXT:    [[C_1:%.*]] = icmp slt i32 [[IV]], 200
-; SCCP-NEXT:    br i1 [[C_1]], label [[LOOP_BODY]], label [[EXIT:%.*]]
-; SCCP:       loop.body:
-; SCCP-NEXT:    [[T_1:%.*]] = icmp slt i32 [[IV]], 200
-; SCCP-NEXT:    call void @use(i1 [[T_1]])
-; SCCP-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
-; SCCP-NEXT:    br label [[LOOP_HEADER]]
-; SCCP:       exit:
-; SCCP-NEXT:    ret void
-;
-; IPSCCP-LABEL: @loop_with_header_2(
-; IPSCCP-NEXT:  entry:
-; IPSCCP-NEXT:    br label [[LOOP_HEADER:%.*]]
-; IPSCCP:       loop.header:
-; IPSCCP-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_BODY:%.*]] ]
-; IPSCCP-NEXT:    [[C_1:%.*]] = icmp slt i32 [[IV]], 200
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[LOOP_BODY]], label [[EXIT:%.*]]
-; IPSCCP:       loop.body:
-; IPSCCP-NEXT:    call void @use(i1 true)
-; IPSCCP-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
-; IPSCCP-NEXT:    br label [[LOOP_HEADER]]
-; IPSCCP:       exit:
-; IPSCCP-NEXT:    ret void
+; CHECK-LABEL: @loop_with_header_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_HEADER:%.*]]
+; CHECK:       loop.header:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP_BODY:%.*]] ]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_BODY]], label [[EXIT:%.*]]
+; CHECK:       loop.body:
+; CHECK-NEXT:    call void @use(i1 true)
+; CHECK-NEXT:    [[IV_NEXT]] = add nsw i32 [[IV]], 1
+; CHECK-NEXT:    br label [[LOOP_HEADER]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop.header
@@ -433,79 +307,42 @@ exit:
 ; can be used as a widening bound. It does not fully contain the range we get
 ; from combining it with the information from %tmp12.
 define void @foo(ptr %arg, i8 %x) {
-; SCCP-LABEL: @foo(
-; SCCP-NEXT:  bb:
-; SCCP-NEXT:    [[TMP:%.*]] = zext i8 [[X:%.*]] to i32
-; SCCP-NEXT:    [[TMP2:%.*]] = load i64, ptr [[ARG:%.*]], align 8
-; SCCP-NEXT:    switch i32 [[TMP]], label [[BB20:%.*]] [
-; SCCP-NEXT:      i32 1, label [[BB3:%.*]]
-; SCCP-NEXT:      i32 2, label [[BB4:%.*]]
-; SCCP-NEXT:      i32 4, label [[BB19:%.*]]
-; SCCP-NEXT:    ]
-; SCCP:       bb3:
-; SCCP-NEXT:    unreachable
-; SCCP:       bb4:
-; SCCP-NEXT:    [[TMP5:%.*]] = add i64 [[TMP2]], 3
-; SCCP-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], 3
-; SCCP-NEXT:    [[TMP7:%.*]] = sub nuw nsw i64 3, [[TMP6]]
-; SCCP-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 1
-; SCCP-NEXT:    [[TMP9:%.*]] = trunc nuw nsw i64 [[TMP8]] to i32
-; SCCP-NEXT:    [[TMP10:%.*]] = zext nneg i32 [[TMP9]] to i64
-; SCCP-NEXT:    br label [[BB11:%.*]]
-; SCCP:       bb11:
-; SCCP-NEXT:    [[TMP12:%.*]] = phi i64 [ [[TMP10]], [[BB4]] ], [ [[TMP17:%.*]], [[BB18:%.*]] ]
-; SCCP-NEXT:    br label [[BB13:%.*]]
-; SCCP:       bb13:
-; SCCP-NEXT:    [[C_1:%.*]] = icmp eq i64 [[TMP12]], 6
-; SCCP-NEXT:    br i1 [[C_1]], label [[BB15:%.*]], label [[BB16:%.*]]
-; SCCP:       bb15:
-; SCCP-NEXT:    unreachable
-; SCCP:       bb16:
-; SCCP-NEXT:    [[TMP17]] = add i64 [[TMP12]], 2
-; SCCP-NEXT:    br label [[BB18]]
-; SCCP:       bb18:
-; SCCP-NEXT:    br label [[BB11]]
-; SCCP:       bb19:
-; SCCP-NEXT:    unreachable
-; SCCP:       bb20:
-; SCCP-NEXT:    ret void
-;
-; IPSCCP-LABEL: @foo(
-; IPSCCP-NEXT:  bb:
-; IPSCCP-NEXT:    [[TMP:%.*]] = zext i8 [[X:%.*]] to i32
-; IPSCCP-NEXT:    [[TMP2:%.*]] = load i64, ptr [[ARG:%.*]], align 8
-; IPSCCP-NEXT:    switch i32 [[TMP]], label [[BB20:%.*]] [
-; IPSCCP-NEXT:      i32 1, label [[BB3:%.*]]
-; IPSCCP-NEXT:      i32 2, label [[BB4:%.*]]
-; IPSCCP-NEXT:      i32 4, label [[BB19:%.*]]
-; IPSCCP-NEXT:    ]
-; IPSCCP:       bb3:
-; IPSCCP-NEXT:    unreachable
-; IPSCCP:       bb4:
-; IPSCCP-NEXT:    [[TMP5:%.*]] = add i64 [[TMP2]], 3
-; IPSCCP-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], 3
-; IPSCCP-NEXT:    [[TMP7:%.*]] = sub nuw nsw i64 3, [[TMP6]]
-; IPSCCP-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 1
-; IPSCCP-NEXT:    [[TMP9:%.*]] = trunc nuw nsw i64 [[TMP8]] to i32
-; IPSCCP-NEXT:    [[TMP10:%.*]] = zext nneg i32 [[TMP9]] to i64
-; IPSCCP-NEXT:    br label [[BB11:%.*]]
-; IPSCCP:       bb11:
-; IPSCCP-NEXT:    [[TMP12:%.*]] = phi i64 [ [[TMP10]], [[BB4]] ], [ [[TMP17:%.*]], [[BB18:%.*]] ]
-; IPSCCP-NEXT:    br label [[BB13:%.*]]
-; IPSCCP:       bb13:
-; IPSCCP-NEXT:    [[C_1:%.*]] = icmp eq i64 [[TMP12]], 6
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[BB15:%.*]], label [[BB16:%.*]]
-; IPSCCP:       bb15:
-; IPSCCP-NEXT:    unreachable
-; IPSCCP:       bb16:
-; IPSCCP-NEXT:    [[TMP17]] = add i64 [[TMP12]], 2
-; IPSCCP-NEXT:    br label [[BB18]]
-; IPSCCP:       bb18:
-; IPSCCP-NEXT:    br label [[BB11]]
-; IPSCCP:       bb19:
-; IPSCCP-NEXT:    unreachable
-; IPSCCP:       bb20:
-; IPSCCP-NEXT:    ret void
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = zext i8 [[X:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[ARG:%.*]], align 8
+; CHECK-NEXT:    switch i32 [[TMP]], label [[BB20:%.*]] [
+; CHECK-NEXT:      i32 1, label [[BB3:%.*]]
+; CHECK-NEXT:      i32 2, label [[BB4:%.*]]
+; CHECK-NEXT:      i32 4, label [[BB19:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb3:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb4:
+; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP6:%.*]] = and i64 [[TMP5]], 3
+; CHECK-NEXT:    [[TMP7:%.*]] = sub nuw nsw i64 3, [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc nuw nsw i64 [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP10:%.*]] = zext nneg i32 [[TMP9]] to i64
+; CHECK-NEXT:    br label [[BB11:%.*]]
+; CHECK:       bb11:
+; CHECK-NEXT:    [[TMP12:%.*]] = phi i64 [ [[TMP10]], [[BB4]] ], [ [[TMP17:%.*]], [[BB18:%.*]] ]
+; CHECK-NEXT:    br label [[BB13:%.*]]
+; CHECK:       bb13:
+; CHECK-NEXT:    [[C_1:%.*]] = icmp eq i64 [[TMP12]], 6
+; CHECK-NEXT:    br i1 [[C_1]], label [[BB15:%.*]], label [[BB16:%.*]]
+; CHECK:       bb15:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb16:
+; CHECK-NEXT:    [[TMP17]] = add i64 [[TMP12]], 2
+; CHECK-NEXT:    br label [[BB18]]
+; CHECK:       bb18:
+; CHECK-NEXT:    br label [[BB11]]
+; CHECK:       bb19:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb20:
+; CHECK-NEXT:    ret void
 ;
 bb:
   %tmp = zext i8 %x to i32
@@ -564,21 +401,13 @@ bb20:                                             ; preds = %bb
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #1
 
 define linkonce_odr dereferenceable(1) ptr @spam(ptr %arg, i32 %arg1) align 2 {
-; SCCP-LABEL: @spam(
-; SCCP-NEXT:  bb:
-; SCCP-NEXT:    [[TMP:%.*]] = getelementptr inbounds nuw [[STRUCT_BAZ_1:%.*]], ptr [[ARG:%.*]], i32 0, i32 3
-; SCCP-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
-; SCCP-NEXT:    [[TMP3:%.*]] = sext i32 [[ARG1:%.*]] to i64
-; SCCP-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 [[TMP3]]
-; SCCP-NEXT:    ret ptr [[TMP4]]
-;
-; IPSCCP-LABEL: @spam(
-; IPSCCP-NEXT:  bb:
-; IPSCCP-NEXT:    [[TMP:%.*]] = getelementptr inbounds nuw [[STRUCT_BAZ_1:%.*]], ptr [[ARG:%.*]], i32 0, i32 3
-; IPSCCP-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
-; IPSCCP-NEXT:    [[TMP3:%.*]] = sext i32 [[ARG1:%.*]] to i64
-; IPSCCP-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 [[TMP3]]
-; IPSCCP-NEXT:    ret ptr [[TMP4]]
+; CHECK-LABEL: @spam(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds nuw [[STRUCT_BAZ_1:%.*]], ptr [[ARG:%.*]], i32 0, i32 3
+; CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = sext i32 [[ARG1:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 [[TMP3]]
+; CHECK-NEXT:    ret ptr [[TMP4]]
 ;
 bb:
   %tmp = getelementptr inbounds %struct.baz.1, ptr %arg, i32 0, i32 3
@@ -589,167 +418,85 @@ bb:
 }
 
 define ptr @wobble(ptr %arg, i32 %arg1) align 2 {
-; SCCP-LABEL: @wobble(
-; SCCP-NEXT:  bb:
-; SCCP-NEXT:    [[TMP:%.*]] = lshr i32 [[ARG1:%.*]], 16
-; SCCP-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP]], [[ARG1]]
-; SCCP-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 65535
-; SCCP-NEXT:    [[TMP4:%.*]] = mul i32 [[ARG1]], 8
-; SCCP-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT_BLAM_2:%.*]], ptr [[ARG:%.*]], i32 0, i32 1
-; SCCP-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 8
-; SCCP-NEXT:    [[TMP7:%.*]] = and i32 [[TMP4]], [[TMP6]]
-; SCCP-NEXT:    br label [[BB8:%.*]]
-; SCCP:       bb8:
-; SCCP-NEXT:    [[TMP9:%.*]] = phi ptr [ undef, [[BB:%.*]] ], [ [[TMP17:%.*]], [[BB29:%.*]] ]
-; SCCP-NEXT:    [[TMP10:%.*]] = phi ptr [ undef, [[BB]] ], [ [[TMP17]], [[BB29]] ]
-; SCCP-NEXT:    [[TMP11:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP30:%.*]], [[BB29]] ]
-; SCCP-NEXT:    [[C_1:%.*]] = icmp slt i32 [[TMP11]], 8
-; SCCP-NEXT:    br i1 [[C_1]], label [[BB13:%.*]], label [[BB31:%.*]]
-; SCCP:       bb13:
-; SCCP-NEXT:    [[TMP15:%.*]] = add i32 [[TMP7]], [[TMP11]]
-; SCCP-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 4
-; SCCP-NEXT:    [[TMP17]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP16]])
-; SCCP-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], i64 2
-; SCCP-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
-; SCCP-NEXT:    [[TMP21:%.*]] = zext i8 [[TMP20]] to i32
-; SCCP-NEXT:    [[TMP22:%.*]] = icmp eq i32 [[TMP21]], 0
-; SCCP-NEXT:    br i1 [[TMP22]], label [[BB23:%.*]], label [[BB25:%.*]]
-; SCCP:       bb23:
-; SCCP-NEXT:    [[TMP24:%.*]] = trunc nuw i32 [[TMP3]] to i16
-; SCCP-NEXT:    store i16 [[TMP24]], ptr [[TMP17]], align 2
-; SCCP-NEXT:    br label [[BB31]]
-; SCCP:       bb25:
-; SCCP-NEXT:    [[TMP26:%.*]] = load i16, ptr [[TMP17]], align 2
-; SCCP-NEXT:    [[TMP27:%.*]] = zext i16 [[TMP26]] to i32
-; SCCP-NEXT:    [[TMP28:%.*]] = icmp eq i32 [[TMP27]], [[TMP3]]
-; SCCP-NEXT:    br i1 [[TMP28]], label [[BB31]], label [[BB29]]
-; SCCP:       bb29:
-; SCCP-NEXT:    [[TMP30]] = add nsw i32 [[TMP11]], 1
-; SCCP-NEXT:    br label [[BB8]]
-; SCCP:       bb31:
-; SCCP-NEXT:    [[TMP32:%.*]] = phi ptr [ [[TMP17]], [[BB23]] ], [ [[TMP17]], [[BB25]] ], [ [[TMP9]], [[BB8]] ]
-; SCCP-NEXT:    [[TMP33:%.*]] = phi ptr [ [[TMP17]], [[BB23]] ], [ [[TMP17]], [[BB25]] ], [ [[TMP10]], [[BB8]] ]
-; SCCP-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[TMP11]], 0
-; SCCP-NEXT:    br i1 [[TMP34]], label [[BB35:%.*]], label [[BB37:%.*]]
-; SCCP:       bb35:
-; SCCP-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP32]], i64 1
-; SCCP-NEXT:    br label [[BB66:%.*]]
-; SCCP:       bb37:
-; SCCP-NEXT:    [[C_2:%.*]] = icmp eq i32 [[TMP11]], 8
-; SCCP-NEXT:    br i1 [[C_2]], label [[BB39:%.*]], label [[BB58:%.*]]
-; SCCP:       bb39:
-; SCCP-NEXT:    [[TMP40:%.*]] = add nsw i32 [[TMP11]], -1
-; SCCP-NEXT:    [[TMP41:%.*]] = trunc nuw i32 [[TMP3]] to i16
-; SCCP-NEXT:    store i16 [[TMP41]], ptr @global.11, align 1
-; SCCP-NEXT:    [[TMP43:%.*]] = add i32 [[TMP7]], [[TMP40]]
-; SCCP-NEXT:    [[TMP44:%.*]] = mul i32 [[TMP43]], 4
-; SCCP-NEXT:    [[TMP45:%.*]] = add i32 [[TMP44]], 2
-; SCCP-NEXT:    [[TMP46:%.*]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP45]])
-; SCCP-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
-; SCCP-NEXT:    [[TMP48:%.*]] = zext i8 [[TMP47]] to i32
-; SCCP-NEXT:    [[TMP49:%.*]] = sub i32 [[TMP43]], 1
-; SCCP-NEXT:    [[TMP50:%.*]] = mul i32 [[TMP49]], 4
-; SCCP-NEXT:    [[TMP51:%.*]] = add i32 [[TMP50]], 2
-; SCCP-NEXT:    [[TMP52:%.*]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP51]])
-; SCCP-NEXT:    [[TMP53:%.*]] = load i8, ptr [[TMP52]], align 1
-; SCCP-NEXT:    [[TMP54:%.*]] = zext i8 [[TMP53]] to i32
-; SCCP-NEXT:    [[TMP55:%.*]] = icmp sgt i32 [[TMP48]], [[TMP54]]
-; SCCP-NEXT:    br i1 [[TMP55]], label [[BB56:%.*]], label [[BB60:%.*]]
-; SCCP:       bb56:
-; SCCP-NEXT:    [[TMP57:%.*]] = add nsw i32 [[TMP40]], -1
-; SCCP-NEXT:    br label [[BB60]]
-; SCCP:       bb58:
-; SCCP-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 @global.11, ptr align 2 [[TMP33]], i64 4, i1 false)
-; SCCP-NEXT:    br label [[BB60]]
-; SCCP:       bb60:
-; SCCP-NEXT:    [[TMP61:%.*]] = phi i32 [ [[TMP57]], [[BB56]] ], [ [[TMP40]], [[BB39]] ], [ [[TMP11]], [[BB58]] ]
-; SCCP-NEXT:    [[TMP63:%.*]] = add i32 [[TMP7]], 1
-; SCCP-NEXT:    [[TMP64:%.*]] = mul i32 [[TMP63]], 4
-; SCCP-NEXT:    [[TMP65:%.*]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP64]])
-; SCCP-NEXT:    br label [[BB66]]
-; SCCP:       bb66:
-; SCCP-NEXT:    [[TMP67:%.*]] = phi ptr [ [[TMP36]], [[BB35]] ], [ null, [[BB60]] ]
-; SCCP-NEXT:    ret ptr [[TMP67]]
-;
-; IPSCCP-LABEL: @wobble(
-; IPSCCP-NEXT:  bb:
-; IPSCCP-NEXT:    [[TMP:%.*]] = lshr i32 [[ARG1:%.*]], 16
-; IPSCCP-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP]], [[ARG1]]
-; IPSCCP-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 65535
-; IPSCCP-NEXT:    [[TMP4:%.*]] = mul i32 [[ARG1]], 8
-; IPSCCP-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT_BLAM_2:%.*]], ptr [[ARG:%.*]], i32 0, i32 1
-; IPSCCP-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 8
-; IPSCCP-NEXT:    [[TMP7:%.*]] = and i32 [[TMP4]], [[TMP6]]
-; IPSCCP-NEXT:    br label [[BB8:%.*]]
-; IPSCCP:       bb8:
-; IPSCCP-NEXT:    [[TMP9:%.*]] = phi ptr [ undef, [[BB:%.*]] ], [ [[TMP17:%.*]], [[BB29:%.*]] ]
-; IPSCCP-NEXT:    [[TMP10:%.*]] = phi ptr [ undef, [[BB]] ], [ [[TMP17]], [[BB29]] ]
-; IPSCCP-NEXT:    [[TMP11:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP30:%.*]], [[BB29]] ]
-; IPSCCP-NEXT:    [[C_1:%.*]] = icmp slt i32 [[TMP11]], 8
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[BB13:%.*]], label [[BB31:%.*]]
-; IPSCCP:       bb13:
-; IPSCCP-NEXT:    [[TMP15:%.*]] = add i32 [[TMP7]], [[TMP11]]
-; IPSCCP-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 4
-; IPSCCP-NEXT:    [[TMP17]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP16]])
-; IPSCCP-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], i64 2
-; IPSCCP-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
-; IPSCCP-NEXT:    [[TMP21:%.*]] = zext i8 [[TMP20]] to i32
-; IPSCCP-NEXT:    [[TMP22:%.*]] = icmp eq i32 [[TMP21]], 0
-; IPSCCP-NEXT:    br i1 [[TMP22]], label [[BB23:%.*]], label [[BB25:%.*]]
-; IPSCCP:       bb23:
-; IPSCCP-NEXT:    [[TMP24:%.*]] = trunc nuw i32 [[TMP3]] to i16
-; IPSCCP-NEXT:    store i16 [[TMP24]], ptr [[TMP17]], align 2
-; IPSCCP-NEXT:    br label [[BB31]]
-; IPSCCP:       bb25:
-; IPSCCP-NEXT:    [[TMP26:%.*]] = load i16, ptr [[TMP17]], align 2
-; IPSCCP-NEXT:    [[TMP27:%.*]] = zext i16 [[TMP26]] to i32
-; IPSCCP-NEXT:    [[TMP28:%.*]] = icmp eq i32 [[TMP27]], [[TMP3]]
-; IPSCCP-NEXT:    br i1 [[TMP28]], label [[BB31]], label [[BB29]]
-; IPSCCP:       bb29:
-; IPSCCP-NEXT:    [[TMP30]] = add nsw i32 [[TMP11]], 1
-; IPSCCP-NEXT:    br label [[BB8]]
-; IPSCCP:       bb31:
-; IPSCCP-NEXT:    [[TMP32:%.*]] = phi ptr [ [[TMP17]], [[BB23]] ], [ [[TMP17]], [[BB25]] ], [ [[TMP9]], [[BB8]] ]
-; IPSCCP-NEXT:    [[TMP33:%.*]] = phi ptr [ [[TMP17]], [[BB23]] ], [ [[TMP17]], [[BB25]] ], [ [[TMP10]], [[BB8]] ]
-; IPSCCP-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[TMP11]], 0
-; IPSCCP-NEXT:    br i1 [[TMP34]], label [[BB35:%.*]], label [[BB37:%.*]]
-; IPSCCP:       bb35:
-; IPSCCP-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP32]], i64 1
-; IPSCCP-NEXT:    br label [[BB66:%.*]]
-; IPSCCP:       bb37:
-; IPSCCP-NEXT:    [[C_2:%.*]] = icmp eq i32 [[TMP11]], 8
-; IPSCCP-NEXT:    br i1 [[C_2]], label [[BB39:%.*]], label [[BB58:%.*]]
-; IPSCCP:       bb39:
-; IPSCCP-NEXT:    [[TMP41:%.*]] = trunc nuw i32 [[TMP3]] to i16
-; IPSCCP-NEXT:    store i16 [[TMP41]], ptr @global.11, align 1
-; IPSCCP-NEXT:    [[TMP43:%.*]] = add i32 [[TMP7]], 7
-; IPSCCP-NEXT:    [[TMP44:%.*]] = mul i32 [[TMP43]], 4
-; IPSCCP-NEXT:    [[TMP45:%.*]] = add i32 [[TMP44]], 2
-; IPSCCP-NEXT:    [[TMP46:%.*]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP45]])
-; IPSCCP-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
-; IPSCCP-NEXT:    [[TMP48:%.*]] = zext i8 [[TMP47]] to i32
-; IPSCCP-NEXT:    [[TMP49:%.*]] = sub i32 [[TMP43]], 1
-; IPSCCP-NEXT:    [[TMP50:%.*]] = mul i32 [[TMP49]], 4
-; IPSCCP-NEXT:    [[TMP51:%.*]] = add i32 [[TMP50]], 2
-; IPSCCP-NEXT:    [[TMP52:%.*]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP51]])
-; IPSCCP-NEXT:    [[TMP53:%.*]] = load i8, ptr [[TMP52]], align 1
-; IPSCCP-NEXT:    [[TMP54:%.*]] = zext i8 [[TMP53]] to i32
-; IPSCCP-NEXT:    [[TMP55:%.*]] = icmp sgt i32 [[TMP48]], [[TMP54]]
-; IPSCCP-NEXT:    br i1 [[TMP55]], label [[BB56:%.*]], label [[BB60:%.*]]
-; IPSCCP:       bb56:
-; IPSCCP-NEXT:    br label [[BB60]]
-; IPSCCP:       bb58:
-; IPSCCP-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 @global.11, ptr align 2 [[TMP33]], i64 4, i1 false)
-; IPSCCP-NEXT:    br label [[BB60]]
-; IPSCCP:       bb60:
-; IPSCCP-NEXT:    [[TMP61:%.*]] = phi i32 [ 6, [[BB56]] ], [ 7, [[BB39]] ], [ [[TMP11]], [[BB58]] ]
-; IPSCCP-NEXT:    [[TMP63:%.*]] = add i32 [[TMP7]], 1
-; IPSCCP-NEXT:    [[TMP64:%.*]] = mul i32 [[TMP63]], 4
-; IPSCCP-NEXT:    [[TMP65:%.*]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP64]])
-; IPSCCP-NEXT:    br label [[BB66]]
-; IPSCCP:       bb66:
-; IPSCCP-NEXT:    [[TMP67:%.*]] = phi ptr [ [[TMP36]], [[BB35]] ], [ null, [[BB60]] ]
-; IPSCCP-NEXT:    ret ptr [[TMP67]]
+; CHECK-LABEL: @wobble(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = lshr i32 [[ARG1:%.*]], 16
+; CHECK-NEXT:    [[TMP2:%.*]] = xor i32 [[TMP]], [[ARG1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 65535
+; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[ARG1]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds nuw [[STRUCT_BLAM_2:%.*]], ptr [[ARG:%.*]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = and i32 [[TMP4]], [[TMP6]]
+; CHECK-NEXT:    br label [[BB8:%.*]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi ptr [ undef, [[BB:%.*]] ], [ [[TMP17:%.*]], [[BB29:%.*]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = phi ptr [ undef, [[BB]] ], [ [[TMP17]], [[BB29]] ]
+; CHECK-NEXT:    [[TMP11:%.*]] = phi i32 [ 0, [[BB]] ], [ [[TMP30:%.*]], [[BB29]] ]
+; CHECK-NEXT:    [[C_1:%.*]] = icmp slt i32 [[TMP11]], 8
+; CHECK-NEXT:    br i1 [[C_1]], label [[BB13:%.*]], label [[BB31:%.*]]
+; CHECK:       bb13:
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP7]], [[TMP11]]
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP17]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP16]])
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP17]], i64 2
+; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-NEXT:    [[TMP21:%.*]] = zext i8 [[TMP20]] to i32
+; CHECK-NEXT:    [[TMP22:%.*]] = icmp eq i32 [[TMP21]], 0
+; CHECK-NEXT:    br i1 [[TMP22]], label [[BB23:%.*]], label [[BB25:%.*]]
+; CHECK:       bb23:
+; CHECK-NEXT:    [[TMP24:%.*]] = trunc nuw i32 [[TMP3]] to i16
+; CHECK-NEXT:    store i16 [[TMP24]], ptr [[TMP17]], align 2
+; CHECK-NEXT:    br label [[BB31]]
+; CHECK:       bb25:
+; CHECK-NEXT:    [[TMP26:%.*]] = load i16, ptr [[TMP17]], align 2
+; CHECK-NEXT:    [[TMP27:%.*]] = zext i16 [[TMP26]] to i32
+; CHECK-NEXT:    [[TMP28:%.*]] = icmp eq i32 [[TMP27]], [[TMP3]]
+; CHECK-NEXT:    br i1 [[TMP28]], label [[BB31]], label [[BB29]]
+; CHECK:       bb29:
+; CHECK-NEXT:    [[TMP30]] = add nsw i32 [[TMP11]], 1
+; CHECK-NEXT:    br label [[BB8]]
+; CHECK:       bb31:
+; CHECK-NEXT:    [[TMP32:%.*]] = phi ptr [ [[TMP17]], [[BB23]] ], [ [[TMP17]], [[BB25]] ], [ [[TMP9]], [[BB8]] ]
+; CHECK-NEXT:    [[TMP33:%.*]] = phi ptr [ [[TMP17]], [[BB23]] ], [ [[TMP17]], [[BB25]] ], [ [[TMP10]], [[BB8]] ]
+; CHECK-NEXT:    [[TMP34:%.*]] = icmp eq i32 [[TMP11]], 0
+; CHECK-NEXT:    br i1 [[TMP34]], label [[BB35:%.*]], label [[BB37:%.*]]
+; CHECK:       bb35:
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP32]], i64 1
+; CHECK-NEXT:    br label [[BB66:%.*]]
+; CHECK:       bb37:
+; CHECK-NEXT:    [[C_2:%.*]] = icmp eq i32 [[TMP11]], 8
+; CHECK-NEXT:    br i1 [[C_2]], label [[BB39:%.*]], label [[BB58:%.*]]
+; CHECK:       bb39:
+; CHECK-NEXT:    [[TMP41:%.*]] = trunc nuw i32 [[TMP3]] to i16
+; CHECK-NEXT:    store i16 [[TMP41]], ptr @global.11, align 1
+; CHECK-NEXT:    [[TMP43:%.*]] = add i32 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP44:%.*]] = mul i32 [[TMP43]], 4
+; CHECK-NEXT:    [[TMP45:%.*]] = add i32 [[TMP44]], 2
+; CHECK-NEXT:    [[TMP46:%.*]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP45]])
+; CHECK-NEXT:    [[TMP47:%.*]] = load i8, ptr [[TMP46]], align 1
+; CHECK-NEXT:    [[TMP48:%.*]] = zext i8 [[TMP47]] to i32
+; CHECK-NEXT:    [[TMP49:%.*]] = sub i32 [[TMP43]], 1
+; CHECK-NEXT:    [[TMP50:%.*]] = mul i32 [[TMP49]], 4
+; CHECK-NEXT:    [[TMP51:%.*]] = add i32 [[TMP50]], 2
+; CHECK-NEXT:    [[TMP52:%.*]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP51]])
+; CHECK-NEXT:    [[TMP53:%.*]] = load i8, ptr [[TMP52]], align 1
+; CHECK-NEXT:    [[TMP54:%.*]] = zext i8 [[TMP53]] to i32
+; CHECK-NEXT:    [[TMP55:%.*]] = icmp sgt i32 [[TMP48]], [[TMP54]]
+; CHECK-NEXT:    br i1 [[TMP55]], label [[BB56:%.*]], label [[BB60:%.*]]
+; CHECK:       bb56:
+; CHECK-NEXT:    br label [[BB60]]
+; CHECK:       bb58:
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 @global.11, ptr align 2 [[TMP33]], i64 4, i1 false)
+; CHECK-NEXT:    br label [[BB60]]
+; CHECK:       bb60:
+; CHECK-NEXT:    [[TMP61:%.*]] = phi i32 [ 6, [[BB56]] ], [ 7, [[BB39]] ], [ [[TMP11]], [[BB58]] ]
+; CHECK-NEXT:    [[TMP63:%.*]] = add i32 [[TMP7]], 1
+; CHECK-NEXT:    [[TMP64:%.*]] = mul i32 [[TMP63]], 4
+; CHECK-NEXT:    [[TMP65:%.*]] = call dereferenceable(1) ptr @spam(ptr [[ARG]], i32 [[TMP64]])
+; CHECK-NEXT:    br label [[BB66]]
+; CHECK:       bb66:
+; CHECK-NEXT:    [[TMP67:%.*]] = phi ptr [ [[TMP36]], [[BB35]] ], [ null, [[BB60]] ]
+; CHECK-NEXT:    ret ptr [[TMP67]]
 ;
 bb:
   %tmp = lshr i32 %arg1, 16
@@ -848,57 +595,31 @@ bb66:                                             ; preds = %bb60, %bb35
 
 
 define i32 @loop_with_multiple_euqal_incomings(i32 %N) {
-; SCCP-LABEL: @loop_with_multiple_euqal_incomings(
-; SCCP-NEXT:  entry:
-; SCCP-NEXT:    br label [[LOOP:%.*]]
-; SCCP:       loop:
-; SCCP-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[P_NEXT:%.*]], [[BB3:%.*]] ], [ 0, [[BB4:%.*]] ], [ 0, [[BB5:%.*]] ], [ 0, [[BB6:%.*]] ]
-; SCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[BB2:%.*]]
-; SCCP:       bb1:
-; SCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_2]], label [[BB3]], label [[BB4]]
-; SCCP:       bb2:
-; SCCP-NEXT:    [[C_4:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_4]], label [[BB5]], label [[BB6]]
-; SCCP:       bb3:
-; SCCP-NEXT:    [[P_NEXT]] = add i32 [[P]], 1
-; SCCP-NEXT:    br label [[LOOP]]
-; SCCP:       bb4:
-; SCCP-NEXT:    [[C_3:%.*]] = call i1 @cond()
-; SCCP-NEXT:    br i1 [[C_3]], label [[LOOP]], label [[END:%.*]]
-; SCCP:       bb5:
-; SCCP-NEXT:    br label [[LOOP]]
-; SCCP:       bb6:
-; SCCP-NEXT:    br label [[LOOP]]
-; SCCP:       end:
-; SCCP-NEXT:    ret i32 [[P]]
-;
-; IPSCCP-LABEL: @loop_with_multiple_euqal_incomings(
-; IPSCCP-NEXT:  entry:
-; IPSCCP-NEXT:    br label [[LOOP:%.*]]
-; IPSCCP:       loop:
-; IPSCCP-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[P_NEXT:%.*]], [[BB3:%.*]] ], [ 0, [[BB4:%.*]] ], [ 0, [[BB5:%.*]] ], [ 0, [[BB6:%.*]] ]
-; IPSCCP-NEXT:    [[C_1:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[BB2:%.*]]
-; IPSCCP:       bb1:
-; IPSCCP-NEXT:    [[C_2:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_2]], label [[BB3]], label [[BB4]]
-; IPSCCP:       bb2:
-; IPSCCP-NEXT:    [[C_4:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_4]], label [[BB5]], label [[BB6]]
-; IPSCCP:       bb3:
-; IPSCCP-NEXT:    [[P_NEXT]] = add i32 [[P]], 1
-; IPSCCP-NEXT:    br label [[LOOP]]
-; IPSCCP:       bb4:
-; IPSCCP-NEXT:    [[C_3:%.*]] = call i1 @cond()
-; IPSCCP-NEXT:    br i1 [[C_3]], label [[LOOP]], label [[END:%.*]]
-; IPSCCP:       bb5:
-; IPSCCP-NEXT:    br label [[LOOP]]
-; IPSCCP:       bb6:
-; IPSCCP-NEXT:    br label [[LOOP]]
-; IPSCCP:       end:
-; IPSCCP-NEXT:    ret i32 [[P]]
+; CHECK-LABEL: @loop_with_multiple_euqal_incomings(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[P:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[P_NEXT:%.*]], [[BB3:%.*]] ], [ 0, [[BB4:%.*]] ], [ 0, [[BB5:%.*]] ], [ 0, [[BB6:%.*]] ]
+; CHECK-NEXT:    [[C_1:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_1]], label [[BB1:%.*]], label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[C_2:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_2]], label [[BB3]], label [[BB4]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[C_4:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_4]], label [[BB5]], label [[BB6]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[P_NEXT]] = add i32 [[P]], 1
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       bb4:
+; CHECK-NEXT:    [[C_3:%.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 [[C_3]], label [[LOOP]], label [[END:%.*]]
+; CHECK:       bb5:
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       bb6:
+; CHECK-NEXT:    br label [[LOOP]]
+; CHECK:       end:
+; CHECK-NEXT:    ret i32 [[P]]
 ;
 entry:
   br label %loop
@@ -933,3 +654,6 @@ bb6:
 end:
   ret i32 %p
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; IPSCCP: {{.*}}
+; SCCP: {{.*}}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/alternate-vectorization-split-node.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/alternate-vectorization-split-node.ll
index 8d44d03e0e5cc..6d961fc3378b4 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/alternate-vectorization-split-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/alternate-vectorization-split-node.ll
@@ -8,11 +8,8 @@ define i32 @test(ptr %c) {
 ; CHECK-NEXT:    [[BITLEN:%.*]] = getelementptr i8, ptr [[C]], i64 136
 ; CHECK-NEXT:    [[INCDEC_PTR_3_1:%.*]] = getelementptr i8, ptr [[C]], i64 115
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[BITLEN]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 0, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr <6 x i64> [[TMP1]], zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <6 x i64> [[TMP2]], <6 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr <8 x i64> [[TMP1]], zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc <8 x i64> [[TMP5]] to <8 x i8>
 ; CHECK-NEXT:    store <8 x i8> [[TMP6]], ptr [[INCDEC_PTR_3_1]], align 1
 ; CHECK-NEXT:    ret i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/commute.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/commute.ll
index 442769937ac12..9e086dcad686c 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/commute.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/commute.ll
@@ -8,15 +8,18 @@ target triple = "aarch64--linux-gnu"
 define void @test1(ptr nocapture readonly %J, i32 %xmin, i32 %ymin) {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[XMIN:%.*]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[YMIN:%.*]], i32 1
 ; CHECK-NEXT:    br label [[FOR_BODY3_LR_PH:%.*]]
 ; CHECK:       for.body3.lr.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <2 x i32> [[TMP1]] to <2 x float>
-; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[J:%.*]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = fsub fast <2 x float> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast <2 x float> [[TMP5]], [[TMP5]]
-; CHECK-NEXT:    [[ADD:%.*]] = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[TMP6]])
+; CHECK-NEXT:    [[CONV5:%.*]] = sitofp i32 [[YMIN:%.*]] to float
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[XMIN:%.*]] to float
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[J:%.*]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast float [[CONV]], [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [[STRUCTA:%.*]], ptr [[J]], i64 0, i32 0, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[SUB10:%.*]] = fsub fast float [[CONV5]], [[TMP1]]
+; CHECK-NEXT:    [[MUL11:%.*]] = fmul fast float [[SUB]], [[SUB]]
+; CHECK-NEXT:    [[MUL12:%.*]] = fmul fast float [[SUB10]], [[SUB10]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[MUL11]], [[MUL12]]
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ADD]], 0.000000e+00
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY3_LR_PH]], label [[FOR_END27:%.*]]
 ; CHECK:       for.end27:
@@ -47,15 +50,18 @@ for.end27:
 define void @test2(ptr nocapture readonly %J, i32 %xmin, i32 %ymin) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[XMIN:%.*]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[YMIN:%.*]], i32 1
 ; CHECK-NEXT:    br label [[FOR_BODY3_LR_PH:%.*]]
 ; CHECK:       for.body3.lr.ph:
-; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <2 x i32> [[TMP1]] to <2 x float>
-; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x float>, ptr [[J:%.*]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = fsub fast <2 x float> [[TMP2]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast <2 x float> [[TMP5]], [[TMP5]]
-; CHECK-NEXT:    [[ADD:%.*]] = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[TMP6]])
+; CHECK-NEXT:    [[CONV5:%.*]] = sitofp i32 [[YMIN:%.*]] to float
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[XMIN:%.*]] to float
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[J:%.*]], align 4
+; CHECK-NEXT:    [[SUB:%.*]] = fsub fast float [[CONV]], [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [[STRUCTA:%.*]], ptr [[J]], i64 0, i32 0, i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[SUB10:%.*]] = fsub fast float [[CONV5]], [[TMP1]]
+; CHECK-NEXT:    [[MUL11:%.*]] = fmul fast float [[SUB]], [[SUB]]
+; CHECK-NEXT:    [[MUL12:%.*]] = fmul fast float [[SUB10]], [[SUB10]]
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[MUL12]], [[MUL11]]
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ADD]], 0.000000e+00
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY3_LR_PH]], label [[FOR_END27:%.*]]
 ; CHECK:       for.end27:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/exp.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/exp.ll
new file mode 100644
index 0000000000000..301e5da28f0d8
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/exp.ll
@@ -0,0 +1,279 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+define void @ldexp_f32i32(ptr %x, ptr %y, i32 %exp) {
+; CHECK-LABEL: @ldexp_f32i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load float, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L0]], i32 [[EXP:%.*]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L2]], i32 [[EXP]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L4]], i32 [[EXP]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L6]], i32 [[EXP]])
+; CHECK-NEXT:    store float [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 1
+; CHECK-NEXT:    store float [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 2
+; CHECK-NEXT:    store float [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 3
+; CHECK-NEXT:    store float [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load float, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1
+  %l2 = load float, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2
+  %l4 = load float, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3
+  %l6 = load float, ptr %arrayidx.3, align 4
+  %l1 = tail call float @llvm.ldexp.f32.i32(float %l0, i32 %exp)
+  %l3 = tail call float @llvm.ldexp.f32.i32(float %l2, i32 %exp)
+  %l5 = tail call float @llvm.ldexp.f32.i32(float %l4, i32 %exp)
+  %l7 = tail call float @llvm.ldexp.f32.i32(float %l6, i32 %exp)
+  store float %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds float, ptr %y, i64 1
+  store float %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds float, ptr %y, i64 2
+  store float %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds float, ptr %y, i64 3
+  store float %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @ldexp_f64i32(ptr %x, ptr %y, i32 %exp) {
+; CHECK-LABEL: @ldexp_f64i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load double, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L0]], i32 [[EXP:%.*]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L2]], i32 [[EXP]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L4]], i32 [[EXP]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L6]], i32 [[EXP]])
+; CHECK-NEXT:    store double [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1
+; CHECK-NEXT:    store double [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2
+; CHECK-NEXT:    store double [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3
+; CHECK-NEXT:    store double [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load double, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1
+  %l2 = load double, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2
+  %l4 = load double, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3
+  %l6 = load double, ptr %arrayidx.3, align 4
+  %l1 = tail call double @llvm.ldexp.f64.i32(double %l0, i32 %exp)
+  %l3 = tail call double @llvm.ldexp.f64.i32(double %l2, i32 %exp)
+  %l5 = tail call double @llvm.ldexp.f64.i32(double %l4, i32 %exp)
+  %l7 = tail call double @llvm.ldexp.f64.i32(double %l6, i32 %exp)
+  store double %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds double, ptr %y, i64 1
+  store double %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds double, ptr %y, i64 2
+  store double %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds double, ptr %y, i64 3
+  store double %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @ldexp_f32i64(ptr %x, ptr %y, i64 %exp) {
+; CHECK-LABEL: @ldexp_f32i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load float, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L0]], i64 [[EXP:%.*]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L2]], i64 [[EXP]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L4]], i64 [[EXP]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L6]], i64 [[EXP]])
+; CHECK-NEXT:    store float [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 1
+; CHECK-NEXT:    store float [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 2
+; CHECK-NEXT:    store float [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 3
+; CHECK-NEXT:    store float [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load float, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1
+  %l2 = load float, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2
+  %l4 = load float, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3
+  %l6 = load float, ptr %arrayidx.3, align 4
+  %l1 = tail call float @llvm.ldexp.f32.i64(float %l0, i64 %exp)
+  %l3 = tail call float @llvm.ldexp.f32.i64(float %l2, i64 %exp)
+  %l5 = tail call float @llvm.ldexp.f32.i64(float %l4, i64 %exp)
+  %l7 = tail call float @llvm.ldexp.f32.i64(float %l6, i64 %exp)
+  store float %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds float, ptr %y, i64 1
+  store float %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds float, ptr %y, i64 2
+  store float %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds float, ptr %y, i64 3
+  store float %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @ldexp_f64i64(ptr %x, ptr %y, i64 %exp) {
+; CHECK-LABEL: @ldexp_f64i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load double, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L0]], i64 [[EXP:%.*]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L2]], i64 [[EXP]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L4]], i64 [[EXP]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L6]], i64 [[EXP]])
+; CHECK-NEXT:    store double [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1
+; CHECK-NEXT:    store double [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2
+; CHECK-NEXT:    store double [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3
+; CHECK-NEXT:    store double [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load double, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1
+  %l2 = load double, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2
+  %l4 = load double, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3
+  %l6 = load double, ptr %arrayidx.3, align 4
+  %l1 = tail call double @llvm.ldexp.f64.i64(double %l0, i64 %exp)
+  %l3 = tail call double @llvm.ldexp.f64.i64(double %l2, i64 %exp)
+  %l5 = tail call double @llvm.ldexp.f64.i64(double %l4, i64 %exp)
+  %l7 = tail call double @llvm.ldexp.f64.i64(double %l6, i64 %exp)
+  store double %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds double, ptr %y, i64 1
+  store double %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds double, ptr %y, i64 2
+  store double %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds double, ptr %y, i64 3
+  store double %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @ldexp_f32i32_i64(ptr %x, ptr %y, i32 %exp32, i64 %exp64) {
+; CHECK-LABEL: @ldexp_f32i32_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load float, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L0]], i32 [[EXP32:%.*]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call float @llvm.ldexp.f32.i32(float [[L2]], i32 [[EXP32]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L4]], i64 [[EXP64:%.*]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call float @llvm.ldexp.f32.i64(float [[L6]], i64 [[EXP64]])
+; CHECK-NEXT:    store float [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 1
+; CHECK-NEXT:    store float [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 2
+; CHECK-NEXT:    store float [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds float, ptr [[Y]], i64 3
+; CHECK-NEXT:    store float [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load float, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1
+  %l2 = load float, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2
+  %l4 = load float, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3
+  %l6 = load float, ptr %arrayidx.3, align 4
+  %l1 = tail call float @llvm.ldexp.f32.i32(float %l0, i32 %exp32)
+  %l3 = tail call float @llvm.ldexp.f32.i32(float %l2, i32 %exp32)
+  %l5 = tail call float @llvm.ldexp.f32.i64(float %l4, i64 %exp64)
+  %l7 = tail call float @llvm.ldexp.f32.i64(float %l6, i64 %exp64)
+  store float %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds float, ptr %y, i64 1
+  store float %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds float, ptr %y, i64 2
+  store float %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds float, ptr %y, i64 3
+  store float %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @ldexp_f64_i32_i64(ptr %x, ptr %y, i32 %exp32, i64 %exp64) {
+; CHECK-LABEL: @ldexp_f64_i32_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load double, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L0]], i32 [[EXP32:%.*]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call double @llvm.ldexp.f64.i32(double [[L2]], i32 [[EXP32]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L4]], i64 [[EXP64:%.*]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call double @llvm.ldexp.f64.i64(double [[L6]], i64 [[EXP64]])
+; CHECK-NEXT:    store double [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 1
+; CHECK-NEXT:    store double [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 2
+; CHECK-NEXT:    store double [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 3
+; CHECK-NEXT:    store double [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load double, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1
+  %l2 = load double, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2
+  %l4 = load double, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3
+  %l6 = load double, ptr %arrayidx.3, align 4
+  %l1 = tail call double @llvm.ldexp.f64.i32(double %l0, i32 %exp32)
+  %l3 = tail call double @llvm.ldexp.f64.i32(double %l2, i32 %exp32)
+  %l5 = tail call double @llvm.ldexp.f64.i64(double %l4, i64 %exp64)
+  %l7 = tail call double @llvm.ldexp.f64.i64(double %l6, i64 %exp64)
+  store double %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds double, ptr %y, i64 1
+  store double %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds double, ptr %y, i64 2
+  store double %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds double, ptr %y, i64 3
+  store double %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+declare float @llvm.ldexp.f32.i32(float, i32)
+declare double @llvm.ldexp.f64.i32(double, i32)
+declare float @llvm.ldexp.f32.i64(float, i64)
+declare double @llvm.ldexp.f64.i64(double, i64)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fround.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fround.ll
new file mode 100644
index 0000000000000..07a3fe7d0bbc5
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fround.ll
@@ -0,0 +1,280 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64 < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+
+define void @lround_i32f32(ptr %x, ptr %y, i32 %n) {
+; CHECK-LABEL: @lround_i32f32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load float, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call i32 @llvm.lround.i32.f32(float [[L0]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call i32 @llvm.lround.i32.f32(float [[L2]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call i32 @llvm.lround.i32.f32(float [[L4]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call i32 @llvm.lround.i32.f32(float [[L6]])
+; CHECK-NEXT:    store i32 [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 1
+; CHECK-NEXT:    store i32 [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 2
+; CHECK-NEXT:    store i32 [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 3
+; CHECK-NEXT:    store i32 [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load float, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1
+  %l2 = load float, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2
+  %l4 = load float, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3
+  %l6 = load float, ptr %arrayidx.3, align 4
+  %l1 = tail call i32 @llvm.lround.i32.f32(float %l0)
+  %l3 = tail call i32 @llvm.lround.i32.f32(float %l2)
+  %l5 = tail call i32 @llvm.lround.i32.f32(float %l4)
+  %l7 = tail call i32 @llvm.lround.i32.f32(float %l6)
+  store i32 %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds i32, ptr %y, i64 1
+  store i32 %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds i32, ptr %y, i64 2
+  store i32 %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds i32, ptr %y, i64 3
+  store i32 %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @lround_i32f64(ptr %x, ptr %y, i32 %n) {
+; CHECK-LABEL: @lround_i32f64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load double, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call i32 @llvm.lround.i32.f64(double [[L0]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call i32 @llvm.lround.i32.f64(double [[L2]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call i32 @llvm.lround.i32.f64(double [[L4]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call i32 @llvm.lround.i32.f64(double [[L6]])
+; CHECK-NEXT:    store i32 [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 1
+; CHECK-NEXT:    store i32 [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 2
+; CHECK-NEXT:    store i32 [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 3
+; CHECK-NEXT:    store i32 [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load double, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1
+  %l2 = load double, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2
+  %l4 = load double, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3
+  %l6 = load double, ptr %arrayidx.3, align 4
+  %l1 = tail call i32 @llvm.lround.i32.f64(double %l0)
+  %l3 = tail call i32 @llvm.lround.i32.f64(double %l2)
+  %l5 = tail call i32 @llvm.lround.i32.f64(double %l4)
+  %l7 = tail call i32 @llvm.lround.i32.f64(double %l6)
+  store i32 %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds i32, ptr %y, i64 1
+  store i32 %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds i32, ptr %y, i64 2
+  store i32 %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds i32, ptr %y, i64 3
+  store i32 %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @lround_i64f32(ptr %x, ptr %y, i64 %n) {
+; CHECK-LABEL: @lround_i64f32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load float, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call i64 @llvm.lround.i64.f32(float [[L0]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call i64 @llvm.lround.i64.f32(float [[L2]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call i64 @llvm.lround.i64.f32(float [[L4]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call i64 @llvm.lround.i64.f32(float [[L6]])
+; CHECK-NEXT:    store i64 [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 1
+; CHECK-NEXT:    store i64 [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 2
+; CHECK-NEXT:    store i64 [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 3
+; CHECK-NEXT:    store i64 [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load float, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1
+  %l2 = load float, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2
+  %l4 = load float, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3
+  %l6 = load float, ptr %arrayidx.3, align 4
+  %l1 = tail call i64 @llvm.lround.i64.f32(float %l0)
+  %l3 = tail call i64 @llvm.lround.i64.f32(float %l2)
+  %l5 = tail call i64 @llvm.lround.i64.f32(float %l4)
+  %l7 = tail call i64 @llvm.lround.i64.f32(float %l6)
+  store i64 %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds i64, ptr %y, i64 1
+  store i64 %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds i64, ptr %y, i64 2
+  store i64 %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds i64, ptr %y, i64 3
+  store i64 %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @lround_i64f64(ptr %x, ptr %y, i64 %n) {
+; CHECK-LABEL: @lround_i64f64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load double, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call i64 @llvm.lround.i64.f64(double [[L0]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call i64 @llvm.lround.i64.f64(double [[L2]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call i64 @llvm.lround.i64.f64(double [[L4]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call i64 @llvm.lround.i64.f64(double [[L6]])
+; CHECK-NEXT:    store i64 [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 1
+; CHECK-NEXT:    store i64 [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 2
+; CHECK-NEXT:    store i64 [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 3
+; CHECK-NEXT:    store i64 [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load double, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1
+  %l2 = load double, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2
+  %l4 = load double, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3
+  %l6 = load double, ptr %arrayidx.3, align 4
+  %l1 = tail call i64 @llvm.lround.i64.f64(double %l0)
+  %l3 = tail call i64 @llvm.lround.i64.f64(double %l2)
+  %l5 = tail call i64 @llvm.lround.i64.f64(double %l4)
+  %l7 = tail call i64 @llvm.lround.i64.f64(double %l6)
+  store i64 %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds i64, ptr %y, i64 1
+  store i64 %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds i64, ptr %y, i64 2
+  store i64 %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds i64, ptr %y, i64 3
+  store i64 %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @llround_i64f32(ptr %x, ptr %y, i64 %n) {
+; CHECK-LABEL: @llround_i64f32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load float, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load float, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load float, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load float, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call i64 @llvm.llround.i64.f32(float [[L0]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call i64 @llvm.llround.i64.f32(float [[L2]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call i64 @llvm.llround.i64.f32(float [[L4]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call i64 @llvm.llround.i64.f32(float [[L6]])
+; CHECK-NEXT:    store i64 [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 1
+; CHECK-NEXT:    store i64 [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 2
+; CHECK-NEXT:    store i64 [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 3
+; CHECK-NEXT:    store i64 [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load float, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds float, ptr %x, i64 1
+  %l2 = load float, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds float, ptr %x, i64 2
+  %l4 = load float, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds float, ptr %x, i64 3
+  %l6 = load float, ptr %arrayidx.3, align 4
+  %l1 = tail call i64 @llvm.llround.i64.f32(float %l0)
+  %l3 = tail call i64 @llvm.llround.i64.f32(float %l2)
+  %l5 = tail call i64 @llvm.llround.i64.f32(float %l4)
+  %l7 = tail call i64 @llvm.llround.i64.f32(float %l6)
+  store i64 %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds i64, ptr %y, i64 1
+  store i64 %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds i64, ptr %y, i64 2
+  store i64 %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds i64, ptr %y, i64 3
+  store i64 %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+define void @llround_i64f64(ptr %x, ptr %y, i64 %n) {
+; CHECK-LABEL: @llround_i64f64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load double, ptr [[X:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 1
+; CHECK-NEXT:    [[L2:%.*]] = load double, ptr [[ARRAYIDX_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 2
+; CHECK-NEXT:    [[L4:%.*]] = load double, ptr [[ARRAYIDX_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 3
+; CHECK-NEXT:    [[L6:%.*]] = load double, ptr [[ARRAYIDX_3]], align 4
+; CHECK-NEXT:    [[L1:%.*]] = tail call i64 @llvm.llround.i64.f64(double [[L0]])
+; CHECK-NEXT:    [[L3:%.*]] = tail call i64 @llvm.llround.i64.f64(double [[L2]])
+; CHECK-NEXT:    [[L5:%.*]] = tail call i64 @llvm.llround.i64.f64(double [[L4]])
+; CHECK-NEXT:    [[L7:%.*]] = tail call i64 @llvm.llround.i64.f64(double [[L6]])
+; CHECK-NEXT:    store i64 [[L1]], ptr [[Y:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 1
+; CHECK-NEXT:    store i64 [[L3]], ptr [[ARRAYIDX2_1]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 2
+; CHECK-NEXT:    store i64 [[L5]], ptr [[ARRAYIDX2_2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i64, ptr [[Y]], i64 3
+; CHECK-NEXT:    store i64 [[L7]], ptr [[ARRAYIDX2_3]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %l0 = load double, ptr %x, align 4
+  %arrayidx.1 = getelementptr inbounds double, ptr %x, i64 1
+  %l2 = load double, ptr %arrayidx.1, align 4
+  %arrayidx.2 = getelementptr inbounds double, ptr %x, i64 2
+  %l4 = load double, ptr %arrayidx.2, align 4
+  %arrayidx.3 = getelementptr inbounds double, ptr %x, i64 3
+  %l6 = load double, ptr %arrayidx.3, align 4
+  %l1 = tail call i64 @llvm.llround.i64.f64(double %l0)
+  %l3 = tail call i64 @llvm.llround.i64.f64(double %l2)
+  %l5 = tail call i64 @llvm.llround.i64.f64(double %l4)
+  %l7 = tail call i64 @llvm.llround.i64.f64(double %l6)
+  store i64 %l1, ptr %y, align 4
+  %arrayidx2.1 = getelementptr inbounds i64, ptr %y, i64 1
+  store i64 %l3, ptr %arrayidx2.1, align 4
+  %arrayidx2.2 = getelementptr inbounds i64, ptr %y, i64 2
+  store i64 %l5, ptr %arrayidx2.2, align 4
+  %arrayidx2.3 = getelementptr inbounds i64, ptr %y, i64 3
+  store i64 %l7, ptr %arrayidx2.3, align 4
+  ret void
+}
+
+declare i32 @llvm.lround.i32.f32(float)
+declare i64 @llvm.lround.i64.f32(float)
+declare i64 @llvm.lround.i64.f64(double)
+declare i64 @llvm.llround.i64.f32(float)
+declare i64 @llvm.llround.i64.f64(double)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll
index 295a71899c338..26ce0fc6e6a3b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reused-scalar-repeated-in-node.ll
@@ -12,7 +12,7 @@ define void @test() {
 ; CHECK:       [[BB63]]:
 ; CHECK-NEXT:    br label %[[BB64]]
 ; CHECK:       [[BB64]]:
-; CHECK-NEXT:    [[TMP25:%.*]] = phi <16 x float> [ poison, %[[BB61]] ], [ poison, %[[BB63]] ], [ poison, %[[BB62]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <16 x float> [ poison, %[[BB61]] ], [ poison, %[[BB63]] ], [ poison, %[[BB62]] ]
 ; CHECK-NEXT:    [[I66:%.*]] = load float, ptr poison, align 16
 ; CHECK-NEXT:    [[I67:%.*]] = load float, ptr poison, align 4
 ; CHECK-NEXT:    [[I68:%.*]] = load float, ptr poison, align 8
@@ -38,43 +38,43 @@ define void @test() {
 ; CHECK-NEXT:    br i1 poison, label %[[BB167:.*]], label %[[BB77:.*]]
 ; CHECK:       [[BB77]]:
 ; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <16 x float> [[TMP11]], <16 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 14, i32 15, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <8 x float> poison, float [[I70]], i32 0
-; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP17]], <8 x i32> <i32 8, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x float> poison, float [[I70]], i32 1
-; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <8 x float> [[TMP14]], float [[I68]], i32 2
-; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <8 x float> [[TMP19]], float [[I66]], i32 3
-; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <8 x float> [[TMP16]], float [[I67]], i32 6
-; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <8 x float> [[TMP20]], float [[I69]], i32 7
-; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <16 x float> [[TMP25]], <16 x float> poison, <16 x i32> <i32 poison, i32 poison, i32 3, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP13:%.*]] = shufflevector <16 x float> [[TMP39]], <16 x float> [[TMP25]], <16 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 18, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 19, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x float> poison, float [[I70]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <8 x float> [[TMP12]], <8 x float> [[TMP13]], <8 x i32> <i32 8, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x float> poison, float [[I70]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <8 x float> [[TMP15]], float [[I68]], i32 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <8 x float> [[TMP16]], float [[I66]], i32 3
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <8 x float> [[TMP17]], float [[I67]], i32 6
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <8 x float> [[TMP18]], float [[I69]], i32 7
+; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <16 x float> [[TMP0]], <16 x float> poison, <16 x i32> <i32 poison, i32 poison, i32 3, i32 2, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP21:%.*]] = shufflevector <16 x float> [[TMP20]], <16 x float> [[TMP0]], <16 x i32> <i32 poison, i32 poison, i32 2, i32 3, i32 18, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 19, i32 poison, i32 poison>
 ; CHECK-NEXT:    br label %[[BB78:.*]]
 ; CHECK:       [[BB78]]:
-; CHECK-NEXT:    [[TMP15:%.*]] = phi <8 x float> [ [[TMP23]], %[[BB77]] ], [ [[TMP36:%.*]], %[[BB78]] ]
-; CHECK-NEXT:    [[TMP22:%.*]] = phi <8 x float> [ [[TMP21]], %[[BB77]] ], [ [[TMP31:%.*]], %[[BB78]] ]
-; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <8 x float> [[TMP22]], <8 x float> poison, <16 x i32> <i32 0, i32 3, i32 1, i32 2, i32 3, i32 0, i32 2, i32 3, i32 2, i32 6, i32 2, i32 3, i32 0, i32 7, i32 6, i32 6>
-; CHECK-NEXT:    [[TMP38:%.*]] = shufflevector <8 x float> [[TMP15]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 1, i32 0, i32 3, i32 1, i32 3, i32 5, i32 3, i32 1, i32 0, i32 4, i32 5, i32 5>
-; CHECK-NEXT:    [[TMP18:%.*]] = fmul fast <16 x float> [[TMP24]], [[TMP13]]
-; CHECK-NEXT:    [[TMP26:%.*]] = fmul fast <16 x float> [[TMP38]], [[TMP25]]
-; CHECK-NEXT:    [[TMP27:%.*]] = fadd fast <16 x float> [[TMP26]], [[TMP18]]
-; CHECK-NEXT:    [[TMP28:%.*]] = fadd fast <16 x float> [[TMP27]], poison
+; CHECK-NEXT:    [[TMP22:%.*]] = phi <8 x float> [ [[TMP14]], %[[BB77]] ], [ [[TMP31:%.*]], %[[BB78]] ]
+; CHECK-NEXT:    [[TMP23:%.*]] = phi <8 x float> [ [[TMP19]], %[[BB77]] ], [ [[TMP32:%.*]], %[[BB78]] ]
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <8 x float> [[TMP23]], <8 x float> poison, <16 x i32> <i32 0, i32 3, i32 1, i32 2, i32 3, i32 0, i32 2, i32 3, i32 2, i32 6, i32 2, i32 3, i32 0, i32 7, i32 6, i32 6>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <8 x float> [[TMP22]], <8 x float> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 1, i32 0, i32 3, i32 1, i32 3, i32 5, i32 3, i32 1, i32 0, i32 4, i32 5, i32 5>
+; CHECK-NEXT:    [[TMP26:%.*]] = fmul fast <16 x float> [[TMP24]], [[TMP21]]
+; CHECK-NEXT:    [[TMP27:%.*]] = fmul fast <16 x float> [[TMP25]], [[TMP0]]
+; CHECK-NEXT:    [[TMP28:%.*]] = fadd fast <16 x float> [[TMP27]], [[TMP26]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = fadd fast <16 x float> [[TMP28]], poison
-; CHECK-NEXT:    [[TMP36]] = shufflevector <16 x float> [[TMP29]], <16 x float> poison, <8 x i32> <i32 5, i32 11, i32 12, i32 10, i32 14, i32 15, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP31]] = shufflevector <16 x float> [[TMP29]], <16 x float> poison, <8 x i32> <i32 12, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP30:%.*]] = fadd fast <16 x float> [[TMP29]], poison
+; CHECK-NEXT:    [[TMP31]] = shufflevector <16 x float> [[TMP30]], <16 x float> poison, <8 x i32> <i32 5, i32 11, i32 12, i32 10, i32 14, i32 15, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP32]] = shufflevector <16 x float> [[TMP30]], <16 x float> poison, <8 x i32> <i32 12, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 14, i32 15>
 ; CHECK-NEXT:    br i1 poison, label %[[BB78]], label %[[BB167]]
 ; CHECK:       [[BB167]]:
-; CHECK-NEXT:    [[TMP32:%.*]] = phi <16 x float> [ [[TMP11]], %[[BB64]] ], [ [[TMP29]], %[[BB78]] ]
-; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x float> [[TMP32]], i32 14
+; CHECK-NEXT:    [[TMP35:%.*]] = phi <16 x float> [ [[TMP11]], %[[BB64]] ], [ [[TMP30]], %[[BB78]] ]
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x float> [[TMP35]], i32 14
 ; CHECK-NEXT:    store float [[TMP33]], ptr poison, align 1
-; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x float> [[TMP32]], i32 13
+; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x float> [[TMP35]], i32 13
 ; CHECK-NEXT:    store float [[TMP34]], ptr poison, align 1
-; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x float> [[TMP32]], i32 15
+; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x float> [[TMP35]], i32 15
 ; CHECK-NEXT:    br i1 poison, label %[[BB186:.*]], label %[[BB184:.*]]
 ; CHECK:       [[BB184]]:
 ; CHECK-NEXT:    br label %[[BB185:.*]]
 ; CHECK:       [[BB185]]:
 ; CHECK-NEXT:    br i1 poison, label %[[BB185]], label %[[BB186]]
 ; CHECK:       [[BB186]]:
-; CHECK-NEXT:    [[I187:%.*]] = phi nsz float [ [[TMP35]], %[[BB167]] ], [ poison, %[[BB185]] ]
+; CHECK-NEXT:    [[I187:%.*]] = phi nsz float [ [[TMP36]], %[[BB167]] ], [ poison, %[[BB185]] ]
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/scalarization-overhead.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/scalarization-overhead.ll
index 64bdcf28af550..5fb468c0b0962 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/scalarization-overhead.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/scalarization-overhead.ll
@@ -15,28 +15,28 @@ define fastcc i64 @zot(float %arg, float %arg1, float %arg2, float %arg3, float
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> <float 0.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00>, [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> <float poison, float 0.000000e+00>, float [[ARG3]], i32 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast <2 x float> [[TMP4]], <float 1.000000e+00, float 0.000000e+00>
-; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP9]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <4 x float> [[TMP6]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP5]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP6]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd fast <4 x float> [[TMP7]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>
 ; CHECK-NEXT:    br i1 [[ARG6:%.*]], label [[BB18:%.*]], label [[BB57:%.*]]
 ; CHECK:       bb18:
-; CHECK-NEXT:    [[TMP8:%.*]] = phi <4 x float> [ [[TMP7]], [[BB:%.*]] ]
-; CHECK-NEXT:    [[VAL16:%.*]] = extractelement <4 x float> [[TMP7]], i32 2
+; CHECK-NEXT:    [[TMP9:%.*]] = phi <4 x float> [ [[TMP8]], [[BB:%.*]] ]
+; CHECK-NEXT:    [[VAL16:%.*]] = extractelement <4 x float> [[TMP8]], i32 2
 ; CHECK-NEXT:    [[VAL23:%.*]] = fmul fast float [[VAL16]], 2.000000e+00
-; CHECK-NEXT:    [[VAL17:%.*]] = extractelement <4 x float> [[TMP7]], i32 3
+; CHECK-NEXT:    [[VAL17:%.*]] = extractelement <4 x float> [[TMP8]], i32 3
 ; CHECK-NEXT:    [[VAL24:%.*]] = fmul fast float [[VAL17]], 3.000000e+00
 ; CHECK-NEXT:    br i1 [[ARG7:%.*]], label [[BB25:%.*]], label [[BB57]]
 ; CHECK:       bb25:
-; CHECK-NEXT:    [[TMP11:%.*]] = phi <4 x float> [ [[TMP8]], [[BB18]] ]
+; CHECK-NEXT:    [[TMP12:%.*]] = phi <4 x float> [ [[TMP9]], [[BB18]] ]
 ; CHECK-NEXT:    br label [[BB30:%.*]]
 ; CHECK:       bb30:
 ; CHECK-NEXT:    [[VAL31:%.*]] = phi float [ [[VAL55:%.*]], [[BB30]] ], [ 0.000000e+00, [[BB25]] ]
 ; CHECK-NEXT:    [[VAL32:%.*]] = phi float [ [[VAL9]], [[BB30]] ], [ 0.000000e+00, [[BB25]] ]
-; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i8>, ptr [[ARG5:%.*]], align 1
-; CHECK-NEXT:    [[TMP13:%.*]] = uitofp <4 x i8> [[TMP12]] to <4 x float>
-; CHECK-NEXT:    [[TMP14:%.*]] = fsub fast <4 x float> [[TMP13]], [[TMP3]]
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul fast <4 x float> [[TMP14]], [[TMP11]]
-; CHECK-NEXT:    [[VAL54:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP15]])
+; CHECK-NEXT:    [[TMP13:%.*]] = load <4 x i8>, ptr [[ARG5:%.*]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = uitofp <4 x i8> [[TMP13]] to <4 x float>
+; CHECK-NEXT:    [[TMP15:%.*]] = fsub fast <4 x float> [[TMP14]], [[TMP3]]
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul fast <4 x float> [[TMP15]], [[TMP12]]
+; CHECK-NEXT:    [[VAL54:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP16]])
 ; CHECK-NEXT:    [[VAL55]] = tail call fast float @llvm.minnum.f32(float [[VAL31]], float [[ARG1:%.*]])
 ; CHECK-NEXT:    [[VAL56:%.*]] = tail call fast float @llvm.maxnum.f32(float [[ARG2:%.*]], float [[VAL54]])
 ; CHECK-NEXT:    call void @ham(float [[VAL55]], float [[VAL56]])
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
index 47153d91956d5..683b92752c702 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
@@ -191,12 +191,12 @@ define i32 @reorder_indices_1(float %0) {
 ; NON-POW2-NEXT:  entry:
 ; NON-POW2-NEXT:    [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4
 ; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x float>, ptr [[NOR1]], align 4
-; NON-POW2-NEXT:    [[TMP3:%.*]] = fneg <3 x float> [[TMP1]]
+; NON-POW2-NEXT:    [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; NON-POW2-NEXT:    [[TMP3:%.*]] = fneg <3 x float> [[TMP2]]
 ; NON-POW2-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0
 ; NON-POW2-NEXT:    [[TMP5:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> poison, <3 x i32> zeroinitializer
 ; NON-POW2-NEXT:    [[TMP6:%.*]] = fmul <3 x float> [[TMP3]], [[TMP5]]
-; NON-POW2-NEXT:    [[TMP10:%.*]] = shufflevector <3 x float> [[TMP6]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
-; NON-POW2-NEXT:    [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> zeroinitializer, <3 x float> [[TMP10]])
+; NON-POW2-NEXT:    [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> zeroinitializer, <3 x float> [[TMP6]])
 ; NON-POW2-NEXT:    [[TMP8:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP5]], <3 x float> [[TMP7]], <3 x float> zeroinitializer)
 ; NON-POW2-NEXT:    [[TMP9:%.*]] = fmul <3 x float> [[TMP8]], zeroinitializer
 ; NON-POW2-NEXT:    store <3 x float> [[TMP9]], ptr [[NOR1]], align 4
@@ -402,19 +402,32 @@ entry:
 }
 
 define void @reuse_shuffle_indices_cost_crash_2(ptr %bezt, float %0) {
-; CHECK-LABEL: define void @reuse_shuffle_indices_cost_crash_2(
-; CHECK-SAME: ptr [[BEZT:%.*]], float [[TMP0:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[FNEG:%.*]] = fmul float [[TMP0]], 0.000000e+00
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float [[FNEG]], float 0.000000e+00)
-; CHECK-NEXT:    store float [[TMP1]], ptr [[BEZT]], align 4
-; CHECK-NEXT:    [[ARRAYIDX5_I:%.*]] = getelementptr float, ptr [[BEZT]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> <float poison, float 0.000000e+00>, float [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[FNEG]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> [[TMP4]], <2 x float> zeroinitializer)
-; CHECK-NEXT:    store <2 x float> [[TMP5]], ptr [[ARRAYIDX5_I]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @reuse_shuffle_indices_cost_crash_2(
+; NON-POW2-SAME: ptr [[BEZT:%.*]], float [[TMP0:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[FNEG:%.*]] = fmul float [[TMP0]], 0.000000e+00
+; NON-POW2-NEXT:    [[TMP1:%.*]] = insertelement <3 x float> poison, float [[FNEG]], i32 0
+; NON-POW2-NEXT:    [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> zeroinitializer
+; NON-POW2-NEXT:    [[TMP3:%.*]] = insertelement <3 x float> <float poison, float poison, float 0.000000e+00>, float [[TMP0]], i32 0
+; NON-POW2-NEXT:    [[TMP4:%.*]] = shufflevector <3 x float> [[TMP3]], <3 x float> poison, <3 x i32> <i32 0, i32 0, i32 2>
+; NON-POW2-NEXT:    [[TMP5:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP2]], <3 x float> [[TMP4]], <3 x float> zeroinitializer)
+; NON-POW2-NEXT:    store <3 x float> [[TMP5]], ptr [[BEZT]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @reuse_shuffle_indices_cost_crash_2(
+; POW2-ONLY-SAME: ptr [[BEZT:%.*]], float [[TMP0:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[FNEG:%.*]] = fmul float [[TMP0]], 0.000000e+00
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[FNEG]], i32 0
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> [[TMP4]], <2 x float> zeroinitializer)
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP5]], ptr [[BEZT]], align 4
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = tail call float @llvm.fmuladd.f32(float [[FNEG]], float 0.000000e+00, float 0.000000e+00)
+; POW2-ONLY-NEXT:    [[ARRAYIDX8_I831:%.*]] = getelementptr float, ptr [[BEZT]], i64 2
+; POW2-ONLY-NEXT:    store float [[TMP6]], ptr [[ARRAYIDX8_I831]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %fneg = fmul float %0, 0.000000e+00
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll
new file mode 100644
index 0000000000000..645dbc49269f0
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/basic-strided-loads.ll
@@ -0,0 +1,741 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+
+; RUN: opt -mtriple=riscv64 -mattr=+m,+v -passes=slp-vectorizer -S < %s | FileCheck %s
+
+define void @const_stride_1_no_reordering(ptr %pl, ptr %ps) {
+; CHECK-LABEL: define void @const_stride_1_no_reordering(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 0
+; CHECK-NEXT:    [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[GEP_L0]], align 16
+; CHECK-NEXT:    store <16 x i8> [[TMP1]], ptr [[GEP_S0]], align 16
+; CHECK-NEXT:    ret void
+;
+  %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 0
+  %gep_l1 = getelementptr inbounds i8, ptr %pl, i64 1
+  %gep_l2 = getelementptr inbounds i8, ptr %pl, i64 2
+  %gep_l3 = getelementptr inbounds i8, ptr %pl, i64 3
+  %gep_l4 = getelementptr inbounds i8, ptr %pl, i64 4
+  %gep_l5 = getelementptr inbounds i8, ptr %pl, i64 5
+  %gep_l6 = getelementptr inbounds i8, ptr %pl, i64 6
+  %gep_l7 = getelementptr inbounds i8, ptr %pl, i64 7
+  %gep_l8 = getelementptr inbounds i8, ptr %pl, i64 8
+  %gep_l9 = getelementptr inbounds i8, ptr %pl, i64 9
+  %gep_l10 = getelementptr inbounds i8, ptr %pl, i64 10
+  %gep_l11 = getelementptr inbounds i8, ptr %pl, i64 11
+  %gep_l12 = getelementptr inbounds i8, ptr %pl, i64 12
+  %gep_l13 = getelementptr inbounds i8, ptr %pl, i64 13
+  %gep_l14 = getelementptr inbounds i8, ptr %pl, i64 14
+  %gep_l15 = getelementptr inbounds i8, ptr %pl, i64 15
+
+  %load0  = load i8, ptr %gep_l0 , align 16
+  %load1  = load i8, ptr %gep_l1 , align 16
+  %load2  = load i8, ptr %gep_l2 , align 16
+  %load3  = load i8, ptr %gep_l3 , align 16
+  %load4  = load i8, ptr %gep_l4 , align 16
+  %load5  = load i8, ptr %gep_l5 , align 16
+  %load6  = load i8, ptr %gep_l6 , align 16
+  %load7  = load i8, ptr %gep_l7 , align 16
+  %load8  = load i8, ptr %gep_l8 , align 16
+  %load9  = load i8, ptr %gep_l9 , align 16
+  %load10 = load i8, ptr %gep_l10, align 16
+  %load11 = load i8, ptr %gep_l11, align 16
+  %load12 = load i8, ptr %gep_l12, align 16
+  %load13 = load i8, ptr %gep_l13, align 16
+  %load14 = load i8, ptr %gep_l14, align 16
+  %load15 = load i8, ptr %gep_l15, align 16
+
+  %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+  %gep_s1 = getelementptr inbounds i8, ptr %ps, i64 1
+  %gep_s2 = getelementptr inbounds i8, ptr %ps, i64 2
+  %gep_s3 = getelementptr inbounds i8, ptr %ps, i64 3
+  %gep_s4 = getelementptr inbounds i8, ptr %ps, i64 4
+  %gep_s5 = getelementptr inbounds i8, ptr %ps, i64 5
+  %gep_s6 = getelementptr inbounds i8, ptr %ps, i64 6
+  %gep_s7 = getelementptr inbounds i8, ptr %ps, i64 7
+  %gep_s8 = getelementptr inbounds i8, ptr %ps, i64 8
+  %gep_s9 = getelementptr inbounds i8, ptr %ps, i64 9
+  %gep_s10 = getelementptr inbounds i8, ptr %ps, i64 10
+  %gep_s11 = getelementptr inbounds i8, ptr %ps, i64 11
+  %gep_s12 = getelementptr inbounds i8, ptr %ps, i64 12
+  %gep_s13 = getelementptr inbounds i8, ptr %ps, i64 13
+  %gep_s14 = getelementptr inbounds i8, ptr %ps, i64 14
+  %gep_s15 = getelementptr inbounds i8, ptr %ps, i64 15
+
+  store i8 %load0, ptr %gep_s0, align 16
+  store i8 %load1, ptr %gep_s1, align 16
+  store i8 %load2, ptr %gep_s2, align 16
+  store i8 %load3, ptr %gep_s3, align 16
+  store i8 %load4, ptr %gep_s4, align 16
+  store i8 %load5, ptr %gep_s5, align 16
+  store i8 %load6, ptr %gep_s6, align 16
+  store i8 %load7, ptr %gep_s7, align 16
+  store i8 %load8, ptr %gep_s8, align 16
+  store i8 %load9, ptr %gep_s9, align 16
+  store i8 %load10, ptr %gep_s10, align 16
+  store i8 %load11, ptr %gep_s11, align 16
+  store i8 %load12, ptr %gep_s12, align 16
+  store i8 %load13, ptr %gep_s13, align 16
+  store i8 %load14, ptr %gep_s14, align 16
+  store i8 %load15, ptr %gep_s15, align 16
+
+  ret void
+}
+
+define void @const_stride_1_with_reordering(ptr %pl, ptr %ps) {
+; CHECK-LABEL: define void @const_stride_1_with_reordering(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 0
+; CHECK-NEXT:    [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[GEP_L0]], align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], ptr [[GEP_S0]], align 16
+; CHECK-NEXT:    ret void
+;
+  %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 0
+  %gep_l1 = getelementptr inbounds i8, ptr %pl, i64 1
+  %gep_l2 = getelementptr inbounds i8, ptr %pl, i64 2
+  %gep_l3 = getelementptr inbounds i8, ptr %pl, i64 3
+  %gep_l4 = getelementptr inbounds i8, ptr %pl, i64 4
+  %gep_l5 = getelementptr inbounds i8, ptr %pl, i64 5
+  %gep_l6 = getelementptr inbounds i8, ptr %pl, i64 6
+  %gep_l7 = getelementptr inbounds i8, ptr %pl, i64 7
+  %gep_l8 = getelementptr inbounds i8, ptr %pl, i64 8
+  %gep_l9 = getelementptr inbounds i8, ptr %pl, i64 9
+  %gep_l10 = getelementptr inbounds i8, ptr %pl, i64 10
+  %gep_l11 = getelementptr inbounds i8, ptr %pl, i64 11
+  %gep_l12 = getelementptr inbounds i8, ptr %pl, i64 12
+  %gep_l13 = getelementptr inbounds i8, ptr %pl, i64 13
+  %gep_l14 = getelementptr inbounds i8, ptr %pl, i64 14
+  %gep_l15 = getelementptr inbounds i8, ptr %pl, i64 15
+
+  %load0  = load i8, ptr %gep_l0 , align 16
+  %load1  = load i8, ptr %gep_l1 , align 16
+  %load2  = load i8, ptr %gep_l2 , align 16
+  %load3  = load i8, ptr %gep_l3 , align 16
+  %load4  = load i8, ptr %gep_l4 , align 16
+  %load5  = load i8, ptr %gep_l5 , align 16
+  %load6  = load i8, ptr %gep_l6 , align 16
+  %load7  = load i8, ptr %gep_l7 , align 16
+  %load8  = load i8, ptr %gep_l8 , align 16
+  %load9  = load i8, ptr %gep_l9 , align 16
+  %load10 = load i8, ptr %gep_l10, align 16
+  %load11 = load i8, ptr %gep_l11, align 16
+  %load12 = load i8, ptr %gep_l12, align 16
+  %load13 = load i8, ptr %gep_l13, align 16
+  %load14 = load i8, ptr %gep_l14, align 16
+  %load15 = load i8, ptr %gep_l15, align 16
+
+  %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+  %gep_s1 = getelementptr inbounds i8, ptr %ps, i64 1
+  %gep_s2 = getelementptr inbounds i8, ptr %ps, i64 2
+  %gep_s3 = getelementptr inbounds i8, ptr %ps, i64 3
+  %gep_s4 = getelementptr inbounds i8, ptr %ps, i64 4
+  %gep_s5 = getelementptr inbounds i8, ptr %ps, i64 5
+  %gep_s6 = getelementptr inbounds i8, ptr %ps, i64 6
+  %gep_s7 = getelementptr inbounds i8, ptr %ps, i64 7
+  %gep_s8 = getelementptr inbounds i8, ptr %ps, i64 8
+  %gep_s9 = getelementptr inbounds i8, ptr %ps, i64 9
+  %gep_s10 = getelementptr inbounds i8, ptr %ps, i64 10
+  %gep_s11 = getelementptr inbounds i8, ptr %ps, i64 11
+  %gep_s12 = getelementptr inbounds i8, ptr %ps, i64 12
+  %gep_s13 = getelementptr inbounds i8, ptr %ps, i64 13
+  %gep_s14 = getelementptr inbounds i8, ptr %ps, i64 14
+  %gep_s15 = getelementptr inbounds i8, ptr %ps, i64 15
+
+  ; NOTE: value from %load1 in stored in  %gep_s0
+  store i8 %load1, ptr %gep_s0, align 16
+  store i8 %load0, ptr %gep_s1, align 16
+  store i8 %load2, ptr %gep_s2, align 16
+  store i8 %load3, ptr %gep_s3, align 16
+  store i8 %load4, ptr %gep_s4, align 16
+  store i8 %load5, ptr %gep_s5, align 16
+  store i8 %load6, ptr %gep_s6, align 16
+  store i8 %load7, ptr %gep_s7, align 16
+  store i8 %load8, ptr %gep_s8, align 16
+  store i8 %load9, ptr %gep_s9, align 16
+  store i8 %load10, ptr %gep_s10, align 16
+  store i8 %load11, ptr %gep_s11, align 16
+  store i8 %load12, ptr %gep_s12, align 16
+  store i8 %load13, ptr %gep_s13, align 16
+  store i8 %load14, ptr %gep_s14, align 16
+  store i8 %load15, ptr %gep_s15, align 16
+
+  ret void
+}
+
+
+define void @const_stride_2_no_reordering(ptr %pl, ptr %ps) {
+; CHECK-LABEL: define void @const_stride_2_no_reordering(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 0
+; CHECK-NEXT:    [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = call <31 x i8> @llvm.masked.load.v31i8.p0(ptr [[GEP_L0]], i32 16, <31 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <31 x i8> poison)
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <31 x i8> [[TMP2]], <31 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:    store <16 x i8> [[TMP1]], ptr [[GEP_S0]], align 16
+; CHECK-NEXT:    ret void
+;
+  %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 0
+  %gep_l1 = getelementptr inbounds i8, ptr %pl, i64 2
+  %gep_l2 = getelementptr inbounds i8, ptr %pl, i64 4
+  %gep_l3 = getelementptr inbounds i8, ptr %pl, i64 6
+  %gep_l4 = getelementptr inbounds i8, ptr %pl, i64 8
+  %gep_l5 = getelementptr inbounds i8, ptr %pl, i64 10
+  %gep_l6 = getelementptr inbounds i8, ptr %pl, i64 12
+  %gep_l7 = getelementptr inbounds i8, ptr %pl, i64 14
+  %gep_l8 = getelementptr inbounds i8, ptr %pl, i64 16
+  %gep_l9 = getelementptr inbounds i8, ptr %pl, i64 18
+  %gep_l10 = getelementptr inbounds i8, ptr %pl, i64 20
+  %gep_l11 = getelementptr inbounds i8, ptr %pl, i64 22
+  %gep_l12 = getelementptr inbounds i8, ptr %pl, i64 24
+  %gep_l13 = getelementptr inbounds i8, ptr %pl, i64 26
+  %gep_l14 = getelementptr inbounds i8, ptr %pl, i64 28
+  %gep_l15 = getelementptr inbounds i8, ptr %pl, i64 30
+
+  %load0  = load i8, ptr %gep_l0 , align 16
+  %load1  = load i8, ptr %gep_l1 , align 16
+  %load2  = load i8, ptr %gep_l2 , align 16
+  %load3  = load i8, ptr %gep_l3 , align 16
+  %load4  = load i8, ptr %gep_l4 , align 16
+  %load5  = load i8, ptr %gep_l5 , align 16
+  %load6  = load i8, ptr %gep_l6 , align 16
+  %load7  = load i8, ptr %gep_l7 , align 16
+  %load8  = load i8, ptr %gep_l8 , align 16
+  %load9  = load i8, ptr %gep_l9 , align 16
+  %load10 = load i8, ptr %gep_l10, align 16
+  %load11 = load i8, ptr %gep_l11, align 16
+  %load12 = load i8, ptr %gep_l12, align 16
+  %load13 = load i8, ptr %gep_l13, align 16
+  %load14 = load i8, ptr %gep_l14, align 16
+  %load15 = load i8, ptr %gep_l15, align 16
+
+  %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+  %gep_s1 = getelementptr inbounds i8, ptr %ps, i64 1
+  %gep_s2 = getelementptr inbounds i8, ptr %ps, i64 2
+  %gep_s3 = getelementptr inbounds i8, ptr %ps, i64 3
+  %gep_s4 = getelementptr inbounds i8, ptr %ps, i64 4
+  %gep_s5 = getelementptr inbounds i8, ptr %ps, i64 5
+  %gep_s6 = getelementptr inbounds i8, ptr %ps, i64 6
+  %gep_s7 = getelementptr inbounds i8, ptr %ps, i64 7
+  %gep_s8 = getelementptr inbounds i8, ptr %ps, i64 8
+  %gep_s9 = getelementptr inbounds i8, ptr %ps, i64 9
+  %gep_s10 = getelementptr inbounds i8, ptr %ps, i64 10
+  %gep_s11 = getelementptr inbounds i8, ptr %ps, i64 11
+  %gep_s12 = getelementptr inbounds i8, ptr %ps, i64 12
+  %gep_s13 = getelementptr inbounds i8, ptr %ps, i64 13
+  %gep_s14 = getelementptr inbounds i8, ptr %ps, i64 14
+  %gep_s15 = getelementptr inbounds i8, ptr %ps, i64 15
+
+  store i8 %load0, ptr %gep_s0, align 16
+  store i8 %load1, ptr %gep_s1, align 16
+  store i8 %load2, ptr %gep_s2, align 16
+  store i8 %load3, ptr %gep_s3, align 16
+  store i8 %load4, ptr %gep_s4, align 16
+  store i8 %load5, ptr %gep_s5, align 16
+  store i8 %load6, ptr %gep_s6, align 16
+  store i8 %load7, ptr %gep_s7, align 16
+  store i8 %load8, ptr %gep_s8, align 16
+  store i8 %load9, ptr %gep_s9, align 16
+  store i8 %load10, ptr %gep_s10, align 16
+  store i8 %load11, ptr %gep_s11, align 16
+  store i8 %load12, ptr %gep_s12, align 16
+  store i8 %load13, ptr %gep_s13, align 16
+  store i8 %load14, ptr %gep_s14, align 16
+  store i8 %load15, ptr %gep_s15, align 16
+
+  ret void
+}
+
+define void @const_stride_2_with_reordering(ptr %pl, ptr %ps) {
+; CHECK-LABEL: define void @const_stride_2_with_reordering(
+; CHECK-SAME: ptr [[PL:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 0
+; CHECK-NEXT:    [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = call <31 x i8> @llvm.masked.load.v31i8.p0(ptr [[GEP_L0]], i32 16, <31 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true>, <31 x i8> poison)
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <31 x i8> [[TMP1]], <31 x i8> poison, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <31 x i8> [[TMP1]], <31 x i8> poison, <16 x i32> <i32 2, i32 0, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], ptr [[GEP_S0]], align 16
+; CHECK-NEXT:    ret void
+;
+  %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 0
+  %gep_l1 = getelementptr inbounds i8, ptr %pl, i64 2
+  %gep_l2 = getelementptr inbounds i8, ptr %pl, i64 4
+  %gep_l3 = getelementptr inbounds i8, ptr %pl, i64 6
+  %gep_l4 = getelementptr inbounds i8, ptr %pl, i64 8
+  %gep_l5 = getelementptr inbounds i8, ptr %pl, i64 10
+  %gep_l6 = getelementptr inbounds i8, ptr %pl, i64 12
+  %gep_l7 = getelementptr inbounds i8, ptr %pl, i64 14
+  %gep_l8 = getelementptr inbounds i8, ptr %pl, i64 16
+  %gep_l9 = getelementptr inbounds i8, ptr %pl, i64 18
+  %gep_l10 = getelementptr inbounds i8, ptr %pl, i64 20
+  %gep_l11 = getelementptr inbounds i8, ptr %pl, i64 22
+  %gep_l12 = getelementptr inbounds i8, ptr %pl, i64 24
+  %gep_l13 = getelementptr inbounds i8, ptr %pl, i64 26
+  %gep_l14 = getelementptr inbounds i8, ptr %pl, i64 28
+  %gep_l15 = getelementptr inbounds i8, ptr %pl, i64 30
+
+  %load0  = load i8, ptr %gep_l0 , align 16
+  %load1  = load i8, ptr %gep_l1 , align 16
+  %load2  = load i8, ptr %gep_l2 , align 16
+  %load3  = load i8, ptr %gep_l3 , align 16
+  %load4  = load i8, ptr %gep_l4 , align 16
+  %load5  = load i8, ptr %gep_l5 , align 16
+  %load6  = load i8, ptr %gep_l6 , align 16
+  %load7  = load i8, ptr %gep_l7 , align 16
+  %load8  = load i8, ptr %gep_l8 , align 16
+  %load9  = load i8, ptr %gep_l9 , align 16
+  %load10 = load i8, ptr %gep_l10, align 16
+  %load11 = load i8, ptr %gep_l11, align 16
+  %load12 = load i8, ptr %gep_l12, align 16
+  %load13 = load i8, ptr %gep_l13, align 16
+  %load14 = load i8, ptr %gep_l14, align 16
+  %load15 = load i8, ptr %gep_l15, align 16
+
+  %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+  %gep_s1 = getelementptr inbounds i8, ptr %ps, i64 1
+  %gep_s2 = getelementptr inbounds i8, ptr %ps, i64 2
+  %gep_s3 = getelementptr inbounds i8, ptr %ps, i64 3
+  %gep_s4 = getelementptr inbounds i8, ptr %ps, i64 4
+  %gep_s5 = getelementptr inbounds i8, ptr %ps, i64 5
+  %gep_s6 = getelementptr inbounds i8, ptr %ps, i64 6
+  %gep_s7 = getelementptr inbounds i8, ptr %ps, i64 7
+  %gep_s8 = getelementptr inbounds i8, ptr %ps, i64 8
+  %gep_s9 = getelementptr inbounds i8, ptr %ps, i64 9
+  %gep_s10 = getelementptr inbounds i8, ptr %ps, i64 10
+  %gep_s11 = getelementptr inbounds i8, ptr %ps, i64 11
+  %gep_s12 = getelementptr inbounds i8, ptr %ps, i64 12
+  %gep_s13 = getelementptr inbounds i8, ptr %ps, i64 13
+  %gep_s14 = getelementptr inbounds i8, ptr %ps, i64 14
+  %gep_s15 = getelementptr inbounds i8, ptr %ps, i64 15
+
+  store i8 %load1, ptr %gep_s0, align 16
+  store i8 %load0, ptr %gep_s1, align 16
+  store i8 %load2, ptr %gep_s2, align 16
+  store i8 %load3, ptr %gep_s3, align 16
+  store i8 %load4, ptr %gep_s4, align 16
+  store i8 %load5, ptr %gep_s5, align 16
+  store i8 %load6, ptr %gep_s6, align 16
+  store i8 %load7, ptr %gep_s7, align 16
+  store i8 %load8, ptr %gep_s8, align 16
+  store i8 %load9, ptr %gep_s9, align 16
+  store i8 %load10, ptr %gep_s10, align 16
+  store i8 %load11, ptr %gep_s11, align 16
+  store i8 %load12, ptr %gep_s12, align 16
+  store i8 %load13, ptr %gep_s13, align 16
+  store i8 %load14, ptr %gep_s14, align 16
+  store i8 %load15, ptr %gep_s15, align 16
+
+  ret void
+}
+
+define void @rt_stride_1_no_reordering(ptr %pl, i64 %stride, ptr %ps) {
+; CHECK-LABEL: define void @rt_stride_1_no_reordering(
+; CHECK-SAME: ptr [[PL:%.*]], i64 [[STRIDE:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[STRIDE0:%.*]] = mul nsw i64 [[STRIDE]], 0
+; CHECK-NEXT:    [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[STRIDE0]]
+; CHECK-NEXT:    [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.experimental.vp.strided.load.v16i8.p0.i64(ptr align 16 [[GEP_L0]], i64 [[TMP1]], <16 x i1> splat (i1 true), i32 16)
+; CHECK-NEXT:    store <16 x i8> [[TMP2]], ptr [[GEP_S0]], align 16
+; CHECK-NEXT:    ret void
+;
+  %stride0  = mul nsw i64 %stride, 0
+  %stride1  = mul nsw i64 %stride, 1
+  %stride2  = mul nsw i64 %stride, 2
+  %stride3  = mul nsw i64 %stride, 3
+  %stride4  = mul nsw i64 %stride, 4
+  %stride5  = mul nsw i64 %stride, 5
+  %stride6  = mul nsw i64 %stride, 6
+  %stride7  = mul nsw i64 %stride, 7
+  %stride8  = mul nsw i64 %stride, 8
+  %stride9  = mul nsw i64 %stride, 9
+  %stride10 = mul nsw i64 %stride, 10
+  %stride11 = mul nsw i64 %stride, 11
+  %stride12 = mul nsw i64 %stride, 12
+  %stride13 = mul nsw i64 %stride, 13
+  %stride14 = mul nsw i64 %stride, 14
+  %stride15 = mul nsw i64 %stride, 15
+
+  %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 %stride0
+  %gep_l1 = getelementptr inbounds i8, ptr %pl, i64 %stride1
+  %gep_l2 = getelementptr inbounds i8, ptr %pl, i64 %stride2
+  %gep_l3 = getelementptr inbounds i8, ptr %pl, i64 %stride3
+  %gep_l4 = getelementptr inbounds i8, ptr %pl, i64 %stride4
+  %gep_l5 = getelementptr inbounds i8, ptr %pl, i64 %stride5
+  %gep_l6 = getelementptr inbounds i8, ptr %pl, i64 %stride6
+  %gep_l7 = getelementptr inbounds i8, ptr %pl, i64 %stride7
+  %gep_l8 = getelementptr inbounds i8, ptr %pl, i64 %stride8
+  %gep_l9 = getelementptr inbounds i8, ptr %pl, i64 %stride9
+  %gep_l10 = getelementptr inbounds i8, ptr %pl, i64 %stride10
+  %gep_l11 = getelementptr inbounds i8, ptr %pl, i64 %stride11
+  %gep_l12 = getelementptr inbounds i8, ptr %pl, i64 %stride12
+  %gep_l13 = getelementptr inbounds i8, ptr %pl, i64 %stride13
+  %gep_l14 = getelementptr inbounds i8, ptr %pl, i64 %stride14
+  %gep_l15 = getelementptr inbounds i8, ptr %pl, i64 %stride15
+
+  %load0  = load i8, ptr %gep_l0 , align 16
+  %load1  = load i8, ptr %gep_l1 , align 16
+  %load2  = load i8, ptr %gep_l2 , align 16
+  %load3  = load i8, ptr %gep_l3 , align 16
+  %load4  = load i8, ptr %gep_l4 , align 16
+  %load5  = load i8, ptr %gep_l5 , align 16
+  %load6  = load i8, ptr %gep_l6 , align 16
+  %load7  = load i8, ptr %gep_l7 , align 16
+  %load8  = load i8, ptr %gep_l8 , align 16
+  %load9  = load i8, ptr %gep_l9 , align 16
+  %load10 = load i8, ptr %gep_l10, align 16
+  %load11 = load i8, ptr %gep_l11, align 16
+  %load12 = load i8, ptr %gep_l12, align 16
+  %load13 = load i8, ptr %gep_l13, align 16
+  %load14 = load i8, ptr %gep_l14, align 16
+  %load15 = load i8, ptr %gep_l15, align 16
+
+  %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+  %gep_s1 = getelementptr inbounds i8, ptr %ps, i64 1
+  %gep_s2 = getelementptr inbounds i8, ptr %ps, i64 2
+  %gep_s3 = getelementptr inbounds i8, ptr %ps, i64 3
+  %gep_s4 = getelementptr inbounds i8, ptr %ps, i64 4
+  %gep_s5 = getelementptr inbounds i8, ptr %ps, i64 5
+  %gep_s6 = getelementptr inbounds i8, ptr %ps, i64 6
+  %gep_s7 = getelementptr inbounds i8, ptr %ps, i64 7
+  %gep_s8 = getelementptr inbounds i8, ptr %ps, i64 8
+  %gep_s9 = getelementptr inbounds i8, ptr %ps, i64 9
+  %gep_s10 = getelementptr inbounds i8, ptr %ps, i64 10
+  %gep_s11 = getelementptr inbounds i8, ptr %ps, i64 11
+  %gep_s12 = getelementptr inbounds i8, ptr %ps, i64 12
+  %gep_s13 = getelementptr inbounds i8, ptr %ps, i64 13
+  %gep_s14 = getelementptr inbounds i8, ptr %ps, i64 14
+  %gep_s15 = getelementptr inbounds i8, ptr %ps, i64 15
+
+  store i8 %load0, ptr %gep_s0, align 16
+  store i8 %load1, ptr %gep_s1, align 16
+  store i8 %load2, ptr %gep_s2, align 16
+  store i8 %load3, ptr %gep_s3, align 16
+  store i8 %load4, ptr %gep_s4, align 16
+  store i8 %load5, ptr %gep_s5, align 16
+  store i8 %load6, ptr %gep_s6, align 16
+  store i8 %load7, ptr %gep_s7, align 16
+  store i8 %load8, ptr %gep_s8, align 16
+  store i8 %load9, ptr %gep_s9, align 16
+  store i8 %load10, ptr %gep_s10, align 16
+  store i8 %load11, ptr %gep_s11, align 16
+  store i8 %load12, ptr %gep_s12, align 16
+  store i8 %load13, ptr %gep_s13, align 16
+  store i8 %load14, ptr %gep_s14, align 16
+  store i8 %load15, ptr %gep_s15, align 16
+
+  ret void
+}
+
+define void @rt_stride_1_with_reordering(ptr %pl, i64 %stride, ptr %ps) {
+; CHECK-LABEL: define void @rt_stride_1_with_reordering(
+; CHECK-SAME: ptr [[PL:%.*]], i64 [[STRIDE:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[STRIDE0:%.*]] = mul nsw i64 [[STRIDE]], 0
+; CHECK-NEXT:    [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[STRIDE0]]
+; CHECK-NEXT:    [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[STRIDE]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.experimental.vp.strided.load.v16i8.p0.i64(ptr align 16 [[GEP_L0]], i64 [[TMP1]], <16 x i1> splat (i1 true), i32 16)
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <16 x i8> [[TMP2]], <16 x i8> poison, <16 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    store <16 x i8> [[TMP3]], ptr [[GEP_S0]], align 16
+; CHECK-NEXT:    ret void
+;
+  %stride0  = mul nsw i64 %stride, 0
+  %stride1  = mul nsw i64 %stride, 1
+  %stride2  = mul nsw i64 %stride, 2
+  %stride3  = mul nsw i64 %stride, 3
+  %stride4  = mul nsw i64 %stride, 4
+  %stride5  = mul nsw i64 %stride, 5
+  %stride6  = mul nsw i64 %stride, 6
+  %stride7  = mul nsw i64 %stride, 7
+  %stride8  = mul nsw i64 %stride, 8
+  %stride9  = mul nsw i64 %stride, 9
+  %stride10 = mul nsw i64 %stride, 10
+  %stride11 = mul nsw i64 %stride, 11
+  %stride12 = mul nsw i64 %stride, 12
+  %stride13 = mul nsw i64 %stride, 13
+  %stride14 = mul nsw i64 %stride, 14
+  %stride15 = mul nsw i64 %stride, 15
+
+  %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 %stride0
+  %gep_l1 = getelementptr inbounds i8, ptr %pl, i64 %stride1
+  %gep_l2 = getelementptr inbounds i8, ptr %pl, i64 %stride2
+  %gep_l3 = getelementptr inbounds i8, ptr %pl, i64 %stride3
+  %gep_l4 = getelementptr inbounds i8, ptr %pl, i64 %stride4
+  %gep_l5 = getelementptr inbounds i8, ptr %pl, i64 %stride5
+  %gep_l6 = getelementptr inbounds i8, ptr %pl, i64 %stride6
+  %gep_l7 = getelementptr inbounds i8, ptr %pl, i64 %stride7
+  %gep_l8 = getelementptr inbounds i8, ptr %pl, i64 %stride8
+  %gep_l9 = getelementptr inbounds i8, ptr %pl, i64 %stride9
+  %gep_l10 = getelementptr inbounds i8, ptr %pl, i64 %stride10
+  %gep_l11 = getelementptr inbounds i8, ptr %pl, i64 %stride11
+  %gep_l12 = getelementptr inbounds i8, ptr %pl, i64 %stride12
+  %gep_l13 = getelementptr inbounds i8, ptr %pl, i64 %stride13
+  %gep_l14 = getelementptr inbounds i8, ptr %pl, i64 %stride14
+  %gep_l15 = getelementptr inbounds i8, ptr %pl, i64 %stride15
+
+  %load0  = load i8, ptr %gep_l0 , align 16
+  %load1  = load i8, ptr %gep_l1 , align 16
+  %load2  = load i8, ptr %gep_l2 , align 16
+  %load3  = load i8, ptr %gep_l3 , align 16
+  %load4  = load i8, ptr %gep_l4 , align 16
+  %load5  = load i8, ptr %gep_l5 , align 16
+  %load6  = load i8, ptr %gep_l6 , align 16
+  %load7  = load i8, ptr %gep_l7 , align 16
+  %load8  = load i8, ptr %gep_l8 , align 16
+  %load9  = load i8, ptr %gep_l9 , align 16
+  %load10 = load i8, ptr %gep_l10, align 16
+  %load11 = load i8, ptr %gep_l11, align 16
+  %load12 = load i8, ptr %gep_l12, align 16
+  %load13 = load i8, ptr %gep_l13, align 16
+  %load14 = load i8, ptr %gep_l14, align 16
+  %load15 = load i8, ptr %gep_l15, align 16
+
+  %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+  %gep_s1 = getelementptr inbounds i8, ptr %ps, i64 1
+  %gep_s2 = getelementptr inbounds i8, ptr %ps, i64 2
+  %gep_s3 = getelementptr inbounds i8, ptr %ps, i64 3
+  %gep_s4 = getelementptr inbounds i8, ptr %ps, i64 4
+  %gep_s5 = getelementptr inbounds i8, ptr %ps, i64 5
+  %gep_s6 = getelementptr inbounds i8, ptr %ps, i64 6
+  %gep_s7 = getelementptr inbounds i8, ptr %ps, i64 7
+  %gep_s8 = getelementptr inbounds i8, ptr %ps, i64 8
+  %gep_s9 = getelementptr inbounds i8, ptr %ps, i64 9
+  %gep_s10 = getelementptr inbounds i8, ptr %ps, i64 10
+  %gep_s11 = getelementptr inbounds i8, ptr %ps, i64 11
+  %gep_s12 = getelementptr inbounds i8, ptr %ps, i64 12
+  %gep_s13 = getelementptr inbounds i8, ptr %ps, i64 13
+  %gep_s14 = getelementptr inbounds i8, ptr %ps, i64 14
+  %gep_s15 = getelementptr inbounds i8, ptr %ps, i64 15
+
+  store i8 %load1, ptr %gep_s0, align 16
+  store i8 %load0, ptr %gep_s1, align 16
+  store i8 %load2, ptr %gep_s2, align 16
+  store i8 %load3, ptr %gep_s3, align 16
+  store i8 %load4, ptr %gep_s4, align 16
+  store i8 %load5, ptr %gep_s5, align 16
+  store i8 %load6, ptr %gep_s6, align 16
+  store i8 %load7, ptr %gep_s7, align 16
+  store i8 %load8, ptr %gep_s8, align 16
+  store i8 %load9, ptr %gep_s9, align 16
+  store i8 %load10, ptr %gep_s10, align 16
+  store i8 %load11, ptr %gep_s11, align 16
+  store i8 %load12, ptr %gep_s12, align 16
+  store i8 %load13, ptr %gep_s13, align 16
+  store i8 %load14, ptr %gep_s14, align 16
+  store i8 %load15, ptr %gep_s15, align 16
+
+  ret void
+}
+
+; TODO: We want to generate this code:
+; define void @constant_stride_widen_no_reordering(ptr %pl, i64 %stride, ptr %ps) {
+; %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 %offset0
+; %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+; %strided_load = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 16 %gep_l0, i64 8, <4 x i1> splat (i1 true), i32 4)
+; %bitcast_ = bitcast <4 x i32> %strided_load to <16 x i8>
+; store <16 x i8> %bitcast_, ptr %gep_s0, align 16
+; ret void
+; }
+define void @constant_stride_widen_no_reordering(ptr %pl, i64 %stride, ptr %ps) {
+; CHECK-LABEL: define void @constant_stride_widen_no_reordering(
+; CHECK-SAME: ptr [[PL:%.*]], i64 [[STRIDE:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 0
+; CHECK-NEXT:    [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = call <28 x i8> @llvm.masked.load.v28i8.p0(ptr [[GEP_L0]], i32 16, <28 x i1> <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <28 x i8> poison)
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <28 x i8> [[TMP1]], <28 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19, i32 24, i32 25, i32 26, i32 27>
+; CHECK-NEXT:    store <16 x i8> [[TMP8]], ptr [[GEP_S0]], align 16
+; CHECK-NEXT:    ret void
+;
+  %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 0
+  %gep_l1 = getelementptr inbounds i8, ptr %pl, i64 1
+  %gep_l2 = getelementptr inbounds i8, ptr %pl, i64 2
+  %gep_l3 = getelementptr inbounds i8, ptr %pl, i64 3
+  %gep_l4 = getelementptr inbounds i8, ptr %pl, i64 8
+  %gep_l5 = getelementptr inbounds i8, ptr %pl, i64 9
+  %gep_l6 = getelementptr inbounds i8, ptr %pl, i64 10
+  %gep_l7 = getelementptr inbounds i8, ptr %pl, i64 11
+  %gep_l8 = getelementptr inbounds i8, ptr %pl, i64 16
+  %gep_l9 = getelementptr inbounds i8, ptr %pl, i64 17
+  %gep_l10 = getelementptr inbounds i8, ptr %pl, i64 18
+  %gep_l11 = getelementptr inbounds i8, ptr %pl, i64 19
+  %gep_l12 = getelementptr inbounds i8, ptr %pl, i64 24
+  %gep_l13 = getelementptr inbounds i8, ptr %pl, i64 25
+  %gep_l14 = getelementptr inbounds i8, ptr %pl, i64 26
+  %gep_l15 = getelementptr inbounds i8, ptr %pl, i64 27
+
+  %load0  = load i8, ptr %gep_l0 , align 16
+  %load1  = load i8, ptr %gep_l1 , align 16
+  %load2  = load i8, ptr %gep_l2 , align 16
+  %load3  = load i8, ptr %gep_l3 , align 16
+  %load4  = load i8, ptr %gep_l4 , align 16
+  %load5  = load i8, ptr %gep_l5 , align 16
+  %load6  = load i8, ptr %gep_l6 , align 16
+  %load7  = load i8, ptr %gep_l7 , align 16
+  %load8  = load i8, ptr %gep_l8 , align 16
+  %load9  = load i8, ptr %gep_l9 , align 16
+  %load10 = load i8, ptr %gep_l10, align 16
+  %load11 = load i8, ptr %gep_l11, align 16
+  %load12 = load i8, ptr %gep_l12, align 16
+  %load13 = load i8, ptr %gep_l13, align 16
+  %load14 = load i8, ptr %gep_l14, align 16
+  %load15 = load i8, ptr %gep_l15, align 16
+
+  %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+  %gep_s1 = getelementptr inbounds i8, ptr %ps, i64 1
+  %gep_s2 = getelementptr inbounds i8, ptr %ps, i64 2
+  %gep_s3 = getelementptr inbounds i8, ptr %ps, i64 3
+  %gep_s4 = getelementptr inbounds i8, ptr %ps, i64 4
+  %gep_s5 = getelementptr inbounds i8, ptr %ps, i64 5
+  %gep_s6 = getelementptr inbounds i8, ptr %ps, i64 6
+  %gep_s7 = getelementptr inbounds i8, ptr %ps, i64 7
+  %gep_s8 = getelementptr inbounds i8, ptr %ps, i64 8
+  %gep_s9 = getelementptr inbounds i8, ptr %ps, i64 9
+  %gep_s10 = getelementptr inbounds i8, ptr %ps, i64 10
+  %gep_s11 = getelementptr inbounds i8, ptr %ps, i64 11
+  %gep_s12 = getelementptr inbounds i8, ptr %ps, i64 12
+  %gep_s13 = getelementptr inbounds i8, ptr %ps, i64 13
+  %gep_s14 = getelementptr inbounds i8, ptr %ps, i64 14
+  %gep_s15 = getelementptr inbounds i8, ptr %ps, i64 15
+
+  store i8 %load0, ptr %gep_s0, align 16
+  store i8 %load1, ptr %gep_s1, align 16
+  store i8 %load2, ptr %gep_s2, align 16
+  store i8 %load3, ptr %gep_s3, align 16
+  store i8 %load4, ptr %gep_s4, align 16
+  store i8 %load5, ptr %gep_s5, align 16
+  store i8 %load6, ptr %gep_s6, align 16
+  store i8 %load7, ptr %gep_s7, align 16
+  store i8 %load8, ptr %gep_s8, align 16
+  store i8 %load9, ptr %gep_s9, align 16
+  store i8 %load10, ptr %gep_s10, align 16
+  store i8 %load11, ptr %gep_s11, align 16
+  store i8 %load12, ptr %gep_s12, align 16
+  store i8 %load13, ptr %gep_s13, align 16
+  store i8 %load14, ptr %gep_s14, align 16
+  store i8 %load15, ptr %gep_s15, align 16
+
+  ret void
+}
+
+; TODO: We want to generate this code:
+; define void @rt_stride_widen_no_reordering(ptr %pl, i64 %stride, ptr %ps) {
+; %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 %offset0
+; %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+; %strided_load = call <4 x i32> @llvm.experimental.vp.strided.load.v4i32.p0.i64(ptr align 16 %gep_l0, i64 %stride, <4 x i1> splat (i1 true), i32 4)
+; %bitcast_ = bitcast <4 x i32> %strided_load to <16 x i8>
+; store <16 x i8> %bitcast_, ptr %gep_s0, align 16
+; ret void
+; }
+define void @rt_stride_widen_no_reordering(ptr %pl, i64 %stride, ptr %ps) {
+; CHECK-LABEL: define void @rt_stride_widen_no_reordering(
+; CHECK-SAME: ptr [[PL:%.*]], i64 [[STRIDE:%.*]], ptr [[PS:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[OFFSET0:%.*]] = mul nsw i64 [[STRIDE]], 0
+; CHECK-NEXT:    [[OFFSET4:%.*]] = mul nsw i64 [[STRIDE]], 1
+; CHECK-NEXT:    [[OFFSET8:%.*]] = mul nsw i64 [[STRIDE]], 2
+; CHECK-NEXT:    [[OFFSET12:%.*]] = mul nsw i64 [[STRIDE]], 3
+; CHECK-NEXT:    [[GEP_L0:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET0]]
+; CHECK-NEXT:    [[GEP_L4:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET4]]
+; CHECK-NEXT:    [[GEP_L8:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET8]]
+; CHECK-NEXT:    [[GEP_L12:%.*]] = getelementptr inbounds i8, ptr [[PL]], i64 [[OFFSET12]]
+; CHECK-NEXT:    [[GEP_S0:%.*]] = getelementptr inbounds i8, ptr [[PS]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[GEP_L0]], align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_L4]], align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i8>, ptr [[GEP_L8]], align 16
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i8>, ptr [[GEP_L12]], align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> [[TMP2]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <4 x i8> [[TMP3]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <16 x i8> [[TMP7]], <16 x i8> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x i8> [[TMP4]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP9]], <16 x i8> [[TMP10]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT:    store <16 x i8> [[TMP8]], ptr [[GEP_S0]], align 16
+; CHECK-NEXT:    ret void
+;
+  %offset0  = mul nsw i64 %stride, 0
+  %offset1  = add nsw i64 %offset0, 1
+  %offset2  = add nsw i64 %offset0, 2
+  %offset3  = add nsw i64 %offset0, 3
+  %offset4  = mul nsw i64 %stride, 1
+  %offset5  = add nsw i64 %offset4, 1
+  %offset6  = add nsw i64 %offset4, 2
+  %offset7  = add nsw i64 %offset4, 3
+  %offset8  = mul nsw i64 %stride, 2
+  %offset9  = add nsw i64 %offset8, 1
+  %offset10  = add nsw i64 %offset8, 2
+  %offset11  = add nsw i64 %offset8, 3
+  %offset12 = mul nsw i64 %stride, 3
+  %offset13 = add nsw i64 %offset12, 1
+  %offset14 = add nsw i64 %offset12, 2
+  %offset15 = add nsw i64 %offset12, 3
+
+  %gep_l0 = getelementptr inbounds i8, ptr %pl, i64 %offset0
+  %gep_l1 = getelementptr inbounds i8, ptr %pl, i64 %offset1
+  %gep_l2 = getelementptr inbounds i8, ptr %pl, i64 %offset2
+  %gep_l3 = getelementptr inbounds i8, ptr %pl, i64 %offset3
+  %gep_l4 = getelementptr inbounds i8, ptr %pl, i64 %offset4
+  %gep_l5 = getelementptr inbounds i8, ptr %pl, i64 %offset5
+  %gep_l6 = getelementptr inbounds i8, ptr %pl, i64 %offset6
+  %gep_l7 = getelementptr inbounds i8, ptr %pl, i64 %offset7
+  %gep_l8 = getelementptr inbounds i8, ptr %pl, i64 %offset8
+  %gep_l9 = getelementptr inbounds i8, ptr %pl, i64 %offset9
+  %gep_l10 = getelementptr inbounds i8, ptr %pl, i64 %offset10
+  %gep_l11 = getelementptr inbounds i8, ptr %pl, i64 %offset11
+  %gep_l12 = getelementptr inbounds i8, ptr %pl, i64 %offset12
+  %gep_l13 = getelementptr inbounds i8, ptr %pl, i64 %offset13
+  %gep_l14 = getelementptr inbounds i8, ptr %pl, i64 %offset14
+  %gep_l15 = getelementptr inbounds i8, ptr %pl, i64 %offset15
+
+  %load0  = load i8, ptr %gep_l0 , align 16
+  %load1  = load i8, ptr %gep_l1 , align 16
+  %load2  = load i8, ptr %gep_l2 , align 16
+  %load3  = load i8, ptr %gep_l3 , align 16
+  %load4  = load i8, ptr %gep_l4 , align 16
+  %load5  = load i8, ptr %gep_l5 , align 16
+  %load6  = load i8, ptr %gep_l6 , align 16
+  %load7  = load i8, ptr %gep_l7 , align 16
+  %load8  = load i8, ptr %gep_l8 , align 16
+  %load9  = load i8, ptr %gep_l9 , align 16
+  %load10 = load i8, ptr %gep_l10, align 16
+  %load11 = load i8, ptr %gep_l11, align 16
+  %load12 = load i8, ptr %gep_l12, align 16
+  %load13 = load i8, ptr %gep_l13, align 16
+  %load14 = load i8, ptr %gep_l14, align 16
+  %load15 = load i8, ptr %gep_l15, align 16
+
+  %gep_s0 = getelementptr inbounds i8, ptr %ps, i64 0
+  %gep_s1 = getelementptr inbounds i8, ptr %ps, i64 1
+  %gep_s2 = getelementptr inbounds i8, ptr %ps, i64 2
+  %gep_s3 = getelementptr inbounds i8, ptr %ps, i64 3
+  %gep_s4 = getelementptr inbounds i8, ptr %ps, i64 4
+  %gep_s5 = getelementptr inbounds i8, ptr %ps, i64 5
+  %gep_s6 = getelementptr inbounds i8, ptr %ps, i64 6
+  %gep_s7 = getelementptr inbounds i8, ptr %ps, i64 7
+  %gep_s8 = getelementptr inbounds i8, ptr %ps, i64 8
+  %gep_s9 = getelementptr inbounds i8, ptr %ps, i64 9
+  %gep_s10 = getelementptr inbounds i8, ptr %ps, i64 10
+  %gep_s11 = getelementptr inbounds i8, ptr %ps, i64 11
+  %gep_s12 = getelementptr inbounds i8, ptr %ps, i64 12
+  %gep_s13 = getelementptr inbounds i8, ptr %ps, i64 13
+  %gep_s14 = getelementptr inbounds i8, ptr %ps, i64 14
+  %gep_s15 = getelementptr inbounds i8, ptr %ps, i64 15
+
+  store i8 %load0, ptr %gep_s0, align 16
+  store i8 %load1, ptr %gep_s1, align 16
+  store i8 %load2, ptr %gep_s2, align 16
+  store i8 %load3, ptr %gep_s3, align 16
+  store i8 %load4, ptr %gep_s4, align 16
+  store i8 %load5, ptr %gep_s5, align 16
+  store i8 %load6, ptr %gep_s6, align 16
+  store i8 %load7, ptr %gep_s7, align 16
+  store i8 %load8, ptr %gep_s8, align 16
+  store i8 %load9, ptr %gep_s9, align 16
+  store i8 %load10, ptr %gep_s10, align 16
+  store i8 %load11, ptr %gep_s11, align 16
+  store i8 %load12, ptr %gep_s12, align 16
+  store i8 %load13, ptr %gep_s13, align 16
+  store i8 %load14, ptr %gep_s14, align 16
+  store i8 %load15, ptr %gep_s15, align 16
+
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll
index 27de36e601512..9cc6b8739b20f 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/vec3-base.ll
@@ -611,16 +611,21 @@ define float @dot_product_fp32(ptr %a, ptr %b) {
 ;
 ; POW2-ONLY-LABEL: @dot_product_fp32(
 ; POW2-ONLY-NEXT:    [[GEP_A_0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[L_A_0:%.*]] = load float, ptr [[GEP_A_0]], align 4
+; POW2-ONLY-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i32 1
+; POW2-ONLY-NEXT:    [[L_A_1:%.*]] = load float, ptr [[GEP_A_1]], align 4
 ; POW2-ONLY-NEXT:    [[GEP_A_2:%.*]] = getelementptr inbounds float, ptr [[A]], i32 2
 ; POW2-ONLY-NEXT:    [[L_A_2:%.*]] = load float, ptr [[GEP_A_2]], align 4
 ; POW2-ONLY-NEXT:    [[GEP_B_0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[L_B_0:%.*]] = load float, ptr [[GEP_B_0]], align 4
+; POW2-ONLY-NEXT:    [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B]], i32 1
+; POW2-ONLY-NEXT:    [[L_B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4
 ; POW2-ONLY-NEXT:    [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i32 2
 ; POW2-ONLY-NEXT:    [[L_B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4
-; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[GEP_A_0]], align 4
-; POW2-ONLY-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[GEP_B_0]], align 4
-; POW2-ONLY-NEXT:    [[TMP3:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP2]]
+; POW2-ONLY-NEXT:    [[MUL_0:%.*]] = fmul fast float [[L_A_0]], [[L_B_0]]
+; POW2-ONLY-NEXT:    [[MUL_1:%.*]] = fmul fast float [[L_A_1]], [[L_B_1]]
 ; POW2-ONLY-NEXT:    [[MUL_2:%.*]] = fmul fast float [[L_A_2]], [[L_B_2]]
-; POW2-ONLY-NEXT:    [[ADD_0:%.*]] = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[TMP3]])
+; POW2-ONLY-NEXT:    [[ADD_0:%.*]] = fadd fast float [[MUL_0]], [[MUL_1]]
 ; POW2-ONLY-NEXT:    [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[MUL_2]]
 ; POW2-ONLY-NEXT:    ret float [[ADD_1]]
 ;
@@ -661,16 +666,21 @@ define float @dot_product_fp32_reorder(ptr %a, ptr %b) {
 ;
 ; POW2-ONLY-LABEL: @dot_product_fp32_reorder(
 ; POW2-ONLY-NEXT:    [[GEP_A_0:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[L_A_0:%.*]] = load float, ptr [[GEP_A_0]], align 4
+; POW2-ONLY-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds float, ptr [[A]], i32 1
+; POW2-ONLY-NEXT:    [[L_A_1:%.*]] = load float, ptr [[GEP_A_1]], align 4
 ; POW2-ONLY-NEXT:    [[GEP_A_2:%.*]] = getelementptr inbounds float, ptr [[A]], i32 2
 ; POW2-ONLY-NEXT:    [[L_A_2:%.*]] = load float, ptr [[GEP_A_2]], align 4
 ; POW2-ONLY-NEXT:    [[GEP_B_0:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[L_B_0:%.*]] = load float, ptr [[GEP_B_0]], align 4
+; POW2-ONLY-NEXT:    [[GEP_B_1:%.*]] = getelementptr inbounds float, ptr [[B]], i32 1
+; POW2-ONLY-NEXT:    [[L_B_1:%.*]] = load float, ptr [[GEP_B_1]], align 4
 ; POW2-ONLY-NEXT:    [[GEP_B_2:%.*]] = getelementptr inbounds float, ptr [[B]], i32 2
 ; POW2-ONLY-NEXT:    [[L_B_2:%.*]] = load float, ptr [[GEP_B_2]], align 4
-; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[GEP_A_0]], align 4
-; POW2-ONLY-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[GEP_B_0]], align 4
-; POW2-ONLY-NEXT:    [[TMP3:%.*]] = fmul fast <2 x float> [[TMP1]], [[TMP2]]
+; POW2-ONLY-NEXT:    [[MUL_0:%.*]] = fmul fast float [[L_A_0]], [[L_B_0]]
+; POW2-ONLY-NEXT:    [[MUL_1:%.*]] = fmul fast float [[L_A_1]], [[L_B_1]]
 ; POW2-ONLY-NEXT:    [[MUL_2:%.*]] = fmul fast float [[L_A_2]], [[L_B_2]]
-; POW2-ONLY-NEXT:    [[ADD_0:%.*]] = call fast float @llvm.vector.reduce.fadd.v2f32(float 0.000000e+00, <2 x float> [[TMP3]])
+; POW2-ONLY-NEXT:    [[ADD_0:%.*]] = fadd fast float [[MUL_1]], [[MUL_0]]
 ; POW2-ONLY-NEXT:    [[ADD_1:%.*]] = fadd fast float [[ADD_0]], [[MUL_2]]
 ; POW2-ONLY-NEXT:    ret float [[ADD_1]]
 ;
@@ -710,16 +720,21 @@ define double @dot_product_fp64(ptr %a, ptr %b) {
 ;
 ; POW2-ONLY-LABEL: @dot_product_fp64(
 ; POW2-ONLY-NEXT:    [[GEP_A_0:%.*]] = getelementptr inbounds double, ptr [[A:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[L_A_0:%.*]] = load double, ptr [[GEP_A_0]], align 4
+; POW2-ONLY-NEXT:    [[GEP_A_1:%.*]] = getelementptr inbounds double, ptr [[A]], i32 1
+; POW2-ONLY-NEXT:    [[L_A_1:%.*]] = load double, ptr [[GEP_A_1]], align 4
 ; POW2-ONLY-NEXT:    [[GEP_A_2:%.*]] = getelementptr inbounds double, ptr [[A]], i32 2
 ; POW2-ONLY-NEXT:    [[L_A_2:%.*]] = load double, ptr [[GEP_A_2]], align 4
 ; POW2-ONLY-NEXT:    [[GEP_B_0:%.*]] = getelementptr inbounds double, ptr [[B:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[L_B_0:%.*]] = load double, ptr [[GEP_B_0]], align 4
+; POW2-ONLY-NEXT:    [[GEP_B_1:%.*]] = getelementptr inbounds double, ptr [[B]], i32 1
+; POW2-ONLY-NEXT:    [[L_B_1:%.*]] = load double, ptr [[GEP_B_1]], align 4
 ; POW2-ONLY-NEXT:    [[GEP_B_2:%.*]] = getelementptr inbounds double, ptr [[B]], i32 2
 ; POW2-ONLY-NEXT:    [[L_B_2:%.*]] = load double, ptr [[GEP_B_2]], align 4
-; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x double>, ptr [[GEP_A_0]], align 4
-; POW2-ONLY-NEXT:    [[TMP2:%.*]] = load <2 x double>, ptr [[GEP_B_0]], align 4
-; POW2-ONLY-NEXT:    [[TMP3:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP2]]
+; POW2-ONLY-NEXT:    [[MUL_0:%.*]] = fmul fast double [[L_A_0]], [[L_B_0]]
+; POW2-ONLY-NEXT:    [[MUL_1:%.*]] = fmul fast double [[L_A_1]], [[L_B_1]]
 ; POW2-ONLY-NEXT:    [[MUL_2:%.*]] = fmul fast double [[L_A_2]], [[L_B_2]]
-; POW2-ONLY-NEXT:    [[ADD_0:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP3]])
+; POW2-ONLY-NEXT:    [[ADD_0:%.*]] = fadd fast double [[MUL_0]], [[MUL_1]]
 ; POW2-ONLY-NEXT:    [[ADD_1:%.*]] = fadd fast double [[ADD_0]], [[MUL_2]]
 ; POW2-ONLY-NEXT:    ret double [[ADD_1]]
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-schedule-for-subvector.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-schedule-for-subvector.ll
index 7408ba10cc772..c791a07993440 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-schedule-for-subvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-schedule-for-subvector.ll
@@ -4,8 +4,7 @@
 define void @test() {
 ; CHECK-LABEL: define void @test() {
 ; CHECK-NEXT:  [[BB:.*:]]
-; CHECK-NEXT:    [[ICMP:%.*]] = icmp samesign ult i32 0, 0
-; CHECK-NEXT:    [[SELECT:%.*]] = select i1 [[ICMP]], i32 0, i32 0
+; CHECK-NEXT:    [[SELECT:%.*]] = select i1 false, i32 0, i32 0
 ; CHECK-NEXT:    [[ZEXT:%.*]] = zext i32 [[SELECT]] to i64
 ; CHECK-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr ptr addrspace(1), ptr addrspace(1) null, i64 [[ZEXT]]
 ; CHECK-NEXT:    store ptr addrspace(1) null, ptr addrspace(1) [[GETELEMENTPTR]], align 8
@@ -13,8 +12,6 @@ define void @test() {
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 null(<2 x double> zeroinitializer)
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 0, i32 poison>, i32 [[CALL]], i32 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP3]], <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i1> [[TMP4]], <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 undef, i1 undef, i1 undef, i1 undef>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
 ; CHECK-NEXT:    ret void
 ;
 bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll b/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll
index c8748f316f024..2d4077f82621a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cast-operand-extracted.ll
@@ -19,14 +19,14 @@ define void @test(ptr %0, i32 %add651) {
 ; CHECK-NEXT:    [[ARRAYIDX660:%.*]] = getelementptr i8, ptr [[TMP4]], i64 7800
 ; CHECK-NEXT:    [[ARRAYIDX689:%.*]] = getelementptr i8, ptr [[TMP4]], i64 7816
 ; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i32> [[TMP3]], splat (i32 1)
-; CHECK-NEXT:    [[TMP8:%.*]] = add <2 x i32> [[TMP6]], [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> <i32 1, i32 poison>, i32 [[TMP5]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = add <2 x i32> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add <2 x i32> [[TMP6]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[ADD651]], i32 0
 ; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP2]], i32 1
 ; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP15:%.*]] = lshr <4 x i32> [[TMP14]], splat (i32 1)
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 1, i32 poison>, i32 [[TMP5]], i32 3
+; CHECK-NEXT:    [[TMP21:%.*]] = add <4 x i32> [[TMP14]], [[TMP20]]
+; CHECK-NEXT:    [[TMP15:%.*]] = lshr <4 x i32> [[TMP21]], splat (i32 1)
 ; CHECK-NEXT:    [[SHR685:%.*]] = lshr i32 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP16:%.*]] = trunc <4 x i32> [[TMP15]] to <4 x i16>
 ; CHECK-NEXT:    [[CONV686:%.*]] = trunc i32 [[SHR685]] to i16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/control-dependent-schedule.ll b/llvm/test/Transforms/SLPVectorizer/X86/control-dependent-schedule.ll
new file mode 100644
index 0000000000000..8602c256ad8c2
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/control-dependent-schedule.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test(i32 %0, i32 %1) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i32 [[TMP0:%.*]], i32 [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[K:%.*]] = alloca [4 x i32], align 16
+; CHECK-NEXT:    [[ADD1:%.*]] = add i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    [[SUB2:%.*]] = add i32 [[ADD1]], -1
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (ptr, ...) @printf(ptr null, i32 [[ADD1]])
+; CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[TMP1]], -1
+; CHECK-NEXT:    [[SUB3:%.*]] = add i32 [[ADD2]], [[CALL]]
+; CHECK-NEXT:    [[ADD4:%.*]] = add i32 [[SUB3]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[ADD4]], ptr [[K]], align 16
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr i8, ptr [[K]], i64 4
+; CHECK-NEXT:    store i32 0, ptr [[ARRAYINIT_ELEMENT]], align 4
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT5:%.*]] = getelementptr i8, ptr [[K]], i64 8
+; CHECK-NEXT:    [[ADD7:%.*]] = add i32 [[ADD2]], [[SUB2]]
+; CHECK-NEXT:    [[SUB8:%.*]] = add i32 [[ADD7]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[SUB8]], ptr [[ARRAYINIT_ELEMENT5]], align 8
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT9:%.*]] = getelementptr i8, ptr [[K]], i64 12
+; CHECK-NEXT:    [[ADD13:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    [[ADD10:%.*]] = add i32 [[ADD13]], [[TMP0]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add i32 [[ADD10]], [[ADD1]]
+; CHECK-NEXT:    [[ADD12:%.*]] = add i32 [[ADD11]], [[TMP0]]
+; CHECK-NEXT:    store i32 [[ADD12]], ptr [[ARRAYINIT_ELEMENT9]], align 4
+; CHECK-NEXT:    [[CALL15:%.*]] = call i32 (ptr, ...) @printf(ptr null, ptr [[K]])
+; CHECK-NEXT:    ret i32 [[CALL15]]
+;
+entry:
+  %k = alloca [4 x i32], align 16
+  %add1 = add i32 %0, %1
+  %sub2 = add i32 %add1, -1
+  %call = tail call i32 (ptr, ...) @printf(ptr null, i32 %add1)
+  %add2 = add i32 %1, -1
+  %sub3 = add i32 %add2, %call
+  %add4 = add i32 %sub3, %0
+  store i32 %add4, ptr %k, align 16
+  %arrayinit.element = getelementptr i8, ptr %k, i64 4
+  store i32 0, ptr %arrayinit.element, align 4
+  %arrayinit.element5 = getelementptr i8, ptr %k, i64 8
+  %add7 = add i32 %add2, %sub2
+  %sub8 = add i32 %add7, %0
+  store i32 %sub8, ptr %arrayinit.element5, align 8
+  %arrayinit.element9 = getelementptr i8, ptr %k, i64 12
+  %add13 = add i32 %1, 1
+  %add10 = add i32 %add13, %0
+  %add11 = add i32 %add10, %add1
+  %add12 = add i32 %add11, %0
+  store i32 %add12, ptr %arrayinit.element9, align 4
+  %call15 = call i32 (ptr, ...) @printf(ptr null, ptr %k)
+  ret i32 %call15
+}
+
+declare i32 @printf(ptr, ...)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll b/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll
new file mode 100644
index 0000000000000..1ec65da663fee
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/control-deps-schedule-data-recalculate.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test(i32 %mul) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: i32 [[MUL:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[H:%.*]] = alloca [4 x i32], align 16
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 0, 0
+; CHECK-NEXT:    [[ADD4:%.*]] = add i32 [[ADD]], 0
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @f1(i32 [[ADD4]])
+; CHECK-NEXT:    [[MUL1:%.*]] = shl i32 0, 1
+; CHECK-NEXT:    [[ADD5:%.*]] = add i32 [[CALL]], [[MUL1]]
+; CHECK-NEXT:    store i32 [[ADD5]], ptr [[H]], align 16
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr i8, ptr [[H]], i64 4
+; CHECK-NEXT:    [[ADD6:%.*]] = add i32 0, 0
+; CHECK-NEXT:    [[ADD7:%.*]] = add i32 [[ADD6]], [[MUL]]
+; CHECK-NEXT:    [[ADD9:%.*]] = add i32 [[ADD7]], [[ADD4]]
+; CHECK-NEXT:    store i32 [[ADD9]], ptr [[ARRAYINIT_ELEMENT]], align 4
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT10:%.*]] = getelementptr i8, ptr [[H]], i64 8
+; CHECK-NEXT:    [[ADD11:%.*]] = or i32 [[ADD]], 0
+; CHECK-NEXT:    [[ADD12:%.*]] = add i32 [[ADD11]], [[ADD4]]
+; CHECK-NEXT:    store i32 [[ADD12]], ptr [[ARRAYINIT_ELEMENT10]], align 8
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT13:%.*]] = getelementptr i8, ptr [[H]], i64 12
+; CHECK-NEXT:    store i32 0, ptr [[ARRAYINIT_ELEMENT13]], align 4
+; CHECK-NEXT:    ret i32 0
+;
+entry:
+  %h = alloca [4 x i32], align 16
+  %add = add i32 0, 0
+  %add4 = add i32 %add, 0
+  %call = tail call i32 @f1(i32 %add4)
+  %mul1 = shl i32 0, 1
+  %add5 = add i32 %call, %mul1
+  store i32 %add5, ptr %h, align 16
+  %arrayinit.element = getelementptr i8, ptr %h, i64 4
+  %add6 = add i32 0, 0
+  %add7 = add i32 %add6, %mul
+  %add9 = add i32 %add7, %add4
+  store i32 %add9, ptr %arrayinit.element, align 4
+  %arrayinit.element10 = getelementptr i8, ptr %h, i64 8
+  %add11 = or i32 %add, 0
+  %add12 = add i32 %add11, %add4
+  store i32 %add12, ptr %arrayinit.element10, align 8
+  %arrayinit.element13 = getelementptr i8, ptr %h, i64 12
+  store i32 0, ptr %arrayinit.element13, align 4
+  ret i32 0
+}
+
+declare i32 @f1(i32)
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll b/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll
index 4a8af6d03da06..a333d162297bc 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll
@@ -2,7 +2,7 @@
 ; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE2
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE4
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX2
 
 ;
 ; dot4(ptr x, ptr y) - ((xptr y[0])+(xptr y[1])+(xptr y[2])+(xptr y[3]))
@@ -371,6 +371,18 @@ define double @dot2f64_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(1
 ; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
 ; AVX-NEXT:    [[DOT01:%.*]] = fadd fast double [[TMP4]], [[TMP5]]
 ; AVX-NEXT:    ret double [[DOT01]]
+;
+; AVX2-LABEL: @dot2f64_fast(
+; AVX2-NEXT:    [[PTRX1:%.*]] = getelementptr inbounds double, ptr [[PTRX:%.*]], i64 1
+; AVX2-NEXT:    [[PTRY1:%.*]] = getelementptr inbounds double, ptr [[PTRY:%.*]], i64 1
+; AVX2-NEXT:    [[X0:%.*]] = load double, ptr [[PTRX]], align 4
+; AVX2-NEXT:    [[Y0:%.*]] = load double, ptr [[PTRY]], align 4
+; AVX2-NEXT:    [[X1:%.*]] = load double, ptr [[PTRX1]], align 4
+; AVX2-NEXT:    [[Y1:%.*]] = load double, ptr [[PTRY1]], align 4
+; AVX2-NEXT:    [[MUL0:%.*]] = fmul double [[X0]], [[Y0]]
+; AVX2-NEXT:    [[MUL1:%.*]] = fmul double [[X1]], [[Y1]]
+; AVX2-NEXT:    [[DOT01:%.*]] = fadd fast double [[MUL0]], [[MUL1]]
+; AVX2-NEXT:    ret double [[DOT01]]
 ;
   %ptrx1 = getelementptr inbounds double, ptr %ptrx, i64 1
   %ptry1 = getelementptr inbounds double, ptr %ptry, i64 1
@@ -409,6 +421,18 @@ define float @dot2f32_fast(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16
 ; AVX-NEXT:    [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
 ; AVX-NEXT:    [[DOT01:%.*]] = fadd fast float [[TMP4]], [[TMP5]]
 ; AVX-NEXT:    ret float [[DOT01]]
+;
+; AVX2-LABEL: @dot2f32_fast(
+; AVX2-NEXT:    [[PTRX1:%.*]] = getelementptr inbounds float, ptr [[PTRX:%.*]], i64 1
+; AVX2-NEXT:    [[PTRY1:%.*]] = getelementptr inbounds float, ptr [[PTRY:%.*]], i64 1
+; AVX2-NEXT:    [[X0:%.*]] = load float, ptr [[PTRX]], align 4
+; AVX2-NEXT:    [[Y0:%.*]] = load float, ptr [[PTRY]], align 4
+; AVX2-NEXT:    [[X1:%.*]] = load float, ptr [[PTRX1]], align 4
+; AVX2-NEXT:    [[Y1:%.*]] = load float, ptr [[PTRY1]], align 4
+; AVX2-NEXT:    [[MUL0:%.*]] = fmul float [[X0]], [[Y0]]
+; AVX2-NEXT:    [[MUL1:%.*]] = fmul float [[X1]], [[Y1]]
+; AVX2-NEXT:    [[DOT01:%.*]] = fadd fast float [[MUL0]], [[MUL1]]
+; AVX2-NEXT:    ret float [[DOT01]]
 ;
   %ptrx1 = getelementptr inbounds float, ptr %ptrx, i64 1
   %ptry1 = getelementptr inbounds float, ptr %ptry, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
index eaa77d74f8df1..378d3b00b1158 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
@@ -31,12 +31,9 @@ define float @baz() {
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr @arr, align 16
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr @arr1, align 16
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
+; THRESHOLD-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], 2.000000e+00
 ; THRESHOLD-NEXT:    [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP3]])
-; THRESHOLD-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> poison, float [[CONV]], i32 0
-; THRESHOLD-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> [[TMP5]], float [[TMP4]], i32 1
-; THRESHOLD-NEXT:    [[TMP7:%.*]] = fmul fast <2 x float> [[TMP6]], splat (float 2.000000e+00)
-; THRESHOLD-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[TMP7]], i32 0
-; THRESHOLD-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP7]], i32 1
+; THRESHOLD-NEXT:    [[TMP9:%.*]] = fmul fast float [[TMP4]], 2.000000e+00
 ; THRESHOLD-NEXT:    [[OP_RDX:%.*]] = fadd fast float [[TMP8]], [[TMP9]]
 ; THRESHOLD-NEXT:    store float [[OP_RDX]], ptr @res, align 4
 ; THRESHOLD-NEXT:    ret float [[OP_RDX]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll
index 4f94784a24dd4..c02ef8388b066 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll
@@ -101,10 +101,8 @@ define <4 x float> @PR16739_byref_alt(ptr nocapture readonly dereferenceable(16)
 define <4 x float> @PR16739_byval(ptr nocapture readonly dereferenceable(16) %x) {
 ; CHECK-LABEL: @PR16739_byval(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr [[X:%.*]], align 16
-; CHECK-NEXT:    [[T1:%.*]] = load i64, ptr [[X]], align 16
-; CHECK-NEXT:    [[T8:%.*]] = lshr i64 [[T1]], 32
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 [[T8]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <4 x i64> [[TMP2]], <i64 0, i64 32, i64 0, i64 0>
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc <4 x i64> [[TMP3]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP4]] to <4 x float>
 ; CHECK-NEXT:    ret <4 x float> [[TMP5]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll b/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
index 700e3ed9effc4..0545e5403f594 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/load-merge.ll
@@ -101,10 +101,8 @@ define <4 x float> @PR16739_byref_alt(ptr nocapture readonly dereferenceable(16)
 define <4 x float> @PR16739_byval(ptr nocapture readonly dereferenceable(16) %x) {
 ; CHECK-LABEL: @PR16739_byval(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, ptr [[X:%.*]], align 16
-; CHECK-NEXT:    [[T1:%.*]] = load i64, ptr [[X]], align 16
-; CHECK-NEXT:    [[T8:%.*]] = lshr i64 [[T1]], 32
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 [[T8]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <4 x i64> [[TMP2]], <i64 0, i64 32, i64 0, i64 0>
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc <4 x i64> [[TMP3]] to <4 x i32>
 ; CHECK-NEXT:    [[TMP5:%.*]] = bitcast <4 x i32> [[TMP4]] to <4 x float>
 ; CHECK-NEXT:    ret <4 x float> [[TMP5]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
index 9fbe0a54b0688..ca14826d57826 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr35497.ll
@@ -8,119 +8,134 @@
 %"class.3" = type { %"struct.1", i64 }
 %"struct.1" = type { [8 x i64] }
 
-$_ZN1C10SwitchModeEv = comdat any
-
 ; Function Attrs: uwtable
-define void @_ZN1C10SwitchModeEv() local_unnamed_addr #0 comdat align 2 {
+define void @_ZN1C10SwitchModeEv(ptr %p, i64 %c) {
 ; SSE-LABEL: @_ZN1C10SwitchModeEv(
 ; SSE-NEXT:  for.body.lr.ph.i:
-; SSE-NEXT:    [[OR_1:%.*]] = or i64 undef, 1
-; SSE-NEXT:    store i64 [[OR_1]], ptr undef, align 8
-; SSE-NEXT:    [[FOO_3:%.*]] = load i64, ptr undef, align 8
-; SSE-NEXT:    [[FOO_2:%.*]] = getelementptr inbounds [[CLASS_1:%.*]], ptr undef, i64 0, i32 0, i32 0, i32 0, i32 0, i64 1
+; SSE-NEXT:    [[BAR5:%.*]] = or i64 [[C:%.*]], 1
+; SSE-NEXT:    store i64 [[BAR5]], ptr [[FOO_2:%.*]], align 8
 ; SSE-NEXT:    [[FOO_4:%.*]] = load i64, ptr [[FOO_2]], align 8
-; SSE-NEXT:    [[BAR5:%.*]] = load i64, ptr undef, align 8
-; SSE-NEXT:    [[AND_2:%.*]] = and i64 [[OR_1]], [[FOO_3]]
+; SSE-NEXT:    [[FOO_3:%.*]] = getelementptr inbounds [[CLASS_1:%.*]], ptr [[FOO_2]], i64 0, i32 0, i32 0, i32 0, i32 0, i64 1
+; SSE-NEXT:    [[FOO_5:%.*]] = load i64, ptr [[FOO_3]], align 8
+; SSE-NEXT:    [[BAR6:%.*]] = load i64, ptr [[FOO_2]], align 8
 ; SSE-NEXT:    [[AND_1:%.*]] = and i64 [[BAR5]], [[FOO_4]]
-; SSE-NEXT:    store i64 [[AND_2]], ptr undef, align 8
-; SSE-NEXT:    [[BAR4:%.*]] = getelementptr inbounds [[CLASS_2:%.*]], ptr undef, i64 0, i32 0, i32 0, i32 0, i64 1
-; SSE-NEXT:    store i64 [[AND_1]], ptr [[BAR4]], align 8
+; SSE-NEXT:    [[AND_2:%.*]] = and i64 [[BAR6]], [[FOO_5]]
+; SSE-NEXT:    store i64 [[AND_1]], ptr [[FOO_2]], align 8
+; SSE-NEXT:    [[BAR4:%.*]] = getelementptr inbounds [[CLASS_2:%.*]], ptr [[FOO_2]], i64 0, i32 0, i32 0, i32 0, i64 1
+; SSE-NEXT:    store i64 [[AND_2]], ptr [[BAR4]], align 8
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @_ZN1C10SwitchModeEv(
 ; AVX-NEXT:  for.body.lr.ph.i:
-; AVX-NEXT:    [[OR_1:%.*]] = or i64 undef, 1
-; AVX-NEXT:    store i64 [[OR_1]], ptr undef, align 8
-; AVX-NEXT:    [[BAR5:%.*]] = load i64, ptr undef, align 8
-; AVX-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr undef, align 8
+; AVX-NEXT:    [[OR_1:%.*]] = or i64 [[C:%.*]], 1
+; AVX-NEXT:    store i64 [[OR_1]], ptr [[P:%.*]], align 8
+; AVX-NEXT:    [[BAR5:%.*]] = load i64, ptr [[P]], align 8
+; AVX-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[P]], align 8
 ; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[OR_1]], i32 0
 ; AVX-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[BAR5]], i32 1
 ; AVX-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[TMP2]], [[TMP0]]
-; AVX-NEXT:    store <2 x i64> [[TMP3]], ptr undef, align 8
+; AVX-NEXT:    store <2 x i64> [[TMP3]], ptr [[P]], align 8
 ; AVX-NEXT:    ret void
 ;
 for.body.lr.ph.i:
-  %or.1 = or i64 undef, 1
-  store i64 %or.1, ptr undef, align 8
-  %foo.3 = load i64, ptr undef, align 8
-  %foo.2 = getelementptr inbounds %class.1, ptr undef, i64 0, i32 0, i32 0, i32 0, i32 0, i64 1
+  %or.1 = or i64 %c, 1
+  store i64 %or.1, ptr %p, align 8
+  %foo.3 = load i64, ptr %p, align 8
+  %foo.2 = getelementptr inbounds %class.1, ptr %p, i64 0, i32 0, i32 0, i32 0, i32 0, i64 1
   %foo.4 = load i64, ptr %foo.2, align 8
-  %bar5 = load i64, ptr undef, align 8
+  %bar5 = load i64, ptr %p, align 8
   %and.2 = and i64 %or.1, %foo.3
   %and.1 = and i64 %bar5, %foo.4
-  store i64 %and.2, ptr undef, align 8
-  %bar4 = getelementptr inbounds %class.2, ptr undef, i64 0, i32 0, i32 0, i32 0, i64 1
+  store i64 %and.2, ptr %p, align 8
+  %bar4 = getelementptr inbounds %class.2, ptr %p, i64 0, i32 0, i32 0, i32 0, i64 1
   store i64 %and.1, ptr %bar4, align 8
   ret void
 }
 
 ; Function Attrs: norecurse nounwind uwtable
-define void @pr35497() local_unnamed_addr #0 {
+define void @pr35497(ptr %p, i64 %c) {
 ; SSE-LABEL: @pr35497(
 ; SSE-NEXT:  entry:
-; SSE-NEXT:    [[TMP0:%.*]] = load i64, ptr undef, align 1
-; SSE-NEXT:    [[ADD:%.*]] = add i64 undef, undef
-; SSE-NEXT:    store i64 [[ADD]], ptr undef, align 1
-; SSE-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds [0 x i64], ptr undef, i64 0, i64 4
-; SSE-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64> <i64 poison, i64 undef>, i64 [[TMP0]], i32 0
-; SSE-NEXT:    [[TMP2:%.*]] = shl <2 x i64> [[TMP1]], splat (i64 2)
-; SSE-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[TMP2]], splat (i64 20)
-; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
-; SSE-NEXT:    [[TMP5:%.*]] = add nuw nsw <2 x i64> [[TMP4]], zeroinitializer
-; SSE-NEXT:    store <2 x i64> [[TMP5]], ptr undef, align 1
-; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
-; SSE-NEXT:    [[TMP7:%.*]] = insertelement <2 x i64> [[TMP6]], i64 [[ADD]], i32 1
-; SSE-NEXT:    [[TMP8:%.*]] = shl <2 x i64> [[TMP7]], splat (i64 2)
-; SSE-NEXT:    [[TMP9:%.*]] = and <2 x i64> [[TMP8]], splat (i64 20)
-; SSE-NEXT:    [[TMP10:%.*]] = lshr <2 x i64> [[TMP5]], splat (i64 6)
-; SSE-NEXT:    [[TMP11:%.*]] = add nuw nsw <2 x i64> [[TMP9]], [[TMP10]]
-; SSE-NEXT:    store <2 x i64> [[TMP11]], ptr [[ARRAYIDX2_2]], align 1
+; SSE-NEXT:    [[TMP0:%.*]] = load i64, ptr [[P:%.*]], align 1
+; SSE-NEXT:    [[AND:%.*]] = shl i64 [[TMP0]], 2
+; SSE-NEXT:    [[SHL:%.*]] = and i64 [[AND]], 20
+; SSE-NEXT:    [[ADD:%.*]] = add i64 [[C:%.*]], [[C]]
+; SSE-NEXT:    store i64 [[ADD]], ptr [[P]], align 1
+; SSE-NEXT:    [[ARRAYIDX2_1:%.*]] = getelementptr inbounds [0 x i64], ptr [[P]], i64 0, i64 5
+; SSE-NEXT:    [[AND_1:%.*]] = shl i64 [[C]], 2
+; SSE-NEXT:    [[SHL_1:%.*]] = and i64 [[AND_1]], 20
+; SSE-NEXT:    [[SHR_1:%.*]] = lshr i64 [[C]], 6
+; SSE-NEXT:    [[ADD_1:%.*]] = add nuw nsw i64 [[SHL]], [[SHR_1]]
+; SSE-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds [0 x i64], ptr [[P]], i64 0, i64 4
+; SSE-NEXT:    [[SHR_2:%.*]] = lshr i64 [[C]], 6
+; SSE-NEXT:    [[ADD_2:%.*]] = add nuw nsw i64 [[SHL_1]], [[SHR_2]]
+; SSE-NEXT:    [[AND_4:%.*]] = shl i64 [[ADD]], 2
+; SSE-NEXT:    [[SHL_4:%.*]] = and i64 [[AND_4]], 20
+; SSE-NEXT:    [[ARRAYIDX2_5:%.*]] = getelementptr inbounds [0 x i64], ptr [[P]], i64 0, i64 1
+; SSE-NEXT:    store i64 [[ADD_1]], ptr [[ARRAYIDX2_5]], align 1
+; SSE-NEXT:    [[AND_5:%.*]] = shl nuw nsw i64 [[ADD_1]], 2
+; SSE-NEXT:    [[SHL_5:%.*]] = and i64 [[AND_5]], 20
+; SSE-NEXT:    [[SHR_5:%.*]] = lshr i64 [[ADD_1]], 6
+; SSE-NEXT:    [[ADD_5:%.*]] = add nuw nsw i64 [[SHL_4]], [[SHR_5]]
+; SSE-NEXT:    store i64 [[ADD_5]], ptr [[ARRAYIDX2_1]], align 1
+; SSE-NEXT:    store i64 [[ADD_2]], ptr [[P]], align 1
+; SSE-NEXT:    [[SHR_6:%.*]] = lshr i64 [[ADD_2]], 6
+; SSE-NEXT:    [[ADD_6:%.*]] = add nuw nsw i64 [[SHL_5]], [[SHR_6]]
+; SSE-NEXT:    store i64 [[ADD_6]], ptr [[ARRAYIDX2_2]], align 1
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @pr35497(
 ; AVX-NEXT:  entry:
-; AVX-NEXT:    [[TMP0:%.*]] = load i64, ptr undef, align 1
-; AVX-NEXT:    [[ADD:%.*]] = add i64 undef, undef
-; AVX-NEXT:    store i64 [[ADD]], ptr undef, align 1
-; AVX-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds [0 x i64], ptr undef, i64 0, i64 4
-; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64> <i64 undef, i64 poison>, i64 [[TMP0]], i32 1
-; AVX-NEXT:    [[TMP2:%.*]] = shl <2 x i64> [[TMP1]], splat (i64 2)
-; AVX-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[TMP2]], splat (i64 20)
-; AVX-NEXT:    [[TMP4:%.*]] = add nuw nsw <2 x i64> [[TMP3]], zeroinitializer
-; AVX-NEXT:    store <2 x i64> [[TMP4]], ptr undef, align 1
-; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> poison, <2 x i32> <i32 1, i32 poison>
-; AVX-NEXT:    [[TMP6:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[ADD]], i32 1
-; AVX-NEXT:    [[TMP7:%.*]] = shl <2 x i64> [[TMP6]], splat (i64 2)
-; AVX-NEXT:    [[TMP8:%.*]] = and <2 x i64> [[TMP7]], splat (i64 20)
-; AVX-NEXT:    [[TMP9:%.*]] = lshr <2 x i64> [[TMP4]], splat (i64 6)
-; AVX-NEXT:    [[TMP10:%.*]] = add nuw nsw <2 x i64> [[TMP8]], [[TMP9]]
-; AVX-NEXT:    store <2 x i64> [[TMP10]], ptr [[ARRAYIDX2_2]], align 1
+; AVX-NEXT:    [[TMP0:%.*]] = load i64, ptr [[P:%.*]], align 1
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[C:%.*]], i32 0
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <2 x i32> zeroinitializer
+; AVX-NEXT:    [[TMP3:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 6)
+; AVX-NEXT:    [[ARRAYIDX2_2:%.*]] = getelementptr inbounds [0 x i64], ptr [[P]], i64 0, i64 4
+; AVX-NEXT:    [[ARRAYIDX2_5:%.*]] = getelementptr inbounds [0 x i64], ptr [[P]], i64 0, i64 1
+; AVX-NEXT:    [[TMP4:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[TMP0]], i32 1
+; AVX-NEXT:    [[TMP5:%.*]] = shl <2 x i64> [[TMP4]], splat (i64 2)
+; AVX-NEXT:    [[TMP6:%.*]] = and <2 x i64> [[TMP5]], splat (i64 20)
+; AVX-NEXT:    [[TMP7:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP4]], <2 x i32> <i32 1, i32 2>
+; AVX-NEXT:    [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP7]], <2 x i32> <i32 1, i32 3>
+; AVX-NEXT:    [[TMP9:%.*]] = add <2 x i64> [[TMP7]], [[TMP8]]
+; AVX-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
+; AVX-NEXT:    store i64 [[TMP10]], ptr [[P]], align 1
+; AVX-NEXT:    [[TMP11:%.*]] = add nuw nsw <2 x i64> [[TMP6]], [[TMP3]]
+; AVX-NEXT:    [[TMP12:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
+; AVX-NEXT:    store i64 [[TMP12]], ptr [[ARRAYIDX2_5]], align 1
+; AVX-NEXT:    [[TMP13:%.*]] = shl <2 x i64> [[TMP9]], splat (i64 2)
+; AVX-NEXT:    [[TMP14:%.*]] = and <2 x i64> [[TMP13]], splat (i64 20)
+; AVX-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[TMP11]], i32 0
+; AVX-NEXT:    store i64 [[TMP15]], ptr [[P]], align 1
+; AVX-NEXT:    [[TMP16:%.*]] = lshr <2 x i64> [[TMP11]], splat (i64 6)
+; AVX-NEXT:    [[TMP17:%.*]] = add nuw nsw <2 x i64> [[TMP14]], [[TMP16]]
+; AVX-NEXT:    store <2 x i64> [[TMP17]], ptr [[ARRAYIDX2_2]], align 1
 ; AVX-NEXT:    ret void
 ;
 entry:
-  %0 = load i64, ptr undef, align 1
+  %0 = load i64, ptr %p, align 1
   %and = shl i64 %0, 2
   %shl = and i64 %and, 20
-  %add = add i64 undef, undef
-  store i64 %add, ptr undef, align 1
-  %arrayidx2.1 = getelementptr inbounds [0 x i64], ptr undef, i64 0, i64 5
-  %and.1 = shl i64 undef, 2
+  %add = add i64 %c, %c
+  store i64 %add, ptr %p, align 1
+  %arrayidx2.1 = getelementptr inbounds [0 x i64], ptr %p, i64 0, i64 5
+  %and.1 = shl i64 %c, 2
   %shl.1 = and i64 %and.1, 20
-  %shr.1 = lshr i64 undef, 6
+  %shr.1 = lshr i64 %c, 6
   %add.1 = add nuw nsw i64 %shl, %shr.1
-  %arrayidx2.2 = getelementptr inbounds [0 x i64], ptr undef, i64 0, i64 4
-  %shr.2 = lshr i64 undef, 6
+  %arrayidx2.2 = getelementptr inbounds [0 x i64], ptr %p, i64 0, i64 4
+  %shr.2 = lshr i64 %c, 6
   %add.2 = add nuw nsw i64 %shl.1, %shr.2
   %and.4 = shl i64 %add, 2
   %shl.4 = and i64 %and.4, 20
-  %arrayidx2.5 = getelementptr inbounds [0 x i64], ptr undef, i64 0, i64 1
+  %arrayidx2.5 = getelementptr inbounds [0 x i64], ptr %p, i64 0, i64 1
   store i64 %add.1, ptr %arrayidx2.5, align 1
   %and.5 = shl nuw nsw i64 %add.1, 2
   %shl.5 = and i64 %and.5, 20
   %shr.5 = lshr i64 %add.1, 6
   %add.5 = add nuw nsw i64 %shl.4, %shr.5
   store i64 %add.5, ptr %arrayidx2.1, align 1
-  store i64 %add.2, ptr undef, align 1
+  store i64 %add.2, ptr %p, align 1
   %shr.6 = lshr i64 %add.2, 6
   %add.6 = add nuw nsw i64 %shl.5, %shr.6
   store i64 %add.6, ptr %arrayidx2.2, align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/schedule-control-deps-after-copyable.ll b/llvm/test/Transforms/SLPVectorizer/X86/schedule-control-deps-after-copyable.ll
new file mode 100644
index 0000000000000..2f5d015bad0fb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/schedule-control-deps-after-copyable.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(i32 %e, i32 %0) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i32 [[E:%.*]], i32 [[TMP0:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[H:%.*]] = alloca [4 x i32], align 16
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP0]], [[E]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[ADD]], 1
+; CHECK-NEXT:    [[ADD4:%.*]] = add i32 [[ADD]], [[ADD2]]
+; CHECK-NEXT:    store i32 [[ADD4]], ptr [[H]], align 16
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr i8, ptr [[H]], i64 4
+; CHECK-NEXT:    [[ADD5:%.*]] = add i32 [[E]], 1
+; CHECK-NEXT:    [[ADD6:%.*]] = add i32 [[ADD5]], [[ADD2]]
+; CHECK-NEXT:    store i32 [[ADD6]], ptr [[ARRAYINIT_ELEMENT]], align 4
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT7:%.*]] = getelementptr i8, ptr [[H]], i64 8
+; CHECK-NEXT:    store i32 [[ADD6]], ptr [[ARRAYINIT_ELEMENT7]], align 8
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT10:%.*]] = getelementptr i8, ptr [[H]], i64 12
+; CHECK-NEXT:    [[CALL12:%.*]] = tail call i32 @fn1(i32 [[ADD]])
+; CHECK-NEXT:    store i32 [[CALL12]], ptr [[ARRAYINIT_ELEMENT10]], align 4
+; CHECK-NEXT:    store volatile i32 0, ptr [[H]], align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %h = alloca [4 x i32], align 16
+  %add = add i32 %0, %e
+  %add2 = add i32 %add, 1
+  %add4 = add i32 %add, %add2
+  store i32 %add4, ptr %h, align 16
+  %arrayinit.element = getelementptr i8, ptr %h, i64 4
+  %add5 = add i32 %e, 1
+  %add6 = add i32 %add5, %add2
+  store i32 %add6, ptr %arrayinit.element, align 4
+  %arrayinit.element7 = getelementptr i8, ptr %h, i64 8
+  store i32 %add6, ptr %arrayinit.element7, align 8
+  %arrayinit.element10 = getelementptr i8, ptr %h, i64 12
+  %call12 = tail call i32 @fn1(i32 %add)
+  store i32 %call12, ptr %arrayinit.element10, align 4
+  store volatile i32 0, ptr %h, align 4
+  ret void
+}
+
+declare i32 @fn1(i32)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/schedule-same-user-with-copyable.ll b/llvm/test/Transforms/SLPVectorizer/X86/schedule-same-user-with-copyable.ll
new file mode 100644
index 0000000000000..c53ccf425a315
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/schedule-same-user-with-copyable.ll
@@ -0,0 +1,64 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test(ptr %o, i32 %b.021.i) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ptr [[O:%.*]], i32 [[B_021_I:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[O1:%.*]] = alloca [3 x i32], align 4
+; CHECK-NEXT:    br label %[[WHILE_BODY:.*]]
+; CHECK:       [[WHILE_BODY]]:
+; CHECK-NEXT:    [[SUB623:%.*]] = phi i32 [ [[SUB6:%.*]], %[[N_EXIT:.*]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[ADD21:%.*]] = phi i32 [ [[ADD:%.*]], %[[N_EXIT]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[ADD419:%.*]] = phi i32 [ [[ADD4:%.*]], %[[N_EXIT]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[ADD18:%.*]] = phi i32 [ [[ADD]], %[[N_EXIT]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT:    store i32 [[ADD419]], ptr [[O1]], align 4
+; CHECK-NEXT:    store i32 [[ADD18]], ptr [[O]], align 4
+; CHECK-NEXT:    br label %[[FOR_BODY4_I:.*]]
+; CHECK:       [[FOR_COND1_I:.*]]:
+; CHECK-NEXT:    ret i32 0
+; CHECK:       [[FOR_BODY4_I]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[O1]], align 4
+; CHECK-NEXT:    [[TOBOOL_NOT_I:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL_NOT_I]], label %[[FOR_COND1_I]], label %[[N_EXIT]]
+; CHECK:       [[N_EXIT]]:
+; CHECK-NEXT:    [[SUB:%.*]] = or i32 [[B_021_I]], [[ADD21]]
+; CHECK-NEXT:    [[ADD]] = or i32 [[SUB]], 1
+; CHECK-NEXT:    [[ADD2:%.*]] = or i32 [[B_021_I]], 1
+; CHECK-NEXT:    [[ADD3:%.*]] = add i32 [[ADD2]], [[SUB623]]
+; CHECK-NEXT:    [[ADD4]] = or i32 [[ADD3]], 1
+; CHECK-NEXT:    [[MUL:%.*]] = shl i32 [[B_021_I]], 1
+; CHECK-NEXT:    [[SUB6]] = or i32 [[MUL]], 1
+; CHECK-NEXT:    br label %[[WHILE_BODY]]
+;
+entry:
+  %o1 = alloca [3 x i32], align 4
+  br label %while.body
+
+while.body:                                       ; preds = %n.exit, %entry
+  %sub623 = phi i32 [ %sub6, %n.exit ], [ 0, %entry ]
+  %add21 = phi i32 [ %add, %n.exit ], [ 0, %entry ]
+  %add419 = phi i32 [ %add4, %n.exit ], [ 0, %entry ]
+  %add18 = phi i32 [ %add, %n.exit ], [ 1, %entry ]
+  store i32 %add419, ptr %o1, align 4
+  store i32 %add18, ptr %o, align 4
+  br label %for.body4.i
+
+for.cond1.i:                                      ; preds = %for.body4.i
+  ret i32 0
+
+for.body4.i:                                      ; preds = %while.body
+  %0 = load i32, ptr %o1, align 4
+  %tobool.not.i = icmp eq i32 %0, 0
+  br i1 %tobool.not.i, label %for.cond1.i, label %n.exit
+
+n.exit:                                           ; preds = %for.body4.i
+  %sub = or i32 %b.021.i, %add21
+  %add = or i32 %sub, 1
+  %add2 = or i32 %b.021.i, 1
+  %add3 = add i32 %add2, %sub623
+  %add4 = or i32 %add3, 1
+  %mul = shl i32 %b.021.i, 1
+  %sub6 = or i32 %mul, 1
+  br label %while.body
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll b/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll
index a17ccb4b46ef9..a56c6b76ba39f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/shuffle-mask-emission.ll
@@ -5,9 +5,11 @@ define i1 @test() {
 ; CHECK-LABEL: define i1 @test() {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[H_PROMOTED118_I_FR:%.*]] = freeze i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[H_PROMOTED118_I_FR]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i32> zeroinitializer, [[TMP3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 0>
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[H_PROMOTED118_I_FR]], i32 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> zeroinitializer, [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> [[TMP1]], <4 x i32> <i32 2, i32 2, i32 7, i32 2>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP5]], <i32 0, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <4 x i32> [[TMP6]], <i32 1, i32 0, i32 0, i32 0>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss.ll b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss.ll
index 8c9f8b5868d4f..82fb5a46fee7c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/slp-fma-loss.ll
@@ -1,27 +1,38 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -S -mcpu=corei7 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s --check-prefixes=CHECK
-; RUN: opt -passes=slp-vectorizer -S -mcpu=bdver2 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s --check-prefixes=CHECK
-; RUN: opt -passes=slp-vectorizer -S -mcpu=core-avx2 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s  --check-prefixes=CHECK
+; RUN: opt -passes=slp-vectorizer -S -mcpu=corei7 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s --check-prefixes=SSE4
+; RUN: opt -passes=slp-vectorizer -S -mcpu=bdver2 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s --check-prefixes=AVX
+; RUN: opt -passes=slp-vectorizer -S -mcpu=core-avx2 -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-2 < %s | FileCheck %s  --check-prefixes=AVX
 
 ; This test checks for a case when a horizontal reduction of floating-point
 ; adds may look profitable, but is not because it eliminates generation of
 ; floating-point FMAs that would be more profitable.
 
-; FIXME: We generate a horizontal reduction today.
-
 define void @hr() {
-; CHECK-LABEL: @hr(
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[PHI0:%.*]] = phi double [ 0.000000e+00, [[TMP0:%.*]] ], [ [[OP_RDX:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[CVT0:%.*]] = uitofp i16 0 to double
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> <double poison, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>, double [[CVT0]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x double> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP2]])
-; CHECK-NEXT:    [[OP_RDX]] = fadd fast double [[TMP3]], [[PHI0]]
-; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[LOOP]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
+; SSE4-LABEL: @hr(
+; SSE4-NEXT:    br label [[LOOP:%.*]]
+; SSE4:       loop:
+; SSE4-NEXT:    [[PHI0:%.*]] = phi double [ 0.000000e+00, [[TMP0:%.*]] ], [ [[OP_RDX:%.*]], [[LOOP]] ]
+; SSE4-NEXT:    [[CVT0:%.*]] = uitofp i16 0 to double
+; SSE4-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> <double poison, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>, double [[CVT0]], i32 0
+; SSE4-NEXT:    [[TMP2:%.*]] = fmul fast <4 x double> zeroinitializer, [[TMP1]]
+; SSE4-NEXT:    [[TMP3:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP2]])
+; SSE4-NEXT:    [[OP_RDX]] = fadd fast double [[TMP3]], [[PHI0]]
+; SSE4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[LOOP]]
+; SSE4:       exit:
+; SSE4-NEXT:    ret void
+;
+; AVX-LABEL: @hr(
+; AVX-NEXT:    br label [[LOOP:%.*]]
+; AVX:       loop:
+; AVX-NEXT:    [[PHI0:%.*]] = phi double [ 0.000000e+00, [[TMP0:%.*]] ], [ [[ADD3:%.*]], [[LOOP]] ]
+; AVX-NEXT:    [[CVT0:%.*]] = uitofp i16 0 to double
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> <double poison, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>, double [[CVT0]], i32 0
+; AVX-NEXT:    [[TMP2:%.*]] = fmul fast <4 x double> zeroinitializer, [[TMP1]]
+; AVX-NEXT:    [[TMP3:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP2]])
+; AVX-NEXT:    [[ADD3]] = fadd fast double [[TMP3]], [[PHI0]]
+; AVX-NEXT:    br i1 true, label [[EXIT:%.*]], label [[LOOP]]
+; AVX:       exit:
+; AVX-NEXT:    ret void
 ;
   br label %loop
 
@@ -47,18 +58,24 @@ exit:
 ; may look profitable; but both are not because this eliminates generation
 ; of floating-point FMAs that would be more profitable.
 
-; FIXME: We generate a horizontal reduction today, and if that's disabled, we
-; still vectorize some of the multiplies.
-
 define double @hr_or_mul() {
-; CHECK-LABEL: @hr_or_mul(
-; CHECK-NEXT:    [[CVT0:%.*]] = uitofp i16 3 to double
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x double> <double 7.000000e+00, double -4.300000e+01, double 2.200000e-02, double 9.500000e+00>, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])
-; CHECK-NEXT:    [[OP_RDX:%.*]] = fadd fast double [[TMP4]], [[CVT0]]
-; CHECK-NEXT:    ret double [[OP_RDX]]
+; SSE4-LABEL: @hr_or_mul(
+; SSE4-NEXT:    [[CVT0:%.*]] = uitofp i16 3 to double
+; SSE4-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i32 0
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; SSE4-NEXT:    [[TMP3:%.*]] = fmul fast <4 x double> <double 7.000000e+00, double -4.300000e+01, double 2.200000e-02, double 9.500000e+00>, [[TMP2]]
+; SSE4-NEXT:    [[TMP4:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])
+; SSE4-NEXT:    [[OP_RDX:%.*]] = fadd fast double [[TMP4]], [[CVT0]]
+; SSE4-NEXT:    ret double [[OP_RDX]]
+;
+; AVX-LABEL: @hr_or_mul(
+; AVX-NEXT:    [[CVT0:%.*]] = uitofp i16 3 to double
+; AVX-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i32 0
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <4 x i32> zeroinitializer
+; AVX-NEXT:    [[TMP3:%.*]] = fmul fast <4 x double> <double 7.000000e+00, double -4.300000e+01, double 2.200000e-02, double 9.500000e+00>, [[TMP2]]
+; AVX-NEXT:    [[TMP4:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP3]])
+; AVX-NEXT:    [[ADD3:%.*]] = fadd fast double [[TMP4]], [[CVT0]]
+; AVX-NEXT:    ret double [[ADD3]]
 ;
   %cvt0 = uitofp i16 3 to double
   %mul0 = fmul fast double 7.000000e+00, %cvt0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll
index 36151df96bfca..4d1f6a1aa074b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll
@@ -190,12 +190,12 @@ define i32 @reorder_indices_1(float %0) {
 ; NON-POW2-NEXT:  entry:
 ; NON-POW2-NEXT:    [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4
 ; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x float>, ptr [[NOR1]], align 4
-; NON-POW2-NEXT:    [[TMP3:%.*]] = fneg <3 x float> [[TMP1]]
+; NON-POW2-NEXT:    [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; NON-POW2-NEXT:    [[TMP3:%.*]] = fneg <3 x float> [[TMP2]]
 ; NON-POW2-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0
 ; NON-POW2-NEXT:    [[TMP5:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> poison, <3 x i32> zeroinitializer
 ; NON-POW2-NEXT:    [[TMP6:%.*]] = fmul <3 x float> [[TMP3]], [[TMP5]]
-; NON-POW2-NEXT:    [[TMP10:%.*]] = shufflevector <3 x float> [[TMP6]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
-; NON-POW2-NEXT:    [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> zeroinitializer, <3 x float> [[TMP10]])
+; NON-POW2-NEXT:    [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> zeroinitializer, <3 x float> [[TMP6]])
 ; NON-POW2-NEXT:    [[TMP8:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP5]], <3 x float> [[TMP7]], <3 x float> zeroinitializer)
 ; NON-POW2-NEXT:    [[TMP9:%.*]] = fmul <3 x float> [[TMP8]], zeroinitializer
 ; NON-POW2-NEXT:    store <3 x float> [[TMP9]], ptr [[NOR1]], align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/extracts-with-undefs.ll b/llvm/test/Transforms/SLPVectorizer/extracts-with-undefs.ll
index a64075db37ba1..5fe02cbc84872 100644
--- a/llvm/test/Transforms/SLPVectorizer/extracts-with-undefs.ll
+++ b/llvm/test/Transforms/SLPVectorizer/extracts-with-undefs.ll
@@ -1,32 +1,57 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=X86 %}
+; RUN: %if aarch64-registered-target %{ opt < %s -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu | FileCheck %s --check-prefix=AARCH64 %}
 
 define void @test() {
-; CHECK-LABEL: @test(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[BODY:%.*]]
-; CHECK:       body:
-; CHECK-NEXT:    [[PHI1:%.*]] = phi double [ 0.000000e+00, [[ENTRY:%.*]] ], [ 0.000000e+00, [[BODY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY]] ], [ zeroinitializer, [[BODY]] ]
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[PHI1]], i32 0
-; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast <2 x double> <double 0.000000e+00, double undef>, [[TMP8]]
-; CHECK-NEXT:    [[ADD8_I_I:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP9]])
-; CHECK-NEXT:    [[CMP42_I:%.*]] = fcmp fast ole double [[ADD8_I_I]], 0.000000e+00
-; CHECK-NEXT:    br i1 false, label [[BODY]], label [[EXIT:%.*]]
-; CHECK:       exit:
-; CHECK-NEXT:    br i1 false, label [[IF_THEN135_I:%.*]], label [[IF_END209_I:%.*]]
-; CHECK:       if.then135.i:
-; CHECK-NEXT:    [[TMP1:%.*]] = fcmp fast olt <2 x double> [[TMP0]], zeroinitializer
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i1> <i1 poison, i1 false>, <2 x i1> [[TMP1]], <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = fmul fast <2 x double> zeroinitializer, [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <2 x double> [[TMP4]], zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = fadd fast <2 x double> [[TMP5]], zeroinitializer
-; CHECK-NEXT:    br label [[IF_END209_I]]
-; CHECK:       if.end209.i:
-; CHECK-NEXT:    [[TMP7:%.*]] = phi <2 x double> [ [[TMP6]], [[IF_THEN135_I]] ], [ zeroinitializer, [[EXIT]] ]
-; CHECK-NEXT:    ret void
+; X86-LABEL: @test(
+; X86-NEXT:  entry:
+; X86-NEXT:    br label [[BODY:%.*]]
+; X86:       body:
+; X86-NEXT:    [[PHI1:%.*]] = phi double [ 0.000000e+00, [[ENTRY:%.*]] ], [ 0.000000e+00, [[BODY]] ]
+; X86-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY]] ], [ zeroinitializer, [[BODY]] ]
+; X86-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[PHI1]], i32 0
+; X86-NEXT:    [[TMP2:%.*]] = fmul fast <2 x double> <double 0.000000e+00, double undef>, [[TMP1]]
+; X86-NEXT:    [[TMP3:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP2]])
+; X86-NEXT:    [[CMP42_I:%.*]] = fcmp fast ole double [[TMP3]], 0.000000e+00
+; X86-NEXT:    br i1 false, label [[BODY]], label [[EXIT:%.*]]
+; X86:       exit:
+; X86-NEXT:    br i1 false, label [[IF_THEN135_I:%.*]], label [[IF_END209_I:%.*]]
+; X86:       if.then135.i:
+; X86-NEXT:    [[TMP4:%.*]] = fcmp fast olt <2 x double> [[TMP0]], zeroinitializer
+; X86-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i1> <i1 poison, i1 false>, <2 x i1> [[TMP4]], <2 x i32> <i32 2, i32 1>
+; X86-NEXT:    [[TMP6:%.*]] = select <2 x i1> [[TMP5]], <2 x double> zeroinitializer, <2 x double> zeroinitializer
+; X86-NEXT:    [[TMP7:%.*]] = fmul fast <2 x double> zeroinitializer, [[TMP6]]
+; X86-NEXT:    [[TMP8:%.*]] = fmul fast <2 x double> [[TMP7]], zeroinitializer
+; X86-NEXT:    [[TMP9:%.*]] = fadd fast <2 x double> [[TMP8]], zeroinitializer
+; X86-NEXT:    br label [[IF_END209_I]]
+; X86:       if.end209.i:
+; X86-NEXT:    [[TMP10:%.*]] = phi <2 x double> [ [[TMP9]], [[IF_THEN135_I]] ], [ zeroinitializer, [[EXIT]] ]
+; X86-NEXT:    ret void
+;
+; AARCH64-LABEL: @test(
+; AARCH64-NEXT:  entry:
+; AARCH64-NEXT:    br label [[BODY:%.*]]
+; AARCH64:       body:
+; AARCH64-NEXT:    [[PHI1:%.*]] = phi double [ 0.000000e+00, [[ENTRY:%.*]] ], [ 0.000000e+00, [[BODY]] ]
+; AARCH64-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY]] ], [ zeroinitializer, [[BODY]] ]
+; AARCH64-NEXT:    [[MUL_I478_I:%.*]] = fmul fast double [[PHI1]], 0.000000e+00
+; AARCH64-NEXT:    [[MUL7_I485_I:%.*]] = fmul fast double undef, 0.000000e+00
+; AARCH64-NEXT:    [[ADD8_I_I:%.*]] = fadd fast double [[MUL_I478_I]], [[MUL7_I485_I]]
+; AARCH64-NEXT:    [[CMP42_I:%.*]] = fcmp fast ole double [[ADD8_I_I]], 0.000000e+00
+; AARCH64-NEXT:    br i1 false, label [[BODY]], label [[EXIT:%.*]]
+; AARCH64:       exit:
+; AARCH64-NEXT:    br i1 false, label [[IF_THEN135_I:%.*]], label [[IF_END209_I:%.*]]
+; AARCH64:       if.then135.i:
+; AARCH64-NEXT:    [[TMP1:%.*]] = fcmp fast olt <2 x double> [[TMP0]], zeroinitializer
+; AARCH64-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i1> <i1 poison, i1 false>, <2 x i1> [[TMP1]], <2 x i32> <i32 2, i32 1>
+; AARCH64-NEXT:    [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x double> zeroinitializer, <2 x double> zeroinitializer
+; AARCH64-NEXT:    [[TMP4:%.*]] = fmul fast <2 x double> zeroinitializer, [[TMP3]]
+; AARCH64-NEXT:    [[TMP5:%.*]] = fmul fast <2 x double> [[TMP4]], zeroinitializer
+; AARCH64-NEXT:    [[TMP6:%.*]] = fadd fast <2 x double> [[TMP5]], zeroinitializer
+; AARCH64-NEXT:    br label [[IF_END209_I]]
+; AARCH64:       if.end209.i:
+; AARCH64-NEXT:    [[TMP7:%.*]] = phi <2 x double> [ [[TMP6]], [[IF_THEN135_I]] ], [ zeroinitializer, [[EXIT]] ]
+; AARCH64-NEXT:    ret void
 ;
 entry:
   br label %body
diff --git a/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll b/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll
index fff988a0a746e..766e1fb50330b 100644
--- a/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll
+++ b/llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll
@@ -6,20 +6,21 @@ define i32 @test(i8 %0) {
 ; CHECK-SAME: i8 [[TMP0:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    [[CMP13_NOT_5:%.*]] = icmp eq i64 0, 0
-; CHECK-NEXT:    [[TMP1:%.*]] = load i8, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 7), align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x i8> <i8 0, i8 poison>, i8 [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <2 x i8> zeroinitializer, [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = load volatile i8, ptr null, align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i8>, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 9), align 1
+; CHECK-NEXT:    [[TEST_STRUCTCOPY_14_S14_CM_COERCE_SROA_2_0_COPYLOAD:%.*]] = load i48, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr addrspace(21) null, align 2
+; CHECK-NEXT:    [[TMP13:%.*]] = load volatile i8, ptr null, align 2
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i8>, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 7), align 1
+; CHECK-NEXT:    [[TMP32:%.*]] = shufflevector <2 x i8> <i8 0, i8 poison>, <2 x i8> [[TMP5]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <2 x i8> zeroinitializer, [[TMP32]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i8> [[TMP5]], <2 x i8> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <8 x i8> <i8 0, i8 0, i8 poison, i8 0, i8 0, i8 poison, i8 0, i8 0>, <8 x i8> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 8, i32 3, i32 4, i32 9, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP33:%.*]] = shufflevector <8 x i8> <i8 0, i8 0, i8 poison, i8 0, i8 0, i8 poison, i8 0, i8 0>, <8 x i8> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 9, i32 3, i32 4, i32 poison, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <8 x i8> [[TMP33]], i8 [[TMP2]], i32 5
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq <8 x i8> zeroinitializer, [[TMP7]]
-; CHECK-NEXT:    [[TEST_STRUCTCOPY_14_S14_CM_COERCE_SROA_2_0_COPYLOAD:%.*]] = load i48, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x i48> <i48 poison, i48 0, i48 0, i48 0>, i48 [[TEST_STRUCTCOPY_14_S14_CM_COERCE_SROA_2_0_COPYLOAD]], i32 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = trunc <4 x i48> [[TMP9]] to <4 x i8>
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq <4 x i8> zeroinitializer, [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = load i8, ptr addrspace(21) null, align 2
-; CHECK-NEXT:    [[TMP13:%.*]] = load volatile i8, ptr null, align 2
 ; CHECK-NEXT:    [[TMP14:%.*]] = load <2 x i8>, ptr addrspace(21) getelementptr inbounds (i8, ptr addrspace(21) null, i64 8), align 8
 ; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x i8> <i8 0, i8 poison, i8 0, i8 poison, i8 poison, i8 0, i8 0, i8 0>, i8 [[TMP12]], i32 1
 ; CHECK-NEXT:    [[TMP16:%.*]] = shufflevector <2 x i8> [[TMP14]], <2 x i8> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/SLPVectorizer/insertelement-postpone.ll b/llvm/test/Transforms/SLPVectorizer/insertelement-postpone.ll
index 1e4b598d9fe92..eefc99feebb95 100644
--- a/llvm/test/Transforms/SLPVectorizer/insertelement-postpone.ll
+++ b/llvm/test/Transforms/SLPVectorizer/insertelement-postpone.ll
@@ -1,24 +1,41 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -mtriple x86_64-unknown-linux-gnu < %s | FileCheck %s %}
-; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -mtriple aarch64-unknown-linux-gnu < %s | FileCheck %s %}
+; RUN: %if x86-registered-target %{ opt -S -passes=slp-vectorizer -mtriple x86_64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=X86 %}
+; RUN: %if aarch64-registered-target %{ opt -S -passes=slp-vectorizer -mtriple aarch64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=AARCH86 %}
 
 define <4 x double> @test(ptr %p2, double %i1754, double %i1781, double %i1778) {
-; CHECK-LABEL: @test(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[I1771:%.*]] = getelementptr inbounds double, ptr [[P2:%.*]], i64 54
-; CHECK-NEXT:    [[I1772:%.*]] = load double, ptr [[I1771]], align 8
-; CHECK-NEXT:    [[I1795:%.*]] = getelementptr inbounds double, ptr [[P2]], i64 55
-; CHECK-NEXT:    [[I1796:%.*]] = load double, ptr [[I1795]], align 8
-; CHECK-NEXT:    [[I1797:%.*]] = fmul fast double [[I1796]], [[I1781:%.*]]
-; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> poison, double [[I1754:%.*]], i32 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[I1778:%.*]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[I1781]], i32 2
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[I1772]], i32 3
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <4 x double> [[TMP3]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double poison>, double [[I1797]], i32 3
-; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast <4 x double> [[TMP5]], [[TMP6]]
-; CHECK-NEXT:    ret <4 x double> [[TMP7]]
+; X86-LABEL: @test(
+; X86-NEXT:  entry:
+; X86-NEXT:    [[I1771:%.*]] = getelementptr inbounds double, ptr [[P2:%.*]], i64 54
+; X86-NEXT:    [[I1772:%.*]] = load double, ptr [[I1771]], align 8
+; X86-NEXT:    [[I1795:%.*]] = getelementptr inbounds double, ptr [[P2]], i64 55
+; X86-NEXT:    [[I1796:%.*]] = load double, ptr [[I1795]], align 8
+; X86-NEXT:    [[I1797:%.*]] = fmul fast double [[I1796]], [[I1781:%.*]]
+; X86-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> poison, double [[I1754:%.*]], i32 0
+; X86-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[I1778:%.*]], i32 1
+; X86-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[I1781]], i32 2
+; X86-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[I1772]], i32 3
+; X86-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> poison, <4 x i32> zeroinitializer
+; X86-NEXT:    [[TMP5:%.*]] = fmul fast <4 x double> [[TMP3]], [[TMP4]]
+; X86-NEXT:    [[TMP6:%.*]] = insertelement <4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double poison>, double [[I1797]], i32 3
+; X86-NEXT:    [[TMP7:%.*]] = fadd fast <4 x double> [[TMP5]], [[TMP6]]
+; X86-NEXT:    ret <4 x double> [[TMP7]]
+;
+; AARCH86-LABEL: @test(
+; AARCH86-NEXT:  entry:
+; AARCH86-NEXT:    [[I1771:%.*]] = getelementptr inbounds double, ptr [[P2:%.*]], i64 54
+; AARCH86-NEXT:    [[I1772:%.*]] = load double, ptr [[I1771]], align 8
+; AARCH86-NEXT:    [[I1795:%.*]] = getelementptr inbounds double, ptr [[P2]], i64 55
+; AARCH86-NEXT:    [[I1796:%.*]] = load double, ptr [[I1795]], align 8
+; AARCH86-NEXT:    [[I1797:%.*]] = fmul fast double [[I1796]], [[I1781:%.*]]
+; AARCH86-NEXT:    [[TMP0:%.*]] = insertelement <4 x double> poison, double [[I1754:%.*]], i32 0
+; AARCH86-NEXT:    [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[I1778:%.*]], i32 1
+; AARCH86-NEXT:    [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[I1781]], i32 2
+; AARCH86-NEXT:    [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[I1772]], i32 3
+; AARCH86-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP3]], <4 x double> poison, <4 x i32> zeroinitializer
+; AARCH86-NEXT:    [[TMP5:%.*]] = fmul fast <4 x double> [[TMP3]], [[TMP4]]
+; AARCH86-NEXT:    [[TMP6:%.*]] = insertelement <4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double poison>, double [[I1797]], i32 3
+; AARCH86-NEXT:    [[I1994:%.*]] = fadd fast <4 x double> [[TMP5]], [[TMP6]]
+; AARCH86-NEXT:    ret <4 x double> [[I1994]]
 ;
 entry:
   %i1771 = getelementptr inbounds double, ptr %p2, i64 54
diff --git a/llvm/test/Transforms/SLPVectorizer/reductions.ll b/llvm/test/Transforms/SLPVectorizer/reductions.ll
new file mode 100644
index 0000000000000..c45c1c3ac32df
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/reductions.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux < %s | FileCheck %s %}
+
+; SLP doesn't currently support sub reductions
+define i64 @reduction_sub(ptr %ptr) {
+; CHECK-LABEL: define i64 @reduction_sub(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[LD0:%.*]] = load i64, ptr [[PTR]], align 8
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 1
+; CHECK-NEXT:    [[LD1:%.*]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[SUB_1:%.*]] = sub nuw nsw i64 [[LD0]], [[LD1]]
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 2
+; CHECK-NEXT:    [[LD2:%.*]] = load i64, ptr [[GEP_1]], align 8
+; CHECK-NEXT:    [[SUB_2:%.*]] = sub nuw nsw i64 [[SUB_1]], [[LD2]]
+; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr inbounds i64, ptr [[PTR]], i64 3
+; CHECK-NEXT:    [[LD3:%.*]] = load i64, ptr [[GEP_2]], align 8
+; CHECK-NEXT:    [[SUB_3:%.*]] = sub nuw nsw i64 [[SUB_2]], [[LD3]]
+; CHECK-NEXT:    ret i64 [[SUB_3]]
+;
+entry:
+  %ld0 = load i64, ptr %ptr
+  %gep = getelementptr inbounds i64, ptr %ptr, i64 1
+  %ld1 = load i64, ptr %gep
+  %sub.1 = sub nuw nsw i64 %ld0, %ld1
+  %gep.1 = getelementptr inbounds i64, ptr %ptr, i64 2
+  %ld2 = load i64, ptr %gep.1
+  %sub.2 = sub nuw nsw i64 %sub.1, %ld2
+  %gep.2 = getelementptr inbounds i64, ptr %ptr, i64 3
+  %ld3 = load i64, ptr %gep.2
+  %sub.3 = sub nuw nsw i64 %sub.2, %ld3
+  ret i64 %sub.3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index ac8b10a0087d0..df42cba7c8d45 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -306,13 +306,11 @@ define void @test11(<2 x i64> %0, i64 %1, <2 x i64> %2) {
 ; CHECK-LABEL: @test11(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i64> [[TMP0:%.*]], i64 [[TMP1:%.*]], i32 1
-; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i64> <i64 5, i64 0>, [[TMP2:%.*]]
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc <2 x i64> [[TMP4]] to <2 x i16>
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i16> [[TMP5]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i16>
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x i16> [[TMP7]], <2 x i16> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x i16> [[TMP6]], <4 x i16> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP9:%.*]] = trunc <4 x i16> [[TMP8]] to <4 x i8>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP2:%.*]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i64> <i64 5, i64 0, i64 0, i64 0>, [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = trunc <4 x i64> [[TMP7]] to <4 x i8>
 ; CHECK-NEXT:    [[TMP11:%.*]] = urem <4 x i8> [[TMP9]], zeroinitializer
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp ne <4 x i8> [[TMP11]], zeroinitializer
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SROA/alloca-address-space.ll b/llvm/test/Transforms/SROA/alloca-address-space.ll
index 31305c8caf05a..941178f0d56a8 100644
--- a/llvm/test/Transforms/SROA/alloca-address-space.ll
+++ b/llvm/test/Transforms/SROA/alloca-address-space.ll
@@ -140,7 +140,7 @@ define void @addressspace_alloca_lifetime() {
 ; CHECK-NEXT:    ret void
 ;
   %alloca = alloca i8, align 8, addrspace(2)
-  call void @llvm.lifetime.start(i64 2, ptr addrspace(2) %alloca)
+  call void @llvm.lifetime.start(ptr addrspace(2) %alloca)
   ret void
 }
 
diff --git a/llvm/test/Transforms/SROA/basictest.ll b/llvm/test/Transforms/SROA/basictest.ll
index 3034aaa53cb22..15803f7b5a25b 100644
--- a/llvm/test/Transforms/SROA/basictest.ll
+++ b/llvm/test/Transforms/SROA/basictest.ll
@@ -4,8 +4,8 @@
 
 target datalayout = "e-p:64:64:64-p1:16:16:16-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define i32 @test0() {
 ; CHECK-LABEL: @test0(
@@ -18,21 +18,21 @@ entry:
   %a1 = alloca i32
   %a2 = alloca float
 
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a1)
+  call void @llvm.lifetime.start.p0(ptr %a1)
 
   store i32 0, ptr %a1
   %v1 = load i32, ptr %a1
 
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a1)
+  call void @llvm.lifetime.end.p0(ptr %a1)
 
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a2)
+  call void @llvm.lifetime.start.p0(ptr %a2)
 
   store float 0.0, ptr %a2
   %v2 = load float , ptr %a2
   %v2.int = bitcast float %v2 to i32
   %sum1 = add i32 %v1, %v2.int
 
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a2)
+  call void @llvm.lifetime.end.p0(ptr %a2)
 
   ret i32 %sum1
 }
@@ -1102,7 +1102,7 @@ define void @PR14059.1(ptr %d) {
 ;
 entry:
   %X.sroa.0.i = alloca double, align 8
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %X.sroa.0.i)
+  call void @llvm.lifetime.start.p0(ptr %X.sroa.0.i)
 
   ; Store to the low 32-bits...
   store i32 0, ptr %X.sroa.0.i, align 8
@@ -1126,7 +1126,7 @@ entry:
   %accum.real.i = load double, ptr %d, align 8
   %add.r.i = fadd double %accum.real.i, %X.sroa.0.0.load1.i
   store double %add.r.i, ptr %d, align 8
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %X.sroa.0.i)
+  call void @llvm.lifetime.end.p0(ptr %X.sroa.0.i)
   ret void
 }
 
@@ -1812,7 +1812,7 @@ define void @PR25873(ptr %outData) {
 ;
 entry:
   %tmpData = alloca %struct.STest, align 8
-  call void @llvm.lifetime.start.p0(i64 16, ptr %tmpData)
+  call void @llvm.lifetime.start.p0(ptr %tmpData)
   store float 1.230000e+02, ptr %tmpData, align 8
   %y = getelementptr inbounds %struct.STest, ptr %tmpData, i64 0, i32 0, i32 1
   store float 4.560000e+02, ptr %y, align 4
@@ -1820,7 +1820,7 @@ entry:
   %0 = load i64, ptr %tmpData, align 8
   store i64 %0, ptr %m_posB, align 8
   call void @llvm.memcpy.p0.p0.i64(ptr align 4 %outData, ptr align 4 %tmpData, i64 16, i1 false)
-  call void @llvm.lifetime.end.p0(i64 16, ptr %tmpData)
+  call void @llvm.lifetime.end.p0(ptr %tmpData)
   ret void
 }
 
@@ -1833,8 +1833,8 @@ define void @PR27999() unnamed_addr {
 ;
 entry-block:
   %0 = alloca [2 x i64], align 8
-  call void @llvm.lifetime.start.p0(i64 16, ptr %0)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %0)
+  call void @llvm.lifetime.start.p0(ptr %0)
+  call void @llvm.lifetime.end.p0(ptr %0)
   ret void
 }
 
@@ -1846,7 +1846,7 @@ define void @PR29139() {
 bb1:
   %e.7.sroa.6.i = alloca i32, align 1
   %e.7.sroa.6.0.load81.i = load i32, ptr %e.7.sroa.6.i, align 1
-  call void @llvm.lifetime.end.p0(i64 2, ptr %e.7.sroa.6.i)
+  call void @llvm.lifetime.end.p0(ptr %e.7.sroa.6.i)
   ret void
 }
 
@@ -1898,8 +1898,8 @@ entry:
   ret void
 }
 
-declare void @llvm.lifetime.start.isVoid.i64.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.isVoid.i64.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.isVoid.i64.p0(ptr nocapture)
+declare void @llvm.lifetime.end.isVoid.i64.p0(ptr nocapture)
 @array = dso_local global [10 x float] zeroinitializer, align 4
 
 define void @test29(i32 %num, i32 %tid) {
@@ -1931,7 +1931,7 @@ define void @test29(i32 %num, i32 %tid) {
 ;
 entry:
   %ra = alloca [10 x float], align 4
-  call void @llvm.lifetime.start.isVoid.i64.p0(i64 40, ptr nonnull %ra)
+  call void @llvm.lifetime.start.isVoid.i64.p0(ptr nonnull %ra)
 
   %cmp1 = icmp sgt i32 %num, 0
   br i1 %cmp1, label %bb1, label %bb7
@@ -1963,7 +1963,7 @@ bb6:
   br label %bb7
 
 bb7:
-  call void @llvm.lifetime.end.isVoid.i64.p0(i64 40, ptr nonnull %ra)
+  call void @llvm.lifetime.end.isVoid.i64.p0(ptr nonnull %ra)
   ret void
 }
 
diff --git a/llvm/test/Transforms/SROA/dead-inst.ll b/llvm/test/Transforms/SROA/dead-inst.ll
index 44ae82173bb84..bf477222ad682 100644
--- a/llvm/test/Transforms/SROA/dead-inst.ll
+++ b/llvm/test/Transforms/SROA/dead-inst.ll
@@ -47,7 +47,7 @@ define void @H(ptr noalias nocapture readnone, [2 x i64], ptr %ptr, i32 signext
 ; CHECK-NEXT:    [[TMP21:%.*]] = phi i64 [ -1, [[TMP12]] ], [ [[TMP20]], [[TMP17]] ]
 ; CHECK-NEXT:    [[TMP22:%.*]] = inttoptr i64 0 to ptr
 ; CHECK-NEXT:    [[TMP23:%.*]] = sub nsw i64 [[TMP21]], [[TMP13]]
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[TMP3]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TMP3]])
 ; CHECK-NEXT:    [[TMP24:%.*]] = icmp ult i64 [[TMP23]], 2
 ; CHECK-NEXT:    br i1 [[TMP24]], label [[G_EXIT:%.*]], label [[TMP25:%.*]]
 ; CHECK:       25:
@@ -60,7 +60,7 @@ define void @H(ptr noalias nocapture readnone, [2 x i64], ptr %ptr, i32 signext
 ; CHECK-NEXT:    call void @D(ptr nonnull sret([[CLASS_B]]) [[TMP3]], ptr nonnull dereferenceable(32) [[PTR2:%.*]])
 ; CHECK-NEXT:    br label [[G_EXIT]]
 ; CHECK:       G.exit:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[TMP3]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TMP3]])
 ; CHECK-NEXT:    br label [[FOO]]
 ; CHECK:       foo:
 ; CHECK-NEXT:    ret void
@@ -105,7 +105,7 @@ a.exit:
   %22 = phi i64 [ -1, %12 ], [ %21, %18 ]
   %23 = load ptr, ptr %13, align 8
   %24 = sub nsw i64 %22, %14
-  call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %3)
+  call void @llvm.lifetime.start.p0(ptr nonnull %3)
   %25 = icmp ult i64 %24, 2
   br i1 %25, label %G.exit, label %26
 
@@ -122,7 +122,7 @@ a.exit:
   br label %G.exit
 
 G.exit:
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %3)
+  call void @llvm.lifetime.end.p0(ptr nonnull %3)
   br label %foo
 
 foo:
@@ -133,10 +133,10 @@ foo:
 declare ptr @memchr(ptr, i32 signext, i64) local_unnamed_addr
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CHECK-MODIFY-CFG: {{.*}}
 ; CHECK-PRESERVE-CFG: {{.*}}
diff --git a/llvm/test/Transforms/SROA/ignore-droppable.ll b/llvm/test/Transforms/SROA/ignore-droppable.ll
index 9c95dc07d04ca..ba581bb7049d8 100644
--- a/llvm/test/Transforms/SROA/ignore-droppable.ll
+++ b/llvm/test/Transforms/SROA/ignore-droppable.ll
@@ -3,8 +3,8 @@
 ; RUN: opt < %s -passes='sroa<modify-cfg>' -S | FileCheck %s --check-prefixes=CHECK,CHECK-MODIFY-CFG
 
 declare void @llvm.assume(i1)
-declare void @llvm.lifetime.start.p0(i64 %size, ptr nocapture %ptr)
-declare void @llvm.lifetime.end.p0(i64 %size, ptr nocapture %ptr)
+declare void @llvm.lifetime.start.p0(ptr nocapture %ptr)
+declare void @llvm.lifetime.end.p0(ptr nocapture %ptr)
 
 define void @positive_assume_uses(ptr %arg) {
 ; CHECK-LABEL: @positive_assume_uses(
@@ -55,10 +55,10 @@ define void @positive_gep_assume_uses() {
 ;
   %A = alloca {i8, i16}
   %B = getelementptr {i8, i16}, ptr %A, i32 0, i32 0
-  call void @llvm.lifetime.start.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.start.p0(ptr %A)
   call void @llvm.assume(i1 true) ["align"(ptr %B, i64 8), "align"(ptr %B, i64 16)]
   store {i8, i16} zeroinitializer, ptr %A
-  call void @llvm.lifetime.end.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.end.p0(ptr %A)
   call void @llvm.assume(i1 true) ["nonnull"(ptr %B), "align"(ptr %B, i64 2)]
   ret void
 }
@@ -71,10 +71,10 @@ define void @positive_mixed_assume_uses() {
 ; CHECK-NEXT:    ret void
 ;
   %A = alloca i8
-  call void @llvm.lifetime.start.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.start.p0(ptr %A)
   call void @llvm.assume(i1 true) ["nonnull"(ptr %A), "align"(ptr %A, i64 8), "align"(ptr %A, i64 16)]
   store i8 1, ptr %A
-  call void @llvm.lifetime.end.p0(i64 2, ptr %A)
+  call void @llvm.lifetime.end.p0(ptr %A)
   call void @llvm.assume(i1 true) ["nonnull"(ptr %A), "align"(ptr %A, i64 2), "nonnull"(ptr %A)]
   call void @llvm.assume(i1 true) ["nonnull"(ptr %A), "align"(ptr %A, i64 2), "nonnull"(ptr %A)]
   ret void
diff --git a/llvm/test/Transforms/SROA/lifetime-intrinsic.ll b/llvm/test/Transforms/SROA/lifetime-intrinsic.ll
index b9e8873b53307..668903dd863a4 100644
--- a/llvm/test/Transforms/SROA/lifetime-intrinsic.ll
+++ b/llvm/test/Transforms/SROA/lifetime-intrinsic.ll
@@ -18,14 +18,14 @@ define i16 @with_lifetime(i32 %a, i32 %b) #0 {
 ; CHECK-NEXT:    ret i16 [[RET]]
 ;
   %arr = alloca %i32x2, align 4
-  call void @llvm.lifetime.start.p0(i64 8, ptr %arr)
+  call void @llvm.lifetime.start.p0(ptr %arr)
   %p1 = getelementptr inbounds %i32x2, ptr %arr, i64 0, i32 0, i32 1
   store i32 %a, ptr %arr, align 4
   store i32 %b, ptr %p1, align 4
   %s0 = load i16, ptr %arr, align 4
   %s2 = load i16, ptr %p1, align 4
   %ret = add i16 %s0, %s2
-  call void @llvm.lifetime.end.p0(i64 8, ptr %arr)
+  call void @llvm.lifetime.end.p0(ptr %arr)
   ret i16 %ret
 }
 
@@ -50,9 +50,9 @@ define i16 @no_lifetime(i32 %a, i32 %b) #0 {
   ret i16 %ret
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 attributes #0 = { alwaysinline nounwind }
 attributes #1 = { argmemonly nounwind }
diff --git a/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll b/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll
index 13808b2aa8916..b86f41b8ff514 100644
--- a/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll
+++ b/llvm/test/Transforms/SROA/non-capturing-call-readonly.ll
@@ -791,13 +791,13 @@ entry:
   ret i32 0
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
 
 define i64 @do_schedule_instrs_for_dce_after_fixups() {
 ; CHECK-LABEL: @do_schedule_instrs_for_dce_after_fixups(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[C:%.*]] = alloca i64, align 2
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr [[C]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[C]])
 ; CHECK-NEXT:    store i64 0, ptr [[C]], align 4
 ; CHECK-NEXT:    br label [[IF_END:%.*]]
 ; CHECK:       if.end:
@@ -807,7 +807,7 @@ define i64 @do_schedule_instrs_for_dce_after_fixups() {
 ;
 entry:
   %c = alloca i64, align 2
-  call void @llvm.lifetime.start.p0(i64 1, ptr %c)
+  call void @llvm.lifetime.start.p0(ptr %c)
   store i64 0, ptr %c
   br label %if.end
 
diff --git a/llvm/test/Transforms/SROA/pr26972.ll b/llvm/test/Transforms/SROA/pr26972.ll
index a2872c70483f3..526db3cf5c567 100644
--- a/llvm/test/Transforms/SROA/pr26972.ll
+++ b/llvm/test/Transforms/SROA/pr26972.ll
@@ -12,11 +12,11 @@ define void @fn1() {
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca [1073741825 x i32], align 16
-  call void @llvm.lifetime.end.p0(i64 4294967300, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CHECK-MODIFY-CFG: {{.*}}
 ; CHECK-PRESERVE-CFG: {{.*}}
diff --git a/llvm/test/Transforms/SROA/readonlynocapture.ll b/llvm/test/Transforms/SROA/readonlynocapture.ll
index 5752fadd76d48..b6f7b1fd1685e 100644
--- a/llvm/test/Transforms/SROA/readonlynocapture.ll
+++ b/llvm/test/Transforms/SROA/readonlynocapture.ll
@@ -284,25 +284,25 @@ define void @incompletestruct(i1 %b, i1 %c) {
 ; CHECK-LABEL: @incompletestruct(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[LII:%.*]] = alloca [[STRUCT_LOADIMMEDIATEINFO:%.*]], align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[LII]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[LII]])
 ; CHECK-NEXT:    [[BF_CLEAR4:%.*]] = and i32 undef, -262144
 ; CHECK-NEXT:    [[BF_SET5:%.*]] = select i1 [[B:%.*]], i32 196608, i32 131072
 ; CHECK-NEXT:    [[BF_SET12:%.*]] = or disjoint i32 [[BF_SET5]], [[BF_CLEAR4]]
 ; CHECK-NEXT:    store i32 [[BF_SET12]], ptr [[LII]], align 4
 ; CHECK-NEXT:    call void @callee(ptr [[LII]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[LII]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[LII]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %LII = alloca %struct.LoadImmediateInfo, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %LII)
+  call void @llvm.lifetime.start.p0(ptr nonnull %LII)
   %bf.load = load i32, ptr %LII, align 4
   %bf.clear4 = and i32 %bf.load, -262144
   %bf.set5 = select i1 %b, i32 196608, i32 131072
   %bf.set12 = or disjoint i32 %bf.set5, %bf.clear4
   store i32 %bf.set12, ptr %LII, align 4
   call void @callee(ptr %LII)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %LII)
+  call void @llvm.lifetime.end.p0(ptr nonnull %LII)
   ret void
 }
 
@@ -312,13 +312,13 @@ define void @incompletestruct_bb(i1 %b, i1 %c) {
 ; CHECK-NEXT:    [[LII:%.*]] = alloca [[STRUCT_LOADIMMEDIATEINFO:%.*]], align 4
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[LII]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[LII]])
 ; CHECK-NEXT:    [[BF_CLEAR4:%.*]] = and i32 undef, -262144
 ; CHECK-NEXT:    [[BF_SET5:%.*]] = select i1 [[B:%.*]], i32 196608, i32 131072
 ; CHECK-NEXT:    [[BF_SET12:%.*]] = or disjoint i32 [[BF_SET5]], [[BF_CLEAR4]]
 ; CHECK-NEXT:    store i32 [[BF_SET12]], ptr [[LII]], align 4
 ; CHECK-NEXT:    call void @callee(ptr [[LII]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[LII]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[LII]])
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    ret void
@@ -328,14 +328,14 @@ entry:
   br i1 %c, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %LII)
+  call void @llvm.lifetime.start.p0(ptr nonnull %LII)
   %bf.load = load i32, ptr %LII, align 4
   %bf.clear4 = and i32 %bf.load, -262144
   %bf.set5 = select i1 %b, i32 196608, i32 131072
   %bf.set12 = or disjoint i32 %bf.set5, %bf.clear4
   store i32 %bf.set12, ptr %LII, align 4
   call void @callee(ptr %LII)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %LII)
+  call void @llvm.lifetime.end.p0(ptr nonnull %LII)
   br label %if.end
 
 if.end:                                           ; preds = %if.then, %entry
@@ -459,35 +459,35 @@ define i32 @provenance_only_capture() {
 define i32 @simple_with_lifetimes() {
 ; CHECK-LABEL: @simple_with_lifetimes(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
 ; CHECK-NEXT:    call void @callee(ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret i32 0
 ;
   %a = alloca i32
-  call void @llvm.lifetime.start(i64 4, ptr %a)
+  call void @llvm.lifetime.start(ptr %a)
   store i32 0, ptr %a
   call void @callee(ptr %a)
   %l1 = load i32, ptr %a
-  call void @llvm.lifetime.end(i64 4, ptr %a)
+  call void @llvm.lifetime.end(ptr %a)
   ret i32 %l1
 }
 
 define i32 @twoalloc_with_lifetimes() {
 ; CHECK-LABEL: @twoalloc_with_lifetimes(
 ; CHECK-NEXT:    [[A:%.*]] = alloca { i32, i32 }, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
 ; CHECK-NEXT:    store i32 0, ptr [[A]], align 4
 ; CHECK-NEXT:    [[B:%.*]] = getelementptr i32, ptr [[A]], i32 1
 ; CHECK-NEXT:    store i32 1, ptr [[B]], align 4
 ; CHECK-NEXT:    call void @callee(ptr [[A]])
 ; CHECK-NEXT:    [[R:%.*]] = add i32 0, 1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %a = alloca {i32, i32}
-  call void @llvm.lifetime.start(i64 8, ptr %a)
+  call void @llvm.lifetime.start(ptr %a)
   store i32 0, ptr %a
   %b = getelementptr i32, ptr %a, i32 1
   store i32 1, ptr %b
@@ -495,7 +495,7 @@ define i32 @twoalloc_with_lifetimes() {
   %l1 = load i32, ptr %a
   %l2 = load i32, ptr %b
   %r = add i32 %l1, %l2
-  call void @llvm.lifetime.end(i64 8, ptr %a)
+  call void @llvm.lifetime.end(ptr %a)
   ret i32 %r
 }
 
diff --git a/llvm/test/Transforms/SROA/select-load.ll b/llvm/test/Transforms/SROA/select-load.ll
index 9de765071b535..359ecaa09111f 100644
--- a/llvm/test/Transforms/SROA/select-load.ll
+++ b/llvm/test/Transforms/SROA/select-load.ll
@@ -118,7 +118,7 @@ define i32 @interfering_lifetime(ptr %data, i64 %indvars.iv) {
   %min = alloca i32, align 4
   %arrayidx = getelementptr inbounds i32, ptr %data, i64 %indvars.iv
   %i1 = load i32, ptr %arrayidx, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %min)
+  call void @llvm.lifetime.start.p0(ptr %min)
   store i32 0, ptr %min, align 4
   %cmp.i.i = icmp slt i32 %i1, 0
   %__b.__a.i.i = select i1 %cmp.i.i, ptr %min, ptr %arrayidx
@@ -132,9 +132,9 @@ define i32 @clamp_load_to_constant_range(ptr %data, i64 %indvars.iv) {
 ; CHECK-PRESERVE-CFG-NEXT:    [[MIN:%.*]] = alloca i32, align 4
 ; CHECK-PRESERVE-CFG-NEXT:    [[MAX:%.*]] = alloca i32, align 4
 ; CHECK-PRESERVE-CFG-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[DATA:%.*]], i64 [[INDVARS_IV:%.*]]
-; CHECK-PRESERVE-CFG-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[MIN]])
+; CHECK-PRESERVE-CFG-NEXT:    call void @llvm.lifetime.start.p0(ptr [[MIN]])
 ; CHECK-PRESERVE-CFG-NEXT:    store i32 0, ptr [[MIN]], align 4
-; CHECK-PRESERVE-CFG-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[MAX]])
+; CHECK-PRESERVE-CFG-NEXT:    call void @llvm.lifetime.start.p0(ptr [[MAX]])
 ; CHECK-PRESERVE-CFG-NEXT:    store i32 4095, ptr [[MAX]], align 4
 ; CHECK-PRESERVE-CFG-NEXT:    [[I1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-PRESERVE-CFG-NEXT:    [[CMP_I_I:%.*]] = icmp slt i32 [[I1]], 0
@@ -167,9 +167,9 @@ define i32 @clamp_load_to_constant_range(ptr %data, i64 %indvars.iv) {
   %min = alloca i32, align 4
   %max = alloca i32, align 4
   %arrayidx = getelementptr inbounds i32, ptr %data, i64 %indvars.iv
-  call void @llvm.lifetime.start.p0(i64 4, ptr %min)
+  call void @llvm.lifetime.start.p0(ptr %min)
   store i32 0, ptr %min, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %max)
+  call void @llvm.lifetime.start.p0(ptr %max)
   store i32 4095, ptr %max, align 4
   %i1 = load i32, ptr %arrayidx, align 4
   %cmp.i.i = icmp slt i32 %i1, 0
@@ -482,6 +482,6 @@ define void @load_of_select_with_noundef_nonnull(ptr %buffer, i1 %b) {
 
 ; Ensure that the branch metadata is reversed to match the reversals above.
 
-declare void @llvm.lifetime.start.p0(i64, ptr )
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr )
+declare void @llvm.lifetime.end.p0(ptr)
 declare i32 @llvm.smax.i32(i32, i32)
diff --git a/llvm/test/Transforms/SROA/vector-conversion.ll b/llvm/test/Transforms/SROA/vector-conversion.ll
index 339810cb03ead..12df63db96a08 100644
--- a/llvm/test/Transforms/SROA/vector-conversion.ll
+++ b/llvm/test/Transforms/SROA/vector-conversion.ll
@@ -15,7 +15,7 @@ define <4 x i64> @vector_ptrtoint({<2 x ptr>, <2 x ptr>} %x) {
 ; CHECK-NEXT:    [[A_SROA_0_16_VECBLEND:%.*]] = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x i64> [[A_SROA_0_16_VEC_EXPAND]], <4 x i64> [[A_SROA_0_0_VECBLEND]]
 ; CHECK-NEXT:    ret <4 x i64> [[A_SROA_0_16_VECBLEND]]
 ;
-  %a = alloca {<2 x ptr>, <2 x ptr>}
+  %a = alloca {<2 x ptr>, <2 x ptr>}, align 32
 
   store {<2 x ptr>, <2 x ptr>} %x, ptr %a
 
@@ -36,7 +36,7 @@ define <4 x ptr> @vector_inttoptr({<2 x i64>, <2 x i64>} %x) {
 ; CHECK-NEXT:    [[A_SROA_0_16_VECBLEND:%.*]] = select <4 x i1> <i1 false, i1 false, i1 true, i1 true>, <4 x ptr> [[A_SROA_0_16_VEC_EXPAND]], <4 x ptr> [[A_SROA_0_0_VECBLEND]]
 ; CHECK-NEXT:    ret <4 x ptr> [[A_SROA_0_16_VECBLEND]]
 ;
-  %a = alloca {<2 x i64>, <2 x i64>}
+  %a = alloca {<2 x i64>, <2 x i64>}, align 32
 
   store {<2 x i64>, <2 x i64>} %x, ptr %a
 
@@ -57,7 +57,7 @@ define <2 x i64> @vector_ptrtointbitcast({<1 x ptr>, <1 x ptr>} %x) {
 ; CHECK-NEXT:    [[A_SROA_0_8_VEC_INSERT:%.*]] = insertelement <2 x i64> [[A_SROA_0_0_VEC_INSERT]], i64 [[TMP4]], i32 1
 ; CHECK-NEXT:    ret <2 x i64> [[A_SROA_0_8_VEC_INSERT]]
 ;
-  %a = alloca {<1 x ptr>, <1 x ptr>}
+  %a = alloca {<1 x ptr>, <1 x ptr>}, align 16
 
   store {<1 x ptr>, <1 x ptr>} %x, ptr %a
 
@@ -74,7 +74,7 @@ define <2 x ptr> @vector_inttoptrbitcast_vector({<16 x i8>, <16 x i8>} %x) {
 ; CHECK-NEXT:    [[X_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[X]], 1
 ; CHECK-NEXT:    ret <2 x ptr> [[TMP2]]
 ;
-  %a = alloca {<16 x i8>, <16 x i8>}
+  %a = alloca {<16 x i8>, <16 x i8>}, align 16
 
   store {<16 x i8>, <16 x i8>} %x, ptr %a
 
@@ -91,7 +91,7 @@ define <16 x i8> @vector_ptrtointbitcast_vector({<2 x ptr>, <2 x ptr>} %x) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to <16 x i8>
 ; CHECK-NEXT:    ret <16 x i8> [[TMP2]]
 ;
-  %a = alloca {<2 x ptr>, <2 x ptr>}
+  %a = alloca {<2 x ptr>, <2 x ptr>}, align 16
 
   store {<2 x ptr>, <2 x ptr>} %x, ptr %a
 
diff --git a/llvm/test/Transforms/SROA/vector-lifetime-intrinsic.ll b/llvm/test/Transforms/SROA/vector-lifetime-intrinsic.ll
index 561315be53368..60228a4239eb1 100644
--- a/llvm/test/Transforms/SROA/vector-lifetime-intrinsic.ll
+++ b/llvm/test/Transforms/SROA/vector-lifetime-intrinsic.ll
@@ -5,10 +5,10 @@
 target datalayout = "e-p:64:32-i64:32-v32:32-n32-S64"
 
 ; Function Attrs: nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #0
+declare void @llvm.lifetime.start.p0(ptr nocapture) #0
 
 ; Function Attrs: nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #0
+declare void @llvm.lifetime.end.p0(ptr nocapture) #0
 
 define void @wombat(<4 x float> %arg1) {
 ; CHECK-LABEL: @wombat(
@@ -19,10 +19,10 @@ define void @wombat(<4 x float> %arg1) {
 ;
 bb:
   %tmp = alloca <4 x float>, align 16
-  call void @llvm.lifetime.start.p0(i64 16, ptr %tmp)
+  call void @llvm.lifetime.start.p0(ptr %tmp)
   store <4 x float> %arg1, ptr %tmp, align 16
   %tmp18 = load <3 x float>, ptr %tmp
-  call void @llvm.lifetime.end.p0(i64 16, ptr %tmp)
+  call void @llvm.lifetime.end.p0(ptr %tmp)
   call void @wombat3(<3 x float> %tmp18)
   ret void
 }
diff --git a/llvm/test/Transforms/SROA/vector-promotion.ll b/llvm/test/Transforms/SROA/vector-promotion.ll
index ffa758e1a85a3..682e8e3c433b5 100644
--- a/llvm/test/Transforms/SROA/vector-promotion.ll
+++ b/llvm/test/Transforms/SROA/vector-promotion.ll
@@ -1534,7 +1534,7 @@ bb.5:
 }
 
 declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; CHECK-MODIFY-CFG: {{.*}}
 ; CHECK-PRESERVE-CFG: {{.*}}
diff --git a/llvm/test/Transforms/SafeStack/ARM/debug.ll b/llvm/test/Transforms/SafeStack/ARM/debug.ll
index a8c534c9b72a3..207475a47a032 100644
--- a/llvm/test/Transforms/SafeStack/ARM/debug.ll
+++ b/llvm/test/Transforms/SafeStack/ARM/debug.ll
@@ -29,15 +29,15 @@ entry:
 define void @f() local_unnamed_addr #1 !dbg !27 {
 entry:
   %c = alloca [16 x i8], align 1
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %c) #5, !dbg !35
+  call void @llvm.lifetime.start.p0(ptr nonnull %c) #5, !dbg !35
   call void @llvm.dbg.declare(metadata ptr %c, metadata !31, metadata !DIExpression()), !dbg !36
   call void @Capture(ptr nonnull %c) #5, !dbg !37
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %c) #5, !dbg !38
+  call void @llvm.lifetime.end.p0(ptr nonnull %c) #5, !dbg !38
   ret void, !dbg !38
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: nounwind readnone speculatable
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #3
@@ -45,7 +45,7 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #3
 declare void @Capture(ptr) local_unnamed_addr #4
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 attributes #0 = { norecurse nounwind readonly safestack "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind safestack "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv7-a,+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SafeStack/X86/call.ll b/llvm/test/Transforms/SafeStack/X86/call.ll
index 9592b33b620ba..f14e6764680aa 100644
--- a/llvm/test/Transforms/SafeStack/X86/call.ll
+++ b/llvm/test/Transforms/SafeStack/X86/call.ll
@@ -152,8 +152,8 @@ define void @call_lifetime(ptr %p) {
   ; CHECK: ret void
 entry:
   %q = alloca [100 x i8], align 16
-  call void @llvm.lifetime.start.p0(i64 100, ptr %q)
-  call void @llvm.lifetime.end.p0(i64 100, ptr %q)
+  call void @llvm.lifetime.start.p0(ptr %q)
+  call void @llvm.lifetime.end.p0(ptr %q)
   ret void
 }
 
@@ -167,5 +167,5 @@ declare void @readnone0(ptr nocapture readnone, ptr nocapture)
 
 declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1) nounwind argmemonly
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind argmemonly
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind argmemonly
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind argmemonly
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind argmemonly
diff --git a/llvm/test/Transforms/SafeStack/X86/coloring-ssp.ll b/llvm/test/Transforms/SafeStack/X86/coloring-ssp.ll
index 8ff369ef063e3..5192e476ed2fc 100644
--- a/llvm/test/Transforms/SafeStack/X86/coloring-ssp.ll
+++ b/llvm/test/Transforms/SafeStack/X86/coloring-ssp.ll
@@ -14,19 +14,19 @@ entry:
   %x = alloca i64, align 8
   %y = alloca i64, align 8
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK:  getelementptr i8, ptr %[[USP]], i32 -16
   call void @capture64(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK:  getelementptr i8, ptr %[[USP]], i32 -16
   call void @capture64(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
 
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @capture64(ptr)
diff --git a/llvm/test/Transforms/SafeStack/X86/coloring.ll b/llvm/test/Transforms/SafeStack/X86/coloring.ll
index 22e1487bdcfce..288ae009581ff 100644
--- a/llvm/test/Transforms/SafeStack/X86/coloring.ll
+++ b/llvm/test/Transforms/SafeStack/X86/coloring.ll
@@ -11,30 +11,30 @@ entry:
   %x = alloca i32, align 4
   %x1 = alloca i32, align 4
   %x2 = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
 
 ; CHECK:  %[[A1:.*]] = getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:  call void @capture(ptr nonnull %[[A1]])
 
   call void @capture(ptr nonnull %x)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x1)
+  call void @llvm.lifetime.end.p0(ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x1)
 
 ; CHECK:  %[[B1:.*]] = getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:  call void @capture(ptr nonnull %[[B1]])
 
   call void @capture(ptr nonnull %x1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x1)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x2)
+  call void @llvm.lifetime.end.p0(ptr %x1)
+  call void @llvm.lifetime.start.p0(ptr %x2)
 
 ; CHECK:  %[[C1:.*]] = getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:  call void @capture(ptr nonnull %[[C1]])
 
   call void @capture(ptr nonnull %x2)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x2)
+  call void @llvm.lifetime.end.p0(ptr %x2)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @capture(ptr)
diff --git a/llvm/test/Transforms/SafeStack/X86/coloring2.ll b/llvm/test/Transforms/SafeStack/X86/coloring2.ll
index ae5f375972365..a4157cbcf3e13 100644
--- a/llvm/test/Transforms/SafeStack/X86/coloring2.ll
+++ b/llvm/test/Transforms/SafeStack/X86/coloring2.ll
@@ -14,21 +14,21 @@ entry:
   %y = alloca i32, align 4
   %z = alloca i32, align 4
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %x)
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %y)
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
   call void @capture32(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -8
   call void @capture32(ptr %z)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %z)
 
   ret void
 }
@@ -42,11 +42,11 @@ entry:
   %x = alloca i32, align 4
   %y = alloca i32, align 4
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -8
   call void @capture32(ptr %y)
@@ -65,21 +65,21 @@ entry:
   %y = alloca i32, align 4
   %z = alloca i64, align 4
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %y)
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -8
   call void @capture32(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %z)
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -8
   call void @capture64(ptr %z)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %z)
 
   ret void
 }
@@ -95,9 +95,9 @@ entry:
   %z = alloca i64, align 4
   %y = alloca i32, align 4
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %z)
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -16
   call void @capture32(ptr %x)
@@ -108,9 +108,9 @@ entry:
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -8
   call void @capture64(ptr %z)
 
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %z)
 
   ret void
 }
@@ -147,8 +147,8 @@ entry:
   %z = alloca i64, align 8
   %z1 = alloca i64, align 8
   %z2 = alloca i64, align 8
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x1)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x2)
+  call void @llvm.lifetime.start.p0(ptr %x1)
+  call void @llvm.lifetime.start.p0(ptr %x2)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -8
 ; CHECK:   call void @capture64(
   call void @capture64(ptr nonnull %x1)
@@ -158,62 +158,62 @@ entry:
   br i1 %a, label %if.then, label %if.else4
 
 if.then:                                          ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -24
 ; CHECK:   call void @capture64(
   call void @capture64(ptr nonnull %y)
   br i1 %b, label %if.then3, label %if.else
 
 if.then3:                                         ; preds = %if.then
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y1)
+  call void @llvm.lifetime.start.p0(ptr %y1)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -32
 ; CHECK:   call void @capture64(
   call void @capture64(ptr nonnull %y1)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y1)
+  call void @llvm.lifetime.end.p0(ptr %y1)
   br label %if.end
 
 if.else:                                          ; preds = %if.then
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y2)
+  call void @llvm.lifetime.start.p0(ptr %y2)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -32
 ; CHECK:   call void @capture64(
   call void @capture64(ptr nonnull %y2)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y2)
+  call void @llvm.lifetime.end.p0(ptr %y2)
   br label %if.end
 
 if.end:                                           ; preds = %if.else, %if.then3
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
   br label %if.end9
 
 if.else4:                                         ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.start.p0(ptr %z)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -24
 ; CHECK:   call void @capture64(
   call void @capture64(ptr nonnull %z)
   br i1 %b, label %if.then6, label %if.else7
 
 if.then6:                                         ; preds = %if.else4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z1)
+  call void @llvm.lifetime.start.p0(ptr %z1)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -32
 ; CHECK:   call void @capture64(
   call void @capture64(ptr nonnull %z1)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z1)
+  call void @llvm.lifetime.end.p0(ptr %z1)
   br label %if.end8
 
 if.else7:                                         ; preds = %if.else4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %z2)
+  call void @llvm.lifetime.start.p0(ptr %z2)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -32
 ; CHECK:   call void @capture64(
   call void @capture64(ptr nonnull %z2)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z2)
+  call void @llvm.lifetime.end.p0(ptr %z2)
   br label %if.end8
 
 if.end8:                                          ; preds = %if.else7, %if.then6
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %z)
   br label %if.end9
 
 if.end9:                                          ; preds = %if.end8, %if.end
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x2)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x1)
+  call void @llvm.lifetime.end.p0(ptr %x2)
+  call void @llvm.lifetime.end.p0(ptr %x1)
   ret void
 }
 
@@ -225,21 +225,21 @@ entry:
 ; CHECK-NEXT:   getelementptr i8, ptr %[[USP]], i32 -16
   %x = alloca i32, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %x)
   br i1 %d, label %bb2, label %bb3
 bb2:
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -8
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %x)
   ret void
 bb3:
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   ret void
 }
 
@@ -250,18 +250,18 @@ entry:
 ; CHECK-NEXT:   getelementptr i8, ptr %[[USP]], i32 -16
   %x = alloca i32, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   br i1 %d, label %bb2, label %bb3
 bb2:
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
   ret void
 bb3:
   ret void
@@ -275,14 +275,14 @@ entry:
 ; CHECK-NEXT:   getelementptr i8, ptr %[[USP]], i32 -16
   %x = alloca i32, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   br i1 %d, label %bb2, label %bb3
 bb2:
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %y)
@@ -299,14 +299,14 @@ entry:
 ; CHECK-NEXT:   getelementptr i8, ptr %[[USP]], i32 -16
   %x = alloca i32, align 4
   %y = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %x)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %x)
   br i1 %d, label %bb2, label %bb3
 bb2:
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %x)
+  call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %y)
@@ -326,10 +326,10 @@ entry:
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %x)
+  call void @llvm.lifetime.end.p0(ptr %x)
   br i1 %d, label %bb2, label %bb3
 bb2:
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %y)
+  call void @llvm.lifetime.start.p0(ptr %y)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -8
 ; CHECK:   call void @capture32(
   call void @capture32(ptr %y)
@@ -347,26 +347,26 @@ entry:
   %B.i2 = alloca [100 x i32], align 4
   %A.i = alloca [100 x i32], align 4
   %B.i = alloca [100 x i32], align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %A.i)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %B.i)
+  call void @llvm.lifetime.start.p0(ptr %A.i)
+  call void @llvm.lifetime.start.p0(ptr %B.i)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -400
 ; CHECK:   call void @capture100x32(
   call void @capture100x32(ptr %A.i)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -800
 ; CHECK:   call void @capture100x32(
   call void @capture100x32(ptr %B.i)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %A.i)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %B.i)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %A.i1)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %B.i2)
+  call void @llvm.lifetime.end.p0(ptr %A.i)
+  call void @llvm.lifetime.end.p0(ptr %B.i)
+  call void @llvm.lifetime.start.p0(ptr %A.i1)
+  call void @llvm.lifetime.start.p0(ptr %B.i2)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -400
 ; CHECK:   call void @capture100x32(
   call void @capture100x32(ptr %A.i1)
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -800
 ; CHECK:   call void @capture100x32(
   call void @capture100x32(ptr %B.i2)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %A.i1)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %B.i2)
+  call void @llvm.lifetime.end.p0(ptr %A.i1)
+  call void @llvm.lifetime.end.p0(ptr %B.i2)
   ret void
 }
 
@@ -378,11 +378,11 @@ entry:
   %buf1 = alloca i8, i32 100000, align 16
   %buf2 = alloca i8, i32 100000, align 16
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %buf1)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %buf1)
+  call void @llvm.lifetime.start.p0(ptr %buf1)
+  call void @llvm.lifetime.end.p0(ptr %buf1)
 
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %buf1)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %buf2)
+  call void @llvm.lifetime.start.p0(ptr %buf1)
+  call void @llvm.lifetime.start.p0(ptr %buf2)
   call void @capture8(ptr %buf1)
   call void @capture8(ptr %buf2)
   ret void
@@ -404,12 +404,12 @@ entry:
   %B.i2 = alloca [100 x i32], align 4
   %A.i = alloca [100 x i32], align 4
   %B.i = alloca [100 x i32], align 4
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %A.i) nounwind
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %B.i) nounwind
+  call void @llvm.lifetime.start.p0(ptr %A.i) nounwind
+  call void @llvm.lifetime.start.p0(ptr %B.i) nounwind
   call void @capture100x32(ptr %A.i)
   call void @capture100x32(ptr %B.i)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %A.i) nounwind
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %B.i) nounwind
+  call void @llvm.lifetime.end.p0(ptr %A.i) nounwind
+  call void @llvm.lifetime.end.p0(ptr %B.i) nounwind
   br label %block2
 
 block2:
@@ -429,13 +429,13 @@ entry:
   %a.i = alloca [4 x %struct.Klass], align 16
   %b.i = alloca [4 x %struct.Klass], align 16
   ; I am used outside the lifetime zone below:
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %a.i)
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %b.i)
+  call void @llvm.lifetime.start.p0(ptr %a.i)
+  call void @llvm.lifetime.start.p0(ptr %b.i)
   call void @capture8(ptr %a.i)
   call void @capture8(ptr %b.i)
   %z3 = load i32, ptr %a.i, align 16
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %a.i)
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %b.i)
+  call void @llvm.lifetime.end.p0(ptr %a.i)
+  call void @llvm.lifetime.end.p0(ptr %b.i)
   ret i32 %z3
 }
 
@@ -445,12 +445,12 @@ entry:
 ; CHECK:        %[[USP:.*]] = load ptr, ptr @__safestack_unsafe_stack_ptr
 ; CHECK-NEXT:   getelementptr i8, ptr %[[USP]], i32 -16
   %x = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x) nounwind
+  call void @llvm.lifetime.start.p0(ptr %x) nounwind
   br label %l2
 
 l2:
   call void @capture8(ptr %x)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %x) nounwind
+  call void @llvm.lifetime.end.p0(ptr %x) nounwind
   br label %l2
 }
 
@@ -463,25 +463,25 @@ entry:
 ; CHECK-NEXT:   getelementptr i8, ptr %[[USP]], i32 -16
   %x = alloca i8, align 4
   %y = alloca i8, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x) nounwind
+  call void @llvm.lifetime.start.p0(ptr %x) nounwind
   br label %l2
 
 l2:
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -8
-  call void @llvm.lifetime.start.p0(i64 4, ptr %y) nounwind
+  call void @llvm.lifetime.start.p0(ptr %y) nounwind
   call void @capture8(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 4, ptr %y) nounwind
+  call void @llvm.lifetime.end.p0(ptr %y) nounwind
 
 ; CHECK:   getelementptr i8, ptr %[[USP]], i32 -4
-  call void @llvm.lifetime.start.p0(i64 4, ptr %x) nounwind
+  call void @llvm.lifetime.start.p0(ptr %x) nounwind
   call void @capture8(ptr %x)
   br label %l2
 }
 
 attributes #0 = { safestack }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @capture8(ptr)
 declare void @capture32(ptr)
 declare void @capture64(ptr)
diff --git a/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll b/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll
index 7a1fdc016ac12..e60522fb3e53b 100644
--- a/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll
+++ b/llvm/test/Transforms/SafeStack/X86/debug-loc2.ll
@@ -43,12 +43,12 @@ entry:
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 declare void @capture(ptr) #2
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #3
diff --git a/llvm/test/Transforms/SafeStack/X86/layout-frag.ll b/llvm/test/Transforms/SafeStack/X86/layout-frag.ll
index b858fd6131537..8a5362b5bef7b 100644
--- a/llvm/test/Transforms/SafeStack/X86/layout-frag.ll
+++ b/llvm/test/Transforms/SafeStack/X86/layout-frag.ll
@@ -13,16 +13,16 @@ entry:
   %x2 = alloca i64, align 8
 
 
-  call void @llvm.lifetime.start.p0(i64 8, ptr %x0)
+  call void @llvm.lifetime.start.p0(ptr %x0)
   call void @capture64(ptr %x0)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %x0)
+  call void @llvm.lifetime.end.p0(ptr %x0)
 
-  call void @llvm.lifetime.start.p0(i64 1, ptr %x1)
-  call void @llvm.lifetime.start.p0(i64 8, ptr %x2)
+  call void @llvm.lifetime.start.p0(ptr %x1)
+  call void @llvm.lifetime.start.p0(ptr %x2)
   call void @capture8(ptr %x1)
   call void @capture64(ptr %x2)
-  call void @llvm.lifetime.end.p0(i64 1, ptr %x1)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %x2)
+  call void @llvm.lifetime.end.p0(ptr %x1)
+  call void @llvm.lifetime.end.p0(ptr %x2)
 
 ; Test that i64 allocas share space.
 ; CHECK: getelementptr i8, ptr %unsafe_stack_ptr, i32 -8
@@ -32,7 +32,7 @@ entry:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare void @capture8(ptr)
 declare void @capture64(ptr)
diff --git a/llvm/test/Transforms/SafeStack/X86/no-crash-on-lifetime.ll b/llvm/test/Transforms/SafeStack/X86/no-crash-on-lifetime.ll
index 76c638ebbd211..c01ca2f97e13d 100644
--- a/llvm/test/Transforms/SafeStack/X86/no-crash-on-lifetime.ll
+++ b/llvm/test/Transforms/SafeStack/X86/no-crash-on-lifetime.ll
@@ -9,9 +9,9 @@ define dso_local void @_ZN1s1tE1F(ptr byval(%class.F) %g) local_unnamed_addr saf
 entry:
   %ref.tmp.i.i.i = alloca i64, align 1
   call void undef(ptr %g)
-  call void @llvm.lifetime.start.p0(i64 3, ptr %ref.tmp.i.i.i)
+  call void @llvm.lifetime.start.p0(ptr %ref.tmp.i.i.i)
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
diff --git a/llvm/test/Transforms/SampleProfile/csspgo-import-list-callee-samples.ll b/llvm/test/Transforms/SampleProfile/csspgo-import-list-callee-samples.ll
index ba66548406f4c..3ae0aea6e7573 100644
--- a/llvm/test/Transforms/SampleProfile/csspgo-import-list-callee-samples.ll
+++ b/llvm/test/Transforms/SampleProfile/csspgo-import-list-callee-samples.ll
@@ -61,10 +61,10 @@ entry:
 }
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.start.p0(ptr nocapture) #3
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.end.p0(ptr nocapture) #3
 
 ; Function Attrs: nounwind uwtable
 define dso_local i32 @main() #0 !dbg !28 {
diff --git a/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll b/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll
index 5fe80dbf6445a..c7617c1de9bd8 100644
--- a/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll
+++ b/llvm/test/Transforms/SampleProfile/entry_counts_cold.ll
@@ -43,7 +43,7 @@ entry:
   %a = alloca i32, align 4
   store ptr %p, ptr %p.addr, align 8, !tbaa !15
   call void @llvm.dbg.declare(metadata ptr %p.addr, metadata !33, metadata !DIExpression()), !dbg !35
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a) #4, !dbg !36
+  call void @llvm.lifetime.start.p0(ptr %a) #4, !dbg !36
   call void @llvm.dbg.declare(metadata ptr %a, metadata !34, metadata !DIExpression()), !dbg !37
   %0 = load ptr, ptr %p.addr, align 8, !dbg !38, !tbaa !15
   %arrayidx = getelementptr inbounds i32, ptr %0, i64 3, !dbg !38
@@ -58,7 +58,7 @@ entry:
   store i32 %call, ptr %a, align 4, !dbg !43, !tbaa !25
   %5 = load i32, ptr %a, align 4, !dbg !44, !tbaa !25
   %add2 = add nsw i32 %5, 1, !dbg !45
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a) #4, !dbg !46
+  call void @llvm.lifetime.end.p0(ptr %a) #4, !dbg !46
   ret i32 %add2, !dbg !47
 }
 
@@ -86,10 +86,10 @@ entry:
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 declare void @baz(...) #3
 
diff --git a/llvm/test/Transforms/SampleProfile/entry_counts_missing_dbginfo.ll b/llvm/test/Transforms/SampleProfile/entry_counts_missing_dbginfo.ll
index b8e106418343a..0e629216d4fca 100644
--- a/llvm/test/Transforms/SampleProfile/entry_counts_missing_dbginfo.ll
+++ b/llvm/test/Transforms/SampleProfile/entry_counts_missing_dbginfo.ll
@@ -53,7 +53,7 @@ entry:
   %a = alloca i32, align 4
   store ptr %p, ptr %p.addr, align 8, !tbaa !15
   call void @llvm.dbg.declare(metadata ptr %p.addr, metadata !33, metadata !DIExpression()), !dbg !35
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a) #4, !dbg !36
+  call void @llvm.lifetime.start.p0(ptr %a) #4, !dbg !36
   call void @llvm.dbg.declare(metadata ptr %a, metadata !34, metadata !DIExpression()), !dbg !37
   %0 = load ptr, ptr %p.addr, align 8, !dbg !38, !tbaa !15
   %arrayidx = getelementptr inbounds i32, ptr %0, i64 3, !dbg !38
@@ -68,7 +68,7 @@ entry:
   store i32 %call, ptr %a, align 4, !dbg !43, !tbaa !25
   %5 = load i32, ptr %a, align 4, !dbg !44, !tbaa !25
   %add2 = add nsw i32 %5, 1, !dbg !45
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a) #4, !dbg !46
+  call void @llvm.lifetime.end.p0(ptr %a) #4, !dbg !46
   ret i32 %add2, !dbg !47
 }
 
@@ -96,10 +96,10 @@ entry:
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 declare void @baz(...) #3
 
diff --git a/llvm/test/Transforms/SampleProfile/non-probe-stale-profile-matching.ll b/llvm/test/Transforms/SampleProfile/non-probe-stale-profile-matching.ll
index 3ca94a4563675..2b091a1e3fda9 100644
--- a/llvm/test/Transforms/SampleProfile/non-probe-stale-profile-matching.ll
+++ b/llvm/test/Transforms/SampleProfile/non-probe-stale-profile-matching.ll
@@ -151,10 +151,10 @@ for.end:                                          ; preds = %cleanup, %if.then
 }
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.start.p0(ptr nocapture) #3
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.end.p0(ptr nocapture) #3
 
 attributes #0 = { noinline nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "use-sample-profile" }
 attributes #1 = { alwaysinline nounwind uwtable "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cmov,+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" "use-sample-profile" }
diff --git a/llvm/test/Transforms/SampleProfile/profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/profile-mismatch.ll
index 42bc1b81f6705..0a1b8960bf97a 100644
--- a/llvm/test/Transforms/SampleProfile/profile-mismatch.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-mismatch.ll
@@ -43,13 +43,13 @@ define dso_local i32 @foo(i32 noundef %x) #0 !dbg !12 {
 entry:
   %y = alloca i32, align 4
   call void @llvm.dbg.value(metadata i32 %x, metadata !16, metadata !DIExpression()), !dbg !18
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %y), !dbg !19
+  call void @llvm.lifetime.start.p0(ptr nonnull %y), !dbg !19
   call void @llvm.dbg.declare(metadata ptr %y, metadata !17, metadata !DIExpression()), !dbg !20
   %add = add nsw i32 %x, 1, !dbg !21
   store volatile i32 %add, ptr %y, align 4, !dbg !20, !tbaa !22
   %y.0. = load volatile i32, ptr %y, align 4, !dbg !26, !tbaa !22
   %add1 = add nsw i32 %y.0., 1, !dbg !27
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %y), !dbg !28
+  call void @llvm.lifetime.end.p0(ptr nonnull %y), !dbg !28
   ret i32 %add1, !dbg !29
 }
 
@@ -57,10 +57,10 @@ entry:
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 ; Function Attrs: argmemonly mustprogress nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly mustprogress nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 ; Function Attrs: noinline nounwind uwtable
 define dso_local i32 @bar(i32 noundef %x) #3 !dbg !30 {
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-discriminator.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-discriminator.ll
index 6d4429b37a52b..26ae1988ed748 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-discriminator.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-discriminator.ll
@@ -31,8 +31,8 @@ bb3:
 }
 
 declare void @_Z3barv() #1
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind argmemonly
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind argmemonly
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind argmemonly
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind argmemonly
 
 attributes #0 = { uwtable "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-icp-factor.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-icp-factor.ll
index b662efab9b2f2..383289edd6947 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-icp-factor.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-icp-factor.ll
@@ -60,12 +60,12 @@ bb:
   %i3 = alloca i32, align 4
   store i32 0, ptr %i, align 4
   call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 1, i32 0, i64 -1), !dbg !62
-  call void @llvm.lifetime.start.p0(i64 8, ptr %i1), !dbg !62
+  call void @llvm.lifetime.start.p0(ptr %i1), !dbg !62
   call void @llvm.dbg.declare(metadata ptr %i1, metadata !57, metadata !DIExpression()), !dbg !63
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i2), !dbg !64
+  call void @llvm.lifetime.start.p0(ptr %i2), !dbg !64
   call void @llvm.dbg.declare(metadata ptr %i2, metadata !59, metadata !DIExpression()), !dbg !65
   store i32 0, ptr %i2, align 4, !dbg !65, !tbaa !19
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i3), !dbg !66
+  call void @llvm.lifetime.start.p0(ptr %i3), !dbg !66
   call void @llvm.dbg.declare(metadata ptr %i3, metadata !60, metadata !DIExpression()), !dbg !67
   store i32 0, ptr %i3, align 4, !dbg !67, !tbaa !19
   br label %bb7, !dbg !66
@@ -78,7 +78,7 @@ bb7:                                              ; preds = %bb25, %bb
 
 bb10:                                             ; preds = %bb7
   call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 3, i32 0, i64 -1), !dbg !72
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i3), !dbg !72
+  call void @llvm.lifetime.end.p0(ptr %i3), !dbg !72
   br label %bb28
 
 bb12:                                             ; preds = %bb7
@@ -119,16 +119,16 @@ bb28:                                             ; preds = %bb10
   call void @llvm.pseudoprobe(i64 -2624081020897602054, i64 9, i32 0, i64 -1), !dbg !92
   %i29 = load i32, ptr %i2, align 4, !dbg !92, !tbaa !19
   %i30 = call i32 (ptr, ...) @printf(ptr @.str, i32 %i29), !dbg !93
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i2), !dbg !95
-  call void @llvm.lifetime.end.p0(i64 8, ptr %i1), !dbg !95
+  call void @llvm.lifetime.end.p0(ptr %i2), !dbg !95
+  call void @llvm.lifetime.end.p0(ptr %i1), !dbg !95
   ret i32 0, !dbg !96
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 declare dso_local i32 @printf(ptr, ...)
 
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
index 22317e60cd081..e1d717c63e9ed 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll
@@ -70,14 +70,14 @@ define dso_local i32 @foo(i32 noundef %x) #0 !dbg !16 {
 entry:
   %y = alloca i32, align 4
   call void @llvm.dbg.value(metadata i32 %x, metadata !20, metadata !DIExpression()), !dbg !22
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %y), !dbg !23
+  call void @llvm.lifetime.start.p0(ptr nonnull %y), !dbg !23
   call void @llvm.dbg.declare(metadata ptr %y, metadata !21, metadata !DIExpression()), !dbg !24
   call void @llvm.pseudoprobe(i64 6699318081062747564, i64 1, i32 0, i64 -1), !dbg !25
   %add = add nsw i32 %x, 1, !dbg !26
   store volatile i32 %add, ptr %y, align 4, !dbg !24, !tbaa !27
   %y.0. = load volatile i32, ptr %y, align 4, !dbg !31, !tbaa !27
   %add1 = add nsw i32 %y.0., 1, !dbg !32
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %y), !dbg !33
+  call void @llvm.lifetime.end.p0(ptr nonnull %y), !dbg !33
   ret i32 %add1, !dbg !34
 }
 
@@ -85,10 +85,10 @@ entry:
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
 
 ; Function Attrs: argmemonly mustprogress nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly mustprogress nocallback nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 ; Function Attrs: noinline nounwind uwtable
 define dso_local i32 @bar(i32 noundef %x) #3 !dbg !35 {
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll
index cdd365b6fb673..c0976dea3c580 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching-LCS.ll
@@ -119,10 +119,10 @@ if.end:                                           ; preds = %if.else, %if.then
 }
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.start.p0(ptr nocapture) #3
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.end.p0(ptr nocapture) #3
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
 declare void @llvm.pseudoprobe(i64, i64, i32, i64) #4
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
index 20be0c2fec7f2..0c38d9cad4520 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-matching.ll
@@ -217,10 +217,10 @@ for.end:                                          ; preds = %cleanup, %if.then
 }
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #4
+declare void @llvm.lifetime.start.p0(ptr nocapture) #4
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #4
+declare void @llvm.lifetime.end.p0(ptr nocapture) #4
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
 declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata) #1
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-name-similarity.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-name-similarity.ll
index 4e435f4586465..dbf3dda46ee28 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-name-similarity.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-name-similarity.ll
@@ -57,10 +57,10 @@ for.body:                                         ; preds = %for.cond
 }
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none)) #2
+declare void @llvm.lifetime.start.p0(ptr captures(none)) #2
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none)) #2
+declare void @llvm.lifetime.end.p0(ptr captures(none)) #2
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
 declare void @llvm.pseudoprobe(i64, i64, i32, i64) #3
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming-recursive.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming-recursive.ll
index d9db804b56364..e246d260c60e0 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming-recursive.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming-recursive.ll
@@ -70,13 +70,13 @@ for.body:                                         ; preds = %for.cond
 }
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind speculatable willreturn memory(none)
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
 declare void @llvm.pseudoprobe(i64, i64, i32, i64) #3
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming.ll
index 6bf09ced6aad1..d1c5a9d9bc2a3 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-renaming.ll
@@ -175,10 +175,10 @@ for.body:                                         ; preds = %for.cond
 }
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.start.p0(ptr nocapture) #3
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #3
+declare void @llvm.lifetime.end.p0(ptr nocapture) #3
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
 declare void @llvm.pseudoprobe(i64, i64, i32, i64) #4
diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-toplev-func.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-toplev-func.ll
index c839364f23553..2ed187227e186 100644
--- a/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-toplev-func.ll
+++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-stale-profile-toplev-func.ll
@@ -85,10 +85,10 @@ for.body:                                         ; preds = %for.cond
 }
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 ; Function Attrs: mustprogress nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
 declare void @llvm.pseudoprobe(i64, i64, i32, i64) #3
diff --git a/llvm/test/Transforms/SampleProfile/remarks.ll b/llvm/test/Transforms/SampleProfile/remarks.ll
index 9c0143ae65ca7..3cb91b717c6ef 100644
--- a/llvm/test/Transforms/SampleProfile/remarks.ll
+++ b/llvm/test/Transforms/SampleProfile/remarks.ll
@@ -121,10 +121,10 @@ define i64 @_Z3foov() #0 !dbg !4 {
 entry:
   %sum = alloca i64, align 8
   %i = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 8, ptr %sum) #4, !dbg !19
+  call void @llvm.lifetime.start.p0(ptr %sum) #4, !dbg !19
   call void @llvm.dbg.declare(metadata ptr %sum, metadata !9, metadata !20), !dbg !21
   store i64 0, ptr %sum, align 8, !dbg !21, !tbaa !22
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #4, !dbg !26
+  call void @llvm.lifetime.start.p0(ptr %i) #4, !dbg !26
   call void @llvm.dbg.declare(metadata ptr %i, metadata !10, metadata !20), !dbg !27
   store i32 0, ptr %i, align 4, !dbg !27, !tbaa !28
   br label %for.cond, !dbg !26
@@ -135,7 +135,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !35
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #4, !dbg !36
+  call void @llvm.lifetime.end.p0(ptr %i) #4, !dbg !36
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -173,12 +173,12 @@ for.inc:                                          ; preds = %if.end
 
 for.end:                                          ; preds = %for.cond.cleanup
   %7 = load i64, ptr %sum, align 8, !dbg !53, !tbaa !22
-  call void @llvm.lifetime.end.p0(i64 8, ptr %sum) #4, !dbg !54
+  call void @llvm.lifetime.end.p0(ptr %sum) #4, !dbg !54
   ret i64 %7, !dbg !55
 }
 
 ; Function Attrs: nounwind argmemonly
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
@@ -189,7 +189,7 @@ define i32 @rand() #3 !dbg !59 {
 }
 
 ; Function Attrs: nounwind argmemonly
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define i32 @main() #0 !dbg !13 {
diff --git a/llvm/test/Transforms/Scalarizer/intrinsics.ll b/llvm/test/Transforms/Scalarizer/intrinsics.ll
index cee44ef434260..070c765294ba1 100644
--- a/llvm/test/Transforms/Scalarizer/intrinsics.ll
+++ b/llvm/test/Transforms/Scalarizer/intrinsics.ll
@@ -8,6 +8,7 @@ declare <2 x float> @llvm.sqrt.v2f32(<2 x float>)
 declare <2 x float> @llvm.minnum.v2f32(<2 x float>, <2 x float>)
 declare <2 x float> @llvm.minimum.v2f32(<2 x float>, <2 x float>)
 declare <2 x float> @llvm.maximum.v2f32(<2 x float>, <2 x float>)
+declare <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float>, <2 x i32>)
 
 ; Ternary fp
 declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
@@ -32,6 +33,8 @@ declare <2 x i32> @llvm.fptoui.sat.v2i32.v2f32(<2 x float>)
 ; Unary fp operand, int return type
 declare <2 x i32> @llvm.lrint.v2i32.v2f32(<2 x float>)
 declare <2 x i32> @llvm.llrint.v2i32.v2f32(<2 x float>)
+declare <2 x i32> @llvm.lround.v2i32.v2f32(<2 x float>)
+declare <2 x i32> @llvm.llround.v2i32.v2f32(<2 x float>)
 
 ; Bool return type, overloaded on fp operand type
 declare <2 x i1> @llvm.is.fpclass(<2 x float>, i32)
@@ -159,6 +162,22 @@ define <2 x float> @scalarize_powi_v2f32(<2 x float> %x, i32 %y) #0 {
   ret <2 x float> %powi
 }
 
+define <2 x float> @scalarize_ldexp_v2f32(<2 x float> %x, <2 x i32> %y) #0 {
+; CHECK-LABEL: @scalarize_ldexp_v2f32(
+; CHECK-NEXT:    [[X_I0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[Y:%.*]] = extractelement <2 x i32> [[Y1:%.*]], i64 0
+; CHECK-NEXT:    [[POWI_I0:%.*]] = call float @llvm.ldexp.f32.i32(float [[X_I0]], i32 [[Y]])
+; CHECK-NEXT:    [[X_I1:%.*]] = extractelement <2 x float> [[X]], i64 1
+; CHECK-NEXT:    [[Y_I1:%.*]] = extractelement <2 x i32> [[Y1]], i64 1
+; CHECK-NEXT:    [[POWI_I1:%.*]] = call float @llvm.ldexp.f32.i32(float [[X_I1]], i32 [[Y_I1]])
+; CHECK-NEXT:    [[POWI_UPTO0:%.*]] = insertelement <2 x float> poison, float [[POWI_I0]], i64 0
+; CHECK-NEXT:    [[POWI:%.*]] = insertelement <2 x float> [[POWI_UPTO0]], float [[POWI_I1]], i64 1
+; CHECK-NEXT:    ret <2 x float> [[POWI]]
+;
+  %powi = call <2 x float> @llvm.ldexp.v2f32.v2i32(<2 x float> %x, <2 x i32> %y)
+  ret <2 x float> %powi
+}
+
 define <2 x i32> @scalarize_smul_fix_sat_v2i32(<2 x i32> %x) #0 {
 ; CHECK-LABEL: @scalarize_smul_fix_sat_v2i32(
 ; CHECK-NEXT:    [[X_I0:%.*]] = extractelement <2 x i32> [[X:%.*]], i64 0
@@ -243,6 +262,34 @@ define <2 x i32> @scalarize_llrint(<2 x float> %x) #0 {
   ret <2 x i32> %rnd
 }
 
+define <2 x i32> @scalarize_lround(<2 x float> %x) #0 {
+; CHECK-LABEL: @scalarize_lround(
+; CHECK-NEXT:    [[X_I0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[RND_I0:%.*]] = call i32 @llvm.lround.i32.f32(float [[X_I0]])
+; CHECK-NEXT:    [[X_I1:%.*]] = extractelement <2 x float> [[X]], i64 1
+; CHECK-NEXT:    [[RND_I1:%.*]] = call i32 @llvm.lround.i32.f32(float [[X_I1]])
+; CHECK-NEXT:    [[RND_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[RND_I0]], i64 0
+; CHECK-NEXT:    [[RND:%.*]] = insertelement <2 x i32> [[RND_UPTO0]], i32 [[RND_I1]], i64 1
+; CHECK-NEXT:    ret <2 x i32> [[RND]]
+;
+  %rnd = call <2 x i32> @llvm.lround.v2i32.v2f32(<2 x float> %x)
+  ret <2 x i32> %rnd
+}
+
+define <2 x i32> @scalarize_llround(<2 x float> %x) #0 {
+; CHECK-LABEL: @scalarize_llround(
+; CHECK-NEXT:    [[X_I0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0
+; CHECK-NEXT:    [[RND_I0:%.*]] = call i32 @llvm.llround.i32.f32(float [[X_I0]])
+; CHECK-NEXT:    [[X_I1:%.*]] = extractelement <2 x float> [[X]], i64 1
+; CHECK-NEXT:    [[RND_I1:%.*]] = call i32 @llvm.llround.i32.f32(float [[X_I1]])
+; CHECK-NEXT:    [[RND_UPTO0:%.*]] = insertelement <2 x i32> poison, i32 [[RND_I0]], i64 0
+; CHECK-NEXT:    [[RND:%.*]] = insertelement <2 x i32> [[RND_UPTO0]], i32 [[RND_I1]], i64 1
+; CHECK-NEXT:    ret <2 x i32> [[RND]]
+;
+  %rnd = call <2 x i32> @llvm.llround.v2i32.v2f32(<2 x float> %x)
+  ret <2 x i32> %rnd
+}
+
 define <2 x i1> @scalarize_is_fpclass(<2 x float> %x) #0 {
 ; CHECK-LABEL: @scalarize_is_fpclass(
 ; CHECK-NEXT:    [[X_I0:%.*]] = extractelement <2 x float> [[X:%.*]], i64 0
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/preserve-inbounds.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/preserve-inbounds.ll
index 2a5b678e91fd8..0d21a99153ea0 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/preserve-inbounds.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/preserve-inbounds.ll
@@ -311,7 +311,7 @@ entry:
 define ptr @nuw_inbounds_implies_nuw_inbounds_trunc_nuw(ptr %p, i128 %i) {
 ; CHECK-LABEL: @nuw_inbounds_implies_nuw_inbounds_trunc_nuw(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw i128 [[I:%.*]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc i128 [[I:%.*]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw i32, ptr [[P:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i64 4
 ; CHECK-NEXT:    ret ptr [[ARRAYIDX2]]
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/rebuild-trunc.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/rebuild-trunc.ll
new file mode 100644
index 0000000000000..a93257f7e564a
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/rebuild-trunc.ll
@@ -0,0 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -passes=separate-const-offset-from-gep -S | FileCheck %s
+
+; Verify that we drop "nuw" from trunc.
+define ptr @pr154116_nuw(ptr %p, i128 %i) {
+; CHECK-LABEL: define ptr @pr154116_nuw(
+; CHECK-SAME: ptr [[P:%.*]], i128 [[I:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i128 [[I]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 80
+; CHECK-NEXT:    ret ptr [[ARRAYIDX2]]
+;
+  %idx = add i128 %i, 20
+  %idx.conv = trunc nuw i128 %idx to i64
+  %arrayidx = getelementptr i32, ptr %p, i64 %idx.conv
+  ret ptr %arrayidx
+}
+
+; Verify that we drop "nsw" from trunc.
+define ptr @pr154116_nsw(ptr %p, i128 %i) {
+; CHECK-LABEL: define ptr @pr154116_nsw(
+; CHECK-SAME: ptr [[P:%.*]], i128 [[I:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i128 [[I]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i32, ptr [[P]], i64 [[TMP1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr i8, ptr [[TMP2]], i64 4
+; CHECK-NEXT:    ret ptr [[ARRAYIDX2]]
+;
+  %idx = add i128 %i, 1
+  %idx.conv = trunc nsw i128 %idx to i64
+  %arrayidx = getelementptr i32, ptr %p, i64 %idx.conv
+  ret ptr %arrayidx
+}
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep-inbounds.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep-inbounds.ll
index 16e47f057babc..5cb3ee6f72e3b 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep-inbounds.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep-inbounds.ll
@@ -279,8 +279,8 @@ define void @addrspace3(ptr addrspace(3) %in.ptr, i64 %in.idx1) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX1_NNEG]] to i32
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i128, ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i64 1024
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i128, ptr addrspace(3) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDX11:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[TMP0]], i32 1024
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -296,8 +296,8 @@ define void @addrspace7(ptr addrspace(7) %in.ptr, i64 %in.idx1) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[IN_IDX1_NNEG:%.*]] = and i64 [[IN_IDX1]], 9223372036854775807
 ; CHECK-NEXT:    [[IDXPROM:%.*]] = trunc i64 [[IN_IDX1_NNEG]] to i32
-; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr i128, ptr addrspace(7) [[IN_PTR]], i32 [[IDXPROM]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(7) [[TMP0]], i64 1024
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i128, ptr addrspace(7) [[IN_PTR]], i32 [[IDXPROM]]
+; CHECK-NEXT:    [[IDX11:%.*]] = getelementptr inbounds i8, ptr addrspace(7) [[TMP0]], i32 1024
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll
index dd12c98af696d..7d8a43f59e367 100644
--- a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/reorder-gep.ll
@@ -74,13 +74,13 @@ define void @reorder_i8half(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1
 ; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
 ; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM2]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i64 256
+; CHECK-NEXT:    [[IDX13:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i32 256
 ; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM3]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP2]], i64 512
+; CHECK-NEXT:    [[IDX25:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP2]], i32 512
 ; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM4]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i64 768
+; CHECK-NEXT:    [[IDX37:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 768
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -169,13 +169,13 @@ define void @bad_index(ptr addrspace(3) %in.ptr, i64 %in.idx0, i64 %in.idx1) {
 ; CHECK-NEXT:    [[IDX0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM1]]
 ; CHECK-NEXT:    [[IDXPROM2:%.*]] = trunc i64 [[IN_IDX1]] to i32
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM2]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i64 1
+; CHECK-NEXT:    [[IDX13:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 1
 ; CHECK-NEXT:    [[IDXPROM3:%.*]] = trunc i64 [[IN_IDX1]] to i32
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM3]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP2]], i64 2
+; CHECK-NEXT:    [[IDX25:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP2]], i32 2
 ; CHECK-NEXT:    [[IDXPROM4:%.*]] = trunc i64 [[IN_IDX1]] to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr half, ptr addrspace(3) [[BASE]], i32 [[IDXPROM4]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i64 3
+; CHECK-NEXT:    [[IDX37:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP0]], i32 3
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/xor-decompose.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/xor-decompose.ll
new file mode 100644
index 0000000000000..056f33e5ee367
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/AMDGPU/xor-decompose.ll
@@ -0,0 +1,435 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; Test the xor with constant operand is decomposed in to gep.
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=separate-const-offset-from-gep \
+; RUN: -S < %s | FileCheck %s
+; Test the gvn pass eliminates the redundant xor instructions from decomposition.
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=separate-const-offset-from-gep,gvn \
+; RUN: -S < %s | FileCheck --check-prefix=GVN %s
+
+; Check that disjoint constants are properly extracted and folded into GEP
+; addressing modes and GVN to eliminate redundant computations
+define amdgpu_kernel void @test1(i1 %0, ptr addrspace(3) %1) {
+; CHECK-LABEL: define amdgpu_kernel void @test1(
+; CHECK-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP6]], i32 8192
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP9]], i32 16384
+; CHECK-NEXT:    [[TMP11:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP12]], i32 24576
+; CHECK-NEXT:    [[TMP14:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP4]], align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP13]], align 16
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <8 x half> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = fadd <8 x half> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = fadd <8 x half> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    store <8 x half> [[TMP20]], ptr addrspace(3) [[TMP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; GVN-LABEL: define amdgpu_kernel void @test1(
+; GVN-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; GVN-NEXT:  [[ENTRY:.*:]]
+; GVN-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; GVN-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; GVN-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; GVN-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 8192
+; GVN-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 16384
+; GVN-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 24576
+; GVN-NEXT:    [[TMP8:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP4]], align 16
+; GVN-NEXT:    [[TMP9:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP5]], align 16
+; GVN-NEXT:    [[TMP10:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP6]], align 16
+; GVN-NEXT:    [[TMP11:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP7]], align 16
+; GVN-NEXT:    [[TMP12:%.*]] = fadd <8 x half> [[TMP8]], [[TMP9]]
+; GVN-NEXT:    [[TMP13:%.*]] = fadd <8 x half> [[TMP10]], [[TMP11]]
+; GVN-NEXT:    [[TMP14:%.*]] = fadd <8 x half> [[TMP12]], [[TMP13]]
+; GVN-NEXT:    store <8 x half> [[TMP14]], ptr addrspace(3) [[TMP1]], align 16
+; GVN-NEXT:    ret void
+;
+entry:
+  %2 = select i1 %0, i32 0, i32 288
+  %3 = xor i32 %2, 32
+  %4 = xor i32 %2, 4128
+  %5 = xor i32 %2, 8224
+  %6 = xor i32 %2, 12320
+  %7 = getelementptr half, ptr addrspace(3) %1, i32 %3
+  %8 = getelementptr half, ptr addrspace(3) %1, i32 %4
+  %9 = getelementptr half, ptr addrspace(3) %1, i32 %5
+  %10 = getelementptr half, ptr addrspace(3) %1, i32 %6
+  %11 = load <8 x half>, ptr addrspace(3) %7, align 16
+  %12 = load <8 x half>, ptr addrspace(3) %8, align 16
+  %13 = load <8 x half>, ptr addrspace(3) %9, align 16
+  %14 = load <8 x half>, ptr addrspace(3) %10, align 16
+  %15 = fadd <8 x half> %11, %12
+  %16 = fadd <8 x half> %13, %14
+  %17 = fadd <8 x half> %15, %16
+  store <8 x half> %17, ptr addrspace(3) %1, align 16
+  ret void
+}
+
+; Check that disjoint constants are properly extracted and folded into GEP
+; addressing modes and GVN to eliminate redundant computations
+define amdgpu_kernel void @test2(i1 %0, ptr addrspace(3) %1) {
+; CHECK-LABEL: define amdgpu_kernel void @test2(
+; CHECK-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP5]], i32 24576
+; CHECK-NEXT:    [[TMP7:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP8]], i32 16384
+; CHECK-NEXT:    [[TMP10:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP11]], i32 8192
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP15:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP9]], align 16
+; CHECK-NEXT:    [[TMP16:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP12]], align 16
+; CHECK-NEXT:    [[TMP17:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP13]], align 16
+; CHECK-NEXT:    [[TMP18:%.*]] = fadd <8 x half> [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    [[TMP19:%.*]] = fadd <8 x half> [[TMP16]], [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = fadd <8 x half> [[TMP18]], [[TMP19]]
+; CHECK-NEXT:    store <8 x half> [[TMP20]], ptr addrspace(3) [[TMP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; GVN-LABEL: define amdgpu_kernel void @test2(
+; GVN-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; GVN-NEXT:  [[ENTRY:.*:]]
+; GVN-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; GVN-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; GVN-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; GVN-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 24576
+; GVN-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 16384
+; GVN-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 8192
+; GVN-NEXT:    [[TMP8:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP5]], align 16
+; GVN-NEXT:    [[TMP9:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP6]], align 16
+; GVN-NEXT:    [[TMP10:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP7]], align 16
+; GVN-NEXT:    [[TMP11:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP4]], align 16
+; GVN-NEXT:    [[TMP12:%.*]] = fadd <8 x half> [[TMP8]], [[TMP9]]
+; GVN-NEXT:    [[TMP13:%.*]] = fadd <8 x half> [[TMP10]], [[TMP11]]
+; GVN-NEXT:    [[TMP14:%.*]] = fadd <8 x half> [[TMP12]], [[TMP13]]
+; GVN-NEXT:    store <8 x half> [[TMP14]], ptr addrspace(3) [[TMP1]], align 16
+; GVN-NEXT:    ret void
+;
+entry:
+  %2 = select i1 %0, i32 0, i32 288
+  %3 = xor i32 %2, 12320
+  %4 = xor i32 %2, 8224
+  %5 = xor i32 %2, 4128
+  %6 = xor i32 %2, 32
+  %7 = getelementptr half, ptr addrspace(3) %1, i32 %3
+  %8 = getelementptr half, ptr addrspace(3) %1, i32 %4
+  %9 = getelementptr half, ptr addrspace(3) %1, i32 %5
+  %10 = getelementptr half, ptr addrspace(3) %1, i32 %6
+  %11 = load <8 x half>, ptr addrspace(3) %7, align 16
+  %12 = load <8 x half>, ptr addrspace(3) %8, align 16
+  %13 = load <8 x half>, ptr addrspace(3) %9, align 16
+  %14 = load <8 x half>, ptr addrspace(3) %10, align 16
+  %15 = fadd <8 x half> %11, %12
+  %16 = fadd <8 x half> %13, %14
+  %17 = fadd <8 x half> %15, %16
+  store <8 x half> %17, ptr addrspace(3) %1, align 16
+  ret void
+}
+
+; Verify that xor instructions with different non-disjoint constants are optimized
+define amdgpu_kernel void @test3(i1 %0, ptr addrspace(3) %1) {
+; CHECK-LABEL: define amdgpu_kernel void @test3(
+; CHECK-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP2]], 288
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP6]], i32 4096
+; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP9]], i32 8192
+; CHECK-NEXT:    [[TMP11:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP4]], align 16
+; CHECK-NEXT:    [[TMP12:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP13:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP10]], align 16
+; CHECK-NEXT:    [[TMP14:%.*]] = fadd <8 x half> [[TMP11]], [[TMP12]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fadd <8 x half> [[TMP13]], [[TMP14]]
+; CHECK-NEXT:    store <8 x half> [[TMP15]], ptr addrspace(3) [[TMP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; GVN-LABEL: define amdgpu_kernel void @test3(
+; GVN-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; GVN-NEXT:  [[ENTRY:.*:]]
+; GVN-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; GVN-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; GVN-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; GVN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP2]], 288
+; GVN-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP5]]
+; GVN-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP6]], i32 4096
+; GVN-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP4]], i32 8192
+; GVN-NEXT:    [[TMP9:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP4]], align 16
+; GVN-NEXT:    [[TMP10:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP7]], align 16
+; GVN-NEXT:    [[TMP11:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP8]], align 16
+; GVN-NEXT:    [[TMP12:%.*]] = fadd <8 x half> [[TMP9]], [[TMP10]]
+; GVN-NEXT:    [[TMP13:%.*]] = fadd <8 x half> [[TMP11]], [[TMP12]]
+; GVN-NEXT:    store <8 x half> [[TMP13]], ptr addrspace(3) [[TMP1]], align 16
+; GVN-NEXT:    ret void
+;
+entry:
+  %2 = select i1 %0, i32 0, i32 288
+  %3 = xor i32 %2, 32
+  %4 = xor i32 %2, 2336
+  %5 = xor i32 %2, 4128
+  %6 = getelementptr half, ptr addrspace(3) %1, i32 %3
+  %7 = getelementptr half, ptr addrspace(3) %1, i32 %4
+  %8 = getelementptr half, ptr addrspace(3) %1, i32 %5
+  %9 = load <8 x half>, ptr addrspace(3) %6, align 16
+  %10 = load <8 x half>, ptr addrspace(3) %7, align 16
+  %11 = load <8 x half>, ptr addrspace(3) %8, align 16
+  %12 = fadd <8 x half> %9, %10
+  %13 = fadd <8 x half> %11, %12
+  store <8 x half> %13, ptr addrspace(3) %1, align 16
+  ret void
+}
+
+; Verify that no optimization occurs when disjoint constants are absent
+define amdgpu_kernel void @test4(i1 %0, ptr addrspace(3) %1) {
+; CHECK-LABEL: define amdgpu_kernel void @test4(
+; CHECK-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP2]], 288
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP5]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd <8 x half> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    store <8 x half> [[TMP9]], ptr addrspace(3) [[TMP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; GVN-LABEL: define amdgpu_kernel void @test4(
+; GVN-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; GVN-NEXT:  [[ENTRY:.*:]]
+; GVN-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; GVN-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; GVN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP2]], 288
+; GVN-NEXT:    [[TMP5:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; GVN-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP4]]
+; GVN-NEXT:    [[TMP7:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP5]], align 16
+; GVN-NEXT:    [[TMP8:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP6]], align 16
+; GVN-NEXT:    [[TMP9:%.*]] = fadd <8 x half> [[TMP7]], [[TMP8]]
+; GVN-NEXT:    store <8 x half> [[TMP9]], ptr addrspace(3) [[TMP1]], align 16
+; GVN-NEXT:    ret void
+;
+entry:
+  %2 = select i1 %0, i32 0, i32 288
+  %3 = xor i32 %2, 32
+  %4 = xor i32 %2, 288
+  %5 = getelementptr half, ptr addrspace(3) %1, i32 %3
+  %6 = getelementptr half, ptr addrspace(3) %1, i32 %4
+  %7 = load <8 x half>, ptr addrspace(3) %5, align 16
+  %8 = load <8 x half>, ptr addrspace(3) %6, align 16
+  %9 = fadd <8 x half> %7, %8
+  store <8 x half> %9, ptr addrspace(3) %1, align 16
+  ret void
+}
+
+
+; Verify that XOR-BinOp-GEP usage chains are properly optimized
+define amdgpu_kernel void @test5(i1 %0, ptr addrspace(3) %1) {
+; CHECK-LABEL: define amdgpu_kernel void @test5(
+; CHECK-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 256
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP7]], i32 8192
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP4]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP8]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <8 x half> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    store <8 x half> [[TMP11]], ptr addrspace(3) [[TMP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; GVN-LABEL: define amdgpu_kernel void @test5(
+; GVN-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; GVN-NEXT:  [[ENTRY:.*:]]
+; GVN-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; GVN-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; GVN-NEXT:    [[TMP4:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; GVN-NEXT:    [[TMP5:%.*]] = add i32 [[TMP3]], 256
+; GVN-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP5]]
+; GVN-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP6]], i32 8192
+; GVN-NEXT:    [[TMP8:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP4]], align 16
+; GVN-NEXT:    [[TMP9:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP7]], align 16
+; GVN-NEXT:    [[TMP10:%.*]] = fadd <8 x half> [[TMP8]], [[TMP9]]
+; GVN-NEXT:    store <8 x half> [[TMP10]], ptr addrspace(3) [[TMP1]], align 16
+; GVN-NEXT:    ret void
+;
+entry:
+  %2 = select i1 %0, i32 0, i32 288
+  %3 = xor i32 %2, 32
+  %4 = xor i32 %2, 4128
+  %5 = add i32 %4, 256
+  %6 = getelementptr half, ptr addrspace(3) %1, i32 %3
+  %7 = getelementptr half, ptr addrspace(3) %1, i32 %5
+  %8 = load <8 x half>, ptr addrspace(3) %6, align 16
+  %9 = load <8 x half>, ptr addrspace(3) %7, align 16
+  %10 = fadd <8 x half> %8, %9
+  store <8 x half> %10, ptr addrspace(3) %1, align 16
+  ret void
+}
+
+; Verify that BinOp-XOR-GEP usage chains are properly optimized.
+; In the below test, make sure we stop processing the chain at xor
+; and not fold the constant from add instruction in to gep. The
+; constant from add can be folded and the future work will cover
+; these cases.
+define amdgpu_kernel void @test6(i1 %0, ptr addrspace(3) %1) {
+; CHECK-LABEL: define amdgpu_kernel void @test6(
+; CHECK-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[TMP2]], 256
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], 32
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP7]], i32 8192
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP5]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP8]], align 16
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <8 x half> [[TMP9]], [[TMP10]]
+; CHECK-NEXT:    store <8 x half> [[TMP11]], ptr addrspace(3) [[TMP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; GVN-LABEL: define amdgpu_kernel void @test6(
+; GVN-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; GVN-NEXT:  [[ENTRY:.*:]]
+; GVN-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; GVN-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; GVN-NEXT:    [[TMP4:%.*]] = add i32 [[TMP2]], 256
+; GVN-NEXT:    [[TMP5:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; GVN-NEXT:    [[TMP6:%.*]] = xor i32 [[TMP4]], 32
+; GVN-NEXT:    [[TMP7:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP6]]
+; GVN-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr addrspace(3) [[TMP7]], i32 8192
+; GVN-NEXT:    [[TMP9:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP5]], align 16
+; GVN-NEXT:    [[TMP10:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP8]], align 16
+; GVN-NEXT:    [[TMP11:%.*]] = fadd <8 x half> [[TMP9]], [[TMP10]]
+; GVN-NEXT:    store <8 x half> [[TMP11]], ptr addrspace(3) [[TMP1]], align 16
+; GVN-NEXT:    ret void
+;
+entry:
+  %2 = select i1 %0, i32 0, i32 288
+  %3 = xor i32 %2, 32
+  %4 = add i32 %2, 256
+  %5 = xor i32 %4, 4128
+  %6 = getelementptr half, ptr addrspace(3) %1, i32 %3
+  %7 = getelementptr half, ptr addrspace(3) %1, i32 %5
+  %8 = load <8 x half>, ptr addrspace(3) %6, align 16
+  %9 = load <8 x half>, ptr addrspace(3) %7, align 16
+  %10 = fadd <8 x half> %8, %9
+  store <8 x half> %10, ptr addrspace(3) %1, align 16
+  ret void
+}
+
+; Verify that BinOp-XOR-GEP usage chains with non disjoint xor works as
+; intended.
+define amdgpu_kernel void @test6a(i1 %0, ptr addrspace(3) %1) {
+; CHECK-LABEL: define amdgpu_kernel void @test6a(
+; CHECK-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = add i32 [[TMP2]], 256
+; CHECK-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP4]], 288
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP7]], align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd <8 x half> [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    store <8 x half> [[TMP10]], ptr addrspace(3) [[TMP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; GVN-LABEL: define amdgpu_kernel void @test6a(
+; GVN-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; GVN-NEXT:  [[ENTRY:.*:]]
+; GVN-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; GVN-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; GVN-NEXT:    [[TMP4:%.*]] = add i32 [[TMP2]], 256
+; GVN-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP4]], 288
+; GVN-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; GVN-NEXT:    [[TMP7:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP5]]
+; GVN-NEXT:    [[TMP8:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP6]], align 16
+; GVN-NEXT:    [[TMP9:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP7]], align 16
+; GVN-NEXT:    [[TMP10:%.*]] = fadd <8 x half> [[TMP8]], [[TMP9]]
+; GVN-NEXT:    store <8 x half> [[TMP10]], ptr addrspace(3) [[TMP1]], align 16
+; GVN-NEXT:    ret void
+;
+entry:
+  %2 = select i1 %0, i32 0, i32 288
+  %3 = xor i32 %2, 32
+  %4 = add i32 %2, 256
+  %5 = xor i32 %4, 288
+  %6 = getelementptr half, ptr addrspace(3) %1, i32 %3
+  %7 = getelementptr half, ptr addrspace(3) %1, i32 %5
+  %8 = load <8 x half>, ptr addrspace(3) %6, align 16
+  %9 = load <8 x half>, ptr addrspace(3) %7, align 16
+  %10 = fadd <8 x half> %8, %9
+  store <8 x half> %10, ptr addrspace(3) %1, align 16
+  ret void
+}
+
+; Ensure disjoint constants exceeding addressing mode limits (e.g., 32768) are
+; not extracted
+define amdgpu_kernel void @test7(i1 %0, ptr addrspace(3) %1) {
+; CHECK-LABEL: define amdgpu_kernel void @test7(
+; CHECK-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; CHECK-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP2]], 32800
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP5]], align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP6]], align 16
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd <8 x half> [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    store <8 x half> [[TMP9]], ptr addrspace(3) [[TMP1]], align 16
+; CHECK-NEXT:    ret void
+;
+; GVN-LABEL: define amdgpu_kernel void @test7(
+; GVN-SAME: i1 [[TMP0:%.*]], ptr addrspace(3) [[TMP1:%.*]]) {
+; GVN-NEXT:  [[ENTRY:.*:]]
+; GVN-NEXT:    [[TMP2:%.*]] = select i1 [[TMP0]], i32 0, i32 288
+; GVN-NEXT:    [[TMP3:%.*]] = xor i32 [[TMP2]], 32
+; GVN-NEXT:    [[TMP4:%.*]] = xor i32 [[TMP2]], 32800
+; GVN-NEXT:    [[TMP5:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP3]]
+; GVN-NEXT:    [[TMP6:%.*]] = getelementptr half, ptr addrspace(3) [[TMP1]], i32 [[TMP4]]
+; GVN-NEXT:    [[TMP7:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP5]], align 16
+; GVN-NEXT:    [[TMP8:%.*]] = load <8 x half>, ptr addrspace(3) [[TMP6]], align 16
+; GVN-NEXT:    [[TMP9:%.*]] = fadd <8 x half> [[TMP7]], [[TMP8]]
+; GVN-NEXT:    store <8 x half> [[TMP9]], ptr addrspace(3) [[TMP1]], align 16
+; GVN-NEXT:    ret void
+;
+entry:
+  %2 = select i1 %0, i32 0, i32 288
+  %3 = xor i32 %2, 32
+  %4 = xor i32 %2, 32800
+  %5 = getelementptr half, ptr addrspace(3) %1, i32 %3
+  %6 = getelementptr half, ptr addrspace(3) %1, i32 %4
+  %7 = load <8 x half>, ptr addrspace(3) %5, align 16
+  %8 = load <8 x half>, ptr addrspace(3) %6, align 16
+  %9 = fadd <8 x half> %7, %8
+  store <8 x half> %9, ptr addrspace(3) %1, align 16
+  ret void
+}
+
diff --git a/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/gep-chain.ll b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/gep-chain.ll
new file mode 100644
index 0000000000000..e4b9dac72e96c
--- /dev/null
+++ b/llvm/test/Transforms/SeparateConstOffsetFromGEP/NVPTX/gep-chain.ll
@@ -0,0 +1,413 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=nvptx64-nvidia-cuda -S -passes=separate-const-offset-from-gep  < %s | FileCheck %s
+
+%struct.uchar4 = type { i8, i8, i8, i8 }
+
+define ptr @basic(ptr %ptr, i64 %offset1, i64 %offset2) {
+; CHECK-LABEL: define ptr @basic(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[OFFSET1:%.*]], i64 [[OFFSET2:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_UCHAR4:%.*]], ptr [[PTR]], i64 [[OFFSET1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr [[STRUCT_UCHAR4]], ptr [[TMP1]], i64 [[OFFSET2]]
+; CHECK-NEXT:    [[GEP24:%.*]] = getelementptr i8, ptr [[TMP2]], i64 72
+; CHECK-NEXT:    ret ptr [[GEP24]]
+;
+  %offset3 = add i64 %offset1, 8
+  %gep1 = getelementptr %struct.uchar4, ptr %ptr, i64 %offset3
+  %offset4 = add i64 %offset2, 10
+  %gep2 = getelementptr %struct.uchar4, ptr %gep1, i64 %offset4
+  ret ptr %gep2
+}
+
+define i32 @more_interesting(ptr %ptr, i32 %offset1, i32 %offset2) {
+; CHECK-LABEL: define i32 @more_interesting(
+; CHECK-SAME: ptr [[PTR:%.*]], i32 [[OFFSET1:%.*]], i32 [[OFFSET2:%.*]]) {
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[OFFSET1]] to i64
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [[STRUCT_UCHAR4:%.*]], ptr [[PTR]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[IDXPROM1:%.*]] = sext i32 [[OFFSET2]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_UCHAR4]], ptr [[GEP1]], i64 [[IDXPROM1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 8
+; CHECK-NEXT:    [[V1:%.*]] = load i32, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[OFFSET2]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_UCHAR4]], ptr [[TMP1]], i64 [[IDXPROM2]]
+; CHECK-NEXT:    [[V2:%.*]] = load i32, ptr [[TMP4]], align 4
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[V1]], [[V2]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %gep1 = getelementptr inbounds %struct.uchar4, ptr %ptr, i32 %offset1
+  %gep2 = getelementptr inbounds nuw i8, ptr %gep1, i32 8
+  %gep3 = getelementptr inbounds %struct.uchar4, ptr %gep2, i32 %offset2
+  %v1 = load i32, ptr %gep3, align 4
+  %gep4 = getelementptr inbounds i8, ptr %gep3, i32 -8
+  %gep5 = getelementptr inbounds %struct.uchar4, ptr %gep4, i32 %offset2
+  %v2 = load i32, ptr %gep5, align 4
+  %r = add i32 %v1, %v2
+  ret i32 %r
+}
+
+;; Check nuw/nusw/inbounds flag propagation
+
+; GEPs with nusw flag. All indices and offsets are non-negative.
+define ptr @test_0(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_0(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw inbounds i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add i64 %i.prom, 1
+  %arrayidx2 = getelementptr nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with inbounds flag. All indices and offsets are non-negative.
+define ptr @test_1(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_1(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr inbounds i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add i64 %i.prom, 1
+  %arrayidx2 = getelementptr nuw inbounds i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nusw flag. All indices and offsets are non-negative.
+define ptr @test_2(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_2(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nusw i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, 1
+  %arrayidx2 = getelementptr nuw inbounds i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with inbounds flag. All indices and offsets are non-negative.
+define ptr @test_3(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_3(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw inbounds i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, 1
+  %arrayidx2 = getelementptr inbounds i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nusw flag. All indices and offsets are non-negative.
+define ptr @test_4(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_4(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw inbounds i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add i64 %i.prom, 1
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with inbounds flag. All indices and offsets are non-negative.
+define ptr @test_5(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_5(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw inbounds i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add i64 %i.prom, 1
+  %arrayidx2 = getelementptr nuw inbounds i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw flag. Negative offsets.
+define ptr @test_6(ptr %p, i64 %i) {
+; CHECK-LABEL: define ptr @test_6(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[I:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 -11
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw nusw i8, ptr %p, i32 -1
+  %idx2 = add nuw i64 %i, -10
+  %arrayidx2 = getelementptr nuw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw and nusw flags. Negative offsets.
+define ptr @test_7(ptr %p, i64 %i) {
+; CHECK-LABEL: define ptr @test_7(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[I:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw nuw i8, ptr [[P]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw nuw i8, ptr [[TMP1]], i64 -11
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw inbounds i8, ptr %p, i32 -1
+  %idx2 = add nuw i64 %i, -10
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw flag. Mixed positive/negative offsets.
+define ptr @test_8(ptr %p, i64 %i) {
+; CHECK-LABEL: define ptr @test_8(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[I:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 -9
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw i8, ptr %p, i32 1
+  %idx2 = add nuw i64 %i, -10
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw and nusw flags. Mixed positive/negative offsets.
+define ptr @test_9(ptr %p, i64 %i) {
+; CHECK-LABEL: define ptr @test_9(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[I:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw nuw i8, ptr [[P]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw nuw i8, ptr [[TMP1]], i64 -9
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw inbounds i8, ptr %p, i32 1
+  %idx2 = add nuw i64 %i, -10
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw flag. Mixed negative/positive offsets.
+define ptr @test_10(ptr %p, i64 %i) {
+; CHECK-LABEL: define ptr @test_10(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[I:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 9
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw i8, ptr %p, i32 -1
+  %idx2 = add nuw i64 %i, 10
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw and nusw flags. Mixed negative/positive offsets.
+define ptr @test_11(ptr %p, i64 %i) {
+; CHECK-LABEL: define ptr @test_11(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[I:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw nuw i8, ptr [[P]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw nuw i8, ptr [[TMP1]], i64 9
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw inbounds i8, ptr %p, i32 -1
+  %idx2 = add nuw i64 %i, 10
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw flag. All positive offsets.
+define ptr @test_12(ptr %p, i64 %i) {
+; CHECK-LABEL: define ptr @test_12(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[I:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw nusw i8, ptr %p, i32 1
+  %idx2 = add nuw i64 %i, 1
+  %arrayidx2 = getelementptr nuw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw and nusw flags. All positive offsets.
+define ptr @test_13(ptr %p, i64 %i) {
+; CHECK-LABEL: define ptr @test_13(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[I:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw nuw i8, ptr [[P]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw nuw i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw inbounds i8, ptr %p, i32 1
+  %idx2 = add nuw i64 %i, 1
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw flag. Zext index with negative offsets.
+define ptr @test_14(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_14(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 -11
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw nusw i8, ptr %p, i32 -1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, -10
+  %arrayidx2 = getelementptr nuw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw and nusw flags. Zext index with negative offsets.
+define ptr @test_15(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_15(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw nuw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw nuw i8, ptr [[TMP1]], i64 -11
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw nusw i8, ptr %p, i32 -1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, -10
+  %arrayidx2 = getelementptr nuw inbounds i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw flag. Zext index with mixed positive/negative offsets.
+define ptr @test_16(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_16(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 -9
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, -10
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw and nusw flags. Zext index with mixed positive/negative offsets.
+define ptr @test_17(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_17(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw nuw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw nuw i8, ptr [[TMP1]], i64 -9
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw nusw i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, -10
+  %arrayidx2 = getelementptr nuw inbounds i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw flag. Zext index with mixed negative/positive offsets.
+define ptr @test_18(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_18(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 9
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw nusw i8, ptr %p, i32 -1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, 10
+  %arrayidx2 = getelementptr nuw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw and nusw flags. Zext index with mixed negative/positive offsets.
+define ptr @test_19(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_19(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw nuw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw nuw i8, ptr [[TMP1]], i64 9
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw nusw i8, ptr %p, i32 -1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, 10
+  %arrayidx2 = getelementptr nuw inbounds i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw flag. Zext index with all positive offsets.
+define ptr @test_20(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_20(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nuw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nuw i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, 1
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+; GEPs with nuw and nusw flags. Zext index with all positive offsets.
+define ptr @test_21(ptr %p, i32 %i) {
+; CHECK-LABEL: define ptr @test_21(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[I_PROM:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr nusw nuw i8, ptr [[P]], i64 [[I_PROM]]
+; CHECK-NEXT:    [[ARRAYIDX22:%.*]] = getelementptr nusw nuw i8, ptr [[TMP1]], i64 2
+; CHECK-NEXT:    ret ptr [[ARRAYIDX22]]
+;
+  %ptradd = getelementptr nuw inbounds i8, ptr %p, i32 1
+  %i.prom = zext i32 %i to i64
+  %idx2 = add nuw i64 %i.prom, 1
+  %arrayidx2 = getelementptr nuw nusw i8, ptr %ptradd, i64 %idx2
+  ret ptr %arrayidx2
+}
+
+define ptr @test_rpot(ptr %ptr, i64 %offset1, i64 %offset2) {
+; CHECK-LABEL: define ptr @test_rpot(
+; CHECK-SAME: ptr [[PTR:%.*]], i64 [[OFFSET1:%.*]], i64 [[OFFSET2:%.*]]) {
+; CHECK-NEXT:  [[A:.*:]]
+; CHECK-NEXT:    br label %[[B:.*]]
+; CHECK:       [[C:.*]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr [[STRUCT_UCHAR4:%.*]], ptr [[TMP0:%.*]], i64 [[OFFSET2]]
+; CHECK-NEXT:    [[GEP24:%.*]] = getelementptr i8, ptr [[TMP1]], i64 72
+; CHECK-NEXT:    br label %[[D:.*]]
+; CHECK:       [[B]]:
+; CHECK-NEXT:    [[TMP0]] = getelementptr [[STRUCT_UCHAR4]], ptr [[PTR]], i64 [[OFFSET1]]
+; CHECK-NEXT:    br label %[[C]]
+; CHECK:       [[D]]:
+; CHECK-NEXT:    ret ptr [[GEP24]]
+;
+A:
+  br label %B
+C:
+  %offset4 = add i64 %offset2, 10
+  %gep2 = getelementptr %struct.uchar4, ptr %gep1, i64 %offset4
+  br label %D
+B:
+  %offset3 = add i64 %offset1, 8
+  %gep1 = getelementptr %struct.uchar4, ptr %ptr, i64 %offset3
+  br label %C
+D:
+  ret ptr %gep2
+}
+
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll
index 6f2833b4f4e76..f82d7309f6d07 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested.ll
@@ -45,7 +45,7 @@
 ;
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
 ; RUN: -passes='loop-mssa(licm,simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | \
-; RUN:	   sort -b -k 1 | FileCheck %s --check-prefixes=LOOP6
+; RUN:	   sort -b -k 1 | FileCheck %s --check-prefixes=LOOP32
 ;
 ; Single loop nest, not unswitched
 ; LOOP1:     Loop at depth 1 containing:
@@ -55,23 +55,23 @@
 ;
 ; Half unswitched loop nests, with unscaled4 and div1 it gets less depth1 loops unswitched
 ; since they have more cost.
-; LOOP-UNSCALE4-DIV1-COUNT-4: Loop at depth 1 containing:
-; LOOP-UNSCALE4-DIV1-COUNT-4: Loop at depth 2 containing:
-; LOOP-UNSCALE4-DIV1-COUNT-4: Loop at depth 3 containing:
+; LOOP-UNSCALE4-DIV1-COUNT-6: Loop at depth 1 containing:
+; LOOP-UNSCALE4-DIV1-COUNT-19: Loop at depth 2 containing:
+; LOOP-UNSCALE4-DIV1-COUNT-29: Loop at depth 3 containing:
 ; LOOP-UNSCALE4-DIV1-NOT:      Loop at depth {{[0-9]+}} containing:
 ;
 ; Half unswitched loop nests, with unscaled4 and div2 it gets more depth1 loops unswitched
 ; as div2 kicks in.
-; LOOP-UNSCALE4-DIV2-COUNT-4: Loop at depth 1 containing:
-; LOOP-UNSCALE4-DIV2-COUNT-4: Loop at depth 2 containing:
-; LOOP-UNSCALE4-DIV2-COUNT-4: Loop at depth 3 containing:
+; LOOP-UNSCALE4-DIV2-COUNT-11: Loop at depth 1 containing:
+; LOOP-UNSCALE4-DIV2-COUNT-22: Loop at depth 2 containing:
+; LOOP-UNSCALE4-DIV2-COUNT-29: Loop at depth 3 containing:
 ; LOOP-UNSCALE4-DIV2-NOT:      Loop at depth {{[0-9]+}} containing:
 ;
-; 6 loop nests, fully unswitched
-; LOOP6-COUNT-6: Loop at depth 1 containing:
-; LOOP6-COUNT-6: Loop at depth 2 containing:
-; LOOP6-COUNT-6: Loop at depth 3 containing:
-; LOOP6-NOT:      Loop at depth {{[0-9]+}} containing:
+; 32 loop nests, fully unswitched
+; LOOP32-COUNT-32: Loop at depth 1 containing:
+; LOOP32-COUNT-32: Loop at depth 2 containing:
+; LOOP32-COUNT-32: Loop at depth 3 containing:
+; LOOP32-NOT:      Loop at depth {{[0-9]+}} containing:
 
 declare void @bar()
 
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll
index ab3b3d26d9975..63d2789da5a82 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch-nested2.ll
@@ -60,7 +60,7 @@
 ;
 ; Half unswitched loop nests, with unscaled3 and div1 it gets less depth1 loops unswitched
 ; since they have more cost.
-; LOOP-UNSCALE3-DIV1-COUNT-2: Loop at depth 1 containing:
+; LOOP-UNSCALE3-DIV1-COUNT-4: Loop at depth 1 containing:
 ; LOOP-UNSCALE3-DIV1-NOT:      Loop at depth 1 containing:
 ; LOOP-UNSCALE3-DIV1-COUNT-1: Loop at depth 2 containing:
 ; LOOP-UNSCALE3-DIV1-NOT:      Loop at depth 2 containing:
@@ -69,7 +69,7 @@
 ;
 ; Half unswitched loop nests, with unscaled3 and div2 it gets more depth1 loops unswitched
 ; as div2 kicks in.
-; LOOP-UNSCALE3-DIV2-COUNT-2: Loop at depth 1 containing:
+; LOOP-UNSCALE3-DIV2-COUNT-6: Loop at depth 1 containing:
 ; LOOP-UNSCALE3-DIV2-NOT:      Loop at depth 1 containing:
 ; LOOP-UNSCALE3-DIV2-COUNT-1: Loop at depth 2 containing:
 ; LOOP-UNSCALE3-DIV2-NOT:      Loop at depth 2 containing:
@@ -77,7 +77,7 @@
 ; LOOP-UNSCALE3-DIV2-NOT:      Loop at depth 3 containing:
 ;
 ; Maximally unswitched (copy of the outer loop per each condition)
-; LOOP-MAX-COUNT-2: Loop at depth 1 containing:
+; LOOP-MAX-COUNT-6: Loop at depth 1 containing:
 ; LOOP-MAX-NOT:      Loop at depth 1 containing:
 ; LOOP-MAX-COUNT-1: Loop at depth 2 containing:
 ; LOOP-MAX-NOT:      Loop at depth 2 containing:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll
index 7515cbbcbf1df..a2a745f46bca7 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-nontrivial-unswitch.ll
@@ -25,37 +25,46 @@
 ;
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
 ; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=1 \
-; RUN: -passes='loop(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP4
+; RUN: -passes='loop(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP5
 ;
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
 ; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=1 \
-; RUN: -passes='loop-mssa(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP4
+; RUN: -passes='loop-mssa(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP5
+;
+; With relaxed candidates multiplier (unscaled candidates == 8) and with relaxed
+; siblings multiplier for top-level loops (toplevel-div == 8) we should get
+;    2^(num conds) == 2^5 == 32
+; copies of the loop:
 ;
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
 ; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=8 \
-; RUN: -passes='loop(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP6
+; RUN: -passes='loop(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
 ;
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=true \
 ; RUN:     -unswitch-num-initial-unscaled-candidates=8 -unswitch-siblings-toplevel-div=8 \
-; RUN: -passes='loop-mssa(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP6
+; RUN: -passes='loop-mssa(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
+;
+; Similarly get
+;    2^(num conds) == 2^5 == 32
+; copies of the loop when cost multiplier is disabled:
 ;
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
-; RUN: -passes='loop(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP6
+; RUN: -passes='loop(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
 ;
 ; RUN: opt < %s -enable-unswitch-cost-multiplier=false \
-; RUN: -passes='loop-mssa(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP6
+; RUN: -passes='loop-mssa(simple-loop-unswitch<nontrivial>),print<loops>' -disable-output 2>&1 | FileCheck %s --check-prefixes=LOOP32
 ;
 ; Single loop, not unswitched
 ; LOOP1:     Loop at depth 1 containing:
 ; LOOP1-NOT: Loop at depth 1 containing:
 
-; 4 loops, unswitched 4 times
-; LOOP4-COUNT-4: Loop at depth 1 containing:
-; LOOP4-NOT:     Loop at depth 1 containing:
+; 5 loops, unswitched 4 times
+; LOOP5-COUNT-5: Loop at depth 1 containing:
+; LOOP5-NOT:     Loop at depth 1 containing:
 
-; 6 loops, fully unswitched
-; LOOP6-COUNT-6: Loop at depth 1 containing:
-; LOOP6-NOT:     Loop at depth 1 containing:
+; 32 loops, fully unswitched
+; LOOP32-COUNT-32: Loop at depth 1 containing:
+; LOOP32-NOT:     Loop at depth 1 containing:
 
 define void @loop_simple5(ptr %addr, i1 %c1, i1 %c2, i1 %c3, i1 %c4, i1 %c5) {
 entry:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll
index 846a7793b6c37..96fe899d69c3b 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/exponential-switch-unswitch.ll
@@ -61,19 +61,19 @@
 ; Somewhat relaxed restrictions on candidates:
 ; LOOP-RELAX-COUNT-5:     Loop at depth 1 containing:
 ; LOOP-RELAX-NOT: Loop at depth 1 containing:
-; LOOP-RELAX-COUNT-5:     Loop at depth 2 containing:
+; LOOP-RELAX-COUNT-32:     Loop at depth 2 containing:
 ; LOOP-RELAX-NOT: Loop at depth 2 containing:
 ;
 ; Even more relaxed restrictions on candidates and siblings.
-; LOOP-RELAX2-COUNT-5:     Loop at depth 1 containing:
+; LOOP-RELAX2-COUNT-11:     Loop at depth 1 containing:
 ; LOOP-RELAX2-NOT: Loop at depth 1 containing:
-; LOOP-RELAX2-COUNT-5:     Loop at depth 2 containing:
+; LOOP-RELAX2-COUNT-40:     Loop at depth 2 containing:
 ; LOOP-RELAX-NOT: Loop at depth 2 containing:
 ;
 ; Unswitched as much as it could (with multiplier disabled).
-; LOOP-MAX-COUNT-6:     Loop at depth 1 containing:
+; LOOP-MAX-COUNT-56:     Loop at depth 1 containing:
 ; LOOP-MAX-NOT: Loop at depth 1 containing:
-; LOOP-MAX-COUNT-11:     Loop at depth 2 containing:
+; LOOP-MAX-COUNT-111:     Loop at depth 2 containing:
 ; LOOP-MAX-NOT: Loop at depth 2 containing:
 
 define i32 @loop_switch(ptr %addr, i32 %c1, i32 %c2) {
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
index c77e7cce77a9c..533b1f691f5ad 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
@@ -38,25 +38,25 @@ exit:
 }
 
 define void @test_two_guards(i1 %cond1, i1 %cond2, i32 %N) {
-; CHECK-LABEL: define void @test_two_guards(i1 %cond1, i1 %cond2, i32 %N) {
+; CHECK-LABEL: @test_two_guards(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 %cond1, label %entry.split.us, label %entry.split
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
-; CHECK-NEXT:    br label %loop.us
-; CHECK:       loop.us:
-; CHECK-NEXT:    %iv.us = phi i32 [ 0, %entry.split.us ], [ %iv.next.us, %guarded.us ]
-; CHECK-NEXT:    br label %guarded.us
-; CHECK:       guarded.us:
-; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 %cond2) [ "deopt"() ]
-; CHECK-NEXT:    %iv.next.us = add i32 %iv.us, 1
-; CHECK-NEXT:    %loop.cond.us = icmp slt i32 %iv.next.us, %N
-; CHECK-NEXT:    br i1 %loop.cond.us, label %loop.us, label %exit.split.us, !llvm.loop !2
-; CHECK:       exit.split.us:
-; CHECK-NEXT:    br label %exit
-; CHECK:       entry.split:
-; CHECK-NEXT:    br label %loop
-; CHECK:       loop:
-; CHECK-NEXT:    br label %deopt
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]]
+; CHECK:       entry.split.us.split.us:
+; CHECK-NEXT:    br label [[LOOP_US_US:%.*]]
+; CHECK:       loop.us.us:
+; CHECK-NEXT:    [[IV_US_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US_SPLIT_US]] ], [ [[IV_NEXT_US_US:%.*]], [[GUARDED_US2:%.*]] ]
+; CHECK-NEXT:    br label [[GUARDED_US_US:%.*]]
+; CHECK:       guarded.us.us:
+; CHECK-NEXT:    br label [[GUARDED_US2]]
+; CHECK:       guarded.us2:
+; CHECK-NEXT:    [[IV_NEXT_US_US]] = add i32 [[IV_US_US]], 1
+; CHECK-NEXT:    [[LOOP_COND_US_US:%.*]] = icmp slt i32 [[IV_NEXT_US_US]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US_US]], label [[LOOP_US_US]], label [[EXIT_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       deopt1:
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
+; CHECK-NEXT:    unreachable
 ; CHECK:       deopt:
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 false) [ "deopt"() ]
 ; CHECK-NEXT:    unreachable
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/inject-invariant-conditions.ll b/llvm/test/Transforms/SimpleLoopUnswitch/inject-invariant-conditions.ll
index 3dc83203f1490..536e0c6a0e74a 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/inject-invariant-conditions.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/inject-invariant-conditions.ll
@@ -5,7 +5,7 @@
 define i32 @test_01(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_01(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0:![0-9]+]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    [[INJECTED_COND:%.*]] = icmp ule i32 [[LIMIT:%.*]], [[X]]
 ; CHECK-NEXT:    br i1 [[INJECTED_COND]], label [[LOOP_US:%.*]], label [[LOOP:%.*]]
 ; CHECK:       loop.us:
@@ -20,7 +20,7 @@ define i32 @test_01(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noun
 ; CHECK-NEXT:    store i32 [[IV_US]], ptr [[ARR_PTR_US]], align 4
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
 ; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -35,7 +35,7 @@ define i32 @test_01(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noun
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[ARR_PTR]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[BACKEDGE]] ], [ 0, [[GUARDED_US]] ], [ -1, [[LOOP]] ], [ -1, [[LOOP_US]] ], [ -2, [[GUARDED]] ]
 ; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
@@ -76,7 +76,7 @@ range_check_failed:                               ; preds = %guarded
 define i32 @test_01_neg_void_profile(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_01_neg_void_profile(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
@@ -133,7 +133,7 @@ range_check_failed:                               ; preds = %guarded
 define i32 @test_01_constants(ptr noundef %p, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_01_constants(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    [[INJECTED_COND:%.*]] = icmp ule i32 200, 300
 ; CHECK-NEXT:    br i1 [[INJECTED_COND]], label [[LOOP_US:%.*]], label [[LOOP:%.*]]
 ; CHECK:       loop.us:
@@ -148,7 +148,7 @@ define i32 @test_01_constants(ptr noundef %p, ptr noundef %arr, ptr noundef %x_p
 ; CHECK-NEXT:    store i32 [[IV_US]], ptr [[ARR_PTR_US]], align 4
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
 ; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], 1000
-; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -160,7 +160,7 @@ define i32 @test_01_constants(ptr noundef %p, ptr noundef %arr, ptr noundef %x_p
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[ARR_PTR]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[BACKEDGE]] ], [ 0, [[GUARDED_US]] ], [ -1, [[LOOP]] ], [ -1, [[LOOP_US]] ]
 ; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
@@ -200,7 +200,7 @@ range_check_failed:                               ; preds = %guarded
 define i32 @test_01_neg_degenerate_profile(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_01_neg_degenerate_profile(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
@@ -210,7 +210,7 @@ define i32 @test_01_neg_degenerate_profile(ptr noundef %p, i32 noundef %n, i32 n
 ; CHECK-NEXT:    br i1 [[BOUND_CHECK]], label [[GUARDED:%.*]], label [[COMMON_RET:%.*]], !prof [[PROF1]]
 ; CHECK:       guarded:
 ; CHECK-NEXT:    [[RANGE_CHECK:%.*]] = icmp ult i32 [[EL]], [[X]]
-; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[COMMON_RET]], !prof [[PROF8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[COMMON_RET]], !prof [[PROF5:![0-9]+]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[ARR_PTR:%.*]] = getelementptr i32, ptr [[ARR:%.*]], i32 [[EL]]
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[ARR_PTR]], align 4
@@ -257,7 +257,7 @@ range_check_failed:                               ; preds = %guarded
 define i32 @test_01_neg_cold(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_01_neg_cold(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
@@ -267,7 +267,7 @@ define i32 @test_01_neg_cold(ptr noundef %p, i32 noundef %n, i32 noundef %limit,
 ; CHECK-NEXT:    br i1 [[BOUND_CHECK]], label [[GUARDED:%.*]], label [[COMMON_RET:%.*]], !prof [[PROF1]]
 ; CHECK:       guarded:
 ; CHECK-NEXT:    [[RANGE_CHECK:%.*]] = icmp ult i32 [[EL]], [[X]]
-; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[COMMON_RET]], !prof [[PROF9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[COMMON_RET]], !prof [[PROF6:![0-9]+]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[ARR_PTR:%.*]] = getelementptr i32, ptr [[ARR:%.*]], i32 [[EL]]
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[ARR_PTR]], align 4
@@ -314,17 +314,17 @@ range_check_failed:                               ; preds = %guarded
 define i32 @test_01_neg_overflowing_metadata(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_01_neg_overflowing_metadata(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ]
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 [[IV]]
 ; CHECK-NEXT:    [[EL:%.*]] = load i32, ptr [[EL_PTR]], align 4
 ; CHECK-NEXT:    [[BOUND_CHECK:%.*]] = icmp ult i32 [[EL]], [[LIMIT:%.*]]
-; CHECK-NEXT:    br i1 [[BOUND_CHECK]], label [[GUARDED:%.*]], label [[COMMON_RET:%.*]], !prof [[PROF10:![0-9]+]]
+; CHECK-NEXT:    br i1 [[BOUND_CHECK]], label [[GUARDED:%.*]], label [[COMMON_RET:%.*]], !prof [[PROF7:![0-9]+]]
 ; CHECK:       guarded:
 ; CHECK-NEXT:    [[RANGE_CHECK:%.*]] = icmp ult i32 [[EL]], [[X]]
-; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[COMMON_RET]], !prof [[PROF10]]
+; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[COMMON_RET]], !prof [[PROF7]]
 ; CHECK:       backedge:
 ; CHECK-NEXT:    [[ARR_PTR:%.*]] = getelementptr i32, ptr [[ARR:%.*]], i32 [[EL]]
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[ARR_PTR]], align 4
@@ -371,7 +371,7 @@ range_check_failed:                               ; preds = %guarded
 define i32 @test_02(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_02(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    [[INJECTED_COND:%.*]] = icmp ule i32 -2147483648, [[X]]
 ; CHECK-NEXT:    br i1 [[INJECTED_COND]], label [[LOOP_US:%.*]], label [[LOOP:%.*]]
 ; CHECK:       loop.us:
@@ -386,7 +386,7 @@ define i32 @test_02(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noun
 ; CHECK-NEXT:    store i32 [[IV_US]], ptr [[ARR_PTR_US]], align 4
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
 ; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -401,7 +401,7 @@ define i32 @test_02(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noun
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[ARR_PTR]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[BACKEDGE]] ], [ 0, [[GUARDED_US]] ], [ -1, [[LOOP]] ], [ -1, [[LOOP_US]] ], [ -2, [[GUARDED]] ]
 ; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
@@ -441,7 +441,7 @@ range_check_failed:                               ; preds = %guarded
 define i32 @test_02_inverse(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_02_inverse(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    [[INJECTED_COND:%.*]] = icmp ule i32 -2147483648, [[X]]
 ; CHECK-NEXT:    br i1 [[INJECTED_COND]], label [[LOOP_US:%.*]], label [[LOOP:%.*]]
 ; CHECK:       loop.us:
@@ -456,7 +456,7 @@ define i32 @test_02_inverse(ptr noundef %p, i32 noundef %n, i32 noundef %limit,
 ; CHECK-NEXT:    store i32 [[IV_US]], ptr [[ARR_PTR_US]], align 4
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
 ; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
@@ -471,7 +471,7 @@ define i32 @test_02_inverse(ptr noundef %p, i32 noundef %n, i32 noundef %limit,
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[ARR_PTR]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[BACKEDGE]] ], [ 0, [[GUARDED_US]] ], [ -1, [[LOOP]] ], [ -1, [[LOOP_US]] ], [ -2, [[GUARDED]] ]
 ; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
@@ -511,7 +511,7 @@ range_check_failed:                               ; preds = %guarded
 define i32 @test_03(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_03(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    [[INJECTED_COND:%.*]] = icmp ule i32 -2147483648, [[X]]
 ; CHECK-NEXT:    br i1 [[INJECTED_COND]], label [[LOOP_US:%.*]], label [[LOOP:%.*]]
 ; CHECK:       loop.us:
@@ -519,20 +519,20 @@ define i32 @test_03(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noun
 ; CHECK-NEXT:    [[EL_PTR_US:%.*]] = getelementptr i32, ptr [[P:%.*]], i32 [[IV_US]]
 ; CHECK-NEXT:    [[EL_US:%.*]] = load i32, ptr [[EL_PTR_US]], align 4
 ; CHECK-NEXT:    [[BOUND_CHECK_US:%.*]] = icmp slt i32 [[EL_US]], 0
-; CHECK-NEXT:    br i1 [[BOUND_CHECK_US]], label [[COMMON_RET:%.*]], label [[GUARDED_US]], !prof [[PROF15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[BOUND_CHECK_US]], label [[COMMON_RET:%.*]], label [[GUARDED_US]], !prof [[PROF10:![0-9]+]]
 ; CHECK:       guarded.us:
 ; CHECK-NEXT:    [[RANGE_CHECK_US:%.*]] = icmp ult i32 [[EL_US]], [[X]]
 ; CHECK-NEXT:    [[ARR_PTR_US:%.*]] = getelementptr i32, ptr [[ARR:%.*]], i32 [[EL_US]]
 ; CHECK-NEXT:    store i32 [[IV_US]], ptr [[ARR_PTR_US]], align 4
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
 ; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i32, ptr [[P]], i32 [[IV]]
 ; CHECK-NEXT:    [[EL:%.*]] = load i32, ptr [[EL_PTR]], align 4
 ; CHECK-NEXT:    [[BOUND_CHECK:%.*]] = icmp slt i32 [[EL]], 0
-; CHECK-NEXT:    br i1 [[BOUND_CHECK]], label [[COMMON_RET]], label [[GUARDED:%.*]], !prof [[PROF15]]
+; CHECK-NEXT:    br i1 [[BOUND_CHECK]], label [[COMMON_RET]], label [[GUARDED:%.*]], !prof [[PROF10]]
 ; CHECK:       guarded:
 ; CHECK-NEXT:    [[RANGE_CHECK:%.*]] = icmp ult i32 [[EL]], [[X]]
 ; CHECK-NEXT:    br i1 [[RANGE_CHECK]], label [[BACKEDGE]], label [[COMMON_RET]]
@@ -541,7 +541,7 @@ define i32 @test_03(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noun
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[ARR_PTR]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[BACKEDGE]] ], [ 0, [[GUARDED_US]] ], [ -1, [[LOOP]] ], [ -1, [[LOOP_US]] ], [ -2, [[GUARDED]] ]
 ; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
@@ -581,7 +581,7 @@ range_check_failed:                               ; preds = %guarded
 define i32 @test_04(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noundef %arr, ptr noundef %x_p) {
 ; CHECK-LABEL: @test_04(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef [[META0]]
+; CHECK-NEXT:    [[X:%.*]] = load i32, ptr [[X_P:%.*]], align 4, !noundef !0
 ; CHECK-NEXT:    [[INJECTED_COND:%.*]] = icmp ule i32 128, [[X]]
 ; CHECK-NEXT:    br i1 [[INJECTED_COND]], label [[LOOP_US:%.*]], label [[LOOP:%.*]]
 ; CHECK:       loop.us:
@@ -589,7 +589,7 @@ define i32 @test_04(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noun
 ; CHECK-NEXT:    [[EL_PTR_US:%.*]] = getelementptr i8, ptr [[P:%.*]], i32 [[IV_US]]
 ; CHECK-NEXT:    [[EL_US:%.*]] = load i8, ptr [[EL_PTR_US]], align 4
 ; CHECK-NEXT:    [[BOUND_CHECK_US:%.*]] = icmp slt i8 [[EL_US]], 0
-; CHECK-NEXT:    br i1 [[BOUND_CHECK_US]], label [[COMMON_RET:%.*]], label [[GUARDED_US]], !prof [[PROF15]]
+; CHECK-NEXT:    br i1 [[BOUND_CHECK_US]], label [[COMMON_RET:%.*]], label [[GUARDED_US]], !prof [[PROF10]]
 ; CHECK:       guarded.us:
 ; CHECK-NEXT:    [[EL_WIDE_US:%.*]] = zext i8 [[EL_US]] to i32
 ; CHECK-NEXT:    [[RANGE_CHECK_US:%.*]] = icmp ult i32 [[EL_WIDE_US]], [[X]]
@@ -597,13 +597,13 @@ define i32 @test_04(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noun
 ; CHECK-NEXT:    store i32 [[IV_US]], ptr [[ARR_PTR_US]], align 4
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
 ; CHECK-NEXT:    [[LOOP_COND_US:%.*]] = icmp slt i32 [[IV_NEXT_US]], [[N:%.*]]
-; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND_US]], label [[LOOP_US]], label [[COMMON_RET]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ 0, [[ENTRY]] ]
 ; CHECK-NEXT:    [[EL_PTR:%.*]] = getelementptr i8, ptr [[P]], i32 [[IV]]
 ; CHECK-NEXT:    [[EL:%.*]] = load i8, ptr [[EL_PTR]], align 4
 ; CHECK-NEXT:    [[BOUND_CHECK:%.*]] = icmp slt i8 [[EL]], 0
-; CHECK-NEXT:    br i1 [[BOUND_CHECK]], label [[COMMON_RET]], label [[GUARDED:%.*]], !prof [[PROF15]]
+; CHECK-NEXT:    br i1 [[BOUND_CHECK]], label [[COMMON_RET]], label [[GUARDED:%.*]], !prof [[PROF10]]
 ; CHECK:       guarded:
 ; CHECK-NEXT:    [[EL_WIDE:%.*]] = zext i8 [[EL]] to i32
 ; CHECK-NEXT:    [[RANGE_CHECK:%.*]] = icmp ult i32 [[EL_WIDE]], [[X]]
@@ -613,7 +613,7 @@ define i32 @test_04(ptr noundef %p, i32 noundef %n, i32 noundef %limit, ptr noun
 ; CHECK-NEXT:    store i32 [[IV]], ptr [[ARR_PTR]], align 4
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[LOOP_COND:%.*]] = icmp slt i32 [[IV_NEXT]], [[N]]
-; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT:    br i1 [[LOOP_COND]], label [[LOOP]], label [[COMMON_RET]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       common.ret:
 ; CHECK-NEXT:    [[COMMON_RET_OP:%.*]] = phi i32 [ 0, [[BACKEDGE]] ], [ 0, [[GUARDED_US]] ], [ -1, [[LOOP]] ], [ -1, [[LOOP_US]] ], [ -2, [[GUARDED]] ]
 ; CHECK-NEXT:    ret i32 [[COMMON_RET_OP]]
@@ -651,24 +651,17 @@ range_check_failed:                               ; preds = %guarded
   ret i32 -2
 }
 ;.
-; CHECK: [[META0]] = !{}
+; CHECK: [[META0:![0-9]+]] = !{}
 ; CHECK: [[PROF1]] = !{!"branch_weights", i32 100, i32 1}
-; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
-; CHECK: [[META3]] = !{!"llvm.loop.unswitch.nontrivial.disable"}
-; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]}
-; CHECK: [[META5]] = !{!"llvm.loop.unswitch.injection.disable"}
-; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]]}
-; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
-; CHECK: [[PROF8]] = !{!"branch_weights", i32 0, i32 0}
-; CHECK: [[PROF9]] = !{!"branch_weights", i32 2, i32 3}
-; CHECK: [[PROF10]] = !{!"branch_weights", i32 -1, i32 -1000}
-; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]]}
-; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META5]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]]}
-; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META5]]}
-; CHECK: [[PROF15]] = !{!"branch_weights", i32 1, i32 100}
-; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]}
-; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META3]]}
-; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]]}
+; CHECK: [[LOOP2]] = distinct !{!2, !3}
+; CHECK: [[META3:![0-9]+]] = !{!"llvm.loop.unswitch.injection.disable"}
+; CHECK: [[LOOP4]] = distinct !{!4, !3}
+; CHECK: [[PROF5]] = !{!"branch_weights", i32 0, i32 0}
+; CHECK: [[PROF6]] = !{!"branch_weights", i32 2, i32 3}
+; CHECK: [[PROF7]] = !{!"branch_weights", i32 -1, i32 -1000}
+; CHECK: [[LOOP8]] = distinct !{!8, !3}
+; CHECK: [[LOOP9]] = distinct !{!9, !3}
+; CHECK: [[PROF10]] = !{!"branch_weights", i32 1, i32 100}
+; CHECK: [[LOOP11]] = distinct !{!11, !3}
+; CHECK: [[LOOP12]] = distinct !{!12, !3}
 ;.
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/invalidate-block-and-loop-dispositions.ll b/llvm/test/Transforms/SimpleLoopUnswitch/invalidate-block-and-loop-dispositions.ll
index 5f713fae9e964..fcef88667449f 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/invalidate-block-and-loop-dispositions.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/invalidate-block-and-loop-dispositions.ll
@@ -14,17 +14,27 @@ define void @test_pr58136(i1 %c.1, i1 %c.2) {
 ; CHECK-NEXT:    [[C_1_FR:%.*]] = freeze i1 [[C_1:%.*]]
 ; CHECK-NEXT:    br i1 [[C_1_FR]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
+; CHECK-NEXT:    [[C_2_FR:%.*]] = freeze i1 [[C_2:%.*]]
+; CHECK-NEXT:    br i1 [[C_2_FR]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]]
+; CHECK:       entry.split.us.split.us:
 ; CHECK-NEXT:    br label [[LOOP_HEADER_US_US:%.*]]
-; CHECK:       loop.header.us:
-; CHECK-NEXT:    [[MUL1_US_US:%.*]] = phi i16 [ [[MUL_US_US:%.*]], [[LOOP_LATCH_US:%.*]] ], [ [[GLOB_PROMOTED]], [[ENTRY_SPLIT_US]] ]
+; CHECK:       loop.header.us.us:
+; CHECK-NEXT:    [[MUL1_US_US:%.*]] = phi i16 [ [[MUL_US_US:%.*]], [[LOOP_LATCH_US_US:%.*]] ], [ [[GLOB_PROMOTED]], [[ENTRY_SPLIT_US_SPLIT_US]] ]
 ; CHECK-NEXT:    [[CALL2_US_US:%.*]] = call i16 @foo()
-; CHECK-NEXT:    br label [[LOOP_LATCH_US_US:%.*]]
-; CHECK:       then.bb.us:
-; CHECK-NEXT:    br i1 [[C_2:%.*]], label [[LOOP_LATCH_US]], label [[EXIT_SPLIT_US:%.*]]
-; CHECK:       loop.latch.us:
+; CHECK-NEXT:    br label [[THEN_BB_US_US:%.*]]
+; CHECK:       then.bb.us.us:
+; CHECK-NEXT:    br label [[LOOP_LATCH_US_US]]
+; CHECK:       loop.latch.us.us:
 ; CHECK-NEXT:    [[MUL_US_US]] = mul nsw i16 [[MUL1_US_US]], [[L_3]]
 ; CHECK-NEXT:    store i16 [[MUL_US_US]], ptr @glob, align 2
-; CHECK-NEXT:    br label [[LOOP_HEADER_US_US]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br label [[LOOP_HEADER_US_US]]
+; CHECK:       entry.split.us.split:
+; CHECK-NEXT:    br label [[LOOP_HEADER_US:%.*]]
+; CHECK:       loop.header.us:
+; CHECK-NEXT:    [[CALL2_US:%.*]] = call i16 @foo()
+; CHECK-NEXT:    br label [[THEN_BB_US:%.*]]
+; CHECK:       then.bb.us:
+; CHECK-NEXT:    br label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -79,7 +89,7 @@ define void @test_pr58158(i1 %c.1) {
 ; CHECK:       outer.loopexit.us:
 ; CHECK-NEXT:    br label [[OUTER_BACKEDGE_US:%.*]]
 ; CHECK:       outer.backedge.us:
-; CHECK-NEXT:    br label [[OUTER_US]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br label [[OUTER_US]]
 ; CHECK:       entry.split:
 ; CHECK-NEXT:    br label [[OUTER:%.*]]
 ; CHECK:       outer:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
index d07c2fa4afd5d..8e97cb5cb42f8 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
@@ -32,7 +32,7 @@ define i32 @test1_freeze(ptr %ptr0, ptr %ptr1, ptr %ptr2) {
 ; CHECK-NEXT:    br label [[LATCH_US:%.*]]
 ; CHECK:       latch.us:
 ; CHECK-NEXT:    [[V_US:%.*]] = load i1, ptr [[PTR0:%.*]], align 1
-; CHECK-NEXT:    br i1 [[V_US]], label [[LOOP_BEGIN_US]], label [[LOOP_EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[V_US]], label [[LOOP_BEGIN_US]], label [[LOOP_EXIT_SPLIT_US:%.*]]
 ; CHECK:       loop_exit.split.us:
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -50,7 +50,7 @@ define i32 @test1_freeze(ptr %ptr0, ptr %ptr1, ptr %ptr2) {
 ; CHECK-NEXT:    br label [[LATCH_US2:%.*]]
 ; CHECK:       latch.us2:
 ; CHECK-NEXT:    [[V_US3:%.*]] = load i1, ptr [[PTR0]], align 1
-; CHECK-NEXT:    br i1 [[V_US3]], label [[LOOP_BEGIN_US1]], label [[LOOP_EXIT_SPLIT_SPLIT_US:%.*]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[V_US3]], label [[LOOP_BEGIN_US1]], label [[LOOP_EXIT_SPLIT_SPLIT_US:%.*]]
 ; CHECK:       loop_exit.split.split.us:
 ; CHECK-NEXT:    br label [[LOOP_EXIT_SPLIT:%.*]]
 ; CHECK:       entry.split.split:
@@ -276,7 +276,7 @@ define i32 @test7b(ptr %ptr, ptr %cond.ptr, ptr %a.ptr, ptr %b.ptr) {
 ; CHECK-NEXT:    [[V4_US:%.*]] = load i1, ptr [[PTR]], align 1
 ; CHECK-NEXT:    br i1 [[V4_US]], label [[INNER_LOOP_EXIT_LOOPEXIT_SPLIT_US:%.*]], label [[INNER_INNER_LOOP_D_US:%.*]]
 ; CHECK:       inner_inner_loop_d.us:
-; CHECK-NEXT:    br label [[INNER_INNER_LOOP_BEGIN_US]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br label [[INNER_INNER_LOOP_BEGIN_US]]
 ; CHECK:       inner_inner_loop_exit.split.us:
 ; CHECK-NEXT:    br label [[INNER_INNER_LOOP_EXIT]]
 ; CHECK:       loop_exit.split.us:
@@ -512,7 +512,7 @@ define i32 @test8b(ptr %ptr, ptr %cond.ptr, ptr %a.ptr, ptr %b.ptr) {
 ; CHECK-NEXT:    [[V2_US:%.*]] = load i1, ptr [[PTR]], align 1
 ; CHECK-NEXT:    br i1 [[V2_US]], label [[INNER_INNER_LOOP_LATCH_US:%.*]], label [[INNER_LOOP_EXIT_LOOPEXIT_SPLIT_US:%.*]]
 ; CHECK:       inner_inner_loop_latch.us:
-; CHECK-NEXT:    br label [[INNER_INNER_LOOP_BEGIN_US]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br label [[INNER_INNER_LOOP_BEGIN_US]]
 ; CHECK:       inner_inner_loop_exit.split.us:
 ; CHECK-NEXT:    br label [[INNER_INNER_LOOP_EXIT]]
 ; CHECK:       inner_loop_exit.loopexit.split.us:
@@ -614,7 +614,7 @@ define i32 @test10a(ptr %ptr, i1 %cond, ptr %a.ptr) {
 ; CHECK-NEXT:    [[V2_US:%.*]] = load i1, ptr [[PTR]], align 1
 ; CHECK-NEXT:    br i1 [[V2_US]], label [[LOOP_EXIT_SPLIT_US_LOOPEXIT:%.*]], label [[LOOP_BEGIN_BACKEDGE_US:%.*]]
 ; CHECK:       loop_begin.backedge.us:
-; CHECK-NEXT:    br label [[LOOP_BEGIN_US]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br label [[LOOP_BEGIN_US]]
 ; CHECK:       loop_exit.split.us.loopexit:
 ; CHECK-NEXT:    [[A_LCSSA_US_PH:%.*]] = phi i32 [ [[A_US]], [[LOOP_A_US]] ]
 ; CHECK-NEXT:    br label [[LOOP_EXIT_SPLIT_US]]
@@ -682,7 +682,7 @@ define i32 @test10b(ptr %ptr, i1 %cond, ptr %a.ptr) {
 ; CHECK-NEXT:    [[V2_US:%.*]] = load i1, ptr [[PTR]], align 1
 ; CHECK-NEXT:    br i1 [[V2_US]], label [[LOOP_BEGIN_BACKEDGE_US]], label [[LOOP_EXIT_SPLIT_US:%.*]]
 ; CHECK:       loop_begin.backedge.us:
-; CHECK-NEXT:    br label [[LOOP_BEGIN_US]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br label [[LOOP_BEGIN_US]]
 ; CHECK:       loop_exit.split.us:
 ; CHECK-NEXT:    [[A_LCSSA_US:%.*]] = phi i32 [ [[A_US]], [[LOOP_A_US]] ]
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
@@ -844,7 +844,7 @@ define i32 @test11b(ptr %ptr, ptr %cond.ptr, ptr %a.ptr, ptr %b.ptr) {
 ; CHECK-NEXT:    br label [[INNER_LOOP_A_US:%.*]]
 ; CHECK:       inner_loop_a.us:
 ; CHECK-NEXT:    [[V2_US:%.*]] = load i1, ptr [[PTR]], align 1
-; CHECK-NEXT:    br i1 [[V2_US]], label [[INNER_LOOP_EXIT_SPLIT_US:%.*]], label [[INNER_LOOP_BEGIN_US]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[V2_US]], label [[INNER_LOOP_EXIT_SPLIT_US:%.*]], label [[INNER_LOOP_BEGIN_US]]
 ; CHECK:       inner_loop_exit.split.us:
 ; CHECK-NEXT:    [[A_INNER_LCSSA_US:%.*]] = phi i32 [ [[A_US]], [[INNER_LOOP_A_US]] ]
 ; CHECK-NEXT:    br label [[INNER_LOOP_EXIT:%.*]]
@@ -1033,7 +1033,7 @@ define i32 @test12b(ptr %ptr, ptr %cond.ptr, ptr %a.ptr, ptr %b.ptr) {
 ; CHECK-NEXT:    br label [[INNER_INNER_LOOP_A_US:%.*]]
 ; CHECK:       inner_inner_loop_a.us:
 ; CHECK-NEXT:    [[V2_US:%.*]] = load i1, ptr [[PTR]], align 1
-; CHECK-NEXT:    br i1 [[V2_US]], label [[INNER_INNER_LOOP_EXIT_SPLIT_US:%.*]], label [[INNER_INNER_LOOP_BEGIN_US]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[V2_US]], label [[INNER_INNER_LOOP_EXIT_SPLIT_US:%.*]], label [[INNER_INNER_LOOP_BEGIN_US]]
 ; CHECK:       inner_inner_loop_exit.split.us:
 ; CHECK-NEXT:    [[A_INNER_INNER_LCSSA_US:%.*]] = phi i32 [ [[A_US]], [[INNER_INNER_LOOP_A_US]] ]
 ; CHECK-NEXT:    br label [[INNER_INNER_LOOP_EXIT:%.*]]
@@ -1142,7 +1142,7 @@ define i32 @test13a(ptr %ptr, i1 %cond, ptr %a.ptr, ptr %b.ptr) {
 ; CHECK-NEXT:    [[V2_US:%.*]] = load i1, ptr [[PTR]], align 1
 ; CHECK-NEXT:    br i1 [[V2_US]], label [[LOOP_EXIT_SPLIT_US:%.*]], label [[LOOP_LATCH_US]]
 ; CHECK:       loop_latch.us:
-; CHECK-NEXT:    br label [[LOOP_BEGIN_US]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br label [[LOOP_BEGIN_US]]
 ; CHECK:       loop_exit.split.us:
 ; CHECK-NEXT:    [[LCSSA_US:%.*]] = phi i32 [ [[A_US]], [[LOOP_A_US]] ]
 ; CHECK-NEXT:    br label [[LOOP_EXIT:%.*]]
@@ -1237,7 +1237,7 @@ define i32 @test13b(ptr %ptr, i1 %cond, ptr %a.ptr, ptr %b.ptr) {
 ; CHECK-NEXT:    [[V2_US:%.*]] = load i1, ptr [[PTR]], align 1
 ; CHECK-NEXT:    br i1 [[V2_US]], label [[LOOP_EXIT_SPLIT_US_LOOPEXIT:%.*]], label [[LOOP_LATCH_US:%.*]]
 ; CHECK:       loop_latch.us:
-; CHECK-NEXT:    br label [[LOOP_BEGIN_US]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    br label [[LOOP_BEGIN_US]]
 ; CHECK:       loop_exit.split.us.loopexit:
 ; CHECK-NEXT:    [[LCSSA_US_PH:%.*]] = phi i32 [ [[A_US]], [[LOOP_A_US]] ]
 ; CHECK-NEXT:    br label [[LOOP_EXIT_SPLIT_US]]
@@ -1356,7 +1356,7 @@ define void @test23(i1 %arg, ptr %ptr) {
 ; CHECK-NEXT:    br label [[OUTER_LATCH_US:%.*]]
 ; CHECK:       outer.latch.us:
 ; CHECK-NEXT:    [[OUTER_COND_US:%.*]] = load i1, ptr [[PTR]], align 1
-; CHECK-NEXT:    br i1 [[OUTER_COND_US]], label [[OUTER_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[OUTER_COND_US]], label [[OUTER_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -1426,10 +1426,10 @@ define i32 @test29(i32 %arg) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
 ; CHECK-NEXT:    switch i32 [[ARG_FR]], label [[ENTRY_SPLIT:%.*]] [
-; CHECK-NEXT:      i32 0, label [[ENTRY_SPLIT_US:%.*]]
-; CHECK-NEXT:      i32 1, label [[ENTRY_SPLIT_US]]
-; CHECK-NEXT:      i32 2, label [[ENTRY_SPLIT_US1:%.*]]
-; CHECK-NEXT:      i32 3, label [[ENTRY_SPLIT]]
+; CHECK-NEXT:    i32 0, label [[ENTRY_SPLIT_US:%.*]]
+; CHECK-NEXT:    i32 1, label [[ENTRY_SPLIT_US]]
+; CHECK-NEXT:    i32 2, label [[ENTRY_SPLIT_US1:%.*]]
+; CHECK-NEXT:    i32 3, label [[ENTRY_SPLIT]]
 ; CHECK-NEXT:    ]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[HEADER_US:%.*]]
@@ -1456,7 +1456,7 @@ define i32 @test29(i32 %arg) {
 ; CHECK-NEXT:    br label [[LATCH_US:%.*]]
 ; CHECK:       latch.us:
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp slt i32 [[TMP_C_SUM_US]], 42
-; CHECK-NEXT:    br i1 [[CMP2_US]], label [[HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP2_US]], label [[HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    [[LCSSA_PHI_US:%.*]] = phi i32 [ [[TMP_C_SUM_US]], [[LATCH_US]] ]
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
@@ -1485,7 +1485,7 @@ define i32 @test29(i32 %arg) {
 ; CHECK-NEXT:    br label [[LATCH_US18:%.*]]
 ; CHECK:       latch.us18:
 ; CHECK-NEXT:    [[CMP2_US19:%.*]] = icmp slt i32 [[TMP_C_SUM_US17]], 42
-; CHECK-NEXT:    br i1 [[CMP2_US19]], label [[HEADER_US2]], label [[EXIT_SPLIT_SPLIT_US:%.*]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP2_US19]], label [[HEADER_US2]], label [[EXIT_SPLIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.split.us:
 ; CHECK-NEXT:    [[LCSSA_PHI_US20:%.*]] = phi i32 [ [[TMP_C_SUM_US17]], [[LATCH_US18]] ]
 ; CHECK-NEXT:    br label [[EXIT_SPLIT:%.*]]
@@ -1587,10 +1587,10 @@ define i32 @test30(i32 %arg) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARG_FR:%.*]] = freeze i32 [[ARG:%.*]]
 ; CHECK-NEXT:    switch i32 [[ARG_FR]], label [[ENTRY_SPLIT:%.*]] [
-; CHECK-NEXT:      i32 -1, label [[ENTRY_SPLIT]]
-; CHECK-NEXT:      i32 0, label [[ENTRY_SPLIT_US:%.*]]
-; CHECK-NEXT:      i32 1, label [[ENTRY_SPLIT_US1:%.*]]
-; CHECK-NEXT:      i32 2, label [[ENTRY_SPLIT_US1]]
+; CHECK-NEXT:    i32 -1, label [[ENTRY_SPLIT]]
+; CHECK-NEXT:    i32 0, label [[ENTRY_SPLIT_US:%.*]]
+; CHECK-NEXT:    i32 1, label [[ENTRY_SPLIT_US1:%.*]]
+; CHECK-NEXT:    i32 2, label [[ENTRY_SPLIT_US1]]
 ; CHECK-NEXT:    ]
 ; CHECK:       entry.split.us:
 ; CHECK-NEXT:    br label [[HEADER_US:%.*]]
@@ -1612,7 +1612,7 @@ define i32 @test30(i32 %arg) {
 ; CHECK-NEXT:    br label [[LATCH_US:%.*]]
 ; CHECK:       latch.us:
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp slt i32 [[TMP_B_SUM_US]], 42
-; CHECK-NEXT:    br i1 [[CMP2_US]], label [[HEADER_US]], label [[LOOP_EXIT2_SPLIT_US:%.*]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP2_US]], label [[HEADER_US]], label [[LOOP_EXIT2_SPLIT_US:%.*]]
 ; CHECK:       loop.exit2.split.us:
 ; CHECK-NEXT:    [[L2_PHI_US:%.*]] = phi i32 [ [[TMP_B_SUM_US]], [[LATCH_US]] ]
 ; CHECK-NEXT:    br label [[LOOP_EXIT2:%.*]]
@@ -1636,7 +1636,7 @@ define i32 @test30(i32 %arg) {
 ; CHECK-NEXT:    br label [[LATCH_US14:%.*]]
 ; CHECK:       latch.us14:
 ; CHECK-NEXT:    [[CMP2_US15:%.*]] = icmp slt i32 [[TMP_B_SUM_US13]], 42
-; CHECK-NEXT:    br i1 [[CMP2_US15]], label [[HEADER_US2]], label [[LOOP_EXIT2_SPLIT_SPLIT_US:%.*]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[CMP2_US15]], label [[HEADER_US2]], label [[LOOP_EXIT2_SPLIT_SPLIT_US:%.*]]
 ; CHECK:       loop.exit2.split.split.us:
 ; CHECK-NEXT:    [[L2_PHI_US16:%.*]] = phi i32 [ [[TMP_B_SUM_US13]], [[LATCH_US14]] ]
 ; CHECK-NEXT:    br label [[LOOP_EXIT2_SPLIT:%.*]]
@@ -2259,9 +2259,9 @@ define void @hoist_inner_loop_switch(ptr %ptr) {
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @cond.i32()
 ; CHECK-NEXT:    [[V1_FR:%.*]] = freeze i32 [[V1]]
 ; CHECK-NEXT:    switch i32 [[V1_FR]], label [[B_HEADER_SPLIT:%.*]] [
-; CHECK-NEXT:      i32 1, label [[B_HEADER_SPLIT_US:%.*]]
-; CHECK-NEXT:      i32 2, label [[B_HEADER_SPLIT_US]]
-; CHECK-NEXT:      i32 3, label [[B_HEADER_SPLIT_US]]
+; CHECK-NEXT:    i32 1, label [[B_HEADER_SPLIT_US:%.*]]
+; CHECK-NEXT:    i32 2, label [[B_HEADER_SPLIT_US]]
+; CHECK-NEXT:    i32 3, label [[B_HEADER_SPLIT_US]]
 ; CHECK-NEXT:    ]
 ; CHECK:       b.header.split.us:
 ; CHECK-NEXT:    br label [[C_HEADER_US:%.*]]
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll
index 64b18291b22d1..c86fa349200c5 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll
@@ -28,7 +28,7 @@ define i32 @basic(i32 %N, i1 %cond, i32 %select_input) {
 ; CHECK-NEXT:    [[UNSWITCHED_SELECT_US:%.*]] = phi i32 [ [[SELECT_INPUT]], [[TMP0]] ]
 ; CHECK-NEXT:    [[ADD_US]] = add nuw nsw i32 [[UNSWITCHED_SELECT_US]], [[RES_US]]
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_US]], 1
-; CHECK-NEXT:    br label [[FOR_COND_US]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br label [[FOR_COND_US]]
 ; CHECK:       for.cond.cleanup.split.us:
 ; CHECK-NEXT:    [[RES_LCSSA_US:%.*]] = phi i32 [ [[RES_US]], [[FOR_COND_US]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
@@ -132,7 +132,7 @@ define i32 @select_phi_input(i32 %N, i1 %cond) {
 ; CHECK-NEXT:    [[UNSWITCHED_SELECT_US:%.*]] = phi i32 [ [[I_US]], [[TMP0]] ]
 ; CHECK-NEXT:    [[ADD_US]] = add nuw nsw i32 [[UNSWITCHED_SELECT_US]], [[RES_US]]
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_US]], 1
-; CHECK-NEXT:    br label [[FOR_COND_US]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br label [[FOR_COND_US]]
 ; CHECK:       for.cond.cleanup.split.us:
 ; CHECK-NEXT:    [[RES_LCSSA_US:%.*]] = phi i32 [ [[RES_US]], [[FOR_COND_US]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
@@ -195,7 +195,7 @@ define i32 @basic_cond_noundef(i32 %N, i1 noundef %cond) {
 ; CHECK-NEXT:    [[UNSWITCHED_SELECT_US:%.*]] = phi i32 [ [[I_US]], [[TMP0]] ]
 ; CHECK-NEXT:    [[ADD_US]] = add nuw nsw i32 [[UNSWITCHED_SELECT_US]], [[RES_US]]
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_US]], 1
-; CHECK-NEXT:    br label [[FOR_COND_US]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br label [[FOR_COND_US]]
 ; CHECK:       for.cond.cleanup.split.us:
 ; CHECK-NEXT:    [[RES_LCSSA_US:%.*]] = phi i32 [ [[RES_US]], [[FOR_COND_US]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
@@ -285,24 +285,55 @@ define i32 @chained_select(i32 %N, i1 %cond, i1 %cond2) {
 ; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[COND]]
 ; CHECK-NEXT:    br i1 [[COND_FR]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
+; CHECK-NEXT:    [[COND2_FR13:%.*]] = freeze i1 [[COND2]]
+; CHECK-NEXT:    br i1 [[COND2_FR13]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]]
+; CHECK:       entry.split.us.split.us:
+; CHECK-NEXT:    br label [[FOR_COND_US_US:%.*]]
+; CHECK:       for.cond.us.us:
+; CHECK-NEXT:    [[RES_US_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US_SPLIT_US]] ], [ [[ADD_US_US:%.*]], [[TMP3:%.*]] ]
+; CHECK-NEXT:    [[I_US_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US_SPLIT_US]] ], [ [[INC_US_US:%.*]], [[TMP3]] ]
+; CHECK-NEXT:    [[CMP_US_US:%.*]] = icmp slt i32 [[I_US_US]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP_US_US]], label [[FOR_BODY_US_US:%.*]], label [[FOR_COND_CLEANUP_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       for.body.us.us:
+; CHECK-NEXT:    br label [[TMP0:%.*]]
+; CHECK:       0:
+; CHECK-NEXT:    br label [[TMP1:%.*]]
+; CHECK:       1:
+; CHECK-NEXT:    [[UNSWITCHED_SELECT_US_US:%.*]] = phi i32 [ [[I_US_US]], [[TMP0]] ]
+; CHECK-NEXT:    br label [[TMP2:%.*]]
+; CHECK:       2:
+; CHECK-NEXT:    br label [[TMP3]]
+; CHECK:       3:
+; CHECK-NEXT:    [[UNSWITCHED_SELECT_US11:%.*]] = phi i32 [ [[UNSWITCHED_SELECT_US_US]], [[TMP2]] ]
+; CHECK-NEXT:    [[ADD_US_US]] = add nuw nsw i32 [[UNSWITCHED_SELECT_US11]], [[RES_US_US]]
+; CHECK-NEXT:    [[INC_US_US]] = add nuw nsw i32 [[I_US_US]], 1
+; CHECK-NEXT:    br label [[FOR_COND_US_US]]
+; CHECK:       for.cond.cleanup.split.us.split.us:
+; CHECK-NEXT:    [[RES_LCSSA_US_US:%.*]] = phi i32 [ [[RES_US_US]], [[FOR_COND_US_US]] ]
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_SPLIT_US:%.*]]
+; CHECK:       entry.split.us.split:
 ; CHECK-NEXT:    br label [[FOR_COND_US:%.*]]
 ; CHECK:       for.cond.us:
-; CHECK-NEXT:    [[RES_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[ADD_US:%.*]], [[TMP1:%.*]] ]
-; CHECK-NEXT:    [[I_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US]] ], [ [[INC_US:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[RES_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US_SPLIT]] ], [ [[ADD_US:%.*]], [[TMP6:%.*]] ]
+; CHECK-NEXT:    [[I_US:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_US_SPLIT]] ], [ [[INC_US:%.*]], [[TMP6]] ]
 ; CHECK-NEXT:    [[CMP_US:%.*]] = icmp slt i32 [[I_US]], [[N]]
-; CHECK-NEXT:    br i1 [[CMP_US]], label [[FOR_BODY_US:%.*]], label [[FOR_COND_CLEANUP_SPLIT_US:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_US]], label [[FOR_BODY_US:%.*]], label [[FOR_COND_CLEANUP_SPLIT_US_SPLIT:%.*]]
 ; CHECK:       for.body.us:
-; CHECK-NEXT:    br label [[TMP0:%.*]]
-; CHECK:       0:
-; CHECK-NEXT:    br label [[TMP1]]
-; CHECK:       1:
-; CHECK-NEXT:    [[UNSWITCHED_SELECT_US:%.*]] = phi i32 [ [[I_US]], [[TMP0]] ]
-; CHECK-NEXT:    [[SELECT2_US:%.*]] = select i1 [[COND2]], i32 [[UNSWITCHED_SELECT_US]], i32 24
-; CHECK-NEXT:    [[ADD_US]] = add nuw nsw i32 [[SELECT2_US]], [[RES_US]]
+; CHECK-NEXT:    br label [[TMP4:%.*]]
+; CHECK:       4:
+; CHECK-NEXT:    br label [[TMP5:%.*]]
+; CHECK:       5:
+; CHECK-NEXT:    [[UNSWITCHED_SELECT_US:%.*]] = phi i32 [ [[I_US]], [[TMP4]] ]
+; CHECK-NEXT:    br label [[TMP6]]
+; CHECK:       6:
+; CHECK-NEXT:    [[ADD_US]] = add nuw nsw i32 24, [[RES_US]]
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_US]], 1
-; CHECK-NEXT:    br label [[FOR_COND_US]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK:       for.cond.cleanup.split.us:
+; CHECK-NEXT:    br label [[FOR_COND_US]]
+; CHECK:       for.cond.cleanup.split.us.split:
 ; CHECK-NEXT:    [[RES_LCSSA_US:%.*]] = phi i32 [ [[RES_US]], [[FOR_COND_US]] ]
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_SPLIT_US]]
+; CHECK:       for.cond.cleanup.split.us:
+; CHECK-NEXT:    [[DOTUS_PHI12:%.*]] = phi i32 [ [[RES_LCSSA_US]], [[FOR_COND_CLEANUP_SPLIT_US_SPLIT]] ], [ [[RES_LCSSA_US_US]], [[FOR_COND_CLEANUP_SPLIT_US_SPLIT_US]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       entry.split:
 ; CHECK-NEXT:    [[COND2_FR:%.*]] = freeze i1 [[COND2]]
@@ -310,36 +341,36 @@ define i32 @chained_select(i32 %N, i1 %cond, i1 %cond2) {
 ; CHECK:       entry.split.split.us:
 ; CHECK-NEXT:    br label [[FOR_COND_US1:%.*]]
 ; CHECK:       for.cond.us1:
-; CHECK-NEXT:    [[RES_US2:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_SPLIT_US]] ], [ [[ADD_US7:%.*]], [[TMP4:%.*]] ]
-; CHECK-NEXT:    [[I_US3:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_SPLIT_US]] ], [ [[INC_US8:%.*]], [[TMP4]] ]
+; CHECK-NEXT:    [[RES_US2:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_SPLIT_US]] ], [ [[ADD_US7:%.*]], [[TMP9:%.*]] ]
+; CHECK-NEXT:    [[I_US3:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_SPLIT_US]] ], [ [[INC_US8:%.*]], [[TMP9]] ]
 ; CHECK-NEXT:    [[CMP_US4:%.*]] = icmp slt i32 [[I_US3]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP_US4]], label [[FOR_BODY_US5:%.*]], label [[FOR_COND_CLEANUP_SPLIT_SPLIT_US:%.*]]
 ; CHECK:       for.body.us5:
-; CHECK-NEXT:    br label [[TMP2:%.*]]
-; CHECK:       2:
-; CHECK-NEXT:    br label [[TMP3:%.*]]
-; CHECK:       3:
-; CHECK-NEXT:    br label [[TMP4]]
-; CHECK:       4:
-; CHECK-NEXT:    [[UNSWITCHED_SELECT_US6:%.*]] = phi i32 [ 42, [[TMP3]] ]
+; CHECK-NEXT:    br label [[TMP7:%.*]]
+; CHECK:       7:
+; CHECK-NEXT:    br label [[TMP8:%.*]]
+; CHECK:       8:
+; CHECK-NEXT:    br label [[TMP9]]
+; CHECK:       9:
+; CHECK-NEXT:    [[UNSWITCHED_SELECT_US6:%.*]] = phi i32 [ 42, [[TMP8]] ]
 ; CHECK-NEXT:    [[ADD_US7]] = add nuw nsw i32 [[UNSWITCHED_SELECT_US6]], [[RES_US2]]
 ; CHECK-NEXT:    [[INC_US8]] = add nuw nsw i32 [[I_US3]], 1
-; CHECK-NEXT:    br label [[FOR_COND_US1]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br label [[FOR_COND_US1]]
 ; CHECK:       for.cond.cleanup.split.split.us:
 ; CHECK-NEXT:    [[RES_LCSSA_US9:%.*]] = phi i32 [ [[RES_US2]], [[FOR_COND_US1]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_SPLIT:%.*]]
 ; CHECK:       entry.split.split:
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]]
 ; CHECK:       for.cond:
-; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_SPLIT]] ], [ [[ADD:%.*]], [[TMP6:%.*]] ]
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_SPLIT]] ], [ [[INC:%.*]], [[TMP6]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_SPLIT]] ], [ [[ADD:%.*]], [[TMP11:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY_SPLIT_SPLIT]] ], [ [[INC:%.*]], [[TMP11]] ]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I]], [[N]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP_SPLIT_SPLIT:%.*]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    br label [[TMP5:%.*]]
-; CHECK:       5:
-; CHECK-NEXT:    br label [[TMP6]]
-; CHECK:       6:
+; CHECK-NEXT:    br label [[TMP10:%.*]]
+; CHECK:       10:
+; CHECK-NEXT:    br label [[TMP11]]
+; CHECK:       11:
 ; CHECK-NEXT:    [[ADD]] = add nuw nsw i32 24, [[RES]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I]], 1
 ; CHECK-NEXT:    br label [[FOR_COND]]
@@ -350,7 +381,7 @@ define i32 @chained_select(i32 %N, i1 %cond, i1 %cond2) {
 ; CHECK-NEXT:    [[DOTUS_PHI10:%.*]] = phi i32 [ [[RES_LCSSA]], [[FOR_COND_CLEANUP_SPLIT_SPLIT]] ], [ [[RES_LCSSA_US9]], [[FOR_COND_CLEANUP_SPLIT_SPLIT_US]] ]
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[DOTUS_PHI:%.*]] = phi i32 [ [[DOTUS_PHI10]], [[FOR_COND_CLEANUP_SPLIT]] ], [ [[RES_LCSSA_US]], [[FOR_COND_CLEANUP_SPLIT_US]] ]
+; CHECK-NEXT:    [[DOTUS_PHI:%.*]] = phi i32 [ [[DOTUS_PHI10]], [[FOR_COND_CLEANUP_SPLIT]] ], [ [[DOTUS_PHI12]], [[FOR_COND_CLEANUP_SPLIT_US]] ]
 ; CHECK-NEXT:    ret i32 [[DOTUS_PHI]]
 ;
 entry:
@@ -396,7 +427,7 @@ define i32 @select_in_if(i32 %N, i1 %cond) {
 ; CHECK-NEXT:    [[P_US:%.*]] = phi i32 [ [[UNSWITCHED_SELECT_US:%.*]], [[TMP1:%.*]] ], [ 24, [[FOR_BODY_US]] ]
 ; CHECK-NEXT:    [[ADD_US]] = add nuw nsw i32 [[P_US]], [[RES_US]]
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_US]], 1
-; CHECK-NEXT:    br label [[FOR_COND_US]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br label [[FOR_COND_US]]
 ; CHECK:       0:
 ; CHECK-NEXT:    br label [[TMP1]]
 ; CHECK:       1:
@@ -486,7 +517,7 @@ define i32 @select_in_if_else(i32 %N, i1 %cond) {
 ; CHECK-NEXT:    [[P_US:%.*]] = phi i32 [ [[COND1A_US]], [[FOR_BODY_IF_US]] ], [ [[UNSWITCHED_SELECT_US:%.*]], [[TMP1:%.*]] ]
 ; CHECK-NEXT:    [[ADD_US]] = add nuw nsw i32 [[P_US]], [[RES_US]]
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_US]], 1
-; CHECK-NEXT:    br label [[FOR_COND_US]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br label [[FOR_COND_US]]
 ; CHECK:       0:
 ; CHECK-NEXT:    br label [[TMP1]]
 ; CHECK:       1:
@@ -575,7 +606,7 @@ define dso_local void @select_nested_loop(i1 noundef zeroext %cond, i32 noundef
 ; CHECK:       for.cond1.for.cond.cleanup3_crit_edge.us.us:
 ; CHECK-NEXT:    [[INC7_US_US]] = add nuw i32 [[I_018_US_US]], 1
 ; CHECK-NEXT:    [[EXITCOND21_NOT_US:%.*]] = icmp eq i32 [[INC7_US_US]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND21_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US_US]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND21_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_COND1_PREHEADER_US_US]]
 ; CHECK:       for.cond1.preheader.us.split.us.us:
 ; CHECK-NEXT:    br label [[FOR_BODY4_US_US_US:%.*]]
 ; CHECK:       for.body4.us.us.us:
@@ -588,7 +619,7 @@ define dso_local void @select_nested_loop(i1 noundef zeroext %cond, i32 noundef
 ; CHECK-NEXT:    tail call void @bar(i32 noundef [[UNSWITCHED_SELECT_US_US]])
 ; CHECK-NEXT:    [[INC_US_US_US]] = add nuw i32 [[J_016_US_US_US]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT_US_US:%.*]] = icmp eq i32 [[INC_US_US_US]], [[M]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US_US]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_SPLIT_US_US:%.*]], label [[FOR_BODY4_US_US_US]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US_US]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_SPLIT_US_US:%.*]], label [[FOR_BODY4_US_US_US]]
 ; CHECK:       for.cond1.for.cond.cleanup3_crit_edge.us.split.us.us:
 ; CHECK-NEXT:    br label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_US]]
 ; CHECK:       for.cond.cleanup.loopexit.split.us:
@@ -676,7 +707,7 @@ define dso_local void @select_invariant_outer_loop(i1 noundef zeroext %cond, i32
 ; CHECK-NEXT:    tail call void @bar(i32 noundef [[UNSWITCHED_SELECT_US]])
 ; CHECK-NEXT:    [[INC_US_US]] = add nuw i32 [[J_019_US_US]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT_US:%.*]] = icmp eq i32 [[INC_US_US]], [[M]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_SPLIT_US:%.*]], label [[FOR_BODY4_US_US]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US_SPLIT_US:%.*]], label [[FOR_BODY4_US_US]]
 ; CHECK:       for.cond1.for.cond.cleanup3_crit_edge.us.split.us:
 ; CHECK-NEXT:    br label [[FOR_COND1_FOR_COND_CLEANUP3_CRIT_EDGE_US]]
 ; CHECK:       for.cond1.preheader.us.split:
@@ -751,7 +782,7 @@ define dso_local i32 @trivial_select_cond(i32 noundef %n, i32 noundef %a, i32 no
 ; CHECK-NEXT:    tail call void @bar(i32 noundef [[UNSWITCHED_SELECT_US]])
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_03_US]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT_US:%.*]] = icmp eq i32 [[INC_US]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]]
 ; CHECK:       for.cond.cleanup.loopexit.split.us:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       for.body.preheader.split:
@@ -808,7 +839,7 @@ define i32 @and_lhs_invariant(i32 %num, i1 %cond) {
 ; CHECK-NEXT:    tail call void @bar(i32 noundef [[UNSWITCHED_SELECT_US]])
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_07_US]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT_US:%.*]] = icmp eq i32 [[INC_US]], [[NUM]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]]
 ; CHECK:       for.cond.cleanup.loopexit.split.us:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       for.body.preheader.split:
@@ -873,7 +904,7 @@ define i32 @and_rhs_invariant(i32 %num, i1 %cond) {
 ; CHECK-NEXT:    tail call void @bar(i32 noundef [[UNSWITCHED_SELECT_US]])
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_07_US]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT_US:%.*]] = icmp eq i32 [[INC_US]], [[NUM]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]]
 ; CHECK:       for.cond.cleanup.loopexit.split.us:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       for.body.preheader.split:
@@ -940,7 +971,7 @@ define i32 @or_lhs_invariant(i32 %num, i1 %cond) {
 ; CHECK-NEXT:    tail call void @bar(i32 noundef [[UNSWITCHED_SELECT_US]])
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_07_US]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT_US:%.*]] = icmp eq i32 [[INC_US]], [[NUM]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]]
 ; CHECK:       for.cond.cleanup.loopexit.split.us:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       for.body.preheader.split:
@@ -1007,7 +1038,7 @@ define i32 @or_rhs_invariant(i32 %num, i1 %cond) {
 ; CHECK-NEXT:    tail call void @bar(i32 noundef [[UNSWITCHED_SELECT_US]])
 ; CHECK-NEXT:    [[INC_US]] = add nuw nsw i32 [[I_07_US]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT_US:%.*]] = icmp eq i32 [[INC_US]], [[NUM]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT_US]], label [[FOR_COND_CLEANUP_LOOPEXIT_SPLIT_US:%.*]], label [[FOR_BODY_US]]
 ; CHECK:       for.cond.cleanup.loopexit.split.us:
 ; CHECK-NEXT:    br label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]]
 ; CHECK:       for.body.preheader.split:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
index 36f7a9e8cd654..9567b6b793239 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
@@ -2626,45 +2626,66 @@ loop_a:
 ; The second unswitched condition.
 ;
 ; CHECK:       entry.split.us:
-; CHECK-NEXT:    br label %loop_begin.us
+; CHECK-NEXT:    br i1 %cond2, label %entry.split.us.split.us, label %entry.split.us.split
 
 loop_a_a:
   call i32 @a()
   br label %latch
 ; The 'loop_a_a' unswitched loop.
 ;
-; CHECK:       loop_begin.us:
-; CHECK-NEXT:    br label %loop_a.us
+; CHECK:       entry.split.us.split.us:
+; CHECK-NEXT:    br label %loop_begin.us.us
 ;
-; CHECK:       loop_a.us:
-; CHECK-NEXT:    br i1 %cond2, label %loop_a_a.us, label %loop_a_c.us
+; CHECK:       loop_begin.us.us:
+; CHECK-NEXT:    br label %loop_a.us.us
 ;
-; The 'loop_a_c' unswitched loop.
+; CHECK:       loop_a.us.us:
+; CHECK-NEXT:    br label %loop_a_a.us.us
 ;
-; CHECK:       loop_a_c.us:
-; CHECK-NEXT:    call i32 @c()
-; CHECK-NEXT:    br label %latch.us
-;
-; CHECK:       loop_a_a.us:
+; CHECK:       loop_a_a.us.us:
 ; CHECK-NEXT:    call i32 @a()
-; CHECK-NEXT:    br label %latch.us
+; CHECK-NEXT:    br label %latch.us.us
 ;
-; CHECK:       latch.us:
+; CHECK:       latch.us.us:
 ; CHECK-NEXT:    %[[V:.*]] = load i1, ptr %ptr
-; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us, !llvm.loop !22
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us.us, label %loop_exit.split.us.split.us
 ;
-; CHECK:       loop_exit.split.us
-; CHECK-NEXT:    br label %loop_exit
+; CHECK:       loop_exit.split.us.split.us:
+; CHECK-NEXT:    br label %loop_exit.split
 
 loop_a_c:
   call i32 @c()
   br label %latch
+; The 'loop_a_c' unswitched loop.
+;
+; CHECK:       entry.split.us.split:
+; CHECK-NEXT:    br label %loop_begin.us
+;
+; CHECK:       loop_begin.us:
+; CHECK-NEXT:    br label %loop_a.us
+;
+; CHECK:       loop_a.us:
+; CHECK-NEXT:    br label %loop_a_c.us
+;
+; CHECK:       loop_a_c.us:
+; CHECK-NEXT:    call i32 @c()
+; CHECK-NEXT:    br label %latch
+;
+; CHECK:       latch.us:
+; CHECK-NEXT:    %[[V:.*]] = load i1, ptr %ptr
+; CHECK-NEXT:    br i1 %[[V]], label %loop_begin.us, label %loop_exit.split.us.split
+;
+; CHECK:       loop_exit.split.us.split:
+; CHECK-NEXT:    br label %loop_exit.split
 
 loop_b:
   call i32 @b()
   br label %latch
 ; The 'loop_b' unswitched loop.
 ;
+; CHECK:       entry.split:
+; CHECK-NEXT:    br label %loop_begin
+;
 ; CHECK:       loop_begin:
 ; CHECK-NEXT:    br label %loop_b
 ;
@@ -2964,9 +2985,9 @@ loop_a:
 ;
 ; CHECK:       [[LOOP_LATCH_A]]:
 ; CHECK-NEXT:    %[[V_A:.*]] = load i1, ptr %ptr
-; CHECK:         br i1 %[[V_A]], label %loop_begin.us, label %loop_exit.split.us, !llvm.loop !26
+; CHECK:         br i1 %[[V_A]], label %[[LOOP_BEGIN_A]], label %[[LOOP_EXIT_A:.*]]
 ;
-; CHECK:       loop_exit.split.us:
+; CHECK:       [[LOOP_EXIT_A]]:
 ; CHECK-NEXT:    br label %loop_exit
 
 loop_b:
@@ -2986,10 +3007,10 @@ loop_b:
 ;
 ; CHECK:       [[LOOP_LATCH_B]]:
 ; CHECK-NEXT:    %[[V_B:.*]] = load i1, ptr %ptr
-; CHECK:         br i1 %[[V_B]], label %loop_begin.us2, label %loop_exit.split.split.us, !llvm.loop !27
+; CHECK:         br i1 %[[V_B]], label %[[LOOP_BEGIN_B]], label %[[LOOP_EXIT_B:.*]]
 ;
-; CHECK:       loop_exit.split.split.us:
-; CHECK-NEXT:    br label %loop_exit.split
+; CHECK:       [[LOOP_EXIT_B]]:
+; CHECK-NEXT:    br label %loop_exit
 
 loop_c:
   call i32 @c()
@@ -3008,10 +3029,10 @@ loop_c:
 ;
 ; CHECK:       [[LOOP_LATCH_C]]:
 ; CHECK-NEXT:    %[[V_C:.*]] = load i1, ptr %ptr
-; CHECK:         br i1 %[[V_C]], label %loop_begin.us6, label %loop_exit.split.split.split.us, !llvm.loop !28
+; CHECK:         br i1 %[[V_C]], label %[[LOOP_BEGIN_C]], label %[[LOOP_EXIT_C:.*]]
 ;
-; CHECK:       loop_exit.split.split.split.us:
-; CHECK-NEXT:    br label %loop_exit.split.split
+; CHECK:       [[LOOP_EXIT_C]]:
+; CHECK-NEXT:    br label %loop_exit
 
 latch:
   %v = load i1, ptr %ptr
@@ -3111,9 +3132,9 @@ body.a:
 ;
 ; CHECK:       [[LATCH_A]]:
 ; CHECK-NEXT:    %[[CMP2_A:.*]] = icmp slt i32 %[[TMP_C_SUM_A]], 42
-; CHECK:         br i1 %[[CMP2_A]], label %header.us, label %exit.split.us, !llvm.loop !29
+; CHECK:         br i1 %[[CMP2_A]], label %[[HEADER_A]], label %[[LOOP_EXIT_A:.*]]
 ;
-; CHECK:       exit.split.us:
+; CHECK:       [[LOOP_EXIT_A]]:
 ; CHECK-NEXT:    %[[LCSSA_A:.*]] = phi i32 [ %[[TMP_C_SUM_A]], %[[LATCH_A]] ]
 ; CHECK-NEXT:    br label %exit
 
@@ -3155,9 +3176,9 @@ body.b:
 ;
 ; CHECK:       [[LATCH_B]]:
 ; CHECK-NEXT:    %[[CMP2_B:.*]] = icmp slt i32 %[[TMP_C_SUM_B]], 42
-; CHECK:         br i1 %[[CMP2_B]], label %header.us2, label %exit.split.split.us, !llvm.loop !30
+; CHECK:         br i1 %[[CMP2_B]], label %[[HEADER_B]], label %[[LOOP_EXIT_B:.*]]
 ;
-; CHECK:       exit.split.split.us:
+; CHECK:       [[LOOP_EXIT_B]]:
 ; CHECK-NEXT:    %[[LCSSA_B:.*]] = phi i32 [ %[[TMP_C_SUM_B]], %[[LATCH_B]] ]
 ; CHECK-NEXT:    br label %[[EXIT_SPLIT:.*]]
 
@@ -3213,11 +3234,11 @@ exit:
   %lcssa.phi = phi i32 [ %tmp.c.sum, %latch ]
   ret i32 %lcssa.phi
 ; CHECK:       [[EXIT_SPLIT]]:
-; CHECK-NEXT:    %[[EXIT_PHI1:.*]] = phi i32 [ %[[LCSSA_C]], %[[LOOP_EXIT_C]] ], [ %[[LCSSA_B]], %exit.split.split.us ]
+; CHECK-NEXT:    %[[EXIT_PHI1:.*]] = phi i32 [ %[[LCSSA_C]], %[[LOOP_EXIT_C]] ], [ %[[LCSSA_B]], %[[LOOP_EXIT_B]] ]
 ; CHECK-NEXT:    br label %exit
 
 ; CHECK:       exit:
-; CHECK-NEXT:    %[[EXIT_PHI2:.*]] = phi i32 [ %[[EXIT_PHI1]], %[[EXIT_SPLIT]] ], [ %[[LCSSA_A]], %exit.split.us ]
+; CHECK-NEXT:    %[[EXIT_PHI2:.*]] = phi i32 [ %[[EXIT_PHI1]], %[[EXIT_SPLIT]] ], [ %[[LCSSA_A]], %[[LOOP_EXIT_A]] ]
 ; CHECK-NEXT:    ret i32 %[[EXIT_PHI2]]
 }
 
@@ -3283,9 +3304,9 @@ body.a:
 ;
 ; CHECK:       [[LATCH_A]]:
 ; CHECK-NEXT:    %[[CMP2_A:.*]] = icmp slt i32 %[[TMP_B_SUM_A]], 42
-; CHECK:         br i1 %[[CMP2_A]], label %header.us, label %loop.exit2.split.us, !llvm.loop !31
+; CHECK:         br i1 %[[CMP2_A]], label %[[HEADER_A]], label %[[LOOP_EXIT_A:.*]]
 ;
-; CHECK:       loop.exit2.split.us:
+; CHECK:       [[LOOP_EXIT_A]]:
 ; CHECK-NEXT:    %[[LCSSA_A:.*]] = phi i32 [ %[[TMP_B_SUM_A]], %[[LATCH_A]] ]
 ; CHECK-NEXT:    br label %loop.exit2
 
@@ -3321,9 +3342,9 @@ body.b:
 ;
 ; CHECK:       [[LATCH_B]]:
 ; CHECK-NEXT:    %[[CMP2_B:.*]] = icmp slt i32 %[[TMP_B_SUM_B]], 42
-; CHECK:         br i1 %[[CMP2_B]], label %header.us2, label %loop.exit2.split.split.us, !llvm.loop !32
+; CHECK:         br i1 %[[CMP2_B]], label %[[HEADER_B]], label %[[LOOP_EXIT_B:.*]]
 ;
-; CHECK:       loop.exit2.split.split.us:
+; CHECK:       [[LOOP_EXIT_B]]:
 ; CHECK-NEXT:    %[[LCSSA_B:.*]] = phi i32 [ %[[TMP_B_SUM_B]], %[[LATCH_B]] ]
 ; CHECK-NEXT:    br label %[[LOOP_EXIT2_SPLIT:.*]]
 
@@ -3376,11 +3397,11 @@ loop.exit2:
   %l2.phi = phi i32 [ %tmp.b.sum, %latch ]
   br label %exit
 ; CHECK:       [[LOOP_EXIT2_SPLIT]]:
-; CHECK-NEXT:    %[[LOOP_EXIT_PHI1:.*]] = phi i32 [ %[[L2_PHI]], %[[LOOP_EXIT_EXIT]] ], [ %[[LCSSA_B]], %loop.exit2.split.split.us ]
+; CHECK-NEXT:    %[[LOOP_EXIT_PHI1:.*]] = phi i32 [ %[[L2_PHI]], %[[LOOP_EXIT_EXIT]] ], [ %[[LCSSA_B]], %[[LOOP_EXIT_B]] ]
 ; CHECK-NEXT:    br label %loop.exit2
 ;
 ; CHECK:       loop.exit2:
-; CHECK-NEXT:    %[[LOOP_EXIT_PHI2:.*]] = phi i32 [ %[[LOOP_EXIT_PHI1]], %[[LOOP_EXIT2_SPLIT]] ], [ %[[LCSSA_A]], %loop.exit2.split.us ]
+; CHECK-NEXT:    %[[LOOP_EXIT_PHI2:.*]] = phi i32 [ %[[LOOP_EXIT_PHI1]], %[[LOOP_EXIT2_SPLIT]] ], [ %[[LCSSA_A]], %[[LOOP_EXIT_A]] ]
 ; CHECK-NEXT:    br label %exit
 
 exit:
@@ -4037,7 +4058,9 @@ entry:
 ; CHECK-NEXT:    ]
 ;
 ; CHECK:       [[ENTRY_SPLIT_US]]:
-; CHECK-NEXT:    br label %outer.header.us
+; CHECK-NEXT:    switch i32 %arg, label %[[ENTRY_SPLIT_US_SPLIT:.*]] [
+; CHECK-NEXT:      i32 1, label %[[ENTRY_SPLIT_US_SPLIT_US:.*]]
+; CHECK-NEXT:    ]
 
 outer.header:
   br label %inner.header
@@ -4051,13 +4074,66 @@ inner.header:
 inner.body1:
   %a = call i32 @a()
   br label %inner.latch
+; The (super convoluted) fully unswitched loop around `@a`.
+;
+; CHECK:       [[ENTRY_SPLIT_US_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_US_US:.*]]
+;
+; CHECK:       [[OUTER_HEADER_US_US]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_SPLIT_US_US:.*]]
+;
+; CHECK:       [[OUTER_LATCH_US_US:.*]]:
+; CHECK-NEXT:    %[[OUTER_COND_US_US:.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 %[[OUTER_COND_US_US]], label %[[OUTER_HEADER_US_US]], label %[[EXIT_SPLIT_US_SPLIT_US:.*]]
+;
+; CHECK:       [[OUTER_HEADER_SPLIT_US_US]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_SPLIT_SPLIT_US_US_US:.*]]
+;
+; CHECK:       [[INNER_LOOPEXIT2_US_US:.*]]:
+; CHECK-NEXT:    br label %[[OUTER_LATCH_US_US]]
+;
+; CHECK:       [[OUTER_HEADER_SPLIT_SPLIT_US_US_US]]:
+; CHECK-NEXT:    br label %[[INNER_HEADER_US_US_US:.*]]
+;
+; CHECK:       [[INNER_HEADER_US_US_US]]:
+; CHECK-NEXT:    br label %[[INNER_BODY1_US_US_US:.*]]
+;
+; CHECK:       [[INNER_BODY1_US_US_US]]:
+; CHECK-NEXT:    %[[A:.*]] = call i32 @a()
+; CHECK-NEXT:    br label %[[INNER_LATCH_US_US_US:.*]]
+;
+; CHECK:       [[INNER_LATCH_US_US_US]]:
+; CHECK-NEXT:    %[[PHI_A:.*]] = phi i32 [ %[[A]], %[[INNER_BODY1_US_US_US]] ]
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 0)
+; CHECK-NEXT:    call void @sink1(i32 %[[PHI_A]])
+; CHECK-NEXT:    %[[INNER_COND_US_US_US:.*]] = call i1 @cond()
+; CHECK-NEXT:    br i1 %[[INNER_COND_US_US_US]], label %[[INNER_HEADER_US_US_US]], label %[[INNER_LOOPEXIT2_SPLIT_US_US_US:.*]]
+;
+; CHECK:       [[INNER_LOOPEXIT2_SPLIT_US_US_US]]:
+; CHECK-NEXT:    br label %[[INNER_LOOPEXIT2_US_US]]
+;
+; CHECK:       [[EXIT_SPLIT_US_SPLIT_US]]:
+; CHECK-NEXT:    br label %[[EXIT_SPLIT_US:.*]]
+
 
 inner.body2:
   %b = call i32 @b()
   br label %inner.latch
 ; The fully unswitched loop around `@b`.
 ;
-; CHECK:       outer.header.us:
+; CHECK:       [[ENTRY_SPLIT_US_SPLIT]]:
+; CHECK-NEXT:    br label %[[OUTER_HEADER_US:.*]]
+;
+; CHECK:       [[OUTER_HEADER_US]]:
 ; CHECK-NEXT:    br label %[[OUTER_HEADER_SPLIT_US:.*]]
 ;
 ; CHECK:       [[INNER_HEADER_US:.*]]:
@@ -4087,51 +4163,18 @@ inner.body2:
 ;
 ; CHECK:       [[OUTER_LATCH_US:.*]]:
 ; CHECK-NEXT:    %[[OUTER_COND_US:.*]] = call i1 @cond()
-; CHECK-NEXT:    br i1 %[[OUTER_COND_US]], label %outer.header.us, label %exit.split.us, !llvm.loop !33
+; CHECK-NEXT:    br i1 %[[OUTER_COND_US]], label %[[OUTER_HEADER_US]], label %[[EXIT_SPLIT_US_SPLIT:.*]]
 ;
 ; CHECK:       [[OUTER_HEADER_SPLIT_US]]:
-; CHECK-NEXT:    switch i32 %arg, label %outer.header.split.split.us5 [
-; CHECK-NEXT:      i32 1, label %outer.header.split.split.us.us
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    br label %[[OUTER_HEADER_SPLIT_SPLIT_US:.*]]
 ;
-; CHECK:       outer.header.split.split.us5:
+; CHECK:       [[OUTER_HEADER_SPLIT_SPLIT_US]]:
 ; CHECK-NEXT:    br label %[[INNER_HEADER_US]]
 ;
 ; CHECK:       [[INNER_LOOPEXIT2_US]]:
 ; CHECK-NEXT:    br label %[[OUTER_LATCH_US]]
-
-; The (super convoluted) fully unswitched loop around `@a`.
-;
-; CHECK:       outer.header.split.split.us.us:
-; CHECK-NEXT:    br label %[[INNER_HEADER_US_US:.*]]
-;
-; CHECK:       [[INNER_HEADER_US_US]]:
-; CHECK-NEXT:    br label %[[INNER_BODY1_US_US:.*]]
-;
-; CHECK:       [[INNER_BODY1_US_US]]:
-; CHECK-NEXT:    %[[A:.*]] = call i32 @a()
-; CHECK-NEXT:    br label %[[INNER_LATCH_US_US:.*]]
-;
-; CHECK:       [[INNER_LATCH_US_US]]:
-; CHECK-NEXT:    %[[PHI_A:.*]] = phi i32 [ %[[A]], %[[INNER_BODY1_US_US]] ]
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 0)
-; CHECK-NEXT:    call void @sink1(i32 %[[PHI_A]])
-; CHECK-NEXT:    %[[INNER_COND_US_US:.*]] = call i1 @cond()
-; CHECK-NEXT:    br i1 %[[INNER_COND_US_US]], label %[[INNER_HEADER_US_US]], label %[[INNER_LOOPEXIT2_SPLIT_US_US:.*]], !llvm.loop !34
-;
-; CHECK:       [[INNER_LOOPEXIT2_SPLIT_US_US]]:
-; CHECK-NEXT:    br label %[[INNER_LOOPEXIT2_US]]
 ;
-; CHECK:       exit.split.us:
+; CHECK:       [[EXIT_SPLIT_US]]:
 ; CHECK-NEXT:    br label %exit
 
 inner.latch:
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/partial-unswitch-loop-and-block-dispositions.ll b/llvm/test/Transforms/SimpleLoopUnswitch/partial-unswitch-loop-and-block-dispositions.ll
index e821dfcd0124c..a169aa47ea7d5 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/partial-unswitch-loop-and-block-dispositions.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/partial-unswitch-loop-and-block-dispositions.ll
@@ -11,43 +11,59 @@ define void @test_pr58564(i16 %a, i1 %c.1, ptr %dst) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i16 [[A:%.*]], -6
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
+; CHECK-NEXT:    br i1 [[C_1:%.*]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]]
+; CHECK:       entry.split.us.split.us:
+; CHECK-NEXT:    br label [[LOOP_1_HEADER_US_US:%.*]]
+; CHECK:       loop.1.header.us.us:
+; CHECK-NEXT:    br label [[LOOP_1_HEADER_SPLIT_US_US_US:%.*]]
+; CHECK:       loop.1.header.split.us.us.us:
+; CHECK-NEXT:    br label [[LOOP_1_HEADER_SPLIT_US_SPLIT_US_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       loop.1.header.split.us.split.us.split.us.split.us:
+; CHECK-NEXT:    br label [[LOOP_1_HEADER_SPLIT_US_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       entry.split.us.split:
 ; CHECK-NEXT:    br label [[LOOP_1_HEADER_US:%.*]]
 ; CHECK:       loop.1.header.us:
 ; CHECK-NEXT:    br label [[LOOP_1_HEADER_SPLIT_US_US:%.*]]
-; CHECK:       loop.4.header.us2:
+; CHECK:       loop.4.header.us5:
 ; CHECK-NEXT:    br label [[LOOP_5_US6:%.*]]
-; CHECK:       loop.5.us3:
+; CHECK:       loop.5.us6:
 ; CHECK-NEXT:    [[IV_US7:%.*]] = phi i16 [ 0, [[LOOP_4_HEADER_US5:%.*]] ], [ [[IV_NEXT_US9:%.*]], [[LOOP_5_US6]] ]
 ; CHECK-NEXT:    [[GEP_US8:%.*]] = getelementptr inbounds ptr, ptr [[DST:%.*]], i16 [[IV_US7]]
 ; CHECK-NEXT:    store ptr null, ptr [[GEP_US8]], align 8
 ; CHECK-NEXT:    [[IV_NEXT_US9]] = add nuw nsw i16 [[IV_US7]], 1
 ; CHECK-NEXT:    [[EC_US10:%.*]] = icmp ne i16 [[IV_US7]], 10000
-; CHECK-NEXT:    br i1 [[EC_US10]], label [[LOOP_5_US6]], label [[LOOP_4_LATCH_US8:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK:       loop.4.latch.us8:
+; CHECK-NEXT:    br i1 [[EC_US10]], label [[LOOP_5_US6]], label [[LOOP_4_LATCH_US11:%.*]]
+; CHECK:       loop.4.latch.us11:
 ; CHECK-NEXT:    br label [[LOOP_1_LATCH_US:%.*]]
 ; CHECK:       loop.1.latch.us:
-; CHECK-NEXT:    br label [[LOOP_1_HEADER_US]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br label [[LOOP_1_HEADER_US]]
 ; CHECK:       loop.4.header.preheader.us:
-; CHECK-NEXT:    br i1 [[C_1:%.*]], label [[LOOP_4_HEADER_PREHEADER_SPLIT1_US_SPLIT_US:%.*]], label [[LOOP_4_HEADER_PREHEADER_SPLIT1_US9:%.*]]
+; CHECK-NEXT:    br i1 false, label [[LOOP_4_HEADER_PREHEADER_SPLIT4_US_SPLIT_US:%.*]], label [[LOOP_4_HEADER_PREHEADER_SPLIT4_US15:%.*]]
 ; CHECK:       loop.1.header.split.us.us:
 ; CHECK-NEXT:    br label [[LOOP_1_HEADER_SPLIT_US_SPLIT_US14:%.*]]
-; CHECK:       loop.2.header.us.us:
+; CHECK:       loop.2.header.us.us12:
 ; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_US_US13:%.*]]
 ; CHECK:       loop.2.latch.us.us:
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1_HEADER_SPLIT_US_SPLIT_US14]], label [[LOOP_4_HEADER_PREHEADER_SPLIT_US_US:%.*]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK:       loop.2.header.split.us.us.us:
+; CHECK-NEXT:    br i1 false, label [[LOOP_2_HEADER_US_US12:%.*]], label [[LOOP_4_HEADER_PREHEADER_SPLIT_US_US:%.*]]
+; CHECK:       loop.2.header.split.us.us.us13:
+; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_SPLIT_US3_US:%.*]]
+; CHECK:       loop.3.header.us.us1.us:
 ; CHECK-NEXT:    br label [[LOOP_3_LATCH_US_US2_US:%.*]]
-; CHECK:       loop.3.header.us.us.us:
+; CHECK:       loop.3.latch.us.us2.us:
 ; CHECK-NEXT:    br label [[LOOP_2_LATCH_SPLIT_US_US_US:%.*]]
-; CHECK:       loop.3.latch.us.us.us:
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_3_LATCH_US_US2_US]], label [[LOOP_2_LATCH_SPLIT_US_US_US1:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       loop.2.latch.split.us.us.us:
+; CHECK-NEXT:    br label [[LOOP_2_LATCH_US_US:%.*]]
+; CHECK:       loop.2.header.split.us.split.us3.us:
 ; CHECK-NEXT:    br label [[LOOP_3_HEADER_US_US1_US:%.*]]
 ; CHECK:       loop.4.header.preheader.split.us.us:
-; CHECK-NEXT:    br label [[LOOP_2_HEADER_US_US12:%.*]]
-; CHECK:       loop.4.header.preheader.split1.us9:
+; CHECK-NEXT:    br label [[LOOP_4_HEADER_PREHEADER_US:%.*]]
+; CHECK:       loop.1.header.split.us.split.us14:
+; CHECK-NEXT:    br label [[LOOP_2_HEADER_US_US12]]
+; CHECK:       loop.4.header.preheader.split4.us15:
 ; CHECK-NEXT:    br label [[LOOP_4_HEADER_US5]]
-; CHECK:       loop.4.header.preheader.split1.us.split.us:
+; CHECK:       loop.4.header.preheader.split4.us.split.us:
+; CHECK-NEXT:    br label [[LOOP_4_HEADER_PREHEADER_SPLIT4_US:%.*]]
+; CHECK:       loop.1.header.split.us.split.us.split.us:
 ; CHECK-NEXT:    br label [[LOOP_1_HEADER_SPLIT_US_SPLIT_US:%.*]]
 ; CHECK:       entry.split:
 ; CHECK-NEXT:    br label [[LOOP_1_HEADER:%.*]]
@@ -55,20 +71,36 @@ define void @test_pr58564(i16 %a, i1 %c.1, ptr %dst) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i16 [[A]], -6
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[LOOP_1_HEADER_SPLIT_US:%.*]], label [[LOOP_1_HEADER_SPLIT:%.*]]
 ; CHECK:       loop.1.header.split.us:
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_1_HEADER_SPLIT_US_SPLIT_US_SPLIT:%.*]], label [[LOOP_1_HEADER_SPLIT_US_SPLIT:%.*]]
+; CHECK:       loop.1.header.split.us.split.us.split:
+; CHECK-NEXT:    br label [[LOOP_1_HEADER_SPLIT_US_SPLIT_US]]
+; CHECK:       loop.1.header.split.us.split.us:
+; CHECK-NEXT:    br label [[LOOP_2_HEADER_US_US:%.*]]
+; CHECK:       loop.2.header.us.us:
+; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_US_US:%.*]]
+; CHECK:       loop.2.header.split.us.us.us:
+; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_SPLIT_US_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       loop.2.header.split.us.split.us.split.us.split.us:
+; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       loop.1.header.split.us.split:
 ; CHECK-NEXT:    br label [[LOOP_2_HEADER_US:%.*]]
 ; CHECK:       loop.2.header.us:
 ; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_US:%.*]]
 ; CHECK:       loop.2.latch.us:
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_2_HEADER_US]], label [[LOOP_4_HEADER_PREHEADER_SPLIT_US:%.*]], !llvm.loop [[LOOP3]]
+; CHECK-NEXT:    br i1 false, label [[LOOP_2_HEADER_US]], label [[LOOP_4_HEADER_PREHEADER_SPLIT_US:%.*]]
 ; CHECK:       loop.2.header.split.us.us:
+; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_SPLIT_US3:%.*]]
+; CHECK:       loop.3.header.us.us1:
 ; CHECK-NEXT:    br label [[LOOP_3_LATCH_US_US2:%.*]]
-; CHECK:       loop.3.header.us.us:
+; CHECK:       loop.3.latch.us.us2:
 ; CHECK-NEXT:    br label [[LOOP_2_LATCH_SPLIT_US_US:%.*]]
-; CHECK:       loop.3.latch.us.us:
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_3_LATCH_US_US2]], label [[LOOP_2_LATCH_SPLIT_US_US1:%.*]], !llvm.loop [[LOOP4]]
 ; CHECK:       loop.2.latch.split.us.us:
+; CHECK-NEXT:    br label [[LOOP_2_LATCH_US:%.*]]
+; CHECK:       loop.2.header.split.us.split.us3:
 ; CHECK-NEXT:    br label [[LOOP_3_HEADER_US_US1:%.*]]
 ; CHECK:       loop.4.header.preheader.split.us:
+; CHECK-NEXT:    br label [[LOOP_4_HEADER_PREHEADER:%.*]]
+; CHECK:       loop.2.header.split.us.split.us.split.us:
 ; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_SPLIT_US:%.*]]
 ; CHECK:       loop.1.header.split:
 ; CHECK-NEXT:    br label [[LOOP_2_HEADER:%.*]]
@@ -76,11 +108,21 @@ define void @test_pr58564(i16 %a, i1 %c.1, ptr %dst) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i16 [[A]], -6
 ; CHECK-NEXT:    br i1 [[TMP2]], label [[LOOP_2_HEADER_SPLIT_US:%.*]], label [[LOOP_2_HEADER_SPLIT:%.*]]
 ; CHECK:       loop.2.header.split.us:
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_2_HEADER_SPLIT_US_SPLIT_US_SPLIT:%.*]], label [[LOOP_2_HEADER_SPLIT_US_SPLIT:%.*]]
+; CHECK:       loop.2.header.split.us.split.us.split:
+; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_SPLIT_US]]
+; CHECK:       loop.2.header.split.us.split.us:
+; CHECK-NEXT:    br label [[LOOP_3_HEADER_US_US:%.*]]
+; CHECK:       loop.3.header.us.us:
+; CHECK-NEXT:    br label [[LOOP_3_LATCH_US_US:%.*]]
+; CHECK:       loop.3.latch.us.us:
+; CHECK-NEXT:    br label [[LOOP_3_HEADER_US_US]]
+; CHECK:       loop.2.header.split.us.split:
 ; CHECK-NEXT:    br label [[LOOP_3_HEADER_US:%.*]]
 ; CHECK:       loop.3.header.us:
 ; CHECK-NEXT:    br label [[LOOP_3_LATCH_US:%.*]]
 ; CHECK:       loop.3.latch.us:
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_3_HEADER_US]], label [[LOOP_2_LATCH_SPLIT_US:%.*]], !llvm.loop [[LOOP4]]
+; CHECK-NEXT:    br label [[LOOP_2_LATCH_SPLIT_US:%.*]]
 ; CHECK:       loop.2.latch.split.us:
 ; CHECK-NEXT:    br label [[LOOP_2_LATCH:%.*]]
 ; CHECK:       loop.2.header.split:
@@ -92,18 +134,18 @@ define void @test_pr58564(i16 %a, i1 %c.1, ptr %dst) {
 ; CHECK-NEXT:    call void @clobber()
 ; CHECK-NEXT:    br label [[LOOP_3_LATCH]]
 ; CHECK:       loop.3.latch:
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_3_HEADER]], label [[LOOP_2_LATCH_SPLIT:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_3_HEADER]], label [[LOOP_2_LATCH_SPLIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       loop.2.latch.split:
 ; CHECK-NEXT:    br label [[LOOP_2_LATCH]]
 ; CHECK:       loop.2.latch:
-; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_2_HEADER]], label [[LOOP_4_HEADER_PREHEADER_SPLIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_2_HEADER]], label [[LOOP_4_HEADER_PREHEADER_SPLIT:%.*]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       loop.4.header.preheader.split:
-; CHECK-NEXT:    br label [[LOOP_2_HEADER_SPLIT_US_SPLIT_US]]
+; CHECK-NEXT:    br label [[LOOP_4_HEADER_PREHEADER]]
 ; CHECK:       loop.4.header.preheader:
 ; CHECK-NEXT:    br i1 [[C_1]], label [[LOOP_4_HEADER_PREHEADER_SPLIT4_US_SPLIT:%.*]], label [[LOOP_4_HEADER_PREHEADER_SPLIT4:%.*]]
-; CHECK:       loop.4.header.preheader.split1.us.split:
-; CHECK-NEXT:    br label [[LOOP_1_HEADER_SPLIT_US_SPLIT_US]]
-; CHECK:       loop.4.header.preheader.split1.us:
+; CHECK:       loop.4.header.preheader.split4.us.split:
+; CHECK-NEXT:    br label [[LOOP_4_HEADER_PREHEADER_SPLIT4_US]]
+; CHECK:       loop.4.header.preheader.split4.us:
 ; CHECK-NEXT:    br label [[LOOP_4_HEADER_US:%.*]]
 ; CHECK:       loop.4.header.us:
 ; CHECK-NEXT:    br label [[LOOP_5_US:%.*]]
@@ -116,7 +158,7 @@ define void @test_pr58564(i16 %a, i1 %c.1, ptr %dst) {
 ; CHECK-NEXT:    br i1 [[EC_US]], label [[LOOP_5_US]], label [[LOOP_4_LATCH_US:%.*]]
 ; CHECK:       loop.4.latch.us:
 ; CHECK-NEXT:    br label [[LOOP_4_HEADER_US]]
-; CHECK:       loop.4.header.preheader.split1:
+; CHECK:       loop.4.header.preheader.split4:
 ; CHECK-NEXT:    br label [[LOOP_4_HEADER:%.*]]
 ; CHECK:       loop.4.header:
 ; CHECK-NEXT:    br label [[LOOP_5:%.*]]
@@ -126,11 +168,11 @@ define void @test_pr58564(i16 %a, i1 %c.1, ptr %dst) {
 ; CHECK-NEXT:    store ptr null, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i16 [[IV]], 1
 ; CHECK-NEXT:    [[EC:%.*]] = icmp ne i16 [[IV]], 10000
-; CHECK-NEXT:    br i1 [[EC]], label [[LOOP_5]], label [[LOOP_4_LATCH:%.*]], !llvm.loop [[LOOP0]]
+; CHECK-NEXT:    br i1 [[EC]], label [[LOOP_5]], label [[LOOP_4_LATCH:%.*]]
 ; CHECK:       loop.4.latch:
 ; CHECK-NEXT:    br label [[LOOP_1_LATCH:%.*]]
 ; CHECK:       loop.1.latch:
-; CHECK-NEXT:    br label [[LOOP_1_HEADER]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br label [[LOOP_1_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
 ;
 entry:
   br label %loop.1.header
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/partial-unswitch.ll b/llvm/test/Transforms/SimpleLoopUnswitch/partial-unswitch.ll
index 108b2406920f2..1d8942079ffd8 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/partial-unswitch.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/partial-unswitch.ll
@@ -19,7 +19,7 @@ define i32 @partial_unswitch_true_successor(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -37,7 +37,7 @@ define i32 @partial_unswitch_true_successor(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -84,7 +84,7 @@ define i32 @partial_unswitch_false_successor(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -102,7 +102,7 @@ define i32 @partial_unswitch_false_successor(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -151,7 +151,7 @@ define i32 @partial_unswtich_gep_load_icmp(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -171,7 +171,7 @@ define i32 @partial_unswtich_gep_load_icmp(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -223,7 +223,7 @@ define i32 @partial_unswitch_reduction_phi(ptr %ptr, i32 %N) {
 ; CHECK-NEXT:    [[RED_NEXT_US]] = phi i32 [ [[ADD_10_US]], [[NOCLOBBER_US]] ]
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    [[RED_NEXT_LCSSA_US:%.*]] = phi i32 [ [[RED_NEXT_US]], [[LOOP_LATCH_US]] ]
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
@@ -246,7 +246,7 @@ define i32 @partial_unswitch_reduction_phi(ptr %ptr, i32 %N) {
 ; CHECK-NEXT:    [[RED_NEXT]] = phi i32 [ [[ADD_5]], [[CLOBBER]] ], [ [[ADD_10]], [[NOCLOBBER]] ]
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    [[RED_NEXT_LCSSA:%.*]] = phi i32 [ [[RED_NEXT]], [[LOOP_LATCH]] ]
 ; CHECK-NEXT:    br label [[EXIT]]
@@ -305,7 +305,7 @@ define i32 @partial_unswitch_true_successor_noclobber(ptr noalias %ptr.1, ptr no
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -325,7 +325,7 @@ define i32 @partial_unswitch_true_successor_noclobber(ptr noalias %ptr.1, ptr no
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -619,7 +619,7 @@ define i32 @partial_unswitch_true_successor_preheader_insertion(ptr %ptr, i32 %N
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_LOOPEXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_LOOPEXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.loopexit.split.us:
 ; CHECK-NEXT:    br label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       loop.ph.split:
@@ -637,7 +637,7 @@ define i32 @partial_unswitch_true_successor_preheader_insertion(ptr %ptr, i32 %N
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_LOOPEXIT_SPLIT:%.*]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_LOOPEXIT_SPLIT:%.*]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       exit.loopexit.split:
 ; CHECK-NEXT:    br label [[EXIT_LOOPEXIT]]
 ; CHECK:       exit.loopexit:
@@ -695,7 +695,7 @@ define i32 @partial_unswitch_true_successor_insert_point(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -713,7 +713,7 @@ define i32 @partial_unswitch_true_successor_insert_point(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP7:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -765,7 +765,7 @@ define i32 @partial_unswitch_true_successor_hoist_invariant(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -784,7 +784,7 @@ define i32 @partial_unswitch_true_successor_hoist_invariant(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP17:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -1057,7 +1057,7 @@ define i32 @partial_unswitch_true_to_latch(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP18:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -1073,7 +1073,7 @@ define i32 @partial_unswitch_true_to_latch(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP19:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP9:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -1112,11 +1112,19 @@ define i32 @partial_unswitch_exiting_block_with_multiple_unswitch_candidates(i32
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 41
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[ENTRY_SPLIT:%.*]], label [[ENTRY_SPLIT_US:%.*]]
 ; CHECK:       entry.split.us:
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]]
+; CHECK:       entry.split.us.split.us:
+; CHECK-NEXT:    br label [[LOOP_US_US:%.*]]
+; CHECK:       loop.us.us:
+; CHECK-NEXT:    br label [[EXITING_US_US:%.*]]
+; CHECK:       exiting.us.us:
+; CHECK-NEXT:    br label [[LOOP_US_US]]
+; CHECK:       entry.split.us.split:
 ; CHECK-NEXT:    br label [[LOOP_US:%.*]]
 ; CHECK:       loop.us:
 ; CHECK-NEXT:    br label [[EXITING_US:%.*]]
 ; CHECK:       exiting.us:
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label [[LOOP_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP20:![0-9]+]]
+; CHECK-NEXT:    br label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    [[RET_VAL_US:%.*]] = phi i32 [ 1, [[EXITING_US]] ]
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
@@ -1130,7 +1138,7 @@ define i32 @partial_unswitch_exiting_block_with_multiple_unswitch_candidates(i32
 ; CHECK-NEXT:    store i32 [[TMP1:%.*]], ptr [[PTR]], align 16
 ; CHECK-NEXT:    br label [[EXITING]]
 ; CHECK:       exiting:
-; CHECK-NEXT:    br i1 [[EXIT_COND]], label [[LOOP]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP21:![0-9]+]]
+; CHECK-NEXT:    br i1 [[EXIT_COND]], label [[LOOP]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    [[RET_VAL:%.*]] = phi i32 [ 1, [[EXITING]] ]
 ; CHECK-NEXT:    br label [[EXIT]]
@@ -1177,7 +1185,7 @@ define i32 @partial_unswitch_true_successor_for_cost_calculation(ptr %ptr, i32 %
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP22:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -1241,7 +1249,7 @@ define i32 @partial_unswitch_true_successor_for_cost_calculation(ptr %ptr, i32 %
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP23:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -1334,7 +1342,7 @@ define i32 @partial_unswitch_true_successor_trunc(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP24:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -1352,7 +1360,7 @@ define i32 @partial_unswitch_true_successor_trunc(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP25:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -1399,7 +1407,7 @@ define i32 @partial_unswitch_false_successor_trunc(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch.us:
 ; CHECK-NEXT:    [[C_US:%.*]] = icmp ult i32 [[IV_US]], [[N:%.*]]
 ; CHECK-NEXT:    [[IV_NEXT_US]] = add i32 [[IV_US]], 1
-; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]], !llvm.loop [[LOOP26:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C_US]], label [[LOOP_HEADER_US]], label [[EXIT_SPLIT_US:%.*]]
 ; CHECK:       exit.split.us:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
 ; CHECK:       entry.split:
@@ -1417,7 +1425,7 @@ define i32 @partial_unswitch_false_successor_trunc(ptr %ptr, i32 %N) {
 ; CHECK:       loop.latch:
 ; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[IV]], [[N]]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
-; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP27:![0-9]+]]
+; CHECK-NEXT:    br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT_SPLIT:%.*]], !llvm.loop [[LOOP13:![0-9]+]]
 ; CHECK:       exit.split:
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
@@ -1448,15 +1456,15 @@ exit:
   ret i32 10
 }
 
-; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[UNSWITCH_PARTIAL_DISABLE:![0-9]+]]}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[UNSWITCH_PARTIAL_DISABLE:![0-9]+]]}
 ; CHECK: [[UNSWITCH_PARTIAL_DISABLE]] = !{!"llvm.loop.unswitch.partial.disable"}
+; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[UNSWITCH_PARTIAL_DISABLE]]}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[UNSWITCH_PARTIAL_DISABLE]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[UNSWITCH_PARTIAL_DISABLE]]}
 ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[UNSWITCH_PARTIAL_DISABLE]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[UNSWITCH_PARTIAL_DISABLE]]}
 ; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[UNSWITCH_PARTIAL_DISABLE]]}
+; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[UNSWITCH_PARTIAL_DISABLE]]}
 ; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[UNSWITCH_PARTIAL_DISABLE]]}
+; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[UNSWITCH_PARTIAL_DISABLE]]}
 ; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[UNSWITCH_PARTIAL_DISABLE]]}
-; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[UNSWITCH_PARTIAL_DISABLE]]}
-; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[UNSWITCH_PARTIAL_DISABLE]]}
-; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[UNSWITCH_PARTIAL_DISABLE]]}
-; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[UNSWITCH_PARTIAL_DISABLE]]}
-; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[UNSWITCH_PARTIAL_DISABLE]]}
-; CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[UNSWITCH_PARTIAL_DISABLE]]}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/pr138509.ll b/llvm/test/Transforms/SimpleLoopUnswitch/pr138509.ll
deleted file mode 100644
index e24d17f088427..0000000000000
--- a/llvm/test/Transforms/SimpleLoopUnswitch/pr138509.ll
+++ /dev/null
@@ -1,49 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -passes="loop-mssa(loop-simplifycfg,licm,loop-rotate,simple-loop-unswitch<nontrivial>)" < %s | FileCheck %s
-
-@a = global i32 0, align 4
-@b = global i32 0, align 4
-@c = global i32 0, align 4
-@d = global i32 0, align 4
-
-define i32 @main() {
-entry:
-  br label %outer.loop.header
-
-outer.loop.header:                                ; preds = %outer.loop.latch, %entry
-  br i1 false, label %exit, label %outer.loop.body
-
-outer.loop.body:                                  ; preds = %inner.loop.header, %outer.loop.header
-  store i32 1, ptr @c, align 4
-  %cmp = icmp sgt i32 0, -1
-  br i1 %cmp, label %outer.loop.latch, label %exit
-
-inner.loop.header:                                ; preds = %outer.loop.latch, %inner.loop.body
-  %a_val = load i32, ptr @a, align 4
-  %c_val = load i32, ptr @c, align 4
-  %mul = mul nsw i32 %c_val, %a_val
-  store i32 %mul, ptr @b, align 4
-  %cmp2 = icmp sgt i32 %mul, -1
-  br i1 %cmp2, label %inner.loop.body, label %outer.loop.body
-
-inner.loop.body:                                  ; preds = %inner.loop.header
-  %mul2 = mul nsw i32 %c_val, 3
-  store i32 %mul2, ptr @c, align 4
-  store i32 %c_val, ptr @d, align 4
-  %mul3 = mul nsw i32 %c_val, %a_val
-  %cmp3 = icmp sgt i32 %mul3, -1
-  br i1 %cmp3, label %inner.loop.header, label %exit
-
-outer.loop.latch:                                 ; preds = %outer.loop.body
-  %d_val = load i32, ptr @d, align 4
-  store i32 %d_val, ptr @b, align 4
-  %cmp4 = icmp eq i32 %d_val, 0
-  br i1 %cmp4, label %inner.loop.header, label %outer.loop.header
-
-exit:                                             ; preds = %inner.loop.body, %outer.loop.body, %outer.loop.header
-  ret i32 0
-}
-
-; CHECK: [[LOOP0:.*]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]}
-; CHECK: [[META1]] = !{!"llvm.loop.unswitch.nontrivial.disable"}
-; CHECK: [[LOOP2:.*]] = distinct !{[[LOOP2]], [[META1]]}
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/update-scev-3.ll b/llvm/test/Transforms/SimpleLoopUnswitch/update-scev-3.ll
index 4e428cbc30bb6..ef00d7ea8f2bb 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/update-scev-3.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/update-scev-3.ll
@@ -19,42 +19,56 @@ define i32 @foo(i1 %not) {
 ; CHECK-NEXT:    [[FALSE:%.*]] = and i1 true, false
 ; CHECK-NEXT:    br i1 [[NOT]], label [[ENTRY_SPLIT_US:%.*]], label [[ENTRY_SPLIT:%.*]]
 ; CHECK:       entry.split.us:
+; CHECK-NEXT:    br i1 [[FALSE]], label [[ENTRY_SPLIT_US_SPLIT_US:%.*]], label [[ENTRY_SPLIT_US_SPLIT:%.*]]
+; CHECK:       entry.split.us.split.us:
+; CHECK-NEXT:    br label [[FOR_COND_US_US:%.*]]
+; CHECK:       for.cond.us.us:
+; CHECK-NEXT:    br label [[FOR_COND_SPLIT_US_US_US:%.*]]
+; CHECK:       for.cond.split.us.us.us:
+; CHECK-NEXT:    br label [[FOR_COND_SPLIT_US_SPLIT_US_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       for.cond.split.us.split.us.split.us.split.us:
+; CHECK-NEXT:    br label [[FOR_COND_SPLIT_US_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       entry.split.us.split:
 ; CHECK-NEXT:    br label [[FOR_COND_US:%.*]]
 ; CHECK:       for.cond.us:
 ; CHECK-NEXT:    br label [[FOR_COND_SPLIT_US_US:%.*]]
 ; CHECK:       for.inc11.us:
-; CHECK-NEXT:    br label [[FOR_COND_US]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br label [[FOR_COND_US]]
 ; CHECK:       for.cond.split.us.us:
-; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_US_US:%.*]]
-; CHECK:       for.cond5.preheader.us.us:
-; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_US_US_US:%.*]]
+; CHECK-NEXT:    br label [[FOR_COND_SPLIT_US_SPLIT_US11:%.*]]
+; CHECK:       for.cond5.preheader.us.us9:
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_US_US_US10:%.*]]
 ; CHECK:       for.inc8.us.us:
-; CHECK-NEXT:    br i1 [[FALSE]], label [[FOR_INC8_FOR_COND5_PREHEADER_CRIT_EDGE_US_US:%.*]], label [[FOR_INC11_SPLIT_US_US:%.*]]
+; CHECK-NEXT:    br i1 false, label [[FOR_INC8_FOR_COND5_PREHEADER_CRIT_EDGE_US_US:%.*]], label [[FOR_INC11_SPLIT_US_US:%.*]]
 ; CHECK:       for.inc8.for.cond5.preheader_crit_edge.us.us:
-; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_US_US]], !llvm.loop [[LOOP2:![0-9]+]]
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_US_US9:%.*]]
 ; CHECK:       for.end.us.us:
-; CHECK-NEXT:    br i1 [[FALSE]], label [[FOR_INC8_US_US:%.*]], label [[CLEANUP15_SPLIT_US_SPLIT_US:%.*]]
-; CHECK:       for.cond5.preheader.split.us.us.us:
-; CHECK-NEXT:    br label [[FOR_BODY7_US_US_US:%.*]]
-; CHECK:       for.body7.us.us.us:
-; CHECK-NEXT:    br label [[HANDLER_POINTER_OVERFLOW_US_US_US:%.*]]
-; CHECK:       handler.pointer_overflow.us.us.us:
-; CHECK-NEXT:    br label [[CONT_US_US_US:%.*]]
-; CHECK:       cont.us.us.us:
-; CHECK-NEXT:    br i1 [[FALSE]], label [[CONT_FOR_BODY7_CRIT_EDGE_US_US_US:%.*]], label [[FOR_END_SPLIT_US_US_US:%.*]]
-; CHECK:       cont.for.body7_crit_edge.us.us.us:
-; CHECK-NEXT:    br label [[FOR_BODY7_US_US_US]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-NEXT:    br i1 false, label [[FOR_INC8_US_US:%.*]], label [[CLEANUP15_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       for.cond5.preheader.split.us.us.us10:
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_US_SPLIT_US7_US:%.*]]
+; CHECK:       for.body7.us.us4.us:
+; CHECK-NEXT:    br label [[HANDLER_POINTER_OVERFLOW_US_US5_US:%.*]]
+; CHECK:       handler.pointer_overflow.us.us5.us:
+; CHECK-NEXT:    br label [[CONT_US_US6_US:%.*]]
+; CHECK:       cont.us.us6.us:
+; CHECK-NEXT:    br label [[FOR_END_SPLIT_US_US_US:%.*]]
 ; CHECK:       for.end.split.us.us.us:
 ; CHECK-NEXT:    br label [[FOR_END_US_US:%.*]]
+; CHECK:       for.cond5.preheader.split.us.split.us7.us:
+; CHECK-NEXT:    br label [[FOR_BODY7_US_US4_US:%.*]]
 ; CHECK:       for.inc11.split.us.us:
 ; CHECK-NEXT:    br label [[FOR_INC11_US:%.*]]
+; CHECK:       for.cond.split.us.split.us11:
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_US_US9]]
+; CHECK:       for.cond.split.us.split.us.split.us:
+; CHECK-NEXT:    br label [[FOR_COND_SPLIT_US_SPLIT_US:%.*]]
 ; CHECK:       cleanup15.split.us.split.us:
 ; CHECK-NEXT:    br label [[CLEANUP15_SPLIT_US:%.*]]
 ; CHECK:       entry.split:
 ; CHECK-NEXT:    br i1 [[FALSE]], label [[ENTRY_SPLIT_SPLIT_US:%.*]], label [[ENTRY_SPLIT_SPLIT:%.*]]
 ; CHECK:       entry.split.split.us:
-; CHECK-NEXT:    br label [[FOR_COND_US5:%.*]]
-; CHECK:       for.cond.us5:
+; CHECK-NEXT:    br label [[FOR_COND_US12:%.*]]
+; CHECK:       for.cond.us12:
 ; CHECK-NEXT:    br label [[FOR_COND_SPLIT_US:%.*]]
 ; CHECK:       for.cond.split.us:
 ; CHECK-NEXT:    br label [[FOR_COND_SPLIT_SPLIT_US_SPLIT_US:%.*]]
@@ -64,13 +78,23 @@ define i32 @foo(i1 %not) {
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]]
 ; CHECK:       for.cond:
 ; CHECK-NEXT:    br label [[FOR_COND_SPLIT:%.*]]
+; CHECK:       for.cond.split.us.split.us:
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_US_US:%.*]]
+; CHECK:       for.cond5.preheader.us.us:
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_US_US_US:%.*]]
+; CHECK:       for.cond5.preheader.split.us.us.us:
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_US_SPLIT_US_SPLIT_US_SPLIT_US:%.*]]
+; CHECK:       for.cond5.preheader.split.us.split.us.split.us.split.us:
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_US_SPLIT_US_SPLIT_US:%.*]]
 ; CHECK:       cleanup15.split.us:
 ; CHECK-NEXT:    br label [[CLEANUP15:%.*]]
+; CHECK:       for.cond5.preheader.split.us.split.us.split.us:
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_US_SPLIT_US:%.*]]
 ; CHECK:       for.cond.split:
 ; CHECK-NEXT:    br label [[FOR_COND_SPLIT_SPLIT:%.*]]
 ; CHECK:       for.cond.split.split.us:
-; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_US4:%.*]]
-; CHECK:       for.cond5.preheader.us4:
+; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_US8:%.*]]
+; CHECK:       for.cond5.preheader.us8:
 ; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_US:%.*]]
 ; CHECK:       for.cond5.preheader.split.us:
 ; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_SPLIT_US_SPLIT_US:%.*]]
@@ -80,6 +104,16 @@ define i32 @foo(i1 %not) {
 ; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER:%.*]]
 ; CHECK:       for.cond5.preheader:
 ; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT:%.*]]
+; CHECK:       for.cond5.preheader.split.us.split.us:
+; CHECK-NEXT:    br label [[FOR_BODY7_US_US:%.*]]
+; CHECK:       for.body7.us.us:
+; CHECK-NEXT:    br label [[HANDLER_POINTER_OVERFLOW_US_US:%.*]]
+; CHECK:       handler.pointer_overflow.us.us:
+; CHECK-NEXT:    br label [[CONT_US_US:%.*]]
+; CHECK:       cont.us.us:
+; CHECK-NEXT:    br label [[CONT_FOR_BODY7_CRIT_EDGE_US_US:%.*]]
+; CHECK:       cont.for.body7_crit_edge.us.us:
+; CHECK-NEXT:    br label [[FOR_BODY7_US_US]]
 ; CHECK:       for.cond5.preheader.split:
 ; CHECK-NEXT:    br label [[FOR_COND5_PREHEADER_SPLIT_SPLIT:%.*]]
 ; CHECK:       for.cond5.preheader.split.split.us:
diff --git a/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll b/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll
index 2e9e7b19c73e2..44d92e1a1c210 100644
--- a/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll
+++ b/llvm/test/Transforms/SimplifyCFG/2008-07-13-InfLoopMiscompile.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
+; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -keep-loops="false" -S | FileCheck %s
 ; PR2540
 ; Outval should end up with a select from 0/2, not all constants.
 
@@ -52,4 +52,3 @@ func_1.exit:		; preds = %cowblock, %entry
 }
 
 declare i32 @printf(ptr, ...) nounwind
-
diff --git a/llvm/test/Transforms/SimplifyCFG/2025-07-29-non-canoncial-loop.ll b/llvm/test/Transforms/SimplifyCFG/2025-07-29-non-canoncial-loop.ll
new file mode 100644
index 0000000000000..322dd98f48df1
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/2025-07-29-non-canoncial-loop.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 --keep-loops="true" -S | FileCheck --check-prefix=NO-THREADING %s
+; Checks that we do not thread the control flow through the loop header loop_header as
+; that will introduce a non-canonical loop.
+
+; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 --keep-loops="false" -S | FileCheck --check-prefix=THREADING %s
+; Checks that we thread the control flow through the loop header loop_header since we
+; do not request --keep-loops.
+
+define void @__start(i1 %cond) {
+; NO-THREADING-LABEL: define void @__start(
+; NO-THREADING-SAME: i1 [[COND:%.*]]) {
+; NO-THREADING-NEXT:  [[ENTRY:.*:]]
+; NO-THREADING-NEXT:    br label %[[LOOP_HEADER:.*]]
+; NO-THREADING:       [[LOOP_HEADER]]:
+; NO-THREADING-NEXT:    br i1 [[COND]], label %[[LOOP_BODY_1:.*]], label %[[LOOP_BODY_0:.*]]
+; NO-THREADING:       [[LOOP_BODY_0]]:
+; NO-THREADING-NEXT:    [[_0_:%.*]] = add i16 0, 0
+; NO-THREADING-NEXT:    br label %[[LOOP_EXIT:.*]]
+; NO-THREADING:       [[LOOP_BODY_1]]:
+; NO-THREADING-NEXT:    [[_1_:%.*]] = add i32 0, 1
+; NO-THREADING-NEXT:    br label %[[LOOP_EXIT]]
+; NO-THREADING:       [[LOOP_EXIT]]:
+; NO-THREADING-NEXT:    br i1 [[COND]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
+; NO-THREADING:       [[EXIT]]:
+; NO-THREADING-NEXT:    ret void
+;
+; THREADING-LABEL: define void @__start(
+; THREADING-SAME: i1 [[COND:%.*]]) {
+; THREADING-NEXT:  [[ENTRY:.*:]]
+; THREADING-NEXT:    br i1 [[COND]], label %[[LOOP_BODY_1:.*]], label %[[LOOP_BODY_0:.*]]
+; THREADING:       [[LOOP_BODY_0]]:
+; THREADING-NEXT:    [[_0_:%.*]] = add i16 0, 0
+; THREADING-NEXT:    br label %[[LOOP_EXIT:.*]]
+; THREADING:       [[LOOP_BODY_1]]:
+; THREADING-NEXT:    [[_1_:%.*]] = add i32 0, 1
+; THREADING-NEXT:    br label %[[LOOP_EXIT]]
+; THREADING:       [[LOOP_EXIT]]:
+; THREADING-NEXT:    br i1 [[COND]], label %[[LOOP_BODY_1]], label %[[EXIT:.*]]
+; THREADING:       [[EXIT]]:
+; THREADING-NEXT:    ret void
+;
+entry:
+  br label %loop_header
+
+loop_header:                                            ; preds = %loop_exit, %entry
+  br i1 %cond, label %loop_body_1, label %loop_body_0
+
+loop_body_0:                                            ; preds = %loop_header
+  %_0_ = add i16 0, 0
+  br label %loop_exit
+
+loop_body_1:                                            ; preds = %loop_header
+  %_1_ = add i32 0, 1
+  br label %loop_exit
+
+loop_exit:                                            ; preds = %loop_body_1, %loop_body_0
+  br i1 %cond, label %loop_header, label %exit
+
+exit:                                            ; preds = %loop_exit
+  ret void
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/critedge-assume.ll b/llvm/test/Transforms/SimplifyCFG/X86/critedge-assume.ll
index 58ca8df8ff6d6..99e908e01bf8d 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/critedge-assume.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/critedge-assume.ll
@@ -56,7 +56,7 @@ while.end:
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 declare i32 @_ZNK1F5beginEv(ptr)
 
@@ -68,7 +68,7 @@ declare noalias nonnull ptr @_Znwm(i64)
 
 declare void @_ZN1B6appendEv(ptr)
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare i1 @llvm.type.test(ptr, metadata)
 
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll b/llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll
index 162a3ab4ee003..1499eecbebca1 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/empty-cleanuppad.ll
@@ -437,7 +437,7 @@ define i32 @f9() personality ptr @__CxxFrameHandler3 {
 ; CHECK-LABEL: @f9(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[S:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[S]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[S]])
 ; CHECK-NEXT:    invoke void @"\01??1S2@@QEAA@XZ"(ptr [[S]])
 ; CHECK-NEXT:            to label [[TRY_CONT:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
 ; CHECK:       catch.dispatch:
@@ -450,13 +450,13 @@ define i32 @f9() personality ptr @__CxxFrameHandler3 {
 ;
 entry:
   %s = alloca i8, align 1
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %s)
+  call void @llvm.lifetime.start.p0(ptr nonnull %s)
   invoke void @"\01??1S2@@QEAA@XZ"(ptr %s)
   to label %try.cont unwind label %ehcleanup
 
 ehcleanup:
   %cleanup.pad = cleanuppad within none []
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %s)
+  call void @llvm.lifetime.end.p0(ptr nonnull %s)
   cleanupret from %cleanup.pad unwind label %catch.dispatch
 
 catch.dispatch:
@@ -534,7 +534,7 @@ invoke.cont2:                                     ; preds = %invoke.cont
 
 ehcleanup:                                        ; preds = %invoke.cont, %entry
   %0 = cleanuppad within none []
-  call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %x)
+  call void @llvm.lifetime.end.p0(ptr nonnull %x)
   cleanupret from %0 unwind label %catch.dispatch
 
 catch.dispatch:                                   ; preds = %ehcleanup, %invoke.cont
@@ -556,8 +556,8 @@ declare void @use_x(i32 %x)
 
 declare i32 @__CxxFrameHandler3(...)
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 ;.
 ; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
 ;.
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/invalidate-dom.ll b/llvm/test/Transforms/SimplifyCFG/X86/invalidate-dom.ll
index a937d9c65017a..ce58e9365db20 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/invalidate-dom.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/invalidate-dom.ll
@@ -79,10 +79,10 @@ for.body:                                         ; preds = %for.cond
 declare i32 @c(...) #0
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 attributes #0 = { "use-soft-float"="false" }
 attributes #1 = { "target-cpu"="x86-64" }
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll
index 62351d766e749..6129e3b957e17 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll
@@ -1338,10 +1338,10 @@ define i32 @test_not_sink_lifetime_marker(i1 zeroext %flag, i32 %x) {
 ; CHECK-NEXT:    [[Z:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    br i1 [[FLAG:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[Y]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[Y]])
 ; CHECK-NEXT:    br label [[IF_END:%.*]]
 ; CHECK:       if.else:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[Z]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[Z]])
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    ret i32 1
@@ -1352,11 +1352,11 @@ entry:
   br i1 %flag, label %if.then, label %if.else
 
 if.then:
-  call void @llvm.lifetime.end.p0(i64 4, ptr %y)
+  call void @llvm.lifetime.end.p0(ptr %y)
   br label %if.end
 
 if.else:
-  call void @llvm.lifetime.end.p0(i64 4, ptr %z)
+  call void @llvm.lifetime.end.p0(ptr %z)
   br label %if.end
 
 if.end:
@@ -1468,8 +1468,8 @@ declare void @direct_callee()
 declare void @direct_callee2()
 declare void @direct_callee3()
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @creating_too_many_phis(i1 %cond, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h) {
 ; CHECK-LABEL: @creating_too_many_phis(
diff --git a/llvm/test/Transforms/SimplifyCFG/branch-phi-thread.ll b/llvm/test/Transforms/SimplifyCFG/branch-phi-thread.ll
index 0afec05ecbd6a..ec9423bd81675 100644
--- a/llvm/test/Transforms/SimplifyCFG/branch-phi-thread.ll
+++ b/llvm/test/Transforms/SimplifyCFG/branch-phi-thread.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=simplifycfg,adce -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
+; RUN: opt < %s -passes=simplifycfg,adce -simplifycfg-require-and-preserve-domtree=1 -keep-loops="false" -S | FileCheck %s
 
 declare void @f1()
 
diff --git a/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll b/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll
index a4303995d2633..307501db02b87 100644
--- a/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll
+++ b/llvm/test/Transforms/SimplifyCFG/common-code-hoisting.ll
@@ -39,8 +39,8 @@ declare void @f0()
 declare void @f1()
 declare void @f2()
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 define void @_Z4loopi(i1 %cmp) {
 ; HOIST-LABEL: @_Z4loopi(
diff --git a/llvm/test/Transforms/SimplifyCFG/invoke_unwind_lifetime.ll b/llvm/test/Transforms/SimplifyCFG/invoke_unwind_lifetime.ll
index ea14b170a51e8..40e9a4984e312 100644
--- a/llvm/test/Transforms/SimplifyCFG/invoke_unwind_lifetime.ll
+++ b/llvm/test/Transforms/SimplifyCFG/invoke_unwind_lifetime.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
 
-declare void @llvm.lifetime.start.p0(i64, ptr)
-declare void @llvm.lifetime.end.p0(i64, ptr)
+declare void @llvm.lifetime.start.p0(ptr)
+declare void @llvm.lifetime.end.p0(ptr)
 
 declare void @escape(ptr)
 
@@ -15,16 +15,16 @@ define void @caller(i1 %c) personality ptr @__gxx_personality_v0 {
 ; CHECK-LABEL: @caller(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[I0:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[I0]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[I0]])
 ; CHECK-NEXT:    call void @escape(ptr [[I0]])
 ; CHECK-NEXT:    [[I2:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[I2]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[I2]])
 ; CHECK-NEXT:    call void @escape(ptr [[I2]])
 ; CHECK-NEXT:    [[I4:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[I4]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[I4]])
 ; CHECK-NEXT:    call void @escape(ptr [[I4]])
 ; CHECK-NEXT:    [[I6:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[I6]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[I6]])
 ; CHECK-NEXT:    call void @escape(ptr [[I6]])
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[V0:%.*]], label [[V1:%.*]]
 ; CHECK:       v0:
@@ -36,19 +36,19 @@ define void @caller(i1 %c) personality ptr @__gxx_personality_v0 {
 ;
 entry:
   %i0 = alloca i32
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %i0)
+  call void @llvm.lifetime.start.p0(ptr nonnull %i0)
   call void @escape(ptr %i0)
 
   %i2 = alloca i32
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %i2)
+  call void @llvm.lifetime.start.p0(ptr nonnull %i2)
   call void @escape(ptr %i2)
 
   %i4 = alloca i32
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %i4)
+  call void @llvm.lifetime.start.p0(ptr nonnull %i4)
   call void @escape(ptr %i4)
 
   %i6 = alloca i32
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %i6)
+  call void @llvm.lifetime.start.p0(ptr nonnull %i6)
   call void @escape(ptr %i6)
 
   br i1 %c, label %v0, label %v1
@@ -66,14 +66,14 @@ invoke.cont:
 
 lpad.v0:
   %i8 = landingpad { ptr, i32 } cleanup
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %i0)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %i4)
+  call void @llvm.lifetime.end.p0(ptr nonnull %i0)
+  call void @llvm.lifetime.end.p0(ptr nonnull %i4)
   br label %end
 
 lpad.v1:
   %i9 = landingpad { ptr, i32 } cleanup
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %i2)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %i6)
+  call void @llvm.lifetime.end.p0(ptr nonnull %i2)
+  call void @llvm.lifetime.end.p0(ptr nonnull %i6)
   br label %end
 
 end:
diff --git a/llvm/test/Transforms/SimplifyCFG/jump-threading.ll b/llvm/test/Transforms/SimplifyCFG/jump-threading.ll
index 50a32413a0551..a4073ae6eb0b4 100644
--- a/llvm/test/Transforms/SimplifyCFG/jump-threading.ll
+++ b/llvm/test/Transforms/SimplifyCFG/jump-threading.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -passes=simplifycfg < %s | FileCheck %s
+; RUN: opt -S -passes=simplifycfg -keep-loops="false" < %s | FileCheck %s
 
 declare void @foo()
 declare void @bar()
diff --git a/llvm/test/Transforms/SimplifyCFG/lifetime-landingpad.ll b/llvm/test/Transforms/SimplifyCFG/lifetime-landingpad.ll
index 0174eb149581c..88395a0600101 100644
--- a/llvm/test/Transforms/SimplifyCFG/lifetime-landingpad.ll
+++ b/llvm/test/Transforms/SimplifyCFG/lifetime-landingpad.ll
@@ -5,32 +5,32 @@ define void @foo() personality ptr @__gxx_personality_v0 {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A:%.*]] = alloca i8, align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[A]]) #[[ATTR1:[0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[A]]) #[[ATTR1:[0-9]+]]
 ; CHECK-NEXT:    call void @bar()
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[A]]) #[[ATTR1]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[A]]) #[[ATTR1]]
 ; CHECK-NEXT:    ret void
 ;
 entry:
   %a = alloca i8
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %a) nounwind
+  call void @llvm.lifetime.start.p0(ptr nonnull %a) nounwind
   invoke void @bar() to label %invoke.cont unwind label %lpad
 
 invoke.cont:
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %a) nounwind
+  call void @llvm.lifetime.end.p0(ptr nonnull %a) nounwind
   ret void
 
 lpad:
   %b = landingpad { ptr, i32 }
   cleanup
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %a) nounwind
+  call void @llvm.lifetime.end.p0(ptr nonnull %a) nounwind
   resume { ptr, i32 } %b
 }
 
 declare void @bar()
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
 
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
 
 declare i32 @__gxx_personality_v0(...)
 ;.
diff --git a/llvm/test/Transforms/SimplifyCFG/lifetime.ll b/llvm/test/Transforms/SimplifyCFG/lifetime.ll
index d6bba2c3c2e9a..fac0b616e44f2 100644
--- a/llvm/test/Transforms/SimplifyCFG/lifetime.ll
+++ b/llvm/test/Transforms/SimplifyCFG/lifetime.ll
@@ -10,11 +10,11 @@
 define void @foo(i1 %x) {
 entry:
   %a = alloca i8
-  call void @llvm.lifetime.start.p0(i64 -1, ptr %a) nounwind
+  call void @llvm.lifetime.start.p0(ptr %a) nounwind
   br i1 %x, label %bb0, label %bb1
 
 bb0:
-  call void @llvm.lifetime.end.p0(i64 -1, ptr %a) nounwind
+  call void @llvm.lifetime.end.p0(ptr %a) nounwind
   br label %bb1
 
 bb1:
@@ -24,6 +24,6 @@ bb1:
 
 declare void @f()
 
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.start.p0(ptr nocapture) nounwind
 
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture) nounwind
+declare void @llvm.lifetime.end.p0(ptr nocapture) nounwind
diff --git a/llvm/test/Transforms/SimplifyCFG/pr50060-constantfold-loopid.ll b/llvm/test/Transforms/SimplifyCFG/pr50060-constantfold-loopid.ll
index 55f1c014c476e..19e1c73cfcba2 100644
--- a/llvm/test/Transforms/SimplifyCFG/pr50060-constantfold-loopid.ll
+++ b/llvm/test/Transforms/SimplifyCFG/pr50060-constantfold-loopid.ll
@@ -22,7 +22,7 @@ define dso_local void @_Z6test01v() addrspace(1) #0 {
 ; CHECK:       do.body:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr @C, align 4, !tbaa [[TBAA2:![0-9]+]]
 ; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP0]], 1
-; CHECK-NEXT:    call addrspace(1) void @llvm.lifetime.start.p0(i64 4, ptr [[J]]) #[[ATTR2:[0-9]+]]
+; CHECK-NEXT:    call addrspace(1) void @llvm.lifetime.start.p0(ptr [[J]]) #[[ATTR2:[0-9]+]]
 ; CHECK-NEXT:    store i32 0, ptr [[J]], align 4, !tbaa [[TBAA2]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]]
 ; CHECK:       for.cond:
@@ -30,11 +30,11 @@ define dso_local void @_Z6test01v() addrspace(1) #0 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP1]], 3
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call addrspace(1) void @llvm.lifetime.end.p0(i64 4, ptr [[J]]) #[[ATTR2]]
+; CHECK-NEXT:    call addrspace(1) void @llvm.lifetime.end.p0(ptr [[J]]) #[[ATTR2]]
 ; CHECK-NEXT:    br label [[DO_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    store i32 undef, ptr [[I]], align 4
-; CHECK-NEXT:    call addrspace(1) void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]]
+; CHECK-NEXT:    call addrspace(1) void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2]]
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !tbaa [[TBAA2]]
 ; CHECK-NEXT:    br label [[FOR_COND1:%.*]]
 ; CHECK:       for.cond1:
@@ -43,7 +43,7 @@ define dso_local void @_Z6test01v() addrspace(1) #0 {
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[FOR_BODY4:%.*]], label [[FOR_COND_CLEANUP3:%.*]]
 ; CHECK:       for.cond.cleanup3:
-; CHECK-NEXT:    call addrspace(1) void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]]
+; CHECK-NEXT:    call addrspace(1) void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[J]], align 4, !tbaa [[TBAA2]]
 ; CHECK-NEXT:    [[INC7:%.*]] = add nsw i32 [[TMP4]], 1
 ; CHECK-NEXT:    store i32 [[INC7]], ptr [[J]], align 4, !tbaa [[TBAA2]]
@@ -64,7 +64,7 @@ entry:
 do.body:                                          ; preds = %do.cond, %entry
   %0 = load i32, ptr @C, align 4, !tbaa !2
   %inc = add nsw i32 %0, 1
-  call addrspace(1) void @llvm.lifetime.start.p0(i64 4, ptr %j) #2
+  call addrspace(1) void @llvm.lifetime.start.p0(ptr %j) #2
   store i32 0, ptr %j, align 4, !tbaa !2
   br label %for.cond
 
@@ -74,12 +74,12 @@ for.cond:                                         ; preds = %for.inc6, %do.body
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call addrspace(1) void @llvm.lifetime.end.p0(i64 4, ptr %j) #2
+  call addrspace(1) void @llvm.lifetime.end.p0(ptr %j) #2
   br label %for.end8
 
 for.body:                                         ; preds = %for.cond
   store i32 undef, ptr %i, align 4
-  call addrspace(1) void @llvm.lifetime.start.p0(i64 4, ptr %i) #2
+  call addrspace(1) void @llvm.lifetime.start.p0(ptr %i) #2
   store i32 0, ptr %i, align 4, !tbaa !2
   br label %for.cond1
 
@@ -90,7 +90,7 @@ for.cond1:                                        ; preds = %for.inc, %for.body
   br i1 %cmp2, label %for.body4, label %for.cond.cleanup3
 
 for.cond.cleanup3:                                ; preds = %for.cond1
-  call addrspace(1) void @llvm.lifetime.end.p0(i64 4, ptr %i) #2
+  call addrspace(1) void @llvm.lifetime.end.p0(ptr %i) #2
   br label %for.end
 
 for.body4:                                        ; preds = %for.cond1
@@ -124,10 +124,10 @@ do.end:                                           ; preds = %do.cond
 }
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) addrspace(1) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) addrspace(1) #1
 
 ; Function Attrs: argmemonly nofree nosync nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) addrspace(1) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) addrspace(1) #1
 
 attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { argmemonly nofree nosync nounwind willreturn }
diff --git a/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
index 87d64932ef093..0014b91dc95bc 100644
--- a/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
+++ b/llvm/test/Transforms/SimplifyCFG/sink-and-convert-switch.ll
@@ -8,16 +8,16 @@ define void @pr104567(i8 %x, ptr %f) {
 ; CHECK-SAME: i8 [[X:%.*]], ptr [[F:%.*]]) {
 ; CHECK-NEXT:  [[START:.*:]]
 ; CHECK-NEXT:    [[Y:%.*]] = alloca [1 x i8], align 1
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 1, ptr nonnull [[Y]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[Y]])
 ; CHECK-NEXT:    [[SWITCH_OFFSET:%.*]] = add nsw i8 [[X]], 4
 ; CHECK-NEXT:    store i8 [[SWITCH_OFFSET]], ptr [[Y]], align 1
 ; CHECK-NEXT:    call void [[F]](ptr [[Y]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 1, ptr nonnull [[Y]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[Y]])
 ; CHECK-NEXT:    ret void
 ;
 start:
   %y = alloca [1 x i8], align 1
-  call void @llvm.lifetime.start.p0(i64 1, ptr nonnull %y)
+  call void @llvm.lifetime.start.p0(ptr nonnull %y)
   switch i8 %x, label %default.unreachable [
   i8 0, label %bb4
   i8 1, label %bb3
@@ -41,7 +41,7 @@ bb2:
 
 bb5:
   call void %f(ptr %y)
-  call void @llvm.lifetime.end.p0(i64 1, ptr nonnull %y)
+  call void @llvm.lifetime.end.p0(ptr nonnull %y)
   ret void
 }
 
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_create.ll b/llvm/test/Transforms/SimplifyCFG/switch_create.ll
index f446d718f8206..a06e64f74b3ac 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch_create.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch_create.ll
@@ -1068,3 +1068,127 @@ if:
 else:
   ret void
 }
+
+define void @trunc_nuw_i1_condition(i32 %V) {
+; CHECK-LABEL: @trunc_nuw_i1_condition(
+; CHECK-NEXT:    switch i32 [[V:%.*]], label [[F:%.*]] [
+; CHECK-NEXT:      i32 2, label [[T:%.*]]
+; CHECK-NEXT:      i32 0, label [[T]]
+; CHECK-NEXT:    ]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret void
+; CHECK:       T:
+; CHECK-NEXT:    call void @foo1()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       F:
+; CHECK-NEXT:    call void @foo2()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  %C1 = icmp eq i32 %V, 2
+  br i1 %C1, label %T, label %N
+N:
+  %C2 = trunc nuw i32 %V to i1
+  br i1 %C2, label %F, label %T
+T:
+  call void @foo1( )
+  ret void
+F:
+  call void @foo2( )
+  ret void
+}
+
+define void @neg_trunc_i1_condition(i32 %V) {
+; CHECK-LABEL: @neg_trunc_i1_condition(
+; CHECK-NEXT:    [[C1:%.*]] = icmp ne i32 [[V:%.*]], 2
+; CHECK-NEXT:    [[C2:%.*]] = trunc i32 [[V]] to i1
+; CHECK-NEXT:    [[OR_COND:%.*]] = and i1 [[C1]], [[C2]]
+; CHECK-NEXT:    br i1 [[OR_COND]], label [[F:%.*]], label [[T:%.*]]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret void
+; CHECK:       T:
+; CHECK-NEXT:    call void @foo1()
+; CHECK-NEXT:    br label [[COMMON_RET:%.*]]
+; CHECK:       F:
+; CHECK-NEXT:    call void @foo2()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+  %C1 = icmp eq i32 %V, 2
+  br i1 %C1, label %T, label %N
+N:
+  %C2 = trunc i32 %V to i1
+  br i1 %C2, label %F, label %T
+T:
+  call void @foo1( )
+  ret void
+F:
+  call void @foo2( )
+  ret void
+}
+
+define void @extra_cond_is_eq_cmp(i8 %c, i32 %x)  {
+; CHECK-LABEL: @extra_cond_is_eq_cmp(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 32
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i1 [[CMP]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN:%.*]], label [[SWITCH_EARLY_TEST:%.*]]
+; CHECK:       switch.early.test:
+; CHECK-NEXT:    switch i8 [[C:%.*]], label [[COMMON_RET:%.*]] [
+; CHECK-NEXT:      i8 99, label [[IF_THEN]]
+; CHECK-NEXT:      i8 97, label [[IF_THEN]]
+; CHECK-NEXT:    ]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret void
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @foo1()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+entry:
+  %cmp = icmp eq i32 %x, 32
+  %cmp4 = icmp eq i8 %c, 97
+  %or.cond = or i1 %cmp, %cmp4
+  %cmp9 = icmp eq i8 %c, 99
+  %or.cond11 = or i1 %or.cond, %cmp9
+  br i1 %or.cond11, label %if.then, label %if.end
+
+if.then:
+  tail call void @foo1()
+  ret void
+
+if.end:
+  ret void
+
+}
+
+define void @extra_cond_is_eq_cmp_c(i8 %c, i32 %x)  {
+; CHECK-LABEL: @extra_cond_is_eq_cmp_c(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 32
+; CHECK-NEXT:    [[TMP0:%.*]] = freeze i1 [[CMP]]
+; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN:%.*]], label [[SWITCH_EARLY_TEST:%.*]]
+; CHECK:       switch.early.test:
+; CHECK-NEXT:    switch i8 [[C:%.*]], label [[COMMON_RET:%.*]] [
+; CHECK-NEXT:      i8 99, label [[IF_THEN]]
+; CHECK-NEXT:      i8 97, label [[IF_THEN]]
+; CHECK-NEXT:    ]
+; CHECK:       common.ret:
+; CHECK-NEXT:    ret void
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @foo1()
+; CHECK-NEXT:    br label [[COMMON_RET]]
+;
+entry:
+  %cmp = icmp eq i32 %x, 32
+  %cmp4 = icmp eq i8 %c, 97
+  %or.cond = or i1 %cmp4, %cmp
+  %cmp9 = icmp eq i8 %c, 99
+  %or.cond11 = or i1 %or.cond, %cmp9
+  br i1 %or.cond11, label %if.then, label %if.end
+
+if.then:
+  tail call void @foo1()
+  ret void
+
+if.end:
+  ret void
+
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/tail-merge-noreturn.ll b/llvm/test/Transforms/SimplifyCFG/tail-merge-noreturn.ll
index 10e4870d504e3..77ce7300e6752 100644
--- a/llvm/test/Transforms/SimplifyCFG/tail-merge-noreturn.ll
+++ b/llvm/test/Transforms/SimplifyCFG/tail-merge-noreturn.ll
@@ -315,8 +315,8 @@ cont3:
 ; from sharing stack slots for x and y.
 
 declare void @escape_i32_ptr(ptr)
-declare void @llvm.lifetime.start(i64, ptr nocapture)
-declare void @llvm.lifetime.end(i64, ptr nocapture)
+declare void @llvm.lifetime.start(ptr nocapture)
+declare void @llvm.lifetime.end(ptr nocapture)
 
 define void @dont_merge_lifetimes(i32 %c1, i32 %c2) {
 ; CHECK-LABEL: @dont_merge_lifetimes(
@@ -328,7 +328,7 @@ define void @dont_merge_lifetimes(i32 %c1, i32 %c2) {
 ; CHECK-NEXT:      i32 42, label [[IF_THEN3:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       if.then:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    store i32 0, ptr [[X]], align 4
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[C2:%.*]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN1:%.*]]
@@ -336,11 +336,11 @@ define void @dont_merge_lifetimes(i32 %c1, i32 %c2) {
 ; CHECK-NEXT:    call void @escape_i32_ptr(ptr nonnull [[X]])
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[X]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[X]])
 ; CHECK-NEXT:    call void @abort()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       if.then3:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[Y]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[Y]])
 ; CHECK-NEXT:    store i32 0, ptr [[Y]], align 4
 ; CHECK-NEXT:    [[TOBOOL5:%.*]] = icmp eq i32 [[C2]], 0
 ; CHECK-NEXT:    br i1 [[TOBOOL5]], label [[IF_END7:%.*]], label [[IF_THEN6:%.*]]
@@ -348,7 +348,7 @@ define void @dont_merge_lifetimes(i32 %c1, i32 %c2) {
 ; CHECK-NEXT:    call void @escape_i32_ptr(ptr nonnull [[Y]])
 ; CHECK-NEXT:    br label [[IF_END7]]
 ; CHECK:       if.end7:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[Y]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[Y]])
 ; CHECK-NEXT:    call void @abort()
 ; CHECK-NEXT:    unreachable
 ; CHECK:       if.end9:
@@ -363,7 +363,7 @@ entry:
   ]
 
 if.then:                                          ; preds = %entry
-  call void @llvm.lifetime.start(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.start(ptr nonnull %x)
   store i32 0, ptr %x, align 4
   %tobool = icmp eq i32 %c2, 0
   br i1 %tobool, label %if.end, label %if.then1
@@ -373,12 +373,12 @@ if.then1:                                         ; preds = %if.then
   br label %if.end
 
 if.end:                                           ; preds = %if.then1, %if.then
-  call void @llvm.lifetime.end(i64 4, ptr nonnull %x)
+  call void @llvm.lifetime.end(ptr nonnull %x)
   call void @abort()
   unreachable
 
 if.then3:                                         ; preds = %entry
-  call void @llvm.lifetime.start(i64 4, ptr nonnull %y)
+  call void @llvm.lifetime.start(ptr nonnull %y)
   store i32 0, ptr %y, align 4
   %tobool5 = icmp eq i32 %c2, 0
   br i1 %tobool5, label %if.end7, label %if.then6
@@ -388,7 +388,7 @@ if.then6:                                         ; preds = %if.then3
   br label %if.end7
 
 if.end7:                                          ; preds = %if.then6, %if.then3
-  call void @llvm.lifetime.end(i64 4, ptr nonnull %y)
+  call void @llvm.lifetime.end(ptr nonnull %y)
   call void @abort()
   unreachable
 
diff --git a/llvm/test/Transforms/SimplifyCFG/two-entry-phi-return.ll b/llvm/test/Transforms/SimplifyCFG/two-entry-phi-return.ll
index 57930c91b9796..f6d71ddda74fe 100644
--- a/llvm/test/Transforms/SimplifyCFG/two-entry-phi-return.ll
+++ b/llvm/test/Transforms/SimplifyCFG/two-entry-phi-return.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s
+; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -keep-loops="false" -S | FileCheck %s
 
 define i1 @qux(ptr %m, ptr %n, ptr %o, ptr %p) nounwind  {
 ; CHECK-LABEL: @qux(
diff --git a/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll b/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll
index 325db79ebbd7e..fa771ad01f3c3 100644
--- a/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll
+++ b/llvm/test/Transforms/TailCallElim/tre-byval-parameter-2.ll
@@ -44,17 +44,17 @@ define dso_local void @_Z7dostuff1AS_i(ptr nocapture byval(%struct.A) align 8 %a
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [[STRUCT_A]], ptr [[B]], i64 0, i32 0, i64 5
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX4]], align 8
 ; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (ptr, ...) @printf(ptr nonnull dereferenceable(1) @.str, i64 [[INC]], i64 [[TMP1]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 80, ptr nonnull [[AGG_TMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[AGG_TMP]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(80) [[AGG_TMP]], ptr nonnull align 8 dereferenceable(80) [[B]], i64 80, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 80, ptr nonnull [[AGG_TMP5]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[AGG_TMP5]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(80) [[AGG_TMP5]], ptr nonnull align 8 dereferenceable(80) [[A]], i64 80, i1 false)
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[I_TR]], 1
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP1]], ptr align 8 [[AGG_TMP]], i64 80, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP52]], ptr align 8 [[AGG_TMP5]], i64 80, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[A]], ptr align 8 [[AGG_TMP1]], i64 80, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[B]], ptr align 8 [[AGG_TMP52]], i64 80, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 80, ptr nonnull [[AGG_TMP]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 80, ptr nonnull [[AGG_TMP5]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[AGG_TMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[AGG_TMP5]])
 ; CHECK-NEXT:    br label [[TAILRECURSE]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret void
@@ -74,14 +74,14 @@ if.end:                                           ; preds = %entry
   %1 = load i64, ptr %arrayidx4, align 8
   %call = call i32 (ptr, ...) @printf(ptr nonnull dereferenceable(1) @.str
 , i64 %inc, i64 %1)
-  call void @llvm.lifetime.start.p0(i64 80, ptr nonnull %agg.tmp)
+  call void @llvm.lifetime.start.p0(ptr nonnull %agg.tmp)
   call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(80) %agg.tmp, ptr nonnull align 8 dereferenceable(80) %b, i64 80, i1 false)
-  call void @llvm.lifetime.start.p0(i64 80, ptr nonnull %agg.tmp5)
+  call void @llvm.lifetime.start.p0(ptr nonnull %agg.tmp5)
   call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(80) %agg.tmp5, ptr nonnull align 8 dereferenceable(80) %a, i64 80, i1 false)
   %add = add nsw i32 %i, 1
   call void @_Z7dostuff1AS_i(ptr nonnull byval(%struct.A) align 8 %agg.tmp, ptr nonnull byval(%struct.A) align 8 %agg.tmp5, i32 %add)
-  call void @llvm.lifetime.end.p0(i64 80, ptr nonnull %agg.tmp)
-  call void @llvm.lifetime.end.p0(i64 80, ptr nonnull %agg.tmp5)
+  call void @llvm.lifetime.end.p0(ptr nonnull %agg.tmp)
+  call void @llvm.lifetime.end.p0(ptr nonnull %agg.tmp5)
   br label %return
 
 return:                                           ; preds = %entry, %if.end
@@ -95,10 +95,10 @@ declare dso_local noundef i32 @printf(ptr nocapture noundef readonly, ...) local
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 ; Function Attrs: noinline norecurse nounwind optnone uwtable
 define dso_local i32 @main() local_unnamed_addr #3 {
diff --git a/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll b/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll
index 256fb04b3ba1d..dedd0811bea51 100644
--- a/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll
+++ b/llvm/test/Transforms/TailCallElim/tre-byval-parameter.ll
@@ -44,14 +44,14 @@ define dso_local i32 @_Z3fooi1S(i32 %count, ptr nocapture readonly byval(%struct
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[COUNT_TR]], 1
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(20) [[AGG_TMP1]], ptr nonnull align 8 dereferenceable(20) [[P1]], i64 20, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr nonnull [[AGG_TMP14]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 20, ptr nonnull [[AGG_TMP_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[AGG_TMP14]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[AGG_TMP_I]])
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(20) [[AGG_TMP14]], ptr nonnull align 8 dereferenceable(20) [[AGG_TMP1]], i64 20, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(20) [[AGG_TMP_I]], ptr nonnull align 8 dereferenceable(20) [[AGG_TMP14]], i64 20, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[AGG_TMP_I1]], ptr align 8 [[AGG_TMP_I]], i64 20, i1 false)
 ; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[P1]], ptr align 8 [[AGG_TMP_I1]], i64 20, i1 false)
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr nonnull [[AGG_TMP14]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 20, ptr nonnull [[AGG_TMP_I]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[AGG_TMP14]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[AGG_TMP_I]])
 ; CHECK-NEXT:    br label [[TAILRECURSE]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret i32 [[CALL]]
@@ -72,13 +72,13 @@ if.then:                                          ; preds = %entry
 if.end:                                           ; preds = %entry
   %add = add nsw i32 %count, 1
   call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(20) %agg.tmp1, ptr nonnull align 8 dereferenceable(20) %p1, i64 20, i1 false)
-  call void @llvm.lifetime.start.p0(i64 20, ptr nonnull %agg.tmp14)
-  call void @llvm.lifetime.start.p0(i64 20, ptr nonnull %agg.tmp.i)
+  call void @llvm.lifetime.start.p0(ptr nonnull %agg.tmp14)
+  call void @llvm.lifetime.start.p0(ptr nonnull %agg.tmp.i)
   call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(20) %agg.tmp14, ptr nonnull align 8 dereferenceable(20) %agg.tmp1, i64 20, i1 false)
   call void @llvm.memcpy.p0.p0.i64(ptr nonnull align 8 dereferenceable(20) %agg.tmp.i, ptr nonnull align 8 dereferenceable(20) %agg.tmp14, i64 20, i1 false)
   %call.i = call i32 @_Z3fooi1S(i32 %add, ptr nonnull byval(%struct.S) align 8 %agg.tmp.i)
-  call void @llvm.lifetime.end.p0(i64 20, ptr nonnull %agg.tmp14)
-  call void @llvm.lifetime.end.p0(i64 20, ptr nonnull %agg.tmp.i)
+  call void @llvm.lifetime.end.p0(ptr nonnull %agg.tmp14)
+  call void @llvm.lifetime.end.p0(ptr nonnull %agg.tmp.i)
   br label %return
 
 return:                                           ; preds = %if.end, %if.then
@@ -89,10 +89,10 @@ return:                                           ; preds = %if.end, %if.then
 declare dso_local i32 @_Z3zoo1S(ptr byval(%struct.S) align 8) local_unnamed_addr #1
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
 declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #2
diff --git a/llvm/test/Transforms/TailCallElim/tre-multiple-exits.ll b/llvm/test/Transforms/TailCallElim/tre-multiple-exits.ll
index 293deca00f7fa..b77ae9c9c7495 100644
--- a/llvm/test/Transforms/TailCallElim/tre-multiple-exits.ll
+++ b/llvm/test/Transforms/TailCallElim/tre-multiple-exits.ll
@@ -49,11 +49,11 @@ define dso_local void @_Z19test_multiple_exitsi(i32 %param) local_unnamed_addr #
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[PARAM_TR]], 10
 ; CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[TEMP]]) #1
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TEMP]]) #1
 ; CHECK-NEXT:    call void @_Z11capture_argPi(ptr nonnull [[TEMP]])
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i32 [[PARAM_TR]], 1
 ; CHECK-NEXT:    call void @_Z19test_multiple_exitsi(i32 [[ADD]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[TEMP]]) #1
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TEMP]]) #1
 ; CHECK-NEXT:    br label [[IF_END14:%.*]]
 ; CHECK:       if.else:
 ; CHECK-NEXT:    [[PARAM_OFF:%.*]] = add i32 [[PARAM_TR]], -10
@@ -80,11 +80,11 @@ entry:
   br i1 %0, label %if.then, label %if.else
 
 if.then:                                          ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %temp) #2
+  call void @llvm.lifetime.start.p0(ptr nonnull %temp) #2
   call void @_Z11capture_argPi(ptr nonnull %temp)
   %add = add nuw nsw i32 %param, 1
   call void @_Z19test_multiple_exitsi(i32 %add)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %temp) #2
+  call void @llvm.lifetime.end.p0(ptr nonnull %temp) #2
   br label %if.end14
 
 if.else:                                          ; preds = %entry
@@ -113,10 +113,10 @@ if.end14:                                         ; preds = %if.then5, %if.then1
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 attributes #0 = { nofree noinline norecurse nounwind uwtable }
 attributes #1 = { nounwind uwtable }
diff --git a/llvm/test/Transforms/TailCallElim/tre-noncapturing-alloca-calls.ll b/llvm/test/Transforms/TailCallElim/tre-noncapturing-alloca-calls.ll
index c9ac9a5d480f2..2f1adedbf7b77 100644
--- a/llvm/test/Transforms/TailCallElim/tre-noncapturing-alloca-calls.ll
+++ b/llvm/test/Transforms/TailCallElim/tre-noncapturing-alloca-calls.ll
@@ -34,11 +34,11 @@ define dso_local void @_Z4testi(i32 %recurseCount) local_unnamed_addr #1 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[RECURSECOUNT_TR]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[IF_END]]
 ; CHECK:       if.end:
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[TEMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr nonnull [[TEMP]])
 ; CHECK-NEXT:    store i32 10, ptr [[TEMP]], align 4
 ; CHECK-NEXT:    call void @_Z15globalIncrementPKi(ptr nonnull [[TEMP]])
 ; CHECK-NEXT:    [[SUB]] = add nsw i32 [[RECURSECOUNT_TR]], -1
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[TEMP]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr nonnull [[TEMP]])
 ; CHECK-NEXT:    br label [[TAILRECURSE]]
 ; CHECK:       return:
 ; CHECK-NEXT:    ret void
@@ -49,12 +49,12 @@ entry:
   br i1 %cmp, label %return, label %if.end
 
 if.end:                                           ; preds = %entry
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %temp) #6
+  call void @llvm.lifetime.start.p0(ptr nonnull %temp) #6
   store i32 10, ptr %temp, align 4
   call void @_Z15globalIncrementPKi(ptr nonnull %temp)
   %sub = add nsw i32 %recurseCount, -1
   call void @_Z4testi(i32 %sub)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %temp) #6
+  call void @llvm.lifetime.end.p0(ptr nonnull %temp) #6
   br label %return
 
 return:                                           ; preds = %entry, %if.end
@@ -62,10 +62,10 @@ return:                                           ; preds = %entry, %if.end
 }
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.start.p0(ptr nocapture) #2
 
 ; Function Attrs: argmemonly nounwind willreturn
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #2
+declare void @llvm.lifetime.end.p0(ptr nocapture) #2
 
 attributes #0 = { nofree noinline norecurse nounwind uwtable }
 attributes #1 = { nounwind uwtable }
diff --git a/llvm/test/Transforms/ThinLTOBitcodeWriter/cfi-check.ll b/llvm/test/Transforms/ThinLTOBitcodeWriter/cfi-check.ll
new file mode 100644
index 0000000000000..b927af6b92f7c
--- /dev/null
+++ b/llvm/test/Transforms/ThinLTOBitcodeWriter/cfi-check.ll
@@ -0,0 +1,19 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
+; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
+; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
+
+; Check that __cfi_check is emitted on the full LTO side with
+; attributes preserved.
+
+; M0: define void @f()
+define void @f() !type !{!"f1", i32 0} {
+  ret void 
+}
+
+; M1: define void @__cfi_check() #0
+define void @__cfi_check() #0 {
+  ret void
+}
+
+; M1: attributes #0 = { "branch-target-enforcement" }
+attributes #0 = { "branch-target-enforcement" }
diff --git a/llvm/test/Transforms/Util/PredicateInfo/branch-on-same-cond.ll b/llvm/test/Transforms/Util/PredicateInfo/branch-on-same-cond.ll
index e4232cf928bd1..0be13ee76bece 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/branch-on-same-cond.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/branch-on-same-cond.ll
@@ -10,23 +10,23 @@ define i32 @test(i32 %x) {
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK: RenamedOp: [[CMP]]
-; CHECK:         [[CMP_0:%.*]] = call i1 @llvm.ssa.copy.{{.*}}(i1 [[CMP]])
+; CHECK:         [[CMP_0:%.*]] = bitcast i1 [[CMP]] to i1
 ; CHECK: RenamedOp: [[X]]
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[EXIT1:%.*]]
 ; CHECK:       bb2:
 ; CHECK: RenamedOp: [[CMP_0]]
-; CHECK:         [[CMP_0_1:%.*]] = call i1 @llvm.ssa.copy.{{.*}}(i1 [[CMP_0]])
+; CHECK:         [[CMP_0_1:%.*]] = bitcast i1 [[CMP_0]] to i1
 ; CHECK: RenamedOp: [[X]]
-; CHECK:         [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X_0]])
+; CHECK:         [[X_0_1:%.*]] = bitcast i32 [[X_0]] to i32
 ; CHECK: RenamedOp: [[X_0]]
-; CHECK:         [[X_0_4:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X_0]])
+; CHECK:         [[X_0_4:%.*]] = bitcast i32 [[X_0]] to i32
 ; CHECK-NEXT:    br i1 [[CMP_0]], label [[BB3:%.*]], label [[EXIT2:%.*]]
 ; CHECK:       bb3:
 ; CHECK: RenamedOp: [[X]]
-; CHECK:         [[X_0_1_2:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X_0_1]])
+; CHECK:         [[X_0_1_2:%.*]] = bitcast i32 [[X_0_1]] to i32
 ; CHECK: RenamedOp: [[X_0_1]]
-; CHECK:         [[X_0_1_3:%.*]] = call i32 @llvm.ssa.copy.{{.*}}(i32 [[X_0_1]])
+; CHECK:         [[X_0_1_3:%.*]] = bitcast i32 [[X_0_1]] to i32
 ; CHECK-NEXT:    br i1 [[CMP_0_1]], label [[EXIT3:%.*]], label [[EXIT4:%.*]]
 ; CHECK:       exit1:
 ; CHECK-NEXT:    ret i32 0
diff --git a/llvm/test/Transforms/Util/PredicateInfo/condprop.ll b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
index e8bb8d410bf88..0235732b95a83 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/condprop.ll
@@ -98,11 +98,11 @@ define void @test3(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[Z]])
-; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[XZ]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
-; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.{{.+}}(i1 [[YZ]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[Y]])
+; CHECK:         [[Z_0:%.*]] = bitcast i1 [[Z]] to i1
+; CHECK:         [[XZ_0:%.*]] = bitcast i1 [[XZ]] to i1
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[YZ_0:%.*]] = bitcast i1 [[YZ]] to i1
+; CHECK:         [[Y_0:%.*]] = bitcast i32 [[Y]] to i32
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH_ZERO:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both_zero:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ_0]])
@@ -139,7 +139,7 @@ define void @test4(i1 %b, i32 %x) {
 ; CHECK-NEXT:    i32 3, label [[CASE3]]
 ; CHECK-NEXT:    i32 4, label [[DEFAULT:%.*]]
 ; CHECK-NEXT:    ] Edge: [label [[SW]],label %case1], RenamedOp: [[X:%.*]] }
-; CHECK-NEXT:    [[X_0:%.*]] = call i32 @llvm.ssa.copy.{{.+}}(i32 [[X]])
+; CHECK-NEXT:    [[X_0:%.*]] = bitcast i32 [[X]] to i32
 ; CHECK-NEXT:    switch i32 [[X]], label [[DEFAULT]] [
 ; CHECK-NEXT:    i32 0, label [[CASE0]]
 ; CHECK-NEXT:    i32 1, label [[CASE1]]
@@ -186,10 +186,10 @@ case3:
 define i1 @test5(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
-; CHECK:         [[Y_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[X_1:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[Y_0:%.*]] = bitcast i32 [[Y]] to i32
+; CHECK:         [[Y_1:%.*]] = bitcast i32 [[Y]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
 ; CHECK:       same:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[X_0]], [[Y_0]]
@@ -259,10 +259,10 @@ different:
 define i1 @test7(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test7(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
-; CHECK:         [[Y_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[X_1:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[Y_0:%.*]] = bitcast i32 [[Y]] to i32
+; CHECK:         [[Y_1:%.*]] = bitcast i32 [[Y]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
 ; CHECK:       same:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[X_0]], [[Y_0]]
@@ -286,10 +286,10 @@ different:
 define i1 @test7_fp(float %x, float %y) {
 ; CHECK-LABEL: @test7_fp(
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[X:%.*]], [[Y:%.*]]
-; CHECK:         [[X_0:%.*]] = call float @llvm.ssa.copy.f32(float [[X]])
-; CHECK:         [[X_1:%.*]] = call float @llvm.ssa.copy.f32(float [[X]])
-; CHECK:         [[Y_0:%.*]] = call float @llvm.ssa.copy.f32(float [[Y]])
-; CHECK:         [[Y_1:%.*]] = call float @llvm.ssa.copy.f32(float [[Y]])
+; CHECK:         [[X_0:%.*]] = bitcast float [[X]] to float
+; CHECK:         [[X_1:%.*]] = bitcast float [[X]] to float
+; CHECK:         [[Y_0:%.*]] = bitcast float [[Y]] to float
+; CHECK:         [[Y_1:%.*]] = bitcast float [[Y]] to float
 ; CHECK-NEXT:    br i1 [[CMP]], label [[SAME:%.*]], label [[DIFFERENT:%.*]]
 ; CHECK:       same:
 ; CHECK-NEXT:    [[CMP2:%.*]] = fcmp ule float [[X_0]], [[Y_0]]
@@ -359,8 +359,8 @@ different:
 define i32 @test9(i32 %i, i32 %j) {
 ; CHECK-LABEL: @test9(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]]
-; CHECK:         [[I_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[I]])
-; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[J]])
+; CHECK:         [[I_0:%.*]] = bitcast i32 [[I]] to i32
+; CHECK:         [[J_0:%.*]] = bitcast i32 [[J]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]]
 ; CHECK:       cond_true:
 ; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[I_0]], [[J_0]]
@@ -382,8 +382,8 @@ ret:
 define i32 @test10(i32 %j, i32 %i) {
 ; CHECK-LABEL: @test10(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], [[J:%.*]]
-; CHECK:         [[I_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[I]])
-; CHECK:         [[J_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[J]])
+; CHECK:         [[I_0:%.*]] = bitcast i32 [[I]] to i32
+; CHECK:         [[J_0:%.*]] = bitcast i32 [[J]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[RET:%.*]]
 ; CHECK:       cond_true:
 ; CHECK-NEXT:    [[DIFF:%.*]] = sub i32 [[I_0]], [[J_0]]
@@ -409,14 +409,14 @@ define i32 @test11(i32 %x) {
 ; CHECK-NEXT:    [[V0:%.*]] = call i32 @yogibar()
 ; CHECK-NEXT:    [[V1:%.*]] = call i32 @yogibar()
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V0]], [[V1]]
-; CHECK:         [[V0_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[V0]])
-; CHECK:         [[V1_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[V1]])
+; CHECK:         [[V0_0:%.*]] = bitcast i32 [[V0]] to i32
+; CHECK:         [[V1_0:%.*]] = bitcast i32 [[V1]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[NEXT:%.*]]
 ; CHECK:       cond_true:
 ; CHECK-NEXT:    ret i32 [[V1_0]]
 ; CHECK:       next:
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp eq i32 [[X:%.*]], [[V0_0]]
-; CHECK:         [[V0_0_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[V0_0]])
+; CHECK:         [[V0_0_1:%.*]] = bitcast i32 [[V0_0]] to i32
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[COND_TRUE2:%.*]], label [[NEXT2:%.*]]
 ; CHECK:       cond_true2:
 ; CHECK-NEXT:    ret i32 [[V0_0_1]]
@@ -445,8 +445,8 @@ next2:
 define i32 @test12(i32 %x) {
 ; CHECK-LABEL: @test12(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[X_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[X_1:%.*]] = bitcast i32 [[X]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond_true:
 ; CHECK-NEXT:    br label [[RET:%.*]]
diff --git a/llvm/test/Transforms/Util/PredicateInfo/diamond.ll b/llvm/test/Transforms/Util/PredicateInfo/diamond.ll
index 17a2c62782546..ac2c9a1026e76 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/diamond.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/diamond.ll
@@ -5,12 +5,12 @@ define i1 @f(i32 %x, i1 %y) {
 ; CHECK-NEXT:    br i1 [[Y:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[BB3:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[X2:%.*]] = add nuw nsw i32 [[X]], 1
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X2]], 2
-; CHECK:         [[X2_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X2]])
+; CHECK:         [[X2_0:%.*]] = bitcast i32 [[X2]] to i32
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[BB2]], label [[BB3]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[X3:%.*]] = phi i32 [ [[X_0]], [[BB0]] ], [ [[X2_0]], [[BB1]] ]
@@ -38,12 +38,12 @@ define i1 @g(i32 %x, i1 %y) {
 ; CHECK-NEXT:    br i1 [[Y:%.*]], label [[BB0:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb0:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB3:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    [[X2:%.*]] = add nuw nsw i32 [[X]], 1
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp sge i32 [[X2]], 2
-; CHECK:         [[X2_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X2]])
+; CHECK:         [[X2_0:%.*]] = bitcast i32 [[X2]] to i32
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[BB3]], label [[BB2]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    [[X3:%.*]] = phi i32 [ [[X_0]], [[BB0]] ], [ [[X2_0]], [[BB1]] ]
diff --git a/llvm/test/Transforms/Util/PredicateInfo/edge.ll b/llvm/test/Transforms/Util/PredicateInfo/edge.ll
index 58674c22002a4..ef757f323921a 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/edge.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/edge.ll
@@ -5,7 +5,7 @@ define i32 @f1(i32 %x) {
 ; CHECK-LABEL: @f1(
 ; CHECK-NEXT:  bb0:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[BB1:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    br label [[BB2]]
@@ -29,7 +29,7 @@ define i32 @f2(i32 %x) {
 ; CHECK-LABEL: @f2(
 ; CHECK-NEXT:  bb0:
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB1:%.*]], label [[BB2:%.*]]
 ; CHECK:       bb1:
 ; CHECK-NEXT:    br label [[BB2]]
@@ -52,7 +52,7 @@ bb2:
 define i32 @f3(i32 %x) {
 ; CHECK-LABEL: @f3(
 ; CHECK-NEXT:  bb0:
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X:%.*]])
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X:%.*]] to i32
 ; CHECK-NEXT:    switch i32 [[X]], label [[BB1:%.*]] [
 ; CHECK-NEXT:    i32 0, label [[BB2:%.*]]
 ; CHECK-NEXT:    ]
@@ -78,7 +78,7 @@ define double @fcmp_oeq_not_zero(double %x, double %y) {
 ; CHECK-LABEL: @fcmp_oeq_not_zero(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y:%.*]], 2.000000e+00
-; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Y]])
+; CHECK:         [[Y_0:%.*]] = bitcast double [[Y]] to double
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[RETURN:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
@@ -105,7 +105,7 @@ define double @fcmp_une_not_zero(double %x, double %y) {
 ; CHECK-LABEL: @fcmp_une_not_zero(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[Y:%.*]], 2.000000e+00
-; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Y]])
+; CHECK:         [[Y_0:%.*]] = bitcast double [[Y]] to double
 ; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
@@ -132,7 +132,7 @@ define double @fcmp_oeq_zero(double %x, double %y) {
 ; CHECK-LABEL: @fcmp_oeq_zero(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y:%.*]], 0.000000e+00
-; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Y]])
+; CHECK:         [[Y_0:%.*]] = bitcast double [[Y]] to double
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[RETURN:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
@@ -159,7 +159,7 @@ define double @fcmp_une_zero(double %x, double %y) {
 ; CHECK-LABEL: @fcmp_une_zero(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[Y:%.*]], -0.000000e+00
-; CHECK:         [[Y_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Y]])
+; CHECK:         [[Y_0:%.*]] = bitcast double [[Y]] to double
 ; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Y_0]]
@@ -188,7 +188,7 @@ define double @fcmp_oeq_maybe_zero(double %x, double %y, double %z1, double %z2)
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[Z:%.*]] = fadd double [[Z1:%.*]], [[Z2:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq double [[Y:%.*]], [[Z]]
-; CHECK:         [[Z_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Z]])
+; CHECK:         [[Z_0:%.*]] = bitcast double [[Z]] to double
 ; CHECK-NEXT:    br i1 [[CMP]], label [[IF:%.*]], label [[RETURN:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Z_0]]
@@ -217,7 +217,7 @@ define double @fcmp_une_maybe_zero(double %x, double %y, double %z1, double %z2)
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[Z:%.*]] = fadd double [[Z1:%.*]], [[Z2:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = fcmp une double [[Y:%.*]], [[Z]]
-; CHECK:         [[Z_0:%.*]] = call double @llvm.ssa.copy.f64(double [[Z]])
+; CHECK:         [[Z_0:%.*]] = bitcast double [[Z]] to double
 ; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[ELSE:%.*]]
 ; CHECK:       else:
 ; CHECK-NEXT:    [[DIV:%.*]] = fdiv double [[X:%.*]], [[Z_0]]
diff --git a/llvm/test/Transforms/Util/PredicateInfo/pr33456.ll b/llvm/test/Transforms/Util/PredicateInfo/pr33456.ll
index 14bfbb12f5ada..36eaf6e66578d 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/pr33456.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/pr33456.ll
@@ -61,8 +61,8 @@ define i32 @main() {
 }
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 
 ; Function Attrs: argmemonly nounwind
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
diff --git a/llvm/test/Transforms/Util/PredicateInfo/testandor.ll b/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
index 4f6488982b94f..2e96a92dd37d4 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/testandor.ll
@@ -10,11 +10,11 @@ define void @test_or(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = or i1 [[XZ]], [[YZ]]
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
-; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
+; CHECK:         [[Z_0:%.*]] = bitcast i1 [[Z]] to i1
+; CHECK:         [[XZ_0:%.*]] = bitcast i1 [[XZ]] to i1
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[YZ_0:%.*]] = bitcast i1 [[YZ]] to i1
+; CHECK:         [[Y_0:%.*]] = bitcast i32 [[Y]] to i32
 ; CHECK-NEXT:    br i1 [[Z]], label [[ONEOF:%.*]], label [[NEITHER:%.*]]
 ; CHECK:       oneof:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ]])
@@ -55,11 +55,11 @@ define void @test_or_logical(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = select i1 [[XZ]], i1 true, i1 [[YZ]]
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
-; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
+; CHECK:         [[Z_0:%.*]] = bitcast i1 [[Z]] to i1
+; CHECK:         [[XZ_0:%.*]] = bitcast i1 [[XZ]] to i1
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[YZ_0:%.*]] = bitcast i1 [[YZ]] to i1
+; CHECK:         [[Y_0:%.*]] = bitcast i32 [[Y]] to i32
 ; CHECK-NEXT:    br i1 [[Z]], label [[ONEOF:%.*]], label [[NEITHER:%.*]]
 ; CHECK:       oneof:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ]])
@@ -100,11 +100,11 @@ define void @test_and(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
-; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
+; CHECK:         [[Z_0:%.*]] = bitcast i1 [[Z]] to i1
+; CHECK:         [[XZ_0:%.*]] = bitcast i1 [[XZ]] to i1
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[YZ_0:%.*]] = bitcast i1 [[YZ]] to i1
+; CHECK:         [[Y_0:%.*]] = bitcast i32 [[Y]] to i32
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ_0]])
@@ -145,11 +145,11 @@ define void @test_and_logical(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XZ:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = select i1 [[XZ]], i1 [[YZ]], i1 false
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
-; CHECK:         [[XZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[YZ_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
-; CHECK:         [[Y_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
+; CHECK:         [[Z_0:%.*]] = bitcast i1 [[Z]] to i1
+; CHECK:         [[XZ_0:%.*]] = bitcast i1 [[XZ]] to i1
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[YZ_0:%.*]] = bitcast i1 [[YZ]] to i1
+; CHECK:         [[Y_0:%.*]] = bitcast i32 [[Y]] to i32
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ_0]])
@@ -190,11 +190,11 @@ define void @testandsame(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[XGT:%.*]] = icmp sgt i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[XLT:%.*]] = icmp slt i32 [[X]], 100
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XGT]], [[XLT]]
-; CHECK:         [[Z_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
-; CHECK:         [[XGT_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XGT]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[X_0_1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X_0]])
-; CHECK:         [[XLT_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XLT]])
+; CHECK:         [[Z_0:%.*]] = bitcast i1 [[Z]] to i1
+; CHECK:         [[XGT_0:%.*]] = bitcast i1 [[XGT]] to i1
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[X_0_1:%.*]] = bitcast i32 [[X_0]] to i32
+; CHECK:         [[XLT_0:%.*]] = bitcast i1 [[XLT]] to i1
 ; CHECK-NEXT:    br i1 [[Z]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[XGT_0]])
@@ -229,16 +229,16 @@ define void @testandassume(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = and i1 [[XZ]], [[YZ]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[Z]])
-; CHECK:         [[TMP1:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[Y]])
-; CHECK:         [[TMP2:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[YZ]])
-; CHECK:         [[TMP3:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
-; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[XZ]])
-; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
-; CHECK:         [[DOT0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP5]])
-; CHECK:         [[DOT01:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP4]])
-; CHECK:         [[DOT02:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[TMP3]])
-; CHECK:         [[DOT03:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP2]])
-; CHECK:         [[DOT04:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[TMP1]])
+; CHECK:         [[TMP1:%.*]] = bitcast i32 [[Y]] to i32
+; CHECK:         [[TMP2:%.*]] = bitcast i1 [[YZ]] to i1
+; CHECK:         [[TMP3:%.*]] = bitcast i32 [[X]] to i32
+; CHECK:         [[TMP4:%.*]] = bitcast i1 [[XZ]] to i1
+; CHECK:         [[TMP5:%.*]] = bitcast i1 [[Z]] to i1
+; CHECK:         [[DOT0:%.*]] = bitcast i1 [[TMP5]] to i1
+; CHECK:         [[DOT01:%.*]] = bitcast i1 [[TMP4]] to i1
+; CHECK:         [[DOT02:%.*]] = bitcast i32 [[TMP3]] to i32
+; CHECK:         [[DOT03:%.*]] = bitcast i1 [[TMP2]] to i1
+; CHECK:         [[DOT04:%.*]] = bitcast i32 [[TMP1]] to i32
 ; CHECK-NEXT:    br i1 [[TMP5]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[DOT01]])
@@ -274,8 +274,8 @@ define void @testorassume(i32 %x, i32 %y) {
 ; CHECK-NEXT:    [[YZ:%.*]] = icmp eq i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    [[Z:%.*]] = or i1 [[XZ]], [[YZ]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[Z]])
-; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[Z]])
-; CHECK:         [[DOT0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[TMP1]])
+; CHECK:         [[TMP1:%.*]] = bitcast i1 [[Z]] to i1
+; CHECK:         [[DOT0:%.*]] = bitcast i1 [[TMP1]] to i1
 ; CHECK-NEXT:    br i1 [[TMP1]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @foo(i1 [[XZ]])
@@ -307,11 +307,11 @@ define void @test_and_one_unknown_cond(i32 %x, i1 %c1) {
 ; CHECK-LABEL: @test_and_one_unknown_cond(
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[A:%.*]] = and i1 [[C1:%.*]], [[C2]]
-; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
-; CHECK:         [[A_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
-; CHECK:         [[C1_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C1]])
-; CHECK:         [[C2_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C2]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[A_0:%.*]] = bitcast i1 [[A]] to i1
+; CHECK:         [[A_1:%.*]] = bitcast i1 [[A]] to i1
+; CHECK:         [[C1_0:%.*]] = bitcast i1 [[C1]] to i1
+; CHECK:         [[C2_0:%.*]] = bitcast i1 [[C2]] to i1
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
 ; CHECK-NEXT:    br i1 [[A]], label [[BOTH:%.*]], label [[NOPE:%.*]]
 ; CHECK:       both:
 ; CHECK-NEXT:    call void @bar(i32 [[X_0]])
@@ -349,11 +349,11 @@ define void @test_or_one_unknown_cond(i32 %x, i1 %c1) {
 ; CHECK-LABEL: @test_or_one_unknown_cond(
 ; CHECK-NEXT:    [[C2:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    [[A:%.*]] = or i1 [[C1:%.*]], [[C2]]
-; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
-; CHECK:         [[A_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
-; CHECK:         [[C1_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C1]])
-; CHECK:         [[C2_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C2]])
-; CHECK:         [[X_0:%.*]] = call i32 @llvm.ssa.copy.i32(i32 [[X]])
+; CHECK:         [[A_0:%.*]] = bitcast i1 [[A]] to i1
+; CHECK:         [[A_1:%.*]] = bitcast i1 [[A]] to i1
+; CHECK:         [[C1_0:%.*]] = bitcast i1 [[C1]] to i1
+; CHECK:         [[C2_0:%.*]] = bitcast i1 [[C2]] to i1
+; CHECK:         [[X_0:%.*]] = bitcast i32 [[X]] to i32
 ; CHECK-NEXT:    br i1 [[A]], label [[NOPE:%.*]], label [[BOTH_INVERTED:%.*]]
 ; CHECK:       both_inverted:
 ; CHECK-NEXT:    call void @bar(i32 [[X_0]])
@@ -391,12 +391,12 @@ define void @test_and_chain(i1 %a, i1 %b, i1 %c) {
 ; CHECK-LABEL: @test_and_chain(
 ; CHECK-NEXT:    [[AND1:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[AND1]], [[C:%.*]]
-; CHECK:         [[AND2_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND2]])
-; CHECK:         [[AND2_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND2]])
-; CHECK:         [[AND1_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND1]])
-; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
-; CHECK:         [[B_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[B]])
-; CHECK:         [[C_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C]])
+; CHECK:         [[AND2_0:%.*]] = bitcast i1 [[AND2]] to i1
+; CHECK:         [[AND2_1:%.*]] = bitcast i1 [[AND2]] to i1
+; CHECK:         [[AND1_0:%.*]] = bitcast i1 [[AND1]] to i1
+; CHECK:         [[A_0:%.*]] = bitcast i1 [[A]] to i1
+; CHECK:         [[B_0:%.*]] = bitcast i1 [[B]] to i1
+; CHECK:         [[C_0:%.*]] = bitcast i1 [[C]] to i1
 ; CHECK-NEXT:    br i1 [[AND2]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A_0]])
@@ -438,12 +438,12 @@ define void @test_or_chain(i1 %a, i1 %b, i1 %c) {
 ; CHECK-LABEL: @test_or_chain(
 ; CHECK-NEXT:    [[OR1:%.*]] = or i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[OR2:%.*]] = or i1 [[OR1]], [[C:%.*]]
-; CHECK:         [[OR2_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR2]])
-; CHECK:         [[OR2_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR2]])
-; CHECK:         [[OR1_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR1]])
-; CHECK:         [[A_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
-; CHECK:         [[B_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[B]])
-; CHECK:         [[C_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C]])
+; CHECK:         [[OR2_0:%.*]] = bitcast i1 [[OR2]] to i1
+; CHECK:         [[OR2_1:%.*]] = bitcast i1 [[OR2]] to i1
+; CHECK:         [[OR1_0:%.*]] = bitcast i1 [[OR1]] to i1
+; CHECK:         [[A_0:%.*]] = bitcast i1 [[A]] to i1
+; CHECK:         [[B_0:%.*]] = bitcast i1 [[B]] to i1
+; CHECK:         [[C_0:%.*]] = bitcast i1 [[C]] to i1
 ; CHECK-NEXT:    br i1 [[OR2]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A]])
@@ -485,10 +485,10 @@ define void @test_and_or_mixed(i1 %a, i1 %b, i1 %c) {
 ; CHECK-LABEL: @test_and_or_mixed(
 ; CHECK-NEXT:    [[OR:%.*]] = or i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[AND:%.*]] = and i1 [[OR]], [[C:%.*]]
-; CHECK:         [[AND_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND]])
-; CHECK:         [[AND_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND]])
-; CHECK:         [[OR_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR]])
-; CHECK:         [[C_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C]])
+; CHECK:         [[AND_0:%.*]] = bitcast i1 [[AND]] to i1
+; CHECK:         [[AND_1:%.*]] = bitcast i1 [[AND]] to i1
+; CHECK:         [[OR_0:%.*]] = bitcast i1 [[OR]] to i1
+; CHECK:         [[C_0:%.*]] = bitcast i1 [[C]] to i1
 ; CHECK-NEXT:    br i1 [[AND]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A]])
@@ -542,15 +542,15 @@ define void @test_deep_and_chain(i1 %a1) {
 ; CHECK-NEXT:    [[A13:%.*]] = and i1 [[A12]], true
 ; CHECK-NEXT:    [[A14:%.*]] = and i1 [[A13]], true
 ; CHECK-NEXT:    [[A15:%.*]] = and i1 [[A14]], true
-; CHECK:         [[A15_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
-; CHECK:         [[A15_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
-; CHECK:         [[A14_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A14]])
-; CHECK:         [[A13_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A13]])
-; CHECK:         [[A12_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A12]])
-; CHECK:         [[A11_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A11]])
-; CHECK:         [[A10_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A10]])
-; CHECK:         [[A9_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A9]])
-; CHECK:         [[A8_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A8]])
+; CHECK:         [[A15_0:%.*]] = bitcast i1 [[A15]] to i1
+; CHECK:         [[A15_1:%.*]] = bitcast i1 [[A15]] to i1
+; CHECK:         [[A14_0:%.*]] = bitcast i1 [[A14]] to i1
+; CHECK:         [[A13_0:%.*]] = bitcast i1 [[A13]] to i1
+; CHECK:         [[A12_0:%.*]] = bitcast i1 [[A12]] to i1
+; CHECK:         [[A11_0:%.*]] = bitcast i1 [[A11]] to i1
+; CHECK:         [[A10_0:%.*]] = bitcast i1 [[A10]] to i1
+; CHECK:         [[A9_0:%.*]] = bitcast i1 [[A9]] to i1
+; CHECK:         [[A8_0:%.*]] = bitcast i1 [[A8]] to i1
 ; CHECK-NEXT:    br i1 [[A15]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A1]])
@@ -656,15 +656,15 @@ define void @test_deep_and_tree(i1 %a1) {
 ; CHECK-NEXT:    [[A13:%.*]] = and i1 [[A12]], [[A12]]
 ; CHECK-NEXT:    [[A14:%.*]] = and i1 [[A13]], [[A13]]
 ; CHECK-NEXT:    [[A15:%.*]] = and i1 [[A14]], [[A14]]
-; CHECK:         [[A15_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
-; CHECK:         [[A15_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
-; CHECK:         [[A14_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A14]])
-; CHECK:         [[A13_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A13]])
-; CHECK:         [[A12_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A12]])
-; CHECK:         [[A11_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A11]])
-; CHECK:         [[A10_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A10]])
-; CHECK:         [[A9_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A9]])
-; CHECK:         [[A8_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A8]])
+; CHECK:         [[A15_0:%.*]] = bitcast i1 [[A15]] to i1
+; CHECK:         [[A15_1:%.*]] = bitcast i1 [[A15]] to i1
+; CHECK:         [[A14_0:%.*]] = bitcast i1 [[A14]] to i1
+; CHECK:         [[A13_0:%.*]] = bitcast i1 [[A13]] to i1
+; CHECK:         [[A12_0:%.*]] = bitcast i1 [[A12]] to i1
+; CHECK:         [[A11_0:%.*]] = bitcast i1 [[A11]] to i1
+; CHECK:         [[A10_0:%.*]] = bitcast i1 [[A10]] to i1
+; CHECK:         [[A9_0:%.*]] = bitcast i1 [[A9]] to i1
+; CHECK:         [[A8_0:%.*]] = bitcast i1 [[A8]] to i1
 ; CHECK-NEXT:    br i1 [[A15]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A1]])
@@ -770,15 +770,15 @@ define void @test_deep_or_tree(i1 %a1) {
 ; CHECK-NEXT:    [[A13:%.*]] = or i1 [[A12]], [[A12]]
 ; CHECK-NEXT:    [[A14:%.*]] = or i1 [[A13]], [[A13]]
 ; CHECK-NEXT:    [[A15:%.*]] = or i1 [[A14]], [[A14]]
-; CHECK:         [[A15_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
-; CHECK:         [[A15_1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
-; CHECK:         [[A14_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A14]])
-; CHECK:         [[A13_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A13]])
-; CHECK:         [[A12_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A12]])
-; CHECK:         [[A11_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A11]])
-; CHECK:         [[A10_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A10]])
-; CHECK:         [[A9_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A9]])
-; CHECK:         [[A8_0:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A8]])
+; CHECK:         [[A15_0:%.*]] = bitcast i1 [[A15]] to i1
+; CHECK:         [[A15_1:%.*]] = bitcast i1 [[A15]] to i1
+; CHECK:         [[A14_0:%.*]] = bitcast i1 [[A14]] to i1
+; CHECK:         [[A13_0:%.*]] = bitcast i1 [[A13]] to i1
+; CHECK:         [[A12_0:%.*]] = bitcast i1 [[A12]] to i1
+; CHECK:         [[A11_0:%.*]] = bitcast i1 [[A11]] to i1
+; CHECK:         [[A10_0:%.*]] = bitcast i1 [[A10]] to i1
+; CHECK:         [[A9_0:%.*]] = bitcast i1 [[A9]] to i1
+; CHECK:         [[A8_0:%.*]] = bitcast i1 [[A8]] to i1
 ; CHECK-NEXT:    br i1 [[A15]], label [[IF:%.*]], label [[ELSE:%.*]]
 ; CHECK:       if:
 ; CHECK-NEXT:    call void @foo(i1 [[A1]])
@@ -873,11 +873,11 @@ define void @test_assume_and_chain(i1 %a, i1 %b, i1 %c) {
 ; CHECK-NEXT:    [[AND1:%.*]] = and i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[AND2:%.*]] = and i1 [[AND1]], [[C:%.*]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[AND2]])
-; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[C]])
-; CHECK:         [[TMP2:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[B]])
-; CHECK:         [[TMP3:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A]])
-; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND1]])
-; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[AND2]])
+; CHECK:         [[TMP1:%.*]] = bitcast i1 [[C]] to i1
+; CHECK:         [[TMP2:%.*]] = bitcast i1 [[B]] to i1
+; CHECK:         [[TMP3:%.*]] = bitcast i1 [[A]] to i1
+; CHECK:         [[TMP4:%.*]] = bitcast i1 [[AND1]] to i1
+; CHECK:         [[TMP5:%.*]] = bitcast i1 [[AND2]] to i1
 ; CHECK-NEXT:    call void @foo(i1 [[TMP3]])
 ; CHECK-NEXT:    call void @foo(i1 [[TMP2]])
 ; CHECK-NEXT:    call void @foo(i1 [[TMP1]])
@@ -901,7 +901,7 @@ define void @test_assume_or_chain(i1 %a, i1 %b, i1 %c) {
 ; CHECK-NEXT:    [[OR1:%.*]] = or i1 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[OR2:%.*]] = or i1 [[OR1]], [[C:%.*]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[OR2]])
-; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[OR2]])
+; CHECK:         [[TMP1:%.*]] = bitcast i1 [[OR2]] to i1
 ; CHECK-NEXT:    call void @foo(i1 [[A]])
 ; CHECK-NEXT:    call void @foo(i1 [[B]])
 ; CHECK-NEXT:    call void @foo(i1 [[C]])
@@ -937,14 +937,14 @@ define void @test_assume_deep_and_tree(i1 %a1) {
 ; CHECK-NEXT:    [[A14:%.*]] = and i1 [[A13]], [[A13]]
 ; CHECK-NEXT:    [[A15:%.*]] = and i1 [[A14]], [[A14]]
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[A15]])
-; CHECK:         [[TMP1:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A8]])
-; CHECK:         [[TMP2:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A9]])
-; CHECK:         [[TMP3:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A10]])
-; CHECK:         [[TMP4:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A11]])
-; CHECK:         [[TMP5:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A12]])
-; CHECK:         [[TMP6:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A13]])
-; CHECK:         [[TMP7:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A14]])
-; CHECK:         [[TMP8:%.*]] = call i1 @llvm.ssa.copy.i1(i1 [[A15]])
+; CHECK:         [[TMP1:%.*]] = bitcast i1 [[A8]] to i1
+; CHECK:         [[TMP2:%.*]] = bitcast i1 [[A9]] to i1
+; CHECK:         [[TMP3:%.*]] = bitcast i1 [[A10]] to i1
+; CHECK:         [[TMP4:%.*]] = bitcast i1 [[A11]] to i1
+; CHECK:         [[TMP5:%.*]] = bitcast i1 [[A12]] to i1
+; CHECK:         [[TMP6:%.*]] = bitcast i1 [[A13]] to i1
+; CHECK:         [[TMP7:%.*]] = bitcast i1 [[A14]] to i1
+; CHECK:         [[TMP8:%.*]] = bitcast i1 [[A15]] to i1
 ; CHECK-NEXT:    call void @foo(i1 [[A1]])
 ; CHECK-NEXT:    call void @foo(i1 [[A2]])
 ; CHECK-NEXT:    call void @foo(i1 [[A3]])
diff --git a/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll b/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
index af5bd94c216f8..d9f6aed7d01c8 100644
--- a/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
+++ b/llvm/test/Transforms/Util/PredicateInfo/unnamed-types.ll
@@ -8,12 +8,12 @@
 ; CHECK-LABEL: bb:
 ; CHECK: Has predicate info
 ; CHECK: branch predicate info { TrueEdge: 1 Comparison:  %cmp1 = icmp ne ptr %arg, null Edge: [label %bb,label %bb1], RenamedOp: %arg }
-; CHECK-NEXT:  %arg.0 = call ptr @llvm.ssa.copy.p0(ptr %arg)
+; CHECK-NEXT:  %arg.0 = bitcast ptr %arg to ptr
 
 ; CHECK-LABEL: bb1:
 ; CHECK: Has predicate info
 ; CHECK-NEXT: branch predicate info { TrueEdge: 0 Comparison:  %cmp2 = icmp ne ptr null, %tmp Edge: [label %bb1,label %bb3], RenamedOp: %tmp }
-; CHECK-NEXT: %tmp.0 = call ptr @llvm.ssa.copy.p0(ptr %tmp)
+; CHECK-NEXT: %tmp.0 = bitcast ptr %tmp to ptr
 
 define void @f0(ptr %arg, ptr %tmp) {
 bb:
diff --git a/llvm/test/Transforms/Util/dbg-call-bitcast.ll b/llvm/test/Transforms/Util/dbg-call-bitcast.ll
index d8d80ab0f457a..f0c579cbd80ca 100644
--- a/llvm/test/Transforms/Util/dbg-call-bitcast.ll
+++ b/llvm/test/Transforms/Util/dbg-call-bitcast.ll
@@ -2,7 +2,7 @@
 
 define dso_local void @_Z1fv() {
   %1 = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %1)
   call void @llvm.dbg.declare(metadata ptr %1, metadata !16, metadata !DIExpression()), !dbg !19
 ; CHECK: %[[A:.*]] = alloca i32, align 4
 ; CHECK: #dbg_value(ptr %[[A]], {{.*}}, !DIExpression(DW_OP_deref)
@@ -11,13 +11,13 @@ define dso_local void @_Z1fv() {
 ; CHECK-NOT: #dbg_value
 ; CHECK: call void @_Z1gPv
   call void @_Z1gPv(ptr nonnull %1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
   ret void, !dbg !21
 }
 
 define dso_local void @_Z2fv() {
   %1 = alloca i32, align 4
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %1)
+  call void @llvm.lifetime.start.p0(ptr nonnull %1)
   call void @llvm.dbg.declare(metadata ptr %1, metadata !16, metadata !DIExpression()), !dbg !19
 ; CHECK: %[[A:.*]] = alloca i32, align 4
 ; CHECK: #dbg_value(ptr %[[A]], {{.*}}, !DIExpression(DW_OP_deref)
@@ -29,14 +29,14 @@ block2:
 ; CHECK: #dbg_value(ptr %[[A]], {{.*}}, !DIExpression(DW_OP_deref)
 ; CHECK: call void @_Z1gPv
   call void @_Z1gPv(ptr nonnull %1)
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %1)
+  call void @llvm.lifetime.end.p0(ptr nonnull %1)
   ret void, !dbg !21
 }
 
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
 declare void @llvm.dbg.declare(metadata, metadata, metadata)
 declare dso_local void @_Z1gPv(ptr)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9, !10}
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/narrow-phi-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/AArch64/narrow-phi-of-shuffles.ll
new file mode 100644
index 0000000000000..0067c922b3892
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AArch64/narrow-phi-of-shuffles.ll
@@ -0,0 +1,1266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=aarch64 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK
+
+define <2 x i8> @shuffle_v2i8(<2 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i8> @shuffle_v2i8(
+; CHECK-SAME: <2 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i8> %val3
+}
+
+define <4 x i8> @shuffle_v4i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i8> @shuffle_v4i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i8> %val3
+}
+
+define <8 x i8> @shuffle_v8i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i8> @shuffle_v8i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i8> %val3
+}
+
+define <16 x i8> @shuffle_v16i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i8> @shuffle_v16i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i8> %val3
+}
+
+define <32 x i8> @shuffle_v32i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i8> @shuffle_v32i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i8> %val3
+}
+
+define <2 x i16> @shuffle_v2i16(<2 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i16> @shuffle_v2i16(
+; CHECK-SAME: <2 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i16> %val3
+}
+
+define <4 x i16> @shuffle_v4i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i16> @shuffle_v4i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i16> %val3
+}
+
+define <8 x i16> @shuffle_v8i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i16> @shuffle_v8i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i16> %val3
+}
+
+define <16 x i16> @shuffle_v16i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i16> @shuffle_v16i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i16> %val3
+}
+
+define <32 x i16> @shuffle_v32i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i16> @shuffle_v32i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i16> %val3
+}
+
+define <2 x i32> @shuffle_v2i32(<2 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i32> @shuffle_v2i32(
+; CHECK-SAME: <2 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i32> %val3
+}
+
+define <4 x i32> @shuffle_v4i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i32> @shuffle_v4i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v8i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i32> @shuffle_v8i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <16 x i32> @shuffle_v16i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i32> @shuffle_v16i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i32> %val3
+}
+
+define <32 x i32> @shuffle_v32i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i32> @shuffle_v32i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i32> %val3
+}
+
+define <2 x bfloat> @shuffle_v2bf16(<2 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x bfloat> @shuffle_v2bf16(
+; CHECK-SAME: <2 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x bfloat> %val3
+}
+
+define <3 x bfloat> @shuffle_v3bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x bfloat> @shuffle_v3bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x bfloat> %val3
+}
+
+define <4 x bfloat> @shuffle_v4bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x bfloat> %val3
+}
+
+define <6 x bfloat> @shuffle_v6bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x bfloat> %val3
+}
+
+define <8 x bfloat> @shuffle_v8bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x bfloat> %val3
+}
+
+define <16 x bfloat> @shuffle_v16bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x bfloat> %val3
+}
+
+define <32 x bfloat> @shuffle_v32bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x bfloat> %val3
+}
+
+define <2 x half> @shuffle_v2f16(<2 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x half> @shuffle_v2f16(
+; CHECK-SAME: <2 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x half> %val3
+}
+
+define <3 x half> @shuffle_v3f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x half> @shuffle_v3f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x half> %val3
+}
+
+define <4 x half> @shuffle_v4f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x half> %val3
+}
+
+define <6 x half> @shuffle_v6f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x half> %val3
+}
+
+define <8 x half> @shuffle_v8f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x half> %val3
+}
+
+define <16 x half> @shuffle_v16f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x half> %val3
+}
+
+define <32 x half> @shuffle_v32f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x half> %val3
+}
+
+define <2 x float> @shuffle_v2f32(<2 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x float> @shuffle_v2f32(
+; CHECK-SAME: <2 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x float> %val3
+}
+
+define <3 x float> @shuffle_v3f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x float> @shuffle_v3f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x float> %val3
+}
+
+define <4 x float> @shuffle_v4f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x float> @shuffle_v4f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x float> %val3
+}
+
+define <6 x float> @shuffle_v6f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x float> @shuffle_v6f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x float> %val3
+}
+
+define <8 x float> @shuffle_v8f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x float> @shuffle_v8f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x float> %val3
+}
+
+define <16 x float> @shuffle_v16f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x float> @shuffle_v16f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x float> %val3
+}
+
+define <32 x float> @shuffle_v32f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x float> @shuffle_v32f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x float> %val3
+}
+
+declare void @func0() local_unnamed_addr
+
+declare void @func1() local_unnamed_addr
diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
index a40d514a520ca..acbc836ffcab0 100644
--- a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
@@ -716,10 +716,9 @@ define <4 x i64> @zext_chain(<4 x i16> %x) {
 
 define <4 x i32> @add_chain(<4 x i32> %x) {
 ; CHECK-LABEL: @add_chain(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i32> [[SHUF]], [[SHUF]]
-; CHECK-NEXT:    [[ADD2:%.*]] = add <4 x i32> [[ADD]], [[ADD]]
-; CHECK-NEXT:    [[REVSHUF:%.*]] = shufflevector <4 x i32> [[ADD2]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i32> [[SHUF:%.*]], [[SHUF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHUF]], [[SHUF]]
+; CHECK-NEXT:    [[REVSHUF:%.*]] = add <4 x i32> [[ADD]], [[TMP2]]
 ; CHECK-NEXT:    ret <4 x i32> [[REVSHUF]]
 ;
   %shuf = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
@@ -731,10 +730,9 @@ define <4 x i32> @add_chain(<4 x i32> %x) {
 
 define <4 x i64> @zext_add_chain(<4 x i32> %x) {
 ; CHECK-LABEL: @zext_add_chain(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-; CHECK-NEXT:    [[ZEXT:%.*]] = zext <4 x i32> [[SHUF]] to <4 x i64>
-; CHECK-NEXT:    [[ADD:%.*]] = add <4 x i64> [[ZEXT]], [[ZEXT]]
-; CHECK-NEXT:    [[REVSHUF:%.*]] = shufflevector <4 x i64> [[ADD]], <4 x i64> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext <4 x i32> [[SHUF:%.*]] to <4 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i32> [[SHUF]] to <4 x i64>
+; CHECK-NEXT:    [[REVSHUF:%.*]] = add <4 x i64> [[ZEXT]], [[TMP2]]
 ; CHECK-NEXT:    ret <4 x i64> [[REVSHUF]]
 ;
   %shuf = shufflevector <4 x i32> %x, <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
diff --git a/llvm/test/Transforms/VectorCombine/AMDGPU/narrow-phi-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/AMDGPU/narrow-phi-of-shuffles.ll
new file mode 100644
index 0000000000000..8c504843d87d8
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AMDGPU/narrow-phi-of-shuffles.ll
@@ -0,0 +1,1266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK
+
+define <2 x i8> @shuffle_v2i8(<2 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i8> @shuffle_v2i8(
+; CHECK-SAME: <2 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i8> %val3
+}
+
+define <4 x i8> @shuffle_v4i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i8> @shuffle_v4i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i8> %val3
+}
+
+define <8 x i8> @shuffle_v8i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i8> @shuffle_v8i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i8> %val3
+}
+
+define <16 x i8> @shuffle_v16i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i8> @shuffle_v16i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i8> %val3
+}
+
+define <32 x i8> @shuffle_v32i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i8> @shuffle_v32i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i8> %val3
+}
+
+define <2 x i16> @shuffle_v2i16(<2 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i16> @shuffle_v2i16(
+; CHECK-SAME: <2 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i16> %val3
+}
+
+define <4 x i16> @shuffle_v4i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i16> @shuffle_v4i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i16> %val3
+}
+
+define <8 x i16> @shuffle_v8i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i16> @shuffle_v8i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i16> %val3
+}
+
+define <16 x i16> @shuffle_v16i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i16> @shuffle_v16i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i16> %val3
+}
+
+define <32 x i16> @shuffle_v32i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i16> @shuffle_v32i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i16> %val3
+}
+
+define <2 x i32> @shuffle_v2i32(<2 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i32> @shuffle_v2i32(
+; CHECK-SAME: <2 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i32> %val3
+}
+
+define <4 x i32> @shuffle_v4i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i32> @shuffle_v4i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x i32> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x i32> [[TMP1]], <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <4 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v8i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i32> @shuffle_v8i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x i32> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x i32> [[TMP1]], <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <16 x i32> @shuffle_v16i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i32> @shuffle_v16i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x i32> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x i32> [[TMP1]], <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <16 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i32> %val3
+}
+
+define <32 x i32> @shuffle_v32i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i32> @shuffle_v32i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x i32> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x i32> [[TMP1]], <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <32 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i32> %val3
+}
+
+define <2 x bfloat> @shuffle_v2bf16(<2 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x bfloat> @shuffle_v2bf16(
+; CHECK-SAME: <2 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x bfloat> %val3
+}
+
+define <3 x bfloat> @shuffle_v3bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x bfloat> @shuffle_v3bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x bfloat> %val3
+}
+
+define <4 x bfloat> @shuffle_v4bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x bfloat> %val3
+}
+
+define <6 x bfloat> @shuffle_v6bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x bfloat> %val3
+}
+
+define <8 x bfloat> @shuffle_v8bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x bfloat> %val3
+}
+
+define <16 x bfloat> @shuffle_v16bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x bfloat> %val3
+}
+
+define <32 x bfloat> @shuffle_v32bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x bfloat> %val3
+}
+
+define <2 x half> @shuffle_v2f16(<2 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x half> @shuffle_v2f16(
+; CHECK-SAME: <2 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x half> %val3
+}
+
+define <3 x half> @shuffle_v3f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x half> @shuffle_v3f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x half> %val3
+}
+
+define <4 x half> @shuffle_v4f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x half> %val3
+}
+
+define <6 x half> @shuffle_v6f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x half> %val3
+}
+
+define <8 x half> @shuffle_v8f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x half> %val3
+}
+
+define <16 x half> @shuffle_v16f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x half> %val3
+}
+
+define <32 x half> @shuffle_v32f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x half> %val3
+}
+
+define <2 x float> @shuffle_v2f32(<2 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x float> @shuffle_v2f32(
+; CHECK-SAME: <2 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x float> %val3
+}
+
+define <3 x float> @shuffle_v3f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x float> @shuffle_v3f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x float> %val3
+}
+
+define <4 x float> @shuffle_v4f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x float> @shuffle_v4f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x float> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <4 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x float> %val3
+}
+
+define <6 x float> @shuffle_v6f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x float> @shuffle_v6f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x float> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <6 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x float> %val3
+}
+
+define <8 x float> @shuffle_v8f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x float> @shuffle_v8f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x float> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <8 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x float> %val3
+}
+
+define <16 x float> @shuffle_v16f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x float> @shuffle_v16f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x float> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <16 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x float> %val3
+}
+
+define <32 x float> @shuffle_v32f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x float> @shuffle_v32f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x float> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <32 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x float> %val3
+}
+
+declare void @func0() local_unnamed_addr
+
+declare void @func1() local_unnamed_addr
diff --git a/llvm/test/Transforms/VectorCombine/ARM/narrow-phi-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/ARM/narrow-phi-of-shuffles.ll
new file mode 100644
index 0000000000000..5263215ae209e
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/ARM/narrow-phi-of-shuffles.ll
@@ -0,0 +1,1266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=arm -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK
+
+define <2 x i8> @shuffle_v2i8(<2 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i8> @shuffle_v2i8(
+; CHECK-SAME: <2 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i8> %val3
+}
+
+define <4 x i8> @shuffle_v4i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i8> @shuffle_v4i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i8> %val3
+}
+
+define <8 x i8> @shuffle_v8i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i8> @shuffle_v8i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i8> %val3
+}
+
+define <16 x i8> @shuffle_v16i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i8> @shuffle_v16i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i8> %val3
+}
+
+define <32 x i8> @shuffle_v32i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i8> @shuffle_v32i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i8> %val3
+}
+
+define <2 x i16> @shuffle_v2i16(<2 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i16> @shuffle_v2i16(
+; CHECK-SAME: <2 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i16> %val3
+}
+
+define <4 x i16> @shuffle_v4i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i16> @shuffle_v4i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i16> %val3
+}
+
+define <8 x i16> @shuffle_v8i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i16> @shuffle_v8i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i16> %val3
+}
+
+define <16 x i16> @shuffle_v16i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i16> @shuffle_v16i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i16> %val3
+}
+
+define <32 x i16> @shuffle_v32i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i16> @shuffle_v32i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i16> %val3
+}
+
+define <2 x i32> @shuffle_v2i32(<2 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i32> @shuffle_v2i32(
+; CHECK-SAME: <2 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i32> %val3
+}
+
+define <4 x i32> @shuffle_v4i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i32> @shuffle_v4i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v8i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i32> @shuffle_v8i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <16 x i32> @shuffle_v16i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i32> @shuffle_v16i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i32> %val3
+}
+
+define <32 x i32> @shuffle_v32i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i32> @shuffle_v32i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i32> %val3
+}
+
+define <2 x bfloat> @shuffle_v2bf16(<2 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x bfloat> @shuffle_v2bf16(
+; CHECK-SAME: <2 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x bfloat> %val3
+}
+
+define <3 x bfloat> @shuffle_v3bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x bfloat> @shuffle_v3bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x bfloat> %val3
+}
+
+define <4 x bfloat> @shuffle_v4bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x bfloat> %val3
+}
+
+define <6 x bfloat> @shuffle_v6bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x bfloat> %val3
+}
+
+define <8 x bfloat> @shuffle_v8bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x bfloat> %val3
+}
+
+define <16 x bfloat> @shuffle_v16bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x bfloat> %val3
+}
+
+define <32 x bfloat> @shuffle_v32bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x bfloat> %val3
+}
+
+define <2 x half> @shuffle_v2f16(<2 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x half> @shuffle_v2f16(
+; CHECK-SAME: <2 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x half> %val3
+}
+
+define <3 x half> @shuffle_v3f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x half> @shuffle_v3f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x half> %val3
+}
+
+define <4 x half> @shuffle_v4f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x half> %val3
+}
+
+define <6 x half> @shuffle_v6f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x half> %val3
+}
+
+define <8 x half> @shuffle_v8f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x half> %val3
+}
+
+define <16 x half> @shuffle_v16f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x half> %val3
+}
+
+define <32 x half> @shuffle_v32f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x half> %val3
+}
+
+define <2 x float> @shuffle_v2f32(<2 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x float> @shuffle_v2f32(
+; CHECK-SAME: <2 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x float> %val3
+}
+
+define <3 x float> @shuffle_v3f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x float> @shuffle_v3f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x float> %val3
+}
+
+define <4 x float> @shuffle_v4f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x float> @shuffle_v4f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x float> %val3
+}
+
+define <6 x float> @shuffle_v6f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x float> @shuffle_v6f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x float> %val3
+}
+
+define <8 x float> @shuffle_v8f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x float> @shuffle_v8f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x float> %val3
+}
+
+define <16 x float> @shuffle_v16f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x float> @shuffle_v16f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x float> %val3
+}
+
+define <32 x float> @shuffle_v32f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x float> @shuffle_v32f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x float> %val3
+}
+
+declare void @func0() local_unnamed_addr
+
+declare void @func1() local_unnamed_addr
diff --git a/llvm/test/Transforms/VectorCombine/Hexagon/narrow-phi-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/Hexagon/narrow-phi-of-shuffles.ll
new file mode 100644
index 0000000000000..d95ead49d38ae
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/Hexagon/narrow-phi-of-shuffles.ll
@@ -0,0 +1,1266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=hexagon-- -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK
+
+define <2 x i8> @shuffle_v2i8(<2 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i8> @shuffle_v2i8(
+; CHECK-SAME: <2 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i8> %val3
+}
+
+define <4 x i8> @shuffle_v4i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i8> @shuffle_v4i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i8> %val3
+}
+
+define <8 x i8> @shuffle_v8i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i8> @shuffle_v8i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i8> %val3
+}
+
+define <16 x i8> @shuffle_v16i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i8> @shuffle_v16i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i8> %val3
+}
+
+define <32 x i8> @shuffle_v32i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i8> @shuffle_v32i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i8> %val3
+}
+
+define <2 x i16> @shuffle_v2i16(<2 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i16> @shuffle_v2i16(
+; CHECK-SAME: <2 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i16> %val3
+}
+
+define <4 x i16> @shuffle_v4i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i16> @shuffle_v4i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i16> %val3
+}
+
+define <8 x i16> @shuffle_v8i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i16> @shuffle_v8i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i16> %val3
+}
+
+define <16 x i16> @shuffle_v16i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i16> @shuffle_v16i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i16> %val3
+}
+
+define <32 x i16> @shuffle_v32i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i16> @shuffle_v32i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i16> %val3
+}
+
+define <2 x i32> @shuffle_v2i32(<2 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i32> @shuffle_v2i32(
+; CHECK-SAME: <2 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i32> %val3
+}
+
+define <4 x i32> @shuffle_v4i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i32> @shuffle_v4i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v8i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i32> @shuffle_v8i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <16 x i32> @shuffle_v16i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i32> @shuffle_v16i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i32> %val3
+}
+
+define <32 x i32> @shuffle_v32i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i32> @shuffle_v32i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i32> %val3
+}
+
+define <2 x bfloat> @shuffle_v2bf16(<2 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x bfloat> @shuffle_v2bf16(
+; CHECK-SAME: <2 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x bfloat> %val3
+}
+
+define <3 x bfloat> @shuffle_v3bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x bfloat> @shuffle_v3bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x bfloat> %val3
+}
+
+define <4 x bfloat> @shuffle_v4bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x bfloat> %val3
+}
+
+define <6 x bfloat> @shuffle_v6bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x bfloat> %val3
+}
+
+define <8 x bfloat> @shuffle_v8bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x bfloat> %val3
+}
+
+define <16 x bfloat> @shuffle_v16bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x bfloat> %val3
+}
+
+define <32 x bfloat> @shuffle_v32bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x bfloat> %val3
+}
+
+define <2 x half> @shuffle_v2f16(<2 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x half> @shuffle_v2f16(
+; CHECK-SAME: <2 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x half> %val3
+}
+
+define <3 x half> @shuffle_v3f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x half> @shuffle_v3f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x half> %val3
+}
+
+define <4 x half> @shuffle_v4f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x half> %val3
+}
+
+define <6 x half> @shuffle_v6f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x half> %val3
+}
+
+define <8 x half> @shuffle_v8f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x half> %val3
+}
+
+define <16 x half> @shuffle_v16f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x half> %val3
+}
+
+define <32 x half> @shuffle_v32f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x half> %val3
+}
+
+define <2 x float> @shuffle_v2f32(<2 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x float> @shuffle_v2f32(
+; CHECK-SAME: <2 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x float> %val3
+}
+
+define <3 x float> @shuffle_v3f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x float> @shuffle_v3f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x float> %val3
+}
+
+define <4 x float> @shuffle_v4f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x float> @shuffle_v4f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x float> %val3
+}
+
+define <6 x float> @shuffle_v6f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x float> @shuffle_v6f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x float> %val3
+}
+
+define <8 x float> @shuffle_v8f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x float> @shuffle_v8f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x float> %val3
+}
+
+define <16 x float> @shuffle_v16f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x float> @shuffle_v16f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x float> %val3
+}
+
+define <32 x float> @shuffle_v32f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x float> @shuffle_v32f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x float> %val3
+}
+
+declare void @func0() local_unnamed_addr
+
+declare void @func1() local_unnamed_addr
diff --git a/llvm/test/Transforms/VectorCombine/RISCV/narrow-phi-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/RISCV/narrow-phi-of-shuffles.ll
new file mode 100644
index 0000000000000..5e3b9a6a7e258
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/RISCV/narrow-phi-of-shuffles.ll
@@ -0,0 +1,1266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=riscv64 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK
+
+define <2 x i8> @shuffle_v2i8(<2 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i8> @shuffle_v2i8(
+; CHECK-SAME: <2 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i8> %val3
+}
+
+define <4 x i8> @shuffle_v4i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i8> @shuffle_v4i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i8> %val3
+}
+
+define <8 x i8> @shuffle_v8i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i8> @shuffle_v8i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i8> %val3
+}
+
+define <16 x i8> @shuffle_v16i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i8> @shuffle_v16i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i8> %val3
+}
+
+define <32 x i8> @shuffle_v32i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i8> @shuffle_v32i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i8> %val3
+}
+
+define <2 x i16> @shuffle_v2i16(<2 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i16> @shuffle_v2i16(
+; CHECK-SAME: <2 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i16> %val3
+}
+
+define <4 x i16> @shuffle_v4i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i16> @shuffle_v4i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i16> %val3
+}
+
+define <8 x i16> @shuffle_v8i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i16> @shuffle_v8i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i16> %val3
+}
+
+define <16 x i16> @shuffle_v16i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i16> @shuffle_v16i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i16> %val3
+}
+
+define <32 x i16> @shuffle_v32i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i16> @shuffle_v32i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i16> %val3
+}
+
+define <2 x i32> @shuffle_v2i32(<2 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i32> @shuffle_v2i32(
+; CHECK-SAME: <2 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i32> %val3
+}
+
+define <4 x i32> @shuffle_v4i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i32> @shuffle_v4i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v8i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i32> @shuffle_v8i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <16 x i32> @shuffle_v16i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i32> @shuffle_v16i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i32> %val3
+}
+
+define <32 x i32> @shuffle_v32i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i32> @shuffle_v32i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i32> %val3
+}
+
+define <2 x bfloat> @shuffle_v2bf16(<2 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x bfloat> @shuffle_v2bf16(
+; CHECK-SAME: <2 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x bfloat> %val3
+}
+
+define <3 x bfloat> @shuffle_v3bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x bfloat> @shuffle_v3bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x bfloat> %val3
+}
+
+define <4 x bfloat> @shuffle_v4bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x bfloat> %val3
+}
+
+define <6 x bfloat> @shuffle_v6bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x bfloat> %val3
+}
+
+define <8 x bfloat> @shuffle_v8bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x bfloat> %val3
+}
+
+define <16 x bfloat> @shuffle_v16bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x bfloat> %val3
+}
+
+define <32 x bfloat> @shuffle_v32bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x bfloat> %val3
+}
+
+define <2 x half> @shuffle_v2f16(<2 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x half> @shuffle_v2f16(
+; CHECK-SAME: <2 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x half> %val3
+}
+
+define <3 x half> @shuffle_v3f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x half> @shuffle_v3f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x half> %val3
+}
+
+define <4 x half> @shuffle_v4f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x half> %val3
+}
+
+define <6 x half> @shuffle_v6f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x half> %val3
+}
+
+define <8 x half> @shuffle_v8f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x half> %val3
+}
+
+define <16 x half> @shuffle_v16f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x half> %val3
+}
+
+define <32 x half> @shuffle_v32f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x half> %val3
+}
+
+define <2 x float> @shuffle_v2f32(<2 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x float> @shuffle_v2f32(
+; CHECK-SAME: <2 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x float> %val3
+}
+
+define <3 x float> @shuffle_v3f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x float> @shuffle_v3f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x float> %val3
+}
+
+define <4 x float> @shuffle_v4f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x float> @shuffle_v4f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x float> %val3
+}
+
+define <6 x float> @shuffle_v6f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x float> @shuffle_v6f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x float> %val3
+}
+
+define <8 x float> @shuffle_v8f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x float> @shuffle_v8f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x float> %val3
+}
+
+define <16 x float> @shuffle_v16f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x float> @shuffle_v16f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x float> %val3
+}
+
+define <32 x float> @shuffle_v32f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x float> @shuffle_v32f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x float> %val3
+}
+
+declare void @func0() local_unnamed_addr
+
+declare void @func1() local_unnamed_addr
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
index d369279c15db4..41d77e89476ba 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
@@ -268,7 +268,7 @@ define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: @ext5_ext0_add(
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 5
@@ -294,7 +294,7 @@ define float @ext1_ext0_fmul(<4 x float> %x) {
 ; CHECK-LABEL: @ext1_ext0_fmul(
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[SHIFT]], [[X]]
-; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %e0 = extractelement <4 x float> %x, i32 1
@@ -363,7 +363,7 @@ define float @ext7_ext4_fmul_v8f32(<8 x float> %x) {
 ; AVX-LABEL: @ext7_ext4_fmul_v8f32(
 ; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 7, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[SHIFT]], [[X]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i64 4
+; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
 ; AVX-NEXT:    ret float [[R]]
 ;
   %e0 = extractelement <8 x float> %x, i32 7
@@ -484,7 +484,7 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP1]], [[SHIFT1]]
 ; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[SHIFT2]], [[TMP2]]
-; CHECK-NEXT:    [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret i32 [[Z0123]]
 ;
   %z = and <4 x i32> %x, %y
@@ -504,7 +504,7 @@ define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
 ; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
-; CHECK-NEXT:    [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
 ; CHECK-NEXT:    ret i32 [[X210]]
 ;
   %x0 = extractelement <4 x i32> %x, i32 0
@@ -523,7 +523,7 @@ define i32 @ext_ext_partial_add_reduction_and_extra_add_v4i32(<4 x i32> %x, <4 x
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
 ; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHIFT2]], [[TMP2]]
-; CHECK-NEXT:    [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret i32 [[X2Y210]]
 ;
   %y0 = extractelement <4 x i32> %y, i32 0
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
index d11fb1426c94e..4c1ca82b2bd06 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop.ll
@@ -268,7 +268,7 @@ define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
 ; CHECK-LABEL: @ext5_ext0_add(
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> poison, <16 x i32> <i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
+; CHECK-NEXT:    [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %e0 = extractelement <16 x i8> %x, i32 5
@@ -294,7 +294,7 @@ define float @ext1_ext0_fmul(<4 x float> %x) {
 ; CHECK-LABEL: @ext1_ext0_fmul(
 ; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[SHIFT]], [[X]]
-; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
+; CHECK-NEXT:    [[R:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
 ; CHECK-NEXT:    ret float [[R]]
 ;
   %e0 = extractelement <4 x float> %x, i32 1
@@ -363,7 +363,7 @@ define float @ext7_ext4_fmul_v8f32(<8 x float> %x) {
 ; AVX-LABEL: @ext7_ext4_fmul_v8f32(
 ; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 7, i32 poison, i32 poison, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[SHIFT]], [[X]]
-; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i64 4
+; AVX-NEXT:    [[R:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
 ; AVX-NEXT:    ret float [[R]]
 ;
   %e0 = extractelement <8 x float> %x, i32 7
@@ -490,7 +490,7 @@ define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = or <4 x i32> [[TMP1]], [[SHIFT1]]
 ; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> poison, <4 x i32> <i32 3, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = or <4 x i32> [[SHIFT2]], [[TMP2]]
-; CHECK-NEXT:    [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret i32 [[Z0123]]
 ;
   %z = and <4 x i32> %x, %y
@@ -510,7 +510,7 @@ define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
 ; CHECK-NEXT:    [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
-; CHECK-NEXT:    [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT:    [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
 ; CHECK-NEXT:    ret i32 [[X210]]
 ;
   %x0 = extractelement <4 x i32> %x, i32 0
@@ -529,7 +529,7 @@ define i32 @ext_ext_partial_add_reduction_and_extra_add_v4i32(<4 x i32> %x, <4 x
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
 ; CHECK-NEXT:    [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHIFT2]], [[TMP2]]
-; CHECK-NEXT:    [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT:    [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
 ; CHECK-NEXT:    ret i32 [[X2Y210]]
 ;
   %y0 = extractelement <4 x i32> %y, i32 0
@@ -573,10 +573,8 @@ define i64 @instsimplify_folder_crash(<4 x i64> %in) {
 ; CHECK-LABEL: @instsimplify_folder_crash(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[SHUFFLE_1:%.*]] = shufflevector <4 x i64> [[IN:%.*]], <4 x i64> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-; CHECK-NEXT:    [[E_0:%.*]] = extractelement <4 x i64> zeroinitializer, i64 0
-; CHECK-NEXT:    [[E_1:%.*]] = extractelement <4 x i64> [[SHUFFLE_1]], i64 1
-; CHECK-NEXT:    [[OR:%.*]] = or i64 [[E_1]], [[E_0]]
-; CHECK-NEXT:    ret i64 [[OR]]
+; CHECK-NEXT:    [[SHIFT:%.*]] = shufflevector <4 x i64> [[SHUFFLE_1]], <4 x i64> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    ret i64 0
 ;
 entry:
   %shuffle.1 = shufflevector <4 x i64> %in, <4 x i64> zeroinitializer, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll b/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
index 3dae93665b1ed..795832f22b096 100644
--- a/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-cmp.ll
@@ -130,7 +130,7 @@ define i1 @cmp10_v2f64(<2 x double> %x, <2 x double> %y) {
 ; AVX-LABEL: @cmp10_v2f64(
 ; AVX-NEXT:    [[SHIFT:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
 ; AVX-NEXT:    [[TMP1:%.*]] = fcmp ule <2 x double> [[SHIFT]], [[Y:%.*]]
-; AVX-NEXT:    [[CMP:%.*]] = extractelement <2 x i1> [[TMP1]], i64 0
+; AVX-NEXT:    [[CMP:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
 ; AVX-NEXT:    ret i1 [[CMP]]
 ;
   %x1 = extractelement <2 x double> %x, i32 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/intrinsic-scalarize.ll b/llvm/test/Transforms/VectorCombine/X86/intrinsic-scalarize.ll
index 5f3229398792a..ad5f5a7107c2b 100644
--- a/llvm/test/Transforms/VectorCombine/X86/intrinsic-scalarize.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/intrinsic-scalarize.ll
@@ -13,8 +13,7 @@ define <2 x float> @maxnum(float %x, float %y) {
 ; AVX2-LABEL: define <2 x float> @maxnum(
 ; AVX2-SAME: float [[X:%.*]], float [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
 ; AVX2-NEXT:    [[V_SCALAR:%.*]] = call float @llvm.maxnum.f32(float [[X]], float [[Y]])
-; AVX2-NEXT:    [[TMP1:%.*]] = call <2 x float> @llvm.maxnum.v2f32(<2 x float> poison, <2 x float> poison)
-; AVX2-NEXT:    [[V:%.*]] = insertelement <2 x float> [[TMP1]], float [[V_SCALAR]], i64 0
+; AVX2-NEXT:    [[V:%.*]] = insertelement <2 x float> poison, float [[V_SCALAR]], i64 0
 ; AVX2-NEXT:    ret <2 x float> [[V]]
 ;
   %x.insert = insertelement <2 x float> poison, float %x, i32 0
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll b/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll
index b26e5ec2698a5..50e32b79a91c2 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-extractelement-scalarization.ll
@@ -27,6 +27,8 @@ define void @multiple_extract(ptr %p) {
 ; infinite loop if we fold an extract that is waiting to be erased
 define void @unused_extract(ptr %p) {
 ; CHECK-LABEL: @unused_extract(
+; CHECK-NEXT:    [[LOAD:%.*]] = load <4 x float>, ptr [[P:%.*]], align 8
+; CHECK-NEXT:    [[EXTRACT:%.*]] = extractelement <4 x float> [[LOAD]], i64 1
 ; CHECK-NEXT:    ret void
 ;
   %load = load <4 x float>, ptr %p, align 8
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
index 977da754ec5a7..e8381d1b206e2 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
@@ -252,9 +252,9 @@ define <4 x i32> @unsafe_load_i32_insert_v4i32_addrspace(ptr align 16 dereferenc
 define <8 x i16> @gep01_load_i16_insert_v8i16(ptr align 16 dereferenceable(18) %p) nofree nosync {
 ; CHECK-LABEL: @gep01_load_i16_insert_v8i16(
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 0, i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[GEP]], align 2
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    ret <8 x i16> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = load <8 x i16>, ptr [[GEP]], align 2
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <8 x i16> [[R]], <8 x i16> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    ret <8 x i16> [[R1]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 0, i64 1
   %s = load i16, ptr %gep, align 2
@@ -341,9 +341,9 @@ define <4 x i32> @gep013_bitcast_load_i32_insert_v4i32(ptr align 1 dereferenceab
 define <8 x i16> @gep10_load_i16_insert_v8i16(ptr align 16 dereferenceable(32) %p) nofree nosync {
 ; CHECK-LABEL: @gep10_load_i16_insert_v8i16(
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds <8 x i16>, ptr [[P:%.*]], i64 1, i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[GEP]], align 16
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    ret <8 x i16> [[R]]
+; CHECK-NEXT:    [[R:%.*]] = load <8 x i16>, ptr [[GEP]], align 16
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <8 x i16> [[R]], <8 x i16> poison, <8 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    ret <8 x i16> [[R1]]
 ;
   %gep = getelementptr inbounds <8 x i16>, ptr %p, i64 1, i64 0
   %s = load i16, ptr %gep, align 16
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-widening.ll b/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
index 30a089818074e..eacc40bfa9b53 100644
--- a/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/load-widening.ll
@@ -443,8 +443,8 @@ define <8 x float> @load_v2f32_v8f32_hwasan(ptr dereferenceable(32) %p) sanitize
 
 define <4 x i32> @load_v2i32_v4i32_asan(ptr dereferenceable(16) %p) sanitize_address {
 ; CHECK-LABEL: @load_v2i32_v4i32_asan(
-; CHECK-NEXT:    [[L:%.*]] = load <2 x i32>, ptr [[P:%.*]], align 1
-; CHECK-NEXT:    [[S:%.*]] = shufflevector <2 x i32> [[L]], <2 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x i32>, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    [[S:%.*]] = shufflevector <1 x i32> [[TMP1]], <1 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[S]]
 ;
   %l = load <2 x i32>, ptr %p, align 1
diff --git a/llvm/test/Transforms/VectorCombine/X86/narrow-phi-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/X86/narrow-phi-of-shuffles.ll
new file mode 100644
index 0000000000000..59422e98cbcc6
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/narrow-phi-of-shuffles.ll
@@ -0,0 +1,1509 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,CHECK-V1
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,CHECK-V2
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,CHECK-V3
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v4 -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK,CHECK-V4
+
+define <2 x i8> @shuffle_v2i8(<2 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i8> @shuffle_v2i8(
+; CHECK-SAME: <2 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i8> %val3
+}
+
+define <4 x i8> @shuffle_v4i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i8> @shuffle_v4i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i8> %val3
+}
+
+define <8 x i8> @shuffle_v8i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i8> @shuffle_v8i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i8> %val3
+}
+
+define <16 x i8> @shuffle_v16i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i8> @shuffle_v16i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i8> %val3
+}
+
+define <32 x i8> @shuffle_v32i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i8> @shuffle_v32i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i8> %val3
+}
+
+define <2 x i16> @shuffle_v2i16(<2 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i16> @shuffle_v2i16(
+; CHECK-SAME: <2 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i16> %val3
+}
+
+define <4 x i16> @shuffle_v4i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i16> @shuffle_v4i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i16> %val3
+}
+
+define <8 x i16> @shuffle_v8i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i16> @shuffle_v8i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i16> %val3
+}
+
+define <16 x i16> @shuffle_v16i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i16> @shuffle_v16i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i16> %val3
+}
+
+define <32 x i16> @shuffle_v32i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i16> @shuffle_v32i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i16> %val3
+}
+
+define <2 x i32> @shuffle_v2i32(<2 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i32> @shuffle_v2i32(
+; CHECK-SAME: <2 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i32> %val3
+}
+
+define <4 x i32> @shuffle_v4i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i32> @shuffle_v4i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v8i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i32> @shuffle_v8i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <16 x i32> @shuffle_v16i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i32> @shuffle_v16i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i32> %val3
+}
+
+define <32 x i32> @shuffle_v32i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i32> @shuffle_v32i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i32> %val3
+}
+
+define <2 x bfloat> @shuffle_v2bf16(<2 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x bfloat> @shuffle_v2bf16(
+; CHECK-SAME: <2 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x bfloat> %val3
+}
+
+define <3 x bfloat> @shuffle_v3bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x bfloat> @shuffle_v3bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x bfloat> %val3
+}
+
+define <4 x bfloat> @shuffle_v4bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <4 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x bfloat> %val3
+}
+
+define <6 x bfloat> @shuffle_v6bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <6 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x bfloat> %val3
+}
+
+define <8 x bfloat> @shuffle_v8bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <8 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x bfloat> %val3
+}
+
+define <16 x bfloat> @shuffle_v16bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <16 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x bfloat> %val3
+}
+
+define <32 x bfloat> @shuffle_v32bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <3 x bfloat> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-NEXT:    [[VAL3:%.*]] = shufflevector <3 x bfloat> [[TMP1]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <32 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x bfloat> %val3
+}
+
+define <2 x half> @shuffle_v2f16(<2 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x half> @shuffle_v2f16(
+; CHECK-SAME: <2 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x half> %val3
+}
+
+define <3 x half> @shuffle_v3f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x half> @shuffle_v3f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x half> %val3
+}
+
+define <4 x half> @shuffle_v4f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-V1-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-V1-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT:  [[ENTRY:.*:]]
+; CHECK-V1-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1:       [[THEN]]:
+; CHECK-V1-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V1-NEXT:    tail call void @func0()
+; CHECK-V1-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V1:       [[ELSE]]:
+; CHECK-V1-NEXT:    tail call void @func1()
+; CHECK-V1-NEXT:    br label %[[FINALLY]]
+; CHECK-V1:       [[FINALLY]]:
+; CHECK-V1-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT:    ret <4 x half> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-V2-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT:  [[ENTRY:.*:]]
+; CHECK-V2-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2:       [[THEN]]:
+; CHECK-V2-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V2-NEXT:    tail call void @func0()
+; CHECK-V2-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V2:       [[ELSE]]:
+; CHECK-V2-NEXT:    tail call void @func1()
+; CHECK-V2-NEXT:    br label %[[FINALLY]]
+; CHECK-V2:       [[FINALLY]]:
+; CHECK-V2-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT:    ret <4 x half> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-V3-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT:  [[ENTRY:.*:]]
+; CHECK-V3-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3:       [[THEN]]:
+; CHECK-V3-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT:    tail call void @func0()
+; CHECK-V3-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V3:       [[ELSE]]:
+; CHECK-V3-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT:    tail call void @func1()
+; CHECK-V3-NEXT:    br label %[[FINALLY]]
+; CHECK-V3:       [[FINALLY]]:
+; CHECK-V3-NEXT:    [[VAL3:%.*]] = phi <4 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT:    ret <4 x half> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-V4-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT:  [[ENTRY:.*:]]
+; CHECK-V4-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4:       [[THEN]]:
+; CHECK-V4-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT:    tail call void @func0()
+; CHECK-V4-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V4:       [[ELSE]]:
+; CHECK-V4-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT:    tail call void @func1()
+; CHECK-V4-NEXT:    br label %[[FINALLY]]
+; CHECK-V4:       [[FINALLY]]:
+; CHECK-V4-NEXT:    [[VAL3:%.*]] = phi <4 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT:    ret <4 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x half> %val3
+}
+
+define <6 x half> @shuffle_v6f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-V1-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-V1-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT:  [[ENTRY:.*:]]
+; CHECK-V1-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1:       [[THEN]]:
+; CHECK-V1-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V1-NEXT:    tail call void @func0()
+; CHECK-V1-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V1:       [[ELSE]]:
+; CHECK-V1-NEXT:    tail call void @func1()
+; CHECK-V1-NEXT:    br label %[[FINALLY]]
+; CHECK-V1:       [[FINALLY]]:
+; CHECK-V1-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT:    ret <6 x half> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-V2-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT:  [[ENTRY:.*:]]
+; CHECK-V2-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2:       [[THEN]]:
+; CHECK-V2-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V2-NEXT:    tail call void @func0()
+; CHECK-V2-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V2:       [[ELSE]]:
+; CHECK-V2-NEXT:    tail call void @func1()
+; CHECK-V2-NEXT:    br label %[[FINALLY]]
+; CHECK-V2:       [[FINALLY]]:
+; CHECK-V2-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT:    ret <6 x half> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-V3-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT:  [[ENTRY:.*:]]
+; CHECK-V3-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3:       [[THEN]]:
+; CHECK-V3-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT:    tail call void @func0()
+; CHECK-V3-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V3:       [[ELSE]]:
+; CHECK-V3-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT:    tail call void @func1()
+; CHECK-V3-NEXT:    br label %[[FINALLY]]
+; CHECK-V3:       [[FINALLY]]:
+; CHECK-V3-NEXT:    [[VAL3:%.*]] = phi <6 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT:    ret <6 x half> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-V4-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT:  [[ENTRY:.*:]]
+; CHECK-V4-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4:       [[THEN]]:
+; CHECK-V4-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT:    tail call void @func0()
+; CHECK-V4-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V4:       [[ELSE]]:
+; CHECK-V4-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT:    tail call void @func1()
+; CHECK-V4-NEXT:    br label %[[FINALLY]]
+; CHECK-V4:       [[FINALLY]]:
+; CHECK-V4-NEXT:    [[VAL3:%.*]] = phi <6 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT:    ret <6 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x half> %val3
+}
+
+define <8 x half> @shuffle_v8f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-V1-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-V1-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT:  [[ENTRY:.*:]]
+; CHECK-V1-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1:       [[THEN]]:
+; CHECK-V1-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V1-NEXT:    tail call void @func0()
+; CHECK-V1-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V1:       [[ELSE]]:
+; CHECK-V1-NEXT:    tail call void @func1()
+; CHECK-V1-NEXT:    br label %[[FINALLY]]
+; CHECK-V1:       [[FINALLY]]:
+; CHECK-V1-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT:    ret <8 x half> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-V2-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT:  [[ENTRY:.*:]]
+; CHECK-V2-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2:       [[THEN]]:
+; CHECK-V2-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V2-NEXT:    tail call void @func0()
+; CHECK-V2-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V2:       [[ELSE]]:
+; CHECK-V2-NEXT:    tail call void @func1()
+; CHECK-V2-NEXT:    br label %[[FINALLY]]
+; CHECK-V2:       [[FINALLY]]:
+; CHECK-V2-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT:    ret <8 x half> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-V3-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT:  [[ENTRY:.*:]]
+; CHECK-V3-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3:       [[THEN]]:
+; CHECK-V3-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT:    tail call void @func0()
+; CHECK-V3-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V3:       [[ELSE]]:
+; CHECK-V3-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT:    tail call void @func1()
+; CHECK-V3-NEXT:    br label %[[FINALLY]]
+; CHECK-V3:       [[FINALLY]]:
+; CHECK-V3-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT:    ret <8 x half> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-V4-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT:  [[ENTRY:.*:]]
+; CHECK-V4-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4:       [[THEN]]:
+; CHECK-V4-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT:    tail call void @func0()
+; CHECK-V4-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V4:       [[ELSE]]:
+; CHECK-V4-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT:    tail call void @func1()
+; CHECK-V4-NEXT:    br label %[[FINALLY]]
+; CHECK-V4:       [[FINALLY]]:
+; CHECK-V4-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT:    ret <8 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x half> %val3
+}
+
+define <16 x half> @shuffle_v16f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-V1-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-V1-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT:  [[ENTRY:.*:]]
+; CHECK-V1-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1:       [[THEN]]:
+; CHECK-V1-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V1-NEXT:    tail call void @func0()
+; CHECK-V1-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V1:       [[ELSE]]:
+; CHECK-V1-NEXT:    tail call void @func1()
+; CHECK-V1-NEXT:    br label %[[FINALLY]]
+; CHECK-V1:       [[FINALLY]]:
+; CHECK-V1-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT:    ret <16 x half> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-V2-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT:  [[ENTRY:.*:]]
+; CHECK-V2-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2:       [[THEN]]:
+; CHECK-V2-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V2-NEXT:    tail call void @func0()
+; CHECK-V2-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V2:       [[ELSE]]:
+; CHECK-V2-NEXT:    tail call void @func1()
+; CHECK-V2-NEXT:    br label %[[FINALLY]]
+; CHECK-V2:       [[FINALLY]]:
+; CHECK-V2-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT:    ret <16 x half> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-V3-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT:  [[ENTRY:.*:]]
+; CHECK-V3-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3:       [[THEN]]:
+; CHECK-V3-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT:    tail call void @func0()
+; CHECK-V3-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V3:       [[ELSE]]:
+; CHECK-V3-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT:    tail call void @func1()
+; CHECK-V3-NEXT:    br label %[[FINALLY]]
+; CHECK-V3:       [[FINALLY]]:
+; CHECK-V3-NEXT:    [[VAL3:%.*]] = phi <16 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT:    ret <16 x half> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-V4-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT:  [[ENTRY:.*:]]
+; CHECK-V4-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4:       [[THEN]]:
+; CHECK-V4-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT:    tail call void @func0()
+; CHECK-V4-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V4:       [[ELSE]]:
+; CHECK-V4-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT:    tail call void @func1()
+; CHECK-V4-NEXT:    br label %[[FINALLY]]
+; CHECK-V4:       [[FINALLY]]:
+; CHECK-V4-NEXT:    [[VAL3:%.*]] = phi <16 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT:    ret <16 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x half> %val3
+}
+
+define <32 x half> @shuffle_v32f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-V1-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-V1-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V1-NEXT:  [[ENTRY:.*:]]
+; CHECK-V1-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V1:       [[THEN]]:
+; CHECK-V1-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V1-NEXT:    tail call void @func0()
+; CHECK-V1-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V1:       [[ELSE]]:
+; CHECK-V1-NEXT:    tail call void @func1()
+; CHECK-V1-NEXT:    br label %[[FINALLY]]
+; CHECK-V1:       [[FINALLY]]:
+; CHECK-V1-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V1-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V1-NEXT:    ret <32 x half> [[VAL3]]
+;
+; CHECK-V2-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-V2-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V2-NEXT:  [[ENTRY:.*:]]
+; CHECK-V2-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V2:       [[THEN]]:
+; CHECK-V2-NEXT:    [[TMP0:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; CHECK-V2-NEXT:    tail call void @func0()
+; CHECK-V2-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V2:       [[ELSE]]:
+; CHECK-V2-NEXT:    tail call void @func1()
+; CHECK-V2-NEXT:    br label %[[FINALLY]]
+; CHECK-V2:       [[FINALLY]]:
+; CHECK-V2-NEXT:    [[TMP1:%.*]] = phi <3 x half> [ [[TMP0]], %[[THEN]] ], [ [[ARG0]], %[[ELSE]] ]
+; CHECK-V2-NEXT:    [[VAL3:%.*]] = shufflevector <3 x half> [[TMP1]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V2-NEXT:    ret <32 x half> [[VAL3]]
+;
+; CHECK-V3-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-V3-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V3-NEXT:  [[ENTRY:.*:]]
+; CHECK-V3-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V3:       [[THEN]]:
+; CHECK-V3-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V3-NEXT:    tail call void @func0()
+; CHECK-V3-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V3:       [[ELSE]]:
+; CHECK-V3-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V3-NEXT:    tail call void @func1()
+; CHECK-V3-NEXT:    br label %[[FINALLY]]
+; CHECK-V3:       [[FINALLY]]:
+; CHECK-V3-NEXT:    [[VAL3:%.*]] = phi <32 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V3-NEXT:    ret <32 x half> [[VAL3]]
+;
+; CHECK-V4-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-V4-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-V4-NEXT:  [[ENTRY:.*:]]
+; CHECK-V4-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK-V4:       [[THEN]]:
+; CHECK-V4-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-V4-NEXT:    tail call void @func0()
+; CHECK-V4-NEXT:    br label %[[FINALLY:.*]]
+; CHECK-V4:       [[ELSE]]:
+; CHECK-V4-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-V4-NEXT:    tail call void @func1()
+; CHECK-V4-NEXT:    br label %[[FINALLY]]
+; CHECK-V4:       [[FINALLY]]:
+; CHECK-V4-NEXT:    [[VAL3:%.*]] = phi <32 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-V4-NEXT:    ret <32 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x half> %val3
+}
+
+define <2 x float> @shuffle_v2f32(<2 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x float> @shuffle_v2f32(
+; CHECK-SAME: <2 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x float> %val3
+}
+
+define <3 x float> @shuffle_v3f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x float> @shuffle_v3f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x float> %val3
+}
+
+define <4 x float> @shuffle_v4f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x float> @shuffle_v4f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x float> %val3
+}
+
+define <6 x float> @shuffle_v6f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x float> @shuffle_v6f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x float> %val3
+}
+
+define <8 x float> @shuffle_v8f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x float> @shuffle_v8f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x float> %val3
+}
+
+define <16 x float> @shuffle_v16f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x float> @shuffle_v16f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x float> %val3
+}
+
+define <32 x float> @shuffle_v32f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x float> @shuffle_v32f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x float> %val3
+}
+
+declare void @func0() local_unnamed_addr
+
+declare void @func1() local_unnamed_addr
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-umin.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-umin.ll
new file mode 100644
index 0000000000000..82b20ccc5b8f5
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-chain-reduction-umin.ll
@@ -0,0 +1,200 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64 -passes=vector-combine -S %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v2 -passes=vector-combine -S %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v3 -passes=vector-combine -S %s | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -mcpu=x86-64-v4 -passes=vector-combine -S %s | FileCheck %s
+
+define i16 @test_reduce_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v8i16(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> [[A0]])
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %a0, <8 x i16> %1)
+  %3 = shufflevector <8 x i16> %2, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %2, <8 x i16> %3)
+  %5 = shufflevector <8 x i16> %4, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %4, <8 x i16> %5)
+  %7 = extractelement <8 x i16> %6, i64 0
+  ret i16 %7
+}
+
+define i8 @test_reduce_v16i8(<16 x i8> %a0) {
+;
+; CHECK-LABEL: define i8 @test_reduce_v16i8(
+; CHECK-SAME: <16 x i8> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP8:%.*]] = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> [[A0]])
+; CHECK-NEXT:    ret i8 [[TMP8]]
+;
+  %1 = shufflevector <16 x i8> %a0, <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %a0, <16 x i8> %1)
+  %3 = shufflevector <16 x i8> %2, <16 x i8> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %2, <16 x i8> %3)
+  %5 = shufflevector <16 x i8> %4, <16 x i8> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %4, <16 x i8> %5)
+  %7 = shufflevector <16 x i8> %6, <16 x i8> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %8 = tail call <16 x i8> @llvm.umin.v16i8(<16 x i8> %6, <16 x i8> %7)
+  %9 = extractelement <16 x i8> %8, i64 0
+  ret i8 %9
+}
+
+define i8 @test_reduce_v32i8(<32 x i8> %a0) {
+; CHECK-LABEL: define i8 @test_reduce_v32i8(
+; CHECK-SAME: <32 x i8> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umin.v32i8(<32 x i8> [[A0]])
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %1 = shufflevector <32 x i8> %a0, <32 x i8> poison, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
+  i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> %a0, <32 x i8> %1)
+  %3 = shufflevector <32 x i8> %2, <32 x i8> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> %2, <32 x i8> %3)
+  %5 = shufflevector <32 x i8> %4, <32 x i8> poison, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> %4, <32 x i8> %5)
+  %7 = shufflevector <32 x i8> %6, <32 x i8> poison, <32 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %8 = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> %6, <32 x i8> %7)
+  %9 = shufflevector <32 x i8> %8, <32 x i8> poison, <32 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %10 = tail call <32 x i8> @llvm.umin.v32i8(<32 x i8> %8, <32 x i8> %9)
+  %11 = extractelement <32 x i8> %10, i64 0
+  ret i8 %11
+}
+
+define i16 @test_reduce_v16i16(<16 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v16i16(
+; CHECK-SAME: <16 x i16> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.umin.v16i16(<16 x i16> [[A0]])
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %1 = shufflevector <16 x i16> %a0, <16 x i16> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <16 x i16> @llvm.umin.v16i16(<16 x i16> %a0, <16 x i16> %1)
+  %3 = shufflevector <16 x i16> %2, <16 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <16 x i16> @llvm.umin.v16i16(<16 x i16> %2, <16 x i16> %3)
+  %5 = shufflevector <16 x i16> %4, <16 x i16> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <16 x i16> @llvm.umin.v16i16(<16 x i16> %4, <16 x i16> %5)
+  %7 = shufflevector <16 x i16> %6, <16 x i16> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %8 = tail call <16 x i16> @llvm.umin.v16i16(<16 x i16> %6, <16 x i16> %7)
+  %9 = extractelement <16 x i16> %8, i64 0
+  ret i16 %9
+}
+
+define i8 @test_reduce_v64i8(<64 x i8> %a0) {
+; CHECK-LABEL: define i8 @test_reduce_v64i8(
+; CHECK-SAME: <64 x i8> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umin.v64i8(<64 x i8> [[A0]])
+; CHECK-NEXT:    ret i8 [[TMP1]]
+;
+  %1 = shufflevector <64 x i8> %a0, <64 x i8> poison, <64 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39,
+  i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47,
+  i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55,
+  i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <64 x i8> @llvm.umin.v64i8(<64 x i8> %a0, <64 x i8> %1)
+  %3 = shufflevector <64 x i8> %2, <64 x i8> poison, <64 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
+  i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <64 x i8> @llvm.umin.v64i8(<64 x i8> %2, <64 x i8> %3)
+  %5 = shufflevector <64 x i8> %4, <64 x i8> poison, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <64 x i8> @llvm.umin.v64i8(<64 x i8> %4, <64 x i8> %5)
+  %7 = shufflevector <64 x i8> %6, <64 x i8> poison, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %8 = tail call <64 x i8> @llvm.umin.v64i8(<64 x i8> %6, <64 x i8> %7)
+  %9 = shufflevector <64 x i8> %8, <64 x i8> poison, <64 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %10 = tail call <64 x i8> @llvm.umin.v64i8(<64 x i8> %8, <64 x i8> %9)
+  %11 = shufflevector <64 x i8> %10, <64 x i8> poison, <64 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %12 = tail call <64 x i8> @llvm.umin.v64i8(<64 x i8> %10, <64 x i8> %11)
+  %13 = extractelement <64 x i8> %12, i64 0
+  ret i8 %13
+}
+
+define i16 @test_reduce_v32i16(<32 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v32i16(
+; CHECK-SAME: <32 x i16> [[A0:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.umin.v32i16(<32 x i16> [[A0]])
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %1 = shufflevector <32 x i16> %a0, <32 x i16> poison, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23,
+  i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <32 x i16> @llvm.umin.v32i16(<32 x i16> %a0, <32 x i16> %1)
+  %3 = shufflevector <32 x i16> %2, <32 x i16> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <32 x i16> @llvm.umin.v32i16(<32 x i16> %2, <32 x i16> %3)
+  %5 = shufflevector <32 x i16> %4, <32 x i16> poison, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <32 x i16> @llvm.umin.v32i16(<32 x i16> %4, <32 x i16> %5)
+  %7 = shufflevector <32 x i16> %6, <32 x i16> poison, <32 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %8 = tail call <32 x i16> @llvm.umin.v32i16(<32 x i16> %6, <32 x i16> %7)
+  %9 = shufflevector <32 x i16> %8, <32 x i16> poison, <32 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison,
+  i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %10 = tail call <32 x i16> @llvm.umin.v32i16(<32 x i16> %8, <32 x i16> %9)
+  %11 = extractelement <32 x i16> %10, i64 0
+  ret i16 %11
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
index b30dc9ffdc596..eddfc57a7d256 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
@@ -47,21 +47,12 @@ define <8 x i32> @concat_extract_subvectors_poison(<8 x i32> %x) {
 ; broadcast loads are free on AVX (and blends are much cheap than general 2-operand shuffles)
 
 define  <4 x double> @blend_broadcasts_v4f64(ptr %p0, ptr %p1)  {
-; SSE-LABEL: define <4 x double> @blend_broadcasts_v4f64(
-; SSE-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR0]] {
-; SSE-NEXT:    [[LD0:%.*]] = load <4 x double>, ptr [[P0]], align 32
-; SSE-NEXT:    [[LD1:%.*]] = load <4 x double>, ptr [[P1]], align 32
-; SSE-NEXT:    [[BLEND:%.*]] = shufflevector <4 x double> [[LD0]], <4 x double> [[LD1]], <4 x i32> <i32 0, i32 4, i32 4, i32 0>
-; SSE-NEXT:    ret <4 x double> [[BLEND]]
-;
-; AVX-LABEL: define <4 x double> @blend_broadcasts_v4f64(
-; AVX-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR0]] {
-; AVX-NEXT:    [[LD0:%.*]] = load <4 x double>, ptr [[P0]], align 32
-; AVX-NEXT:    [[LD1:%.*]] = load <4 x double>, ptr [[P1]], align 32
-; AVX-NEXT:    [[BCST0:%.*]] = shufflevector <4 x double> [[LD0]], <4 x double> undef, <4 x i32> zeroinitializer
-; AVX-NEXT:    [[BCST1:%.*]] = shufflevector <4 x double> [[LD1]], <4 x double> undef, <4 x i32> zeroinitializer
-; AVX-NEXT:    [[BLEND:%.*]] = shufflevector <4 x double> [[BCST0]], <4 x double> [[BCST1]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
-; AVX-NEXT:    ret <4 x double> [[BLEND]]
+; CHECK-LABEL: define <4 x double> @blend_broadcasts_v4f64(
+; CHECK-SAME: ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = load <1 x double>, ptr [[P0]], align 32
+; CHECK-NEXT:    [[TMP2:%.*]] = load <1 x double>, ptr [[P1]], align 32
+; CHECK-NEXT:    [[BLEND:%.*]] = shufflevector <1 x double> [[TMP1]], <1 x double> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT:    ret <4 x double> [[BLEND]]
 ;
   %ld0 = load <4 x double>, ptr %p0, align 32
   %ld1 = load <4 x double>, ptr %p1, align 32
@@ -81,3 +72,6 @@ define <2 x float> @PR86068(<2 x float> %a0, <2 x float> %a1) {
   %s2 = shufflevector <2 x float> %s1, <2 x float> %a0, <2 x i32> <i32 0, i32 3>
   ret <2 x float> %s2
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
+; SSE: {{.*}}
diff --git a/llvm/test/Transforms/VectorCombine/alias.ll b/llvm/test/Transforms/VectorCombine/alias.ll
new file mode 100644
index 0000000000000..459956cd997d8
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/alias.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=vector-combine -S | FileCheck %s --check-prefixes=CHECK
+
+define <4 x i32> @quux(ptr addrspace(3) %arg) {
+; CHECK-LABEL: define <4 x i32> @quux(
+; CHECK-SAME: ptr addrspace(3) [[ARG:%.*]]) {
+; CHECK-NEXT:  [[BB:.*:]]
+; CHECK-NEXT:    [[EXTRACTELEMENT:%.*]] = load i8, ptr addrspace(3) [[ARG]], align 4, !tbaa [[TBAA0:![0-9]+]], !alias.scope [[META0:![0-9]+]], !noalias [[META0]]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds <4 x i8>, ptr addrspace(3) [[ARG]], i32 0, i64 1
+; CHECK-NEXT:    [[EXTRACTELEMENT1:%.*]] = load i8, ptr addrspace(3) [[TMP0]], align 1, !tbaa [[TBAA0]], !alias.scope [[META0]], !noalias [[META0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds <4 x i8>, ptr addrspace(3) [[ARG]], i32 0, i64 2
+; CHECK-NEXT:    [[EXTRACTELEMENT2:%.*]] = load i8, ptr addrspace(3) [[TMP1]], align 2, !tbaa [[TBAA0]], !alias.scope [[META0]], !noalias [[META0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds <4 x i8>, ptr addrspace(3) [[ARG]], i32 0, i64 3
+; CHECK-NEXT:    [[EXTRACTELEMENT3:%.*]] = load i8, ptr addrspace(3) [[TMP2]], align 1, !tbaa [[TBAA0]], !alias.scope [[META0]], !noalias [[META0]]
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[EXTRACTELEMENT]] to i32
+; CHECK-NEXT:    [[ZEXT4:%.*]] = zext i8 [[EXTRACTELEMENT1]] to i32
+; CHECK-NEXT:    [[ZEXT5:%.*]] = zext i8 [[EXTRACTELEMENT2]] to i32
+; CHECK-NEXT:    [[ZEXT6:%.*]] = zext i8 [[EXTRACTELEMENT3]] to i32
+; CHECK-NEXT:    [[INSERTELEMENT:%.*]] = insertelement <4 x i32> poison, i32 [[ZEXT]], i64 0
+; CHECK-NEXT:    [[INSERTELEMENT7:%.*]] = insertelement <4 x i32> [[INSERTELEMENT]], i32 [[ZEXT4]], i64 1
+; CHECK-NEXT:    [[INSERTELEMENT8:%.*]] = insertelement <4 x i32> [[INSERTELEMENT7]], i32 [[ZEXT5]], i64 2
+; CHECK-NEXT:    [[INSERTELEMENT9:%.*]] = insertelement <4 x i32> [[INSERTELEMENT8]], i32 [[ZEXT6]], i64 3
+; CHECK-NEXT:    ret <4 x i32> [[INSERTELEMENT9]]
+;
+bb:
+  %load = load <4 x i8>, ptr addrspace(3) %arg, align 4, !alias.scope !0, !noalias !0, !tbaa !5
+  %extractelement = extractelement <4 x i8> %load, i64 0
+  %extractelement1 = extractelement <4 x i8> %load, i64 1
+  %extractelement2 = extractelement <4 x i8> %load, i64 2
+  %extractelement3 = extractelement <4 x i8> %load, i64 3
+  %zext = zext i8 %extractelement to i32
+  %zext4 = zext i8 %extractelement1 to i32
+  %zext5 = zext i8 %extractelement2 to i32
+  %zext6 = zext i8 %extractelement3 to i32
+  %insertelement = insertelement <4 x i32> poison, i32 %zext, i64 0
+  %insertelement7 = insertelement <4 x i32> %insertelement, i32 %zext4, i64 1
+  %insertelement8 = insertelement <4 x i32> %insertelement7, i32 %zext5, i64 2
+  %insertelement9 = insertelement <4 x i32> %insertelement8, i32 %zext6, i64 3
+  ret <4 x i32> %insertelement9
+}
+
+!0 = !{!1}
+!1 = distinct !{!1, !2}
+!2 = distinct !{!2}
+!3 = !{!"Simple C/C++ TBAA"}
+!4 = !{!"omnipotent char", !3, i64 0}
+!5 = !{!"i8", !4, i64 0}
+;.
+; CHECK: [[TBAA0]] = !{[[META3:![0-9]+]], [[META3]], i64 0, i64 0}
+; CHECK: [[META3]] = !{!"i8", [[META4:![0-9]+]]}
+; CHECK: [[META4]] = !{!"omnipotent char", [[META5:![0-9]+]], i64 0}
+; CHECK: [[META5]] = !{!"Simple C/C++ TBAA"}
+; CHECK: [[META0]] = !{[[META1:![0-9]+]]}
+; CHECK: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = distinct !{[[META2]]}
+;.
\ No newline at end of file
diff --git a/llvm/test/Transforms/VectorCombine/binop-scalarize.ll b/llvm/test/Transforms/VectorCombine/binop-scalarize.ll
index 52a706a0b59a7..bc07f8b086496 100644
--- a/llvm/test/Transforms/VectorCombine/binop-scalarize.ll
+++ b/llvm/test/Transforms/VectorCombine/binop-scalarize.ll
@@ -20,3 +20,22 @@ define <4 x i8> @udiv_ub(i8 %x, i8 %y) {
   %v = udiv <4 x i8> %x.insert, %y.insert
   ret <4 x i8> %v
 }
+
+
+; Unfoldable constant expression may cause infinite loop between 
+; scalarizing insertelement and folding binop(insert(x,a,idx),insert(y,b,idx))
+@val = external hidden global ptr, align 8
+
+define <2 x i64> @pr153012(i64 %idx) #0 {
+; CHECK-LABEL: define <2 x i64> @pr153012(
+; CHECK-SAME: i64 [[IDX:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[A:%.*]] = insertelement <2 x i64> <i64 5, i64 ptrtoint (ptr @val to i64)>, i64 [[IDX]], i32 0
+; CHECK-NEXT:    [[B:%.*]] = or disjoint <2 x i64> splat (i64 2), [[A]]
+; CHECK-NEXT:    ret <2 x i64> [[B]]
+;
+entry:
+  %a = insertelement <2 x i64> <i64 5, i64 ptrtoint (ptr @val to i64)>, i64 %idx, i32 0
+  %b = or disjoint <2 x i64> splat (i64 2), %a
+  ret <2 x i64> %b
+}
diff --git a/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
new file mode 100644
index 0000000000000..403ce33b5344e
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/fold-shuffle-chains-to-reduce.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=vector-combine -S | FileCheck %s
+
+define i16 @test_reduce_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v8i16(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> [[A0]])
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %a0, <8 x i16> %1)
+  %3 = shufflevector <8 x i16> %2, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %2, <8 x i16> %3)
+  %5 = shufflevector <8 x i16> %4, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %4, <8 x i16> %5)
+  %7 = extractelement <8 x i16> %6, i64 0
+  ret i16 %7
+}
+
+define i16 @test_reduce_v7i16_or(<7 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v7i16_or(
+; CHECK-SAME: <7 x i16> [[A0:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.or.v7i16(<7 x i16> [[A0]])
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %1 = shufflevector <7 x i16> %a0, <7 x i16> poison, <7 x i32> <i32 3, i32 4, i32 5, i32 6, i32 poison, i32 poison, i32 poison>
+  %2 = or <7 x i16> %a0, %1
+  %3 = shufflevector <7 x i16> %2, <7 x i16> poison, <7 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = or <7 x i16> %2, %3
+  %5 = shufflevector <7 x i16> %4, <7 x i16> poison, <7 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = or <7 x i16> %4, %5
+  %7 = extractelement <7 x i16> %6, i64 0
+  ret i16 %7
+}
+
+define i16 @test_reduce_v3i16_and(<3 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v3i16_and(
+; CHECK-SAME: <3 x i16> [[A0:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.and.v3i16(<3 x i16> [[A0]])
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %1 = shufflevector <3 x i16> %a0, <3 x i16> poison, <3 x i32> <i32 1, i32 2, i32 poison>
+  %2 = and <3 x i16> %a0, %1
+  %3 = shufflevector <3 x i16> %2, <3 x i16> poison, <3 x i32> <i32 1, i32 poison, i32 poison>
+  %4 = and <3 x i16> %2, %3
+  %5 = extractelement <3 x i16> %4, i64 0
+  ret i16 %5
+}
+
+define i16 @test_reduce_v6i16_xor(<6 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v6i16_xor(
+; CHECK-SAME: <6 x i16> [[A0:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16 @llvm.vector.reduce.xor.v6i16(<6 x i16> [[A0]])
+; CHECK-NEXT:    ret i16 [[TMP1]]
+;
+  %1 = shufflevector <6 x i16> %a0, <6 x i16> poison, <6 x i32> <i32 3, i32 4, i32 5, i32 poison, i32 poison, i32 poison>
+  %2 = xor <6 x i16> %a0, %1
+  %3 = shufflevector <6 x i16> %2, <6 x i16> poison, <6 x i32> <i32 1, i32 2, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = xor <6 x i16> %2, %3
+  %5 = shufflevector <6 x i16> %4, <6 x i16> poison, <6 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = xor <6 x i16> %4, %5
+  %7 = extractelement <6 x i16> %6, i64 0
+  ret i16 %7
+}
+
+define i16 @test_reduce_v8i16_2(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v8i16_2(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP5]])
+; CHECK-NEXT:    [[TMP13:%.*]] = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> [[A0]])
+; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <8 x i16> [[TMP6]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP9:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP8]])
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <8 x i16> [[TMP9]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP9]], <8 x i16> [[TMP10]])
+; CHECK-NEXT:    [[TMP12:%.*]] = shufflevector <8 x i16> [[TMP11]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP16:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP11]], <8 x i16> [[TMP12]])
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i16> [[TMP16]], i64 0
+; CHECK-NEXT:    [[TMP15:%.*]] = tail call i16 @llvm.umin.i16(i16 [[TMP13]], i16 [[TMP14]])
+; CHECK-NEXT:    ret i16 [[TMP15]]
+;
+  %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %a0, <8 x i16> %1)
+  %3 = shufflevector <8 x i16> %2, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %2, <8 x i16> %3)
+  %5 = shufflevector <8 x i16> %4, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %4, <8 x i16> %5)
+  %7 = extractelement <8 x i16> %6, i64 0
+
+  %8 = shufflevector <8 x i16> %6, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+  %9 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %a0, <8 x i16> %8)
+  %10 = shufflevector <8 x i16> %9, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %11 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %9, <8 x i16> %10)
+  %12 = shufflevector <8 x i16> %11, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %13 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %11, <8 x i16> %12)
+  %14 = extractelement <8 x i16> %13, i64 0
+
+  %15 = tail call i16 @llvm.umin.i16(i16 %7, i16 %14)
+
+  ret i16 %15
+}
+
+define i16 @test_reduce_v8i16_neg1(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v8i16_neg1(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP6]], i64 0
+; CHECK-NEXT:    ret i16 [[TMP7]]
+;
+  %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %a0, <8 x i16> %1)
+  %3 = shufflevector <8 x i16> %2, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %2, <8 x i16> %3)
+  %5 = shufflevector <8 x i16> %4, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %4, <8 x i16> %5)
+  %7 = extractelement <8 x i16> %6, i64 0
+  ret i16 %7
+}
+
+define i16 @test_reduce_v8i16_neg2(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v8i16_neg2(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = tail call <8 x i16> @llvm.umax.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i16> [[TMP6]], i64 0
+; CHECK-NEXT:    ret i16 [[TMP7]]
+;
+  %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %a0, <8 x i16> %1)
+  %3 = shufflevector <8 x i16> %2, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %2, <8 x i16> %3)
+  %5 = shufflevector <8 x i16> %4, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = tail call <8 x i16> @llvm.umax.v8i16(<8 x i16> %4, <8 x i16> %5)
+  %7 = extractelement <8 x i16> %6, i64 0
+  ret i16 %7
+}
+
+define i16 @test_reduce_v8i16_neg3(<8 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v8i16_neg3(
+; CHECK-SAME: <8 x i16> [[A0:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[A0]], <8 x i16> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP3]])
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <8 x i16> [[TMP4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP7:%.*]] = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> [[TMP5]], <8 x i16> [[TMP6]])
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i16> [[TMP7]], i64 0
+; CHECK-NEXT:    ret i16 [[TMP8]]
+;
+  %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+  %2 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %a0, <8 x i16> %1)
+  %3 = shufflevector <8 x i16> %2, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %2, <8 x i16> %3)
+  %5 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %2, <8 x i16> %3)
+  %6 = shufflevector <8 x i16> %4, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %7 = tail call <8 x i16> @llvm.umin.v8i16(<8 x i16> %5, <8 x i16> %6)
+  %8 = extractelement <8 x i16> %7, i64 0
+  ret i16 %8
+}
+
+define i16 @test_reduce_v6i16_xor_neg(<6 x i16> %a0) {
+; CHECK-LABEL: define i16 @test_reduce_v6i16_xor_neg(
+; CHECK-SAME: <6 x i16> [[A0:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <6 x i16> [[A0]], <6 x i16> poison, <6 x i32> <i32 3, i32 4, i32 5, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = xor <6 x i16> [[A0]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <6 x i16> [[TMP2]], <6 x i16> poison, <6 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP4:%.*]] = xor <6 x i16> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <6 x i16> [[TMP4]], <6 x i16> poison, <6 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP6:%.*]] = xor <6 x i16> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <6 x i16> [[TMP6]], i64 0
+; CHECK-NEXT:    ret i16 [[TMP7]]
+;
+  %1 = shufflevector <6 x i16> %a0, <6 x i16> poison, <6 x i32> <i32 3, i32 4, i32 5, i32 poison, i32 poison, i32 poison>
+  %2 = xor <6 x i16> %a0, %1
+  %3 = shufflevector <6 x i16> %2, <6 x i16> poison, <6 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+  %4 = xor <6 x i16> %2, %3
+  %5 = shufflevector <6 x i16> %4, <6 x i16> poison, <6 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %6 = xor <6 x i16> %4, %5
+  %7 = extractelement <6 x i16> %6, i64 0
+  ret i16 %7
+}
diff --git a/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll b/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll
index 9e43a28bf1e59..abd98a4dc64b8 100644
--- a/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll
+++ b/llvm/test/Transforms/VectorCombine/intrinsic-scalarize.ll
@@ -5,8 +5,7 @@ define <4 x i32> @umax_fixed(i32 %x, i32 %y) {
 ; CHECK-LABEL: define <4 x i32> @umax_fixed(
 ; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call i32 @llvm.umax.i32(i32 [[X]], i32 [[Y]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> poison, <4 x i32> poison)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x i32> poison, i32 [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <4 x i32> [[V]]
 ;
   %x.insert = insertelement <4 x i32> poison, i32 %x, i32 0
@@ -19,8 +18,7 @@ define <vscale x 4 x i32> @umax_scalable(i32 %x, i32 %y) {
 ; CHECK-LABEL: define <vscale x 4 x i32> @umax_scalable(
 ; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call i32 @llvm.umax.i32(i32 [[X]], i32 [[Y]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> poison)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x i32> [[TMP1]], i32 [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <vscale x 4 x i32> [[V]]
 ;
   %x.insert = insertelement <vscale x 4 x i32> poison, i32 %x, i32 0
@@ -33,8 +31,7 @@ define <4 x i32> @umax_fixed_lhs_const(i32 %x) {
 ; CHECK-LABEL: define <4 x i32> @umax_fixed_lhs_const(
 ; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call i32 @llvm.umax.i32(i32 1, i32 [[X]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> poison)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x i32> poison, i32 [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <4 x i32> [[V]]
 ;
   %x.insert = insertelement <4 x i32> poison, i32 %x, i32 0
@@ -46,8 +43,7 @@ define <4 x i32> @umax_fixed_rhs_const(i32 %x) {
 ; CHECK-LABEL: define <4 x i32> @umax_fixed_rhs_const(
 ; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call i32 @llvm.umax.i32(i32 [[X]], i32 1)
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.umax.v4i32(<4 x i32> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 4>)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x i32> poison, i32 [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <4 x i32> [[V]]
 ;
   %x.insert = insertelement <4 x i32> poison, i32 %x, i32 0
@@ -59,8 +55,7 @@ define <vscale x 4 x i32> @umax_scalable_lhs_const(i32 %x) {
 ; CHECK-LABEL: define <vscale x 4 x i32> @umax_scalable_lhs_const(
 ; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call i32 @llvm.umax.i32(i32 42, i32 [[X]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32> splat (i32 42), <vscale x 4 x i32> poison)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x i32> [[TMP1]], i32 [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <vscale x 4 x i32> [[V]]
 ;
   %x.insert = insertelement <vscale x 4 x i32> poison, i32 %x, i32 0
@@ -72,8 +67,7 @@ define <vscale x 4 x i32> @umax_scalable_rhs_const(i32 %x) {
 ; CHECK-LABEL: define <vscale x 4 x i32> @umax_scalable_rhs_const(
 ; CHECK-SAME: i32 [[X:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call i32 @llvm.umax.i32(i32 [[X]], i32 42)
-; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.umax.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i32> splat (i32 42))
-; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x i32> [[TMP1]], i32 [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <vscale x 4 x i32> [[V]]
 ;
   %x.insert = insertelement <vscale x 4 x i32> poison, i32 %x, i32 0
@@ -100,8 +94,7 @@ define <4 x float> @fabs_fixed(float %x) {
 ; CHECK-LABEL: define <4 x float> @fabs_fixed(
 ; CHECK-SAME: float [[X:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fabs.v4f32(<4 x float> poison)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x float> [[TMP1]], float [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x float> poison, float [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[V]]
 ;
   %x.insert = insertelement <4 x float> poison, float %x, i32 0
@@ -113,8 +106,7 @@ define <vscale x 4 x float> @fabs_scalable(float %x) {
 ; CHECK-LABEL: define <vscale x 4 x float> @fabs_scalable(
 ; CHECK-SAME: float [[X:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call float @llvm.fabs.f32(float [[X]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.fabs.nxv4f32(<vscale x 4 x float> poison)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x float> [[TMP1]], float [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x float> poison, float [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <vscale x 4 x float> [[V]]
 ;
   %x.insert = insertelement <vscale x 4 x float> poison, float %x, i32 0
@@ -126,8 +118,7 @@ define <4 x float> @fma_fixed(float %x, float %y, float %z) {
 ; CHECK-LABEL: define <4 x float> @fma_fixed(
 ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call float @llvm.fma.f32(float [[X]], float [[Y]], float [[Z]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> poison, <4 x float> poison, <4 x float> poison)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x float> [[TMP1]], float [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x float> poison, float [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[V]]
 ;
   %x.insert = insertelement <4 x float> poison, float %x, i32 0
@@ -141,8 +132,7 @@ define <vscale x 4 x float> @fma_scalable(float %x, float %y, float %z) {
 ; CHECK-LABEL: define <vscale x 4 x float> @fma_scalable(
 ; CHECK-SAME: float [[X:%.*]], float [[Y:%.*]], float [[Z:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call float @llvm.fma.f32(float [[X]], float [[Y]], float [[Z]])
-; CHECK-NEXT:    [[TMP1:%.*]] = call <vscale x 4 x float> @llvm.fma.nxv4f32(<vscale x 4 x float> poison, <vscale x 4 x float> poison, <vscale x 4 x float> poison)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x float> [[TMP1]], float [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <vscale x 4 x float> poison, float [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <vscale x 4 x float> [[V]]
 ;
   %x.insert = insertelement <vscale x 4 x float> poison, float %x, i32 0
@@ -156,8 +146,7 @@ define <4 x float> @scalar_argument(float %x) {
 ; CHECK-LABEL: define <4 x float> @scalar_argument(
 ; CHECK-SAME: float [[X:%.*]]) {
 ; CHECK-NEXT:    [[V_SCALAR:%.*]] = call float @llvm.powi.f32.i32(float [[X]], i32 42)
-; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.powi.v4f32.i32(<4 x float> poison, i32 42)
-; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x float> [[TMP1]], float [[V_SCALAR]], i64 0
+; CHECK-NEXT:    [[V:%.*]] = insertelement <4 x float> poison, float [[V_SCALAR]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[V]]
 ;
   %x.insert = insertelement <4 x float> poison, float %x, i32 0
diff --git a/llvm/test/Transforms/VectorCombine/load-shufflevector.ll b/llvm/test/Transforms/VectorCombine/load-shufflevector.ll
new file mode 100644
index 0000000000000..7d9393ab77f20
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/load-shufflevector.ll
@@ -0,0 +1,404 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=vector-combine -S < %s | FileCheck %s
+
+define <8 x half> @shuffle_v4_v8f16_r0_1_volatile(ptr addrspace(1) nocapture readonly %arg0) local_unnamed_addr {
+; CHECK-LABEL: define <8 x half> @shuffle_v4_v8f16_r0_1_volatile(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load volatile <4 x half>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x half> [[TMP0]], <4 x half> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <8 x half> [[TMP1]]
+;
+entry:
+  %val0 = load volatile <4 x half>, ptr addrspace(1) %arg0, align 32
+  %val1 = shufflevector <4 x half> %val0, <4 x half> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
+  ret <8 x half> %val1
+}
+
+define <8 x half> @shuffle_v4_v8f16_r0_1(ptr addrspace(1) nocapture readonly %arg0) local_unnamed_addr {
+; CHECK-LABEL: define <8 x half> @shuffle_v4_v8f16_r0_1(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x half>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x half> [[TMP0]], <2 x half> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <8 x half> [[TMP1]]
+;
+entry:
+  %val0 = load <4 x half>, ptr addrspace(1) %arg0, align 32
+  %val1 = shufflevector <4 x half> %val0, <4 x half> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
+  ret <8 x half> %val1
+}
+
+define <8 x half> @shuffle_v4_v8f16_r0_2(ptr addrspace(1) nocapture readonly %arg0) local_unnamed_addr {
+; CHECK-LABEL: define <8 x half> @shuffle_v4_v8f16_r0_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x half>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x half> [[TMP0]], <3 x half> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <8 x half> [[TMP1]]
+;
+entry:
+  %val0 = load <4 x half>, ptr addrspace(1) %arg0, align 32
+  %val1 = shufflevector <4 x half> %val0, <4 x half> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 2, i32 2, i32 2, i32 2>
+  ret <8 x half> %val1
+}
+
+define <4 x half> @shuffle_v4_v4f16_r1_2(ptr addrspace(1) nocapture readonly %arg0) local_unnamed_addr {
+; CHECK-LABEL: define <4 x half> @shuffle_v4_v4f16_r1_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x half>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x half> [[TMP0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 2>
+; CHECK-NEXT:    ret <4 x half> [[TMP1]]
+;
+entry:
+  %val0 = load <4 x half>, ptr addrspace(1) %arg0, align 32
+  %val1 = shufflevector <4 x half> %val0, <4 x half> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 2>
+  ret <4 x half> %val1
+}
+
+define <8 x half> @shuffle_v4_v8f16_r1_2(ptr addrspace(1) nocapture readonly %arg0) local_unnamed_addr {
+; CHECK-LABEL: define <8 x half> @shuffle_v4_v8f16_r1_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x half>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x half> [[TMP0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <8 x half> [[TMP1]]
+;
+entry:
+  %val0 = load <4 x half>, ptr addrspace(1) %arg0, align 32
+  %val1 = shufflevector <4 x half> %val0, <4 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2>
+  ret <8 x half> %val1
+}
+
+define <8 x half> @shuffle_v4_v8f16_cond_r0_1(ptr addrspace(1) nocapture readonly %arg0, i1 %cond) local_unnamed_addr {
+; CHECK-LABEL: define <8 x half> @shuffle_v4_v8f16_cond_r0_1(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]], i1 [[COND:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x half>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x half> [[TMP0]], <2 x half> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x half> [[TMP0]], <2 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x half> [[VAL3]]
+;
+entry:
+  %val0 = load <4 x half>, ptr addrspace(1) %arg0, align 32
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <4 x half> %val0, <4 x half> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  br label %finally
+
+else:
+  %val2 = shufflevector <4 x half> %val0, <4 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  br label %finally
+
+finally:
+  %val3 = phi <8 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x half> %val3
+}
+
+define <4 x half> @shuffle_v4_v4f16_cond_r1_2(ptr addrspace(1) nocapture readonly %arg0, i1 %cond) local_unnamed_addr {
+; CHECK-LABEL: define <4 x half> @shuffle_v4_v4f16_cond_r1_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]], i1 [[COND:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x half>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[TMP0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[TMP0]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x half> [[VAL3]]
+;
+entry:
+  %val0 = load <4 x half>, ptr addrspace(1) %arg0, align 32
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <4 x half> %val0, <4 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  br label %finally
+
+else:
+  %val2 = shufflevector <4 x half> %val0, <4 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  br label %finally
+
+finally:
+  %val3 = phi <4 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x half> %val3
+}
+
+define <8 x half> @shuffle_v4_v8f16_cond_r1_2(ptr addrspace(1) nocapture readonly %arg0, i1 %cond) local_unnamed_addr {
+; CHECK-LABEL: define <8 x half> @shuffle_v4_v8f16_cond_r1_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]], i1 [[COND:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x half>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[TMP0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[TMP0]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x half> [[VAL3]]
+;
+entry:
+  %val0 = load <4 x half>, ptr addrspace(1) %arg0, align 32
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <4 x half> %val0, <4 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  br label %finally
+
+else:
+  %val2 = shufflevector <4 x half> %val0, <4 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  br label %finally
+
+finally:
+  %val3 = phi <8 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x half> %val3
+}
+
+define <8 x i32> @shuffle_v4_v8i32_r0_1(ptr addrspace(1) nocapture readonly %arg0) local_unnamed_addr {
+; CHECK-LABEL: define <8 x i32> @shuffle_v4_v8i32_r0_1(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0, align 32
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
+  ret <8 x i32> %val1
+}
+
+define <8 x i32> @shuffle_v4_v8i32_r0_2(ptr addrspace(1) nocapture readonly %arg0) local_unnamed_addr {
+; CHECK-LABEL: define <8 x i32> @shuffle_v4_v8i32_r0_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x i32>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[TMP0]], <3 x i32> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0, align 32
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 2, i32 2, i32 2, i32 2>
+  ret <8 x i32> %val1
+}
+
+define <4 x i32> @shuffle_v4_v4i32_r1_2(ptr addrspace(1) nocapture readonly %arg0) local_unnamed_addr {
+; CHECK-LABEL: define <4 x i32> @shuffle_v4_v4i32_r1_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x i32>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[TMP0]], <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 2>
+; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+;
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0, align 32
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 2>
+  ret <4 x i32> %val1
+}
+
+define <8 x i32> @shuffle_v4_v8i32_r1_2(ptr addrspace(1) nocapture readonly %arg0) local_unnamed_addr {
+; CHECK-LABEL: define <8 x i32> @shuffle_v4_v8i32_r1_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x i32>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x i32> [[TMP0]], <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
+;
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0, align 32
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2>
+  ret <8 x i32> %val1
+}
+
+define <8 x i32> @shuffle_v4_v8i32_cond_r0_1(ptr addrspace(1) nocapture readonly %arg0, i1 %cond) local_unnamed_addr {
+; CHECK-LABEL: define <8 x i32> @shuffle_v4_v8i32_cond_r0_1(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]], i1 [[COND:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0, align 32
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  br label %finally
+
+else:
+  %val2 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v4_v8i32_cond_r0_2(ptr addrspace(1) nocapture readonly %arg0, i1 %cond) local_unnamed_addr {
+; CHECK-LABEL: define <8 x i32> @shuffle_v4_v8i32_cond_r0_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]], i1 [[COND:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x i32>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[TMP0]], <3 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[TMP0]], <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0, align 32
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+  br label %finally
+
+else:
+  %val2 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <4 x i32> @shuffle_v4_v4i32_cond_r1_2(ptr addrspace(1) nocapture readonly %arg0, i1 %cond) local_unnamed_addr {
+; CHECK-LABEL: define <4 x i32> @shuffle_v4_v4i32_cond_r1_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]], i1 [[COND:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x i32>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[TMP0]], <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[TMP0]], <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i32> [[VAL3]]
+;
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0, align 32
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  br label %finally
+
+else:
+  %val2 = shufflevector <4 x i32> %val0, <4 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v4_v8i32_cond_r1_2(ptr addrspace(1) nocapture readonly %arg0, i1 %cond) local_unnamed_addr {
+; CHECK-LABEL: define <8 x i32> @shuffle_v4_v8i32_cond_r1_2(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]], i1 [[COND:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <3 x i32>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[TMP0]], <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[TMP0]], <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0, align 32
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  br label %finally
+
+else:
+  %val2 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v4_v8i32_cond_r1_4(ptr addrspace(1) nocapture readonly %arg0, i1 %cond) local_unnamed_addr {
+; CHECK-LABEL: define <8 x i32> @shuffle_v4_v8i32_cond_r1_4(
+; CHECK-SAME: ptr addrspace(1) readonly captures(none) [[ARG0:%.*]], i1 [[COND:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[VAL0:%.*]] = load <4 x i32>, ptr addrspace(1) [[ARG0]], align 32
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <4 x i32> [[VAL0]], <4 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <4 x i32> [[VAL0]], <4 x i32> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4>
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  %val0 = load <4 x i32>, ptr addrspace(1) %arg0, align 32
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2>
+  br label %finally
+
+else:
+  %val2 = shufflevector <4 x i32> %val0, <4 x i32> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4>
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <16 x i8> @shuffle_v16_v16i8_r0_31(ptr %arg) {
+; CHECK-LABEL: define <16 x i8> @shuffle_v16_v16i8_r0_31(
+; CHECK-SAME: ptr [[ARG:%.*]]) {
+; CHECK-NEXT:    [[LOAD:%.*]] = load <16 x i8>, ptr [[ARG]], align 1
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <16 x i8> [[LOAD]], <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT:    ret <16 x i8> [[SHUF]]
+;
+  %load= load <16 x i8>, ptr %arg, align 1
+  %shuf = shufflevector <16 x i8> %load, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  ret <16 x i8> %shuf
+}
diff --git a/llvm/test/Transforms/VectorCombine/narrow-phi-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/narrow-phi-of-shuffles.ll
new file mode 100644
index 0000000000000..b70840f85fbf2
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/narrow-phi-of-shuffles.ll
@@ -0,0 +1,1266 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -passes=vector-combine -S %s | FileCheck %s --check-prefixes=CHECK
+
+define <2 x i8> @shuffle_v2i8(<2 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i8> @shuffle_v2i8(
+; CHECK-SAME: <2 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i8> [[ARG0]], <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i8> %arg0, <2 x i8> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i8> %val3
+}
+
+define <4 x i8> @shuffle_v4i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i8> @shuffle_v4i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i8> %val3
+}
+
+define <8 x i8> @shuffle_v8i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i8> @shuffle_v8i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i8> %val3
+}
+
+define <16 x i8> @shuffle_v16i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i8> @shuffle_v16i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i8> %val3
+}
+
+define <32 x i8> @shuffle_v32i8(<3 x i8> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i8> @shuffle_v32i8(
+; CHECK-SAME: <3 x i8> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i8> [[ARG0]], <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i8> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i8> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i8> %arg0, <3 x i8> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i8> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i8> %val3
+}
+
+define <2 x i16> @shuffle_v2i16(<2 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i16> @shuffle_v2i16(
+; CHECK-SAME: <2 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i16> [[ARG0]], <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i16> %arg0, <2 x i16> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i16> %val3
+}
+
+define <4 x i16> @shuffle_v4i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i16> @shuffle_v4i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i16> %val3
+}
+
+define <8 x i16> @shuffle_v8i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i16> @shuffle_v8i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i16> %val3
+}
+
+define <16 x i16> @shuffle_v16i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i16> @shuffle_v16i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i16> %val3
+}
+
+define <32 x i16> @shuffle_v32i16(<3 x i16> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i16> @shuffle_v32i16(
+; CHECK-SAME: <3 x i16> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i16> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i16> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i16> %arg0, <3 x i16> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i16> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i16> %val3
+}
+
+define <2 x i32> @shuffle_v2i32(<2 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x i32> @shuffle_v2i32(
+; CHECK-SAME: <2 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x i32> [[ARG0]], <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x i32> %arg0, <2 x i32> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x i32> %val3
+}
+
+define <4 x i32> @shuffle_v4i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x i32> @shuffle_v4i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x i32> %val3
+}
+
+define <8 x i32> @shuffle_v8i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x i32> @shuffle_v8i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x i32> %val3
+}
+
+define <16 x i32> @shuffle_v16i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x i32> @shuffle_v16i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x i32> %val3
+}
+
+define <32 x i32> @shuffle_v32i32(<3 x i32> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x i32> @shuffle_v32i32(
+; CHECK-SAME: <3 x i32> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x i32> [[ARG0]], <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x i32> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x i32> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x i32> %arg0, <3 x i32> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x i32> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x i32> %val3
+}
+
+define <2 x bfloat> @shuffle_v2bf16(<2 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x bfloat> @shuffle_v2bf16(
+; CHECK-SAME: <2 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x bfloat> [[ARG0]], <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x bfloat> %arg0, <2 x bfloat> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x bfloat> %val3
+}
+
+define <3 x bfloat> @shuffle_v3bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x bfloat> @shuffle_v3bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x bfloat> %val3
+}
+
+define <4 x bfloat> @shuffle_v4bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x bfloat> @shuffle_v4bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x bfloat> %val3
+}
+
+define <6 x bfloat> @shuffle_v6bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x bfloat> @shuffle_v6bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x bfloat> %val3
+}
+
+define <8 x bfloat> @shuffle_v8bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x bfloat> @shuffle_v8bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x bfloat> %val3
+}
+
+define <16 x bfloat> @shuffle_v16bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x bfloat> @shuffle_v16bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x bfloat> %val3
+}
+
+define <32 x bfloat> @shuffle_v32bf16(<3 x bfloat> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x bfloat> @shuffle_v32bf16(
+; CHECK-SAME: <3 x bfloat> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x bfloat> [[ARG0]], <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x bfloat> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x bfloat> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x bfloat> %arg0, <3 x bfloat> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x bfloat> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x bfloat> %val3
+}
+
+define <2 x half> @shuffle_v2f16(<2 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x half> @shuffle_v2f16(
+; CHECK-SAME: <2 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x half> [[ARG0]], <2 x half> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x half> %arg0, <2 x half> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x half> %val3
+}
+
+define <3 x half> @shuffle_v3f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x half> @shuffle_v3f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x half> %val3
+}
+
+define <4 x half> @shuffle_v4f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x half> @shuffle_v4f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x half> %val3
+}
+
+define <6 x half> @shuffle_v6f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x half> @shuffle_v6f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x half> %val3
+}
+
+define <8 x half> @shuffle_v8f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x half> @shuffle_v8f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x half> %val3
+}
+
+define <16 x half> @shuffle_v16f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x half> @shuffle_v16f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x half> %val3
+}
+
+define <32 x half> @shuffle_v32f16(<3 x half> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x half> @shuffle_v32f16(
+; CHECK-SAME: <3 x half> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x half> [[ARG0]], <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x half> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x half> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x half> %arg0, <3 x half> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x half> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x half> %val3
+}
+
+define <2 x float> @shuffle_v2f32(<2 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <2 x float> @shuffle_v2f32(
+; CHECK-SAME: <2 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <2 x float> [[ARG0]], <2 x float> poison, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <2 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <2 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <2 x float> %arg0, <2 x float> poison, <2 x i32> <i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <2 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <2 x float> %val3
+}
+
+define <3 x float> @shuffle_v3f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <3 x float> @shuffle_v3f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <3 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <3 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <3 x i32> <i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <3 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <3 x float> %val3
+}
+
+define <4 x float> @shuffle_v4f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <4 x float> @shuffle_v4f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <4 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <4 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <4 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <4 x float> %val3
+}
+
+define <6 x float> @shuffle_v6f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <6 x float> @shuffle_v6f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <6 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <6 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <6 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <6 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <6 x float> %val3
+}
+
+define <8 x float> @shuffle_v8f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <8 x float> @shuffle_v8f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <8 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <8 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <8 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <8 x float> %val3
+}
+
+define <16 x float> @shuffle_v16f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <16 x float> @shuffle_v16f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <16 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <16 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <16 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <16 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <16 x float> %val3
+}
+
+define <32 x float> @shuffle_v32f32(<3 x float> %arg0, i1 %cond) {
+; CHECK-LABEL: define <32 x float> @shuffle_v32f32(
+; CHECK-SAME: <3 x float> [[ARG0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]]
+; CHECK:       [[THEN]]:
+; CHECK-NEXT:    [[VAL1:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    tail call void @func0()
+; CHECK-NEXT:    br label %[[FINALLY:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[VAL2:%.*]] = shufflevector <3 x float> [[ARG0]], <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    tail call void @func1()
+; CHECK-NEXT:    br label %[[FINALLY]]
+; CHECK:       [[FINALLY]]:
+; CHECK-NEXT:    [[VAL3:%.*]] = phi <32 x float> [ [[VAL1]], %[[THEN]] ], [ [[VAL2]], %[[ELSE]] ]
+; CHECK-NEXT:    ret <32 x float> [[VAL3]]
+;
+entry:
+  br i1 %cond, label %then, label %else
+
+then:
+  %val1 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  tail call void @func0()
+  br label %finally
+
+else:
+  %val2 = shufflevector <3 x float> %arg0, <3 x float> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
+  tail call void @func1()
+  br label %finally
+
+finally:
+  %val3 = phi <32 x float> [ %val1, %then ], [ %val2, %else ]
+  ret <32 x float> %val3
+}
+
+declare void @func0() local_unnamed_addr
+
+declare void @func1() local_unnamed_addr
diff --git a/llvm/test/Verifier/AMDGPU/intrinsic-amdgcn-call-whole-wave.ll b/llvm/test/Verifier/AMDGPU/intrinsic-amdgcn-call-whole-wave.ll
new file mode 100644
index 0000000000000..a744bf318be9a
--- /dev/null
+++ b/llvm/test/Verifier/AMDGPU/intrinsic-amdgcn-call-whole-wave.ll
@@ -0,0 +1,46 @@
+; RUN: not llvm-as %s -disable-output 2>&1 | FileCheck %s
+
+define amdgpu_cs void @indirect(ptr %fn, i32 %x) {
+  ; CHECK: Indirect whole wave calls are not allowed
+  %whatever = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr %fn, i32 %x)
+  ret void
+}
+
+declare amdgpu_gfx_whole_wave void @variadic_callee(i1 %active, i32 %x, ...)
+
+define amdgpu_cs void @variadic(ptr %fn, i32 %x) {
+  ; CHECK: Variadic whole wave calls are not allowed
+  %whatever = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @variadic_callee, i32 %x)
+  ret void
+}
+
+declare amdgpu_gfx void @bad_cc_callee(i1 %active, i32 %x)
+
+define amdgpu_cs void @bad_cc(i32 %x) {
+  ; CHECK: Callee must have the amdgpu_gfx_whole_wave calling convention
+  %whatever = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @bad_cc_callee, i32 %x)
+  ret void
+}
+
+declare amdgpu_gfx_whole_wave i32 @no_i1_callee(i32 %active, i32 %y, i32 %z)
+
+define amdgpu_cs void @no_i1(i32 %x) {
+  ; CHECK: Callee must have i1 as its first argument
+  %whatever = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @no_i1_callee, i32 %x, i32 0)
+  ret void
+}
+
+declare amdgpu_gfx_whole_wave i32 @good_callee(i1 %active, i32 %x, i32 inreg %y)
+
+define amdgpu_cs void @bad_args(i32 %x) {
+  ; CHECK: Call argument count must match callee argument count
+  %whatever.0 = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @good_callee, i32 %x)
+
+  ; CHECK: Argument types must match
+  %whatever.1 = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @good_callee, i32 %x, i64 inreg 0)
+
+  ; CHECK: Argument inreg attributes must match
+  %whatever.2 = call i32(ptr, ...) @llvm.amdgcn.call.whole.wave(ptr @good_callee, i32 %x, i32 0)
+
+  ret void
+}
diff --git a/llvm/test/Verifier/intrinsic-immarg.ll b/llvm/test/Verifier/intrinsic-immarg.ll
index c1bb9323b8701..d5aef3dcbbfe0 100644
--- a/llvm/test/Verifier/intrinsic-immarg.ll
+++ b/llvm/test/Verifier/intrinsic-immarg.ll
@@ -163,26 +163,6 @@ define void @test_scatter_8i32(<8 x i32> %a1, <8 x ptr> %ptr, <8 x i1> %mask, i3
   ret void
 }
 
-declare void @llvm.lifetime.start.p0(i64, ptr)
-define void @test_lifetime_start(i64 %arg0) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i64 %arg0
-  ; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 %arg0, ptr %ptr)
-  %ptr = alloca i64
-  call void @llvm.lifetime.start.p0(i64 %arg0, ptr %ptr)
-  ret void
-}
-
-declare void @llvm.lifetime.end.p0(i64, ptr)
-define void @test_lifetime_end(i64 %arg0) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i64 %arg0
-  ; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 %arg0, ptr %ptr)
-  %ptr = alloca i64
-  call void @llvm.lifetime.end.p0(i64 %arg0, ptr %ptr)
-  ret void
-}
-
 declare ptr @llvm.invariant.start.p0(i64, ptr)
 define void @test_invariant_start(i64 %arg0, ptr %ptr) {
   ; CHECK: immarg operand has non-immediate parameter
diff --git a/llvm/test/Verifier/opaque-ptr.ll b/llvm/test/Verifier/opaque-ptr.ll
index 10e43a45e5740..3ac90448228be 100644
--- a/llvm/test/Verifier/opaque-ptr.ll
+++ b/llvm/test/Verifier/opaque-ptr.ll
@@ -40,13 +40,13 @@ define void @atomicrmw(ptr %a, i32 %i) {
 define void @opaque_mangle() {
 ; CHECK-LABEL: @opaque_mangle(
 ; CHECK-NEXT:    [[A:%.*]] = alloca i64, align 8
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 8, ptr [[A]])
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 8, ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[A]])
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[A]])
 ; CHECK-NEXT:    ret void
 ;
   %a = alloca i64
-  call void @llvm.lifetime.start.p0(i64 8, ptr %a)
-  call void @llvm.lifetime.end.p0(i64 8, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -65,10 +65,8 @@ define void @intrinsic_calls(ptr %a) {
   ret void
 }
 
-; CHECK: @llvm.lifetime.start.p0
-; CHECK: @llvm.lifetime.end.p0
-declare void @llvm.lifetime.start.p0(i64, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 
 declare <2 x i32> @llvm.masked.load.v2i32.p0(ptr, i32, <2 x i1>, <2 x i32>)
 declare void @llvm.masked.store.v2i32.p0(<2 x i32>, ptr, i32, <2 x i1>)
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 8c2d1a454e8f9..bc240425d6d0e 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -715,3 +715,13 @@ def host_unwind_supports_jit():
 
 if config.has_logf128:
     config.available_features.add("has_logf128")
+
+if lit_config.update_tests:
+    import sys
+    import os
+
+    utilspath = os.path.join(config.llvm_src_root, "utils")
+    sys.path.append(utilspath)
+    from update_any_test_checks import utc_lit_plugin
+
+    lit_config.test_updaters.append(utc_lit_plugin)
diff --git a/llvm/test/lit.site.cfg.py.in b/llvm/test/lit.site.cfg.py.in
index 893e2cbd4f62b..973e0ec934a52 100644
--- a/llvm/test/lit.site.cfg.py.in
+++ b/llvm/test/lit.site.cfg.py.in
@@ -66,7 +66,6 @@ config.spirv_tools_tests = @LLVM_INCLUDE_SPIRV_TOOLS_TESTS@
 config.have_vc_rev = @LLVM_APPEND_VC_REV@
 config.force_vc_rev = "@LLVM_FORCE_VC_REVISION@"
 config.has_logf128 = @LLVM_HAS_LOGF128@
-config.has_key_instructions = @LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS@
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)
diff --git a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected
index 5751738f7bb3b..65904d13c1c96 100644
--- a/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_analyze_test_checks/Inputs/loop-distribute.ll.expected
@@ -72,12 +72,12 @@ define void @ldist(i1 %cond, ptr %A, ptr %B, ptr %C) {
 ; CHECK-NEXT:  LDist: Partition 0:
 ; CHECK-NEXT:  for.body.ldist1: ; preds = %if.end.ldist1, %for.body.ph.ldist1
 ; CHECK-NEXT:    %iv.ldist1 = phi i16 [ 0, %for.body.ph.ldist1 ], [ %iv.next.ldist1, %if.end.ldist1 ]
-; CHECK-NEXT:    %lv.ldist1 = load i16, ptr %A, align 1, !alias.scope !0, !noalias !3
-; CHECK-NEXT:    store i16 %lv.ldist1, ptr %A, align 1, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    %lv.ldist1 = load i16, ptr %A, align 1, !alias.scope !2, !noalias !5
+; CHECK-NEXT:    store i16 %lv.ldist1, ptr %A, align 1, !alias.scope !2, !noalias !5
 ; CHECK-NEXT:    br i1 %cond, label %if.then.ldist1, label %if.end.ldist1
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  if.then.ldist1: ; preds = %for.body.ldist1
-; CHECK-NEXT:    %lv2.ldist1 = load i16, ptr %A, align 1, !alias.scope !0, !noalias !3
+; CHECK-NEXT:    %lv2.ldist1 = load i16, ptr %A, align 1, !alias.scope !2, !noalias !5
 ; CHECK-NEXT:    br label %if.end.ldist1
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  if.end.ldist1: ; preds = %if.then.ldist1, %for.body.ldist1
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
index e1da112ceebb5..c368a1c7e8ebe 100644
--- a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/nvptx-basic.ll.expected
@@ -6,8 +6,8 @@
 define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct.St8x4) align 8 %in, ptr nocapture noundef writeonly %ret) {
 ; CHECK-LABEL: caller_St8x4(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<3>;
-; CHECK-NEXT:    .reg .b64 %rd<13>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<9>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    { // callseq 0, 0
@@ -23,11 +23,11 @@ define dso_local void @caller_St8x4(ptr nocapture noundef readonly byval(%struct
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd5, %rd6}, [retval0];
 ; CHECK-NEXT:    ld.param.v2.b64 {%rd7, %rd8}, [retval0+16];
 ; CHECK-NEXT:    } // callseq 0
-; CHECK-NEXT:    ld.param.b32 %r2, [caller_St8x4_param_1];
-; CHECK-NEXT:    st.b64 [%r2], %rd5;
-; CHECK-NEXT:    st.b64 [%r2+8], %rd6;
-; CHECK-NEXT:    st.b64 [%r2+16], %rd7;
-; CHECK-NEXT:    st.b64 [%r2+24], %rd8;
+; CHECK-NEXT:    ld.param.b32 %r1, [caller_St8x4_param_1];
+; CHECK-NEXT:    st.b64 [%r1], %rd5;
+; CHECK-NEXT:    st.b64 [%r1+8], %rd6;
+; CHECK-NEXT:    st.b64 [%r1+16], %rd7;
+; CHECK-NEXT:    st.b64 [%r1+24], %rd8;
 ; CHECK-NEXT:    ret;
   %call = tail call fastcc [4 x i64] @callee_St8x4(ptr noundef nonnull byval(%struct.St8x4) align 8 %in) #2
   %.fca.0.extract = extractvalue [4 x i64] %call, 0
@@ -48,7 +48,6 @@ define internal fastcc [4 x i64] @callee_St8x4(ptr nocapture noundef readonly by
 ; CHECK-LABEL: callee_St8x4(
 ; CHECK:         // @callee_St8x4
 ; CHECK-NEXT:  {
-; CHECK-NEXT:    .reg .b32 %r<2>;
 ; CHECK-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/target-triple-mismatch.ll b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/target-triple-mismatch.ll
new file mode 100644
index 0000000000000..3da27cbacd172
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/Inputs/target-triple-mismatch.ll
@@ -0,0 +1,7 @@
+; RUN: llc < %s -mtriple=arm64-apple-darwin | FileCheck %s
+
+define i64 @foo(i64 %a) {
+entry:
+  %b = add i64 %a, 1
+  ret i64 %b
+}
diff --git a/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/target-triple-mismatch.test b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/target-triple-mismatch.test
new file mode 100644
index 0000000000000..3bbf14d469d4b
--- /dev/null
+++ b/llvm/test/tools/UpdateTestChecks/update_llc_test_checks/target-triple-mismatch.test
@@ -0,0 +1,11 @@
+# REQUIRES: aarch64-registered-target
+## Check that arm64-apple-darwin target triple is wrongly captured as arm64 (non-Apple)
+
+# RUN: cp -f %S/Inputs/target-triple-mismatch.ll %t.ll
+# RUN: %update_llc_test_checks %t.ll 2>&1 | FileCheck %s --check-prefix=LOG
+# RUN: FileCheck --input-file=%t.ll %s --check-prefix=AUTOGEN
+
+# LOG: WARNING: Couldn't match any function. Possibly the wrong target triple has been provided
+
+# AUTOGEN: ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# AUTOGEN-NEXT: ; CHECK: {{.*}}
diff --git a/llvm/test/tools/UpdateTestChecks/update_mir_test_checks/Inputs/x86-MIFlags.mir b/llvm/test/tools/UpdateTestChecks/update_mir_test_checks/Inputs/x86-MIFlags.mir
index ef6632711e463..3b03b131a1e14 100644
--- a/llvm/test/tools/UpdateTestChecks/update_mir_test_checks/Inputs/x86-MIFlags.mir
+++ b/llvm/test/tools/UpdateTestChecks/update_mir_test_checks/Inputs/x86-MIFlags.mir
@@ -51,8 +51,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/tools/UpdateTestChecks/update_mir_test_checks/Inputs/x86-MIFlags.mir.expected b/llvm/test/tools/UpdateTestChecks/update_mir_test_checks/Inputs/x86-MIFlags.mir.expected
index f4027c1e0db51..4b7a7108256d7 100644
--- a/llvm/test/tools/UpdateTestChecks/update_mir_test_checks/Inputs/x86-MIFlags.mir.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_mir_test_checks/Inputs/x86-MIFlags.mir.expected
@@ -52,8 +52,8 @@ frameInfo:
   hasMustTailInVarArgFunc: false
   hasTailCall:     false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 callSites:       []
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll
index 96ff2d7ea5fed..ef601189855b4 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll
@@ -14,7 +14,7 @@ entry:
     #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17)
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !22
     #dbg_declare(ptr %i, !14, !DIExpression(), !23)
   store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24
   br label %for.cond, !dbg !22
@@ -27,7 +27,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !33
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -49,10 +49,10 @@ for.end:                                          ; preds = %for.cond.cleanup
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define dso_local void @bar(ptr %A) #0 !dbg !41 {
@@ -61,7 +61,7 @@ entry:
   %i = alloca i32, align 4
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !47
     #dbg_declare(ptr %i, !44, !DIExpression(), !48)
   store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24
   br label %for.cond, !dbg !47
@@ -74,7 +74,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !55
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.expected
index 65048305a2277..4bae52e7b3815 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.expected
@@ -16,7 +16,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:      #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]])
 ; CHECK-NEXT:    store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG22]]
@@ -27,7 +27,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]]
@@ -50,7 +50,7 @@ entry:
     #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17)
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !22
     #dbg_declare(ptr %i, !14, !DIExpression(), !23)
   store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24
   br label %for.cond, !dbg !22
@@ -63,7 +63,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !33
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -85,10 +85,10 @@ for.end:                                          ; preds = %for.cond.cleanup
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define dso_local void @bar(ptr %A) #0 !dbg !41 {
@@ -98,7 +98,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG47]]
@@ -109,7 +109,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]]
@@ -131,7 +131,7 @@ entry:
   %i = alloca i32, align 4
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !47
     #dbg_declare(ptr %i, !44, !DIExpression(), !48)
   store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24
   br label %for.cond, !dbg !47
@@ -144,7 +144,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !55
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected
index 7c1ea5e716b99..12c6e4eee0147 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.expected
@@ -17,7 +17,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:      #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]])
 ; CHECK-NEXT:    store ptr [[A]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG22]]
@@ -28,7 +28,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]]
@@ -51,7 +51,7 @@ entry:
     #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17)
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !22
     #dbg_declare(ptr %i, !14, !DIExpression(), !23)
   store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24
   br label %for.cond, !dbg !22
@@ -64,7 +64,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !33
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -86,10 +86,10 @@ for.end:                                          ; preds = %for.cond.cleanup
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define dso_local void @bar(ptr %A) #0 !dbg !41 {
@@ -100,7 +100,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store ptr [[A]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG47]]
@@ -111,7 +111,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]]
@@ -133,7 +133,7 @@ entry:
   %i = alloca i32, align 4
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !47
     #dbg_declare(ptr %i, !44, !DIExpression(), !48)
   store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24
   br label %for.cond, !dbg !47
@@ -146,7 +146,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !55
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected
index 94af952353a3c..d67a303236369 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.globals.expected
@@ -17,7 +17,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:      #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]])
 ; CHECK-NEXT:    store ptr [[A]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG22]]
@@ -28,7 +28,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]]
@@ -51,7 +51,7 @@ entry:
     #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17)
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !22
     #dbg_declare(ptr %i, !14, !DIExpression(), !23)
   store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24
   br label %for.cond, !dbg !22
@@ -64,7 +64,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !33
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -86,10 +86,10 @@ for.end:                                          ; preds = %for.cond.cleanup
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define dso_local void @bar(ptr %A) #0 !dbg !41 {
@@ -100,7 +100,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store ptr [[A]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG47]]
@@ -111,7 +111,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]]
@@ -133,7 +133,7 @@ entry:
   %i = alloca i32, align 4
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !47
     #dbg_declare(ptr %i, !44, !DIExpression(), !48)
   store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24
   br label %for.cond, !dbg !47
@@ -146,7 +146,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !55
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.noglobals.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.noglobals.expected
index 65048305a2277..4bae52e7b3815 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.noglobals.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.noglobals.expected
@@ -16,7 +16,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:      #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]])
 ; CHECK-NEXT:    store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG22]]
@@ -27,7 +27,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]]
@@ -50,7 +50,7 @@ entry:
     #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17)
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !22
     #dbg_declare(ptr %i, !14, !DIExpression(), !23)
   store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24
   br label %for.cond, !dbg !22
@@ -63,7 +63,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !33
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -85,10 +85,10 @@ for.end:                                          ; preds = %for.cond.cleanup
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define dso_local void @bar(ptr %A) #0 !dbg !41 {
@@ -98,7 +98,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG47]]
@@ -109,7 +109,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]]
@@ -131,7 +131,7 @@ entry:
   %i = alloca i32, align 4
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !47
     #dbg_declare(ptr %i, !44, !DIExpression(), !48)
   store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24
   br label %for.cond, !dbg !47
@@ -144,7 +144,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !55
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
diff --git a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.transitiveglobals.expected b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.transitiveglobals.expected
index a656c4aefa0e1..fb3a76f305e6a 100644
--- a/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.transitiveglobals.expected
+++ b/llvm/test/tools/UpdateTestChecks/update_test_checks/Inputs/various_ir_values_dbgrecords.ll.funcsig.transitiveglobals.expected
@@ -16,7 +16,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:      #dbg_assign(i1 undef, [[META13:![0-9]+]], !DIExpression(), [[DIASSIGNID16]], ptr [[A_ADDR]], !DIExpression(), [[META17:![0-9]+]])
 ; CHECK-NEXT:    store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META13]], !DIExpression(), [[META17]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2:[0-9]+]], !dbg [[DBG22:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META14:![0-9]+]], !DIExpression(), [[META23:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META23]], !tbaa [[TBAA24:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG22]]
@@ -27,7 +27,7 @@ define dso_local void @foo(ptr %A) #0 !dbg !7 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG30:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG31:![0-9]+]], !prof [[PROF32:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG33:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG34:![0-9]+]], !tbaa [[TBAA18]]
@@ -50,7 +50,7 @@ entry:
     #dbg_assign(i1 undef, !13, !DIExpression(), !16, ptr %A.addr, !DIExpression(), !17)
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !13, !DIExpression(), !17)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !22
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !22
     #dbg_declare(ptr %i, !14, !DIExpression(), !23)
   store i32 0, ptr %i, align 4, !dbg !23, !tbaa !24
   br label %for.cond, !dbg !22
@@ -63,7 +63,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !31, !prof !32
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !33
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !33
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
@@ -85,10 +85,10 @@ for.end:                                          ; preds = %for.cond.cleanup
 }
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.start.p0(ptr nocapture) #1
 
 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1
+declare void @llvm.lifetime.end.p0(ptr nocapture) #1
 
 ; Function Attrs: nounwind uwtable
 define dso_local void @bar(ptr %A) #0 !dbg !41 {
@@ -98,7 +98,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    store ptr [[A:%.*]], ptr [[A_ADDR]], align 8, !tbaa [[TBAA18]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[A_ADDR]], [[META43:![0-9]+]], !DIExpression(), [[META46:![0-9]+]])
-; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG47:![0-9]+]]
 ; CHECK-NEXT:      #dbg_declare(ptr [[I]], [[META44:![0-9]+]], !DIExpression(), [[META48:![0-9]+]])
 ; CHECK-NEXT:    store i32 0, ptr [[I]], align 4, !dbg [[META48]], !tbaa [[TBAA24]]
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]], !dbg [[DBG47]]
@@ -109,7 +109,7 @@ define dso_local void @bar(ptr %A) #0 !dbg !41 {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], [[TMP2]], !dbg [[DBG53:![0-9]+]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_COND_CLEANUP:%.*]], !dbg [[DBG54:![0-9]+]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    call void @llvm.lifetime.end.p0(i64 4, ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0(ptr [[I]]) #[[ATTR2]], !dbg [[DBG55:![0-9]+]]
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[A_ADDR]], align 8, !dbg [[DBG56:![0-9]+]], !tbaa [[TBAA18]]
@@ -131,7 +131,7 @@ entry:
   %i = alloca i32, align 4
   store ptr %A, ptr %A.addr, align 8, !tbaa !18
     #dbg_declare(ptr %A.addr, !43, !DIExpression(), !46)
-  call void @llvm.lifetime.start.p0(i64 4, ptr %i) #2, !dbg !47
+  call void @llvm.lifetime.start.p0(ptr %i) #2, !dbg !47
     #dbg_declare(ptr %i, !44, !DIExpression(), !48)
   store i32 0, ptr %i, align 4, !dbg !48, !tbaa !24
   br label %for.cond, !dbg !47
@@ -144,7 +144,7 @@ for.cond:                                         ; preds = %for.inc, %entry
   br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !54
 
 for.cond.cleanup:                                 ; preds = %for.cond
-  call void @llvm.lifetime.end.p0(i64 4, ptr %i) #2, !dbg !55
+  call void @llvm.lifetime.end.p0(ptr %i) #2, !dbg !55
   br label %for.end
 
 for.body:                                         ; preds = %for.cond
diff --git a/llvm/test/tools/dxil-dis/lifetimes.ll b/llvm/test/tools/dxil-dis/lifetimes.ll
index cb3e6291c7bc0..af7a19a4d4f66 100644
--- a/llvm/test/tools/dxil-dis/lifetimes.ll
+++ b/llvm/test/tools/dxil-dis/lifetimes.ll
@@ -4,19 +4,17 @@ target triple = "dxil-unknown-shadermodel6.7-library"
 define void @test_lifetimes()  {
 ; CHECK-LABEL: test_lifetimes
 ; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [2 x i32], align 4
-; CHECK-NEXT: [[GEP:%.*]] = getelementptr [2 x i32], [2 x i32]* [[ALLOCA]], i32 0, i32 0
 ; CHECK-NEXT: [[BITCAST:%.*]] = bitcast [2 x i32]* [[ALLOCA]] to i8*
-; CHECK-NEXT: call void @llvm.lifetime.start(i64 4, i8* nonnull [[BITCAST]])
-; CHECK-NEXT: store i32 0, i32* [[GEP]], align 4
+; CHECK-NEXT: call void @llvm.lifetime.start(i64 8, i8* nonnull [[BITCAST]])
 ; CHECK-NEXT: [[BITCAST:%.*]] = bitcast [2 x i32]* [[ALLOCA]] to i8*
-; CHECK-NEXT: call void @llvm.lifetime.end(i64 4, i8* nonnull [[BITCAST]])
+; CHECK-NEXT: call void @llvm.lifetime.end(i64 8, i8* nonnull [[BITCAST]])
 ; CHECK-NEXT: ret void
 ;
   %a = alloca [2 x i32], align 4
   %gep = getelementptr [2 x i32], ptr %a, i32 0, i32 0
-  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %a)
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
   store i32 0, ptr %gep, align 4
-  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %a)
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret void
 }
 
@@ -29,10 +27,10 @@ define void @test_lifetimes()  {
 ; CHECK-DAG: declare void @llvm.lifetime.end(i64, i8* nocapture) [[LIFETIME_ATTRS]]
 
 ; Function Attrs: nounwind memory(argmem: readwrite)
-declare void @llvm.lifetime.end.p0(i64, ptr) #0
+declare void @llvm.lifetime.end.p0(ptr) #0
 
 ; Function Attrs: nounwind memory(argmem: readwrite)
-declare void @llvm.lifetime.start.p0(i64, ptr) #0
+declare void @llvm.lifetime.start.p0(ptr) #0
 
 attributes #0 = { nounwind memory(argmem: readwrite) }
 
diff --git a/llvm/test/tools/dxil-dis/opaque-value_as_metadata.ll b/llvm/test/tools/dxil-dis/opaque-value_as_metadata.ll
index 165d5d2e520df..c53bb2127ebcc 100644
--- a/llvm/test/tools/dxil-dis/opaque-value_as_metadata.ll
+++ b/llvm/test/tools/dxil-dis/opaque-value_as_metadata.ll
@@ -7,6 +7,7 @@ target triple = "dxil-unknown-shadermodel6.7-library"
 @CBV = external constant %"$Globals"
 
 define void @main() #0 {
+  %1 = load float, ptr @CBV, align 4
   ret void
 }
 
diff --git a/llvm/test/tools/llvm-cgdata/empty.test b/llvm/test/tools/llvm-cgdata/empty.test
index 0d2b0e848a2c9..2082eca58f073 100644
--- a/llvm/test/tools/llvm-cgdata/empty.test
+++ b/llvm/test/tools/llvm-cgdata/empty.test
@@ -16,7 +16,7 @@ RUN: llvm-cgdata --show %t_emptyheader.cgdata | count 0
 
 # The version number appears when asked, as it's in the header
 RUN: llvm-cgdata --show --cgdata-version %t_emptyheader.cgdata | FileCheck %s --check-prefix=VERSION
-VERSION: Version: 3
+VERSION: Version: 4
 
 # When converting a binary file (w/ the header only) to a text file, it's an empty file as the text format does not have an explicit header.
 RUN: llvm-cgdata --convert %t_emptyheader.cgdata --format text | count 0
@@ -30,7 +30,7 @@ RUN: llvm-cgdata --convert %t_emptyheader.cgdata --format text | count 0
 #   uint64_t StableFunctionMapOffset;
 # }
 RUN: printf '\xffcgdata\x81' > %t_header.cgdata
-RUN: printf '\x03\x00\x00\x00' >> %t_header.cgdata
+RUN: printf '\x04\x00\x00\x00' >> %t_header.cgdata
 RUN: printf '\x00\x00\x00\x00' >> %t_header.cgdata
 RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata
 RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata
diff --git a/llvm/test/tools/llvm-cgdata/error.test b/llvm/test/tools/llvm-cgdata/error.test
index 92ff484e31caf..9484371848a72 100644
--- a/llvm/test/tools/llvm-cgdata/error.test
+++ b/llvm/test/tools/llvm-cgdata/error.test
@@ -22,9 +22,9 @@ RUN: printf '\xffcgdata\x81' > %t_corrupt.cgdata
 RUN: not llvm-cgdata --show %t_corrupt.cgdata 2>&1 | FileCheck %s  --check-prefix=CORRUPT
 CORRUPT: {{.}}cgdata: invalid codegen data (file header is corrupt)
 
-# The current version 3 while the header says 4.
+# The current version 4 while the header says 5.
 RUN: printf '\xffcgdata\x81' > %t_version.cgdata
-RUN: printf '\x04\x00\x00\x00' >> %t_version.cgdata
+RUN: printf '\x05\x00\x00\x00' >> %t_version.cgdata
 RUN: printf '\x00\x00\x00\x00' >> %t_version.cgdata
 RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata
 RUN: printf '\x20\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata
diff --git a/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test b/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test
index b060872113b1b..70b83af407e5a 100644
--- a/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test
+++ b/llvm/test/tools/llvm-cgdata/merge-combined-funcmap-hashtree.test
@@ -23,6 +23,8 @@ RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-both-hashtree-funcma
 # Merge an object file having cgdata (__llvm_outline and __llvm_merge)
 RUN: llvm-cgdata -m --skip-trim %t/merge-both-hashtree-funcmap.o -o %t/merge-both-hashtree-funcmap.cgdata
 RUN: llvm-cgdata -s %t/merge-both-hashtree-funcmap.cgdata | FileCheck %s
+RUN: llvm-cgdata -m --skip-trim %t/merge-both-hashtree-funcmap.o -o %t/merge-both-hashtree-funcmap-lazy.cgdata -indexed-codegen-data-lazy-loading
+RUN: llvm-cgdata -s %t/merge-both-hashtree-funcmap-lazy.cgdata -indexed-codegen-data-lazy-loading | FileCheck %s
 
 CHECK: Outlined hash tree:
 CHECK-NEXT:  Total Node Count: 3
@@ -63,4 +65,4 @@ CHECK-NEXT:  Mergeable function Count: 0
 
 ;--- merge-both-template.ll
 @.data1 = private unnamed_addr constant [72 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_outline"
-@.data2 = private unnamed_addr constant [68 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
+@.data2 = private unnamed_addr constant [84 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test
index 2936086321028..c088ffbb4e83f 100644
--- a/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test
+++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-archive.test
@@ -23,8 +23,8 @@ RUN: llvm-ar rcs %t/merge-archive.a %t/merge-1.o %t/merge-2.o
 # Merge the archive into the codegen data file.
 RUN: llvm-cgdata --merge --skip-trim %t/merge-archive.a -o %t/merge-archive.cgdata
 RUN: llvm-cgdata --show %t/merge-archive.cgdata | FileCheck %s
-
-RUN: llvm-cgdata --show %t/merge-archive.cgdata| FileCheck %s
+RUN: llvm-cgdata --merge --skip-trim %t/merge-archive.a -o %t/merge-archive-lazy.cgdata -indexed-codegen-data-lazy-loading
+RUN: llvm-cgdata --show %t/merge-archive-lazy.cgdata -indexed-codegen-data-lazy-loading | FileCheck %s
 CHECK: Stable function map:
 CHECK-NEXT:  Unique hash Count: 1
 CHECK-NEXT:  Total function Count: 2
@@ -65,7 +65,7 @@ MAP-NEXT: ...
 ...
 
 ;--- merge-1-template.ll
-@.data = private unnamed_addr constant [68 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [84 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
 
 ;--- raw-2.cgtext
 :stable_function_map
@@ -80,4 +80,4 @@ MAP-NEXT: ...
 ...
 
 ;--- merge-2-template.ll
-@.data = private unnamed_addr constant [68 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [84 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test
index d2965456a1999..90b5992973b49 100644
--- a/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test
+++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-concat.test
@@ -17,6 +17,8 @@ RUN: sed "s/<RAW_2_BYTES>/$(cat %t/raw-2-bytes.txt)/g" %t/merge-concat-template-
 RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-concat.ll -o %t/merge-concat.o
 RUN: llvm-cgdata --merge --skip-trim %t/merge-concat.o -o %t/merge-concat.cgdata
 RUN: llvm-cgdata --show %t/merge-concat.cgdata | FileCheck %s
+RUN: llvm-cgdata --merge --skip-trim %t/merge-concat.o -o %t/merge-concat-lazy.cgdata -indexed-codegen-data-lazy-loading
+RUN: llvm-cgdata --show %t/merge-concat-lazy.cgdata -indexed-codegen-data-lazy-loading | FileCheck %s
 
 CHECK: Stable function map:
 CHECK-NEXT:  Unique hash Count: 1
@@ -74,5 +76,5 @@ MAP-NEXT: ...
 ; In an linked executable (as opposed to an object file), cgdata in __llvm_merge might be concatenated.
 ; Although this is not a typical workflow, we simply support this case to parse cgdata that is concatenated.
 ; In other words, the following two trees are encoded back-to-back in a binary format.
-@.data1 = private unnamed_addr constant [68 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
-@.data2 = private unnamed_addr constant [68 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
+@.data1 = private unnamed_addr constant [84 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
+@.data2 = private unnamed_addr constant [84 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test
index 8277e3272d77e..b986aef26f1d7 100644
--- a/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test
+++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-double.test
@@ -19,8 +19,9 @@ RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o
 
 # Merge two object files into the codegen data file.
 RUN: llvm-cgdata --merge --skip-trim %t/merge-1.o %t/merge-2.o -o %t/merge.cgdata
-
 RUN: llvm-cgdata --show %t/merge.cgdata | FileCheck %s
+RUN: llvm-cgdata --merge --skip-trim %t/merge-1.o %t/merge-2.o -o %t/merge-lazy.cgdata -indexed-codegen-data-lazy-loading
+RUN: llvm-cgdata --show %t/merge-lazy.cgdata -indexed-codegen-data-lazy-loading  | FileCheck %s
 CHECK: Stable function map:
 CHECK-NEXT:  Unique hash Count: 1
 CHECK-NEXT:  Total function Count: 2
@@ -61,7 +62,7 @@ MAP-NEXT: ...
 ...
 
 ;--- merge-1-template.ll
-@.data = private unnamed_addr constant [68 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [84 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
 
 ;--- raw-2.cgtext
 :stable_function_map
@@ -76,4 +77,4 @@ MAP-NEXT: ...
 ...
 
 ;--- merge-2-template.ll
-@.data = private unnamed_addr constant [68 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [84 x i8] c"<RAW_2_BYTES>", section "__DATA,__llvm_merge"
diff --git a/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test b/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test
index 9469f1cbda331..eac852ff7e710 100644
--- a/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test
+++ b/llvm/test/tools/llvm-cgdata/merge-funcmap-single.test
@@ -15,6 +15,8 @@ RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-single.ll -o %t/merg
 # Merge an object file having cgdata (__llvm_merge)
 RUN: llvm-cgdata -m --skip-trim %t/merge-single.o -o %t/merge-single.cgdata
 RUN: llvm-cgdata -s %t/merge-single.cgdata | FileCheck %s
+RUN: llvm-cgdata -m --skip-trim %t/merge-single.o -o %t/merge-single-lazy.cgdata -indexed-codegen-data-lazy-loading 
+RUN: llvm-cgdata -s %t/merge-single-lazy.cgdata -indexed-codegen-data-lazy-loading | FileCheck %s
 CHECK: Stable function map:
 CHECK-NEXT:  Unique hash Count: 1
 CHECK-NEXT:  Total function Count: 1
@@ -33,4 +35,4 @@ CHECK-NEXT:  Mergeable function Count: 0
 ...
 
 ;--- merge-single-template.ll
-@.data = private unnamed_addr constant [68 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
+@.data = private unnamed_addr constant [84 x i8] c"<RAW_1_BYTES>", section "__DATA,__llvm_merge"
diff --git a/llvm/test/tools/llvm-exegesis/AArch64/debug-gen-asm.s b/llvm/test/tools/llvm-exegesis/AArch64/debug-gen-asm.s
new file mode 100644
index 0000000000000..7cd088fa7f20e
--- /dev/null
+++ b/llvm/test/tools/llvm-exegesis/AArch64/debug-gen-asm.s
@@ -0,0 +1,21 @@
+REQUIRES: aarch64-registered-target, asserts
+
+RUN: llvm-exegesis -mcpu=neoverse-v2 --use-dummy-perf-counters --min-instructions=100 --mode=latency --debug-only=preview-gen-assembly --opcode-name=ADDVv4i16v 2>&1 | FileCheck %s -check-prefix=PREVIEW
+
+PREVIEW: Generated assembly snippet:
+PREVIEW-NEXT: ```
+PREVIEW:      {{[04]}}: {{.*}}      movi    d{{[0-9]+}}, #0000000000000000
+PREVIEW-NEXT: {{[48]}}: {{.*}}      addv    h{{[0-9]+}}, v{{[0-9]+}}.4h
+PREVIEW:      ...      ({{[0-9]+}} more instructions)
+PREVIEW-NEXT: {{.*}}   addv    h{{[0-9]+}}, v{{[0-9]+}}.4h
+PREVIEW:      {{.*}}   ret
+PREVIEW-NEXT:```
+
+RUN: llvm-exegesis -mcpu=neoverse-v2 --use-dummy-perf-counters --min-instructions=100 --mode=latency --debug-only=print-gen-assembly  --opcode-name=ADDVv4i16v 2>&1 | FileCheck %s -check-prefix=PRINT
+PRINT: Generated assembly snippet:
+PRINT-NEXT: ```
+PRINT:      {{[04]}}: {{.*}}      movi    d{{[0-9]+}}, #0000000000000000
+PRINT-NEXT: {{[48]}}: {{.*}}      addv    h{{[0-9]+}}, v{{[0-9]+}}.4h
+PRINT-NEXT: {{.*}}   addv    h{{[0-9]+}}, v{{[0-9]+}}.4h
+PRINT:      {{.*}}   ret
+PRINT-NEXT:```
diff --git a/llvm/test/tools/llvm-ir2vec/entities.ll b/llvm/test/tools/llvm-ir2vec/entities.ll
index 737044cc62007..4ed6400d7a195 100644
--- a/llvm/test/tools/llvm-ir2vec/entities.ll
+++ b/llvm/test/tools/llvm-ir2vec/entities.ll
@@ -1,6 +1,6 @@
 ; RUN: llvm-ir2vec entities | FileCheck %s
 
-CHECK: 92
+CHECK: 93
 CHECK-NEXT: Ret     0
 CHECK-NEXT: Br      1
 CHECK-NEXT: Switch  2
@@ -48,48 +48,49 @@ CHECK-NEXT: SIToFP  43
 CHECK-NEXT: FPTrunc 44
 CHECK-NEXT: FPExt   45
 CHECK-NEXT: PtrToInt        46
-CHECK-NEXT: IntToPtr        47
-CHECK-NEXT: BitCast 48
-CHECK-NEXT: AddrSpaceCast   49
-CHECK-NEXT: CleanupPad      50
-CHECK-NEXT: CatchPad        51
-CHECK-NEXT: ICmp    52
-CHECK-NEXT: FCmp    53
-CHECK-NEXT: PHI     54
-CHECK-NEXT: Call    55
-CHECK-NEXT: Select  56
-CHECK-NEXT: UserOp1 57
-CHECK-NEXT: UserOp2 58
-CHECK-NEXT: VAArg   59
-CHECK-NEXT: ExtractElement  60
-CHECK-NEXT: InsertElement   61
-CHECK-NEXT: ShuffleVector   62
-CHECK-NEXT: ExtractValue    63
-CHECK-NEXT: InsertValue     64
-CHECK-NEXT: LandingPad      65
-CHECK-NEXT: Freeze  66
-CHECK-NEXT: FloatTy 67
+CHECK-NEXT: PtrToAddr	47
+CHECK-NEXT: IntToPtr        48
+CHECK-NEXT: BitCast 49
+CHECK-NEXT: AddrSpaceCast   50
+CHECK-NEXT: CleanupPad      51
+CHECK-NEXT: CatchPad        52
+CHECK-NEXT: ICmp    53
+CHECK-NEXT: FCmp    54
+CHECK-NEXT: PHI     55
+CHECK-NEXT: Call    56
+CHECK-NEXT: Select  57
+CHECK-NEXT: UserOp1 58
+CHECK-NEXT: UserOp2 59
+CHECK-NEXT: VAArg   60
+CHECK-NEXT: ExtractElement  61
+CHECK-NEXT: InsertElement   62
+CHECK-NEXT: ShuffleVector   63
+CHECK-NEXT: ExtractValue    64
+CHECK-NEXT: InsertValue     65
+CHECK-NEXT: LandingPad      66
+CHECK-NEXT: Freeze  67
 CHECK-NEXT: FloatTy 68
 CHECK-NEXT: FloatTy 69
 CHECK-NEXT: FloatTy 70
 CHECK-NEXT: FloatTy 71
 CHECK-NEXT: FloatTy 72
 CHECK-NEXT: FloatTy 73
-CHECK-NEXT: VoidTy  74
-CHECK-NEXT: LabelTy 75
-CHECK-NEXT: MetadataTy      76
-CHECK-NEXT: UnknownTy       77
-CHECK-NEXT: TokenTy 78
-CHECK-NEXT: IntegerTy       79
-CHECK-NEXT: FunctionTy      80
-CHECK-NEXT: PointerTy       81
-CHECK-NEXT: StructTy        82
-CHECK-NEXT: ArrayTy 83
-CHECK-NEXT: VectorTy        84
+CHECK-NEXT: FloatTy 74
+CHECK-NEXT: VoidTy  75
+CHECK-NEXT: LabelTy 76
+CHECK-NEXT: MetadataTy      77
+CHECK-NEXT: UnknownTy       78
+CHECK-NEXT: TokenTy 79
+CHECK-NEXT: IntegerTy       80
+CHECK-NEXT: FunctionTy      81
+CHECK-NEXT: PointerTy       82
+CHECK-NEXT: StructTy        83
+CHECK-NEXT: ArrayTy 84
 CHECK-NEXT: VectorTy        85
-CHECK-NEXT: PointerTy       86
-CHECK-NEXT: UnknownTy       87
-CHECK-NEXT: Function        88
-CHECK-NEXT: Pointer 89
-CHECK-NEXT: Constant        90
-CHECK-NEXT: Variable        91
+CHECK-NEXT: VectorTy        86
+CHECK-NEXT: PointerTy       87
+CHECK-NEXT: UnknownTy       88
+CHECK-NEXT: Function        89
+CHECK-NEXT: Pointer 90
+CHECK-NEXT: Constant        91
+CHECK-NEXT: Variable        92
diff --git a/llvm/test/tools/llvm-ir2vec/triplets.ll b/llvm/test/tools/llvm-ir2vec/triplets.ll
index a7fd9e4a3745a..6f64bab888f6b 100644
--- a/llvm/test/tools/llvm-ir2vec/triplets.ll
+++ b/llvm/test/tools/llvm-ir2vec/triplets.ll
@@ -25,41 +25,41 @@ entry:
 }
 
 ; TRIPLETS: MAX_RELATION=3
-; TRIPLETS-NEXT: 12      79      0
-; TRIPLETS-NEXT: 12      91      2
-; TRIPLETS-NEXT: 12      91      3
+; TRIPLETS-NEXT: 12      80      0
+; TRIPLETS-NEXT: 12      92      2
+; TRIPLETS-NEXT: 12      92      3
 ; TRIPLETS-NEXT: 12      0       1
-; TRIPLETS-NEXT: 0       74      0
-; TRIPLETS-NEXT: 0       91      2
-; TRIPLETS-NEXT: 16      79      0
-; TRIPLETS-NEXT: 16      91      2
-; TRIPLETS-NEXT: 16      91      3
+; TRIPLETS-NEXT: 0       75      0
+; TRIPLETS-NEXT: 0       92      2
+; TRIPLETS-NEXT: 16      80      0
+; TRIPLETS-NEXT: 16      92      2
+; TRIPLETS-NEXT: 16      92      3
 ; TRIPLETS-NEXT: 16      0       1
-; TRIPLETS-NEXT: 0       74      0
-; TRIPLETS-NEXT: 0       91      2
-; TRIPLETS-NEXT: 30      81      0
-; TRIPLETS-NEXT: 30      90      2
+; TRIPLETS-NEXT: 0       75      0
+; TRIPLETS-NEXT: 0       92      2
+; TRIPLETS-NEXT: 30      82      0
+; TRIPLETS-NEXT: 30      91      2
 ; TRIPLETS-NEXT: 30      30      1
-; TRIPLETS-NEXT: 30      81      0
-; TRIPLETS-NEXT: 30      90      2
+; TRIPLETS-NEXT: 30      82      0
+; TRIPLETS-NEXT: 30      91      2
 ; TRIPLETS-NEXT: 30      32      1
-; TRIPLETS-NEXT: 32      74      0
-; TRIPLETS-NEXT: 32      91      2
-; TRIPLETS-NEXT: 32      89      3
+; TRIPLETS-NEXT: 32      75      0
+; TRIPLETS-NEXT: 32      92      2
+; TRIPLETS-NEXT: 32      90      3
 ; TRIPLETS-NEXT: 32      32      1
-; TRIPLETS-NEXT: 32      74      0
-; TRIPLETS-NEXT: 32      91      2
-; TRIPLETS-NEXT: 32      89      3
+; TRIPLETS-NEXT: 32      75      0
+; TRIPLETS-NEXT: 32      92      2
+; TRIPLETS-NEXT: 32      90      3
 ; TRIPLETS-NEXT: 32      31      1
-; TRIPLETS-NEXT: 31      79      0
-; TRIPLETS-NEXT: 31      89      2
+; TRIPLETS-NEXT: 31      80      0
+; TRIPLETS-NEXT: 31      90      2
 ; TRIPLETS-NEXT: 31      31      1
-; TRIPLETS-NEXT: 31      79      0
-; TRIPLETS-NEXT: 31      89      2
+; TRIPLETS-NEXT: 31      80      0
+; TRIPLETS-NEXT: 31      90      2
 ; TRIPLETS-NEXT: 31      12      1
-; TRIPLETS-NEXT: 12      79      0
-; TRIPLETS-NEXT: 12      91      2
-; TRIPLETS-NEXT: 12      91      3
+; TRIPLETS-NEXT: 12      80      0
+; TRIPLETS-NEXT: 12      92      2
+; TRIPLETS-NEXT: 12      92      3
 ; TRIPLETS-NEXT: 12      0       1
-; TRIPLETS-NEXT: 0       74      0
-; TRIPLETS-NEXT: 0       91      2
+; TRIPLETS-NEXT: 0       75      0
+; TRIPLETS-NEXT: 0       92      2
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s
index d8051e7ecb4fe..cd4135af1c5c4 100644
--- a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s
@@ -3966,18 +3966,18 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      4     0.50                        fabs	z31.d, p7/m, z31.d
 # CHECK-NEXT:  1      4     0.50                        fabs	z31.h, p7/m, z31.h
 # CHECK-NEXT:  1      4     0.50                        fabs	z31.s, p7/m, z31.s
-# CHECK-NEXT:  1      4     0.50                        facge	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  1      4     0.50                        facge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  1      4     0.50                        facge	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  1      4     0.50                        facge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  1      4     0.50                        facge	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  1      4     0.50                        facge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  1      4     0.50                        facgt	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  1      4     0.50                        facgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  1      4     0.50                        facgt	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  1      4     0.50                        facgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  1      4     0.50                        facgt	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  1      4     0.50                        facgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        facge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     1.00                        facge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        facge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        facge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      4     1.00                        facge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     1.00                        facge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        facgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     1.00                        facgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        facgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        facgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      4     1.00                        facgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     1.00                        facgt	p0.s, p0/z, z1.s, z0.s
 # CHECK-NEXT:  1      4     0.50                        fadd	z0.d, p0/m, z0.d, #0.5
 # CHECK-NEXT:  1      4     0.50                        fadd	z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  1      4     0.50                        fadd	z0.d, z1.d, z31.d
@@ -4005,30 +4005,30 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      4     0.50                        fcadd	z31.d, p7/m, z31.d, z31.d, #270
 # CHECK-NEXT:  1      4     0.50                        fcadd	z31.h, p7/m, z31.h, z31.h, #270
 # CHECK-NEXT:  1      4     0.50                        fcadd	z31.s, p7/m, z31.s, z31.s, #270
-# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  1      4     0.50                        fcmge	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmge	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  1      4     0.50                        fcmge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  1      4     0.50                        fcmge	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmge	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  1      4     0.50                        fcmge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  1      4     0.50                        fcmge	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmge	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  1      4     0.50                        fcmge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        fcmeq	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmeq	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     1.00                        fcmeq	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmeq	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        fcmeq	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmeq	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     1.00                        fcmge	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     1.00                        fcmge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        fcmge	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        fcmge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcmge	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     1.00                        fcmge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        fcmgt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     1.00                        fcmgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        fcmgt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        fcmgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcmgt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     1.00                        fcmgt	p0.s, p0/z, z1.s, z0.s
 # CHECK-NEXT:  1      4     0.50                        fcmla	z0.d, p0/m, z0.d, z0.d, #0
 # CHECK-NEXT:  1      4     0.50                        fcmla	z0.d, p0/m, z1.d, z2.d, #90
 # CHECK-NEXT:  1      4     0.50                        fcmla	z0.h, p0/m, z0.h, z0.h, #0
@@ -4045,21 +4045,21 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  1      4     0.50                        fcmla	z31.h, p7/m, z31.h, z31.h, #270
 # CHECK-NEXT:  1      4     0.50                        fcmla	z31.h, z31.h, z7.h[3], #270
 # CHECK-NEXT:  1      4     0.50                        fcmla	z31.s, p7/m, z31.s, z31.s, #270
-# CHECK-NEXT:  1      4     0.50                        fcmle	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmle	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmle	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmlt	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmlt	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmlt	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmne	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmne	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  1      4     0.50                        fcmne	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmne	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  1      4     0.50                        fcmne	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  1      4     0.50                        fcmne	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  1      4     0.50                        fcmuo	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  1      4     0.50                        fcmuo	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  1      4     0.50                        fcmuo	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     1.00                        fcmle	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmle	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmle	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmlt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmlt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmlt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmne	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmne	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     1.00                        fcmne	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmne	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        fcmne	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmne	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     1.00                        fcmuo	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     1.00                        fcmuo	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        fcmuo	p0.s, p0/z, z0.s, z1.s
 # CHECK-NEXT:  1      4     0.50                        fcvt	z0.d, p0/m, z0.h
 # CHECK-NEXT:  1      4     0.50                        fcvt	z0.d, p0/m, z0.s
 # CHECK-NEXT:  1      4     0.50                        fcvt	z0.h, p0/m, z0.d
@@ -6848,7 +6848,7 @@ zip2	z31.s, z31.s, z31.s
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]
-# CHECK-NEXT: 79.00  75.00  75.00  9.00    -     209.00 3667.00  -     -     1290.00 924.00 199.50 199.50 670.00
+# CHECK-NEXT: 79.00  75.00  75.00  9.00    -     209.00 3667.00  -     -     1315.50 949.50 199.50 199.50 670.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]   Instructions:
@@ -7398,18 +7398,18 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	z31.d, p7/m, z31.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	z31.h, p7/m, z31.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	z31.s, p7/m, z31.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     facgt	p0.s, p0/z, z1.s, z0.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.d, p0/m, z0.d, #0.5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.d, p7/m, z0.d, z31.d
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.d, z1.d, z31.d
@@ -7437,30 +7437,30 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcadd	z31.d, p7/m, z31.d, z31.d, #270
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcadd	z31.h, p7/m, z31.h, z31.h, #270
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcadd	z31.s, p7/m, z31.s, z31.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.s, p0/z, z1.s, z0.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.d, p0/z, z1.d, z0.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.h, p0/z, z1.h, z0.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmeq	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmeq	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmeq	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmgt	p0.s, p0/z, z1.s, z0.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.d, p0/m, z0.d, z0.d, #0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.d, p0/m, z1.d, z2.d, #90
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.h, p0/m, z0.h, z0.h, #0
@@ -7477,21 +7477,21 @@ zip2	z31.s, z31.s, z31.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z31.h, p7/m, z31.h, z31.h, #270
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z31.h, z31.h, z7.h[3], #270
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z31.s, p7/m, z31.s, z31.s, #270
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmle	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmle	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmle	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmlt	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmlt	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmlt	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.d, p0/z, z0.d, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.h, p0/z, z0.h, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.s, p0/z, z0.s, #0.0
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.s, p0/z, z0.s, z1.s
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmuo	p0.d, p0/z, z0.d, z1.d
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmuo	p0.h, p0/z, z0.h, z1.h
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmuo	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmle	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmle	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmle	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmlt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmlt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmlt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmne	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmne	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmne	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmuo	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmuo	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     fcmuo	p0.s, p0/z, z0.s, z1.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	z0.d, p0/m, z0.h
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	z0.d, p0/m, z0.s
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	z0.h, p0/m, z0.d
diff --git a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-arithmetic.s b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-arithmetic.s
index 5cf5ed575a3e2..234a3e2b6689c 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-arithmetic.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-arithmetic.s
@@ -3002,357 +3002,357 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
 # CHECK-NEXT:  1      8     4.00                         8     SMX60_VIEU[4]                              VWSUB_VX                   vwsub.vx	v8, v16, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VV                  vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADDU_VV                  vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADDU_VX                  vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADDU_VX                  vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VV                   vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADD_VV                   vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VAADD_VX                   vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VAADD_VX                   vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VV                  vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUBU_VV                  vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUBU_VX                  vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUBU_VX                  vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VV                   vasub.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUB_VV                   vasub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VASUB_VX                   vasub.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VASUB_VX                   vasub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VMADC_VI                   vmadc.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
@@ -3882,445 +3882,445 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
 # CHECK-NEXT:  1      16    4.00                         16    SMX60_VIEU[4]                              VRSUB_VX                   vrsub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VI                  vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VI                  vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VV                  vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VV                  vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADDU_VX                  vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADDU_VX                  vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VI                   vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VI                   vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VV                   vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VV                   vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSADD_VX                   vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSADD_VX                   vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VV                  vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUBU_VV                  vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUBU_VX                  vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUBU_VX                  vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VV                   vssub.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUB_VV                   vssub.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      5     4.00                         5     SMX60_VIEU[4]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSUB_VX                   vssub.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSUB_VX                   vssub.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VWADDU_WV                  vwaddu.wv	v8, v16, v24
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
@@ -4574,7 +4574,7 @@ vwsub.wx v8, v16, x30
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3.0]  [3.1]  [4]    [5]    [6]
-# CHECK-NEXT:  -     1120.00  -     -      -      -     3292.00  -
+# CHECK-NEXT:  -     1120.00  -     -      -      -     4084.00  -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3.0]  [3.1]  [4]    [5]    [6]    Instructions:
@@ -5267,11 +5267,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -5279,29 +5279,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -5311,11 +5311,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -5323,29 +5323,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -5355,11 +5355,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -5367,29 +5367,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -5399,11 +5399,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -5411,29 +5411,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vaadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vaadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -5443,11 +5443,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -5455,29 +5455,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -5487,11 +5487,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -5499,29 +5499,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -5531,11 +5531,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -5543,29 +5543,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -5575,11 +5575,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -5587,29 +5587,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vasub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vasub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vmadc.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6147,11 +6147,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6159,29 +6159,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6191,11 +6191,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6203,29 +6203,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6235,11 +6235,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6247,29 +6247,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsaddu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsaddu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6279,11 +6279,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6291,29 +6291,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6323,11 +6323,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6335,29 +6335,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6367,11 +6367,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6379,29 +6379,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsadd.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsadd.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6411,11 +6411,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6423,29 +6423,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssubu.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6455,11 +6455,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6467,29 +6467,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssubu.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssubu.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6499,11 +6499,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6511,29 +6511,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssub.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -6543,11 +6543,11 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
@@ -6555,29 +6555,29 @@ vwsub.wx v8, v16, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssub.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssub.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vwaddu.wv	v8, v16, v24
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
diff --git a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-bitwise.s b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-bitwise.s
index 89d38728be107..5a5f366aa79f3 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-bitwise.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-bitwise.s
@@ -2630,269 +2630,269 @@ vssrl.vx v8, v8, x30
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
 # CHECK-NEXT:  1      16    4.00                         16    SMX60_VIEU[4]                              VSRL_VX                    vsrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VI                   vssra.vi	v8, v8, 12
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VI                   vssra.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VV                   vssra.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VV                   vssra.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRA_VX                   vssra.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRA_VX                   vssra.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VI                   vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VI                   vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VV                   vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VV                   vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     1.00                         4     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     2.00                         4     SMX60_VIEU[2]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      4     4.00                         4     SMX60_VIEU[4]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSSRL_VX                   vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSSRL_VX                   vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSSRL_VX                   vssrl.vx	v8, v8, t5
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SMX60_FP
@@ -2906,7 +2906,7 @@ vssrl.vx v8, v8, x30
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3.0]  [3.1]  [4]    [5]    [6]
-# CHECK-NEXT:  -     708.00  -      -      -      -     2436.00  -
+# CHECK-NEXT:  -     708.00  -      -      -      -     3060.00  -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3.0]  [3.1]  [4]    [5]    [6]    Instructions:
@@ -4069,43 +4069,43 @@ vssrl.vx v8, v8, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -4113,43 +4113,43 @@ vssrl.vx v8, v8, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -4157,43 +4157,43 @@ vssrl.vx v8, v8, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssra.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssra.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -4201,43 +4201,43 @@ vssrl.vx v8, v8, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vi	v8, v8, 12
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vi	v8, v8, 12
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -4245,43 +4245,43 @@ vssrl.vx v8, v8, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -4289,40 +4289,40 @@ vssrl.vx v8, v8, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vssrl.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vssrl.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vssrl.vx	v8, v8, t5
diff --git a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-mul-div.s b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-mul-div.s
index 572ebf22692b3..a166f15a99b8d 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-mul-div.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/rvv-mul-div.s
@@ -1906,93 +1906,93 @@ vsmul.vx v8, v8, x30
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
 # CHECK-NEXT:  1      8     4.00                         8     SMX60_VIEU[4]                              VWMULSU_VX                 vwmulsu.vx	v8, v16, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     4.00                         7     SMX60_VIEU[4]                              VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      32    32.00                        32    SMX60_VIEU[32]                             VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     4.00                         7     SMX60_VIEU[4]                              VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      32    32.00                        32    SMX60_VIEU[32]                             VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     4.00                         7     SMX60_VIEU[4]                              VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      32    32.00                        32    SMX60_VIEU[32]                             VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      7     4.00                         7     SMX60_VIEU[4]                              VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VV                   vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  1      32    32.00                        32    SMX60_VIEU[32]                             VSMUL_VV                   vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, mf8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     4.00                         7     SMX60_VIEU[4]                              VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      32    32.00                        32    SMX60_VIEU[32]                             VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, mf4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     4.00                         7     SMX60_VIEU[4]                              VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      32    32.00                        32    SMX60_VIEU[32]                             VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, mf2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     1.00                         7     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     4.00                         7     SMX60_VIEU[4]                              VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      32    32.00                        32    SMX60_VIEU[32]                             VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      7     4.00                         7     SMX60_VIEU[4]                              VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      8     8.00                         8     SMX60_VIEU[8]                              VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      16    16.00                        16    SMX60_VIEU[16]                             VSMUL_VX                   vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  1      1     1.00                  U      1     SMX60_IEU,SMX60_IEUA                       VSETVLI                    vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  1      1     1.00                         1     SMX60_VIEU                                 VSMUL_VX                   vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  1      32    32.00                        32    SMX60_VIEU[32]                             VSMUL_VX                   vsmul.vx	v8, v8, t5
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SMX60_FP
@@ -2006,7 +2006,7 @@ vsmul.vx v8, v8, x30
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3.0]  [3.1]  [4]    [5]    [6]
-# CHECK-NEXT:  -     486.00  -      -      -      -     3748.00  -
+# CHECK-NEXT:  -     486.00  -      -      -      -     4196.00  -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3.0]  [3.1]  [4]    [5]    [6]    Instructions:
@@ -2901,43 +2901,43 @@ vsmul.vx v8, v8, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     32.00   -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     32.00   -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     32.00   -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vv	v8, v8, v8
+# CHECK-NEXT:  -      -      -      -      -      -     32.00   -     vsmul.vv	v8, v8, v8
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf4, tu, mu
@@ -2945,40 +2945,40 @@ vsmul.vx v8, v8, x30
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, mf8, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e8, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     32.00   -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, mf4, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e16, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     32.00   -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, mf2, tu, mu
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e32, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     32.00   -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m1, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m2, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m4, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     16.00   -     vsmul.vx	v8, v8, t5
 # CHECK-NEXT:  -     1.00    -      -      -      -      -      -     vsetvli	t3, zero, e64, m8, tu, mu
-# CHECK-NEXT:  -      -      -      -      -      -     1.00    -     vsmul.vx	v8, v8, t5
+# CHECK-NEXT:  -      -      -      -      -      -     32.00   -     vsmul.vx	v8, v8, t5
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/zero-idioms.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/zero-idioms.s
index 0f19ef26806a9..2dce795839090 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/zero-idioms.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/zero-idioms.s
@@ -105,12 +105,12 @@ vpxor  %ymm3, %ymm3, %ymm5
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      83
-# CHECK-NEXT: Total Cycles:      17
+# CHECK-NEXT: Total Cycles:      19
 # CHECK-NEXT: Total uOps:        83
 
 # CHECK:      Dispatch Width:    6
-# CHECK-NEXT: uOps Per Cycle:    4.88
-# CHECK-NEXT: IPC:               4.88
+# CHECK-NEXT: uOps Per Cycle:    4.37
+# CHECK-NEXT: IPC:               4.37
 # CHECK-NEXT: Block RThroughput: 13.8
 
 # CHECK:      Instruction Info:
@@ -208,7 +208,7 @@ vpxor  %ymm3, %ymm3, %ymm5
 
 # CHECK:      Register File statistics:
 # CHECK-NEXT: Total number of mappings created:    24
-# CHECK-NEXT: Max number of mappings used:         15
+# CHECK-NEXT: Max number of mappings used:         16
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SKLDivider
@@ -313,92 +313,92 @@ vpxor  %ymm3, %ymm3, %ymm5
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     vpxor	%ymm3, %ymm3, %ymm5
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456
+# CHECK-NEXT:                     012345678
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DR   .    .    ..   subl	%eax, %eax
-# CHECK-NEXT: [0,1]     DR   .    .    ..   subq	%rax, %rax
-# CHECK-NEXT: [0,2]     DR   .    .    ..   xorl	%eax, %eax
-# CHECK-NEXT: [0,3]     DR   .    .    ..   xorq	%rax, %rax
-# CHECK-NEXT: [0,4]     DeER .    .    ..   pcmpgtb	%mm2, %mm2
-# CHECK-NEXT: [0,5]     D=eER.    .    ..   pcmpgtd	%mm2, %mm2
-# CHECK-NEXT: [0,6]     .D=eER    .    ..   pcmpgtw	%mm2, %mm2
-# CHECK-NEXT: [0,7]     .D---R    .    ..   pcmpgtb	%xmm2, %xmm2
-# CHECK-NEXT: [0,8]     .D---R    .    ..   pcmpgtd	%xmm2, %xmm2
-# CHECK-NEXT: [0,9]     .D---R    .    ..   pcmpgtq	%xmm2, %xmm2
-# CHECK-NEXT: [0,10]    .D---R    .    ..   pcmpgtw	%xmm2, %xmm2
-# CHECK-NEXT: [0,11]    .D---R    .    ..   vpcmpgtb	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,12]    . D--R    .    ..   vpcmpgtd	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,13]    . D--R    .    ..   vpcmpgtq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,14]    . D--R    .    ..   vpcmpgtw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,15]    . D--R    .    ..   vpcmpgtb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,16]    . D--R    .    ..   vpcmpgtd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,17]    . D--R    .    ..   vpcmpgtq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,18]    .  D-R    .    ..   vpcmpgtw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,19]    .  D-R    .    ..   vpcmpgtb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,20]    .  D-R    .    ..   vpcmpgtd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,21]    .  D-R    .    ..   vpcmpgtq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,22]    .  D-R    .    ..   vpcmpgtw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,23]    .  D-R    .    ..   vpcmpgtb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,24]    .   DR    .    ..   vpcmpgtd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,25]    .   DR    .    ..   vpcmpgtq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,26]    .   DR    .    ..   vpcmpgtw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,27]    .   DeER  .    ..   psubb	%mm2, %mm2
-# CHECK-NEXT: [0,28]    .   D=eER .    ..   psubd	%mm2, %mm2
-# CHECK-NEXT: [0,29]    .   D==eER.    ..   psubq	%mm2, %mm2
-# CHECK-NEXT: [0,30]    .    D==eER    ..   psubw	%mm2, %mm2
-# CHECK-NEXT: [0,31]    .    D----R    ..   psubb	%xmm2, %xmm2
-# CHECK-NEXT: [0,32]    .    D----R    ..   psubd	%xmm2, %xmm2
-# CHECK-NEXT: [0,33]    .    D----R    ..   psubq	%xmm2, %xmm2
-# CHECK-NEXT: [0,34]    .    D----R    ..   psubw	%xmm2, %xmm2
-# CHECK-NEXT: [0,35]    .    D----R    ..   vpsubb	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,36]    .    .D---R    ..   vpsubd	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,37]    .    .D---R    ..   vpsubq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,38]    .    .D---R    ..   vpsubw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,39]    .    .D---R    ..   vpsubb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,40]    .    .D---R    ..   vpsubd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,41]    .    .D---R    ..   vpsubq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,42]    .    . D--R    ..   vpsubw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,43]    .    . D--R    ..   vpsubb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,44]    .    . D--R    ..   vpsubd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,45]    .    . D--R    ..   vpsubq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,46]    .    . D--R    ..   vpsubw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,47]    .    . D--R    ..   vpsubb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,48]    .    .  D-R    ..   vpsubd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,49]    .    .  D-R    ..   vpsubq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,50]    .    .  D-R    ..   vpsubw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,51]    .    .  DeER   ..   andnps	%xmm0, %xmm0
-# CHECK-NEXT: [0,52]    .    .  DeER   ..   andnpd	%xmm1, %xmm1
-# CHECK-NEXT: [0,53]    .    .  DeER   ..   vandnps	%xmm2, %xmm2, %xmm2
-# CHECK-NEXT: [0,54]    .    .   DeER  ..   vandnpd	%xmm1, %xmm1, %xmm1
-# CHECK-NEXT: [0,55]    .    .   DeER  ..   vandnps	%ymm2, %ymm2, %ymm2
-# CHECK-NEXT: [0,56]    .    .   D=eER ..   vandnpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT: [0,57]    .    .   DeE-R ..   pandn	%mm2, %mm2
-# CHECK-NEXT: [0,58]    .    .   D=eER ..   pandn	%xmm2, %xmm2
-# CHECK-NEXT: [0,59]    .    .   D=eER ..   vpandn	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,60]    .    .    D=eER..   vpandn	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,61]    .    .    D=eER..   vandnps	%xmm2, %xmm2, %xmm5
-# CHECK-NEXT: [0,62]    .    .    D=eER..   vandnpd	%xmm1, %xmm1, %xmm5
-# CHECK-NEXT: [0,63]    .    .    D==eER.   vpandn	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,64]    .    .    D==eER.   vandnps	%ymm2, %ymm2, %ymm5
-# CHECK-NEXT: [0,65]    .    .    D==eER.   vandnpd	%ymm1, %ymm1, %ymm5
-# CHECK-NEXT: [0,66]    .    .    .D==eER   vpandn	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,67]    .    .    .D----R   xorps	%xmm0, %xmm0
-# CHECK-NEXT: [0,68]    .    .    .D----R   xorpd	%xmm1, %xmm1
-# CHECK-NEXT: [0,69]    .    .    .D----R   vxorps	%xmm2, %xmm2, %xmm2
-# CHECK-NEXT: [0,70]    .    .    .D----R   vxorpd	%xmm1, %xmm1, %xmm1
-# CHECK-NEXT: [0,71]    .    .    .D----R   vxorps	%ymm2, %ymm2, %ymm2
-# CHECK-NEXT: [0,72]    .    .    . D---R   vxorpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT: [0,73]    .    .    . D=eER   pxor	%mm2, %mm2
-# CHECK-NEXT: [0,74]    .    .    . D---R   pxor	%xmm2, %xmm2
-# CHECK-NEXT: [0,75]    .    .    . D---R   vpxor	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,76]    .    .    . D---R   vpxor	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,77]    .    .    . D---R   vxorps	%xmm4, %xmm4, %xmm5
-# CHECK-NEXT: [0,78]    .    .    .  D--R   vxorpd	%xmm1, %xmm1, %xmm3
-# CHECK-NEXT: [0,79]    .    .    .  D--R   vxorps	%ymm4, %ymm4, %ymm5
-# CHECK-NEXT: [0,80]    .    .    .  D--R   vxorpd	%ymm1, %ymm1, %ymm3
-# CHECK-NEXT: [0,81]    .    .    .  D--R   vpxor	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,82]    .    .    .  D--R   vpxor	%ymm3, %ymm3, %ymm5
+# CHECK:      [0,0]     DR   .    .    .  .   subl	%eax, %eax
+# CHECK-NEXT: [0,1]     DR   .    .    .  .   subq	%rax, %rax
+# CHECK-NEXT: [0,2]     DR   .    .    .  .   xorl	%eax, %eax
+# CHECK-NEXT: [0,3]     DR   .    .    .  .   xorq	%rax, %rax
+# CHECK-NEXT: [0,4]     DeER .    .    .  .   pcmpgtb	%mm2, %mm2
+# CHECK-NEXT: [0,5]     D=eER.    .    .  .   pcmpgtd	%mm2, %mm2
+# CHECK-NEXT: [0,6]     .D=eER    .    .  .   pcmpgtw	%mm2, %mm2
+# CHECK-NEXT: [0,7]     .D---R    .    .  .   pcmpgtb	%xmm2, %xmm2
+# CHECK-NEXT: [0,8]     .D---R    .    .  .   pcmpgtd	%xmm2, %xmm2
+# CHECK-NEXT: [0,9]     .D---R    .    .  .   pcmpgtq	%xmm2, %xmm2
+# CHECK-NEXT: [0,10]    .D---R    .    .  .   pcmpgtw	%xmm2, %xmm2
+# CHECK-NEXT: [0,11]    .D---R    .    .  .   vpcmpgtb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,12]    . D--R    .    .  .   vpcmpgtd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,13]    . D--R    .    .  .   vpcmpgtq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,14]    . D---R   .    .  .   vpcmpgtw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,15]    . D---R   .    .  .   vpcmpgtb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,16]    . D---R   .    .  .   vpcmpgtd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,17]    . D---R   .    .  .   vpcmpgtq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,18]    .  D--R   .    .  .   vpcmpgtw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,19]    .  D--R   .    .  .   vpcmpgtb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,20]    .  D--R   .    .  .   vpcmpgtd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,21]    .  D--R   .    .  .   vpcmpgtq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,22]    .  D---R  .    .  .   vpcmpgtw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,23]    .  D---R  .    .  .   vpcmpgtb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,24]    .   D--R  .    .  .   vpcmpgtd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,25]    .   D--R  .    .  .   vpcmpgtq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,26]    .   D--R  .    .  .   vpcmpgtw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,27]    .   DeER  .    .  .   psubb	%mm2, %mm2
+# CHECK-NEXT: [0,28]    .   D=eER .    .  .   psubd	%mm2, %mm2
+# CHECK-NEXT: [0,29]    .   D==eER.    .  .   psubq	%mm2, %mm2
+# CHECK-NEXT: [0,30]    .    D==eER    .  .   psubw	%mm2, %mm2
+# CHECK-NEXT: [0,31]    .    D----R    .  .   psubb	%xmm2, %xmm2
+# CHECK-NEXT: [0,32]    .    D----R    .  .   psubd	%xmm2, %xmm2
+# CHECK-NEXT: [0,33]    .    D----R    .  .   psubq	%xmm2, %xmm2
+# CHECK-NEXT: [0,34]    .    D----R    .  .   psubw	%xmm2, %xmm2
+# CHECK-NEXT: [0,35]    .    D----R    .  .   vpsubb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,36]    .    .D---R    .  .   vpsubd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,37]    .    .D---R    .  .   vpsubq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,38]    .    .D----R   .  .   vpsubw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,39]    .    .D----R   .  .   vpsubb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,40]    .    .D----R   .  .   vpsubd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,41]    .    .D----R   .  .   vpsubq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,42]    .    . D---R   .  .   vpsubw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,43]    .    . D---R   .  .   vpsubb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,44]    .    . D---R   .  .   vpsubd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,45]    .    . D---R   .  .   vpsubq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,46]    .    . D----R  .  .   vpsubw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,47]    .    . D----R  .  .   vpsubb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,48]    .    .  D---R  .  .   vpsubd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,49]    .    .  D---R  .  .   vpsubq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,50]    .    .  D---R  .  .   vpsubw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,51]    .    .  DeE-R  .  .   andnps	%xmm0, %xmm0
+# CHECK-NEXT: [0,52]    .    .  DeE-R  .  .   andnpd	%xmm1, %xmm1
+# CHECK-NEXT: [0,53]    .    .  DeE-R  .  .   vandnps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: [0,54]    .    .   DeE-R .  .   vandnpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: [0,55]    .    .   DeE-R .  .   vandnps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: [0,56]    .    .   D=eER .  .   vandnpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: [0,57]    .    .   DeE-R .  .   pandn	%mm2, %mm2
+# CHECK-NEXT: [0,58]    .    .   D=eER .  .   pandn	%xmm2, %xmm2
+# CHECK-NEXT: [0,59]    .    .   D=eER .  .   vpandn	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,60]    .    .    D=eER.  .   vpandn	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,61]    .    .    D=eER.  .   vandnps	%xmm2, %xmm2, %xmm5
+# CHECK-NEXT: [0,62]    .    .    D=eER.  .   vandnpd	%xmm1, %xmm1, %xmm5
+# CHECK-NEXT: [0,63]    .    .    D==eER  .   vpandn	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,64]    .    .    D==eER  .   vandnps	%ymm2, %ymm2, %ymm5
+# CHECK-NEXT: [0,65]    .    .    D==eER  .   vandnpd	%ymm1, %ymm1, %ymm5
+# CHECK-NEXT: [0,66]    .    .    .D==eER .   vpandn	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,67]    .    .    .D----R .   xorps	%xmm0, %xmm0
+# CHECK-NEXT: [0,68]    .    .    .D----R .   xorpd	%xmm1, %xmm1
+# CHECK-NEXT: [0,69]    .    .    .D----R .   vxorps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: [0,70]    .    .    .D----R .   vxorpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: [0,71]    .    .    .D----R .   vxorps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: [0,72]    .    .    . D---R .   vxorpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: [0,73]    .    .    . D=eER .   pxor	%mm2, %mm2
+# CHECK-NEXT: [0,74]    .    .    . D----R.   pxor	%xmm2, %xmm2
+# CHECK-NEXT: [0,75]    .    .    . D----R.   vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,76]    .    .    . D----R.   vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,77]    .    .    . D----R.   vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT: [0,78]    .    .    .  D---R.   vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT: [0,79]    .    .    .  D---R.   vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT: [0,80]    .    .    .  D---R.   vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT: [0,81]    .    .    .  D---R.   vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,82]    .    .    .  D----R   vpxor	%ymm3, %ymm3, %ymm5
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -421,19 +421,19 @@ vpxor  %ymm3, %ymm3, %ymm5
 # CHECK-NEXT: 11.    1     0.0    0.0    3.0       vpcmpgtb	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT: 12.    1     0.0    0.0    2.0       vpcmpgtd	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT: 13.    1     0.0    0.0    2.0       vpcmpgtq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 14.    1     0.0    0.0    2.0       vpcmpgtw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 15.    1     0.0    0.0    2.0       vpcmpgtb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 16.    1     0.0    0.0    2.0       vpcmpgtd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 17.    1     0.0    0.0    2.0       vpcmpgtq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 18.    1     0.0    0.0    1.0       vpcmpgtw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 19.    1     0.0    0.0    1.0       vpcmpgtb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 20.    1     0.0    0.0    1.0       vpcmpgtd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 21.    1     0.0    0.0    1.0       vpcmpgtq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 22.    1     0.0    0.0    1.0       vpcmpgtw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 23.    1     0.0    0.0    1.0       vpcmpgtb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 24.    1     0.0    0.0    0.0       vpcmpgtd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 25.    1     0.0    0.0    0.0       vpcmpgtq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 26.    1     0.0    0.0    0.0       vpcmpgtw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 14.    1     0.0    0.0    3.0       vpcmpgtw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 15.    1     0.0    0.0    3.0       vpcmpgtb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 16.    1     0.0    0.0    3.0       vpcmpgtd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 17.    1     0.0    0.0    3.0       vpcmpgtq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 18.    1     0.0    0.0    2.0       vpcmpgtw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 19.    1     0.0    0.0    2.0       vpcmpgtb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 20.    1     0.0    0.0    2.0       vpcmpgtd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 21.    1     0.0    0.0    2.0       vpcmpgtq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 22.    1     0.0    0.0    3.0       vpcmpgtw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 23.    1     0.0    0.0    3.0       vpcmpgtb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 24.    1     0.0    0.0    2.0       vpcmpgtd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 25.    1     0.0    0.0    2.0       vpcmpgtq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 26.    1     0.0    0.0    2.0       vpcmpgtw	%ymm3, %ymm3, %ymm5
 # CHECK-NEXT: 27.    1     1.0    1.0    0.0       psubb	%mm2, %mm2
 # CHECK-NEXT: 28.    1     2.0    0.0    0.0       psubd	%mm2, %mm2
 # CHECK-NEXT: 29.    1     3.0    0.0    0.0       psubq	%mm2, %mm2
@@ -445,24 +445,24 @@ vpxor  %ymm3, %ymm3, %ymm5
 # CHECK-NEXT: 35.    1     0.0    0.0    4.0       vpsubb	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT: 36.    1     0.0    0.0    3.0       vpsubd	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT: 37.    1     0.0    0.0    3.0       vpsubq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 38.    1     0.0    0.0    3.0       vpsubw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 39.    1     0.0    0.0    3.0       vpsubb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 40.    1     0.0    0.0    3.0       vpsubd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 41.    1     0.0    0.0    3.0       vpsubq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 42.    1     0.0    0.0    2.0       vpsubw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 43.    1     0.0    0.0    2.0       vpsubb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 44.    1     0.0    0.0    2.0       vpsubd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 45.    1     0.0    0.0    2.0       vpsubq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 46.    1     0.0    0.0    2.0       vpsubw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 47.    1     0.0    0.0    2.0       vpsubb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 48.    1     0.0    0.0    1.0       vpsubd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 49.    1     0.0    0.0    1.0       vpsubq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 50.    1     0.0    0.0    1.0       vpsubw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 51.    1     1.0    1.0    0.0       andnps	%xmm0, %xmm0
-# CHECK-NEXT: 52.    1     1.0    1.0    0.0       andnpd	%xmm1, %xmm1
-# CHECK-NEXT: 53.    1     1.0    1.0    0.0       vandnps	%xmm2, %xmm2, %xmm2
-# CHECK-NEXT: 54.    1     1.0    0.0    0.0       vandnpd	%xmm1, %xmm1, %xmm1
-# CHECK-NEXT: 55.    1     1.0    0.0    0.0       vandnps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: 38.    1     0.0    0.0    4.0       vpsubw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 39.    1     0.0    0.0    4.0       vpsubb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 40.    1     0.0    0.0    4.0       vpsubd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 41.    1     0.0    0.0    4.0       vpsubq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 42.    1     0.0    0.0    3.0       vpsubw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 43.    1     0.0    0.0    3.0       vpsubb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 44.    1     0.0    0.0    3.0       vpsubd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 45.    1     0.0    0.0    3.0       vpsubq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 46.    1     0.0    0.0    4.0       vpsubw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 47.    1     0.0    0.0    4.0       vpsubb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 48.    1     0.0    0.0    3.0       vpsubd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 49.    1     0.0    0.0    3.0       vpsubq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 50.    1     0.0    0.0    3.0       vpsubw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 51.    1     1.0    1.0    1.0       andnps	%xmm0, %xmm0
+# CHECK-NEXT: 52.    1     1.0    1.0    1.0       andnpd	%xmm1, %xmm1
+# CHECK-NEXT: 53.    1     1.0    1.0    1.0       vandnps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: 54.    1     1.0    0.0    1.0       vandnpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: 55.    1     1.0    0.0    1.0       vandnps	%ymm2, %ymm2, %ymm2
 # CHECK-NEXT: 56.    1     2.0    0.0    0.0       vandnpd	%ymm1, %ymm1, %ymm1
 # CHECK-NEXT: 57.    1     1.0    1.0    1.0       pandn	%mm2, %mm2
 # CHECK-NEXT: 58.    1     2.0    0.0    0.0       pandn	%xmm2, %xmm2
@@ -481,13 +481,13 @@ vpxor  %ymm3, %ymm3, %ymm5
 # CHECK-NEXT: 71.    1     0.0    0.0    4.0       vxorps	%ymm2, %ymm2, %ymm2
 # CHECK-NEXT: 72.    1     0.0    0.0    3.0       vxorpd	%ymm1, %ymm1, %ymm1
 # CHECK-NEXT: 73.    1     2.0    2.0    0.0       pxor	%mm2, %mm2
-# CHECK-NEXT: 74.    1     0.0    0.0    3.0       pxor	%xmm2, %xmm2
-# CHECK-NEXT: 75.    1     0.0    0.0    3.0       vpxor	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 76.    1     0.0    0.0    3.0       vpxor	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 77.    1     0.0    0.0    3.0       vxorps	%xmm4, %xmm4, %xmm5
-# CHECK-NEXT: 78.    1     0.0    0.0    2.0       vxorpd	%xmm1, %xmm1, %xmm3
-# CHECK-NEXT: 79.    1     0.0    0.0    2.0       vxorps	%ymm4, %ymm4, %ymm5
-# CHECK-NEXT: 80.    1     0.0    0.0    2.0       vxorpd	%ymm1, %ymm1, %ymm3
-# CHECK-NEXT: 81.    1     0.0    0.0    2.0       vpxor	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 82.    1     0.0    0.0    2.0       vpxor	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT:        1     0.6    0.2    1.6       <total>
+# CHECK-NEXT: 74.    1     0.0    0.0    4.0       pxor	%xmm2, %xmm2
+# CHECK-NEXT: 75.    1     0.0    0.0    4.0       vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 76.    1     0.0    0.0    4.0       vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 77.    1     0.0    0.0    4.0       vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT: 78.    1     0.0    0.0    3.0       vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT: 79.    1     0.0    0.0    3.0       vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT: 80.    1     0.0    0.0    3.0       vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT: 81.    1     0.0    0.0    3.0       vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 82.    1     0.0    0.0    4.0       vpxor	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT:        1     0.6    0.2    2.2       <total>
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeServer/zero-idioms.s b/llvm/test/tools/llvm-mca/X86/SkylakeServer/zero-idioms.s
index d3f94c27641ed..dbe3827aa9384 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeServer/zero-idioms.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeServer/zero-idioms.s
@@ -167,12 +167,12 @@ vpxorq  %zmm19, %zmm19, %zmm21
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      139
-# CHECK-NEXT: Total Cycles:      27
+# CHECK-NEXT: Total Cycles:      31
 # CHECK-NEXT: Total uOps:        139
 
 # CHECK:      Dispatch Width:    6
-# CHECK-NEXT: uOps Per Cycle:    5.15
-# CHECK-NEXT: IPC:               5.15
+# CHECK-NEXT: uOps Per Cycle:    4.48
+# CHECK-NEXT: IPC:               4.48
 # CHECK-NEXT: Block RThroughput: 23.2
 
 # CHECK:      Instruction Info:
@@ -487,148 +487,148 @@ vpxorq  %zmm19, %zmm19, %zmm21
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     vpxorq	%zmm19, %zmm19, %zmm21
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0123456789
-# CHECK-NEXT: Index     0123456789          0123456
-
-# CHECK:      [0,0]     DR   .    .    .    .    ..   subl	%eax, %eax
-# CHECK-NEXT: [0,1]     DR   .    .    .    .    ..   subq	%rax, %rax
-# CHECK-NEXT: [0,2]     DR   .    .    .    .    ..   xorl	%eax, %eax
-# CHECK-NEXT: [0,3]     DR   .    .    .    .    ..   xorq	%rax, %rax
-# CHECK-NEXT: [0,4]     DeER .    .    .    .    ..   pcmpgtb	%mm2, %mm2
-# CHECK-NEXT: [0,5]     D=eER.    .    .    .    ..   pcmpgtd	%mm2, %mm2
-# CHECK-NEXT: [0,6]     .D=eER    .    .    .    ..   pcmpgtw	%mm2, %mm2
-# CHECK-NEXT: [0,7]     .D---R    .    .    .    ..   pcmpgtb	%xmm2, %xmm2
-# CHECK-NEXT: [0,8]     .D---R    .    .    .    ..   pcmpgtd	%xmm2, %xmm2
-# CHECK-NEXT: [0,9]     .D---R    .    .    .    ..   pcmpgtq	%xmm2, %xmm2
-# CHECK-NEXT: [0,10]    .D---R    .    .    .    ..   pcmpgtw	%xmm2, %xmm2
-# CHECK-NEXT: [0,11]    .D---R    .    .    .    ..   vpcmpgtb	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,12]    . D--R    .    .    .    ..   vpcmpgtd	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,13]    . D--R    .    .    .    ..   vpcmpgtq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,14]    . D--R    .    .    .    ..   vpcmpgtw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,15]    . D--R    .    .    .    ..   vpcmpgtb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,16]    . D--R    .    .    .    ..   vpcmpgtd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,17]    . D--R    .    .    .    ..   vpcmpgtq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,18]    .  D-R    .    .    .    ..   vpcmpgtw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,19]    .  D-R    .    .    .    ..   vpcmpgtb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,20]    .  D-R    .    .    .    ..   vpcmpgtd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,21]    .  D-R    .    .    .    ..   vpcmpgtq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,22]    .  D-R    .    .    .    ..   vpcmpgtw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,23]    .  D-R    .    .    .    ..   vpcmpgtb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,24]    .   DR    .    .    .    ..   vpcmpgtd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,25]    .   DR    .    .    .    ..   vpcmpgtq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,26]    .   DR    .    .    .    ..   vpcmpgtw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,27]    .   DeER  .    .    .    ..   psubb	%mm2, %mm2
-# CHECK-NEXT: [0,28]    .   D=eER .    .    .    ..   psubd	%mm2, %mm2
-# CHECK-NEXT: [0,29]    .   D==eER.    .    .    ..   psubq	%mm2, %mm2
-# CHECK-NEXT: [0,30]    .    D==eER    .    .    ..   psubw	%mm2, %mm2
-# CHECK-NEXT: [0,31]    .    D----R    .    .    ..   psubb	%xmm2, %xmm2
-# CHECK-NEXT: [0,32]    .    D----R    .    .    ..   psubd	%xmm2, %xmm2
-# CHECK-NEXT: [0,33]    .    D----R    .    .    ..   psubq	%xmm2, %xmm2
-# CHECK-NEXT: [0,34]    .    D----R    .    .    ..   psubw	%xmm2, %xmm2
-# CHECK-NEXT: [0,35]    .    D----R    .    .    ..   vpsubb	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,36]    .    .D---R    .    .    ..   vpsubd	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,37]    .    .D---R    .    .    ..   vpsubq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,38]    .    .D---R    .    .    ..   vpsubw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,39]    .    .D---R    .    .    ..   vpsubb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,40]    .    .D---R    .    .    ..   vpsubd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,41]    .    .D---R    .    .    ..   vpsubq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,42]    .    . D--R    .    .    ..   vpsubw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,43]    .    . D--R    .    .    ..   vpsubb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,44]    .    . D--R    .    .    ..   vpsubd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,45]    .    . D--R    .    .    ..   vpsubq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,46]    .    . D--R    .    .    ..   vpsubw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,47]    .    . D--R    .    .    ..   vpsubb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,48]    .    .  D-R    .    .    ..   vpsubd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,49]    .    .  D-R    .    .    ..   vpsubq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,50]    .    .  D-R    .    .    ..   vpsubw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,51]    .    .  D-R    .    .    ..   vpsubb	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,52]    .    .  D-R    .    .    ..   vpsubd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,53]    .    .  D-R    .    .    ..   vpsubq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,54]    .    .   DR    .    .    ..   vpsubw	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,55]    .    .   DR    .    .    ..   vpsubb	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,56]    .    .   DR    .    .    ..   vpsubd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,57]    .    .   DR    .    .    ..   vpsubq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,58]    .    .   DR    .    .    ..   vpsubw	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,59]    .    .   DR    .    .    ..   vpsubb	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,60]    .    .    DR   .    .    ..   vpsubd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,61]    .    .    DR   .    .    ..   vpsubq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,62]    .    .    DR   .    .    ..   vpsubw	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,63]    .    .    DR   .    .    ..   vpsubb	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,64]    .    .    DR   .    .    ..   vpsubd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,65]    .    .    DR   .    .    ..   vpsubq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,66]    .    .    .DR  .    .    ..   vpsubw	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,67]    .    .    .DR  .    .    ..   vpsubb	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,68]    .    .    .DR  .    .    ..   vpsubd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,69]    .    .    .DR  .    .    ..   vpsubq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,70]    .    .    .DR  .    .    ..   vpsubw	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,71]    .    .    .DR  .    .    ..   vpsubb	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,72]    .    .    . DR .    .    ..   vpsubd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,73]    .    .    . DR .    .    ..   vpsubq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,74]    .    .    . DR .    .    ..   vpsubw	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,75]    .    .    . DeER    .    ..   andnps	%xmm0, %xmm0
-# CHECK-NEXT: [0,76]    .    .    . DeER    .    ..   andnpd	%xmm1, %xmm1
-# CHECK-NEXT: [0,77]    .    .    . DeER    .    ..   vandnps	%xmm2, %xmm2, %xmm2
-# CHECK-NEXT: [0,78]    .    .    .  DeER   .    ..   vandnpd	%xmm1, %xmm1, %xmm1
-# CHECK-NEXT: [0,79]    .    .    .  DeER   .    ..   vandnps	%ymm2, %ymm2, %ymm2
-# CHECK-NEXT: [0,80]    .    .    .  D=eER  .    ..   vandnpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT: [0,81]    .    .    .  D=eER  .    ..   vandnps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT: [0,82]    .    .    .  D==eER .    ..   vandnpd	%zmm1, %zmm1, %zmm1
-# CHECK-NEXT: [0,83]    .    .    .  DeE--R .    ..   pandn	%mm2, %mm2
-# CHECK-NEXT: [0,84]    .    .    .   D=eER .    ..   pandn	%xmm2, %xmm2
-# CHECK-NEXT: [0,85]    .    .    .   DeE-R .    ..   vpandn	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,86]    .    .    .   D=eER .    ..   vpandn	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,87]    .    .    .   D==eER.    ..   vpandnd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,88]    .    .    .   D===eER    ..   vpandnq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,89]    .    .    .   D====eER   ..   vpandnd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,90]    .    .    .    D====eER  ..   vpandnq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,91]    .    .    .    D=====eER ..   vpandnd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,92]    .    .    .    D======eER..   vpandnq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,93]    .    .    .    D=eE-----R..   vandnps	%xmm2, %xmm2, %xmm5
-# CHECK-NEXT: [0,94]    .    .    .    D=eE-----R..   vandnpd	%xmm1, %xmm1, %xmm5
-# CHECK-NEXT: [0,95]    .    .    .    D==eE----R..   vpandn	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,96]    .    .    .    .D=eE----R..   vandnps	%ymm2, %ymm2, %ymm5
-# CHECK-NEXT: [0,97]    .    .    .    .D==eE---R..   vandnpd	%ymm1, %ymm1, %ymm5
-# CHECK-NEXT: [0,98]    .    .    .    .D==eE---R..   vpandn	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,99]    .    .    .    .D===eE--R..   vandnps	%zmm2, %zmm2, %zmm5
-# CHECK-NEXT: [0,100]   .    .    .    .D===eE--R..   vandnpd	%zmm1, %zmm1, %zmm5
-# CHECK-NEXT: [0,101]   .    .    .    .D======eER.   vpandnd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,102]   .    .    .    . D=====eER.   vpandnq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,103]   .    .    .    . D=====eER.   vpandnd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,104]   .    .    .    . D======eER   vpandnq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,105]   .    .    .    . D======eER   vpandnd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,106]   .    .    .    . D======eER   vpandnq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,107]   .    .    .    . D--------R   xorps	%xmm0, %xmm0
-# CHECK-NEXT: [0,108]   .    .    .    .  D-------R   xorpd	%xmm1, %xmm1
-# CHECK-NEXT: [0,109]   .    .    .    .  D-------R   vxorps	%xmm2, %xmm2, %xmm2
-# CHECK-NEXT: [0,110]   .    .    .    .  D-------R   vxorpd	%xmm1, %xmm1, %xmm1
-# CHECK-NEXT: [0,111]   .    .    .    .  D-------R   vxorps	%ymm2, %ymm2, %ymm2
-# CHECK-NEXT: [0,112]   .    .    .    .  D-------R   vxorpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT: [0,113]   .    .    .    .  D-------R   vxorps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT: [0,114]   .    .    .    .   D------R   vxorpd	%zmm1, %zmm1, %zmm1
-# CHECK-NEXT: [0,115]   .    .    .    .   D=eE---R   pxor	%mm2, %mm2
-# CHECK-NEXT: [0,116]   .    .    .    .   D------R   pxor	%xmm2, %xmm2
-# CHECK-NEXT: [0,117]   .    .    .    .   D------R   vpxor	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: [0,118]   .    .    .    .   D------R   vpxor	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: [0,119]   .    .    .    .   D------R   vpxord	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,120]   .    .    .    .    D-----R   vpxorq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: [0,121]   .    .    .    .    D-----R   vpxord	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,122]   .    .    .    .    D-----R   vpxorq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: [0,123]   .    .    .    .    D-----R   vpxord	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,124]   .    .    .    .    D-----R   vpxorq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: [0,125]   .    .    .    .    D-----R   vxorps	%xmm4, %xmm4, %xmm5
-# CHECK-NEXT: [0,126]   .    .    .    .    .D----R   vxorpd	%xmm1, %xmm1, %xmm3
-# CHECK-NEXT: [0,127]   .    .    .    .    .D----R   vxorps	%ymm4, %ymm4, %ymm5
-# CHECK-NEXT: [0,128]   .    .    .    .    .D----R   vxorpd	%ymm1, %ymm1, %ymm3
-# CHECK-NEXT: [0,129]   .    .    .    .    .D----R   vxorps	%zmm4, %zmm4, %zmm5
-# CHECK-NEXT: [0,130]   .    .    .    .    .D----R   vxorpd	%zmm1, %zmm1, %zmm3
-# CHECK-NEXT: [0,131]   .    .    .    .    .D----R   vpxor	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: [0,132]   .    .    .    .    . D---R   vpxor	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: [0,133]   .    .    .    .    . D---R   vpxord	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,134]   .    .    .    .    . D---R   vpxorq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: [0,135]   .    .    .    .    . D---R   vpxord	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,136]   .    .    .    .    . D---R   vpxorq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: [0,137]   .    .    .    .    . D---R   vpxord	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: [0,138]   .    .    .    .    .  D--R   vpxorq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:                     0123456789          0
+# CHECK-NEXT: Index     0123456789          0123456789
+
+# CHECK:      [0,0]     DR   .    .    .    .    .    .   subl	%eax, %eax
+# CHECK-NEXT: [0,1]     DR   .    .    .    .    .    .   subq	%rax, %rax
+# CHECK-NEXT: [0,2]     DR   .    .    .    .    .    .   xorl	%eax, %eax
+# CHECK-NEXT: [0,3]     DR   .    .    .    .    .    .   xorq	%rax, %rax
+# CHECK-NEXT: [0,4]     DeER .    .    .    .    .    .   pcmpgtb	%mm2, %mm2
+# CHECK-NEXT: [0,5]     D=eER.    .    .    .    .    .   pcmpgtd	%mm2, %mm2
+# CHECK-NEXT: [0,6]     .D=eER    .    .    .    .    .   pcmpgtw	%mm2, %mm2
+# CHECK-NEXT: [0,7]     .D---R    .    .    .    .    .   pcmpgtb	%xmm2, %xmm2
+# CHECK-NEXT: [0,8]     .D---R    .    .    .    .    .   pcmpgtd	%xmm2, %xmm2
+# CHECK-NEXT: [0,9]     .D---R    .    .    .    .    .   pcmpgtq	%xmm2, %xmm2
+# CHECK-NEXT: [0,10]    .D---R    .    .    .    .    .   pcmpgtw	%xmm2, %xmm2
+# CHECK-NEXT: [0,11]    .D---R    .    .    .    .    .   vpcmpgtb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,12]    . D--R    .    .    .    .    .   vpcmpgtd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,13]    . D--R    .    .    .    .    .   vpcmpgtq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,14]    . D---R   .    .    .    .    .   vpcmpgtw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,15]    . D---R   .    .    .    .    .   vpcmpgtb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,16]    . D---R   .    .    .    .    .   vpcmpgtd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,17]    . D---R   .    .    .    .    .   vpcmpgtq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,18]    .  D--R   .    .    .    .    .   vpcmpgtw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,19]    .  D--R   .    .    .    .    .   vpcmpgtb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,20]    .  D--R   .    .    .    .    .   vpcmpgtd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,21]    .  D--R   .    .    .    .    .   vpcmpgtq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,22]    .  D---R  .    .    .    .    .   vpcmpgtw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,23]    .  D---R  .    .    .    .    .   vpcmpgtb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,24]    .   D--R  .    .    .    .    .   vpcmpgtd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,25]    .   D--R  .    .    .    .    .   vpcmpgtq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,26]    .   D--R  .    .    .    .    .   vpcmpgtw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,27]    .   DeER  .    .    .    .    .   psubb	%mm2, %mm2
+# CHECK-NEXT: [0,28]    .   D=eER .    .    .    .    .   psubd	%mm2, %mm2
+# CHECK-NEXT: [0,29]    .   D==eER.    .    .    .    .   psubq	%mm2, %mm2
+# CHECK-NEXT: [0,30]    .    D==eER    .    .    .    .   psubw	%mm2, %mm2
+# CHECK-NEXT: [0,31]    .    D----R    .    .    .    .   psubb	%xmm2, %xmm2
+# CHECK-NEXT: [0,32]    .    D----R    .    .    .    .   psubd	%xmm2, %xmm2
+# CHECK-NEXT: [0,33]    .    D----R    .    .    .    .   psubq	%xmm2, %xmm2
+# CHECK-NEXT: [0,34]    .    D----R    .    .    .    .   psubw	%xmm2, %xmm2
+# CHECK-NEXT: [0,35]    .    D----R    .    .    .    .   vpsubb	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,36]    .    .D---R    .    .    .    .   vpsubd	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,37]    .    .D---R    .    .    .    .   vpsubq	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,38]    .    .D----R   .    .    .    .   vpsubw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,39]    .    .D----R   .    .    .    .   vpsubb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,40]    .    .D----R   .    .    .    .   vpsubd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,41]    .    .D----R   .    .    .    .   vpsubq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,42]    .    . D---R   .    .    .    .   vpsubw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,43]    .    . D---R   .    .    .    .   vpsubb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,44]    .    . D---R   .    .    .    .   vpsubd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,45]    .    . D---R   .    .    .    .   vpsubq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,46]    .    . D----R  .    .    .    .   vpsubw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,47]    .    . D----R  .    .    .    .   vpsubb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,48]    .    .  D---R  .    .    .    .   vpsubd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,49]    .    .  D---R  .    .    .    .   vpsubq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,50]    .    .  D---R  .    .    .    .   vpsubw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,51]    .    .  D---R  .    .    .    .   vpsubb	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,52]    .    .  D---R  .    .    .    .   vpsubd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,53]    .    .  D---R  .    .    .    .   vpsubq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,54]    .    .   D---R .    .    .    .   vpsubw	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,55]    .    .   D---R .    .    .    .   vpsubb	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,56]    .    .   D---R .    .    .    .   vpsubd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,57]    .    .   D---R .    .    .    .   vpsubq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,58]    .    .   D---R .    .    .    .   vpsubw	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,59]    .    .   D---R .    .    .    .   vpsubb	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,60]    .    .    D--R .    .    .    .   vpsubd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,61]    .    .    D--R .    .    .    .   vpsubq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,62]    .    .    D---R.    .    .    .   vpsubw	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,63]    .    .    D---R.    .    .    .   vpsubb	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,64]    .    .    D---R.    .    .    .   vpsubd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,65]    .    .    D---R.    .    .    .   vpsubq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,66]    .    .    .D--R.    .    .    .   vpsubw	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,67]    .    .    .D--R.    .    .    .   vpsubb	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,68]    .    .    .D--R.    .    .    .   vpsubd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,69]    .    .    .D--R.    .    .    .   vpsubq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,70]    .    .    .D---R    .    .    .   vpsubw	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,71]    .    .    .D---R    .    .    .   vpsubb	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,72]    .    .    . D--R    .    .    .   vpsubd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,73]    .    .    . D--R    .    .    .   vpsubq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,74]    .    .    . D--R    .    .    .   vpsubw	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,75]    .    .    . DeER    .    .    .   andnps	%xmm0, %xmm0
+# CHECK-NEXT: [0,76]    .    .    . DeER    .    .    .   andnpd	%xmm1, %xmm1
+# CHECK-NEXT: [0,77]    .    .    . DeER    .    .    .   vandnps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: [0,78]    .    .    .  DeER   .    .    .   vandnpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: [0,79]    .    .    .  DeER   .    .    .   vandnps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: [0,80]    .    .    .  D=eER  .    .    .   vandnpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: [0,81]    .    .    .  D=eER  .    .    .   vandnps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT: [0,82]    .    .    .  D==eER .    .    .   vandnpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT: [0,83]    .    .    .  DeE--R .    .    .   pandn	%mm2, %mm2
+# CHECK-NEXT: [0,84]    .    .    .   D=eER .    .    .   pandn	%xmm2, %xmm2
+# CHECK-NEXT: [0,85]    .    .    .   DeE-R .    .    .   vpandn	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,86]    .    .    .   D=eER .    .    .   vpandn	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,87]    .    .    .   D==eER.    .    .   vpandnd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,88]    .    .    .   D===eER    .    .   vpandnq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,89]    .    .    .   D====eER   .    .   vpandnd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,90]    .    .    .    D====eER  .    .   vpandnq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,91]    .    .    .    D=====eER .    .   vpandnd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,92]    .    .    .    D======eER.    .   vpandnq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,93]    .    .    .    D=eE-----R.    .   vandnps	%xmm2, %xmm2, %xmm5
+# CHECK-NEXT: [0,94]    .    .    .    D=eE-----R.    .   vandnpd	%xmm1, %xmm1, %xmm5
+# CHECK-NEXT: [0,95]    .    .    .    D==eE----R.    .   vpandn	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,96]    .    .    .    .D=eE----R.    .   vandnps	%ymm2, %ymm2, %ymm5
+# CHECK-NEXT: [0,97]    .    .    .    .D==eE---R.    .   vandnpd	%ymm1, %ymm1, %ymm5
+# CHECK-NEXT: [0,98]    .    .    .    .D==eE---R.    .   vpandn	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,99]    .    .    .    .D===eE--R.    .   vandnps	%zmm2, %zmm2, %zmm5
+# CHECK-NEXT: [0,100]   .    .    .    .D===eE---R    .   vandnpd	%zmm1, %zmm1, %zmm5
+# CHECK-NEXT: [0,101]   .    .    .    .D======eER    .   vpandnd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,102]   .    .    .    . D=====eER    .   vpandnq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,103]   .    .    .    . D=====eER    .   vpandnd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,104]   .    .    .    . D======eER   .   vpandnq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,105]   .    .    .    . D======eER   .   vpandnd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,106]   .    .    .    . D======eER   .   vpandnq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,107]   .    .    .    . D--------R   .   xorps	%xmm0, %xmm0
+# CHECK-NEXT: [0,108]   .    .    .    .  D-------R   .   xorpd	%xmm1, %xmm1
+# CHECK-NEXT: [0,109]   .    .    .    .  D-------R   .   vxorps	%xmm2, %xmm2, %xmm2
+# CHECK-NEXT: [0,110]   .    .    .    .  D-------R   .   vxorpd	%xmm1, %xmm1, %xmm1
+# CHECK-NEXT: [0,111]   .    .    .    .  D-------R   .   vxorps	%ymm2, %ymm2, %ymm2
+# CHECK-NEXT: [0,112]   .    .    .    .  D--------R  .   vxorpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: [0,113]   .    .    .    .  D--------R  .   vxorps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT: [0,114]   .    .    .    .   D-------R  .   vxorpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT: [0,115]   .    .    .    .   D=eE----R  .   pxor	%mm2, %mm2
+# CHECK-NEXT: [0,116]   .    .    .    .   D-------R  .   pxor	%xmm2, %xmm2
+# CHECK-NEXT: [0,117]   .    .    .    .   D-------R  .   vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: [0,118]   .    .    .    .   D-------R  .   vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: [0,119]   .    .    .    .   D-------R  .   vpxord	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,120]   .    .    .    .    D-------R .   vpxorq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: [0,121]   .    .    .    .    D-------R .   vpxord	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,122]   .    .    .    .    D-------R .   vpxorq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: [0,123]   .    .    .    .    D-------R .   vpxord	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,124]   .    .    .    .    D-------R .   vpxorq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: [0,125]   .    .    .    .    D-------R .   vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT: [0,126]   .    .    .    .    .D------R .   vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT: [0,127]   .    .    .    .    .D------R .   vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT: [0,128]   .    .    .    .    .D-------R.   vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT: [0,129]   .    .    .    .    .D-------R.   vxorps	%zmm4, %zmm4, %zmm5
+# CHECK-NEXT: [0,130]   .    .    .    .    .D-------R.   vxorpd	%zmm1, %zmm1, %zmm3
+# CHECK-NEXT: [0,131]   .    .    .    .    .D-------R.   vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: [0,132]   .    .    .    .    . D------R.   vpxor	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: [0,133]   .    .    .    .    . D------R.   vpxord	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,134]   .    .    .    .    . D------R.   vpxorq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: [0,135]   .    .    .    .    . D------R.   vpxord	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,136]   .    .    .    .    . D-------R   vpxorq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: [0,137]   .    .    .    .    . D-------R   vpxord	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: [0,138]   .    .    .    .    .  D------R   vpxorq	%zmm19, %zmm19, %zmm21
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -651,19 +651,19 @@ vpxorq  %zmm19, %zmm19, %zmm21
 # CHECK-NEXT: 11.    1     0.0    0.0    3.0       vpcmpgtb	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT: 12.    1     0.0    0.0    2.0       vpcmpgtd	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT: 13.    1     0.0    0.0    2.0       vpcmpgtq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 14.    1     0.0    0.0    2.0       vpcmpgtw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 15.    1     0.0    0.0    2.0       vpcmpgtb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 16.    1     0.0    0.0    2.0       vpcmpgtd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 17.    1     0.0    0.0    2.0       vpcmpgtq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 18.    1     0.0    0.0    1.0       vpcmpgtw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 19.    1     0.0    0.0    1.0       vpcmpgtb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 20.    1     0.0    0.0    1.0       vpcmpgtd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 21.    1     0.0    0.0    1.0       vpcmpgtq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 22.    1     0.0    0.0    1.0       vpcmpgtw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 23.    1     0.0    0.0    1.0       vpcmpgtb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 24.    1     0.0    0.0    0.0       vpcmpgtd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 25.    1     0.0    0.0    0.0       vpcmpgtq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 26.    1     0.0    0.0    0.0       vpcmpgtw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 14.    1     0.0    0.0    3.0       vpcmpgtw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 15.    1     0.0    0.0    3.0       vpcmpgtb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 16.    1     0.0    0.0    3.0       vpcmpgtd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 17.    1     0.0    0.0    3.0       vpcmpgtq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 18.    1     0.0    0.0    2.0       vpcmpgtw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 19.    1     0.0    0.0    2.0       vpcmpgtb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 20.    1     0.0    0.0    2.0       vpcmpgtd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 21.    1     0.0    0.0    2.0       vpcmpgtq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 22.    1     0.0    0.0    3.0       vpcmpgtw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 23.    1     0.0    0.0    3.0       vpcmpgtb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 24.    1     0.0    0.0    2.0       vpcmpgtd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 25.    1     0.0    0.0    2.0       vpcmpgtq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 26.    1     0.0    0.0    2.0       vpcmpgtw	%ymm3, %ymm3, %ymm5
 # CHECK-NEXT: 27.    1     1.0    1.0    0.0       psubb	%mm2, %mm2
 # CHECK-NEXT: 28.    1     2.0    0.0    0.0       psubd	%mm2, %mm2
 # CHECK-NEXT: 29.    1     3.0    0.0    0.0       psubq	%mm2, %mm2
@@ -675,43 +675,43 @@ vpxorq  %zmm19, %zmm19, %zmm21
 # CHECK-NEXT: 35.    1     0.0    0.0    4.0       vpsubb	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT: 36.    1     0.0    0.0    3.0       vpsubd	%xmm3, %xmm3, %xmm3
 # CHECK-NEXT: 37.    1     0.0    0.0    3.0       vpsubq	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 38.    1     0.0    0.0    3.0       vpsubw	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 39.    1     0.0    0.0    3.0       vpsubb	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 40.    1     0.0    0.0    3.0       vpsubd	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 41.    1     0.0    0.0    3.0       vpsubq	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 42.    1     0.0    0.0    2.0       vpsubw	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 43.    1     0.0    0.0    2.0       vpsubb	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 44.    1     0.0    0.0    2.0       vpsubd	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 45.    1     0.0    0.0    2.0       vpsubq	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 46.    1     0.0    0.0    2.0       vpsubw	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 47.    1     0.0    0.0    2.0       vpsubb	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 48.    1     0.0    0.0    1.0       vpsubd	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 49.    1     0.0    0.0    1.0       vpsubq	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 50.    1     0.0    0.0    1.0       vpsubw	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 51.    1     0.0    0.0    1.0       vpsubb	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 52.    1     0.0    0.0    1.0       vpsubd	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 53.    1     0.0    0.0    1.0       vpsubq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 54.    1     0.0    0.0    0.0       vpsubw	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 55.    1     0.0    0.0    0.0       vpsubb	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 56.    1     0.0    0.0    0.0       vpsubd	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 57.    1     0.0    0.0    0.0       vpsubq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 58.    1     0.0    0.0    0.0       vpsubw	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 59.    1     0.0    0.0    0.0       vpsubb	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 60.    1     0.0    0.0    0.0       vpsubd	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 61.    1     0.0    0.0    0.0       vpsubq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 62.    1     0.0    0.0    0.0       vpsubw	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 63.    1     0.0    0.0    0.0       vpsubb	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 64.    1     0.0    0.0    0.0       vpsubd	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 65.    1     0.0    0.0    0.0       vpsubq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 66.    1     0.0    0.0    0.0       vpsubw	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 67.    1     0.0    0.0    0.0       vpsubb	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 68.    1     0.0    0.0    0.0       vpsubd	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 69.    1     0.0    0.0    0.0       vpsubq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 70.    1     0.0    0.0    0.0       vpsubw	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 71.    1     0.0    0.0    0.0       vpsubb	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 72.    1     0.0    0.0    0.0       vpsubd	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 73.    1     0.0    0.0    0.0       vpsubq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 74.    1     0.0    0.0    0.0       vpsubw	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 38.    1     0.0    0.0    4.0       vpsubw	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 39.    1     0.0    0.0    4.0       vpsubb	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 40.    1     0.0    0.0    4.0       vpsubd	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 41.    1     0.0    0.0    4.0       vpsubq	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 42.    1     0.0    0.0    3.0       vpsubw	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 43.    1     0.0    0.0    3.0       vpsubb	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 44.    1     0.0    0.0    3.0       vpsubd	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 45.    1     0.0    0.0    3.0       vpsubq	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 46.    1     0.0    0.0    4.0       vpsubw	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 47.    1     0.0    0.0    4.0       vpsubb	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 48.    1     0.0    0.0    3.0       vpsubd	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 49.    1     0.0    0.0    3.0       vpsubq	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 50.    1     0.0    0.0    3.0       vpsubw	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 51.    1     0.0    0.0    3.0       vpsubb	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 52.    1     0.0    0.0    3.0       vpsubd	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 53.    1     0.0    0.0    3.0       vpsubq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 54.    1     0.0    0.0    3.0       vpsubw	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 55.    1     0.0    0.0    3.0       vpsubb	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 56.    1     0.0    0.0    3.0       vpsubd	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 57.    1     0.0    0.0    3.0       vpsubq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 58.    1     0.0    0.0    3.0       vpsubw	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 59.    1     0.0    0.0    3.0       vpsubb	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 60.    1     0.0    0.0    2.0       vpsubd	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 61.    1     0.0    0.0    2.0       vpsubq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 62.    1     0.0    0.0    3.0       vpsubw	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 63.    1     0.0    0.0    3.0       vpsubb	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 64.    1     0.0    0.0    3.0       vpsubd	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 65.    1     0.0    0.0    3.0       vpsubq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 66.    1     0.0    0.0    2.0       vpsubw	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 67.    1     0.0    0.0    2.0       vpsubb	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 68.    1     0.0    0.0    2.0       vpsubd	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 69.    1     0.0    0.0    2.0       vpsubq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 70.    1     0.0    0.0    3.0       vpsubw	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 71.    1     0.0    0.0    3.0       vpsubb	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 72.    1     0.0    0.0    2.0       vpsubd	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 73.    1     0.0    0.0    2.0       vpsubq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 74.    1     0.0    0.0    2.0       vpsubw	%zmm19, %zmm19, %zmm21
 # CHECK-NEXT: 75.    1     1.0    1.0    0.0       andnps	%xmm0, %xmm0
 # CHECK-NEXT: 76.    1     1.0    1.0    0.0       andnpd	%xmm1, %xmm1
 # CHECK-NEXT: 77.    1     1.0    1.0    0.0       vandnps	%xmm2, %xmm2, %xmm2
@@ -737,7 +737,7 @@ vpxorq  %zmm19, %zmm19, %zmm21
 # CHECK-NEXT: 97.    1     3.0    2.0    3.0       vandnpd	%ymm1, %ymm1, %ymm5
 # CHECK-NEXT: 98.    1     3.0    2.0    3.0       vpandn	%ymm3, %ymm3, %ymm5
 # CHECK-NEXT: 99.    1     4.0    3.0    2.0       vandnps	%zmm2, %zmm2, %zmm5
-# CHECK-NEXT: 100.   1     4.0    3.0    2.0       vandnpd	%zmm1, %zmm1, %zmm5
+# CHECK-NEXT: 100.   1     4.0    3.0    3.0       vandnpd	%zmm1, %zmm1, %zmm5
 # CHECK-NEXT: 101.   1     7.0    0.0    0.0       vpandnd	%xmm19, %xmm19, %xmm21
 # CHECK-NEXT: 102.   1     6.0    0.0    0.0       vpandnq	%xmm19, %xmm19, %xmm21
 # CHECK-NEXT: 103.   1     6.0    0.0    0.0       vpandnd	%ymm19, %ymm19, %ymm21
@@ -749,31 +749,31 @@ vpxorq  %zmm19, %zmm19, %zmm21
 # CHECK-NEXT: 109.   1     0.0    0.0    7.0       vxorps	%xmm2, %xmm2, %xmm2
 # CHECK-NEXT: 110.   1     0.0    0.0    7.0       vxorpd	%xmm1, %xmm1, %xmm1
 # CHECK-NEXT: 111.   1     0.0    0.0    7.0       vxorps	%ymm2, %ymm2, %ymm2
-# CHECK-NEXT: 112.   1     0.0    0.0    7.0       vxorpd	%ymm1, %ymm1, %ymm1
-# CHECK-NEXT: 113.   1     0.0    0.0    7.0       vxorps	%zmm2, %zmm2, %zmm2
-# CHECK-NEXT: 114.   1     0.0    0.0    6.0       vxorpd	%zmm1, %zmm1, %zmm1
-# CHECK-NEXT: 115.   1     2.0    2.0    3.0       pxor	%mm2, %mm2
-# CHECK-NEXT: 116.   1     0.0    0.0    6.0       pxor	%xmm2, %xmm2
-# CHECK-NEXT: 117.   1     0.0    0.0    6.0       vpxor	%xmm3, %xmm3, %xmm3
-# CHECK-NEXT: 118.   1     0.0    0.0    6.0       vpxor	%ymm3, %ymm3, %ymm3
-# CHECK-NEXT: 119.   1     0.0    0.0    6.0       vpxord	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 120.   1     0.0    0.0    5.0       vpxorq	%xmm19, %xmm19, %xmm19
-# CHECK-NEXT: 121.   1     0.0    0.0    5.0       vpxord	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 122.   1     0.0    0.0    5.0       vpxorq	%ymm19, %ymm19, %ymm19
-# CHECK-NEXT: 123.   1     0.0    0.0    5.0       vpxord	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 124.   1     0.0    0.0    5.0       vpxorq	%zmm19, %zmm19, %zmm19
-# CHECK-NEXT: 125.   1     0.0    0.0    5.0       vxorps	%xmm4, %xmm4, %xmm5
-# CHECK-NEXT: 126.   1     0.0    0.0    4.0       vxorpd	%xmm1, %xmm1, %xmm3
-# CHECK-NEXT: 127.   1     0.0    0.0    4.0       vxorps	%ymm4, %ymm4, %ymm5
-# CHECK-NEXT: 128.   1     0.0    0.0    4.0       vxorpd	%ymm1, %ymm1, %ymm3
-# CHECK-NEXT: 129.   1     0.0    0.0    4.0       vxorps	%zmm4, %zmm4, %zmm5
-# CHECK-NEXT: 130.   1     0.0    0.0    4.0       vxorpd	%zmm1, %zmm1, %zmm3
-# CHECK-NEXT: 131.   1     0.0    0.0    4.0       vpxor	%xmm3, %xmm3, %xmm5
-# CHECK-NEXT: 132.   1     0.0    0.0    3.0       vpxor	%ymm3, %ymm3, %ymm5
-# CHECK-NEXT: 133.   1     0.0    0.0    3.0       vpxord	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 134.   1     0.0    0.0    3.0       vpxorq	%xmm19, %xmm19, %xmm21
-# CHECK-NEXT: 135.   1     0.0    0.0    3.0       vpxord	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 136.   1     0.0    0.0    3.0       vpxorq	%ymm19, %ymm19, %ymm21
-# CHECK-NEXT: 137.   1     0.0    0.0    3.0       vpxord	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT: 138.   1     0.0    0.0    2.0       vpxorq	%zmm19, %zmm19, %zmm21
-# CHECK-NEXT:        1     0.9    0.2    2.0       <total>
+# CHECK-NEXT: 112.   1     0.0    0.0    8.0       vxorpd	%ymm1, %ymm1, %ymm1
+# CHECK-NEXT: 113.   1     0.0    0.0    8.0       vxorps	%zmm2, %zmm2, %zmm2
+# CHECK-NEXT: 114.   1     0.0    0.0    7.0       vxorpd	%zmm1, %zmm1, %zmm1
+# CHECK-NEXT: 115.   1     2.0    2.0    4.0       pxor	%mm2, %mm2
+# CHECK-NEXT: 116.   1     0.0    0.0    7.0       pxor	%xmm2, %xmm2
+# CHECK-NEXT: 117.   1     0.0    0.0    7.0       vpxor	%xmm3, %xmm3, %xmm3
+# CHECK-NEXT: 118.   1     0.0    0.0    7.0       vpxor	%ymm3, %ymm3, %ymm3
+# CHECK-NEXT: 119.   1     0.0    0.0    7.0       vpxord	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 120.   1     0.0    0.0    7.0       vpxorq	%xmm19, %xmm19, %xmm19
+# CHECK-NEXT: 121.   1     0.0    0.0    7.0       vpxord	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 122.   1     0.0    0.0    7.0       vpxorq	%ymm19, %ymm19, %ymm19
+# CHECK-NEXT: 123.   1     0.0    0.0    7.0       vpxord	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 124.   1     0.0    0.0    7.0       vpxorq	%zmm19, %zmm19, %zmm19
+# CHECK-NEXT: 125.   1     0.0    0.0    7.0       vxorps	%xmm4, %xmm4, %xmm5
+# CHECK-NEXT: 126.   1     0.0    0.0    6.0       vxorpd	%xmm1, %xmm1, %xmm3
+# CHECK-NEXT: 127.   1     0.0    0.0    6.0       vxorps	%ymm4, %ymm4, %ymm5
+# CHECK-NEXT: 128.   1     0.0    0.0    7.0       vxorpd	%ymm1, %ymm1, %ymm3
+# CHECK-NEXT: 129.   1     0.0    0.0    7.0       vxorps	%zmm4, %zmm4, %zmm5
+# CHECK-NEXT: 130.   1     0.0    0.0    7.0       vxorpd	%zmm1, %zmm1, %zmm3
+# CHECK-NEXT: 131.   1     0.0    0.0    7.0       vpxor	%xmm3, %xmm3, %xmm5
+# CHECK-NEXT: 132.   1     0.0    0.0    6.0       vpxor	%ymm3, %ymm3, %ymm5
+# CHECK-NEXT: 133.   1     0.0    0.0    6.0       vpxord	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 134.   1     0.0    0.0    6.0       vpxorq	%xmm19, %xmm19, %xmm21
+# CHECK-NEXT: 135.   1     0.0    0.0    6.0       vpxord	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 136.   1     0.0    0.0    7.0       vpxorq	%ymm19, %ymm19, %ymm21
+# CHECK-NEXT: 137.   1     0.0    0.0    7.0       vpxord	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT: 138.   1     0.0    0.0    6.0       vpxorq	%zmm19, %zmm19, %zmm21
+# CHECK-NEXT:        1     0.9    0.2    3.1       <total>
diff --git a/llvm/test/tools/llvm-mca/X86/stack-engine-pop.s b/llvm/test/tools/llvm-mca/X86/stack-engine-pop.s
new file mode 100644
index 0000000000000..2ffb52ae61fc4
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/stack-engine-pop.s
@@ -0,0 +1,92 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=skylake -timeline -iterations=2 < %s | FileCheck %s
+
+movq $0x80, %rsp
+popq %rax
+popq %rcx
+popq %rdx
+popq %rbx
+popq %r12
+
+# CHECK:      Iterations:        2
+# CHECK-NEXT: Instructions:      12
+# CHECK-NEXT: Total Cycles:      14
+# CHECK-NEXT: Total uOps:        22
+
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    1.57
+# CHECK-NEXT: IPC:               0.86
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.25                        movq	$128, %rsp
+# CHECK-NEXT:  2      6     0.50    *                   popq	%rax
+# CHECK-NEXT:  2      6     0.50    *                   popq	%rcx
+# CHECK-NEXT:  2      6     0.50    *                   popq	%rdx
+# CHECK-NEXT:  2      6     0.50    *                   popq	%rbx
+# CHECK-NEXT:  2      6     0.50    *                   popq	%r12
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SKLDivider
+# CHECK-NEXT: [1]   - SKLFPDivider
+# CHECK-NEXT: [2]   - SKLPort0
+# CHECK-NEXT: [3]   - SKLPort1
+# CHECK-NEXT: [4]   - SKLPort2
+# CHECK-NEXT: [5]   - SKLPort3
+# CHECK-NEXT: [6]   - SKLPort4
+# CHECK-NEXT: [7]   - SKLPort5
+# CHECK-NEXT: [8]   - SKLPort6
+# CHECK-NEXT: [9]   - SKLPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     1.50   1.50   2.50   2.50    -     1.50   1.50    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -     movq	$128, %rsp
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     popq	%rax
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -      -     0.50    -     popq	%rcx
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     popq	%rdx
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -      -     0.50    -     popq	%rbx
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     popq	%r12
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeER .    .  .   movq	$128, %rsp
+# CHECK-NEXT: [0,1]     D=eeeeeeER.  .   popq	%rax
+# CHECK-NEXT: [0,2]     D=eeeeeeER.  .   popq	%rcx
+# CHECK-NEXT: [0,3]     .D=eeeeeeER  .   popq	%rdx
+# CHECK-NEXT: [0,4]     .D=eeeeeeER  .   popq	%rbx
+# CHECK-NEXT: [0,5]     .D==eeeeeeER .   popq	%r12
+# CHECK-NEXT: [1,0]     . DeE------R .   movq	$128, %rsp
+# CHECK-NEXT: [1,1]     . D=eeeeeeER .   popq	%rax
+# CHECK-NEXT: [1,2]     . D==eeeeeeER.   popq	%rcx
+# CHECK-NEXT: [1,3]     .  D=eeeeeeER.   popq	%rdx
+# CHECK-NEXT: [1,4]     .  D==eeeeeeER   popq	%rbx
+# CHECK-NEXT: [1,5]     .  D==eeeeeeER   popq	%r12
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     1.0    1.0    3.0       movq	$128, %rsp
+# CHECK-NEXT: 1.     2     2.0    0.0    0.0       popq	%rax
+# CHECK-NEXT: 2.     2     2.5    0.5    0.0       popq	%rcx
+# CHECK-NEXT: 3.     2     2.0    1.0    0.0       popq	%rdx
+# CHECK-NEXT: 4.     2     2.5    1.5    0.0       popq	%rbx
+# CHECK-NEXT: 5.     2     3.0    2.0    0.0       popq	%r12
+# CHECK-NEXT:        2     2.2    1.0    0.5       <total>
diff --git a/llvm/test/tools/llvm-mca/X86/stack-engine-push.s b/llvm/test/tools/llvm-mca/X86/stack-engine-push.s
new file mode 100644
index 0000000000000..fc394d4c1e7d3
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/X86/stack-engine-push.s
@@ -0,0 +1,92 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=skylake -timeline -iterations=2 < %s | FileCheck %s
+
+movq $0x80, %rsp
+pushq %rax
+pushq %rcx
+pushq %rdx
+pushq %rbx
+pushq %r12
+
+# CHECK:      Iterations:        2
+# CHECK-NEXT: Instructions:      12
+# CHECK-NEXT: Total Cycles:      15
+# CHECK-NEXT: Total uOps:        32
+
+# CHECK:      Dispatch Width:    6
+# CHECK-NEXT: uOps Per Cycle:    2.13
+# CHECK-NEXT: IPC:               0.80
+# CHECK-NEXT: Block RThroughput: 5.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.25                        movq	$128, %rsp
+# CHECK-NEXT:  3      2     1.00           *            pushq	%rax
+# CHECK-NEXT:  3      2     1.00           *            pushq	%rcx
+# CHECK-NEXT:  3      2     1.00           *            pushq	%rdx
+# CHECK-NEXT:  3      2     1.00           *            pushq	%rbx
+# CHECK-NEXT:  3      2     1.00           *            pushq	%r12
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SKLDivider
+# CHECK-NEXT: [1]   - SKLFPDivider
+# CHECK-NEXT: [2]   - SKLPort0
+# CHECK-NEXT: [3]   - SKLPort1
+# CHECK-NEXT: [4]   - SKLPort2
+# CHECK-NEXT: [5]   - SKLPort3
+# CHECK-NEXT: [6]   - SKLPort4
+# CHECK-NEXT: [7]   - SKLPort5
+# CHECK-NEXT: [8]   - SKLPort6
+# CHECK-NEXT: [9]   - SKLPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     1.50   1.50   1.50   1.50   5.00   1.50   1.50   2.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -     movq	$128, %rsp
+# CHECK-NEXT:  -      -     0.50    -     0.50    -     1.00   0.50    -     0.50   pushq	%rax
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   1.00    -     0.50   0.50   pushq	%rcx
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50   1.00   0.50    -      -     pushq	%rdx
+# CHECK-NEXT:  -      -      -     0.50   0.50    -     1.00   0.50    -     0.50   pushq	%rbx
+# CHECK-NEXT:  -      -     0.50   0.50    -     0.50   1.00    -      -     0.50   pushq	%r12
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeER .    .   .   movq	$128, %rsp
+# CHECK-NEXT: [0,1]     D=eeER    .   .   pushq	%rax
+# CHECK-NEXT: [0,2]     .D=eeER   .   .   pushq	%rcx
+# CHECK-NEXT: [0,3]     .D==eeER  .   .   pushq	%rdx
+# CHECK-NEXT: [0,4]     . D==eeER .   .   pushq	%rbx
+# CHECK-NEXT: [0,5]     . D===eeER.   .   pushq	%r12
+# CHECK-NEXT: [1,0]     .  DeE---R.   .   movq	$128, %rsp
+# CHECK-NEXT: [1,1]     .  D===eeER   .   pushq	%rax
+# CHECK-NEXT: [1,2]     .   D===eeER  .   pushq	%rcx
+# CHECK-NEXT: [1,3]     .   D====eeER .   pushq	%rdx
+# CHECK-NEXT: [1,4]     .    D====eeER.   pushq	%rbx
+# CHECK-NEXT: [1,5]     .    D=====eeER   pushq	%r12
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     2     1.0    1.0    1.5       movq	$128, %rsp
+# CHECK-NEXT: 1.     2     3.0    0.5    0.0       pushq	%rax
+# CHECK-NEXT: 2.     2     3.0    1.0    0.0       pushq	%rcx
+# CHECK-NEXT: 3.     2     4.0    1.0    0.0       pushq	%rdx
+# CHECK-NEXT: 4.     2     4.0    1.0    0.0       pushq	%rbx
+# CHECK-NEXT: 5.     2     5.0    1.0    0.0       pushq	%r12
+# CHECK-NEXT:        2     3.3    0.9    0.3       <total>
diff --git a/llvm/test/tools/llvm-objcopy/DXContainer/copy-basic.test b/llvm/test/tools/llvm-objcopy/DXContainer/copy-basic.test
new file mode 100644
index 0000000000000..16ac30f059487
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/DXContainer/copy-basic.test
@@ -0,0 +1,294 @@
+## Tests that the copied DXContainer is identical to the original.
+
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objcopy %t %t.out
+# RUN: cmp %t %t.out
+
+## The DXContainer described below was generated with:
+
+## `clang-dxc -T cs_6_7 test.hlsl /Fo temp.dxo`
+## `obj2yaml temp.dxo`
+
+## ``` test.hlsl
+## [RootSignature("")]
+## [numthreads(1,1,1)]
+## void main() {}
+## ```
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  FileSize:        1984
+  PartCount:       7
+  PartOffsets:     [ 60, 1792, 1808, 1836, 1852, 1868, 1900 ]
+Parts:
+  - Name:            DXIL
+    Size:            1724
+    Program:
+      MajorVersion:    6
+      MinorVersion:    7
+      ShaderKind:      5
+      Size:            431
+      DXILMajorVersion: 1
+      DXILMinorVersion: 7
+      DXILSize:        1700
+      DXIL:            [ 0x42, 0x43, 0xC0, 0xDE, 0x21, 0xC, 0x0, 0x0, 0xA6,
+                         0x1, 0x0, 0x0, 0xB, 0x82, 0x20, 0x0, 0x2, 0x0,
+                         0x0, 0x0, 0x13, 0x0, 0x0, 0x0, 0x7, 0x81, 0x23,
+                         0x91, 0x41, 0xC8, 0x4, 0x49, 0x6, 0x10, 0x32,
+                         0x39, 0x92, 0x1, 0x84, 0xC, 0x25, 0x5, 0x8, 0x19,
+                         0x1E, 0x4, 0x8B, 0x62, 0x80, 0x10, 0x45, 0x2,
+                         0x42, 0x92, 0xB, 0x42, 0x84, 0x10, 0x32, 0x14,
+                         0x38, 0x8, 0x18, 0x4B, 0xA, 0x32, 0x42, 0x88,
+                         0x48, 0x90, 0x14, 0x20, 0x43, 0x46, 0x88, 0xA5,
+                         0x0, 0x19, 0x32, 0x42, 0xE4, 0x48, 0xE, 0x90,
+                         0x11, 0x22, 0xC4, 0x50, 0x41, 0x51, 0x81, 0x8C,
+                         0xE1, 0x83, 0xE5, 0x8A, 0x4, 0x21, 0x46, 0x6,
+                         0x51, 0x18, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x1B,
+                         0x90, 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0xC0,
+                         0x1, 0x24, 0x80, 0x2, 0x0, 0x0, 0x0, 0x49, 0x18,
+                         0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x13, 0x82, 0x0,
+                         0x0, 0x89, 0x20, 0x0, 0x0, 0x11, 0x0, 0x0, 0x0,
+                         0x32, 0x22, 0x8, 0x9, 0x20, 0x64, 0x85, 0x4, 0x13,
+                         0x22, 0xA4, 0x84, 0x4, 0x13, 0x22, 0xE3, 0x84,
+                         0xA1, 0x90, 0x14, 0x12, 0x4C, 0x88, 0x8C, 0xB,
+                         0x84, 0x84, 0x4C, 0x10, 0x20, 0x73, 0x4, 0x8,
+                         0xC1, 0x65, 0xC3, 0x85, 0x2C, 0xE8, 0x3, 0x40,
+                         0x14, 0x91, 0x4E, 0xD1, 0x4A, 0x48, 0x44, 0x54,
+                         0x11, 0xC3, 0x9, 0x30, 0xC4, 0x18, 0x1, 0x30,
+                         0x2, 0x50, 0x82, 0x21, 0x1A, 0x8, 0x98, 0x23,
+                         0x0, 0x3, 0x0, 0x13, 0x14, 0x72, 0xC0, 0x87, 0x74,
+                         0x60, 0x87, 0x36, 0x68, 0x87, 0x79, 0x68, 0x3,
+                         0x72, 0xC0, 0x87, 0xD, 0xAE, 0x50, 0xE, 0x6D,
+                         0xD0, 0xE, 0x7A, 0x50, 0xE, 0x6D, 0x0, 0xF, 0x7A,
+                         0x30, 0x7, 0x72, 0xA0, 0x7, 0x73, 0x20, 0x7, 0x6D,
+                         0x90, 0xE, 0x71, 0xA0, 0x7, 0x73, 0x20, 0x7, 0x6D,
+                         0x90, 0xE, 0x78, 0xA0, 0x7, 0x78, 0xD0, 0x6, 0xE9,
+                         0x10, 0x7, 0x76, 0xA0, 0x7, 0x71, 0x60, 0x7, 0x6D,
+                         0x90, 0xE, 0x73, 0x20, 0x7, 0x7A, 0x30, 0x7, 0x72,
+                         0xD0, 0x6, 0xE9, 0x60, 0x7, 0x74, 0xA0, 0x7, 0x76,
+                         0x40, 0x7, 0x6D, 0x60, 0xE, 0x71, 0x60, 0x7, 0x7A,
+                         0x10, 0x7, 0x76, 0xD0, 0x6, 0xE6, 0x30, 0x7, 0x72,
+                         0xA0, 0x7, 0x73, 0x20, 0x7, 0x6D, 0x60, 0xE, 0x76,
+                         0x40, 0x7, 0x7A, 0x60, 0x7, 0x74, 0xD0, 0x6, 0xEE,
+                         0x80, 0x7, 0x7A, 0x10, 0x7, 0x76, 0xA0, 0x7, 0x73,
+                         0x20, 0x7, 0x7A, 0x60, 0x7, 0x74, 0x30, 0xE4,
+                         0x21, 0x0, 0x0, 0x8, 0x0, 0x0, 0x0, 0x2, 0x0,
+                         0x0, 0x0, 0x20, 0xB, 0x4, 0x7, 0x0, 0x0, 0x0,
+                         0x32, 0x1E, 0x98, 0xC, 0x19, 0x11, 0x4C, 0x90,
+                         0x8C, 0x9, 0x26, 0x47, 0xC6, 0x4, 0x43, 0xBA,
+                         0x12, 0x28, 0x88, 0x62, 0x28, 0x87, 0x42, 0x28,
+                         0x2, 0x0, 0x0, 0x0, 0x79, 0x18, 0x0, 0x0, 0xE2,
+                         0x0, 0x0, 0x0, 0x33, 0x8, 0x80, 0x1C, 0xC4, 0xE1,
+                         0x1C, 0x66, 0x14, 0x1, 0x3D, 0x88, 0x43, 0x38,
+                         0x84, 0xC3, 0x8C, 0x42, 0x80, 0x7, 0x79, 0x78,
+                         0x7, 0x73, 0x98, 0x71, 0xC, 0xE6, 0x0, 0xF, 0xED,
+                         0x10, 0xE, 0xF4, 0x80, 0xE, 0x33, 0xC, 0x42, 0x1E,
+                         0xC2, 0xC1, 0x1D, 0xCE, 0xA1, 0x1C, 0x66, 0x30,
+                         0x5, 0x3D, 0x88, 0x43, 0x38, 0x84, 0x83, 0x1B,
+                         0xCC, 0x3, 0x3D, 0xC8, 0x43, 0x3D, 0x8C, 0x3,
+                         0x3D, 0xCC, 0x78, 0x8C, 0x74, 0x70, 0x7, 0x7B,
+                         0x8, 0x7, 0x79, 0x48, 0x87, 0x70, 0x70, 0x7, 0x7A,
+                         0x70, 0x3, 0x76, 0x78, 0x87, 0x70, 0x20, 0x87,
+                         0x19, 0xCC, 0x11, 0xE, 0xEC, 0x90, 0xE, 0xE1,
+                         0x30, 0xF, 0x6E, 0x30, 0xF, 0xE3, 0xF0, 0xE, 0xF0,
+                         0x50, 0xE, 0x33, 0x10, 0xC4, 0x1D, 0xDE, 0x21,
+                         0x1C, 0xD8, 0x21, 0x1D, 0xC2, 0x61, 0x1E, 0x66,
+                         0x30, 0x89, 0x3B, 0xBC, 0x83, 0x3B, 0xD0, 0x43,
+                         0x39, 0xB4, 0x3, 0x3C, 0xBC, 0x83, 0x3C, 0x84,
+                         0x3, 0x3B, 0xCC, 0xF0, 0x14, 0x76, 0x60, 0x7,
+                         0x7B, 0x68, 0x7, 0x37, 0x68, 0x87, 0x72, 0x68,
+                         0x7, 0x37, 0x80, 0x87, 0x70, 0x90, 0x87, 0x70,
+                         0x60, 0x7, 0x76, 0x28, 0x7, 0x76, 0xF8, 0x5, 0x76,
+                         0x78, 0x87, 0x77, 0x80, 0x87, 0x5F, 0x8, 0x87,
+                         0x71, 0x18, 0x87, 0x72, 0x98, 0x87, 0x79, 0x98,
+                         0x81, 0x2C, 0xEE, 0xF0, 0xE, 0xEE, 0xE0, 0xE,
+                         0xF5, 0xC0, 0xE, 0xEC, 0x30, 0x3, 0x62, 0xC8,
+                         0xA1, 0x1C, 0xE4, 0xA1, 0x1C, 0xCC, 0xA1, 0x1C,
+                         0xE4, 0xA1, 0x1C, 0xDC, 0x61, 0x1C, 0xCA, 0x21,
+                         0x1C, 0xC4, 0x81, 0x1D, 0xCA, 0x61, 0x6, 0xD6,
+                         0x90, 0x43, 0x39, 0xC8, 0x43, 0x39, 0x98, 0x43,
+                         0x39, 0xC8, 0x43, 0x39, 0xB8, 0xC3, 0x38, 0x94,
+                         0x43, 0x38, 0x88, 0x3, 0x3B, 0x94, 0xC3, 0x2F,
+                         0xBC, 0x83, 0x3C, 0xFC, 0x82, 0x3B, 0xD4, 0x3,
+                         0x3B, 0xB0, 0xC3, 0xC, 0xC7, 0x69, 0x87, 0x70,
+                         0x58, 0x87, 0x72, 0x70, 0x83, 0x74, 0x68, 0x7,
+                         0x78, 0x60, 0x87, 0x74, 0x18, 0x87, 0x74, 0xA0,
+                         0x87, 0x19, 0xCE, 0x53, 0xF, 0xEE, 0x0, 0xF, 0xF2,
+                         0x50, 0xE, 0xE4, 0x90, 0xE, 0xE3, 0x40, 0xF, 0xE1,
+                         0x20, 0xE, 0xEC, 0x50, 0xE, 0x33, 0x20, 0x28,
+                         0x1D, 0xDC, 0xC1, 0x1E, 0xC2, 0x41, 0x1E, 0xD2,
+                         0x21, 0x1C, 0xDC, 0x81, 0x1E, 0xDC, 0xE0, 0x1C,
+                         0xE4, 0xE1, 0x1D, 0xEA, 0x1, 0x1E, 0x66, 0x18,
+                         0x51, 0x38, 0xB0, 0x43, 0x3A, 0x9C, 0x83, 0x3B,
+                         0xCC, 0x50, 0x24, 0x76, 0x60, 0x7, 0x7B, 0x68,
+                         0x7, 0x37, 0x60, 0x87, 0x77, 0x78, 0x7, 0x78,
+                         0x98, 0x51, 0x4C, 0xF4, 0x90, 0xF, 0xF0, 0x50,
+                         0xE, 0x33, 0x1E, 0x6A, 0x1E, 0xCA, 0x61, 0x1C,
+                         0xE8, 0x21, 0x1D, 0xDE, 0xC1, 0x1D, 0x7E, 0x1,
+                         0x1E, 0xE4, 0xA1, 0x1C, 0xCC, 0x21, 0x1D, 0xF0,
+                         0x61, 0x6, 0x54, 0x85, 0x83, 0x38, 0xCC, 0xC3,
+                         0x3B, 0xB0, 0x43, 0x3D, 0xD0, 0x43, 0x39, 0xFC,
+                         0xC2, 0x3C, 0xE4, 0x43, 0x3B, 0x88, 0xC3, 0x3B,
+                         0xB0, 0xC3, 0x8C, 0xC5, 0xA, 0x87, 0x79, 0x98,
+                         0x87, 0x77, 0x18, 0x87, 0x74, 0x8, 0x7, 0x7A,
+                         0x28, 0x7, 0x72, 0x98, 0x81, 0x5C, 0xE3, 0x10,
+                         0xE, 0xEC, 0xC0, 0xE, 0xE5, 0x50, 0xE, 0xF3, 0x30,
+                         0x23, 0xC1, 0xD2, 0x41, 0x1E, 0xE4, 0xE1, 0x17,
+                         0xD8, 0xE1, 0x1D, 0xDE, 0x1, 0x1E, 0x66, 0x48,
+                         0x19, 0x3B, 0xB0, 0x83, 0x3D, 0xB4, 0x83, 0x1B,
+                         0x84, 0xC3, 0x38, 0x8C, 0x43, 0x39, 0xCC, 0xC3,
+                         0x3C, 0xB8, 0xC1, 0x39, 0xC8, 0xC3, 0x3B, 0xD4,
+                         0x3, 0x3C, 0xCC, 0x48, 0xB4, 0x71, 0x8, 0x7, 0x76,
+                         0x60, 0x7, 0x71, 0x8, 0x87, 0x71, 0x58, 0x87,
+                         0x19, 0xDB, 0xC6, 0xE, 0xEC, 0x60, 0xF, 0xED,
+                         0xE0, 0x6, 0xF0, 0x20, 0xF, 0xE5, 0x30, 0xF, 0xE5,
+                         0x20, 0xF, 0xF6, 0x50, 0xE, 0x6E, 0x10, 0xE, 0xE3,
+                         0x30, 0xE, 0xE5, 0x30, 0xF, 0xF3, 0xE0, 0x6, 0xE9,
+                         0xE0, 0xE, 0xE4, 0x50, 0xE, 0xF8, 0x30, 0x23,
+                         0xE2, 0xEC, 0x61, 0x1C, 0xC2, 0x81, 0x1D, 0xD8,
+                         0xE1, 0x17, 0xEC, 0x21, 0x1D, 0xE6, 0x21, 0x1D,
+                         0xC4, 0x21, 0x1D, 0xD8, 0x21, 0x1D, 0xE8, 0x21,
+                         0x1F, 0x66, 0x20, 0x9D, 0x3B, 0xBC, 0x43, 0x3D,
+                         0xB8, 0x3, 0x39, 0x94, 0x83, 0x39, 0xCC, 0x58,
+                         0xBC, 0x70, 0x70, 0x7, 0x77, 0x78, 0x7, 0x7A,
+                         0x8, 0x7, 0x7A, 0x48, 0x87, 0x77, 0x70, 0x87,
+                         0x19, 0xCB, 0xE7, 0xE, 0xEF, 0x30, 0xF, 0xE1,
+                         0xE0, 0xE, 0xE9, 0x40, 0xF, 0xE9, 0xA0, 0xF, 0xE5,
+                         0x30, 0xC3, 0x1, 0x3, 0x73, 0xA8, 0x7, 0x77, 0x18,
+                         0x87, 0x5F, 0x98, 0x87, 0x70, 0x70, 0x87, 0x74,
+                         0xA0, 0x87, 0x74, 0xD0, 0x87, 0x72, 0x98, 0x81,
+                         0x84, 0x41, 0x39, 0xE0, 0xC3, 0x38, 0xB0, 0x43,
+                         0x3D, 0x90, 0x43, 0x39, 0xCC, 0x40, 0xC4, 0xA0,
+                         0x1D, 0xCA, 0xA1, 0x1D, 0xE0, 0x41, 0x1E, 0xDE,
+                         0xC1, 0x1C, 0x66, 0x24, 0x63, 0x30, 0xE, 0xE1,
+                         0xC0, 0xE, 0xEC, 0x30, 0xF, 0xE9, 0x40, 0xF, 0xE5,
+                         0x30, 0x43, 0x21, 0x83, 0x75, 0x18, 0x7, 0x73,
+                         0x48, 0x87, 0x5F, 0xA0, 0x87, 0x7C, 0x80, 0x87,
+                         0x72, 0x98, 0xB1, 0x94, 0x1, 0x3C, 0x8C, 0xC3,
+                         0x3C, 0x94, 0xC3, 0x38, 0xD0, 0x43, 0x3A, 0xBC,
+                         0x83, 0x3B, 0xCC, 0xC3, 0x8C, 0xC5, 0xC, 0x48,
+                         0x21, 0x15, 0x42, 0x61, 0x1E, 0xE6, 0x21, 0x1D,
+                         0xCE, 0xC1, 0x1D, 0x52, 0x81, 0x14, 0x66, 0x4C,
+                         0x67, 0x30, 0xE, 0xEF, 0x20, 0xF, 0xEF, 0xE0,
+                         0x6, 0xEF, 0x50, 0xF, 0xF4, 0x30, 0xF, 0xE9, 0x40,
+                         0xE, 0xE5, 0xE0, 0x6, 0xE6, 0x20, 0xF, 0xE1, 0xD0,
+                         0xE, 0xE5, 0x30, 0xA3, 0x40, 0x83, 0x76, 0x68,
+                         0x7, 0x79, 0x8, 0x87, 0x19, 0x52, 0x1A, 0xB8,
+                         0xC3, 0x3B, 0x84, 0x3, 0x3B, 0xA4, 0x43, 0x38,
+                         0xCC, 0x83, 0x1B, 0x84, 0x3, 0x39, 0x90, 0x83,
+                         0x3C, 0xCC, 0x3, 0x3C, 0x84, 0xC3, 0x38, 0x94,
+                         0xC3, 0xC, 0x46, 0xD, 0xC6, 0x21, 0x1C, 0xD8,
+                         0x81, 0x1D, 0xCA, 0xA1, 0x1C, 0x7E, 0x81, 0x1E,
+                         0xF2, 0x1, 0x1E, 0xCA, 0x61, 0x86, 0xB3, 0x6,
+                         0xE4, 0x80, 0xF, 0x6E, 0xE0, 0xE, 0xEF, 0xE0,
+                         0xE, 0xF5, 0xE0, 0xE, 0xE9, 0x60, 0xE, 0xEF, 0x20,
+                         0xF, 0xED, 0x30, 0xA3, 0x62, 0x3, 0x72, 0xC0,
+                         0x7, 0x37, 0x18, 0x87, 0x77, 0x70, 0x7, 0x7A,
+                         0x90, 0x87, 0x77, 0x60, 0x7, 0x73, 0x60, 0x87,
+                         0x77, 0xB8, 0x7, 0x37, 0x40, 0x87, 0x74, 0x70,
+                         0x7, 0x7A, 0x98, 0x87, 0x19, 0x4B, 0x1B, 0x90,
+                         0x3, 0x3E, 0xB8, 0x1, 0x3C, 0xC8, 0x43, 0x39,
+                         0x8C, 0x43, 0x3A, 0xCC, 0x43, 0x39, 0x0, 0x0,
+                         0x79, 0x28, 0x0, 0x0, 0x53, 0x0, 0x0, 0x0, 0xC2,
+                         0x3C, 0x90, 0x40, 0x86, 0x10, 0x19, 0x32, 0xE2,
+                         0x64, 0x90, 0x40, 0x46, 0x2, 0x19, 0x23, 0x23,
+                         0x46, 0x2, 0x13, 0x24, 0xC6, 0x0, 0x13, 0x74,
+                         0xD4, 0x61, 0x8C, 0x2D, 0xCC, 0xED, 0xC, 0xC4,
+                         0xAE, 0x4C, 0x6E, 0x2E, 0xED, 0xCD, 0xD, 0x44,
+                         0x46, 0xC6, 0x5, 0xC6, 0x5, 0xE6, 0x2C, 0x8D,
+                         0xE, 0x4, 0xE5, 0x2C, 0x8D, 0xE, 0xE8, 0x2C, 0x8D,
+                         0xE, 0xAD, 0x4E, 0xCC, 0x65, 0xEC, 0xAD, 0x4D,
+                         0x27, 0xCD, 0x4D, 0xAC, 0x8C, 0x2D, 0x6D, 0xEC,
+                         0x85, 0x8D, 0xCD, 0xAE, 0xAD, 0x5, 0x4E, 0xEE,
+                         0x4D, 0xAD, 0x6C, 0x8C, 0xCE, 0xE5, 0x2C, 0x8D,
+                         0xE, 0x84, 0x86, 0xC6, 0xCC, 0xC6, 0x86, 0x4C,
+                         0xC, 0x87, 0x6C, 0xEC, 0x26, 0x67, 0x46, 0x26,
+                         0x67, 0x6C, 0xA6, 0xCC, 0x66, 0x8C, 0xC6, 0x2C,
+                         0xEC, 0x26, 0xC, 0x26, 0x2C, 0xEC, 0x26, 0xCC,
+                         0xCC, 0x86, 0x6, 0xE6, 0x6, 0x26, 0xE7, 0x86,
+                         0xE6, 0x26, 0xE5, 0x8, 0x63, 0x73, 0x87, 0x68,
+                         0xB, 0x4B, 0x73, 0x3B, 0xCA, 0xDD, 0x18, 0x5A,
+                         0x98, 0xDC, 0xD7, 0x5C, 0x9A, 0x5E, 0xD9, 0x69,
+                         0xCC, 0xE4, 0xC2, 0xDA, 0xCA, 0x5A, 0xE0, 0xDE,
+                         0xD2, 0xDC, 0xE8, 0xCA, 0xE4, 0x86, 0x20, 0x1C,
+                         0xC1, 0x10, 0x84, 0x43, 0x18, 0x82, 0x70, 0xC,
+                         0x43, 0x10, 0xE, 0x62, 0x8, 0x42, 0x1, 0xC, 0x41,
+                         0x38, 0x8A, 0x21, 0x8, 0x87, 0x31, 0x6, 0xC1,
+                         0x38, 0xC6, 0x10, 0x4, 0x63, 0x18, 0x4, 0x24,
+                         0x19, 0x83, 0x60, 0x24, 0x63, 0x18, 0xC, 0xC3,
+                         0x18, 0x83, 0xB0, 0x44, 0x63, 0x28, 0x94, 0x1,
+                         0x0, 0xA4, 0x31, 0xC, 0x6, 0xB1, 0x8C, 0x61, 0x60,
+                         0xA, 0xC6, 0x24, 0x64, 0x78, 0x2E, 0x76, 0x61,
+                         0x6C, 0x76, 0x65, 0x72, 0x43, 0x9, 0x18, 0xA3,
+                         0xB0, 0xB1, 0xD9, 0xB5, 0xB9, 0xA4, 0x91, 0x95,
+                         0xB9, 0xD1, 0xD, 0x25, 0x68, 0x8C, 0x43, 0x86,
+                         0xE7, 0x32, 0x87, 0x16, 0x46, 0x56, 0x26, 0xD7,
+                         0xF4, 0x46, 0x56, 0xC6, 0x36, 0x94, 0xC0, 0x31,
+                         0xA, 0x19, 0x9E, 0x8B, 0x5D, 0x99, 0xDC, 0x5C,
+                         0xDA, 0x9B, 0xDB, 0x50, 0x82, 0xC7, 0x38, 0x64,
+                         0x78, 0x2E, 0x65, 0x6E, 0x74, 0x72, 0x79, 0x50,
+                         0x6F, 0x69, 0x6E, 0x74, 0x73, 0x43, 0x9, 0x24,
+                         0x13, 0xB1, 0xB1, 0xD9, 0xB5, 0xB9, 0xB4, 0xBD,
+                         0x91, 0xD5, 0xB1, 0x95, 0xB9, 0x98, 0xB1, 0x85,
+                         0x9D, 0xCD, 0xD, 0x45, 0x98, 0x28, 0x0, 0x0, 0x71,
+                         0x20, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x6, 0x40,
+                         0x30, 0x0, 0xD2, 0x0, 0x0, 0x0, 0x61, 0x20, 0x0,
+                         0x0, 0x6, 0x0, 0x0, 0x0, 0x13, 0x4, 0x1, 0x86,
+                         0x3, 0x1, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x7, 0x50,
+                         0x10, 0xCD, 0x14, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0,
+                         0x0, 0x0, 0x0, 0x0, 0x0 ]
+  - Name:            SFI0
+    Size:            8
+  - Name:            HASH
+    Size:            20
+    Hash:
+      IncludesSource:  false
+      Digest:          [ 0x9F, 0xD1, 0xD9, 0xE2, 0x49, 0xFB, 0x3A, 0x6C,
+                         0x8C, 0x14, 0x8A, 0x96, 0x1C, 0x7D, 0x85, 0xA9 ]
+  - Name:            ISG1
+    Size:            8
+    Signature:
+      Parameters:      []
+  - Name:            OSG1
+    Size:            8
+    Signature:
+      Parameters:      []
+  - Name:            RTS0
+    Size:            24
+    RootSignature:
+      Version:         2
+      NumRootParameters: 0
+      RootParametersOffset: 24
+      NumStaticSamplers: 0
+      StaticSamplersOffset: 0
+      Parameters:      []
+  - Name:            PSV0
+    Size:            76
+    PSVInfo:
+      Version:         3
+      ShaderStage:     5
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      SigInputVectors: 0
+      SigOutputVectors: [ 0, 0, 0, 0 ]
+      NumThreadsX:     1
+      NumThreadsY:     1
+      NumThreadsZ:     1
+      EntryName:       main
+      ResourceStride:  24
+      Resources:       []
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+...
diff --git a/llvm/test/tools/llvm-objcopy/DXContainer/copy-headers.test b/llvm/test/tools/llvm-objcopy/DXContainer/copy-headers.test
new file mode 100644
index 0000000000000..b7720c8d57fbb
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/DXContainer/copy-headers.test
@@ -0,0 +1,42 @@
+## Tests that the copied DXContainer is identical to the original, ensuring all
+## the different headers are correctly handled.
+
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objcopy %t %t.out
+# RUN: cmp %t %t.out
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  FileSize:        1996
+  PartCount:       7
+  PartOffsets:     [ 60, 76, 92, 108, 236, 1932, 1960 ]
+Parts:
+  - Name:            FKE0
+    Size:            8
+  - Name:            FKE1
+    Size:            8
+  - Name:            FKE2
+    Size:            8
+  - Name:            FKE3
+    Size:            120
+  - Name:            FKE4
+    Size:            1688
+  - Name:            FKE5
+    Size:            20
+  - Name:            DXIL
+    Size:            28
+    Program:
+      MajorVersion:    6
+      MinorVersion:    5
+      ShaderKind:      5
+      Size:            8
+      DXILMajorVersion: 1
+      DXILMinorVersion: 5
+      DXILSize:        4
+      DXIL:            [ 0x42, 0x43, 0xC0, 0xDE, ]
+...
diff --git a/llvm/test/tools/llvm-objcopy/DXContainer/reading-errs.test b/llvm/test/tools/llvm-objcopy/DXContainer/reading-errs.test
new file mode 100644
index 0000000000000..32099552a4525
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/DXContainer/reading-errs.test
@@ -0,0 +1,43 @@
+## Check that llvm-objcopy reports a suitable error when it
+## encounters an invalid input during reading.
+
+## We can't have multiple DXIL parts.
+# RUN: yaml2obj %s --docnum=1 -o %t1
+# RUN: not llvm-objcopy %t1 %t1.out 2>&1 | FileCheck %s -DFILE=%t1 --check-prefix=ERROR1
+
+# ERROR1: error: '[[FILE]]': More than one DXIL part is present in the file
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       2
+Parts:
+  - Name:            DXIL
+    Size:            28
+  - Name:            DXIL
+    Size:            28
+...
+
+## The first part offset is out of file bounds.
+# RUN: yaml2obj %s --docnum=2 -o %t2
+# RUN: not llvm-objcopy %t2 %t2.out 2>&1 | FileCheck %s -DFILE=%t2 --check-prefix=ERROR2
+
+# ERROR2: error: '[[FILE]]': Reading structure out of file bounds
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           2
+    Minor:           0
+  PartCount:       1
+  PartOffsets:     [ 48 ]
+Parts:
+  - Name:            DXIL
+    Size:            28
+...
diff --git a/llvm/test/tools/llvm-objcopy/DXContainer/remove-headers.test b/llvm/test/tools/llvm-objcopy/DXContainer/remove-headers.test
new file mode 100644
index 0000000000000..4f4a138a4852c
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/DXContainer/remove-headers.test
@@ -0,0 +1,51 @@
+## Tests that the copied DXContainer correctly has the specified headers
+## removed.
+
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objcopy --remove-section=FKE1 --remove-section=FKE4 %t %t.out
+# RUN: obj2yaml %t.out | FileCheck %s
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+## FileSize = 1996 - 8 (FKE1 content) - 1688 (FKE4 content)
+##          - 8 (2 part offsets) - 16 (2 part headers)
+##          = 276
+# CHECK: FileSize:       276
+  FileSize:        1996
+# CHECK-NEXT: PartCount:      5
+  PartCount:       7
+# CHECK-NEXT: PartOffsets:    [ 52, 68, 84, 212, 240 ]
+  PartOffsets:     [ 60, 76, 92, 108, 236, 1932, 1960 ]
+Parts:
+# CHECK-NEXT: Parts
+# CHECK-NOT: FKE1
+# CHECK-NOT: FKE4
+  - Name:            FKE0
+    Size:            8
+  - Name:            FKE1
+    Size:            8
+  - Name:            FKE2
+    Size:            8
+  - Name:            FKE3
+    Size:            120
+  - Name:            FKE4
+    Size:            1688
+  - Name:            FKE5
+    Size:            20
+  - Name:            DXIL
+    Size:            28
+    Program:
+      MajorVersion:    6
+      MinorVersion:    5
+      ShaderKind:      5
+      Size:            8
+      DXILMajorVersion: 1
+      DXILMinorVersion: 5
+      DXILSize:        4
+      DXIL:            [ 0x42, 0x43, 0xC0, 0xDE, ]
+...
diff --git a/llvm/test/tools/llvm-objcopy/DXContainer/remove-root-signature.test b/llvm/test/tools/llvm-objcopy/DXContainer/remove-root-signature.test
new file mode 100644
index 0000000000000..3bb11787b646a
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/DXContainer/remove-root-signature.test
@@ -0,0 +1,302 @@
+## Tests that the RTS0 (root signature) part is correctly removed from the
+## copied DXContainer.
+
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objcopy --remove-section=RTS0 %t %t.out
+# RUN: obj2yaml %t.out | FileCheck %s
+
+## The DXContainer described below was generated with:
+
+## `clang-dxc -T cs_6_7 test.hlsl /Fo temp.dxo`
+## `obj2yaml temp.dxo`
+
+## ``` test.hlsl
+## [RootSignature("")]
+## [numthreads(1,1,1)]
+## void main() {}
+## ```
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+## FileSize = 1984 - 24 (RTS0 content) - 4 (1 part offset) - 8 (1 part header)
+##          = 1948
+# CHECK: FileSize:       1948
+  FileSize:        1984
+# CHECK-NEXT: PartCount:      6
+  PartCount:       7
+# CHECK-NEXT: PartOffsets:     [ 56, 1788, 1804, 1832, 1848, 1864 ]
+  PartOffsets:     [ 60, 1792, 1808, 1836, 1852, 1868, 1900 ]
+Parts:
+# CHECK-NEXT: Parts
+# CHECK-NOT: RTS0
+  - Name:            DXIL
+    Size:            1724
+    Program:
+      MajorVersion:    6
+      MinorVersion:    7
+      ShaderKind:      5
+      Size:            431
+      DXILMajorVersion: 1
+      DXILMinorVersion: 7
+      DXILSize:        1700
+      DXIL:            [ 0x42, 0x43, 0xC0, 0xDE, 0x21, 0xC, 0x0, 0x0, 0xA6,
+                         0x1, 0x0, 0x0, 0xB, 0x82, 0x20, 0x0, 0x2, 0x0,
+                         0x0, 0x0, 0x13, 0x0, 0x0, 0x0, 0x7, 0x81, 0x23,
+                         0x91, 0x41, 0xC8, 0x4, 0x49, 0x6, 0x10, 0x32,
+                         0x39, 0x92, 0x1, 0x84, 0xC, 0x25, 0x5, 0x8, 0x19,
+                         0x1E, 0x4, 0x8B, 0x62, 0x80, 0x10, 0x45, 0x2,
+                         0x42, 0x92, 0xB, 0x42, 0x84, 0x10, 0x32, 0x14,
+                         0x38, 0x8, 0x18, 0x4B, 0xA, 0x32, 0x42, 0x88,
+                         0x48, 0x90, 0x14, 0x20, 0x43, 0x46, 0x88, 0xA5,
+                         0x0, 0x19, 0x32, 0x42, 0xE4, 0x48, 0xE, 0x90,
+                         0x11, 0x22, 0xC4, 0x50, 0x41, 0x51, 0x81, 0x8C,
+                         0xE1, 0x83, 0xE5, 0x8A, 0x4, 0x21, 0x46, 0x6,
+                         0x51, 0x18, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x1B,
+                         0x90, 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0x7, 0xC0,
+                         0x1, 0x24, 0x80, 0x2, 0x0, 0x0, 0x0, 0x49, 0x18,
+                         0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x13, 0x82, 0x0,
+                         0x0, 0x89, 0x20, 0x0, 0x0, 0x11, 0x0, 0x0, 0x0,
+                         0x32, 0x22, 0x8, 0x9, 0x20, 0x64, 0x85, 0x4, 0x13,
+                         0x22, 0xA4, 0x84, 0x4, 0x13, 0x22, 0xE3, 0x84,
+                         0xA1, 0x90, 0x14, 0x12, 0x4C, 0x88, 0x8C, 0xB,
+                         0x84, 0x84, 0x4C, 0x10, 0x20, 0x73, 0x4, 0x8,
+                         0xC1, 0x65, 0xC3, 0x85, 0x2C, 0xE8, 0x3, 0x40,
+                         0x14, 0x91, 0x4E, 0xD1, 0x4A, 0x48, 0x44, 0x54,
+                         0x11, 0xC3, 0x9, 0x30, 0xC4, 0x18, 0x1, 0x30,
+                         0x2, 0x50, 0x82, 0x21, 0x1A, 0x8, 0x98, 0x23,
+                         0x0, 0x3, 0x0, 0x13, 0x14, 0x72, 0xC0, 0x87, 0x74,
+                         0x60, 0x87, 0x36, 0x68, 0x87, 0x79, 0x68, 0x3,
+                         0x72, 0xC0, 0x87, 0xD, 0xAE, 0x50, 0xE, 0x6D,
+                         0xD0, 0xE, 0x7A, 0x50, 0xE, 0x6D, 0x0, 0xF, 0x7A,
+                         0x30, 0x7, 0x72, 0xA0, 0x7, 0x73, 0x20, 0x7, 0x6D,
+                         0x90, 0xE, 0x71, 0xA0, 0x7, 0x73, 0x20, 0x7, 0x6D,
+                         0x90, 0xE, 0x78, 0xA0, 0x7, 0x78, 0xD0, 0x6, 0xE9,
+                         0x10, 0x7, 0x76, 0xA0, 0x7, 0x71, 0x60, 0x7, 0x6D,
+                         0x90, 0xE, 0x73, 0x20, 0x7, 0x7A, 0x30, 0x7, 0x72,
+                         0xD0, 0x6, 0xE9, 0x60, 0x7, 0x74, 0xA0, 0x7, 0x76,
+                         0x40, 0x7, 0x6D, 0x60, 0xE, 0x71, 0x60, 0x7, 0x7A,
+                         0x10, 0x7, 0x76, 0xD0, 0x6, 0xE6, 0x30, 0x7, 0x72,
+                         0xA0, 0x7, 0x73, 0x20, 0x7, 0x6D, 0x60, 0xE, 0x76,
+                         0x40, 0x7, 0x7A, 0x60, 0x7, 0x74, 0xD0, 0x6, 0xEE,
+                         0x80, 0x7, 0x7A, 0x10, 0x7, 0x76, 0xA0, 0x7, 0x73,
+                         0x20, 0x7, 0x7A, 0x60, 0x7, 0x74, 0x30, 0xE4,
+                         0x21, 0x0, 0x0, 0x8, 0x0, 0x0, 0x0, 0x2, 0x0,
+                         0x0, 0x0, 0x20, 0xB, 0x4, 0x7, 0x0, 0x0, 0x0,
+                         0x32, 0x1E, 0x98, 0xC, 0x19, 0x11, 0x4C, 0x90,
+                         0x8C, 0x9, 0x26, 0x47, 0xC6, 0x4, 0x43, 0xBA,
+                         0x12, 0x28, 0x88, 0x62, 0x28, 0x87, 0x42, 0x28,
+                         0x2, 0x0, 0x0, 0x0, 0x79, 0x18, 0x0, 0x0, 0xE2,
+                         0x0, 0x0, 0x0, 0x33, 0x8, 0x80, 0x1C, 0xC4, 0xE1,
+                         0x1C, 0x66, 0x14, 0x1, 0x3D, 0x88, 0x43, 0x38,
+                         0x84, 0xC3, 0x8C, 0x42, 0x80, 0x7, 0x79, 0x78,
+                         0x7, 0x73, 0x98, 0x71, 0xC, 0xE6, 0x0, 0xF, 0xED,
+                         0x10, 0xE, 0xF4, 0x80, 0xE, 0x33, 0xC, 0x42, 0x1E,
+                         0xC2, 0xC1, 0x1D, 0xCE, 0xA1, 0x1C, 0x66, 0x30,
+                         0x5, 0x3D, 0x88, 0x43, 0x38, 0x84, 0x83, 0x1B,
+                         0xCC, 0x3, 0x3D, 0xC8, 0x43, 0x3D, 0x8C, 0x3,
+                         0x3D, 0xCC, 0x78, 0x8C, 0x74, 0x70, 0x7, 0x7B,
+                         0x8, 0x7, 0x79, 0x48, 0x87, 0x70, 0x70, 0x7, 0x7A,
+                         0x70, 0x3, 0x76, 0x78, 0x87, 0x70, 0x20, 0x87,
+                         0x19, 0xCC, 0x11, 0xE, 0xEC, 0x90, 0xE, 0xE1,
+                         0x30, 0xF, 0x6E, 0x30, 0xF, 0xE3, 0xF0, 0xE, 0xF0,
+                         0x50, 0xE, 0x33, 0x10, 0xC4, 0x1D, 0xDE, 0x21,
+                         0x1C, 0xD8, 0x21, 0x1D, 0xC2, 0x61, 0x1E, 0x66,
+                         0x30, 0x89, 0x3B, 0xBC, 0x83, 0x3B, 0xD0, 0x43,
+                         0x39, 0xB4, 0x3, 0x3C, 0xBC, 0x83, 0x3C, 0x84,
+                         0x3, 0x3B, 0xCC, 0xF0, 0x14, 0x76, 0x60, 0x7,
+                         0x7B, 0x68, 0x7, 0x37, 0x68, 0x87, 0x72, 0x68,
+                         0x7, 0x37, 0x80, 0x87, 0x70, 0x90, 0x87, 0x70,
+                         0x60, 0x7, 0x76, 0x28, 0x7, 0x76, 0xF8, 0x5, 0x76,
+                         0x78, 0x87, 0x77, 0x80, 0x87, 0x5F, 0x8, 0x87,
+                         0x71, 0x18, 0x87, 0x72, 0x98, 0x87, 0x79, 0x98,
+                         0x81, 0x2C, 0xEE, 0xF0, 0xE, 0xEE, 0xE0, 0xE,
+                         0xF5, 0xC0, 0xE, 0xEC, 0x30, 0x3, 0x62, 0xC8,
+                         0xA1, 0x1C, 0xE4, 0xA1, 0x1C, 0xCC, 0xA1, 0x1C,
+                         0xE4, 0xA1, 0x1C, 0xDC, 0x61, 0x1C, 0xCA, 0x21,
+                         0x1C, 0xC4, 0x81, 0x1D, 0xCA, 0x61, 0x6, 0xD6,
+                         0x90, 0x43, 0x39, 0xC8, 0x43, 0x39, 0x98, 0x43,
+                         0x39, 0xC8, 0x43, 0x39, 0xB8, 0xC3, 0x38, 0x94,
+                         0x43, 0x38, 0x88, 0x3, 0x3B, 0x94, 0xC3, 0x2F,
+                         0xBC, 0x83, 0x3C, 0xFC, 0x82, 0x3B, 0xD4, 0x3,
+                         0x3B, 0xB0, 0xC3, 0xC, 0xC7, 0x69, 0x87, 0x70,
+                         0x58, 0x87, 0x72, 0x70, 0x83, 0x74, 0x68, 0x7,
+                         0x78, 0x60, 0x87, 0x74, 0x18, 0x87, 0x74, 0xA0,
+                         0x87, 0x19, 0xCE, 0x53, 0xF, 0xEE, 0x0, 0xF, 0xF2,
+                         0x50, 0xE, 0xE4, 0x90, 0xE, 0xE3, 0x40, 0xF, 0xE1,
+                         0x20, 0xE, 0xEC, 0x50, 0xE, 0x33, 0x20, 0x28,
+                         0x1D, 0xDC, 0xC1, 0x1E, 0xC2, 0x41, 0x1E, 0xD2,
+                         0x21, 0x1C, 0xDC, 0x81, 0x1E, 0xDC, 0xE0, 0x1C,
+                         0xE4, 0xE1, 0x1D, 0xEA, 0x1, 0x1E, 0x66, 0x18,
+                         0x51, 0x38, 0xB0, 0x43, 0x3A, 0x9C, 0x83, 0x3B,
+                         0xCC, 0x50, 0x24, 0x76, 0x60, 0x7, 0x7B, 0x68,
+                         0x7, 0x37, 0x60, 0x87, 0x77, 0x78, 0x7, 0x78,
+                         0x98, 0x51, 0x4C, 0xF4, 0x90, 0xF, 0xF0, 0x50,
+                         0xE, 0x33, 0x1E, 0x6A, 0x1E, 0xCA, 0x61, 0x1C,
+                         0xE8, 0x21, 0x1D, 0xDE, 0xC1, 0x1D, 0x7E, 0x1,
+                         0x1E, 0xE4, 0xA1, 0x1C, 0xCC, 0x21, 0x1D, 0xF0,
+                         0x61, 0x6, 0x54, 0x85, 0x83, 0x38, 0xCC, 0xC3,
+                         0x3B, 0xB0, 0x43, 0x3D, 0xD0, 0x43, 0x39, 0xFC,
+                         0xC2, 0x3C, 0xE4, 0x43, 0x3B, 0x88, 0xC3, 0x3B,
+                         0xB0, 0xC3, 0x8C, 0xC5, 0xA, 0x87, 0x79, 0x98,
+                         0x87, 0x77, 0x18, 0x87, 0x74, 0x8, 0x7, 0x7A,
+                         0x28, 0x7, 0x72, 0x98, 0x81, 0x5C, 0xE3, 0x10,
+                         0xE, 0xEC, 0xC0, 0xE, 0xE5, 0x50, 0xE, 0xF3, 0x30,
+                         0x23, 0xC1, 0xD2, 0x41, 0x1E, 0xE4, 0xE1, 0x17,
+                         0xD8, 0xE1, 0x1D, 0xDE, 0x1, 0x1E, 0x66, 0x48,
+                         0x19, 0x3B, 0xB0, 0x83, 0x3D, 0xB4, 0x83, 0x1B,
+                         0x84, 0xC3, 0x38, 0x8C, 0x43, 0x39, 0xCC, 0xC3,
+                         0x3C, 0xB8, 0xC1, 0x39, 0xC8, 0xC3, 0x3B, 0xD4,
+                         0x3, 0x3C, 0xCC, 0x48, 0xB4, 0x71, 0x8, 0x7, 0x76,
+                         0x60, 0x7, 0x71, 0x8, 0x87, 0x71, 0x58, 0x87,
+                         0x19, 0xDB, 0xC6, 0xE, 0xEC, 0x60, 0xF, 0xED,
+                         0xE0, 0x6, 0xF0, 0x20, 0xF, 0xE5, 0x30, 0xF, 0xE5,
+                         0x20, 0xF, 0xF6, 0x50, 0xE, 0x6E, 0x10, 0xE, 0xE3,
+                         0x30, 0xE, 0xE5, 0x30, 0xF, 0xF3, 0xE0, 0x6, 0xE9,
+                         0xE0, 0xE, 0xE4, 0x50, 0xE, 0xF8, 0x30, 0x23,
+                         0xE2, 0xEC, 0x61, 0x1C, 0xC2, 0x81, 0x1D, 0xD8,
+                         0xE1, 0x17, 0xEC, 0x21, 0x1D, 0xE6, 0x21, 0x1D,
+                         0xC4, 0x21, 0x1D, 0xD8, 0x21, 0x1D, 0xE8, 0x21,
+                         0x1F, 0x66, 0x20, 0x9D, 0x3B, 0xBC, 0x43, 0x3D,
+                         0xB8, 0x3, 0x39, 0x94, 0x83, 0x39, 0xCC, 0x58,
+                         0xBC, 0x70, 0x70, 0x7, 0x77, 0x78, 0x7, 0x7A,
+                         0x8, 0x7, 0x7A, 0x48, 0x87, 0x77, 0x70, 0x87,
+                         0x19, 0xCB, 0xE7, 0xE, 0xEF, 0x30, 0xF, 0xE1,
+                         0xE0, 0xE, 0xE9, 0x40, 0xF, 0xE9, 0xA0, 0xF, 0xE5,
+                         0x30, 0xC3, 0x1, 0x3, 0x73, 0xA8, 0x7, 0x77, 0x18,
+                         0x87, 0x5F, 0x98, 0x87, 0x70, 0x70, 0x87, 0x74,
+                         0xA0, 0x87, 0x74, 0xD0, 0x87, 0x72, 0x98, 0x81,
+                         0x84, 0x41, 0x39, 0xE0, 0xC3, 0x38, 0xB0, 0x43,
+                         0x3D, 0x90, 0x43, 0x39, 0xCC, 0x40, 0xC4, 0xA0,
+                         0x1D, 0xCA, 0xA1, 0x1D, 0xE0, 0x41, 0x1E, 0xDE,
+                         0xC1, 0x1C, 0x66, 0x24, 0x63, 0x30, 0xE, 0xE1,
+                         0xC0, 0xE, 0xEC, 0x30, 0xF, 0xE9, 0x40, 0xF, 0xE5,
+                         0x30, 0x43, 0x21, 0x83, 0x75, 0x18, 0x7, 0x73,
+                         0x48, 0x87, 0x5F, 0xA0, 0x87, 0x7C, 0x80, 0x87,
+                         0x72, 0x98, 0xB1, 0x94, 0x1, 0x3C, 0x8C, 0xC3,
+                         0x3C, 0x94, 0xC3, 0x38, 0xD0, 0x43, 0x3A, 0xBC,
+                         0x83, 0x3B, 0xCC, 0xC3, 0x8C, 0xC5, 0xC, 0x48,
+                         0x21, 0x15, 0x42, 0x61, 0x1E, 0xE6, 0x21, 0x1D,
+                         0xCE, 0xC1, 0x1D, 0x52, 0x81, 0x14, 0x66, 0x4C,
+                         0x67, 0x30, 0xE, 0xEF, 0x20, 0xF, 0xEF, 0xE0,
+                         0x6, 0xEF, 0x50, 0xF, 0xF4, 0x30, 0xF, 0xE9, 0x40,
+                         0xE, 0xE5, 0xE0, 0x6, 0xE6, 0x20, 0xF, 0xE1, 0xD0,
+                         0xE, 0xE5, 0x30, 0xA3, 0x40, 0x83, 0x76, 0x68,
+                         0x7, 0x79, 0x8, 0x87, 0x19, 0x52, 0x1A, 0xB8,
+                         0xC3, 0x3B, 0x84, 0x3, 0x3B, 0xA4, 0x43, 0x38,
+                         0xCC, 0x83, 0x1B, 0x84, 0x3, 0x39, 0x90, 0x83,
+                         0x3C, 0xCC, 0x3, 0x3C, 0x84, 0xC3, 0x38, 0x94,
+                         0xC3, 0xC, 0x46, 0xD, 0xC6, 0x21, 0x1C, 0xD8,
+                         0x81, 0x1D, 0xCA, 0xA1, 0x1C, 0x7E, 0x81, 0x1E,
+                         0xF2, 0x1, 0x1E, 0xCA, 0x61, 0x86, 0xB3, 0x6,
+                         0xE4, 0x80, 0xF, 0x6E, 0xE0, 0xE, 0xEF, 0xE0,
+                         0xE, 0xF5, 0xE0, 0xE, 0xE9, 0x60, 0xE, 0xEF, 0x20,
+                         0xF, 0xED, 0x30, 0xA3, 0x62, 0x3, 0x72, 0xC0,
+                         0x7, 0x37, 0x18, 0x87, 0x77, 0x70, 0x7, 0x7A,
+                         0x90, 0x87, 0x77, 0x60, 0x7, 0x73, 0x60, 0x87,
+                         0x77, 0xB8, 0x7, 0x37, 0x40, 0x87, 0x74, 0x70,
+                         0x7, 0x7A, 0x98, 0x87, 0x19, 0x4B, 0x1B, 0x90,
+                         0x3, 0x3E, 0xB8, 0x1, 0x3C, 0xC8, 0x43, 0x39,
+                         0x8C, 0x43, 0x3A, 0xCC, 0x43, 0x39, 0x0, 0x0,
+                         0x79, 0x28, 0x0, 0x0, 0x53, 0x0, 0x0, 0x0, 0xC2,
+                         0x3C, 0x90, 0x40, 0x86, 0x10, 0x19, 0x32, 0xE2,
+                         0x64, 0x90, 0x40, 0x46, 0x2, 0x19, 0x23, 0x23,
+                         0x46, 0x2, 0x13, 0x24, 0xC6, 0x0, 0x13, 0x74,
+                         0xD4, 0x61, 0x8C, 0x2D, 0xCC, 0xED, 0xC, 0xC4,
+                         0xAE, 0x4C, 0x6E, 0x2E, 0xED, 0xCD, 0xD, 0x44,
+                         0x46, 0xC6, 0x5, 0xC6, 0x5, 0xE6, 0x2C, 0x8D,
+                         0xE, 0x4, 0xE5, 0x2C, 0x8D, 0xE, 0xE8, 0x2C, 0x8D,
+                         0xE, 0xAD, 0x4E, 0xCC, 0x65, 0xEC, 0xAD, 0x4D,
+                         0x27, 0xCD, 0x4D, 0xAC, 0x8C, 0x2D, 0x6D, 0xEC,
+                         0x85, 0x8D, 0xCD, 0xAE, 0xAD, 0x5, 0x4E, 0xEE,
+                         0x4D, 0xAD, 0x6C, 0x8C, 0xCE, 0xE5, 0x2C, 0x8D,
+                         0xE, 0x84, 0x86, 0xC6, 0xCC, 0xC6, 0x86, 0x4C,
+                         0xC, 0x87, 0x6C, 0xEC, 0x26, 0x67, 0x46, 0x26,
+                         0x67, 0x6C, 0xA6, 0xCC, 0x66, 0x8C, 0xC6, 0x2C,
+                         0xEC, 0x26, 0xC, 0x26, 0x2C, 0xEC, 0x26, 0xCC,
+                         0xCC, 0x86, 0x6, 0xE6, 0x6, 0x26, 0xE7, 0x86,
+                         0xE6, 0x26, 0xE5, 0x8, 0x63, 0x73, 0x87, 0x68,
+                         0xB, 0x4B, 0x73, 0x3B, 0xCA, 0xDD, 0x18, 0x5A,
+                         0x98, 0xDC, 0xD7, 0x5C, 0x9A, 0x5E, 0xD9, 0x69,
+                         0xCC, 0xE4, 0xC2, 0xDA, 0xCA, 0x5A, 0xE0, 0xDE,
+                         0xD2, 0xDC, 0xE8, 0xCA, 0xE4, 0x86, 0x20, 0x1C,
+                         0xC1, 0x10, 0x84, 0x43, 0x18, 0x82, 0x70, 0xC,
+                         0x43, 0x10, 0xE, 0x62, 0x8, 0x42, 0x1, 0xC, 0x41,
+                         0x38, 0x8A, 0x21, 0x8, 0x87, 0x31, 0x6, 0xC1,
+                         0x38, 0xC6, 0x10, 0x4, 0x63, 0x18, 0x4, 0x24,
+                         0x19, 0x83, 0x60, 0x24, 0x63, 0x18, 0xC, 0xC3,
+                         0x18, 0x83, 0xB0, 0x44, 0x63, 0x28, 0x94, 0x1,
+                         0x0, 0xA4, 0x31, 0xC, 0x6, 0xB1, 0x8C, 0x61, 0x60,
+                         0xA, 0xC6, 0x24, 0x64, 0x78, 0x2E, 0x76, 0x61,
+                         0x6C, 0x76, 0x65, 0x72, 0x43, 0x9, 0x18, 0xA3,
+                         0xB0, 0xB1, 0xD9, 0xB5, 0xB9, 0xA4, 0x91, 0x95,
+                         0xB9, 0xD1, 0xD, 0x25, 0x68, 0x8C, 0x43, 0x86,
+                         0xE7, 0x32, 0x87, 0x16, 0x46, 0x56, 0x26, 0xD7,
+                         0xF4, 0x46, 0x56, 0xC6, 0x36, 0x94, 0xC0, 0x31,
+                         0xA, 0x19, 0x9E, 0x8B, 0x5D, 0x99, 0xDC, 0x5C,
+                         0xDA, 0x9B, 0xDB, 0x50, 0x82, 0xC7, 0x38, 0x64,
+                         0x78, 0x2E, 0x65, 0x6E, 0x74, 0x72, 0x79, 0x50,
+                         0x6F, 0x69, 0x6E, 0x74, 0x73, 0x43, 0x9, 0x24,
+                         0x13, 0xB1, 0xB1, 0xD9, 0xB5, 0xB9, 0xB4, 0xBD,
+                         0x91, 0xD5, 0xB1, 0x95, 0xB9, 0x98, 0xB1, 0x85,
+                         0x9D, 0xCD, 0xD, 0x45, 0x98, 0x28, 0x0, 0x0, 0x71,
+                         0x20, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x6, 0x40,
+                         0x30, 0x0, 0xD2, 0x0, 0x0, 0x0, 0x61, 0x20, 0x0,
+                         0x0, 0x6, 0x0, 0x0, 0x0, 0x13, 0x4, 0x1, 0x86,
+                         0x3, 0x1, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x7, 0x50,
+                         0x10, 0xCD, 0x14, 0x61, 0x0, 0x0, 0x0, 0x0, 0x0,
+                         0x0, 0x0, 0x0, 0x0, 0x0 ]
+  - Name:            SFI0
+    Size:            8
+  - Name:            HASH
+    Size:            20
+    Hash:
+      IncludesSource:  false
+      Digest:          [ 0x9F, 0xD1, 0xD9, 0xE2, 0x49, 0xFB, 0x3A, 0x6C,
+                         0x8C, 0x14, 0x8A, 0x96, 0x1C, 0x7D, 0x85, 0xA9 ]
+  - Name:            ISG1
+    Size:            8
+    Signature:
+      Parameters:      []
+  - Name:            OSG1
+    Size:            8
+    Signature:
+      Parameters:      []
+  - Name:            RTS0
+    Size:            24
+    RootSignature:
+      Version:         2
+      NumRootParameters: 0
+      RootParametersOffset: 24
+      NumStaticSamplers: 0
+      StaticSamplersOffset: 0
+      Parameters:      []
+  - Name:            PSV0
+    Size:            76
+    PSVInfo:
+      Version:         3
+      ShaderStage:     5
+      MinimumWaveLaneCount: 0
+      MaximumWaveLaneCount: 4294967295
+      UsesViewID:      0
+      SigInputVectors: 0
+      SigOutputVectors: [ 0, 0, 0, 0 ]
+      NumThreadsX:     1
+      NumThreadsY:     1
+      NumThreadsZ:     1
+      EntryName:       main
+      ResourceStride:  24
+      Resources:       []
+      SigInputElements: []
+      SigOutputElements: []
+      SigPatchOrPrimElements: []
+      InputOutputMap:
+        - [  ]
+        - [  ]
+        - [  ]
+        - [  ]
+...
diff --git a/llvm/test/tools/llvm-objdump/DXContainer/input-output-signatures.yaml b/llvm/test/tools/llvm-objdump/DXContainer/input-output-signatures.yaml
new file mode 100644
index 0000000000000..ad979d2dcb7ea
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/DXContainer/input-output-signatures.yaml
@@ -0,0 +1,167 @@
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-objdump -p %t | FileCheck %s --match-full-lines --strict-whitespace
+
+## This test covers llvm-objdump printing private headers for the ISG1, OSG1,
+## and PSG1 "parts" of the DX container file format. The test uses a few
+## absurdly large values and long string names to ensure that the columns in the
+## printed table widen correctly.
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  FileSize:        630
+  PartCount:       3
+  PartOffsets:     [ 64, 124, 184 ]
+Parts:
+  - Name:            ISG1
+    Size:            52
+    Signature:
+      Parameters:
+        - Stream:          0
+          Name:            AAA_HSFoo
+          Index:           4391238    # This value forces the index column to widen
+          SystemValue:     Undefined
+          CompType:        Float32
+          Register:        0
+          Mask:            7
+          ExclusiveMask:   2
+          MinPrecision:    Default
+  - Name:            OSG1
+    Size:            52
+    Signature:
+      Parameters:
+        - Stream:          0
+          Name:            SV_Position
+          Index:           0
+          SystemValue:     Position
+          CompType:        Float32
+          Register:        2147483647 # This value forces the register column to widen
+          Mask:            15
+          ExclusiveMask:   0
+          MinPrecision:    Default
+  - Name:            PSG1
+    Size:            402
+    Signature:
+      Parameters:
+        - Stream:          0
+          Name:            SV_TessFactor
+          Index:           0
+          SystemValue:     FinalQuadEdgeTessfactor  # The tessfactor forces the SysVal column to widen
+          CompType:        Float32
+          Register:        0
+          Mask:            8
+          ExclusiveMask:   8
+          MinPrecision:    Default
+        - Stream:          0
+          Name:            BBB
+          Index:           0
+          SystemValue:     Undefined
+          CompType:        Float32
+          Register:        0
+          Mask:            7
+          ExclusiveMask:   0
+          MinPrecision:    Default
+        - Stream:          0
+          Name:            SV_TessFactor
+          Index:           1
+          SystemValue:     FinalQuadEdgeTessfactor
+          CompType:        Float32
+          Register:        1
+          Mask:            8
+          ExclusiveMask:   8
+          MinPrecision:    Default
+        - Stream:          0
+          Name:            BBB
+          Index:           1
+          SystemValue:     Undefined
+          CompType:        Float32
+          Register:        1
+          Mask:            7
+          ExclusiveMask:   0
+          MinPrecision:    Default
+        - Stream:          0
+          Name:            SV_TessFactor
+          Index:           2
+          SystemValue:     FinalQuadEdgeTessfactor
+          CompType:        Float32
+          Register:        2
+          Mask:            8
+          ExclusiveMask:   8
+          MinPrecision:    Default
+        - Stream:          0
+          Name:            BBB
+          Index:           2
+          SystemValue:     Undefined
+          CompType:        Float32
+          Register:        2
+          Mask:            7
+          ExclusiveMask:   0
+          MinPrecision:    Default
+        - Stream:          0
+          Name:            SV_TessFactor
+          Index:           3
+          SystemValue:     FinalQuadEdgeTessfactor
+          CompType:        Float32
+          Register:        3
+          Mask:            8
+          ExclusiveMask:   8
+          MinPrecision:    Default
+        - Stream:          0
+          Name:            SV_InsideTessFactor
+          Index:           0
+          SystemValue:     FinalQuadInsideTessfactor
+          CompType:        Float32
+          Register:        4
+          Mask:            8
+          ExclusiveMask:   0
+          MinPrecision:    Default
+        - Stream:          0
+          Name:            SV_InsideTessFactor
+          Index:           1
+          SystemValue:     FinalQuadInsideTessfactor
+          CompType:        Float32
+          Register:        5
+          Mask:            8
+          ExclusiveMask:   0
+          MinPrecision:    Default
+        - Stream:          0
+          Name:            AVeryLongStringThatWillForceWidening # This value forces name column to widen
+          Index:           0
+          SystemValue:     Undefined
+          CompType:        Float32
+          Register:        6
+          Mask:            15
+          ExclusiveMask:   4
+          MinPrecision:    Default
+...
+
+# CHECK:; Input signature:
+# CHECK-NEXT:;
+# CHECK-NEXT:; Name                       Index  Mask Register   SysValue  Format  Used
+# CHECK-NEXT:; ------------------------ ------- ----- -------- ---------- ------- -----
+# CHECK-NEXT:; AAA_HSFoo                4391238  xyz         0  Undefined Float32   y
+
+# CHECK:; Output signature:
+# CHECK-NEXT:;
+# CHECK-NEXT:; Name                     Index  Mask   Register   SysValue  Format  Used
+# CHECK-NEXT:; ------------------------ ----- ----- ---------- ---------- ------- -----
+# CHECK-NEXT:; SV_Position                  0  xyzw 2147483647   Position Float32
+
+# CHECK:; Patch Constant signature:
+# CHECK-NEXT:;
+# CHECK-NEXT:; Name                                 Index  Mask Register                  SysValue  Format  Used
+# CHECK-NEXT:; ------------------------------------ ----- ----- -------- ------------------------- ------- -----
+# CHECK-NEXT:; SV_TessFactor                            0     w        0   FinalQuadEdgeTessfactor Float32     w
+# CHECK-NEXT:; BBB                                      0  xyz         0                 Undefined Float32
+# CHECK-NEXT:; SV_TessFactor                            1     w        1   FinalQuadEdgeTessfactor Float32     w
+# CHECK-NEXT:; BBB                                      1  xyz         1                 Undefined Float32
+# CHECK-NEXT:; SV_TessFactor                            2     w        2   FinalQuadEdgeTessfactor Float32     w
+# CHECK-NEXT:; BBB                                      2  xyz         2                 Undefined Float32
+# CHECK-NEXT:; SV_TessFactor                            3     w        3   FinalQuadEdgeTessfactor Float32     w
+# CHECK-NEXT:; SV_InsideTessFactor                      0     w        4 FinalQuadInsideTessfactor Float32
+# CHECK-NEXT:; SV_InsideTessFactor                      1     w        5 FinalQuadInsideTessfactor Float32
+# CHECK-NEXT:; AVeryLongStringThatWillForceWidening     0  xyzw        6                 Undefined Float32    z
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx1250.s b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx1250.s
new file mode 100644
index 0000000000000..99a4df3e5adfb
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/kd-gfx1250.s
@@ -0,0 +1,121 @@
+;; Test disassembly for gfx1250 kernel descriptor.
+
+; RUN: rm -rf %t && split-file %s %t && cd %t
+
+;--- 1.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -filetype=obj -mcpu=gfx1250 < 1.s > 1.o
+; RUN: llvm-objdump --disassemble-symbols=kernel.kd 1.o | tail -n +7 | tee 1-disasm.s | FileCheck 1.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -filetype=obj -mcpu=gfx1250 < 1-disasm.s > 1-disasm.o
+; RUN: cmp 1.o 1-disasm.o
+; CHECK: .amdhsa_kernel kernel
+; CHECK-NEXT: .amdhsa_group_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_kernarg_size 0
+; CHECK-NEXT: .amdhsa_inst_pref_size 0
+; CHECK-NEXT: ; GLG_EN 0
+; CHECK-NEXT: .amdhsa_named_barrier_count 0
+; CHECK-NEXT: ; ENABLE_DYNAMIC_VGPR 0
+; CHECK-NEXT: ; TCP_SPLIT 0
+; CHECK-NEXT: ; ENABLE_DIDT_THROTTLE 0
+; CHECK-NEXT: ; IMAGE_OP 0
+; CHECK-NEXT: .amdhsa_next_free_vgpr 32
+; CHECK-NEXT: .amdhsa_reserve_vcc 0
+; CHECK-NEXT: .amdhsa_reserve_xnack_mask 0
+; CHECK-NEXT: .amdhsa_next_free_sgpr 8
+; CHECK-NEXT: .amdhsa_float_round_mode_32 0
+; CHECK-NEXT: .amdhsa_float_round_mode_16_64 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_32 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_16_64 3
+; CHECK-NEXT: .amdhsa_fp16_overflow 0
+; CHECK-NEXT: ; FLAT_SCRATCH_IS_NV 0
+; CHECK-NEXT: .amdhsa_memory_ordered 1
+; CHECK-NEXT: .amdhsa_forward_progress 1
+; CHECK-NEXT: .amdhsa_round_robin_scheduling 0
+; CHECK-NEXT: .amdhsa_enable_private_segment 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; CHECK-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0
+; CHECK-NEXT: .amdhsa_exception_fp_denorm_src 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_div_zero 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_overflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_underflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_inexact 0
+; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_uses_cu_stores 1
+; CHECK-NEXT: .amdhsa_wavefront_size32 1
+; CHECK-NEXT: .amdhsa_uses_dynamic_stack 0
+; CHECK-NEXT: .end_amdhsa_kernel
+.amdhsa_kernel kernel
+  .amdhsa_next_free_vgpr 32
+  .amdhsa_next_free_sgpr 32
+  .amdhsa_inst_pref_size 0
+  .amdhsa_uses_cu_stores 1
+.end_amdhsa_kernel
+
+;--- 2.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -filetype=obj -mcpu=gfx1250 < 2.s > 2.o
+; RUN: llvm-objdump --disassemble-symbols=kernel.kd 2.o | tail -n +7 | tee 2-disasm.s | FileCheck 2.s
+; RUN: llvm-mc --triple=amdgcn-amd-amdhsa -filetype=obj -mcpu=gfx1250 < 2-disasm.s > 2-disasm.o
+; RUN: cmp 2.o 2-disasm.o
+; CHECK: .amdhsa_kernel kernel
+; CHECK-NEXT: .amdhsa_group_segment_fixed_size 393216
+; CHECK-NEXT: .amdhsa_private_segment_fixed_size 0
+; CHECK-NEXT: .amdhsa_kernarg_size 0
+; CHECK-NEXT: .amdhsa_inst_pref_size 63
+; CHECK-NEXT: ; GLG_EN 0
+; CHECK-NEXT: .amdhsa_named_barrier_count 7
+; CHECK-NEXT: ; ENABLE_DYNAMIC_VGPR 0
+; CHECK-NEXT: ; TCP_SPLIT 0
+; CHECK-NEXT: ; ENABLE_DIDT_THROTTLE 0
+; CHECK-NEXT: ; IMAGE_OP 0
+; CHECK-NEXT: .amdhsa_next_free_vgpr 32
+; CHECK-NEXT: .amdhsa_reserve_vcc 0
+; CHECK-NEXT: .amdhsa_reserve_xnack_mask 0
+; CHECK-NEXT: .amdhsa_next_free_sgpr 8
+; CHECK-NEXT: .amdhsa_float_round_mode_32 0
+; CHECK-NEXT: .amdhsa_float_round_mode_16_64 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_32 0
+; CHECK-NEXT: .amdhsa_float_denorm_mode_16_64 3
+; CHECK-NEXT: .amdhsa_fp16_overflow 0
+; CHECK-NEXT: ; FLAT_SCRATCH_IS_NV 0
+; CHECK-NEXT: .amdhsa_memory_ordered 1
+; CHECK-NEXT: .amdhsa_forward_progress 1
+; CHECK-NEXT: .amdhsa_round_robin_scheduling 0
+; CHECK-NEXT: .amdhsa_enable_private_segment 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_x 1
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_y 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_id_z 0
+; CHECK-NEXT: .amdhsa_system_sgpr_workgroup_info 0
+; CHECK-NEXT: .amdhsa_system_vgpr_workitem_id 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_invalid_op 0
+; CHECK-NEXT: .amdhsa_exception_fp_denorm_src 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_div_zero 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_overflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_underflow 0
+; CHECK-NEXT: .amdhsa_exception_fp_ieee_inexact 0
+; CHECK-NEXT: .amdhsa_exception_int_div_zero 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_queue_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 0
+; CHECK-NEXT: .amdhsa_user_sgpr_dispatch_id 0
+; CHECK-NEXT: .amdhsa_user_sgpr_private_segment_size 0
+; CHECK-NEXT: .amdhsa_uses_cu_stores 0
+; CHECK-NEXT: .amdhsa_wavefront_size32 1
+; CHECK-NEXT: .amdhsa_uses_dynamic_stack 0
+; CHECK-NEXT: .end_amdhsa_kernel
+.amdhsa_kernel kernel
+  .amdhsa_group_segment_fixed_size 393216
+  .amdhsa_next_free_vgpr 32
+  .amdhsa_next_free_sgpr 32
+  .amdhsa_named_barrier_count 7
+  .amdhsa_uses_cu_stores 0
+  .amdhsa_inst_pref_size 63
+.end_amdhsa_kernel
diff --git a/llvm/test/tools/llvm-objdump/ELF/ARM/undefined-in-it.s b/llvm/test/tools/llvm-objdump/ELF/ARM/undefined-in-it.s
new file mode 100644
index 0000000000000..c5dc5cb248744
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/ELF/ARM/undefined-in-it.s
@@ -0,0 +1,45 @@
+@RUN: llvm-mc -triple arm-none-eabi -mcpu=cortex-m33 -filetype=obj %s | llvm-objdump -d --mcpu=cortex-m3 - | FileCheck %s
+
+@ Check that instructions that are disassembled as <undefined> within an IT
+@ block advance the IT state. This prevents the IT state spilling over into
+@ the next instruction.
+
+@ The vldmiaeq instruction is disassembled as <undefined> with
+@ -mcpu=cortex-m3 as this does not have a fpu.
+.text
+.fpu fp-armv8
+.thumb
+ ite eq
+ vldmiaeq r0!, {s16-s31}
+ addne    r0, r0, r0
+ add      r1, r1, r1
+
+ itet eq
+ vldmiaeq r0!, {s16-s31}
+ vldmiane r0!, {s16-s31}
+ vldmiaeq r0!, {s16-s31}
+ add      r0, r0, r0
+ add      r1, r1, r1
+ add      r2, r2, r2
+
+ it eq
+ vldmiaeq r0!, {s16-s31}
+
+ it ne
+ addne      r0, r0, r0
+
+@ CHECK:             0: bf0c          ite     eq
+@ CHECK-NEXT:        2: ecb0 8a10     <unknown>
+@ CHECK-NEXT:        6: 1800          addne   r0, r0, r0
+@ CHECK-NEXT:        8: 4409          add     r1, r1
+@ CHECK-NEXT:        a: bf0a          itet    eq
+@ CHECK-NEXT:        c: ecb0 8a10     <unknown>
+@ CHECK-NEXT:       10: ecb0 8a10     <unknown>
+@ CHECK-NEXT:       14: ecb0 8a10     <unknown>
+@ CHECK-NEXT:       18: 4400          add     r0, r0
+@ CHECK-NEXT:       1a: 4409          add     r1, r1
+@ CHECK-NEXT:       1c: 4412          add     r2, r2
+@ CHECK-NEXT:       1e: bf08          it      eq
+@ CHECK-NEXT:       20: ecb0 8a10     <unknown>
+@ CHECK-NEXT:       24: bf18          it      ne
+@ CHECK-NEXT:       26: 1800          addne   r0, r0, r0
diff --git a/llvm/test/tools/llvm-objdump/MachO/bad-trie.test b/llvm/test/tools/llvm-objdump/MachO/bad-trie.test
index 8b29d30ef0618..e4d0ed58744f9 100644
--- a/llvm/test/tools/llvm-objdump/MachO/bad-trie.test
+++ b/llvm/test/tools/llvm-objdump/MachO/bad-trie.test
@@ -11,7 +11,7 @@ RUN: not llvm-objdump --macho --exports-trie %p/Inputs/macho-trie-export-info-si
 EXPORT_INFO_SIZE_TOO_BIG: macho-trie-export-info-size-too-big': truncated or malformed object (export info size: 0x1234 in export trie data at node: 0x33 too big and extends past end of trie data)
 
 RUN: not llvm-objdump --macho --exports-trie %p/Inputs/macho-trie-children-count-byte 2>&1 | FileCheck --check-prefix CHILDREN_COUNT_BYTE %s 
-CHILDREN_COUNT_BYTE: macho-trie-children-count-byte': truncated or malformed object (byte for count of childern in export trie data at node: 0x5 extends past end of trie data)
+CHILDREN_COUNT_BYTE: macho-trie-children-count-byte': truncated or malformed object (byte for count of children in export trie data at node: 0x5 extends past end of trie data)
 
 RUN: not llvm-objdump --macho --exports-trie %p/Inputs/macho-trie-import-name-start 2>&1 | FileCheck --check-prefix IMPORT_NAME_START %s 
 IMPORT_NAME_START: macho-trie-import-name-start': truncated or malformed object (import name of re-export in export trie data at node: 0x33 starts past end of trie data)
@@ -25,8 +25,8 @@ EDGE_STRING_END: macho-trie-edge-string-end': truncated or malformed object (edg
 RUN: not llvm-objdump --macho --exports-trie %p/Inputs/macho-trie-not-export-node 2>&1 | FileCheck --check-prefix NOT_EXPORT_NODE %s 
 NOT_EXPORT_NODE: macho-trie-not-export-node': truncated or malformed object (node is not an export node in export trie data at node: 0x5a)
 
-RUN: not llvm-objdump --macho --exports-trie %p/Inputs/macho-trie-node-loop 2>&1 | FileCheck --check-prefix LOOP_OF_CHILDERN %s 
-LOOP_OF_CHILDERN: macho-trie-node-loop': truncated or malformed object (loop in childern in export trie data at node: 0x42 back to node: 0x5)
+RUN: not llvm-objdump --macho --exports-trie %p/Inputs/macho-trie-node-loop 2>&1 | FileCheck --check-prefix LOOP_OF_CHILDREN %s
+LOOP_OF_CHILDREN: macho-trie-node-loop': truncated or malformed object (loop in children in export trie data at node: 0x42 back to node: 0x5)
 
 RUN: not llvm-objdump --macho --exports-trie %p/Inputs/macho-trie-bad-library-ordinal 2>&1 | FileCheck --check-prefix BAD_LIBRARY_ORDINAL %s 
 BAD_LIBRARY_ORDINAL: macho-trie-bad-library-ordinal': truncated or malformed object (bad library ordinal: 69 (max 3) in export trie data at node: 0x33)
diff --git a/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin-reduced.html b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin-reduced.html
new file mode 100644
index 0000000000000..c419744905fe1
--- /dev/null
+++ b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin-reduced.html
@@ -0,0 +1,124 @@
+ <html>
+  <head>
+  <style>
+  table, th, td {
+    border: 1px solid black;
+  }
+  table.center {
+    margin-left: auto;
+    margin-right: auto;
+  }
+  </style>
+  </head>
+  <body>
+  <table>
+  <caption><b>Location Bugs found by the Debugify</b></caption>
+  <tr>
+      <th>File</th>
+    <th>LLVM Pass Name</th>
+    <th>LLVM IR Instruction</th>
+    <th>Function Name</th>
+    <th>Basic Block Name</th>
+    <th>Action</th>
+    <th>Origin</th>
+  </tr>
+  </tr>
+    <tr>
+    <td>test.ll</td>
+    <td>LoopVectorizePass</td>
+    <td>add</td>
+    <td>fn</td>
+    <td>no-name</td>
+    <td>not-generate</td>
+    <td><details><summary>View Origin StackTrace</summary><pre>Stack Trace 0:
+ #0 0x00005895d035c935 llvm::DbgLocOrigin::DbgLocOrigin(bool) /tmp/llvm-project/llvm/lib/IR/DebugLoc.cpp:22:9
+ #1 0x00005895d03af013 llvm::DILocAndCoverageTracking::DILocAndCoverageTracking() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:90:11
+ #2 0x00005895d03af013 llvm::DebugLoc::DebugLoc() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:133:5
+ #3 0x00005895d03af013 llvm::Instruction::Instruction(llvm::Type*, unsigned int, llvm::User::AllocInfo, llvm::InsertPosition) /tmp/llvm-project/llvm/lib/IR/Instruction.cpp:37:14
+ #4 0x00005895d06862b5 llvm::PHINode::PHINode(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:0:9
+ #5 0x00005895d06862b5 llvm::PHINode::Create(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:2651:9
+ #6 0x00005895d06862b5 llvm::InstCombinerImpl::foldPHIArgGEPIntoPHI(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:617:9
+ #7 0x00005895d0688fe0 llvm::InstCombinerImpl::visitPHINode(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:1456:22
+ #8 0x00005895d05cd21f llvm::InstCombinerImpl::run() /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5327:22
+ #9 0x00005895d05d067e combineInstructionsOverFunction(llvm::Function&, llvm::InstructionWorklist&, llvm::AAResults*, llvm::AssumptionCache&, llvm::TargetLibraryInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::OptimizationRemarkEmitter&, llvm::BlockFrequencyInfo*, llvm::BranchProbabilityInfo*, llvm::ProfileSummaryInfo*, llvm::InstCombineOptions const&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5643:31
+#10 0x00005895d05cf9a9 llvm::InstCombinePass::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5706:8
+#11 0x00005895d107d07d llvm::detail::PassModel>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#12 0x00005895d04204a7 llvm::PassManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h:85:8
+#13 0x00005895ce4cb09d llvm::detail::PassModel>, llvm::AnalysisManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#14 0x00005895cfae2865 llvm::CGSCCToFunctionPassAdaptor::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38
+#15 0x00005895ce4cad5d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#16 0x00005895cfade813 llvm::PassManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:93:12
+#17 0x00005895d1e3968d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>, llvm::AnalysisManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#18 0x00005895cfae1224 llvm::DevirtSCCRepeatedPass::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38
+#19 0x00005895d1e5067d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5</pre></details></td>
+    </tr>
+  <tr>
+</table>
+<br>
+<table>
+  <caption><b>Summary of Location Bugs</b></caption>
+  <tr>
+      <th>LLVM Pass Name</th>
+    <th>Number of bugs</th>
+  </tr>
+    <tr>
+    <td>LoopVectorizePass</td>
+    <td>2</td>
+    </tr>
+  <tr>
+</table>
+<br>
+<br>
+<table>
+  <caption><b>SP Bugs found by the Debugify</b></caption>
+  <tr>
+      <th>File</th>
+    <th>LLVM Pass Name</th>
+    <th>Function Name</th>
+    <th>Action</th>
+  </tr>
+<tr>
+        <td colspan='4'> No bugs found </td>
+      </tr>
+    </table>
+<br>
+<table>
+  <caption><b>Summary of SP Bugs</b></caption>
+  <tr>
+      <th>LLVM Pass Name</th>
+    <th>Number of bugs</th>
+  </tr>
+  <tr>
+<tr>
+        <td colspan='2'> No bugs found </td>
+      </tr>
+    </table>
+<br>
+<br>
+<table>
+  <caption><b>Variable Location Bugs found by the Debugify</b></caption>
+  <tr>
+      <th>File</th>
+    <th>LLVM Pass Name</th>
+    <th>Variable</th>
+    <th>Function</th>
+    <th>Action</th>
+  </tr>
+<tr>
+        <td colspan='4'> No bugs found </td>
+      </tr>
+    </table>
+<br>
+<table>
+  <caption><b>Summary of Variable Location Bugs</b></caption>
+  <tr>
+      <th>LLVM Pass Name</th>
+    <th>Number of bugs</th>
+  </tr>
+  <tr>
+<tr>
+        <td colspan='2'> No bugs found </td>
+      </tr>
+    </table>
+</body>
+  </html>
\ No newline at end of file
diff --git a/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html
index ed08c850cb4ad..4de4642edbe8b 100644
--- a/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html
+++ b/llvm/test/tools/llvm-original-di-preservation/Inputs/expected-origin.html
@@ -53,6 +53,37 @@
 #19 0x00005895d1e5067d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5</pre></details></td>
     </tr>
   <tr>
+  </tr>
+    <tr>
+    <td>test2.ll</td>
+    <td>LoopVectorizePass</td>
+    <td>add</td>
+    <td>foo</td>
+    <td>no-name</td>
+    <td>not-generate</td>
+    <td><details><summary>View Origin StackTrace</summary><pre>Stack Trace 0:
+ #0 0x000046afd035c935 llvm::DbgLocOrigin::DbgLocOrigin(bool) /tmp/llvm-project/llvm/lib/IR/DebugLoc.cpp:22:9
+ #1 0x000046afd03af013 llvm::DILocAndCoverageTracking::DILocAndCoverageTracking() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:90:11
+ #2 0x000046afd03af013 llvm::DebugLoc::DebugLoc() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:133:5
+ #3 0x000046afd03af013 llvm::Instruction::Instruction(llvm::Type*, unsigned int, llvm::User::AllocInfo, llvm::InsertPosition) /tmp/llvm-project/llvm/lib/IR/Instruction.cpp:37:14
+ #4 0x000046afd06862b5 llvm::PHINode::PHINode(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:0:9
+ #5 0x000046afd06862b5 llvm::PHINode::Create(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:2651:9
+ #6 0x000046afd06862b5 llvm::InstCombinerImpl::foldPHIArgGEPIntoPHI(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:617:9
+ #7 0x000046afd0688fe0 llvm::InstCombinerImpl::visitPHINode(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:1456:22
+ #8 0x000046afd05cd21f llvm::InstCombinerImpl::run() /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5327:22
+ #9 0x000046afd05d067e combineInstructionsOverFunction(llvm::Function&, llvm::InstructionWorklist&, llvm::AAResults*, llvm::AssumptionCache&, llvm::TargetLibraryInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::OptimizationRemarkEmitter&, llvm::BlockFrequencyInfo*, llvm::BranchProbabilityInfo*, llvm::ProfileSummaryInfo*, llvm::InstCombineOptions const&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5643:31
+#10 0x000046afd05cf9a9 llvm::InstCombinePass::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5706:8
+#11 0x000046afd107d07d llvm::detail::PassModel>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#12 0x000046afd04204a7 llvm::PassManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h:85:8
+#13 0x000046afce4cb09d llvm::detail::PassModel>, llvm::AnalysisManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#14 0x000046afcfae2865 llvm::CGSCCToFunctionPassAdaptor::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38
+#15 0x000046afce4cad5d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#16 0x000046afcfade813 llvm::PassManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:93:12
+#17 0x000046afd1e3968d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>, llvm::AnalysisManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5
+#18 0x000046afcfae1224 llvm::DevirtSCCRepeatedPass::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38
+#19 0x000046afd1e5067d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5</pre></details></td>
+    </tr>
+  <tr>
 </table>
 <br>
 <table>
@@ -63,7 +94,7 @@
   </tr>
     <tr>
     <td>LoopVectorizePass</td>
-    <td>1</td>
+    <td>2</td>
     </tr>
   <tr>
 </table>
diff --git a/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json b/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json
index ab59bc295add1..5a06a1feb9854 100644
--- a/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json
+++ b/llvm/test/tools/llvm-original-di-preservation/Inputs/origin.json
@@ -1 +1,2 @@
 {"file":"test.ll", "pass":"LoopVectorizePass", "bugs": [[{"action":"not-generate","bb-name":"no-name","fn-name":"fn","instr":"add","metadata":"DILocation", "origin": "Stack Trace 0:\n #0 0x00005895d035c935 llvm::DbgLocOrigin::DbgLocOrigin(bool) /tmp/llvm-project/llvm/lib/IR/DebugLoc.cpp:22:9\n #1 0x00005895d03af013 llvm::DILocAndCoverageTracking::DILocAndCoverageTracking() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:90:11\n #2 0x00005895d03af013 llvm::DebugLoc::DebugLoc() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:133:5\n #3 0x00005895d03af013 llvm::Instruction::Instruction(llvm::Type*, unsigned int, llvm::User::AllocInfo, llvm::InsertPosition) /tmp/llvm-project/llvm/lib/IR/Instruction.cpp:37:14\n #4 0x00005895d06862b5 llvm::PHINode::PHINode(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:0:9\n #5 0x00005895d06862b5 llvm::PHINode::Create(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:2651:9\n #6 0x00005895d06862b5 llvm::InstCombinerImpl::foldPHIArgGEPIntoPHI(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:617:9\n #7 0x00005895d0688fe0 llvm::InstCombinerImpl::visitPHINode(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:1456:22\n #8 0x00005895d05cd21f llvm::InstCombinerImpl::run() /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5327:22\n #9 0x00005895d05d067e combineInstructionsOverFunction(llvm::Function&, llvm::InstructionWorklist&, llvm::AAResults*, llvm::AssumptionCache&, llvm::TargetLibraryInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::OptimizationRemarkEmitter&, llvm::BlockFrequencyInfo*, llvm::BranchProbabilityInfo*, llvm::ProfileSummaryInfo*, llvm::InstCombineOptions const&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5643:31\n#10 0x00005895d05cf9a9 llvm::InstCombinePass::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5706:8\n#11 0x00005895d107d07d llvm::detail::PassModel>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#12 0x00005895d04204a7 llvm::PassManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h:85:8\n#13 0x00005895ce4cb09d llvm::detail::PassModel>, llvm::AnalysisManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#14 0x00005895cfae2865 llvm::CGSCCToFunctionPassAdaptor::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38\n#15 0x00005895ce4cad5d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#16 0x00005895cfade813 llvm::PassManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:93:12\n#17 0x00005895d1e3968d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>, llvm::AnalysisManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#18 0x00005895cfae1224 llvm::DevirtSCCRepeatedPass::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38\n#19 0x00005895d1e5067d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5"}]]}
+{"file":"test2.ll", "pass":"LoopVectorizePass", "bugs": [[{"action":"not-generate","bb-name":"no-name","fn-name":"foo","instr":"add","metadata":"DILocation", "origin": "Stack Trace 0:\n #0 0x000046afd035c935 llvm::DbgLocOrigin::DbgLocOrigin(bool) /tmp/llvm-project/llvm/lib/IR/DebugLoc.cpp:22:9\n #1 0x000046afd03af013 llvm::DILocAndCoverageTracking::DILocAndCoverageTracking() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:90:11\n #2 0x000046afd03af013 llvm::DebugLoc::DebugLoc() /tmp/llvm-project/llvm/include/llvm/IR/DebugLoc.h:133:5\n #3 0x000046afd03af013 llvm::Instruction::Instruction(llvm::Type*, unsigned int, llvm::User::AllocInfo, llvm::InsertPosition) /tmp/llvm-project/llvm/lib/IR/Instruction.cpp:37:14\n #4 0x000046afd06862b5 llvm::PHINode::PHINode(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:0:9\n #5 0x000046afd06862b5 llvm::PHINode::Create(llvm::Type*, unsigned int, llvm::Twine const&, llvm::InsertPosition) /tmp/llvm-project/llvm/include/llvm/IR/Instructions.h:2651:9\n #6 0x000046afd06862b5 llvm::InstCombinerImpl::foldPHIArgGEPIntoPHI(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:617:9\n #7 0x000046afd0688fe0 llvm::InstCombinerImpl::visitPHINode(llvm::PHINode&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp:1456:22\n #8 0x000046afd05cd21f llvm::InstCombinerImpl::run() /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5327:22\n #9 0x000046afd05d067e combineInstructionsOverFunction(llvm::Function&, llvm::InstructionWorklist&, llvm::AAResults*, llvm::AssumptionCache&, llvm::TargetLibraryInfo&, llvm::TargetTransformInfo&, llvm::DominatorTree&, llvm::OptimizationRemarkEmitter&, llvm::BlockFrequencyInfo*, llvm::BranchProbabilityInfo*, llvm::ProfileSummaryInfo*, llvm::InstCombineOptions const&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5643:31\n#10 0x000046afd05cf9a9 llvm::InstCombinePass::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp:5706:8\n#11 0x000046afd107d07d llvm::detail::PassModel>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#12 0x000046afd04204a7 llvm::PassManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerImpl.h:85:8\n#13 0x000046afce4cb09d llvm::detail::PassModel>, llvm::AnalysisManager>::run(llvm::Function&, llvm::AnalysisManager&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#14 0x000046afcfae2865 llvm::CGSCCToFunctionPassAdaptor::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38\n#15 0x000046afce4cad5d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#16 0x000046afcfade813 llvm::PassManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:93:12\n#17 0x000046afd1e3968d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>, llvm::AnalysisManager, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5\n#18 0x000046afcfae1224 llvm::DevirtSCCRepeatedPass::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/lib/Analysis/CGSCCPassManager.cpp:0:38\n#19 0x000046afd1e5067d llvm::detail::PassModel, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&>::run(llvm::LazyCallGraph::SCC&, llvm::AnalysisManager&, llvm::LazyCallGraph&, llvm::CGSCCUpdateResult&) /tmp/llvm-project/llvm/include/llvm/IR/PassManagerInternal.h:91:5"}]]}
diff --git a/llvm/test/tools/llvm-original-di-preservation/basic.test b/llvm/test/tools/llvm-original-di-preservation/basic.test
index df43fbb3b5b9f..5bd7e1378b870 100644
--- a/llvm/test/tools/llvm-original-di-preservation/basic.test
+++ b/llvm/test/tools/llvm-original-di-preservation/basic.test
@@ -15,3 +15,6 @@ REDUCE-NOT: Skipped lines:
 
 RUN: %llvm-original-di-preservation %p/Inputs/origin.json --report-html-file %t4.html | FileCheck %s
 RUN: diff -w %p/Inputs/expected-origin.html %t4.html
+
+RUN: %llvm-original-di-preservation --reduce %p/Inputs/origin.json --report-html-file %t5.html | FileCheck %s
+RUN: diff -w %p/Inputs/expected-origin-reduced.html %t5.html
diff --git a/llvm/test/tools/llvm-profdata/merge-traces.proftext b/llvm/test/tools/llvm-profdata/merge-traces.proftext
index bcf29ba634ea6..3512f33cd06a9 100644
--- a/llvm/test/tools/llvm-profdata/merge-traces.proftext
+++ b/llvm/test/tools/llvm-profdata/merge-traces.proftext
@@ -1,24 +1,36 @@
-# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s -o %t.profdata
-# RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefixes=SAMPLE1,SEEN1
-# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %t.profdata -o %t.profdata
-# RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefixes=SAMPLE2,SEEN2
-# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %t.profdata -o %t.profdata
-# RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefixes=SAMPLE2,SEEN3
-# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %t.profdata -o %t.profdata
-# RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefixes=SAMPLE2,SEEN4
-
-# SEEN1: Temporal Profile Traces (samples=1 seen=1):
-# SEEN2: Temporal Profile Traces (samples=2 seen=2):
-# SEEN3: Temporal Profile Traces (samples=2 seen=3):
-# SEEN4: Temporal Profile Traces (samples=2 seen=4):
-# SAMPLE1: Temporal Profile Trace 0 (weight=1 count=3):
-# SAMPLE1:   a
-# SAMPLE1:   b
-# SAMPLE1:   c
-# SAMPLE2: Temporal Profile Trace 1 (weight=1 count=3):
-# SAMPLE2:   a
-# SAMPLE2:   b
-# SAMPLE2:   c
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s --text | FileCheck %s --check-prefixes=CHECK,SEEN1,SAMPLE1
+
+# Merge %s twice so it has two traces
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %s --text | FileCheck %s --check-prefixes=CHECK,SEEN2,SAMPLE2
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %s -o %t-2.profdata
+
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %s %s --text | FileCheck %s --check-prefixes=CHECK,SEEN3,SAMPLE2
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %t-2.profdata %s --text | FileCheck %s --check-prefixes=CHECK,SEEN3,SAMPLE2
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %t-2.profdata --text | FileCheck %s --check-prefixes=CHECK,SEEN3,SAMPLE2
+
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %s %s %s --text | FileCheck %s --check-prefixes=CHECK,SEEN4,SAMPLE2
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %t-2.profdata %s %s --text | FileCheck %s --check-prefixes=CHECK,SEEN4,SAMPLE2
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %t-2.profdata %t-2.profdata --text | FileCheck %s --check-prefixes=CHECK,SEEN4,SAMPLE2
+
+# Test that we can increase the reservoir size, even if inputs are sampled
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=2 %s %s %s %s -o %t-4.profdata
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=4 %t-4.profdata %t-4.profdata --text | FileCheck %s --check-prefixes=CHECK,SEEN8,SAMPLE4
+
+# Test that decreasing the reservoir size truncates traces
+# RUN: llvm-profdata merge --temporal-profile-trace-reservoir-size=1 %t-4.profdata --text | FileCheck %s --check-prefixes=CHECK,SEEN4,SAMPLE1
+
+# CHECK: :temporal_prof_traces
+# CHECK: # Num Temporal Profile Traces:
+# SAMPLE1: 1
+# SAMPLE2: 2
+# SAMPLE4: 4
+# CHECK: # Temporal Profile Trace Stream Size:
+# SEEN1: 1
+# SEEN2: 2
+# SEEN3: 3
+# SEEN4: 4
+# SEEN8: 8
+# CHECK: a,b,c,
 
 # Header
 :ir
diff --git a/llvm/test/tools/llvm-profdata/overlap.test b/llvm/test/tools/llvm-profdata/overlap.test
index 91379d5722e6e..6470e3f19712b 100644
--- a/llvm/test/tools/llvm-profdata/overlap.test
+++ b/llvm/test/tools/llvm-profdata/overlap.test
@@ -9,7 +9,7 @@ MAINFUNC:   # of edge counters overlap: 2
 MAINFUNC:   Edge profile overlap: 100.000%
 MAINFUNC:   Edge profile base count sum: 200000
 MAINFUNC:   Edge profile test count sum: 20000
-OVERLAP: Profile overlap infomation for base_profile: {{.*}} and test_profile:
+OVERLAP: Profile overlap information for base_profile: {{.*}} and test_profile:
 OVERLAP: Program level:
 OVERLAP:   # of functions overlap: 2
 OVERLAP:   # of functions mismatch: 1
diff --git a/llvm/test/tools/llvm-profdata/overlap_cs.test b/llvm/test/tools/llvm-profdata/overlap_cs.test
index 3f2086a27c770..ba15418f87da8 100644
--- a/llvm/test/tools/llvm-profdata/overlap_cs.test
+++ b/llvm/test/tools/llvm-profdata/overlap_cs.test
@@ -2,7 +2,7 @@ RUN: llvm-profdata overlap -cs %p/Inputs/overlap_1_cs.proftext %p/Inputs/overlap
 RUN: llvm-profdata merge %p/Inputs/overlap_1_cs.proftext -o %t_1_cs.profdata
 RUN: llvm-profdata merge %p/Inputs/overlap_2_cs.proftext -o %t_2_cs.profdata
 RUN: llvm-profdata overlap -cs %t_1_cs.profdata %t_2_cs.profdata | FileCheck %s -check-prefix=OVERLAP
-OVERLAP: Profile overlap infomation for base_profile: {{.*}} and test_profile:
+OVERLAP: Profile overlap information for base_profile: {{.*}} and test_profile:
 OVERLAP: Program level:
 OVERLAP:  # of functions overlap: 1
 OVERLAP:  Edge profile overlap: 80.000%
diff --git a/llvm/test/tools/llvm-profdata/overlap_vp.test b/llvm/test/tools/llvm-profdata/overlap_vp.test
index 228b809e218e4..4425696ab419c 100644
--- a/llvm/test/tools/llvm-profdata/overlap_vp.test
+++ b/llvm/test/tools/llvm-profdata/overlap_vp.test
@@ -2,7 +2,7 @@ RUN: llvm-profdata overlap %p/Inputs/overlap_1_vp.proftext %p/Inputs/overlap_2_v
 RUN: llvm-profdata merge %p/Inputs/overlap_1_vp.proftext -o %t_1_vp.profdata
 RUN: llvm-profdata merge %p/Inputs/overlap_2_vp.proftext -o %t_2_vp.profdata
 RUN: llvm-profdata overlap %t_1_vp.profdata %t_2_vp.profdata | FileCheck %s -check-prefix=OVERLAP
-OVERLAP: Profile overlap infomation for base_profile: {{.*}} and test_profile:
+OVERLAP: Profile overlap information for base_profile: {{.*}} and test_profile:
 OVERLAP: Program level:
 OVERLAP:   # of functions overlap: 1
 OVERLAP:   Edge profile overlap: 80.000%
diff --git a/llvm/test/tools/llvm-profdata/read-traces.proftext b/llvm/test/tools/llvm-profdata/read-traces.proftext
index 87f69fe0d7610..5e822a9ea53ec 100644
--- a/llvm/test/tools/llvm-profdata/read-traces.proftext
+++ b/llvm/test/tools/llvm-profdata/read-traces.proftext
@@ -3,19 +3,16 @@
 # RUN: llvm-profdata merge -text %t.2.profdata -o %t.3.proftext
 # RUN: diff %t.1.proftext %t.3.proftext
 
-# RUN: llvm-profdata show --temporal-profile-traces %t.1.proftext | FileCheck %s
+# RUN: llvm-profdata merge -text %s | FileCheck %s
 
-# CHECK: Temporal Profile Traces (samples=3 seen=3):
-# CHECK: Temporal Profile Trace 0 (weight=1 count=3):
-# CHECK:   foo
-# CHECK:   bar
-# CHECK:   goo
-# CHECK: Temporal Profile Trace 1 (weight=3 count=3):
-# CHECK:   foo
-# CHECK:   goo
-# CHECK:   bar
-# CHECK: Temporal Profile Trace 2 (weight=1 count=1):
-# CHECK:   goo
+# CHECK:      :temporal_prof_traces
+# CHECK:      # Num Temporal Profile Traces:
+# CHECK-NEXT: 3
+# CHECK:      # Temporal Profile Trace Stream Size:
+# CHECK-NEXT: 3
+# CHECK-DAG:  foo,bar,goo,
+# CHECK-DAG:  foo,goo,bar,
+# CHECK-DAG:  goo,
 
 # Header
 :ir
diff --git a/llvm/test/tools/llvm-profdata/trace-limit.proftext b/llvm/test/tools/llvm-profdata/trace-limit.proftext
index e246ee890ba31..6b4f974add169 100644
--- a/llvm/test/tools/llvm-profdata/trace-limit.proftext
+++ b/llvm/test/tools/llvm-profdata/trace-limit.proftext
@@ -11,7 +11,7 @@
 # RUN: llvm-profdata merge --temporal-profile-max-trace-length=1000 %s -o %t.profdata
 # RUN: llvm-profdata show --temporal-profile-traces %t.profdata | FileCheck %s --check-prefixes=CHECK,ALL
 
-# NONE: Temporal Profile Traces (samples=0
+# NONE: Temporal Profile Traces (samples=0 seen=0):
 # CHECK: Temporal Profile Traces (samples=1 seen=1):
 # SOME:   Trace 0 (weight=1 count=2):
 # ALL:    Trace 0 (weight=1 count=3):
diff --git a/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test b/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test
index d7fe77cf658af..d97cb70f7591e 100644
--- a/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test
+++ b/llvm/test/tools/llvm-readobj/ELF/reloc-types-aarch64.test
@@ -57,6 +57,7 @@
 # CHECK: Type: R_AARCH64_LD64_GOT_LO12_NC (312)
 # CHECK: Type: R_AARCH64_LD64_GOTPAGE_LO15 (313)
 # CHECK: Type: R_AARCH64_PLT32 (314)
+# CHECK: Type: R_AARCH64_PATCHINST (316)
 # CHECK: Type: R_AARCH64_TLSGD_ADR_PREL21 (512)
 # CHECK: Type: R_AARCH64_TLSGD_ADR_PAGE21 (513)
 # CHECK: Type: R_AARCH64_TLSGD_ADD_LO12_NC (514)
@@ -214,6 +215,7 @@ Sections:
       - Type: R_AARCH64_LD64_GOT_LO12_NC
       - Type: R_AARCH64_LD64_GOTPAGE_LO15
       - Type: R_AARCH64_PLT32
+      - Type: R_AARCH64_PATCHINST
       - Type: R_AARCH64_TLSGD_ADR_PREL21
       - Type: R_AARCH64_TLSGD_ADR_PAGE21
       - Type: R_AARCH64_TLSGD_ADD_LO12_NC
diff --git a/llvm/test/tools/llvm-readobj/ELF/sframe-fde.test b/llvm/test/tools/llvm-readobj/ELF/sframe-fde.test
index dee40180c42e6..b9075a81eba4b 100644
--- a/llvm/test/tools/llvm-readobj/ELF/sframe-fde.test
+++ b/llvm/test/tools/llvm-readobj/ELF/sframe-fde.test
@@ -108,6 +108,8 @@ Sections:
 #  CASE1-NEXT:      }
 #  CASE1-NEXT:      Repetitive block size: 0xDE
 #  CASE1-NEXT:      Padding2: 0xAD
+#  CASE1-NEXT:      FREs [
+#  CASE1-NEXT:      ]
 #  CASE1-NEXT:    }
 #  CASE1-NEXT:  ]
 #  CASE1-NEXT:}
@@ -169,6 +171,8 @@ Sections:
 #  CASE1-NEXT:      }
 #  CASE1-NEXT:      Repetitive block size (unused): 0xDE
 #  CASE1-NEXT:      Padding2: 0xAD
+#  CASE1-NEXT:      FREs [
+#  CASE1-NEXT:      ]
 #  CASE1-NEXT:    }
 #  CASE1-NEXT:  ]
 #  CASE1-NEXT:}
@@ -196,7 +200,7 @@ Sections:
       0x00, 0xde, 0xad, 0x00,  # Start Address
       0x00, 0x00, 0x01, 0xbe,  # Size
       0x00, 0x00, 0x00, 0x10,  # Start FRE Offset
-      0x00, 0x00, 0x00, 0x10,  # Number of FREs
+      0x00, 0x00, 0x00, 0x00,  # Number of FREs
       0x02, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
     ]
 # CASE2-LABEL:SFrame section '.sframe' {
@@ -223,7 +227,7 @@ Sections:
 #  CASE2-NEXT:      PC: 0xDEAD1C
 #  CASE2-NEXT:      Size: 0x1BE
 #  CASE2-NEXT:      Start FRE Offset: 0x10
-#  CASE2-NEXT:      Num FREs: 16
+#  CASE2-NEXT:      Num FREs: 0
 #  CASE2-NEXT:      Info {
 #  CASE2-NEXT:        FRE Type: Addr4 (0x2)
 #  CASE2-NEXT:        FDE Type: PCInc (0x0)
@@ -232,6 +236,8 @@ Sections:
 #  CASE2-NEXT:      }
 #  CASE2-NEXT:      Repetitive block size (unused): 0xDE
 #  CASE2-NEXT:      Padding2: 0xAD00
+#  CASE2-NEXT:      FREs [
+#  CASE2-NEXT:      ]
 #  CASE2-NEXT:    }
 #  CASE2-NEXT:  ]
 #  CASE2-NEXT:}
diff --git a/llvm/test/tools/llvm-readobj/ELF/sframe-fre.test b/llvm/test/tools/llvm-readobj/ELF/sframe-fre.test
new file mode 100644
index 0000000000000..3f1e7d667f47e
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/sframe-fre.test
@@ -0,0 +1,303 @@
+# RUN: yaml2obj --docnum=1 %s -o %t.1
+# RUN: llvm-readobj --sframe=.sframe_eof_address --sframe=.sframe_eof_offset --sframe \
+# RUN:   %t.1 2>&1 | \
+# RUN:   FileCheck %s --strict-whitespace --match-full-lines \
+# RUN:   -DFILE=%t.1 --check-prefix=CASE1
+
+# RUN: yaml2obj --docnum=2 %s -o %t.2
+# RUN: llvm-readobj --sframe %t.2 2>&1 | \
+# RUN:   FileCheck %s --strict-whitespace --match-full-lines \
+# RUN:   -DFILE=%t.2 --check-prefix=CASE2
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+Sections:
+  - Name:  .sframe_eof_address
+    Type:  SHT_GNU_SFRAME
+    Flags: [ SHF_ALLOC ]
+    ContentArray: [
+      0xe2, 0xde, 0x02, 0x04,  # Preamble (magic, version, flags)
+      # Header:
+      0x03, 0x42, 0x47, 0x00,  # ABI, Fixed FP offset, Fixed RA Offset, AUX header length
+      0x01, 0x00, 0x00, 0x00,  # Number of FDEs
+      0x01, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0x10, 0x00, 0x00,  # FRE length
+      0x00, 0x00, 0x00, 0x00,  # FDE offset
+      0x00, 0x00, 0x00, 0x00,  # FRE offset
+
+      # FDE[0]:
+      0x00, 0xde, 0xad, 0x00,  # Start Address
+      0xbe, 0x01, 0x00, 0x00,  # Size
+      0x00, 0x00, 0x00, 0x00,  # Start FRE Offset
+      0x01, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
+
+      # FRE[0]:
+      0x00                     # Start Address, (missing) Info
+    ]
+# CASE1-LABEL:SFrame section '.sframe_eof_address' {
+#       CASE1:    FuncDescEntry [0] {
+#       CASE1:      Info {
+#  CASE1-NEXT:        FRE Type: Addr1 (0x0)
+#  CASE1-NEXT:        FDE Type: PCInc (0x0)
+#       CASE1:      FREs [
+#  CASE1-NEXT:{{.*}}: warning: '[[FILE]]': unexpected end of data at offset 0x31 while reading [0x30, 0x32)
+#  CASE1-NEXT:      ]
+
+  - Name:  .sframe_eof_offset
+    Type:  SHT_GNU_SFRAME
+    Flags: [ SHF_ALLOC ]
+    ContentArray: [
+      0xe2, 0xde, 0x02, 0x04,  # Preamble (magic, version, flags)
+      # Header:
+      0x03, 0x42, 0x47, 0x00,  # ABI, Fixed FP offset, Fixed RA Offset, AUX header length
+      0x01, 0x00, 0x00, 0x00,  # Number of FDEs
+      0x01, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0x10, 0x00, 0x00,  # FRE length
+      0x00, 0x00, 0x00, 0x00,  # FDE offset
+      0x00, 0x00, 0x00, 0x00,  # FRE offset
+
+      # FDE[0]:
+      0x00, 0xde, 0xad, 0x00,  # Start Address
+      0xbe, 0x01, 0x00, 0x00,  # Size
+      0x00, 0x00, 0x00, 0x00,  # Start FRE Offset
+      0x01, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
+
+      # FRE[0]:
+      0x00, 0x02               # Start Address, Info, (missing) Offsets
+    ]
+# CASE1-LABEL:SFrame section '.sframe_eof_offset' {
+#       CASE1:    FuncDescEntry [0] {
+#       CASE1:      Info {
+#  CASE1-NEXT:        FRE Type: Addr1 (0x0)
+#  CASE1-NEXT:        FDE Type: PCInc (0x0)
+#       CASE1:      FREs [
+#  CASE1-NEXT:{{.*}}: warning: '[[FILE]]': unexpected end of data at offset 0x32 while reading [0x32, 0x33)
+#  CASE1-NEXT:      ]
+
+  - Name:  .sframe
+    Type:  SHT_GNU_SFRAME
+    Flags: [ SHF_ALLOC ]
+    ContentArray: [
+      0xe2, 0xde, 0x02, 0x04,  # Preamble (magic, version, flags)
+      # Header:
+      0x03, 0x42, 0x40, 0x00,  # ABI, Fixed FP offset, Fixed RA Offset, AUX header length
+      0x03, 0x00, 0x00, 0x00,  # Number of FDEs
+      0x01, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0x10, 0x00, 0x00,  # FRE length
+      0x00, 0x00, 0x00, 0x00,  # FDE offset
+      0x00, 0x00, 0x00, 0x00,  # FRE offset
+
+      # FDE[0]:
+      0x00, 0x00, 0xde, 0x00,  # Start Address
+      0xbe, 0x01, 0x00, 0x00,  # Size
+      0x00, 0x00, 0x00, 0x00,  # Start FRE Offset
+      0x02, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
+
+      # FDE[1]:
+      0x00, 0x00, 0xad, 0x00,  # Start Address
+      0xbe, 0x01, 0x00, 0x00,  # Size
+      0x08, 0x00, 0x00, 0x00,  # Start FRE Offset
+      0x03, 0x00, 0x00, 0x00,  # Number of FREs
+      0x11, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
+
+      # FDE[2]:
+      0x00, 0x00, 0xbe, 0x00,  # Start Address
+      0xbe, 0x01, 0x00, 0x00,  # Size
+      0x08, 0x00, 0x00, 0x00,  # Start FRE Offset
+      0x01, 0x00, 0x00, 0x00,  # Number of FREs
+      0x03, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
+
+      # FRE[0,0]: Zero offsets
+      0x00, 0x00,              # Start Address, Info
+
+      # FRE[0,1]: One four-byte offset, SP-based
+      0x01, 0x43,              # Start Address, Info
+      0x74, 0x00, 0x00, 0x00,  # Offset[0]
+
+      # FRE[1,0]: Two two-byte offsets, FP-based
+      0x00, 0x00, 0x24,        # Start Address, Info
+      0x10, 0x00, 0x20, 0x00,  # Offset[0], Offset[1]
+
+      # FRE[1,1]: Four one-byte offsets, FP-based
+      0x00, 0x01, 0x08,        # Start Address, Info
+      0x10, 0x20, 0x30, 0x40,  # Offset[0-3]
+    ]
+## Testing:
+## - little-endian support
+## - AMD64 ABI
+## - one and two byte address sizes
+## - PCInc and PCMask FDE types
+## - all offset sizes
+## - various offset counts
+## - hitting EOF after printing some number of FREs
+## - invalid FRE type
+# CASE1-LABEL:SFrame section '.sframe' {
+#       CASE1:    ABI: AMD64EndianLittle (0x3)
+#       CASE1:    CFA fixed RA offset: 64
+#       CASE1:    FuncDescEntry [0] {
+#  CASE1-NEXT:      PC: 0xDE007F
+#       CASE1:      Info {
+#  CASE1-NEXT:        FRE Type: Addr1 (0x0)
+#  CASE1-NEXT:        FDE Type: PCInc (0x0)
+#       CASE1:      FREs [
+#  CASE1-NEXT:        Frame Row Entry {
+#  CASE1-NEXT:          Start Address: 0xDE007F
+#  CASE1-NEXT:          Return Address Signed: No
+#  CASE1-NEXT:          Offset Size: B1 (0x0)
+#  CASE1-NEXT:          Base Register: FP (0x0)
+#  CASE1-NEXT:          RA Offset: 64
+#  CASE1-NEXT:        }
+#  CASE1-NEXT:        Frame Row Entry {
+#  CASE1-NEXT:          Start Address: 0xDE0080
+#  CASE1-NEXT:          Return Address Signed: No
+#  CASE1-NEXT:          Offset Size: B4 (0x2)
+#  CASE1-NEXT:          Base Register: SP (0x1)
+#  CASE1-NEXT:          CFA Offset: 116
+#  CASE1-NEXT:          RA Offset: 64
+#  CASE1-NEXT:        }
+#  CASE1-NEXT:      ]
+#       CASE1:    FuncDescEntry [1] {
+#  CASE1-NEXT:      PC: 0xAD0093
+#       CASE1:      Info {
+#  CASE1-NEXT:        FRE Type: Addr2 (0x1)
+#  CASE1-NEXT:        FDE Type: PCMask (0x1)
+#       CASE1:      FREs [
+#  CASE1-NEXT:        Frame Row Entry {
+#  CASE1-NEXT:          Start Address: 0x0
+#  CASE1-NEXT:          Return Address Signed: No
+#  CASE1-NEXT:          Offset Size: B2 (0x1)
+#  CASE1-NEXT:          Base Register: FP (0x0)
+#  CASE1-NEXT:          CFA Offset: 16
+#  CASE1-NEXT:          RA Offset: 64
+#  CASE1-NEXT:          FP Offset: 32
+#  CASE1-NEXT:        }
+#  CASE1-NEXT:        Frame Row Entry {
+#  CASE1-NEXT:          Start Address: 0x100
+#  CASE1-NEXT:          Return Address Signed: No
+#  CASE1-NEXT:          Offset Size: B1 (0x0)
+#  CASE1-NEXT:          Base Register: FP (0x0)
+#  CASE1-NEXT:          CFA Offset: 16
+#  CASE1-NEXT:          RA Offset: 64
+#  CASE1-NEXT:          FP Offset: 32
+#  CASE1-NEXT:          Extra Offsets: [48, 64]
+#  CASE1-NEXT:        }
+#  CASE1-NEXT:{{.*}}: warning: '[[FILE]]': unexpected end of data at offset 0x6e while reading [0x6e, 0x71)
+#       CASE1:    FuncDescEntry [2] {
+#       CASE1:      Info {
+#  CASE1-NEXT:        FRE Type: 0x3
+#       CASE1:      FREs [
+#  CASE1-NEXT:{{.*}}: warning: '[[FILE]]': unsupported FRE type 3 at offset 0x60
+
+--- !ELF
+FileHeader:
+  Class: ELFCLASS64
+  Data:  ELFDATA2MSB
+  Type:  ET_EXEC
+Sections:
+  - Name:  .sframe
+    Type:  SHT_GNU_SFRAME
+    Flags: [ SHF_ALLOC ]
+    ContentArray: [
+      0xde, 0xe2, 0x02, 0x05,  # Preamble (magic, version, flags)
+      # Header:
+      0x01, 0x42, 0x47, 0x00,  # ABI, Fixed FP offset, Fixed RA Offset, AUX header length
+      0x00, 0x00, 0x00, 0x01,  # Number of FDEs
+      0x00, 0x00, 0x00, 0x10,  # Number of FREs
+      0x00, 0x00, 0x10, 0x00,  # FRE length
+      0x00, 0x00, 0x00, 0x00,  # FDE offset
+      0x00, 0x00, 0x01, 0x00,  # FRE offset
+
+      # FDE:
+      0x00, 0xde, 0x00, 0x00,  # Start Address
+      0x00, 0x00, 0x01, 0xbe,  # Size
+      0x00, 0x00, 0x00, 0x00,  # Start FRE Offset
+      0x00, 0x00, 0x00, 0x06,  # Number of FREs
+      0x02, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
+
+      # FRE[0]: Zero offsets
+      0x00, 0x00, 0x00, 0x00,  # Start Address
+      0x00,                    # Info
+
+      # FRE[1]: One offset
+      0x00, 0x00, 0x00, 0x01,  # Start Address
+      0x82, 0x10,              # Info, Offset[0]
+
+      # FRE[2]: Two offsets
+      0x00, 0x00, 0x00, 0x02,  # Start Address
+      0x04, 0x10, 0x20,        # Info, Offset[0-1]
+
+      # FRE[3]: Three offsets
+      0x00, 0x00, 0x00, 0x03,  # Start Address
+      0x86, 0x10, 0x20, 0x30,  # Info, Offset[0-2]
+
+      # FRE[4]: Four offsets
+      0x00, 0x00, 0x00, 0x04,  # Start Address
+      0x08,                    # Info
+      0x10, 0x20, 0x30, 0x40,  # Offset[0-3]
+
+      # FRE[4]: Invalid offset size
+      0x00, 0x00, 0x00, 0x05,  # Start Address
+      0x60,                    # Info
+    ]
+## Testing:
+## - big-endian support
+## - ARM64 ABI
+## - four-byte address sizes
+## - return address signing
+## - various offset counts
+## - invalid offset size
+# CASE2-LABEL:SFrame section '.sframe' {
+#       CASE2:    ABI: AArch64EndianBig (0x1)
+#       CASE2:    FuncDescEntry [0] {
+#  CASE2-NEXT:      PC: 0xDE001C
+#       CASE2:      Info {
+#  CASE2-NEXT:        FRE Type: Addr4 (0x2)
+#  CASE2-NEXT:        FDE Type: PCInc (0x0)
+#       CASE2:      FREs [
+#  CASE2-NEXT:        Frame Row Entry {
+#  CASE2-NEXT:          Start Address: 0xDE001C
+#  CASE2-NEXT:          Return Address Signed: No
+#  CASE2-NEXT:          Offset Size: B1 (0x0)
+#  CASE2-NEXT:          Base Register: FP (0x0)
+#  CASE2-NEXT:        }
+#  CASE2-NEXT:        Frame Row Entry {
+#  CASE2-NEXT:          Start Address: 0xDE001D
+#  CASE2-NEXT:          Return Address Signed: Yes
+#  CASE2-NEXT:          Offset Size: B1 (0x0)
+#  CASE2-NEXT:          Base Register: FP (0x0)
+#  CASE2-NEXT:          CFA Offset: 16
+#  CASE2-NEXT:        }
+#  CASE2-NEXT:        Frame Row Entry {
+#  CASE2-NEXT:          Start Address: 0xDE001E
+#  CASE2-NEXT:          Return Address Signed: No
+#  CASE2-NEXT:          Offset Size: B1 (0x0)
+#  CASE2-NEXT:          Base Register: FP (0x0)
+#  CASE2-NEXT:          CFA Offset: 16
+#  CASE2-NEXT:          RA Offset: 32
+#  CASE2-NEXT:        }
+#  CASE2-NEXT:        Frame Row Entry {
+#  CASE2-NEXT:          Start Address: 0xDE001F
+#  CASE2-NEXT:          Return Address Signed: Yes
+#  CASE2-NEXT:          Offset Size: B1 (0x0)
+#  CASE2-NEXT:          Base Register: FP (0x0)
+#  CASE2-NEXT:          CFA Offset: 16
+#  CASE2-NEXT:          RA Offset: 32
+#  CASE2-NEXT:          FP Offset: 48
+#  CASE2-NEXT:        }
+#  CASE2-NEXT:        Frame Row Entry {
+#  CASE2-NEXT:          Start Address: 0xDE0020
+#  CASE2-NEXT:          Return Address Signed: No
+#  CASE2-NEXT:          Offset Size: B1 (0x0)
+#  CASE2-NEXT:          Base Register: FP (0x0)
+#  CASE2-NEXT:          CFA Offset: 16
+#  CASE2-NEXT:          RA Offset: 32
+#  CASE2-NEXT:          FP Offset: 48
+#  CASE2-NEXT:          Extra Offsets: [64]
+#  CASE2-NEXT:        }
+#  CASE2-NEXT:{{.*}}: warning: '[[FILE]]': unsupported FRE offset size 3 at offset 0x58
diff --git a/llvm/test/tools/llvm-readobj/ELF/sframe-reloc.test b/llvm/test/tools/llvm-readobj/ELF/sframe-reloc.test
new file mode 100644
index 0000000000000..e96ae023e460f
--- /dev/null
+++ b/llvm/test/tools/llvm-readobj/ELF/sframe-reloc.test
@@ -0,0 +1,93 @@
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-readobj --sframe %t 2>&1 | \
+# RUN:   FileCheck %s --strict-whitespace --match-full-lines -DFILE=%t
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name:  .text
+    Type:  SHT_PROGBITS
+    Flags: [ SHF_ALLOC, SHF_EXECINSTR ]
+    Size:  0x1000
+  - Name:  .sframe
+    Type:  SHT_GNU_SFRAME
+    Flags: [ SHF_ALLOC ]
+    ContentArray: [
+      0xe2, 0xde, 0x02, 0x04,  # Preamble (magic, version, flags)
+      # Header:
+      0x03, 0x42, 0x40, 0x00,  # ABI, Fixed FP offset, Fixed RA Offset, AUX header length
+      0x03, 0x00, 0x00, 0x00,  # Number of FDEs
+      0x01, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0x10, 0x00, 0x00,  # FRE length
+      0x00, 0x00, 0x00, 0x00,  # FDE offset
+      0x00, 0x00, 0x00, 0x00,  # FRE offset
+
+      # FDE[0]:
+      0x00, 0x00, 0xde, 0x00,  # Start Address
+      0xbe, 0x01, 0x00, 0x00,  # Size
+      0x00, 0x00, 0x00, 0x00,  # Start FRE Offset
+      0x01, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
+
+      # FDE[1]:
+      0x00, 0x00, 0xad, 0x00,  # Start Address
+      0xbe, 0x01, 0x00, 0x00,  # Size
+      0x00, 0x00, 0x00, 0x00,  # Start FRE Offset
+      0x00, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
+
+      # FDE[2]:
+      0x00, 0x00, 0xbe, 0x00,  # Start Address
+      0xbe, 0x01, 0x00, 0x00,  # Size
+      0x00, 0x00, 0x00, 0x00,  # Start FRE Offset
+      0x00, 0x00, 0x00, 0x00,  # Number of FREs
+      0x00, 0xde, 0xad, 0x00,  # Info, RepSize, Padding2
+
+      # FRE[0]:
+      0x05, 0x02, 0x10,        # Start Address, Info, Offset[0]
+
+    ]
+  - Name:            .rela.sframe
+    Type:            SHT_RELA
+    Flags:           [ SHF_INFO_LINK ]
+    Link:            .symtab
+    AddressAlign:    0x8
+    Info:            .sframe
+    Relocations:
+      - Offset:          0x1c
+        Symbol:          .text
+        Type:            R_X86_64_PC32
+        Addend:          0x42
+      - Offset:          0x30
+        Symbol:          .text
+        Type:            R_X86_64_PC32
+      - Offset:          0x30
+        Symbol:          .text
+        Type:            R_X86_64_PC32
+      - Offset:          0x44
+        Symbol:          4747
+        Type:            R_X86_64_PC32
+Symbols:
+  - Name:            .text
+    Type:            STT_SECTION
+    Section:         .text
+# CHECK-LABEL:SFrame section '.sframe' {
+#       CHECK:    ABI: AMD64EndianLittle (0x3)
+#       CHECK:    FuncDescEntry [0] {
+#  CHECK-NEXT:      PC {
+#  CHECK-NEXT:        Relocation: R_X86_64_PC32
+#  CHECK-NEXT:        Symbol Name: .text
+#  CHECK-NEXT:        Start Address: 0xDE0042
+#  CHECK-NEXT:      }
+#       CHECK:      FREs [
+#  CHECK-NEXT:        Frame Row Entry {
+#  CHECK-NEXT:          Start Address: 0xDE0047
+#       CHECK:    FuncDescEntry [1] {
+#  CHECK-NEXT:{{.*}}: warning: '[[FILE]]': more than one relocation at offset 0x30
+#  CHECK-NEXT:      PC: 0xAD0030
+#       CHECK:    FuncDescEntry [2] {
+#  CHECK-NEXT:{{.*}}: warning: '[[FILE]]': unable to read an entry with index 4747 from SHT_SYMTAB section with index {{[0-9]*}}: can't read an entry at 0x{{[0-9a-f]*}}: it goes past the end of the section (0x30)
diff --git a/llvm/test/tools/llvm-reduce/mir/preserve-frame-info.mir b/llvm/test/tools/llvm-reduce/mir/preserve-frame-info.mir
index d7ad5f88874d7..9af4ac3863405 100644
--- a/llvm/test/tools/llvm-reduce/mir/preserve-frame-info.mir
+++ b/llvm/test/tools/llvm-reduce/mir/preserve-frame-info.mir
@@ -20,8 +20,10 @@
 # RESULT-NEXT: hasVAStart:      true
 # RESULT-NEXT: hasMustTailInVarArgFunc: true
 # RESULT-NEXT: hasTailCall:     true
-# RESULT-NEXT: savePoint:       '%bb.1'
-# RESULT-NEXT: restorePoint:    '%bb.2'
+# RESULT-NEXT: savePoint:
+# RESULT-NEXT:   - point:           '%bb.1'
+# RESULT-NEXT: restorePoint:
+# RESULT-NEXT:   - point:           '%bb.2'
 
 # RESULT-NEXT: fixedStack:
 # RESULT-NEXT:  - { id: 0, offset: 56, size: 4, alignment: 8, callee-saved-register: '$sgpr44',
@@ -116,8 +118,10 @@ frameInfo:
   hasMustTailInVarArgFunc: true
   hasTailCall:     true
   localFrameSize:  0
-  savePoint:       '%bb.1'
-  restorePoint:    '%bb.2'
+  savePoint:
+    - point:           '%bb.1'
+  restorePoint:
+    - point:           '%bb.2'
 
 fixedStack:
   - { id: 0, offset: 0, size: 8, alignment: 4, isImmutable: true, isAliased: false }
diff --git a/llvm/test/tools/llvm-reduce/operands-to-args-lifetimes.ll b/llvm/test/tools/llvm-reduce/operands-to-args-lifetimes.ll
index d9ed9dff02b6e..5db1989968455 100644
--- a/llvm/test/tools/llvm-reduce/operands-to-args-lifetimes.ll
+++ b/llvm/test/tools/llvm-reduce/operands-to-args-lifetimes.ll
@@ -4,15 +4,15 @@
 ; INTERESTING: store
 ; REDUCED: define void @test(ptr %a) {
 ; REDUCED-NEXT: %a1 = alloca i32
-; REDUCED-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr %a1)
+; REDUCED-NEXT: call void @llvm.lifetime.start.p0(ptr %a1)
 ; REDUCED-NEXT: store i32 0, ptr %a
 ; REDUCED-NEXT: store i32 1, ptr %a
-; REDUCED-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr %a1)
+; REDUCED-NEXT: call void @llvm.lifetime.end.p0(ptr %a1)
 define void @test() {
   %a = alloca i32
-  call void @llvm.lifetime.start.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.start.p0(ptr %a)
   store i32 0, ptr %a
   store i32 1, ptr %a
-  call void @llvm.lifetime.end.p0(i64 4, ptr %a)
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
diff --git a/llvm/test/tools/llvm-reduce/reduce-operands-alloca.ll b/llvm/test/tools/llvm-reduce/reduce-operands-alloca.ll
index b68f7182feaa2..75b152fadd09e 100644
--- a/llvm/test/tools/llvm-reduce/reduce-operands-alloca.ll
+++ b/llvm/test/tools/llvm-reduce/reduce-operands-alloca.ll
@@ -69,13 +69,13 @@ define void @alloca_constexpr_elt() {
 }
 
 ; CHECK-LABEL: @alloca_lifetimes(
-; ZERO: call void @llvm.lifetime.start.p0(i64 4, ptr %alloca)
-; ONE: call void @llvm.lifetime.start.p0(i64 4, ptr %alloca)
-; POISON: call void @llvm.lifetime.start.p0(i64 4, ptr %alloca)
+; ZERO: call void @llvm.lifetime.start.p0(ptr %alloca)
+; ONE: call void @llvm.lifetime.start.p0(ptr %alloca)
+; POISON: call void @llvm.lifetime.start.p0(ptr %alloca)
 define void @alloca_lifetimes() {
   %alloca = alloca i32
-  call void @llvm.lifetime.start.p0(i64 4, ptr %alloca)
+  call void @llvm.lifetime.start.p0(ptr %alloca)
   store i32 0, ptr %alloca
-  call void @llvm.lifetime.end.p0(i64 4, ptr %alloca)
+  call void @llvm.lifetime.end.p0(ptr %alloca)
   ret void
 }
diff --git a/llvm/tools/llvm-c-test/debuginfo.c b/llvm/tools/llvm-c-test/debuginfo.c
index e73f69743805c..0f09c74a476bb 100644
--- a/llvm/tools/llvm-c-test/debuginfo.c
+++ b/llvm/tools/llvm-c-test/debuginfo.c
@@ -416,3 +416,41 @@ int llvm_di_type_get_name(void) {
 
   return 0;
 }
+
+int llvm_add_globaldebuginfo(void) {
+  const char *Filename = "debuginfo.c";
+  LLVMModuleRef M = LLVMModuleCreateWithName(Filename);
+  LLVMDIBuilderRef Builder = LLVMCreateDIBuilder(M);
+  LLVMMetadataRef File =
+      LLVMDIBuilderCreateFile(Builder, Filename, strlen(Filename), ".", 1);
+
+  LLVMMetadataRef GlobalVarValueExpr =
+      LLVMDIBuilderCreateConstantValueExpression(Builder, 0);
+  LLVMMetadataRef Int64Ty =
+      LLVMDIBuilderCreateBasicType(Builder, "Int64", 5, 64, 0, LLVMDIFlagZero);
+  LLVMMetadataRef Int64TypeDef = LLVMDIBuilderCreateTypedef(
+      Builder, Int64Ty, "int64_t", 7, File, 42, File, 0);
+
+  LLVMMetadataRef GVE = LLVMDIBuilderCreateGlobalVariableExpression(
+      Builder, File, "global", 6, "", 0, File, 1, Int64TypeDef, true,
+      GlobalVarValueExpr, NULL, 0);
+
+  LLVMTypeRef RecType =
+      LLVMStructCreateNamed(LLVMGetModuleContext(M), "struct");
+  LLVMValueRef Global = LLVMAddGlobal(M, RecType, "global");
+
+  LLVMGlobalAddDebugInfo(Global, GVE);
+  // use AddMetadata to add twice
+  int kindId = LLVMGetMDKindID("dbg", 3);
+  LLVMGlobalAddMetadata(Global, kindId, GVE);
+  size_t numEntries;
+  LLVMValueMetadataEntry *ME = LLVMGlobalCopyAllMetadata(Global, &numEntries);
+  assert(ME != NULL);
+  assert(numEntries == 2);
+
+  LLVMDisposeValueMetadataEntries(ME);
+  LLVMDisposeDIBuilder(Builder);
+  LLVMDisposeModule(M);
+
+  return 0;
+}
diff --git a/llvm/tools/llvm-c-test/llvm-c-test.h b/llvm/tools/llvm-c-test/llvm-c-test.h
index 1da6596cd5a8f..4c5a88ce0447e 100644
--- a/llvm/tools/llvm-c-test/llvm-c-test.h
+++ b/llvm/tools/llvm-c-test/llvm-c-test.h
@@ -45,6 +45,7 @@ int llvm_add_named_metadata_operand(void);
 int llvm_set_metadata(void);
 int llvm_replace_md_operand(void);
 int llvm_is_a_value_as_metadata(void);
+int llvm_add_globaldebuginfo(void);
 
 // object.c
 int llvm_object_list_sections(void);
diff --git a/llvm/tools/llvm-c-test/main.c b/llvm/tools/llvm-c-test/main.c
index badbe4b13b6ba..d1963b702888b 100644
--- a/llvm/tools/llvm-c-test/main.c
+++ b/llvm/tools/llvm-c-test/main.c
@@ -101,6 +101,8 @@ int main(int argc, char **argv) {
     return llvm_replace_md_operand();
   } else if (argc == 2 && !strcmp(argv[1], "--is-a-value-as-metadata")) {
     return llvm_is_a_value_as_metadata();
+  } else if (argc == 2 && !strcmp(argv[1], "--add-globaldebuginfo")) {
+    return llvm_add_globaldebuginfo();
   } else if (argc == 2 && !strcmp(argv[1], "--test-function-attributes")) {
     return llvm_test_function_attributes();
   } else if (argc == 2 && !strcmp(argv[1], "--test-callsite-attributes")) {
diff --git a/llvm/tools/llvm-cgdata/Opts.td b/llvm/tools/llvm-cgdata/Opts.td
index 8da933f744e87..2b515a0140e67 100644
--- a/llvm/tools/llvm-cgdata/Opts.td
+++ b/llvm/tools/llvm-cgdata/Opts.td
@@ -31,3 +31,4 @@ def : JoinedOrSeparate<["-"], "o">, Alias<output>, MetaVarName<"<file>">, HelpTe
 def format : Option<["--"], "format", KIND_SEPARATE>,
              HelpText<"Specify the output format (text or binary)">, MetaVarName<"<value>">;
 def : JoinedOrSeparate<["-"], "f">, Alias<format>, HelpText<"Alias for --format">;
+def indexed_codegen_data_lazy_loading : F<"indexed-codegen-data-lazy-loading", "Lazily load indexed CodeGenData for testing purpose.">, Flags<[HelpHidden]>;
diff --git a/llvm/tools/llvm-cgdata/llvm-cgdata.cpp b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp
index 98fa5c5657353..047557e5a7fae 100644
--- a/llvm/tools/llvm-cgdata/llvm-cgdata.cpp
+++ b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp
@@ -83,6 +83,8 @@ static CGDataAction Action;
 static std::optional<CGDataFormat> OutputFormat;
 static std::vector<std::string> InputFilenames;
 
+extern cl::opt<bool> IndexedCodeGenDataLazyLoading;
+
 static void exitWithError(Twine Message, StringRef Whence = "",
                           StringRef Hint = "") {
   WithColor::error();
@@ -361,6 +363,9 @@ static void parseArgs(int argc, char **argv) {
   default:
     llvm_unreachable("unrecognized action");
   }
+
+  IndexedCodeGenDataLazyLoading =
+      Args.hasArg(OPT_indexed_codegen_data_lazy_loading);
 }
 
 int llvm_cgdata_main(int argc, char **argvNonConst, const llvm::ToolContext &) {
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
index a7771b99e97b1..1fd0a15dcfa91 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
@@ -8,6 +8,7 @@
 
 #include "BenchmarkRunner.h"
 #include "Assembler.h"
+#include "DisassemblerHelper.h"
 #include "Error.h"
 #include "MCInstrDescView.h"
 #include "MmapUtils.h"
@@ -20,6 +21,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
 #include "llvm/Support/CrashRecoveryContext.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -588,6 +590,91 @@ class SubProcessFunctionExecutorImpl
   const std::optional<int> BenchmarkProcessCPU;
 };
 #endif // __linux__
+
+// Structure to hold instruction information for assembly printing
+struct InstructionInfo {
+  std::string Text;
+  uint64_t Address;
+  std::string HexBytes;
+};
+
+#ifndef NDEBUG
+// Helper function to print generated assembly snippets
+void printInstructions(const std::vector<InstructionInfo> &Instructions,
+                       int InitialLinesCount, int LastLinesCount) {
+  int N = Instructions.size();
+  dbgs() << "Generated assembly snippet:\n```\n";
+
+  // Print initial lines
+  for (int i = 0; i < InitialLinesCount; ++i)
+    dbgs() << format_hex_no_prefix(Instructions[i].Address, 0) << ":\t"
+           << Instructions[i].HexBytes << Instructions[i].Text << '\n';
+
+  // Show truncation message if needed
+  int SkippedInstructions = N - InitialLinesCount - LastLinesCount;
+  if (SkippedInstructions > 0)
+    dbgs() << "...\t(" << SkippedInstructions << " more instructions)\n";
+
+  // Print last min(PreviewLast, N - PreviewFirst) lines
+  int LastLinesToPrint = std::min(
+      LastLinesCount, N > InitialLinesCount ? N - InitialLinesCount : 0);
+  for (int i = N - LastLinesToPrint; i < N; ++i)
+    dbgs() << format_hex_no_prefix(Instructions[i].Address, 0) << ":\t"
+           << Instructions[i].HexBytes << Instructions[i].Text << '\n';
+  dbgs() << "```\n";
+}
+#endif // NDEBUG
+
+// Function to extract and print assembly from snippet
+Error printAssembledSnippet(const LLVMState &State,
+                            const SmallString<0> &Snippet) {
+  // Extract the actual function bytes from the object file
+  std::vector<uint8_t> FunctionBytes;
+  if (auto Err = getBenchmarkFunctionBytes(Snippet, FunctionBytes))
+    return make_error<Failure>("Failed to extract function bytes: " +
+                               toString(std::move(Err)));
+
+  // Decode all instructions first
+  DisassemblerHelper DisHelper(State);
+  uint64_t Address = 0;
+  std::vector<InstructionInfo> Instructions;
+  const size_t FunctionBytesSize = FunctionBytes.size();
+
+  while (Address < FunctionBytesSize) {
+    MCInst Inst;
+    uint64_t Size;
+    ArrayRef<uint8_t> Bytes(FunctionBytes.data() + Address,
+                            FunctionBytesSize - Address);
+
+    if (!DisHelper.decodeInst(Inst, Size, Bytes)) {
+      Instructions.push_back({"<decode error>", Address, ""});
+      break;
+    }
+
+    // Format instruction text
+    std::string InstStr;
+    raw_string_ostream OS(InstStr);
+    DisHelper.printInst(&Inst, OS);
+
+    // Create hex string for this instruction (big-endian order)
+    std::string HexStr;
+    raw_string_ostream HexOS(HexStr);
+    for (int i = Size - 1; i >= 0; --i)
+      HexOS << format_hex_no_prefix(Bytes[i], 2);
+
+    Instructions.push_back({OS.str(), Address, HexOS.str()});
+    Address += Size;
+  }
+
+#undef DEBUG_TYPE
+#define DEBUG_TYPE "preview-gen-assembly"
+  LLVM_DEBUG(printInstructions(Instructions, 10, 3));
+#undef DEBUG_TYPE
+#define DEBUG_TYPE "print-gen-assembly"
+  LLVM_DEBUG(printInstructions(Instructions, Instructions.size(), 0));
+#undef DEBUG_TYPE
+  return Error::success();
+}
 } // namespace
 
 Expected<SmallString<0>> BenchmarkRunner::assembleSnippet(
@@ -655,6 +742,10 @@ BenchmarkRunner::getRunnableConfiguration(
     if (Error E = Snippet.takeError())
       return std::move(E);
     RC.ObjectFile = getObjectFromBuffer(*Snippet);
+
+    // Print the assembled snippet by disassembling the binary data
+    if (Error E = printAssembledSnippet(State, *Snippet))
+      return std::move(E);
   }
 
   return std::move(RC);
diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
index babcffeb9666a..b5820f8669f1d 100644
--- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
+++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
@@ -482,7 +482,8 @@ void benchmarkMain() {
   InitializeAllExegesisTargets();
 #define LLVM_EXEGESIS(TargetName)                                              \
   LLVMInitialize##TargetName##AsmPrinter();                                    \
-  LLVMInitialize##TargetName##AsmParser();
+  LLVMInitialize##TargetName##AsmParser();                                     \
+  LLVMInitialize##TargetName##Disassembler();
 #include "llvm/Config/TargetExegesis.def"
 
   const LLVMState State = ExitOnErr(
diff --git a/llvm/tools/llvm-jitlink/CMakeLists.txt b/llvm/tools/llvm-jitlink/CMakeLists.txt
index e337fe5f99865..19e3edaf63d86 100644
--- a/llvm/tools/llvm-jitlink/CMakeLists.txt
+++ b/llvm/tools/llvm-jitlink/CMakeLists.txt
@@ -30,10 +30,10 @@ add_llvm_tool(llvm-jitlink
   EXPORT_SYMBOLS
   )
 
-if(${CMAKE_SYSTEM_NAME} MATCHES "Haiku")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "Haiku")
   target_link_libraries(llvm-jitlink PRIVATE network)
 endif()
 
-if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "SunOS")
   target_link_libraries(llvm-jitlink PRIVATE socket)
 endif()
diff --git a/llvm/tools/llvm-objdump/DXContainerDump.cpp b/llvm/tools/llvm-objdump/DXContainerDump.cpp
index 2fb073473de53..52963e0f7d1b6 100644
--- a/llvm/tools/llvm-objdump/DXContainerDump.cpp
+++ b/llvm/tools/llvm-objdump/DXContainerDump.cpp
@@ -12,16 +12,152 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm-objdump.h"
+#include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/Object/DXContainer.h"
+#include "llvm/Support/ScopedPrinter.h"
 
 using namespace llvm;
+using namespace llvm::object;
+
+static llvm::SmallString<4> maskToString(uint8_t Mask,
+                                         bool StripTrailing = false) {
+  llvm::SmallString<4> Result("    ");
+  if (Mask & 1)
+    Result[0] = 'x';
+  if (Mask & 2)
+    Result[1] = 'y';
+  if (Mask & 4)
+    Result[2] = 'z';
+  if (Mask & 8)
+    Result[3] = 'w';
+  if (!StripTrailing)
+    return Result;
+  int Size = 8 - countl_zero(Mask);
+  return Result.slice(0, Size);
+}
+
+static void printColumnHeader(raw_ostream &OS, size_t Length) {
+  for (size_t I = 0; I < Length; ++I)
+    OS << "-";
+}
+
+static void printColumnHeaders(raw_ostream &OS, ArrayRef<size_t> Lengths) {
+  // Generate the header in a temporary to avoid trailing whitespace.
+  SmallString<256> Str;
+  raw_svector_ostream Tmp(Str);
+  for (auto L : Lengths) {
+    printColumnHeader(Tmp, L);
+    Tmp << " ";
+  }
+  Str.back() = '\n';
+  OS << Str;
+}
+
+static size_t digitsForNumber(size_t N) {
+  if (N == 0)
+    return 1;
+  return static_cast<size_t>(log10(static_cast<double>(N))) + 1;
+}
 
 namespace {
 class DXContainerDumper : public objdump::Dumper {
+  const DXContainerObjectFile &Obj;
+
 public:
-  DXContainerDumper(const object::DXContainerObjectFile &Obj)
-      : objdump::Dumper(Obj) {}
+  DXContainerDumper(const DXContainerObjectFile &O)
+      : objdump::Dumper(O), Obj(O) {}
+
+  void printPrivateHeaders() override;
+  void printSignature(const DirectX::Signature &S);
 };
+
+void DXContainerDumper::printSignature(const DirectX::Signature &S) {
+  // DXC prints a table like this as part of the shader disassembly:
+  //; Name                 Index   Mask Register SysValue  Format   Used
+  //; -------------------- ----- ------ -------- -------- ------- ------
+  //; NORMAL                   0   xyz         0     NONE   float   xyz
+  //; TEXCOORD                 0   xy          1     NONE   float   xy
+
+  // DXC's implementation doesn't scale columns entirely completely for the
+  // provided input, so this implementation is a bit more complicated in
+  // formatting logic to scale with the size of the printed text.
+
+  // DXC gives names 21 characters for some unknown reason, I arbitrarily chose
+  // to start at 24 so that we're not going shorter but are using a round
+  // number.
+  size_t LongestName = 24;
+  size_t LongestSV = 10;
+  size_t LongestIndex = strlen("Index");
+  size_t LongestRegister = strlen("Register");
+  size_t LongestFormat = strlen("Format");
+  const size_t MaskWidth = 5;
+  // Compute the column widths. Skip calculating the "Mask" and "Used" columns
+  // since they both have widths of 4.
+  for (auto El : S) {
+    LongestName = std::max(LongestName, S.getName(El.NameOffset).size());
+    LongestSV = std::max(
+        LongestSV,
+        enumToStringRef(El.SystemValue, dxbc::getD3DSystemValues()).size());
+    LongestIndex = std::max(LongestIndex, digitsForNumber(El.Index));
+    LongestRegister = std::max(LongestRegister, digitsForNumber(El.Register));
+    LongestFormat = std::max(
+        LongestFormat,
+        enumToStringRef(El.CompType, dxbc::getSigComponentTypes()).size());
+  }
+
+  // Print Column headers.
+  OS << "; ";
+  OS << left_justify("Name", LongestName) << " ";
+  OS << right_justify("Index", LongestIndex) << " ";
+  OS << right_justify("Mask", MaskWidth) << " ";
+  OS << right_justify("Register", LongestRegister) << " ";
+  OS << right_justify("SysValue", LongestSV) << " ";
+  OS << right_justify("Format", LongestFormat) << " ";
+  OS << right_justify("Used", MaskWidth) << "\n";
+  OS << "; ";
+  printColumnHeaders(OS, {LongestName, LongestIndex, MaskWidth, LongestRegister,
+                          LongestSV, LongestFormat, MaskWidth});
+
+  for (auto El : S) {
+    OS << "; " << left_justify(S.getName(El.NameOffset), LongestName) << " ";
+    OS << right_justify(std::to_string(El.Index), LongestIndex) << " ";
+    OS << right_justify(maskToString(El.Mask), MaskWidth) << " ";
+    OS << right_justify(std::to_string(El.Register), LongestRegister) << " ";
+    OS << right_justify(
+              enumToStringRef(El.SystemValue, dxbc::getD3DSystemValues()),
+              LongestSV)
+       << " ";
+    OS << right_justify(
+        enumToStringRef(El.CompType, dxbc::getSigComponentTypes()),
+        LongestFormat);
+    if (El.ExclusiveMask)
+      OS << "  " << maskToString(El.ExclusiveMask, true);
+    OS << "\n";
+  }
+}
+
+void DXContainerDumper::printPrivateHeaders() {
+  const DXContainer &C =
+      cast<object::DXContainerObjectFile>(Obj).getDXContainer();
+
+  if (!C.getInputSignature().isEmpty()) {
+    OS << "; Input signature:\n;\n";
+    printSignature(C.getInputSignature());
+    OS << ";\n";
+  }
+
+  if (!C.getOutputSignature().isEmpty()) {
+    OS << "; Output signature:\n;\n";
+    printSignature(C.getOutputSignature());
+    OS << ";\n";
+  }
+
+  if (!C.getPatchConstantSignature().isEmpty()) {
+    OS << "; Patch Constant signature:\n;\n";
+    printSignature(C.getPatchConstantSignature());
+    OS << ";\n";
+  }
+}
 } // namespace
 
 std::unique_ptr<objdump::Dumper> llvm::objdump::createDXContainerDumper(
diff --git a/llvm/tools/llvm-objdump/MachODump.cpp b/llvm/tools/llvm-objdump/MachODump.cpp
index d6e29a3a94265..8105a248a33be 100644
--- a/llvm/tools/llvm-objdump/MachODump.cpp
+++ b/llvm/tools/llvm-objdump/MachODump.cpp
@@ -1749,7 +1749,7 @@ static void DumpLiteralPointerSection(MachOObjectFile *O,
 
     StringRef BytesStr = unwrapOrError(Sect->getContents(), O->getFileName());
 
-    const char *Contents = reinterpret_cast<const char *>(BytesStr.data());
+    const char *Contents = BytesStr.data();
 
     switch (section_type) {
     case MachO::S_CSTRING_LITERALS:
@@ -1965,7 +1965,7 @@ static void DumpSectionContents(StringRef Filename, MachOObjectFile *O,
 
         StringRef BytesStr =
             unwrapOrError(Section.getContents(), O->getFileName());
-        const char *sect = reinterpret_cast<const char *>(BytesStr.data());
+        const char *sect = BytesStr.data();
         uint32_t sect_size = BytesStr.size();
         uint64_t sect_addr = Section.getAddress();
 
@@ -2049,7 +2049,7 @@ static void DumpInfoPlistSectionContents(StringRef Filename,
         outs() << "Contents of (" << SegName << "," << SectName << ") section\n";
       StringRef BytesStr =
           unwrapOrError(Section.getContents(), O->getFileName());
-      const char *sect = reinterpret_cast<const char *>(BytesStr.data());
+      const char *sect = BytesStr.data();
       outs() << format("%.*s", BytesStr.size(), sect) << "\n";
       return;
     }
@@ -3237,7 +3237,7 @@ static const char *GuessCstringPointer(uint64_t ReferenceValue,
           uint64_t object_offset = Sec.offset + sect_offset;
           StringRef MachOContents = info->O->getData();
           uint64_t object_size = MachOContents.size();
-          const char *object_addr = (const char *)MachOContents.data();
+          const char *object_addr = MachOContents.data();
           if (object_offset < object_size) {
             const char *name = object_addr + object_offset;
             return name;
@@ -3258,7 +3258,7 @@ static const char *GuessCstringPointer(uint64_t ReferenceValue,
           uint64_t object_offset = Sec.offset + sect_offset;
           StringRef MachOContents = info->O->getData();
           uint64_t object_size = MachOContents.size();
-          const char *object_addr = (const char *)MachOContents.data();
+          const char *object_addr = MachOContents.data();
           if (object_offset < object_size) {
             const char *name = object_addr + object_offset;
             return name;
@@ -3447,7 +3447,7 @@ static uint64_t GuessPointerPointer(uint64_t ReferenceValue,
           uint64_t object_offset = Sec.offset + sect_offset;
           StringRef MachOContents = info->O->getData();
           uint64_t object_size = MachOContents.size();
-          const char *object_addr = (const char *)MachOContents.data();
+          const char *object_addr = MachOContents.data();
           if (object_offset < object_size) {
             uint64_t pointer_value;
             memcpy(&pointer_value, object_addr + object_offset,
@@ -4350,7 +4350,7 @@ walk_pointer_list_64(const char *listname, const SectionRef S,
   outs() << "Contents of (" << SegName << "," << SectName << ") section\n";
 
   StringRef BytesStr = unwrapOrError(S.getContents(), O->getFileName());
-  const char *Contents = reinterpret_cast<const char *>(BytesStr.data());
+  const char *Contents = BytesStr.data();
 
   for (uint32_t i = 0; i < S.getSize(); i += sizeof(uint64_t)) {
     uint32_t left = S.getSize() - i;
@@ -4399,7 +4399,7 @@ walk_pointer_list_32(const char *listname, const SectionRef S,
   outs() << "Contents of (" << SegName << "," << SectName << ") section\n";
 
   StringRef BytesStr = unwrapOrError(S.getContents(), O->getFileName());
-  const char *Contents = reinterpret_cast<const char *>(BytesStr.data());
+  const char *Contents = BytesStr.data();
 
   for (uint32_t i = 0; i < S.getSize(); i += sizeof(uint32_t)) {
     uint32_t left = S.getSize() - i;
@@ -9063,7 +9063,7 @@ static void PrintDyldLoadCommand(MachO::dylinker_command dyld,
   if (dyld.name >= dyld.cmdsize)
     outs() << "         name ?(bad offset " << dyld.name << ")\n";
   else {
-    const char *P = (const char *)(Ptr) + dyld.name;
+    const char *P = Ptr + dyld.name;
     outs() << "         name " << P << " (offset " << dyld.name << ")\n";
   }
 }
@@ -9094,7 +9094,7 @@ static void PrintRpathLoadCommand(MachO::rpath_command rpath, const char *Ptr) {
   if (rpath.path >= rpath.cmdsize)
     outs() << "         path ?(bad offset " << rpath.path << ")\n";
   else {
-    const char *P = (const char *)(Ptr) + rpath.path;
+    const char *P = Ptr + rpath.path;
     outs() << "         path " << P << " (offset " << rpath.path << ")\n";
   }
 }
@@ -10041,7 +10041,7 @@ static void PrintDylibCommand(MachO::dylib_command dl, const char *Ptr) {
   else
     outs() << "\n";
   if (dl.dylib.name < dl.cmdsize) {
-    const char *P = (const char *)(Ptr) + dl.dylib.name;
+    const char *P = Ptr + dl.dylib.name;
     outs() << "         name " << P << " (offset " << dl.dylib.name << ")\n";
   } else {
     outs() << "         name ?(bad offset " << dl.dylib.name << ")\n";
diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp
index c19c698ac793b..815759dfec309 100644
--- a/llvm/tools/llvm-objdump/llvm-objdump.cpp
+++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp
@@ -636,8 +636,14 @@ static bool isCSKYElf(const ObjectFile &Obj) {
   return Elf && Elf->getEMachine() == ELF::EM_CSKY;
 }
 
+static bool isRISCVElf(const ObjectFile &Obj) {
+  const auto *Elf = dyn_cast<ELFObjectFileBase>(&Obj);
+  return Elf && Elf->getEMachine() == ELF::EM_RISCV;
+}
+
 static bool hasMappingSymbols(const ObjectFile &Obj) {
-  return isArmElf(Obj) || isAArch64Elf(Obj) || isCSKYElf(Obj);
+  return isArmElf(Obj) || isAArch64Elf(Obj) || isCSKYElf(Obj) ||
+         isRISCVElf(Obj);
 }
 
 static void printRelocation(formatted_raw_ostream &OS, StringRef FileName,
diff --git a/llvm/tools/llvm-offload-wrapper/CMakeLists.txt b/llvm/tools/llvm-offload-wrapper/CMakeLists.txt
new file mode 100644
index 0000000000000..18acbe2caaded
--- /dev/null
+++ b/llvm/tools/llvm-offload-wrapper/CMakeLists.txt
@@ -0,0 +1,16 @@
+set(LLVM_LINK_COMPONENTS
+  BitWriter
+  Core
+  Object
+  Option
+  FrontendOffloading
+  Support
+  TargetParser
+  )
+
+add_llvm_tool(llvm-offload-wrapper
+  llvm-offload-wrapper.cpp
+
+  DEPENDS
+  intrinsics_gen
+  )
diff --git a/llvm/tools/llvm-offload-wrapper/llvm-offload-wrapper.cpp b/llvm/tools/llvm-offload-wrapper/llvm-offload-wrapper.cpp
new file mode 100644
index 0000000000000..9dac1646b1e26
--- /dev/null
+++ b/llvm/tools/llvm-offload-wrapper/llvm-offload-wrapper.cpp
@@ -0,0 +1,135 @@
+//===- llvm-offload-wrapper: Create runtime registration code for devices -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Provides a utility for generating runtime registration code for device code.
+// We take a binary image (CUDA fatbinary, HIP offload bundle, LLVM binary) and
+// create a new IR module that calls the respective runtime to load it on the
+// device.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/Frontend/Offloading/OffloadWrapper.h"
+#include "llvm/Frontend/Offloading/Utility.h"
+#include "llvm/Object/OffloadBinary.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileOutputBuffer.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/StringSaver.h"
+#include "llvm/Support/WithColor.h"
+#include "llvm/TargetParser/Host.h"
+
+using namespace llvm;
+
+static cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden);
+
+static cl::OptionCategory
+    OffloadWrapeprCategory("llvm-offload-wrapper options");
+
+static cl::opt<object::OffloadKind> Kind(
+    "kind", cl::desc("Wrap for offload kind:"), cl::cat(OffloadWrapeprCategory),
+    cl::Required,
+    cl::values(clEnumValN(object::OFK_OpenMP, "openmp", "Wrap OpenMP binaries"),
+               clEnumValN(object::OFK_Cuda, "cuda", "Wrap CUDA binaries"),
+               clEnumValN(object::OFK_HIP, "hip", "Wrap HIP binaries")));
+
+static cl::opt<std::string> OutputFile("o", cl::desc("Write output to <file>."),
+                                       cl::value_desc("file"),
+                                       cl::cat(OffloadWrapeprCategory));
+
+static cl::list<std::string> InputFiles(cl::Positional,
+                                        cl::desc("Wrap input from <file>"),
+                                        cl::value_desc("file"), cl::OneOrMore,
+                                        cl::cat(OffloadWrapeprCategory));
+
+static cl::opt<std::string>
+    TheTriple("triple", cl::desc("Target triple for the wrapper module"),
+              cl::init(sys::getDefaultTargetTriple()),
+              cl::cat(OffloadWrapeprCategory));
+
+static Error wrapImages(ArrayRef<ArrayRef<char>> BuffersToWrap) {
+  if (BuffersToWrap.size() > 1 &&
+      (Kind == llvm::object::OFK_Cuda || Kind == llvm::object::OFK_HIP))
+    return createStringError(
+        "CUDA / HIP offloading uses a single fatbinary or offload bundle");
+
+  LLVMContext Context;
+  Module M("offload.wrapper.module", Context);
+  M.setTargetTriple(Triple());
+
+  switch (Kind) {
+  case llvm::object::OFK_OpenMP:
+    if (Error Err = offloading::wrapOpenMPBinaries(
+            M, BuffersToWrap, offloading::getOffloadEntryArray(M),
+            /*Suffix=*/"", /*Relocatable=*/false))
+      return Err;
+    break;
+  case llvm::object::OFK_Cuda:
+    if (Error Err = offloading::wrapCudaBinary(
+            M, BuffersToWrap.front(), offloading::getOffloadEntryArray(M),
+            /*Suffix=*/"", /*EmitSurfacesAndTextures=*/false))
+      return Err;
+    break;
+  case llvm::object::OFK_HIP:
+    if (Error Err = offloading::wrapHIPBinary(
+            M, BuffersToWrap.front(), offloading::getOffloadEntryArray(M)))
+      return Err;
+    break;
+  default:
+    return createStringError(getOffloadKindName(Kind) +
+                             " wrapping is not supported");
+  }
+
+  int FD = -1;
+  if (std::error_code EC = sys::fs::openFileForWrite(OutputFile, FD))
+    return errorCodeToError(EC);
+  llvm::raw_fd_ostream OS(FD, true);
+  WriteBitcodeToFile(M, OS);
+
+  return Error::success();
+}
+
+int main(int argc, char **argv) {
+  InitLLVM X(argc, argv);
+  cl::HideUnrelatedOptions(OffloadWrapeprCategory);
+  cl::ParseCommandLineOptions(
+      argc, argv,
+      "Generate runtime registration code for a device binary image\n");
+
+  if (Help) {
+    cl::PrintHelpMessage();
+    return EXIT_SUCCESS;
+  }
+
+  auto ReportError = [argv](Error E) {
+    logAllUnhandledErrors(std::move(E), WithColor::error(errs(), argv[0]));
+    exit(EXIT_FAILURE);
+  };
+
+  SmallVector<std::unique_ptr<MemoryBuffer>> Buffers;
+  SmallVector<ArrayRef<char>> BuffersToWrap;
+  for (StringRef Input : InputFiles) {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
+        MemoryBuffer::getFileOrSTDIN(Input);
+    if (std::error_code EC = BufferOrErr.getError())
+      ReportError(createFileError(Input, EC));
+    std::unique_ptr<MemoryBuffer> &Buffer =
+        Buffers.emplace_back(std::move(*BufferOrErr));
+    BuffersToWrap.emplace_back(
+        ArrayRef<char>(Buffer->getBufferStart(), Buffer->getBufferSize()));
+  }
+
+  if (Error Err = wrapImages(BuffersToWrap))
+    ReportError(std::move(Err));
+
+  return EXIT_SUCCESS;
+}
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 207ae2ddd4cf2..8fe0887c59774 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -2629,7 +2629,7 @@ void SampleOverlapAggregator::dumpFuncSimilarity(raw_fd_ostream &OS) const {
 }
 
 void SampleOverlapAggregator::dumpProgramSummary(raw_fd_ostream &OS) const {
-  OS << "Profile overlap infomation for base_profile: "
+  OS << "Profile overlap information for base_profile: "
      << ProfOverlap.BaseName.toString()
      << " and test_profile: " << ProfOverlap.TestName.toString()
      << "\nProgram level:\n";
diff --git a/llvm/tools/llvm-profgen/CSPreInliner.h b/llvm/tools/llvm-profgen/CSPreInliner.h
index 8a3f16a4f13cb..022c3f8d0daed 100644
--- a/llvm/tools/llvm-profgen/CSPreInliner.h
+++ b/llvm/tools/llvm-profgen/CSPreInliner.h
@@ -16,9 +16,6 @@
 #include "llvm/Transforms/IPO/ProfiledCallGraph.h"
 #include "llvm/Transforms/IPO/SampleContextTracker.h"
 
-using namespace llvm;
-using namespace sampleprof;
-
 namespace llvm {
 namespace sampleprof {
 
diff --git a/llvm/tools/llvm-profgen/ErrorHandling.h b/llvm/tools/llvm-profgen/ErrorHandling.h
index b797add8a892f..17084bd785e64 100644
--- a/llvm/tools/llvm-profgen/ErrorHandling.h
+++ b/llvm/tools/llvm-profgen/ErrorHandling.h
@@ -16,7 +16,7 @@
 #include "llvm/Support/WithColor.h"
 #include <system_error>
 
-using namespace llvm;
+namespace llvm {
 
 [[noreturn]] inline void exitWithError(const Twine &Message,
                                        StringRef Whence = StringRef(),
@@ -53,4 +53,6 @@ inline void emitWarningSummary(uint64_t Num, uint64_t Total, StringRef Msg) {
                        << "%(" << Num << "/" << Total << ") " << Msg << "\n";
 }
 
+} // end namespace llvm
+
 #endif
diff --git a/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp b/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
index edfe8979c7121..7ebca23ba7956 100644
--- a/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
+++ b/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MissingFrameInferrer.h"
+#include "Options.h"
 #include "PerfReader.h"
 #include "ProfiledBinary.h"
 #include "llvm/ADT/SCCIterator.h"
@@ -37,7 +38,8 @@ STATISTIC(TailCallMaxTailCallPath, "Length of the longest tail call path");
 static cl::opt<uint32_t>
     MaximumSearchDepth("max-search-depth", cl::init(UINT32_MAX - 1),
                        cl::desc("The maximum levels the DFS-based missing "
-                                "frame search should go with"));
+                                "frame search should go with"),
+                       cl::cat(ProfGenCategory));
 
 void MissingFrameInferrer::initialize(
     const ContextSampleCounterMap *SampleCounters) {
diff --git a/llvm/tools/llvm-profgen/Options.h b/llvm/tools/llvm-profgen/Options.h
new file mode 100644
index 0000000000000..f94cf9118c06a
--- /dev/null
+++ b/llvm/tools/llvm-profgen/Options.h
@@ -0,0 +1,28 @@
+//===-- Options.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TOOLS_LLVM_PROFGEN_OPTIONS_H
+#define LLVM_TOOLS_LLVM_PROFGEN_OPTIONS_H
+
+#include "llvm/Support/CommandLine.h"
+
+namespace llvm {
+
+extern cl::OptionCategory ProfGenCategory;
+
+extern cl::opt<std::string> OutputFilename;
+extern cl::opt<bool> ShowDisassemblyOnly;
+extern cl::opt<bool> ShowSourceLocations;
+extern cl::opt<bool> SkipSymbolization;
+extern cl::opt<bool> ShowDetailedWarning;
+extern cl::opt<bool> InferMissingFrames;
+extern cl::opt<bool> EnableCSPreInliner;
+extern cl::opt<bool> UseContextCostForPreInliner;
+
+} // end namespace llvm
+
+#endif
diff --git a/llvm/tools/llvm-profgen/PerfReader.cpp b/llvm/tools/llvm-profgen/PerfReader.cpp
index ad113eda27914..9a805f2941753 100644
--- a/llvm/tools/llvm-profgen/PerfReader.cpp
+++ b/llvm/tools/llvm-profgen/PerfReader.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "PerfReader.h"
+#include "Options.h"
 #include "ProfileGenerator.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
@@ -15,44 +16,47 @@
 
 #define DEBUG_TYPE "perf-reader"
 
+namespace llvm {
+
 cl::opt<bool> SkipSymbolization("skip-symbolization",
                                 cl::desc("Dump the unsymbolized profile to the "
                                          "output file. It will show unwinder "
-                                         "output for CS profile generation."));
+                                         "output for CS profile generation."),
+                                cl::cat(ProfGenCategory));
 
 static cl::opt<bool> ShowMmapEvents("show-mmap-events",
-                                    cl::desc("Print binary load events."));
+                                    cl::desc("Print binary load events."),
+                                    cl::cat(ProfGenCategory));
 
 static cl::opt<bool>
     UseOffset("use-offset", cl::init(true),
               cl::desc("Work with `--skip-symbolization` or "
                        "`--unsymbolized-profile` to write/read the "
-                       "offset instead of virtual address."));
+                       "offset instead of virtual address."),
+              cl::cat(ProfGenCategory));
 
 static cl::opt<bool> UseLoadableSegmentAsBase(
     "use-first-loadable-segment-as-base",
     cl::desc("Use first loadable segment address as base address "
              "for offsets in unsymbolized profile. By default "
-             "first executable segment address is used"));
+             "first executable segment address is used"),
+    cl::cat(ProfGenCategory));
 
 static cl::opt<bool>
     IgnoreStackSamples("ignore-stack-samples",
                        cl::desc("Ignore call stack samples for hybrid samples "
-                                "and produce context-insensitive profile."));
+                                "and produce context-insensitive profile."),
+                       cl::cat(ProfGenCategory));
 cl::opt<bool> ShowDetailedWarning("show-detailed-warning",
-                                  cl::desc("Show detailed warning message."));
+                                  cl::desc("Show detailed warning message."),
+                                  cl::cat(ProfGenCategory));
 
 static cl::opt<int> CSProfMaxUnsymbolizedCtxDepth(
     "csprof-max-unsymbolized-context-depth", cl::init(-1),
     cl::desc("Keep the last K contexts while merging unsymbolized profile. -1 "
-             "means no depth limit."));
+             "means no depth limit."),
+    cl::cat(ProfGenCategory));
 
-extern cl::opt<std::string> PerfTraceFilename;
-extern cl::opt<bool> ShowDisassemblyOnly;
-extern cl::opt<bool> ShowSourceLocations;
-extern cl::opt<std::string> OutputFilename;
-
-namespace llvm {
 namespace sampleprof {
 
 void VirtualUnwinder::unwindCall(UnwindState &State) {
diff --git a/llvm/tools/llvm-profgen/PerfReader.h b/llvm/tools/llvm-profgen/PerfReader.h
index 4b3ac8f569755..19451915812e1 100644
--- a/llvm/tools/llvm-profgen/PerfReader.h
+++ b/llvm/tools/llvm-profgen/PerfReader.h
@@ -17,9 +17,6 @@
 #include <fstream>
 #include <map>
 
-using namespace llvm;
-using namespace sampleprof;
-
 namespace llvm {
 
 class CleanupInstaller;
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
index db686c3b597eb..33575b9c67625 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -8,6 +8,7 @@
 #include "ProfileGenerator.h"
 #include "ErrorHandling.h"
 #include "MissingFrameInferrer.h"
+#include "Options.h"
 #include "PerfReader.h"
 #include "ProfiledBinary.h"
 #include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
@@ -17,20 +18,24 @@
 #include <unordered_set>
 #include <utility>
 
+namespace llvm {
+
 cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
                                     cl::Required,
-                                    cl::desc("Output profile file"));
+                                    cl::desc("Output profile file"),
+                                    cl::cat(ProfGenCategory));
 static cl::alias OutputA("o", cl::desc("Alias for --output"),
                          cl::aliasopt(OutputFilename));
 
 static cl::opt<SampleProfileFormat> OutputFormat(
     "format", cl::desc("Format of output profile"), cl::init(SPF_Ext_Binary),
-    cl::values(
-        clEnumValN(SPF_Binary, "binary", "Binary encoding (default)"),
-        clEnumValN(SPF_Ext_Binary, "extbinary", "Extensible binary encoding"),
-        clEnumValN(SPF_Text, "text", "Text encoding"),
-        clEnumValN(SPF_GCC, "gcc",
-                   "GCC encoding (only meaningful for -sample)")));
+    cl::values(clEnumValN(SPF_Binary, "binary", "Binary encoding (default)"),
+               clEnumValN(SPF_Ext_Binary, "extbinary",
+                          "Extensible binary encoding"),
+               clEnumValN(SPF_Text, "text", "Text encoding"),
+               clEnumValN(SPF_GCC, "gcc",
+                          "GCC encoding (only meaningful for -sample)")),
+    cl::cat(ProfGenCategory));
 
 static cl::opt<bool> UseMD5(
     "use-md5", cl::Hidden,
@@ -56,58 +61,57 @@ static cl::opt<int32_t, true> RecursionCompression(
 static cl::opt<bool>
     TrimColdProfile("trim-cold-profile",
                     cl::desc("If the total count of the profile is smaller "
-                             "than threshold, it will be trimmed."));
+                             "than threshold, it will be trimmed."),
+                    cl::cat(ProfGenCategory));
 
 static cl::opt<bool> CSProfMergeColdContext(
     "csprof-merge-cold-context", cl::init(true),
     cl::desc("If the total count of context profile is smaller than "
              "the threshold, it will be merged into context-less base "
-             "profile."));
+             "profile."),
+    cl::cat(ProfGenCategory));
 
 static cl::opt<uint32_t> CSProfMaxColdContextDepth(
     "csprof-max-cold-context-depth", cl::init(1),
     cl::desc("Keep the last K contexts while merging cold profile. 1 means the "
-             "context-less base profile"));
+             "context-less base profile"),
+    cl::cat(ProfGenCategory));
 
 static cl::opt<int, true> CSProfMaxContextDepth(
     "csprof-max-context-depth",
     cl::desc("Keep the last K contexts while merging profile. -1 means no "
              "depth limit."),
-    cl::location(llvm::sampleprof::CSProfileGenerator::MaxContextDepth));
+    cl::location(llvm::sampleprof::CSProfileGenerator::MaxContextDepth),
+    cl::cat(ProfGenCategory));
 
 static cl::opt<double> ProfileDensityThreshold(
-    "profile-density-threshold", llvm::cl::init(50),
-    llvm::cl::desc("If the profile density is below the given threshold, it "
-                   "will be suggested to increase the sampling rate."),
-    llvm::cl::Optional);
-static cl::opt<bool> ShowDensity("show-density", llvm::cl::init(false),
-                                 llvm::cl::desc("show profile density details"),
-                                 llvm::cl::Optional);
+    "profile-density-threshold", cl::init(50),
+    cl::desc("If the profile density is below the given threshold, it "
+             "will be suggested to increase the sampling rate."),
+    cl::Optional, cl::cat(ProfGenCategory));
+static cl::opt<bool> ShowDensity("show-density", cl::init(false),
+                                 cl::desc("show profile density details"),
+                                 cl::Optional, cl::cat(ProfGenCategory));
 static cl::opt<int> ProfileDensityCutOffHot(
-    "profile-density-cutoff-hot", llvm::cl::init(990000),
-    llvm::cl::desc("Total samples cutoff for functions used to calculate "
-                   "profile density."));
+    "profile-density-cutoff-hot", cl::init(990000),
+    cl::desc("Total samples cutoff for functions used to calculate "
+             "profile density."),
+    cl::cat(ProfGenCategory));
 
 static cl::opt<bool> UpdateTotalSamples(
-    "update-total-samples", llvm::cl::init(false),
-    llvm::cl::desc(
-        "Update total samples by accumulating all its body samples."),
-    llvm::cl::Optional);
+    "update-total-samples", cl::init(false),
+    cl::desc("Update total samples by accumulating all its body samples."),
+    cl::Optional, cl::cat(ProfGenCategory));
 
 static cl::opt<bool> GenCSNestedProfile(
     "gen-cs-nested-profile", cl::Hidden, cl::init(true),
     cl::desc("Generate nested function profiles for CSSPGO"));
 
 cl::opt<bool> InferMissingFrames(
-    "infer-missing-frames", llvm::cl::init(true),
-    llvm::cl::desc(
+    "infer-missing-frames", cl::init(true),
+    cl::desc(
         "Infer missing call frames due to compiler tail call elimination."),
-    llvm::cl::Optional);
-
-using namespace llvm;
-using namespace sampleprof;
-
-namespace llvm {
+    cl::Optional, cl::cat(ProfGenCategory));
 
 namespace sampleprof {
 
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.h b/llvm/tools/llvm-profgen/ProfileGenerator.h
index 5e36128530cd9..d3e04563a81c2 100644
--- a/llvm/tools/llvm-profgen/ProfileGenerator.h
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.h
@@ -17,9 +17,6 @@
 #include <memory>
 #include <unordered_set>
 
-using namespace llvm;
-using namespace sampleprof;
-
 namespace llvm {
 namespace sampleprof {
 
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.cpp b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
index 6847ba1b21b1f..31cac4d5c7721 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.cpp
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.cpp
@@ -9,6 +9,7 @@
 #include "ProfiledBinary.h"
 #include "ErrorHandling.h"
 #include "MissingFrameInferrer.h"
+#include "Options.h"
 #include "ProfileGenerator.h"
 #include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
 #include "llvm/Demangle/Demangle.h"
@@ -24,46 +25,51 @@
 
 #define DEBUG_TYPE "load-binary"
 
-using namespace llvm;
-using namespace sampleprof;
+namespace llvm {
+
+using namespace object;
 
 cl::opt<bool> ShowDisassemblyOnly("show-disassembly-only",
-                                  cl::desc("Print disassembled code."));
+                                  cl::desc("Print disassembled code."),
+                                  cl::cat(ProfGenCategory));
 
 cl::opt<bool> ShowSourceLocations("show-source-locations",
-                                  cl::desc("Print source locations."));
+                                  cl::desc("Print source locations."),
+                                  cl::cat(ProfGenCategory));
 
 static cl::opt<bool>
     ShowCanonicalFnName("show-canonical-fname",
-                        cl::desc("Print canonical function name."));
+                        cl::desc("Print canonical function name."),
+                        cl::cat(ProfGenCategory));
 
 static cl::opt<bool> ShowPseudoProbe(
     "show-pseudo-probe",
-    cl::desc("Print pseudo probe section and disassembled info."));
+    cl::desc("Print pseudo probe section and disassembled info."),
+    cl::cat(ProfGenCategory));
 
 static cl::opt<bool> UseDwarfCorrelation(
     "use-dwarf-correlation",
     cl::desc("Use dwarf for profile correlation even when binary contains "
-             "pseudo probe."));
+             "pseudo probe."),
+    cl::cat(ProfGenCategory));
 
 static cl::opt<std::string>
     DWPPath("dwp", cl::init(""),
             cl::desc("Path of .dwp file. When not specified, it will be "
-                     "<binary>.dwp in the same directory as the main binary."));
+                     "<binary>.dwp in the same directory as the main binary."),
+            cl::cat(ProfGenCategory));
 
 static cl::list<std::string> DisassembleFunctions(
     "disassemble-functions", cl::CommaSeparated,
     cl::desc("List of functions to print disassembly for. Accept demangled "
-             "names only. Only work with show-disassembly-only"));
+             "names only. Only work with show-disassembly-only"),
+    cl::cat(ProfGenCategory));
 
 static cl::opt<bool>
     KernelBinary("kernel",
-                 cl::desc("Generate the profile for Linux kernel binary."));
+                 cl::desc("Generate the profile for Linux kernel binary."),
+                 cl::cat(ProfGenCategory));
 
-extern cl::opt<bool> ShowDetailedWarning;
-extern cl::opt<bool> InferMissingFrames;
-
-namespace llvm {
 namespace sampleprof {
 
 static const Target *getTarget(const ObjectFile *Obj) {
diff --git a/llvm/tools/llvm-profgen/ProfiledBinary.h b/llvm/tools/llvm-profgen/ProfiledBinary.h
index 0588cb48b2af6..9c0bff591337a 100644
--- a/llvm/tools/llvm-profgen/ProfiledBinary.h
+++ b/llvm/tools/llvm-profgen/ProfiledBinary.h
@@ -41,15 +41,6 @@
 #include <unordered_set>
 #include <vector>
 
-namespace llvm {
-extern cl::opt<bool> EnableCSPreInliner;
-extern cl::opt<bool> UseContextCostForPreInliner;
-} // namespace llvm
-
-using namespace llvm;
-using namespace sampleprof;
-using namespace llvm::object;
-
 namespace llvm {
 namespace sampleprof {
 
@@ -303,34 +294,34 @@ class ProfiledBinary {
 
   bool IsCOFF = false;
 
-  void setPreferredTextSegmentAddresses(const ObjectFile *O);
+  void setPreferredTextSegmentAddresses(const object::ObjectFile *O);
 
   template <class ELFT>
-  void setPreferredTextSegmentAddresses(const ELFFile<ELFT> &Obj,
+  void setPreferredTextSegmentAddresses(const object::ELFFile<ELFT> &Obj,
                                         StringRef FileName);
-  void setPreferredTextSegmentAddresses(const COFFObjectFile *Obj,
+  void setPreferredTextSegmentAddresses(const object::COFFObjectFile *Obj,
                                         StringRef FileName);
 
-  void checkPseudoProbe(const ELFObjectFileBase *Obj);
+  void checkPseudoProbe(const object::ELFObjectFileBase *Obj);
 
-  void decodePseudoProbe(const ELFObjectFileBase *Obj);
+  void decodePseudoProbe(const object::ELFObjectFileBase *Obj);
 
-  void
-  checkUseFSDiscriminator(const ObjectFile *Obj,
-                          std::map<SectionRef, SectionSymbolsTy> &AllSymbols);
+  void checkUseFSDiscriminator(
+      const object::ObjectFile *Obj,
+      std::map<object::SectionRef, SectionSymbolsTy> &AllSymbols);
 
   // Set up disassembler and related components.
-  void setUpDisassembler(const ObjectFile *Obj);
+  void setUpDisassembler(const object::ObjectFile *Obj);
   symbolize::LLVMSymbolizer::Options getSymbolizerOpts() const;
 
   // Load debug info of subprograms from DWARF section.
-  void loadSymbolsFromDWARF(ObjectFile &Obj);
+  void loadSymbolsFromDWARF(object::ObjectFile &Obj);
 
   // Load debug info from DWARF unit.
   void loadSymbolsFromDWARFUnit(DWARFUnit &CompilationUnit);
 
   // Create elf symbol to its start address mapping.
-  void populateElfSymbolAddressList(const ELFObjectFileBase *O);
+  void populateElfSymbolAddressList(const object::ELFObjectFileBase *O);
 
   // A function may be spilt into multiple non-continuous address ranges. We use
   // this to set whether start a function range is the real entry of the
@@ -341,11 +332,12 @@ class ProfiledBinary {
   void warnNoFuncEntry();
 
   /// Dissassemble the text section and build various address maps.
-  void disassemble(const ObjectFile *O);
+  void disassemble(const object::ObjectFile *O);
 
   /// Helper function to dissassemble the symbol and extract info for unwinding
   bool dissassembleSymbol(std::size_t SI, ArrayRef<uint8_t> Bytes,
-                          SectionSymbolsTy &Symbols, const SectionRef &Section);
+                          SectionSymbolsTy &Symbols,
+                          const object::SectionRef &Section);
   /// Symbolize a given instruction pointer and return a full call context.
   SampleContextFrameVector symbolize(const InstructionPointer &IP,
                                      bool UseCanonicalFnName = false,
diff --git a/llvm/tools/llvm-profgen/llvm-profgen.cpp b/llvm/tools/llvm-profgen/llvm-profgen.cpp
index 3b974e25103ad..7e070a1ea6489 100644
--- a/llvm/tools/llvm-profgen/llvm-profgen.cpp
+++ b/llvm/tools/llvm-profgen/llvm-profgen.cpp
@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ErrorHandling.h"
+#include "Options.h"
 #include "PerfReader.h"
 #include "ProfileGenerator.h"
 #include "ProfiledBinary.h"
@@ -21,7 +22,12 @@
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/VirtualFileSystem.h"
 
-static cl::OptionCategory ProfGenCategory("ProfGen Options");
+using namespace llvm;
+using namespace sampleprof;
+
+namespace llvm {
+
+cl::OptionCategory ProfGenCategory("ProfGen Options");
 
 static cl::opt<std::string> PerfScriptFilename(
     "perfscript", cl::value_desc("perfscript"),
@@ -67,13 +73,6 @@ static cl::opt<std::string> DebugBinPath(
              "from it instead of the executable binary."),
     cl::cat(ProfGenCategory));
 
-extern cl::opt<bool> ShowDisassemblyOnly;
-extern cl::opt<bool> ShowSourceLocations;
-extern cl::opt<bool> SkipSymbolization;
-
-using namespace llvm;
-using namespace sampleprof;
-
 // Validate the command line input.
 static void validateCommandLine() {
   // Allow the missing perfscript if we only use to show binary disassembly.
@@ -138,6 +137,8 @@ static PerfInputFile getPerfInputFile() {
   return File;
 }
 
+} // end namespace llvm
+
 int main(int argc, const char *argv[]) {
   InitLLVM X(argc, argv);
 
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 26a84fae5f7e0..ade025fd3d892 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -425,7 +425,13 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
   ArrayRef<Elf_Word> getShndxTable(const Elf_Shdr *Symtab) const;
 
   void printSFrameHeader(const SFrameParser<ELFT::Endianness> &Parser);
-  void printSFrameFDEs(const SFrameParser<ELFT::Endianness> &Parser);
+  void printSFrameFDEs(const SFrameParser<ELFT::Endianness> &Parser,
+                       ArrayRef<Relocation<ELFT>> Relocations,
+                       const Elf_Shdr *RelocSymTab);
+  uint64_t getAndPrintSFrameFDEStartAddress(
+      const SFrameParser<ELFT::Endianness> &Parser,
+      const typename SFrameParser<ELFT::Endianness>::FDERange::iterator FDE,
+      ArrayRef<Relocation<ELFT>> Relocations, const Elf_Shdr *RelocSymTab);
 
 private:
   mutable SmallVector<std::optional<VersionEntry>, 0> VersionMap;
@@ -6479,15 +6485,16 @@ void ELFDumper<ELFT>::printSFrameHeader(
   if (Expected<ArrayRef<uint8_t>> Aux = Parser.getAuxHeader())
     W.printHexList("Auxiliary header", *Aux);
   else
-    reportWarning(Aux.takeError(), FileName);
+    reportUniqueWarning(Aux.takeError());
 }
 
 template <typename ELFT>
 void ELFDumper<ELFT>::printSFrameFDEs(
-    const SFrameParser<ELFT::Endianness> &Parser) {
+    const SFrameParser<ELFT::Endianness> &Parser,
+    ArrayRef<Relocation<ELFT>> Relocations, const Elf_Shdr *RelocSymTab) {
   typename SFrameParser<ELFT::Endianness>::FDERange FDEs;
   if (Error Err = Parser.fdes().moveInto(FDEs)) {
-    reportWarning(std::move(Err), FileName);
+    reportUniqueWarning(std::move(Err));
     return;
   }
 
@@ -6497,19 +6504,21 @@ void ELFDumper<ELFT>::printSFrameFDEs(
         W,
         formatv("FuncDescEntry [{0}]", std::distance(FDEs.begin(), It)).str());
 
-    W.printHex("PC", Parser.getAbsoluteStartAddress(It));
+    uint64_t FDEStartAddress =
+        getAndPrintSFrameFDEStartAddress(Parser, It, Relocations, RelocSymTab);
     W.printHex("Size", It->Size);
     W.printHex("Start FRE Offset", It->StartFREOff);
     W.printNumber("Num FREs", It->NumFREs);
 
     {
       DictScope InfoScope(W, "Info");
-      W.printEnum("FRE Type", It->getFREType(), sframe::getFRETypes());
-      W.printEnum("FDE Type", It->getFDEType(), sframe::getFDETypes());
+      W.printEnum("FRE Type", It->Info.getFREType(), sframe::getFRETypes());
+      W.printEnum("FDE Type", It->Info.getFDEType(), sframe::getFDETypes());
       switch (Parser.getHeader().ABIArch) {
       case sframe::ABI::AArch64EndianBig:
       case sframe::ABI::AArch64EndianLittle:
-        W.printEnum("PAuth Key", sframe::AArch64PAuthKey(It->getPAuthKey()),
+        W.printEnum("PAuth Key",
+                    sframe::AArch64PAuthKey(It->Info.getPAuthKey()),
                     sframe::getAArch64PAuthKeys());
         break;
       case sframe::ABI::AMD64EndianLittle:
@@ -6517,22 +6526,86 @@ void ELFDumper<ELFT>::printSFrameFDEs(
         break;
       }
 
-      W.printHex("Raw", It->Info);
+      W.printHex("Raw", It->Info.Info);
     }
 
     W.printHex(
         ("Repetitive block size" +
-         Twine(It->getFDEType() == sframe::FDEType::PCMask ? "" : " (unused)"))
+         Twine(It->Info.getFDEType() == sframe::FDEType::PCMask ? ""
+                                                                : " (unused)"))
             .str(),
         It->RepSize);
 
     W.printHex("Padding2", It->Padding2);
+
+    ListScope FREListScope(W, "FREs");
+    Error Err = Error::success();
+    for (const typename SFrameParser<ELFT::Endianness>::FrameRowEntry &FRE :
+         Parser.fres(*It, Err)) {
+      DictScope FREScope(W, "Frame Row Entry");
+      W.printHex("Start Address",
+                 (It->Info.getFDEType() == sframe::FDEType::PCInc
+                      ? FDEStartAddress
+                      : 0) +
+                     FRE.StartAddress);
+      W.printBoolean("Return Address Signed", FRE.Info.isReturnAddressSigned());
+      W.printEnum("Offset Size", FRE.Info.getOffsetSize(),
+                  sframe::getFREOffsets());
+      W.printEnum("Base Register", FRE.Info.getBaseRegister(),
+                  sframe::getBaseRegisters());
+      if (std::optional<int32_t> Off = Parser.getCFAOffset(FRE))
+        W.printNumber("CFA Offset", *Off);
+      if (std::optional<int32_t> Off = Parser.getRAOffset(FRE))
+        W.printNumber("RA Offset", *Off);
+      if (std::optional<int32_t> Off = Parser.getFPOffset(FRE))
+        W.printNumber("FP Offset", *Off);
+      if (ArrayRef<int32_t> Offs = Parser.getExtraOffsets(FRE); !Offs.empty())
+        W.printList("Extra Offsets", Offs);
+    }
+    if (Err)
+      reportUniqueWarning(std::move(Err));
+  }
+}
+
+template <typename ELFT>
+uint64_t ELFDumper<ELFT>::getAndPrintSFrameFDEStartAddress(
+    const SFrameParser<ELFT::Endianness> &Parser,
+    const typename SFrameParser<ELFT::Endianness>::FDERange::iterator FDE,
+    ArrayRef<Relocation<ELFT>> Relocations, const Elf_Shdr *RelocSymTab) {
+  uint64_t Address = Parser.getAbsoluteStartAddress(FDE);
+  uint64_t Offset = Parser.offsetOf(FDE);
+
+  auto Reloc = llvm::lower_bound(
+      Relocations, Offset, [](auto R, uint64_t O) { return R.Offset < O; });
+  if (Reloc == Relocations.end() || Reloc->Offset != Offset) {
+    W.printHex("PC", Address);
+  } else if (std::next(Reloc) != Relocations.end() &&
+             std::next(Reloc)->Offset == Offset) {
+    reportUniqueWarning(
+        formatv("more than one relocation at offset {0:x+}", Offset));
+    W.printHex("PC", Address);
+  } else if (Expected<RelSymbol<ELFT>> RelSym =
+                 getRelocationTarget(*Reloc, RelocSymTab);
+             !RelSym) {
+    reportUniqueWarning(RelSym.takeError());
+    W.printHex("PC", Address);
+  } else {
+    // Exactly one relocation at the given offset. Print it.
+    DictScope PCScope(W, "PC");
+    SmallString<32> RelocName;
+    Obj.getRelocationTypeName(Reloc->Type, RelocName);
+    W.printString("Relocation", RelocName);
+    W.printString("Symbol Name", RelSym->Name);
+    Address = FDE->StartAddress + Reloc->Addend.value_or(0);
+    W.printHex("Start Address", Address);
   }
+  return Address;
 }
 
 template <typename ELFT>
 void ELFDumper<ELFT>::printSectionsAsSFrame(ArrayRef<std::string> Sections) {
   constexpr endianness E = ELFT::Endianness;
+
   for (object::SectionRef Section :
        getSectionRefsByNameOrIndex(ObjF, Sections)) {
     // Validity of sections names checked in getSectionRefsByNameOrIndex.
@@ -6543,21 +6616,42 @@ void ELFDumper<ELFT>::printSectionsAsSFrame(ArrayRef<std::string> Sections) {
 
     StringRef SectionContent;
     if (Error Err = Section.getContents().moveInto(SectionContent)) {
-      reportWarning(std::move(Err), FileName);
+      reportUniqueWarning(std::move(Err));
       continue;
     }
 
     Expected<object::SFrameParser<E>> Parser = object::SFrameParser<E>::create(
         arrayRefFromStringRef(SectionContent), Section.getAddress());
     if (!Parser) {
-      reportWarning(createError("invalid sframe section: " +
-                                toString(Parser.takeError())),
-                    FileName);
+      reportUniqueWarning("invalid sframe section: " +
+                          toString(Parser.takeError()));
       continue;
     }
 
+    const Elf_Shdr *ELFSection = ObjF.getSection(Section.getRawDataRefImpl());
+    MapVector<const Elf_Shdr *, const Elf_Shdr *> RelocationMap;
+    if (Error Err = Obj.getSectionAndRelocations(
+                           [&](const Elf_Shdr &S) { return &S == ELFSection; })
+                        .moveInto(RelocationMap)) {
+      reportUniqueWarning(std::move(Err));
+    }
+
+    std::vector<Relocation<ELFT>> Relocations;
+    const Elf_Shdr *RelocSymTab = nullptr;
+    if (const Elf_Shdr *RelocSection = RelocationMap.lookup(ELFSection)) {
+      forEachRelocationDo(*RelocSection,
+                          [&](const Relocation<ELFT> &R, unsigned Ndx,
+                              const Elf_Shdr &Sec, const Elf_Shdr *SymTab) {
+                            RelocSymTab = SymTab;
+                            Relocations.push_back(R);
+                          });
+      llvm::stable_sort(Relocations, [](const auto &LHS, const auto &RHS) {
+        return LHS.Offset < RHS.Offset;
+      });
+    }
+
     printSFrameHeader(*Parser);
-    printSFrameFDEs(*Parser);
+    printSFrameFDEs(*Parser, Relocations, RelocSymTab);
   }
 }
 
diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
index 3f84e8f6c8d40..b1138ef9d5289 100644
--- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
+++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
@@ -62,6 +62,15 @@ static cl::opt<bool> TmpFilesAsBitcode(
     cl::desc("Always write temporary files as bitcode instead of textual IR"),
     cl::init(false), cl::cat(LLVMReduceOptions));
 
+static SmallVector<MachineBasicBlock *> constructSaveRestorePoints(
+    ArrayRef<MachineBasicBlock *> SRPoints,
+    const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &BBMap) {
+  SmallVector<MachineBasicBlock *> Pts;
+  for (auto &Src : SRPoints)
+    Pts.push_back(BBMap.find(Src)->second);
+  return Pts;
+}
+
 static void cloneFrameInfo(
     MachineFrameInfo &DstMFI, const MachineFrameInfo &SrcMFI,
     const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) {
@@ -92,11 +101,17 @@ static void cloneFrameInfo(
   DstMFI.setCVBytesOfCalleeSavedRegisters(
       SrcMFI.getCVBytesOfCalleeSavedRegisters());
 
-  if (MachineBasicBlock *SavePt = SrcMFI.getSavePoint())
-    DstMFI.setSavePoint(Src2DstMBB.find(SavePt)->second);
-  if (MachineBasicBlock *RestorePt = SrcMFI.getRestorePoint())
-    DstMFI.setRestorePoint(Src2DstMBB.find(RestorePt)->second);
+  assert(SrcMFI.getSavePoints().size() < 2 &&
+         "Multiple restore points not yet supported!");
+
+  DstMFI.setSavePoints(
+      constructSaveRestorePoints(SrcMFI.getSavePoints(), Src2DstMBB));
+
+  assert(SrcMFI.getRestorePoints().size() < 2 &&
+         "Multiple restore points not yet supported!");
 
+  DstMFI.setRestorePoints(
+      constructSaveRestorePoints(SrcMFI.getRestorePoints(), Src2DstMBB));
 
   auto CopyObjectProperties = [](MachineFrameInfo &DstMFI,
                                  const MachineFrameInfo &SrcMFI, int FI) {
diff --git a/llvm/tools/opt/optdriver.cpp b/llvm/tools/opt/optdriver.cpp
index a185ea464decc..26902b213571f 100644
--- a/llvm/tools/opt/optdriver.cpp
+++ b/llvm/tools/opt/optdriver.cpp
@@ -340,7 +340,6 @@ static bool shouldPinPassToLegacyPM(StringRef Pass) {
       "amdgpu-lower-kernel-attributes",
       "amdgpu-propagate-attributes-early",
       "amdgpu-propagate-attributes-late",
-      "amdgpu-unify-metadata",
       "amdgpu-printf-runtime-binding",
       "amdgpu-always-inline"};
   if (llvm::is_contained(PassNameExactToIgnore, Pass))
diff --git a/llvm/unittests/ADT/APFloatTest.cpp b/llvm/unittests/ADT/APFloatTest.cpp
index 9609e8e22a3ed..141282ea254b4 100644
--- a/llvm/unittests/ADT/APFloatTest.cpp
+++ b/llvm/unittests/ADT/APFloatTest.cpp
@@ -16,9 +16,11 @@
 #include "llvm/Support/FormatVariadic.h"
 #include "gtest/gtest.h"
 #include <cmath>
+#include <limits>
 #include <ostream>
 #include <string>
 #include <tuple>
+#include <type_traits>
 
 using namespace llvm;
 
@@ -1658,6 +1660,90 @@ TEST(APFloatTest, toInteger) {
   EXPECT_EQ(APSInt::getMaxValue(5, false), result);
 }
 
+class APFloatConvertFromAPIntParamTest
+    : public ::testing::TestWithParam<const fltSemantics *> {
+protected:
+  // Helper to run a conversion and compare the integer result directly.
+  static void testConversionAndCompareInt(const APInt &InputValue,
+                                          const bool IsSigned,
+                                          APFloat::roundingMode RM,
+                                          const APInt &ExpectedIntValue) {
+    const fltSemantics &Sem = *GetParam();
+    APFloat F(Sem);
+    F.convertFromAPInt(InputValue, /*IsSigned=*/IsSigned, RM);
+
+    APSInt ResultInt(InputValue.getBitWidth(), /*isUnsigned=*/!IsSigned);
+    bool IsExact;
+    F.convertToInteger(ResultInt, APFloat::rmTowardZero, &IsExact);
+
+    EXPECT_TRUE(IsExact);
+    EXPECT_TRUE(ResultInt.eq(ExpectedIntValue))
+        << "InputValue: " << InputValue << "\n"
+        << ResultInt << " vs " << ExpectedIntValue << "\n";
+  }
+};
+
+TEST_P(APFloatConvertFromAPIntParamTest, HalfwayRounding) {
+  const fltSemantics &Sem = *GetParam();
+  const unsigned Precision = APFloat::semanticsPrecision(Sem);
+
+  if (Precision == 0)
+    GTEST_SKIP() << "Skipping test for semantics with no significand.";
+
+  for (bool IsSigned : {false, true}) {
+    const unsigned BitWidth = Precision + 1 + (IsSigned ? 1 : 0);
+
+    const APInt RoundedDownVal = APInt::getOneBitSet(BitWidth, Precision);
+    const APInt HalfwayVal = RoundedDownVal + 1;
+    const APInt RoundedUpVal = RoundedDownVal + 2;
+
+    testConversionAndCompareInt(HalfwayVal, IsSigned,
+                                APFloat::rmNearestTiesToEven, RoundedDownVal);
+    testConversionAndCompareInt(HalfwayVal, IsSigned,
+                                APFloat::rmNearestTiesToAway, RoundedUpVal);
+    testConversionAndCompareInt(HalfwayVal, IsSigned, APFloat::rmTowardPositive,
+                                RoundedUpVal);
+    testConversionAndCompareInt(HalfwayVal, IsSigned, APFloat::rmTowardNegative,
+                                RoundedDownVal);
+    testConversionAndCompareInt(HalfwayVal, IsSigned, APFloat::rmTowardZero,
+                                RoundedDownVal);
+  }
+}
+
+TEST_P(APFloatConvertFromAPIntParamTest, MaxMagnitude) {
+  const fltSemantics &Sem = *GetParam();
+  const unsigned Precision = APFloat::semanticsPrecision(Sem);
+
+  if (Precision == 0)
+    GTEST_SKIP() << "Skipping test for semantics with no significand.";
+
+  const APFloat Largest = APFloat::getLargest(Sem, /*Negative=*/false);
+  const int Exp = ilogb(Largest);
+  for (bool IsSigned : {false, true}) {
+    const unsigned BitWidth = Exp + 1 + (IsSigned ? 1 : 0);
+
+    bool IsExact;
+    APSInt LargestAsInt{BitWidth, /*IsUnsigned=*/!IsSigned};
+    const APFloat::opStatus ToIntStatus =
+        Largest.convertToInteger(LargestAsInt, APFloat::rmTowardZero, &IsExact);
+    EXPECT_EQ(ToIntStatus, APFloat::opOK);
+
+    for (const APFloat::roundingMode RM :
+         {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+          APFloat::rmTowardPositive, APFloat::rmTowardZero,
+          APFloat::rmNearestTiesToEven}) {
+      testConversionAndCompareInt(LargestAsInt, IsSigned, RM, LargestAsInt);
+    }
+  }
+}
+
+INSTANTIATE_TEST_SUITE_P(IEEESemantics, APFloatConvertFromAPIntParamTest,
+                         ::testing::Values(&APFloat::IEEEhalf(),
+                                           &APFloat::BFloat(),
+                                           &APFloat::IEEEsingle(),
+                                           &APFloat::IEEEdouble(),
+                                           &APFloat::IEEEquad()));
+
 static APInt nanbitsFromAPInt(const fltSemantics &Sem, bool SNaN, bool Negative,
                               uint64_t payload) {
   APInt appayload(64, payload);
@@ -1916,6 +2002,15 @@ TEST(APFloatTest, exactInverse) {
   EXPECT_TRUE(inv.bitwiseIsEqual(APFloat(APFloat::PPCDoubleDouble(), "0.5")));
   EXPECT_TRUE(APFloat(APFloat::x87DoubleExtended(), "2.0").getExactInverse(&inv));
   EXPECT_TRUE(inv.bitwiseIsEqual(APFloat(APFloat::x87DoubleExtended(), "0.5")));
+  // 0x1p1022 has a normal inverse for IEEE 754 binary64: 0x1p-1022.
+  EXPECT_TRUE(APFloat(0x1p1022).getExactInverse(&inv));
+  EXPECT_TRUE(inv.bitwiseIsEqual(APFloat(0x1p-1022)));
+  // With regards to getExactInverse, IEEEdouble and PPCDoubleDouble should
+  // behave the same.
+  EXPECT_TRUE(
+      APFloat(APFloat::PPCDoubleDouble(), "0x1p1022").getExactInverse(&inv));
+  EXPECT_TRUE(
+      inv.bitwiseIsEqual(APFloat(APFloat::PPCDoubleDouble(), "0x1p-1022")));
 
   // FLT_MIN
   EXPECT_TRUE(APFloat(1.17549435e-38f).getExactInverse(&inv));
@@ -1927,6 +2022,8 @@ TEST(APFloatTest, exactInverse) {
   EXPECT_FALSE(APFloat(0.0).getExactInverse(nullptr));
   // Denormalized float
   EXPECT_FALSE(APFloat(1.40129846e-45f).getExactInverse(nullptr));
+  // Largest subnormal
+  EXPECT_FALSE(APFloat(0x1p-127f).getExactInverse(nullptr));
 }
 
 TEST(APFloatTest, roundToIntegral) {
@@ -2661,6 +2758,37 @@ TEST(APFloatTest, Float8UZConvert) {
   }
 }
 
+struct DD {
+  double Hi;
+  double Lo;
+};
+
+template <typename T, typename U>
+static APFloat makeDoubleAPFloat(T Hi, U Lo) {
+  APFloat HiFloat{APFloat::IEEEdouble(), APFloat::uninitialized};
+  if constexpr (std::is_same_v<decltype(Hi), APFloat>) {
+    HiFloat = Hi;
+  } else if constexpr (std::is_same_v<decltype(Hi), double>) {
+    HiFloat = APFloat{Hi};
+  } else {
+    HiFloat = {APFloat::IEEEdouble(), Hi};
+  }
+
+  APFloat LoFloat{APFloat::IEEEdouble(), APFloat::uninitialized};
+  if constexpr (std::is_same_v<decltype(Lo), APFloat>) {
+    LoFloat = Lo;
+  } else if constexpr (std::is_same_v<decltype(Lo), double>) {
+    LoFloat = APFloat{Lo};
+  } else {
+    LoFloat = {APFloat::IEEEdouble(), Lo};
+  }
+
+  APInt Bits = LoFloat.bitcastToAPInt().concat(HiFloat.bitcastToAPInt());
+  return APFloat(APFloat::PPCDoubleDouble(), Bits);
+}
+
+static APFloat makeDoubleAPFloat(DD X) { return makeDoubleAPFloat(X.Hi, X.Lo); }
+
 TEST(APFloatTest, PPCDoubleDouble) {
   APFloat test(APFloat::PPCDoubleDouble(), "1.0");
   EXPECT_EQ(0x3ff0000000000000ull, test.bitcastToAPInt().getRawData()[0]);
@@ -5315,18 +5443,809 @@ TEST(APFloatTest, PPCDoubleDoubleFMA) {
             APFloat(APFloat::PPCDoubleDouble(), "10").compare(A));
 }
 
-TEST(APFloatTest, PPCDoubleDoubleRoundToIntegral) {
-  {
-    APFloat A(APFloat::PPCDoubleDouble(), "1.5");
-    A.roundToIntegral(APFloat::rmNearestTiesToEven);
-    EXPECT_EQ(APFloat::cmpEqual,
-              APFloat(APFloat::PPCDoubleDouble(), "2").compare(A));
+struct PPCDoubleDoubleRoundToIntegralTestCase {
+  DD Input;
+  DD Rounded[5] = {};
+  constexpr PPCDoubleDoubleRoundToIntegralTestCase &
+  withRounded(DD R, APFloat::roundingMode RM) {
+    Rounded[static_cast<std::underlying_type_t<APFloat::roundingMode>>(RM)] = R;
+    return *this;
   }
-  {
-    APFloat A(APFloat::PPCDoubleDouble(), "2.5");
-    A.roundToIntegral(APFloat::rmNearestTiesToEven);
-    EXPECT_EQ(APFloat::cmpEqual,
-              APFloat(APFloat::PPCDoubleDouble(), "2").compare(A));
+};
+
+auto ppcDoubleDoubleRoundToIntegralTests() {
+  constexpr double Eps = std::numeric_limits<double>::epsilon();
+  constexpr double HalfEps = Eps / 2.0;
+  constexpr double QuarterEps = Eps / 4.0;
+  constexpr double SmallestNormal = std::numeric_limits<double>::min();
+  constexpr double EvenIntegerThreshold{uint64_t{1}
+                                        << std::numeric_limits<double>::digits};
+  constexpr double Inf = std::numeric_limits<double>::infinity();
+  constexpr double QNaN = std::numeric_limits<double>::quiet_NaN();
+  using TestCase = PPCDoubleDoubleRoundToIntegralTestCase;
+  static constexpr auto TestCases = std::array{
+      // 1. Zeros and Basic Integers
+      // Input: Positive Zero (0.0, 0.0)
+      TestCase({{0.0, 0.0}})
+          .withRounded({0.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({0.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({0.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({0.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({0.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Negative Zero (-0.0, 0.0)
+      TestCase({{-0.0, 0.0}})
+          .withRounded({-0.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-0.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-0.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-0.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-0.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Positive Even (2.0, 0.0)
+      TestCase({{2.0, 0.0}})
+          .withRounded({2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Positive Odd (3.0, 0.0)
+      TestCase({{3.0, 0.0}})
+          .withRounded({3.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({3.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({3.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({3.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({3.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Negative Even (-2.0, 0.0)
+      TestCase({{-2.0, 0.0}})
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // 2. General Fractions (Non-Ties)
+      // Input: 2.3
+      TestCase({{2.3, 0.0}})
+          .withRounded({2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({3.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: 2.7
+      TestCase({{2.7, 0.0}})
+          .withRounded({2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({3.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({3.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({3.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: -2.3
+      TestCase({{-2.3, 0.0}})
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-3.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: -2.7
+      TestCase({{-2.7, 0.0}})
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-3.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-3.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-3.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: 2.3 + Tiny
+      TestCase({{2.3, SmallestNormal}})
+          .withRounded({2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({3.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // 3. Exact Midpoints (Ties at N.5)
+      // Input: 0.5
+      TestCase({{0.5, 0.0}})
+          .withRounded({0.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({0.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({1.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({1.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({0.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: 1.5 (Odd base)
+      TestCase({{1.5, 0.0}})
+          .withRounded({1.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({1.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: 2.5 (Even base)
+      TestCase({{2.5, 0.0}})
+          .withRounded({2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({3.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({3.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: -0.5
+      TestCase({{-0.5, 0.0}})
+          .withRounded({-0.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-1.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-0.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-1.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-0.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: -1.5 (Odd base)
+      TestCase({{-1.5, 0.0}})
+          .withRounded({-1.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-1.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: -2.5 (Even base)
+      TestCase({{-2.5, 0.0}})
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-3.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-3.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // 4. Near Midpoints (lo breaks the tie)
+      // Input: Slightly > 2.5
+      TestCase({{2.5, SmallestNormal}})
+          .withRounded({2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({3.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({3.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({3.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly < 2.5
+      TestCase({{2.5, -SmallestNormal}})
+          .withRounded({2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({3.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly > 1.5
+      TestCase({{1.5, SmallestNormal}})
+          .withRounded({1.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({1.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly < 1.5
+      TestCase({{1.5, -SmallestNormal}})
+          .withRounded({1.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({1.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({1.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({1.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly > -2.5 (closer to 0)
+      TestCase({{-2.5, SmallestNormal}})
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-3.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly < -2.5 (further from 0)
+      TestCase({{-2.5, -SmallestNormal}})
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-3.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-3.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-3.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // 5. Near Integers (lo crosses the integer boundary)
+      // Input: Slightly > 2.0
+      TestCase({{2.0, SmallestNormal}})
+          .withRounded({2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({3.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly < 2.0 (1.99...)
+      TestCase({{2.0, -SmallestNormal}})
+          .withRounded({1.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({1.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly > -2.0 (-1.99...)
+      TestCase({{-2.0, SmallestNormal}})
+          .withRounded({-1.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-1.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly < -2.0
+      TestCase({{-2.0, -SmallestNormal}})
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-3.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-2.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly > 0.0
+      TestCase({{SmallestNormal, 0.0}})
+          .withRounded({0.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({0.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({1.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({0.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({0.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: Slightly < 0.0
+      TestCase({{-SmallestNormal, 0.0}})
+          .withRounded({-0.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-1.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-0.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-0.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-0.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // 6. Boundary of Canonicalization (Maximum lo)
+      // Input: 1.0 + Max lo (1 + 2^-53)
+      TestCase({{1.0, HalfEps}})
+          .withRounded({1.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({1.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({2.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({1.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({1.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: 1.0 - Max lo (1 - 2^-54)
+      TestCase({{1.0, -QuarterEps}})
+          .withRounded({0.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({0.0, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({1.0, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({1.0, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({1.0, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // 7. Large Magnitudes (Beyond 2^53). N = EvenIntegerThreshold (Even)
+      // Input: EvenIntegerThreshold (Exact)
+      TestCase({{EvenIntegerThreshold, 0.0}})
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold, 0.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold, 0.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Input: EvenIntegerThreshold+1 (Exact)
+      TestCase({{EvenIntegerThreshold, 1.0}})
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold, 1.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold, 1.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Fractions
+      // Input: EvenIntegerThreshold+0.25
+      TestCase({{EvenIntegerThreshold, 0.25}})
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold, 0.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold, 0.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Input: EvenIntegerThreshold+0.75
+      TestCase({{EvenIntegerThreshold, 0.75}})
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold, 1.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold, 1.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Ties (Midpoints)
+      // Input: EvenIntegerThreshold-0.5
+      TestCase({{EvenIntegerThreshold, -0.5}})
+          .withRounded({EvenIntegerThreshold - 1.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold - 1.0, 0.0},
+                       APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold, 0.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold, 0.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Input: EvenIntegerThreshold+0.5
+      TestCase({{EvenIntegerThreshold, 0.5}})
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold, 1.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold, 0.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Input: EvenIntegerThreshold+1.5
+      TestCase({{EvenIntegerThreshold + 2.0, -0.5}})
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold + 2.0, 0.0},
+                       APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold + 2.0, 0.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold + 2.0, 0.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Input: EvenIntegerThreshold+2.5
+      TestCase({{EvenIntegerThreshold + 2.0, 0.5}})
+          .withRounded({EvenIntegerThreshold + 2.0, 0.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold + 2.0, 0.0},
+                       APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold + 4.0, -1.0},
+                       APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold + 4.0, -1.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold + 2.0, 0.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Near Ties
+      // Input: EvenIntegerThreshold+0.5+HalfEps
+      TestCase({{EvenIntegerThreshold, 0.5 + HalfEps}})
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold, 1.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold, 1.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Input: EvenIntegerThreshold+0.5-QuarterEps
+      TestCase({{EvenIntegerThreshold, 0.5 - QuarterEps}})
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold, 0.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold, 0.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // Canonical Boundary (Max lo for EvenIntegerThreshold is 1.0)
+      // Input: EvenIntegerThreshold+1.0
+      TestCase({{EvenIntegerThreshold, 1.0}})
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardZero)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardNegative)
+          .withRounded({EvenIntegerThreshold, 1.0}, APFloat::rmTowardPositive)
+          .withRounded({EvenIntegerThreshold, 1.0},
+                       APFloat::rmNearestTiesToAway)
+          .withRounded({EvenIntegerThreshold, 1.0},
+                       APFloat::rmNearestTiesToEven),
+
+      // 8. Special Values
+      // Input: +Inf
+      TestCase({{Inf, 0.0}})
+          .withRounded({Inf, 0.0}, APFloat::rmTowardZero)
+          .withRounded({Inf, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({Inf, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({Inf, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({Inf, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: -Inf
+      TestCase({{-Inf, 0.0}})
+          .withRounded({-Inf, 0.0}, APFloat::rmTowardZero)
+          .withRounded({-Inf, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({-Inf, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({-Inf, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({-Inf, 0.0}, APFloat::rmNearestTiesToEven),
+
+      // Input: NaN input hi. Expected output canonical (NaN, 0.0).
+      TestCase({{QNaN, 0.0}})
+          .withRounded({QNaN, 0.0}, APFloat::rmTowardZero)
+          .withRounded({QNaN, 0.0}, APFloat::rmTowardNegative)
+          .withRounded({QNaN, 0.0}, APFloat::rmTowardPositive)
+          .withRounded({QNaN, 0.0}, APFloat::rmNearestTiesToAway)
+          .withRounded({QNaN, 0.0}, APFloat::rmNearestTiesToEven),
+  };
+  return TestCases;
+}
+
+class PPCDoubleDoubleRoundToIntegralValueTest
+    : public testing::Test,
+      public ::testing::WithParamInterface<
+          PPCDoubleDoubleRoundToIntegralTestCase> {};
+
+INSTANTIATE_TEST_SUITE_P(
+    PPCDoubleDoubleRoundToIntegralValueParamTests,
+    PPCDoubleDoubleRoundToIntegralValueTest,
+    ::testing::ValuesIn(ppcDoubleDoubleRoundToIntegralTests()));
+
+TEST_P(PPCDoubleDoubleRoundToIntegralValueTest,
+       PPCDoubleDoubleRoundToIntegral) {
+  const PPCDoubleDoubleRoundToIntegralTestCase TestCase = GetParam();
+  const APFloat Input = makeDoubleAPFloat(TestCase.Input);
+  EXPECT_FALSE(Input.isDenormal())
+      << TestCase.Input.Hi << " + " << TestCase.Input.Lo;
+  for (size_t I = 0, E = std::size(TestCase.Rounded); I != E; ++I) {
+    const auto RM = static_cast<APFloat::roundingMode>(I);
+    const APFloat Expected = makeDoubleAPFloat(TestCase.Rounded[I]);
+    EXPECT_FALSE(Expected.isDenormal())
+        << TestCase.Rounded[I].Hi << " + " << TestCase.Input.Lo;
+    APFloat Actual = Input;
+    Actual.roundToIntegral(RM);
+    if (Actual.isNaN())
+      EXPECT_TRUE(Actual.isNaN());
+    else
+      EXPECT_EQ(Actual.compare(Expected), APFloat::cmpEqual)
+          << "RM: " << RM << " Input.Hi: " << TestCase.Input.Hi
+          << " Input.Lo: " << TestCase.Input.Lo << " Actual: " << Actual
+          << " Expected.Hi: " << TestCase.Rounded[I].Hi
+          << " Expected.Lo: " << TestCase.Rounded[I].Lo
+          << " Expected: " << Expected;
+  }
+}
+
+namespace PPCDoubleDoubleConvertToIntegerTestDetails {
+// Define the rounding modes for easier readability.
+static constexpr auto RNE = APFloat::rmNearestTiesToEven;
+static constexpr auto RNA = APFloat::rmNearestTiesToAway;
+static constexpr auto RTZ = APFloat::rmTowardZero;
+static constexpr auto RUP = APFloat::rmTowardPositive;
+static constexpr auto RDN = APFloat::rmTowardNegative;
+
+struct TestCase {
+  // Structure to hold the expected result of a conversion
+  struct ExpectedConversion {
+    // The expected integer value represented as a string (decimal).
+    // We use a string to easily represent arbitrary precision values in
+    // constexpr. The test runner should parse this into an APSInt matching the
+    // test configuration.
+    const char *ExpectedIntStr;
+    APFloat::opStatus Status;
+  };
+
+  DD Input;
+  unsigned IntegerWidth;
+  bool IsSigned;
+  // Array indexed by the rounding mode enum value.
+  std::array<ExpectedConversion, 5> Rounded = {};
+
+  // Helper to define the expected results for a specific rounding mode.
+  constexpr TestCase &with(APFloat::roundingMode RM, const char *ExpectedStr,
+                           APFloat::opStatus Status) {
+    Rounded[static_cast<std::underlying_type_t<APFloat::roundingMode>>(RM)] = {
+        ExpectedStr,
+        Status,
+    };
+    return *this;
+  }
+
+  // Helper to define the same result for all rounding modes.
+  constexpr TestCase &withAll(const char *ExpectedStr,
+                              APFloat::opStatus Status) {
+    return with(RNE, ExpectedStr, Status)
+        .with(RNA, ExpectedStr, Status)
+        .with(RTZ, ExpectedStr, Status)
+        .with(RUP, ExpectedStr, Status)
+        .with(RDN, ExpectedStr, Status);
+  }
+};
+
+auto testCases() {
+  // Define the status codes.
+  constexpr auto OK = llvm::APFloat::opOK;
+  constexpr auto Inexact = llvm::APFloat::opInexact;
+  // The API specifies opInvalidOp for out-of-range (overflow/underflow) and
+  // NaN.
+  constexpr auto Invalid = llvm::APFloat::opInvalidOp;
+
+  // Helper constants for constructing specific DD values.
+  constexpr double Infinity = std::numeric_limits<double>::infinity();
+  constexpr double NaN = std::numeric_limits<double>::quiet_NaN();
+  constexpr double DMAX = std::numeric_limits<double>::max();
+
+  // Powers of 2
+  constexpr double P53 = 0x1p53;
+  constexpr double P63 = 0x1p63;
+  constexpr double P64 = 0x1p64;
+  // 2^-100 (A very small delta demonstrating extended precision)
+  constexpr double PM100 = 0x1p-100;
+
+  static constexpr auto ConvertToIntegerTestCases = std::array{
+      // 1. Zeros, NaNs, and Infinities (Target: 64-bit Signed)
+      // INT64_MAX = 9223372036854775807
+      // INT64_MIN = -9223372036854775808
+
+      // Input: Positive Zero (0.0, 0.0)
+      TestCase{{0.0, 0.0}, 64, true}.withAll("0", OK),
+
+      // Input: Negative Zero (-0.0, 0.0)
+      TestCase{{-0.0, 0.0}, 64, true}.withAll("0", OK),
+
+      // Input: NaN. Expected behavior: Invalid, deterministic result of 0.
+      TestCase{{NaN, 0.0}, 64, true}.withAll("0", Invalid),
+
+      // Input: +Infinity. Expected behavior: Invalid, deterministic result of
+      // INT64_MAX.
+      TestCase{{Infinity, 0.0}, 64, true}.withAll("9223372036854775807",
+                                                  Invalid),
+
+      // Input: -Infinity. Expected behavior: Invalid, deterministic result of
+      // INT64_MIN.
+      TestCase{{-Infinity, 0.0}, 64, true}.withAll("-9223372036854775808",
+                                                   Invalid),
+
+      // 2. Basic Rounding and Tie-Breaking (Target: 32-bit Signed)
+
+      // Input: 2.5 (Tie, preceding integer is Even)
+      TestCase{{2.5, 0.0}, 32, true}
+          .with(RTZ, "2", Inexact)
+          .with(RDN, "2", Inexact)
+          .with(RUP, "3", Inexact)
+          .with(RNA, "3", Inexact)
+          .with(RNE, "2", Inexact),
+
+      // Input: 3.5 (Tie, preceding integer is Odd)
+      TestCase{{3.5, 0.0}, 32, true}
+          .with(RTZ, "3", Inexact)
+          .with(RDN, "3", Inexact)
+          .with(RUP, "4", Inexact)
+          .with(RNA, "4", Inexact)
+          .with(RNE, "4", Inexact),
+
+      // Input: -2.5 (Tie, preceding integer is Even)
+      TestCase{{-2.5, 0.0}, 32, true}
+          .with(RTZ, "-2", Inexact)
+          .with(RDN, "-3", Inexact)
+          .with(RUP, "-2", Inexact)
+          .with(RNA, "-3", Inexact)
+          .with(RNE, "-2", Inexact),
+
+      // 3. Double-Double Precision (The role of 'lo')
+      // Testing how extended precision affects rounding decisions.
+
+      // Input: 2.5 + Epsilon (Slightly above tie)
+      TestCase{{2.5, PM100}, 32, true}
+          .with(RTZ, "2", Inexact)
+          .with(RDN, "2", Inexact)
+          .with(RUP, "3", Inexact)
+          .with(RNA, "3", Inexact)
+          .with(RNE, "3", Inexact),
+
+      // Input: 2.5 - Epsilon (Slightly below tie)
+      TestCase{{2.5, -PM100}, 32, true}
+          .with(RTZ, "2", Inexact)
+          .with(RDN, "2", Inexact)
+          .with(RUP, "3", Inexact)
+          .with(RNA, "2", Inexact)
+          .with(RNE, "2", Inexact),
+
+      // Input: 1.0 + Epsilon (Just above 1.0, e.g., 1.00...1)
+      TestCase{{1.0, PM100}, 32, true}
+          .with(RTZ, "1", Inexact)
+          .with(RDN, "1", Inexact)
+          .with(RUP, "2", Inexact)
+          .with(RNA, "1", Inexact)
+          .with(RNE, "1", Inexact),
+
+      // Input: 1.0 - Epsilon (Just below 1.0, e.g. 0.999...)
+      TestCase{{1.0, -PM100}, 32, true}
+          .with(RTZ, "0", Inexact)
+          .with(RDN, "0", Inexact)
+          .with(RUP, "1", Inexact)
+          .with(RNA, "1", Inexact)
+          .with(RNE, "1", Inexact),
+
+      // Input: Large number tie-breaking (Crucial test for DD implementation)
+      // Input: 2^53 + 1.5.
+      // A standard double(2^53 + 1.5) rounds to 2^53 + 2.0.
+      // The DD representation must precisely hold 2^53 + 1.5.
+      // The canonical DD representation is {2^53 + 2.0, -0.5}.
+      // Value is 9007199254740993.5
+      TestCase{{P53 + 2.0, -0.5}, 64, true}
+          .with(RTZ, "9007199254740993", Inexact)
+          .with(RDN, "9007199254740993", Inexact)
+          .with(RUP, "9007199254740994", Inexact)
+          .with(RNA, "9007199254740994", Inexact)
+          .with(RNE, "9007199254740994", Inexact),
+
+      // 4. Overflow Boundaries (Signed)
+
+      // Input: Exactly INT64_MAX. (2^63 - 1)
+      // Represented precisely as (2^63, -1.0)
+      TestCase{{P63, -1.0}, 64, true}.withAll("9223372036854775807", OK),
+
+      // Input: INT64_MAX + 0.3.
+      // Represented as (2^63, -0.7)
+      TestCase{{P63, -0.7}, 64, true}
+          .with(RTZ, "9223372036854775807", Inexact)
+          .with(RDN, "9223372036854775807", Inexact)
+          .with(RNA, "9223372036854775807", Inexact)
+          .with(RNE, "9223372036854775807", Inexact)
+          .with(RUP, "9223372036854775807", Invalid),
+
+      // Input: INT64_MAX + 0.5 (Tie at the boundary)
+      // Represented as (2^63, -0.5). Target integers are MAX (odd) and 2^63
+      // (even).
+      TestCase{{P63, -0.5}, 64, true}
+          .with(RTZ, "9223372036854775807", Inexact)
+          .with(RDN, "9223372036854775807", Inexact)
+          .with(RUP, "9223372036854775807", Invalid)
+          .with(RNA, "9223372036854775807", Invalid)
+          .with(RNE, "9223372036854775807", Invalid),
+
+      // Input: 2^55 - 2^1 - 2^-52 to signed integer.
+      // Represented as (2^55 - 2^2, 2^1 - 2^-1).
+      TestCase{{0x1.fffffffffffffp+54, 0x1.8p0}, 56, true}
+          .with(RTZ, "36028797018963965", Inexact)
+          .with(RDN, "36028797018963965", Inexact)
+          .with(RUP, "36028797018963966", Inexact)
+          .with(RNA, "36028797018963966", Inexact)
+          .with(RNE, "36028797018963966", Inexact),
+
+      // Input: 2^55 - 2^1 - 2^-52 to signed integer.
+      // Represented as (2^55 - 2^2, 2^1 - 2^-52).
+      TestCase{{0x1.fffffffffffffp+54, 0x1.fffffffffffffp0}, 56, true}
+          .with(RTZ, "36028797018963965", Inexact)
+          .with(RDN, "36028797018963965", Inexact)
+          .with(RUP, "36028797018963966", Inexact)
+          .with(RNA, "36028797018963966", Inexact)
+          .with(RNE, "36028797018963966", Inexact),
+
+      // Input: Exactly 2^63 (One past INT64_MAX)
+      TestCase{{P63, 0.0}, 64, true}.withAll("9223372036854775807", Invalid),
+
+      // Input: Exactly INT64_MIN (-2^63)
+      TestCase{{-P63, 0.0}, 64, true}.withAll("-9223372036854775808", OK),
+
+      // Input: INT64_MIN - 0.5 (Tie at the lower boundary)
+      // Target integers are -2^63-1 (odd) and MIN (even).
+      TestCase{{-P63, -0.5}, 64, true}
+          .with(RTZ, "-9223372036854775808", Inexact)
+          .with(RUP, "-9223372036854775808", Inexact)
+          // RDN rounds down, causing overflow.
+          .with(RDN, "-9223372036854775808", Invalid)
+          // RNA rounds away (down), causing overflow.
+          .with(RNA, "-9223372036854775808", Invalid)
+          // RNE rounds to even (up to -2^63), which is OK.
+          .with(RNE, "-9223372036854775808", Inexact),
+
+      // 5. Overflow Boundaries (Unsigned)
+      // UINT64_MAX = 18446744073709551615 (2^64 - 1)
+
+      // Input: Exactly UINT64_MAX. (2^64 - 1)
+      // Represented precisely as (2^64, -1.0)
+      TestCase{{P64, -1.0}, 64, false}.withAll("18446744073709551615", OK),
+
+      // Input: UINT64_MAX + 0.5 (Tie at the boundary)
+      // Represented as (2^64, -0.5)
+      TestCase{{P64, -0.5}, 64, false}
+          .with(RTZ, "18446744073709551615", Inexact)
+          .with(RDN, "18446744073709551615", Inexact)
+          // RUP rounds up (2^64), causing overflow.
+          .with(RUP, "18446744073709551615", Invalid)
+          // RNA rounds away (up), causing overflow.
+          .with(RNA, "18446744073709551615", Invalid)
+          // RNE rounds to even (up to 2^64), causing overflow.
+          .with(RNE, "18446744073709551615", Invalid),
+
+      // Input: 2^55 - 2^1 - 2^-52 to unsigned integer.
+      // Represented as (2^55 - 2^2, 2^1 - 2^-1).
+      TestCase{{0x1.fffffffffffffp+54, 0x1.8p0}, 55, false}
+          .with(RTZ, "36028797018963965", Inexact)
+          .with(RDN, "36028797018963965", Inexact)
+          .with(RUP, "36028797018963966", Inexact)
+          .with(RNA, "36028797018963966", Inexact)
+          .with(RNE, "36028797018963966", Inexact),
+
+      // Input: 2^55 - 2^1 - 2^-52 to unsigned integer.
+      // Represented as (2^55 - 2^2, 2^1 - 2^-52).
+      TestCase{{0x1.fffffffffffffp+54, 0x1.fffffffffffffp0}, 55, false}
+          .with(RTZ, "36028797018963965", Inexact)
+          .with(RDN, "36028797018963965", Inexact)
+          .with(RUP, "36028797018963966", Inexact)
+          .with(RNA, "36028797018963966", Inexact)
+          .with(RNE, "36028797018963966", Inexact),
+
+      // Input: -0.3 (Slightly below zero)
+      TestCase{{-0.3, 0.0}, 64, false}
+          .with(RTZ, "0", Inexact)
+          .with(RUP, "0", Inexact)
+          .with(RNA, "0", Inexact)
+          .with(RNE, "0", Inexact)
+          .with(RDN, "0", Invalid),
+
+      // Input: -0.5 (Tie at zero)
+      TestCase{{-0.5, 0.0}, 64, false}
+          .with(RTZ, "0", Inexact)
+          .with(RUP, "0", Inexact)
+          // RNE rounds to even (0).
+          .with(RNE, "0", Inexact)
+          .with(RDN, "0", Invalid)
+          // RNA rounds away (-1), causing overflow.
+          .with(RNA, "0", Invalid),
+
+      // Input: -1.0 (Negative integer)
+      TestCase{{-1.0, 0.0}, 64, false}.withAll("0", Invalid),
+
+      // 6. High Precision Integers (Target: 128-bit Signed)
+      // INT128_MAX = 170141183460469231731687303715884105727
+
+      // Input: 2^100 (Exactly representable in DD)
+      // 2^100 = 1267650600228229401496703205376.0
+      TestCase{{1267650600228229401496703205376.0, 0.0}, 128, true}.withAll(
+          "1267650600228229401496703205376", OK),
+
+      // Input: DMAX. (Approx 1.8e308).
+      // This is vastly larger than INT128_MAX (Approx 1.7e38).
+      TestCase{{DMAX, 0.0}, 128, true}.withAll(
+          "170141183460469231731687303715884105727", Invalid),
+
+      // Input: Largest semPPCDoubleDoubleLegacy
+      TestCase{{DMAX, 0x1.ffffffffffffep+969}, 128, true}.withAll(
+          "170141183460469231731687303715884105727", Invalid),
+
+      // 7. Round to negative -0
+      TestCase{{-PM100, 0.0}, 32, true}
+          .with(RTZ, "0", Inexact)
+          .with(RUP, "0", Inexact)
+          .with(RNA, "0", Inexact)
+          .with(RNE, "0", Inexact)
+          .with(RDN, "-1", Inexact),
+  };
+  return ConvertToIntegerTestCases;
+}
+} // namespace PPCDoubleDoubleConvertToIntegerTestDetails
+
+class PPCDoubleDoubleConvertToIntegerValueTest
+    : public testing::Test,
+      public ::testing::WithParamInterface<
+          PPCDoubleDoubleConvertToIntegerTestDetails::TestCase> {};
+
+INSTANTIATE_TEST_SUITE_P(
+    PPCDoubleDoubleConvertToIntegerValueParamTests,
+    PPCDoubleDoubleConvertToIntegerValueTest,
+    ::testing::ValuesIn(
+        PPCDoubleDoubleConvertToIntegerTestDetails::testCases()));
+
+TEST_P(PPCDoubleDoubleConvertToIntegerValueTest,
+       PPCDoubleDoubleConvertToInteger) {
+  const PPCDoubleDoubleConvertToIntegerTestDetails::TestCase Params =
+      GetParam();
+  const APFloat Input = makeDoubleAPFloat(Params.Input);
+  EXPECT_FALSE(Input.isDenormal())
+      << Params.Input.Hi << " + " << Params.Input.Lo;
+
+  for (size_t I = 0, E = std::size(Params.Rounded); I != E; ++I) {
+    const auto RM = static_cast<APFloat::roundingMode>(I);
+    const auto &Expected = Params.Rounded[I];
+    APSInt ActualInteger(Params.IntegerWidth, /*isUnsigned=*/!Params.IsSigned);
+
+    APSInt ExpectedInteger{Expected.ExpectedIntStr};
+    EXPECT_LE(ExpectedInteger.getBitWidth(), Params.IntegerWidth);
+    ExpectedInteger = ExpectedInteger.extend(Params.IntegerWidth);
+    if (ExpectedInteger.isUnsigned() && Params.IsSigned) {
+      ExpectedInteger.setIsSigned(Params.IsSigned);
+      EXPECT_FALSE(ExpectedInteger.isNegative());
+    }
+
+    const bool NegativeUnderflow =
+        ExpectedInteger.isZero() && Input.isNegative();
+    const bool ExpectedIsExact =
+        Expected.Status == APFloat::opOK && !NegativeUnderflow;
+    bool ActualIsExact;
+    const auto ActualStatus =
+        Input.convertToInteger(ActualInteger, RM, &ActualIsExact);
+    EXPECT_EQ(ActualStatus, Expected.Status);
+    EXPECT_EQ(ActualIsExact, ExpectedIsExact);
+    EXPECT_EQ(ActualInteger, ExpectedInteger);
   }
 }
 
@@ -5372,6 +6291,102 @@ TEST(APFloatTest, PPCDoubleDoubleCompare) {
   }
 }
 
+namespace PPCDoubleDoubleCompareAbsoluteValueTestDetails {
+struct TestCase {
+  DD LHS;
+  DD RHS;
+  APFloat::cmpResult Result;
+};
+
+auto testCases() {
+  static constexpr auto CompareAbsoluteValueTestCases = std::array{
+      TestCase{
+          {1.0, 0.0},
+          {1.0, 0.0},
+          APFloat::cmpEqual,
+      },
+      TestCase{
+          {1.0, -0.0},
+          {1.0, +0.0},
+          APFloat::cmpEqual,
+      },
+      TestCase{
+          {1.0, 0.0},
+          {0x1.0000000000001p+0, 0.0},
+          APFloat::cmpLessThan,
+      },
+      TestCase{
+          {0x1.0000000000001p+0, 0.0},
+          {1.0, 0.0},
+          APFloat::cmpGreaterThan,
+      },
+      TestCase{
+          {0x1.0000000000001p+0, +0x1p-1074},
+          {1.0, -0x1p-1074},
+          APFloat::cmpGreaterThan,
+      },
+      TestCase{
+          {0x1.0000000000001p+0, -0x1p-1074},
+          {1.0, +0x1p-1074},
+          APFloat::cmpGreaterThan,
+      },
+      TestCase{
+          {1.0, 0.0},
+          {1.0, -0x1p-1074},
+          APFloat::cmpGreaterThan,
+      },
+      TestCase{
+          {1.0, 0.0},
+          {1.0, +0x1p-1074},
+          APFloat::cmpLessThan,
+      },
+      TestCase{
+          {1.0, +0x1p-1073},
+          {1.0, -0x1p-1074},
+          APFloat::cmpGreaterThan,
+      },
+      TestCase{
+          {1.0, +0x1p-1074},
+          {1.0, -0x1p-1074},
+          APFloat::cmpGreaterThan,
+      },
+  };
+  return CompareAbsoluteValueTestCases;
+}
+} // namespace PPCDoubleDoubleCompareAbsoluteValueTestDetails
+
+class PPCDoubleDoubleCompareAbsoluteValueValueTest
+    : public testing::Test,
+      public ::testing::WithParamInterface<
+          PPCDoubleDoubleCompareAbsoluteValueTestDetails::TestCase> {};
+
+INSTANTIATE_TEST_SUITE_P(
+    PPCDoubleDoubleCompareAbsoluteValueValueParamTests,
+    PPCDoubleDoubleCompareAbsoluteValueValueTest,
+    ::testing::ValuesIn(
+        PPCDoubleDoubleCompareAbsoluteValueTestDetails::testCases()));
+
+TEST_P(PPCDoubleDoubleCompareAbsoluteValueValueTest,
+       PPCDoubleDoubleCompareAbsoluteValue) {
+  auto Param = GetParam();
+  for (bool LHSNegate : {false, true}) {
+    auto LHS = llvm::detail::DoubleAPFloat{APFloat::PPCDoubleDouble(),
+                                           APFloat{Param.LHS.Hi},
+                                           APFloat{Param.LHS.Lo}};
+    if (LHSNegate)
+      LHS.changeSign();
+    for (bool RHSNegate : {false, true}) {
+      auto RHS = llvm::detail::DoubleAPFloat{APFloat::PPCDoubleDouble(),
+                                             APFloat{Param.RHS.Hi},
+                                             APFloat{Param.RHS.Lo}};
+      if (RHSNegate)
+        RHS.changeSign();
+
+      EXPECT_EQ(LHS.compareAbsoluteValue(RHS), Param.Result);
+    }
+  }
+}
+
 TEST(APFloatTest, PPCDoubleDoubleBitwiseIsEqual) {
   using DataType = std::tuple<uint64_t, uint64_t, uint64_t, uint64_t, bool>;
 
@@ -5525,19 +6540,328 @@ TEST(APFloatTest, PPCDoubleDoubleScalbn) {
   EXPECT_EQ(0x3cc8000000000000ull, Result.bitcastToAPInt().getRawData()[1]);
 }
 
-TEST(APFloatTest, PPCDoubleDoubleFrexp) {
-  // 3.0 + 3.0 << 53
-  uint64_t Input[] = {
-      0x4008000000000000ull, 0x3cb8000000000000ull,
+namespace PPCDoubleDoubleFrexpTestDetails {
+// Define the rounding modes for easier readability.
+static constexpr auto RNE = APFloat::rmNearestTiesToEven;
+static constexpr auto RNA = APFloat::rmNearestTiesToAway;
+static constexpr auto RTZ = APFloat::rmTowardZero;
+static constexpr auto RUP = APFloat::rmTowardPositive;
+static constexpr auto RDN = APFloat::rmTowardNegative;
+
+struct TestCase {
+  // Structure to hold the expected result of a conversion
+  struct ExpectedFractionExponent {
+    DD Fraction;
+    int Exponent;
+    friend APFloat::cmpResult compare(const ExpectedFractionExponent &Lhs,
+                                      const ExpectedFractionExponent &Rhs) {
+      const APFloat LhsFraction = makeDoubleAPFloat(Lhs.Fraction);
+      const APFloat RhsFraction = makeDoubleAPFloat(Rhs.Fraction);
+      const APFloat::cmpResult FractionRelation =
+          LhsFraction.compare(RhsFraction);
+      if (FractionRelation == APFloat::cmpUnordered)
+        return APFloat::cmpUnordered;
+      if (LhsFraction.isZero() && RhsFraction.isZero())
+        return APFloat::cmpEqual;
+      if (!LhsFraction.isNegative() &&
+          (RhsFraction.isNegative() || RhsFraction.isZero()))
+        return APFloat::cmpGreaterThan;
+      if (!RhsFraction.isNegative() &&
+          (LhsFraction.isNegative() || LhsFraction.isZero()))
+        return APFloat::cmpLessThan;
+      if (Lhs.Exponent > Rhs.Exponent)
+        return LhsFraction.isNegative() ? APFloat::cmpLessThan
+                                        : APFloat::cmpGreaterThan;
+      if (Lhs.Exponent < Rhs.Exponent)
+        return RhsFraction.isNegative() ? APFloat::cmpGreaterThan
+                                        : APFloat::cmpLessThan;
+      return FractionRelation;
+    }
   };
-  int Exp;
-  // 0.75 + 0.75 << 53
-  APFloat Result =
-      frexp(APFloat(APFloat::PPCDoubleDouble(), APInt(128, 2, Input)), Exp,
-            APFloat::rmNearestTiesToEven);
-  EXPECT_EQ(2, Exp);
-  EXPECT_EQ(0x3fe8000000000000ull, Result.bitcastToAPInt().getRawData()[0]);
-  EXPECT_EQ(0x3c98000000000000ull, Result.bitcastToAPInt().getRawData()[1]);
+
+  DD Input;
+  // Array indexed by the rounding mode enum value.
+  std::array<ExpectedFractionExponent, 5> Rounded = {};
+
+  // Helper to define the expected results for a specific rounding mode.
+  constexpr TestCase &with(APFloat::roundingMode RM, DD ExpectedDD,
+                           int ExpectedExponent) {
+    Rounded[static_cast<std::underlying_type_t<APFloat::roundingMode>>(RM)] = {
+        ExpectedDD,
+        ExpectedExponent,
+    };
+    return *this;
+  }
+
+  // Helper to define the same result for all rounding modes.
+  constexpr TestCase &withAll(DD ExpectedDD, int ExpectedExponent) {
+    return with(RNE, ExpectedDD, ExpectedExponent)
+        .with(RNA, ExpectedDD, ExpectedExponent)
+        .with(RTZ, ExpectedDD, ExpectedExponent)
+        .with(RUP, ExpectedDD, ExpectedExponent)
+        .with(RDN, ExpectedDD, ExpectedExponent);
+  }
+};
+
+auto testCases() {
+  static constexpr auto FrexpTestCases = std::array{
+      // Input: +infinity
+      TestCase{{std::numeric_limits<double>::infinity(), 0.0}}.withAll(
+          {std::numeric_limits<double>::infinity(), 0.0}, INT_MAX),
+
+      // Input: -infinity
+      TestCase{{-std::numeric_limits<double>::infinity(), 0.0}}.withAll(
+          {-std::numeric_limits<double>::infinity(), 0.0}, INT_MAX),
+
+      // Input: NaN
+      TestCase{{std::numeric_limits<double>::quiet_NaN(), 0.0}}.withAll(
+          {std::numeric_limits<double>::quiet_NaN(), 0.0}, INT_MIN),
+
+      // Input: 2^-1074
+      TestCase{{0x1p-1074, 0.0}}.withAll({0x1p-1, 0.0}, -1073),
+      TestCase{{-0x1p-1074, 0.0}}.withAll({-0x1p-1, 0.0}, -1073),
+
+      // Input: (2^1, -2^-1073 + -2^-1074)
+      TestCase{{0x1p1, -0x1.8p-1073}}
+          .withAll({0x1p0, -0x1p-1073}, 1)
+          .with(RNA, {0x1p0, -0x1p-1074}, 1)
+          .with(RUP, {0x1p0, -0x1p-1074}, 1),
+      TestCase{{-0x1p1, 0x1.8p-1073}}
+          .withAll({-0x1p0, 0x1p-1073}, 1)
+          .with(RNA, {-0x1p0, 0x1p-1074}, 1)
+          .with(RDN, {-0x1p0, 0x1p-1074}, 1),
+
+      // Input: (2^1, -2^-1073)
+      TestCase{{0x1p1, -0x1p-1073}}.withAll({0x1p0, -0x1p-1074}, 1),
+
+      // Input: (2^1, -2^-1074)
+      TestCase{{0x1p1, -0x1p-1074}}
+          .withAll({0x1p-1, -0.0}, 2)
+          .with(RDN, {0x1p0, -0x1p-1074}, 1)
+          .with(RTZ, {0x1p0, -0x1p-1074}, 1),
+
+      // Input: (2^2, -2^-1072 + -2^-1073 + -2^-1074)
+      TestCase{{0x1p2, -0x1.cp-1072}}
+          .withAll({0x1p0, -0x1p-1073}, 2)
+          .with(RUP, {0x1p0, -0x1p-1074}, 2),
+
+      // Input: (2^2, -2^-1072 + -2^-1073)
+      TestCase{{0x1p2, -0x1.8p-1072}}
+          .withAll({0x1p0, -0x1p-1073}, 2)
+          .with(RNA, {0x1p0, -0x1p-1074}, 2)
+          .with(RUP, {0x1p0, -0x1p-1074}, 2),
+      TestCase{{-0x1p2, 0x1.8p-1072}}
+          .withAll({-0x1p0, 0x1p-1073}, 2)
+          .with(RNA, {-0x1p0, 0x1p-1074}, 2)
+          .with(RDN, {-0x1p0, 0x1p-1074}, 2),
+
+      // Input: (2^2, -2^-1072 + -2^-1074)
+      TestCase{{0x1p2, -0x1.4cp-1072}}
+          .withAll({0x1p0, -0x1p-1074}, 2)
+          .with(RDN, {0x1p0, -0x1p-1073}, 2)
+          .with(RTZ, {0x1p0, -0x1p-1073}, 2),
+
+      // Input: (2^2, -2^-1072)
+      TestCase{{0x1p2, -0x1p-1072}}.withAll({0x1p0, -0x1p-1074}, 2),
+
+      // Input: (2^2, -2^-1073 + -2^-1074)
+      TestCase{{0x1p2, -0x1.8p-1073}}
+          .withAll({0x1p0, -0x1p-1074}, 2)
+          .with(RUP, {0x1p-1, -0.0}, 3),
+
+      // Input: (2^2, -2^-1073)
+      TestCase{{0x1p2, -0x1p-1073}}
+          .withAll({0x1p-1, -0.0}, 3)
+          .with(RDN, {0x1p0, -0x1p-1074}, 2)
+          .with(RTZ, {0x1p0, -0x1p-1074}, 2),
+
+      // Input: (2^2, -2^-1074)
+      TestCase{{0x1p2, -0x1p-1074}}
+          .withAll({0x1p-1, -0.0}, 3)
+          .with(RDN, {0x1p0, -0x1p-1074}, 2)
+          .with(RTZ, {0x1p0, -0x1p-1074}, 2),
+
+      // Input: 3+3*2^-53 canonicalized to (3+2^-51, -2^-53)
+      // Output: 0.75+0.75*2^-53 canonicalized to (.75+2^-53, -2^-55)
+      TestCase{{0x1.8000000000001p1, -0x1p-53}}.withAll(
+          {0x1.8000000000001p-1, -0x1p-55}, 2),
+      TestCase{{-0x1.8000000000001p1, 0x1p-53}}.withAll(
+          {-0x1.8000000000001p-1, 0x1p-55}, 2),
+
+      // Input: (2^1021+2^969, 2^968-2^915)
+      TestCase{{0x1.0000000000001p1021, 0x1.fffffffffffffp967}}.withAll(
+          {0x1.0000000000001p-1, 0x1.fffffffffffffp-55}, 1022),
+      TestCase{{-0x1.0000000000001p1021, -0x1.fffffffffffffp967}}.withAll(
+          {-0x1.0000000000001p-1, -0x1.fffffffffffffp-55}, 1022),
+
+      // Input: (2^1023, -2^-1)
+      TestCase{{0x1p+1023, -0x1p-1}}.withAll({0x1p0, -0x1p-1024}, 1023),
+      TestCase{{-0x1p+1023, 0x1p-1}}.withAll({-0x1p0, 0x1p-1024}, 1023),
+
+      // Input: (2^1023, -2^-51)
+      TestCase{{0x1p+1023, -0x1p-51}}.withAll({0x1p0, -0x1p-1074}, 1023),
+      TestCase{{-0x1p+1023, 0x1p-51}}.withAll({-0x1p0, 0x1p-1074}, 1023),
+
+      // Input: (2^1023, -2^-52)
+      TestCase{{0x1p+1023, -0x1p-52}}
+          .withAll({0x1p-1, -0x0p0}, 1024)
+          .with(RDN, {0x1p0, -0x1p-1074}, 1023)
+          .with(RTZ, {0x1p0, -0x1p-1074}, 1023),
+      TestCase{{-0x1p+1023, 0x1p-52}}
+          .withAll({-0x1p-1, 0x0p0}, 1024)
+          .with(RUP, {-0x1p0, 0x1p-1074}, 1023)
+          .with(RTZ, {-0x1p0, 0x1p-1074}, 1023),
+
+      // Input: (2^1023, 2^-1074)
+      TestCase{{0x1p+1023, 0x1p-1074}}
+          .withAll({0x1p-1, 0x0p+0}, 1024)
+          .with(RUP, {0x1p-1, 0x1p-1074}, 1024),
+      TestCase{{-0x1p+1023, -0x1p-1074}}
+          .withAll({-0x1p-1, -0x0p+0}, 1024)
+          .with(RDN, {-0x1p-1, -0x1p-1074}, 1024),
+
+      // Input: (2^1024-2^971, 2^970-2^918)
+      TestCase{{0x1.fffffffffffffp+1023, 0x1.ffffffffffffep+969}}.withAll(
+          {0x1.fffffffffffffp-1, 0x1.ffffffffffffep-55}, 1024),
+      TestCase{{-0x1.fffffffffffffp+1023, -0x1.ffffffffffffep+969}}.withAll(
+          {-0x1.fffffffffffffp-1, -0x1.ffffffffffffep-55}, 1024),
+  };
+  return FrexpTestCases;
+}
+} // namespace PPCDoubleDoubleFrexpTestDetails
+
+class PPCDoubleDoubleFrexpValueTest
+    : public testing::Test,
+      public ::testing::WithParamInterface<
+          PPCDoubleDoubleFrexpTestDetails::TestCase> {};
+
+INSTANTIATE_TEST_SUITE_P(
+    PPCDoubleDoubleFrexpValueParamTests, PPCDoubleDoubleFrexpValueTest,
+    ::testing::ValuesIn(PPCDoubleDoubleFrexpTestDetails::testCases()));
+
+TEST_P(PPCDoubleDoubleFrexpValueTest, PPCDoubleDoubleFrexp) {
+  const PPCDoubleDoubleFrexpTestDetails::TestCase Params = GetParam();
+  const APFloat Input = makeDoubleAPFloat(Params.Input);
+  auto RmToIdx = [](APFloat::roundingMode RM) {
+    return static_cast<std::underlying_type_t<APFloat::roundingMode>>(RM);
+  };
+  // First, make sure our expected results are consistent with each other before
+  // bothering to test the implementation.
+  if (Input.isFinite()) {
+    // Make sure the input is canonical.
+    EXPECT_EQ(APFloat{Params.Input.Hi},
+              APFloat{Params.Input.Hi} + APFloat{Params.Input.Lo})
+        << Params.Input.Hi << " + " << Params.Input.Lo;
+
+    const auto Dn = Params.Rounded[RmToIdx(APFloat::rmTowardNegative)];
+    const auto Up = Params.Rounded[RmToIdx(APFloat::rmTowardPositive)];
+    const auto Tz = Params.Rounded[RmToIdx(APFloat::rmTowardZero)];
+    const auto Ne = Params.Rounded[RmToIdx(APFloat::rmNearestTiesToEven)];
+    const auto Na = Params.Rounded[RmToIdx(APFloat::rmNearestTiesToAway)];
+
+    // The rdn result must be no larger than the rup result.
+    const APFloat::cmpResult DnVsUp = compare(Dn, Up);
+    EXPECT_TRUE(DnVsUp == APFloat::cmpLessThan || DnVsUp == APFloat::cmpEqual);
+
+    for (size_t I = 0, E = std::size(Params.Rounded); I != E; ++I) {
+      const APFloat RoundedFraction =
+          makeDoubleAPFloat(Params.Rounded[I].Fraction);
+      // All possible results should be bracketed by [Dn, Up].
+      const APFloat::cmpResult VsDn = compare(Params.Rounded[I], Dn);
+      EXPECT_TRUE(VsDn == APFloat::cmpGreaterThan || VsDn == APFloat::cmpEqual);
+      const APFloat::cmpResult VsUp = compare(Params.Rounded[I], Up);
+      EXPECT_TRUE(VsUp == APFloat::cmpLessThan || VsUp == APFloat::cmpEqual);
+      // A rounding result is either equal to the rup or rdn result.
+      EXPECT_TRUE(VsUp == APFloat::cmpEqual || VsDn == APFloat::cmpEqual);
+      // frexp returns a result whose magnitude is in in [.5, 1) so its exponent
+      // should be -1.
+      if (!RoundedFraction.isZero())
+        EXPECT_EQ(ilogb(RoundedFraction), -1)
+            << static_cast<APFloat::roundingMode>(I);
+      // Decomposition preserves sign.
+      EXPECT_EQ(RoundedFraction.isNegative(), Input.isNegative());
+      // A rounding result must be canonical.
+      EXPECT_EQ(APFloat{Params.Rounded[I].Fraction.Hi},
+                APFloat{Params.Rounded[I].Fraction.Hi} +
+                    APFloat{Params.Rounded[I].Fraction.Lo})
+          << Params.Rounded[I].Fraction.Hi << " + "
+          << Params.Rounded[I].Fraction.Lo;
+    }
+
+    // The rtz result must be either rup or rdn depending on the sign.
+    if (Input.isNegative()) {
+      const APFloat::cmpResult TzVsUp = compare(Tz, Up);
+      EXPECT_EQ(TzVsUp, APFloat::cmpEqual);
+    } else {
+      const APFloat::cmpResult TzVsDn = compare(Tz, Dn);
+      EXPECT_EQ(TzVsDn, APFloat::cmpEqual);
+    }
+
+    // The recomposed up should be at least as big as the input.
+    const APFloat RecomposedUp =
+        scalbn(makeDoubleAPFloat(Up.Fraction), Up.Exponent,
+               APFloat::rmNearestTiesToEven);
+    EXPECT_TRUE(RecomposedUp >= Input);
+    // The recomposed down can't be larger than the input.
+    const APFloat RecomposedDn =
+        scalbn(makeDoubleAPFloat(Dn.Fraction), Dn.Exponent,
+               APFloat::rmNearestTiesToEven);
+    EXPECT_TRUE(RecomposedDn <= Input);
+    // The recomposed tz must have a smaller magnitude.
+    const APFloat RecomposedTz =
+        scalbn(makeDoubleAPFloat(Tz.Fraction), Tz.Exponent,
+               APFloat::rmNearestTiesToEven);
+    EXPECT_TRUE(abs(RecomposedTz) <= abs(Input));
+    // Either both or neither of the recomposed round-to-nearest results are
+    // equal to the input.
+    const APFloat RecomposedNe =
+        scalbn(makeDoubleAPFloat(Ne.Fraction), Ne.Exponent,
+               APFloat::rmNearestTiesToEven);
+    const APFloat RecomposedNa =
+        scalbn(makeDoubleAPFloat(Na.Fraction), Na.Exponent,
+               APFloat::rmNearestTiesToEven);
+    EXPECT_EQ(RecomposedNe == Input, RecomposedNa == Input);
+    // Either the ne result equals the na result or the na result has a bigger
+    // magnitude.
+    const APFloat::cmpResult NeVsNa =
+        abs(RecomposedNe).compare(abs(RecomposedNa));
+    EXPECT_TRUE(NeVsNa == APFloat::cmpLessThan || NeVsNa == APFloat::cmpEqual);
+    // ne and na may only disagree if they broke a tie differently.
+    if (NeVsNa == APFloat::cmpLessThan) {
+      // ne's magnitude should be lower than input.
+      const APFloat::cmpResult NeVsInput =
+          abs(RecomposedNe).compare(abs(Input));
+      EXPECT_EQ(NeVsInput, APFloat::cmpLessThan);
+      // na's magnitude should be greater than input.
+      const APFloat::cmpResult NaVsInput =
+          abs(RecomposedNa).compare(abs(Input));
+      EXPECT_EQ(NaVsInput, APFloat::cmpGreaterThan);
+    }
+    // If up or down perfectly reconstructs the input, the round-to-nearest
+    // results should too.
+    if (RecomposedUp == Input || RecomposedDn == Input) {
+      EXPECT_EQ(RecomposedNe, Input);
+      EXPECT_EQ(RecomposedNa, Input);
+    }
+  }
+
+  for (size_t I = 0, E = std::size(Params.Rounded); I != E; ++I) {
+    const auto RM = static_cast<APFloat::roundingMode>(I);
+    const auto &Expected = Params.Rounded[I];
+    const APFloat ExpectedFraction = makeDoubleAPFloat(Expected.Fraction);
+
+    int ActualExponent;
+    const APFloat ActualFraction = frexp(Input, ActualExponent, RM);
+    if (ExpectedFraction.isNaN())
+      EXPECT_TRUE(ActualFraction.isNaN());
+    else
+      EXPECT_EQ(ActualFraction.compare(ExpectedFraction), APFloat::cmpEqual)
+          << ActualFraction << " vs " << ExpectedFraction << " for input "
+          << Params.Input.Hi << " + " << Params.Input.Lo << " RM " << RM;
+    EXPECT_EQ(ActualExponent, Expected.Exponent)
+        << "for input " << Params.Input.Hi << " + " << Params.Input.Lo
+        << " RM " << RM;
+  }
 }
 
 TEST(APFloatTest, PPCDoubleDoubleNext) {
@@ -5551,13 +6875,9 @@ TEST(APFloatTest, PPCDoubleDoubleNext) {
     return X;
   };
 
-  auto Zero = [] {
-    return APFloat::getZero(APFloat::IEEEdouble());
-  };
+  auto Zero = [] { return APFloat::getZero(APFloat::IEEEdouble()); };
 
-  auto One = [] {
-    return APFloat::getOne(APFloat::IEEEdouble());
-  };
+  auto One = [] { return APFloat::getOne(APFloat::IEEEdouble()); };
 
   // 0x1p-1074
   auto MinSubnormal = [] {
@@ -5574,24 +6894,6 @@ TEST(APFloatTest, PPCDoubleDoubleNext) {
   // 2^-53
   auto EpsNeg = [&] { return scalbn(Eps(), -1, APFloat::rmNearestTiesToEven); };
 
-  auto MakeDoubleAPFloat = [](auto Hi, auto Lo) {
-    APFloat HiFloat{APFloat::IEEEdouble(), APFloat::uninitialized};
-    if constexpr (std::is_same_v<decltype(Hi), APFloat>) {
-      HiFloat = Hi;
-    } else {
-      HiFloat = {APFloat::IEEEdouble(), Hi};
-    }
-
-    APFloat LoFloat{APFloat::IEEEdouble(), APFloat::uninitialized};
-    if constexpr (std::is_same_v<decltype(Lo), APFloat>) {
-      LoFloat = Lo;
-    } else {
-      LoFloat = {APFloat::IEEEdouble(), Lo};
-    }
-
-    APInt Bits = LoFloat.bitcastToAPInt().concat(HiFloat.bitcastToAPInt());
-    return APFloat(APFloat::PPCDoubleDouble(), Bits);
-  };
   APFloat Test(APFloat::PPCDoubleDouble(), APFloat::uninitialized);
   APFloat Expected(APFloat::PPCDoubleDouble(), APFloat::uninitialized);
 
@@ -5719,55 +7021,55 @@ TEST(APFloatTest, PPCDoubleDoubleNext) {
 
   // 2b. |hi| >= 2*DBL_MIN_NORMAL (DD precision > D precision)
   // Test at hi = 1.0, lo = 0.
-  Test = MakeDoubleAPFloat(One(), Zero());
-  Expected = MakeDoubleAPFloat(One(), MinSubnormal());
+  Test = makeDoubleAPFloat(One(), Zero());
+  Expected = makeDoubleAPFloat(One(), MinSubnormal());
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
 
   // Test at hi = -1.0. delta = 2^-1074 (positive, moving towards +Inf).
-  Test = MakeDoubleAPFloat(-One(), Zero());
-  Expected = MakeDoubleAPFloat(-One(), MinSubnormal());
+  Test = makeDoubleAPFloat(-One(), Zero());
+  Expected = makeDoubleAPFloat(-One(), MinSubnormal());
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
 
   // Testing the boundary where calculated delta equals DBL_TRUE_MIN.
   // Requires ilogb(hi) = E = -968.
   // delta = 2^(-968 - 106) = 2^-1074 = DBL_TRUE_MIN.
-  Test = MakeDoubleAPFloat("0x1p-968", Zero());
-  Expected = MakeDoubleAPFloat("0x1p-968", MinSubnormal());
+  Test = makeDoubleAPFloat("0x1p-968", Zero());
+  Expected = makeDoubleAPFloat("0x1p-968", MinSubnormal());
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
 
   // Testing below the boundary (E < -968). Delta clamps to DBL_TRUE_MIN.
-  Test = MakeDoubleAPFloat("0x1p-969", Zero());
-  Expected = MakeDoubleAPFloat("0x1p-969", MinSubnormal());
+  Test = makeDoubleAPFloat("0x1p-969", Zero());
+  Expected = makeDoubleAPFloat("0x1p-969", MinSubnormal());
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
 
   // 3. Standard Increment (No rollover)
   // hi=1.0, lo=2^-1074.
-  Test = MakeDoubleAPFloat(One(), MinSubnormal());
-  Expected = MakeDoubleAPFloat(One(), NextUp(MinSubnormal()));
+  Test = makeDoubleAPFloat(One(), MinSubnormal());
+  Expected = makeDoubleAPFloat(One(), NextUp(MinSubnormal()));
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
 
   // Incrementing negative lo.
-  Test = MakeDoubleAPFloat(One(), -MinSubnormal());
-  Expected = MakeDoubleAPFloat(One(), Zero());
+  Test = makeDoubleAPFloat(One(), -MinSubnormal());
+  Expected = makeDoubleAPFloat(One(), Zero());
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_EQ(Test.compare(Expected), APFloat::cmpEqual);
 
   // Crossing lo=0.
-  Test = MakeDoubleAPFloat(One(), -MinSubnormal());
-  Expected = MakeDoubleAPFloat(One(), Zero());
+  Test = makeDoubleAPFloat(One(), -MinSubnormal());
+  Expected = makeDoubleAPFloat(One(), Zero());
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_EQ(Test.compare(Expected), APFloat::cmpEqual);
 
   // 4. Rollover Cases around 1.0 (Positive hi)
   // hi=1.0, lo=nextDown(2^-53).
-  Test = MakeDoubleAPFloat(One(), NextDown(EpsNeg()));
+  Test = makeDoubleAPFloat(One(), NextDown(EpsNeg()));
   EXPECT_FALSE(Test.isDenormal());
-  Expected = MakeDoubleAPFloat(One(), EpsNeg());
+  Expected = makeDoubleAPFloat(One(), EpsNeg());
   EXPECT_FALSE(Test.isDenormal());
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
@@ -5778,17 +7080,17 @@ TEST(APFloatTest, PPCDoubleDoubleNext) {
   // Can't naively TwoSum(0x1p+0, nextUp(0x1p-53)):
   //   It gives {nextUp(0x1p+0), nextUp(nextUp(-0x1p-53))} but the next
   //   number should be {nextUp(0x1p+0), nextUp(-0x1p-53)}.
-  Test = MakeDoubleAPFloat(One(), EpsNeg());
+  Test = makeDoubleAPFloat(One(), EpsNeg());
   EXPECT_FALSE(Test.isDenormal());
-  Expected = MakeDoubleAPFloat(NextUp(One()), NextUp(-EpsNeg()));
+  Expected = makeDoubleAPFloat(NextUp(One()), NextUp(-EpsNeg()));
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
   EXPECT_FALSE(Test.isDenormal());
 
   // hi = nextDown(1), lo = nextDown(0x1p-54)
-  Test = MakeDoubleAPFloat(NextDown(One()), NextDown(APFloat(0x1p-54)));
+  Test = makeDoubleAPFloat(NextDown(One()), NextDown(APFloat(0x1p-54)));
   EXPECT_FALSE(Test.isDenormal());
-  Expected = MakeDoubleAPFloat(One(), APFloat(-0x1p-54));
+  Expected = makeDoubleAPFloat(One(), APFloat(-0x1p-54));
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
   EXPECT_FALSE(Test.isDenormal());
@@ -5796,31 +7098,274 @@ TEST(APFloatTest, PPCDoubleDoubleNext) {
   // 5. Negative Rollover (Moving towards Zero / +Inf)
 
   // hi = -1, lo = nextDown(0x1p-54)
-  Test = MakeDoubleAPFloat(APFloat(-1.0), NextDown(APFloat(0x1p-54)));
+  Test = makeDoubleAPFloat(APFloat(-1.0), NextDown(APFloat(0x1p-54)));
   EXPECT_FALSE(Test.isDenormal());
-  Expected = MakeDoubleAPFloat(APFloat(-1.0), APFloat(0x1p-54));
+  Expected = makeDoubleAPFloat(APFloat(-1.0), APFloat(0x1p-54));
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
   EXPECT_FALSE(Test.isDenormal());
 
   // hi = -1, lo = 0x1p-54
-  Test = MakeDoubleAPFloat(APFloat(-1.0), APFloat(0x1p-54));
+  Test = makeDoubleAPFloat(APFloat(-1.0), APFloat(0x1p-54));
   EXPECT_FALSE(Test.isDenormal());
   Expected =
-      MakeDoubleAPFloat(NextUp(APFloat(-1.0)), NextUp(APFloat(-0x1p-54)));
+      makeDoubleAPFloat(NextUp(APFloat(-1.0)), NextUp(APFloat(-0x1p-54)));
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
   EXPECT_FALSE(Test.isDenormal());
 
   // 6. Rollover across Power of 2 boundary (Exponent change)
-  Test = MakeDoubleAPFloat(NextDown(APFloat(2.0)), NextDown(EpsNeg()));
+  Test = makeDoubleAPFloat(NextDown(APFloat(2.0)), NextDown(EpsNeg()));
   EXPECT_FALSE(Test.isDenormal());
-  Expected = MakeDoubleAPFloat(APFloat(2.0), -EpsNeg());
+  Expected = makeDoubleAPFloat(APFloat(2.0), -EpsNeg());
   EXPECT_EQ(Test.next(false), APFloat::opOK);
   EXPECT_TRUE(Test.bitwiseIsEqual(Expected));
   EXPECT_FALSE(Test.isDenormal());
 }
 
+TEST(APFloatTest, PPCDoubleDoubleConvertFromAPIntInexact) {
+  // Create an integer which would not be exactly representable in
+  // PPCDoubleDoubleLegacy.
+  for (bool IsSigned : {false, true}) {
+    const unsigned BitWidth =
+        APFloat::semanticsPrecision(APFloat::IEEEdouble()) * 3 +
+        (IsSigned ? 1 : 0);
+
+    for (bool Negative :
+         IsSigned ? std::vector{false, true} : std::vector{false}) {
+      APInt Huge = APInt{BitWidth, 0};
+      // Set the highest bit without making Huge negative..
+      Huge.setBit(BitWidth - (IsSigned ? 2 : 1));
+      // Set the low bit.
+      Huge.setBit(0);
+      if (Negative)
+        Huge.negate();
+      for (const APFloat::roundingMode RM :
+           {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+            APFloat::rmTowardPositive, APFloat::rmTowardZero,
+            APFloat::rmNearestTiesToEven}) {
+        APFloat F{APFloat::PPCDoubleDouble()};
+        const APFloat::opStatus ConvertFromStatus =
+            F.convertFromAPInt(Huge, /*IsSigned=*/IsSigned, RM);
+        EXPECT_EQ(ConvertFromStatus, APFloat::opOK);
+
+        bool IsExact;
+        APSInt ResultInt{Huge.getBitWidth(), /*isUnsigned=*/!IsSigned};
+        const APFloat::opStatus ConvertToStatus =
+            F.convertToInteger(ResultInt, APFloat::rmTowardZero, &IsExact);
+
+        EXPECT_TRUE(IsExact) << "RM: " << RM;
+        EXPECT_TRUE(ResultInt.eq(Huge)) << ResultInt << " vs " << Huge << "\n";
+        EXPECT_EQ(ConvertToStatus, APFloat::opOK);
+      }
+    }
+  }
+}
+
+TEST(APFloatTest, PPCDoubleDoubleConvertFromAPIntBoundary) {
+  const unsigned Binary64Precision =
+      APFloat::semanticsPrecision(APFloat::IEEEdouble());
+  APSInt Boundary =
+      APSInt::getMaxValue(Binary64Precision + 1, /*Unsigned=*/true);
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    const APFloat Exact = makeDoubleAPFloat(0x1p54, -0x1p0);
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(Boundary, /*IsSigned=*/false, RM);
+    EXPECT_EQ(ConvertFromStatus, APFloat::opOK);
+    EXPECT_EQ(F, Exact);
+  }
+
+  Boundary = APSInt{APInt::getHighBitsSet(/*numBits=*/128,
+                                          /*hiBitsSet=*/Binary64Precision + 1),
+                    /*isUnsigned=*/true};
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    const APFloat Exact = makeDoubleAPFloat(0x1p128, -0x1p74);
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(Boundary, /*IsSigned=*/false, RM);
+    EXPECT_EQ(ConvertFromStatus, APFloat::opOK);
+    EXPECT_EQ(F, Exact);
+  }
+}
+
+TEST(APFloatTest, PPCDoubleDoubleConvertFromAPIntEnormous) {
+  APFloat Largest = APFloat::getLargest(APFloat::PPCDoubleDouble());
+  int Exponent = ilogb(Largest);
+  unsigned BitWidth = Exponent + 1;
+  APSInt HugeInt{BitWidth, /*isUnsigned=*/true};
+  bool IsExact;
+  APFloat::opStatus Status =
+      Largest.convertToInteger(HugeInt, APFloat::rmTowardPositive, &IsExact);
+  ASSERT_EQ(Status, APFloat::opOK);
+  ASSERT_TRUE(IsExact);
+
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(HugeInt, /*IsSigned=*/false, RM);
+    EXPECT_EQ(ConvertFromStatus, APFloat::opOK);
+    EXPECT_EQ(F, Largest);
+  }
+
+  const unsigned MaxExponent =
+      APFloat::semanticsMaxExponent(APFloat::IEEEdouble());
+  const unsigned Binary64Precision =
+      APFloat::semanticsPrecision(APFloat::IEEEdouble());
+  const unsigned UlpOfLargest = MaxExponent - (2 * Binary64Precision);
+  const unsigned HalfUlpOfLargest = UlpOfLargest - 1;
+
+  // Add just under a half-ulp. This should never overflow for
+  // round-ties-to-nearest modes.
+  HugeInt.setLowBits(HalfUlpOfLargest);
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(HugeInt, /*IsSigned=*/false, RM);
+    if (RM == APFloat::rmTowardPositive) {
+      EXPECT_TRUE(F.isPosInfinity()) << F;
+      EXPECT_EQ(ConvertFromStatus, APFloat::opInexact | APFloat::opOverflow);
+    } else {
+      EXPECT_EQ(F, Largest);
+      EXPECT_EQ(ConvertFromStatus, APFloat::opInexact);
+    }
+  }
+
+  // Now test adding a half-ulp. This should cause overflow for ties-to-away.
+  // ties-to-even will not overflow if the max finite value has a clear low bit.
+  ++HugeInt;
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(HugeInt, /*IsSigned=*/false, RM);
+    const bool Overflow =
+        RM == APFloat::rmTowardPositive || RM == APFloat::rmNearestTiesToAway ||
+        (RM == APFloat::rmNearestTiesToEven && HugeInt[UlpOfLargest]);
+    if (Overflow) {
+      EXPECT_TRUE(F.isPosInfinity()) << F;
+      EXPECT_EQ(ConvertFromStatus, APFloat::opInexact | APFloat::opOverflow);
+    } else {
+      EXPECT_EQ(F, Largest);
+      EXPECT_EQ(ConvertFromStatus, APFloat::opInexact);
+    }
+  }
+
+  // Now test adding just over a half-ulp. This should break all ties.
+  ++HugeInt;
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(HugeInt, /*IsSigned=*/false, RM);
+    const bool Overflow = RM == APFloat::rmTowardPositive ||
+                          RM == APFloat::rmNearestTiesToAway ||
+                          RM == APFloat::rmNearestTiesToEven;
+    if (Overflow) {
+      EXPECT_TRUE(F.isPosInfinity()) << F;
+      EXPECT_EQ(ConvertFromStatus, APFloat::opInexact | APFloat::opOverflow);
+    } else {
+      EXPECT_EQ(F, Largest);
+      EXPECT_EQ(ConvertFromStatus, APFloat::opInexact);
+    }
+  }
+
+  HugeInt.setAllBits();
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(HugeInt, /*IsSigned=*/false, RM);
+    const bool Overflow = RM == APFloat::rmTowardPositive ||
+                          RM == APFloat::rmNearestTiesToAway ||
+                          RM == APFloat::rmNearestTiesToEven;
+    if (Overflow) {
+      EXPECT_TRUE(F.isPosInfinity()) << F;
+      EXPECT_EQ(ConvertFromStatus, APFloat::opInexact | APFloat::opOverflow);
+    } else {
+      EXPECT_EQ(F, Largest);
+      EXPECT_EQ(ConvertFromStatus, APFloat::opInexact);
+    }
+  }
+
+  HugeInt.clearAllBits();
+  HugeInt.setBit(2 * Binary64Precision + 1);
+  HugeInt.setLowBits(Binary64Precision + 1);
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    const APFloat RoundUp = makeDoubleAPFloat(0x1p107, 0x1p54);
+    const APFloat RoundDown = makeDoubleAPFloat(0x1p107, 0x1.fffffffffffffp53);
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(HugeInt, /*IsSigned=*/false, RM);
+    EXPECT_EQ(ConvertFromStatus, APFloat::opInexact);
+
+    if (RM == APFloat::rmNearestTiesToEven ||
+        RM == APFloat::rmNearestTiesToAway || RM == APFloat::rmTowardPositive)
+      EXPECT_EQ(F, RoundUp);
+    else
+      EXPECT_EQ(F, RoundDown);
+  }
+
+  ++HugeInt;
+  // 162259276829213381405976519770112 can be represented exactly.
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    const APFloat Exact = makeDoubleAPFloat(0x1p107, 0x1p54);
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(HugeInt, /*IsSigned=*/false, RM);
+    EXPECT_EQ(ConvertFromStatus, APFloat::opOK);
+    EXPECT_EQ(F, Exact);
+  }
+
+  ++HugeInt;
+  // 162259276829213381405976519770113 rounds to either:
+  // 162259276829213381405976519770112
+  // 162259276829213381405976519770114
+  for (const APFloat::roundingMode RM :
+       {APFloat::rmNearestTiesToAway, APFloat::rmTowardNegative,
+        APFloat::rmTowardPositive, APFloat::rmTowardZero,
+        APFloat::rmNearestTiesToEven}) {
+    const APFloat RoundUp =
+        makeDoubleAPFloat(0x1.0000000000001p107, -0x1.fffffffffffffp53);
+    const APFloat RoundDown = makeDoubleAPFloat(0x1p107, 0x1p54);
+    EXPECT_LT(RoundDown, RoundUp);
+
+    APFloat F{APFloat::PPCDoubleDouble()};
+    const APFloat::opStatus ConvertFromStatus =
+        F.convertFromAPInt(HugeInt, /*IsSigned=*/false, RM);
+    EXPECT_EQ(ConvertFromStatus, APFloat::opInexact);
+
+    if (RM == APFloat::rmNearestTiesToAway || RM == APFloat::rmTowardPositive)
+      EXPECT_EQ(F, RoundUp);
+    else
+      EXPECT_EQ(F, RoundDown);
+  }
+}
+
 TEST(APFloatTest, x87Largest) {
   APFloat MaxX87Val = APFloat::getLargest(APFloat::x87DoubleExtended());
   EXPECT_TRUE(MaxX87Val.isLargest());
@@ -7812,13 +9357,8 @@ TEST(APFloatTest, getExactLog2) {
       continue;
 
     APFloat One(Semantics, "1.0");
-
-    if (I == APFloat::S_PPCDoubleDouble) {
-      // Not implemented
-      EXPECT_EQ(INT_MIN, One.getExactLog2());
-      EXPECT_EQ(INT_MIN, One.getExactLog2Abs());
-      continue;
-    }
+    APFloat Smallest = APFloat::getSmallest(Semantics);
+    APFloat Largest = APFloat::getLargest(Semantics);
 
     int MinExp = APFloat::semanticsMinExponent(Semantics);
     int MaxExp = APFloat::semanticsMaxExponent(Semantics);
@@ -7865,16 +9405,15 @@ TEST(APFloatTest, getExactLog2) {
       EXPECT_EQ(INT_MIN, APFloat::getNaN(Semantics, true).getExactLog2Abs());
     }
 
-    EXPECT_EQ(INT_MIN,
-              scalbn(One, MinExp - Precision - 1, APFloat::rmNearestTiesToEven)
-                  .getExactLog2());
-    EXPECT_EQ(INT_MIN,
-              scalbn(One, MinExp - Precision, APFloat::rmNearestTiesToEven)
-                  .getExactLog2());
-
     EXPECT_EQ(
         INT_MIN,
-        scalbn(One, MaxExp + 1, APFloat::rmNearestTiesToEven).getExactLog2());
+        scalbn(Smallest, -2, APFloat::rmNearestTiesToEven).getExactLog2());
+    EXPECT_EQ(
+        INT_MIN,
+        scalbn(Smallest, -1, APFloat::rmNearestTiesToEven).getExactLog2());
+
+    EXPECT_EQ(INT_MIN,
+              scalbn(Largest, 1, APFloat::rmNearestTiesToEven).getExactLog2());
 
     for (int i = MinExp - Precision + 1; i <= MaxExp; ++i) {
       EXPECT_EQ(i, scalbn(One, i, APFloat::rmNearestTiesToEven).getExactLog2());
diff --git a/llvm/unittests/ADT/APIntTest.cpp b/llvm/unittests/ADT/APIntTest.cpp
index 4741c7bcc140f..116693c873f30 100644
--- a/llvm/unittests/ADT/APIntTest.cpp
+++ b/llvm/unittests/ADT/APIntTest.cpp
@@ -3103,6 +3103,53 @@ TEST(APIntOpsTest, Mulh) {
   EXPECT_EQ(APInt(128, "FFEB498812C66C68D4552DB89B8EBF8F", 16), i128Res);
 }
 
+TEST(APIntOpsTest, muli) {
+  APInt u32a(32, 0x0001'E235);
+  APInt u32b(32, 0xF623'55AD);
+  EXPECT_EQ(0x0001'CFA1'7CA0'76D1, APIntOps::muluExtended(u32a, u32b));
+
+  APInt u64a(64, 0x1234'5678'90AB'CDEF);
+  APInt u64b(64, 0xFEDC'BA09'8765'4321);
+  EXPECT_EQ(APInt(128, "121FA000A3723A57C24A442FE55618CF", 16),
+            APIntOps::muluExtended(u64a, u64b));
+
+  APInt u128a(128, "1234567890ABCDEF1234567890ABCDEF", 16);
+  APInt u128b(128, "FEDCBA0987654321FEDCBA0987654321", 16);
+  EXPECT_EQ(
+      APInt(256,
+            "121FA000A3723A57E68984312C3A8D7E96B428606E1E6BF5C24A442FE55618CF",
+            16),
+      APIntOps::muluExtended(u128a, u128b));
+
+  APInt s32a(32, 0x1234'5678);
+  APInt s32b(32, 0x10AB'CDEF);
+  APInt s32c(32, 0xFEDC'BA09);
+  EXPECT_EQ(0x012F'7D02'2A42'D208, APIntOps::mulsExtended(s32a, s32b));
+  EXPECT_EQ(0xFFEB'4988'09CA'3A38, APIntOps::mulsExtended(s32a, s32c));
+
+  APInt s64a(64, 0x1234'5678'90AB'CDEF);
+  APInt s64b(64, 0x1234'5678'90FE'DCBA);
+  APInt s64c(64, 0xFEDC'BA09'8765'4321);
+  EXPECT_EQ(APInt(128, "014B66DC328E10C1FB99704184EF03A6", 16),
+            APIntOps::mulsExtended(s64a, s64b));
+  EXPECT_EQ(APInt(128, "FFEB498812C66C68C24A442FE55618CF", 16),
+            APIntOps::mulsExtended(s64a, s64c));
+
+  APInt s128a(128, "1234567890ABCDEF1234567890ABCDEF", 16);
+  APInt s128b(128, "1234567890FEDCBA1234567890FEDCBA", 16);
+  APInt s128c(128, "FEDCBA0987654321FEDCBA0987654321", 16);
+  EXPECT_EQ(
+      APInt(256,
+            "014B66DC328E10C1FE303DF9EA0B2529F87E475F3C6C180DFB99704184EF03A6",
+            16),
+      APIntOps::mulsExtended(s128a, s128b));
+  EXPECT_EQ(
+      APInt(256,
+            "FFEB498812C66C68D4552DB89B8EBF8F96B428606E1E6BF5C24A442FE55618CF",
+            16),
+      APIntOps::mulsExtended(s128a, s128c));
+}
+
 TEST(APIntTest, RoundingUDiv) {
   for (uint64_t Ai = 1; Ai <= 255; Ai++) {
     APInt A(8, Ai);
@@ -3697,4 +3744,82 @@ TEST(APIntTest, TryExt) {
   ASSERT_EQ(42, APInt(128, -1).trySExtValue().value_or(42));
 }
 
+TEST(APIntTest, Fshl) {
+  EXPECT_EQ(
+      APIntOps::fshl(APInt(8, 0), APInt(8, 255), APInt(8, 8)).getZExtValue(),
+      0U);
+  EXPECT_EQ(
+      APIntOps::fshl(APInt(8, 255), APInt(8, 0), APInt(8, 8)).getZExtValue(),
+      255U);
+  EXPECT_EQ(
+      APIntOps::fshl(APInt(8, 255), APInt(8, 0), APInt(8, 15)).getZExtValue(),
+      128U);
+  EXPECT_EQ(
+      APIntOps::fshl(APInt(8, 15), APInt(8, 15), APInt(8, 11)).getZExtValue(),
+      120U);
+  EXPECT_EQ(
+      APIntOps::fshl(APInt(8, 2), APInt(8, 1), APInt(8, 3)).getZExtValue(),
+      16U);
+  EXPECT_EQ(
+      APIntOps::fshl(APInt(8, 2), APInt(8, 1), APInt(8, 1)).getZExtValue(),
+      APIntOps::fshl(APInt(8, 2), APInt(8, 1), APInt(8, 9)).getZExtValue());
+  EXPECT_EQ(
+      APIntOps::fshl(APInt(8, 2), APInt(8, 1), APInt(8, 7)).getZExtValue(),
+      APIntOps::fshl(APInt(8, 2), APInt(8, 1), APInt(8, 15)).getZExtValue());
+  EXPECT_EQ(APIntOps::fshl(APInt(32, 0, /*isSigned*/ true),
+                           APInt(32, 2147483647, /*isSigned*/ true),
+                           APInt(32, 32, /*isSigned*/ true))
+                .getSExtValue(),
+            0);
+  EXPECT_EQ(APIntOps::fshl(APInt(64, 1, /*isSigned*/ true),
+                           APInt(64, 2, /*isSigned*/ true),
+                           APInt(64, 3, /*isSigned*/ true))
+                .getSExtValue(),
+            8);
+  EXPECT_EQ(APIntOps::fshl(APInt(16, -2, /*isSigned*/ true),
+                           APInt(16, -1, /*isSigned*/ true),
+                           APInt(16, 3, /*isSigned*/ true))
+                .getSExtValue(),
+            -9);
+}
+
+TEST(APIntTest, Fshr) {
+  EXPECT_EQ(
+      APIntOps::fshr(APInt(8, 0), APInt(8, 255), APInt(8, 8)).getZExtValue(),
+      255U);
+  EXPECT_EQ(
+      APIntOps::fshr(APInt(8, 255), APInt(8, 0), APInt(8, 8)).getZExtValue(),
+      0U);
+  EXPECT_EQ(
+      APIntOps::fshr(APInt(8, 255), APInt(8, 0), APInt(8, 15)).getZExtValue(),
+      254U);
+  EXPECT_EQ(
+      APIntOps::fshr(APInt(8, 15), APInt(8, 15), APInt(8, 11)).getZExtValue(),
+      225U);
+  EXPECT_EQ(
+      APIntOps::fshr(APInt(8, 1), APInt(8, 2), APInt(8, 3)).getZExtValue(),
+      32U);
+  EXPECT_EQ(
+      APIntOps::fshr(APInt(8, 1), APInt(8, 2), APInt(8, 1)).getZExtValue(),
+      APIntOps::fshr(APInt(8, 1), APInt(8, 2), APInt(8, 9)).getZExtValue());
+  EXPECT_EQ(
+      APIntOps::fshr(APInt(8, 1), APInt(8, 2), APInt(8, 7)).getZExtValue(),
+      APIntOps::fshr(APInt(8, 1), APInt(8, 2), APInt(8, 15)).getZExtValue());
+  EXPECT_EQ(APIntOps::fshr(APInt(64, 0, /*isSigned*/ true),
+                           APInt(64, 9223372036854775807, /*isSigned*/ true),
+                           APInt(64, 64, /*isSigned*/ true))
+                .getSExtValue(),
+            9223372036854775807);
+  EXPECT_EQ(APIntOps::fshr(APInt(64, 1, /*isSigned*/ true),
+                           APInt(64, 2, /*isSigned*/ true),
+                           APInt(64, 3, /*isSigned*/ true))
+                .getSExtValue(),
+            2305843009213693952);
+  EXPECT_EQ(APIntOps::fshr(APInt(16, -2, /*isSigned*/ true),
+                           APInt(16, -1, /*isSigned*/ true),
+                           APInt(16, 3, /*isSigned*/ true))
+                .getSExtValue(),
+            -8193);
+}
+
 } // end anonymous namespace
diff --git a/llvm/unittests/ADT/DenseMapTest.cpp b/llvm/unittests/ADT/DenseMapTest.cpp
index bdfbc8557859a..785ab16271d93 100644
--- a/llvm/unittests/ADT/DenseMapTest.cpp
+++ b/llvm/unittests/ADT/DenseMapTest.cpp
@@ -10,6 +10,7 @@
 #include "CountCopyAndMove.h"
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/DenseMapInfoVariant.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "gmock/gmock.h"
@@ -249,6 +250,25 @@ TYPED_TEST(DenseMapTest, CopyConstructorNotSmallTest) {
     EXPECT_EQ(this->getValue(Key), copyMap[this->getKey(Key)]);
 }
 
+// Test range constructors.
+TYPED_TEST(DenseMapTest, RangeConstructorTest) {
+  using KeyAndValue =
+      std::pair<typename TypeParam::key_type, typename TypeParam::mapped_type>;
+  KeyAndValue PlainArray[] = {{this->getKey(0), this->getValue(0)},
+                              {this->getKey(1), this->getValue(1)}};
+
+  TypeParam MapFromRange(llvm::from_range, PlainArray);
+  EXPECT_EQ(2u, MapFromRange.size());
+  EXPECT_EQ(this->getValue(0), MapFromRange[this->getKey(0)]);
+  EXPECT_EQ(this->getValue(1), MapFromRange[this->getKey(1)]);
+
+  TypeParam MapFromInitList({{this->getKey(0), this->getValue(1)},
+                             {this->getKey(1), this->getValue(2)}});
+  EXPECT_EQ(2u, MapFromInitList.size());
+  EXPECT_EQ(this->getValue(1), MapFromInitList[this->getKey(0)]);
+  EXPECT_EQ(this->getValue(2), MapFromInitList[this->getKey(1)]);
+}
+
 // Test copying from a default-constructed map.
 TYPED_TEST(DenseMapTest, CopyConstructorFromDefaultTest) {
   TypeParam copyMap(this->Map);
@@ -367,9 +387,16 @@ TYPED_TEST(DenseMapTest, ConstIteratorTest) {
   EXPECT_TRUE(cit == cit2);
 }
 
+// TYPED_TEST below cycles through different types.  We define UniversalSmallSet
+// here so that we'll use SmallSet or SmallPtrSet depending on whether the
+// element type is a pointer.
+template <typename T, unsigned N>
+using UniversalSmallSet =
+    std::conditional_t<std::is_pointer_v<T>, SmallPtrSet<T, N>, SmallSet<T, N>>;
+
 TYPED_TEST(DenseMapTest, KeysValuesIterator) {
-  SmallSet<typename TypeParam::key_type, 10> Keys;
-  SmallSet<typename TypeParam::mapped_type, 10> Values;
+  UniversalSmallSet<typename TypeParam::key_type, 10> Keys;
+  UniversalSmallSet<typename TypeParam::mapped_type, 10> Values;
   for (int I = 0; I < 10; ++I) {
     auto K = this->getKey(I);
     auto V = this->getValue(I);
@@ -378,8 +405,8 @@ TYPED_TEST(DenseMapTest, KeysValuesIterator) {
     this->Map[K] = V;
   }
 
-  SmallSet<typename TypeParam::key_type, 10> ActualKeys;
-  SmallSet<typename TypeParam::mapped_type, 10> ActualValues;
+  UniversalSmallSet<typename TypeParam::key_type, 10> ActualKeys;
+  UniversalSmallSet<typename TypeParam::mapped_type, 10> ActualValues;
   for (auto K : this->Map.keys())
     ActualKeys.insert(K);
   for (auto V : this->Map.values())
@@ -390,8 +417,8 @@ TYPED_TEST(DenseMapTest, KeysValuesIterator) {
 }
 
 TYPED_TEST(DenseMapTest, ConstKeysValuesIterator) {
-  SmallSet<typename TypeParam::key_type, 10> Keys;
-  SmallSet<typename TypeParam::mapped_type, 10> Values;
+  UniversalSmallSet<typename TypeParam::key_type, 10> Keys;
+  UniversalSmallSet<typename TypeParam::mapped_type, 10> Values;
   for (int I = 0; I < 10; ++I) {
     auto K = this->getKey(I);
     auto V = this->getValue(I);
@@ -401,8 +428,8 @@ TYPED_TEST(DenseMapTest, ConstKeysValuesIterator) {
   }
 
   const TypeParam &ConstMap = this->Map;
-  SmallSet<typename TypeParam::key_type, 10> ActualKeys;
-  SmallSet<typename TypeParam::mapped_type, 10> ActualValues;
+  UniversalSmallSet<typename TypeParam::key_type, 10> ActualKeys;
+  UniversalSmallSet<typename TypeParam::mapped_type, 10> ActualValues;
   for (auto K : ConstMap.keys())
     ActualKeys.insert(K);
   for (auto V : ConstMap.values())
@@ -726,6 +753,15 @@ TEST(DenseMapCustomTest, FindAsTest) {
   EXPECT_TRUE(map.find_as("d") == map.end());
 }
 
+TEST(DenseMapCustomTest, SmallDenseMapFromRange) {
+  std::pair<int, StringRef> PlainArray[] = {{0, "0"}, {1, "1"}, {2, "2"}};
+  SmallDenseMap<int, StringRef> M(llvm::from_range, PlainArray);
+  EXPECT_EQ(3u, M.size());
+  using testing::Pair;
+  EXPECT_THAT(M, testing::UnorderedElementsAre(Pair(0, "0"), Pair(1, "1"),
+                                               Pair(2, "2")));
+}
+
 TEST(DenseMapCustomTest, SmallDenseMapInitializerList) {
   SmallDenseMap<int, int> M = {{0, 0}, {0, 1}, {1, 2}};
   EXPECT_EQ(2u, M.size());
diff --git a/llvm/unittests/ADT/SmallPtrSetTest.cpp b/llvm/unittests/ADT/SmallPtrSetTest.cpp
index 4ea7403b65abc..a627091b90c70 100644
--- a/llvm/unittests/ADT/SmallPtrSetTest.cpp
+++ b/llvm/unittests/ADT/SmallPtrSetTest.cpp
@@ -475,4 +475,8 @@ TEST(SmallPtrSetTest, Reserve) {
   EXPECT_EQ(Set.capacity(), 128u);
   EXPECT_EQ(Set.size(), 6u);
   EXPECT_THAT(Set, UnorderedElementsAre(&Vals[0], &Vals[1], &Vals[2], &Vals[3], &Vals[4], &Vals[5]));
+
+  // Reserving 192 should result in 256 buckets.
+  Set.reserve(192);
+  EXPECT_EQ(Set.capacity(), 256u);
 }
diff --git a/llvm/unittests/ADT/StringRefTest.cpp b/llvm/unittests/ADT/StringRefTest.cpp
index ec9cdc197597d..d5f8dc41cdb6b 100644
--- a/llvm/unittests/ADT/StringRefTest.cpp
+++ b/llvm/unittests/ADT/StringRefTest.cpp
@@ -619,6 +619,19 @@ TEST(StringRefTest, Hashing) {
             hash_value(StringRef("hello world").slice(1, -1)));
 }
 
+TEST(StringRefTest, getAutoSenseRadix) {
+  struct RadixPair {
+    const char *Str;
+    unsigned Expected;
+  } RadixNumbers[] = {{"123", 10}, {"1", 10}, {"0b1", 2}, {"01", 8}, {"0o1", 8},
+                      {"0x1", 16}, {"0", 10}, {"00", 8},  {"", 10}};
+  for (size_t i = 0; i < std::size(RadixNumbers); ++i) {
+    StringRef number = RadixNumbers[i].Str;
+    unsigned radix = getAutoSenseRadix(number);
+    EXPECT_EQ(radix, RadixNumbers[i].Expected);
+  }
+}
+
 struct UnsignedPair {
   const char *Str;
   uint64_t Expected;
diff --git a/llvm/unittests/Analysis/IR2VecTest.cpp b/llvm/unittests/Analysis/IR2VecTest.cpp
index e288585033c53..f7838cc4068ce 100644
--- a/llvm/unittests/Analysis/IR2VecTest.cpp
+++ b/llvm/unittests/Analysis/IR2VecTest.cpp
@@ -320,11 +320,13 @@ TEST_F(IR2VecTestFixture, GetInstVecMap) {
   EXPECT_TRUE(InstMap.count(AddInst));
   EXPECT_TRUE(InstMap.count(RetInst));
 
-  EXPECT_EQ(InstMap.at(AddInst).size(), 2u);
-  EXPECT_EQ(InstMap.at(RetInst).size(), 2u);
+  const auto &AddEmb = InstMap.at(AddInst);
+  const auto &RetEmb = InstMap.at(RetInst);
+  EXPECT_EQ(AddEmb.size(), 2u);
+  EXPECT_EQ(RetEmb.size(), 2u);
 
-  EXPECT_TRUE(InstMap.at(AddInst).approximatelyEquals(Embedding(2, 27.6)));
-  EXPECT_TRUE(InstMap.at(RetInst).approximatelyEquals(Embedding(2, 16.8)));
+  EXPECT_TRUE(AddEmb.approximatelyEquals(Embedding(2, 27.9)));
+  EXPECT_TRUE(RetEmb.approximatelyEquals(Embedding(2, 17.0)));
 }
 
 TEST_F(IR2VecTestFixture, GetBBVecMap) {
@@ -337,9 +339,9 @@ TEST_F(IR2VecTestFixture, GetBBVecMap) {
   EXPECT_TRUE(BBMap.count(BB));
   EXPECT_EQ(BBMap.at(BB).size(), 2u);
 
-  // BB vector should be sum of add and ret: {27.6, 27.6} + {16.8, 16.8} =
-  // {44.4, 44.4}
-  EXPECT_TRUE(BBMap.at(BB).approximatelyEquals(Embedding(2, 44.4)));
+  // BB vector should be sum of add and ret: {27.9, 27.9} + {17.0, 17.0} =
+  // {44.9, 44.9}
+  EXPECT_TRUE(BBMap.at(BB).approximatelyEquals(Embedding(2, 44.9)));
 }
 
 TEST_F(IR2VecTestFixture, GetBBVector) {
@@ -349,7 +351,7 @@ TEST_F(IR2VecTestFixture, GetBBVector) {
   const auto &BBVec = Emb->getBBVector(*BB);
 
   EXPECT_EQ(BBVec.size(), 2u);
-  EXPECT_TRUE(BBVec.approximatelyEquals(Embedding(2, 44.4)));
+  EXPECT_TRUE(BBVec.approximatelyEquals(Embedding(2, 44.9)));
 }
 
 TEST_F(IR2VecTestFixture, GetFunctionVector) {
@@ -360,8 +362,8 @@ TEST_F(IR2VecTestFixture, GetFunctionVector) {
 
   EXPECT_EQ(FuncVec.size(), 2u);
 
-  // Function vector should match BB vector (only one BB): {44.4, 44.4}
-  EXPECT_TRUE(FuncVec.approximatelyEquals(Embedding(2, 44.4)));
+  // Function vector should match BB vector (only one BB): {44.9, 44.9}
+  EXPECT_TRUE(FuncVec.approximatelyEquals(Embedding(2, 44.9)));
 }
 
 static constexpr unsigned MaxOpcodes = Vocabulary::MaxOpcodes;
diff --git a/llvm/unittests/Analysis/MemorySSATest.cpp b/llvm/unittests/Analysis/MemorySSATest.cpp
index 1a9296f042578..fceefbcec1ef1 100644
--- a/llvm/unittests/Analysis/MemorySSATest.cpp
+++ b/llvm/unittests/Analysis/MemorySSATest.cpp
@@ -1092,8 +1092,8 @@ TEST_F(MemorySSATest, LifetimeMarkersAreClobbers) {
   //   %baz = getelementptr i8, ptr %foo, i64 2
   //   store i8 0, ptr %foo
   //   store i8 0, ptr %bar
-  //   call void @llvm.lifetime.end.p0(i64 3, ptr %foo)
-  //   call void @llvm.lifetime.start.p0(i64 3, ptr %foo)
+  //   call void @llvm.lifetime.end.p0(ptr %foo)
+  //   call void @llvm.lifetime.start.p0(ptr %foo)
   //   store i8 0, ptr %foo
   //   store i8 0, ptr %bar
   //   call void @llvm.memset.p0i8(ptr %baz, i8 0, i64 1)
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index 6af20065213ac..559a0b724f383 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/FloatingPointPredicateUtils.h"
 #include "llvm/AsmParser/Parser.h"
@@ -2208,6 +2209,41 @@ TEST_F(ComputeKnownFPClassTest, Constants) {
   }
 }
 
+TEST_F(ComputeKnownFPClassTest, fcmpImpliesClass_fabs_zero) {
+  parseAssembly("define float @test(float %x) {\n"
+                "  %A = call float @llvm.fabs.f32(float %x)\n"
+                "  ret float %A\n"
+                "}\n");
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_OEQ, *F, A, fcZero)),
+            fcZero);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_UEQ, *F, A, fcZero)),
+            fcZero | fcNan);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_UNE, *F, A, fcZero)),
+            ~fcZero);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_ONE, *F, A, fcZero)),
+            ~fcNan & ~fcZero);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_ORD, *F, A, fcZero)),
+            ~fcNan);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_UNO, *F, A, fcZero)),
+            fcNan);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_OGT, *F, A, fcZero)),
+            fcSubnormal | fcNormal | fcInf);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_UGT, *F, A, fcZero)),
+            fcSubnormal | fcNormal | fcInf | fcNan);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_OGE, *F, A, fcZero)),
+            ~fcNan);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_UGE, *F, A, fcZero)),
+            fcAllFlags);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_OLT, *F, A, fcZero)),
+            fcNone);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_ULT, *F, A, fcZero)),
+            fcNan);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_OLE, *F, A, fcZero)),
+            fcZero);
+  EXPECT_EQ(std::get<1>(fcmpImpliesClass(FCmpInst::FCMP_ULE, *F, A, fcZero)),
+            fcZero | fcNan);
+}
+
 TEST_F(ValueTrackingTest, isNonZeroRecurrence) {
   parseAssembly(R"(
     define i1 @test(i8 %n, i8 %r) {
diff --git a/llvm/unittests/BinaryFormat/MsgPackDocumentTest.cpp b/llvm/unittests/BinaryFormat/MsgPackDocumentTest.cpp
index a8db0f1ad0cca..6a6ad7010f629 100644
--- a/llvm/unittests/BinaryFormat/MsgPackDocumentTest.cpp
+++ b/llvm/unittests/BinaryFormat/MsgPackDocumentTest.cpp
@@ -22,12 +22,58 @@ TEST(MsgPackDocument, DocNodeTest) {
   ASSERT_TRUE(Str1 == Str2);
 }
 
+TEST(MsgPackDocument, TestReadBoolean) {
+  Document Doc1;
+  bool Ok = Doc1.readFromBlob(StringRef("\xC2", 1), /*Multi=*/false);
+  ASSERT_TRUE(Ok);
+  ASSERT_EQ(Doc1.getRoot().getKind(), Type::Boolean);
+  ASSERT_EQ(Doc1.getRoot().getBool(), false);
+  Document Doc2;
+  Ok = Doc2.readFromBlob(StringRef("\xC3", 1), /*Multi=*/false);
+  ASSERT_TRUE(Ok);
+  ASSERT_EQ(Doc2.getRoot().getKind(), Type::Boolean);
+  ASSERT_EQ(Doc2.getRoot().getBool(), true);
+}
+
 TEST(MsgPackDocument, TestReadInt) {
-  Document Doc;
-  bool Ok = Doc.readFromBlob(StringRef("\xd0\x00", 2), /*Multi=*/false);
+  Document Doc1;
+  bool Ok = Doc1.readFromBlob(StringRef("\xD0\x00", 2), /*Multi=*/false);
+  ASSERT_TRUE(Ok);
+  ASSERT_EQ(Doc1.getRoot().getKind(), Type::Int);
+  ASSERT_EQ(Doc1.getRoot().getInt(), 0);
+  Document Doc2;
+  Ok = Doc2.readFromBlob(StringRef("\xFF", 1), /*Multi=*/false);
+  ASSERT_TRUE(Ok);
+  ASSERT_EQ(Doc2.getRoot().getKind(), Type::Int);
+  ASSERT_EQ(Doc2.getRoot().getInt(), -1);
+}
+
+TEST(MsgPackDocument, TestReadUInt) {
+  Document Doc1;
+  bool Ok = Doc1.readFromBlob(StringRef("\xCC\x00", 2), /*Multi=*/false);
+  ASSERT_TRUE(Ok);
+  ASSERT_EQ(Doc1.getRoot().getKind(), Type::UInt);
+  ASSERT_EQ(Doc1.getRoot().getUInt(), 0U);
+  Document Doc2;
+  Ok = Doc2.readFromBlob(StringRef("\x01", 1), /*Multi=*/false);
   ASSERT_TRUE(Ok);
-  ASSERT_EQ(Doc.getRoot().getKind(), Type::Int);
-  ASSERT_EQ(Doc.getRoot().getInt(), 0);
+  ASSERT_EQ(Doc2.getRoot().getKind(), Type::UInt);
+  ASSERT_EQ(Doc2.getRoot().getUInt(), 1U);
+}
+
+TEST(MsgPackDocument, TestReadFloat) {
+  Document Doc1;
+  bool Ok =
+      Doc1.readFromBlob(StringRef("\xCA\x3F\x80\x00\x00", 5), /*Multi=*/false);
+  ASSERT_TRUE(Ok);
+  ASSERT_EQ(Doc1.getRoot().getKind(), Type::Float);
+  ASSERT_EQ(Doc1.getRoot().getFloat(), 1.0);
+  Document Doc2;
+  Ok = Doc2.readFromBlob(StringRef("\xCB\x48\x3D\x63\x29\xF1\xC3\x5C\xA5", 9),
+                         /*Multi=*/false);
+  ASSERT_TRUE(Ok);
+  ASSERT_EQ(Doc2.getRoot().getKind(), Type::Float);
+  ASSERT_EQ(Doc2.getRoot().getFloat(), 1e40);
 }
 
 TEST(MsgPackDocument, TestReadBinary) {
@@ -192,12 +238,54 @@ TEST(MsgPackDocument, TestReadMergeMap) {
   ASSERT_EQ(BayS.getInt(), 8);
 }
 
+TEST(MsgPackDocument, TestWriteBoolean) {
+  Document Doc;
+  Doc.getRoot() = true;
+  std::string Buffer;
+  Doc.writeToBlob(Buffer);
+  ASSERT_EQ(Buffer, "\xc3");
+  Doc.getRoot() = false;
+  Doc.writeToBlob(Buffer);
+  ASSERT_EQ(Buffer, "\xc2");
+}
+
 TEST(MsgPackDocument, TestWriteInt) {
   Document Doc;
   Doc.getRoot() = 1;
   std::string Buffer;
   Doc.writeToBlob(Buffer);
   ASSERT_EQ(Buffer, "\x01");
+  Doc.getRoot() = -1;
+  Doc.writeToBlob(Buffer);
+  ASSERT_EQ(Buffer, "\xFF");
+  Doc.getRoot() = -4096;
+  Doc.writeToBlob(Buffer);
+  ASSERT_EQ(Buffer, StringRef("\xD1\xF0\x00", 3));
+}
+
+TEST(MsgPackDocument, TestWriteUInt) {
+  Document Doc;
+  Doc.getRoot() = 1U;
+  std::string Buffer;
+  Doc.writeToBlob(Buffer);
+  ASSERT_EQ(Buffer, "\x01");
+  Doc.getRoot() = 4096U;
+  Doc.writeToBlob(Buffer);
+  ASSERT_EQ(Buffer, StringRef("\xCD\x10\x00", 3));
+}
+
+TEST(MsgPackDocument, TestWriteFloat) {
+  Document Doc;
+  Doc.getRoot() = 1.0;
+  std::string Buffer;
+  Doc.writeToBlob(Buffer);
+  ASSERT_EQ(Buffer, StringRef("\xCA\x3F\x80\x00\x00", 5));
+  Doc.getRoot() = 1.0f;
+  Doc.writeToBlob(Buffer);
+  ASSERT_EQ(Buffer, StringRef("\xCA\x3F\x80\x00\x00", 5));
+  Doc.getRoot() = 1e40;
+  Doc.writeToBlob(Buffer);
+  ASSERT_EQ(Buffer, "\xCB\x48\x3D\x63\x29\xF1\xC3\x5C\xA5");
 }
 
 TEST(MsgPackDocument, TestWriteBinary) {
diff --git a/llvm/unittests/BinaryFormat/SFrameTest.cpp b/llvm/unittests/BinaryFormat/SFrameTest.cpp
index 394e382e041e9..ab7b0fe20b750 100644
--- a/llvm/unittests/BinaryFormat/SFrameTest.cpp
+++ b/llvm/unittests/BinaryFormat/SFrameTest.cpp
@@ -54,28 +54,28 @@ TYPED_TEST_SUITE(SFrameTest, Types, NameGenerator);
 
 TYPED_TEST(SFrameTest, FDEFlags) {
   FuncDescEntry<TestFixture::Endian> FDE = {};
-  EXPECT_EQ(FDE.Info, 0u);
-  EXPECT_EQ(FDE.getPAuthKey(), 0);
-  EXPECT_EQ(FDE.getFDEType(), FDEType::PCInc);
-  EXPECT_EQ(FDE.getFREType(), FREType::Addr1);
-
-  FDE.setPAuthKey(1);
-  EXPECT_EQ(FDE.Info, 0x20u);
-  EXPECT_EQ(FDE.getPAuthKey(), 1);
-  EXPECT_EQ(FDE.getFDEType(), FDEType::PCInc);
-  EXPECT_EQ(FDE.getFREType(), FREType::Addr1);
-
-  FDE.setFDEType(FDEType::PCMask);
-  EXPECT_EQ(FDE.Info, 0x30u);
-  EXPECT_EQ(FDE.getPAuthKey(), 1);
-  EXPECT_EQ(FDE.getFDEType(), FDEType::PCMask);
-  EXPECT_EQ(FDE.getFREType(), FREType::Addr1);
-
-  FDE.setFREType(FREType::Addr4);
-  EXPECT_EQ(FDE.Info, 0x32u);
-  EXPECT_EQ(FDE.getPAuthKey(), 1);
-  EXPECT_EQ(FDE.getFDEType(), FDEType::PCMask);
-  EXPECT_EQ(FDE.getFREType(), FREType::Addr4);
+  EXPECT_EQ(FDE.Info.Info, 0u);
+  EXPECT_EQ(FDE.Info.getPAuthKey(), 0);
+  EXPECT_EQ(FDE.Info.getFDEType(), FDEType::PCInc);
+  EXPECT_EQ(FDE.Info.getFREType(), FREType::Addr1);
+
+  FDE.Info.setPAuthKey(1);
+  EXPECT_EQ(FDE.Info.Info, 0x20u);
+  EXPECT_EQ(FDE.Info.getPAuthKey(), 1);
+  EXPECT_EQ(FDE.Info.getFDEType(), FDEType::PCInc);
+  EXPECT_EQ(FDE.Info.getFREType(), FREType::Addr1);
+
+  FDE.Info.setFDEType(FDEType::PCMask);
+  EXPECT_EQ(FDE.Info.Info, 0x30u);
+  EXPECT_EQ(FDE.Info.getPAuthKey(), 1);
+  EXPECT_EQ(FDE.Info.getFDEType(), FDEType::PCMask);
+  EXPECT_EQ(FDE.Info.getFREType(), FREType::Addr1);
+
+  FDE.Info.setFREType(FREType::Addr4);
+  EXPECT_EQ(FDE.Info.Info, 0x32u);
+  EXPECT_EQ(FDE.Info.getPAuthKey(), 1);
+  EXPECT_EQ(FDE.Info.getFDEType(), FDEType::PCMask);
+  EXPECT_EQ(FDE.Info.getFREType(), FREType::Addr4);
 }
 
 TYPED_TEST(SFrameTest, FREFlags) {
diff --git a/llvm/unittests/CAS/ActionCacheTest.cpp b/llvm/unittests/CAS/ActionCacheTest.cpp
new file mode 100644
index 0000000000000..db67e30ca203b
--- /dev/null
+++ b/llvm/unittests/CAS/ActionCacheTest.cpp
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the tests for ActionCaches.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/ActionCache.h"
+#include "CASTestConfig.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+TEST_P(CASTest, ActionCacheHit) {
+  std::shared_ptr<ObjectStore> CAS = createObjectStore();
+  std::unique_ptr<ActionCache> Cache = createActionCache();
+
+  std::optional<ObjectProxy> ID;
+  ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID), Succeeded());
+  std::optional<CASID> ResultID;
+  ASSERT_THAT_ERROR(Cache->put(*ID, *ID), Succeeded());
+  ASSERT_THAT_ERROR(Cache->get(*ID).moveInto(ResultID), Succeeded());
+  ASSERT_TRUE(ResultID);
+  std::optional<ObjectRef> Result = CAS->getReference(*ResultID);
+  ASSERT_TRUE(Result);
+  ASSERT_EQ(*ID, *Result);
+}
+
+TEST_P(CASTest, ActionCacheMiss) {
+  std::shared_ptr<ObjectStore> CAS = createObjectStore();
+  std::unique_ptr<ActionCache> Cache = createActionCache();
+
+  std::optional<ObjectProxy> ID1, ID2;
+  ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID1), Succeeded());
+  ASSERT_THAT_ERROR(CAS->createProxy({}, "2").moveInto(ID2), Succeeded());
+  ASSERT_THAT_ERROR(Cache->put(*ID1, *ID2), Succeeded());
+  // This is a cache miss for looking up a key doesn't exist.
+  std::optional<CASID> Result1;
+  ASSERT_THAT_ERROR(Cache->get(*ID2).moveInto(Result1), Succeeded());
+  ASSERT_FALSE(Result1);
+
+  ASSERT_THAT_ERROR(Cache->put(*ID2, *ID1), Succeeded());
+  // Cache hit after adding the value.
+  std::optional<CASID> Result2;
+  ASSERT_THAT_ERROR(Cache->get(*ID2).moveInto(Result2), Succeeded());
+  ASSERT_TRUE(Result2);
+  std::optional<ObjectRef> Ref = CAS->getReference(*Result2);
+  ASSERT_TRUE(Ref);
+  ASSERT_EQ(*ID1, *Ref);
+}
+
+TEST_P(CASTest, ActionCacheRewrite) {
+  std::shared_ptr<ObjectStore> CAS = createObjectStore();
+  std::unique_ptr<ActionCache> Cache = createActionCache();
+
+  std::optional<ObjectProxy> ID1, ID2;
+  ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID1), Succeeded());
+  ASSERT_THAT_ERROR(CAS->createProxy({}, "2").moveInto(ID2), Succeeded());
+  ASSERT_THAT_ERROR(Cache->put(*ID1, *ID1), Succeeded());
+  // Writing to the same key with different value is error.
+  ASSERT_THAT_ERROR(Cache->put(*ID1, *ID2), Failed());
+  // Writing the same value multiple times to the same key is fine.
+  ASSERT_THAT_ERROR(Cache->put(*ID1, *ID1), Succeeded());
+}
diff --git a/llvm/unittests/CAS/CASTestConfig.cpp b/llvm/unittests/CAS/CASTestConfig.cpp
new file mode 100644
index 0000000000000..29e2db48db5c0
--- /dev/null
+++ b/llvm/unittests/CAS/CASTestConfig.cpp
@@ -0,0 +1,21 @@
+//===- CASTestConfig.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CASTestConfig.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+static CASTestingEnv createInMemory(int I) {
+  return CASTestingEnv{createInMemoryCAS(), createInMemoryActionCache()};
+}
+
+INSTANTIATE_TEST_SUITE_P(InMemoryCAS, CASTest,
+                         ::testing::Values(createInMemory));
diff --git a/llvm/unittests/CAS/CASTestConfig.h b/llvm/unittests/CAS/CASTestConfig.h
new file mode 100644
index 0000000000000..8093a0b0864f9
--- /dev/null
+++ b/llvm/unittests/CAS/CASTestConfig.h
@@ -0,0 +1,38 @@
+//===- CASTestConfig.h ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "gtest/gtest.h"
+
+#ifndef LLVM_UNITTESTS_CASTESTCONFIG_H
+#define LLVM_UNITTESTS_CASTESTCONFIG_H
+
+struct CASTestingEnv {
+  std::unique_ptr<llvm::cas::ObjectStore> CAS;
+  std::unique_ptr<llvm::cas::ActionCache> Cache;
+};
+
+class CASTest
+    : public testing::TestWithParam<std::function<CASTestingEnv(int)>> {
+protected:
+  std::optional<int> NextCASIndex;
+
+  std::unique_ptr<llvm::cas::ObjectStore> createObjectStore() {
+    auto TD = GetParam()(++(*NextCASIndex));
+    return std::move(TD.CAS);
+  }
+  std::unique_ptr<llvm::cas::ActionCache> createActionCache() {
+    auto TD = GetParam()(++(*NextCASIndex));
+    return std::move(TD.Cache);
+  }
+  void SetUp() { NextCASIndex = 0; }
+  void TearDown() { NextCASIndex = std::nullopt; }
+};
+
+#endif
diff --git a/llvm/unittests/CAS/CMakeLists.txt b/llvm/unittests/CAS/CMakeLists.txt
new file mode 100644
index 0000000000000..ff081007f31bc
--- /dev/null
+++ b/llvm/unittests/CAS/CMakeLists.txt
@@ -0,0 +1,13 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  CAS
+  TestingSupport
+  )
+
+add_llvm_unittest(CASTests
+  ActionCacheTest.cpp
+  CASTestConfig.cpp
+  ObjectStoreTest.cpp
+  )
+
+target_link_libraries(CASTests PRIVATE LLVMTestingSupport)
diff --git a/llvm/unittests/CAS/ObjectStoreTest.cpp b/llvm/unittests/CAS/ObjectStoreTest.cpp
new file mode 100644
index 0000000000000..54083fdb408f6
--- /dev/null
+++ b/llvm/unittests/CAS/ObjectStoreTest.cpp
@@ -0,0 +1,348 @@
+//===- ObjectStoreTest.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/Process.h"
+#include "llvm/Support/RandomNumberGenerator.h"
+#include "llvm/Support/ThreadPool.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+#include "CASTestConfig.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+
+TEST_P(CASTest, PrintIDs) {
+  std::unique_ptr<ObjectStore> CAS = createObjectStore();
+
+  std::optional<CASID> ID1, ID2;
+  ASSERT_THAT_ERROR(CAS->createProxy({}, "1").moveInto(ID1), Succeeded());
+  ASSERT_THAT_ERROR(CAS->createProxy({}, "2").moveInto(ID2), Succeeded());
+  EXPECT_NE(ID1, ID2);
+  std::string PrintedID1 = ID1->toString();
+  std::string PrintedID2 = ID2->toString();
+  EXPECT_NE(PrintedID1, PrintedID2);
+
+  std::optional<CASID> ParsedID1, ParsedID2;
+  ASSERT_THAT_ERROR(CAS->parseID(PrintedID1).moveInto(ParsedID1), Succeeded());
+  ASSERT_THAT_ERROR(CAS->parseID(PrintedID2).moveInto(ParsedID2), Succeeded());
+  EXPECT_EQ(ID1, ParsedID1);
+  EXPECT_EQ(ID2, ParsedID2);
+}
+
+TEST_P(CASTest, Blobs) {
+  std::unique_ptr<ObjectStore> CAS1 = createObjectStore();
+  StringRef ContentStrings[] = {
+      "word",
+      "some longer text std::string's local memory",
+      R"(multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text)",
+  };
+
+  SmallVector<CASID> IDs;
+  for (StringRef Content : ContentStrings) {
+    // Use StringRef::str() to create a temporary std::string. This could cause
+    // problems if the CAS is storing references to the input string instead of
+    // copying it.
+    std::optional<ObjectProxy> Blob;
+    ASSERT_THAT_ERROR(CAS1->createProxy({}, Content).moveInto(Blob),
+                      Succeeded());
+    IDs.push_back(Blob->getID());
+
+    // Check basic printing of IDs.
+    EXPECT_EQ(IDs.back().toString(), IDs.back().toString());
+    if (IDs.size() > 2)
+      EXPECT_NE(IDs.front().toString(), IDs.back().toString());
+  }
+
+  // Check that the blobs give the same IDs later.
+  for (int I = 0, E = IDs.size(); I != E; ++I) {
+    std::optional<ObjectProxy> Blob;
+    ASSERT_THAT_ERROR(CAS1->createProxy({}, ContentStrings[I]).moveInto(Blob),
+                      Succeeded());
+    EXPECT_EQ(IDs[I], Blob->getID());
+  }
+
+  // Run validation on all CASIDs.
+  for (int I = 0, E = IDs.size(); I != E; ++I)
+    ASSERT_THAT_ERROR(CAS1->validate(IDs[I]), Succeeded());
+
+  // Check that the blobs can be retrieved multiple times.
+  for (int I = 0, E = IDs.size(); I != E; ++I) {
+    for (int J = 0, JE = 3; J != JE; ++J) {
+      std::optional<ObjectProxy> Buffer;
+      ASSERT_THAT_ERROR(CAS1->getProxy(IDs[I]).moveInto(Buffer), Succeeded());
+      EXPECT_EQ(ContentStrings[I], Buffer->getData());
+    }
+  }
+
+  // Confirm these blobs don't exist in a fresh CAS instance.
+  std::unique_ptr<ObjectStore> CAS2 = createObjectStore();
+  for (int I = 0, E = IDs.size(); I != E; ++I) {
+    std::optional<ObjectProxy> Proxy;
+    EXPECT_THAT_ERROR(CAS2->getProxy(IDs[I]).moveInto(Proxy), Failed());
+  }
+
+  // Insert into the second CAS and confirm the IDs are stable. Getting them
+  // should work now.
+  for (int I = IDs.size(), E = 0; I != E; --I) {
+    auto &ID = IDs[I - 1];
+    auto &Content = ContentStrings[I - 1];
+    std::optional<ObjectProxy> Blob;
+    ASSERT_THAT_ERROR(CAS2->createProxy({}, Content).moveInto(Blob),
+                      Succeeded());
+    EXPECT_EQ(ID, Blob->getID());
+
+    std::optional<ObjectProxy> Buffer;
+    ASSERT_THAT_ERROR(CAS2->getProxy(ID).moveInto(Buffer), Succeeded());
+    EXPECT_EQ(Content, Buffer->getData());
+  }
+}
+
+TEST_P(CASTest, BlobsBig) {
+  // A little bit of validation that bigger blobs are okay. Climb up to 1MB.
+  std::unique_ptr<ObjectStore> CAS = createObjectStore();
+  SmallString<256> String1 = StringRef("a few words");
+  SmallString<256> String2 = StringRef("others");
+  while (String1.size() < 1024U * 1024U) {
+    std::optional<CASID> ID1;
+    std::optional<CASID> ID2;
+    ASSERT_THAT_ERROR(CAS->createProxy({}, String1).moveInto(ID1), Succeeded());
+    ASSERT_THAT_ERROR(CAS->createProxy({}, String1).moveInto(ID2), Succeeded());
+    ASSERT_THAT_ERROR(CAS->validate(*ID1), Succeeded());
+    ASSERT_THAT_ERROR(CAS->validate(*ID2), Succeeded());
+    ASSERT_EQ(ID1, ID2);
+
+    String1.append(String2);
+    ASSERT_THAT_ERROR(CAS->createProxy({}, String2).moveInto(ID1), Succeeded());
+    ASSERT_THAT_ERROR(CAS->createProxy({}, String2).moveInto(ID2), Succeeded());
+    ASSERT_THAT_ERROR(CAS->validate(*ID1), Succeeded());
+    ASSERT_THAT_ERROR(CAS->validate(*ID2), Succeeded());
+    ASSERT_EQ(ID1, ID2);
+    String2.append(String1);
+  }
+
+  // Specifically check near 1MB for objects large enough they're likely to be
+  // stored externally in an on-disk CAS and will be near a page boundary.
+  SmallString<0> Storage;
+  const size_t InterestingSize = 1024U * 1024ULL;
+  const size_t SizeE = InterestingSize + 2;
+  if (Storage.size() < SizeE)
+    Storage.resize(SizeE, '\01');
+  for (size_t Size = InterestingSize - 2; Size != SizeE; ++Size) {
+    StringRef Data(Storage.data(), Size);
+    std::optional<ObjectProxy> Blob;
+    ASSERT_THAT_ERROR(CAS->createProxy({}, Data).moveInto(Blob), Succeeded());
+    ASSERT_EQ(Data, Blob->getData());
+    ASSERT_EQ(0, Blob->getData().end()[0]);
+  }
+}
+
+TEST_P(CASTest, LeafNodes) {
+  std::unique_ptr<ObjectStore> CAS1 = createObjectStore();
+  StringRef ContentStrings[] = {
+      "word",
+      "some longer text std::string's local memory",
+      R"(multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text
+multiline text multiline text multiline text multiline text multiline text)",
+  };
+
+  SmallVector<ObjectRef> Nodes;
+  SmallVector<CASID> IDs;
+  for (StringRef Content : ContentStrings) {
+    // Use StringRef::str() to create a temporary std::string. This could cause
+    // problems if the CAS is storing references to the input string instead of
+    // copying it.
+    std::optional<ObjectRef> Node;
+    ASSERT_THAT_ERROR(
+        CAS1->store({}, arrayRefFromStringRef<char>(Content)).moveInto(Node),
+        Succeeded());
+    Nodes.push_back(*Node);
+
+    // Check basic printing of IDs.
+    IDs.push_back(CAS1->getID(*Node));
+    auto ID = CAS1->getID(Nodes.back());
+    EXPECT_EQ(ID.toString(), IDs.back().toString());
+    EXPECT_EQ(*Node, Nodes.back());
+    EXPECT_EQ(ID, IDs.back());
+    if (Nodes.size() <= 1)
+      continue;
+    EXPECT_NE(Nodes.front(), Nodes.back());
+    EXPECT_NE(IDs.front(), IDs.back());
+  }
+
+  // Check that the blobs give the same IDs later.
+  for (int I = 0, E = IDs.size(); I != E; ++I) {
+    std::optional<ObjectRef> Node;
+    ASSERT_THAT_ERROR(
+        CAS1->store({}, arrayRefFromStringRef<char>(ContentStrings[I]))
+            .moveInto(Node),
+        Succeeded());
+    EXPECT_EQ(IDs[I], CAS1->getID(*Node));
+  }
+
+  // Check that the blobs can be retrieved multiple times.
+  for (int I = 0, E = IDs.size(); I != E; ++I) {
+    for (int J = 0, JE = 3; J != JE; ++J) {
+      std::optional<ObjectProxy> Object;
+      ASSERT_THAT_ERROR(CAS1->getProxy(IDs[I]).moveInto(Object), Succeeded());
+      ASSERT_TRUE(Object);
+      EXPECT_EQ(ContentStrings[I], Object->getData());
+    }
+  }
+
+  // Confirm these blobs don't exist in a fresh CAS instance.
+  std::unique_ptr<ObjectStore> CAS2 = createObjectStore();
+  for (int I = 0, E = IDs.size(); I != E; ++I) {
+    std::optional<ObjectProxy> Object;
+    EXPECT_THAT_ERROR(CAS2->getProxy(IDs[I]).moveInto(Object), Failed());
+  }
+
+  // Insert into the second CAS and confirm the IDs are stable. Getting them
+  // should work now.
+  for (int I = IDs.size(), E = 0; I != E; --I) {
+    auto &ID = IDs[I - 1];
+    auto &Content = ContentStrings[I - 1];
+    std::optional<ObjectRef> Node;
+    ASSERT_THAT_ERROR(
+        CAS2->store({}, arrayRefFromStringRef<char>(Content)).moveInto(Node),
+        Succeeded());
+    EXPECT_EQ(ID, CAS2->getID(*Node));
+
+    std::optional<ObjectProxy> Object;
+    ASSERT_THAT_ERROR(CAS2->getProxy(ID).moveInto(Object), Succeeded());
+    ASSERT_TRUE(Object);
+    EXPECT_EQ(Content, Object->getData());
+  }
+}
+
+TEST_P(CASTest, NodesBig) {
+  std::unique_ptr<ObjectStore> CAS = createObjectStore();
+
+  // Specifically check near 1MB for objects large enough they're likely to be
+  // stored externally in an on-disk CAS, and such that one of them will be
+  // near a page boundary.
+  SmallString<0> Storage;
+  constexpr size_t InterestingSize = 1024U * 1024ULL;
+  constexpr size_t WordSize = sizeof(void *);
+
+  // Start much smaller to account for headers.
+  constexpr size_t SizeB = InterestingSize - 8 * WordSize;
+  constexpr size_t SizeE = InterestingSize + 1;
+  if (Storage.size() < SizeE)
+    Storage.resize(SizeE, '\01');
+
+  SmallVector<ObjectRef, 4> CreatedNodes;
+  // Avoid checking every size because this is an expensive test. Just check
+  // for data that is 8B-word-aligned, and one less. Also appending the created
+  // nodes as the references in the next block to check references are created
+  // correctly.
+  for (size_t Size = SizeB; Size < SizeE; Size += WordSize) {
+    for (bool IsAligned : {false, true}) {
+      StringRef Data(Storage.data(), Size - (IsAligned ? 0 : 1));
+      std::optional<ObjectProxy> Node;
+      ASSERT_THAT_ERROR(CAS->createProxy(CreatedNodes, Data).moveInto(Node),
+                        Succeeded());
+      ASSERT_EQ(Data, Node->getData());
+      ASSERT_EQ(0, Node->getData().end()[0]);
+      ASSERT_EQ(Node->getNumReferences(), CreatedNodes.size());
+      CreatedNodes.emplace_back(Node->getRef());
+    }
+  }
+
+  for (auto ID : CreatedNodes)
+    ASSERT_THAT_ERROR(CAS->validate(CAS->getID(ID)), Succeeded());
+}
+
+#if LLVM_ENABLE_THREADS
+/// Common test functionality for creating blobs in parallel. You can vary which
+/// cas instances are the same or different, and the size of the created blobs.
+static void testBlobsParallel(ObjectStore &Read1, ObjectStore &Read2,
+                              ObjectStore &Write1, ObjectStore &Write2,
+                              uint64_t BlobSize) {
+  SCOPED_TRACE("testBlobsParallel");
+  unsigned BlobCount = 100;
+  std::vector<std::string> Blobs;
+  Blobs.reserve(BlobCount);
+  for (unsigned I = 0; I < BlobCount; ++I) {
+    std::string Blob;
+    Blob.resize(BlobSize);
+    getRandomBytes(Blob.data(), BlobSize);
+    Blobs.push_back(std::move(Blob));
+  }
+
+  std::mutex NodesMtx;
+  std::vector<std::optional<CASID>> CreatedNodes(BlobCount);
+
+  auto Producer = [&](unsigned I, ObjectStore *CAS) {
+    std::optional<ObjectProxy> Node;
+    EXPECT_THAT_ERROR(CAS->createProxy({}, Blobs[I]).moveInto(Node),
+                      Succeeded());
+    {
+      std::lock_guard<std::mutex> L(NodesMtx);
+      CreatedNodes[I] = Node ? Node->getID() : CASID::getDenseMapTombstoneKey();
+    }
+  };
+
+  auto Consumer = [&](unsigned I, ObjectStore *CAS) {
+    std::optional<CASID> ID;
+    while (!ID) {
+      // Busy wait.
+      std::lock_guard<std::mutex> L(NodesMtx);
+      ID = CreatedNodes[I];
+    }
+    if (ID == CASID::getDenseMapTombstoneKey())
+      // Producer failed; already reported.
+      return;
+
+    std::optional<ObjectProxy> Node;
+    ASSERT_THAT_ERROR(CAS->getProxy(*ID).moveInto(Node), Succeeded());
+    EXPECT_EQ(Node->getData(), Blobs[I]);
+  };
+
+  DefaultThreadPool Threads;
+  for (unsigned I = 0; I < BlobCount; ++I) {
+    Threads.async(Producer, I, &Write1);
+    Threads.async(Producer, I, &Write2);
+    Threads.async(Consumer, I, &Read1);
+    Threads.async(Consumer, I, &Read2);
+  }
+
+  Threads.wait();
+}
+
+static void testBlobsParallel1(ObjectStore &CAS, uint64_t BlobSize) {
+  SCOPED_TRACE("testBlobsParallel1");
+  testBlobsParallel(CAS, CAS, CAS, CAS, BlobSize);
+}
+
+TEST_P(CASTest, BlobsParallel) {
+  std::shared_ptr<ObjectStore> CAS = createObjectStore();
+  uint64_t Size = 1ULL * 1024;
+  ASSERT_NO_FATAL_FAILURE(testBlobsParallel1(*CAS, Size));
+}
+
+#ifdef EXPENSIVE_CHECKS
+TEST_P(CASTest, BlobsBigParallel) {
+  std::shared_ptr<ObjectStore> CAS = createObjectStore();
+  // 100k is large enough to be standalone files in our on-disk cas.
+  uint64_t Size = 100ULL * 1024;
+  ASSERT_NO_FATAL_FAILURE(testBlobsParallel1(*CAS, Size));
+}
+#endif // EXPENSIVE_CHECKS
+#endif // LLVM_ENABLE_THREADS
diff --git a/llvm/unittests/CGData/StableFunctionMapTest.cpp b/llvm/unittests/CGData/StableFunctionMapTest.cpp
index d551ac8a814f4..5cf62ae0b3943 100644
--- a/llvm/unittests/CGData/StableFunctionMapTest.cpp
+++ b/llvm/unittests/CGData/StableFunctionMapTest.cpp
@@ -117,7 +117,7 @@ TEST(StableFunctionMap, Finalize3) {
   Map.finalize();
   auto &M = Map.getFunctionMap();
   EXPECT_THAT(M, SizeIs(1));
-  auto &FuncEntries = M.begin()->second;
+  auto &FuncEntries = M.begin()->second.Entries;
   for (auto &FuncEntry : FuncEntries) {
     EXPECT_THAT(*FuncEntry->IndexOperandHashMap, SizeIs(1));
     ASSERT_THAT(*FuncEntry->IndexOperandHashMap,
diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt
index 81abce51b8939..d22613d12d6dc 100644
--- a/llvm/unittests/CMakeLists.txt
+++ b/llvm/unittests/CMakeLists.txt
@@ -34,6 +34,7 @@ add_subdirectory(AsmParser)
 add_subdirectory(BinaryFormat)
 add_subdirectory(Bitcode)
 add_subdirectory(Bitstream)
+add_subdirectory(CAS)
 add_subdirectory(CGData)
 add_subdirectory(CodeGen)
 add_subdirectory(DebugInfo)
diff --git a/llvm/unittests/CodeGen/DroppedVariableStatsMIRTest.cpp b/llvm/unittests/CodeGen/DroppedVariableStatsMIRTest.cpp
index e72b4e4aea49f..c0cf4ff546d7b 100644
--- a/llvm/unittests/CodeGen/DroppedVariableStatsMIRTest.cpp
+++ b/llvm/unittests/CodeGen/DroppedVariableStatsMIRTest.cpp
@@ -158,8 +158,8 @@ tracksDebugUserValues: false
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -307,8 +307,8 @@ tracksDebugUserValues: false
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -450,8 +450,8 @@ tracksDebugUserValues: false
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -593,8 +593,8 @@ tracksDebugUserValues: false
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -738,8 +738,8 @@ tracksDebugUserValues: false
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -883,8 +883,8 @@ tracksDebugUserValues: false
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
@@ -1029,8 +1029,8 @@ tracksDebugUserValues: false
   hasTailCall:     false
   isCalleeSavedInfoValid: false
   localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
+  savePoint:       []
+  restorePoint:    []
 fixedStack:      []
 stack:           []
 entry_values:    []
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
index 67e6d140a445f..73ddf0c88d3ed 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsVectorTest.cpp
@@ -1548,48 +1548,3 @@ TEST_F(AArch64GISelMITest, TestNumSignBitsUAddoOverflow) {
   // Assert sign-extension from vector boolean
   EXPECT_EQ(32u, Info.computeNumSignBits(CopyOverflow));
 }
-
-TEST_F(AArch64GISelMITest, TestKnwonBitsUnmergeVectorScalar) {
-  StringRef MIRString = R"(
-   %copy_x0:_(<2 x s16>) = COPY $w0
-   %maskf:_(s16) = G_CONSTANT i16 15
-   %x0_x1:_(<2 x s16>) = G_BUILD_VECTOR %maskf, %maskf
-   %and:_(<2 x s16>) = G_AND %copy_x0, %x0_x1
-   %x0_0:_(s16), %x0_1:_(s16) = G_UNMERGE_VALUES %and
-   %result:_(s16) = COPY %x0_0
-)";
-
-  setUp(MIRString);
-  if (!TM)
-    GTEST_SKIP();
-
-  Register CopyOverflow = Copies[Copies.size() - 1];
-
-  GISelValueTracking Info(*MF);
-
-  EXPECT_EQ(0xFFF0u, Info.getKnownBits(CopyOverflow).Zero.getZExtValue());
-}
-
-TEST_F(AArch64GISelMITest, TestKnwonBitsUnmergeVectorVector) {
-  StringRef MIRString = R"(
-   %copy_x0:_(<4 x s8>) = COPY $w0
-   %maskff:_(s8) = G_CONSTANT i8 255
-   %maskf:_(s8) = G_CONSTANT i8 15
-   %x0_x1:_(<4 x s8>) = G_BUILD_VECTOR %maskf, %maskf, %maskff, %maskff
-   %and:_(<4 x s8>) = G_AND %copy_x0, %x0_x1
-   %x0_0:_(<2 x s8>), %x0_1:_(<2 x s8>) = G_UNMERGE_VALUES %and
-   %result1:_(<2 x s8>) = COPY %x0_0
-   %result2:_(<2 x s8>) = COPY %x0_1
-)";
-
-  setUp(MIRString);
-  if (!TM)
-    GTEST_SKIP();
-
-  GISelValueTracking Info(*MF);
-
-  Register CopyOverflow1 = Copies[Copies.size() - 2];
-  EXPECT_EQ(0xF0u, Info.getKnownBits(CopyOverflow1).Zero.getZExtValue());
-  Register CopyOverflow2 = Copies[Copies.size() - 1];
-  EXPECT_EQ(0x00u, Info.getKnownBits(CopyOverflow2).Zero.getZExtValue());
-}
diff --git a/llvm/unittests/CodeGen/SelectionDAGTestBase.h b/llvm/unittests/CodeGen/SelectionDAGTestBase.h
index edc730d7f9b45..8a0a05fc2f8b0 100644
--- a/llvm/unittests/CodeGen/SelectionDAGTestBase.h
+++ b/llvm/unittests/CodeGen/SelectionDAGTestBase.h
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
@@ -72,12 +71,8 @@ class SelectionDAGTestBase : public testing::Test {
     if (!DAG)
       reportFatalUsageError("Failed to create SelectionDAG?");
     OptimizationRemarkEmitter ORE(F);
-    FunctionAnalysisManager FAM;
-    FAM.registerPass([&] { return TM->getTargetIRAnalysis(); });
-
-    TargetTransformInfo TTI = TM->getTargetIRAnalysis().run(*F, FAM);
     DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
-              nullptr, TTI.hasBranchDivergence(F));
+              nullptr);
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {
diff --git a/llvm/unittests/CodeGen/TypeTraitsTest.cpp b/llvm/unittests/CodeGen/TypeTraitsTest.cpp
index 0ca4fa4e82c7c..dde86280cff6a 100644
--- a/llvm/unittests/CodeGen/TypeTraitsTest.cpp
+++ b/llvm/unittests/CodeGen/TypeTraitsTest.cpp
@@ -16,7 +16,6 @@
 
 using namespace llvm;
 
-#if __has_feature(is_trivially_copyable) || (defined(__GNUC__) && __GNUC__ >= 5)
 static_assert(std::is_trivially_copyable_v<PressureChange>,
               "trivially copyable");
 static_assert(std::is_trivially_copyable_v<SDep>, "trivially copyable");
@@ -24,5 +23,3 @@ static_assert(std::is_trivially_copyable_v<SDValue>, "trivially copyable");
 static_assert(std::is_trivially_copyable_v<SlotIndex>, "trivially copyable");
 static_assert(std::is_trivially_copyable_v<IdentifyingPassPtr>,
               "trivially copyable");
-#endif
-
diff --git a/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp b/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
index bbb9e8d3d6a75..7db561ce84f7d 100644
--- a/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
@@ -31,15 +31,21 @@ const llvm::StringRef FooSrc = R"(
   }
 )";
 
-static ThreadSafeModule parseModule(llvm::StringRef Source,
-                                    llvm::StringRef Name) {
-  auto Ctx = std::make_unique<LLVMContext>();
+static std::unique_ptr<Module>
+parseModuleRaw(llvm::StringRef Source, llvm::StringRef Name, LLVMContext &Ctx) {
   SMDiagnostic Err;
-  auto M = parseIR(MemoryBufferRef(Source, Name), Err, *Ctx);
+  auto M = parseIR(MemoryBufferRef(Source, Name), Err, Ctx);
   if (!M) {
     Err.print("Testcase source failed to parse: ", errs());
     exit(1);
   }
+  return M;
+}
+
+static ThreadSafeModule parseModule(llvm::StringRef Source,
+                                    llvm::StringRef Name) {
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = parseModuleRaw(Source, Name, *Ctx);
   return ThreadSafeModule(std::move(M), std::move(Ctx));
 }
 
@@ -128,6 +134,20 @@ TEST(ThreadSafeModuleTest, ConsumingModuleDo) {
   TSM.consumingModuleDo([](std::unique_ptr<Module> M) {});
 }
 
+TEST(ThreadSafeModuleTest, CloneExternalModuleToNewContext) {
+  auto Ctx = std::make_unique<LLVMContext>();
+  auto M = parseModuleRaw(FooSrc, "foo.ll", *Ctx);
+  auto TSCtx = ThreadSafeContext(std::make_unique<LLVMContext>());
+  auto TSM = cloneExternalModuleToContext(*M, TSCtx);
+  TSM.withModuleDo([&](Module &NewM) {
+    EXPECT_NE(&NewM.getContext(), Ctx.get());
+    TSCtx.withContextDo(
+        [&](LLVMContext *NewCtx) { EXPECT_EQ(&NewM.getContext(), NewCtx); });
+    EXPECT_FALSE(NewM.empty());
+    EXPECT_FALSE(verifyModule(NewM, &errs()));
+  });
+}
+
 TEST(ThreadSafeModuleTest, CloneToNewContext) {
   auto TSM1 = parseModule(FooSrc, "foo.ll");
   auto TSM2 = cloneToNewContext(TSM1);
diff --git a/llvm/unittests/Frontend/HLSLBindingTest.cpp b/llvm/unittests/Frontend/HLSLBindingTest.cpp
index ca2f7b55dd613..87106c29fbf12 100644
--- a/llvm/unittests/Frontend/HLSLBindingTest.cpp
+++ b/llvm/unittests/Frontend/HLSLBindingTest.cpp
@@ -273,3 +273,21 @@ TEST(HLSLBindingTest, TestFindAvailable) {
   // In an empty space we find the slot at the beginning.
   EXPECT_THAT(V, HasSpecificValue(0u));
 }
+
+// Test that add duplicate bindings are correctly de-duplicated
+TEST(HLSLBindingTest, TestNoOverlapWithDuplicates) {
+  hlsl::BindingInfoBuilder Builder;
+
+  // We add the same binding three times, and just use `nullptr` for the cookie
+  // so that they should all be uniqued away.
+  Builder.trackBinding(ResourceClass::SRV, /*Space=*/0, /*LowerBound=*/5,
+                       /*UpperBound=*/5, /*Cookie=*/nullptr);
+  Builder.trackBinding(ResourceClass::SRV, /*Space=*/0, /*LowerBound=*/5,
+                       /*UpperBound=*/5, /*Cookie=*/nullptr);
+  Builder.trackBinding(ResourceClass::SRV, /*Space=*/0, /*LowerBound=*/5,
+                       /*UpperBound=*/5, /*Cookie=*/nullptr);
+  bool HasOverlap;
+  hlsl::BindingInfo Info = Builder.calculateBindingInfo(HasOverlap);
+
+  EXPECT_FALSE(HasOverlap);
+}
diff --git a/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp b/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
index 98b33fdfb8c12..1eb03f16527ec 100644
--- a/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
+++ b/llvm/unittests/Frontend/HLSLRootSignatureDumpTest.cpp
@@ -10,12 +10,13 @@
 #include "gtest/gtest.h"
 
 using namespace llvm::hlsl::rootsig;
+using llvm::dxil::ResourceClass;
 
 namespace {
 
 TEST(HLSLRootSignatureTest, DescriptorCBVClauseDump) {
   DescriptorTableClause Clause;
-  Clause.Type = ClauseType::CBuffer;
+  Clause.Type = ResourceClass::CBuffer;
   Clause.Reg = {RegisterType::BReg, 0};
   Clause.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_1);
 
@@ -32,7 +33,7 @@ TEST(HLSLRootSignatureTest, DescriptorCBVClauseDump) {
 
 TEST(HLSLRootSignatureTest, DescriptorSRVClauseDump) {
   DescriptorTableClause Clause;
-  Clause.Type = ClauseType::SRV;
+  Clause.Type = ResourceClass::SRV;
   Clause.Reg = {RegisterType::TReg, 0};
   Clause.NumDescriptors = NumDescriptorsUnbounded;
   Clause.Space = 42;
@@ -52,7 +53,7 @@ TEST(HLSLRootSignatureTest, DescriptorSRVClauseDump) {
 TEST(HLSLRootSignatureTest, DescriptorUAVClauseDump) {
   using llvm::dxbc::DescriptorRangeFlags;
   DescriptorTableClause Clause;
-  Clause.Type = ClauseType::UAV;
+  Clause.Type = ResourceClass::UAV;
   Clause.Reg = {RegisterType::UReg, 92374};
   Clause.NumDescriptors = 3298;
   Clause.Space = 932847;
@@ -82,7 +83,7 @@ TEST(HLSLRootSignatureTest, DescriptorUAVClauseDump) {
 
 TEST(HLSLRootSignatureTest, DescriptorSamplerClauseDump) {
   DescriptorTableClause Clause;
-  Clause.Type = ClauseType::Sampler;
+  Clause.Type = ResourceClass::Sampler;
   Clause.Reg = {RegisterType::SReg, 0};
   Clause.NumDescriptors = 2;
   Clause.Space = 42;
@@ -102,7 +103,7 @@ TEST(HLSLRootSignatureTest, DescriptorSamplerClauseDump) {
 
 TEST(HLSLRootSignatureTest, DescriptorCBVV10ClauseDump) {
   DescriptorTableClause Clause;
-  Clause.Type = ClauseType::CBuffer;
+  Clause.Type = ResourceClass::CBuffer;
   Clause.Reg = {RegisterType::BReg, 0};
   Clause.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
 
@@ -119,7 +120,7 @@ TEST(HLSLRootSignatureTest, DescriptorCBVV10ClauseDump) {
 
 TEST(HLSLRootSignatureTest, DescriptorSamplerV10ClauseDump) {
   DescriptorTableClause Clause;
-  Clause.Type = ClauseType::Sampler;
+  Clause.Type = ResourceClass::Sampler;
   Clause.Reg = {RegisterType::SReg, 0};
   Clause.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
 
@@ -151,7 +152,7 @@ TEST(HLSLRootSignatureTest, DescriptorTableDump) {
 
 TEST(HLSLRootSignatureTest, RootCBVDump) {
   RootDescriptor Descriptor;
-  Descriptor.Type = DescriptorType::CBuffer;
+  Descriptor.Type = ResourceClass::CBuffer;
   Descriptor.Reg = {RegisterType::BReg, 0};
   Descriptor.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_1);
 
@@ -168,7 +169,7 @@ TEST(HLSLRootSignatureTest, RootCBVDump) {
 
 TEST(HLSLRootSignatureTest, RootSRV10Dump) {
   RootDescriptor Descriptor;
-  Descriptor.Type = DescriptorType::SRV;
+  Descriptor.Type = ResourceClass::SRV;
   Descriptor.Reg = {RegisterType::TReg, 0};
   Descriptor.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
 
@@ -185,7 +186,7 @@ TEST(HLSLRootSignatureTest, RootSRV10Dump) {
 
 TEST(HLSLRootSignatureTest, RootUAVV10Dump) {
   RootDescriptor Descriptor;
-  Descriptor.Type = DescriptorType::UAV;
+  Descriptor.Type = ResourceClass::UAV;
   Descriptor.Reg = {RegisterType::UReg, 0};
   Descriptor.setDefaultFlags(llvm::dxbc::RootSignatureVersion::V1_0);
 
@@ -202,7 +203,7 @@ TEST(HLSLRootSignatureTest, RootUAVV10Dump) {
 
 TEST(HLSLRootSignatureTest, RootSRVDump) {
   RootDescriptor Descriptor;
-  Descriptor.Type = DescriptorType::SRV;
+  Descriptor.Type = ResourceClass::SRV;
   Descriptor.Reg = {RegisterType::TReg, 0};
   Descriptor.Space = 42;
   Descriptor.Visibility = llvm::dxbc::ShaderVisibility::Geometry;
@@ -221,7 +222,7 @@ TEST(HLSLRootSignatureTest, RootSRVDump) {
 TEST(HLSLRootSignatureTest, RootUAVDump) {
   using llvm::dxbc::RootDescriptorFlags;
   RootDescriptor Descriptor;
-  Descriptor.Type = DescriptorType::UAV;
+  Descriptor.Type = ResourceClass::UAV;
   Descriptor.Reg = {RegisterType::UReg, 92374};
   Descriptor.Space = 932847;
   Descriptor.Visibility = llvm::dxbc::ShaderVisibility::Hull;
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index d6b578aa8ffd1..b7a060bb3563d 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
+#include <cstdlib>
 #include <optional>
 
 using namespace llvm;
@@ -5360,6 +5361,144 @@ TEST_F(OpenMPIRBuilderTest, CreateReductions) {
   EXPECT_TRUE(findGEPZeroOne(ReductionFn->getArg(1), FirstRHS, SecondRHS));
 }
 
+static void createScan(llvm::Value *scanVar, llvm::Type *scanType,
+                       OpenMPIRBuilder &OMPBuilder, IRBuilder<> &Builder,
+                       OpenMPIRBuilder::LocationDescription Loc,
+                       OpenMPIRBuilder::InsertPointTy &allocaIP,
+                       ScanInfo *&ScanRedInfo) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  ASSERT_EXPECTED_INIT(InsertPointTy, retIp,
+                       OMPBuilder.createScan(Loc, allocaIP, {scanVar},
+                                             {scanType}, true, ScanRedInfo));
+  Builder.restoreIP(retIp);
+}
+/*
+ Following is the pseudocode of the code generated by the test case
+ <declare pointer to buffer> ptr
+  size num_iters = 100
+  // temp buffer allocation
+  omp masked {
+    buff = malloc(num_iters*scanvarstype)
+    *ptr = buff
+  }
+ barrier;
+  // input phase loop
+  for (i: 0..<num_iters>) {
+    <input phase>;
+    buffer = *ptr;
+    buffer[i] = red;
+  }
+  // scan reduction
+  omp masked
+  {
+    for (int k = 0; k != ceil(log2(num_iters)); ++k) {
+      i=pow(2,k)
+      for (size cnt = last_iter; cnt >= i; --cnt) {
+        buffer = *ptr;
+        buffer[cnt] op= buffer[cnt-i];
+      }
+    }
+  }
+ barrier;
+ // scan phase loop
+  for (0..<num_iters>) {
+    buffer = *ptr;
+    red = buffer[i] ;
+    <scan phase>;
+  }
+  // temp buffer deletion
+  omp masked {
+    free(*ptr)
+  }
+  barrier;
+*/
+TEST_F(OpenMPIRBuilderTest, ScanReduction) {
+  using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
+  OpenMPIRBuilder OMPBuilder(*M);
+  OMPBuilder.initialize();
+  IRBuilder<> Builder(BB);
+  OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
+  Value *TripCount = F->getArg(0);
+  Type *LCTy = TripCount->getType();
+  Value *StartVal = ConstantInt::get(LCTy, 1);
+  Value *StopVal = ConstantInt::get(LCTy, 100);
+  Value *Step = ConstantInt::get(LCTy, 1);
+  auto AllocaIP = Builder.saveIP();
+
+  llvm::Value *ScanVar = Builder.CreateAlloca(Builder.getFloatTy());
+  llvm::Value *OrigVar = Builder.CreateAlloca(Builder.getFloatTy());
+  unsigned NumBodiesGenerated = 0;
+  ScanInfo *ScanRedInfo;
+  ASSERT_EXPECTED_INIT(ScanInfo *, ScanInformation,
+                       OMPBuilder.scanInfoInitialize());
+  ScanRedInfo = ScanInformation;
+  auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {
+    NumBodiesGenerated += 1;
+    Builder.restoreIP(CodeGenIP);
+    createScan(ScanVar, Builder.getFloatTy(), OMPBuilder, Builder, Loc,
+               AllocaIP, ScanRedInfo);
+    return Error::success();
+  };
+  llvm::SmallVector<CanonicalLoopInfo *> loops;
+  ASSERT_EXPECTED_INIT(llvm::SmallVector<CanonicalLoopInfo *>, loopvec,
+                       OMPBuilder.createCanonicalScanLoops(
+                           Loc, LoopBodyGenCB, StartVal, StopVal, Step, false,
+                           false, Builder.saveIP(), "scan", ScanRedInfo));
+  loops = loopvec;
+  CanonicalLoopInfo *InputLoop = loops.front();
+  CanonicalLoopInfo *ScanLoop = loops.back();
+  Builder.restoreIP(ScanLoop->getAfterIP());
+  InputLoop->assertOK();
+  ScanLoop->assertOK();
+
+  EXPECT_EQ(ScanLoop->getAfter(), Builder.GetInsertBlock());
+  EXPECT_EQ(NumBodiesGenerated, 2U);
+  SmallVector<OpenMPIRBuilder::ReductionInfo> ReductionInfos = {
+      {Builder.getFloatTy(), OrigVar, ScanVar,
+       /*EvaluationKind=*/OpenMPIRBuilder::EvalKind::Scalar, sumReduction,
+       /*ReductionGenClang=*/nullptr, sumAtomicReduction}};
+  OpenMPIRBuilder::LocationDescription RedLoc({InputLoop->getAfterIP(), DL});
+  llvm::BasicBlock *Cont = splitBB(Builder, false, "omp.scan.loop.cont");
+  ASSERT_EXPECTED_INIT(
+      InsertPointTy, retIp,
+      OMPBuilder.emitScanReduction(RedLoc, ReductionInfos, ScanRedInfo));
+  Builder.restoreIP(retIp);
+  Builder.CreateBr(Cont);
+  Builder.SetInsertPoint(Cont);
+  unsigned NumMallocs = 0;
+  unsigned NumFrees = 0;
+  unsigned NumMasked = 0;
+  unsigned NumEndMasked = 0;
+  unsigned NumLog = 0;
+  unsigned NumCeil = 0;
+  for (Instruction &I : instructions(F)) {
+    if (!isa<CallInst>(I))
+      continue;
+    CallInst *Call = dyn_cast<CallInst>(&I);
+    StringRef Name = Call->getCalledFunction()->getName();
+    if (Name.equals_insensitive("malloc")) {
+      NumMallocs += 1;
+    } else if (Name.equals_insensitive("free")) {
+      NumFrees += 1;
+    } else if (Name.equals_insensitive("__kmpc_masked")) {
+      NumMasked += 1;
+    } else if (Name.equals_insensitive("__kmpc_end_masked")) {
+      NumEndMasked += 1;
+    } else if (Name.equals_insensitive("llvm.log2.f64")) {
+      NumLog += 1;
+    } else if (Name.equals_insensitive("llvm.ceil.f64")) {
+      NumCeil += 1;
+    }
+  }
+  EXPECT_EQ(NumBodiesGenerated, 2U);
+  EXPECT_EQ(NumMasked, 3U);
+  EXPECT_EQ(NumEndMasked, 3U);
+  EXPECT_EQ(NumMallocs, 1U);
+  EXPECT_EQ(NumFrees, 1U);
+  EXPECT_EQ(NumLog, 1U);
+  EXPECT_EQ(NumCeil, 1U);
+}
+
 TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
   OpenMPIRBuilder OMPBuilder(*M);
diff --git a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
index aac0f96845f2f..7780bbaf66f4f 100644
--- a/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
+++ b/llvm/unittests/IR/BasicBlockDbgInfoTest.cpp
@@ -189,11 +189,7 @@ TEST(BasicBlockDbgInfoTest, DropSourceAtomOnSplit) {
     ASSERT_TRUE(After);
     const DebugLoc &OrigTerminatorDL = After->getTerminator()->getDebugLoc();
     ASSERT_TRUE(OrigTerminatorDL);
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
     EXPECT_EQ(OrigTerminatorDL->getAtomGroup(), 1u);
-#else
-    EXPECT_EQ(OrigTerminatorDL->getAtomGroup(), 0u);
-#endif
   }
 
   // Test splitBasicBlock.
@@ -204,11 +200,7 @@ TEST(BasicBlockDbgInfoTest, DropSourceAtomOnSplit) {
 
     const DebugLoc &OrigTerminatorDL = After->getTerminator()->getDebugLoc();
     ASSERT_TRUE(OrigTerminatorDL);
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
     EXPECT_EQ(OrigTerminatorDL->getAtomGroup(), 1u);
-#else
-    EXPECT_EQ(OrigTerminatorDL->getAtomGroup(), 0u);
-#endif
 
     BasicBlock *Before = After->getSinglePredecessor();
     ASSERT_TRUE(Before);
diff --git a/llvm/unittests/IR/CMakeLists.txt b/llvm/unittests/IR/CMakeLists.txt
index b66eae93f9339..8b7bd3997ea27 100644
--- a/llvm/unittests/IR/CMakeLists.txt
+++ b/llvm/unittests/IR/CMakeLists.txt
@@ -43,6 +43,7 @@ add_llvm_unittest(IRTests
   PatternMatch.cpp
   ShuffleVectorInstTest.cpp
   StructuralHashTest.cpp
+  RuntimeLibcallsTest.cpp
   TimePassesTest.cpp
   TypesTest.cpp
   UseTest.cpp
diff --git a/llvm/unittests/IR/ConstantRangeTest.cpp b/llvm/unittests/IR/ConstantRangeTest.cpp
index bcb5d498c8cb9..53d581c8db7c9 100644
--- a/llvm/unittests/IR/ConstantRangeTest.cpp
+++ b/llvm/unittests/IR/ConstantRangeTest.cpp
@@ -451,6 +451,65 @@ TEST_F(ConstantRangeTest, Trunc) {
   EXPECT_EQ(SevenOne.truncate(2), ConstantRange(APInt(2, 3), APInt(2, 1)));
 }
 
+TEST_F(ConstantRangeTest, TruncNuw) {
+  auto Range = [](unsigned NumBits, unsigned Lower, unsigned Upper) {
+    return ConstantRange(APInt(NumBits, Lower), APInt(NumBits, Upper));
+  };
+  // trunc([0, 4), 3->2) = full
+  EXPECT_TRUE(
+      Range(3, 0, 4).truncate(2, TruncInst::NoUnsignedWrap).isFullSet());
+  // trunc([0, 3), 3->2) = [0, 3)
+  EXPECT_EQ(Range(3, 0, 3).truncate(2, TruncInst::NoUnsignedWrap),
+            Range(2, 0, 3));
+  // trunc([1, 3), 3->2) = [1, 3)
+  EXPECT_EQ(Range(3, 1, 3).truncate(2, TruncInst::NoUnsignedWrap),
+            Range(2, 1, 3));
+  // trunc([1, 5), 3->2) = [1, 0)
+  EXPECT_EQ(Range(3, 1, 5).truncate(2, TruncInst::NoUnsignedWrap),
+            Range(2, 1, 0));
+  // trunc([4, 7), 3->2) = empty
+  EXPECT_TRUE(
+      Range(3, 4, 7).truncate(2, TruncInst::NoUnsignedWrap).isEmptySet());
+  // trunc([4, 0), 3->2) = empty
+  EXPECT_TRUE(
+      Range(3, 4, 0).truncate(2, TruncInst::NoUnsignedWrap).isEmptySet());
+  // trunc([4, 1), 3->2) = [0, 1)
+  EXPECT_EQ(Range(3, 4, 1).truncate(2, TruncInst::NoUnsignedWrap),
+            Range(2, 0, 1));
+  // trunc([3, 1), 3->2) = [3, 1)
+  EXPECT_EQ(Range(3, 3, 1).truncate(2, TruncInst::NoUnsignedWrap),
+            Range(2, 3, 1));
+  // trunc([3, 0), 3->2) = [3, 0)
+  EXPECT_EQ(Range(3, 3, 0).truncate(2, TruncInst::NoUnsignedWrap),
+            Range(2, 3, 0));
+  // trunc([1, 0), 2->1) = [1, 0)
+  EXPECT_EQ(Range(2, 1, 0).truncate(1, TruncInst::NoUnsignedWrap),
+            Range(1, 1, 0));
+  // trunc([2, 1), 2->1) = [0, 1)
+  EXPECT_EQ(Range(2, 2, 1).truncate(1, TruncInst::NoUnsignedWrap),
+            Range(1, 0, 1));
+}
+
+TEST_F(ConstantRangeTest, TruncNuwExhaustive) {
+  EnumerateConstantRanges(4, [&](const ConstantRange &CR) {
+    unsigned NumBits = 3;
+    ConstantRange Trunc = CR.truncate(NumBits, TruncInst::NoUnsignedWrap);
+    SmallBitVector Elems(1 << NumBits);
+    ForeachNumInConstantRange(CR, [&](const APInt &N) {
+      if (N.isIntN(NumBits))
+        Elems.set(N.getZExtValue());
+    });
+    TestRange(Trunc, Elems, PreferSmallest, {CR});
+  });
+  EnumerateConstantRanges(3, [&](const ConstantRange &CR) {
+    ConstantRange Trunc = CR.truncate(1, TruncInst::NoUnsignedWrap);
+    EXPECT_EQ(CR.contains(APInt::getZero(3)),
+              Trunc.contains(APInt::getZero(1)));
+    EXPECT_EQ(CR.contains(APInt::getOneBitSet(3, 0)),
+              Trunc.contains(APInt::getAllOnes(1)));
+  });
+}
+
 TEST_F(ConstantRangeTest, ZExt) {
   ConstantRange ZFull = Full.zeroExtend(20);
   ConstantRange ZEmpty = Empty.zeroExtend(20);
diff --git a/llvm/unittests/IR/DebugInfoTest.cpp b/llvm/unittests/IR/DebugInfoTest.cpp
index 0065615cbfe13..03333d5872d2f 100644
--- a/llvm/unittests/IR/DebugInfoTest.cpp
+++ b/llvm/unittests/IR/DebugInfoTest.cpp
@@ -553,17 +553,15 @@ TEST(DIBuilder, FixedPointType) {
   EXPECT_TRUE(Ty->getTag() == dwarf::DW_TAG_base_type);
 }
 
-TEST(DbgAssignIntrinsicTest, replaceVariableLocationOp) {
+TEST(DbgAssignRecordTest, replaceVariableLocationOp) {
   LLVMContext C;
   std::unique_ptr<Module> M = parseIR(C, R"(
     define dso_local void @fun(i32 %v1, ptr %p1, ptr %p2) !dbg !7 {
     entry:
-      call void @llvm.dbg.assign(metadata i32 %v1, metadata !14, metadata !DIExpression(), metadata !17, metadata ptr %p1, metadata !DIExpression()), !dbg !16
+        #dbg_assign(i32 %v1, !14, !DIExpression(), !17, ptr %p1, !DIExpression(), !16)
       ret void
     }
 
-    declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata)
-
     !llvm.dbg.cu = !{!0}
     !llvm.module.flags = !{!3}
 
@@ -629,28 +627,28 @@ TEST(AssignmentTrackingTest, Utils) {
   std::unique_ptr<Module> M = parseIR(C, R"(
     define dso_local void @fun1() !dbg !7 {
     entry:
-      call void @llvm.dbg.assign(metadata i32 undef, metadata !10, metadata !DIExpression(), metadata !12, metadata i32 undef, metadata !DIExpression()), !dbg !13
+        #dbg_assign(i32 undef, !10, !DIExpression(), !12, i32 undef, !DIExpression(), !13)
       %local = alloca i32, align 4, !DIAssignID !12
-      call void @llvm.dbg.assign(metadata i32 undef, metadata !16, metadata !DIExpression(), metadata !12, metadata i32 undef, metadata !DIExpression()), !dbg !15
+        #dbg_assign(i32 undef, !16, !DIExpression(), !12, i32 undef, !DIExpression(), !15)
+        #dbg_assign(i32 undef, !16, !DIExpression(), !25, i32 undef, !DIExpression(), !15)
+        #dbg_assign(i32 undef, !16, !DIExpression(), !25, i32 undef, !DIExpression(), !15)
       ret void, !dbg !15
     }
 
     define dso_local void @fun2() !dbg !17 {
     entry:
       %local = alloca i32, align 4, !DIAssignID !20
-      call void @llvm.dbg.assign(metadata i32 undef, metadata !18, metadata !DIExpression(), metadata !20, metadata i32 undef, metadata !DIExpression()), !dbg !19
+        #dbg_assign(i32 undef, !18, !DIExpression(), !20, i32 undef, !DIExpression(), !19)
       ret void, !dbg !19
     }
 
     define dso_local void @fun3() !dbg !21 {
     entry:
       %local = alloca i32, align 4, !DIAssignID !24
-      call void @llvm.dbg.assign(metadata i32 undef, metadata !22, metadata !DIExpression(), metadata !24, metadata i32* undef, metadata !DIExpression()), !dbg !23
+        #dbg_assign(i32 undef, !22, !DIExpression(), !24, i32* undef, !DIExpression(), !23)
       ret void
     }
 
-    declare void @llvm.dbg.assign(metadata, metadata, metadata, metadata, metadata, metadata)
-
     !llvm.dbg.cu = !{!0}
     !llvm.module.flags = !{!3, !4, !5}
     !llvm.ident = !{!6}
@@ -680,6 +678,7 @@ TEST(AssignmentTrackingTest, Utils) {
     !22 = !DILocalVariable(name: "local4", scope: !21, file: !1, line: 2, type: !11)
     !23 = !DILocation(line: 4, column: 1, scope: !21)
     !24 = distinct !DIAssignID()
+    !25 = distinct !DIAssignID()
     )");
 
   // Check the test IR isn't malformed.
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index 4f2ede3321080..773c32e7d9b43 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -466,15 +466,11 @@ TEST_F(IRBuilderTest, Lifetime) {
 
   CallInst *Start1 = Builder.CreateLifetimeStart(Var1);
   CallInst *Start2 = Builder.CreateLifetimeStart(Var2);
-  CallInst *Start3 = Builder.CreateLifetimeStart(Var3, Builder.getInt64(100));
+  CallInst *Start3 = Builder.CreateLifetimeStart(Var3);
 
-  EXPECT_EQ(Start1->getArgOperand(0), Builder.getInt64(-1));
-  EXPECT_EQ(Start2->getArgOperand(0), Builder.getInt64(-1));
-  EXPECT_EQ(Start3->getArgOperand(0), Builder.getInt64(100));
-
-  EXPECT_EQ(Start1->getArgOperand(1), Var1);
-  EXPECT_EQ(Start2->getArgOperand(1)->stripPointerCasts(), Var2);
-  EXPECT_EQ(Start3->getArgOperand(1), Var3);
+  EXPECT_EQ(Start1->getArgOperand(0), Var1);
+  EXPECT_EQ(Start2->getArgOperand(0), Var2);
+  EXPECT_EQ(Start3->getArgOperand(0), Var3);
 
   Value *End1 = Builder.CreateLifetimeEnd(Var1);
   Builder.CreateLifetimeEnd(Var2);
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index fd8838e06f34c..21d45960dce7c 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -1935,5 +1935,22 @@ TEST(InstructionsTest, CmpPredicate) {
   EXPECT_EQ(P2, R2);
 }
 
+TEST(InstructionsTest, StripAndAccumulateConstantOffset) {
+  LLVMContext C;
+  DataLayout DL;
+  std::unique_ptr<Module> M = parseIR(C, R"(
+  define void @foo(ptr %ptr, i64 %offset) {
+    %gep = getelementptr inbounds [1 x i8], ptr %ptr, i64 4, i64 %offset
+    ret void
+  })");
+  ASSERT_TRUE(M);
+  Value *GEP = &M->getFunction("foo")->getEntryBlock().front();
+  APInt Offset(DL.getIndexTypeSizeInBits(GEP->getType()), 0);
+  Value *Stripped = GEP->stripAndAccumulateConstantOffsets(
+      DL, Offset, /*AllowNonInBounds=*/true);
+  EXPECT_EQ(Stripped, GEP);
+  EXPECT_TRUE(Offset.isZero());
+}
+
 } // end anonymous namespace
 } // end namespace llvm
diff --git a/llvm/unittests/IR/MetadataTest.cpp b/llvm/unittests/IR/MetadataTest.cpp
index ba8367f4a396e..7425703606381 100644
--- a/llvm/unittests/IR/MetadataTest.cpp
+++ b/llvm/unittests/IR/MetadataTest.cpp
@@ -1470,17 +1470,10 @@ TEST_F(DILocationTest, Merge) {
     PickMergedSourceLocations = false;
   }
 
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
 #define EXPECT_ATOM(Loc, Group, Rank)                                          \
   EXPECT_EQ(Group, M->getAtomGroup());                                         \
   EXPECT_EQ(Rank, M->getAtomRank());
-#else
-#define EXPECT_ATOM(Loc, Group, Rank)                                          \
-  EXPECT_EQ(0u, M->getAtomGroup());                                            \
-  EXPECT_EQ(0u, M->getAtomRank());                                             \
-  (void)Group;                                                                 \
-  (void)Rank;
-#endif
+
   // Identical, including source atom numbers.
   {
     auto *A = DILocation::get(Context, 2, 7, N, nullptr, false, /*AtomGroup*/ 1,
@@ -1753,15 +1746,8 @@ TEST_F(DILocationTest, KeyInstructions) {
   EXPECT_EQ(Context.pImpl->NextAtomGroup, 1u);
   DILocation *A1 =
       DILocation::get(Context, 1, 0, getSubprogram(), nullptr, false, 1, 2);
-  // The group is only applied to the DILocation if we've built LLVM with
-  // EXPERIMENTAL_KEY_INSTRUCTIONS.
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
   EXPECT_EQ(A1->getAtomGroup(), 1u);
   EXPECT_EQ(A1->getAtomRank(), 2u);
-#else
-  EXPECT_EQ(A1->getAtomGroup(), 0u);
-  EXPECT_EQ(A1->getAtomRank(), 0u);
-#endif
 
   // Group number 1 has been "used" so next available is 2.
   EXPECT_EQ(Context.pImpl->NextAtomGroup, 2u);
diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp
index bb7cc0802b1df..972dac82d3331 100644
--- a/llvm/unittests/IR/PatternMatch.cpp
+++ b/llvm/unittests/IR/PatternMatch.cpp
@@ -2621,4 +2621,40 @@ TEST_F(PatternMatchTest, PtrAdd) {
   EXPECT_FALSE(match(OtherGEP, m_PtrAdd(m_Value(A), m_Value(B))));
 }
 
+TEST_F(PatternMatchTest, ShiftOrSelf) {
+  Type *I64Ty = Type::getInt64Ty(Ctx);
+  Constant *LHS = ConstantInt::get(I64Ty, 7);
+  Constant *ShAmt = ConstantInt::get(I64Ty, 16);
+  Value *Shl = IRB.CreateShl(LHS, ShAmt);
+  Value *LShr = IRB.CreateLShr(LHS, ShAmt);
+  Value *AShr = IRB.CreateAShr(LHS, ShAmt);
+  Value *Add = IRB.CreateAdd(LHS, LHS);
+
+  uint64_t ShAmtC;
+  Value *A;
+  EXPECT_TRUE(match(Shl, m_ShlOrSelf(m_Value(A), ShAmtC)));
+  EXPECT_EQ(A, LHS);
+  EXPECT_EQ(ShAmtC, 16U);
+
+  EXPECT_TRUE(match(Add, m_ShlOrSelf(m_Value(A), ShAmtC)));
+  EXPECT_EQ(A, Add);
+  EXPECT_EQ(ShAmtC, 0U);
+
+  EXPECT_TRUE(match(LShr, m_LShrOrSelf(m_Value(A), ShAmtC)));
+  EXPECT_EQ(A, LHS);
+  EXPECT_EQ(ShAmtC, 16U);
+
+  EXPECT_TRUE(match(Add, m_LShrOrSelf(m_Value(A), ShAmtC)));
+  EXPECT_EQ(A, Add);
+  EXPECT_EQ(ShAmtC, 0U);
+
+  EXPECT_TRUE(match(AShr, m_AShrOrSelf(m_Value(A), ShAmtC)));
+  EXPECT_EQ(A, LHS);
+  EXPECT_EQ(ShAmtC, 16U);
+
+  EXPECT_TRUE(match(Add, m_AShrOrSelf(m_Value(A), ShAmtC)));
+  EXPECT_EQ(A, Add);
+  EXPECT_EQ(ShAmtC, 0U);
+}
+
 } // anonymous namespace.
diff --git a/llvm/unittests/IR/RuntimeLibcallsTest.cpp b/llvm/unittests/IR/RuntimeLibcallsTest.cpp
new file mode 100644
index 0000000000000..012316801859c
--- /dev/null
+++ b/llvm/unittests/IR/RuntimeLibcallsTest.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/RuntimeLibcalls.h"
+#include "llvm/ADT/STLExtras.h"
+#include "gtest/gtest.h"
+using namespace llvm;
+
+namespace {
+
+TEST(RuntimeLibcallsTest, LibcallImplByName) {
+  EXPECT_TRUE(RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName("").empty());
+  EXPECT_TRUE(
+      RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName("unknown").empty());
+  EXPECT_TRUE(
+      RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName("Unsupported").empty());
+  EXPECT_TRUE(
+      RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName("unsupported").empty());
+
+  for (RTLIB::LibcallImpl LC : RTLIB::libcall_impls()) {
+    StringRef Name = RTLIB::RuntimeLibcallsInfo::getLibcallImplName(LC);
+    EXPECT_TRUE(is_contained(
+        RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName(Name), LC));
+  }
+
+  // Test first libcall name
+  EXPECT_EQ(
+      RTLIB::arm64ec__Unwind_Resume,
+      *RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName("#_Unwind_Resume")
+           .begin());
+  // Test longest libcall names
+  EXPECT_EQ(RTLIB::__hexagon_memcpy_likely_aligned_min32bytes_mult8bytes,
+            *RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName(
+                 "__hexagon_memcpy_likely_aligned_min32bytes_mult8bytes")
+                 .begin());
+
+  {
+    auto SquirtleSquad =
+        RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName("sqrtl");
+    ASSERT_EQ(size(SquirtleSquad), 3);
+    auto I = SquirtleSquad.begin();
+    EXPECT_EQ(*I++, RTLIB::sqrt_f128);
+    EXPECT_EQ(*I++, RTLIB::sqrt_f80);
+    EXPECT_EQ(*I++, RTLIB::sqrt_ppcf128);
+  }
+
+  // Last libcall
+  {
+    auto Truncs = RTLIB::RuntimeLibcallsInfo::lookupLibcallImplName("truncl");
+    ASSERT_EQ(size(Truncs), 3);
+    auto I = Truncs.begin();
+    EXPECT_EQ(*I++, RTLIB::trunc_f128);
+    EXPECT_EQ(*I++, RTLIB::trunc_f80);
+    EXPECT_EQ(*I++, RTLIB::trunc_ppcf128);
+  }
+}
+
+} // namespace
diff --git a/llvm/unittests/Support/AlignOfTest.cpp b/llvm/unittests/Support/AlignOfTest.cpp
index f84895c18602d..979f2cf18cc15 100644
--- a/llvm/unittests/Support/AlignOfTest.cpp
+++ b/llvm/unittests/Support/AlignOfTest.cpp
@@ -30,7 +30,7 @@ namespace {
 #ifdef __clang__
 #pragma clang diagnostic ignored "-Wunknown-pragmas"
 #pragma clang diagnostic ignored "-Winaccessible-base"
-#elif ((__GNUC__ * 100) + __GNUC_MINOR__) >= 402
+#elif defined(__GNUC__)
 // Pragma based warning suppression was introduced in GGC 4.2.  Additionally
 // this warning is "enabled by default".  The warning still appears if -Wall is
 // suppressed.  Apparently GCC suppresses it when -w is specifed, which is odd.
diff --git a/llvm/unittests/Support/DebugLogTest.cpp b/llvm/unittests/Support/DebugLogTest.cpp
index c24d1a5693169..e087705b72586 100644
--- a/llvm/unittests/Support/DebugLogTest.cpp
+++ b/llvm/unittests/Support/DebugLogTest.cpp
@@ -6,11 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-// This macro is defined in the LLVM build system, but we undefine it here
-// so that we test at least once in-tree the case where __SHORT_FILE__ is not
-// defined.
-#undef __SHORT_FILE__
-
 #include "llvm/Support/DebugLog.h"
 #include "llvm/ADT/Sequence.h"
 #include "llvm/Support/raw_ostream.h"
@@ -120,8 +115,18 @@ TEST(DebugLogTest, StreamPrefix) {
     ldbg_osA << "5";
     EXPECT_EQ(os.str(), expected);
   }
-  // After destructors, there was a pending newline for stream B.
-  EXPECT_EQ(os.str(), expected + "PrefixB ");
+  EXPECT_EQ(os.str(), expected);
+}
+
+TEST(DebugLogTest, DestructorPrefix) {
+  llvm::DebugFlag = true;
+  std::string str;
+  raw_string_ostream os(str);
+  {
+    llvm::impl::raw_ldbg_ostream ldbg_osB("PrefixB ", os);
+  }
+  // After destructors, nothing should have been printed.
+  EXPECT_EQ(os.str(), "");
 }
 #else
 TEST(DebugLogTest, Basic) {
diff --git a/llvm/unittests/Support/DebugTest.cpp b/llvm/unittests/Support/DebugTest.cpp
index e8b754892a4c2..ed8bfc63f0648 100644
--- a/llvm/unittests/Support/DebugTest.cpp
+++ b/llvm/unittests/Support/DebugTest.cpp
@@ -50,4 +50,46 @@ TEST(DebugTest, CommaInDebugBlock) {
   });
   EXPECT_EQ("ZYX", os1.str());
 }
+
+TEST(DebugTest, DebugWithType) {
+  llvm::DebugFlag = true;
+
+  // Check if the DEBUG_WITH_TYPE macro is enabled for the given type.
+  auto CheckDebugWithType = [](const char *Type) {
+    bool Visited = false;
+    DEBUG_WITH_TYPE(Type, { Visited = true; });
+    return Visited;
+  };
+
+  {
+    static const char *DT[] = {"A", "B"};
+    setCurrentDebugTypes(DT, sizeof(DT) / sizeof(DT[0]));
+    EXPECT_TRUE(CheckDebugWithType("A"));
+    EXPECT_TRUE(CheckDebugWithType("B"));
+    EXPECT_FALSE(CheckDebugWithType("C"));
+  }
+  {
+    static const char *DT[] = {"A:"};
+    setCurrentDebugTypes(DT, sizeof(DT) / sizeof(DT[0]));
+    EXPECT_FALSE(CheckDebugWithType("A"));
+    EXPECT_TRUE(CheckDebugWithType("B"));
+    EXPECT_TRUE(CheckDebugWithType("C"));
+  }
+  {
+    static const char *DT[] = {"A:", "B"};
+    setCurrentDebugTypes(DT, sizeof(DT) / sizeof(DT[0]));
+    EXPECT_FALSE(CheckDebugWithType("A"));
+    EXPECT_TRUE(CheckDebugWithType("B"));
+    EXPECT_FALSE(CheckDebugWithType("C"));
+  }
+  {
+    static const char *DT[] = {"A:3", "B:", "C"};
+    setCurrentDebugTypes(DT, sizeof(DT) / sizeof(DT[0]));
+    EXPECT_TRUE(CheckDebugWithType("A"));
+    EXPECT_FALSE(isCurrentDebugType("A", 4));
+    EXPECT_FALSE(CheckDebugWithType("B"));
+    EXPECT_TRUE(isCurrentDebugType("C", 10));
+    EXPECT_FALSE(CheckDebugWithType("D"));
+  }
+}
 #endif
diff --git a/llvm/unittests/Support/DynamicLibrary/CMakeLists.txt b/llvm/unittests/Support/DynamicLibrary/CMakeLists.txt
index 2366209c25aee..98b7f2d19ee46 100644
--- a/llvm/unittests/Support/DynamicLibrary/CMakeLists.txt
+++ b/llvm/unittests/Support/DynamicLibrary/CMakeLists.txt
@@ -16,7 +16,7 @@ set_output_directory(DynamicLibraryLib
   )
 
 # FIXME: Find out why AIX fails with new DynamicLibrary symbols behavior.
-if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   add_llvm_unittest(DynamicLibraryTests
     DynamicLibraryTest.cpp
     )
@@ -28,7 +28,7 @@ else()
     )
 endif()
 target_link_libraries(DynamicLibraryTests PRIVATE DynamicLibraryLib)
-if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   export_executable_symbols(DynamicLibraryTests)
 endif()
 
@@ -64,7 +64,7 @@ endfunction(dynlib_add_module)
 
 # Revert -Wl,-z,nodelete on this test since it relies on the file
 # being unloaded.
-if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "Linux")
   string(REPLACE "-Wl,-z,nodelete" "" CMAKE_MODULE_LINKER_FLAGS
     ${CMAKE_MODULE_LINKER_FLAGS})
 endif()
diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp
index 355aa6b9ade06..888729b9dd249 100644
--- a/llvm/unittests/Support/Path.cpp
+++ b/llvm/unittests/Support/Path.cpp
@@ -1471,6 +1471,43 @@ TEST_F(FileSystemTest, FileMapping) {
   ASSERT_NO_ERROR(fs::remove(TempPath));
 }
 
+TEST_F(FileSystemTest, FileMappingSync) {
+  // Create a temp file.
+  SmallString<0> TempPath(TestDirectory);
+  sys::path::append(TempPath, "test-%%%%");
+  auto TempFileOrError = fs::TempFile::create(TempPath);
+  ASSERT_TRUE((bool)TempFileOrError);
+  fs::TempFile File = std::move(*TempFileOrError);
+  StringRef Content("hello there");
+  std::string FileName = File.TmpName;
+  ASSERT_NO_ERROR(
+      fs::resize_file_before_mapping_readwrite(File.FD, Content.size()));
+  {
+    // Map in the file and write some content.
+    std::error_code EC;
+    fs::mapped_file_region MFR(fs::convertFDToNativeFile(File.FD),
+                               fs::mapped_file_region::readwrite,
+                               Content.size(), 0, EC);
+
+    // Keep the file so it can be read.
+    ASSERT_FALSE((bool)File.keep());
+
+    // Write content through mapped memory.
+    ASSERT_NO_ERROR(EC);
+    std::copy(Content.begin(), Content.end(), MFR.data());
+
+    // Synchronize to file system.
+    ASSERT_FALSE((bool)MFR.sync());
+
+    // Check the file content using file IO APIs.
+    auto Buffer = MemoryBuffer::getFile(FileName);
+    ASSERT_TRUE((bool)Buffer);
+    ASSERT_EQ(Content, Buffer->get()->getBuffer());
+  }
+  // Manually remove the test file.
+  ASSERT_FALSE((bool)fs::remove(FileName));
+}
+
 TEST(Support, NormalizePath) {
   //                           Input,        Expected Win, Expected Posix
   using TestTuple = std::tuple<const char *, const char *, const char *>;
diff --git a/llvm/unittests/Support/ProgramTest.cpp b/llvm/unittests/Support/ProgramTest.cpp
index 693b53b0a9781..d30bf458f233c 100644
--- a/llvm/unittests/Support/ProgramTest.cpp
+++ b/llvm/unittests/Support/ProgramTest.cpp
@@ -10,6 +10,7 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ConvertUTF.h"
+#include "llvm/Support/ExponentialBackoff.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Signals.h"
@@ -573,6 +574,88 @@ TEST_F(ProgramEnvTest, TestLockFile) {
   sys::fs::remove(LockedFile);
 }
 
+TEST_F(ProgramEnvTest, TestLockFileExclusive) {
+  using namespace llvm::sys;
+  using namespace std::chrono_literals;
+
+  if (const char *LockedFile = getenv("LLVM_PROGRAM_TEST_LOCKED_FILE")) {
+    // Child process.
+    int FD2;
+    ASSERT_NO_ERROR(fs::openFileForReadWrite(LockedFile, FD2,
+                                             fs::CD_OpenExisting, fs::OF_None));
+
+    // File should currently be non-exclusive locked by the main process, thus
+    // trying to acquire exclusive lock will fail and trying to acquire
+    // non-exclusive will succeed.
+    EXPECT_TRUE(
+        fs::tryLockFile(FD2, std::chrono::seconds(0), fs::LockKind::Exclusive));
+
+    EXPECT_FALSE(
+        fs::tryLockFile(FD2, std::chrono::seconds(0), fs::LockKind::Shared));
+
+    close(FD2);
+    // Write a file to indicate just finished.
+    std::string FinishFile = std::string(LockedFile) + "-finished";
+    int FD3;
+    ASSERT_NO_ERROR(fs::openFileForReadWrite(FinishFile, FD3, fs::CD_CreateNew,
+                                             fs::OF_None));
+    close(FD3);
+    exit(0);
+  }
+
+  // Create file that will be locked.
+  SmallString<64> LockedFile;
+  int FD1;
+  ASSERT_NO_ERROR(
+      fs::createUniqueDirectory("TestLockFileExclusive", LockedFile));
+  sys::path::append(LockedFile, "file");
+  ASSERT_NO_ERROR(
+      fs::openFileForReadWrite(LockedFile, FD1, fs::CD_CreateNew, fs::OF_None));
+
+  std::string Executable =
+      sys::fs::getMainExecutable(TestMainArgv0, &ProgramTestStringArg1);
+  StringRef argv[] = {Executable,
+                      "--gtest_filter=ProgramEnvTest.TestLockFileExclusive"};
+
+  // Add LLVM_PROGRAM_TEST_LOCKED_FILE to the environment of the child.
+  std::string EnvVar = "LLVM_PROGRAM_TEST_LOCKED_FILE=";
+  EnvVar += LockedFile.str();
+  addEnvVar(EnvVar);
+
+  // Lock the file.
+  ASSERT_NO_ERROR(
+      fs::tryLockFile(FD1, std::chrono::seconds(0), fs::LockKind::Exclusive));
+
+  std::string Error;
+  bool ExecutionFailed;
+  ProcessInfo PI2 = ExecuteNoWait(Executable, argv, getEnviron(), {}, 0, &Error,
+                                  &ExecutionFailed);
+  ASSERT_FALSE(ExecutionFailed) << Error;
+  ASSERT_TRUE(Error.empty());
+  ASSERT_NE(PI2.Pid, ProcessInfo::InvalidPid) << "Invalid process id";
+
+  std::string FinishFile = std::string(LockedFile) + "-finished";
+  // Wait till child process writes the file to indicate the job finished.
+  bool Finished = false;
+  ExponentialBackoff Backoff(5s); // timeout 5s.
+  do {
+    if (fs::exists(FinishFile)) {
+      Finished = true;
+      break;
+    }
+  } while (Backoff.waitForNextAttempt());
+
+  ASSERT_TRUE(Finished);
+  ASSERT_NO_ERROR(fs::unlockFile(FD1));
+  ProcessInfo WaitResult = llvm::sys::Wait(PI2, /*SecondsToWait=*/1, &Error);
+  ASSERT_TRUE(Error.empty());
+  ASSERT_EQ(0, WaitResult.ReturnCode);
+  ASSERT_EQ(WaitResult.Pid, PI2.Pid);
+  sys::fs::remove(LockedFile);
+  sys::fs::remove(FinishFile);
+  sys::fs::remove_directories(sys::path::parent_path(LockedFile));
+}
+
 TEST_F(ProgramEnvTest, TestExecuteWithNoStacktraceHandler) {
   using namespace llvm::sys;
 
diff --git a/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp b/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
index 28d69f30bfd1a..f06f03bb35a5d 100644
--- a/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/Target/AArch64/AArch64SelectionDAGTest.cpp
@@ -8,7 +8,6 @@
 #include "AArch64SelectionDAGInfo.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
@@ -63,12 +62,8 @@ class AArch64SelectionDAGTest : public testing::Test {
     if (!DAG)
       report_fatal_error("DAG?");
     OptimizationRemarkEmitter ORE(F);
-    FunctionAnalysisManager FAM;
-    FAM.registerPass([&] { return TM->getTargetIRAnalysis(); });
-
-    TargetTransformInfo TTI = TM->getTargetIRAnalysis().run(*F, FAM);
     DAG->init(*MF, ORE, nullptr, nullptr, nullptr, nullptr, nullptr, MMI,
-              nullptr, TTI.hasBranchDivergence(F));
+              nullptr);
   }
 
   TargetLoweringBase::LegalizeTypeAction getTypeAction(EVT VT) {
diff --git a/llvm/unittests/Target/AArch64/SMEAttributesTest.cpp b/llvm/unittests/Target/AArch64/SMEAttributesTest.cpp
index f13252f3a4c28..bd0e53c69622e 100644
--- a/llvm/unittests/Target/AArch64/SMEAttributesTest.cpp
+++ b/llvm/unittests/Target/AArch64/SMEAttributesTest.cpp
@@ -78,7 +78,7 @@ TEST(SMEAttributes, Constructors) {
                             "ret void\n}");
   CallBase &Call =
       cast<CallBase>((CallModule->getFunction("foo")->begin()->front()));
-  ASSERT_TRUE(SMECallAttrs(Call).callsite().hasUndefZT0());
+  ASSERT_TRUE(SMECallAttrs(Call, nullptr).callsite().hasUndefZT0());
 
   // Invalid combinations.
   EXPECT_DEBUG_DEATH(SA(SA::SM_Enabled | SA::SM_Compatible),
@@ -309,7 +309,7 @@ TEST(SMEAttributes, Transitions) {
 
   // Shared ZA -> Private ZA Interface
   ASSERT_FALSE(CA(ZA_Shared, Private_ZA).requiresDisablingZABeforeCall());
-  ASSERT_TRUE(CA(ZA_Shared, Private_ZA).requiresEnablingZAAfterCall());
+  ASSERT_FALSE(CA(ZA_Shared, Private_ZA).requiresEnablingZAAfterCall());
 
   // Shared ZT0 -> Private ZA Interface
   ASSERT_TRUE(CA(ZT0_Shared, Private_ZA).requiresDisablingZABeforeCall());
@@ -328,7 +328,7 @@ TEST(SMEAttributes, Transitions) {
   // Shared ZA & ZT0 -> Private ZA Interface
   ASSERT_FALSE(CA(ZA_ZT0_Shared, Private_ZA).requiresDisablingZABeforeCall());
   ASSERT_TRUE(CA(ZA_ZT0_Shared, Private_ZA).requiresPreservingZT0());
-  ASSERT_TRUE(CA(ZA_ZT0_Shared, Private_ZA).requiresEnablingZAAfterCall());
+  ASSERT_FALSE(CA(ZA_ZT0_Shared, Private_ZA).requiresEnablingZAAfterCall());
 
   // Shared ZA -> Shared ZA Interface
   ASSERT_FALSE(CA(ZA_Shared, ZT0_Shared).requiresDisablingZABeforeCall());
diff --git a/llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp b/llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp
index ac08501817340..593991c71d706 100644
--- a/llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp
+++ b/llvm/unittests/Target/AMDGPU/AMDGPUUnitTests.cpp
@@ -320,3 +320,23 @@ TEST(AMDGPU, TestReverseComposeSubRegIndices) {
     }
   }
 }
+
+TEST(AMDGPU, TestGetNamedOperandIdx) {
+  std::unique_ptr<const GCNTargetMachine> TM =
+      createAMDGPUTargetMachine("amdgcn-amd-", "gfx900", "");
+  if (!TM)
+    return;
+  const MCInstrInfo *MCII = TM->getMCInstrInfo();
+
+  for (unsigned Opcode = 0, E = MCII->getNumOpcodes(); Opcode != E; ++Opcode) {
+    const MCInstrDesc &Desc = MCII->get(Opcode);
+    for (unsigned Idx = 0; Idx < Desc.getNumOperands(); ++Idx) {
+      AMDGPU::OpName OpName = AMDGPU::getOperandIdxName(Opcode, Idx);
+      if (OpName == AMDGPU::OpName::NUM_OPERAND_NAMES)
+        continue;
+      int16_t RetrievedIdx = AMDGPU::getNamedOperandIdx(Opcode, OpName);
+      EXPECT_EQ(Idx, static_cast<unsigned>(RetrievedIdx))
+          << "Opcode " << Opcode << " (" << MCII->getName(Opcode) << ')';
+    }
+  }
+}
diff --git a/llvm/unittests/Target/AMDGPU/CSETest.cpp b/llvm/unittests/Target/AMDGPU/CSETest.cpp
index 3de5b88599649..ff44ff184234c 100644
--- a/llvm/unittests/Target/AMDGPU/CSETest.cpp
+++ b/llvm/unittests/Target/AMDGPU/CSETest.cpp
@@ -10,6 +10,7 @@
 #include "AMDGPUUnitTests.h"
 #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
 #include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
diff --git a/llvm/unittests/TargetParser/Host.cpp b/llvm/unittests/TargetParser/Host.cpp
index 0a9ac9bb0596d..be8548ebf8551 100644
--- a/llvm/unittests/TargetParser/Host.cpp
+++ b/llvm/unittests/TargetParser/Host.cpp
@@ -59,16 +59,28 @@ Serial          : 0000000000000000
 
   EXPECT_EQ(sys::detail::getHostCPUNameForARM(CortexA9ProcCpuinfo),
             "cortex-a9");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM(
+                0x4100c090, ArrayRef<uint64_t>{0x4100c090, 0x4100c090}),
+            "cortex-a9");
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x41\n"
                                               "CPU part        : 0xc0f"),
             "cortex-a15");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM(0x4100c0f0,
+                                              ArrayRef<uint64_t>{0x4100c0f0}),
+            "cortex-a15");
   // Verify that both CPU implementer and CPU part are checked:
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x40\n"
                                               "CPU part        : 0xc0f"),
             "generic");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM(0x4000c0f0,
+                                              ArrayRef<uint64_t>{0x4000c0f0}),
+            "generic");
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x51\n"
                                               "CPU part        : 0x06f"),
             "krait");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM(0x510006f0,
+                                              ArrayRef<uint64_t>{0x510006f0}),
+            "krait");
 }
 
 TEST(getLinuxHostCPUName, AArch64) {
@@ -126,10 +138,16 @@ TEST(getLinuxHostCPUName, AArch64) {
                                               "CPU part        : 0xd85\n"
                                               "CPU part        : 0xd87"),
             "cortex-x925");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM(
+                0x4100d850, ArrayRef<uint64_t>{0x4100d850, 0x4100d870}),
+            "cortex-x925");
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x41\n"
                                               "CPU part        : 0xd87\n"
                                               "CPU part        : 0xd85"),
             "cortex-x925");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM(
+                0x4100d870, ArrayRef<uint64_t>{0x4100d870, 0x4100d850}),
+            "cortex-x925");
   EXPECT_EQ(sys::detail::getHostCPUNameForARM("CPU implementer : 0x51\n"
                                               "CPU part        : 0xc00"),
             "falkor");
@@ -200,16 +218,25 @@ CPU architecture: 8
                                               "CPU variant     : 0xc\n"
                                               "CPU part        : 0xafe"),
             "exynos-m3");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM(
+                0x53c0afe0, ArrayRef<uint64_t>{0x53c0afe0, 0x5300d050}),
+            "exynos-m3");
   // Verify Exynos M3.
   EXPECT_EQ(sys::detail::getHostCPUNameForARM(ExynosProcCpuInfo +
                                               "CPU variant     : 0x1\n"
                                               "CPU part        : 0x002"),
             "exynos-m3");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM(
+                0x53100020, ArrayRef<uint64_t>{0x53100020, 0x5300d050}),
+            "exynos-m3");
   // Verify Exynos M4.
   EXPECT_EQ(sys::detail::getHostCPUNameForARM(ExynosProcCpuInfo +
                                               "CPU variant     : 0x1\n"
                                               "CPU part        : 0x003"),
             "exynos-m4");
+  EXPECT_EQ(sys::detail::getHostCPUNameForARM(
+                0x53100030, ArrayRef<uint64_t>{0x53100030, 0x5300d050}),
+            "exynos-m4");
 
   const std::string ThunderX2T99ProcCpuInfo = R"(
 processor	: 0
diff --git a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
index 319538eaea135..9cee4ff776da9 100644
--- a/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/TargetParser/RISCVISAInfoTest.cpp
@@ -1099,6 +1099,7 @@ R"(All available -march extensions for RISC-V
     smcdeleg             1.0
     smcntrpmf            1.0
     smcsrind             1.0
+    smctr                1.0
     smdbltrp             1.0
     smepmp               1.0
     smmpm                1.0
@@ -1111,6 +1112,7 @@ R"(All available -march extensions for RISC-V
     sscofpmf             1.0
     sscounterenw         1.0
     sscsrind             1.0
+    ssctr                1.0
     ssdbltrp             1.0
     ssnpm                1.0
     sspm                 1.0
@@ -1163,6 +1165,7 @@ R"(All available -march extensions for RISC-V
     xsfvqmaccqoq         1.0
     xsifivecdiscarddlone 1.0
     xsifivecflushdlone   1.0
+    xsmtvdot             1.0
     xtheadba             1.0
     xtheadbb             1.0
     xtheadbs             1.0
@@ -1178,15 +1181,13 @@ R"(All available -march extensions for RISC-V
     xwchc                2.2
 
 Experimental extensions
-    p                    0.14
+    p                    0.15
     zicfilp              1.0       This is a long dummy description
     zicfiss              1.0
     zalasr               0.1
     zvbc32e              0.7
     zvkgs                0.7
     zvqdotq              0.0
-    smctr                1.0
-    ssctr                1.0
     svukte               0.3
     xqccmp               0.3
     xqcia                0.7
diff --git a/llvm/unittests/TextAPI/TextStubV5Tests.cpp b/llvm/unittests/TextAPI/TextStubV5Tests.cpp
index 24577b3ec6148..f6689f7524a72 100644
--- a/llvm/unittests/TextAPI/TextStubV5Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV5Tests.cpp
@@ -1167,6 +1167,33 @@ TEST(TBDv5, InvalidMinOS) {
   EXPECT_EQ("invalid min_deployment section\n", ErrorMessage);
 }
 
+TEST(TBDv5, RISCV) {
+  static const char TBDv5File[] = R"({ 
+"tapi_tbd_version": 5,
+"main_library": {
+  "target_info": [
+    {
+      "target": "riscv32-ios",
+      "min_deployment": "34.1" 
+    }
+  ],
+  "install_names":[
+    { "name":"/S/L/F/Foo.framework/Foo" }
+  ]
+}})";
+
+  Expected<TBDFile> Result =
+      TextAPIReader::get(MemoryBufferRef(TBDv5File, "Test.tbd"));
+  EXPECT_TRUE(!!Result);
+  Target ExpectedTarget = Target(AK_riscv32, PLATFORM_IOS, VersionTuple(34, 1));
+  TBDFile ReadFile = std::move(Result.get());
+  EXPECT_EQ(FileType::TBD_V5, ReadFile->getFileType());
+  EXPECT_EQ(std::string("/S/L/F/Foo.framework/Foo"),
+            ReadFile->getInstallName());
+  EXPECT_TRUE(ReadFile->targets().begin() != ReadFile->targets().end());
+  EXPECT_EQ(*ReadFile->targets().begin(), ExpectedTarget);
+}
+
 TEST(TBDv5, SimSupport) {
   static const char TBDv5File[] = R"({ 
 "tapi_tbd_version": 5,
diff --git a/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp b/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp
index 9f76e9ff11c3a..49373d367783b 100644
--- a/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp
+++ b/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp
@@ -27,12 +27,11 @@ namespace llvm {
 static void removeSSACopy(Function &F) {
   for (BasicBlock &BB : F) {
     for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
-      if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
-        if (II->getIntrinsicID() != Intrinsic::ssa_copy)
-          continue;
-        Inst.replaceAllUsesWith(II->getOperand(0));
-        Inst.eraseFromParent();
-      }
+      auto *BC = dyn_cast<BitCastInst>(&Inst);
+      if (!BC || BC->getType() != BC->getOperand(0)->getType())
+        continue;
+      Inst.replaceAllUsesWith(BC->getOperand(0));
+      Inst.eraseFromParent();
     }
   }
 }
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index b8b035751637a..fe81986aee7b9 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -1203,13 +1203,9 @@ TEST_F(CloneInstruction, cloneKeyInstructions) {
 
   ASSERT_FALSE(verifyModule(*M, &errs()));
 
-#ifdef EXPERIMENTAL_KEY_INSTRUCTIONS
 #define EXPECT_ATOM(Inst, G)                                                   \
   EXPECT_TRUE(Inst->getDebugLoc());                                            \
   EXPECT_EQ(Inst->getDebugLoc()->getAtomGroup(), uint64_t(G));
-#else
-#define EXPECT_ATOM(Inst, G) (void)Inst;
-#endif
 
   Function *F = M->getFunction("test");
   BasicBlock *BB = &*F->begin();
diff --git a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
index 7f9a59bd428a0..31b4a5ec9e391 100644
--- a/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/SandboxVectorizer/SeedCollectorTest.cpp
@@ -259,10 +259,10 @@ define void @foo(ptr %ptrA, float %val, ptr %ptrB) {
   // Check begin() end() when empty.
   EXPECT_EQ(SC.begin(), SC.end());
 
-  SC.insert(S0);
-  SC.insert(S1);
-  SC.insert(S2);
-  SC.insert(S3);
+  SC.insert(S0, /*AllowDiffTypes=*/false);
+  SC.insert(S1, /*AllowDiffTypes=*/false);
+  SC.insert(S2, /*AllowDiffTypes=*/false);
+  SC.insert(S3, /*AllowDiffTypes=*/false);
   unsigned Cnt = 0;
   SmallVector<sandboxir::SeedBundle *> Bndls;
   for (auto &SeedBndl : SC) {
@@ -480,6 +480,45 @@ define void @foo(ptr noalias %ptr, float %v, <2 x float> %val) {
   ExpectThatElementsAre(SB, {St0, St1, St3});
 }
 
+TEST_F(SeedBundleTest, DiffTypes) {
+  parseIR(C, R"IR(
+define void @foo(ptr noalias %ptr, i8 %v, i16 %v16) {
+bb:
+  %ptr0 = getelementptr i8, ptr %ptr, i32 0
+  %ptr1 = getelementptr i8, ptr %ptr, i32 1
+  %ptr3 = getelementptr i8, ptr %ptr, i32 3
+  store i8 %v, ptr %ptr0
+  store i8 %v, ptr %ptr3
+  store i16 %v16, ptr %ptr1
+  ret void
+}
+)IR");
+  Function &LLVMF = *M->getFunction("foo");
+  DominatorTree DT(LLVMF);
+  TargetLibraryInfoImpl TLII(M->getTargetTriple());
+  TargetLibraryInfo TLI(TLII);
+  DataLayout DL(M->getDataLayout());
+  LoopInfo LI(DT);
+  AssumptionCache AC(LLVMF);
+  ScalarEvolution SE(LLVMF, TLI, AC, DT, LI);
+
+  sandboxir::Context Ctx(C);
+  auto &F = *Ctx.createFunction(&LLVMF);
+  auto BB = F.begin();
+  auto It = std::next(BB->begin(), 3);
+  auto *St0 = &*It++;
+  auto *St3 = &*It++;
+  auto *St1 = &*It++;
+
+  sandboxir::SeedCollector SC(&*BB, SE, /*CollectStores=*/true,
+                              /*CollectLoads=*/false, /*AllowDiffTypes=*/true);
+
+  auto StoreSeedsRange = SC.getStoreSeeds();
+  EXPECT_EQ(range_size(StoreSeedsRange), 1u);
+  auto &SB = *StoreSeedsRange.begin();
+  ExpectThatElementsAre(SB, {St0, St1, St3});
+}
+
 TEST_F(SeedBundleTest, VectorLoads) {
   parseIR(C, R"IR(
 define void @foo(ptr noalias %ptr, <2 x float> %val0) {
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 123f6de823162..a943e7ac12b1b 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -59,7 +59,7 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
   auto Iter = VecBB->begin();
   auto *CanIV = dyn_cast<VPCanonicalIVPHIRecipe>(&*Iter++);
   EXPECT_NE(nullptr, CanIV);
-  VPWidenPHIRecipe *Phi = dyn_cast<VPWidenPHIRecipe>(&*Iter++);
+  auto *Phi = dyn_cast<VPPhi>(&*Iter++);
   EXPECT_NE(nullptr, Phi);
 
   VPInstruction *Idx = dyn_cast<VPInstruction>(&*Iter++);
@@ -138,7 +138,7 @@ compound=true
     N4 [label =
       "vector.body:\l" +
       "  EMIT vp\<%2\> = CANONICAL-INDUCTION ir\<0\>, vp\<%index.next\>\l" +
-      "  WIDEN-PHI ir\<%indvars.iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%indvars.iv.next\>, vector.body ]\l" +
+      "  EMIT-SCALAR ir\<%indvars.iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%indvars.iv.next\>, vector.body ]\l" +
       "  EMIT ir\<%arr.idx\> = getelementptr ir\<%A\>, ir\<%indvars.iv\>\l" +
       "  EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" +
       "  EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" +
@@ -203,7 +203,7 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
           VPInstruction::BranchOnCond,
           {Plan->getOrAddLiveIn(ConstantInt::getTrue(F->getContext()))}));
   VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
-      Plan, [](PHINode *P) { return nullptr; }, *SE, TLI);
+      Plan, [](PHINode *P) { return nullptr; }, TLI);
 
   VPBlockBase *Entry = Plan->getEntry()->getEntryBasicBlock();
   EXPECT_EQ(0u, Entry->getNumPredecessors());
@@ -304,7 +304,7 @@ compound=true
     N4 [label =
       "vector.body:\l" +
       "  EMIT vp\<%2\> = CANONICAL-INDUCTION ir\<0\>, vp\<%index.next\>\l" +
-      "  WIDEN-PHI ir\<%iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%iv.next\>, loop.latch ]\l" +
+      "  EMIT-SCALAR ir\<%iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%iv.next\>, loop.latch ]\l" +
       "  EMIT ir\<%arr.idx\> = getelementptr ir\<%A\>, ir\<%iv\>\l" +
       "  EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" +
       "  EMIT ir\<%c\> = icmp ir\<%l1\>, ir\<0\>\l" +
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 94b74d2117e18..db64c755d005f 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1088,7 +1088,7 @@ TEST_F(VPRecipeTest, CastVPInterleaveRecipeToVPUser) {
   VPValue *Addr = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
   VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
   InterleaveGroup<Instruction> IG(4, false, Align(4));
-  VPInterleaveRecipe Recipe(&IG, Addr, {}, Mask, false, DebugLoc());
+  VPInterleaveRecipe Recipe(&IG, Addr, {}, Mask, false, {}, DebugLoc());
   EXPECT_TRUE(isa<VPUser>(&Recipe));
   VPRecipeBase *BaseR = &Recipe;
   EXPECT_TRUE(isa<VPUser>(BaseR));
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
index 7dfd11a48b595..56f685801151a 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -72,10 +72,13 @@ class VPlanTestIRBase : public testing::Test {
 
     Loop *L = LI->getLoopFor(LoopHeader);
     PredicatedScalarEvolution PSE(*SE, *L);
-    auto Plan = VPlanTransforms::buildPlainCFG(L, *LI);
+    auto Plan = VPlanTransforms::buildVPlan0(L, *LI, IntegerType::get(*Ctx, 64),
+                                             {}, PSE);
+
     VFRange R(ElementCount::getFixed(1), ElementCount::getFixed(2));
-    VPlanTransforms::prepareForVectorization(*Plan, IntegerType::get(*Ctx, 64),
-                                             PSE, true, false, L, {}, false, R);
+    VPlanTransforms::handleEarlyExits(*Plan, false, R);
+    VPlanTransforms::addMiddleCheck(*Plan, true, false);
+
     VPlanTransforms::createLoopRegions(*Plan);
     return Plan;
   }
diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
index b698b287178f7..c2f045bf524e9 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
@@ -170,15 +170,14 @@ TEST_F(VPVerifierTest, VPPhiIncomingValueDoesntDominateIncomingBlock) {
   EXPECT_FALSE(verifyVPlanIsValid(Plan));
 #if GTEST_HAS_STREAM_REDIRECTION
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  EXPECT_STREQ("Incoming def at index 0 does not dominate incoming block!\n"
+  EXPECT_STREQ("Incoming def does not dominate incoming block!\n"
                "  EMIT vp<%2> = add ir<0>\n"
                "  does not dominate preheader for\n"
                "  EMIT-SCALAR vp<%1> = phi [ vp<%2>, preheader ]",
                ::testing::internal::GetCapturedStderr().c_str());
 #else
-  EXPECT_STREQ("Incoming def at index 0 does not dominate incoming block!\n", ::
-                   testing::internal::GetCapturedStderr()
-                       .c_str());
+  EXPECT_STREQ("Incoming def does not dominate incoming block!\n",
+               ::testing::internal::GetCapturedStderr().c_str());
 #endif
 #endif
 }
diff --git a/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp b/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
index ac5c455ed63ce..04707084a4166 100644
--- a/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/IntrinsicEmitter.cpp
@@ -455,15 +455,9 @@ struct FnAttributeComparator {
 
 struct AttributeComparator {
   bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
-    // Order all intrinsics with no functiona attributes before all intrinsics
-    // with function attributes.
-    bool HasFnAttrLHS = hasFnAttributes(*L);
-    bool HasFnAttrRHS = hasFnAttributes(*R);
-
-    // Order by argument attributes if function `hasFnAttributes` is equal.
-    // This is reliable because each side is already sorted internally.
-    return std::tie(HasFnAttrLHS, L->ArgumentAttributes) <
-           std::tie(HasFnAttrRHS, R->ArgumentAttributes);
+    // This comparator is used to unique just the argument attributes of an
+    // intrinsic without considering any function attributes.
+    return L->ArgumentAttributes < R->ArgumentAttributes;
   }
 };
 } // End anonymous namespace
@@ -561,7 +555,8 @@ static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID,
 } // getIntrinsicArgAttributeSet
 )";
 
-  // Compute unique function attribute sets.
+  // Compute unique function attribute sets. Note that ID 255 will be used for
+  // intrinsics with no function attributes.
   std::map<const CodeGenIntrinsic *, unsigned, FnAttributeComparator>
       UniqFnAttributes;
   OS << R"(
@@ -570,6 +565,8 @@ static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {
     default: llvm_unreachable("Invalid attribute set number");)";
 
   for (const CodeGenIntrinsic &Int : Ints) {
+    if (!hasFnAttributes(Int))
+      continue;
     unsigned ID = UniqFnAttributes.size();
     if (!UniqFnAttributes.try_emplace(&Int, ID).second)
       continue;
@@ -621,9 +618,7 @@ static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {
 
 static constexpr uint16_t IntrinsicsToAttributesMap[] = {)";
 
-  // Compute the maximum number of attribute arguments and the map. For function
-  // attributes, we only consider whether the intrinsics has any function
-  // arguments or not.
+  // Compute unique argument attributes.
   std::map<const CodeGenIntrinsic *, unsigned, AttributeComparator>
       UniqAttributes;
   for (const CodeGenIntrinsic &Int : Ints) {
@@ -631,78 +626,153 @@ static constexpr uint16_t IntrinsicsToAttributesMap[] = {)";
     UniqAttributes.try_emplace(&Int, ID);
   }
 
-  // Emit an array of AttributeList.  Most intrinsics will have at least one
-  // entry, for the function itself (index ~1), which is usually nounwind.
-  for (const CodeGenIntrinsic &Int : Ints) {
-    uint16_t FnAttrIndex = UniqFnAttributes[&Int];
-    OS << formatv("\n    {} << 8 | {}, // {}", FnAttrIndex,
-                  UniqAttributes[&Int], Int.Name);
-  }
+  constexpr uint16_t NoFunctionAttrsID = 255;
+  if (UniqAttributes.size() > 256)
+    PrintFatalError("Too many unique argument attributes for table!");
+  // Note, ID 255 is used to indicate no function attributes.
+  if (UniqFnAttributes.size() > 255)
+    PrintFatalError("Too many unique function attributes for table!");
 
   // Assign a 16-bit packed ID for each intrinsic. The lower 8-bits will be its
   // "argument attribute ID" (index in UniqAttributes) and upper 8 bits will be
   // its "function attribute ID" (index in UniqFnAttributes).
-  if (UniqAttributes.size() > 256)
-    PrintFatalError("Too many unique argument attributes for table!");
-  if (UniqFnAttributes.size() > 256)
-    PrintFatalError("Too many unique function attributes for table!");
+  for (const CodeGenIntrinsic &Int : Ints) {
+    uint16_t FnAttrIndex =
+        hasFnAttributes(Int) ? UniqFnAttributes[&Int] : NoFunctionAttrsID;
+    OS << formatv("\n    {} << 8 | {}, // {}", FnAttrIndex,
+                  UniqAttributes[&Int], Int.Name);
+  }
 
   OS << R"(
-};
-
-AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id,
-                                       FunctionType *FT) {)";
-
-  OS << formatv(R"(
-  if (id == 0)
-    return AttributeList();
-
-  uint16_t PackedID = IntrinsicsToAttributesMap[id - 1];
-  uint8_t FnAttrID = PackedID >> 8;
-  switch(PackedID & 0xFF) {{
-    default: llvm_unreachable("Invalid attribute number");
-)");
+}; // IntrinsicsToAttributesMap
+)";
 
+  // For a given intrinsic, its attributes are constructed by populating the
+  // local array `AS` below with its non-empty argument attributes followed by
+  // function attributes if any. Each argument attribute is constructed as:
+  //
+  //   getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo));
+  //
+  // Create a table that records, for each argument attributes, the list of
+  // <ArgNo, ArgAttrID> pairs that are needed to construct its argument
+  // attributes. These tables for all intrinsics will be concatenated into one
+  // large table and then for each intrinsic, we remember the Staring index and
+  // number of size of its slice of entries (i.e., number of arguments with
+  // non-empty attributes), so that we can build the attribute list for an
+  // intrinsic without using a switch-case.
+
+  using ArgNoAttrIDPair = std::pair<uint16_t, uint16_t>;
+
+  // Emit the table of concatenated <ArgNo, AttrId> using SequenceToOffsetTable
+  // so that entries can be reused if possible. Individual sequences in this
+  // table do not have any terminator.
+  using ArgAttrIDSubTable = SmallVector<ArgNoAttrIDPair>;
+  SequenceToOffsetTable<ArgAttrIDSubTable> ArgAttrIdSequenceTable(std::nullopt);
+  SmallVector<ArgAttrIDSubTable> ArgAttrIdSubTables(
+      UniqAttributes.size()); // Indexed by UniqueID.
+
+  // Find the max number of attributes to create the local array.
+  unsigned MaxNumAttrs = 0;
   for (const auto [IntPtr, UniqueID] : UniqAttributes) {
-    OS << formatv("  case {}:\n", UniqueID);
     const CodeGenIntrinsic &Int = *IntPtr;
+    ArgAttrIDSubTable SubTable;
 
-    // Keep track of the number of attributes we're writing out.
-    unsigned NumAttrs =
-        llvm::count_if(Int.ArgumentAttributes,
-                       [](const auto &Attrs) { return !Attrs.empty(); });
-    NumAttrs += hasFnAttributes(Int);
-    if (NumAttrs == 0) {
-      OS << "    return AttributeList();\n";
-      continue;
-    }
-
-    OS << "    return AttributeList::get(C, {\n";
-    ListSeparator LS(",\n");
-    for (const auto &[AttrIdx, Attrs] : enumerate(Int.ArgumentAttributes)) {
+    for (const auto &[ArgNo, Attrs] : enumerate(Int.ArgumentAttributes)) {
       if (Attrs.empty())
         continue;
 
-      unsigned ArgAttrID = UniqArgAttributes.find(Attrs)->second;
-      OS << LS
-         << formatv("      {{{}, getIntrinsicArgAttributeSet(C, {}, "
-                    "FT->getContainedType({}))}",
-                    AttrIdx, ArgAttrID, AttrIdx);
+      uint16_t ArgAttrID = UniqArgAttributes.find(Attrs)->second;
+      SubTable.emplace_back((uint16_t)ArgNo, ArgAttrID);
     }
+    ArgAttrIdSubTables[UniqueID] = SubTable;
+    if (!SubTable.empty())
+      ArgAttrIdSequenceTable.add(SubTable);
+    unsigned NumAttrs = SubTable.size() + hasFnAttributes(Int);
+    MaxNumAttrs = std::max(MaxNumAttrs, NumAttrs);
+  }
 
-    if (hasFnAttributes(Int)) {
-      OS << LS
-         << "      {AttributeList::FunctionIndex, "
-            "getIntrinsicFnAttributeSet(C, FnAttrID)}";
-    }
-    OS << "\n    });\n";
+  ArgAttrIdSequenceTable.layout();
+
+  if (ArgAttrIdSequenceTable.size() >= std::numeric_limits<uint16_t>::max())
+    PrintFatalError("Size of ArgAttrIdTable exceeds supported limit");
+
+  // Emit the 2 tables (flattened ArgNo, ArgAttrID) and ArgAttributesInfoTable.
+  OS << formatv(R"(
+namespace {{
+struct ArgNoAttrIDPair {{
+  uint16_t ArgNo, ArgAttrID;
+};
+} // namespace
+
+// Number of entries: {}
+static constexpr ArgNoAttrIDPair ArgAttrIdTable[] = {{
+)",
+                ArgAttrIdSequenceTable.size());
+
+  ArgAttrIdSequenceTable.emit(OS, [](raw_ostream &OS, ArgNoAttrIDPair Elem) {
+    OS << formatv("{{{}, {}}", Elem.first, Elem.second);
+  });
+
+  OS << formatv(R"(}; // ArgAttrIdTable
+
+namespace {{
+struct ArgAttributesInfo {{
+  uint16_t StartIndex;
+  uint16_t NumAttrs;
+};
+} // namespace
+
+// Number of entries: {}
+static constexpr ArgAttributesInfo ArgAttributesInfoTable[] = {{
+)",
+                ArgAttrIdSubTables.size());
+
+  for (const auto &SubTable : ArgAttrIdSubTables) {
+    unsigned NumAttrs = SubTable.size();
+    unsigned StartIndex = NumAttrs ? ArgAttrIdSequenceTable.get(SubTable) : 0;
+    OS << formatv("  {{{}, {}},\n", StartIndex, NumAttrs);
   }
+  OS << "}; // ArgAttributesInfoTable\n";
 
-  OS << R"(  }
+  // Now emit the Intrinsic::getAttributes function. This will first map
+  // from intrinsic ID -> unique arg/function attr ID (using the
+  // IntrinsicsToAttributesMap) table. Then it will use the unique arg ID to
+  // construct all the argument attributes (using the ArgAttributesInfoTable and
+  // ArgAttrIdTable) and then add on the function attributes if any.
+  OS << formatv(R"(
+AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id,
+                                       FunctionType *FT) {{
+  if (id == 0)
+    return AttributeList();
+
+  uint16_t PackedID = IntrinsicsToAttributesMap[id - 1];
+  uint8_t FnAttrID = PackedID >> 8;
+  uint8_t ArgAttrID = PackedID & 0xFF;
+  using PairTy = std::pair<unsigned, AttributeSet>;
+  alignas(PairTy) char ASStorage[sizeof(PairTy) * {}];
+  PairTy *AS = reinterpret_cast<PairTy *>(ASStorage);
+
+  // Construct an ArrayRef for easier range checking.
+  ArrayRef<ArgAttributesInfo> ArgAttributesInfoTableAR(ArgAttributesInfoTable);
+  if (ArgAttrID >= ArgAttributesInfoTableAR.size())
+    llvm_unreachable("Invalid arguments attribute ID");
+
+  auto [StartIndex, NumAttrs] = ArgAttributesInfoTableAR[ArgAttrID];
+  for (unsigned Idx = 0; Idx < NumAttrs; ++Idx) {{
+    auto [ArgNo, ArgAttrID] = ArgAttrIdTable[StartIndex + Idx];
+    AS[Idx] = {{ArgNo,
+        getIntrinsicArgAttributeSet(C, ArgAttrID, FT->getContainedType(ArgNo))};
+  }
+  if (FnAttrID != {}) {
+    AS[NumAttrs++] = {{AttributeList::FunctionIndex,
+                      getIntrinsicFnAttributeSet(C, FnAttrID)};
+  }
+  return AttributeList::get(C, ArrayRef(AS, NumAttrs));
 }
 #endif // GET_INTRINSIC_ATTRIBUTES
 
-)";
+)",
+                MaxNumAttrs, NoFunctionAttrsID);
 }
 
 void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
diff --git a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
index 412431b96d030..a1020d5a0db1f 100644
--- a/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
+++ b/llvm/utils/TableGen/Basic/RuntimeLibcallsEmitter.cpp
@@ -6,10 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#define DEBUG_TYPE "runtime-libcall-emitter"
+
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/xxhash.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/SetTheory.h"
@@ -215,6 +220,9 @@ class RuntimeLibcallEmitter {
 private:
   void emitGetRuntimeLibcallEnum(raw_ostream &OS) const;
 
+  void emitNameMatchHashTable(raw_ostream &OS,
+                              StringToOffsetTable &OffsetTable) const;
+
   void emitGetInitRuntimeLibcallNames(raw_ostream &OS) const;
 
   void emitSystemRuntimeLibrarySetCalls(raw_ostream &OS) const;
@@ -255,12 +263,9 @@ class RuntimeLibcallEmitter {
       RuntimeLibcallImplDefList.emplace_back(LibCallImplDef, Def2RuntimeLibcall,
                                              LibCallImplEnumVal++);
 
-      RuntimeLibcallImpl &LibCallImpl = RuntimeLibcallImplDefList.back();
-
+      const RuntimeLibcallImpl &LibCallImpl = RuntimeLibcallImplDefList.back();
       Def2RuntimeLibcallImpl[LibCallImplDef] = &LibCallImpl;
 
-      // const RuntimeLibcallImpl &LibCallImpl =
-      // RuntimeLibcallImplDefList.back();
       if (LibCallImpl.isDefault()) {
         const RuntimeLibcall *Provides = LibCallImpl.getProvides();
         if (!Provides)
@@ -295,8 +300,6 @@ void RuntimeLibcallEmitter::emitGetRuntimeLibcallEnum(raw_ostream &OS) const {
     OS << "  " << Name << " = " << LibCall.getEnumVal() << ",\n";
   }
 
-  // TODO: Emit libcall names as string offset table.
-
   OS << "  UNKNOWN_LIBCALL = " << RuntimeLibcallDefList.size()
      << "\n};\n\n"
         "enum LibcallImpl : unsigned short {\n"
@@ -315,8 +318,176 @@ void RuntimeLibcallEmitter::emitGetRuntimeLibcallEnum(raw_ostream &OS) const {
         "#endif\n\n";
 }
 
+// StringMap uses xxh3_64bits, truncated to uint32_t.
+static uint64_t hash(StringRef Str) {
+  return static_cast<uint32_t>(xxh3_64bits(Str));
+}
+
+static void emitHashFunction(raw_ostream &OS) {
+  OS << "static inline uint64_t hash(StringRef Str) {\n"
+        "  return static_cast<uint32_t>(xxh3_64bits(Str));\n"
+        "}\n\n";
+}
+
+/// Return the table size, maximum number of collisions for the set of hashes
+static std::pair<int, int>
+computePerfectHashParameters(ArrayRef<uint64_t> Hashes) {
+  // Chosen based on experimentation with llvm/benchmarks/RuntimeLibcalls.cpp
+  const int SizeOverhead = 4;
+
+  // Index derived from hash -> number of collisions.
+  DenseMap<uint64_t, int> Table;
+
+  unsigned NumHashes = Hashes.size();
+
+  for (int MaxCollisions = 1;; ++MaxCollisions) {
+    for (unsigned N = NextPowerOf2(NumHashes - 1); N < SizeOverhead * NumHashes;
+         N <<= 1) {
+      Table.clear();
+
+      bool NeedResize = false;
+      for (uint64_t H : Hashes) {
+        uint64_t Idx = H % static_cast<uint64_t>(N);
+        if (++Table[Idx] > MaxCollisions) {
+          // Need to resize the final table if we increased the collision count.
+          NeedResize = true;
+          break;
+        }
+      }
+
+      if (!NeedResize)
+        return {N, MaxCollisions};
+    }
+  }
+}
+
+static std::vector<unsigned>
+constructPerfectHashTable(ArrayRef<RuntimeLibcallImpl> Keywords,
+                          ArrayRef<uint64_t> Hashes,
+                          ArrayRef<unsigned> TableValues, int Size,
+                          int Collisions, StringToOffsetTable &OffsetTable) {
+  std::vector<unsigned> Lookup(Size * Collisions);
+
+  for (auto [HashValue, TableValue] : zip(Hashes, TableValues)) {
+    uint64_t Idx = (HashValue % static_cast<uint64_t>(Size)) *
+                   static_cast<uint64_t>(Collisions);
+
+    bool Found = false;
+    for (int J = 0; J < Collisions; ++J) {
+      unsigned &Entry = Lookup[Idx + J];
+      if (Entry == 0) {
+        Entry = TableValue;
+        Found = true;
+        break;
+      }
+    }
+
+    if (!Found)
+      reportFatalInternalError("failure to hash");
+  }
+
+  return Lookup;
+}
+
+/// Generate hash table based lookup by name.
+void RuntimeLibcallEmitter::emitNameMatchHashTable(
+    raw_ostream &OS, StringToOffsetTable &OffsetTable) const {
+  std::vector<uint64_t> Hashes(RuntimeLibcallImplDefList.size());
+  std::vector<unsigned> TableValues(RuntimeLibcallImplDefList.size());
+  DenseSet<StringRef> SeenFuncNames;
+
+  size_t MaxFuncNameSize = 0;
+  size_t Index = 0;
+
+  for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
+    StringRef ImplName = LibCallImpl.getLibcallFuncName();
+    if (SeenFuncNames.insert(ImplName).second) {
+      MaxFuncNameSize = std::max(MaxFuncNameSize, ImplName.size());
+      TableValues[Index] = LibCallImpl.getEnumVal();
+      Hashes[Index++] = hash(ImplName);
+    }
+  }
+
+  // Trim excess elements from non-unique entries.
+  Hashes.resize(SeenFuncNames.size());
+  TableValues.resize(SeenFuncNames.size());
+
+  LLVM_DEBUG({
+    for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
+      StringRef ImplName = LibCallImpl.getLibcallFuncName();
+      if (ImplName.size() == MaxFuncNameSize) {
+        dbgs() << "Maximum runtime libcall name size: " << ImplName << '('
+               << MaxFuncNameSize << ")\n";
+      }
+    }
+  });
+
+  // Early exiting on the symbol name provides a significant speedup in the miss
+  // case on the set of symbols in a clang binary. Emit this as an inlinable
+  // precondition in the header.
+  //
+  // The empty check is also used to get sensible behavior on anonymous
+  // functions.
+  //
+  // TODO: It may make more sense to split the search by string size more. There
+  // are a few outliers, most call names are small.
+  OS << "#ifdef GET_LOOKUP_LIBCALL_IMPL_NAME_BODY\n"
+        "  size_t Size = Name.size();\n"
+        "  if (Size == 0 || Size > "
+     << MaxFuncNameSize
+     << ")\n"
+        "    return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);\n"
+        " return lookupLibcallImplNameImpl(Name);\n"
+        "#endif\n";
+
+  auto [Size, Collisions] = computePerfectHashParameters(Hashes);
+  std::vector<unsigned> Lookup =
+      constructPerfectHashTable(RuntimeLibcallImplDefList, Hashes, TableValues,
+                                Size, Collisions, OffsetTable);
+
+  LLVM_DEBUG(dbgs() << "Runtime libcall perfect hashing parameters: Size = "
+                    << Size << ", maximum collisions = " << Collisions << '\n');
+
+  OS << "#ifdef DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME\n";
+  emitHashFunction(OS);
+
+  OS << "iota_range<RTLIB::LibcallImpl> RTLIB::RuntimeLibcallsInfo::"
+        "lookupLibcallImplNameImpl(StringRef Name) {\n";
+
+  // Emit RTLIB::LibcallImpl values
+  OS << "  static constexpr uint16_t HashTableNameToEnum[" << Lookup.size()
+     << "] = {\n";
+
+  for (unsigned TableVal : Lookup)
+    OS << "    " << TableVal << ",\n";
+
+  OS << "  };\n\n";
+
+  OS << "  unsigned Idx = (hash(Name) % " << Size << ") * " << Collisions
+     << ";\n\n"
+        "  for (int I = 0; I != "
+     << Collisions << R"(; ++I) {
+    const uint16_t Entry = HashTableNameToEnum[Idx + I];
+    const uint16_t StrOffset = RuntimeLibcallNameOffsetTable[Entry];
+    const uint8_t StrSize = RuntimeLibcallNameSizeTable[Entry];
+    StringRef Str(
+      &RTLIB::RuntimeLibcallsInfo::RuntimeLibcallImplNameTableStorage[StrOffset],
+      StrSize);
+    if (Str == Name)
+      return libcallImplNameHit(Entry, StrOffset);
+  }
+
+  return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);
+}
+)";
+
+  OS << "#endif\n\n";
+}
+
 void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallNames(
     raw_ostream &OS) const {
+  OS << "#ifdef GET_INIT_RUNTIME_LIBCALL_NAMES\n";
+
   // Emit the implementation names
   StringToOffsetTable Table(/*AppendZero=*/true,
                             "RTLIB::RuntimeLibcallsInfo::");
@@ -337,6 +508,15 @@ const uint16_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameOffsetTable[] = {
   }
   OS << "};\n";
 
+  OS << R"(
+const uint8_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameSizeTable[] = {
+)";
+
+  OS << "  0,\n";
+  for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList)
+    OS << "  " << LibCallImpl.getLibcallFuncName().size() << ",\n";
+  OS << "};\n\n";
+
   // Emit the reverse mapping from implementation libraries to RTLIB::Libcall
   OS << "const RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::"
         "ImplToLibcall[RTLIB::NumLibcallImpls] = {\n"
@@ -351,15 +531,31 @@ const uint16_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameOffsetTable[] = {
     OS << '\n';
   }
   OS << "};\n\n";
+
+  OS << "#endif\n\n";
+
+  emitNameMatchHashTable(OS, Table);
 }
 
 void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
     raw_ostream &OS) const {
   OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets("
-        "const llvm::Triple &TT, FloatABI::ABIType FloatABI) {\n"
+        "const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
+        "FloatABI::ABIType FloatABI, EABI EABIVersion, "
+        "StringRef ABIName) {\n"
         "  struct LibcallImplPair {\n"
         "    RTLIB::Libcall Func;\n"
         "    RTLIB::LibcallImpl Impl;\n"
+        "  };\n"
+        "  auto setLibcallsImpl = [this](\n"
+        "    ArrayRef<LibcallImplPair> Libcalls,\n"
+        "    std::optional<llvm::CallingConv::ID> CC = {})\n"
+        "  {\n"
+        "    for (const auto [Func, Impl] : Libcalls) {\n"
+        "      setLibcallImpl(Func, Impl);\n"
+        "      if (CC)\n"
+        "        setLibcallImplCallingConv(Impl, *CC);\n"
+        "    }\n"
         "  };\n";
   ArrayRef<const Record *> AllLibs =
       Records.getAllDerivedDefinitions("SystemRuntimeLibrary");
@@ -485,31 +681,18 @@ void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
 
       Funcs.erase(UniqueI, Funcs.end());
 
-      OS << indent(IndentDepth + 2)
-         << "static const LibcallImplPair LibraryCalls";
-      SubsetPredicate.emitTableVariableNameSuffix(OS);
-      OS << "[] = {\n";
+      OS << indent(IndentDepth + 2) << "setLibcallsImpl({\n";
       for (const RuntimeLibcallImpl *LibCallImpl : Funcs) {
-        OS << indent(IndentDepth + 6);
+        OS << indent(IndentDepth + 4);
         LibCallImpl->emitTableEntry(OS);
       }
-
-      OS << indent(IndentDepth + 2) << "};\n\n"
-         << indent(IndentDepth + 2)
-         << "for (const auto [Func, Impl] : LibraryCalls";
-      SubsetPredicate.emitTableVariableNameSuffix(OS);
-      OS << ") {\n"
-         << indent(IndentDepth + 4) << "setLibcallImpl(Func, Impl);\n";
-
+      OS << indent(IndentDepth + 2) << "}";
       if (FuncsWithCC.CallingConv) {
         StringRef CCEnum =
             FuncsWithCC.CallingConv->getValueAsString("CallingConv");
-        OS << indent(IndentDepth + 4) << "setLibcallImplCallingConv(Impl, "
-           << CCEnum << ");\n";
+        OS << ", " << CCEnum;
       }
-
-      OS << indent(IndentDepth + 2) << "}\n";
-      OS << '\n';
+      OS << ");\n\n";
 
       if (!SubsetPredicate.isAlwaysAvailable()) {
         OS << indent(IndentDepth);
@@ -533,9 +716,7 @@ void RuntimeLibcallEmitter::run(raw_ostream &OS) {
   emitSourceFileHeader("Runtime LibCalls Source Fragment", OS, Records);
   emitGetRuntimeLibcallEnum(OS);
 
-  OS << "#ifdef GET_INIT_RUNTIME_LIBCALL_NAMES\n";
   emitGetInitRuntimeLibcallNames(OS);
-  OS << "#endif\n\n";
 
   OS << "#ifdef GET_SET_TARGET_RUNTIME_LIBCALL_SETS\n";
   emitSystemRuntimeLibrarySetCalls(OS);
@@ -565,7 +746,9 @@ void LibcallPredicateExpander::expand(SetTheory &ST, const Record *Def,
       auto [It, Inserted] = Func2Preds.insert({LibcallImpl, {{}, CCClass}});
       if (!Inserted) {
         PrintError(
-            Def, "combining nested libcall set predicates currently unhandled");
+            Def,
+            "combining nested libcall set predicates currently unhandled: '" +
+                LibcallImpl->getLibcallFuncName() + "'");
       }
 
       It->second.first.push_back(AP.getDef());
diff --git a/llvm/utils/TableGen/CallingConvEmitter.cpp b/llvm/utils/TableGen/CallingConvEmitter.cpp
index 3084d90d444aa..e0d933723ad66 100644
--- a/llvm/utils/TableGen/CallingConvEmitter.cpp
+++ b/llvm/utils/TableGen/CallingConvEmitter.cpp
@@ -77,7 +77,7 @@ void CallingConvEmitter::run(raw_ostream &O) {
       O << CC->getName() << "(unsigned ValNo, MVT ValVT,\n"
         << std::string(Pad, ' ') << "MVT LocVT, CCValAssign::LocInfo LocInfo,\n"
         << std::string(Pad, ' ')
-        << "ISD::ArgFlagsTy ArgFlags, CCState &State);\n";
+        << "ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State);\n";
     }
   }
 
@@ -115,7 +115,8 @@ void CallingConvEmitter::emitCallingConv(const Record *CC, raw_ostream &O) {
   }
   O << CurrentAction << "(unsigned ValNo, MVT ValVT,\n"
     << std::string(Pad, ' ') << "MVT LocVT, CCValAssign::LocInfo LocInfo,\n"
-    << std::string(Pad, ' ') << "ISD::ArgFlagsTy ArgFlags, CCState &State) {\n";
+    << std::string(Pad, ' ') << "ISD::ArgFlagsTy ArgFlags, Type *OrigTy, "
+    << "CCState &State) {\n";
   // Emit all of the actions, in order.
   for (unsigned I = 0, E = CCActions->size(); I != E; ++I) {
     const Record *Action = CCActions->getElementAsRecord(I);
@@ -227,7 +228,7 @@ void CallingConvEmitter::emitAction(const Record *Action, indent Indent,
   if (Action->isSubClassOf("CCDelegateTo")) {
     const Record *CC = Action->getValueAsDef("CC");
     O << Indent << "if (!" << CC->getName()
-      << "(ValNo, ValVT, LocVT, LocInfo, ArgFlags, State))\n"
+      << "(ValNo, ValVT, LocVT, LocInfo, ArgFlags, OrigTy, State))\n"
       << Indent + 2 << "return false;\n";
     DelegateToMap[CurrentAction].insert(CC->getName().str());
   } else if (Action->isSubClassOf("CCAssignToReg") ||
diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp
index 3ab6f40a6deec..7479a31e5078e 100644
--- a/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -153,11 +153,10 @@ bool CodeEmitterGen::addCodeToMergeInOperand(const Record *R,
   if (!EncoderMethodName.empty()) {
     raw_string_ostream CaseOS(Case);
     CaseOS << indent(6);
-    if (UseAPInt) {
-      CaseOS << EncoderMethodName << "(MI, " + utostr(OpIdx) << ", op";
-    } else {
-      CaseOS << "op = " << EncoderMethodName << "(MI, " << utostr(OpIdx);
-    }
+    if (UseAPInt)
+      CaseOS << EncoderMethodName << "(MI, " << OpIdx << ", op";
+    else
+      CaseOS << "op = " << EncoderMethodName << "(MI, " << OpIdx;
     CaseOS << ", Fixups, STI);\n";
   } else {
     if (UseAPInt) {
@@ -290,55 +289,52 @@ CodeEmitterGen::getInstructionCases(const Record *R,
     BitOffsetCase += S;
   };
 
-  if (const RecordVal *RV = R->getValue("EncodingInfos")) {
-    if (const auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
-      const CodeGenHwModes &HWM = Target.getHwModes();
-      EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-
-      // Invoke the interface to obtain the HwMode ID controlling the
-      // EncodingInfo for the current subtarget. This interface will
-      // mask off irrelevant HwMode IDs.
-      Append("      unsigned HwMode = "
-             "STI.getHwMode(MCSubtargetInfo::HwMode_EncodingInfo);\n");
-      Case += "      switch (HwMode) {\n";
-      Case += "      default: llvm_unreachable(\"Unknown hardware mode!\"); "
-              "break;\n";
-      for (auto &[ModeId, Encoding] : EBM) {
-        if (ModeId == DefaultMode) {
-          Case +=
-              "      case " + itostr(DefaultMode) + ": InstBitsByHw = InstBits";
-        } else {
-          Case += "      case " + itostr(ModeId) +
-                  ": InstBitsByHw = InstBits_" + HWM.getMode(ModeId).Name.str();
-        }
-        Case += "; break;\n";
-      }
-      Case += "      };\n";
-
-      // We need to remodify the 'Inst' value from the table we found above.
-      if (UseAPInt) {
-        int NumWords = APInt::getNumWords(BitWidth);
-        Case += "      Inst = APInt(" + itostr(BitWidth);
-        Case += ", ArrayRef(InstBitsByHw + opcode * " + itostr(NumWords) +
-                ", " + itostr(NumWords);
-        Case += "));\n";
-        Case += "      Value = Inst;\n";
+  if (const Record *RV = R->getValueAsOptionalDef("EncodingInfos")) {
+    const CodeGenHwModes &HWM = Target.getHwModes();
+    EncodingInfoByHwMode EBM(RV, HWM);
+
+    // Invoke the interface to obtain the HwMode ID controlling the
+    // EncodingInfo for the current subtarget. This interface will
+    // mask off irrelevant HwMode IDs.
+    Append("      unsigned HwMode = "
+           "STI.getHwMode(MCSubtargetInfo::HwMode_EncodingInfo);\n");
+    Case += "      switch (HwMode) {\n";
+    Case += "      default: llvm_unreachable(\"Unknown hardware mode!\"); "
+            "break;\n";
+    for (auto &[ModeId, Encoding] : EBM) {
+      if (ModeId == DefaultMode) {
+        Case +=
+            "      case " + itostr(DefaultMode) + ": InstBitsByHw = InstBits";
       } else {
-        Case += "      Value = InstBitsByHw[opcode];\n";
+        Case += "      case " + itostr(ModeId) + ": InstBitsByHw = InstBits_" +
+                HWM.getMode(ModeId).Name.str();
       }
+      Case += "; break;\n";
+    }
+    Case += "      };\n";
 
-      Append("      switch (HwMode) {\n");
-      Append("      default: llvm_unreachable(\"Unhandled HwMode\");\n");
-      for (auto &[ModeId, Encoding] : EBM) {
-        Append("      case " + itostr(ModeId) + ": {\n");
-        addInstructionCasesForEncoding(R, Encoding, Target, Case,
-                                       BitOffsetCase);
-        Append("      break;\n");
-        Append("      }\n");
-      }
+    // We need to remodify the 'Inst' value from the table we found above.
+    if (UseAPInt) {
+      int NumWords = APInt::getNumWords(BitWidth);
+      Case += "      Inst = APInt(" + itostr(BitWidth);
+      Case += ", ArrayRef(InstBitsByHw + opcode * " + itostr(NumWords) + ", " +
+              itostr(NumWords);
+      Case += "));\n";
+      Case += "      Value = Inst;\n";
+    } else {
+      Case += "      Value = InstBitsByHw[opcode];\n";
+    }
+
+    Append("      switch (HwMode) {\n");
+    Append("      default: llvm_unreachable(\"Unhandled HwMode\");\n");
+    for (auto &[ModeId, Encoding] : EBM) {
+      Append("      case " + itostr(ModeId) + ": {\n");
+      addInstructionCasesForEncoding(R, Encoding, Target, Case, BitOffsetCase);
+      Append("      break;\n");
       Append("      }\n");
-      return {std::move(Case), std::move(BitOffsetCase)};
     }
+    Append("      }\n");
+    return {std::move(Case), std::move(BitOffsetCase)};
   }
   addInstructionCasesForEncoding(R, R, Target, Case, BitOffsetCase);
   return {std::move(Case), std::move(BitOffsetCase)};
@@ -391,8 +387,7 @@ void CodeEmitterGen::addInstructionCasesForEncoding(
 
 static void emitInstBits(raw_ostream &OS, const APInt &Bits) {
   for (unsigned I = 0; I < Bits.getNumWords(); ++I)
-    OS << ((I > 0) ? ", " : "") << "UINT64_C(" << utostr(Bits.getRawData()[I])
-       << ")";
+    OS << ((I > 0) ? ", " : "") << "UINT64_C(" << Bits.getRawData()[I] << ")";
 }
 
 void CodeEmitterGen::emitInstructionBaseValues(
@@ -417,20 +412,18 @@ void CodeEmitterGen::emitInstructionBaseValues(
     }
 
     const Record *EncodingDef = R;
-    if (const RecordVal *RV = R->getValue("EncodingInfos")) {
-      if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
-        EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-        if (EBM.hasMode(HwMode)) {
-          EncodingDef = EBM.get(HwMode);
-        } else {
-          // If the HwMode does not match, then Encoding '0'
-          // should be generated.
-          APInt Value(BitWidth, 0);
-          O << "    ";
-          emitInstBits(O, Value);
-          O << "," << '\t' << "// " << R->getName() << "\n";
-          continue;
-        }
+    if (const Record *RV = R->getValueAsOptionalDef("EncodingInfos")) {
+      EncodingInfoByHwMode EBM(RV, HWM);
+      if (EBM.hasMode(HwMode)) {
+        EncodingDef = EBM.get(HwMode);
+      } else {
+        // If the HwMode does not match, then Encoding '0'
+        // should be generated.
+        APInt Value(BitWidth, 0);
+        O << "    ";
+        emitInstBits(O, Value);
+        O << "," << '\t' << "// " << R->getName() << "\n";
+        continue;
       }
     }
     const BitsInit *BI = EncodingDef->getValueAsBitsInit("Inst");
@@ -445,7 +438,7 @@ void CodeEmitterGen::emitInstructionBaseValues(
     emitInstBits(O, Value);
     O << "," << '\t' << "// " << R->getName() << "\n";
   }
-  O << "    UINT64_C(0)\n  };\n";
+  O << "  };\n";
 }
 
 void CodeEmitterGen::emitCaseMap(
@@ -490,16 +483,14 @@ void CodeEmitterGen::run(raw_ostream &O) {
           R->getValueAsBit("isPseudo"))
         continue;
 
-      if (const RecordVal *RV = R->getValue("EncodingInfos")) {
-        if (const DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
-          EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-          for (const auto &[Key, Value] : EBM) {
-            const BitsInit *BI = Value->getValueAsBitsInit("Inst");
-            BitWidth = std::max(BitWidth, BI->getNumBits());
-            HwModes.insert(Key);
-          }
-          continue;
+      if (const Record *RV = R->getValueAsOptionalDef("EncodingInfos")) {
+        EncodingInfoByHwMode EBM(RV, HWM);
+        for (const auto &[Key, Value] : EBM) {
+          const BitsInit *BI = Value->getValueAsBitsInit("Inst");
+          BitWidth = std::max(BitWidth, BI->getNumBits());
+          HwModes.insert(Key);
         }
+        continue;
       }
       const BitsInit *BI = R->getValueAsBitsInit("Inst");
       BitWidth = std::max(BitWidth, BI->getNumBits());
diff --git a/llvm/utils/TableGen/Common/CodeGenInstruction.cpp b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
index 3dbf2b9b446e8..d1acd721813f3 100644
--- a/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenInstruction.cpp
@@ -227,7 +227,7 @@ CGIOperandList::findSubOperandAlias(StringRef Name) const {
 }
 
 std::pair<unsigned, unsigned>
-CGIOperandList::ParseOperandName(StringRef Op, bool AllowWholeOp) {
+CGIOperandList::parseOperandName(StringRef Op, bool AllowWholeOp) const {
   if (!Op.starts_with("$"))
     PrintFatalError(TheDef->getLoc(),
                     TheDef->getName() + ": Illegal operand name: '" + Op + "'");
@@ -307,7 +307,7 @@ static void ParseConstraint(StringRef CStr, CGIOperandList &Ops,
                       "Illegal format for @earlyclobber constraint in '" +
                           Rec->getName() + "': '" + CStr + "'");
     Name = Name.substr(wpos);
-    std::pair<unsigned, unsigned> Op = Ops.ParseOperandName(Name, false);
+    std::pair<unsigned, unsigned> Op = Ops.parseOperandName(Name, false);
 
     // Build the string for the operand
     if (!Ops[Op.first].Constraints[Op.second].isNone())
@@ -335,7 +335,7 @@ static void ParseConstraint(StringRef CStr, CGIOperandList &Ops,
                     "Illegal format for tied-to constraint in '" +
                         Rec->getName() + "': '" + CStr + "'");
   StringRef LHSOpName = CStr.substr(start, wpos - start);
-  std::pair<unsigned, unsigned> LHSOp = Ops.ParseOperandName(LHSOpName, false);
+  std::pair<unsigned, unsigned> LHSOp = Ops.parseOperandName(LHSOpName, false);
 
   wpos = CStr.find_first_not_of(" \t", pos + 1);
   if (wpos == StringRef::npos)
@@ -343,7 +343,7 @@ static void ParseConstraint(StringRef CStr, CGIOperandList &Ops,
                     "Illegal format for tied-to constraint: '" + CStr + "'");
 
   StringRef RHSOpName = CStr.substr(wpos);
-  std::pair<unsigned, unsigned> RHSOp = Ops.ParseOperandName(RHSOpName, false);
+  std::pair<unsigned, unsigned> RHSOp = Ops.parseOperandName(RHSOpName, false);
 
   // Sort the operands into order, which should put the output one
   // first. But keep the original order, for use in diagnostics.
@@ -414,7 +414,7 @@ void CGIOperandList::ProcessDisableEncoding(StringRef DisableEncoding) {
       break;
 
     // Figure out which operand this is.
-    std::pair<unsigned, unsigned> Op = ParseOperandName(OpName, false);
+    std::pair<unsigned, unsigned> Op = parseOperandName(OpName, false);
 
     // Mark the operand as not-to-be encoded.
     OperandList[Op.first].DoNotEncode[Op.second] = true;
diff --git a/llvm/utils/TableGen/Common/CodeGenInstruction.h b/llvm/utils/TableGen/Common/CodeGenInstruction.h
index 1ae913c262b2e..591a7cc811dd6 100644
--- a/llvm/utils/TableGen/Common/CodeGenInstruction.h
+++ b/llvm/utils/TableGen/Common/CodeGenInstruction.h
@@ -183,12 +183,12 @@ class CGIOperandList {
   std::optional<std::pair<unsigned, unsigned>>
   findSubOperandAlias(StringRef Name) const;
 
-  /// ParseOperandName - Parse an operand name like "$foo" or "$foo.bar",
+  /// Parses an operand name like "$foo" or "$foo.bar",
   /// where $foo is a whole operand and $foo.bar refers to a suboperand.
   /// This aborts if the name is invalid.  If AllowWholeOp is true, references
   /// to operands with suboperands are allowed, otherwise not.
-  std::pair<unsigned, unsigned> ParseOperandName(StringRef Op,
-                                                 bool AllowWholeOp = true);
+  std::pair<unsigned, unsigned>
+  parseOperandName(StringRef Op, bool AllowWholeOp = true) const;
 
   /// getFlattenedOperandNumber - Flatten a operand/suboperand pair into a
   /// flat machineinstr operand #.
diff --git a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
index f78427940b276..e873b3eaa4b7e 100644
--- a/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenRegisters.cpp
@@ -57,9 +57,8 @@ CodeGenSubRegIndex::CodeGenSubRegIndex(const Record *R, unsigned Enum,
   if (R->getValue("Namespace"))
     Namespace = R->getValueAsString("Namespace").str();
 
-  if (const RecordVal *RV = R->getValue("SubRegRanges"))
-    if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue()))
-      Range = SubRegRangeByHwMode(DI->getDef(), CGH);
+  if (const Record *RV = R->getValueAsOptionalDef("SubRegRanges"))
+    Range = SubRegRangeByHwMode(RV, CGH);
   if (!Range.hasDefault())
     Range.insertSubRegRangeForMode(DefaultMode, SubRegRange(R));
 }
@@ -732,9 +731,8 @@ CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank,
 
   Namespace = R->getValueAsString("Namespace");
 
-  if (const RecordVal *RV = R->getValue("RegInfos"))
-    if (const DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue()))
-      RSI = RegSizeInfoByHwMode(DI->getDef(), RegBank.getHwModes());
+  if (const Record *RV = R->getValueAsOptionalDef("RegInfos"))
+    RSI = RegSizeInfoByHwMode(RV, RegBank.getHwModes());
   unsigned Size = R->getValueAsInt("Size");
   assert((RSI.hasDefault() || Size != 0 || VTs[0].isSimple()) &&
          "Impossible to determine register size");
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index 70141ba738bdb..efaf05adc8f06 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -53,26 +53,26 @@ constexpr StringLiteral EncodeMacroName = "GIMT_Encode";
 void emitEncodingMacrosDef(raw_ostream &OS) {
   OS << "#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__\n"
      << "#define " << EncodeMacroName << "2(Val)"
-     << " uint8_t(Val), uint8_t((uint16_t)Val >> 8)\n"
+     << " uint8_t(Val), uint8_t((Val) >> 8)\n"
      << "#define " << EncodeMacroName << "4(Val)"
-     << " uint8_t(Val), uint8_t((uint32_t)Val >> 8), "
-        "uint8_t((uint32_t)Val >> 16), uint8_t((uint32_t)Val >> 24)\n"
+     << " uint8_t(Val), uint8_t((Val) >> 8), "
+        "uint8_t((Val) >> 16), uint8_t((Val) >> 24)\n"
      << "#define " << EncodeMacroName << "8(Val)"
-     << " uint8_t(Val), uint8_t((uint64_t)Val >> 8), "
-        "uint8_t((uint64_t)Val >> 16), uint8_t((uint64_t)Val >> 24),  "
-        "uint8_t((uint64_t)Val >> 32), uint8_t((uint64_t)Val >> 40), "
-        "uint8_t((uint64_t)Val >> 48), uint8_t((uint64_t)Val >> 56)\n"
+     << " uint8_t(Val), uint8_t((Val) >> 8), "
+        "uint8_t((Val) >> 16), uint8_t((Val) >> 24),  "
+        "uint8_t(uint64_t(Val) >> 32), uint8_t(uint64_t(Val) >> 40), "
+        "uint8_t(uint64_t(Val) >> 48), uint8_t(uint64_t(Val) >> 56)\n"
      << "#else\n"
      << "#define " << EncodeMacroName << "2(Val)"
-     << " uint8_t((uint16_t)Val >> 8), uint8_t(Val)\n"
+     << " uint8_t((Val) >> 8), uint8_t(Val)\n"
      << "#define " << EncodeMacroName << "4(Val)"
-     << " uint8_t((uint32_t)Val >> 24), uint8_t((uint32_t)Val >> 16), "
-        "uint8_t((uint32_t)Val >> 8), uint8_t(Val)\n"
+     << " uint8_t((Val) >> 24), uint8_t((Val) >> 16), "
+        "uint8_t((Val) >> 8), uint8_t(Val)\n"
      << "#define " << EncodeMacroName << "8(Val)"
-     << " uint8_t((uint64_t)Val >> 56), uint8_t((uint64_t)Val >> 48), "
-        "uint8_t((uint64_t)Val >> 40), uint8_t((uint64_t)Val >> 32),  "
-        "uint8_t((uint64_t)Val >> 24), uint8_t((uint64_t)Val >> 16), "
-        "uint8_t((uint64_t)Val >> 8), uint8_t(Val)\n"
+     << " uint8_t(uint64_t(Val) >> 56), uint8_t(uint64_t(Val) >> 48), "
+        "uint8_t(uint64_t(Val) >> 40), uint8_t(uint64_t(Val) >> 32),  "
+        "uint8_t((Val) >> 24), uint8_t((Val) >> 16), "
+        "uint8_t((Val) >> 8), uint8_t(Val)\n"
      << "#endif\n";
 }
 
@@ -237,9 +237,12 @@ MatchTableRecord MatchTable::NamedValue(unsigned NumBytes, StringRef Namespace,
 
 MatchTableRecord MatchTable::IntValue(unsigned NumBytes, int64_t IntValue) {
   assert(isUIntN(NumBytes * 8, IntValue) || isIntN(NumBytes * 8, IntValue));
-  auto Str = llvm::to_string(IntValue);
-  if (NumBytes == 1 && IntValue < 0)
-    Str = "uint8_t(" + Str + ")";
+  uint64_t UIntValue = IntValue;
+  if (NumBytes < 8)
+    UIntValue &= (UINT64_C(1) << NumBytes * 8) - 1;
+  std::string Str = llvm::to_string(UIntValue);
+  if (UIntValue > INT64_MAX)
+    Str += 'u';
   // TODO: Could optimize this directly to save the compiler some work when
   // building the file
   return MatchTableRecord(std::nullopt, Str, NumBytes,
diff --git a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
index 77a2ae60ad6f9..b617a3dbca586 100644
--- a/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/Common/VarLenCodeEmitterGen.cpp
@@ -236,18 +236,16 @@ void VarLenCodeEmitterGen::run(raw_ostream &OS) {
       continue;
 
     // Setup alternative encodings according to HwModes
-    if (const RecordVal *RV = R->getValue("EncodingInfos")) {
-      if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
-        const CodeGenHwModes &HWM = Target.getHwModes();
-        EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-        for (const auto [Mode, EncodingDef] : EBM) {
-          Modes.try_emplace(Mode, "_" + HWM.getMode(Mode).Name.str());
-          const RecordVal *RV = EncodingDef->getValue("Inst");
-          const DagInit *DI = cast<DagInit>(RV->getValue());
-          VarLenInsts[R].try_emplace(Mode, VarLenInst(DI, RV));
-        }
-        continue;
+    if (const Record *RV = R->getValueAsOptionalDef("EncodingInfos")) {
+      const CodeGenHwModes &HWM = Target.getHwModes();
+      EncodingInfoByHwMode EBM(RV, HWM);
+      for (const auto [Mode, EncodingDef] : EBM) {
+        Modes.try_emplace(Mode, "_" + HWM.getMode(Mode).Name.str());
+        const RecordVal *RV = EncodingDef->getValue("Inst");
+        const DagInit *DI = cast<DagInit>(RV->getValue());
+        VarLenInsts[R].try_emplace(Mode, VarLenInst(DI, RV));
       }
+      continue;
     }
     const RecordVal *RV = R->getValue("Inst");
     const DagInit *DI = cast<DagInit>(RV->getValue());
@@ -449,7 +447,7 @@ VarLenCodeEmitterGen::getInstructionCases(const Record *R,
 std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
     const Record *R, AltEncodingTy Mode, const VarLenInst &VLI,
     const CodeGenTarget &Target, int Indent) {
-  CodeGenInstruction &CGI = Target.getInstruction(R);
+  const CodeGenInstruction &CGI = Target.getInstruction(R);
 
   std::string Case;
   raw_string_ostream SS(Case);
@@ -476,7 +474,7 @@ std::string VarLenCodeEmitterGen::getInstructionCaseForEncoding(
         LoBit = static_cast<unsigned>(cast<IntInit>(DV->getArg(2))->getValue());
       }
 
-      auto OpIdx = CGI.Operands.ParseOperandName(OperandName);
+      auto OpIdx = CGI.Operands.parseOperandName(OperandName);
       unsigned FlatOpIdx = CGI.Operands.getFlattenedOperandNumber(OpIdx);
       StringRef CustomEncoder =
           CGI.Operands[OpIdx.first].EncoderMethodNames[OpIdx.second];
diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 21c611e364a8d..4f6eeb3922f9a 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -34,6 +34,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/FormattedStream.h"
+#include "llvm/Support/KnownBits.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
@@ -101,6 +102,22 @@ STATISTIC(NumEncodingsOmitted, "Number of encodings omitted");
 
 static unsigned getNumToSkipInBytes() { return LargeTable ? 3 : 2; }
 
+/// Similar to KnownBits::print(), but allows you to specify a character to use
+/// to print unknown bits.
+static void printKnownBits(raw_ostream &OS, const KnownBits &Bits,
+                           char Unknown) {
+  for (unsigned I = Bits.getBitWidth(); I--;) {
+    if (Bits.Zero[I] && Bits.One[I])
+      OS << '!';
+    else if (Bits.Zero[I])
+      OS << '0';
+    else if (Bits.One[I])
+      OS << '1';
+    else
+      OS << Unknown;
+  }
+}
+
 namespace {
 
 struct EncodingField {
@@ -129,6 +146,102 @@ struct OperandInfo {
   const_iterator end() const { return Fields.end(); }
 };
 
+/// Represents a parsed InstructionEncoding record or a record derived from it.
+class InstructionEncoding {
+  /// The Record this encoding originates from.
+  const Record *EncodingDef;
+
+  /// The instruction this encoding is for.
+  const CodeGenInstruction *Inst;
+
+  /// The name of this encoding (for debugging purposes).
+  std::string Name;
+
+  /// Known bits of this encoding. This is the value of the `Inst` field
+  /// with any variable references replaced with '?'.
+  KnownBits InstBits;
+
+  /// Mask of bits that should be considered unknown during decoding.
+  /// This is the value of the `SoftFail` field.
+  APInt SoftFailBits;
+
+  /// The name of the function to use for decoding. May be an empty string,
+  /// meaning the decoder is generated.
+  StringRef DecoderMethod;
+
+  /// Whether the custom decoding function always succeeds. If a custom decoder
+  /// function is specified, the value is taken from the target description,
+  /// otherwise it is inferred.
+  bool HasCompleteDecoder;
+
+  /// Information about the operands' contribution to this encoding.
+  SmallVector<OperandInfo, 16> Operands;
+
+public:
+  InstructionEncoding(const Record *EncodingDef,
+                      const CodeGenInstruction *Inst);
+
+  /// Returns the Record this encoding originates from.
+  const Record *getRecord() const { return EncodingDef; }
+
+  /// Returns the instruction this encoding is for.
+  const CodeGenInstruction *getInstruction() const { return Inst; }
+
+  /// Returns the name of this encoding, for debugging purposes.
+  StringRef getName() const { return Name; }
+
+  /// Returns the size of this encoding, in bits.
+  unsigned getBitWidth() const { return InstBits.getBitWidth(); }
+
+  /// Returns the known bits of this encoding.
+  const KnownBits &getInstBits() const { return InstBits; }
+
+  /// Returns a mask of bits that should be considered unknown during decoding.
+  const APInt &getSoftFailBits() const { return SoftFailBits; }
+
+  /// Returns the known bits of this encoding that must match for
+  /// successful decoding.
+  KnownBits getMandatoryBits() const {
+    KnownBits EncodingBits = InstBits;
+    // Mark all bits that are allowed to change according to SoftFail mask
+    // as unknown.
+    EncodingBits.Zero &= ~SoftFailBits;
+    EncodingBits.One &= ~SoftFailBits;
+    return EncodingBits;
+  }
+
+  /// Returns the name of the function to use for decoding, or an empty string
+  /// if the decoder is generated.
+  StringRef getDecoderMethod() const { return DecoderMethod; }
+
+  /// Returns whether the decoder (either generated or specified by the user)
+  /// always succeeds.
+  bool hasCompleteDecoder() const { return HasCompleteDecoder; }
+
+  /// Returns information about the operands' contribution to this encoding.
+  ArrayRef<OperandInfo> getOperands() const { return Operands; }
+
+private:
+  void parseVarLenEncoding(const VarLenInst &VLI);
+  void parseFixedLenEncoding(const BitsInit &RecordInstBits);
+
+  void parseVarLenOperands(const VarLenInst &VLI);
+  void parseFixedLenOperands(const BitsInit &Bits);
+};
+
+/// Sorting predicate to sort encoding IDs by encoding width.
+class LessEncodingIDByWidth {
+  ArrayRef<InstructionEncoding> Encodings;
+
+public:
+  explicit LessEncodingIDByWidth(ArrayRef<InstructionEncoding> Encodings)
+      : Encodings(Encodings) {}
+
+  bool operator()(unsigned ID1, unsigned ID2) const {
+    return Encodings[ID1].getBitWidth() < Encodings[ID2].getBitWidth();
+  }
+};
+
 typedef std::vector<uint32_t> FixupList;
 typedef std::vector<FixupList> FixupScopeList;
 typedef SmallSetVector<CachedHashString, 16> PredicateSet;
@@ -139,7 +252,6 @@ class DecoderTable {
   DecoderTable() { Data.reserve(16384); }
 
   void clear() { Data.clear(); }
-  void push_back(uint8_t Item) { Data.push_back(Item); }
   size_t size() const { return Data.size(); }
   const uint8_t *data() const { return Data.data(); }
 
@@ -147,7 +259,16 @@ class DecoderTable {
   const_iterator begin() const { return Data.begin(); }
   const_iterator end() const { return Data.end(); }
 
-  // Insert a ULEB128 encoded value into the table.
+  /// Inserts a state machine opcode into the table.
+  void insertOpcode(MCD::DecoderOps Opcode) { Data.push_back(Opcode); }
+
+  /// Inserts a uint8 encoded value into the table.
+  void insertUInt8(unsigned Value) {
+    assert(isUInt<8>(Value));
+    Data.push_back(Value);
+  }
+
+  /// Inserts a ULEB128 encoded value into the table.
   void insertULEB128(uint64_t Value) {
     // Encode and emit the value to filter against.
     uint8_t Buffer[16];
@@ -192,43 +313,42 @@ struct DecoderTableInfo {
   DecoderSet Decoders;
 
   bool isOutermostScope() const { return FixupStack.size() == 1; }
-};
-
-struct EncodingAndInst {
-  const Record *EncodingDef;
-  const CodeGenInstruction *Inst;
-  StringRef HwModeName;
-
-  EncodingAndInst(const Record *EncodingDef, const CodeGenInstruction *Inst,
-                  StringRef HwModeName = "")
-      : EncodingDef(EncodingDef), Inst(Inst), HwModeName(HwModeName) {}
-};
 
-struct EncodingIDAndOpcode {
-  unsigned EncodingID;
-  unsigned Opcode;
+  void pushScope() { FixupStack.emplace_back(); }
 
-  EncodingIDAndOpcode() : EncodingID(0), Opcode(0) {}
-  EncodingIDAndOpcode(unsigned EncodingID, unsigned Opcode)
-      : EncodingID(EncodingID), Opcode(Opcode) {}
+  void popScope() {
+    // Resolve any remaining fixups in the current scope before popping it.
+    // All fixups resolve to the current location.
+    uint32_t DestIdx = Table.size();
+    for (uint32_t FixupIdx : FixupStack.back())
+      Table.patchNumToSkip(FixupIdx, DestIdx);
+    FixupStack.pop_back();
+  }
 };
 
-using EncodingIDsVec = std::vector<EncodingIDAndOpcode>;
-using NamespacesHwModesMap = std::map<std::string, std::set<StringRef>>;
+using NamespacesHwModesMap = std::map<std::string, std::set<unsigned>>;
 
 class DecoderEmitter {
   const RecordKeeper &RK;
-  std::vector<EncodingAndInst> NumberedEncodings;
+  CodeGenTarget Target;
+  const CodeGenHwModes &CGH;
+
+  /// All parsed encodings.
+  std::vector<InstructionEncoding> Encodings;
+
+  /// Encodings IDs for each HwMode. An ID is an index into Encodings.
+  SmallDenseMap<unsigned, std::vector<unsigned>> EncodingIDsByHwMode;
 
 public:
-  DecoderEmitter(const RecordKeeper &R, StringRef PredicateNamespace)
-      : RK(R), Target(R), PredicateNamespace(PredicateNamespace) {}
+  DecoderEmitter(const RecordKeeper &RK, StringRef PredicateNamespace);
+
+  const CodeGenTarget &getTarget() const { return Target; }
 
   // Emit the decoder state machine table. Returns a mask of MCD decoder ops
   // that were emitted.
   unsigned emitTable(formatted_raw_ostream &OS, DecoderTable &Table,
-                     unsigned BitWidth, StringRef Namespace,
-                     const EncodingIDsVec &EncodingIDs) const;
+                     StringRef Namespace, unsigned HwModeID, unsigned BitWidth,
+                     ArrayRef<unsigned> EncodingIDs) const;
   void emitInstrLenTable(formatted_raw_ostream &OS,
                          ArrayRef<unsigned> InstrLen) const;
   void emitPredicateFunction(formatted_raw_ostream &OS,
@@ -237,113 +357,28 @@ class DecoderEmitter {
                            DecoderSet &Decoders) const;
 
   // run - Output the code emitter
-  void run(raw_ostream &o);
+  void run(raw_ostream &o) const;
 
 private:
-  CodeGenTarget Target;
-
-public:
-  StringRef PredicateNamespace;
-};
+  void collectHwModesReferencedForEncodings(
+      std::vector<unsigned> &HwModeIDs,
+      NamespacesHwModesMap &NamespacesWithHwModes) const;
 
-// The set (BIT_TRUE, BIT_FALSE, BIT_UNSET) represents a ternary logic system
-// for a bit value.
-//
-// BIT_UNFILTERED is used as the init value for a filter position.  It is used
-// only for filter processings.
-struct BitValue {
-  enum bit_value_t : uint8_t {
-    BIT_FALSE,     // '0'
-    BIT_TRUE,      // '1'
-    BIT_UNSET,     // '?', printed as '_'
-    BIT_UNFILTERED // unfiltered, printed as '.'
-  };
-
-  BitValue(bit_value_t V) : V(V) {}
-  explicit BitValue(const Init *Init) {
-    if (const auto *Bit = dyn_cast<BitInit>(Init))
-      V = Bit->getValue() ? BIT_TRUE : BIT_FALSE;
-    else
-      V = BIT_UNSET;
-  }
-  BitValue(const BitsInit &Bits, unsigned Idx) : BitValue(Bits.getBit(Idx)) {}
-
-  bool isSet() const { return V == BIT_TRUE || V == BIT_FALSE; }
-  bool isUnset() const { return V == BIT_UNSET; }
-  std::optional<uint64_t> getValue() const {
-    if (isSet())
-      return static_cast<uint64_t>(V);
-    return std::nullopt;
-  }
-
-  // For printing a bit value.
-  operator StringRef() const {
-    switch (V) {
-    case BIT_FALSE:
-      return "0";
-    case BIT_TRUE:
-      return "1";
-    case BIT_UNSET:
-      return "_";
-    case BIT_UNFILTERED:
-      return ".";
-    }
-    llvm_unreachable("Unknow bit value");
-  }
+  void
+  handleHwModesUnrelatedEncodings(unsigned EncodingID,
+                                  ArrayRef<unsigned> HwModeIDs,
+                                  NamespacesHwModesMap &NamespacesWithHwModes);
 
-  bool operator==(bit_value_t Other) const { return Other == V; }
-  bool operator!=(bit_value_t Other) const { return Other != V; }
+  void parseInstructionEncodings();
 
-private:
-  bit_value_t V;
+public:
+  StringRef PredicateNamespace;
 };
 
 } // end anonymous namespace
 
-static raw_ostream &operator<<(raw_ostream &OS, const EncodingAndInst &Value) {
-  if (Value.EncodingDef != Value.Inst->TheDef)
-    OS << Value.EncodingDef->getName() << ":";
-  OS << Value.Inst->TheDef->getName();
-  return OS;
-}
-
-// Prints the bit value for each position.
-static void dumpBits(raw_ostream &OS, const BitsInit &Bits) {
-  for (const Init *Bit : reverse(Bits.getBits()))
-    OS << BitValue(Bit);
-}
-
-static const BitsInit &getBitsField(const Record &Def, StringRef FieldName) {
-  const RecordVal *RV = Def.getValue(FieldName);
-  if (const BitsInit *Bits = dyn_cast<BitsInit>(RV->getValue()))
-    return *Bits;
-
-  // Handle variable length instructions.
-  VarLenInst VLI(cast<DagInit>(RV->getValue()), RV);
-  SmallVector<const Init *, 16> Bits;
-
-  for (const auto &SI : VLI) {
-    if (const BitsInit *BI = dyn_cast<BitsInit>(SI.Value))
-      llvm::append_range(Bits, BI->getBits());
-    else if (const BitInit *BI = dyn_cast<BitInit>(SI.Value))
-      Bits.push_back(BI);
-    else
-      Bits.append(SI.BitWidth, UnsetInit::get(Def.getRecords()));
-  }
-
-  return *BitsInit::get(Def.getRecords(), Bits);
-}
-
-// Representation of the instruction to work on.
-typedef std::vector<BitValue> insn_t;
-
 namespace {
 
-static constexpr uint64_t NO_FIXED_SEGMENTS_SENTINEL =
-    std::numeric_limits<uint64_t>::max();
-
-class FilterChooser;
-
 /// Filter - Filter works with FilterChooser to produce the decoding tree for
 /// the ISA.
 ///
@@ -360,16 +395,16 @@ class FilterChooser;
 ///
 /// An example of a conflict is
 ///
-/// Conflict:
-///                     111101000.00........00010000....
-///                     111101000.00........0001........
-///                     1111010...00........0001........
-///                     1111010...00....................
-///                     1111010.........................
-///                     1111............................
-///                     ................................
-///     VST4q8a         111101000_00________00010000____
-///     VST4q8b         111101000_00________00010000____
+/// Decoding Conflict:
+///     ................................
+///     1111............................
+///     1111010.........................
+///     1111010...00....................
+///     1111010...00........0001........
+///     111101000.00........0001........
+///     111101000.00........00010000....
+///     111101000_00________00010000____  VST4q8a
+///     111101000_00________00010000____  VST4q8b
 ///
 /// The Debug output shows the path that the decoding tree follows to reach the
 /// the conclusion that there is a conflict.  VST4q8a is a vst4 to double-spaced
@@ -380,61 +415,28 @@ class FilterChooser;
 /// decoder could try to decode the even/odd register numbering and assign to
 /// VST4q8a or VST4q8b, but for the time being, the decoder chooses the "a"
 /// version and return the Opcode since the two have the same Asm format string.
-class Filter {
-protected:
-  const FilterChooser &Owner; // FilterChooser who owns this filter
+struct Filter {
   unsigned StartBit; // the starting bit position
   unsigned NumBits;  // number of bits to filter
-  bool Mixed;        // a mixed region contains both set and unset bits
 
   // Map of well-known segment value to the set of uid's with that value.
-  std::map<uint64_t, std::vector<EncodingIDAndOpcode>> FilteredInstructions;
+  std::map<uint64_t, std::vector<unsigned>> FilteredIDs;
 
   // Set of uid's with non-constant segment values.
-  std::vector<EncodingIDAndOpcode> VariableInstructions;
-
-  // Map of well-known segment value to its delegate.
-  std::map<uint64_t, std::unique_ptr<const FilterChooser>> FilterChooserMap;
+  std::vector<unsigned> VariableIDs;
 
-  // Number of instructions which fall under FilteredInstructions category.
-  unsigned NumFiltered;
+  Filter(ArrayRef<InstructionEncoding> Encodings,
+         ArrayRef<unsigned> EncodingIDs, unsigned StartBit, unsigned NumBits);
 
-  // Keeps track of the last opcode in the filtered bucket.
-  EncodingIDAndOpcode LastOpcFiltered;
-
-public:
-  Filter(Filter &&f);
-  Filter(const FilterChooser &owner, unsigned startBit, unsigned numBits,
-         bool mixed);
-
-  ~Filter() = default;
-
-  unsigned getNumFiltered() const { return NumFiltered; }
-
-  EncodingIDAndOpcode getSingletonOpc() const {
-    assert(NumFiltered == 1);
-    return LastOpcFiltered;
+  bool hasSingleFilteredID() const {
+    return FilteredIDs.size() == 1 && FilteredIDs.begin()->second.size() == 1;
   }
 
-  // Return the filter chooser for the group of instructions without constant
-  // segment values.
-  const FilterChooser &getVariableFC() const {
-    assert(NumFiltered == 1 && FilterChooserMap.size() == 1);
-    return *(FilterChooserMap.find(NO_FIXED_SEGMENTS_SENTINEL)->second);
+  unsigned getSingletonEncodingID() const {
+    assert(hasSingleFilteredID());
+    return FilteredIDs.begin()->second.front();
   }
 
-  // Divides the decoding task into sub tasks and delegates them to the
-  // inferior FilterChooser's.
-  //
-  // A special case arises when there's only one entry in the filtered
-  // instructions.  In order to unambiguously decode the singleton, we need to
-  // match the remaining undecoded encoding bits against the singleton.
-  void recurse();
-
-  // Emit table entries to decode instructions given a segment or segments of
-  // bits.
-  void emitTableEntry(DecoderTableInfo &TableInfo) const;
-
   // Returns the number of fanout produced by the filter.  More fanout implies
   // the filter distinguishes more categories of instructions.
   unsigned usefulness() const;
@@ -467,39 +469,51 @@ enum bitAttr_t {
 /// decide what further remaining bits to look at.
 
 class FilterChooser {
-protected:
-  friend class Filter;
-
-  // Vector of codegen instructions to choose our filter.
-  ArrayRef<EncodingAndInst> AllInstructions;
+  // Vector of encodings to choose our filter.
+  ArrayRef<InstructionEncoding> Encodings;
 
-  // Vector of uid's for this filter chooser to work on.
-  // The first member of the pair is the opcode id being decoded, the second is
-  // the opcode id that should be emitted.
-  ArrayRef<EncodingIDAndOpcode> Opcodes;
-
-  // Lookup table for the operand decoding of instructions.
-  const std::map<unsigned, std::vector<OperandInfo>> &Operands;
-
-  // Vector of candidate filters.
-  std::vector<Filter> Filters;
+  /// Encoding IDs for this filter chooser to work on.
+  /// Sorted by non-decreasing encoding width.
+  SmallVector<unsigned, 0> EncodingIDs;
 
   // Array of bit values passed down from our parent.
-  // Set to all BIT_UNFILTERED's for Parent == NULL.
-  std::vector<BitValue> FilterBitValues;
+  // Set to all unknown for Parent == nullptr.
+  KnownBits FilterBits;
 
   // Links to the FilterChooser above us in the decoding tree.
   const FilterChooser *Parent;
 
-  // Index of the best filter from Filters.
-  int BestIndex;
-
-  // Width of instructions
-  unsigned BitWidth;
+  /// Some targets (ARM) specify more encoding bits in Inst that Size allows.
+  /// This field allows us to ignore the extra bits.
+  unsigned MaxFilterWidth;
 
   // Parent emitter
   const DecoderEmitter *Emitter;
 
+  /// If the selected filter matches multiple encodings, then this is the
+  /// starting position and the width of the filtered range.
+  unsigned StartBit;
+  unsigned NumBits;
+
+  /// If the selected filter matches multiple encodings, and there is
+  /// *exactly one* encoding in which all bits are known in the filtered range,
+  /// then this is the ID of that encoding.
+  /// Also used when there is only one encoding.
+  std::optional<unsigned> SingletonEncodingID;
+
+  /// If the selected filter matches multiple encodings, and there is
+  /// *at least one* encoding in which all bits are known in the filtered range,
+  /// then this is the FilterChooser created for the subset of encodings that
+  /// contain some unknown bits in the filtered range.
+  std::unique_ptr<const FilterChooser> VariableFC;
+
+  /// If the selected filter matches multiple encodings, and there is
+  /// *more than one* encoding in which all bits are known in the filtered
+  /// range, then this is a map of field values to FilterChoosers created for
+  /// the subset of encodings sharing that field value.
+  /// The "field value" here refers to the encoding bits in the filtered range.
+  std::map<uint64_t, std::unique_ptr<const FilterChooser>> FilterChooserMap;
+
   struct Island {
     unsigned StartBit;
     unsigned NumBits;
@@ -507,124 +521,104 @@ class FilterChooser {
   };
 
 public:
-  FilterChooser(ArrayRef<EncodingAndInst> Insts,
-                ArrayRef<EncodingIDAndOpcode> IDs,
-                const std::map<unsigned, std::vector<OperandInfo>> &Ops,
-                unsigned BW, const DecoderEmitter *E)
-      : AllInstructions(Insts), Opcodes(IDs), Operands(Ops),
-        FilterBitValues(BW, BitValue::BIT_UNFILTERED), Parent(nullptr),
-        BestIndex(-1), BitWidth(BW), Emitter(E) {
+  /// Constructs a top-level filter chooser.
+  FilterChooser(ArrayRef<InstructionEncoding> Encodings,
+                ArrayRef<unsigned> EncodingIDs, unsigned MaxFilterWidth,
+                const DecoderEmitter *E)
+      : Encodings(Encodings), EncodingIDs(EncodingIDs), Parent(nullptr),
+        MaxFilterWidth(MaxFilterWidth), Emitter(E) {
+    // Sort encoding IDs once.
+    stable_sort(this->EncodingIDs, LessEncodingIDByWidth(Encodings));
+    // Filter width is the width of the smallest encoding.
+    unsigned FilterWidth = Encodings[this->EncodingIDs.front()].getBitWidth();
+    // Cap it as necessary.
+    FilterWidth = std::min(FilterWidth, MaxFilterWidth);
+    FilterBits = KnownBits(FilterWidth);
     doFilter();
   }
 
-  FilterChooser(ArrayRef<EncodingAndInst> Insts,
-                ArrayRef<EncodingIDAndOpcode> IDs,
-                const std::map<unsigned, std::vector<OperandInfo>> &Ops,
-                const std::vector<BitValue> &ParentFilterBitValues,
-                const FilterChooser &parent)
-      : AllInstructions(Insts), Opcodes(IDs), Operands(Ops),
-        FilterBitValues(ParentFilterBitValues), Parent(&parent), BestIndex(-1),
-        BitWidth(parent.BitWidth), Emitter(parent.Emitter) {
+  /// Constructs an inferior filter chooser.
+  FilterChooser(ArrayRef<InstructionEncoding> Encodings,
+                ArrayRef<unsigned> EncodingIDs, const KnownBits &FilterBits,
+                const FilterChooser &Parent)
+      : Encodings(Encodings), EncodingIDs(EncodingIDs), Parent(&Parent),
+        MaxFilterWidth(Parent.MaxFilterWidth), Emitter(Parent.Emitter) {
+    // Inferior filter choosers are created from sorted array of encoding IDs.
+    assert(is_sorted(EncodingIDs, LessEncodingIDByWidth(Encodings)));
+    assert(!FilterBits.hasConflict() && "Broken filter");
+    // Filter width is the width of the smallest encoding.
+    unsigned FilterWidth = Encodings[EncodingIDs.front()].getBitWidth();
+    // Cap it as necessary.
+    FilterWidth = std::min(FilterWidth, MaxFilterWidth);
+    this->FilterBits = FilterBits.anyext(FilterWidth);
     doFilter();
   }
 
   FilterChooser(const FilterChooser &) = delete;
   void operator=(const FilterChooser &) = delete;
 
-  unsigned getBitWidth() const { return BitWidth; }
-
-protected:
-  // Populates the insn given the uid.
-  void insnWithID(insn_t &Insn, unsigned Opcode) const {
-    const Record *EncodingDef = AllInstructions[Opcode].EncodingDef;
-    const BitsInit &Bits = getBitsField(*EncodingDef, "Inst");
-    Insn.resize(std::max(BitWidth, Bits.getNumBits()), BitValue::BIT_UNSET);
-    // We may have a SoftFail bitmask, which specifies a mask where an encoding
-    // may differ from the value in "Inst" and yet still be valid, but the
-    // disassembler should return SoftFail instead of Success.
-    //
-    // This is used for marking UNPREDICTABLE instructions in the ARM world.
-    const RecordVal *RV = EncodingDef->getValue("SoftFail");
-    const BitsInit *SFBits = RV ? dyn_cast<BitsInit>(RV->getValue()) : nullptr;
-    for (unsigned i = 0; i < Bits.getNumBits(); ++i) {
-      if (SFBits && BitValue(*SFBits, i) == BitValue::BIT_TRUE)
-        Insn[i] = BitValue::BIT_UNSET;
-      else
-        Insn[i] = BitValue(Bits, i);
-    }
+  /// Returns the width of the largest encoding.
+  unsigned getMaxEncodingWidth() const {
+    // The last encoding ID is the ID of an encoding with the largest width.
+    return Encodings[EncodingIDs.back()].getBitWidth();
   }
 
-  // Populates the field of the insn given the start position and the number of
-  // consecutive bits to scan for.
-  //
-  // Returns a pair of values (indicator, field), where the indicator is false
-  // if there exists any uninitialized bit value in the range and true if all
-  // bits are well-known. The second value is the potentially populated field.
-  std::pair<bool, uint64_t> fieldFromInsn(const insn_t &Insn, unsigned StartBit,
-                                          unsigned NumBits) const;
-
-  /// dumpFilterArray - dumpFilterArray prints out debugging info for the given
-  /// filter array as a series of chars.
-  void dumpFilterArray(raw_ostream &OS, ArrayRef<BitValue> Filter) const;
+private:
+  /// Applies the given filter to the set of encodings this FilterChooser
+  /// works with, creating inferior FilterChoosers as necessary.
+  void applyFilter(const Filter &F);
 
   /// dumpStack - dumpStack traverses the filter chooser chain and calls
   /// dumpFilterArray on each filter chooser up to the top level one.
-  void dumpStack(raw_ostream &OS, const char *prefix) const;
-
-  Filter &bestFilter() {
-    assert(BestIndex != -1 && "BestIndex not set");
-    return Filters[BestIndex];
-  }
+  void dumpStack(raw_ostream &OS, indent Indent, unsigned PadToWidth) const;
 
-  bool PositionFiltered(unsigned Idx) const {
-    return FilterBitValues[Idx].isSet();
+  bool isPositionFiltered(unsigned Idx) const {
+    return FilterBits.Zero[Idx] || FilterBits.One[Idx];
   }
 
   // Calculates the island(s) needed to decode the instruction.
   // This returns a list of undecoded bits of an instructions, for example,
   // Inst{20} = 1 && Inst{3-0} == 0b1111 represents two islands of yet-to-be
   // decoded bits in order to verify that the instruction matches the Opcode.
-  unsigned getIslands(std::vector<Island> &Islands, const insn_t &Insn) const;
+  std::vector<Island> getIslands(const KnownBits &EncodingBits) const;
 
   // Emits code to check the Predicates member of an instruction are true.
   // Returns true if predicate matches were emitted, false otherwise.
-  bool emitPredicateMatch(raw_ostream &OS, unsigned Opc) const;
+  bool emitPredicateMatch(raw_ostream &OS, unsigned EncodingID) const;
   bool emitPredicateMatchAux(const Init &Val, bool ParenIfBinOp,
                              raw_ostream &OS) const;
 
-  bool doesOpcodeNeedPredicate(unsigned Opc) const;
+  bool doesOpcodeNeedPredicate(unsigned EncodingID) const;
   unsigned getPredicateIndex(DecoderTableInfo &TableInfo, StringRef P) const;
-  void emitPredicateTableEntry(DecoderTableInfo &TableInfo, unsigned Opc) const;
+  void emitPredicateTableEntry(DecoderTableInfo &TableInfo,
+                               unsigned EncodingID) const;
 
-  void emitSoftFailTableEntry(DecoderTableInfo &TableInfo, unsigned Opc) const;
+  void emitSoftFailTableEntry(DecoderTableInfo &TableInfo,
+                              unsigned EncodingID) const;
 
   // Emits table entries to decode the singleton.
   void emitSingletonTableEntry(DecoderTableInfo &TableInfo,
-                               EncodingIDAndOpcode Opc) const;
+                               unsigned EncodingID) const;
 
-  // Emits code to decode the singleton, and then to decode the rest.
-  void emitSingletonTableEntry(DecoderTableInfo &TableInfo,
-                               const Filter &Best) const;
-
-  bool emitBinaryParser(raw_ostream &OS, indent Indent,
+  void emitBinaryParser(raw_ostream &OS, indent Indent,
                         const OperandInfo &OpInfo) const;
 
-  bool emitDecoder(raw_ostream &OS, indent Indent, unsigned Opc) const;
-  std::pair<unsigned, bool> getDecoderIndex(DecoderSet &Decoders,
-                                            unsigned Opc) const;
-
-  // Assign a single filter and run with it.
-  void runSingleFilter(unsigned startBit, unsigned numBit, bool mixed);
+  void emitDecoder(raw_ostream &OS, indent Indent, unsigned EncodingID) const;
+  unsigned getDecoderIndex(DecoderSet &Decoders, unsigned EncodingID) const;
 
   // reportRegion is a helper function for filterProcessor to mark a region as
   // eligible for use as a filter region.
-  void reportRegion(bitAttr_t RA, unsigned StartBit, unsigned BitIndex,
-                    bool AllowMixed);
+  void reportRegion(std::vector<std::unique_ptr<Filter>> &Filters, bitAttr_t RA,
+                    unsigned StartBit, unsigned BitIndex,
+                    bool AllowMixed) const;
 
-  // FilterProcessor scans the well-known encoding bits of the instructions and
-  // builds up a list of candidate filters.  It chooses the best filter and
-  // recursively descends down the decoding tree.
-  bool filterProcessor(bool AllowMixed, bool Greedy = true);
+  /// Scans the well-known encoding bits of the encodings and, builds up a list
+  /// of candidate filters, and then returns the best one, if any.
+  std::unique_ptr<Filter> findBestFilter(ArrayRef<bitAttr_t> BitAttrs,
+                                         bool AllowMixed,
+                                         bool Greedy = true) const;
+
+  std::unique_ptr<Filter> findBestFilter() const;
 
   // Decides on the best configuration of filter(s) to use in order to decode
   // the instructions.  A conflict of instructions may occur, in which case we
@@ -632,9 +626,10 @@ class FilterChooser {
   void doFilter();
 
 public:
-  // emitTableEntries - Emit state machine entries to decode our share of
-  // instructions.
+  /// Emits state machine entries to decode our share of instructions.
   void emitTableEntries(DecoderTableInfo &TableInfo) const;
+
+  void dump() const;
 };
 
 } // end anonymous namespace
@@ -645,183 +640,71 @@ class FilterChooser {
 //                       //
 ///////////////////////////
 
-Filter::Filter(Filter &&f)
-    : Owner(f.Owner), StartBit(f.StartBit), NumBits(f.NumBits), Mixed(f.Mixed),
-      FilteredInstructions(std::move(f.FilteredInstructions)),
-      VariableInstructions(std::move(f.VariableInstructions)),
-      FilterChooserMap(std::move(f.FilterChooserMap)),
-      NumFiltered(f.NumFiltered), LastOpcFiltered(f.LastOpcFiltered) {}
-
-Filter::Filter(const FilterChooser &owner, unsigned startBit, unsigned numBits,
-               bool mixed)
-    : Owner(owner), StartBit(startBit), NumBits(numBits), Mixed(mixed) {
-  assert(StartBit + NumBits - 1 < Owner.BitWidth);
-
-  NumFiltered = 0;
-  LastOpcFiltered = {0, 0};
-
-  for (const auto &OpcPair : Owner.Opcodes) {
-    insn_t Insn;
-
-    // Populates the insn given the uid.
-    Owner.insnWithID(Insn, OpcPair.EncodingID);
+Filter::Filter(ArrayRef<InstructionEncoding> Encodings,
+               ArrayRef<unsigned> EncodingIDs, unsigned StartBit,
+               unsigned NumBits)
+    : StartBit(StartBit), NumBits(NumBits) {
+  for (unsigned EncodingID : EncodingIDs) {
+    const InstructionEncoding &Encoding = Encodings[EncodingID];
+    KnownBits EncodingBits = Encoding.getMandatoryBits();
 
     // Scans the segment for possibly well-specified encoding bits.
-    auto [Ok, Field] = Owner.fieldFromInsn(Insn, StartBit, NumBits);
+    KnownBits FieldBits = EncodingBits.extractBits(NumBits, StartBit);
 
-    if (Ok) {
+    if (FieldBits.isConstant()) {
       // The encoding bits are well-known.  Lets add the uid of the
       // instruction into the bucket keyed off the constant field value.
-      LastOpcFiltered = OpcPair;
-      FilteredInstructions[Field].push_back(LastOpcFiltered);
-      ++NumFiltered;
+      FilteredIDs[FieldBits.getConstant().getZExtValue()].push_back(EncodingID);
     } else {
       // Some of the encoding bit(s) are unspecified.  This contributes to
       // one additional member of "Variable" instructions.
-      VariableInstructions.push_back(OpcPair);
+      VariableIDs.push_back(EncodingID);
     }
   }
 
-  assert((FilteredInstructions.size() + VariableInstructions.size() > 0) &&
+  assert((FilteredIDs.size() + VariableIDs.size() > 0) &&
          "Filter returns no instruction categories");
 }
 
-// Divides the decoding task into sub tasks and delegates them to the
-// inferior FilterChooser's.
-//
-// A special case arises when there's only one entry in the filtered
-// instructions.  In order to unambiguously decode the singleton, we need to
-// match the remaining undecoded encoding bits against the singleton.
-void Filter::recurse() {
-  // Starts by inheriting our parent filter chooser's filter bit values.
-  std::vector<BitValue> BitValueArray(Owner.FilterBitValues);
-
-  if (!VariableInstructions.empty()) {
-    // Conservatively marks each segment position as BIT_UNSET.
-    for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex)
-      BitValueArray[StartBit + bitIndex] = BitValue::BIT_UNSET;
+void FilterChooser::applyFilter(const Filter &F) {
+  StartBit = F.StartBit;
+  NumBits = F.NumBits;
+  assert(FilterBits.extractBits(NumBits, StartBit).isUnknown());
 
+  if (!F.VariableIDs.empty()) {
     // Delegates to an inferior filter chooser for further processing on this
     // group of instructions whose segment values are variable.
-    FilterChooserMap.try_emplace(
-        NO_FIXED_SEGMENTS_SENTINEL,
-        std::make_unique<FilterChooser>(Owner.AllInstructions,
-                                        VariableInstructions, Owner.Operands,
-                                        BitValueArray, Owner));
+    VariableFC = std::make_unique<FilterChooser>(Encodings, F.VariableIDs,
+                                                 FilterBits, *this);
   }
 
   // No need to recurse for a singleton filtered instruction.
   // See also Filter::emit*().
-  if (getNumFiltered() == 1) {
-    assert(FilterChooserMap.size() == 1);
+  if (F.hasSingleFilteredID()) {
+    SingletonEncodingID = F.getSingletonEncodingID();
+    assert(VariableFC && "Shouldn't have created a filter for one encoding!");
     return;
   }
 
   // Otherwise, create sub choosers.
-  for (const auto &Inst : FilteredInstructions) {
-    // Marks all the segment positions with either BIT_TRUE or BIT_FALSE.
-    for (unsigned bitIndex = 0; bitIndex < NumBits; ++bitIndex)
-      BitValueArray[StartBit + bitIndex] = Inst.first & (1ULL << bitIndex)
-                                               ? BitValue::BIT_TRUE
-                                               : BitValue::BIT_FALSE;
+  for (const auto &[FilterVal, InferiorEncodingIDs] : F.FilteredIDs) {
+    // Create a new filter by inserting the field bits into the parent filter.
+    APInt FieldBits(NumBits, FilterVal);
+    KnownBits InferiorFilterBits = FilterBits;
+    InferiorFilterBits.insertBits(KnownBits::makeConstant(FieldBits), StartBit);
 
     // Delegates to an inferior filter chooser for further processing on this
     // category of instructions.
-    FilterChooserMap.try_emplace(
-        Inst.first,
-        std::make_unique<FilterChooser>(Owner.AllInstructions, Inst.second,
-                                        Owner.Operands, BitValueArray, Owner));
-  }
-}
-
-static void resolveTableFixups(DecoderTable &Table, const FixupList &Fixups,
-                               uint32_t DestIdx) {
-  // Any NumToSkip fixups in the current scope can resolve to the
-  // current location.
-  for (uint32_t FixupIdx : Fixups)
-    Table.patchNumToSkip(FixupIdx, DestIdx);
-}
-
-// Emit table entries to decode instructions given a segment or segments
-// of bits.
-void Filter::emitTableEntry(DecoderTableInfo &TableInfo) const {
-  assert(isUInt<8>(NumBits) && "NumBits overflowed uint8 table entry!");
-  TableInfo.Table.push_back(MCD::OPC_ExtractField);
-
-  TableInfo.Table.insertULEB128(StartBit);
-  TableInfo.Table.push_back(NumBits);
-
-  // If the NO_FIXED_SEGMENTS_SENTINEL is present, we need to add a new scope
-  // for this filter. Otherwise, we can skip adding a new scope and any
-  // patching added will automatically be added to the enclosing scope.
-
-  // If NO_FIXED_SEGMENTS_SENTINEL is present, it will be last entry in
-  // FilterChooserMap.
-
-  const uint64_t LastFilter = FilterChooserMap.rbegin()->first;
-  bool HasFallthrough = LastFilter == NO_FIXED_SEGMENTS_SENTINEL;
-  if (HasFallthrough)
-    TableInfo.FixupStack.emplace_back();
-
-  DecoderTable &Table = TableInfo.Table;
-
-  size_t PrevFilter = 0;
-  for (const auto &[FilterVal, Delegate] : FilterChooserMap) {
-    // Field value NO_FIXED_SEGMENTS_SENTINEL implies a non-empty set of
-    // variable instructions. See also recurse().
-    if (FilterVal == NO_FIXED_SEGMENTS_SENTINEL) {
-      // Each scope should always have at least one filter value to check
-      // for.
-      assert(PrevFilter != 0 && "empty filter set!");
-      FixupList &CurScope = TableInfo.FixupStack.back();
-      // Resolve any NumToSkip fixups in the current scope.
-      resolveTableFixups(Table, CurScope, Table.size());
-
-      // Delete the scope we have added here.
-      TableInfo.FixupStack.pop_back();
-
-      PrevFilter = 0; // Don't re-process the filter's fallthrough.
-    } else {
-      // The last filtervalue emitted can be OPC_FilterValue if we are at
-      // outermost scope.
-      const uint8_t DecoderOp =
-          FilterVal == LastFilter && TableInfo.isOutermostScope()
-              ? MCD::OPC_FilterValueOrFail
-              : MCD::OPC_FilterValue;
-      Table.push_back(DecoderOp);
-      Table.insertULEB128(FilterVal);
-      if (DecoderOp == MCD::OPC_FilterValue) {
-        // Reserve space for the NumToSkip entry. We'll backpatch the value
-        // later.
-        PrevFilter = Table.insertNumToSkip();
-      } else {
-        PrevFilter = 0;
-      }
-    }
-
-    // We arrive at a category of instructions with the same segment value.
-    // Now delegate to the sub filter chooser for further decodings.
-    // The case may fallthrough, which happens if the remaining well-known
-    // encoding bits do not match exactly.
-    Delegate->emitTableEntries(TableInfo);
-
-    // Now that we've emitted the body of the handler, update the NumToSkip
-    // of the filter itself to be able to skip forward when false.
-    if (PrevFilter)
-      Table.patchNumToSkip(PrevFilter, Table.size());
+    FilterChooserMap.try_emplace(FilterVal, std::make_unique<FilterChooser>(
+                                                Encodings, InferiorEncodingIDs,
+                                                InferiorFilterBits, *this));
   }
-
-  // If there is no fallthrough and the final filter was not in the outermost
-  // scope, then it must be fixed up according to the enclosing scope rather
-  // than the current position.
-  if (PrevFilter)
-    TableInfo.FixupStack.back().push_back(PrevFilter);
 }
 
 // Returns the number of fanout produced by the filter.  More fanout implies
 // the filter distinguishes more categories of instructions.
 unsigned Filter::usefulness() const {
-  return FilteredInstructions.size() + VariableInstructions.empty();
+  return FilteredIDs.size() + VariableIDs.empty();
 }
 
 //////////////////////////////////
@@ -833,19 +716,23 @@ unsigned Filter::usefulness() const {
 // Emit the decoder state machine table. Returns a mask of MCD decoder ops
 // that were emitted.
 unsigned DecoderEmitter::emitTable(formatted_raw_ostream &OS,
-                                   DecoderTable &Table, unsigned BitWidth,
-                                   StringRef Namespace,
-                                   const EncodingIDsVec &EncodingIDs) const {
+                                   DecoderTable &Table, StringRef Namespace,
+                                   unsigned HwModeID, unsigned BitWidth,
+                                   ArrayRef<unsigned> EncodingIDs) const {
   // We'll need to be able to map from a decoded opcode into the corresponding
   // EncodingID for this specific combination of BitWidth and Namespace. This
-  // is used below to index into NumberedEncodings.
+  // is used below to index into Encodings.
   DenseMap<unsigned, unsigned> OpcodeToEncodingID;
   OpcodeToEncodingID.reserve(EncodingIDs.size());
-  for (const auto &EI : EncodingIDs)
-    OpcodeToEncodingID[EI.Opcode] = EI.EncodingID;
+  for (unsigned EncodingID : EncodingIDs) {
+    const Record *InstDef = Encodings[EncodingID].getInstruction()->TheDef;
+    OpcodeToEncodingID[Target.getInstrIntValue(InstDef)] = EncodingID;
+  }
 
-  OS << "static const uint8_t DecoderTable" << Namespace << BitWidth
-     << "[] = {\n";
+  OS << "static const uint8_t DecoderTable" << Namespace;
+  if (HwModeID != DefaultMode)
+    OS << '_' << Target.getHwModes().getModeName(HwModeID);
+  OS << BitWidth << "[" << Table.size() << "] = {\n";
 
   // Emit ULEB128 encoded value to OS, returning the number of bytes emitted.
   auto emitULEB128 = [](DecoderTable::const_iterator &I,
@@ -989,7 +876,7 @@ unsigned DecoderEmitter::emitTable(formatted_raw_ostream &OS,
       auto EncodingID = EncI->second;
 
       if (!IsTry) {
-        OS << "// Opcode: " << NumberedEncodings[EncodingID]
+        OS << "// Opcode: " << Encodings[EncodingID].getName()
            << ", DecodeIdx: " << DecodeIdx << '\n';
         break;
       }
@@ -997,7 +884,7 @@ unsigned DecoderEmitter::emitTable(formatted_raw_ostream &OS,
       // Fallthrough for OPC_TryDecode.
       if (!IsFail) {
         uint32_t NumToSkip = emitNumToSkip(I, OS);
-        OS << "// Opcode: " << NumberedEncodings[EncodingID]
+        OS << "// Opcode: " << Encodings[EncodingID].getName()
            << ", DecodeIdx: " << DecodeIdx;
         emitNumToSkipComment(NumToSkip, /*InComment=*/true);
       }
@@ -1028,7 +915,6 @@ unsigned DecoderEmitter::emitTable(formatted_raw_ostream &OS,
       break;
     }
   }
-  OS << "  0\n";
   OS << "};\n\n";
 
   return OpcodeMask;
@@ -1077,7 +963,9 @@ void DecoderEmitter::emitDecoderFunction(formatted_raw_ostream &OS,
     // Emit a function for each case first.
     for (const auto &[Index, Decoder] : enumerate(Decoders)) {
       OS << "template <typename InsnType>\n";
-      OS << "DecodeStatus decodeFn" << Index << "(" << DecodeParams << ") {\n";
+      OS << "static DecodeStatus decodeFn" << Index << "(" << DecodeParams
+         << ") {\n";
+      OS << "  using namespace llvm::MCD;\n";
       OS << "  " << TmpTypeDecl;
       OS << "  [[maybe_unused]] TmpType tmp;\n";
       OS << Decoder;
@@ -1090,24 +978,26 @@ void DecoderEmitter::emitDecoderFunction(formatted_raw_ostream &OS,
   OS << "template <typename InsnType>\n";
   OS << "static DecodeStatus decodeToMCInst(unsigned Idx, " << DecodeParams
      << ") {\n";
+  OS << "  using namespace llvm::MCD;\n";
   OS << "  DecodeComplete = true;\n";
 
   if (UseFnTableInDecodeToMCInst) {
-    // Build a table of function pointers.
+    // Build a table of function pointers
     OS << "  using DecodeFnTy = DecodeStatus (*)(" << DecodeParams << ");\n";
     OS << "  static constexpr DecodeFnTy decodeFnTable[] = {\n";
     for (size_t Index : llvm::seq(Decoders.size()))
       OS << "    decodeFn" << Index << ",\n";
     OS << "  };\n";
     OS << "  if (Idx >= " << Decoders.size() << ")\n";
-    OS << "    llvm_unreachable(\"Invalid index!\");\n";
+    OS << "    llvm_unreachable(\"Invalid decoder index!\");\n";
+
     OS << "  return decodeFnTable[Idx](S, insn, MI, Address, Decoder, "
           "DecodeComplete);\n";
   } else {
     OS << "  " << TmpTypeDecl;
     OS << "  TmpType tmp;\n";
     OS << "  switch (Idx) {\n";
-    OS << "  default: llvm_unreachable(\"Invalid index!\");\n";
+    OS << "  default: llvm_unreachable(\"Invalid decoder index!\");\n";
     for (const auto &[Index, Decoder] : enumerate(Decoders)) {
       OS << "  case " << Index << ":\n";
       OS << Decoder;
@@ -1118,55 +1008,25 @@ void DecoderEmitter::emitDecoderFunction(formatted_raw_ostream &OS,
   OS << "}\n";
 }
 
-// Populates the field of the insn given the start position and the number of
-// consecutive bits to scan for.
-//
-// Returns a pair of values (indicator, field), where the indicator is false
-// if there exists any uninitialized bit value in the range and true if all
-// bits are well-known. The second value is the potentially populated field.
-std::pair<bool, uint64_t> FilterChooser::fieldFromInsn(const insn_t &Insn,
-                                                       unsigned StartBit,
-                                                       unsigned NumBits) const {
-  uint64_t Field = 0;
-
-  for (unsigned i = 0; i < NumBits; ++i) {
-    if (Insn[StartBit + i] == BitValue::BIT_UNSET)
-      return {false, Field};
-
-    if (Insn[StartBit + i] == BitValue::BIT_TRUE)
-      Field = Field | (1ULL << i);
-  }
-
-  return {true, Field};
-}
-
-/// dumpFilterArray - dumpFilterArray prints out debugging info for the given
-/// filter array as a series of chars.
-void FilterChooser::dumpFilterArray(raw_ostream &OS,
-                                    ArrayRef<BitValue> Filter) const {
-  for (unsigned bitIndex = BitWidth; bitIndex > 0; bitIndex--)
-    OS << Filter[bitIndex - 1];
-}
-
 /// dumpStack - dumpStack traverses the filter chooser chain and calls
 /// dumpFilterArray on each filter chooser up to the top level one.
-void FilterChooser::dumpStack(raw_ostream &OS, const char *prefix) const {
-  const FilterChooser *current = this;
-
-  while (current) {
-    OS << prefix;
-    dumpFilterArray(OS, current->FilterBitValues);
-    OS << '\n';
-    current = current->Parent;
-  }
+void FilterChooser::dumpStack(raw_ostream &OS, indent Indent,
+                              unsigned PadToWidth) const {
+  if (Parent)
+    Parent->dumpStack(OS, Indent, PadToWidth);
+  assert(PadToWidth >= FilterBits.getBitWidth());
+  OS << Indent << indent(PadToWidth - FilterBits.getBitWidth());
+  printKnownBits(OS, FilterBits, '.');
+  OS << '\n';
 }
 
 // Calculates the island(s) needed to decode the instruction.
 // This returns a list of undecoded bits of an instructions, for example,
 // Inst{20} = 1 && Inst{3-0} == 0b1111 represents two islands of yet-to-be
 // decoded bits in order to verify that the instruction matches the Opcode.
-unsigned FilterChooser::getIslands(std::vector<Island> &Islands,
-                                   const insn_t &Insn) const {
+std::vector<FilterChooser::Island>
+FilterChooser::getIslands(const KnownBits &EncodingBits) const {
+  std::vector<Island> Islands;
   uint64_t FieldVal;
   unsigned StartBit;
 
@@ -1175,41 +1035,43 @@ unsigned FilterChooser::getIslands(std::vector<Island> &Islands,
   // 2: Island (well-known bit value needed for decoding)
   unsigned State = 0;
 
-  for (unsigned i = 0; i < BitWidth; ++i) {
-    std::optional<uint64_t> Val = Insn[i].getValue();
-    bool Filtered = PositionFiltered(i);
+  unsigned FilterWidth = FilterBits.getBitWidth();
+  for (unsigned i = 0; i != FilterWidth; ++i) {
+    bool IsKnown = EncodingBits.Zero[i] || EncodingBits.One[i];
+    bool Filtered = isPositionFiltered(i);
     switch (State) {
     default:
       llvm_unreachable("Unreachable code!");
     case 0:
     case 1:
-      if (Filtered || !Val) {
+      if (Filtered || !IsKnown) {
         State = 1; // Still in Water
       } else {
         State = 2; // Into the Island
         StartBit = i;
-        FieldVal = *Val;
+        FieldVal = static_cast<uint64_t>(EncodingBits.One[i]);
       }
       break;
     case 2:
-      if (Filtered || !Val) {
+      if (Filtered || !IsKnown) {
         State = 1; // Into the Water
         Islands.push_back({StartBit, i - StartBit, FieldVal});
       } else {
         State = 2; // Still in Island
-        FieldVal |= *Val << (i - StartBit);
+        FieldVal |= static_cast<uint64_t>(EncodingBits.One[i])
+                    << (i - StartBit);
       }
       break;
     }
   }
   // If we are still in Island after the loop, do some housekeeping.
   if (State == 2)
-    Islands.push_back({StartBit, BitWidth - StartBit, FieldVal});
+    Islands.push_back({StartBit, FilterWidth - StartBit, FieldVal});
 
-  return Islands.size();
+  return Islands;
 }
 
-bool FilterChooser::emitBinaryParser(raw_ostream &OS, indent Indent,
+void FilterChooser::emitBinaryParser(raw_ostream &OS, indent Indent,
                                      const OperandInfo &OpInfo) const {
   const std::string &Decoder = OpInfo.Decoder;
 
@@ -1235,49 +1097,44 @@ bool FilterChooser::emitBinaryParser(raw_ostream &OS, indent Indent,
     OS << ";\n";
   }
 
-  bool OpHasCompleteDecoder;
   if (!Decoder.empty()) {
-    OpHasCompleteDecoder = OpInfo.HasCompleteDecoder;
     OS << Indent << "if (!Check(S, " << Decoder
        << "(MI, tmp, Address, Decoder))) { "
-       << (OpHasCompleteDecoder ? "" : "DecodeComplete = false; ")
+       << (OpInfo.HasCompleteDecoder ? "" : "DecodeComplete = false; ")
        << "return MCDisassembler::Fail; }\n";
   } else {
-    OpHasCompleteDecoder = true;
     OS << Indent << "MI.addOperand(MCOperand::createImm(tmp));\n";
   }
-  return OpHasCompleteDecoder;
 }
 
-bool FilterChooser::emitDecoder(raw_ostream &OS, indent Indent,
-                                unsigned Opc) const {
-  bool HasCompleteDecoder = true;
-
-  for (const auto &Op : Operands.find(Opc)->second) {
-    // If a custom instruction decoder was specified, use that.
-    if (Op.numFields() == 0 && !Op.Decoder.empty()) {
-      HasCompleteDecoder = Op.HasCompleteDecoder;
-      OS << Indent << "if (!Check(S, " << Op.Decoder
-         << "(MI, insn, Address, Decoder))) { "
-         << (HasCompleteDecoder ? "" : "DecodeComplete = false; ")
-         << "return MCDisassembler::Fail; }\n";
-      break;
-    }
+void FilterChooser::emitDecoder(raw_ostream &OS, indent Indent,
+                                unsigned EncodingID) const {
+  const InstructionEncoding &Encoding = Encodings[EncodingID];
 
-    HasCompleteDecoder &= emitBinaryParser(OS, Indent, Op);
+  // If a custom instruction decoder was specified, use that.
+  StringRef DecoderMethod = Encoding.getDecoderMethod();
+  if (!DecoderMethod.empty()) {
+    OS << Indent << "if (!Check(S, " << DecoderMethod
+       << "(MI, insn, Address, Decoder))) { "
+       << (Encoding.hasCompleteDecoder() ? "" : "DecodeComplete = false; ")
+       << "return MCDisassembler::Fail; }\n";
+    return;
   }
-  return HasCompleteDecoder;
+
+  for (const OperandInfo &Op : Encoding.getOperands())
+    if (Op.numFields())
+      emitBinaryParser(OS, Indent, Op);
 }
 
-std::pair<unsigned, bool> FilterChooser::getDecoderIndex(DecoderSet &Decoders,
-                                                         unsigned Opc) const {
+unsigned FilterChooser::getDecoderIndex(DecoderSet &Decoders,
+                                        unsigned EncodingID) const {
   // Build up the predicate string.
   SmallString<256> Decoder;
   // FIXME: emitDecoder() function can take a buffer directly rather than
   // a stream.
   raw_svector_ostream S(Decoder);
   indent Indent(UseFnTableInDecodeToMCInst ? 2 : 4);
-  bool HasCompleteDecoder = emitDecoder(S, Indent, Opc);
+  emitDecoder(S, Indent, EncodingID);
 
   // Using the full decoder string as the key value here is a bit
   // heavyweight, but is effective. If the string comparisons become a
@@ -1289,7 +1146,7 @@ std::pair<unsigned, bool> FilterChooser::getDecoderIndex(DecoderSet &Decoders,
   Decoders.insert(CachedHashString(Decoder));
   // Now figure out the index for when we write out the table.
   DecoderSet::const_iterator P = find(Decoders, Decoder.str());
-  return {(unsigned)(P - Decoders.begin()), HasCompleteDecoder};
+  return std::distance(Decoders.begin(), P);
 }
 
 // If ParenIfBinOp is true, print a surrounding () if Val uses && or ||.
@@ -1326,9 +1183,10 @@ bool FilterChooser::emitPredicateMatchAux(const Init &Val, bool ParenIfBinOp,
   return true;
 }
 
-bool FilterChooser::emitPredicateMatch(raw_ostream &OS, unsigned Opc) const {
+bool FilterChooser::emitPredicateMatch(raw_ostream &OS,
+                                       unsigned EncodingID) const {
   const ListInit *Predicates =
-      AllInstructions[Opc].EncodingDef->getValueAsListInit("Predicates");
+      Encodings[EncodingID].getRecord()->getValueAsListInit("Predicates");
   bool IsFirstEmission = true;
   for (unsigned i = 0; i < Predicates->size(); ++i) {
     const Record *Pred = Predicates->getElementAsRecord(i);
@@ -1348,9 +1206,9 @@ bool FilterChooser::emitPredicateMatch(raw_ostream &OS, unsigned Opc) const {
   return !Predicates->empty();
 }
 
-bool FilterChooser::doesOpcodeNeedPredicate(unsigned Opc) const {
+bool FilterChooser::doesOpcodeNeedPredicate(unsigned EncodingID) const {
   const ListInit *Predicates =
-      AllInstructions[Opc].EncodingDef->getValueAsListInit("Predicates");
+      Encodings[EncodingID].getRecord()->getValueAsListInit("Predicates");
   for (unsigned i = 0; i < Predicates->size(); ++i) {
     const Record *Pred = Predicates->getElementAsRecord(i);
     if (!Pred->getValue("AssemblerMatcherPredicate"))
@@ -1378,8 +1236,8 @@ unsigned FilterChooser::getPredicateIndex(DecoderTableInfo &TableInfo,
 }
 
 void FilterChooser::emitPredicateTableEntry(DecoderTableInfo &TableInfo,
-                                            unsigned Opc) const {
-  if (!doesOpcodeNeedPredicate(Opc))
+                                            unsigned EncodingID) const {
+  if (!doesOpcodeNeedPredicate(EncodingID))
     return;
 
   // Build up the predicate string.
@@ -1387,16 +1245,16 @@ void FilterChooser::emitPredicateTableEntry(DecoderTableInfo &TableInfo,
   // FIXME: emitPredicateMatch() functions can take a buffer directly rather
   // than a stream.
   raw_svector_ostream PS(Predicate);
-  emitPredicateMatch(PS, Opc);
+  emitPredicateMatch(PS, EncodingID);
 
   // Figure out the index into the predicate table for the predicate just
   // computed.
   unsigned PIdx = getPredicateIndex(TableInfo, PS.str());
 
-  const uint8_t DecoderOp = TableInfo.isOutermostScope()
-                                ? MCD::OPC_CheckPredicateOrFail
-                                : MCD::OPC_CheckPredicate;
-  TableInfo.Table.push_back(DecoderOp);
+  const MCD::DecoderOps DecoderOp = TableInfo.isOutermostScope()
+                                        ? MCD::OPC_CheckPredicateOrFail
+                                        : MCD::OPC_CheckPredicate;
+  TableInfo.Table.insertOpcode(DecoderOp);
   TableInfo.Table.insertULEB128(PIdx);
 
   if (DecoderOp == MCD::OPC_CheckPredicate) {
@@ -1406,38 +1264,27 @@ void FilterChooser::emitPredicateTableEntry(DecoderTableInfo &TableInfo,
 }
 
 void FilterChooser::emitSoftFailTableEntry(DecoderTableInfo &TableInfo,
-                                           unsigned Opc) const {
-  const Record *EncodingDef = AllInstructions[Opc].EncodingDef;
-  const RecordVal *RV = EncodingDef->getValue("SoftFail");
-  const BitsInit *SFBits = RV ? dyn_cast<BitsInit>(RV->getValue()) : nullptr;
+                                           unsigned EncodingID) const {
+  const InstructionEncoding &Encoding = Encodings[EncodingID];
+  const KnownBits &InstBits = Encoding.getInstBits();
+  const APInt &SFBits = Encoding.getSoftFailBits();
 
-  if (!SFBits)
+  if (SFBits.isZero())
     return;
-  const BitsInit *InstBits = EncodingDef->getValueAsBitsInit("Inst");
 
-  APInt PositiveMask(BitWidth, 0ULL);
-  APInt NegativeMask(BitWidth, 0ULL);
-  for (unsigned i = 0; i < BitWidth; ++i) {
-    BitValue B(*SFBits, i);
-    BitValue IB(*InstBits, i);
-
-    if (B != BitValue::BIT_TRUE)
+  unsigned EncodingWidth = InstBits.getBitWidth();
+  APInt PositiveMask(EncodingWidth, 0);
+  APInt NegativeMask(EncodingWidth, 0);
+  for (unsigned i = 0; i != EncodingWidth; ++i) {
+    if (!SFBits[i])
       continue;
 
-    if (IB == BitValue::BIT_FALSE) {
+    if (InstBits.Zero[i]) {
       // The bit is meant to be false, so emit a check to see if it is true.
       PositiveMask.setBit(i);
-    } else if (IB == BitValue::BIT_TRUE) {
+    } else if (InstBits.One[i]) {
       // The bit is meant to be true, so emit a check to see if it is false.
       NegativeMask.setBit(i);
-    } else {
-      // The bit is not set; this must be an error!
-      errs() << "SoftFail Conflict: bit SoftFail{" << i << "} in "
-             << AllInstructions[Opc] << " is set but Inst{" << i
-             << "} is unset!\n"
-             << "  - You can only mark a bit as SoftFail if it is fully defined"
-             << " (1/0 - not '?') in Inst\n";
-      return;
     }
   }
 
@@ -1447,35 +1294,31 @@ void FilterChooser::emitSoftFailTableEntry(DecoderTableInfo &TableInfo,
   if (!NeedPositiveMask && !NeedNegativeMask)
     return;
 
-  TableInfo.Table.push_back(MCD::OPC_SoftFail);
+  TableInfo.Table.insertOpcode(MCD::OPC_SoftFail);
   TableInfo.Table.insertULEB128(PositiveMask.getZExtValue());
   TableInfo.Table.insertULEB128(NegativeMask.getZExtValue());
 }
 
 // Emits table entries to decode the singleton.
 void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
-                                            EncodingIDAndOpcode Opc) const {
-  std::vector<Island> Islands;
-  insn_t Insn;
-  insnWithID(Insn, Opc.EncodingID);
+                                            unsigned EncodingID) const {
+  const InstructionEncoding &Encoding = Encodings[EncodingID];
+  KnownBits EncodingBits = Encoding.getMandatoryBits();
 
   // Look for islands of undecoded bits of the singleton.
-  getIslands(Islands, Insn);
+  std::vector<Island> Islands = getIslands(EncodingBits);
 
   // Emit the predicate table entry if one is needed.
-  emitPredicateTableEntry(TableInfo, Opc.EncodingID);
+  emitPredicateTableEntry(TableInfo, EncodingID);
 
   // Check any additional encoding fields needed.
   for (const Island &Ilnd : reverse(Islands)) {
-    unsigned NumBits = Ilnd.NumBits;
-    assert(isUInt<8>(NumBits) && "NumBits overflowed uint8 table entry!");
-    const uint8_t DecoderOp = TableInfo.isOutermostScope()
-                                  ? MCD::OPC_CheckFieldOrFail
-                                  : MCD::OPC_CheckField;
-    TableInfo.Table.push_back(DecoderOp);
-
+    const MCD::DecoderOps DecoderOp = TableInfo.isOutermostScope()
+                                          ? MCD::OPC_CheckFieldOrFail
+                                          : MCD::OPC_CheckField;
+    TableInfo.Table.insertOpcode(DecoderOp);
     TableInfo.Table.insertULEB128(Ilnd.StartBit);
-    TableInfo.Table.push_back(NumBits);
+    TableInfo.Table.insertUInt8(Ilnd.NumBits);
     TableInfo.Table.insertULEB128(Ilnd.FieldVal);
 
     if (DecoderOp == MCD::OPC_CheckField) {
@@ -1488,10 +1331,9 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
   }
 
   // Check for soft failure of the match.
-  emitSoftFailTableEntry(TableInfo, Opc.EncodingID);
+  emitSoftFailTableEntry(TableInfo, EncodingID);
 
-  auto [DIdx, HasCompleteDecoder] =
-      getDecoderIndex(TableInfo.Decoders, Opc.EncodingID);
+  unsigned DIdx = getDecoderIndex(TableInfo.Decoders, EncodingID);
 
   // Produce OPC_Decode or OPC_TryDecode opcode based on the information
   // whether the instruction decoder is complete or not. If it is complete
@@ -1502,13 +1344,13 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
   // decoder method indicates that additional processing should be done to see
   // if there is any other instruction that also matches the bitpattern and
   // can decode it.
-  const uint8_t DecoderOp = HasCompleteDecoder ? MCD::OPC_Decode
-                                               : (TableInfo.isOutermostScope()
-                                                      ? MCD::OPC_TryDecodeOrFail
-                                                      : MCD::OPC_TryDecode);
-  TableInfo.Table.push_back(DecoderOp);
-  NumEncodingsSupported++;
-  TableInfo.Table.insertULEB128(Opc.Opcode);
+  const MCD::DecoderOps DecoderOp =
+      Encoding.hasCompleteDecoder()  ? MCD::OPC_Decode
+      : TableInfo.isOutermostScope() ? MCD::OPC_TryDecodeOrFail
+                                     : MCD::OPC_TryDecode;
+  TableInfo.Table.insertOpcode(DecoderOp);
+  const Record *InstDef = Encodings[EncodingID].getInstruction()->TheDef;
+  TableInfo.Table.insertULEB128(Emitter->getTarget().getInstrIntValue(InstDef));
   TableInfo.Table.insertULEB128(DIdx);
 
   if (DecoderOp == MCD::OPC_TryDecode) {
@@ -1517,129 +1359,36 @@ void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
   }
 }
 
-// Emits table entries to decode the singleton, and then to decode the rest.
-void FilterChooser::emitSingletonTableEntry(DecoderTableInfo &TableInfo,
-                                            const Filter &Best) const {
-  EncodingIDAndOpcode Opc = Best.getSingletonOpc();
-
-  // complex singletons need predicate checks from the first singleton
-  // to refer forward to the variable filterchooser that follows.
-  TableInfo.FixupStack.emplace_back();
-
-  emitSingletonTableEntry(TableInfo, Opc);
-
-  resolveTableFixups(TableInfo.Table, TableInfo.FixupStack.back(),
-                     TableInfo.Table.size());
-  TableInfo.FixupStack.pop_back();
-
-  Best.getVariableFC().emitTableEntries(TableInfo);
-}
-
-// Assign a single filter and run with it.  Top level API client can initialize
-// with a single filter to start the filtering process.
-void FilterChooser::runSingleFilter(unsigned startBit, unsigned numBit,
-                                    bool mixed) {
-  Filters.clear();
-  Filters.emplace_back(*this, startBit, numBit, true);
-  BestIndex = 0; // Sole Filter instance to choose from.
-  bestFilter().recurse();
-}
-
 // reportRegion is a helper function for filterProcessor to mark a region as
 // eligible for use as a filter region.
-void FilterChooser::reportRegion(bitAttr_t RA, unsigned StartBit,
-                                 unsigned BitIndex, bool AllowMixed) {
-  if (RA == ATTR_MIXED && AllowMixed)
-    Filters.emplace_back(*this, StartBit, BitIndex - StartBit, true);
-  else if (RA == ATTR_ALL_SET && !AllowMixed)
-    Filters.emplace_back(*this, StartBit, BitIndex - StartBit, false);
+void FilterChooser::reportRegion(std::vector<std::unique_ptr<Filter>> &Filters,
+                                 bitAttr_t RA, unsigned StartBit,
+                                 unsigned BitIndex, bool AllowMixed) const {
+  if (AllowMixed ? RA == ATTR_MIXED : RA == ATTR_ALL_SET)
+    Filters.push_back(std::make_unique<Filter>(Encodings, EncodingIDs, StartBit,
+                                               BitIndex - StartBit));
 }
 
-// FilterProcessor scans the well-known encoding bits of the instructions and
-// builds up a list of candidate filters.  It chooses the best filter and
-// recursively descends down the decoding tree.
-bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
-  Filters.clear();
-  BestIndex = -1;
-  unsigned numInstructions = Opcodes.size();
-
-  assert(numInstructions && "Filter created with no instructions");
-
-  // No further filtering is necessary.
-  if (numInstructions == 1)
-    return true;
+std::unique_ptr<Filter>
+FilterChooser::findBestFilter(ArrayRef<bitAttr_t> BitAttrs, bool AllowMixed,
+                              bool Greedy) const {
+  assert(EncodingIDs.size() >= 2 && "Nothing to filter");
 
   // Heuristics.  See also doFilter()'s "Heuristics" comment when num of
   // instructions is 3.
   if (AllowMixed && !Greedy) {
-    assert(numInstructions == 3);
+    assert(EncodingIDs.size() == 3);
 
-    for (const auto &Opcode : Opcodes) {
-      std::vector<Island> Islands;
-      insn_t Insn;
-
-      insnWithID(Insn, Opcode.EncodingID);
+    for (unsigned EncodingID : EncodingIDs) {
+      const InstructionEncoding &Encoding = Encodings[EncodingID];
+      KnownBits EncodingBits = Encoding.getMandatoryBits();
 
       // Look for islands of undecoded bits of any instruction.
-      if (getIslands(Islands, Insn) > 0) {
+      std::vector<Island> Islands = getIslands(EncodingBits);
+      if (!Islands.empty()) {
         // Found an instruction with island(s).  Now just assign a filter.
-        runSingleFilter(Islands[0].StartBit, Islands[0].NumBits, true);
-        return true;
-      }
-    }
-  }
-
-  unsigned BitIndex;
-
-  // We maintain BIT_WIDTH copies of the bitAttrs automaton.
-  // The automaton consumes the corresponding bit from each
-  // instruction.
-  //
-  //   Input symbols: 0, 1, and _ (unset).
-  //   States:        NONE, FILTERED, ALL_SET, ALL_UNSET, and MIXED.
-  //   Initial state: NONE.
-  //
-  // (NONE) ------- [01] -> (ALL_SET)
-  // (NONE) ------- _ ----> (ALL_UNSET)
-  // (ALL_SET) ---- [01] -> (ALL_SET)
-  // (ALL_SET) ---- _ ----> (MIXED)
-  // (ALL_UNSET) -- [01] -> (MIXED)
-  // (ALL_UNSET) -- _ ----> (ALL_UNSET)
-  // (MIXED) ------ . ----> (MIXED)
-  // (FILTERED)---- . ----> (FILTERED)
-
-  std::vector<bitAttr_t> bitAttrs(BitWidth, ATTR_NONE);
-
-  // FILTERED bit positions provide no entropy and are not worthy of pursuing.
-  // Filter::recurse() set either BIT_TRUE or BIT_FALSE for each position.
-  for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex)
-    if (FilterBitValues[BitIndex].isSet())
-      bitAttrs[BitIndex] = ATTR_FILTERED;
-
-  for (const auto &OpcPair : Opcodes) {
-    insn_t insn;
-
-    insnWithID(insn, OpcPair.EncodingID);
-
-    for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex) {
-      switch (bitAttrs[BitIndex]) {
-      case ATTR_NONE:
-        if (insn[BitIndex] == BitValue::BIT_UNSET)
-          bitAttrs[BitIndex] = ATTR_ALL_UNSET;
-        else
-          bitAttrs[BitIndex] = ATTR_ALL_SET;
-        break;
-      case ATTR_ALL_SET:
-        if (insn[BitIndex] == BitValue::BIT_UNSET)
-          bitAttrs[BitIndex] = ATTR_MIXED;
-        break;
-      case ATTR_ALL_UNSET:
-        if (insn[BitIndex] != BitValue::BIT_UNSET)
-          bitAttrs[BitIndex] = ATTR_MIXED;
-        break;
-      case ATTR_MIXED:
-      case ATTR_FILTERED:
-        break;
+        return std::make_unique<Filter>(
+            Encodings, EncodingIDs, Islands[0].StartBit, Islands[0].NumBits);
       }
     }
   }
@@ -1667,8 +1416,10 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
   bitAttr_t RA = ATTR_NONE;
   unsigned StartBit = 0;
 
-  for (BitIndex = 0; BitIndex < BitWidth; ++BitIndex) {
-    bitAttr_t bitAttr = bitAttrs[BitIndex];
+  std::vector<std::unique_ptr<Filter>> Filters;
+  unsigned FilterWidth = FilterBits.getBitWidth();
+  for (unsigned BitIndex = 0; BitIndex != FilterWidth; ++BitIndex) {
+    bitAttr_t bitAttr = BitAttrs[BitIndex];
 
     assert(bitAttr != ATTR_NONE && "Bit without attributes");
 
@@ -1694,17 +1445,17 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
     case ATTR_ALL_SET:
       switch (bitAttr) {
       case ATTR_FILTERED:
-        reportRegion(RA, StartBit, BitIndex, AllowMixed);
+        reportRegion(Filters, RA, StartBit, BitIndex, AllowMixed);
         RA = ATTR_NONE;
         break;
       case ATTR_ALL_SET:
         break;
       case ATTR_ALL_UNSET:
-        reportRegion(RA, StartBit, BitIndex, AllowMixed);
+        reportRegion(Filters, RA, StartBit, BitIndex, AllowMixed);
         RA = ATTR_NONE;
         break;
       case ATTR_MIXED:
-        reportRegion(RA, StartBit, BitIndex, AllowMixed);
+        reportRegion(Filters, RA, StartBit, BitIndex, AllowMixed);
         StartBit = BitIndex;
         RA = ATTR_MIXED;
         break;
@@ -1715,17 +1466,17 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
     case ATTR_MIXED:
       switch (bitAttr) {
       case ATTR_FILTERED:
-        reportRegion(RA, StartBit, BitIndex, AllowMixed);
+        reportRegion(Filters, RA, StartBit, BitIndex, AllowMixed);
         StartBit = BitIndex;
         RA = ATTR_NONE;
         break;
       case ATTR_ALL_SET:
-        reportRegion(RA, StartBit, BitIndex, AllowMixed);
+        reportRegion(Filters, RA, StartBit, BitIndex, AllowMixed);
         StartBit = BitIndex;
         RA = ATTR_ALL_SET;
         break;
       case ATTR_ALL_UNSET:
-        reportRegion(RA, StartBit, BitIndex, AllowMixed);
+        reportRegion(Filters, RA, StartBit, BitIndex, AllowMixed);
         RA = ATTR_NONE;
         break;
       case ATTR_MIXED:
@@ -1748,23 +1499,23 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
   case ATTR_FILTERED:
     break;
   case ATTR_ALL_SET:
-    reportRegion(RA, StartBit, BitIndex, AllowMixed);
+    reportRegion(Filters, RA, StartBit, FilterWidth, AllowMixed);
     break;
   case ATTR_ALL_UNSET:
     break;
   case ATTR_MIXED:
-    reportRegion(RA, StartBit, BitIndex, AllowMixed);
+    reportRegion(Filters, RA, StartBit, FilterWidth, AllowMixed);
     break;
   }
 
   // We have finished with the filter processings.  Now it's time to choose
   // the best performing filter.
-  BestIndex = 0;
+  unsigned BestIndex = 0;
   bool AllUseless = true;
   unsigned BestScore = 0;
 
   for (const auto &[Idx, Filter] : enumerate(Filters)) {
-    unsigned Usefulness = Filter.usefulness();
+    unsigned Usefulness = Filter->usefulness();
 
     if (Usefulness)
       AllUseless = false;
@@ -1775,75 +1526,181 @@ bool FilterChooser::filterProcessor(bool AllowMixed, bool Greedy) {
     }
   }
 
-  if (!AllUseless)
-    bestFilter().recurse();
+  if (AllUseless)
+    return nullptr;
 
-  return !AllUseless;
-} // end of FilterChooser::filterProcessor(bool)
+  return std::move(Filters[BestIndex]);
+}
 
-// Decides on the best configuration of filter(s) to use in order to decode
-// the instructions.  A conflict of instructions may occur, in which case we
-// dump the conflict set to the standard error.
-void FilterChooser::doFilter() {
-  unsigned Num = Opcodes.size();
-  assert(Num && "FilterChooser created with no instructions");
+std::unique_ptr<Filter> FilterChooser::findBestFilter() const {
+  // We maintain BIT_WIDTH copies of the bitAttrs automaton.
+  // The automaton consumes the corresponding bit from each
+  // instruction.
+  //
+  //   Input symbols: 0, 1, _ (unset), and . (any of the above).
+  //   States:        NONE, FILTERED, ALL_SET, ALL_UNSET, and MIXED.
+  //   Initial state: NONE.
+  //
+  // (NONE) ------- [01] -> (ALL_SET)
+  // (NONE) ------- _ ----> (ALL_UNSET)
+  // (ALL_SET) ---- [01] -> (ALL_SET)
+  // (ALL_SET) ---- _ ----> (MIXED)
+  // (ALL_UNSET) -- [01] -> (MIXED)
+  // (ALL_UNSET) -- _ ----> (ALL_UNSET)
+  // (MIXED) ------ . ----> (MIXED)
+  // (FILTERED)---- . ----> (FILTERED)
+
+  unsigned FilterWidth = FilterBits.getBitWidth();
+  SmallVector<bitAttr_t, 128> BitAttrs(FilterWidth, ATTR_NONE);
+
+  // FILTERED bit positions provide no entropy and are not worthy of pursuing.
+  // Filter::recurse() set either 1 or 0 for each position.
+  for (unsigned BitIndex = 0; BitIndex != FilterWidth; ++BitIndex)
+    if (isPositionFiltered(BitIndex))
+      BitAttrs[BitIndex] = ATTR_FILTERED;
+
+  for (unsigned EncodingID : EncodingIDs) {
+    const InstructionEncoding &Encoding = Encodings[EncodingID];
+    KnownBits EncodingBits = Encoding.getMandatoryBits();
+
+    for (unsigned BitIndex = 0; BitIndex != FilterWidth; ++BitIndex) {
+      bool IsKnown = EncodingBits.Zero[BitIndex] || EncodingBits.One[BitIndex];
+      switch (BitAttrs[BitIndex]) {
+      case ATTR_NONE:
+        if (IsKnown)
+          BitAttrs[BitIndex] = ATTR_ALL_SET;
+        else
+          BitAttrs[BitIndex] = ATTR_ALL_UNSET;
+        break;
+      case ATTR_ALL_SET:
+        if (!IsKnown)
+          BitAttrs[BitIndex] = ATTR_MIXED;
+        break;
+      case ATTR_ALL_UNSET:
+        if (IsKnown)
+          BitAttrs[BitIndex] = ATTR_MIXED;
+        break;
+      case ATTR_MIXED:
+      case ATTR_FILTERED:
+        break;
+      }
+    }
+  }
 
   // Try regions of consecutive known bit values first.
-  if (filterProcessor(false))
-    return;
+  if (std::unique_ptr<Filter> F =
+          findBestFilter(BitAttrs, /*AllowMixed=*/false))
+    return F;
 
   // Then regions of mixed bits (both known and unitialized bit values allowed).
-  if (filterProcessor(true))
-    return;
+  if (std::unique_ptr<Filter> F = findBestFilter(BitAttrs, /*AllowMixed=*/true))
+    return F;
 
   // Heuristics to cope with conflict set {t2CMPrs, t2SUBSrr, t2SUBSrs} where
   // no single instruction for the maximum ATTR_MIXED region Inst{14-4} has a
   // well-known encoding pattern.  In such case, we backtrack and scan for the
   // the very first consecutive ATTR_ALL_SET region and assign a filter to it.
-  if (Num == 3 && filterProcessor(true, false))
-    return;
+  if (EncodingIDs.size() == 3) {
+    if (std::unique_ptr<Filter> F =
+            findBestFilter(BitAttrs, /*AllowMixed=*/true, /*Greedy=*/false))
+      return F;
+  }
 
-  // If we come to here, the instruction decoding has failed.
-  // Set the BestIndex to -1 to indicate so.
-  BestIndex = -1;
+  // There is a conflict we could not resolve.
+  return nullptr;
 }
 
-// emitTableEntries - Emit state machine entries to decode our share of
-// instructions.
-void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const {
-  if (Opcodes.size() == 1) {
-    // There is only one instruction in the set, which is great!
-    // Call emitSingletonDecoder() to see whether there are any remaining
-    // encodings bits.
-    emitSingletonTableEntry(TableInfo, Opcodes[0]);
+// Decides on the best configuration of filter(s) to use in order to decode
+// the instructions.  A conflict of instructions may occur, in which case we
+// dump the conflict set to the standard error.
+void FilterChooser::doFilter() {
+  assert(!EncodingIDs.empty() && "FilterChooser created with no instructions");
+
+  // No filter needed.
+  if (EncodingIDs.size() == 1) {
+    SingletonEncodingID = EncodingIDs.front();
     return;
   }
 
-  // Choose the best filter to do the decodings!
-  if (BestIndex != -1) {
-    const Filter &Best = Filters[BestIndex];
-    if (Best.getNumFiltered() == 1)
-      emitSingletonTableEntry(TableInfo, Best);
-    else
-      Best.emitTableEntry(TableInfo);
+  std::unique_ptr<Filter> BestFilter = findBestFilter();
+  if (BestFilter) {
+    applyFilter(*BestFilter);
     return;
   }
 
-  // We don't know how to decode these instructions! Dump the
-  // conflict set and bail.
-
   // Print out useful conflict information for postmortem analysis.
   errs() << "Decoding Conflict:\n";
+  dump();
+  PrintFatalError("Decoding conflict encountered");
+}
+
+void FilterChooser::dump() const {
+  indent Indent(4);
+  // Helps to keep the output right-justified.
+  unsigned PadToWidth = getMaxEncodingWidth();
 
-  dumpStack(errs(), "\t\t");
+  // Dump filter stack.
+  dumpStack(errs(), Indent, PadToWidth);
 
-  for (auto Opcode : Opcodes) {
-    const EncodingAndInst &Enc = AllInstructions[Opcode.EncodingID];
-    errs() << '\t' << Enc << ' ';
-    dumpBits(errs(), getBitsField(*Enc.EncodingDef, "Inst"));
-    errs() << '\n';
+  // Dump encodings.
+  for (unsigned EncodingID : EncodingIDs) {
+    const InstructionEncoding &Encoding = Encodings[EncodingID];
+    errs() << Indent << indent(PadToWidth - Encoding.getBitWidth());
+    printKnownBits(errs(), Encoding.getMandatoryBits(), '_');
+    errs() << "  " << Encoding.getName() << '\n';
+  }
+}
+
+void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const {
+  // If there are other encodings that could match if those with all bits
+  // known don't, enter a scope so that they have a chance.
+  if (VariableFC)
+    TableInfo.pushScope();
+
+  if (SingletonEncodingID) {
+    assert(FilterChooserMap.empty());
+    // There is only one encoding in which all bits in the filtered range are
+    // fully defined, but we still need to check if the remaining (unfiltered)
+    // bits are valid for this encoding. We also need to check predicates etc.
+    emitSingletonTableEntry(TableInfo, *SingletonEncodingID);
+  } else {
+    // The general case: emit a switch over the field value.
+    DecoderTable &Table = TableInfo.Table;
+    Table.insertOpcode(MCD::OPC_ExtractField);
+    Table.insertULEB128(StartBit);
+    Table.insertUInt8(NumBits);
+
+    // Emit switch cases for all but the last element.
+    for (const auto &[FilterVal, Delegate] : drop_end(FilterChooserMap)) {
+      Table.insertOpcode(MCD::OPC_FilterValue);
+      Table.insertULEB128(FilterVal);
+      size_t FixupPos = Table.insertNumToSkip();
+
+      // Emit table entries for this case.
+      Delegate->emitTableEntries(TableInfo);
+
+      // Patch the previous OPC_FilterValue to fall through to the next case.
+      Table.patchNumToSkip(FixupPos, Table.size());
+    }
+
+    // Emit a switch case for the last element. It never falls through;
+    // if it doesn't match, we leave the current scope.
+    const auto &[FilterVal, Delegate] = *FilterChooserMap.rbegin();
+    Table.insertOpcode(!TableInfo.isOutermostScope()
+                           ? MCD::OPC_FilterValue
+                           : MCD::OPC_FilterValueOrFail);
+    Table.insertULEB128(FilterVal);
+    if (!TableInfo.isOutermostScope())
+      TableInfo.FixupStack.back().push_back(Table.insertNumToSkip());
+
+    // Emit table entries for the last case.
+    Delegate->emitTableEntries(TableInfo);
+  }
+
+  if (VariableFC) {
+    TableInfo.popScope();
+    VariableFC->emitTableEntries(TableInfo);
   }
-  PrintFatalError("Decoding conflict encountered");
 }
 
 static std::string findOperandDecoderMethod(const Record *Record) {
@@ -1885,15 +1742,128 @@ OperandInfo getOpInfo(const Record *TypeRecord) {
   return OperandInfo(findOperandDecoderMethod(TypeRecord), HasCompleteDecoder);
 }
 
-static void parseVarLenInstOperand(const Record &Def,
-                                   std::vector<OperandInfo> &Operands,
-                                   const CodeGenInstruction &CGI) {
+void InstructionEncoding::parseVarLenEncoding(const VarLenInst &VLI) {
+  InstBits = KnownBits(VLI.size());
+  SoftFailBits = APInt(VLI.size(), 0);
+
+  // Parse Inst field.
+  unsigned I = 0;
+  for (const EncodingSegment &S : VLI) {
+    if (const auto *SegmentBits = dyn_cast<BitsInit>(S.Value)) {
+      for (const Init *V : SegmentBits->getBits()) {
+        if (const auto *B = dyn_cast<BitInit>(V)) {
+          if (B->getValue())
+            InstBits.One.setBit(I);
+          else
+            InstBits.Zero.setBit(I);
+        }
+        ++I;
+      }
+    } else if (const auto *B = dyn_cast<BitInit>(S.Value)) {
+      if (B->getValue())
+        InstBits.One.setBit(I);
+      else
+        InstBits.Zero.setBit(I);
+      ++I;
+    } else {
+      I += S.BitWidth;
+    }
+  }
+  assert(I == VLI.size());
+}
+
+void InstructionEncoding::parseFixedLenEncoding(
+    const BitsInit &RecordInstBits) {
+  // For fixed length instructions, sometimes the `Inst` field specifies more
+  // bits than the actual size of the instruction, which is specified in `Size`.
+  // In such cases, we do some basic validation and drop the upper bits.
+  unsigned BitWidth = EncodingDef->getValueAsInt("Size") * 8;
+  unsigned InstNumBits = RecordInstBits.getNumBits();
+
+  // Returns true if all bits in `Bits` are zero or unset.
+  auto CheckAllZeroOrUnset = [&](ArrayRef<const Init *> Bits,
+                                 const RecordVal *Field) {
+    bool AllZeroOrUnset = llvm::all_of(Bits, [](const Init *Bit) {
+      if (const auto *BI = dyn_cast<BitInit>(Bit))
+        return !BI->getValue();
+      return isa<UnsetInit>(Bit);
+    });
+    if (AllZeroOrUnset)
+      return;
+    PrintNote([Field](raw_ostream &OS) { Field->print(OS); });
+    PrintFatalError(EncodingDef, Twine(Name) + ": Size is " + Twine(BitWidth) +
+                                     " bits, but " + Field->getName() +
+                                     " bits beyond that are    not zero/unset");
+  };
+
+  if (InstNumBits < BitWidth)
+    PrintFatalError(EncodingDef, Twine(Name) + ": Size is " + Twine(BitWidth) +
+                                     " bits, but Inst specifies only " +
+                                     Twine(InstNumBits) + " bits");
+
+  if (InstNumBits > BitWidth) {
+    // Ensure that all the bits beyond 'Size' are 0 or unset (i.e., carry no
+    // actual encoding).
+    ArrayRef<const Init *> UpperBits =
+        RecordInstBits.getBits().drop_front(BitWidth);
+    const RecordVal *InstField = EncodingDef->getValue("Inst");
+    CheckAllZeroOrUnset(UpperBits, InstField);
+  }
+
+  ArrayRef<const Init *> ActiveInstBits =
+      RecordInstBits.getBits().take_front(BitWidth);
+  InstBits = KnownBits(BitWidth);
+  SoftFailBits = APInt(BitWidth, 0);
+
+  // Parse Inst field.
+  for (auto [I, V] : enumerate(ActiveInstBits)) {
+    if (const auto *B = dyn_cast<BitInit>(V)) {
+      if (B->getValue())
+        InstBits.One.setBit(I);
+      else
+        InstBits.Zero.setBit(I);
+    }
+  }
 
-  const RecordVal *RV = Def.getValue("Inst");
-  VarLenInst VLI(cast<DagInit>(RV->getValue()), RV);
+  // Parse SoftFail field.
+  const RecordVal *SoftFailField = EncodingDef->getValue("SoftFail");
+  if (!SoftFailField)
+    return;
+
+  const auto *SFBits = dyn_cast<BitsInit>(SoftFailField->getValue());
+  if (!SFBits || SFBits->getNumBits() != InstNumBits) {
+    PrintNote(EncodingDef->getLoc(), "in record");
+    PrintFatalError(SoftFailField,
+                    formatv("SoftFail field, if defined, must be "
+                            "of the same type as Inst, which is bits<{}>",
+                            InstNumBits));
+  }
+
+  if (InstNumBits > BitWidth) {
+    // Ensure that all upper bits of `SoftFail` are 0 or unset.
+    ArrayRef<const Init *> UpperBits = SFBits->getBits().drop_front(BitWidth);
+    CheckAllZeroOrUnset(UpperBits, SoftFailField);
+  }
+
+  ArrayRef<const Init *> ActiveSFBits = SFBits->getBits().take_front(BitWidth);
+  for (auto [I, V] : enumerate(ActiveSFBits)) {
+    if (const auto *B = dyn_cast<BitInit>(V); B && B->getValue()) {
+      if (!InstBits.Zero[I] && !InstBits.One[I]) {
+        PrintNote(EncodingDef->getLoc(), "in record");
+        PrintError(SoftFailField,
+                   formatv("SoftFail{{{0}} = 1 requires Inst{{{0}} "
+                           "to be fully defined (0 or 1, not '?')",
+                           I));
+      }
+      SoftFailBits.setBit(I);
+    }
+  }
+}
+
+void InstructionEncoding::parseVarLenOperands(const VarLenInst &VLI) {
   SmallVector<int> TiedTo;
 
-  for (const auto &[Idx, Op] : enumerate(CGI.Operands)) {
+  for (const auto &[Idx, Op] : enumerate(Inst->Operands)) {
     if (Op.MIOperandInfo && Op.MIOperandInfo->getNumArgs() > 0)
       for (auto *Arg : Op.MIOperandInfo->getArgs())
         Operands.push_back(getOpInfo(cast<DefInit>(Arg)->getDef()));
@@ -1921,17 +1891,15 @@ static void parseVarLenInstOperand(const Record &Def,
     }
 
     if (!OpName.empty()) {
-      auto OpSubOpPair =
-          const_cast<CodeGenInstruction &>(CGI).Operands.ParseOperandName(
-              OpName);
-      unsigned OpIdx = CGI.Operands.getFlattenedOperandNumber(OpSubOpPair);
+      auto OpSubOpPair = Inst->Operands.parseOperandName(OpName);
+      unsigned OpIdx = Inst->Operands.getFlattenedOperandNumber(OpSubOpPair);
       Operands[OpIdx].addField(CurrBitPos, EncodingSegment.BitWidth, Offset);
       if (!EncodingSegment.CustomDecoder.empty())
         Operands[OpIdx].Decoder = EncodingSegment.CustomDecoder.str();
 
       int TiedReg = TiedTo[OpSubOpPair.first];
       if (TiedReg != -1) {
-        unsigned OpIdx = CGI.Operands.getFlattenedOperandNumber(
+        unsigned OpIdx = Inst->Operands.getFlattenedOperandNumber(
             {TiedReg, OpSubOpPair.second});
         Operands[OpIdx].addField(CurrBitPos, EncodingSegment.BitWidth, Offset);
       }
@@ -1952,12 +1920,12 @@ static void debugDumpRecord(const Record &Rec) {
 /// For an operand field named OpName: populate OpInfo.InitValue with the
 /// constant-valued bit values, and OpInfo.Fields with the ranges of bits to
 /// insert from the decoded instruction.
-static void addOneOperandFields(const Record &EncodingDef, const BitsInit &Bits,
+static void addOneOperandFields(const Record *EncodingDef, const BitsInit &Bits,
                                 std::map<StringRef, StringRef> &TiedNames,
                                 StringRef OpName, OperandInfo &OpInfo) {
   // Some bits of the operand may be required to be 1 depending on the
   // instruction's encoding. Collect those bits.
-  if (const RecordVal *EncodedValue = EncodingDef.getValue(OpName))
+  if (const RecordVal *EncodedValue = EncodingDef->getValue(OpName))
     if (const BitsInit *OpBits = dyn_cast<BitsInit>(EncodedValue->getValue()))
       for (unsigned I = 0; I < OpBits->getNumBits(); ++I)
         if (const BitInit *OpBit = dyn_cast<BitInit>(OpBits->getBit(I)))
@@ -1989,37 +1957,8 @@ static void addOneOperandFields(const Record &EncodingDef, const BitsInit &Bits,
   }
 }
 
-static unsigned
-populateInstruction(const CodeGenTarget &Target, const Record &EncodingDef,
-                    const CodeGenInstruction &CGI, unsigned Opc,
-                    std::map<unsigned, std::vector<OperandInfo>> &Operands,
-                    bool IsVarLenInst) {
-  const Record &Def = *CGI.TheDef;
-  // If all the bit positions are not specified; do not decode this instruction.
-  // We are bound to fail!  For proper disassembly, the well-known encoding bits
-  // of the instruction must be fully specified.
-
-  const BitsInit &Bits = getBitsField(EncodingDef, "Inst");
-  if (Bits.allInComplete())
-    return 0;
-
-  std::vector<OperandInfo> InsnOperands;
-
-  // If the instruction has specified a custom decoding hook, use that instead
-  // of trying to auto-generate the decoder.
-  StringRef InstDecoder = EncodingDef.getValueAsString("DecoderMethod");
-  if (!InstDecoder.empty()) {
-    bool HasCompleteInstDecoder =
-        EncodingDef.getValueAsBit("hasCompleteDecoder");
-    InsnOperands.push_back(
-        OperandInfo(InstDecoder.str(), HasCompleteInstDecoder));
-    Operands[Opc] = std::move(InsnOperands);
-    return Bits.getNumBits();
-  }
-
-  // Generate a description of the operand of the instruction that we know
-  // how to decode automatically.
-  // FIXME: We'll need to have a way to manually override this as needed.
+void InstructionEncoding::parseFixedLenOperands(const BitsInit &Bits) {
+  const Record &Def = *Inst->TheDef;
 
   // Gather the outputs/inputs of the instruction, so we can find their
   // positions in the encoding.  This assumes for now that they appear in the
@@ -2035,15 +1974,15 @@ populateInstruction(const CodeGenTarget &Target, const Record &EncodingDef,
   // Search for tied operands, so that we can correctly instantiate
   // operands that are not explicitly represented in the encoding.
   std::map<StringRef, StringRef> TiedNames;
-  for (const auto &Op : CGI.Operands) {
+  for (const auto &Op : Inst->Operands) {
     for (const auto &[J, CI] : enumerate(Op.Constraints)) {
       if (!CI.isTied())
         continue;
       std::pair<unsigned, unsigned> SO =
-          CGI.Operands.getSubOperandNumber(CI.getTiedOperand());
-      StringRef TiedName = CGI.Operands[SO.first].SubOpNames[SO.second];
+          Inst->Operands.getSubOperandNumber(CI.getTiedOperand());
+      StringRef TiedName = Inst->Operands[SO.first].SubOpNames[SO.second];
       if (TiedName.empty())
-        TiedName = CGI.Operands[SO.first].Name;
+        TiedName = Inst->Operands[SO.first].Name;
       StringRef MyName = Op.SubOpNames[J];
       if (MyName.empty())
         MyName = Op.Name;
@@ -2053,168 +1992,123 @@ populateInstruction(const CodeGenTarget &Target, const Record &EncodingDef,
     }
   }
 
-  if (IsVarLenInst) {
-    parseVarLenInstOperand(EncodingDef, InsnOperands, CGI);
-  } else {
-    // For each operand, see if we can figure out where it is encoded.
-    for (const auto &Op : InOutOperands) {
-      const Init *OpInit = Op.first;
-      StringRef OpName = Op.second;
-
-      // We're ready to find the instruction encoding locations for this
-      // operand.
-
-      // First, find the operand type ("OpInit"), and sub-op names
-      // ("SubArgDag") if present.
-      const DagInit *SubArgDag = dyn_cast<DagInit>(OpInit);
-      if (SubArgDag)
-        OpInit = SubArgDag->getOperator();
-      const Record *OpTypeRec = cast<DefInit>(OpInit)->getDef();
-      // Lookup the sub-operands from the operand type record (note that only
-      // Operand subclasses have MIOperandInfo, see CodeGenInstruction.cpp).
-      const DagInit *SubOps = OpTypeRec->isSubClassOf("Operand")
-                                  ? OpTypeRec->getValueAsDag("MIOperandInfo")
-                                  : nullptr;
-
-      // Lookup the decoder method and construct a new OperandInfo to hold our
-      // result.
-      OperandInfo OpInfo = getOpInfo(OpTypeRec);
-
-      // If we have named sub-operands...
-      if (SubArgDag) {
-        // Then there should not be a custom decoder specified on the top-level
-        // type.
-        if (!OpInfo.Decoder.empty()) {
-          PrintError(EncodingDef.getLoc(),
-                     "DecoderEmitter: operand \"" + OpName + "\" has type \"" +
-                         OpInit->getAsString() +
-                         "\" with a custom DecoderMethod, but also named "
-                         "sub-operands.");
-          continue;
-        }
-
-        // Decode each of the sub-ops separately.
-        assert(SubOps && SubArgDag->getNumArgs() == SubOps->getNumArgs());
-        for (const auto &[I, Arg] : enumerate(SubOps->getArgs())) {
-          StringRef SubOpName = SubArgDag->getArgNameStr(I);
-          OperandInfo SubOpInfo = getOpInfo(cast<DefInit>(Arg)->getDef());
-
-          addOneOperandFields(EncodingDef, Bits, TiedNames, SubOpName,
-                              SubOpInfo);
-          InsnOperands.push_back(std::move(SubOpInfo));
-        }
+  // For each operand, see if we can figure out where it is encoded.
+  for (const auto &Op : InOutOperands) {
+    const Init *OpInit = Op.first;
+    StringRef OpName = Op.second;
+
+    // We're ready to find the instruction encoding locations for this
+    // operand.
+
+    // First, find the operand type ("OpInit"), and sub-op names
+    // ("SubArgDag") if present.
+    const DagInit *SubArgDag = dyn_cast<DagInit>(OpInit);
+    if (SubArgDag)
+      OpInit = SubArgDag->getOperator();
+    const Record *OpTypeRec = cast<DefInit>(OpInit)->getDef();
+    // Lookup the sub-operands from the operand type record (note that only
+    // Operand subclasses have MIOperandInfo, see CodeGenInstruction.cpp).
+    const DagInit *SubOps = OpTypeRec->isSubClassOf("Operand")
+                                ? OpTypeRec->getValueAsDag("MIOperandInfo")
+                                : nullptr;
+
+    // Lookup the decoder method and construct a new OperandInfo to hold our
+    // result.
+    OperandInfo OpInfo = getOpInfo(OpTypeRec);
+
+    // If we have named sub-operands...
+    if (SubArgDag) {
+      // Then there should not be a custom decoder specified on the top-level
+      // type.
+      if (!OpInfo.Decoder.empty()) {
+        PrintError(EncodingDef,
+                   "DecoderEmitter: operand \"" + OpName + "\" has type \"" +
+                       OpInit->getAsString() +
+                       "\" with a custom DecoderMethod, but also named "
+                       "sub-operands.");
         continue;
       }
 
-      // Otherwise, if we have an operand with sub-operands, but they aren't
-      // named...
-      if (SubOps && OpInfo.Decoder.empty()) {
-        // If it's a single sub-operand, and no custom decoder, use the decoder
-        // from the one sub-operand.
-        if (SubOps->getNumArgs() == 1)
-          OpInfo = getOpInfo(cast<DefInit>(SubOps->getArg(0))->getDef());
-
-        // If we have multiple sub-ops, there'd better have a custom
-        // decoder. (Otherwise we don't know how to populate them properly...)
-        if (SubOps->getNumArgs() > 1) {
-          PrintError(EncodingDef.getLoc(),
-                     "DecoderEmitter: operand \"" + OpName +
-                         "\" uses MIOperandInfo with multiple ops, but doesn't "
-                         "have a custom decoder!");
-          debugDumpRecord(EncodingDef);
-          continue;
-        }
-      }
+      // Decode each of the sub-ops separately.
+      assert(SubOps && SubArgDag->getNumArgs() == SubOps->getNumArgs());
+      for (const auto &[I, Arg] : enumerate(SubOps->getArgs())) {
+        StringRef SubOpName = SubArgDag->getArgNameStr(I);
+        OperandInfo SubOpInfo = getOpInfo(cast<DefInit>(Arg)->getDef());
 
-      addOneOperandFields(EncodingDef, Bits, TiedNames, OpName, OpInfo);
-      // FIXME: it should be an error not to find a definition for a given
-      // operand, rather than just failing to add it to the resulting
-      // instruction! (This is a longstanding bug, which will be addressed in an
-      // upcoming change.)
-      if (OpInfo.numFields() > 0)
-        InsnOperands.push_back(std::move(OpInfo));
+        addOneOperandFields(EncodingDef, Bits, TiedNames, SubOpName, SubOpInfo);
+        Operands.push_back(std::move(SubOpInfo));
+      }
+      continue;
     }
-  }
-  Operands[Opc] = std::move(InsnOperands);
-
-#if 0
-  LLVM_DEBUG({
-      // Dumps the instruction encoding bits.
-      dumpBits(errs(), Bits);
 
-      errs() << '\n';
-
-      // Dumps the list of operand info.
-      for (unsigned i = 0, e = CGI.Operands.size(); i != e; ++i) {
-        const CGIOperandList::OperandInfo &Info = CGI.Operands[i];
-        const std::string &OperandName = Info.Name;
-        const Record &OperandDef = *Info.Rec;
-
-        errs() << "\t" << OperandName << " (" << OperandDef.getName() << ")\n";
+    // Otherwise, if we have an operand with sub-operands, but they aren't
+    // named...
+    if (SubOps && OpInfo.Decoder.empty()) {
+      // If it's a single sub-operand, and no custom decoder, use the decoder
+      // from the one sub-operand.
+      if (SubOps->getNumArgs() == 1)
+        OpInfo = getOpInfo(cast<DefInit>(SubOps->getArg(0))->getDef());
+
+      // If we have multiple sub-ops, there'd better have a custom
+      // decoder. (Otherwise we don't know how to populate them properly...)
+      if (SubOps->getNumArgs() > 1) {
+        PrintError(EncodingDef,
+                   "DecoderEmitter: operand \"" + OpName +
+                       "\" uses MIOperandInfo with multiple ops, but doesn't "
+                       "have a custom decoder!");
+        debugDumpRecord(*EncodingDef);
+        continue;
       }
-    });
-#endif
-
-  return Bits.getNumBits();
-}
-
-// emitFieldFromInstruction - Emit the templated helper function
-// fieldFromInstruction().
-// On Windows we make sure that this function is not inlined when
-// using the VS compiler. It has a bug which causes the function
-// to be optimized out in some circumstances. See llvm.org/pr38292
-static void emitFieldFromInstruction(formatted_raw_ostream &OS) {
-  OS << R"(
-// Helper functions for extracting fields from encoded instructions.
-// InsnType must either be integral or an APInt-like object that must:
-// * be default-constructible and copy-constructible
-// * Support extractBitsAsZExtValue(numBits, startBit)
-// * Support the ~, &, ==, and != operators with other objects of the same type
-// * Support the != and bitwise & with uint64_t
-template <typename InsnType>
-#if defined(_MSC_VER) && !defined(__clang__)
-__declspec(noinline)
-#endif
-static std::enable_if_t<std::is_integral<InsnType>::value, InsnType>
-fieldFromInstruction(const InsnType &insn, unsigned startBit,
-                     unsigned numBits) {
-  assert(startBit + numBits <= 64 && "Cannot support >64-bit extractions!");
-  assert(startBit + numBits <= (sizeof(InsnType) * 8) &&
-         "Instruction field out of bounds!");
-  InsnType fieldMask;
-  if (numBits == sizeof(InsnType) * 8)
-    fieldMask = (InsnType)(-1LL);
-  else
-    fieldMask = (((InsnType)1 << numBits) - 1) << startBit;
-  return (insn & fieldMask) >> startBit;
-}
+    }
 
-template <typename InsnType>
-static std::enable_if_t<!std::is_integral<InsnType>::value, uint64_t>
-fieldFromInstruction(const InsnType &insn, unsigned startBit,
-                     unsigned numBits) {
-  return insn.extractBitsAsZExtValue(numBits, startBit);
-}
-)";
+    addOneOperandFields(EncodingDef, Bits, TiedNames, OpName, OpInfo);
+    // FIXME: it should be an error not to find a definition for a given
+    // operand, rather than just failing to add it to the resulting
+    // instruction! (This is a longstanding bug, which will be addressed in an
+    // upcoming change.)
+    if (OpInfo.numFields() > 0)
+      Operands.push_back(std::move(OpInfo));
+  }
 }
 
-// emitInsertBits - Emit the templated helper function insertBits().
-static void emitInsertBits(formatted_raw_ostream &OS) {
-  OS << R"(
-// Helper function for inserting bits extracted from an encoded instruction into
-// an integer-typed field.
-template <typename IntType>
-static std::enable_if_t<std::is_integral_v<IntType>, void>
-insertBits(IntType &field, IntType bits, unsigned startBit, unsigned numBits) {
-  // Check that no bit beyond numBits is set, so that a simple bitwise |
-  // is sufficient.
-  assert((~(((IntType)1 << numBits) - 1) & bits) == 0 &&
-           "bits has more than numBits bits set");
-  assert(startBit + numBits <= sizeof(IntType) * 8);
-  (void)numBits;
-  field |= bits << startBit;
-}
-)";
+InstructionEncoding::InstructionEncoding(const Record *EncodingDef,
+                                         const CodeGenInstruction *Inst)
+    : EncodingDef(EncodingDef), Inst(Inst) {
+  const Record *InstDef = Inst->TheDef;
+
+  // Give this encoding a name.
+  if (EncodingDef != InstDef)
+    Name = (EncodingDef->getName() + Twine(':')).str();
+  Name.append(InstDef->getName());
+
+  DecoderMethod = EncodingDef->getValueAsString("DecoderMethod");
+  if (!DecoderMethod.empty())
+    HasCompleteDecoder = EncodingDef->getValueAsBit("hasCompleteDecoder");
+
+  const RecordVal *InstField = EncodingDef->getValue("Inst");
+  if (const auto *DI = dyn_cast<DagInit>(InstField->getValue())) {
+    VarLenInst VLI(DI, InstField);
+    parseVarLenEncoding(VLI);
+    // If the encoding has a custom decoder, don't bother parsing the operands.
+    if (DecoderMethod.empty())
+      parseVarLenOperands(VLI);
+  } else {
+    const auto *BI = cast<BitsInit>(InstField->getValue());
+    parseFixedLenEncoding(*BI);
+    // If the encoding has a custom decoder, don't bother parsing the operands.
+    if (DecoderMethod.empty())
+      parseFixedLenOperands(*BI);
+  }
+
+  if (DecoderMethod.empty()) {
+    // A generated decoder is always successful if none of the operand
+    // decoders can fail (all are always successful).
+    HasCompleteDecoder = all_of(Operands, [](const OperandInfo &Op) {
+      // By default, a generated operand decoder is assumed to always succeed.
+      // This can be overridden by the user.
+      return Op.Decoder.empty() || Op.HasCompleteDecoder;
+    });
+  }
 }
 
 // emitDecodeInstruction - Emit the templated helper function
@@ -2250,6 +2144,7 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
   OS << ") {\n";
   if (HasCheckPredicate)
     OS << "  const FeatureBitset &Bits = STI.getFeatureBits();\n";
+  OS << "  using namespace llvm::MCD;\n";
 
   OS << R"(
   const uint8_t *Ptr = DecodeTable;
@@ -2420,16 +2315,16 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
   }
   if (HasSoftFail) {
     OS << R"(
-      case MCD::OPC_SoftFail: {
-        // Decode the mask values.
-        uint64_t PositiveMask = decodeULEB128AndIncUnsafe(Ptr);
-        uint64_t NegativeMask = decodeULEB128AndIncUnsafe(Ptr);
-        bool Failed = (insn & PositiveMask) != 0 || (~insn & NegativeMask) != 0;
-        if (Failed)
-          S = MCDisassembler::SoftFail;
-        LLVM_DEBUG(dbgs() << Loc << ": OPC_SoftFail: " << (Failed ? "FAIL\n" : "PASS\n"));
-        break;
-  })";
+    case MCD::OPC_SoftFail: {
+      // Decode the mask values.
+      uint64_t PositiveMask = decodeULEB128AndIncUnsafe(Ptr);
+      uint64_t NegativeMask = decodeULEB128AndIncUnsafe(Ptr);
+      bool Failed = (insn & PositiveMask) != 0 || (~insn & NegativeMask) != 0;
+      if (Failed)
+        S = MCDisassembler::SoftFail;
+      LLVM_DEBUG(dbgs() << Loc << ": OPC_SoftFail: " << (Failed ? "FAIL\n" : "PASS\n"));
+      break;
+    })";
   }
   OS << R"(
     case MCD::OPC_Fail: {
@@ -2444,83 +2339,179 @@ static DecodeStatus decodeInstruction(const uint8_t DecodeTable[], MCInst &MI,
 )";
 }
 
-// Helper to propagate SoftFail status. Returns false if the status is Fail;
-// callers are expected to early-exit in that condition. (Note, the '&' operator
-// is correct to propagate the values of this enum; see comment on 'enum
-// DecodeStatus'.)
-static void emitCheck(formatted_raw_ostream &OS) {
-  OS << R"(
-static bool Check(DecodeStatus &Out, DecodeStatus In) {
-  Out = static_cast<DecodeStatus>(Out & In);
-  return Out != MCDisassembler::Fail;
-}
-
-)";
-}
-
-// Collect all HwModes referenced by the target for encoding purposes,
-// returning a vector of corresponding names.
-static void collectHwModesReferencedForEncodings(
-    const CodeGenHwModes &HWM, std::vector<StringRef> &Names,
-    NamespacesHwModesMap &NamespacesWithHwModes) {
-  SmallBitVector BV(HWM.getNumModeIds());
-  for (const auto &MS : HWM.getHwModeSelects()) {
-    for (const HwModeSelect::PairType &P : MS.second.Items) {
-      if (P.second->isSubClassOf("InstructionEncoding")) {
+/// Collects all HwModes referenced by the target for encoding purposes.
+void DecoderEmitter::collectHwModesReferencedForEncodings(
+    std::vector<unsigned> &HwModeIDs,
+    NamespacesHwModesMap &NamespacesWithHwModes) const {
+  SmallBitVector BV(CGH.getNumModeIds());
+  for (const auto &MS : CGH.getHwModeSelects()) {
+    for (auto [HwModeID, EncodingDef] : MS.second.Items) {
+      if (EncodingDef->isSubClassOf("InstructionEncoding")) {
         std::string DecoderNamespace =
-            P.second->getValueAsString("DecoderNamespace").str();
-        if (P.first == DefaultMode) {
-          NamespacesWithHwModes[DecoderNamespace].insert("");
-        } else {
-          NamespacesWithHwModes[DecoderNamespace].insert(
-              HWM.getMode(P.first).Name);
-        }
-        BV.set(P.first);
+            EncodingDef->getValueAsString("DecoderNamespace").str();
+        NamespacesWithHwModes[DecoderNamespace].insert(HwModeID);
+        BV.set(HwModeID);
       }
     }
   }
-  transform(BV.set_bits(), std::back_inserter(Names), [&HWM](const int &M) {
-    if (M == DefaultMode)
-      return StringRef("");
-    return HWM.getModeName(M, /*IncludeDefault=*/true);
-  });
+  // FIXME: Can't do `HwModeIDs.assign(BV.set_bits_begin(), BV.set_bits_end())`
+  //   because const_set_bits_iterator_impl is not copy-assignable.
+  //   This breaks some MacOS builds.
+  llvm::copy(BV.set_bits(), std::back_inserter(HwModeIDs));
 }
 
-static void
-handleHwModesUnrelatedEncodings(const CodeGenInstruction *Instr,
-                                ArrayRef<StringRef> HwModeNames,
-                                NamespacesHwModesMap &NamespacesWithHwModes,
-                                std::vector<EncodingAndInst> &GlobalEncodings) {
-  const Record *InstDef = Instr->TheDef;
-
+void DecoderEmitter::handleHwModesUnrelatedEncodings(
+    unsigned EncodingID, ArrayRef<unsigned> HwModeIDs,
+    NamespacesHwModesMap &NamespacesWithHwModes) {
   switch (DecoderEmitterSuppressDuplicates) {
   case SUPPRESSION_DISABLE: {
-    for (StringRef HwModeName : HwModeNames)
-      GlobalEncodings.emplace_back(InstDef, Instr, HwModeName);
+    for (unsigned HwModeID : HwModeIDs)
+      EncodingIDsByHwMode[HwModeID].push_back(EncodingID);
     break;
   }
   case SUPPRESSION_LEVEL1: {
+    const Record *InstDef = Encodings[EncodingID].getInstruction()->TheDef;
     std::string DecoderNamespace =
         InstDef->getValueAsString("DecoderNamespace").str();
     auto It = NamespacesWithHwModes.find(DecoderNamespace);
     if (It != NamespacesWithHwModes.end()) {
-      for (StringRef HwModeName : It->second)
-        GlobalEncodings.emplace_back(InstDef, Instr, HwModeName);
+      for (unsigned HwModeID : It->second)
+        EncodingIDsByHwMode[HwModeID].push_back(EncodingID);
     } else {
       // Only emit the encoding once, as it's DecoderNamespace doesn't
       // contain any HwModes.
-      GlobalEncodings.emplace_back(InstDef, Instr, "");
+      EncodingIDsByHwMode[DefaultMode].push_back(EncodingID);
     }
     break;
   }
   case SUPPRESSION_LEVEL2:
-    GlobalEncodings.emplace_back(InstDef, Instr, "");
+    EncodingIDsByHwMode[DefaultMode].push_back(EncodingID);
     break;
   }
 }
 
+/// Checks if the given target-specific non-pseudo instruction
+/// is a candidate for decoding.
+static bool isDecodableInstruction(const Record *InstDef) {
+  return !InstDef->getValueAsBit("isAsmParserOnly") &&
+         !InstDef->getValueAsBit("isCodeGenOnly");
+}
+
+/// Checks if the given encoding is valid.
+static bool isValidEncoding(const Record *EncodingDef) {
+  const RecordVal *InstField = EncodingDef->getValue("Inst");
+  if (!InstField)
+    return false;
+
+  if (const auto *InstInit = dyn_cast<BitsInit>(InstField->getValue())) {
+    // Fixed-length encoding. Size must be non-zero.
+    if (!EncodingDef->getValueAsInt("Size"))
+      return false;
+
+    // At least one of the encoding bits must be complete (not '?').
+    // FIXME: This should take SoftFail field into account.
+    return !InstInit->allInComplete();
+  }
+
+  if (const auto *InstInit = dyn_cast<DagInit>(InstField->getValue())) {
+    // Variable-length encoding.
+    // At least one of the encoding bits must be complete (not '?').
+    VarLenInst VLI(InstInit, InstField);
+    return !all_of(VLI, [](const EncodingSegment &Segment) {
+      return isa<UnsetInit>(Segment.Value);
+    });
+  }
+
+  // Inst field is neither BitsInit nor DagInit. This is something unsupported.
+  return false;
+}
+
+/// Parses all InstructionEncoding instances and fills internal data structures.
+void DecoderEmitter::parseInstructionEncodings() {
+  // First, collect all encoding-related HwModes referenced by the target.
+  // And establish a mapping table between DecoderNamespace and HwMode.
+  // If HwModeNames is empty, add the default mode so we always have one HwMode.
+  std::vector<unsigned> HwModeIDs;
+  NamespacesHwModesMap NamespacesWithHwModes;
+  collectHwModesReferencedForEncodings(HwModeIDs, NamespacesWithHwModes);
+  if (HwModeIDs.empty())
+    HwModeIDs.push_back(DefaultMode);
+
+  ArrayRef<const CodeGenInstruction *> Instructions =
+      Target.getTargetNonPseudoInstructions();
+  Encodings.reserve(Instructions.size());
+
+  for (const CodeGenInstruction *Inst : Instructions) {
+    const Record *InstDef = Inst->TheDef;
+    if (!isDecodableInstruction(InstDef)) {
+      ++NumEncodingsLackingDisasm;
+      continue;
+    }
+
+    if (const Record *RV = InstDef->getValueAsOptionalDef("EncodingInfos")) {
+      EncodingInfoByHwMode EBM(RV, CGH);
+      for (auto [HwModeID, EncodingDef] : EBM) {
+        if (!isValidEncoding(EncodingDef)) {
+          // TODO: Should probably give a warning.
+          ++NumEncodingsOmitted;
+          continue;
+        }
+        unsigned EncodingID = Encodings.size();
+        Encodings.emplace_back(EncodingDef, Inst);
+        EncodingIDsByHwMode[HwModeID].push_back(EncodingID);
+      }
+      continue; // Ignore encoding specified by Instruction itself.
+    }
+
+    if (!isValidEncoding(InstDef)) {
+      ++NumEncodingsOmitted;
+      continue;
+    }
+
+    unsigned EncodingID = Encodings.size();
+    Encodings.emplace_back(InstDef, Inst);
+
+    // This instruction is encoded the same on all HwModes.
+    // According to user needs, add it to all, some, or only the default HwMode.
+    handleHwModesUnrelatedEncodings(EncodingID, HwModeIDs,
+                                    NamespacesWithHwModes);
+  }
+
+  for (const Record *EncodingDef :
+       RK.getAllDerivedDefinitions("AdditionalEncoding")) {
+    const Record *InstDef = EncodingDef->getValueAsDef("AliasOf");
+    // TODO: Should probably give a warning in these cases.
+    //   What's the point of specifying an additional encoding
+    //   if it is invalid or if the instruction is not decodable?
+    if (!isDecodableInstruction(InstDef)) {
+      ++NumEncodingsLackingDisasm;
+      continue;
+    }
+    if (!isValidEncoding(EncodingDef)) {
+      ++NumEncodingsOmitted;
+      continue;
+    }
+    unsigned EncodingID = Encodings.size();
+    Encodings.emplace_back(EncodingDef, &Target.getInstruction(InstDef));
+    EncodingIDsByHwMode[DefaultMode].push_back(EncodingID);
+  }
+
+  // Do some statistics.
+  NumInstructions = Instructions.size();
+  NumEncodingsSupported = Encodings.size();
+  NumEncodings = NumEncodingsSupported + NumEncodingsOmitted;
+}
+
+DecoderEmitter::DecoderEmitter(const RecordKeeper &RK,
+                               StringRef PredicateNamespace)
+    : RK(RK), Target(RK), CGH(Target.getHwModes()),
+      PredicateNamespace(PredicateNamespace) {
+  Target.reverseBitsForLittleEndianEncoding();
+  parseInstructionEncodings();
+}
+
 // Emits disassembler code for instruction decoding.
-void DecoderEmitter::run(raw_ostream &o) {
+void DecoderEmitter::run(raw_ostream &o) const {
   formatted_raw_ostream OS(o);
   OS << R"(
 #include "llvm/MC/MCInst.h"
@@ -2535,130 +2526,58 @@ void DecoderEmitter::run(raw_ostream &o) {
 namespace {
 )";
 
-  emitFieldFromInstruction(OS);
-  emitInsertBits(OS);
-  emitCheck(OS);
-
-  Target.reverseBitsForLittleEndianEncoding();
-
-  // Parameterize the decoders based on namespace and instruction width.
-
-  // First, collect all encoding-related HwModes referenced by the target.
-  // And establish a mapping table between DecoderNamespace and HwMode.
-  // If HwModeNames is empty, add the empty string so we always have one HwMode.
-  const CodeGenHwModes &HWM = Target.getHwModes();
-  std::vector<StringRef> HwModeNames;
-  NamespacesHwModesMap NamespacesWithHwModes;
-  collectHwModesReferencedForEncodings(HWM, HwModeNames, NamespacesWithHwModes);
-  if (HwModeNames.empty())
-    HwModeNames.push_back("");
-
-  const auto &NumberedInstructions = Target.getInstructions();
-  NumberedEncodings.reserve(NumberedInstructions.size());
-  for (const auto &NumberedInstruction : NumberedInstructions) {
-    const Record *InstDef = NumberedInstruction->TheDef;
-    if (const RecordVal *RV = InstDef->getValue("EncodingInfos")) {
-      if (const DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
-        EncodingInfoByHwMode EBM(DI->getDef(), HWM);
-        for (auto &[ModeId, Encoding] : EBM) {
-          // DecoderTables with DefaultMode should not have any suffix.
-          if (ModeId == DefaultMode) {
-            NumberedEncodings.emplace_back(Encoding, NumberedInstruction, "");
-          } else {
-            NumberedEncodings.emplace_back(Encoding, NumberedInstruction,
-                                           HWM.getMode(ModeId).Name);
-          }
-        }
-        continue;
-      }
-    }
-    // This instruction is encoded the same on all HwModes.
-    // According to user needs, provide varying degrees of suppression.
-    handleHwModesUnrelatedEncodings(NumberedInstruction, HwModeNames,
-                                    NamespacesWithHwModes, NumberedEncodings);
-  }
-  for (const Record *NumberedAlias :
-       RK.getAllDerivedDefinitions("AdditionalEncoding"))
-    NumberedEncodings.emplace_back(
-        NumberedAlias,
-        &Target.getInstruction(NumberedAlias->getValueAsDef("AliasOf")));
-
-  std::map<std::pair<std::string, unsigned>, std::vector<EncodingIDAndOpcode>>
-      OpcMap;
-  std::map<unsigned, std::vector<OperandInfo>> Operands;
+  // Do extra bookkeeping for variable-length encodings.
   std::vector<unsigned> InstrLen;
   bool IsVarLenInst = Target.hasVariableLengthEncodings();
   unsigned MaxInstLen = 0;
-
-  for (const auto &[NEI, NumberedEncoding] : enumerate(NumberedEncodings)) {
-    const Record *EncodingDef = NumberedEncoding.EncodingDef;
-    const CodeGenInstruction *Inst = NumberedEncoding.Inst;
-    const Record *Def = Inst->TheDef;
-    unsigned Size = EncodingDef->getValueAsInt("Size");
-    if (Def->getValueAsString("Namespace") == "TargetOpcode" ||
-        Def->getValueAsBit("isPseudo") ||
-        Def->getValueAsBit("isAsmParserOnly") ||
-        Def->getValueAsBit("isCodeGenOnly")) {
-      NumEncodingsLackingDisasm++;
-      continue;
+  if (IsVarLenInst) {
+    InstrLen.resize(Target.getInstructions().size(), 0);
+    for (const InstructionEncoding &Encoding : Encodings) {
+      MaxInstLen = std::max(MaxInstLen, Encoding.getBitWidth());
+      InstrLen[Target.getInstrIntValue(Encoding.getInstruction()->TheDef)] =
+          Encoding.getBitWidth();
     }
+  }
 
-    if (NEI < NumberedInstructions.size())
-      NumInstructions++;
-    NumEncodings++;
-
-    if (!Size && !IsVarLenInst)
-      continue;
-
-    if (IsVarLenInst)
-      InstrLen.resize(NumberedInstructions.size(), 0);
-
-    if (unsigned Len = populateInstruction(Target, *EncodingDef, *Inst, NEI,
-                                           Operands, IsVarLenInst)) {
-      if (IsVarLenInst) {
-        MaxInstLen = std::max(MaxInstLen, Len);
-        InstrLen[NEI] = Len;
-      }
-      std::string DecoderNamespace =
-          EncodingDef->getValueAsString("DecoderNamespace").str();
-      if (!NumberedEncoding.HwModeName.empty())
-        DecoderNamespace += "_" + NumberedEncoding.HwModeName.str();
-      OpcMap[{DecoderNamespace, Size}].emplace_back(
-          NEI, Target.getInstrIntValue(Def));
-    } else {
-      NumEncodingsOmitted++;
+  // Map of (namespace, hwmode, size) tuple to encoding IDs.
+  std::map<std::tuple<StringRef, unsigned, unsigned>, std::vector<unsigned>>
+      EncMap;
+  for (const auto &[HwModeID, EncodingIDs] : EncodingIDsByHwMode) {
+    for (unsigned EncodingID : EncodingIDs) {
+      const InstructionEncoding &Encoding = Encodings[EncodingID];
+      const Record *EncodingDef = Encoding.getRecord();
+      unsigned Size = EncodingDef->getValueAsInt("Size");
+      StringRef DecoderNamespace =
+          EncodingDef->getValueAsString("DecoderNamespace");
+      EncMap[{DecoderNamespace, HwModeID, Size}].push_back(EncodingID);
     }
   }
 
   DecoderTableInfo TableInfo;
   unsigned OpcodeMask = 0;
-  for (const auto &[NSAndByteSize, EncodingIDs] : OpcMap) {
-    const std::string &DecoderNamespace = NSAndByteSize.first;
-    const unsigned BitWidth = 8 * NSAndByteSize.second;
-    // Emit the decoder for this namespace+width combination.
-    FilterChooser FC(NumberedEncodings, EncodingIDs, Operands,
-                     IsVarLenInst ? MaxInstLen : BitWidth, this);
+
+  for (const auto &[Key, EncodingIDs] : EncMap) {
+    auto [DecoderNamespace, HwModeID, Size] = Key;
+    const unsigned BitWidth = IsVarLenInst ? MaxInstLen : 8 * Size;
+    // Emit the decoder for this (namespace, hwmode, width) combination.
+    FilterChooser FC(Encodings, EncodingIDs, BitWidth, this);
 
     // The decode table is cleared for each top level decoder function. The
     // predicates and decoders themselves, however, are shared across all
     // decoders to give more opportunities for uniqueing.
     TableInfo.Table.clear();
-    TableInfo.FixupStack.clear();
-    TableInfo.FixupStack.emplace_back();
+    TableInfo.pushScope();
     FC.emitTableEntries(TableInfo);
     // Any NumToSkip fixups in the top level scope can resolve to the
     // OPC_Fail at the end of the table.
-    assert(TableInfo.FixupStack.size() == 1 && "fixup stack phasing error!");
-    // Resolve any NumToSkip fixups in the current scope.
-    resolveTableFixups(TableInfo.Table, TableInfo.FixupStack.back(),
-                       TableInfo.Table.size());
-    TableInfo.FixupStack.clear();
+    assert(TableInfo.isOutermostScope() && "fixup stack phasing error!");
+    TableInfo.popScope();
 
-    TableInfo.Table.push_back(MCD::OPC_Fail);
+    TableInfo.Table.insertOpcode(MCD::OPC_Fail);
 
     // Print the table to the output stream.
-    OpcodeMask |= emitTable(OS, TableInfo.Table, FC.getBitWidth(),
-                            DecoderNamespace, EncodingIDs);
+    OpcodeMask |= emitTable(OS, TableInfo.Table, DecoderNamespace, HwModeID,
+                            BitWidth, EncodingIDs);
   }
 
   // For variable instruction, we emit a instruction length table
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index 6f72b51963f87..26d93fc13c9ba 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -223,17 +223,104 @@ void InstrInfoEmitter::EmitOperandInfo(raw_ostream &OS,
   }
 }
 
+static void emitGetInstructionIndexForOpLookup(
+    raw_ostream &OS, const MapVector<SmallVector<int>, unsigned> &OperandMap,
+    ArrayRef<unsigned> InstructionIndex) {
+  StringRef Type = OperandMap.size() <= UINT8_MAX + 1 ? "uint8_t" : "uint16_t";
+  OS << "LLVM_READONLY static " << Type
+     << " getInstructionIndexForOpLookup(uint16_t Opcode) {\n"
+        "  static constexpr "
+     << Type << " InstructionIndex[] = {";
+  for (auto [TableIndex, Entry] : enumerate(InstructionIndex))
+    OS << (TableIndex % 16 == 0 ? "\n    " : " ") << Entry << ',';
+  OS << "\n  };\n"
+        "  return InstructionIndex[Opcode];\n"
+        "}\n";
+}
+
+static void
+emitGetNamedOperandIdx(raw_ostream &OS,
+                       const MapVector<SmallVector<int>, unsigned> &OperandMap,
+                       unsigned MaxOperandNo, unsigned NumOperandNames) {
+  OS << "LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, OpName "
+        "Name) {\n";
+  OS << "  assert(Name != OpName::NUM_OPERAND_NAMES);\n";
+  if (!NumOperandNames) {
+    // There are no operands, so no need to emit anything
+    OS << "  return -1;\n}\n";
+    return;
+  }
+  assert(MaxOperandNo <= INT16_MAX &&
+         "Too many operands for the operand name -> index table");
+  StringRef Type = MaxOperandNo <= INT8_MAX ? "int8_t" : "int16_t";
+  OS << "  static constexpr " << Type << " OperandMap[][" << NumOperandNames
+     << "] = {\n";
+  for (const auto &[OpList, _] : OperandMap) {
+    // Emit a row of the OperandMap table.
+    OS << "    {";
+    for (unsigned ID = 0; ID < NumOperandNames; ++ID)
+      OS << (ID < OpList.size() ? OpList[ID] : -1) << ", ";
+    OS << "},\n";
+  }
+  OS << "  };\n";
+
+  OS << "  unsigned InstrIdx = getInstructionIndexForOpLookup(Opcode);\n"
+        "  return OperandMap[InstrIdx][(unsigned)Name];\n"
+        "}\n";
+}
+
+static void
+emitGetOperandIdxName(raw_ostream &OS,
+                      MapVector<StringRef, unsigned> OperandNameToID,
+                      const MapVector<SmallVector<int>, unsigned> &OperandMap,
+                      unsigned MaxNumOperands, unsigned NumOperandNames) {
+  OS << "LLVM_READONLY OpName getOperandIdxName(uint16_t Opcode, int16_t Idx) "
+        "{\n";
+  OS << "  assert(Idx >= 0 && Idx < " << MaxNumOperands << ");\n";
+  if (!MaxNumOperands) {
+    // There are no operands, so no need to emit anything
+    OS << "  return -1;\n}\n";
+    return;
+  }
+  OS << "  static constexpr OpName OperandMap[][" << MaxNumOperands
+     << "] = {\n";
+  for (const auto &[OpList, _] : OperandMap) {
+    SmallVector<unsigned> IDs(MaxNumOperands, NumOperandNames);
+    for (const auto &[ID, Idx] : enumerate(OpList)) {
+      if (Idx >= 0)
+        IDs[Idx] = ID;
+    }
+    // Emit a row of the OperandMap table. Map operand indices to enum values.
+    OS << "    {";
+    for (unsigned ID : IDs) {
+      if (ID == NumOperandNames)
+        OS << "OpName::NUM_OPERAND_NAMES, ";
+      else
+        OS << "OpName::" << OperandNameToID.getArrayRef()[ID].first << ", ";
+    }
+    OS << "},\n";
+  }
+  OS << "  };\n";
+
+  OS << "  unsigned InstrIdx = getInstructionIndexForOpLookup(Opcode);\n"
+        "  return OperandMap[InstrIdx][(unsigned)Idx];\n"
+        "}\n";
+}
+
 /// Generate a table and function for looking up the indices of operands by
 /// name.
 ///
 /// This code generates:
 /// - An enum in the llvm::TargetNamespace::OpName namespace, with one entry
 ///   for each operand name.
-/// - A 2-dimensional table called OperandMap for mapping OpName enum values to
-///   operand indices.
-/// - A function called getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
+/// - A 2-dimensional table for mapping OpName enum values to operand indices.
+/// - A function called getNamedOperandIdx(uint16_t Opcode, OpName Name)
 ///   for looking up the operand index for an instruction, given a value from
 ///   OpName enum
+/// - A 2-dimensional table for mapping operand indices to OpName enum values.
+/// - A function called getOperandIdxName(uint16_t Opcode, int16_t Idx)
+///   for looking up the OpName enum for an instruction, given the operand
+///   index. This is the inverse of getNamedOperandIdx().
 ///
 /// Fixed/Predefined instructions do not have UseNamedOperandTable enabled, so
 /// we can just skip them. Hence accept just the TargetInstructions.
@@ -242,11 +329,6 @@ void InstrInfoEmitter::emitOperandNameMappings(
     ArrayRef<const CodeGenInstruction *> TargetInstructions) {
   StringRef Namespace = Target.getInstNamespace();
 
-  /// To facilitate assigning OpName enum values in the sorted alphabetical
-  /// order, we go through an indirection from OpName -> ID, and Enum -> ID.
-  /// This allows us to build the OpList and assign IDs to OpNames in a single
-  /// scan of the instructions below.
-
   // Map of operand names to their ID.
   MapVector<StringRef, unsigned> OperandNameToID;
 
@@ -285,53 +367,38 @@ void InstrInfoEmitter::emitOperandNameMappings(
   }
 
   const size_t NumOperandNames = OperandNameToID.size();
+  const unsigned MaxNumOperands = MaxOperandNo + 1;
 
   OS << "#ifdef GET_INSTRINFO_OPERAND_ENUM\n";
   OS << "#undef GET_INSTRINFO_OPERAND_ENUM\n";
   OS << "namespace llvm::" << Namespace << " {\n";
-  OS << "enum class OpName {\n";
+
+  assert(NumOperandNames <= UINT16_MAX &&
+         "Too many operands for the operand index -> name table");
+  StringRef EnumType = getMinimalTypeForRange(NumOperandNames);
+  OS << "enum class OpName : " << EnumType << " {\n";
   for (const auto &[Op, I] : OperandNameToID)
     OS << "  " << Op << " = " << I << ",\n";
   OS << "  NUM_OPERAND_NAMES = " << NumOperandNames << ",\n";
   OS << "}; // enum class OpName\n\n";
-  OS << "LLVM_READONLY\n";
-  OS << "int16_t getNamedOperandIdx(uint16_t Opcode, OpName Name);\n";
+
+  OS << "LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, OpName "
+        "Name);\n";
+  OS << "LLVM_READONLY OpName getOperandIdxName(uint16_t Opcode, int16_t "
+        "Idx);\n";
   OS << "} // end namespace llvm::" << Namespace << '\n';
   OS << "#endif //GET_INSTRINFO_OPERAND_ENUM\n\n";
 
   OS << "#ifdef GET_INSTRINFO_NAMED_OPS\n";
   OS << "#undef GET_INSTRINFO_NAMED_OPS\n";
   OS << "namespace llvm::" << Namespace << " {\n";
-  OS << "LLVM_READONLY\n";
-  OS << "int16_t getNamedOperandIdx(uint16_t Opcode, OpName Name) {\n";
-  OS << "  assert(Name != OpName::NUM_OPERAND_NAMES);\n";
-  if (NumOperandNames != 0) {
-    assert(MaxOperandNo <= INT16_MAX &&
-           "Too many operands for the operand name -> index table");
-    StringRef Type = MaxOperandNo <= INT8_MAX ? "int8_t" : "int16_t";
-    OS << "  static constexpr " << Type << " OperandMap[][" << NumOperandNames
-       << "] = {\n";
-    for (const auto &[OpList, _] : OperandMap) {
-      // Emit a row of the OperandMap table.
-      OS << "    {";
-      for (unsigned ID = 0; ID < NumOperandNames; ++ID)
-        OS << (ID < OpList.size() ? OpList[ID] : -1) << ", ";
-      OS << "},\n";
-    }
-    OS << "  };\n";
 
-    Type = OperandMap.size() <= UINT8_MAX + 1 ? "uint8_t" : "uint16_t";
-    OS << "  static constexpr " << Type << " InstructionIndex[] = {";
-    for (auto [TableIndex, Entry] : enumerate(InstructionIndex))
-      OS << (TableIndex % 16 == 0 ? "\n    " : " ") << Entry << ',';
-    OS << "\n  };\n";
+  emitGetInstructionIndexForOpLookup(OS, OperandMap, InstructionIndex);
+
+  emitGetNamedOperandIdx(OS, OperandMap, MaxOperandNo, NumOperandNames);
+  emitGetOperandIdxName(OS, OperandNameToID, OperandMap, MaxNumOperands,
+                        NumOperandNames);
 
-    OS << "  return OperandMap[InstructionIndex[Opcode]][(unsigned)Name];\n";
-  } else {
-    // There are no operands, so no need to emit anything
-    OS << "  return -1;\n";
-  }
-  OS << "}\n";
   OS << "} // end namespace llvm::" << Namespace << '\n';
   OS << "#endif //GET_INSTRINFO_NAMED_OPS\n\n";
 }
diff --git a/llvm/utils/UpdateTestChecks/asm.py b/llvm/utils/UpdateTestChecks/asm.py
index 98b4b5da8a3df..27b2aa8d0bdbe 100644
--- a/llvm/utils/UpdateTestChecks/asm.py
+++ b/llvm/utils/UpdateTestChecks/asm.py
@@ -32,7 +32,7 @@ class string:
 )
 
 ASM_FUNCTION_ARM_RE = re.compile(
-    r'^(?P<func>[0-9a-zA-Z_$]+):[ \t]*@+[ \t]*@"?(?P=func)"?\n'  # f: (name of function)
+    r'^(?P<func>[0-9a-zA-Z_$]+):[ \t]*(@+[ \t]*@"?(?P=func)"?)?\n'  # f: (name of function)
     r"(?:\.L(?P=func)\$local:\n)?"  # drop .L<func>$local:
     r"(?:\s*\.type\s+\.L(?P=func)\$local,@function\n)?"  # drop .type .L<func>$local
     r"\s+\.fnstart\n"  # .fnstart
@@ -42,11 +42,11 @@ class string:
 )
 
 ASM_FUNCTION_AARCH64_RE = re.compile(
-    r'^_?(?P<func>[^:]+):[ \t]*\/\/[ \t]*@"?(?P=func)"?( (Function|Tail Call))?\n'
+    r'^(?P<func>_?[a-zA-Z][^:]*):[ \t]*(\/\/[ \t]*@"?(?P=func)"?)?( (Function|Tail Call))?\n'
     r"(?:[ \t]+.cfi_startproc\n)?"  # drop optional cfi noise
-    r"(?P<body>.*?)\n"
+    r"(?P<body>.*?)"
     # This list is incomplete
-    r"^\s*(\.Lfunc_end[0-9]+|// -- End function)",
+    r"^(\.Lfunc_end[0-9]+:|// -- End function)",
     flags=(re.M | re.S),
 )
 
@@ -173,7 +173,7 @@ class string:
 )
 
 ASM_FUNCTION_ARM_MACHO_RE = re.compile(
-    r'^_(?P<func>[^:]+):[ \t]*@[ \t]@"?(?P=func)"?\n'
+    r'^_(?P<func>[^:]+):[ \t]*(@[ \t]@"?(?P=func)"?)?\n'
     r"([ \t]*.cfi_startproc\n[ \t]*)?"
     r"(?P<body>.*?)\n"
     r"[ \t]*\.cfi_endproc\n",
@@ -181,21 +181,21 @@ class string:
 )
 
 ASM_FUNCTION_THUMBS_DARWIN_RE = re.compile(
-    r'^_(?P<func>[^:]+):[ \t]*@[ \t]@"?(?P=func)"?\n'
+    r'^_(?P<func>[^:]+):[ \t]*(@[ \t]@"?(?P=func)"?)?\n'
     r"(?P<body>.*?)\n"
     r"[ \t]*\.data_region\n",
     flags=(re.M | re.S),
 )
 
 ASM_FUNCTION_THUMB_DARWIN_RE = re.compile(
-    r'^_(?P<func>[^:]+):[ \t]*@[ \t]@"?(?P=func)"?\n'
+    r'^_(?P<func>[^:]+):[ \t]*(@[ \t]@"?(?P=func)"?)?\n'
     r"(?P<body>.*?)\n"
     r"^[ \t]*@[ \t]--[ \t]End[ \t]function",
     flags=(re.M | re.S),
 )
 
 ASM_FUNCTION_ARM_IOS_RE = re.compile(
-    r'^_(?P<func>[^:]+):[ \t]*@[ \t]@"?(?P=func)"?\n'
+    r'^_(?P<func>[^:]+):[ \t]*(@[ \t]@"?(?P=func)"?)?\n'
     r"(?P<body>.*?)"
     r"^[ \t]*@[ \t]--[ \t]End[ \t]function",
     flags=(re.M | re.S),
diff --git a/llvm/utils/UpdateTestChecks/common.py b/llvm/utils/UpdateTestChecks/common.py
index 178c623e33e0e..835092c468695 100644
--- a/llvm/utils/UpdateTestChecks/common.py
+++ b/llvm/utils/UpdateTestChecks/common.py
@@ -36,6 +36,7 @@
     "Branch Probability Analysis",
     "Cost Model Analysis",
     "Dependence Analysis",
+    "Delinearization",
     "Loop Access Analysis",
     "Scalar Evolution Analysis",
 }
@@ -878,12 +879,17 @@ def is_filtered(self):
         return False
 
     def process_run_line(self, function_re, scrubber, raw_tool_output, prefixes):
+        """
+        Returns the number of functions processed from the output by the regex.
+        """
         build_global_values_dictionary(
             self._global_var_dict, raw_tool_output, prefixes, self._ginfo
         )
+        processed_func_count = 0
         for m in function_re.finditer(raw_tool_output):
             if not m:
                 continue
+            processed_func_count += 1
             func = m.group("func")
             body = m.group("body")
             # func_name_separator is the string that is placed right after function name at the
@@ -1000,6 +1006,7 @@ def process_run_line(self, function_re, scrubber, raw_tool_output, prefixes):
                         # preprocesser directives to exclude individual functions from some
                         # RUN lines.
                         self._func_dict[prefix][func] = None
+        return processed_func_count
 
     def processed_prefixes(self, prefixes):
         """
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
index 4f3ef5bf174b7..da380c9c1f438 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
@@ -33,6 +33,7 @@ static_library("cppcoreguidelines") {
     "OwningMemoryCheck.cpp",
     "PreferMemberInitializerCheck.cpp",
     "ProBoundsArrayToPointerDecayCheck.cpp",
+    "ProBoundsAvoidUncheckedContainerAccess.cpp",
     "ProBoundsConstantArrayIndexCheck.cpp",
     "ProBoundsPointerArithmeticCheck.cpp",
     "ProTypeConstCastCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
index 0dc5efc981c87..a6848b3c9f241 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
@@ -46,6 +46,7 @@ static_library("misc") {
     "NoRecursionCheck.cpp",
     "NonCopyableObjects.cpp",
     "NonPrivateMemberVariablesInClassesCheck.cpp",
+    "OverrideWithDifferentVisibilityCheck.cpp",
     "RedundantExpressionCheck.cpp",
     "StaticAssertCheck.cpp",
     "ThrowByValueCatchByReferenceCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
index b609d4a7462fb..f8c4838ab7ee3 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
@@ -122,6 +122,7 @@ static_library("clangd") {
     "SemanticHighlighting.cpp",
     "SemanticSelection.cpp",
     "SourceCode.cpp",
+    "SymbolDocumentation.cpp",
     "SystemIncludeExtractor.cpp",
     "TUScheduler.cpp",
     "TidyProvider.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
index ad32aa9ce7bf2..8aba04a4fc47d 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
@@ -108,6 +108,7 @@ unittest("ClangdTests") {
     "SourceCodeTests.cpp",
     "StdLibTests.cpp",
     "SymbolCollectorTests.cpp",
+    "SymbolDocumentationTests.cpp",
     "SymbolInfoTests.cpp",
     "SyncAPI.cpp",
     "TUSchedulerTests.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Analysis/FlowSensitive/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Analysis/FlowSensitive/BUILD.gn
index 0b6fa7cc5f5ce..74b2fe204537a 100644
--- a/llvm/utils/gn/secondary/clang/lib/Analysis/FlowSensitive/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Analysis/FlowSensitive/BUILD.gn
@@ -31,6 +31,7 @@ static_library("FlowSensitive") {
     "DataflowEnvironment.cpp",
     "DebugSupport.cpp",
     "Formula.cpp",
+    "FormulaSerialization.cpp",
     "HTMLLogger.cpp",
     "Logger.cpp",
     "RecordOps.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Format/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Format/BUILD.gn
index 58cd59a7c941b..93c0f3c51fe89 100644
--- a/llvm/utils/gn/secondary/clang/lib/Format/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Format/BUILD.gn
@@ -21,6 +21,7 @@ static_library("Format") {
     "MacroExpander.cpp",
     "MatchFilePath.cpp",
     "NamespaceEndCommentsFixer.cpp",
+    "NumericLiteralInfo.cpp",
     "ObjCPropertyAttributeOrderFixer.cpp",
     "QualifierAlignmentFixer.cpp",
     "SortJavaScriptImports.cpp",
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index ed68387321d53..020e35e2fd467 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -60,7 +60,6 @@ write_lit_config("lit_site_cfg") {
     "CLANG_VENDOR_UTI=org.llvm.clang",
     "ENABLE_BACKTRACES=1",
     "ENABLE_SHARED=0",
-    "LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS=0",
     "LLVM_EXTERNAL_LIT=",
     "LLVM_HOST_TRIPLE=$llvm_current_triple",
     "LLVM_INCLUDE_SPIRV_TOOLS_TESTS=0",
diff --git a/llvm/utils/gn/secondary/clang/unittests/Analysis/FlowSensitive/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Analysis/FlowSensitive/BUILD.gn
index e4727d5a3298c..1afd342f67ce4 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Analysis/FlowSensitive/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Analysis/FlowSensitive/BUILD.gn
@@ -27,6 +27,7 @@ unittest("ClangAnalysisFlowSensitiveTests") {
     "DataflowEnvironmentTest.cpp",
     "DebugSupportTest.cpp",
     "DeterminismTest.cpp",
+    "FormulaTest.cpp",
     "LoggerTest.cpp",
     "MapLatticeTest.cpp",
     "MatchSwitchTest.cpp",
diff --git a/llvm/utils/gn/secondary/clang/unittests/Format/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Format/BUILD.gn
index b35061970c226..0b3f00c3f0518 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Format/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Format/BUILD.gn
@@ -37,6 +37,7 @@ unittest("FormatTests") {
     "MacroExpanderTest.cpp",
     "MatchFilePathTest.cpp",
     "NamespaceEndCommentsFixerTest.cpp",
+    "NumericLiteralInfoTest.cpp",
     "ObjCPropertyAttributeOrderFixerTest.cpp",
     "QualifierFixerTest.cpp",
     "SortImportsTestJS.cpp",
diff --git a/llvm/utils/gn/secondary/clang/unittests/Lex/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Lex/BUILD.gn
index 16abe7a6e95e4..a0f72494a2bd9 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Lex/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Lex/BUILD.gn
@@ -20,6 +20,7 @@ unittest("LexTests") {
     "LexHLSLRootSignatureTest.cpp",
     "LexerTest.cpp",
     "ModuleDeclStateTest.cpp",
+    "NoTrivialPPDirectiveTracerTest.cpp",
     "PPCallbacksTest.cpp",
     "PPConditionalDirectiveRecordTest.cpp",
     "PPDependencyDirectivesTest.cpp",
diff --git a/llvm/utils/gn/secondary/compiler-rt/lib/asan/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/lib/asan/BUILD.gn
index 42467c21aa24c..1d24d049a0ae4 100644
--- a/llvm/utils/gn/secondary/compiler-rt/lib/asan/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/lib/asan/BUILD.gn
@@ -68,6 +68,7 @@ if (current_toolchain == host_toolchain) {
     if (current_os != "win") {
       cflags += [ "-ftls-model=initial-exec" ]
     } else {
+      cflags += [ "/MD" ]
       ldflags = [ "/OPT:NOICF" ]
     }
 
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index d270686a2a87d..5f2817c6d80ed 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -1104,14 +1104,12 @@ if (current_toolchain == default_toolchain) {
       "__fwd/get.h",
       "__fwd/ios.h",
       "__fwd/istream.h",
-      "__fwd/map.h",
       "__fwd/mdspan.h",
       "__fwd/memory.h",
       "__fwd/memory_resource.h",
       "__fwd/ostream.h",
       "__fwd/pair.h",
       "__fwd/queue.h",
-      "__fwd/set.h",
       "__fwd/span.h",
       "__fwd/sstream.h",
       "__fwd/stack.h",
diff --git a/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn
index b00442d8e1ebb..10c5f95edf5fd 100644
--- a/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Host/BUILD.gn
@@ -16,7 +16,6 @@ static_library("Host") {
   ]
   public_deps = [ "//llvm/utils/gn/build/libs/xml" ]
   sources = [
-    "aix/Support.cpp",
     "common/File.cpp",
     "common/FileAction.cpp",
     "common/FileCache.cpp",
@@ -51,7 +50,6 @@ static_library("Host") {
     "common/UDPSocket.cpp",
     "common/XML.cpp",
     "common/ZipFileResolver.cpp",
-    "posix/Support.cpp",
   ]
 
   if (lldb_enable_libedit) {
@@ -89,6 +87,7 @@ static_library("Host") {
       "posix/MainLoopPosix.cpp",
       "posix/PipePosix.cpp",
       "posix/ProcessLauncherPosixFork.cpp",
+      "posix/Support.cpp",
     ]
   }
 
@@ -96,6 +95,7 @@ static_library("Host") {
     sources += [
       "aix/Host.cpp",
       "aix/HostInfoAIX.cpp",
+      "aix/Support.cpp",
     ]
   }
 
diff --git a/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn b/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn
index 30a1e03e3bffa..a42f781271a21 100644
--- a/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/tools/lldb-dap/BUILD.gn
@@ -5,6 +5,7 @@ static_library("lib") {
   configs += [ "//llvm/utils/gn/build:lldb_code" ]
   deps = [
     "//lldb/source/API:liblldb",
+    "//lldb/source/Host",
     "//llvm/lib/Option",
     "//llvm/lib/Support",
   ]
@@ -14,7 +15,6 @@ static_library("lib") {
 
   include_dirs = [ "." ]
 
-  # FIXME: link in //lldb/source/Host on Win (and netbsd)
   # FIXME: link in pthread
   # FIXME: rpath/install_name stuff on macOS for framework on macOS
 
@@ -44,6 +44,7 @@ static_library("lib") {
     "Handler/InitializeRequestHandler.cpp",
     "Handler/LaunchRequestHandler.cpp",
     "Handler/LocationsRequestHandler.cpp",
+    "Handler/ModuleSymbolsRequestHandler.cpp",
     "Handler/ModulesRequestHandler.cpp",
     "Handler/NextRequestHandler.cpp",
     "Handler/PauseRequestHandler.cpp",
@@ -72,6 +73,7 @@ static_library("lib") {
     "LLDBUtils.cpp",
     "OutputRedirector.cpp",
     "ProgressEvent.cpp",
+    "Protocol/DAPTypes.cpp",
     "Protocol/ProtocolBase.cpp",
     "Protocol/ProtocolEvents.cpp",
     "Protocol/ProtocolRequests.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/CAS/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CAS/BUILD.gn
new file mode 100644
index 0000000000000..8b67ac3a8888d
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/CAS/BUILD.gn
@@ -0,0 +1,10 @@
+static_library("CAS") {
+  output_name = "LLVMCAS"
+  sources = [
+    "ActionCache.cpp",
+    "ActionCaches.cpp",
+    "BuiltinCAS.cpp",
+    "InMemoryCAS.cpp",
+    "ObjectStore.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
index eac2cd476ddb3..9d518dfeff50e 100644
--- a/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/MC/BUILD.gn
@@ -54,6 +54,7 @@ static_library("MC") {
     "MCObjectWriter.cpp",
     "MCPseudoProbe.cpp",
     "MCRegisterInfo.cpp",
+    "MCSFrame.cpp",
     "MCSPIRVStreamer.cpp",
     "MCSchedule.cpp",
     "MCSection.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ObjCopy/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ObjCopy/BUILD.gn
index 24ebf96906d5c..8aacbbc2f16f0 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ObjCopy/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ObjCopy/BUILD.gn
@@ -14,6 +14,10 @@ static_library("ObjCopy") {
     "COFF/COFFWriter.cpp",
     "CommonConfig.cpp",
     "ConfigManager.cpp",
+    "DXContainer/DXContainerObjcopy.cpp",
+    "DXContainer/DXContainerObject.cpp",
+    "DXContainer/DXContainerReader.cpp",
+    "DXContainer/DXContainerWriter.cpp",
     "ELF/ELFObjcopy.cpp",
     "ELF/ELFObject.cpp",
     "MachO/MachOLayoutBuilder.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 0ea1c6fcee5ee..58ab4a55d4a56 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -72,6 +72,7 @@ static_library("Support") {
     "CrashRecoveryContext.cpp",
     "DAGDeltaAlgorithm.cpp",
     "DJB.cpp",
+    "DXILABI.cpp",
     "DataExtractor.cpp",
     "Debug.cpp",
     "DebugCounter.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
index 7ef94737bb418..cd2d9f1a697c0 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
@@ -174,6 +174,7 @@ static_library("LLVMAArch64CodeGen") {
     "GISel/AArch64PostSelectOptimize.cpp",
     "GISel/AArch64PreLegalizerCombiner.cpp",
     "GISel/AArch64RegisterBankInfo.cpp",
+    "MachineSMEABIPass.cpp",
     "SMEABIPass.cpp",
     "SMEPeepholeOpt.cpp",
     "SVEIntrinsicOpts.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 3d11ce566207a..74f6374e444c3 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -201,7 +201,6 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUTargetObjectFile.cpp",
     "AMDGPUTargetTransformInfo.cpp",
     "AMDGPUUnifyDivergentExitNodes.cpp",
-    "AMDGPUUnifyMetadata.cpp",
     "AMDGPUWaitSGPRHazards.cpp",
     "GCNCreateVOPD.cpp",
     "GCNDPPCombine.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/BUILD.gn
index 2838901eecd1e..449adf4777069 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/BUILD.gn
@@ -49,6 +49,7 @@ static_library("LLVMAVRCodeGen") {
     "AVRSubtarget.cpp",
     "AVRTargetMachine.cpp",
     "AVRTargetObjectFile.cpp",
+    "AVRTargetTransformInfo.cpp",
   ]
 }
 
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
index ea3615cee392a..c304e9382de35 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
@@ -63,7 +63,6 @@ static_library("LLVMPowerPCCodeGen") {
     "PPCBoolRetToInt.cpp",
     "PPCBranchCoalescing.cpp",
     "PPCBranchSelector.cpp",
-    "PPCCCState.cpp",
     "PPCCTRLoops.cpp",
     "PPCCTRLoopsVerify.cpp",
     "PPCCallingConv.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/TargetParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/TargetParser/BUILD.gn
index 3dbc803d0d483..183fa57d47a63 100644
--- a/llvm/utils/gn/secondary/llvm/lib/TargetParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/TargetParser/BUILD.gn
@@ -20,5 +20,6 @@ static_library("TargetParser") {
     "TargetParser.cpp",
     "Triple.cpp",
     "X86TargetParser.cpp",
+    "XtensaTargetParser.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
index 08cddc1f90415..eee482016c243 100644
--- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn
@@ -65,7 +65,6 @@ write_lit_config("lit_site_cfg") {
     "LLVM_ENABLE_FFI=0",
     "LLVM_ENABLE_HTTPLIB=0",
     "LLVM_ENABLE_PROFCHECK=0",
-    "LLVM_EXPERIMENTAL_KEY_INSTRUCTIONS=0",
     "LLVM_FORCE_VC_REVISION=",
     "LLVM_HAS_LOGF128=0",
     "LLVM_HAVE_OPT_VIEWER_MODULES=0",
@@ -302,6 +301,7 @@ group("test") {
     "//llvm/tools/llvm-modextract",
     "//llvm/tools/llvm-mt",
     "//llvm/tools/llvm-nm",
+    "//llvm/tools/llvm-offload-wrapper",
     "//llvm/tools/llvm-objcopy:symlinks",
     "//llvm/tools/llvm-objdump:symlinks",
     "//llvm/tools/llvm-opt-fuzzer",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-offload-wrapper/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-offload-wrapper/BUILD.gn
new file mode 100644
index 0000000000000..27cc7aa4a849e
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-offload-wrapper/BUILD.gn
@@ -0,0 +1,12 @@
+executable("llvm-offload-wrapper") {
+  deps = [
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/Frontend/Offloading",
+    "//llvm/lib/Object",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+    "//llvm/lib/TargetParser",
+  ]
+  sources = [ "llvm-offload-wrapper.cpp" ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
index b8291ecf7271a..22026664ae596 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
@@ -8,6 +8,7 @@ group("unittests") {
     "BinaryFormat:BinaryFormatTests",
     "Bitcode:BitcodeTests",
     "Bitstream:BitstreamTests",
+    "CAS:CASTests",
     "CGData:CodeGenDataTests",
     "CodeGen:CodeGenTests",
 
diff --git a/llvm/utils/gn/secondary/llvm/unittests/CAS/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/CAS/BUILD.gn
new file mode 100644
index 0000000000000..efd4f19b675f2
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/unittests/CAS/BUILD.gn
@@ -0,0 +1,14 @@
+import("//third-party/unittest/unittest.gni")
+
+unittest("CASTests") {
+  deps = [
+    "//llvm/lib/CAS",
+    "//llvm/lib/Support",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "ActionCacheTest.cpp",
+    "CASTestConfig.cpp",
+    "ObjectStoreTest.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
index 0d162ff0f9d57..229586a9b0e3c 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
@@ -43,6 +43,7 @@ unittest("IRTests") {
     "PassBuilderCallbacksTest.cpp",
     "PassManagerTest.cpp",
     "PatternMatch.cpp",
+    "RuntimeLibcallsTest.cpp",
     "ShuffleVectorInstTest.cpp",
     "StructuralHashTest.cpp",
     "TimePassesTest.cpp",
diff --git a/llvm/utils/lit/lit/LitConfig.py b/llvm/utils/lit/lit/LitConfig.py
index cb4aef6f72a87..df297b91be1b6 100644
--- a/llvm/utils/lit/lit/LitConfig.py
+++ b/llvm/utils/lit/lit/LitConfig.py
@@ -39,6 +39,7 @@ def __init__(
         parallelism_groups={},
         per_test_coverage=False,
         gtest_sharding=True,
+        update_tests=False,
     ):
         # The name of the test runner.
         self.progname = progname
@@ -91,6 +92,8 @@ def __init__(
         self.parallelism_groups = parallelism_groups
         self.per_test_coverage = per_test_coverage
         self.gtest_sharding = bool(gtest_sharding)
+        self.update_tests = update_tests
+        self.test_updaters = []
 
     @property
     def maxIndividualTestTime(self):
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index 7290977724170..83a09eb784fd3 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -307,6 +307,11 @@ def isFailure(self):
     def getFullName(self):
         return self.suite.config.name + " :: " + "/".join(self.path_in_suite)
 
+    def getSummaryName(self, printPathRelativeCWD):
+        if printPathRelativeCWD:
+            return os.path.relpath(self.getFilePath())
+        return self.getFullName()
+
     def getFilePath(self):
         if self.file_path:
             return self.file_path
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index e7cd70766a3dd..f2c5c6d0dbe93 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -1192,6 +1192,18 @@ def executeScriptInternal(
                 str(result.timeoutReached),
             )
 
+        if litConfig.update_tests:
+            for test_updater in litConfig.test_updaters:
+                try:
+                    update_output = test_updater(result, test)
+                except Exception as e:
+                    out += f"Exception occurred in test updater: {e}"
+                    continue
+                if update_output:
+                    for line in update_output.splitlines():
+                        out += f"# {line}\n"
+                    break
+
     return out, err, exitCode, timeoutInfo
 
 
diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py
index e88951520e660..8238bc42395af 100644
--- a/llvm/utils/lit/lit/cl_arguments.py
+++ b/llvm/utils/lit/lit/cl_arguments.py
@@ -91,6 +91,13 @@ def parse_args():
         help="Enable -v, but for all tests not just failed tests.",
         action="store_true",
     )
+    format_group.add_argument(
+        "-r",
+        "--relative-paths",
+        dest="printPathRelativeCWD",
+        help="Print paths relative to CWD",
+        action="store_true",
+    )
     format_group.add_argument(
         "-o",
         "--output",
@@ -230,6 +237,12 @@ def parse_args():
         action="store_true",
         help="Exit with status zero even if some tests fail",
     )
+    execution_group.add_argument(
+        "--update-tests",
+        dest="update_tests",
+        action="store_true",
+        help="Try to update regression tests to reflect current behavior, if possible",
+    )
     execution_test_time_group = execution_group.add_mutually_exclusive_group()
     execution_test_time_group.add_argument(
         "--skip-test-time-recording",
diff --git a/llvm/utils/lit/lit/display.py b/llvm/utils/lit/lit/display.py
index 3a224f734bf71..b565bbc7a4f93 100644
--- a/llvm/utils/lit/lit/display.py
+++ b/llvm/utils/lit/lit/display.py
@@ -136,9 +136,7 @@ def print_result(self, test):
         # Show the test failure output, if requested.
         if (test.isFailure() and self.opts.showOutput) or self.opts.showAllOutput:
             if test.isFailure():
-                print(
-                    "%s TEST '%s' FAILED %s" % ("*" * 20, test.getFullName(), "*" * 20)
-                )
+                print("%s TEST '%s' FAILED %s" % ("*" * 20, test_name, "*" * 20))
             out = test.result.output
             # Encode/decode so that, when using Python 3.6.5 in Windows 10,
             # print(out) doesn't raise UnicodeEncodeError if out contains
@@ -159,7 +157,7 @@ def print_result(self, test):
 
         # Report test metrics, if present.
         if test.result.metrics:
-            print("%s TEST '%s' RESULTS %s" % ("*" * 10, test.getFullName(), "*" * 10))
+            print("%s TEST '%s' RESULTS %s" % ("*" * 10, test_name, "*" * 10))
             items = sorted(test.result.metrics.items())
             for metric_name, value in items:
                 print("%s: %s " % (metric_name, value.format()))
diff --git a/llvm/utils/lit/lit/llvm/config.py b/llvm/utils/lit/lit/llvm/config.py
index 649636d4bcf4c..56aa5eb64fa36 100644
--- a/llvm/utils/lit/lit/llvm/config.py
+++ b/llvm/utils/lit/lit/llvm/config.py
@@ -64,12 +64,17 @@ def __init__(self, lit_config, config):
             self.with_environment("_TAG_REDIR_ERR", "TXT")
             self.with_environment("_CEE_RUNOPTS", "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)")
 
+        if lit_config.update_tests:
+            self.use_lit_shell = True
+
         # Choose between lit's internal shell pipeline runner and a real shell.
         # If LIT_USE_INTERNAL_SHELL is in the environment, we use that as an
         # override.
         lit_shell_env = os.environ.get("LIT_USE_INTERNAL_SHELL")
         if lit_shell_env:
             self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env)
+            if not self.use_lit_shell and lit_config.update_tests:
+                print("note: --update-tests is not supported when using external shell")
 
         if not self.use_lit_shell:
             features.add("shell")
@@ -107,6 +112,8 @@ def __init__(self, lit_config, config):
             features.add("system-solaris")
         elif platform.system() == "OS/390":
             features.add("system-zos")
+        elif sys.platform == "cygwin":
+            features.add("system-cygwin")
 
         # Native compilation: host arch == default triple arch
         # Both of these values should probably be in every site config (e.g. as
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index 9650a0e901173..a585cc0abdd48 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -43,6 +43,7 @@ def main(builtin_params={}):
         per_test_coverage=opts.per_test_coverage,
         gtest_sharding=opts.gtest_sharding,
         maxRetriesPerTest=opts.maxRetriesPerTest,
+        update_tests=opts.update_tests,
     )
 
     discovered_tests = lit.discovery.find_tests_for_inputs(
@@ -328,12 +329,13 @@ def print_results(tests, elapsed, opts):
             sorted(tests_by_code[code], key=lambda t: t.getFullName()),
             code,
             opts.shown_codes,
+            opts.printPathRelativeCWD,
         )
 
     print_summary(total_tests, tests_by_code, opts.quiet, elapsed)
 
 
-def print_group(tests, code, shown_codes):
+def print_group(tests, code, shown_codes, printPathRelativeCWD):
     if not tests:
         return
     if not code.isFailure and code not in shown_codes:
@@ -341,7 +343,7 @@ def print_group(tests, code, shown_codes):
     print("*" * 20)
     print("{} Tests ({}):".format(code.label, len(tests)))
     for test in tests:
-        print("  %s" % test.getFullName())
+        print("  %s" % test.getSummaryName(printPathRelativeCWD))
     sys.stdout.write("\n")
 
 
diff --git a/llvm/utils/lit/tests/Inputs/print-relative-path/lit.cfg b/llvm/utils/lit/tests/Inputs/print-relative-path/lit.cfg
new file mode 100644
index 0000000000000..2cf5104e124dc
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/print-relative-path/lit.cfg
@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "print-relative-path"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
diff --git a/llvm/utils/lit/tests/Inputs/print-relative-path/test.txt b/llvm/utils/lit/tests/Inputs/print-relative-path/test.txt
new file mode 100644
index 0000000000000..0ddee88ac0bf5
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/print-relative-path/test.txt
@@ -0,0 +1 @@
+# RUN: echo %var
diff --git a/llvm/utils/lit/tests/Inputs/print-relative-path/test2.txt b/llvm/utils/lit/tests/Inputs/print-relative-path/test2.txt
new file mode 100644
index 0000000000000..3c0fa612b5fb1
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/print-relative-path/test2.txt
@@ -0,0 +1,2 @@
+# RUN: false
+
diff --git a/llvm/utils/lit/tests/print-relative-path.py b/llvm/utils/lit/tests/print-relative-path.py
new file mode 100644
index 0000000000000..c66f0150ca70f
--- /dev/null
+++ b/llvm/utils/lit/tests/print-relative-path.py
@@ -0,0 +1,18 @@
+# RUN: %{lit} --ignore-fail --show-pass %{inputs}/print-relative-path | FileCheck --check-prefix=CHECK-DEFAULT %s
+# RUN: %{lit} --ignore-fail --show-pass -r %{inputs}/print-relative-path | FileCheck --check-prefix=CHECK-RELATIVE %s
+# RUN: %{lit} --ignore-fail --show-pass --relative-paths %{inputs}/print-relative-path | FileCheck --check-prefix=CHECK-RELATIVE %s
+
+
+# CHECK-DEFAULT: PASS: print-relative-path :: test.txt (1 of 2)
+# CHECK-DEFAULT: FAIL: print-relative-path :: test2.txt (2 of 2)
+# CHECK-DEFAULT: Passed Tests (1):
+# CHECK-DEFAULT:  print-relative-path :: test.txt
+# CHECK-DEFAULT: Failed Tests (1):
+# CHECK-DEFAULT:  print-relative-path :: test2.txt
+
+# CHECK-RELATIVE: PASS: print-relative-path :: test.txt (1 of 2)
+# CHECK-RELATIVE: FAIL: print-relative-path :: test2.txt (2 of 2)
+# CHECK-RELATIVE: Passed Tests (1):
+# CHECK-RELATIVE:  Inputs{{[/\\]}}print-relative-path{{[/\\]}}test.txt
+# CHECK-RELATIVE: Failed Tests (1):
+# CHECK-RELATIVE:  Inputs{{[/\\]}}print-relative-path{{[/\\]}}test2.txt
diff --git a/llvm/utils/lldbDataFormatters.py b/llvm/utils/lldbDataFormatters.py
index 7fbeabe64ab9e..5e553caa4446d 100644
--- a/llvm/utils/lldbDataFormatters.py
+++ b/llvm/utils/lldbDataFormatters.py
@@ -8,7 +8,6 @@
 
 import collections
 import lldb
-import json
 
 
 def __lldb_init_module(debugger, internal_dict):
@@ -192,28 +191,19 @@ def SmallStringSummaryProvider(valobj, internal_dict):
 
 
 def StringRefSummaryProvider(valobj, internal_dict):
-    if valobj.GetNumChildren() == 2:
-        # StringRef's are also used to point at binary blobs in memory,
-        # so filter out suspiciously long strings.
-        max_length = 1024
-        actual_length = valobj.GetChildAtIndex(1).GetValueAsUnsigned()
-        truncate = actual_length > max_length
-        length = min(max_length, actual_length)
-        if length == 0:
-            return '""'
-
-        data = valobj.GetChildAtIndex(0).GetPointeeData(item_count=length)
-        error = lldb.SBError()
-        string = data.ReadRawData(error, 0, data.GetByteSize()).decode()
-        if error.Fail():
-            return "<error: %s>" % error.description
-
-        # json.dumps conveniently escapes the string for us.
-        string = json.dumps(string)
-        if truncate:
-            string += "..."
-        return string
-    return None
+    data_pointer = valobj.GetChildMemberWithName("Data")
+    length = valobj.GetChildMemberWithName("Length").unsigned
+    if data_pointer.unsigned == 0 or length == 0:
+        return '""'
+
+    data = data_pointer.deref
+    # Get a char[N] type, from the underlying char type.
+    array_type = data.type.GetArrayType(length)
+    # Cast the char* string data to a char[N] array.
+    char_array = data.Cast(array_type)
+    # Use the builtin summary for its support of max-string-summary-length and
+    # display of non-printable bytes.
+    return char_array.summary
 
 
 def ConstStringSummaryProvider(valobj, internal_dict):
diff --git a/llvm/utils/llvm-original-di-preservation.py b/llvm/utils/llvm-original-di-preservation.py
index b5ccd7a3224f8..cdca2ec8153b8 100755
--- a/llvm/utils/llvm-original-di-preservation.py
+++ b/llvm/utils/llvm-original-di-preservation.py
@@ -6,6 +6,7 @@
 from __future__ import print_function
 import argparse
 import os
+import re
 import sys
 from json import loads
 from collections import defaultdict
@@ -22,6 +23,14 @@ def __init__(self, origin, action, bb_name, fn_name, instr):
     def key(self):
         return self.action + self.bb_name + self.fn_name + self.instr
 
+    def reduced_key(self, bug_pass):
+        if self.origin is not None:
+            # If we have the origin stacktrace available, we can use it to efficiently deduplicate identical errors. We
+            # just need to remove the pointer values from the string first, so that we can deduplicate across files.
+            origin_no_addr = re.sub(r"0x[0-9a-fA-F]+", "", self.origin)
+            return origin_no_addr
+        return bug_pass + self.instr
+
     def to_dict(self):
         result = {
             "instr": self.instr,
@@ -42,6 +51,9 @@ def __init__(self, action, fn_name):
     def key(self):
         return self.action + self.fn_name
 
+    def reduced_key(self, bug_pass):
+        return bug_pass + self.fn_name
+
     def to_dict(self):
         return {
             "fn_name": self.fn_name,
@@ -58,6 +70,9 @@ def __init__(self, action, name, fn_name):
     def key(self):
         return self.action + self.name + self.fn_name
 
+    def reduced_key(self, bug_pass):
+        return bug_pass + self.name
+
     def to_dict(self):
         return {
             "fn_name": self.fn_name,
@@ -478,7 +493,11 @@ def get_json_chunk(file, start, size):
 # Parse the program arguments.
 def parse_program_args(parser):
     parser.add_argument("file_name", type=str, help="json file to process")
-    parser.add_argument("--reduce", action="store_true", help="create reduced report")
+    parser.add_argument(
+        "--reduce",
+        action="store_true",
+        help="create reduced report by deduplicating bugs within and across files",
+    )
 
     report_type_group = parser.add_mutually_exclusive_group(required=True)
     report_type_group.add_argument(
@@ -523,13 +542,10 @@ def Main():
     di_sp_bugs_summary = OrderedDict()
     di_var_bugs_summary = OrderedDict()
 
-    # Compress similar bugs.
-    # DILocBugs with same pass & instruction name.
-    di_loc_pass_instr_set = set()
-    # DISPBugs with same pass & function name.
-    di_sp_pass_fn_set = set()
-    # DIVarBugs with same pass & variable name.
-    di_var_pass_var_set = set()
+    # If we are using --reduce, use these sets to deduplicate similar bugs within and across files.
+    di_loc_reduced_set = set()
+    di_sp_reduced_set = set()
+    di_var_reduced_set = set()
 
     start_line = 0
     chunk_size = 1000000
@@ -585,9 +601,9 @@ def Main():
                     if not di_loc_bug.key() in di_loc_set:
                         di_loc_set.add(di_loc_bug.key())
                         if opts.reduce:
-                            pass_instr = bugs_pass + instr
-                            if not pass_instr in di_loc_pass_instr_set:
-                                di_loc_pass_instr_set.add(pass_instr)
+                            reduced_key = di_loc_bug.reduced_key(bugs_pass)
+                            if not reduced_key in di_loc_reduced_set:
+                                di_loc_reduced_set.add(reduced_key)
                                 di_loc_bugs.append(di_loc_bug)
                         else:
                             di_loc_bugs.append(di_loc_bug)
@@ -608,9 +624,9 @@ def Main():
                     if not di_sp_bug.key() in di_sp_set:
                         di_sp_set.add(di_sp_bug.key())
                         if opts.reduce:
-                            pass_fn = bugs_pass + name
-                            if not pass_fn in di_sp_pass_fn_set:
-                                di_sp_pass_fn_set.add(pass_fn)
+                            reduced_key = di_sp_bug.reduced_key(bugs_pass)
+                            if not reduced_key in di_sp_reduced_set:
+                                di_sp_reduced_set.add(reduced_key)
                                 di_sp_bugs.append(di_sp_bug)
                         else:
                             di_sp_bugs.append(di_sp_bug)
@@ -632,9 +648,9 @@ def Main():
                     if not di_var_bug.key() in di_var_set:
                         di_var_set.add(di_var_bug.key())
                         if opts.reduce:
-                            pass_var = bugs_pass + name
-                            if not pass_var in di_var_pass_var_set:
-                                di_var_pass_var_set.add(pass_var)
+                            reduced_key = di_var_bug.reduced_key(bugs_pass)
+                            if not reduced_key in di_var_reduced_set:
+                                di_var_reduced_set.add(reduced_key)
                                 di_var_bugs.append(di_var_bug)
                         else:
                             di_var_bugs.append(di_var_bug)
diff --git a/llvm/utils/profcheck-xfail.txt b/llvm/utils/profcheck-xfail.txt
new file mode 100644
index 0000000000000..2f12386003b33
--- /dev/null
+++ b/llvm/utils/profcheck-xfail.txt
@@ -0,0 +1,1979 @@
+Analysis/LoopAccessAnalysis/memcheck-ni.ll
+Analysis/MemorySSA/pr116227.ll
+Analysis/MemorySSA/pr40038.ll
+Analysis/MemorySSA/pr43641.ll
+Analysis/MemorySSA/pr46574.ll
+Analysis/MemorySSA/update-remove-dead-blocks.ll
+Analysis/StackSafetyAnalysis/ipa.ll
+Analysis/ValueTracking/known-power-of-two-urem.ll
+Bitcode/fcmp-fast.ll
+Bitcode/flags.ll
+CodeGen/AArch64/cgdata-merge-local.ll
+CodeGen/AArch64/llvm-masked-gather-legal-for-sve.ll
+CodeGen/AArch64/llvm-masked-scatter-legal-for-sve.ll
+CodeGen/AArch64/selectopt-cast.ll
+CodeGen/AArch64/selectopt.ll
+CodeGen/AMDGPU/amdgpu-attributor-no-agpr.ll
+CodeGen/AMDGPU/amdgpu-codegenprepare-fdiv.ll
+CodeGen/AMDGPU/amdgpu-codegenprepare-sqrt.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-dynamic-indirect-access-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-dynamic-indirect-access.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-dynamic-lds-test-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-dynamic-lds-test.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-lower-all.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-multiple-blocks-return-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-multiple-blocks-return.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-multi-static-dynamic-indirect-access-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-multi-static-dynamic-indirect-access.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-non-kernel-declaration.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-dynamic-indirect-access-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-dynamic-indirect-access.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-dynamic-lds-test-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-dynamic-lds-test.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-indirect-access-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-indirect-access-function-param-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-indirect-access-function-param.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-indirect-access.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-indirect-access-lower-all.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-indirect-access-nested-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-indirect-access-nested.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-indirect-access-no-kernel-lds-id.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-no-heap-ptr.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-O0.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-test-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-test-atomic-cmpxchg-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-test-atomicrmw-asan.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-test.ll
+CodeGen/AMDGPU/amdgpu-sw-lower-lds-static-lds-vector-ptrs.ll
+CodeGen/AMDGPU/atomic_optimization_split_dt_update.ll
+CodeGen/AMDGPU/atomic-optimizer-promote-i8.ll
+CodeGen/AMDGPU/attributor-flatscratchinit.ll
+CodeGen/AMDGPU/global_atomic_optimizer_fp_rtn.ll
+CodeGen/AMDGPU/global-atomic-scan.ll
+CodeGen/AMDGPU/global_atomics_iterative_scan_fp.ll
+CodeGen/AMDGPU/global_atomics_iterative_scan.ll
+CodeGen/AMDGPU/global_atomics_optimizer_fp_no_rtn.ll
+CodeGen/AMDGPU/GlobalISel/atomic_optimizations_mul_one.ll
+CodeGen/AMDGPU/lower-buffer-fat-pointers-mem-transfer.ll
+CodeGen/AMDGPU/lower-ctor-dtor.ll
+CodeGen/AMDGPU/lower-mem-intrinsics.ll
+CodeGen/AMDGPU/lower-mem-intrinsics-threshold.ll
+CodeGen/AMDGPU/opencl-printf-and-hostcall.ll
+CodeGen/AMDGPU/opencl-printf.ll
+CodeGen/AMDGPU/opencl-printf-pipeline.ll
+CodeGen/AMDGPU/printf_builtin.ll
+CodeGen/AMDGPU/printf-existing-format-strings.ll
+CodeGen/AMDGPU/printf_nobuiltin.ll
+CodeGen/AMDGPU/private-memory-atomics.ll
+CodeGen/AMDGPU/si-annotate-nested-control-flows.ll
+CodeGen/AMDGPU/simple-indirect-call-2.ll
+CodeGen/ARM/loopvectorize_pr33804.ll
+CodeGen/ARM/sjljeh-swifterror.ll
+CodeGen/BPF/adjust-opt-icmp1.ll
+CodeGen/BPF/adjust-opt-icmp2.ll
+CodeGen/BPF/adjust-opt-icmp5.ll
+CodeGen/BPF/adjust-opt-icmp6.ll
+CodeGen/Hexagon/autohvx/interleave.ll
+CodeGen/Hexagon/loop-idiom/hexagon-memmove1.ll
+CodeGen/Hexagon/loop-idiom/hexagon-memmove2.ll
+CodeGen/Hexagon/loop-idiom/memmove-rt-check.ll
+CodeGen/NVPTX/lower-ctor-dtor.ll
+CodeGen/PowerPC/P10-stack-alignment.ll
+CodeGen/X86/masked_gather_scatter.ll
+CodeGen/X86/nocfivalue.ll
+DebugInfo/AArch64/ir-outliner.ll
+DebugInfo/assignment-tracking/X86/hotcoldsplit.ll
+DebugInfo/debugify-each.ll
+DebugInfo/Generic/block-asan.ll
+DebugInfo/KeyInstructions/Generic/loop-unswitch.ll
+DebugInfo/KeyInstructions/Generic/simplifycfg-branch-fold.ll
+DebugInfo/simplify-cfg-preserve-dbg-values.ll
+DebugInfo/X86/asan_debug_info.ll
+Instrumentation/AddressSanitizer/aarch64be.ll
+Instrumentation/AddressSanitizer/adaptive_global_redzones.ll
+Instrumentation/AddressSanitizer/alloca-offset-lifetime.ll
+Instrumentation/AddressSanitizer/AMDGPU/adaptive_constant_global_redzones.ll
+Instrumentation/AddressSanitizer/AMDGPU/adaptive_global_redzones.ll
+Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_lds.ll
+Instrumentation/AddressSanitizer/AMDGPU/asan_do_not_instrument_scratch.ll
+Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_constant_address_space.ll
+Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_generic_address_space.ll
+Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_global_address_space.ll
+Instrumentation/AddressSanitizer/AMDGPU/asan_instrument_mem_intrinsics.ll
+Instrumentation/AddressSanitizer/AMDGPU/global_metadata_addrspacecasts.ll
+Instrumentation/AddressSanitizer/AMDGPU/instrument-stack.ll
+Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_lds_globals.ll
+Instrumentation/AddressSanitizer/AMDGPU/no_redzones_in_scratch_globals.ll
+Instrumentation/AddressSanitizer/asan_address_space_attr.ll
+Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
+Instrumentation/AddressSanitizer/asan-disable-sanitizer-instrumentation.ll
+Instrumentation/AddressSanitizer/asan-funclet.ll
+Instrumentation/AddressSanitizer/asan-masked-load-store.ll
+Instrumentation/AddressSanitizer/asan-optimize-callbacks.ll
+Instrumentation/AddressSanitizer/asan-pass-second-run.ll
+Instrumentation/AddressSanitizer/asan-stack-safety.ll
+Instrumentation/AddressSanitizer/asan-struct-scalable.ll
+Instrumentation/AddressSanitizer/asan-vp-load-store.ll
+Instrumentation/AddressSanitizer/asan-vs-gvn.ll
+Instrumentation/AddressSanitizer/basic.ll
+Instrumentation/AddressSanitizer/basic-msvc64.ll
+Instrumentation/AddressSanitizer/byref-args.ll
+Instrumentation/AddressSanitizer/byval-args.ll
+Instrumentation/AddressSanitizer/calls-only.ll
+Instrumentation/AddressSanitizer/calls-only-smallfn.ll
+Instrumentation/AddressSanitizer/debug-info-alloca.ll
+Instrumentation/AddressSanitizer/debug-info-global-var.ll
+Instrumentation/AddressSanitizer/debug_info.ll
+Instrumentation/AddressSanitizer/debug_info_noninstrumented_alloca2.ll
+Instrumentation/AddressSanitizer/debug_info_noninstrumented_alloca.ll
+Instrumentation/AddressSanitizer/do-not-instrument-globals-darwin.ll
+Instrumentation/AddressSanitizer/do-not-instrument-globals-linux.ll
+Instrumentation/AddressSanitizer/do-not-instrument-globals-windows.ll
+Instrumentation/AddressSanitizer/do-not-instrument-internal-globals.ll
+Instrumentation/AddressSanitizer/do-not-instrument-netbsd-link_set.ll
+Instrumentation/AddressSanitizer/do-not-instrument-profiling-globals.ll
+Instrumentation/AddressSanitizer/do-not-instrument-promotable-allocas.ll
+Instrumentation/AddressSanitizer/do-not-instrument-sanitizers.ll
+Instrumentation/AddressSanitizer/do-not-touch-comdat-global.ll
+Instrumentation/AddressSanitizer/do-not-touch-odr-global.ll
+Instrumentation/AddressSanitizer/do-not-touch-threadlocal.ll
+Instrumentation/AddressSanitizer/dynamic-shadow-darwin.ll
+Instrumentation/AddressSanitizer/experiment-call.ll
+Instrumentation/AddressSanitizer/experiment.ll
+Instrumentation/AddressSanitizer/fake-stack.ll
+Instrumentation/AddressSanitizer/force-dynamic-shadow.ll
+Instrumentation/AddressSanitizer/freebsd.ll
+Instrumentation/AddressSanitizer/global_addrspace.ll
+Instrumentation/AddressSanitizer/global_cstring_darwin.ll
+Instrumentation/AddressSanitizer/global_lto_merge.ll
+Instrumentation/AddressSanitizer/global_metadata_array.ll
+Instrumentation/AddressSanitizer/global_metadata_bitcasts.ll
+Instrumentation/AddressSanitizer/global-metadata-code-model-medium.ll
+Instrumentation/AddressSanitizer/global-metadata-code-model-small.ll
+Instrumentation/AddressSanitizer/global_metadata_darwin.ll
+Instrumentation/AddressSanitizer/global_metadata_external_comdat.ll
+Instrumentation/AddressSanitizer/global_metadata.ll
+Instrumentation/AddressSanitizer/global_metadata_windows.ll
+Instrumentation/AddressSanitizer/global_with_comdat.ll
+Instrumentation/AddressSanitizer/hoist-argument-init-insts.ll
+Instrumentation/AddressSanitizer/instrumentation-with-call-threshold.ll
+Instrumentation/AddressSanitizer/instrument-dynamic-allocas.ll
+Instrumentation/AddressSanitizer/instrument_global.ll
+Instrumentation/AddressSanitizer/instrument_initializer_metadata.ll
+Instrumentation/AddressSanitizer/instrument_initializer_without_global.ll
+Instrumentation/AddressSanitizer/instrument_late_initializer.ll
+Instrumentation/AddressSanitizer/instrument_load_then_store.ll
+Instrumentation/AddressSanitizer/instrument-no-return.ll
+Instrumentation/AddressSanitizer/instrument-section-invalid-c-ident.ll
+Instrumentation/AddressSanitizer/instrument-stack.ll
+Instrumentation/AddressSanitizer/kcfi.ll
+Instrumentation/AddressSanitizer/kcfi-offset.ll
+Instrumentation/AddressSanitizer/keep_going.ll
+Instrumentation/AddressSanitizer/lifetime.ll
+Instrumentation/AddressSanitizer/lifetime-throw.ll
+Instrumentation/AddressSanitizer/lifetime-uar-uas.ll
+Instrumentation/AddressSanitizer/local_alias.ll
+Instrumentation/AddressSanitizer/localescape.ll
+Instrumentation/AddressSanitizer/local_stack_base.ll
+Instrumentation/AddressSanitizer/mem-intrinsics.ll
+Instrumentation/AddressSanitizer/missing_dbg.ll
+Instrumentation/AddressSanitizer/module-flags-aarch64.ll
+Instrumentation/AddressSanitizer/module-flags.ll
+Instrumentation/AddressSanitizer/musttail.ll
+Instrumentation/AddressSanitizer/no-global-ctors.ll
+Instrumentation/AddressSanitizer/no_global_dtors.ll
+Instrumentation/AddressSanitizer/no-globals.ll
+Instrumentation/AddressSanitizer/odr-check-ignore.ll
+Instrumentation/AddressSanitizer/program-addrspace.ll
+Instrumentation/AddressSanitizer/ps4.ll
+Instrumentation/AddressSanitizer/remove-memory-effects.ll
+Instrumentation/AddressSanitizer/scale-offset.ll
+Instrumentation/AddressSanitizer/skip-coro.ll
+Instrumentation/AddressSanitizer/stack_dynamic_alloca.ll
+Instrumentation/AddressSanitizer/stack_layout.ll
+Instrumentation/AddressSanitizer/stack-poisoning-and-lifetime-be.ll
+Instrumentation/AddressSanitizer/stack-poisoning-and-lifetime.ll
+Instrumentation/AddressSanitizer/stack-poisoning-byval-args.ll
+Instrumentation/AddressSanitizer/stack-poisoning.ll
+Instrumentation/AddressSanitizer/str-nobuiltin.ll
+Instrumentation/AddressSanitizer/test64.ll
+Instrumentation/AddressSanitizer/twice.ll
+Instrumentation/AddressSanitizer/ubsan.ll
+Instrumentation/AddressSanitizer/vector-load-store.ll
+Instrumentation/AddressSanitizer/version-mismatch-check.ll
+Instrumentation/AddressSanitizer/win-sorted-sections.ll
+Instrumentation/AddressSanitizer/win-string-literal.ll
+Instrumentation/AddressSanitizer/with-ifunc.ll
+Instrumentation/AddressSanitizer/X86/asm_cpuid.ll
+Instrumentation/AddressSanitizer/X86/asm_more_registers_than_available.ll
+Instrumentation/AddressSanitizer/X86/bug_11395.ll
+Instrumentation/BoundsChecking/many-trap.ll
+Instrumentation/BoundsChecking/many-traps-2.ll
+Instrumentation/BoundsChecking/opt.ll
+Instrumentation/BoundsChecking/phi.ll
+Instrumentation/BoundsChecking/runtimes.ll
+Instrumentation/BoundsChecking/simple.ll
+Instrumentation/DataFlowSanitizer/abilist_aggregate.ll
+Instrumentation/DataFlowSanitizer/abilist.ll
+Instrumentation/DataFlowSanitizer/atomics.ll
+Instrumentation/DataFlowSanitizer/basic.ll
+Instrumentation/DataFlowSanitizer/custom_fun_callback_attributes.ll
+Instrumentation/DataFlowSanitizer/custom_fun_varargs_attributes.ll
+Instrumentation/DataFlowSanitizer/dataflow-disable-sanitizer-instrumentation.ll
+Instrumentation/DataFlowSanitizer/debug.ll
+Instrumentation/DataFlowSanitizer/extern_weak.ll
+Instrumentation/DataFlowSanitizer/ignore_persnality_routine.ll
+Instrumentation/DataFlowSanitizer/origin_abilist.ll
+Instrumentation/DataFlowSanitizer/origin_cached_shadows.ll
+Instrumentation/DataFlowSanitizer/origin_load.ll
+Instrumentation/DataFlowSanitizer/origin_other_ops.ll
+Instrumentation/DataFlowSanitizer/origin_phi.ll
+Instrumentation/DataFlowSanitizer/origin_select.ll
+Instrumentation/DataFlowSanitizer/origin_store.ll
+Instrumentation/DataFlowSanitizer/origin_track_load.ll
+Instrumentation/DataFlowSanitizer/select.ll
+Instrumentation/DataFlowSanitizer/shadow-args-zext.ll
+Instrumentation/DataFlowSanitizer/struct.ll
+Instrumentation/DataFlowSanitizer/uninstrumented_local_functions.ll
+Instrumentation/HeapProfiler/basic-histogram.ll
+Instrumentation/HeapProfiler/basic.ll
+Instrumentation/HeapProfiler/filename.ll
+Instrumentation/HeapProfiler/instrumentation-use-callbacks.ll
+Instrumentation/HeapProfiler/masked-load-store.ll
+Instrumentation/HeapProfiler/memprof-options.ll
+Instrumentation/HeapProfiler/no-instrumentation.ll
+Instrumentation/HeapProfiler/scale-granularity.ll
+Instrumentation/HeapProfiler/shadow.ll
+Instrumentation/HeapProfiler/skip-compiler-inserted.ll
+Instrumentation/HeapProfiler/stack.ll
+Instrumentation/HeapProfiler/version-mismatch-check.ll
+Instrumentation/HWAddressSanitizer/alloca-array.ll
+Instrumentation/HWAddressSanitizer/alloca-compat.ll
+Instrumentation/HWAddressSanitizer/alloca.ll
+Instrumentation/HWAddressSanitizer/alloca-uninteresting.ll
+Instrumentation/HWAddressSanitizer/alloca-with-calls.ll
+Instrumentation/HWAddressSanitizer/atomic.ll
+Instrumentation/HWAddressSanitizer/basic-compat.ll
+Instrumentation/HWAddressSanitizer/basic.ll
+Instrumentation/HWAddressSanitizer/dbg-assign-tag-offset.ll
+Instrumentation/HWAddressSanitizer/dbg-declare-tag-offset.ll
+Instrumentation/HWAddressSanitizer/dbg-value-tag-offset.ll
+Instrumentation/HWAddressSanitizer/dbg-value-tag-offset-nopad.ll
+Instrumentation/HWAddressSanitizer/exception-lifetime.ll
+Instrumentation/HWAddressSanitizer/fixed-shadow.ll
+Instrumentation/HWAddressSanitizer/fuchsia.ll
+Instrumentation/HWAddressSanitizer/globals-access.ll
+Instrumentation/HWAddressSanitizer/globals.ll
+Instrumentation/HWAddressSanitizer/globals-tag.ll
+Instrumentation/HWAddressSanitizer/hwasan-pass-second-run.ll
+Instrumentation/HWAddressSanitizer/kernel-inline.ll
+Instrumentation/HWAddressSanitizer/landingpad.ll
+Instrumentation/HWAddressSanitizer/mapping-override.ll
+Instrumentation/HWAddressSanitizer/memaccess-clobber.ll
+Instrumentation/HWAddressSanitizer/mem-attr.ll
+Instrumentation/HWAddressSanitizer/mem-intrinsics.ll
+Instrumentation/HWAddressSanitizer/musttail.ll
+Instrumentation/HWAddressSanitizer/personality-bti.ll
+Instrumentation/HWAddressSanitizer/personality.ll
+Instrumentation/HWAddressSanitizer/pgo-opt-out.ll
+Instrumentation/HWAddressSanitizer/pgo-opt-out-no-ps.ll
+Instrumentation/HWAddressSanitizer/prologue.ll
+Instrumentation/HWAddressSanitizer/RISCV/alloca.ll
+Instrumentation/HWAddressSanitizer/RISCV/alloca-with-calls.ll
+Instrumentation/HWAddressSanitizer/RISCV/atomic.ll
+Instrumentation/HWAddressSanitizer/RISCV/basic.ll
+Instrumentation/HWAddressSanitizer/RISCV/exception-lifetime.ll
+Instrumentation/HWAddressSanitizer/RISCV/use-after-scope-setjmp.ll
+Instrumentation/HWAddressSanitizer/RISCV/with-calls.ll
+Instrumentation/HWAddressSanitizer/stack-coloring.ll
+Instrumentation/HWAddressSanitizer/stack-safety-analysis.ll
+Instrumentation/HWAddressSanitizer/str-nobuiltin.ll
+Instrumentation/HWAddressSanitizer/use-after-scope.ll
+Instrumentation/HWAddressSanitizer/use-after-scope-setjmp.ll
+Instrumentation/HWAddressSanitizer/vector-load-store.ll
+Instrumentation/HWAddressSanitizer/with-calls.ll
+Instrumentation/HWAddressSanitizer/X86/alloca-array.ll
+Instrumentation/HWAddressSanitizer/X86/alloca.ll
+Instrumentation/HWAddressSanitizer/X86/alloca-with-calls.ll
+Instrumentation/HWAddressSanitizer/X86/atomic.ll
+Instrumentation/HWAddressSanitizer/X86/basic.ll
+Instrumentation/HWAddressSanitizer/X86/globals.ll
+Instrumentation/HWAddressSanitizer/X86/with-calls.ll
+Instrumentation/HWAddressSanitizer/zero-ptr.ll
+Instrumentation/InstrProfiling/always_inline.ll
+Instrumentation/InstrProfiling/atomic-updates.ll
+Instrumentation/InstrProfiling/comdat.ll
+Instrumentation/InstrProfiling/conditional-counter-updates.ll
+Instrumentation/InstrProfiling/early-exit.ll
+Instrumentation/InstrProfiling/icall-nocomdat.ll
+Instrumentation/InstrProfiling/no-counters.ll
+Instrumentation/InstrProfiling/platform.ll
+Instrumentation/InstrProfiling/profiling.ll
+Instrumentation/JustMyCode/jmc-instrument-elf.ll
+Instrumentation/JustMyCode/jmc-instrument.ll
+Instrumentation/JustMyCode/jmc-instrument-x86.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-cvt.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-fminv.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-ld1.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-smaxv.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-sminv.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-st1_lane.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-st1.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-st1_origins.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-tbl.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-umaxv.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-uminv.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-vadd.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-vaddlv.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-vaddv.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-vcvt.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-vmax.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-vmovn.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-vmul.ll
+Instrumentation/MemorySanitizer/AArch64/arm64-vshift.ll
+Instrumentation/MemorySanitizer/AArch64/module-flags-aarch64.ll
+Instrumentation/MemorySanitizer/AArch64/neon_vst_float.ll
+Instrumentation/MemorySanitizer/AArch64/qshrn.ll
+Instrumentation/MemorySanitizer/AArch64/vararg.ll
+Instrumentation/MemorySanitizer/AArch64/vararg_shadow.ll
+Instrumentation/MemorySanitizer/abs-vector.ll
+Instrumentation/MemorySanitizer/alloca.ll
+Instrumentation/MemorySanitizer/ARM32/vararg-arm32.ll
+Instrumentation/MemorySanitizer/array_types.ll
+Instrumentation/MemorySanitizer/atomics.ll
+Instrumentation/MemorySanitizer/attributes.ll
+Instrumentation/MemorySanitizer/bitreverse.ll
+Instrumentation/MemorySanitizer/bmi.ll
+Instrumentation/MemorySanitizer/byval-alignment.ll
+Instrumentation/MemorySanitizer/byval.ll
+Instrumentation/MemorySanitizer/check_access_address.ll
+Instrumentation/MemorySanitizer/check-array.ll
+Instrumentation/MemorySanitizer/check-constant-shadow.ll
+Instrumentation/MemorySanitizer/check-struct.ll
+Instrumentation/MemorySanitizer/clmul.ll
+Instrumentation/MemorySanitizer/count-zeroes.ll
+Instrumentation/MemorySanitizer/csr.ll
+Instrumentation/MemorySanitizer/disambiguate-origin.ll
+Instrumentation/MemorySanitizer/expand-experimental-reductions.ll
+Instrumentation/MemorySanitizer/freeze.ll
+Instrumentation/MemorySanitizer/funnel_shift.ll
+Instrumentation/MemorySanitizer/i386/avx2-intrinsics-i386.ll
+Instrumentation/MemorySanitizer/i386/avx-intrinsics-i386.ll
+Instrumentation/MemorySanitizer/i386/mmx-intrinsics.ll
+Instrumentation/MemorySanitizer/i386/msan_i386intrinsics.ll
+Instrumentation/MemorySanitizer/i386/sse2-intrinsics-i386.ll
+Instrumentation/MemorySanitizer/i386/sse41-intrinsics-i386.ll
+Instrumentation/MemorySanitizer/i386/sse-intrinsics-i386.ll
+Instrumentation/MemorySanitizer/i386/vararg_call.ll
+Instrumentation/MemorySanitizer/i386/vararg.ll
+Instrumentation/MemorySanitizer/i386/vararg_shadow.ll
+Instrumentation/MemorySanitizer/i386/vararg-too-large.ll
+Instrumentation/MemorySanitizer/instrumentation-with-call-threshold.ll
+Instrumentation/MemorySanitizer/invalidate_global_aa.ll
+Instrumentation/MemorySanitizer/is-fpclass.ll
+Instrumentation/MemorySanitizer/libatomic.ll
+Instrumentation/MemorySanitizer/LoongArch/vararg.ll
+Instrumentation/MemorySanitizer/LoongArch/vararg-loongarch64.ll
+Instrumentation/MemorySanitizer/manual-shadow.ll
+Instrumentation/MemorySanitizer/masked-store-load.ll
+Instrumentation/MemorySanitizer/Mips32/vararg-mipsel.ll
+Instrumentation/MemorySanitizer/Mips32/vararg-mips.ll
+Instrumentation/MemorySanitizer/Mips/vararg-mips64el.ll
+Instrumentation/MemorySanitizer/Mips/vararg-mips64.ll
+Instrumentation/MemorySanitizer/missing_origin.ll
+Instrumentation/MemorySanitizer/msan_asm_conservative.ll
+Instrumentation/MemorySanitizer/msan_basic.ll
+Instrumentation/MemorySanitizer/msan_debug_info.ll
+Instrumentation/MemorySanitizer/msan-disable-checks.ll
+Instrumentation/MemorySanitizer/msan_eager.ll
+Instrumentation/MemorySanitizer/msan_invalidate.ll
+Instrumentation/MemorySanitizer/msan_llvm_is_constant.ll
+Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll
+Instrumentation/MemorySanitizer/msan_llvm_strip_invariant.ll
+Instrumentation/MemorySanitizer/msan-pass-second-run.ll
+Instrumentation/MemorySanitizer/mul_by_constant.ll
+Instrumentation/MemorySanitizer/no-check-rt-unaligned.ll
+Instrumentation/MemorySanitizer/nosanitize.ll
+Instrumentation/MemorySanitizer/opaque-ptr.ll
+Instrumentation/MemorySanitizer/origin-alignment.ll
+Instrumentation/MemorySanitizer/origin-array.ll
+Instrumentation/MemorySanitizer/or.ll
+Instrumentation/MemorySanitizer/overflow.ll
+Instrumentation/MemorySanitizer/partial-poison.ll
+Instrumentation/MemorySanitizer/PowerPC32/vararg-ppcle.ll
+Instrumentation/MemorySanitizer/PowerPC32/vararg-ppc.ll
+Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64le.ll
+Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
+Instrumentation/MemorySanitizer/pr32842.ll
+Instrumentation/MemorySanitizer/reduce.ll
+Instrumentation/MemorySanitizer/return_from_main.ll
+Instrumentation/MemorySanitizer/RISCV32/vararg-riscv32.ll
+Instrumentation/MemorySanitizer/saturating.ll
+Instrumentation/MemorySanitizer/scmp.ll
+Instrumentation/MemorySanitizer/stable_set_alloca_origin.ll
+Instrumentation/MemorySanitizer/store-long-origin.ll
+Instrumentation/MemorySanitizer/store-origin.ll
+Instrumentation/MemorySanitizer/str-nobuiltin.ll
+Instrumentation/MemorySanitizer/SystemZ/vararg.ll
+Instrumentation/MemorySanitizer/ucmp.ll
+Instrumentation/MemorySanitizer/unreachable.ll
+Instrumentation/MemorySanitizer/unsized_type.ll
+Instrumentation/MemorySanitizer/vector_arith.ll
+Instrumentation/MemorySanitizer/vector_cmp.ll
+Instrumentation/MemorySanitizer/vector_cvt.ll
+Instrumentation/MemorySanitizer/vector-load-store.ll
+Instrumentation/MemorySanitizer/vector_pack.ll
+Instrumentation/MemorySanitizer/vector-reduce-fadd.ll
+Instrumentation/MemorySanitizer/vector-reduce-fmul.ll
+Instrumentation/MemorySanitizer/vector_shift.ll
+Instrumentation/MemorySanitizer/vector-track-origins-neon.ll
+Instrumentation/MemorySanitizer/vector-track-origins-struct.ll
+Instrumentation/MemorySanitizer/vscale.ll
+Instrumentation/MemorySanitizer/with-call-type-size.ll
+Instrumentation/MemorySanitizer/X86/avx10_2_512ni-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx10_2ni-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx2-intrinsics-x86.ll
+Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx512bw-intrinsics-upgrade.ll
+Instrumentation/MemorySanitizer/X86/avx512fp16-arith-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx512fp16-arith-vl-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx512fp16-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx512-gfni-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx512-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx512-intrinsics-upgrade.ll
+Instrumentation/MemorySanitizer/X86/avx512vl-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx512vl_vnni-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx512vl_vnni-intrinsics-upgrade.ll
+Instrumentation/MemorySanitizer/X86/avx512vnni-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx512vnni-intrinsics-upgrade.ll
+Instrumentation/MemorySanitizer/X86/avx-intrinsics-x86.ll
+Instrumentation/MemorySanitizer/X86/avxvnniint16-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avxvnniint8-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/avx_vnni-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/f16c-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/f16c-intrinsics-upgrade.ll
+Instrumentation/MemorySanitizer/X86/mmx-intrinsics.ll
+Instrumentation/MemorySanitizer/X86/msan_x86intrinsics.ll
+Instrumentation/MemorySanitizer/X86/sse2-intrinsics-x86.ll
+Instrumentation/MemorySanitizer/X86/sse41-intrinsics-x86.ll
+Instrumentation/MemorySanitizer/X86/sse-intrinsics-x86.ll
+Instrumentation/MemorySanitizer/X86/vararg_call.ll
+Instrumentation/MemorySanitizer/X86/vararg_shadow.ll
+Instrumentation/MemorySanitizer/X86/vararg-too-large.ll
+Instrumentation/MemorySanitizer/X86/x86-vpermi2.ll
+Instrumentation/NumericalStabilitySanitizer/basic.ll
+Instrumentation/NumericalStabilitySanitizer/cfg.ll
+Instrumentation/NumericalStabilitySanitizer/invoke.ll
+Instrumentation/NumericalStabilitySanitizer/memory.ll
+Instrumentation/NumericalStabilitySanitizer/non_float_store.ll
+Instrumentation/NumericalStabilitySanitizer/scalable_vector.ll
+Instrumentation/RealtimeSanitizer/rtsan_blocking.ll
+Instrumentation/RealtimeSanitizer/rtsan.ll
+Instrumentation/RealtimeSanitizer/rtsan_multi_return.ll
+Instrumentation/SanitizerBinaryMetadata/atomics.ll
+Instrumentation/SanitizerBinaryMetadata/ctor.ll
+Instrumentation/SanitizerBinaryMetadata/pretend-atomic-access.ll
+Instrumentation/SanitizerBinaryMetadata/shared-mutable.ll
+Instrumentation/SanitizerCoverage/backedge-pruning.ll
+Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_32.ll
+Instrumentation/SanitizerCoverage/cmp-tracing-api-x86_64.ll
+Instrumentation/SanitizerCoverage/cmp-tracing.ll
+Instrumentation/SanitizerCoverage/coff-comdat.ll
+Instrumentation/SanitizerCoverage/coff-pc-table-inline-8bit-counters.ll
+Instrumentation/SanitizerCoverage/coff-pc-table-inline-bool-flag.ll
+Instrumentation/SanitizerCoverage/coff-used-ctor.ll
+Instrumentation/SanitizerCoverage/const-cmp-tracing.ll
+Instrumentation/SanitizerCoverage/control-flow.ll
+Instrumentation/SanitizerCoverage/coverage2-dbg.ll
+Instrumentation/SanitizerCoverage/coverage-dbg.ll
+Instrumentation/SanitizerCoverage/coverage-disable-sanitizer-instrumentation.ll
+Instrumentation/SanitizerCoverage/coverage.ll
+Instrumentation/SanitizerCoverage/div-tracing.ll
+Instrumentation/SanitizerCoverage/gep-tracing.ll
+Instrumentation/SanitizerCoverage/inline-8bit-counters.ll
+Instrumentation/SanitizerCoverage/inline-bool-flag.ll
+Instrumentation/SanitizerCoverage/interposable-symbol.ll
+Instrumentation/SanitizerCoverage/missing_dbg.ll
+Instrumentation/SanitizerCoverage/opaque-ptr.ll
+Instrumentation/SanitizerCoverage/pc-table.ll
+Instrumentation/SanitizerCoverage/seh.ll
+Instrumentation/SanitizerCoverage/stack-depth.ll
+Instrumentation/SanitizerCoverage/switch-tracing.ll
+Instrumentation/SanitizerCoverage/trace-pc-guard-inline-8bit-counters.ll
+Instrumentation/SanitizerCoverage/trace-pc-guard-inline-bool-flag.ll
+Instrumentation/SanitizerCoverage/trace-pc-guard.ll
+Instrumentation/SanitizerCoverage/tracing-comdat.ll
+Instrumentation/SanitizerCoverage/tracing.ll
+Instrumentation/SanitizerCoverage/unreachable-critedge.ll
+Instrumentation/sanitizers-naked.ll
+Instrumentation/ThreadSanitizer/do-not-instrument-memory-access.ll
+Instrumentation/ThreadSanitizer/tsan_basic.ll
+Instrumentation/ThreadSanitizer/tsan-pass-second-run.ll
+Instrumentation/TypeSanitizer/access-with-offset.ll
+Instrumentation/TypeSanitizer/alloca.ll
+Instrumentation/TypeSanitizer/alloca-only.ll
+Instrumentation/TypeSanitizer/anon.ll
+Instrumentation/TypeSanitizer/basic.ll
+Instrumentation/TypeSanitizer/basic-nosan.ll
+Instrumentation/TypeSanitizer/byval.ll
+Instrumentation/TypeSanitizer/globals.ll
+Instrumentation/TypeSanitizer/invalid-metadata.ll
+Instrumentation/TypeSanitizer/memintrinsics.ll
+Instrumentation/TypeSanitizer/nosanitize.ll
+Instrumentation/TypeSanitizer/sanitize-no-tbaa.ll
+Instrumentation/TypeSanitizer/swifterror.ll
+Other/ChangePrinters/DotCfg/print-changed-dot-cfg.ll
+Other/opt-bisect-print-ir-path.ll
+Other/printer.ll
+Other/X86/debugcounter-partiallyinlinelibcalls.ll
+tools/not/disable-symbolization.test
+tools/UpdateTestChecks/update_analyze_test_checks/loop-access-analysis.test
+tools/UpdateTestChecks/update_analyze_test_checks/loop-distribute.test
+tools/UpdateTestChecks/update_test_checks/argument_name_reuse.test
+tools/UpdateTestChecks/update_test_checks/basic.test
+tools/UpdateTestChecks/update_test_checks/check_attrs.test
+tools/UpdateTestChecks/update_test_checks/difile_absolute_filenames.test
+tools/UpdateTestChecks/update_test_checks/filter_out_after.test
+tools/UpdateTestChecks/update_test_checks/generated_funcs_prefix_reuse.test
+tools/UpdateTestChecks/update_test_checks/generated_funcs.test
+tools/UpdateTestChecks/update_test_checks/global_preserve_name.test
+tools/UpdateTestChecks/update_test_checks/if_target.test
+tools/UpdateTestChecks/update_test_checks/named_function_arguments_split.test
+tools/UpdateTestChecks/update_test_checks/on_the_fly_arg_change.test
+tools/UpdateTestChecks/update_test_checks/phi-labels.test
+tools/UpdateTestChecks/update_test_checks/pre-process.test
+tools/UpdateTestChecks/update_test_checks/stable_ir_values2.test
+tools/UpdateTestChecks/update_test_checks/stable_ir_values3.test
+tools/UpdateTestChecks/update_test_checks/stable_ir_values4.test
+tools/UpdateTestChecks/update_test_checks/stable_ir_values5.test
+tools/UpdateTestChecks/update_test_checks/stable_ir_values6.test
+tools/UpdateTestChecks/update_test_checks/stable_ir_values_funcs.test
+tools/UpdateTestChecks/update_test_checks/stable_ir_values.test
+tools/UpdateTestChecks/update_test_checks/various_ir_values_dbgrecords.test
+Transforms/AggressiveInstCombine/inline-strcmp-debugloc.ll
+Transforms/AggressiveInstCombine/lower-table-based-cttz-basics.ll
+Transforms/AggressiveInstCombine/lower-table-based-cttz-dereferencing-pointer.ll
+Transforms/AggressiveInstCombine/lower-table-based-cttz-non-argument-value.ll
+Transforms/AggressiveInstCombine/lower-table-based-cttz-zero-element.ll
+Transforms/AggressiveInstCombine/memchr.ll
+Transforms/AggressiveInstCombine/strncmp-1.ll
+Transforms/AggressiveInstCombine/strncmp-2.ll
+Transforms/AggressiveInstCombine/trunc_select_cmp.ll
+Transforms/AggressiveInstCombine/trunc_select.ll
+Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll
+Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll
+Transforms/AtomicExpand/AArch64/pcsections.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-agent.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-f32-system.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-agent.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-f64-system.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-i16-system.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-agent.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-i32-system.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-agent.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-i64-system.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-i8-system.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd-flat-specialization.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd-flat-specialization-preserve-name.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-flat-noalias-addrspace.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fmax.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fmin.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-fp-vector.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-nand.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-simplify-cfg-CAS-block.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-v2bf16-agent.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-v2bf16-system.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-v2f16-agent.ll
+Transforms/AtomicExpand/AMDGPU/expand-atomic-v2f16-system.ll
+Transforms/AtomicExpand/AMDGPU/expand-cmpxchg-flat-maybe-private.ll
+Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll
+Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll
+Transforms/AtomicExpand/ARM/atomicrmw-fp.ll
+Transforms/AtomicExpand/ARM/cmpxchg-weak.ll
+Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
+Transforms/AtomicExpand/LoongArch/atomicrmw-fp.ll
+Transforms/AtomicExpand/Mips/atomicrmw-fp.ll
+Transforms/AtomicExpand/PowerPC/atomicrmw-fp.ll
+Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll
+Transforms/AtomicExpand/SPARC/libcalls.ll
+Transforms/AtomicExpand/SPARC/partword.ll
+Transforms/AtomicExpand/X86/expand-atomic-rmw-fp.ll
+Transforms/AtomicExpand/X86/expand-atomic-rmw-initial-load.ll
+Transforms/AtomicExpand/X86/expand-atomic-xchg-fp.ll
+Transforms/Attributor/align.ll
+Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
+Transforms/Attributor/ArgumentPromotion/2008-07-02-array-indexing.ll
+Transforms/Attributor/ArgumentPromotion/2008-09-07-CGUpdate.ll
+Transforms/Attributor/ArgumentPromotion/2008-09-08-CGUpdateSelfEdge.ll
+Transforms/Attributor/ArgumentPromotion/aggregate-promote.ll
+Transforms/Attributor/ArgumentPromotion/alignment.ll
+Transforms/Attributor/ArgumentPromotion/alloca-as.ll
+Transforms/Attributor/ArgumentPromotion/array.ll
+Transforms/Attributor/ArgumentPromotion/attrs.ll
+Transforms/Attributor/ArgumentPromotion/basictest.ll
+Transforms/Attributor/ArgumentPromotion/byval-2.ll
+Transforms/Attributor/ArgumentPromotion/byval.ll
+Transforms/Attributor/ArgumentPromotion/chained.ll
+Transforms/Attributor/ArgumentPromotion/control-flow2.ll
+Transforms/Attributor/ArgumentPromotion/crash.ll
+Transforms/Attributor/ArgumentPromotion/dbg.ll
+Transforms/Attributor/ArgumentPromotion/fp80.ll
+Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
+Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
+Transforms/Attributor/ArgumentPromotion/nonzero-address-spaces.ll
+Transforms/Attributor/ArgumentPromotion/pr27568.ll
+Transforms/Attributor/ArgumentPromotion/profile.ll
+Transforms/Attributor/ArgumentPromotion/reserve-tbaa.ll
+Transforms/Attributor/ArgumentPromotion/tail.ll
+Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
+Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
+Transforms/Attributor/callgraph.ll
+Transforms/Attributor/call-simplify-pointer-info.ll
+Transforms/Attributor/internal-noalias.ll
+Transforms/Attributor/IPConstantProp/2009-09-24-byval-ptr.ll
+Transforms/Attributor/IPConstantProp/recursion.ll
+Transforms/Attributor/IPConstantProp/remove-call-inst.ll
+Transforms/Attributor/liveness.ll
+Transforms/Attributor/memory_locations.ll
+Transforms/Attributor/misc_crash.ll
+Transforms/Attributor/noalias.ll
+Transforms/Attributor/nodelete.ll
+Transforms/Attributor/nofpclass.ll
+Transforms/Attributor/nounwind.ll
+Transforms/Attributor/potential.ll
+Transforms/Attributor/range.ll
+Transforms/Attributor/reduced/clear_cached_analysis_for_deleted_functions.ll
+Transforms/Attributor/reduced/register_benchmark_test.ll
+Transforms/Attributor/undefined_behavior.ll
+Transforms/Attributor/value-simplify-gpu.ll
+Transforms/Attributor/value-simplify.ll
+Transforms/Attributor/value-simplify-local-remote.ll
+Transforms/Attributor/value-simplify-pointer-info.ll
+Transforms/Attributor/value-simplify-reachability.ll
+Transforms/BlockExtractor/extract-blocks.ll
+Transforms/BlockExtractor/extract-blocks-with-groups.ll
+Transforms/CodeExtractor/2004-03-13-LoopExtractorCrash.ll
+Transforms/CodeExtractor/2004-03-14-DominanceProblem.ll
+Transforms/CodeExtractor/2004-03-14-NoSwitchSupport.ll
+Transforms/CodeExtractor/2004-03-17-MissedLiveIns.ll
+Transforms/CodeExtractor/2004-03-17-UpdatePHIsOutsideRegion.ll
+Transforms/CodeExtractor/extract-assume.ll
+Transforms/CodeExtractor/input-value-debug.ll
+Transforms/CodeExtractor/LoopExtractor_alloca.ll
+Transforms/CodeExtractor/LoopExtractor_crash.ll
+Transforms/CodeExtractor/LoopExtractor_infinite.ll
+Transforms/CodeExtractor/LoopExtractor.ll
+Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-constant-numerator.ll
+Transforms/CodeGenPrepare/NVPTX/bypass-slow-div.ll
+Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-not-exact.ll
+Transforms/CodeGenPrepare/NVPTX/bypass-slow-div-special-cases.ll
+Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll
+Transforms/CodeGenPrepare/X86/vec-shift.ll
+Transforms/Coroutines/coro-alloca-outside-frame.ll
+Transforms/Coroutines/coro-await-suspend-lower-invoke.ll
+Transforms/Coroutines/coro-await-suspend-lower.ll
+Transforms/Coroutines/coro-byval-param.ll
+Transforms/Coroutines/coro-catchswitch-cleanuppad.ll
+Transforms/Coroutines/coro-debug-coro-frame.ll
+Transforms/Coroutines/coro-debug-dbg.values.ll
+Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll
+Transforms/Coroutines/coro-debug-frame-variable.ll
+Transforms/Coroutines/coro-debug-O2.ll
+Transforms/Coroutines/coro-debug-spill-dbg.declare.ll
+Transforms/Coroutines/coro-materialize.ll
+Transforms/Coroutines/coro-noop.ll
+Transforms/Coroutines/coro-noop-pacbti.ll
+Transforms/Coroutines/coro-only-destroy-when-complete.ll
+Transforms/Coroutines/coro-spill-suspend.ll
+Transforms/Coroutines/coro-split-00.ll
+Transforms/Coroutines/coro-split-alloc.ll
+Transforms/Coroutines/coro-split-dbg-labels.ll
+Transforms/Coroutines/coro-split-final-suspend.ll
+Transforms/Coroutines/coro-split-hidden.ll
+Transforms/Coroutines/coro-split-musttail10.ll
+Transforms/Coroutines/coro-split-musttail12.ll
+Transforms/Coroutines/coro-split-musttail13.ll
+Transforms/Coroutines/coro-split-musttail1.ll
+Transforms/Coroutines/coro-split-musttail2.ll
+Transforms/Coroutines/coro-split-musttail3.ll
+Transforms/Coroutines/coro-split-musttail4.ll
+Transforms/Coroutines/coro-split-musttail5.ll
+Transforms/Coroutines/coro-split-musttail6.ll
+Transforms/Coroutines/coro-split-musttail7.ll
+Transforms/Coroutines/coro-split-musttail8.ll
+Transforms/Coroutines/coro-split-musttail9.ll
+Transforms/Coroutines/coro-split-musttail-chain-pgo-counter-promo.ll
+Transforms/Coroutines/coro-split-musttail.ll
+Transforms/Coroutines/coro-split-musttail-ppc64le.ll
+Transforms/Coroutines/coro-split-noinline.ll
+Transforms/Coroutines/no-suspend.ll
+Transforms/CorrelatedValuePropagation/cond-at-use.ll
+Transforms/CorrelatedValuePropagation/urem-expansion.ll
+Transforms/CorrelatedValuePropagation/urem.ll
+Transforms/CrossDSOCFI/basic.ll
+Transforms/CrossDSOCFI/cfi_functions.ll
+Transforms/CrossDSOCFI/thumb.ll
+Transforms/DFAJumpThreading/dfa-jump-threading-analysis.ll
+Transforms/DFAJumpThreading/dfa-jump-threading-transform.ll
+Transforms/DFAJumpThreading/dfa-unfold-select.ll
+Transforms/DFAJumpThreading/max-path-length.ll
+Transforms/DFAJumpThreading/negative.ll
+Transforms/ExpandLargeDivRem/X86/sdiv129.ll
+Transforms/ExpandLargeDivRem/X86/srem129.ll
+Transforms/ExpandLargeDivRem/X86/udiv129.ll
+Transforms/ExpandLargeDivRem/X86/urem129.ll
+Transforms/ExpandLargeDivRem/X86/vector.ll
+Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptosi129.ll
+Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-fptoui129.ll
+Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-si129tofp.ll
+Transforms/ExpandLargeFpConvert/X86/expand-large-fp-convert-ui129tofp.ll
+Transforms/ExpandMemCmp/AArch64/memcmp.ll
+Transforms/ExpandMemCmp/X86/memcmp.ll
+Transforms/ExpandMemCmp/X86/memcmp-x32.ll
+Transforms/ExpandVariadics/expand-va-intrinsic-split-linkage.ll
+Transforms/ExpandVariadics/expand-va-intrinsic-split-simple.ll
+Transforms/ExpandVariadics/intrinsics.ll
+Transforms/FixIrreducible/basic.ll
+Transforms/FixIrreducible/bug45623.ll
+Transforms/FixIrreducible/nested.ll
+Transforms/FixIrreducible/switch.ll
+Transforms/FunctionSpecialization/function-specialization3.ll
+Transforms/GCOVProfiling/atomic-counter.ll
+Transforms/GCOVProfiling/exit-block.ll
+Transforms/GCOVProfiling/function-numbering.ll
+Transforms/GCOVProfiling/global-ctor.ll
+Transforms/GCOVProfiling/kcfi.ll
+Transforms/GCOVProfiling/kcfi-normalize.ll
+Transforms/GCOVProfiling/linezero.ll
+Transforms/GCOVProfiling/linkagename.ll
+Transforms/GCOVProfiling/module-flags.ll
+Transforms/GCOVProfiling/noprofile.ll
+Transforms/GCOVProfiling/nosanitize.ll
+Transforms/GCOVProfiling/reset.ll
+Transforms/GCOVProfiling/split-indirectbr-critical-edges.ll
+Transforms/GCOVProfiling/three-element-mdnode.ll
+Transforms/GCOVProfiling/version.ll
+Transforms/GlobalOpt/disable-globals-aa.ll
+Transforms/GlobalOpt/global-demotion.ll
+Transforms/GlobalOpt/globalvar-code-model.ll
+Transforms/GlobalOpt/large-int-crash.ll
+Transforms/GlobalOpt/shrink-global-to-bool-check-debug.ll
+Transforms/GlobalOpt/shrink-global-to-bool-opaque-ptrs.ll
+Transforms/GVN/debugloc-load-select.ll
+Transforms/GVN/load-through-select-dbg.ll
+Transforms/GVN/opaque-ptr.ll
+Transforms/GVN/pr69301.ll
+Transforms/GVN/pre-invalid-prof-metadata.ll
+Transforms/GVN/PRE/pre-load-through-select.ll
+Transforms/GVN/PRE/pre-loop-load-through-select.ll
+Transforms/HotColdSplit/addr-taken.ll
+Transforms/HotColdSplit/apply-noreturn-bonus.ll
+Transforms/HotColdSplit/assumption-cache-invalidation.ll
+Transforms/HotColdSplit/delete-use-without-def-dbg-val.ll
+Transforms/HotColdSplit/duplicate-phi-preds-crash.ll
+Transforms/HotColdSplit/eh-pads.ll
+Transforms/HotColdSplit/forward-dfs-reaches-marked-block.ll
+Transforms/HotColdSplit/invalid-dbg-assign.ll
+Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll
+Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll
+Transforms/HotColdSplit/lifetime-markers-on-inputs-3.ll
+Transforms/HotColdSplit/minsize.ll
+Transforms/HotColdSplit/multiple-exits.ll
+Transforms/HotColdSplit/musttail.ll
+Transforms/HotColdSplit/noreturn.ll
+Transforms/HotColdSplit/outline-cold-asm.ll
+Transforms/HotColdSplit/outline-disjoint-diamonds.ll
+Transforms/HotColdSplit/outline-if-then-else.ll
+Transforms/HotColdSplit/outline-inner-region.ll
+Transforms/HotColdSplit/outline-multiple-entry-region.ll
+Transforms/HotColdSplit/outline-outer-region.ll
+Transforms/HotColdSplit/outline-while-loop.ll
+Transforms/HotColdSplit/phi-with-distinct-outlined-values.ll
+Transforms/HotColdSplit/region-overlap.ll
+Transforms/HotColdSplit/resume.ll
+Transforms/HotColdSplit/retain-section.ll
+Transforms/HotColdSplit/section-splitting-custom.ll
+Transforms/HotColdSplit/section-splitting-default.ll
+Transforms/HotColdSplit/split-cold-2.ll
+Transforms/HotColdSplit/split-out-dbg-label.ll
+Transforms/HotColdSplit/split-out-dbg-val-of-arg.ll
+Transforms/HotColdSplit/split-phis-in-exit-blocks.ll
+Transforms/HotColdSplit/split-static-profile.ll
+Transforms/HotColdSplit/stale-assume-in-original-func.ll
+Transforms/HotColdSplit/succ-block-with-self-edge.ll
+Transforms/HotColdSplit/swifterror.ll
+Transforms/HotColdSplit/transfer-debug-info.ll
+Transforms/HotColdSplit/unwind.ll
+Transforms/HotColdSplit/update-split-loop-metadata.ll
+Transforms/IndirectBrExpand/basic.ll
+Transforms/IndVarSimplify/debugloc-rem-subst.ll
+Transforms/IndVarSimplify/eliminate-backedge.ll
+Transforms/IndVarSimplify/eliminate-rem.ll
+Transforms/IndVarSimplify/invalidate-modified-lcssa-phi.ll
+Transforms/IndVarSimplify/pr45835.ll
+Transforms/IndVarSimplify/preserving-debugloc-rem-div.ll
+Transforms/Inline/optimization-remarks-hotness-threshold.ll
+Transforms/InstCombine/2004-09-20-BadLoadCombine2.ll
+Transforms/InstCombine/2004-09-20-BadLoadCombine.ll
+Transforms/InstCombine/2005-04-07-UDivSelectCrash.ll
+Transforms/InstCombine/2011-02-14-InfLoop.ll
+Transforms/InstCombine/AArch64/sve-intrinsic-sel.ll
+Transforms/InstCombine/AArch64/sve-intrinsic-simplify-binop.ll
+Transforms/InstCombine/AArch64/sve-intrinsic-simplify-shift.ll
+Transforms/InstCombine/add2.ll
+Transforms/InstCombine/add.ll
+Transforms/InstCombine/add-mask.ll
+Transforms/InstCombine/add-shl-mul-umax.ll
+Transforms/InstCombine/add-shl-sdiv-to-srem.ll
+Transforms/InstCombine/AMDGPU/addrspacecast.ll
+Transforms/InstCombine/and2.ll
+Transforms/InstCombine/and-fcmp.ll
+Transforms/InstCombine/and.ll
+Transforms/InstCombine/and-or-icmp-nullptr.ll
+Transforms/InstCombine/and-or-icmps.ll
+Transforms/InstCombine/and-or-implied-cond-not.ll
+Transforms/InstCombine/apint-div1.ll
+Transforms/InstCombine/apint-div2.ll
+Transforms/InstCombine/apint-rem1.ll
+Transforms/InstCombine/apint-rem2.ll
+Transforms/InstCombine/ashr-demand.ll
+Transforms/InstCombine/atomic.ll
+Transforms/InstCombine/binop-cast.ll
+Transforms/InstCombine/binop-select-cast-of-select-cond.ll
+Transforms/InstCombine/binop-select.ll
+Transforms/InstCombine/bit-checks.ll
+Transforms/InstCombine/bitreverse.ll
+Transforms/InstCombine/branch.ll
+Transforms/InstCombine/builtin-dynamic-object-size.ll
+Transforms/InstCombine/canonicalize-clamp-like-pattern-between-negative-and-positive-thresholds.ll
+Transforms/InstCombine/canonicalize-clamp-like-pattern-between-zero-and-positive-threshold.ll
+Transforms/InstCombine/cast-mul-select.ll
+Transforms/InstCombine/clamp-to-minmax.ll
+Transforms/InstCombine/conditional-negation.ll
+Transforms/InstCombine/cttz.ll
+Transforms/InstCombine/debuginfo-invert.ll
+Transforms/InstCombine/demorgan.ll
+Transforms/InstCombine/div.ll
+Transforms/InstCombine/div-shift.ll
+Transforms/InstCombine/fabs.ll
+Transforms/InstCombine/fcmp-select.ll
+Transforms/InstCombine/ffs-1.ll
+Transforms/InstCombine/ffs-i16.ll
+Transforms/InstCombine/fmul-bool.ll
+Transforms/InstCombine/fmul.ll
+Transforms/InstCombine/fneg.ll
+Transforms/InstCombine/fold-ctpop-of-not.ll
+Transforms/InstCombine/fold-ext-eq-c-with-op.ll
+Transforms/InstCombine/free-inversion.ll
+Transforms/InstCombine/icmp-and-lowbit-mask.ll
+Transforms/InstCombine/icmp-equality-test.ll
+Transforms/InstCombine/icmp.ll
+Transforms/InstCombine/icmp-mul-and.ll
+Transforms/InstCombine/icmp-of-and-x.ll
+Transforms/InstCombine/icmp-of-or-x.ll
+Transforms/InstCombine/icmp-select-implies-common-op.ll
+Transforms/InstCombine/icmp-select.ll
+Transforms/InstCombine/icmp-with-selects.ll
+Transforms/InstCombine/intrinsic-select.ll
+Transforms/InstCombine/known-never-nan.ll
+Transforms/InstCombine/ldexp-ext.ll
+Transforms/InstCombine/ldexp.ll
+Transforms/InstCombine/load-bitcast-select.ll
+Transforms/InstCombine/load.ll
+Transforms/InstCombine/load-select.ll
+Transforms/InstCombine/loadstore-metadata.ll
+Transforms/InstCombine/logical-select-inseltpoison.ll
+Transforms/InstCombine/logical-select.ll
+Transforms/InstCombine/lshr.ll
+Transforms/InstCombine/masked_intrinsics-inseltpoison.ll
+Transforms/InstCombine/masked_intrinsics.ll
+Transforms/InstCombine/memchr-11.ll
+Transforms/InstCombine/memchr-2.ll
+Transforms/InstCombine/memchr-3.ll
+Transforms/InstCombine/memchr-6.ll
+Transforms/InstCombine/memchr-7.ll
+Transforms/InstCombine/memchr-9.ll
+Transforms/InstCombine/memchr.ll
+Transforms/InstCombine/mem-gep-zidx.ll
+Transforms/InstCombine/memrchr-3.ll
+Transforms/InstCombine/memrchr-4.ll
+Transforms/InstCombine/minmax-fold.ll
+Transforms/InstCombine/minmax-fp.ll
+Transforms/InstCombine/minmax-intrinsics.ll
+Transforms/InstCombine/mul-inseltpoison.ll
+Transforms/InstCombine/mul.ll
+Transforms/InstCombine/mul-masked-bits.ll
+Transforms/InstCombine/mul-pow2.ll
+Transforms/InstCombine/multiple-uses-load-bitcast-select.ll
+Transforms/InstCombine/narrow.ll
+Transforms/InstCombine/negated-bitmask.ll
+Transforms/InstCombine/nested-select.ll
+Transforms/InstCombine/not.ll
+Transforms/InstCombine/or-bitmask.ll
+Transforms/InstCombine/or-fcmp.ll
+Transforms/InstCombine/or.ll
+Transforms/InstCombine/phi-select-constant.ll
+Transforms/InstCombine/pow-1.ll
+Transforms/InstCombine/pow-3.ll
+Transforms/InstCombine/pow-sqrt.ll
+Transforms/InstCombine/pr24354.ll
+Transforms/InstCombine/pr35515.ll
+Transforms/InstCombine/ptrtoint-nullgep.ll
+Transforms/InstCombine/pull-conditional-binop-through-shift.ll
+Transforms/InstCombine/rem.ll
+Transforms/InstCombine/sdiv-canonicalize.ll
+Transforms/InstCombine/sdiv-guard.ll
+Transforms/InstCombine/select-and-cmp.ll
+Transforms/InstCombine/select-and-or.ll
+Transforms/InstCombine/select_arithmetic.ll
+Transforms/InstCombine/select-bitext.ll
+Transforms/InstCombine/select-cmp-br.ll
+Transforms/InstCombine/select-cmp.ll
+Transforms/InstCombine/select-factorize.ll
+Transforms/InstCombine/select_frexp.ll
+Transforms/InstCombine/select-icmp-and.ll
+Transforms/InstCombine/select.ll
+Transforms/InstCombine/select-min-max.ll
+Transforms/InstCombine/select-of-symmetric-selects.ll
+Transforms/InstCombine/select-or-cmp.ll
+Transforms/InstCombine/select-safe-bool-transforms.ll
+Transforms/InstCombine/select-safe-impliedcond-transforms.ll
+Transforms/InstCombine/select-safe-transforms.ll
+Transforms/InstCombine/select-select.ll
+Transforms/InstCombine/select-with-extreme-eq-cond.ll
+Transforms/InstCombine/shift.ll
+Transforms/InstCombine/shuffle-select-narrow-inseltpoison.ll
+Transforms/InstCombine/shuffle-select-narrow.ll
+Transforms/InstCombine/simplify-demanded-fpclass.ll
+Transforms/InstCombine/sink-not-into-another-hand-of-logical-and.ll
+Transforms/InstCombine/sink-not-into-another-hand-of-logical-or.ll
+Transforms/InstCombine/sink-not-into-logical-and.ll
+Transforms/InstCombine/sink-not-into-logical-or.ll
+Transforms/InstCombine/strchr-1.ll
+Transforms/InstCombine/strchr-3.ll
+Transforms/InstCombine/strlen-1.ll
+Transforms/InstCombine/strrchr-3.ll
+Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll
+Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll
+Transforms/InstCombine/sub.ll
+Transforms/InstCombine/sub-xor-cmp.ll
+Transforms/InstCombine/truncating-saturate.ll
+Transforms/InstCombine/trunc-inseltpoison.ll
+Transforms/InstCombine/trunc.ll
+Transforms/InstCombine/unordered-fcmp-select.ll
+Transforms/InstCombine/urem-via-cmp-select.ll
+Transforms/InstCombine/vec_sext.ll
+Transforms/InstCombine/vector-urem.ll
+Transforms/InstCombine/wcslen-1.ll
+Transforms/InstCombine/wcslen-3.ll
+Transforms/InstCombine/X86/blend_x86.ll
+Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll
+Transforms/InstCombine/X86/x86-avx512.ll
+Transforms/InstCombine/xor-and-or.ll
+Transforms/InstCombine/xor-ashr.ll
+Transforms/InstCombine/xor.ll
+Transforms/InstCombine/zext-bool-add-sub.ll
+Transforms/InstCombine/zext-or-icmp.ll
+Transforms/IRCE/add-metadata-pre-post-loops.ll
+Transforms/IRCE/bad_expander.ll
+Transforms/IRCE/bug-mismatched-types.ll
+Transforms/IRCE/clamp.ll
+Transforms/IRCE/compound-loop-bound.ll
+Transforms/IRCE/conjunctive-checks.ll
+Transforms/IRCE/correct-loop-info.ll
+Transforms/IRCE/decrementing-loop.ll
+Transforms/IRCE/eq_ne.ll
+Transforms/IRCE/iv-plus-offset-range-check.ll
+Transforms/IRCE/low-iterations.ll
+Transforms/IRCE/multiple-access-no-preloop.ll
+Transforms/IRCE/non_known_positive_end.ll
+Transforms/IRCE/non-loop-invariant-rhs-instr.ll
+Transforms/IRCE/only-lower-check.ll
+Transforms/IRCE/only-upper-check.ll
+Transforms/IRCE/optimistic_scev.ll
+Transforms/IRCE/pr57335.ll
+Transforms/IRCE/pre_post_loops.ll
+Transforms/IRCE/profitability.ll
+Transforms/IRCE/range_intersect_miscompile.ll
+Transforms/IRCE/ranges_of_different_types.ll
+Transforms/IRCE/rc-negative-bound.ll
+Transforms/IRCE/single-access-no-preloop.ll
+Transforms/IRCE/single-access-with-preloop.ll
+Transforms/IRCE/skip-profitability-checks.ll
+Transforms/IRCE/stride_more_than_1.ll
+Transforms/IRCE/unsigned_comparisons_ugt.ll
+Transforms/IRCE/unsigned_comparisons_ult.ll
+Transforms/IRCE/variable-loop-bounds.ll
+Transforms/IRCE/wide_indvar.ll
+Transforms/IRCE/with-parent-loops.ll
+Transforms/IROutliner/alloca-addrspace-1.ll
+Transforms/IROutliner/alloca-addrspace.ll
+Transforms/IROutliner/different-intrinsics.ll
+Transforms/IROutliner/different-order-phi-merges.ll
+Transforms/IROutliner/duplicate-merging-phis.ll
+Transforms/IROutliner/exit-block-phi-node-value-attribution.ll
+Transforms/IROutliner/exit-phi-nodes-incoming-value-constant-argument.ll
+Transforms/IROutliner/extraction.ll
+Transforms/IROutliner/gvn-output-set-overload.ll
+Transforms/IROutliner/gvn-phi-debug.ll
+Transforms/IROutliner/illegal-assumes.ll
+Transforms/IROutliner/illegal-branches.ll
+Transforms/IROutliner/illegal-callbr.ll
+Transforms/IROutliner/illegal-catchpad.ll
+Transforms/IROutliner/illegal-cleanup.ll
+Transforms/IROutliner/illegal-frozen.ll
+Transforms/IROutliner/illegal-indirect-calls.ll
+Transforms/IROutliner/illegal-invoke.ll
+Transforms/IROutliner/illegal-landingpad.ll
+Transforms/IROutliner/illegal-memcpy.ll
+Transforms/IROutliner/illegal-memmove.ll
+Transforms/IROutliner/illegal-memset.ll
+Transforms/IROutliner/illegal-phi-nodes.ll
+Transforms/IROutliner/illegal-returns-twice.ll
+Transforms/IROutliner/illegal-vaarg.ll
+Transforms/IROutliner/included-phi-nodes-begin.ll
+Transforms/IROutliner/included-phi-nodes-end.ll
+Transforms/IROutliner/legal-debug.ll
+Transforms/IROutliner/legal-indirect-calls.ll
+Transforms/IROutliner/mismatched-phi-exits.ll
+Transforms/IROutliner/mismatched-phi-exits-not-in-first-outlined.ll
+Transforms/IROutliner/mismatched-phi-outputs-ordering.ll
+Transforms/IROutliner/must-capture-all-phi-nodes-begin.ll
+Transforms/IROutliner/no-external-block-entries.ll
+Transforms/IROutliner/one-external-incoming-block-phi-node.ll
+Transforms/IROutliner/opt-remarks.ll
+Transforms/IROutliner/outline-memcpy.ll
+Transforms/IROutliner/outline-memmove.ll
+Transforms/IROutliner/outline-memset.ll
+Transforms/IROutliner/outline-vaarg-intrinsic.ll
+Transforms/IROutliner/outlining-across-branch.ll
+Transforms/IROutliner/outlining-address-taken.ll
+Transforms/IROutliner/outlining-basic-branches.ll
+Transforms/IROutliner/outlining-bitcasts.ll
+Transforms/IROutliner/outlining-branches-phi-nodes.ll
+Transforms/IROutliner/outlining-call-and-indirect.ll
+Transforms/IROutliner/outlining-calls.ll
+Transforms/IROutliner/outlining-calls-names-must-match.ll
+Transforms/IROutliner/outlining-commutative-fp.ll
+Transforms/IROutliner/outlining-commutative.ll
+Transforms/IROutliner/outlining-commutative-operands-opposite-order.ll
+Transforms/IROutliner/outlining-compatible-and-attribute-transfer.ll
+Transforms/IROutliner/outlining-compatible-or-attribute-transfer.ll
+Transforms/IROutliner/outlining-constants-vs-registers.ll
+Transforms/IROutliner/outlining-cost-model.ll
+Transforms/IROutliner/outlining-debug-statements.ll
+Transforms/IROutliner/outlining-different-constants.ll
+Transforms/IROutliner/outlining-different-globals.ll
+Transforms/IROutliner/outlining-different-output-blocks.ll
+Transforms/IROutliner/outlining-different-structure.ll
+Transforms/IROutliner/outlining-exits-to-phi-node.ll
+Transforms/IROutliner/outlining-first-instruction.ll
+Transforms/IROutliner/outlining-gep.ll
+Transforms/IROutliner/outlining-isomorphic-predicates.ll
+Transforms/IROutliner/outlining-larger-size-commutative.ll
+Transforms/IROutliner/outlining-multiple-exits-diff-outputs.ll
+Transforms/IROutliner/outlining-multiple-exits.ll
+Transforms/IROutliner/outlining-multiple-exits-one-output-set.ll
+Transforms/IROutliner/outlining-no-return-functions.ll
+Transforms/IROutliner/outlining-odr.ll
+Transforms/IROutliner/outlining-remapped-outputs.ll
+Transforms/IROutliner/outlining-same-constants.ll
+Transforms/IROutliner/outlining-same-globals.ll
+Transforms/IROutliner/outlining-same-output-blocks.ll
+Transforms/IROutliner/outlining-strip-loop-info.ll
+Transforms/IROutliner/outlining-swift-error.ll
+Transforms/IROutliner/phi-node-exit-path-order.ll
+Transforms/IROutliner/phi-nodes-non-constant.ll
+Transforms/IROutliner/phi-nodes-output-overload.ll
+Transforms/IROutliner/phi-nodes-parent-block-referential.ll
+Transforms/IROutliner/phi-nodes-simple.ll
+Transforms/IROutliner/region-end-of-module.ll
+Transforms/IROutliner/region-inputs-in-phi-nodes.ll
+Transforms/LoopBoundSplit/bug51866.ll
+Transforms/LoopBoundSplit/bug-loop-bound-split-phi-in-exit-block.ll
+Transforms/LoopBoundSplit/loop-bound-split.ll
+Transforms/LoopDeletion/invalidate-scev-after-hoisting.ll
+Transforms/LoopDistribute/basic-with-memchecks.ll
+Transforms/LoopDistribute/bounds-expansion-bug.ll
+Transforms/LoopDistribute/cross-partition-access.ll
+Transforms/LoopDistribute/debug-loc.ll
+Transforms/LoopDistribute/debug-print.ll
+Transforms/LoopDistribute/diagnostics.ll
+Transforms/LoopDistribute/followup.ll
+Transforms/LoopDistribute/laa-invalidation.ll
+Transforms/LoopDistribute/outside-use.ll
+Transforms/LoopDistribute/pointer-phi-in-loop.ll
+Transforms/LoopDistribute/scev-inserted-runtime-check.ll
+Transforms/LoopDistribute/symbolic-stride.ll
+Transforms/LoopFlatten/loop-flatten-version.ll
+Transforms/LoopFlatten/widen-iv2.ll
+Transforms/LoopFlatten/widen-iv.ll
+Transforms/LoopIdiom/AArch64/byte-compare-index.ll
+Transforms/LoopIdiom/AArch64/find-first-byte.ll
+Transforms/LoopIdiom/memset-runtime-32bit.ll
+Transforms/LoopIdiom/memset-runtime-64bit.ll
+Transforms/LoopIdiom/RISCV/byte-compare-index.ll
+Transforms/LoopIdiom/X86/arithmetic-right-shift-until-zero.ll
+Transforms/LoopIdiom/X86/left-shift-until-bittest.ll
+Transforms/LoopIdiom/X86/left-shift-until-zero.ll
+Transforms/LoopIdiom/X86/logical-right-shift-until-zero-debuginfo.ll
+Transforms/LoopIdiom/X86/logical-right-shift-until-zero.ll
+Transforms/LoopLoadElim/forward.ll
+Transforms/LoopLoadElim/invalidate-laa-after-versioning.ll
+Transforms/LoopLoadElim/memcheck.ll
+Transforms/LoopLoadElim/pr47457.ll
+Transforms/LoopLoadElim/symbolic-stride.ll
+Transforms/LoopLoadElim/unknown-stride-known-dep.ll
+Transforms/LoopLoadElim/versioning-scev-invalidation.ll
+Transforms/LoopPredication/preserve-bpi.ll
+Transforms/LoopSimplifyCFG/constant-fold-branch.ll
+Transforms/LoopSimplifyCFG/handle_dead_exits.ll
+Transforms/LoopSimplifyCFG/invalidate-scev-dispositions-2.ll
+Transforms/LoopSimplifyCFG/invalidate-scev-dispositions.ll
+Transforms/LoopSimplifyCFG/lcssa.ll
+Transforms/LoopSimplifyCFG/live_block_marking.ll
+Transforms/LoopSimplifyCFG/mssa_update.ll
+Transforms/LoopSimplifyCFG/pr117537.ll
+Transforms/LoopSimplifyCFG/update_parents.ll
+Transforms/LoopSimplify/pr26682.ll
+Transforms/LoopSimplify/preserve-llvm-loop-metadata.ll
+Transforms/LoopUnroll/AArch64/apple-unrolling-multi-exit.ll
+Transforms/LoopUnroll/AArch64/unrolling-multi-exit.ll
+Transforms/LoopUnroll/peel-last-iteration-expansion-cost.ll
+Transforms/LoopUnroll/peel-last-iteration-with-guards.ll
+Transforms/LoopUnroll/peel-last-iteration-with-variable-trip-count.ll
+Transforms/LoopVectorize/12-12-11-if-conv.ll
+Transforms/LoopVectorize/AArch64/aarch64-predication.ll
+Transforms/LoopVectorize/AArch64/arith-fp-frem-costs.ll
+Transforms/LoopVectorize/AArch64/blend-costs.ll
+Transforms/LoopVectorize/AArch64/check-prof-info.ll
+Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
+Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+Transforms/LoopVectorize/AArch64/deterministic-type-shrinkage.ll
+Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
+Transforms/LoopVectorize/AArch64/drop-poison-generating-flags.ll
+Transforms/LoopVectorize/AArch64/early_exit_costs.ll
+Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
+Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
+Transforms/LoopVectorize/AArch64/epilog-vectorization-widen-inductions.ll
+Transforms/LoopVectorize/AArch64/extend-vectorization-factor-for-unprofitable-memops.ll
+Transforms/LoopVectorize/AArch64/f128-fmuladd-reduction.ll
+Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
+Transforms/LoopVectorize/AArch64/first-order-recurrence.ll
+Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll
+Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll
+Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+Transforms/LoopVectorize/AArch64/induction-costs.ll
+Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+Transforms/LoopVectorize/AArch64/interleave_count_for_estimated_tc.ll
+Transforms/LoopVectorize/AArch64/interleaved_cost.ll
+Transforms/LoopVectorize/AArch64/interleave-with-gaps.ll
+Transforms/LoopVectorize/AArch64/interleave-with-runtime-checks.ll
+Transforms/LoopVectorize/AArch64/interleaving-load-store.ll
+Transforms/LoopVectorize/AArch64/interleaving-reduction.ll
+Transforms/LoopVectorize/AArch64/intrinsiccost.ll
+Transforms/LoopVectorize/AArch64/invariant-replicate-region.ll
+Transforms/LoopVectorize/AArch64/loop-vectorization-factors.ll
+Transforms/LoopVectorize/AArch64/loopvectorize_pr33804_double.ll
+Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
+Transforms/LoopVectorize/AArch64/low_trip_memcheck_cost.ll
+Transforms/LoopVectorize/AArch64/masked-call.ll
+Transforms/LoopVectorize/AArch64/masked-call-scalarize.ll
+Transforms/LoopVectorize/AArch64/multiple-result-intrinsics.ll
+Transforms/LoopVectorize/AArch64/neoverse-epilogue-vect.ll
+Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
+Transforms/LoopVectorize/AArch64/outer_loop_test1_no_explicit_vect_width.ll
+Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
+Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-neon.ll
+Transforms/LoopVectorize/AArch64/partial-reduce.ll
+Transforms/LoopVectorize/AArch64/pr31900.ll
+Transforms/LoopVectorize/AArch64/pr33053.ll
+Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
+Transforms/LoopVectorize/AArch64/predicated-costs.ll
+Transforms/LoopVectorize/AArch64/predication_costs.ll
+Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+Transforms/LoopVectorize/AArch64/reduction-small-size.ll
+Transforms/LoopVectorize/AArch64/reg-usage.ll
+Transforms/LoopVectorize/AArch64/runtime-check-trip-count-decisions.ll
+Transforms/LoopVectorize/AArch64/scalable-call.ll
+Transforms/LoopVectorize/AArch64/scalable-predicate-instruction.ll
+Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll
+Transforms/LoopVectorize/AArch64/scalable-reductions.ll
+Transforms/LoopVectorize/AArch64/scalable-reductions-tf.ll
+Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+Transforms/LoopVectorize/AArch64/scalable-struct-return.ll
+Transforms/LoopVectorize/AArch64/scalable-vectorization-cost-tuning.ll
+Transforms/LoopVectorize/AArch64/scalable-vectorization.ll
+Transforms/LoopVectorize/AArch64/scalarize-store-with-predication.ll
+Transforms/LoopVectorize/AArch64/sdiv-pow2.ll
+Transforms/LoopVectorize/AArch64/select-costs.ll
+Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll
+Transforms/LoopVectorize/AArch64/smallest-and-widest-types.ll
+Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+Transforms/LoopVectorize/AArch64/strict-fadd.ll
+Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+Transforms/LoopVectorize/AArch64/sve-basic-vec.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
+Transforms/LoopVectorize/AArch64/sve-epilog-vect-vscale-tune.ll
+Transforms/LoopVectorize/AArch64/sve-gather-scatter-cost.ll
+Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll
+Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
+Transforms/LoopVectorize/AArch64/sve-large-strides.ll
+Transforms/LoopVectorize/AArch64/sve-multiexit.ll
+Transforms/LoopVectorize/AArch64/sve-select-cmp.ll
+Transforms/LoopVectorize/AArch64/sve-tail-folding-cost.ll
+Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
+Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
+Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
+Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll
+Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
+Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
+Transforms/LoopVectorize/AArch64/tail-fold-uniform-memops.ll
+Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-cost.ll
+Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-remove-loop-region.ll
+Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
+Transforms/LoopVectorize/AArch64/unsafe-vf-hint-remark.ll
+Transforms/LoopVectorize/AArch64/veclib-function-calls.ll
+Transforms/LoopVectorize/AArch64/veclib-intrinsic-calls.ll
+Transforms/LoopVectorize/AArch64/vplan-printing.ll
+Transforms/LoopVectorize/ARM/mve-gather-scatter-tailpred.ll
+Transforms/LoopVectorize/ARM/mve-icmpcost.ll
+Transforms/LoopVectorize/ARM/mve-multiexit.ll
+Transforms/LoopVectorize/ARM/mve-qabs.ll
+Transforms/LoopVectorize/ARM/mve-reduction-predselect.ll
+Transforms/LoopVectorize/ARM/mve-reductions.ll
+Transforms/LoopVectorize/ARM/mve-reduction-types.ll
+Transforms/LoopVectorize/ARM/mve-selectandorcost.ll
+Transforms/LoopVectorize/ARM/optsize_minsize.ll
+Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
+Transforms/LoopVectorize/ARM/scalar-block-cost.ll
+Transforms/LoopVectorize/ARM/tail-folding-allowed.ll
+Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
+Transforms/LoopVectorize/ARM/tail-folding-loop-hint.ll
+Transforms/LoopVectorize/ARM/tail-folding-not-allowed.ll
+Transforms/LoopVectorize/ARM/tail-folding-prefer-flag.ll
+Transforms/LoopVectorize/ARM/tail-folding-reductions-allowed.ll
+Transforms/LoopVectorize/as_cast.ll
+Transforms/LoopVectorize/assume.ll
+Transforms/LoopVectorize/bzip_reverse_loops.ll
+Transforms/LoopVectorize/calloc.ll
+Transforms/LoopVectorize/cast-induction.ll
+Transforms/LoopVectorize/consecutive-ptr-uniforms.ll
+Transforms/LoopVectorize/dbg-outer-loop-vect.ll
+Transforms/LoopVectorize/debugloc.ll
+Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
+Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
+Transforms/LoopVectorize/diag-with-hotness-info.ll
+Transforms/LoopVectorize/dont-fold-tail-for-divisible-TC.ll
+Transforms/LoopVectorize/early_exit_legality.ll
+Transforms/LoopVectorize/epilog-iv-select-cmp.ll
+Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
+Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+Transforms/LoopVectorize/epilog-vectorization-trunc-induction-steps.ll
+Transforms/LoopVectorize/explicit_outer_detection.ll
+Transforms/LoopVectorize/explicit_outer_uniform_diverg_branch.ll
+Transforms/LoopVectorize/first-order-recurrence-complex.ll
+Transforms/LoopVectorize/first-order-recurrence.ll
+Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
+Transforms/LoopVectorize/float-induction.ll
+Transforms/LoopVectorize/float-minmax-instruction-flag.ll
+Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll
+Transforms/LoopVectorize/fmax-without-fast-math-flags.ll
+Transforms/LoopVectorize/fmin-without-fast-math-flags.ll
+Transforms/LoopVectorize/forked-pointers.ll
+Transforms/LoopVectorize/gcc-examples.ll
+Transforms/LoopVectorize/Hexagon/invalidate-cm-after-invalidating-interleavegroups.ll
+Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll
+Transforms/LoopVectorize/hoist-loads.ll
+Transforms/LoopVectorize/i8-induction.ll
+Transforms/LoopVectorize/icmp-uniforms.ll
+Transforms/LoopVectorize/if-conversion.ll
+Transforms/LoopVectorize/if-conversion-nest.ll
+Transforms/LoopVectorize/if-pred-non-void.ll
+Transforms/LoopVectorize/if-pred-not-when-safe.ll
+Transforms/LoopVectorize/if-pred-stores.ll
+Transforms/LoopVectorize/if-reduction.ll
+Transforms/LoopVectorize/induction.ll
+Transforms/LoopVectorize/instruction-only-used-outside-of-loop.ll
+Transforms/LoopVectorize/interleave-and-scalarize-only.ll
+Transforms/LoopVectorize/interleaved-accesses-2.ll
+Transforms/LoopVectorize/interleaved-accesses-3.ll
+Transforms/LoopVectorize/interleaved-accesses-different-insert-position.ll
+Transforms/LoopVectorize/interleaved-accesses.ll
+Transforms/LoopVectorize/interleaved-accesses-masked-group.ll
+Transforms/LoopVectorize/interleaved-accesses-pred-stores.ll
+Transforms/LoopVectorize/interleaved-accesses-uniform-load.ll
+Transforms/LoopVectorize/invariant-store-vectorization-2.ll
+Transforms/LoopVectorize/invariant-store-vectorization.ll
+Transforms/LoopVectorize/is_fpclass.ll
+Transforms/LoopVectorize/iv-select-cmp-decreasing.ll
+Transforms/LoopVectorize/iv-select-cmp.ll
+Transforms/LoopVectorize/iv-select-cmp-nested-loop.ll
+Transforms/LoopVectorize/iv-select-cmp-no-wrap.ll
+Transforms/LoopVectorize/iv-select-cmp-trunc.ll
+Transforms/LoopVectorize/lcssa-crashes.ll
+Transforms/LoopVectorize/load-deref-pred-align.ll
+Transforms/LoopVectorize/load-deref-pred-neg-off.ll
+Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll
+Transforms/LoopVectorize/load-of-struct-deref-pred.ll
+Transforms/LoopVectorize/loop-form.ll
+Transforms/LoopVectorize/loop-with-constant-exit-condition.ll
+Transforms/LoopVectorize/memdep-fold-tail.ll
+Transforms/LoopVectorize/metadata.ll
+Transforms/LoopVectorize/minmax_reduction.ll
+Transforms/LoopVectorize/multiple-exits-versioning.ll
+Transforms/LoopVectorize/multiple-result-intrinsics.ll
+Transforms/LoopVectorize/noalias-scope-decl.ll
+Transforms/LoopVectorize/no_outside_user.ll
+Transforms/LoopVectorize/no_switch.ll
+Transforms/LoopVectorize/optimal-epilog-vectorization-liveout.ll
+Transforms/LoopVectorize/optimal-epilog-vectorization.ll
+Transforms/LoopVectorize/optimal-epilog-vectorization-scalable.ll
+Transforms/LoopVectorize/optsize.ll
+Transforms/LoopVectorize/outer_loop_hcfg_construction.ll
+Transforms/LoopVectorize/outer-loop-inner-latch-successors.ll
+Transforms/LoopVectorize/outer_loop_scalable.ll
+Transforms/LoopVectorize/outer_loop_test1.ll
+Transforms/LoopVectorize/outer_loop_test2.ll
+Transforms/LoopVectorize/outer-loop-vec-phi-predecessor-order.ll
+Transforms/LoopVectorize/outer-loop-wide-phis.ll
+Transforms/LoopVectorize/phi-cost.ll
+Transforms/LoopVectorize/pointer-induction.ll
+Transforms/LoopVectorize/PowerPC/exit-branch-cost.ll
+Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization.ll
+Transforms/LoopVectorize/PowerPC/optimal-epilog-vectorization-profitability.ll
+Transforms/LoopVectorize/PowerPC/small-loop-rdx.ll
+Transforms/LoopVectorize/PowerPC/vplan-scalarivsext-crash.ll
+Transforms/LoopVectorize/pr154045-dont-fold-extractelement-livein.ll
+Transforms/LoopVectorize/pr30654-phiscev-sext-trunc.ll
+Transforms/LoopVectorize/pr32859.ll
+Transforms/LoopVectorize/pr34681.ll
+Transforms/LoopVectorize/pr37248.ll
+Transforms/LoopVectorize/pr39099.ll
+Transforms/LoopVectorize/pr44488-predication.ll
+Transforms/LoopVectorize/pr45525.ll
+Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll
+Transforms/LoopVectorize/pr48832.ll
+Transforms/LoopVectorize/pr51614-fold-tail-by-masking.ll
+Transforms/LoopVectorize/pr55100-expand-scev-predicate-used.ll
+Transforms/LoopVectorize/pr55167-fold-tail-live-out.ll
+Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
+Transforms/LoopVectorize/predicate-switch.ll
+Transforms/LoopVectorize/preserve-dbg-loc-and-loop-metadata.ll
+Transforms/LoopVectorize/reduction-inloop-cond.ll
+Transforms/LoopVectorize/reduction-inloop.ll
+Transforms/LoopVectorize/reduction-inloop-pred.ll
+Transforms/LoopVectorize/reduction-inloop-uf4.ll
+Transforms/LoopVectorize/reduction.ll
+Transforms/LoopVectorize/reduction-order.ll
+Transforms/LoopVectorize/reduction-predselect.ll
+Transforms/LoopVectorize/reduction-small-size.ll
+Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll
+Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
+Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+Transforms/LoopVectorize/RISCV/divrem.ll
+Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
+Transforms/LoopVectorize/RISCV/mask-index-type.ll
+Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
+Transforms/LoopVectorize/RISCV/pr88802.ll
+Transforms/LoopVectorize/RISCV/preserve-dbg-loc.ll
+Transforms/LoopVectorize/RISCV/reductions.ll
+Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
+Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll
+Transforms/LoopVectorize/RISCV/strided-accesses.ll
+Transforms/LoopVectorize/RISCV/tail-folding-cond-reduction.ll
+Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll
+Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll
+Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll
+Transforms/LoopVectorize/RISCV/tail-folding-safe-dep-distance.ll
+Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
+Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+Transforms/LoopVectorize/runtime-checks-difference.ll
+Transforms/LoopVectorize/same-base-access.ll
+Transforms/LoopVectorize/scalable-assume.ll
+Transforms/LoopVectorize/scalable-first-order-recurrence.ll
+Transforms/LoopVectorize/scalable-noalias-scope-decl.ll
+Transforms/LoopVectorize/scalarized-bitcast.ll
+Transforms/LoopVectorize/scalarize-masked-call.ll
+Transforms/LoopVectorize/scalar-select.ll
+Transforms/LoopVectorize/scev-predicate-reasoning.ll
+Transforms/LoopVectorize/select-cmp.ll
+Transforms/LoopVectorize/select-cmp-multiuse.ll
+Transforms/LoopVectorize/select-cmp-predicated.ll
+Transforms/LoopVectorize/select-neg-cond.ll
+Transforms/LoopVectorize/select-reduction.ll
+Transforms/LoopVectorize/select-reduction-start-value-may-be-undef-or-poison.ll
+Transforms/LoopVectorize/select-with-fastflags.ll
+Transforms/LoopVectorize/single-early-exit-cond-poison.ll
+Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll
+Transforms/LoopVectorize/single-early-exit-interleave-hint.ll
+Transforms/LoopVectorize/single-early-exit-interleave.ll
+Transforms/LoopVectorize/single_early_exit_live_outs.ll
+Transforms/LoopVectorize/single_early_exit.ll
+Transforms/LoopVectorize/single_early_exit_with_outer_loop.ll
+Transforms/LoopVectorize/single-value-blend-phis.ll
+Transforms/LoopVectorize/skip-iterations.ll
+Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll
+Transforms/LoopVectorize/strict-fadd-interleave-only.ll
+Transforms/LoopVectorize/struct-return.ll
+Transforms/LoopVectorize/SystemZ/branch-for-predicated-block.ll
+Transforms/LoopVectorize/SystemZ/force-target-instruction-cost.ll
+Transforms/LoopVectorize/SystemZ/load-scalarization-cost-0.ll
+Transforms/LoopVectorize/SystemZ/mem-interleaving-costs.ll
+Transforms/LoopVectorize/SystemZ/pr38110.ll
+Transforms/LoopVectorize/SystemZ/pr47665.ll
+Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll
+Transforms/LoopVectorize/SystemZ/scalar-steps-with-users-demanding-all-lanes-and-first-lane-only.ll
+Transforms/LoopVectorize/tail-folding-alloca-in-loop.ll
+Transforms/LoopVectorize/tail-folding-counting-down.ll
+Transforms/LoopVectorize/tail-folding-optimize-vector-induction-width.ll
+Transforms/LoopVectorize/tail-folding-switch.ll
+Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+Transforms/LoopVectorize/trip-count-expansion-may-introduce-ub.ll
+Transforms/LoopVectorize/tripcount.ll
+Transforms/LoopVectorize/trunc-extended-icmps.ll
+Transforms/LoopVectorize/uncountable-early-exit-vplan.ll
+Transforms/LoopVectorize/uniform-blend.ll
+Transforms/LoopVectorize/unroll_nonlatch.ll
+Transforms/LoopVectorize/unused-blend-mask-for-first-operand.ll
+Transforms/LoopVectorize/vectorize-pointer-phis.ll
+Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll
+Transforms/LoopVectorize/vect-phiscev-sext-trunc.ll
+Transforms/LoopVectorize/vect.stats.ll
+Transforms/LoopVectorize/VE/disable_lv.ll
+Transforms/LoopVectorize/version-stride-with-integer-casts.ll
+Transforms/LoopVectorize/vplan-predicate-switch.ll
+Transforms/LoopVectorize/vplan-printing.ll
+Transforms/LoopVectorize/vplan-printing-outer-loop.ll
+Transforms/LoopVectorize/vplan-printing-reductions.ll
+Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
+Transforms/LoopVectorize/vplan-vectorize-inner-loop-reduction.ll
+Transforms/LoopVectorize/vplan-widen-call-instruction.ll
+Transforms/LoopVectorize/vplan-widen-select-instruction.ll
+Transforms/LoopVectorize/X86/avx1.ll
+Transforms/LoopVectorize/X86/avx512.ll
+Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll
+Transforms/LoopVectorize/X86/constant-fold.ll
+Transforms/LoopVectorize/X86/conversion-cost.ll
+Transforms/LoopVectorize/X86/cost-constant-known-via-scev.ll
+Transforms/LoopVectorize/X86/CostModel/handle-iptr-with-data-layout-to-not-assert.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f32-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-f64-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i16-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-01u.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3-indices-0uu.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i32-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i64-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-load-i8-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f32-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-f64-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-6.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i16-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i32-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i64-stride-7.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-3.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-5.ll
+Transforms/LoopVectorize/X86/CostModel/interleaved-store-i8-stride-6.ll
+Transforms/LoopVectorize/X86/cost-model.ll
+Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll
+Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll
+Transforms/LoopVectorize/X86/CostModel/masked-interleaved-store-i16.ll
+Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll
+Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll
+Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll
+Transforms/LoopVectorize/X86/CostModel/masked-load-i8.ll
+Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll
+Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll
+Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll
+Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll
+Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll
+Transforms/LoopVectorize/X86/CostModel/masked-store-i8.ll
+Transforms/LoopVectorize/X86/divs-with-tail-folding.ll
+Transforms/LoopVectorize/X86/drop-inbounds-flags-for-reverse-vector-pointer.ll
+Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
+Transforms/LoopVectorize/X86/epilog-vectorization-inductions.ll
+Transforms/LoopVectorize/X86/fixed-order-recurrence.ll
+Transforms/LoopVectorize/X86/float-induction-x86.ll
+Transforms/LoopVectorize/X86/fp32_to_uint32-cost-model.ll
+Transforms/LoopVectorize/X86/fp64_to_uint32-cost-model.ll
+Transforms/LoopVectorize/X86/gather_scatter.ll
+Transforms/LoopVectorize/X86/imprecise-through-phis.ll
+Transforms/LoopVectorize/X86/induction-costs.ll
+Transforms/LoopVectorize/X86/interleaved-accesses-use-after-free.ll
+Transforms/LoopVectorize/X86/interleaved-accesses-waw-dependency.ll
+Transforms/LoopVectorize/X86/intrinsiccost.ll
+Transforms/LoopVectorize/X86/invariant-load-gather.ll
+Transforms/LoopVectorize/X86/invariant-store-vectorization.ll
+Transforms/LoopVectorize/X86/limit-vf-by-tripcount.ll
+Transforms/LoopVectorize/X86/load-deref-pred.ll
+Transforms/LoopVectorize/X86/masked_load_store.ll
+Transforms/LoopVectorize/X86/masked-store-cost.ll
+Transforms/LoopVectorize/X86/multi-exit-cost.ll
+Transforms/LoopVectorize/X86/no_fpmath.ll
+Transforms/LoopVectorize/X86/no_fpmath_with_hotness.ll
+Transforms/LoopVectorize/X86/optsize.ll
+Transforms/LoopVectorize/X86/outer_loop_test1_no_explicit_vect_width.ll
+Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
+Transforms/LoopVectorize/X86/pr141968-instsimplifyfolder.ll
+Transforms/LoopVectorize/X86/pr23997.ll
+Transforms/LoopVectorize/X86/pr51366-sunk-instruction-used-outside-of-loop.ll
+Transforms/LoopVectorize/X86/pr54634.ll
+Transforms/LoopVectorize/X86/pr55096-scalarize-add.ll
+Transforms/LoopVectorize/X86/pr56319-vector-exit-cond-optimization-epilogue-vectorization.ll
+Transforms/LoopVectorize/X86/pr81872.ll
+Transforms/LoopVectorize/X86/predicate-switch.ll
+Transforms/LoopVectorize/X86/propagate-metadata.ll
+Transforms/LoopVectorize/X86/reduction-fastmath.ll
+Transforms/LoopVectorize/X86/reg-usage.ll
+Transforms/LoopVectorize/X86/replicate-recipe-with-only-first-lane-used.ll
+Transforms/LoopVectorize/X86/replicate-uniform-call.ll
+Transforms/LoopVectorize/X86/scatter_crash.ll
+Transforms/LoopVectorize/X86/small-size.ll
+Transforms/LoopVectorize/X86/strided_load_cost.ll
+Transforms/LoopVectorize/X86/tail_folding_and_assume_safety.ll
+Transforms/LoopVectorize/X86/tail_loop_folding.ll
+Transforms/LoopVectorize/X86/transform-narrow-interleave-to-widen-memory.ll
+Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
+Transforms/LoopVectorize/X86/vectorize-force-tail-with-evl.ll
+Transforms/LoopVectorize/X86/vectorize-interleaved-accesses-gap.ll
+Transforms/LoopVectorize/X86/vector_max_bandwidth.ll
+Transforms/LoopVectorize/X86/vector_ptr_load_store.ll
+Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll
+Transforms/LoopVectorize/X86/x86-interleaved-accesses-masked-group.ll
+Transforms/LoopVectorize/X86/x86-interleaved-store-accesses-with-gaps.ll
+Transforms/LoopVectorize/X86/x86-pr39099.ll
+Transforms/LoopVectorize/X86/x86-predication.ll
+Transforms/LoopVersioning/add-phi-update-users.ll
+Transforms/LoopVersioning/basic.ll
+Transforms/LoopVersioning/bound-check-partially-known.ll
+Transforms/LoopVersioning/crash-36998.ll
+Transforms/LoopVersioning/exit-block-dominates-rt-check-block.ll
+Transforms/LoopVersioning/incorrect-phi.ll
+Transforms/LoopVersioning/invalidate-laa-after-versioning.ll
+Transforms/LoopVersioning/lcssa.ll
+Transforms/LoopVersioningLICM/load-from-unknown-address.ll
+Transforms/LoopVersioningLICM/loopversioningLICM1.ll
+Transforms/LoopVersioningLICM/loopversioningLICM2.ll
+Transforms/LoopVersioningLICM/metadata.ll
+Transforms/LoopVersioning/loop-invariant-bound.ll
+Transforms/LoopVersioning/noalias.ll
+Transforms/LoopVersioning/noalias-version-twice.ll
+Transforms/LoopVersioning/single-iteration.ll
+Transforms/LoopVersioning/wrapping-pointer-non-integral-addrspace.ll
+Transforms/LoopVersioning/wrapping-pointer-versioning.ll
+Transforms/LowerAtomic/atomic-load.ll
+Transforms/LowerAtomic/atomic-swap.ll
+Transforms/LowerConstantIntrinsics/builtin-object-size-phi.ll
+Transforms/LowerConstantIntrinsics/objectsize_basic.ll
+Transforms/LowerGlobalDestructors/lower-global-dtors-existing-dos_handle.ll
+Transforms/LowerGlobalDestructors/lower-global-dtors.ll
+Transforms/LowerGlobalDestructors/non-literal-type.ll
+Transforms/LowerIFunc/ifunc-alias.ll
+Transforms/LowerIFunc/ifunc-nonsense-resolvers.ll
+Transforms/LowerIFunc/ifunc-program-addrspace.ll
+Transforms/LowerIFunc/lower-ifunc.ll
+Transforms/LowerMatrixIntrinsics/multiply-fused-dominance.ll
+Transforms/LowerMatrixIntrinsics/multiply-fused.ll
+Transforms/LowerMatrixIntrinsics/multiply-fused-loops.ll
+Transforms/LowerMatrixIntrinsics/multiply-fused-multiple-blocks.ll
+Transforms/LowerMatrixIntrinsics/multiply-fused-volatile.ll
+Transforms/LowerMatrixIntrinsics/select.ll
+Transforms/LowerSwitch/2003-05-01-PHIProblem.ll
+Transforms/LowerSwitch/2004-03-13-SwitchIsDefaultCrash.ll
+Transforms/LowerSwitch/2014-06-10-SwitchContiguousOpt.ll
+Transforms/LowerSwitch/2014-06-11-SwitchDefaultUnreachableOpt.ll
+Transforms/LowerSwitch/2014-06-23-PHIlowering.ll
+Transforms/LowerSwitch/93152.ll
+Transforms/LowerSwitch/condition-phi-unreachable-default.ll
+Transforms/LowerSwitch/delete-default-block-crash.ll
+Transforms/LowerSwitch/do-not-handle-impossible-values.ll
+Transforms/LowerSwitch/feature.ll
+Transforms/LowerSwitch/fold-popular-case-to-unreachable-default.ll
+Transforms/LowerSwitch/pr59316.ll
+Transforms/LowerTypeTests/aarch64-jumptable.ll
+Transforms/LowerTypeTests/blockaddress-2.ll
+Transforms/LowerTypeTests/blockaddress.ll
+Transforms/LowerTypeTests/cfi-annotation.ll
+Transforms/LowerTypeTests/cfi-coff-comdat-rename.ll
+Transforms/LowerTypeTests/cfi-direct-call1.ll
+Transforms/LowerTypeTests/cfi-icall-alias.ll
+Transforms/LowerTypeTests/cfi-nounwind-direct-call.ll
+Transforms/LowerTypeTests/cfi-unwind-direct-call.ll
+Transforms/LowerTypeTests/export-alias.ll
+Transforms/LowerTypeTests/export-cross-dso-cfi.ll
+Transforms/LowerTypeTests/export-icall.ll
+Transforms/LowerTypeTests/export-rename-local.ll
+Transforms/LowerTypeTests/export-symver.ll
+Transforms/LowerTypeTests/function-arm-thumb.ll
+Transforms/LowerTypeTests/function-disjoint.ll
+Transforms/LowerTypeTests/function-ext.ll
+Transforms/LowerTypeTests/function.ll
+Transforms/LowerTypeTests/function-thumb-bti.ll
+Transforms/LowerTypeTests/function-weak.ll
+Transforms/LowerTypeTests/icall-branch-funnel.ll
+Transforms/LowerTypeTests/import.ll
+Transforms/LowerTypeTests/nocfivalue.ll
+Transforms/LowerTypeTests/pr37625.ll
+Transforms/LowerTypeTests/section.ll
+Transforms/LowerTypeTests/simple.ll
+Transforms/LowerTypeTests/x86-jumptable.ll
+Transforms/MemCpyOpt/memset-memcpy-dbgloc.ll
+Transforms/MemCpyOpt/memset-memcpy-redundant-memset.ll
+Transforms/MemCpyOpt/opaque-ptr.ll
+Transforms/MergeFunc/2011-02-08-RemoveEqual.ll
+Transforms/MergeFunc/apply_function_attributes.ll
+Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
+Transforms/MergeFunc/call-and-invoke-with-ranges.ll
+Transforms/MergeFunc/cfi-thunk-merging.ll
+Transforms/MergeFunc/comdat.ll
+Transforms/MergeFunc/crash-cast-arrays.ll
+Transforms/MergeFunc/crash.ll
+Transforms/MergeFunc/different-index-width-gep-crash.ll
+Transforms/MergeFunc/external-before-local.ll
+Transforms/MergeFunc/fold-weak.ll
+Transforms/MergeFunc/inline-asm.ll
+Transforms/MergeFunc/inttoptr-address-space.ll
+Transforms/MergeFunc/inttoptr.ll
+Transforms/MergeFunc/linkonce.ll
+Transforms/MergeFunc/linkonce_odr.ll
+Transforms/MergeFunc/merge-block-address.ll
+Transforms/MergeFunc/merge-fp-intrinsics.ll
+Transforms/MergeFunc/mergefunc-preserve-nonnull.ll
+Transforms/MergeFunc/mergefunc-preserve-vfe-intrinsics.ll
+Transforms/MergeFunc/mergefunc-struct-return.ll
+Transforms/MergeFunc/merge-linkonce-odr-used.ll
+Transforms/MergeFunc/merge-linkonce-odr-weak-odr-mixed-used.ll
+Transforms/MergeFunc/merge-ptr-and-int.ll
+Transforms/MergeFunc/merge-used.ll
+Transforms/MergeFunc/merge-weak-crash.ll
+Transforms/MergeFunc/merge-weak-odr.ll
+Transforms/MergeFunc/merge-weak-odr-used.ll
+Transforms/MergeFunc/metadata-call-arguments.ll
+Transforms/MergeFunc/nonzero-address-spaces.ll
+Transforms/MergeFunc/phi-speculation2.ll
+Transforms/MergeFunc/ptr-int-transitivity-2.ll
+Transforms/MergeFunc/ranges.ll
+Transforms/MergeFunc/ranges-multiple.ll
+Transforms/MergeFunc/self-referential-global.ll
+Transforms/MergeFunc/unnamed-addr-reprocessing.ll
+Transforms/MergeFunc/vector-GEP-crash.ll
+Transforms/MergeICmps/X86/alias-merge-blocks.ll
+Transforms/MergeICmps/X86/entry-block-shuffled-2.ll
+Transforms/MergeICmps/X86/entry-block-shuffled.ll
+Transforms/MergeICmps/X86/pr59740.ll
+Transforms/OpenMP/always_inline_device.ll
+Transforms/OpenMP/custom_state_machines.ll
+Transforms/OpenMP/custom_state_machines_remarks.ll
+Transforms/OpenMP/get_hardware_num_threads_in_block_fold.ll
+Transforms/OpenMP/gpu_state_machine_function_ptr_replacement.ll
+Transforms/OpenMP/parallel_region_merging.ll
+Transforms/OpenMP/single_threaded_execution.ll
+Transforms/OpenMP/spmdization_assumes.ll
+Transforms/OpenMP/spmdization_guarding.ll
+Transforms/OpenMP/spmdization_guarding_two_reaching_kernels.ll
+Transforms/OpenMP/spmdization_indirect.ll
+Transforms/OpenMP/spmdization.ll
+Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
+Transforms/OpenMP/spmdization_remarks.ll
+Transforms/PartiallyInlineLibCalls/X86/good-prototype.ll
+Transforms/PGOProfile/chr-dead-pred.ll
+Transforms/PGOProfile/chr-dup-threshold.ll
+Transforms/PGOProfile/chr.ll
+Transforms/PGOProfile/chr-poison.ll
+Transforms/PGOProfile/comdat.ll
+Transforms/PGOProfile/cspgo_profile_summary.ll
+Transforms/PGOProfile/memop_profile_funclet_wasm.ll
+Transforms/PGOProfile/profcheck-select.ll
+Transforms/PGOProfile/prof-verify.ll
+Transforms/PGOProfile/prof-verify-no-entrycount.ll
+Transforms/PGOProfile/X86/macho.ll
+Transforms/PhaseOrdering/AArch64/globals-aa-required-for-vectorization.ll
+Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
+Transforms/PhaseOrdering/AArch64/loopflatten.ll
+Transforms/PhaseOrdering/AArch64/matrix-extract-insert.ll
+Transforms/PhaseOrdering/AArch64/predicated-reduction.ll
+Transforms/PhaseOrdering/AArch64/quant_4x4.ll
+Transforms/PhaseOrdering/ARM/arm_mean_q7.ll
+Transforms/PhaseOrdering/lower-table-based-cttz.ll
+Transforms/PhaseOrdering/pr44461-br-to-switch-rotate.ll
+Transforms/PhaseOrdering/simplifycfg-switch-lowering-vs-correlatedpropagation.ll
+Transforms/PhaseOrdering/vector-select.ll
+Transforms/PhaseOrdering/X86/blendv-select.ll
+Transforms/PhaseOrdering/X86/merge-functions2.ll
+Transforms/PhaseOrdering/X86/merge-functions3.ll
+Transforms/PhaseOrdering/X86/merge-functions.ll
+Transforms/PhaseOrdering/X86/pr48844-br-to-switch-vectorization.ll
+Transforms/PhaseOrdering/X86/pr52078.ll
+Transforms/PhaseOrdering/X86/pr67803.ll
+Transforms/PhaseOrdering/X86/preserve-access-group.ll
+Transforms/PhaseOrdering/X86/simplifycfg-late.ll
+Transforms/PhaseOrdering/X86/vdiv.ll
+Transforms/PhaseOrdering/X86/vector-reductions.ll
+Transforms/PreISelIntrinsicLowering/AArch64/expand-exp.ll
+Transforms/PreISelIntrinsicLowering/AArch64/expand-log.ll
+Transforms/PreISelIntrinsicLowering/expand-vp.ll
+Transforms/PreISelIntrinsicLowering/PowerPC/memset-pattern.ll
+Transforms/PreISelIntrinsicLowering/RISCV/memset-pattern.ll
+Transforms/PreISelIntrinsicLowering/X86/memcpy-inline-non-constant-len.ll
+Transforms/PreISelIntrinsicLowering/X86/memset-inline-non-constant-len.ll
+Transforms/PreISelIntrinsicLowering/X86/memset-pattern.ll
+Transforms/Reassociate/basictest.ll
+Transforms/SampleProfile/pseudo-probe-dangle.ll
+Transforms/SampleProfile/pseudo-probe-emit.ll
+Transforms/SampleProfile/pseudo-probe-profile.ll
+Transforms/SampleProfile/pseudo-probe-profile-mismatch-thinlto.ll
+Transforms/SampleProfile/remarks.ll
+Transforms/SandboxVectorizer/special_opcodes.ll
+Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-load.ll
+Transforms/ScalarizeMaskedMemIntrin/AArch64/expand-masked-store.ll
+Transforms/ScalarizeMaskedMemIntrin/AArch64/streaming-compatible-expand-masked-gather-scatter.ll
+Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expamd-masked-load.ll
+Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-gather.ll
+Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-scatter.ll
+Transforms/ScalarizeMaskedMemIntrin/AMDGPU/expand-masked-store.ll
+Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-compressstore.ll
+Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-expandload.ll
+Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-gather.ll
+Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-load.ll
+Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-scatter.ll
+Transforms/ScalarizeMaskedMemIntrin/X86/expand-masked-store.ll
+Transforms/Scalarizer/basic-inseltpoison.ll
+Transforms/Scalarizer/basic.ll
+Transforms/Scalarizer/min-bits.ll
+Transforms/Scalarizer/scatter-order.ll
+Transforms/Scalarizer/variable-extractelement.ll
+Transforms/Scalarizer/variable-insertelement.ll
+Transforms/Scalarizer/vector-of-pointer-to-vector.ll
+Transforms/SimpleLoopUnswitch/debuginfo.ll
+Transforms/SimpleLoopUnswitch/delete-dead-blocks.ll
+Transforms/SimpleLoopUnswitch/endless-unswitch.ll
+Transforms/SimpleLoopUnswitch/guards.ll
+Transforms/SimpleLoopUnswitch/inject-invariant-conditions-exponential.ll
+Transforms/SimpleLoopUnswitch/inject-invariant-conditions.ll
+Transforms/SimpleLoopUnswitch/LIV-loop-condtion.ll
+Transforms/SimpleLoopUnswitch/nontrivial-unswitch-freeze.ll
+Transforms/SimpleLoopUnswitch/nontrivial-unswitch.ll
+Transforms/SimpleLoopUnswitch/nontrivial-unswitch-select.ll
+Transforms/SimpleLoopUnswitch/nontrivial-unswitch-skip-selects-in-guards.ll
+Transforms/SimpleLoopUnswitch/partial-unswitch.ll
+Transforms/SimpleLoopUnswitch/partial-unswitch-loop-and-block-dispositions.ll
+Transforms/SimpleLoopUnswitch/partial-unswitch-mssa-threshold.ll
+Transforms/SimpleLoopUnswitch/partial-unswitch-update-memoryssa.ll
+Transforms/SimpleLoopUnswitch/pr59546.ll
+Transforms/SimpleLoopUnswitch/pr60736.ll
+Transforms/SimpleLoopUnswitch/trivial-unswitch-freeze-individual-conditions.ll
+Transforms/SimpleLoopUnswitch/trivial-unswitch.ll
+Transforms/SimpleLoopUnswitch/trivial-unswitch-logical-and-or.ll
+Transforms/SimplifyCFG/2006-12-08-Ptr-ICmp-Branch.ll
+Transforms/SimplifyCFG/2008-10-03-SpeculativelyExecuteBeforePHI.ll
+Transforms/SimplifyCFG/annotations.ll
+Transforms/SimplifyCFG/ARM/branch-fold-threshold.ll
+Transforms/SimplifyCFG/ARM/phi-eliminate.ll
+Transforms/SimplifyCFG/ARM/select-trunc-i64.ll
+Transforms/SimplifyCFG/ARM/switch-to-lookup-table.ll
+Transforms/SimplifyCFG/basictest.ll
+Transforms/SimplifyCFG/branch-cond-dont-merge.ll
+Transforms/SimplifyCFG/branch-fold-dbg.ll
+Transforms/SimplifyCFG/branch-fold.ll
+Transforms/SimplifyCFG/branch-fold-multiple.ll
+Transforms/SimplifyCFG/branch-fold-threshold.ll
+Transforms/SimplifyCFG/branch-nested.ll
+Transforms/SimplifyCFG/clamp.ll
+Transforms/SimplifyCFG/common-code-hoisting.ll
+Transforms/SimplifyCFG/common-dest-folding.ll
+Transforms/SimplifyCFG/extract-cost.ll
+Transforms/SimplifyCFG/fold-branch-to-common-dest-free-cost.ll
+Transforms/SimplifyCFG/fold-branch-to-common-dest.ll
+Transforms/SimplifyCFG/fold-branch-to-common-dest-two-preds-cost.ll
+Transforms/SimplifyCFG/fold-debug-location.ll
+Transforms/SimplifyCFG/Hexagon/switch-to-lookup-table.ll
+Transforms/SimplifyCFG/hoist-dbgvalue.ll
+Transforms/SimplifyCFG/indirectbr.ll
+Transforms/SimplifyCFG/merge-cond-stores-2.ll
+Transforms/SimplifyCFG/merge-cond-stores.ll
+Transforms/SimplifyCFG/multiple-phis.ll
+Transforms/SimplifyCFG/PhiBlockMerge.ll
+Transforms/SimplifyCFG/pr48641.ll
+Transforms/SimplifyCFG/preserve-branchweights.ll
+Transforms/SimplifyCFG/preserve-store-alignment.ll
+Transforms/SimplifyCFG/rangereduce.ll
+Transforms/SimplifyCFG/RISCV/select-trunc-i64.ll
+Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv32.ll
+Transforms/SimplifyCFG/RISCV/switch_to_lookup_table-rv64.ll
+Transforms/SimplifyCFG/safe-abs.ll
+Transforms/SimplifyCFG/SimplifyEqualityComparisonWithOnlyPredecessor-domtree-preservation-edgecase.ll
+Transforms/SimplifyCFG/speculate-blocks.ll
+Transforms/SimplifyCFG/speculate-derefable-load.ll
+Transforms/SimplifyCFG/suppress-zero-branch-weights.ll
+Transforms/SimplifyCFG/switch_create-custom-dl.ll
+Transforms/SimplifyCFG/switch_create.ll
+Transforms/SimplifyCFG/switch-dup-bbs.ll
+Transforms/SimplifyCFG/switch_mask.ll
+Transforms/SimplifyCFG/switch_msan.ll
+Transforms/SimplifyCFG/switch-on-const-select.ll
+Transforms/SimplifyCFG/switchToSelect-domtree-preservation-edgecase.ll
+Transforms/SimplifyCFG/switch-to-select-multiple-edge-per-block-phi.ll
+Transforms/SimplifyCFG/switch-to-select-two-case.ll
+Transforms/SimplifyCFG/wc-widen-block.ll
+Transforms/SimplifyCFG/X86/disable-lookup-table.ll
+Transforms/SimplifyCFG/X86/hoist-loads-stores-with-cf.ll
+Transforms/SimplifyCFG/X86/SpeculativeExec.ll
+Transforms/SimplifyCFG/X86/switch-to-lookup-globals.ll
+Transforms/SimplifyCFG/X86/switch-to-lookup-large-types.ll
+Transforms/SimplifyCFG/X86/switch_to_lookup_table_big.ll
+Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
+Transforms/SLPVectorizer/AArch64/gather-root.ll
+Transforms/SLPVectorizer/AArch64/horizontal.ll
+Transforms/SLPVectorizer/AArch64/loadi8.ll
+Transforms/SLPVectorizer/AArch64/phi-node-bitwidt-op-not.ll
+Transforms/SLPVectorizer/AArch64/uselistorder.ll
+Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
+Transforms/SLPVectorizer/AArch64/vectorizable-selects-min-max.ll
+Transforms/SLPVectorizer/AArch64/vectorizable-selects-uniform-cmps.ll
+Transforms/SLPVectorizer/AMDGPU/horizontal-store.ll
+Transforms/SLPVectorizer/bool-logical-op-reduction-with-poison.ll
+Transforms/SLPVectorizer/call-arg-reduced-by-minbitwidth.ll
+Transforms/SLPVectorizer/const-bool-logical-or-reduction.ll
+Transforms/SLPVectorizer/extracts-with-undefs.ll
+Transforms/SLPVectorizer/freeze-signedness-missed.ll
+Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll
+Transforms/SLPVectorizer/gather_extract_from_vectorbuild.ll
+Transforms/SLPVectorizer/insert-element-build-vector-const.ll
+Transforms/SLPVectorizer/insert-element-build-vector-inseltpoison.ll
+Transforms/SLPVectorizer/insert-element-build-vector.ll
+Transforms/SLPVectorizer/logical-ops-poisonous-repeated.ll
+Transforms/SLPVectorizer/minbitwidth-node-with-multi-users.ll
+Transforms/SLPVectorizer/minbitwidth-user-not-min.ll
+Transforms/SLPVectorizer/partial-register-extract.ll
+Transforms/SLPVectorizer/reduction-gather-non-scheduled-extracts.ll
+Transforms/SLPVectorizer/reorder-node.ll
+Transforms/SLPVectorizer/reused-buildvector-matching-vectorized-node.ll
+Transforms/SLPVectorizer/revec.ll
+Transforms/SLPVectorizer/RISCV/long-gep-chains.ll
+Transforms/SLPVectorizer/RISCV/remarks_cmp_sel_min_max.ll
+Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll
+Transforms/SLPVectorizer/RISCV/reordered-interleaved-loads.ll
+Transforms/SLPVectorizer/RISCV/revec.ll
+Transforms/SLPVectorizer/RISCV/select-profitability.ll
+Transforms/SLPVectorizer/RISCV/shuffled-gather-casted.ll
+Transforms/SLPVectorizer/RISCV/unsigned-node-trunc-with-signed-users.ll
+Transforms/SLPVectorizer/slp-deleted-inst.ll
+Transforms/SLPVectorizer/SystemZ/cmp-ptr-minmax.ll
+Transforms/SLPVectorizer/SystemZ/ext-not-resized-op-resized.ll
+Transforms/SLPVectorizer/SystemZ/minbitwidth-trunc.ll
+Transforms/SLPVectorizer/X86/bool-mask.ll
+Transforms/SLPVectorizer/X86/bv-root-part-of-graph.ll
+Transforms/SLPVectorizer/X86/cmp-after-intrinsic-call-minbitwidth.ll
+Transforms/SLPVectorizer/X86/cmp-as-alternate-ops.ll
+Transforms/SLPVectorizer/X86/cmp_sel.ll
+Transforms/SLPVectorizer/X86/crash_7zip.ll
+Transforms/SLPVectorizer/X86/crash_clear_undefs.ll
+Transforms/SLPVectorizer/X86/crash_cmpop.ll
+Transforms/SLPVectorizer/X86/debug-counter.ll
+Transforms/SLPVectorizer/X86/debug-info-salvage.ll
+Transforms/SLPVectorizer/X86/extractelement-single-use-many-nodes.ll
+Transforms/SLPVectorizer/X86/extracts-non-extendable.ll
+Transforms/SLPVectorizer/X86/ext-used-scalar-different-bitwidth.ll
+Transforms/SLPVectorizer/X86/gather-node-same-as-vect-but-order.ll
+Transforms/SLPVectorizer/X86/horizontal-minmax.ll
+Transforms/SLPVectorizer/X86/insert-after-bundle.ll
+Transforms/SLPVectorizer/X86/jumbled-load-multiuse.ll
+Transforms/SLPVectorizer/X86/minbitwidth-icmp-to-trunc.ll
+Transforms/SLPVectorizer/X86/minbw-user-non-sizable.ll
+Transforms/SLPVectorizer/X86/non-load-reduced-as-part-of-bv.ll
+Transforms/SLPVectorizer/X86/ordering-bug.ll
+Transforms/SLPVectorizer/X86/phi-node-bitwidt-op-not.ll
+Transforms/SLPVectorizer/X86/phi-node-reshuffled-part.ll
+Transforms/SLPVectorizer/X86/pr46983.ll
+Transforms/SLPVectorizer/X86/pr49933.ll
+Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
+Transforms/SLPVectorizer/X86/reduction-bool-logic-op-inside.ll
+Transforms/SLPVectorizer/X86/reduction-logical.ll
+Transforms/SLPVectorizer/X86/resized-bv-values-non-power-of2-node.ll
+Transforms/SLPVectorizer/X86/reused-reductions-with-minbitwidth.ll
+Transforms/SLPVectorizer/X86/select-reduction-op.ll
+Transforms/SLPVectorizer/X86/shrink_after_reorder.ll
+Transforms/SLPVectorizer/X86/subvector-minbitwidth-unsigned-value.ll
+Transforms/SLPVectorizer/X86/undef_vect.ll
+Transforms/SLPVectorizer/X86/used-reduced-op.ll
+Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll
+Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll
+Transforms/SLPVectorizer/X86/whole-registers-compare.ll
+Transforms/SROA/addrspacecast.ll
+Transforms/SROA/phi-and-select.ll
+Transforms/SROA/phi-gep.ll
+Transforms/SROA/scalable-vectors-with-known-vscale.ll
+Transforms/SROA/select-gep.ll
+Transforms/SROA/select-load.ll
+Transforms/SROA/slice-width.ll
+Transforms/SROA/std-clamp.ll
+Transforms/SROA/vector-conversion.ll
+Transforms/SROA/vector-promotion.ll
+Transforms/StackProtector/cross-dso-cfi-stack-chk-fail.ll
+Transforms/StructurizeCFG/AMDGPU/uniform-regions.ll
+Transforms/StructurizeCFG/hoist-zerocost.ll
+Transforms/StructurizeCFG/loop-break-phi.ll
+Transforms/StructurizeCFG/nested-loop-order.ll
+Transforms/StructurizeCFG/simple-structurizecfg-crash.ll
+Transforms/StructurizeCFG/structurizer-keep-perf-md.ll
+Transforms/StructurizeCFG/switch.ll
+Transforms/TailCallElim/2010-06-26-MultipleReturnValues.ll
+Transforms/TailCallElim/accum_recursion.ll
+Transforms/TailCallElim/basic.ll
+Transforms/TailCallElim/debugloc.ll
+Transforms/TailCallElim/dropping_debugloc_acc_rec_inst_rnew.ll
+Transforms/TailCallElim/inf-recursion.ll
+Transforms/UnifyLoopExits/basic.ll
+Transforms/UnifyLoopExits/integer_guards.ll
+Transforms/UnifyLoopExits/nested.ll
+Transforms/UnifyLoopExits/restore-ssa.ll
+Transforms/UnifyLoopExits/switch.ll
+Transforms/UnifyLoopExits/undef-phis.ll
+Transforms/Util/libcalls-opt-remarks.ll
+Transforms/Util/lowerswitch.ll
+Transforms/VectorCombine/AArch64/shuffletoidentity.ll
+Transforms/VectorCombine/X86/shuffle-of-selects.ll
+Transforms/WholeProgramDevirt/branch-funnel.ll
+Transforms/WholeProgramDevirt/branch-funnel-threshold.ll
+Transforms/WholeProgramDevirt/devirt_single_after_filtering_unreachable_function.ll
+Transforms/WholeProgramDevirt/expand-check.ll
+Transforms/WholeProgramDevirt/unique-retval-same-vtable.ll
+Transforms/WholeProgramDevirt/vcp-accesses-memory.ll
+Transforms/WholeProgramDevirt/vcp-decl.ll
+Transforms/WholeProgramDevirt/vcp-non-constant-arg.ll
+Transforms/WholeProgramDevirt/vcp-no-this.ll
+Transforms/WholeProgramDevirt/vcp-too-wide-ints.ll
+Transforms/WholeProgramDevirt/vcp-type-mismatch.ll
+Transforms/WholeProgramDevirt/vcp-uses-this.ll
diff --git a/llvm/utils/release/build_llvm_release.bat b/llvm/utils/release/build_llvm_release.bat
index 3042fc2d77dd1..54645d0c6369f 100755
--- a/llvm/utils/release/build_llvm_release.bat
+++ b/llvm/utils/release/build_llvm_release.bat
@@ -169,9 +169,9 @@ set common_cmake_flags=^
 
 if "%force-msvc%" == "" (
   where /q clang-cl
-  if errorlevel 0 (
+  if %errorlevel% EQU 0 (
     where /q lld-link
-    if errorlevel 0 (
+    if %errorlevel% EQU 0 (
       set common_compiler_flags=%common_compiler_flags% -fuse-ld=lld
       
       set common_cmake_flags=%common_cmake_flags%^
diff --git a/llvm/utils/release/clear-release-notes.py b/llvm/utils/release/clear-release-notes.py
new file mode 100755
index 0000000000000..3ea0298b43d43
--- /dev/null
+++ b/llvm/utils/release/clear-release-notes.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+# ===-- clear-release-notes.py  ---------------------------------------------===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===------------------------------------------------------------------------===#
+#
+# Clear release notes, which is needed when bumping trunk version.
+#
+# ===------------------------------------------------------------------------===#
+
+import argparse
+from pathlib import Path
+
+
+def process_file(fpath: Path) -> None:
+    # ReleaseNotes.rst/.md -> ReleaseNotesTemplate.txt
+    template_path = fpath.with_name(f"{fpath.stem}Template.txt")
+    fpath.write_text(template_path.read_text(), newline="\n")
+    print(f"{fpath} updated.")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-s",
+        "--source-root",
+        default=None,
+        help="LLVM source root (/path/llvm-project). Defaults to the "
+        "llvm-project the script is located in.",
+    )
+
+    args = parser.parse_args()
+
+    # Find llvm-project root
+    source_root = Path(__file__).resolve().parents[3]
+
+    if args.source_root:
+        source_root = Path(args.source_root).resolve()
+
+    files_to_update = (
+        "clang/docs/ReleaseNotes.rst",
+        "clang-tools-extra/docs/ReleaseNotes.rst",
+        "flang/docs/ReleaseNotes.md",
+        "lld/docs/ReleaseNotes.rst",
+        "llvm/docs/ReleaseNotes.md",
+    )
+
+    for f in files_to_update:
+        process_file(source_root / f)
diff --git a/llvm/utils/release/github-upload-release.py b/llvm/utils/release/github-upload-release.py
index 5ed037ee9522a..2b4e57d6348e9 100755
--- a/llvm/utils/release/github-upload-release.py
+++ b/llvm/utils/release/github-upload-release.py
@@ -58,7 +58,7 @@ def create_release(repo, release, tag=None, name=None, message=None):
 * [Linux x86_64](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/LLVM-{release}-Linux-X64.tar.xz) ([signature](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/LLVM-{release}-Linux-X64.tar.xz.jsonl))
 * [Linux Arm64](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/LLVM-{release}-Linux-ARM64.tar.xz) ([signature](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/LLVM-{release}-Linux-ARM64.tar.xz.jsonl))
 AUTOMATIC_DOWNLOAD_LINKS_END -->
-<!-- * [Linux Armv7-a](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/clang+llvm-{release}-armv7a-linux-gnueabihf.tar.gz) ([signature](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}}/clang+llvm-{release}-armv7a-linux-gnueabihf.tar.gz.sig)) -->
+<!-- * [Linux Armv7-a](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/clang+llvm-{release}-armv7a-linux-gnueabihf.tar.gz) ([signature](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/clang+llvm-{release}-armv7a-linux-gnueabihf.tar.gz.sig)) -->
 
 <!-- AUTOMATIC_DOWNLOAD_LINKS_BEGIN
 * [macOS Apple Silicon](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/LLVM-{release}-macOS-ARM64.tar.xz) (ARM64) ([signature](https://github.com/llvm/llvm-project/releases/download/llvmorg-{release}/LLVM-{release}-macOS-ARM64.tar.xz.jsonl))
diff --git a/llvm/utils/revert_checker.py b/llvm/utils/revert_checker.py
index 3c3ff3e6b846f..ced68bc8475d5 100755
--- a/llvm/utils/revert_checker.py
+++ b/llvm/utils/revert_checker.py
@@ -41,6 +41,7 @@
 
 import argparse
 import collections
+import itertools
 import logging
 import re
 import subprocess
@@ -246,7 +247,11 @@ def _load_pr_commit_mappings(
 # enough for the 99% case of reverts: rarely should someone land a cleanish
 # revert of a >6 month old change...
 def find_reverts(
-    git_dir: str, across_ref: str, root: str, max_pr_lookback: int = 20000
+    git_dir: str,
+    across_ref: str,
+    root: str,
+    max_pr_lookback: int = 20000,
+    stop_at_sha: Optional[str] = None,
 ) -> List[Revert]:
     """Finds reverts across `across_ref` in `git_dir`, starting from `root`.
 
@@ -260,6 +265,9 @@ def find_reverts(
             SHAs. These heuristics require that commit history from `root` to
             `some_parent_of_root` is loaded in memory. `max_pr_lookback` is how
             many commits behind `across_ref` should be loaded in memory.
+        stop_at_sha: If non-None and `stop_at_sha` is encountered while walking
+            to `across_ref` from `root`, stop checking for reverts. This allows for
+            faster incremental checking between `find_reverts` calls.
     """
     across_sha = _rev_parse(git_dir, across_ref)
     root_sha = _rev_parse(git_dir, root)
@@ -281,10 +289,16 @@ def find_reverts(
         root_sha,
     )
 
+    commit_log_stream: Iterable[_LogEntry] = _log_stream(git_dir, root_sha, across_sha)
+    if stop_at_sha:
+        commit_log_stream = itertools.takewhile(
+            lambda x: x.sha != stop_at_sha, commit_log_stream
+        )
+
     all_reverts = []
     # Lazily load PR <-> commit mappings, since it can be expensive.
     pr_commit_mappings = None
-    for sha, commit_message in _log_stream(git_dir, root_sha, across_sha):
+    for sha, commit_message in commit_log_stream:
         reverts, pr_reverts = _try_parse_reverts_from_commit_message(
             commit_message,
         )
diff --git a/llvm/utils/revert_checker_test.py b/llvm/utils/revert_checker_test.py
index c149be8dc0dd1..b99a920402a80 100755
--- a/llvm/utils/revert_checker_test.py
+++ b/llvm/utils/revert_checker_test.py
@@ -130,6 +130,45 @@ def test_known_reverts_across_arbitrary_llvm_rev(self) -> None:
             ],
         )
 
+    def test_stop_at_sha_stops_early(self) -> None:
+        reverts = revert_checker.find_reverts(
+            git_dir=get_llvm_project_path(),
+            # This SHA is a direct child of the reverted SHA expected below.
+            across_ref="2d5f3b0a61fb171617012a2c3ba05fd31fb3bb1d",
+            # This SHA is a direct child of the revert SHA listed below.
+            root="2c01b278580212914ec037bb5dd9b73702dfe7f1",
+            max_pr_lookback=50,
+            # This SHA is the first revert that would be returned, if not for
+            # `stop_at_sha`.
+            stop_at_sha="50866e84d1da8462aeb96607bf6d9e5bbd5869c5",
+        )
+        self.assertEqual(reverts, [])
+
+    def test_stop_at_sha_still_catches_reverts_in_range(self) -> None:
+        reverts = revert_checker.find_reverts(
+            git_dir=get_llvm_project_path(),
+            # This SHA is a direct child of the reverted SHA expected below.
+            across_ref="2d5f3b0a61fb171617012a2c3ba05fd31fb3bb1d",
+            # This SHA is the direct child of the revert mentioned in
+            # `assertEqual` below.
+            root="2c01b278580212914ec037bb5dd9b73702dfe7f1",
+            max_pr_lookback=50,
+            # This SHA is the direct parent of the revert mentioned in
+            # `assertEqual` below.
+            stop_at_sha="b96ebee1fab2b281c97deb54f3d61c469fe07d01",
+        )
+        self.assertEqual(
+            reverts,
+            [
+                revert_checker.Revert(
+                    # This SHA is a `Reverts ${PR}` for #111004.
+                    sha="50866e84d1da8462aeb96607bf6d9e5bbd5869c5",
+                    # ...And this was the commit for #111004.
+                    reverted_sha="67160c5ab5f5b7fd5fa7851abcfde367c8a9f91b",
+                ),
+            ],
+        )
+
     def test_pr_based_revert_works(self) -> None:
         reverts = revert_checker.find_reverts(
             git_dir=get_llvm_project_path(),
diff --git a/llvm/utils/update_any_test_checks.py b/llvm/utils/update_any_test_checks.py
index e8eef1a46c504..76fe336593929 100755
--- a/llvm/utils/update_any_test_checks.py
+++ b/llvm/utils/update_any_test_checks.py
@@ -34,9 +34,12 @@ def find_utc_tool(search_path, utc_name):
     return None
 
 
-def run_utc_tool(utc_name, utc_tool, testname):
+def run_utc_tool(utc_name, utc_tool, testname, environment):
     result = subprocess.run(
-        [utc_tool, testname], stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        [utc_tool, testname],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=environment,
     )
     return (result.returncode, result.stdout, result.stderr)
 
@@ -60,6 +63,42 @@ def expand_listfile_args(arg_list):
     return exp_arg_list
 
 
+def utc_lit_plugin(result, test):
+    testname = test.getFilePath()
+    if not testname:
+        return None
+
+    script_name = os.path.abspath(__file__)
+    utc_search_path = os.path.join(os.path.dirname(script_name), os.path.pardir)
+
+    with open(testname, "r") as f:
+        header = f.readline().strip()
+
+    m = RE_ASSERTIONS.search(header)
+    if m is None:
+        return None
+
+    utc_name = m.group(1)
+    utc_tool = find_utc_tool([utc_search_path], utc_name)
+    if not utc_tool:
+        return f"update-utc-tests: {utc_name} not found"
+
+    return_code, stdout, stderr = run_utc_tool(
+        utc_name, utc_tool, testname, test.config.environment
+    )
+
+    stderr = stderr.decode(errors="replace")
+    if return_code != 0:
+        if stderr:
+            return f"update-utc-tests: {utc_name} exited with return code {return_code}\n{stderr.rstrip()}"
+        return f"update-utc-tests: {utc_name} exited with return code {return_code}"
+
+    stdout = stdout.decode(errors="replace")
+    if stdout:
+        return f"update-utc-tests: updated {testname}\n{stdout.rstrip()}"
+    return f"update-utc-tests: updated {testname}"
+
+
 def main():
     from argparse import RawTextHelpFormatter
 
@@ -78,6 +117,11 @@ def main():
         nargs="*",
         help="Additional directories to scan for update_*_test_checks scripts",
     )
+    parser.add_argument(
+        "--path",
+        help="""Additional directories to scan for executables invoked by the update_*_test_checks scripts,
+separated by the platform path separator""",
+    )
     parser.add_argument("tests", nargs="+")
     config = parser.parse_args()
 
@@ -88,6 +132,10 @@ def main():
     script_name = os.path.abspath(__file__)
     utc_search_path.append(os.path.join(os.path.dirname(script_name), os.path.pardir))
 
+    local_env = os.environ.copy()
+    if config.path:
+        local_env["PATH"] = config.path + os.pathsep + local_env["PATH"]
+
     not_autogenerated = []
     utc_tools = {}
     have_error = False
@@ -117,7 +165,7 @@ def main():
                         continue
 
                 future = executor.submit(
-                    run_utc_tool, utc_name, utc_tools[utc_name], testname
+                    run_utc_tool, utc_name, utc_tools[utc_name], testname, local_env
                 )
                 jobs.append((testname, future))
 
diff --git a/llvm/utils/update_llc_test_checks.py b/llvm/utils/update_llc_test_checks.py
index 07216d7dbfbb2..8c57e75f34f75 100755
--- a/llvm/utils/update_llc_test_checks.py
+++ b/llvm/utils/update_llc_test_checks.py
@@ -142,7 +142,12 @@ def update_test(ti: common.TestInfo):
             triple = common.get_triple_from_march(march_in_cmd)
 
         scrubber, function_re = output_type.get_run_handler(triple)
-        builder.process_run_line(function_re, scrubber, raw_tool_output, prefixes)
+        if 0 == builder.process_run_line(
+            function_re, scrubber, raw_tool_output, prefixes
+        ):
+            common.warn(
+                "Couldn't match any function. Possibly the wrong target triple has been provided"
+            )
         builder.processed_prefixes(prefixes)
 
     func_dict = builder.finish_and_get_func_dict()
diff --git a/mlir/CMakeLists.txt b/mlir/CMakeLists.txt
index a1ad81f625cd6..f58a4c6f506ec 100644
--- a/mlir/CMakeLists.txt
+++ b/mlir/CMakeLists.txt
@@ -137,9 +137,18 @@ else()
   set(MLIR_ENABLE_ROCM_CONVERSIONS 0)
 endif()
 
+# Build the XeVM conversions and run according tests if the SPIRV backend
+# is available.
+if ("SPIRV" IN_LIST LLVM_TARGETS_TO_BUILD)
+  set(MLIR_ENABLE_XEVM_CONVERSIONS 1)
+else()
+  set(MLIR_ENABLE_XEVM_CONVERSIONS 0)
+endif()
+
 set(MLIR_ENABLE_CUDA_RUNNER 0 CACHE BOOL "Enable building the MLIR CUDA runner")
 set(MLIR_ENABLE_ROCM_RUNNER 0 CACHE BOOL "Enable building the MLIR ROCm runner")
 set(MLIR_ENABLE_SYCL_RUNNER 0 CACHE BOOL "Enable building the MLIR SYCL runner")
+set(MLIR_ENABLE_LEVELZERO_RUNNER 0 CACHE BOOL "Enable building the MLIR LevelZero runner")
 set(MLIR_ENABLE_SPIRV_CPU_RUNNER 0 CACHE BOOL "Enable building the MLIR SPIR-V cpu runner")
 set(MLIR_ENABLE_VULKAN_RUNNER 0 CACHE BOOL "Enable building the MLIR Vulkan runner")
 set(MLIR_ENABLE_NVPTXCOMPILER 0 CACHE BOOL
diff --git a/mlir/Maintainers.md b/mlir/Maintainers.md
index c31b5c635fba3..02e93eb658279 100644
--- a/mlir/Maintainers.md
+++ b/mlir/Maintainers.md
@@ -27,6 +27,42 @@ dialects, build system and language bindings.
   [@joker-eph](https://github.com/joker-eph) (GitHub),
   mehdi_amini (Discourse)
 
+### Code
+
+#### Standalone subcategories
+* Core tooling (ODS, DRR, PDLL, LSP) (core)
+* CMake ([christopherbate](https://github.com/christopherbate))
+* Dialect Conversion ([matthias-springer](https://github.com/matthias-springer), [zero9178](https://github.com/zero9178))
+* Python Bindings ([makslevental](https://github.com/makslevental), [rolfmorel](https://github.com/rolfmorel))
+
+### Dialects
+
+#### Code Structure Dialects
+* Builtin Dialect (core)
+* ‘func’ Dialect (core)
+* ‘scf’ Dialect (core)
+* ‘cf’ Dialect (core)
+* ‘index’ Dialect (core)
+* ‘ptr’ Dialect ([fabianmcg](https://github.com/fabianmcg))
+
+#### Basic Compute Dialects
+* ‘arith’ Dialect (core)
+* ‘math’ Dialect (core)
+* Rewrite System Dialects (core)
+* Transform Dialect ([martin-luecke](https://github.com/martin-luecke), [ftynse](https://github.com/ftynse), [rolfmorel](https://github.com/rolfmorel))
+* ‘pdl_interp’ Dialect ([jpienaar](https://github.com/jpienaar))
+* ‘pdl’ Dialect ([jpienaar](https://github.com/jpienaar))
+
+#### Accessory Dialects
+* ‘affine’ Dialect ([ftynse](https://github.com/ftynse))
+* ‘dlti’ Dialect ([rolfmorel](https://github.com/rolfmorel))
+* ‘irdl’ Dialect ([math-fehr](https://github.com/math-fehr), [moxinilian](https://github.com/moxinilian))
+* ‘shape’ Dialect ([jpienaar](https://github.com/jpienaar))
+* ‘smt’ Dialect ([fabianschuiki](https://github.com/fabianschuiki), [maerhart](https://github.com/maerhart))
+* ‘ub’ Dialect ([Hardcode84](https://github.com/Hardcode84))
+* ‘complex’ Dialect (core)
+* ‘async’ Dialect (unmaintained)
+
 ## Egress
 
 MLIR components pertaining to egress flows from MLIR, in particular to LLVM IR.
@@ -44,6 +80,40 @@ MLIR components pertaining to egress flows from MLIR, in particular to LLVM IR.
   [@gysit](https://github.com/gysit) (GitHub),
   gysit (Discourse)
 
+### Dialects
+
+The `egress` maintainer refers to the people working in the Egress category,
+with the point-of-contact being the maintainers above. Named maintainers, if
+available, should be contacted first, as they're more active in those areas.
+
+#### Lowering Dialects
+* ‘llvm’ Dialect (egress)
+* ‘SPIR-V’ Dialect ([@kuhar](https://github.com/kuhar), [@antiagainst](https://github.com/antiagainst))
+* ‘emitc’ Dialect ([@aniragil](https://github.com/aniragil), [@marbre](https://github.com/marbre))
+
+#### GPU Dialects
+* ‘gpu’ Dialect ([@fabianmcg](https://github.com/fabianmcg))
+* ‘amdgpu’ Dialect ([@krzysz00](https://github.com/krzysz00))
+* ‘rocdl’ Dialect ([@krzysz00](https://github.com/krzysz00))
+* ‘nvgpu’ Dialect ([@grypp](https://github.com/grypp))
+* ‘nvvm’ Dialect ([@grypp](https://github.com/grypp))
+* ‘xegpu’ Dialect ([@chencha3](https://github.com/chencha3), [@Jianhui-Li](https://github.com/Jianhui-Li))
+* 'xevm' Dialect ([@silee2](https://github.com/silee2))
+
+#### CPU Dialects
+* ‘arm_neon’ Dialect ([@banach-space](https://github.com/banach-space))
+* ‘arm_sve’ Dialect ([@banach-space](https://github.com/banach-space))
+* ‘ArmSME’ Dialect ([@banach-space](https://github.com/banach-space))
+* ‘amx’ Dialect ([@adam-smnk](https://github.com/adam-smnk))
+* ‘x86vector’ Dialect ([@adam-smnk](https://github.com/adam-smnk))
+* ‘vcix’ Dialect ([@mshockwave](https://github.com/mshockwave))
+
+#### Paradigm Dialects
+* ‘omp’ Dialect ([@tblah](https://github.com/tblah), [@skatrak](https://github.com/skatrak))
+* ‘acc’ Dialect ([@clementval](https://github.com/clementval), [@razvanlupusoru](https://github.com/razvanlupusoru))
+* ‘mpi’ Dialect ([@fschlimb](https://github.com/fschlimb))
+* ‘shard’ Dialect ([@fschlimb](https://github.com/fschlimb))
+
 ## Tensor Compiler
 
 MLIR components specific to construction of compilers for tensor algebra, in
diff --git a/mlir/cmake/modules/AddMLIR.cmake b/mlir/cmake/modules/AddMLIR.cmake
index ff4269ed7acd2..38839679ef8b1 100644
--- a/mlir/cmake/modules/AddMLIR.cmake
+++ b/mlir/cmake/modules/AddMLIR.cmake
@@ -388,6 +388,9 @@ function(add_mlir_library name)
 
   if(TARGET ${name})
     target_link_libraries(${name} INTERFACE ${LLVM_COMMON_LIBS})
+    if(ARG_INSTALL_WITH_TOOLCHAIN)
+      set_target_properties(${name} PROPERTIES MLIR_INSTALL_WITH_TOOLCHAIN TRUE)
+    endif()
     if(NOT ARG_DISABLE_INSTALL)
       add_mlir_library_install(${name})
     endif()
@@ -617,26 +620,27 @@ endfunction(add_mlir_aggregate)
 # This is usually done as part of add_mlir_library but is broken out for cases
 # where non-standard library builds can be installed.
 function(add_mlir_library_install name)
-  if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
-  get_target_export_arg(${name} MLIR export_to_mlirtargets UMBRELLA mlir-libraries)
-  install(TARGETS ${name}
-    COMPONENT ${name}
-    ${export_to_mlirtargets}
-    LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX}
-    ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX}
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    # Note that CMake will create a directory like:
-    #   objects-${CMAKE_BUILD_TYPE}/obj.LibName
-    # and put object files there.
-    OBJECTS DESTINATION lib${LLVM_LIBDIR_SUFFIX}
-  )
+  get_target_property(_install_with_toolchain ${name} MLIR_INSTALL_WITH_TOOLCHAIN)
+  if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY OR _install_with_toolchain)
+    get_target_export_arg(${name} MLIR export_to_mlirtargets UMBRELLA mlir-libraries)
+    install(TARGETS ${name}
+      COMPONENT ${name}
+      ${export_to_mlirtargets}
+      LIBRARY DESTINATION lib${LLVM_LIBDIR_SUFFIX}
+      ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX}
+      RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
+      # Note that CMake will create a directory like:
+      #   objects-${CMAKE_BUILD_TYPE}/obj.LibName
+      # and put object files there.
+      OBJECTS DESTINATION lib${LLVM_LIBDIR_SUFFIX}
+    )
 
-  if (NOT LLVM_ENABLE_IDE)
-    add_llvm_install_targets(install-${name}
-                            DEPENDS ${name}
-                            COMPONENT ${name})
-  endif()
-  set_property(GLOBAL APPEND PROPERTY MLIR_ALL_LIBS ${name})
+    if (NOT LLVM_ENABLE_IDE)
+      add_llvm_install_targets(install-${name}
+                              DEPENDS ${name}
+                              COMPONENT ${name})
+    endif()
+    set_property(GLOBAL APPEND PROPERTY MLIR_ALL_LIBS ${name})
   endif()
   set_property(GLOBAL APPEND PROPERTY MLIR_EXPORTS ${name})
 endfunction()
diff --git a/mlir/cmake/modules/AddMLIRPython.cmake b/mlir/cmake/modules/AddMLIRPython.cmake
index c14e614ed7d92..2b883558d33c6 100644
--- a/mlir/cmake/modules/AddMLIRPython.cmake
+++ b/mlir/cmake/modules/AddMLIRPython.cmake
@@ -704,7 +704,12 @@ function(add_mlir_python_extension libname extname)
       # NanobindAdaptors.h uses PyClassMethod_New to build `pure_subclass`es but nanobind
       # doesn't declare this API as undefined in its linker flags. So we need to declare it as such
       # for downstream users that do not do something like `-undefined dynamic_lookup`.
-      target_link_options(${libname} PUBLIC "LINKER:-U,_PyClassMethod_New")
+      # Same for the rest.
+      target_link_options(${libname} PUBLIC
+        "LINKER:-U,_PyClassMethod_New"
+        "LINKER:-U,_PyCode_Addr2Location"
+        "LINKER:-U,_PyFrame_GetLasti"
+      )
     endif()
   endif()
 
diff --git a/mlir/cmake/modules/FindLevelZero.cmake b/mlir/cmake/modules/FindLevelZeroRuntime.cmake
similarity index 66%
rename from mlir/cmake/modules/FindLevelZero.cmake
rename to mlir/cmake/modules/FindLevelZeroRuntime.cmake
index 012187f0afc0b..2a8fb3a16d16f 100644
--- a/mlir/cmake/modules/FindLevelZero.cmake
+++ b/mlir/cmake/modules/FindLevelZeroRuntime.cmake
@@ -20,7 +20,6 @@ include(FindPackageHandleStandardArgs)
 # Search path priority
 # 1. CMake Variable LEVEL_ZERO_DIR
 # 2. Environment Variable LEVEL_ZERO_DIR
-
 if(NOT LEVEL_ZERO_DIR)
     if(DEFINED ENV{LEVEL_ZERO_DIR})
         set(LEVEL_ZERO_DIR "$ENV{LEVEL_ZERO_DIR}")
@@ -28,32 +27,32 @@ if(NOT LEVEL_ZERO_DIR)
 endif()
 
 if(LEVEL_ZERO_DIR)
-    find_path(LevelZero_INCLUDE_DIR
+    find_path(LevelZeroRuntime_INCLUDE_DIR
         NAMES level_zero/ze_api.h
         PATHS ${LEVEL_ZERO_DIR}/include
         NO_DEFAULT_PATH
     )
 
     if(LINUX)
-        find_library(LevelZero_LIBRARY
+        find_library(LevelZeroRuntime_LIBRARY
             NAMES ze_loader
             PATHS ${LEVEL_ZERO_DIR}/lib
-                  ${LEVEL_ZERO_DIR}/lib/x86_64-linux-gnu
+            ${LEVEL_ZERO_DIR}/lib/x86_64-linux-gnu
             NO_DEFAULT_PATH
         )
     else()
-        find_library(LevelZero_LIBRARY
+        find_library(LevelZeroRuntime_LIBRARY
             NAMES ze_loader
             PATHS ${LEVEL_ZERO_DIR}/lib
             NO_DEFAULT_PATH
         )
     endif()
 else()
-    find_path(LevelZero_INCLUDE_DIR
+    find_path(LevelZeroRuntime_INCLUDE_DIR
         NAMES level_zero/ze_api.h
     )
 
-    find_library(LevelZero_LIBRARY
+    find_library(LevelZeroRuntime_LIBRARY
         NAMES ze_loader
     )
 endif()
@@ -64,12 +63,14 @@ endif()
 # lists of equal lengths, with the shorter string getting zero-padded.
 function(compare_versions VERSION_STR1 VERSION_STR2 OUTPUT)
     # Convert the strings to list
-    string(REPLACE  "." ";" VL1 ${VERSION_STR1})
-    string(REPLACE  "." ";" VL2 ${VERSION_STR2})
+    string(REPLACE "." ";" VL1 ${VERSION_STR1})
+    string(REPLACE "." ";" VL2 ${VERSION_STR2})
+
     # get lengths of both lists
     list(LENGTH VL1 VL1_LEN)
     list(LENGTH VL2 VL2_LEN)
     set(LEN ${VL1_LEN})
+
     # If they differ in size pad the shorter list with 0s
     if(VL1_LEN GREATER VL2_LEN)
         math(EXPR DIFF "${VL1_LEN} - ${VL2_LEN}" OUTPUT_FORMAT DECIMAL)
@@ -98,12 +99,10 @@ function(compare_versions VERSION_STR1 VERSION_STR2 OUTPUT)
             set(${OUTPUT} TRUE PARENT_SCOPE)
         endif()
     endforeach()
-
-    endfunction(compare_versions)
+endfunction(compare_versions)
 
 # Creates a small function to run and extract the LevelZero loader version.
 function(get_l0_loader_version)
-
     set(L0_VERSIONEER_SRC
         [====[
         #include <iostream>
@@ -142,19 +141,20 @@ function(get_l0_loader_version)
 
     # We need both the directories in the include path as ze_loader.h
     # includes "ze_api.h" and not "level_zero/ze_api.h".
-    list(APPEND INCLUDE_DIRS ${LevelZero_INCLUDE_DIR})
-    list(APPEND INCLUDE_DIRS ${LevelZero_INCLUDE_DIR}/level_zero)
+    list(APPEND INCLUDE_DIRS ${LevelZeroRuntime_INCLUDE_DIR})
+    list(APPEND INCLUDE_DIRS ${LevelZeroRuntime_INCLUDE_DIR}/level_zero)
     list(JOIN INCLUDE_DIRS ";" INCLUDE_DIRS_STR)
     try_run(L0_VERSIONEER_RUN L0_VERSIONEER_COMPILE
-            "${CMAKE_BINARY_DIR}"
-            "${L0_VERSIONEER_FILE}"
-            LINK_LIBRARIES ${LevelZero_LIBRARY}
-            CMAKE_FLAGS
-                "-DINCLUDE_DIRECTORIES=${INCLUDE_DIRS_STR}"
-            RUN_OUTPUT_VARIABLE L0_VERSION
+        "${CMAKE_BINARY_DIR}"
+        "${L0_VERSIONEER_FILE}"
+        LINK_LIBRARIES ${LevelZeroRuntime_LIBRARY}
+        CMAKE_FLAGS
+        "-DINCLUDE_DIRECTORIES=${INCLUDE_DIRS_STR}"
+        RUN_OUTPUT_VARIABLE L0_VERSION
     )
-    if(${L0_VERSIONEER_COMPILE} AND (DEFINED L0_VERSIONEER_RUN))
-        set(LevelZero_VERSION ${L0_VERSION} PARENT_SCOPE)
+
+    if(${L0_VERSIONEER_COMPILE} AND(DEFINED L0_VERSIONEER_RUN))
+        set(LevelZeroRuntime_VERSION ${L0_VERSION} PARENT_SCOPE)
         message(STATUS "Found Level Zero of version: ${L0_VERSION}")
     else()
         message(FATAL_ERROR
@@ -163,59 +163,61 @@ function(get_l0_loader_version)
     endif()
 endfunction(get_l0_loader_version)
 
-if(LevelZero_INCLUDE_DIR AND LevelZero_LIBRARY)
-    list(APPEND LevelZero_LIBRARIES "${LevelZero_LIBRARY}")
-    list(APPEND LevelZero_INCLUDE_DIRS ${LevelZero_INCLUDE_DIR})
+if(LevelZeroRuntime_INCLUDE_DIR AND LevelZeroRuntime_LIBRARY)
+    list(APPEND LevelZeroRuntime_LIBRARIES "${LevelZeroRuntime_LIBRARY}")
+    list(APPEND LevelZeroRuntime_INCLUDE_DIRS ${LevelZeroRuntime_INCLUDE_DIR})
+
     if(OpenCL_FOUND)
-      list(APPEND LevelZero_INCLUDE_DIRS ${OpenCL_INCLUDE_DIRS})
+        list(APPEND LevelZeroRuntime_INCLUDE_DIRS ${OpenCL_INCLUDE_DIRS})
     endif()
 
-    cmake_path(GET LevelZero_LIBRARY PARENT_PATH LevelZero_LIBRARIES_PATH)
-    set(LevelZero_LIBRARIES_DIR ${LevelZero_LIBRARIES_PATH})
-
-    if(NOT TARGET LevelZero::LevelZero)
-      add_library(LevelZero::LevelZero INTERFACE IMPORTED)
-      set_target_properties(LevelZero::LevelZero
-        PROPERTIES INTERFACE_LINK_LIBRARIES "${LevelZero_LIBRARIES}"
-      )
-      set_target_properties(LevelZero::LevelZero
-        PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LevelZero_INCLUDE_DIRS}"
-      )
+    cmake_path(GET LevelZeroRuntime_LIBRARY PARENT_PATH LevelZeroRuntime_LIBRARIES_PATH)
+    set(LevelZeroRuntime_LIBRARIES_DIR ${LevelZeroRuntime_LIBRARIES_PATH})
+
+    if(NOT TARGET LevelZeroRuntime::LevelZeroRuntime)
+        add_library(LevelZeroRuntime::LevelZeroRuntime INTERFACE IMPORTED)
+        set_target_properties(LevelZeroRuntime::LevelZeroRuntime
+            PROPERTIES INTERFACE_LINK_LIBRARIES "${LevelZeroRuntime_LIBRARIES}"
+        )
+        set_target_properties(LevelZeroRuntime::LevelZeroRuntime
+            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${LevelZeroRuntime_INCLUDE_DIRS}"
+        )
     endif()
 endif()
 
 # Check if a specific version of Level Zero is required
-if(LevelZero_FIND_VERSION)
+if(LevelZeroRuntime_FIND_VERSION)
     get_l0_loader_version()
     set(VERSION_GT_FIND_VERSION FALSE)
     compare_versions(
-        ${LevelZero_VERSION}
-        ${LevelZero_FIND_VERSION}
+        ${LevelZeroRuntime_VERSION}
+        ${LevelZeroRuntime_FIND_VERSION}
         VERSION_GT_FIND_VERSION
     )
+
     if(${VERSION_GT_FIND_VERSION})
-        set(LevelZero_FOUND TRUE)
+        set(LevelZeroRuntime_FOUND TRUE)
     else()
-        set(LevelZero_FOUND FALSE)
+        set(LevelZeroRuntime_FOUND FALSE)
     endif()
 else()
-    set(LevelZero_FOUND TRUE)
+    set(LevelZeroRuntime_FOUND TRUE)
 endif()
 
-find_package_handle_standard_args(LevelZero
+find_package_handle_standard_args(LevelZeroRuntime
     REQUIRED_VARS
-        LevelZero_FOUND
-        LevelZero_INCLUDE_DIRS
-        LevelZero_LIBRARY
-        LevelZero_LIBRARIES_DIR
+    LevelZeroRuntime_FOUND
+    LevelZeroRuntime_INCLUDE_DIRS
+    LevelZeroRuntime_LIBRARY
+    LevelZeroRuntime_LIBRARIES_DIR
     HANDLE_COMPONENTS
 )
-mark_as_advanced(LevelZero_LIBRARY LevelZero_INCLUDE_DIRS)
+mark_as_advanced(LevelZeroRuntime_LIBRARY LevelZeroRuntime_INCLUDE_DIRS)
 
-if(LevelZero_FOUND)
-    find_package_message(LevelZero "Found LevelZero: ${LevelZero_LIBRARY}"
-        "(found version ${LevelZero_VERSION})"
+if(LevelZeroRuntime_FOUND)
+    find_package_message(LevelZeroRuntime "Found LevelZero: ${LevelZeroRuntime_LIBRARY}"
+        "(found version ${LevelZeroRuntime_VERSION})"
     )
 else()
-    find_package_message(LevelZero "Could not find LevelZero" "")
+    find_package_message(LevelZeroRuntime "Could not find LevelZero" "")
 endif()
diff --git a/mlir/cmake/modules/FindSyclRuntime.cmake b/mlir/cmake/modules/FindSyclRuntime.cmake
index 9e6ae04025848..5986895cb1628 100644
--- a/mlir/cmake/modules/FindSyclRuntime.cmake
+++ b/mlir/cmake/modules/FindSyclRuntime.cmake
@@ -19,7 +19,7 @@ if(NOT DEFINED ENV{CMPLR_ROOT})
 else()
     get_filename_component(ONEAPI_VER "$ENV{CMPLR_ROOT}" NAME)
     if(ONEAPI_VER VERSION_LESS 2024.0)
-        if(LINUX OR (${CMAKE_SYSTEM_NAME} MATCHES "Linux"))
+        if(LINUX OR ("${CMAKE_SYSTEM_NAME}" MATCHES "Linux"))
             set(SyclRuntime_ROOT "$ENV{CMPLR_ROOT}/linux")
         elseif(WIN32)
             set(SyclRuntime_ROOT "$ENV{CMPLR_ROOT}/windows")
diff --git a/mlir/docs/DialectConversion.md b/mlir/docs/DialectConversion.md
index cf577eca5b9a6..556e73c2d56c7 100644
--- a/mlir/docs/DialectConversion.md
+++ b/mlir/docs/DialectConversion.md
@@ -202,17 +202,62 @@ struct MyConversionPattern : public ConversionPattern {
 
 #### Type Safety
 
-The types of the remapped operands provided to a conversion pattern must be of a
-type expected by the pattern. The expected types of a pattern are determined by
-a provided [TypeConverter](#type-converter). If no type converter is provided,
-the types of the remapped operands are expected to match the types of the
-original operands. If a type converter is provided, the types of the remapped
-operands are expected to be legal as determined by the converter. If the
-remapped operand types are not of an expected type, and a materialization to the
-expected type could not be performed, the pattern fails application before the
-`matchAndRewrite` hook is invoked. This ensures that patterns do not have to
-explicitly ensure type safety, or sanitize the types of the incoming remapped
-operands. More information on type conversion is detailed in the
+The types of the remapped operands provided to a conversion pattern (through
+the adaptor or `ArrayRef` of operands) depend on type conversion rules.
+
+If the pattern was initialized with a [type converter](#type-converter), the
+conversion driver passes values whose types match the legalized types of the
+operands of the matched operation as per the type converter. To that end, the
+conversion driver may insert target materializations to convert the most
+recently mapped values to the expected legalized types. The driver tries to
+reuse existing materializations on a best-effort basis, but this is not
+guaranteed by the infrastructure. If the operand types of the matched op could
+not be legalized, the pattern fails to apply before the `matchAndRewrite` hook
+is invoked.
+
+Example:
+```c++
+// Type converter that converts all FloatTypes to IntegerTypes.
+TypeConverter converter;
+converter.addConversion([](FloatType t) {
+    return IntegerType::get(t.getContext(), t.getWidth());
+});
+
+// Assuming that `MyConversionPattern` was initialized with `converter`.
+struct MyConversionPattern : public ConversionPattern {
+  virtual LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands, /* ... */) const {
+//                                               ^^^^^^^^
+//      If `op` has a FloatType operand, the respective value in `operands`
+//      is guaranteed to have the legalized IntegerType. If another pattern
+//      previously replaced the operand SSA value with an SSA value of the
+//      legalized type (via "replaceOp" or "applySignatureConversion"), you
+//      will get that SSA value directly (unless the replacement value was
+//      also replaced). Otherwise, you will get a materialization to the
+//      legalized type.
+```
+
+If the pattern was initialized without a type converter, the conversion driver
+passes the most recently mapped values to the pattern, excluding any
+materializations. If a value with the same type as the original operand is
+desired, users can directly take the respective operand from the matched
+operation.
+
+Example: When initializing the pattern from the above example without a type
+converter, `operands` contains the most recent replacement values, regardless
+of their types.
+
+Note: When running without a type converter, materializations are intentionally
+excluded from the lookup process because their presence may depend on other
+patterns. Passing materializations would make the conversion infrastructure
+fragile and unpredictable. Moreover, there could be multiple materializations
+to different types. (This can be the case when multiple patterns are running
+with different type converters.) In such a case, it would be unclear which
+materialization to pass.
+
+The above rules ensure that patterns do not have to explicitly ensure type
+safety, or sanitize the types of the incoming remapped operands. More
+information on type conversion is detailed in the
 [dedicated section](#type-conversion) below.
 
 ## Type Conversion
diff --git a/mlir/docs/Dialects/GPU.md b/mlir/docs/Dialects/GPU.md
index 94b053daa1615..8d4d2ca3e5743 100644
--- a/mlir/docs/Dialects/GPU.md
+++ b/mlir/docs/Dialects/GPU.md
@@ -193,10 +193,25 @@ llvm.func @foo() {
 // mlir-translate --mlir-to-llvmir:
 @binary_bin_cst = internal constant [6 x i8] c"AMDGPU", align 8
 @binary_func_kernel_name = private unnamed_addr constant [7 x i8] c"func\00", align 1
+@binary_module = internal global ptr null
+@llvm.global_ctors = appending global [1 x {i32, ptr, ptr}] [{i32 123, ptr @binary_load, ptr null}]
+@llvm.global_dtors = appending global [1 x {i32, ptr, ptr}] [{i32 123, ptr @binary_unload, ptr null}]
+define internal void @binary_load() section ".text.startup" {
+entry:
+  %0 = call ptr @mgpuModuleLoad(ptr @binary_bin_cst)
+  store ptr %0, ptr @binary_module
+  ...
+}
+define internal void @binary_unload() section ".text.startup" {
+entry:
+  %0 = load ptr, ptr @binary_module, align 8
+  call void @mgpuModuleUnload(ptr %0)
+  ...
+}
 ...
 define void @foo() {
   ...
-  %module = call ptr @mgpuModuleLoad(ptr @binary_bin_cst)
+  %module = load ptr, ptr @binary_module, align 8
   %kernel = call ptr @mgpuModuleGetFunction(ptr %module, ptr @binary_func_kernel_name)
   call void @mgpuLaunchKernel(ptr %kernel, ...) ; Launch the kernel
   ...
diff --git a/mlir/docs/Remarks.md b/mlir/docs/Remarks.md
new file mode 100644
index 0000000000000..ee2ea855b8a6a
--- /dev/null
+++ b/mlir/docs/Remarks.md
@@ -0,0 +1,259 @@
+# Remark Infrastructure
+
+Remarks are **structured, human- and machine-readable notes** emitted by the
+compiler to explain:
+
+- What was transformed
+- What was missed
+- Why it happened
+
+The **`RemarkEngine`** collects finalized remarks during compilation and sends
+them to a pluggable **streamer**. By default, MLIR integrates with LLVM’s
+[`llvm::remarks`](https://llvm.org/docs/Remarks.html), allowing you to:
+
+- Stream remarks as passes run
+- Serialize them to **YAML** or **LLVM bitstream** for tooling
+
+***
+
+## Key Points
+
+- **Opt-in** – Disabled by default; zero overhead unless enabled.
+- **Per-context** – Configured on `MLIRContext`.
+- **Formats** – LLVM Remark engine (YAML / Bitstream) or custom streamers.
+- **Kinds** – `Passed`, `Missed`, `Failure`, `Analysis`.
+- **API** – Lightweight streaming interface using `<<` (like MLIR diagnostics).
+
+***
+
+## How It Works
+
+Two main components:
+
+- **`RemarkEngine`** (owned by `MLIRContext`): Receives finalized
+  `InFlightRemark`s, optionally mirrors them to the `DiagnosticEngine`, and
+  dispatches to the installed streamer.
+
+- **`MLIRRemarkStreamerBase`** (abstract): Backend interface with a single hook:
+
+  ```c++
+  virtual void streamOptimizationRemark(const Remark &remark) = 0;
+  ```
+
+**Default backend – `MLIRLLVMRemarkStreamer`** Adapts `mlir::Remark` to LLVM’s
+remark format and writes YAML/bitstream via `llvm::remarks::RemarkStreamer`.
+
+**Ownership flow:** `MLIRContext` → `RemarkEngine` → `MLIRRemarkStreamerBase`
+
+***
+
+## Categories
+
+MLIR provides four built-in remark categories (extendable if needed):
+
+#### 1. **Passed**
+
+Optimization/transformation succeeded.
+
+```
+[Passed] RemarkName | Category:Vectorizer:myPass1 | Function=foo | Remark="vectorized loop", tripCount=128
+```
+
+#### 2. **Missed**
+
+Optimization/transformation didn’t apply — ideally with actionable feedback.
+
+```
+[Missed]  | Category:Unroll | Function=foo | Reason="tripCount=4 < threshold=256", Suggestion="increase unroll to 128"
+```
+
+#### 3. **Failure**
+
+Optimization/transformation attempted but failed. This is slightly different
+from the `Missed` category.
+
+For example, the user specifies `-use-max-register=100` when invoking the
+compiler, but the attempt fails for some reason:
+
+```bash
+$ your-compiler -use-max-register=100 mycode.xyz
+```
+
+```
+[Failed] Category:RegisterAllocator | Reason="Limiting to use-max-register=100 failed; it now uses 104 registers for better performance"
+```
+
+#### 4. **Analysis**
+
+Neutral analysis results.
+
+```
+[Analysis] Category:Register | Remark="Kernel uses 168 registers"
+[Analysis] Category:Register | Remark="Kernel uses 10kB local memory"
+```
+
+***
+
+## Emitting Remarks
+
+The `remark::*` helpers return an **in-flight remark**.
+You append strings or key–value metrics using `<<`.
+
+### Remark Options
+
+When constructing a remark, you typically provide four fields that are `StringRef`:
+
+1. **Remark name** – identifiable name
+2. **Category** – high-level classification
+3. **Sub-category** – more fine-grained classification
+4. **Function name** – the function where the remark originates
+
+
+### Example
+
+```c++
+#include "mlir/IR/Remarks.h"
+
+LogicalResult MyPass::runOnOperation() {
+  Location loc = getOperation()->getLoc();
+
+  remark::RemarkOpts opts = remark::RemarkOpts::name(MyRemarkName1)
+                                .category(categoryVectorizer)
+                                .function(fName)
+                                .subCategory(myPassname1);
+
+  // PASSED
+  remark::passed(loc, opts)
+      << "vectorized loop"
+      << remark::metric("tripCount", 128);
+
+  // ANALYSIS
+  remark::analysis(loc, opts)
+      << "Kernel uses 168 registers";
+
+  // MISSED (with reason + suggestion)
+  int tripBad = 4, threshold = 256, target = 128;
+  remark::missed(loc, opts)
+      << remark::reason("tripCount={0} < threshold={1}", tripBad, threshold)
+      << remark::suggest("increase unroll to {0}", target);
+
+  // FAILURE
+  remark::failed(loc, opts)
+      << remark::reason("failed due to unsupported pattern");
+
+  return success();
+}
+```
+
+***
+
+### Metrics and Shortcuts
+
+Helper functions accept
+[LLVM format](https://llvm.org/docs/ProgrammersManual.html#formatting-strings-the-formatv-function)
+style strings. This format builds lazily, so remarks are zero-cost when
+disabled.
+
+#### Adding Remarks
+
+- **`remark::add(fmt, ...)`** – Shortcut for `metric("Remark", ...)`.
+
+#### Adding Reasons
+
+- **`remark::reason(fmt, ...)`** – Shortcut for `metric("Reason", ...)`. Used to
+  explain why a remark was missed or failed.
+
+#### Adding Suggestions
+
+- **`remark::suggest(fmt, ...)`** – Shortcut for `metric("Suggestion", ...)`.
+  Used to provide actionable feedback.
+
+#### Adding Custom Metrics
+
+- **`remark::metric(key, value)`** – Adds a structured key–value metric.
+
+Example: tracking `TripCount`. When exported to YAML, it appears under `args`
+for machine readability:
+
+```cpp
+remark::metric("TripCount", value)
+```
+
+#### String Metrics
+
+Passing a plain string (e.g. `<< "vectorized loop"`) is equivalent to:
+
+```cpp
+metric("Remark", "vectorized loop")
+```
+
+***
+
+## Enabling Remarks
+
+### 1. **With LLVMRemarkStreamer (YAML or Bitstream)**
+
+Persists remarks to a file in the chosen format.
+
+```c++
+mlir::remark::RemarkCategories cats{/*passed=*/categoryLoopunroll,
+                                     /*missed=*/std::nullopt,
+                                     /*analysis=*/std::nullopt,
+                                     /*failed=*/categoryLoopunroll};
+
+mlir::remark::enableOptimizationRemarksWithLLVMStreamer(
+    context, yamlFile, llvm::remarks::Format::YAML, cats);
+```
+
+**YAML format** – human-readable, easy to diff:
+
+```yaml
+--- !Passed
+pass:            Category:SubCategory
+name:            MyRemarkName1
+function:        myFunc
+loc:             myfile.mlir:12:3
+args:
+  - Remark:          vectorized loop
+  - tripCount:       128
+```
+
+**Bitstream format** – compact binary for large runs.
+
+***
+
+### 2. **With `mlir::emitRemarks` (No Streamer)**
+
+If the streamer isn't passed, the remarks are mirrored to the `DiagnosticEngine`
+using `mlir::emitRemarks`
+
+```c++
+mlir::remark::RemarkCategories cats{/*passed=*/categoryLoopunroll,
+                                     /*missed=*/std::nullopt,
+                                     /*analysis=*/std::nullopt,
+                                     /*failed=*/categoryLoopunroll};
+remark::enableOptimizationRemarks(
+    /*streamer=*/nullptr, cats,
+    /*printAsEmitRemarks=*/true);
+```
+
+***
+
+### 3. **With a Custom Streamer**
+
+You can implement a custom streamer by inheriting `MLIRRemarkStreamerBase` to
+consume remarks in any format.
+
+```c++
+class MyStreamer : public MLIRRemarkStreamerBase {
+public:
+  void streamOptimizationRemark(const Remark &remark) override {
+    // Convert and write remark to your custom format
+  }
+};
+
+auto myStreamer = std::make_unique<MyStreamer>();
+remark::enableOptimizationRemarks(
+    /*streamer=*/myStreamer, cats,
+    /*printAsEmitRemarks=*/true);
+```
diff --git a/mlir/docs/Tutorials/Toy/Ch-4.md b/mlir/docs/Tutorials/Toy/Ch-4.md
index e9abe36afc4d7..621f6a684285a 100644
--- a/mlir/docs/Tutorials/Toy/Ch-4.md
+++ b/mlir/docs/Tutorials/Toy/Ch-4.md
@@ -170,7 +170,7 @@ let arguments = (ins
   OptionalAttr<DictArrayAttr>:$arg_attrs,
   OptionalAttr<DictArrayAttr>:$res_attrs
 );
-
+```
 
 We have already provided the definition in the `extraClassDeclaration`
 field of the `FuncOp` class:
diff --git a/mlir/examples/standalone/CMakeLists.txt b/mlir/examples/standalone/CMakeLists.txt
index 038242ba1437a..42b487fe2d40f 100644
--- a/mlir/examples/standalone/CMakeLists.txt
+++ b/mlir/examples/standalone/CMakeLists.txt
@@ -8,6 +8,10 @@ set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to")
 if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
   find_package(MLIR REQUIRED CONFIG)
 
+
+  # Define the default argument to use by `lit` when testing.
+  set(LLVM_LIT_ARGS "-sv" CACHE STRING "Default options for lit")
+
   message(STATUS "Using MLIRConfig.cmake in: ${MLIR_DIR}")
   message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}")
 
diff --git a/mlir/examples/toy/Ch1/parser/AST.cpp b/mlir/examples/toy/Ch1/parser/AST.cpp
index 2546f2a9725d6..841642488db68 100644
--- a/mlir/examples/toy/Ch1/parser/AST.cpp
+++ b/mlir/examples/toy/Ch1/parser/AST.cpp
@@ -120,7 +120,7 @@ void ASTDumper::dump(NumberExprAST *num) {
 ///    [ [ 1, 2 ], [ 3, 4 ] ]
 /// We print out such array with the dimensions spelled out at every level:
 ///    <2,2>[<2>[ 1, 2 ], <2>[ 3, 4 ] ]
-void printLitHelper(ExprAST *litOrNum) {
+static void printLitHelper(ExprAST *litOrNum) {
   // Inside a literal expression we can have either a number or another literal
   if (auto *num = llvm::dyn_cast<NumberExprAST>(litOrNum)) {
     llvm::errs() << num->getValue();
diff --git a/mlir/examples/toy/Ch1/toyc.cpp b/mlir/examples/toy/Ch1/toyc.cpp
index fb7b484a92fb9..b9f3a2d3ae11f 100644
--- a/mlir/examples/toy/Ch1/toyc.cpp
+++ b/mlir/examples/toy/Ch1/toyc.cpp
@@ -39,7 +39,8 @@ static cl::opt<enum Action>
                cl::values(clEnumValN(DumpAST, "ast", "output the AST dump")));
 
 /// Returns a Toy AST resulting from parsing the file or a nullptr on error.
-std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
+static std::unique_ptr<toy::ModuleAST>
+parseInputFile(llvm::StringRef filename) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileOrErr =
       llvm::MemoryBuffer::getFileOrSTDIN(filename);
   if (std::error_code ec = fileOrErr.getError()) {
diff --git a/mlir/examples/toy/Ch2/parser/AST.cpp b/mlir/examples/toy/Ch2/parser/AST.cpp
index 2546f2a9725d6..841642488db68 100644
--- a/mlir/examples/toy/Ch2/parser/AST.cpp
+++ b/mlir/examples/toy/Ch2/parser/AST.cpp
@@ -120,7 +120,7 @@ void ASTDumper::dump(NumberExprAST *num) {
 ///    [ [ 1, 2 ], [ 3, 4 ] ]
 /// We print out such array with the dimensions spelled out at every level:
 ///    <2,2>[<2>[ 1, 2 ], <2>[ 3, 4 ] ]
-void printLitHelper(ExprAST *litOrNum) {
+static void printLitHelper(ExprAST *litOrNum) {
   // Inside a literal expression we can have either a number or another literal
   if (auto *num = llvm::dyn_cast<NumberExprAST>(litOrNum)) {
     llvm::errs() << num->getValue();
diff --git a/mlir/examples/toy/Ch2/toyc.cpp b/mlir/examples/toy/Ch2/toyc.cpp
index e33b49b41c5a1..a60738da9d4ae 100644
--- a/mlir/examples/toy/Ch2/toyc.cpp
+++ b/mlir/examples/toy/Ch2/toyc.cpp
@@ -58,7 +58,8 @@ static cl::opt<enum Action> emitAction(
     cl::values(clEnumValN(DumpMLIR, "mlir", "output the MLIR dump")));
 
 /// Returns a Toy AST resulting from parsing the file or a nullptr on error.
-std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
+static std::unique_ptr<toy::ModuleAST>
+parseInputFile(llvm::StringRef filename) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileOrErr =
       llvm::MemoryBuffer::getFileOrSTDIN(filename);
   if (std::error_code ec = fileOrErr.getError()) {
@@ -71,7 +72,7 @@ std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
   return parser.parseModule();
 }
 
-int dumpMLIR() {
+static int dumpMLIR() {
   mlir::MLIRContext context;
   // Load our Dialect in this MLIR Context.
   context.getOrLoadDialect<mlir::toy::ToyDialect>();
@@ -112,7 +113,7 @@ int dumpMLIR() {
   return 0;
 }
 
-int dumpAST() {
+static int dumpAST() {
   if (inputType == InputType::MLIR) {
     llvm::errs() << "Can't dump a Toy AST when the input is MLIR\n";
     return 5;
diff --git a/mlir/examples/toy/Ch3/parser/AST.cpp b/mlir/examples/toy/Ch3/parser/AST.cpp
index 2546f2a9725d6..841642488db68 100644
--- a/mlir/examples/toy/Ch3/parser/AST.cpp
+++ b/mlir/examples/toy/Ch3/parser/AST.cpp
@@ -120,7 +120,7 @@ void ASTDumper::dump(NumberExprAST *num) {
 ///    [ [ 1, 2 ], [ 3, 4 ] ]
 /// We print out such array with the dimensions spelled out at every level:
 ///    <2,2>[<2>[ 1, 2 ], <2>[ 3, 4 ] ]
-void printLitHelper(ExprAST *litOrNum) {
+static void printLitHelper(ExprAST *litOrNum) {
   // Inside a literal expression we can have either a number or another literal
   if (auto *num = llvm::dyn_cast<NumberExprAST>(litOrNum)) {
     llvm::errs() << num->getValue();
diff --git a/mlir/examples/toy/Ch3/toyc.cpp b/mlir/examples/toy/Ch3/toyc.cpp
index f8aa846582267..3094935567d94 100644
--- a/mlir/examples/toy/Ch3/toyc.cpp
+++ b/mlir/examples/toy/Ch3/toyc.cpp
@@ -64,7 +64,8 @@ static cl::opt<enum Action> emitAction(
 static cl::opt<bool> enableOpt("opt", cl::desc("Enable optimizations"));
 
 /// Returns a Toy AST resulting from parsing the file or a nullptr on error.
-std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
+static std::unique_ptr<toy::ModuleAST>
+parseInputFile(llvm::StringRef filename) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileOrErr =
       llvm::MemoryBuffer::getFileOrSTDIN(filename);
   if (std::error_code ec = fileOrErr.getError()) {
@@ -77,8 +78,8 @@ std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
   return parser.parseModule();
 }
 
-int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context,
-             mlir::OwningOpRef<mlir::ModuleOp> &module) {
+static int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context,
+                    mlir::OwningOpRef<mlir::ModuleOp> &module) {
   // Handle '.toy' input to the compiler.
   if (inputType != InputType::MLIR &&
       !llvm::StringRef(inputFilename).ends_with(".mlir")) {
@@ -107,7 +108,7 @@ int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context,
   return 0;
 }
 
-int dumpMLIR() {
+static int dumpMLIR() {
   mlir::MLIRContext context;
   // Load our Dialect in this MLIR Context.
   context.getOrLoadDialect<mlir::toy::ToyDialect>();
@@ -134,7 +135,7 @@ int dumpMLIR() {
   return 0;
 }
 
-int dumpAST() {
+static int dumpAST() {
   if (inputType == InputType::MLIR) {
     llvm::errs() << "Can't dump a Toy AST when the input is MLIR\n";
     return 5;
diff --git a/mlir/examples/toy/Ch4/parser/AST.cpp b/mlir/examples/toy/Ch4/parser/AST.cpp
index 2546f2a9725d6..841642488db68 100644
--- a/mlir/examples/toy/Ch4/parser/AST.cpp
+++ b/mlir/examples/toy/Ch4/parser/AST.cpp
@@ -120,7 +120,7 @@ void ASTDumper::dump(NumberExprAST *num) {
 ///    [ [ 1, 2 ], [ 3, 4 ] ]
 /// We print out such array with the dimensions spelled out at every level:
 ///    <2,2>[<2>[ 1, 2 ], <2>[ 3, 4 ] ]
-void printLitHelper(ExprAST *litOrNum) {
+static void printLitHelper(ExprAST *litOrNum) {
   // Inside a literal expression we can have either a number or another literal
   if (auto *num = llvm::dyn_cast<NumberExprAST>(litOrNum)) {
     llvm::errs() << num->getValue();
diff --git a/mlir/examples/toy/Ch4/toyc.cpp b/mlir/examples/toy/Ch4/toyc.cpp
index ae02bc4314bab..36816f0327dee 100644
--- a/mlir/examples/toy/Ch4/toyc.cpp
+++ b/mlir/examples/toy/Ch4/toyc.cpp
@@ -65,7 +65,8 @@ static cl::opt<enum Action> emitAction(
 static cl::opt<bool> enableOpt("opt", cl::desc("Enable optimizations"));
 
 /// Returns a Toy AST resulting from parsing the file or a nullptr on error.
-std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
+static std::unique_ptr<toy::ModuleAST>
+parseInputFile(llvm::StringRef filename) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileOrErr =
       llvm::MemoryBuffer::getFileOrSTDIN(filename);
   if (std::error_code ec = fileOrErr.getError()) {
@@ -78,8 +79,8 @@ std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
   return parser.parseModule();
 }
 
-int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context,
-             mlir::OwningOpRef<mlir::ModuleOp> &module) {
+static int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context,
+                    mlir::OwningOpRef<mlir::ModuleOp> &module) {
   // Handle '.toy' input to the compiler.
   if (inputType != InputType::MLIR &&
       !llvm::StringRef(inputFilename).ends_with(".mlir")) {
@@ -108,7 +109,7 @@ int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context,
   return 0;
 }
 
-int dumpMLIR() {
+static int dumpMLIR() {
   mlir::MLIRContext context;
   // Load our Dialect in this MLIR Context.
   context.getOrLoadDialect<mlir::toy::ToyDialect>();
@@ -143,7 +144,7 @@ int dumpMLIR() {
   return 0;
 }
 
-int dumpAST() {
+static int dumpAST() {
   if (inputType == InputType::MLIR) {
     llvm::errs() << "Can't dump a Toy AST when the input is MLIR\n";
     return 5;
diff --git a/mlir/examples/toy/Ch5/parser/AST.cpp b/mlir/examples/toy/Ch5/parser/AST.cpp
index 2546f2a9725d6..841642488db68 100644
--- a/mlir/examples/toy/Ch5/parser/AST.cpp
+++ b/mlir/examples/toy/Ch5/parser/AST.cpp
@@ -120,7 +120,7 @@ void ASTDumper::dump(NumberExprAST *num) {
 ///    [ [ 1, 2 ], [ 3, 4 ] ]
 /// We print out such array with the dimensions spelled out at every level:
 ///    <2,2>[<2>[ 1, 2 ], <2>[ 3, 4 ] ]
-void printLitHelper(ExprAST *litOrNum) {
+static void printLitHelper(ExprAST *litOrNum) {
   // Inside a literal expression we can have either a number or another literal
   if (auto *num = llvm::dyn_cast<NumberExprAST>(litOrNum)) {
     llvm::errs() << num->getValue();
diff --git a/mlir/examples/toy/Ch5/toyc.cpp b/mlir/examples/toy/Ch5/toyc.cpp
index afdf782d8ea47..3760a88d6f698 100644
--- a/mlir/examples/toy/Ch5/toyc.cpp
+++ b/mlir/examples/toy/Ch5/toyc.cpp
@@ -71,7 +71,8 @@ static cl::opt<enum Action> emitAction(
 static cl::opt<bool> enableOpt("opt", cl::desc("Enable optimizations"));
 
 /// Returns a Toy AST resulting from parsing the file or a nullptr on error.
-std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
+static std::unique_ptr<toy::ModuleAST>
+parseInputFile(llvm::StringRef filename) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileOrErr =
       llvm::MemoryBuffer::getFileOrSTDIN(filename);
   if (std::error_code ec = fileOrErr.getError()) {
@@ -84,8 +85,8 @@ std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
   return parser.parseModule();
 }
 
-int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context,
-             mlir::OwningOpRef<mlir::ModuleOp> &module) {
+static int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context,
+                    mlir::OwningOpRef<mlir::ModuleOp> &module) {
   // Handle '.toy' input to the compiler.
   if (inputType != InputType::MLIR &&
       !llvm::StringRef(inputFilename).ends_with(".mlir")) {
@@ -114,7 +115,7 @@ int loadMLIR(llvm::SourceMgr &sourceMgr, mlir::MLIRContext &context,
   return 0;
 }
 
-int dumpMLIR() {
+static int dumpMLIR() {
   mlir::DialectRegistry registry;
   mlir::func::registerAllExtensions(registry);
 
@@ -171,7 +172,7 @@ int dumpMLIR() {
   return 0;
 }
 
-int dumpAST() {
+static int dumpAST() {
   if (inputType == InputType::MLIR) {
     llvm::errs() << "Can't dump a Toy AST when the input is MLIR\n";
     return 5;
diff --git a/mlir/examples/toy/Ch6/parser/AST.cpp b/mlir/examples/toy/Ch6/parser/AST.cpp
index 2546f2a9725d6..841642488db68 100644
--- a/mlir/examples/toy/Ch6/parser/AST.cpp
+++ b/mlir/examples/toy/Ch6/parser/AST.cpp
@@ -120,7 +120,7 @@ void ASTDumper::dump(NumberExprAST *num) {
 ///    [ [ 1, 2 ], [ 3, 4 ] ]
 /// We print out such array with the dimensions spelled out at every level:
 ///    <2,2>[<2>[ 1, 2 ], <2>[ 3, 4 ] ]
-void printLitHelper(ExprAST *litOrNum) {
+static void printLitHelper(ExprAST *litOrNum) {
   // Inside a literal expression we can have either a number or another literal
   if (auto *num = llvm::dyn_cast<NumberExprAST>(litOrNum)) {
     llvm::errs() << num->getValue();
diff --git a/mlir/examples/toy/Ch6/toyc.cpp b/mlir/examples/toy/Ch6/toyc.cpp
index 4a5e10973f0c0..c31c53a229fbf 100644
--- a/mlir/examples/toy/Ch6/toyc.cpp
+++ b/mlir/examples/toy/Ch6/toyc.cpp
@@ -96,7 +96,8 @@ static cl::opt<enum Action> emitAction(
 static cl::opt<bool> enableOpt("opt", cl::desc("Enable optimizations"));
 
 /// Returns a Toy AST resulting from parsing the file or a nullptr on error.
-std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
+static std::unique_ptr<toy::ModuleAST>
+parseInputFile(llvm::StringRef filename) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileOrErr =
       llvm::MemoryBuffer::getFileOrSTDIN(filename);
   if (std::error_code ec = fileOrErr.getError()) {
@@ -109,8 +110,8 @@ std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
   return parser.parseModule();
 }
 
-int loadMLIR(mlir::MLIRContext &context,
-             mlir::OwningOpRef<mlir::ModuleOp> &module) {
+static int loadMLIR(mlir::MLIRContext &context,
+                    mlir::OwningOpRef<mlir::ModuleOp> &module) {
   // Handle '.toy' input to the compiler.
   if (inputType != InputType::MLIR &&
       !llvm::StringRef(inputFilename).ends_with(".mlir")) {
@@ -140,8 +141,8 @@ int loadMLIR(mlir::MLIRContext &context,
   return 0;
 }
 
-int loadAndProcessMLIR(mlir::MLIRContext &context,
-                       mlir::OwningOpRef<mlir::ModuleOp> &module) {
+static int loadAndProcessMLIR(mlir::MLIRContext &context,
+                              mlir::OwningOpRef<mlir::ModuleOp> &module) {
   if (int error = loadMLIR(context, module))
     return error;
 
@@ -196,7 +197,7 @@ int loadAndProcessMLIR(mlir::MLIRContext &context,
   return 0;
 }
 
-int dumpAST() {
+static int dumpAST() {
   if (inputType == InputType::MLIR) {
     llvm::errs() << "Can't dump a Toy AST when the input is MLIR\n";
     return 5;
@@ -210,7 +211,7 @@ int dumpAST() {
   return 0;
 }
 
-int dumpLLVMIR(mlir::ModuleOp module) {
+static int dumpLLVMIR(mlir::ModuleOp module) {
   // Register the translation to LLVM IR with the MLIR context.
   mlir::registerBuiltinDialectTranslation(*module->getContext());
   mlir::registerLLVMDialectTranslation(*module->getContext());
@@ -254,7 +255,7 @@ int dumpLLVMIR(mlir::ModuleOp module) {
   return 0;
 }
 
-int runJit(mlir::ModuleOp module) {
+static int runJit(mlir::ModuleOp module) {
   // Initialize LLVM targets.
   llvm::InitializeNativeTarget();
   llvm::InitializeNativeTargetAsmPrinter();
diff --git a/mlir/examples/toy/Ch7/parser/AST.cpp b/mlir/examples/toy/Ch7/parser/AST.cpp
index e38a743a769c6..aa2c7846329c3 100644
--- a/mlir/examples/toy/Ch7/parser/AST.cpp
+++ b/mlir/examples/toy/Ch7/parser/AST.cpp
@@ -123,7 +123,7 @@ void ASTDumper::dump(NumberExprAST *num) {
 ///    [ [ 1, 2 ], [ 3, 4 ] ]
 /// We print out such array with the dimensions spelled out at every level:
 ///    <2,2>[<2>[ 1, 2 ], <2>[ 3, 4 ] ]
-void printLitHelper(ExprAST *litOrNum) {
+static void printLitHelper(ExprAST *litOrNum) {
   // Inside a literal expression we can have either a number or another literal
   if (auto *num = llvm::dyn_cast<NumberExprAST>(litOrNum)) {
     llvm::errs() << num->getValue();
diff --git a/mlir/examples/toy/Ch7/toyc.cpp b/mlir/examples/toy/Ch7/toyc.cpp
index 32208eccaba5f..553ca6ba613d9 100644
--- a/mlir/examples/toy/Ch7/toyc.cpp
+++ b/mlir/examples/toy/Ch7/toyc.cpp
@@ -96,7 +96,8 @@ static cl::opt<enum Action> emitAction(
 static cl::opt<bool> enableOpt("opt", cl::desc("Enable optimizations"));
 
 /// Returns a Toy AST resulting from parsing the file or a nullptr on error.
-std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
+static std::unique_ptr<toy::ModuleAST>
+parseInputFile(llvm::StringRef filename) {
   llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileOrErr =
       llvm::MemoryBuffer::getFileOrSTDIN(filename);
   if (std::error_code ec = fileOrErr.getError()) {
@@ -109,8 +110,8 @@ std::unique_ptr<toy::ModuleAST> parseInputFile(llvm::StringRef filename) {
   return parser.parseModule();
 }
 
-int loadMLIR(mlir::MLIRContext &context,
-             mlir::OwningOpRef<mlir::ModuleOp> &module) {
+static int loadMLIR(mlir::MLIRContext &context,
+                    mlir::OwningOpRef<mlir::ModuleOp> &module) {
   // Handle '.toy' input to the compiler.
   if (inputType != InputType::MLIR &&
       !llvm::StringRef(inputFilename).ends_with(".mlir")) {
@@ -140,8 +141,8 @@ int loadMLIR(mlir::MLIRContext &context,
   return 0;
 }
 
-int loadAndProcessMLIR(mlir::MLIRContext &context,
-                       mlir::OwningOpRef<mlir::ModuleOp> &module) {
+static int loadAndProcessMLIR(mlir::MLIRContext &context,
+                              mlir::OwningOpRef<mlir::ModuleOp> &module) {
   if (int error = loadMLIR(context, module))
     return error;
 
@@ -197,7 +198,7 @@ int loadAndProcessMLIR(mlir::MLIRContext &context,
   return 0;
 }
 
-int dumpAST() {
+static int dumpAST() {
   if (inputType == InputType::MLIR) {
     llvm::errs() << "Can't dump a Toy AST when the input is MLIR\n";
     return 5;
@@ -211,7 +212,7 @@ int dumpAST() {
   return 0;
 }
 
-int dumpLLVMIR(mlir::ModuleOp module) {
+static int dumpLLVMIR(mlir::ModuleOp module) {
   // Register the translation to LLVM IR with the MLIR context.
   mlir::registerBuiltinDialectTranslation(*module->getContext());
   mlir::registerLLVMDialectTranslation(*module->getContext());
@@ -255,7 +256,7 @@ int dumpLLVMIR(mlir::ModuleOp module) {
   return 0;
 }
 
-int runJit(mlir::ModuleOp module) {
+static int runJit(mlir::ModuleOp module) {
   // Initialize LLVM targets.
   llvm::InitializeNativeTarget();
   llvm::InitializeNativeTargetAsmPrinter();
diff --git a/mlir/include/mlir-c/ExecutionEngine.h b/mlir/include/mlir-c/ExecutionEngine.h
index 99cddc5c2598d..1a58d68533f24 100644
--- a/mlir/include/mlir-c/ExecutionEngine.h
+++ b/mlir/include/mlir-c/ExecutionEngine.h
@@ -46,6 +46,13 @@ MLIR_CAPI_EXPORTED MlirExecutionEngine mlirExecutionEngineCreate(
     MlirModule op, int optLevel, int numPaths,
     const MlirStringRef *sharedLibPaths, bool enableObjectDump);
 
+/// Initialize the ExecutionEngine. Global constructors specified by
+/// `llvm.mlir.global_ctors` will be run. One common scenario is that kernel
+/// binary compiled from `gpu.module` gets loaded during initialization. Make
+/// sure all symbols are resolvable before initialization by calling
+/// `mlirExecutionEngineRegisterSymbol` or including shared libraries.
+MLIR_CAPI_EXPORTED void mlirExecutionEngineInitialize(MlirExecutionEngine jit);
+
 /// Destroy an ExecutionEngine instance.
 MLIR_CAPI_EXPORTED void mlirExecutionEngineDestroy(MlirExecutionEngine jit);
 
diff --git a/mlir/include/mlir/Analysis/Presburger/IntegerRelation.h b/mlir/include/mlir/Analysis/Presburger/IntegerRelation.h
index 4b1802413f75f..f86535740fec9 100644
--- a/mlir/include/mlir/Analysis/Presburger/IntegerRelation.h
+++ b/mlir/include/mlir/Analysis/Presburger/IntegerRelation.h
@@ -479,10 +479,28 @@ class IntegerRelation {
   /// respect to a positive constant `divisor`. Two constraints are added to the
   /// system to capture equivalence with the floordiv:
   /// q = dividend floordiv c    <=>   c*q <= dividend <= c*q + c - 1.
-  void addLocalFloorDiv(ArrayRef<DynamicAPInt> dividend,
-                        const DynamicAPInt &divisor);
-  void addLocalFloorDiv(ArrayRef<int64_t> dividend, int64_t divisor) {
-    addLocalFloorDiv(getDynamicAPIntVec(dividend), DynamicAPInt(divisor));
+  /// Returns the column position of the new local variable.
+  unsigned addLocalFloorDiv(ArrayRef<DynamicAPInt> dividend,
+                            const DynamicAPInt &divisor);
+  unsigned addLocalFloorDiv(ArrayRef<int64_t> dividend, int64_t divisor) {
+    return addLocalFloorDiv(getDynamicAPIntVec(dividend),
+                            DynamicAPInt(divisor));
+  }
+
+  /// Adds a new local variable as the modulus of an affine function of other
+  /// variables, the coefficients of which are provided in `exprs`. The modulus
+  /// is with respect to a positive constant `modulus`. The function returns the
+  /// absolute index of the new local variable representing the result of the
+  /// modulus operation. Two new local variables are added to the system, one
+  /// representing the floor div with respect to the modulus and one
+  /// representing the mod. Three constraints are added to the system to capture
+  /// the equivalance. The first two are required to compute the result of the
+  /// floor division `q`, and the third computes the equality relation:
+  /// result =  exprs - modulus * q.
+  unsigned addLocalModulo(ArrayRef<DynamicAPInt> exprs,
+                          const DynamicAPInt &modulus);
+  unsigned addLocalModulo(ArrayRef<int64_t> exprs, int64_t modulus) {
+    return addLocalModulo(getDynamicAPIntVec(exprs), DynamicAPInt(modulus));
   }
 
   /// Projects out (aka eliminates) `num` variables starting at position
@@ -905,6 +923,11 @@ class IntegerRelation {
   IntMatrix inequalities;
 };
 
+inline raw_ostream &operator<<(raw_ostream &os, const IntegerRelation &rel) {
+  rel.print(os);
+  return os;
+}
+
 /// An IntegerPolyhedron represents the set of points from a PresburgerSpace
 /// that satisfy a list of affine constraints. Affine constraints can be
 /// inequalities or equalities in the form:
diff --git a/mlir/include/mlir/Analysis/Presburger/Matrix.h b/mlir/include/mlir/Analysis/Presburger/Matrix.h
index 054eb7b26d06e..15069fa2a55f1 100644
--- a/mlir/include/mlir/Analysis/Presburger/Matrix.h
+++ b/mlir/include/mlir/Analysis/Presburger/Matrix.h
@@ -322,7 +322,7 @@ class FracMatrix : public Matrix<Fraction> {
   // The parameter is what [the original
   // paper](https://www.cs.cmu.edu/~avrim/451f11/lectures/lect1129_LLL.pdf)
   // calls `y`, usually 3/4.
-  void LLL(Fraction delta);
+  void LLL(const Fraction &delta);
 
   // Multiply each row of the matrix by the LCM of the denominators, thereby
   // converting it to an integer matrix.
diff --git a/mlir/include/mlir/CMakeLists.txt b/mlir/include/mlir/CMakeLists.txt
index 9cf3b442aa4ad..f88a35ba1efc7 100644
--- a/mlir/include/mlir/CMakeLists.txt
+++ b/mlir/include/mlir/CMakeLists.txt
@@ -4,4 +4,5 @@ add_subdirectory(Dialect)
 add_subdirectory(IR)
 add_subdirectory(Interfaces)
 add_subdirectory(Reducer)
+add_subdirectory(Target)
 add_subdirectory(Transforms)
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
index d5055f023cdc8..8e86808cc424a 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/MemRefBuilder.h
@@ -189,15 +189,13 @@ class UnrankedMemRefDescriptor : public StructBuilder {
   /// `unpack`.
   static unsigned getNumUnpackedValues() { return 2; }
 
-  /// Builds IR computing the sizes in bytes (suitable for opaque allocation)
-  /// and appends the corresponding values into `sizes`. `addressSpaces`
-  /// which must have the same length as `values`, is needed to handle layouts
-  /// where sizeof(ptr addrspace(N)) != sizeof(ptr addrspace(0)).
-  static void computeSizes(OpBuilder &builder, Location loc,
+  /// Builds and returns IR computing the size in bytes (suitable for opaque
+  /// allocation). `addressSpace` is needed to handle layouts where
+  /// sizeof(ptr addrspace(N)) != sizeof(ptr addrspace(0)).
+  static Value computeSize(OpBuilder &builder, Location loc,
                            const LLVMTypeConverter &typeConverter,
-                           ArrayRef<UnrankedMemRefDescriptor> values,
-                           ArrayRef<unsigned> addressSpaces,
-                           SmallVectorImpl<Value> &sizes);
+                           UnrankedMemRefDescriptor desc,
+                           unsigned addressSpace);
 
   /// TODO: The following accessors don't take alignment rules between elements
   /// of the descriptor struct into account. For some architectures, it might be
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
index 969154abe8830..c292e3727f46c 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/Pattern.h
@@ -183,10 +183,20 @@ class ConvertToLLVMPattern : public ConversionPattern {
                          ArrayRef<Value> sizes, ArrayRef<Value> strides,
                          ConversionPatternRewriter &rewriter) const;
 
+  /// Copies the given unranked memory descriptor to heap-allocated memory (if
+  /// toDynamic is true) or to stack-allocated memory (otherwise) and returns
+  /// the new descriptor. Also frees the previously used memory (that is assumed
+  /// to be heap-allocated) if toDynamic is false. Returns a "null" SSA value
+  /// on failure.
+  Value copyUnrankedDescriptor(OpBuilder &builder, Location loc,
+                               UnrankedMemRefType memRefType, Value operand,
+                               bool toDynamic) const;
+
   /// Copies the memory descriptor for any operands that were unranked
   /// descriptors originally to heap-allocated memory (if toDynamic is true) or
-  /// to stack-allocated memory (otherwise). Also frees the previously used
-  /// memory (that is assumed to be heap-allocated) if toDynamic is false.
+  /// to stack-allocated memory (otherwise). The vector of descriptors is
+  /// updated in place. Also frees the previously used memory (that is assumed
+  /// to be heap-allocated) if toDynamic is false.
   LogicalResult copyUnrankedDescriptors(OpBuilder &builder, Location loc,
                                         TypeRange origTypes,
                                         SmallVectorImpl<Value> &operands,
@@ -233,9 +243,7 @@ class ConvertOpToLLVMPattern : public ConvertToLLVMPattern {
   virtual LogicalResult
   matchAndRewrite(SourceOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const {
-    SmallVector<Value> oneToOneOperands =
-        getOneToOneAdaptorOperands(adaptor.getOperands());
-    return matchAndRewrite(op, OpAdaptor(oneToOneOperands, adaptor), rewriter);
+    return dispatchTo1To1(*this, op, adaptor, rewriter);
   }
 
 private:
@@ -276,7 +284,7 @@ class ConvertOpInterfaceToLLVMPattern : public ConvertToLLVMPattern {
   virtual LogicalResult
   matchAndRewrite(SourceOp op, ArrayRef<ValueRange> operands,
                   ConversionPatternRewriter &rewriter) const {
-    return matchAndRewrite(op, getOneToOneAdaptorOperands(operands), rewriter);
+    return dispatchTo1To1(*this, op, operands, rewriter);
   }
 
 private:
diff --git a/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h b/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h
index 38b5e492a8ed8..2096bcb9896a5 100644
--- a/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h
+++ b/mlir/include/mlir/Conversion/LLVMCommon/TypeConverter.h
@@ -74,8 +74,14 @@ class LLVMTypeConverter : public TypeConverter {
   /// LLVM-compatible type. In particular, if more than one value is returned,
   /// create an LLVM dialect structure type with elements that correspond to
   /// each of the types converted with `convertCallingConventionType`.
-  Type packFunctionResults(TypeRange types,
-                           bool useBarePointerCallConv = false) const;
+  ///
+  /// Populate the converted (unpacked) types into `groupedTypes`, if provided.
+  /// `groupedType` contains one nested vector per input type. In case of a 1:N
+  /// conversion, a nested vector may contain 0 or more then 1 converted type.
+  Type
+  packFunctionResults(TypeRange types, bool useBarePointerCallConv = false,
+                      SmallVector<SmallVector<Type>> *groupedTypes = nullptr,
+                      int64_t *numConvertedTypes = nullptr) const;
 
   /// Convert a non-empty list of types of values produced by an operation into
   /// an LLVM-compatible type. In particular, if more than one value is
@@ -88,15 +94,9 @@ class LLVMTypeConverter : public TypeConverter {
   /// UnrankedMemRefType, are converted following the specific rules for the
   /// calling convention. Calling convention independent types are converted
   /// following the default LLVM type conversions.
-  Type convertCallingConventionType(Type type,
-                                    bool useBarePointerCallConv = false) const;
-
-  /// Promote the bare pointers in 'values' that resulted from memrefs to
-  /// descriptors. 'stdTypes' holds the types of 'values' before the conversion
-  /// to the LLVM-IR dialect (i.e., MemRefType, or any other builtin type).
-  void promoteBarePtrsToDescriptors(ConversionPatternRewriter &rewriter,
-                                    Location loc, ArrayRef<Type> stdTypes,
-                                    SmallVectorImpl<Value> &values) const;
+  LogicalResult
+  convertCallingConventionType(Type type, SmallVectorImpl<Type> &result,
+                               bool useBarePointerCallConv = false) const;
 
   /// Returns the MLIR context.
   MLIRContext &getContext() const;
@@ -109,9 +109,14 @@ class LLVMTypeConverter : public TypeConverter {
   /// Promote the LLVM representation of all operands including promoting MemRef
   /// descriptors to stack and use pointers to struct to avoid the complexity
   /// of the platform-specific C/C++ ABI lowering related to struct argument
-  /// passing.
+  /// passing. (The ArrayRef variant is for 1:N.)
+  SmallVector<Value, 4> promoteOperands(Location loc, ValueRange opOperands,
+                                        ArrayRef<ValueRange> adaptorOperands,
+                                        OpBuilder &builder,
+                                        bool useBarePtrCallConv = false) const;
   SmallVector<Value, 4> promoteOperands(Location loc, ValueRange opOperands,
-                                        ValueRange operands, OpBuilder &builder,
+                                        ValueRange adaptorOperands,
+                                        OpBuilder &builder,
                                         bool useBarePtrCallConv = false) const;
 
   /// Promote the LLVM struct representation of one MemRef descriptor to stack
diff --git a/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h
index b595b6a308bea..5abfb3d7e72dd 100644
--- a/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h
+++ b/mlir/include/mlir/Conversion/MemRefToEmitC/MemRefToEmitC.h
@@ -10,8 +10,11 @@
 
 constexpr const char *alignedAllocFunctionName = "aligned_alloc";
 constexpr const char *mallocFunctionName = "malloc";
+constexpr const char *memcpyFunctionName = "memcpy";
 constexpr const char *cppStandardLibraryHeader = "cstdlib";
 constexpr const char *cStandardLibraryHeader = "stdlib.h";
+constexpr const char *cppStringLibraryHeader = "cstring";
+constexpr const char *cStringLibraryHeader = "string.h";
 
 namespace mlir {
 class DialectRegistry;
diff --git a/mlir/include/mlir/Conversion/Passes.h b/mlir/include/mlir/Conversion/Passes.h
index 3dc48b2201cf2..91b2ecf8922a3 100644
--- a/mlir/include/mlir/Conversion/Passes.h
+++ b/mlir/include/mlir/Conversion/Passes.h
@@ -75,6 +75,7 @@
 #include "mlir/Conversion/TosaToTensor/TosaToTensor.h"
 #include "mlir/Conversion/UBToLLVM/UBToLLVM.h"
 #include "mlir/Conversion/UBToSPIRV/UBToSPIRV.h"
+#include "mlir/Conversion/VectorToAMX/VectorToAMX.h"
 #include "mlir/Conversion/VectorToArmSME/VectorToArmSME.h"
 #include "mlir/Conversion/VectorToGPU/VectorToGPU.h"
 #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h"
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 6e1baaf23fcf7..2058aba7f9e37 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -52,6 +52,8 @@ def ConvertToLLVMPass : Pass<"convert-to-llvm"> {
                "Test conversion patterns of only the specified dialects">,
     Option<"useDynamic", "dynamic", "bool", "false",
            "Use op conversion attributes to configure the conversion">,
+    Option<"allowPatternRollback", "allow-pattern-rollback", "bool", "true",
+           "Experimental performance flag to disallow pattern rollback">
   ];
 }
 
@@ -1531,6 +1533,19 @@ def ConvertVectorToXeGPU : Pass<"convert-vector-to-xegpu"> {
   ];
 }
 
+//===----------------------------------------------------------------------===//
+// VectorToAMX
+//===----------------------------------------------------------------------===//
+
+def ConvertVectorToAMX : Pass<"convert-vector-to-amx"> {
+  let summary = "Lower the operations from the vector dialect into the AMX "
+                "dialect";
+  let dependentDialects = [
+    "affine::AffineDialect", "amx::AMXDialect", "arith::ArithDialect",
+    "memref::MemRefDialect", "scf::SCFDialect", "vector::VectorDialect"
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // XeVMToLLVM
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Conversion/VectorToAMX/VectorToAMX.h b/mlir/include/mlir/Conversion/VectorToAMX/VectorToAMX.h
new file mode 100644
index 0000000000000..b075ac92990a2
--- /dev/null
+++ b/mlir/include/mlir/Conversion/VectorToAMX/VectorToAMX.h
@@ -0,0 +1,26 @@
+//===- VectorToAMX.h - Convert vector to AMX dialect ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_CONVERSION_VECTORTOAMX_VECTORTOAMX_H
+#define MLIR_CONVERSION_VECTORTOAMX_VECTORTOAMX_H
+
+#include "mlir/IR/PatternMatch.h"
+
+namespace mlir {
+class Pass;
+class RewritePatternSet;
+
+#define GEN_PASS_DECL_CONVERTVECTORTOAMX
+#include "mlir/Conversion/Passes.h.inc"
+
+/// Collect a set of patterns to convert from the vector to AMX ops.
+void populateVectorToAMXConversionPatterns(RewritePatternSet &patterns);
+
+} // namespace mlir
+
+#endif // MLIR_CONVERSION_VECTORTOAMX_VECTORTOAMX_H
diff --git a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
index 92aacdaef4136..2ccf350a359a8 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
+++ b/mlir/include/mlir/Dialect/AMDGPU/IR/AMDGPU.td
@@ -656,6 +656,48 @@ def AMDGPU_SwizzleBitModeOp : AMDGPU_Op<"swizzle_bitmode",
   }];
 }
 
+def AMDGPU_PermlaneSwapOp : AMDGPU_Op<"permlane_swap", [Pure, AllTypesMatch<["result", "src"]>]> {
+  let summary = "AMDGPU permlane swap op";
+  let description = [{
+    High-level wrapper on `rocdl.permlane{16,32}.swap` variants for permutations
+    on rows of lanes in a subgroup.
+
+    Supports arbitrary int/float/vector types, which will be repacked to i32 and
+    one or more `rocdl.permlane_swap` ops during lowering.
+    Supported lane permutations:
+    - Swap the data between odd and even rows of 16 lanes
+    - Swap the data between the first 32 lanes and the last 32 lanes
+
+    Example:
+    ```mlir
+    %0 = amdgpu.permlane_swap %src 16 : f16
+    %1 = amdgpu.permlane_swap %src 32 { fetch_inactive = true, bound_ctrl = true } : f16
+    ```
+
+    Operands:
+    * `$src`: Vector register to permute across lanes of the subgroup.
+    * `$row_length`: The length of a row to permute in number of lanes (valid values are 16 and 32).
+    * `$fetch_inactive`: Optional. Used to dertermine behavior of a fetch from a disabled lane.
+      `fetch_inactive = false`: If the source lane is disabled, use `bound_ctrl` to determine the source value.
+      `fetch_inactive = true`: If the source lane is disabled, fetch the source value anyway (ignoring `bound_ctrl`).
+    * `$bound_ctrl`: Optional. Used to determine what a thread should do if its source operand is from 
+      a disabled lane: use the value zero, or disable the write. 
+      `bound_ctrl = false`: Do not write when source is from a disabled lane
+      `bound_ctrl = true`: Use zero as input if source is from a disabled lane
+
+    Note: Lowering is only supported on gfx950 and up.
+  }];
+  let arguments = (ins AnyIntegerOrFloatOr1DVector:$src,
+                       I32Attr:$row_length,
+                       DefaultValuedAttr<BoolAttr, "false">:$fetch_inactive,
+                       DefaultValuedAttr<BoolAttr, "false">:$bound_ctrl);
+  let results = (outs AnyIntegerOrFloatOr1DVector:$result);
+  let assemblyFormat = [{
+    $src $row_length attr-dict `:` type($result)
+  }];
+  let hasVerifier = 1;
+}
+
 def AMDGPU_LDSBarrierOp : AMDGPU_Op<"lds_barrier"> {
   let summary = "Barrier that includes a wait for LDS memory operations.";
   let description = [{
@@ -907,7 +949,8 @@ def AMDGPU_GatherToLDSOp :
       The elements gathered by the subgroup will be written contiguously in order of lane ID
       starting at `$dst[$dstIndices]`. Byte-sized (ex. i8) or short-sized (ex. i16)
       types will be zero-padded/extended to 32 bits before being written. 96-bit types
-      (ex. vector<3xf32>) will be zero-padded to 128 bits before being written.
+      (ex. vector<3xf32>) will be zero-padded to 128 bits before being written. Only the
+      offsets held by lane 0 are used.
     * `$transferType`: type of the data to be transferred by each thread. This is used to determine
       the size of the data to be transferred and the number of threads in the subgroup.
       The transfer type must be a scalar type or a vector type with a single element type.
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
index 19a2ade2e95a0..a3e9fc3c57bb1 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithBase.td
@@ -20,10 +20,10 @@ def Arith_Dialect : Dialect {
     mathematical operations. This includes unary, binary, and ternary arithmetic
     ops, bitwise and shift ops, cast ops, and compare ops. Operations in this
     dialect also accept vectors and tensors of integers or floats. The dialect
-    assumes integers are represented by bitvectors with a two's complement 
-    representation. Unless otherwise stated, the operations within this dialect 
-    propagate poison values, i.e., if any of its inputs are poison, then the 
-    output is poison. Unless otherwise stated, operations applied to `vector` 
+    assumes integers are represented by bitvectors with a two's complement
+    representation. Unless otherwise stated, the operations within this dialect
+    propagate poison values, i.e., if any of its inputs are poison, then the
+    output is poison. Unless otherwise stated, operations applied to `vector`
     and `tensor` values propagates poison elementwise.
   }];
 
@@ -76,27 +76,29 @@ def Arith_CmpIPredicateAttr : I64EnumAttr<
 
 def ATOMIC_RMW_KIND_ADDF     : I64EnumAttrCase<"addf", 0>;
 def ATOMIC_RMW_KIND_ADDI     : I64EnumAttrCase<"addi", 1>;
-def ATOMIC_RMW_KIND_ASSIGN   : I64EnumAttrCase<"assign", 2>;
-def ATOMIC_RMW_KIND_MAXIMUMF : I64EnumAttrCase<"maximumf", 3>;
-def ATOMIC_RMW_KIND_MAXS     : I64EnumAttrCase<"maxs", 4>;
-def ATOMIC_RMW_KIND_MAXU     : I64EnumAttrCase<"maxu", 5>;
-def ATOMIC_RMW_KIND_MINIMUMF : I64EnumAttrCase<"minimumf", 6>;
-def ATOMIC_RMW_KIND_MINS     : I64EnumAttrCase<"mins", 7>;
-def ATOMIC_RMW_KIND_MINU     : I64EnumAttrCase<"minu", 8>;
-def ATOMIC_RMW_KIND_MULF     : I64EnumAttrCase<"mulf", 9>;
-def ATOMIC_RMW_KIND_MULI     : I64EnumAttrCase<"muli", 10>;
-def ATOMIC_RMW_KIND_ORI      : I64EnumAttrCase<"ori", 11>;
-def ATOMIC_RMW_KIND_ANDI     : I64EnumAttrCase<"andi", 12>;
-def ATOMIC_RMW_KIND_MAXNUMF  : I64EnumAttrCase<"maxnumf", 13>;
-def ATOMIC_RMW_KIND_MINNUMF  : I64EnumAttrCase<"minnumf", 14>;
+def ATOMIC_RMW_KIND_ANDI     : I64EnumAttrCase<"andi", 2>;
+def ATOMIC_RMW_KIND_ASSIGN   : I64EnumAttrCase<"assign", 3>;
+def ATOMIC_RMW_KIND_MAXIMUMF : I64EnumAttrCase<"maximumf", 4>;
+def ATOMIC_RMW_KIND_MAXNUMF  : I64EnumAttrCase<"maxnumf", 5>;
+def ATOMIC_RMW_KIND_MAXS     : I64EnumAttrCase<"maxs", 6>;
+def ATOMIC_RMW_KIND_MAXU     : I64EnumAttrCase<"maxu", 7>;
+def ATOMIC_RMW_KIND_MINIMUMF : I64EnumAttrCase<"minimumf", 8>;
+def ATOMIC_RMW_KIND_MINNUMF  : I64EnumAttrCase<"minnumf", 9>;
+def ATOMIC_RMW_KIND_MINS     : I64EnumAttrCase<"mins", 10>;
+def ATOMIC_RMW_KIND_MINU     : I64EnumAttrCase<"minu", 11>;
+def ATOMIC_RMW_KIND_MULF     : I64EnumAttrCase<"mulf", 12>;
+def ATOMIC_RMW_KIND_MULI     : I64EnumAttrCase<"muli", 13>;
+def ATOMIC_RMW_KIND_ORI      : I64EnumAttrCase<"ori", 14>;
+def ATOMIC_RMW_KIND_XORI     : I64EnumAttrCase<"xori", 15>;
 
 def AtomicRMWKindAttr : I64EnumAttr<
     "AtomicRMWKind", "",
-    [ATOMIC_RMW_KIND_ADDF, ATOMIC_RMW_KIND_ADDI, ATOMIC_RMW_KIND_ASSIGN,
-     ATOMIC_RMW_KIND_MAXIMUMF, ATOMIC_RMW_KIND_MAXS, ATOMIC_RMW_KIND_MAXU,
-     ATOMIC_RMW_KIND_MINIMUMF, ATOMIC_RMW_KIND_MINS, ATOMIC_RMW_KIND_MINU,
+    [ATOMIC_RMW_KIND_ADDF, ATOMIC_RMW_KIND_ADDI, ATOMIC_RMW_KIND_ANDI,
+     ATOMIC_RMW_KIND_ASSIGN, ATOMIC_RMW_KIND_MAXIMUMF, ATOMIC_RMW_KIND_MAXNUMF,
+     ATOMIC_RMW_KIND_MAXS, ATOMIC_RMW_KIND_MAXU, ATOMIC_RMW_KIND_MINIMUMF,
+     ATOMIC_RMW_KIND_MINNUMF, ATOMIC_RMW_KIND_MINS, ATOMIC_RMW_KIND_MINU,
      ATOMIC_RMW_KIND_MULF, ATOMIC_RMW_KIND_MULI, ATOMIC_RMW_KIND_ORI,
-     ATOMIC_RMW_KIND_ANDI, ATOMIC_RMW_KIND_MAXNUMF, ATOMIC_RMW_KIND_MINNUMF]> {
+     ATOMIC_RMW_KIND_XORI]> {
   let cppNamespace = "::mlir::arith";
 }
 
diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
index ef9ccb7e87946..20c9097b51e6d 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
@@ -216,14 +216,14 @@ def Arith_ConstantOp : Op<Arith_Dialect, "constant",
 def Arith_AddIOp : Arith_IntBinaryOpWithOverflowFlags<"addi", [Commutative]> {
   let summary = "integer addition operation";
   let description = [{
-    Performs N-bit addition on the operands. The operands are interpreted as 
-    unsigned bitvectors. The result is represented by a bitvector containing the 
-    mathematical value of the addition modulo 2^n, where `n` is the bitwidth. 
-    Because `arith` integers use a two's complement representation, this operation 
+    Performs N-bit addition on the operands. The operands are interpreted as
+    unsigned bitvectors. The result is represented by a bitvector containing the
+    mathematical value of the addition modulo 2^n, where `n` is the bitwidth.
+    Because `arith` integers use a two's complement representation, this operation
     is applicable on both signed and unsigned integer operands.
 
     The `addi` operation takes two operands and returns one result, each of
-    these is required to be the same type. This type may be an integer scalar type, 
+    these is required to be the same type. This type may be an integer scalar type,
     a vector whose element type is integer, or a tensor of integers.
 
     This op supports `nuw`/`nsw` overflow flags which stands for
@@ -489,8 +489,8 @@ def Arith_DivUIOp : Arith_IntBinaryOp<"divui", [ConditionallySpeculatable]> {
     the most significant, i.e. for `i16` given two's complement representation,
     `6 / -2 = 6 / (2^16 - 2) = 0`.
 
-    Division by zero is undefined behavior. When applied to `vector` and 
-    `tensor` values, the behavior is undefined if _any_ elements are divided by 
+    Division by zero is undefined behavior. When applied to `vector` and
+    `tensor` values, the behavior is undefined if _any_ elements are divided by
     zero.
 
     Example:
@@ -525,9 +525,9 @@ def Arith_DivSIOp : Arith_IntBinaryOp<"divsi", [ConditionallySpeculatable]> {
     Signed integer division. Rounds towards zero. Treats the leading bit as
     sign, i.e. `6 / -2 = -3`.
 
-    Divison by zero, or signed division overflow (minimum value divided by -1) 
-    is undefined behavior. When applied to `vector` and `tensor` values, the 
-    behavior is undefined if _any_ of its elements are divided by zero or has a 
+    Divison by zero, or signed division overflow (minimum value divided by -1)
+    is undefined behavior. When applied to `vector` and `tensor` values, the
+    behavior is undefined if _any_ of its elements are divided by zero or has a
     signed division overflow.
 
     Example:
@@ -562,10 +562,10 @@ def Arith_CeilDivUIOp : Arith_IntBinaryOp<"ceildivui",
   let description = [{
     Unsigned integer division. Rounds towards positive infinity. Treats the
     leading bit as the most significant, i.e. for `i16` given two's complement
-    representation, `6 / -2 = 6 / (2^16 - 2) = 1`. 
+    representation, `6 / -2 = 6 / (2^16 - 2) = 1`.
 
-    Division by zero is undefined behavior. When applied to `vector` and 
-    `tensor` values, the behavior is undefined if _any_ elements are divided by 
+    Division by zero is undefined behavior. When applied to `vector` and
+    `tensor` values, the behavior is undefined if _any_ elements are divided by
     zero.
 
     Example:
@@ -594,9 +594,9 @@ def Arith_CeilDivSIOp : Arith_IntBinaryOp<"ceildivsi",
   let description = [{
     Signed integer division. Rounds towards positive infinity, i.e. `7 / -2 = -3`.
 
-    Divison by zero, or signed division overflow (minimum value divided by -1) 
-    is undefined behavior. When applied to `vector` and `tensor` values, the 
-    behavior is undefined if _any_ of its elements are divided by zero or has a 
+    Divison by zero, or signed division overflow (minimum value divided by -1)
+    is undefined behavior. When applied to `vector` and `tensor` values, the
+    behavior is undefined if _any_ of its elements are divided by zero or has a
     signed division overflow.
 
     Example:
@@ -624,9 +624,9 @@ def Arith_FloorDivSIOp : Arith_TotalIntBinaryOp<"floordivsi"> {
   let description = [{
     Signed integer division. Rounds towards negative infinity, i.e. `5 / -2 = -3`.
 
-    Divison by zero, or signed division overflow (minimum value divided by -1) 
-    is undefined behavior. When applied to `vector` and `tensor` values, the 
-    behavior is undefined if _any_ of its elements are divided by zero or has a 
+    Divison by zero, or signed division overflow (minimum value divided by -1)
+    is undefined behavior. When applied to `vector` and `tensor` values, the
+    behavior is undefined if _any_ of its elements are divided by zero or has a
     signed division overflow.
 
     Example:
@@ -650,8 +650,8 @@ def Arith_RemUIOp : Arith_TotalIntBinaryOp<"remui"> {
     Unsigned integer division remainder. Treats the leading bit as the most
     significant, i.e. for `i16`, `6 % -2 = 6 % (2^16 - 2) = 6`.
 
-    Division by zero is undefined behavior. When applied to `vector` and 
-    `tensor` values, the behavior is undefined if _any_ elements are divided by 
+    Division by zero is undefined behavior. When applied to `vector` and
+    `tensor` values, the behavior is undefined if _any_ elements are divided by
     zero.
 
     Example:
@@ -680,8 +680,8 @@ def Arith_RemSIOp : Arith_TotalIntBinaryOp<"remsi"> {
     Signed integer division remainder. Treats the leading bit as sign, i.e. `6 %
     -2 = 0`.
 
-    Division by zero is undefined behavior. When applied to `vector` and 
-    `tensor` values, the behavior is undefined if _any_ elements are divided by 
+    Division by zero is undefined behavior. When applied to `vector` and
+    `tensor` values, the behavior is undefined if _any_ elements are divided by
     zero.
 
     Example:
@@ -794,9 +794,9 @@ def Arith_XOrIOp : Arith_TotalIntBinaryOp<"xori", [Commutative]> {
 def Arith_ShLIOp : Arith_IntBinaryOpWithOverflowFlags<"shli"> {
   let summary = "integer left-shift";
   let description = [{
-    The `shli` operation shifts the integer value of the first operand to the left 
-    by the integer value of the second operand. The second operand is interpreted as 
-    unsigned. The low order bits are filled with zeros. If the value of the second 
+    The `shli` operation shifts the integer value of the first operand to the left
+    by the integer value of the second operand. The second operand is interpreted as
+    unsigned. The low order bits are filled with zeros. If the value of the second
     operand is greater or equal than the bitwidth of the first operand, then the
     operation returns poison.
 
@@ -811,7 +811,7 @@ def Arith_ShLIOp : Arith_IntBinaryOpWithOverflowFlags<"shli"> {
     %1 = arith.constant 5 : i8  // %1 is 0b00000101
     %2 = arith.constant 3 : i8
     %3 = arith.shli %1, %2 : i8 // %3 is 0b00101000
-    %4 = arith.shli %1, %2 overflow<nsw, nuw> : i8  
+    %4 = arith.shli %1, %2 overflow<nsw, nuw> : i8
     ```
   }];
   let hasFolder = 1;
@@ -824,9 +824,9 @@ def Arith_ShLIOp : Arith_IntBinaryOpWithOverflowFlags<"shli"> {
 def Arith_ShRUIOp : Arith_TotalIntBinaryOp<"shrui"> {
   let summary = "unsigned integer right-shift";
   let description = [{
-    The `shrui` operation shifts an integer value of the first operand to the right 
+    The `shrui` operation shifts an integer value of the first operand to the right
     by the value of the second operand. The first operand is interpreted as unsigned,
-    and the second operand is interpreted as unsigned. The high order bits are always 
+    and the second operand is interpreted as unsigned. The high order bits are always
     filled with zeros. If the value of the second operand is greater or equal than the
     bitwidth of the first operand, then the operation returns poison.
 
@@ -848,11 +848,11 @@ def Arith_ShRUIOp : Arith_TotalIntBinaryOp<"shrui"> {
 def Arith_ShRSIOp : Arith_TotalIntBinaryOp<"shrsi"> {
   let summary = "signed integer right-shift";
   let description = [{
-    The `shrsi` operation shifts an integer value of the first operand to the right 
-    by the value of the second operand. The first operand is interpreted as signed, 
-    and the second operand is interpreter as unsigned. The high order bits in the 
-    output are filled with copies of the most-significant bit of the shifted value 
-    (which means that the sign of the value is preserved). If the value of the second 
+    The `shrsi` operation shifts an integer value of the first operand to the right
+    by the value of the second operand. The first operand is interpreted as signed,
+    and the second operand is interpreter as unsigned. The high order bits in the
+    output are filled with copies of the most-significant bit of the shifted value
+    (which means that the sign of the value is preserved). If the value of the second
     operand is greater or equal than bitwidth of the first operand, then the operation
     returns poison.
 
@@ -1229,28 +1229,28 @@ def Arith_ScalingExtFOp
   let summary = "Upcasts input floats using provided scales values following "
                 "OCP MXFP Spec";
   let description = [{
-  This operation upcasts input floating-point values using provided scale 
-  values. It expects both scales and the input operand to be of the same shape, 
-  making the operation elementwise. Scales are usually calculated per block 
+  This operation upcasts input floating-point values using provided scale
+  values. It expects both scales and the input operand to be of the same shape,
+  making the operation elementwise. Scales are usually calculated per block
   following the OCP MXFP spec as described in https://arxiv.org/abs/2310.10537.
 
-  If scales are calculated per block where blockSize != 1, then scales may 
-  require broadcasting to make this operation elementwise. For example, let's 
-  say the input is of shape `<dim1 x dim2 x ... dimN>`. Given blockSize != 1 and 
-  assuming quantization happens on the last axis, the input can be reshaped to 
-  `<dim1 x dim2 x ... (dimN/blockSize) x blockSize>`. Scales will be calculated 
-  per block on the last axis. Therefore, scales will be of shape 
-  `<dim1 x dim2 x ... (dimN/blockSize) x 1>`. Scales could also be of some other 
-  shape as long as it is broadcast compatible with the input, e.g., 
+  If scales are calculated per block where blockSize != 1, then scales may
+  require broadcasting to make this operation elementwise. For example, let's
+  say the input is of shape `<dim1 x dim2 x ... dimN>`. Given blockSize != 1 and
+  assuming quantization happens on the last axis, the input can be reshaped to
+  `<dim1 x dim2 x ... (dimN/blockSize) x blockSize>`. Scales will be calculated
+  per block on the last axis. Therefore, scales will be of shape
+  `<dim1 x dim2 x ... (dimN/blockSize) x 1>`. Scales could also be of some other
+  shape as long as it is broadcast compatible with the input, e.g.,
   `<1 x 1 x ... (dimN/blockSize) x 1>`.
 
-  In this example, before calling into `arith.scaling_extf`, scales must be 
-  broadcasted to `<dim1 x dim2 x dim3 ... (dimN/blockSize) x blockSize>`. Note 
-  that there could be multiple quantization axes. Internally, 
+  In this example, before calling into `arith.scaling_extf`, scales must be
+  broadcasted to `<dim1 x dim2 x dim3 ... (dimN/blockSize) x blockSize>`. Note
+  that there could be multiple quantization axes. Internally,
   `arith.scaling_extf` would perform the following:
- 
+
     ```
-    resultTy = get_type(result) 
+    resultTy = get_type(result)
     scaleTy  = get_type(scale)
     inputTy = get_type(input)
     scale.exponent = arith.truncf(scale) : scaleTy to f8E8M0
@@ -1258,12 +1258,12 @@ def Arith_ScalingExtFOp
     input.extf = arith.extf(input) : inputTy to resultTy
     result = arith.mulf(scale.extf, input.extf)
     ```
-    It propagates NaN values. Therefore, if either scale or the input element 
+    It propagates NaN values. Therefore, if either scale or the input element
     contains NaN, then the output element value will also be a NaN.
   }];
   let hasVerifier = 1;
   let assemblyFormat =
-      [{ $in `,` $scale (`fastmath` `` $fastmath^)? attr-dict `:` 
+      [{ $in `,` $scale (`fastmath` `` $fastmath^)? attr-dict `:`
       type($in) `,` type($scale) `to` type($out)}];
 }
 
@@ -1373,28 +1373,28 @@ def Arith_ScalingTruncFOp
   let summary = "Downcasts input floating point values using provided scales "
                 "values following OCP MXFP Spec";
   let description = [{
-    This operation downcasts input using the provided scale values. It expects 
-    both scales and the input operand to be of the same shape and, therefore, 
-    makes the operation elementwise. Scales are usually calculated per block 
+    This operation downcasts input using the provided scale values. It expects
+    both scales and the input operand to be of the same shape and, therefore,
+    makes the operation elementwise. Scales are usually calculated per block
     following the OCP MXFP spec as described in https://arxiv.org/abs/2310.10537.
     Users are required to normalize and clamp the scales as necessary before calling
     passing them to this operation.  OCP MXFP spec also does the flushing of denorms
-    on the input operand, which should be handled during lowering by passing appropriate 
-    fastMath flag to this operation. 
-
-    If scales are calculated per block where blockSize != 1, scales may require 
-    broadcasting to make this operation elementwise. For example, let's say the 
-    input is of shape `<dim1 x dim2 x ... dimN>`. Given blockSize != 1 and 
-    assuming quantization happens on the last axis, the input can be reshaped to 
-    `<dim1 x dim2 x ... (dimN/blockSize) x blockSize>`. Scales will be calculated 
-    per block on the last axis. Therefore, scales will be of shape 
-    `<dim1 x dim2 x ... (dimN/blockSize) x 1>`. Scales could also be of some other 
-    shape as long as it is broadcast compatible with the input, e.g., 
+    on the input operand, which should be handled during lowering by passing appropriate
+    fastMath flag to this operation.
+
+    If scales are calculated per block where blockSize != 1, scales may require
+    broadcasting to make this operation elementwise. For example, let's say the
+    input is of shape `<dim1 x dim2 x ... dimN>`. Given blockSize != 1 and
+    assuming quantization happens on the last axis, the input can be reshaped to
+    `<dim1 x dim2 x ... (dimN/blockSize) x blockSize>`. Scales will be calculated
+    per block on the last axis. Therefore, scales will be of shape
+    `<dim1 x dim2 x ... (dimN/blockSize) x 1>`. Scales could also be of some other
+    shape as long as it is broadcast compatible with the input, e.g.,
     `<1 x 1 x ... (dimN/blockSize) x 1>`.
 
-    In this example, before calling into `arith.scaling_truncf`, scales must be 
-    broadcasted to `<dim1 x dim2 x dim3 ... (dimN/blockSize) x blockSize>`. Note 
-    that there could be multiple quantization axes. Internally, 
+    In this example, before calling into `arith.scaling_truncf`, scales must be
+    broadcasted to `<dim1 x dim2 x dim3 ... (dimN/blockSize) x blockSize>`. Note
+    that there could be multiple quantization axes. Internally,
     `arith.scaling_truncf` would perform the following:
 
     ```
@@ -1409,7 +1409,7 @@ def Arith_ScalingTruncFOp
   }];
   let hasVerifier = 1;
   let assemblyFormat =
-      [{ $in `,` $scale ($roundingmode^)? (`fastmath` `` $fastmath^)? attr-dict `:` 
+      [{ $in `,` $scale ($roundingmode^)? (`fastmath` `` $fastmath^)? attr-dict `:`
       type($in) `,` type($scale) `to` type($out)}];
 }
 
diff --git a/mlir/include/mlir/Dialect/CommonFolders.h b/mlir/include/mlir/Dialect/CommonFolders.h
index b5a12426aff80..113765157946d 100644
--- a/mlir/include/mlir/Dialect/CommonFolders.h
+++ b/mlir/include/mlir/Dialect/CommonFolders.h
@@ -15,10 +15,16 @@
 #ifndef MLIR_DIALECT_COMMONFOLDERS_H
 #define MLIR_DIALECT_COMMONFOLDERS_H
 
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/BuiltinAttributeInterfaces.h"
 #include "mlir/IR/BuiltinAttributes.h"
-#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/BuiltinTypeInterfaces.h"
+#include "mlir/IR/Types.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
+
+#include <cassert>
+#include <cstddef>
 #include <optional>
 
 namespace mlir {
@@ -30,11 +36,13 @@ class PoisonAttr;
 /// Uses `resultType` for the type of the returned attribute.
 /// Optional PoisonAttr template argument allows to specify 'poison' attribute
 /// which will be directly propagated to result.
-template <class AttrElementT,
+template <class AttrElementT, //
           class ElementValueT = typename AttrElementT::ValueType,
           class PoisonAttr = ub::PoisonAttr,
+          class ResultAttrElementT = AttrElementT,
+          class ResultElementValueT = typename ResultAttrElementT::ValueType,
           class CalculationT = function_ref<
-              std::optional<ElementValueT>(ElementValueT, ElementValueT)>>
+              std::optional<ResultElementValueT>(ElementValueT, ElementValueT)>>
 Attribute constFoldBinaryOpConditional(ArrayRef<Attribute> operands,
                                        Type resultType,
                                        CalculationT &&calculate) {
@@ -65,7 +73,7 @@ Attribute constFoldBinaryOpConditional(ArrayRef<Attribute> operands,
     if (!calRes)
       return {};
 
-    return AttrElementT::get(resultType, *calRes);
+    return ResultAttrElementT::get(resultType, *calRes);
   }
 
   if (isa<SplatElementsAttr>(operands[0]) &&
@@ -99,7 +107,7 @@ Attribute constFoldBinaryOpConditional(ArrayRef<Attribute> operands,
       return {};
     auto lhsIt = *maybeLhsIt;
     auto rhsIt = *maybeRhsIt;
-    SmallVector<ElementValueT, 4> elementResults;
+    SmallVector<ResultElementValueT, 4> elementResults;
     elementResults.reserve(lhs.getNumElements());
     for (size_t i = 0, e = lhs.getNumElements(); i < e; ++i, ++lhsIt, ++rhsIt) {
       auto elementResult = calculate(*lhsIt, *rhsIt);
@@ -119,11 +127,13 @@ Attribute constFoldBinaryOpConditional(ArrayRef<Attribute> operands,
 /// attribute.
 /// Optional PoisonAttr template argument allows to specify 'poison' attribute
 /// which will be directly propagated to result.
-template <class AttrElementT,
+template <class AttrElementT, //
           class ElementValueT = typename AttrElementT::ValueType,
           class PoisonAttr = ub::PoisonAttr,
+          class ResultAttrElementT = AttrElementT,
+          class ResultElementValueT = typename ResultAttrElementT::ValueType,
           class CalculationT = function_ref<
-              std::optional<ElementValueT>(ElementValueT, ElementValueT)>>
+              std::optional<ResultElementValueT>(ElementValueT, ElementValueT)>>
 Attribute constFoldBinaryOpConditional(ArrayRef<Attribute> operands,
                                        CalculationT &&calculate) {
   assert(operands.size() == 2 && "binary op takes two operands");
@@ -139,64 +149,73 @@ Attribute constFoldBinaryOpConditional(ArrayRef<Attribute> operands,
       return operands[1];
   }
 
-  auto getResultType = [](Attribute attr) -> Type {
+  auto getAttrType = [](Attribute attr) -> Type {
     if (auto typed = dyn_cast_or_null<TypedAttr>(attr))
       return typed.getType();
     return {};
   };
 
-  Type lhsType = getResultType(operands[0]);
-  Type rhsType = getResultType(operands[1]);
+  Type lhsType = getAttrType(operands[0]);
+  Type rhsType = getAttrType(operands[1]);
   if (!lhsType || !rhsType)
     return {};
   if (lhsType != rhsType)
     return {};
 
   return constFoldBinaryOpConditional<AttrElementT, ElementValueT, PoisonAttr,
+                                      ResultAttrElementT, ResultElementValueT,
                                       CalculationT>(
       operands, lhsType, std::forward<CalculationT>(calculate));
 }
 
 template <class AttrElementT,
           class ElementValueT = typename AttrElementT::ValueType,
-          class PoisonAttr = void,
+          class PoisonAttr = void, //
+          class ResultAttrElementT = AttrElementT,
+          class ResultElementValueT = typename ResultAttrElementT::ValueType,
           class CalculationT =
-              function_ref<ElementValueT(ElementValueT, ElementValueT)>>
+              function_ref<ResultElementValueT(ElementValueT, ElementValueT)>>
 Attribute constFoldBinaryOp(ArrayRef<Attribute> operands, Type resultType,
                             CalculationT &&calculate) {
-  return constFoldBinaryOpConditional<AttrElementT, ElementValueT, PoisonAttr>(
+  return constFoldBinaryOpConditional<AttrElementT, ElementValueT, PoisonAttr,
+                                      ResultAttrElementT>(
       operands, resultType,
-      [&](ElementValueT a, ElementValueT b) -> std::optional<ElementValueT> {
-        return calculate(a, b);
-      });
+      [&](ElementValueT a, ElementValueT b)
+          -> std::optional<ResultElementValueT> { return calculate(a, b); });
 }
 
-template <class AttrElementT,
+template <class AttrElementT, //
           class ElementValueT = typename AttrElementT::ValueType,
           class PoisonAttr = ub::PoisonAttr,
+          class ResultAttrElementT = AttrElementT,
+          class ResultElementValueT = typename ResultAttrElementT::ValueType,
           class CalculationT =
-              function_ref<ElementValueT(ElementValueT, ElementValueT)>>
+              function_ref<ResultElementValueT(ElementValueT, ElementValueT)>>
 Attribute constFoldBinaryOp(ArrayRef<Attribute> operands,
                             CalculationT &&calculate) {
-  return constFoldBinaryOpConditional<AttrElementT, ElementValueT, PoisonAttr>(
+  return constFoldBinaryOpConditional<AttrElementT, ElementValueT, PoisonAttr,
+                                      ResultAttrElementT>(
       operands,
-      [&](ElementValueT a, ElementValueT b) -> std::optional<ElementValueT> {
-        return calculate(a, b);
-      });
+      [&](ElementValueT a, ElementValueT b)
+          -> std::optional<ResultElementValueT> { return calculate(a, b); });
 }
 
 /// Performs constant folding `calculate` with element-wise behavior on the one
 /// attributes in `operands` and returns the result if possible.
+/// Uses `resultType` for the type of the returned attribute.
 /// Optional PoisonAttr template argument allows to specify 'poison' attribute
 /// which will be directly propagated to result.
-template <class AttrElementT,
+template <class AttrElementT, //
           class ElementValueT = typename AttrElementT::ValueType,
           class PoisonAttr = ub::PoisonAttr,
+          class ResultAttrElementT = AttrElementT,
+          class ResultElementValueT = typename ResultAttrElementT::ValueType,
           class CalculationT =
-              function_ref<std::optional<ElementValueT>(ElementValueT)>>
+              function_ref<std::optional<ResultElementValueT>(ElementValueT)>>
 Attribute constFoldUnaryOpConditional(ArrayRef<Attribute> operands,
+                                      Type resultType,
                                       CalculationT &&calculate) {
-  if (!llvm::getSingleElement(operands))
+  if (!resultType || !llvm::getSingleElement(operands))
     return {};
 
   static_assert(
@@ -214,7 +233,7 @@ Attribute constFoldUnaryOpConditional(ArrayRef<Attribute> operands,
     auto res = calculate(op.getValue());
     if (!res)
       return {};
-    return AttrElementT::get(op.getType(), *res);
+    return ResultAttrElementT::get(resultType, *res);
   }
   if (isa<SplatElementsAttr>(operands[0])) {
     // Both operands are splats so we can avoid expanding the values out and
@@ -224,7 +243,7 @@ Attribute constFoldUnaryOpConditional(ArrayRef<Attribute> operands,
     auto elementResult = calculate(op.getSplatValue<ElementValueT>());
     if (!elementResult)
       return {};
-    return DenseElementsAttr::get(op.getType(), *elementResult);
+    return DenseElementsAttr::get(cast<ShapedType>(resultType), *elementResult);
   } else if (isa<ElementsAttr>(operands[0])) {
     // Operands are ElementsAttr-derived; perform an element-wise fold by
     // expanding the values.
@@ -234,7 +253,7 @@ Attribute constFoldUnaryOpConditional(ArrayRef<Attribute> operands,
     if (!maybeOpIt)
       return {};
     auto opIt = *maybeOpIt;
-    SmallVector<ElementValueT> elementResults;
+    SmallVector<ResultElementValueT> elementResults;
     elementResults.reserve(op.getNumElements());
     for (size_t i = 0, e = op.getNumElements(); i < e; ++i, ++opIt) {
       auto elementResult = calculate(*opIt);
@@ -242,19 +261,81 @@ Attribute constFoldUnaryOpConditional(ArrayRef<Attribute> operands,
         return {};
       elementResults.push_back(*elementResult);
     }
-    return DenseElementsAttr::get(op.getShapedType(), elementResults);
+    return DenseElementsAttr::get(cast<ShapedType>(resultType), elementResults);
   }
   return {};
 }
 
-template <class AttrElementT,
+/// Performs constant folding `calculate` with element-wise behavior on the one
+/// attributes in `operands` and returns the result if possible.
+/// Uses the operand element type for the element type of the returned
+/// attribute.
+/// Optional PoisonAttr template argument allows to specify 'poison' attribute
+/// which will be directly propagated to result.
+template <class AttrElementT, //
+          class ElementValueT = typename AttrElementT::ValueType,
+          class PoisonAttr = ub::PoisonAttr,
+          class ResultAttrElementT = AttrElementT,
+          class ResultElementValueT = typename ResultAttrElementT::ValueType,
+          class CalculationT =
+              function_ref<std::optional<ResultElementValueT>(ElementValueT)>>
+Attribute constFoldUnaryOpConditional(ArrayRef<Attribute> operands,
+                                      CalculationT &&calculate) {
+  if (!llvm::getSingleElement(operands))
+    return {};
+
+  static_assert(
+      std::is_void_v<PoisonAttr> || !llvm::is_incomplete_v<PoisonAttr>,
+      "PoisonAttr is undefined, either add a dependency on UB dialect or pass "
+      "void as template argument to opt-out from poison semantics.");
+  if constexpr (!std::is_void_v<PoisonAttr>) {
+    if (isa<PoisonAttr>(operands[0]))
+      return operands[0];
+  }
+
+  auto getAttrType = [](Attribute attr) -> Type {
+    if (auto typed = dyn_cast_or_null<TypedAttr>(attr))
+      return typed.getType();
+    return {};
+  };
+
+  Type operandType = getAttrType(operands[0]);
+  if (!operandType)
+    return {};
+
+  return constFoldUnaryOpConditional<AttrElementT, ElementValueT, PoisonAttr,
+                                     ResultAttrElementT, ResultElementValueT,
+                                     CalculationT>(
+      operands, operandType, std::forward<CalculationT>(calculate));
+}
+
+template <class AttrElementT, //
+          class ElementValueT = typename AttrElementT::ValueType,
+          class PoisonAttr = ub::PoisonAttr,
+          class ResultAttrElementT = AttrElementT,
+          class ResultElementValueT = typename ResultAttrElementT::ValueType,
+          class CalculationT = function_ref<ResultElementValueT(ElementValueT)>>
+Attribute constFoldUnaryOp(ArrayRef<Attribute> operands, Type resultType,
+                           CalculationT &&calculate) {
+  return constFoldUnaryOpConditional<AttrElementT, ElementValueT, PoisonAttr,
+                                     ResultAttrElementT>(
+      operands, resultType,
+      [&](ElementValueT a) -> std::optional<ResultElementValueT> {
+        return calculate(a);
+      });
+}
+
+template <class AttrElementT, //
           class ElementValueT = typename AttrElementT::ValueType,
           class PoisonAttr = ub::PoisonAttr,
-          class CalculationT = function_ref<ElementValueT(ElementValueT)>>
+          class ResultAttrElementT = AttrElementT,
+          class ResultElementValueT = typename ResultAttrElementT::ValueType,
+          class CalculationT = function_ref<ResultElementValueT(ElementValueT)>>
 Attribute constFoldUnaryOp(ArrayRef<Attribute> operands,
                            CalculationT &&calculate) {
-  return constFoldUnaryOpConditional<AttrElementT, ElementValueT, PoisonAttr>(
-      operands, [&](ElementValueT a) -> std::optional<ElementValueT> {
+  return constFoldUnaryOpConditional<AttrElementT, ElementValueT, PoisonAttr,
+                                     ResultAttrElementT>(
+      operands, [&](ElementValueT a) -> std::optional<ResultElementValueT> {
         return calculate(a);
       });
 }
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index 937b34a625628..59beac7d64154 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -1697,6 +1697,7 @@ def EmitC_GetFieldOp
   let arguments = (ins FlatSymbolRefAttr:$field_name);
   let results = (outs EmitCType:$result);
   let assemblyFormat = "$field_name `:` type($result) attr-dict";
+  let hasVerifier = 1;
 }
 
 #endif // MLIR_DIALECT_EMITC_IR_EMITC
diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index 2ed7d3810b918..a5c3a92f1b7a5 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -804,8 +804,8 @@ def GPU_LaunchOp : GPU_Op<"launch", [
                Optional<Index>:$clusterSizeY,
                Optional<Index>:$clusterSizeZ,
                Optional<I32>:$dynamicSharedMemorySize,
-               OptionalAttr<SymbolRefAttr>:$kernelFunc,
-               OptionalAttr<SymbolRefAttr>:$kernelModule)>,
+               OptionalAttr<FlatSymbolRefAttr>:$module,
+               OptionalAttr<FlatSymbolRefAttr>:$function)>,
     Results<(outs Optional<GPU_AsyncToken>:$asyncToken)> {
   let summary = "GPU kernel launch operation";
 
@@ -839,7 +839,7 @@ def GPU_LaunchOp : GPU_Op<"launch", [
     -   a variadic number of Workgroup memory attributions.
     -   a variadic number of Private memory attributions.
 
-    The `kernelFunc` and `kernelModule` attributes are optional and specifies
+    The `function` and `module` attributes are optional and specifies
     the kernel name and a module in which the kernel should be outlined.
 
     Syntax:
@@ -850,6 +850,8 @@ def GPU_LaunchOp : GPU_Op<"launch", [
                              `blocks` `(` ssa-id-list `)` `in` ssa-reassignment
                              `threads` `(` ssa-id-list `)` `in` ssa-reassignment
                              (dynamic_shared_memory_size ssa-use)?
+                             (`module(` symbol-ref-id `)`)?
+                             (`function(` symbol-ref-id `)`)?
                              memory-attribution
                              region attr-dict?
     ssa-reassignment ::= `(` ssa-id `=` ssa-use (`,` ssa-id `=` ssa-use)* `)`
@@ -907,6 +909,14 @@ def GPU_LaunchOp : GPU_Op<"launch", [
       // sizes are immediately usable inside body region.
       "some_op"(%cx, %bx, %tx) : (index, index, index) -> ()
     }
+
+    // Launch with module and function attributes.
+    gpu.launch blocks(%bx, %by, %bz) in (%sz_bx = %0, %sz_by = %1, %sz_bz = %2)
+               threads(%tx, %ty, %tz) in (%sz_tx = %3, %sz_ty = %4, %sz_tz = %5)
+               module(@kernel_module) function(@kernel_func) {
+      "some_op"(%bx, %tx) : (index, index) -> ()
+      %42 = load %val1[%bx] : memref<?xf32, 1>
+    }
     ```
 
     Rationale: using operation/block arguments gives analyses a clear way of
@@ -931,7 +941,9 @@ def GPU_LaunchOp : GPU_Op<"launch", [
       CArg<"TypeRange", "{}">:$privateAttributions,
       CArg<"Value", "nullptr">:$clusterSizeX,
       CArg<"Value", "nullptr">:$clusterSizeY,
-      CArg<"Value", "nullptr">:$clusterSizeZ)>
+      CArg<"Value", "nullptr">:$clusterSizeZ,
+      CArg<"FlatSymbolRefAttr", "nullptr">:$module,
+      CArg<"FlatSymbolRefAttr", "nullptr">:$function)>,
   ];
 
   let extraClassDeclaration = [{
@@ -3197,6 +3209,9 @@ def GPU_WarpExecuteOnLane0Op : GPU_Op<"warp_execute_on_lane_0",
     bool isDefinedOutsideOfRegion(Value value) {
       return !getRegion().isAncestor(value.getParentRegion());
     }
+
+    /// Get the terminator of the warp region.
+    gpu::YieldOp getTerminator();
   }];
 }
 
diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
index 87423c639945f..3a8caf8aa42e2 100644
--- a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
+++ b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
@@ -331,7 +331,10 @@ def ApplyGPUPromoteShuffleToAMDGPUPatternsOp : Op<Transform_Dialect,
     Collects patterns that are tryin to promote `gpu.shuffle`s to specialized
     AMDGPU intrinsics.
   }];
-  let assemblyFormat = "attr-dict";
+  let arguments = (ins OptionalAttr<StrAttr>:$chipset);
+  let assemblyFormat = [{
+    (`chipset` `=` $chipset^)? attr-dict
+  }];
 }
 
 
diff --git a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h
index b4fd55e6c22e6..d5c253d6c9c08 100644
--- a/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/GPU/Transforms/Passes.h
@@ -114,7 +114,8 @@ void populateGpuDecomposeMemrefsPatterns(RewritePatternSet &patterns);
 void populateGpuEliminateBarriersPatterns(RewritePatternSet &patterns);
 
 /// Tries to promote `gpu.shuffle`s to specialized AMDGPU intrinsics.
-void populateGpuPromoteShuffleToAMDGPUPatterns(RewritePatternSet &patterns);
+void populateGpuPromoteShuffleToAMDGPUPatterns(
+    RewritePatternSet &patterns, std::optional<amdgpu::Chipset> maybeChipset);
 
 /// Generate the code for registering passes.
 #define GEN_PASS_REGISTRATION
diff --git a/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h b/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h
index 5b7df69b62e0b..21331e5aa89f3 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h
@@ -26,13 +26,26 @@ namespace NVVM {
 enum class PTXRegisterMod {
   /// Read register with no modifier
   Read = 0,
-  /// Read register with '+' modifier
+  /// Write register with '=' modifier
   Write = 2,
-  /// Read register with '=' modifier.
-  /// Note that, this is not natively supported by LLVM, but it is possible to
-  /// set read and write for the same operand.
+  /// ReadWrite register with '+' modifier.
+  /// Note that, this is not natively supported by LLVM, the Interface does
+  /// mapping
   ReadWrite = 1,
 };
+
+inline llvm::raw_ostream &operator<<(llvm::raw_ostream &os,
+                                     PTXRegisterMod mod) {
+  switch (mod) {
+  case PTXRegisterMod::Read:
+    return os << "Read";
+  case PTXRegisterMod::Write:
+    return os << "Write";
+  case PTXRegisterMod::ReadWrite:
+    return os << "ReadWrite";
+  }
+  llvm_unreachable("Unknown PTXRegisterMod value");
+}
 } // namespace NVVM
 } // namespace mlir
 
@@ -54,13 +67,19 @@ class PtxBuilder {
   SmallVector<Value> ptxOperands;
   // Register constraints (read, write, readwrite) and register data types
   std::string registerConstraints;
-
+  // Modifiers
+  SmallVector<PTXRegisterMod> registerModifiers;
+  // Has return value as write-only or read-write
   bool hasResult = false;
+  // Indicates if the Op will handle the register mapping manually.
+  bool needsManualRegisterMapping = false;
 
 public:
   /// Single constructor that only initializes members.
-  PtxBuilder(Operation *op, PatternRewriter &rewriter)
-      : interfaceOp(op), rewriter(rewriter) {}
+  PtxBuilder(Operation *op, PatternRewriter &rewriter,
+             bool needsManualRegisterMapping = false)
+      : interfaceOp(op), rewriter(rewriter),
+        needsManualRegisterMapping(needsManualRegisterMapping) {}
 
   /// Add an operand with the read/write input type.
   void insertValue(Value v, PTXRegisterMod itype = PTXRegisterMod::Read);
@@ -74,6 +93,16 @@ class PtxBuilder {
   void buildAndReplaceOp();
 };
 
+/// Count the number of placeholder variables such as {$r}, {$w}, {$rw} in the
+/// PTX code.
+void countPlaceholderNumbers(StringRef ptxCode,
+                             llvm::SmallDenseSet<unsigned> &seenRW,
+                             llvm::SmallDenseSet<unsigned> &seenW,
+                             llvm::SmallDenseSet<unsigned> &seenR,
+                             llvm::SmallVectorImpl<unsigned> &rwNums,
+                             llvm::SmallVectorImpl<unsigned> &wNums,
+                             llvm::SmallVectorImpl<unsigned> &rNums);
+
 } // namespace NVVM
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td b/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td
index e98b94b5b3052..086cdccb01221 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.td
@@ -124,19 +124,21 @@ def BasicPtxBuilderOpInterface : OpInterface<"BasicPtxBuilderInterface"> {
             following this order:
              1) Adds results 
              2) Adds operands 
-             3) Adds attributes             
+             3) Adds attributes
+             Returns true if the OP is going to do register mapping itself
           }],
-         /*retType=*/"void",
+         /*retType=*/"bool",
          /*methodName=*/"getAsmValues",
          /*args=*/(ins "::mlir::RewriterBase &":$rewriter, 
-         "llvm::SmallVectorImpl<std::pair<mlir::Value, mlir::NVVM::PTXRegisterMod>>&" : $asmValues),
+         "llvm::SmallVectorImpl<std::pair<mlir::Value, mlir::NVVM::PTXRegisterMod>>&" : $asmValues         
+         ),
          /*methodBody=*/"",
          /*defaultImpl=*/ [{         
            mlir::Operation* op = $_op;
            
            // Step 1. Add results
-           for (auto val : op->getResults()) 
-            asmValues.push_back({val, mlir::NVVM::PTXRegisterMod::Write});
+            for (auto val : op->getResults()) 
+              asmValues.push_back({val, mlir::NVVM::PTXRegisterMod::Write});
 
            // Step 2. Add operands
            for (auto val : op->getOperands()) 
@@ -149,6 +151,7 @@ def BasicPtxBuilderOpInterface : OpInterface<"BasicPtxBuilderInterface"> {
              asmValues.push_back({val, mlir::NVVM::PTXRegisterMod::Read});
              }
            }
+           return false; // No manual mapping needed
          }]
        >
   ];
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index 790d2e77ea874..138dd7703a5e7 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -13,6 +13,7 @@ include "mlir/Dialect/LLVMIR/LLVMDialect.td"
 include "mlir/Dialect/LLVMIR/LLVMInterfaces.td"
 include "mlir/IR/AttrTypeBase.td"
 include "mlir/IR/CommonAttrConstraints.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
 
 // All of the attributes will extend this class.
 class LLVM_Attr<string name, string attrMnemonic,
@@ -1304,6 +1305,34 @@ def LLVM_TargetFeaturesAttr : LLVM_Attr<"TargetFeatures", "target_features">
   let genVerifyDecl = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// LLVM_TargetAttr
+//===----------------------------------------------------------------------===//
+
+def LLVM_TargetAttr : LLVM_Attr<"Target", "target",
+                                [LLVM_TargetAttrInterface]> {
+  let summary = "LLVM target info: triple, chip, features";
+  let description = [{
+    An attribute to hold LLVM target information, specifying LLVM's target
+    `triple` string, the target `chip` string (i.e. the `cpu` string), and
+    target `features` string as an attribute. The latter is optional.
+
+    Responds to DLTI-queries on the keys:
+      * A query for `"triple"` returns the `StringAttr` for the `triple`.
+      * A query for `"chip"` returns the `StringAttr` for the `chip`/`cpu`.
+      * A query for `"features"` returns the `StringAttr`, if provided.
+  }];
+  let parameters = (ins "StringAttr":$triple,
+                        "StringAttr":$chip,
+                        OptionalParameter<"StringAttr", "">:$features);
+
+  let assemblyFormat = [{`<` struct($triple, $chip, $features) `>`}];
+
+  let extraClassDeclaration = [{
+    FailureOr<Attribute> query(DataLayoutEntryKey key);
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // UndefAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
index 3ede857733242..1ceeb7e4ba2a5 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrs.h
@@ -16,6 +16,7 @@
 
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/OpImplementation.h"
+#include "mlir/Interfaces/DataLayoutInterfaces.h"
 #include <optional>
 
 #include "mlir/Dialect/LLVMIR/LLVMOpsEnums.h.inc"
@@ -89,8 +90,8 @@ class TBAANodeAttr : public Attribute {
 // TODO: this shouldn't be needed after we unify the attribute generation, i.e.
 // --gen-attr-* and --gen-attrdef-*.
 using cconv::CConv;
-using tailcallkind::TailCallKind;
 using linkage::Linkage;
+using tailcallkind::TailCallKind;
 } // namespace LLVM
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
index 107bf3edb657a..ab0462f945a33 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMDialect.td
@@ -27,6 +27,7 @@ def LLVM_Dialect : Dialect {
   );
 
   let extraClassDeclaration = [{
+    static StringRef getTargetAttrName() { return "llvm.target"; }
     /// Name of the data layout attributes.
     static StringRef getDataLayoutAttrName() { return "llvm.data_layout"; }
     static StringRef getNoAliasScopesAttrName() { return "noalias_scopes"; }
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
index 138170f8c8762..64600e86bedfb 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td
@@ -14,6 +14,7 @@
 #define LLVMIR_INTERFACES
 
 include "mlir/IR/OpBase.td"
+include "mlir/Interfaces/DataLayoutInterfaces.td"
 
 def FastmathFlagsInterface : OpInterface<"FastmathFlagsInterface"> {
   let description = [{
@@ -532,4 +533,39 @@ def LLVM_DIRecursiveTypeAttrInterface
   ];
 }
 
+def LLVM_TargetAttrInterface
+  : AttrInterface<"TargetAttrInterface", [DLTIQueryInterface]> {
+  let description = [{
+    Interface for attributes that describe LLVM targets.
+
+    These attributes should be able to return the specified target `triple`,
+    `chip` and `features`.
+
+    Implementing attributes should provide a `DLTIQueryInterface::query()`
+    implementation which responds to keys `"triple"`, `"chip"` and `"features"`
+    by returning appropriate `StringAttr`s.
+  }];
+  let cppNamespace = "::mlir::LLVM";
+  let methods = [
+    InterfaceMethod<
+      /*description=*/"Returns the target triple identifier.",
+      /*retTy=*/"StringAttr",
+      /*methodName=*/"getTriple",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the target chip (i.e. \"cpu\") identifier.",
+      /*retTy=*/"StringAttr",
+      /*methodName=*/"getChip",
+      /*args=*/(ins)
+    >,
+    InterfaceMethod<
+      /*description=*/"Returns the target features as a string.",
+      /*retTy=*/"StringAttr",
+      /*methodName=*/"getFeatures",
+      /*args=*/(ins)
+    >
+  ];
+}
+
 #endif // LLVMIR_INTERFACES
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index d38298fcb2a38..5e2461b4508e4 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -87,21 +87,21 @@ class LLVM_TernarySameArgsIntrOpF<string func, list<Trait> traits = []> :
 class LLVM_CountZerosIntrOp<string func, list<Trait> traits = []> :
     LLVM_OneResultIntrOp<func, [], [0],
            !listconcat([Pure, SameOperandsAndResultType], traits),
-            /*requiresFastmath=*/0,
+            /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
             /*immArgPositions=*/[1], /*immArgAttrNames=*/["is_zero_poison"]> {
   let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
                    I1Attr:$is_zero_poison);
 }
 
 def LLVM_AbsOp : LLVM_OneResultIntrOp<"abs", [], [0], [Pure],
-    /*requiresFastmath=*/0,
+    /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
     /*immArgPositions=*/[1], /*immArgAttrNames=*/["is_int_min_poison"]> {
   let arguments = (ins LLVM_ScalarOrVectorOf<AnySignlessInteger>:$in,
                    I1Attr:$is_int_min_poison);
 }
 
 def LLVM_IsFPClass : LLVM_OneResultIntrOp<"is.fpclass", [], [0], [Pure],
-  /*requiresFastmath=*/0,
+  /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
   /*immArgPositions=*/[1], /*immArgAttrNames=*/["bit"]> {
   let arguments = (ins LLVM_ScalarOrVectorOf<LLVM_AnyFloat>:$in, I32Attr:$bit);
 }
@@ -348,15 +348,11 @@ def LLVM_PtrMaskOp
 // Memory marker intrinsics.
 //
 
-/// Base operation for lifetime markers. The LLVM intrinsics require the size
-/// operand to be an immediate. In MLIR it is encoded as an attribute.
-class LLVM_LifetimeBaseOp<string opName> : LLVM_ZeroResultIntrOp<opName, [1],
-    [DeclareOpInterfaceMethods<PromotableOpInterface>],
-    /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
-    /*requiresArgAndResultAttrs=*/0, /*requiresOpBundles=*/0,
-    /*immArgPositions=*/[0], /*immArgAttrNames=*/["size"]> {
-  let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
-  let assemblyFormat = "$size `,` $ptr attr-dict `:` qualified(type($ptr))";
+/// Base operation for lifetime markers.
+class LLVM_LifetimeBaseOp<string opName> : LLVM_ZeroResultIntrOp<opName, [0],
+    [DeclareOpInterfaceMethods<PromotableOpInterface>]> {
+  let arguments = (ins LLVM_AnyPointer:$ptr);
+  let assemblyFormat = "$ptr attr-dict `:` qualified(type($ptr))";
 }
 
 def LLVM_LifetimeStartOp : LLVM_LifetimeBaseOp<"lifetime.start">;
@@ -364,8 +360,8 @@ def LLVM_LifetimeEndOp : LLVM_LifetimeBaseOp<"lifetime.end">;
 
 def LLVM_InvariantStartOp : LLVM_OneResultIntrOp<"invariant.start", [], [1],
     [DeclareOpInterfaceMethods<PromotableOpInterface>],
-    /*requiresFastmath=*/0, /*immArgPositions=*/[0],
-    /*immArgAttrNames=*/["size"]> {
+    /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
+    /*immArgPositions=*/[0], /*immArgAttrNames=*/["size"]> {
   let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
   let results = (outs LLVM_DefaultPointer:$res);
   let assemblyFormat = "$size `,` $ptr attr-dict `:` qualified(type($ptr))";
@@ -416,6 +412,7 @@ class LLVM_ConstrainedIntr<string mnem, int numArgs,
                                !gt(hasRoundingMode, 0) : [DeclareOpInterfaceMethods<RoundingModeOpInterface>],
                                true : []),
                            /*requiresFastmath=*/0,
+                           /*requiresArgAndResultAttrs=*/0,
                            /*immArgPositions=*/[],
                            /*immArgAttrNames=*/[]> {
   dag regularArgs = !dag(ins, !listsplat(LLVM_Type, numArgs), !foreach(i, !range(numArgs), "arg_" #i));
@@ -593,7 +590,7 @@ def LLVM_ExpectOp
 def LLVM_ExpectWithProbabilityOp
   : LLVM_OneResultIntrOp<"expect.with.probability", [], [0],
                          [Pure, AllTypesMatch<["val", "expected", "res"]>],
-                         /*requiresFastmath=*/0,
+                         /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
                          /*immArgPositions=*/[2], /*immArgAttrNames=*/["prob"]> {
   let arguments = (ins AnySignlessInteger:$val,
                        AnySignlessInteger:$expected,
@@ -829,7 +826,7 @@ class LLVM_VecReductionAccBase<string mnem, Type element>
                            /*overloadedResults=*/[],
                            /*overloadedOperands=*/[1],
                            /*traits=*/[Pure, SameOperandsAndResultElementType],
-                           /*equiresFastmath=*/1>,
+                           /*requiresFastmath=*/1>,
       Arguments<(ins element:$start_value,
                      LLVM_VectorOf<element>:$input,
                      DefaultValuedAttr<LLVM_FastmathFlagsAttr, "{}">:$fastmathFlags)>;
@@ -1073,14 +1070,36 @@ def LLVM_masked_scatter : LLVM_ZeroResultIntrOp<"masked.scatter"> {
 }
 
 /// Create a call to Masked Expand Load intrinsic.
-def LLVM_masked_expandload : LLVM_IntrOp<"masked.expandload", [0], [], [], 1> {
-  let arguments = (ins LLVM_AnyPointer, LLVM_VectorOf<I1>, LLVM_AnyVector);
+def LLVM_masked_expandload
+  : LLVM_OneResultIntrOp<"masked.expandload", [0], [],
+  /*traits=*/[], /*requiresFastMath=*/0, /*requiresArgAndResultAttrs=*/1,
+  /*immArgPositions=*/[], /*immArgAttrNames=*/[]> {
+  dag args = (ins LLVM_AnyPointer:$ptr,
+                       LLVM_VectorOf<I1>:$mask,
+                       LLVM_AnyVector:$passthru);
+
+  let arguments = !con(args, baseArgs);
+
+  let builders = [
+    OpBuilder<(ins "TypeRange":$resTy, "Value":$ptr, "Value":$mask, "Value":$passthru, CArg<"uint64_t", "1">:$align)>
+  ];
 }
 
 /// Create a call to Masked Compress Store intrinsic.
 def LLVM_masked_compressstore
-    : LLVM_IntrOp<"masked.compressstore", [], [0], [], 0> {
-  let arguments = (ins LLVM_AnyVector, LLVM_AnyPointer, LLVM_VectorOf<I1>);
+    : LLVM_ZeroResultIntrOp<"masked.compressstore", [0],
+  /*traits=*/[], /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
+  /*requiresArgAndResultAttrs=*/1, /*requiresOpBundles=*/0,
+  /*immArgPositions=*/[], /*immArgAttrNames=*/[]> {
+  dag args = (ins LLVM_AnyVector:$value,
+                       LLVM_AnyPointer:$ptr,
+                       LLVM_VectorOf<I1>:$mask);
+
+  let arguments = !con(args, baseArgs);
+
+  let builders = [
+    OpBuilder<(ins "Value":$value, "Value":$ptr, "Value":$mask, CArg<"uint64_t", "1">:$align)>
+  ];
 }
 
 //
@@ -1159,7 +1178,7 @@ def LLVM_vector_insert
                   PredOpTrait<"it is not inserting scalable into fixed-length vectors.",
                     CPred<"!isScalableVectorType($srcvec.getType()) || "
                           "isScalableVectorType($dstvec.getType())">>],
-                  /*requiresFastmath=*/0,
+                  /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
                   /*immArgPositions=*/[2], /*immArgAttrNames=*/["pos"]> {
   let arguments = (ins LLVM_AnyVector:$dstvec, LLVM_AnyVector:$srcvec,
                        I64Attr:$pos);
@@ -1193,7 +1212,7 @@ def LLVM_vector_extract
                   PredOpTrait<"it is not extracting scalable from fixed-length vectors.",
                     CPred<"!isScalableVectorType($res.getType()) || "
                           "isScalableVectorType($srcvec.getType())">>],
-                  /*requiresFastmath=*/0,
+                  /*requiresFastmath=*/0, /*requiresArgAndResultAttrs=*/0,
                   /*immArgPositions=*/[1], /*immArgAttrNames=*/["pos"]> {
   let arguments = (ins LLVM_AnyVector:$srcvec, I64Attr:$pos);
   let results = (outs LLVM_AnyVector:$res);
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index a8d7cf2069547..d6aa9580870a8 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -475,11 +475,12 @@ class LLVM_OneResultIntrOp<string mnem, list<int> overloadedResults = [],
                            list<int> overloadedOperands = [],
                            list<Trait> traits = [],
                            bit requiresFastmath = 0,
+                           bit requiresArgAndResultAttrs = 0,
                            list<int> immArgPositions = [],
                            list<string> immArgAttrNames = []>
     : LLVM_IntrOp<mnem, overloadedResults, overloadedOperands, traits, 1,
                   /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
-                  requiresFastmath, /*requiresArgAndResultAttrs=*/0,
+                  requiresFastmath, requiresArgAndResultAttrs,
                   /*requiresOpBundles=*/0, immArgPositions,
                   immArgAttrNames>;
 
diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 30df3b739e5ca..85e6e064f22a9 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -83,6 +83,15 @@ def NVVM_Dialect : Dialect {
     /// are grid constants.
     static StringRef getGridConstantAttrName() { return "nvvm.grid_constant"; }
 
+    /// Get the name of the attribute used to annotate the `.blocksareclusters`
+    /// PTX directive for kernel functions.
+    /// This attribute implies that the grid launch configuration for the
+    /// corresponding kernel function is specifying the number of clusters
+    /// instead of the number of thread blocks. This attribute is only
+    /// allowed for kernel functions and requires nvvm.reqntid and
+    /// nvvm.cluster_dim attributes.
+    static StringRef getBlocksAreClustersAttrName() { return "nvvm.blocksareclusters"; }
+
     /// Verify an attribute from this dialect on the argument at 'argIndex' for
     /// the region at 'regionIndex' on the given operation. Returns failure if
     /// the verification failed, success otherwise. This hook may optionally be
@@ -258,6 +267,7 @@ def NVVM_ClusterDim : NVVM_PureSpecialRangeableRegisterOp<"read.ptx.sreg.cluster
 def NVVM_ClockOp : NVVM_SpecialRegisterOp<"read.ptx.sreg.clock">;
 def NVVM_Clock64Op : NVVM_SpecialRegisterOp<"read.ptx.sreg.clock64">;
 def NVVM_GlobalTimerOp : NVVM_SpecialRegisterOp<"read.ptx.sreg.globaltimer">;
+def NVVM_GlobalTimerLoOp : NVVM_SpecialRegisterOp<"read.ptx.sreg.globaltimer.lo">;
 
 //===----------------------------------------------------------------------===//
 // envreg registers
@@ -315,16 +325,19 @@ def NVVM_InlinePtxOp : NVVM_Op<"inline_ptx",
   }];
 
   let arguments = (ins Variadic<AnyType>:$readOnlyArgs, 
+                       Variadic<AnyType>:$readWriteArgs, 
                        StrAttr:$ptxCode,
                        PtxPredicate:$predicate);
                       
   let results = (outs Variadic<AnyType>:$writeOnlyArgs);
-
-  let assemblyFormat = [{ 
-    $ptxCode `(` $readOnlyArgs `)` 
-    (`,` `predicate` `=` $predicate^)? attr-dict 
-    `:` type(operands) 
-    (`->` type($writeOnlyArgs)^)?
+  
+  let assemblyFormat = [{
+    $ptxCode
+    ( `ro` `(` $readOnlyArgs^ `:` type($readOnlyArgs) `)` )?
+    ( `rw` `(` $readWriteArgs^ `:` type($readWriteArgs) `)` )?
+    (`,` `predicate` `=` $predicate^)? 
+    attr-dict
+    ( `->` type($writeOnlyArgs)^ )?
   }];
   
   let extraClassDefinition = [{
@@ -333,6 +346,10 @@ def NVVM_InlinePtxOp : NVVM_Op<"inline_ptx",
       return std::string(ptxInstStr.data());
     }
   }];
+
+  let extraClassDeclaration = [{
+    bool getAsmValues(RewriterBase &, llvm::SmallVectorImpl<std::pair<mlir::Value, mlir::NVVM::PTXRegisterMod>> &);
+  }];
 }
 
 //===----------------------------------------------------------------------===//
@@ -401,6 +418,74 @@ def NVVM_ReduxOp :
    }];   
 }
 
+//===----------------------------------------------------------------------===//
+// NVVM nanosleep
+//===----------------------------------------------------------------------===//
+
+def NVVM_NanosleepOp : NVVM_Op<"nanosleep">,
+  Arguments<(ins 
+  ConfinedAttr<I32Attr, [IntMinValue<1>, IntMaxValue<1000000>]>:$duration)> 
+{
+  let summary = "Suspends the thread for a specified duration.";
+
+  let description = [{
+    The op suspends the thread for a sleep duration approximately close to the 
+    delay `$duration`, specified in nanoseconds. 
+
+    The sleep duration is approximated, but guaranteed to be in the 
+    interval [0, 2*t]. The maximum sleep duration is 1 millisecond. 
+    The implementation may reduce the sleep duration for individual threads 
+    within a warp such that all sleeping threads in the warp wake up together.
+
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-nanosleep)
+  }];
+  
+  string llvmBuilder = [{
+      createIntrinsicCall(builder, 
+                          llvm::Intrinsic::nvvm_nanosleep, 
+                          {builder.getInt32($duration)});
+  }];
+  let assemblyFormat = "attr-dict $duration";
+}
+
+//===----------------------------------------------------------------------===//
+// NVVM Performance Monitor events
+//===----------------------------------------------------------------------===//
+
+def NVVM_PMEventOp : NVVM_PTXBuilder_Op<"pmevent">,
+  Arguments<(ins OptionalAttr<I16Attr>:$maskedEventId, 
+                 OptionalAttr<I32Attr>:$eventId)> {
+  let summary = "Trigger one or more Performance Monitor events.";
+
+  let description = [{
+    Triggers one or more of a fixed number of performance monitor events, with
+    event index or mask specified by immediate operand.
+
+    Without `mask` it triggers a single performance monitor event indexed by
+    immediate operand a, in the range 0..15.
+
+    With `mask` it triggers one or more of the performance monitor events. Each
+    bit in the 16-bit immediate operand controls an event.
+
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-pmevent)
+  }];
+  
+  string llvmBuilder = [{
+      llvm::Value *mId = builder.getInt16(* $maskedEventId);
+      createIntrinsicCall(builder, llvm::Intrinsic::nvvm_pm_event_mask, {mId});
+  }];
+
+  let assemblyFormat = "attr-dict (`id` `=` $eventId^)? (`mask` `=` $maskedEventId^)?";
+
+  let extraClassDeclaration = [{
+    bool hasIntrinsic() { return !getEventId(); }
+  }];
+  let extraClassDefinition = [{
+    std::string $cppClass::getPtx() { return std::string("pmevent %0;"); }
+  }];
+  let hasVerifier = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // NVVM Split arrive/wait barrier
 //===----------------------------------------------------------------------===//
@@ -2032,13 +2117,16 @@ def NVVM_StMatrixOp: NVVM_Op<"stmatrix">,
 
 def NVVM_LdMatrixOp: NVVM_Op<"ldmatrix">,
   Results<(outs AnyType:$res)>,
-  Arguments<(ins LLVM_AnyPointer: $ptr, I32Attr:$num, MMALayoutAttr:$layout)> {
+  Arguments<(ins LLVM_PointerShared:$ptr, I32Attr:$num,
+                 MMALayoutAttr:$layout,
+                 LdStMatrixShapeAttr:$shape,
+                 LdStMatrixEltTypeAttr:$eltType)> {
 
   let summary = "cooperative matrix load";
 
   string llvmBuilder = [{
       auto operands = moduleTranslation.lookupValues(opInst.getOperands());
-      auto intId = getLdMatrixIntrinsicId($layout, $num);
+      auto intId = getLdMatrixIntrinsicId($layout, $num, $shape, $eltType);
       $res = createIntrinsicCall(builder, intId, operands, {operands[0]->getType()});
   }];
 
@@ -2215,6 +2303,56 @@ def NVVM_MmaOp : NVVM_Op<"mma.sync", [AttrSizedOperandSegments]> {
 // NVVM TMA Ops
 //===----------------------------------------------------------------------===//
 
+// List of modes supported for TMA Load and Prefetch Ops
+def TMALoadModeTile   : I32EnumAttrCase<"TILE", 0, "tile">;
+def TMALoadModeIm2Col : I32EnumAttrCase<"IM2COL", 1, "im2col">;
+def TMALoadModeIm2ColW : I32EnumAttrCase<"IM2COL_W", 2, "im2col_w">;
+def TMALoadModeIm2ColW128 : I32EnumAttrCase<"IM2COL_W_128", 3, "im2col_w_128">;
+def TMALoadModeTileGather4 : I32EnumAttrCase<"TILE_GATHER4", 4, "tile_gather4">;
+
+def TMALoadMode : I32EnumAttr<"TMALoadMode", "NVVM TMA Load Mode",
+    [TMALoadModeTile, TMALoadModeIm2Col,
+     TMALoadModeIm2ColW, TMALoadModeIm2ColW128,
+     TMALoadModeTileGather4]> {
+  let genSpecializedAttr = 0;
+  let cppNamespace = "::mlir::NVVM";
+}
+def TMALoadModeAttr : EnumAttr<NVVM_Dialect, TMALoadMode, "tma_load_mode"> {
+  let summary = "List of Load-Modes supported for TMA Tensor Ops";
+  let description = [{
+    TMA Tensor Ops support the following modes, when copying data from
+    global memory to shared memory (i.e. load):
+
+    Tile Mode: It's the default mode. The source multi-dimensional tensor
+    layout is preserved at the destination.
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-tiled-mode)
+
+    Im2col Mode: This mode is used when `im2colOffsets` operands are present.
+    The elements in the Bounding Box of the source tensor are rearranged into
+    columns at the destination. In this mode, the tensor has to be at least
+    3-dimensional. The number of `im2colOffsets` is `dims - 2` where `dims`
+    is the dimension of the tensor.
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-im2col-mode)
+
+    Im2col_W Mode: This mode is similar to Im2Col mode with the restriction that
+    elements are accessed across the W dimension only. The number of `im2colOffsets`
+    are always two, referred as `wHalo` and `wOffset`.
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-im2col-w-w128-modes)
+
+    Im2col_W_128 Mode: This mode is similar to Im2Col_W mode with the number of
+    elements accessed across the W dimension is always 128 only.
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-im2col-w-w128-modes)
+
+    Tile_Gather4 Mode: This mode is similar to Tile mode but works only on 2D tensor.
+    In gather4 mode, four rows in the source 2D tensor are combined to form a single
+    2D tensor at the destination. This mode requires five co-ordinates. The first one
+    represents the column-index followed by four row indices.
+    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensor-tiled-scatter4-gather4-modes)
+  }];
+
+  let assemblyFormat = "`<` $value `>`";
+}
+
 def NVVM_CpAsyncBulkCommitGroupOp : NVVM_Op<"cp.async.bulk.commit.group">,
   Arguments<(ins )> {
   let assemblyFormat = "attr-dict";
@@ -2483,23 +2621,16 @@ def NVVM_CpAsyncBulkPrefetchOp : NVVM_Op<"cp.async.bulk.prefetch"> {
 def NVVM_CpAsyncBulkTensorPrefetchOp :
   NVVM_Op<"cp.async.bulk.tensor.prefetch", [AttrSizedOperandSegments]> {
   let arguments = (ins
-    LLVM_AnyPointer:$tmaDescriptor,
+    LLVM_PointerGeneric:$tmaDescriptor,
     Variadic<I32>:$coordinates,
     Variadic<I16>:$im2colOffsets,
+    DefaultValuedAttr<TMALoadModeAttr, "TMALoadMode::TILE">:$mode,
     Optional<I64>:$l2CacheHint);
 
   let description = [{
     Initiates an asynchronous prefetch operation on the tensor data from global
-    memory to L2 cache.
-
-    The Op has two modes:
-    1) Tiled Mode: It's the default mode. The source multi-dimensional tensor
-    layout is preserved at the destination.
-
-    2) Im2col Mode: This mode is used when `im2colOffsets` operands are present.
-    the elements in the Bounding Box of the source tensor are rearranged into
-    columns at the destination. In this mode, the tensor has to be at least
-    3-dimensional.
+    memory to L2 cache. This Op supports all the load modes specified in
+    `TMALoadMode`.
 
     The `l2CacheHint` operand is optional, and it is used to specify cache
     eviction policy that may be used during the memory access.
@@ -2516,34 +2647,17 @@ def NVVM_CpAsyncBulkTensorPrefetchOp :
   }];
 
   let extraClassDeclaration = [{
-    static llvm::Intrinsic::ID getIntrinsicID(int tensorDims, bool isIm2Col);
+    static mlir::NVVM::IDArgPair
+    getIntrinsicIDAndArgs(Operation &op, LLVM::ModuleTranslation &mt,
+                          llvm::IRBuilderBase& builder);
   }];
 
   let hasVerifier = 1;
 
   string llvmBuilder = [{
-    // Arguments to the intrinsic:
-    // tmaDesc, tensorDims, im2colOffsets
-    // cache_hint(if applicable) and flag(boolean)
-    llvm::SmallVector<llvm::Value *> translatedOperands;
-    translatedOperands.push_back($tmaDescriptor);
-
-    for (auto v : op.getCoordinates())
-      translatedOperands.push_back(moduleTranslation.lookupValue(v));
-
-    for (auto v : op.getIm2colOffsets())
-      translatedOperands.push_back(moduleTranslation.lookupValue(v));
-
-    llvm::LLVMContext &ctx = moduleTranslation.getLLVMContext();
-    auto *i64Unused = llvm::ConstantInt::get(llvm::Type::getInt64Ty(ctx), 0);
-
-    bool isCacheHint = op.getL2CacheHint() ? true : false;
-    translatedOperands.push_back(isCacheHint ? $l2CacheHint : i64Unused);
-    translatedOperands.push_back(builder.getInt1(isCacheHint));
-
-    auto intId = NVVM::CpAsyncBulkTensorPrefetchOp::getIntrinsicID(
-        op.getCoordinates().size(), op.getIm2colOffsets().size() > 0);
-    createIntrinsicCall(builder, intId, translatedOperands);
+    auto [id, args] = NVVM::CpAsyncBulkTensorPrefetchOp::getIntrinsicIDAndArgs(
+                      *op, moduleTranslation, builder);
+    createIntrinsicCall(builder, id, args);
   }];
 }
 
@@ -2986,8 +3100,7 @@ def NVVM_WgmmaMmaAsyncOp : NVVM_Op<"wgmma.mma_async",
   let hasVerifier = 1;
 
   let extraClassDeclaration = [{
-    void getAsmValues(RewriterBase &rewriter, 
-        llvm::SmallVectorImpl<std::pair<mlir::Value, mlir::NVVM::PTXRegisterMod>> &asmValues);
+    bool getAsmValues(RewriterBase &, llvm::SmallVectorImpl<std::pair<mlir::Value, mlir::NVVM::PTXRegisterMod>> &);
   }];
 }
 
@@ -2995,30 +3108,46 @@ def NVVM_WgmmaMmaAsyncOp : NVVM_Op<"wgmma.mma_async",
 // NVVM Griddepcontrol Ops
 //===----------------------------------------------------------------------===//
 
-def NVVM_GriddepcontrolWaitOp : NVVM_IntrOp<"griddepcontrol.wait", [], 0> {
-  let assemblyFormat = "attr-dict";
+def GridDepActionWait : I32EnumCase<"wait", 0>;
+def GridDepActionLaunchDependent : I32EnumCase<"launch_dependents", 1>;
 
+def GridDepActionKind : I32Enum<"GridDepActionKind", "Action kind for grid dependency control",
+  [GridDepActionWait, GridDepActionLaunchDependent]> {
+  let cppNamespace = "::mlir::NVVM";
+}
+
+def GridDepActionAttr : EnumAttr<NVVM_Dialect, GridDepActionKind, "grid_dep_action">;
+
+def NVVM_GriddepcontrolOp : NVVM_Op<"griddepcontrol", []> {
   let description = [{
-    Causes the executing thread to wait until all prerequisite grids in flight 
+    If the $kind attribute is set to `wait`, it causes the 
+    executing thread to wait until all prerequisite grids in flight 
     have completed and all the memory operations from the prerequisite grids 
     are performed and made visible to the current grid.
 
+    When the $kind is launch_dependents, it signals that specific dependents 
+    the runtime system designated to react to this instruction can be scheduled 
+    as soon as all other CTAs in the grid issue the same instruction or have 
+    completed.
 
     [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-griddepcontrol)
   }];
-}
-
-def NVVM_GriddepcontrolLaunchDependentsOp
-    : NVVM_IntrOp<"griddepcontrol.launch.dependents", [], 0> {
-  let assemblyFormat = "attr-dict";
 
-  let description = [{
-    Signals that specific dependents the runtime system designated to react to 
-    this instruction can be scheduled as soon as all other CTAs in the grid 
-    issue the same instruction or have completed.
+  let arguments = (ins GridDepActionAttr:$kind);
 
+  let assemblyFormat = "$kind attr-dict";
 
-    [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-griddepcontrol)
+  string llvmBuilder = [{
+    llvm::Intrinsic::ID id;
+      switch ($kind) {
+        case NVVM::GridDepActionKind::wait:
+          id = llvm::Intrinsic::nvvm_griddepcontrol_wait;
+          break;
+        case NVVM::GridDepActionKind::launch_dependents:
+          id = llvm::Intrinsic::nvvm_griddepcontrol_launch_dependents;
+          break;
+      }
+      createIntrinsicCall(builder, id);
   }];
 }
 
@@ -3027,9 +3156,10 @@ def NVVM_GriddepcontrolLaunchDependentsOp
 //===----------------------------------------------------------------------===//
 
 def NVVM_MapaOp: NVVM_Op<"mapa",
-    [TypesMatchWith<"`res` and `a` should have the same type",
-                    "a", "res", "$_self">, NVVMRequiresSM<90>]> {
-  let results = (outs AnyTypeOf<[LLVM_PointerGeneric, LLVM_PointerShared]>:$res);
+  [InputAddressIsCombinationOf<["a", "res"],
+    [[LLVM_PointerShared, LLVM_PointerSharedCluster], [LLVM_PointerGeneric, LLVM_PointerGeneric]],
+    "Valid address-space check(or mapping) for mapa Op">, NVVMRequiresSM<90>]> {
+  let results = (outs AnyTypeOf<[LLVM_PointerGeneric, LLVM_PointerSharedCluster]>:$res);
   let arguments = (ins AnyTypeOf<[LLVM_PointerGeneric, LLVM_PointerShared]>:$a, I32:$b);
 
   string llvmBuilder = [{
diff --git a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
index a2354e22e2745..9fa3ec1fc4b21 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td
@@ -93,19 +93,22 @@ class ROCDL_IntrPure1Op<string mnemonic> :
 
 class ROCDL_IntrOp<string mnemonic, list<int> overloadedResults,
   list<int> overloadedOperands, list<Trait> traits, int numResults,
-  int requiresAccessGroup = 0, int requiresAliasAnalysis = 0, list<int> immArgPositions = [],
+  int requiresAccessGroup = 0, int requiresAliasAnalysis = 0,
+  int requiresArgAndResultAttrs = 0,
+  list<int> immArgPositions = [],
   list<string> immArgAttrNames = []> :
   LLVM_IntrOpBase<ROCDL_Dialect,  mnemonic,
     "amdgcn_" # !subst(".", "_", mnemonic), overloadedResults,
     overloadedOperands, traits, numResults, requiresAccessGroup,
-    requiresAliasAnalysis, 0, 0, 0, immArgPositions, immArgAttrNames>;
+    requiresAliasAnalysis, 0, requiresArgAndResultAttrs, 0,
+    immArgPositions, immArgAttrNames>;
 
 // Subclass to save typing and ease readibility when there aren't overloaded
 // operands or memory accesses.
 class ROCDL_ConcreteNonMemIntrOp<string mnemonic, list<Trait> traits,
     int numResults, list<int> immArgPositions = [],
     list<string> immArgNames = []>
-  : ROCDL_IntrOp<mnemonic, [], [], traits, numResults, 0, 0,
+  : ROCDL_IntrOp<mnemonic, [], [], traits, numResults, 0, 0, 0,
       immArgPositions, immArgNames>;
 //===----------------------------------------------------------------------===//
 // ROCDL special register op definitions
@@ -148,8 +151,11 @@ class ROCDL_DimGetterFunctionOp<string mnemonic, string device_function,
 //===----------------------------------------------------------------------===//
 
 class ROCDL_MbcntOp<string mnemonic> :
-    ROCDL_IntrPure1Op<"mbcnt." # mnemonic>,
-  Arguments<(ins I32:$in0, I32:$in1)> {
+    ROCDL_IntrOp<"mbcnt." # mnemonic, [], [], [Pure], 1,
+    0, 0, /*requiresArgAndResultAttrs=*/1> {
+  dag args = (ins I32:$in0, I32:$in1);
+  let arguments = !con(args, baseArgs);
+  let results = (outs I32:$res);
   let assemblyFormat = [{
     $in0 `,` $in1  attr-dict `:` `(` type($in0) `,` type($in1) `)` `->` type($res)
    }];
@@ -189,6 +195,20 @@ def ROCDL_BallotOp :
   let assemblyFormat = "$pred attr-dict `:` type($res)";
 }
 
+def ROCDL_ReadfirstlaneOp : ROCDL_IntrOp<"readfirstlane", [], [0], [AllTypesMatch<["res", "src"]>], 1>,
+  Arguments<(ins LLVM_Type:$src)> {
+  let results = (outs LLVM_Type:$res);
+  let summary = "Get the value in first active lane.";
+
+  let description = [{
+    Returns the value in the lowest active lane of the input operand.
+  }];
+
+  let assemblyFormat = [{
+    $src attr-dict `:` type($res)
+  }];
+}
+
 def ROCDL_ReadlaneOp : ROCDL_IntrOp<"readlane", [], [0], [AllTypesMatch<["res", "src0"]>], 1>,
   Arguments<(ins LLVM_Type:$src0,
                  I32:$src1)> {
@@ -201,7 +221,7 @@ def ROCDL_ReadlaneOp : ROCDL_IntrOp<"readlane", [], [0], [AllTypesMatch<["res",
 
   let assemblyFormat = [{
     $src0 `,` $src1  attr-dict `:` `(` type($src0) `,` type($src1) `)` `->` type($res)
-   }];
+  }];
 }
 
 //===----------------------------------------------------------------------===//
@@ -501,7 +521,7 @@ def ROCDL_ds_read_tr16_b64 : ROCDL_LDS_Read_Tr_IntrOp<"ds.read.tr16.b64">;
 //===---------------------------------------------------------------------===//
 
 def ROCDL_LoadToLDSOp :
-  ROCDL_IntrOp<"load.to.lds", [], [0], [], 0, 0, 1, [2, 3, 4], ["size", "offset", "aux"]> {
+  ROCDL_IntrOp<"load.to.lds", [], [0], [], 0, 0, 1, 0, [2, 3, 4], ["size", "offset", "aux"]> {
   dag args = (ins Arg<LLVM_AnyPointer, "", [MemRead]>:$globalPtr,
                  Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
                  I32Attr:$size,
@@ -520,7 +540,7 @@ def ROCDL_LoadToLDSOp :
 }
 
 def ROCDL_GlobalLoadLDSOp :
-  ROCDL_IntrOp<"global.load.lds", [], [], [], 0, 0, 1, [2, 3, 4], ["size", "offset", "aux"]> {
+  ROCDL_IntrOp<"global.load.lds", [], [], [], 0, 0, 1, 0, [2, 3, 4], ["size", "offset", "aux"]> {
   dag args = (ins Arg<ROCDLGlobalBuffer, "", [MemRead]>:$globalPtr,
                  Arg<ROCDLBufferLDS, "", [MemWrite]>:$ldsPtr,
                  I32Attr:$size,
@@ -734,7 +754,7 @@ def ROCDL_RawBufferAtomicUMinOp :
 
 // DPP Update intrinsic
 def ROCDL_DPPUpdateOp : ROCDL_IntrOp<"update.dpp", [], [0],
-    [AllTypesMatch<["res", "src", "old"]>], 1, 0, 0,
+    [AllTypesMatch<["res", "src", "old"]>], 1, 0, 0, 0,
       [2, 3, 4, 5], ["dppCtrl", "rowMask", "bankMask", "boundCtrl"]>,
   Arguments<(ins LLVM_Type:$old, LLVM_Type:$src, I32Attr:$dppCtrl, I32Attr:$rowMask,
       I32Attr:$bankMask, I1Attr:$boundCtrl)> {
@@ -746,7 +766,7 @@ def ROCDL_DPPUpdateOp : ROCDL_IntrOp<"update.dpp", [], [0],
 
 // PermLaneX16 intrinsic operation
 def ROCDL_PermlaneX16Op : ROCDL_IntrOp<"permlanex16", [], [0],
-    [AllTypesMatch<["res", "old", "src0"]>, AllTypesMatch<["src1", "src2"]>], 1, 0, 0,
+    [AllTypesMatch<["res", "old", "src0"]>, AllTypesMatch<["src1", "src2"]>], 1, 0, 0, 0,
     [4, 5], ["fi", "boundControl"]>,
   Arguments<(ins LLVM_Type:$old, LLVM_Type:$src0, LLVM_Type:$src1, LLVM_Type:$src2,
              I1Attr:$fi, I1Attr:$boundControl)> {
@@ -760,6 +780,53 @@ def ROCDL_PermlaneX16Op : ROCDL_IntrOp<"permlanex16", [], [0],
   }];
 }
 
+class ROCDL_ConcretePair<Type elem0, Type elem1> :
+  Type<And<[
+    LLVM_AnyStruct.predicate,
+    SubstLeaves<
+         "$_self",
+         "::llvm::cast<::mlir::LLVM::LLVMStructType>($_self).getBody()[0]",
+         elem0.predicate>,
+      SubstLeaves<
+         "$_self",
+         "::llvm::cast<::mlir::LLVM::LLVMStructType>($_self).getBody()[1]",
+         elem1.predicate>
+  ]>,
+  "LLVM dialect-compatible struct of " # elem0.summary # "and" # elem1.summary,
+  "::mlir::LLVM::LLVMStructType">,
+  BuildableType<"::mlir::LLVM::LLVMStructType::getLiteral($_builder.getContext(), "
+  "{" # elem0.builderCall # ", " # elem1.builderCall # "})">;
+
+// Permlane16 swap intrinsic operation
+def ROCDL_Permlane16SwapOp : ROCDL_IntrOp<"permlane16.swap", [], [],
+    [], 1, 0, 0, 0,
+    [2, 3], ["fi", "boundControl"]>,
+  Arguments<(ins I32:$old, I32:$src, I1Attr:$fi, I1Attr:$boundControl)> {
+  let results = (outs ROCDL_ConcretePair<I32, I32>:$res);
+  let assemblyFormat = [{
+    attr-dict $old `,` $src `,` $fi `,` $boundControl `:` `(` type($old) `,` type($src) `)` `->` type($res)
+  }];
+  let description = [{
+    Performs a `permlane16.swap` operation with the given operands, applying the
+    permutation specified by $fi to the provided inputs.
+  }];
+}
+
+// Permlane32 swap intrinsic operation
+def ROCDL_Permlane32SwapOp : ROCDL_IntrOp<"permlane32.swap", [], [],
+    [], 1, 0, 0, 0,
+    [2, 3], ["fi", "boundControl"]>,
+  Arguments<(ins I32:$old, I32:$src, I1Attr:$fi, I1Attr:$boundControl)> {
+  let results = (outs ROCDL_ConcretePair<I32, I32>:$res);
+  let assemblyFormat = [{
+    attr-dict $old `,` $src `,` $fi `,` $boundControl `:` `(` type($old) `,` type($src) `)` `->` type($res)
+  }];
+  let description = [{
+    Performs a `permlane32.swap` operation with the given operands, applying the
+    permutation specified by $fi to the provided inputs.
+  }];
+}
+
 class ROCDL_ConcreteVector<Type elem, int length> :
   FixedVectorOfLengthAndType<[length], [elem]>,
   BuildableType<
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/Linalg.h b/mlir/include/mlir/Dialect/Linalg/IR/Linalg.h
index 62c04bb2ee1ab..eb4e3810f0d07 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/Linalg.h
+++ b/mlir/include/mlir/Dialect/Linalg/IR/Linalg.h
@@ -145,8 +145,7 @@ std::pair<int64_t, int64_t> getFmrFromWinogradConv2DFmr(WinogradConv2DFmr fmr);
 #define GET_OP_CLASSES
 #include "mlir/Dialect/Linalg/IR/LinalgRelayoutOps.h.inc"
 
-namespace mlir {
-namespace linalg {
+namespace mlir::linalg {
 
 /// Returns the outer shape in the packed domain before applying the
 /// transposition.
@@ -155,7 +154,194 @@ template <typename OpTy,
                                       std::is_same_v<OpTy, linalg::UnPackOp>>>
 SmallVector<int64_t> getPackedOuterShapeWithoutTransposition(OpTy packOrUnPack);
 
-} // namespace linalg
-} // namespace mlir
+/// Specialization of `linalg.matmul` op that has a transpose map on A
+class MatmulTransposeAOp : public MatmulOp {
+  /// Create an affine map for a transpose-A matmul. Used only in the builders.
+  static SmallVector<AffineMap> getDefaultIndexingMaps(OpBuilder &builder);
+
+public:
+  using MatmulOp::MatmulOp;
+  static ::mlir::TypeID resolveTypeID() { return TypeID::get<MatmulOp>(); }
+
+  /// Build a transpose A matmul.
+  static void build(OpBuilder &builder, OperationState &result,
+                    ValueRange inputs, ValueRange outputs,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static MatmulTransposeAOp create(OpBuilder &builder, Location location,
+                                   ValueRange inputs, ValueRange outputs,
+                                   ArrayRef<NamedAttribute> attributes = {});
+
+  /// Build a transpose A matmul with a specific result type.
+  static void build(OpBuilder &builder, OperationState &result,
+                    TypeRange resultTensorTypes, ValueRange inputs,
+                    ValueRange outputs,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static MatmulTransposeAOp create(OpBuilder &builder, Location location,
+                                   TypeRange resultTensorTypes,
+                                   ValueRange inputs, ValueRange outputs,
+                                   ArrayRef<NamedAttribute> attributes = {});
+
+  /// Build a transpose A matmul with a specific result type and a cast type.
+  static void build(OpBuilder &builder, OperationState &result,
+                    TypeRange resultTensorTypes, ValueRange inputs,
+                    ValueRange outputs, Attribute cast,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static MatmulTransposeAOp create(OpBuilder &builder, Location location,
+                                   TypeRange resultTensorTypes,
+                                   ValueRange inputs, ValueRange outputs,
+                                   Attribute cast,
+                                   ArrayRef<NamedAttribute> attributes = {});
+
+  /// Checks if the affine map is the expected one for this operation
+  static bool isDefaultIndexingMaps(Attribute attr);
+
+  static bool classof(Operation *op);
+};
+
+/// Specialization of `linalg.matmul` op that has a transpose map on B
+class MatmulTransposeBOp : public MatmulOp {
+  /// Create an affine map for a transpose-B matmul. Used only in the builders.
+  static SmallVector<AffineMap> getDefaultIndexingMaps(OpBuilder &builder);
+
+public:
+  using MatmulOp::MatmulOp;
+  static ::mlir::TypeID resolveTypeID() { return TypeID::get<MatmulOp>(); }
+
+  /// Build a transpose B matmul.
+  static void build(OpBuilder &builder, OperationState &result,
+                    ValueRange inputs, ValueRange outputs,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static MatmulTransposeBOp create(OpBuilder &builder, Location location,
+                                   ValueRange inputs, ValueRange outputs,
+                                   ArrayRef<NamedAttribute> attributes = {});
+
+  /// Build a transpose B matmul with a specific result type.
+  static void build(OpBuilder &builder, OperationState &result,
+                    TypeRange resultTensorTypes, ValueRange inputs,
+                    ValueRange outputs,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static MatmulTransposeBOp create(OpBuilder &builder, Location location,
+                                   TypeRange resultTensorTypes,
+                                   ValueRange inputs, ValueRange outputs,
+                                   ArrayRef<NamedAttribute> attributes = {});
+
+  /// Build a transpose B matmul with a specific result type and a cast type.
+  static void build(OpBuilder &builder, OperationState &result,
+                    TypeRange resultTensorTypes, ValueRange inputs,
+                    ValueRange outputs, Attribute cast,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static MatmulTransposeBOp create(OpBuilder &builder, Location location,
+                                   TypeRange resultTensorTypes,
+                                   ValueRange inputs, ValueRange outputs,
+                                   Attribute cast,
+                                   ArrayRef<NamedAttribute> attributes = {});
+
+  /// Checks if the affine map is the expected one for this operation
+  static bool isDefaultIndexingMaps(Attribute attr);
+
+  static bool classof(Operation *op);
+};
+
+/// Specialization of `linalg.batch_matmul` op that has a transpose map on A
+class BatchMatmulTransposeAOp : public BatchMatmulOp {
+  /// Create an affine map for a transpose-A batch_matmul. Used only in the
+  /// builders.
+  static SmallVector<AffineMap> getDefaultIndexingMaps(OpBuilder &builder);
+
+public:
+  using BatchMatmulOp::BatchMatmulOp;
+  static ::mlir::TypeID resolveTypeID() { return TypeID::get<BatchMatmulOp>(); }
+
+  /// Build a transpose A matmul.
+  static void build(OpBuilder &builder, OperationState &result,
+                    ValueRange inputs, ValueRange outputs,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static BatchMatmulTransposeAOp
+  create(OpBuilder &builder, Location location, ValueRange inputs,
+         ValueRange outputs, ArrayRef<NamedAttribute> attributes = {});
+
+  /// Build a transpose A matmul with a specific result type.
+  static void build(OpBuilder &builder, OperationState &result,
+                    TypeRange resultTensorTypes, ValueRange inputs,
+                    ValueRange outputs,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static BatchMatmulTransposeAOp
+  create(OpBuilder &builder, Location location, TypeRange resultTensorTypes,
+         ValueRange inputs, ValueRange outputs,
+         ArrayRef<NamedAttribute> attributes = {});
+
+  /// Build a transpose A matmul with a specific result type and a cast type.
+  static void build(OpBuilder &builder, OperationState &result,
+                    TypeRange resultTensorTypes, ValueRange inputs,
+                    ValueRange outputs, Attribute cast,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static BatchMatmulTransposeAOp
+  create(OpBuilder &builder, Location location, TypeRange resultTensorTypes,
+         ValueRange inputs, ValueRange outputs, Attribute cast,
+         ArrayRef<NamedAttribute> attributes = {});
+
+  /// Checks if the affine map is the expected one for this operation
+  static bool isDefaultIndexingMaps(Attribute attr);
+
+  static bool classof(Operation *op);
+};
+
+/// Specialization of `linalg.batch_matmul` op that has a transpose map on B
+class BatchMatmulTransposeBOp : public BatchMatmulOp {
+  /// Create an affine map for a transpose-B batch_matmul. Used only in the
+  /// builders.
+  static SmallVector<AffineMap> getDefaultIndexingMaps(OpBuilder &builder);
+
+public:
+  using BatchMatmulOp::BatchMatmulOp;
+  static ::mlir::TypeID resolveTypeID() { return TypeID::get<BatchMatmulOp>(); }
+
+  /// Build a transpose B matmul.
+  static void build(OpBuilder &builder, OperationState &result,
+                    ValueRange inputs, ValueRange outputs,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static BatchMatmulTransposeBOp
+  create(OpBuilder &builder, Location location, ValueRange inputs,
+         ValueRange outputs, ArrayRef<NamedAttribute> attributes = {});
+
+  /// Build a transpose B matmul with a specific result type.
+  static void build(OpBuilder &builder, OperationState &result,
+                    TypeRange resultTensorTypes, ValueRange inputs,
+                    ValueRange outputs,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static BatchMatmulTransposeBOp
+  create(OpBuilder &builder, Location location, TypeRange resultTensorTypes,
+         ValueRange inputs, ValueRange outputs,
+         ArrayRef<NamedAttribute> attributes = {});
+
+  /// Build a transpose B matmul with a specific result type and a cast type.
+  static void build(OpBuilder &builder, OperationState &result,
+                    TypeRange resultTensorTypes, ValueRange inputs,
+                    ValueRange outputs, Attribute cast,
+                    ArrayRef<NamedAttribute> attributes = {});
+
+  static BatchMatmulTransposeBOp
+  create(OpBuilder &builder, Location location, TypeRange resultTensorTypes,
+         ValueRange inputs, ValueRange outputs, Attribute cast,
+         ArrayRef<NamedAttribute> attributes = {});
+
+  /// Checks if the affine map is the expected one for this operation
+  static bool isDefaultIndexingMaps(Attribute attr);
+
+  static bool classof(Operation *op);
+};
+
+} // namespace mlir::linalg
 
 #endif // MLIR_DIALECT_LINALG_IR_LINALG_H
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
index ba73cfbbed845..9f1e88a040f5f 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
@@ -474,7 +474,7 @@ def LinalgStructuredInterface
         int64_t resultIndex =
             opOperand->getOperandNumber() - $_op.getNumDpsInputs();
         assert(resultIndex >= 0 &&
-               resultIndex < this->getOperation()->getNumResults());
+               resultIndex < $_op.getNumDpsInits());
         Operation *yieldOp = getBlock()->getTerminator();
         return &yieldOp->getOpOperand(resultIndex);
       }]
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
index 3637147c5a90d..9aae1b850c3a0 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yaml
@@ -1055,152 +1055,6 @@ structured_op: !LinalgStructuredOpConfig
                     - !ScalarExpression
                       scalar_arg: BZp
 --- !LinalgOpConfig
-metadata: !LinalgOpMetadata
-  name: matmul_transpose_a
-  cpp_class_name: MatmulTransposeAOp
-  doc: |-
-    Performs a matrix multiplication of two 2D inputs with lhs operand
-    transposed.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-  implements:
-  - LinalgContractionOpInterface
-structured_op: !LinalgStructuredOpConfig
-  args:
-  - !LinalgOperandDefConfig
-    name: A
-    kind: input_tensor
-    type_var: T1
-    shape_map: affine_map<()[s0, s1, s2] -> (s0, s1)>
-  - !LinalgOperandDefConfig
-    name: B
-    kind: input_tensor
-    type_var: T2
-    shape_map: affine_map<()[s0, s1, s2] -> (s0, s2)>
-  - !LinalgOperandDefConfig
-    name: C
-    kind: output_tensor
-    type_var: U
-    shape_map: affine_map<()[s0, s1, s2] -> (s2, s1)>
-  - !LinalgOperandDefConfig
-    name: cast
-    kind: type_fn_attr
-    default_fn: cast_signed
-  indexing_maps: !LinalgIndexingMapsConfig
-    static_indexing_maps:
-    - affine_map<(d0, d1, d2)[s0, s1, s2] -> (d2, d0)>
-    - affine_map<(d0, d1, d2)[s0, s1, s2] -> (d2, d1)>
-    - affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d1)>
-  iterator_types:
-  - parallel
-  - parallel
-  - reduction
-  assignments:
-  - !ScalarAssign
-    arg: C
-    value: !ScalarExpression
-      scalar_fn:
-        kind: binary
-        fn_name: add
-        operands:
-        - !ScalarExpression
-          scalar_arg: C
-        - !ScalarExpression
-          scalar_fn:
-            kind: binary
-            fn_name: mul
-            operands:
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                attr_name: cast
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: A
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                attr_name: cast
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: B
---- !LinalgOpConfig
-metadata: !LinalgOpMetadata
-  name: matmul_transpose_b
-  cpp_class_name: MatmulTransposeBOp
-  doc: |-
-    Performs a matrix multiplication of two 2D inputs with rhs operand
-    transposed.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-  implements:
-  - LinalgContractionOpInterface
-structured_op: !LinalgStructuredOpConfig
-  args:
-  - !LinalgOperandDefConfig
-    name: A
-    kind: input_tensor
-    type_var: T1
-    shape_map: affine_map<()[s0, s1, s2] -> (s0, s1)>
-  - !LinalgOperandDefConfig
-    name: B
-    kind: input_tensor
-    type_var: T2
-    shape_map: affine_map<()[s0, s1, s2] -> (s2, s1)>
-  - !LinalgOperandDefConfig
-    name: C
-    kind: output_tensor
-    type_var: U
-    shape_map: affine_map<()[s0, s1, s2] -> (s0, s2)>
-  - !LinalgOperandDefConfig
-    name: cast
-    kind: type_fn_attr
-    default_fn: cast_signed
-  indexing_maps: !LinalgIndexingMapsConfig
-    static_indexing_maps:
-    - affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d2)>
-    - affine_map<(d0, d1, d2)[s0, s1, s2] -> (d1, d2)>
-    - affine_map<(d0, d1, d2)[s0, s1, s2] -> (d0, d1)>
-  iterator_types:
-  - parallel
-  - parallel
-  - reduction
-  assignments:
-  - !ScalarAssign
-    arg: C
-    value: !ScalarExpression
-      scalar_fn:
-        kind: binary
-        fn_name: add
-        operands:
-        - !ScalarExpression
-          scalar_arg: C
-        - !ScalarExpression
-          scalar_fn:
-            kind: binary
-            fn_name: mul
-            operands:
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                attr_name: cast
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: A
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                attr_name: cast
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: B
---- !LinalgOpConfig
 metadata: !LinalgOpMetadata
   name: mmt4d
   cpp_class_name: Mmt4DOp
@@ -1358,146 +1212,6 @@ structured_op: !LinalgStructuredOpConfig
                 - !ScalarExpression
                   scalar_arg: rhs
 --- !LinalgOpConfig
-metadata: !LinalgOpMetadata
-  name: batch_matmul_transpose_a
-  cpp_class_name: BatchMatmulTransposeAOp
-  doc: |-
-    Performs a batched matrix multiplication of two 3D inputs where lhs operand
-    has its non-batch dimensions transposed.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-  implements:
-  - LinalgContractionOpInterface
-structured_op: !LinalgStructuredOpConfig
-  args:
-  - !LinalgOperandDefConfig
-    name: A
-    kind: input_tensor
-    type_var: T1
-    shape_map: affine_map<()[s0, s1, s2, s3] -> (s0, s1, s2)>
-  - !LinalgOperandDefConfig
-    name: B
-    kind: input_tensor
-    type_var: T2
-    shape_map: affine_map<()[s0, s1, s2, s3] -> (s0, s1, s3)>
-  - !LinalgOperandDefConfig
-    name: C
-    kind: output_tensor
-    type_var: U
-    shape_map: affine_map<()[s0, s1, s2, s3] -> (s0, s2, s3)>
-  indexing_maps: !LinalgIndexingMapsConfig
-    static_indexing_maps:
-    - affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d3, d1)>
-    - affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d3, d2)>
-    - affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d1, d2)>
-  iterator_types:
-  - parallel
-  - parallel
-  - parallel
-  - reduction
-  assignments:
-  - !ScalarAssign
-    arg: C
-    value: !ScalarExpression
-      scalar_fn:
-        kind: binary
-        fn_name: add
-        operands:
-        - !ScalarExpression
-          scalar_arg: C
-        - !ScalarExpression
-          scalar_fn:
-            kind: binary
-            fn_name: mul
-            operands:
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                fn_name: cast_signed
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: A
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                fn_name: cast_signed
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: B
---- !LinalgOpConfig
-metadata: !LinalgOpMetadata
-  name: batch_matmul_transpose_b
-  cpp_class_name: BatchMatmulTransposeBOp
-  doc: |-
-    Performs a batched matrix multiplication of two 3D inputs where rhs operand
-    has its non-batch dimensions transposed.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-  implements:
-  - LinalgContractionOpInterface
-structured_op: !LinalgStructuredOpConfig
-  args:
-  - !LinalgOperandDefConfig
-    name: A
-    kind: input_tensor
-    type_var: T1
-    shape_map: affine_map<()[s0, s1, s2, s3] -> (s0, s1, s2)>
-  - !LinalgOperandDefConfig
-    name: B
-    kind: input_tensor
-    type_var: T2
-    shape_map: affine_map<()[s0, s1, s2, s3] -> (s0, s3, s2)>
-  - !LinalgOperandDefConfig
-    name: C
-    kind: output_tensor
-    type_var: U
-    shape_map: affine_map<()[s0, s1, s2, s3] -> (s0, s1, s3)>
-  indexing_maps: !LinalgIndexingMapsConfig
-    static_indexing_maps:
-    - affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d1, d3)>
-    - affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d2, d3)>
-    - affine_map<(d0, d1, d2, d3)[s0, s1, s2, s3] -> (d0, d1, d2)>
-  iterator_types:
-  - parallel
-  - parallel
-  - parallel
-  - reduction
-  assignments:
-  - !ScalarAssign
-    arg: C
-    value: !ScalarExpression
-      scalar_fn:
-        kind: binary
-        fn_name: add
-        operands:
-        - !ScalarExpression
-          scalar_arg: C
-        - !ScalarExpression
-          scalar_fn:
-            kind: binary
-            fn_name: mul
-            operands:
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                fn_name: cast_signed
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: A
-            - !ScalarExpression
-              scalar_fn:
-                kind: type
-                fn_name: cast_signed
-                type_var: U
-                operands:
-                - !ScalarExpression
-                  scalar_arg: B
---- !LinalgOpConfig
 metadata: !LinalgOpMetadata
   name: quantized_batch_matmul
   cpp_class_name: QuantizedBatchMatmulOp
diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
index ca0cc03acd7ad..f3674c3eecfe6 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgStructuredOps.td
@@ -253,10 +253,12 @@ def MapOp : LinalgStructuredBase_Op<"map", [
           }
     ```
 
-    Shortened print form is available. Applies to simple maps with one
-    non-yield operation inside the body.
+    Shortened print form is available for simple maps where the body contains exactly
+    two operations (the payload operation and a yield), the payload operation has
+    the same number of operands as block arguments with operands matching block
+    arguments in order, and the yield operand is the result of the payload operation.
 
-    The example above will be printed as:
+    The example above will be printed using the shortened form as:
     ```mlir
       %add = linalg.map { arith.addf }
           ins(%lhs, %rhs : tensor<64xf32>, tensor<64xf32>)
@@ -340,13 +342,15 @@ def ReduceOp : LinalgStructuredBase_Op<"reduce", [
           }
     ```
 
-    Shortened print form is available. Applies to simple (not variadic) reduces
-    with one non-yield operation inside the body. Applies only if the operation
-    takes `%out` as the first argument.
+    Shortened print form is available for simple reduces where the body contains exactly
+    two operations (the payload operation and a yield), the payload operation has the
+    same number of operands as block arguments, the first block argument (init) is the
+    last operand of the payload operation with remaining operands matching remaining
+    block arguments in order, and the yield operand is the result of the payload operation.
 
-    The example above will be printed as:
+    The example above will be printed using the shortened form as:
     ```mlir
-          %reduce = linalg.reduce { arith.addf }
+      %reduce = linalg.reduce { arith.addf }
           ins(%input:tensor<16x32x64xf32>)
           outs(%init:tensor<16x64xf32>)
           dimensions = [1]
@@ -785,6 +789,9 @@ def MatmulOp : LinalgStructuredBase_Op<"matmul", [
       /// Returns a list of AffineMap with the default matmul indexing charactristic.
       static SmallVector<AffineMap> getDefaultIndexingMaps(MLIRContext *context);
 
+      /// Returns true if the AffineMap is the default matmul indexing charactristic.
+      static bool isDefaultIndexingMaps(Attribute attr);
+
       /// Returns true if the given broadcast map \p bcastMap is valid for this op.
       bool isValidLhsRhsBroadcastMap(AffineMap bcastMap);
 
@@ -1057,6 +1064,9 @@ def BatchMatmulOp : LinalgStructuredBase_Op<"batch_matmul", !listconcat([AttrSiz
       /// Returns a list with default AffineMap(s), i.e. without broadcasts and transpositions.
       static SmallVector<AffineMap> getDefaultIndexingMaps(MLIRContext *context);
 
+      /// Returns true if the AffineMap is the default batch matmul indexing charactristic.
+      static bool isDefaultIndexingMaps(Attribute attr);
+
       /// Returns true if the given broadcast map \p bcastMap is valid for this op.
       bool isValidLhsRhsBroadcastMap(AffineMap bcastMap, bool isLHS = true);
 
@@ -1181,6 +1191,9 @@ def BatchReduceMatmulOp : LinalgStructuredBase_Op<"batch_reduce_matmul", [
       /// Returns a list of AffineMap with the default batch_reduce_matmul indexing charactristic.
       static SmallVector<AffineMap> getDefaultIndexingMaps(MLIRContext *context);
 
+      /// Returns true if the AffineMap is the default batch reduce matmul indexing charactristic.
+      static bool isDefaultIndexingMaps(Attribute attr);
+
       /// Returns true if the given broadcast map \p bcastMap is valid for this op.
       bool isValidLhsRhsBroadcastMap(AffineMap bcastMap, bool isLHS = true);
 
diff --git a/mlir/include/mlir/Dialect/Linalg/Passes.td b/mlir/include/mlir/Dialect/Linalg/Passes.td
index 373842c9b03de..44da2965e6892 100644
--- a/mlir/include/mlir/Dialect/Linalg/Passes.td
+++ b/mlir/include/mlir/Dialect/Linalg/Passes.td
@@ -10,6 +10,88 @@
 #define MLIR_DIALECT_LINALG_PASSES
 
 include "mlir/Pass/PassBase.td"
+include "mlir/IR/Constraints.td"
+
+// ------------------ Begin of "form" conversions
+//
+// These conversions allow for the transformation of linalg ops between
+// different forms. Structured ops can be represented in different forms,
+// such as generic ops, category ops, and named ops.
+//
+// The operation tree is as follows:
+//   generic     category      named
+//  ---------|-------------|----------
+//  generic ---> contract ----> matmul
+//           |              \-> batch_matmul
+//           |              \-> batch_reduce_matmul
+//           |              \-> ...
+//           \-> elementwise -> add
+//                          \-> sub
+//                          \-> ...
+//
+// Morphisms between representations can happen in the following 6 ways:
+//  generic <---> category <---> named
+//      \-------------------------/
+//
+// generic subsumes category which subsumes structured named (not softmax,
+// convolutions, etc). The generalization path is guaranteed, the
+// specialization path is not.
+
+def LinalgMorphOpsPass : Pass<"linalg-morph-ops"> {
+  let summary = "Convert linalg ops between forms";
+
+  let description = [{
+    Convert a linalg op from one representation to another equivalent.
+    For example, a linalg named op `linalg.add` can also be written as an
+    category op `linalg.elementwise`, and can also be re-written as
+    a `linalg.generic`, giving the morphism:
+
+      named-op <--> category_op (elementwise, contraction, ..) <--> generic
+
+    Note that the set of `linalg.generic` subsumes named and category ops
+    and therefore not all `linalg.genric` can be converted to  named or
+    category op. Similarly, catgory ops subsume named ops.
+
+    Note:
+     Legacy converters:
+     `--linalg-generalize-named-ops` is the path `named-op --> generic-op`
+     `--linalg-specialize-generic-ops` is the path `named-op <-- generic-op`
+  }];
+  let dependentDialects = ["linalg::LinalgDialect"];
+
+  let options = [
+    // Generalization path is guaranteed.
+    Option<"namedToCategory", "named-to-category", "bool", /*default=*/"false",
+           "convert named ops to category op e.g. `linalg.elementwise`">,
+    Option<"categoryToGeneric", "category-to-generic", "bool", /*default=*/"false",
+           "convert category ops e.g. `linalg.elementwise` to `linalg.generic`">,
+    Option<"namedToGeneric", "named-to-generic", "bool", /*default=*/"false",
+           "convert named ops e.g. `linalg.add` to `linalg.generic`">,
+    
+    // Specialization path is not guaranteed.
+    Option<"genericToNamed", "generic-to-named", "bool", /*default=*/"false",
+           "convert linalg.generic to equivalent named ops"> ];
+    //  TODOs: `generic-to-category`, `category-to-named`
+}
+
+def LinalgGeneralizeNamedOpsPass : Pass<"linalg-generalize-named-ops">,
+                                   Deprecated<"Use 'linalg-morph-ops' instead."> {
+  let summary = "Convert named ops into generic ops";
+  let dependentDialects = ["linalg::LinalgDialect"];
+}
+
+def LinalgSpecializeGenericOpsPass : Pass<"linalg-specialize-generic-ops">,
+                                     Deprecated<"Use 'linalg-morph-ops' instead."> {
+  let summary = "Convert generic ops back to named ops";
+  let dependentDialects = ["linalg::LinalgDialect"];
+}
+
+// ------------------ End of "form" conversions
+
+def SimplifyDepthwiseConvPass: Pass<"simplify-depthwise-conv"> {
+  let summary = "Simplify depthwise convolution.";
+  let dependentDialects = ["linalg::LinalgDialect", "tensor::TensorDialect"];
+}
 
 def ConvertElementwiseToLinalgPass : Pass<"convert-elementwise-to-linalg", ""> {
   let summary = "Convert ElementwiseMappable ops to linalg";
@@ -77,11 +159,6 @@ def LinalgElementwiseOpFusionPass : Pass<"linalg-fuse-elementwise-ops"> {
   ];
 }
 
-def LinalgNamedOpConversionPass: Pass<"linalg-named-op-conversion"> {
-  let summary = "Convert from one named linalg op to another.";
-  let dependentDialects = ["linalg::LinalgDialect", "tensor::TensorDialect"];
-}
-
 def LinalgInlineScalarOperandsPass : Pass<"linalg-inline-scalar-operands"> {
   let summary = "Inline scalar operands into linalg generic ops";
   let dependentDialects = [
@@ -89,16 +166,6 @@ def LinalgInlineScalarOperandsPass : Pass<"linalg-inline-scalar-operands"> {
   ];
 }
 
-def LinalgGeneralizeNamedOpsPass : Pass<"linalg-generalize-named-ops"> {
-  let summary = "Convert named ops into generic ops";
-  let dependentDialects = ["linalg::LinalgDialect"];
-}
-
-def LinalgSpecializeGenericOpsPass : Pass<"linalg-specialize-generic-ops"> {
-  let summary = "Convert generic ops back to named ops";
-  let dependentDialects = ["linalg::LinalgDialect"];
-}
-
 def LinalgFoldIntoElementwisePass : Pass<"linalg-fold-into-elementwise"> {
   let summary = "Fold transform, broadcast and other ops into elementwise";
   let dependentDialects = ["linalg::LinalgDialect"];
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 61ce23f07faa8..a19cce4b919a8 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -2348,6 +2348,9 @@ def VectorizeChildrenAndApplyPatternsOp :
     operation that is contained inside the vectorization target.
 
     This transformation supports the following attributes:
+    - `fold_type_extensions_into_contract`: a `UnitAttr` to enable the folding of
+      type extension operations into `vector.contract` to create a mixed precision
+      operation.
     - `vectorize_padding`: a `UnitAttr` to activate the vectorization of
       `tensor.pad` ops. Different pipelines may prefer to lower such ops to
       loops.
@@ -2368,6 +2371,7 @@ def VectorizeChildrenAndApplyPatternsOp :
   }];
 
   let arguments = (ins TransformHandleTypeInterface:$target,
+                   UnitAttr:$fold_type_extensions_into_contract,
                    UnitAttr:$vectorize_padding,
                    UnitAttr:$vectorize_nd_extract,
                    UnitAttr:$flatten_1d_depthwise_conv,
@@ -2381,6 +2385,7 @@ def VectorizeChildrenAndApplyPatternsOp :
 
   let builders = [
     OpBuilder<(ins "Value":$target,
+               CArg<"bool", "false">:$foldTypeExtensionsIntoContract,
                CArg<"bool", "false">:$vectorizePadding,
                CArg<"bool", "false">:$vectorizeNDExtract,
                CArg<"bool", "false">:$flatten1DDepthwise)>
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index d4ffe0a91fcfe..0cfc8821c0add 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -1690,7 +1690,7 @@ struct DecomposeOuterUnitDimsPackOpPattern
 /// Rewrites a linalg::UnPackOp into a sequence of rank-reduced
 ///   * tensor::ExtractSliceOp + linalg::TransposeOp + tensor::InsertSliceOp
 ///
-/// Requires that all the outer dims of the input linalg::PackOp are 1.
+/// Requires that all the tiled outer dims of the input linalg::PackOp are 1.
 ///
 /// Before:
 /// ```
@@ -1831,6 +1831,10 @@ void populateLinalgNamedOpsGeneralizationPatterns(RewritePatternSet &patterns);
 void populateLinalgGenericOpsSpecializationPatterns(
     RewritePatternSet &patterns);
 
+/// Populates `patterns` that convert linalg named ops e.g. `linalg.add`
+/// to equivalent `linalg.elementwise`.
+void populateLinalgNamedToElementwisePatterns(RewritePatternSet &patterns);
+
 /// Populates `patterns` with patterns that fold operations like
 /// `linalg.transform` into elementwise op map.
 void populateLinalgFoldIntoElementwisePatterns(RewritePatternSet &patterns);
@@ -1958,9 +1962,8 @@ void populateFoldAddIntoDestPatterns(RewritePatternSet &patterns);
 void populateFuseTensorPadWithProducerLinalgOpPatterns(
     RewritePatternSet &patterns);
 
-/// Patterns to convert from one named op to another. These can be seen as
-/// canonicalizations of named ops into another named op.
-void populateLinalgNamedOpConversionPatterns(RewritePatternSet &patterns);
+/// Patterns to simplify depthwise convolutions.
+void populateSimplifyDepthwiseConvPatterns(RewritePatternSet &patterns);
 
 /// Patterns to fold unit-extent dimensions in operands/results of linalg ops on
 /// tensors via reassociative reshape ops.
diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
index 18d5f2db12e9c..0db11aa9af683 100644
--- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
+++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td
@@ -1536,6 +1536,15 @@ def OpenACC_ParallelOp : OpenACC_Op<"parallel",
     /// Adds a private clause variable to this operation, including its recipe.
     void addPrivatization(MLIRContext *, mlir::acc::PrivateOp op,
                           mlir::acc::PrivateRecipeOp recipe);
+    /// Adds a firstprivate clause variable to this operation, including its
+    /// recipe.
+    void addFirstPrivatization(MLIRContext *, mlir::acc::FirstprivateOp op,
+                               mlir::acc::FirstprivateRecipeOp recipe);
+
+    /// Adds a reduction clause variable to this operation, including its
+    /// recipe.
+    void addReduction(MLIRContext *, mlir::acc::ReductionOp op,
+                      mlir::acc::ReductionRecipeOp recipe);
   }];
 
   let assemblyFormat = [{
@@ -1681,6 +1690,14 @@ def OpenACC_SerialOp : OpenACC_Op<"serial",
     /// Adds a private clause variable to this operation, including its recipe.
     void addPrivatization(MLIRContext *, mlir::acc::PrivateOp op,
                           mlir::acc::PrivateRecipeOp recipe);
+    /// Adds a firstprivate clause variable to this operation, including its
+    /// recipe.
+    void addFirstPrivatization(MLIRContext *, mlir::acc::FirstprivateOp op,
+                               mlir::acc::FirstprivateRecipeOp recipe);
+    /// Adds a reduction clause variable to this operation, including its
+    /// recipe.
+    void addReduction(MLIRContext *, mlir::acc::ReductionOp op,
+                      mlir::acc::ReductionRecipeOp recipe);
   }];
 
   let assemblyFormat = [{
@@ -2407,6 +2424,10 @@ def OpenACC_LoopOp : OpenACC_Op<"loop",
     /// Adds a private clause variable to this operation, including its recipe.
     void addPrivatization(MLIRContext *, mlir::acc::PrivateOp op,
                           mlir::acc::PrivateRecipeOp recipe);
+    /// Adds a reduction clause variable to this operation, including its
+    /// recipe.
+    void addReduction(MLIRContext *, mlir::acc::ReductionOp op,
+                      mlir::acc::ReductionRecipeOp recipe);
   }];
 
   let hasCustomAssemblyFormat = 1;
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
index 72ce4c6a21cb3..c9e6764e7d634 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPAttrDefs.td
@@ -42,10 +42,10 @@ def AtomicControlAttr : OpenMP_Attr<"AtomicControl", "atomic_control"> {
 //===----------------------------------------------------------------------===//
 
 def DeclareTargetAttr : OpenMP_Attr<"DeclareTarget", "declaretarget"> {
-  let parameters = (ins
-    OptionalParameter<"DeclareTargetDeviceTypeAttr">:$device_type,
-    OptionalParameter<"DeclareTargetCaptureClauseAttr">:$capture_clause
-  );
+  let parameters =
+      (ins OptionalParameter<"DeclareTargetDeviceTypeAttr">:$device_type,
+          OptionalParameter<"DeclareTargetCaptureClauseAttr">:$capture_clause,
+          OptionalParameter<"BoolAttr">:$automap);
 
   let assemblyFormat = "`<` struct(params) `>`";
 }
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index be114ea4fb631..2548a8ab4aac6 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -2115,4 +2115,121 @@ def AllocateDirOp : OpenMP_Op<"allocate_dir", clauses = [
   let hasVerifier = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetAllocMemOp : OpenMP_Op<"target_allocmem",
+    [MemoryEffects<[MemAlloc<DefaultResource>]>, AttrSizedOperandSegments]> {
+  let summary = "allocate storage on an openmp device for an object of a given type";
+
+  let description = [{
+    Allocates memory on the specified OpenMP device for an object of the given type.
+    Returns an integer value representing the device pointer to the allocated memory.
+    The memory is uninitialized after allocation. Operations must be paired with 
+    `omp.target_freemem` to avoid memory leaks.
+
+    * `$device`: The integer ID of the OpenMP device where the memory will be allocated.
+    * `$in_type`: The type of the object for which memory is being allocated. 
+      For arrays, this can be a static or dynamic array type.
+    * `$uniq_name`: An optional unique name for the allocated memory.
+    * `$bindc_name`: An optional name used for C interoperability.
+    * `$typeparams`: Runtime type parameters for polymorphic or parameterized types. 
+      These are typically integer values that define aspects of a type not fixed at compile time.
+    * `$shape`: Runtime shape operands for dynamic arrays. 
+      Each operand is an integer value representing the extent of a specific dimension. 
+
+  ```mlir
+    // Allocate a static 3x3 integer vector on device 0
+    %device_0 = arith.constant 0 : i32
+    %ptr_static = omp.target_allocmem %device_0 : i32, vector<3x3xi32>
+    // ... use %ptr_static ...
+    omp.target_freemem %device_0, %ptr_static : i32, i64
+
+    // Allocate a dynamic 2D Fortran array (fir.array) on device 1
+    %device_1 = arith.constant 1 : i32
+    %rows = arith.constant 10 : index
+    %cols = arith.constant 20 : index
+    %ptr_dynamic = omp.target_allocmem %device_1 : i32, !fir.array<?x?xf32>, %rows, %cols : index, index
+    // ... use %ptr_dynamic ...
+    omp.target_freemem %device_1, %ptr_dynamic : i32, i64
+  ```
+  }];
+
+  let arguments = (ins
+    Arg<AnyInteger>:$device,
+    TypeAttr:$in_type,
+    OptionalAttr<StrAttr>:$uniq_name,
+    OptionalAttr<StrAttr>:$bindc_name,
+    Variadic<IntLikeType>:$typeparams,
+    Variadic<IntLikeType>:$shape
+  );
+  let results = (outs I64);
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+
+  let extraClassDeclaration = [{
+    mlir::Type getAllocatedType();
+  }];
+}
+
+//===----------------------------------------------------------------------===//
+// TargetFreeMemOp
+//===----------------------------------------------------------------------===//
+
+def TargetFreeMemOp : OpenMP_Op<"target_freemem",
+  [MemoryEffects<[MemFree]>]> {
+  let summary = "free memory on an openmp device";
+
+  let description = [{
+    Deallocates memory on the specified OpenMP device that was previously
+    allocated by an `omp.target_allocmem` operation. After this operation, the
+    deallocated memory is in an undefined state and should not be accessed.
+    It is crucial to ensure that all accesses to the memory region are completed
+    before `omp.target_freemem` is called to avoid undefined behavior.
+
+    * `$device`: The integer ID of the OpenMP device from which the memory will be freed.
+    * `$heapref`: The integer value representing the device pointer to the memory
+      to be deallocated, which was previously returned by `omp.target_allocmem`.
+
+    ```mlir
+      // Example of allocating and freeing memory on an OpenMP device
+      %device_id = arith.constant 0 : i32
+      %allocated_ptr = omp.target_allocmem %device_id : i32, vector<3x3xi32>
+      // ... operations using %allocated_ptr on the device ...
+      omp.target_freemem %device_id, %allocated_ptr : i32, i64
+    ```
+  }];
+
+  let arguments = (ins
+  Arg<AnyInteger, "", [MemFree]>:$device,
+  Arg<I64, "", [MemFree]>:$heapref
+  );
+  let assemblyFormat = "$device `,` $heapref attr-dict `:` type($device) `,` qualified(type($heapref))";
+}
+
+//===----------------------------------------------------------------------===//
+// workdistribute Construct
+//===----------------------------------------------------------------------===//
+
+def WorkdistributeOp : OpenMP_Op<"workdistribute"> {
+  let summary = "workdistribute directive";
+  let description = [{
+    workdistribute divides execution of the enclosed structured block into
+    separate units of work, each executed only once by each
+    initial thread in the league.
+    ```
+    !$omp target teams
+        !$omp workdistribute
+        y = a * x + y 
+        !$omp end workdistribute
+    !$omp end target teams
+    ```
+  }];
+  let regions = (region AnyRegion:$region);
+  let hasVerifier = 1;
+  let assemblyFormat = "$region attr-dict";
+}
+
 #endif // OPENMP_OPS
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
index 0dc385b08a823..d471e6c0ed70b 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
@@ -329,14 +329,16 @@ def DeclareTargetInterface : OpInterface<"DeclareTargetInterface"> {
       /*retTy=*/"void",
       /*methodName=*/"setDeclareTarget",
       (ins "mlir::omp::DeclareTargetDeviceType":$deviceType,
-            "mlir::omp::DeclareTargetCaptureClause":$captureClause), [{}], [{
+           "mlir::omp::DeclareTargetCaptureClause":$captureClause,
+           "bool":$automap), [{}], [{
         $_op->setAttr("omp.declare_target",
                   mlir::omp::DeclareTargetAttr::get(
                       $_op->getContext(),
                       mlir::omp::DeclareTargetDeviceTypeAttr::get(
                           $_op->getContext(), deviceType),
                       mlir::omp::DeclareTargetCaptureClauseAttr::get(
-                          $_op->getContext(), captureClause)));
+                          $_op->getContext(), captureClause),
+                      mlir::BoolAttr::get($_op->getContext(), automap)));
       }]>,
       InterfaceMethod<
       /*description=*/[{
@@ -374,6 +376,19 @@ def DeclareTargetInterface : OpInterface<"DeclareTargetInterface"> {
           if (auto dAttr = llvm::dyn_cast_or_null<mlir::omp::DeclareTargetAttr>(dTar))
             return dAttr.getCaptureClause().getValue();
         return {};
+      }]>,
+      InterfaceMethod<
+      /*description=*/[{
+        Return true if the DeclareTarget attribute has the AUTOMAP modifier.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"getDeclareTargetAutomap",
+      (ins), [{}], [{
+        if (mlir::Attribute dTar = $_op->getAttr("omp.declare_target"))
+          if (auto dAttr = llvm::dyn_cast_or_null<mlir::omp::DeclareTargetAttr>(dTar))
+            if (auto autoVal = dAttr.getAutomap())
+              return autoVal.getValue();
+         return false;
       }]>
   ];
 }
diff --git a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
index 0c1c15b85f4c9..88df54174da24 100644
--- a/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
+++ b/mlir/include/mlir/Dialect/SCF/IR/SCFOps.td
@@ -169,9 +169,13 @@ def ForOp : SCF_Op<"for",
     region capturing the loop body. The induction variable is represented as an
     argument of this region. This SSA value is a signless integer or index.
     The step is a value of same type but required to be positive, the lower and
-    upper bounds can be also negative or zero. The lower and upper bounds specify
-    a half-open range: the iteration is executed iff the signed comparison of induction
-    variable value is less than the upper bound and bigger or equal to the lower bound.
+    upper bounds can be also negative or zero. The lower and upper bounds
+    specify a half-open range: the iteration is executed iff the comparison of
+    induction variable value is less than the upper bound and bigger or equal
+    to the lower bound.
+
+    By default, the integer comparison is signed. If the `unsignedCmp` unit
+    attribute is specified, the integer comparison is unsigned.
 
     The body region must contain exactly one block that terminates with
     `scf.yield`. Calling ForOp::build will create such a region and insert
@@ -184,8 +188,8 @@ def ForOp : SCF_Op<"for",
       ... // body
     }
     ...
-    // Integer case.
-    scf.for %iv_32 = %lb_32 to %ub_32 step %step_32 : i32 {
+    // Unsigned integer case.
+    scf.for unsigned %iv_32 = %lb_32 to %ub_32 step %step_32 : i32 {
       ... // body
     }
     ```
@@ -258,7 +262,8 @@ def ForOp : SCF_Op<"for",
   let arguments = (ins AnySignlessIntegerOrIndex:$lowerBound,
                        AnySignlessIntegerOrIndex:$upperBound,
                        AnySignlessIntegerOrIndex:$step,
-                       Variadic<AnyType>:$initArgs);
+                       Variadic<AnyType>:$initArgs,
+                       UnitAttr:$unsignedCmp);
   let results = (outs Variadic<AnyType>:$results);
   let regions = (region SizedRegion<1>:$region);
 
@@ -266,7 +271,8 @@ def ForOp : SCF_Op<"for",
   let builders = [OpBuilder<(ins "Value":$lowerBound, "Value":$upperBound,
       "Value":$step, CArg<"ValueRange", "{}">:$initArgs,
       CArg<"function_ref<void(OpBuilder &, Location, Value, ValueRange)>",
-           "nullptr">)>];
+           "nullptr">,
+      CArg<"bool", "false">:$unsignedCmp)>];
 
   let extraClassDeclaration = [{
     using BodyBuilderFn =
diff --git a/mlir/include/mlir/Dialect/SPIRV/Interfaces/SPIRVImageInterfaces.h b/mlir/include/mlir/Dialect/SPIRV/Interfaces/SPIRVImageInterfaces.h
index ca5a1dcd88171..0c70d4fafd246 100644
--- a/mlir/include/mlir/Dialect/SPIRV/Interfaces/SPIRVImageInterfaces.h
+++ b/mlir/include/mlir/Dialect/SPIRV/Interfaces/SPIRVImageInterfaces.h
@@ -9,6 +9,8 @@
 #ifndef MLIR_DIALECT_SPIRV_IMAGE_INTERFACES_H_
 #define MLIR_DIALECT_SPIRV_IMAGE_INTERFACES_H_
 
+#include "mlir/IR/OpDefinition.h"
+
 #include "mlir/Dialect/SPIRV/Interfaces/SPIRVImageInterfaces.h.inc"
 
 #endif // MLIR_DIALECT_SPIRV_IMAGE_INTERFACES_H_
diff --git a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
index 87deef9ca7466..3e4da94bd714e 100644
--- a/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Tensor/Transforms/Transforms.h
@@ -142,6 +142,32 @@ FailureOr<Value> buildIndependentOp(OpBuilder &b, tensor::PadOp padOp,
 FailureOr<Value> buildIndependentOp(OpBuilder &b, tensor::EmptyOp emptyOp,
                                     ValueRange independencies);
 
+/// Computes the offsets, sizes, and strides needed to build a collapsed
+/// `sliceOp`. The dimensions to collapse are specified by `reassociation`.
+///
+/// This fails when the specified collapse cannot be represented by a valid
+/// ExtractSliceOp.
+LogicalResult
+getCollapsedExtractSliceInfo(OpBuilder &b, tensor::ExtractSliceOp sliceOp,
+                             ArrayRef<ReassociationIndices> reassociation,
+                             SmallVectorImpl<OpFoldResult> &collapsedOffsets,
+                             SmallVectorImpl<OpFoldResult> &collapsedSizes,
+                             SmallVectorImpl<OpFoldResult> &collapsedStrides);
+
+/// Computes the offsets, sizes, and strides needed to build an expanded
+/// `sliceOp`. The dimensions to expand are specified by `reassociation` and
+/// `expandedShape`.
+///
+/// This fails when the specified expansion cannot be represented by a valid
+/// ExtractSliceOp.
+LogicalResult
+getExpandedExtractSliceInfo(OpBuilder &b, tensor::ExtractSliceOp sliceOp,
+                            ArrayRef<ReassociationIndices> reassociation,
+                            ArrayRef<int64_t> expandedShape,
+                            SmallVectorImpl<OpFoldResult> &expandedOffsets,
+                            SmallVectorImpl<OpFoldResult> &expandedSizes,
+                            SmallVectorImpl<OpFoldResult> &expandedStrides);
+
 } // namespace tensor
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
index 364c1728715e8..63410b8bea747 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.h
@@ -32,6 +32,7 @@
 #include "mlir/Interfaces/ViewLikeInterface.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Alignment.h"
 
 // Pull in all enum type definitions and utility function declarations.
 #include "mlir/Dialect/Vector/IR/VectorEnums.h.inc"
diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index dc55704c36183..bcc423a634148 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -754,7 +754,7 @@ def Vector_FMAOp :
 }
 
 def Vector_ToElementsOp : Vector_Op<"to_elements", [
-    Pure,
+    InferTypeOpAdaptor, Pure,
     ShapedTypeMatchesElementCountAndTypes<"source", "elements">]> {
   let summary = "operation that decomposes a vector into all its scalar elements";
   let description = [{
@@ -1714,7 +1714,6 @@ def Vector_LoadOp : Vector_Op<"load", [
     load operation. It must be a positive power of 2. The operation must access
     memory at an address aligned to this boundary. Violations may lead to
     architecture-specific faults or performance penalties.
-    A value of 0 indicates no specific alignment requirement.
   }];
 
   let arguments = (ins Arg<AnyMemRef, "the reference to load from",
@@ -1729,18 +1728,18 @@ def Vector_LoadOp : Vector_Op<"load", [
                    "Value":$base,
                    "ValueRange":$indices,
                    CArg<"bool", "false">:$nontemporal,
-                   CArg<"uint64_t", "0">:$alignment), [{
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
       return build($_builder, $_state, resultType, base, indices, nontemporal,
-                   alignment != 0 ? $_builder.getI64IntegerAttr(alignment) :
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
                                     nullptr);
     }]>,
     OpBuilder<(ins "TypeRange":$resultTypes,
                    "Value":$base,
                    "ValueRange":$indices,
                    CArg<"bool", "false">:$nontemporal,
-                   CArg<"uint64_t", "0">:$alignment), [{
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
       return build($_builder, $_state, resultTypes, base, indices, nontemporal,
-                   alignment != 0 ? $_builder.getI64IntegerAttr(alignment) :
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
                                     nullptr);
     }]>
   ];
@@ -1830,7 +1829,6 @@ def Vector_StoreOp : Vector_Op<"store", [
     store operation. It must be a positive power of 2. The operation must access
     memory at an address aligned to this boundary. Violations may lead to
     architecture-specific faults or performance penalties.
-    A value of 0 indicates no specific alignment requirement.
   }];
 
   let arguments = (ins
@@ -1847,9 +1845,9 @@ def Vector_StoreOp : Vector_Op<"store", [
                    "Value":$base,
                    "ValueRange":$indices,
                    CArg<"bool", "false">:$nontemporal,
-                   CArg<"uint64_t", "0">:$alignment), [{
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
       return build($_builder, $_state, valueToStore, base, indices, nontemporal,
-                   alignment != 0 ? $_builder.getI64IntegerAttr(alignment) :
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
                                     nullptr);
     }]>
   ];
@@ -1876,7 +1874,9 @@ def Vector_MaskedLoadOp :
     Arguments<(ins Arg<AnyMemRef, "", [MemRead]>:$base,
                Variadic<Index>:$indices,
                VectorOfNonZeroRankOf<[I1]>:$mask,
-               AnyVectorOfNonZeroRank:$pass_thru)>,
+               AnyVectorOfNonZeroRank:$pass_thru,
+               ConfinedAttr<OptionalAttr<I64Attr>,
+                   [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)>,
     Results<(outs AnyVectorOfNonZeroRank:$result)> {
 
   let summary = "loads elements from memory into a vector as defined by a mask vector";
@@ -1912,6 +1912,11 @@ def Vector_MaskedLoadOp :
     %1 = vector.maskedload %base[%i, %j], %mask, %pass_thru
        : memref<?x?xf32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
     ```
+
+    An optional `alignment` attribute allows to specify the byte alignment of the
+    load operation. It must be a positive power of 2. The operation must access
+    memory at an address aligned to this boundary. Violations may lead to
+    architecture-specific faults or performance penalties.
   }];
   let extraClassDeclaration = [{
     MemRefType getMemRefType() {
@@ -1932,6 +1937,29 @@ def Vector_MaskedLoadOp :
   let hasCanonicalizer = 1;
   let hasFolder = 1;
   let hasVerifier = 1;
+
+  let builders = [
+    OpBuilder<(ins "VectorType":$resultType,
+                   "Value":$base,
+                   "ValueRange":$indices,
+                   "Value":$mask,
+                   "Value":$passthrough,
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
+      return build($_builder, $_state, resultType, base, indices, mask, passthrough,
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
+                                    nullptr);
+    }]>,
+    OpBuilder<(ins "TypeRange":$resultTypes,
+                   "Value":$base,
+                   "ValueRange":$indices,
+                   "Value":$mask,
+                   "Value":$passthrough,
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
+      return build($_builder, $_state, resultTypes, base, indices, mask, passthrough,
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
+                                    nullptr);
+    }]>
+  ];
 }
 
 def Vector_MaskedStoreOp :
@@ -1939,7 +1967,9 @@ def Vector_MaskedStoreOp :
     Arguments<(ins Arg<AnyMemRef, "", [MemWrite]>:$base,
                Variadic<Index>:$indices,
                VectorOfNonZeroRankOf<[I1]>:$mask,
-               AnyVectorOfNonZeroRank:$valueToStore)> {
+               AnyVectorOfNonZeroRank:$valueToStore,
+               ConfinedAttr<OptionalAttr<I64Attr>,
+                   [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)> {
 
   let summary = "stores elements from a vector into memory as defined by a mask vector";
 
@@ -1974,6 +2004,11 @@ def Vector_MaskedStoreOp :
     vector.maskedstore %base[%i, %j], %mask, %value
       : memref<?x?xf32>, vector<16xi1>, vector<16xf32>
     ```
+
+    An optional `alignment` attribute allows to specify the byte alignment of the
+    store operation. It must be a positive power of 2. The operation must access
+    memory at an address aligned to this boundary. Violations may lead to
+    architecture-specific faults or performance penalties.
   }];
   let extraClassDeclaration = [{
     MemRefType getMemRefType() {
@@ -1992,6 +2027,18 @@ def Vector_MaskedStoreOp :
   let hasCanonicalizer = 1;
   let hasFolder = 1;
   let hasVerifier = 1;
+
+  let builders = [
+    OpBuilder<(ins "Value":$base,
+                   "ValueRange":$indices,
+                   "Value":$mask,
+                   "Value":$valueToStore,
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
+      return build($_builder, $_state, base, indices, mask, valueToStore,
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
+                                    nullptr);
+    }]>
+  ];
 }
 
 def Vector_GatherOp :
@@ -2003,27 +2050,44 @@ def Vector_GatherOp :
                Variadic<Index>:$indices,
                VectorOfNonZeroRankOf<[AnyInteger, Index]>:$index_vec,
                VectorOfNonZeroRankOf<[I1]>:$mask,
-               AnyVectorOfNonZeroRank:$pass_thru)>,
+               AnyVectorOfNonZeroRank:$pass_thru,
+               ConfinedAttr<OptionalAttr<I64Attr>,
+                   [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)>,
     Results<(outs AnyVectorOfNonZeroRank:$result)> {
 
   let summary = [{
-    gathers elements from memory or ranked tensor into a vector as defined by an
-    index vector and a mask vector
+    Gathers elements from memory or ranked tensor into a vector as defined by an
+    index vector and a mask vector.
   }];
 
   let description = [{
     The gather operation returns an n-D vector whose elements are either loaded
-    from memory or ranked tensor, or taken from a pass-through vector, depending
+    from a k-D memref or tensor, or taken from an n-D pass-through vector, depending
     on the values of an n-D mask vector.
-    If a mask bit is set, the corresponding result element is defined by the base
-    with indices and the n-D index vector (each index is a 1-D offset on the base).
-    Otherwise, the corresponding element is taken from the n-D pass-through vector.
-    Informally the semantics are:
+
+    If a mask bit is set, the corresponding result element is taken from `base`
+    at an index defined by k indices and n-D `index_vec`. Otherwise, the element
+    is taken from the pass-through vector. As an example, suppose that `base` is
+    3-D and the result is 2-D:
+
+    ```mlir
+    func.func @gather_3D_to_2D(
+        %base: memref<?x10x?xf32>, %i0: index, %i1: index, %i2: index,
+        %index_vec: vector<2x3xi32>, %mask: vector<2x3xi1>,
+        %fall_thru: vector<2x3xf32>) -> vector<2x3xf32> {
+            %result = vector.gather %base[%i0, %i1, %i2]
+                                   [%index_vec], %mask, %fall_thru : [...]
+            return %result : vector<2x3xf32>
+    }
     ```
-    result[0] := if mask[0] then base[index[0]] else pass_thru[0]
-    result[1] := if mask[1] then base[index[1]] else pass_thru[1]
-    etc.
+
+    The indexing semantics are then,
+
     ```
+    result[i,j] := if mask[i,j] then base[i0, i1, i2 + index_vec[i,j]]
+                   else pass_thru[i,j]
+    ```
+    The index into `base` only varies in the innermost ((k-1)-th) dimension.
 
     If a mask bit is set and the corresponding index is out-of-bounds for the
     given base, the behavior is undefined. If a mask bit is not set, the value
@@ -2034,12 +2098,19 @@ def Vector_GatherOp :
     during progressively lowering to bring other memory operations closer to
     hardware ISA support for a gather.
 
+    An optional `alignment` attribute allows to specify the byte alignment of the
+    gather operation. It must be a positive power of 2. The operation must access
+    memory at an address aligned to this boundary. Violations may lead to
+    architecture-specific faults or performance penalties.
+
     Examples:
 
     ```mlir
+    // 1-D memref gathered to 2-D vector.
     %0 = vector.gather %base[%c0][%v], %mask, %pass_thru
        : memref<?xf32>, vector<2x16xi32>, vector<2x16xi1>, vector<2x16xf32> into vector<2x16xf32>
 
+    // 2-D memref gathered to 1-D vector.
     %1 = vector.gather %base[%i, %j][%v], %mask, %pass_thru
        : memref<16x16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
     ```
@@ -2060,6 +2131,20 @@ def Vector_GatherOp :
     "`into` type($result)";
   let hasCanonicalizer = 1;
   let hasVerifier = 1;
+
+  let builders = [
+    OpBuilder<(ins "VectorType":$resultType,
+                   "Value":$base,
+                   "ValueRange":$indices,
+                   "Value":$index_vec,
+                   "Value":$mask,
+                   "Value":$passthrough,
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
+      return build($_builder, $_state, resultType, base, indices, index_vec, mask, passthrough,
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
+                                    nullptr);
+    }]>
+  ];
 }
 
 def Vector_ScatterOp :
@@ -2068,7 +2153,9 @@ def Vector_ScatterOp :
                Variadic<Index>:$indices,
                VectorOfNonZeroRankOf<[AnyInteger, Index]>:$index_vec,
                VectorOfNonZeroRankOf<[I1]>:$mask,
-               AnyVectorOfNonZeroRank:$valueToStore)> {
+               AnyVectorOfNonZeroRank:$valueToStore,
+               ConfinedAttr<OptionalAttr<I64Attr>,
+                   [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)> {
 
   let summary = [{
     scatters elements from a vector into memory as defined by an index vector
@@ -2102,6 +2189,11 @@ def Vector_ScatterOp :
     correspond to those of the `llvm.masked.scatter`
     [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-scatter-intrinsics).
 
+    An optional `alignment` attribute allows to specify the byte alignment of the
+    scatter operation. It must be a positive power of 2. The operation must access
+    memory at an address aligned to this boundary. Violations may lead to
+    architecture-specific faults or performance penalties.
+
     Examples:
 
     ```mlir
@@ -2126,6 +2218,19 @@ def Vector_ScatterOp :
       "type($index_vec)  `,` type($mask) `,` type($valueToStore)";
   let hasCanonicalizer = 1;
   let hasVerifier = 1;
+
+  let builders = [
+    OpBuilder<(ins "Value":$base,
+                   "ValueRange":$indices,
+                   "Value":$index_vec,
+                   "Value":$mask,
+                   "Value":$valueToStore,
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">: $alignment), [{
+      return build($_builder, $_state, base, indices, index_vec, mask, valueToStore,
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
+                                    nullptr);
+    }]>
+  ];
 }
 
 def Vector_ExpandLoadOp :
@@ -2133,7 +2238,9 @@ def Vector_ExpandLoadOp :
     Arguments<(ins Arg<AnyMemRef, "", [MemRead]>:$base,
                Variadic<Index>:$indices,
                FixedVectorOfNonZeroRankOf<[I1]>:$mask,
-               AnyVectorOfNonZeroRank:$pass_thru)>,
+               AnyVectorOfNonZeroRank:$pass_thru,
+               ConfinedAttr<OptionalAttr<I64Attr>,
+                            [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)>,
     Results<(outs AnyVectorOfNonZeroRank:$result)> {
 
   let summary = "reads elements from memory and spreads them into a vector as defined by a mask";
@@ -2165,6 +2272,11 @@ def Vector_ExpandLoadOp :
     correspond to those of the `llvm.masked.expandload`
     [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics).
 
+    An optional `alignment` attribute allows to specify the byte alignment of the
+    load operation. It must be a positive power of 2. The operation must access
+    memory at an address aligned to this boundary. Violations may lead to
+    architecture-specific faults or performance penalties.
+
     Note, at the moment this Op is only available for fixed-width vectors.
 
     Examples:
@@ -2195,6 +2307,19 @@ def Vector_ExpandLoadOp :
     "type($base) `,` type($mask) `,` type($pass_thru) `into` type($result)";
   let hasCanonicalizer = 1;
   let hasVerifier = 1;
+
+  let builders = [
+    OpBuilder<(ins "VectorType":$resultType,
+                   "Value":$base,
+                   "ValueRange":$indices,
+                   "Value":$mask,
+                   "Value":$passthrough,
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
+      return build($_builder, $_state, resultType, base, indices, mask, passthrough,
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
+                                    nullptr);
+    }]>
+  ];
 }
 
 def Vector_CompressStoreOp :
@@ -2202,7 +2327,9 @@ def Vector_CompressStoreOp :
     Arguments<(ins Arg<AnyMemRef, "", [MemWrite]>:$base,
                Variadic<Index>:$indices,
                FixedVectorOfNonZeroRankOf<[I1]>:$mask,
-               AnyVectorOfNonZeroRank:$valueToStore)> {
+               AnyVectorOfNonZeroRank:$valueToStore,
+               ConfinedAttr<OptionalAttr<I64Attr>,
+                            [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)> {
 
   let summary = "writes elements selectively from a vector as defined by a mask";
 
@@ -2233,6 +2360,11 @@ def Vector_CompressStoreOp :
     correspond to those of the `llvm.masked.compressstore`
     [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics).
 
+    An optional `alignment` attribute allows to specify the byte alignment of the
+    store operation. It must be a positive power of 2. The operation must access
+    memory at an address aligned to this boundary. Violations may lead to
+    architecture-specific faults or performance penalties.
+
     Note, at the moment this Op is only available for fixed-width vectors.
 
     Examples:
@@ -2261,6 +2393,17 @@ def Vector_CompressStoreOp :
       "type($base) `,` type($mask) `,` type($valueToStore)";
   let hasCanonicalizer = 1;
   let hasVerifier = 1;
+  let builders = [
+    OpBuilder<(ins "Value":$base,
+                   "ValueRange":$indices,
+                   "Value":$mask,
+                   "Value":$valueToStore,
+                   CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{
+      return build($_builder, $_state, base, indices, valueToStore, mask,
+                   alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) :
+                                    nullptr);
+    }]>
+  ];
 }
 
 def Vector_ShapeCastOp :
diff --git a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
index 299f198e4ab9c..07a4117a37b2c 100644
--- a/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
+++ b/mlir/include/mlir/Dialect/Vector/TransformOps/VectorTransformOps.td
@@ -254,6 +254,17 @@ def ApplyLowerGatherPatternsOp : Op<Transform_Dialect,
   let assemblyFormat = "attr-dict";
 }
 
+def ApplyUnrollFromElementsPatternsOp : Op<Transform_Dialect,
+    "apply_patterns.vector.unroll_from_elements",
+    [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
+  let description = [{
+    Indicates that vector from_elements operations should be unrolled
+    along the outermost dimension.
+  }];
+
+  let assemblyFormat = "attr-dict";
+}
+
 def ApplyLowerScanPatternsOp : Op<Transform_Dialect,
     "apply_patterns.vector.lower_scan",
     [DeclareOpInterfaceMethods<PatternDescriptorOpInterface>]> {
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
index e03f0dabece52..47f96112a9433 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/LoweringPatterns.h
@@ -303,6 +303,14 @@ void populateVectorRankReducingFMAPattern(RewritePatternSet &patterns);
 void populateVectorToFromElementsToShuffleTreePatterns(
     RewritePatternSet &patterns, PatternBenefit benefit = 1);
 
+/// Populate the pattern set with the following patterns:
+///
+/// [UnrollFromElements]
+/// Unrolls 2 or more dimensional `vector.from_elements` ops by unrolling the
+/// outermost dimension.
+void populateVectorFromElementsLoweringPatterns(RewritePatternSet &patterns,
+                                                PatternBenefit benefit = 1);
+
 /// Populate the pattern set with the following patterns:
 ///
 /// [ContractionOpToMatmulOpLowering]
diff --git a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h
index 7cd70e42d363c..ace26990601c8 100644
--- a/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h
+++ b/mlir/include/mlir/Dialect/Vector/Utils/VectorUtils.h
@@ -12,6 +12,7 @@
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinAttributes.h"
@@ -228,7 +229,7 @@ bool isLinearizableVector(VectorType type);
 Value createReadOrMaskedRead(OpBuilder &builder, Location loc, Value source,
                              ArrayRef<int64_t> inputVectorSizes, Value padValue,
                              bool useInBoundsInsteadOfMasking = false,
-                             ArrayRef<bool> scalableDims = {});
+                             ArrayRef<bool> inputScalableVecDims = {});
 
 /// Returns success if `inputVectorSizes` is a valid masking configuraion for
 /// given `shape`, i.e., it meets:
@@ -238,6 +239,22 @@ Value createReadOrMaskedRead(OpBuilder &builder, Location loc, Value source,
 ///      static sizes in `shape`.
 LogicalResult isValidMaskedInputVector(ArrayRef<int64_t> shape,
                                        ArrayRef<int64_t> inputVectorSizes);
+
+/// Generic utility for unrolling n-D vector operations to (n-1)-D operations.
+/// This handles the common pattern of:
+/// 1. Check if already 1-D. If so, return failure.
+/// 2. Check for scalable dimensions. If so, return failure.
+/// 3. Create poison initialized result.
+/// 4. Loop through the outermost dimension, execute the UnrollVectorOpFn to
+/// create sub vectors.
+/// 5. Insert the sub vectors back into the final vector.
+/// 6. Replace the original op with the new result.
+using UnrollVectorOpFn =
+    function_ref<Value(PatternRewriter &, Location, VectorType, int64_t)>;
+
+LogicalResult unrollVectorOp(Operation *op, PatternRewriter &rewriter,
+                             UnrollVectorOpFn unrollFn);
+
 } // namespace vector
 
 /// Constructs a permutation map of invariant memref indices to vector
diff --git a/mlir/include/mlir/Dialect/WasmSSA/IR/WasmSSAOps.td b/mlir/include/mlir/Dialect/WasmSSA/IR/WasmSSAOps.td
index 676621b176f5c..b80ee2c38546a 100644
--- a/mlir/include/mlir/Dialect/WasmSSA/IR/WasmSSAOps.td
+++ b/mlir/include/mlir/Dialect/WasmSSA/IR/WasmSSAOps.td
@@ -329,11 +329,11 @@ def WasmSSA_IfOp : WasmSSA_Op<"if", [Terminator,
 
      ```mlir
      // Runs the if clause is %a is non-zero
-     "wasmssa.if"(%a)[^bb1] ({
+     wasmssa.if %a {
         // Execute if %a is non-zero
-     },{
+     } else {
         // else clause
-     }) : (i32) -> ()
+     }
      ```
     }];
   let arguments = (ins I32:$condition, Variadic<WasmSSA_ValType>: $inputs);
@@ -359,6 +359,7 @@ def WasmSSA_IfOp : WasmSSA_Op<"if", [Terminator,
       return createBlock(getElse());
     }
   }];
+  let assemblyFormat = "$condition (`(`$inputs^`)` `:` type($inputs))? attr-dict  `:` $if custom<ElseRegion>($else) `>` $target";
 }
 
 def WasmSSA_LocalOp : WasmSSA_Op<"local", [
@@ -445,7 +446,7 @@ def WasmSSA_MemOp : WasmSSA_Op<"memory", [Symbol]> {
 
      ```mlir
      // Define the `mem_0` memory with defined bounds of 0 -> 65536
-     "wasmssa.memory"() <{limits = !wasmssa<limit[0:65536]>, sym_name = "mem_0"}> : () -> ()
+     wasmssa.memory @mem_0 !wasmssa<limit[0:65536]>
      ```
     }];
   let arguments = (ins SymbolNameAttr: $sym_name,
@@ -456,6 +457,8 @@ def WasmSSA_MemOp : WasmSSA_Op<"memory", [Symbol]> {
     "::llvm::StringRef":$symbol,
     "wasmssa::LimitType":$limit)>
   ];
+
+  let assemblyFormat = "$sym_name custom<WasmVisibility>($sym_visibility) $limits attr-dict";
 }
 
 def WasmSSA_MemImportOp : WasmSSA_Op<"import_mem", [Symbol, ImportOpInterface]> {
@@ -494,6 +497,7 @@ def WasmSSA_TableOp : WasmSSA_Op<"table", [Symbol]> {
   let builders = [OpBuilder<(ins
       "::llvm::StringRef":$symbol,
       "wasmssa::TableType":$type)>];
+  let assemblyFormat = "$sym_name custom<WasmVisibility>($sym_visibility) $type attr-dict";
 }
 
 def WasmSSA_TableImportOp : WasmSSA_Op<"import_table", [Symbol, ImportOpInterface]> {
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/XeGPU/IR/CMakeLists.txt
index 3f8cac4dc07c3..728f1aa859061 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/CMakeLists.txt
@@ -12,3 +12,9 @@ mlir_tablegen(XeGPUEnums.h.inc -gen-enum-decls)
 mlir_tablegen(XeGPUEnums.cpp.inc -gen-enum-defs)
 add_public_tablegen_target(MLIRXeGPUEnumsIncGen)
 add_dependencies(mlir-headers MLIRXeGPUEnumsIncGen)
+
+set(LLVM_TARGET_DEFINITIONS XeGPUAttrs.td)
+mlir_tablegen(XeGPUAttrInterface.h.inc -gen-attr-interface-decls)
+mlir_tablegen(XeGPUAttrInterface.cpp.inc -gen-attr-interface-defs)
+add_public_tablegen_target(MLIRXeGPUAttrInterfaceIncGen)
+add_dependencies(mlir-headers MLIRXeGPUAttrInterfaceIncGen)
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h
index 8e2784f40ad39..1481859e94a92 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h
@@ -11,10 +11,12 @@
 
 #include "mlir/Bytecode/BytecodeOpInterface.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/TypeUtilities.h"
+#include "mlir/IR/Value.h"
 #include "mlir/Interfaces/ShapedOpInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
@@ -22,17 +24,20 @@
 namespace mlir {
 namespace xegpu {
 class TensorDescType;
+class DistributeLayoutAttr;
+class LayoutAttr;
+class SliceAttr;
 } // namespace xegpu
 } // namespace mlir
 
+#include <mlir/Dialect/XeGPU/IR/XeGPUAttrInterface.h.inc>
+#include <mlir/Dialect/XeGPU/IR/XeGPUDialect.h.inc>
 #include <mlir/Dialect/XeGPU/IR/XeGPUEnums.h.inc>
+
 #define GET_ATTRDEF_CLASSES
 #include <mlir/Dialect/XeGPU/IR/XeGPUAttrs.h.inc>
 #define GET_TYPEDEF_CLASSES
 #include <mlir/Dialect/XeGPU/IR/XeGPUTypes.h.inc>
-
-#include <mlir/Dialect/XeGPU/IR/XeGPUDialect.h.inc>
-
 #define GET_OP_CLASSES
 #include <mlir/Dialect/XeGPU/IR/XeGPU.h.inc>
 
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
index 64eb21cbc3c4c..b4d696444cc44 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
@@ -175,7 +175,52 @@ def XeGPU_FenceScopeAttr:
     let assemblyFormat = "$value";
 }
 
-def XeGPU_LayoutAttr : XeGPUAttr<"Layout", "layout"> {
+def DistributeLayoutAttr: AttrInterface<"DistributeLayoutAttr"> {
+  let cppNamespace = "::mlir::xegpu";
+  let description = [{
+    Common trait for all XeGPU layouts.
+  }];
+
+  let methods = [
+    InterfaceMethod<"Check the availability of workgroup level layouts",
+                    "bool",
+                    "isForWorkgroup">,
+    InterfaceMethod<"Get the rank of attribute",
+                    "int64_t",
+                    "getRank">,
+    InterfaceMethod<"Get the num of effective subgroups",
+                    "int64_t",
+                    "getNumSubgroups", (ins), [{
+                        std::optional<SmallVector<int64_t>> sgLayout = llvm::cast<ConcreteAttr>(tablegen_opaque_val).getSgLayoutAsInt();
+                        if (sgLayout.has_value())
+                          return computeProduct(*sgLayout);
+                        return 0;
+                    }], [{}]>,
+    InterfaceMethod<"Get the SgLayout field of the attribute as integer array",
+                    "std::optional<SmallVector<int64_t>>",
+                    "getSgLayoutAsInt">,
+    InterfaceMethod<"Get the SgData field of the attribute as integer array",
+                    "std::optional<SmallVector<int64_t>>",
+                    "getSgDataAsInt">,
+    InterfaceMethod<"Derive a new layout by dropping sgLayout and sgData",
+                    "xegpu::DistributeLayoutAttr",
+                    "dropSgLayoutAndData">,
+    InterfaceMethod<[{Delinearizes a linear subgroup ID into its multidimensional
+                      indices based on the effective subgroup layout.}],
+                    "FailureOr<SmallVector<Value>>",
+                    "delinearizeSubgroupId",
+                    (ins "OpBuilder &": $builder, "Location":$loc, "Value":$linearId)>,
+    InterfaceMethod<[{Generates instructions to compute multidimensional offsets for blocks
+                      assigned to a subgroup identified by linearId. The shape parameter
+                      represents the workgroup-level problem size. Each subgroup may access
+                      multiple blocks according to round-robin distribution rules.}],
+                    "FailureOr<SmallVector<SmallVector<Value>>>",
+                    "getOffsets",
+                    (ins "OpBuilder &": $builder, "Location":$loc, "Value":$linearId, "ArrayRef<int64_t>":$shape)>
+  ];
+}
+
+def XeGPU_LayoutAttr : XeGPUAttr<"Layout", "layout", [DistributeLayoutAttr]> {
   let summary = [{
     Describes the data distribution to subgroups and work-items for a tensor
     specified by the tensor descriptor.
@@ -297,12 +342,12 @@ def XeGPU_LayoutAttr : XeGPUAttr<"Layout", "layout"> {
   ];
 
   let extraClassDeclaration = [{
-    bool isWgLayout() {
+    bool isForWorkgroup() {
       return getSgLayout() != nullptr;
     }
 
-    bool isSgLayout() {
-      return !isWgLayout();
+    bool isForSubgroup() {
+      return !isForWorkgroup();
     }
 
     int64_t getRank() {
@@ -330,12 +375,157 @@ def XeGPU_LayoutAttr : XeGPUAttr<"Layout", "layout"> {
       return LayoutAttr::get(getContext(), getSgLayout(), getSgData(), nullptr,
                              getLaneLayout(), getLaneData(), getOrder());
     }
+
+    std::optional<SmallVector<int64_t>> getSgLayoutAsInt() const {
+      if (DenseI32ArrayAttr layout = getSgLayout())
+        return llvm::to_vector_of<int64_t>(layout.asArrayRef());
+      return std::nullopt;
+    }
+
+    std::optional<SmallVector<int64_t>> getSgDataAsInt() const {
+      if (DenseI32ArrayAttr data = getSgData())
+        return llvm::to_vector_of<int64_t>(data.asArrayRef());
+      return std::nullopt;
+    }
+
+    /// Delinearizes a linear subgroup ID into its multidimensional indices
+    /// based on the effective subgroup layout.
+    FailureOr<SmallVector<Value>>
+    delinearizeSubgroupId(OpBuilder &builder, Location loc, Value linearId);
+
+    /// Generates instructions to compute multidimensional offsets for blocks
+    /// assigned to a subgroup identified by linearId. The shape parameter
+    /// represents the workgroup-level problem size. Each subgroup may access
+    /// multiple blocks according to round-robin distribution rules.
+    FailureOr<SmallVector<SmallVector<Value>>>
+    getOffsets(OpBuilder &builder, Location loc, Value linearId, ArrayRef<int64_t> shape);
+
   }];
 
   let assemblyFormat = "`<` struct(params) `>`";
   let genVerifyDecl = 1;
 }
 
+
+def XeGPU_SliceAttr : XeGPUAttr<"Slice", "slice", [DistributeLayoutAttr]> {
+  let summary = [{Describes the data distribution and sharing among subgroups or work-items.}];
+
+  let description = [{
+    Like LayoutAttr, SliceAttr describes data distribution among subgroups or work-items.
+    However, whereas LayoutAttr requires the data to have the same rank as the attribute,
+    SliceAttr permits the data to have a lower rank. In this case, compute units in the
+    specified dimensions (given by `$dims`) share the data, provided that the remaining
+    ranks match the data rank. SliceAttr is commonly used by operations such as
+    vector.multi_reduction and vector.broadcast.
+
+    Example:
+    ```
+    #l = #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32]>
+    #r = #xegpu.slice<#l, dim = [0]>
+
+    %exp = math.exp %input {layout_result_0 = #l}: vector<256x128xf32>
+    %red = vector.multi_reduction<add>, %exp, %acc [0] {layout_result_0 = #r}: vector<256x128xf32> to vector<128xf32>
+    %bcast = vector.broadcast %red {layout_result_0 = #l} : vector<128xf32> to vector<256x128xf32>
+    ```
+    In this example, %red is conceptually divided into 4 vectors of type vector<32xf32>, each assigned to
+    a group of subgroups. Each group consists of 8 subgroups from the same column of sg_layout, sharing a
+    single reduction result of type vector<32xf32>.
+
+  }];
+
+  let parameters = (ins
+    "xegpu::DistributeLayoutAttr": $parent,
+    "DenseI64ArrayAttr": $dims
+  );
+
+  let extraClassDeclaration = [{
+
+    int64_t getRank() const {
+      SliceAttr attr = flatten();
+      auto parent = dyn_cast<LayoutAttr>(attr.getParent());
+      return parent.getRank() - attr.getDims().size();
+    }
+
+    DenseI32ArrayAttr getOrder() const {
+      SliceAttr attr = flatten();
+      auto parent = dyn_cast<LayoutAttr>(attr.getParent());
+      return parent.getOrder();
+    }
+
+    bool isForWorkgroup() const {
+      SliceAttr attr = flatten();
+      auto parent = dyn_cast<LayoutAttr>(attr.getParent());
+      return parent.isForWorkgroup();
+    }
+
+    bool isForSubgroup() const {
+      SliceAttr attr = flatten();
+      auto parent = dyn_cast<LayoutAttr>(attr.getParent());
+      return parent.isForSubgroup();
+    }
+
+    /// Returns the SgLayout of the attribute, computed by applying
+    /// the slice dimensions to the underlying LayoutAttr.
+    std::optional<SmallVector<int64_t>> getSgLayoutAsInt() const {
+      SliceAttr attr = flatten();
+      auto parent = dyn_cast<LayoutAttr>(attr.getParent());
+      if (auto layout = parent.getSgLayoutAsInt()) {
+        ArrayRef<int64_t> dims = attr.getDims().asArrayRef();
+        return XeGPUDialect::slice(llvm::ArrayRef<int64_t>(*layout), dims);
+      }
+      return std::nullopt;
+    }
+
+    /// Returns the SgData of the attribute, computed by applying
+    /// the slice dimensions to the underlying LayoutAttr.
+    std::optional<SmallVector<int64_t>> getSgDataAsInt() const {
+      SliceAttr attr = flatten();
+      auto parent = dyn_cast<LayoutAttr>(attr.getParent());
+      if (auto data = parent.getSgDataAsInt()) {
+        ArrayRef<int64_t> dims = attr.getDims().asArrayRef();
+        return XeGPUDialect::slice(llvm::ArrayRef<int64_t>(*data), dims);
+      }
+      return std::nullopt;
+    }
+
+    SliceAttr dropSgLayoutAndData() {
+      SliceAttr attr = flatten();
+      auto parent = dyn_cast<LayoutAttr>(attr.getParent());
+      parent = parent.dropSgLayoutAndData();
+      return SliceAttr::get(getContext(), parent, attr.getDims());
+    }
+
+    SliceAttr dropInstData() {
+      SliceAttr attr = flatten();
+      auto parent = dyn_cast<LayoutAttr>(attr.getParent());
+      parent = parent.dropInstData();
+      return SliceAttr::get(getContext(), parent, attr.getDims());
+    }
+
+    /// flatten a nested SliceAttr, e.g., for 2-level nested SliceAttr
+    /// #xegpu.slice<#xegpu.slice<#xegpu.layout<sg_layout = [4, 8, 12]>, dims = [0]>, dims = [0]>
+    /// it will coalese two slice operations and return a simplified SliceAttr
+    /// #xegpu.slice<#xegpu.layout<sg_layout = [4, 8, 12]>, dims = [0, 1]>
+    SliceAttr flatten() const;
+
+    /// Delinearizes a linear subgroup ID into its multidimensional indices
+    /// based on the effective subgroup layout.
+    FailureOr<SmallVector<Value>>
+    delinearizeSubgroupId(OpBuilder &builder, Location loc, Value linearId);
+
+    /// Generates instructions to compute multidimensional offsets for blocks
+    /// assigned to a subgroup identified by linearId. The shape parameter
+    /// represents the workgroup-level problem size. Each subgroup may access
+    /// multiple blocks according to round-robin distribution rules.
+    FailureOr<SmallVector<SmallVector<Value>>>
+    getOffsets(OpBuilder &builder, Location loc, Value linearId, ArrayRef<int64_t> shape);
+
+  }];
+
+  let assemblyFormat = "`<` qualified($parent) `,` `dims` `=` $dims `>`";
+  let genVerifyDecl = 1;
+}
+
 def XeGPU_RangeAttr : XeGPUAttr<"Range", "range"> {
   let summary = [{Specifies a half-open range}];
   let description = [{
@@ -365,4 +555,34 @@ def XeGPU_RangeAttr : XeGPUAttr<"Range", "range"> {
   let genVerifyDecl = 1;
 }
 
+def XeGPU_MemLayoutAttr : XeGPUAttr<"MemLayout", "mem_layout"> {
+  let summary = [{Specifies memory layouts with named attributes.}];
+
+  let description = [{
+    This attribute stores a collection of named attributes that describe
+    memory layout properties such as stride, block, etc.
+  }];
+
+  let parameters = (ins "DictionaryAttr": $attrs);
+  let hasCustomAssemblyFormat = 1;
+
+  let extraClassDeclaration = [{
+    /// Get a specific attribute by name
+    Attribute getAttr(StringRef name) const {
+      return getAttrs().get(name);
+    }
+
+    /// Check if a specific attribute exists
+    bool hasAttr(StringRef name) const {
+      return getAttrs().contains(name);
+    }
+
+    ArrayAttr getStrides() {
+      return getAttrs().getAs<ArrayAttr>("stride");
+    }
+
+  }];
+
+}
+
 #endif // MLIR_DIALECT_XEGPU_IR_XEGPUATTRS_TD
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td
index 549018b61d6fb..76d58e5ea2424 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUDialect.td
@@ -41,6 +41,18 @@ def XeGPU_Dialect : Dialect {
       /// Checks if the given shape can be evenly distributed based on the layout
       /// and data factors provided by the LayoutAttr.
       static bool isEvenlyDistributable(llvm::ArrayRef<int64_t> shape, xegpu::LayoutAttr attr);
+
+      /// drops/slices the shape in the specified dims, and return the rest. e.g.,
+      /// for shape = [32, 64, 8], dims = [0, 2], it will return [64]
+      template<typename T, typename U>
+      static llvm::SmallVector<T> slice(llvm::ArrayRef<T> shape, llvm::ArrayRef<U> dims) {
+        llvm::SmallVector<T> result;
+        for (auto [i, v]: llvm::enumerate(shape)) {
+          if (!llvm::is_contained(dims, i))
+            result.push_back(v);
+        }
+        return result;
+      }
     }];
 }
 
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
index 75b16a87e03c6..f0b325cd1d593 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
@@ -29,7 +29,7 @@ class XeGPU_Op<string mnemonic, list<Trait> traits = []>:
     void printProperties(::mlir::MLIRContext *ctx,
             ::mlir::OpAsmPrinter &p, const Properties &prop,
             ::mlir::ArrayRef<::llvm::StringRef> elidedProps) {
-      
+
       DictionaryAttr propAttr = dyn_cast_if_present<mlir::DictionaryAttr>(getPropertiesAsAttr(ctx, prop));
 
       // filter out the elidedProps from propAttr, and get the resultAttr
@@ -43,7 +43,7 @@ class XeGPU_Op<string mnemonic, list<Trait> traits = []>:
       }
 
       if (!filteredAttrs.empty()) {
-        p << "<" << DictionaryAttr::get(ctx, filteredAttrs) << ">"; 
+        p << "<" << DictionaryAttr::get(ctx, filteredAttrs) << ">";
       }
     }
 
@@ -60,8 +60,7 @@ class XeGPU_Op<string mnemonic, list<Trait> traits = []>:
 }
 
 
-def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface,
-                        AttrSizedOperandSegments, OffsetSizeAndStrideOpInterface]> {
+def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface, AttrSizedOperandSegments]> {
 
   let summary = "Create nd-tensor descriptor operation";
   let description = [{
@@ -71,28 +70,32 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
     future). Elements in the subview continuous in each dimension. It encodes the
     following important information for supporting Intel hardware features:
 
-    * source: an object representing (starting address/pointer of) a memory region.
+    Arguments:
+    - `source`: an object representing (starting address/pointer of) a memory region.
        It can be either a memref object, or simply a pointer represented by uint64_t type.
        For the case of dynamic memrefs or pointer, the shape and layout information of the
        memory region should be explicitly passed via `shape` and `strides` parameters.
 
-    * offsets: index values represents offsets from the "source" at the each dimension
+    - `offsets`: index values represents offsets from the "source" at the each dimension
         at which the subview of the target memory will be created. It is encoded via
         "offsets" and "const_offsets", such that it can accept various forms, such as,
         operands (e.g., [%c0, %c]) and attributes (e.g., [2, 4]).
 
-    * shape: the shape information of the memory region pointed by the "source". It is
+    - `shape`: the shape information of the memory region pointed by the "source". It is
          typically encoded via the MemRefType of the source, e.g., memref<4096x4096xf16>.
         But if "source" is simply a pointer represented as uint64_t type, or a memref
         type without shape information e.g., memref<?x?xf16>, the shape information has
         to be explicitly passed via the "shape" and "const_shape" arguments.
 
-    * strides: the strides of the memory region pointed by the "source". Similar to shape,
+    - `strides`: the strides of the memory region pointed by the "source". Similar to shape,
         it is typically encoded via the MemRefType of the source too. But if "source" is
         simply a pointer represented as uint64_t type, or a memref type without shape
         information e.g., memref<?x?xf16>, the strides information has to be explicitly
         passed via the "strides" and "const_strides" argument.
 
+    Results:
+    - `res`: nd tensor descriptor
+
     Example 1 (suppose the tensor shape inferred by the compiler is 8x16):
     ```mlir
     %0 = memref.alloc() : memref<1024x1024xf32>
@@ -143,11 +146,7 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
   let builders = [
     OpBuilder<(ins "Type": $tdesc, "TypedValue<MemRefType>": $source)>,
 
-    OpBuilder<(ins "Type": $tdesc, "TypedValue<MemRefType> ": $source,
-                   "llvm::ArrayRef<OpFoldResult>": $shape,
-                   "llvm::ArrayRef<OpFoldResult>": $strides)>,
-
-    OpBuilder<(ins "Type": $tdesc, "TypedValue<IntegerType> ": $source,
+    OpBuilder<(ins "Type": $tdesc, "Value ": $source,
                    "llvm::ArrayRef<OpFoldResult>": $shape,
                    "llvm::ArrayRef<OpFoldResult>": $strides)>,
 
@@ -181,82 +180,38 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
       return getType().getShape();
     }
 
-    /// wrapper for matching with OffsetSizeAndStrideOpInterface
-    OperandRange getSizes() {
-      return getShape();
+    SmallVector<OpFoldResult> getMixedOffsets() {
+      auto statics = getConstOffsets().value_or(SmallVector<int64_t>());
+      auto dynamics = getOffsets();
+      if (statics.size() == 0 && dynamics.size() == 0)
+        return {};
+      return getMixedValues(statics, dynamics, getContext());
     }
 
-    ArrayRef<int64_t> getStaticOffsets(){
-      auto attr = getConstOffsetsAttr();
+    SmallVector<OpFoldResult> getMixedSizes() {
+      SmallVector<int64_t> statics;
 
-      if (attr) 
-        return attr;
+      /// Get the static sizes/shape, the value passed to const_shape
+      /// will overide the value in memref shape.
+      if (auto memrefTy = llvm::dyn_cast<MemRefType>(getSourceType()))
+        statics = llvm::to_vector(memrefTy.getShape());
+      if (auto attr = getConstShapeAttr())
+        statics = llvm::to_vector(attr.asArrayRef());
 
-      int64_t rank = getMixedSizes().size();
-      
-      setConstOffsets(llvm::SmallVector<int64_t, 4>(rank, 0));
-
-      attr = getConstOffsetsAttr();
-      return attr;
+      return getMixedValues(statics, getShape(), getContext());
     }
 
-    /// wrapper for matching with OffsetSizeAndStrideOpInterface
-    /// If source is IntegerType or `const_shape` is filled,
-    /// it will return `const_shape`, such that mixes of `shape`
-    /// and `const_shape` will be used to represent the shape of
-    /// source operand. They overide static shape from source memref type.
-    ArrayRef<int64_t> getStaticSizes() {
-      /// To be compatible with OffsetSizeAndStrideOpInterface, which expects valid return value and perform checks
-      static  llvm::SmallVector<int64_t, 4> emptyShape;
-
-      auto attr = getConstShapeAttr();
-      if (attr)
-        return attr;
-
-      if (llvm::isa<IntegerType>(getSourceType()))
-        return emptyShape;
-
-      auto memrefType = llvm::dyn_cast<MemRefType>(getSourceType());
-      assert(memrefType && "Incorrect use of getStaticSizes");
-      return memrefType.getShape();
-    }
+    SmallVector<OpFoldResult> getMixedStrides() {
+      SmallVector<int64_t> statics;
 
-    /// wrapper for matching with OffsetSizeAndStrideOpInterface
-    /// If source is IntegerType or `const_strides` is filled, it
-    /// will return `const_strides`, such that mixes of `strides`
-    /// and `const_strides` will be used to represent the strides of
-    /// source operand. They overide static strides from source memref type.
-    ArrayRef<int64_t> getStaticStrides() {
-      /// To be compatible with OffsetSizeAndStrideOpInterface, which expects valid return value and perform checks
-      static llvm::SmallVector<int64_t, 4> emptyStrides;
-
-      auto attr = getConstStridesAttr();
-      if (attr)
-        return attr;
-      
-      if (llvm::isa<IntegerType>(getSourceType()))
-        return emptyStrides;
-
-      auto memrefType = llvm::dyn_cast<MemRefType>(getSourceType());
-      assert(memrefType && "Incorrect use of getStaticStrides");
-      auto [strides, _] = memrefType.getStridesAndOffset();
-      // reuse the storage of ConstStridesAttr since strides from
-      // memref is not persistant
-      setConstStrides(strides);
-      attr = getConstStridesAttr();
-      return attr;
-    }
+      /// Get the static strides, the value passed to const_strides
+      /// will overide the value in memref.
+      if (auto memrefTy = llvm::dyn_cast<MemRefType>(getSourceType()))
+        statics = memrefTy.getStridesAndOffset().first;
+      if (auto attr = getConstStridesAttr())
+        statics = llvm::to_vector(attr.asArrayRef());
 
-    /// Return the expected rank of each of the`static_offsets`,
-    /// `static_shape` and `static_strides` attributes.
-    std::array<unsigned, 3> getArrayAttrMaxRanks() {
-      unsigned rank;
-      if (auto ty = llvm::dyn_cast<MemRefType>(getSourceType())) {
-        rank = ty.getRank();
-      } else {
-        rank = (unsigned)getMixedOffsets().size();
-      }
-      return {rank, rank, rank};
+      return getMixedValues(statics, getStrides(), getContext());
     }
 
     /// Return the number of leading operands before the `offsets`,
@@ -281,6 +236,14 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
       return static_cast<unsigned>(MemorySpace::Global);
     }
 
+    xegpu::DistributeLayoutAttr getLayoutAttr() {
+      return dyn_cast_if_present<xegpu::DistributeLayoutAttr>(getType().getLayout());
+    }
+
+    ArrayRef<int64_t> getDataShape() {
+      return getTensorDescShape();
+    }
+
   }];
 }
 
@@ -311,18 +274,40 @@ def XeGPU_PrefetchNdOp : XeGPU_Op<"prefetch_nd", []> {
     xegpu::TensorDescType getTensorDescType() {
       return getTensorDesc().getType();
     }
+
+   SmallVector<OpFoldResult> getMixedOffsets() {
+      auto statics = getConstOffsets().value_or(SmallVector<int64_t>());
+      auto dynamics = getOffsets();
+      if (statics.size() == 0 && dynamics.size() == 0)
+        return {};
+      return getMixedValues(statics, dynamics, getContext());
+    }
+
+    xegpu::DistributeLayoutAttr getLayoutAttr() {
+      return dyn_cast_if_present<xegpu::DistributeLayoutAttr>(getTensorDescType().getLayout());
+    }
+
+    ArrayRef<int64_t> getDataShape() {
+      return getTensorDescType().getShape();
+    }
+
   }];
 
   let assemblyFormat = [{
-    $TensorDesc `` 
-    custom<OptionalDynamicIndexList>($offsets, $const_offsets) 
+    $TensorDesc ``
+    custom<OptionalDynamicIndexList>($offsets, $const_offsets)
     prop-dict attr-dict `:` qualified(type($TensorDesc))
   }];
 
   let builders = [
-    OpBuilder<(ins "Value": $TensorDesc, 
-                   "xegpu::CachePolicyAttr": $l1_hint, 
-                   "xegpu::CachePolicyAttr": $l2_hint, 
+    OpBuilder<(ins "Value": $TensorDesc,
+                   "xegpu::CachePolicyAttr": $l1_hint,
+                   "xegpu::CachePolicyAttr": $l2_hint,
+                   "xegpu::CachePolicyAttr": $l3_hint)>,
+    OpBuilder<(ins "Value": $TensorDesc,
+                   "ArrayRef<OpFoldResult>": $offsets,
+                   "xegpu::CachePolicyAttr": $l1_hint,
+                   "xegpu::CachePolicyAttr": $l2_hint,
                    "xegpu::CachePolicyAttr": $l3_hint)>
   ];
 
@@ -370,7 +355,7 @@ def XeGPU_LoadNdOp : XeGPU_Op<"load_nd", [
 
   let arguments = (ins XeGPU_TensorDesc: $TensorDesc,
                        Variadic<Index>: $offsets,
-                       OptionalAttr<DenseI64ArrayAttr>: $const_offsets,  
+                       OptionalAttr<DenseI64ArrayAttr>: $const_offsets,
                        OptionalAttr<UnitAttr>: $packed,
                        OptionalAttr<DenseI64ArrayAttr>: $transpose,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
@@ -387,19 +372,43 @@ def XeGPU_LoadNdOp : XeGPU_Op<"load_nd", [
     xegpu::TensorDescType getTensorDescType() {
       return getTensorDesc().getType();
     }
+
+    SmallVector<OpFoldResult> getMixedOffsets() {
+      auto statics = getConstOffsets().value_or(SmallVector<int64_t>());
+      auto dynamics = getOffsets();
+      if (statics.size() == 0 && dynamics.size() == 0)
+        return {};
+      return getMixedValues(statics, dynamics, getContext());
+    }
+
+    xegpu::DistributeLayoutAttr getLayoutAttr() {
+      return dyn_cast_if_present<xegpu::DistributeLayoutAttr>(getTensorDescType().getLayout());
+    }
+
+    ArrayRef<int64_t> getDataShape() {
+      return getTensorDescType().getShape();
+    }
+
+
   }];
 
   let assemblyFormat = [{
-    $TensorDesc `` 
-    custom<OptionalDynamicIndexList>($offsets, $const_offsets) 
+    $TensorDesc ``
+    custom<OptionalDynamicIndexList>($offsets, $const_offsets)
     prop-dict attr-dict `:` qualified(type($TensorDesc)) `->` type($value)
   }];
 
   let builders = [
-    OpBuilder<(ins "Type": $value, "Value": $TensorDesc, 
+    OpBuilder<(ins "Type": $value, "Value": $TensorDesc,
+                    "UnitAttr": $packed, "DenseI64ArrayAttr": $transpose,
+                    "xegpu::CachePolicyAttr": $l1_hint,
+                    "xegpu::CachePolicyAttr": $l2_hint,
+                    "xegpu::CachePolicyAttr": $l3_hint)>,
+    OpBuilder<(ins "Type": $value, "Value": $TensorDesc,
+                    "ArrayRef<OpFoldResult>": $offsets,
                     "UnitAttr": $packed, "DenseI64ArrayAttr": $transpose,
-                    "xegpu::CachePolicyAttr": $l1_hint, 
-                    "xegpu::CachePolicyAttr": $l2_hint, 
+                    "xegpu::CachePolicyAttr": $l1_hint,
+                    "xegpu::CachePolicyAttr": $l2_hint,
                     "xegpu::CachePolicyAttr": $l3_hint)>
   ];
 
@@ -442,7 +451,7 @@ def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", [
   let arguments = (ins XeGPU_ValueType: $value,
                        XeGPU_TensorDesc: $TensorDesc,
                        Variadic<Index>: $offsets,
-                       OptionalAttr<DenseI64ArrayAttr>: $const_offsets,  
+                       OptionalAttr<DenseI64ArrayAttr>: $const_offsets,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
@@ -455,20 +464,42 @@ def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", [
     xegpu::TensorDescType getTensorDescType() {
       return getTensorDesc().getType();
     }
+
+    SmallVector<OpFoldResult> getMixedOffsets() {
+      auto statics = getConstOffsets().value_or(SmallVector<int64_t>());
+      auto dynamics = getOffsets();
+      if (statics.size() == 0 && dynamics.size() == 0)
+        return {};
+      return getMixedValues(statics, dynamics, getContext());
+    }
+
+    xegpu::DistributeLayoutAttr getLayoutAttr() {
+      return dyn_cast_if_present<xegpu::DistributeLayoutAttr>(getTensorDescType().getLayout());
+    }
+
+    ArrayRef<int64_t> getDataShape() {
+      return getTensorDescType().getShape();
+    }
+
   }];
 
    let assemblyFormat = [{
-    $value `,` 
-    $TensorDesc `` 
-    custom<OptionalDynamicIndexList>($offsets, $const_offsets) 
+    $value `,`
+    $TensorDesc ``
+    custom<OptionalDynamicIndexList>($offsets, $const_offsets)
     prop-dict attr-dict `:`  type($value) `,` qualified(type($TensorDesc))
   }];
 
   let builders = [
-    OpBuilder<(ins "Value": $value, "Value": $TensorDesc, 
-                   "xegpu::CachePolicyAttr": $l1_hint, 
-                   "xegpu::CachePolicyAttr": $l2_hint, 
-                   "xegpu::CachePolicyAttr": $l3_hint)>
+    OpBuilder<(ins "Value": $value, "Value": $TensorDesc,
+                   "xegpu::CachePolicyAttr": $l1_hint,
+                   "xegpu::CachePolicyAttr": $l2_hint,
+                   "xegpu::CachePolicyAttr": $l3_hint)>,
+    OpBuilder<(ins "Value": $value, "Value": $TensorDesc,
+                  "ArrayRef<OpFoldResult>": $offsets,
+                  "xegpu::CachePolicyAttr": $l1_hint,
+                  "xegpu::CachePolicyAttr": $l2_hint,
+                  "xegpu::CachePolicyAttr": $l3_hint)>
   ];
 
 
@@ -533,12 +564,17 @@ def XeGPU_CreateDescOp: XeGPU_Op<"create_tdesc", [Pure, ViewLikeOpInterface]> {
     (scattered) subviews, allowing each work-item in a subgroup specifying their own offset.
     It accepts the following parameters:
 
-    * source: a 1D memref or pointer (uint64_t) represents the flattened memory object.
-    * offsets: a vector containing offsets of each access point. Its size
+    Arguments:
+    - `source`: a 1D memref or pointer (i64, i32, ui64, ui32) represents the flattened
+      memory object.
+    - `offsets`: a vector containing offsets of each access point. Its size
       is fixed to the hardware supportted subgroup size, e.g., 16 on PVC,
       implying each element in the vector corresponds to a work-item (SIMT lane)
       in the subgroup.
 
+    Results:
+    - `res`: scattered tensor descriptor
+
     The first dimension of the result TensorDesc corresponds to work-items, so it should
     match the dimension of offsets. It may also has a second dimension corresponding to
     the chunk_size if the chunk size is larger than 1.
@@ -569,8 +605,8 @@ def XeGPU_CreateDescOp: XeGPU_Op<"create_tdesc", [Pure, ViewLikeOpInterface]> {
     ```
   }];
 
-  let arguments = (ins XeGPU_BaseAddrType: $source,
-                       XeGPU_OffsetType: $offsets);
+  let arguments = (ins XeGPU_GatherScatterBaseAddrType:$source,
+      XeGPU_OffsetType:$offsets);
   let results = (outs XeGPU_TensorDesc:$TensorDesc);
 
   let builders = [
@@ -628,6 +664,18 @@ def XeGPU_PrefetchOp : XeGPU_Op<"prefetch", []> {
     As compared to prefetch_nd, which works on non-scattered TensorDesc,
     it works on scattered TensorDesc instead.
 
+    Arguments:
+    - `source`: represents the memory region to be loaded from, which can be either a
+        tensor_desc or a 1D memref or pointer (ui64, ui32, i64 or i32).
+        In case of tensor_desc, offsets come from the producer create_tdesc op.
+        tensor_desc cannot be used in SIMT mode.
+    - `offsets`: represents offsets from source. required if `source` in not a TensorDescType.
+        offsets is a vector of `index` type and vector length is either the subgroup size
+        or 1 in SIMT mode. scalar offset is also valid for SIMT mode.
+    - `l1_hint`, `l2_hint`, `l3_hint`: are optional cache hints for each level of cache.
+    - `offset_align_byte`: required if `source` is a pointer. If `source` is not a pointer,
+        it is not allowed. Represents the alignment in bytes of each offset in offsets.
+
     Example 1:
     ```mlir
       xegpu.prefetch %tdesc {l1_hint = #xegpu.cache_hint<cached>,
@@ -635,12 +683,12 @@ def XeGPU_PrefetchOp : XeGPU_Op<"prefetch", []> {
                              l3_hint = #xegpu.cache_hint<cached>}
         : !xegpu.tensor_desc<16xf16>
     ```
-    
+
     Example 2:
     A variant accepts memref as base pointer and an offset instead of scattered TensorTdesc.
     It combines "create scattered TensorTdesc" and "prefetch with scattered TensorTdesc".
-    The source operand could be a raw pointer (uint64_t).
-    Please refer to create_tdesc for the restriction of memref. 
+    The source operand could be a raw pointer (ui64, ui32, i64, i32).
+    Please refer to create_tdesc for the restriction of memref.
     ```mlir
       %a = memref.alloc() : memref<1024xf32>
       %0 = arith.constant dense<[0, 16, 32, 64]> : vector<4xindex>
@@ -650,13 +698,33 @@ def XeGPU_PrefetchOp : XeGPU_Op<"prefetch", []> {
         : memref<1024xf32>, vector<4xindex>
     ```
 
+    Example 3 (SIMT mode):
+    SIMT mode only accepts the offsets variant.
+    ```mlir
+      xegpu.prefetch %0[%1] {l1_hint = #xegpu.cache_hint<cached>,
+                             l2_hint = #xegpu.cache_hint<cached>,
+                             l3_hint = #xegpu.cache_hint<cached>}
+        : memref<256xf32>, vector<1xindex>
+    ```
+
+    Example 4 (SIMT mode):
+    SIMT mode only accepts the offsets variant.
+    ```mlir
+      xegpu.prefetch %0[%1] {l1_hint = #xegpu.cache_hint<cached>,
+                             l2_hint = #xegpu.cache_hint<cached>,
+                             l3_hint = #xegpu.cache_hint<cached>,
+                             offset_align_byte = 2}
+        : i64, vector<1xindex>
+    ```
+
   }];
 
-  let arguments = (ins XeGPU_GatherScatterSourceType: $source,
-                       Optional<XeGPU_OffsetType>: $offsets,
-                       OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
-                       OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
-                       OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
+  let arguments = (ins XeGPU_GatherScatterSourceType:$source,
+      Optional<AnyTypeOf<[XeGPU_OffsetType, Index]>>:$offsets,
+      OptionalAttr<XeGPU_CacheHintAttr>:$l1_hint,
+      OptionalAttr<XeGPU_CacheHintAttr>:$l2_hint,
+      OptionalAttr<XeGPU_CacheHintAttr>:$l3_hint,
+      OptionalAttr<I64Attr>:$offset_align_byte);
 
   let extraClassDeclaration = extraBaseClassDeclaration # [{
     Type getSourceType() {
@@ -673,19 +741,20 @@ def XeGPU_PrefetchOp : XeGPU_Op<"prefetch", []> {
     xegpu::TensorDescType getTensorDescType() {
       return dyn_cast<xegpu::TensorDescType>(getSourceType());
     }
+
   }];
 
   let assemblyFormat = [{
-    $source 
+    $source
     (`[` $offsets^ `]`)?
     prop-dict
-    attr-dict `:` type(operands) 
+    attr-dict `:` type(operands)
   }];
-    
+
   let builders = [
     OpBuilder<(ins "Value": $source,
-                    "xegpu::CachePolicyAttr": $l1_hint, 
-                    "xegpu::CachePolicyAttr": $l2_hint, 
+                    "xegpu::CachePolicyAttr": $l1_hint,
+                    "xegpu::CachePolicyAttr": $l2_hint,
                     "xegpu::CachePolicyAttr": $l3_hint)>
   ];
 
@@ -703,8 +772,26 @@ def XeGPU_LoadGatherOp : XeGPU_Op<"load", [MemoryEffects<[MemRead]>]> {
     The mask operand masks out memory access so that it is safe to pass out-of-boundary
     addresses/offsets as long as they are masked. It applies to slots of SIMD lanes.
 
-    In SIMT mode, the result vector represents the data to be loaded by each work-item.
-    Each work-item recieves a `chunk_size` number of elements.
+    In SIMT mode, the result is a 1D vector that represents the data to be loaded by
+    each work-item. If size is not 1, size should be equal to the chunk size,
+
+    Arguments:
+    - `source`: represents the memory region to be loaded from, which can be either a
+        tensor_desc or a 1D memref or pointer (ui64, ui32, i64 or i32).
+        In case of tensor_desc, offsets come from the producer create_tdesc op.
+        tensor_desc cannot be used in SIMT mode.
+    - `offsets`: represents offsets from source. required if `source` in not a TensorDescType.
+        offsets is a vector of `index` type and vector length is either the subgroup size
+        or 1 in SIMT mode. scalar offset is also valid for SIMT mode.
+    - `mask`: is a vector of `i1` type, which is used to mask out the memory access.
+        mask is a vector of size equal to the subgroup size, or 1 in SIMT mode.
+        scalar mask is also valid for SIMT mode.
+    - `chunk_size`: (optional) represents contiguous number of elements to load from per work item.
+    - `l1_hint`, `l2_hint`, `l3_hint`: are optional cache hints for each level of cache.
+
+    Results:
+    - `res`: represents loaded data
+
 
   Example 1:
   ```mlir
@@ -723,21 +810,12 @@ def XeGPU_LoadGatherOp : XeGPU_Op<"load", [MemoryEffects<[MemRead]>]> {
           : !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<memory_space=global, chunk_size=8>>,
             vector<16xi1> -> vector<16x8xf32>
   ```
-  
-  Example 3 (SIMT mode):
-  ```mlir
-    %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>,
-                             l2_hint = #xegpu.cache_hint<uncached>,
-                             l3_hint = #xegpu.cache_hint<uncached>}>
-          : !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<memory_space=global, chunk_size=8>>
-            vector<16xi1> -> vector<8xf32>
-  ```
-  
-  Example 4:
+
+  Example 3:
   A variant accepts memref as base pointer and an offset instead of scattered TensorTdesc.
   It combines "create scattered TensorTdesc" and "load with scattered TensorTdesc".
-  The source operand could be a raw pointer (uint64_t). Please refer to create_tdesc
-  for the restriction of memref. 
+  The source operand could be a raw pointer (ui64, ui32, i64, i32). Please refer to create_tdesc
+  for the restriction of memref.
   ```mlir
     %a = memref.alloc() : memref<1024xf32>
     %offsets = vector.step : vector<16xindex>
@@ -748,16 +826,25 @@ def XeGPU_LoadGatherOp : XeGPU_Op<"load", [MemoryEffects<[MemRead]>]> {
       : memref<1024xf32>, vector<16xi1>, vector<16xindex> -> vector<16xf32>
   ```
 
+  Example 4 (SIMT mode):
+  SIMT mode only accepts the offsets variant. chunk_size can be inferred from result
+  type. In this example, chunk_size is 8.
+  ```mlir
+    %2 = xegpu.load %1[%2], %0 <{l1_hint = #xegpu.cache_hint<cached>,
+                             l2_hint = #xegpu.cache_hint<uncached>,
+                             l3_hint = #xegpu.cache_hint<uncached>}>
+          : memref<128xf32>, vector<1xindex>, vector<1xi1> -> vector<8xf32>
+  ```
+
   }];
 
-  let arguments = (ins XeGPU_GatherScatterSourceType: $source,
-                       Optional<XeGPU_OffsetType>: $offsets,
-                       XeGPU_MaskType: $mask,
-                       OptionalAttr<I64Attr>: $chunk_size,
-                       OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
-                       OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
-                       OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
-  let results = (outs XeGPU_ValueType: $value);
+  let arguments = (ins XeGPU_GatherScatterSourceType:$source,
+      Optional<AnyTypeOf<[XeGPU_OffsetType, Index]>>:$offsets,
+      AnyTypeOf<[XeGPU_MaskType, I1]>:$mask, OptionalAttr<I64Attr>:$chunk_size,
+      OptionalAttr<XeGPU_CacheHintAttr>:$l1_hint,
+      OptionalAttr<XeGPU_CacheHintAttr>:$l2_hint,
+      OptionalAttr<XeGPU_CacheHintAttr>:$l3_hint);
+  let results = (outs AnyTypeOf<[XeGPU_ValueType, XeGPU_ScalarType]>:$value);
 
   let extraClassDeclaration = extraBaseClassDeclaration # [{
 
@@ -794,14 +881,14 @@ def XeGPU_LoadGatherOp : XeGPU_Op<"load", [MemoryEffects<[MemRead]>]> {
   let assemblyFormat = [{
     $source
     (`[` $offsets^ `]`)? `,`
-    $mask prop-dict 
+    $mask prop-dict
     attr-dict `:` type(operands) `->` type($value)
   }];
 
   let builders = [
     OpBuilder<(ins "Type": $value, "Value": $source, "Value": $mask,
-                    "xegpu::CachePolicyAttr": $l1_hint, 
-                    "xegpu::CachePolicyAttr": $l2_hint, 
+                    "xegpu::CachePolicyAttr": $l1_hint,
+                    "xegpu::CachePolicyAttr": $l2_hint,
                     "xegpu::CachePolicyAttr": $l3_hint)>
    ];
 
@@ -810,15 +897,31 @@ def XeGPU_LoadGatherOp : XeGPU_Op<"load", [MemoryEffects<[MemRead]>]> {
 
 def XeGPU_StoreScatterOp : XeGPU_Op<"store", [MemoryEffects<[MemWrite]>]> {
   let summary = "store data to scattered memory locations.";
-  let description = [{ It (aka. store) stores data to scattered memory locations. The value is
+  let description =
+      [{ It (aka. store) stores data to scattered memory locations. The value is
   typically a 1D vector. But when the chunk size of the TensorDesc is larger than 1, it will be
   a 2D vector instead. For the later case, dim-1 of the value correspods to the simd lanes
   and the dim-0 of the value corresponds to the chunk size stored per lane. So `store_scatter`
   has transpose effect, which is similar to `load_gather`. Therefore, a transpose attribute is
   introduced on purpose, making sure users are aware of this implicit transformation.
 
-  In SIMT mode, the input vector represents the data to be stored by each work-item.
-  Each work-item stores a `chunk_size` number of elements.
+  In SIMT mode, the result is a 1D vector that represents the data to be stored by
+  each work-item. If size is not 1, size should be equal to the chunk size.
+
+    Arguments:
+    - `value`: represents the data to be stored.
+    - `dest`: represents the memory region to be stored to, which can be either a
+        tensor_desc or a 1D memref or pointer (ui64, ui32, i64 or i32).
+        In case of tensor_desc, offsets come from the producer create_tdesc op.
+        tensor_desc cannot be used in SIMT mode.
+    - `offsets`: represents offsets from dest. required if `source` in not a TensorDescType.
+        offsets is a vector of `index` type and vector length is either the subgroup size
+        or 1 in SIMT mode. scalar offset is also valid for SIMT mode.
+    - `mask`: is a vector of `i1` type, which is used to mask out the memory access.
+        mask is a vector of size equal to the subgroup size, or 1 in SIMT mode.
+        scalar mask is also valid for SIMT mode.
+    - `chunk_size`: (optional) represents contiguous number of elements to store to per work item.
+    - `l1_hint`, `l2_hint`, `l3_hint`: are optional cache hints for each level of cache.
 
   Example 1:
   ```mlir
@@ -836,19 +939,11 @@ def XeGPU_StoreScatterOp : XeGPU_Op<"store", [MemoryEffects<[MemWrite]>]> {
           : vector<16x8xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scattered_tdesc_attr<chunk_size=8>>, vector<16xi1>
   ```
 
-  Example 3 (SIMT mode):
-  ```mlir
-    xegpu.store %0, %1, %2 <{l1_hint = #xegpu.cache_hint<uncached>,
-                             l2_hint = #xegpu.cache_hint<write_back>,
-                             l3_hint = #xegpu.cache_hint<write_through>}>
-          : vector<8xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scattered_tdesc_attr<chunk_size=8>> vector<16xi1>
-  ```
-
-  Example 4:
+  Example 3:
   A variant accepts memref as base pointer and an offset instead of scattered TensorTdesc.
   It combines "create scattered TensorTdesc" and "store with scattered TensorTdesc".
   The dest operand could be a raw pointer (uint64_t).
-  Please refer to create_tdesc for the restriction of memref. 
+  Please refer to create_tdesc for the restriction of memref.
   ```mlir
     %a = memref.alloc() : memref<1024xf32>
     %val = arith.constant dense<0.0> : vector<16xf32>
@@ -860,19 +955,27 @@ def XeGPU_StoreScatterOp : XeGPU_Op<"store", [MemoryEffects<[MemWrite]>]> {
       : memref<1024xf32>, vector<16xi1>, vector<16xindex> -> vector<16xf32>
   ```
 
+  Example 4 (SIMT mode):
+  SIMT mode only accepts the offsets variant. chunk_size can be inferred from value
+  type. In this example, chunk_size is 8.
+  ```mlir
+    xegpu.store %0, %1[%2], %3 <{l1_hint = #xegpu.cache_hint<uncached>,
+                             l2_hint = #xegpu.cache_hint<write_back>,
+                             l3_hint = #xegpu.cache_hint<write_through>}>
+          : vector<8xf32>, memref<256xf32>, vector<1xindex>, vector<1xi1>
+  ```
+
   }];
 
-  let arguments = (ins
-    XeGPU_ValueType: $value,
-    XeGPU_GatherScatterSourceType: $dest,
-    Optional<XeGPU_OffsetType>: $offsets,
-    XeGPU_MaskType: $mask,
-    OptionalAttr<I64Attr>: $chunk_size,
-    OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
-    OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
-    OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
+  let arguments = (ins AnyTypeOf<[XeGPU_ValueType, XeGPU_ScalarType]>:$value,
+      XeGPU_GatherScatterSourceType:$dest,
+      Optional<AnyTypeOf<[XeGPU_OffsetType, Index]>>:$offsets,
+      AnyTypeOf<[XeGPU_MaskType, I1]>:$mask, OptionalAttr<I64Attr>:$chunk_size,
+      OptionalAttr<XeGPU_CacheHintAttr>:$l1_hint,
+      OptionalAttr<XeGPU_CacheHintAttr>:$l2_hint,
+      OptionalAttr<XeGPU_CacheHintAttr>:$l3_hint);
 
-  let extraClassDeclaration = extraBaseClassDeclaration # [{
+  let extraClassDeclaration = extraBaseClassDeclaration#[{
     Type getDestType() {
       return getDest().getType();
     }
@@ -888,6 +991,11 @@ def XeGPU_StoreScatterOp : XeGPU_Op<"store", [MemoryEffects<[MemWrite]>]> {
       return dyn_cast<xegpu::TensorDescType>(getDestType());
     }
 
+    mlir::Type getElementType() {
+      auto type = getValue().getType();
+      return getElementTypeOrSelf(type);
+    }
+
     VectorType getValueType() {
       return llvm::dyn_cast<VectorType>(getValue().getType());
     }
@@ -901,15 +1009,15 @@ def XeGPU_StoreScatterOp : XeGPU_Op<"store", [MemoryEffects<[MemWrite]>]> {
     $value `,`
     $dest
     (`[` $offsets^ `]`)? `,`
-    $mask 
-    prop-dict 
+    $mask
+    prop-dict
     attr-dict `:`  type(operands)
   }];
 
   let builders = [
     OpBuilder<(ins "Value": $value, "Value": $dest, "Value": $mask,
-                    "xegpu::CachePolicyAttr": $l1_hint, 
-                    "xegpu::CachePolicyAttr": $l2_hint, 
+                    "xegpu::CachePolicyAttr": $l1_hint,
+                    "xegpu::CachePolicyAttr": $l2_hint,
                     "xegpu::CachePolicyAttr": $l3_hint)>
    ];
 
@@ -1146,4 +1254,161 @@ def XeGPU_ConvertLayoutOp: XeGPU_Op<"convert_layout", [Pure, AllTypesMatch<["sou
     let hasCanonicalizer = 1;
 }
 
+def isSharedPred : CPred<"isSharedMemory(llvm::cast<mlir::MemRefType>($_self))">;
+class StaticShared1DMemRefOf<list<Type> allowedTypes> :
+  ConfinedType<MemRefRankOf<allowedTypes, [1]>, [HasStaticShapePred, isSharedPred],
+     "statically shaped " # MemRefOf<allowedTypes>.summary # " for shared memory",
+     "mlir::MemRefType">;
+
+class SizeInBits<string name> :
+  StrFunc<"llvm::cast<mlir::ShapedType>($" # name # ".getType()).getNumElements()"
+          "*llvm::cast<mlir::ShapedType>($" # name # ".getType()).getElementTypeBitWidth()">;
+class AllMemSizesMatch<list<string> names> :
+    AllMatchSameOperatorTrait<names, SizeInBits<"_self">.result,
+                              "size in bits">;
+
+def XeGPU_CreateMemDescOp: XeGPU_Op<"create_mem_desc", [Pure,
+      AllMemSizesMatch<["source", "mem_desc"]>]>  {
+  let summary = "Create a memory descriptor.";
+  let description = [{
+    Creates a memory descriptor from a shared local memory (SLM) buffer, and xegpu
+    specific memory layout. The resulting memory descriptor has to have the same size
+    as the underlying shared local memory.
+
+    Arguments:
+     - `source` : a 1D statically shaped memref with element type i8, representing the raw SLM buffer.
+    Results:
+     - `mem_desc` : the memory descriptor.
+  }];
+  let arguments = (ins StaticShared1DMemRefOf<[I8]>:$source);
+  let results = (outs XeGPU_MemDesc:$mem_desc);
+  let assemblyFormat = "$source prop-dict attr-dict `` `:` type($source) `->` qualified(type($mem_desc))";
+}
+
+def XeGPU_LoadMatrixOp: XeGPU_Op<"load_matrix", [MemoryEffects<[MemRead]>,
+                              AllElementTypesMatch<["mem_desc", "res"]>,
+                              AllRanksMatch<["mem_desc", "res"]>]>  {
+  let arguments = (ins XeGPU_MemDesc:$mem_desc,
+    Variadic<Index>: $offsets,
+    DenseI64ArrayAttr: $const_offsets,
+    OptionalAttr<DistributeLayoutAttr>:$layout
+  );
+  let results = (outs XeGPU_ValueType:$res);
+  let assemblyFormat = [{
+    $mem_desc `` custom<DynamicIndexList>($offsets, $const_offsets)
+    prop-dict attr-dict `` `:` type(operands) `->` type(results)
+  }];
+
+  let description = [{
+    This operation loads a 2D block of data from shared local memory (SLM) as specified
+    by the provided 2D `mem_desc`. Only 2D memory descriptors are supported; use the
+    subview operation to obtain a compatible 2D `mem_desc` from a higher-rank descriptor if needed.
+
+    Arguments:
+     - `mem_desc`: the memory descriptor identifying the SLM region.
+     - `offsets`: the coordinates within the matrix to read from.
+     - `layout`: [optional] An attribute for guiding distributions among
+                 subgroups and/or work-items. It currently can accept either
+                 LayoutAttr or SliceAttr.
+    Results:
+     - `res`: the matrix elements loaded from SLM.
+  }];
+
+  let builders = [
+    OpBuilder<(ins "Type":$res, "TypedValue<MemDescType>": $mem_desc,
+                    "llvm::ArrayRef<OpFoldResult>": $offsets, "DistributeLayoutAttr": $layout)>,
+  ];
+  let extraClassDeclaration = [{
+    SmallVector<OpFoldResult> getMixedOffsets() {
+      return getMixedValues(getConstOffsets(), getOffsets(), getContext());
+    }
+
+    ArrayRef<int64_t> getDataShape() {
+      return getRes().getType().getShape();
+    }
+  }];
+
+  let hasVerifier = 1;
+}
+
+def XeGPU_StoreMatrixOp: XeGPU_Op<"store_matrix", [MemoryEffects<[MemWrite]>,
+                              AllElementTypesMatch<["mem_desc", "data"]>,
+                              AllRanksMatch<["mem_desc", "data"]>]> {
+  let arguments = (ins
+    XeGPU_ValueType:$data,
+    XeGPU_MemDesc:$mem_desc,
+    Variadic<Index>: $offsets,
+    DenseI64ArrayAttr: $const_offsets,
+    OptionalAttr<DistributeLayoutAttr>:$layout
+  );
+  let assemblyFormat = [{ $data `,` $mem_desc `` custom<DynamicIndexList>($offsets, $const_offsets)
+                          prop-dict attr-dict `` `:` type(operands)}];
+  let description = [{
+    This operation stores a 2D `data` fragment into the shared local memory region
+    specified by a 2D `mem_desc`. Only 2D memory descriptors are supported; use the
+    subview operation to obtain a 2D `mem_desc` from a higher-rank descriptor if needed.
+
+    Arguments:
+     - `mem_desc`: the memory descriptor specifying the SLM region.
+     - `offsets`: the coordinates within the matrix where the data will be written.
+     - `data`: the values to be stored in the matrix.
+     - `layout`: [optional] An attribute for guiding distributions among
+                 subgroups and/or work-items. It currently can accept either
+                 LayoutAttr or SliceAttr.
+  }];
+  let builders = [
+    OpBuilder<(ins "Value" : $data, "TypedValue<MemDescType>": $mem_desc,
+                   "llvm::ArrayRef<OpFoldResult>": $offsets, "DistributeLayoutAttr": $layout)>,
+  ];
+  let extraClassDeclaration = [{
+    SmallVector<OpFoldResult> getMixedOffsets() {
+      return getMixedValues(getConstOffsets(), getOffsets(), getContext());
+    }
+
+    ArrayRef<int64_t> getDataShape() {
+      return getData().getType().getShape();
+    }
+
+  }];
+
+  let hasVerifier = 1;
+}
+
+def XeGPU_MemDescSubviewOp: XeGPU_Op<"mem_desc_subview",
+          [Pure, ViewLikeOpInterface, AllElementTypesMatch<["src", "res"]>]> {
+  let description = [{
+    Creates a subview of a memory descriptor. The resulting memory descriptor can have
+    a lower rank than the source; in this case, the result dimensions correspond to the
+    higher-order dimensions of the source memory descriptor.
+
+    Arguments:
+     - `src` : a memory descriptor.
+     - `offsets` : the coordinates within the matrix the subview will be created from.
+
+    Results:
+    - `res` : a memory descriptor with smaller size.
+
+  }];
+  let arguments = (ins XeGPU_MemDesc:$src,
+                       Variadic<Index>:$offsets,
+                       DenseI64ArrayAttr:$const_offsets);
+  let results = (outs XeGPU_MemDesc:$res);
+  let assemblyFormat = [{$src `` custom<DynamicIndexList>($offsets, $const_offsets) prop-dict
+                         attr-dict `` `:` qualified(type($src)) `->` qualified(type($res))}];
+  let builders = [
+    OpBuilder<(ins "Type": $res, "Value":$src, "llvm::ArrayRef<OpFoldResult>": $offsets)>
+  ];
+
+  let extraClassDeclaration = [{
+    mlir::Value getViewSource() { return getSrc(); }
+
+    SmallVector<OpFoldResult> getMixedOffsets() {
+      return getMixedValues(getConstOffsets(), getOffsets(), getContext());
+    }
+  }];
+
+  let hasVerifier = 1;
+}
+
+
 #endif // MLIR_DIALECT_XEGPU_IR_XEGPUOPS_TD
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
index b268cabb5d266..84902b2039643 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
@@ -16,13 +16,17 @@ include "mlir/IR/BuiltinTypes.td"
 def XeGPU_IntType: AnyTypeOf<[I1, I8, I16, I32, I64, SI1, SI8, SI16, SI32, SI64, UI1, UI8, UI16, UI32, UI64]>;
 def XeGPU_FloatType: AnyTypeOf<[F16, F32, F64, BF16, TF32]>;
 def XeGPU_ScalarType: AnyTypeOf<[XeGPU_IntType, XeGPU_FloatType]>;
-def XeGPU_BaseAddrType: AnyTypeOf<[Non0RankedMemRefOf<[XeGPU_ScalarType]>, UI64, UI32, I64, I32]>;
+def XeGPU_PointerType : AnyTypeOf<[UI64, UI32, I64, I32]>;
+def XeGPU_BaseAddrType
+    : AnyTypeOf<[Non0RankedMemRefOf<[XeGPU_ScalarType]>, XeGPU_PointerType]>;
 def XeGPU_DpasOprType: FixedVectorOfRankAndType<[1, 2, 3], [XeGPU_ScalarType]>;
 def XeGPU_DpasResType: FixedVectorOfRankAndType<[1, 2], [XeGPU_ScalarType]>;
 def XeGPU_OffsetType: FixedVectorOfNonZeroRankOf<[Index]>;
 def XeGPU_MaskType: FixedVectorOfNonZeroRankOf<[I1]>;
 def XeGPU_ValueType: FixedVectorOfNonZeroRankOf<[XeGPU_ScalarType]>;
 def XeGPU_VectorType: VectorOfRankAndType<[1,2,3,4,5,6], [XeGPU_ScalarType]>;
+def XeGPU_GatherScatterBaseAddrType
+    : AnyTypeOf<[MemRefRankOf<[XeGPU_ScalarType], [1]>, XeGPU_PointerType]>;
 
 // common base class for types in XeGPU dialect
 class XeGPUTypeDef<string name, string typeMnemonic, list<Trait> traits = [],
@@ -189,7 +193,8 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
   let genVerifyDecl = 1;
 }
 
-def XeGPU_GatherScatterSourceType : AnyTypeOf<[XeGPU_TensorDesc,Non0RankedMemRefOf<[XeGPU_ScalarType]>, UI64]>;
+def XeGPU_GatherScatterSourceType
+    : AnyTypeOf<[XeGPU_TensorDesc, XeGPU_GatherScatterBaseAddrType]>;
 
 def XeGPU_Nbarrier: XeGPUTypeDef<"Nbarrier", "nbarrier", [], "mlir::Type"> {
   let summary = "!xegpu.nbarrier a custom XeGPU type representing a barrier.";
@@ -201,4 +206,53 @@ def XeGPU_Nbarrier: XeGPUTypeDef<"Nbarrier", "nbarrier", [], "mlir::Type"> {
   }];
 }
 
+def XeGPU_MemDesc: XeGPUTypeDef<"MemDesc", "mem_desc", [ShapedTypeInterface], "mlir::Type"> {
+  let summary = "MemDesc describing the data in SLM";
+  let description = [{
+    MemDesc represents a block of data stored in shared local memory.
+    By default, unless a layout attribute is provided, the data is stored
+    contiguously in row-major order within the region.
+
+    Examples:
+    ```mlir
+    // A multi-dimensional array stored in column-major order.
+    !xegpu.mem_desc<128x128xf16, #xegpu.mem_layout<stride = [1, 128]>>
+
+    // A multi-dimensional array stored in a blocked layout. Elements within the same block
+    // are stored contiguously in memory. Blocks are stored in row-major order.
+    !xegpu.mem_desc<128x128xf16, #xegpu.mem_layout<block = [8, 8]>>
+
+    // A multi-dimensional array stored in column-major order with blocked layout.
+    !xegpu.mem_desc<128x128xf16, #xegpu.mem_layout<stride = [1, 128], block = [8, 8]>>
+    ```
+  }];
+  let parameters = (ins ArrayRefParameter<"int64_t">: $shape,
+                        "mlir::Type": $elementType,
+                        OptionalParameter<"MemLayoutAttr">: $mem_layout);
+
+  let extraClassDeclaration = [{
+    bool hasRank() const { return true; }
+
+    MemDescType cloneWith(std::optional<llvm::ArrayRef<int64_t>> shape, Type elementType) const {
+      return MemDescType::get(getContext(), shape.value_or(getShape()), elementType, getMemLayout());
+    }
+
+    ArrayAttr getStrides() {
+      auto layout = getMemLayout();
+      if (layout && layout.hasAttr("stride")) {
+        return layout.getStrides();
+      }
+
+      // derive and return default strides
+      SmallVector<int64_t> defaultStrides;
+      llvm::append_range(defaultStrides, getShape().drop_front());
+      llvm::append_values(defaultStrides, 1);
+      Builder builder(getContext());
+      return builder.getI64ArrayAttr(defaultStrides);
+    }
+  }];
+
+  let hasCustomAssemblyFormat = true;
+}
+
 #endif // MLIR_DIALECT_XEGPU_IR_XEGPUTYPES_TD
diff --git a/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h b/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
index 488f358ff3802..b2b2d3ab85231 100644
--- a/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
+++ b/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
@@ -10,6 +10,7 @@
 #define MLIR_DIALECT_XEGPU_UTILS_XEGPUUTILS_H_
 
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/OpDefinition.h"
 namespace mlir {
 
 class VectorType;
@@ -123,6 +124,25 @@ Value createVectorWithShapeFromValues(OpBuilder &builder, Location loc,
 void doSCFStructuralTypeConversionWithTensorType(Operation *op,
                                                  TypeConverter converter);
 
+/// Retrieves the chip string from the XeVM target attribute of the parent
+/// GPU module operation. Returns the chip identifier if found, or nullopt
+/// if no GPU module parent or XeVM target attribute exists.
+std::optional<std::string> getChipStr(Operation *op);
+
+/// Generates element-wise addition ops of two arrays with automatic alignment.
+/// When the input arrays have different sizes, the shorter array is
+/// right-aligned with the longer array, and the unmatched leading elements from
+/// the longer array are preserved unchanged. This is commonly used for offset
+/// computation where higher-dimensional offsets need to be added to
+/// lower-dimensional adjustments.
+///
+/// Example:
+///   lhs = [l1, l2, l3], rhs = [r1, r2]
+///   Result: [11, l2+r1, l3+r2]
+SmallVector<OpFoldResult> addWithRightAligned(OpBuilder &builder, Location loc,
+                                              ArrayRef<OpFoldResult> lhs,
+                                              ArrayRef<OpFoldResult> rhs);
+
 } // namespace xegpu
 
 } // namespace mlir
diff --git a/mlir/include/mlir/ExecutionEngine/ExecutionEngine.h b/mlir/include/mlir/ExecutionEngine/ExecutionEngine.h
index 96ccebcd5685e..5bd71d68d253a 100644
--- a/mlir/include/mlir/ExecutionEngine/ExecutionEngine.h
+++ b/mlir/include/mlir/ExecutionEngine/ExecutionEngine.h
@@ -227,6 +227,13 @@ class ExecutionEngine {
       llvm::function_ref<llvm::orc::SymbolMap(llvm::orc::MangleAndInterner)>
           symbolMap);
 
+  /// Initialize the ExecutionEngine. Global constructors specified by
+  /// `llvm.mlir.global_ctors` will be run. One common scenario is that kernel
+  /// binary compiled from `gpu.module` gets loaded during initialization. Make
+  /// sure all symbols are resolvable before initialization by calling
+  /// `registerSymbols` or including shared libraries.
+  void initialize();
+
 private:
   /// Ordering of llvmContext and jit is important for destruction purposes: the
   /// jit must be destroyed before the context.
@@ -250,6 +257,8 @@ class ExecutionEngine {
   /// Destroy functions in the libraries loaded by the ExecutionEngine that are
   /// called when this ExecutionEngine is destructed.
   SmallVector<LibraryDestroyFn> destroyFns;
+
+  bool isInitialized = false;
 };
 
 } // namespace mlir
diff --git a/mlir/include/mlir/ExecutionEngine/MemRefUtils.h b/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
index 6e72f7c23bdcf..d66d757cb7a8e 100644
--- a/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
+++ b/mlir/include/mlir/ExecutionEngine/MemRefUtils.h
@@ -151,7 +151,7 @@ class OwningMemRef {
       AllocFunType allocFun = &::malloc,
       std::function<void(StridedMemRefType<T, Rank>)> freeFun =
           [](StridedMemRefType<T, Rank> descriptor) {
-            ::free(descriptor.data);
+            ::free(descriptor.basePtr);
           })
       : freeFunc(freeFun) {
     if (shapeAlloc.empty())
diff --git a/mlir/include/mlir/IR/Block.h b/mlir/include/mlir/IR/Block.h
index e486bb627474d..85ce66f69df48 100644
--- a/mlir/include/mlir/IR/Block.h
+++ b/mlir/include/mlir/IR/Block.h
@@ -205,12 +205,14 @@ class alignas(8) Block : public IRObjectWithUseList<BlockOperand>,
   }
 
   /// Return an iterator range over the operation within this block excluding
-  /// the terminator operation at the end.
+  /// the terminator operation at the end. If the block has no terminator,
+  /// return an iterator range over the entire block. If it is unknown if the
+  /// block has a terminator (i.e., last block operation is unregistered), also
+  /// return an iterator range over the entire block.
   iterator_range<iterator> without_terminator() {
     if (begin() == end())
       return {begin(), end()};
-    auto endIt = --end();
-    return {begin(), endIt};
+    return without_terminator_impl();
   }
 
   //===--------------------------------------------------------------------===//
@@ -221,7 +223,8 @@ class alignas(8) Block : public IRObjectWithUseList<BlockOperand>,
   /// the block might have a valid terminator operation.
   Operation *getTerminator();
 
-  /// Check whether this block might have a terminator.
+  /// Return "true" if this block might have a terminator. Return "true" if
+  /// the last operation is unregistered.
   bool mightHaveTerminator();
 
   //===--------------------------------------------------------------------===//
@@ -402,6 +405,9 @@ class alignas(8) Block : public IRObjectWithUseList<BlockOperand>,
   void printAsOperand(raw_ostream &os, AsmState &state);
 
 private:
+  /// Same as `without_terminator`, but assumes that the block is not empty.
+  iterator_range<iterator> without_terminator_impl();
+
   /// Pair of the parent object that owns this block and a bit that signifies if
   /// the operations within this block have a valid ordering.
   llvm::PointerIntPair<Region *, /*IntBits=*/1, bool> parentValidOpOrderPair;
diff --git a/mlir/include/mlir/IR/CommonTypeConstraints.td b/mlir/include/mlir/IR/CommonTypeConstraints.td
index 45ec1846580f2..b682f4c025a46 100644
--- a/mlir/include/mlir/IR/CommonTypeConstraints.td
+++ b/mlir/include/mlir/IR/CommonTypeConstraints.td
@@ -576,7 +576,7 @@ class NormalizeIndex<int value> {
 class IsNthDimSizeIsOneOfPred<int n, list<int> allowedSizes>
   : And<[
       CPred<"::llvm::cast<::mlir::ShapedType>($_self).getRank() >= " # NormalizeIndex<n>.ret>,
-      CPred<"::llvm::is_contained(ArrayRef<int64_t>({" # !interleave(allowedSizes, ", ") # "}), "
+      CPred<"::llvm::is_contained(::llvm::ArrayRef<int64_t>({" # !interleave(allowedSizes, ", ") # "}), "
         # "::llvm::cast<::mlir::ShapedType>($_self).getDimSize("
         #   !if(!lt(n, 0),
               "::llvm::cast<::mlir::ShapedType>($_self).getRank() + " # n,
@@ -585,7 +585,7 @@ class IsNthDimSizeIsOneOfPred<int n, list<int> allowedSizes>
 
 // Whether the shape of a vector matches the given `shape` list.
 class IsVectorOfShape<list<int> shape>
-  : CPred<"::llvm::cast<::mlir::VectorType>($_self).getShape() == ArrayRef<int64_t>({" # !interleave(shape, ", ") # "})">;
+  : CPred<"::llvm::cast<::mlir::VectorType>($_self).getShape() == ::llvm::ArrayRef<int64_t>({" # !interleave(shape, ", ") # "})">;
 
 // Any vector where the number of elements is from the given
 // `allowedLengths` list
diff --git a/mlir/include/mlir/IR/MLIRContext.h b/mlir/include/mlir/IR/MLIRContext.h
index ef8dab87f131a..9690029256474 100644
--- a/mlir/include/mlir/IR/MLIRContext.h
+++ b/mlir/include/mlir/IR/MLIRContext.h
@@ -34,6 +34,9 @@ class MLIRContextImpl;
 class RegisteredOperationName;
 class StorageUniquer;
 class IRUnit;
+namespace remark::detail {
+class RemarkEngine;
+} // namespace remark::detail
 
 /// MLIRContext is the top-level object for a collection of MLIR operations. It
 /// holds immortal uniqued objects like types, and the tables used to unique
@@ -212,6 +215,13 @@ class MLIRContext {
   /// Returns the diagnostic engine for this context.
   DiagnosticEngine &getDiagEngine();
 
+  /// Returns the remark engine for this context, or nullptr if none has been
+  /// set.
+  remark::detail::RemarkEngine *getRemarkEngine();
+
+  /// Set the remark engine for this context.
+  void setRemarkEngine(std::unique_ptr<remark::detail::RemarkEngine> engine);
+
   /// Returns the storage uniquer used for creating affine constructs.
   StorageUniquer &getAffineUniquer();
 
diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td
index 9e5fb5659a22b..af8c072a7a364 100644
--- a/mlir/include/mlir/IR/OpBase.td
+++ b/mlir/include/mlir/IR/OpBase.td
@@ -603,6 +603,51 @@ class RangedTypesMatchWith<string summary, string lhsArg, string rhsArg,
                            string transform>
   : TypesMatchWith<summary, lhsArg, rhsArg, transform, "llvm::equal">;
 
+// Checks that each inputArg has the same type as the corresponding entry
+// in allowedTypes
+class InputMatchesTypes<list<string> inputArgs, list<Type> allowedTypes> :
+    PredOpTrait<"operands {" # !interleave(inputArgs, ", ") # "} match expected types",
+                !foldl(TruePred, !range(!size(inputArgs)), acc, i,
+                       And<[acc,
+                           SubstLeaves<"$_self", "$" # inputArgs[i] # ".getType()",
+                                      allowedTypes[i].predicate>
+                       ]>)> {
+    assert !eq(!size(inputArgs), !size(allowedTypes)),
+           "inputArgs and allowedTypes lists must have the same length";
+
+    list<string> inputArgList = inputArgs;
+    list<Type> allowedTypeList = allowedTypes;
+}
+
+// Checks that inputArgs match one of the allowed type combinations.
+// Each combination in allowedCombinations must have the same number of types
+// as there are inputArgs.
+class InputAddressIsCombinationOf<list<string> inputArgs, 
+                                  list<list<Type>> allowedCombinations,
+                                  string description = ""> :
+    PredOpTrait<!if(!empty(description), 
+                    "operands {" # !interleave(inputArgs, ", ") # "} match one of the allowed type combinations",
+                    description),
+                Or<!foreach(combination, allowedCombinations,
+                           !foldl(TruePred, !range(!size(inputArgs)), acc, i,
+                                  And<[acc,
+                                      SubstLeaves<"$_self", "$" # inputArgs[i] # ".getType()",
+                                                 combination[i].predicate>
+                                  ]>))>> {
+    assert !gt(!size(allowedCombinations), 0),
+           "allowedCombinations must not be empty";
+    
+    // Validate that each combination has the same number of types as inputArgs
+    defvar inputArgSize = !size(inputArgs);
+    defvar validSizes = !foldl(1, allowedCombinations, acc, combination,
+                              !and(acc, !eq(inputArgSize, !size(combination))));
+    assert validSizes,
+           "each combination in allowedCombinations must have the same length as inputArgs";
+    
+    list<string> inputArgList = inputArgs;
+    list<list<Type>> allowedCombinationList = allowedCombinations;
+}
+
 // Type Constraint operand `idx`'s Element type is `type`.
 class TCopVTEtIs<int idx, Type type> : And<[
    CPred<"$_op.getNumOperands() > " # idx>,
diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h
index 4f89f8b85861e..5569392cf0b41 100644
--- a/mlir/include/mlir/IR/Operation.h
+++ b/mlir/include/mlir/IR/Operation.h
@@ -1106,6 +1106,8 @@ inline raw_ostream &operator<<(raw_ostream &os, const Operation &op) {
 /// useful to act as a "stream modifier" to customize printing an operation
 /// with a stream using the operator<< overload, e.g.:
 ///   llvm::dbgs() << OpWithFlags(op, OpPrintingFlags().skipRegions());
+/// This always prints the operation with the local scope, to avoid introducing
+/// spurious newlines in the stream.
 class OpWithFlags {
 public:
   OpWithFlags(Operation *op, OpPrintingFlags flags = {})
@@ -1116,11 +1118,11 @@ class OpWithFlags {
 private:
   Operation *op;
   OpPrintingFlags theFlags;
-  friend raw_ostream &operator<<(raw_ostream &os, const OpWithFlags &op);
+  friend raw_ostream &operator<<(raw_ostream &os, OpWithFlags op);
 };
 
-inline raw_ostream &operator<<(raw_ostream &os,
-                               const OpWithFlags &opWithFlags) {
+inline raw_ostream &operator<<(raw_ostream &os, OpWithFlags opWithFlags) {
+  opWithFlags.flags().useLocalScope();
   opWithFlags.op->print(os, opWithFlags.flags());
   return os;
 }
diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h
index b5a93a0c5a898..57e73c1d8c7c1 100644
--- a/mlir/include/mlir/IR/PatternMatch.h
+++ b/mlir/include/mlir/IR/PatternMatch.h
@@ -311,14 +311,14 @@ struct OpOrInterfaceRewritePatternBase : public RewritePattern {
 /// opposed to a raw Operation.
 template <typename SourceOp>
 struct OpRewritePattern
-    : public detail::OpOrInterfaceRewritePatternBase<SourceOp> {
+    : public mlir::detail::OpOrInterfaceRewritePatternBase<SourceOp> {
 
   /// Patterns must specify the root operation name they match against, and can
   /// also specify the benefit of the pattern matching and a list of generated
   /// ops.
   OpRewritePattern(MLIRContext *context, PatternBenefit benefit = 1,
                    ArrayRef<StringRef> generatedNames = {})
-      : detail::OpOrInterfaceRewritePatternBase<SourceOp>(
+      : mlir::detail::OpOrInterfaceRewritePatternBase<SourceOp>(
             SourceOp::getOperationName(), benefit, context, generatedNames) {}
 };
 
@@ -327,10 +327,10 @@ struct OpRewritePattern
 /// of a raw Operation.
 template <typename SourceOp>
 struct OpInterfaceRewritePattern
-    : public detail::OpOrInterfaceRewritePatternBase<SourceOp> {
+    : public mlir::detail::OpOrInterfaceRewritePatternBase<SourceOp> {
 
   OpInterfaceRewritePattern(MLIRContext *context, PatternBenefit benefit = 1)
-      : detail::OpOrInterfaceRewritePatternBase<SourceOp>(
+      : mlir::detail::OpOrInterfaceRewritePatternBase<SourceOp>(
             Pattern::MatchInterfaceOpTypeTag(), SourceOp::getInterfaceID(),
             benefit, context) {}
 };
diff --git a/mlir/include/mlir/IR/Remarks.h b/mlir/include/mlir/IR/Remarks.h
new file mode 100644
index 0000000000000..26d65472f2b1c
--- /dev/null
+++ b/mlir/include/mlir/IR/Remarks.h
@@ -0,0 +1,520 @@
+//===- Remarks.h - MLIR Optimization Remark ----------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines utilities for emitting optimization remarks.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_IR_REMARKS_H
+#define MLIR_IR_REMARKS_H
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/IR/DiagnosticInfo.h"
+#include "llvm/Remarks/Remark.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/Regex.h"
+#include <optional>
+
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/Value.h"
+
+namespace mlir::remark {
+
+/// Define an the set of categories to accept. By default none are, the provided
+/// regex matches against the category names for each kind of remark.
+struct RemarkCategories {
+  std::optional<std::string> passed, missed, analysis, failed;
+};
+
+/// Categories describe the outcome of an transformation, not the mechanics of
+/// emitting/serializing remarks.
+enum class RemarkKind {
+  RemarkUnknown = 0,
+
+  /// An optimization was applied.
+  RemarkPassed,
+
+  /// A profitable optimization opportunity was found but not applied.
+  RemarkMissed,
+
+  /// The compiler attempted the optimization but failed (e.g., legality
+  /// checks, or better opportunites).
+  RemarkFailure,
+
+  /// Informational context (e.g., analysis numbers) without a pass/fail
+  /// outcome.
+  RemarkAnalysis,
+};
+
+using namespace llvm;
+
+/// Options to create a Remark
+struct RemarkOpts {
+  StringRef remarkName;      // Identifiable name
+  StringRef categoryName;    // Category name (subject to regex filtering)
+  StringRef subCategoryName; // Subcategory name
+  StringRef functionName;    // Function name if available
+
+  // Construct RemarkOpts from a remark name.
+  static constexpr RemarkOpts name(StringRef n) {
+    return RemarkOpts{n, {}, {}, {}};
+  }
+  /// Return a copy with the category set.
+  constexpr RemarkOpts category(StringRef v) const {
+    return {remarkName, v, subCategoryName, functionName};
+  }
+  /// Return a copy with the subcategory set.
+  constexpr RemarkOpts subCategory(StringRef v) const {
+    return {remarkName, categoryName, v, functionName};
+  }
+  /// Return a copy with the function name set.
+  constexpr RemarkOpts function(StringRef v) const {
+    return {remarkName, categoryName, subCategoryName, v};
+  }
+};
+
+} // namespace mlir::remark
+
+namespace mlir::remark::detail {
+//===----------------------------------------------------------------------===//
+// Remark Base Class
+//===----------------------------------------------------------------------===//
+class Remark {
+
+public:
+  Remark(RemarkKind remarkKind, DiagnosticSeverity severity, Location loc,
+         RemarkOpts opts)
+      : remarkKind(remarkKind), functionName(opts.functionName), loc(loc),
+        categoryName(opts.categoryName), subCategoryName(opts.subCategoryName),
+        remarkName(opts.remarkName) {
+    if (!categoryName.empty() && !subCategoryName.empty()) {
+      (llvm::Twine(categoryName) + ":" + subCategoryName)
+          .toStringRef(fullCategoryName);
+    }
+  }
+
+  // Remark argument that is a key-value pair that can be printed as machine
+  // parsable args.
+  struct Arg {
+    std::string key;
+    std::string val;
+    Arg(llvm::StringRef m) : key("Remark"), val(m) {}
+    Arg(llvm::StringRef k, llvm::StringRef v) : key(k), val(v) {}
+    Arg(llvm::StringRef k, std::string v) : key(k), val(std::move(v)) {}
+    Arg(llvm::StringRef k, const char *v) : Arg(k, llvm::StringRef(v)) {}
+    Arg(llvm::StringRef k, Value v);
+    Arg(llvm::StringRef k, Type t);
+    Arg(llvm::StringRef k, bool b) : key(k), val(b ? "true" : "false") {}
+
+    // One constructor for all arithmetic types except bool.
+    template <typename T, typename = std::enable_if_t<std::is_arithmetic_v<T> &&
+                                                      !std::is_same_v<T, bool>>>
+    Arg(llvm::StringRef k, T v) : key(k) {
+      if constexpr (std::is_floating_point_v<T>) {
+        llvm::raw_string_ostream os(val);
+        os << v;
+      } else if constexpr (std::is_signed_v<T>) {
+        val = llvm::itostr(static_cast<long long>(v));
+      } else {
+        val = llvm::utostr(static_cast<unsigned long long>(v));
+      }
+    }
+  };
+
+  void insert(llvm::StringRef s);
+  void insert(Arg a);
+
+  void print(llvm::raw_ostream &os, bool printLocation = false) const;
+
+  Location getLocation() const { return loc; }
+  /// Diagnostic -> Remark
+  llvm::remarks::Remark generateRemark() const;
+
+  StringRef getFunction() const {
+    if (!functionName.empty())
+      return functionName;
+    return "<unknown function>";
+  }
+
+  llvm::StringRef getCategoryName() const { return categoryName; }
+
+  llvm::StringRef getFullCategoryName() const {
+    if (categoryName.empty() && subCategoryName.empty())
+      return {};
+    if (subCategoryName.empty())
+      return categoryName;
+    if (categoryName.empty())
+      return subCategoryName;
+    return fullCategoryName;
+  }
+
+  StringRef getRemarkName() const {
+    if (remarkName.empty())
+      return "<unknown remark name>";
+    return remarkName;
+  }
+
+  std::string getMsg() const;
+
+  ArrayRef<Arg> getArgs() const { return args; }
+
+  llvm::remarks::Type getRemarkType() const;
+
+  StringRef getRemarkTypeString() const;
+
+protected:
+  /// Keeps the MLIR diagnostic kind, which is used to determine the
+  /// diagnostic kind in the LLVM remark streamer.
+  RemarkKind remarkKind;
+  /// Name of the convering function like interface
+  StringRef functionName;
+
+  Location loc;
+  /// Sub category passname e.g., "Unroll" or "UnrollAndJam"
+  StringRef categoryName;
+
+  /// Sub category name "Loop Optimizer"
+  StringRef subCategoryName;
+
+  /// Combined name for category and sub-category
+  SmallString<64> fullCategoryName;
+
+  /// Remark identifier
+  StringRef remarkName;
+
+  /// Args collected via the streaming interface.
+  SmallVector<Arg, 4> args;
+
+private:
+  /// Convert the MLIR diagnostic severity to LLVM diagnostic severity.
+  static llvm::DiagnosticSeverity
+  makeLLVMSeverity(DiagnosticSeverity severity) {
+    switch (severity) {
+    case DiagnosticSeverity::Note:
+      return llvm::DiagnosticSeverity::DS_Note;
+    case DiagnosticSeverity::Warning:
+      return llvm::DiagnosticSeverity::DS_Warning;
+    case DiagnosticSeverity::Error:
+      return llvm::DiagnosticSeverity::DS_Error;
+    case DiagnosticSeverity::Remark:
+      return llvm::DiagnosticSeverity::DS_Remark;
+    }
+    llvm_unreachable("Unknown diagnostic severity");
+  }
+  /// Convert the MLIR remark kind to LLVM diagnostic kind.
+  static llvm::DiagnosticKind makeLLVMKind(RemarkKind remarkKind) {
+    switch (remarkKind) {
+    case RemarkKind::RemarkUnknown:
+      return llvm::DiagnosticKind::DK_Generic;
+    case RemarkKind::RemarkPassed:
+      return llvm::DiagnosticKind::DK_OptimizationRemark;
+    case RemarkKind::RemarkMissed:
+      return llvm::DiagnosticKind::DK_OptimizationRemarkMissed;
+    case RemarkKind::RemarkFailure:
+      return llvm::DiagnosticKind::DK_OptimizationFailure;
+    case RemarkKind::RemarkAnalysis:
+      return llvm::DiagnosticKind::DK_OptimizationRemarkAnalysis;
+    }
+    llvm_unreachable("Unknown diagnostic kind");
+  }
+};
+
+inline Remark &operator<<(Remark &r, StringRef s) {
+  r.insert(s);
+  return r;
+}
+inline Remark &&operator<<(Remark &&r, StringRef s) {
+  r.insert(s);
+  return std::move(r);
+}
+inline Remark &operator<<(Remark &r, const Remark::Arg &kv) {
+  r.insert(kv);
+  return r;
+}
+
+//===----------------------------------------------------------------------===//
+// Shorthand aliases for different kinds of remarks.
+//===----------------------------------------------------------------------===//
+
+template <RemarkKind K, DiagnosticSeverity S>
+class OptRemarkBase final : public Remark {
+public:
+  explicit OptRemarkBase(Location loc, RemarkOpts opts)
+      : Remark(K, S, loc, opts) {}
+};
+
+using OptRemarkAnalysis =
+    OptRemarkBase<RemarkKind::RemarkAnalysis, DiagnosticSeverity::Remark>;
+
+using OptRemarkPass =
+    OptRemarkBase<RemarkKind::RemarkPassed, DiagnosticSeverity::Remark>;
+
+using OptRemarkMissed =
+    OptRemarkBase<RemarkKind::RemarkMissed, DiagnosticSeverity::Remark>;
+
+using OptRemarkFailure =
+    OptRemarkBase<RemarkKind::RemarkFailure, DiagnosticSeverity::Remark>;
+
+class RemarkEngine;
+
+//===----------------------------------------------------------------------===//
+// InFlightRemark
+//===----------------------------------------------------------------------===//
+
+/// Lazy text building for zero cost string formatting.
+struct LazyTextBuild {
+  llvm::StringRef key;
+  std::function<std::string()> thunk;
+};
+
+/// InFlightRemark is a RAII class that holds a reference to a Remark
+/// instance and allows to build the remark using the << operator. The remark
+/// is emitted when the InFlightRemark instance is destroyed, which happens
+/// when the scope ends or when the InFlightRemark instance is moved.
+/// Similar to InFlightDiagnostic, but for remarks.
+class InFlightRemark {
+public:
+  explicit InFlightRemark(std::unique_ptr<Remark> diag)
+      : remark(std::move(diag)) {}
+
+  InFlightRemark(RemarkEngine &eng, std::unique_ptr<Remark> diag)
+      : owner(&eng), remark(std::move(diag)) {}
+
+  InFlightRemark() = default; // empty ctor
+
+  InFlightRemark &operator<<(const LazyTextBuild &l) {
+    if (remark)
+      *remark << Remark::Arg(l.key, l.thunk());
+    return *this;
+  }
+
+  // Generic path, but *not* for Lazy
+  template <typename T, typename = std::enable_if_t<
+                            !std::is_same_v<std::decay_t<T>, LazyTextBuild>>>
+  InFlightRemark &operator<<(T &&arg) {
+    if (remark)
+      *remark << std::forward<T>(arg);
+    return *this;
+  }
+
+  explicit operator bool() const { return remark != nullptr; }
+
+  ~InFlightRemark();
+
+  InFlightRemark(const InFlightRemark &) = delete;
+  InFlightRemark &operator=(const InFlightRemark &) = delete;
+  InFlightRemark(InFlightRemark &&) = default;
+  InFlightRemark &operator=(InFlightRemark &&) = default;
+
+private:
+  RemarkEngine *owner{nullptr};
+  std::unique_ptr<Remark> remark;
+};
+
+//===----------------------------------------------------------------------===//
+// MLIR Remark Streamer
+//===----------------------------------------------------------------------===//
+
+/// Base class for MLIR remark streamers that is used to stream
+/// optimization remarks to the underlying remark streamer. The derived classes
+/// should implement the `streamOptimizationRemark` method to provide the
+/// actual streaming implementation.
+class MLIRRemarkStreamerBase {
+public:
+  virtual ~MLIRRemarkStreamerBase() = default;
+  /// Stream an optimization remark to the underlying remark streamer. It is
+  /// called by the RemarkEngine to stream the optimization remarks.
+  ///
+  /// It must be overridden by the derived classes to provide
+  /// the actual streaming implementation.
+  virtual void streamOptimizationRemark(const Remark &remark) = 0;
+
+  virtual void finalize() {} // optional
+};
+
+//===----------------------------------------------------------------------===//
+// Remark Engine (MLIR Context will own this class)
+//===----------------------------------------------------------------------===//
+
+class RemarkEngine {
+private:
+  /// Regex that filters missed optimization remarks: only matching one are
+  /// reported.
+  std::optional<llvm::Regex> missFilter;
+  /// The category for passed optimization remarks.
+  std::optional<llvm::Regex> passedFilter;
+  /// The category for analysis remarks.
+  std::optional<llvm::Regex> analysisFilter;
+  /// The category for failed optimization remarks.
+  std::optional<llvm::Regex> failedFilter;
+  /// The MLIR remark streamer that will be used to emit the remarks.
+  std::unique_ptr<MLIRRemarkStreamerBase> remarkStreamer;
+  /// When is enabled, engine also prints remarks as mlir::emitRemarks.
+  bool printAsEmitRemarks = false;
+
+  /// Return true if missed optimization remarks are enabled, override
+  /// to provide different implementation.
+  bool isMissedOptRemarkEnabled(StringRef categoryName) const;
+
+  /// Return true if passed optimization remarks are enabled, override
+  /// to provide different implementation.
+  bool isPassedOptRemarkEnabled(StringRef categoryName) const;
+
+  /// Return true if analysis optimization remarks are enabled, override
+  /// to provide different implementation.
+  bool isAnalysisOptRemarkEnabled(StringRef categoryName) const;
+
+  /// Return true if analysis optimization remarks are enabled, override
+  /// to provide different implementation.
+  bool isFailedOptRemarkEnabled(StringRef categoryName) const;
+
+  /// Return true if any type of remarks are enabled for this pass.
+  bool isAnyRemarkEnabled(StringRef categoryName) const {
+    return isMissedOptRemarkEnabled(categoryName) ||
+           isPassedOptRemarkEnabled(categoryName) ||
+           isFailedOptRemarkEnabled(categoryName) ||
+           isAnalysisOptRemarkEnabled(categoryName);
+  }
+
+  /// Emit a remark using the given maker function, which should return
+  /// a Remark instance. The remark will be emitted using the main
+  /// remark streamer.
+  template <typename RemarkT, typename... Args>
+  InFlightRemark makeRemark(Args &&...args);
+
+  template <typename RemarkT>
+  InFlightRemark emitIfEnabled(Location loc, RemarkOpts opts,
+                               bool (RemarkEngine::*isEnabled)(StringRef)
+                                   const);
+
+public:
+  /// Default constructor is deleted, use the other constructor.
+  RemarkEngine() = delete;
+
+  /// Constructs Remark engine with optional category names. If a category
+  /// name is not provided, it is not enabled. The category names are used to
+  /// filter the remarks that are emitted.
+  RemarkEngine(bool printAsEmitRemarks, const RemarkCategories &cats);
+
+  /// Destructor that will close the output file and reset the
+  /// main remark streamer.
+  ~RemarkEngine();
+
+  /// Setup the remark engine with the given output path and format.
+  LogicalResult initialize(std::unique_ptr<MLIRRemarkStreamerBase> streamer,
+                           std::string *errMsg);
+
+  /// Report a remark.
+  void report(const Remark &&remark);
+
+  /// Report a successful remark, this will create an InFlightRemark
+  /// that can be used to build the remark using the << operator.
+  InFlightRemark emitOptimizationRemark(Location loc, RemarkOpts opts);
+
+  /// Report a missed optimization remark
+  /// that can be used to build the remark using the << operator.
+  InFlightRemark emitOptimizationRemarkMiss(Location loc, RemarkOpts opts);
+
+  /// Report a failed optimization remark, this will create an InFlightRemark
+  /// that can be used to build the remark using the << operator.
+  InFlightRemark emitOptimizationRemarkFailure(Location loc, RemarkOpts opts);
+
+  /// Report an analysis remark, this will create an InFlightRemark
+  /// that can be used to build the remark using the << operator.
+  InFlightRemark emitOptimizationRemarkAnalysis(Location loc, RemarkOpts opts);
+};
+
+template <typename Fn, typename... Args>
+inline InFlightRemark withEngine(Fn fn, Location loc, Args &&...args) {
+  MLIRContext *ctx = loc->getContext();
+
+  RemarkEngine *enginePtr = ctx->getRemarkEngine();
+
+  if (LLVM_UNLIKELY(enginePtr))
+    return (enginePtr->*fn)(loc, std::forward<Args>(args)...);
+
+  return {};
+}
+
+} // namespace mlir::remark::detail
+
+namespace mlir::remark {
+
+/// Create a Reason with llvm::formatv formatting.
+template <class... Ts>
+inline detail::LazyTextBuild reason(const char *fmt, Ts &&...ts) {
+  return {"Reason", [=] { return llvm::formatv(fmt, ts...).str(); }};
+}
+
+/// Create a Suggestion with llvm::formatv formatting.
+template <class... Ts>
+inline detail::LazyTextBuild suggest(const char *fmt, Ts &&...ts) {
+  return {"Suggestion", [=] { return llvm::formatv(fmt, ts...).str(); }};
+}
+
+/// Create a Remark with llvm::formatv formatting.
+template <class... Ts>
+inline detail::LazyTextBuild add(const char *fmt, Ts &&...ts) {
+  return {"Remark", [=] { return llvm::formatv(fmt, ts...).str(); }};
+}
+
+template <class V>
+inline detail::LazyTextBuild metric(StringRef key, V &&v) {
+  using DV = std::decay_t<V>;
+  return {key, [key, vv = DV(std::forward<V>(v))]() mutable {
+            // Reuse Arg's formatting logic and return just the value string.
+            return detail::Remark::Arg(key, std::move(vv)).val;
+          }};
+}
+//===----------------------------------------------------------------------===//
+// Emitters
+//===----------------------------------------------------------------------===//
+
+/// Report an optimization remark that was passed.
+inline detail::InFlightRemark passed(Location loc, RemarkOpts opts) {
+  return withEngine(&detail::RemarkEngine::emitOptimizationRemark, loc, opts);
+}
+
+/// Report an optimization remark that was missed.
+inline detail::InFlightRemark missed(Location loc, RemarkOpts opts) {
+  return withEngine(&detail::RemarkEngine::emitOptimizationRemarkMiss, loc,
+                    opts);
+}
+
+/// Report an optimization remark that failed.
+inline detail::InFlightRemark failed(Location loc, RemarkOpts opts) {
+  return withEngine(&detail::RemarkEngine::emitOptimizationRemarkFailure, loc,
+                    opts);
+}
+
+/// Report an optimization analysis remark.
+inline detail::InFlightRemark analysis(Location loc, RemarkOpts opts) {
+  return withEngine(&detail::RemarkEngine::emitOptimizationRemarkAnalysis, loc,
+                    opts);
+}
+
+//===----------------------------------------------------------------------===//
+// Setup
+//===----------------------------------------------------------------------===//
+
+/// Setup remarks for the context. This function will enable the remark engine
+/// and set the streamer to be used for optimization remarks. The remark
+/// categories are used to filter the remarks that will be emitted by the remark
+/// engine. If a category is not specified, it will not be emitted. If
+/// `printAsEmitRemarks` is true, the remarks will be printed as
+/// mlir::emitRemarks. 'streamer' must inherit from MLIRRemarkStreamerBase and
+/// will be used to stream the remarks.
+LogicalResult enableOptimizationRemarks(
+    MLIRContext &ctx,
+    std::unique_ptr<remark::detail::MLIRRemarkStreamerBase> streamer,
+    const remark::RemarkCategories &cats, bool printAsEmitRemarks = false);
+
+} // namespace mlir::remark
+
+#endif // MLIR_IR_REMARKS_H
diff --git a/mlir/include/mlir/InitAllTranslations.h b/mlir/include/mlir/InitAllTranslations.h
index 1ab80fb27fa9a..622024db5a8a2 100644
--- a/mlir/include/mlir/InitAllTranslations.h
+++ b/mlir/include/mlir/InitAllTranslations.h
@@ -20,6 +20,7 @@ namespace mlir {
 
 void registerFromLLVMIRTranslation();
 void registerFromSPIRVTranslation();
+void registerFromWasmTranslation();
 void registerToCppTranslation();
 void registerToLLVMIRTranslation();
 void registerToSPIRVTranslation();
@@ -36,6 +37,7 @@ inline void registerAllTranslations() {
     registerFromLLVMIRTranslation();
     registerFromSPIRVTranslation();
     registerIRDLToCppTranslation();
+    registerFromWasmTranslation();
     registerToCppTranslation();
     registerToLLVMIRTranslation();
     registerToSPIRVTranslation();
diff --git a/mlir/include/mlir/Interfaces/SideEffectInterfaces.h b/mlir/include/mlir/Interfaces/SideEffectInterfaces.h
index aef7ec622fe4f..9de20f0c69f1a 100644
--- a/mlir/include/mlir/Interfaces/SideEffectInterfaces.h
+++ b/mlir/include/mlir/Interfaces/SideEffectInterfaces.h
@@ -383,33 +383,71 @@ struct Write : public Effect::Base<Write> {};
 // SideEffect Utilities
 //===----------------------------------------------------------------------===//
 
-/// Returns true if `op` has only an effect of type `EffectTy`.
+/// Return "true" if `op` has unknown effects. I.e., the effects of the
+/// operation itself are unknown and the operation does not derive its effects
+/// from its nested operations. (`HasRecursiveMemoryEffects` trait is not
+/// implemented or it is unknown whether it is implemented or not.)
+bool hasUnknownEffects(Operation *op);
+
+/// Returns "true" if `op` has only an effect of type `EffectTy`. Returns
+/// "false" if `op` has unknown effects or other/additional effects. Recursive
+/// effects are not taken into account.
 template <typename EffectTy>
 bool hasSingleEffect(Operation *op);
 
-/// Returns true if `op` has only an effect of type `EffectTy` (and of no other
-/// type) on `value`.
+/// Returns "true" if `op` has only an effect of type `EffectTy` on `value`.
+/// Returns "false" if `op` has unknown effects or other/additional effects.
+/// Recursive effects are not taken into account.
 template <typename EffectTy>
 bool hasSingleEffect(Operation *op, Value value);
 
-/// Returns true if `op` has only an effect of type `EffectTy` (and of no other
-/// type) on `value` of type `ValueTy`.
+/// Returns "true" if `op` has only an effect of type `EffectTy` on `value` of
+/// type `ValueTy`. Returns "false" if `op` has unknown effects or
+/// other/additional effects. Recursive effects are not taken into account.
 template <typename ValueTy, typename EffectTy>
 bool hasSingleEffect(Operation *op, ValueTy value);
 
-/// Returns true if `op` has an effect of type `EffectTy`.
+/// Returns "true" if `op` has an effect of type `EffectTy`. Returns "false" if
+/// `op` has unknown effects. Recursive effects are not taken into account.
 template <typename... EffectTys>
 bool hasEffect(Operation *op);
 
-/// Returns true if `op` has an effect of type `EffectTy` on `value`.
+/// Returns "true" if `op` has an effect of type `EffectTy` on `value`. Returns
+/// "false" if `op` has unknown effects. Recursive effects are not taken into
+/// account.
 template <typename... EffectTys>
 bool hasEffect(Operation *op, Value value);
 
-/// Returns true if `op` has an effect of type `EffectTy` on `value` of type
-/// `ValueTy`.
+/// Returns "true" if `op` has an effect of type `EffectTy` on `value` of type
+/// `ValueTy`. Returns "false" if `op` has unknown effects. Recursive effects
+/// are not taken into account.
 template <typename ValueTy, typename... EffectTys>
 bool hasEffect(Operation *op, ValueTy value);
 
+/// Returns "true" if `op` might have an effect of type `EffectTy`. Returns
+/// "true" if the op has unknown effects. Recursive effects are not taken into
+/// account.
+template <typename... EffectTys>
+bool mightHaveEffect(Operation *op) {
+  return hasUnknownEffects(op) || hasEffect<EffectTys...>(op);
+}
+
+/// Returns "true" if `op` might have an effect of type `EffectTy` on `value`.
+/// Returns "true" if the op has unknown effects. Recursive effects are not
+/// taken into account.
+template <typename... EffectTys>
+bool mightHaveEffect(Operation *op, Value value) {
+  return hasUnknownEffects(op) || hasEffect<EffectTys...>(op, value);
+}
+
+/// Returns "true" if `op` might have an effect of type `EffectTy` on `value`
+/// of type `ValueTy`. Returns "true" if the op has unknown effects. Recursive
+/// effects are not taken into account.
+template <typename ValueTy, typename... EffectTys>
+bool mightHaveEffect(Operation *op, ValueTy value) {
+  return hasUnknownEffects(op) || hasEffect<EffectTys...>(op, value);
+}
+
 /// Return true if the given operation is unused, and has no side effects on
 /// memory that prevent erasing.
 bool isOpTriviallyDead(Operation *op);
diff --git a/mlir/include/mlir/Interfaces/ViewLikeInterface.td b/mlir/include/mlir/Interfaces/ViewLikeInterface.td
index d1401c238381e..ed213bfdae337 100644
--- a/mlir/include/mlir/Interfaces/ViewLikeInterface.td
+++ b/mlir/include/mlir/Interfaces/ViewLikeInterface.td
@@ -26,7 +26,17 @@ def ViewLikeOpInterface : OpInterface<"ViewLikeOpInterface"> {
   let methods = [
     InterfaceMethod<
       "Returns the source buffer from which the view is created.",
-      "::mlir::Value", "getViewSource">
+      "::mlir::Value", "getViewSource">,
+    InterfaceMethod<
+      /*desc=*/[{ Returns the buffer which the view created. }],
+      /*retTy=*/"::mlir::Value",
+      /*methodName=*/"getViewDest",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        return $_op->getResult(0);
+      }]
+    >
   ];
 }
 
diff --git a/mlir/include/mlir/Pass/PassOptions.h b/mlir/include/mlir/Pass/PassOptions.h
index e1f16c6158ad5..0c71f78b52d3d 100644
--- a/mlir/include/mlir/Pass/PassOptions.h
+++ b/mlir/include/mlir/Pass/PassOptions.h
@@ -377,7 +377,7 @@ class PassOptions : protected llvm::cl::SubCommand {
 ///   ListOption<int> someListFlag{*this, "flag-name", llvm::cl::desc("...")};
 /// };
 template <typename T>
-class PassPipelineOptions : public detail::PassOptions {
+class PassPipelineOptions : public virtual detail::PassOptions {
 public:
   /// Factory that parses the provided options and returns a unique_ptr to the
   /// struct.
diff --git a/mlir/include/mlir/Remark/RemarkStreamer.h b/mlir/include/mlir/Remark/RemarkStreamer.h
new file mode 100644
index 0000000000000..8bfd176d9bade
--- /dev/null
+++ b/mlir/include/mlir/Remark/RemarkStreamer.h
@@ -0,0 +1,49 @@
+//===- RemarkStreamer.h - MLIR Optimization Remark ---------------*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines LLVMRemarkStreamer plugging class that uses LLVM's
+// streamer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/Remarks.h"
+
+#include "llvm/Remarks/RemarkStreamer.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+namespace mlir::remark::detail {
+
+/// Concrete streamer that writes LLVM optimization remarks to a file
+/// (YAML or Bitstream). Lives outside core.
+class LLVMRemarkStreamer final : public MLIRRemarkStreamerBase {
+public:
+  static FailureOr<std::unique_ptr<MLIRRemarkStreamerBase>>
+  createToFile(llvm::StringRef path, llvm::remarks::Format fmt);
+
+  void streamOptimizationRemark(const Remark &remark) override;
+  void finalize() override {}
+  ~LLVMRemarkStreamer() override;
+
+private:
+  LLVMRemarkStreamer() = default;
+
+  std::unique_ptr<class llvm::remarks::RemarkStreamer> remarkStreamer;
+  std::unique_ptr<class llvm::ToolOutputFile> file;
+};
+} // namespace mlir::remark::detail
+
+namespace mlir::remark {
+/// Enable optimization remarks to a file with the given path and format.
+/// The remark categories are used to filter the remarks that are emitted.
+/// If the printAsEmitRemarks flag is set, remarks will also be printed using
+/// mlir::emitRemarks.
+LogicalResult enableOptimizationRemarksWithLLVMStreamer(
+    MLIRContext &ctx, StringRef filePath, llvm::remarks::Format fmt,
+    const RemarkCategories &cat, bool printAsEmitRemarks = false);
+
+} // namespace mlir::remark
diff --git a/mlir/include/mlir/Target/CMakeLists.txt b/mlir/include/mlir/Target/CMakeLists.txt
new file mode 100644
index 0000000000000..39d31dc9b5e9c
--- /dev/null
+++ b/mlir/include/mlir/Target/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(LLVMIR)
diff --git a/mlir/include/mlir/Target/LLVM/XeVM/Target.h b/mlir/include/mlir/Target/LLVM/XeVM/Target.h
new file mode 100644
index 0000000000000..6aab15c9ba07e
--- /dev/null
+++ b/mlir/include/mlir/Target/LLVM/XeVM/Target.h
@@ -0,0 +1,30 @@
+//===-- Target.h - MLIR XeVM target registration ----------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This provides registration calls for attaching the XeVM target interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TARGET_LLVM_XEVM_TARGET_H
+#define MLIR_TARGET_LLVM_XEVM_TARGET_H
+
+namespace mlir {
+class DialectRegistry;
+class MLIRContext;
+namespace xevm {
+/// Registers the `TargetAttrInterface` for the `#xevm.target` attribute in
+/// the given registry.
+void registerXeVMTargetInterfaceExternalModels(mlir::DialectRegistry &registry);
+
+/// Registers the `TargetAttrInterface` for the `#xevm.target` attribute in
+/// the registry associated with the given context.
+void registerXeVMTargetInterfaceExternalModels(mlir::MLIRContext &context);
+} // namespace xevm
+} // namespace mlir
+
+#endif // MLIR_TARGET_LLVM_XEVM_TARGET_H
diff --git a/mlir/include/mlir/Target/LLVM/XeVM/Utils.h b/mlir/include/mlir/Target/LLVM/XeVM/Utils.h
new file mode 100644
index 0000000000000..5d523f1ec457f
--- /dev/null
+++ b/mlir/include/mlir/Target/LLVM/XeVM/Utils.h
@@ -0,0 +1,63 @@
+//===-- Utils.h - MLIR XeVM target utils ------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This files declares XeVM target related utility classes and functions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TARGET_LLVM_XEVM_UTILS_H
+#define MLIR_TARGET_LLVM_XEVM_UTILS_H
+
+#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/Target/LLVM/ModuleToObject.h"
+
+namespace mlir {
+namespace xevm {
+
+/// Base class for all XeVM serializations from GPU modules into binary strings.
+/// By default this class serializes into LLVM bitcode.
+class SerializeGPUModuleBase : public LLVM::ModuleToObject {
+public:
+  SerializeGPUModuleBase(Operation &module, XeVMTargetAttr target,
+                         const gpu::TargetOptions &targetOptions = {});
+
+  /// Returns the target attribute.
+  XeVMTargetAttr getTarget() const;
+
+  /// Loads the bitcode files in `librariesToLink`.
+  std::optional<SmallVector<std::unique_ptr<llvm::Module>>>
+  loadBitcodeFiles(llvm::Module &module) override;
+
+  /// Returns the gpu module being serialized.
+  gpu::GPUModuleOp getGPUModuleOp();
+
+  /// Compiles to native code using `ocloc`.
+  std::optional<SmallVector<char, 0>> compileToBinary(const std::string &asmStr,
+                                                      StringRef inputFormat);
+
+protected:
+  /// XeVM Target attribute.
+  XeVMTargetAttr xeTarget;
+  /// List of LLVM bitcode to link into after translation to LLVM IR.
+  /// The attributes can be StringAttr pointing to a file path, or
+  /// a Resource blob pointing to the LLVM bitcode in-memory.
+  SmallVector<Attribute> librariesToLink;
+
+  /// Returns the path to the tool used for serialization.
+  std::optional<std::string> findTool(StringRef tool);
+
+  /// GPU compilation target options.
+  gpu::TargetOptions targetOptions;
+};
+} // namespace xevm
+} // namespace mlir
+
+#endif // MLIR_TARGET_LLVM_XEVM_UTILS_H
diff --git a/mlir/include/mlir/Target/LLVMIR/CMakeLists.txt b/mlir/include/mlir/Target/LLVMIR/CMakeLists.txt
new file mode 100644
index 0000000000000..e31af32661164
--- /dev/null
+++ b/mlir/include/mlir/Target/LLVMIR/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(Transforms)
diff --git a/mlir/lib/Target/LLVMIR/DataLayoutImporter.h b/mlir/include/mlir/Target/LLVMIR/DataLayoutImporter.h
similarity index 71%
rename from mlir/lib/Target/LLVMIR/DataLayoutImporter.h
rename to mlir/include/mlir/Target/LLVMIR/DataLayoutImporter.h
index 88ceaf1a74e62..4d432df493c3f 100644
--- a/mlir/lib/Target/LLVMIR/DataLayoutImporter.h
+++ b/mlir/include/mlir/Target/LLVMIR/DataLayoutImporter.h
@@ -11,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_LIB_TARGET_LLVMIR_DATALAYOUTIMPORTER_H_
-#define MLIR_LIB_TARGET_LLVMIR_DATALAYOUTIMPORTER_H_
+#ifndef MLIR_TARGET_LLVMIR_DATALAYOUTIMPORTER_H
+#define MLIR_TARGET_LLVMIR_DATALAYOUTIMPORTER_H
 
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/IR/BuiltinAttributes.h"
@@ -38,23 +38,31 @@ namespace detail {
 /// null if the bit width is not supported.
 FloatType getFloatType(MLIRContext *context, unsigned width);
 
-/// Helper class that translates an LLVM data layout to an MLIR data layout
-/// specification. Only integer, float, pointer, alloca memory space, stack
-/// alignment, and endianness entries are translated. The class also returns all
-/// entries from the default data layout specification found in the language
-/// reference (https://llvm.org/docs/LangRef.html#data-layout) if they are not
-/// overwritten by the provided data layout.
+/// Helper class that translates an LLVM data layout string to an MLIR data
+/// layout specification. Only integer, float, pointer, alloca memory space,
+/// stack alignment, and endianness entries are translated. The class also
+/// returns all entries from the default data layout specification found in the
+/// language reference (https://llvm.org/docs/LangRef.html#data-layout) if they
+/// are not overwritten by the provided data layout.
 class DataLayoutImporter {
 public:
-  DataLayoutImporter(MLIRContext *context,
-                     const llvm::DataLayout &llvmDataLayout)
-      : context(context) {
-    translateDataLayout(llvmDataLayout);
+  DataLayoutImporter(MLIRContext *context, StringRef dataLayoutStr)
+      : dataLayoutStr(dataLayoutStr), context(context) {
+    // Translate the `dataLayoutStr`. First, append the default data layout
+    // string specified in the language reference
+    // (https://llvm.org/docs/LangRef.html#data-layout) to the supplied string.
+    // The translation then parses the string and ignores the default value if a
+    // specific kind occurs in both strings. Additionally, the following default
+    // values exist:
+    // - non-default address space pointer specifications default to the default
+    //   address space pointer specification
+    // - the alloca address space defaults to the default address space.
+    dataLayoutSpec = dataLayoutSpecFromDataLayoutStr();
   }
 
   /// Returns the MLIR data layout specification translated from the LLVM
   /// data layout.
-  DataLayoutSpecInterface getDataLayout() const { return dataLayout; }
+  DataLayoutSpecInterface getDataLayoutSpec() const { return dataLayoutSpec; }
 
   /// Returns the last data layout token that has been processed before
   /// the data layout translation failed.
@@ -65,8 +73,9 @@ class DataLayoutImporter {
   ArrayRef<StringRef> getUnhandledTokens() const { return unhandledTokens; }
 
 private:
-  /// Translates the LLVM `dataLayout` to an MLIR data layout specification.
-  void translateDataLayout(const llvm::DataLayout &llvmDataLayout);
+  /// Translate the LLVM data layout string to an MLIR data layout
+  /// specification.
+  DataLayoutSpecInterface dataLayoutSpecFromDataLayoutStr();
 
   /// Tries to parse the letter only prefix that identifies the specification
   /// and removes the consumed characters from the beginning of the string.
@@ -116,17 +125,18 @@ class DataLayoutImporter {
   /// Adds legal int widths entry if there is none yet.
   LogicalResult tryToEmplaceLegalIntWidthsEntry(StringRef token);
 
-  std::string layoutStr = {};
+  std::string dataLayoutStr = {};
+  DataLayoutSpecInterface dataLayoutSpec;
+
   StringRef lastToken = {};
   SmallVector<StringRef> unhandledTokens;
   llvm::MapVector<StringAttr, DataLayoutEntryInterface> keyEntries;
   llvm::MapVector<TypeAttr, DataLayoutEntryInterface> typeEntries;
   MLIRContext *context;
-  DataLayoutSpecInterface dataLayout;
 };
 
 } // namespace detail
 } // namespace LLVM
 } // namespace mlir
 
-#endif // MLIR_LIB_TARGET_LLVMIR_DATALAYOUTIMPORTER_H_
+#endif // MLIR_TARGET_LLVMIR_DATALAYOUTIMPORTER_H
diff --git a/mlir/include/mlir/Target/LLVMIR/Transforms/CMakeLists.txt b/mlir/include/mlir/Target/LLVMIR/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..b1a3c79b9b837
--- /dev/null
+++ b/mlir/include/mlir/Target/LLVMIR/Transforms/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(LLVM_TARGET_DEFINITIONS Passes.td)
+mlir_tablegen(Passes.h.inc -gen-pass-decls -name TargetLLVMIRTransforms)
+add_public_tablegen_target(MLIRTargetLLVMIRTransformsIncGen)
+
+add_mlir_doc(Passes TargetLLVMIRTransforms ./ -gen-pass-doc)
diff --git a/mlir/include/mlir/Target/LLVMIR/Transforms/Passes.h b/mlir/include/mlir/Target/LLVMIR/Transforms/Passes.h
new file mode 100644
index 0000000000000..1e6419154108c
--- /dev/null
+++ b/mlir/include/mlir/Target/LLVMIR/Transforms/Passes.h
@@ -0,0 +1,27 @@
+//===- Passes.h - LLVM Target Pass Construction and Registration ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TARGET_LLVMIR_TRANSFORMS_PASSES_H
+#define MLIR_TARGET_LLVMIR_TRANSFORMS_PASSES_H
+
+#include "mlir/Pass/Pass.h"
+
+namespace mlir {
+
+namespace LLVM {
+
+#define GEN_PASS_DECL
+#define GEN_PASS_REGISTRATION
+#include "mlir/Target/LLVMIR/Transforms/Passes.h.inc"
+
+void registerTargetLLVMPasses();
+
+} // namespace LLVM
+} // namespace mlir
+
+#endif // MLIR_TARGET_LLVMIR_TRANSFORMS_PASSES_H
diff --git a/mlir/include/mlir/Target/LLVMIR/Transforms/Passes.td b/mlir/include/mlir/Target/LLVMIR/Transforms/Passes.td
new file mode 100644
index 0000000000000..906f6e82efa50
--- /dev/null
+++ b/mlir/include/mlir/Target/LLVMIR/Transforms/Passes.td
@@ -0,0 +1,30 @@
+//===-- Passes.td - LLVM Target pass definition file -------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TARGET_LLVMIR_TRANSFORMS_PASSES
+#define MLIR_TARGET_LLVMIR_TRANSFORMS_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+def LLVMTargetToDataLayout : Pass<"llvm-target-to-data-layout"> {
+  let summary = "Derive data layout attributes from LLVM target attributes";
+  let dependentDialects = ["mlir::DLTIDialect"];
+  let description = [{
+    Derive a `DataLayoutSpecInterface`-implementing data layout attribute from
+    the LLVM-backend target specified by the `TargetAttrInterface`-implementing
+    attribute attached to the target op at the name `llvm.target`.
+  }];
+  let options = [
+    Option<"initializeLLVMTargets", "initialize-llvm-targets", "bool",
+           /*default=*/"true",
+           "Whether to pre-load all available target machines, that LLVM is "
+           "configured to support, into the TargetRegistry.">
+  ];
+}
+
+#endif // MLIR_TARGET_LLVMIR_TRANSFORMS_PASSES
diff --git a/mlir/include/mlir/Target/SPIRV/Serialization.h b/mlir/include/mlir/Target/SPIRV/Serialization.h
index 225777e25d607..e474101402e58 100644
--- a/mlir/include/mlir/Target/SPIRV/Serialization.h
+++ b/mlir/include/mlir/Target/SPIRV/Serialization.h
@@ -15,6 +15,7 @@
 
 #include "mlir/Support/LLVM.h"
 #include <cstdint>
+#include <string>
 
 namespace mlir {
 class MLIRContext;
@@ -27,12 +28,33 @@ struct SerializationOptions {
   bool emitSymbolName = true;
   /// Whether to emit `OpLine` location information for SPIR-V ops.
   bool emitDebugInfo = false;
+  /// Whether to store a module to an additional file during
+  /// serialization. This is used to store the SPIR-V module to the
+  /// file in addition to writing it to `os` passed from the calling
+  /// tool. This saved file is later used for validation.
+  bool saveModuleForValidation = false;
+  /// A prefix prepended to the file used when `saveModuleForValidation`
+  /// is set to `true`. This can either be a file prefix, or a relative or
+  /// or an absolute path followed by the prefix. For example:
+  ///
+  ///   * "foo" - Create files with a `foo` prefix in the current working
+  ///     directory. For example: `fooXYZ123.spv`, `fooABC456.spv` ...
+  ///     `fooXXXXXX.spv`. The last 6 characters will be a unique combination
+  ///     as generated by `llvm::sys::fs::createUniqueFile`.
+  ///
+  ///   * "my/dir/foo" - Create files in `my/dir` with a `foo` prefix. The
+  ///     `my/dir` need to exists. For example: `fooXYZ123.spv`,
+  ///     `fooABC456.spv` ... `fooXXXXXX.spv` will be created and stored in
+  ///     `/my/dir`. Filenames follow the same pattern as above.
+  ///
+  ///   * "/home/user/my/dir" - Same as above but using an absolute path.
+  std::string validationFilePrefix = "";
 };
 
-/// Serializes the given SPIR-V `module` and writes to `binary`. On failure,
+/// Serializes the given SPIR-V `moduleOp` and writes to `binary`. On failure,
 /// reports errors to the error handler registered with the MLIR context for
-/// `module`.
-LogicalResult serialize(ModuleOp module, SmallVectorImpl<uint32_t> &binary,
+/// `moduleOp`.
+LogicalResult serialize(ModuleOp moduleOp, SmallVectorImpl<uint32_t> &binary,
                         const SerializationOptions &options = {});
 
 } // namespace spirv
diff --git a/mlir/include/mlir/Target/Wasm/WasmBinaryEncoding.h b/mlir/include/mlir/Target/Wasm/WasmBinaryEncoding.h
new file mode 100644
index 0000000000000..21adde878994e
--- /dev/null
+++ b/mlir/include/mlir/Target/Wasm/WasmBinaryEncoding.h
@@ -0,0 +1,143 @@
+//===- WasmBinaryEncoding.h - Byte encodings for Wasm binary format ===----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// Define various flags used to encode instructions, types, etc. in
+// WebAssembly binary format.
+//
+// These encodings are defined in the WebAssembly binary format specification.
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_TARGET_WASMBINARYENCODING
+#define MLIR_TARGET_WASMBINARYENCODING
+
+#include <cstddef>
+
+namespace mlir {
+struct WasmBinaryEncoding {
+  /// Byte encodings for Wasm instructions.
+  struct OpCode {
+    // Locals, globals, constants.
+    static constexpr std::byte localGet{0x20};
+    static constexpr std::byte localSet{0x21};
+    static constexpr std::byte localTee{0x22};
+    static constexpr std::byte globalGet{0x23};
+    static constexpr std::byte constI32{0x41};
+    static constexpr std::byte constI64{0x42};
+    static constexpr std::byte constFP32{0x43};
+    static constexpr std::byte constFP64{0x44};
+
+    // Numeric operations.
+    static constexpr std::byte clzI32{0x67};
+    static constexpr std::byte ctzI32{0x68};
+    static constexpr std::byte popcntI32{0x69};
+    static constexpr std::byte addI32{0x6A};
+    static constexpr std::byte subI32{0x6B};
+    static constexpr std::byte mulI32{0x6C};
+    static constexpr std::byte divSI32{0x6d};
+    static constexpr std::byte divUI32{0x6e};
+    static constexpr std::byte remSI32{0x6f};
+    static constexpr std::byte remUI32{0x70};
+    static constexpr std::byte andI32{0x71};
+    static constexpr std::byte orI32{0x72};
+    static constexpr std::byte xorI32{0x73};
+    static constexpr std::byte shlI32{0x74};
+    static constexpr std::byte shrSI32{0x75};
+    static constexpr std::byte shrUI32{0x76};
+    static constexpr std::byte rotlI32{0x77};
+    static constexpr std::byte rotrI32{0x78};
+    static constexpr std::byte clzI64{0x79};
+    static constexpr std::byte ctzI64{0x7A};
+    static constexpr std::byte popcntI64{0x7B};
+    static constexpr std::byte addI64{0x7C};
+    static constexpr std::byte subI64{0x7D};
+    static constexpr std::byte mulI64{0x7E};
+    static constexpr std::byte divSI64{0x7F};
+    static constexpr std::byte divUI64{0x80};
+    static constexpr std::byte remSI64{0x81};
+    static constexpr std::byte remUI64{0x82};
+    static constexpr std::byte andI64{0x83};
+    static constexpr std::byte orI64{0x84};
+    static constexpr std::byte xorI64{0x85};
+    static constexpr std::byte shlI64{0x86};
+    static constexpr std::byte shrSI64{0x87};
+    static constexpr std::byte shrUI64{0x88};
+    static constexpr std::byte rotlI64{0x89};
+    static constexpr std::byte rotrI64{0x8A};
+    static constexpr std::byte absF32{0x8B};
+    static constexpr std::byte negF32{0x8C};
+    static constexpr std::byte ceilF32{0x8D};
+    static constexpr std::byte floorF32{0x8E};
+    static constexpr std::byte truncF32{0x8F};
+    static constexpr std::byte sqrtF32{0x91};
+    static constexpr std::byte addF32{0x92};
+    static constexpr std::byte subF32{0x93};
+    static constexpr std::byte mulF32{0x94};
+    static constexpr std::byte divF32{0x95};
+    static constexpr std::byte minF32{0x96};
+    static constexpr std::byte maxF32{0x97};
+    static constexpr std::byte copysignF32{0x98};
+    static constexpr std::byte absF64{0x99};
+    static constexpr std::byte negF64{0x9A};
+    static constexpr std::byte ceilF64{0x9B};
+    static constexpr std::byte floorF64{0x9C};
+    static constexpr std::byte truncF64{0x9D};
+    static constexpr std::byte sqrtF64{0x9F};
+    static constexpr std::byte addF64{0xA0};
+    static constexpr std::byte subF64{0xA1};
+    static constexpr std::byte mulF64{0xA2};
+    static constexpr std::byte divF64{0xA3};
+    static constexpr std::byte minF64{0xA4};
+    static constexpr std::byte maxF64{0xA5};
+    static constexpr std::byte copysignF64{0xA6};
+    static constexpr std::byte wrap{0xA7};
+  };
+
+  /// Byte encodings of types in Wasm binaries
+  struct Type {
+    static constexpr std::byte emptyBlockType{0x40};
+    static constexpr std::byte funcType{0x60};
+    static constexpr std::byte externRef{0x6F};
+    static constexpr std::byte funcRef{0x70};
+    static constexpr std::byte v128{0x7B};
+    static constexpr std::byte f64{0x7C};
+    static constexpr std::byte f32{0x7D};
+    static constexpr std::byte i64{0x7E};
+    static constexpr std::byte i32{0x7F};
+  };
+
+  /// Byte encodings of Wasm imports.
+  struct Import {
+    static constexpr std::byte typeID{0x00};
+    static constexpr std::byte tableType{0x01};
+    static constexpr std::byte memType{0x02};
+    static constexpr std::byte globalType{0x03};
+  };
+
+  /// Byte encodings for Wasm limits.
+  struct LimitHeader {
+    static constexpr std::byte lowLimitOnly{0x00};
+    static constexpr std::byte bothLimits{0x01};
+  };
+
+  /// Byte encodings describing the mutability of globals.
+  struct GlobalMutability {
+    static constexpr std::byte isConst{0x00};
+    static constexpr std::byte isMutable{0x01};
+  };
+
+  /// Byte encodings describing Wasm exports.
+  struct Export {
+    static constexpr std::byte function{0x00};
+    static constexpr std::byte table{0x01};
+    static constexpr std::byte memory{0x02};
+    static constexpr std::byte global{0x03};
+  };
+
+  static constexpr std::byte endByte{0x0B};
+};
+} // namespace mlir
+
+#endif
diff --git a/mlir/include/mlir/Target/Wasm/WasmImporter.h b/mlir/include/mlir/Target/Wasm/WasmImporter.h
new file mode 100644
index 0000000000000..3f05bbecefc8a
--- /dev/null
+++ b/mlir/include/mlir/Target/Wasm/WasmImporter.h
@@ -0,0 +1,31 @@
+//===- WasmImporter.h - Helpers to create WebAssembly emitter ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines helpers to import WebAssembly code using the WebAssembly
+// dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_TARGET_WASM_WASMIMPORTER_H
+#define MLIR_TARGET_WASM_WASMIMPORTER_H
+
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/OwningOpRef.h"
+#include "llvm/Support/SourceMgr.h"
+
+namespace mlir::wasm {
+
+/// If `source` contains a valid Wasm binary file, this function returns a
+/// a ModuleOp containing the representation of the Wasm module encoded in
+/// the source file in the `wasmssa` dialect.
+OwningOpRef<ModuleOp> importWebAssemblyToModule(llvm::SourceMgr &source,
+                                                MLIRContext *context);
+} // namespace mlir::wasm
+
+#endif // MLIR_TARGET_WASM_WASMIMPORTER_H
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index f6437657c9a93..f23a70601fc0a 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -521,8 +521,8 @@ class ConversionPattern : public RewritePattern {
 
   /// Hook for derived classes to implement combined matching and rewriting.
   /// This overload supports only 1:1 replacements. The 1:N overload is called
-  /// by the driver. By default, it calls this 1:1 overload or reports a fatal
-  /// error if 1:N replacements were found.
+  /// by the driver. By default, it calls this 1:1 overload or fails to match
+  /// if 1:N replacements were found.
   virtual LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<Value> operands,
                   ConversionPatternRewriter &rewriter) const {
@@ -534,7 +534,7 @@ class ConversionPattern : public RewritePattern {
   virtual LogicalResult
   matchAndRewrite(Operation *op, ArrayRef<ValueRange> operands,
                   ConversionPatternRewriter &rewriter) const {
-    return matchAndRewrite(op, getOneToOneAdaptorOperands(operands), rewriter);
+    return dispatchTo1To1(*this, op, operands, rewriter);
   }
 
   /// Attempt to match and rewrite the IR root at the specified operation.
@@ -567,11 +567,26 @@ class ConversionPattern : public RewritePattern {
   /// try to extract the single value of each range to construct a the inputs
   /// for a 1:1 adaptor.
   ///
-  /// This function produces a fatal error if at least one range has 0 or
-  /// more than 1 value: "pattern 'name' does not support 1:N conversion"
-  SmallVector<Value>
+  /// Returns failure if at least one range has 0 or more than 1 value.
+  FailureOr<SmallVector<Value>>
   getOneToOneAdaptorOperands(ArrayRef<ValueRange> operands) const;
 
+  /// Overloaded method used to dispatch to the 1:1 'matchAndRewrite' method
+  /// if possible and emit diagnostic with a failure return value otherwise.
+  /// 'self' should be '*this' of the derived-pattern and is used to dispatch
+  /// to the correct 'matchAndRewrite' method in the derived pattern.
+  template <typename SelfPattern, typename SourceOp>
+  static LogicalResult dispatchTo1To1(const SelfPattern &self, SourceOp op,
+                                      ArrayRef<ValueRange> operands,
+                                      ConversionPatternRewriter &rewriter);
+
+  /// Same as above, but accepts an adaptor as operand.
+  template <typename SelfPattern, typename SourceOp>
+  static LogicalResult dispatchTo1To1(
+      const SelfPattern &self, SourceOp op,
+      typename SourceOp::template GenericAdaptor<ArrayRef<ValueRange>> adaptor,
+      ConversionPatternRewriter &rewriter);
+
 protected:
   /// An optional type converter for use by this pattern.
   const TypeConverter *typeConverter = nullptr;
@@ -620,9 +635,7 @@ class OpConversionPattern : public ConversionPattern {
   virtual LogicalResult
   matchAndRewrite(SourceOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const {
-    SmallVector<Value> oneToOneOperands =
-        getOneToOneAdaptorOperands(adaptor.getOperands());
-    return matchAndRewrite(op, OpAdaptor(oneToOneOperands, adaptor), rewriter);
+    return dispatchTo1To1(*this, op, adaptor, rewriter);
   }
 
 private:
@@ -666,7 +679,7 @@ class OpInterfaceConversionPattern : public ConversionPattern {
   virtual LogicalResult
   matchAndRewrite(SourceOp op, ArrayRef<ValueRange> operands,
                   ConversionPatternRewriter &rewriter) const {
-    return matchAndRewrite(op, getOneToOneAdaptorOperands(operands), rewriter);
+    return dispatchTo1To1(*this, op, operands, rewriter);
   }
 
 private:
@@ -728,6 +741,9 @@ class ConversionPatternRewriter final : public PatternRewriter {
 public:
   ~ConversionPatternRewriter() override;
 
+  /// Return the configuration of the current dialect conversion.
+  const ConversionConfig &getConfig() const;
+
   /// Apply a signature conversion to given block. This replaces the block with
   /// a new block containing the updated signature. The operations of the given
   /// block are inlined into the newly-created block, which is returned.
@@ -766,6 +782,12 @@ class ConversionPatternRewriter final : public PatternRewriter {
 
   /// Replace all the uses of the block argument `from` with `to`. This
   /// function supports both 1:1 and 1:N replacements.
+  ///
+  /// Note: If `allowPatternRollback` is set to "true", this function replaces
+  /// all current and future uses of the block argument. This same block
+  /// block argument must not be replaced multiple times. Uses are not replaced
+  /// immediately but in a delayed fashion. Patterns may still see the original
+  /// uses when inspecting IR.
   void replaceUsesOfBlockArgument(BlockArgument from, ValueRange to);
 
   /// Return the converted value of 'key' with a type defined by the type
@@ -862,6 +884,35 @@ class ConversionPatternRewriter final : public PatternRewriter {
   std::unique_ptr<detail::ConversionPatternRewriterImpl> impl;
 };
 
+template <typename SelfPattern, typename SourceOp>
+LogicalResult
+ConversionPattern::dispatchTo1To1(const SelfPattern &self, SourceOp op,
+                                  ArrayRef<ValueRange> operands,
+                                  ConversionPatternRewriter &rewriter) {
+  FailureOr<SmallVector<Value>> oneToOneOperands =
+      self.getOneToOneAdaptorOperands(operands);
+  if (failed(oneToOneOperands))
+    return rewriter.notifyMatchFailure(op,
+                                       "pattern '" + self.getDebugName() +
+                                           "' does not support 1:N conversion");
+  return self.matchAndRewrite(op, *oneToOneOperands, rewriter);
+}
+
+template <typename SelfPattern, typename SourceOp>
+LogicalResult ConversionPattern::dispatchTo1To1(
+    const SelfPattern &self, SourceOp op,
+    typename SourceOp::template GenericAdaptor<ArrayRef<ValueRange>> adaptor,
+    ConversionPatternRewriter &rewriter) {
+  FailureOr<SmallVector<Value>> oneToOneOperands =
+      self.getOneToOneAdaptorOperands(adaptor.getOperands());
+  if (failed(oneToOneOperands))
+    return rewriter.notifyMatchFailure(op,
+                                       "pattern '" + self.getDebugName() +
+                                           "' does not support 1:N conversion");
+  return self.matchAndRewrite(
+      op, typename SourceOp::Adaptor(*oneToOneOperands, adaptor), rewriter);
+}
+
 //===----------------------------------------------------------------------===//
 // ConversionTarget
 //===----------------------------------------------------------------------===//
@@ -1158,6 +1209,16 @@ class PDLConversionConfig final {
 // ConversionConfig
 //===----------------------------------------------------------------------===//
 
+/// An enum to control folding behavior during dialect conversion.
+enum class DialectConversionFoldingMode {
+  /// Never attempt to fold.
+  Never,
+  /// Only attempt to fold not legal operations before applying patterns.
+  BeforePatterns,
+  /// Only attempt to fold not legal operations after applying patterns.
+  AfterPatterns,
+};
+
 /// Dialect conversion configuration.
 struct ConversionConfig {
   /// An optional callback used to notify about match failure diagnostics during
@@ -1228,18 +1289,29 @@ struct ConversionConfig {
   /// 2. Pattern produces IR (in-place modification or new IR) that is illegal
   ///    and cannot be legalized by subsequent foldings / pattern applications.
   ///
-  /// If set to "false", the conversion driver will produce an LLVM fatal error
-  /// instead of rolling back IR modifications. Moreover, in case of a failed
-  /// conversion, the original IR is not restored. The resulting IR may be a
-  /// mix of original and rewritten IR. (Same as a failed greedy pattern
-  /// rewrite.)
+  /// Experimental: If set to "false", the conversion driver will produce an
+  /// LLVM fatal error instead of rolling back IR modifications. Moreover, in
+  /// case of a failed conversion, the original IR is not restored. The
+  /// resulting IR may be a mix of original and rewritten IR. (Same as a failed
+  /// greedy pattern rewrite.) Use the cmake build option
+  /// `-DMLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS=ON` (ideally together with
+  /// ASAN) to detect invalid pattern API usage.
   ///
-  /// Note: This flag was added in preparation of the One-Shot Dialect
-  /// Conversion refactoring, which will remove the ability to roll back IR
-  /// modifications from the conversion driver. Use this flag to ensure that
-  /// your patterns do not trigger any IR rollbacks. For details, see
+  /// When pattern rollback is disabled, the conversion driver has to maintain
+  /// less internal state. This is more efficient, but not supported by all
+  /// lowering patterns. For details, see
   /// https://discourse.llvm.org/t/rfc-a-new-one-shot-dialect-conversion-driver/79083.
   bool allowPatternRollback = true;
+
+  /// The folding mode to use during conversion.
+  DialectConversionFoldingMode foldingMode =
+      DialectConversionFoldingMode::BeforePatterns;
+
+  /// If set to "true", the materialization kind ("source" or "target") will be
+  /// attached to "builtin.unrealized_conversion_cast" ops. This flag is useful
+  /// for debugging, to find out what kind of materialization rule may be
+  /// missing.
+  bool attachDebugMaterializationKind = false;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Transforms/WalkPatternRewriteDriver.h b/mlir/include/mlir/Transforms/WalkPatternRewriteDriver.h
index 6d62ae3dd43dc..7d5c1d5cebb26 100644
--- a/mlir/include/mlir/Transforms/WalkPatternRewriteDriver.h
+++ b/mlir/include/mlir/Transforms/WalkPatternRewriteDriver.h
@@ -27,6 +27,8 @@ namespace mlir {
 /// This is intended as the simplest and most lightweight pattern rewriter in
 /// cases when a simple walk gets the job done.
 ///
+/// The driver will skip unreachable blocks.
+///
 /// Note: Does not apply patterns to the given operation itself.
 void walkAndApplyPatterns(Operation *op,
                           const FrozenRewritePatternSet &patterns,
diff --git a/mlir/lib/Analysis/AliasAnalysis/LocalAliasAnalysis.cpp b/mlir/lib/Analysis/AliasAnalysis/LocalAliasAnalysis.cpp
index 6cece4630a0e5..8062b474539fd 100644
--- a/mlir/lib/Analysis/AliasAnalysis/LocalAliasAnalysis.cpp
+++ b/mlir/lib/Analysis/AliasAnalysis/LocalAliasAnalysis.cpp
@@ -127,9 +127,12 @@ static void collectUnderlyingAddressValues(OpResult result, unsigned maxDepth,
   Operation *op = result.getOwner();
 
   // If this is a view, unwrap to the source.
-  if (ViewLikeOpInterface view = dyn_cast<ViewLikeOpInterface>(op))
-    return collectUnderlyingAddressValues(view.getViewSource(), maxDepth,
-                                          visited, output);
+  if (ViewLikeOpInterface view = dyn_cast<ViewLikeOpInterface>(op)) {
+    if (result == view.getViewDest()) {
+      return collectUnderlyingAddressValues(view.getViewSource(), maxDepth,
+                                            visited, output);
+    }
+  }
   // Check to see if we can reason about the control flow of this op.
   if (auto branch = dyn_cast<RegionBranchOpInterface>(op)) {
     return collectUnderlyingAddressValues(branch, /*region=*/nullptr, result,
diff --git a/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp b/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp
index 10874fd0feb5a..9424eff3e6b6f 100644
--- a/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/DeadCodeAnalysis.cpp
@@ -15,6 +15,7 @@
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Location.h"
 #include "mlir/IR/Operation.h"
+#include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/SymbolTable.h"
 #include "mlir/IR/Value.h"
 #include "mlir/IR/ValueRange.h"
@@ -78,9 +79,17 @@ void Executable::onUpdate(DataFlowSolver *solver) const {
 void PredecessorState::print(raw_ostream &os) const {
   if (allPredecessorsKnown())
     os << "(all) ";
-  os << "predecessors:\n";
-  for (Operation *op : getKnownPredecessors())
-    os << "  " << *op << "\n";
+  os << "predecessors:";
+  if (getKnownPredecessors().empty())
+    os << " (none)";
+  else
+    os << "\n";
+  llvm::interleave(
+      getKnownPredecessors(), os,
+      [&](Operation *op) {
+        os << "  " << OpWithFlags(op, OpPrintingFlags().skipRegions());
+      },
+      "\n");
 }
 
 ChangeResult PredecessorState::join(Operation *predecessor) {
@@ -127,7 +136,7 @@ DeadCodeAnalysis::DeadCodeAnalysis(DataFlowSolver &solver)
 
 LogicalResult DeadCodeAnalysis::initialize(Operation *top) {
   LDBG() << "Initializing DeadCodeAnalysis for top-level op: "
-         << top->getName();
+         << OpWithFlags(top, OpPrintingFlags().skipRegions());
   // Mark the top-level blocks as executable.
   for (Region &region : top->getRegions()) {
     if (region.empty())
@@ -135,7 +144,8 @@ LogicalResult DeadCodeAnalysis::initialize(Operation *top) {
     auto *state =
         getOrCreate<Executable>(getProgramPointBefore(&region.front()));
     propagateIfChanged(state, state->setToLive());
-    LDBG() << "Marked entry block live for region in op: " << top->getName();
+    LDBG() << "Marked entry block live for region in op: "
+           << OpWithFlags(top, OpPrintingFlags().skipRegions());
   }
 
   // Mark as overdefined the predecessors of symbol callables with potentially
@@ -147,17 +157,19 @@ LogicalResult DeadCodeAnalysis::initialize(Operation *top) {
 
 void DeadCodeAnalysis::initializeSymbolCallables(Operation *top) {
   LDBG() << "[init] Entering initializeSymbolCallables for top-level op: "
-         << top->getName();
+         << OpWithFlags(top, OpPrintingFlags().skipRegions());
   analysisScope = top;
   auto walkFn = [&](Operation *symTable, bool allUsesVisible) {
-    LDBG() << "[init] Processing symbol table op: " << symTable->getName();
+    LDBG() << "[init] Processing symbol table op: "
+           << OpWithFlags(symTable, OpPrintingFlags().skipRegions());
     Region &symbolTableRegion = symTable->getRegion(0);
     Block *symbolTableBlock = &symbolTableRegion.front();
 
     bool foundSymbolCallable = false;
     for (auto callable : symbolTableBlock->getOps<CallableOpInterface>()) {
       LDBG() << "[init] Found CallableOpInterface: "
-             << callable.getOperation()->getName();
+             << OpWithFlags(callable.getOperation(),
+                            OpPrintingFlags().skipRegions());
       Region *callableRegion = callable.getCallableRegion();
       if (!callableRegion)
         continue;
@@ -172,7 +184,8 @@ void DeadCodeAnalysis::initializeSymbolCallables(Operation *top) {
             getOrCreate<PredecessorState>(getProgramPointAfter(callable));
         propagateIfChanged(state, state->setHasUnknownPredecessors());
         LDBG() << "[init] Marked callable as having unknown predecessors: "
-               << callable.getOperation()->getName();
+               << OpWithFlags(callable.getOperation(),
+                              OpPrintingFlags().skipRegions());
       }
       foundSymbolCallable = true;
     }
@@ -195,7 +208,8 @@ void DeadCodeAnalysis::initializeSymbolCallables(Operation *top) {
         propagateIfChanged(state, state->setHasUnknownPredecessors());
         LDBG() << "[init] Marked nested callable as "
                   "having unknown predecessors: "
-               << callable.getOperation()->getName();
+               << OpWithFlags(callable.getOperation(),
+                              OpPrintingFlags().skipRegions());
       });
     }
 
@@ -211,13 +225,13 @@ void DeadCodeAnalysis::initializeSymbolCallables(Operation *top) {
       propagateIfChanged(state, state->setHasUnknownPredecessors());
       LDBG() << "[init] Found non-call use for symbol, "
                 "marked as having unknown predecessors: "
-             << symbol->getName();
+             << OpWithFlags(symbol, OpPrintingFlags().skipRegions());
     }
   };
   SymbolTable::walkSymbolTables(top, /*allSymUsesVisible=*/!top->getBlock(),
                                 walkFn);
   LDBG() << "[init] Finished initializeSymbolCallables for top-level op: "
-         << top->getName();
+         << OpWithFlags(top, OpPrintingFlags().skipRegions());
 }
 
 /// Returns true if the operation is a returning terminator in region
@@ -229,12 +243,13 @@ static bool isRegionOrCallableReturn(Operation *op) {
 }
 
 LogicalResult DeadCodeAnalysis::initializeRecursively(Operation *op) {
-  LDBG() << "[init] Entering initializeRecursively for op: " << op->getName()
-         << " at " << op;
+  LDBG() << "[init] Entering initializeRecursively for op: "
+         << OpWithFlags(op, OpPrintingFlags().skipRegions());
   // Initialize the analysis by visiting every op with control-flow semantics.
   if (op->getNumRegions() || op->getNumSuccessors() ||
       isRegionOrCallableReturn(op) || isa<CallOpInterface>(op)) {
-    LDBG() << "[init] Visiting op with control-flow semantics: " << *op;
+    LDBG() << "[init] Visiting op with control-flow semantics: "
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
     // When the liveness of the parent block changes, make sure to
     // re-invoke the analysis on the op.
     if (op->getBlock())
@@ -246,16 +261,17 @@ LogicalResult DeadCodeAnalysis::initializeRecursively(Operation *op) {
   }
   // Recurse on nested operations.
   for (Region &region : op->getRegions()) {
-    LDBG() << "[init] Recursing into region of op: " << op->getName();
+    LDBG() << "[init] Recursing into region of op: "
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
     for (Operation &nestedOp : region.getOps()) {
-      LDBG() << "[init] Recursing into nested op: " << nestedOp.getName()
-             << " at " << &nestedOp;
+      LDBG() << "[init] Recursing into nested op: "
+             << OpWithFlags(&nestedOp, OpPrintingFlags().skipRegions());
       if (failed(initializeRecursively(&nestedOp)))
         return failure();
     }
   }
-  LDBG() << "[init] Finished initializeRecursively for op: " << op->getName()
-         << " at " << op;
+  LDBG() << "[init] Finished initializeRecursively for op: "
+         << OpWithFlags(op, OpPrintingFlags().skipRegions());
   return success();
 }
 
@@ -269,35 +285,40 @@ void DeadCodeAnalysis::markEdgeLive(Block *from, Block *to) {
 }
 
 void DeadCodeAnalysis::markEntryBlocksLive(Operation *op) {
-  LDBG() << "Marking entry blocks live for op: " << op->getName();
+  LDBG() << "Marking entry blocks live for op: "
+         << OpWithFlags(op, OpPrintingFlags().skipRegions());
   for (Region &region : op->getRegions()) {
     if (region.empty())
       continue;
     auto *state =
         getOrCreate<Executable>(getProgramPointBefore(&region.front()));
     propagateIfChanged(state, state->setToLive());
-    LDBG() << "Marked entry block live for region in op: " << op->getName();
+    LDBG() << "Marked entry block live for region in op: "
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
   }
 }
 
 LogicalResult DeadCodeAnalysis::visit(ProgramPoint *point) {
-  LDBG() << "Visiting program point: " << point << " " << *point;
+  LDBG() << "Visiting program point: " << *point;
   if (point->isBlockStart())
     return success();
   Operation *op = point->getPrevOp();
-  LDBG() << "Visiting operation: " << *op;
+  LDBG() << "Visiting operation: "
+         << OpWithFlags(op, OpPrintingFlags().skipRegions());
 
   // If the parent block is not executable, there is nothing to do.
   if (op->getBlock() != nullptr &&
       !getOrCreate<Executable>(getProgramPointBefore(op->getBlock()))
            ->isLive()) {
-    LDBG() << "Parent block not live, skipping op: " << *op;
+    LDBG() << "Parent block not live, skipping op: "
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
     return success();
   }
 
   // We have a live call op. Add this as a live predecessor of the callee.
   if (auto call = dyn_cast<CallOpInterface>(op)) {
-    LDBG() << "Visiting call operation: " << *op;
+    LDBG() << "Visiting call operation: "
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
     visitCallOperation(call);
   }
 
@@ -305,12 +326,14 @@ LogicalResult DeadCodeAnalysis::visit(ProgramPoint *point) {
   if (op->getNumRegions()) {
     // Check if we can reason about the region control-flow.
     if (auto branch = dyn_cast<RegionBranchOpInterface>(op)) {
-      LDBG() << "Visiting region branch operation: " << *op;
+      LDBG() << "Visiting region branch operation: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions());
       visitRegionBranchOperation(branch);
 
       // Check if this is a callable operation.
     } else if (auto callable = dyn_cast<CallableOpInterface>(op)) {
-      LDBG() << "Visiting callable operation: " << *op;
+      LDBG() << "Visiting callable operation: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions());
       const auto *callsites = getOrCreateFor<PredecessorState>(
           getProgramPointAfter(op), getProgramPointAfter(callable));
 
@@ -322,19 +345,22 @@ LogicalResult DeadCodeAnalysis::visit(ProgramPoint *point) {
 
       // Otherwise, conservatively mark all entry blocks as executable.
     } else {
-      LDBG() << "Marking all entry blocks live for op: " << *op;
+      LDBG() << "Marking all entry blocks live for op: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions());
       markEntryBlocksLive(op);
     }
   }
 
   if (isRegionOrCallableReturn(op)) {
     if (auto branch = dyn_cast<RegionBranchOpInterface>(op->getParentOp())) {
-      LDBG() << "Visiting region terminator: " << *op;
+      LDBG() << "Visiting region terminator: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions());
       // Visit the exiting terminator of a region.
       visitRegionTerminator(op, branch);
     } else if (auto callable =
                    dyn_cast<CallableOpInterface>(op->getParentOp())) {
-      LDBG() << "Visiting callable terminator: " << *op;
+      LDBG() << "Visiting callable terminator: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions());
       // Visit the exiting terminator of a callable.
       visitCallableTerminator(op, callable);
     }
@@ -343,12 +369,14 @@ LogicalResult DeadCodeAnalysis::visit(ProgramPoint *point) {
   if (op->getNumSuccessors()) {
     // Check if we can reason about the control-flow.
     if (auto branch = dyn_cast<BranchOpInterface>(op)) {
-      LDBG() << "Visiting branch operation: " << *op;
+      LDBG() << "Visiting branch operation: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions());
       visitBranchOperation(branch);
 
       // Otherwise, conservatively mark all successors as exectuable.
     } else {
-      LDBG() << "Marking all successors live for op: " << *op;
+      LDBG() << "Marking all successors live for op: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions());
       for (Block *successor : op->getSuccessors())
         markEdgeLive(op->getBlock(), successor);
     }
@@ -358,7 +386,8 @@ LogicalResult DeadCodeAnalysis::visit(ProgramPoint *point) {
 }
 
 void DeadCodeAnalysis::visitCallOperation(CallOpInterface call) {
-  LDBG() << "visitCallOperation: " << call.getOperation()->getName();
+  LDBG() << "visitCallOperation: "
+         << OpWithFlags(call.getOperation(), OpPrintingFlags().skipRegions());
   Operation *callableOp = call.resolveCallableInTable(&symbolTable);
 
   // A call to a externally-defined callable has unknown predecessors.
@@ -382,14 +411,14 @@ void DeadCodeAnalysis::visitCallOperation(CallOpInterface call) {
         getOrCreate<PredecessorState>(getProgramPointAfter(callableOp));
     propagateIfChanged(callsites, callsites->join(call));
     LDBG() << "Added callsite as predecessor for callable: "
-           << callableOp->getName();
+           << OpWithFlags(callableOp, OpPrintingFlags().skipRegions());
   } else {
     // Mark this call op's predecessors as overdefined.
     auto *predecessors =
         getOrCreate<PredecessorState>(getProgramPointAfter(call));
     propagateIfChanged(predecessors, predecessors->setHasUnknownPredecessors());
     LDBG() << "Marked call op's predecessors as unknown for: "
-           << call.getOperation()->getName();
+           << OpWithFlags(call.getOperation(), OpPrintingFlags().skipRegions());
   }
 }
 
@@ -421,7 +450,8 @@ DeadCodeAnalysis::getOperandValues(Operation *op) {
 }
 
 void DeadCodeAnalysis::visitBranchOperation(BranchOpInterface branch) {
-  LDBG() << "visitBranchOperation: " << branch.getOperation()->getName();
+  LDBG() << "visitBranchOperation: "
+         << OpWithFlags(branch.getOperation(), OpPrintingFlags().skipRegions());
   // Try to deduce a single successor for the branch.
   std::optional<SmallVector<Attribute>> operands = getOperandValues(branch);
   if (!operands)
@@ -440,7 +470,8 @@ void DeadCodeAnalysis::visitBranchOperation(BranchOpInterface branch) {
 
 void DeadCodeAnalysis::visitRegionBranchOperation(
     RegionBranchOpInterface branch) {
-  LDBG() << "visitRegionBranchOperation: " << branch.getOperation()->getName();
+  LDBG() << "visitRegionBranchOperation: "
+         << OpWithFlags(branch.getOperation(), OpPrintingFlags().skipRegions());
   // Try to deduce which regions are executable.
   std::optional<SmallVector<Attribute>> operands = getOperandValues(branch);
   if (!operands)
@@ -517,14 +548,14 @@ void DeadCodeAnalysis::visitCallableTerminator(Operation *op,
     if (canResolve) {
       propagateIfChanged(predecessors, predecessors->join(op));
       LDBG() << "Added callable terminator as predecessor for callsite: "
-             << predecessor->getName();
+             << OpWithFlags(predecessor, OpPrintingFlags().skipRegions());
     } else {
       // If the terminator is not a return-like, then conservatively assume we
       // can't resolve the predecessor.
       propagateIfChanged(predecessors,
                          predecessors->setHasUnknownPredecessors());
       LDBG() << "Could not resolve callable terminator for callsite: "
-             << predecessor->getName();
+             << OpWithFlags(predecessor, OpPrintingFlags().skipRegions());
     }
   }
 }
diff --git a/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp b/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp
index c7a950d9a8871..e79f6a8aec1cf 100644
--- a/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp
@@ -19,6 +19,8 @@
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/Dialect.h"
 #include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
@@ -28,6 +30,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include <cassert>
 #include <optional>
 #include <utility>
@@ -87,7 +90,8 @@ LogicalResult IntegerRangeAnalysis::visitOperation(
     return success();
   }
 
-  LLVM_DEBUG(llvm::dbgs() << "Inferring ranges for " << *op << "\n");
+  LDBG() << "Inferring ranges for "
+         << OpWithFlags(op, OpPrintingFlags().skipRegions());
   auto argRanges = llvm::map_to_vector(
       operands, [](const IntegerValueRangeLattice *lattice) {
         return lattice->getValue();
@@ -99,7 +103,7 @@ LogicalResult IntegerRangeAnalysis::visitOperation(
       return;
     assert(llvm::is_contained(op->getResults(), result));
 
-    LLVM_DEBUG(llvm::dbgs() << "Inferred range " << attrs << "\n");
+    LDBG() << "Inferred range " << attrs;
     IntegerValueRangeLattice *lattice = results[result.getResultNumber()];
     IntegerValueRange oldRange = lattice->getValue();
 
@@ -114,7 +118,7 @@ LogicalResult IntegerRangeAnalysis::visitOperation(
     });
     if (isYieldedResult && !oldRange.isUninitialized() &&
         !(lattice->getValue() == oldRange)) {
-      LLVM_DEBUG(llvm::dbgs() << "Loop variant loop result detected\n");
+      LDBG() << "Loop variant loop result detected";
       changed |= lattice->join(IntegerValueRange::getMaxRange(v));
     }
     propagateIfChanged(lattice, changed);
@@ -128,7 +132,8 @@ void IntegerRangeAnalysis::visitNonControlFlowArguments(
     Operation *op, const RegionSuccessor &successor,
     ArrayRef<IntegerValueRangeLattice *> argLattices, unsigned firstIndex) {
   if (auto inferrable = dyn_cast<InferIntRangeInterface>(op)) {
-    LLVM_DEBUG(llvm::dbgs() << "Inferring ranges for " << *op << "\n");
+    LDBG() << "Inferring ranges for "
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
 
     auto argRanges = llvm::map_to_vector(op->getOperands(), [&](Value value) {
       return getLatticeElementFor(getProgramPointAfter(op), value)->getValue();
@@ -141,7 +146,7 @@ void IntegerRangeAnalysis::visitNonControlFlowArguments(
       if (!llvm::is_contained(successor.getSuccessor()->getArguments(), arg))
         return;
 
-      LLVM_DEBUG(llvm::dbgs() << "Inferred range " << attrs << "\n");
+      LDBG() << "Inferred range " << attrs;
       IntegerValueRangeLattice *lattice = argLattices[arg.getArgNumber()];
       IntegerValueRange oldRange = lattice->getValue();
 
@@ -156,7 +161,7 @@ void IntegerRangeAnalysis::visitNonControlFlowArguments(
       });
       if (isYieldedValue && !oldRange.isUninitialized() &&
           !(lattice->getValue() == oldRange)) {
-        LLVM_DEBUG(llvm::dbgs() << "Loop variant loop result detected\n");
+        LDBG() << "Loop variant loop result detected";
         changed |= lattice->join(IntegerValueRange::getMaxRange(v));
       }
       propagateIfChanged(lattice, changed);
diff --git a/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp b/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp
index 509f5202be08d..65df355216f74 100644
--- a/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp
@@ -294,7 +294,34 @@ RunLivenessAnalysis::RunLivenessAnalysis(Operation *op) {
   solver.load<LivenessAnalysis>(symbolTable);
   LDBG() << "Initializing and running solver";
   (void)solver.initializeAndRun(op);
-  LDBG() << "RunLivenessAnalysis initialized for op: " << op->getName();
+  LDBG() << "RunLivenessAnalysis initialized for op: " << op->getName()
+         << " check on unreachable code now:";
+  // The framework doesn't visit operations in dead blocks, so we need to
+  // explicitly mark them as dead.
+  op->walk([&](Operation *op) {
+    if (op->getNumResults() == 0)
+      return;
+    for (auto result : llvm::enumerate(op->getResults())) {
+      if (getLiveness(result.value()))
+        continue;
+      LDBG() << "Result: " << result.index() << " of "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions())
+             << " has no liveness info (unreachable), mark dead";
+      solver.getOrCreateState<Liveness>(result.value());
+    }
+    for (auto &region : op->getRegions()) {
+      for (auto &block : region) {
+        for (auto blockArg : llvm::enumerate(block.getArguments())) {
+          if (getLiveness(blockArg.value()))
+            continue;
+          LDBG() << "Block argument: " << blockArg.index() << " of "
+                 << OpWithFlags(op, OpPrintingFlags().skipRegions())
+                 << " has no liveness info, mark dead";
+          solver.getOrCreateState<Liveness>(blockArg.value());
+        }
+      }
+    }
+  });
 }
 
 const Liveness *RunLivenessAnalysis::getLiveness(Value val) {
diff --git a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
index e625f626d12fd..13a3e1480c836 100644
--- a/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/SparseAnalysis.cpp
@@ -19,12 +19,15 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/DebugLog.h"
 #include <cassert>
 #include <optional>
 
 using namespace mlir;
 using namespace mlir::dataflow;
 
+#define DEBUG_TYPE "dataflow"
+
 //===----------------------------------------------------------------------===//
 // AbstractSparseLattice
 //===----------------------------------------------------------------------===//
@@ -64,22 +67,36 @@ AbstractSparseForwardDataFlowAnalysis::initialize(Operation *top) {
 
 LogicalResult
 AbstractSparseForwardDataFlowAnalysis::initializeRecursively(Operation *op) {
+  LDBG() << "Initializing recursively for operation: " << op->getName();
+
   // Initialize the analysis by visiting every owner of an SSA value (all
   // operations and blocks).
-  if (failed(visitOperation(op)))
+  if (failed(visitOperation(op))) {
+    LDBG() << "Failed to visit operation: " << op->getName();
     return failure();
+  }
 
   for (Region &region : op->getRegions()) {
+    LDBG() << "Processing region with " << region.getBlocks().size()
+           << " blocks";
     for (Block &block : region) {
+      LDBG() << "Processing block with " << block.getNumArguments()
+             << " arguments";
       getOrCreate<Executable>(getProgramPointBefore(&block))
           ->blockContentSubscribe(this);
       visitBlock(&block);
-      for (Operation &op : block)
-        if (failed(initializeRecursively(&op)))
+      for (Operation &op : block) {
+        LDBG() << "Recursively initializing nested operation: " << op.getName();
+        if (failed(initializeRecursively(&op))) {
+          LDBG() << "Failed to initialize nested operation: " << op.getName();
           return failure();
+        }
+      }
     }
   }
 
+  LDBG() << "Successfully completed recursive initialization for operation: "
+         << op->getName();
   return success();
 }
 
@@ -409,11 +426,20 @@ static MutableArrayRef<OpOperand> operandsToOpOperands(OperandRange &operands) {
 
 LogicalResult
 AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
+  LDBG() << "Visiting operation: " << op->getName() << " with "
+         << op->getNumOperands() << " operands and " << op->getNumResults()
+         << " results";
+
   // If we're in a dead block, bail out.
   if (op->getBlock() != nullptr &&
-      !getOrCreate<Executable>(getProgramPointBefore(op->getBlock()))->isLive())
+      !getOrCreate<Executable>(getProgramPointBefore(op->getBlock()))
+           ->isLive()) {
+    LDBG() << "Operation is in dead block, bailing out";
     return success();
+  }
 
+  LDBG() << "Creating lattice elements for " << op->getNumOperands()
+         << " operands and " << op->getNumResults() << " results";
   SmallVector<AbstractSparseLattice *> operandLattices =
       getLatticeElements(op->getOperands());
   SmallVector<const AbstractSparseLattice *> resultLattices =
@@ -422,11 +448,15 @@ AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
   // Block arguments of region branch operations flow back into the operands
   // of the parent op
   if (auto branch = dyn_cast<RegionBranchOpInterface>(op)) {
+    LDBG() << "Processing RegionBranchOpInterface operation";
     visitRegionSuccessors(branch, operandLattices);
     return success();
   }
 
   if (auto branch = dyn_cast<BranchOpInterface>(op)) {
+    LDBG() << "Processing BranchOpInterface operation with "
+           << op->getNumSuccessors() << " successors";
+
     // Block arguments of successor blocks flow back into our operands.
 
     // We remember all operands not forwarded to any block in a BitVector.
@@ -463,6 +493,7 @@ AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
   // For function calls, connect the arguments of the entry blocks to the
   // operands of the call op that are forwarded to these arguments.
   if (auto call = dyn_cast<CallOpInterface>(op)) {
+    LDBG() << "Processing CallOpInterface operation";
     Operation *callableOp = call.resolveCallableInTable(&symbolTable);
     if (auto callable = dyn_cast_or_null<CallableOpInterface>(callableOp)) {
       // Not all operands of a call op forward to arguments. Such operands are
@@ -513,6 +544,7 @@ AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
   // of this op itself and the operands of the terminators of the regions of
   // this op.
   if (auto terminator = dyn_cast<RegionBranchTerminatorOpInterface>(op)) {
+    LDBG() << "Processing RegionBranchTerminatorOpInterface operation";
     if (auto branch = dyn_cast<RegionBranchOpInterface>(op->getParentOp())) {
       visitRegionSuccessorsFromTerminator(terminator, branch);
       return success();
@@ -520,12 +552,16 @@ AbstractSparseBackwardDataFlowAnalysis::visitOperation(Operation *op) {
   }
 
   if (op->hasTrait<OpTrait::ReturnLike>()) {
+    LDBG() << "Processing ReturnLike operation";
     // Going backwards, the operands of the return are derived from the
     // results of all CallOps calling this CallableOp.
-    if (auto callable = dyn_cast<CallableOpInterface>(op->getParentOp()))
+    if (auto callable = dyn_cast<CallableOpInterface>(op->getParentOp())) {
+      LDBG() << "Callable parent found, visiting callable operation";
       return visitCallableOperation(op, callable, operandLattices);
+    }
   }
 
+  LDBG() << "Using default visitOperationImpl for operation: " << op->getName();
   return visitOperationImpl(op, operandLattices, resultLattices);
 }
 
diff --git a/mlir/lib/Analysis/DataFlowFramework.cpp b/mlir/lib/Analysis/DataFlowFramework.cpp
index 16f7033da7edf..7e1b4052027d3 100644
--- a/mlir/lib/Analysis/DataFlowFramework.cpp
+++ b/mlir/lib/Analysis/DataFlowFramework.cpp
@@ -45,7 +45,7 @@ void AnalysisState::addDependency(ProgramPoint *dependent,
   DATAFLOW_DEBUG({
     if (inserted) {
       LDBG() << "Creating dependency between " << debugName << " of " << anchor
-             << "\nand " << debugName << " on " << dependent;
+             << "\nand " << debugName << " on " << *dependent;
     }
   });
 }
@@ -62,11 +62,12 @@ void ProgramPoint::print(raw_ostream &os) const {
     return;
   }
   if (!isBlockStart()) {
-    os << "<after operation>:";
-    return getPrevOp()->print(os, OpPrintingFlags().skipRegions());
+    os << "<after operation>:"
+       << OpWithFlags(getPrevOp(), OpPrintingFlags().skipRegions());
+    return;
   }
-  os << "<before operation>:";
-  return getNextOp()->print(os, OpPrintingFlags().skipRegions());
+  os << "<before operation>:"
+     << OpWithFlags(getNextOp(), OpPrintingFlags().skipRegions());
 }
 
 //===----------------------------------------------------------------------===//
@@ -78,8 +79,8 @@ void LatticeAnchor::print(raw_ostream &os) const {
     os << "<NULL POINT>";
     return;
   }
-  if (auto *LatticeAnchor = llvm::dyn_cast<GenericLatticeAnchor *>(*this))
-    return LatticeAnchor->print(os);
+  if (auto *latticeAnchor = llvm::dyn_cast<GenericLatticeAnchor *>(*this))
+    return latticeAnchor->print(os);
   if (auto value = llvm::dyn_cast<Value>(*this)) {
     return value.print(os, OpPrintingFlags().skipRegions());
   }
@@ -88,8 +89,8 @@ void LatticeAnchor::print(raw_ostream &os) const {
 }
 
 Location LatticeAnchor::getLoc() const {
-  if (auto *LatticeAnchor = llvm::dyn_cast<GenericLatticeAnchor *>(*this))
-    return LatticeAnchor->getLoc();
+  if (auto *latticeAnchor = llvm::dyn_cast<GenericLatticeAnchor *>(*this))
+    return latticeAnchor->getLoc();
   if (auto value = llvm::dyn_cast<Value>(*this))
     return value.getLoc();
 
@@ -128,7 +129,7 @@ LogicalResult DataFlowSolver::initializeAndRun(Operation *top) {
     worklist.pop();
 
     DATAFLOW_DEBUG(LDBG() << "Invoking '" << analysis->debugName
-                          << "' on: " << point);
+                          << "' on: " << *point);
     if (failed(analysis->visit(point)))
       return failure();
   }
diff --git a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
index f4b02b496a5c5..30ce1fb320017 100644
--- a/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
+++ b/mlir/lib/Analysis/FlatLinearValueConstraints.cpp
@@ -60,7 +60,7 @@ struct AffineExprFlattener : public SimpleAffineExprFlattener {
                           AffineExpr localExpr) override {
     SimpleAffineExprFlattener::addLocalFloorDivId(dividend, divisor, localExpr);
     // Update localVarCst.
-    localVarCst.addLocalFloorDiv(dividend, divisor);
+    (void)localVarCst.addLocalFloorDiv(dividend, divisor);
   }
 
   LogicalResult addLocalIdSemiAffine(ArrayRef<int64_t> lhs,
diff --git a/mlir/lib/Analysis/Presburger/Barvinok.cpp b/mlir/lib/Analysis/Presburger/Barvinok.cpp
index 4546e49639fb9..75d592e976edf 100644
--- a/mlir/lib/Analysis/Presburger/Barvinok.cpp
+++ b/mlir/lib/Analysis/Presburger/Barvinok.cpp
@@ -554,7 +554,7 @@ QuasiPolynomial mlir::presburger::detail::getCoefficientInRationalFunction(
 /// t^num / \prod_j (1 - t^dens[j]).
 /// v represents the affine functions whose floors are multiplied by the
 /// generators, and ds represents the list of generators.
-std::pair<QuasiPolynomial, std::vector<Fraction>>
+static std::pair<QuasiPolynomial, std::vector<Fraction>>
 substituteMuInTerm(unsigned numParams, const ParamPoint &v,
                    const std::vector<Point> &ds, const Point &mu) {
   unsigned numDims = mu.size();
@@ -606,8 +606,8 @@ substituteMuInTerm(unsigned numParams, const ParamPoint &v,
 /// Here, sign = ± 1,
 /// num is a QuasiPolynomial, and
 /// each dens[j] is a Fraction.
-void normalizeDenominatorExponents(int &sign, QuasiPolynomial &num,
-                                   std::vector<Fraction> &dens) {
+static void normalizeDenominatorExponents(int &sign, QuasiPolynomial &num,
+                                          std::vector<Fraction> &dens) {
   // We track the number of exponents that are negative in the
   // denominator, and convert them to their absolute values.
   unsigned numNegExps = 0;
@@ -634,8 +634,8 @@ void normalizeDenominatorExponents(int &sign, QuasiPolynomial &num,
 
 /// Compute the binomial coefficients nCi for 0 ≤ i ≤ r,
 /// where n is a QuasiPolynomial.
-std::vector<QuasiPolynomial> getBinomialCoefficients(const QuasiPolynomial &n,
-                                                     unsigned r) {
+static std::vector<QuasiPolynomial>
+getBinomialCoefficients(const QuasiPolynomial &n, unsigned r) {
   unsigned numParams = n.getNumInputs();
   std::vector<QuasiPolynomial> coefficients;
   coefficients.reserve(r + 1);
@@ -651,8 +651,8 @@ std::vector<QuasiPolynomial> getBinomialCoefficients(const QuasiPolynomial &n,
 
 /// Compute the binomial coefficients nCi for 0 ≤ i ≤ r,
 /// where n is a QuasiPolynomial.
-std::vector<Fraction> getBinomialCoefficients(const Fraction &n,
-                                              const Fraction &r) {
+static std::vector<Fraction> getBinomialCoefficients(const Fraction &n,
+                                                     const Fraction &r) {
   std::vector<Fraction> coefficients;
   coefficients.reserve((int64_t)floor(r));
   coefficients.emplace_back(1);
diff --git a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
index 5c4d4d13580a0..0dcdd5bb97bc8 100644
--- a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
+++ b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
@@ -1500,12 +1500,13 @@ void IntegerRelation::addBound(BoundType type, ArrayRef<DynamicAPInt> expr,
 /// respect to a positive constant 'divisor'. Two constraints are added to the
 /// system to capture equivalence with the floordiv.
 ///      q = expr floordiv c    <=>   c*q <= expr <= c*q + c - 1.
-void IntegerRelation::addLocalFloorDiv(ArrayRef<DynamicAPInt> dividend,
-                                       const DynamicAPInt &divisor) {
+/// Returns the column position of the new local variable.
+unsigned IntegerRelation::addLocalFloorDiv(ArrayRef<DynamicAPInt> dividend,
+                                           const DynamicAPInt &divisor) {
   assert(dividend.size() == getNumCols() && "incorrect dividend size");
   assert(divisor > 0 && "positive divisor expected");
 
-  appendVar(VarKind::Local);
+  unsigned newVar = appendVar(VarKind::Local);
 
   SmallVector<DynamicAPInt, 8> dividendCopy(dividend);
   dividendCopy.insert(dividendCopy.end() - 1, DynamicAPInt(0));
@@ -1513,6 +1514,28 @@ void IntegerRelation::addLocalFloorDiv(ArrayRef<DynamicAPInt> dividend,
       getDivLowerBound(dividendCopy, divisor, dividendCopy.size() - 2));
   addInequality(
       getDivUpperBound(dividendCopy, divisor, dividendCopy.size() - 2));
+  return newVar;
+}
+
+unsigned IntegerRelation::addLocalModulo(ArrayRef<DynamicAPInt> exprs,
+                                         const DynamicAPInt &modulus) {
+  assert(exprs.size() == getNumCols() && "incorrect exprs size");
+  assert(modulus > 0 && "positive modulus expected");
+
+  /// Add a local variable for q = expr floordiv modulus
+  addLocalFloorDiv(exprs, modulus);
+
+  /// Add a local var to represent the result
+  auto resultIndex = appendVar(VarKind::Local);
+
+  SmallVector<DynamicAPInt, 8> exprsCopy(exprs);
+  /// Insert the two new locals before the constant
+  /// Add locals that correspond to `q` and `result` to compute
+  /// 0 = (expr - modulus * q) - result
+  exprsCopy.insert(exprsCopy.end() - 1,
+                   {DynamicAPInt(-modulus), DynamicAPInt(-1)});
+  addEquality(exprsCopy);
+  return resultIndex;
 }
 
 int IntegerRelation::findEqualityToConstant(unsigned pos, bool symbolic) const {
diff --git a/mlir/lib/Analysis/Presburger/Matrix.cpp b/mlir/lib/Analysis/Presburger/Matrix.cpp
index 9fc6205eb5ed5..bb6056487512a 100644
--- a/mlir/lib/Analysis/Presburger/Matrix.cpp
+++ b/mlir/lib/Analysis/Presburger/Matrix.cpp
@@ -402,10 +402,10 @@ void Matrix<T>::print(raw_ostream &os) const {
   for (unsigned row = 0; row < nRows; ++row)
     for (unsigned column = 0; column < nColumns; ++column)
       updatePrintMetrics<T>(at(row, column), ptm);
-  unsigned MIN_SPACING = 1;
+  unsigned minSpacing = 1;
   for (unsigned row = 0; row < nRows; ++row) {
     for (unsigned column = 0; column < nColumns; ++column) {
-      printWithPrintMetrics<T>(os, at(row, column), MIN_SPACING, ptm);
+      printWithPrintMetrics<T>(os, at(row, column), minSpacing, ptm);
     }
     os << "\n";
   }
@@ -721,7 +721,7 @@ FracMatrix FracMatrix::gramSchmidt() const {
 // Otherwise, we swap b_k and b_{k-1} and decrement k.
 //
 // We repeat this until k = n and return.
-void FracMatrix::LLL(Fraction delta) {
+void FracMatrix::LLL(const Fraction &delta) {
   DynamicAPInt nearest;
   Fraction mu;
 
diff --git a/mlir/lib/Analysis/Presburger/QuasiPolynomial.cpp b/mlir/lib/Analysis/Presburger/QuasiPolynomial.cpp
index 84d885fd96c21..4e374d0178d4c 100644
--- a/mlir/lib/Analysis/Presburger/QuasiPolynomial.cpp
+++ b/mlir/lib/Analysis/Presburger/QuasiPolynomial.cpp
@@ -112,14 +112,14 @@ QuasiPolynomial QuasiPolynomial::simplify() {
     // A term is zero if its coefficient is zero, or
     if (coefficients[i] == Fraction(0, 1))
       continue;
-    bool product_is_zero =
+    bool productIsZero =
         // if any of the affine functions in the product
-        llvm::any_of(affine[i], [](const SmallVector<Fraction> &affine_ij) {
+        llvm::any_of(affine[i], [](const SmallVector<Fraction> &affineIj) {
           // has all its coefficients as zero.
-          return llvm::all_of(affine_ij,
+          return llvm::all_of(affineIj,
                               [](const Fraction &f) { return f == 0; });
         });
-    if (product_is_zero)
+    if (productIsZero)
       continue;
 
     // Now, we know the term is nonzero.
diff --git a/mlir/lib/Analysis/Presburger/Simplex.cpp b/mlir/lib/Analysis/Presburger/Simplex.cpp
index 08290db55f2c7..a1cbe2990c67d 100644
--- a/mlir/lib/Analysis/Presburger/Simplex.cpp
+++ b/mlir/lib/Analysis/Presburger/Simplex.cpp
@@ -433,7 +433,7 @@ LogicalResult SymbolicLexSimplex::addSymbolicCut(unsigned row) {
   normalizeDiv(divCoeffs, divDenom);
 
   domainSimplex.addDivisionVariable(divCoeffs, divDenom);
-  domainPoly.addLocalFloorDiv(divCoeffs, divDenom);
+  (void)domainPoly.addLocalFloorDiv(divCoeffs, divDenom);
 
   // Update `this` to account for the additional symbol we just added.
   appendSymbol();
@@ -1663,7 +1663,7 @@ class presburger::GBRSimplex {
   /// First pushes a snapshot for the current simplex state to the stack so
   /// that this can be rolled back later.
   void addEqualityForDirection(ArrayRef<DynamicAPInt> dir) {
-    assert(llvm::any_of(dir, [](const DynamicAPInt &x) { return x != 0; }) &&
+    assert(llvm::any_of(dir, [](const DynamicAPInt &X) { return X != 0; }) &&
            "Direction passed is the zero vector!");
     snapshotStack.emplace_back(simplex.getSnapshot());
     simplex.addEquality(getCoeffsForDirection(dir));
@@ -2156,10 +2156,10 @@ void SimplexBase::print(raw_ostream &os) const {
   for (unsigned row = 0, numRows = getNumRows(); row < numRows; ++row)
     for (unsigned col = 0, numCols = getNumColumns(); col < numCols; ++col)
       updatePrintMetrics<DynamicAPInt>(tableau(row, col), ptm);
-  unsigned MIN_SPACING = 1;
+  unsigned minSpacing = 1;
   for (unsigned row = 0, numRows = getNumRows(); row < numRows; ++row) {
     for (unsigned col = 0, numCols = getNumColumns(); col < numCols; ++col) {
-      printWithPrintMetrics<DynamicAPInt>(os, tableau(row, col), MIN_SPACING,
+      printWithPrintMetrics<DynamicAPInt>(os, tableau(row, col), minSpacing,
                                           ptm);
     }
     os << '\n';
diff --git a/mlir/lib/Analysis/TopologicalSortUtils.cpp b/mlir/lib/Analysis/TopologicalSortUtils.cpp
index a2fd14910892a..99546e71533ae 100644
--- a/mlir/lib/Analysis/TopologicalSortUtils.cpp
+++ b/mlir/lib/Analysis/TopologicalSortUtils.cpp
@@ -101,12 +101,7 @@ bool mlir::sortTopologically(
 
 bool mlir::sortTopologically(
     Block *block, function_ref<bool(Value, Operation *)> isOperandReady) {
-  if (block->empty())
-    return true;
-  if (block->back().hasTrait<OpTrait::IsTerminator>())
-    return sortTopologically(block, block->without_terminator(),
-                             isOperandReady);
-  return sortTopologically(block, *block, isOperandReady);
+  return sortTopologically(block, block->without_terminator(), isOperandReady);
 }
 
 bool mlir::computeTopologicalSorting(
diff --git a/mlir/lib/Bindings/Python/DialectGPU.cpp b/mlir/lib/Bindings/Python/DialectGPU.cpp
index e5045cf0bba35..a21176fffb441 100644
--- a/mlir/lib/Bindings/Python/DialectGPU.cpp
+++ b/mlir/lib/Bindings/Python/DialectGPU.cpp
@@ -9,8 +9,8 @@
 #include "mlir-c/Dialect/GPU.h"
 #include "mlir-c/IR.h"
 #include "mlir-c/Support.h"
-#include "mlir/Bindings/Python/NanobindAdaptors.h"
 #include "mlir/Bindings/Python/Nanobind.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
 
 namespace nb = nanobind;
 using namespace nanobind::literals;
@@ -34,7 +34,7 @@ NB_MODULE(_mlirDialectsGPU, m) {
 
   mlirGPUAsyncTokenType.def_classmethod(
       "get",
-      [](nb::object cls, MlirContext ctx) {
+      [](const nb::object &cls, MlirContext ctx) {
         return cls(mlirGPUAsyncTokenTypeGet(ctx));
       },
       "Gets an instance of AsyncTokenType in the same context", nb::arg("cls"),
@@ -47,8 +47,9 @@ NB_MODULE(_mlirDialectsGPU, m) {
   mlir_attribute_subclass(m, "ObjectAttr", mlirAttributeIsAGPUObjectAttr)
       .def_classmethod(
           "get",
-          [](nb::object cls, MlirAttribute target, uint32_t format,
-             nb::bytes object, std::optional<MlirAttribute> mlirObjectProps,
+          [](const nb::object &cls, MlirAttribute target, uint32_t format,
+             const nb::bytes &object,
+             std::optional<MlirAttribute> mlirObjectProps,
              std::optional<MlirAttribute> mlirKernelsAttr) {
             MlirStringRef objectStrRef = mlirStringRefCreate(
                 static_cast<char *>(const_cast<void *>(object.data())),
diff --git a/mlir/lib/Bindings/Python/DialectLLVM.cpp b/mlir/lib/Bindings/Python/DialectLLVM.cpp
index f211e769d66be..ee106c0321669 100644
--- a/mlir/lib/Bindings/Python/DialectLLVM.cpp
+++ b/mlir/lib/Bindings/Python/DialectLLVM.cpp
@@ -12,8 +12,8 @@
 #include "mlir-c/IR.h"
 #include "mlir-c/Support.h"
 #include "mlir/Bindings/Python/Diagnostics.h"
-#include "mlir/Bindings/Python/NanobindAdaptors.h"
 #include "mlir/Bindings/Python/Nanobind.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
 
 namespace nb = nanobind;
 
@@ -24,7 +24,7 @@ using namespace mlir;
 using namespace mlir::python;
 using namespace mlir::python::nanobind_adaptors;
 
-void populateDialectLLVMSubmodule(const nanobind::module_ &m) {
+static void populateDialectLLVMSubmodule(const nanobind::module_ &m) {
 
   //===--------------------------------------------------------------------===//
   // StructType
@@ -35,8 +35,8 @@ void populateDialectLLVMSubmodule(const nanobind::module_ &m) {
 
   llvmStructType.def_classmethod(
       "get_literal",
-      [](nb::object cls, const std::vector<MlirType> &elements, bool packed,
-         MlirLocation loc) {
+      [](const nb::object &cls, const std::vector<MlirType> &elements,
+         bool packed, MlirLocation loc) {
         CollectDiagnosticsToStringScope scope(mlirLocationGetContext(loc));
 
         MlirType type = mlirLLVMStructTypeLiteralGetChecked(
@@ -51,7 +51,7 @@ void populateDialectLLVMSubmodule(const nanobind::module_ &m) {
 
   llvmStructType.def_classmethod(
       "get_identified",
-      [](nb::object cls, const std::string &name, MlirContext context) {
+      [](const nb::object &cls, const std::string &name, MlirContext context) {
         return cls(mlirLLVMStructTypeIdentifiedGet(
             context, mlirStringRefCreate(name.data(), name.size())));
       },
@@ -59,7 +59,7 @@ void populateDialectLLVMSubmodule(const nanobind::module_ &m) {
 
   llvmStructType.def_classmethod(
       "get_opaque",
-      [](nb::object cls, const std::string &name, MlirContext context) {
+      [](const nb::object &cls, const std::string &name, MlirContext context) {
         return cls(mlirLLVMStructTypeOpaqueGet(
             context, mlirStringRefCreate(name.data(), name.size())));
       },
@@ -79,7 +79,7 @@ void populateDialectLLVMSubmodule(const nanobind::module_ &m) {
 
   llvmStructType.def_classmethod(
       "new_identified",
-      [](nb::object cls, const std::string &name,
+      [](const nb::object &cls, const std::string &name,
          const std::vector<MlirType> &elements, bool packed, MlirContext ctx) {
         return cls(mlirLLVMStructTypeIdentifiedNewGet(
             ctx, mlirStringRefCreate(name.data(), name.length()),
@@ -123,7 +123,7 @@ void populateDialectLLVMSubmodule(const nanobind::module_ &m) {
   mlir_type_subclass(m, "PointerType", mlirTypeIsALLVMPointerType)
       .def_classmethod(
           "get",
-          [](nb::object cls, std::optional<unsigned> addressSpace,
+          [](const nb::object &cls, std::optional<unsigned> addressSpace,
              MlirContext context) {
             CollectDiagnosticsToStringScope scope(context);
             MlirType type = mlirLLVMPointerTypeGet(
diff --git a/mlir/lib/Bindings/Python/DialectNVGPU.cpp b/mlir/lib/Bindings/Python/DialectNVGPU.cpp
index a0d6a4b4c73f9..bb3f519c962f7 100644
--- a/mlir/lib/Bindings/Python/DialectNVGPU.cpp
+++ b/mlir/lib/Bindings/Python/DialectNVGPU.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir-c/Dialect/NVGPU.h"
 #include "mlir-c/IR.h"
-#include "mlir/Bindings/Python/NanobindAdaptors.h"
 #include "mlir/Bindings/Python/Nanobind.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
 
 namespace nb = nanobind;
 using namespace llvm;
@@ -23,8 +23,8 @@ static void populateDialectNVGPUSubmodule(const nb::module_ &m) {
 
   nvgpuTensorMapDescriptorType.def_classmethod(
       "get",
-      [](nb::object cls, MlirType tensorMemrefType, int swizzle, int l2promo,
-         int oobFill, int interleave, MlirContext ctx) {
+      [](const nb::object &cls, MlirType tensorMemrefType, int swizzle,
+         int l2promo, int oobFill, int interleave, MlirContext ctx) {
         return cls(mlirNVGPUTensorMapDescriptorTypeGet(
             ctx, tensorMemrefType, swizzle, l2promo, oobFill, interleave));
       },
diff --git a/mlir/lib/Bindings/Python/DialectPDL.cpp b/mlir/lib/Bindings/Python/DialectPDL.cpp
index bcc6ff406c952..2acedbc26b072 100644
--- a/mlir/lib/Bindings/Python/DialectPDL.cpp
+++ b/mlir/lib/Bindings/Python/DialectPDL.cpp
@@ -8,8 +8,8 @@
 
 #include "mlir-c/Dialect/PDL.h"
 #include "mlir-c/IR.h"
-#include "mlir/Bindings/Python/NanobindAdaptors.h"
 #include "mlir/Bindings/Python/Nanobind.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
 
 namespace nb = nanobind;
 using namespace llvm;
@@ -17,7 +17,7 @@ using namespace mlir;
 using namespace mlir::python;
 using namespace mlir::python::nanobind_adaptors;
 
-void populateDialectPDLSubmodule(const nanobind::module_ &m) {
+static void populateDialectPDLSubmodule(const nanobind::module_ &m) {
   //===-------------------------------------------------------------------===//
   // PDLType
   //===-------------------------------------------------------------------===//
@@ -32,7 +32,7 @@ void populateDialectPDLSubmodule(const nanobind::module_ &m) {
       mlir_type_subclass(m, "AttributeType", mlirTypeIsAPDLAttributeType);
   attributeType.def_classmethod(
       "get",
-      [](nb::object cls, MlirContext ctx) {
+      [](const nb::object &cls, MlirContext ctx) {
         return cls(mlirPDLAttributeTypeGet(ctx));
       },
       "Get an instance of AttributeType in given context.", nb::arg("cls"),
@@ -46,7 +46,7 @@ void populateDialectPDLSubmodule(const nanobind::module_ &m) {
       mlir_type_subclass(m, "OperationType", mlirTypeIsAPDLOperationType);
   operationType.def_classmethod(
       "get",
-      [](nb::object cls, MlirContext ctx) {
+      [](const nb::object &cls, MlirContext ctx) {
         return cls(mlirPDLOperationTypeGet(ctx));
       },
       "Get an instance of OperationType in given context.", nb::arg("cls"),
@@ -59,7 +59,7 @@ void populateDialectPDLSubmodule(const nanobind::module_ &m) {
   auto rangeType = mlir_type_subclass(m, "RangeType", mlirTypeIsAPDLRangeType);
   rangeType.def_classmethod(
       "get",
-      [](nb::object cls, MlirType elementType) {
+      [](const nb::object &cls, MlirType elementType) {
         return cls(mlirPDLRangeTypeGet(elementType));
       },
       "Gets an instance of RangeType in the same context as the provided "
@@ -77,7 +77,7 @@ void populateDialectPDLSubmodule(const nanobind::module_ &m) {
   auto typeType = mlir_type_subclass(m, "TypeType", mlirTypeIsAPDLTypeType);
   typeType.def_classmethod(
       "get",
-      [](nb::object cls, MlirContext ctx) {
+      [](const nb::object &cls, MlirContext ctx) {
         return cls(mlirPDLTypeTypeGet(ctx));
       },
       "Get an instance of TypeType in given context.", nb::arg("cls"),
@@ -90,7 +90,7 @@ void populateDialectPDLSubmodule(const nanobind::module_ &m) {
   auto valueType = mlir_type_subclass(m, "ValueType", mlirTypeIsAPDLValueType);
   valueType.def_classmethod(
       "get",
-      [](nb::object cls, MlirContext ctx) {
+      [](const nb::object &cls, MlirContext ctx) {
         return cls(mlirPDLValueTypeGet(ctx));
       },
       "Get an instance of TypeType in given context.", nb::arg("cls"),
diff --git a/mlir/lib/Bindings/Python/DialectQuant.cpp b/mlir/lib/Bindings/Python/DialectQuant.cpp
index 55571cd1e50a6..a5220fcc00604 100644
--- a/mlir/lib/Bindings/Python/DialectQuant.cpp
+++ b/mlir/lib/Bindings/Python/DialectQuant.cpp
@@ -165,7 +165,7 @@ static void populateDialectQuantSubmodule(const nb::module_ &m) {
                          quantizedType.get_class());
   anyQuantizedType.def_classmethod(
       "get",
-      [](nb::object cls, unsigned flags, MlirType storageType,
+      [](const nb::object &cls, unsigned flags, MlirType storageType,
          MlirType expressedType, int64_t storageTypeMin,
          int64_t storageTypeMax) {
         return cls(mlirAnyQuantizedTypeGet(flags, storageType, expressedType,
@@ -186,7 +186,7 @@ static void populateDialectQuantSubmodule(const nb::module_ &m) {
       quantizedType.get_class());
   uniformQuantizedType.def_classmethod(
       "get",
-      [](nb::object cls, unsigned flags, MlirType storageType,
+      [](const nb::object &cls, unsigned flags, MlirType storageType,
          MlirType expressedType, double scale, int64_t zeroPoint,
          int64_t storageTypeMin, int64_t storageTypeMax) {
         return cls(mlirUniformQuantizedTypeGet(flags, storageType,
@@ -221,7 +221,7 @@ static void populateDialectQuantSubmodule(const nb::module_ &m) {
       quantizedType.get_class());
   uniformQuantizedPerAxisType.def_classmethod(
       "get",
-      [](nb::object cls, unsigned flags, MlirType storageType,
+      [](const nb::object &cls, unsigned flags, MlirType storageType,
          MlirType expressedType, std::vector<double> scales,
          std::vector<int64_t> zeroPoints, int32_t quantizedDimension,
          int64_t storageTypeMin, int64_t storageTypeMax) {
@@ -293,7 +293,7 @@ static void populateDialectQuantSubmodule(const nb::module_ &m) {
       mlirTypeIsAUniformQuantizedSubChannelType, quantizedType.get_class());
   uniformQuantizedSubChannelType.def_classmethod(
       "get",
-      [](nb::object cls, unsigned flags, MlirType storageType,
+      [](const nb::object &cls, unsigned flags, MlirType storageType,
          MlirType expressedType, MlirAttribute scales, MlirAttribute zeroPoints,
          std::vector<int32_t> quantizedDimensions,
          std::vector<int64_t> blockSizes, int64_t storageTypeMin,
@@ -367,7 +367,8 @@ static void populateDialectQuantSubmodule(const nb::module_ &m) {
       quantizedType.get_class());
   calibratedQuantizedType.def_classmethod(
       "get",
-      [](nb::object cls, MlirType expressedType, double min, double max) {
+      [](const nb::object &cls, MlirType expressedType, double min,
+         double max) {
         return cls(mlirCalibratedQuantizedTypeGet(expressedType, min, max));
       },
       "Gets an instance of CalibratedQuantizedType in the same context as the "
diff --git a/mlir/lib/Bindings/Python/DialectSMT.cpp b/mlir/lib/Bindings/Python/DialectSMT.cpp
index 4e7647729fb0a..cab4219fea72b 100644
--- a/mlir/lib/Bindings/Python/DialectSMT.cpp
+++ b/mlir/lib/Bindings/Python/DialectSMT.cpp
@@ -24,7 +24,7 @@ using namespace mlir;
 using namespace mlir::python;
 using namespace mlir::python::nanobind_adaptors;
 
-void populateDialectSMTSubmodule(nanobind::module_ &m) {
+static void populateDialectSMTSubmodule(nanobind::module_ &m) {
 
   auto smtBoolType = mlir_type_subclass(m, "BoolType", mlirSMTTypeIsABool)
                          .def_classmethod(
diff --git a/mlir/lib/Bindings/Python/DialectSparseTensor.cpp b/mlir/lib/Bindings/Python/DialectSparseTensor.cpp
index 97cebcceebd9a..9d7dc110764fb 100644
--- a/mlir/lib/Bindings/Python/DialectSparseTensor.cpp
+++ b/mlir/lib/Bindings/Python/DialectSparseTensor.cpp
@@ -12,8 +12,8 @@
 #include "mlir-c/AffineMap.h"
 #include "mlir-c/Dialect/SparseTensor.h"
 #include "mlir-c/IR.h"
-#include "mlir/Bindings/Python/NanobindAdaptors.h"
 #include "mlir/Bindings/Python/Nanobind.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
 
 namespace nb = nanobind;
 using namespace llvm;
@@ -38,7 +38,8 @@ static void populateDialectSparseTensorSubmodule(const nb::module_ &m) {
                           mlirAttributeIsASparseTensorEncodingAttr)
       .def_classmethod(
           "get",
-          [](nb::object cls, std::vector<MlirSparseTensorLevelType> lvlTypes,
+          [](const nb::object &cls,
+             std::vector<MlirSparseTensorLevelType> lvlTypes,
              std::optional<MlirAffineMap> dimToLvl,
              std::optional<MlirAffineMap> lvlToDim, int posWidth, int crdWidth,
              std::optional<MlirAttribute> explicitVal,
@@ -58,7 +59,7 @@ static void populateDialectSparseTensorSubmodule(const nb::module_ &m) {
           "Gets a sparse_tensor.encoding from parameters.")
       .def_classmethod(
           "build_level_type",
-          [](nb::object cls, MlirSparseTensorLevelFormat lvlFmt,
+          [](const nb::object &cls, MlirSparseTensorLevelFormat lvlFmt,
              const std::vector<MlirSparseTensorLevelPropertyNondefault>
                  &properties,
              unsigned n, unsigned m) {
diff --git a/mlir/lib/Bindings/Python/DialectTransform.cpp b/mlir/lib/Bindings/Python/DialectTransform.cpp
index 59a030ac67f57..0190edf79118e 100644
--- a/mlir/lib/Bindings/Python/DialectTransform.cpp
+++ b/mlir/lib/Bindings/Python/DialectTransform.cpp
@@ -11,8 +11,8 @@
 #include "mlir-c/Dialect/Transform.h"
 #include "mlir-c/IR.h"
 #include "mlir-c/Support.h"
-#include "mlir/Bindings/Python/NanobindAdaptors.h"
 #include "mlir/Bindings/Python/Nanobind.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
 
 namespace nb = nanobind;
 using namespace mlir;
diff --git a/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp b/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp
index 81dada3553622..4885d62c56e6e 100644
--- a/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp
+++ b/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp
@@ -7,8 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir-c/ExecutionEngine.h"
-#include "mlir/Bindings/Python/NanobindAdaptors.h"
 #include "mlir/Bindings/Python/Nanobind.h"
+#include "mlir/Bindings/Python/NanobindAdaptors.h"
 
 namespace nb = nanobind;
 using namespace mlir;
@@ -124,6 +124,17 @@ NB_MODULE(_mlirExecutionEngine, m) {
           },
           nb::arg("name"), nb::arg("callback"),
           "Register `callback` as the runtime symbol `name`.")
+      .def(
+          "initialize",
+          [](PyExecutionEngine &executionEngine) {
+            mlirExecutionEngineInitialize(executionEngine.get());
+          },
+          "Initialize the ExecutionEngine. Global constructors specified by "
+          "`llvm.mlir.global_ctors` will be run. One common scenario is that "
+          "kernel binary compiled from `gpu.module` gets loaded during "
+          "initialization. Make sure all symbols are resolvable before "
+          "initialization by calling `register_runtime` or including "
+          "shared libraries.")
       .def(
           "dump_to_object_file",
           [](PyExecutionEngine &executionEngine, const std::string &fileName) {
diff --git a/mlir/lib/Bindings/Python/Globals.h b/mlir/lib/Bindings/Python/Globals.h
index 826a34a535176..71a051cb3d9f5 100644
--- a/mlir/lib/Bindings/Python/Globals.h
+++ b/mlir/lib/Bindings/Python/Globals.h
@@ -10,15 +10,19 @@
 #define MLIR_BINDINGS_PYTHON_GLOBALS_H
 
 #include <optional>
+#include <regex>
 #include <string>
+#include <unordered_set>
 #include <vector>
 
 #include "NanobindUtils.h"
 #include "mlir-c/IR.h"
 #include "mlir/CAPI/Support.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Regex.h"
 
 namespace mlir {
 namespace python {
@@ -114,6 +118,39 @@ class PyGlobals {
   std::optional<nanobind::object>
   lookupOperationClass(llvm::StringRef operationName);
 
+  class TracebackLoc {
+  public:
+    bool locTracebacksEnabled();
+
+    void setLocTracebacksEnabled(bool value);
+
+    size_t locTracebackFramesLimit();
+
+    void setLocTracebackFramesLimit(size_t value);
+
+    void registerTracebackFileInclusion(const std::string &file);
+
+    void registerTracebackFileExclusion(const std::string &file);
+
+    bool isUserTracebackFilename(llvm::StringRef file);
+
+    static constexpr size_t kMaxFrames = 512;
+
+  private:
+    nanobind::ft_mutex mutex;
+    bool locTracebackEnabled_ = false;
+    size_t locTracebackFramesLimit_ = 10;
+    std::unordered_set<std::string> userTracebackIncludeFiles;
+    std::unordered_set<std::string> userTracebackExcludeFiles;
+    std::regex userTracebackIncludeRegex;
+    bool rebuildUserTracebackIncludeRegex = false;
+    std::regex userTracebackExcludeRegex;
+    bool rebuildUserTracebackExcludeRegex = false;
+    llvm::StringMap<bool> isUserTracebackFilenameCache;
+  };
+
+  TracebackLoc &getTracebackLoc() { return tracebackLoc; }
+
 private:
   static PyGlobals *instance;
 
@@ -134,6 +171,8 @@ class PyGlobals {
   /// Set of dialect namespaces that we have attempted to import implementation
   /// modules for.
   llvm::StringSet<> loadedDialectModules;
+
+  TracebackLoc tracebackLoc;
 };
 
 } // namespace python
diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp
index 5feed95f96f53..4b3a06cbce854 100644
--- a/mlir/lib/Bindings/Python/IRCore.cpp
+++ b/mlir/lib/Bindings/Python/IRCore.cpp
@@ -20,11 +20,8 @@
 #include "nanobind/nanobind.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/raw_ostream.h"
 
 #include <optional>
-#include <system_error>
-#include <utility>
 
 namespace nb = nanobind;
 using namespace nb::literals;
@@ -1523,7 +1520,7 @@ nb::object PyOperation::create(std::string_view name,
                                llvm::ArrayRef<MlirValue> operands,
                                std::optional<nb::dict> attributes,
                                std::optional<std::vector<PyBlock *>> successors,
-                               int regions, DefaultingPyLocation location,
+                               int regions, PyLocation &location,
                                const nb::object &maybeIp, bool inferType) {
   llvm::SmallVector<MlirType, 4> mlirResults;
   llvm::SmallVector<MlirBlock, 4> mlirSuccessors;
@@ -1627,7 +1624,7 @@ nb::object PyOperation::create(std::string_view name,
   if (!operation.ptr)
     throw nb::value_error("Operation creation failed");
   PyOperationRef created =
-      PyOperation::createDetached(location->getContext(), operation);
+      PyOperation::createDetached(location.getContext(), operation);
   maybeInsertOperation(created, maybeIp);
 
   return created.getObject();
@@ -1937,9 +1934,9 @@ nb::object PyOpView::buildGeneric(
     std::optional<nb::list> resultTypeList, nb::list operandList,
     std::optional<nb::dict> attributes,
     std::optional<std::vector<PyBlock *>> successors,
-    std::optional<int> regions, DefaultingPyLocation location,
+    std::optional<int> regions, PyLocation &location,
     const nb::object &maybeIp) {
-  PyMlirContextRef context = location->getContext();
+  PyMlirContextRef context = location.getContext();
 
   // Class level operation construction metadata.
   // Operand and result segment specs are either none, which does no
@@ -2789,6 +2786,156 @@ class PyOpAttributeMap {
   PyOperationRef operation;
 };
 
+// see
+// https://raw.githubusercontent.com/python/pythoncapi_compat/master/pythoncapi_compat.h
+
+#ifndef _Py_CAST
+#define _Py_CAST(type, expr) ((type)(expr))
+#endif
+
+// Static inline functions should use _Py_NULL rather than using directly NULL
+// to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer,
+// _Py_NULL is defined as nullptr.
+#ifndef _Py_NULL
+#if (defined(__STDC_VERSION__) && __STDC_VERSION__ > 201710L) ||               \
+    (defined(__cplusplus) && __cplusplus >= 201103)
+#define _Py_NULL nullptr
+#else
+#define _Py_NULL NULL
+#endif
+#endif
+
+// Python 3.10.0a3
+#if PY_VERSION_HEX < 0x030A00A3
+
+// bpo-42262 added Py_XNewRef()
+#if !defined(Py_XNewRef)
+[[maybe_unused]] PyObject *_Py_XNewRef(PyObject *obj) {
+  Py_XINCREF(obj);
+  return obj;
+}
+#define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj))
+#endif
+
+// bpo-42262 added Py_NewRef()
+#if !defined(Py_NewRef)
+[[maybe_unused]] PyObject *_Py_NewRef(PyObject *obj) {
+  Py_INCREF(obj);
+  return obj;
+}
+#define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj))
+#endif
+
+#endif // Python 3.10.0a3
+
+// Python 3.9.0b1
+#if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION)
+
+// bpo-40429 added PyThreadState_GetFrame()
+PyFrameObject *PyThreadState_GetFrame(PyThreadState *tstate) {
+  assert(tstate != _Py_NULL && "expected tstate != _Py_NULL");
+  return _Py_CAST(PyFrameObject *, Py_XNewRef(tstate->frame));
+}
+
+// bpo-40421 added PyFrame_GetBack()
+PyFrameObject *PyFrame_GetBack(PyFrameObject *frame) {
+  assert(frame != _Py_NULL && "expected frame != _Py_NULL");
+  return _Py_CAST(PyFrameObject *, Py_XNewRef(frame->f_back));
+}
+
+// bpo-40421 added PyFrame_GetCode()
+PyCodeObject *PyFrame_GetCode(PyFrameObject *frame) {
+  assert(frame != _Py_NULL && "expected frame != _Py_NULL");
+  assert(frame->f_code != _Py_NULL && "expected frame->f_code != _Py_NULL");
+  return _Py_CAST(PyCodeObject *, Py_NewRef(frame->f_code));
+}
+
+#endif // Python 3.9.0b1
+
+MlirLocation tracebackToLocation(MlirContext ctx) {
+  size_t framesLimit =
+      PyGlobals::get().getTracebackLoc().locTracebackFramesLimit();
+  // Use a thread_local here to avoid requiring a large amount of space.
+  thread_local std::array<MlirLocation, PyGlobals::TracebackLoc::kMaxFrames>
+      frames;
+  size_t count = 0;
+
+  nb::gil_scoped_acquire acquire;
+  PyThreadState *tstate = PyThreadState_GET();
+  PyFrameObject *next;
+  PyFrameObject *pyFrame = PyThreadState_GetFrame(tstate);
+  // In the increment expression:
+  // 1. get the next prev frame;
+  // 2. decrement the ref count on the current frame (in order that it can get
+  //    gc'd, along with any objects in its closure and etc);
+  // 3. set current = next.
+  for (; pyFrame != nullptr && count < framesLimit;
+       next = PyFrame_GetBack(pyFrame), Py_XDECREF(pyFrame), pyFrame = next) {
+    PyCodeObject *code = PyFrame_GetCode(pyFrame);
+    auto fileNameStr =
+        nb::cast<std::string>(nb::borrow<nb::str>(code->co_filename));
+    llvm::StringRef fileName(fileNameStr);
+    if (!PyGlobals::get().getTracebackLoc().isUserTracebackFilename(fileName))
+      continue;
+
+    // co_qualname and PyCode_Addr2Location added in py3.11
+#if PY_VERSION_HEX < 0x030B00F0
+    std::string name =
+        nb::cast<std::string>(nb::borrow<nb::str>(code->co_name));
+    llvm::StringRef funcName(name);
+    int startLine = PyFrame_GetLineNumber(pyFrame);
+    MlirLocation loc =
+        mlirLocationFileLineColGet(ctx, wrap(fileName), startLine, 0);
+#else
+    std::string name =
+        nb::cast<std::string>(nb::borrow<nb::str>(code->co_qualname));
+    llvm::StringRef funcName(name);
+    int startLine, startCol, endLine, endCol;
+    int lasti = PyFrame_GetLasti(pyFrame);
+    if (!PyCode_Addr2Location(code, lasti, &startLine, &startCol, &endLine,
+                              &endCol)) {
+      throw nb::python_error();
+    }
+    MlirLocation loc = mlirLocationFileLineColRangeGet(
+        ctx, wrap(fileName), startLine, startCol, endLine, endCol);
+#endif
+
+    frames[count] = mlirLocationNameGet(ctx, wrap(funcName), loc);
+    ++count;
+  }
+  // When the loop breaks (after the last iter), current frame (if non-null)
+  // is leaked without this.
+  Py_XDECREF(pyFrame);
+
+  if (count == 0)
+    return mlirLocationUnknownGet(ctx);
+
+  MlirLocation callee = frames[0];
+  assert(!mlirLocationIsNull(callee) && "expected non-null callee location");
+  if (count == 1)
+    return callee;
+
+  MlirLocation caller = frames[count - 1];
+  assert(!mlirLocationIsNull(caller) && "expected non-null caller location");
+  for (int i = count - 2; i >= 1; i--)
+    caller = mlirLocationCallSiteGet(frames[i], caller);
+
+  return mlirLocationCallSiteGet(callee, caller);
+}
+
+PyLocation
+maybeGetTracebackLocation(const std::optional<PyLocation> &location) {
+  if (location.has_value())
+    return location.value();
+  if (!PyGlobals::get().getTracebackLoc().locTracebacksEnabled())
+    return DefaultingPyLocation::resolve();
+
+  PyMlirContext &ctx = DefaultingPyMlirContext::resolve();
+  MlirLocation mlirLoc = tracebackToLocation(ctx.get());
+  PyMlirContextRef ref = PyMlirContext::forContext(ctx.get());
+  return {ref, mlirLoc};
+}
+
 } // namespace
 
 //------------------------------------------------------------------------------
@@ -3052,10 +3199,10 @@ void mlir::python::populateIRCore(nb::module_ &m) {
       .def("__eq__", [](PyLocation &self, nb::object other) { return false; })
       .def_prop_ro_static(
           "current",
-          [](nb::object & /*class*/) {
+          [](nb::object & /*class*/) -> std::optional<PyLocation *> {
             auto *loc = PyThreadContextEntry::getDefaultLocation();
             if (!loc)
-              throw nb::value_error("No current Location");
+              return std::nullopt;
             return loc;
           },
           "Gets the Location bound to the current thread or raises ValueError")
@@ -3240,8 +3387,9 @@ void mlir::python::populateIRCore(nb::module_ &m) {
           kModuleParseDocstring)
       .def_static(
           "create",
-          [](DefaultingPyLocation loc) {
-            MlirModule module = mlirModuleCreateEmpty(loc);
+          [](const std::optional<PyLocation> &loc) {
+            PyLocation pyLoc = maybeGetTracebackLocation(loc);
+            MlirModule module = mlirModuleCreateEmpty(pyLoc.get());
             return PyModule::forModule(module).releaseObject();
           },
           nb::arg("loc").none() = nb::none(), "Creates an empty module")
@@ -3442,6 +3590,14 @@ void mlir::python::populateIRCore(nb::module_ &m) {
             return operation.createOpView();
           },
           "Detaches the operation from its parent block.")
+      .def_prop_ro(
+          "attached",
+          [](PyOperationBase &self) {
+            PyOperation &operation = self.getOperation();
+            operation.checkValid();
+            return operation.isAttached();
+          },
+          "Reports if the operation is attached to its parent block.")
       .def("erase", [](PyOperationBase &self) { self.getOperation().erase(); })
       .def("walk", &PyOperationBase::walk, nb::arg("callback"),
            nb::arg("walk_order") = MlirWalkPostOrder);
@@ -3454,8 +3610,8 @@ void mlir::python::populateIRCore(nb::module_ &m) {
              std::optional<std::vector<PyValue *>> operands,
              std::optional<nb::dict> attributes,
              std::optional<std::vector<PyBlock *>> successors, int regions,
-             DefaultingPyLocation location, const nb::object &maybeIp,
-             bool inferType) {
+             const std::optional<PyLocation> &location,
+             const nb::object &maybeIp, bool inferType) {
             // Unpack/validate operands.
             llvm::SmallVector<MlirValue, 4> mlirOperands;
             if (operands) {
@@ -3467,8 +3623,9 @@ void mlir::python::populateIRCore(nb::module_ &m) {
               }
             }
 
+            PyLocation pyLoc = maybeGetTracebackLocation(location);
             return PyOperation::create(name, results, mlirOperands, attributes,
-                                       successors, regions, location, maybeIp,
+                                       successors, regions, pyLoc, maybeIp,
                                        inferType);
           },
           nb::arg("name"), nb::arg("results").none() = nb::none(),
@@ -3512,12 +3669,14 @@ void mlir::python::populateIRCore(nb::module_ &m) {
                  std::optional<nb::list> resultTypeList, nb::list operandList,
                  std::optional<nb::dict> attributes,
                  std::optional<std::vector<PyBlock *>> successors,
-                 std::optional<int> regions, DefaultingPyLocation location,
+                 std::optional<int> regions,
+                 const std::optional<PyLocation> &location,
                  const nb::object &maybeIp) {
+                PyLocation pyLoc = maybeGetTracebackLocation(location);
                 new (self) PyOpView(PyOpView::buildGeneric(
                     name, opRegionSpec, operandSegmentSpecObj,
                     resultSegmentSpecObj, resultTypeList, operandList,
-                    attributes, successors, regions, location, maybeIp));
+                    attributes, successors, regions, pyLoc, maybeIp));
               },
               nb::arg("name"), nb::arg("opRegionSpec"),
               nb::arg("operandSegmentSpecObj").none() = nb::none(),
@@ -3551,17 +3710,18 @@ void mlir::python::populateIRCore(nb::module_ &m) {
       [](nb::handle cls, std::optional<nb::list> resultTypeList,
          nb::list operandList, std::optional<nb::dict> attributes,
          std::optional<std::vector<PyBlock *>> successors,
-         std::optional<int> regions, DefaultingPyLocation location,
+         std::optional<int> regions, std::optional<PyLocation> location,
          const nb::object &maybeIp) {
         std::string name = nb::cast<std::string>(cls.attr("OPERATION_NAME"));
         std::tuple<int, bool> opRegionSpec =
             nb::cast<std::tuple<int, bool>>(cls.attr("_ODS_REGIONS"));
         nb::object operandSegmentSpec = cls.attr("_ODS_OPERAND_SEGMENTS");
         nb::object resultSegmentSpec = cls.attr("_ODS_RESULT_SEGMENTS");
+        PyLocation pyLoc = maybeGetTracebackLocation(location);
         return PyOpView::buildGeneric(name, opRegionSpec, operandSegmentSpec,
                                       resultSegmentSpec, resultTypeList,
                                       operandList, attributes, successors,
-                                      regions, location, maybeIp);
+                                      regions, pyLoc, maybeIp);
       },
       nb::arg("cls"), nb::arg("results").none() = nb::none(),
       nb::arg("operands").none() = nb::none(),
diff --git a/mlir/lib/Bindings/Python/IRModule.cpp b/mlir/lib/Bindings/Python/IRModule.cpp
index e600f1bbd4493..0de2f1711829b 100644
--- a/mlir/lib/Bindings/Python/IRModule.cpp
+++ b/mlir/lib/Bindings/Python/IRModule.cpp
@@ -13,9 +13,9 @@
 
 #include "Globals.h"
 #include "NanobindUtils.h"
+#include "mlir-c/Bindings/Python/Interop.h"
 #include "mlir-c/Support.h"
 #include "mlir/Bindings/Python/Nanobind.h"
-#include "mlir-c/Bindings/Python/Interop.h" // This is expected after nanobind.
 
 namespace nb = nanobind;
 using namespace mlir;
@@ -197,3 +197,71 @@ PyGlobals::lookupOperationClass(llvm::StringRef operationName) {
   // Not found and loading did not yield a registration.
   return std::nullopt;
 }
+
+bool PyGlobals::TracebackLoc::locTracebacksEnabled() {
+  nanobind::ft_lock_guard lock(mutex);
+  return locTracebackEnabled_;
+}
+
+void PyGlobals::TracebackLoc::setLocTracebacksEnabled(bool value) {
+  nanobind::ft_lock_guard lock(mutex);
+  locTracebackEnabled_ = value;
+}
+
+size_t PyGlobals::TracebackLoc::locTracebackFramesLimit() {
+  nanobind::ft_lock_guard lock(mutex);
+  return locTracebackFramesLimit_;
+}
+
+void PyGlobals::TracebackLoc::setLocTracebackFramesLimit(size_t value) {
+  nanobind::ft_lock_guard lock(mutex);
+  locTracebackFramesLimit_ = std::min(value, kMaxFrames);
+}
+
+void PyGlobals::TracebackLoc::registerTracebackFileInclusion(
+    const std::string &file) {
+  nanobind::ft_lock_guard lock(mutex);
+  auto reg = "^" + llvm::Regex::escape(file);
+  if (userTracebackIncludeFiles.insert(reg).second)
+    rebuildUserTracebackIncludeRegex = true;
+  if (userTracebackExcludeFiles.count(reg)) {
+    if (userTracebackExcludeFiles.erase(reg))
+      rebuildUserTracebackExcludeRegex = true;
+  }
+}
+
+void PyGlobals::TracebackLoc::registerTracebackFileExclusion(
+    const std::string &file) {
+  nanobind::ft_lock_guard lock(mutex);
+  auto reg = "^" + llvm::Regex::escape(file);
+  if (userTracebackExcludeFiles.insert(reg).second)
+    rebuildUserTracebackExcludeRegex = true;
+  if (userTracebackIncludeFiles.count(reg)) {
+    if (userTracebackIncludeFiles.erase(reg))
+      rebuildUserTracebackIncludeRegex = true;
+  }
+}
+
+bool PyGlobals::TracebackLoc::isUserTracebackFilename(
+    const llvm::StringRef file) {
+  nanobind::ft_lock_guard lock(mutex);
+  if (rebuildUserTracebackIncludeRegex) {
+    userTracebackIncludeRegex.assign(
+        llvm::join(userTracebackIncludeFiles, "|"));
+    rebuildUserTracebackIncludeRegex = false;
+    isUserTracebackFilenameCache.clear();
+  }
+  if (rebuildUserTracebackExcludeRegex) {
+    userTracebackExcludeRegex.assign(
+        llvm::join(userTracebackExcludeFiles, "|"));
+    rebuildUserTracebackExcludeRegex = false;
+    isUserTracebackFilenameCache.clear();
+  }
+  if (!isUserTracebackFilenameCache.contains(file)) {
+    std::string fileStr = file.str();
+    bool include = std::regex_search(fileStr, userTracebackIncludeRegex);
+    bool exclude = std::regex_search(fileStr, userTracebackExcludeRegex);
+    isUserTracebackFilenameCache[file] = include || !exclude;
+  }
+  return isUserTracebackFilenameCache[file];
+}
diff --git a/mlir/lib/Bindings/Python/IRModule.h b/mlir/lib/Bindings/Python/IRModule.h
index 9c22dea157c06..fa16ae3ce3294 100644
--- a/mlir/lib/Bindings/Python/IRModule.h
+++ b/mlir/lib/Bindings/Python/IRModule.h
@@ -192,16 +192,6 @@ class PyMlirContext {
   PyMlirContext(const PyMlirContext &) = delete;
   PyMlirContext(PyMlirContext &&) = delete;
 
-  /// For the case of a python __init__ (nanobind::init) method, pybind11 is
-  /// quite strict about needing to return a pointer that is not yet associated
-  /// to an nanobind::object. Since the forContext() method acts like a pool,
-  /// possibly returning a recycled context, it does not satisfy this need. The
-  /// usual way in python to accomplish such a thing is to override __new__, but
-  /// that is also not supported by pybind11. Instead, we use this entry
-  /// point which always constructs a fresh context (which cannot alias an
-  /// existing one because it is fresh).
-  static PyMlirContext *createNewContextForInit();
-
   /// Returns a context reference for the singleton PyMlirContext wrapper for
   /// the given context.
   static PyMlirContextRef forContext(MlirContext context);
@@ -722,8 +712,7 @@ class PyOperation : public PyOperationBase, public BaseContextObject {
          llvm::ArrayRef<MlirValue> operands,
          std::optional<nanobind::dict> attributes,
          std::optional<std::vector<PyBlock *>> successors, int regions,
-         DefaultingPyLocation location, const nanobind::object &ip,
-         bool inferType);
+         PyLocation &location, const nanobind::object &ip, bool inferType);
 
   /// Creates an OpView suitable for this operation.
   nanobind::object createOpView();
@@ -781,7 +770,7 @@ class PyOpView : public PyOperationBase {
                nanobind::list operandList,
                std::optional<nanobind::dict> attributes,
                std::optional<std::vector<PyBlock *>> successors,
-               std::optional<int> regions, DefaultingPyLocation location,
+               std::optional<int> regions, PyLocation &location,
                const nanobind::object &maybeIp);
 
   /// Construct an instance of a class deriving from OpView, bypassing its
diff --git a/mlir/lib/Bindings/Python/MainModule.cpp b/mlir/lib/Bindings/Python/MainModule.cpp
index 6f49431006605..278847e7ac7f5 100644
--- a/mlir/lib/Bindings/Python/MainModule.cpp
+++ b/mlir/lib/Bindings/Python/MainModule.cpp
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 #include "Globals.h"
 #include "IRModule.h"
 #include "NanobindUtils.h"
@@ -44,7 +43,27 @@ NB_MODULE(_mlir, m) {
       .def("_register_operation_impl", &PyGlobals::registerOperationImpl,
            "operation_name"_a, "operation_class"_a, nb::kw_only(),
            "replace"_a = false,
-           "Testing hook for directly registering an operation");
+           "Testing hook for directly registering an operation")
+      .def("loc_tracebacks_enabled",
+           [](PyGlobals &self) {
+             return self.getTracebackLoc().locTracebacksEnabled();
+           })
+      .def("set_loc_tracebacks_enabled",
+           [](PyGlobals &self, bool enabled) {
+             self.getTracebackLoc().setLocTracebacksEnabled(enabled);
+           })
+      .def("set_loc_tracebacks_frame_limit",
+           [](PyGlobals &self, int n) {
+             self.getTracebackLoc().setLocTracebackFramesLimit(n);
+           })
+      .def("register_traceback_file_inclusion",
+           [](PyGlobals &self, const std::string &filename) {
+             self.getTracebackLoc().registerTracebackFileInclusion(filename);
+           })
+      .def("register_traceback_file_exclusion",
+           [](PyGlobals &self, const std::string &filename) {
+             self.getTracebackLoc().registerTracebackFileExclusion(filename);
+           });
 
   // Aside from making the globals accessible to python, having python manage
   // it is necessary to make sure it is destroyed (and releases its python
diff --git a/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp b/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
index 6ebeac5ae69f4..eacb936c763b7 100644
--- a/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
+++ b/mlir/lib/Bytecode/Writer/BytecodeWriter.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
@@ -150,8 +151,7 @@ class EncodingEmitter {
 
   /// Backpatch a byte in the result buffer at the given offset.
   void patchByte(uint64_t offset, uint8_t value, StringLiteral desc) {
-    LLVM_DEBUG(llvm::dbgs() << "patchByte(" << offset << ',' << uint64_t(value)
-                            << ")\t" << desc << '\n');
+    LDBG() << "patchByte(" << offset << ',' << uint64_t(value) << ")\t" << desc;
     assert(offset < size() && offset >= prevResultSize &&
            "cannot patch previously emitted data");
     currentResult[offset - prevResultSize] = value;
@@ -160,8 +160,7 @@ class EncodingEmitter {
   /// Emit the provided blob of data, which is owned by the caller and is
   /// guaranteed to not die before the end of the bytecode process.
   void emitOwnedBlob(ArrayRef<uint8_t> data, StringLiteral desc) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "emitOwnedBlob(" << data.size() << "b)\t" << desc << '\n');
+    LDBG() << "emitOwnedBlob(" << data.size() << "b)\t" << desc;
     // Push the current buffer before adding the provided data.
     appendResult(std::move(currentResult));
     appendOwnedResult(data);
@@ -209,15 +208,13 @@ class EncodingEmitter {
   /// Emit a single byte.
   template <typename T>
   void emitByte(T byte, StringLiteral desc) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "emitByte(" << uint64_t(byte) << ")\t" << desc << '\n');
+    LDBG() << "emitByte(" << uint64_t(byte) << ")\t" << desc;
     currentResult.push_back(static_cast<uint8_t>(byte));
   }
 
   /// Emit a range of bytes.
   void emitBytes(ArrayRef<uint8_t> bytes, StringLiteral desc) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "emitBytes(" << bytes.size() << "b)\t" << desc << '\n');
+    LDBG() << "emitBytes(" << bytes.size() << "b)\t" << desc;
     llvm::append_range(currentResult, bytes);
   }
 
@@ -229,7 +226,7 @@ class EncodingEmitter {
   /// additional bytes, provide the value of the integer encoded in
   /// little-endian order.
   void emitVarInt(uint64_t value, StringLiteral desc) {
-    LLVM_DEBUG(llvm::dbgs() << "emitVarInt(" << value << ")\t" << desc << '\n');
+    LDBG() << "emitVarInt(" << value << ")\t" << desc;
 
     // In the most common case, the value can be represented in a single byte.
     // Given how hot this case is, explicitly handle that here.
diff --git a/mlir/lib/CAPI/ExecutionEngine/ExecutionEngine.cpp b/mlir/lib/CAPI/ExecutionEngine/ExecutionEngine.cpp
index 306cebd236be9..2dbb993b1640f 100644
--- a/mlir/lib/CAPI/ExecutionEngine/ExecutionEngine.cpp
+++ b/mlir/lib/CAPI/ExecutionEngine/ExecutionEngine.cpp
@@ -68,6 +68,10 @@ mlirExecutionEngineCreate(MlirModule op, int optLevel, int numPaths,
   return wrap(jitOrError->release());
 }
 
+extern "C" void mlirExecutionEngineInitialize(MlirExecutionEngine jit) {
+  unwrap(jit)->initialize();
+}
+
 extern "C" void mlirExecutionEngineDestroy(MlirExecutionEngine jit) {
   delete (unwrap(jit));
 }
@@ -106,9 +110,8 @@ extern "C" void mlirExecutionEngineRegisterSymbol(MlirExecutionEngine jit,
                                                   void *sym) {
   unwrap(jit)->registerSymbols([&](llvm::orc::MangleAndInterner interner) {
     llvm::orc::SymbolMap symbolMap;
-    symbolMap[interner(unwrap(name))] =
-        { llvm::orc::ExecutorAddr::fromPtr(sym),
-          llvm::JITSymbolFlags::Exported };
+    symbolMap[interner(unwrap(name))] = {llvm::orc::ExecutorAddr::fromPtr(sym),
+                                         llvm::JITSymbolFlags::Exported};
     return symbolMap;
   });
 }
diff --git a/mlir/lib/CMakeLists.txt b/mlir/lib/CMakeLists.txt
index 191b5ab6a6f56..91ed05f6548d7 100644
--- a/mlir/lib/CMakeLists.txt
+++ b/mlir/lib/CMakeLists.txt
@@ -13,6 +13,7 @@ add_subdirectory(Parser)
 add_subdirectory(Pass)
 add_subdirectory(Query)
 add_subdirectory(Reducer)
+add_subdirectory(Remark)
 add_subdirectory(Rewrite)
 add_subdirectory(Support)
 add_subdirectory(TableGen)
diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 64720bfe6cf50..203790ed95153 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
 #include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Dialect/LLVMIR/ROCDLDialect.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/TypeUtilities.h"
@@ -1876,6 +1877,54 @@ struct AMDGPUSwizzleBitModeLowering
   }
 };
 
+struct AMDGPUPermlaneLowering : public ConvertOpToLLVMPattern<PermlaneSwapOp> {
+  using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+
+  AMDGPUPermlaneLowering(const LLVMTypeConverter &converter, Chipset chipset)
+      : ConvertOpToLLVMPattern<PermlaneSwapOp>(converter), chipset(chipset) {}
+  Chipset chipset;
+
+  LogicalResult
+  matchAndRewrite(PermlaneSwapOp op, OpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    if (chipset < kGfx950)
+      return op->emitOpError("permlane_swap is only supported on gfx950+");
+
+    Location loc = op.getLoc();
+    Type i32 = rewriter.getI32Type();
+    Value src = adaptor.getSrc();
+    unsigned rowLength = op.getRowLength();
+    bool fi = op.getFetchInactive();
+    bool boundctrl = op.getBoundCtrl();
+
+    SmallVector<Value> decomposed =
+        LLVM::decomposeValue(rewriter, loc, src, i32);
+
+    SmallVector<Value> permuted;
+    for (Value v : decomposed) {
+      Value res;
+      Type i32pair = LLVM::LLVMStructType::getLiteral(
+          rewriter.getContext(), {v.getType(), v.getType()});
+
+      if (rowLength == 16)
+        res = ROCDL::Permlane16SwapOp::create(rewriter, loc, i32pair, v, v, fi,
+                                              boundctrl);
+      else if (rowLength == 32)
+        res = ROCDL::Permlane32SwapOp::create(rewriter, loc, i32pair, v, v, fi,
+                                              boundctrl);
+      else
+        llvm_unreachable("unsupported row length");
+
+      Value vdstNew = LLVM::ExtractValueOp::create(rewriter, loc, res, {0});
+      permuted.emplace_back(vdstNew);
+    }
+
+    Value result = LLVM::composeValue(rewriter, loc, permuted, src.getType());
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
 struct ConvertAMDGPUToROCDLPass
     : public impl::ConvertAMDGPUToROCDLPassBase<ConvertAMDGPUToROCDLPass> {
   using Base::Base;
@@ -1944,6 +1993,6 @@ void mlir::populateAMDGPUToROCDLConversionPatterns(LLVMTypeConverter &converter,
            WMMAOpLowering, ExtPackedFp8OpLowering, ScaledExtPackedOpLowering,
            PackedScaledTruncOpLowering, PackedTrunc2xFp8OpLowering,
            PackedStochRoundFp8OpLowering, GatherToLDSOpLowering,
-           TransposeLoadOpLowering>(converter, chipset);
+           TransposeLoadOpLowering, AMDGPUPermlaneLowering>(converter, chipset);
   patterns.add<AMDGPUSwizzleBitModeLowering>(converter);
 }
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index 18e857c81af8d..cb0c829719565 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -238,6 +238,16 @@ struct CmpFOpLowering : public ConvertOpToLLVMPattern<arith::CmpFOp> {
                   ConversionPatternRewriter &rewriter) const override;
 };
 
+struct SelectOpOneToNLowering : public ConvertOpToLLVMPattern<arith::SelectOp> {
+  using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
+  using Adaptor =
+      typename ConvertOpToLLVMPattern<arith::SelectOp>::OneToNOpAdaptor;
+
+  LogicalResult
+  matchAndRewrite(arith::SelectOp op, Adaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override;
+};
+
 } // namespace
 
 //===----------------------------------------------------------------------===//
@@ -479,6 +489,32 @@ CmpFOpLowering::matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
       rewriter);
 }
 
+//===----------------------------------------------------------------------===//
+// SelectOpOneToNLowering
+//===----------------------------------------------------------------------===//
+
+/// Pattern for arith.select where the true/false values lower to multiple
+/// SSA values (1:N conversion). This pattern generates multiple arith.select
+/// than can be lowered by the 1:1 arith.select pattern.
+LogicalResult SelectOpOneToNLowering::matchAndRewrite(
+    arith::SelectOp op, Adaptor adaptor,
+    ConversionPatternRewriter &rewriter) const {
+  // In case of a 1:1 conversion, the 1:1 pattern will match.
+  if (llvm::hasSingleElement(adaptor.getTrueValue()))
+    return rewriter.notifyMatchFailure(
+        op, "not a 1:N conversion, 1:1 pattern will match");
+  if (!op.getCondition().getType().isInteger(1))
+    return rewriter.notifyMatchFailure(op,
+                                       "non-i1 conditions are not supported");
+  SmallVector<Value> results;
+  for (auto [trueValue, falseValue] :
+       llvm::zip_equal(adaptor.getTrueValue(), adaptor.getFalseValue()))
+    results.push_back(arith::SelectOp::create(
+        rewriter, op.getLoc(), op.getCondition(), trueValue, falseValue));
+  rewriter.replaceOpWithMultiple(op, {results});
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Pass Definition
 //===----------------------------------------------------------------------===//
@@ -587,6 +623,7 @@ void mlir::arith::populateArithToLLVMConversionPatterns(
     RemSIOpLowering,
     RemUIOpLowering,
     SelectOpLowering,
+    SelectOpOneToNLowering,
     ShLIOpLowering,
     ShRSIOpLowering,
     ShRUIOpLowering,
diff --git a/mlir/lib/Conversion/CMakeLists.txt b/mlir/lib/Conversion/CMakeLists.txt
index 785cb8293810c..171f7169fd41d 100644
--- a/mlir/lib/Conversion/CMakeLists.txt
+++ b/mlir/lib/Conversion/CMakeLists.txt
@@ -68,6 +68,7 @@ add_subdirectory(TosaToSCF)
 add_subdirectory(TosaToTensor)
 add_subdirectory(UBToLLVM)
 add_subdirectory(UBToSPIRV)
+add_subdirectory(VectorToAMX)
 add_subdirectory(VectorToArmSME)
 add_subdirectory(VectorToGPU)
 add_subdirectory(VectorToLLVM)
diff --git a/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
index 35ad99c7791db..7a3a7fdb73e5e 100644
--- a/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
+++ b/mlir/lib/Conversion/ComplexToROCDLLibraryCalls/ComplexToROCDLLibraryCalls.cpp
@@ -56,22 +56,30 @@ struct ComplexOpToROCDLLibraryCalls : public OpRewritePattern<Op> {
 private:
   std::string funcName;
 };
+
+// Rewrite complex.pow(z, w) -> complex.exp(w * complex.log(z))
+struct PowOpToROCDLLibraryCalls : public OpRewritePattern<complex::PowOp> {
+  using OpRewritePattern<complex::PowOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(complex::PowOp op,
+                                PatternRewriter &rewriter) const final {
+    Location loc = op.getLoc();
+    Value logBase = rewriter.create<complex::LogOp>(loc, op.getLhs());
+    Value mul = rewriter.create<complex::MulOp>(loc, op.getRhs(), logBase);
+    Value exp = rewriter.create<complex::ExpOp>(loc, mul);
+    rewriter.replaceOp(op, exp);
+    return success();
+  }
+};
 } // namespace
 
 void mlir::populateComplexToROCDLLibraryCallsConversionPatterns(
     RewritePatternSet &patterns) {
+  patterns.add<PowOpToROCDLLibraryCalls>(patterns.getContext());
   patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float32Type>>(
       patterns.getContext(), "__ocml_cabs_f32");
   patterns.add<ComplexOpToROCDLLibraryCalls<complex::AbsOp, Float64Type>>(
       patterns.getContext(), "__ocml_cabs_f64");
-  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AngleOp, Float32Type>>(
-      patterns.getContext(), "__ocml_carg_f32");
-  patterns.add<ComplexOpToROCDLLibraryCalls<complex::AngleOp, Float64Type>>(
-      patterns.getContext(), "__ocml_carg_f64");
-  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ConjOp, Float32Type>>(
-      patterns.getContext(), "__ocml_conj_f32");
-  patterns.add<ComplexOpToROCDLLibraryCalls<complex::ConjOp, Float64Type>>(
-      patterns.getContext(), "__ocml_conj_f64");
   patterns.add<ComplexOpToROCDLLibraryCalls<complex::CosOp, Float32Type>>(
       patterns.getContext(), "__ocml_ccos_f32");
   patterns.add<ComplexOpToROCDLLibraryCalls<complex::CosOp, Float64Type>>(
@@ -84,10 +92,6 @@ void mlir::populateComplexToROCDLLibraryCallsConversionPatterns(
       patterns.getContext(), "__ocml_clog_f32");
   patterns.add<ComplexOpToROCDLLibraryCalls<complex::LogOp, Float64Type>>(
       patterns.getContext(), "__ocml_clog_f64");
-  patterns.add<ComplexOpToROCDLLibraryCalls<complex::PowOp, Float32Type>>(
-      patterns.getContext(), "__ocml_cpow_f32");
-  patterns.add<ComplexOpToROCDLLibraryCalls<complex::PowOp, Float64Type>>(
-      patterns.getContext(), "__ocml_cpow_f64");
   patterns.add<ComplexOpToROCDLLibraryCalls<complex::SinOp, Float32Type>>(
       patterns.getContext(), "__ocml_csin_f32");
   patterns.add<ComplexOpToROCDLLibraryCalls<complex::SinOp, Float64Type>>(
@@ -122,10 +126,10 @@ void ConvertComplexToROCDLLibraryCallsPass::runOnOperation() {
 
   ConversionTarget target(getContext());
   target.addLegalDialect<func::FuncDialect>();
-  target.addIllegalOp<complex::AbsOp, complex::AngleOp, complex::ConjOp,
-                      complex::CosOp, complex::ExpOp, complex::LogOp,
-                      complex::PowOp, complex::SinOp, complex::SqrtOp,
-                      complex::TanOp, complex::TanhOp>();
+  target.addLegalOp<complex::MulOp>();
+  target.addIllegalOp<complex::AbsOp, complex::CosOp, complex::ExpOp,
+                      complex::LogOp, complex::PowOp, complex::SinOp,
+                      complex::SqrtOp, complex::TanOp, complex::TanhOp>();
   if (failed(applyPartialConversion(op, target, std::move(patterns))))
     signalPassFailure();
 }
diff --git a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
index ff6d369176393..798d8b04eed76 100644
--- a/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
+++ b/mlir/lib/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.cpp
@@ -125,22 +125,33 @@ static FailureOr<Block *> getConvertedBlock(ConversionPatternRewriter &rewriter,
   return rewriter.applySignatureConversion(block, *conversion, converter);
 }
 
+/// Flatten the given value ranges into a single vector of values.
+static SmallVector<Value> flattenValues(ArrayRef<ValueRange> values) {
+  SmallVector<Value> result;
+  for (const ValueRange &vals : values)
+    llvm::append_range(result, vals);
+  return result;
+}
+
 /// Convert the destination block signature (if necessary) and lower the branch
 /// op to llvm.br.
 struct BranchOpLowering : public ConvertOpToLLVMPattern<cf::BranchOp> {
   using ConvertOpToLLVMPattern<cf::BranchOp>::ConvertOpToLLVMPattern;
+  using Adaptor =
+      typename ConvertOpToLLVMPattern<cf::BranchOp>::OneToNOpAdaptor;
 
   LogicalResult
-  matchAndRewrite(cf::BranchOp op, typename cf::BranchOp::Adaptor adaptor,
+  matchAndRewrite(cf::BranchOp op, Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
+    SmallVector<Value> flattenedAdaptor = flattenValues(adaptor.getOperands());
     FailureOr<Block *> convertedBlock =
         getConvertedBlock(rewriter, getTypeConverter(), op, op.getSuccessor(),
-                          TypeRange(adaptor.getOperands()));
+                          TypeRange(ValueRange(flattenedAdaptor)));
     if (failed(convertedBlock))
       return failure();
     DictionaryAttr attrs = op->getAttrDictionary();
     Operation *newOp = rewriter.replaceOpWithNewOp<LLVM::BrOp>(
-        op, adaptor.getOperands(), *convertedBlock);
+        op, flattenedAdaptor, *convertedBlock);
     // TODO: We should not just forward all attributes like that. But there are
     // existing Flang tests that depend on this behavior.
     newOp->setAttrs(attrs);
@@ -152,29 +163,37 @@ struct BranchOpLowering : public ConvertOpToLLVMPattern<cf::BranchOp> {
 /// branch op to llvm.cond_br.
 struct CondBranchOpLowering : public ConvertOpToLLVMPattern<cf::CondBranchOp> {
   using ConvertOpToLLVMPattern<cf::CondBranchOp>::ConvertOpToLLVMPattern;
+  using Adaptor =
+      typename ConvertOpToLLVMPattern<cf::CondBranchOp>::OneToNOpAdaptor;
 
   LogicalResult
-  matchAndRewrite(cf::CondBranchOp op,
-                  typename cf::CondBranchOp::Adaptor adaptor,
+  matchAndRewrite(cf::CondBranchOp op, Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
+    SmallVector<Value> flattenedAdaptorTrue =
+        flattenValues(adaptor.getTrueDestOperands());
+    SmallVector<Value> flattenedAdaptorFalse =
+        flattenValues(adaptor.getFalseDestOperands());
+    if (!llvm::hasSingleElement(adaptor.getCondition()))
+      return rewriter.notifyMatchFailure(op,
+                                         "expected single element condition");
     FailureOr<Block *> convertedTrueBlock =
         getConvertedBlock(rewriter, getTypeConverter(), op, op.getTrueDest(),
-                          TypeRange(adaptor.getTrueDestOperands()));
+                          TypeRange(ValueRange(flattenedAdaptorTrue)));
     if (failed(convertedTrueBlock))
       return failure();
     FailureOr<Block *> convertedFalseBlock =
         getConvertedBlock(rewriter, getTypeConverter(), op, op.getFalseDest(),
-                          TypeRange(adaptor.getFalseDestOperands()));
+                          TypeRange(ValueRange(flattenedAdaptorFalse)));
     if (failed(convertedFalseBlock))
       return failure();
-    DictionaryAttr attrs = op->getAttrDictionary();
+    DictionaryAttr attrs = op->getDiscardableAttrDictionary();
     auto newOp = rewriter.replaceOpWithNewOp<LLVM::CondBrOp>(
-        op, adaptor.getCondition(), adaptor.getTrueDestOperands(),
-        adaptor.getFalseDestOperands(), op.getBranchWeightsAttr(),
+        op, llvm::getSingleElement(adaptor.getCondition()),
+        flattenedAdaptorTrue, flattenedAdaptorFalse, op.getBranchWeightsAttr(),
         *convertedTrueBlock, *convertedFalseBlock);
     // TODO: We should not just forward all attributes like that. But there are
     // existing Flang tests that depend on this behavior.
-    newOp->setAttrs(attrs);
+    newOp->setDiscardableAttrs(attrs);
     return success();
   }
 };
diff --git a/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp b/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp
index ed5d6d4a7fe40..764ad2e766c73 100644
--- a/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp
+++ b/mlir/lib/Conversion/ConvertToLLVM/ConvertToLLVMPass.cpp
@@ -14,6 +14,7 @@
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Rewrite/FrozenRewritePatternSet.h"
 #include "mlir/Transforms/DialectConversion.h"
+#include "llvm/Support/DebugLog.h"
 #include <memory>
 
 #define DEBUG_TYPE "convert-to-llvm"
@@ -31,7 +32,8 @@ namespace {
 class ConvertToLLVMPassInterface {
 public:
   ConvertToLLVMPassInterface(MLIRContext *context,
-                             ArrayRef<std::string> filterDialects);
+                             ArrayRef<std::string> filterDialects,
+                             bool allowPatternRollback = true);
   virtual ~ConvertToLLVMPassInterface() = default;
 
   /// Get the dependent dialects used by `convert-to-llvm`.
@@ -60,6 +62,9 @@ class ConvertToLLVMPassInterface {
   MLIRContext *context;
   /// List of dialects names to use as filters.
   ArrayRef<std::string> filterDialects;
+  /// An experimental flag to disallow pattern rollback. This is more efficient
+  /// but not supported by all lowering patterns.
+  bool allowPatternRollback;
 };
 
 /// This DialectExtension can be attached to the context, which will invoke the
@@ -75,13 +80,13 @@ class LoadDependentDialectExtension : public DialectExtensionBase {
 
   void apply(MLIRContext *context,
              MutableArrayRef<Dialect *> dialects) const final {
-    LLVM_DEBUG(llvm::dbgs() << "Convert to LLVM extension load\n");
+    LDBG() << "Convert to LLVM extension load";
     for (Dialect *dialect : dialects) {
       auto *iface = dyn_cast<ConvertToLLVMPatternInterface>(dialect);
       if (!iface)
         continue;
-      LLVM_DEBUG(llvm::dbgs() << "Convert to LLVM found dialect interface for "
-                              << dialect->getNamespace() << "\n");
+      LDBG() << "Convert to LLVM found dialect interface for "
+             << dialect->getNamespace();
       iface->loadDependentDialects(context);
     }
   }
@@ -128,7 +133,9 @@ struct StaticConvertToLLVM : public ConvertToLLVMPassInterface {
 
   /// Apply the conversion driver.
   LogicalResult transform(Operation *op, AnalysisManager manager) const final {
-    if (failed(applyPartialConversion(op, *target, *patterns)))
+    ConversionConfig config;
+    config.allowPatternRollback = allowPatternRollback;
+    if (failed(applyPartialConversion(op, *target, *patterns, config)))
       return failure();
     return success();
   }
@@ -179,7 +186,9 @@ struct DynamicConvertToLLVM : public ConvertToLLVMPassInterface {
                                               patterns);
 
     // Apply the conversion.
-    if (failed(applyPartialConversion(op, target, std::move(patterns))))
+    ConversionConfig config;
+    config.allowPatternRollback = allowPatternRollback;
+    if (failed(applyPartialConversion(op, target, std::move(patterns), config)))
       return failure();
     return success();
   }
@@ -206,9 +215,11 @@ class ConvertToLLVMPass
     std::shared_ptr<ConvertToLLVMPassInterface> impl;
     // Choose the pass implementation.
     if (useDynamic)
-      impl = std::make_shared<DynamicConvertToLLVM>(context, filterDialects);
+      impl = std::make_shared<DynamicConvertToLLVM>(context, filterDialects,
+                                                    allowPatternRollback);
     else
-      impl = std::make_shared<StaticConvertToLLVM>(context, filterDialects);
+      impl = std::make_shared<StaticConvertToLLVM>(context, filterDialects,
+                                                   allowPatternRollback);
     if (failed(impl->initialize()))
       return failure();
     this->impl = impl;
@@ -228,8 +239,10 @@ class ConvertToLLVMPass
 //===----------------------------------------------------------------------===//
 
 ConvertToLLVMPassInterface::ConvertToLLVMPassInterface(
-    MLIRContext *context, ArrayRef<std::string> filterDialects)
-    : context(context), filterDialects(filterDialects) {}
+    MLIRContext *context, ArrayRef<std::string> filterDialects,
+    bool allowPatternRollback)
+    : context(context), filterDialects(filterDialects),
+      allowPatternRollback(allowPatternRollback) {}
 
 void ConvertToLLVMPassInterface::getDependentDialects(
     DialectRegistry &registry) {
diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 67bb1c14c99a2..42c76ed475b4c 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -527,19 +527,21 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
   using ConvertOpToLLVMPattern<CallOpType>::ConvertOpToLLVMPattern;
   using Super = CallOpInterfaceLowering<CallOpType>;
   using Base = ConvertOpToLLVMPattern<CallOpType>;
+  using Adaptor = typename ConvertOpToLLVMPattern<CallOpType>::OneToNOpAdaptor;
 
-  LogicalResult matchAndRewriteImpl(CallOpType callOp,
-                                    typename CallOpType::Adaptor adaptor,
+  LogicalResult matchAndRewriteImpl(CallOpType callOp, Adaptor adaptor,
                                     ConversionPatternRewriter &rewriter,
                                     bool useBarePtrCallConv = false) const {
     // Pack the result types into a struct.
     Type packedResult = nullptr;
+    SmallVector<SmallVector<Type>> groupedResultTypes;
     unsigned numResults = callOp.getNumResults();
     auto resultTypes = llvm::to_vector<4>(callOp.getResultTypes());
-
+    int64_t numConvertedTypes = 0;
     if (numResults != 0) {
       if (!(packedResult = this->getTypeConverter()->packFunctionResults(
-                resultTypes, useBarePtrCallConv)))
+                resultTypes, useBarePtrCallConv, &groupedResultTypes,
+                &numConvertedTypes)))
         return failure();
     }
 
@@ -565,34 +567,64 @@ struct CallOpInterfaceLowering : public ConvertOpToLLVMPattern<CallOpType> {
         static_cast<int32_t>(promoted.size()), 0};
     newOp.getProperties().op_bundle_sizes = rewriter.getDenseI32ArrayAttr({});
 
-    SmallVector<Value, 4> results;
-    if (numResults < 2) {
-      // If < 2 results, packing did not do anything and we can just return.
-      results.append(newOp.result_begin(), newOp.result_end());
-    } else {
-      // Otherwise, it had been converted to an operation producing a structure.
-      // Extract individual results from the structure and return them as list.
-      results.reserve(numResults);
-      for (unsigned i = 0; i < numResults; ++i) {
-        results.push_back(LLVM::ExtractValueOp::create(
-            rewriter, callOp.getLoc(), newOp->getResult(0), i));
+    // Helper function that extracts an individual result from the return value
+    // of the new call op. llvm.call ops support only 0 or 1 result. In case of
+    // 2 or more results, the results are packed into a structure.
+    //
+    // The new call op may have more than 2 results because:
+    // a. The original call op has more than 2 results.
+    // b. An original op result type-converted to more than 1 result.
+    auto getUnpackedResult = [&](unsigned i) -> Value {
+      assert(numConvertedTypes > 0 && "convert op has no results");
+      if (numConvertedTypes == 1) {
+        assert(i == 0 && "out of bounds: converted op has only one result");
+        return newOp->getResult(0);
       }
+      // Results have been converted to a structure. Extract individual results
+      // from the structure.
+      return LLVM::ExtractValueOp::create(rewriter, callOp.getLoc(),
+                                          newOp->getResult(0), i);
+    };
+
+    // Group the results into a vector of vectors, such that it is clear which
+    // original op result is replaced with which range of values. (In case of a
+    // 1:N conversion, there can be multiple replacements for a single result.)
+    SmallVector<SmallVector<Value>> results;
+    results.reserve(numResults);
+    unsigned counter = 0;
+    for (unsigned i = 0; i < numResults; ++i) {
+      SmallVector<Value> &group = results.emplace_back();
+      for (unsigned j = 0, e = groupedResultTypes[i].size(); j < e; ++j)
+        group.push_back(getUnpackedResult(counter++));
     }
 
-    if (useBarePtrCallConv) {
-      // For the bare-ptr calling convention, promote memref results to
-      // descriptors.
-      assert(results.size() == resultTypes.size() &&
-             "The number of arguments and types doesn't match");
-      this->getTypeConverter()->promoteBarePtrsToDescriptors(
-          rewriter, callOp.getLoc(), resultTypes, results);
-    } else if (failed(this->copyUnrankedDescriptors(rewriter, callOp.getLoc(),
-                                                    resultTypes, results,
-                                                    /*toDynamic=*/false))) {
-      return failure();
+    // Special handling for MemRef types.
+    for (unsigned i = 0; i < numResults; ++i) {
+      Type origType = resultTypes[i];
+      auto memrefType = dyn_cast<MemRefType>(origType);
+      auto unrankedMemrefType = dyn_cast<UnrankedMemRefType>(origType);
+      if (useBarePtrCallConv && memrefType) {
+        // For the bare-ptr calling convention, promote memref results to
+        // descriptors.
+        assert(results[i].size() == 1 && "expected one converted result");
+        results[i].front() = MemRefDescriptor::fromStaticShape(
+            rewriter, callOp.getLoc(), *this->getTypeConverter(), memrefType,
+            results[i].front());
+      }
+      if (unrankedMemrefType) {
+        assert(!useBarePtrCallConv && "unranked memref is not supported in the "
+                                      "bare-ptr calling convention");
+        assert(results[i].size() == 1 && "expected one converted result");
+        Value desc = this->copyUnrankedDescriptor(
+            rewriter, callOp.getLoc(), unrankedMemrefType, results[i].front(),
+            /*toDynamic=*/false);
+        if (!desc)
+          return failure();
+        results[i].front() = desc;
+      }
     }
 
-    rewriter.replaceOp(callOp, results);
+    rewriter.replaceOpWithMultiple(callOp, results);
     return success();
   }
 };
@@ -606,7 +638,7 @@ class CallOpLowering : public CallOpInterfaceLowering<func::CallOp> {
         symbolTables(symbolTables) {}
 
   LogicalResult
-  matchAndRewrite(func::CallOp callOp, OpAdaptor adaptor,
+  matchAndRewrite(func::CallOp callOp, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     bool useBarePtrCallConv = false;
     if (getTypeConverter()->getOptions().useBarePtrCallConv) {
@@ -636,7 +668,7 @@ struct CallIndirectOpLowering
   using Super::Super;
 
   LogicalResult
-  matchAndRewrite(func::CallIndirectOp callIndirectOp, OpAdaptor adaptor,
+  matchAndRewrite(func::CallIndirectOp callIndirectOp, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     return matchAndRewriteImpl(callIndirectOp, adaptor, rewriter);
   }
@@ -679,41 +711,50 @@ struct ReturnOpLowering : public ConvertOpToLLVMPattern<func::ReturnOp> {
   using ConvertOpToLLVMPattern<func::ReturnOp>::ConvertOpToLLVMPattern;
 
   LogicalResult
-  matchAndRewrite(func::ReturnOp op, OpAdaptor adaptor,
+  matchAndRewrite(func::ReturnOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
-    unsigned numArguments = op.getNumOperands();
     SmallVector<Value, 4> updatedOperands;
 
     auto funcOp = op->getParentOfType<LLVM::LLVMFuncOp>();
     bool useBarePtrCallConv =
         shouldUseBarePtrCallConv(funcOp, this->getTypeConverter());
-    if (useBarePtrCallConv) {
-      // For the bare-ptr calling convention, extract the aligned pointer to
-      // be returned from the memref descriptor.
-      for (auto it : llvm::zip(op->getOperands(), adaptor.getOperands())) {
-        Type oldTy = std::get<0>(it).getType();
-        Value newOperand = std::get<1>(it);
-        if (isa<MemRefType>(oldTy) && getTypeConverter()->canConvertToBarePtr(
-                                          cast<BaseMemRefType>(oldTy))) {
-          MemRefDescriptor memrefDesc(newOperand);
-          newOperand = memrefDesc.allocatedPtr(rewriter, loc);
-        } else if (isa<UnrankedMemRefType>(oldTy)) {
+
+    for (auto [oldOperand, newOperands] :
+         llvm::zip_equal(op->getOperands(), adaptor.getOperands())) {
+      Type oldTy = oldOperand.getType();
+      if (auto memRefType = dyn_cast<MemRefType>(oldTy)) {
+        assert(newOperands.size() == 1 && "expected one converted result");
+        if (useBarePtrCallConv &&
+            getTypeConverter()->canConvertToBarePtr(memRefType)) {
+          // For the bare-ptr calling convention, extract the aligned pointer to
+          // be returned from the memref descriptor.
+          MemRefDescriptor memrefDesc(newOperands.front());
+          updatedOperands.push_back(memrefDesc.allocatedPtr(rewriter, loc));
+          continue;
+        }
+      } else if (auto unrankedMemRefType =
+                     dyn_cast<UnrankedMemRefType>(oldTy)) {
+        assert(newOperands.size() == 1 && "expected one converted result");
+        if (useBarePtrCallConv) {
           // Unranked memref is not supported in the bare pointer calling
           // convention.
           return failure();
         }
-        updatedOperands.push_back(newOperand);
+        Value updatedDesc =
+            copyUnrankedDescriptor(rewriter, loc, unrankedMemRefType,
+                                   newOperands.front(), /*toDynamic=*/true);
+        if (!updatedDesc)
+          return failure();
+        updatedOperands.push_back(updatedDesc);
+        continue;
       }
-    } else {
-      updatedOperands = llvm::to_vector<4>(adaptor.getOperands());
-      (void)copyUnrankedDescriptors(rewriter, loc, op.getOperands().getTypes(),
-                                    updatedOperands,
-                                    /*toDynamic=*/true);
+
+      llvm::append_range(updatedOperands, newOperands);
     }
 
     // If ReturnOp has 0 or 1 operand, create it and return immediately.
-    if (numArguments <= 1) {
+    if (updatedOperands.size() <= 1) {
       rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(
           op, TypeRange(), updatedOperands, op->getAttrs());
       return success();
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 3cfbd898e49e2..e516118f75207 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -532,6 +532,9 @@ void GpuToLLVMConversionPass::runOnOperation() {
     // Vector transfer ops with rank > 1 should be lowered with VectorToSCF.
     vector::populateVectorTransferLoweringPatterns(patterns,
                                                    /*maxTransferRank=*/1);
+    // Transform N-D vector.from_elements to 1-D vector.from_elements before
+    // conversion.
+    vector::populateVectorFromElementsLoweringPatterns(patterns);
     if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))))
       return signalPassFailure();
   }
diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 317bfc2970cf5..50ac1c60184eb 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -27,6 +27,7 @@
 #include "mlir/Dialect/Math/IR/Math.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
+#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
@@ -369,6 +370,9 @@ struct LowerGpuOpsToNVVMOpsPass final
     {
       RewritePatternSet patterns(m.getContext());
       populateGpuRewritePatterns(patterns);
+      // Transform N-D vector.from_elements to 1-D vector.from_elements before
+      // conversion.
+      vector::populateVectorFromElementsLoweringPatterns(patterns);
       if (failed(applyPatternsGreedily(m, std::move(patterns))))
         return signalPassFailure();
     }
diff --git a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
index d22364e1ef441..e39fcd579b890 100644
--- a/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
+++ b/mlir/lib/Conversion/GPUToROCDL/LowerGpuOpsToROCDLOps.cpp
@@ -79,17 +79,30 @@ static bool canBeCalledWithBarePointers(gpu::GPUFuncOp func) {
   return canBeBare;
 }
 
-static Value getLaneId(ConversionPatternRewriter &rewriter, Location loc,
-                       const unsigned indexBitwidth) {
+static Value getLaneId(RewriterBase &rewriter, Location loc) {
   auto int32Type = IntegerType::get(rewriter.getContext(), 32);
   Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
   Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
-  Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, int32Type,
-                                           ValueRange{minus1, zero});
-  Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, int32Type,
-                                          ValueRange{minus1, mbcntLo});
+  NamedAttribute noundef = rewriter.getNamedAttr(
+      LLVM::LLVMDialect::getNoUndefAttrName(), rewriter.getUnitAttr());
+  NamedAttribute lowRange = rewriter.getNamedAttr(
+      LLVM::LLVMDialect::getRangeAttrName(),
+      LLVM::ConstantRangeAttr::get(rewriter.getContext(), APInt::getZero(32),
+                                   APInt(32, 32)));
+  NamedAttribute highRange = rewriter.getNamedAttr(
+      LLVM::LLVMDialect::getRangeAttrName(),
+      LLVM::ConstantRangeAttr::get(rewriter.getContext(), APInt::getZero(32),
+                                   APInt(32, 64)));
+  Value mbcntLo = ROCDL::MbcntLoOp::create(
+      rewriter, loc, int32Type, minus1, zero, /*arg_attrs=*/{},
+      /*res_attrs=*/
+      rewriter.getArrayAttr(rewriter.getDictionaryAttr({noundef, lowRange})));
+  Value laneId = ROCDL::MbcntHiOp::create(
+      rewriter, loc, int32Type, minus1, mbcntLo, /*arg_attrs=*/{},
+      rewriter.getArrayAttr(rewriter.getDictionaryAttr({noundef, highRange})));
   return laneId;
 }
+
 static constexpr StringLiteral amdgcnDataLayout =
     "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
     "-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:"
@@ -104,18 +117,16 @@ struct GPULaneIdOpToROCDL : ConvertOpToLLVMPattern<gpu::LaneIdOp> {
   LogicalResult
   matchAndRewrite(gpu::LaneIdOp op, gpu::LaneIdOp::Adaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    auto loc = op->getLoc();
+    Location loc = op.getLoc();
     MLIRContext *context = rewriter.getContext();
-    // convert to:  %mlo = call @llvm.amdgcn.mbcnt.lo(-1, 0)
-    // followed by: %lid = call @llvm.amdgcn.mbcnt.hi(-1, %mlo)
-
-    Type intTy = IntegerType::get(context, 32);
-    Value zero = arith::ConstantIntOp::create(rewriter, loc, 0, 32);
-    Value minus1 = arith::ConstantIntOp::create(rewriter, loc, -1, 32);
-    Value mbcntLo = ROCDL::MbcntLoOp::create(rewriter, loc, intTy,
-                                             ValueRange{minus1, zero});
-    Value laneId = ROCDL::MbcntHiOp::create(rewriter, loc, intTy,
-                                            ValueRange{minus1, mbcntLo});
+    // convert to:
+    //   %mlo = call noundef range(i32 0, 32)
+    //     @llvm.amdgcn.mbcnt.lo(-1, 0)
+    // followed by:
+    //   %lid = call noundef range(i32 0, 64)
+    //     @llvm.amdgcn.mbcnt.hi(-1, %mlo)
+
+    Value laneId = getLaneId(rewriter, loc);
     // Truncate or extend the result depending on the index bitwidth specified
     // by the LLVMTypeConverter options.
     const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
@@ -185,8 +196,7 @@ struct GPUShuffleOpLowering : public ConvertOpToLLVMPattern<gpu::ShuffleOp> {
     Location loc = op->getLoc();
     Value initShflValue = adaptor.getValue();
 
-    const unsigned indexBitwidth = getTypeConverter()->getIndexTypeBitwidth();
-    Value srcLaneId = getLaneId(rewriter, loc, indexBitwidth);
+    Value srcLaneId = getLaneId(rewriter, loc);
 
     auto int32Type = IntegerType::get(rewriter.getContext(), 32);
     Value width = adaptor.getWidth();
@@ -317,7 +327,7 @@ struct LowerGpuOpsToROCDLOpsPass final
     {
       RewritePatternSet patterns(ctx);
       populateGpuRewritePatterns(patterns);
-      populateGpuPromoteShuffleToAMDGPUPatterns(patterns);
+      populateGpuPromoteShuffleToAMDGPUPatterns(patterns, maybeChipset);
       (void)applyPatternsGreedily(m, std::move(patterns));
     }
 
diff --git a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
index fce7a3f324b86..522e91421ff55 100644
--- a/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/MemRefBuilder.cpp
@@ -353,14 +353,9 @@ void UnrankedMemRefDescriptor::unpack(OpBuilder &builder, Location loc,
   results.push_back(d.memRefDescPtr(builder, loc));
 }
 
-void UnrankedMemRefDescriptor::computeSizes(
+Value UnrankedMemRefDescriptor::computeSize(
     OpBuilder &builder, Location loc, const LLVMTypeConverter &typeConverter,
-    ArrayRef<UnrankedMemRefDescriptor> values, ArrayRef<unsigned> addressSpaces,
-    SmallVectorImpl<Value> &sizes) {
-  if (values.empty())
-    return;
-  assert(values.size() == addressSpaces.size() &&
-         "must provide address space for each descriptor");
+    UnrankedMemRefDescriptor desc, unsigned addressSpace) {
   // Cache the index type.
   Type indexType = typeConverter.getIndexType();
 
@@ -371,34 +366,31 @@ void UnrankedMemRefDescriptor::computeSizes(
       builder, loc, indexType,
       llvm::divideCeil(typeConverter.getIndexTypeBitwidth(), 8));
 
-  sizes.reserve(sizes.size() + values.size());
-  for (auto [desc, addressSpace] : llvm::zip(values, addressSpaces)) {
-    // Emit IR computing the memory necessary to store the descriptor. This
-    // assumes the descriptor to be
-    //   { type*, type*, index, index[rank], index[rank] }
-    // and densely packed, so the total size is
-    //   2 * sizeof(pointer) + (1 + 2 * rank) * sizeof(index).
-    // TODO: consider including the actual size (including eventual padding due
-    // to data layout) into the unranked descriptor.
-    Value pointerSize = createIndexAttrConstant(
-        builder, loc, indexType,
-        llvm::divideCeil(typeConverter.getPointerBitwidth(addressSpace), 8));
-    Value doublePointerSize =
-        LLVM::MulOp::create(builder, loc, indexType, two, pointerSize);
-
-    // (1 + 2 * rank) * sizeof(index)
-    Value rank = desc.rank(builder, loc);
-    Value doubleRank = LLVM::MulOp::create(builder, loc, indexType, two, rank);
-    Value doubleRankIncremented =
-        LLVM::AddOp::create(builder, loc, indexType, doubleRank, one);
-    Value rankIndexSize = LLVM::MulOp::create(builder, loc, indexType,
-                                              doubleRankIncremented, indexSize);
-
-    // Total allocation size.
-    Value allocationSize = LLVM::AddOp::create(
-        builder, loc, indexType, doublePointerSize, rankIndexSize);
-    sizes.push_back(allocationSize);
-  }
+  // Emit IR computing the memory necessary to store the descriptor. This
+  // assumes the descriptor to be
+  //   { type*, type*, index, index[rank], index[rank] }
+  // and densely packed, so the total size is
+  //   2 * sizeof(pointer) + (1 + 2 * rank) * sizeof(index).
+  // TODO: consider including the actual size (including eventual padding due
+  // to data layout) into the unranked descriptor.
+  Value pointerSize = createIndexAttrConstant(
+      builder, loc, indexType,
+      llvm::divideCeil(typeConverter.getPointerBitwidth(addressSpace), 8));
+  Value doublePointerSize =
+      LLVM::MulOp::create(builder, loc, indexType, two, pointerSize);
+
+  // (1 + 2 * rank) * sizeof(index)
+  Value rank = desc.rank(builder, loc);
+  Value doubleRank = LLVM::MulOp::create(builder, loc, indexType, two, rank);
+  Value doubleRankIncremented =
+      LLVM::AddOp::create(builder, loc, indexType, doubleRank, one);
+  Value rankIndexSize = LLVM::MulOp::create(builder, loc, indexType,
+                                            doubleRankIncremented, indexSize);
+
+  // Total allocation size.
+  Value allocationSize = LLVM::AddOp::create(builder, loc, indexType,
+                                             doublePointerSize, rankIndexSize);
+  return allocationSize;
 }
 
 Value UnrankedMemRefDescriptor::allocatedPtr(
diff --git a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
index 2568044f1fd32..48a03198fd465 100644
--- a/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/Pattern.cpp
@@ -216,34 +216,14 @@ MemRefDescriptor ConvertToLLVMPattern::createMemRefDescriptor(
   return memRefDescriptor;
 }
 
-LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
-    OpBuilder &builder, Location loc, TypeRange origTypes,
-    SmallVectorImpl<Value> &operands, bool toDynamic) const {
-  assert(origTypes.size() == operands.size() &&
-         "expected as may original types as operands");
-
-  // Find operands of unranked memref type and store them.
-  SmallVector<UnrankedMemRefDescriptor> unrankedMemrefs;
-  SmallVector<unsigned> unrankedAddressSpaces;
-  for (unsigned i = 0, e = operands.size(); i < e; ++i) {
-    if (auto memRefType = dyn_cast<UnrankedMemRefType>(origTypes[i])) {
-      unrankedMemrefs.emplace_back(operands[i]);
-      FailureOr<unsigned> addressSpace =
-          getTypeConverter()->getMemRefAddressSpace(memRefType);
-      if (failed(addressSpace))
-        return failure();
-      unrankedAddressSpaces.emplace_back(*addressSpace);
-    }
-  }
-
-  if (unrankedMemrefs.empty())
-    return success();
-
-  // Compute allocation sizes.
-  SmallVector<Value> sizes;
-  UnrankedMemRefDescriptor::computeSizes(builder, loc, *getTypeConverter(),
-                                         unrankedMemrefs, unrankedAddressSpaces,
-                                         sizes);
+Value ConvertToLLVMPattern::copyUnrankedDescriptor(
+    OpBuilder &builder, Location loc, UnrankedMemRefType memRefType,
+    Value operand, bool toDynamic) const {
+  // Convert memory space.
+  FailureOr<unsigned> addressSpace =
+      getTypeConverter()->getMemRefAddressSpace(memRefType);
+  if (failed(addressSpace))
+    return {};
 
   // Get frequently used types.
   Type indexType = getTypeConverter()->getIndexType();
@@ -254,52 +234,61 @@ LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
   if (toDynamic) {
     mallocFunc = LLVM::lookupOrCreateMallocFn(builder, module, indexType);
     if (failed(mallocFunc))
-      return failure();
+      return {};
   }
   if (!toDynamic) {
     freeFunc = LLVM::lookupOrCreateFreeFn(builder, module);
     if (failed(freeFunc))
-      return failure();
+      return {};
   }
 
-  unsigned unrankedMemrefPos = 0;
-  for (unsigned i = 0, e = operands.size(); i < e; ++i) {
-    Type type = origTypes[i];
-    if (!isa<UnrankedMemRefType>(type))
-      continue;
-    Value allocationSize = sizes[unrankedMemrefPos++];
-    UnrankedMemRefDescriptor desc(operands[i]);
-
-    // Allocate memory, copy, and free the source if necessary.
-    Value memory =
-        toDynamic ? LLVM::CallOp::create(builder, loc, mallocFunc.value(),
-                                         allocationSize)
-                        .getResult()
-                  : LLVM::AllocaOp::create(builder, loc, getPtrType(),
-                                           IntegerType::get(getContext(), 8),
-                                           allocationSize,
-                                           /*alignment=*/0);
-    Value source = desc.memRefDescPtr(builder, loc);
-    LLVM::MemcpyOp::create(builder, loc, memory, source, allocationSize, false);
-    if (!toDynamic)
-      LLVM::CallOp::create(builder, loc, freeFunc.value(), source);
-
-    // Create a new descriptor. The same descriptor can be returned multiple
-    // times, attempting to modify its pointer can lead to memory leaks
-    // (allocated twice and overwritten) or double frees (the caller does not
-    // know if the descriptor points to the same memory).
-    Type descriptorType = getTypeConverter()->convertType(type);
-    if (!descriptorType)
-      return failure();
-    auto updatedDesc =
-        UnrankedMemRefDescriptor::poison(builder, loc, descriptorType);
-    Value rank = desc.rank(builder, loc);
-    updatedDesc.setRank(builder, loc, rank);
-    updatedDesc.setMemRefDescPtr(builder, loc, memory);
+  UnrankedMemRefDescriptor desc(operand);
+  Value allocationSize = UnrankedMemRefDescriptor::computeSize(
+      builder, loc, *getTypeConverter(), desc, *addressSpace);
+
+  // Allocate memory, copy, and free the source if necessary.
+  Value memory = toDynamic
+                     ? LLVM::CallOp::create(builder, loc, mallocFunc.value(),
+                                            allocationSize)
+                           .getResult()
+                     : LLVM::AllocaOp::create(builder, loc, getPtrType(),
+                                              IntegerType::get(getContext(), 8),
+                                              allocationSize,
+                                              /*alignment=*/0);
+  Value source = desc.memRefDescPtr(builder, loc);
+  LLVM::MemcpyOp::create(builder, loc, memory, source, allocationSize, false);
+  if (!toDynamic)
+    LLVM::CallOp::create(builder, loc, freeFunc.value(), source);
+
+  // Create a new descriptor. The same descriptor can be returned multiple
+  // times, attempting to modify its pointer can lead to memory leaks
+  // (allocated twice and overwritten) or double frees (the caller does not
+  // know if the descriptor points to the same memory).
+  Type descriptorType = getTypeConverter()->convertType(memRefType);
+  if (!descriptorType)
+    return {};
+  auto updatedDesc =
+      UnrankedMemRefDescriptor::poison(builder, loc, descriptorType);
+  Value rank = desc.rank(builder, loc);
+  updatedDesc.setRank(builder, loc, rank);
+  updatedDesc.setMemRefDescPtr(builder, loc, memory);
+  return updatedDesc;
+}
 
-    operands[i] = updatedDesc;
+LogicalResult ConvertToLLVMPattern::copyUnrankedDescriptors(
+    OpBuilder &builder, Location loc, TypeRange origTypes,
+    SmallVectorImpl<Value> &operands, bool toDynamic) const {
+  assert(origTypes.size() == operands.size() &&
+         "expected as may original types as operands");
+  for (unsigned i = 0, e = operands.size(); i < e; ++i) {
+    if (auto memRefType = dyn_cast<UnrankedMemRefType>(origTypes[i])) {
+      Value updatedDesc = copyUnrankedDescriptor(builder, loc, memRefType,
+                                                 operands[i], toDynamic);
+      if (!updatedDesc)
+        return failure();
+      operands[i] = updatedDesc;
+    }
   }
-
   return success();
 }
 
diff --git a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
index 1a9bf569086da..cb9dea108cc48 100644
--- a/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
+++ b/mlir/lib/Conversion/LLVMCommon/TypeConverter.cpp
@@ -365,6 +365,7 @@ Type LLVMTypeConverter::convertFunctionSignatureImpl(
   useBarePtrCallConv = useBarePtrCallConv || options.useBarePtrCallConv;
   auto funcArgConverter = useBarePtrCallConv ? barePtrFuncArgTypeConverter
                                              : structFuncArgTypeConverter;
+
   // Convert argument types one by one and check for errors.
   for (auto [idx, type] : llvm::enumerate(funcTy.getInputs())) {
     SmallVector<Type, 8> converted;
@@ -658,27 +659,19 @@ FailureOr<Type> LLVMTypeConverter::convertVectorType(VectorType type) const {
 /// UnrankedMemRefType, are converted following the specific rules for the
 /// calling convention. Calling convention independent types are converted
 /// following the default LLVM type conversions.
-Type LLVMTypeConverter::convertCallingConventionType(
-    Type type, bool useBarePtrCallConv) const {
-  if (useBarePtrCallConv)
-    if (auto memrefTy = dyn_cast<BaseMemRefType>(type))
-      return convertMemRefToBarePtr(memrefTy);
-
-  return convertType(type);
-}
+LogicalResult LLVMTypeConverter::convertCallingConventionType(
+    Type type, SmallVectorImpl<Type> &result, bool useBarePtrCallConv) const {
+  if (useBarePtrCallConv) {
+    if (auto memrefTy = dyn_cast<BaseMemRefType>(type)) {
+      Type converted = convertMemRefToBarePtr(memrefTy);
+      if (!converted)
+        return failure();
+      result.push_back(converted);
+      return success();
+    }
+  }
 
-/// Promote the bare pointers in 'values' that resulted from memrefs to
-/// descriptors. 'stdTypes' holds they types of 'values' before the conversion
-/// to the LLVM-IR dialect (i.e., MemRefType, or any other builtin type).
-void LLVMTypeConverter::promoteBarePtrsToDescriptors(
-    ConversionPatternRewriter &rewriter, Location loc, ArrayRef<Type> stdTypes,
-    SmallVectorImpl<Value> &values) const {
-  assert(stdTypes.size() == values.size() &&
-         "The number of types and values doesn't match");
-  for (unsigned i = 0, end = values.size(); i < end; ++i)
-    if (auto memrefTy = dyn_cast<MemRefType>(stdTypes[i]))
-      values[i] = MemRefDescriptor::fromStaticShape(rewriter, loc, *this,
-                                                    memrefTy, values[i]);
+  return convertType(type, result);
 }
 
 /// Convert a non-empty list of types of values produced by an operation into an
@@ -706,23 +699,35 @@ Type LLVMTypeConverter::packOperationResults(TypeRange types) const {
 /// LLVM-compatible type. In particular, if more than one value is returned,
 /// create an LLVM dialect structure type with elements that correspond to each
 /// of the types converted with `convertCallingConventionType`.
-Type LLVMTypeConverter::packFunctionResults(TypeRange types,
-                                            bool useBarePtrCallConv) const {
+Type LLVMTypeConverter::packFunctionResults(
+    TypeRange types, bool useBarePtrCallConv,
+    SmallVector<SmallVector<Type>> *groupedTypes,
+    int64_t *numConvertedTypes) const {
   assert(!types.empty() && "expected non-empty list of type");
+  assert((!groupedTypes || groupedTypes->empty()) &&
+         "expected groupedTypes to be empty");
 
   useBarePtrCallConv |= options.useBarePtrCallConv;
-  if (types.size() == 1)
-    return convertCallingConventionType(types.front(), useBarePtrCallConv);
-
   SmallVector<Type> resultTypes;
   resultTypes.reserve(types.size());
+  size_t sizeBefore = 0;
   for (auto t : types) {
-    auto converted = convertCallingConventionType(t, useBarePtrCallConv);
-    if (!converted || !LLVM::isCompatibleType(converted))
+    if (failed(
+            convertCallingConventionType(t, resultTypes, useBarePtrCallConv)))
       return {};
-    resultTypes.push_back(converted);
+    if (groupedTypes) {
+      SmallVector<Type> &group = groupedTypes->emplace_back();
+      llvm::append_range(group, ArrayRef(resultTypes).drop_front(sizeBefore));
+    }
+    sizeBefore = resultTypes.size();
   }
 
+  if (numConvertedTypes)
+    *numConvertedTypes = resultTypes.size();
+  if (resultTypes.size() == 1)
+    return resultTypes.front();
+  if (resultTypes.empty())
+    return {};
   return LLVM::LLVMStructType::getLiteral(&getContext(), resultTypes);
 }
 
@@ -740,40 +745,50 @@ Value LLVMTypeConverter::promoteOneMemRefDescriptor(Location loc, Value operand,
   return allocated;
 }
 
-SmallVector<Value, 4>
-LLVMTypeConverter::promoteOperands(Location loc, ValueRange opOperands,
-                                   ValueRange operands, OpBuilder &builder,
-                                   bool useBarePtrCallConv) const {
+SmallVector<Value, 4> LLVMTypeConverter::promoteOperands(
+    Location loc, ValueRange opOperands, ValueRange adaptorOperands,
+    OpBuilder &builder, bool useBarePtrCallConv) const {
+  SmallVector<ValueRange> ranges;
+  for (size_t i = 0, e = adaptorOperands.size(); i < e; i++)
+    ranges.push_back(adaptorOperands.slice(i, 1));
+  return promoteOperands(loc, opOperands, ranges, builder, useBarePtrCallConv);
+}
+
+SmallVector<Value, 4> LLVMTypeConverter::promoteOperands(
+    Location loc, ValueRange opOperands, ArrayRef<ValueRange> adaptorOperands,
+    OpBuilder &builder, bool useBarePtrCallConv) const {
   SmallVector<Value, 4> promotedOperands;
-  promotedOperands.reserve(operands.size());
+  promotedOperands.reserve(adaptorOperands.size());
   useBarePtrCallConv |= options.useBarePtrCallConv;
-  for (auto it : llvm::zip(opOperands, operands)) {
-    auto operand = std::get<0>(it);
-    auto llvmOperand = std::get<1>(it);
-
+  for (auto [operand, llvmOperand] :
+       llvm::zip_equal(opOperands, adaptorOperands)) {
     if (useBarePtrCallConv) {
       // For the bare-ptr calling convention, we only have to extract the
       // aligned pointer of a memref.
       if (isa<MemRefType>(operand.getType())) {
-        MemRefDescriptor desc(llvmOperand);
-        llvmOperand = desc.alignedPtr(builder, loc);
+        assert(llvmOperand.size() == 1 && "Expected a single operand");
+        MemRefDescriptor desc(llvmOperand.front());
+        promotedOperands.push_back(desc.alignedPtr(builder, loc));
+        continue;
       } else if (isa<UnrankedMemRefType>(operand.getType())) {
         llvm_unreachable("Unranked memrefs are not supported");
       }
     } else {
       if (isa<UnrankedMemRefType>(operand.getType())) {
-        UnrankedMemRefDescriptor::unpack(builder, loc, llvmOperand,
+        assert(llvmOperand.size() == 1 && "Expected a single operand");
+        UnrankedMemRefDescriptor::unpack(builder, loc, llvmOperand.front(),
                                          promotedOperands);
         continue;
       }
       if (auto memrefType = dyn_cast<MemRefType>(operand.getType())) {
-        MemRefDescriptor::unpack(builder, loc, llvmOperand, memrefType,
+        assert(llvmOperand.size() == 1 && "Expected a single operand");
+        MemRefDescriptor::unpack(builder, loc, llvmOperand.front(), memrefType,
                                  promotedOperands);
         continue;
       }
     }
 
-    promotedOperands.push_back(llvmOperand);
+    llvm::append_range(promotedOperands, llvmOperand);
   }
   return promotedOperands;
 }
@@ -802,11 +817,7 @@ mlir::structFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type,
     result.append(converted.begin(), converted.end());
     return success();
   }
-  auto converted = converter.convertType(type);
-  if (!converted)
-    return failure();
-  result.push_back(converted);
-  return success();
+  return converter.convertType(type, result);
 }
 
 /// Callback to convert function argument types. It converts MemRef function
@@ -814,11 +825,7 @@ mlir::structFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type,
 LogicalResult
 mlir::barePtrFuncArgTypeConverter(const LLVMTypeConverter &converter, Type type,
                                   SmallVectorImpl<Type> &result) {
-  auto llvmTy = converter.convertCallingConventionType(
-      type, /*useBarePointerCallConv=*/true);
-  if (!llvmTy)
-    return failure();
-
-  result.push_back(llvmTy);
-  return success();
+  return converter.convertCallingConventionType(
+      type, result,
+      /*useBarePointerCallConv=*/true);
 }
diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
index 6bd0e2d4d4b08..2b7bdc9a7b7f8 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp
@@ -17,11 +17,13 @@
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/TypeRange.h"
 #include "mlir/IR/Value.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include <cstdint>
+#include <numeric>
 
 using namespace mlir;
 
@@ -97,6 +99,48 @@ Type convertMemRefType(MemRefType opTy, const TypeConverter *typeConverter) {
   return resultTy;
 }
 
+static Value calculateMemrefTotalSizeBytes(Location loc, MemRefType memrefType,
+                                           OpBuilder &builder) {
+  assert(isMemRefTypeLegalForEmitC(memrefType) &&
+         "incompatible memref type for EmitC conversion");
+  emitc::CallOpaqueOp elementSize = emitc::CallOpaqueOp::create(
+      builder, loc, emitc::SizeTType::get(builder.getContext()),
+      builder.getStringAttr("sizeof"), ValueRange{},
+      ArrayAttr::get(builder.getContext(),
+                     {TypeAttr::get(memrefType.getElementType())}));
+
+  IndexType indexType = builder.getIndexType();
+  int64_t numElements = std::accumulate(memrefType.getShape().begin(),
+                                        memrefType.getShape().end(), int64_t{1},
+                                        std::multiplies<int64_t>());
+  emitc::ConstantOp numElementsValue = emitc::ConstantOp::create(
+      builder, loc, indexType, builder.getIndexAttr(numElements));
+
+  Type sizeTType = emitc::SizeTType::get(builder.getContext());
+  emitc::MulOp totalSizeBytes = emitc::MulOp::create(
+      builder, loc, sizeTType, elementSize.getResult(0), numElementsValue);
+
+  return totalSizeBytes.getResult();
+}
+
+static emitc::ApplyOp
+createPointerFromEmitcArray(Location loc, OpBuilder &builder,
+                            TypedValue<emitc::ArrayType> arrayValue) {
+
+  emitc::ConstantOp zeroIndex = emitc::ConstantOp::create(
+      builder, loc, builder.getIndexType(), builder.getIndexAttr(0));
+
+  emitc::ArrayType arrayType = arrayValue.getType();
+  llvm::SmallVector<mlir::Value> indices(arrayType.getRank(), zeroIndex);
+  emitc::SubscriptOp subPtr =
+      emitc::SubscriptOp::create(builder, loc, arrayValue, ValueRange(indices));
+  emitc::ApplyOp ptr = emitc::ApplyOp::create(
+      builder, loc, emitc::PointerType::get(arrayType.getElementType()),
+      builder.getStringAttr("&"), subPtr);
+
+  return ptr;
+}
+
 struct ConvertAlloc final : public OpConversionPattern<memref::AllocOp> {
   using OpConversionPattern::OpConversionPattern;
   LogicalResult
@@ -112,19 +156,21 @@ struct ConvertAlloc final : public OpConversionPattern<memref::AllocOp> {
     Type sizeTType = emitc::SizeTType::get(rewriter.getContext());
     Type elementType = memrefType.getElementType();
     IndexType indexType = rewriter.getIndexType();
-    emitc::CallOpaqueOp sizeofElementOp = rewriter.create<emitc::CallOpaqueOp>(
-        loc, sizeTType, rewriter.getStringAttr("sizeof"), ValueRange{},
+    emitc::CallOpaqueOp sizeofElementOp = emitc::CallOpaqueOp::create(
+        rewriter, loc, sizeTType, rewriter.getStringAttr("sizeof"),
+        ValueRange{},
         ArrayAttr::get(rewriter.getContext(), {TypeAttr::get(elementType)}));
 
     int64_t numElements = 1;
     for (int64_t dimSize : memrefType.getShape()) {
       numElements *= dimSize;
     }
-    Value numElementsValue = rewriter.create<emitc::ConstantOp>(
-        loc, indexType, rewriter.getIndexAttr(numElements));
+    Value numElementsValue = emitc::ConstantOp::create(
+        rewriter, loc, indexType, rewriter.getIndexAttr(numElements));
 
-    Value totalSizeBytes = rewriter.create<emitc::MulOp>(
-        loc, sizeTType, sizeofElementOp.getResult(0), numElementsValue);
+    Value totalSizeBytes =
+        emitc::MulOp::create(rewriter, loc, sizeTType,
+                             sizeofElementOp.getResult(0), numElementsValue);
 
     emitc::CallOpaqueOp allocCall;
     StringAttr allocFunctionName;
@@ -132,8 +178,8 @@ struct ConvertAlloc final : public OpConversionPattern<memref::AllocOp> {
     SmallVector<Value, 2> argsVec;
     if (allocOp.getAlignment()) {
       allocFunctionName = rewriter.getStringAttr(alignedAllocFunctionName);
-      alignmentValue = rewriter.create<emitc::ConstantOp>(
-          loc, sizeTType,
+      alignmentValue = emitc::ConstantOp::create(
+          rewriter, loc, sizeTType,
           rewriter.getIntegerAttr(indexType,
                                   allocOp.getAlignment().value_or(0)));
       argsVec.push_back(alignmentValue);
@@ -144,21 +190,62 @@ struct ConvertAlloc final : public OpConversionPattern<memref::AllocOp> {
     argsVec.push_back(totalSizeBytes);
     ValueRange args(argsVec);
 
-    allocCall = rewriter.create<emitc::CallOpaqueOp>(
-        loc,
+    allocCall = emitc::CallOpaqueOp::create(
+        rewriter, loc,
         emitc::PointerType::get(
             emitc::OpaqueType::get(rewriter.getContext(), "void")),
         allocFunctionName, args);
 
     emitc::PointerType targetPointerType = emitc::PointerType::get(elementType);
-    emitc::CastOp castOp = rewriter.create<emitc::CastOp>(
-        loc, targetPointerType, allocCall.getResult(0));
+    emitc::CastOp castOp = emitc::CastOp::create(
+        rewriter, loc, targetPointerType, allocCall.getResult(0));
 
     rewriter.replaceOp(allocOp, castOp);
     return success();
   }
 };
 
+struct ConvertCopy final : public OpConversionPattern<memref::CopyOp> {
+  using OpConversionPattern::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(memref::CopyOp copyOp, OpAdaptor operands,
+                  ConversionPatternRewriter &rewriter) const override {
+    Location loc = copyOp.getLoc();
+    MemRefType srcMemrefType = cast<MemRefType>(copyOp.getSource().getType());
+    MemRefType targetMemrefType =
+        cast<MemRefType>(copyOp.getTarget().getType());
+
+    if (!isMemRefTypeLegalForEmitC(srcMemrefType))
+      return rewriter.notifyMatchFailure(
+          loc, "incompatible source memref type for EmitC conversion");
+
+    if (!isMemRefTypeLegalForEmitC(targetMemrefType))
+      return rewriter.notifyMatchFailure(
+          loc, "incompatible target memref type for EmitC conversion");
+
+    auto srcArrayValue =
+        cast<TypedValue<emitc::ArrayType>>(operands.getSource());
+    emitc::ApplyOp srcPtr =
+        createPointerFromEmitcArray(loc, rewriter, srcArrayValue);
+
+    auto targetArrayValue =
+        cast<TypedValue<emitc::ArrayType>>(operands.getTarget());
+    emitc::ApplyOp targetPtr =
+        createPointerFromEmitcArray(loc, rewriter, targetArrayValue);
+
+    emitc::CallOpaqueOp memCpyCall = emitc::CallOpaqueOp::create(
+        rewriter, loc, TypeRange{}, "memcpy",
+        ValueRange{
+            targetPtr.getResult(), srcPtr.getResult(),
+            calculateMemrefTotalSizeBytes(loc, srcMemrefType, rewriter)});
+
+    rewriter.replaceOp(copyOp, memCpyCall.getResults());
+
+    return success();
+  }
+};
+
 struct ConvertGlobal final : public OpConversionPattern<memref::GlobalOp> {
   using OpConversionPattern::OpConversionPattern;
 
@@ -320,6 +407,7 @@ void mlir::populateMemRefToEmitCTypeConversion(TypeConverter &typeConverter) {
 
 void mlir::populateMemRefToEmitCConversionPatterns(
     RewritePatternSet &patterns, const TypeConverter &converter) {
-  patterns.add<ConvertAlloca, ConvertAlloc, ConvertGlobal, ConvertGetGlobal,
-               ConvertLoad, ConvertStore>(converter, patterns.getContext());
+  patterns.add<ConvertAlloca, ConvertAlloc, ConvertCopy, ConvertGlobal,
+               ConvertGetGlobal, ConvertLoad, ConvertStore>(
+      converter, patterns.getContext());
 }
diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp
index e78dd76d6e256..a073a9acf752f 100644
--- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp
+++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitCPass.cpp
@@ -18,6 +18,8 @@
 #include "mlir/IR/Attributes.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/DialectConversion.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/StringRef.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_CONVERTMEMREFTOEMITC
@@ -27,6 +29,15 @@ namespace mlir {
 using namespace mlir;
 
 namespace {
+
+emitc::IncludeOp addStandardHeader(OpBuilder &builder, ModuleOp module,
+                                   StringRef headerName) {
+  StringAttr includeAttr = builder.getStringAttr(headerName);
+  return emitc::IncludeOp::create(
+      builder, module.getLoc(), includeAttr,
+      /*is_standard_include=*/builder.getUnitAttr());
+}
+
 struct ConvertMemRefToEmitCPass
     : public impl::ConvertMemRefToEmitCBase<ConvertMemRefToEmitCPass> {
   using Base::Base;
@@ -55,34 +66,29 @@ struct ConvertMemRefToEmitCPass
       return signalPassFailure();
 
     mlir::ModuleOp module = getOperation();
+    llvm::SmallSet<StringRef, 4> existingHeaders;
+    mlir::OpBuilder builder(module.getBody(), module.getBody()->begin());
+    module.walk([&](mlir::emitc::IncludeOp includeOp) {
+      if (includeOp.getIsStandardInclude())
+        existingHeaders.insert(includeOp.getInclude());
+    });
+
     module.walk([&](mlir::emitc::CallOpaqueOp callOp) {
-      if (callOp.getCallee() != alignedAllocFunctionName &&
-          callOp.getCallee() != mallocFunctionName) {
+      StringRef expectedHeader;
+      if (callOp.getCallee() == alignedAllocFunctionName ||
+          callOp.getCallee() == mallocFunctionName)
+        expectedHeader = options.lowerToCpp ? cppStandardLibraryHeader
+                                            : cStandardLibraryHeader;
+      else if (callOp.getCallee() == memcpyFunctionName)
+        expectedHeader =
+            options.lowerToCpp ? cppStringLibraryHeader : cStringLibraryHeader;
+      else
         return mlir::WalkResult::advance();
+      if (!existingHeaders.contains(expectedHeader)) {
+        addStandardHeader(builder, module, expectedHeader);
+        existingHeaders.insert(expectedHeader);
       }
-
-      for (auto &op : *module.getBody()) {
-        emitc::IncludeOp includeOp = llvm::dyn_cast<mlir::emitc::IncludeOp>(op);
-        if (!includeOp) {
-          continue;
-        }
-        if (includeOp.getIsStandardInclude() &&
-            ((options.lowerToCpp &&
-              includeOp.getInclude() == cppStandardLibraryHeader) ||
-             (!options.lowerToCpp &&
-              includeOp.getInclude() == cStandardLibraryHeader))) {
-          return mlir::WalkResult::interrupt();
-        }
-      }
-
-      mlir::OpBuilder builder(module.getBody(), module.getBody()->begin());
-      StringAttr includeAttr =
-          builder.getStringAttr(options.lowerToCpp ? cppStandardLibraryHeader
-                                                   : cStandardLibraryHeader);
-      builder.create<mlir::emitc::IncludeOp>(
-          module.getLoc(), includeAttr,
-          /*is_standard_include=*/builder.getUnitAttr());
-      return mlir::WalkResult::interrupt();
+      return mlir::WalkResult::advance();
     });
   }
 };
diff --git a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
index d6bdd342c9045..262e0e7a30c63 100644
--- a/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
+++ b/mlir/lib/Conversion/MemRefToLLVM/MemRefToLLVM.cpp
@@ -1246,10 +1246,8 @@ struct MemorySpaceCastOpLowering
       auto result = UnrankedMemRefDescriptor::poison(
           rewriter, loc, typeConverter->convertType(resultTypeU));
       result.setRank(rewriter, loc, rank);
-      SmallVector<Value, 1> sizes;
-      UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
-                                             result, resultAddrSpace, sizes);
-      Value resultUnderlyingSize = sizes.front();
+      Value resultUnderlyingSize = UnrankedMemRefDescriptor::computeSize(
+          rewriter, loc, *getTypeConverter(), result, resultAddrSpace);
       Value resultUnderlyingDesc =
           LLVM::AllocaOp::create(rewriter, loc, getPtrType(),
                                  rewriter.getI8Type(), resultUnderlyingSize);
@@ -1530,12 +1528,11 @@ struct MemRefReshapeOpLowering
     auto targetDesc = UnrankedMemRefDescriptor::poison(
         rewriter, loc, typeConverter->convertType(targetType));
     targetDesc.setRank(rewriter, loc, resultRank);
-    SmallVector<Value, 4> sizes;
-    UnrankedMemRefDescriptor::computeSizes(rewriter, loc, *getTypeConverter(),
-                                           targetDesc, addressSpace, sizes);
+    Value allocationSize = UnrankedMemRefDescriptor::computeSize(
+        rewriter, loc, *getTypeConverter(), targetDesc, addressSpace);
     Value underlyingDescPtr = LLVM::AllocaOp::create(
         rewriter, loc, getPtrType(), IntegerType::get(getContext(), 8),
-        sizes.front());
+        allocationSize);
     targetDesc.setMemRefDescPtr(rewriter, loc, underlyingDescPtr);
 
     // Extract pointers and offset from the source memref.
@@ -1872,6 +1869,8 @@ matchSimpleAtomicOp(memref::AtomicRMWOp atomicOp) {
     return LLVM::AtomicBinOp::umin;
   case arith::AtomicRMWKind::ori:
     return LLVM::AtomicBinOp::_or;
+  case arith::AtomicRMWKind::xori:
+    return LLVM::AtomicBinOp::_xor;
   case arith::AtomicRMWKind::andi:
     return LLVM::AtomicBinOp::_and;
   default:
diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
index c7ecd8334da42..2e00b42f4a56d 100644
--- a/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
+++ b/mlir/lib/Conversion/MemRefToSPIRV/MemRefToSPIRV.cpp
@@ -22,6 +22,7 @@
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/IR/Visitors.h"
 #include <cassert>
+#include <limits>
 #include <optional>
 
 #define DEBUG_TYPE "memref-to-spirv-pattern"
@@ -475,7 +476,12 @@ struct MemoryRequirements {
 /// Given an accessed SPIR-V pointer, calculates its alignment requirements, if
 /// any.
 static FailureOr<MemoryRequirements>
-calculateMemoryRequirements(Value accessedPtr, bool isNontemporal) {
+calculateMemoryRequirements(Value accessedPtr, bool isNontemporal,
+                            uint64_t preferredAlignment) {
+  if (preferredAlignment >= std::numeric_limits<uint32_t>::max()) {
+    return failure();
+  }
+
   MLIRContext *ctx = accessedPtr.getContext();
 
   auto memoryAccess = spirv::MemoryAccess::None;
@@ -484,7 +490,10 @@ calculateMemoryRequirements(Value accessedPtr, bool isNontemporal) {
   }
 
   auto ptrType = cast<spirv::PointerType>(accessedPtr.getType());
-  if (ptrType.getStorageClass() != spirv::StorageClass::PhysicalStorageBuffer) {
+  bool mayOmitAlignment =
+      !preferredAlignment &&
+      ptrType.getStorageClass() != spirv::StorageClass::PhysicalStorageBuffer;
+  if (mayOmitAlignment) {
     if (memoryAccess == spirv::MemoryAccess::None) {
       return MemoryRequirements{spirv::MemoryAccessAttr{}, IntegerAttr{}};
     }
@@ -493,6 +502,7 @@ calculateMemoryRequirements(Value accessedPtr, bool isNontemporal) {
   }
 
   // PhysicalStorageBuffers require the `Aligned` attribute.
+  // Other storage types may show an `Aligned` attribute.
   auto pointeeType = dyn_cast<spirv::ScalarType>(ptrType.getPointeeType());
   if (!pointeeType)
     return failure();
@@ -504,7 +514,8 @@ calculateMemoryRequirements(Value accessedPtr, bool isNontemporal) {
 
   memoryAccess = memoryAccess | spirv::MemoryAccess::Aligned;
   auto memAccessAttr = spirv::MemoryAccessAttr::get(ctx, memoryAccess);
-  auto alignment = IntegerAttr::get(IntegerType::get(ctx, 32), *sizeInBytes);
+  auto alignmentValue = preferredAlignment ? preferredAlignment : *sizeInBytes;
+  auto alignment = IntegerAttr::get(IntegerType::get(ctx, 32), alignmentValue);
   return MemoryRequirements{memAccessAttr, alignment};
 }
 
@@ -518,16 +529,9 @@ calculateMemoryRequirements(Value accessedPtr, LoadOrStoreOp loadOrStoreOp) {
       llvm::is_one_of<LoadOrStoreOp, memref::LoadOp, memref::StoreOp>::value,
       "Must be called on either memref::LoadOp or memref::StoreOp");
 
-  Operation *memrefAccessOp = loadOrStoreOp.getOperation();
-  auto memrefMemAccess = memrefAccessOp->getAttrOfType<spirv::MemoryAccessAttr>(
-      spirv::attributeName<spirv::MemoryAccess>());
-  auto memrefAlignment =
-      memrefAccessOp->getAttrOfType<IntegerAttr>("alignment");
-  if (memrefMemAccess && memrefAlignment)
-    return MemoryRequirements{memrefMemAccess, memrefAlignment};
-
   return calculateMemoryRequirements(accessedPtr,
-                                     loadOrStoreOp.getNontemporal());
+                                     loadOrStoreOp.getNontemporal(),
+                                     loadOrStoreOp.getAlignment().value_or(0));
 }
 
 LogicalResult
diff --git a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
index 2549a9c631c24..c6c5ab356f256 100644
--- a/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
+++ b/mlir/lib/Conversion/NVGPUToNVVM/NVGPUToNVVM.cpp
@@ -283,11 +283,13 @@ struct MmaLdMatrixOpToNVVM : public ConvertOpToLLVMPattern<nvgpu::LdMatrixOp> {
     Value srcPtr =
         getStridedElementPtr(rewriter, b.getLoc(), srcMemrefType,
                              adaptor.getSrcMemref(), adaptor.getIndices());
+    auto shape = NVVM::LdStMatrixShapeAttr::get(rewriter.getContext(), 8, 8);
     Value ldMatrixResult = NVVM::LdMatrixOp::create(
         b, ldMatrixResultType, srcPtr,
         /*num=*/op.getNumTiles(),
         /*layout=*/op.getTranspose() ? NVVM::MMALayout::col
-                                     : NVVM::MMALayout::row);
+                                     : NVVM::MMALayout::row,
+        /*shape=*/shape, /*eltType=*/NVVM::LdStMatrixEltType::B16);
 
     // The ldmatrix operation returns either a single i32 value or a struct of
     // i32 values. Here we unpack those values and cast them back to their
@@ -1104,12 +1106,10 @@ struct NVGPUGenerateWarpgroupDescriptorLowering
     // // [0,14)   start_address
     dsc = insertBit(dsc, basePtr14bit, startBaseAddrBit);
 
-    LDBG() << "Generating warpgroup.descriptor: "
-           << "leading_off:" << leadDimVal << "\t"
-           << "stride_off :" << strideDimVal << "\t"
-           << "base_offset:" << offsetVal << "\t"
-           << "layout_type:" << swizzle << " ("
-           << nvgpu::stringifyTensorMapSwizzleKind(swizzleKind)
+    LDBG() << "Generating warpgroup.descriptor: " << "leading_off:"
+           << leadDimVal << "\t" << "stride_off :" << strideDimVal << "\t"
+           << "base_offset:" << offsetVal << "\t" << "layout_type:" << swizzle
+           << " (" << nvgpu::stringifyTensorMapSwizzleKind(swizzleKind)
            << ")\n start_addr :  " << baseAddr;
 
     rewriter.replaceOp(op, dsc);
@@ -1399,14 +1399,12 @@ struct NVGPUWarpgroupMmaOpLowering
     /// This function generates a WgmmaMmaAsyncOp using provided GMMA matrix
     /// descriptors and arranges them based on induction variables: i, j, and k.
     Value generateWgmma(int i, int j, int k, Value matrixC) {
-      LDBG() << "\t wgmma."
-             << "m" << wgmmaM << "n" << wgmmaN << "k" << wgmmaK << "(A["
-             << (iterationM * wgmmaM) << ":" << (iterationM * wgmmaM) + wgmmaM
-             << "][" << (iterationK * wgmmaK) << ":"
-             << (iterationK * wgmmaK + wgmmaK) << "] * "
-             << " B[" << (iterationK * wgmmaK) << ":"
-             << (iterationK * wgmmaK + wgmmaK) << "][" << 0 << ":" << wgmmaN
-             << "])";
+      LDBG() << "\t wgmma." << "m" << wgmmaM << "n" << wgmmaN << "k" << wgmmaK
+             << "(A[" << (iterationM * wgmmaM) << ":"
+             << (iterationM * wgmmaM) + wgmmaM << "][" << (iterationK * wgmmaK)
+             << ":" << (iterationK * wgmmaK + wgmmaK) << "] * " << " B["
+             << (iterationK * wgmmaK) << ":" << (iterationK * wgmmaK + wgmmaK)
+             << "][" << 0 << ":" << wgmmaN << "])";
 
       Value descriptorA = iterateDescriptorA(adaptor.getDescriptorA(), i, j, k);
       Value descriptorB = iterateDescriptorB(adaptor.getDescriptorB(), i, j, k);
diff --git a/mlir/lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp b/mlir/lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp
index 91788f9848fe6..c67ec3642f121 100644
--- a/mlir/lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp
+++ b/mlir/lib/Conversion/NVVMToLLVM/NVVMToLLVM.cpp
@@ -57,11 +57,11 @@ struct PtxLowering
 
     SmallVector<std::pair<Value, PTXRegisterMod>> asmValues;
     LDBG() << op.getPtx();
-    PtxBuilder generator(op, rewriter);
 
-    op.getAsmValues(rewriter, asmValues);
+    bool needsManualMapping = op.getAsmValues(rewriter, asmValues);
+    PtxBuilder generator(op, rewriter, needsManualMapping);
     for (auto &[asmValue, modifier] : asmValues) {
-      LDBG() << asmValue << "\t Modifier : " << &modifier;
+      LDBG() << asmValue << "\t Modifier : " << modifier;
       generator.insertValue(asmValue, modifier);
     }
 
diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
index 5bd1d49dae0c9..7a5ea4f29056b 100644
--- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
+++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include <queue>
 
 #define DEBUG_TYPE "pdl-predicate-tree"
@@ -544,7 +545,7 @@ static void visitUpward(std::vector<PositionalPredicate> &predList,
   Value value = opIndex.parent;
   TypeSwitch<Operation *>(value.getDefiningOp())
       .Case<pdl::OperationOp>([&](auto operationOp) {
-        LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n");
+        LDBG() << "  * Value: " << value;
 
         // Get users and iterate over them.
         Position *usersPos = builder.getUsers(pos, /*useRepresentative=*/true);
@@ -618,19 +619,15 @@ static Value buildPredicateList(pdl::PatternOp pattern,
   RootOrderingGraph graph;
   ParentMaps parentMaps;
   buildCostGraph(roots, graph, parentMaps);
-  LLVM_DEBUG({
-    llvm::dbgs() << "Graph:\n";
-    for (auto &target : graph) {
-      llvm::dbgs() << "  * " << target.first.getLoc() << " " << target.first
-                   << "\n";
-      for (auto &source : target.second) {
-        RootOrderingEntry &entry = source.second;
-        llvm::dbgs() << "      <- " << source.first << ": " << entry.cost.first
-                     << ":" << entry.cost.second << " via "
-                     << entry.connector.getLoc() << "\n";
-      }
+  LDBG() << "Graph:";
+  for (auto &target : graph) {
+    LDBG() << "  * " << target.first.getLoc() << " " << target.first;
+    for (auto &source : target.second) {
+      RootOrderingEntry &entry = source.second;
+      LDBG() << "      <- " << source.first << ": " << entry.cost.first << ":"
+             << entry.cost.second << " via " << entry.connector.getLoc();
     }
-  });
+  }
 
   // Solve the optimal branching problem for each candidate root, or use the
   // provided one.
@@ -638,11 +635,11 @@ static Value buildPredicateList(pdl::PatternOp pattern,
   OptimalBranching::EdgeList bestEdges;
   if (!bestRoot) {
     unsigned bestCost = 0;
-    LLVM_DEBUG(llvm::dbgs() << "Candidate roots:\n");
+    LDBG() << "Candidate roots:";
     for (Value root : roots) {
       OptimalBranching solver(graph, root);
       unsigned cost = solver.solve();
-      LLVM_DEBUG(llvm::dbgs() << "  * " << root << ": " << cost << "\n");
+      LDBG() << "  * " << root << ": " << cost;
       if (!bestRoot || bestCost > cost) {
         bestCost = cost;
         bestRoot = root;
@@ -656,18 +653,15 @@ static Value buildPredicateList(pdl::PatternOp pattern,
   }
 
   // Print the best solution.
-  LLVM_DEBUG({
-    llvm::dbgs() << "Best tree:\n";
-    for (const std::pair<Value, Value> &edge : bestEdges) {
-      llvm::dbgs() << "  * " << edge.first;
-      if (edge.second)
-        llvm::dbgs() << " <- " << edge.second;
-      llvm::dbgs() << "\n";
-    }
-  });
+  LDBG() << "Best tree:";
+  for (const std::pair<Value, Value> &edge : bestEdges) {
+    if (edge.second)
+      LDBG() << "  * " << edge.first << " <- " << edge.second;
+    else
+      LDBG() << "  * " << edge.first;
+  }
 
-  LLVM_DEBUG(llvm::dbgs() << "Calling key getTreePredicates:\n");
-  LLVM_DEBUG(llvm::dbgs() << "  * Value: " << bestRoot << "\n");
+  LDBG() << "Calling key getTreePredicates (Value: " << bestRoot << ")";
 
   // The best root is the starting point for the traversal. Get the tree
   // predicates for the DAG rooted at bestRoot.
@@ -691,7 +685,7 @@ static Value buildPredicateList(pdl::PatternOp pattern,
     // Determine the connector.
     Value connector = graph[target][source].connector;
     assert(connector && "invalid edge");
-    LLVM_DEBUG(llvm::dbgs() << "  * Connector: " << connector.getLoc() << "\n");
+    LDBG() << "  * Connector: " << connector.getLoc();
     DenseMap<Value, OpIndex> parentMap = parentMaps.lookup(target);
     Position *pos = valueToPosition.lookup(connector);
     assert(pos && "connector has not been traversed yet");
diff --git a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
index ba448e46913ac..37cfc9f2c23e6 100644
--- a/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
+++ b/mlir/lib/Conversion/SCFToControlFlow/SCFToControlFlow.cpp
@@ -382,8 +382,11 @@ LogicalResult ForLowering::matchAndRewrite(ForOp forOp,
 
   // With the body block done, we can fill in the condition block.
   rewriter.setInsertionPointToEnd(conditionBlock);
-  auto comparison = arith::CmpIOp::create(
-      rewriter, loc, arith::CmpIPredicate::slt, iv, upperBound);
+  arith::CmpIPredicate predicate = forOp.getUnsignedCmp()
+                                       ? arith::CmpIPredicate::ult
+                                       : arith::CmpIPredicate::slt;
+  auto comparison =
+      arith::CmpIOp::create(rewriter, loc, predicate, iv, upperBound);
 
   cf::CondBranchOp::create(rewriter, loc, comparison, firstBodyBlock,
                            ArrayRef<Value>(), endBlock, ArrayRef<Value>());
diff --git a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
index 84cbd869c78ef..1f239aa5d1aa3 100644
--- a/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
+++ b/mlir/lib/Conversion/SCFToEmitC/SCFToEmitC.cpp
@@ -154,6 +154,10 @@ ForLowering::matchAndRewrite(ForOp forOp, OpAdaptor adaptor,
                              ConversionPatternRewriter &rewriter) const {
   Location loc = forOp.getLoc();
 
+  if (forOp.getUnsignedCmp())
+    return rewriter.notifyMatchFailure(forOp,
+                                       "unsigned loops are not supported");
+
   // Create an emitc::variable op for each result. These variables will be
   // assigned to by emitc::assign ops within the loop body.
   SmallVector<Value> resultVariables;
diff --git a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
index badd2f643925b..7d0a236b6f69a 100644
--- a/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
+++ b/mlir/lib/Conversion/SCFToGPU/SCFToGPU.cpp
@@ -27,7 +27,7 @@
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/RegionUtils.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include <optional>
 
 #define DEBUG_TYPE "loops-to-gpu"
@@ -134,7 +134,7 @@ static LogicalResult checkAffineLoopNestMappable(AffineForOp forOp,
                                                  unsigned numBlockDims,
                                                  unsigned numThreadDims) {
   if (numBlockDims < 1 || numThreadDims < 1) {
-    LLVM_DEBUG(llvm::dbgs() << "nothing to map");
+    LDBG() << "nothing to map";
     return success();
   }
 
diff --git a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
index dc92367fc58cd..55ed31ee695ff 100644
--- a/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
+++ b/mlir/lib/Conversion/SCFToSPIRV/SCFToSPIRV.cpp
@@ -178,8 +178,14 @@ struct ForOpConversion final : SCFToSPIRVPattern<scf::ForOp> {
     // Generate the rest of the loop header.
     rewriter.setInsertionPointToEnd(header);
     auto *mergeBlock = loopOp.getMergeBlock();
-    auto cmpOp = spirv::SLessThanOp::create(rewriter, loc, rewriter.getI1Type(),
-                                            newIndVar, adaptor.getUpperBound());
+    Value cmpOp;
+    if (forOp.getUnsignedCmp()) {
+      cmpOp = spirv::ULessThanOp::create(rewriter, loc, rewriter.getI1Type(),
+                                         newIndVar, adaptor.getUpperBound());
+    } else {
+      cmpOp = spirv::SLessThanOp::create(rewriter, loc, rewriter.getI1Type(),
+                                         newIndVar, adaptor.getUpperBound());
+    }
 
     spirv::BranchConditionalOp::create(rewriter, loc, cmpOp, body,
                                        ArrayRef<Value>(), mergeBlock,
diff --git a/mlir/lib/Conversion/VectorToAMX/CMakeLists.txt b/mlir/lib/Conversion/VectorToAMX/CMakeLists.txt
new file mode 100644
index 0000000000000..2d4b2b6e9283c
--- /dev/null
+++ b/mlir/lib/Conversion/VectorToAMX/CMakeLists.txt
@@ -0,0 +1,19 @@
+add_mlir_conversion_library(MLIRVectorToAMX
+  VectorToAMX.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Conversion/VectorToAMX
+
+  DEPENDS
+  MLIRConversionPassIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRAMXDialect
+  MLIRAffineUtils
+  MLIRArithDialect
+  MLIRLinalgUtils
+  MLIRMemRefDialect
+  MLIRSCFDialect
+  MLIRTransforms
+  MLIRVectorDialect
+  )
diff --git a/mlir/lib/Conversion/VectorToAMX/VectorToAMX.cpp b/mlir/lib/Conversion/VectorToAMX/VectorToAMX.cpp
new file mode 100644
index 0000000000000..a11e9b2624300
--- /dev/null
+++ b/mlir/lib/Conversion/VectorToAMX/VectorToAMX.cpp
@@ -0,0 +1,283 @@
+//===- VectorToAMX.cpp - Convert vector to AMX dialect ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Conversion/VectorToAMX/VectorToAMX.h"
+
+#include "mlir/Dialect/AMX/AMXDialect.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Affine/ViewLikeInterfaceUtils.h"
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+#include <numeric>
+
+namespace mlir {
+#define GEN_PASS_DEF_CONVERTVECTORTOAMX
+#include "mlir/Conversion/Passes.h.inc"
+} // namespace mlir
+
+using namespace mlir;
+
+namespace {
+
+/// Return true if vector shape is compatible with AMX tiles.
+/// The validation accounts for VNNI packing.
+static bool verifyAmxShape(VectorType vec) {
+  // Check overall shape:
+  //   - 2D for plain layout input or output
+  //   - 3D for VNNI packed input
+  if (vec.getRank() != 2 && vec.getRank() != 3)
+    return false;
+
+  ArrayRef<int64_t> shape = vec.getShape();
+  int64_t rows = shape[0];
+  int64_t cols = shape[1];
+  unsigned elemBitWidth = vec.getElementType().getIntOrFloatBitWidth();
+
+  // 3D shape indicates VNNI packed layout.
+  if (vec.getRank() == 3) {
+    int64_t vnniFactor = 32 / elemBitWidth;
+    if (shape.back() != vnniFactor)
+      return false;
+    cols *= vnniFactor;
+  }
+
+  // AMX tile supports up to 16 rows of 64 bytes each.
+  constexpr unsigned maxRows = 16;
+  constexpr unsigned maxBitsPerRow = 64 * 8;
+  return rows <= maxRows && (cols * elemBitWidth) <= maxBitsPerRow;
+}
+
+/// Checks if contraction operands are in AMX-compatible packed VNNI layout.
+static LogicalResult isAmxVnniLayout(PatternRewriter &rewriter,
+                                     vector::ContractionOp contractOp) {
+  VectorType accType = dyn_cast<VectorType>(contractOp.getAcc().getType());
+  if (!accType || accType.getRank() != 2)
+    return rewriter.notifyMatchFailure(contractOp, "Expects acc 2D vector");
+
+  // Expect 3D inputs for VNNI packed data.
+  VectorType lhsType = contractOp.getLhs().getType();
+  VectorType rhsType = contractOp.getRhs().getType();
+  if (lhsType.getRank() != 3 || rhsType.getRank() != 3)
+    return rewriter.notifyMatchFailure(contractOp,
+                                       "Expects lhs and rhs 3D vectors");
+
+  // Check if shapes are compatible with AMX tile.
+  if (!verifyAmxShape(lhsType) || !verifyAmxShape(rhsType) ||
+      !verifyAmxShape(accType))
+    return rewriter.notifyMatchFailure(contractOp, "Invalid operand shape");
+
+  // Validate affine maps.
+  //
+  // Iterators can be ordered arbitrarily. Indexing map positions are based on
+  // operands' target shapes.
+  // The matrix layouts must match the following:
+  //   - matrix A - [M]x[K/vnniFactor]x[vnniFactor]
+  //   - matrix B - [K/vnniFactor]x[N]x[vnniFactor]
+  //   - matrix C - [M]x[N]
+  SmallVector<AffineMap, 4> indexingMaps = contractOp.getIndexingMapsArray();
+  AffineMap mapA = indexingMaps[0];
+  AffineMap mapB = indexingMaps[1];
+  if (mapA.getNumInputs() != 4 || mapA.getNumResults() != 3 ||
+      mapB.getNumResults() != 3)
+    return rewriter.notifyMatchFailure(contractOp,
+                                       "Invalid input indexing maps");
+  FailureOr<linalg::ContractionDimensions> dims =
+      linalg::inferContractionDims(indexingMaps);
+  if (failed(dims))
+    return rewriter.notifyMatchFailure(contractOp,
+                                       "Failed to infer contraction dims");
+  // Two reduction dimensions are expected:
+  //   - one for the K dimension
+  //   - one for the VNNI factor
+  if (dims->k.size() != 2)
+    return rewriter.notifyMatchFailure(contractOp,
+                                       "Expected two reduction dims");
+  assert(dims->m.size() == 1 && dims->n.size() == 1 &&
+         "Invalid parallel contraction dims");
+
+  SmallVector<vector::IteratorType> iteratorTypes =
+      contractOp.getIteratorTypesArray();
+  // Check VNNI dim maps - the innermost dim for A and B inputs.
+  auto vnniDimA = dyn_cast<AffineDimExpr>(mapA.getResult(2));
+  auto vnniDimB = dyn_cast<AffineDimExpr>(mapB.getResult(2));
+  if (!vnniDimA || !vnniDimB || vnniDimA != vnniDimB ||
+      iteratorTypes[vnniDimA.getPosition()] != vector::IteratorType::reduction)
+    return rewriter.notifyMatchFailure(contractOp, "Invalid VNNI dim map");
+  // Check K dim maps - non-transposed row-major layout.
+  auto redDimA = dyn_cast<AffineDimExpr>(mapA.getResult(1));
+  auto redDimB = dyn_cast<AffineDimExpr>(mapB.getResult(0));
+  if (!redDimA || !redDimB || redDimA != redDimB ||
+      iteratorTypes[redDimA.getPosition()] != vector::IteratorType::reduction)
+    return rewriter.notifyMatchFailure(contractOp, "Invalid K dim map");
+  // Check M and N dim maps - map to non-transposed output.
+  AffineMap mapC = indexingMaps[2];
+  auto mDimC = dyn_cast<AffineDimExpr>(mapC.getResult(0));
+  auto nDimC = dyn_cast<AffineDimExpr>(mapC.getResult(1));
+  if (!mDimC || !nDimC)
+    return rewriter.notifyMatchFailure(contractOp, "Invalid acc maps");
+  auto parallelDimA = dyn_cast<AffineDimExpr>(mapA.getResult(0));
+  if (!parallelDimA ||
+      iteratorTypes[parallelDimA.getPosition()] !=
+          vector::IteratorType::parallel ||
+      parallelDimA != mDimC)
+    return rewriter.notifyMatchFailure(contractOp, "Invalid M dim map");
+  auto parallelDimB = dyn_cast<AffineDimExpr>(mapB.getResult(1));
+  if (!parallelDimB ||
+      iteratorTypes[parallelDimB.getPosition()] !=
+          vector::IteratorType::parallel ||
+      parallelDimB != nDimC)
+    return rewriter.notifyMatchFailure(contractOp, "Invalid N dim map");
+
+  return success();
+}
+
+/// Validate contraction operands for AMX lowering.
+static LogicalResult validateOperands(PatternRewriter &rewriter,
+                                      vector::ContractionOp contractOp) {
+  VectorType accType = dyn_cast<VectorType>(contractOp.getAcc().getType());
+  if (!accType)
+    return rewriter.notifyMatchFailure(contractOp, "Expects vector acc");
+
+  // Check if operand types are compatible with AMX compute ops.
+  bool validElemTypes = false;
+  Type lhsElemType = contractOp.getLhs().getType().getElementType();
+  Type rhsElemType = contractOp.getRhs().getType().getElementType();
+  Type accElemType = accType.getElementType();
+  if (accElemType.isInteger(32)) {
+    validElemTypes = lhsElemType.isInteger(8) && rhsElemType.isInteger(8);
+  } else if (accElemType.isF32()) {
+    validElemTypes = (lhsElemType.isF16() && rhsElemType.isF16()) ||
+                     (lhsElemType.isBF16() && rhsElemType.isBF16());
+  }
+  if (!validElemTypes)
+    return rewriter.notifyMatchFailure(contractOp,
+                                       "Invalid combination of operand types");
+
+  if (failed(isAmxVnniLayout(rewriter, contractOp)))
+    return failure();
+
+  return success();
+}
+
+/// Collapses the two innermost dimensions together.
+static Value collapseLastDim(PatternRewriter &rewriter,
+                             TypedValue<MemRefType> memref) {
+  int64_t rank = memref.getType().getRank();
+  SmallVector<ReassociationIndices> reassocIndices;
+  for (auto i : llvm::seq<int64_t>(0, rank - 2))
+    reassocIndices.push_back({i});
+  reassocIndices.push_back({rank - 2, rank - 1});
+  return memref::CollapseShapeOp::create(rewriter, memref.getLoc(), memref,
+                                         reassocIndices);
+}
+
+/// Loads vector values to an AMX tile.
+static TypedValue<amx::TileType> loadTile(PatternRewriter &rewriter,
+                                          TypedValue<VectorType> vec) {
+  Location loc = vec.getLoc();
+  Value zeroIndex = rewriter.createOrFold<arith::ConstantIndexOp>(loc, 0);
+
+  // Transfer the vector to a tile through an intermediate buffer.
+  VectorType vecTy = vec.getType();
+  Value buf = memref::AllocaOp::create(
+      rewriter, loc, MemRefType::get(vecTy.getShape(), vecTy.getElementType()));
+  SmallVector<Value> indices(vecTy.getRank(), zeroIndex);
+  vector::TransferWriteOp::create(rewriter, loc, vec, buf, indices);
+
+  // Collapse the VNNI dimension in case of packing.
+  bool isPacked = vecTy.getRank() == 3;
+  if (isPacked)
+    buf = collapseLastDim(rewriter, cast<TypedValue<MemRefType>>(buf));
+
+  ArrayRef<int64_t> shape = vecTy.getShape();
+  int64_t rows = shape[0];
+  int64_t cols = std::accumulate(shape.begin() + 1, shape.end(), 1,
+                                 std::multiplies<int64_t>());
+  auto tileType = amx::TileType::get({rows, cols}, vecTy.getElementType());
+
+  return amx::TileLoadOp::create(rewriter, loc, tileType, buf,
+                                 {zeroIndex, zeroIndex});
+}
+
+/// Stores an AMX tile in a vector.
+static TypedValue<VectorType> storeTile(PatternRewriter &rewriter,
+                                        TypedValue<amx::TileType> tile) {
+  Location loc = tile.getLoc();
+  Value zeroIndex = rewriter.createOrFold<arith::ConstantIndexOp>(loc, 0);
+
+  // Transfer the tile to a vector through an intermediate buffer.
+  amx::TileType tileTy = tile.getType();
+  Value buf = memref::AllocaOp::create(
+      rewriter, loc,
+      MemRefType::get(tileTy.getShape(), tileTy.getElementType()));
+  SmallVector<Value> indices(2, zeroIndex);
+  amx::TileStoreOp::create(rewriter, loc, buf, indices, tile);
+
+  auto vecTy = VectorType::get(tileTy.getShape(), tileTy.getElementType());
+  return vector::TransferReadOp::create(rewriter, loc, vecTy, buf, indices, {});
+}
+
+struct ContractionToAMX : public OpRewritePattern<vector::ContractionOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::ContractionOp contractOp,
+                                PatternRewriter &rewriter) const override {
+    Location loc = contractOp.getLoc();
+
+    if (contractOp.getKind() != vector::CombiningKind::ADD)
+      return rewriter.notifyMatchFailure(contractOp,
+                                         "Expects add combining kind");
+    if (failed(validateOperands(rewriter, contractOp)))
+      return failure();
+
+    TypedValue<amx::TileType> lhsTile = loadTile(rewriter, contractOp.getLhs());
+    TypedValue<amx::TileType> rhsTile = loadTile(rewriter, contractOp.getRhs());
+    auto acc = dyn_cast<TypedValue<VectorType>>(contractOp.getAcc());
+    assert(acc && "Invalid accumulator type");
+    TypedValue<amx::TileType> accTile = loadTile(rewriter, acc);
+
+    TypedValue<amx::TileType> tileMul;
+    if (acc.getType().getElementType().isFloat()) {
+      tileMul = amx::TileMulFOp::create(rewriter, loc, accTile.getType(),
+                                        lhsTile, rhsTile, accTile);
+    } else {
+      tileMul = amx::TileMulIOp::create(rewriter, loc, accTile.getType(),
+                                        lhsTile, rhsTile, accTile);
+    }
+
+    Value res = storeTile(rewriter, tileMul);
+    rewriter.replaceOp(contractOp, res);
+
+    return success();
+  }
+};
+
+struct ConvertVectorToAMXPass
+    : public impl::ConvertVectorToAMXBase<ConvertVectorToAMXPass> {
+  void runOnOperation() override {
+    MLIRContext &ctx = getContext();
+    RewritePatternSet patterns(&ctx);
+    populateVectorToAMXConversionPatterns(patterns);
+    if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))))
+      return signalPassFailure();
+  }
+};
+
+} // namespace
+
+void mlir::populateVectorToAMXConversionPatterns(RewritePatternSet &patterns) {
+  patterns.add<ContractionToAMX>(patterns.getContext());
+}
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
index 17a79e3815b97..afc3d1b12ac0d 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVM.cpp
@@ -247,8 +247,9 @@ class VectorLoadStoreConversion : public ConvertOpToLLVMPattern<LoadOrStoreOp> {
     MemRefType memRefTy = loadOrStoreOp.getMemRefType();
 
     // Resolve alignment.
-    unsigned align;
-    if (failed(getVectorToLLVMAlignment(*this->getTypeConverter(), vectorTy,
+    unsigned align = loadOrStoreOp.getAlignment().value_or(0);
+    if (!align &&
+        failed(getVectorToLLVMAlignment(*this->getTypeConverter(), vectorTy,
                                         memRefTy, align, useVectorAlignment)))
       return rewriter.notifyMatchFailure(loadOrStoreOp,
                                          "could not resolve alignment");
@@ -1890,15 +1891,21 @@ struct VectorFromElementsLowering
                   ConversionPatternRewriter &rewriter) const override {
     Location loc = fromElementsOp.getLoc();
     VectorType vectorType = fromElementsOp.getType();
-    // TODO: Multi-dimensional vectors lower to !llvm.array<... x vector<>>.
-    // Such ops should be handled in the same way as vector.insert.
+    // Only support 1-D vectors. Multi-dimensional vectors should have been
+    // transformed to 1-D vectors by the vector-to-vector transformations before
+    // this.
     if (vectorType.getRank() > 1)
       return rewriter.notifyMatchFailure(fromElementsOp,
                                          "rank > 1 vectors are not supported");
     Type llvmType = typeConverter->convertType(vectorType);
+    Type llvmIndexType = typeConverter->convertType(rewriter.getIndexType());
     Value result = LLVM::PoisonOp::create(rewriter, loc, llvmType);
-    for (auto [idx, val] : llvm::enumerate(adaptor.getElements()))
-      result = vector::InsertOp::create(rewriter, loc, val, result, idx);
+    for (auto [idx, val] : llvm::enumerate(adaptor.getElements())) {
+      auto constIdx =
+          LLVM::ConstantOp::create(rewriter, loc, llvmIndexType, idx);
+      result = LLVM::InsertElementOp::create(rewriter, loc, llvmType, result,
+                                             val, constIdx);
+    }
     rewriter.replaceOp(fromElementsOp, result);
     return success();
   }
diff --git a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
index cf108690c3741..9852df6970fdc 100644
--- a/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
+++ b/mlir/lib/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.cpp
@@ -94,6 +94,7 @@ void ConvertVectorToLLVMPass::runOnOperation() {
     populateVectorStepLoweringPatterns(patterns);
     populateVectorRankReducingFMAPattern(patterns);
     populateVectorGatherLoweringPatterns(patterns);
+    populateVectorFromElementsLoweringPatterns(patterns);
     if (armI8MM) {
       if (armNeon)
         arm_neon::populateLowerContractionToNeonI8MMPatterns(patterns);
diff --git a/mlir/lib/Conversion/VectorToXeGPU/CMakeLists.txt b/mlir/lib/Conversion/VectorToXeGPU/CMakeLists.txt
index 567083da00239..e9ad67c52820d 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/CMakeLists.txt
+++ b/mlir/lib/Conversion/VectorToXeGPU/CMakeLists.txt
@@ -13,4 +13,5 @@ add_mlir_conversion_library(MLIRVectorToXeGPU
   MLIRTransforms
   MLIRVectorDialect
   MLIRXeGPUDialect
+  MLIRXeGPUUtils
   )
diff --git a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
index 80107554144cf..819c2e5973ffd 100644
--- a/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
+++ b/mlir/lib/Conversion/VectorToXeGPU/VectorToXeGPU.cpp
@@ -14,9 +14,11 @@
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/StructuredOpsUtils.h"
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
+#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/TypeSwitch.h"
@@ -68,11 +70,6 @@ static LogicalResult transferPreconditions(PatternRewriter &rewriter,
   if (!srcTy)
     return rewriter.notifyMatchFailure(xferOp, "Expects memref source");
 
-  // Perform common data transfer checks.
-  VectorType vecTy = xferOp.getVectorType();
-  if (failed(storeLoadPreconditions(rewriter, xferOp, vecTy)))
-    return failure();
-
   // Validate further transfer op semantics.
   SmallVector<int64_t> strides;
   int64_t offset;
@@ -80,6 +77,7 @@ static LogicalResult transferPreconditions(PatternRewriter &rewriter,
     return rewriter.notifyMatchFailure(
         xferOp, "Buffer must be contiguous in the innermost dimension");
 
+  VectorType vecTy = xferOp.getVectorType();
   unsigned vecRank = vecTy.getRank();
   if (xferOp.hasOutOfBoundsDim() && vecRank < 2)
     return rewriter.notifyMatchFailure(
@@ -155,6 +153,277 @@ createNdDescriptor(PatternRewriter &rewriter, Location loc,
   return ndDesc;
 }
 
+// Adjusts the strides of a memref according to a given permutation map for
+// vector operations.
+//
+// This function updates the innermost strides in the `strides` array to
+// reflect the permutation specified by `permMap`. The permutation is computed
+// using the inverse and broadcasting-aware version of the permutation map,
+// and is applied to the relevant strides. This ensures that memory accesses
+// are consistent with the logical permutation of vector elements.
+//
+// Example:
+//   Suppose we have a memref of rank 4 with strides `[s0, s1, s2, s3]`.
+//   If the permutation map swaps the last two dimensions (e.g., [0, 1] -> [1,
+//   0]), then after calling this function, the last two strides will be
+//   swapped:
+//     Original strides: [s0, s1, s2, s3]
+//     After permutation: [s0, s1, s3, s2]
+//
+static void adjustStridesForPermutation(AffineMap permMap,
+                                        SmallVectorImpl<Value> &strides) {
+
+  AffineMap invMap = inverseAndBroadcastProjectedPermutation(permMap);
+  SmallVector<unsigned> perms;
+  invMap.isPermutationOfMinorIdentityWithBroadcasting(perms);
+  SmallVector<int64_t> perms64(perms.begin(), perms.end());
+  strides = applyPermutation(strides, perms64);
+}
+
+// Computes memory strides for vector transfer operations, handling both
+// static and dynamic memrefs while applying permutation transformations
+// for XeGPU lowering.
+static SmallVector<Value> computeStrides(VectorTransferOpInterface xferOp,
+                                         PatternRewriter &rewriter) {
+  SmallVector<Value> strides;
+  Value baseMemref = xferOp.getBase();
+  AffineMap permMap = xferOp.getPermutationMap();
+  MemRefType memrefType = dyn_cast<MemRefType>(baseMemref.getType());
+
+  Location loc = xferOp.getLoc();
+  if (memrefType.hasStaticShape()) {
+    int64_t offset;
+    SmallVector<int64_t> intStrides;
+    if (failed(memrefType.getStridesAndOffset(intStrides, offset)))
+      return {};
+    // Wrap static strides as MLIR values
+    for (int64_t s : intStrides)
+      strides.push_back(arith::ConstantIndexOp::create(rewriter, loc, s));
+  } else {
+    // For dynamic shape memref, use memref.extract_strided_metadata to get
+    // stride values
+    unsigned rank = memrefType.getRank();
+    Type indexType = rewriter.getIndexType();
+
+    // Result types: [base_memref, offset, stride0, stride1, ..., strideN-1,
+    // size0, size1, ..., sizeN-1]
+    SmallVector<Type> resultTypes;
+    resultTypes.push_back(MemRefType::get(
+        {}, memrefType.getElementType())); // base memref (unranked)
+    resultTypes.push_back(indexType);      // offset
+
+    for (unsigned i = 0; i < rank; ++i)
+      resultTypes.push_back(indexType); // strides
+
+    for (unsigned i = 0; i < rank; ++i)
+      resultTypes.push_back(indexType); // sizes
+
+    auto meta = memref::ExtractStridedMetadataOp::create(
+        rewriter, loc, resultTypes, baseMemref);
+    strides.append(meta.getStrides().begin(), meta.getStrides().end());
+  }
+  // Adjust strides according to the permutation map (e.g., for transpose)
+  adjustStridesForPermutation(permMap, strides);
+  return strides;
+}
+
+// This function compute the vectors of localOffsets for scattered load/stores.
+// It is used in the lowering of vector.transfer_read/write to
+// load_gather/store_scatter Example:
+//   %0 = vector.transfer_read %expand_shape[%block_id_y, %c0, %c0, %c0, %c0],
+//               %cst {in_bounds = [true, true, true, true]}>} :
+//               memref<8x4x2x6x32xbf16>, vector<4x2x6x32xbf16>
+//
+//   %6 = vector.step: vector<4xindex>
+//   %7 = vector.step: vector<2xindex>
+//   %8 = vector.step: vector<6xindex>
+//   %9 = vector.step: vector<32xindex>
+//   %10 = arith.mul %6, 384
+//   %11 = arith.mul %7, 192
+//   %12 = arith.mul %8, 32
+//   %13 = arith.mul %9, 1
+//   %14 = vector.shape_cast %10: vector<4xindex> -> vector<4x1x1x1xbf16>
+//   %15 = vector.shape_cast %11: vector<2xindex> -> vector<1x2x1x1xbf16>
+//   %16 = vector.shape_cast %12: vector<6xindex> -> vector<1x1x6x1xbf16>
+//   %17 = vector.shape_cast %13: vector<32xindex> -> vector<1x1x1x32xbf16>
+//   %18 = vector.broadcast %14: vector<4x1x1x1xbf16> -> vector<4x2x6x32xindex>
+//   %19 = vector.broadcast %15: vector<1x2x1x1xbf16> -> vector<4x2x6x32xindex>
+//   %20 = vector.broadcast %16: vector<1x1x6x1xbf16> -> vector<4x2x6x32xindex>
+//   %21 = vector.broadcast %17: vector<1x1x1x32xbf16> -> vector<4x2x6x32xindex>
+//   %22 = arith.add %18, %19
+//   %23 = arith.add %20, %21
+//   %local_offsets = arith.add %22, %23
+//   %orig_offset = %block_id_y * 4x2x6x32 // consider using affine map
+//   %offsets =  orig_offset + local_offsets
+static Value computeOffsets(VectorTransferOpInterface xferOp,
+                            PatternRewriter &rewriter,
+                            ArrayRef<Value> strides) {
+  Location loc = xferOp.getLoc();
+  VectorType vectorType = xferOp.getVectorType();
+  SmallVector<Value> indices(xferOp.getIndices().begin(),
+                             xferOp.getIndices().end());
+  ArrayRef<int64_t> vectorShape = vectorType.getShape();
+
+  // Create vector.step operations for each dimension
+  SmallVector<Value> stepVectors;
+  llvm::map_to_vector(vectorShape, [&](int64_t dim) {
+    auto stepType = VectorType::get({dim}, rewriter.getIndexType());
+    auto stepOp = vector::StepOp::create(rewriter, loc, stepType);
+    stepVectors.push_back(stepOp);
+    return stepOp;
+  });
+
+  // Multiply step vectors by corresponding strides
+  size_t memrefRank = strides.size();
+  size_t vectorRank = vectorShape.size();
+  SmallVector<Value> strideMultiplied;
+  for (size_t i = 0; i < vectorRank; ++i) {
+    size_t memrefDim = memrefRank - vectorRank + i;
+    Value strideValue = strides[memrefDim];
+    auto mulType = dyn_cast<VectorType>(stepVectors[i].getType());
+    auto bcastOp =
+        vector::BroadcastOp::create(rewriter, loc, mulType, strideValue);
+    auto mulOp = arith::MulIOp::create(rewriter, loc, stepVectors[i], bcastOp);
+    strideMultiplied.push_back(mulOp);
+  }
+
+  // Shape cast each multiplied vector to add singleton dimensions
+  SmallVector<Value> shapeCasted;
+  for (size_t i = 0; i < vectorRank; ++i) {
+    SmallVector<int64_t> newShape(vectorRank, 1);
+    newShape[i] = vectorShape[i];
+    auto newType = VectorType::get(newShape, rewriter.getIndexType());
+    auto castOp = vector::ShapeCastOp::create(rewriter, loc, newType,
+                                              strideMultiplied[i]);
+    shapeCasted.push_back(castOp);
+  }
+
+  // Broadcast each shape-casted vector to full vector shape
+  SmallVector<Value> broadcasted;
+  auto fullIndexVectorType =
+      VectorType::get(vectorShape, rewriter.getIndexType());
+  for (Value shapeCastVal : shapeCasted) {
+    auto broadcastOp = vector::BroadcastOp::create(
+        rewriter, loc, fullIndexVectorType, shapeCastVal);
+    broadcasted.push_back(broadcastOp);
+  }
+
+  // Add all broadcasted vectors together to compute local offsets
+  Value localOffsets = broadcasted[0];
+  for (size_t i = 1; i < broadcasted.size(); ++i)
+    localOffsets =
+        arith::AddIOp::create(rewriter, loc, localOffsets, broadcasted[i]);
+
+  // Compute base offset from transfer read indices
+  Value baseOffset = nullptr;
+  if (!indices.empty()) {
+    baseOffset = arith::ConstantIndexOp::create(rewriter, loc, 0);
+    for (size_t i = 0; i < indices.size(); ++i) {
+      Value strideVal = strides[i];
+      Value offsetContrib =
+          arith::MulIOp::create(rewriter, loc, indices[i], strideVal);
+      baseOffset =
+          arith::AddIOp::create(rewriter, loc, baseOffset, offsetContrib);
+    }
+    // Broadcast base offset to match vector shape
+    Value bcastBase = vector::BroadcastOp::create(
+        rewriter, loc, fullIndexVectorType, baseOffset);
+    localOffsets =
+        arith::AddIOp::create(rewriter, loc, bcastBase, localOffsets);
+  }
+  return localOffsets;
+}
+
+// Collapse memref shape to 1D
+static Value collapseMemrefTo1D(VectorTransferOpInterface xferOp,
+                                PatternRewriter &rewriter) {
+  Location loc = xferOp.getLoc();
+
+  Value baseMemref = xferOp.getBase();
+  MemRefType memrefType = dyn_cast<MemRefType>(baseMemref.getType());
+  Type elementType = memrefType.getElementType();
+
+  // Compute the total number of elements in the memref
+  MemRefType flatMemrefType;
+  if (memrefType.hasStaticShape()) {
+    auto totalElements = memrefType.getNumElements();
+    flatMemrefType = MemRefType::get({totalElements}, elementType);
+  } else {
+    flatMemrefType = MemRefType::get({ShapedType::kDynamic}, elementType);
+  }
+
+  SmallVector<ReassociationIndices> reassociation;
+  ReassociationIndices allDims =
+      llvm::to_vector(llvm::seq<int64_t>(0, memrefType.getRank()));
+  reassociation.push_back(allDims);
+
+  auto collapseOp = memref::CollapseShapeOp::create(
+      rewriter, loc, flatMemrefType, baseMemref, reassociation);
+  return collapseOp;
+}
+
+static LogicalResult lowerToScatteredLoadOp(vector::TransferReadOp readOp,
+                                            PatternRewriter &rewriter) {
+
+  Location loc = readOp.getLoc();
+  VectorType vectorType = readOp.getVectorType();
+  ArrayRef<int64_t> vectorShape = vectorType.getShape();
+  auto memrefType = dyn_cast<MemRefType>(readOp.getShapedType());
+  if (!memrefType)
+    return rewriter.notifyMatchFailure(readOp, "Expected memref source");
+
+  SmallVector<Value> strides = computeStrides(readOp, rewriter);
+  if (strides.empty())
+    return rewriter.notifyMatchFailure(readOp, "Failed to compute strides");
+
+  Value localOffsets = computeOffsets(readOp, rewriter, strides);
+
+  Value flatMemref = collapseMemrefTo1D(readOp, rewriter);
+
+  Value mask = vector::ConstantMaskOp::create(
+      rewriter, loc, VectorType::get(vectorShape, rewriter.getI1Type()),
+      vectorShape);
+  auto gatherOp = xegpu::LoadGatherOp::create(
+      rewriter, loc, vectorType, flatMemref, localOffsets, mask,
+      /*chunk_size=*/IntegerAttr{},
+      /*l1_hint=*/xegpu::CachePolicyAttr{},
+      /*l2_hint=*/xegpu::CachePolicyAttr{},
+      /*l3_hint=*/xegpu::CachePolicyAttr{});
+
+  rewriter.replaceOp(readOp, gatherOp.getResult());
+  return success();
+}
+
+static LogicalResult lowerToScatteredStoreOp(vector::TransferWriteOp writeOp,
+                                             PatternRewriter &rewriter) {
+
+  Location loc = writeOp.getLoc();
+  VectorType vectorType = writeOp.getVectorType();
+  ArrayRef<int64_t> vectorShape = vectorType.getShape();
+
+  auto memrefType = dyn_cast<MemRefType>(writeOp.getShapedType());
+  if (!memrefType)
+    return rewriter.notifyMatchFailure(writeOp, "Expected memref source");
+
+  SmallVector<Value> strides = computeStrides(writeOp, rewriter);
+
+  Value localOffsets = computeOffsets(writeOp, rewriter, strides);
+
+  Value flatMemref = collapseMemrefTo1D(writeOp, rewriter);
+
+  Value mask = vector::ConstantMaskOp::create(
+      rewriter, loc, VectorType::get(vectorShape, rewriter.getI1Type()),
+      vectorShape);
+  xegpu::StoreScatterOp::create(rewriter, loc, writeOp.getVector(), flatMemref,
+                                localOffsets, mask,
+                                /*chunk_size=*/IntegerAttr{},
+                                /*l1_hint=*/xegpu::CachePolicyAttr{},
+                                /*l2_hint=*/xegpu::CachePolicyAttr{},
+                                /*l3_hint=*/xegpu::CachePolicyAttr{});
+  rewriter.eraseOp(writeOp);
+  return success();
+}
+
 struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
   using OpRewritePattern<vector::TransferReadOp>::OpRewritePattern;
 
@@ -165,6 +434,22 @@ struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
     if (failed(transferPreconditions(rewriter, readOp)))
       return failure();
 
+    // TODO:This check needs to be replaced with proper uArch capability check
+    auto chip = xegpu::getChipStr(readOp);
+    if (chip != "pvc" && chip != "bmg") {
+      // lower to scattered load Op if the target HW doesn't have 2d block load
+      // support
+      // TODO: add support for OutOfBound access
+      if (readOp.hasOutOfBoundsDim())
+        return failure();
+      return lowerToScatteredLoadOp(readOp, rewriter);
+    }
+
+    // Perform common data transfer checks.
+    VectorType vecTy = readOp.getVectorType();
+    if (failed(storeLoadPreconditions(rewriter, readOp, vecTy)))
+      return failure();
+
     bool isOutOfBounds = readOp.hasOutOfBoundsDim();
     if (isOutOfBounds && !isZeroConstant(readOp.getPadding()))
       return rewriter.notifyMatchFailure(
@@ -173,7 +458,6 @@ struct TransferReadLowering : public OpRewritePattern<vector::TransferReadOp> {
     AffineMap readMap = readOp.getPermutationMap();
     bool isTransposeLoad = !readMap.isMinorIdentity();
 
-    VectorType vecTy = readOp.getVectorType();
     Type elementType = vecTy.getElementType();
     unsigned minTransposeBitWidth = 32;
     if (isTransposeLoad &&
@@ -221,11 +505,26 @@ struct TransferWriteLowering
     if (failed(transferPreconditions(rewriter, writeOp)))
       return failure();
 
+    // TODO:This check needs to be replaced with proper uArch capability check
+    auto chip = xegpu::getChipStr(writeOp);
+    if (chip != "pvc" && chip != "bmg") {
+      // lower to scattered store Op if the target HW doesn't have 2d block
+      // store support
+      // TODO: add support for OutOfBound access
+      if (writeOp.hasOutOfBoundsDim())
+        return failure();
+      return lowerToScatteredStoreOp(writeOp, rewriter);
+    }
+
+    // Perform common data transfer checks.
+    VectorType vecTy = writeOp.getVectorType();
+    if (failed(storeLoadPreconditions(rewriter, writeOp, vecTy)))
+      return failure();
+
     AffineMap map = writeOp.getPermutationMap();
     if (!map.isMinorIdentity())
       return rewriter.notifyMatchFailure(writeOp, "Expects identity map");
 
-    VectorType vecTy = writeOp.getVectorType();
     auto descType = xegpu::TensorDescType::get(
         vecTy.getShape(), vecTy.getElementType(),
         /*array_length=*/1, /*boundary_check=*/writeOp.hasOutOfBoundsDim(),
diff --git a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
index 9a0a230e8abca..11a40d663a201 100644
--- a/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
+++ b/mlir/lib/Dialect/AMDGPU/IR/AMDGPUDialect.cpp
@@ -510,6 +510,18 @@ LogicalResult DPPOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// PermlaneSwapOp
+//===----------------------------------------------------------------------===//
+LogicalResult PermlaneSwapOp::verify() {
+  unsigned rowLength = getRowLength();
+
+  if (rowLength != 16 && rowLength != 32)
+    return emitOpError("row_length attribute must either be 16 or 32.");
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // GatherToLDSOp
 //===----------------------------------------------------------------------===//
@@ -518,8 +530,8 @@ LogicalResult GatherToLDSOp::verify() {
   MemRefType srcType = cast<MemRefType>(getSrc().getType());
   MemRefType dstType = cast<MemRefType>(getDst().getType());
 
-  if (!dstType.areTrailingDimsContiguous(dstType.getRank()))
-    return emitOpError("destination types must be contiguous");
+  if (!dstType.areTrailingDimsContiguous(1))
+    return emitOpError("destination type inner most dim must be contiguous");
 
   auto elemType = srcType.getElementType();
   // Check $src and $dst element types are the same.
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/FoldMemRefsOps.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/FoldMemRefsOps.cpp
index a3fdc7ee385ed..d54751098410b 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/FoldMemRefsOps.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/FoldMemRefsOps.cpp
@@ -28,62 +28,79 @@ struct AmdgpuFoldMemRefOpsPass final
   }
 };
 
+static LogicalResult foldMemrefViewOp(PatternRewriter &rewriter, Location loc,
+                                      Value view, mlir::OperandRange indices,
+                                      SmallVectorImpl<Value> &resolvedIndices,
+                                      Value &memrefBase, StringRef role) {
+  Operation *defOp = view.getDefiningOp();
+  if (!defOp) {
+    return failure();
+  }
+  return llvm::TypeSwitch<Operation *, LogicalResult>(defOp)
+      .Case<memref::SubViewOp>([&](memref::SubViewOp subviewOp) {
+        mlir::affine::resolveIndicesIntoOpWithOffsetsAndStrides(
+            rewriter, loc, subviewOp.getMixedOffsets(),
+            subviewOp.getMixedStrides(), subviewOp.getDroppedDims(), indices,
+            resolvedIndices);
+        memrefBase = subviewOp.getSource();
+        return success();
+      })
+      .Case<memref::ExpandShapeOp>([&](memref::ExpandShapeOp expandShapeOp) {
+        if (failed(mlir::memref::resolveSourceIndicesExpandShape(
+                loc, rewriter, expandShapeOp, indices, resolvedIndices,
+                false))) {
+          return failure();
+        }
+        memrefBase = expandShapeOp.getViewSource();
+        return success();
+      })
+      .Case<memref::CollapseShapeOp>(
+          [&](memref::CollapseShapeOp collapseShapeOp) {
+            if (failed(mlir::memref::resolveSourceIndicesCollapseShape(
+                    loc, rewriter, collapseShapeOp, indices,
+                    resolvedIndices))) {
+              return failure();
+            }
+            memrefBase = collapseShapeOp.getViewSource();
+            return success();
+          })
+      .Default([&](Operation *op) {
+        return rewriter.notifyMatchFailure(
+            op, (role + " producer is not one of SubViewOp, ExpandShapeOp, or "
+                        "CollapseShapeOp")
+                    .str());
+      });
+}
+
 struct FoldMemRefOpsIntoGatherToLDSOp final : OpRewritePattern<GatherToLDSOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(GatherToLDSOp op,
                                 PatternRewriter &rewriter) const override {
     Location loc = op.getLoc();
 
-    Value memrefSource;
-    SmallVector<Value> sourceIndices;
-    auto foldResult =
-        llvm::TypeSwitch<Operation *, LogicalResult>(
-            op.getSrc().getDefiningOp())
-            .Case<memref::SubViewOp>([&](memref::SubViewOp subviewOp) {
-              // If the source is a SubViewOp, we can directly rewrite the
-              // GatherToLDSOp.
-              mlir::affine::resolveIndicesIntoOpWithOffsetsAndStrides(
-                  rewriter, loc, subviewOp.getMixedOffsets(),
-                  subviewOp.getMixedStrides(), subviewOp.getDroppedDims(),
-                  op.getSrcIndices(), sourceIndices);
-              memrefSource = subviewOp.getSource();
-              return success();
-            })
-            .Case<memref::ExpandShapeOp>(
-                [&](memref::ExpandShapeOp expandShapeOp) {
-                  if (failed(mlir::memref::resolveSourceIndicesExpandShape(
-                          loc, rewriter, expandShapeOp, op.getSrcIndices(),
-                          sourceIndices, false))) {
-                    return failure();
-                  }
-                  memrefSource = expandShapeOp.getViewSource();
-                  return success();
-                })
-            .Case<memref::CollapseShapeOp>(
-                [&](memref::CollapseShapeOp collapseShapeOp) {
-                  if (failed(mlir::memref::resolveSourceIndicesCollapseShape(
-                          loc, rewriter, collapseShapeOp, op.getSrcIndices(),
-                          sourceIndices))) {
-                    return failure();
-                  }
-                  memrefSource = collapseShapeOp.getViewSource();
-                  return success();
-                })
-            .Default([&](Operation *op) {
-              // If the source is not a SubViewOp, ExpandShapeOp, or
-              // CollapseShapeOp, we cannot fold the GatherToLDSOp.
-              return rewriter.notifyMatchFailure(
-                  op,
-                  "source producer is not one of SubViewOp, ExpandShapeOp, or "
-                  "CollapseShapeOp");
-            });
+    SmallVector<Value> sourceIndices, destIndices;
+    Value memrefSource, memrefDest;
+
+    auto foldSrcResult =
+        foldMemrefViewOp(rewriter, loc, op.getSrc(), op.getSrcIndices(),
+                         sourceIndices, memrefSource, "source");
+
+    if (failed(foldSrcResult)) {
+      memrefSource = op.getSrc();
+      sourceIndices = op.getSrcIndices();
+    }
+
+    auto foldDstResult =
+        foldMemrefViewOp(rewriter, loc, op.getDst(), op.getDstIndices(),
+                         destIndices, memrefDest, "destination");
 
-    if (failed(foldResult)) {
-      return failure();
+    if (failed(foldDstResult)) {
+      memrefDest = op.getDst();
+      destIndices = op.getDstIndices();
     }
 
     rewriter.replaceOpWithNewOp<GatherToLDSOp>(op, memrefSource, sourceIndices,
-                                               op.getDst(), op.getDstIndices(),
+                                               memrefDest, destIndices,
                                                op.getTransferType());
 
     return success();
diff --git a/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp b/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
index 86edc2bcc2761..b405ec2201bf8 100644
--- a/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/AffineStructures.cpp
@@ -93,13 +93,13 @@ FlatAffineValueConstraints::addAffineForOpDomain(AffineForOp forOp) {
       int64_t lb = forOp.getConstantLowerBound();
       dividend[pos] = 1;
       dividend.back() -= lb;
-      addLocalFloorDiv(dividend, step);
+      unsigned qPos = addLocalFloorDiv(dividend, step);
       // Second constraint: (iv - lb) - step * q = 0.
       SmallVector<int64_t, 8> eq(getNumCols(), 0);
       eq[pos] = 1;
       eq.back() -= lb;
       // For the local var just added above.
-      eq[getNumCols() - 2] = -step;
+      eq[qPos] = -step;
       addEquality(eq);
     }
   }
diff --git a/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp b/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
index 2f85e0b5da043..166d39e88d41e 100644
--- a/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/LoopAnalysis.cpp
@@ -21,6 +21,7 @@
 #include "llvm/Support/MathExtras.h"
 
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include <numeric>
 #include <optional>
 
@@ -548,19 +549,19 @@ bool mlir::affine::isTilingValid(ArrayRef<AffineForOp> loops) {
         // Check whether there is any negative direction vector in the
         // dependence components found above, which means that dependence is
         // violated by the default hyper-rect tiling method.
-        LLVM_DEBUG(llvm::dbgs() << "Checking whether tiling legality violated "
-                                   "for dependence at depth: "
-                                << Twine(d) << " between:\n";);
-        LLVM_DEBUG(srcAccess.opInst->dump());
-        LLVM_DEBUG(dstAccess.opInst->dump());
+        LDBG() << "Checking whether tiling legality violated "
+               << "for dependence at depth: " << Twine(d) << " between:"
+               << OpWithFlags(srcAccess.opInst, OpPrintingFlags().skipRegions())
+               << "\nand:\n"
+               << OpWithFlags(dstAccess.opInst,
+                              OpPrintingFlags().skipRegions());
         for (const DependenceComponent &depComp : depComps) {
           if (depComp.lb.has_value() && depComp.ub.has_value() &&
               *depComp.lb < *depComp.ub && *depComp.ub < 0) {
-            LLVM_DEBUG(llvm::dbgs()
-                       << "Dependence component lb = " << Twine(*depComp.lb)
-                       << " ub = " << Twine(*depComp.ub)
-                       << " is negative  at depth: " << Twine(d)
-                       << " and thus violates the legality rule.\n");
+            LDBG() << "Dependence component lb = " << Twine(*depComp.lb)
+                   << " ub = " << Twine(*depComp.ub)
+                   << " is negative  at depth: " << Twine(d)
+                   << " and thus violates the legality rule.";
             return false;
           }
         }
diff --git a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
index a89c1ae475b96..99ea20bf13b49 100644
--- a/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
+++ b/mlir/lib/Dialect/Affine/Analysis/Utils.cpp
@@ -22,6 +22,7 @@
 #include "mlir/IR/IntegerSet.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
@@ -241,7 +242,7 @@ addNodeToMDG(Operation *nodeOp, MemRefDependenceGraph &mdg,
 }
 
 bool MemRefDependenceGraph::init() {
-  LLVM_DEBUG(llvm::dbgs() << "--- Initializing MDG ---\n");
+  LDBG() << "--- Initializing MDG ---";
   // Map from a memref to the set of ids of the nodes that have ops accessing
   // the memref.
   DenseMap<Value, SetVector<unsigned>> memrefAccesses;
@@ -288,8 +289,7 @@ bool MemRefDependenceGraph::init() {
       // Return false if non-handled/unknown region-holding ops are found. We
       // won't know what such ops do or what its regions mean; for e.g., it may
       // not be an imperative op.
-      LLVM_DEBUG(llvm::dbgs()
-                 << "MDG init failed; unknown region-holding op found!\n");
+      LDBG() << "MDG init failed; unknown region-holding op found!";
       return false;
     }
     // We aren't creating nodes for memory-effect free ops either with no
@@ -297,7 +297,7 @@ bool MemRefDependenceGraph::init() {
     // interface.
   }
 
-  LLVM_DEBUG(llvm::dbgs() << "Created " << nodes.size() << " nodes\n");
+  LDBG() << "Created " << nodes.size() << " nodes";
 
   // Add dependence edges between nodes which produce SSA values and their
   // users. Load ops can be considered as the ones producing SSA values.
@@ -556,9 +556,8 @@ MemRefDependenceGraph::getFusedLoopNestInsertionPoint(unsigned srcId,
   gatherDefiningNodes(dstId, definingNodes);
   if (llvm::any_of(definingNodes,
                    [&](unsigned id) { return hasDependencePath(srcId, id); })) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "Can't fuse: a defining op with a user in the dst "
-                  "loop has dependence from the src loop\n");
+    LDBG() << "Can't fuse: a defining op with a user in the dst "
+           << "loop has dependence from the src loop";
     return nullptr;
   }
 
@@ -957,20 +956,20 @@ std::optional<bool> ComputationSliceState::isSliceValid() const {
   FlatAffineValueConstraints srcConstraints;
   // TODO: Store the source's domain to avoid computation at each depth.
   if (failed(getSourceAsConstraints(srcConstraints))) {
-    LLVM_DEBUG(llvm::dbgs() << "Unable to compute source's domain\n");
+    LDBG() << "Unable to compute source's domain";
     return std::nullopt;
   }
   // As the set difference utility currently cannot handle symbols in its
   // operands, validity of the slice cannot be determined.
   if (srcConstraints.getNumSymbolVars() > 0) {
-    LLVM_DEBUG(llvm::dbgs() << "Cannot handle symbols in source domain\n");
+    LDBG() << "Cannot handle symbols in source domain";
     return std::nullopt;
   }
   // TODO: Handle local vars in the source domains while using the 'projectOut'
   // utility below. Currently, aligning is not done assuming that there will be
   // no local vars in the source domain.
   if (srcConstraints.getNumLocalVars() != 0) {
-    LLVM_DEBUG(llvm::dbgs() << "Cannot handle locals in source domain\n");
+    LDBG() << "Cannot handle locals in source domain";
     return std::nullopt;
   }
 
@@ -978,7 +977,7 @@ std::optional<bool> ComputationSliceState::isSliceValid() const {
   // fusion succeeds.
   FlatAffineValueConstraints sliceConstraints;
   if (failed(getAsConstraints(&sliceConstraints))) {
-    LLVM_DEBUG(llvm::dbgs() << "Unable to compute slice's domain\n");
+    LDBG() << "Unable to compute slice's domain";
     return std::nullopt;
   }
 
@@ -987,11 +986,11 @@ std::optional<bool> ComputationSliceState::isSliceValid() const {
   sliceConstraints.projectOut(ivs.size(),
                               sliceConstraints.getNumVars() - ivs.size());
 
-  LLVM_DEBUG(llvm::dbgs() << "Domain of the source of the slice:\n");
-  LLVM_DEBUG(srcConstraints.dump());
-  LLVM_DEBUG(llvm::dbgs() << "Domain of the slice if this fusion succeeds "
-                             "(expressed in terms of its source's IVs):\n");
-  LLVM_DEBUG(sliceConstraints.dump());
+  LDBG() << "Domain of the source of the slice:\n"
+         << "Source constraints:" << srcConstraints
+         << "\nDomain of the slice if this fusion succeeds "
+         << "(expressed in terms of its source's IVs):\n"
+         << "Slice constraints:" << sliceConstraints;
 
   // TODO: Store 'srcSet' to avoid recalculating for each depth.
   PresburgerSet srcSet(srcConstraints);
@@ -999,7 +998,7 @@ std::optional<bool> ComputationSliceState::isSliceValid() const {
   PresburgerSet diffSet = sliceSet.subtract(srcSet);
 
   if (!diffSet.isIntegerEmpty()) {
-    LLVM_DEBUG(llvm::dbgs() << "Incorrect slice\n");
+    LDBG() << "Incorrect slice";
     return false;
   }
   return true;
@@ -1172,8 +1171,7 @@ LogicalResult MemRefRegion::compute(Operation *op, unsigned loopDepth,
 
   unsigned rank = access.getRank();
 
-  LLVM_DEBUG(llvm::dbgs() << "MemRefRegion::compute: " << *op
-                          << "\ndepth: " << loopDepth << "\n";);
+  LDBG() << "MemRefRegion::compute: " << *op << " depth: " << loopDepth;
 
   // 0-d memrefs.
   if (rank == 0) {
@@ -1236,7 +1234,7 @@ LogicalResult MemRefRegion::compute(Operation *op, unsigned loopDepth,
       if (auto constVal = getConstantIntValue(symbol))
         cst.addBound(BoundType::EQ, symbol, constVal.value());
     } else {
-      LLVM_DEBUG(llvm::dbgs() << "unknown affine dimensional value");
+      LDBG() << "unknown affine dimensional value";
       return failure();
     }
   }
@@ -1260,7 +1258,7 @@ LogicalResult MemRefRegion::compute(Operation *op, unsigned loopDepth,
   // Add access function equalities to connect loop IVs to data dimensions.
   if (failed(cst.composeMap(&accessValueMap))) {
     op->emitError("getMemRefRegion: compose affine map failed");
-    LLVM_DEBUG(accessValueMap.getAffineMap().dump());
+    LDBG() << "Access map: " << accessValueMap.getAffineMap();
     return failure();
   }
 
@@ -1317,8 +1315,7 @@ LogicalResult MemRefRegion::compute(Operation *op, unsigned loopDepth,
   }
   cst.removeTrivialRedundancy();
 
-  LLVM_DEBUG(llvm::dbgs() << "Memory region:\n");
-  LLVM_DEBUG(cst.dump());
+  LDBG() << "Memory region: " << cst;
   return success();
 }
 
@@ -1346,14 +1343,14 @@ std::optional<int64_t> MemRefRegion::getRegionSize() {
   auto memRefType = cast<MemRefType>(memref.getType());
 
   if (!memRefType.getLayout().isIdentity()) {
-    LLVM_DEBUG(llvm::dbgs() << "Non-identity layout map not yet supported\n");
+    LDBG() << "Non-identity layout map not yet supported";
     return false;
   }
 
   // Compute the extents of the buffer.
   std::optional<int64_t> numElements = getConstantBoundingSizeAndShape();
   if (!numElements) {
-    LLVM_DEBUG(llvm::dbgs() << "Dynamic shapes not yet supported\n");
+    LDBG() << "Dynamic shapes not yet supported";
     return std::nullopt;
   }
   auto eltSize = getMemRefIntOrFloatEltSizeInBytes(memRefType);
@@ -1397,8 +1394,7 @@ LogicalResult mlir::affine::boundCheckLoadOrStoreOp(LoadOrStoreOp loadOrStoreOp,
                             /*addMemRefDimBounds=*/false)))
     return success();
 
-  LLVM_DEBUG(llvm::dbgs() << "Memory region");
-  LLVM_DEBUG(region.getConstraints()->dump());
+  LDBG() << "Memory region: " << region.getConstraints();
 
   bool outOfBounds = false;
   unsigned rank = loadOrStoreOp.getMemRefType().getRank();
@@ -1558,7 +1554,7 @@ mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
       // Check if 'loopDepth' exceeds nesting depth of src/dst ops.
       if ((!isBackwardSlice && loopDepth > getNestingDepth(a)) ||
           (isBackwardSlice && loopDepth > getNestingDepth(b))) {
-        LLVM_DEBUG(llvm::dbgs() << "Invalid loop depth\n");
+        LDBG() << "Invalid loop depth";
         return SliceComputationResult::GenericFailure;
       }
 
@@ -1571,7 +1567,7 @@ mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
           &dependenceConstraints, /*dependenceComponents=*/nullptr,
           /*allowRAR=*/readReadAccesses);
       if (result.value == DependenceResult::Failure) {
-        LLVM_DEBUG(llvm::dbgs() << "Dependence check failed\n");
+        LDBG() << "Dependence check failed";
         return SliceComputationResult::GenericFailure;
       }
       if (result.value == DependenceResult::NoDependence)
@@ -1586,8 +1582,7 @@ mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
       if (sliceUnionCst.getNumDimAndSymbolVars() == 0) {
         // Initialize 'sliceUnionCst' with the bounds computed in previous step.
         if (failed(tmpSliceState.getAsConstraints(&sliceUnionCst))) {
-          LLVM_DEBUG(llvm::dbgs()
-                     << "Unable to compute slice bound constraints\n");
+          LDBG() << "Unable to compute slice bound constraints";
           return SliceComputationResult::GenericFailure;
         }
         assert(sliceUnionCst.getNumDimAndSymbolVars() > 0);
@@ -1597,8 +1592,7 @@ mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
       // Compute constraints for 'tmpSliceState' in 'tmpSliceCst'.
       FlatAffineValueConstraints tmpSliceCst;
       if (failed(tmpSliceState.getAsConstraints(&tmpSliceCst))) {
-        LLVM_DEBUG(llvm::dbgs()
-                   << "Unable to compute slice bound constraints\n");
+        LDBG() << "Unable to compute slice bound constraints";
         return SliceComputationResult::GenericFailure;
       }
 
@@ -1630,8 +1624,7 @@ mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
       if (sliceUnionCst.getNumLocalVars() > 0 ||
           tmpSliceCst.getNumLocalVars() > 0 ||
           failed(sliceUnionCst.unionBoundingBox(tmpSliceCst))) {
-        LLVM_DEBUG(llvm::dbgs()
-                   << "Unable to compute union bounding box of slice bounds\n");
+        LDBG() << "Unable to compute union bounding box of slice bounds";
         return SliceComputationResult::GenericFailure;
       }
     }
@@ -1639,7 +1632,7 @@ mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
 
   // Empty union.
   if (sliceUnionCst.getNumDimAndSymbolVars() == 0) {
-    LLVM_DEBUG(llvm::dbgs() << "empty slice union - unexpected\n");
+    LDBG() << "empty slice union - unexpected";
     return SliceComputationResult::GenericFailure;
   }
 
@@ -1652,7 +1645,7 @@ mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
   unsigned innermostCommonLoopDepth =
       getInnermostCommonLoopDepth(ops, &surroundingLoops);
   if (loopDepth > innermostCommonLoopDepth) {
-    LLVM_DEBUG(llvm::dbgs() << "Exceeds max loop depth\n");
+    LDBG() << "Exceeds max loop depth";
     return SliceComputationResult::GenericFailure;
   }
 
@@ -1696,7 +1689,7 @@ mlir::affine::computeSliceUnion(ArrayRef<Operation *> opsA,
   // that the slice is valid, otherwise return appropriate failure status.
   std::optional<bool> isSliceValid = sliceUnion->isSliceValid();
   if (!isSliceValid) {
-    LLVM_DEBUG(llvm::dbgs() << "Cannot determine if the slice is valid\n");
+    LDBG() << "Cannot determine if the slice is valid";
     return SliceComputationResult::GenericFailure;
   }
   if (!*isSliceValid)
@@ -2050,7 +2043,8 @@ static std::optional<int64_t> getMemoryFootprintBytes(Block &block,
     if (failed(
             region->compute(opInst,
                             /*loopDepth=*/getNestingDepth(&*block.begin())))) {
-      LLVM_DEBUG(opInst->emitError("error obtaining memory region"));
+      LDBG() << "Error obtaining memory region";
+      opInst->emitError("error obtaining memory region");
       return failure();
     }
 
@@ -2058,9 +2052,11 @@ static std::optional<int64_t> getMemoryFootprintBytes(Block &block,
     if (inserted) {
       it->second = std::move(region);
     } else if (failed(it->second->unionBoundingBox(*region))) {
-      LLVM_DEBUG(opInst->emitWarning(
+      LDBG() << "getMemoryFootprintBytes: unable to perform a union on a "
+                "memory region";
+      opInst->emitWarning(
           "getMemoryFootprintBytes: unable to perform a union on a memory "
-          "region"));
+          "region");
       return failure();
     }
     return WalkResult::advance();
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 22608a16cc1ab..7e5ce26b5f733 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -427,6 +427,21 @@ bool mlir::affine::isValidSymbol(Value value) {
   return false;
 }
 
+/// A utility function to check if a value is defined at the top level of
+/// `region` or is an argument of `region` or is defined above the region.
+static bool isTopLevelValueOrAbove(Value value, Region *region) {
+  Region *parentRegion = value.getParentRegion();
+  do {
+    if (parentRegion == region)
+      return true;
+    Operation *regionOp = region->getParentOp();
+    if (regionOp->hasTrait<OpTrait::IsIsolatedFromAbove>())
+      break;
+    region = region->getParentOp()->getParentRegion();
+  } while (region);
+  return false;
+}
+
 /// A value can be used as a symbol for `region` iff it meets one of the
 /// following conditions:
 /// *) It is a constant.
@@ -445,19 +460,12 @@ bool mlir::affine::isValidSymbol(Value value, Region *region) {
     return false;
 
   // A top-level value is a valid symbol.
-  if (region && ::isTopLevelValue(value, region))
+  if (region && isTopLevelValueOrAbove(value, region))
     return true;
 
   auto *defOp = value.getDefiningOp();
-  if (!defOp) {
-    // A block argument that is not a top-level value is a valid symbol if it
-    // dominates region's parent op.
-    Operation *regionOp = region ? region->getParentOp() : nullptr;
-    if (regionOp && !regionOp->hasTrait<OpTrait::IsIsolatedFromAbove>())
-      if (auto *parentOpRegion = region->getParentOp()->getParentRegion())
-        return isValidSymbol(value, parentOpRegion);
+  if (!defOp)
     return false;
-  }
 
   // Constant operation is ok.
   Attribute operandCst;
@@ -475,12 +483,6 @@ bool mlir::affine::isValidSymbol(Value value, Region *region) {
   if (auto dimOp = dyn_cast<ShapedDimOpInterface>(defOp))
     return isDimOpValidSymbol(dimOp, region);
 
-  // Check for values dominating `region`'s parent op.
-  Operation *regionOp = region ? region->getParentOp() : nullptr;
-  if (regionOp && !regionOp->hasTrait<OpTrait::IsIsolatedFromAbove>())
-    if (auto *parentRegion = region->getParentOp()->getParentRegion())
-      return isValidSymbol(value, parentRegion);
-
   return false;
 }
 
diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
index 6c9adff7e9106..ff0157eb9e4f3 100644
--- a/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/LoopFusion.cpp
@@ -26,6 +26,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/raw_ostream.h"
 #include <iomanip>
 #include <optional>
@@ -95,8 +96,8 @@ static bool canRemoveSrcNodeAfterFusion(
     // Otherwise, the src loop can't be removed.
     if (fusedLoopInsPoint != depNodeOp &&
         !fusedLoopInsPoint->isBeforeInBlock(depNodeOp)) {
-      LLVM_DEBUG(llvm::dbgs() << "Src loop can't be removed: dst loop doesn't "
-                                 "dominate dependence\n");
+      LDBG() << "Src loop can't be removed: dst loop doesn't "
+             << "dominate dependence";
       return false;
     }
 
@@ -109,14 +110,13 @@ static bool canRemoveSrcNodeAfterFusion(
   if (hasOutDepsAfterFusion || !escapingMemRefs.empty()) {
     std::optional<bool> isMaximal = fusionSlice.isMaximal();
     if (!isMaximal) {
-      LLVM_DEBUG(llvm::dbgs() << "Src loop can't be removed: can't determine "
-                                 "if fusion is maximal\n");
+      LDBG() << "Src loop can't be removed: can't determine "
+             << "if fusion is maximal";
       return false;
     }
 
     if (!*isMaximal) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << "Src loop can't be removed: fusion is not maximal\n");
+      LDBG() << "Src loop can't be removed: fusion is not maximal";
       return false;
     }
   }
@@ -190,7 +190,8 @@ static bool isEscapingMemref(Value memref, Block *block) {
 
   // Check if this is defined to be an alias of another memref.
   if (auto viewOp = dyn_cast<mlir::ViewLikeOpInterface>(defOp))
-    if (isEscapingMemref(viewOp.getViewSource(), block))
+    if (memref == viewOp.getViewDest() &&
+        isEscapingMemref(viewOp.getViewSource(), block))
       return true;
 
   // Any op besides allocating ops wouldn't guarantee alias freedom
@@ -279,19 +280,19 @@ static std::optional<double> getAdditionalComputeFraction(
     AffineForOp srcForOp, AffineForOp dstForOp, unsigned depth,
     ArrayRef<ComputationSliceState> depthSliceUnions, int64_t &sliceCost,
     int64_t &fusedLoopNestComputeCost) {
-  LLVM_DEBUG(llvm::dbgs() << "Determining additional compute fraction...\n";);
+  LDBG() << "Determining additional compute fraction...";
   // Compute cost of sliced and unsliced src loop nest.
   // Walk src loop nest and collect stats.
   LoopNestStats srcLoopNestStats;
   if (!getLoopNestStats(srcForOp, &srcLoopNestStats)) {
-    LLVM_DEBUG(llvm::dbgs() << "Failed to get source loop nest stats.\n");
+    LDBG() << "Failed to get source loop nest stats.";
     return std::nullopt;
   }
 
   // Compute cost of dst loop nest.
   LoopNestStats dstLoopNestStats;
   if (!getLoopNestStats(dstForOp, &dstLoopNestStats)) {
-    LLVM_DEBUG(llvm::dbgs() << "Failed to get destination loop nest stats.\n");
+    LDBG() << "Failed to get destination loop nest stats.";
     return std::nullopt;
   }
 
@@ -304,14 +305,14 @@ static std::optional<double> getAdditionalComputeFraction(
   const ComputationSliceState &slice = depthSliceUnions[depth - 1];
   // Skip slice union if it wasn't computed for this depth.
   if (slice.isEmpty()) {
-    LLVM_DEBUG(llvm::dbgs() << "Slice wasn't computed.\n");
+    LDBG() << "Slice wasn't computed.";
     return std::nullopt;
   }
 
   if (!getFusionComputeCost(srcForOp, srcLoopNestStats, dstForOp,
                             dstLoopNestStats, slice,
                             &fusedLoopNestComputeCost)) {
-    LLVM_DEBUG(llvm::dbgs() << "Unable to compute fusion compute cost\n");
+    LDBG() << "Unable to compute fusion compute cost";
     return std::nullopt;
   }
 
@@ -348,9 +349,8 @@ static Value createPrivateMemRef(AffineForOp forOp,
                     MemRefAccess bM(cast<AffineWriteOpInterface>(b));
                     return aM == bM;
                   })) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "Private memref creation unsupported for multiple producer "
-                  "stores with different access functions.\n");
+    LDBG() << "Private memref creation unsupported for multiple producer "
+           << "stores with different access functions.";
     return nullptr;
   }
 
@@ -455,8 +455,7 @@ static Value createPrivateMemRef(AffineForOp forOp,
   assert(succeeded(res) &&
          "replaceAllMemrefUsesWith should always succeed here");
   (void)res;
-  LLVM_DEBUG(llvm::dbgs() << "Created private memref of type: " << newMemRefType
-                          << '\n');
+  LDBG() << "Created private memref of type: " << newMemRefType;
   return newMemRef;
 }
 
@@ -505,15 +504,12 @@ static bool isFusionProfitable(AffineForOp srcForOp,
                                unsigned maxLegalFusionDepth,
                                unsigned *dstLoopDepth,
                                double computeToleranceThreshold) {
-  LLVM_DEBUG({
-    llvm::dbgs()
-        << "Checking whether fusion is profitable between source nest:\n";
-    llvm::dbgs() << ' ' << srcForOp << " and destination nest:\n";
-    llvm::dbgs() << dstForOp << "\n";
-  });
+  LDBG() << "Checking whether fusion is profitable between source nest:";
+  LDBG() << ' ' << srcForOp << " and destination nest:";
+  LDBG() << dstForOp;
 
   if (maxLegalFusionDepth == 0) {
-    LLVM_DEBUG(llvm::dbgs() << "Can't fuse: maxLegalFusionDepth is 0\n");
+    LDBG() << "Can't fuse: maxLegalFusionDepth is 0";
     return false;
   }
 
@@ -537,8 +533,8 @@ static bool isFusionProfitable(AffineForOp srcForOp,
   // TODO: Suppport multiple producer stores in profitability
   // analysis.
   if (producerStores.size() > 1) {
-    LLVM_DEBUG(llvm::dbgs() << "Limited profitability analysis. Not "
-                               "supported for multiple producer store case.\n");
+    LDBG() << "Limited profitability analysis. Not "
+           << "supported for multiple producer store case.";
     int64_t sliceCost;
     int64_t fusedLoopNestComputeCost;
     // We will still fuse if fusion obeys the specified compute
@@ -547,12 +543,11 @@ static bool isFusionProfitable(AffineForOp srcForOp,
         srcForOp, dstForOp, maxLegalFusionDepth, depthSliceUnions, sliceCost,
         fusedLoopNestComputeCost);
     if (!fraction || fraction > computeToleranceThreshold) {
-      LLVM_DEBUG(llvm::dbgs() << "Additional computation exceeds "
-                                 "compute tolerance. Not fusing.\n");
+      LDBG() << "Additional computation exceeds "
+             << "compute tolerance. Not fusing.";
       return false;
     }
-    LLVM_DEBUG(llvm::dbgs()
-               << "Considering fusion profitable at max legal depth.\n");
+    LDBG() << "Considering fusion profitable at max legal depth.";
     return true;
   }
 
@@ -574,8 +569,7 @@ static bool isFusionProfitable(AffineForOp srcForOp,
   // Compute src loop nest write region size.
   MemRefRegion srcWriteRegion(srcStoreOp->getLoc());
   if (failed(srcWriteRegion.compute(srcStoreOp, /*loopDepth=*/0))) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "Unable to compute MemRefRegion for source operation\n");
+    LDBG() << "Unable to compute MemRefRegion for source operation";
     return false;
   }
 
@@ -609,8 +603,7 @@ static bool isFusionProfitable(AffineForOp srcForOp,
         getAdditionalComputeFraction(srcForOp, dstForOp, i, depthSliceUnions,
                                      sliceCost, fusedLoopNestComputeCost);
     if (!mayAdditionalComputeFraction) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << "Can't determine additional compute fraction.\n");
+      LDBG() << "Can't determine additional compute fraction.";
       continue;
     }
     double additionalComputeFraction = *mayAdditionalComputeFraction;
@@ -620,9 +613,7 @@ static bool isFusionProfitable(AffineForOp srcForOp,
     // depth 'i'.
     MemRefRegion sliceWriteRegion(srcStoreOp->getLoc());
     if (failed(sliceWriteRegion.compute(srcStoreOp, /*loopDepth=*/0, &slice))) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << "Failed to compute slice write region at loopDepth: " << i
-                 << "\n");
+      LDBG() << "Failed to compute slice write region at loopDepth: " << i;
       continue;
     }
 
@@ -630,9 +621,7 @@ static bool isFusionProfitable(AffineForOp srcForOp,
         sliceWriteRegion.getRegionSize();
     if (!maybeSliceWriteRegionSizeBytes.has_value() ||
         *maybeSliceWriteRegionSizeBytes == 0) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << "Failed to get slice write region size at loopDepth: " << i
-                 << "\n");
+      LDBG() << "Failed to get slice write region size at loopDepth: " << i;
       continue;
     }
     int64_t sliceWriteRegionSizeBytes = *maybeSliceWriteRegionSizeBytes;
@@ -649,9 +638,8 @@ static bool isFusionProfitable(AffineForOp srcForOp,
           << "   storage reduction factor: " << storageReduction << "x\n"
           << "   fused nest cost: " << fusedLoopNestComputeCost << "\n"
           << "   src write region size: " << srcWriteRegionSizeBytes << "\n"
-          << "   slice write region size: " << sliceWriteRegionSizeBytes
-          << "\n";
-      llvm::dbgs() << msg.str();
+          << "   slice write region size: " << sliceWriteRegionSizeBytes;
+      LDBG() << msg.str();
     });
 
     // TODO: This is a placeholder cost model.
@@ -670,28 +658,24 @@ static bool isFusionProfitable(AffineForOp srcForOp,
   // A simple cost model: fuse if it reduces the memory footprint.
 
   if (!bestDstLoopDepth) {
-    LLVM_DEBUG(
-        llvm::dbgs()
-        << "All fusion choices involve more than the threshold amount of "
-           "redundant computation; NOT fusing.\n");
+    LDBG() << "All fusion choices involve more than the threshold amount of "
+           << "redundant computation; NOT fusing.";
     return false;
   }
 
   if (!bestDstLoopDepth) {
-    LLVM_DEBUG(llvm::dbgs() << "no fusion depth could be evaluated.\n");
+    LDBG() << "no fusion depth could be evaluated.";
     return false;
   }
 
   // Set dstLoopDepth based on best values from search.
   *dstLoopDepth = *bestDstLoopDepth;
 
-  LLVM_DEBUG(
-      llvm::dbgs() << " LoopFusion fusion stats:"
-                   << "\n  best loop depth: " << bestDstLoopDepth
-                   << "\n  src loop nest compute cost: " << srcLoopNestCost
-                   << "\n  dst loop nest compute cost: " << dstLoopNestCost
-                   << "\n  fused loop nest compute cost: "
-                   << minFusedLoopNestComputeCost << "\n");
+  LDBG() << " LoopFusion fusion stats:";
+  LDBG() << "  best loop depth: " << bestDstLoopDepth;
+  LDBG() << "  src loop nest compute cost: " << srcLoopNestCost;
+  LDBG() << "  dst loop nest compute cost: " << dstLoopNestCost;
+  LDBG() << "  fused loop nest compute cost: " << minFusedLoopNestComputeCost;
 
   auto dstMemSize = getMemoryFootprintBytes(dstForOp);
   auto srcMemSize = getMemoryFootprintBytes(srcForOp);
@@ -699,8 +683,7 @@ static bool isFusionProfitable(AffineForOp srcForOp,
   std::optional<double> storageReduction;
 
   if (!dstMemSize || !srcMemSize) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "  fusion memory benefit cannot be evaluated; NOT fusing.\n");
+    LDBG() << "  fusion memory benefit cannot be evaluated; NOT fusing.";
     return false;
   }
 
@@ -710,13 +693,13 @@ static bool isFusionProfitable(AffineForOp srcForOp,
   assert(sliceMemEstimate && "expected value");
   auto fusedMem = dstMemSizeVal + *sliceMemEstimate;
 
-  LLVM_DEBUG(llvm::dbgs() << "   src mem: " << srcMemSizeVal << "\n"
-                          << "   dst mem: " << dstMemSizeVal << "\n"
-                          << "   fused mem: " << fusedMem << "\n"
-                          << "   slice mem: " << sliceMemEstimate << "\n");
+  LDBG() << "   src mem: " << srcMemSizeVal;
+  LDBG() << "   dst mem: " << dstMemSizeVal;
+  LDBG() << "   fused mem: " << fusedMem;
+  LDBG() << "   slice mem: " << sliceMemEstimate;
 
   if (static_cast<long>(fusedMem) > srcMemSizeVal + dstMemSizeVal) {
-    LLVM_DEBUG(llvm::dbgs() << "Fusion is not profitable; NOT fusing.\n");
+    LDBG() << "Fusion is not profitable; NOT fusing.";
     return false;
   }
   storageReduction =
@@ -734,8 +717,8 @@ static bool isFusionProfitable(AffineForOp srcForOp,
         << std::setprecision(2) << additionalComputeFraction
         << "% redundant computation and a ";
     msg << (storageReduction ? std::to_string(*storageReduction) : "<unknown>");
-    msg << "% storage reduction.\n";
-    llvm::dbgs() << msg.str();
+    msg << "% storage reduction.";
+    LDBG() << msg.str();
   });
 
   return true;
@@ -895,7 +878,7 @@ struct GreedyFusion {
   /// No fusion is performed when producers with a user count greater than
   /// `maxSrcUserCount` for any of the memrefs involved.
   void performFusionsIntoDest(unsigned dstId, unsigned maxSrcUserCount) {
-    LLVM_DEBUG(llvm::dbgs() << "Evaluating dst loop " << dstId << "\n");
+    LDBG() << "Evaluating dst loop " << dstId;
     // Skip if this node was removed (fused into another node).
     if (mdg->nodes.count(dstId) == 0)
       return;
@@ -909,7 +892,7 @@ struct GreedyFusion {
     if (dstNode->op->getNumResults() > 0)
       return;
 
-    LLVM_DEBUG(llvm::dbgs() << "Evaluating dst loop " << dstId << "\n");
+    LDBG() << "Evaluating dst loop " << dstId;
 
     // Sink sequential loops in 'dstNode' (and thus raise parallel loops)
     // while preserving relative order. This can increase the maximum loop
@@ -936,18 +919,14 @@ struct GreedyFusion {
         auto *srcNode = mdg->getNode(srcId);
         auto srcAffineForOp = cast<AffineForOp>(srcNode->op);
 
-        LLVM_DEBUG(llvm::dbgs()
-                   << "Trying to fuse producer loop nest " << srcId
-                   << " with consumer loop nest " << dstId << "\n");
-        LLVM_DEBUG(llvm::dbgs() << "Compute tolerance threshold: "
-                                << computeToleranceThreshold << '\n');
-        LLVM_DEBUG(llvm::dbgs()
-                   << "Producer loop nest:\n"
-                   << *srcNode->op << "\n and consumer loop nest:\n"
-                   << *dstNode->op << '\n');
+        LDBG() << "Trying to fuse producer loop nest " << srcId
+               << " with consumer loop nest " << dstId;
+        LDBG() << "Compute tolerance threshold: " << computeToleranceThreshold;
+        LDBG() << "Producer loop nest:";
+        LDBG() << *srcNode->op << " and consumer loop nest:";
+        LDBG() << *dstNode->op;
 
-        LLVM_DEBUG(llvm::dbgs() << "Evaluating src loop " << srcId
-                                << " for dst loop " << dstId << "\n");
+        LDBG() << "Evaluating src loop " << srcId << " for dst loop " << dstId;
 
         // Skip if 'srcNode' is a loop nest returning values.
         // TODO: support loop nests that return values.
@@ -1018,19 +997,16 @@ struct GreedyFusion {
                                    &depthSliceUnions[i - 1], strategy);
           if (result.value == FusionResult::Success) {
             maxLegalFusionDepth = i;
-            LLVM_DEBUG(llvm::dbgs()
-                       << "Found valid slice for depth: " << i << '\n');
+            LDBG() << "Found valid slice for depth: " << i;
           }
         }
 
         if (maxLegalFusionDepth == 0) {
-          LLVM_DEBUG(llvm::dbgs()
-                     << "Can't fuse: fusion is not legal at any depth\n");
+          LDBG() << "Can't fuse: fusion is not legal at any depth";
           continue;
         }
 
-        LLVM_DEBUG(llvm::dbgs() << "Max legal depth for fusion: "
-                                << maxLegalFusionDepth << '\n');
+        LDBG() << "Max legal depth for fusion: " << maxLegalFusionDepth;
 
         double computeToleranceThresholdToUse = computeToleranceThreshold;
 
@@ -1040,7 +1016,7 @@ struct GreedyFusion {
         // producer-consumer memref access for example). Check this and allow
         // fusion accordingly.
         if (hasCyclicDependence(srcAffineForOp)) {
-          LLVM_DEBUG(llvm::dbgs() << "Source nest has a cyclic dependence.\n");
+          LDBG() << "Source nest has a cyclic dependence.";
           // Maximal fusion does not check for compute tolerance threshold; so
           // perform the maximal fusion only when the redundanation computation
           // is zero.
@@ -1053,18 +1029,15 @@ struct GreedyFusion {
                 srcForOp, dstForOp, maxLegalFusionDepth, depthSliceUnions,
                 sliceCost, fusedLoopNestComputeCost);
             if (!fraction || fraction > 0) {
-              LLVM_DEBUG(
-                  llvm::dbgs()
-                  << "Can't perform maximal fusion with a cyclic dependence "
-                     "and non-zero additional compute.\n");
+              LDBG() << "Can't perform maximal fusion with a cyclic dependence "
+                     << "and non-zero additional compute.";
               return;
             }
           } else {
             // Set redundant computation tolerance to zero regardless of what
             // the user specified. Without this, fusion would be invalid.
-            LLVM_DEBUG(llvm::dbgs()
-                       << "Setting compute tolerance to zero since "
-                          "source has a cylic dependence.\n");
+            LDBG() << "Setting compute tolerance to zero since "
+                   << "source has a cylic dependence.";
             computeToleranceThresholdToUse = 0;
           }
         }
@@ -1107,8 +1080,7 @@ struct GreedyFusion {
           if (canCreatePrivateMemRef(memref, srcEscapingMemRefs, srcId, dstId,
                                      removeSrcNode)) {
             // Create a private version of this memref.
-            LLVM_DEBUG(llvm::dbgs()
-                       << "Creating private memref for " << memref << '\n');
+            LDBG() << "Creating private memref for " << memref;
             // Create a private version of this memref.
             privateMemrefs.insert(memref);
           }
@@ -1118,10 +1090,9 @@ struct GreedyFusion {
         fuseLoops(srcAffineForOp, dstAffineForOp, bestSlice);
         dstNodeChanged = true;
 
-        LLVM_DEBUG(llvm::dbgs()
-                   << "Fused src loop " << srcId << " into dst loop " << dstId
-                   << " at depth " << bestDstLoopDepth << ":\n"
-                   << dstAffineForOp << "\n");
+        LDBG() << "Fused src loop " << srcId << " into dst loop " << dstId
+               << " at depth " << bestDstLoopDepth << ":";
+        LDBG() << dstAffineForOp;
 
         // Move 'dstAffineForOp' before 'insertPointInst' if needed.
         if (fusedLoopInsPoint != dstAffineForOp)
@@ -1179,8 +1150,7 @@ struct GreedyFusion {
             dstLoopCollector.memrefFrees);
 
         if (removeSrcNode) {
-          LLVM_DEBUG(llvm::dbgs()
-                     << "Removing src loop " << srcId << " after fusion\n");
+          LDBG() << "Removing src loop " << srcId << " after fusion";
           // srcNode is no longer valid after it is removed from mdg.
           srcAffineForOp.erase();
           mdg->removeNode(srcId);
@@ -1195,7 +1165,7 @@ struct GreedyFusion {
   /// user count greater than `maxSrcUserCount` for any of the memrefs involved
   /// are encountered.
   void fuseProducerConsumerNodes(unsigned maxSrcUserCount) {
-    LLVM_DEBUG(llvm::dbgs() << "--- Producer/Consumer Fusion ---\n");
+    LDBG() << "--- Producer/Consumer Fusion ---";
     init();
     while (!worklist.empty()) {
       unsigned dstId = worklist.back();
@@ -1207,7 +1177,7 @@ struct GreedyFusion {
   // Visits each node in the graph, and for each node, attempts to fuse it with
   // its sibling nodes (nodes which share a parent, but no dependence edges).
   void fuseSiblingNodes() {
-    LLVM_DEBUG(llvm::dbgs() << "--- Sibling Fusion ---\n");
+    LDBG() << "--- Sibling Fusion ---";
     init();
     while (!worklist.empty()) {
       unsigned dstId = worklist.back();
@@ -1289,8 +1259,7 @@ struct GreedyFusion {
           maxLegalFusionDepth = i;
       }
 
-      LLVM_DEBUG(llvm::dbgs() << "Max legal depth for fusion: "
-                              << maxLegalFusionDepth << '\n');
+      LDBG() << "Max legal depth for fusion: " << maxLegalFusionDepth;
 
       // Skip if fusion is not feasible at any loop depths.
       if (maxLegalFusionDepth == 0)
@@ -1304,7 +1273,7 @@ struct GreedyFusion {
       // producer-consumer memref access for example). Check this and allow
       // fusion accordingly.
       if (hasCyclicDependence(sibAffineForOp)) {
-        LLVM_DEBUG(llvm::dbgs() << "Source nest has a cyclic dependence.\n");
+        LDBG() << "Source nest has a cyclic dependence.";
         // Maximal fusion does not check for compute tolerance threshold; so
         // perform the maximal fusion only when the redundanation computation is
         // zero.
@@ -1316,17 +1285,15 @@ struct GreedyFusion {
               sibAffineForOp, dstForOp, maxLegalFusionDepth, depthSliceUnions,
               sliceCost, fusedLoopNestComputeCost);
           if (!fraction || fraction > 0) {
-            LLVM_DEBUG(
-                llvm::dbgs()
-                << "Can't perform maximal fusion with a cyclic dependence "
-                   "and non-zero additional compute.\n");
+            LDBG() << "Can't perform maximal fusion with a cyclic dependence "
+                   << "and non-zero additional compute.";
             return;
           }
         } else {
           // Set redundant computation tolerance to zero regardless of what the
           // user specified. Without this, fusion would be invalid.
-          LLVM_DEBUG(llvm::dbgs() << "Setting compute tolerance to zero since "
-                                     "source has a cyclic dependence.\n");
+          LDBG() << "Setting compute tolerance to zero since "
+                 << "source has a cyclic dependence.";
           computeToleranceThresholdToUse = 0.0;
         }
       }
@@ -1356,8 +1323,7 @@ struct GreedyFusion {
       // slice is used in the destination.
       auto isMaximal = bestSlice.isMaximal();
       if (!isMaximal.value_or(false)) {
-        LLVM_DEBUG(llvm::dbgs()
-                   << "Slice isn't maximal; not performing sibling fusion.\n");
+        LDBG() << "Slice isn't maximal; not performing sibling fusion.";
         continue;
       }
 
@@ -1374,10 +1340,9 @@ struct GreedyFusion {
       if (insertPointInst != dstForInst)
         dstForInst->moveBefore(insertPointInst);
 
-      LLVM_DEBUG(llvm::dbgs()
-                 << "Fused sibling nest " << sibId << " into destination nest "
-                 << dstNode->id << " at depth " << bestDstLoopDepth << ":\n"
-                 << dstAffineForOp << "\n");
+      LDBG() << "Fused sibling nest " << sibId << " into destination nest "
+             << dstNode->id << " at depth " << bestDstLoopDepth << ":";
+      LDBG() << dstAffineForOp;
 
       // Update data dependence graph state post fusion.
       updateStateAfterSiblingFusion(sibNode, dstNode);
@@ -1555,7 +1520,7 @@ struct GreedyFusion {
 void LoopFusion::runOnBlock(Block *block) {
   MemRefDependenceGraph g(*block);
   if (!g.init()) {
-    LLVM_DEBUG(llvm::dbgs() << "MDG init failed\n");
+    LDBG() << "MDG init failed";
     return;
   }
 
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp
index 41cd7395b1836..c6abb0d734d88 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopFusionUtils.cpp
@@ -22,6 +22,7 @@
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
@@ -251,20 +252,20 @@ FusionResult mlir::affine::canFuseLoops(AffineForOp srcForOp,
                                         FusionStrategy fusionStrategy) {
   // Return 'failure' if 'dstLoopDepth == 0'.
   if (dstLoopDepth == 0) {
-    LLVM_DEBUG(llvm::dbgs() << "Cannot fuse loop nests at depth 0\n");
+    LDBG() << "Cannot fuse loop nests at depth 0";
     return FusionResult::FailPrecondition;
   }
   // Return 'failure' if 'srcForOp' and 'dstForOp' are not in the same block.
   auto *block = srcForOp->getBlock();
   if (block != dstForOp->getBlock()) {
-    LLVM_DEBUG(llvm::dbgs() << "Cannot fuse loop nests in different blocks\n");
+    LDBG() << "Cannot fuse loop nests in different blocks";
     return FusionResult::FailPrecondition;
   }
 
   // Return 'failure' if no valid insertion point for fused loop nest in 'block'
   // exists which would preserve dependences.
   if (!getFusedLoopNestInsertionPoint(srcForOp, dstForOp)) {
-    LLVM_DEBUG(llvm::dbgs() << "Fusion would violate dependences in block\n");
+    LDBG() << "Fusion would violate dependences in block";
     return FusionResult::FailBlockDependence;
   }
 
@@ -277,14 +278,14 @@ FusionResult mlir::affine::canFuseLoops(AffineForOp srcForOp,
   // Gather all load and store from 'forOpA' which precedes 'forOpB' in 'block'.
   SmallVector<Operation *, 4> opsA;
   if (!gatherLoadsAndStores(forOpA, opsA)) {
-    LLVM_DEBUG(llvm::dbgs() << "Fusing loops with affine.if unsupported\n");
+    LDBG() << "Fusing loops with affine.if unsupported";
     return FusionResult::FailPrecondition;
   }
 
   // Gather all load and store from 'forOpB' which succeeds 'forOpA' in 'block'.
   SmallVector<Operation *, 4> opsB;
   if (!gatherLoadsAndStores(forOpB, opsB)) {
-    LLVM_DEBUG(llvm::dbgs() << "Fusing loops with affine.if unsupported\n");
+    LDBG() << "Fusing loops with affine.if unsupported";
     return FusionResult::FailPrecondition;
   }
 
@@ -296,7 +297,7 @@ FusionResult mlir::affine::canFuseLoops(AffineForOp srcForOp,
     // TODO: 'getMaxLoopDepth' does not support forward slice fusion.
     assert(isSrcForOpBeforeDstForOp && "Unexpected forward slice fusion");
     if (getMaxLoopDepth(opsA, opsB) < dstLoopDepth) {
-      LLVM_DEBUG(llvm::dbgs() << "Fusion would violate loop dependences\n");
+      LDBG() << "Fusion would violate loop dependences";
       return FusionResult::FailFusionDependence;
     }
   }
@@ -339,12 +340,12 @@ FusionResult mlir::affine::canFuseLoops(AffineForOp srcForOp,
       strategyOpsA, opsB, dstLoopDepth, numCommonLoops,
       isSrcForOpBeforeDstForOp, srcSlice);
   if (sliceComputationResult.value == SliceComputationResult::GenericFailure) {
-    LLVM_DEBUG(llvm::dbgs() << "computeSliceUnion failed\n");
+    LDBG() << "computeSliceUnion failed";
     return FusionResult::FailPrecondition;
   }
   if (sliceComputationResult.value ==
       SliceComputationResult::IncorrectSliceFailure) {
-    LLVM_DEBUG(llvm::dbgs() << "Incorrect slice computation\n");
+    LDBG() << "Incorrect slice computation";
     return FusionResult::FailIncorrectSlice;
   }
 
@@ -477,7 +478,7 @@ bool mlir::affine::getLoopNestStats(AffineForOp forOpRoot,
     auto *parentForOp = forOp->getParentOp();
     if (forOp != forOpRoot) {
       if (!isa<AffineForOp>(parentForOp)) {
-        LLVM_DEBUG(llvm::dbgs() << "Expected parent AffineForOp\n");
+        LDBG() << "Expected parent AffineForOp";
         return WalkResult::interrupt();
       }
       // Add mapping to 'forOp' from its parent AffineForOp.
@@ -498,7 +499,7 @@ bool mlir::affine::getLoopNestStats(AffineForOp forOpRoot,
     std::optional<uint64_t> maybeConstTripCount = getConstantTripCount(forOp);
     if (!maybeConstTripCount) {
       // Currently only constant trip count loop nests are supported.
-      LLVM_DEBUG(llvm::dbgs() << "Non-constant trip count unsupported\n");
+      LDBG() << "Non-constant trip count unsupported";
       return WalkResult::interrupt();
     }
 
diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
index 2de057d1d0758..cd216efb66992 100644
--- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
+++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp
@@ -21,9 +21,11 @@
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/IntegerSet.h"
+#include "mlir/IR/OperationSupport.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
@@ -365,12 +367,11 @@ checkIfHyperRectangular(MutableArrayRef<AffineForOp> input) {
   if (input.size() <= 1)
     return success();
   if (failed(getIndexSet(ops, &cst))) {
-    LLVM_DEBUG(llvm::dbgs() << "Index set computation failed!\n");
+    LDBG() << "Index set computation failed!";
     return failure();
   }
   if (!cst.isHyperRectangular(0, input.size())) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "Non-hyperrectangular nests not supported for tiling!\n");
+    LDBG() << "Non-hyperrectangular nests not supported for tiling!";
     return failure();
   }
   return success();
@@ -385,14 +386,13 @@ static LogicalResult performPreTilingChecks(MutableArrayRef<AffineForOp> input,
 
   if (llvm::any_of(input,
                    [](AffineForOp op) { return op.getNumResults() > 0; })) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "Cannot tile nest where a loop has yield values\n");
+    LDBG() << "Cannot tile nest where a loop has yield values";
     return failure();
   }
 
   // Check if the supplied `for` ops are all successively nested.
   if (!isPerfectlyNested(input)) {
-    LLVM_DEBUG(llvm::dbgs() << "input loops not perfectly nested");
+    LDBG() << "input loops not perfectly nested";
     return failure();
   }
 
@@ -1098,7 +1098,7 @@ LogicalResult mlir::affine::loopUnrollJamByFactor(AffineForOp forOp,
 
   // If the trip count is lower than the unroll jam factor, no unroll jam.
   if (mayBeConstantTripCount && *mayBeConstantTripCount < unrollJamFactor) {
-    LLVM_DEBUG(llvm::dbgs() << "[failed] trip count < unroll-jam factor\n");
+    LDBG() << "[failed] trip count < unroll-jam factor";
     return failure();
   }
 
@@ -1339,6 +1339,15 @@ bool mlir::affine::isValidLoopInterchangePermutation(
   unsigned maxLoopDepth = loops.size();
   if (maxLoopDepth == 1)
     return true;
+
+  // We cannot guarantee the validity of the interchange if the loops have
+  // iter_args, since the dependence analysis does not take them into account.
+  // Conservatively return false in such cases.
+  if (llvm::any_of(loops, [](AffineForOp loop) {
+        return loop.getNumIterOperands() > 0;
+      }))
+    return false;
+
   // Gather dependence components for dependences between all ops in loop nest
   // rooted at 'loops[0]', at loop depths in range [1, maxLoopDepth].
   std::vector<SmallVector<DependenceComponent, 2>> depCompsVec;
@@ -1766,9 +1775,7 @@ findHighestBlockForPlacement(const MemRefRegion &region, Block &block,
     // We can't hoist past the definition of the memref being copied.
     Value memref = region.memref;
     if (!memref.getParentRegion()->isAncestor(enclosingOp->getParentRegion())) {
-      LLVM_DEBUG(
-          llvm::dbgs()
-          << "memref definition will end up not dominating hoist location\n");
+      LDBG() << "memref definition will end up not dominating hoist location";
       break;
     }
 
@@ -1977,7 +1984,7 @@ static LogicalResult generateCopy(
   auto memRefType = cast<MemRefType>(memref.getType());
 
   if (!memRefType.getLayout().isIdentity()) {
-    LLVM_DEBUG(llvm::dbgs() << "Non-identity layout map not yet supported\n");
+    LDBG() << "Non-identity layout map not yet supported";
     return failure();
   }
 
@@ -1989,7 +1996,7 @@ static LogicalResult generateCopy(
 
   unsigned rank = memRefType.getRank();
   if (rank == 0) {
-    LLVM_DEBUG(llvm::dbgs() << "Non-zero ranked memrefs supported\n");
+    LDBG() << "Non-zero ranked memrefs supported";
     return failure();
   }
 
@@ -2001,19 +2008,18 @@ static LogicalResult generateCopy(
   std::optional<int64_t> numElements =
       region.getConstantBoundingSizeAndShape(&fastBufferShape, &lbs);
   if (!numElements) {
-    LLVM_DEBUG(llvm::dbgs() << "Non-constant region size not supported\n");
+    LDBG() << "Non-constant region size not supported";
     return failure();
   }
 
   if (llvm::any_of(lbs, [](AffineMap lb) { return lb.getNumResults() > 1; })) {
     // This can be supported in the future if needed.
-    LLVM_DEBUG(llvm::dbgs()
-               << "Max lower bound for memref region start not supported\n");
+    LDBG() << "Max lower bound for memref region start not supported";
     return failure();
   }
 
   if (*numElements == 0) {
-    LLVM_DEBUG(llvm::dbgs() << "Nothing to copy\n");
+    LDBG() << "Nothing to copy";
     return success();
   }
 
@@ -2021,9 +2027,8 @@ static LogicalResult generateCopy(
   for (unsigned i = 0; i < rank; ++i) {
     region.getLowerAndUpperBound(i, lbMaps[i], ubMaps[i]);
     if (lbMaps[i].getNumResults() == 0 || ubMaps[i].getNumResults() == 0) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << "Missing lower or upper bound for region along dimension: "
-                 << i << '\n');
+      LDBG() << "Missing lower or upper bound for region along dimension: "
+             << i;
       return failure();
     }
   }
@@ -2122,7 +2127,7 @@ static LogicalResult generateCopy(
     // TODO: use all stride levels once DmaStartOp is extended for
     // multi-level strides.
     if (dmaStrideInfos.size() > 1) {
-      LLVM_DEBUG(llvm::dbgs() << "Only up to one level of stride supported\n");
+      LDBG() << "Only up to one level of stride supported";
       return failure();
     }
 
@@ -2309,10 +2314,11 @@ mlir::affine::affineDataCopyGenerate(Block::iterator begin, Block::iterator end,
   // surrounding the this block range.
   unsigned copyDepth = getNestingDepth(&*begin);
 
-  LLVM_DEBUG(llvm::dbgs() << "Generating copies at depth " << copyDepth
-                          << "\n");
-  LLVM_DEBUG(llvm::dbgs() << "from begin: " << *begin << "\n");
-  LLVM_DEBUG(llvm::dbgs() << "to inclusive end: " << *std::prev(end) << "\n");
+  LDBG() << "Generating copies at depth " << copyDepth;
+  LDBG() << "from begin: "
+         << OpWithFlags(&*begin, OpPrintingFlags().skipRegions());
+  LDBG() << "to inclusive end: "
+         << OpWithFlags(&*std::prev(end), OpPrintingFlags().skipRegions());
 
   // List of memory regions to copy for. We need a map vector to have a
   // guaranteed iteration order to write test cases. CHECK-DAG doesn't help here
@@ -2349,8 +2355,8 @@ mlir::affine::affineDataCopyGenerate(Block::iterator begin, Block::iterator end,
       return;
 
     if (!memref.getParentRegion()->isAncestor(block->getParent())) {
-      LLVM_DEBUG(llvm::dbgs() << "memref definition is inside of the depth at "
-                                 "which copy-in/copy-out would happen\n");
+      LDBG() << "memref definition is inside of the depth at "
+             << "which copy-in/copy-out would happen";
       return;
     }
 
@@ -2358,12 +2364,10 @@ mlir::affine::affineDataCopyGenerate(Block::iterator begin, Block::iterator end,
     auto region = std::make_unique<MemRefRegion>(opInst->getLoc());
     if (failed(region->compute(opInst, copyDepth, /*sliceState=*/nullptr,
                                /*addMemRefDimBounds=*/false))) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << "Error obtaining memory region: semi-affine maps?\n");
-      LLVM_DEBUG(llvm::dbgs() << "over-approximating to the entire memref\n");
+      LDBG() << "Error obtaining memory region: semi-affine maps?";
+      LDBG() << "over-approximating to the entire memref";
       if (!getFullMemRefAsRegion(opInst, copyDepth, region.get())) {
-        LLVM_DEBUG(
-            opInst->emitError("non-constant memref sizes not yet supported"));
+        LDBG() << "non-constant memref sizes not yet supported";
         error = true;
         return;
       }
@@ -2392,13 +2396,11 @@ mlir::affine::affineDataCopyGenerate(Block::iterator begin, Block::iterator end,
 
           // Perform a union with the existing region.
           if (failed(it->second->unionBoundingBox(*region))) {
-            LLVM_DEBUG(llvm::dbgs()
-                       << "Memory region bounding box failed; "
-                          "over-approximating to the entire memref\n");
+            LDBG() << "Memory region bounding box failed; "
+                   << "over-approximating to the entire memref";
             // If the union fails, we will overapproximate.
             if (!getFullMemRefAsRegion(opInst, copyDepth, region.get())) {
-              LLVM_DEBUG(opInst->emitError(
-                  "non-constant memref sizes not yet supported"));
+              LDBG() << "non-constant memref sizes not yet supported";
               error = true;
               return true;
             }
@@ -2428,8 +2430,7 @@ mlir::affine::affineDataCopyGenerate(Block::iterator begin, Block::iterator end,
   });
 
   if (error) {
-    LLVM_DEBUG(begin->emitError(
-        "copy generation failed for one or more memref's in this block\n"));
+    LDBG() << "copy generation failed for one or more memref's in this block";
     return failure();
   }
 
@@ -2466,8 +2467,7 @@ mlir::affine::affineDataCopyGenerate(Block::iterator begin, Block::iterator end,
   processRegions(writeRegions);
 
   if (!ret) {
-    LLVM_DEBUG(begin->emitError(
-        "copy generation failed for one or more memref's in this block\n"));
+    LDBG() << "copy generation failed for one or more memref's in this block";
     return failure();
   }
 
@@ -2608,7 +2608,7 @@ static AffineIfOp createSeparationCondition(MutableArrayRef<AffineForOp> loops,
                                         /*boundFloorDivisor=*/nullptr,
                                         /*ub=*/nullptr, &fullTileLbPos,
                                         &fullTileUbPos)) {
-      LLVM_DEBUG(llvm::dbgs() << "Can't get constant diff pair for a loop\n");
+      LDBG() << "Can't get constant diff pair for a loop";
       return nullptr;
     }
 
@@ -2667,8 +2667,7 @@ createFullTiles(MutableArrayRef<AffineForOp> inputNest,
   for (auto loop : inputNest) {
     // TODO: straightforward to generalize to a non-unit stride.
     if (loop.getStepAsInt() != 1) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << "[tile separation] non-unit stride not implemented\n");
+      LDBG() << "[tile separation] non-unit stride not implemented";
       return failure();
     }
     SmallVector<Operation *, 1> loopOp{loop.getOperation()};
@@ -2682,8 +2681,8 @@ createFullTiles(MutableArrayRef<AffineForOp> inputNest,
                                         /*boundFloorDivisor=*/nullptr,
                                         /*ub=*/nullptr, &lbPos, &ubPos) ||
         lbPos == ubPos) {
-      LLVM_DEBUG(llvm::dbgs() << "[tile separation] Can't get constant diff / "
-                                 "equalities not yet handled\n");
+      LDBG() << "[tile separation] Can't get constant diff / "
+             << "equalities not yet handled";
       return failure();
     }
 
@@ -2741,8 +2740,8 @@ mlir::affine::separateFullTiles(MutableArrayRef<AffineForOp> inputNest,
   AffineIfOp ifOp = createSeparationCondition(inputNest, b);
   if (!ifOp) {
     fullTileLoops.front().erase();
-    LLVM_DEBUG(llvm::dbgs() << "All tiles are full tiles, or failure creating "
-                               "separation condition\n");
+    LDBG() << "All tiles are full tiles, or failure creating "
+           << "separation condition";
     return failure();
   }
 
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 488c3c369afed..7d4d818ee448b 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -2678,6 +2678,7 @@ TypedAttr mlir::arith::getIdentityValueAttr(AtomicRMWKind kind, Type resultType,
   case AtomicRMWKind::addi:
   case AtomicRMWKind::maxu:
   case AtomicRMWKind::ori:
+  case AtomicRMWKind::xori:
     return builder.getZeroAttr(resultType);
   case AtomicRMWKind::andi:
     return builder.getIntegerAttr(
@@ -2736,7 +2737,7 @@ std::optional<TypedAttr> mlir::arith::getNeutralElement(Operation *op) {
           // Integer operations.
           .Case([](arith::AddIOp op) { return AtomicRMWKind::addi; })
           .Case([](arith::OrIOp op) { return AtomicRMWKind::ori; })
-          .Case([](arith::XOrIOp op) { return AtomicRMWKind::ori; })
+          .Case([](arith::XOrIOp op) { return AtomicRMWKind::xori; })
           .Case([](arith::AndIOp op) { return AtomicRMWKind::andi; })
           .Case([](arith::MaxUIOp op) { return AtomicRMWKind::maxu; })
           .Case([](arith::MinUIOp op) { return AtomicRMWKind::minu; })
@@ -2806,6 +2807,8 @@ Value mlir::arith::getReductionOp(AtomicRMWKind op, OpBuilder &builder,
     return arith::OrIOp::create(builder, loc, lhs, rhs);
   case AtomicRMWKind::andi:
     return arith::AndIOp::create(builder, loc, lhs, rhs);
+  case AtomicRMWKind::xori:
+    return arith::XOrIOp::create(builder, loc, lhs, rhs);
   // TODO: Add remaining reduction operations.
   default:
     (void)emitOptionalError(loc, "Reduction operation type not supported");
diff --git a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
index ddc64ea7ebf9a..91e37dd9ac36e 100644
--- a/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
+++ b/mlir/lib/Dialect/Async/Transforms/AsyncRuntimeRefCounting.cpp
@@ -248,7 +248,7 @@ LogicalResult AsyncRuntimeRefCountingPass::addDropRefAfterLastUse(Value value) {
   Region *definingRegion = value.getParentRegion();
 
   // Last users of the `value` inside all blocks where the value dies.
-  llvm::SmallSet<Operation *, 4> lastUsers;
+  llvm::SmallPtrSet<Operation *, 4> lastUsers;
 
   // Find blocks in the `definingRegion` that have users of the `value` (if
   // there are multiple users in the block, which one will be selected is
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index f1f12f4bca70e..56ff2121e4620 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -463,8 +463,12 @@ struct SimplifyClones : public OpRewritePattern<CloneOp> {
     // which otherwise could prevent removal of unnecessary allocs.
     Value canonicalSource = source;
     while (auto iface = dyn_cast_or_null<ViewLikeOpInterface>(
-               canonicalSource.getDefiningOp()))
+               canonicalSource.getDefiningOp())) {
+      if (canonicalSource != iface.getViewDest()) {
+        break;
+      }
       canonicalSource = iface.getViewSource();
+    }
 
     std::optional<Operation *> maybeCloneDeallocOp =
         memref::findDealloc(cloneOp.getOutput());
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
index 891652670dd5b..a465c957d063e 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferDeallocationSimplification.cpp
@@ -37,8 +37,12 @@ using namespace mlir::bufferization;
 /// Given a memref value, return the "base" value by skipping over all
 /// ViewLikeOpInterface ops (if any) in the reverse use-def chain.
 static Value getViewBase(Value value) {
-  while (auto viewLikeOp = value.getDefiningOp<ViewLikeOpInterface>())
+  while (auto viewLikeOp = value.getDefiningOp<ViewLikeOpInterface>()) {
+    if (value != viewLikeOp.getViewDest()) {
+      break;
+    }
     value = viewLikeOp.getViewSource();
+  }
   return value;
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
index 8f983ab1eae36..0b2e080e52b75 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/BufferViewFlowAnalysis.cpp
@@ -121,7 +121,7 @@ void BufferViewFlowAnalysis::build(Operation *op) {
     // Add additional dependencies created by view changes to the alias list.
     if (auto viewInterface = dyn_cast<ViewLikeOpInterface>(op)) {
       registerDependencies(viewInterface.getViewSource(),
-                           viewInterface->getResult(0));
+                           viewInterface.getViewDest());
       return WalkResult::advance();
     }
 
@@ -231,8 +231,12 @@ static bool isFunctionArgument(Value v) {
 /// Given a memref value, return the "base" value by skipping over all
 /// ViewLikeOpInterface ops (if any) in the reverse use-def chain.
 static Value getViewBase(Value value) {
-  while (auto viewLikeOp = value.getDefiningOp<ViewLikeOpInterface>())
+  while (auto viewLikeOp = value.getDefiningOp<ViewLikeOpInterface>()) {
+    if (value != viewLikeOp.getViewDest()) {
+      break;
+    }
     value = viewLikeOp.getViewSource();
+  }
   return value;
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 91f6f25e3d096..68ef51992efee 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -20,6 +20,7 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Pass/PassManager.h"
+#include "llvm/Support/DebugLog.h"
 #include <optional>
 
 namespace mlir {
@@ -328,20 +329,16 @@ LogicalResult bufferization::bufferizeOp(Operation *op,
               "blocks");
 
     // Bufferize the op.
-    LLVM_DEBUG(llvm::dbgs()
-               << "//===-------------------------------------------===//\n"
-               << "IR after bufferizing: " << nextOp->getName() << "\n");
+    LDBG(3) << "//===-------------------------------------------===//\n"
+            << "IR after bufferizing: " << nextOp->getName();
     rewriter.setInsertionPoint(nextOp);
     if (failed(
             bufferizableOp.bufferize(rewriter, options, bufferizationState))) {
-      LLVM_DEBUG(llvm::dbgs()
-                 << "failed to bufferize\n"
-                 << "//===-------------------------------------------===//\n");
+      LDBG(2) << "failed to bufferize\n"
+              << "//===-------------------------------------------===//";
       return nextOp->emitError("failed to bufferize op");
     }
-    LLVM_DEBUG(llvm::dbgs()
-               << *op
-               << "\n//===-------------------------------------------===//\n");
+    LDBG(3) << *op << "\n//===-------------------------------------------===//";
   }
 
   // Return early if the top-level op is entirely gone.
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
index a8e8353ca6191..fb7f2bb5f01d8 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OneShotAnalysis.cpp
@@ -56,6 +56,7 @@
 #include "mlir/Interfaces/SubsetOpInterface.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/Support/DebugLog.h"
 
 MLIR_DEFINE_EXPLICIT_TYPE_ID(mlir::bufferization::OneShotAnalysisState)
 
@@ -616,13 +617,10 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
         if (getParallelRegion(def.getParentRegion(), options) !=
             getParallelRegion(uConflictingWrite->getOwner()->getParentRegion(),
                               options)) {
-          LLVM_DEBUG(
-              llvm::dbgs()
-              << "\n- bufferizes out-of-place due to parallel region:\n");
-          LLVM_DEBUG(llvm::dbgs()
-                     << "  unConflictingWrite = operand "
-                     << uConflictingWrite->getOperandNumber() << " of "
-                     << *uConflictingWrite->getOwner() << "\n");
+          LDBG() << "\n- bufferizes out-of-place due to parallel region:\n"
+                 << "  unConflictingWrite = operand "
+                 << uConflictingWrite->getOperandNumber() << " of "
+                 << *uConflictingWrite->getOwner();
           return true;
         }
       }
@@ -631,9 +629,9 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
 
   for (OpOperand *uRead : usesRead) {
     Operation *readingOp = uRead->getOwner();
-    LLVM_DEBUG(llvm::dbgs() << "\n- check conflict:\n");
-    LLVM_DEBUG(llvm::dbgs() << "  uRead = operand " << uRead->getOperandNumber()
-                            << " of " << *readingOp << "\n");
+    LDBG() << "\n- check conflict:\n"
+           << "  uRead = operand " << uRead->getOperandNumber() << " of "
+           << *readingOp;
 
     // Find the definition of uRead by following the SSA use-def chain.
     // E.g.:
@@ -648,23 +646,22 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
     const SetVector<Value> &definitions = state.findDefinitionsCached(uRead);
     if (definitions.empty()) {
       // Fast path: No conflict if there are no definitions.
-      LLVM_DEBUG(llvm::dbgs()
-                 << "  no conflict: read value has no definitions\n");
+      LDBG() << "  no conflict: read value has no definitions";
       continue;
     }
 
     // Look for conflicting memory writes. Potential conflicts are writes to an
     // alias that have been decided to bufferize inplace.
     for (OpOperand *uConflictingWrite : usesWrite) {
-      LLVM_DEBUG(llvm::dbgs() << "  unConflictingWrite = operand "
-                              << uConflictingWrite->getOperandNumber() << " of "
-                              << *uConflictingWrite->getOwner() << "\n");
+      LDBG() << "  unConflictingWrite = operand "
+             << uConflictingWrite->getOperandNumber() << " of "
+             << *uConflictingWrite->getOwner();
 
       // Check if op dominance can be used to rule out read-after-write
       // conflicts.
       bool useDominance =
           canUseOpDominance(uRead, uConflictingWrite, definitions, state);
-      LLVM_DEBUG(llvm::dbgs() << "\n- useDominance = " << useDominance << "\n");
+      LDBG() << "\n- useDominance = " << useDominance;
 
       // Throughout this loop, check for multiple requirements that have to be
       // met for uConflictingWrite to be an actual conflict.
@@ -680,8 +677,7 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
         //       inside a loop), there may be no meaningful `happensBefore`
         //       relationship.
         if (happensBefore(readingOp, conflictingWritingOp, domInfo)) {
-          LLVM_DEBUG(llvm::dbgs()
-                     << "  no conflict: read happens before write\n");
+          LDBG() << "  no conflict: read happens before write";
           continue;
         }
 
@@ -693,8 +689,7 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
         // Note: If the op is executed multiple times (e.g., because it is
         //       inside a loop), it may be conflicting with itself.
         if (uConflictingWrite == uRead) {
-          LLVM_DEBUG(llvm::dbgs()
-                     << "  no conflict: read and write are same use\n");
+          LDBG() << "  no conflict: read and write are same use";
           continue;
         }
 
@@ -705,8 +700,8 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
         //       multiple times.
         if (state.insideMutuallyExclusiveRegions(readingOp,
                                                  conflictingWritingOp)) {
-          LLVM_DEBUG(llvm::dbgs() << "  no conflict: read and write are in "
-                                     "mutually exclusive regions\n");
+          LDBG() << "  no conflict: read and write are in "
+                    "mutually exclusive regions";
           continue;
         }
 
@@ -721,9 +716,7 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
                       state, uRead, uConflictingWrite->get()) ||
                   hasEquivalentValueInReverseUseDefChain(
                       state, uConflictingWrite, uRead->get())) {
-                LLVM_DEBUG(
-                    llvm::dbgs()
-                    << "  no conflict: op bufferizes to element-wise access\n");
+                LDBG() << "  no conflict: op bufferizes to element-wise access";
                 continue;
               }
             }
@@ -733,15 +726,14 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
 
       // No conflict if the operands are non-conflicting subsets.
       if (areNonConflictingSubsets(uRead, uConflictingWrite, state)) {
-        LLVM_DEBUG(llvm::dbgs() << "  no conflict: non-conflicting subsets\n");
+        LDBG() << "  no conflict: non-conflicting subsets";
         continue;
       }
 
       // No conflict if the op interface says so.
       if (auto bufferizableOp = options.dynCastBufferizableOp(readingOp)) {
         if (bufferizableOp.isNotConflicting(uRead, uConflictingWrite, state)) {
-          LLVM_DEBUG(llvm::dbgs()
-                     << "  no conflict: op interace of reading op says 'no'\n");
+          LDBG() << "  no conflict: op interace of reading op says 'no'";
           continue;
         }
       }
@@ -751,9 +743,7 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
                 options.dynCastBufferizableOp(conflictingWritingOp)) {
           if (bufferizableOp.isNotConflicting(uRead, uConflictingWrite,
                                               state)) {
-            LLVM_DEBUG(
-                llvm::dbgs()
-                << "  no conflict: op interace of writing op says 'no'\n");
+            LDBG() << "  no conflict: op interace of writing op says 'no'";
             continue;
           }
         }
@@ -761,29 +751,26 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
 
       // Check all possible definitions.
       for (Value definition : definitions) {
-        LLVM_DEBUG(llvm::dbgs() << "  * definition = " << definition << "\n");
+        LDBG() << "  * definition = " << definition;
 
         // No conflict if the conflicting write happens before the definition.
         if (Operation *defOp = definition.getDefiningOp()) {
           if (happensBefore(conflictingWritingOp, defOp, domInfo)) {
             // conflictingWritingOp happens before defOp. No conflict.
-            LLVM_DEBUG(llvm::dbgs()
-                       << "    no conflict: write happens before definition\n");
+            LDBG() << "    no conflict: write happens before definition";
             continue;
           }
           // No conflict if conflictingWritingOp is contained in defOp.
           if (defOp->isProperAncestor(conflictingWritingOp)) {
-            LLVM_DEBUG(
-                llvm::dbgs()
-                << "    no conflict: write is contained in definition\n");
+            LDBG() << "    no conflict: write is contained in definition";
             continue;
           }
         } else {
           auto bbArg = cast<BlockArgument>(definition);
           Block *block = bbArg.getOwner();
           if (!block->findAncestorOpInBlock(*conflictingWritingOp)) {
-            LLVM_DEBUG(llvm::dbgs() << "    no conflict: definition is bbArg "
-                                       "and write happens outside of block\n");
+            LDBG() << "    no conflict: definition is bbArg "
+                      "and write happens outside of block";
             // conflictingWritingOp happens outside of the block. No
             // conflict.
             continue;
@@ -795,8 +782,7 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
         AliasingValueList aliases = state.getAliasingValues(*uConflictingWrite);
         if (aliases.getNumAliases() == 1 &&
             aliases.getAliases()[0].value == definition) {
-          LLVM_DEBUG(llvm::dbgs()
-                     << "    no conflict: definition and write are same\n");
+          LDBG() << "    no conflict: definition and write are same";
           continue;
         }
 
@@ -804,7 +790,7 @@ hasReadAfterWriteInterference(const DenseSet<OpOperand *> &usesRead,
 
         if (options.printConflicts)
           annotateConflict(uRead, uConflictingWrite, definition);
-        LLVM_DEBUG(llvm::dbgs() << "  => RaW CONFLICT FOUND\n");
+        LDBG() << "  => RaW CONFLICT FOUND";
         return true;
       }
     }
@@ -958,7 +944,7 @@ wouldCreateWriteToNonWritableBuffer(OpOperand &operand,
   for (AliasingValue alias : state.getAliasingValues(operand))
     state.applyOnAliases(alias.value, checkReadOnly);
   if (foundReadOnly) {
-    LLVM_DEBUG(llvm::dbgs() << "=> NOT WRITABLE\n");
+    LDBG() << "=> NOT WRITABLE";
     return true;
   }
 
@@ -987,10 +973,9 @@ void OneShotAnalysisState::resetCache() {
 static LogicalResult
 bufferizableInPlaceAnalysisImpl(OpOperand &operand, OneShotAnalysisState &state,
                                 const DominanceInfo &domInfo) {
-  LLVM_DEBUG(
-      llvm::dbgs() << "//===-------------------------------------------===//\n"
-                   << "Analyzing operand #" << operand.getOperandNumber()
-                   << " of " << *operand.getOwner() << "\n");
+  LDBG() << "//===-------------------------------------------===//\n"
+         << "Analyzing operand #" << operand.getOperandNumber() << " of "
+         << *operand.getOwner();
 
   bool foundInterference =
       wouldCreateWriteToNonWritableBuffer(operand, state) ||
@@ -1001,8 +986,7 @@ bufferizableInPlaceAnalysisImpl(OpOperand &operand, OneShotAnalysisState &state,
   else
     state.bufferizeInPlace(operand);
 
-  LLVM_DEBUG(llvm::dbgs()
-             << "//===-------------------------------------------===//\n");
+  LDBG() << "//===-------------------------------------------===//";
   return success();
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
index 725fa2429ce5b..b593ccab060c7 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/OwnershipBasedBufferDeallocation.cpp
@@ -51,14 +51,8 @@ static bool isMemref(Value v) { return isa<BaseMemRefType>(v.getType()); }
 /// Return "true" if the given op is guaranteed to have neither "Allocate" nor
 /// "Free" side effects.
 static bool hasNeitherAllocateNorFreeSideEffect(Operation *op) {
-  if (isa<MemoryEffectOpInterface>(op))
-    return !hasEffect<MemoryEffects::Allocate>(op) &&
-           !hasEffect<MemoryEffects::Free>(op);
-  // If the op does not implement the MemoryEffectOpInterface but has has
-  // recursive memory effects, then this op in isolation (without its body) does
-  // not have any side effects. All the ops inside the regions of this op will
-  // be processed separately.
-  return op->hasTrait<OpTrait::HasRecursiveMemoryEffects>();
+  return !mightHaveEffect<MemoryEffects::Allocate>(op) &&
+         !mightHaveEffect<MemoryEffects::Free>(op);
 }
 
 /// Return "true" if the given op has buffer semantics. I.e., it has buffer
@@ -517,9 +511,7 @@ LogicalResult BufferDeallocation::verifyOperationPreconditions(Operation *op) {
   //   MemoryEffectOpInterface. They usually do not have side effects apart
   //   from the callee, which will be analyzed separately. (This is similar to
   //   "recursive memory effects".)
-  if (!isa<MemoryEffectOpInterface>(op) &&
-      !op->hasTrait<OpTrait::HasRecursiveMemoryEffects>() &&
-      !isa<CallOpInterface>(op))
+  if (hasUnknownEffects(op) && !isa<CallOpInterface>(op))
     return op->emitError(
         "ops with unknown memory side effects are not supported");
 
diff --git a/mlir/lib/Dialect/DLTI/Traits.cpp b/mlir/lib/Dialect/DLTI/Traits.cpp
index 34f2dd5896083..3f6dd2900a915 100644
--- a/mlir/lib/Dialect/DLTI/Traits.cpp
+++ b/mlir/lib/Dialect/DLTI/Traits.cpp
@@ -24,7 +24,7 @@ LogicalResult mlir::impl::verifyHasDefaultDLTIDataLayoutTrait(Operation *op) {
 }
 
 DataLayoutSpecInterface mlir::impl::getDataLayoutSpec(Operation *op) {
-  return op->getAttrOfType<DataLayoutSpecAttr>(
+  return op->getAttrOfType<DataLayoutSpecInterface>(
       DLTIDialect::kDataLayoutAttrName);
 }
 
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index e6a3154721faa..5359826eed0fd 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -114,11 +114,8 @@ bool mlir::emitc::isIntegerIndexOrOpaqueType(Type type) {
 bool mlir::emitc::isSupportedFloatType(Type type) {
   if (auto floatType = llvm::dyn_cast<FloatType>(type)) {
     switch (floatType.getWidth()) {
-    case 16: {
-      if (llvm::isa<Float16Type, BFloat16Type>(type))
-        return true;
-      return false;
-    }
+    case 16:
+      return llvm::isa<Float16Type, BFloat16Type>(type);
     case 32:
     case 64:
       return true;
@@ -1398,6 +1395,7 @@ void FileOp::build(OpBuilder &builder, OperationState &state, StringRef id) {
 //===----------------------------------------------------------------------===//
 // FieldOp
 //===----------------------------------------------------------------------===//
+
 static void printEmitCFieldOpTypeAndInitialValue(OpAsmPrinter &p, FieldOp op,
                                                  TypeAttr type,
                                                  Attribute initialValue) {
@@ -1455,6 +1453,15 @@ LogicalResult FieldOp::verify() {
 //===----------------------------------------------------------------------===//
 // GetFieldOp
 //===----------------------------------------------------------------------===//
+
+LogicalResult GetFieldOp::verify() {
+  auto parentClassOp = getOperation()->getParentOfType<emitc::ClassOp>();
+  if (!parentClassOp.getOperation())
+    return emitOpError(" must be nested within an emitc.class operation");
+
+  return success();
+}
+
 LogicalResult GetFieldOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   mlir::FlatSymbolRefAttr fieldNameAttr = getFieldNameAttr();
   FieldOp fieldOp =
diff --git a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
index c55e26e722f33..06d7e07005f8a 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/WrapFuncInClass.cpp
@@ -64,8 +64,8 @@ class WrapFuncInClass : public OpRewritePattern<emitc::FuncOp> {
       TypeAttr typeAttr = TypeAttr::get(val.getType());
       fields.push_back({fieldName, typeAttr});
 
-      FieldOp fieldop = rewriter.create<emitc::FieldOp>(
-          funcOp->getLoc(), fieldName, typeAttr, nullptr);
+      FieldOp fieldop = emitc::FieldOp::create(rewriter, funcOp->getLoc(),
+                                               fieldName, typeAttr, nullptr);
 
       if (argAttrs && idx < argAttrs->size()) {
         fieldop->setDiscardableAttrs(funcOp.getArgAttrDict(idx));
diff --git a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
index 5a72ef17db7f0..cc77aa6711c42 100644
--- a/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
+++ b/mlir/lib/Dialect/GPU/IR/GPUDialect.cpp
@@ -756,7 +756,8 @@ void LaunchOp::build(OpBuilder &builder, OperationState &result,
                      Type asyncTokenType, ValueRange asyncDependencies,
                      TypeRange workgroupAttributions,
                      TypeRange privateAttributions, Value clusterSizeX,
-                     Value clusterSizeY, Value clusterSizeZ) {
+                     Value clusterSizeY, Value clusterSizeZ,
+                     FlatSymbolRefAttr module, FlatSymbolRefAttr function) {
   OpBuilder::InsertionGuard g(builder);
 
   // Add a WorkGroup attribution attribute. This attribute is required to
@@ -781,6 +782,12 @@ void LaunchOp::build(OpBuilder &builder, OperationState &result,
   if (dynamicSharedMemorySize)
     result.addOperands(dynamicSharedMemorySize);
 
+  // Add optional module and function attributes.
+  if (module)
+    result.addAttribute(getModuleAttrName(result.name), module);
+  if (function)
+    result.addAttribute(getFunctionAttrName(result.name), function);
+
   // Create a kernel body region with kNumConfigRegionAttributes + N memory
   // attributions, where the first kNumConfigRegionAttributes arguments have
   // `index` type and the rest have the same types as the data operands.
@@ -944,6 +951,21 @@ void LaunchOp::print(OpAsmPrinter &p) {
     p << ' ' << getDynamicSharedMemorySizeKeyword() << ' '
       << getDynamicSharedMemorySize();
 
+  // Print optional module attribute.
+  StringRef moduleAttrName = getModuleAttrName();
+  if (auto module = getModule()) {
+    p << ' ' << moduleAttrName << '(';
+    p.printSymbolName(*module);
+    p << ')';
+  }
+  // Print optional function attribute.
+  StringRef functionAttrName = getFunctionAttrName();
+  if (auto function = getFunction()) {
+    p << ' ' << functionAttrName << '(';
+    p.printSymbolName(*function);
+    p << ')';
+  }
+
   printAttributions(p, getWorkgroupKeyword(), getWorkgroupAttributions());
   printAttributions(p, getPrivateKeyword(), getPrivateAttributions());
 
@@ -952,7 +974,8 @@ void LaunchOp::print(OpAsmPrinter &p) {
   p.printRegion(getBody(), /*printEntryBlockArgs=*/false);
   p.printOptionalAttrDict((*this)->getAttrs(), /*elidedAttrs=*/{
                               LaunchOp::getOperandSegmentSizeAttr(),
-                              getNumWorkgroupAttributionsAttrName()});
+                              getNumWorkgroupAttributionsAttrName(),
+                              moduleAttrName, functionAttrName});
 }
 
 // Parse the size assignment blocks for blocks and threads.  These have the form
@@ -990,6 +1013,9 @@ parseSizeAssignment(OpAsmParser &parser,
 ///       `clusters` `(` ssa-id-list `)` `in` ssa-reassignment (Optional)
 ///       `blocks` `(` ssa-id-list `)` `in` ssa-reassignment
 ///       `threads` `(` ssa-id-list `)` `in` ssa-reassignment
+///       (`dynamic_shared_memory_size` ssa-use)?
+///       (`module(` symbol-ref-id `)`)?
+///       (`function(` symbol-ref-id `)`)?
 ///       memory-attribution
 ///       region attr-dict?
 /// ssa-reassignment ::= `(` ssa-id `=` ssa-use (`,` ssa-id `=` ssa-use)* `)`
@@ -1060,6 +1086,27 @@ ParseResult LaunchOp::parse(OpAsmParser &parser, OperationState &result) {
       return failure();
   }
 
+  // Parse optional module attribute.
+  StringRef moduleAttrName = getModuleAttrName(result.name);
+  if (succeeded(parser.parseOptionalKeyword(moduleAttrName))) {
+    FlatSymbolRefAttr moduleSymbol;
+    if (parser.parseLParen() ||
+        parser.parseAttribute(moduleSymbol, Type(), moduleAttrName,
+                              result.attributes) ||
+        parser.parseRParen())
+      return failure();
+  }
+  // Parse optional function attribute.
+  StringRef functionAttrName = getFunctionAttrName(result.name);
+  if (succeeded(parser.parseOptionalKeyword(functionAttrName))) {
+    FlatSymbolRefAttr funcSymbol;
+    if (parser.parseLParen() ||
+        parser.parseAttribute(funcSymbol, Type(), functionAttrName,
+                              result.attributes) ||
+        parser.parseRParen())
+      return failure();
+  }
+
   // Create the region arguments, it has kNumConfigRegionAttributes arguments
   // that correspond to block/thread identifiers and grid/block sizes, all
   // having `index` type, a variadic number of WorkGroup Attributions and
@@ -2439,8 +2486,7 @@ LogicalResult WarpExecuteOnLane0Op::verify() {
   if (getArgs().size() != getWarpRegion().getNumArguments())
     return emitOpError(
         "expected same number op arguments and block arguments.");
-  auto yield =
-      cast<YieldOp>(getWarpRegion().getBlocks().begin()->getTerminator());
+  gpu::YieldOp yield = getTerminator();
   if (yield.getNumOperands() != getNumResults())
     return emitOpError(
         "expected same number of yield operands and return values.");
@@ -2464,6 +2510,10 @@ bool WarpExecuteOnLane0Op::areTypesCompatible(Type lhs, Type rhs) {
       verifyDistributedType(lhs, rhs, getWarpSize(), getOperation()));
 }
 
+gpu::YieldOp WarpExecuteOnLane0Op::getTerminator() {
+  return cast<gpu::YieldOp>(getBody()->getTerminator());
+}
+
 //===----------------------------------------------------------------------===//
 // GPU KernelMetadataAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index 21cb2f6d0a8e0..c766539f9d91a 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -13,6 +13,7 @@
 #include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
 #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
+#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/TransformOps/Utils.h"
@@ -43,6 +44,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/LogicalResult.h"
+#include <optional>
 #include <type_traits>
 
 using namespace mlir;
@@ -170,7 +172,16 @@ void ApplyGPURewritePatternsOp::populatePatterns(RewritePatternSet &patterns) {
 
 void transform::ApplyGPUPromoteShuffleToAMDGPUPatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
-  populateGpuPromoteShuffleToAMDGPUPatterns(patterns);
+  std::optional<StringRef> chipsetName = getChipset();
+  std::optional<amdgpu::Chipset> maybeChipset;
+  if (chipsetName) {
+    FailureOr<amdgpu::Chipset> parsedChipset =
+        amdgpu::Chipset::parse(*chipsetName);
+    assert(llvm::succeeded(parsedChipset) && "expected valid chipset");
+    maybeChipset = parsedChipset;
+  }
+
+  populateGpuPromoteShuffleToAMDGPUPatterns(patterns, maybeChipset);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
index 99f5c5b0cf139..97adad64d78c4 100644
--- a/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/KernelOutlining.cpp
@@ -356,8 +356,8 @@ class GpuKernelOutliningPass
       auto funcWalkResult = func.walk([&](gpu::LaunchOp op) {
         SetVector<Value> operands;
         std::string kernelFnName;
-        if (op.getKernelFunc()) {
-          kernelFnName = op.getKernelFunc()->getRootReference().str();
+        if (op.getFunction()) {
+          kernelFnName = op.getFunction()->str();
         } else {
           kernelFnName =
               Twine(op->getParentOfType<SymbolOpInterface>().getName(),
@@ -403,9 +403,8 @@ class GpuKernelOutliningPass
     OpBuilder builder(context);
     std::string kernelModuleName;
     gpu::GPUModuleOp kernelModule;
-    if (gpuLaunchOp.getKernelModule()) {
-      kernelModuleName =
-          gpuLaunchOp.getKernelModule()->getRootReference().str();
+    if (gpuLaunchOp.getModule()) {
+      kernelModuleName = gpuLaunchOp.getModule()->str();
       kernelModule =
           parentSymbolTable.lookup<gpu::GPUModuleOp>(kernelModuleName);
     } else {
@@ -432,8 +431,7 @@ class GpuKernelOutliningPass
       if (std::optional<SymbolTable::UseRange> symbolUses =
               SymbolTable::getSymbolUses(symbolDefWorklist.pop_back_val())) {
         for (SymbolTable::SymbolUse symbolUse : *symbolUses) {
-          StringRef symbolName =
-              cast<FlatSymbolRefAttr>(symbolUse.getSymbolRef()).getValue();
+          StringAttr symbolName = symbolUse.getSymbolRef().getLeafReference();
           if (symbolTable.lookup(symbolName))
             continue;
 
diff --git a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
index 18c69f5f30e5d..67cef8af1e3b5 100644
--- a/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/PromoteShuffleToAMDGPU.cpp
@@ -11,16 +11,21 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
 #include "mlir/Dialect/GPU/Transforms/Passes.h"
 
 #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/IR/PatternMatch.h"
+#include <optional>
 
 using namespace mlir;
 
 namespace {
+
+constexpr amdgpu::Chipset kGfx950 = amdgpu::Chipset(9, 5, 0);
+
 /// Try to promote `gpu.shuffle` to `amdgpu.swizzle_bitmode`, width must be 64
 /// and offset must be a constant integer in the range [0, 31].
 struct PromoteShuffleToSwizzlePattern
@@ -56,9 +61,48 @@ struct PromoteShuffleToSwizzlePattern
     return success();
   }
 };
+
+/// Try to promote `gpu.shuffle` to `amdgpu.permlane_swap`, width must be 64
+/// and offset must be a constant integer in the set {16, 32}.
+struct PromoteShuffleToPermlanePattern
+    : public OpRewritePattern<gpu::ShuffleOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(gpu::ShuffleOp op,
+                                PatternRewriter &rewriter) const override {
+    if (op.getMode() != gpu::ShuffleMode::XOR)
+      return rewriter.notifyMatchFailure(op,
+                                         "only xor shuffle mode is supported");
+
+    if (!isConstantIntValue(op.getWidth(), 64))
+      return rewriter.notifyMatchFailure(op,
+                                         "only 64 width shuffle is supported");
+
+    std::optional<int64_t> offset = getConstantIntValue(op.getOffset());
+    if (!offset)
+      return rewriter.notifyMatchFailure(op,
+                                         "offset must be a constant integer");
+
+    int64_t offsetValue = *offset;
+    if (offsetValue != 16 && offsetValue != 32)
+      return rewriter.notifyMatchFailure(op, "offset must be either 15 or 31");
+
+    Location loc = op.getLoc();
+    Value res = amdgpu::PermlaneSwapOp::create(
+        rewriter, loc, op.getResult(0).getType(), op.getValue(), offsetValue);
+    Value valid = arith::ConstantIntOp::create(rewriter, loc, 1, /*width*/ 1);
+    rewriter.replaceOp(op, {res, valid});
+    return success();
+  }
+};
+
 } // namespace
 
 void mlir::populateGpuPromoteShuffleToAMDGPUPatterns(
-    RewritePatternSet &patterns) {
-  patterns.add<PromoteShuffleToSwizzlePattern>(patterns.getContext());
+    RewritePatternSet &patterns, std::optional<amdgpu::Chipset> maybeChipset) {
+  patterns.add<PromoteShuffleToSwizzlePattern>(patterns.getContext(),
+                                               /*benefit*/ 1);
+  if (maybeChipset && *maybeChipset >= kGfx950)
+    patterns.add<PromoteShuffleToPermlanePattern>(patterns.getContext(),
+                                                  /*benefit*/ 2);
 }
diff --git a/mlir/lib/Dialect/GPU/Transforms/XeVMAttachTarget.cpp b/mlir/lib/Dialect/GPU/Transforms/XeVMAttachTarget.cpp
index e9cf4939a13b8..6da76e9e7a331 100644
--- a/mlir/lib/Dialect/GPU/Transforms/XeVMAttachTarget.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/XeVMAttachTarget.cpp
@@ -17,6 +17,7 @@
 #include "mlir/Dialect/LLVMIR/XeVMDialect.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/Pass/Pass.h"
+#include "mlir/Target/LLVM/XeVM/Target.h"
 #include "llvm/Support/Regex.h"
 
 namespace mlir {
diff --git a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
index 384d1a0ddccd2..88f531f394765 100644
--- a/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
+++ b/mlir/lib/Dialect/GPU/Utils/DistributionUtils.cpp
@@ -14,6 +14,7 @@
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/IR/Value.h"
+#include "llvm/ADT/DenseMap.h"
 
 #include <numeric>
 
@@ -55,28 +56,30 @@ WarpDistributionPattern::moveRegionToNewWarpOpAndAppendReturns(
     SmallVector<size_t> &indices) const {
   SmallVector<Type> types(warpOp.getResultTypes().begin(),
                           warpOp.getResultTypes().end());
-  auto yield = cast<gpu::YieldOp>(
-      warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
-  llvm::SmallSetVector<Value, 32> yieldValues(yield.getOperands().begin(),
-                                              yield.getOperands().end());
+  gpu::YieldOp yield = warpOp.getTerminator();
+  SmallVector<Value> yieldValues(yield.getOperands().begin(),
+                                 yield.getOperands().end());
+  llvm::SmallDenseMap<Value, unsigned> indexLookup;
+  // Record the value -> first index mapping for faster lookup.
+  for (auto [i, v] : llvm::enumerate(yieldValues)) {
+    if (!indexLookup.count(v))
+      indexLookup[v] = i;
+  }
+
   for (auto [value, type] : llvm::zip_equal(newYieldedValues, newReturnTypes)) {
-    if (yieldValues.insert(value)) {
+    // If the value already exists in the yield, don't create a new output.
+    if (indexLookup.count(value)) {
+      indices.push_back(indexLookup[value]);
+    } else {
+      // If the value is new, add it to the yield and to the types.
+      yieldValues.push_back(value);
       types.push_back(type);
       indices.push_back(yieldValues.size() - 1);
-    } else {
-      // If the value already exit the region don't create a new output.
-      for (auto [idx, yieldOperand] :
-           llvm::enumerate(yieldValues.getArrayRef())) {
-        if (yieldOperand == value) {
-          indices.push_back(idx);
-          break;
-        }
-      }
     }
   }
-  yieldValues.insert_range(newYieldedValues);
+
   WarpExecuteOnLane0Op newWarpOp = moveRegionToNewWarpOpAndReplaceReturns(
-      rewriter, warpOp, yieldValues.getArrayRef(), types);
+      rewriter, warpOp, yieldValues, types);
   rewriter.replaceOp(warpOp,
                      newWarpOp.getResults().take_front(warpOp.getNumResults()));
   return newWarpOp;
@@ -85,8 +88,7 @@ WarpDistributionPattern::moveRegionToNewWarpOpAndAppendReturns(
 OpOperand *WarpDistributionPattern::getWarpResult(
     WarpExecuteOnLane0Op warpOp,
     llvm::function_ref<bool(Operation *)> fn) const {
-  auto yield = cast<gpu::YieldOp>(
-      warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
+  gpu::YieldOp yield = warpOp.getTerminator();
   for (OpOperand &yieldOperand : yield->getOpOperands()) {
     Value yieldValues = yieldOperand.get();
     Operation *definedOp = yieldValues.getDefiningOp();
diff --git a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
index 894de4408c375..6d2a64f94e3ca 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/BasicPtxBuilderInterface.cpp
@@ -13,9 +13,12 @@
 
 #include "mlir/Dialect/LLVMIR/BasicPtxBuilderInterface.h"
 
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/DebugLog.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/Regex.h"
+
 #define DEBUG_TYPE "ptx-builder"
-#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
-#define DBGSNL() (llvm::dbgs() << "\n")
 
 //===----------------------------------------------------------------------===//
 // BasicPtxBuilderInterface
@@ -59,19 +62,37 @@ static char getRegisterType(Value v) {
   return getRegisterType(v.getType());
 }
 
+/// Extract every element of a struct value.
+static SmallVector<Value> extractStructElements(PatternRewriter &rewriter,
+                                                Location loc, Value structVal) {
+  auto structTy = dyn_cast<LLVM::LLVMStructType>(structVal.getType());
+  assert(structTy && "expected LLVM struct");
+
+  SmallVector<Value> elems;
+  for (unsigned i : llvm::seq<unsigned>(0, structTy.getBody().size()))
+    elems.push_back(rewriter.create<LLVM::ExtractValueOp>(loc, structVal, i));
+
+  return elems;
+}
+
 void PtxBuilder::insertValue(Value v, PTXRegisterMod itype) {
-  LLVM_DEBUG(DBGS() << v << "\t Modifier : " << &itype << "\n");
+  LDBG() << v << "\t Modifier : " << itype << "\n";
+  registerModifiers.push_back(itype);
+
   auto getModifier = [&]() -> const char * {
-    if (itype == PTXRegisterMod::ReadWrite) {
-      assert(false && "Read-Write modifier is not supported. Try setting the "
-                      "same value as Write and Read separately.");
-      return "+";
-    }
-    if (itype == PTXRegisterMod::Write) {
+    switch (itype) {
+    case PTXRegisterMod::Read:
+      return "";
+    case PTXRegisterMod::Write:
       return "=";
+    case PTXRegisterMod::ReadWrite:
+      // "Read-Write modifier is not actually supported
+      // Interface will change it to "=" later and add integer mapping
+      return "+";
     }
-    return "";
+    llvm_unreachable("Unknown PTX register modifier");
   };
+
   auto addValue = [&](Value v) {
     if (itype == PTXRegisterMod::Read) {
       ptxOperands.push_back(v);
@@ -107,18 +128,248 @@ void PtxBuilder::insertValue(Value v, PTXRegisterMod itype) {
   ss << getModifier() << getRegisterType(v) << ",";
 }
 
+/// Check if the operation needs to pack and unpack results.
+static bool
+needsPackUnpack(BasicPtxBuilderInterface interfaceOp,
+                bool needsManualRegisterMapping,
+                SmallVectorImpl<PTXRegisterMod> &registerModifiers) {
+  if (needsManualRegisterMapping)
+    return false;
+  const unsigned writeOnlyVals = interfaceOp->getNumResults();
+  const unsigned readWriteVals =
+      llvm::count_if(registerModifiers, [](PTXRegisterMod m) {
+        return m == PTXRegisterMod::ReadWrite;
+      });
+  return (writeOnlyVals + readWriteVals) > 1;
+}
+
+/// Pack the result types of the interface operation.
+/// If the operation has multiple results, it packs them into a struct
+/// type. Otherwise, it returns the original result types.
+static SmallVector<Type>
+packResultTypes(BasicPtxBuilderInterface interfaceOp,
+                bool needsManualRegisterMapping,
+                SmallVectorImpl<PTXRegisterMod> &registerModifiers,
+                SmallVectorImpl<Value> &ptxOperands) {
+  MLIRContext *ctx = interfaceOp->getContext();
+  TypeRange resultRange = interfaceOp->getResultTypes();
+
+  if (!needsPackUnpack(interfaceOp, needsManualRegisterMapping,
+                       registerModifiers)) {
+    // Single value path:
+    if (interfaceOp->getResults().size() == 1)
+      return SmallVector<Type>{resultRange.front()};
+
+    // No declared results: if there is an RW, forward its type.
+    for (auto [m, v] : llvm::zip(registerModifiers, ptxOperands))
+      if (m == PTXRegisterMod::ReadWrite)
+        return SmallVector<Type>{v.getType()};
+  }
+
+  SmallVector<Type> packed;
+  for (auto [m, v] : llvm::zip(registerModifiers, ptxOperands))
+    if (m == PTXRegisterMod::ReadWrite)
+      packed.push_back(v.getType());
+  for (Type t : resultRange)
+    packed.push_back(t);
+
+  if (packed.empty())
+    return {};
+
+  auto sTy = LLVM::LLVMStructType::getLiteral(ctx, packed, /*isPacked=*/false);
+  return SmallVector<Type>{sTy};
+}
+
+/// Canonicalize the register constraints:
+///  - Turn every "+X" into "=X"
+///  - Append (at the very end) the 0-based indices of tokens that were "+X"
+/// Examples:
+///  "+f,+f,+r,=r,=r,r,r" -> "=f,=f,=r,=r,=r,r,r,0,1,2"
+///  "+f,+f,+r,=r,=r"     -> "=f,=f,=r,=r,=r,0,1,2"
+static std::string canonicalizeRegisterConstraints(llvm::StringRef csv) {
+  SmallVector<llvm::StringRef> toks;
+  SmallVector<std::string> out;
+  SmallVector<unsigned> plusIdx;
+
+  csv.split(toks, ',');
+  out.reserve(toks.size() + 8);
+
+  for (unsigned i = 0, e = toks.size(); i < e; ++i) {
+    StringRef t = toks[i].trim();
+    if (t.consume_front("+")) {
+      plusIdx.push_back(i);
+      out.push_back(("=" + t).str());
+    } else {
+      out.push_back(t.str());
+    }
+  }
+
+  // Append indices of original "+X" tokens.
+  for (unsigned idx : plusIdx)
+    out.push_back(std::to_string(idx));
+
+  // Join back to CSV.
+  std::string result;
+  result.reserve(csv.size() + plusIdx.size() * 2);
+  llvm::raw_string_ostream os(result);
+  for (size_t i = 0; i < out.size(); ++i) {
+    if (i)
+      os << ',';
+    os << out[i];
+  }
+  return os.str();
+}
+
+constexpr llvm::StringLiteral kReadWritePrefix{"rw"};
+constexpr llvm::StringLiteral kWriteOnlyPrefix{"w"};
+constexpr llvm::StringLiteral kReadOnlyPrefix{"r"};
+
+/// Returns a regex that matches {$rwN}, {$wN}, {$rN}
+static llvm::Regex getPredicateMappingRegex() {
+  llvm::Regex rx(llvm::formatv(R"(\{\$({0}|{1}|{2})([0-9]+)\})",
+                               kReadWritePrefix, kWriteOnlyPrefix,
+                               kReadOnlyPrefix)
+                     .str());
+  return rx;
+}
+
+void mlir::NVVM::countPlaceholderNumbers(
+    StringRef ptxCode, llvm::SmallDenseSet<unsigned int> &seenRW,
+    llvm::SmallDenseSet<unsigned int> &seenW,
+    llvm::SmallDenseSet<unsigned int> &seenR,
+    llvm::SmallVectorImpl<unsigned int> &rwNums,
+    llvm::SmallVectorImpl<unsigned int> &wNums,
+    llvm::SmallVectorImpl<unsigned int> &rNums) {
+
+  llvm::Regex rx = getPredicateMappingRegex();
+  StringRef rest = ptxCode;
+
+  SmallVector<StringRef, 3> m; // 0: full, 1: kind, 2: number
+  while (!rest.empty() && rx.match(rest, &m)) {
+    unsigned num = 0;
+    (void)m[2].getAsInteger(10, num);
+    // Insert it into the vector only the first time we see this number
+    if (m[1].equals_insensitive(kReadWritePrefix)) {
+      if (seenRW.insert(num).second)
+        rwNums.push_back(num);
+    } else if (m[1].equals_insensitive(kWriteOnlyPrefix)) {
+      if (seenW.insert(num).second)
+        wNums.push_back(num);
+    } else {
+      if (seenR.insert(num).second)
+        rNums.push_back(num);
+    }
+
+    const size_t advance = (size_t)(m[0].data() - rest.data()) + m[0].size();
+    rest = rest.drop_front(advance);
+  }
+}
+
+/// Rewrites `{$rwN}`, `{$wN}`, and `{$rN}` placeholders in `ptxCode` into
+/// compact `$K` indices:
+///   - All `rw*` first (sorted by N),
+///   - Then `w*`,
+///   - Then `r*`.
+/// If there a predicate, it comes always in the end.
+/// Each number is assigned once; duplicates are ignored.
+///
+/// Example Input:
+/// "{
+///       reg .pred p;
+///       setp.ge.s32 p,   {$r0}, {$r1};"
+///       selp.s32 {$rw0}, {$r0}, {$r1}, p;
+///       selp.s32 {$rw1}, {$r0}, {$r1}, p;
+///       selp.s32 {$w0},  {$r0}, {$r1}, p;
+///       selp.s32 {$w1},  {$r0}, {$r1}, p;
+/// }\n"
+/// Example Output:
+/// "{
+///       reg .pred p;
+///       setp.ge.s32 p, $4, $5;"
+///       selp.s32 $0,   $4, $5, p;
+///       selp.s32 $1,   $4, $5, p;
+///       selp.s32 $2,   $4, $5, p;
+///       selp.s32 $3,   $4, $5, p;
+/// }\n"
+static std::string rewriteAsmPlaceholders(llvm::StringRef ptxCode) {
+  llvm::SmallDenseSet<unsigned> seenRW, seenW, seenR;
+  llvm::SmallVector<unsigned> rwNums, wNums, rNums;
+
+  // Step 1. Count Register Placeholder numbers
+  countPlaceholderNumbers(ptxCode, seenRW, seenW, seenR, rwNums, wNums, rNums);
+
+  // Step 2. Sort the Register Placeholder numbers
+  llvm::sort(rwNums);
+  llvm::sort(wNums);
+  llvm::sort(rNums);
+
+  // Step 3. Create mapping from original to new IDs
+  llvm::DenseMap<unsigned, unsigned> rwMap, wMap, rMap;
+  unsigned nextId = 0;
+  for (unsigned n : rwNums)
+    rwMap[n] = nextId++;
+  for (unsigned n : wNums)
+    wMap[n] = nextId++;
+  for (unsigned n : rNums)
+    rMap[n] = nextId++;
+
+  // Step 4. Rewrite the PTX code with new IDs
+  std::string out;
+  out.reserve(ptxCode.size());
+  size_t prev = 0;
+  StringRef rest = ptxCode;
+  SmallVector<StringRef, 3> matches;
+  llvm::Regex rx = getPredicateMappingRegex();
+  while (!rest.empty() && rx.match(rest, &matches)) {
+    // Compute absolute match bounds in the original buffer.
+    size_t absStart = (size_t)(matches[0].data() - ptxCode.data());
+    size_t absEnd = absStart + matches[0].size();
+
+    // Emit text before the match.
+    out.append(ptxCode.data() + prev, ptxCode.data() + absStart);
+
+    // Emit compact $K
+    unsigned num = 0;
+    (void)matches[2].getAsInteger(10, num);
+    unsigned id = 0;
+    if (matches[1].equals_insensitive(kReadWritePrefix))
+      id = rwMap.lookup(num);
+    else if (matches[1].equals_insensitive(kWriteOnlyPrefix))
+      id = wMap.lookup(num);
+    else
+      id = rMap.lookup(num);
+
+    out.push_back('$');
+    out += std::to_string(id);
+
+    prev = absEnd;
+
+    const size_t advance =
+        (size_t)(matches[0].data() - rest.data()) + matches[0].size();
+    rest = rest.drop_front(advance);
+  }
+
+  // Step 5. Tail.
+  out.append(ptxCode.data() + prev, ptxCode.data() + ptxCode.size());
+  return out;
+}
+
 LLVM::InlineAsmOp PtxBuilder::build() {
   auto asmDialectAttr = LLVM::AsmDialectAttr::get(interfaceOp->getContext(),
                                                   LLVM::AsmDialect::AD_ATT);
 
-  auto resultTypes = interfaceOp->getResultTypes();
+  SmallVector<Type> resultTypes = packResultTypes(
+      interfaceOp, needsManualRegisterMapping, registerModifiers, ptxOperands);
 
   // Remove the last comma from the constraints string.
   if (!registerConstraints.empty() &&
       registerConstraints[registerConstraints.size() - 1] == ',')
     registerConstraints.pop_back();
+  registerConstraints = canonicalizeRegisterConstraints(registerConstraints);
 
   std::string ptxInstruction = interfaceOp.getPtx();
+  if (!needsManualRegisterMapping)
+    ptxInstruction = rewriteAsmPlaceholders(ptxInstruction);
 
   // Add the predicate to the asm string.
   if (interfaceOp.getPredicate().has_value() &&
@@ -136,7 +387,7 @@ LLVM::InlineAsmOp PtxBuilder::build() {
       rewriter, interfaceOp->getLoc(),
       /*result types=*/resultTypes,
       /*operands=*/ptxOperands,
-      /*asm_string=*/llvm::StringRef(ptxInstruction),
+      /*asm_string=*/ptxInstruction,
       /*constraints=*/registerConstraints.data(),
       /*has_side_effects=*/interfaceOp.hasSideEffect(),
       /*is_align_stack=*/false, LLVM::TailCallKind::None,
@@ -146,10 +397,89 @@ LLVM::InlineAsmOp PtxBuilder::build() {
 
 void PtxBuilder::buildAndReplaceOp() {
   LLVM::InlineAsmOp inlineAsmOp = build();
-  LLVM_DEBUG(DBGS() << "\n Generated PTX \n\t" << inlineAsmOp << "\n");
-  if (inlineAsmOp->getNumResults() == interfaceOp->getNumResults()) {
-    rewriter.replaceOp(interfaceOp, inlineAsmOp);
-  } else {
+  LDBG() << "\n Generated PTX \n\t" << inlineAsmOp;
+
+  // Case 0: no result at all → just erase wrapper op.
+  if (!hasResult) {
+    rewriter.eraseOp(interfaceOp);
+    return;
+  }
+
+  if (needsManualRegisterMapping) {
+    rewriter.replaceOp(interfaceOp, inlineAsmOp->getResults());
+    return;
+  }
+
+  // Case 1: Simple path, return single scalar
+  if (!needsPackUnpack(interfaceOp, needsManualRegisterMapping,
+                       registerModifiers)) {
+    if (inlineAsmOp->getNumResults() > 0) {
+      rewriter.replaceOp(interfaceOp, inlineAsmOp->getResults());
+    } else {
+      // RW-only case with no declared results: forward the RW value.
+      SmallVector<Value> results;
+      for (auto [m, v] : llvm::zip(registerModifiers, ptxOperands))
+        if (m == PTXRegisterMod::ReadWrite) {
+          results.push_back(v);
+          break;
+        }
+      rewriter.replaceOp(interfaceOp, results);
+    }
+    return;
+  }
+
+  const bool hasRW = llvm::any_of(registerModifiers, [](PTXRegisterMod m) {
+    return m == PTXRegisterMod::ReadWrite;
+  });
+
+  // All multi-value paths produce a single struct result we need to unpack.
+  assert(LLVM::LLVMStructType::classof(inlineAsmOp.getResultTypes().front()) &&
+         "expected struct return for multi-result inline asm");
+  Value structVal = inlineAsmOp.getResult(0);
+  SmallVector<Value> unpacked =
+      extractStructElements(rewriter, interfaceOp->getLoc(), structVal);
+
+  // Case 2: only declared results (no RW): replace the op with all unpacked.
+  if (!hasRW && interfaceOp->getResults().size() > 0) {
+    rewriter.replaceOp(interfaceOp, unpacked);
+    return;
+  }
+
+  // Case 3: RW-only (no declared results): update RW uses and erase wrapper.
+  if (hasRW && interfaceOp->getResults().size() == 0) {
+    unsigned idx = 0;
+    for (auto [m, v] : llvm::zip(registerModifiers, ptxOperands)) {
+      if (m != PTXRegisterMod::ReadWrite)
+        continue;
+      Value repl = unpacked[idx++];
+      v.replaceUsesWithIf(repl, [&](OpOperand &use) {
+        Operation *owner = use.getOwner();
+        return owner != interfaceOp && owner != inlineAsmOp;
+      });
+    }
     rewriter.eraseOp(interfaceOp);
+    return;
+  }
+
+  // Case 4: mixed (RW + declared results).
+  {
+    // First rewrite RW operands in place.
+    unsigned idx = 0;
+    for (auto [m, v] : llvm::zip(registerModifiers, ptxOperands)) {
+      if (m != PTXRegisterMod::ReadWrite)
+        continue;
+      Value repl = unpacked[idx++];
+      v.replaceUsesWithIf(repl, [&](OpOperand &use) {
+        Operation *owner = use.getOwner();
+        return owner != interfaceOp && owner != inlineAsmOp;
+      });
+    }
+    // The remaining unpacked values correspond to the declared results.
+    SmallVector<Value> tail;
+    tail.reserve(unpacked.size() - idx);
+    for (unsigned i = idx, e = unpacked.size(); i < e; ++i)
+      tail.push_back(unpacked[i]);
+
+    rewriter.replaceOp(interfaceOp, tail);
   }
 }
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
index 1e02bfe62f379..634efcaea794e 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMAttrs.cpp
@@ -402,3 +402,20 @@ ModuleFlagAttr::verify(function_ref<InFlightDiagnostic()> emitError,
                         "supported for unknown key '"
                      << key << "'";
 }
+
+//===----------------------------------------------------------------------===//
+// LLVM_TargetAttr
+//===----------------------------------------------------------------------===//
+
+FailureOr<::mlir::Attribute> TargetAttr::query(DataLayoutEntryKey key) {
+  if (auto stringAttrKey = dyn_cast<StringAttr>(key)) {
+    if (stringAttrKey.getValue() == "triple")
+      return getTriple();
+    if (stringAttrKey.getValue() == "chip")
+      return getChip();
+    if (stringAttrKey.getValue() == "features" && getFeatures())
+      return getFeatures();
+  }
+
+  return failure();
+}
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 422039f81855a..a6e89f63c822b 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -141,6 +141,38 @@ static ParseResult parseLLVMLinkage(OpAsmParser &p, LinkageAttr &val) {
   return success();
 }
 
+static ArrayAttr getLLVMAlignParamForCompressExpand(OpBuilder &builder,
+                                                    bool isExpandLoad,
+                                                    uint64_t alignment = 1) {
+  // From
+  // https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics
+  // https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics
+  //
+  //   The pointer alignment defaults to 1.
+  if (alignment == 1) {
+    return nullptr;
+  }
+
+  auto emptyDictAttr = builder.getDictionaryAttr({});
+  auto alignmentAttr = builder.getI64IntegerAttr(alignment);
+  auto namedAttr =
+      builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
+  SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
+  auto alignDictAttr = builder.getDictionaryAttr(attrs);
+  // From
+  // https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics
+  // https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics
+  //
+  //   The align parameter attribute can be provided for [expandload]'s first
+  //   argument. The align parameter attribute can be provided for
+  //   [compressstore]'s second argument.
+  int pos = isExpandLoad ? 0 : 1;
+  return pos == 0 ? builder.getArrayAttr(
+                        {alignDictAttr, emptyDictAttr, emptyDictAttr})
+                  : builder.getArrayAttr(
+                        {emptyDictAttr, alignDictAttr, emptyDictAttr});
+}
+
 //===----------------------------------------------------------------------===//
 // Operand bundle helpers.
 //===----------------------------------------------------------------------===//
@@ -4116,6 +4148,32 @@ LogicalResult LLVM::masked_scatter::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// masked_expandload (intrinsic)
+//===----------------------------------------------------------------------===//
+
+void LLVM::masked_expandload::build(OpBuilder &builder, OperationState &state,
+                                    mlir::TypeRange resTys, Value ptr,
+                                    Value mask, Value passthru,
+                                    uint64_t align) {
+  ArrayAttr argAttrs = getLLVMAlignParamForCompressExpand(builder, true, align);
+  build(builder, state, resTys, ptr, mask, passthru, /*arg_attrs=*/argAttrs,
+        /*res_attrs=*/nullptr);
+}
+
+//===----------------------------------------------------------------------===//
+// masked_compressstore (intrinsic)
+//===----------------------------------------------------------------------===//
+
+void LLVM::masked_compressstore::build(OpBuilder &builder,
+                                       OperationState &state, Value value,
+                                       Value ptr, Value mask, uint64_t align) {
+  ArrayAttr argAttrs =
+      getLLVMAlignParamForCompressExpand(builder, false, align);
+  build(builder, state, value, ptr, mask, /*arg_attrs=*/argAttrs,
+        /*res_attrs=*/nullptr);
+}
+
 //===----------------------------------------------------------------------===//
 // InlineAsmOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index e7d5dad4f71ef..ef3802758cc23 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Interfaces/MemorySlotInterfaces.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/DebugLog.h"
 
 #define DEBUG_TYPE "sroa"
 
@@ -734,9 +735,8 @@ static std::optional<uint64_t> gepToByteOffset(const DataLayout &dataLayout,
               return false;
             })
             .Default([&](Type type) {
-              LLVM_DEBUG(llvm::dbgs()
-                         << "[sroa] Unsupported type for offset computations"
-                         << type << "\n");
+              LDBG() << "[sroa] Unsupported type for offset computations"
+                     << type;
               return true;
             });
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
index e0977f5b616c1..ec1e6edc9bcb4 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/NVVMDialect.cpp
@@ -50,7 +50,6 @@ using namespace NVVM;
 
 // This verifier is shared among the following Ops:
 // CpAsyncBulkTensorGlobalToSharedClusterOp (TMA Load)
-// CpAsyncBulkTensorPrefetchOp (TMA Prefetch)
 // CpAsyncBulkTensorReduceOp (TMA Store-Reduce)
 static LogicalResult cpAsyncBulkTensorCommonVerifier(size_t tensorDims,
                                                      bool isIm2Col,
@@ -98,11 +97,45 @@ LogicalResult CpAsyncOp::verify() {
   return success();
 }
 
+// This verify params can be shared across TMA Load and Prefetch Ops.
+static LogicalResult verifyTMALoadParams(size_t tensorDims, size_t numIm2colOff,
+                                         TMALoadMode mode, Location loc) {
+  if (tensorDims < 1 || tensorDims > 5)
+    return emitError(loc, "expects coordinates between 1 to 5 dimension");
+
+  auto checkTMALoadParams = [&](TMALoadMode mode, bool isIm2col,
+                                size_t expectedIm2colOff) -> LogicalResult {
+    if (isIm2col && (tensorDims < 3))
+      return emitError(loc)
+             << "to use " << stringifyEnum(mode)
+             << " mode, the tensor has to be at least 3-dimensional";
+
+    if (numIm2colOff != expectedIm2colOff)
+      return emitError(loc) << " im2col offsets expected " << expectedIm2colOff
+                            << " (provided " << numIm2colOff << ")";
+
+    return success();
+  };
+
+  switch (mode) {
+  case TMALoadMode::TILE:
+    return checkTMALoadParams(mode, false, 0);
+  case TMALoadMode::IM2COL:
+    return checkTMALoadParams(mode, true, tensorDims - 2);
+  case TMALoadMode::IM2COL_W:
+  case TMALoadMode::IM2COL_W_128:
+    return checkTMALoadParams(mode, true, 2);
+  case TMALoadMode::TILE_GATHER4:
+    return (tensorDims == 5)
+               ? checkTMALoadParams(mode, false, 0)
+               : emitError(loc, "Gather4 mode expects 5 coordinates");
+  }
+  return success();
+}
+
 LogicalResult CpAsyncBulkTensorPrefetchOp::verify() {
-  size_t numIm2ColOffsets = getIm2colOffsets().size();
-  bool isIm2Col = numIm2ColOffsets > 0;
-  return cpAsyncBulkTensorCommonVerifier(getCoordinates().size(), isIm2Col,
-                                         numIm2ColOffsets, getLoc());
+  return verifyTMALoadParams(getCoordinates().size(), getIm2colOffsets().size(),
+                             getMode(), getLoc());
 }
 
 LogicalResult CpAsyncBulkTensorReduceOp::verify() {
@@ -189,6 +222,26 @@ LogicalResult BulkStoreOp::verify() {
   return success();
 }
 
+LogicalResult PMEventOp::verify() {
+  auto eventId = getEventId();
+  auto maskedEventId = getMaskedEventId();
+  if (!maskedEventId && !eventId) {
+    return emitOpError() << "either `id` or `mask` must be set";
+  }
+
+  if (maskedEventId && eventId) {
+    return emitOpError() << "`id` and `mask` cannot be set at the same time";
+  }
+
+  if (eventId) {
+    if (eventId < 0 || eventId > 15) {
+      return emitOpError() << "`id` must be between 0 and 15";
+    }
+  }
+
+  return llvm::success();
+}
+
 // Given the element type of an operand and whether or not it is an accumulator,
 // this function returns the PTX type (`NVVM::MMATypes`) that corresponds to the
 // operand's element type.
@@ -791,24 +844,58 @@ LogicalResult NVVM::WMMAMmaOp::verify() {
 }
 
 LogicalResult NVVM::LdMatrixOp::verify() {
-  unsigned addressSpace =
-      llvm::cast<LLVM::LLVMPointerType>(getPtr().getType()).getAddressSpace();
-  if (addressSpace != NVVM::kSharedMemorySpace)
-    return emitOpError("expected source pointer in memory space 3");
-
-  if (getNum() != 1 && getNum() != 2 && getNum() != 4)
-    return emitOpError("expected num attribute to be 1, 2 or 4");
+  uint32_t num = getNum(), m = getShape().getM(), n = getShape().getN();
+  if (m == 8 && n == 8) {
+    if (num != 1 && num != 2 && num != 4) {
+      return emitOpError("expected num attribute to be 1, 2 or 4 for 8x8 "
+                         "matrix");
+    }
+    if (getEltType() != LdStMatrixEltType::B16) {
+      return emitOpError("expected element type to be b16 for 8x8 matrix");
+    }
+  } else if (m == 8 && n == 16) {
+    if (num != 1 && num != 2 && num != 4) {
+      return emitOpError("expected num attribute to be 1, 2 or 4 for 8x16 "
+                         "matrix");
+    }
+    if (getLayout() != MMALayout::row) {
+      return emitOpError("expected layout to be row for 8x16 matrix");
+    }
+    if (getEltType() != LdStMatrixEltType::B8X16_B4X16_P64 &&
+        getEltType() != LdStMatrixEltType::B8X16_B6X16_P32) {
+      return emitOpError("expected element type to be b8x16.b4x16_p64 or "
+                         "b8x16.b6x16_p32 for 8x16 matrix");
+    }
+  } else if (m == 16 && n == 16) {
+    if (num != 1 && num != 2) {
+      return emitOpError("expected num attribute to be 1 or 2 for 16x16 "
+                         "matrix");
+    }
+    if (getLayout() != MMALayout::col) {
+      return emitOpError("expected layout to be col for 16x16 matrix");
+    }
+    if (getEltType() != LdStMatrixEltType::B8 &&
+        getEltType() != LdStMatrixEltType::B8X16_B4X16_P64 &&
+        getEltType() != LdStMatrixEltType::B8X16_B6X16_P32) {
+      return emitOpError("expected element type to be b8, b8x16.b4x16_p64 or "
+                         "b8x16.b6x16_p32 for 16x16 matrix");
+    }
+  } else {
+    return emitOpError("expected shape to be 8x8, 8x16 or 16x16");
+  }
 
   Type i32 = IntegerType::get(getContext(), 32);
-  if (getNum() == 1 && getType() != i32)
+  uint32_t numElements = (m == 16 && n == 16 ? num * 2 : num);
+  if (numElements == 1 && getType() != i32)
     return emitOpError("expected destination type is i32");
-  if (getNum() == 2 || getNum() == 4) {
+  if (numElements == 2 || numElements == 4) {
     Type dstType = LLVM::LLVMStructType::getLiteral(
-        getContext(), SmallVector<Type>(getNum(), i32));
+        getContext(), SmallVector<Type>(numElements, i32));
     if (getType() != dstType)
       return emitOpError("expected destination type is a structure of ")
-             << getNum() << " elements of type i32";
+             << numElements << " elements of type i32";
   }
+
   return success();
 }
 
@@ -1069,7 +1156,7 @@ std::string NVVM::WgmmaMmaAsyncOp::getPtx() {
   return ptx;
 }
 
-void NVVM::WgmmaMmaAsyncOp::getAsmValues(
+bool NVVM::WgmmaMmaAsyncOp::getAsmValues(
     RewriterBase &rewriter,
     llvm::SmallVectorImpl<std::pair<mlir::Value, mlir::NVVM::PTXRegisterMod>>
         &asmValues) {
@@ -1100,7 +1187,9 @@ void NVVM::WgmmaMmaAsyncOp::getAsmValues(
         {makeConstantI32(rewriter, 1 - static_cast<int>(getLayoutB())),
          mlir::NVVM::PTXRegisterMod::Read});
   }
+  return true; // Has manual mapping
 }
+
 LogicalResult NVVM::FenceProxyOp::verify() {
   if (getKind() == NVVM::ProxyKind::TENSORMAP)
     return emitOpError() << "tensormap proxy is not a supported proxy kind";
@@ -1379,28 +1468,57 @@ mlir::NVVM::IDArgPair CpAsyncBulkSharedCTAToGlobalOp::getIntrinsicIDAndArgs(
   return {id, std::move(args)};
 }
 
-llvm::Intrinsic::ID CpAsyncBulkTensorPrefetchOp::getIntrinsicID(int tensorDims,
-                                                                bool isIm2Col) {
-  switch (tensorDims) {
-  case 1:
-    return llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_1d;
-  case 2:
-    return llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_2d;
-  case 3:
-    return isIm2Col
-               ? llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_3d
-               : llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_3d;
-  case 4:
-    return isIm2Col
-               ? llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_4d
-               : llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_4d;
-  case 5:
-    return isIm2Col
-               ? llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_5d
-               : llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_5d;
-  default:
-    llvm_unreachable("Invalid TensorDim in CpAsyncBulkTensorPrefetchOp.");
-  }
+mlir::NVVM::IDArgPair CpAsyncBulkTensorPrefetchOp::getIntrinsicIDAndArgs(
+    Operation &op, LLVM::ModuleTranslation &mt, llvm::IRBuilderBase &builder) {
+  auto thisOp = cast<NVVM::CpAsyncBulkTensorPrefetchOp>(op);
+  llvm::SmallVector<llvm::Value *> args;
+
+  // Fill the Intrinsic Args
+  args.push_back(mt.lookupValue(thisOp.getTmaDescriptor()));
+
+  for (auto v : thisOp.getCoordinates())
+    args.push_back(mt.lookupValue(v));
+  for (auto v : thisOp.getIm2colOffsets())
+    args.push_back(mt.lookupValue(v));
+
+  mlir::Value cacheHint = thisOp.getL2CacheHint();
+  const bool hasCacheHint = static_cast<bool>(cacheHint);
+  llvm::Value *i64Unused =
+      llvm::ConstantInt::get(llvm::Type::getInt64Ty(mt.getLLVMContext()), 0);
+  args.push_back(hasCacheHint ? mt.lookupValue(cacheHint) : i64Unused);
+  args.push_back(builder.getInt1(hasCacheHint));
+
+  const unsigned NI = llvm::Intrinsic::not_intrinsic;
+  static constexpr llvm::Intrinsic::ID IDTable[][6] = {
+      {NI, llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_1d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_2d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_3d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_4d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_5d},
+      {NI, NI, NI,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_3d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_4d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_5d},
+      {NI, NI, NI,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_w_3d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_w_4d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_w_5d},
+      {NI, NI, NI,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_w_128_3d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_w_128_4d,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_im2col_w_128_5d},
+      {NI, NI, NI, NI, NI,
+       llvm::Intrinsic::nvvm_cp_async_bulk_tensor_prefetch_tile_gather4_2d}};
+
+  static_assert(getMaxEnumValForTMALoadMode() == std::size(IDTable) - 1,
+                "TMALoadModes must match number of rows in IDTable");
+  size_t mode = static_cast<size_t>(thisOp.getMode());
+  size_t dim = thisOp.getCoordinates().size();
+  llvm::Intrinsic::ID id = IDTable[mode][dim];
+  if (id == llvm::Intrinsic::not_intrinsic)
+    llvm_unreachable("Invalid intrinsic for CpAsyncBulkTensorPrefetchOp.");
+
+  return {id, std::move(args)};
 }
 
 #define CP_ASYNC_BULK_TENSOR_REDUCE_MODE(op, dim, mode)                        \
@@ -1816,6 +1934,21 @@ llvm::Intrinsic::ID PrefetchOp::getIntrinsicID(NVVM::PrefetchOp &op) {
   }
 }
 
+bool NVVM::InlinePtxOp::getAsmValues(
+    RewriterBase &rewriter,
+    llvm::SmallVectorImpl<std::pair<mlir::Value, mlir::NVVM::PTXRegisterMod>>
+        &asmValues) {
+  for (auto arg : getReadWriteArgs())
+    asmValues.push_back({arg, mlir::NVVM::PTXRegisterMod::ReadWrite});
+  for (auto arg : getResults())
+    asmValues.push_back({arg, mlir::NVVM::PTXRegisterMod::Write});
+  for (auto arg : getReadOnlyArgs())
+    asmValues.push_back({arg, mlir::NVVM::PTXRegisterMod::Read});
+  if (getPredicate())
+    asmValues.push_back({getPredicate(), mlir::NVVM::PTXRegisterMod::Read});
+  return false; // No manual mapping needed
+}
+
 //===----------------------------------------------------------------------===//
 // NVVMDialect initialization, type parsing, and registration.
 //===----------------------------------------------------------------------===//
@@ -1854,19 +1987,31 @@ LogicalResult NVVMDialect::verifyOperationAttribute(Operation *op,
       attrName == NVVMDialect::getReqntidAttrName() ||
       attrName == NVVMDialect::getClusterDimAttrName()) {
     auto values = llvm::dyn_cast<DenseI32ArrayAttr>(attr.getValue());
-    if (!values || values.empty() || values.size() > 3)
+    if (!values || values.empty() || values.size() > 3) {
       return op->emitError()
              << "'" << attrName
              << "' attribute must be integer array with maximum 3 index";
+    }
   }
   // If minctasm / maxnreg / cluster_max_blocks exist, it must be an integer
   // attribute
   if (attrName == NVVMDialect::getMinctasmAttrName() ||
       attrName == NVVMDialect::getMaxnregAttrName() ||
       attrName == NVVMDialect::getClusterMaxBlocksAttrName()) {
-    if (!llvm::dyn_cast<IntegerAttr>(attr.getValue()))
+    if (!llvm::dyn_cast<IntegerAttr>(attr.getValue())) {
       return op->emitError()
              << "'" << attrName << "' attribute must be integer constant";
+    }
+  }
+  // blocksareclusters must be used along with reqntid and cluster_dim
+  if (attrName == NVVMDialect::getBlocksAreClustersAttrName()) {
+    if (!op->hasAttr(NVVMDialect::getReqntidAttrName()) ||
+        !op->hasAttr(NVVMDialect::getClusterDimAttrName())) {
+      return op->emitError()
+             << "'" << attrName << "' attribute must be used along with "
+             << "'" << NVVMDialect::getReqntidAttrName() << "' and "
+             << "'" << NVVMDialect::getClusterDimAttrName() << "'";
+    }
   }
 
   return success();
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/DIExpressionRewriter.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/DIExpressionRewriter.cpp
index 8317b67a905e6..23b4130482314 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/DIExpressionRewriter.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/DIExpressionRewriter.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/LLVMIR/Transforms/DIExpressionRewriter.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 
 using namespace mlir;
 using namespace LLVM;
@@ -63,9 +63,8 @@ DIExpressionRewriter::simplify(DIExpressionAttr expr,
   }
 
   if (maxNumRewrites && numRewrites >= *maxNumRewrites) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "LLVMDIExpressionSimplifier exceeded max num rewrites ("
-               << maxNumRewrites << ")\n");
+    LDBG() << "LLVMDIExpressionSimplifier exceeded max num rewrites ("
+           << maxNumRewrites << ")";
     // Skip rewriting the rest.
     result.append(inputs.begin(), inputs.end());
   }
diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
index b951df8af3550..4ea2ac957fa1a 100644
--- a/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.cpp
@@ -129,7 +129,6 @@ handleInlinedAllocas(Operation *call,
       OpBuilder::InsertionGuard insertionGuard(builder);
       builder.setInsertionPoint(allocaOp);
       LLVM::LifetimeStartOp::create(builder, allocaOp.getLoc(),
-                                    arraySize.getValue().getLimitedValue(),
                                     allocaOp.getResult());
     }
     allocaOp->moveAfter(newConstant);
@@ -147,7 +146,6 @@ handleInlinedAllocas(Operation *call,
     for (auto &[allocaOp, arraySize, shouldInsertLifetime] : allocasToMove) {
       if (shouldInsertLifetime)
         LLVM::LifetimeEndOp::create(builder, allocaOp.getLoc(),
-                                    arraySize.getValue().getLimitedValue(),
                                     allocaOp.getResult());
     }
   }
@@ -237,8 +235,10 @@ getUnderlyingObjectSet(Value pointerValue) {
   WalkContinuation walkResult = walkSlice(pointerValue, [&](Value val) {
     // Attempt to advance to the source of the underlying view-like operation.
     // Examples of view-like operations include GEPOp and AddrSpaceCastOp.
-    if (auto viewOp = val.getDefiningOp<ViewLikeOpInterface>())
-      return WalkContinuation::advanceTo(viewOp.getViewSource());
+    if (auto viewOp = val.getDefiningOp<ViewLikeOpInterface>()) {
+      if (val == viewOp.getViewDest())
+        return WalkContinuation::advanceTo(viewOp.getViewSource());
+    }
 
     // Attempt to advance to control flow predecessors.
     std::optional<SmallVector<Value>> controlFlowPredecessors =
diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
index 34c63d378e1ca..e0e371617729a 100644
--- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
+++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp
@@ -194,9 +194,10 @@ static void buildMatmulOp(OpBuilder &b, OperationState &state,
                           ArrayRef<AffineMap> indexingMaps) {
   // Initialize indexingMaps attribute, for MatmulOp.
   SmallVector<Attribute, 3> indexingMapsAttrVal;
-  indexingMapsAttrVal = llvm::map_to_vector(
-      MatmulOp::getDefaultIndexingMaps(b.getContext()),
-      [](AffineMap map) -> Attribute { return AffineMapAttr::get(map); });
+  indexingMapsAttrVal =
+      llvm::map_to_vector(indexingMaps, [](AffineMap map) -> Attribute {
+        return AffineMapAttr::get(map);
+      });
   state.addAttribute("indexing_maps", b.getArrayAttr(indexingMapsAttrVal));
   return buildStructuredOp(b, state, resultTensorTypes, inputs, outputs,
                            attributes, regionBuilder);
@@ -1569,40 +1570,50 @@ ParseResult MapOp::parse(OpAsmParser &parser, OperationState &result) {
   return success();
 }
 
-// Retrieve the operation from the body, if it is the only one (except
-// yield) and if it gets the same amount of arguments as the body does.
-// If initFirst flag is enabled, we check that init takes the first position in
-// operands of payload.
-static Operation *findPayloadOp(Block *body, bool initFirst = false) {
+static bool canUseShortForm(Block *body, bool initFirst = false) {
+  // Check if the body can be printed in short form. The following 4 conditions
+  // must be satisfied:
+
+  // 1) The body must contain exactly 2 operations: the payload op and a yield.
   if (body->getOperations().size() != 2)
-    return nullptr;
+    return false;
   Operation &payload = body->getOperations().front();
-  assert(isa<YieldOp>(body->getOperations().back()));
 
+  // 2) The payload op must have the same number of operands as the number of
+  //    block arguments.
   if (payload.getNumOperands() == 0 ||
       payload.getNumOperands() != body->getNumArguments())
-    return nullptr;
+    return false;
+
+  // 3) If `initFirst` is true (e.g., for reduction ops), the init block
+  //    must be the first operand of the payload op, otherwise, the operands
+  //    must match the block arguments in order.
   if (initFirst) {
     // check init
     if (payload.getOperands().back() != body->getArgument(0))
-      return nullptr;
+      return false;
     // check rest
     for (const auto &[operand, bbArg] :
          llvm::zip(payload.getOperands(), body->getArguments().drop_front())) {
       if (bbArg != operand)
-        return nullptr;
+        return false;
     }
   } else {
     for (const auto &[operand, bbArg] :
          llvm::zip(payload.getOperands(), body->getArguments())) {
       if (bbArg != operand)
-        return nullptr;
+        return false;
     }
   }
-  return &payload;
+
+  // 4) The `yield` operand must be the result of the payload op.
+  auto yieldOp = cast<YieldOp>(body->getTerminator());
+  return yieldOp.getNumOperands() == 1 &&
+         yieldOp.getOperand(0).getDefiningOp() &&
+         yieldOp.getOperand(0).getDefiningOp() == &payload;
 }
 
-void printShortForm(OpAsmPrinter &p, Operation *payloadOp) {
+static void printShortForm(OpAsmPrinter &p, Operation *payloadOp) {
   SmallVector<StringRef> elidedAttrs;
   std::string attrToElide;
   p << " { " << payloadOp->getName().getStringRef();
@@ -1621,15 +1632,15 @@ void printShortForm(OpAsmPrinter &p, Operation *payloadOp) {
 
 void MapOp::print(OpAsmPrinter &p) {
   Block *mapper = getBody();
-  Operation *payloadOp = findPayloadOp(mapper);
-  if (payloadOp) {
-    printShortForm(p, payloadOp);
+  bool useShortForm = canUseShortForm(mapper);
+  if (useShortForm) {
+    printShortForm(p, &mapper->getOperations().front());
   }
 
   printCommonStructuredOpParts(p, getDpsInputs(), getDpsInits());
   p.printOptionalAttrDict((*this)->getAttrs());
 
-  if (!payloadOp) {
+  if (!useShortForm) {
     // Print region if the payload op was not detected.
     p.increaseIndent();
     p.printNewline();
@@ -1828,15 +1839,15 @@ static void printDenseI64ArrayAttr(OpAsmPrinter &p, StringRef attributeName,
 
 void ReduceOp::print(OpAsmPrinter &p) {
   Block *mapper = getBody();
-  Operation *payloadOp = findPayloadOp(mapper, /*initFirst=*/true);
-  if (payloadOp) {
-    printShortForm(p, payloadOp);
+  bool useShortForm = canUseShortForm(mapper, /*initFirst=*/true);
+  if (useShortForm) {
+    printShortForm(p, &mapper->getOperations().front());
   }
 
   printCommonStructuredOpParts(p, getDpsInputs(), getDpsInits());
   printDenseI64ArrayAttr(p, getDimensionsAttrName(), getDimensions());
   p.printOptionalAttrDict((*this)->getAttrs(), {getDimensionsAttrName()});
-  if (!payloadOp) {
+  if (!useShortForm) {
     // Print region if the payload op was not detected.
     p.increaseIndent();
     p.printNewline();
@@ -3749,6 +3760,25 @@ std::pair<int64_t, int64_t> getFmrFromWinogradConv2DFmr(WinogradConv2DFmr fmr) {
 // MatMulOp
 //===----------------------------------------------------------------------===//
 
+static FailureOr<SmallVector<SmallVector<int64_t>>>
+getAffineResultPositions(ArrayAttr maps) {
+  SmallVector<SmallVector<int64_t>> positions;
+  for (auto map : maps) {
+    AffineMapAttr attr = dyn_cast<AffineMapAttr>(map);
+    if (!attr)
+      return failure();
+    SmallVector<int64_t> pos;
+    for (auto result : attr.getAffineMap().getResults()) {
+      auto dim = dyn_cast<AffineDimExpr>(result);
+      if (!dim)
+        return failure();
+      pos.push_back(dim.getPosition());
+    }
+    positions.push_back(pos);
+  }
+  return positions;
+}
+
 /// Returns a list of AffineMap with the typical matmul indexing charactristic.
 SmallVector<AffineMap> MatmulOp::getDefaultIndexingMaps(MLIRContext *context) {
   AffineExpr d0, d1, d2;
@@ -3760,6 +3790,20 @@ SmallVector<AffineMap> MatmulOp::getDefaultIndexingMaps(MLIRContext *context) {
   return indexingMaps;
 }
 
+bool MatmulOp::isDefaultIndexingMaps(Attribute attr) {
+  ArrayAttr maps = dyn_cast<ArrayAttr>(attr);
+  if (!maps)
+    return false;
+  if (maps.size() != 3)
+    return false;
+  auto positions = getAffineResultPositions(maps);
+  if (failed(positions))
+    return false;
+  return (*positions)[0] == SmallVector<int64_t>{0, 2} &&
+         (*positions)[1] == SmallVector<int64_t>{2, 1} &&
+         (*positions)[2] == SmallVector<int64_t>{0, 1};
+}
+
 SmallVector<utils::IteratorType> MatmulOp::getIteratorTypesArray() {
   return SmallVector<utils::IteratorType>{utils::IteratorType::parallel,
                                           utils::IteratorType::parallel,
@@ -3912,6 +3956,380 @@ Speculation::Speculatability MatmulOp::getSpeculatability() {
   return getGenericSpeculatabilityImpl(cast<LinalgOp>(getOperation()));
 }
 
+SmallVector<AffineMap>
+MatmulTransposeAOp::getDefaultIndexingMaps(OpBuilder &builder) {
+  AffineExpr d0, d1, d2;
+  MLIRContext *context = builder.getContext();
+  bindDims(context, d0, d1, d2);
+  AffineMap mapLHS = AffineMap::get(3, 0, {d2, d0}, context);
+  AffineMap mapRHS = AffineMap::get(3, 0, {d2, d1}, context);
+  AffineMap mapOut = AffineMap::get(3, 0, {d0, d1}, context);
+  return {mapLHS, mapRHS, mapOut};
+}
+
+bool MatmulTransposeAOp::isDefaultIndexingMaps(Attribute attr) {
+  ArrayAttr maps = dyn_cast<ArrayAttr>(attr);
+  if (!maps)
+    return false;
+  if (maps.size() != 3)
+    return false;
+  auto positions = getAffineResultPositions(maps);
+  if (failed(positions))
+    return false;
+  return (*positions)[0] == SmallVector<int64_t>{2, 0} &&
+         (*positions)[1] == SmallVector<int64_t>{2, 1} &&
+         (*positions)[2] == SmallVector<int64_t>{0, 1};
+}
+
+void linalg::MatmulTransposeAOp::build(OpBuilder &builder,
+                                       OperationState &result,
+                                       ValueRange inputs, ValueRange outputs,
+                                       ArrayRef<NamedAttribute> attributes) {
+  buildMatmulOp(builder, result, std::nullopt, inputs, outputs, attributes,
+                MatmulOp::getRegionBuilder(), getDefaultIndexingMaps(builder));
+}
+
+MatmulTransposeAOp
+MatmulTransposeAOp::create(OpBuilder &builder, Location location,
+                           ValueRange inputs, ValueRange outputs,
+                           ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, inputs, outputs, attributes);
+  auto res = dyn_cast<MatmulTransposeAOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+void linalg::MatmulTransposeAOp::build(OpBuilder &builder,
+                                       OperationState &result,
+                                       TypeRange resultTensorTypes,
+                                       ValueRange inputs, ValueRange outputs,
+                                       ArrayRef<NamedAttribute> attributes) {
+  buildMatmulOp(builder, result, resultTensorTypes, inputs, outputs, attributes,
+                MatmulOp::getRegionBuilder(), getDefaultIndexingMaps(builder));
+}
+
+MatmulTransposeAOp
+MatmulTransposeAOp::create(OpBuilder &builder, Location location,
+                           TypeRange resultTensorTypes, ValueRange inputs,
+                           ValueRange outputs,
+                           ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, resultTensorTypes, inputs, outputs, attributes);
+  auto res = dyn_cast<MatmulTransposeAOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+void linalg::MatmulTransposeAOp::build(OpBuilder &builder,
+                                       OperationState &result,
+                                       TypeRange resultTensorTypes,
+                                       ValueRange inputs, ValueRange outputs,
+                                       Attribute cast,
+                                       ArrayRef<NamedAttribute> attributes) {
+  result.addAttribute("cast", cast);
+  buildMatmulOp(builder, result, resultTensorTypes, inputs, outputs, attributes,
+                MatmulOp::getRegionBuilder(), getDefaultIndexingMaps(builder));
+}
+
+MatmulTransposeAOp
+MatmulTransposeAOp::create(OpBuilder &builder, Location location,
+                           TypeRange resultTensorTypes, ValueRange inputs,
+                           ValueRange outputs, Attribute cast,
+                           ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, resultTensorTypes, inputs, outputs, cast, attributes);
+  auto res = dyn_cast<MatmulTransposeAOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+bool MatmulTransposeAOp::classof(Operation *op) {
+  return dyn_cast_or_null<linalg::MatmulOp>(op) &&
+         MatmulTransposeAOp::isDefaultIndexingMaps(
+             op->getAttr("indexing_maps"));
+}
+
+SmallVector<AffineMap>
+MatmulTransposeBOp::getDefaultIndexingMaps(OpBuilder &builder) {
+  AffineExpr d0, d1, d2;
+  MLIRContext *context = builder.getContext();
+  bindDims(context, d0, d1, d2);
+  AffineMap mapLHS = AffineMap::get(3, 0, {d0, d2}, context);
+  AffineMap mapRHS = AffineMap::get(3, 0, {d1, d2}, context);
+  AffineMap mapOut = AffineMap::get(3, 0, {d0, d1}, context);
+  return {mapLHS, mapRHS, mapOut};
+}
+
+bool MatmulTransposeBOp::isDefaultIndexingMaps(Attribute attr) {
+  ArrayAttr maps = dyn_cast<ArrayAttr>(attr);
+  if (!maps)
+    return false;
+  if (maps.size() != 3)
+    return false;
+  auto positions = getAffineResultPositions(maps);
+  if (failed(positions))
+    return false;
+  return (*positions)[0] == SmallVector<int64_t>{0, 2} &&
+         (*positions)[1] == SmallVector<int64_t>{1, 2} &&
+         (*positions)[2] == SmallVector<int64_t>{0, 1};
+}
+
+void linalg::MatmulTransposeBOp::build(OpBuilder &builder,
+                                       OperationState &result,
+                                       ValueRange inputs, ValueRange outputs,
+                                       ArrayRef<NamedAttribute> attributes) {
+  buildMatmulOp(builder, result, std::nullopt, inputs, outputs, attributes,
+                MatmulOp::getRegionBuilder(), getDefaultIndexingMaps(builder));
+}
+
+MatmulTransposeBOp
+MatmulTransposeBOp::create(OpBuilder &builder, Location location,
+                           ValueRange inputs, ValueRange outputs,
+                           ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, inputs, outputs, attributes);
+  auto res = dyn_cast<MatmulTransposeBOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+void linalg::MatmulTransposeBOp::build(OpBuilder &builder,
+                                       OperationState &result,
+                                       TypeRange resultTensorTypes,
+                                       ValueRange inputs, ValueRange outputs,
+                                       ArrayRef<NamedAttribute> attributes) {
+  buildMatmulOp(builder, result, resultTensorTypes, inputs, outputs, attributes,
+                MatmulOp::getRegionBuilder(), getDefaultIndexingMaps(builder));
+}
+
+MatmulTransposeBOp
+MatmulTransposeBOp::create(OpBuilder &builder, Location location,
+                           TypeRange resultTensorTypes, ValueRange inputs,
+                           ValueRange outputs,
+                           ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, resultTensorTypes, inputs, outputs, attributes);
+  auto res = dyn_cast<MatmulTransposeBOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+void linalg::MatmulTransposeBOp::build(OpBuilder &builder,
+                                       OperationState &result,
+                                       TypeRange resultTensorTypes,
+                                       ValueRange inputs, ValueRange outputs,
+                                       Attribute cast,
+                                       ArrayRef<NamedAttribute> attributes) {
+  result.addAttribute("cast", cast);
+  buildMatmulOp(builder, result, resultTensorTypes, inputs, outputs, attributes,
+                MatmulOp::getRegionBuilder(), getDefaultIndexingMaps(builder));
+}
+
+MatmulTransposeBOp
+MatmulTransposeBOp::create(OpBuilder &builder, Location location,
+                           TypeRange resultTensorTypes, ValueRange inputs,
+                           ValueRange outputs, Attribute cast,
+                           ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, resultTensorTypes, inputs, outputs, cast, attributes);
+  auto res = dyn_cast<MatmulTransposeBOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+bool MatmulTransposeBOp::classof(Operation *op) {
+  return dyn_cast_or_null<linalg::MatmulOp>(op) &&
+         MatmulTransposeBOp::isDefaultIndexingMaps(
+             op->getAttr("indexing_maps"));
+}
+
+SmallVector<AffineMap>
+BatchMatmulTransposeAOp::getDefaultIndexingMaps(OpBuilder &builder) {
+  AffineExpr d0, d1, d2, d3;
+  MLIRContext *context = builder.getContext();
+  bindDims(context, d0, d1, d2, d3);
+  AffineMap mapLHS = AffineMap::get(4, 0, {d0, d3, d1}, context);
+  AffineMap mapRHS = AffineMap::get(4, 0, {d0, d3, d2}, context);
+  AffineMap mapOut = AffineMap::get(4, 0, {d0, d1, d2}, context);
+  return {mapLHS, mapRHS, mapOut};
+}
+
+bool BatchMatmulTransposeAOp::isDefaultIndexingMaps(Attribute attr) {
+  ArrayAttr maps = dyn_cast<ArrayAttr>(attr);
+  if (!maps)
+    return false;
+  if (maps.size() != 3)
+    return false;
+  auto positions = getAffineResultPositions(maps);
+  if (failed(positions))
+    return false;
+  return (*positions)[0] == SmallVector<int64_t>{0, 3, 1} &&
+         (*positions)[1] == SmallVector<int64_t>{0, 3, 2} &&
+         (*positions)[2] == SmallVector<int64_t>{0, 1, 2};
+}
+
+void linalg::BatchMatmulTransposeAOp::build(
+    OpBuilder &builder, OperationState &result, ValueRange inputs,
+    ValueRange outputs, ArrayRef<NamedAttribute> attributes) {
+  buildMatmulOp(builder, result, std::nullopt, inputs, outputs, attributes,
+                BatchMatmulOp::getRegionBuilder(),
+                getDefaultIndexingMaps(builder));
+}
+
+BatchMatmulTransposeAOp
+BatchMatmulTransposeAOp::create(OpBuilder &builder, Location location,
+                                ValueRange inputs, ValueRange outputs,
+                                ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, inputs, outputs, attributes);
+  auto res = dyn_cast<BatchMatmulTransposeAOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+void linalg::BatchMatmulTransposeAOp::build(
+    OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
+    ValueRange inputs, ValueRange outputs,
+    ArrayRef<NamedAttribute> attributes) {
+  buildMatmulOp(builder, result, resultTensorTypes, inputs, outputs, attributes,
+                BatchMatmulOp::getRegionBuilder(),
+                getDefaultIndexingMaps(builder));
+}
+
+BatchMatmulTransposeAOp
+BatchMatmulTransposeAOp::create(OpBuilder &builder, Location location,
+                                TypeRange resultTensorTypes, ValueRange inputs,
+                                ValueRange outputs,
+                                ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, resultTensorTypes, inputs, outputs, attributes);
+  auto res = dyn_cast<BatchMatmulTransposeAOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+void linalg::BatchMatmulTransposeAOp::build(
+    OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
+    ValueRange inputs, ValueRange outputs, Attribute cast,
+    ArrayRef<NamedAttribute> attributes) {
+  result.addAttribute("cast", cast);
+  buildMatmulOp(builder, result, resultTensorTypes, inputs, outputs, attributes,
+                BatchMatmulOp::getRegionBuilder(),
+                getDefaultIndexingMaps(builder));
+}
+
+BatchMatmulTransposeAOp
+BatchMatmulTransposeAOp::create(OpBuilder &builder, Location location,
+                                TypeRange resultTensorTypes, ValueRange inputs,
+                                ValueRange outputs, Attribute cast,
+                                ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, resultTensorTypes, inputs, outputs, cast, attributes);
+  auto res = dyn_cast<BatchMatmulTransposeAOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+bool BatchMatmulTransposeAOp::classof(Operation *op) {
+  return dyn_cast_or_null<linalg::BatchMatmulOp>(op) &&
+         BatchMatmulTransposeAOp::isDefaultIndexingMaps(
+             op->getAttr("indexing_maps"));
+}
+
+SmallVector<AffineMap>
+BatchMatmulTransposeBOp::getDefaultIndexingMaps(OpBuilder &builder) {
+  AffineExpr d0, d1, d2, d3;
+  MLIRContext *context = builder.getContext();
+  bindDims(context, d0, d1, d2, d3);
+  AffineMap mapLHS = AffineMap::get(4, 0, {d0, d1, d3}, context);
+  AffineMap mapRHS = AffineMap::get(4, 0, {d0, d2, d3}, context);
+  AffineMap mapOut = AffineMap::get(4, 0, {d0, d1, d2}, context);
+  return {mapLHS, mapRHS, mapOut};
+}
+
+bool BatchMatmulTransposeBOp::isDefaultIndexingMaps(Attribute attr) {
+  ArrayAttr maps = dyn_cast<ArrayAttr>(attr);
+  if (!maps)
+    return false;
+  if (maps.size() != 3)
+    return false;
+  auto positions = getAffineResultPositions(maps);
+  if (failed(positions))
+    return false;
+  return (*positions)[0] == SmallVector<int64_t>{0, 1, 3} &&
+         (*positions)[1] == SmallVector<int64_t>{0, 2, 3} &&
+         (*positions)[2] == SmallVector<int64_t>{0, 1, 2};
+}
+
+void linalg::BatchMatmulTransposeBOp::build(
+    OpBuilder &builder, OperationState &result, ValueRange inputs,
+    ValueRange outputs, ArrayRef<NamedAttribute> attributes) {
+  buildMatmulOp(builder, result, std::nullopt, inputs, outputs, attributes,
+                BatchMatmulOp::getRegionBuilder(),
+                getDefaultIndexingMaps(builder));
+}
+
+BatchMatmulTransposeBOp
+BatchMatmulTransposeBOp::create(OpBuilder &builder, Location location,
+                                ValueRange inputs, ValueRange outputs,
+                                ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, inputs, outputs, attributes);
+  auto res = dyn_cast<BatchMatmulTransposeBOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+void linalg::BatchMatmulTransposeBOp::build(
+    OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
+    ValueRange inputs, ValueRange outputs,
+    ArrayRef<NamedAttribute> attributes) {
+  buildMatmulOp(builder, result, resultTensorTypes, inputs, outputs, attributes,
+                BatchMatmulOp::getRegionBuilder(),
+                getDefaultIndexingMaps(builder));
+}
+
+BatchMatmulTransposeBOp
+BatchMatmulTransposeBOp::create(OpBuilder &builder, Location location,
+                                TypeRange resultTensorTypes, ValueRange inputs,
+                                ValueRange outputs,
+                                ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, resultTensorTypes, inputs, outputs, attributes);
+  auto res = dyn_cast<BatchMatmulTransposeBOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+void linalg::BatchMatmulTransposeBOp::build(
+    OpBuilder &builder, OperationState &result, TypeRange resultTensorTypes,
+    ValueRange inputs, ValueRange outputs, Attribute cast,
+    ArrayRef<NamedAttribute> attributes) {
+  result.addAttribute("cast", cast);
+  buildMatmulOp(builder, result, resultTensorTypes, inputs, outputs, attributes,
+                BatchMatmulOp::getRegionBuilder(),
+                getDefaultIndexingMaps(builder));
+}
+
+BatchMatmulTransposeBOp
+BatchMatmulTransposeBOp::create(OpBuilder &builder, Location location,
+                                TypeRange resultTensorTypes, ValueRange inputs,
+                                ValueRange outputs, Attribute cast,
+                                ArrayRef<NamedAttribute> attributes) {
+  OperationState state(location, getOperationName());
+  build(builder, state, resultTensorTypes, inputs, outputs, cast, attributes);
+  auto res = dyn_cast<BatchMatmulTransposeBOp>(builder.create(state));
+  assert(res && "builder didn't return the right type");
+  return res;
+}
+
+bool BatchMatmulTransposeBOp::classof(Operation *op) {
+  return dyn_cast_or_null<linalg::BatchMatmulOp>(op) &&
+         BatchMatmulTransposeBOp::isDefaultIndexingMaps(
+             op->getAttr("indexing_maps"));
+}
+
 //===----------------------------------------------------------------------===//
 // ContractOp
 //===----------------------------------------------------------------------===//
@@ -4120,6 +4538,20 @@ BatchMatmulOp::getDefaultIndexingMaps(MLIRContext *context) {
   return indexingMaps;
 }
 
+bool BatchMatmulOp::isDefaultIndexingMaps(Attribute attr) {
+  ArrayAttr maps = dyn_cast<ArrayAttr>(attr);
+  if (!maps)
+    return false;
+  if (maps.size() != 3)
+    return false;
+  auto positions = getAffineResultPositions(maps);
+  if (failed(positions))
+    return false;
+  return (*positions)[0] == SmallVector<int64_t>{0, 1, 3} &&
+         (*positions)[1] == SmallVector<int64_t>{0, 3, 2} &&
+         (*positions)[2] == SmallVector<int64_t>{0, 1, 2};
+}
+
 SmallVector<utils::IteratorType> BatchMatmulOp::getIteratorTypesArray() {
   return SmallVector<utils::IteratorType>{
       utils::IteratorType::parallel, utils::IteratorType::parallel,
@@ -5345,11 +5777,18 @@ ArrayRef<int64_t> UnPackOp::getAllOuterDims() {
 
 SmallVector<int64_t> UnPackOp::getTiledOuterDims() {
   auto innerDimsPos = getInnerDimsPos();
-  auto packedShape = getSourceType().getShape();
+  SmallVector<int64_t> outerDims(getAllOuterDims());
   SmallVector<int64_t> res;
 
+  // Recover the original order of the outer dims.
+  SmallVector<int64_t> outerDimPermInv(getOuterDimsPerm());
+  invertPermutationVector(outerDimPermInv);
+  if (!outerDimPermInv.empty())
+    applyPermutationToVector(outerDims, outerDimPermInv);
+
+  // Collect the outer dims corresponding to the tilled inner dims.
   for (auto index : innerDimsPos)
-    res.push_back(packedShape[index]);
+    res.push_back(outerDims[index]);
 
   return res;
 }
@@ -5646,6 +6085,19 @@ BatchReduceMatmulOp::getDefaultIndexingMaps(MLIRContext *context) {
   return indexingMaps;
 }
 
+bool BatchReduceMatmulOp::isDefaultIndexingMaps(Attribute attr) {
+  ArrayAttr maps = dyn_cast<ArrayAttr>(attr);
+  if (!maps)
+    return false;
+  if (maps.size() != 3)
+    return false;
+  auto positions = getAffineResultPositions(maps);
+  if (failed(positions))
+    return false;
+  return (*positions)[0] == SmallVector<int64_t>{0, 1, 3} &&
+         (*positions)[1] == SmallVector<int64_t>{0, 3, 2} &&
+         (*positions)[2] == SmallVector<int64_t>{1, 2};
+}
 unsigned BatchReduceMatmulOp::getNumRegionArgs() { return 3; }
 
 std::string BatchReduceMatmulOp::getLibraryCallName() {
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index bdfc8d020e58f..639e0feabc9bd 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -15,6 +15,7 @@
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Bufferization/Transforms/OneShotAnalysis.h"
+#include "mlir/Dialect/CommonFolders.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Linalg/TransformOps/GPUHeuristics.h"
@@ -27,6 +28,7 @@
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
 #include "mlir/Dialect/Transform/Utils/Utils.h"
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
@@ -1985,14 +1987,19 @@ transform::PadOp::apply(transform::TransformRewriter &rewriter,
 
     // Convert the padding values to attributes.
     SmallVector<Attribute> paddingValues;
-    for (auto const &it :
+    for (auto const &[untypedAttr, elementOrTensorType] :
          llvm::zip(getPaddingValues(), linalgTarget->getOperandTypes())) {
-      auto attr = dyn_cast<TypedAttr>(std::get<0>(it));
+
+      if (isa<ub::PoisonAttr>(untypedAttr)) {
+        paddingValues.push_back(untypedAttr);
+        continue;
+      }
+      auto attr = dyn_cast<TypedAttr>(untypedAttr);
       if (!attr) {
-        emitOpError("expects padding values to be typed attributes");
+        emitOpError("expects padding values to be typed attributes or poison");
         return DiagnosedSilenceableFailure::definiteFailure();
       }
-      Type elementType = getElementTypeOrSelf(std::get<1>(it));
+      Type elementType = getElementTypeOrSelf(elementOrTensorType);
       // Try to parse string attributes to obtain an attribute of element type.
       if (auto stringAttr = dyn_cast<StringAttr>(attr)) {
         auto parsedAttr = dyn_cast_if_present<TypedAttr>(parseAttribute(
@@ -2000,7 +2007,7 @@ transform::PadOp::apply(transform::TransformRewriter &rewriter,
             /*numRead=*/nullptr, /*isKnownNullTerminated=*/true));
         if (!parsedAttr || parsedAttr.getType() != elementType) {
           auto diag = this->emitOpError("expects a padding that parses to ")
-                      << elementType << ", got " << std::get<0>(it);
+                      << elementType << ", got " << untypedAttr;
           diag.attachNote(linalgTarget.getLoc()) << "when applied to this op";
           return DiagnosedSilenceableFailure::definiteFailure();
         }
@@ -2235,8 +2242,13 @@ transform::PadTilingInterfaceOp::apply(transform::TransformRewriter &rewriter,
          llvm::zip(getPaddingValues(), targetOp->getOperandTypes())) {
       auto attr = dyn_cast<TypedAttr>(untypedAttr);
       Type elementType = getElementTypeOrSelf(elementOrTensorType);
+
+      if (isa<ub::PoisonAttr>(untypedAttr)) {
+        paddingValues.push_back(untypedAttr);
+        continue;
+      }
       if (!attr) {
-        emitOpError("expects padding values to be typed attributes");
+        emitOpError("expects padding values to be typed attributes or poison");
         return DiagnosedSilenceableFailure::definiteFailure();
       }
       // Try to parse string attributes to obtain an attribute of element type.
@@ -3783,8 +3795,15 @@ LogicalResult TileUsingForallOp::verify() {
 
 void transform::VectorizeChildrenAndApplyPatternsOp::build(
     OpBuilder &builder, OperationState &result, Value target,
-    bool vectorizePadding, bool vectorizeExtract, bool flatten1DDepthwiseConv) {
+    bool foldTypeExtensionsIntoContract, bool vectorizePadding,
+    bool vectorizeExtract, bool flatten1DDepthwiseConv) {
   result.addOperands(target);
+  if (foldTypeExtensionsIntoContract) {
+    result.addAttribute(
+        VectorizeChildrenAndApplyPatternsOp::
+            getFoldTypeExtensionsIntoContractAttrName(result.name),
+        builder.getUnitAttr());
+  }
   if (vectorizePadding) {
     result.addAttribute(
         VectorizeChildrenAndApplyPatternsOp::getVectorizePaddingAttrName(
@@ -3875,6 +3894,9 @@ transform::VectorizeChildrenAndApplyPatternsOp::applyToOne(
 
   patterns.add<CopyVectorizationPattern>(ctx);
 
+  if (getFoldTypeExtensionsIntoContract())
+    vector::populateFoldArithExtensionPatterns(patterns);
+
   if (getVectorizePadding()) {
     linalg::populatePadOpVectorizationPatterns(patterns);
     // This creates an alternative path for lowering tensor.pad - by
diff --git a/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
index 3908d73f5e0e1..b4507a93e0098 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/BlockPackMatmul.cpp
@@ -320,10 +320,6 @@ void linalg::populateBlockPackMatmulPatterns(
     RewritePatternSet &patterns, const ControlBlockPackMatmulFn &controlFn) {
   patterns.add<BlockPackMatmul<linalg::GenericOp>,
                BlockPackMatmul<linalg::MatmulOp>,
-               BlockPackMatmul<linalg::BatchMatmulOp>,
-               BlockPackMatmul<linalg::MatmulTransposeAOp>,
-               BlockPackMatmul<linalg::BatchMatmulTransposeAOp>,
-               BlockPackMatmul<linalg::MatmulTransposeBOp>,
-               BlockPackMatmul<linalg::BatchMatmulTransposeBOp>>(
-      patterns.getContext(), controlFn);
+               BlockPackMatmul<linalg::BatchMatmulOp>>(patterns.getContext(),
+                                                       controlFn);
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
index 70f846e5bbd20..fb39e18691e03 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
@@ -23,9 +23,11 @@ add_mlir_dialect_library(MLIRLinalgTransforms
   InlineScalarOperands.cpp
   Interchange.cpp
   Loops.cpp
+  MorphOps.cpp
   TransposeMatmul.cpp
   ShardingInterfaceImpl.cpp
-  NamedOpConversions.cpp
+  SimplifyDepthwiseConv.cpp
+  NamedToElementwise.cpp
   BlockPackMatmul.cpp
   PackAndUnpackPatterns.cpp
   Padding.cpp
diff --git a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
index bf66ed01ef158..22690daa4f9e1 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp
@@ -691,9 +691,9 @@ struct DropPadUnitDims : public OpRewritePattern<tensor::PadOp> {
 
     auto newResultType = RankedTensorType::get(
         newResultShape, padOp.getResultType().getElementType());
-    auto newPadOp = rewriter.create<tensor::PadOp>(
-        padOp.getLoc(), /*result=*/newResultType, collapsedSource, newLowPad,
-        newHighPad, paddingVal, padOp.getNofold());
+    auto newPadOp = tensor::PadOp::create(
+        rewriter, padOp.getLoc(), /*result=*/newResultType, collapsedSource,
+        newLowPad, newHighPad, paddingVal, padOp.getNofold());
 
     Value dest = padOp.getResult();
     if (options.rankReductionStrategy ==
@@ -1052,12 +1052,8 @@ struct RankReduceMatmul : RankReduceContractionOps<FromOpTy, ToOpTy> {
   static bool constexpr reduceLeft =
       (std::is_same_v<FromOpTy, BatchMatmulOp> &&
        std::is_same_v<ToOpTy, BatchVecmatOp>) ||
-      (std::is_same_v<FromOpTy, BatchMatmulTransposeAOp> &&
-       std::is_same_v<ToOpTy, BatchVecmatOp>) ||
       (std::is_same_v<FromOpTy, MatmulOp> &&
        std::is_same_v<ToOpTy, VecmatOp>) ||
-      (std::is_same_v<FromOpTy, MatmulTransposeAOp> &&
-       std::is_same_v<ToOpTy, VecmatOp>) ||
       (std::is_same_v<FromOpTy, MatvecOp> && std::is_same_v<ToOpTy, DotOp>);
 
   /// Look for non-batch spatial dims to collapse.
@@ -1113,27 +1109,15 @@ void mlir::linalg::populateContractionOpRankReducingPatterns(
   MLIRContext *context = patterns.getContext();
   // Unbatching patterns for unit batch size
   patterns.add<RankReduceToUnBatched<BatchMatmulOp, MatmulOp>>(context);
-  patterns
-      .add<RankReduceToUnBatched<BatchMatmulTransposeAOp, MatmulTransposeAOp>>(
-          context);
-  patterns
-      .add<RankReduceToUnBatched<BatchMatmulTransposeBOp, MatmulTransposeBOp>>(
-          context);
   patterns.add<RankReduceToUnBatched<BatchMatvecOp, MatvecOp>>(context);
   patterns.add<RankReduceToUnBatched<BatchVecmatOp, VecmatOp>>(context);
 
   // Non-batch rank 1 reducing patterns
   patterns.add<RankReduceMatmul<MatmulOp, VecmatOp>>(context);
   patterns.add<RankReduceMatmul<MatmulOp, MatvecOp>>(context);
-  patterns.add<RankReduceMatmul<MatmulTransposeAOp, VecmatOp>>(context);
-  patterns.add<RankReduceMatmul<MatmulTransposeBOp, MatvecOp>>(context);
   // Batch rank 1 reducing patterns
   patterns.add<RankReduceMatmul<BatchMatmulOp, BatchVecmatOp>>(context);
   patterns.add<RankReduceMatmul<BatchMatmulOp, BatchMatvecOp>>(context);
-  patterns.add<RankReduceMatmul<BatchMatmulTransposeAOp, BatchVecmatOp>>(
-      context);
-  patterns.add<RankReduceMatmul<BatchMatmulTransposeBOp, BatchMatvecOp>>(
-      context);
 
   // Non-batch rank 0 reducing patterns
   patterns.add<RankReduceMatmul<MatvecOp, DotOp>>(context);
diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
index fd530f26db526..9436f1c6cd9b0 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp
@@ -594,7 +594,8 @@ static FailureOr<PackingResult> buildPackingLoopNestImpl(
     auto clonedForOp = scf::ForOp::create(
         rewriter, loc, bvm.lookupOrDefault(forOp.getLowerBound()),
         bvm.lookupOrDefault(forOp.getUpperBound()),
-        bvm.lookupOrDefault(forOp.getStep()), hoistedPackedTensor);
+        bvm.lookupOrDefault(forOp.getStep()), hoistedPackedTensor,
+        /*bodyBuilder=*/nullptr, forOp.getUnsignedCmp());
 
     // Map the induction var, region args and results to the `clonedForOp`.
     bvm.map(forOp.getInductionVar(), clonedForOp.getInductionVar());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
index 58986a6009995..36434cf2d2ae2 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Hoisting.cpp
@@ -55,7 +55,8 @@ static scf::ForOp replaceWithDifferentYield(RewriterBase &rewriter,
 
   scf::ForOp newLoop = scf::ForOp::create(
       rewriter, loop.getLoc(), loop.getLowerBound(), loop.getUpperBound(),
-      loop.getStep(), inits, [](OpBuilder &, Location, Value, ValueRange) {});
+      loop.getStep(), inits, [](OpBuilder &, Location, Value, ValueRange) {},
+      loop.getUnsignedCmp());
 
   // Generate the new yield with the replaced operand.
   auto yieldOp = cast<scf::YieldOp>(loop.getBody()->getTerminator());
@@ -165,8 +166,12 @@ static bool noAliasingUseInLoop(vector::TransferReadOp transferRead,
   Value source = transferRead.getBase();
 
   // Skip view-like Ops and retrive the actual soruce Operation
-  while (auto srcOp = source.getDefiningOp<ViewLikeOpInterface>())
-    source = srcOp.getViewSource();
+  while (auto viewLike = source.getDefiningOp<ViewLikeOpInterface>()) {
+    if (viewLike.getViewDest() != source) {
+      break;
+    }
+    source = viewLike.getViewSource();
+  }
 
   llvm::SmallVector<Operation *, 32> users(source.getUsers().begin(),
                                            source.getUsers().end());
@@ -177,7 +182,8 @@ static bool noAliasingUseInLoop(vector::TransferReadOp transferRead,
     if (!processed.insert(user).second)
       continue;
     if (auto viewLike = dyn_cast<ViewLikeOpInterface>(user)) {
-      users.append(viewLike->getUsers().begin(), viewLike->getUsers().end());
+      Value viewDest = viewLike.getViewDest();
+      users.append(viewDest.getUsers().begin(), viewDest.getUsers().end());
       continue;
     }
     if (isMemoryEffectFree(user) || isa<vector::TransferReadOp>(user))
diff --git a/mlir/lib/Dialect/Linalg/Transforms/MorphOps.cpp b/mlir/lib/Dialect/Linalg/Transforms/MorphOps.cpp
new file mode 100644
index 0000000000000..f261ccb1415fe
--- /dev/null
+++ b/mlir/lib/Dialect/Linalg/Transforms/MorphOps.cpp
@@ -0,0 +1,62 @@
+//===- MorphOps.cpp - conversion between named,category and generic ops ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements conversions between linalg ops:
+//    named <--> category (elementwise, contraction, ..) <--> generic.
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Complex/IR/Complex.h"
+#include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h"
+#include "mlir/Dialect/Linalg/Passes.h"
+#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
+#include "mlir/Dialect/Math/IR/Math.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+namespace mlir {
+#define GEN_PASS_DEF_LINALGMORPHOPSPASS
+#include "mlir/Dialect/Linalg/Passes.h.inc"
+} // namespace mlir
+
+#define DEBUG_TYPE "linalg-morphism"
+
+using namespace mlir;
+using namespace mlir::linalg;
+
+namespace {
+struct LinalgMorphOpsPass
+    : public impl::LinalgMorphOpsPassBase<LinalgMorphOpsPass> {
+
+  using impl::LinalgMorphOpsPassBase<
+      LinalgMorphOpsPass>::LinalgMorphOpsPassBase;
+
+  void runOnOperation() override;
+};
+
+void LinalgMorphOpsPass::runOnOperation() {
+
+  RewritePatternSet patterns(&getContext());
+
+  // Lowering paths (named -> category -> generic)
+  if (namedToCategory) {
+    populateLinalgNamedToElementwisePatterns(patterns);
+  }
+  if (namedToGeneric || categoryToGeneric) {
+    populateLinalgNamedOpsGeneralizationPatterns(patterns);
+  }
+
+  // Lifting paths (named <- category <- generic)
+  if (genericToNamed) {
+    populateLinalgGenericOpsSpecializationPatterns(patterns);
+  }
+
+  if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))))
+    signalPassFailure();
+}
+} // namespace
diff --git a/mlir/lib/Dialect/Linalg/Transforms/NamedToElementwise.cpp b/mlir/lib/Dialect/Linalg/Transforms/NamedToElementwise.cpp
new file mode 100644
index 0000000000000..00a076b6e9746
--- /dev/null
+++ b/mlir/lib/Dialect/Linalg/Transforms/NamedToElementwise.cpp
@@ -0,0 +1,98 @@
+//===- NamedToElementwise.cpp - convert linalg named op into elementwise --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements rewriting those linalg named ops that are essentially
+// elementwise e.g. `linalg.exp`, to `linalg.elementwise`. This allows further
+// optimization on `linalg.elementwise` such as folding transpose, broadcast.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Linalg/IR/Linalg.h"
+#include "mlir/Dialect/Linalg/Passes.h"
+#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/TypeSwitch.h"
+
+using namespace mlir;
+using namespace mlir::linalg;
+
+#define DEBUG_TYPE "linalg-named-to-elementwise"
+
+namespace {
+ElementwiseKind getKind(Operation *op) {
+  return llvm::TypeSwitch<Operation *, ElementwiseKind>(op)
+      .Case([](SelectOp) { return ElementwiseKind::select; })
+      .Case([](AddOp) { return ElementwiseKind::add; })
+      .Case([](SubOp) { return ElementwiseKind::sub; })
+      .Case([](MulOp) { return ElementwiseKind::mul; })
+      .Case([](DivOp) { return ElementwiseKind::div; })
+      .Case([](DivUnsignedOp) { return ElementwiseKind::div_unsigned; })
+      .Case([](PowFOp) { return ElementwiseKind::powf; })
+      .Case([](ExpOp) { return ElementwiseKind::exp; })
+      .Case([](LogOp) { return ElementwiseKind::log; })
+      .Case([](AbsOp) { return ElementwiseKind::abs; })
+      .Case([](CeilOp) { return ElementwiseKind::ceil; })
+      .Case([](FloorOp) { return ElementwiseKind::floor; })
+      .Case([](NegFOp) { return ElementwiseKind::negf; })
+      .Case([](ReciprocalOp) { return ElementwiseKind::reciprocal; })
+      .Case([](RoundOp) { return ElementwiseKind::round; })
+      .Case([](SqrtOp) { return ElementwiseKind::sqrt; })
+      .Case([](RsqrtOp) { return ElementwiseKind::rsqrt; })
+      .Case([](SquareOp) { return ElementwiseKind::square; })
+      .Case([](TanhOp) { return ElementwiseKind::tanh; })
+      .Case([](ErfOp) { return ElementwiseKind::erf; })
+      .Default([&](Operation *op) {
+        llvm_unreachable("unhandled case in named to elementwise");
+        return ElementwiseKind::sub;
+      });
+}
+
+template <typename NamedOpTy>
+struct NamedToElementwisePattern : public OpRewritePattern<NamedOpTy> {
+  using OpRewritePattern<NamedOpTy>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(NamedOpTy op,
+                                PatternRewriter &rewriter) const override {
+    SmallVector<NamedAttribute> attrs;
+    auto kindAttr = ElementwiseKindAttr::get(op.getContext(), getKind(op));
+    attrs.push_back(rewriter.getNamedAttr("kind", kindAttr));
+    attrs.push_back(
+        rewriter.getNamedAttr("indexing_maps", op.getIndexingMaps()));
+
+    rewriter.replaceOpWithNewOp<ElementwiseOp>(op, op.getDpsInputs(),
+                                               op.getDpsInits(), attrs);
+    return success();
+  }
+};
+} // namespace
+
+void mlir::linalg::populateLinalgNamedToElementwisePatterns(
+    RewritePatternSet &patterns) {
+  patterns.add<NamedToElementwisePattern<SelectOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<AddOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<SubOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<MulOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<DivOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<DivUnsignedOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<PowFOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<ExpOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<LogOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<AbsOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<CeilOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<FloorOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<NegFOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<ReciprocalOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<RoundOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<SqrtOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<RsqrtOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<SquareOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<TanhOp>>(patterns.getContext());
+  patterns.add<NamedToElementwisePattern<ErfOp>>(patterns.getContext());
+}
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index 2e6252336dfeb..3d12bc397813b 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -11,6 +11,7 @@
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/UB/IR/UBOps.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/IR/AffineExpr.h"
 #include "mlir/IR/BuiltinAttributes.h"
@@ -230,13 +231,18 @@ static Value padOperand(RewriterBase &rewriter, TilingInterface opToPad,
   Value paddingValue;
   if (auto complexTy =
           dyn_cast<ComplexType>(getElementTypeOrSelf(v.getType()))) {
-    auto complexAttr = cast<ArrayAttr>(paddingValueAttr);
-    paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
-                                               complexTy, complexAttr);
-  } else {
-    paddingValue = arith::ConstantOp::create(rewriter, opToPad.getLoc(),
-                                             cast<TypedAttr>(paddingValueAttr));
+    if (auto complexAttr = dyn_cast<ArrayAttr>(paddingValueAttr)) {
+      paddingValue = complex::ConstantOp::create(rewriter, opToPad.getLoc(),
+                                                 complexTy, complexAttr);
+    }
+  } else if (isa<ub::PoisonAttr>(paddingValueAttr)) {
+    paddingValue = ub::PoisonOp::create(rewriter, opToPad.getLoc(),
+                                        getElementTypeOrSelf(v.getType()));
+  } else if (auto typedAttr = dyn_cast<TypedAttr>(paddingValueAttr)) {
+    paddingValue =
+        arith::ConstantOp::create(rewriter, opToPad.getLoc(), typedAttr);
   }
+  assert(paddingValue && "failed to create value from padding attribute");
 
   // Pad the operand to the bounding box defined by `paddedShape`.
   SmallVector<int64_t> tensorShape;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp b/mlir/lib/Dialect/Linalg/Transforms/SimplifyDepthwiseConv.cpp
similarity index 93%
rename from mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
rename to mlir/lib/Dialect/Linalg/Transforms/SimplifyDepthwiseConv.cpp
index a2bd9d92815a0..27ccf3c2ba148 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/NamedOpConversions.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/SimplifyDepthwiseConv.cpp
@@ -21,7 +21,7 @@
 #include "llvm/ADT/TypeSwitch.h"
 
 namespace mlir {
-#define GEN_PASS_DEF_LINALGNAMEDOPCONVERSIONPASS
+#define GEN_PASS_DEF_SIMPLIFYDEPTHWISECONVPASS
 #include "mlir/Dialect/Linalg/Passes.h.inc"
 } // namespace mlir
 
@@ -143,23 +143,22 @@ struct SimplifyDepthwiseConvQOp
   }
 };
 
-struct LinalgNamedOpConversionPass
-    : public impl::LinalgNamedOpConversionPassBase<
-          LinalgNamedOpConversionPass> {
-  using impl::LinalgNamedOpConversionPassBase<
-      LinalgNamedOpConversionPass>::LinalgNamedOpConversionPassBase;
+struct SimplifyDepthwiseConvPass
+    : public impl::SimplifyDepthwiseConvPassBase<SimplifyDepthwiseConvPass> {
+  using impl::SimplifyDepthwiseConvPassBase<
+      SimplifyDepthwiseConvPass>::SimplifyDepthwiseConvPassBase;
 
   void runOnOperation() override {
     Operation *op = getOperation();
     RewritePatternSet patterns(op->getContext());
-    populateLinalgNamedOpConversionPatterns(patterns);
+    populateSimplifyDepthwiseConvPatterns(patterns);
     if (failed(applyPatternsGreedily(op, std::move(patterns))))
       return signalPassFailure();
   }
 };
 } // namespace
 
-void mlir::linalg::populateLinalgNamedOpConversionPatterns(
+void mlir::linalg::populateSimplifyDepthwiseConvPatterns(
     RewritePatternSet &patterns) {
   patterns.add<SimplifyDepthwiseConvOp, SimplifyDepthwiseConvQOp>(
       patterns.getContext());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp b/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
index 455e1a6d146d1..35ba4f159113f 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Specialize.cpp
@@ -234,19 +234,8 @@ static FailureOr<LinalgOp> specializeLinalgContractions(RewriterBase &rewriter,
 
   /// Codegen the different matmul variants.
   if (numOfBatchDims) {
-    if (a == IndexMatchResult::Transposed)
-      return replaceWithMatmulVariant<BatchMatmulTransposeAOp>(rewriter,
-                                                               genericOp);
-    if (b == IndexMatchResult::Transposed)
-      return replaceWithMatmulVariant<BatchMatmulTransposeBOp>(rewriter,
-                                                               genericOp);
     return replaceWithMatmulVariant<BatchMatmulOp>(rewriter, genericOp);
   }
-
-  if (a == IndexMatchResult::Transposed)
-    return replaceWithMatmulVariant<MatmulTransposeAOp>(rewriter, genericOp);
-  if (b == IndexMatchResult::Transposed)
-    return replaceWithMatmulVariant<MatmulTransposeBOp>(rewriter, genericOp);
   return replaceWithMatmulVariant<MatmulOp>(rewriter, genericOp);
 }
 
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
index bb725f2ad6a68..e9a8b253eea35 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp
@@ -29,6 +29,7 @@
 #include "mlir/Support/LLVM.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/InterleavedRange.h"
 #include "llvm/Support/raw_ostream.h"
 #include <utility>
@@ -38,9 +39,6 @@
 using namespace mlir;
 using namespace mlir::linalg;
 
-#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ")
-#define DBGSNL() (llvm::dbgs() << "\n")
-
 //===----------------------------------------------------------------------===//
 // Transformations exposed as functional-style API calls.
 //===----------------------------------------------------------------------===//
@@ -91,11 +89,11 @@ static bool hasAtMostOneResultFunctionOfDim(AffineMap map, int64_t dim) {
   }
   return true;
 }
+#endif // NDEBUG
 
 static std::string stringifyReassocIndices(ReassociationIndicesRef ri) {
   return llvm::interleaved(ri, ", ", /*Prefix=*/"|", /*Suffix=*/"");
 }
-#endif // NDEBUG
 
 /// Return the index of the first result of `map` that is a function of
 /// AffineDimExpr(dim), std::nullopt otherwise.
@@ -276,23 +274,18 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
       tensor::PadOp::create(rewriter, loc, collapsed, packOp.getSource(), lows,
                             highs, paddingValue, /*nofold=*/false);
 
-  LLVM_DEBUG(
-      DBGSNL(); DBGSNL();
-      DBGS() << "insertPositions: "
-             << llvm::interleaved(packingMetadata.insertPositions);
-      DBGSNL(); DBGS() << "outerPositions: "
-                       << llvm::interleaved(packingMetadata.outerPositions);
-      DBGSNL(); DBGS() << "packedShape: "
-                       << llvm::interleaved(packedTensorType.getShape());
-      DBGSNL(); DBGS() << "packedToStripMinedShapePerm: "
-                       << llvm::interleaved(packedToStripMinedShapePerm);
-      DBGSNL();
-      DBGS() << "reassociations: "
-             << llvm::interleaved(llvm::map_range(
-                    packingMetadata.reassociations, stringifyReassocIndices));
-      DBGSNL();
-      DBGS() << "stripMinedShape: " << llvm::interleaved(stripMinedShape);
-      DBGSNL(); DBGS() << "collapsed type: " << collapsed; DBGSNL(););
+  LDBG() << "insertPositions: "
+         << llvm::interleaved(packingMetadata.insertPositions);
+  LDBG() << "outerPositions: "
+         << llvm::interleaved(packingMetadata.outerPositions);
+  LDBG() << "packedShape: " << llvm::interleaved(packedTensorType.getShape());
+  LDBG() << "packedToStripMinedShapePerm: "
+         << llvm::interleaved(packedToStripMinedShapePerm);
+  LDBG() << "reassociations: "
+         << llvm::interleaved(llvm::map_range(packingMetadata.reassociations,
+                                              stringifyReassocIndices));
+  LDBG() << "stripMinedShape: " << llvm::interleaved(stripMinedShape);
+  LDBG() << "collapsed type: " << collapsed;
 
   if (lowerPadLikeWithInsertSlice && packOp.isLikePad()) {
     // Pack ops which operate as simple pads may not produce legal
@@ -317,7 +310,7 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
           rewriter, loc, /*source=*/padOp, /*dest=*/packOp.getDest(),
           /*offsets=*/zeros, sizes, /*strides=*/ones);
 
-      LLVM_DEBUG(DBGS() << "insert_slice op: " << insertSliceOp; DBGSNL(););
+      LDBG() << "insert_slice op: " << insertSliceOp;
 
       rewriter.replaceOp(packOp, insertSliceOp->getResults());
 
@@ -339,10 +332,9 @@ FailureOr<LowerPackResult> linalg::lowerPack(RewriterBase &rewriter,
   auto transposeOp = linalg::TransposeOp::create(
       rewriter, loc, reshapeOp.getResult(), packOp.getDest(), transpPerm);
 
-  LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
-             DBGS() << "reshape op: " << reshapeOp; DBGSNL();
-             DBGS() << "transpPerm: " << llvm::interleaved(transpPerm);
-             DBGSNL(); DBGS() << "transpose op: " << transposeOp; DBGSNL(););
+  LDBG() << "reshape op: " << reshapeOp;
+  LDBG() << "transpPerm: " << llvm::interleaved(transpPerm);
+  LDBG() << "transpose op: " << transposeOp;
 
   // 7. Replace packOp by transposeOp.
   rewriter.replaceOp(packOp, transposeOp->getResults());
@@ -410,21 +402,16 @@ linalg::lowerUnPack(RewriterBase &rewriter, linalg::UnPackOp unPackOp,
       linalg::TransposeOp::create(rewriter, loc, unPackOp.getSource(), emptyOp,
                                   packedToStripMinedShapePerm);
 
-  LLVM_DEBUG(
-      DBGSNL(); DBGSNL();
-      DBGS() << "insertPositions: "
-             << llvm::interleaved(packingMetadata.insertPositions);
-      DBGSNL(); DBGS() << "packedShape: "
-                       << llvm::interleaved(packedTensorType.getShape());
-      DBGSNL(); DBGS() << "packedToStripMinedShapePerm: "
-                       << llvm::interleaved(packedToStripMinedShapePerm);
-      DBGSNL();
-      DBGS() << "reassociations: "
-             << llvm::interleaved(llvm::map_range(
-                    packingMetadata.reassociations, stringifyReassocIndices));
-      DBGSNL();
-      DBGS() << "stripMinedShape: " << llvm::interleaved(stripMinedShape);
-      DBGSNL(); DBGS() << "collapsed type: " << collapsedType; DBGSNL(););
+  LDBG() << "insertPositions: "
+         << llvm::interleaved(packingMetadata.insertPositions);
+  LDBG() << "packedShape: " << llvm::interleaved(packedTensorType.getShape());
+  LDBG() << "packedToStripMinedShapePerm: "
+         << llvm::interleaved(packedToStripMinedShapePerm);
+  LDBG() << "reassociations: "
+         << llvm::interleaved(llvm::map_range(packingMetadata.reassociations,
+                                              stringifyReassocIndices));
+  LDBG() << "stripMinedShape: " << llvm::interleaved(stripMinedShape);
+  LDBG() << "collapsed type: " << collapsedType;
 
   // 4. Collapse from the stripMinedShape to the padded result.
   auto reshapeOp = tensor::CollapseShapeOp::create(
@@ -486,10 +473,9 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
   SmallVector<AffineMap> indexingMaps = linalgOp.getIndexingMapsArray();
   SmallVector<utils::IteratorType> iteratorTypes =
       linalgOp.getIteratorTypesArray();
-  LLVM_DEBUG(DBGS() << "Start packing: " << linalgOp << "\n"
-                    << "maps: " << llvm::interleaved(indexingMaps) << "\n"
-                    << "iterators: " << llvm::interleaved(iteratorTypes)
-                    << "\n");
+  LDBG() << "Start packing: " << linalgOp;
+  LDBG() << "maps: " << llvm::interleaved(indexingMaps);
+  LDBG() << "iterators: " << llvm::interleaved(iteratorTypes);
 
   SmallVector<linalg::PackOp> packOps;
   SmallVector<linalg::UnPackOp> unPackOps;
@@ -511,14 +497,11 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
     packedOperandsDims.packedDimForEachOperand = *maybePackedDimForEachOperand;
     listOfPackedOperandsDim.pushBack(std::move(packedOperandsDims));
 
-    LLVM_DEBUG(
-        DBGS() << "++++ After pack size #" << i << ": " << packedSizes[i]
-               << "\n"
-               << "maps: " << llvm::interleaved(indexingMaps) << "\n"
-               << "iterators: " << llvm::interleaved(iteratorTypes) << "\n"
-               << "packedDimForEachOperand: "
-               << llvm::interleaved(packedOperandsDims.packedDimForEachOperand)
-               << "\n");
+    LDBG() << "++++ After pack size #" << i << ": " << packedSizes[i];
+    LDBG() << "maps: " << llvm::interleaved(indexingMaps);
+    LDBG() << "iterators: " << llvm::interleaved(iteratorTypes);
+    LDBG() << "packedDimForEachOperand: "
+           << llvm::interleaved(packedOperandsDims.packedDimForEachOperand);
   }
 
   // Step 2. Propagate packing to all LinalgOp operands.
@@ -534,10 +517,9 @@ FailureOr<PackResult> linalg::pack(RewriterBase &rewriter,
           listOfPackedOperandsDim.extractPackedDimsForOperand(pos);
       SmallVector<OpFoldResult> innerPackSizes =
           listOfPackedOperandsDim.extractPackSizesForOperand(pos);
-      LLVM_DEBUG(DBGS() << "operand: " << operand << "\n"
-                        << "innerPos: " << llvm::interleaved(innerPos) << "\n"
-                        << "innerPackSizes: "
-                        << llvm::interleaved(innerPackSizes) << "\n");
+      LDBG() << "operand: " << operand;
+      LDBG() << "innerPos: " << llvm::interleaved(innerPos);
+      LDBG() << "innerPackSizes: " << llvm::interleaved(innerPackSizes);
       if (innerPackSizes.empty()) {
         inputsAndInits.push_back(operand);
         continue;
@@ -776,8 +758,8 @@ linalg::packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
 
   int64_t numLoops = linalgOp.getNumLoops();
   if (numLoops <= 2) {
-    LLVM_DEBUG(DBGS() << "need 3+ loops to find a matmul to pack, got "
-                      << numLoops << "\nin: " << linalgOp << "\n");
+    LDBG() << "need 3+ loops to find a matmul to pack, got " << numLoops
+           << " in: " << linalgOp;
     return rewriter.notifyMatchFailure(
         linalgOp, "need 3+ loops to find a matmul to pack");
   }
@@ -801,8 +783,7 @@ linalg::packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
   FailureOr<ContractionDimensions> maybeDimensions =
       inferContractionDims(linalgOp);
   if (failed(maybeDimensions)) {
-    LLVM_DEBUG(DBGS() << "couldn't infer matmul iterators in: " << linalgOp
-                      << "\n");
+    LDBG() << "couldn't infer matmul iterators in: " << linalgOp;
     return rewriter.notifyMatchFailure(linalgOp,
                                        "couldn't infer matmul iterators");
   }
@@ -814,10 +795,8 @@ linalg::packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
   // to plug a heuristic.
   int64_t mPos = maybeDimensions->m.back(), nPos = maybeDimensions->n.back(),
           kPos = maybeDimensions->k.back();
-  LLVM_DEBUG(DBGSNL(); DBGSNL(); DBGSNL();
-             DBGS() << "Start packing generic op greedily with (m@" << mPos
-                    << ", n@" << nPos << ", k@" << kPos << "): " << linalgOp
-                    << "\n";);
+  LDBG() << "Start packing generic op greedily with (m@" << mPos << ", n@"
+         << nPos << ", k@" << kPos << "): " << linalgOp;
 
   // 2.a. Rewrite as a generic.
   auto genericOp = dyn_cast<GenericOp>(linalgOp.getOperation());
@@ -833,14 +812,14 @@ linalg::packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
   // not change the indexings of any operand.
   SmallVector<int64_t> permutation =
       computePermutationVector(numLoops, {mPos, nPos, kPos}, mmnnkkPos);
-  LLVM_DEBUG(DBGS() << "perm: " << llvm::interleaved(permutation) << "\n");
+  LDBG() << "perm: " << llvm::interleaved(permutation);
   // Sign .. unsigned pollution.
   SmallVector<unsigned> unsignedPerm(permutation.begin(), permutation.end());
   FailureOr<GenericOp> interchangeResult =
       interchangeGenericOp(rewriter, genericOp, unsignedPerm);
   assert(succeeded(interchangeResult) && "unexpected failure interchanging op");
   genericOp = *interchangeResult;
-  LLVM_DEBUG(DBGS() << "Generalized Op to pack: " << genericOp << "\n";);
+  LDBG() << "Generalized Op to pack: " << genericOp;
 
   // At this point, the op iterators are normalized to {leading, k, m, n}.
   // The layouts induced by packing will always be:
@@ -862,12 +841,11 @@ linalg::packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
 
   // Add leading zeros to match numLoops, we only pack the last 3 dimensions
   // post interchange.
-  LLVM_DEBUG(DBGS() << "paddedSizesNextMultipleOf: "
-                    << llvm::interleaved(paddedSizesNextMultipleOf) << "\n"
-                    << "loopRanges: "
-                    << llvm::interleaved(llvm::map_range(
-                           loopRanges, [](Range r) { return r.size; }))
-                    << "\n");
+  LDBG() << "paddedSizesNextMultipleOf: "
+         << llvm::interleaved(paddedSizesNextMultipleOf);
+  LDBG() << "loopRanges: "
+         << llvm::interleaved(
+                llvm::map_range(loopRanges, [](Range r) { return r.size; }));
   SmallVector<OpFoldResult> adjustedPackedSizes(numLoops - packedSizes.size(),
                                                 rewriter.getIndexAttr(0));
   for (int64_t i = 0, e = numPackedDims; i < e; ++i) {
@@ -883,8 +861,7 @@ linalg::packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
         {loopRanges[adjustedPackedSizes.size()].size,
          rewriter.getIndexAttr(paddedSizesNextMultipleOf[i])}));
   }
-  LLVM_DEBUG(DBGS() << "adjustedPackedSizes: "
-                    << llvm::interleaved(adjustedPackedSizes) << "\n");
+  LDBG() << "adjustedPackedSizes: " << llvm::interleaved(adjustedPackedSizes);
 
   // TODO: If we wanted to give the genericOp a name after packing, after
   // calling `pack` would be a good time. One would still need to check that
@@ -1214,9 +1191,8 @@ LogicalResult DecomposeOuterUnitDimsPackOpPattern::matchAndRewrite(
   }
   srcPermForTranspose.append(innerDimPos.begin(), innerDimPos.end());
 
-  LLVM_DEBUG(DBGS() << "Pack permutation: " << packOp << "\n"
-                    << "perm: " << llvm::interleaved(srcPermForTranspose)
-                    << "\n");
+  LDBG() << "Pack permutation: " << packOp;
+  LDBG() << "perm: " << llvm::interleaved(srcPermForTranspose);
 
   // 2.1 Create tensor.empty (init value for TransposeOp)
   SmallVector<OpFoldResult> transShapeForEmptyOp(srcRank - numTiles,
diff --git a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
index a2a4335ef8e22..2650488c17993 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/TransposeMatmul.cpp
@@ -59,12 +59,12 @@ FailureOr<Operation *> mlir::linalg::transposeMatmul(RewriterBase &rewriter,
                                                  ArrayRef<int64_t>{1, 0});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = linalg::MatmulTransposeAOp::create(
+    newMatmulOp = MatmulTransposeAOp::create(
         rewriter, loc, matmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), matmulOp.getInputs()[1]},
         matmulOp.getOutputs());
   } else {
-    newMatmulOp = linalg::MatmulTransposeBOp::create(
+    newMatmulOp = MatmulTransposeBOp::create(
         rewriter, loc, matmulOp.getResultTypes(),
         ValueRange{matmulOp.getInputs()[0], transposeOp->getResult(0)},
         matmulOp.getOutputs());
@@ -116,12 +116,12 @@ mlir::linalg::transposeBatchMatmul(RewriterBase &rewriter,
                                                  ArrayRef<int64_t>{0, 2, 1});
   Operation *newMatmulOp;
   if (transposeLHS) {
-    newMatmulOp = linalg::BatchMatmulTransposeAOp::create(
+    newMatmulOp = BatchMatmulTransposeAOp::create(
         rewriter, loc, batchMatmulOp.getResultTypes(),
         ValueRange{transposeOp->getResult(0), batchMatmulOp.getInputs()[1]},
         batchMatmulOp.getOutputs());
   } else {
-    newMatmulOp = linalg::BatchMatmulTransposeBOp::create(
+    newMatmulOp = BatchMatmulTransposeBOp::create(
         rewriter, loc, batchMatmulOp.getResultTypes(),
         ValueRange{batchMatmulOp.getInputs()[0], transposeOp->getResult(0)},
         batchMatmulOp.getOutputs());
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
index 0860ceafa0270..406f05c1b08ef 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp
@@ -1805,7 +1805,8 @@ vectorizeAsTensorPackOp(RewriterBase &rewriter, linalg::PackOp packOp,
     inputShape[innerDimsPos[idx]] *= size;
   auto maskedRead = vector::createReadOrMaskedRead(
       rewriter, loc, packOp.getSource(), inputShape, padValue,
-      useInBoundsInsteadOfMasking);
+      useInBoundsInsteadOfMasking,
+      /*inputScalableVecSizes=*/{});
 
   // Create ShapeCastOp.
   SmallVector<int64_t> destShape(inputVectorSizes);
@@ -1878,19 +1879,46 @@ static VectorType getCollapsedVecType(VectorType type,
   return VectorType::get(newShape, type.getElementType(), newScalableFlags);
 }
 
-/// Vectorize a `linalg::UnPackOp` to these 4 Ops:
-///   Vector::TransferReadOp - Reads a vector from the source tensor
-///   vector::TransposeOp - Transpose the Source tensor
-///   ShapeCastOp - Reshape the data based on the target.
-///   vector::TransferWriteOp. - Write the result vector back to the destination
-///   tensor.
-///   If the vector sizes are not provided:
-///   * the vector sizes are determined by the input operand and attributes,
-///   * update the inBounds attribute instead of masking.
+/// Vectorize `linalg.unpack` as:
+///   * xfer_read -> vector.transpose -> vector.shape_cast -> xfer_write
+///
+/// The input-vector-sizes specify the read vector sizes (i.e. the vector sizes
+/// for the xfer_read operation). This is sufficient to infer the other vector
+/// sizes required here.
+///
+/// If the vector sizes are not provided:
+///  * the vector sizes are determined from the input tensor static shape.
+///  * the inBounds attribute is used instead of masking.
+///
+/// EXAMPLE (no vector sizes):
+/// ```
+///   %unpack = linalg.unpack  %src
+///    inner_dims_pos = [0, 1]
+///    inner_tiles = [8, 8]
+///    into %dest : tensor<1x1x8x8xf32> -> tensor<8x8xf32>
+/// ```
+/// is vectorized as:
+/// ```
+///   %read = vector.transfer_read %src
+///     : tensor<1x1x8x8xf32>, vector<1x1x8x8xf32>
+///   %tr = vector.transpose %read, [0, 2, 1, 3]
+///     : vector<1x1x8x8xf32> to vector<1x8x1x8xf32>
+///   %sc = vector.shape_cast %tr
+///     : vector<1x8x1x8xf32> to vector<8x8xf32>
+///   %vector = vector.transfer_write %sc into %dest
+///     : vector<8x8xf32>, tensor<8x8xf32>
+/// ```
 static LogicalResult
 vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
                           ArrayRef<int64_t> inputVectorSizes,
+                          ArrayRef<bool> inputScalableVecDims,
                           SmallVectorImpl<Value> &newResults) {
+  if (!inputVectorSizes.empty()) {
+    assert(inputVectorSizes.size() == unpackOp.getSourceRank() &&
+           "Invalid number of input vector sizes!");
+    assert(inputVectorSizes.size() == inputScalableVecDims.size() &&
+           "Incompatible number of vector sizes and vector scalable flags!");
+  }
 
   // TODO: Introduce a parent class that will handle the insertion point update.
   OpBuilder::InsertionGuard g(rewriter);
@@ -1898,88 +1926,40 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
 
   RankedTensorType unpackTensorType = unpackOp.getSourceType();
 
-  ArrayRef<int64_t> innerDimPos = unpackOp.getInnerDimsPos();
-  ArrayRef<int64_t> innerTiles = unpackOp.getStaticInnerTiles();
   ArrayRef<int64_t> sourceShape = unpackTensorType.getShape();
   bool useInBoundsInsteadOfMasking = false;
-  ArrayRef<int64_t> outerDimsPerm = unpackOp.getOuterDimsPerm();
-
-  auto destSize = unpackOp.getDestRank();
-
-  if (!inputVectorSizes.empty())
-    assert(inputVectorSizes.size() == destSize &&
-           "Incorrect number of input vector sizes");
-
-  // vectorSizes is the shape of the vector that will be used to do final
-  // write on the destination tensor. It is set like this: Let's say the
-  // source tensor is rank 'M' and the dest tensor rank 'N', where N <= M.
-  // Thus:
-  // 1. vectorSizes = sourceShape.take_front(N)
-  // 2. if outer_dims_perms is present: do that permutation on vectorSizes.
-  // 3. multiply all the locations in vectorSize pointed by innerDimPos by the
-  //    innerTiles attribute value.
-  SmallVector<int64_t> vectorSizes(inputVectorSizes);
-  if (vectorSizes.empty()) {
-    llvm::append_range(vectorSizes, sourceShape.take_front(destSize));
-    if (!outerDimsPerm.empty())
-      applyPermutationToVector(vectorSizes, outerDimsPerm);
-    for (auto [i, pos] : llvm::enumerate(innerDimPos))
-      vectorSizes[pos] *= innerTiles[i];
 
-    useInBoundsInsteadOfMasking = true;
-  }
+  Location loc = unpackOp->getLoc();
 
-  // readVectorSizes is the size of tensor used to read and apply mask. It is
-  // set like this: Let's say the vectorSize (VS) array is size 'N' and
-  // the sourceShape(SS) is 'M' where M >= N and InnerTileSizes (IT) of
-  // size M-N
-  // Thus:
-  // - initially: readVectorSizes = vectorInputSizes
-  // - Divide all the readMaskShape locations pointed by innerDimPos
-  //   by the innerTileSize attribute value.
-  // - if outer_dims_perms is present: do that permutation on readVectorSizes.
-  // - Append the remaining shape from SS
-  // E.g. let's say let's say unpackTensorType.getShape() = <8x8x32x16>
-  // inner Dim Pos = [0, 1] and Inner Tiles = [32, 16], vector_sizes are [512,
-  // 128] and outer_dims_perm is [1, 0] then read shape is:
-  //   ReadVectorSizes(initial): [512, 128]
-  //   Final Value(after innerDim Adjustment): [512/32, 128/16]
-  //                                           = [16, 8]
-  //   After applying outer_dims_perm: [8, 16]
-  //   After appending the rest of the sourceShape: [8, 16, 32, 16]
-
-  SmallVector<int64_t> readVectorSizes(vectorSizes.begin(), vectorSizes.end());
-
-  for (auto [index, size] : enumerate(innerTiles)) {
-    readVectorSizes[innerDimPos[index]] =
-        llvm::divideCeil(readVectorSizes[innerDimPos[index]], size);
-  }
-  if (!outerDimsPerm.empty()) {
-    applyPermutationToVector(readVectorSizes, outerDimsPerm);
-  }
-  readVectorSizes.append(sourceShape.begin() + vectorSizes.size(),
-                         sourceShape.end());
+  // Obtain vector sizes for the read operation.
+  SmallVector<int64_t> readVectorSizes(inputVectorSizes);
+  SmallVector<bool> readScalableVectorFlags(inputScalableVecDims);
 
-  Location loc = unpackOp->getLoc();
+  // In the absence of input-vector-sizes, use the _static_ input tensor shape.
+  if (inputVectorSizes.empty()) {
+    if (ShapedType::isDynamicShape(sourceShape))
+      return failure();
 
+    readVectorSizes.assign(sourceShape.begin(), sourceShape.end());
+    useInBoundsInsteadOfMasking = true;
+  }
+
+  // -- Generate the read operation --
   auto padValue = arith::ConstantOp::create(
       rewriter, loc,
       rewriter.getZeroAttr(unpackOp.getSourceType().getElementType()));
-
-  // Read result, mask if necessary. If transferReadOp shape is not equal
-  // to shape of source, then a mask is necessary.
   Value readResult = vector::createReadOrMaskedRead(
       rewriter, loc, unpackOp.getSource(), readVectorSizes, padValue,
-      /*useInBoundsInsteadOfMasking=*/false);
+      useInBoundsInsteadOfMasking, readScalableVectorFlags);
 
+  // -- Generate the transpose operation --
   PackingMetadata packMetadata;
   SmallVector<int64_t> lastDimToInsertPosPerm =
       getUnPackInverseSrcPerm(unpackOp, packMetadata);
-  // Transpose the appropriate rows to match output.
   vector::TransposeOp transposeOp = vector::TransposeOp::create(
       rewriter, loc, readResult, lastDimToInsertPosPerm);
 
-  // Collapse the vector to the size required by result.
+  // -- Generate the shape_cast operation --
   VectorType collapsedVecType = getCollapsedVecType(
       transposeOp.getType(),
       getSymbolLessAffineMaps(convertReassociationIndicesToExprs(
@@ -1987,9 +1967,11 @@ vectorizeAsTensorUnpackOp(RewriterBase &rewriter, linalg::UnPackOp unpackOp,
   vector::ShapeCastOp shapeCastOp = vector::ShapeCastOp::create(
       rewriter, loc, collapsedVecType, transposeOp->getResult(0));
 
+  // -- Generate the write operation --
   Operation *write = createWriteOrMaskedWrite(
       rewriter, loc, shapeCastOp.getResult(), unpackOp.getDest(),
       /*writeIndices=*/{}, useInBoundsInsteadOfMasking);
+
   newResults.push_back(write->getResult(0));
   return success();
 }
@@ -2016,7 +1998,7 @@ vectorizeAsTensorPadOp(RewriterBase &rewriter, tensor::PadOp padOp,
   assert(succeeded(status) && "failed to reify result shapes");
   auto maskedRead = vector::createReadOrMaskedRead(
       rewriter, loc, padOp.getSource(), inputVectorSizes, padValue,
-      /*useInBoundsInsteadOfMasking=*/false);
+      /*useInBoundsInsteadOfMasking=*/false, /*inputScalableVecSizes=*/{});
 
   // Create Xfer write Op
   Value dest = tensor::EmptyOp::create(rewriter, loc, reifiedReturnShapes[0],
@@ -2095,24 +2077,34 @@ vectorizeDynamicLinalgOpPrecondition(linalg::LinalgOp op,
   return success();
 }
 
-/// Need to check if the inner-tiles are static/constant.
+//// This hook considers two cases:
+///   (1) If the input-vector-sizes are empty, then the vector sizes will be
+///       infered. This is only possible when all shapes are static.
+///   (2) If the input-vector-sizes are non-empty (i.e. user provided), then
+///       carry out basic sanity-checking.
 static LogicalResult
 vectorizeUnPackOpPrecondition(linalg::UnPackOp unpackOp,
                               ArrayRef<int64_t> inputVectorSizes) {
+  // If there are no input vector sizes and all shapes are static, there is
+  // nothing left to check.
+  if (inputVectorSizes.empty() && unpackOp.getDestType().hasStaticShape() &&
+      unpackOp.getSourceType().hasStaticShape())
+    return success();
 
-  if (llvm::any_of(unpackOp.getInnerTiles(), [](OpFoldResult res) {
-        return !getConstantIntValue(res).has_value();
-      })) {
-    LDBG() << "Inner-tiles must be constant: " << unpackOp;
+  // The number of input vector sizes must be equal to:
+  //  * read-vector-rank
+  if (!inputVectorSizes.empty() &&
+      (inputVectorSizes.size() != unpackOp.getSourceRank())) {
+    LDBG() << "Incorrect number of input vector sizes";
     return failure();
   }
-  ArrayRef<int64_t> resultShape = unpackOp.getDestType().getShape();
-  bool satisfyEmptyCond = inputVectorSizes.empty() &&
-                          unpackOp.getDestType().hasStaticShape() &&
-                          unpackOp.getSourceType().hasStaticShape();
-  if (!satisfyEmptyCond &&
-      failed(vector::isValidMaskedInputVector(resultShape, inputVectorSizes)))
+
+  // Check the vector sizes for the read operation.
+  if (failed(vector::isValidMaskedInputVector(
+          unpackOp.getSourceType().getShape(), inputVectorSizes))) {
+    LDBG() << "Invalid vector sizes for the read operation";
     return failure();
+  }
 
   return success();
 }
@@ -2436,6 +2428,7 @@ vectorizePackOpPrecondition(linalg::PackOp packOp,
     LDBG() << "pad value is not constant: " << packOp;
     return failure();
   }
+
   ArrayRef<int64_t> resultTensorShape = packOp.getDestType().getShape();
   bool satisfyEmptyCond = true;
   if (inputVectorSizes.empty()) {
@@ -2499,8 +2492,12 @@ vectorizePadOpPrecondition(tensor::PadOp padOp,
   return success();
 }
 
-/// Preconditions for scalable vectors. This is quite restrictive - it models
-/// the fact that in practice we would only make selected dimensions scalable.
+/// Preconditions for scalable vectors.
+///
+/// For Ops implementing the LinalgOp interface, this is quite restrictive - it
+/// models the fact that in practice we would only make selected dimensions
+/// scalable. For other Ops (e.g. `linalg.unpack`), this will succeed
+/// unconditionally - we are yet to identify meaningful conditions.
 static LogicalResult
 vectorizeScalableVectorPrecondition(Operation *op,
                                     ArrayRef<int64_t> inputVectorSizes,
@@ -2516,10 +2513,11 @@ vectorizeScalableVectorPrecondition(Operation *op,
 
   auto linalgOp = dyn_cast<LinalgOp>(op);
 
-  // Cond 1: There's been no need for scalable vectorisation of
-  // non-linalg Ops so far
-  if (!linalgOp)
-    return failure();
+  // Cond 1: Reject Ops that don't implement the LinalgOp interface, with the
+  // exception of UnpackOp for which there is a dedicated hook.
+  if (!linalgOp) {
+    return success(isa<linalg::UnPackOp>(op));
+  }
 
   // Cond 2: There's been no need for more than 2 scalable dims so far
   if (numOfScalableDims > 2)
@@ -2565,7 +2563,7 @@ vectorizeScalableVectorPrecondition(Operation *op,
                 "vectorization";
       return failure();
     }
-    if (isa<linalg::MatmulOp>(op) || isa<linalg::MatmulTransposeAOp>(op)) {
+    if (isa<linalg::MatmulOp>(op)) {
       LDBG()
           << "Scalable vectorization of the reduction dim in Matmul-like ops "
              "is not supported";
@@ -2606,17 +2604,12 @@ vectorizeScalableVectorPrecondition(Operation *op,
       return failure();
   }
 
-  // Check to not let go the matmul with extended semantic, through this
-  // transform.
-  if (linalgOp.hasUserDefinedMaps())
-    return failure();
-
   // Cond 4: Only the following ops are supported in the
   // presence of scalable vectors
   return success(isElementwise(linalgOp) || isa<linalg::MatmulOp>(op) ||
-                 isa<linalg::MatmulTransposeAOp>(op) ||
                  isa<linalg::DepthwiseConv1DNwcWcOp>(op) ||
                  isa<linalg::MatvecOp>(op) || isa<linalg::Mmt4DOp>(op) ||
+                 isa<linalg::BatchMmt4DOp>(op) ||
                  hasReductionIterator(linalgOp));
 }
 
@@ -2750,7 +2743,8 @@ FailureOr<VectorizationResult> mlir::linalg::vectorize(
           })
           .Case<linalg::UnPackOp>([&](auto unpackOp) {
             return vectorizeAsTensorUnpackOp(rewriter, unpackOp,
-                                             inputVectorSizes, results);
+                                             inputVectorSizes,
+                                             inputScalableVecDims, results);
           })
           .Case<tensor::InsertSliceOp>([&](auto sliceOp) {
             return vectorizeAsInsertSliceOp(rewriter, sliceOp, inputVectorSizes,
@@ -3142,7 +3136,8 @@ vectorizeAsInsertSliceOp(RewriterBase &rewriter, tensor::InsertSliceOp sliceOp,
       vecType.getRank(), arith::ConstantIndexOp::create(rewriter, loc, 0));
   Value read = mlir::vector::createReadOrMaskedRead(
       rewriter, loc, source, vecType.getShape(), padValue,
-      /*useInBoundsInsteadOfMasking=*/inputVectorSizes.empty());
+      /*useInBoundsInsteadOfMasking=*/inputVectorSizes.empty(),
+      /*inputScalableVecSizes=*/{});
 
   // Create write
   auto writeIndices =
diff --git a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
index 74b968c27a62a..b59d73d1291c8 100644
--- a/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
+++ b/mlir/lib/Dialect/MemRef/IR/MemRefOps.cpp
@@ -3558,6 +3558,7 @@ LogicalResult AtomicRMWOp::verify() {
   case arith::AtomicRMWKind::minu:
   case arith::AtomicRMWKind::muli:
   case arith::AtomicRMWKind::ori:
+  case arith::AtomicRMWKind::xori:
   case arith::AtomicRMWKind::andi:
     if (!llvm::isa<IntegerType>(getValue().getType()))
       return emitOpError() << "with kind '"
diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
index 9771bd2aaa143..d35566a9c0d29 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandStridedMetadata.cpp
@@ -959,7 +959,7 @@ class RewriteExtractAlignedPointerAsIndexOfViewLikeOp
                   PatternRewriter &rewriter) const override {
     auto viewLikeOp =
         extractOp.getSource().getDefiningOp<ViewLikeOpInterface>();
-    if (!viewLikeOp)
+    if (!viewLikeOp || extractOp.getSource() != viewLikeOp.getViewDest())
       return rewriter.notifyMatchFailure(extractOp, "not a ViewLike source");
     rewriter.modifyOpInPlace(extractOp, [&]() {
       extractOp.getSourceMutable().assign(viewLikeOp.getViewSource());
diff --git a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
index 5af46a48f124f..3de9c3898c713 100644
--- a/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
+++ b/mlir/lib/Dialect/MemRef/Utils/MemRefUtils.cpp
@@ -210,8 +210,10 @@ MemrefValue skipFullyAliasingOperations(MemrefValue source) {
 MemrefValue skipViewLikeOps(MemrefValue source) {
   while (auto op = source.getDefiningOp()) {
     if (auto viewLike = dyn_cast<ViewLikeOpInterface>(op)) {
-      source = cast<MemrefValue>(viewLike.getViewSource());
-      continue;
+      if (source == viewLike.getViewDest()) {
+        source = cast<MemrefValue>(viewLike.getViewSource());
+        continue;
+      }
     }
     return source;
   }
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index 34c95e314f38b..8474244c7d7c0 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -422,6 +422,12 @@ std::optional<InFlightDiagnostic> verifyTmaDescriptorWithMemref(
                            << descMemref << " != " << dstMemref;
   }
 
+  int lastDimBytes =
+      descMemref.getShape().back() * descMemref.getElementTypeBitWidth() / 8;
+  if (lastDimBytes % 16 != 0) {
+    return op->emitError() << "the bytes in the last dimension of the tensor "
+                              "map must be a multiple of 16";
+  }
   return std::nullopt;
 }
 
diff --git a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
index 485bb73672398..5fd426eee1dfd 100644
--- a/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
+++ b/mlir/lib/Dialect/OpenACC/IR/OpenACC.cpp
@@ -1390,6 +1390,36 @@ void acc::ParallelOp::addPrivatization(MLIRContext *context,
   setPrivatizationRecipesAttr(mlir::ArrayAttr::get(context, recipes));
 }
 
+void acc::ParallelOp::addFirstPrivatization(
+    MLIRContext *context, mlir::acc::FirstprivateOp op,
+    mlir::acc::FirstprivateRecipeOp recipe) {
+  getFirstprivateOperandsMutable().append(op.getResult());
+
+  llvm::SmallVector<mlir::Attribute> recipes;
+
+  if (getFirstprivatizationRecipesAttr())
+    llvm::copy(getFirstprivatizationRecipesAttr(), std::back_inserter(recipes));
+
+  recipes.push_back(
+      mlir::SymbolRefAttr::get(context, recipe.getSymName().str()));
+  setFirstprivatizationRecipesAttr(mlir::ArrayAttr::get(context, recipes));
+}
+
+void acc::ParallelOp::addReduction(MLIRContext *context,
+                                   mlir::acc::ReductionOp op,
+                                   mlir::acc::ReductionRecipeOp recipe) {
+  getReductionOperandsMutable().append(op.getResult());
+
+  llvm::SmallVector<mlir::Attribute> recipes;
+
+  if (getReductionRecipesAttr())
+    llvm::copy(getReductionRecipesAttr(), std::back_inserter(recipes));
+
+  recipes.push_back(
+      mlir::SymbolRefAttr::get(context, recipe.getSymName().str()));
+  setReductionRecipesAttr(mlir::ArrayAttr::get(context, recipes));
+}
+
 static ParseResult parseNumGangs(
     mlir::OpAsmParser &parser,
     llvm::SmallVectorImpl<mlir::OpAsmParser::UnresolvedOperand> &operands,
@@ -2041,6 +2071,36 @@ void acc::SerialOp::addPrivatization(MLIRContext *context,
   setPrivatizationRecipesAttr(mlir::ArrayAttr::get(context, recipes));
 }
 
+void acc::SerialOp::addFirstPrivatization(
+    MLIRContext *context, mlir::acc::FirstprivateOp op,
+    mlir::acc::FirstprivateRecipeOp recipe) {
+  getFirstprivateOperandsMutable().append(op.getResult());
+
+  llvm::SmallVector<mlir::Attribute> recipes;
+
+  if (getFirstprivatizationRecipesAttr())
+    llvm::copy(getFirstprivatizationRecipesAttr(), std::back_inserter(recipes));
+
+  recipes.push_back(
+      mlir::SymbolRefAttr::get(context, recipe.getSymName().str()));
+  setFirstprivatizationRecipesAttr(mlir::ArrayAttr::get(context, recipes));
+}
+
+void acc::SerialOp::addReduction(MLIRContext *context,
+                                 mlir::acc::ReductionOp op,
+                                 mlir::acc::ReductionRecipeOp recipe) {
+  getReductionOperandsMutable().append(op.getResult());
+
+  llvm::SmallVector<mlir::Attribute> recipes;
+
+  if (getReductionRecipesAttr())
+    llvm::copy(getReductionRecipesAttr(), std::back_inserter(recipes));
+
+  recipes.push_back(
+      mlir::SymbolRefAttr::get(context, recipe.getSymName().str()));
+  setReductionRecipesAttr(mlir::ArrayAttr::get(context, recipes));
+}
+
 //===----------------------------------------------------------------------===//
 // KernelsOp
 //===----------------------------------------------------------------------===//
@@ -3059,6 +3119,20 @@ void acc::LoopOp::addPrivatization(MLIRContext *context,
   setPrivatizationRecipesAttr(mlir::ArrayAttr::get(context, recipes));
 }
 
+void acc::LoopOp::addReduction(MLIRContext *context, mlir::acc::ReductionOp op,
+                               mlir::acc::ReductionRecipeOp recipe) {
+  getReductionOperandsMutable().append(op.getResult());
+
+  llvm::SmallVector<mlir::Attribute> recipes;
+
+  if (getReductionRecipesAttr())
+    llvm::copy(getReductionRecipesAttr(), std::back_inserter(recipes));
+
+  recipes.push_back(
+      mlir::SymbolRefAttr::get(context, recipe.getSymName().str()));
+  setReductionRecipesAttr(mlir::ArrayAttr::get(context, recipes));
+}
+
 //===----------------------------------------------------------------------===//
 // DataOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index c1c1767ef90b0..6e43f28e8d93d 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -3874,6 +3874,159 @@ LogicalResult AllocateDirOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// TargetAllocMemOp
+//===----------------------------------------------------------------------===//
+
+mlir::Type omp::TargetAllocMemOp::getAllocatedType() {
+  return getInTypeAttr().getValue();
+}
+
+/// operation ::= %res = (`omp.target_alloc_mem`) $device : devicetype,
+///                      $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
+///                      attr-dict-without-keyword
+static mlir::ParseResult parseTargetAllocMemOp(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  auto &builder = parser.getBuilder();
+  bool hasOperands = false;
+  std::int32_t typeparamsSize = 0;
+
+  // Parse device number as a new operand
+  mlir::OpAsmParser::UnresolvedOperand deviceOperand;
+  mlir::Type deviceType;
+  if (parser.parseOperand(deviceOperand) || parser.parseColonType(deviceType))
+    return mlir::failure();
+  if (parser.resolveOperand(deviceOperand, deviceType, result.operands))
+    return mlir::failure();
+  if (parser.parseComma())
+    return mlir::failure();
+
+  mlir::Type intype;
+  if (parser.parseType(intype))
+    return mlir::failure();
+  result.addAttribute("in_type", mlir::TypeAttr::get(intype));
+  llvm::SmallVector<mlir::OpAsmParser::UnresolvedOperand> operands;
+  llvm::SmallVector<mlir::Type> typeVec;
+  if (!parser.parseOptionalLParen()) {
+    // parse the LEN params of the derived type. (<params> : <types>)
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None) ||
+        parser.parseColonTypeList(typeVec) || parser.parseRParen())
+      return mlir::failure();
+    typeparamsSize = operands.size();
+    hasOperands = true;
+  }
+  std::int32_t shapeSize = 0;
+  if (!parser.parseOptionalComma()) {
+    // parse size to scale by, vector of n dimensions of type index
+    if (parser.parseOperandList(operands, mlir::OpAsmParser::Delimiter::None))
+      return mlir::failure();
+    shapeSize = operands.size() - typeparamsSize;
+    auto idxTy = builder.getIndexType();
+    for (std::int32_t i = typeparamsSize, end = operands.size(); i != end; ++i)
+      typeVec.push_back(idxTy);
+    hasOperands = true;
+  }
+  if (hasOperands &&
+      parser.resolveOperands(operands, typeVec, parser.getNameLoc(),
+                             result.operands))
+    return mlir::failure();
+
+  mlir::Type restype = builder.getIntegerType(64);
+  if (!restype) {
+    parser.emitError(parser.getNameLoc(), "invalid allocate type: ") << intype;
+    return mlir::failure();
+  }
+  llvm::SmallVector<std::int32_t> segmentSizes{1, typeparamsSize, shapeSize};
+  result.addAttribute("operandSegmentSizes",
+                      builder.getDenseI32ArrayAttr(segmentSizes));
+  if (parser.parseOptionalAttrDict(result.attributes) ||
+      parser.addTypeToList(restype, result.types))
+    return mlir::failure();
+  return mlir::success();
+}
+
+mlir::ParseResult omp::TargetAllocMemOp::parse(mlir::OpAsmParser &parser,
+                                               mlir::OperationState &result) {
+  return parseTargetAllocMemOp(parser, result);
+}
+
+void omp::TargetAllocMemOp::print(mlir::OpAsmPrinter &p) {
+  p << " ";
+  p.printOperand(getDevice());
+  p << " : ";
+  p << getDevice().getType();
+  p << ", ";
+  p << getInType();
+  if (!getTypeparams().empty()) {
+    p << '(' << getTypeparams() << " : " << getTypeparams().getTypes() << ')';
+  }
+  for (auto sh : getShape()) {
+    p << ", ";
+    p.printOperand(sh);
+  }
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          {"in_type", "operandSegmentSizes"});
+}
+
+llvm::LogicalResult omp::TargetAllocMemOp::verify() {
+  mlir::Type outType = getType();
+  if (!mlir::dyn_cast<IntegerType>(outType))
+    return emitOpError("must be a integer type");
+  return mlir::success();
+}
+
+//===----------------------------------------------------------------------===//
+// WorkdistributeOp
+//===----------------------------------------------------------------------===//
+
+LogicalResult WorkdistributeOp::verify() {
+  // Check that region exists and is not empty
+  Region &region = getRegion();
+  if (region.empty())
+    return emitOpError("region cannot be empty");
+  // Verify single entry point.
+  Block &entryBlock = region.front();
+  if (entryBlock.empty())
+    return emitOpError("region must contain a structured block");
+  // Verify single exit point.
+  bool hasTerminator = false;
+  for (Block &block : region) {
+    if (isa<TerminatorOp>(block.back())) {
+      if (hasTerminator) {
+        return emitOpError("region must have exactly one terminator");
+      }
+      hasTerminator = true;
+    }
+  }
+  if (!hasTerminator) {
+    return emitOpError("region must be terminated with omp.terminator");
+  }
+  auto walkResult = region.walk([&](Operation *op) -> WalkResult {
+    // No implicit barrier at end
+    if (isa<BarrierOp>(op)) {
+      return emitOpError(
+          "explicit barriers are not allowed in workdistribute region");
+    }
+    // Check for invalid nested constructs
+    if (isa<ParallelOp>(op)) {
+      return emitOpError(
+          "nested parallel constructs not allowed in workdistribute");
+    }
+    if (isa<TeamsOp>(op)) {
+      return emitOpError(
+          "nested teams constructs not allowed in workdistribute");
+    }
+    return WalkResult::advance();
+  });
+  if (walkResult.wasInterrupted())
+    return failure();
+
+  Operation *parentOp = (*this)->getParentOp();
+  if (!llvm::dyn_cast<TeamsOp>(parentOp))
+    return emitOpError("workdistribute must be nested under teams");
+  return success();
+}
+
 #define GET_ATTRDEF_CLASSES
 #include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp
index 0262a1b8a3893..84f9777a443fd 100644
--- a/mlir/lib/Dialect/SCF/IR/SCF.cpp
+++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp
@@ -157,8 +157,7 @@ void ExecuteRegionOp::print(OpAsmPrinter &p) {
   p.printRegion(getRegion(),
                 /*printEntryBlockArgs=*/false,
                 /*printBlockTerminators=*/true);
-
-  p.printOptionalAttrDict((*this)->getAttrs());
+  p.printOptionalAttrDict((*this)->getAttrs(), /*elidedAttrs=*/{"no_inline"});
 }
 
 LogicalResult ExecuteRegionOp::verify() {
@@ -318,9 +317,12 @@ void ConditionOp::getSuccessorRegions(
 
 void ForOp::build(OpBuilder &builder, OperationState &result, Value lb,
                   Value ub, Value step, ValueRange initArgs,
-                  BodyBuilderFn bodyBuilder) {
+                  BodyBuilderFn bodyBuilder, bool unsignedCmp) {
   OpBuilder::InsertionGuard guard(builder);
 
+  if (unsignedCmp)
+    result.addAttribute(getUnsignedCmpAttrName(result.name),
+                        builder.getUnitAttr());
   result.addOperands({lb, ub, step});
   result.addOperands(initArgs);
   for (Value v : initArgs)
@@ -450,6 +452,9 @@ static void printInitializationList(OpAsmPrinter &p,
 }
 
 void ForOp::print(OpAsmPrinter &p) {
+  if (getUnsignedCmp())
+    p << " unsigned";
+
   p << " " << getInductionVar() << " = " << getLowerBound() << " to "
     << getUpperBound() << " step " << getStep();
 
@@ -462,7 +467,8 @@ void ForOp::print(OpAsmPrinter &p) {
   p.printRegion(getRegion(),
                 /*printEntryBlockArgs=*/false,
                 /*printBlockTerminators=*/!getInitArgs().empty());
-  p.printOptionalAttrDict((*this)->getAttrs());
+  p.printOptionalAttrDict((*this)->getAttrs(),
+                          /*elidedAttrs=*/getUnsignedCmpAttrName().strref());
 }
 
 ParseResult ForOp::parse(OpAsmParser &parser, OperationState &result) {
@@ -472,6 +478,10 @@ ParseResult ForOp::parse(OpAsmParser &parser, OperationState &result) {
   OpAsmParser::Argument inductionVariable;
   OpAsmParser::UnresolvedOperand lb, ub, step;
 
+  if (succeeded(parser.parseOptionalKeyword("unsigned")))
+    result.addAttribute(getUnsignedCmpAttrName(result.name),
+                        builder.getUnitAttr());
+
   // Parse the induction variable followed by '='.
   if (parser.parseOperand(inductionVariable.ssaName) || parser.parseEqual() ||
       // Parse loop bounds.
@@ -562,7 +572,7 @@ ForOp::replaceWithAdditionalYields(RewriterBase &rewriter,
   inits.append(newInitOperands.begin(), newInitOperands.end());
   scf::ForOp newLoop = scf::ForOp::create(
       rewriter, getLoc(), getLowerBound(), getUpperBound(), getStep(), inits,
-      [](OpBuilder &, Location, Value, ValueRange) {});
+      [](OpBuilder &, Location, Value, ValueRange) {}, getUnsignedCmp());
   newLoop->setAttrs(getPrunedAttributeList(getOperation(), {}));
 
   // Generate the new yield values and append them to the scf.yield operation.
@@ -806,7 +816,8 @@ mlir::scf::replaceAndCastForOpIterArg(RewriterBase &rewriter, scf::ForOp forOp,
   // 2. Create the new forOp shell.
   scf::ForOp newForOp = scf::ForOp::create(
       rewriter, forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-      forOp.getStep(), newIterOperands);
+      forOp.getStep(), newIterOperands, /*bodyBuilder=*/nullptr,
+      forOp.getUnsignedCmp());
   newForOp->setAttrs(forOp->getAttrs());
   Block &newBlock = newForOp.getRegion().front();
   SmallVector<Value, 4> newBlockTransferArgs(newBlock.getArguments().begin(),
@@ -931,7 +942,8 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
 
     scf::ForOp newForOp =
         scf::ForOp::create(rewriter, forOp.getLoc(), forOp.getLowerBound(),
-                           forOp.getUpperBound(), forOp.getStep(), newIterArgs);
+                           forOp.getUpperBound(), forOp.getStep(), newIterArgs,
+                           /*bodyBuilder=*/nullptr, forOp.getUnsignedCmp());
     newForOp->setAttrs(forOp->getAttrs());
     Block &newBlock = newForOp.getRegion().front();
 
@@ -989,12 +1001,12 @@ struct ForOpIterArgsFolder : public OpRewritePattern<scf::ForOp> {
 /// Util function that tries to compute a constant diff between u and l.
 /// Returns std::nullopt when the difference between two AffineValueMap is
 /// dynamic.
-static std::optional<int64_t> computeConstDiff(Value l, Value u) {
+static std::optional<APInt> computeConstDiff(Value l, Value u) {
   IntegerAttr clb, cub;
   if (matchPattern(l, m_Constant(&clb)) && matchPattern(u, m_Constant(&cub))) {
     llvm::APInt lbValue = clb.getValue();
     llvm::APInt ubValue = cub.getValue();
-    return (ubValue - lbValue).getSExtValue();
+    return ubValue - lbValue;
   }
 
   // Else a simple pattern match for x + c or c + x
@@ -1003,7 +1015,7 @@ static std::optional<int64_t> computeConstDiff(Value l, Value u) {
           u, m_Op<arith::AddIOp>(matchers::m_Val(l), m_ConstantInt(&diff))) ||
       matchPattern(
           u, m_Op<arith::AddIOp>(m_ConstantInt(&diff), matchers::m_Val(l))))
-    return diff.getSExtValue();
+    return diff;
   return std::nullopt;
 }
 
@@ -1022,13 +1034,15 @@ struct SimplifyTrivialLoops : public OpRewritePattern<ForOp> {
       return success();
     }
 
-    std::optional<int64_t> diff =
+    std::optional<APInt> diff =
         computeConstDiff(op.getLowerBound(), op.getUpperBound());
     if (!diff)
       return failure();
 
     // If the loop is known to have 0 iterations, remove it.
-    if (*diff <= 0) {
+    bool zeroOrLessIterations =
+        diff->isZero() || (!op.getUnsignedCmp() && diff->isNegative());
+    if (zeroOrLessIterations) {
       rewriter.replaceOp(op, op.getInitArgs());
       return success();
     }
@@ -3384,9 +3398,8 @@ ParseResult scf::WhileOp::parse(OpAsmParser &parser, OperationState &result) {
 
   if (functionType.getNumInputs() != operands.size()) {
     return parser.emitError(typeLoc)
-           << "expected as many input types as operands "
-           << "(expected " << operands.size() << " got "
-           << functionType.getNumInputs() << ")";
+           << "expected as many input types as operands " << "(expected "
+           << operands.size() << " got " << functionType.getNumInputs() << ")";
   }
 
   // Resolve input operands.
@@ -4222,14 +4235,15 @@ LogicalResult scf::IndexSwitchOp::verify() {
              << "see yield operation here";
     }
     for (auto [idx, result, operand] :
-         llvm::zip(llvm::seq<unsigned>(0, getNumResults()), getResultTypes(),
-                   yield.getOperandTypes())) {
-      if (result == operand)
+         llvm::enumerate(getResultTypes(), yield.getOperands())) {
+      if (!operand)
+        return yield.emitOpError() << "operand " << idx << " is null\n";
+      if (result == operand.getType())
         continue;
       return (emitOpError("expected result #")
               << idx << " of each region to be " << result)
                  .attachNote(yield.getLoc())
-             << name << " returns " << operand << " here";
+             << name << " returns " << operand.getType() << " here";
     }
     return success();
   };
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index f8799c52e8797..fb179e64d8e7b 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -769,7 +769,8 @@ struct ForOpInterface
     // Construct a new scf.for op with memref instead of tensor values.
     auto newForOp = scf::ForOp::create(
         rewriter, forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-        forOp.getStep(), castedInitArgs);
+        forOp.getStep(), castedInitArgs, /*bodyBuilder=*/nullptr,
+        forOp.getUnsignedCmp());
     newForOp->setAttrs(forOp->getAttrs());
     Block *loopBody = newForOp.getBody();
 
diff --git a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
index bee77800e9d44..ae52af5009dc9 100644
--- a/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/ForToWhile.cpp
@@ -58,9 +58,12 @@ struct ForLoopLoweringPattern : public OpRewritePattern<ForOp> {
     auto *beforeBlock = rewriter.createBlock(
         &whileOp.getBefore(), whileOp.getBefore().begin(), lcvTypes, lcvLocs);
     rewriter.setInsertionPointToStart(whileOp.getBeforeBody());
-    auto cmpOp = arith::CmpIOp::create(
-        rewriter, whileOp.getLoc(), arith::CmpIPredicate::slt,
-        beforeBlock->getArgument(0), forOp.getUpperBound());
+    arith::CmpIPredicate predicate = forOp.getUnsignedCmp()
+                                         ? arith::CmpIPredicate::ult
+                                         : arith::CmpIPredicate::slt;
+    auto cmpOp = arith::CmpIOp::create(rewriter, whileOp.getLoc(), predicate,
+                                       beforeBlock->getArgument(0),
+                                       forOp.getUpperBound());
     scf::ConditionOp::create(rewriter, whileOp.getLoc(), cmpOp.getResult(),
                              beforeBlock->getArguments());
 
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index 1130538e51fb7..7e7fba43fe9e0 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -791,6 +791,11 @@ FailureOr<ForOp> mlir::scf::pipelineForLoop(RewriterBase &rewriter, ForOp forOp,
                                             bool *modifiedIR) {
   if (modifiedIR)
     *modifiedIR = false;
+
+  // TODO: Add support for unsigned loops.
+  if (forOp.getUnsignedCmp())
+    return failure();
+
   LoopPipelinerInternal pipeliner;
   if (!pipeliner.initializeLoopInfo(forOp, options))
     return failure();
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
index 4752c0837c1c5..f1203b2bdfee5 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopSpecialization.cpp
@@ -256,6 +256,10 @@ struct ForLoopPeelingPattern : public OpRewritePattern<ForOp> {
 
   LogicalResult matchAndRewrite(ForOp forOp,
                                 PatternRewriter &rewriter) const override {
+    if (forOp.getUnsignedCmp())
+      return rewriter.notifyMatchFailure(forOp,
+                                         "unsigned loops are not supported");
+
     // Do not peel already peeled loops.
     if (forOp->hasAttr(kPeeledLoopLabel))
       return failure();
diff --git a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
index 1b07b77546030..3b75970c98ad4 100644
--- a/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/StructuralTypeConversions.cpp
@@ -116,7 +116,8 @@ class ConvertForOpTypes
                                 llvm::getSingleElement(adaptor.getLowerBound()),
                                 llvm::getSingleElement(adaptor.getUpperBound()),
                                 llvm::getSingleElement(adaptor.getStep()),
-                                flattenValues(adaptor.getInitArgs()));
+                                flattenValues(adaptor.getInitArgs()),
+                                /*bodyBuilder=*/nullptr, op.getUnsignedCmp());
 
     // Reserve whatever attributes in the original op.
     newOp->setAttrs(op->getAttrs());
diff --git a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
index c0e47ee1e74fc..250c413eff9e5 100644
--- a/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/TileUsingInterface.cpp
@@ -797,7 +797,8 @@ FailureOr<LoopLikeOpInterface> yieldTiledValuesAndReplaceLoop<scf::ForOp>(
   inits.append(newInitOperands.begin(), newInitOperands.end());
   auto newLoop = scf::ForOp::create(
       rewriter, loc, loopOp.getLowerBound(), loopOp.getUpperBound(),
-      loopOp.getStep(), inits, [](OpBuilder &, Location, Value, ValueRange) {});
+      loopOp.getStep(), inits, [](OpBuilder &, Location, Value, ValueRange) {},
+      loopOp.getUnsignedCmp());
 
   // Move the loop body to the new op.
   Block *loopBody = loopOp.getBody();
@@ -935,7 +936,8 @@ static LogicalResult addInitOperandsToLoopNest(
     auto newLoop = scf::ForOp::create(
         rewriter, forLoop.getLoc(), forLoop.getLowerBound(),
         forLoop.getUpperBound(), forLoop.getStep(), newInits,
-        [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {});
+        [&](OpBuilder &b, Location loc, Value iv, ValueRange iterArgs) {},
+        forLoop.getUnsignedCmp());
 
     // Merge the body of the new loop with the body of the old loops.
     SmallVector<Value> sourceBlockArgs;
diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
index 57317951d609c..49102583ec5e7 100644
--- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp
@@ -1233,6 +1233,7 @@ static void getPerfectlyNestedLoopsImpl(
 
 static Loops stripmineSink(scf::ForOp forOp, Value factor,
                            ArrayRef<scf::ForOp> targets) {
+  assert(!forOp.getUnsignedCmp() && "unsigned loops are not supported");
   auto originalStep = forOp.getStep();
   auto iv = forOp.getInductionVar();
 
@@ -1241,6 +1242,8 @@ static Loops stripmineSink(scf::ForOp forOp, Value factor,
 
   Loops innerLoops;
   for (auto t : targets) {
+    assert(!t.getUnsignedCmp() && "unsigned loops are not supported");
+
     // Save information for splicing ops out of t when done
     auto begin = t.getBody()->begin();
     auto nOps = t.getBody()->getOperations().size();
@@ -1415,6 +1418,8 @@ scf::ForallOp mlir::fuseIndependentSiblingForallLoops(scf::ForallOp target,
 scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
                                                 scf::ForOp source,
                                                 RewriterBase &rewriter) {
+  assert(source.getUnsignedCmp() == target.getUnsignedCmp() &&
+         "incompatible signedness");
   unsigned numTargetOuts = target.getNumResults();
   unsigned numSourceOuts = source.getNumResults();
 
@@ -1428,7 +1433,8 @@ scf::ForOp mlir::fuseIndependentSiblingForLoops(scf::ForOp target,
   rewriter.setInsertionPointAfter(source);
   scf::ForOp fusedLoop = scf::ForOp::create(
       rewriter, source.getLoc(), source.getLowerBound(), source.getUpperBound(),
-      source.getStep(), fusedInitArgs);
+      source.getStep(), fusedInitArgs, /*bodyBuilder=*/nullptr,
+      source.getUnsignedCmp());
 
   // Map original induction variables and operands to those of the fused loop.
   IRMapping mapping;
diff --git a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
index 3b97786e58155..dabbea1bdec63 100644
--- a/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Pipelines/SparseTensorPipelines.cpp
@@ -71,7 +71,6 @@ void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm,
   pm.addPass(createLowerAffinePass());
   pm.addPass(
       createConvertVectorToLLVMPass(options.convertVectorToLLVMOptions()));
-  pm.addPass(createFinalizeMemRefToLLVMConversionPass());
   pm.addNestedPass<func::FuncOp>(createConvertComplexToStandardPass());
   pm.addNestedPass<func::FuncOp>(arith::createArithExpandOpsPass());
   pm.addNestedPass<func::FuncOp>(createConvertMathToLLVMPass());
@@ -79,12 +78,6 @@ void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm,
   pm.addPass(createConvertComplexToLibm());
   pm.addPass(
       createConvertVectorToLLVMPass(options.convertVectorToLLVMOptions()));
-  pm.addPass(createConvertComplexToLLVMPass());
-  pm.addPass(
-      createConvertVectorToLLVMPass(options.convertVectorToLLVMOptions()));
-  pm.addPass(createConvertFuncToLLVMPass());
-  pm.addPass(createArithToLLVMConversionPass());
-  pm.addPass(createConvertControlFlowToLLVMPass());
 
   // Finalize GPU code generation.
   if (gpuCodegen) {
@@ -99,8 +92,8 @@ void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm,
     pm.addPass(createGpuModuleToBinaryPass(gpuModuleToBinaryPassOptions));
   }
 
-  // Convert poison values.
-  pm.addPass(createUBToLLVMConversionPass());
+  // Convert to LLVM.
+  pm.addPass(createConvertToLLVMPass());
 
   // Ensure all casts are realized.
   pm.addPass(createReconcileUnrealizedCastsPass());
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 134aef3a6c719..0e88d31dae8e8 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -730,9 +730,9 @@ class SparseTensorCompressConverter : public OpConversionPattern<CompressOp> {
                    {tensor, lvlCoords, values, filled, added, count},
                    EmitCInterface::On);
     Operation *parent = getTop(op);
+    rewriter.setInsertionPointAfter(parent);
     rewriter.replaceOp(op, adaptor.getTensor());
     // Deallocate the buffers on exit of the loop nest.
-    rewriter.setInsertionPointAfter(parent);
     memref::DeallocOp::create(rewriter, loc, values);
     memref::DeallocOp::create(rewriter, loc, filled);
     memref::DeallocOp::create(rewriter, loc, added);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
index 4464450fd328f..febec6d2d2ce4 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
@@ -533,8 +533,10 @@ static bool vectorizeStmt(PatternRewriter &rewriter, scf::ForOp forOp, VL vl,
       VectorType vtp = vectorType(vl, init.getType());
       Value vinit = genVectorReducInit(rewriter, loc, yield->getOperand(0),
                                        forOp.getRegionIterArg(0), init, vtp);
-      forOpNew = scf::ForOp::create(rewriter, loc, forOp.getLowerBound(),
-                                    forOp.getUpperBound(), step, vinit);
+      forOpNew =
+          scf::ForOp::create(rewriter, loc, forOp.getLowerBound(),
+                             forOp.getUpperBound(), step, vinit,
+                             /*bodyBuilder=*/nullptr, forOp.getUnsignedCmp());
       forOpNew->setAttr(
           LoopEmitter::getLoopEmitterLoopAttrName(),
           forOp->getAttr(LoopEmitter::getLoopEmitterLoopAttrName()));
@@ -605,8 +607,8 @@ struct ForOpRewriter : public OpRewritePattern<scf::ForOp> {
 
   ForOpRewriter(MLIRContext *context, unsigned vectorLength,
                 bool enableVLAVectorization, bool enableSIMDIndex32)
-      : OpRewritePattern(context), vl{vectorLength, enableVLAVectorization,
-                                      enableSIMDIndex32} {}
+      : OpRewritePattern(context),
+        vl{vectorLength, enableVLAVectorization, enableSIMDIndex32} {}
 
   LogicalResult matchAndRewrite(scf::ForOp op,
                                 PatternRewriter &rewriter) const override {
diff --git a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
index 2ec23e1fb35ce..dfce835a1954b 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/ReshapePatterns.cpp
@@ -327,172 +327,31 @@ struct BubbleUpExpandShapeThroughExtractSlice
                                 PatternRewriter &rewriter) const override {
     auto expandShapeOp =
         sliceOp.getSource().getDefiningOp<tensor::ExpandShapeOp>();
+    if (!expandShapeOp) {
+      return rewriter.notifyMatchFailure(
+          sliceOp, "tensor.extract_slice source not produced by expand_shape");
+    }
+    SmallVector<ReassociationIndices> reassociation =
+        expandShapeOp.getReassociationIndices();
 
-    if (checkPreconditionForBubbleUpExtractSlice(sliceOp, expandShapeOp,
-                                                 rewriter)
-            .failed())
+    SmallVector<OpFoldResult> offsets, sizes, strides;
+    if (failed(getCollapsedExtractSliceInfo(rewriter, sliceOp, reassociation,
+                                            offsets, sizes, strides)))
       return failure();
 
-    // The tensor.extract_slice before applying the pattern works on the result
-    // of the tensor.expand_shape, so variables (i.e. inputs for ExtractSliceOp)
-    // referring to the state before applying the pattern are named with the
-    // prefix "expanded", and ones referring to the state after applying the
-    // pattern are named with the prefix "collapsed".
-    SmallVector<OpFoldResult> expandedOffsets = sliceOp.getMixedOffsets();
-    SmallVector<OpFoldResult> expandedSizes = sliceOp.getMixedSizes();
-    SmallVector<OpFoldResult> expandedShape =
-        getMixedValues(expandShapeOp.getStaticOutputShape(),
-                       expandShapeOp.getOutputShape(), rewriter);
-
-    // Helper variables and function for accumulating the size values.
-    Location loc = expandShapeOp->getLoc();
-    AffineExpr d0, d1, d2;
-    bindDims(rewriter.getContext(), d0, d1, d2);
-    // Multiply two integers.
-    auto mul = [&](OpFoldResult v1, OpFoldResult v2) {
-      auto mulMap = AffineMap::get(2, 0, {d0 * d1});
-      return affine::makeComposedFoldedAffineApply(rewriter, loc, mulMap,
-                                                   {v1, v2});
-    };
-
-    // Compute new offsets, sizes, and strides for tensor.extract_slice.
-    // The new tensor.extract_slice will work on a tensor that has has a rank of
-    // ReassociationIndices.size(). In the loop a single offset, size, and
-    // stride value is computed per reassociation group.
-    SmallVector<OpFoldResult> collapsedOffsets, collapsedSizes,
-        collapsedStrides;
-    for (const ReassociationIndices &indices :
-         expandShapeOp.getReassociationIndices()) {
-      // collapsedSize will hold the size of the single dim that represents the
-      // reassociation group in the non expanded tensor.
-      OpFoldResult collapsedSize = rewriter.getIndexAttr(1);
-      // The reassocGroupSizes and reassocGroupOffsets are used to create an
-      // affine.linearize_index op to linearize the single offset value required
-      // for this reassociation group.
-      SmallVector<OpFoldResult> reassocGroupSizes, reassocGroupOffsets;
-
-      for (long expandedDim : indices) {
-        // reassocGroupSizes and reassocGroupOffsets can be obtained directly
-        // from the expanded state, but the collapsed size requires calculation
-        // as it did not previously exist.
-        reassocGroupSizes.push_back(expandedShape[expandedDim]);
-        reassocGroupOffsets.push_back(expandedOffsets[expandedDim]);
-        collapsedSize = mul(collapsedSize, expandedSizes[expandedDim]);
-      }
-
-      SmallVector<Value> offsetVals =
-          llvm::map_to_vector(reassocGroupOffsets, [&](OpFoldResult ofr) {
-            return getValueOrCreateConstantIndexOp(rewriter, loc, ofr);
-          });
-      OpFoldResult collapsedOffset =
-          affine::AffineLinearizeIndexOp::create(rewriter, loc, offsetVals,
-                                                 reassocGroupSizes,
-                                                 /*disjoint=*/true)
-              .getResult();
-      collapsedOffsets.push_back(collapsedOffset);
-      collapsedSizes.push_back(collapsedSize);
-
-      // Only unit stride is supported.
-      collapsedStrides.push_back(rewriter.getIndexAttr(1));
-    }
-
     // The shape of the result can be obtained from the sizes passed in.
-    SmallVector<Value> dynDims;
-    SmallVector<int64_t> shape;
-    dispatchIndexOpFoldResults(expandedSizes, dynDims, shape);
-    RankedTensorType resultType = RankedTensorType::get(
-        shape, expandShapeOp.getResultType().getElementType());
+    SmallVector<OpFoldResult> expandedSizes = sliceOp.getMixedSizes();
+    RankedTensorType resultType = sliceOp.getResultType();
 
     // Create a new ExtractSliceOp and ExpandShapeOp.
+    Location loc = sliceOp.getLoc();
     Value newSliceOp = tensor::ExtractSliceOp::create(
-        rewriter, loc, expandShapeOp.getSrc(), collapsedOffsets, collapsedSizes,
-        collapsedStrides);
+        rewriter, loc, expandShapeOp.getSrc(), offsets, sizes, strides);
     rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
         sliceOp, resultType, newSliceOp,
         expandShapeOp.getReassociationIndices(), expandedSizes);
     return success();
   }
-
-  // Helper function to check if all the required conditions for the
-  // tensor.extract_slice to be bubbled up through the tensor.expand_shape are
-  // met.
-  LogicalResult
-  checkPreconditionForBubbleUpExtractSlice(tensor::ExtractSliceOp sliceOp,
-                                           tensor::ExpandShapeOp expandShapeOp,
-                                           PatternRewriter &rewriter) const {
-
-    if (!expandShapeOp) {
-      return rewriter.notifyMatchFailure(
-          sliceOp, "tensor.extract_slice source not produced by expand_shape");
-    }
-
-    if (!sliceOp.hasUnitStride()) {
-      return rewriter.notifyMatchFailure(
-          sliceOp, "unsupported: non-unit stride. Only contiguous slices can "
-                   "be supported in this transformation.");
-    }
-
-    SmallVector<OpFoldResult> offsets = sliceOp.getMixedOffsets();
-    SmallVector<OpFoldResult> sizes = sliceOp.getMixedSizes();
-
-    if (static_cast<size_t>(sliceOp.getResultType().getRank()) !=
-        sizes.size()) {
-      return rewriter.notifyMatchFailure(sliceOp,
-                                         "unimplemented: rank reducing slice");
-    }
-
-    SmallVector<OpFoldResult> outputShape =
-        getMixedValues(expandShapeOp.getStaticOutputShape(),
-                       expandShapeOp.getOutputShape(), rewriter);
-
-    std::function<bool(OpFoldResult, OpFoldResult, OpFoldResult)>
-        isZeroOffsetAndFullSize =
-            [](OpFoldResult offset, OpFoldResult sliceSize, OpFoldResult size) {
-              if (!isZeroInteger(offset))
-                return false;
-              FailureOr<bool> maybeEqual =
-                  ValueBoundsConstraintSet::areEqual(sliceSize, size);
-              return llvm::succeeded(maybeEqual) && maybeEqual.value();
-            };
-
-    // Check that the slice is contiguous within each reassociation group.
-    // The slice is contiguous only if after the first dimension where a non
-    // unit slice is taken, the slice size on all subsequent dimensions of the
-    // group is equal to the entire size of the dimension.
-    // Examples of contiguous slices:
-    //   full sizes: [8, 8, 10] slice offsets: [0, 0, 0] slice sizes: [1, 1, 10]
-    //   full sizes: [5, 10] slice offsets: [3, 0] slice sizes: [2, 10]
-    // Examples of non contiguous slices:
-    //   full sizes: [8, 8, 10] slice offsets: [0, 0, 0] slice sizes: [1, 2, 5]
-    //   full sizes: [5, 10] slice offsets: [0, 4] slice sizes: [2, 5]
-    for (const ReassociationIndices &indices :
-         expandShapeOp.getReassociationIndices()) {
-      int64_t i = 0;
-      int64_t e = indices.size();
-      // Find the first expanded dim after the first dim with non-unit extracted
-      // size.
-      for (; i < e; ++i) {
-        if (!isOneInteger(sizes[indices[i]])) {
-          // +1 to skip the first non-unit size dim.
-          i++;
-          break;
-        }
-      }
-
-      // Verify that all subsequent dimensions extract the full size of the
-      // source tensor.
-      for (; i < e; ++i) {
-        int64_t expandedDim = indices[i];
-        if (!isZeroOffsetAndFullSize(offsets[expandedDim], sizes[expandedDim],
-                                     outputShape[expandedDim])) {
-          return rewriter.notifyMatchFailure(
-              sliceOp, "Not a contiguous slice of the expanded tensor.");
-        }
-      }
-    }
-
-    return success();
-  }
 };
 
 /// Converts `tensor.extract_slice(tensor.collapse_shape)` to
@@ -582,170 +441,281 @@ struct BubbleUpCollapseShapeThroughExtractSlice
           "tensor.extract_slice source not produced by tensor.collapse_shape");
     }
 
-    if (!sliceOp.hasUnitStride()) {
-      return rewriter.notifyMatchFailure(
-          sliceOp, "unsupported: non-unit stride. Only contiguous slices can "
-                   "be supported in this transformation.");
-    }
+    SmallVector<OpFoldResult> offsets, sizes, strides;
+    if (failed(getExpandedExtractSliceInfo(
+            rewriter, sliceOp, collapseShapeOp.getReassociationIndices(),
+            collapseShapeOp.getSrcType().getShape(), offsets, sizes, strides)))
+      return failure();
 
-    // The tensor.extract_slice before applying the pattern works on the result
-    // of the tensor.collapse_shape, so variables (i.e. inputs for
-    // ExtractSliceOp) referring to the state before applying the pattern are
-    // named with the prefix "collapsed", and ones referring to the state after
-    // applying the pattern are named with the prefix "expanded".
-    SmallVector<OpFoldResult> collapsedOffsets = sliceOp.getMixedOffsets();
-    SmallVector<OpFoldResult> collapsedSizes = sliceOp.getMixedSizes();
-
-    if (static_cast<size_t>(sliceOp.getResultType().getRank()) !=
-        collapsedSizes.size()) {
-      return rewriter.notifyMatchFailure(sliceOp,
-                                         "unimplemented: rank reducing slice");
-    }
+    Value newSliceOp = tensor::ExtractSliceOp::create(
+        rewriter, collapseShapeOp->getLoc(), collapseShapeOp.getSrc(), offsets,
+        sizes, strides);
+    rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
+        sliceOp, sliceOp.getResultType(), newSliceOp,
+        collapseShapeOp.getReassociationIndices());
 
-    ArrayRef<int64_t> srcShape = collapseShapeOp.getSrcType().getShape();
-    SmallVector<ReassociationIndices, 4> reassociationIndices =
-        collapseShapeOp.getReassociationIndices();
-
-    // Compute new offsets, sizes, and strides for tensor.extract_slice.
-    // The new tensor.extract_slice will work on a tensor that has has a rank
-    // equal to the rank of the src of the collapse_shape. In each iteration of
-    // the loop, the offsets and sizes will be computed per reassociation group.
-    SmallVector<OpFoldResult> expandedOffsets, expandedSizes;
-    SmallVector<OpFoldResult> expandedStrides(srcShape.size(),
-                                              rewriter.getIndexAttr(1));
-
-    for (auto [collapsedSize, collapsedOffset, reassocIndices] :
-         llvm::zip_equal(collapsedSizes, collapsedOffsets,
-                         collapseShapeOp.getReassociationIndices())) {
-      // CASE #1 - size and/or offset are dynamic.
-      // In this case, the slice can be represented as a contiguous slice only
-      // if there is a single dimension in the reassociation group that has a
-      // size not equal to 1.
-      if (isa<Value>(collapsedSize) || isa<Value>(collapsedOffset)) {
-        int nonUnitSizeCount = 0;
-        for (int64_t expandedShapeIdx : reassocIndices) {
-          if (srcShape[expandedShapeIdx] != 1) {
-            nonUnitSizeCount++;
-            expandedSizes.push_back(collapsedSize);
-            expandedOffsets.push_back(collapsedOffset);
-            continue;
-          }
-
-          expandedSizes.push_back(rewriter.getIndexAttr(1));
-          expandedOffsets.push_back(rewriter.getIndexAttr(0));
-        }
+    return success();
+  }
+};
 
-        if (nonUnitSizeCount != 1) {
-          return rewriter.notifyMatchFailure(
-              sliceOp,
-              "unsupported: slice cannot be verified to be contiguous");
-        }
-        continue;
-      }
+} // namespace
 
-      // CASE #2 = size and offset are static.
-      // Verify that the slice can be represented as a contiguous slice of the
-      // src of the collapse_shape.
-      // Checking this is done on order of most internal dimensions first,
-      // so traversal is done in reverse order of the reassociation group.
-      // If the expected slice shape is [1, 1, ..., 1, Sk, Ak + 1, Ak + 2,
-      // ...,An] then we first find the size and offset for n...k+1 then for k
-      // and then for k-1...0.
-
-      // currentCollapsedsize and currentCollapsedOffset are initialized with
-      // the original collapsed size and offset and divided by the expanded
-      // shape size in each dimension as we go along the reassociation group.
-      // In essence we are spreading the original collapsed size and offset over
-      // the various expanded slice dimensions.
-      // The variables are used both to check the validity of the slice and to
-      // compute the expanded sizes and offsets.
-      int64_t currentCollapsedsize = getConstantIntValue(collapsedSize).value();
-      int64_t currentCollapsedOffset =
-          getConstantIntValue(collapsedOffset).value();
-
-      SmallVector<OpFoldResult> groupExpandedSizes, groupExpandedOffsets;
-
-      ReassociationIndices reversedReassocIndices(reassocIndices.rbegin(),
-                                                  reassocIndices.rend());
-      int64_t idx = 0;
-      int64_t reassocGroupSize = reassocIndices.size();
-
-      // First handle the trailing dimensions where the slice size should be
-      // equal to the tensor shape and the offset should be 0 (n...k+1).
-      for (; idx < reassocGroupSize; ++idx) {
-        int64_t expandedShapeSize = srcShape[reversedReassocIndices[idx]];
-
-        if (currentCollapsedsize < expandedShapeSize)
-          break;
-
-        // We need to make sure that the slice size can be set to the shape size
-        // and the offset to 0.
-        if ((currentCollapsedsize % expandedShapeSize) != 0 ||
-            (currentCollapsedOffset % expandedShapeSize) != 0) {
-          return rewriter.notifyMatchFailure(
-              sliceOp, "unsupported: cannot be extracted as a contiguous slice "
-                       "of the src of the collapse_shape");
-        }
+LogicalResult mlir::tensor::getCollapsedExtractSliceInfo(
+    OpBuilder &b, tensor::ExtractSliceOp sliceOp,
+    ArrayRef<ReassociationIndices> reassociation,
+    SmallVectorImpl<OpFoldResult> &collapsedOffsets,
+    SmallVectorImpl<OpFoldResult> &collapsedSizes,
+    SmallVectorImpl<OpFoldResult> &collapsedStrides) {
+  if (!sliceOp.hasUnitStride()) {
+    return failure();
+  }
+
+  SmallVector<OpFoldResult> offsets = sliceOp.getMixedOffsets();
+  SmallVector<OpFoldResult> sizes = sliceOp.getMixedSizes();
 
-        groupExpandedSizes.push_back(rewriter.getIndexAttr(expandedShapeSize));
-        groupExpandedOffsets.push_back(rewriter.getIndexAttr(0));
+  if (static_cast<size_t>(sliceOp.getResultType().getRank()) != sizes.size()) {
+    return failure();
+  }
 
-        currentCollapsedsize /= expandedShapeSize;
-        currentCollapsedOffset /= expandedShapeSize;
+  auto isZeroOffsetAndFullSize = [&](OpFoldResult offset,
+                                     OpFoldResult sliceSize, int64_t inputDim) {
+    if (!isZeroInteger(offset))
+      return false;
+    ValueBoundsConstraintSet::Variable inputSize(sliceOp.getSource(), inputDim);
+    FailureOr<bool> maybeEqual =
+        ValueBoundsConstraintSet::areEqual(sliceSize, inputSize);
+    return llvm::succeeded(maybeEqual) && maybeEqual.value();
+  };
+
+  // Check that the slice is contiguous within each reassociation group.
+  // The slice is contiguous only if after the first dimension where a non
+  // unit slice is taken, the slice size on all subsequent dimensions of the
+  // group is equal to the entire size of the dimension.
+  // Examples of contiguous slices:
+  //   full sizes: [8, 8, 10] slice offsets: [0, 0, 0] slice sizes: [1, 1, 10]
+  //   full sizes: [5, 10] slice offsets: [3, 0] slice sizes: [2, 10]
+  // Examples of non contiguous slices:
+  //   full sizes: [8, 8, 10] slice offsets: [0, 0, 0] slice sizes: [1, 2, 5]
+  //   full sizes: [5, 10] slice offsets: [0, 4] slice sizes: [2, 5]
+  for (const ReassociationIndices &indices : reassociation) {
+    int64_t i = 0;
+    int64_t e = indices.size();
+    // Find the first expanded dim after the first dim with non-unit extracted
+    // size.
+    for (; i < e; ++i) {
+      if (!isOneInteger(sizes[indices[i]])) {
+        // +1 to skip the first non-unit size dim.
+        i++;
+        break;
       }
+    }
+
+    // Verify that all subsequent dimensions extract the full size of the
+    // source tensor.
+    for (; i < e; ++i) {
+      int64_t expandedDim = indices[i];
+      if (!isZeroOffsetAndFullSize(offsets[expandedDim], sizes[expandedDim],
+                                   expandedDim)) {
+        return failure();
+      }
+    }
+  }
+
+  // The tensor.extract_slice before applying the pattern works on the result
+  // of the tensor.expand_shape, so variables (i.e. inputs for ExtractSliceOp)
+  // referring to the state before applying the pattern are named with the
+  // prefix "expanded", and ones referring to the state after applying the
+  // pattern are named with the prefix "collapsed".
+  Location loc = sliceOp.getLoc();
+  SmallVector<OpFoldResult> expandedOffsets = sliceOp.getMixedOffsets();
+  SmallVector<OpFoldResult> expandedSizes = sliceOp.getMixedSizes();
+  SmallVector<OpFoldResult> expandedShape =
+      getMixedSizes(b, loc, sliceOp.getSource());
+
+  // Helper variables and function for accumulating the size values.
+  AffineExpr d0, d1, d2;
+  bindDims(b.getContext(), d0, d1, d2);
+  // Multiply two integers.
+  auto mul = [&](OpFoldResult v1, OpFoldResult v2) {
+    auto mulMap = AffineMap::get(2, 0, {d0 * d1});
+    return affine::makeComposedFoldedAffineApply(b, loc, mulMap, {v1, v2});
+  };
+
+  // Compute new offsets, sizes, and strides for tensor.extract_slice.
+  // The new tensor.extract_slice will work on a tensor that has has a rank of
+  // ReassociationIndices.size(). In the loop a single offset, size, and
+  // stride value is computed per reassociation group.
+  for (const ReassociationIndices &indices : reassociation) {
+    // collapsedSize will hold the size of the single dim that represents the
+    // reassociation group in the non expanded tensor.
+    OpFoldResult collapsedSize = b.getIndexAttr(1);
+    // The reassocGroupSizes and reassocGroupOffsets are used to create an
+    // affine.linearize_index op to linearize the single offset value required
+    // for this reassociation group.
+    SmallVector<OpFoldResult> reassocGroupSizes, reassocGroupOffsets;
+
+    for (long expandedDim : indices) {
+      // reassocGroupSizes and reassocGroupOffsets can be obtained directly
+      // from the expanded state, but the collapsed size requires calculation
+      // as it did not previously exist.
+      reassocGroupSizes.push_back(expandedShape[expandedDim]);
+      reassocGroupOffsets.push_back(expandedOffsets[expandedDim]);
+      collapsedSize = mul(collapsedSize, expandedSizes[expandedDim]);
+    }
+
+    SmallVector<Value> offsetVals =
+        llvm::map_to_vector(reassocGroupOffsets, [&](OpFoldResult ofr) {
+          return getValueOrCreateConstantIndexOp(b, loc, ofr);
+        });
+    OpFoldResult collapsedOffset = affine::AffineLinearizeIndexOp::create(
+                                       b, loc, offsetVals, reassocGroupSizes,
+                                       /*disjoint=*/true)
+                                       .getResult();
+    collapsedOffsets.push_back(collapsedOffset);
+    collapsedSizes.push_back(collapsedSize);
+
+    // Only unit stride is supported.
+    collapsedStrides.push_back(b.getIndexAttr(1));
+  }
+  return success();
+}
+
+LogicalResult mlir::tensor::getExpandedExtractSliceInfo(
+    OpBuilder &b, tensor::ExtractSliceOp sliceOp,
+    ArrayRef<ReassociationIndices> reassociation,
+    ArrayRef<int64_t> expandedShape,
+    SmallVectorImpl<OpFoldResult> &expandedOffsets,
+    SmallVectorImpl<OpFoldResult> &expandedSizes,
+    SmallVectorImpl<OpFoldResult> &expandedStrides) {
+  if (!sliceOp.hasUnitStride()) {
+    return failure();
+  }
+
+  // The tensor.extract_slice before applying the pattern works on the result
+  // of the tensor.collapse_shape, so variables (i.e. inputs for
+  // ExtractSliceOp) referring to the state before applying the pattern are
+  // named with the prefix "collapsed", and ones referring to the state after
+  // applying the pattern are named with the prefix "expanded".
+  SmallVector<OpFoldResult> collapsedOffsets = sliceOp.getMixedOffsets();
+  SmallVector<OpFoldResult> collapsedSizes = sliceOp.getMixedSizes();
+  if (static_cast<size_t>(sliceOp.getResultType().getRank()) !=
+      collapsedSizes.size()) {
+    return failure();
+  }
 
-      // Now handle the first dim where slicing occurs on (k).
-      if (idx < reassocGroupSize) {
-        int64_t expandedShapeSize = srcShape[reversedReassocIndices[idx]];
-        int64_t offsetInDim = currentCollapsedOffset % expandedShapeSize;
-        // We need to make sure that the slice size in this dim + offset will
-        // not exceed the shape size.
-        if ((currentCollapsedsize + offsetInDim) >= expandedShapeSize) {
-          return rewriter.notifyMatchFailure(
-              sliceOp, "unsupported: slice cannot be extracted as a contiguous "
-                       "slice of the src of the collapse_shape");
+  // Compute new offsets, sizes, and strides for tensor.extract_slice.
+  // The new tensor.extract_slice will work on a tensor that has has a rank
+  // equal to the rank of the src of the collapse_shape. In each iteration of
+  // the loop, the offsets and sizes will be computed per reassociation group.
+  expandedStrides.resize(expandedShape.size(), b.getIndexAttr(1));
+  for (auto [collapsedSize, collapsedOffset, reassocIndices] :
+       llvm::zip_equal(collapsedSizes, collapsedOffsets, reassociation)) {
+    // CASE #1 - size and/or offset are dynamic.
+    // In this case, the slice can be represented as a contiguous slice only
+    // if there is a single dimension in the reassociation group that has a
+    // size not equal to 1.
+    if (isa<Value>(collapsedSize) || isa<Value>(collapsedOffset)) {
+      int nonUnitSizeCount = 0;
+      for (int64_t expandedShapeIdx : reassocIndices) {
+        if (expandedShape[expandedShapeIdx] != 1) {
+          nonUnitSizeCount++;
+          expandedSizes.push_back(collapsedSize);
+          expandedOffsets.push_back(collapsedOffset);
+          continue;
         }
 
-        groupExpandedSizes.push_back(
-            rewriter.getIndexAttr(currentCollapsedsize));
-        groupExpandedOffsets.push_back(rewriter.getIndexAttr(offsetInDim));
+        expandedSizes.push_back(b.getIndexAttr(1));
+        expandedOffsets.push_back(b.getIndexAttr(0));
+      }
 
-        currentCollapsedOffset /= expandedShapeSize;
+      if (nonUnitSizeCount != 1) {
+        return failure();
       }
+      continue;
+    }
 
-      // Now handle the leading dimensions where the slice size is equal to 1
-      // (k-1...0).
-      // The size for these dimensions must be 1 because of how we constructed
-      // the slice size of the expanded shape. We spread the original collapsed
-      // size over the expanded shape sizes until we reached dimension k where
-      // the remaining size was smaller than the expanded shape size, and spread
-      // the remaining size on it. So, now we are left with only 1s.
-      for (idx++; idx < reassocGroupSize; ++idx) {
-        int64_t expandedShapeSize = srcShape[reversedReassocIndices[idx]];
-        int64_t offsetInDim = currentCollapsedOffset % expandedShapeSize;
-        groupExpandedSizes.push_back(rewriter.getIndexAttr(1));
-        groupExpandedOffsets.push_back(rewriter.getIndexAttr(offsetInDim));
-        currentCollapsedOffset /= expandedShapeSize;
+    // CASE #2 = size and offset are static.
+    // Verify that the slice can be represented as a contiguous slice of the
+    // src of the collapse_shape.
+    // Checking this is done on order of most internal dimensions first,
+    // so traversal is done in reverse order of the reassociation group.
+    // If the expected slice shape is [1, 1, ..., 1, Sk, Ak + 1, Ak + 2,
+    // ...,An] then we first find the size and offset for n...k+1 then for k
+    // and then for k-1...0.
+
+    // currentCollapsedsize and currentCollapsedOffset are initialized with
+    // the original collapsed size and offset and divided by the expanded
+    // shape size in each dimension as we go along the reassociation group.
+    // In essence we are spreading the original collapsed size and offset over
+    // the various expanded slice dimensions.
+    // The variables are used both to check the validity of the slice and to
+    // compute the expanded sizes and offsets.
+    int64_t currentCollapsedsize = getConstantIntValue(collapsedSize).value();
+    int64_t currentCollapsedOffset =
+        getConstantIntValue(collapsedOffset).value();
+    SmallVector<OpFoldResult> groupExpandedSizes, groupExpandedOffsets;
+    ReassociationIndices reversedReassocIndices(reassocIndices.rbegin(),
+                                                reassocIndices.rend());
+    int64_t idx = 0;
+    int64_t reassocGroupSize = reassocIndices.size();
+
+    // First handle the trailing dimensions where the slice size should be
+    // equal to the tensor shape and the offset should be 0 (n...k+1).
+    for (; idx < reassocGroupSize; ++idx) {
+      int64_t expandedShapeSize = expandedShape[reversedReassocIndices[idx]];
+
+      if (currentCollapsedsize < expandedShapeSize)
+        break;
+
+      // We need to make sure that the slice size can be set to the shape size
+      // and the offset to 0.
+      if ((currentCollapsedsize % expandedShapeSize) != 0 ||
+          (currentCollapsedOffset % expandedShapeSize) != 0) {
+        return failure();
       }
 
-      expandedSizes.append(groupExpandedSizes.rbegin(),
-                           groupExpandedSizes.rend());
-      expandedOffsets.append(groupExpandedOffsets.rbegin(),
-                             groupExpandedOffsets.rend());
+      groupExpandedSizes.push_back(b.getIndexAttr(expandedShapeSize));
+      groupExpandedOffsets.push_back(b.getIndexAttr(0));
+
+      currentCollapsedsize /= expandedShapeSize;
+      currentCollapsedOffset /= expandedShapeSize;
     }
 
-    Value newSliceOp = tensor::ExtractSliceOp::create(
-        rewriter, collapseShapeOp->getLoc(), collapseShapeOp.getSrc(),
-        expandedOffsets, expandedSizes, expandedStrides);
-    rewriter.replaceOpWithNewOp<tensor::CollapseShapeOp>(
-        sliceOp, sliceOp.getResultType(), newSliceOp,
-        collapseShapeOp.getReassociationIndices());
+    // Now handle the first dim where slicing occurs on (k).
+    if (idx < reassocGroupSize) {
+      int64_t expandedShapeSize = expandedShape[reversedReassocIndices[idx]];
+      int64_t offsetInDim = currentCollapsedOffset % expandedShapeSize;
+      // We need to make sure that the slice size in this dim + offset will
+      // not exceed the shape size.
+      if ((currentCollapsedsize + offsetInDim) >= expandedShapeSize) {
+        return failure();
+      }
+      groupExpandedSizes.push_back(b.getIndexAttr(currentCollapsedsize));
+      groupExpandedOffsets.push_back(b.getIndexAttr(offsetInDim));
+      currentCollapsedOffset /= expandedShapeSize;
+    }
 
-    return success();
+    // Now handle the leading dimensions where the slice size is equal to 1
+    // (k-1...0).
+    // The size for these dimensions must be 1 because of how we constructed
+    // the slice size of the expanded shape. We spread the original collapsed
+    // size over the expanded shape sizes until we reached dimension k where
+    // the remaining size was smaller than the expanded shape size, and spread
+    // the remaining size on it. So, now we are left with only 1s.
+    for (idx++; idx < reassocGroupSize; ++idx) {
+      int64_t expandedShapeSize = expandedShape[reversedReassocIndices[idx]];
+      int64_t offsetInDim = currentCollapsedOffset % expandedShapeSize;
+      groupExpandedSizes.push_back(b.getIndexAttr(1));
+      groupExpandedOffsets.push_back(b.getIndexAttr(offsetInDim));
+      currentCollapsedOffset /= expandedShapeSize;
+    }
+    expandedSizes.append(groupExpandedSizes.rbegin(),
+                         groupExpandedSizes.rend());
+    expandedOffsets.append(groupExpandedOffsets.rbegin(),
+                           groupExpandedOffsets.rend());
   }
-};
-
-} // namespace
+  return success();
+}
 
 void mlir::tensor::populateReassociativeReshapeFoldingPatterns(
     RewritePatternSet &patterns) {
diff --git a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
index e3cba38871909..fce61f27ca3ea 100644
--- a/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
+++ b/mlir/lib/Dialect/Tosa/IR/TosaCanonicalizations.cpp
@@ -1120,13 +1120,14 @@ OpFoldResult MulOp::fold(FoldAdaptor adaptor) {
   }
 
   if (rhsTy == resultTy) {
-    if (isSplatZero(resultETy, lhsAttr))
+    if (isSplatZero(resultETy, lhsAttr) && resultTy.hasStaticShape())
+      // constant values can only be resized if resulting type is static
       return lhsAttr.resizeSplat(resultTy);
     if (isSplatOne(resultETy, lhsAttr, shift))
       return rhs;
   }
   if (lhsTy == resultTy) {
-    if (isSplatZero(resultETy, rhsAttr))
+    if (isSplatZero(resultETy, rhsAttr) && resultTy.hasStaticShape())
       return rhsAttr.resizeSplat(resultTy);
     if (isSplatOne(resultETy, rhsAttr, shift))
       return lhs;
diff --git a/mlir/lib/Dialect/Transform/IR/Utils.cpp b/mlir/lib/Dialect/Transform/IR/Utils.cpp
index d66639042295a..773eb138cdf62 100644
--- a/mlir/lib/Dialect/Transform/IR/Utils.cpp
+++ b/mlir/lib/Dialect/Transform/IR/Utils.cpp
@@ -11,6 +11,7 @@
 #include "mlir/IR/Verifier.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 
 using namespace mlir;
 
@@ -90,7 +91,7 @@ transform::detail::mergeSymbolsInto(Operation *target,
   //
   // Rename private symbols in both ops in order to resolve conflicts that can
   // be resolved that way.
-  LLVM_DEBUG(DBGS() << "renaming private symbols to resolve conflicts:\n");
+  LDBG() << "renaming private symbols to resolve conflicts:";
   // TODO: Do we *actually* need to test in both directions?
   for (auto &&[symbolTable, otherSymbolTable] : llvm::zip(
            SmallVector<SymbolTable *, 2>{&targetSymbolTable, &otherSymbolTable},
@@ -102,7 +103,7 @@ transform::detail::mergeSymbolsInto(Operation *target,
       if (!symbolOp)
         continue;
       StringAttr name = symbolOp.getNameAttr();
-      LLVM_DEBUG(DBGS() << "  found @" << name.getValue() << "\n");
+      LDBG() << "  found @" << name.getValue();
 
       // Check if there is a colliding op in the other module.
       auto collidingOp =
@@ -110,7 +111,7 @@ transform::detail::mergeSymbolsInto(Operation *target,
       if (!collidingOp)
         continue;
 
-      LLVM_DEBUG(DBGS() << "    collision found for @" << name.getValue());
+      LDBG() << "    collision found for @" << name.getValue();
 
       // Collisions are fine if both opt are functions and can be merged.
       if (auto funcOp = dyn_cast<FunctionOpInterface>(op),
@@ -119,13 +120,12 @@ transform::detail::mergeSymbolsInto(Operation *target,
           funcOp && collidingFuncOp) {
         if (canMergeInto(funcOp, collidingFuncOp) ||
             canMergeInto(collidingFuncOp, funcOp)) {
-          LLVM_DEBUG(llvm::dbgs() << " but both ops are functions and "
-                                     "will be merged\n");
+          LDBG() << " but both ops are functions and will be merged";
           continue;
         }
 
         // If they can't be merged, proceed like any other collision.
-        LLVM_DEBUG(llvm::dbgs() << " and both ops are function definitions");
+        LDBG() << " and both ops are function definitions";
       }
 
       // Collision can be resolved by renaming if one of the ops is private.
@@ -133,7 +133,7 @@ transform::detail::mergeSymbolsInto(Operation *target,
           [&](SymbolOpInterface op, SymbolOpInterface otherOp,
               SymbolTable &symbolTable,
               SymbolTable &otherSymbolTable) -> InFlightDiagnostic {
-        LLVM_DEBUG(llvm::dbgs() << ", renaming\n");
+        LDBG() << ", renaming";
         FailureOr<StringAttr> maybeNewName =
             symbolTable.renameToUnique(op, {&otherSymbolTable});
         if (failed(maybeNewName)) {
@@ -142,8 +142,7 @@ transform::detail::mergeSymbolsInto(Operation *target,
               << "attempted renaming due to collision with this op";
           return diag;
         }
-        LLVM_DEBUG(DBGS() << "      renamed to @" << maybeNewName->getValue()
-                          << "\n");
+        LDBG() << "      renamed to @" << maybeNewName->getValue();
         return InFlightDiagnostic();
       };
 
@@ -161,7 +160,7 @@ transform::detail::mergeSymbolsInto(Operation *target,
           return diag;
         continue;
       }
-      LLVM_DEBUG(llvm::dbgs() << ", emitting error\n");
+      LDBG() << ", emitting error";
       InFlightDiagnostic diag = symbolOp.emitError()
                                 << "doubly defined symbol @" << name.getValue();
       diag.attachNote(collidingOp->getLoc()) << "previously defined here";
@@ -179,7 +178,7 @@ transform::detail::mergeSymbolsInto(Operation *target,
   // Step 2:
   //
   // Move all ops from `other` into target and merge public symbols.
-  LLVM_DEBUG(DBGS() << "moving all symbols into target\n");
+  LDBG() << "moving all symbols into target";
   {
     SmallVector<SymbolOpInterface> opsToMove;
     for (Operation &op : other->getRegion(0).front()) {
@@ -193,13 +192,13 @@ transform::detail::mergeSymbolsInto(Operation *target,
           targetSymbolTable.lookup(op.getNameAttr()));
 
       // Move op even if we get a collision.
-      LLVM_DEBUG(DBGS() << "  moving @" << op.getName());
+      LDBG() << "  moving @" << op.getName();
       op->moveBefore(&target->getRegion(0).front(),
                      target->getRegion(0).front().end());
 
       // If there is no collision, we are done.
       if (!collidingOp) {
-        LLVM_DEBUG(llvm::dbgs() << " without collision\n");
+        LDBG() << " without collision";
         continue;
       }
 
@@ -217,9 +216,9 @@ transform::detail::mergeSymbolsInto(Operation *target,
       }
       assert(canMergeInto(funcOp, collidingFuncOp));
 
-      LLVM_DEBUG(llvm::dbgs() << " with collision, trying to keep op at "
-                              << collidingFuncOp.getLoc() << ":\n"
-                              << collidingFuncOp << "\n");
+      LDBG() << " with collision, trying to keep op at "
+             << collidingFuncOp.getLoc() << ":\n"
+             << collidingFuncOp;
 
       // Update symbol table. This works with or without the previous `swap`.
       targetSymbolTable.remove(funcOp);
@@ -239,6 +238,6 @@ transform::detail::mergeSymbolsInto(Operation *target,
     return target->emitError()
            << "failed to verify target op after merging symbols";
 
-  LLVM_DEBUG(DBGS() << "done merging ops\n");
+  LDBG() << "done merging ops";
   return InFlightDiagnostic();
 }
diff --git a/mlir/lib/Dialect/Utils/StaticValueUtils.cpp b/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
index e6ef0282101d2..34385d76f133a 100644
--- a/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
+++ b/mlir/lib/Dialect/Utils/StaticValueUtils.cpp
@@ -276,7 +276,7 @@ std::optional<int64_t> constantTripCount(OpFoldResult lb, OpFoldResult ub,
   if (!ubConstant)
     return std::nullopt;
   std::optional<int64_t> stepConstant = getConstantIntValue(step);
-  if (!stepConstant)
+  if (!stepConstant || *stepConstant == 0)
     return std::nullopt;
 
   return llvm::divideCeilSigned(*ubConstant - *lbConstant, *stepConstant);
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index a450056a3041a..2b2581d353673 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -2402,6 +2402,16 @@ LogicalResult ToElementsOp::fold(FoldAdaptor adaptor,
   return foldToElementsFromElements(*this, results);
 }
 
+LogicalResult
+ToElementsOp::inferReturnTypes(MLIRContext *ctx, std::optional<Location> loc,
+                               ToElementsOp::Adaptor adaptor,
+                               SmallVectorImpl<Type> &inferredReturnTypes) {
+  auto vecType = cast<VectorType>(adaptor.getSource().getType());
+  Type elType = vecType.getElementType();
+  inferredReturnTypes.append(vecType.getNumElements(), elType);
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // FromElementsOp
 //===----------------------------------------------------------------------===//
@@ -2841,9 +2851,47 @@ LogicalResult BroadcastOp::verify() {
   llvm_unreachable("unexpected vector.broadcast op error");
 }
 
+// Fold broadcast(shape_cast(x)) into broadcast(x) if x's type is compatible
+// with broadcast's result type and shape_cast only adds or removes ones in the
+// leading dimensions.
+static LogicalResult foldBroadcastOfShapeCast(BroadcastOp broadcastOp) {
+  auto srcShapeCast = broadcastOp.getSource().getDefiningOp<ShapeCastOp>();
+  if (!srcShapeCast)
+    return failure();
+
+  VectorType srcType = srcShapeCast.getSourceVectorType();
+  VectorType destType = broadcastOp.getResultVectorType();
+  // Check type compatibility.
+  if (vector::isBroadcastableTo(srcType, destType) !=
+      BroadcastableToResult::Success)
+    return failure();
+
+  ArrayRef<int64_t> srcShape = srcType.getShape();
+  ArrayRef<int64_t> shapecastShape =
+      srcShapeCast.getResultVectorType().getShape();
+  // Trailing dimensions should be the same if shape_cast only alters the
+  // leading dimensions.
+  unsigned numTrailingDims = std::min(srcShape.size(), shapecastShape.size());
+  if (!llvm::equal(srcShape.take_back(numTrailingDims),
+                   shapecastShape.take_back(numTrailingDims)))
+    return failure();
+
+  assert(all_of(srcShape.drop_back(numTrailingDims),
+                [](int64_t E) { return E == 1; }) &&
+         all_of(shapecastShape.drop_back(numTrailingDims),
+                [](int64_t E) { return E == 1; }) &&
+         "ill-formed shape_cast");
+
+  broadcastOp.getSourceMutable().assign(srcShapeCast.getSource());
+  return success();
+}
+
 OpFoldResult BroadcastOp::fold(FoldAdaptor adaptor) {
   if (getSourceType() == getResultVectorType())
     return getSource();
+  if (succeeded(foldBroadcastOfShapeCast(*this)))
+    return getResult();
+
   if (!adaptor.getSource())
     return {};
   auto vectorType = getResultVectorType();
@@ -3238,6 +3286,18 @@ LogicalResult InsertOp::verify() {
   return success();
 }
 
+// Calculate the linearized position of the continuous chunk of elements to
+// insert, based on the shape of the value to insert and the positions to insert
+// at.
+static int64_t calculateInsertPosition(VectorType destTy,
+                                       ArrayRef<int64_t> positions) {
+  llvm::SmallVector<int64_t> completePositions(destTy.getRank(), 0);
+  assert(positions.size() <= completePositions.size() &&
+         "positions size must be less than or equal to destTy rank");
+  copy(positions, completePositions.begin());
+  return linearize(completePositions, computeStrides(destTy.getShape()));
+}
+
 namespace {
 
 // If insertOp is only inserting unit dimensions it can be transformed to a
@@ -3275,6 +3335,132 @@ class InsertSplatToSplat final : public OpRewritePattern<InsertOp> {
     return success();
   }
 };
+
+/// Pattern to optimize a chain of insertions.
+///
+/// This pattern identifies chains of vector.insert operations that:
+/// 1. Only insert values at static positions.
+/// 2. Completely initialize all elements in the resulting vector.
+/// 3. All intermediate insert operations have only one use.
+///
+/// When these conditions are met, the entire chain can be replaced with a
+/// single vector.from_elements operation.
+///
+/// To keep this pattern simple, and avoid spending too much time on matching
+/// fragmented insert chains, this pattern only considers the last insert op in
+/// the chain.
+///
+/// Example transformation:
+///   %poison = ub.poison : vector<2xi32>
+///   %0 = vector.insert %c1, %poison[0] : i32 into vector<2xi32>
+///   %1 = vector.insert %c2, %0[1] : i32 into vector<2xi32>
+/// ->
+///   %result = vector.from_elements %c1, %c2 : vector<2xi32>
+class InsertChainFullyInitialized final : public OpRewritePattern<InsertOp> {
+public:
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(InsertOp op,
+                                PatternRewriter &rewriter) const override {
+
+    VectorType destTy = op.getDestVectorType();
+    if (destTy.isScalable())
+      return failure();
+    // Ensure this is the trailing vector.insert op in a chain of inserts.
+    for (Operation *user : op.getResult().getUsers())
+      if (auto insertOp = dyn_cast<InsertOp>(user))
+        if (insertOp.getDest() == op.getResult())
+          return failure();
+
+    InsertOp currentOp = op;
+    SmallVector<InsertOp> chainInsertOps;
+    while (currentOp) {
+      // Check cond 1: Dynamic position is not supported.
+      if (currentOp.hasDynamicPosition())
+        return failure();
+
+      chainInsertOps.push_back(currentOp);
+      currentOp = currentOp.getDest().getDefiningOp<InsertOp>();
+      // Check cond 3: Intermediate inserts have only one use to avoid an
+      // explosion of vectors.
+      if (currentOp && !currentOp->hasOneUse())
+        return failure();
+    }
+
+    int64_t vectorSize = destTy.getNumElements();
+    int64_t initializedCount = 0;
+    SmallVector<bool> initializedDestIdxs(vectorSize, false);
+    SmallVector<int64_t> pendingInsertPos;
+    SmallVector<int64_t> pendingInsertSize;
+    SmallVector<Value> pendingInsertValues;
+
+    for (auto insertOp : chainInsertOps) {
+      // This pattern can do nothing with poison index.
+      if (is_contained(insertOp.getStaticPosition(), InsertOp::kPoisonIndex))
+        return failure();
+
+      // Calculate the linearized position for inserting elements.
+      int64_t insertBeginPosition =
+          calculateInsertPosition(destTy, insertOp.getStaticPosition());
+
+      // The valueToStore operand may be a vector or a scalar. Need to handle
+      // both cases.
+      int64_t insertSize = 1;
+      if (auto srcVectorType =
+              llvm::dyn_cast<VectorType>(insertOp.getValueToStoreType()))
+        insertSize = srcVectorType.getNumElements();
+
+      assert(insertBeginPosition + insertSize <= vectorSize &&
+             "insert would overflow the vector");
+
+      for (auto index : llvm::seq<int64_t>(insertBeginPosition,
+                                           insertBeginPosition + insertSize)) {
+        if (initializedDestIdxs[index])
+          continue;
+        initializedDestIdxs[index] = true;
+        ++initializedCount;
+      }
+
+      // Defer the creation of ops before we can make sure the pattern can
+      // succeed.
+      pendingInsertPos.push_back(insertBeginPosition);
+      pendingInsertSize.push_back(insertSize);
+      pendingInsertValues.push_back(insertOp.getValueToStore());
+
+      if (initializedCount == vectorSize)
+        break;
+    }
+
+    // Check cond 2: all positions must be initialized.
+    if (initializedCount != vectorSize)
+      return failure();
+
+    SmallVector<Value> elements(vectorSize);
+    for (auto [insertBeginPosition, insertSize, valueToStore] :
+         llvm::reverse(llvm::zip(pendingInsertPos, pendingInsertSize,
+                                 pendingInsertValues))) {
+      auto srcVectorType = llvm::dyn_cast<VectorType>(valueToStore.getType());
+
+      if (!srcVectorType) {
+        elements[insertBeginPosition] = valueToStore;
+        continue;
+      }
+
+      SmallVector<Type> elementToInsertTypes(insertSize,
+                                             srcVectorType.getElementType());
+      // Get all elements from the vector in row-major order.
+      auto elementsToInsert = rewriter.create<vector::ToElementsOp>(
+          op.getLoc(), elementToInsertTypes, valueToStore);
+      for (int64_t linearIdx = 0; linearIdx < insertSize; linearIdx++) {
+        elements[insertBeginPosition + linearIdx] =
+            elementsToInsert.getResult(linearIdx);
+      }
+    }
+
+    rewriter.replaceOpWithNewOp<vector::FromElementsOp>(op, destTy, elements);
+    return success();
+  }
+};
+
 } // namespace
 
 static Attribute
@@ -3301,13 +3487,9 @@ foldDenseElementsAttrDestInsertOp(InsertOp insertOp, Attribute srcAttr,
       !insertOp->hasOneUse())
     return {};
 
-  // Calculate the linearized position of the continuous chunk of elements to
-  // insert.
-  llvm::SmallVector<int64_t> completePositions(destTy.getRank(), 0);
-  copy(insertOp.getStaticPosition(), completePositions.begin());
+  // Calculate the linearized position for inserting elements.
   int64_t insertBeginPosition =
-      linearize(completePositions, computeStrides(destTy.getShape()));
-
+      calculateInsertPosition(destTy, insertOp.getStaticPosition());
   SmallVector<Attribute> insertedValues;
   Type destEltType = destTy.getElementType();
 
@@ -3343,7 +3525,8 @@ static Value foldInsertUseChain(InsertOp insertOp) {
 
 void InsertOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                            MLIRContext *context) {
-  results.add<InsertToBroadcast, BroadcastFolder, InsertSplatToSplat>(context);
+  results.add<InsertToBroadcast, BroadcastFolder, InsertSplatToSplat,
+              InsertChainFullyInitialized>(context);
 }
 
 OpFoldResult InsertOp::fold(FoldAdaptor adaptor) {
diff --git a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
index 2d5cc070558c3..fe066dc04ad55 100644
--- a/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
+++ b/mlir/lib/Dialect/Vector/TransformOps/VectorTransformOps.cpp
@@ -139,6 +139,11 @@ void transform::ApplyLowerGatherPatternsOp::populatePatterns(
   vector::populateVectorGatherLoweringPatterns(patterns);
 }
 
+void transform::ApplyUnrollFromElementsPatternsOp::populatePatterns(
+    RewritePatternSet &patterns) {
+  vector::populateVectorFromElementsLoweringPatterns(patterns);
+}
+
 void transform::ApplyLowerScanPatternsOp::populatePatterns(
     RewritePatternSet &patterns) {
   vector::populateVectorScanLoweringPatterns(patterns);
diff --git a/mlir/lib/Dialect/Vector/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Vector/Transforms/CMakeLists.txt
index 9e287fc109990..acbf2b746037b 100644
--- a/mlir/lib/Dialect/Vector/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Vector/Transforms/CMakeLists.txt
@@ -3,6 +3,7 @@ add_mlir_dialect_library(MLIRVectorTransforms
   LowerVectorBitCast.cpp
   LowerVectorBroadcast.cpp
   LowerVectorContract.cpp
+  LowerVectorFromElements.cpp
   LowerVectorGather.cpp
   LowerVectorInterleave.cpp
   LowerVectorMask.cpp
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorFromElements.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorFromElements.cpp
new file mode 100644
index 0000000000000..c22fd54cef46b
--- /dev/null
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorFromElements.cpp
@@ -0,0 +1,65 @@
+//===- LowerVectorFromElements.cpp - Lower 'vector.from_elements' op -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements target-independent rewrites and utilities to lower the
+// 'vector.from_elements' operation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
+
+#define DEBUG_TYPE "lower-vector-from-elements"
+
+using namespace mlir;
+
+namespace {
+
+/// Unrolls 2 or more dimensional `vector.from_elements` ops by unrolling the
+/// outermost dimension. For example:
+/// ```
+/// %v = vector.from_elements %e0, %e1, %e2, %e3, %e4, %e5 : vector<2x3xf32>
+///
+/// ==>
+///
+/// %0   = ub.poison : vector<2x3xf32>
+/// %v0  = vector.from_elements %e0, %e1, %e2 : vector<3xf32>
+/// %1   = vector.insert %v0, %0 [0] : vector<3xf32> into vector<2x3xf32>
+/// %v1  = vector.from_elements %e3, %e4, %e5 : vector<3xf32>
+/// %v   = vector.insert %v1, %1 [1] : vector<3xf32> into vector<2x3xf32>
+/// ```
+///
+/// When applied exhaustively, this will produce a sequence of 1-d from_elements
+/// ops.
+struct UnrollFromElements : OpRewritePattern<vector::FromElementsOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(vector::FromElementsOp op,
+                                PatternRewriter &rewriter) const override {
+    ValueRange allElements = op.getElements();
+
+    auto unrollFromElementsFn = [&](PatternRewriter &rewriter, Location loc,
+                                    VectorType subTy, int64_t index) {
+      size_t subTyNumElements = subTy.getNumElements();
+      assert((index + 1) * subTyNumElements <= allElements.size() &&
+             "out of bounds");
+      ValueRange subElements =
+          allElements.slice(index * subTyNumElements, subTyNumElements);
+      return vector::FromElementsOp::create(rewriter, loc, subTy, subElements);
+    };
+
+    return unrollVectorOp(op, rewriter, unrollFromElementsFn);
+  }
+};
+
+} // namespace
+
+void mlir::vector::populateVectorFromElementsLoweringPatterns(
+    RewritePatternSet &patterns, PatternBenefit benefit) {
+  patterns.add<UnrollFromElements>(patterns.getContext(), benefit);
+}
diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
index e062f55f87679..90f21c53246b0 100644
--- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorGather.cpp
@@ -54,27 +54,13 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
 
   LogicalResult matchAndRewrite(vector::GatherOp op,
                                 PatternRewriter &rewriter) const override {
-    VectorType resultTy = op.getType();
-    if (resultTy.getRank() < 2)
-      return rewriter.notifyMatchFailure(op, "already 1-D");
-
-    // Unrolling doesn't take vscale into account. Pattern is disabled for
-    // vectors with leading scalable dim(s).
-    if (resultTy.getScalableDims().front())
-      return rewriter.notifyMatchFailure(op, "cannot unroll scalable dim");
-
-    Location loc = op.getLoc();
     Value indexVec = op.getIndexVec();
     Value maskVec = op.getMask();
     Value passThruVec = op.getPassThru();
 
-    Value result = arith::ConstantOp::create(rewriter, loc, resultTy,
-                                             rewriter.getZeroAttr(resultTy));
-
-    VectorType subTy = VectorType::Builder(resultTy).dropDim(0);
-
-    for (int64_t i = 0, e = resultTy.getShape().front(); i < e; ++i) {
-      int64_t thisIdx[1] = {i};
+    auto unrollGatherFn = [&](PatternRewriter &rewriter, Location loc,
+                              VectorType subTy, int64_t index) {
+      int64_t thisIdx[1] = {index};
 
       Value indexSubVec =
           vector::ExtractOp::create(rewriter, loc, indexVec, thisIdx);
@@ -82,15 +68,12 @@ struct UnrollGather : OpRewritePattern<vector::GatherOp> {
           vector::ExtractOp::create(rewriter, loc, maskVec, thisIdx);
       Value passThruSubVec =
           vector::ExtractOp::create(rewriter, loc, passThruVec, thisIdx);
-      Value subGather = vector::GatherOp::create(
-          rewriter, loc, subTy, op.getBase(), op.getIndices(), indexSubVec,
-          maskSubVec, passThruSubVec);
-      result =
-          vector::InsertOp::create(rewriter, loc, subGather, result, thisIdx);
-    }
+      return vector::GatherOp::create(rewriter, loc, subTy, op.getBase(),
+                                      op.getIndices(), indexSubVec, maskSubVec,
+                                      passThruSubVec);
+    };
 
-    rewriter.replaceOp(op, result);
-    return success();
+    return unrollVectorOp(op, rewriter, unrollGatherFn);
   }
 };
 
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
index bb0f339a26e43..60aa0e9bae64a 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorDistribute.cpp
@@ -528,8 +528,7 @@ struct WarpOpTransferWrite : public WarpDistributionPattern {
 
   LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
                                 PatternRewriter &rewriter) const override {
-    auto yield = cast<gpu::YieldOp>(
-        warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
+    gpu::YieldOp yield = warpOp.getTerminator();
     Operation *lastNode = yield->getPrevNode();
     auto writeOp = dyn_cast_or_null<vector::TransferWriteOp>(lastNode);
     if (!writeOp)
@@ -846,8 +845,7 @@ struct WarpOpDeadResult : public WarpDistributionPattern {
     newYieldValues.reserve(warpOp->getNumResults());
     DenseMap<Value, int64_t> dedupYieldOperandPositionMap;
     DenseMap<OpResult, int64_t> dedupResultPositionMap;
-    auto yield = cast<gpu::YieldOp>(
-        warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
+    gpu::YieldOp yield = warpOp.getTerminator();
 
     // Some values may be yielded multiple times and correspond to multiple
     // results. Deduplicating occurs by taking each result with its matching
@@ -901,8 +899,7 @@ struct WarpOpForwardOperand : public WarpDistributionPattern {
   using Base::Base;
   LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
                                 PatternRewriter &rewriter) const override {
-    auto yield = cast<gpu::YieldOp>(
-        warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
+    gpu::YieldOp yield = warpOp.getTerminator();
     Value valForwarded;
     unsigned resultIndex;
     for (OpOperand &operand : yield->getOpOperands()) {
@@ -1708,8 +1705,7 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
       : WarpDistributionPattern(ctx, b), distributionMapFn(std::move(fn)) {}
   LogicalResult matchAndRewrite(WarpExecuteOnLane0Op warpOp,
                                 PatternRewriter &rewriter) const override {
-    auto warpOpYield = cast<gpu::YieldOp>(
-        warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
+    gpu::YieldOp warpOpYield = warpOp.getTerminator();
     // Only pick up `ForOp` if it is the last op in the region.
     Operation *lastNode = warpOpYield->getPrevNode();
     auto forOp = dyn_cast_or_null<scf::ForOp>(lastNode);
@@ -1826,7 +1822,8 @@ struct WarpOpScfForOp : public WarpDistributionPattern {
     rewriter.setInsertionPointAfter(newWarpOp);
     auto newForOp = scf::ForOp::create(
         rewriter, forOp.getLoc(), forOp.getLowerBound(), forOp.getUpperBound(),
-        forOp.getStep(), newForOpOperands);
+        forOp.getStep(), newForOpOperands, /*bodyBuilder=*/nullptr,
+        forOp.getUnsignedCmp());
     // Next, we insert a new `WarpOp` (called inner `WarpOp`) inside the
     // newly created `ForOp`. This `WarpOp` will contain all ops that were
     // contained within the original `ForOp` body.
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index c707f38d9081c..369857fac7a06 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -98,8 +98,9 @@ void TransferOptimization::deadStoreOp(vector::TransferWriteOp write) {
     // If the user has already been processed skip.
     if (!processed.insert(user).second)
       continue;
-    if (isa<ViewLikeOpInterface>(user)) {
-      users.append(user->getUsers().begin(), user->getUsers().end());
+    if (auto viewLike = dyn_cast<ViewLikeOpInterface>(user)) {
+      Value viewDest = viewLike.getViewDest();
+      users.append(viewDest.getUsers().begin(), viewDest.getUsers().end());
       continue;
     }
     if (isMemoryEffectFree(user))
@@ -182,8 +183,9 @@ void TransferOptimization::storeToLoadForwarding(vector::TransferReadOp read) {
     // If the user has already been processed skip.
     if (!processed.insert(user).second)
       continue;
-    if (isa<ViewLikeOpInterface>(user)) {
-      users.append(user->getUsers().begin(), user->getUsers().end());
+    if (auto viewLike = dyn_cast<ViewLikeOpInterface>(user)) {
+      Value viewDest = viewLike.getViewDest();
+      users.append(viewDest.getUsers().begin(), viewDest.getUsers().end());
       continue;
     }
     if (isMemoryEffectFree(user) || isa<vector::TransferReadOp>(user))
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index 2269a40ec8ef1..023c4da7dffdf 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -2274,7 +2274,7 @@ struct FoldArithToVectorOuterProduct : public OpRewritePattern<MulOpType> {
 
   LogicalResult matchAndRewrite(MulOpType mulOp,
                                 PatternRewriter &rewriter) const override {
-    auto resType = llvm::cast<VectorType>(mulOp.getResult().getType());
+    auto resType = llvm::dyn_cast<VectorType>(mulOp.getResult().getType());
     if (!resType)
       return failure();
     if (resType.getRank() != 2)
diff --git a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
index 10ed2bcfb35a3..841e1384e03b3 100644
--- a/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
+++ b/mlir/lib/Dialect/Vector/Utils/VectorUtils.cpp
@@ -279,14 +279,16 @@ vector::createUnrollIterator(VectorType vType, int64_t targetRank) {
   // Attempt to unroll until targetRank or the first scalable dimension (which
   // cannot be unrolled).
   auto shapeToUnroll = vType.getShape().drop_back(targetRank);
-  auto scalableDimsToUnroll = vType.getScalableDims().drop_back(targetRank);
-  auto it = llvm::find(scalableDimsToUnroll, true);
-  auto firstScalableDim = it - scalableDimsToUnroll.begin();
+  auto inputScalableVecDimsToUnroll =
+      vType.getScalableDims().drop_back(targetRank);
+  auto it = llvm::find(inputScalableVecDimsToUnroll, true);
+  auto firstScalableDim = it - inputScalableVecDimsToUnroll.begin();
   if (firstScalableDim == 0)
     return {};
   // All scalable dimensions should be removed now.
-  scalableDimsToUnroll = scalableDimsToUnroll.slice(0, firstScalableDim);
-  assert(!llvm::is_contained(scalableDimsToUnroll, true) &&
+  inputScalableVecDimsToUnroll =
+      inputScalableVecDimsToUnroll.slice(0, firstScalableDim);
+  assert(!llvm::is_contained(inputScalableVecDimsToUnroll, true) &&
          "unexpected leading scalable dimension");
   // Create an unroll iterator for leading dimensions.
   shapeToUnroll = shapeToUnroll.slice(0, firstScalableDim);
@@ -319,15 +321,15 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
                                      ArrayRef<int64_t> inputVectorSizes,
                                      Value padValue,
                                      bool useInBoundsInsteadOfMasking,
-                                     ArrayRef<bool> scalableDims) {
+                                     ArrayRef<bool> inputScalableVecDims) {
   assert(!llvm::is_contained(inputVectorSizes, ShapedType::kDynamic) &&
          "invalid input vector sizes");
   auto sourceShapedType = cast<ShapedType>(source.getType());
   auto sourceShape = sourceShapedType.getShape();
   assert(sourceShape.size() == inputVectorSizes.size() &&
          "expected same ranks.");
-  auto vectorType =
-      VectorType::get(inputVectorSizes, padValue.getType(), scalableDims);
+  auto vectorType = VectorType::get(inputVectorSizes, padValue.getType(),
+                                    inputScalableVecDims);
   assert(padValue.getType() == sourceShapedType.getElementType() &&
          "expected same pad element type to match source element type");
   int64_t readRank = inputVectorSizes.size();
@@ -356,8 +358,8 @@ Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
           ? memref::getMixedSizes(builder, loc, source)
           : tensor::getMixedSizes(builder, loc, source);
 
-  auto maskType =
-      VectorType::get(inputVectorSizes, builder.getI1Type(), scalableDims);
+  auto maskType = VectorType::get(inputVectorSizes, builder.getI1Type(),
+                                  inputScalableVecDims);
   Value mask =
       vector::CreateMaskOp::create(builder, loc, maskType, mixedSourceDims);
   return mlir::vector::maskOperation(builder, transferReadOp, mask)
@@ -385,9 +387,34 @@ vector::isValidMaskedInputVector(ArrayRef<int64_t> shape,
                              staticSize <= inputSize;
                     })) {
     LDBG() << "Input vector sizes must be greater than or equal to iteration "
-              "space "
-              "static sizes";
+              "space static sizes";
     return failure();
   }
   return success();
 }
+
+LogicalResult vector::unrollVectorOp(Operation *op, PatternRewriter &rewriter,
+                                     vector::UnrollVectorOpFn unrollFn) {
+  assert(op->getNumResults() == 1 && "expected single result");
+  assert(isa<VectorType>(op->getResult(0).getType()) && "expected vector type");
+  VectorType resultTy = cast<VectorType>(op->getResult(0).getType());
+  if (resultTy.getRank() < 2)
+    return rewriter.notifyMatchFailure(op, "already 1-D");
+
+  // Unrolling doesn't take vscale into account. Pattern is disabled for
+  // vectors with leading scalable dim(s).
+  if (resultTy.getScalableDims().front())
+    return rewriter.notifyMatchFailure(op, "cannot unroll scalable dim");
+
+  Location loc = op->getLoc();
+  Value result = ub::PoisonOp::create(rewriter, loc, resultTy);
+  VectorType subTy = VectorType::Builder(resultTy).dropDim(0);
+
+  for (int64_t i = 0, e = resultTy.getShape().front(); i < e; ++i) {
+    Value subVector = unrollFn(rewriter, loc, subTy, i);
+    result = vector::InsertOp::create(rewriter, loc, subVector, result, i);
+  }
+
+  rewriter.replaceOp(op, result);
+  return success();
+}
diff --git a/mlir/lib/Dialect/WasmSSA/IR/WasmSSAOps.cpp b/mlir/lib/Dialect/WasmSSA/IR/WasmSSAOps.cpp
index 2ce32fe895205..89b62a21c4f52 100644
--- a/mlir/lib/Dialect/WasmSSA/IR/WasmSSAOps.cpp
+++ b/mlir/lib/Dialect/WasmSSA/IR/WasmSSAOps.cpp
@@ -22,6 +22,47 @@
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
 
+using namespace mlir;
+namespace {
+ParseResult parseElseRegion(OpAsmParser &opParser, Region &elseRegion) {
+  std::string keyword;
+  std::ignore = opParser.parseOptionalKeywordOrString(&keyword);
+  if (keyword == "else")
+    return opParser.parseRegion(elseRegion);
+  return ParseResult::success();
+}
+
+void printElseRegion(OpAsmPrinter &opPrinter, Operation *op,
+                     Region &elseRegion) {
+  if (elseRegion.empty())
+    return;
+  opPrinter.printKeywordOrString("else ");
+  opPrinter.printRegion(elseRegion);
+}
+
+ParseResult parseWasmVisibility(OpAsmParser &opParser, StringAttr &visibility) {
+  std::string keyword;
+  auto initLocation = opParser.getCurrentLocation();
+  std::ignore = opParser.parseOptionalKeywordOrString(&keyword);
+  if (keyword == "nested" or keyword == "") {
+    visibility = StringAttr::get(opParser.getContext(), "nested");
+    return ParseResult::success();
+  }
+
+  if (keyword == "public" || keyword == "private") {
+    visibility = StringAttr::get(opParser.getContext(), keyword);
+    return ParseResult::success();
+  }
+  opParser.emitError(initLocation, "expecting symbol visibility");
+  return ParseResult::failure();
+}
+
+void printWasmVisibility(OpAsmPrinter &opPrinter, Operation *op,
+                         Attribute visibility) {
+  opPrinter.printKeywordOrString(cast<StringAttr>(visibility).strref());
+}
+} // namespace
+
 #define GET_OP_CLASSES
 #include "mlir/Dialect/WasmSSA/IR/WasmSSAOps.cpp.inc"
 
@@ -29,7 +70,6 @@
 #include "mlir/IR/Types.h"
 #include "llvm/Support/LogicalResult.h"
 
-using namespace mlir;
 using namespace wasmssa;
 
 namespace {
diff --git a/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt b/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt
index 242a97ccfdf6d..7869a28dfed57 100644
--- a/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/XeGPU/IR/CMakeLists.txt
@@ -7,13 +7,18 @@ add_mlir_dialect_library(MLIRXeGPUDialect
 
   DEPENDS
   MLIRXeGPUIncGen
+  MLIRXeGPUAttrInterfaceIncGen
   MLIRXeGPUAttrsIncGen
   MLIRXeGPUEnumsIncGen
 
   LINK_LIBS PUBLIC
   MLIRArithDialect
+  MLIRIndexDialect
+  MLIRAffineUtils
   MLIRArithUtils
   MLIRDialectUtils
+  MLIRGPUDialect
+  MLIRXeVMDialect
   MLIRIR
   MLIRViewLikeInterface
   MLIRVectorDialect
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
index 3c0ca114a62d4..a2d708be0e937 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
@@ -6,12 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/Affine/Utils.h"
+#include "mlir/Dialect/Arith/Utils/Utils.h"
+#include "mlir/Dialect/Index/IR/IndexOps.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPUTargetInfo.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/DialectImplementation.h"
 #include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/Debug.h"
 
 using std::optional;
 
@@ -33,6 +37,57 @@ void XeGPUDialect::initialize() {
       >();
 }
 
+/// Generates instructions to compute offsets for a subgroup identified by
+/// its multidimensional indices (sgId), using the specified subgroup layout
+/// (sgLayout), subgroup data dimensions (sizePerSg), and the overall data
+/// dimensions (sizePerWg).
+static SmallVector<SmallVector<Value>>
+genOffsetsComputingInsts(OpBuilder &builder, Location loc,
+                         SmallVector<Value> sgId, ArrayRef<int64_t> sgLayout,
+                         ArrayRef<int64_t> sizePerSg,
+                         ArrayRef<int64_t> sizePerWg) {
+
+  SmallVector<SmallVector<Value>> offsets;
+
+  // nd local offset, localOffset[i] = sgId[i] * sizePerSg[i]
+  SmallVector<Value> localOffsets = llvm::map_to_vector(
+      llvm::zip(sgId, sizePerSg), [&](const auto &t) -> Value {
+        return builder.createOrFold<index::MulOp>(
+            loc, std::get<0>(t),
+            builder.createOrFold<arith::ConstantIndexOp>(loc, std::get<1>(t)));
+      });
+
+  // distUnit[i] is the minimum value between sizePerWg[i] and
+  // sgLayout[i] * sizePerSg[i]
+  SmallVector<int64_t> distUnit = llvm::map_to_vector(
+      llvm::zip_equal(sizePerWg, computeElementwiseMul(sgLayout, sizePerSg)),
+      [](const auto &t) { return std::min(std::get<0>(t), std::get<1>(t)); });
+
+  for (SmallVector<int64_t> unitOffs :
+       StaticTileOffsetRange(sizePerWg, distUnit)) {
+    SmallVector<Value> base =
+        llvm::map_to_vector(unitOffs, [&](int64_t d) -> Value {
+          return arith::ConstantIndexOp::create(builder, loc, d);
+        });
+
+    SmallVector<Value> adds = llvm::map_to_vector(
+        llvm::zip_equal(base, localOffsets), [&](const auto &t) -> Value {
+          return builder.createOrFold<arith::AddIOp>(loc, std::get<0>(t),
+                                                     std::get<1>(t));
+        });
+
+    SmallVector<Value> mods = llvm::map_to_vector(
+        llvm::zip_equal(adds, sizePerWg), [&](const auto &t) -> Value {
+          return builder.createOrFold<index::RemUOp>(
+              loc, std::get<0>(t),
+              arith::ConstantIndexOp::create(builder, loc, std::get<1>(t)));
+        });
+
+    offsets.push_back(mods);
+  }
+  return offsets;
+}
+
 // Checks if the given shape can be evenly distributed based on the layout
 // and data factors provided by the LayoutAttr.
 bool XeGPUDialect::isEvenlyDistributable(llvm::ArrayRef<int64_t> shape,
@@ -211,6 +266,150 @@ LayoutAttr::verify(llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
   return success();
 }
 
+FailureOr<SmallVector<Value>>
+LayoutAttr::delinearizeSubgroupId(OpBuilder &builder, Location loc,
+                                  Value linearId) {
+  // delinearizeSubgroupId is only available for
+  // workgroup-level layout attribute
+  if (!isForWorkgroup())
+    return failure();
+
+  // TODO: handle order attribute
+  auto hasDefaultOrder = [&]() {
+    DenseI32ArrayAttr order = getOrder();
+    return !order || isIdentityPermutation(llvm::to_vector_of<int64_t>(
+                         llvm::reverse(order.asArrayRef())));
+  };
+  if (!hasDefaultOrder())
+    return mlir::emitError(loc, "order attribute is currently not supported.");
+
+  auto dims = llvm::map_to_vector(*getSgLayoutAsInt(), [&](int64_t d) -> Value {
+    return builder.createOrFold<arith::ConstantIndexOp>(loc, d);
+  });
+
+  return affine::delinearizeIndex(builder, loc, linearId, dims);
+}
+
+/// Implements DistributeLayoutAttr::getOffsets to generate
+/// instructions for computing multi-dimensional offsets when distributed by
+/// LayoutAttr.
+FailureOr<SmallVector<SmallVector<Value>>>
+LayoutAttr::getOffsets(OpBuilder &builder, Location loc, Value linearId,
+                       ArrayRef<int64_t> shape) {
+  if (!isForWorkgroup())
+    return failure();
+
+  SmallVector<int64_t> sgLayout = getSgLayoutAsInt().value();
+  SmallVector<int64_t> sgShape;
+  if (auto maybeSgShape = getSgDataAsInt())
+    sgShape = maybeSgShape.value();
+  else if (auto derivedShape = computeShapeRatio(shape, sgLayout))
+    sgShape = derivedShape.value();
+  else
+    return failure();
+
+  // delinearize Ids
+  auto maybeIds = delinearizeSubgroupId(builder, loc, linearId);
+  if (failed(maybeIds))
+    return failure();
+  SmallVector<Value> sgIds = *maybeIds;
+
+  return genOffsetsComputingInsts(builder, loc, sgIds, sgLayout, sgShape,
+                                  shape);
+}
+
+//===----------------------------------------------------------------------===//
+// XeGPU_SliceAttr
+//===----------------------------------------------------------------------===//
+LogicalResult
+SliceAttr::verify(llvm::function_ref<InFlightDiagnostic()> emitError,
+                  xegpu::DistributeLayoutAttr parent, DenseI64ArrayAttr dims) {
+  if (!parent || !dims)
+    return emitError() << "expected parent layout and dims attribute";
+
+  int64_t rank = parent.getRank();
+
+  // check every element in dims is unique and smaller than rank
+  llvm::SmallDenseSet<int64_t> seen;
+  for (int64_t dim : dims.asArrayRef()) {
+    if (dim < 0 || dim >= rank)
+      return emitError() << "invalid dim (" << dim << ") in slice attribute.";
+    if (!seen.insert(dim).second)
+      return emitError() << "repeated dim (" << dim << ") in slice attribute.";
+  }
+  return success();
+}
+
+SliceAttr SliceAttr::flatten() const {
+  xegpu::DistributeLayoutAttr parent = getParent();
+  SmallVector<DenseI64ArrayAttr> slicedDims({getDims()});
+
+  while (auto sliceAttr = dyn_cast<xegpu::SliceAttr>(parent)) {
+    parent = sliceAttr.getParent();
+    slicedDims.push_back(sliceAttr.getDims());
+  }
+
+  auto layoutAttr = dyn_cast<xegpu::LayoutAttr>(parent);
+  SmallVector<int64_t> indices =
+      llvm::to_vector(llvm::seq<int64_t>(0, layoutAttr.getRank()));
+
+  // get remaining dims (flattend) by applying slice ops with all slicedDims
+  SmallVector<int64_t> remainingDims(indices);
+  for (auto dim : llvm::reverse(slicedDims))
+    remainingDims = XeGPUDialect::slice(llvm::ArrayRef<int64_t>(remainingDims),
+                                        dim.asArrayRef());
+
+  // get flattend sliced dims by applying slice ops with the remaining dims
+  SmallVector<int64_t> flattendDims = XeGPUDialect::slice(
+      llvm::ArrayRef<int64_t>(indices), llvm::ArrayRef<int64_t>(remainingDims));
+
+  return xegpu::SliceAttr::get(
+      getContext(), layoutAttr,
+      DenseI64ArrayAttr::get(getContext(), flattendDims));
+}
+
+FailureOr<SmallVector<Value>>
+SliceAttr::delinearizeSubgroupId(OpBuilder &builder, Location loc,
+                                 Value linearId) {
+  SliceAttr attr = flatten();
+  auto parent = dyn_cast<LayoutAttr>(attr.getParent());
+  return parent.delinearizeSubgroupId(builder, loc, linearId);
+}
+
+/// Implements DistributeLayoutAttr::getOffsets to generate
+/// instructions for computing multi-dimensional offsets when distributed by
+/// SliceAttr.
+FailureOr<SmallVector<SmallVector<Value>>>
+SliceAttr::getOffsets(OpBuilder &builder, Location loc, Value linearId,
+                      ArrayRef<int64_t> shape) {
+  assert(getRank() == static_cast<int64_t>(shape.size()) && "invalid shape.");
+  if (!isForWorkgroup())
+    return failure();
+
+  SmallVector<int64_t> sgLayout = getSgLayoutAsInt().value();
+  SmallVector<int64_t> sgShape;
+  if (auto maybeSgShape = getSgDataAsInt())
+    sgShape = maybeSgShape.value();
+  else if (auto derivedShape = computeShapeRatio(shape, sgLayout))
+    sgShape = derivedShape.value();
+  else
+    return failure();
+
+  // delinearize Ids
+  auto maybeIds = delinearizeSubgroupId(builder, loc, linearId);
+  if (failed(maybeIds))
+    return failure();
+
+  // The effective sgIds for offsets computing correspond
+  // to the dims that are not sliced.
+  ArrayRef<int64_t> dims = flatten().getDims().asArrayRef();
+  SmallVector<Value> sgIds =
+      XeGPUDialect::slice(ArrayRef<Value>(*maybeIds), dims);
+
+  return genOffsetsComputingInsts(builder, loc, sgIds, sgLayout, sgShape,
+                                  shape);
+}
+
 //===----------------------------------------------------------------------===//
 // XeGPU_RangeAttr
 //===----------------------------------------------------------------------===//
@@ -230,7 +429,7 @@ RangeAttr::verify(llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
 // XeGPU_TensorDescType
 //===----------------------------------------------------------------------===//
 
-mlir::Type TensorDescType::parse(::mlir::AsmParser &parser) {
+mlir::Type TensorDescType::parse(AsmParser &parser) {
   llvm::SmallVector<int64_t> shape;
   mlir::Type elementType;
   mlir::FailureOr<mlir::Attribute> encoding;
@@ -280,7 +479,7 @@ mlir::Type TensorDescType::parse(::mlir::AsmParser &parser) {
       layout.value_or(mlir::Attribute()));
 }
 
-void TensorDescType::print(::mlir::AsmPrinter &printer) const {
+void TensorDescType::print(AsmPrinter &printer) const {
   printer << "<";
 
   auto shape = getShape();
@@ -325,10 +524,10 @@ TensorDescType TensorDescType::get(llvm::ArrayRef<int64_t> shape,
   return Base::get(context, shape, elementType, attr, layout);
 }
 
-LogicalResult TensorDescType::verify(
-    llvm::function_ref<::mlir::InFlightDiagnostic()> emitError,
-    llvm::ArrayRef<int64_t> shape, mlir::Type elementType,
-    mlir::Attribute encoding, mlir::Attribute layout) {
+LogicalResult
+TensorDescType::verify(llvm::function_ref<InFlightDiagnostic()> emitError,
+                       llvm::ArrayRef<int64_t> shape, mlir::Type elementType,
+                       mlir::Attribute encoding, mlir::Attribute layout) {
   size_t rank = shape.size();
 
   if (rank == 0)
@@ -394,6 +593,119 @@ LogicalResult TensorDescType::verify(
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// XeGPU_MemDescType
+//===----------------------------------------------------------------------===//
+mlir::Type MemDescType::parse(AsmParser &parser) {
+  llvm::SmallVector<int64_t> shape;
+  mlir::Type elementType;
+  mlir::FailureOr<MemLayoutAttr> layout;
+
+  // Parse literal '<'
+  if (parser.parseLess())
+    return {};
+
+  auto shapeLoc = parser.getCurrentLocation();
+  if (mlir::failed(parser.parseDimensionList(shape, false, true))) {
+    parser.emitError(shapeLoc, "failed to parse parameter 'shape'");
+    return {};
+  }
+
+  auto elemTypeLoc = parser.getCurrentLocation();
+  if (mlir::failed(parser.parseType(elementType))) {
+    parser.emitError(elemTypeLoc, "failed to parse parameter 'elementType'");
+    return {};
+  }
+
+  // parse optional attributes
+  if (mlir::succeeded(parser.parseOptionalComma())) {
+    MemLayoutAttr attr;
+    ParseResult res = parser.parseAttribute(attr);
+    if (mlir::failed(res))
+      return {};
+    layout = attr;
+  }
+
+  // Parse literal '>'
+  if (parser.parseGreater())
+    return {};
+
+  MLIRContext *ctxt = parser.getContext();
+  return MemDescType::getChecked(
+      [&]() { return parser.emitError(parser.getNameLoc()); }, ctxt, shape,
+      elementType, layout.value_or(MemLayoutAttr()));
+}
+
+void MemDescType::print(AsmPrinter &printer) const {
+  printer << "<";
+
+  printer.printDimensionList(getShape());
+  printer << 'x';
+  printer << getElementType();
+
+  if (auto layout = getMemLayout())
+    printer << ", " << layout;
+
+  printer << ">";
+}
+
+//===----------------------------------------------------------------------===//
+// XeGPU_MemDescType
+//===----------------------------------------------------------------------===//
+
+Attribute MemLayoutAttr::parse(AsmParser &parser, Type type) {
+
+  auto context = parser.getContext();
+  llvm::SMLoc loc = parser.getCurrentLocation();
+
+  llvm::SmallDenseSet<StringRef> seenKeys;
+  SmallVector<NamedAttribute> attributes;
+
+  auto parseElt = [&]() -> ParseResult {
+    StringRef nameId;
+    if (failed(parser.parseKeyword(&nameId)))
+      return parser.emitError(loc, "expected valid attribute name");
+
+    if (!seenKeys.insert(nameId).second)
+      return parser.emitError(loc, "duplicate key '")
+             << nameId << " in mem layout attribute";
+
+    if (failed(parser.parseEqual()))
+      return failure();
+
+    Attribute attr;
+    if (failed(parser.parseAttribute(attr)))
+      return failure();
+    attributes.emplace_back(nameId, attr);
+    return success();
+  };
+
+  // Parse literal '<'
+  if (parser.parseLess())
+    return {};
+
+  if (failed(parser.parseCommaSeparatedList(parseElt)))
+    return {};
+
+  // Parse literal '>'
+  if (parser.parseGreater())
+    return {};
+
+  return parser.getChecked<MemLayoutAttr>(
+      loc, context, DictionaryAttr::get(context, attributes));
+}
+
+void MemLayoutAttr::print(AsmPrinter &printer) const {
+  printer << "<";
+  ArrayRef<NamedAttribute> attrs = getAttrs().getValue();
+  for (size_t i = 0; i < attrs.size(); i++) {
+    printer << attrs[i].getName().str() << " = " << attrs[i].getValue();
+    if (i < attrs.size() - 1)
+      printer << ", ";
+  }
+  printer << ">";
+}
+
 } // namespace xegpu
 } // namespace mlir
 
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 33450f3fa229e..7036996a68e0d 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Arith/Utils/Utils.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
@@ -21,6 +23,17 @@
 namespace mlir {
 namespace xegpu {
 
+bool isSharedMemory(const MemRefType &memrefTy) {
+  Attribute attr = memrefTy.getMemorySpace();
+  if (auto intAttr = llvm::dyn_cast<IntegerAttr>(attr))
+    return intAttr.getInt() == 3;
+  if (auto memrefSpace = llvm::dyn_cast<MemorySpaceAttr>(attr))
+    return memrefSpace.getValue() == MemorySpace::SLM;
+  if (auto xevmSpace = llvm::dyn_cast<xevm::AddrSpaceAttr>(attr))
+    return xevmSpace.getValue() == xevm::AddrSpace::SHARED;
+  return gpu::GPUDialect::isWorkgroupMemoryAddressSpace(attr);
+}
+
 template <typename T>
 static std::string makeString(T array, bool breakline = false) {
   std::string buf;
@@ -45,13 +58,6 @@ static SmallVector<int64_t> getShapeOf(Type type) {
   return shape;
 }
 
-static int64_t getRankOf(Value val) {
-  auto type = val.getType();
-  if (auto ty = llvm::dyn_cast<ShapedType>(type))
-    return ty.getRank();
-  return 0;
-}
-
 static bool isReadHintOrNone(const CachePolicyAttr &attr) {
   if (!attr)
     return true;
@@ -76,13 +82,18 @@ isValidGatherScatterParams(Type maskTy, VectorType valueTy,
   if (!tdescTy.isScattered())
     return emitError() << "Expects a scattered TensorDesc.";
 
-  if (!valueTy)
-    return emitError() << "Expecting a vector type result.";
+  auto chunkSize = tdescTy.getChunkSizeAsInt();
+  if (!valueTy) {
+    if (chunkSize > 1)
+      return emitError() << "Expecting chunk size == 1 for scalar result";
+    if (dyn_cast<VectorType>(maskTy))
+      return emitError() << "Expecting a vector type result.";
+    return success();
+  }
 
   auto maskShape = getShapeOf(maskTy);
   auto valueShape = getShapeOf(valueTy);
   auto tdescShape = getShapeOf(tdescTy);
-  auto chunkSize = tdescTy.getChunkSizeAsInt();
 
   if (valueTy.getElementType() != tdescTy.getElementType())
     return emitError()
@@ -111,25 +122,49 @@ isValidGatherScatterParams(Type maskTy, VectorType valueTy,
 }
 
 static LogicalResult
-isValidGatherScatterBufferParams(Type maskTy, VectorType valueTy,
-                                 int64_t chunkSize,
+isValidGatherScatterBufferParams(Type offsetsTy, Type maskTy,
+                                 VectorType valueTy, int64_t chunkSize,
                                  function_ref<InFlightDiagnostic()> emitError) {
 
-  if (!valueTy)
-    return emitError() << "Expecting a vector type result.";
+  auto maskVecTy = dyn_cast<VectorType>(maskTy);
+  auto offsetsVecTy = dyn_cast<VectorType>(offsetsTy);
+  if (!valueTy) {
+    if (chunkSize > 1)
+      return emitError() << "Expecting chunk size == 1 for scalar result";
+    if (maskVecTy || offsetsVecTy)
+      return emitError() << "Expecting scalar mask and offsets.";
+    else if (maskVecTy && offsetsVecTy)
+      return emitError() << "Expecting a vector type result.";
+    return success();
+  }
 
+  auto valueSize = valueTy.getNumElements();
+  // SIMT mode with scalar mask and offsets.
+  if (!maskVecTy && !offsetsVecTy) {
+    if (valueSize != chunkSize)
+      return emitError() << "value elements must match chunk size "
+                         << chunkSize;
+    return success();
+  }
   auto maskShape = getShapeOf(maskTy);
   auto valueShape = getShapeOf(valueTy);
 
-  // a valid shape for SIMT case
-  if (valueTy.getRank() == 1) {
-    if (valueTy.getNumElements() != chunkSize)
-      return emitError() << "value elements must match chunk size " << chunkSize
-                         << " for SIMT code.";
-    return success();
+  if (!maskVecTy)
+    return emitError() << "Expecting a vector type mask.";
+  int64_t maskSize = maskVecTy.getNumElements();
+
+  if (chunkSize > 1) {
+    if ((valueTy.getRank() == 1) && (valueSize != chunkSize))
+      return emitError() << "value elements must match chunk size "
+                         << chunkSize;
+  } else {
+    if (valueSize != maskSize)
+      return emitError()
+             << "Mask should match value except the chunk size dim.";
   }
-
   llvm::SmallVector<int64_t> expectedMaskShape(valueShape);
+  if (maskSize == 1)
+    return success();
   if (chunkSize > 1)
     expectedMaskShape.pop_back();
   if (expectedMaskShape != maskShape)
@@ -156,41 +191,18 @@ void CreateNdDescOp::build(OpBuilder &builder, OperationState &state,
 }
 
 void CreateNdDescOp::build(OpBuilder &builder, OperationState &state,
-                           Type tdesc, TypedValue<MemRefType> source,
+                           Type tdesc, Value source,
                            llvm::ArrayRef<OpFoldResult> shape,
                            llvm::ArrayRef<OpFoldResult> strides) {
-  assert(shape.size() && strides.size() && shape.size() == strides.size() &&
-         "Shape and strides must be present and of equal size for ui64 "
-         "initialization.");
+  Type srcTy = source.getType();
+  assert((isa<IntegerType, MemRefType>(srcTy)) &&
+         "Source has to be either int or memref.");
 
-  llvm::SmallVector<int64_t> staticShape;
-  llvm::SmallVector<int64_t> staticStrides;
   llvm::SmallVector<Value> dynamicShape;
   llvm::SmallVector<Value> dynamicStrides;
 
-  dispatchIndexOpFoldResults(shape, dynamicShape, staticShape);
-  dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides);
-
-  auto staticShapeAttr = builder.getDenseI64ArrayAttr(staticShape);
-  auto staticStridesAttr = builder.getDenseI64ArrayAttr(staticStrides);
-
-  build(builder, state, tdesc, source, ValueRange({}), dynamicShape,
-        dynamicStrides, builder.getDenseI64ArrayAttr({}), staticShapeAttr,
-        staticStridesAttr);
-}
-
-void CreateNdDescOp::build(OpBuilder &builder, OperationState &state,
-                           Type tdesc, TypedValue<IntegerType> source,
-                           llvm::ArrayRef<OpFoldResult> shape,
-                           llvm::ArrayRef<OpFoldResult> strides) {
-  assert(shape.size() && strides.size() && shape.size() == strides.size() &&
-         "Shape and strides must be present and of equal size for ui64 "
-         "initialization.");
-
   llvm::SmallVector<int64_t> staticShape;
   llvm::SmallVector<int64_t> staticStrides;
-  llvm::SmallVector<Value> dynamicShape;
-  llvm::SmallVector<Value> dynamicStrides;
 
   dispatchIndexOpFoldResults(shape, dynamicShape, staticShape);
   dispatchIndexOpFoldResults(strides, dynamicStrides, staticStrides);
@@ -198,6 +210,18 @@ void CreateNdDescOp::build(OpBuilder &builder, OperationState &state,
   auto staticShapeAttr = builder.getDenseI64ArrayAttr(staticShape);
   auto staticStridesAttr = builder.getDenseI64ArrayAttr(staticStrides);
 
+  if (auto memrefTy = dyn_cast<MemRefType>(srcTy)) {
+    auto memrefShape = memrefTy.getShape();
+    auto [memrefStrides, _] = memrefTy.getStridesAndOffset();
+
+    // if shape and strides are from Memref, we don't need attributes for them
+    // to keep the IR print clean.
+    if (staticShape == memrefShape && staticStrides == memrefStrides) {
+      staticShapeAttr = DenseI64ArrayAttr();
+      staticStridesAttr = DenseI64ArrayAttr();
+    }
+  }
+
   build(builder, state, tdesc, source, ValueRange({}), dynamicShape,
         dynamicStrides, builder.getDenseI64ArrayAttr({}), staticShapeAttr,
         staticStridesAttr);
@@ -265,8 +289,8 @@ void CreateNdDescOp::build(OpBuilder &builder, OperationState &state,
 }
 
 LogicalResult CreateNdDescOp::verify() {
-  auto rank = (int64_t)getMixedOffsets().size();
-  bool invalidRank = false;
+  size_t rank = getMixedSizes().size();
+  bool invalidRank = rank != getMixedStrides().size();
   bool invalidElemTy = false;
 
   // Memory space of created TensorDesc should match with the source.
@@ -280,31 +304,28 @@ LogicalResult CreateNdDescOp::verify() {
            << " Source: " << srcMemorySpace
            << ", TensorDesc: " << tdescMemorySpace;
 
+  if (size_t offsetRank = getMixedOffsets().size())
+    invalidRank |= (offsetRank != rank);
+
   // check source type matches the rank if it is a memref.
   // It also should have the same ElementType as TensorDesc.
-  auto memrefTy = dyn_cast<MemRefType>(getSourceType());
-  if (memrefTy) {
-    invalidRank |= (memrefTy.getRank() != rank);
+  if (auto memrefTy = dyn_cast<MemRefType>(getSourceType()))
     invalidElemTy |= memrefTy.getElementType() != getElementType();
-  }
 
   if (llvm::isa<IntegerType>(getSourceType())) {
     // strides and shape must present for integer source.
     if (getMixedStrides().empty() || getMixedSizes().empty())
-      return emitOpError("Expecting strides and shape to be present for "
+      return emitOpError("expecting strides and shape to be present for "
                          "integer source.");
   }
 
-  // mismatches among shape, strides, and offsets are
-  // already handeled by OffsetSizeAndStrideOpInterface.
-  // So they are not check here.
   if (invalidRank)
     return emitOpError(
         "Expecting the rank of shape, strides, offsets, and source (if source "
         "is a memref) should match with each other.");
 
   // check result TensorDesc rank
-  if (getType().getRank() > rank)
+  if (getType().getRank() > (int64_t)rank)
     return emitOpError(
         "Expecting the TensorDesc rank is not greater than the "
         "ranks of shape, strides, offsets or the memref source.");
@@ -360,13 +381,10 @@ ParseResult parseOptionalDynamicIndexList(
 void printOptionalDynamicIndexList(OpAsmPrinter &printer, Operation *op,
                                    OperandRange values,
                                    DenseI64ArrayAttr integers) {
-
-  if (!integers)
+  if (!integers || integers.empty())
     return;
-
-  return printDynamicIndexList(printer, op, values, integers,
-                               /*scalableFlags=*/{}, {},
-                               AsmParser::Delimiter::Square);
+  printDynamicIndexList(printer, op, values, integers,
+                        /*scalableFlags=*/{}, {}, AsmParser::Delimiter::Square);
 }
 //===----------------------------------------------------------------------===//
 // XeGPU_PrefetchNdOp
@@ -381,6 +399,21 @@ void PrefetchNdOp::build(OpBuilder &builder, OperationState &state,
                l1_hint, l2_hint, l3_hint);
 }
 
+void PrefetchNdOp::build(OpBuilder &builder, OperationState &state,
+                         Value tensorDesc, ArrayRef<OpFoldResult> offsets,
+                         xegpu::CachePolicyAttr l1_hint,
+                         xegpu::CachePolicyAttr l2_hint,
+                         xegpu::CachePolicyAttr l3_hint) {
+  SmallVector<Value> dynamicOffsets;
+  SmallVector<int64_t> staticOffsets;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
+
+  auto staticOffsetsAttr = builder.getDenseI64ArrayAttr(staticOffsets);
+
+  build(builder, state, tensorDesc, dynamicOffsets, staticOffsetsAttr, l1_hint,
+        l2_hint, l3_hint);
+}
+
 LogicalResult PrefetchNdOp::verify() {
   auto tdescTy = getTensorDescType();
   if (tdescTy.isScattered())
@@ -423,6 +456,22 @@ void LoadNdOp::build(OpBuilder &builder, OperationState &state, Type retType,
                l3_hint);
 }
 
+void LoadNdOp::build(OpBuilder &builder, OperationState &state, Type retType,
+                     Value tensorDesc, ArrayRef<OpFoldResult> offsets,
+                     UnitAttr packed, DenseI64ArrayAttr transpose,
+                     xegpu::CachePolicyAttr l1_hint,
+                     xegpu::CachePolicyAttr l2_hint,
+                     xegpu::CachePolicyAttr l3_hint) {
+  SmallVector<Value> dynamicOffsets;
+  SmallVector<int64_t> staticOffsets;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
+
+  auto staticOffsetsAttr = builder.getDenseI64ArrayAttr(staticOffsets);
+
+  build(builder, state, retType, tensorDesc, dynamicOffsets, staticOffsetsAttr,
+        packed, transpose, l1_hint, l2_hint, l3_hint);
+}
+
 LogicalResult LoadNdOp::verify() {
   auto tdescTy = getTensorDescType();
   auto valueTy = getType();
@@ -529,6 +578,21 @@ void StoreNdOp::build(OpBuilder &builder, OperationState &state, Value value,
                DenseI64ArrayAttr(), l1_hint, l2_hint, l3_hint);
 }
 
+void StoreNdOp::build(OpBuilder &builder, OperationState &state, Value value,
+                      Value tensorDesc, ArrayRef<OpFoldResult> offsets,
+                      xegpu::CachePolicyAttr l1_hint,
+                      xegpu::CachePolicyAttr l2_hint,
+                      xegpu::CachePolicyAttr l3_hint) {
+  SmallVector<Value> dynamicOffsets;
+  SmallVector<int64_t> staticOffsets;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
+
+  auto staticOffsetsAttr = builder.getDenseI64ArrayAttr(staticOffsets);
+
+  build(builder, state, value, tensorDesc, dynamicOffsets, staticOffsetsAttr,
+        l1_hint, l2_hint, l3_hint);
+}
+
 LogicalResult StoreNdOp::verify() {
   auto dstTy = getTensorDescType(); // Tile
   auto valTy = getValueType();      // Vector
@@ -635,10 +699,6 @@ void CreateDescOp::build(OpBuilder &builder, OperationState &state,
 LogicalResult CreateDescOp::verify() {
   auto tdescTy = getTensorDescType();
 
-  if (getRankOf(getSource()) > 1)
-    return emitOpError(
-        "Expecting the source is a 1D memref or pointer (uint64_t).");
-
   if (!tdescTy.isScattered())
     return emitOpError("Expects a scattered TensorDesc.\n");
 
@@ -673,12 +733,14 @@ LogicalResult CreateDescOp::verify() {
 LogicalResult PrefetchOp::verify() {
   auto tdescTy = getTensorDescType();
 
-  if (tdescTy && !tdescTy.isScattered())
-    return emitOpError("Expects a scattered TensorDesc.\n");
+  if (!tdescTy && !getOffsets())
+    return emitOpError("Expects offsets.");
 
-  if (!tdescTy && getRankOf(getSource()) > 1)
-    return emitOpError(
-        "Expecting the source is a 1D memref or pointer (uint64_t).");
+  if (tdescTy && getOffsets())
+    return emitOpError("offsets not allowed.");
+
+  if (tdescTy && !tdescTy.isScattered())
+    return emitOpError("Expects a scattered TensorDesc.");
 
   if (!isReadHintOrNone(getL1HintAttr()))
     return emitOpError("invalid l1_hint: ") << getL1HintAttr();
@@ -689,6 +751,13 @@ LogicalResult PrefetchOp::verify() {
   if (!isReadHintOrNone(getL3HintAttr()))
     return emitOpError("invalid l3_hint: ") << getL3HintAttr();
 
+  auto srcTy = getSourceType();
+  if (srcTy.isInteger() && !getOffsetAlignByteAttr())
+    return emitOpError("offset_align_byte is required with integer source.");
+
+  if (getOffsetAlignByteAttr() && !srcTy.isInteger())
+    return emitOpError("offset_align_byte only allowed with integer source.");
+
   return success();
 }
 
@@ -696,7 +765,8 @@ void PrefetchOp::build(OpBuilder &builder, OperationState &state, Value source,
                        xegpu::CachePolicyAttr l1_hint,
                        xegpu::CachePolicyAttr l2_hint,
                        xegpu::CachePolicyAttr l3_hint) {
-  build(builder, state, source, Value(), l1_hint, l2_hint, l3_hint);
+  build(builder, state, source, Value(), l1_hint, l2_hint, l3_hint,
+        IntegerAttr{});
 }
 
 //===----------------------------------------------------------------------===//
@@ -707,13 +777,15 @@ LogicalResult LoadGatherOp::verify() {
   auto maskTy = getMaskType();
   auto valueTy = getValueType();
 
+  if (!tdescTy && !getOffsets())
+    return emitOpError("Expects offsets.");
+
+  if (tdescTy && getOffsets())
+    return emitOpError("offsets not allowed.");
+
   if (tdescTy && !tdescTy.isScattered())
     return emitOpError("Expects a scattered TensorDesc.");
 
-  if (!tdescTy && getRankOf(getSource()) > 1)
-    return emitOpError(
-        "Expecting the source is a 1D memref or pointer (uint64_t).");
-
   if (!isReadHintOrNone(getL1HintAttr()))
     return emitOpError("invalid l1_hint: ") << getL1HintAttr();
 
@@ -730,10 +802,11 @@ LogicalResult LoadGatherOp::verify() {
   uint64_t chunkSize = static_cast<int64_t>(getChunkSize().value_or(1));
   auto memTy = dyn_cast<MemRefType>(srcTy);
 
-  if (memTy && (valueTy.getElementType() != memTy.getElementType()))
+  if (memTy && (getElementType() != memTy.getElementType()))
     return emitError() << "Value should have the same element type as MemRef.";
 
-  return isValidGatherScatterBufferParams(maskTy, valueTy, chunkSize,
+  auto offsetsTy = getOffsets().getType();
+  return isValidGatherScatterBufferParams(offsetsTy, maskTy, valueTy, chunkSize,
                                           [&]() { return emitOpError(); });
 }
 
@@ -754,12 +827,14 @@ LogicalResult StoreScatterOp::verify() {
   auto maskTy = getMaskType();
   auto valueTy = getValueType();
 
-  if (tdescTy && !tdescTy.isScattered())
-    return emitOpError("Expects a scattered TensorDesc.\n");
+  if (!tdescTy && !getOffsets())
+    return emitOpError("Expects offsets.");
 
-  if (!tdescTy && getRankOf(getDest()) > 1)
-    return emitOpError(
-        "Expecting the dest is a 1D memref or pointer (uint64_t).");
+  if (tdescTy && getOffsets())
+    return emitOpError("offsets not allowed.");
+
+  if (tdescTy && !tdescTy.isScattered())
+    return emitOpError("Expects a scattered TensorDesc.");
 
   if (!isWriteHintOrNone(getL1HintAttr()))
     return emitOpError("invalid l1_hint: ") << getL1HintAttr();
@@ -778,10 +853,11 @@ LogicalResult StoreScatterOp::verify() {
   uint64_t chunkSize = static_cast<int64_t>(getChunkSize().value_or(1));
   auto memTy = dyn_cast<MemRefType>(destTy);
 
-  if (memTy && (valueTy.getElementType() != memTy.getElementType()))
+  if (memTy && (getElementType() != memTy.getElementType()))
     return emitError() << "Value should have the same element type as MemRef.";
 
-  return isValidGatherScatterBufferParams(maskTy, valueTy, chunkSize,
+  auto offsetsTy = getOffsets().getType();
+  return isValidGatherScatterBufferParams(offsetsTy, maskTy, valueTy, chunkSize,
                                           [&]() { return emitOpError(); });
 }
 
@@ -888,8 +964,8 @@ LogicalResult ConvertLayoutOp::verify() {
 
   // both input and target layouts should be WgLayout or SgLayout at the same
   // time.
-  if ((!srcLayout.isWgLayout() || !resLayout.isWgLayout()) &&
-      (!srcLayout.isSgLayout() || !resLayout.isSgLayout()))
+  if ((!srcLayout.isForWorkgroup() || !resLayout.isForWorkgroup()) &&
+      (!srcLayout.isForSubgroup() || !resLayout.isForSubgroup()))
     return emitOpError("expected input layout and target layout be WgLayout or "
                        "SgLayout at the same time.");
 
@@ -928,9 +1004,107 @@ void ConvertLayoutOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
   patterns.add<FoldConvertLayoutOp>(context);
 }
 
+//===----------------------------------------------------------------------===//
+// XeGPU_LoadMatrixOp
+//===----------------------------------------------------------------------===//
+void LoadMatrixOp::build(OpBuilder &builder, OperationState &state, Type res,
+                         TypedValue<MemDescType> memDesc,
+                         llvm::ArrayRef<OpFoldResult> offsets,
+                         DistributeLayoutAttr layout) {
+  llvm::SmallVector<Value> dynamicOffsets;
+  llvm::SmallVector<int64_t> staticOffsets;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
+  auto staticOffsetsAttr = builder.getDenseI64ArrayAttr(staticOffsets);
+  build(builder, state, res, memDesc, dynamicOffsets, staticOffsetsAttr,
+        layout);
+}
+
+LogicalResult LoadMatrixOp::verify() {
+  VectorType resTy = getRes().getType();
+  MemDescType mdescTy = getMemDesc().getType();
+
+  if (mdescTy.getRank() != 2)
+    return emitOpError("mem_desc must be 2D.");
+
+  ArrayRef<int64_t> valueShape = resTy.getShape();
+  ArrayRef<int64_t> mdescShape = mdescTy.getShape();
+  if (llvm::any_of(llvm::zip_equal(valueShape, mdescShape),
+                   [](auto p) { return std::get<0>(p) > std::get<1>(p); }))
+    return emitOpError("result shape must not exceed mem_desc shape.");
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// XeGPU_StoreMatrixOp
+//===----------------------------------------------------------------------===//
+void StoreMatrixOp::build(OpBuilder &builder, OperationState &state, Value data,
+                          TypedValue<MemDescType> memDesc,
+                          llvm::ArrayRef<OpFoldResult> offsets,
+                          DistributeLayoutAttr layout) {
+  llvm::SmallVector<Value> dynamicOffsets;
+  llvm::SmallVector<int64_t> staticOffsets;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
+  auto staticOffsetsAttr = builder.getDenseI64ArrayAttr(staticOffsets);
+  build(builder, state, data, memDesc, dynamicOffsets, staticOffsetsAttr,
+        layout);
+}
+
+LogicalResult StoreMatrixOp::verify() {
+  VectorType dataTy = getData().getType();
+  MemDescType mdescTy = getMemDesc().getType();
+
+  if (mdescTy.getRank() != 2)
+    return emitOpError("mem_desc must be 2D.");
+
+  ArrayRef<int64_t> dataShape = dataTy.getShape();
+  ArrayRef<int64_t> mdescShape = mdescTy.getShape();
+  if (llvm::any_of(llvm::zip_equal(dataShape, mdescShape),
+                   [](auto p) { return std::get<0>(p) > std::get<1>(p); }))
+    return emitOpError("data shape must not exceed mem_desc shape.");
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// XeGPU_MemDescSubviewOp
+//===----------------------------------------------------------------------===//
+
+void MemDescSubviewOp::build(OpBuilder &builder, OperationState &state,
+                             Type resTy, Value src,
+                             llvm::ArrayRef<OpFoldResult> offsets) {
+  llvm::SmallVector<Value> dynamicOffsets;
+  llvm::SmallVector<int64_t> staticOffsets;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
+  auto staticOffsetsAttr = builder.getDenseI64ArrayAttr(staticOffsets);
+  build(builder, state, resTy, src, dynamicOffsets, staticOffsetsAttr);
+}
+
+LogicalResult MemDescSubviewOp::verify() {
+  MemDescType srcTy = getSrc().getType();
+  MemDescType resTy = getRes().getType();
+  ArrayRef<int64_t> srcShape = srcTy.getShape();
+  ArrayRef<int64_t> resShape = resTy.getShape();
+
+  if (srcTy.getRank() < resTy.getRank())
+    return emitOpError("result rank must not exceed source rank.");
+
+  if (llvm::any_of(
+          llvm::zip_equal(resShape, srcShape.take_back(resShape.size())),
+          [](auto p) { return std::get<0>(p) > std::get<1>(p); }))
+    return emitOpError("result shape must not exceed source shape.");
+
+  if (srcTy.getStrides() != resTy.getStrides())
+    return emitOpError("result must inherit the source strides.");
+
+  return success();
+}
+
 } // namespace xegpu
 } // namespace mlir
 
+namespace mlir {
+#include <mlir/Dialect/XeGPU/IR/XeGPUAttrInterface.cpp.inc>
+} // namespace mlir
 #include <mlir/Dialect/XeGPU/IR/XeGPUEnums.cpp.inc>
 #define GET_OP_CLASSES
 #include <mlir/Dialect/XeGPU/IR/XeGPU.cpp.inc>
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp
index d82c541f31359..b3144e4c1e55d 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp
@@ -141,7 +141,7 @@ XeGPUBlockingPass::getTileShape(const T &operandOrResult) const {
     value = (Value)operandOrResult;
 
   xegpu::LayoutAttr layout = xegpu::getLayoutAttr(operandOrResult);
-  if (layout && layout.isSgLayout()) {
+  if (layout && layout.isForSubgroup()) {
     if (auto inst_data = layout.getInstData())
       return llvm::to_vector_of<int64_t>(inst_data.asArrayRef());
 
@@ -205,12 +205,12 @@ bool XeGPUBlockingPass::needsUnroll(Operation *op) const {
   bool hasWgLayoutOperands =
       llvm::any_of(op->getOpOperands(), [](OpOperand &opr) {
         xegpu::LayoutAttr layout = xegpu::getLayoutAttr(opr);
-        return layout && layout.isWgLayout();
+        return layout && layout.isForWorkgroup();
       });
   bool hasWgLayoutResults =
       llvm::any_of(op->getOpResults(), [](OpResult result) {
         xegpu::LayoutAttr layout = xegpu::getLayoutAttr(result);
-        return layout && layout.isWgLayout();
+        return layout && layout.isForWorkgroup();
       });
   if (hasWgLayoutOperands || hasWgLayoutResults) {
     LDBG() << "skip unrolling for op with workgroup level layout: " << *op;
@@ -272,7 +272,7 @@ void XeGPUBlockingPass::runOnOperation() {
 
         auto layout =
             llvm::dyn_cast_if_present<xegpu::LayoutAttr>(type.getEncoding());
-        if (layout && layout.isWgLayout())
+        if (layout && layout.isForWorkgroup())
           return failure();
 
         int count;
@@ -289,7 +289,7 @@ void XeGPUBlockingPass::runOnOperation() {
         ArrayRef<int64_t> shape = type.getShape();
 
         xegpu::LayoutAttr layout = type.getLayoutAttr();
-        if (layout && layout.isWgLayout())
+        if (layout && layout.isForWorkgroup())
           return failure();
 
         int count;
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index 2088c3c7fc5ec..8e47968609d32 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -336,8 +336,7 @@ struct StoreNdDistribution final : public gpu::WarpDistributionPattern {
   using gpu::WarpDistributionPattern::WarpDistributionPattern;
   LogicalResult matchAndRewrite(gpu::WarpExecuteOnLane0Op warpOp,
                                 PatternRewriter &rewriter) const override {
-    auto yield = cast<gpu::YieldOp>(
-        warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
+    gpu::YieldOp yield = warpOp.getTerminator();
     Operation *lastNode = yield->getPrevNode();
     auto storeOp = dyn_cast_or_null<xegpu::StoreNdOp>(lastNode);
     if (!storeOp)
@@ -449,8 +448,7 @@ struct LoadNdDistribution final : public gpu::WarpDistributionPattern {
       // Make sure the same load op is the last operation in the warp op body.
       // This ensure that load op is not sinked earlier violating any barrier
       // synchronizations.
-      auto yield = cast<gpu::YieldOp>(
-          warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
+      gpu::YieldOp yield = warpOp.getTerminator();
       return yield->getPrevNode() == op;
     });
 
@@ -752,8 +750,7 @@ struct PrefetchNdDistribution final : public gpu::WarpDistributionPattern {
   using gpu::WarpDistributionPattern::WarpDistributionPattern;
   LogicalResult matchAndRewrite(gpu::WarpExecuteOnLane0Op warpOp,
                                 PatternRewriter &rewriter) const override {
-    auto yield = cast<gpu::YieldOp>(
-        warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
+    gpu::YieldOp yield = warpOp.getTerminator();
     Operation *lastNode = yield->getPrevNode();
     auto prefetchOp = dyn_cast_or_null<xegpu::PrefetchNdOp>(lastNode);
     if (!prefetchOp)
@@ -794,8 +791,7 @@ struct GpuBarrierDistribution final : public gpu::WarpDistributionPattern {
   using gpu::WarpDistributionPattern::WarpDistributionPattern;
   LogicalResult matchAndRewrite(gpu::WarpExecuteOnLane0Op warpOp,
                                 PatternRewriter &rewriter) const override {
-    auto yield = cast<gpu::YieldOp>(
-        warpOp.getBodyRegion().getBlocks().begin()->getTerminator());
+    gpu::YieldOp yield = warpOp.getTerminator();
     Operation *lastNode = yield->getPrevNode();
     // The last node must be a gpu::BarrierOp.
     auto barrierOp = dyn_cast_or_null<gpu::BarrierOp>(lastNode);
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
index 850f70cca288f..93b4efcd125ec 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUWgToSgDistribute.cpp
@@ -34,38 +34,29 @@ using namespace mlir;
 
 namespace {
 
-// Check if there is sg id range attached to the scf.if op.
-static bool isSgIdRangeSpecified(Operation *op, int64_t &startOfRange,
-                                 int64_t &endOfRange) {
-  Operation *parent = op->getParentOp();
-  // Find the outermost scf::IfOp with xegpu.sg_id_range.
+// Retrieve the RangeAttr if it is specified.
+static xegpu::RangeAttr getRangeSpecAttr(Operation *op) {
+  Operation *parent = op->getParentOfType<scf::IfOp>();
   while (parent) {
-    if (auto ifOp = dyn_cast<scf::IfOp>(parent)) {
-      if (auto attr = llvm::dyn_cast_or_null<xegpu::RangeAttr>(
-              ifOp->getAttr("sg_id_range"))) {
-        startOfRange = attr.getStart().getInt();
-        endOfRange = attr.getEnd().getInt();
-        break;
-      }
-    }
-    parent = parent->getParentOp();
+    if (auto attr = llvm::dyn_cast_if_present<xegpu::RangeAttr>(
+            parent->getAttr("sg_id_range")))
+      return attr;
+    parent = parent->getParentOfType<scf::IfOp>();
   }
-  // Return false if startOfRange is 0
-  return (startOfRange > 0 && endOfRange > startOfRange);
+  return {};
 }
 
 static std::pair<SmallVector<int64_t>, int>
-getSgShapeAndCount(ArrayRef<int64_t> shape, xegpu::LayoutAttr layout) {
+getSgShapeAndCount(ArrayRef<int64_t> shape,
+                   xegpu::DistributeLayoutAttr layout) {
   int count = 1;
   SmallVector<int64_t> sgShape(shape);
-
-  if (layout && layout.isWgLayout()) {
-    DenseI32ArrayAttr sgLayoutAttr = layout.getSgLayout();
-    auto sgLayout = llvm::to_vector_of<int64_t>(sgLayoutAttr.asArrayRef());
-    if (DenseI32ArrayAttr sgDataAttr = layout.getSgData())
-      sgShape = llvm::to_vector_of<int64_t>(sgDataAttr.asArrayRef());
-    else
-      sgShape = computeShapeRatio(shape, sgLayout).value_or(sgShape);
+  if (layout && layout.isForWorkgroup()) {
+    SmallVector<int64_t> sgLayout = layout.getSgLayoutAsInt().value();
+    if (auto maybeSgData = layout.getSgDataAsInt())
+      sgShape = *maybeSgData;
+    else if (auto maybeDerivedSgData = computeShapeRatio(shape, sgLayout))
+      sgShape = *maybeDerivedSgData;
     SmallVector<int64_t> distUnit = computeElementwiseMul(sgLayout, sgShape);
     // Clamp distUnit to the original shape to handle cases where data is
     // shared among subgroups, which may cause distUnit to exceed the original
@@ -77,6 +68,67 @@ getSgShapeAndCount(ArrayRef<int64_t> shape, xegpu::LayoutAttr layout) {
   return std::make_pair(sgShape, count);
 }
 
+/// Utility helper for deriving a list of offsets for each sub-TensorDescs
+/// or sub-MemDescs to be accessed by current subgroup (sgId) based on the
+/// associated distribute layout attribute, the shape, subgroup id and the
+/// original offsets of the op
+template <
+    typename OpType,
+    typename = std::enable_if_t<llvm::is_one_of<
+        OpType, xegpu::CreateNdDescOp, xegpu::LoadNdOp, xegpu::StoreNdOp,
+        xegpu::PrefetchNdOp, xegpu::LoadMatrixOp, xegpu::StoreMatrixOp>::value>>
+static LogicalResult
+genOffsetsList(ConversionPatternRewriter &rewriter, OpType op,
+               SmallVector<SmallVector<OpFoldResult>> &offsetsList) {
+  Location loc = op.getLoc();
+  SmallVector<OpFoldResult> origOffsets = op.getMixedOffsets();
+  // not applicable to ops without offsets operands.
+  if (origOffsets.empty())
+    return failure();
+
+  // not applicable to ops without workgroup layout attributes
+  xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
+  if (!layout || !layout.isForWorkgroup())
+    return failure();
+
+  Value sgId = rewriter.create<gpu::SubgroupIdOp>(loc, /*upper_bound=*/nullptr);
+
+  // verify and adjust the sgId if the range specifier is present
+  xegpu::RangeAttr sgIdRange = getRangeSpecAttr(op);
+  if (sgIdRange) {
+    int64_t startOfRange = sgIdRange.getStart().getInt();
+    int64_t endOfRange = sgIdRange.getEnd().getInt();
+    // verify the RangeAttr against the layout attribute
+    if (layout.getNumSubgroups() != endOfRange - startOfRange)
+      return rewriter.notifyMatchFailure(
+          op, "sg_layout size must match the sg_id_range");
+    // adjust the sgId if necessary
+    if (startOfRange > 0) {
+      Value startOfRangeVal =
+          rewriter.create<arith::ConstantIndexOp>(loc, startOfRange);
+      sgId = rewriter.create<index::SubOp>(loc, sgId, startOfRangeVal);
+    }
+  }
+
+  // Compute the list of subgroup-relative offsets for sub-tensors or sub-memory
+  // descriptors to be accessed, based on the layout information.
+  ArrayRef<int64_t> wgShape = op.getDataShape();
+  auto maybeDescOffsets = layout.getOffsets(rewriter, loc, sgId, wgShape);
+  if (failed(maybeDescOffsets))
+    return failure();
+
+  // Compute the final global offsets for each accessed sub-tensor
+  // or sub-memory descriptor.
+  for (const auto &sgOffsets : *maybeDescOffsets) {
+    SmallVector<OpFoldResult> newOffsets = xegpu::addWithRightAligned(
+        rewriter, loc, getAsOpFoldResult(sgOffsets), origOffsets);
+    offsetsList.push_back(std::move(newOffsets));
+  }
+
+  // callback(offsetsList);
+  return success();
+}
+
 /// This pattern transforms the CreateNdDescOp to create a subgroup descriptor
 /// from a workgroup descriptor. It replaces the offsets and sizes with
 /// appropriate values for the subgroup.
@@ -125,125 +177,74 @@ getSgShapeAndCount(ArrayRef<int64_t> shape, xegpu::LayoutAttr layout) {
 struct WgToSgCreateNdOp : public OpConversionPattern<xegpu::CreateNdDescOp> {
   using OpConversionPattern<xegpu::CreateNdDescOp>::OpConversionPattern;
 
-  // Calculate offset for each subgroup
-  static SmallVector<OpFoldResult>
-  calculateGlobalOffsets(ConversionPatternRewriter &rewriter, Location loc,
-                         const SmallVector<OpFoldResult> &originalOffsets,
-                         const SmallVector<Value> &localOffset,
-                         const SmallVector<int64_t> &distUnitBaseAddr,
-                         const SmallVector<int64_t> &distUnitShape) {
-    assert(localOffset.size() == distUnitBaseAddr.size() &&
-           "localOffset and distUnitBaseAddr must have the same rank");
-
-    SmallVector<OpFoldResult> globalOffsets(originalOffsets.begin(),
-                                            originalOffsets.end());
-    size_t rank = localOffset.size();
-    for (size_t i = 0; i < rank; ++i) {
-      size_t dimIdx = originalOffsets.size() - rank + i;
-      Value constOffset =
-          arith::ConstantIndexOp::create(rewriter, loc, distUnitBaseAddr[i]);
-      Value offset =
-          rewriter.createOrFold<index::AddOp>(loc, localOffset[i], constOffset);
-      Value modValue =
-          arith::ConstantIndexOp::create(rewriter, loc, distUnitShape[i]);
-      Value offsetMod =
-          rewriter.createOrFold<index::RemUOp>(loc, offset, modValue);
-      Value origOffset = getValueOrCreateConstantIndexOp(
-          rewriter, loc, originalOffsets[dimIdx]);
-      Value globalOffset =
-          rewriter.createOrFold<index::AddOp>(loc, origOffset, offsetMod);
-      globalOffsets[dimIdx] = globalOffset;
-    }
-
-    return globalOffsets;
-  }
-
   LogicalResult
   matchAndRewrite(xegpu::CreateNdDescOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    Location loc = op.getLoc();
+    SmallVector<SmallVector<OpFoldResult>> offsetsList;
+    if (failed(genOffsetsList(rewriter, op, offsetsList)))
+      return failure();
+
     MLIRContext *ctx = op.getContext();
     xegpu::TensorDescType tdescTy = op.getType();
-    auto layout = dyn_cast<xegpu::LayoutAttr>(tdescTy.getLayout());
-    if (!layout)
-      return failure();
-    Type elemTy = tdescTy.getElementType();
     ArrayRef<int64_t> wgShape = tdescTy.getShape();
-    // sgLayout must be present for workgroup-level distribution.
-    SmallVector<int64_t> sgLayout;
-    if (auto sgLayoutAttr = layout.getSgLayout())
-      sgLayout = llvm::to_vector_of<int64_t>(sgLayoutAttr.asArrayRef());
-    else
-      return rewriter.notifyMatchFailure(
-          op, "sgLayout attribute is required in layout");
-
+    Type elemTy = tdescTy.getElementType();
+    xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
     SmallVector<int64_t> sgShape = getSgShapeAndCount(wgShape, layout).first;
+    auto newTdescTy =
+        xegpu::TensorDescType::get(ctx, sgShape, elemTy, tdescTy.getEncoding(),
+                                   layout.dropSgLayoutAndData());
 
-    // TODO : Handle order attribute
-    // Get the subgroup ID
-    auto linearSgId =
-        gpu::SubgroupIdOp::create(rewriter, loc, /*upper_bound=*/nullptr);
-
-    // Create constants for layout dimensions
-    SmallVector<Value> sgLayoutDim(sgLayout.size());
-    SmallVector<Value> sgDataDim(sgShape.size());
+    SmallVector<Value> newOps;
+    for (auto offsets : offsetsList) {
+      auto newOp = xegpu::CreateNdDescOp::create(
+          rewriter, op.getLoc(), newTdescTy, op.getSource(), offsets,
+          op.getMixedSizes(), op.getMixedStrides());
 
-    for (size_t i = 0; i < sgLayout.size(); i++) {
-      sgLayoutDim[i] =
-          arith::ConstantIndexOp::create(rewriter, loc, sgLayout[i]);
-      sgDataDim[i] = arith::ConstantIndexOp::create(rewriter, loc, sgShape[i]);
+      newOps.push_back(newOp);
     }
+    rewriter.replaceOpWithMultiple(op, {newOps});
 
-    int64_t startOfRange = -1, endOfRange = -1;
-    bool sgIdRangeSpecified =
-        isSgIdRangeSpecified(op, startOfRange, endOfRange);
-
-    Value adjustedSgId = linearSgId;
-    if (sgIdRangeSpecified) {
-      int64_t sgCount = endOfRange - startOfRange;
-      if (computeProduct(sgLayout) != sgCount)
-        return rewriter.notifyMatchFailure(
-            op, "sg_layout size must match the sg_id_range");
-      // Subtract startOfRange from the original subgroup id to get the adjusted
-      // sg id
-      Value startOfRangeVal =
-          arith::ConstantIndexOp::create(rewriter, loc, startOfRange);
-      adjustedSgId =
-          rewriter.createOrFold<index::SubOp>(loc, linearSgId, startOfRangeVal);
-    }
+    return success();
+  }
+};
+
+// This pattern transforms the CreateNdDescOp without offsets to create a
+// subgroup descriptor from a workgroup descriptor
+struct WgToSgCreateNdOpNoOffset
+    : public OpConversionPattern<xegpu::CreateNdDescOp> {
+  using OpConversionPattern<xegpu::CreateNdDescOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(xegpu::CreateNdDescOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
 
-    auto deLinearizeSgId =
-        affine::delinearizeIndex(rewriter, loc, adjustedSgId, sgLayoutDim);
-    if (failed(deLinearizeSgId))
+    // Check no offsets are specified.
+    if (!op.getMixedOffsets().empty())
       return failure();
-    SmallVector<Value> sgIds = *deLinearizeSgId;
-
-    // Calculate distribution unit shape and local offsets for subgroup
-    SmallVector<int64_t> distUnitShape(sgLayout.size());
-    SmallVector<Value> localOffset(sgLayout.size());
-    for (size_t i = 0; i < sgLayout.size(); i++) {
-      distUnitShape[i] = std::min(sgLayout[i] * sgShape[i], wgShape[i]);
-      localOffset[i] =
-          rewriter.createOrFold<index::MulOp>(loc, sgIds[i], sgDataDim[i]);
-    }
 
-    SmallVector<OpFoldResult> originalOffsets = op.getMixedOffsets();
+    Location loc = op.getLoc();
+    MLIRContext *ctx = op.getContext();
+    xegpu::TensorDescType tdescTy = op.getType();
+    auto layout = dyn_cast<xegpu::LayoutAttr>(tdescTy.getLayout());
+    if (!layout || !layout.isForWorkgroup())
+      return failure();
 
+    Type elemTy = tdescTy.getElementType();
+    ArrayRef<int64_t> wgShape = tdescTy.getShape();
+
+    SmallVector<int64_t> sgShape;
+    int count;
+    std::tie(sgShape, count) = getSgShapeAndCount(wgShape, layout);
     xegpu::TensorDescType newTdescTy =
         xegpu::TensorDescType::get(ctx, sgShape, elemTy, tdescTy.getEncoding(),
                                    layout.dropSgLayoutAndData());
-    SmallVector<Value> newCreateNdOps;
-    for (SmallVector<int64_t> distUnitBaseAddr :
-         StaticTileOffsetRange(wgShape, distUnitShape)) {
-      SmallVector<OpFoldResult> globalOffsets =
-          calculateGlobalOffsets(rewriter, loc, originalOffsets, localOffset,
-                                 distUnitBaseAddr, distUnitShape);
-
-      auto newCreateNdOp = xegpu::CreateNdDescOp::create(
-          rewriter, loc, newTdescTy, op.getSource(), globalOffsets,
-          op.getMixedSizes(), op.getMixedStrides());
-      newCreateNdOps.push_back(newCreateNdOp);
-    }
+
+    SmallVector<Value> newCreateNdOps(count);
+    std::generate(newCreateNdOps.begin(), newCreateNdOps.end(), [&]() {
+      return xegpu::CreateNdDescOp::create(rewriter, loc, newTdescTy,
+                                           op.getSource(), op.getMixedSizes(),
+                                           op.getMixedStrides());
+    });
 
     rewriter.replaceOpWithMultiple(op, {newCreateNdOps});
     return success();
@@ -256,12 +257,10 @@ struct WgToSgLoadNdOp : public OpConversionPattern<xegpu::LoadNdOp> {
   LogicalResult
   matchAndRewrite(xegpu::LoadNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    SmallVector<Value> newLoadOps;
-
-    int64_t offsetSize = static_cast<int64_t>(op.getOffsets().size());
-    if ((offsetSize != 0) || op.getConstOffsetsAttr())
+    if (!op.getMixedOffsets().empty())
       return failure();
 
+    SmallVector<Value> newLoadOps;
     for (auto src : adaptor.getTensorDesc()) {
       xegpu::TensorDescType tdescTy =
           dyn_cast<xegpu::TensorDescType>(src.getType());
@@ -284,9 +283,7 @@ struct WgToSgStoreNdOp : public OpConversionPattern<xegpu::StoreNdOp> {
   LogicalResult
   matchAndRewrite(xegpu::StoreNdOp op, OneToNOpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-
-    int64_t offsetSize = static_cast<int64_t>(op.getOffsets().size());
-    if ((offsetSize != 0) || op.getConstOffsetsAttr())
+    if (!op.getMixedOffsets().empty())
       return failure();
 
     for (auto [v, t] : llvm::zip(adaptor.getValue(), adaptor.getTensorDesc()))
@@ -298,6 +295,84 @@ struct WgToSgStoreNdOp : public OpConversionPattern<xegpu::StoreNdOp> {
   }
 };
 
+// This pattern transforms the LoadNdOp with explicit offsets to load
+// subgroup data.
+struct WgToSgLoadNdOpWithOffset : public OpConversionPattern<xegpu::LoadNdOp> {
+  using OpConversionPattern<xegpu::LoadNdOp>::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(xegpu::LoadNdOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    SmallVector<SmallVector<OpFoldResult>> offsetsList;
+    if (failed(genOffsetsList(rewriter, op, offsetsList)))
+      return failure();
+
+    SmallVector<Value> newOps;
+    for (auto [tdesc, offsets] :
+         llvm::zip(adaptor.getTensorDesc(), offsetsList)) {
+      auto tdescTy = dyn_cast<xegpu::TensorDescType>(tdesc.getType());
+      VectorType newResTy =
+          VectorType::get(tdescTy.getShape(), tdescTy.getElementType());
+      auto newOp = xegpu::LoadNdOp::create(
+          rewriter, op.getLoc(), newResTy, tdesc, offsets,
+          /*packed = */ nullptr, /*transpose = */ nullptr, op.getL1HintAttr(),
+          op.getL2HintAttr(), op.getL3HintAttr());
+      newOps.push_back(newOp);
+    }
+    rewriter.replaceOpWithMultiple(op, {newOps});
+
+    return success();
+  }
+};
+
+// This pattern transforms the StoreNdOp with explicit offsets to store
+// subgroup data.
+struct WgToSgStoreNdOpWithOffset
+    : public OpConversionPattern<xegpu::StoreNdOp> {
+  using OpConversionPattern<xegpu::StoreNdOp>::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(xegpu::StoreNdOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    SmallVector<SmallVector<OpFoldResult>> offsetsList;
+    if (failed(genOffsetsList(rewriter, op, offsetsList)))
+      return failure();
+
+    for (auto [v, tdesc, offsets] :
+         llvm::zip(adaptor.getValue(), adaptor.getTensorDesc(), offsetsList)) {
+      rewriter.create<xegpu::StoreNdOp>(op.getLoc(), v, tdesc, offsets,
+                                        op.getL1HintAttr(), op.getL2HintAttr(),
+                                        op.getL3HintAttr());
+    }
+    rewriter.eraseOp(op);
+
+    return success();
+  }
+};
+
+// This pattern transforms the PrefetchNdOp with explicit offsets to prefetch
+// subgroup data.
+struct WgToSgPrefetchNdOpWithOffset
+    : public OpConversionPattern<xegpu::PrefetchNdOp> {
+  using OpConversionPattern<xegpu::PrefetchNdOp>::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(xegpu::PrefetchNdOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    SmallVector<SmallVector<OpFoldResult>> offsetsList;
+    if (failed(genOffsetsList(rewriter, op, offsetsList)))
+      return failure();
+
+    for (auto [tdesc, offsets] :
+         llvm::zip(adaptor.getTensorDesc(), offsetsList)) {
+      rewriter.create<xegpu::PrefetchNdOp>(
+          op.getLoc(), tdesc, offsets, op.getL1HintAttr(), op.getL2HintAttr(),
+          op.getL3HintAttr());
+    }
+    rewriter.eraseOp(op);
+
+    return success();
+  }
+};
+
 /// This pattern transforms the UpdateNdOffsetOp to update the offsets of a
 /// subgroup descriptor. It creates an UpdateNdOffsetOp op to update the
 /// offsets of the new subgroup src tensor descriptors.
@@ -526,8 +601,8 @@ struct WgToSgElementwiseOp : public ConversionPattern {
 // is lowered to:
 //   #a = #xegpu.layout<inst_data = [16, 16]>
 //   #b = #xegpu.layout<inst_data = [8, 16]>
-//   store_matrix %1, %slm <{layout_input_0 = #a}> : vector<32x16>, matrix_desc<32x64xf32>
-//   %d = load_matrix %slm <{layout_result_0 = #a}> : matrix_desc<32x64xf32> -> vector<16x32xf32>
+//   store_matrix %1, %slm <{layout_input_0 = #a}> : vector<32x16>, mem_desc<32x64xf32>
+//   %d = load_matrix %slm <{layout_result_0 = #a}> : mem_desc<32x64xf32> -> vector<16x32xf32>
 //   xegpu.convert_layout %d <{input_layout = #a, target_layout = #b}> : vector<16x32xf32>
 // clang-format on
 struct WgToSgConvertLayoutOp
@@ -539,7 +614,8 @@ struct WgToSgConvertLayoutOp
     xegpu::LayoutAttr input = op.getInputLayout();
     xegpu::LayoutAttr target = op.getTargetLayout();
 
-    if (!input || !target || !input.isWgLayout() || !target.isWgLayout())
+    if (!input || !target || !input.isForWorkgroup() ||
+        !target.isForWorkgroup())
       return rewriter.notifyMatchFailure(
           op, "Input and target layouts must have subgroup layout");
 
@@ -649,16 +725,107 @@ struct UnrealizedConversionCastOpPattern
   }
 };
 
+// This pattern distributes arith.constant op into subgroup-level constants
+struct WgToSgArithConstantOp : public OpConversionPattern<arith::ConstantOp> {
+  using OpConversionPattern<arith::ConstantOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(arith::ConstantOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+    auto vecAttr = dyn_cast<DenseElementsAttr>(op.getValue());
+    auto vecType = dyn_cast<VectorType>(op.getType());
+    if (!vecAttr || !vecAttr.isSplat() || !vecType)
+      return failure();
+
+    xegpu::LayoutAttr layout = xegpu::getLayoutAttr(op.getResult());
+    if (!layout || !layout.getSgLayout())
+      return failure();
+
+    ArrayRef<int64_t> wgShape = vecType.getShape();
+    SmallVector<int64_t> sgShape;
+    int count;
+    std::tie(sgShape, count) = getSgShapeAndCount(wgShape, layout);
+
+    // Current limitation: constant of vector with single value.
+    // TODO: support more complex cases, e.g., vector with multiple values.
+    Attribute singleVal = vecAttr.getSplatValue<Attribute>();
+
+    auto newType = VectorType::get(sgShape, vecType.getElementType());
+    auto sgAttr = DenseElementsAttr::get(newType, singleVal);
+    auto cstOp =
+        arith::ConstantOp::create(rewriter, op.getLoc(), newType, sgAttr);
+    if (auto newLayout = layout.dropSgLayoutAndData())
+      xegpu::setLayoutAttr(cstOp->getResult(0), newLayout);
+    SmallVector<Value> newConsts(count, cstOp);
+
+    rewriter.replaceOpWithMultiple(op, {newConsts});
+    return success();
+  }
+};
+
+struct WgToSgLoadMatrixOp : public OpConversionPattern<xegpu::LoadMatrixOp> {
+  using OpConversionPattern<xegpu::LoadMatrixOp>::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(xegpu::LoadMatrixOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    SmallVector<SmallVector<OpFoldResult>> offsetsList;
+    if (failed(genOffsetsList(rewriter, op, offsetsList)))
+      return failure();
+
+    ArrayRef<int64_t> wgShape = op.getDataShape();
+    VectorType valueTy = op.getRes().getType();
+    Type elemTy = valueTy.getElementType();
+
+    xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
+    SmallVector<int64_t> sgShape = getSgShapeAndCount(wgShape, layout).first;
+    VectorType newResTy = VectorType::get(sgShape, elemTy);
+    SmallVector<Value> newOps;
+    for (auto offsets : offsetsList) {
+      auto newOp = rewriter.create<xegpu::LoadMatrixOp>(
+          op.getLoc(), newResTy, op.getMemDesc(), offsets,
+          layout.dropSgLayoutAndData());
+      newOps.push_back(newOp);
+    }
+    rewriter.replaceOpWithMultiple(op, {newOps});
+
+    return success();
+  }
+};
+
+struct WgToSgStoreMatrixOp : public OpConversionPattern<xegpu::StoreMatrixOp> {
+  using OpConversionPattern<xegpu::StoreMatrixOp>::OpConversionPattern;
+  LogicalResult
+  matchAndRewrite(xegpu::StoreMatrixOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    SmallVector<SmallVector<OpFoldResult>> offsetsList;
+    if (failed(genOffsetsList(rewriter, op, offsetsList)))
+      return failure();
+
+    xegpu::DistributeLayoutAttr layout = op.getLayoutAttr();
+    for (auto [v, offsets] : llvm::zip(adaptor.getData(), offsetsList))
+      rewriter.create<xegpu::StoreMatrixOp>(op.getLoc(), v, op.getMemDesc(),
+                                            offsets,
+                                            layout.dropSgLayoutAndData());
+    rewriter.eraseOp(op);
+    return success();
+  }
+};
+
 } // namespace
 
 namespace mlir {
 namespace xegpu {
 void populateXeGPUWgToSgDistributePatterns(RewritePatternSet &patterns) {
-  patterns.add<WgToSgCreateNdOp, WgToSgLoadNdOp, WgToSgStoreNdOp,
-               WgToSgUpdateNdOffsetOp, WgToSgDpasOp, WgToSgPrefetchNdOp,
-               UnrealizedConversionCastOpPattern, WgToSgElementwiseOp,
-               WgToSgVectorBroadcastOp, WgToSgConvertLayoutOp>(
-      patterns.getContext());
+  patterns
+      .add<WgToSgCreateNdOp, WgToSgCreateNdOpNoOffset, WgToSgLoadNdOp,
+           WgToSgLoadNdOpWithOffset, WgToSgStoreNdOp, WgToSgStoreNdOpWithOffset,
+           WgToSgUpdateNdOffsetOp, WgToSgDpasOp, WgToSgPrefetchNdOp,
+           WgToSgPrefetchNdOpWithOffset, UnrealizedConversionCastOpPattern,
+           WgToSgElementwiseOp, WgToSgVectorBroadcastOp, WgToSgConvertLayoutOp,
+           WgToSgArithConstantOp, WgToSgLoadMatrixOp, WgToSgStoreMatrixOp>(
+          patterns.getContext());
 }
 } // namespace xegpu
 } // namespace mlir
@@ -748,8 +915,8 @@ void XeGPUWgToSgDistributePass::runOnOperation() {
     return xegpu::TensorDescType();
   };
 
-  auto isLegal = [&](xegpu::LayoutAttr layout) -> bool {
-    return !layout || !layout.isWgLayout();
+  auto isLegal = [&](xegpu::DistributeLayoutAttr layout) -> bool {
+    return !layout || !layout.isForWorkgroup();
   };
 
   target.addDynamicallyLegalOp<xegpu::CreateNdDescOp, xegpu::LoadNdOp,
@@ -765,6 +932,24 @@ void XeGPUWgToSgDistributePass::runOnOperation() {
     return isLegal(layout);
   });
 
+  target.addDynamicallyLegalOp<xegpu::LoadMatrixOp>(
+      [=](xegpu::LoadMatrixOp op) -> bool {
+        return isLegal(op.getLayoutAttr());
+      });
+
+  target.addDynamicallyLegalOp<xegpu::StoreMatrixOp>(
+      [=](xegpu::StoreMatrixOp op) -> bool {
+        return isLegal(op.getLayoutAttr());
+      });
+
+  target.addDynamicallyLegalOp<arith::ConstantOp>(
+      [=](arith::ConstantOp op) -> bool {
+        auto vecType = dyn_cast<VectorType>(op.getType());
+        if (!vecType)
+          return true;
+        return isLegal(xegpu::getLayoutAttr(op.getResult()));
+      });
+
   target.addDynamicallyLegalOp<vector::BroadcastOp>(
       [=](vector::BroadcastOp op) -> bool {
         return isLegal(xegpu::getLayoutAttr(op.getResult()));
diff --git a/mlir/lib/Dialect/XeGPU/Utils/CMakeLists.txt b/mlir/lib/Dialect/XeGPU/Utils/CMakeLists.txt
index 98e84a4420722..d9bf4a1461c27 100644
--- a/mlir/lib/Dialect/XeGPU/Utils/CMakeLists.txt
+++ b/mlir/lib/Dialect/XeGPU/Utils/CMakeLists.txt
@@ -7,5 +7,7 @@ add_mlir_dialect_library(MLIRXeGPUUtils
   LINK_LIBS PUBLIC
   MLIRIR
   MLIRSCFTransforms
+  MLIRGPUDialect
+  MLIRXeVMDialect
   MLIRXeGPUDialect
   )
diff --git a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
index 2cf21fb802ba3..6835f64ad8ef7 100644
--- a/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
+++ b/mlir/lib/Dialect/XeGPU/Utils/XeGPUUtils.cpp
@@ -11,6 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/Index/IR/IndexOps.h"
+#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
 #include "mlir/Dialect/SCF/Transforms/Patterns.h"
 #include "mlir/Dialect/Utils/IndexingUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
@@ -38,7 +41,7 @@ mlir::xegpu::getDistributedVectorType(xegpu::TensorDescType tdescTy) {
   auto layout = llvm::dyn_cast_if_present<LayoutAttr>(tdescTy.getLayout());
   // It only works for subgroup level layout, which only has lane_layout
   // and lane_data, and is to distribute a SIMD code into SIMT code.
-  if (!layout || !layout.isSgLayout())
+  if (!layout || !layout.isForSubgroup())
     return failure();
 
   SmallVector<int64_t> laneData(layout.getLaneData().asArrayRef());
@@ -404,3 +407,49 @@ void xegpu::doSCFStructuralTypeConversionWithTensorType(
     (void)mlir::applyPartialConversion(op, target, std::move(patterns));
   }
 }
+
+std::optional<std::string> xegpu::getChipStr(Operation *op) {
+  auto gpuModuleOp = op->getParentOfType<gpu::GPUModuleOp>();
+
+  if (!gpuModuleOp)
+    return std::nullopt;
+
+  auto targetAttrs = gpuModuleOp.getTargets();
+  if (targetAttrs) {
+    for (auto &attr : *targetAttrs) {
+      auto xevmAttr = llvm::dyn_cast<xevm::XeVMTargetAttr>(attr);
+      if (xevmAttr)
+        return xevmAttr.getChip().str();
+    }
+  }
+
+  return std::nullopt;
+}
+
+/// Generates element-wise addition ops of two arrays with automatic alignment.
+/// When the input arrays have different sizes, the shorter array is
+/// right-aligned with the longer array, and the unmatched leading elements from
+/// the longer array are preserved unchanged. This is commonly used for offset
+/// computation where higher-dimensional offsets need to be added to
+/// lower-dimensional adjustments.
+///
+/// Example:
+///   lhs = [l1, l2, l3], rhs = [r1, r2]
+///   Result: [11, l2+r1, l3+r2]
+SmallVector<OpFoldResult>
+xegpu::addWithRightAligned(OpBuilder &builder, Location loc,
+                           ArrayRef<OpFoldResult> lhs,
+                           ArrayRef<OpFoldResult> rhs) {
+  // ensure a is longer than b
+  ArrayRef<OpFoldResult> a = lhs.size() >= rhs.size() ? lhs : rhs;
+  ArrayRef<OpFoldResult> b = lhs.size() >= rhs.size() ? rhs : lhs;
+  SmallVector<OpFoldResult> results(a.take_front(a.size() - b.size()));
+  a = a.slice(a.size() - b.size());
+  for (auto [l, r] : llvm::zip(a, b)) {
+    auto lval = getValueOrCreateConstantIndexOp(builder, loc, l);
+    auto rval = getValueOrCreateConstantIndexOp(builder, loc, r);
+    results.push_back(builder.createOrFold<index::AddOp>(loc, lval, rval));
+  }
+  return results;
+  return {};
+}
diff --git a/mlir/lib/ExecutionEngine/CMakeLists.txt b/mlir/lib/ExecutionEngine/CMakeLists.txt
index dd2ac75b88798..fdeb4dacf9278 100644
--- a/mlir/lib/ExecutionEngine/CMakeLists.txt
+++ b/mlir/lib/ExecutionEngine/CMakeLists.txt
@@ -14,6 +14,7 @@ set(LLVM_OPTIONAL_SOURCES
   RunnerUtils.cpp
   OptUtils.cpp
   JitRunner.cpp
+  LevelZeroRuntimeWrappers.cpp
   SpirvCpuRuntimeWrappers.cpp
   SyclRuntimeWrappers.cpp
   VulkanRuntimeWrappers.cpp
@@ -374,6 +375,15 @@ if(LLVM_ENABLE_PIC)
     )
   endif()
 
+  if(MLIR_ENABLE_SYCL_RUNNER OR MLIR_ENABLE_LEVELZERO_RUNNER)
+    # Both runtimes require LevelZero, so we can find it once.
+    find_package(LevelZeroRuntime)
+
+    if(NOT LevelZeroRuntime_FOUND)
+      message(FATAL_ERROR "LevelZero not found. Please set LEVEL_ZERO_DIR.")
+    endif()
+  endif()
+
   if(MLIR_ENABLE_SYCL_RUNNER)
     find_package(SyclRuntime)
 
@@ -381,12 +391,6 @@ if(LLVM_ENABLE_PIC)
       message(FATAL_ERROR "syclRuntime not found. Please set check oneapi installation and run setvars.sh.")
     endif()
 
-    find_package(LevelZero)
-
-    if(NOT LevelZero_FOUND)
-      message(FATAL_ERROR "LevelZero not found. Please set LEVEL_ZERO_DIR.")
-    endif()
-
     add_mlir_library(mlir_sycl_runtime
       SHARED
       SyclRuntimeWrappers.cpp
@@ -404,9 +408,28 @@ if(LLVM_ENABLE_PIC)
       ${MLIR_INCLUDE_DIRS}
     )
 
-    target_link_libraries(mlir_sycl_runtime PRIVATE LevelZero::LevelZero SyclRuntime::SyclRuntime)
+    target_link_libraries(mlir_sycl_runtime PRIVATE LevelZeroRuntime::LevelZeroRuntime SyclRuntime::SyclRuntime)
+
+    set_property(TARGET mlir_sycl_runtime APPEND PROPERTY BUILD_RPATH "${LevelZeroRuntime_LIBRARIES_DIR}" "${SyclRuntime_LIBRARIES_DIR}")
+  endif()
+
+  if(MLIR_ENABLE_LEVELZERO_RUNNER)
+    add_mlir_library(mlir_levelzero_runtime
+      SHARED
+      LevelZeroRuntimeWrappers.cpp
+
+      EXCLUDE_FROM_LIBMLIR
+    )
+
+    target_compile_options(mlir_levelzero_runtime PUBLIC -fexceptions -frtti)
+
+    target_include_directories(mlir_levelzero_runtime PRIVATE
+      ${MLIR_INCLUDE_DIRS}
+    )
+
+    target_link_libraries(mlir_levelzero_runtime PRIVATE LevelZeroRuntime::LevelZeroRuntime)
 
-    set_property(TARGET mlir_sycl_runtime APPEND PROPERTY BUILD_RPATH "${LevelZero_LIBRARIES_DIR}" "${SyclRuntime_LIBRARIES_DIR}")
+    set_property(TARGET mlir_levelzero_runtime APPEND PROPERTY BUILD_RPATH "${LevelZeroRuntime_LIBRARIES_DIR}")
   endif()
 
   if(MLIR_ENABLE_SPIRV_CPU_RUNNER)
diff --git a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
index f704fbfbe8fff..52162a43aeae3 100644
--- a/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/mlir/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -106,7 +106,7 @@ void ExecutionEngine::dumpToObjectFile(StringRef filename) {
   }
   // Compilation is lazy and it doesn't populate object cache unless requested.
   // In case object dump is requested before cache is populated, we need to
-  // force compilation manually. 
+  // force compilation manually.
   if (cache->isEmpty()) {
     for (std::string &functionName : functionNames) {
       auto result = lookupPacked(functionName);
@@ -400,13 +400,6 @@ ExecutionEngine::create(Operation *m, const ExecutionEngineOptions &options,
     return symbolMap;
   };
   engine->registerSymbols(runtimeSymbolMap);
-
-  // Execute the global constructors from the module being processed.
-  // TODO: Allow JIT initialize for AArch64. Currently there's a bug causing a
-  // crash for AArch64 see related issue #71963.
-  if (!engine->jit->getTargetTriple().isAArch64())
-    cantFail(engine->jit->initialize(engine->jit->getMainJITDylib()));
-
   return std::move(engine);
 }
 
@@ -442,6 +435,7 @@ Expected<void *> ExecutionEngine::lookup(StringRef name) const {
 
 Error ExecutionEngine::invokePacked(StringRef name,
                                     MutableArrayRef<void *> args) {
+  initialize();
   auto expectedFPtr = lookupPacked(name);
   if (!expectedFPtr)
     return expectedFPtr.takeError();
@@ -451,3 +445,13 @@ Error ExecutionEngine::invokePacked(StringRef name,
 
   return Error::success();
 }
+
+void ExecutionEngine::initialize() {
+  if (isInitialized)
+    return;
+  // TODO: Allow JIT initialize for AArch64. Currently there's a bug causing a
+  // crash for AArch64 see related issue #71963.
+  if (!jit->getTargetTriple().isAArch64())
+    cantFail(jit->initialize(jit->getMainJITDylib()));
+  isInitialized = true;
+}
diff --git a/mlir/lib/ExecutionEngine/JitRunner.cpp b/mlir/lib/ExecutionEngine/JitRunner.cpp
index 2107df37d1997..0ada4cc96570a 100644
--- a/mlir/lib/ExecutionEngine/JitRunner.cpp
+++ b/mlir/lib/ExecutionEngine/JitRunner.cpp
@@ -202,6 +202,8 @@ compileAndExecute(Options &options, Operation *module, StringRef entryPoint,
 
   auto engine = std::move(*expectedEngine);
 
+  engine->initialize();
+
   auto expectedFPtr = engine->lookupPacked(entryPoint);
   if (!expectedFPtr)
     return expectedFPtr.takeError();
diff --git a/mlir/lib/ExecutionEngine/LevelZeroRuntimeWrappers.cpp b/mlir/lib/ExecutionEngine/LevelZeroRuntimeWrappers.cpp
new file mode 100644
index 0000000000000..21eaf28c9f214
--- /dev/null
+++ b/mlir/lib/ExecutionEngine/LevelZeroRuntimeWrappers.cpp
@@ -0,0 +1,573 @@
+//===- LevelZeroRuntimeWrappers.cpp - MLIR Level Zero (L0) wrapper library-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements wrappers around the Level Zero (L0) runtime library with C linkage
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Twine.h"
+
+#include "level_zero/ze_api.h"
+#include <cassert>
+#include <deque>
+#include <exception>
+#include <functional>
+#include <iostream>
+#include <limits>
+#include <unordered_set>
+#include <vector>
+
+namespace {
+template <typename F>
+auto catchAll(F &&func) {
+  try {
+    return func();
+  } catch (const std::exception &e) {
+    std::cerr << "An exception was thrown: " << e.what() << std::endl;
+    std::abort();
+  } catch (...) {
+    std::cerr << "An unknown exception was thrown." << std::endl;
+    std::abort();
+  }
+}
+
+#define L0_SAFE_CALL(call)                                                     \
+  {                                                                            \
+    ze_result_t status = (call);                                               \
+    if (status != ZE_RESULT_SUCCESS) {                                         \
+      const char *errorString;                                                 \
+      zeDriverGetLastErrorDescription(NULL, &errorString);                     \
+      std::cerr << "L0 error " << status << ": " << errorString << std::endl;  \
+      std::abort();                                                            \
+    }                                                                          \
+  }
+} // namespace
+
+//===----------------------------------------------------------------------===//
+// L0 RT context & device setters
+//===----------------------------------------------------------------------===//
+
+// Returns the L0 driver handle for the given index. Default index is 0
+// (i.e., returns the first driver handle of the available drivers).
+
+static ze_driver_handle_t getDriver(uint32_t idx = 0) {
+  ze_init_driver_type_desc_t driver_type = {};
+  driver_type.stype = ZE_STRUCTURE_TYPE_INIT_DRIVER_TYPE_DESC;
+  driver_type.flags = ZE_INIT_DRIVER_TYPE_FLAG_GPU;
+  driver_type.pNext = nullptr;
+  uint32_t driverCount{0};
+  thread_local static std::vector<ze_driver_handle_t> drivers;
+  thread_local static bool isDriverInitialised{false};
+  if (isDriverInitialised && idx < drivers.size())
+    return drivers[idx];
+  L0_SAFE_CALL(zeInitDrivers(&driverCount, nullptr, &driver_type));
+  if (!driverCount)
+    throw std::runtime_error("No L0 drivers found.");
+  drivers.resize(driverCount);
+  L0_SAFE_CALL(zeInitDrivers(&driverCount, drivers.data(), &driver_type));
+  if (idx >= driverCount)
+    throw std::runtime_error((llvm::Twine("Requested driver idx out-of-bound, "
+                                          "number of availabe drivers: ") +
+                              std::to_string(driverCount))
+                                 .str());
+  isDriverInitialised = true;
+  return drivers[idx];
+}
+
+static ze_device_handle_t getDevice(const uint32_t driverIdx = 0,
+                                    const int32_t devIdx = 0) {
+  thread_local static ze_device_handle_t l0Device;
+  thread_local int32_t currDevIdx{-1};
+  thread_local uint32_t currDriverIdx{0};
+  if (currDriverIdx == driverIdx && currDevIdx == devIdx)
+    return l0Device;
+  auto driver = getDriver(driverIdx);
+  uint32_t deviceCount{0};
+  L0_SAFE_CALL(zeDeviceGet(driver, &deviceCount, nullptr));
+  if (!deviceCount)
+    throw std::runtime_error("getDevice failed: did not find L0 device.");
+  if (static_cast<int>(deviceCount) < devIdx + 1)
+    throw std::runtime_error("getDevice failed: devIdx out-of-bounds.");
+  std::vector<ze_device_handle_t> devices(deviceCount);
+  L0_SAFE_CALL(zeDeviceGet(driver, &deviceCount, devices.data()));
+  l0Device = devices[devIdx];
+  currDriverIdx = driverIdx;
+  currDevIdx = devIdx;
+  return l0Device;
+}
+
+// Returns the default L0 context of the defult driver.
+static ze_context_handle_t getContext(ze_driver_handle_t driver) {
+  thread_local static ze_context_handle_t context;
+  thread_local static bool isContextInitialised{false};
+  if (isContextInitialised)
+    return context;
+  ze_context_desc_t ctxtDesc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
+  L0_SAFE_CALL(zeContextCreate(driver, &ctxtDesc, &context));
+  isContextInitialised = true;
+  return context;
+}
+
+//===----------------------------------------------------------------------===//
+// L0 RT helper structs
+//===----------------------------------------------------------------------===//
+
+struct ZeContextDeleter {
+  void operator()(ze_context_handle_t ctx) const {
+    if (ctx)
+      L0_SAFE_CALL(zeContextDestroy(ctx));
+  }
+};
+
+struct ZeCommandListDeleter {
+  void operator()(ze_command_list_handle_t cmdList) const {
+    if (cmdList)
+      L0_SAFE_CALL(zeCommandListDestroy(cmdList));
+  }
+};
+using UniqueZeContext =
+    std::unique_ptr<std::remove_pointer<ze_context_handle_t>::type,
+                    ZeContextDeleter>;
+using UniqueZeCommandList =
+    std::unique_ptr<std::remove_pointer<ze_command_list_handle_t>::type,
+                    ZeCommandListDeleter>;
+struct L0RTContextWrapper {
+  ze_driver_handle_t driver{nullptr};
+  ze_device_handle_t device{nullptr};
+  UniqueZeContext context;
+  // Usually, one immediate command list with ordinal 0 suffices for
+  // both copy and compute ops, but leaves HW underutilized.
+  UniqueZeCommandList immCmdListCompute;
+  // Copy engines can be used for both memcpy and memset, but
+  // they have limitations for memset pattern size (e.g., 1 byte).
+  UniqueZeCommandList immCmdListCopy;
+  uint32_t copyEngineMaxMemoryFillPatternSize{-1u};
+
+  L0RTContextWrapper() = default;
+  L0RTContextWrapper(const uint32_t driverIdx = 0, const int32_t devIdx = 0)
+      : driver(getDriver(driverIdx)), device(getDevice(devIdx)) {
+    // Create context
+    ze_context_handle_t ctx = getContext(driver);
+    context.reset(ctx);
+
+    // Determine ordinals
+    uint32_t computeEngineOrdinal = -1u, copyEngineOrdinal = -1u;
+    ze_device_properties_t deviceProperties{};
+    L0_SAFE_CALL(zeDeviceGetProperties(device, &deviceProperties));
+    uint32_t queueGroupCount = 0;
+    L0_SAFE_CALL(zeDeviceGetCommandQueueGroupProperties(
+        device, &queueGroupCount, nullptr));
+    std::vector<ze_command_queue_group_properties_t> queueGroupProperties(
+        queueGroupCount);
+    L0_SAFE_CALL(zeDeviceGetCommandQueueGroupProperties(
+        device, &queueGroupCount, queueGroupProperties.data()));
+
+    for (uint32_t queueGroupIdx = 0; queueGroupIdx < queueGroupCount;
+         ++queueGroupIdx) {
+      const auto &group = queueGroupProperties[queueGroupIdx];
+      if (group.flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE)
+        computeEngineOrdinal = queueGroupIdx;
+      else if (group.flags & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COPY) {
+        copyEngineOrdinal = queueGroupIdx;
+        copyEngineMaxMemoryFillPatternSize = group.maxMemoryFillPatternSize;
+      }
+      if (copyEngineOrdinal != -1u && computeEngineOrdinal != -1u)
+        break;
+    }
+
+    // Fallback to the default queue if no dedicated copy queue is available.
+    if (copyEngineOrdinal == -1u)
+      copyEngineOrdinal = computeEngineOrdinal;
+
+    assert(copyEngineOrdinal != -1u && computeEngineOrdinal != -1u &&
+           "Expected two engines to be available.");
+
+    // Create copy command list
+    ze_command_queue_desc_t cmdQueueDesc{
+        ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC,
+        nullptr,
+        copyEngineOrdinal, // ordinal
+        0,                 // index (assume one physical engine in the group)
+        0,                 // flags
+        ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS,
+        ZE_COMMAND_QUEUE_PRIORITY_NORMAL};
+
+    ze_command_list_handle_t rawCmdListCopy = nullptr;
+    L0_SAFE_CALL(zeCommandListCreateImmediate(context.get(), device,
+                                              &cmdQueueDesc, &rawCmdListCopy));
+    immCmdListCopy.reset(rawCmdListCopy);
+
+    // Create compute command list
+    cmdQueueDesc.ordinal = computeEngineOrdinal;
+    ze_command_list_handle_t rawCmdListCompute = nullptr;
+    L0_SAFE_CALL(zeCommandListCreateImmediate(
+        context.get(), device, &cmdQueueDesc, &rawCmdListCompute));
+    immCmdListCompute.reset(rawCmdListCompute);
+  }
+  L0RTContextWrapper(const L0RTContextWrapper &) = delete;
+  L0RTContextWrapper &operator=(const L0RTContextWrapper &) = delete;
+  // Allow move
+  L0RTContextWrapper(L0RTContextWrapper &&) noexcept = default;
+  L0RTContextWrapper &operator=(L0RTContextWrapper &&) noexcept = default;
+  ~L0RTContextWrapper() = default;
+};
+
+struct ZeEventDeleter {
+  void operator()(ze_event_handle_t event) const {
+    if (event)
+      L0_SAFE_CALL(zeEventDestroy(event));
+  }
+};
+
+struct ZeEventPoolDeleter {
+  void operator()(ze_event_pool_handle_t pool) const {
+    if (pool)
+      L0_SAFE_CALL(zeEventPoolDestroy(pool));
+  }
+};
+
+using UniqueZeEvent =
+    std::unique_ptr<std::remove_pointer<ze_event_handle_t>::type,
+                    ZeEventDeleter>;
+using UniqueZeEventPool =
+    std::unique_ptr<std::remove_pointer<ze_event_pool_handle_t>::type,
+                    ZeEventPoolDeleter>;
+
+// L0 only supports pre-determined sizes of event pools,
+// implement a runtime data structure to avoid running out of events.
+
+struct DynamicEventPool {
+  constexpr static size_t numEventsPerPool{128};
+
+  std::vector<UniqueZeEventPool> eventPools;
+  std::vector<UniqueZeEvent> availableEvents;
+  std::unordered_map<ze_event_handle_t, UniqueZeEvent> takenEvents;
+
+  // Limit the number of events to avoid running out of memory.
+  // The limit is set to 32K events, which should be sufficient for most use
+  // cases.
+  size_t maxEventsCount{32768}; // 32K events
+  size_t currentEventsLimit{0};
+  size_t currentEventsCnt{0};
+  L0RTContextWrapper *rtCtx;
+
+  DynamicEventPool(L0RTContextWrapper *rtCtx) : rtCtx(rtCtx) {
+    createNewPool(numEventsPerPool);
+  }
+
+  DynamicEventPool(const DynamicEventPool &) = delete;
+  DynamicEventPool &operator=(const DynamicEventPool &) = delete;
+
+  // Allow move
+  DynamicEventPool(DynamicEventPool &&) noexcept = default;
+  DynamicEventPool &operator=(DynamicEventPool &&) noexcept = default;
+
+  ~DynamicEventPool() {
+    assert(takenEvents.empty() && "Some events were not released");
+  }
+
+  void createNewPool(size_t numEvents) {
+    ze_event_pool_desc_t eventPoolDesc = {};
+    eventPoolDesc.flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
+    eventPoolDesc.count = numEvents;
+
+    ze_event_pool_handle_t rawPool = nullptr;
+    L0_SAFE_CALL(zeEventPoolCreate(rtCtx->context.get(), &eventPoolDesc, 1,
+                                   &rtCtx->device, &rawPool));
+
+    eventPools.emplace_back(UniqueZeEventPool(rawPool));
+    currentEventsLimit += numEvents;
+  }
+
+  ze_event_handle_t takeEvent() {
+    ze_event_handle_t rawEvent = nullptr;
+
+    if (!availableEvents.empty()) {
+      // Reuse one
+      auto uniqueEvent = std::move(availableEvents.back());
+      availableEvents.pop_back();
+      rawEvent = uniqueEvent.get();
+      takenEvents[rawEvent] = std::move(uniqueEvent);
+    } else {
+      if (currentEventsCnt >= maxEventsCount) {
+        throw std::runtime_error("DynamicEventPool: reached max events limit");
+      }
+      if (currentEventsCnt == currentEventsLimit)
+        createNewPool(numEventsPerPool);
+
+      ze_event_desc_t eventDesc = {
+          ZE_STRUCTURE_TYPE_EVENT_DESC, nullptr,
+          static_cast<uint32_t>(currentEventsCnt % numEventsPerPool),
+          ZE_EVENT_SCOPE_FLAG_DEVICE, ZE_EVENT_SCOPE_FLAG_HOST};
+
+      ze_event_handle_t newEvent = nullptr;
+      L0_SAFE_CALL(
+          zeEventCreate(eventPools.back().get(), &eventDesc, &newEvent));
+
+      takenEvents[newEvent] = UniqueZeEvent(newEvent);
+      rawEvent = newEvent;
+      currentEventsCnt++;
+    }
+
+    return rawEvent;
+  }
+
+  void releaseEvent(ze_event_handle_t event) {
+    auto it = takenEvents.find(event);
+    assert(it != takenEvents.end() &&
+           "Attempting to release unknown or already released event");
+
+    L0_SAFE_CALL(zeEventHostReset(event));
+    availableEvents.emplace_back(std::move(it->second));
+    takenEvents.erase(it);
+  }
+};
+
+L0RTContextWrapper &getRtContext() {
+  thread_local static L0RTContextWrapper rtContext(0);
+  return rtContext;
+}
+
+DynamicEventPool &getDynamicEventPool() {
+  thread_local static DynamicEventPool dynEventPool{&getRtContext()};
+  return dynEventPool;
+}
+
+struct StreamWrapper {
+  // avoid event pointer invalidations
+  std::deque<ze_event_handle_t> implicitEventStack;
+  DynamicEventPool &dynEventPool;
+
+  StreamWrapper(DynamicEventPool &dynEventPool) : dynEventPool(dynEventPool) {}
+  ~StreamWrapper() { sync(); }
+
+  ze_event_handle_t *getLastImplicitEventPtr() {
+    // Assume current implicit events will not be used after `sync`.
+    return implicitEventStack.size() ? &implicitEventStack.back() : nullptr;
+  }
+
+  void sync(ze_event_handle_t explicitEvent = nullptr) {
+    ze_event_handle_t syncEvent{nullptr};
+    if (!explicitEvent) {
+      ze_event_handle_t *lastImplicitEventPtr = getLastImplicitEventPtr();
+      syncEvent = lastImplicitEventPtr ? *lastImplicitEventPtr : nullptr;
+    } else {
+      syncEvent = explicitEvent;
+    }
+    if (syncEvent)
+      L0_SAFE_CALL(zeEventHostSynchronize(
+          syncEvent, std::numeric_limits<uint64_t>::max()));
+    // All of the "implicit" events were signaled and are of no use, release
+    // them. "explicit" event must be "released" via mgpuEventDestroy
+    for (auto event : implicitEventStack)
+      dynEventPool.releaseEvent(event);
+    implicitEventStack.clear();
+  }
+
+  template <typename Func>
+  void enqueueOp(Func &&op) {
+    ze_event_handle_t newImplicitEvent = dynEventPool.takeEvent();
+    ze_event_handle_t *lastImplicitEventPtr = getLastImplicitEventPtr();
+    const uint32_t numWaitEvents = lastImplicitEventPtr ? 1 : 0;
+    std::forward<Func>(op)(newImplicitEvent, numWaitEvents,
+                           lastImplicitEventPtr);
+    implicitEventStack.push_back(newImplicitEvent);
+  }
+};
+
+static ze_module_handle_t loadModule(const void *data, size_t dataSize) {
+  assert(data);
+  ze_module_handle_t zeModule;
+  ze_module_desc_t desc = {ZE_STRUCTURE_TYPE_MODULE_DESC,
+                           nullptr,
+                           ZE_MODULE_FORMAT_IL_SPIRV,
+                           dataSize,
+                           (const uint8_t *)data,
+                           nullptr,
+                           nullptr};
+  ze_module_build_log_handle_t buildLogHandle;
+  ze_result_t result =
+      zeModuleCreate(getRtContext().context.get(), getRtContext().device, &desc,
+                     &zeModule, &buildLogHandle);
+  if (result != ZE_RESULT_SUCCESS) {
+    std::cerr << "Error creating module, error code: " << result << std::endl;
+    size_t logSize = 0;
+    L0_SAFE_CALL(zeModuleBuildLogGetString(buildLogHandle, &logSize, nullptr));
+    std::string buildLog(" ", logSize);
+    L0_SAFE_CALL(
+        zeModuleBuildLogGetString(buildLogHandle, &logSize, buildLog.data()));
+    std::cerr << "Build log:\n" << buildLog << std::endl;
+    std::abort();
+  }
+  return zeModule;
+}
+
+//===----------------------------------------------------------------------===//
+// L0 Wrappers definition
+//===----------------------------------------------------------------------===//
+
+extern "C" StreamWrapper *mgpuStreamCreate() {
+  return new StreamWrapper(getDynamicEventPool());
+}
+
+extern "C" void mgpuStreamSynchronize(StreamWrapper *stream) {
+  if (stream)
+    stream->sync();
+}
+
+extern "C" void mgpuStreamDestroy(StreamWrapper *stream) { delete stream; }
+
+extern "C" void mgpuStreamWaitEvent(StreamWrapper *stream,
+                                    ze_event_handle_t event) {
+  assert(stream && "Invalid stream");
+  assert(event && "Invalid event");
+  stream->sync(event);
+}
+
+extern "C" ze_event_handle_t mgpuEventCreate() {
+  return getDynamicEventPool().takeEvent();
+}
+
+extern "C" void mgpuEventDestroy(ze_event_handle_t event) {
+  return getDynamicEventPool().releaseEvent(event);
+}
+
+extern "C" void mgpuEventSynchronize(ze_event_handle_t event) {
+  L0_SAFE_CALL(
+      zeEventHostSynchronize(event, std::numeric_limits<uint64_t>::max()));
+  L0_SAFE_CALL(zeEventHostReset(event));
+}
+
+extern "C" void mgpuEventRecord(ze_event_handle_t event,
+                                StreamWrapper *stream) {
+  L0_SAFE_CALL(zeCommandListAppendSignalEvent(
+      getRtContext().immCmdListCopy.get(), event));
+  L0_SAFE_CALL(zeCommandListAppendSignalEvent(
+      getRtContext().immCmdListCompute.get(), event));
+}
+
+extern "C" void *mgpuMemAlloc(uint64_t size, StreamWrapper *stream,
+                              bool isShared) {
+  return catchAll([&]() {
+    void *memPtr = nullptr;
+    constexpr size_t alignment{64};
+    ze_device_mem_alloc_desc_t deviceDesc = {};
+    deviceDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC;
+    if (isShared) {
+      ze_host_mem_alloc_desc_t hostDesc = {};
+      hostDesc.stype = ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC;
+      L0_SAFE_CALL(zeMemAllocShared(getRtContext().context.get(), &deviceDesc,
+                                    &hostDesc, size, alignment,
+                                    getRtContext().device, &memPtr));
+    } else {
+      L0_SAFE_CALL(zeMemAllocDevice(getRtContext().context.get(), &deviceDesc,
+                                    size, alignment, getRtContext().device,
+                                    &memPtr));
+    }
+    if (!memPtr)
+      throw std::runtime_error("mem allocation failed!");
+    return memPtr;
+  });
+}
+
+extern "C" void mgpuMemFree(void *ptr, StreamWrapper *stream) {
+  stream->sync();
+  if (ptr)
+    L0_SAFE_CALL(zeMemFree(getRtContext().context.get(), ptr));
+}
+
+extern "C" void mgpuMemcpy(void *dst, void *src, size_t sizeBytes,
+                           StreamWrapper *stream) {
+  stream->enqueueOp([&](ze_event_handle_t newEvent, uint32_t numWaitEvents,
+                        ze_event_handle_t *waitEvents) {
+    L0_SAFE_CALL(zeCommandListAppendMemoryCopy(
+        getRtContext().immCmdListCopy.get(), dst, src, sizeBytes, newEvent,
+        numWaitEvents, waitEvents));
+  });
+}
+
+template <typename PATTERN_TYPE>
+void mgpuMemset(void *dst, PATTERN_TYPE value, size_t count,
+                StreamWrapper *stream) {
+  L0RTContextWrapper &rtContext = getRtContext();
+  auto listType =
+      rtContext.copyEngineMaxMemoryFillPatternSize >= sizeof(PATTERN_TYPE)
+          ? rtContext.immCmdListCopy.get()
+          : rtContext.immCmdListCompute.get();
+  stream->enqueueOp([&](ze_event_handle_t newEvent, uint32_t numWaitEvents,
+                        ze_event_handle_t *waitEvents) {
+    L0_SAFE_CALL(zeCommandListAppendMemoryFill(
+        listType, dst, &value, sizeof(PATTERN_TYPE),
+        count * sizeof(PATTERN_TYPE), newEvent, numWaitEvents, waitEvents));
+  });
+}
+extern "C" void mgpuMemset32(void *dst, unsigned int value, size_t count,
+                             StreamWrapper *stream) {
+  mgpuMemset<unsigned int>(dst, value, count, stream);
+}
+
+extern "C" void mgpuMemset16(void *dst, unsigned short value, size_t count,
+                             StreamWrapper *stream) {
+  mgpuMemset<unsigned short>(dst, value, count, stream);
+}
+
+extern "C" ze_module_handle_t mgpuModuleLoad(const void *data,
+                                             size_t gpuBlobSize) {
+  return catchAll([&]() { return loadModule(data, gpuBlobSize); });
+}
+
+extern "C" ze_kernel_handle_t mgpuModuleGetFunction(ze_module_handle_t module,
+                                                    const char *name) {
+  assert(module && name);
+  ze_kernel_handle_t zeKernel;
+  ze_kernel_desc_t desc = {};
+  desc.pKernelName = name;
+  L0_SAFE_CALL(zeKernelCreate(module, &desc, &zeKernel));
+  return zeKernel;
+}
+
+extern "C" void mgpuLaunchKernel(ze_kernel_handle_t kernel, size_t gridX,
+                                 size_t gridY, size_t gridZ, size_t blockX,
+                                 size_t blockY, size_t blockZ,
+                                 size_t sharedMemBytes, StreamWrapper *stream,
+                                 void **params, void ** /*extra*/,
+                                 size_t paramsCount) {
+
+  if (sharedMemBytes > 0) {
+    paramsCount = paramsCount - 1; // Last param is shared memory size
+    L0_SAFE_CALL(
+        zeKernelSetArgumentValue(kernel, paramsCount, sharedMemBytes, nullptr));
+  }
+  for (size_t i = 0; i < paramsCount; ++i)
+    L0_SAFE_CALL(zeKernelSetArgumentValue(kernel, static_cast<uint32_t>(i),
+                                          sizeof(void *), params[i]));
+  L0_SAFE_CALL(zeKernelSetGroupSize(kernel, blockX, blockY, blockZ));
+  ze_group_count_t dispatch;
+  dispatch.groupCountX = static_cast<uint32_t>(gridX);
+  dispatch.groupCountY = static_cast<uint32_t>(gridY);
+  dispatch.groupCountZ = static_cast<uint32_t>(gridZ);
+  stream->enqueueOp([&](ze_event_handle_t newEvent, uint32_t numWaitEvents,
+                        ze_event_handle_t *waitEvents) {
+    L0_SAFE_CALL(zeCommandListAppendLaunchKernel(
+        getRtContext().immCmdListCompute.get(), kernel, &dispatch, newEvent,
+        numWaitEvents, waitEvents));
+  });
+}
+
+extern "C" void mgpuModuleUnload(ze_module_handle_t module) {
+  L0_SAFE_CALL(zeModuleDestroy(module));
+}
+
+extern "C" void mgpuSetDefaultDevice(int32_t devIdx) {
+  catchAll([&]() {
+    // For now, a user must ensure that streams and events complete
+    // and are destroyed before switching a device.
+    getRtContext() = L0RTContextWrapper(devIdx);
+    getDynamicEventPool() = DynamicEventPool(&getRtContext());
+  });
+}
diff --git a/mlir/lib/IR/Block.cpp b/mlir/lib/IR/Block.cpp
index 57825d9b42178..27b47e2d2653d 100644
--- a/mlir/lib/IR/Block.cpp
+++ b/mlir/lib/IR/Block.cpp
@@ -251,6 +251,16 @@ bool Block::mightHaveTerminator() {
   return !empty() && back().mightHaveTrait<OpTrait::IsTerminator>();
 }
 
+iterator_range<Block::iterator> Block::without_terminator_impl() {
+  // Note: When the op is unregistered, we do not know for sure if the last
+  // op is a terminator. In that case, we include it in `without_terminator`,
+  // but that decision is somewhat arbitrary.
+  if (!back().hasTrait<OpTrait::IsTerminator>())
+    return {begin(), end()};
+  auto endIt = --end();
+  return {begin(), endIt};
+}
+
 // Indexed successor access.
 unsigned Block::getNumSuccessors() {
   return empty() ? 0 : back().getNumSuccessors();
diff --git a/mlir/lib/IR/BuiltinAttributes.cpp b/mlir/lib/IR/BuiltinAttributes.cpp
index fd898b7493c7f..6f880f810d651 100644
--- a/mlir/lib/IR/BuiltinAttributes.cpp
+++ b/mlir/lib/IR/BuiltinAttributes.cpp
@@ -19,6 +19,7 @@
 #include "mlir/IR/Types.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/Endian.h"
 #include <optional>
 
@@ -1119,9 +1120,8 @@ static bool isValidIntOrFloat(Type type, int64_t dataEltSize, bool isInt,
   auto denseEltBitWidth = getDenseElementBitWidth(type);
   auto dataSize = static_cast<size_t>(dataEltSize * CHAR_BIT);
   if (denseEltBitWidth != dataSize) {
-    LLVM_DEBUG(llvm::dbgs() << "expected dense element bit width "
-                            << denseEltBitWidth << " to match data size "
-                            << dataSize << " for type " << type << "\n");
+    LDBG() << "expected dense element bit width " << denseEltBitWidth
+           << " to match data size " << dataSize << " for type " << type;
     return false;
   }
 
@@ -1129,9 +1129,7 @@ static bool isValidIntOrFloat(Type type, int64_t dataEltSize, bool isInt,
   if (!isInt) {
     bool valid = llvm::isa<FloatType>(type);
     if (!valid)
-      LLVM_DEBUG(llvm::dbgs()
-                 << "expected float type when isInt is false, but found "
-                 << type << "\n");
+      LDBG() << "expected float type when isInt is false, but found " << type;
     return valid;
   }
   if (type.isIndex())
@@ -1139,9 +1137,7 @@ static bool isValidIntOrFloat(Type type, int64_t dataEltSize, bool isInt,
 
   auto intType = llvm::dyn_cast<IntegerType>(type);
   if (!intType) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "expected integer type when isInt is true, but found " << type
-               << "\n");
+    LDBG() << "expected integer type when isInt is true, but found " << type;
     return false;
   }
 
@@ -1151,8 +1147,7 @@ static bool isValidIntOrFloat(Type type, int64_t dataEltSize, bool isInt,
 
   bool valid = intType.isSigned() == isSigned;
   if (!valid)
-    LLVM_DEBUG(llvm::dbgs() << "expected signedness " << isSigned
-                            << " to match type " << type << "\n");
+    LDBG() << "expected signedness " << isSigned << " to match type " << type;
   return valid;
 }
 
diff --git a/mlir/lib/IR/CMakeLists.txt b/mlir/lib/IR/CMakeLists.txt
index 3ef69cea18f0a..d95bdc957e3c2 100644
--- a/mlir/lib/IR/CMakeLists.txt
+++ b/mlir/lib/IR/CMakeLists.txt
@@ -33,6 +33,7 @@ add_mlir_library(MLIRIR
   PatternMatch.cpp
   Region.cpp
   RegionKindInterface.cpp
+  Remarks.cpp
   SymbolTable.cpp
   TensorEncoding.cpp
   Types.cpp
diff --git a/mlir/lib/IR/Dialect.cpp b/mlir/lib/IR/Dialect.cpp
index f84fe89157074..952619b4477a7 100644
--- a/mlir/lib/IR/Dialect.cpp
+++ b/mlir/lib/IR/Dialect.cpp
@@ -19,7 +19,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVectorExtras.h"
 #include "llvm/ADT/Twine.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/Regex.h"
 #include <memory>
 
@@ -104,14 +104,8 @@ void Dialect::addInterface(std::unique_ptr<DialectInterface> interface) {
 
   auto it = registeredInterfaces.try_emplace(interface->getID(),
                                              std::move(interface));
-  (void)it;
-  LLVM_DEBUG({
-    if (!it.second) {
-      llvm::dbgs() << "[" DEBUG_TYPE
-                      "] repeated interface registration for dialect "
-                   << getNamespace();
-    }
-  });
+  if (!it.second)
+    LDBG() << "repeated interface registration for dialect " << getNamespace();
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index 2d5381d43f863..1fa04ed8e738f 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -25,12 +25,13 @@
 #include "mlir/IR/Location.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/OperationSupport.h"
+#include "mlir/IR/Remarks.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Mutex.h"
 #include "llvm/Support/RWMutex.h"
@@ -133,6 +134,11 @@ class MLIRContextImpl {
   //===--------------------------------------------------------------------===//
   DiagnosticEngine diagEngine;
 
+  //===--------------------------------------------------------------------===//
+  // Remark
+  //===--------------------------------------------------------------------===//
+  std::unique_ptr<remark::detail::RemarkEngine> remarkEngine;
+
   //===--------------------------------------------------------------------===//
   // Options
   //===--------------------------------------------------------------------===//
@@ -387,6 +393,19 @@ bool MLIRContext::hasActionHandler() { return (bool)getImpl().actionHandler; }
 /// Returns the diagnostic engine for this context.
 DiagnosticEngine &MLIRContext::getDiagEngine() { return getImpl().diagEngine; }
 
+//===----------------------------------------------------------------------===//
+// Remark Handlers
+//===----------------------------------------------------------------------===//
+
+void MLIRContext::setRemarkEngine(
+    std::unique_ptr<remark::detail::RemarkEngine> engine) {
+  getImpl().remarkEngine = std::move(engine);
+}
+
+remark::detail::RemarkEngine *MLIRContext::getRemarkEngine() {
+  return getImpl().remarkEngine.get();
+}
+
 //===----------------------------------------------------------------------===//
 // Dialect and Operation Registration
 //===----------------------------------------------------------------------===//
@@ -455,8 +474,7 @@ MLIRContext::getOrLoadDialect(StringRef dialectNamespace, TypeID dialectID,
   auto dialectIt = impl.loadedDialects.try_emplace(dialectNamespace, nullptr);
 
   if (dialectIt.second) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "Load new dialect in Context " << dialectNamespace << "\n");
+    LDBG() << "Load new dialect in Context " << dialectNamespace;
 #ifndef NDEBUG
     if (impl.multiThreadedExecutionContext != 0)
       llvm::report_fatal_error(
@@ -525,8 +543,7 @@ DynamicDialect *MLIRContext::getOrLoadDynamicDialect(
                              "' has already been registered");
   }
 
-  LLVM_DEBUG(llvm::dbgs() << "Load new dynamic dialect in Context "
-                          << dialectNamespace << "\n");
+  LDBG() << "Load new dynamic dialect in Context " << dialectNamespace;
 #ifndef NDEBUG
   if (impl.multiThreadedExecutionContext != 0)
     llvm::report_fatal_error(
@@ -1192,11 +1209,10 @@ willBeValidAffineMap(unsigned dimCount, unsigned symbolCount,
   getMaxDimAndSymbol(ArrayRef<ArrayRef<AffineExpr>>(results), maxDimPosition,
                      maxSymbolPosition);
   if ((maxDimPosition >= dimCount) || (maxSymbolPosition >= symbolCount)) {
-    LLVM_DEBUG(
-        llvm::dbgs()
+    LDBG()
         << "maximum dimensional identifier position in result expression must "
            "be less than `dimCount` and maximum symbolic identifier position "
-           "in result expression must be less than `symbolCount`\n");
+           "in result expression must be less than `symbolCount`";
     return false;
   }
   return true;
diff --git a/mlir/lib/IR/Remarks.cpp b/mlir/lib/IR/Remarks.cpp
new file mode 100644
index 0000000000000..78c964427868f
--- /dev/null
+++ b/mlir/lib/IR/Remarks.cpp
@@ -0,0 +1,279 @@
+//===- Remarks.cpp - MLIR Remarks -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/Remarks.h"
+
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/Value.h"
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+
+using namespace mlir::remark::detail;
+
+//------------------------------------------------------------------------------
+// Remark
+//------------------------------------------------------------------------------
+
+Remark::Arg::Arg(llvm::StringRef k, Value v) : key(k) {
+  llvm::raw_string_ostream os(val);
+  os << v;
+}
+
+Remark::Arg::Arg(llvm::StringRef k, Type t) : key(k) {
+  llvm::raw_string_ostream os(val);
+  os << t;
+}
+
+void Remark::insert(llvm::StringRef s) { args.emplace_back(s); }
+void Remark::insert(Arg a) { args.push_back(std::move(a)); }
+
+// Simple helper to print key=val list (sorted).
+static void printArgs(llvm::raw_ostream &os, llvm::ArrayRef<Remark::Arg> args) {
+  if (args.empty())
+    return;
+
+  llvm::SmallVector<Remark::Arg, 8> sorted(args.begin(), args.end());
+  llvm::sort(sorted, [](const Remark::Arg &a, const Remark::Arg &b) {
+    return a.key < b.key;
+  });
+
+  for (size_t i = 0; i < sorted.size(); ++i) {
+    const auto &a = sorted[i];
+    os << a.key << "=";
+
+    llvm::StringRef val(a.val);
+    bool needsQuote = val.contains(' ') || val.contains(',') ||
+                      val.contains('{') || val.contains('}');
+    if (needsQuote)
+      os << '"' << val << '"';
+    else
+      os << val;
+
+    if (i + 1 < sorted.size())
+      os << ", ";
+  }
+}
+
+/// Print the remark to the given output stream.
+/// Example output:
+// clang-format off
+/// [Missed] Category: Loop | Pass:Unroller |  Function=main | Reason="tripCount=4 < threshold=256"
+/// [Failure] LoopOptimizer | Reason="failed due to unsupported pattern"
+// clang-format on
+void Remark::print(llvm::raw_ostream &os, bool printLocation) const {
+  // Header: [Type] pass:remarkName
+  StringRef type = getRemarkTypeString();
+  StringRef categoryName = getFullCategoryName();
+  StringRef name = remarkName;
+
+  os << '[' << type << "] ";
+  os << name << " | ";
+  if (!categoryName.empty())
+    os << "Category:" << categoryName << " | ";
+  if (!functionName.empty())
+    os << "Function=" << getFunction() << " | ";
+
+  if (printLocation) {
+    if (auto flc = mlir::dyn_cast<mlir::FileLineColLoc>(getLocation()))
+      os << " @" << flc.getFilename() << ":" << flc.getLine() << ":"
+         << flc.getColumn();
+  }
+
+  printArgs(os, getArgs());
+}
+
+std::string Remark::getMsg() const {
+  std::string s;
+  llvm::raw_string_ostream os(s);
+  print(os);
+  os.flush();
+  return s;
+}
+
+llvm::StringRef Remark::getRemarkTypeString() const {
+  switch (remarkKind) {
+  case RemarkKind::RemarkUnknown:
+    return "Unknown";
+  case RemarkKind::RemarkPassed:
+    return "Passed";
+  case RemarkKind::RemarkMissed:
+    return "Missed";
+  case RemarkKind::RemarkFailure:
+    return "Failure";
+  case RemarkKind::RemarkAnalysis:
+    return "Analysis";
+  }
+  llvm_unreachable("Unknown remark kind");
+}
+
+llvm::remarks::Type Remark::getRemarkType() const {
+  switch (remarkKind) {
+  case RemarkKind::RemarkUnknown:
+    return llvm::remarks::Type::Unknown;
+  case RemarkKind::RemarkPassed:
+    return llvm::remarks::Type::Passed;
+  case RemarkKind::RemarkMissed:
+    return llvm::remarks::Type::Missed;
+  case RemarkKind::RemarkFailure:
+    return llvm::remarks::Type::Failure;
+  case RemarkKind::RemarkAnalysis:
+    return llvm::remarks::Type::Analysis;
+  }
+  llvm_unreachable("Unknown remark kind");
+}
+
+llvm::remarks::Remark Remark::generateRemark() const {
+  auto locLambda = [&]() -> llvm::remarks::RemarkLocation {
+    if (auto flc = dyn_cast<FileLineColLoc>(getLocation()))
+      return {flc.getFilename(), flc.getLine(), flc.getColumn()};
+    return {"<unknown file>", 0, 0};
+  };
+
+  llvm::remarks::Remark r; // The result.
+  r.RemarkType = getRemarkType();
+  r.RemarkName = getRemarkName();
+  // MLIR does not use passes; instead, it has categories and sub-categories.
+  r.PassName = getFullCategoryName();
+  r.FunctionName = getFunction();
+  r.Loc = locLambda();
+  for (const Remark::Arg &arg : getArgs()) {
+    r.Args.emplace_back();
+    r.Args.back().Key = arg.key;
+    r.Args.back().Val = arg.val;
+  }
+  return r;
+}
+
+//===----------------------------------------------------------------------===//
+// InFlightRemark
+//===----------------------------------------------------------------------===//
+
+InFlightRemark::~InFlightRemark() {
+  if (remark && owner)
+    owner->report(std::move(*remark));
+  owner = nullptr;
+}
+
+//===----------------------------------------------------------------------===//
+// Remark Engine
+//===----------------------------------------------------------------------===//
+
+template <typename RemarkT, typename... Args>
+InFlightRemark RemarkEngine::makeRemark(Args &&...args) {
+  static_assert(std::is_base_of_v<Remark, RemarkT>,
+                "RemarkT must derive from Remark");
+  return InFlightRemark(*this,
+                        std::make_unique<RemarkT>(std::forward<Args>(args)...));
+}
+
+template <typename RemarkT>
+InFlightRemark
+RemarkEngine::emitIfEnabled(Location loc, RemarkOpts opts,
+                            bool (RemarkEngine::*isEnabled)(StringRef) const) {
+  return (this->*isEnabled)(opts.categoryName) ? makeRemark<RemarkT>(loc, opts)
+                                               : InFlightRemark{};
+}
+
+bool RemarkEngine::isMissedOptRemarkEnabled(StringRef categoryName) const {
+  return missFilter && missFilter->match(categoryName);
+}
+
+bool RemarkEngine::isPassedOptRemarkEnabled(StringRef categoryName) const {
+  return passedFilter && passedFilter->match(categoryName);
+}
+
+bool RemarkEngine::isAnalysisOptRemarkEnabled(StringRef categoryName) const {
+  return analysisFilter && analysisFilter->match(categoryName);
+}
+
+bool RemarkEngine::isFailedOptRemarkEnabled(StringRef categoryName) const {
+  return failedFilter && failedFilter->match(categoryName);
+}
+
+InFlightRemark RemarkEngine::emitOptimizationRemark(Location loc,
+                                                    RemarkOpts opts) {
+  return emitIfEnabled<OptRemarkPass>(loc, opts,
+                                      &RemarkEngine::isPassedOptRemarkEnabled);
+}
+
+InFlightRemark RemarkEngine::emitOptimizationRemarkMiss(Location loc,
+                                                        RemarkOpts opts) {
+  return emitIfEnabled<OptRemarkMissed>(
+      loc, opts, &RemarkEngine::isMissedOptRemarkEnabled);
+}
+
+InFlightRemark RemarkEngine::emitOptimizationRemarkFailure(Location loc,
+                                                           RemarkOpts opts) {
+  return emitIfEnabled<OptRemarkFailure>(
+      loc, opts, &RemarkEngine::isFailedOptRemarkEnabled);
+}
+
+InFlightRemark RemarkEngine::emitOptimizationRemarkAnalysis(Location loc,
+                                                            RemarkOpts opts) {
+  return emitIfEnabled<OptRemarkAnalysis>(
+      loc, opts, &RemarkEngine::isAnalysisOptRemarkEnabled);
+}
+
+//===----------------------------------------------------------------------===//
+// RemarkEngine
+//===----------------------------------------------------------------------===//
+
+void RemarkEngine::report(const Remark &&remark) {
+  // Stream the remark
+  if (remarkStreamer)
+    remarkStreamer->streamOptimizationRemark(remark);
+
+  // Print using MLIR's diagnostic
+  if (printAsEmitRemarks)
+    emitRemark(remark.getLocation(), remark.getMsg());
+}
+
+RemarkEngine::~RemarkEngine() {
+  if (remarkStreamer)
+    remarkStreamer->finalize();
+}
+
+llvm::LogicalResult
+RemarkEngine::initialize(std::unique_ptr<MLIRRemarkStreamerBase> streamer,
+                         std::string *errMsg) {
+  // If you need to validate categories/filters, do so here and set errMsg.
+  remarkStreamer = std::move(streamer);
+  return success();
+}
+
+RemarkEngine::RemarkEngine(bool printAsEmitRemarks,
+                           const RemarkCategories &cats)
+    : printAsEmitRemarks(printAsEmitRemarks) {
+  if (cats.passed)
+    passedFilter = llvm::Regex(cats.passed.value());
+  if (cats.missed)
+    missFilter = llvm::Regex(cats.missed.value());
+  if (cats.analysis)
+    analysisFilter = llvm::Regex(cats.analysis.value());
+  if (cats.failed)
+    failedFilter = llvm::Regex(cats.failed.value());
+}
+
+llvm::LogicalResult mlir::remark::enableOptimizationRemarks(
+    MLIRContext &ctx,
+    std::unique_ptr<remark::detail::MLIRRemarkStreamerBase> streamer,
+    const remark::RemarkCategories &cats, bool printAsEmitRemarks) {
+  auto engine =
+      std::make_unique<remark::detail::RemarkEngine>(printAsEmitRemarks, cats);
+
+  std::string errMsg;
+  if (failed(engine->initialize(std::move(streamer), &errMsg))) {
+    llvm::report_fatal_error(
+        llvm::Twine("Failed to initialize remark engine. Error: ") + errMsg);
+  }
+  ctx.setRemarkEngine(std::move(engine));
+
+  return success();
+}
diff --git a/mlir/lib/Interfaces/SideEffectInterfaces.cpp b/mlir/lib/Interfaces/SideEffectInterfaces.cpp
index 266f6dbacce89..b5a6888e5e1a4 100644
--- a/mlir/lib/Interfaces/SideEffectInterfaces.cpp
+++ b/mlir/lib/Interfaces/SideEffectInterfaces.cpp
@@ -304,6 +304,11 @@ template bool
 mlir::hasEffect<BlockArgument, MemoryEffects::Write, MemoryEffects::Free>(
     Operation *, BlockArgument);
 
+bool mlir::hasUnknownEffects(Operation *op) {
+  return !isa<MemoryEffectOpInterface>(op) &&
+         !op->hasTrait<OpTrait::HasRecursiveMemoryEffects>();
+}
+
 bool mlir::wouldOpBeTriviallyDead(Operation *op) {
   if (op->mightHaveTrait<OpTrait::IsTerminator>())
     return false;
diff --git a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
index 2f47939df5a02..af4ea5ac1cec8 100644
--- a/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
+++ b/mlir/lib/Interfaces/Utils/InferIntRangeCommon.cpp
@@ -290,8 +290,7 @@ static ConstantIntRanges inferDivURange(const ConstantIntRanges &lhs,
                                         DivisionFixupFn fixup) {
   const APInt &lhsMin = lhs.umin(), &lhsMax = lhs.umax(), &rhsMin = rhs.umin(),
               &rhsMax = rhs.umax();
-
-  if (!rhsMin.isZero()) {
+  if (!rhsMin.isZero() && !rhsMax.isZero()) {
     auto udiv = [&fixup](const APInt &a,
                          const APInt &b) -> std::optional<APInt> {
       return fixup(a, b, a.udiv(b));
diff --git a/mlir/lib/RegisterAllDialects.cpp b/mlir/lib/RegisterAllDialects.cpp
index 950b85e2543bc..258fed135a3e5 100644
--- a/mlir/lib/RegisterAllDialects.cpp
+++ b/mlir/lib/RegisterAllDialects.cpp
@@ -102,6 +102,7 @@
 #include "mlir/Interfaces/CastInterfaces.h"
 #include "mlir/Target/LLVM/NVVM/Target.h"
 #include "mlir/Target/LLVM/ROCDL/Target.h"
+#include "mlir/Target/LLVM/XeVM/Target.h"
 #include "mlir/Target/SPIRV/Target.h"
 
 /// Add all the MLIR dialects to the provided registry.
@@ -199,6 +200,7 @@ void mlir::registerAllDialects(DialectRegistry &registry) {
   NVVM::registerNVVMTargetInterfaceExternalModels(registry);
   ROCDL::registerROCDLTargetInterfaceExternalModels(registry);
   spirv::registerSPIRVTargetInterfaceExternalModels(registry);
+  xevm::registerXeVMTargetInterfaceExternalModels(registry);
 }
 
 /// Append all the MLIR dialects to the registry contained in the given context.
diff --git a/mlir/lib/RegisterAllExtensions.cpp b/mlir/lib/RegisterAllExtensions.cpp
index 8f7c67c8cd031..232ddaf6762c4 100644
--- a/mlir/lib/RegisterAllExtensions.cpp
+++ b/mlir/lib/RegisterAllExtensions.cpp
@@ -58,6 +58,7 @@
 #include "mlir/Target/LLVMIR/Dialect/GPU/GPUToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/ROCDL/ROCDLToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Dialect/XeVM/XeVMToLLVMIRTranslation.h"
 
 /// This function may be called to register all MLIR dialect extensions with the
 /// provided registry.
diff --git a/mlir/lib/RegisterAllPasses.cpp b/mlir/lib/RegisterAllPasses.cpp
index 1ed3a3798b0d2..c67b24226ae45 100644
--- a/mlir/lib/RegisterAllPasses.cpp
+++ b/mlir/lib/RegisterAllPasses.cpp
@@ -45,6 +45,7 @@
 #include "mlir/Dialect/Transform/Transforms/Passes.h"
 #include "mlir/Dialect/Vector/Transforms/Passes.h"
 #include "mlir/Dialect/XeGPU/Transforms/Passes.h"
+#include "mlir/Target/LLVMIR/Transforms/Passes.h"
 #include "mlir/Transforms/Passes.h"
 
 // This function may be called to register the MLIR passes with the
@@ -74,6 +75,7 @@ void mlir::registerAllPasses() {
   registerNVGPUPasses();
   registerSparseTensorPasses();
   LLVM::registerLLVMPasses();
+  LLVM::registerTargetLLVMIRTransformsPasses();
   math::registerMathPasses();
   memref::registerMemRefPasses();
   shard::registerShardPasses();
diff --git a/mlir/lib/Remark/CMakeLists.txt b/mlir/lib/Remark/CMakeLists.txt
new file mode 100644
index 0000000000000..920a95db4f07b
--- /dev/null
+++ b/mlir/lib/Remark/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_mlir_library(MLIRRemarkStreamer
+  RemarkStreamer.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Remark
+
+  LINK_LIBS PUBLIC
+    MLIRIR
+
+  LINK_COMPONENTS
+    Remarks
+    Core
+    BitstreamReader
+  )
diff --git a/mlir/lib/Remark/RemarkStreamer.cpp b/mlir/lib/Remark/RemarkStreamer.cpp
new file mode 100644
index 0000000000000..8e3544ff2c34c
--- /dev/null
+++ b/mlir/lib/Remark/RemarkStreamer.cpp
@@ -0,0 +1,69 @@
+#include "mlir/Remark/RemarkStreamer.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/Remarks.h"
+
+#include "llvm/Remarks/RemarkSerializer.h"
+#include "llvm/Remarks/RemarkStreamer.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/ToolOutputFile.h"
+
+namespace mlir::remark::detail {
+
+FailureOr<std::unique_ptr<MLIRRemarkStreamerBase>>
+LLVMRemarkStreamer::createToFile(llvm::StringRef path,
+                                 llvm::remarks::Format fmt) {
+  std::error_code ec;
+  // Use error_code ctor; YAML is text. (Bitstream also works fine here.)
+  auto f =
+      std::make_unique<llvm::ToolOutputFile>(path, ec, llvm::sys::fs::OF_Text);
+  if (ec)
+    return failure();
+
+  auto serOr = llvm::remarks::createRemarkSerializer(
+      fmt, llvm::remarks::SerializerMode::Separate, f->os());
+  if (!serOr) {
+    llvm::consumeError(serOr.takeError());
+    return failure();
+  }
+
+  auto rs =
+      std::make_unique<llvm::remarks::RemarkStreamer>(std::move(*serOr), path);
+
+  auto impl = std::unique_ptr<LLVMRemarkStreamer>(new LLVMRemarkStreamer());
+  impl->remarkStreamer = std::move(rs);
+  impl->file = std::move(f);
+  return std::unique_ptr<MLIRRemarkStreamerBase>(std::move(impl));
+}
+
+void LLVMRemarkStreamer::streamOptimizationRemark(const Remark &remark) {
+  if (!remarkStreamer->matchesFilter(remark.getCategoryName()))
+    return;
+
+  // First, convert the diagnostic to a remark.
+  llvm::remarks::Remark r = remark.generateRemark();
+  // Then, emit the remark through the serializer.
+  remarkStreamer->getSerializer().emit(r);
+}
+
+LLVMRemarkStreamer::~LLVMRemarkStreamer() {
+  if (file && remarkStreamer)
+    file->keep();
+}
+} // namespace mlir::remark::detail
+
+namespace mlir::remark {
+LogicalResult enableOptimizationRemarksWithLLVMStreamer(
+    MLIRContext &ctx, StringRef path, llvm::remarks::Format fmt,
+    const RemarkCategories &cat, bool printAsEmitRemarks) {
+
+  FailureOr<std::unique_ptr<detail::MLIRRemarkStreamerBase>> sOr =
+      detail::LLVMRemarkStreamer::createToFile(path, fmt);
+  if (failed(sOr))
+    return failure();
+
+  return remark::enableOptimizationRemarks(ctx, std::move(*sOr), cat,
+                                           printAsEmitRemarks);
+}
+
+} // namespace mlir::remark
diff --git a/mlir/lib/Rewrite/ByteCode.cpp b/mlir/lib/Rewrite/ByteCode.cpp
index ae3f22dec9f2c..5cbea5d2c6cee 100644
--- a/mlir/lib/Rewrite/ByteCode.cpp
+++ b/mlir/lib/Rewrite/ByteCode.cpp
@@ -20,8 +20,10 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/InterleavedRange.h"
 #include <numeric>
 #include <optional>
 
@@ -707,10 +709,8 @@ void Generator::allocateMemoryIndices(pdl_interp::FuncOp matcherFunc,
   }
 
   // Print the index usage and ensure that we did not run out of index space.
-  LLVM_DEBUG({
-    llvm::dbgs() << "Allocated " << allocatedIndices.size() << " indices "
-                 << "(down from initial " << valueDefRanges.size() << ").\n";
-  });
+  LDBG() << "Allocated " << allocatedIndices.size() << " indices "
+         << "(down from initial " << valueDefRanges.size() << ").";
   assert(allocatedIndices.size() <= std::numeric_limits<ByteCodeField>::max() &&
          "Ran out of memory for allocated indices");
 
@@ -736,6 +736,7 @@ void Generator::generate(Region *region, ByteCodeWriter &writer) {
 }
 
 void Generator::generate(Operation *op, ByteCodeWriter &writer) {
+  LDBG() << "Generating bytecode for operation: " << op->getName();
   LLVM_DEBUG({
     // The following list must contain all the operations that do not
     // produce any bytecode.
@@ -1275,12 +1276,8 @@ class ByteCodeExecutor {
   /// Handle a switch operation with the provided value and cases.
   template <typename T, typename RangeT, typename Comparator = std::equal_to<T>>
   void handleSwitch(const T &value, RangeT &&cases, Comparator cmp = {}) {
-    LLVM_DEBUG({
-      llvm::dbgs() << "  * Value: " << value << "\n"
-                   << "  * Cases: ";
-      llvm::interleaveComma(cases, llvm::dbgs());
-      llvm::dbgs() << "\n";
-    });
+    LDBG() << "Switch operation:\n  * Value: " << value
+           << "\n  * Cases: " << llvm::interleaved(cases);
 
     // Check to see if the attribute value is within the case list. Jump to
     // the correct successor index based on the result.
@@ -1424,38 +1421,27 @@ class ByteCodeExecutor {
 } // namespace
 
 void ByteCodeExecutor::executeApplyConstraint(PatternRewriter &rewriter) {
-  LLVM_DEBUG(llvm::dbgs() << "Executing ApplyConstraint:\n");
+  LDBG() << "Executing ApplyConstraint:";
   ByteCodeField fun_idx = read();
   SmallVector<PDLValue, 16> args;
   readList<PDLValue>(args);
 
-  LLVM_DEBUG({
-    llvm::dbgs() << "  * Arguments: ";
-    llvm::interleaveComma(args, llvm::dbgs());
-    llvm::dbgs() << "\n";
-  });
+  LDBG() << "  * Arguments: " << llvm::interleaved(args);
 
   ByteCodeField isNegated = read();
-  LLVM_DEBUG({
-    llvm::dbgs() << "  * isNegated: " << isNegated << "\n";
-    llvm::interleaveComma(args, llvm::dbgs());
-  });
+  LDBG() << "  * isNegated: " << isNegated;
 
   ByteCodeField numResults = read();
   const PDLRewriteFunction &constraintFn = constraintFunctions[fun_idx];
   ByteCodeRewriteResultList results(numResults);
   LogicalResult rewriteResult = constraintFn(rewriter, results, args);
   [[maybe_unused]] ArrayRef<PDLValue> constraintResults = results.getResults();
-  LLVM_DEBUG({
-    if (succeeded(rewriteResult)) {
-      llvm::dbgs() << "  * Constraint succeeded\n";
-      llvm::dbgs() << "  * Results: ";
-      llvm::interleaveComma(constraintResults, llvm::dbgs());
-      llvm::dbgs() << "\n";
-    } else {
-      llvm::dbgs() << "  * Constraint failed\n";
-    }
-  });
+  if (succeeded(rewriteResult)) {
+    LDBG() << "  * Constraint succeeded, results: "
+           << llvm::interleaved(constraintResults);
+  } else {
+    LDBG() << "  * Constraint failed";
+  }
   assert((failed(rewriteResult) || constraintResults.size() == numResults) &&
          "native PDL rewrite function succeeded but returned "
          "unexpected number of results");
@@ -1466,15 +1452,12 @@ void ByteCodeExecutor::executeApplyConstraint(PatternRewriter &rewriter) {
 }
 
 LogicalResult ByteCodeExecutor::executeApplyRewrite(PatternRewriter &rewriter) {
-  LLVM_DEBUG(llvm::dbgs() << "Executing ApplyRewrite:\n");
+  LDBG() << "Executing ApplyRewrite:";
   const PDLRewriteFunction &rewriteFn = rewriteFunctions[read()];
   SmallVector<PDLValue, 16> args;
   readList<PDLValue>(args);
 
-  LLVM_DEBUG({
-    llvm::dbgs() << "  * Arguments: ";
-    llvm::interleaveComma(args, llvm::dbgs());
-  });
+  LDBG() << "  * Arguments: " << llvm::interleaved(args);
 
   // Execute the rewrite function.
   ByteCodeField numResults = read();
@@ -1487,7 +1470,7 @@ LogicalResult ByteCodeExecutor::executeApplyRewrite(PatternRewriter &rewriter) {
   processNativeFunResults(results, numResults, rewriteResult);
 
   if (failed(rewriteResult)) {
-    LLVM_DEBUG(llvm::dbgs() << "  - Failed");
+    LDBG() << "  - Failed";
     return failure();
   }
   return success();
@@ -1516,7 +1499,7 @@ void ByteCodeExecutor::processNativeFunResults(
     PDLValue::Kind resultKind = read<PDLValue::Kind>();
     (void)resultKind;
     PDLValue result = results.getResults()[resultIdx];
-    LLVM_DEBUG(llvm::dbgs() << "  * Result: " << result << "\n");
+    LDBG() << "  * Result: " << result;
     assert(result.getKind() == resultKind &&
            "native PDL rewrite function returned an unexpected type of "
            "result");
@@ -1544,16 +1527,16 @@ void ByteCodeExecutor::processNativeFunResults(
 }
 
 void ByteCodeExecutor::executeAreEqual() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing AreEqual:\n");
+  LDBG() << "Executing AreEqual:";
   const void *lhs = read<const void *>();
   const void *rhs = read<const void *>();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * " << lhs << " == " << rhs << "\n");
+  LDBG() << "  * " << lhs << " == " << rhs;
   selectJump(lhs == rhs);
 }
 
 void ByteCodeExecutor::executeAreRangesEqual() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing AreRangesEqual:\n");
+  LDBG() << "Executing AreRangesEqual:";
   PDLValue::Kind valueKind = read<PDLValue::Kind>();
   const void *lhs = read<const void *>();
   const void *rhs = read<const void *>();
@@ -1562,14 +1545,14 @@ void ByteCodeExecutor::executeAreRangesEqual() {
   case PDLValue::Kind::TypeRange: {
     const TypeRange *lhsRange = reinterpret_cast<const TypeRange *>(lhs);
     const TypeRange *rhsRange = reinterpret_cast<const TypeRange *>(rhs);
-    LLVM_DEBUG(llvm::dbgs() << "  * " << lhs << " == " << rhs << "\n\n");
+    LDBG() << "  * " << lhs << " == " << rhs;
     selectJump(*lhsRange == *rhsRange);
     break;
   }
   case PDLValue::Kind::ValueRange: {
     const auto *lhsRange = reinterpret_cast<const ValueRange *>(lhs);
     const auto *rhsRange = reinterpret_cast<const ValueRange *>(rhs);
-    LLVM_DEBUG(llvm::dbgs() << "  * " << lhs << " == " << rhs << "\n\n");
+    LDBG() << "  * " << lhs << " == " << rhs;
     selectJump(*lhsRange == *rhsRange);
     break;
   }
@@ -1579,20 +1562,19 @@ void ByteCodeExecutor::executeAreRangesEqual() {
 }
 
 void ByteCodeExecutor::executeBranch() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing Branch\n");
+  LDBG() << "Executing Branch";
   curCodeIt = &code[read<ByteCodeAddr>()];
 }
 
 void ByteCodeExecutor::executeCheckOperandCount() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing CheckOperandCount:\n");
+  LDBG() << "Executing CheckOperandCount:";
   Operation *op = read<Operation *>();
   uint32_t expectedCount = read<uint32_t>();
   bool compareAtLeast = read();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Found: " << op->getNumOperands() << "\n"
-                          << "  * Expected: " << expectedCount << "\n"
-                          << "  * Comparator: "
-                          << (compareAtLeast ? ">=" : "==") << "\n");
+  LDBG() << "  * Found: " << op->getNumOperands()
+         << "\n  * Expected: " << expectedCount
+         << "\n  * Comparator: " << (compareAtLeast ? ">=" : "==");
   if (compareAtLeast)
     selectJump(op->getNumOperands() >= expectedCount);
   else
@@ -1600,25 +1582,24 @@ void ByteCodeExecutor::executeCheckOperandCount() {
 }
 
 void ByteCodeExecutor::executeCheckOperationName() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing CheckOperationName:\n");
+  LDBG() << "Executing CheckOperationName:";
   Operation *op = read<Operation *>();
   OperationName expectedName = read<OperationName>();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Found: \"" << op->getName() << "\"\n"
-                          << "  * Expected: \"" << expectedName << "\"\n");
+  LDBG() << "  * Found: \"" << op->getName() << "\"\n  * Expected: \""
+         << expectedName << "\"";
   selectJump(op->getName() == expectedName);
 }
 
 void ByteCodeExecutor::executeCheckResultCount() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing CheckResultCount:\n");
+  LDBG() << "Executing CheckResultCount:";
   Operation *op = read<Operation *>();
   uint32_t expectedCount = read<uint32_t>();
   bool compareAtLeast = read();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Found: " << op->getNumResults() << "\n"
-                          << "  * Expected: " << expectedCount << "\n"
-                          << "  * Comparator: "
-                          << (compareAtLeast ? ">=" : "==") << "\n");
+  LDBG() << "  * Found: " << op->getNumResults()
+         << "\n  * Expected: " << expectedCount
+         << "\n  * Comparator: " << (compareAtLeast ? ">=" : "==");
   if (compareAtLeast)
     selectJump(op->getNumResults() >= expectedCount);
   else
@@ -1626,36 +1607,35 @@ void ByteCodeExecutor::executeCheckResultCount() {
 }
 
 void ByteCodeExecutor::executeCheckTypes() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing AreEqual:\n");
+  LDBG() << "Executing AreEqual:";
   TypeRange *lhs = read<TypeRange *>();
   Attribute rhs = read<Attribute>();
-  LLVM_DEBUG(llvm::dbgs() << "  * " << lhs << " == " << rhs << "\n\n");
+  LDBG() << "  * " << lhs << " == " << rhs;
 
   selectJump(*lhs == cast<ArrayAttr>(rhs).getAsValueRange<TypeAttr>());
 }
 
 void ByteCodeExecutor::executeContinue() {
   ByteCodeField level = read();
-  LLVM_DEBUG(llvm::dbgs() << "Executing Continue\n"
-                          << "  * Level: " << level << "\n");
+  LDBG() << "Executing Continue\n  * Level: " << level;
   ++loopIndex[level];
   popCodeIt();
 }
 
 void ByteCodeExecutor::executeCreateConstantTypeRange() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing CreateConstantTypeRange:\n");
+  LDBG() << "Executing CreateConstantTypeRange:";
   unsigned memIndex = read();
   unsigned rangeIndex = read();
   ArrayAttr typesAttr = cast<ArrayAttr>(read<Attribute>());
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Types: " << typesAttr << "\n\n");
+  LDBG() << "  * Types: " << typesAttr;
   assignRangeToMemory(typesAttr.getAsValueRange<TypeAttr>(), memIndex,
                       rangeIndex);
 }
 
 void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter,
                                               Location mainRewriteLoc) {
-  LLVM_DEBUG(llvm::dbgs() << "Executing CreateOperation:\n");
+  LDBG() << "Executing CreateOperation:";
 
   unsigned memIndex = read();
   OperationState state(mainRewriteLoc, read<OperationName>());
@@ -1696,45 +1676,37 @@ void ByteCodeExecutor::executeCreateOperation(PatternRewriter &rewriter,
   Operation *resultOp = rewriter.create(state);
   memory[memIndex] = resultOp;
 
-  LLVM_DEBUG({
-    llvm::dbgs() << "  * Attributes: "
-                 << state.attributes.getDictionary(state.getContext())
-                 << "\n  * Operands: ";
-    llvm::interleaveComma(state.operands, llvm::dbgs());
-    llvm::dbgs() << "\n  * Result Types: ";
-    llvm::interleaveComma(state.types, llvm::dbgs());
-    llvm::dbgs() << "\n  * Result: " << *resultOp << "\n";
-  });
+  LDBG() << "  * Attributes: "
+         << state.attributes.getDictionary(state.getContext())
+         << "\n  * Operands: " << llvm::interleaved(state.operands)
+         << "\n  * Result Types: " << llvm::interleaved(state.types)
+         << "\n  * Result: " << *resultOp;
 }
 
 template <typename T>
 void ByteCodeExecutor::executeDynamicCreateRange(StringRef type) {
-  LLVM_DEBUG(llvm::dbgs() << "Executing CreateDynamic" << type << "Range:\n");
+  LDBG() << "Executing CreateDynamic" << type << "Range:";
   unsigned memIndex = read();
   unsigned rangeIndex = read();
   SmallVector<T> values;
   readList(values);
 
-  LLVM_DEBUG({
-    llvm::dbgs() << "\n  * " << type << "s: ";
-    llvm::interleaveComma(values, llvm::dbgs());
-    llvm::dbgs() << "\n";
-  });
+  LDBG() << "  * " << type << "s: " << llvm::interleaved(values);
 
   assignRangeToMemory(values, memIndex, rangeIndex);
 }
 
 void ByteCodeExecutor::executeEraseOp(PatternRewriter &rewriter) {
-  LLVM_DEBUG(llvm::dbgs() << "Executing EraseOp:\n");
+  LDBG() << "Executing EraseOp:";
   Operation *op = read<Operation *>();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n");
+  LDBG() << "  * Operation: " << *op;
   rewriter.eraseOp(op);
 }
 
 template <typename T, typename Range, PDLValue::Kind kind>
 void ByteCodeExecutor::executeExtract() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing Extract" << kind << ":\n");
+  LDBG() << "Executing Extract" << kind << ":";
   Range *range = read<Range *>();
   unsigned index = read<uint32_t>();
   unsigned memIndex = read();
@@ -1745,18 +1717,16 @@ void ByteCodeExecutor::executeExtract() {
   }
 
   T result = index < range->size() ? (*range)[index] : T();
-  LLVM_DEBUG(llvm::dbgs() << "  * " << kind << "s(" << range->size() << ")\n"
-                          << "  * Index: " << index << "\n"
-                          << "  * Result: " << result << "\n");
+  LDBG() << "  * " << kind << "s(" << range->size() << ")";
+  LDBG() << "  * Index: " << index;
+  LDBG() << "  * Result: " << result;
   storeToMemory(memIndex, result);
 }
 
-void ByteCodeExecutor::executeFinalize() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing Finalize\n");
-}
+void ByteCodeExecutor::executeFinalize() { LDBG() << "Executing Finalize"; }
 
 void ByteCodeExecutor::executeForEach() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing ForEach:\n");
+  LDBG() << "Executing ForEach:";
   const ByteCodeField *prevCodeIt = getPrevCodeIt();
   unsigned rangeIndex = read();
   unsigned memIndex = read();
@@ -1768,12 +1738,12 @@ void ByteCodeExecutor::executeForEach() {
     ArrayRef<Operation *> array = opRangeMemory[rangeIndex];
     assert(index <= array.size() && "iterated past the end");
     if (index < array.size()) {
-      LLVM_DEBUG(llvm::dbgs() << "  * Result: " << array[index] << "\n");
+      LDBG() << "  * Result: " << array[index];
       value = array[index];
       break;
     }
 
-    LLVM_DEBUG(llvm::dbgs() << "  * Done\n");
+    LDBG() << "  * Done";
     index = 0;
     selectJump(size_t(0));
     return;
@@ -1791,49 +1761,47 @@ void ByteCodeExecutor::executeForEach() {
 }
 
 void ByteCodeExecutor::executeGetAttribute() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing GetAttribute:\n");
+  LDBG() << "Executing GetAttribute:";
   unsigned memIndex = read();
   Operation *op = read<Operation *>();
   StringAttr attrName = read<StringAttr>();
   Attribute attr = op->getAttr(attrName);
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n"
-                          << "  * Attribute: " << attrName << "\n"
-                          << "  * Result: " << attr << "\n");
+  LDBG() << "  * Operation: " << *op << "\n  * Attribute: " << attrName
+         << "\n  * Result: " << attr;
   memory[memIndex] = attr.getAsOpaquePointer();
 }
 
 void ByteCodeExecutor::executeGetAttributeType() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing GetAttributeType:\n");
+  LDBG() << "Executing GetAttributeType:";
   unsigned memIndex = read();
   Attribute attr = read<Attribute>();
   Type type;
   if (auto typedAttr = dyn_cast<TypedAttr>(attr))
     type = typedAttr.getType();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Attribute: " << attr << "\n"
-                          << "  * Result: " << type << "\n");
+  LDBG() << "  * Attribute: " << attr << "\n  * Result: " << type;
   memory[memIndex] = type.getAsOpaquePointer();
 }
 
 void ByteCodeExecutor::executeGetDefiningOp() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing GetDefiningOp:\n");
+  LDBG() << "Executing GetDefiningOp:";
   unsigned memIndex = read();
   Operation *op = nullptr;
   if (read<PDLValue::Kind>() == PDLValue::Kind::Value) {
     Value value = read<Value>();
     if (value)
       op = value.getDefiningOp();
-    LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n");
+    LDBG() << "  * Value: " << value;
   } else {
     ValueRange *values = read<ValueRange *>();
     if (values && !values->empty()) {
       op = values->front().getDefiningOp();
     }
-    LLVM_DEBUG(llvm::dbgs() << "  * Values: " << values << "\n");
+    LDBG() << "  * Values: " << values;
   }
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Result: " << op << "\n");
+  LDBG() << "  * Result: " << op;
   memory[memIndex] = op;
 }
 
@@ -1843,9 +1811,8 @@ void ByteCodeExecutor::executeGetOperand(unsigned index) {
   Value operand =
       index < op->getNumOperands() ? op->getOperand(index) : Value();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n"
-                          << "  * Index: " << index << "\n"
-                          << "  * Result: " << operand << "\n");
+  LDBG() << "  * Operation: " << *op << "\n  * Index: " << index
+         << "\n  * Result: " << operand;
   memory[memIndex] = operand.getAsOpaquePointer();
 }
 
@@ -1860,13 +1827,12 @@ executeGetOperandsResults(RangeT values, Operation *op, unsigned index,
   // Check for the sentinel index that signals that all values should be
   // returned.
   if (index == std::numeric_limits<uint32_t>::max()) {
-    LLVM_DEBUG(llvm::dbgs() << "  * Getting all values\n");
+    LDBG() << "  * Getting all values";
     // `values` is already the full value range.
 
     // Otherwise, check to see if this operation uses AttrSizedSegments.
   } else if (op->hasTrait<AttrSizedSegmentsT>()) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "  * Extracting values from `" << attrSizedSegments << "`\n");
+    LDBG() << "  * Extracting values from `" << attrSizedSegments << "`";
 
     auto segmentAttr = op->getAttrOfType<DenseI32ArrayAttr>(attrSizedSegments);
     if (!segmentAttr || segmentAttr.asArrayRef().size() <= index)
@@ -1877,16 +1843,15 @@ executeGetOperandsResults(RangeT values, Operation *op, unsigned index,
         std::accumulate(segments.begin(), segments.begin() + index, 0);
     values = values.slice(startIndex, *std::next(segments.begin(), index));
 
-    LLVM_DEBUG(llvm::dbgs() << "  * Extracting range[" << startIndex << ", "
-                            << *std::next(segments.begin(), index) << "]\n");
+    LDBG() << "  * Extracting range[" << startIndex << ", "
+           << *std::next(segments.begin(), index) << "]";
 
     // Otherwise, assume this is the last operand group of the operation.
     // FIXME: We currently don't support operations with
     // SameVariadicOperandSize/SameVariadicResultSize here given that we don't
     // have a way to detect it's presence.
   } else if (values.size() >= index) {
-    LLVM_DEBUG(llvm::dbgs()
-               << "  * Treating values as trailing variadic range\n");
+    LDBG() << "  * Treating values as trailing variadic range";
     values = values.drop_front(index);
 
     // If we couldn't detect a way to compute the values, bail out.
@@ -1905,7 +1870,7 @@ executeGetOperandsResults(RangeT values, Operation *op, unsigned index,
 }
 
 void ByteCodeExecutor::executeGetOperands() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing GetOperands:\n");
+  LDBG() << "Executing GetOperands:";
   unsigned index = read<uint32_t>();
   Operation *op = read<Operation *>();
   ByteCodeField rangeIndex = read();
@@ -1914,7 +1879,7 @@ void ByteCodeExecutor::executeGetOperands() {
       op->getOperands(), op, index, rangeIndex, "operandSegmentSizes",
       valueRangeMemory);
   if (!result)
-    LLVM_DEBUG(llvm::dbgs() << "  * Invalid operand range\n");
+    LDBG() << "  * Invalid operand range";
   memory[read()] = result;
 }
 
@@ -1924,14 +1889,13 @@ void ByteCodeExecutor::executeGetResult(unsigned index) {
   OpResult result =
       index < op->getNumResults() ? op->getResult(index) : OpResult();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n"
-                          << "  * Index: " << index << "\n"
-                          << "  * Result: " << result << "\n");
+  LDBG() << "  * Operation: " << *op << "\n  * Index: " << index
+         << "\n  * Result: " << result;
   memory[memIndex] = result.getAsOpaquePointer();
 }
 
 void ByteCodeExecutor::executeGetResults() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing GetResults:\n");
+  LDBG() << "Executing GetResults:";
   unsigned index = read<uint32_t>();
   Operation *op = read<Operation *>();
   ByteCodeField rangeIndex = read();
@@ -1940,12 +1904,12 @@ void ByteCodeExecutor::executeGetResults() {
       op->getResults(), op, index, rangeIndex, "resultSegmentSizes",
       valueRangeMemory);
   if (!result)
-    LLVM_DEBUG(llvm::dbgs() << "  * Invalid result range\n");
+    LDBG() << "  * Invalid result range";
   memory[read()] = result;
 }
 
 void ByteCodeExecutor::executeGetUsers() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing GetUsers:\n");
+  LDBG() << "Executing GetUsers:";
   unsigned memIndex = read();
   unsigned rangeIndex = read();
   OwningOpRange &range = opRangeMemory[rangeIndex];
@@ -1957,7 +1921,7 @@ void ByteCodeExecutor::executeGetUsers() {
     Value value = read<Value>();
     if (!value)
       return;
-    LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n");
+    LDBG() << "  * Value: " << value;
 
     // Extract the users of a single value.
     range = OwningOpRange(std::distance(value.user_begin(), value.user_end()));
@@ -1967,11 +1931,8 @@ void ByteCodeExecutor::executeGetUsers() {
     ValueRange *values = read<ValueRange *>();
     if (!values)
       return;
-    LLVM_DEBUG({
-      llvm::dbgs() << "  * Values (" << values->size() << "): ";
-      llvm::interleaveComma(*values, llvm::dbgs());
-      llvm::dbgs() << "\n";
-    });
+    LDBG() << "  * Values (" << values->size()
+           << "): " << llvm::interleaved(*values);
 
     // Extract all the users of a range of values.
     SmallVector<Operation *> users;
@@ -1981,54 +1942,49 @@ void ByteCodeExecutor::executeGetUsers() {
     llvm::copy(users, range.begin());
   }
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Result: " << range.size() << " operations\n");
+  LDBG() << "  * Result: " << range.size() << " operations";
 }
 
 void ByteCodeExecutor::executeGetValueType() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing GetValueType:\n");
+  LDBG() << "Executing GetValueType:";
   unsigned memIndex = read();
   Value value = read<Value>();
   Type type = value ? value.getType() : Type();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n"
-                          << "  * Result: " << type << "\n");
+  LDBG() << "  * Value: " << value << "\n  * Result: " << type;
   memory[memIndex] = type.getAsOpaquePointer();
 }
 
 void ByteCodeExecutor::executeGetValueRangeTypes() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing GetValueRangeTypes:\n");
+  LDBG() << "Executing GetValueRangeTypes:";
   unsigned memIndex = read();
   unsigned rangeIndex = read();
   ValueRange *values = read<ValueRange *>();
   if (!values) {
-    LLVM_DEBUG(llvm::dbgs() << "  * Values: <NULL>\n\n");
+    LDBG() << "  * Values: <NULL>";
     memory[memIndex] = nullptr;
     return;
   }
 
-  LLVM_DEBUG({
-    llvm::dbgs() << "  * Values (" << values->size() << "): ";
-    llvm::interleaveComma(*values, llvm::dbgs());
-    llvm::dbgs() << "\n  * Result: ";
-    llvm::interleaveComma(values->getType(), llvm::dbgs());
-    llvm::dbgs() << "\n";
-  });
+  LDBG() << "  * Values (" << values->size()
+         << "): " << llvm::interleaved(*values)
+         << "\n  * Result: " << llvm::interleaved(values->getType());
   typeRangeMemory[rangeIndex] = values->getType();
   memory[memIndex] = &typeRangeMemory[rangeIndex];
 }
 
 void ByteCodeExecutor::executeIsNotNull() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing IsNotNull:\n");
+  LDBG() << "Executing IsNotNull:";
   const void *value = read<const void *>();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Value: " << value << "\n");
+  LDBG() << "  * Value: " << value;
   selectJump(value != nullptr);
 }
 
 void ByteCodeExecutor::executeRecordMatch(
     PatternRewriter &rewriter,
     SmallVectorImpl<PDLByteCode::MatchResult> &matches) {
-  LLVM_DEBUG(llvm::dbgs() << "Executing RecordMatch:\n");
+  LDBG() << "Executing RecordMatch:";
   unsigned patternIndex = read();
   PatternBenefit benefit = currentPatternBenefits[patternIndex];
   const ByteCodeField *dest = &code[read<ByteCodeAddr>()];
@@ -2036,7 +1992,7 @@ void ByteCodeExecutor::executeRecordMatch(
   // If the benefit of the pattern is impossible, skip the processing of the
   // rest of the pattern.
   if (benefit.isImpossibleToMatch()) {
-    LLVM_DEBUG(llvm::dbgs() << "  * Benefit: Impossible To Match\n");
+    LDBG() << "  * Benefit: Impossible To Match";
     curCodeIt = dest;
     return;
   }
@@ -2052,8 +2008,8 @@ void ByteCodeExecutor::executeRecordMatch(
     matchLocs.push_back(read<Operation *>()->getLoc());
   Location matchLoc = rewriter.getFusedLoc(matchLocs);
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Benefit: " << benefit.getBenefit() << "\n"
-                          << "  * Location: " << matchLoc << "\n");
+  LDBG() << "  * Benefit: " << benefit.getBenefit();
+  LDBG() << "  * Location: " << matchLoc;
   matches.emplace_back(matchLoc, patterns[patternIndex], benefit);
   PDLByteCode::MatchResult &match = matches.back();
 
@@ -2083,38 +2039,34 @@ void ByteCodeExecutor::executeRecordMatch(
 }
 
 void ByteCodeExecutor::executeReplaceOp(PatternRewriter &rewriter) {
-  LLVM_DEBUG(llvm::dbgs() << "Executing ReplaceOp:\n");
+  LDBG() << "Executing ReplaceOp:";
   Operation *op = read<Operation *>();
   SmallVector<Value, 16> args;
   readList(args);
 
-  LLVM_DEBUG({
-    llvm::dbgs() << "  * Operation: " << *op << "\n"
-                 << "  * Values: ";
-    llvm::interleaveComma(args, llvm::dbgs());
-    llvm::dbgs() << "\n";
-  });
+  LDBG() << "  * Operation: " << *op
+         << "\n  * Values: " << llvm::interleaved(args);
   rewriter.replaceOp(op, args);
 }
 
 void ByteCodeExecutor::executeSwitchAttribute() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing SwitchAttribute:\n");
+  LDBG() << "Executing SwitchAttribute:";
   Attribute value = read<Attribute>();
   ArrayAttr cases = read<ArrayAttr>();
   handleSwitch(value, cases);
 }
 
 void ByteCodeExecutor::executeSwitchOperandCount() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing SwitchOperandCount:\n");
+  LDBG() << "Executing SwitchOperandCount:";
   Operation *op = read<Operation *>();
   auto cases = read<DenseIntOrFPElementsAttr>().getValues<uint32_t>();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n");
+  LDBG() << "  * Operation: " << *op;
   handleSwitch(op->getNumOperands(), cases);
 }
 
 void ByteCodeExecutor::executeSwitchOperationName() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing SwitchOperationName:\n");
+  LDBG() << "Executing SwitchOperationName:";
   OperationName value = read<Operation *>()->getName();
   size_t caseCount = read();
 
@@ -2123,13 +2075,11 @@ void ByteCodeExecutor::executeSwitchOperationName() {
   // switch so that we can display all of the possible values.
   LLVM_DEBUG({
     const ByteCodeField *prevCodeIt = curCodeIt;
-    llvm::dbgs() << "  * Value: " << value << "\n"
-                 << "  * Cases: ";
-    llvm::interleaveComma(
-        llvm::map_range(llvm::seq<size_t>(0, caseCount),
-                        [&](size_t) { return read<OperationName>(); }),
-        llvm::dbgs());
-    llvm::dbgs() << "\n";
+    LDBG() << "  * Value: " << value << "\n  * Cases: "
+           << llvm::interleaved(
+                  llvm::map_range(llvm::seq<size_t>(0, caseCount), [&](size_t) {
+                    return read<OperationName>();
+                  }));
     curCodeIt = prevCodeIt;
   });
 
@@ -2144,27 +2094,27 @@ void ByteCodeExecutor::executeSwitchOperationName() {
 }
 
 void ByteCodeExecutor::executeSwitchResultCount() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing SwitchResultCount:\n");
+  LDBG() << "Executing SwitchResultCount:";
   Operation *op = read<Operation *>();
   auto cases = read<DenseIntOrFPElementsAttr>().getValues<uint32_t>();
 
-  LLVM_DEBUG(llvm::dbgs() << "  * Operation: " << *op << "\n");
+  LDBG() << "  * Operation: " << *op;
   handleSwitch(op->getNumResults(), cases);
 }
 
 void ByteCodeExecutor::executeSwitchType() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing SwitchType:\n");
+  LDBG() << "Executing SwitchType:";
   Type value = read<Type>();
   auto cases = read<ArrayAttr>().getAsValueRange<TypeAttr>();
   handleSwitch(value, cases);
 }
 
 void ByteCodeExecutor::executeSwitchTypes() {
-  LLVM_DEBUG(llvm::dbgs() << "Executing SwitchTypes:\n");
+  LDBG() << "Executing SwitchTypes:";
   TypeRange *value = read<TypeRange *>();
   auto cases = read<ArrayAttr>().getAsRange<ArrayAttr>();
   if (!value) {
-    LLVM_DEBUG(llvm::dbgs() << "Types: <NULL>\n");
+    LDBG() << "Types: <NULL>";
     return selectJump(size_t(0));
   }
   handleSwitch(*value, cases, [](ArrayAttr caseValue, const TypeRange &value) {
@@ -2178,7 +2128,7 @@ ByteCodeExecutor::execute(PatternRewriter &rewriter,
                           std::optional<Location> mainRewriteLoc) {
   while (true) {
     // Print the location of the operation being executed.
-    LLVM_DEBUG(llvm::dbgs() << readInline<Location>() << "\n");
+    LDBG() << readInline<Location>();
 
     OpCode opCode = static_cast<OpCode>(read());
     switch (opCode) {
@@ -2239,7 +2189,7 @@ ByteCodeExecutor::execute(PatternRewriter &rewriter,
       break;
     case Finalize:
       executeFinalize();
-      LLVM_DEBUG(llvm::dbgs() << "\n");
+      LDBG() << "";
       return success();
     case ForEach:
       executeForEach();
@@ -2258,12 +2208,12 @@ ByteCodeExecutor::execute(PatternRewriter &rewriter,
     case GetOperand2:
     case GetOperand3: {
       unsigned index = opCode - GetOperand0;
-      LLVM_DEBUG(llvm::dbgs() << "Executing GetOperand" << index << ":\n");
+      LDBG() << "Executing GetOperand" << index << ":";
       executeGetOperand(index);
       break;
     }
     case GetOperandN:
-      LLVM_DEBUG(llvm::dbgs() << "Executing GetOperandN:\n");
+      LDBG() << "Executing GetOperandN:";
       executeGetOperand(read<uint32_t>());
       break;
     case GetOperands:
@@ -2274,12 +2224,12 @@ ByteCodeExecutor::execute(PatternRewriter &rewriter,
     case GetResult2:
     case GetResult3: {
       unsigned index = opCode - GetResult0;
-      LLVM_DEBUG(llvm::dbgs() << "Executing GetResult" << index << ":\n");
+      LDBG() << "Executing GetResult" << index << ":";
       executeGetResult(index);
       break;
     }
     case GetResultN:
-      LLVM_DEBUG(llvm::dbgs() << "Executing GetResultN:\n");
+      LDBG() << "Executing GetResultN:";
       executeGetResult(read<uint32_t>());
       break;
     case GetResults:
@@ -2324,7 +2274,7 @@ ByteCodeExecutor::execute(PatternRewriter &rewriter,
       executeSwitchTypes();
       break;
     }
-    LLVM_DEBUG(llvm::dbgs() << "\n");
+    LDBG() << "";
   }
 }
 
@@ -2383,7 +2333,7 @@ LogicalResult PDLByteCode::rewrite(PatternRewriter &rewriter,
   // bug in the user code (i.e. failable rewrites should not be used with
   // pattern rewriters that don't support it).
   if (failed(result) && !rewriter.canRecoverFromRewriteFailure()) {
-    LLVM_DEBUG(llvm::dbgs() << " and rollback is not supported - aborting");
+    LDBG() << " and rollback is not supported - aborting";
     llvm::report_fatal_error(
         "Native PDL Rewrite failed, but the pattern "
         "rewriter doesn't support recovery. Failable pattern rewrites should "
diff --git a/mlir/lib/Rewrite/PatternApplicator.cpp b/mlir/lib/Rewrite/PatternApplicator.cpp
index e13bcff2767ec..23ae95a8cca47 100644
--- a/mlir/lib/Rewrite/PatternApplicator.cpp
+++ b/mlir/lib/Rewrite/PatternApplicator.cpp
@@ -37,9 +37,9 @@ PatternApplicator::~PatternApplicator() = default;
 #ifndef NDEBUG
 /// Log a message for a pattern that is impossible to match.
 static void logImpossibleToMatch(const Pattern &pattern) {
-  llvm::dbgs() << "Ignoring pattern '" << pattern.getRootKind()
-               << "' because it is impossible to match or cannot lead "
-                  "to legal IR (by cost model)\n";
+  LDBG() << "Ignoring pattern '" << pattern.getRootKind()
+         << "' because it is impossible to match or cannot lead "
+            "to legal IR (by cost model)";
 }
 
 /// Log IR after pattern application.
diff --git a/mlir/lib/Target/CMakeLists.txt b/mlir/lib/Target/CMakeLists.txt
index 6eb0abc214d38..f0c3ac4d511c1 100644
--- a/mlir/lib/Target/CMakeLists.txt
+++ b/mlir/lib/Target/CMakeLists.txt
@@ -4,3 +4,4 @@ add_subdirectory(SPIRV)
 add_subdirectory(LLVMIR)
 add_subdirectory(LLVM)
 add_subdirectory(SMTLIB)
+add_subdirectory(Wasm)
diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
index 8e83e455d1a7f..a5ee64c7fe38b 100644
--- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp
+++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
@@ -1447,7 +1447,7 @@ LogicalResult CppEmitter::emitAttribute(Location loc, Attribute attr) {
   }
   if (auto dense = dyn_cast<DenseIntElementsAttr>(attr)) {
     if (auto iType = dyn_cast<IntegerType>(
-            cast<TensorType>(dense.getType()).getElementType())) {
+            cast<ShapedType>(dense.getType()).getElementType())) {
       os << '{';
       interleaveComma(dense, os, [&](const APInt &val) {
         printInt(val, shouldMapToUnsigned(iType.getSignedness()));
@@ -1456,7 +1456,7 @@ LogicalResult CppEmitter::emitAttribute(Location loc, Attribute attr) {
       return success();
     }
     if (auto iType = dyn_cast<IndexType>(
-            cast<TensorType>(dense.getType()).getElementType())) {
+            cast<ShapedType>(dense.getType()).getElementType())) {
       os << '{';
       interleaveComma(dense, os,
                       [&](const APInt &val) { printInt(val, false); });
diff --git a/mlir/lib/Target/LLVM/CMakeLists.txt b/mlir/lib/Target/LLVM/CMakeLists.txt
index f6e44c60e65f6..9a0e4d45693b8 100644
--- a/mlir/lib/Target/LLVM/CMakeLists.txt
+++ b/mlir/lib/Target/LLVM/CMakeLists.txt
@@ -210,3 +210,27 @@ if(MLIR_ENABLE_ROCM_CONVERSIONS)
   )
 endif()
 
+if ("SPIRV" IN_LIST LLVM_TARGETS_TO_BUILD)
+  set(SPIRV_LIBS
+    SPIRVCodeGen
+    SPIRVDesc
+    SPIRVInfo
+  )
+endif()
+
+add_mlir_dialect_library(MLIRXeVMTarget
+  XeVM/Target.cpp
+
+  OBJECT
+
+  LINK_COMPONENTS
+  ${SPIRV_LIBS}
+
+  LINK_LIBS PUBLIC
+  MLIRIR
+  MLIRExecutionEngineUtils
+  MLIRSupport
+  MLIRGPUDialect
+  MLIRTargetLLVM
+  MLIRXeVMToLLVMIRTranslation
+)
diff --git a/mlir/lib/Target/LLVM/NVVM/Target.cpp b/mlir/lib/Target/LLVM/NVVM/Target.cpp
index 55c8a64de9ded..8760ea8588e2c 100644
--- a/mlir/lib/Target/LLVM/NVVM/Target.cpp
+++ b/mlir/lib/Target/LLVM/NVVM/Target.cpp
@@ -24,9 +24,11 @@
 #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.h"
 #include "mlir/Target/LLVMIR/Export.h"
+#include "llvm/Support/InterleavedRange.h"
 
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/Config/Targets.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -265,6 +267,8 @@ NVPTXSerializer::NVPTXSerializer(Operation &module, NVVMTargetAttr target,
 std::optional<NVPTXSerializer::TmpFile>
 NVPTXSerializer::createTemp(StringRef name, StringRef suffix) {
   llvm::SmallString<128> filename;
+  if (name.size() > 80)
+    name = name.substr(0, 80);
   std::error_code ec =
       llvm::sys::fs::createTemporaryFile(name, suffix, filename);
   if (ec) {
@@ -452,17 +456,11 @@ NVPTXSerializer::compileToBinary(const std::string &ptxCode) {
 
   // Dump tool invocation commands.
 #define DEBUG_TYPE "serialize-to-binary"
-  LLVM_DEBUG({
-    llvm::dbgs() << "Tool invocation for module: "
-                 << getOperation().getNameAttr() << "\n";
-    llvm::dbgs() << "ptxas executable:" << ptxasCompiler.value() << "\n";
-    llvm::interleave(ptxasArgs, llvm::dbgs(), " ");
-    llvm::dbgs() << "\n";
-    if (createFatbin) {
-      llvm::interleave(fatbinArgs, llvm::dbgs(), " ");
-      llvm::dbgs() << "\n";
-    }
-  });
+  LDBG() << "Tool invocation for module: " << getOperation().getNameAttr()
+         << "\nptxas executable:" << ptxasCompiler.value()
+         << "\nptxas args: " << llvm::interleaved(ptxasArgs, " ");
+  if (createFatbin)
+    LDBG() << "fatbin args: " << llvm::interleaved(fatbinArgs, " ");
 #undef DEBUG_TYPE
 
   // Helper function for printing tool error logs.
@@ -507,7 +505,7 @@ NVPTXSerializer::compileToBinary(const std::string &ptxCode) {
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> logBuffer =
         llvm::MemoryBuffer::getFile(logFile->first);
     if (logBuffer && !(*logBuffer)->getBuffer().empty()) {
-      llvm::dbgs() << "Output:\n" << (*logBuffer)->getBuffer() << "\n";
+      LDBG() << "Output:\n" << (*logBuffer)->getBuffer();
       llvm::dbgs().flush();
     }
   });
@@ -529,7 +527,7 @@ NVPTXSerializer::compileToBinary(const std::string &ptxCode) {
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> logBuffer =
         llvm::MemoryBuffer::getFile(logFile->first);
     if (logBuffer && !(*logBuffer)->getBuffer().empty()) {
-      llvm::dbgs() << "Output:\n" << (*logBuffer)->getBuffer() << "\n";
+      LDBG() << "Output:\n" << (*logBuffer)->getBuffer();
       llvm::dbgs().flush();
     }
   });
@@ -629,12 +627,11 @@ NVPTXSerializer::compileToBinaryNVPTX(const std::string &ptxCode) {
       SmallVector<char> log(logSize + 1, 0);
       RETURN_ON_NVPTXCOMPILER_ERROR(
           nvPTXCompilerGetInfoLog(compiler, log.data()));
-      llvm::dbgs() << "NVPTX compiler invocation for module: "
-                   << getOperation().getNameAttr() << "\n";
-      llvm::dbgs() << "Arguments: ";
-      llvm::interleave(cmdOpts.second, llvm::dbgs(), " ");
-      llvm::dbgs() << "\nOutput\n" << log.data() << "\n";
-      llvm::dbgs().flush();
+      LDBG() << "NVPTX compiler invocation for module: "
+             << getOperation().getNameAttr()
+             << "\nArguments: " << llvm::interleaved(cmdOpts.second, " ")
+             << "\nOutput\n"
+             << log.data();
     }
   });
 #undef DEBUG_TYPE
@@ -678,10 +675,8 @@ NVPTXSerializer::moduleToObject(llvm::Module &llvmModule) {
   // Return LLVM IR if the compilation target is `offload`.
 #define DEBUG_TYPE "serialize-to-llvm"
   LLVM_DEBUG({
-    llvm::dbgs() << "LLVM IR for module: " << getOperation().getNameAttr()
-                 << "\n";
-    llvm::dbgs() << llvmModule << "\n";
-    llvm::dbgs().flush();
+    LDBG() << "LLVM IR for module: " << getOperation().getNameAttr();
+    LDBG() << llvmModule;
   });
 #undef DEBUG_TYPE
   if (targetOptions.getCompilationTarget() == gpu::CompilationTarget::Offload)
@@ -716,11 +711,8 @@ NVPTXSerializer::moduleToObject(llvm::Module &llvmModule) {
     isaCallback(serializedISA.value());
 
 #define DEBUG_TYPE "serialize-to-isa"
-  LLVM_DEBUG({
-    llvm::dbgs() << "PTX for module: " << getOperation().getNameAttr() << "\n";
-    llvm::dbgs() << *serializedISA << "\n";
-    llvm::dbgs().flush();
-  });
+  LDBG() << "PTX for module: " << getOperation().getNameAttr() << "\n"
+         << *serializedISA;
 #undef DEBUG_TYPE
 
   // Return PTX if the compilation target is `assembly`.
diff --git a/mlir/lib/Target/LLVM/XeVM/Target.cpp b/mlir/lib/Target/LLVM/XeVM/Target.cpp
new file mode 100644
index 0000000000000..1e6784a25d64a
--- /dev/null
+++ b/mlir/lib/Target/LLVM/XeVM/Target.cpp
@@ -0,0 +1,418 @@
+//===- Target.cpp - MLIR LLVM XeVM target compilation -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This files defines XeVM target related functions including registration
+// calls for the `#xevm.target` compilation attribute.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Target/LLVM/XeVM/Target.h"
+
+#include "mlir/Dialect/GPU/IR/CompilationInterfaces.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/LLVMIR/XeVMDialect.h"
+#include "mlir/IR/BuiltinAttributeInterfaces.h"
+#include "mlir/IR/BuiltinDialect.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/DialectResourceBlobManager.h"
+#include "mlir/Target/LLVM/XeVM/Utils.h"
+#include "mlir/Target/LLVMIR/Dialect/GPU/GPUToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Dialect/XeVM/XeVMToLLVMIRTranslation.h"
+#include "mlir/Target/LLVMIR/Export.h"
+#include "llvm/IR/LegacyPassManager.h"
+#include "llvm/Target/TargetMachine.h"
+
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/Config/Targets.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <cstdint>
+#include <cstdlib>
+
+using namespace mlir;
+using namespace mlir::xevm;
+
+namespace {
+// XeVM implementation of the gpu:TargetAttrInterface.
+class XeVMTargetAttrImpl
+    : public gpu::TargetAttrInterface::FallbackModel<XeVMTargetAttrImpl> {
+public:
+  std::optional<SmallVector<char, 0>>
+  serializeToObject(Attribute attribute, Operation *module,
+                    const gpu::TargetOptions &options) const;
+
+  Attribute createObject(Attribute attribute, Operation *module,
+                         const SmallVector<char, 0> &object,
+                         const gpu::TargetOptions &options) const;
+};
+} // namespace
+
+void mlir::xevm::registerXeVMTargetInterfaceExternalModels(
+    DialectRegistry &registry) {
+  registry.addExtension(+[](MLIRContext *ctx, XeVMDialect *dialect) {
+    XeVMTargetAttr::attachInterface<XeVMTargetAttrImpl>(*ctx);
+  });
+}
+
+void mlir::xevm::registerXeVMTargetInterfaceExternalModels(
+    MLIRContext &context) {
+  DialectRegistry registry;
+  registerXeVMTargetInterfaceExternalModels(registry);
+  context.appendDialectRegistry(registry);
+}
+
+SerializeGPUModuleBase::SerializeGPUModuleBase(
+    Operation &module, XeVMTargetAttr xeTarget,
+    const gpu::TargetOptions &targetOptions)
+    : ModuleToObject(module, xeTarget.getTriple(), "", {}, xeTarget.getO()),
+      xeTarget(xeTarget), librariesToLink(targetOptions.getLibrariesToLink()),
+      targetOptions(targetOptions) {
+  if (xeTarget.getLinkFiles())
+    librariesToLink.append(xeTarget.getLinkFiles().begin(),
+                           xeTarget.getLinkFiles().end());
+}
+
+XeVMTargetAttr SerializeGPUModuleBase::getTarget() const { return xeTarget; }
+
+std::optional<SmallVector<std::unique_ptr<llvm::Module>>>
+SerializeGPUModuleBase::loadBitcodeFiles(llvm::Module &module) {
+  if (librariesToLink.empty())
+    return SmallVector<std::unique_ptr<llvm::Module>>();
+  SmallVector<std::unique_ptr<llvm::Module>> bcFiles;
+  if (failed(loadBitcodeFilesFromList(module.getContext(), librariesToLink,
+                                      bcFiles)))
+    return std::nullopt;
+  return std::move(bcFiles);
+}
+
+gpu::GPUModuleOp SerializeGPUModuleBase::getGPUModuleOp() {
+  return dyn_cast<gpu::GPUModuleOp>(&SerializeGPUModuleBase::getOperation());
+}
+
+// There is 1 way to finalize IL to native code: IGC
+// There are 2 ways to access IGC: AOT (ocloc) and JIT (L0 runtime).
+// - L0 runtime consumes IL and is external to MLIR codebase (rt wrappers).
+// - `ocloc` tool can be "queried" from within MLIR.
+std::optional<SmallVector<char, 0>>
+SerializeGPUModuleBase::compileToBinary(const std::string &asmStr,
+                                        StringRef inputFormat) {
+  using TmpFile = std::pair<llvm::SmallString<128>, llvm::FileRemover>;
+  // Find the `ocloc` tool.
+  std::optional<std::string> oclocCompiler = findTool("ocloc");
+  if (!oclocCompiler)
+    return std::nullopt;
+  Location loc = getGPUModuleOp().getLoc();
+  std::string basename = llvm::formatv(
+      "mlir-{0}-{1}-{2}", getGPUModuleOp().getNameAttr().getValue(),
+      getTarget().getTriple(), getTarget().getChip());
+
+  auto createTemp = [&](StringRef name,
+                        StringRef suffix) -> std::optional<TmpFile> {
+    llvm::SmallString<128> filePath;
+    if (auto ec = llvm::sys::fs::createTemporaryFile(name, suffix, filePath)) {
+      getGPUModuleOp().emitError()
+          << "Couldn't create the temp file: `" << filePath
+          << "`, error message: " << ec.message();
+      return std::nullopt;
+    }
+    return TmpFile(filePath, llvm::FileRemover(filePath.c_str()));
+  };
+  // Create temp file
+  std::optional<TmpFile> asmFile = createTemp(basename, "asm");
+  std::optional<TmpFile> binFile = createTemp(basename, "");
+  std::optional<TmpFile> logFile = createTemp(basename, "log");
+  if (!logFile || !asmFile || !binFile)
+    return std::nullopt;
+  // Dump the assembly to a temp file
+  std::error_code ec;
+  {
+    llvm::raw_fd_ostream asmStream(asmFile->first, ec);
+    if (ec) {
+      emitError(loc) << "Couldn't open the file: `" << asmFile->first
+                     << "`, error message: " << ec.message();
+      return std::nullopt;
+    }
+    asmStream << asmStr;
+    if (asmStream.has_error()) {
+      emitError(loc) << "An error occurred while writing the assembly to: `"
+                     << asmFile->first << "`.";
+      return std::nullopt;
+    }
+    asmStream.flush();
+  }
+  // Set cmd options
+  std::pair<llvm::BumpPtrAllocator, SmallVector<const char *>> cmdOpts =
+      targetOptions.tokenizeCmdOptions();
+  // Example: --gpu-module-to-binary="opts='opt1 opt2'"
+  const std::string cmdOptsStr = "\"" + llvm::join(cmdOpts.second, " ") + "\"";
+  SmallVector<StringRef, 12> oclocArgs(
+      {"ocloc", "compile", "-file", asmFile->first, inputFormat, "-device",
+       getTarget().getChip(), "-output", binFile->first, "-output_no_suffix",
+       "-options", cmdOptsStr});
+
+// Dump tool invocation commands.
+#define DEBUG_TYPE "serialize-to-binary"
+  LLVM_DEBUG({
+    llvm::dbgs() << "Tool invocation for module: "
+                 << getGPUModuleOp().getNameAttr() << "\n";
+    llvm::interleave(oclocArgs, llvm::dbgs(), " ");
+    llvm::dbgs() << "\n";
+  });
+#undef DEBUG_TYPE
+  // Helper function for printing tool error logs.
+  std::string message;
+  auto emitLogError =
+      [&](StringRef toolName) -> std::optional<SmallVector<char, 0>> {
+    if (message.empty()) {
+      llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> toolStderr =
+          llvm::MemoryBuffer::getFile(logFile->first);
+      if (toolStderr)
+        emitError(loc) << toolName << " invocation failed. Log:\n"
+                       << toolStderr->get()->getBuffer();
+      else
+        emitError(loc) << toolName << " invocation failed.";
+      return std::nullopt;
+    }
+    emitError(loc) << toolName
+                   << " invocation failed, error message: " << message;
+    return std::nullopt;
+  };
+  std::optional<StringRef> redirects[] = {
+      std::nullopt,
+      logFile->first,
+      logFile->first,
+  };
+  // Invoke ocloc.
+  if (llvm::sys::ExecuteAndWait(oclocCompiler.value(), oclocArgs, std::nullopt,
+                                redirects, 0, 0, &message))
+    return emitLogError("`ocloc`");
+  binFile->first.append(".bin");
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> binaryBuffer =
+      llvm::MemoryBuffer::getFile(binFile->first);
+  if (!binaryBuffer) {
+    emitError(loc) << "Couldn't open the file: `" << binFile->first
+                   << "`, error message: " << binaryBuffer.getError().message();
+    return std::nullopt;
+  }
+  StringRef bin = (*binaryBuffer)->getBuffer();
+  return SmallVector<char, 0>(bin.begin(), bin.end());
+}
+
+std::optional<std::string> SerializeGPUModuleBase::findTool(StringRef tool) {
+  // 1. Check the toolkit path given in the command line.
+  StringRef pathRef = targetOptions.getToolkitPath();
+  SmallVector<char, 256> path;
+  if (!pathRef.empty()) {
+    path.insert(path.begin(), pathRef.begin(), pathRef.end());
+    llvm::sys::path::append(path, "bin", tool);
+    if (llvm::sys::fs::can_execute(path))
+      return StringRef(path.data(), path.size()).str();
+  }
+  // 2. Check PATH.
+  if (std::optional<std::string> toolPath =
+          llvm::sys::Process::FindInEnvPath("PATH", tool))
+    return *toolPath;
+
+  getGPUModuleOp().emitError()
+      << "Couldn't find the `" << tool
+      << "` binary. Please specify the toolkit "
+         "path via GpuModuleToBinaryPass or add the compiler to $PATH`.";
+  return std::nullopt;
+}
+
+namespace {
+class SPIRVSerializer : public SerializeGPUModuleBase {
+public:
+  SPIRVSerializer(Operation &module, XeVMTargetAttr xeTarget,
+                  const gpu::TargetOptions &targetOptions)
+      : SerializeGPUModuleBase(module, xeTarget, targetOptions) {}
+
+  static void init();
+
+  /// Serializes the LLVM module to an object format, depending on the
+  /// compilation target selected in target options.
+  std::optional<SmallVector<char, 0>>
+  moduleToObject(llvm::Module &llvmModule) override;
+
+private:
+  /// Translates the LLVM module to SPIR-V binary using LLVM's
+  /// SPIR-V target.
+  std::optional<std::string>
+  translateToSPIRVBinary(llvm::Module &llvmModule,
+                         llvm::TargetMachine &targetMachine);
+};
+} // namespace
+
+void SPIRVSerializer::init() {
+  static llvm::once_flag initializeBackendOnce;
+  llvm::call_once(initializeBackendOnce, []() {
+#if LLVM_HAS_SPIRV_TARGET
+    LLVMInitializeSPIRVTarget();
+    LLVMInitializeSPIRVTargetInfo();
+    LLVMInitializeSPIRVTargetMC();
+    LLVMInitializeSPIRVAsmPrinter();
+#endif
+  });
+}
+
+std::optional<SmallVector<char, 0>>
+SPIRVSerializer::moduleToObject(llvm::Module &llvmModule) {
+#define DEBUG_TYPE "serialize-to-llvm"
+  LLVM_DEBUG({
+    llvm::dbgs() << "LLVM IR for module: " << getGPUModuleOp().getNameAttr()
+                 << "\n";
+    llvm::dbgs() << llvmModule << "\n";
+    llvm::dbgs().flush();
+  });
+#undef DEBUG_TYPE
+
+  // Return LLVM IR if the compilation target is `offload`.
+  if (targetOptions.getCompilationTarget() == gpu::CompilationTarget::Offload)
+    return SerializeGPUModuleBase::moduleToObject(llvmModule);
+
+#if !LLVM_HAS_SPIRV_TARGET
+  getGPUModuleOp()->emitError("The `SPIRV` target was not built. Please enable "
+                              "it when building LLVM.");
+  return std::nullopt;
+#endif // LLVM_HAS_SPIRV_TARGET
+
+  std::optional<llvm::TargetMachine *> targetMachine =
+      getOrCreateTargetMachine();
+  if (!targetMachine) {
+    getGPUModuleOp().emitError() << "Target Machine unavailable for triple "
+                                 << triple << ", can't optimize with LLVM\n";
+    return std::nullopt;
+  }
+
+  // Return SPIRV if the compilation target is `assembly`.
+  if (targetOptions.getCompilationTarget() ==
+      gpu::CompilationTarget::Assembly) {
+    std::optional<std::string> serializedISA =
+        translateToISA(llvmModule, **targetMachine);
+    if (!serializedISA) {
+      getGPUModuleOp().emitError() << "Failed translating the module to ISA."
+                                   << triple << ", can't compile with LLVM\n";
+      return std::nullopt;
+    }
+
+#define DEBUG_TYPE "serialize-to-isa"
+    LLVM_DEBUG({
+      llvm::dbgs() << "SPIR-V for module: " << getGPUModuleOp().getNameAttr()
+                   << "\n";
+      llvm::dbgs() << *serializedISA << "\n";
+      llvm::dbgs().flush();
+    });
+#undef DEBUG_TYPE
+
+    // Make sure to include the null terminator.
+    StringRef bin(serializedISA->c_str(), serializedISA->size() + 1);
+    return SmallVector<char, 0>(bin.begin(), bin.end());
+  }
+
+  // Level zero runtime is set up to accept SPIR-V binary
+  // translateToSPIRVBinary translates the LLVM module to SPIR-V binary
+  // using LLVM's SPIRV target.
+  // compileToBinary can be used in the future if level zero runtime
+  // implementation switches to native XeVM binary format.
+  std::optional<std::string> serializedSPIRVBinary =
+      translateToSPIRVBinary(llvmModule, **targetMachine);
+  if (!serializedSPIRVBinary) {
+    getGPUModuleOp().emitError() << "Failed translating the module to Binary.";
+    return std::nullopt;
+  }
+  if (serializedSPIRVBinary->size() % 4) {
+    getGPUModuleOp().emitError() << "SPIRV code size must be a multiple of 4.";
+    return std::nullopt;
+  }
+  StringRef bin(serializedSPIRVBinary->c_str(), serializedSPIRVBinary->size());
+  return SmallVector<char, 0>(bin.begin(), bin.end());
+}
+
+std::optional<std::string>
+SPIRVSerializer::translateToSPIRVBinary(llvm::Module &llvmModule,
+                                        llvm::TargetMachine &targetMachine) {
+  std::string targetISA;
+  llvm::raw_string_ostream stream(targetISA);
+
+  { // Drop pstream after this to prevent the ISA from being stuck buffering
+    llvm::buffer_ostream pstream(stream);
+    llvm::legacy::PassManager codegenPasses;
+    if (targetMachine.addPassesToEmitFile(codegenPasses, pstream, nullptr,
+                                          llvm::CodeGenFileType::ObjectFile))
+      return std::nullopt;
+
+    codegenPasses.run(llvmModule);
+  }
+  return targetISA;
+}
+
+std::optional<SmallVector<char, 0>>
+XeVMTargetAttrImpl::serializeToObject(Attribute attribute, Operation *module,
+                                      const gpu::TargetOptions &options) const {
+  if (!module)
+    return std::nullopt;
+  auto gpuMod = dyn_cast<gpu::GPUModuleOp>(module);
+  if (!gpuMod) {
+    module->emitError("expected to be a gpu.module op");
+    return std::nullopt;
+  }
+  auto xeTarget = cast<XeVMTargetAttr>(attribute);
+  if (xeTarget.getTriple().starts_with("spirv")) {
+    gpuMod.walk([&](LLVM::LLVMFuncOp funcOp) {
+      if (funcOp->hasAttr(gpu::GPUDialect::getKernelFuncAttrName())) {
+        funcOp.setIntelReqdSubGroupSize(16);
+        return WalkResult::interrupt();
+      }
+      return WalkResult::advance();
+    });
+
+    SPIRVSerializer serializer(*module, cast<XeVMTargetAttr>(attribute),
+                               options);
+    serializer.init();
+
+#if !LLVM_HAS_SPIRV_TARGET
+    module->emitError("Cannot run `TargetRegistry::lookupTarget()` for SPIRV "
+                      "without having the target built.");
+#endif
+
+    return serializer.run();
+  }
+  module->emitError("Unsupported XeVM target triple: ") << xeTarget.getTriple();
+  return std::nullopt;
+}
+
+Attribute
+XeVMTargetAttrImpl::createObject(Attribute attribute, Operation *module,
+                                 const SmallVector<char, 0> &object,
+                                 const gpu::TargetOptions &options) const {
+  Builder builder(attribute.getContext());
+  gpu::CompilationTarget format = options.getCompilationTarget();
+  auto xeTarget = cast<XeVMTargetAttr>(attribute);
+  SmallVector<NamedAttribute, 2> properties;
+  if (format == gpu::CompilationTarget::Assembly)
+    properties.push_back(
+        builder.getNamedAttr("O", builder.getI32IntegerAttr(xeTarget.getO())));
+
+  DictionaryAttr objectProps;
+  if (!properties.empty())
+    objectProps = builder.getDictionaryAttr(properties);
+
+  return builder.getAttr<gpu::ObjectAttr>(
+      attribute, format,
+      builder.getStringAttr(StringRef(object.data(), object.size())),
+      objectProps, /*kernels=*/nullptr);
+}
diff --git a/mlir/lib/Target/LLVMIR/CMakeLists.txt b/mlir/lib/Target/LLVMIR/CMakeLists.txt
index 9ea5c6835e8ef..d39b35526daf9 100644
--- a/mlir/lib/Target/LLVMIR/CMakeLists.txt
+++ b/mlir/lib/Target/LLVMIR/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_subdirectory(Dialect)
+add_subdirectory(Transforms)
 
 set(LLVM_OPTIONAL_SOURCES
   ConvertFromLLVMIR.cpp
diff --git a/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp b/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
index fbad5c2fb78d9..8bd07cd4e2ed9 100644
--- a/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DataLayoutImporter.cpp
@@ -6,13 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "DataLayoutImporter.h"
+#include "mlir/Target/LLVMIR/DataLayoutImporter.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
 #include "mlir/Target/LLVMIR/Import.h"
+
 #include "llvm/IR/DataLayout.h"
 
 using namespace mlir;
@@ -274,101 +275,88 @@ DataLayoutImporter::tryToEmplaceLegalIntWidthsEntry(StringRef token) {
   return success();
 }
 
-void DataLayoutImporter::translateDataLayout(
-    const llvm::DataLayout &llvmDataLayout) {
-  dataLayout = {};
-
-  // Transform the data layout to its string representation and append the
-  // default data layout string specified in the language reference
-  // (https://llvm.org/docs/LangRef.html#data-layout). The translation then
-  // parses the string and ignores the default value if a specific kind occurs
-  // in both strings. Additionally, the following default values exist:
-  // - non-default address space pointer specifications default to the default
-  //   address space pointer specification
-  // - the alloca address space defaults to the default address space.
-  layoutStr = llvmDataLayout.getStringRepresentation();
-  if (!layoutStr.empty())
-    layoutStr += "-";
-  layoutStr += kDefaultDataLayout;
-  StringRef layout(layoutStr);
+DataLayoutSpecInterface DataLayoutImporter::dataLayoutSpecFromDataLayoutStr() {
+  if (!dataLayoutStr.empty())
+    dataLayoutStr += "-";
+  dataLayoutStr += kDefaultDataLayout;
 
   // Split the data layout string into tokens separated by a dash.
   SmallVector<StringRef> tokens;
-  layout.split(tokens, '-');
+  StringRef(dataLayoutStr).split(tokens, '-');
 
   for (StringRef token : tokens) {
     lastToken = token;
     FailureOr<StringRef> prefix = tryToParseAlphaPrefix(token);
     if (failed(prefix))
-      return;
+      return {};
 
     // Parse the endianness.
     if (*prefix == "e") {
       if (failed(tryToEmplaceEndiannessEntry(
               DLTIDialect::kDataLayoutEndiannessLittle, token)))
-        return;
+        return {};
       continue;
     }
     if (*prefix == "E") {
       if (failed(tryToEmplaceEndiannessEntry(
               DLTIDialect::kDataLayoutEndiannessBig, token)))
-        return;
+        return {};
       continue;
     }
     // Parse the program address space.
     if (*prefix == "P") {
       if (failed(tryToEmplaceAddrSpaceEntry(
               token, DLTIDialect::kDataLayoutProgramMemorySpaceKey)))
-        return;
+        return {};
       continue;
     }
     // Parse the mangling mode.
     if (*prefix == "m") {
       if (failed(tryToEmplaceManglingModeEntry(
               token, DLTIDialect::kDataLayoutManglingModeKey)))
-        return;
+        return {};
       continue;
     }
     // Parse the global address space.
     if (*prefix == "G") {
       if (failed(tryToEmplaceAddrSpaceEntry(
               token, DLTIDialect::kDataLayoutGlobalMemorySpaceKey)))
-        return;
+        return {};
       continue;
     }
     // Parse the alloca address space.
     if (*prefix == "A") {
       if (failed(tryToEmplaceAddrSpaceEntry(
               token, DLTIDialect::kDataLayoutAllocaMemorySpaceKey)))
-        return;
+        return {};
       continue;
     }
     // Parse the stack alignment.
     if (*prefix == "S") {
       if (failed(tryToEmplaceStackAlignmentEntry(token)))
-        return;
+        return {};
       continue;
     }
     // Parse integer alignment specifications.
     if (*prefix == "i") {
       FailureOr<uint64_t> width = tryToParseInt(token);
       if (failed(width))
-        return;
+        return {};
 
       Type type = IntegerType::get(context, *width);
       if (failed(tryToEmplaceAlignmentEntry(type, token)))
-        return;
+        return {};
       continue;
     }
     // Parse float alignment specifications.
     if (*prefix == "f") {
       FailureOr<uint64_t> width = tryToParseInt(token);
       if (failed(width))
-        return;
+        return {};
 
       Type type = getFloatType(context, *width);
       if (failed(tryToEmplaceAlignmentEntry(type, token)))
-        return;
+        return {};
       continue;
     }
     // Parse pointer alignment specifications.
@@ -376,17 +364,17 @@ void DataLayoutImporter::translateDataLayout(
       FailureOr<uint64_t> space =
           token.starts_with(":") ? 0 : tryToParseInt(token);
       if (failed(space))
-        return;
+        return {};
 
       auto type = LLVMPointerType::get(context, *space);
       if (failed(tryToEmplacePointerAlignmentEntry(type, token)))
-        return;
+        return {};
       continue;
     }
     // Parse native integer widths specifications.
     if (*prefix == "n") {
       if (failed(tryToEmplaceLegalIntWidthsEntry(token)))
-        return;
+        return {};
       continue;
     }
     // Parse function pointer alignment specifications.
@@ -394,7 +382,7 @@ void DataLayoutImporter::translateDataLayout(
     if (prefix->starts_with("F")) {
       StringRef nextPrefix = prefix->drop_front(1);
       if (failed(tryToEmplaceFunctionPointerAlignmentEntry(nextPrefix, token)))
-        return;
+        return {};
       continue;
     }
 
@@ -409,11 +397,12 @@ void DataLayoutImporter::translateDataLayout(
     entries.push_back(it.second);
   for (const auto &it : keyEntries)
     entries.push_back(it.second);
-  dataLayout = DataLayoutSpecAttr::get(context, entries);
+  return DataLayoutSpecAttr::get(context, entries);
 }
 
 DataLayoutSpecInterface
 mlir::translateDataLayout(const llvm::DataLayout &dataLayout,
                           MLIRContext *context) {
-  return DataLayoutImporter(context, dataLayout).getDataLayout();
+  return DataLayoutImporter(context, dataLayout.getStringRepresentation())
+      .getDataLayoutSpec();
 }
diff --git a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
index 90462d16c874e..a20701ce75bc0 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/NVVM/NVVMToLLVMIRTranslation.cpp
@@ -135,33 +135,83 @@ static llvm::Intrinsic::ID getVoteSyncIntrinsicId(NVVM::VoteSyncKind kind) {
   llvm_unreachable("unsupported vote kind");
 }
 
-/// Return the intrinsic ID associated with ldmatrix for the given paramters.
-static llvm::Intrinsic::ID getLdMatrixIntrinsicId(NVVM::MMALayout layout,
-                                                  int32_t num) {
-  if (layout == NVVM::MMALayout::row) {
+static llvm::Intrinsic::ID
+getLdMatrixIntrinsicId(NVVM::MMALayout layout, int32_t num,
+                       NVVM::LdStMatrixShapeAttr shape,
+                       NVVM::LdStMatrixEltType eltType) {
+  if (shape.getM() == 8 && shape.getN() == 8) {
     switch (num) {
     case 1:
-      return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16;
+      return (layout == NVVM::MMALayout::row)
+                 ? llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_b16
+                 : llvm::Intrinsic::
+                       nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16;
     case 2:
-      return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16;
+      return (layout == NVVM::MMALayout::row)
+                 ? llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_b16
+                 : llvm::Intrinsic::
+                       nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16;
     case 4:
-      return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16;
-    default:
-      llvm_unreachable("unsupported number of matrix");
+      return (layout == NVVM::MMALayout::row)
+                 ? llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_b16
+                 : llvm::Intrinsic::
+                       nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16;
     }
-
-  } else {
-    switch (num) {
-    case 1:
-      return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x1_trans_b16;
-    case 2:
-      return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x2_trans_b16;
-    case 4:
-      return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m8n8_x4_trans_b16;
-    default:
-      llvm_unreachable("unsupported number of matrix");
+  } else if (shape.getM() == 8 && shape.getN() == 16) {
+    if (eltType == NVVM::LdStMatrixEltType::B8X16_B6X16_P32) {
+      switch (num) {
+      case 1:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b6x16_p32;
+      case 2:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b6x16_p32;
+      case 4:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b6x16_p32;
+      }
+    } else if (eltType == NVVM::LdStMatrixEltType::B8X16_B4X16_P64) {
+      switch (num) {
+      case 1:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m8n16_x1_b8x16_b4x16_p64;
+      case 2:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m8n16_x2_b8x16_b4x16_p64;
+      case 4:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m8n16_x4_b8x16_b4x16_p64;
+      }
+    }
+  } else if (shape.getM() == 16 && shape.getN() == 16) {
+    if (eltType == NVVM::LdStMatrixEltType::B8) {
+      switch (num) {
+      case 1:
+        return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8;
+      case 2:
+        return llvm::Intrinsic::nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8;
+      }
+    } else if (eltType == NVVM::LdStMatrixEltType::B8X16_B6X16_P32) {
+      switch (num) {
+      case 1:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b6x16_p32;
+      case 2:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b6x16_p32;
+      }
+    } else if (eltType == NVVM::LdStMatrixEltType::B8X16_B4X16_P64) {
+      switch (num) {
+      case 1:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m16n16_x1_trans_b8x16_b4x16_p64;
+      case 2:
+        return llvm::Intrinsic::
+            nvvm_ldmatrix_sync_aligned_m16n16_x2_trans_b8x16_b4x16_p64;
+      }
     }
   }
+  llvm_unreachable("unknown ldmatrix kind");
 }
 
 /// Return the intrinsic ID associated with stmatrix for the given paramters.
@@ -418,7 +468,11 @@ class NVVMDialectLLVMIRTranslationInterface
     } else if (attribute.getName() ==
                NVVM::NVVMDialect::getKernelFuncAttrName()) {
       llvmFunc->setCallingConv(llvm::CallingConv::PTX_Kernel);
+    } else if (attribute.getName() ==
+               NVVM::NVVMDialect::getBlocksAreClustersAttrName()) {
+      llvmFunc->addFnAttr("nvvm.blocksareclusters");
     }
+
     return success();
   }
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 762cc88d9fc3d..6694de8383534 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -3205,6 +3205,23 @@ llvm::AtomicRMWInst::BinOp convertBinOpToAtomic(Operation &op) {
       .Default(llvm::AtomicRMWInst::BinOp::BAD_BINOP);
 }
 
+void extractAtomicControlFlags(omp::AtomicUpdateOp atomicUpdateOp,
+                               bool &isIgnoreDenormalMode,
+                               bool &isFineGrainedMemory,
+                               bool &isRemoteMemory) {
+  isIgnoreDenormalMode = false;
+  isFineGrainedMemory = false;
+  isRemoteMemory = false;
+  if (atomicUpdateOp &&
+      atomicUpdateOp->hasAttr(atomicUpdateOp.getAtomicControlAttrName())) {
+    mlir::omp::AtomicControlAttr atomicControlAttr =
+        atomicUpdateOp.getAtomicControlAttr();
+    isIgnoreDenormalMode = atomicControlAttr.getIgnoreDenormalMode();
+    isFineGrainedMemory = atomicControlAttr.getFineGrainedMemory();
+    isRemoteMemory = atomicControlAttr.getRemoteMemory();
+  }
+}
+
 /// Converts an OpenMP atomic update operation using OpenMPIRBuilder.
 static LogicalResult
 convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
@@ -3269,13 +3286,19 @@ convertOmpAtomicUpdate(omp::AtomicUpdateOp &opInst,
     return moduleTranslation.lookupValue(yieldop.getResults()[0]);
   };
 
+  bool isIgnoreDenormalMode;
+  bool isFineGrainedMemory;
+  bool isRemoteMemory;
+  extractAtomicControlFlags(opInst, isIgnoreDenormalMode, isFineGrainedMemory,
+                            isRemoteMemory);
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
       ompBuilder->createAtomicUpdate(ompLoc, allocaIP, llvmAtomicX, llvmExpr,
                                      atomicOrdering, binop, updateFn,
-                                     isXBinopExpr);
+                                     isXBinopExpr, isIgnoreDenormalMode,
+                                     isFineGrainedMemory, isRemoteMemory);
 
   if (failed(handleError(afterIP, *opInst)))
     return failure();
@@ -3364,13 +3387,19 @@ convertOmpAtomicCapture(omp::AtomicCaptureOp atomicCaptureOp,
     return moduleTranslation.lookupValue(yieldop.getResults()[0]);
   };
 
+  bool isIgnoreDenormalMode;
+  bool isFineGrainedMemory;
+  bool isRemoteMemory;
+  extractAtomicControlFlags(atomicUpdateOp, isIgnoreDenormalMode,
+                            isFineGrainedMemory, isRemoteMemory);
   // Handle ambiguous alloca, if any.
   auto allocaIP = findAllocaInsertPoint(builder, moduleTranslation);
   llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
       ompBuilder->createAtomicCapture(
           ompLoc, allocaIP, llvmAtomicX, llvmAtomicV, llvmExpr, atomicOrdering,
-          binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr);
+          binop, updateFn, atomicUpdateOp, isPostfixUpdate, isXBinopExpr,
+          isIgnoreDenormalMode, isFineGrainedMemory, isRemoteMemory);
 
   if (failed(handleError(afterIP, *atomicCaptureOp)))
     return failure();
@@ -4327,9 +4356,11 @@ createAlteredByCaptureMap(MapInfoData &mapData,
 
         if (!isPtrTy) {
           auto curInsert = builder.saveIP();
+          llvm::DebugLoc DbgLoc = builder.getCurrentDebugLocation();
           builder.restoreIP(findAllocaInsertPoint(builder, moduleTranslation));
           auto *memTempAlloc =
               builder.CreateAlloca(builder.getPtrTy(), nullptr, ".casted");
+          builder.SetCurrentDebugLocation(DbgLoc);
           builder.restoreIP(curInsert);
 
           builder.CreateStore(newV, memTempAlloc);
@@ -5836,6 +5867,10 @@ static bool isTargetDeviceOp(Operation *op) {
   if (mlir::isa<omp::ThreadprivateOp>(op))
     return true;
 
+  if (mlir::isa<omp::TargetAllocMemOp>(op) ||
+      mlir::isa<omp::TargetFreeMemOp>(op))
+    return true;
+
   if (auto parentFn = op->getParentOfType<LLVM::LLVMFuncOp>())
     if (auto declareTargetIface =
             llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(
@@ -5848,6 +5883,85 @@ static bool isTargetDeviceOp(Operation *op) {
   return false;
 }
 
+static llvm::Function *getOmpTargetAlloc(llvm::IRBuilderBase &builder,
+                                         llvm::Module *llvmModule) {
+  llvm::Type *i64Ty = builder.getInt64Ty();
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *returnType = builder.getPtrTy(0);
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(returnType, {i64Ty, i32Ty}, false);
+  llvm::Function *func = cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_alloc", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetAllocMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                        LLVM::ModuleTranslation &moduleTranslation) {
+  auto allocMemOp = cast<omp::TargetAllocMemOp>(opInst);
+  if (!allocMemOp)
+    return failure();
+
+  // Get "omp_target_alloc" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTargetAllocFunc = getOmpTargetAlloc(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = allocMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the allocation size.
+  llvm::DataLayout dataLayout = llvmModule->getDataLayout();
+  mlir::Type heapTy = allocMemOp.getAllocatedType();
+  llvm::Type *llvmHeapTy = moduleTranslation.convertType(heapTy);
+  llvm::TypeSize typeSize = dataLayout.getTypeStoreSize(llvmHeapTy);
+  llvm::Value *allocSize = builder.getInt64(typeSize.getFixedValue());
+  for (auto typeParam : allocMemOp.getTypeparams())
+    allocSize =
+        builder.CreateMul(allocSize, moduleTranslation.lookupValue(typeParam));
+  // Create call to "omp_target_alloc" with the args as translated llvm values.
+  llvm::CallInst *call =
+      builder.CreateCall(ompTargetAllocFunc, {allocSize, llvmDeviceNum});
+  llvm::Value *resultI64 = builder.CreatePtrToInt(call, builder.getInt64Ty());
+
+  // Map the result
+  moduleTranslation.mapValue(allocMemOp.getResult(), resultI64);
+  return success();
+}
+
+static llvm::Function *getOmpTargetFree(llvm::IRBuilderBase &builder,
+                                        llvm::Module *llvmModule) {
+  llvm::Type *ptrTy = builder.getPtrTy(0);
+  llvm::Type *i32Ty = builder.getInt32Ty();
+  llvm::Type *voidTy = builder.getVoidTy();
+  llvm::FunctionType *fnType =
+      llvm::FunctionType::get(voidTy, {ptrTy, i32Ty}, false);
+  llvm::Function *func = dyn_cast<llvm::Function>(
+      llvmModule->getOrInsertFunction("omp_target_free", fnType).getCallee());
+  return func;
+}
+
+static LogicalResult
+convertTargetFreeMemOp(Operation &opInst, llvm::IRBuilderBase &builder,
+                       LLVM::ModuleTranslation &moduleTranslation) {
+  auto freeMemOp = cast<omp::TargetFreeMemOp>(opInst);
+  if (!freeMemOp)
+    return failure();
+
+  // Get "omp_target_free" function
+  llvm::Module *llvmModule = moduleTranslation.getLLVMModule();
+  llvm::Function *ompTragetFreeFunc = getOmpTargetFree(builder, llvmModule);
+  // Get the corresponding device value in llvm
+  mlir::Value deviceNum = freeMemOp.getDevice();
+  llvm::Value *llvmDeviceNum = moduleTranslation.lookupValue(deviceNum);
+  // Get the corresponding heapref value in llvm
+  mlir::Value heapref = freeMemOp.getHeapref();
+  llvm::Value *llvmHeapref = moduleTranslation.lookupValue(heapref);
+  // Convert heapref int to ptr and call "omp_target_free"
+  llvm::Value *intToPtr =
+      builder.CreateIntToPtr(llvmHeapref, builder.getPtrTy(0));
+  builder.CreateCall(ompTragetFreeFunc, {intToPtr, llvmDeviceNum});
+  return success();
+}
+
 /// Given an OpenMP MLIR operation, create the corresponding LLVM IR (including
 /// OpenMP runtime calls).
 static LogicalResult
@@ -6022,6 +6136,12 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder,
             // the omp.canonical_loop.
             return applyUnrollHeuristic(op, builder, moduleTranslation);
           })
+          .Case([&](omp::TargetAllocMemOp) {
+            return convertTargetAllocMemOp(*op, builder, moduleTranslation);
+          })
+          .Case([&](omp::TargetFreeMemOp) {
+            return convertTargetFreeMemOp(*op, builder, moduleTranslation);
+          })
           .Default([&](Operation *inst) {
             return inst->emitError()
                    << "not yet implemented: " << inst->getName();
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 6325480e63657..40726365cd06e 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -16,7 +16,6 @@
 #include "mlir/Target/LLVMIR/Import.h"
 
 #include "AttrKindDetail.h"
-#include "DataLayoutImporter.h"
 #include "DebugImporter.h"
 #include "LoopAnnotationImporter.h"
 
@@ -25,6 +24,7 @@
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/Matchers.h"
 #include "mlir/Interfaces/DataLayoutInterfaces.h"
+#include "mlir/Target/LLVMIR/DataLayoutImporter.h"
 #include "mlir/Tools/mlir-translate/Translation.h"
 
 #include "llvm/ADT/DepthFirstIterator.h"
@@ -1045,8 +1045,9 @@ LogicalResult ModuleImport::convertIFuncs() {
 
 LogicalResult ModuleImport::convertDataLayout() {
   Location loc = mlirModule.getLoc();
-  DataLayoutImporter dataLayoutImporter(context, llvmModule->getDataLayout());
-  if (!dataLayoutImporter.getDataLayout())
+  DataLayoutImporter dataLayoutImporter(
+      context, llvmModule->getDataLayout().getStringRepresentation());
+  if (!dataLayoutImporter.getDataLayoutSpec())
     return emitError(loc, "cannot translate data layout: ")
            << dataLayoutImporter.getLastToken();
 
@@ -1054,7 +1055,7 @@ LogicalResult ModuleImport::convertDataLayout() {
     emitWarning(loc, "unhandled data layout token: ") << token;
 
   mlirModule->setAttr(DLTIDialect::kDataLayoutAttrName,
-                      dataLayoutImporter.getDataLayout());
+                      dataLayoutImporter.getDataLayoutSpec());
   return success();
 }
 
diff --git a/mlir/lib/Target/LLVMIR/Transforms/CMakeLists.txt b/mlir/lib/Target/LLVMIR/Transforms/CMakeLists.txt
new file mode 100644
index 0000000000000..a0232601c5f9c
--- /dev/null
+++ b/mlir/lib/Target/LLVMIR/Transforms/CMakeLists.txt
@@ -0,0 +1,21 @@
+add_mlir_dialect_library(MLIRTargetLLVMIRTransforms
+  TargetToDataLayout.cpp
+
+  DEPENDS
+  MLIRTargetLLVMIRTransformsIncGen
+
+  LINK_COMPONENTS
+  MC
+  Target
+  TargetParser
+  AllTargetsAsmParsers
+  AllTargetsCodeGens
+  AllTargetsDescs
+  AllTargetsInfos
+
+  LINK_LIBS PUBLIC
+  MLIRDLTIDialect
+  MLIRLLVMDialect
+  MLIRPass
+  MLIRTargetLLVMIRImport
+  )
diff --git a/mlir/lib/Target/LLVMIR/Transforms/TargetToDataLayout.cpp b/mlir/lib/Target/LLVMIR/Transforms/TargetToDataLayout.cpp
new file mode 100644
index 0000000000000..d41d441812039
--- /dev/null
+++ b/mlir/lib/Target/LLVMIR/Transforms/TargetToDataLayout.cpp
@@ -0,0 +1,104 @@
+//===- TargetToDataLayout.cpp - extract data layout from TargetMachine ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "mlir/Target/LLVMIR/Transforms/Passes.h"
+
+#include "mlir/Dialect/DLTI/DLTI.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Target/LLVMIR/Import.h"
+
+#include "llvm/MC/TargetRegistry.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Target/TargetMachine.h"
+
+#define DEBUG_TYPE "mlir-llvm-target-to-data-layout"
+#define DBGS() (llvm::dbgs() << '[' << DEBUG_TYPE << "] ")
+#define LDBG(X) LLVM_DEBUG(DBGS() << X << "\n")
+
+namespace mlir {
+namespace LLVM {
+#define GEN_PASS_DEF_LLVMTARGETTODATALAYOUT
+#include "mlir/Target/LLVMIR/Transforms/Passes.h.inc"
+} // namespace LLVM
+} // namespace mlir
+
+using namespace mlir;
+
+static FailureOr<std::unique_ptr<llvm::TargetMachine>>
+getTargetMachine(LLVM::TargetAttrInterface attr) {
+  StringRef triple = attr.getTriple();
+  StringRef chipAKAcpu = attr.getChip();
+  StringRef features = attr.getFeatures() ? attr.getFeatures().getValue() : "";
+
+  std::string error;
+  const llvm::Target *target =
+      llvm::TargetRegistry::lookupTarget(triple, error);
+  if (!target || !error.empty()) {
+    LDBG("Looking up target '" << triple << "' failed: " << error << "\n");
+    return failure();
+  }
+
+  return std::unique_ptr<llvm::TargetMachine>(target->createTargetMachine(
+      llvm::Triple(triple), chipAKAcpu, features, {}, {}));
+}
+
+static FailureOr<llvm::DataLayout>
+getDataLayout(LLVM::TargetAttrInterface attr) {
+  FailureOr<std::unique_ptr<llvm::TargetMachine>> targetMachine =
+      getTargetMachine(attr);
+  if (failed(targetMachine)) {
+    LDBG("Failed to retrieve the target machine for data layout.\n");
+    return failure();
+  }
+  return (targetMachine.value())->createDataLayout();
+}
+
+struct TargetToDataLayoutPass
+    : public LLVM::impl::LLVMTargetToDataLayoutBase<TargetToDataLayoutPass> {
+  using LLVM::impl::LLVMTargetToDataLayoutBase<
+      TargetToDataLayoutPass>::LLVMTargetToDataLayoutBase;
+
+  void runOnOperation() override {
+    Operation *op = getOperation();
+
+    if (initializeLLVMTargets) {
+      static llvm::once_flag initializeBackendsOnce;
+      llvm::call_once(initializeBackendsOnce, []() {
+        // Ensure that the targets, that LLVM has been configured to support,
+        // are loaded into the TargetRegistry.
+        llvm::InitializeAllTargets();
+        llvm::InitializeAllTargetMCs();
+      });
+    }
+
+    auto targetAttr = op->getAttrOfType<LLVM::TargetAttrInterface>(
+        LLVM::LLVMDialect::getTargetAttrName());
+    if (!targetAttr) {
+      op->emitError()
+          << "no TargetAttrInterface-implementing attribute at key \""
+          << LLVM::LLVMDialect::getTargetAttrName() << "\"";
+      return signalPassFailure();
+    }
+
+    FailureOr<llvm::DataLayout> dataLayout = getDataLayout(targetAttr);
+    if (failed(dataLayout)) {
+      op->emitError() << "failed to obtain llvm::DataLayout for " << targetAttr;
+      return signalPassFailure();
+    }
+
+    DataLayoutSpecInterface dataLayoutSpec =
+        mlir::translateDataLayout(dataLayout.value(), &getContext());
+
+    if (auto existingDlSpec = op->getAttrOfType<DataLayoutSpecInterface>(
+            DLTIDialect::kDataLayoutAttrName)) {
+      dataLayoutSpec = existingDlSpec.combineWith({dataLayoutSpec});
+    }
+
+    op->setAttr(DLTIDialect::kDataLayoutAttrName, dataLayoutSpec);
+  }
+};
diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index 750821833224f..d8c54ec5f88c3 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -344,6 +344,8 @@ LogicalResult spirv::Deserializer::processDecoration(ArrayRef<uint32_t> words) {
   case spirv::Decoration::RestrictPointer:
   case spirv::Decoration::NoContraction:
   case spirv::Decoration::Constant:
+  case spirv::Decoration::Invariant:
+  case spirv::Decoration::Patch:
     if (words.size() != 2) {
       return emitError(unknownLoc, "OpDecoration with ")
              << decorationName << "needs a single target <id>";
@@ -1558,7 +1560,19 @@ spirv::Deserializer::processConstantComposite(ArrayRef<uint32_t> operands) {
   }
 
   auto resultID = operands[1];
-  if (auto shapedType = dyn_cast<ShapedType>(resultType)) {
+  if (auto tensorType = dyn_cast<TensorArmType>(resultType)) {
+    SmallVector<Attribute> flattenedElems;
+    for (Attribute element : elements) {
+      if (auto denseElemAttr = dyn_cast<DenseElementsAttr>(element)) {
+        for (auto value : denseElemAttr.getValues<Attribute>())
+          flattenedElems.push_back(value);
+      } else {
+        flattenedElems.push_back(element);
+      }
+    }
+    auto attr = DenseElementsAttr::get(tensorType, flattenedElems);
+    constantMap.try_emplace(resultID, attr, tensorType);
+  } else if (auto shapedType = dyn_cast<ShapedType>(resultType)) {
     auto attr = DenseElementsAttr::get(shapedType, elements);
     // For normal constants, we just record the attribute (and its type) for
     // later materialization at use sites.
diff --git a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
index 30536638b56f7..7fc779587f4f1 100644
--- a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
+++ b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
@@ -16,6 +16,7 @@
 #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
 #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
+#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
 #include "mlir/Target/SPIRV/SPIRVBinaryUtils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/Sequence.h"
@@ -112,7 +113,9 @@ LogicalResult Serializer::serialize() {
 
   // TODO: handle the other sections
   processCapability();
-  processExtension();
+  if (failed(processExtension())) {
+    return failure();
+  }
   processMemoryModel();
   processDebugInfo();
 
@@ -204,13 +207,24 @@ void Serializer::processDebugInfo() {
   // TODO: Encode more debug instructions.
 }
 
-void Serializer::processExtension() {
+LogicalResult Serializer::processExtension() {
   llvm::SmallVector<uint32_t, 16> extName;
-  for (spirv::Extension ext : module.getVceTriple()->getExtensions()) {
+  llvm::SmallSet<Extension, 4> deducedExts(
+      llvm::from_range, module.getVceTriple()->getExtensions());
+  auto nonSemanticInfoExt = spirv::Extension::SPV_KHR_non_semantic_info;
+  if (options.emitDebugInfo && !deducedExts.contains(nonSemanticInfoExt)) {
+    TargetEnvAttr targetEnvAttr = lookupTargetEnvOrDefault(module);
+    if (!is_contained(targetEnvAttr.getExtensions(), nonSemanticInfoExt))
+      return module.emitError(
+          "SPV_KHR_non_semantic_info extension not available");
+    deducedExts.insert(nonSemanticInfoExt);
+  }
+  for (spirv::Extension ext : deducedExts) {
     extName.clear();
     spirv::encodeStringLiteralInto(extName, spirv::stringifyExtension(ext));
     encodeInstructionInto(extensions, spirv::Opcode::OpExtension, extName);
   }
+  return success();
 }
 
 void Serializer::processMemoryModel() {
@@ -338,6 +352,8 @@ LogicalResult Serializer::processDecorationAttr(Location loc, uint32_t resultID,
   case spirv::Decoration::NoContraction:
   case spirv::Decoration::Constant:
   case spirv::Decoration::Block:
+  case spirv::Decoration::Invariant:
+  case spirv::Decoration::Patch:
     // For unit attributes and decoration attributes, the args list
     // has no values so we do nothing.
     if (isa<UnitAttr, DecorationAttr>(attr))
@@ -954,6 +970,11 @@ Serializer::prepareDenseElementsConstant(Location loc, Type constType,
   uint32_t resultID = getNextID();
   SmallVector<uint32_t, 4> operands = {typeID, resultID};
   auto elementType = cast<spirv::CompositeType>(constType).getElementType(0);
+  if (auto tensorArmType = dyn_cast<spirv::TensorArmType>(constType)) {
+    ArrayRef<int64_t> innerShape = tensorArmType.getShape().drop_front();
+    if (!innerShape.empty())
+      elementType = spirv::TensorArmType::get(innerShape, elementType);
+  }
 
   // "If the Result Type is a cooperative matrix type, then there must be only
   // one Constituent, with scalar type matching the cooperative matrix Component
@@ -977,30 +998,10 @@ Serializer::prepareDenseElementsConstant(Location loc, Type constType,
     } else {
       return 0;
     }
-  } else if (isa<spirv::TensorArmType>(constType)) {
-    if (isZeroValue(valueAttr)) {
-      encodeInstructionInto(typesGlobalValues, spirv::Opcode::OpConstantNull,
-                            {typeID, resultID});
-      return resultID;
-    }
-    numberOfConstituents = shapedType.getNumElements();
-    operands.reserve(numberOfConstituents + 2);
-    for (int i = 0; i < numberOfConstituents; ++i) {
-      uint32_t elementID = 0;
-      if (auto attr = dyn_cast<DenseIntElementsAttr>(valueAttr)) {
-        elementID =
-            elementType.isInteger(1)
-                ? prepareConstantBool(loc, attr.getValues<BoolAttr>()[i])
-                : prepareConstantInt(loc, attr.getValues<IntegerAttr>()[i]);
-      }
-      if (auto attr = dyn_cast<DenseFPElementsAttr>(valueAttr)) {
-        elementID = prepareConstantFp(loc, attr.getValues<FloatAttr>()[i]);
-      }
-      if (!elementID) {
-        return 0;
-      }
-      operands.push_back(elementID);
-    }
+  } else if (isa<spirv::TensorArmType>(constType) && isZeroValue(valueAttr)) {
+    encodeInstructionInto(typesGlobalValues, spirv::Opcode::OpConstantNull,
+                          {typeID, resultID});
+    return resultID;
   } else {
     operands.reserve(numberOfConstituents + 2);
     for (int i = 0; i < numberOfConstituents; ++i) {
diff --git a/mlir/lib/Target/SPIRV/Serialization/Serializer.h b/mlir/lib/Target/SPIRV/Serialization/Serializer.h
index 7047869bca4cd..fb2cecdff8e43 100644
--- a/mlir/lib/Target/SPIRV/Serialization/Serializer.h
+++ b/mlir/lib/Target/SPIRV/Serialization/Serializer.h
@@ -102,7 +102,7 @@ class Serializer {
 
   void processDebugInfo();
 
-  void processExtension();
+  LogicalResult processExtension();
 
   void processMemoryModel();
 
diff --git a/mlir/lib/Target/SPIRV/TranslateRegistration.cpp b/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
index ac338d555e320..796354e154c02 100644
--- a/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
+++ b/mlir/lib/Target/SPIRV/TranslateRegistration.cpp
@@ -21,8 +21,11 @@
 #include "mlir/Target/SPIRV/Serialization.h"
 #include "mlir/Tools/mlir-translate/Translation.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/ToolOutputFile.h"
 
 using namespace mlir;
 
@@ -76,24 +79,66 @@ void registerFromSPIRVTranslation() {
 // Serialization registration
 //===----------------------------------------------------------------------===//
 
-static LogicalResult serializeModule(spirv::ModuleOp module,
-                                     raw_ostream &output) {
+static LogicalResult
+serializeModule(spirv::ModuleOp moduleOp, raw_ostream &output,
+                const spirv::SerializationOptions &options) {
   SmallVector<uint32_t, 0> binary;
-  if (failed(spirv::serialize(module, binary)))
+  if (failed(spirv::serialize(moduleOp, binary)))
     return failure();
 
-  output.write(reinterpret_cast<char *>(binary.data()),
-               binary.size() * sizeof(uint32_t));
+  size_t sizeInBytes = binary.size() * sizeof(uint32_t);
+
+  output.write(reinterpret_cast<char *>(binary.data()), sizeInBytes);
+
+  if (options.saveModuleForValidation) {
+    size_t dirSeparator =
+        options.validationFilePrefix.find(llvm::sys::path::get_separator());
+    // If file prefix includes directory check if that directory exists.
+    if (dirSeparator != std::string::npos) {
+      llvm::StringRef parentDir =
+          llvm::sys::path::parent_path(options.validationFilePrefix);
+      if (!llvm::sys::fs::is_directory(parentDir))
+        return moduleOp.emitError(
+            "validation prefix directory does not exist\n");
+    }
+
+    SmallString<128> filename;
+    int fd = 0;
+
+    std::error_code errorCode = llvm::sys::fs::createUniqueFile(
+        options.validationFilePrefix + "%%%%%%.spv", fd, filename);
+    if (errorCode)
+      return moduleOp.emitError("error creating validation output file: ")
+             << errorCode.message() << "\n";
+
+    llvm::raw_fd_ostream validationOutput(fd, /*shouldClose=*/true);
+    validationOutput.write(reinterpret_cast<char *>(binary.data()),
+                           sizeInBytes);
+    validationOutput.flush();
+  }
 
   return mlir::success();
 }
 
 namespace mlir {
 void registerToSPIRVTranslation() {
+  static llvm::cl::opt<std::string> validationFilesPrefix(
+      "spirv-save-validation-files-with-prefix",
+      llvm::cl::desc(
+          "When non-empty string is passed each serialized SPIR-V module is "
+          "saved to an additional file that starts with the given prefix. This "
+          "is used to generate separate binaries for validation, where "
+          "`--split-input-file` normally combines all outputs into one. The "
+          "one combined output (`-o`) is still written. Created files need to "
+          "be removed manually once processed."),
+      llvm::cl::init(""));
+
   TranslateFromMLIRRegistration toBinary(
       "serialize-spirv", "serialize SPIR-V dialect",
-      [](spirv::ModuleOp module, raw_ostream &output) {
-        return serializeModule(module, output);
+      [](spirv::ModuleOp moduleOp, raw_ostream &output) {
+        return serializeModule(moduleOp, output,
+                               {true, false, !validationFilesPrefix.empty(),
+                                validationFilesPrefix});
       },
       [](DialectRegistry &registry) {
         registry.insert<spirv::SPIRVDialect>();
diff --git a/mlir/lib/Target/Wasm/CMakeLists.txt b/mlir/lib/Target/Wasm/CMakeLists.txt
new file mode 100644
index 0000000000000..890fc0ecfbeb6
--- /dev/null
+++ b/mlir/lib/Target/Wasm/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_mlir_translation_library(MLIRTargetWasmImport
+  TranslateRegistration.cpp
+  TranslateFromWasm.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/Target/Wasm
+
+  LINK_LIBS PUBLIC
+  MLIRWasmSSADialect
+  MLIRIR
+  MLIRSupport
+  MLIRTranslateLib
+)
diff --git a/mlir/lib/Target/Wasm/TranslateFromWasm.cpp b/mlir/lib/Target/Wasm/TranslateFromWasm.cpp
new file mode 100644
index 0000000000000..6afbe0505e649
--- /dev/null
+++ b/mlir/lib/Target/Wasm/TranslateFromWasm.cpp
@@ -0,0 +1,1522 @@
+//===- TranslateFromWasm.cpp - Translating to WasmSSA dialect -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the WebAssembly importer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/WasmSSA/IR/WasmSSA.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinAttributeInterfaces.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/Location.h"
+#include "mlir/Support/LLVM.h"
+#include "mlir/Target/Wasm/WasmBinaryEncoding.h"
+#include "mlir/Target/Wasm/WasmImporter.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/LEB128.h"
+#include "llvm/Support/LogicalResult.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <variant>
+
+#define DEBUG_TYPE "wasm-translate"
+
+static_assert(CHAR_BIT == 8,
+              "This code expects std::byte to be exactly 8 bits");
+
+using namespace mlir;
+using namespace mlir::wasm;
+using namespace mlir::wasmssa;
+
+namespace {
+using section_id_t = uint8_t;
+enum struct WasmSectionType : section_id_t {
+  CUSTOM = 0,
+  TYPE = 1,
+  IMPORT = 2,
+  FUNCTION = 3,
+  TABLE = 4,
+  MEMORY = 5,
+  GLOBAL = 6,
+  EXPORT = 7,
+  START = 8,
+  ELEMENT = 9,
+  CODE = 10,
+  DATA = 11,
+  DATACOUNT = 12
+};
+
+constexpr section_id_t highestWasmSectionID{
+    static_cast<section_id_t>(WasmSectionType::DATACOUNT)};
+
+#define APPLY_WASM_SEC_TRANSFORM                                               \
+  WASM_SEC_TRANSFORM(CUSTOM)                                                   \
+  WASM_SEC_TRANSFORM(TYPE)                                                     \
+  WASM_SEC_TRANSFORM(IMPORT)                                                   \
+  WASM_SEC_TRANSFORM(FUNCTION)                                                 \
+  WASM_SEC_TRANSFORM(TABLE)                                                    \
+  WASM_SEC_TRANSFORM(MEMORY)                                                   \
+  WASM_SEC_TRANSFORM(GLOBAL)                                                   \
+  WASM_SEC_TRANSFORM(EXPORT)                                                   \
+  WASM_SEC_TRANSFORM(START)                                                    \
+  WASM_SEC_TRANSFORM(ELEMENT)                                                  \
+  WASM_SEC_TRANSFORM(CODE)                                                     \
+  WASM_SEC_TRANSFORM(DATA)                                                     \
+  WASM_SEC_TRANSFORM(DATACOUNT)
+
+template <WasmSectionType>
+constexpr const char *wasmSectionName = "";
+
+#define WASM_SEC_TRANSFORM(section)                                            \
+  template <>                                                                  \
+  [[maybe_unused]] constexpr const char                                        \
+      *wasmSectionName<WasmSectionType::section> = #section;
+APPLY_WASM_SEC_TRANSFORM
+#undef WASM_SEC_TRANSFORM
+
+constexpr bool sectionShouldBeUnique(WasmSectionType secType) {
+  return secType != WasmSectionType::CUSTOM;
+}
+
+template <std::byte... Bytes>
+struct ByteSequence {};
+
+/// Template class for representing a byte sequence of only one byte
+template <std::byte Byte>
+struct UniqueByte : ByteSequence<Byte> {};
+
+[[maybe_unused]] constexpr ByteSequence<
+    WasmBinaryEncoding::Type::i32, WasmBinaryEncoding::Type::i64,
+    WasmBinaryEncoding::Type::f32, WasmBinaryEncoding::Type::f64,
+    WasmBinaryEncoding::Type::v128> valueTypesEncodings{};
+
+template <std::byte... allowedFlags>
+constexpr bool isValueOneOf(std::byte value,
+                            ByteSequence<allowedFlags...> = {}) {
+  return ((value == allowedFlags) | ... | false);
+}
+
+template <std::byte... flags>
+constexpr bool isNotIn(std::byte value, ByteSequence<flags...> = {}) {
+  return !isValueOneOf<flags...>(value);
+}
+
+struct GlobalTypeRecord {
+  Type type;
+  bool isMutable;
+};
+
+struct TypeIdxRecord {
+  size_t id;
+};
+
+struct SymbolRefContainer {
+  FlatSymbolRefAttr symbol;
+};
+
+struct GlobalSymbolRefContainer : SymbolRefContainer {
+  Type globalType;
+};
+
+struct FunctionSymbolRefContainer : SymbolRefContainer {
+  FunctionType functionType;
+};
+
+using ImportDesc =
+    std::variant<TypeIdxRecord, TableType, LimitType, GlobalTypeRecord>;
+
+using parsed_inst_t = FailureOr<SmallVector<Value>>;
+
+struct WasmModuleSymbolTables {
+  SmallVector<FunctionSymbolRefContainer> funcSymbols;
+  SmallVector<GlobalSymbolRefContainer> globalSymbols;
+  SmallVector<SymbolRefContainer> memSymbols;
+  SmallVector<SymbolRefContainer> tableSymbols;
+  SmallVector<FunctionType> moduleFuncTypes;
+
+  std::string getNewSymbolName(StringRef prefix, size_t id) const {
+    return (prefix + Twine{id}).str();
+  }
+
+  std::string getNewFuncSymbolName() const {
+    size_t id = funcSymbols.size();
+    return getNewSymbolName("func_", id);
+  }
+
+  std::string getNewGlobalSymbolName() const {
+    size_t id = globalSymbols.size();
+    return getNewSymbolName("global_", id);
+  }
+
+  std::string getNewMemorySymbolName() const {
+    size_t id = memSymbols.size();
+    return getNewSymbolName("mem_", id);
+  }
+
+  std::string getNewTableSymbolName() const {
+    size_t id = tableSymbols.size();
+    return getNewSymbolName("table_", id);
+  }
+};
+
+class ParserHead;
+
+/// Wrapper around SmallVector to only allow access as push and pop on the
+/// stack. Makes sure that there are no "free accesses" on the stack to preserve
+/// its state.
+class ValueStack {
+private:
+  struct LabelLevel {
+    size_t stackIdx;
+    LabelLevelOpInterface levelOp;
+  };
+
+public:
+  bool empty() const { return values.empty(); }
+
+  size_t size() const { return values.size(); }
+
+  /// Pops values from the stack because they are being used in an operation.
+  /// @param operandTypes The list of expected types of the operation, used
+  ///   to know how many values to pop and check if the types match the
+  ///   expectation.
+  /// @param opLoc Location of the caller, used to report accurately the
+  /// location
+  ///   if an error occurs.
+  /// @return Failure or the vector of popped values.
+  FailureOr<SmallVector<Value>> popOperands(TypeRange operandTypes,
+                                            Location *opLoc);
+
+  /// Push the results of an operation to the stack so they can be used in a
+  /// following operation.
+  /// @param results The list of results of the operation
+  /// @param opLoc Location of the caller, used to report accurately the
+  /// location
+  ///   if an error occurs.
+  LogicalResult pushResults(ValueRange results, Location *opLoc);
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  /// A simple dump function for debugging.
+  /// Writes output to llvm::dbgs().
+  LLVM_DUMP_METHOD void dump() const;
+#endif
+
+private:
+  SmallVector<Value> values;
+};
+
+using local_val_t = TypedValue<wasmssa::LocalRefType>;
+
+class ExpressionParser {
+public:
+  using locals_t = SmallVector<local_val_t>;
+  ExpressionParser(ParserHead &parser, WasmModuleSymbolTables const &symbols,
+                   ArrayRef<local_val_t> initLocal)
+      : parser{parser}, symbols{symbols}, locals{initLocal} {}
+
+private:
+  template <std::byte opCode>
+  inline parsed_inst_t parseSpecificInstruction(OpBuilder &builder);
+
+  template <typename valueT>
+  parsed_inst_t
+  parseConstInst(OpBuilder &builder,
+                 std::enable_if_t<std::is_arithmetic_v<valueT>> * = nullptr);
+
+  /// Construct an operation with \p numOperands operands and a single result.
+  /// Each operand must have the same type. Suitable for e.g. binops, unary
+  /// ops, etc.
+  ///
+  /// \p opcode - The WASM opcode to build.
+  /// \p valueType - The operand and result type for the built instruction.
+  /// \p numOperands - The number of operands for the built operation.
+  ///
+  /// \returns The parsed instruction result, or failure.
+  template <typename opcode, typename valueType, unsigned int numOperands>
+  inline parsed_inst_t
+  buildNumericOp(OpBuilder &builder,
+                 std::enable_if_t<std::is_arithmetic_v<valueType>> * = nullptr);
+
+  /// This function generates a dispatch tree to associate an opcode with a
+  /// parser. Parsers are registered by specialising the
+  /// `parseSpecificInstruction` function for the op code to handle.
+  ///
+  /// The dispatcher is generated by recursively creating all possible patterns
+  /// for an opcode and calling the relevant parser on the leaf.
+  ///
+  /// @tparam patternBitSize is the first bit for which the pattern is not fixed
+  ///
+  /// @tparam highBitPattern is the fixed pattern that this instance handles for
+  /// the 8-patternBitSize bits
+  template <size_t patternBitSize = 0, std::byte highBitPattern = std::byte{0}>
+  inline parsed_inst_t dispatchToInstParser(std::byte opCode,
+                                            OpBuilder &builder) {
+    static_assert(patternBitSize <= 8,
+                  "PatternBitSize is outside of range of opcode space! "
+                  "(expected at most 8 bits)");
+    if constexpr (patternBitSize < 8) {
+      constexpr std::byte bitSelect{1 << (7 - patternBitSize)};
+      constexpr std::byte nextHighBitPatternStem = highBitPattern << 1;
+      constexpr size_t nextPatternBitSize = patternBitSize + 1;
+      if ((opCode & bitSelect) != std::byte{0})
+        return dispatchToInstParser<nextPatternBitSize,
+                                    nextHighBitPatternStem | std::byte{1}>(
+            opCode, builder);
+      return dispatchToInstParser<nextPatternBitSize, nextHighBitPatternStem>(
+          opCode, builder);
+    } else {
+      return parseSpecificInstruction<highBitPattern>(builder);
+    }
+  }
+
+  struct ParseResultWithInfo {
+    SmallVector<Value> opResults;
+    std::byte endingByte;
+  };
+
+public:
+  template <std::byte ParseEndByte = WasmBinaryEncoding::endByte>
+  parsed_inst_t parse(OpBuilder &builder, UniqueByte<ParseEndByte> = {});
+
+  template <std::byte... ExpressionParseEnd>
+  FailureOr<ParseResultWithInfo>
+  parse(OpBuilder &builder,
+        ByteSequence<ExpressionParseEnd...> parsingEndFilters);
+
+  FailureOr<SmallVector<Value>> popOperands(TypeRange operandTypes) {
+    return valueStack.popOperands(operandTypes, &currentOpLoc.value());
+  }
+
+  LogicalResult pushResults(ValueRange results) {
+    return valueStack.pushResults(results, &currentOpLoc.value());
+  }
+
+  /// The local.set and local.tee operations behave similarly and only differ
+  /// on their return value. This function factorizes the behavior of the two
+  /// operations in one place.
+  template <typename OpToCreate>
+  parsed_inst_t parseSetOrTee(OpBuilder &);
+
+private:
+  std::optional<Location> currentOpLoc;
+  ParserHead &parser;
+  WasmModuleSymbolTables const &symbols;
+  locals_t locals;
+  ValueStack valueStack;
+};
+
+class ParserHead {
+public:
+  ParserHead(StringRef src, StringAttr name) : head{src}, locName{name} {}
+  ParserHead(ParserHead &&) = default;
+
+private:
+  ParserHead(ParserHead const &other) = default;
+
+public:
+  auto getLocation() const {
+    return FileLineColLoc::get(locName, 0, anchorOffset + offset);
+  }
+
+  FailureOr<StringRef> consumeNBytes(size_t nBytes) {
+    LDBG() << "Consume " << nBytes << " bytes";
+    LDBG() << "  Bytes remaining: " << size();
+    LDBG() << "  Current offset: " << offset;
+    if (nBytes > size())
+      return emitError(getLocation(), "trying to extract ")
+             << nBytes << "bytes when only " << size() << "are available";
+
+    StringRef res = head.slice(offset, offset + nBytes);
+    offset += nBytes;
+    LDBG() << "  Updated offset (+" << nBytes << "): " << offset;
+    return res;
+  }
+
+  FailureOr<std::byte> consumeByte() {
+    FailureOr<StringRef> res = consumeNBytes(1);
+    if (failed(res))
+      return failure();
+    return std::byte{*res->bytes_begin()};
+  }
+
+  template <typename T>
+  FailureOr<T> parseLiteral();
+
+  FailureOr<uint32_t> parseVectorSize();
+
+private:
+  // TODO: This is equivalent to parseLiteral<uint32_t> and could be removed
+  // if parseLiteral specialization were moved here, but default GCC on Ubuntu
+  // 22.04 has bug with template specialization in class declaration
+  inline FailureOr<uint32_t> parseUI32();
+  inline FailureOr<int64_t> parseI64();
+
+public:
+  FailureOr<StringRef> parseName() {
+    FailureOr<uint32_t> size = parseVectorSize();
+    if (failed(size))
+      return failure();
+
+    return consumeNBytes(*size);
+  }
+
+  FailureOr<WasmSectionType> parseWasmSectionType() {
+    FailureOr<std::byte> id = consumeByte();
+    if (failed(id))
+      return failure();
+    if (std::to_integer<unsigned>(*id) > highestWasmSectionID)
+      return emitError(getLocation(), "invalid section ID: ")
+             << static_cast<int>(*id);
+    return static_cast<WasmSectionType>(*id);
+  }
+
+  FailureOr<LimitType> parseLimit(MLIRContext *ctx) {
+    using WasmLimits = WasmBinaryEncoding::LimitHeader;
+    FileLineColLoc limitLocation = getLocation();
+    FailureOr<std::byte> limitHeader = consumeByte();
+    if (failed(limitHeader))
+      return failure();
+
+    if (isNotIn<WasmLimits::bothLimits, WasmLimits::lowLimitOnly>(*limitHeader))
+      return emitError(limitLocation, "invalid limit header: ")
+             << static_cast<int>(*limitHeader);
+    FailureOr<uint32_t> minParse = parseUI32();
+    if (failed(minParse))
+      return failure();
+    std::optional<uint32_t> max{std::nullopt};
+    if (*limitHeader == WasmLimits::bothLimits) {
+      FailureOr<uint32_t> maxParse = parseUI32();
+      if (failed(maxParse))
+        return failure();
+      max = *maxParse;
+    }
+    return LimitType::get(ctx, *minParse, max);
+  }
+
+  FailureOr<Type> parseValueType(MLIRContext *ctx) {
+    FileLineColLoc typeLoc = getLocation();
+    FailureOr<std::byte> typeEncoding = consumeByte();
+    if (failed(typeEncoding))
+      return failure();
+    switch (*typeEncoding) {
+    case WasmBinaryEncoding::Type::i32:
+      return IntegerType::get(ctx, 32);
+    case WasmBinaryEncoding::Type::i64:
+      return IntegerType::get(ctx, 64);
+    case WasmBinaryEncoding::Type::f32:
+      return Float32Type::get(ctx);
+    case WasmBinaryEncoding::Type::f64:
+      return Float64Type::get(ctx);
+    case WasmBinaryEncoding::Type::v128:
+      return IntegerType::get(ctx, 128);
+    case WasmBinaryEncoding::Type::funcRef:
+      return wasmssa::FuncRefType::get(ctx);
+    case WasmBinaryEncoding::Type::externRef:
+      return wasmssa::ExternRefType::get(ctx);
+    default:
+      return emitError(typeLoc, "invalid value type encoding: ")
+             << static_cast<int>(*typeEncoding);
+    }
+  }
+
+  FailureOr<GlobalTypeRecord> parseGlobalType(MLIRContext *ctx) {
+    using WasmGlobalMut = WasmBinaryEncoding::GlobalMutability;
+    FailureOr<Type> typeParsed = parseValueType(ctx);
+    if (failed(typeParsed))
+      return failure();
+    FileLineColLoc mutLoc = getLocation();
+    FailureOr<std::byte> mutSpec = consumeByte();
+    if (failed(mutSpec))
+      return failure();
+    if (isNotIn<WasmGlobalMut::isConst, WasmGlobalMut::isMutable>(*mutSpec))
+      return emitError(mutLoc, "invalid global mutability specifier: ")
+             << static_cast<int>(*mutSpec);
+    return GlobalTypeRecord{*typeParsed, *mutSpec == WasmGlobalMut::isMutable};
+  }
+
+  FailureOr<TupleType> parseResultType(MLIRContext *ctx) {
+    FailureOr<uint32_t> nParamsParsed = parseVectorSize();
+    if (failed(nParamsParsed))
+      return failure();
+    uint32_t nParams = *nParamsParsed;
+    SmallVector<Type> res{};
+    res.reserve(nParams);
+    for (size_t i = 0; i < nParams; ++i) {
+      FailureOr<Type> parsedType = parseValueType(ctx);
+      if (failed(parsedType))
+        return failure();
+      res.push_back(*parsedType);
+    }
+    return TupleType::get(ctx, res);
+  }
+
+  FailureOr<FunctionType> parseFunctionType(MLIRContext *ctx) {
+    FileLineColLoc typeLoc = getLocation();
+    FailureOr<std::byte> funcTypeHeader = consumeByte();
+    if (failed(funcTypeHeader))
+      return failure();
+    if (*funcTypeHeader != WasmBinaryEncoding::Type::funcType)
+      return emitError(typeLoc, "invalid function type header byte. Expecting ")
+             << std::to_integer<unsigned>(WasmBinaryEncoding::Type::funcType)
+             << " got " << std::to_integer<unsigned>(*funcTypeHeader);
+    FailureOr<TupleType> inputTypes = parseResultType(ctx);
+    if (failed(inputTypes))
+      return failure();
+
+    FailureOr<TupleType> resTypes = parseResultType(ctx);
+    if (failed(resTypes))
+      return failure();
+
+    return FunctionType::get(ctx, inputTypes->getTypes(), resTypes->getTypes());
+  }
+
+  FailureOr<TypeIdxRecord> parseTypeIndex() {
+    FailureOr<uint32_t> res = parseUI32();
+    if (failed(res))
+      return failure();
+    return TypeIdxRecord{*res};
+  }
+
+  FailureOr<TableType> parseTableType(MLIRContext *ctx) {
+    FailureOr<Type> elmTypeParse = parseValueType(ctx);
+    if (failed(elmTypeParse))
+      return failure();
+    if (!isWasmRefType(*elmTypeParse))
+      return emitError(getLocation(), "invalid element type for table");
+    FailureOr<LimitType> limitParse = parseLimit(ctx);
+    if (failed(limitParse))
+      return failure();
+    return TableType::get(ctx, *elmTypeParse, *limitParse);
+  }
+
+  FailureOr<ImportDesc> parseImportDesc(MLIRContext *ctx) {
+    FileLineColLoc importLoc = getLocation();
+    FailureOr<std::byte> importType = consumeByte();
+    auto packager = [](auto parseResult) -> FailureOr<ImportDesc> {
+      if (failed(parseResult))
+        return failure();
+      return {*parseResult};
+    };
+    if (failed(importType))
+      return failure();
+    switch (*importType) {
+    case WasmBinaryEncoding::Import::typeID:
+      return packager(parseTypeIndex());
+    case WasmBinaryEncoding::Import::tableType:
+      return packager(parseTableType(ctx));
+    case WasmBinaryEncoding::Import::memType:
+      return packager(parseLimit(ctx));
+    case WasmBinaryEncoding::Import::globalType:
+      return packager(parseGlobalType(ctx));
+    default:
+      return emitError(importLoc, "invalid import type descriptor: ")
+             << static_cast<int>(*importType);
+    }
+  }
+
+  parsed_inst_t parseExpression(OpBuilder &builder,
+                                WasmModuleSymbolTables const &symbols,
+                                ArrayRef<local_val_t> locals = {}) {
+    auto eParser = ExpressionParser{*this, symbols, locals};
+    return eParser.parse(builder);
+  }
+
+  LogicalResult parseCodeFor(FuncOp func,
+                             WasmModuleSymbolTables const &symbols) {
+    SmallVector<local_val_t> locals{};
+    // Populating locals with function argument
+    Block &block = func.getBody().front();
+    // Delete temporary return argument which was only created for IR validity
+    assert(func.getBody().getBlocks().size() == 1 &&
+           "Function should only have its default created block at this point");
+    assert(block.getOperations().size() == 1 &&
+           "Only the placeholder return op should be present at this point");
+    auto returnOp = cast<ReturnOp>(&block.back());
+    assert(returnOp);
+
+    FailureOr<uint32_t> codeSizeInBytes = parseUI32();
+    if (failed(codeSizeInBytes))
+      return failure();
+    FailureOr<StringRef> codeContent = consumeNBytes(*codeSizeInBytes);
+    if (failed(codeContent))
+      return failure();
+    auto name = StringAttr::get(func->getContext(),
+                                locName.str() + "::" + func.getSymName());
+    auto cParser = ParserHead{*codeContent, name};
+    FailureOr<uint32_t> localVecSize = cParser.parseVectorSize();
+    if (failed(localVecSize))
+      return failure();
+    OpBuilder builder{&func.getBody().front().back()};
+    for (auto arg : block.getArguments())
+      locals.push_back(cast<TypedValue<LocalRefType>>(arg));
+    // Declare the local ops
+    uint32_t nVarVec = *localVecSize;
+    for (size_t i = 0; i < nVarVec; ++i) {
+      FileLineColLoc varLoc = cParser.getLocation();
+      FailureOr<uint32_t> nSubVar = cParser.parseUI32();
+      if (failed(nSubVar))
+        return failure();
+      FailureOr<Type> varT = cParser.parseValueType(func->getContext());
+      if (failed(varT))
+        return failure();
+      for (size_t j = 0; j < *nSubVar; ++j) {
+        auto local = builder.create<LocalOp>(varLoc, *varT);
+        locals.push_back(local.getResult());
+      }
+    }
+    parsed_inst_t res = cParser.parseExpression(builder, symbols, locals);
+    if (failed(res))
+      return failure();
+    if (!cParser.end())
+      return emitError(cParser.getLocation(),
+                       "unparsed garbage remaining at end of code block");
+    builder.create<ReturnOp>(func->getLoc(), *res);
+    returnOp->erase();
+    return success();
+  }
+
+  bool end() const { return curHead().empty(); }
+
+  ParserHead copy() const { return *this; }
+
+private:
+  StringRef curHead() const { return head.drop_front(offset); }
+
+  FailureOr<std::byte> peek() const {
+    if (end())
+      return emitError(
+          getLocation(),
+          "trying to peek at next byte, but input stream is empty");
+    return static_cast<std::byte>(curHead().front());
+  }
+
+  size_t size() const { return head.size() - offset; }
+
+  StringRef head;
+  StringAttr locName;
+  unsigned anchorOffset{0};
+  unsigned offset{0};
+};
+
+template <>
+FailureOr<float> ParserHead::parseLiteral<float>() {
+  FailureOr<StringRef> bytes = consumeNBytes(4);
+  if (failed(bytes))
+    return failure();
+  return llvm::support::endian::read<float>(bytes->bytes_begin(),
+                                            llvm::endianness::little);
+}
+
+template <>
+FailureOr<double> ParserHead::parseLiteral<double>() {
+  FailureOr<StringRef> bytes = consumeNBytes(8);
+  if (failed(bytes))
+    return failure();
+  return llvm::support::endian::read<double>(bytes->bytes_begin(),
+                                             llvm::endianness::little);
+}
+
+template <>
+FailureOr<uint32_t> ParserHead::parseLiteral<uint32_t>() {
+  char const *error = nullptr;
+  uint32_t res{0};
+  unsigned encodingSize{0};
+  StringRef src = curHead();
+  uint64_t decoded = llvm::decodeULEB128(src.bytes_begin(), &encodingSize,
+                                         src.bytes_end(), &error);
+  if (error)
+    return emitError(getLocation(), error);
+
+  if (std::isgreater(decoded, std::numeric_limits<uint32_t>::max()))
+    return emitError(getLocation()) << "literal does not fit on 32 bits";
+
+  res = static_cast<uint32_t>(decoded);
+  offset += encodingSize;
+  return res;
+}
+
+template <>
+FailureOr<int32_t> ParserHead::parseLiteral<int32_t>() {
+  char const *error = nullptr;
+  int32_t res{0};
+  unsigned encodingSize{0};
+  StringRef src = curHead();
+  int64_t decoded = llvm::decodeSLEB128(src.bytes_begin(), &encodingSize,
+                                        src.bytes_end(), &error);
+  if (error)
+    return emitError(getLocation(), error);
+  if (std::isgreater(decoded, std::numeric_limits<int32_t>::max()) ||
+      std::isgreater(std::numeric_limits<int32_t>::min(), decoded))
+    return emitError(getLocation()) << "literal does not fit on 32 bits";
+
+  res = static_cast<int32_t>(decoded);
+  offset += encodingSize;
+  return res;
+}
+
+template <>
+FailureOr<int64_t> ParserHead::parseLiteral<int64_t>() {
+  char const *error = nullptr;
+  unsigned encodingSize{0};
+  StringRef src = curHead();
+  int64_t res = llvm::decodeSLEB128(src.bytes_begin(), &encodingSize,
+                                    src.bytes_end(), &error);
+  if (error)
+    return emitError(getLocation(), error);
+
+  offset += encodingSize;
+  return res;
+}
+
+FailureOr<uint32_t> ParserHead::parseVectorSize() {
+  return parseLiteral<uint32_t>();
+}
+
+inline FailureOr<uint32_t> ParserHead::parseUI32() {
+  return parseLiteral<uint32_t>();
+}
+
+inline FailureOr<int64_t> ParserHead::parseI64() {
+  return parseLiteral<int64_t>();
+}
+
+template <std::byte opCode>
+inline parsed_inst_t ExpressionParser::parseSpecificInstruction(OpBuilder &) {
+  return emitError(*currentOpLoc, "unknown instruction opcode: ")
+         << static_cast<int>(opCode);
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void ValueStack::dump() const {
+  llvm::dbgs() << "================= Wasm ValueStack =======================\n";
+  llvm::dbgs() << "size: " << size() << "\n";
+  llvm::dbgs() << "<Top>"
+               << "\n";
+  // Stack is pushed to via push_back. Therefore the top of the stack is the
+  // end of the vector. Iterate in reverse so that the first thing we print
+  // is the top of the stack.
+  size_t stackSize = size();
+  for (size_t idx = 0; idx < stackSize; idx++) {
+    size_t actualIdx = stackSize - 1 - idx;
+    llvm::dbgs() << "  ";
+    values[actualIdx].dump();
+  }
+  llvm::dbgs() << "<Bottom>"
+               << "\n";
+  llvm::dbgs() << "=========================================================\n";
+}
+#endif
+
+parsed_inst_t ValueStack::popOperands(TypeRange operandTypes, Location *opLoc) {
+  LDBG() << "Popping from ValueStack\n"
+         << "  Elements(s) to pop: " << operandTypes.size() << "\n"
+         << "  Current stack size: " << values.size();
+  if (operandTypes.size() > values.size())
+    return emitError(*opLoc,
+                     "stack doesn't contain enough values. trying to get ")
+           << operandTypes.size() << " operands on a stack containing only "
+           << values.size() << " values.";
+  size_t stackIdxOffset = values.size() - operandTypes.size();
+  SmallVector<Value> res{};
+  res.reserve(operandTypes.size());
+  for (size_t i{0}; i < operandTypes.size(); ++i) {
+    Value operand = values[i + stackIdxOffset];
+    Type stackType = operand.getType();
+    if (stackType != operandTypes[i])
+      return emitError(*opLoc, "invalid operand type on stack. expecting ")
+             << operandTypes[i] << ", value on stack is of type " << stackType
+             << ".";
+    LDBG() << "    POP: " << operand;
+    res.push_back(operand);
+  }
+  values.resize(values.size() - operandTypes.size());
+  LDBG() << "  Updated stack size: " << values.size();
+  return res;
+}
+
+LogicalResult ValueStack::pushResults(ValueRange results, Location *opLoc) {
+  LDBG() << "Pushing to ValueStack\n"
+         << "  Elements(s) to push: " << results.size() << "\n"
+         << "  Current stack size: " << values.size();
+  for (Value val : results) {
+    if (!isWasmValueType(val.getType()))
+      return emitError(*opLoc, "invalid value type on stack: ")
+             << val.getType();
+    LDBG() << "    PUSH: " << val;
+    values.push_back(val);
+  }
+
+  LDBG() << "  Updated stack size: " << values.size();
+  return success();
+}
+
+template <std::byte EndParseByte>
+parsed_inst_t ExpressionParser::parse(OpBuilder &builder,
+                                      UniqueByte<EndParseByte> endByte) {
+  auto res = parse(builder, ByteSequence<EndParseByte>{});
+  if (failed(res))
+    return failure();
+  return res->opResults;
+}
+
+template <std::byte... ExpressionParseEnd>
+FailureOr<ExpressionParser::ParseResultWithInfo>
+ExpressionParser::parse(OpBuilder &builder,
+                        ByteSequence<ExpressionParseEnd...> parsingEndFilters) {
+  SmallVector<Value> res;
+  for (;;) {
+    currentOpLoc = parser.getLocation();
+    FailureOr<std::byte> opCode = parser.consumeByte();
+    if (failed(opCode))
+      return failure();
+    if (isValueOneOf(*opCode, parsingEndFilters))
+      return {{res, *opCode}};
+    parsed_inst_t resParsed;
+    resParsed = dispatchToInstParser(*opCode, builder);
+    if (failed(resParsed))
+      return failure();
+    std::swap(res, *resParsed);
+    if (failed(pushResults(res)))
+      return failure();
+  }
+}
+
+template <>
+inline parsed_inst_t ExpressionParser::parseSpecificInstruction<
+    WasmBinaryEncoding::OpCode::localGet>(OpBuilder &builder) {
+  FailureOr<uint32_t> id = parser.parseLiteral<uint32_t>();
+  Location instLoc = *currentOpLoc;
+  if (failed(id))
+    return failure();
+  if (*id >= locals.size())
+    return emitError(instLoc, "invalid local index. function has ")
+           << locals.size() << " accessible locals, received index " << *id;
+  return {{builder.create<LocalGetOp>(instLoc, locals[*id]).getResult()}};
+}
+
+template <>
+inline parsed_inst_t ExpressionParser::parseSpecificInstruction<
+    WasmBinaryEncoding::OpCode::globalGet>(OpBuilder &builder) {
+  FailureOr<uint32_t> id = parser.parseLiteral<uint32_t>();
+  Location instLoc = *currentOpLoc;
+  if (failed(id))
+    return failure();
+  if (*id >= symbols.globalSymbols.size())
+    return emitError(instLoc, "invalid global index. function has ")
+           << symbols.globalSymbols.size()
+           << " accessible globals, received index " << *id;
+  GlobalSymbolRefContainer globalVar = symbols.globalSymbols[*id];
+  auto globalOp = builder.create<GlobalGetOp>(instLoc, globalVar.globalType,
+                                              globalVar.symbol);
+
+  return {{globalOp.getResult()}};
+}
+
+template <typename OpToCreate>
+parsed_inst_t ExpressionParser::parseSetOrTee(OpBuilder &builder) {
+  FailureOr<uint32_t> id = parser.parseLiteral<uint32_t>();
+  if (failed(id))
+    return failure();
+  if (*id >= locals.size())
+    return emitError(*currentOpLoc, "invalid local index. function has ")
+           << locals.size() << " accessible locals, received index " << *id;
+  if (valueStack.empty())
+    return emitError(
+        *currentOpLoc,
+        "invalid stack access, trying to access a value on an empty stack.");
+
+  parsed_inst_t poppedOp = popOperands(locals[*id].getType().getElementType());
+  if (failed(poppedOp))
+    return failure();
+  return {
+      builder.create<OpToCreate>(*currentOpLoc, locals[*id], poppedOp->front())
+          ->getResults()};
+}
+
+template <>
+inline parsed_inst_t ExpressionParser::parseSpecificInstruction<
+    WasmBinaryEncoding::OpCode::localSet>(OpBuilder &builder) {
+  return parseSetOrTee<LocalSetOp>(builder);
+}
+
+template <>
+inline parsed_inst_t ExpressionParser::parseSpecificInstruction<
+    WasmBinaryEncoding::OpCode::localTee>(OpBuilder &builder) {
+  return parseSetOrTee<LocalTeeOp>(builder);
+}
+
+template <typename T>
+inline Type buildLiteralType(OpBuilder &);
+
+template <>
+inline Type buildLiteralType<int32_t>(OpBuilder &builder) {
+  return builder.getI32Type();
+}
+
+template <>
+inline Type buildLiteralType<int64_t>(OpBuilder &builder) {
+  return builder.getI64Type();
+}
+
+template <>
+[[maybe_unused]] inline Type buildLiteralType<uint32_t>(OpBuilder &builder) {
+  return builder.getI32Type();
+}
+
+template <>
+[[maybe_unused]] inline Type buildLiteralType<uint64_t>(OpBuilder &builder) {
+  return builder.getI64Type();
+}
+
+template <>
+inline Type buildLiteralType<float>(OpBuilder &builder) {
+  return builder.getF32Type();
+}
+
+template <>
+inline Type buildLiteralType<double>(OpBuilder &builder) {
+  return builder.getF64Type();
+}
+
+template <typename ValT,
+          typename E = std::enable_if_t<std::is_arithmetic_v<ValT>>>
+struct AttrHolder;
+
+template <typename ValT>
+struct AttrHolder<ValT, std::enable_if_t<std::is_integral_v<ValT>>> {
+  using type = IntegerAttr;
+};
+
+template <typename ValT>
+struct AttrHolder<ValT, std::enable_if_t<std::is_floating_point_v<ValT>>> {
+  using type = FloatAttr;
+};
+
+template <typename ValT>
+using attr_holder_t = typename AttrHolder<ValT>::type;
+
+template <typename ValT,
+          typename EnableT = std::enable_if_t<std::is_arithmetic_v<ValT>>>
+attr_holder_t<ValT> buildLiteralAttr(OpBuilder &builder, ValT val) {
+  return attr_holder_t<ValT>::get(buildLiteralType<ValT>(builder), val);
+}
+
+template <typename valueT>
+parsed_inst_t ExpressionParser::parseConstInst(
+    OpBuilder &builder, std::enable_if_t<std::is_arithmetic_v<valueT>> *) {
+  auto parsedConstant = parser.parseLiteral<valueT>();
+  if (failed(parsedConstant))
+    return failure();
+  auto constOp =
+      ConstOp::create(builder, *currentOpLoc,
+                      buildLiteralAttr<valueT>(builder, *parsedConstant));
+  return {{constOp.getResult()}};
+}
+
+template <>
+inline parsed_inst_t ExpressionParser::parseSpecificInstruction<
+    WasmBinaryEncoding::OpCode::constI32>(OpBuilder &builder) {
+  return parseConstInst<int32_t>(builder);
+}
+
+template <>
+inline parsed_inst_t ExpressionParser::parseSpecificInstruction<
+    WasmBinaryEncoding::OpCode::constI64>(OpBuilder &builder) {
+  return parseConstInst<int64_t>(builder);
+}
+
+template <>
+inline parsed_inst_t ExpressionParser::parseSpecificInstruction<
+    WasmBinaryEncoding::OpCode::constFP32>(OpBuilder &builder) {
+  return parseConstInst<float>(builder);
+}
+
+template <>
+inline parsed_inst_t ExpressionParser::parseSpecificInstruction<
+    WasmBinaryEncoding::OpCode::constFP64>(OpBuilder &builder) {
+  return parseConstInst<double>(builder);
+}
+
+template <typename opcode, typename valueType, unsigned int numOperands>
+inline parsed_inst_t ExpressionParser::buildNumericOp(
+    OpBuilder &builder, std::enable_if_t<std::is_arithmetic_v<valueType>> *) {
+  auto ty = buildLiteralType<valueType>(builder);
+  LDBG() << "*** buildNumericOp: numOperands = " << numOperands
+         << ", type = " << ty << " ***";
+  auto tysToPop = SmallVector<Type, numOperands>();
+  tysToPop.resize(numOperands);
+  std::fill(tysToPop.begin(), tysToPop.end(), ty);
+  auto operands = popOperands(tysToPop);
+  if (failed(operands))
+    return failure();
+  auto op = builder.create<opcode>(*currentOpLoc, *operands).getResult();
+  LDBG() << "Built operation: " << op;
+  return {{op}};
+}
+
+// Convenience macro for generating numerical operations.
+#define BUILD_NUMERIC_OP(OP_NAME, N_ARGS, PREFIX, SUFFIX, TYPE)                \
+  template <>                                                                  \
+  inline parsed_inst_t ExpressionParser::parseSpecificInstruction<             \
+      WasmBinaryEncoding::OpCode::PREFIX##SUFFIX>(OpBuilder & builder) {       \
+    return buildNumericOp<OP_NAME, TYPE, N_ARGS>(builder);                     \
+  }
+
+// Macro to define binops that only support integer types.
+#define BUILD_NUMERIC_BINOP_INT(OP_NAME, PREFIX)                               \
+  BUILD_NUMERIC_OP(OP_NAME, 2, PREFIX, I32, int32_t)                           \
+  BUILD_NUMERIC_OP(OP_NAME, 2, PREFIX, I64, int64_t)
+
+// Macro to define binops that only support floating point types.
+#define BUILD_NUMERIC_BINOP_FP(OP_NAME, PREFIX)                                \
+  BUILD_NUMERIC_OP(OP_NAME, 2, PREFIX, F32, float)                             \
+  BUILD_NUMERIC_OP(OP_NAME, 2, PREFIX, F64, double)
+
+// Macro to define binops that support both floating point and integer types.
+#define BUILD_NUMERIC_BINOP_INTFP(OP_NAME, PREFIX)                             \
+  BUILD_NUMERIC_BINOP_INT(OP_NAME, PREFIX)                                     \
+  BUILD_NUMERIC_BINOP_FP(OP_NAME, PREFIX)
+
+// Macro to implement unary ops that only support integers.
+#define BUILD_NUMERIC_UNARY_OP_INT(OP_NAME, PREFIX)                            \
+  BUILD_NUMERIC_OP(OP_NAME, 1, PREFIX, I32, int32_t)                           \
+  BUILD_NUMERIC_OP(OP_NAME, 1, PREFIX, I64, int64_t)
+
+// Macro to implement unary ops that support integer and floating point types.
+#define BUILD_NUMERIC_UNARY_OP_FP(OP_NAME, PREFIX)                             \
+  BUILD_NUMERIC_OP(OP_NAME, 1, PREFIX, F32, float)                             \
+  BUILD_NUMERIC_OP(OP_NAME, 1, PREFIX, F64, double)
+
+BUILD_NUMERIC_BINOP_FP(CopySignOp, copysign)
+BUILD_NUMERIC_BINOP_FP(DivOp, div)
+BUILD_NUMERIC_BINOP_FP(MaxOp, max)
+BUILD_NUMERIC_BINOP_FP(MinOp, min)
+BUILD_NUMERIC_BINOP_INT(AndOp, and)
+BUILD_NUMERIC_BINOP_INT(DivSIOp, divS)
+BUILD_NUMERIC_BINOP_INT(DivUIOp, divU)
+BUILD_NUMERIC_BINOP_INT(OrOp, or)
+BUILD_NUMERIC_BINOP_INT(RemSIOp, remS)
+BUILD_NUMERIC_BINOP_INT(RemUIOp, remU)
+BUILD_NUMERIC_BINOP_INT(RotlOp, rotl)
+BUILD_NUMERIC_BINOP_INT(RotrOp, rotr)
+BUILD_NUMERIC_BINOP_INT(ShLOp, shl)
+BUILD_NUMERIC_BINOP_INT(ShRSOp, shrS)
+BUILD_NUMERIC_BINOP_INT(ShRUOp, shrU)
+BUILD_NUMERIC_BINOP_INT(XOrOp, xor)
+BUILD_NUMERIC_BINOP_INTFP(AddOp, add)
+BUILD_NUMERIC_BINOP_INTFP(MulOp, mul)
+BUILD_NUMERIC_BINOP_INTFP(SubOp, sub)
+BUILD_NUMERIC_UNARY_OP_FP(AbsOp, abs)
+BUILD_NUMERIC_UNARY_OP_FP(CeilOp, ceil)
+BUILD_NUMERIC_UNARY_OP_FP(FloorOp, floor)
+BUILD_NUMERIC_UNARY_OP_FP(NegOp, neg)
+BUILD_NUMERIC_UNARY_OP_FP(SqrtOp, sqrt)
+BUILD_NUMERIC_UNARY_OP_FP(TruncOp, trunc)
+BUILD_NUMERIC_UNARY_OP_INT(ClzOp, clz)
+BUILD_NUMERIC_UNARY_OP_INT(CtzOp, ctz)
+BUILD_NUMERIC_UNARY_OP_INT(PopCntOp, popcnt)
+
+// Don't need these anymore so let's undef them.
+#undef BUILD_NUMERIC_BINOP_FP
+#undef BUILD_NUMERIC_BINOP_INT
+#undef BUILD_NUMERIC_BINOP_INTFP
+#undef BUILD_NUMERIC_UNARY_OP_FP
+#undef BUILD_NUMERIC_UNARY_OP_INT
+#undef BUILD_NUMERIC_OP
+#undef BUILD_NUMERIC_CAST_OP
+
+class WasmBinaryParser {
+private:
+  struct SectionRegistry {
+    using section_location_t = StringRef;
+
+    std::array<SmallVector<section_location_t>, highestWasmSectionID + 1>
+        registry;
+
+    template <WasmSectionType SecType>
+    std::conditional_t<sectionShouldBeUnique(SecType),
+                       std::optional<section_location_t>,
+                       ArrayRef<section_location_t>>
+    getContentForSection() const {
+      constexpr auto idx = static_cast<size_t>(SecType);
+      if constexpr (sectionShouldBeUnique(SecType)) {
+        return registry[idx].empty() ? std::nullopt
+                                     : std::make_optional(registry[idx][0]);
+      } else {
+        return registry[idx];
+      }
+    }
+
+    bool hasSection(WasmSectionType secType) const {
+      return !registry[static_cast<size_t>(secType)].empty();
+    }
+
+    ///
+    /// @returns success if registration valid, failure in case registration
+    /// can't be done (if another section of same type already exist and this
+    /// section type should only be present once)
+    ///
+    LogicalResult registerSection(WasmSectionType secType,
+                                  section_location_t location, Location loc) {
+      if (sectionShouldBeUnique(secType) && hasSection(secType))
+        return emitError(loc,
+                         "trying to add a second instance of unique section");
+
+      registry[static_cast<size_t>(secType)].push_back(location);
+      emitRemark(loc, "Adding section with section ID ")
+          << static_cast<uint8_t>(secType);
+      return success();
+    }
+
+    LogicalResult populateFromBody(ParserHead ph) {
+      while (!ph.end()) {
+        FileLineColLoc sectionLoc = ph.getLocation();
+        FailureOr<WasmSectionType> secType = ph.parseWasmSectionType();
+        if (failed(secType))
+          return failure();
+
+        FailureOr<uint32_t> secSizeParsed = ph.parseLiteral<uint32_t>();
+        if (failed(secSizeParsed))
+          return failure();
+
+        uint32_t secSize = *secSizeParsed;
+        FailureOr<StringRef> sectionContent = ph.consumeNBytes(secSize);
+        if (failed(sectionContent))
+          return failure();
+
+        LogicalResult registration =
+            registerSection(*secType, *sectionContent, sectionLoc);
+
+        if (failed(registration))
+          return failure();
+      }
+      return success();
+    }
+  };
+
+  auto getLocation(int offset = 0) const {
+    return FileLineColLoc::get(srcName, 0, offset);
+  }
+
+  template <WasmSectionType>
+  LogicalResult parseSectionItem(ParserHead &, size_t);
+
+  template <WasmSectionType section>
+  LogicalResult parseSection() {
+    auto secName = std::string{wasmSectionName<section>};
+    auto sectionNameAttr =
+        StringAttr::get(ctx, srcName.strref() + ":" + secName + "-SECTION");
+    unsigned offset = 0;
+    auto getLocation = [sectionNameAttr, &offset]() {
+      return FileLineColLoc::get(sectionNameAttr, 0, offset);
+    };
+    auto secContent = registry.getContentForSection<section>();
+    if (!secContent) {
+      LDBG() << secName << " section is not present in file.";
+      return success();
+    }
+
+    auto secSrc = secContent.value();
+    ParserHead ph{secSrc, sectionNameAttr};
+    FailureOr<uint32_t> nElemsParsed = ph.parseVectorSize();
+    if (failed(nElemsParsed))
+      return failure();
+    uint32_t nElems = *nElemsParsed;
+    LDBG() << "starting to parse " << nElems << " items for section "
+           << secName;
+    for (size_t i = 0; i < nElems; ++i) {
+      if (failed(parseSectionItem<section>(ph, i)))
+        return failure();
+    }
+
+    if (!ph.end())
+      return emitError(getLocation(), "unparsed garbage at end of section ")
+             << secName;
+    return success();
+  }
+
+  /// Handles the registration of a function import
+  LogicalResult visitImport(Location loc, StringRef moduleName,
+                            StringRef importName, TypeIdxRecord tid) {
+    using llvm::Twine;
+    if (tid.id >= symbols.moduleFuncTypes.size())
+      return emitError(loc, "invalid type id: ")
+             << tid.id << ". Only " << symbols.moduleFuncTypes.size()
+             << " type registration.";
+    FunctionType type = symbols.moduleFuncTypes[tid.id];
+    std::string symbol = symbols.getNewFuncSymbolName();
+    auto funcOp = FuncImportOp::create(builder, loc, symbol, moduleName,
+                                       importName, type);
+    symbols.funcSymbols.push_back({{FlatSymbolRefAttr::get(funcOp)}, type});
+    return funcOp.verify();
+  }
+
+  /// Handles the registration of a memory import
+  LogicalResult visitImport(Location loc, StringRef moduleName,
+                            StringRef importName, LimitType limitType) {
+    std::string symbol = symbols.getNewMemorySymbolName();
+    auto memOp = MemImportOp::create(builder, loc, symbol, moduleName,
+                                     importName, limitType);
+    symbols.memSymbols.push_back({FlatSymbolRefAttr::get(memOp)});
+    return memOp.verify();
+  }
+
+  /// Handles the registration of a table import
+  LogicalResult visitImport(Location loc, StringRef moduleName,
+                            StringRef importName, TableType tableType) {
+    std::string symbol = symbols.getNewTableSymbolName();
+    auto tableOp = TableImportOp::create(builder, loc, symbol, moduleName,
+                                         importName, tableType);
+    symbols.tableSymbols.push_back({FlatSymbolRefAttr::get(tableOp)});
+    return tableOp.verify();
+  }
+
+  /// Handles the registration of a global variable import
+  LogicalResult visitImport(Location loc, StringRef moduleName,
+                            StringRef importName, GlobalTypeRecord globalType) {
+    std::string symbol = symbols.getNewGlobalSymbolName();
+    auto giOp =
+        GlobalImportOp::create(builder, loc, symbol, moduleName, importName,
+                               globalType.type, globalType.isMutable);
+    symbols.globalSymbols.push_back(
+        {{FlatSymbolRefAttr::get(giOp)}, giOp.getType()});
+    return giOp.verify();
+  }
+
+  // Detect occurence of errors
+  LogicalResult peekDiag(Diagnostic &diag) {
+    if (diag.getSeverity() == DiagnosticSeverity::Error)
+      isValid = false;
+    return failure();
+  }
+
+public:
+  WasmBinaryParser(llvm::SourceMgr &sourceMgr, MLIRContext *ctx)
+      : builder{ctx}, ctx{ctx} {
+    ctx->getDiagEngine().registerHandler(
+        [this](Diagnostic &diag) { return peekDiag(diag); });
+    ctx->loadAllAvailableDialects();
+    if (sourceMgr.getNumBuffers() != 1) {
+      emitError(UnknownLoc::get(ctx), "one source file should be provided");
+      return;
+    }
+    uint32_t sourceBufId = sourceMgr.getMainFileID();
+    StringRef source = sourceMgr.getMemoryBuffer(sourceBufId)->getBuffer();
+    srcName = StringAttr::get(
+        ctx, sourceMgr.getMemoryBuffer(sourceBufId)->getBufferIdentifier());
+
+    auto parser = ParserHead{source, srcName};
+    auto const wasmHeader = StringRef{"\0asm", 4};
+    FileLineColLoc magicLoc = parser.getLocation();
+    FailureOr<StringRef> magic = parser.consumeNBytes(wasmHeader.size());
+    if (failed(magic) || magic->compare(wasmHeader)) {
+      emitError(magicLoc, "source file does not contain valid Wasm header.");
+      return;
+    }
+    auto const expectedVersionString = StringRef{"\1\0\0\0", 4};
+    FileLineColLoc versionLoc = parser.getLocation();
+    FailureOr<StringRef> version =
+        parser.consumeNBytes(expectedVersionString.size());
+    if (failed(version))
+      return;
+    if (version->compare(expectedVersionString)) {
+      emitError(versionLoc,
+                "unsupported Wasm version. only version 1 is supported");
+      return;
+    }
+    LogicalResult fillRegistry = registry.populateFromBody(parser.copy());
+    if (failed(fillRegistry))
+      return;
+
+    mOp = ModuleOp::create(builder, getLocation());
+    builder.setInsertionPointToStart(&mOp.getBodyRegion().front());
+    LogicalResult parsingTypes = parseSection<WasmSectionType::TYPE>();
+    if (failed(parsingTypes))
+      return;
+
+    LogicalResult parsingImports = parseSection<WasmSectionType::IMPORT>();
+    if (failed(parsingImports))
+      return;
+
+    firstInternalFuncID = symbols.funcSymbols.size();
+
+    LogicalResult parsingFunctions = parseSection<WasmSectionType::FUNCTION>();
+    if (failed(parsingFunctions))
+      return;
+
+    LogicalResult parsingTables = parseSection<WasmSectionType::TABLE>();
+    if (failed(parsingTables))
+      return;
+
+    LogicalResult parsingMems = parseSection<WasmSectionType::MEMORY>();
+    if (failed(parsingMems))
+      return;
+
+    LogicalResult parsingGlobals = parseSection<WasmSectionType::GLOBAL>();
+    if (failed(parsingGlobals))
+      return;
+
+    LogicalResult parsingCode = parseSection<WasmSectionType::CODE>();
+    if (failed(parsingCode))
+      return;
+
+    LogicalResult parsingExports = parseSection<WasmSectionType::EXPORT>();
+    if (failed(parsingExports))
+      return;
+
+    // Copy over sizes of containers into statistics.
+    LDBG() << "WASM Imports:"
+           << "\n"
+           << " - Num functions: " << symbols.funcSymbols.size() << "\n"
+           << " - Num globals: " << symbols.globalSymbols.size() << "\n"
+           << " - Num memories: " << symbols.memSymbols.size() << "\n"
+           << " - Num tables: " << symbols.tableSymbols.size();
+  }
+
+  ModuleOp getModule() {
+    if (isValid)
+      return mOp;
+    if (mOp)
+      mOp.erase();
+    return ModuleOp{};
+  }
+
+private:
+  mlir::StringAttr srcName;
+  OpBuilder builder;
+  WasmModuleSymbolTables symbols;
+  MLIRContext *ctx;
+  ModuleOp mOp;
+  SectionRegistry registry;
+  size_t firstInternalFuncID{0};
+  bool isValid{true};
+};
+
+template <>
+LogicalResult
+WasmBinaryParser::parseSectionItem<WasmSectionType::IMPORT>(ParserHead &ph,
+                                                            size_t) {
+  FileLineColLoc importLoc = ph.getLocation();
+  auto moduleName = ph.parseName();
+  if (failed(moduleName))
+    return failure();
+
+  auto importName = ph.parseName();
+  if (failed(importName))
+    return failure();
+
+  FailureOr<ImportDesc> import = ph.parseImportDesc(ctx);
+  if (failed(import))
+    return failure();
+
+  return std::visit(
+      [this, importLoc, &moduleName, &importName](auto import) {
+        return visitImport(importLoc, *moduleName, *importName, import);
+      },
+      *import);
+}
+
+template <>
+LogicalResult
+WasmBinaryParser::parseSectionItem<WasmSectionType::EXPORT>(ParserHead &ph,
+                                                            size_t) {
+  FileLineColLoc exportLoc = ph.getLocation();
+
+  auto exportName = ph.parseName();
+  if (failed(exportName))
+    return failure();
+
+  FailureOr<std::byte> opcode = ph.consumeByte();
+  if (failed(opcode))
+    return failure();
+
+  FailureOr<uint32_t> idx = ph.parseLiteral<uint32_t>();
+  if (failed(idx))
+    return failure();
+
+  using SymbolRefDesc = std::variant<SmallVector<SymbolRefContainer>,
+                                     SmallVector<GlobalSymbolRefContainer>,
+                                     SmallVector<FunctionSymbolRefContainer>>;
+
+  SymbolRefDesc currentSymbolList;
+  std::string symbolType = "";
+  switch (*opcode) {
+  case WasmBinaryEncoding::Export::function:
+    symbolType = "function";
+    currentSymbolList = symbols.funcSymbols;
+    break;
+  case WasmBinaryEncoding::Export::table:
+    symbolType = "table";
+    currentSymbolList = symbols.tableSymbols;
+    break;
+  case WasmBinaryEncoding::Export::memory:
+    symbolType = "memory";
+    currentSymbolList = symbols.memSymbols;
+    break;
+  case WasmBinaryEncoding::Export::global:
+    symbolType = "global";
+    currentSymbolList = symbols.globalSymbols;
+    break;
+  default:
+    return emitError(exportLoc, "invalid value for export type: ")
+           << std::to_integer<unsigned>(*opcode);
+  }
+
+  auto currentSymbol = std::visit(
+      [&](const auto &list) -> FailureOr<FlatSymbolRefAttr> {
+        if (*idx > list.size()) {
+          emitError(
+              exportLoc,
+              llvm::formatv(
+                  "trying to export {0} {1} which is undefined in this scope",
+                  symbolType, *idx));
+          return failure();
+        }
+        return list[*idx].symbol;
+      },
+      currentSymbolList);
+
+  if (failed(currentSymbol))
+    return failure();
+
+  Operation *op = SymbolTable::lookupSymbolIn(mOp, *currentSymbol);
+  SymbolTable::setSymbolVisibility(op, SymbolTable::Visibility::Public);
+  StringAttr symName = SymbolTable::getSymbolName(op);
+  return SymbolTable{mOp}.rename(symName, *exportName);
+}
+
+template <>
+LogicalResult
+WasmBinaryParser::parseSectionItem<WasmSectionType::TABLE>(ParserHead &ph,
+                                                           size_t) {
+  FileLineColLoc opLocation = ph.getLocation();
+  FailureOr<TableType> tableType = ph.parseTableType(ctx);
+  if (failed(tableType))
+    return failure();
+  LDBG() << "  Parsed table description: " << *tableType;
+  StringAttr symbol = builder.getStringAttr(symbols.getNewTableSymbolName());
+  auto tableOp =
+      TableOp::create(builder, opLocation, symbol.strref(), *tableType);
+  symbols.tableSymbols.push_back({SymbolRefAttr::get(tableOp)});
+  return success();
+}
+
+template <>
+LogicalResult
+WasmBinaryParser::parseSectionItem<WasmSectionType::FUNCTION>(ParserHead &ph,
+                                                              size_t) {
+  FileLineColLoc opLoc = ph.getLocation();
+  auto typeIdxParsed = ph.parseLiteral<uint32_t>();
+  if (failed(typeIdxParsed))
+    return failure();
+  uint32_t typeIdx = *typeIdxParsed;
+  if (typeIdx >= symbols.moduleFuncTypes.size())
+    return emitError(getLocation(), "invalid type index: ") << typeIdx;
+  std::string symbol = symbols.getNewFuncSymbolName();
+  auto funcOp =
+      FuncOp::create(builder, opLoc, symbol, symbols.moduleFuncTypes[typeIdx]);
+  Block *block = funcOp.addEntryBlock();
+  OpBuilder::InsertionGuard guard{builder};
+  builder.setInsertionPointToEnd(block);
+  ReturnOp::create(builder, opLoc);
+  symbols.funcSymbols.push_back(
+      {{FlatSymbolRefAttr::get(funcOp.getSymNameAttr())},
+       symbols.moduleFuncTypes[typeIdx]});
+  return funcOp.verify();
+}
+
+template <>
+LogicalResult
+WasmBinaryParser::parseSectionItem<WasmSectionType::TYPE>(ParserHead &ph,
+                                                          size_t) {
+  FailureOr<FunctionType> funcType = ph.parseFunctionType(ctx);
+  if (failed(funcType))
+    return failure();
+  LDBG() << "Parsed function type " << *funcType;
+  symbols.moduleFuncTypes.push_back(*funcType);
+  return success();
+}
+
+template <>
+LogicalResult
+WasmBinaryParser::parseSectionItem<WasmSectionType::MEMORY>(ParserHead &ph,
+                                                            size_t) {
+  FileLineColLoc opLocation = ph.getLocation();
+  FailureOr<LimitType> memory = ph.parseLimit(ctx);
+  if (failed(memory))
+    return failure();
+
+  LDBG() << "  Registering memory " << *memory;
+  std::string symbol = symbols.getNewMemorySymbolName();
+  auto memOp = MemOp::create(builder, opLocation, symbol, *memory);
+  symbols.memSymbols.push_back({SymbolRefAttr::get(memOp)});
+  return success();
+}
+
+template <>
+LogicalResult
+WasmBinaryParser::parseSectionItem<WasmSectionType::GLOBAL>(ParserHead &ph,
+                                                            size_t) {
+  FileLineColLoc globalLocation = ph.getLocation();
+  auto globalTypeParsed = ph.parseGlobalType(ctx);
+  if (failed(globalTypeParsed))
+    return failure();
+
+  GlobalTypeRecord globalType = *globalTypeParsed;
+  auto symbol = builder.getStringAttr(symbols.getNewGlobalSymbolName());
+  auto globalOp = builder.create<wasmssa::GlobalOp>(
+      globalLocation, symbol, globalType.type, globalType.isMutable);
+  symbols.globalSymbols.push_back(
+      {{FlatSymbolRefAttr::get(globalOp)}, globalOp.getType()});
+  OpBuilder::InsertionGuard guard{builder};
+  Block *block = builder.createBlock(&globalOp.getInitializer());
+  builder.setInsertionPointToStart(block);
+  parsed_inst_t expr = ph.parseExpression(builder, symbols);
+  if (failed(expr))
+    return failure();
+  if (block->empty())
+    return emitError(globalLocation, "global with empty initializer");
+  if (expr->size() != 1 && (*expr)[0].getType() != globalType.type)
+    return emitError(
+        globalLocation,
+        "initializer result type does not match global declaration type");
+  builder.create<ReturnOp>(globalLocation, *expr);
+  return success();
+}
+
+template <>
+LogicalResult WasmBinaryParser::parseSectionItem<WasmSectionType::CODE>(
+    ParserHead &ph, size_t innerFunctionId) {
+  unsigned long funcId = innerFunctionId + firstInternalFuncID;
+  FunctionSymbolRefContainer symRef = symbols.funcSymbols[funcId];
+  auto funcOp =
+      dyn_cast<FuncOp>(SymbolTable::lookupSymbolIn(mOp, symRef.symbol));
+  assert(funcOp);
+  if (failed(ph.parseCodeFor(funcOp, symbols)))
+    return failure();
+  return success();
+}
+} // namespace
+
+namespace mlir::wasm {
+OwningOpRef<ModuleOp> importWebAssemblyToModule(llvm::SourceMgr &source,
+                                                MLIRContext *context) {
+  WasmBinaryParser wBN{source, context};
+  ModuleOp mOp = wBN.getModule();
+  if (mOp)
+    return {mOp};
+
+  return {nullptr};
+}
+} // namespace mlir::wasm
diff --git a/mlir/lib/Target/Wasm/TranslateRegistration.cpp b/mlir/lib/Target/Wasm/TranslateRegistration.cpp
new file mode 100644
index 0000000000000..03b97846d45d3
--- /dev/null
+++ b/mlir/lib/Target/Wasm/TranslateRegistration.cpp
@@ -0,0 +1,28 @@
+//===- TranslateRegistration.cpp - Register translation -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "mlir/Dialect/WasmSSA/IR/WasmSSA.h"
+#include "mlir/IR/DialectRegistry.h"
+#include "mlir/IR/OwningOpRef.h"
+#include "mlir/Target/Wasm/WasmImporter.h"
+#include "mlir/Tools/mlir-translate/Translation.h"
+
+using namespace mlir;
+
+namespace mlir {
+void registerFromWasmTranslation() {
+  TranslateToMLIRRegistration registration{
+      "import-wasm", "Translate WASM to MLIR",
+      [](llvm::SourceMgr &sourceMgr,
+         MLIRContext *context) -> OwningOpRef<Operation *> {
+        return wasm::importWebAssemblyToModule(sourceMgr, context);
+      },
+      [](DialectRegistry &registry) {
+        registry.insert<wasmssa::WasmSSADialect>();
+      }};
+}
+} // namespace mlir
diff --git a/mlir/lib/Transforms/CSE.cpp b/mlir/lib/Transforms/CSE.cpp
index 09e5a02e3802a..8eaac308755fd 100644
--- a/mlir/lib/Transforms/CSE.cpp
+++ b/mlir/lib/Transforms/CSE.cpp
@@ -177,11 +177,10 @@ void CSEDriver::replaceUsesAndDelete(ScopedMapTy &knownValues, Operation *op,
 bool CSEDriver::hasOtherSideEffectingOpInBetween(Operation *fromOp,
                                                  Operation *toOp) {
   assert(fromOp->getBlock() == toOp->getBlock());
-  assert(
-      isa<MemoryEffectOpInterface>(fromOp) &&
-      cast<MemoryEffectOpInterface>(fromOp).hasEffect<MemoryEffects::Read>() &&
-      isa<MemoryEffectOpInterface>(toOp) &&
-      cast<MemoryEffectOpInterface>(toOp).hasEffect<MemoryEffects::Read>());
+  assert(hasEffect<MemoryEffects::Read>(fromOp) &&
+         "expected read effect on fromOp");
+  assert(hasEffect<MemoryEffects::Read>(toOp) &&
+         "expected read effect on toOp");
   Operation *nextOp = fromOp->getNextNode();
   auto result =
       memEffectsCache.try_emplace(fromOp, std::make_pair(fromOp, nullptr));
@@ -245,11 +244,10 @@ LogicalResult CSEDriver::simplifyOperation(ScopedMapTy &knownValues,
   // Some simple use case of operation with memory side-effect are dealt with
   // here. Operations with no side-effect are done after.
   if (!isMemoryEffectFree(op)) {
-    auto memEffects = dyn_cast<MemoryEffectOpInterface>(op);
     // TODO: Only basic use case for operations with MemoryEffects::Read can be
     // eleminated now. More work needs to be done for more complicated patterns
     // and other side-effects.
-    if (!memEffects || !memEffects.onlyHasEffect<MemoryEffects::Read>())
+    if (!hasSingleEffect<MemoryEffects::Read>(op))
       return failure();
 
     // Look for an existing definition for the operation.
diff --git a/mlir/lib/Transforms/InlinerPass.cpp b/mlir/lib/Transforms/InlinerPass.cpp
index 703e517d45374..b07363c3dafe4 100644
--- a/mlir/lib/Transforms/InlinerPass.cpp
+++ b/mlir/lib/Transforms/InlinerPass.cpp
@@ -18,6 +18,7 @@
 #include "mlir/Analysis/CallGraph.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Transforms/Inliner.h"
+#include "llvm/Support/DebugLog.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_INLINER
@@ -120,8 +121,8 @@ static bool isProfitableToInline(const Inliner::ResolvedCall &resolvedCall,
     return true;
 
   unsigned ratio = countOps(calleeRegion) * 100 / callerOps;
-  LLVM_DEBUG(llvm::dbgs() << "Callee / caller operation ratio (max: "
-                          << inliningThreshold << "%): " << ratio << "%\n");
+  LDBG() << "Callee / caller operation ratio (max: " << inliningThreshold
+         << "%): " << ratio << "%";
   return ratio <= inliningThreshold;
 }
 
diff --git a/mlir/lib/Transforms/Mem2Reg.cpp b/mlir/lib/Transforms/Mem2Reg.cpp
index cf039c328e07f..d36a3c1362c19 100644
--- a/mlir/lib/Transforms/Mem2Reg.cpp
+++ b/mlir/lib/Transforms/Mem2Reg.cpp
@@ -19,6 +19,7 @@
 #include "mlir/Interfaces/MemorySlotInterfaces.h"
 #include "mlir/Transforms/Passes.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/GenericIteratedDominanceFrontier.h"
 
 namespace mlir {
@@ -632,8 +633,7 @@ MemorySlotPromoter::promoteSlot() {
     }
   }
 
-  LLVM_DEBUG(llvm::dbgs() << "[mem2reg] Promoted memory slot: " << slot.ptr
-                          << "\n");
+  LDBG() << "Promoted memory slot: " << slot.ptr;
 
   if (statistics.promotedAmount)
     (*statistics.promotedAmount)++;
diff --git a/mlir/lib/Transforms/RemoveDeadValues.cpp b/mlir/lib/Transforms/RemoveDeadValues.cpp
index 4ccb83f3ad298..02dad69e49614 100644
--- a/mlir/lib/Transforms/RemoveDeadValues.cpp
+++ b/mlir/lib/Transforms/RemoveDeadValues.cpp
@@ -258,18 +258,17 @@ static SmallVector<OpOperand *> operandsToOpOperands(OperandRange operands) {
 static void processSimpleOp(Operation *op, RunLivenessAnalysis &la,
                             DenseSet<Value> &nonLiveSet,
                             RDVFinalCleanupList &cl) {
-  LDBG() << "Processing simple op: " << *op;
   if (!isMemoryEffectFree(op) || hasLive(op->getResults(), nonLiveSet, la)) {
-    LDBG()
-        << "Simple op is not memory effect free or has live results, skipping: "
-        << *op;
+    LDBG() << "Simple op is not memory effect free or has live results, "
+              "preserving it: "
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
     return;
   }
 
   LDBG()
       << "Simple op has all dead results and is memory effect free, scheduling "
          "for removal: "
-      << *op;
+      << OpWithFlags(op, OpPrintingFlags().skipRegions());
   cl.operations.push_back(op);
   collectNonLiveValues(nonLiveSet, op->getResults(),
                        BitVector(op->getNumResults(), true));
@@ -728,19 +727,31 @@ static void processBranchOp(BranchOpInterface branchOp, RunLivenessAnalysis &la,
 /// Removes dead values collected in RDVFinalCleanupList.
 /// To be run once when all dead values have been collected.
 static void cleanUpDeadVals(RDVFinalCleanupList &list) {
+  LDBG() << "Starting cleanup of dead values...";
+
   // 1. Operations
+  LDBG() << "Cleaning up " << list.operations.size() << " operations";
   for (auto &op : list.operations) {
+    LDBG() << "Erasing operation: "
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
     op->dropAllUses();
     op->erase();
   }
 
   // 2. Values
+  LDBG() << "Cleaning up " << list.values.size() << " values";
   for (auto &v : list.values) {
+    LDBG() << "Dropping all uses of value: " << v;
     v.dropAllUses();
   }
 
   // 3. Functions
+  LDBG() << "Cleaning up " << list.functions.size() << " functions";
   for (auto &f : list.functions) {
+    LDBG() << "Cleaning up function: " << f.funcOp.getOperation()->getName();
+    LDBG() << "  Erasing " << f.nonLiveArgs.count() << " non-live arguments";
+    LDBG() << "  Erasing " << f.nonLiveRets.count()
+           << " non-live return values";
     // Some functions may not allow erasing arguments or results. These calls
     // return failure in such cases without modifying the function, so it's okay
     // to proceed.
@@ -749,44 +760,67 @@ static void cleanUpDeadVals(RDVFinalCleanupList &list) {
   }
 
   // 4. Operands
+  LDBG() << "Cleaning up " << list.operands.size() << " operand lists";
   for (OperationToCleanup &o : list.operands) {
-    if (o.op->getNumOperands() > 0)
+    if (o.op->getNumOperands() > 0) {
+      LDBG() << "Erasing " << o.nonLive.count()
+             << " non-live operands from operation: "
+             << OpWithFlags(o.op, OpPrintingFlags().skipRegions());
       o.op->eraseOperands(o.nonLive);
+    }
   }
 
   // 5. Results
+  LDBG() << "Cleaning up " << list.results.size() << " result lists";
   for (auto &r : list.results) {
+    LDBG() << "Erasing " << r.nonLive.count()
+           << " non-live results from operation: "
+           << OpWithFlags(r.op, OpPrintingFlags().skipRegions());
     dropUsesAndEraseResults(r.op, r.nonLive);
   }
 
   // 6. Blocks
+  LDBG() << "Cleaning up " << list.blocks.size() << " block argument lists";
   for (auto &b : list.blocks) {
     // blocks that are accessed via multiple codepaths processed once
     if (b.b->getNumArguments() != b.nonLiveArgs.size())
       continue;
+    LDBG() << "Erasing " << b.nonLiveArgs.count()
+           << " non-live arguments from block: " << b.b;
     // it iterates backwards because erase invalidates all successor indexes
     for (int i = b.nonLiveArgs.size() - 1; i >= 0; --i) {
       if (!b.nonLiveArgs[i])
         continue;
+      LDBG() << "  Erasing block argument " << i << ": " << b.b->getArgument(i);
       b.b->getArgument(i).dropAllUses();
       b.b->eraseArgument(i);
     }
   }
 
   // 7. Successor Operands
+  LDBG() << "Cleaning up " << list.successorOperands.size()
+         << " successor operand lists";
   for (auto &op : list.successorOperands) {
     SuccessorOperands successorOperands =
         op.branch.getSuccessorOperands(op.successorIndex);
     // blocks that are accessed via multiple codepaths processed once
     if (successorOperands.size() != op.nonLiveOperands.size())
       continue;
+    LDBG() << "Erasing " << op.nonLiveOperands.count()
+           << " non-live successor operands from successor "
+           << op.successorIndex << " of branch: "
+           << OpWithFlags(op.branch, OpPrintingFlags().skipRegions());
     // it iterates backwards because erase invalidates all successor indexes
     for (int i = successorOperands.size() - 1; i >= 0; --i) {
       if (!op.nonLiveOperands[i])
         continue;
+      LDBG() << "  Erasing successor operand " << i << ": "
+             << successorOperands[i];
       successorOperands.erase(i);
     }
   }
+
+  LDBG() << "Finished cleanup of dead values";
 }
 
 struct RemoveDeadValues : public impl::RemoveDeadValuesBase<RemoveDeadValues> {
diff --git a/mlir/lib/Transforms/SROA.cpp b/mlir/lib/Transforms/SROA.cpp
index 67f536a5cfdfa..859c0302185e6 100644
--- a/mlir/lib/Transforms/SROA.cpp
+++ b/mlir/lib/Transforms/SROA.cpp
@@ -12,6 +12,7 @@
 #include "mlir/Analysis/TopologicalSortUtils.h"
 #include "mlir/Interfaces/MemorySlotInterfaces.h"
 #include "mlir/Transforms/Passes.h"
+#include "llvm/Support/DebugLog.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_SROA
@@ -180,8 +181,7 @@ static void destructureSlot(
   assert(slot.ptr.use_empty() && "after destructuring, the original slot "
                                  "pointer should no longer be used");
 
-  LLVM_DEBUG(llvm::dbgs() << "[sroa] Destructured memory slot: " << slot.ptr
-                          << "\n");
+  LDBG() << "Destructured memory slot: " << slot.ptr;
 
   if (statistics.destructuredAmount)
     (*statistics.destructuredAmount)++;
diff --git a/mlir/lib/Transforms/SymbolDCE.cpp b/mlir/lib/Transforms/SymbolDCE.cpp
index 0a925c47a9e8b..87885be00daa3 100644
--- a/mlir/lib/Transforms/SymbolDCE.cpp
+++ b/mlir/lib/Transforms/SymbolDCE.cpp
@@ -13,8 +13,11 @@
 
 #include "mlir/Transforms/Passes.h"
 
+#include "mlir/IR/Operation.h"
 #include "mlir/IR/SymbolTable.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
+#include "llvm/Support/InterleavedRange.h"
 
 namespace mlir {
 #define GEN_PASS_DEF_SYMBOLDCE
@@ -87,8 +90,8 @@ LogicalResult SymbolDCE::computeLiveness(Operation *symbolTableOp,
                                          SymbolTableCollection &symbolTable,
                                          bool symbolTableIsHidden,
                                          DenseSet<Operation *> &liveSymbols) {
-  LLVM_DEBUG(llvm::dbgs() << "computeLiveness: " << symbolTableOp->getName()
-                          << "\n");
+  LDBG() << "computeLiveness: "
+         << OpWithFlags(symbolTableOp, OpPrintingFlags().skipRegions());
   // A worklist of live operations to propagate uses from.
   SmallVector<Operation *, 16> worklist;
 
@@ -116,7 +119,8 @@ LogicalResult SymbolDCE::computeLiveness(Operation *symbolTableOp,
   // consideration.
   while (!worklist.empty()) {
     Operation *op = worklist.pop_back_val();
-    LLVM_DEBUG(llvm::dbgs() << "processing: " << op->getName() << "\n");
+    LDBG() << "processing: "
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
 
     // If this is a symbol table, recursively compute its liveness.
     if (op->hasTrait<OpTrait::SymbolTable>()) {
@@ -124,13 +128,14 @@ LogicalResult SymbolDCE::computeLiveness(Operation *symbolTableOp,
       // symbol, or if it is a private symbol.
       SymbolOpInterface symbol = dyn_cast<SymbolOpInterface>(op);
       bool symIsHidden = symbolTableIsHidden || !symbol || symbol.isPrivate();
-      LLVM_DEBUG(llvm::dbgs() << "\tsymbol table: " << op->getName()
-                              << " is hidden: " << symIsHidden << "\n");
+      LDBG() << "\tsymbol table: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions())
+             << " is hidden: " << symIsHidden;
       if (failed(computeLiveness(op, symbolTable, symIsHidden, liveSymbols)))
         return failure();
     } else {
-      LLVM_DEBUG(llvm::dbgs()
-                 << "\tnon-symbol table: " << op->getName() << "\n");
+      LDBG() << "\tnon-symbol table: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions());
       // If the op is not a symbol table, then, unless op itself is dead which
       // would be handled by DCE, we need to check all the regions and blocks
       // within the op to find the uses (e.g., consider visibility within op as
@@ -160,20 +165,17 @@ LogicalResult SymbolDCE::computeLiveness(Operation *symbolTableOp,
     }
 
     SmallVector<Operation *, 4> resolvedSymbols;
-    LLVM_DEBUG(llvm::dbgs() << "uses of " << op->getName() << "\n");
+    LDBG() << "uses of " << OpWithFlags(op, OpPrintingFlags().skipRegions());
     for (const SymbolTable::SymbolUse &use : *uses) {
-      LLVM_DEBUG(llvm::dbgs() << "\tuse: " << use.getUser() << "\n");
+      LDBG() << "\tuse: " << use.getUser();
       // Lookup the symbols referenced by this use.
       resolvedSymbols.clear();
       if (failed(symbolTable.lookupSymbolIn(parentOp, use.getSymbolRef(),
                                             resolvedSymbols)))
         // Ignore references to unknown symbols.
         continue;
-      LLVM_DEBUG({
-        llvm::dbgs() << "\t\tresolved symbols: ";
-        llvm::interleaveComma(resolvedSymbols, llvm::dbgs());
-        llvm::dbgs() << "\n";
-      });
+      LDBG() << "\t\tresolved symbols: "
+             << llvm::interleaved(resolvedSymbols, ", ");
 
       // Mark each of the resolved symbols as live.
       for (Operation *resolvedSymbol : resolvedSymbols)
diff --git a/mlir/lib/Transforms/Utils/ControlFlowSinkUtils.cpp b/mlir/lib/Transforms/Utils/ControlFlowSinkUtils.cpp
index cfd568f028721..19cf4646bddd4 100644
--- a/mlir/lib/Transforms/Utils/ControlFlowSinkUtils.cpp
+++ b/mlir/lib/Transforms/Utils/ControlFlowSinkUtils.cpp
@@ -21,7 +21,10 @@
 #include "mlir/Transforms/ControlFlowSinkUtils.h"
 #include "mlir/IR/Dominance.h"
 #include "mlir/IR/Matchers.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/OperationSupport.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
+#include "llvm/Support/DebugLog.h"
 #include <vector>
 
 #define DEBUG_TYPE "cf-sink"
@@ -84,13 +87,15 @@ bool Sinker::allUsersDominatedBy(Operation *op, Region *region) {
 
 void Sinker::tryToSinkPredecessors(Operation *user, Region *region,
                                    std::vector<Operation *> &stack) {
-  LLVM_DEBUG(user->print(llvm::dbgs() << "\nContained op:\n"));
+  LDBG() << "Contained op: "
+         << OpWithFlags(user, OpPrintingFlags().skipRegions());
   for (Value value : user->getOperands()) {
     Operation *op = value.getDefiningOp();
     // Ignore block arguments and ops that are already inside the region.
     if (!op || op->getParentRegion() == region)
       continue;
-    LLVM_DEBUG(op->print(llvm::dbgs() << "\nTry to sink:\n"));
+    LDBG() << "Try to sink:\n"
+           << OpWithFlags(op, OpPrintingFlags().skipRegions());
 
     // If the op's users are all in the region and it can be moved, then do so.
     if (allUsersDominatedBy(op, region) && shouldMoveIntoRegion(op, region)) {
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index f23c6197accd5..e3248204d6694 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -121,17 +121,8 @@ struct ConversionValueMapping {
   /// false positives.
   bool isMappedTo(Value value) const { return mappedTo.contains(value); }
 
-  /// Lookup the most recently mapped values with the desired types in the
-  /// mapping.
-  ///
-  /// Special cases:
-  /// - If the desired type range is empty, simply return the most recently
-  ///   mapped values.
-  /// - If there is no mapping to the desired types, also return the most
-  ///   recently mapped values.
-  /// - If there is no mapping for the given values at all, return the given
-  ///   value.
-  ValueVector lookupOrDefault(Value from, TypeRange desiredTypes = {}) const;
+  /// Lookup a value in the mapping.
+  ValueVector lookup(const ValueVector &from) const;
 
   template <typename T>
   struct IsValueVector : std::is_same<std::decay_t<T>, ValueVector> {};
@@ -185,54 +176,40 @@ struct ConversionValueMapping {
 };
 } // namespace
 
-ValueVector
-ConversionValueMapping::lookupOrDefault(Value from,
-                                        TypeRange desiredTypes) const {
-  // Try to find the deepest values that have the desired types. If there is no
-  // such mapping, simply return the deepest values.
-  ValueVector desiredValue;
-  ValueVector current{from};
-  do {
-    // Store the current value if the types match.
-    if (TypeRange(ValueRange(current)) == desiredTypes)
-      desiredValue = current;
-
-    // If possible, Replace each value with (one or multiple) mapped values.
-    ValueVector next;
-    for (Value v : current) {
-      auto it = mapping.find({v});
-      if (it != mapping.end()) {
-        llvm::append_range(next, it->second);
-      } else {
-        next.push_back(v);
-      }
-    }
-    if (next != current) {
-      // If at least one value was replaced, continue the lookup from there.
-      current = std::move(next);
-      continue;
-    }
-
-    // Otherwise: Check if there is a mapping for the entire vector. Such
-    // mappings are materializations. (N:M mapping are not supported for value
-    // replacements.)
-    //
-    // Note: From a correctness point of view, materializations do not have to
-    // be stored (and looked up) in the mapping. But for performance reasons,
-    // we choose to reuse existing IR (when possible) instead of creating it
-    // multiple times.
-    auto it = mapping.find(current);
-    if (it == mapping.end()) {
-      // No mapping found: The lookup stops here.
-      break;
-    }
-    current = it->second;
-  } while (true);
-
-  // If the desired values were found use them, otherwise default to the leaf
-  // values.
-  // Note: If `desiredTypes` is empty, this function always returns `current`.
-  return !desiredValue.empty() ? std::move(desiredValue) : std::move(current);
+/// Marker attribute for pure type conversions. I.e., mappings whose only
+/// purpose is to resolve a type mismatch. (In contrast, mappings that point to
+/// the replacement values of a "replaceOp" call, etc., are not pure type
+/// conversions.)
+static const StringRef kPureTypeConversionMarker = "__pure_type_conversion__";
+
+/// Return the operation that defines all values in the vector. Return nullptr
+/// if the values are not defined by the same operation.
+static Operation *getCommonDefiningOp(const ValueVector &values) {
+  assert(!values.empty() && "expected non-empty value vector");
+  Operation *op = values.front().getDefiningOp();
+  for (Value v : llvm::drop_begin(values)) {
+    if (v.getDefiningOp() != op)
+      return nullptr;
+  }
+  return op;
+}
+
+/// A vector of values is a pure type conversion if all values are defined by
+/// the same operation and the operation has the `kPureTypeConversionMarker`
+/// attribute.
+static bool isPureTypeConversion(const ValueVector &values) {
+  assert(!values.empty() && "expected non-empty value vector");
+  Operation *op = getCommonDefiningOp(values);
+  return op && op->hasAttr(kPureTypeConversionMarker);
+}
+
+ValueVector ConversionValueMapping::lookup(const ValueVector &from) const {
+  auto it = mapping.find(from);
+  if (it == mapping.end()) {
+    // No mapping found: The lookup stops here.
+    return {};
+  }
+  return it->second;
 }
 
 //===----------------------------------------------------------------------===//
@@ -873,7 +850,7 @@ namespace detail {
 struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   explicit ConversionPatternRewriterImpl(MLIRContext *ctx,
                                          const ConversionConfig &config)
-      : context(ctx), config(config) {}
+      : context(ctx), config(config), notifyingRewriter(ctx, config.listener) {}
 
   //===--------------------------------------------------------------------===//
   // State Management
@@ -895,6 +872,7 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   /// failure.
   template <typename RewriteTy, typename... Args>
   void appendRewrite(Args &&...args) {
+    assert(config.allowPatternRollback && "appending rewrites is not allowed");
     rewrites.push_back(
         std::make_unique<RewriteTy>(*this, std::forward<Args>(args)...));
   }
@@ -921,16 +899,13 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   bool wasOpReplaced(Operation *op) const;
 
   /// Lookup the most recently mapped values with the desired types in the
-  /// mapping.
+  /// mapping, taking into account only replacements. Perform a best-effort
+  /// search for existing materializations with the desired types.
   ///
-  /// Special cases:
-  /// - If the desired type range is empty, simply return the most recently
-  ///   mapped values.
-  /// - If there is no mapping to the desired types, also return the most
-  ///   recently mapped values.
-  /// - If there is no mapping for the given values at all, return the given
-  ///   value.
-  ValueVector lookupOrDefault(Value from, TypeRange desiredTypes = {}) const;
+  /// If `skipPureTypeConversions` is "true", materializations that are pure
+  /// type conversions are not considered.
+  ValueVector lookupOrDefault(Value from, TypeRange desiredTypes = {},
+                              bool skipPureTypeConversions = false) const;
 
   /// Lookup the given value within the map, or return an empty vector if the
   /// value is not mapped. If it is mapped, this follows the same behavior
@@ -993,11 +968,19 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   /// If `valuesToMap` is set to a non-null Value, then that value is mapped to
   /// the results of the unresolved materialization in the conversion value
   /// mapping.
+  ///
+  /// If `isPureTypeConversion` is "true", the materialization is created only
+  /// to resolve a type mismatch. That means it is not a regular value
+  /// replacement issued by the user. (Replacement values that are created
+  /// "out of thin air" appear like unresolved materializations because they are
+  /// unrealized_conversion_cast ops. However, they must be treated like
+  /// regular value replacements.)
   ValueRange buildUnresolvedMaterialization(
       MaterializationKind kind, OpBuilder::InsertPoint ip, Location loc,
       ValueVector valuesToMap, ValueRange inputs, TypeRange outputTypes,
       Type originalType, const TypeConverter *converter,
-      UnrealizedConversionCastOp *castOp = nullptr);
+      UnrealizedConversionCastOp *castOp = nullptr,
+      bool isPureTypeConversion = true);
 
   /// Find a replacement value for the given SSA value in the conversion value
   /// mapping. The replacement value must have the same type as the given SSA
@@ -1086,6 +1069,9 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   ConversionValueMapping mapping;
 
   /// Ordered list of block operations (creations, splits, motions).
+  /// This vector is maintained only if `allowPatternRollback` is set to
+  /// "true". Otherwise, all IR rewrites are materialized immediately and no
+  /// bookkeeping is needed.
   SmallVector<std::unique_ptr<IRRewrite>> rewrites;
 
   /// A set of operations that should no longer be considered for legalization.
@@ -1109,6 +1095,10 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   /// by the current pattern.
   SetVector<Block *> patternInsertedBlocks;
 
+  /// A list of unresolved materializations that were created by the current
+  /// pattern.
+  DenseSet<UnrealizedConversionCastOp> patternMaterializations;
+
   /// A mapping for looking up metadata of unresolved materializations.
   DenseMap<UnrealizedConversionCastOp, UnresolvedMaterializationInfo>
       unresolvedMaterializations;
@@ -1124,15 +1114,37 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   /// Dialect conversion configuration.
   const ConversionConfig &config;
 
+  /// A set of erased operations. This set is utilized only if
+  /// `allowPatternRollback` is set to "false". Conceptually, this set is
+  /// similar to `replacedOps` (which is maintained when the flag is set to
+  /// "true"). However, erasing from a DenseSet is more efficient than erasing
+  /// from a SetVector.
+  DenseSet<Operation *> erasedOps;
+
+  /// A set of erased blocks. This set is utilized only if
+  /// `allowPatternRollback` is set to "false".
+  DenseSet<Block *> erasedBlocks;
+
+  /// A rewriter that notifies the listener (if any) about all IR
+  /// modifications. This rewriter is utilized only if `allowPatternRollback`
+  /// is set to "false". If the flag is set to "true", the listener is notified
+  /// with a separate mechanism (e.g., in `IRRewrite::commit`).
+  IRRewriter notifyingRewriter;
+
 #ifndef NDEBUG
+  /// A set of replaced block arguments. This set is for debugging purposes
+  /// only and it is maintained only if `allowPatternRollback` is set to
+  /// "true".
+  DenseSet<BlockArgument> replacedArgs;
+
   /// A set of operations that have pending updates. This tracking isn't
   /// strictly necessary, and is thus only active during debug builds for extra
   /// verification.
   SmallPtrSet<Operation *, 1> pendingRootUpdates;
 
   /// A raw output stream used to prefix the debug log.
-  llvm::impl::raw_ldbg_ostream os{(Twine("[") + DEBUG_TYPE + "] ").str(),
-                                  llvm::dbgs(), /*HasPendingNewline=*/false};
+  llvm::impl::raw_ldbg_ostream os{(Twine("[") + DEBUG_TYPE + ":1] ").str(),
+                                  llvm::dbgs()};
 
   /// A logger used to emit diagnostics during the conversion process.
   llvm::ScopedPrinter logger{os};
@@ -1160,11 +1172,8 @@ void BlockTypeConversionRewrite::rollback() {
   getNewBlock()->replaceAllUsesWith(getOrigBlock());
 }
 
-void ReplaceBlockArgRewrite::commit(RewriterBase &rewriter) {
-  Value repl = rewriterImpl.findOrBuildReplacementValue(arg, converter);
-  if (!repl)
-    return;
-
+static void performReplaceBlockArg(RewriterBase &rewriter, BlockArgument arg,
+                                   Value repl) {
   if (isa<BlockArgument>(repl)) {
     rewriter.replaceAllUsesWith(arg, repl);
     return;
@@ -1181,6 +1190,13 @@ void ReplaceBlockArgRewrite::commit(RewriterBase &rewriter) {
   });
 }
 
+void ReplaceBlockArgRewrite::commit(RewriterBase &rewriter) {
+  Value repl = rewriterImpl.findOrBuildReplacementValue(arg, converter);
+  if (!repl)
+    return;
+  performReplaceBlockArg(rewriter, arg, repl);
+}
+
 void ReplaceBlockArgRewrite::rollback() { rewriterImpl.mapping.erase({arg}); }
 
 void ReplaceOperationRewrite::commit(RewriterBase &rewriter) {
@@ -1264,10 +1280,101 @@ void ConversionPatternRewriterImpl::applyRewrites() {
 // State Management
 //===----------------------------------------------------------------------===//
 
-ValueVector
-ConversionPatternRewriterImpl::lookupOrDefault(Value from,
-                                               TypeRange desiredTypes) const {
-  return mapping.lookupOrDefault(from, desiredTypes);
+ValueVector ConversionPatternRewriterImpl::lookupOrDefault(
+    Value from, TypeRange desiredTypes, bool skipPureTypeConversions) const {
+  // Helper function that looks up a single value.
+  auto lookup = [&](const ValueVector &values) -> ValueVector {
+    assert(!values.empty() && "expected non-empty value vector");
+
+    // If the pattern rollback is enabled, use the mapping to look up the
+    // values.
+    if (config.allowPatternRollback)
+      return mapping.lookup(values);
+
+    // Otherwise, look up values by examining the IR. All replacements have
+    // already been materialized in IR.
+    Operation *op = getCommonDefiningOp(values);
+    if (!op)
+      return {};
+    auto castOp = dyn_cast<UnrealizedConversionCastOp>(op);
+    if (!castOp)
+      return {};
+    if (!this->unresolvedMaterializations.contains(castOp))
+      return {};
+    if (castOp.getOutputs() != values)
+      return {};
+    return castOp.getInputs();
+  };
+
+  // Helper function that looks up each value in `values` individually and then
+  // composes the results. If that fails, it tries to look up the entire vector
+  // at once.
+  auto composedLookup = [&](const ValueVector &values) -> ValueVector {
+    // If possible, replace each value with (one or multiple) mapped values.
+    ValueVector next;
+    for (Value v : values) {
+      ValueVector r = lookup({v});
+      if (!r.empty()) {
+        llvm::append_range(next, r);
+      } else {
+        next.push_back(v);
+      }
+    }
+    if (next != values) {
+      // At least one value was replaced.
+      return next;
+    }
+
+    // Otherwise: Check if there is a mapping for the entire vector. Such
+    // mappings are materializations. (N:M mapping are not supported for value
+    // replacements.)
+    //
+    // Note: From a correctness point of view, materializations do not have to
+    // be stored (and looked up) in the mapping. But for performance reasons,
+    // we choose to reuse existing IR (when possible) instead of creating it
+    // multiple times.
+    ValueVector r = lookup(values);
+    if (r.empty()) {
+      // No mapping found: The lookup stops here.
+      return {};
+    }
+    return r;
+  };
+
+  // Try to find the deepest values that have the desired types. If there is no
+  // such mapping, simply return the deepest values.
+  ValueVector desiredValue;
+  ValueVector current{from};
+  ValueVector lastNonMaterialization{from};
+  do {
+    // Store the current value if the types match.
+    bool match = TypeRange(ValueRange(current)) == desiredTypes;
+    if (skipPureTypeConversions) {
+      // Skip pure type conversions, if requested.
+      bool pureConversion = isPureTypeConversion(current);
+      match &= !pureConversion;
+      // Keep track of the last mapped value that was not a pure type
+      // conversion.
+      if (!pureConversion)
+        lastNonMaterialization = current;
+    }
+    if (match)
+      desiredValue = current;
+
+    // Lookup next value in the mapping.
+    ValueVector next = composedLookup(current);
+    if (next.empty())
+      break;
+    current = std::move(next);
+  } while (true);
+
+  // If the desired values were found use them, otherwise default to the leaf
+  // values. (Skip pure type conversions, if requested.)
+  if (!desiredTypes.empty())
+    return desiredValue;
+  if (skipPureTypeConversions)
+    return lastNonMaterialization;
+  return current;
 }
 
 ValueVector
@@ -1300,15 +1407,8 @@ void ConversionPatternRewriterImpl::resetState(RewriterState state,
 void ConversionPatternRewriterImpl::undoRewrites(unsigned numRewritesToKeep,
                                                  StringRef patternName) {
   for (auto &rewrite :
-       llvm::reverse(llvm::drop_begin(rewrites, numRewritesToKeep))) {
-    if (!config.allowPatternRollback &&
-        !isa<UnresolvedMaterializationRewrite>(rewrite)) {
-      // Unresolved materializations can always be rolled back (erased).
-      llvm::report_fatal_error("pattern '" + patternName +
-                               "' rollback of IR modifications requested");
-    }
+       llvm::reverse(llvm::drop_begin(rewrites, numRewritesToKeep)))
     rewrite->rollback();
-  }
   rewrites.resize(numRewritesToKeep);
 }
 
@@ -1324,10 +1424,13 @@ LogicalResult ConversionPatternRewriterImpl::remapValues(
     Location operandLoc = inputLoc ? *inputLoc : operand.getLoc();
 
     if (!currentTypeConverter) {
-      // The current pattern does not have a type converter. I.e., it does not
-      // distinguish between legal and illegal types. For each operand, simply
-      // pass through the most recently mapped values.
-      remapped.push_back(lookupOrDefault(operand));
+      // The current pattern does not have a type converter. Pass the most
+      // recently mapped values, excluding materializations. Materializations
+      // are intentionally excluded because their presence may depend on other
+      // patterns. Including materializations would make the lookup fragile
+      // and unpredictable.
+      remapped.push_back(lookupOrDefault(operand, /*desiredTypes=*/{},
+                                         /*skipPureTypeConversions=*/true));
       continue;
     }
 
@@ -1356,7 +1459,8 @@ LogicalResult ConversionPatternRewriterImpl::remapValues(
     }
 
     // Create a materialization for the most recently mapped values.
-    repl = lookupOrDefault(operand);
+    repl = lookupOrDefault(operand, /*desiredTypes=*/{},
+                           /*skipPureTypeConversions=*/true);
     ValueRange castValues = buildUnresolvedMaterialization(
         MaterializationKind::Target, computeInsertPoint(repl), operandLoc,
         /*valuesToMap=*/repl, /*inputs=*/repl, /*outputTypes=*/legalTypes,
@@ -1368,12 +1472,12 @@ LogicalResult ConversionPatternRewriterImpl::remapValues(
 
 bool ConversionPatternRewriterImpl::isOpIgnored(Operation *op) const {
   // Check to see if this operation is ignored or was replaced.
-  return replacedOps.count(op) || ignoredOps.count(op);
+  return wasOpReplaced(op) || ignoredOps.count(op);
 }
 
 bool ConversionPatternRewriterImpl::wasOpReplaced(Operation *op) const {
   // Check to see if this operation was replaced.
-  return replacedOps.count(op);
+  return replacedOps.count(op) || erasedOps.count(op);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1457,7 +1561,8 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
   // a bit more efficient, so we try to do that when possible.
   bool fastPath = !config.listener;
   if (fastPath) {
-    appendRewrite<InlineBlockRewrite>(newBlock, block, newBlock->end());
+    if (config.allowPatternRollback)
+      appendRewrite<InlineBlockRewrite>(newBlock, block, newBlock->end());
     newBlock->getOperations().splice(newBlock->end(), block->getOperations());
   } else {
     while (!block->empty())
@@ -1482,7 +1587,8 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
               OpBuilder::InsertPoint(newBlock, newBlock->begin()),
               origArg.getLoc(),
               /*valuesToMap=*/{}, /*inputs=*/ValueRange(),
-              /*outputTypes=*/origArgType, /*originalType=*/Type(), converter)
+              /*outputTypes=*/origArgType, /*originalType=*/Type(), converter,
+              /*castOp=*/nullptr, /*isPureTypeConversion=*/false)
               .front();
       replaceUsesOfBlockArgument(origArg, mat, converter);
       continue;
@@ -1504,7 +1610,8 @@ Block *ConversionPatternRewriterImpl::applySignatureConversion(
     replaceUsesOfBlockArgument(origArg, replArgs, converter);
   }
 
-  appendRewrite<BlockTypeConversionRewrite>(/*origBlock=*/block, newBlock);
+  if (config.allowPatternRollback)
+    appendRewrite<BlockTypeConversionRewrite>(/*origBlock=*/block, newBlock);
 
   // Erase the old block. (It is just unlinked for now and will be erased during
   // cleanup.)
@@ -1523,7 +1630,7 @@ ValueRange ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
     MaterializationKind kind, OpBuilder::InsertPoint ip, Location loc,
     ValueVector valuesToMap, ValueRange inputs, TypeRange outputTypes,
     Type originalType, const TypeConverter *converter,
-    UnrealizedConversionCastOp *castOp) {
+    UnrealizedConversionCastOp *castOp, bool isPureTypeConversion) {
   assert((!originalType || kind == MaterializationKind::Target) &&
          "original type is valid only for target materializations");
   assert(TypeRange(inputs) != outputTypes &&
@@ -1533,21 +1640,37 @@ ValueRange ConversionPatternRewriterImpl::buildUnresolvedMaterialization(
   // tracking the materialization like we do for other operations.
   OpBuilder builder(outputTypes.front().getContext());
   builder.setInsertionPoint(ip.getBlock(), ip.getPoint());
-  auto convertOp =
+  UnrealizedConversionCastOp convertOp =
       UnrealizedConversionCastOp::create(builder, loc, outputTypes, inputs);
-  if (!valuesToMap.empty())
-    mapping.map(valuesToMap, convertOp.getResults());
+  if (config.attachDebugMaterializationKind) {
+    StringRef kindStr =
+        kind == MaterializationKind::Source ? "source" : "target";
+    convertOp->setAttr("__kind__", builder.getStringAttr(kindStr));
+  }
+  if (isPureTypeConversion)
+    convertOp->setAttr(kPureTypeConversionMarker, builder.getUnitAttr());
+
+  // Register the materialization.
   if (castOp)
     *castOp = convertOp;
   unresolvedMaterializations[convertOp] =
       UnresolvedMaterializationInfo(converter, kind, originalType);
-  appendRewrite<UnresolvedMaterializationRewrite>(convertOp,
-                                                  std::move(valuesToMap));
+  if (config.allowPatternRollback) {
+    if (!valuesToMap.empty())
+      mapping.map(valuesToMap, convertOp.getResults());
+    appendRewrite<UnresolvedMaterializationRewrite>(convertOp,
+                                                    std::move(valuesToMap));
+  } else {
+    patternMaterializations.insert(convertOp);
+  }
   return convertOp.getResults();
 }
 
 Value ConversionPatternRewriterImpl::findOrBuildReplacementValue(
     Value value, const TypeConverter *converter) {
+  assert(config.allowPatternRollback &&
+         "this code path is valid only in rollback mode");
+
   // Try to find a replacement value with the same type in the conversion value
   // mapping. This includes cached materializations. We try to reuse those
   // instead of generating duplicate IR.
@@ -1609,26 +1732,119 @@ void ConversionPatternRewriterImpl::notifyOperationInserted(
       logger.getOStream() << " (was detached)";
     logger.getOStream() << "\n";
   });
-  assert(!wasOpReplaced(op->getParentOp()) &&
+
+  // In rollback mode, it is easier to misuse the API, so perform extra error
+  // checking.
+  assert(!(config.allowPatternRollback && wasOpReplaced(op->getParentOp())) &&
          "attempting to insert into a block within a replaced/erased op");
 
+  // In "no rollback" mode, the listener is always notified immediately.
+  if (!config.allowPatternRollback && config.listener)
+    config.listener->notifyOperationInserted(op, previous);
+
   if (wasDetached) {
-    // If the op was detached, it is most likely a newly created op.
-    // TODO: If the same op is inserted multiple times from a detached state,
-    // the rollback mechanism may erase the same op multiple times. This is a
-    // bug in the rollback-based dialect conversion driver.
-    appendRewrite<CreateOperationRewrite>(op);
+    // If the op was detached, it is most likely a newly created op. Add it the
+    // set of newly created ops, so that it will be legalized. If this op is
+    // not a newly created op, it will be legalized a second time, which is
+    // inefficient but harmless.
     patternNewOps.insert(op);
+
+    if (config.allowPatternRollback) {
+      // TODO: If the same op is inserted multiple times from a detached
+      // state, the rollback mechanism may erase the same op multiple times.
+      // This is a bug in the rollback-based dialect conversion driver.
+      appendRewrite<CreateOperationRewrite>(op);
+    } else {
+      // In "no rollback" mode, there is an extra data structure for tracking
+      // erased operations that must be kept up to date.
+      erasedOps.erase(op);
+    }
     return;
   }
 
   // The op was moved from one place to another.
-  appendRewrite<MoveOperationRewrite>(op, previous);
+  if (config.allowPatternRollback)
+    appendRewrite<MoveOperationRewrite>(op, previous);
+}
+
+/// Given that `fromRange` is about to be replaced with `toRange`, compute
+/// replacement values with the types of `fromRange`.
+static SmallVector<Value>
+getReplacementValues(ConversionPatternRewriterImpl &impl, ValueRange fromRange,
+                     const SmallVector<SmallVector<Value>> &toRange,
+                     const TypeConverter *converter) {
+  assert(!impl.config.allowPatternRollback &&
+         "this code path is valid only in 'no rollback' mode");
+  SmallVector<Value> repls;
+  for (auto [from, to] : llvm::zip_equal(fromRange, toRange)) {
+    if (from.use_empty()) {
+      // The replaced value is dead. No replacement value is needed.
+      repls.push_back(Value());
+      continue;
+    }
+
+    if (to.empty()) {
+      // The replaced value is dropped. Materialize a replacement value "out of
+      // thin air".
+      Value srcMat = impl.buildUnresolvedMaterialization(
+          MaterializationKind::Source, computeInsertPoint(from), from.getLoc(),
+          /*valuesToMap=*/{}, /*inputs=*/ValueRange(),
+          /*outputTypes=*/from.getType(), /*originalType=*/Type(),
+          converter)[0];
+      repls.push_back(srcMat);
+      continue;
+    }
+
+    if (TypeRange(ValueRange(to)) == TypeRange(from.getType())) {
+      // The replacement value already has the correct type. Use it directly.
+      repls.push_back(to[0]);
+      continue;
+    }
+
+    // The replacement value has the wrong type. Build a source materialization
+    // to the original type.
+    // TODO: This is a bit inefficient. We should try to reuse existing
+    // materializations if possible. This would require an extension of the
+    // `lookupOrDefault` API.
+    Value srcMat = impl.buildUnresolvedMaterialization(
+        MaterializationKind::Source, computeInsertPoint(to), from.getLoc(),
+        /*valuesToMap=*/{}, /*inputs=*/to, /*outputTypes=*/from.getType(),
+        /*originalType=*/Type(), converter)[0];
+    repls.push_back(srcMat);
+  }
+
+  return repls;
 }
 
 void ConversionPatternRewriterImpl::replaceOp(
     Operation *op, SmallVector<SmallVector<Value>> &&newValues) {
-  assert(newValues.size() == op->getNumResults());
+  assert(newValues.size() == op->getNumResults() &&
+         "incorrect number of replacement values");
+
+  if (!config.allowPatternRollback) {
+    // Pattern rollback is not allowed: materialize all IR changes immediately.
+    SmallVector<Value> repls = getReplacementValues(
+        *this, op->getResults(), newValues, currentTypeConverter);
+    // Update internal data structures, so that there are no dangling pointers
+    // to erased IR.
+    op->walk([&](Operation *op) {
+      erasedOps.insert(op);
+      ignoredOps.remove(op);
+      if (auto castOp = dyn_cast<UnrealizedConversionCastOp>(op)) {
+        unresolvedMaterializations.erase(castOp);
+        patternMaterializations.erase(castOp);
+      }
+      // The original op will be erased, so remove it from the set of
+      // unlegalized ops.
+      if (config.unlegalizedOps)
+        config.unlegalizedOps->erase(op);
+    });
+    op->walk([&](Block *block) { erasedBlocks.insert(block); });
+    // Replace the op with the replacement values and notify the listener.
+    notifyingRewriter.replaceOp(op, repls);
+    return;
+  }
+
   assert(!ignoredOps.contains(op) && "operation was already replaced");
 
   // Check if replaced op is an unresolved materialization, i.e., an
@@ -1650,7 +1866,8 @@ void ConversionPatternRewriterImpl::replaceOp(
           MaterializationKind::Source, computeInsertPoint(result),
           result.getLoc(), /*valuesToMap=*/{result}, /*inputs=*/ValueRange(),
           /*outputTypes=*/result.getType(), /*originalType=*/Type(),
-          currentTypeConverter);
+          currentTypeConverter, /*castOp=*/nullptr,
+          /*isPureTypeConversion=*/false);
       continue;
     }
 
@@ -1667,11 +1884,59 @@ void ConversionPatternRewriterImpl::replaceOp(
 
 void ConversionPatternRewriterImpl::replaceUsesOfBlockArgument(
     BlockArgument from, ValueRange to, const TypeConverter *converter) {
+  if (!config.allowPatternRollback) {
+    SmallVector<Value> toConv = llvm::to_vector(to);
+    SmallVector<Value> repls =
+        getReplacementValues(*this, from, {toConv}, converter);
+    IRRewriter r(from.getContext());
+    Value repl = repls.front();
+    if (!repl)
+      return;
+
+    performReplaceBlockArg(r, from, repl);
+    return;
+  }
+
+#ifndef NDEBUG
+  // Make sure that a block argument is not replaced multiple times. In
+  // rollback mode, `replaceUsesOfBlockArgument` replaces not only all current
+  // uses of the given block argument, but also all future uses that may be
+  // introduced by future pattern applications. Therefore, it does not make
+  // sense to call `replaceUsesOfBlockArgument` multiple times with the same
+  // block argument. Doing so would overwrite the mapping and mess with the
+  // internal state of the dialect conversion driver.
+  assert(!replacedArgs.contains(from) &&
+         "attempting to replace a block argument that was already replaced");
+  replacedArgs.insert(from);
+#endif // NDEBUG
+
   appendRewrite<ReplaceBlockArgRewrite>(from.getOwner(), from, converter);
   mapping.map(from, to);
 }
 
 void ConversionPatternRewriterImpl::eraseBlock(Block *block) {
+  if (!config.allowPatternRollback) {
+    // Pattern rollback is not allowed: materialize all IR changes immediately.
+    // Update internal data structures, so that there are no dangling pointers
+    // to erased IR.
+    block->walk([&](Operation *op) {
+      erasedOps.insert(op);
+      ignoredOps.remove(op);
+      if (auto castOp = dyn_cast<UnrealizedConversionCastOp>(op)) {
+        unresolvedMaterializations.erase(castOp);
+        patternMaterializations.erase(castOp);
+      }
+      // The original op will be erased, so remove it from the set of
+      // unlegalized ops.
+      if (config.unlegalizedOps)
+        config.unlegalizedOps->erase(op);
+    });
+    block->walk([&](Block *block) { erasedBlocks.insert(block); });
+    // Erase the block and notify the listener.
+    notifyingRewriter.eraseBlock(block);
+    return;
+  }
+
   assert(!wasOpReplaced(block->getParentOp()) &&
          "attempting to erase a block within a replaced/erased op");
   appendRewrite<EraseBlockRewrite>(block);
@@ -1705,23 +1970,37 @@ void ConversionPatternRewriterImpl::notifyBlockInserted(
           logger.getOStream() << " (was detached)";
         logger.getOStream() << "\n";
       });
-  assert(!wasOpReplaced(newParentOp) &&
+
+  // In rollback mode, it is easier to misuse the API, so perform extra error
+  // checking.
+  assert(!(config.allowPatternRollback && wasOpReplaced(newParentOp)) &&
          "attempting to insert into a region within a replaced/erased op");
   (void)newParentOp;
 
+  // In "no rollback" mode, the listener is always notified immediately.
+  if (!config.allowPatternRollback && config.listener)
+    config.listener->notifyBlockInserted(block, previous, previousIt);
+
   patternInsertedBlocks.insert(block);
 
   if (wasDetached) {
     // If the block was detached, it is most likely a newly created block.
-    // TODO: If the same block is inserted multiple times from a detached state,
-    // the rollback mechanism may erase the same block multiple times. This is a
-    // bug in the rollback-based dialect conversion driver.
-    appendRewrite<CreateBlockRewrite>(block);
+    if (config.allowPatternRollback) {
+      // TODO: If the same block is inserted multiple times from a detached
+      // state, the rollback mechanism may erase the same block multiple times.
+      // This is a bug in the rollback-based dialect conversion driver.
+      appendRewrite<CreateBlockRewrite>(block);
+    } else {
+      // In "no rollback" mode, there is an extra data structure for tracking
+      // erased blocks that must be kept up to date.
+      erasedBlocks.erase(block);
+    }
     return;
   }
 
   // The block was moved from one place to another.
-  appendRewrite<MoveBlockRewrite>(block, previous, previousIt);
+  if (config.allowPatternRollback)
+    appendRewrite<MoveBlockRewrite>(block, previous, previousIt);
 }
 
 void ConversionPatternRewriterImpl::inlineBlockBefore(Block *source,
@@ -1754,6 +2033,10 @@ ConversionPatternRewriter::ConversionPatternRewriter(
 
 ConversionPatternRewriter::~ConversionPatternRewriter() = default;
 
+const ConversionConfig &ConversionPatternRewriter::getConfig() const {
+  return impl->config;
+}
+
 void ConversionPatternRewriter::replaceOp(Operation *op, Operation *newOp) {
   assert(op && newOp && "expected non-null op");
   replaceOp(op, newOp->getResults());
@@ -1895,9 +2178,9 @@ void ConversionPatternRewriter::inlineBlockBefore(Block *source, Block *dest,
   // ops should be moved one-by-one ("slow path"), so that a separate
   // `MoveOperationRewrite` is enqueued for each moved op. Moving ops in bulk is
   // a bit more efficient, so we try to do that when possible.
-  bool fastPath = !impl->config.listener;
+  bool fastPath = !getConfig().listener;
 
-  if (fastPath)
+  if (fastPath && impl->config.allowPatternRollback)
     impl->inlineBlockBefore(source, dest, before);
 
   // Replace all uses of block arguments.
@@ -1923,6 +2206,11 @@ void ConversionPatternRewriter::inlineBlockBefore(Block *source, Block *dest,
 }
 
 void ConversionPatternRewriter::startOpModification(Operation *op) {
+  if (!impl->config.allowPatternRollback) {
+    // Pattern rollback is not allowed: no extra bookkeeping is needed.
+    PatternRewriter::startOpModification(op);
+    return;
+  }
   assert(!impl->wasOpReplaced(op) &&
          "attempting to modify a replaced/erased op");
 #ifndef NDEBUG
@@ -1932,20 +2220,29 @@ void ConversionPatternRewriter::startOpModification(Operation *op) {
 }
 
 void ConversionPatternRewriter::finalizeOpModification(Operation *op) {
-  assert(!impl->wasOpReplaced(op) &&
-         "attempting to modify a replaced/erased op");
-  PatternRewriter::finalizeOpModification(op);
   impl->patternModifiedOps.insert(op);
+  if (!impl->config.allowPatternRollback) {
+    PatternRewriter::finalizeOpModification(op);
+    if (getConfig().listener)
+      getConfig().listener->notifyOperationModified(op);
+    return;
+  }
 
   // There is nothing to do here, we only need to track the operation at the
   // start of the update.
 #ifndef NDEBUG
+  assert(!impl->wasOpReplaced(op) &&
+         "attempting to modify a replaced/erased op");
   assert(impl->pendingRootUpdates.erase(op) &&
          "operation did not have a pending in-place update");
 #endif
 }
 
 void ConversionPatternRewriter::cancelOpModification(Operation *op) {
+  if (!impl->config.allowPatternRollback) {
+    PatternRewriter::cancelOpModification(op);
+    return;
+  }
 #ifndef NDEBUG
   assert(impl->pendingRootUpdates.erase(op) &&
          "operation did not have a pending in-place update");
@@ -1970,17 +2267,17 @@ detail::ConversionPatternRewriterImpl &ConversionPatternRewriter::getImpl() {
 // ConversionPattern
 //===----------------------------------------------------------------------===//
 
-SmallVector<Value> ConversionPattern::getOneToOneAdaptorOperands(
+FailureOr<SmallVector<Value>> ConversionPattern::getOneToOneAdaptorOperands(
     ArrayRef<ValueRange> operands) const {
   SmallVector<Value> oneToOneOperands;
   oneToOneOperands.reserve(operands.size());
   for (ValueRange operand : operands) {
     if (operand.size() != 1)
-      llvm::report_fatal_error("pattern '" + getDebugName() +
-                               "' does not support 1:N conversion");
+      return failure();
+
     oneToOneOperands.push_back(operand.front());
   }
-  return oneToOneOperands;
+  return std::move(oneToOneOperands);
 }
 
 LogicalResult
@@ -2018,8 +2315,7 @@ class OperationLegalizer {
   using LegalizationAction = ConversionTarget::LegalizationAction;
 
   OperationLegalizer(const ConversionTarget &targetInfo,
-                     const FrozenRewritePatternSet &patterns,
-                     const ConversionConfig &config);
+                     const FrozenRewritePatternSet &patterns);
 
   /// Returns true if the given operation is known to be illegal on the target.
   bool isIllegal(Operation *op) const;
@@ -2116,16 +2412,12 @@ class OperationLegalizer {
 
   /// The pattern applicator to use for conversions.
   PatternApplicator applicator;
-
-  /// Dialect conversion configuration.
-  const ConversionConfig &config;
 };
 } // namespace
 
 OperationLegalizer::OperationLegalizer(const ConversionTarget &targetInfo,
-                                       const FrozenRewritePatternSet &patterns,
-                                       const ConversionConfig &config)
-    : target(targetInfo), applicator(patterns), config(config) {
+                                       const FrozenRewritePatternSet &patterns)
+    : target(targetInfo), applicator(patterns) {
   // The set of patterns that can be applied to illegal operations to transform
   // them into legal ones.
   DenseMap<OperationName, LegalizationPatterns> legalizerPatterns;
@@ -2203,15 +2495,17 @@ OperationLegalizer::legalize(Operation *op,
     return success();
   }
 
-  // If the operation isn't legal, try to fold it in-place.
-  // TODO: Should we always try to do this, even if the op is
-  // already legal?
-  if (succeeded(legalizeWithFold(op, rewriter))) {
-    LLVM_DEBUG({
-      logSuccess(logger, "operation was folded");
-      logger.startLine() << logLineComment;
-    });
-    return success();
+  // If the operation is not legal, try to fold it in-place if the folding mode
+  // is 'BeforePatterns'. 'Never' will skip this.
+  const ConversionConfig &config = rewriter.getConfig();
+  if (config.foldingMode == DialectConversionFoldingMode::BeforePatterns) {
+    if (succeeded(legalizeWithFold(op, rewriter))) {
+      LLVM_DEBUG({
+        logSuccess(logger, "operation was folded");
+        logger.startLine() << logLineComment;
+      });
+      return success();
+    }
   }
 
   // Otherwise, we need to apply a legalization pattern to this operation.
@@ -2223,6 +2517,18 @@ OperationLegalizer::legalize(Operation *op,
     return success();
   }
 
+  // If the operation can't be legalized via patterns, try to fold it in-place
+  // if the folding mode is 'AfterPatterns'.
+  if (config.foldingMode == DialectConversionFoldingMode::AfterPatterns) {
+    if (succeeded(legalizeWithFold(op, rewriter))) {
+      LLVM_DEBUG({
+        logSuccess(logger, "operation was folded");
+        logger.startLine() << logLineComment;
+      });
+      return success();
+    }
+  }
+
   LLVM_DEBUG({
     logFailure(logger, "no matched legalization pattern");
     logger.startLine() << logLineComment;
@@ -2286,7 +2592,7 @@ OperationLegalizer::legalizeWithFold(Operation *op,
       LLVM_DEBUG(logFailure(rewriterImpl.logger,
                             "failed to legalize generated constant '{0}'",
                             newOp->getName()));
-      if (!config.allowPatternRollback) {
+      if (!rewriter.getConfig().allowPatternRollback) {
         // Rolling back a folder is like rolling back a pattern.
         llvm::report_fatal_error(
             "op '" + opName +
@@ -2302,10 +2608,36 @@ OperationLegalizer::legalizeWithFold(Operation *op,
   return success();
 }
 
+/// Report a fatal error indicating that newly produced or modified IR could
+/// not be legalized.
+static void
+reportNewIrLegalizationFatalError(const Pattern &pattern,
+                                  const SetVector<Operation *> &newOps,
+                                  const SetVector<Operation *> &modifiedOps,
+                                  const SetVector<Block *> &insertedBlocks) {
+  auto newOpNames = llvm::map_range(
+      newOps, [](Operation *op) { return op->getName().getStringRef(); });
+  auto modifiedOpNames = llvm::map_range(
+      modifiedOps, [](Operation *op) { return op->getName().getStringRef(); });
+  StringRef detachedBlockStr = "(detached block)";
+  auto insertedBlockNames = llvm::map_range(insertedBlocks, [&](Block *block) {
+    if (block->getParentOp())
+      return block->getParentOp()->getName().getStringRef();
+    return detachedBlockStr;
+  });
+  llvm::report_fatal_error(
+      "pattern '" + pattern.getDebugName() +
+      "' produced IR that could not be legalized. " + "new ops: {" +
+      llvm::join(newOpNames, ", ") + "}, " + "modified ops: {" +
+      llvm::join(modifiedOpNames, ", ") + "}, " + "inserted block into ops: {" +
+      llvm::join(insertedBlockNames, ", ") + "}");
+}
+
 LogicalResult
 OperationLegalizer::legalizeWithPattern(Operation *op,
                                         ConversionPatternRewriter &rewriter) {
   auto &rewriterImpl = rewriter.getImpl();
+  const ConversionConfig &config = rewriter.getConfig();
 
 #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
   Operation *checkOp;
@@ -2345,17 +2677,23 @@ OperationLegalizer::legalizeWithPattern(Operation *op,
   RewriterState curState = rewriterImpl.getCurrentState();
   auto onFailure = [&](const Pattern &pattern) {
     assert(rewriterImpl.pendingRootUpdates.empty() && "dangling root updates");
-#if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
     if (!rewriterImpl.config.allowPatternRollback) {
-      // Returning "failure" after modifying IR is not allowed.
+      // Erase all unresolved materializations.
+      for (auto op : rewriterImpl.patternMaterializations) {
+        rewriterImpl.unresolvedMaterializations.erase(op);
+        op.erase();
+      }
+      rewriterImpl.patternMaterializations.clear();
+#if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
+      // Expensive pattern check that can detect API violations.
       if (checkOp) {
         OperationFingerPrint fingerPrintAfterPattern(checkOp);
         if (fingerPrintAfterPattern != *topLevelFingerPrint)
           llvm::report_fatal_error("pattern '" + pattern.getDebugName() +
                                    "' returned failure but IR did change");
       }
-    }
 #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
+    }
     rewriterImpl.patternNewOps.clear();
     rewriterImpl.patternModifiedOps.clear();
     rewriterImpl.patternInsertedBlocks.clear();
@@ -2379,6 +2717,16 @@ OperationLegalizer::legalizeWithPattern(Operation *op,
   // successfully applied.
   auto onSuccess = [&](const Pattern &pattern) {
     assert(rewriterImpl.pendingRootUpdates.empty() && "dangling root updates");
+    if (!rewriterImpl.config.allowPatternRollback) {
+      // Eagerly erase unused materializations.
+      for (auto op : rewriterImpl.patternMaterializations) {
+        if (op->use_empty()) {
+          rewriterImpl.unresolvedMaterializations.erase(op);
+          op.erase();
+        }
+      }
+      rewriterImpl.patternMaterializations.clear();
+    }
     SetVector<Operation *> newOps = moveAndReset(rewriterImpl.patternNewOps);
     SetVector<Operation *> modifiedOps =
         moveAndReset(rewriterImpl.patternModifiedOps);
@@ -2389,8 +2737,8 @@ OperationLegalizer::legalizeWithPattern(Operation *op,
     appliedPatterns.erase(&pattern);
     if (failed(result)) {
       if (!rewriterImpl.config.allowPatternRollback)
-        llvm::report_fatal_error("pattern '" + pattern.getDebugName() +
-                                 "' produced IR that could not be legalized");
+        reportNewIrLegalizationFatalError(pattern, newOps, modifiedOps,
+                                          insertedBlocks);
       rewriterImpl.resetState(curState, pattern.getDebugName());
     }
     if (config.listener)
@@ -2469,6 +2817,9 @@ LogicalResult OperationLegalizer::legalizePatternBlockRewrites(
   // If the pattern moved or created any blocks, make sure the types of block
   // arguments get legalized.
   for (Block *block : insertedBlocks) {
+    if (impl.erasedBlocks.contains(block))
+      continue;
+
     // Only check blocks outside of the current operation.
     Operation *parentOp = block->getParentOp();
     if (!parentOp || parentOp == op || block->getNumArguments() == 0)
@@ -2749,8 +3100,7 @@ struct OperationConverter {
                               const FrozenRewritePatternSet &patterns,
                               const ConversionConfig &config,
                               OpConversionMode mode)
-      : config(config), opLegalizer(target, patterns, this->config),
-        mode(mode) {}
+      : config(config), opLegalizer(target, patterns), mode(mode) {}
 
   /// Converts the given operations to the conversion target.
   LogicalResult convertOperations(ArrayRef<Operation *> ops);
@@ -2902,6 +3252,10 @@ LogicalResult OperationConverter::convertOperations(ArrayRef<Operation *> ops) {
   SmallVector<UnrealizedConversionCastOp> remainingCastOps;
   reconcileUnrealizedCasts(allCastOps, &remainingCastOps);
 
+  // Drop markers.
+  for (UnrealizedConversionCastOp castOp : remainingCastOps)
+    castOp->removeAttr(kPureTypeConversionMarker);
+
   // Try to legalize all unresolved materializations.
   if (config.buildMaterializations) {
     IRRewriter rewriter(rewriterImpl.context, config.listener);
diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
index 607b86cb86315..0324588ac6691 100644
--- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp
@@ -15,6 +15,8 @@
 #include "mlir/Config/mlir-config.h"
 #include "mlir/IR/Action.h"
 #include "mlir/IR/Matchers.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/Verifier.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Rewrite/PatternApplicator.h"
@@ -23,7 +25,7 @@
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/ScopeExit.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -178,9 +180,8 @@ static Operation *getDumpRootOp(Operation *op) {
   return op;
 }
 static void logSuccessfulFolding(Operation *op) {
-  llvm::dbgs() << "// *** IR Dump After Successful Folding ***\n";
-  op->dump();
-  llvm::dbgs() << "\n\n";
+  LDBG() << "// *** IR Dump After Successful Folding ***\n"
+         << OpWithFlags(op, OpPrintingFlags().elideLargeElementsAttrs());
 }
 #endif // NDEBUG
 
@@ -394,8 +395,12 @@ class GreedyPatternRewriteDriver : public RewriterBase::Listener {
                      function_ref<void(Diagnostic &)> reasonCallback) override;
 
 #ifndef NDEBUG
+  /// A raw output stream used to prefix the debug log.
+
+  llvm::impl::raw_ldbg_ostream os{(Twine("[") + DEBUG_TYPE + ":1] ").str(),
+                                  llvm::dbgs()};
   /// A logger used to emit information during the application process.
-  llvm::ScopedPrinter logger{llvm::dbgs()};
+  llvm::ScopedPrinter logger{os};
 #endif
 
   /// The low-level pattern applicator.
@@ -871,7 +876,18 @@ LogicalResult RegionPatternRewriteDriver::simplify(bool *changed) && {
 
     ctx->executeAction<GreedyPatternRewriteIteration>(
         [&] {
-          continueRewrites = processWorklist();
+          continueRewrites = false;
+
+          // Erase unreachable blocks
+          // Operations like:
+          //   %add = arith.addi %add, %add : i64
+          // are legal in unreachable code. Unfortunately many patterns would be
+          // unsafe to apply on such IR and can lead to crashes or infinite
+          // loops.
+          continueRewrites |=
+              succeeded(eraseUnreachableBlocks(rewriter, region));
+
+          continueRewrites |= processWorklist();
 
           // After applying patterns, make sure that the CFG of each of the
           // regions is kept up to date.
@@ -917,10 +933,9 @@ mlir::applyPatternsGreedily(Region &region,
   RegionPatternRewriteDriver driver(region.getContext(), patterns, config,
                                     region);
   LogicalResult converged = std::move(driver).simplify(changed);
-  LLVM_DEBUG(if (failed(converged)) {
-    llvm::dbgs() << "The pattern rewrite did not converge after scanning "
-                 << config.getMaxIterations() << " times\n";
-  });
+  if (failed(converged))
+    LDBG() << "The pattern rewrite did not converge after scanning "
+           << config.getMaxIterations() << " times";
   return converged;
 }
 
@@ -1052,9 +1067,8 @@ LogicalResult mlir::applyOpPatternsGreedily(
   LogicalResult converged = std::move(driver).simplify(ops, changed);
   if (allErased)
     *allErased = surviving.empty();
-  LLVM_DEBUG(if (failed(converged)) {
-    llvm::dbgs() << "The pattern rewrite did not converge after "
-                 << config.getMaxNumRewrites() << " rewrites";
-  });
+  if (failed(converged))
+    LDBG() << "The pattern rewrite did not converge after "
+           << config.getMaxNumRewrites() << " rewrites";
   return converged;
 }
diff --git a/mlir/lib/Transforms/Utils/InliningUtils.cpp b/mlir/lib/Transforms/Utils/InliningUtils.cpp
index eeb40529cc2fe..73107cfc36ea9 100644
--- a/mlir/lib/Transforms/Utils/InliningUtils.cpp
+++ b/mlir/lib/Transforms/Utils/InliningUtils.cpp
@@ -13,10 +13,12 @@
 #include "mlir/Transforms/InliningUtils.h"
 
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/IRMapping.h"
 #include "mlir/IR/Operation.h"
 #include "mlir/Interfaces/CallInterfaces.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/raw_ostream.h"
 #include <optional>
 
@@ -182,13 +184,16 @@ static bool isLegalToInline(InlinerInterface &interface, Region *src,
                             IRMapping &valueMapping) {
   for (auto &block : *src) {
     for (auto &op : block) {
+      // UnrealizedConversionCastOp is inlineable but cannot implement the
+      // inliner interface due to layering constraints.
+      if (isa<UnrealizedConversionCastOp>(op))
+        continue;
+
       // Check this operation.
       if (!interface.isLegalToInline(&op, insertRegion,
                                      shouldCloneInlinedRegion, valueMapping)) {
-        LLVM_DEBUG({
-          llvm::dbgs() << "* Illegal to inline because of op: ";
-          op.dump();
-        });
+        LDBG() << "* Illegal to inline because of op: "
+               << OpWithFlags(&op, OpPrintingFlags().skipRegions());
         return false;
       }
       // Check any nested regions.
diff --git a/mlir/lib/Transforms/Utils/LoopInvariantCodeMotionUtils.cpp b/mlir/lib/Transforms/Utils/LoopInvariantCodeMotionUtils.cpp
index cb3f2c52e2116..33f648269faa4 100644
--- a/mlir/lib/Transforms/Utils/LoopInvariantCodeMotionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/LoopInvariantCodeMotionUtils.cpp
@@ -13,11 +13,13 @@
 #include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"
 
 #include "mlir/IR/Operation.h"
+#include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/SubsetOpInterface.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/DebugLog.h"
 #include <queue>
 
 #define DEBUG_TYPE "licm"
@@ -64,8 +66,7 @@ size_t mlir::moveLoopInvariantCode(
   size_t numMoved = 0;
 
   for (Region *region : regions) {
-    LLVM_DEBUG(llvm::dbgs() << "Original loop:\n"
-                            << *region->getParentOp() << "\n");
+    LDBG() << "Original loop:\n" << *region->getParentOp();
 
     std::queue<Operation *> worklist;
     // Add top-level operations in the loop body to the worklist.
@@ -83,12 +84,13 @@ size_t mlir::moveLoopInvariantCode(
       if (op->getParentRegion() != region)
         continue;
 
-      LLVM_DEBUG(llvm::dbgs() << "Checking op: " << *op << "\n");
+      LDBG() << "Checking op: "
+             << OpWithFlags(op, OpPrintingFlags().skipRegions());
       if (!shouldMoveOutOfRegion(op, region) ||
           !canBeHoisted(op, definedOutside))
         continue;
 
-      LLVM_DEBUG(llvm::dbgs() << "Moving loop-invariant op: " << *op << "\n");
+      LDBG() << "Moving loop-invariant op: " << *op;
       moveOutOfRegion(op, region);
       ++numMoved;
 
diff --git a/mlir/lib/Transforms/Utils/RegionUtils.cpp b/mlir/lib/Transforms/Utils/RegionUtils.cpp
index a1d975dfb1476..31ae1d1895b81 100644
--- a/mlir/lib/Transforms/Utils/RegionUtils.cpp
+++ b/mlir/lib/Transforms/Utils/RegionUtils.cpp
@@ -23,12 +23,15 @@
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/DebugLog.h"
 
 #include <deque>
 #include <iterator>
 
 using namespace mlir;
 
+#define DEBUG_TYPE "region-utils"
+
 void mlir::replaceAllUsesInRegionWith(Value orig, Value replacement,
                                       Region &region) {
   for (auto &use : llvm::make_early_inc_range(orig.getUses())) {
@@ -182,19 +185,34 @@ SmallVector<Value> mlir::makeRegionIsolatedFromAbove(
 // TODO: We could likely merge this with the DCE algorithm below.
 LogicalResult mlir::eraseUnreachableBlocks(RewriterBase &rewriter,
                                            MutableArrayRef<Region> regions) {
+  LDBG() << "Starting eraseUnreachableBlocks with " << regions.size()
+         << " regions";
+
   // Set of blocks found to be reachable within a given region.
   llvm::df_iterator_default_set<Block *, 16> reachable;
   // If any blocks were found to be dead.
-  bool erasedDeadBlocks = false;
+  int erasedDeadBlocks = 0;
 
   SmallVector<Region *, 1> worklist;
   worklist.reserve(regions.size());
   for (Region &region : regions)
     worklist.push_back(&region);
+
+  LDBG(2) << "Initial worklist size: " << worklist.size();
+
   while (!worklist.empty()) {
     Region *region = worklist.pop_back_val();
-    if (region->empty())
+    if (region->empty()) {
+      LDBG(2) << "Skipping empty region";
       continue;
+    }
+
+    LDBG(2) << "Processing region with " << region->getBlocks().size()
+            << " blocks";
+    if (region->getParentOp())
+      LDBG(2) << " -> for operation:  "
+              << OpWithFlags(region->getParentOp(),
+                             OpPrintingFlags().skipRegions());
 
     // If this is a single block region, just collect the nested regions.
     if (region->hasOneBlock()) {
@@ -209,13 +227,17 @@ LogicalResult mlir::eraseUnreachableBlocks(RewriterBase &rewriter,
     for (Block *block : depth_first_ext(&region->front(), reachable))
       (void)block /* Mark all reachable blocks */;
 
+    LDBG(2) << "Found " << reachable.size() << " reachable blocks out of "
+            << region->getBlocks().size() << " total blocks";
+
     // Collect all of the dead blocks and push the live regions onto the
     // worklist.
     for (Block &block : llvm::make_early_inc_range(*region)) {
       if (!reachable.count(&block)) {
+        LDBG() << "Erasing unreachable block: " << &block;
         block.dropAllDefinedValueUses();
         rewriter.eraseBlock(&block);
-        erasedDeadBlocks = true;
+        ++erasedDeadBlocks;
         continue;
       }
 
@@ -226,7 +248,10 @@ LogicalResult mlir::eraseUnreachableBlocks(RewriterBase &rewriter,
     }
   }
 
-  return success(erasedDeadBlocks);
+  LDBG() << "Finished eraseUnreachableBlocks, erased " << erasedDeadBlocks
+         << " dead blocks";
+
+  return success(erasedDeadBlocks > 0);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Transforms/Utils/WalkPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/WalkPatternRewriteDriver.cpp
index ee5c642c943c4..1382550e0f7e6 100644
--- a/mlir/lib/Transforms/Utils/WalkPatternRewriteDriver.cpp
+++ b/mlir/lib/Transforms/Utils/WalkPatternRewriteDriver.cpp
@@ -13,18 +13,40 @@
 #include "mlir/Transforms/WalkPatternRewriteDriver.h"
 
 #include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/Operation.h"
 #include "mlir/IR/OperationSupport.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/IR/Verifier.h"
 #include "mlir/IR/Visitors.h"
 #include "mlir/Rewrite/PatternApplicator.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/DebugLog.h"
 #include "llvm/Support/ErrorHandling.h"
 
 #define DEBUG_TYPE "walk-rewriter"
 
 namespace mlir {
 
+// Find all reachable blocks in the region and add them to the visitedBlocks
+// set.
+static void findReachableBlocks(Region &region,
+                                DenseSet<Block *> &reachableBlocks) {
+  Block *entryBlock = &region.front();
+  reachableBlocks.insert(entryBlock);
+  // Traverse the CFG and add all reachable blocks to the blockList.
+  SmallVector<Block *> worklist({entryBlock});
+  while (!worklist.empty()) {
+    Block *block = worklist.pop_back_val();
+    Operation *terminator = &block->back();
+    for (Block *successor : terminator->getSuccessors()) {
+      if (reachableBlocks.contains(successor))
+        continue;
+      worklist.push_back(successor);
+      reachableBlocks.insert(successor);
+    }
+  }
+}
+
 namespace {
 struct WalkAndApplyPatternsAction final
     : tracing::ActionImpl<WalkAndApplyPatternsAction> {
@@ -88,20 +110,104 @@ void walkAndApplyPatterns(Operation *op,
   PatternApplicator applicator(patterns);
   applicator.applyDefaultCostModel();
 
+  // Iterator on all reachable operations in the region.
+  // Also keep track if we visited the nested regions of the current op
+  // already to drive the post-order traversal.
+  struct RegionReachableOpIterator {
+    RegionReachableOpIterator(Region *region) : region(region) {
+      regionIt = region->begin();
+      if (regionIt != region->end())
+        blockIt = regionIt->begin();
+      if (!llvm::hasSingleElement(*region))
+        findReachableBlocks(*region, reachableBlocks);
+    }
+    // Advance the iterator to the next reachable operation.
+    void advance() {
+      assert(regionIt != region->end());
+      hasVisitedRegions = false;
+      if (blockIt == regionIt->end()) {
+        ++regionIt;
+        while (regionIt != region->end() &&
+               !reachableBlocks.contains(&*regionIt))
+          ++regionIt;
+        if (regionIt != region->end())
+          blockIt = regionIt->begin();
+        return;
+      }
+      ++blockIt;
+      if (blockIt != regionIt->end()) {
+        LDBG() << "Incrementing block iterator, next op: "
+               << OpWithFlags(&*blockIt, OpPrintingFlags().skipRegions());
+      }
+    }
+    // The region we're iterating over.
+    Region *region;
+    // The Block currently being iterated over.
+    Region::iterator regionIt;
+    // The Operation currently being iterated over.
+    Block::iterator blockIt;
+    // The set of blocks that are reachable in the current region.
+    DenseSet<Block *> reachableBlocks;
+    // Whether we've visited the nested regions of the current op already.
+    bool hasVisitedRegions = false;
+  };
+
+  // Worklist of regions to visit to drive the post-order traversal.
+  SmallVector<RegionReachableOpIterator> worklist;
+
+  LDBG() << "Starting walk-based pattern rewrite driver";
   ctx->executeAction<WalkAndApplyPatternsAction>(
       [&] {
+        // Perform a post-order traversal of the regions, visiting each
+        // reachable operation.
         for (Region &region : op->getRegions()) {
-          region.walk([&](Operation *visitedOp) {
-            LLVM_DEBUG(llvm::dbgs() << "Visiting op: "; visitedOp->print(
-                llvm::dbgs(), OpPrintingFlags().skipRegions());
-                       llvm::dbgs() << "\n";);
+          assert(worklist.empty());
+          if (region.empty())
+            continue;
+
+          // Prime the worklist with the entry block of this region.
+          worklist.push_back({&region});
+          while (!worklist.empty()) {
+            RegionReachableOpIterator &it = worklist.back();
+            if (it.regionIt == it.region->end()) {
+              // We're done with this region.
+              worklist.pop_back();
+              continue;
+            }
+            if (it.blockIt == it.regionIt->end()) {
+              // We're done with this block.
+              it.advance();
+              continue;
+            }
+            Operation *op = &*it.blockIt;
+            // If we haven't visited the nested regions of this op yet,
+            // enqueue them.
+            if (!it.hasVisitedRegions) {
+              it.hasVisitedRegions = true;
+              for (Region &nestedRegion : llvm::reverse(op->getRegions())) {
+                if (nestedRegion.empty())
+                  continue;
+                worklist.push_back({&nestedRegion});
+              }
+            }
+            // If we're not at the back of the worklist, we've enqueued some
+            // nested region for processing. We'll come back to this op later
+            // (post-order)
+            if (&it != &worklist.back())
+              continue;
+
+            // Preemptively increment the iterator, in case the current op
+            // would be erased.
+            it.advance();
+
+            LDBG() << "Visiting op: "
+                   << OpWithFlags(op, OpPrintingFlags().skipRegions());
 #if MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
-            erasedListener.visitedOp = visitedOp;
+            erasedListener.visitedOp = op;
 #endif // MLIR_ENABLE_EXPENSIVE_PATTERN_API_CHECKS
-            if (succeeded(applicator.matchAndRewrite(visitedOp, rewriter))) {
-              LLVM_DEBUG(llvm::dbgs() << "\tOp matched and rewritten\n";);
-            }
-          });
+            if (succeeded(applicator.matchAndRewrite(op, rewriter)))
+              LDBG() << "\tOp matched and rewritten";
+          }
         }
       },
       {op});
diff --git a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
index be71737e4b5b4..dcae3dd742940 100644
--- a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
+++ b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
@@ -194,6 +194,13 @@ class _OperationBase:
         """
         Detaches the operation from its parent block.
         """
+
+    @property
+    def attached(self) -> bool:
+        """
+        Reports if the operation is attached to its parent block.
+        """
+
     def erase(self) -> None: ...
 
     @overload
diff --git a/mlir/python/mlir/_mlir_libs/_mlirExecutionEngine.pyi b/mlir/python/mlir/_mlir_libs/_mlirExecutionEngine.pyi
index 58d453d2b2d37..4b82c78489295 100644
--- a/mlir/python/mlir/_mlir_libs/_mlirExecutionEngine.pyi
+++ b/mlir/python/mlir/_mlir_libs/_mlirExecutionEngine.pyi
@@ -19,5 +19,6 @@ class ExecutionEngine:
     def dump_to_object_file(self, file_name: str) -> None: ...
     def raw_lookup(self, func_name: str) -> int: ...
     def raw_register_runtime(self, name: str, callback: object) -> None: ...
+    def init() -> None: ...
     @property
     def _CAPIPtr(self) -> object: ...
diff --git a/mlir/python/mlir/dialects/_ods_common.py b/mlir/python/mlir/dialects/_ods_common.py
index a5efa057cfb64..10abd06ff266e 100644
--- a/mlir/python/mlir/dialects/_ods_common.py
+++ b/mlir/python/mlir/dialects/_ods_common.py
@@ -78,12 +78,12 @@ def equally_sized_accessor(
 def get_default_loc_context(location=None):
     """
     Returns a context in which the defaulted location is created. If the location
-    is None, takes the current location from the stack, raises ValueError if there
-    is no location on the stack.
+    is None, takes the current location from the stack.
     """
     if location is None:
-        # Location.current raises ValueError if there is no current location.
-        return _cext.ir.Location.current.context
+        if _cext.ir.Location.current:
+            return _cext.ir.Location.current.context
+        return None
     return location.context
 
 
diff --git a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
index 1b359da40a291..fd4a5a848f1e3 100644
--- a/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
+++ b/mlir/python/mlir/dialects/linalg/opdsl/ops/core_named_ops.py
@@ -373,42 +373,6 @@ def quantized_matmul(
     )
 
 
-@linalg_structured_op
-def matmul_transpose_a(
-    A=TensorDef(T1, S.K, S.N),
-    B=TensorDef(T2, S.K, S.M),
-    C=TensorDef(U, S.M, S.N, output=True),
-    cast=TypeFnAttrDef(default=TypeFn.cast_signed),
-):
-    """Performs a matrix multiplication of two 2D inputs with lhs operand
-    transposed.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-    """
-    domain(D.m, D.n, D.k)
-    implements(ContractionOpInterface)
-    C[D.m, D.n] += cast(U, A[D.k, D.m]) * cast(U, B[D.k, D.n])
-
-
-@linalg_structured_op
-def matmul_transpose_b(
-    A=TensorDef(T1, S.M, S.K),
-    B=TensorDef(T2, S.N, S.K),
-    C=TensorDef(U, S.M, S.N, output=True),
-    cast=TypeFnAttrDef(default=TypeFn.cast_signed),
-):
-    """Performs a matrix multiplication of two 2D inputs with rhs operand
-    transposed.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-    """
-    domain(D.m, D.n, D.k)
-    implements(ContractionOpInterface)
-    C[D.m, D.n] += cast(U, A[D.m, D.k]) * cast(U, B[D.n, D.k])
-
-
 @linalg_structured_op
 def mmt4d(
     lhs=TensorDef(TV.LhsType, S.M, S.K, S.M0, S.K0),
@@ -453,44 +417,6 @@ def batch_mmt4d(
     ) * TypeFn.cast_signed(TV.AccumType, rhs[D.b, D.n, D.k, D.n0, D.k0])
 
 
-@linalg_structured_op
-def batch_matmul_transpose_a(
-    A=TensorDef(T1, Batch, S.K, S.M),
-    B=TensorDef(T2, Batch, S.K, S.N),
-    C=TensorDef(U, Batch, S.M, S.N, output=True),
-):
-    """Performs a batched matrix multiplication of two 3D inputs where lhs operand
-    has its non-batch dimensions transposed.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-    """
-    domain(D.b, D.m, D.n, D.k)
-    implements(ContractionOpInterface)
-    C[D.b, D.m, D.n] += TypeFn.cast_signed(U, A[D.b, D.k, D.m]) * TypeFn.cast_signed(
-        U, B[D.b, D.k, D.n]
-    )
-
-
-@linalg_structured_op
-def batch_matmul_transpose_b(
-    A=TensorDef(T1, Batch, S.M, S.K),
-    B=TensorDef(T2, Batch, S.N, S.K),
-    C=TensorDef(U, Batch, S.M, S.N, output=True),
-):
-    """Performs a batched matrix multiplication of two 3D inputs where rhs operand
-    has its non-batch dimensions transposed.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-    """
-    domain(D.b, D.m, D.n, D.k)
-    implements(ContractionOpInterface)
-    C[D.b, D.m, D.n] += TypeFn.cast_signed(U, A[D.b, D.m, D.k]) * TypeFn.cast_signed(
-        U, B[D.b, D.n, D.k]
-    )
-
-
 @linalg_structured_op
 def quantized_batch_matmul(
     A=TensorDef(T1, Batch, S.M, S.K),
@@ -512,25 +438,6 @@ def quantized_batch_matmul(
     ) * (TypeFn.cast_signed(U, B[D.b, D.k, D.n]) - TypeFn.cast_signed(U, BZp))
 
 
-@linalg_structured_op
-def batch_reduce_matmul(
-    A=TensorDef(T1, Batch, S.M, S.K),
-    B=TensorDef(T2, Batch, S.K, S.N),
-    C=TensorDef(U, S.M, S.N, output=True),
-):
-    """Performs a batch-reduce matrix multiplication of two 3D inputs.
-    The partial multiplication results are reduced into a 2D output.
-
-    Numeric casting is performed on the operands to the inner multiply, promoting
-    them to the same data type as the accumulator/output.
-    """
-    domain(D.b, D.m, D.n, D.k)
-    implements(ContractionOpInterface)
-    C[D.m, D.n] += TypeFn.cast_signed(U, A[D.b, D.m, D.k]) * TypeFn.cast_signed(
-        U, B[D.b, D.k, D.n]
-    )
-
-
 @linalg_structured_op
 def matvec(
     A=TensorDef(T1, S.M, S.N), y=TensorDef(T2, S.N), x=TensorDef(U, S.M, output=True)
diff --git a/mlir/test/Analysis/DataFlow/test-liveness-analysis.mlir b/mlir/test/Analysis/DataFlow/test-liveness-analysis.mlir
index a89a0f4084e99..3748be74eb0f3 100644
--- a/mlir/test/Analysis/DataFlow/test-liveness-analysis.mlir
+++ b/mlir/test/Analysis/DataFlow/test-liveness-analysis.mlir
@@ -283,3 +283,23 @@ func.func @test_10_negative() -> (i32) {
   %0:2 = func.call @private_1() : () -> (i32, i32)
   return %0#0 : i32
 }
+
+// -----
+
+// Test that we correctly set a liveness value for operations in dead block.
+// These won't be visited by the dataflow framework so the analysis need to
+// explicitly manage them.
+// CHECK-LABEL: test_tag: dead_block_cmpi:
+// CHECK-NEXT: operand #0: not live
+// CHECK-NEXT: operand #1: not live
+// CHECK-NEXT: result #0: not live
+func.func @dead_block() {
+  %false = arith.constant false
+  %zero = arith.constant 0 : i64
+  cf.cond_br %false, ^bb1, ^bb4
+  ^bb1:
+    %3 = arith.cmpi eq, %zero, %zero  {tag = "dead_block_cmpi"} : i64
+    cf.br ^bb1
+  ^bb4:
+    return
+}
diff --git a/mlir/test/CAPI/CMakeLists.txt b/mlir/test/CAPI/CMakeLists.txt
index a7f9eb9b4efe8..d45142510a496 100644
--- a/mlir/test/CAPI/CMakeLists.txt
+++ b/mlir/test/CAPI/CMakeLists.txt
@@ -30,6 +30,13 @@ if(MLIR_ENABLE_EXECUTION_ENGINE)
     MLIRCAPIConversion
     MLIRCAPIExecutionEngine
     MLIRCAPIRegisterEverything
+)
+  _add_capi_test_executable(mlir-capi-global-constructors-test
+    global_constructors.c
+  LINK_LIBS PRIVATE
+    MLIRCAPIConversion
+    MLIRCAPIExecutionEngine
+    MLIRCAPIRegisterEverything
 )
 endif()
 
diff --git a/mlir/test/CAPI/global_constructors.c b/mlir/test/CAPI/global_constructors.c
new file mode 100644
index 0000000000000..bd2fe1416f0df
--- /dev/null
+++ b/mlir/test/CAPI/global_constructors.c
@@ -0,0 +1,113 @@
+//===- global_constructors.c - Test JIT with the global constructors ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM
+// Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: target=aarch64{{.*}}, target=arm64{{.*}}
+/* RUN: mlir-capi-global-constructors-test 2>&1 | FileCheck %s
+ */
+/* REQUIRES: host-supports-jit
+ */
+
+#include "mlir-c/Conversion.h"
+#include "mlir-c/ExecutionEngine.h"
+#include "mlir-c/IR.h"
+#include "mlir-c/RegisterEverything.h"
+
+#include <assert.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static void registerAllUpstreamDialects(MlirContext ctx) {
+  MlirDialectRegistry registry = mlirDialectRegistryCreate();
+  mlirRegisterAllDialects(registry);
+  mlirContextAppendDialectRegistry(ctx, registry);
+  mlirDialectRegistryDestroy(registry);
+}
+
+void lowerModuleToLLVM(MlirContext ctx, MlirModule module) {
+  MlirPassManager pm = mlirPassManagerCreate(ctx);
+  MlirOpPassManager opm = mlirPassManagerGetNestedUnder(
+      pm, mlirStringRefCreateFromCString("func.func"));
+  mlirPassManagerAddOwnedPass(pm, mlirCreateConversionConvertFuncToLLVMPass());
+  mlirOpPassManagerAddOwnedPass(
+      opm, mlirCreateConversionArithToLLVMConversionPass());
+  MlirLogicalResult status =
+      mlirPassManagerRunOnOp(pm, mlirModuleGetOperation(module));
+  if (mlirLogicalResultIsFailure(status)) {
+    fprintf(stderr, "Unexpected failure running pass pipeline\n");
+    exit(2);
+  }
+  mlirPassManagerDestroy(pm);
+}
+
+// Helper variable to track callback invocations
+static int initCnt = 0;
+
+// Callback function that will be called during JIT initialization
+static void initCallback(void) { initCnt += 1; }
+
+// CHECK-LABEL: Running test 'testGlobalCtorJitCallback'
+void testGlobalCtorJitCallback(void) {
+  MlirContext ctx = mlirContextCreate();
+  registerAllUpstreamDialects(ctx);
+
+  // Create module with global constructor that calls our callback
+  MlirModule module = mlirModuleCreateParse(
+      ctx, mlirStringRefCreateFromCString(
+               // clang-format off
+"module {                                                                       \n"
+"  llvm.mlir.global_ctors ctors = [@ctor], priorities = [0 : i32], data = [#llvm.zero] \n"
+"  llvm.func @ctor() {                                                          \n"
+"    func.call @init_callback() : () -> ()                                      \n"
+"    llvm.return                                                                \n"
+"  }                                                                            \n"
+"  func.func private @init_callback() attributes { llvm.emit_c_interface }      \n"
+"}                                                                              \n"
+               // clang-format on
+               ));
+
+  lowerModuleToLLVM(ctx, module);
+  mlirRegisterAllLLVMTranslations(ctx);
+
+  // Create execution engine with initialization disabled
+  MlirExecutionEngine jit = mlirExecutionEngineCreate(
+      module, /*optLevel=*/2, /*numPaths=*/0, /*sharedLibPaths=*/NULL,
+      /*enableObjectDump=*/false);
+
+  if (mlirExecutionEngineIsNull(jit)) {
+    fprintf(stderr, "Execution engine creation failed");
+    exit(2);
+  }
+
+  // Register callback symbol before initialization
+  mlirExecutionEngineRegisterSymbol(
+      jit, mlirStringRefCreateFromCString("_mlir_ciface_init_callback"),
+      (void *)(uintptr_t)initCallback);
+
+  mlirExecutionEngineInitialize(jit);
+
+  // CHECK: Init count: 1
+  printf("Init count: %d\n", initCnt);
+
+  mlirExecutionEngineDestroy(jit);
+  mlirModuleDestroy(module);
+  mlirContextDestroy(ctx);
+}
+
+int main(void) {
+
+#define _STRINGIFY(x) #x
+#define STRINGIFY(x) _STRINGIFY(x)
+#define TEST(test)                                                             \
+  printf("Running test '" STRINGIFY(test) "'\n");                              \
+  test();
+  TEST(testGlobalCtorJitCallback);
+  return 0;
+}
diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt
index 89568e7766ae5..8ddc620ae33be 100644
--- a/mlir/test/CMakeLists.txt
+++ b/mlir/test/CMakeLists.txt
@@ -123,7 +123,7 @@ set(MLIR_TEST_DEPENDS
   tblgen-to-irdl
   )
 if(NOT MLIR_STANDALONE_BUILD)
-  list(APPEND MLIR_TEST_DEPENDS FileCheck count not split-file)
+  list(APPEND MLIR_TEST_DEPENDS FileCheck count not split-file yaml2obj)
 endif()
 
 set(MLIR_TEST_DEPENDS ${MLIR_TEST_DEPENDS}
@@ -141,6 +141,7 @@ if(LLVM_ENABLE_PIC AND TARGET ${LLVM_NATIVE_ARCH})
     llc
     mlir_async_runtime
     mlir-capi-execution-engine-test
+    mlir-capi-global-constructors-test
     mlir_c_runner_utils
     mlir_runner_utils
     mlir_float16_utils
@@ -156,7 +157,10 @@ if(MLIR_ENABLE_CUDA_RUNNER)
 endif()
 
 if(MLIR_ENABLE_EXECUTION_ENGINE)
-  list(APPEND MLIR_TEST_DEPENDS mlir-capi-execution-engine-test)
+  list(APPEND MLIR_TEST_DEPENDS
+    mlir-capi-execution-engine-test
+    mlir-capi-global-constructors-test
+  )
 endif()
 
 if(MLIR_ENABLE_ROCM_RUNNER)
@@ -167,6 +171,10 @@ if(MLIR_ENABLE_SYCL_RUNNER)
   list(APPEND MLIR_TEST_DEPENDS mlir_sycl_runtime)
 endif()
 
+if(MLIR_ENABLE_LEVELZERO_RUNNER)
+  list(APPEND MLIR_TEST_DEPENDS mlir_levelzero_runtime)
+endif()
+
 if (MLIR_RUN_ARM_SME_TESTS AND NOT ARM_SME_ABI_ROUTINES_SHLIB)
   list(APPEND MLIR_TEST_DEPENDS mlir_arm_sme_abi_stubs)
 endif()
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/permlane.mlir b/mlir/test/Conversion/AMDGPUToROCDL/permlane.mlir
new file mode 100644
index 0000000000000..aae2b1d0fd90c
--- /dev/null
+++ b/mlir/test/Conversion/AMDGPUToROCDL/permlane.mlir
@@ -0,0 +1,163 @@
+// RUN: mlir-opt --convert-amdgpu-to-rocdl=chipset=gfx950 --canonicalize %s | FileCheck %s
+
+// CHECK-LABEL: func @test_permlane16_i32
+// CHECK-SAME: (%[[ARG0:.*]]: i32)
+func.func @test_permlane16_i32(%arg0 : i32) -> i32 {
+// CHECK:  %[[PERM:.*]] = rocdl.permlane16.swap %[[ARG0]], %[[ARG0]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:  %[[RES:.*]] = llvm.extractvalue %[[PERM]][0] : !llvm.struct<(i32, i32)>
+// CHECK:  return %[[RES]] : i32
+  %0 = amdgpu.permlane_swap %arg0 16 : i32
+  return %0 : i32
+}
+
+// CHECK-LABEL: func @test_permlane16_i32_optional_attr
+// CHECK-SAME: (%[[ARG0:.*]]: i32)
+func.func @test_permlane16_i32_optional_attr(%arg0 : i32) -> i32 {
+// CHECK:  %[[PERM:.*]] = rocdl.permlane16.swap %[[ARG0]], %[[ARG0]], true, true : (i32, i32) -> <(i32, i32)>
+// CHECK:  %[[RES:.*]] = llvm.extractvalue %[[PERM]][0] : !llvm.struct<(i32, i32)>
+// CHECK:  return %[[RES]] : i32
+  %0 = amdgpu.permlane_swap %arg0 16 { fetch_inactive = true, bound_ctrl = true }  : i32
+  return %0 : i32
+}
+
+// CHECK-LABEL: func @test_permlane32_i32
+// CHECK-SAME: (%[[ARG0:.*]]: i32)
+func.func @test_permlane32_i32(%arg0 : i32) -> i32 {
+// CHECK:  %[[PERM:.*]] = rocdl.permlane32.swap %[[ARG0]], %[[ARG0]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:  %[[RES:.*]] = llvm.extractvalue %[[PERM]][0] : !llvm.struct<(i32, i32)>
+// CHECK:  return %[[RES]] : i32
+  %0 = amdgpu.permlane_swap %arg0 32 : i32
+  return %0 : i32
+}
+
+// CHECK-LABEL: func @test_permlane16_f32
+// CHECK-SAME: (%[[ARG0:.*]]: f32)
+func.func @test_permlane16_f32(%arg0 : f32) -> f32 {
+// CHECK:  %[[CAST:.*]] = llvm.bitcast %[[ARG0]] : f32 to i32
+// CHECK:  %[[PERM:.*]] = rocdl.permlane16.swap %[[CAST]], %[[CAST]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:  %[[RES:.*]] = llvm.extractvalue %[[PERM]][0] : !llvm.struct<(i32, i32)>
+// CHECK:  %[[RES_CAST:.*]] = llvm.bitcast %[[RES]] : i32 to f32
+// CHECK:  return %[[RES_CAST]] : f32
+  %0 = amdgpu.permlane_swap %arg0 16 : f32
+  return %0 : f32
+}
+
+// CHECK-LABEL: func @test_permlane32_f32
+// CHECK-SAME: (%[[ARG0:.*]]: f32)
+func.func @test_permlane32_f32(%arg0 : f32) -> f32 {
+// CHECK:  %[[CAST:.*]] = llvm.bitcast %[[ARG0]] : f32 to i32
+// CHECK:  %[[PERM:.*]] = rocdl.permlane32.swap %[[CAST]], %[[CAST]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:  %[[RES:.*]] = llvm.extractvalue %[[PERM]][0] : !llvm.struct<(i32, i32)>
+// CHECK:  %[[RES_CAST:.*]] = llvm.bitcast %[[RES]] : i32 to f32
+// CHECK:  return %[[RES_CAST]] : f32
+  %0 = amdgpu.permlane_swap %arg0 32 : f32
+  return %0 : f32
+}
+
+// CHECK-LABEL: func @test_permlane16_f16
+// CHECK-SAME: (%[[ARG0:.*]]: f16)
+func.func @test_permlane16_f16(%arg0 : f16) -> f16 {
+// CHECK:  %[[CAST:.*]] = llvm.bitcast %[[ARG0]] : f16 to i16
+// CHECK:  %[[ZEXT:.*]] = llvm.zext %[[CAST]] : i16 to i32
+// CHECK:  %[[PERM:.*]] = rocdl.permlane16.swap %[[ZEXT]], %[[ZEXT]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:  %[[RES:.*]] = llvm.extractvalue %[[PERM]][0] : !llvm.struct<(i32, i32)>
+// CHECK:  %[[TRUNC:.*]] = llvm.trunc %[[RES]] : i32 to i16
+// CHECK:  %[[RES_CAST:.*]] = llvm.bitcast %[[TRUNC]] : i16 to f16
+// CHECK:  return %[[RES_CAST]] : f16
+  %0 = amdgpu.permlane_swap %arg0 16 : f16
+  return %0 : f16
+}
+
+// CHECK-LABEL: func @test_permlane32_f16
+// CHECK-SAME: (%[[ARG0:.*]]: f16)
+func.func @test_permlane32_f16(%arg0 : f16) -> f16 {
+// CHECK:  %[[CAST:.*]] = llvm.bitcast %[[ARG0]] : f16 to i16
+// CHECK:  %[[ZEXT:.*]] = llvm.zext %[[CAST]] : i16 to i32
+// CHECK:  %[[PERM:.*]] = rocdl.permlane32.swap %[[ZEXT]], %[[ZEXT]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:  %[[RES:.*]] = llvm.extractvalue %[[PERM]][0] : !llvm.struct<(i32, i32)>
+// CHECK:  %[[TRUNC:.*]] = llvm.trunc %[[RES]] : i32 to i16
+// CHECK:  %[[RES_CAST:.*]] = llvm.bitcast %[[TRUNC]] : i16 to f16
+// CHECK:  return %[[RES_CAST]] : f16
+  %0 = amdgpu.permlane_swap %arg0 32 : f16
+  return %0 : f16
+}
+
+// CHECK-LABEL: func @test_permlane16_2xi32
+// CHECK-SAME: (%[[ARG0:.*]]: vector<2xi32>)
+func.func @test_permlane16_2xi32(%arg0 : vector<2xi32>) -> vector<2xi32> {
+// CHECK-DAG:  %[[POISON:.*]] = llvm.mlir.poison : vector<2xi32>
+// CHECK-DAG:  %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK-DAG:  %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:      %[[ELEM0:.*]] = llvm.extractelement %[[ARG0]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:      %[[ELEM1:.*]] = llvm.extractelement %[[ARG0]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:      %[[PERM0_TUPLE:.*]] = rocdl.permlane16.swap %[[ELEM0]], %[[ELEM0]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:      %[[PERM0:.*]] = llvm.extractvalue %[[PERM0_TUPLE]][0] : !llvm.struct<(i32, i32)>
+// CHECK:      %[[PERM1_TUPLE:.*]] = rocdl.permlane16.swap %[[ELEM1]], %[[ELEM1]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:      %[[PERM1:.*]] = llvm.extractvalue %[[PERM1_TUPLE]][0] : !llvm.struct<(i32, i32)>
+// CHECK:      %[[VEC_INSERT0:.*]] = llvm.insertelement %[[PERM0]], %[[POISON]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:      %[[VEC_INSERT1:.*]] = llvm.insertelement %[[PERM1]], %[[VEC_INSERT0]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:      return %[[VEC_INSERT1]] : vector<2xi32>
+  %0 = amdgpu.permlane_swap %arg0 16 : vector<2xi32>
+  return %0 : vector<2xi32>
+}
+
+// CHECK-LABEL: func @test_permlane32_2xi32
+// CHECK-SAME: (%[[ARG0:.*]]: vector<2xi32>)
+func.func @test_permlane32_2xi32(%arg0 : vector<2xi32>) -> vector<2xi32> {
+// CHECK-DAG:  %[[POISON:.*]] = llvm.mlir.poison : vector<2xi32>
+// CHECK-DAG:  %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK-DAG:  %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:      %[[ELEM0:.*]] = llvm.extractelement %[[ARG0]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:      %[[ELEM1:.*]] = llvm.extractelement %[[ARG0]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:      %[[PERM0_TUPLE:.*]] = rocdl.permlane32.swap %[[ELEM0]], %[[ELEM0]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:      %[[PERM0:.*]] = llvm.extractvalue %[[PERM0_TUPLE]][0] : !llvm.struct<(i32, i32)>
+// CHECK:      %[[PERM1_TUPLE:.*]] = rocdl.permlane32.swap %[[ELEM1]], %[[ELEM1]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:      %[[PERM1:.*]] = llvm.extractvalue %[[PERM1_TUPLE]][0] : !llvm.struct<(i32, i32)>
+// CHECK:      %[[VEC_INSERT0:.*]] = llvm.insertelement %[[PERM0]], %[[POISON]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:      %[[VEC_INSERT1:.*]] = llvm.insertelement %[[PERM1]], %[[VEC_INSERT0]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:      return %[[VEC_INSERT1]] : vector<2xi32>
+  %0 = amdgpu.permlane_swap %arg0 32 : vector<2xi32>
+  return %0 : vector<2xi32>
+}
+
+// CHECK-LABEL: func @test_permlane16_4xf16
+// CHECK-SAME: (%[[ARG0:.*]]: vector<4xf16>)
+func.func @test_permlane16_4xf16(%arg0 : vector<4xf16>) -> vector<4xf16> {
+// CHECK-DAG:  %[[POISON:.*]] = llvm.mlir.poison : vector<2xi32>
+// CHECK-DAG:  %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK-DAG:  %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:      %[[CAST1:.*]] = llvm.bitcast %[[ARG0]] : vector<4xf16> to vector<2xi32>
+// CHECK:      %[[ELEM0:.*]] = llvm.extractelement %[[CAST1]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:      %[[ELEM1:.*]] = llvm.extractelement %[[CAST1]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:      %[[PERM0_TUPLE:.*]] = rocdl.permlane16.swap %[[ELEM0]], %[[ELEM0]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:      %[[PERM0:.*]] = llvm.extractvalue %[[PERM0_TUPLE]][0] : !llvm.struct<(i32, i32)>
+// CHECK:      %[[PERM1_TUPLE:.*]] = rocdl.permlane16.swap %[[ELEM1]], %[[ELEM1]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:      %[[PERM1:.*]] = llvm.extractvalue %[[PERM1_TUPLE]][0] : !llvm.struct<(i32, i32)>
+// CHECK:      %[[VEC_INSERT0:.*]] = llvm.insertelement %[[PERM0]], %[[POISON]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:      %[[VEC_INSERT1:.*]] = llvm.insertelement %[[PERM1]], %[[VEC_INSERT0]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:      %[[CAST2:.*]] = llvm.bitcast %[[VEC_INSERT1]] : vector<2xi32> to vector<4xf16>
+// CHECK:      return %[[CAST2]] : vector<4xf16>
+  %0 = amdgpu.permlane_swap %arg0 16 : vector<4xf16>
+  return %0 : vector<4xf16>
+}
+
+// CHECK-LABEL: func @test_permlane32_4xf16
+// CHECK-SAME: (%[[ARG0:.*]]: vector<4xf16>)
+func.func @test_permlane32_4xf16(%arg0 : vector<4xf16>) -> vector<4xf16> {
+// CHECK-DAG:  %[[POISON:.*]] = llvm.mlir.poison : vector<2xi32>
+// CHECK-DAG:  %[[C1:.*]] = llvm.mlir.constant(1 : i32) : i32
+// CHECK-DAG:  %[[C0:.*]] = llvm.mlir.constant(0 : i32) : i32
+// CHECK:      %[[CAST1:.*]] = llvm.bitcast %[[ARG0]] : vector<4xf16> to vector<2xi32>
+// CHECK:      %[[ELEM0:.*]] = llvm.extractelement %[[CAST1]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:      %[[ELEM1:.*]] = llvm.extractelement %[[CAST1]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:      %[[PERM0_TUPLE:.*]] = rocdl.permlane32.swap %[[ELEM0]], %[[ELEM0]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:      %[[PERM0:.*]] = llvm.extractvalue %[[PERM0_TUPLE]][0] : !llvm.struct<(i32, i32)>
+// CHECK:      %[[PERM1_TUPLE:.*]] = rocdl.permlane32.swap %[[ELEM1]], %[[ELEM1]], false, false : (i32, i32) -> <(i32, i32)>
+// CHECK:      %[[PERM1:.*]] = llvm.extractvalue %[[PERM1_TUPLE]][0] : !llvm.struct<(i32, i32)>
+// CHECK:      %[[VEC_INSERT0:.*]] = llvm.insertelement %[[PERM0]], %[[POISON]][%[[C0]] : i32] : vector<2xi32>
+// CHECK:      %[[VEC_INSERT1:.*]] = llvm.insertelement %[[PERM1]], %[[VEC_INSERT0]][%[[C1]] : i32] : vector<2xi32>
+// CHECK:      %[[CAST2:.*]] = llvm.bitcast %[[VEC_INSERT1]] : vector<2xi32> to vector<4xf16>
+// CHECK:      return %[[CAST2]] : vector<4xf16>
+  %0 = amdgpu.permlane_swap %arg0 32 : vector<4xf16>
+  return %0 : vector<4xf16>
+}
diff --git a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
index 83bdbe1f67118..ba12ff29ebef9 100644
--- a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
+++ b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
@@ -3,6 +3,7 @@
 // Same below, but using the `ConvertToLLVMPatternInterface` entry point
 // and the generic `convert-to-llvm` pass.
 // RUN: mlir-opt --convert-to-llvm="filter-dialects=arith" --split-input-file %s | FileCheck %s
+// RUN: mlir-opt --convert-to-llvm="filter-dialects=arith allow-pattern-rollback=0" --split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @vector_ops
 func.func @vector_ops(%arg0: vector<4xf32>, %arg1: vector<4xi1>, %arg2: vector<4xi64>, %arg3: vector<4xi64>) -> vector<4xf32> {
@@ -373,12 +374,11 @@ func.func @integer_extension_and_truncation(%arg0 : i3) {
 
 // CHECK-LABEL: @integer_cast_0d_vector
 func.func @integer_cast_0d_vector(%arg0 : vector<i3>) {
-// CHECK: %[[ARG0:.*]] = builtin.unrealized_conversion_cast
-// CHECK-NEXT: = llvm.sext %[[ARG0]] : vector<1xi3> to vector<1xi6>
+// CHECK: = llvm.sext %{{.*}}: vector<1xi3> to vector<1xi6>
   %0 = arith.extsi %arg0 : vector<i3> to vector<i6>
-// CHECK-NEXT: = llvm.zext %[[ARG0]] : vector<1xi3> to vector<1xi6>
+// CHECK-NEXT: = llvm.zext %{{.*}} : vector<1xi3> to vector<1xi6>
   %1 = arith.extui %arg0 : vector<i3> to vector<i6>
-// CHECK-NEXT: = llvm.trunc %[[ARG0]] : vector<1xi3> to vector<1xi2>
+// CHECK-NEXT: = llvm.trunc %{{.*}} : vector<1xi3> to vector<1xi2>
   %2 = arith.trunci %arg0 : vector<i3> to vector<i2>
   return
 }
diff --git a/mlir/test/Conversion/ArithToLLVM/type-conversion.mlir b/mlir/test/Conversion/ArithToLLVM/type-conversion.mlir
new file mode 100644
index 0000000000000..e3a0c82a628ba
--- /dev/null
+++ b/mlir/test/Conversion/ArithToLLVM/type-conversion.mlir
@@ -0,0 +1,15 @@
+// RUN: mlir-opt %s -test-llvm-legalize-patterns -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -test-llvm-legalize-patterns="allow-pattern-rollback=0" -split-input-file | FileCheck %s
+
+// CHECK-LABEL: llvm.func @arith_select(
+//  CHECK-SAME:     %[[arg0:.*]]: i1, %[[arg1:.*]]: i18, %[[arg2:.*]]: i18, %[[arg3:.*]]: i18, %[[arg4:.*]]: i18) -> !llvm.struct<(i18, i18)>
+//       CHECK:   %[[select0:.*]] = llvm.select %[[arg0]], %[[arg1]], %[[arg3]] : i1, i18
+//       CHECK:   %[[select1:.*]] = llvm.select %[[arg0]], %[[arg2]], %[[arg4]] : i1, i18
+//       CHECK:   %[[i0:.*]] = llvm.mlir.poison : !llvm.struct<(i18, i18)>
+//       CHECK:   %[[i1:.*]] = llvm.insertvalue %[[select0]], %[[i0]][0] : !llvm.struct<(i18, i18)>
+//       CHECK:   %[[i2:.*]] = llvm.insertvalue %[[select1]], %[[i1]][1] : !llvm.struct<(i18, i18)>
+//       CHECK:   llvm.return %[[i2]]
+func.func @arith_select(%arg0: i1, %arg1: i17, %arg2: i17) -> (i17) {
+  %0 = arith.select %arg0, %arg1, %arg2 : i17
+  return %0 : i17
+}
diff --git a/mlir/test/Conversion/ComplexToLLVM/convert-to-llvm.mlir b/mlir/test/Conversion/ComplexToLLVM/convert-to-llvm.mlir
index ad1b6658fbe78..4d2c12a56eaca 100644
--- a/mlir/test/Conversion/ComplexToLLVM/convert-to-llvm.mlir
+++ b/mlir/test/Conversion/ComplexToLLVM/convert-to-llvm.mlir
@@ -3,6 +3,7 @@
 // Same below, but using the `ConvertToLLVMPatternInterface` entry point
 // and the generic `convert-to-llvm` pass.
 // RUN: mlir-opt --convert-to-llvm="filter-dialects=complex" --split-input-file %s | FileCheck %s
+// RUN: mlir-opt --convert-to-llvm="filter-dialects=complex allow-pattern-rollback=0" --split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: func @complex_create
 // CHECK-SAME:    (%[[REAL0:.*]]: f32, %[[IMAG0:.*]]: f32)
@@ -23,9 +24,9 @@ func.func @complex_constant() -> complex<f64> {
 
 // CHECK-LABEL: func @complex_extract
 // CHECK-SAME:    (%[[CPLX:.*]]: complex<f32>)
-// CHECK-NEXT:    %[[CAST0:.*]] = builtin.unrealized_conversion_cast %[[CPLX]] : complex<f32> to !llvm.struct<(f32, f32)>
-// CHECK-NEXT:    %[[REAL:.*]] = llvm.extractvalue %[[CAST0]][0] : !llvm.struct<(f32, f32)>
-// CHECK-NEXT:    %[[IMAG:.*]] = llvm.extractvalue %[[CAST0]][1] : !llvm.struct<(f32, f32)>
+// CHECK:    builtin.unrealized_conversion_cast %[[CPLX]] : complex<f32> to !llvm.struct<(f32, f32)>
+// CHECK:    %[[REAL:.*]] = llvm.extractvalue %{{.*}}[0] : !llvm.struct<(f32, f32)>
+// CHECK:    %[[IMAG:.*]] = llvm.extractvalue %{{.*}}[1] : !llvm.struct<(f32, f32)>
 func.func @complex_extract(%cplx: complex<f32>) {
   %real1 = complex.re %cplx : complex<f32>
   %imag1 = complex.im %cplx : complex<f32>
diff --git a/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
index ae59f28b46392..080ba4f0ff67b 100644
--- a/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
+++ b/mlir/test/Conversion/ComplexToROCDLLibraryCalls/complex-to-rocdl-library-calls.mlir
@@ -1,19 +1,13 @@
-// RUN: mlir-opt %s -convert-complex-to-rocdl-library-calls | FileCheck %s
+// RUN: mlir-opt %s --allow-unregistered-dialect -convert-complex-to-rocdl-library-calls | FileCheck %s
 
 // CHECK-DAG: @__ocml_cabs_f32(complex<f32>) -> f32
 // CHECK-DAG: @__ocml_cabs_f64(complex<f64>) -> f64
-// CHECK-DAG: @__ocml_carg_f32(complex<f32>) -> f32
-// CHECK-DAG: @__ocml_carg_f64(complex<f64>) -> f64
 // CHECK-DAG: @__ocml_ccos_f32(complex<f32>) -> complex<f32>
 // CHECK-DAG: @__ocml_ccos_f64(complex<f64>) -> complex<f64>
 // CHECK-DAG: @__ocml_cexp_f32(complex<f32>) -> complex<f32>
 // CHECK-DAG: @__ocml_cexp_f64(complex<f64>) -> complex<f64>
 // CHECK-DAG: @__ocml_clog_f32(complex<f32>) -> complex<f32>
 // CHECK-DAG: @__ocml_clog_f64(complex<f64>) -> complex<f64>
-// CHECK-DAG: @__ocml_conj_f32(complex<f32>) -> complex<f32>
-// CHECK-DAG: @__ocml_conj_f64(complex<f64>) -> complex<f64>
-// CHECK-DAG: @__ocml_cpow_f32(complex<f32>, complex<f32>) -> complex<f32>
-// CHECK-DAG: @__ocml_cpow_f64(complex<f64>, complex<f64>) -> complex<f64>
 // CHECK-DAG: @__ocml_csin_f32(complex<f32>) -> complex<f32>
 // CHECK-DAG: @__ocml_csin_f64(complex<f64>) -> complex<f64>
 // CHECK-DAG: @__ocml_csqrt_f32(complex<f32>) -> complex<f32>
@@ -33,16 +27,6 @@ func.func @abs_caller(%f: complex<f32>, %d: complex<f64>) -> (f32, f64) {
   return %rf, %rd : f32, f64
 }
 
-//CHECK-LABEL: @angle_caller
-func.func @angle_caller(%f: complex<f32>, %d: complex<f64>) -> (f32, f64) {
-  // CHECK: %[[AF:.*]] = call @__ocml_carg_f32(%{{.*}})
-  %af = complex.angle %f : complex<f32>
-  // CHECK: %[[AD:.*]] = call @__ocml_carg_f64(%{{.*}})
-  %ad = complex.angle %d : complex<f64>
-  // CHECK: return %[[AF]], %[[AD]]
-  return %af, %ad : f32, f64
-}
-
 //CHECK-LABEL: @cos_caller
 func.func @cos_caller(%f: complex<f32>, %d: complex<f64>) -> (complex<f32>, complex<f64>) {
   // CHECK: %[[CF:.*]] = call @__ocml_ccos_f32(%{{.*}})
@@ -73,24 +57,15 @@ func.func @log_caller(%f: complex<f32>, %d: complex<f64>) -> (complex<f32>, comp
   return %lf, %ld : complex<f32>, complex<f64>
 }
 
-//CHECK-LABEL: @conj_caller
-func.func @conj_caller(%f: complex<f32>, %d: complex<f64>) -> (complex<f32>, complex<f64>) {
-  // CHECK: %[[CF:.*]] = call @__ocml_conj_f32(%{{.*}})
-  %cf2 = complex.conj %f : complex<f32>
-  // CHECK: %[[CD:.*]] = call @__ocml_conj_f64(%{{.*}})
-  %cd2 = complex.conj %d : complex<f64>
-  // CHECK: return %[[CF]], %[[CD]]
-  return %cf2, %cd2 : complex<f32>, complex<f64>
-}
-
 //CHECK-LABEL: @pow_caller
-func.func @pow_caller(%f: complex<f32>, %d: complex<f64>) -> (complex<f32>, complex<f64>) {
-  // CHECK: %[[PF:.*]] = call @__ocml_cpow_f32(%{{.*}}, %{{.*}})
-  %pf = complex.pow %f, %f : complex<f32>
-  // CHECK: %[[PD:.*]] = call @__ocml_cpow_f64(%{{.*}}, %{{.*}})
-  %pd = complex.pow %d, %d : complex<f64>
-  // CHECK: return %[[PF]], %[[PD]]
-  return %pf, %pd : complex<f32>, complex<f64>
+//CHECK:          (%[[Z:.*]]: complex<f32>, %[[W:.*]]: complex<f32>)
+func.func @pow_caller(%z: complex<f32>, %w: complex<f32>) -> complex<f32> {
+  // CHECK: %[[LOG:.*]] = call @__ocml_clog_f32(%[[Z]])
+  // CHECK: %[[MUL:.*]] = complex.mul %[[W]], %[[LOG]]
+  // CHECK: %[[EXP:.*]] = call @__ocml_cexp_f32(%[[MUL]])
+  // CHECK: return %[[EXP]]
+  %r = complex.pow %z, %w : complex<f32>
+  return %r : complex<f32>
 }
 
 //CHECK-LABEL: @sin_caller
diff --git a/mlir/test/Conversion/ControlFlowToLLVM/assert.mlir b/mlir/test/Conversion/ControlFlowToLLVM/assert.mlir
index 3ec8f1fa1e567..18d0526ecf1a9 100644
--- a/mlir/test/Conversion/ControlFlowToLLVM/assert.mlir
+++ b/mlir/test/Conversion/ControlFlowToLLVM/assert.mlir
@@ -3,6 +3,7 @@
 // Same below, but using the `ConvertToLLVMPatternInterface` entry point
 // and the generic `convert-to-llvm` pass.
 // RUN: mlir-opt --convert-to-llvm="filter-dialects=cf" --split-input-file %s | FileCheck %s
+// RUN: mlir-opt --convert-to-llvm="filter-dialects=cf allow-pattern-rollback=0" --split-input-file %s | FileCheck %s
 
 func.func @main() {
   %a = arith.constant 0 : i1
diff --git a/mlir/test/Conversion/ConvertToSPIRV/vector-unroll.mlir b/mlir/test/Conversion/ConvertToSPIRV/vector-unroll.mlir
index d68ba44ee8840..c85f4334ff2e5 100644
--- a/mlir/test/Conversion/ConvertToSPIRV/vector-unroll.mlir
+++ b/mlir/test/Conversion/ConvertToSPIRV/vector-unroll.mlir
@@ -83,20 +83,16 @@ func.func @vaddi_reduction(%arg0 : vector<8xi32>, %arg1 : vector<8xi32>) -> (i32
 // CHECK-LABEL: @transpose
 // CHECK-SAME: (%[[ARG0:.+]]: vector<3xi32>, %[[ARG1:.+]]: vector<3xi32>)
 func.func @transpose(%arg0 : vector<2x3xi32>) -> (vector<3x2xi32>) {
-  // CHECK: %[[UB:.*]] = ub.poison : vector<2xi32>
   // CHECK: %[[EXTRACT0:.*]] = vector.extract %[[ARG0]][0] : i32 from vector<3xi32>
-  // CHECK: %[[INSERT0:.*]]= vector.insert %[[EXTRACT0]], %[[UB]] [0] : i32 into vector<2xi32>
   // CHECK: %[[EXTRACT1:.*]] = vector.extract %[[ARG1]][0] : i32 from vector<3xi32>
-  // CHECK: %[[INSERT1:.*]] = vector.insert %[[EXTRACT1]], %[[INSERT0]][1] : i32 into vector<2xi32>
+  // CHECK: %[[FROM_ELEMENTS0:.*]] = vector.from_elements %[[EXTRACT0]], %[[EXTRACT1]] : vector<2xi32>
   // CHECK: %[[EXTRACT2:.*]] = vector.extract %[[ARG0]][1] : i32 from vector<3xi32>
-  // CHECK: %[[INSERT2:.*]] = vector.insert %[[EXTRACT2]], %[[UB]] [0] : i32 into vector<2xi32>
   // CHECK: %[[EXTRACT3:.*]] = vector.extract %[[ARG1]][1] : i32 from vector<3xi32>
-  // CHECK: %[[INSERT3:.*]] = vector.insert %[[EXTRACT3]], %[[INSERT2]] [1] : i32 into vector<2xi32>
+  // CHECK: %[[FROM_ELEMENTS1:.*]] = vector.from_elements %[[EXTRACT2]], %[[EXTRACT3]] : vector<2xi32>
   // CHECK: %[[EXTRACT4:.*]] = vector.extract %[[ARG0]][2] : i32 from vector<3xi32>
-  // CHECK: %[[INSERT4:.*]] = vector.insert %[[EXTRACT4]], %[[UB]] [0] : i32 into vector<2xi32>
   // CHECK: %[[EXTRACT5:.*]] = vector.extract %[[ARG1]][2] : i32 from vector<3xi32>
-  // CHECK: %[[INSERT5:.*]] = vector.insert %[[EXTRACT5]], %[[INSERT4]] [1] : i32 into vector<2xi32>
-  // CHECK: return %[[INSERT1]], %[[INSERT3]], %[[INSERT5]] : vector<2xi32>, vector<2xi32>, vector<2xi32>
+  // CHECK: %[[FROM_ELEMENTS2:.*]] = vector.from_elements %[[EXTRACT4]], %[[EXTRACT5]] : vector<2xi32>
+  // CHECK: return %[[FROM_ELEMENTS0]], %[[FROM_ELEMENTS1]], %[[FROM_ELEMENTS2]] : vector<2xi32>, vector<2xi32>, vector<2xi32>
   %0 = vector.transpose %arg0, [1, 0] : vector<2x3xi32> to vector<3x2xi32>
   return %0 : vector<3x2xi32>
 }
diff --git a/mlir/test/Conversion/FuncToLLVM/func-to-llvm.mlir b/mlir/test/Conversion/FuncToLLVM/func-to-llvm.mlir
index 2113557fbbb15..94dfceadbc449 100644
--- a/mlir/test/Conversion/FuncToLLVM/func-to-llvm.mlir
+++ b/mlir/test/Conversion/FuncToLLVM/func-to-llvm.mlir
@@ -9,6 +9,7 @@
 // Same below, but using the `ConvertToLLVMPatternInterface` entry point
 // and the generic `convert-to-llvm` pass.
 // RUN: mlir-opt --convert-to-llvm="filter-dialects=arith,cf,func,math" %s | FileCheck %s
+// RUN: mlir-opt --convert-to-llvm="filter-dialects=arith,cf,func,math allow-pattern-rollback=0" %s | FileCheck %s
 
 // CHECK-LABEL: func @empty() {
 // CHECK-NEXT:  llvm.return
diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm-target-attr.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm-target-attr.mlir
index ed7fa6508d5ad..0016db5084584 100644
--- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm-target-attr.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm-target-attr.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt %s --pass-pipeline="builtin.module(gpu.module(convert-to-llvm{dynamic=true}))" | FileCheck %s
+// RUN: mlir-opt %s --pass-pipeline="builtin.module(gpu.module(convert-to-llvm{dynamic=true allow-pattern-rollback=0}))" | FileCheck %s
 
 // CHECK-LABEL: gpu.module @nvvm_module
 gpu.module @nvvm_module [#nvvm.target] {
diff --git a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
index 2b6adffc81f72..71c3e9974611e 100644
--- a/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
+++ b/mlir/test/Conversion/GPUToROCDL/gpu-to-rocdl.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt %s -convert-gpu-to-rocdl -split-input-file | FileCheck %s
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='allowed-dialects=func,arith,math' -split-input-file | FileCheck %s
-// RUN: mlir-opt %s -convert-gpu-to-rocdl='index-bitwidth=32' -split-input-file | FileCheck --check-prefix=CHECK32 %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx950' -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx950 allowed-dialects=func,arith,math' -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -convert-gpu-to-rocdl='chipset=gfx950 index-bitwidth=32' -split-input-file | FileCheck --check-prefix=CHECK32 %s
 
 // CHECK-LABEL: @test_module
 // CHECK-SAME: llvm.data_layout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9"
@@ -54,8 +54,8 @@ gpu.module @test_module {
     // CHECK: = llvm.sext %{{.*}} : i32 to i64
     %gDimZ = gpu.grid_dim z
 
-    // CHECK: = rocdl.mbcnt.lo %{{.*}}, %{{.*}} : (i32, i32) -> i32
-    // CHECK: = rocdl.mbcnt.hi %{{.*}}, %{{.*}} : (i32, i32) -> i32
+    // CHECK: = rocdl.mbcnt.lo %{{.*}}, %{{.*}} {res_attrs = [{llvm.noundef, llvm.range = #llvm.constant_range<i32, 0, 32>}]} : (i32, i32) -> i32
+    // CHECK: = rocdl.mbcnt.hi %{{.*}}, %{{.*}} {res_attrs = [{llvm.noundef, llvm.range = #llvm.constant_range<i32, 0, 64>}]} : (i32, i32) -> i32
     // CHECK: = llvm.sext %{{.*}} : i32 to i64
     %laneId = gpu.lane_id
 
@@ -701,7 +701,7 @@ gpu.module @test_module {
     // CHECK: %[[#CAST_VALUE:]] = llvm.bitcast %[[#VALUE]] : f32 to i32
     // CHECK: %[[#PERMUTE:]] = rocdl.ds_bpermute %[[#ALIGNED_DST_LANE]], %[[#CAST_VALUE]] : (i32, i32) -> i32
     // CHECK: %[[#CAST_SHFL_VALUE:]] = llvm.bitcast %[[#PERMUTE]] : i32 to f32
-    %shfli, %predi = gpu.shuffle idx %arg0, %arg1, %arg2 : f32 
+    %shfli, %predi = gpu.shuffle idx %arg0, %arg1, %arg2 : f32
     // *** UP mode shuffle ***
     // CHECK: %[[#LANE_ID:]] = rocdl.mbcnt.hi
     // CHECK: %[[#ZERO:]] = llvm.mlir.constant(0 : i32) : i32
@@ -734,14 +734,40 @@ gpu.module @test_module {
     func.return %shfl, %shfli, %shflu, %shfld : f32, f32, f32, f32
   }
 
+  // CHECK-LABEL: func @gpu_shuffle_promote()
+  func.func @gpu_shuffle_promote() -> (f32, f32, f32) {
+    // CHECK: %[[#VALUE:]] = llvm.mlir.constant(1.000000e+00 : f32) : f32
+    %arg0 = arith.constant 1.0 : f32
+    %arg1 = arith.constant 4 : i32
+    %arg2 = arith.constant 16 : i32
+    %arg3 = arith.constant 32 : i32
+    %arg4 = arith.constant 64 : i32
+    // CHECK: %[[#CAST_VALUE:]] = llvm.bitcast %[[#VALUE]] : f32 to i32
+    // CHECK: %[[#MASK:]] = llvm.mlir.constant(4127 : i32) : i32
+    // CHECK: %[[#PERMUTE:]] = rocdl.ds_swizzle %[[#CAST_VALUE]], %[[#MASK]] : (i32, i32) -> i32
+    // CHECK: %[[#CAST_SHFL_VALUE:]] = llvm.bitcast %[[#PERMUTE]] : i32 to f32
+    %shfl1, %pred1 = gpu.shuffle xor %arg0, %arg1, %arg4 : f32
+    // CHECK: %[[#CAST_VALUE:]] = llvm.bitcast %[[#VALUE]] : f32 to i32
+    // CHECK: %[[#PERMUTE:]] = rocdl.permlane16.swap %[[#CAST_VALUE]], %[[#CAST_VALUE]], false, false : (i32, i32) -> <(i32, i32)>
+    // CHECK: %[[#EXTRACT:]] = llvm.extractvalue %[[#PERMUTE:]][0] : !llvm.struct<(i32, i32)>
+    // CHECK: %[[#CAST_SHFL_VALUE:]] = llvm.bitcast %[[#EXTRACT]] : i32 to f32
+    %shfl2, %pred2 = gpu.shuffle xor %arg0, %arg2, %arg4 : f32
+    // CHECK: %[[#CAST_VALUE:]] = llvm.bitcast %[[#VALUE]] : f32 to i32
+    // CHECK: %[[#PERMUTE:]] = rocdl.permlane32.swap %[[#CAST_VALUE]], %[[#CAST_VALUE]], false, false : (i32, i32) -> <(i32, i32)>
+    // CHECK: %[[#EXTRACT:]] = llvm.extractvalue %[[#PERMUTE:]][0] : !llvm.struct<(i32, i32)>
+    // CHECK: %[[#CAST_SHFL_VALUE:]] = llvm.bitcast %[[#EXTRACT]] : i32 to f32
+    %shfl3, %pred3 = gpu.shuffle xor  %arg0, %arg3, %arg4 : f32
+    func.return %shfl1, %shfl2, %shfl3 : f32, f32, f32
+  }
+
   // CHECK-LABEL: func @gpu_shuffle_vec
   //  CHECK-SAME: (%[[ARG:.*]]: vector<4xf16>, %{{.*}}: i32, %{{.*}}: i32)
   func.func @gpu_shuffle_vec(%arg0: vector<4xf16>, %arg1: i32, %arg2: i32) -> vector<4xf16> {
     // CHECK: %[[CAST1:.*]] = llvm.bitcast %[[ARG]] : vector<4xf16> to vector<2xi32>
     // CHECK: %[[IDX0:.*]] = llvm.mlir.constant(0 : i32) : i32
-    // CHECK: %[[ELEM0:.*]] = llvm.extractelement %13[%[[IDX0]] : i32] : vector<2xi32>
+    // CHECK: %[[ELEM0:.*]] = llvm.extractelement %[[CAST1]][%[[IDX0]] : i32] : vector<2xi32>
     // CHECK: %[[IDX1:.*]] = llvm.mlir.constant(1 : i32) : i32
-    // CHECK: %[[ELEM1:.*]] = llvm.extractelement %13[%[[IDX1]] : i32] : vector<2xi32>
+    // CHECK: %[[ELEM1:.*]] = llvm.extractelement %[[CAST1]][%[[IDX1]] : i32] : vector<2xi32>
     // CHECK: %[[PERM0:.*]] = rocdl.ds_bpermute %{{.*}}, %[[ELEM0]] : (i32, i32) -> i32
     // CHECK: %[[PERM1:.*]] = rocdl.ds_bpermute %{{.*}}, %[[ELEM1]] : (i32, i32) -> i32
     // CHECK: %[[V0:.*]] = llvm.mlir.poison : vector<2xi32>
diff --git a/mlir/test/Conversion/IndexToLLVM/index-to-llvm.mlir b/mlir/test/Conversion/IndexToLLVM/index-to-llvm.mlir
index 26abb3bdc23a1..007929ed677fa 100644
--- a/mlir/test/Conversion/IndexToLLVM/index-to-llvm.mlir
+++ b/mlir/test/Conversion/IndexToLLVM/index-to-llvm.mlir
@@ -5,6 +5,7 @@
 // Same below, but using the `ConvertToLLVMPatternInterface` entry point
 // and the generic `convert-to-llvm` pass.
 // RUN: mlir-opt --convert-to-llvm="filter-dialects=index" --split-input-file %s | FileCheck %s
+// RUN: mlir-opt --convert-to-llvm="filter-dialects=index allow-pattern-rollback=0" --split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @trivial_ops
 func.func @trivial_ops(%a: index, %b: index) {
diff --git a/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir b/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir
index 92904082a6f46..f4541220fe4d2 100644
--- a/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir
+++ b/mlir/test/Conversion/MathToLLVM/math-to-llvm.mlir
@@ -3,6 +3,7 @@
 // Same below, but using the `ConvertToLLVMPatternInterface` entry point
 // and the generic `convert-to-llvm` pass.
 // RUN: mlir-opt --convert-to-llvm="filter-dialects=math" --split-input-file %s | FileCheck %s
+// RUN: mlir-opt --convert-to-llvm="filter-dialects=math allow-pattern-rollback=0" --split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @ops
 func.func @ops(%arg0: f32, %arg1: f32, %arg2: i32, %arg3: i32, %arg4: f64) {
diff --git a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir
index 08354dbf280c1..26b54566cb2cd 100644
--- a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir
+++ b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir
@@ -79,21 +79,17 @@ func.func @absf_caller(%float: f32, %double: f64) -> (f32, f64)  {
 // CHECK-LABEL:   func @absf_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @fabsf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @fabsf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @fabs(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @fabs(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 func.func @absf_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
   %float_result = math.absf %float : vector<2xf32>
@@ -116,21 +112,17 @@ func.func @acos_caller(%float: f32, %double: f64) -> (f32, f64)  {
 // CHECK-LABEL:   func @acos_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @acosf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @acosf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @acos(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @acos(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 func.func @acos_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
   %float_result = math.acos %float : vector<2xf32>
@@ -153,21 +145,17 @@ func.func @acosh_caller(%float: f32, %double: f64) -> (f32, f64)  {
 // CHECK-LABEL:   func @acosh_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @acoshf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @acoshf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @acosh(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @acosh(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 func.func @acosh_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
   %float_result = math.acosh %float : vector<2xf32>
@@ -190,21 +178,17 @@ func.func @asin_caller(%float: f32, %double: f64) -> (f32, f64)  {
 // CHECK-LABEL:   func @asin_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @asinf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @asinf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @asin(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @asin(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 func.func @asin_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
   %float_result = math.asin %float : vector<2xf32>
@@ -227,21 +211,17 @@ func.func @asinh_caller(%float: f32, %double: f64) -> (f32, f64)  {
 // CHECK-LABEL:   func @asinh_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @asinhf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @asinhf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @asinh(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @asinh(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 func.func @asinh_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
   %float_result = math.asinh %float : vector<2xf32>
@@ -274,21 +254,17 @@ func.func @atan_caller(%float: f32, %double: f64, %half: f16, %bfloat: bf16) ->
 // CHECK-LABEL:   func @atan_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @atanf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @atanf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @atan(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @atan(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 func.func @atan_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
   %float_result = math.atan %float : vector<2xf32>
@@ -321,21 +297,17 @@ func.func @atanh_caller(%float: f32, %double: f64, %half: f16, %bfloat: bf16) ->
 // CHECK-LABEL:   func @atanh_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @atanhf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @atanhf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @atanh(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @atanh(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 func.func @atanh_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
   %float_result = math.atanh %float : vector<2xf32>
@@ -419,23 +391,19 @@ func.func @erf_caller(%float: f32, %double: f64) -> (f32, f64)  {
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
 func.func @erf_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-  // CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-  // CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
   // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
   // CHECK:           %[[OUT0_F32:.*]] = call @erff(%[[IN0_F32]]) : (f32) -> f32
-  // CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
   // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
   // CHECK:           %[[OUT1_F32:.*]] = call @erff(%[[IN1_F32]]) : (f32) -> f32
-  // CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+  // CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
   %float_result = math.erf %float : vector<2xf32>
   // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
   // CHECK:           %[[OUT0_F64:.*]] = call @erf(%[[IN0_F64]]) : (f64) -> f64
-  // CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
   // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
   // CHECK:           %[[OUT1_F64:.*]] = call @erf(%[[IN1_F64]]) : (f64) -> f64
-  // CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
+  // CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
   %double_result = math.erf %double : vector<2xf64>
-  // CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+  // CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
   return %float_result, %double_result : vector<2xf32>, vector<2xf64>
 }
 
@@ -459,21 +427,17 @@ func.func @exp_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vec
 // CHECK-LABEL:   func @exp_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @expf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @expf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @exp(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @exp(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 
 // CHECK-LABEL: func @exp2_caller
@@ -496,21 +460,17 @@ func.func @exp2_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (ve
 // CHECK-LABEL:   func @exp2_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @exp2f(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @exp2f(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @exp2(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @exp2(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 
 // CHECK-LABEL: func @log_caller
@@ -533,21 +493,17 @@ func.func @log_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vec
 // CHECK-LABEL:   func @log_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @logf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @logf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @log(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @log(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 
 // CHECK-LABEL: func @log2_caller
@@ -570,21 +526,17 @@ func.func @log2_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (ve
 // CHECK-LABEL:   func @log2_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @log2f(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @log2f(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @log2(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @log2(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 
 // CHECK-LABEL: func @log10_caller
@@ -607,21 +559,17 @@ func.func @log10_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (v
 // CHECK-LABEL:   func @log10_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @log10f(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @log10f(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @log10(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @log10(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 
 // CHECK-LABEL: func @expm1_caller
@@ -644,21 +592,17 @@ func.func @expm1_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (v
 // CHECK-LABEL:   func @expm1_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @expm1f(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @expm1f(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @expm1(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @expm1(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 
 func.func @expm1_multidim_vec_caller(%float: vector<2x2xf32>) -> (vector<2x2xf32>) {
@@ -667,20 +611,16 @@ func.func @expm1_multidim_vec_caller(%float: vector<2x2xf32>) -> (vector<2x2xf32
 }
 // CHECK-LABEL:   func @expm1_multidim_vec_caller(
 // CHECK-SAME:                           %[[VAL:.*]]: vector<2x2xf32>
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2x2xf32>
 // CHECK:           %[[IN0_0_F32:.*]] = vector.extract %[[VAL]][0, 0] : f32 from vector<2x2xf32>
 // CHECK:           %[[OUT0_0_F32:.*]] = call @expm1f(%[[IN0_0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_1:.*]] = vector.insert %[[OUT0_0_F32]], %[[CVF]] [0, 0] : f32 into vector<2x2xf32>
 // CHECK:           %[[IN0_1_F32:.*]] = vector.extract %[[VAL]][0, 1] : f32 from vector<2x2xf32>
 // CHECK:           %[[OUT0_1_F32:.*]] = call @expm1f(%[[IN0_1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_2:.*]] = vector.insert %[[OUT0_1_F32]], %[[VAL_1]] [0, 1] : f32 into vector<2x2xf32>
 // CHECK:           %[[IN1_0_F32:.*]] = vector.extract %[[VAL]][1, 0] : f32 from vector<2x2xf32>
 // CHECK:           %[[OUT1_0_F32:.*]] = call @expm1f(%[[IN1_0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_3:.*]] = vector.insert %[[OUT1_0_F32]], %[[VAL_2]] [1, 0] : f32 into vector<2x2xf32>
 // CHECK:           %[[IN1_1_F32:.*]] = vector.extract %[[VAL]][1, 1] : f32 from vector<2x2xf32>
 // CHECK:           %[[OUT1_1_F32:.*]] = call @expm1f(%[[IN1_1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_4:.*]] = vector.insert %[[OUT1_1_F32]], %[[VAL_3]] [1, 1] : f32 into vector<2x2xf32>
-// CHECK:           return %[[VAL_4]] : vector<2x2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_0_F32]], %[[OUT0_1_F32]], %[[OUT1_0_F32]], %[[OUT1_1_F32]] : vector<2x2xf32>
+// CHECK:           return %[[RES_F32]] : vector<2x2xf32>
 // CHECK:         }
 
 // CHECK-LABEL: func @fma_caller(
@@ -704,29 +644,25 @@ func.func @fma_vec_caller(%float_a: vector<2xf32>, %float_b: vector<2xf32>, %flo
 // CHECK-SAME:                           %[[VAL_0A:.*]]: vector<2xf32>, %[[VAL_0B:.*]]: vector<2xf32>, %[[VAL_0C:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1A:.*]]: vector<2xf64>, %[[VAL_1B:.*]]: vector<2xf64>, %[[VAL_1C:.*]]: vector<2xf64>
 // CHECK-SAME:                           ) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32A:.*]] = vector.extract %[[VAL_0A]][0] : f32 from vector<2xf32>
 // CHECK:           %[[IN0_F32B:.*]] = vector.extract %[[VAL_0B]][0] : f32 from vector<2xf32>
 // CHECK:           %[[IN0_F32C:.*]] = vector.extract %[[VAL_0C]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @fmaf(%[[IN0_F32A]], %[[IN0_F32B]], %[[IN0_F32C]]) : (f32, f32, f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32A:.*]] = vector.extract %[[VAL_0A]][1] : f32 from vector<2xf32>
 // CHECK:           %[[IN1_F32B:.*]] = vector.extract %[[VAL_0B]][1] : f32 from vector<2xf32>
 // CHECK:           %[[IN1_F32C:.*]] = vector.extract %[[VAL_0C]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @fmaf(%[[IN1_F32A]], %[[IN1_F32B]], %[[IN1_F32C]]) : (f32, f32, f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64A:.*]] = vector.extract %[[VAL_1A]][0] : f64 from vector<2xf64>
 // CHECK:           %[[IN0_F64B:.*]] = vector.extract %[[VAL_1B]][0] : f64 from vector<2xf64>
 // CHECK:           %[[IN0_F64C:.*]] = vector.extract %[[VAL_1C]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @fma(%[[IN0_F64A]], %[[IN0_F64B]], %[[IN0_F64C]]) : (f64, f64, f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64A:.*]] = vector.extract %[[VAL_1A]][1] : f64 from vector<2xf64>
 // CHECK:           %[[IN1_F64B:.*]] = vector.extract %[[VAL_1B]][1] : f64 from vector<2xf64>
 // CHECK:           %[[IN1_F64C:.*]] = vector.extract %[[VAL_1C]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @fma(%[[IN1_F64A]], %[[IN1_F64B]], %[[IN1_F64C]]) : (f64, f64, f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 
 // CHECK-LABEL: func @round_caller
@@ -814,23 +750,19 @@ func.func @sin_caller(%float: f32, %double: f64) -> (f32, f64)  {
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
 func.func @round_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-  // CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-  // CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
   // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
   // CHECK:           %[[OUT0_F32:.*]] = call @roundf(%[[IN0_F32]]) : (f32) -> f32
-  // CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
   // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
   // CHECK:           %[[OUT1_F32:.*]] = call @roundf(%[[IN1_F32]]) : (f32) -> f32
-  // CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+  // CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
   %float_result = math.round %float : vector<2xf32>
   // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
   // CHECK:           %[[OUT0_F64:.*]] = call @round(%[[IN0_F64]]) : (f64) -> f64
-  // CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
   // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
   // CHECK:           %[[OUT1_F64:.*]] = call @round(%[[IN1_F64]]) : (f64) -> f64
-  // CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
+  // CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
   %double_result = math.round %double : vector<2xf64>
-  // CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+  // CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
   return %float_result, %double_result : vector<2xf32>, vector<2xf64>
 }
 
@@ -838,23 +770,19 @@ func.func @round_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (v
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
 func.func @roundeven_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-  // CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-  // CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
   // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
   // CHECK:           %[[OUT0_F32:.*]] = call @roundevenf(%[[IN0_F32]]) : (f32) -> f32
-  // CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
   // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
   // CHECK:           %[[OUT1_F32:.*]] = call @roundevenf(%[[IN1_F32]]) : (f32) -> f32
-  // CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+  // CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
   %float_result = math.roundeven %float : vector<2xf32>
   // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
   // CHECK:           %[[OUT0_F64:.*]] = call @roundeven(%[[IN0_F64]]) : (f64) -> f64
-  // CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
   // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
   // CHECK:           %[[OUT1_F64:.*]] = call @roundeven(%[[IN1_F64]]) : (f64) -> f64
-  // CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
+  // CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
   %double_result = math.roundeven %double : vector<2xf64>
-  // CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+  // CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
   return %float_result, %double_result : vector<2xf32>, vector<2xf64>
 }
 
@@ -862,23 +790,19 @@ func.func @roundeven_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
 func.func @trunc_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-  // CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-  // CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
   // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
   // CHECK:           %[[OUT0_F32:.*]] = call @truncf(%[[IN0_F32]]) : (f32) -> f32
-  // CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
   // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
   // CHECK:           %[[OUT1_F32:.*]] = call @truncf(%[[IN1_F32]]) : (f32) -> f32
-  // CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+  // CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
   %float_result = math.trunc %float : vector<2xf32>
   // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
   // CHECK:           %[[OUT0_F64:.*]] = call @trunc(%[[IN0_F64]]) : (f64) -> f64
-  // CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
   // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
   // CHECK:           %[[OUT1_F64:.*]] = call @trunc(%[[IN1_F64]]) : (f64) -> f64
-  // CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
+  // CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
   %double_result = math.trunc %double : vector<2xf64>
-  // CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+  // CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
   return %float_result, %double_result : vector<2xf32>, vector<2xf64>
 }
 
@@ -907,21 +831,17 @@ func.func @tan_caller(%float: f32, %double: f64, %half: f16, %bfloat: bf16) -> (
 // CHECK-LABEL:   func @tan_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @tanf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @tanf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @tan(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @tan(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 func.func @tan_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
   %float_result = math.tan %float : vector<2xf32>
@@ -985,21 +905,17 @@ func.func @sqrt_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (ve
 // CHECK-LABEL:   func @sqrt_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @sqrtf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @sqrtf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @sqrt(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @sqrt(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 
 // CHECK-LABEL: func @rsqrt_caller
@@ -1022,21 +938,17 @@ func.func @rsqrt_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (v
 // CHECK-LABEL:   func @rsqrt_vec_caller(
 // CHECK-SAME:                           %[[VAL_0:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @rsqrtf(%[[IN0_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @rsqrtf(%[[IN1_F32]]) : (f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @rsqrt(%[[IN0_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @rsqrt(%[[IN1_F64]]) : (f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
 
 // CHECK-LABEL: func @powf_caller(
@@ -1060,23 +972,19 @@ func.func @powf_vec_caller(%float_a: vector<2xf32>, %float_b: vector<2xf32>, %do
 // CHECK-SAME:                           %[[VAL_0A:.*]]: vector<2xf32>, %[[VAL_0B:.*]]: vector<2xf32>,
 // CHECK-SAME:                           %[[VAL_1A:.*]]: vector<2xf64>, %[[VAL_1B:.*]]: vector<2xf64>
 // CHECK-SAME:                           ) -> (vector<2xf32>, vector<2xf64>) {
-// CHECK-DAG:       %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32>
-// CHECK-DAG:       %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64>
 // CHECK:           %[[IN0_F32A:.*]] = vector.extract %[[VAL_0A]][0] : f32 from vector<2xf32>
 // CHECK:           %[[IN0_F32B:.*]] = vector.extract %[[VAL_0B]][0] : f32 from vector<2xf32>
 // CHECK:           %[[OUT0_F32:.*]] = call @powf(%[[IN0_F32A]], %[[IN0_F32B]]) : (f32, f32) -> f32
-// CHECK:           %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32>
 // CHECK:           %[[IN1_F32A:.*]] = vector.extract %[[VAL_0A]][1] : f32 from vector<2xf32>
 // CHECK:           %[[IN1_F32B:.*]] = vector.extract %[[VAL_0B]][1] : f32 from vector<2xf32>
 // CHECK:           %[[OUT1_F32:.*]] = call @powf(%[[IN1_F32A]], %[[IN1_F32B]]) : (f32, f32) -> f32
-// CHECK:           %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32>
+// CHECK:           %[[RES_F32:.*]] = vector.from_elements %[[OUT0_F32]], %[[OUT1_F32]] : vector<2xf32>
 // CHECK:           %[[IN0_F64A:.*]] = vector.extract %[[VAL_1A]][0] : f64 from vector<2xf64>
 // CHECK:           %[[IN0_F64B:.*]] = vector.extract %[[VAL_1B]][0] : f64 from vector<2xf64>
 // CHECK:           %[[OUT0_F64:.*]] = call @pow(%[[IN0_F64A]], %[[IN0_F64B]]) : (f64, f64) -> f64
-// CHECK:           %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64>
 // CHECK:           %[[IN1_F64A:.*]] = vector.extract %[[VAL_1A]][1] : f64 from vector<2xf64>
 // CHECK:           %[[IN1_F64B:.*]] = vector.extract %[[VAL_1B]][1] : f64 from vector<2xf64>
 // CHECK:           %[[OUT1_F64:.*]] = call @pow(%[[IN1_F64A]], %[[IN1_F64B]]) : (f64, f64) -> f64
-// CHECK:           %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64>
-// CHECK:           return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64>
+// CHECK:           %[[RES_F64:.*]] = vector.from_elements %[[OUT0_F64]], %[[OUT1_F64]] : vector<2xf64>
+// CHECK:           return %[[RES_F32]], %[[RES_F64]] : vector<2xf32>, vector<2xf64>
 // CHECK:         }
diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-alloc-copy.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-alloc-copy.mlir
new file mode 100644
index 0000000000000..c1627a0d4d023
--- /dev/null
+++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-alloc-copy.mlir
@@ -0,0 +1,50 @@
+// RUN: mlir-opt -convert-memref-to-emitc="lower-to-cpp=true" %s -split-input-file | FileCheck %s --check-prefix=CPP
+// RUN: mlir-opt -convert-memref-to-emitc="lower-to-cpp=false" %s -split-input-file | FileCheck %s --check-prefix=NOCPP
+
+func.func @alloc_copy(%arg0: memref<999xi32>) {
+  %alloc = memref.alloc() : memref<999xi32>
+  memref.copy %arg0, %alloc : memref<999xi32> to memref<999xi32>
+  %alloc_1 = memref.alloc() : memref<999xi32>
+  memref.copy %arg0, %alloc_1 : memref<999xi32> to memref<999xi32>
+  return
+} 
+
+// CHECK: module {
+// NOCPP:  emitc.include <"stdlib.h">
+// NOCPP-NEXT:  emitc.include <"string.h">
+
+// CPP:  emitc.include <"cstdlib">
+// CPP-NEXT:  emitc.include <"cstring">
+
+// CHECK-LABEL: alloc_copy
+// CHECK-SAME: %[[arg0:.*]]: memref<999xi32>
+// CHECK-NEXT:  builtin.unrealized_conversion_cast %arg0 : memref<999xi32> to !emitc.array<999xi32> 
+// CHECK-NEXT:  emitc.call_opaque "sizeof"() {args = [i32]} : () -> !emitc.size_t 
+// CHECK-NEXT:  "emitc.constant"() <{value = 999 : index}> : () -> index 
+// CHECK-NEXT:  emitc.mul %1, %2 : (!emitc.size_t, index) -> !emitc.size_t 
+// CHECK-NEXT:  emitc.call_opaque "malloc"(%3) : (!emitc.size_t) -> !emitc.ptr<!emitc.opaque<"void">> 
+// CHECK-NEXT:  emitc.cast %4 : !emitc.ptr<!emitc.opaque<"void">> to !emitc.ptr<i32> 
+// CHECK-NEXT:  builtin.unrealized_conversion_cast %5 : !emitc.ptr<i32> to !emitc.array<999xi32> 
+// CHECK-NEXT:  "emitc.constant"() <{value = 0 : index}> : () -> index 
+// CHECK-NEXT:  emitc.subscript %0[%7] : (!emitc.array<999xi32>, index) -> !emitc.lvalue<i32> 
+// CHECK-NEXT:  emitc.apply "&"(%8) : (!emitc.lvalue<i32>) -> !emitc.ptr<i32> 
+// CHECK-NEXT:  emitc.call_opaque "sizeof"() {args = [i32]} : () -> !emitc.size_t
+// CHECK-NEXT:  "emitc.constant"() <{value = 999 : index}> : () -> index
+// CHECK-NEXT:  emitc.mul %12, %13 : (!emitc.size_t, index) -> !emitc.size_t
+// CHECK-NEXT:  emitc.call_opaque "memcpy"(%11, %9, %14) : (!emitc.ptr<i32>, !emitc.ptr<i32>, !emitc.size_t) -> ()
+// CHECK-NEXT:  emitc.call_opaque "sizeof"() {args = [i32]} : () -> !emitc.size_t
+// CHECK-NEXT:  "emitc.constant"() <{value = 999 : index}> : () -> index
+// CHECK-NEXT:  emitc.mul %15, %16 : (!emitc.size_t, index) -> !emitc.size_t
+// CHECK-NEXT:  emitc.call_opaque "malloc"(%17) : (!emitc.size_t) -> !emitc.ptr<!emitc.opaque<"void">>
+// CHECK-NEXT:  emitc.cast %18 : !emitc.ptr<!emitc.opaque<"void">> to !emitc.ptr<i32>
+// CHECK-NEXT:  builtin.unrealized_conversion_cast %19 : !emitc.ptr<i32> to !emitc.array<999xi32>
+// CHECK-NEXT:  "emitc.constant"() <{value = 0 : index}> : () -> index
+// CHECK-NEXT:  emitc.subscript %0[%21] : (!emitc.array<999xi32>, index) -> !emitc.lvalue<i32>
+// CHECK-NEXT:  emitc.apply "&"(%22) : (!emitc.lvalue<i32>) -> !emitc.ptr<i32>
+// CHECK-NEXT:  emitc.subscript %20[%21] : (!emitc.array<999xi32>, index) -> !emitc.lvalue<i32>
+// CHECK-NEXT:  emitc.apply "&"(%24) : (!emitc.lvalue<i32>) -> !emitc.ptr<i32>
+// CHECK-NEXT:  emitc.call_opaque "sizeof"() {args = [i32]} : () -> !emitc.size_t
+// CHECK-NEXT:  "emitc.constant"() <{value = 999 : index}> : () -> index
+// CHECK-NEXT:  emitc.mul %26, %27 : (!emitc.size_t, index) -> !emitc.size_t
+// CHECK-NEXT:  emitc.call_opaque "memcpy"(%25, %23, %28) : (!emitc.ptr<i32>, !emitc.ptr<i32>, !emitc.size_t) -> ()
+// CHECK-NEXT:    return
diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-copy.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-copy.mlir
new file mode 100644
index 0000000000000..d151d1bd53458
--- /dev/null
+++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-copy.mlir
@@ -0,0 +1,29 @@
+// RUN: mlir-opt -convert-memref-to-emitc="lower-to-cpp=true" %s -split-input-file | FileCheck %s --check-prefix=CPP
+// RUN: mlir-opt -convert-memref-to-emitc="lower-to-cpp=false" %s -split-input-file | FileCheck %s --check-prefix=NOCPP
+
+func.func @copying(%arg0 : memref<9x4x5x7xf32>, %arg1 : memref<9x4x5x7xf32>) {
+  memref.copy %arg0, %arg1 : memref<9x4x5x7xf32> to memref<9x4x5x7xf32>
+  return
+}
+
+// CHECK: module {
+// NOCPP:  emitc.include <"string.h">
+// CPP:  emitc.include <"cstring">
+
+// CHECK-LABEL:  copying
+// CHECK-SAME: %[[arg0:.*]]: memref<9x4x5x7xf32>, %[[arg1:.*]]: memref<9x4x5x7xf32>
+// CHECK-NEXT: %0 = builtin.unrealized_conversion_cast %arg1 : memref<9x4x5x7xf32> to !emitc.array<9x4x5x7xf32>
+// CHECK-NEXT: %1 = builtin.unrealized_conversion_cast %arg0 : memref<9x4x5x7xf32> to !emitc.array<9x4x5x7xf32>
+// CHECK-NEXT: %2 = "emitc.constant"() <{value = 0 : index}> : () -> index
+// CHECK-NEXT: %3 = emitc.subscript %1[%2, %2, %2, %2] : (!emitc.array<9x4x5x7xf32>, index, index, index, index) -> !emitc.lvalue<f32>
+// CHECK-NEXT: %4 = emitc.apply "&"(%3) : (!emitc.lvalue<f32>) -> !emitc.ptr<f32>
+// CHECK-NEXT: %5 = emitc.subscript %0[%2, %2, %2, %2] : (!emitc.array<9x4x5x7xf32>, index, index, index, index) -> !emitc.lvalue<f32>
+// CHECK-NEXT: %6 = emitc.apply "&"(%5) : (!emitc.lvalue<f32>) -> !emitc.ptr<f32>
+// CHECK-NEXT: %7 = emitc.call_opaque "sizeof"() {args = [f32]} : () -> !emitc.size_t
+// CHECK-NEXT: %8 = "emitc.constant"() <{value = 1260 : index}> : () -> index
+// CHECK-NEXT: %9 = emitc.mul %7, %8 : (!emitc.size_t, index) -> !emitc.size_t
+// CHECK-NEXT: emitc.call_opaque "memcpy"(%6, %4, %9) : (!emitc.ptr<f32>, !emitc.ptr<f32>, !emitc.size_t) -> ()
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+// CHECK-NEXT:}
+
diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir
index fda01974d3fc8..b6eccfc8f0050 100644
--- a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir
+++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc-failed.mlir
@@ -1,13 +1,5 @@
 // RUN: mlir-opt -convert-memref-to-emitc %s -split-input-file -verify-diagnostics
 
-func.func @memref_op(%arg0 : memref<2x4xf32>) {
-  // expected-error@+1 {{failed to legalize operation 'memref.copy'}}
-  memref.copy %arg0, %arg0 : memref<2x4xf32> to memref<2x4xf32>
-  return
-}
-
-// -----
-
 func.func @alloca_with_dynamic_shape() {
   %0 = index.constant 1
   // expected-error@+1 {{failed to legalize operation 'memref.alloca'}}
diff --git a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
index ad9d649590e66..45b1a1f1ca40c 100644
--- a/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/memref-to-llvm.mlir
@@ -464,7 +464,9 @@ func.func @atomic_rmw(%I : memref<10xi32>, %ival : i32, %F : memref<10xf32>, %fv
   // CHECK: llvm.atomicrmw _or %{{.*}}, %{{.*}} acq_rel
   memref.atomic_rmw andi %ival, %I[%i] : (i32, memref<10xi32>) -> i32
   // CHECK: llvm.atomicrmw _and %{{.*}}, %{{.*}} acq_rel
-  // CHECK-INTERFACE-COUNT-13: llvm.atomicrmw
+  memref.atomic_rmw xori %ival, %I[%i] : (i32, memref<10xi32>) -> i32
+  // CHECK: llvm.atomicrmw _xor %{{.*}}, %{{.*}} acq_rel
+  // CHECK-INTERFACE-COUNT-14: llvm.atomicrmw
   return
 }
 
diff --git a/mlir/test/Conversion/MemRefToLLVM/type-conversion.mlir b/mlir/test/Conversion/MemRefToLLVM/type-conversion.mlir
index 0288aa11313c7..6c6756f5097b4 100644
--- a/mlir/test/Conversion/MemRefToLLVM/type-conversion.mlir
+++ b/mlir/test/Conversion/MemRefToLLVM/type-conversion.mlir
@@ -1,12 +1,13 @@
-// RUN: mlir-opt %s -test-llvm-legalize-patterns -split-input-file
+// RUN: mlir-opt %s -test-llvm-legalize-patterns -split-input-file | FileCheck %s
+// RUN: mlir-opt %s -test-llvm-legalize-patterns="allow-pattern-rollback=0" -split-input-file | FileCheck %s
 
 // Test the argument materializer for ranked MemRef types.
 
 //   CHECK-LABEL: func @construct_ranked_memref_descriptor(
-//         CHECK:   llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
+//         CHECK:   llvm.mlir.poison : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)>
 // CHECK-COUNT-7:   llvm.insertvalue
 //         CHECK:   builtin.unrealized_conversion_cast %{{.*}} : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> to memref<5x4xf32>
-func.func @construct_ranked_memref_descriptor(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: i64, %arg3: i64, %arg4: i64, %arg5: i64, %arg6: i64) {
+func.func @construct_ranked_memref_descriptor(%arg0: !llvm.ptr, %arg1: !llvm.ptr, %arg2: i64, %arg3: i64, %arg4: i64, %arg5: i64, %arg6: i64) attributes {is_legal} {
   %0 = "test.direct_replacement"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %arg6) : (!llvm.ptr, !llvm.ptr, i64, i64, i64, i64, i64) -> (memref<5x4xf32>)
   "test.legal_op"(%0) : (memref<5x4xf32>) -> ()
   return
@@ -21,7 +22,7 @@ func.func @construct_ranked_memref_descriptor(%arg0: !llvm.ptr, %arg1: !llvm.ptr
 // CHECK-LABEL: func @invalid_ranked_memref_descriptor(
 //       CHECK:   %[[cast:.*]] = builtin.unrealized_conversion_cast %{{.*}} : i1 to memref<5x4xf32>
 //       CHECK:   "test.legal_op"(%[[cast]])
-func.func @invalid_ranked_memref_descriptor(%arg0: i1) {
+func.func @invalid_ranked_memref_descriptor(%arg0: i1) attributes {is_legal} {
   %0 = "test.direct_replacement"(%arg0) : (i1) -> (memref<5x4xf32>)
   "test.legal_op"(%0) : (memref<5x4xf32>) -> ()
   return
@@ -32,10 +33,10 @@ func.func @invalid_ranked_memref_descriptor(%arg0: i1) {
 // Test the argument materializer for unranked MemRef types.
 
 //   CHECK-LABEL: func @construct_unranked_memref_descriptor(
-//         CHECK:   llvm.mlir.undef : !llvm.struct<(i64, ptr)>
+//         CHECK:   llvm.mlir.poison : !llvm.struct<(i64, ptr)>
 // CHECK-COUNT-2:   llvm.insertvalue
 //         CHECK:   builtin.unrealized_conversion_cast %{{.*}} : !llvm.struct<(i64, ptr)> to memref<*xf32>
-func.func @construct_unranked_memref_descriptor(%arg0: i64, %arg1: !llvm.ptr) {
+func.func @construct_unranked_memref_descriptor(%arg0: i64, %arg1: !llvm.ptr) attributes {is_legal} {
   %0 = "test.direct_replacement"(%arg0, %arg1) : (i64, !llvm.ptr) -> (memref<*xf32>)
   "test.legal_op"(%0) : (memref<*xf32>) -> ()
   return
@@ -50,8 +51,107 @@ func.func @construct_unranked_memref_descriptor(%arg0: i64, %arg1: !llvm.ptr) {
 // CHECK-LABEL: func @invalid_unranked_memref_descriptor(
 //       CHECK:   %[[cast:.*]] = builtin.unrealized_conversion_cast %{{.*}} : i1 to memref<*xf32>
 //       CHECK:   "test.legal_op"(%[[cast]])
-func.func @invalid_unranked_memref_descriptor(%arg0: i1) {
+func.func @invalid_unranked_memref_descriptor(%arg0: i1) attributes {is_legal} {
   %0 = "test.direct_replacement"(%arg0) : (i1) -> (memref<*xf32>)
   "test.legal_op"(%0) : (memref<*xf32>) -> ()
   return
 }
+
+// -----
+
+// CHECK-LABEL: llvm.func @simple_func_conversion(
+//  CHECK-SAME:     %[[arg0:.*]]: i64) -> i64
+//       CHECK:   llvm.return %[[arg0]] : i64
+func.func @simple_func_conversion(%arg0: i64) -> i64 {
+  return %arg0 : i64
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @one_to_n_argument_conversion(
+//  CHECK-SAME:     %[[arg0:.*]]: i18, %[[arg1:.*]]: i18)
+//       CHECK:   %[[cast:.*]] = builtin.unrealized_conversion_cast %[[arg0]], %[[arg1]] : i18, i18 to i17
+//       CHECK:   "test.legal_op"(%[[cast]]) : (i17) -> ()
+func.func @one_to_n_argument_conversion(%arg0: i17) {
+  "test.legal_op"(%arg0) : (i17) -> ()
+  return
+}
+
+// CHECK: llvm.func @caller(%[[arg0:.*]]: i18, %[[arg1:.*]]: i18)
+// CHECK:   llvm.call @one_to_n_argument_conversion(%[[arg0]], %[[arg1]]) : (i18, i18) -> ()
+func.func @caller(%arg0: i17) {
+  func.call @one_to_n_argument_conversion(%arg0) : (i17) -> ()
+  return
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @one_to_n_return_conversion(
+//  CHECK-SAME:     %[[arg0:.*]]: i18, %[[arg1:.*]]: i18) -> !llvm.struct<(i18, i18)>
+//       CHECK:   %[[p1:.*]] = llvm.mlir.poison : !llvm.struct<(i18, i18)>
+//       CHECK:   %[[p2:.*]] = llvm.insertvalue %[[arg0]], %[[p1]][0] : !llvm.struct<(i18, i18)>
+//       CHECK:   %[[p3:.*]] = llvm.insertvalue %[[arg1]], %[[p2]][1] : !llvm.struct<(i18, i18)>
+//       CHECK:   llvm.return %[[p3]]
+func.func @one_to_n_return_conversion(%arg0: i17) -> i17 {
+  return %arg0 : i17
+}
+
+// CHECK: llvm.func @caller(%[[arg0:.*]]: i18, %[[arg1:.*]]: i18)
+// CHECK:   %[[res:.*]] = llvm.call @one_to_n_return_conversion(%[[arg0]], %[[arg1]]) : (i18, i18) -> !llvm.struct<(i18, i18)>
+// CHECK:   %[[e0:.*]] = llvm.extractvalue %[[res]][0] : !llvm.struct<(i18, i18)>
+// CHECK:   %[[e1:.*]] = llvm.extractvalue %[[res]][1] : !llvm.struct<(i18, i18)>
+// CHECK:   %[[i0:.*]] = llvm.mlir.poison : !llvm.struct<(i18, i18)>
+// CHECK:   %[[i1:.*]] = llvm.insertvalue %[[e0]], %[[i0]][0] : !llvm.struct<(i18, i18)>
+// CHECK:   %[[i2:.*]] = llvm.insertvalue %[[e1]], %[[i1]][1] : !llvm.struct<(i18, i18)>
+// CHECK:   llvm.return %[[i2]]
+func.func @caller(%arg0: i17) -> (i17) {
+  %res = func.call @one_to_n_return_conversion(%arg0) : (i17) -> (i17)
+  return %res : i17
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @multi_return(
+//  CHECK-SAME:     %[[arg0:.*]]: i18, %[[arg1:.*]]: i18, %[[arg2:.*]]: i1) -> !llvm.struct<(i18, i18, i1)>
+//       CHECK:   %[[p1:.*]] = llvm.mlir.poison : !llvm.struct<(i18, i18, i1)>
+//       CHECK:   %[[p2:.*]] = llvm.insertvalue %[[arg0]], %[[p1]][0] : !llvm.struct<(i18, i18, i1)>
+//       CHECK:   %[[p3:.*]] = llvm.insertvalue %[[arg1]], %[[p2]][1] : !llvm.struct<(i18, i18, i1)>
+//       CHECK:   %[[p4:.*]] = llvm.insertvalue %[[arg2]], %[[p3]][2] : !llvm.struct<(i18, i18, i1)>
+//       CHECK:   llvm.return %[[p4]]
+func.func @multi_return(%arg0: i17, %arg1: i1) -> (i17, i1) {
+  return %arg0, %arg1 : i17, i1
+}
+
+// CHECK: llvm.func @caller(%[[arg0:.*]]: i1, %[[arg1:.*]]: i18, %[[arg2:.*]]: i18)
+// CHECK:   %[[res:.*]] = llvm.call @multi_return(%[[arg1]], %[[arg2]], %[[arg0]]) : (i18, i18, i1) -> !llvm.struct<(i18, i18, i1)>
+// CHECK:   %[[e0:.*]] = llvm.extractvalue %[[res]][0] : !llvm.struct<(i18, i18, i1)>
+// CHECK:   %[[e1:.*]] = llvm.extractvalue %[[res]][1] : !llvm.struct<(i18, i18, i1)>
+// CHECK:   %[[e2:.*]] = llvm.extractvalue %[[res]][2] : !llvm.struct<(i18, i18, i1)>
+// CHECK:   %[[i0:.*]] = llvm.mlir.poison : !llvm.struct<(i18, i18, i1, i18, i18)>
+// CHECK:   %[[i1:.*]] = llvm.insertvalue %[[e0]], %[[i0]][0]
+// CHECK:   %[[i2:.*]] = llvm.insertvalue %[[e1]], %[[i1]][1]
+// CHECK:   %[[i3:.*]] = llvm.insertvalue %[[e2]], %[[i2]][2]
+// CHECK:   %[[i4:.*]] = llvm.insertvalue %[[e0]], %[[i3]][3]
+// CHECK:   %[[i5:.*]] = llvm.insertvalue %[[e1]], %[[i4]][4]
+// CHECK:   llvm.return %[[i5]]
+func.func @caller(%arg0: i1, %arg1: i17) -> (i17, i1, i17) {
+  %res:2 = func.call @multi_return(%arg1, %arg0) : (i17, i1) -> (i17, i1)
+  return %res#0, %res#1, %res#0 : i17, i1, i17
+}
+
+// -----
+
+// CHECK-LABEL: llvm.func @branch(
+//  CHECK-SAME:     %[[arg0:.*]]: i1, %[[arg1:.*]]: i18, %[[arg2:.*]]: i18)
+//       CHECK:   llvm.br ^[[bb1:.*]](%[[arg1]], %[[arg2]], %[[arg0]] : i18, i18, i1)
+//       CHECK: ^[[bb1]](%[[arg3:.*]]: i18, %[[arg4:.*]]: i18, %[[arg5:.*]]: i1):
+//       CHECK:   llvm.cond_br %[[arg5]], ^[[bb1]](%[[arg1]], %[[arg2]], %[[arg5]] : i18, i18, i1), ^[[bb2:.*]](%[[arg3]], %[[arg4]] : i18, i18)
+//       CHECK: ^bb2(%{{.*}}: i18, %{{.*}}: i18):
+//       CHECK:   llvm.return
+func.func @branch(%arg0: i1, %arg1: i17) {
+  cf.br ^bb1(%arg1, %arg0: i17, i1)
+^bb1(%arg2: i17, %arg3: i1):
+  cf.cond_br %arg3, ^bb1(%arg1, %arg3 : i17, i1), ^bb2(%arg2 : i17)
+^bb2(%arg4: i17):
+  return
+}
diff --git a/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir b/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir
index 2a7be0be7477a..e6321e99693ac 100644
--- a/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir
+++ b/mlir/test/Conversion/MemRefToSPIRV/memref-to-spirv.mlir
@@ -85,6 +85,28 @@ func.func @load_i1(%src: memref<4xi1, #spirv.storage_class<StorageBuffer>>, %i :
   return %0: i1
 }
 
+// CHECK-LABEL: func @load_aligned
+//  CHECK-SAME: (%[[SRC:.+]]: memref<4xi1, #spirv.storage_class<StorageBuffer>>, %[[IDX:.+]]: index)
+func.func @load_aligned(%src: memref<4xi1, #spirv.storage_class<StorageBuffer>>, %i : index) -> i1 {
+  // CHECK: spirv.Load "StorageBuffer" {{.*}} ["Aligned", 32] : i8
+  %0 = memref.load %src[%i] { alignment = 32 } : memref<4xi1, #spirv.storage_class<StorageBuffer>>
+  return %0: i1
+}
+
+// CHECK-LABEL: func @load_aligned_nontemporal
+func.func @load_aligned_nontemporal(%src: memref<4xi1, #spirv.storage_class<StorageBuffer>>, %i : index) -> i1 {
+  // CHECK: spirv.Load "StorageBuffer" {{.*}} ["Aligned|Nontemporal", 32] : i8
+  %0 = memref.load %src[%i] { alignment = 32, nontemporal = true } : memref<4xi1, #spirv.storage_class<StorageBuffer>>
+  return %0: i1
+}
+
+// CHECK-LABEL: func @load_aligned_psb
+func.func @load_aligned_psb(%src: memref<4xi1, #spirv.storage_class<PhysicalStorageBuffer>>, %i : index) -> i1 {
+  // CHECK: %[[VAL:.+]] = spirv.Load "PhysicalStorageBuffer" {{.*}} ["Aligned", 32] : i8
+  %0 = memref.load %src[%i] { alignment = 32 } : memref<4xi1, #spirv.storage_class<PhysicalStorageBuffer>>
+  return %0: i1
+}
+
 // CHECK-LABEL: func @store_i1
 //  CHECK-SAME: %[[DST:.+]]: memref<4xi1, #spirv.storage_class<StorageBuffer>>,
 //  CHECK-SAME: %[[IDX:.+]]: index
diff --git a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir
index 8d4f9478e7d67..c4cf4f7337d81 100644
--- a/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/NVGPUToNVVM/nvgpu-to-nvvm.mlir
@@ -159,7 +159,7 @@ func.func @m8n8k4_f64(%arg0: vector<1x1xf64>, %arg1: vector<1x1xf64>, %arg2: vec
 // CHECK-LABEL: @ldmatrix_x4
 func.func @ldmatrix_x4(%arg0: memref<128x128xf16, 3>) ->  vector<4x2xf16> {
   %c0  = arith.constant 0 : index
-  // CHECK: nvvm.ldmatrix {{%.+}} {layout = #nvvm.mma_layout<row>, num = 4 : i32} {{.*}} -> !llvm.struct<(i32, i32, i32, i32)
+  // CHECK: nvvm.ldmatrix {{%.+}} {eltType = #nvvm.ld_st_matrix_elt_type<b16>, layout = #nvvm.mma_layout<row>, num = 4 : i32, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>} : {{.*}} -> !llvm.struct<(i32, i32, i32, i32)>
   %a = nvgpu.ldmatrix %arg0[%c0, %c0] {transpose = false, numTiles = 4 : i32} : memref<128x128xf16, 3> -> vector<4x2xf16>
   // CHECK: llvm.extractvalue
   // CHECK: llvm.bitcast
@@ -179,7 +179,7 @@ func.func @ldmatrix_x4(%arg0: memref<128x128xf16, 3>) ->  vector<4x2xf16> {
 // CHECK-LABEL: @ldmatrix_x1
 func.func @ldmatrix_x1(%arg0: memref<128x128xf16, 3>) ->  vector<1x2xf16> {
   %c0  = arith.constant 0 : index
-  // CHECK: nvvm.ldmatrix {{%.+}} {layout = #nvvm.mma_layout<row>, num = 1 : i32} {{.*}} -> i32
+  // CHECK: nvvm.ldmatrix {{%.+}} {eltType = #nvvm.ld_st_matrix_elt_type<b16>, layout = #nvvm.mma_layout<row>, num = 1 : i32, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>} : {{.*}} -> i32
   %a = nvgpu.ldmatrix %arg0[%c0, %c0] {transpose = false, numTiles = 1 : i32} : memref<128x128xf16, 3> -> vector<1x2xf16>
   // CHECK: llvm.bitcast
   // CHECK: llvm.insertvalue
diff --git a/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir b/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir
index 580b09d70c480..2a19c72ab0840 100644
--- a/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir
+++ b/mlir/test/Conversion/NVVMToLLVM/nvvm-to-llvm.mlir
@@ -3,6 +3,7 @@
 // Same below, but using the `ConvertToLLVMPatternInterface` entry point
 // and the generic `convert-to-llvm` pass.
 // RUN: mlir-opt --convert-to-llvm --split-input-file %s | FileCheck %s
+// RUN: mlir-opt --convert-to-llvm="allow-pattern-rollback=0" --split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @init_mbarrier
 llvm.func @init_mbarrier(%barrier_gen : !llvm.ptr, %barrier : !llvm.ptr<3>, %count : i32, %pred : i1) {
@@ -666,18 +667,92 @@ llvm.func @init_mbarrier(
     %count : i32, 
     %pred : i1) {
   // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "mbarrier.init.b64 [$0], $1;", "l,r" 
-  nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count)  : !llvm.ptr, i32 
+  nvvm.inline_ptx "mbarrier.init.b64 [{$r0}], {$r1};" ro (%barrier_gen, %count : !llvm.ptr, i32)
   // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "@$2 mbarrier.init.b64 [$0], $1;", "l,r,b"
-  nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count), predicate = %pred : !llvm.ptr, i32, i1
+  nvvm.inline_ptx "mbarrier.init.b64 [{$r0}], {$r1};" ro (%barrier_gen, %count : !llvm.ptr, i32), predicate = %pred
   llvm.return
 }
 // -----
 
 llvm.func @ex2(%input : f32, %pred : i1) {
   // CHECK: %{{.*}} = llvm.inline_asm has_side_effects asm_dialect = att "ex2.approx.ftz.f32 $0, $1;", "=f,f" %{{.*}} : (f32) -> f32
-  %0 = nvvm.inline_ptx "ex2.approx.ftz.f32 $0, $1;" (%input) : f32 -> f32
+  %0 = nvvm.inline_ptx "ex2.approx.ftz.f32 {$w0}, {$r0};" ro (%input : f32) -> f32
   
   // CHECK: %{{.*}} =  llvm.inline_asm has_side_effects asm_dialect = att "@$1 ex2.approx.ftz.f32 $0, $1;", "=f,f,b" %{{.*}}, %{{.*}} : (f32, i1) -> f32
-  %1 = nvvm.inline_ptx "ex2.approx.ftz.f32 $0, $1;" (%input), predicate = %pred  : f32, i1 -> f32
+  %1 = nvvm.inline_ptx "ex2.approx.ftz.f32 {$w0}, {$r0};" ro (%input : f32), predicate = %pred  -> f32
+  llvm.return
+}
+
+// CHECK-LABEL: @multi_return(
+// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: i32, %[[arg1:[a-zA-Z0-9_]+]]: i32)
+llvm.func @multi_return(%a : i32, %b : i32) -> i32 {
+  // CHECK: %[[S1:.+]] = llvm.inline_asm has_side_effects asm_dialect = att "{.reg .pred p; setp.ge.s32 p, $2, $3; selp.s32 $0, $2,$3, p; selp.s32 $1, $2,$3, p;}", "=r,=r,r,r" %[[arg0]], %[[arg1]] : (i32, i32) -> !llvm.struct<(i32, i32)>
+  // CHECK: %[[S2:.+]] = llvm.extractvalue %[[S1]][0] : !llvm.struct<(i32, i32)> 
+  // CHECK: %[[S3:.+]] = llvm.extractvalue %[[S1]][1] : !llvm.struct<(i32, i32)> 
+  // CHECK: %[[S4:.+]] = llvm.add %[[S2]], %[[S3]] : i32
+  // CHECK: llvm.return %[[S4]] : i32
+   %r1, %r2 = nvvm.inline_ptx "{.reg .pred p; setp.ge.s32 p, {$r0}, {$r1}; selp.s32 {$w0}, {$r0},{$r1}, p; selp.s32 {$w1}, {$r0},{$r1}, p;}"
+                  ro (%a, %b : i32,i32) -> i32,i32
+   %r3 = llvm.add %r1, %r2 : i32
+   llvm.return %r3 : i32
+}
+
+// CHECK-LABEL: @inline_ptx_multi_rw(
+// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: i32, %[[arg1:[a-zA-Z0-9_]+]]: i32, %[[arg2:[a-zA-Z0-9_]+]]: f32, %[[arg3:[a-zA-Z0-9_]+]]: f32)
+llvm.func @inline_ptx_multi_rw(%a : i32, %b : i32,  %rw_c : f32, %rw_d : f32) -> f32 {
+// CHECK: %[[S0:.+]] = llvm.inline_asm has_side_effects asm_dialect = att "{.reg .pred p; setp.ge.s32 p, $2, $3; selp.s32 $0, $2,$3, p; selp.s32 $1, $2,$3, p;}", 
+// CHECK-SAME: "=f,=f,r,r,0,1" 
+// CHECK-SAME: %[[arg2]], %[[arg3]], %[[arg0]], %[[arg1]] 
+// CHECK-SAME: : (f32, f32, i32, i32) -> !llvm.struct<(f32, f32)>
+// CHECK: %[[S1:.+]] = llvm.extractvalue %[[S0]][0] : !llvm.struct<(f32, f32)> 
+// CHECK: %[[S2:.+]] = llvm.extractvalue %[[S0]][1] : !llvm.struct<(f32, f32)> 
+// CHECK: %[[S3:.+]] = llvm.fadd %[[S1]], %[[S2]] : f32
+// CHECK: llvm.return %[[S3]] : f32
+    nvvm.inline_ptx "{.reg .pred p; setp.ge.s32 p, {$r0}, {$r1}; selp.s32 {$rw0}, {$r0},{$r1}, p; selp.s32 {$rw1}, {$r0},{$r1}, p;}"
+    ro (%a, %b : i32,i32) 
+    rw (%rw_c, %rw_d: f32,f32)
+   %r4 = llvm.fadd %rw_c, %rw_d : f32
+   llvm.return %r4 : f32
+}
+
+// CHECK-LABEL: @inline_ptx_multi_rw_r(
+// CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: i32, %[[arg1:[a-zA-Z0-9_]+]]: i32, %[[arg2:[a-zA-Z0-9_]+]]: f32, %[[arg3:[a-zA-Z0-9_]+]]: f32)
+llvm.func @inline_ptx_multi_rw_r(%a : i32, %b : i32,  %rw_c : f32, %rw_d : f32) -> f32 {
+// CHECK: %[[S0:.+]] = llvm.inline_asm has_side_effects asm_dialect = att "{.reg .pred p; setp.ge.s32 p, $4, $5; selp.s32 $0, $4,$5, p; selp.s32 $1, $4,$5, p; selp.s32 $2, $4,$5, p; selp.s32 $3, $4,$5, p;}", 
+// CHECK-SAME: "=f,=f,=r,=r,r,r,0,1" 
+// CHECK-SAME: %[[arg2]], %[[arg3]], %[[arg0]], %[[arg1]] : 
+// CHECK-SAME: (f32, f32, i32, i32) -> !llvm.struct<(f32, f32, i32, i32)>
+// CHECK: %[[S1:.+]] = llvm.extractvalue %[[S0]][0] : !llvm.struct<(f32, f32, i32, i32)> 
+// CHECK: %[[S2:.+]] = llvm.extractvalue %[[S0]][1] : !llvm.struct<(f32, f32, i32, i32)> 
+// CHECK: %[[S3:.+]] = llvm.extractvalue %[[S0]][2] : !llvm.struct<(f32, f32, i32, i32)> 
+// CHECK: %[[S4:.+]] = llvm.extractvalue %[[S0]][3] : !llvm.struct<(f32, f32, i32, i32)> 
+// CHECK: %[[S5:.+]] = llvm.add %[[S3]], %[[S4]] : i32
+// CHECK: %[[S6:.+]] = llvm.sitofp %[[S5]] : i32 to f32
+// CHECK: %[[S7:.+]] = llvm.fadd %[[S1]], %[[S2]] : f32
+// CHECK: %[[S8:.+]] = llvm.fadd %[[S6]], %[[S2]] : f32
+// CHECK: llvm.return %[[S8]] : f32
+
+  %wo0, %wo1 = nvvm.inline_ptx "{.reg .pred p; setp.ge.s32 p, {$r0}, {$r1}; selp.s32 {$rw0}, {$r0},{$r1}, p; selp.s32 {$rw1}, {$r0},{$r1}, p; selp.s32 {$w0}, {$r0},{$r1}, p; selp.s32 {$w1}, {$r0},{$r1}, p;}"
+      ro (%a, %b : i32,i32) 
+      rw (%rw_c, %rw_d: f32,f32) -> i32,i32
+   %r3 = llvm.add %wo0, %wo1 : i32
+   %r3f = llvm.sitofp %r3 : i32 to f32
+   %r4 = llvm.fadd %rw_c, %rw_d : f32
+   %r5 = llvm.fadd %r3f, %rw_d : f32
+   llvm.return %r5 : f32
+}
+
+
+// -----
+
+// CHECK-LABEL: @nvvm_pmevent
+llvm.func @nvvm_pmevent() {
+  // CHECK: %[[S0:.+]] = llvm.mlir.constant(10 : i32) : i32
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "pmevent $0;", "n" %[[S0]] : (i32) -> ()
+  
+  nvvm.pmevent id = 10
+  // CHECK: %[[S1:.+]] = llvm.mlir.constant(4 : i32) : i32
+  // CHECK: llvm.inline_asm has_side_effects asm_dialect = att "pmevent $0;", "n" %[[S1]] : (i32) -> ()
+  nvvm.pmevent id = 4
   llvm.return
 }
diff --git a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
index d69de998346b5..7d8ccd910cdf4 100644
--- a/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
+++ b/mlir/test/Conversion/OpenMPToLLVM/convert-to-llvmir.mlir
@@ -1,5 +1,6 @@
 // RUN: mlir-opt -convert-openmp-to-llvm -split-input-file %s | FileCheck %s
 // RUN: mlir-opt -convert-to-llvm -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -convert-to-llvm="allow-pattern-rollback=0" -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: llvm.func @foo(i64, i64)
 func.func private @foo(index, index)
diff --git a/mlir/test/Conversion/SCFToControlFlow/convert-to-cfg.mlir b/mlir/test/Conversion/SCFToControlFlow/convert-to-cfg.mlir
index ef0fa083a021a..483c7b35c6ec8 100644
--- a/mlir/test/Conversion/SCFToControlFlow/convert-to-cfg.mlir
+++ b/mlir/test/Conversion/SCFToControlFlow/convert-to-cfg.mlir
@@ -18,6 +18,24 @@ func.func @simple_std_for_loop(%arg0 : index, %arg1 : index, %arg2 : index) {
   return
 }
 
+// CHECK-LABEL: func @unsigned_loop(%{{.*}}: index, %{{.*}}: index, %{{.*}}: index) {
+//  CHECK-NEXT:  cf.br ^bb1(%{{.*}} : index)
+//  CHECK-NEXT:  ^bb1(%{{.*}}: index):    // 2 preds: ^bb0, ^bb2
+//  CHECK-NEXT:    %{{.*}} = arith.cmpi ult, %{{.*}}, %{{.*}} : index
+//  CHECK-NEXT:    cf.cond_br %{{.*}}, ^bb2, ^bb3
+//  CHECK-NEXT:  ^bb2:   // pred: ^bb1
+//  CHECK-NEXT:    %{{.*}} = arith.constant 1 : index
+//  CHECK-NEXT:    %[[iv:.*]] = arith.addi %{{.*}}, %{{.*}} : index
+//  CHECK-NEXT:    cf.br ^bb1(%[[iv]] : index)
+//  CHECK-NEXT:  ^bb3:   // pred: ^bb1
+//  CHECK-NEXT:    return
+func.func @unsigned_loop(%arg0 : index, %arg1 : index, %arg2 : index) {
+  scf.for unsigned %i0 = %arg0 to %arg1 step %arg2 {
+    %c1 = arith.constant 1 : index
+  }
+  return
+}
+
 // CHECK-LABEL: func @simple_std_2_for_loops(%{{.*}}: index, %{{.*}}: index, %{{.*}}: index) {
 //  CHECK-NEXT:    cf.br ^bb1(%{{.*}} : index)
 //  CHECK-NEXT:  ^bb1(%[[ub0:.*]]: index):    // 2 preds: ^bb0, ^bb5
diff --git a/mlir/test/Conversion/SCFToSPIRV/for.mlir b/mlir/test/Conversion/SCFToSPIRV/for.mlir
index 81661ec7a3a06..9c552166cd72d 100644
--- a/mlir/test/Conversion/SCFToSPIRV/for.mlir
+++ b/mlir/test/Conversion/SCFToSPIRV/for.mlir
@@ -5,6 +5,7 @@ module attributes {
     #spirv.vce<v1.0, [Shader], [SPV_KHR_storage_buffer_storage_class]>, #spirv.resource_limits<>>
 } {
 
+// CHECK-LABEL: @loop_kernel
 func.func @loop_kernel(%arg2 : memref<10xf32, #spirv.storage_class<StorageBuffer>>, %arg3 : memref<10xf32, #spirv.storage_class<StorageBuffer>>) {
   // CHECK: %[[LB:.*]] = spirv.Constant 4 : i32
   %lb = arith.constant 4 : index
@@ -34,6 +35,19 @@ func.func @loop_kernel(%arg2 : memref<10xf32, #spirv.storage_class<StorageBuffer
   return
 }
 
+// CHECK-LABEL: @unsigned_loop
+//       CHECK:   spirv.ULessThan
+func.func @unsigned_loop(%arg2 : memref<10xf32, #spirv.storage_class<StorageBuffer>>, %arg3 : memref<10xf32, #spirv.storage_class<StorageBuffer>>) {
+  %lb = arith.constant 4 : index
+  %ub = arith.constant 42 : index
+  %step = arith.constant 2 : index
+  scf.for unsigned %arg4 = %lb to %ub step %step {
+    %1 = memref.load %arg2[%arg4] : memref<10xf32, #spirv.storage_class<StorageBuffer>>
+    memref.store %1, %arg3[%arg4] : memref<10xf32, #spirv.storage_class<StorageBuffer>>
+  }
+  return
+}
+
 // CHECK-LABEL: @loop_yield
 func.func @loop_yield(%arg2 : memref<10xf32, #spirv.storage_class<StorageBuffer>>, %arg3 : memref<10xf32, #spirv.storage_class<StorageBuffer>>) {
   // CHECK: %[[LB:.*]] = spirv.Constant 4 : i32
diff --git a/mlir/test/Conversion/UBToLLVM/ub-to-llvm.mlir b/mlir/test/Conversion/UBToLLVM/ub-to-llvm.mlir
index 5307e477b8786..6c0b111d4c2c5 100644
--- a/mlir/test/Conversion/UBToLLVM/ub-to-llvm.mlir
+++ b/mlir/test/Conversion/UBToLLVM/ub-to-llvm.mlir
@@ -3,6 +3,7 @@
 // Same below, but using the `ConvertToLLVMPatternInterface` entry point
 // and the generic `convert-to-llvm` pass.
 // RUN: mlir-opt --convert-to-llvm="filter-dialects=ub" --split-input-file %s | FileCheck %s
+// RUN: mlir-opt --convert-to-llvm="filter-dialects=ub allow-pattern-rollback=0" --split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: @check_poison
 func.func @check_poison() {
diff --git a/mlir/test/Conversion/VectorToAMX/contract-to-amx.mlir b/mlir/test/Conversion/VectorToAMX/contract-to-amx.mlir
new file mode 100644
index 0000000000000..4fb88dd165126
--- /dev/null
+++ b/mlir/test/Conversion/VectorToAMX/contract-to-amx.mlir
@@ -0,0 +1,310 @@
+// RUN: mlir-opt %s -convert-vector-to-amx -split-input-file | FileCheck %s
+
+/// VNNI format is Intel's packed data layout.
+/// For matrix multiplication, elements from the reduction dimension `k`
+/// are packed into 32-bit tuples. Then the appropriate AMX operations can
+/// perform tile multiplication directly on the packed data.
+///
+/// These packed elements are represented in the indexing maps by a separate
+/// reduction dimension `vnni`.
+
+#map = affine_map<(m, n, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, n, k, vnni) -> (k, n, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (m, n)>
+func.func @contract_vnni_f16(%A: vector<4x8x2xf16>, %B: vector<8x16x2xf16>,
+    %C: vector<4x16xf32>) -> vector<4x16xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<4x8x2xf16>, vector<8x16x2xf16> into vector<4x16xf32>
+  return %0 : vector<4x16xf32>
+}
+
+// CHECK-LABEL: @contract_vnni_f16(
+// CHECK-SAME:    %[[A:.+]]: vector<4x8x2xf16>,
+// CHECK-SAME:    %[[B:.+]]: vector<8x16x2xf16>,
+// CHECK-SAME:    %[[C:.+]]: vector<4x16xf32>
+
+/// AMX hardware has no direct access to the registers. Thus, data must
+/// be transfered through intermediate buffers.
+///
+/// Load A vector into an AMX tile
+// CHECK:       %[[A_BUF:.+]] = memref.alloca() : memref<4x8x2xf16>
+// CHECK:       vector.transfer_write %[[A]], %[[A_BUF]]
+// CHECK:       %[[A_BUF_2D:.+]] = memref.collapse_shape %[[A_BUF]]
+// CHECK-SAME:    {{\[}}[0], [1, 2]] : memref<4x8x2xf16> into memref<4x16xf16>
+// CHECK:       %[[A_TILE:.+]] = amx.tile_load %[[A_BUF_2D]]
+
+/// Load B vector into an AMX tile
+// CHECK:       %[[B_BUF:.+]] = memref.alloca() : memref<8x16x2xf16>
+// CHECK:       vector.transfer_write %[[B]], %[[B_BUF]]
+// CHECK:       %[[B_BUF_2D:.+]] = memref.collapse_shape %[[B_BUF]]
+// CHECK-SAME:    {{\[}}[0], [1, 2]] : memref<8x16x2xf16> into memref<8x32xf16>
+// CHECK:       %[[B_TILE:.+]] = amx.tile_load %[[B_BUF_2D]]
+
+/// Load C vector into an AMX tile
+// CHECK:       %[[C_BUF:.+]] = memref.alloca() : memref<4x16xf32>
+// CHECK:       vector.transfer_write %[[C]], %[[C_BUF]]
+// CHECK:       %[[C_TILE:.+]] = amx.tile_load %[[C_BUF]]
+
+/// Perform tile multiplication
+// CHECK:       %[[RES:.+]] = amx.tile_mulf
+// CHECK-SAME:    %[[A_TILE]], %[[B_TILE]], %[[C_TILE]]
+
+/// Load the result back into a vector
+// CHECK:       %[[RES_BUF:.+]] = memref.alloca() : memref<4x16xf32>
+// CHECK:       amx.tile_store %[[RES_BUF]]{{.*}}, %[[RES]]
+// CHECK:       %[[RES_VEC:.+]] = vector.transfer_read %[[RES_BUF]]
+
+// CHECK:       return %[[RES_VEC]]
+
+// -----
+
+#map = affine_map<(m, n, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, n, k, vnni) -> (k, n, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (m, n)>
+func.func @contract_vnni_bf16(%A: vector<4x8x2xbf16>, %B: vector<8x16x2xbf16>,
+    %C: vector<4x16xf32>) -> vector<4x16xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<4x8x2xbf16>, vector<8x16x2xbf16> into vector<4x16xf32>
+  return %0 : vector<4x16xf32>
+}
+
+// CHECK-LABEL: @contract_vnni_bf16(
+// CHECK-COUNT-3: amx.tile_load
+// CHECK: amx.tile_mulf
+// CHECK: amx.tile_store
+
+// -----
+
+#map = affine_map<(m, n, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, n, k, vnni) -> (k, n, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (m, n)>
+func.func @contract_vnni_i8(%A: vector<4x16x4xi8>, %B: vector<16x8x4xi8>,
+    %C: vector<4x8xi32>) -> vector<4x8xi32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<4x16x4xi8>, vector<16x8x4xi8> into vector<4x8xi32>
+  return %0 : vector<4x8xi32>
+}
+
+// CHECK-LABEL: @contract_vnni_i8(
+// CHECK-COUNT-3: amx.tile_load
+// CHECK: amx.tile_muli
+// CHECK: amx.tile_store
+
+// -----
+
+#map = affine_map<(vnni, m, k, n) -> (m, k, vnni)>
+#map1 = affine_map<(vnni, m, k, n) -> (k, n, vnni)>
+#map2 = affine_map<(vnni, m, k, n) -> (m, n)>
+func.func @contract_shuffled_iterators(%A: vector<4x16x4xi8>, %B: vector<16x8x4xi8>,
+    %C: vector<4x8xi32>) -> vector<4x8xi32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["reduction", "parallel", "reduction", "parallel"]}
+    %A, %B, %C : vector<4x16x4xi8>, vector<16x8x4xi8> into vector<4x8xi32>
+  return %0 : vector<4x8xi32>
+}
+
+// CHECK-LABEL: @contract_shuffled_iterators(
+// CHECK-COUNT-3: amx.tile_load
+// CHECK: amx.tile_muli
+// CHECK: amx.tile_store
+
+// -----
+
+#map = affine_map<(m, n, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, n, k, vnni) -> (k, n, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (m, n)>
+func.func @negative_invalid_kind(%A: vector<4x8x2xf16>, %B: vector<8x16x2xf16>,
+    %C: vector<4x16xf32>) -> vector<4x16xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<mul>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<4x8x2xf16>, vector<8x16x2xf16> into vector<4x16xf32>
+  return %0 : vector<4x16xf32>
+}
+
+// CHECK-LABEL: @negative_invalid_kind(
+// CHECK-NOT: amx
+// CHECK: vector.contract
+
+// -----
+
+#map = affine_map<(m, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, k, vnni) -> (k, m, vnni)>
+#map2 = affine_map<(m, k, vnni) -> ()>
+func.func @negative_non_vector_acc(%A: vector<4x8x2xf16>, %B: vector<8x4x2xf16>,
+    %C: f32) -> f32 {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["reduction", "reduction", "reduction"]}
+    %A, %B, %C : vector<4x8x2xf16>, vector<8x4x2xf16> into f32
+  return %0 : f32
+}
+
+// CHECK-LABEL: @negative_non_vector_acc(
+// CHECK-NOT: amx
+// CHECK: vector.contract
+
+// -----
+
+#map = affine_map<(m, n, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, n, k, vnni) -> (k, n, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (m, n)>
+func.func @negative_invalid_operand_types(%A: vector<4x8x2xf32>,
+    %B: vector<8x16x2xf32>, %C: vector<4x16xf32>) -> vector<4x16xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<4x8x2xf32>, vector<8x16x2xf32> into vector<4x16xf32>
+  return %0 : vector<4x16xf32>
+}
+
+// CHECK-LABEL: @negative_invalid_operand_types(
+// CHECK-NOT: amx
+// CHECK: vector.contract
+
+// -----
+
+#map = affine_map<(m, n, k) -> (m, k)>
+#map1 = affine_map<(m, n, k) -> (k, n)>
+#map2 = affine_map<(m, n, k) -> (m, n)>
+func.func @negative_non_packed_layout(%A: vector<4x16xf16>, %B: vector<16x16xf16>,
+    %C: vector<4x16xf32>) -> vector<4x16xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction"]}
+    %A, %B, %C : vector<4x16xf16>, vector<16x16xf16> into vector<4x16xf32>
+  return %0 : vector<4x16xf32>
+}
+
+// CHECK-LABEL: @negative_non_packed_layout(
+// CHECK-NOT: amx
+// CHECK: vector.contract
+
+// -----
+
+#map = affine_map<(m, n, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, n, k, vnni) -> (k, n, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (m, n)>
+func.func @negative_invalid_vnni_factor(%A: vector<4x2x4xf16>, %B: vector<2x2x4xf16>,
+    %C: vector<4x2xf32>) -> vector<4x2xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<4x2x4xf16>, vector<2x2x4xf16> into vector<4x2xf32>
+  return %0 : vector<4x2xf32>
+}
+
+// CHECK-LABEL: @negative_invalid_vnni_factor(
+// CHECK-NOT: amx
+// CHECK: vector.contract
+
+// -----
+
+#map = affine_map<(batch, m, n, k, vnni) -> (batch, m, k, vnni)>
+#map1 = affine_map<(batch, m, n, k, vnni) -> (batch, k, n, vnni)>
+#map2 = affine_map<(batch, m, n, k, vnni) -> (batch, m, n)>
+func.func @negative_invalid_operands_shapes(%A: vector<1x4x8x2xf16>,
+    %B: vector<1x8x16x2xf16>, %C: vector<1x4x16xf32>) -> vector<1x4x16xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<1x4x8x2xf16>, vector<1x8x16x2xf16> into vector<1x4x16xf32>
+  return %0 : vector<1x4x16xf32>
+}
+
+// CHECK-LABEL: @negative_invalid_operands_shapes(
+// CHECK-NOT: amx
+// CHECK: vector.contract
+
+// -----
+
+#map = affine_map<(m, n, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, n, k, vnni) -> (k, n, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (m, n)>
+func.func @negative_too_many_rows(%A: vector<32x8x2xf16>, %B: vector<8x16x2xf16>,
+    %C: vector<32x16xf32>) -> vector<32x16xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<32x8x2xf16>, vector<8x16x2xf16> into vector<32x16xf32>
+  return %0 : vector<32x16xf32>
+}
+
+// CHECK-LABEL: @negative_too_many_rows(
+// CHECK-NOT: amx
+// CHECK: vector.contract
+
+// -----
+
+#map = affine_map<(m, n, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, n, k, vnni) -> (k, n, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (m, n)>
+func.func @negative_too_wide_rows(%A: vector<4x32x2xf16>, %B: vector<32x16x2xf16>,
+    %C: vector<4x16xf32>) -> vector<4x16xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<4x32x2xf16>, vector<32x16x2xf16> into vector<4x16xf32>
+  return %0 : vector<4x16xf32>
+}
+
+// CHECK-LABEL: @negative_too_wide_rows(
+// CHECK-NOT: amx
+// CHECK: vector.contract
+
+// -----
+
+#map = affine_map<(m, n, k, vnni) -> (k, vnni, m)>
+#map1 = affine_map<(m, n, k, vnni) -> (n, k, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (m, n)>
+func.func @negative_input_dim_permutation(%A: vector<2x2x2xf16>,
+    %B: vector<2x2x2xf16>, %C: vector<2x2xf32>) -> vector<2x2xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<2x2x2xf16>, vector<2x2x2xf16> into vector<2x2xf32>
+  return %0 : vector<2x2xf32>
+}
+
+// CHECK-LABEL: @negative_input_dim_permutation(
+// CHECK-NOT: amx
+// CHECK: vector.contract
+
+// -----
+
+#map = affine_map<(m, n, k, vnni) -> (m, k, vnni)>
+#map1 = affine_map<(m, n, k, vnni) -> (k, n, vnni)>
+#map2 = affine_map<(m, n, k, vnni) -> (n, m)>
+func.func @negative_output_dim_permutation(%A: vector<4x8x2xf16>,
+    %B: vector<8x16x2xf16>, %C: vector<16x4xf32>) -> vector<16x4xf32> {
+  %0 = vector.contract
+    {kind = #vector.kind<add>,
+    indexing_maps = [#map, #map1, #map2],
+    iterator_types = ["parallel", "parallel", "reduction", "reduction"]}
+    %A, %B, %C : vector<4x8x2xf16>, vector<8x16x2xf16> into vector<16x4xf32>
+  return %0 : vector<16x4xf32>
+}
+
+// CHECK-LABEL: @negative_output_dim_permutation(
+// CHECK-NOT: amx
+// CHECK: vector.contract
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
index 31e17fb3e3cc6..9b57b1b6fb4c7 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm-interface.mlir
@@ -1,4 +1,5 @@
 // RUN: mlir-opt --convert-to-llvm="filter-dialects=vector" --split-input-file %s | FileCheck %s
+// RUN: mlir-opt --convert-to-llvm="filter-dialects=vector allow-pattern-rollback=0" --split-input-file %s | FileCheck %s
 // RUN: mlir-opt %s -convert-vector-to-llvm -split-input-file | FileCheck %s
 
 //===========================================================================//
@@ -182,8 +183,7 @@ func.func @shuffle_0D_direct(%arg0: vector<f32>) -> vector<3xf32> {
 }
 // CHECK-LABEL: @shuffle_0D_direct(
 //  CHECK-SAME:     %[[A:.*]]: vector<f32>
-//       CHECK:   %[[c:.*]] = builtin.unrealized_conversion_cast %[[A]] : vector<f32> to vector<1xf32>
-//       CHECK:   %[[s:.*]] = llvm.shufflevector %[[c]], %[[c]] [0, 1, 0] : vector<1xf32>
+//       CHECK:   %[[s:.*]] = llvm.shufflevector %{{.*}}, %{{.*}} [0, 1, 0] : vector<1xf32>
 //       CHECK:   return %[[s]] : vector<3xf32>
 
 // -----
@@ -1679,6 +1679,16 @@ func.func @load_0d(%memref : memref<200x100xf32>, %i : index, %j : index) -> vec
 
 // -----
 
+func.func @load_with_alignment(%memref : memref<200x100xf32>, %i : index, %j : index) -> vector<8xf32> {
+  %0 = vector.load %memref[%i, %j] { alignment = 8 } : memref<200x100xf32>, vector<8xf32>
+  return %0 : vector<8xf32>
+}
+
+// CHECK-LABEL: func @load_with_alignment
+// CHECK: llvm.load {{.*}} {alignment = 8 : i64} : !llvm.ptr -> vector<8xf32>
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // vector.store
 //===----------------------------------------------------------------------===//
@@ -1785,6 +1795,16 @@ func.func @store_0d(%memref : memref<200x100xf32>, %i : index, %j : index) {
 
 // -----
 
+func.func @store_with_alignment(%memref : memref<200x100xf32>, %i : index, %j : index, %val : vector<4xf32>) {
+  vector.store %val, %memref[%i, %j] {alignment = 8} : memref<200x100xf32>, vector<4xf32>
+  return
+}
+
+// CHECK-LABEL: func @store_with_alignment
+// CHECK: llvm.store %{{.*}} {alignment = 8 : i64} :  vector<4xf32>, !llvm.ptr
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // vector.maskedload
 //===----------------------------------------------------------------------===//
@@ -1839,6 +1859,16 @@ func.func @masked_load_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16]
 
 // -----
 
+func.func @masked_load_with_alignment(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector<16xf32>, %arg3: index) -> vector<16xf32> {
+  %0 = vector.maskedload %arg0[%arg3], %arg1, %arg2 { alignment = 2 } : memref<?xf32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
+  return %0 : vector<16xf32>
+}
+
+// CHECK-LABEL: func @masked_load_with_alignment
+// CHECK: llvm.intr.masked.load %{{.*}} {alignment = 2 : i32} : (!llvm.ptr, vector<16xi1>, vector<16xf32>) -> vector<16xf32>
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // vector.maskedstore
 //===----------------------------------------------------------------------===//
@@ -1891,6 +1921,16 @@ func.func @masked_store_index_scalable(%arg0: memref<?xindex>, %arg1: vector<[16
 
 // -----
 
+func.func @masked_store_with_alignment(%arg0: memref<?xf32>, %arg1: vector<16xi1>, %arg2: vector<16xf32>, %arg3: index) {
+  vector.maskedstore %arg0[%arg3], %arg1, %arg2 { alignment = 2 } : memref<?xf32>, vector<16xi1>, vector<16xf32>
+  return
+}
+
+// CHECK-LABEL: func @masked_store_with_alignment
+// CHECK: llvm.intr.masked.store %{{.*}} {alignment = 2 : i32} : vector<16xf32>, vector<16xi1> into !llvm.ptr
+
+// -----
+
 //===----------------------------------------------------------------------===//
 // vector.gather
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
index 72810b5dddaa3..07d335117de01 100644
--- a/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
+++ b/mlir/test/Conversion/VectorToLLVM/vector-to-llvm.mlir
@@ -1737,3 +1737,40 @@ func.func @step() -> vector<4xindex> {
   %0 = vector.step : vector<4xindex>
   return %0 : vector<4xindex>
 }
+
+
+// -----
+
+//===----------------------------------------------------------------------===//
+// vector.from_elements
+//===----------------------------------------------------------------------===//
+
+// NOTE: We unroll multi-dimensional from_elements ops with pattern `UnrollFromElements`
+// and then convert the 1-D from_elements ops to llvm.
+
+// CHECK-LABEL: func @from_elements_3d
+//  CHECK-SAME:  %[[ARG_0:.*]]: f32, %[[ARG_1:.*]]: f32, %[[ARG_2:.*]]: f32, %[[ARG_3:.*]]: f32)
+//       CHECK:  %[[UNDEF_RES:.*]] = ub.poison : vector<2x1x2xf32>
+//       CHECK:  %[[UNDEF_RES_LLVM:.*]] = builtin.unrealized_conversion_cast %[[UNDEF_RES]] : vector<2x1x2xf32> to !llvm.array<2 x array<1 x vector<2xf32>>>
+//       CHECK:  %[[UNDEF_VEC_RANK_2:.*]] = ub.poison : vector<1x2xf32>
+//       CHECK:  %[[UNDEF_VEC_RANK_2_LLVM:.*]] = builtin.unrealized_conversion_cast %[[UNDEF_VEC_RANK_2]] : vector<1x2xf32> to !llvm.array<1 x vector<2xf32>>
+//       CHECK:  %[[UNDEF_VEC0:.*]] = llvm.mlir.poison : vector<2xf32>
+//       CHECK:  %[[C0_0:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:  %[[VEC0_0:.*]] = llvm.insertelement %[[ARG_0]], %[[UNDEF_VEC0]][%[[C0_0]] : i64] : vector<2xf32>
+//       CHECK:  %[[C1_0:.*]] = llvm.mlir.constant(1 : i64) : i64
+//       CHECK:  %[[VEC0_1:.*]] = llvm.insertelement %[[ARG_1]], %[[VEC0_0]][%[[C1_0]] : i64] : vector<2xf32>
+//       CHECK:  %[[RES_RANK_2_0:.*]] = llvm.insertvalue %[[VEC0_1]], %[[UNDEF_VEC_RANK_2_LLVM]][0] : !llvm.array<1 x vector<2xf32>>
+//       CHECK:  %[[RES_0:.*]] = llvm.insertvalue %[[RES_RANK_2_0]], %[[UNDEF_RES_LLVM]][0] : !llvm.array<2 x array<1 x vector<2xf32>>>
+//       CHECK:  %[[UNDEF_VEC1:.*]] = llvm.mlir.poison : vector<2xf32>
+//       CHECK:  %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64
+//       CHECK:  %[[VEC1_0:.*]] = llvm.insertelement %[[ARG_2]], %[[UNDEF_VEC1]][%[[C0_1]] : i64] : vector<2xf32>
+//       CHECK:  %[[C1_1:.*]] = llvm.mlir.constant(1 : i64) : i64
+//       CHECK:  %[[VEC1_1:.*]] = llvm.insertelement %[[ARG_3]], %[[VEC1_0]][%[[C1_1]] : i64] : vector<2xf32>
+//       CHECK:  %[[RES_RANK_2_1:.*]] = llvm.insertvalue %[[VEC1_1]], %[[UNDEF_VEC_RANK_2_LLVM]][0] : !llvm.array<1 x vector<2xf32>>
+//       CHECK:  %[[RES_1:.*]] = llvm.insertvalue %[[RES_RANK_2_1]], %[[RES_0]][1] : !llvm.array<2 x array<1 x vector<2xf32>>>
+//       CHECK:  %[[CAST:.*]] = builtin.unrealized_conversion_cast %[[RES_1]] : !llvm.array<2 x array<1 x vector<2xf32>>> to vector<2x1x2xf32>
+//       CHECK:  return %[[CAST]]
+func.func @from_elements_3d(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32) -> vector<2x1x2xf32> {
+  %0 = vector.from_elements %arg0, %arg1, %arg2, %arg3 : vector<2x1x2xf32>
+  return %0 : vector<2x1x2xf32>
+}
diff --git a/mlir/test/Conversion/VectorToXeGPU/transfer-read-to-xegpu.mlir b/mlir/test/Conversion/VectorToXeGPU/transfer-read-to-xegpu.mlir
index d1e5a62ad3e9b..b373bdab80567 100644
--- a/mlir/test/Conversion/VectorToXeGPU/transfer-read-to-xegpu.mlir
+++ b/mlir/test/Conversion/VectorToXeGPU/transfer-read-to-xegpu.mlir
@@ -1,212 +1,398 @@
-// RUN: mlir-opt %s -convert-vector-to-xegpu -split-input-file | FileCheck %s
+// RUN: mlir-opt %s --xevm-attach-target='module=xevm_* O=3 chip=pvc' -convert-vector-to-xegpu -split-input-file | FileCheck %s --check-prefix=LOAD-ND
+// RUN: mlir-opt %s -convert-vector-to-xegpu -split-input-file | FileCheck %s --check-prefix=LOAD-GATHER
 
-func.func @load_1D_vector(%source: memref<8x16x32xf32>, %offset: index) -> vector<8xf32> {
+gpu.module @xevm_module {
+gpu.func @load_1D_vector(%source: memref<8x16x32xf32>, %offset: index) -> vector<8xf32> {
   %c0 = arith.constant 0.0 : f32
   %0 = vector.transfer_read %source[%offset, %offset, %offset], %c0
     {in_bounds = [true]} : memref<8x16x32xf32>, vector<8xf32>
-  return %0 : vector<8xf32>
+  gpu.return %0 : vector<8xf32>
 }
 
-// CHECK-LABEL: @load_1D_vector(
-// CHECK-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
-// CHECK-SAME:  %[[OFFSET:.+]]: index
-// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
-// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
-// CHECK-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8xf32,
-// CHECK-SAME:    boundary_check = false
-// CHECK:       %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8xf32>
-// CHECK:       return %[[VEC]]
+// LOAD-ND-LABEL:  @load_1D_vector(
+// LOAD-ND-SAME:   %[[SRC:.+]]: memref<8x16x32xf32>,
+// LOAD-ND-SAME:   %[[OFFSET:.+]]: index
+// LOAD-ND:        %[[DESC:.+]] = xegpu.create_nd_tdesc
+// LOAD-ND-SAME:     %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// LOAD-ND-SAME:     memref<8x16x32xf32> -> !xegpu.tensor_desc<8xf32,
+// LOAD-ND-SAME:     boundary_check = false
+// LOAD-ND:        %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8xf32>
+// LOAD-ND:        return %[[VEC]]
+
+// LOAD-GATHER-LABEL:  @load_1D_vector(
+// LOAD-GATHER-SAME:   %[[SRC:.+]]: memref<8x16x32xf32>,
+// LOAD-GATHER:        %[[CST:.+]] = arith.constant dense<true> : vector<8xi1>
+// LOAD-GATHER:        %[[STEP:.+]] = vector.step : vector<8xindex>
+// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index
+// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index
+// LOAD-GATHER:        %[[SPLAT:.+]] = vector.broadcast {{.*}}:  index to vector<8xindex>
+// LOAD-GATHER:        %[[IDX:.+]] = arith.addi %[[SPLAT]], %[[STEP]] : vector<8xindex>
+// LOAD-GATHER:        %[[COLLAPSE:.+]] = memref.collapse_shape %[[SRC]] {{\[}}[0, 1, 2]{{\]}} : memref<8x16x32xf32> into memref<4096xf32>
+// LOAD-GATHER:        %[[VEC:.+]] = xegpu.load %[[COLLAPSE]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : memref<4096xf32>, vector<8xindex>, vector<8xi1> -> vector<8xf32>
 
-// -----
+}
 
-func.func @load_2D_vector(%source: memref<8x16x32xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @load_2D_vector(%source: memref<8x16x32xf32>,
     %offset: index) -> vector<8x16xf32> {
   %c0 = arith.constant 0.0 : f32
   %0 = vector.transfer_read %source[%offset, %offset, %offset], %c0
     {in_bounds = [true, true]} : memref<8x16x32xf32>, vector<8x16xf32>
-  return %0 : vector<8x16xf32>
+  gpu.return %0 : vector<8x16xf32>
 }
 
-// CHECK-LABEL: @load_2D_vector(
-// CHECK-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
-// CHECK-SAME:  %[[OFFSET:.+]]: index
-// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
-// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
-// CHECK-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8x16xf32,
-// CHECK-SAME:    boundary_check = false
-// CHECK:       %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8x16xf32>
-// CHECK:       return %[[VEC]]
+// LOAD-ND-LABEL:  @load_2D_vector(
+// LOAD-ND-SAME:   %[[SRC:.+]]: memref<8x16x32xf32>,
+// LOAD-ND-SAME:   %[[OFFSET:.+]]: index
+// LOAD-ND:        %[[DESC:.+]] = xegpu.create_nd_tdesc
+// LOAD-ND-SAME:     %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// LOAD-ND-SAME:     memref<8x16x32xf32> -> !xegpu.tensor_desc<8x16xf32,
+// LOAD-ND-SAME:     boundary_check = false
+// LOAD-ND:        %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8x16xf32>
+// LOAD-ND:        return %[[VEC]]
+
+// LOAD-GATHER-LABEL:  @load_2D_vector(
+// LOAD-GATHER-SAME:   %[[SRC:.+]]: memref<8x16x32xf32>,
+// LOAD-GATHER:        %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>
+// LOAD-GATHER-COUNT2: vector.step
+// LOAD-GATHER-COUNT2: vector.shape_cast
+// LOAD-GATHER-COUNT2: vector.broadcast
+// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index
+// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index
+// LOAD-GATHER:        %[[SPLAT:.+]] = vector.broadcast {{.*}}: index to vector<8x16xindex>
+// LOAD-GATHER:        %[[IDX:.+]] = arith.addi %[[SPLAT]], {{.*}}: vector<8x16xindex>
+// LOAD-GATHER:        %[[COLLAPSE:.+]] = memref.collapse_shape %[[SRC]] {{\[}}[0, 1, 2]{{\]}} : memref<8x16x32xf32> into memref<4096xf32>
+// LOAD-GATHER:        %[[VEC:.+]] = xegpu.load %[[COLLAPSE]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : memref<4096xf32>, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32>
 
-// -----
+}
 
-func.func @load_zero_pad_out_of_bounds(%source: memref<32x64xf32>,
+
+// -----
+gpu.module @xevm_module {
+gpu.func @load_zero_pad_out_of_bounds(%source: memref<32x64xf32>,
     %offset: index) -> vector<8x16xf32> {
   %c0 = arith.constant 0.0 : f32
   %0 = vector.transfer_read %source[%offset, %offset], %c0
     {in_bounds = [false, true]} : memref<32x64xf32>, vector<8x16xf32>
-  return %0 : vector<8x16xf32>
+  gpu.return %0 : vector<8x16xf32>
 }
 
-// CHECK-LABEL: @load_zero_pad_out_of_bounds(
-// CHECK-SAME:  %[[SRC:.+]]: memref<32x64xf32>,
-// CHECK-SAME:  %[[OFFSET:.+]]: index
-// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET]], %[[OFFSET]]]
-// CHECK-SAME:    memref<32x64xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK:       %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8x16xf32>
-// CHECK:       return %[[VEC]]
+// LOAD-ND-LABEL:  @load_zero_pad_out_of_bounds(
+// LOAD-ND-SAME:   %[[SRC:.+]]: memref<32x64xf32>,
+// LOAD-ND-SAME:   %[[OFFSET:.+]]: index
+// LOAD-ND:        %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET]], %[[OFFSET]]]
+// LOAD-ND-SAME:     memref<32x64xf32> -> !xegpu.tensor_desc<8x16xf32>
+// LOAD-ND:        %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8x16xf32>
+// LOAD-ND:        return %[[VEC]]
 
-// -----
+// LOAD-GATHER-LABEL:  @load_zero_pad_out_of_bounds(
+// LOAD-GATHER:        vector.transfer_read
 
-func.func @load_transposed(%source: memref<32x64xf32>,
-    %offset: index) -> vector<8x16xf32> {
+}
+
+
+// -----
+gpu.module @xevm_module {
+gpu.func @load_transposed(%source: memref<32x64xf32>,
+    %i: index, %j: index) -> vector<8x16xf32> {
   %c0 = arith.constant 0.0 : f32
-  %0 = vector.transfer_read %source[%offset, %offset], %c0
+  %0 = vector.transfer_read %source[%i, %j], %c0
     {permutation_map = affine_map<(d0, d1) -> (d1, d0)>,
     in_bounds = [true, true]} : memref<32x64xf32>, vector<8x16xf32>
-  return %0 : vector<8x16xf32>
+  gpu.return %0 : vector<8x16xf32>
 }
 
-// CHECK-LABEL: @load_transposed(
-// CHECK-SAME:  %[[SRC:.+]]: memref<32x64xf32>,
-// CHECK-SAME:  %[[OFFSET:.+]]: index
-// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET]], %[[OFFSET]]]
-// CHECK-SAME:    memref<32x64xf32> -> !xegpu.tensor_desc<16x8xf32
-// CHECK:       %[[VEC:.+]] = xegpu.load_nd %[[DESC]] <{transpose = array<i64: 1, 0>}>
-// CHECK-SAME:    -> vector<8x16xf32>
-// CHECK:       return %[[VEC]]
+// LOAD-ND-LABEL:  @load_transposed(
+// LOAD-ND-SAME:   %[[SRC:.+]]: memref<32x64xf32>,
+// LOAD-ND-SAME:   %[[OFFSET1:.+]]: index, 
+// LOAD-ND-SAME:   %[[OFFSET2:.+]]: index  
+// LOAD-ND:        %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET1]], %[[OFFSET2]]]
+// LOAD-ND-SAME:     memref<32x64xf32> -> !xegpu.tensor_desc<16x8xf32
+// LOAD-ND:        %[[VEC:.+]] = xegpu.load_nd %[[DESC]] <{transpose = array<i64: 1, 0>}>
+// LOAD-ND-SAME:     -> vector<8x16xf32>
+// LOAD-ND:        return %[[VEC]]
+
+
+// LOAD-GATHER-LABEL:  @load_transposed(
+// LOAD-GATHER-SAME:    %[[SRC:.+]]: memref<32x64xf32>,
+// LOAD-GATHER:         %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>
+// LOAD-GATHER-COUNT2:  vector.step
+// LOAD-GATHER-COUNT2:  vector.shape_cast
+// LOAD-GATHER-COUNT2: vector.broadcast
+// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index
+// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index
+// LOAD-GATHER:        %[[BCAST2:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>
+// LOAD-GATHER:        %[[IDX:.+]] = arith.addi %[[BCAST2]], {{.*}}: vector<8x16xindex>
+// LOAD-GATHER:        %[[COLLAPSE:.*]] = memref.collapse_shape %arg0 {{\[\[}}0, 1{{\]\]}} : memref<32x64xf32> into memref<2048xf32>
+// LOAD-GATHER:        %[[LOAD:.*]] = xegpu.load %[[COLLAPSE]][%[[IDX]]], %[[CST]] : memref<2048xf32>, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32>
 
-// -----
+}
 
-func.func @load_dynamic_source(%source: memref<?x?x?xf32>,
-    %offset: index) -> vector<8x16xf32> {
+// -----
+gpu.module @xevm_module {
+gpu.func @load_dynamic_source(%source: memref<?x?x?xf32>,
+    %i: index, %j: index, %k: index) -> vector<8x16xf32> {
   %c0 = arith.constant 0.0 : f32
-  %0 = vector.transfer_read %source[%offset, %offset, %offset], %c0
+  %0 = vector.transfer_read %source[%i, %j, %k], %c0
     {in_bounds = [true, true]} : memref<?x?x?xf32>, vector<8x16xf32>
-  return %0 : vector<8x16xf32>
-}
-
-// CHECK-LABEL: @load_dynamic_source(
-// CHECK-SAME:  %[[SRC:.+]]: memref<?x?x?xf32>,
-// CHECK-SAME:  %[[OFFSET:.+]]: index
-// CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
-// CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
-// CHECK-DAG:   %[[C2:.+]] = arith.constant 2 : index
-// CHECK-DAG:   %[[DIM_0:.+]] = memref.dim %[[SRC]], %[[C0]]
-// CHECK-DAG:   %[[DIM_1:.+]] = memref.dim %[[SRC]], %[[C1]]
-// CHECK-DAG:   %[[DIM_2:.+]] = memref.dim %[[SRC]], %[[C2]]
-// CHECK:       %[[DIM_0_STRIDE:.+]] = arith.muli %[[DIM_2]], %[[DIM_1]]
-// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
-// CHECK-SAME:  , shape : [%[[DIM_0]], %[[DIM_1]], %[[DIM_2]]], strides : [%[[DIM_0_STRIDE]], %[[DIM_2]], 1]
-// CHECK-SAME:    memref<?x?x?xf32> -> !xegpu.tensor_desc<8x16xf32
-// CHECK:       %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8x16xf32>
-// CHECK:       return %[[VEC]]
+  gpu.return %0 : vector<8x16xf32>
+}
+// LOAD-ND-LABEL:  @load_dynamic_source(
+// LOAD-ND-SAME:   %[[SRC:.+]]: memref<?x?x?xf32>,
+// LOAD-ND-SAME:   %[[OFFSET:.+]]: index
+// LOAD-ND:        %[[C2:.+]] = arith.constant 2 : index
+// LOAD-ND:        %[[C1:.+]] = arith.constant 1 : index
+// LOAD-ND:        %[[C0:.+]] = arith.constant 0 : index
+// LOAD-ND-DAG:    %[[DIM_0:.+]] = memref.dim %[[SRC]], %[[C0]]
+// LOAD-ND-DAG:    %[[DIM_1:.+]] = memref.dim %[[SRC]], %[[C1]]
+// LOAD-ND-DAG:    %[[DIM_2:.+]] = memref.dim %[[SRC]], %[[C2]]
+// LOAD-ND:        %[[DIM_0_STRIDE:.+]] = arith.muli %[[DIM_2]], %[[DIM_1]]
+// LOAD-ND:        %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET:.+]], %[[OFFSET:.+]], %[[OFFSET:.+]]]
+// LOAD-ND:        %[[VEC:.+]] = xegpu.load_nd %[[DESC]]{{.*}}-> vector<8x16xf32>
+// LOAD-ND:        return %[[VEC]]
+
+
+// LOAD-GATHER-LABEL:  @load_dynamic_source(
+// LOAD-GATHER-SAME:   %[[ARG0:.+]]: memref<?x?x?xf32>,
+// LOAD-GATHER:        %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>
+// LOAD-GATHER:        memref.extract_strided_metadata %[[ARG0]]
+// LOAD-GATHER-COUNT2: vector.step
+// LOAD-GATHER-COUNT2: vector.shape_cast
+// LOAD-GATHER-COUNT2: vector.broadcast
+// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index
+// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index
+// LOAD-GATHER:        %[[BROADIDX:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>
+// LOAD-GATHER:        %[[FINALIDX:.+]] = arith.addi %[[BROADIDX]], {{.*}} : vector<8x16xindex>
+// LOAD-GATHER:        %[[COLLAPSE:.+]] = memref.collapse_shape %[[ARG0]] {{\[}}[0, 1, 2]{{\]}} : memref<?x?x?xf32> into memref<?xf32>
+// LOAD-GATHER:        %[[RES:.+]] = xegpu.load %[[COLLAPSE]][%[[FINALIDX]]], %[[CST]] : memref<?xf32>, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32>
+// LOAD-GATHER:        gpu.return %[[RES]] : vector<8x16xf32>
+}
+
+// -----
+gpu.module @xevm_module {
+gpu.func @load_dynamic_source2(%source: memref<?x8x16xf32>,
+    %i: index, %j: index, %k: index) -> vector<8x16xf32> {
+  %c0 = arith.constant 0.0 : f32
+  %0 = vector.transfer_read %source[%i, %j, %k], %c0
+    {in_bounds = [true, true]} : memref<?x8x16xf32>, vector<8x16xf32>
+  gpu.return %0 : vector<8x16xf32>
+}
+
+// LOAD-ND-LABEL:  @load_dynamic_source2(
+// LOAD-ND-DAG:    %[[C0:.+]] = arith.constant 0 : index
+// LOAD-ND-DAG:    %[[DIM:.+]] = memref.dim %{{.*}}, %[[C0]] : memref<?x8x16xf32>
+// LOAD-ND:        %[[DESC:.+]] = xegpu.create_nd_tdesc %{{.*}}[%{{.*}}, %{{.*}}, %{{.*}}], shape : [%[[DIM]], 8, 16], strides : [128, 16, 1] : memref<?x8x16xf32> -> !xegpu.tensor_desc<8x16xf32, #xegpu.block_tdesc_attr<boundary_check = false>>
+// LOAD-ND:        %[[VEC:.+]] = xegpu.load_nd %[[DESC]] : !xegpu.tensor_desc<8x16xf32, #xegpu.block_tdesc_attr<boundary_check = false>> -> vector<8x16xf32>
+// LOAD-ND:        return %[[VEC]] : vector<8x16xf32>
+
+// LOAD-GATHER-LABEL:  @load_dynamic_source2(
+// LOAD-GATHER-DAG:    %[[CST_0:.+]] = arith.constant dense<true> : vector<8x16xi1>
+// LOAD-GATHER-COUNT2: vector.step
+// LOAD-GATHER-COUNT2: vector.shape_cast
+// LOAD-GATHER-COUNT2: vector.broadcast
+// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index
+// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index
+// LOAD-GATHER-DAG:    %[[BCASTIDX:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>
+// LOAD-GATHER-DAG:    %[[OFFSETS:.+]] = arith.addi %[[BCASTIDX]], {{.*}} : vector<8x16xindex>
+// LOAD-GATHER-DAG:    %[[COLLAPSE:.+]] = memref.collapse_shape %arg0 {{\[}}[0, 1, 2]{{\]}} : memref<?x8x16xf32> into memref<?xf32>
+// LOAD-GATHER:        %[[VEC:.+]] = xegpu.load %[[COLLAPSE]]{{\[}}%[[OFFSETS]]{{\]}}, %[[CST_0]] : memref<?xf32>, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf32> 
+
+}
+
+// -----
+gpu.module @xevm_module {
+gpu.func @load_dynamic_source3(%source: memref<?x?x?x?x?xf32>,
+    %i: index, %j: index, %k: index, %l: index, %m: index) -> vector<2x4x8x16xf32> {
+  %c0 = arith.constant 0.0 : f32
+  %0 = vector.transfer_read %source[%i, %j, %k, %l, %m], %c0
+    {in_bounds = [true, true, true, true]} : memref<?x?x?x?x?xf32>, vector<2x4x8x16xf32>
+  gpu.return %0 : vector<2x4x8x16xf32>
+}
+
+// LOAD-ND-LABEL:  @load_dynamic_source3(
+// LOAD-ND:        vector.transfer_read
+
+// LOAD-GATHER-LABEL:  @load_dynamic_source3(
+// LOAD-GATHER-SAME:   %[[SRC:.+]]: memref<?x?x?x?x?xf32>
+// LOAD-GATHER:        %[[CST:.+]] = arith.constant dense<true> : vector<2x4x8x16xi1>
+// LOAD-GATHER:        memref.extract_strided_metadata %[[SRC]] : memref<?x?x?x?x?xf32> -> memref<f32>, index, index, index, index, index, index, index, index, index, index, index
+// LOAD-GATHER-COUNT4: vector.step
+// LOAD-GATHER-COUNT3: vector.broadcast
+// LOAD-GATHER-COUNT4: vector.shape_cast
+// LOAD-GATHER-COUNT4: vector.broadcast {{.*}} : vector<2x4x8x16xindex>
+// LOAD-GATHER-COUNT3: arith.addi {{.*}} : vector<2x4x8x16xindex>
+// LOAD-GATHER:        %[[SPLAT:.+]] = vector.broadcast {{.*}} : index to vector<2x4x8x16xindex>
+// LOAD-GATHER:        %[[IDX:.+]] = arith.addi %[[SPLAT]], {{.*}} : vector<2x4x8x16xindex>
+// LOAD-GATHER:        %[[COLLAPSE:.+]] = memref.collapse_shape %[[SRC]] {{\[}}[0, 1, 2, 3, 4]{{\]}} : memref<?x?x?x?x?xf32> into memref<?xf32>
+// LOAD-GATHER:        %[[VEC:.+]] = xegpu.load %[[COLLAPSE]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : memref<?xf32>, vector<2x4x8x16xindex>, vector<2x4x8x16xi1> -> vector<2x4x8x16xf32>
+// LOAD-GATHER:        return %[[VEC]]
+}
+
+// -----
+gpu.module @xevm_module {
+gpu.func @load_high_dim_vector(%source: memref<16x32x64xf32>,
+    %offset: index, %arg2: index) -> vector<8x16x32xf32> {
+  %c0 = arith.constant 0.0 : f32
+  %0 = vector.transfer_read %source[%offset, %arg2, %offset], %c0
+    {in_bounds = [true, true, true]} : memref<16x32x64xf32>, vector<8x16x32xf32>
+  gpu.return %0 : vector<8x16x32xf32>
+}
+
+// LOAD-ND-LABEL:  @load_high_dim_vector(
+// LOAD-ND:        vector.transfer_read
+
+// LOAD-GATHER-LABEL:  @load_high_dim_vector(
+// LOAD-GATHER:        %[[CST:.+]] = arith.constant dense<true> : vector<8x16x32xi1>
+// LOAD-GATHER:        %[[CST_0:.+]] = arith.constant dense<64> : vector<16xindex>
+// LOAD-GATHER:        %[[CST_1:.+]] = arith.constant dense<2048> : vector<8xindex>
+// LOAD-GATHER:        %[[C2048:.+]] = arith.constant 2048 : index
+// LOAD-GATHER:        %[[C64:.+]] = arith.constant 64 : index
+// LOAD-GATHER-COUNT3: vector.step
+// LOAD-GATHER-COUNT3: vector.shape_cast
+// LOAD-GATHER-COUNT3: vector.broadcast {{.*}} : vector<8x16x32xindex>
+// LOAD-GATHER-COUNT2: arith.addi {{.*}} : vector<8x16x32xindex>
+// LOAD-GATHER:        %[[BCASTOFF:.+]] = vector.broadcast {{.*}} : index to vector<8x16x32xindex>
+// LOAD-GATHER:        %[[IDX:.+]] = arith.addi %[[BCASTOFF]], {{.*}} : vector<8x16x32xindex>
+// LOAD-GATHER:        %[[COLLAPSE:.+]] = memref.collapse_shape %arg0 {{\[}}[0, 1, 2]{{\]}} : memref<16x32x64xf32> into memref<32768xf32>
+// LOAD-GATHER:        %[[VEC:.+]] = xegpu.load %[[COLLAPSE]][%[[IDX]]], %[[CST]] : memref<32768xf32>, vector<8x16x32xindex>, vector<8x16x32xi1> -> vector<8x16x32xf32>
+
+}
 
 // -----
+gpu.module @xevm_module {
+gpu.func @load_transpose_f16(%source: memref<32x64xf16>,
+    %offset: index) -> vector<8x16xf16> {
+  %c0 = arith.constant 0.0 : f16
+  %0 = vector.transfer_read %source[%offset, %offset], %c0
+    {permutation_map = affine_map<(d0, d1) -> (d1, d0)>,
+    in_bounds = [true, true]} : memref<32x64xf16>, vector<8x16xf16>
+  gpu.return %0 : vector<8x16xf16>
+}
 
-func.func @no_load_out_of_bounds_non_zero_pad(%source: memref<32x64xf32>,
+// LOAD-ND-LABEL:  @load_transpose_f16(
+// LOAD-ND:        vector.transfer_read
+
+// LOAD-GATHER-LABEL:  @load_transpose_f16(
+// LOAD-GATHER-SAME:    %[[SRC:.+]]: memref<32x64xf16>,
+// LOAD-GATHER:         %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>
+// LOAD-GATHER-COUNT2:  vector.step
+// LOAD-GATHER-COUNT2:  vector.shape_cast
+// LOAD-GATHER-COUNT2: vector.broadcast
+// LOAD-GATHER-COUNT2: arith.muli {{.*}} : index
+// LOAD-GATHER-COUNT2: arith.addi {{.*}} : index
+// LOAD-GATHER:        %[[BCAST2:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>
+// LOAD-GATHER:        %[[IDX:.+]] = arith.addi %[[BCAST2]], {{.*}}: vector<8x16xindex>
+// LOAD-GATHER:        %[[COLLAPSE:.*]] = memref.collapse_shape %arg0 {{\[\[}}0, 1{{\]\]}} : memref<32x64xf16> into memref<2048xf16>
+// LOAD-GATHER:        %[[LOAD:.*]] = xegpu.load %[[COLLAPSE]][%[[IDX]]], %[[CST]] : memref<2048xf16>, vector<8x16xindex>, vector<8x16xi1> -> vector<8x16xf16>
+}
+
+// -----
+gpu.module @xevm_module {
+gpu.func @no_load_out_of_bounds_non_zero_pad(%source: memref<32x64xf32>,
     %offset: index, %arg2: index, %pad: f32) -> (vector<8x16xf32>, vector<8x16xf32>) {
   %c1 = arith.constant 1.0 : f32
   %0 = vector.transfer_read %source[%offset, %arg2], %c1
     {in_bounds = [true, false]} : memref<32x64xf32>, vector<8x16xf32>
   %1 = vector.transfer_read %source[%arg2, %offset], %pad
     {in_bounds = [false, true]} : memref<32x64xf32>, vector<8x16xf32>
-  return %0, %1 : vector<8x16xf32>, vector<8x16xf32>
+  gpu.return %0, %1 : vector<8x16xf32>, vector<8x16xf32>
 }
 
-// CHECK-LABEL:   @no_load_out_of_bounds_non_zero_pad(
-// CHECK-COUNT-2: vector.transfer_read
+// LOAD-ND-LABEL:    @no_load_out_of_bounds_non_zero_pad(
+// LOAD-ND-COUNT-2: vector.transfer_read
 
-// -----
+// LOAD-GATHER-LABEL: @no_load_out_of_bounds_non_zero_pad(
+// LOAD-GATHER-COUNT-2: vector.transfer_read
+}
 
-func.func @no_load_out_of_bounds_1D_vector(%source: memref<8x16x32xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @no_load_out_of_bounds_1D_vector(%source: memref<8x16x32xf32>,
     %offset: index) -> vector<8xf32> {
   %c0 = arith.constant 0.0 : f32
   %0 = vector.transfer_read %source[%offset, %offset, %offset], %c0
     {in_bounds = [false]} : memref<8x16x32xf32>, vector<8xf32>
-  return %0 : vector<8xf32>
+  gpu.return %0 : vector<8xf32>
 }
 
-// CHECK-LABEL: @no_load_out_of_bounds_1D_vector(
-// CHECK:       vector.transfer_read
+// LOAD-ND-LABEL:  @no_load_out_of_bounds_1D_vector(
+// LOAD-ND:        vector.transfer_read
 
-// -----
+// LOAD-GATHER-LABEL:  @no_load_out_of_bounds_1D_vector(
+// LOAD-GATHER:        vector.transfer_read
+}
 
-func.func @no_load_masked(%source : memref<4xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @no_load_masked(%source : memref<4xf32>,
     %offset : index) -> vector<4xf32> {
   %c0 = arith.constant 0.0 : f32
   %mask = arith.constant dense<[0, 1, 0, 1]> : vector<4xi1>
   %0 = vector.transfer_read %source[%offset], %c0, %mask
     {in_bounds = [true]} : memref<4xf32>, vector<4xf32>
-  return %0 : vector<4xf32>
+  gpu.return %0 : vector<4xf32>
 }
 
-// CHECK-LABEL: @no_load_masked(
-// CHECK:       vector.transfer_read
+// LOAD-ND-LABEL:  @no_load_masked(
+// LOAD-ND:        vector.transfer_read
 
-// -----
+// LOAD-GATHER-LABEL:  @no_load_masked(
+// LOAD-GATHER:        vector.transfer_read
+}
 
-func.func @no_load_tensor(%source: tensor<32x64xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @no_load_tensor(%source: tensor<32x64xf32>,
     %offset: index, %arg2: index) -> vector<8x16xf32> {
   %c0 = arith.constant 0.0 : f32
   %0 = vector.transfer_read %source[%offset, %arg2], %c0
     {in_bounds = [true, true]} : tensor<32x64xf32>, vector<8x16xf32>
-  return %0 : vector<8x16xf32>
+  gpu.return %0 : vector<8x16xf32>
 }
 
-// CHECK-LABEL: @no_load_tensor(
-// CHECK:       vector.transfer_read
+// LOAD-ND-LABEL:  @no_load_tensor(
+// LOAD-ND:        vector.transfer_read
 
-// -----
-
-func.func @no_load_high_dim_vector(%source: memref<16x32x64xf32>,
-    %offset: index, %arg2: index) -> vector<8x16x32xf32> {
-  %c0 = arith.constant 0.0 : f32
-  %0 = vector.transfer_read %source[%offset, %arg2, %offset], %c0
-    {in_bounds = [true, true, true]} : memref<16x32x64xf32>, vector<8x16x32xf32>
-  return %0 : vector<8x16x32xf32>
+// LOAD-GATHER-LABEL:  @no_load_tensor(
+// LOAD-GATHER:        vector.transfer_read
 }
 
-// CHECK-LABEL: @no_load_high_dim_vector(
-// CHECK:       vector.transfer_read
 
 // -----
-
-func.func @no_load_non_unit_inner_stride(
+gpu.module @xevm_module {
+gpu.func @no_load_non_unit_inner_stride(
     %source: memref<32xf32, strided<[?], offset: ?>>,
     %offset: index) -> vector<8xf32> {
   %c0 = arith.constant 0.0 : f32
   %0 = vector.transfer_read %source[%offset], %c0 {in_bounds = [true]}
     : memref<32xf32, strided<[?], offset: ?>>, vector<8xf32>
-  return %0 : vector<8xf32>
+  gpu.return %0 : vector<8xf32>
 }
 
-// CHECK-LABEL: @no_load_non_unit_inner_stride(
-// CHECK:       vector.transfer_read
+// LOAD-ND-LABEL:  @no_load_non_unit_inner_stride(
+// LOAD-ND:        vector.transfer_read
+
+// LOAD-GATHER-LABEL:  @no_load_non_unit_inner_stride(
+// LOAD-GATHER:        vector.transfer_read
+}
 
-// -----
 
-func.func @no_load_unsupported_map(%source: memref<16x32x64xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @no_load_unsupported_map(%source: memref<16x32x64xf32>,
     %offset: index) -> vector<8x16xf32> {
   %c0 = arith.constant 0.0 : f32
   %0 = vector.transfer_read %source[%offset, %offset, %offset], %c0
     {permutation_map = affine_map<(d0, d1, d2) -> (d0, d2)>,
     in_bounds = [true, true]} : memref<16x32x64xf32>, vector<8x16xf32>
-  return %0 : vector<8x16xf32>
+  gpu.return %0 : vector<8x16xf32>
 }
 
-// CHECK-LABEL: @no_load_unsupported_map(
-// CHECK:       vector.transfer_read
-
-// -----
+// LOAD-ND-LABEL:  @no_load_unsupported_map(
+// LOAD-ND:        vector.transfer_read
 
-func.func @no_load_transpose_unsupported_data_type(%source: memref<32x64xf16>,
-    %offset: index) -> vector<8x16xf16> {
-  %c0 = arith.constant 0.0 : f16
-  %0 = vector.transfer_read %source[%offset, %offset], %c0
-    {permutation_map = affine_map<(d0, d1) -> (d1, d0)>,
-    in_bounds = [true, true]} : memref<32x64xf16>, vector<8x16xf16>
-  return %0 : vector<8x16xf16>
+// LOAD-GATHER-LABEL:  @no_load_unsupported_map(
+// LOAD-GATHER:        vector.transfer_read
 }
 
-// CHECK-LABEL: @no_load_transpose_unsupported_data_type(
-// CHECK:       vector.transfer_read
diff --git a/mlir/test/Conversion/VectorToXeGPU/transfer-write-to-xegpu.mlir b/mlir/test/Conversion/VectorToXeGPU/transfer-write-to-xegpu.mlir
index d5f1221aebed5..b3f761a545ee1 100644
--- a/mlir/test/Conversion/VectorToXeGPU/transfer-write-to-xegpu.mlir
+++ b/mlir/test/Conversion/VectorToXeGPU/transfer-write-to-xegpu.mlir
@@ -1,178 +1,278 @@
-// RUN: mlir-opt %s -convert-vector-to-xegpu -split-input-file | FileCheck %s
+// RUN: mlir-opt %s --xevm-attach-target='module=xevm.* O=3 chip=pvc' -convert-vector-to-xegpu -split-input-file | FileCheck %s --check-prefix=STORE-ND
+// RUN: mlir-opt %s -convert-vector-to-xegpu -split-input-file | FileCheck %s --check-prefix=STORE-SCATTER
 
-func.func @store_1D_vector(%vec: vector<8xf32>,
+
+gpu.module @xevm_module {
+gpu.func @store_1D_vector(%vec: vector<8xf32>,
     %source: memref<8x16x32xf32>, %offset: index) {
   vector.transfer_write %vec, %source[%offset, %offset, %offset]
     {in_bounds = [true]}
     : vector<8xf32>, memref<8x16x32xf32>
-  return
+  gpu.return
 }
 
-// CHECK-LABEL: @store_1D_vector(
-// CHECK-SAME:  %[[VEC:.+]]: vector<8xf32>,
-// CHECK-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
-// CHECK-SAME:  %[[OFFSET:.+]]: index
-// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
-// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
-// CHECK-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8xf32,
-// CHECK-SAME:    boundary_check = false
-// CHECK:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8xf32>
+// STORE-ND-LABEL: @store_1D_vector(
+// STORE-ND-SAME:  %[[VEC:.+]]: vector<8xf32>,
+// STORE-ND-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
+// STORE-ND-SAME:  %[[OFFSET:.+]]: index
+// STORE-ND:       %[[DESC:.+]] = xegpu.create_nd_tdesc
+// STORE-ND-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// STORE-ND-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8xf32,
+// STORE-ND-SAME:    boundary_check = false
+// STORE-ND:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8xf32>
+
+// STORE-SCATTER-LABEL:  @store_1D_vector(
+// STORE-SCATTER-SAME:   %[[VEC:.+]]: vector<8xf32>,
+// STORE-SCATTER-SAME:   %[[SRC:.+]]: memref<8x16x32xf32>,
+// STORE-SCATTER-DAG:        %[[CST:.+]] = arith.constant dense<true> : vector<8xi1>
+// STORE-SCATTER-DAG:        %[[STEP:.+]] = vector.step
+// STORE-SCATTER-COUNT2: arith.muli {{.*}} : index
+// STORE-SCATTER-COUNT2: arith.addi {{.*}} : index
+// STORE-SCATTER-DAG:    %[[BCAST:.+]] = vector.broadcast {{.*}} : index to vector<8xindex>
+// STORE-SCATTER-DAG:    %[[IDX:.+]] = arith.addi %[[BCAST]], %{{.*}} : vector<8xindex>
+// STORE-SCATTER-DAG:    %[[COLLAPSE:.+]] = memref.collapse_shape %[[SRC]] {{\[}}[0, 1, 2]{{\]}} : memref<8x16x32xf32> into memref<4096xf32>
+// STORE-SCATTER:       xegpu.store %[[VEC]], %[[COLLAPSE]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : vector<8xf32>, memref<4096xf32>, vector<8xindex>, vector<8xi1>
+}
 
 // -----
-
-func.func @store_2D_vector(%vec: vector<8x16xf32>,
+gpu.module @xevm_module {
+gpu.func @store_2D_vector(%vec: vector<8x16xf32>,
     %source: memref<8x16x32xf32>, %offset: index) {
   vector.transfer_write %vec, %source[%offset, %offset, %offset]
     {in_bounds = [true, true]}
     : vector<8x16xf32>, memref<8x16x32xf32>
-  return
+  gpu.return
 }
 
-// CHECK-LABEL: @store_2D_vector(
-// CHECK-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
-// CHECK-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
-// CHECK-SAME:  %[[OFFSET:.+]]: index
-// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
-// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
-// CHECK-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8x16xf32,
-// CHECK-SAME:    boundary_check = false
-// CHECK:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8x16xf32>
+// STORE-ND-LABEL: @store_2D_vector(
+// STORE-ND-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
+// STORE-ND-SAME:  %[[SRC:.+]]: memref<8x16x32xf32>,
+// STORE-ND-SAME:  %[[OFFSET:.+]]: index
+// STORE-ND:       %[[DESC:.+]] = xegpu.create_nd_tdesc
+// STORE-ND-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// STORE-ND-SAME:    memref<8x16x32xf32> -> !xegpu.tensor_desc<8x16xf32,
+// STORE-ND-SAME:    boundary_check = false
+// STORE-ND:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8x16xf32>
+
+// STORE-SCATTER-LABEL:  @store_2D_vector(
+// STORE-SCATTER-SAME:   %[[VEC:.+]]: vector<8x16xf32>,
+// STORE-SCATTER-SAME:   %[[SRC:.+]]: memref<8x16x32xf32>,
+// STORE-SCATTER-SAME:   %[[OFFSET:.+]]: index
+// STORE-SCATTER:        %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>
+// STORE-SCATTER-COUNT2: %[[STEP:.+]] = vector.step
+// STORE-SCATTER-COUNT2: vector.shape_cast {{.*}}
+// STORE-SCATTER-COUNT2: vector.broadcast {{.*}} : vector<8x16xindex>
+// STORE-SCATTER-DAG:    %[[BCAST2:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>
+// STORE-SCATTER-DAG:    %[[IDX:.+]] = arith.addi %[[BCAST2]], {{.*}} : vector<8x16xindex>
+// STORE-SCATTER-DAG:    %[[COLLAPSE:.+]] = memref.collapse_shape %[[SRC]] {{\[}}[0, 1, 2]{{\]}} : memref<8x16x32xf32> into memref<4096xf32>
+// STORE-SCATTER:        xegpu.store %[[VEC]], %[[COLLAPSE]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : vector<8x16xf32>, memref<4096xf32>, vector<8x16xindex>, vector<8x16xi1>
+}
 
 // -----
-
-func.func @store_dynamic_source(%vec: vector<8x16xf32>,
+gpu.module @xevm_module {
+gpu.func @store_dynamic_source(%vec: vector<8x16xf32>,
     %source: memref<?x?x?xf32>, %offset: index) {
   vector.transfer_write %vec, %source[%offset, %offset, %offset]
     {in_bounds = [true, true]}
     : vector<8x16xf32>, memref<?x?x?xf32>
-  return
-}
-
-// CHECK-LABEL: @store_dynamic_source(
-// CHECK-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
-// CHECK-SAME:  %[[SRC:.+]]: memref<?x?x?xf32>,
-// CHECK-SAME:  %[[OFFSET:.+]]: index
-// CHECK-DAG:   %[[C0:.+]] = arith.constant 0 : index
-// CHECK-DAG:   %[[C1:.+]] = arith.constant 1 : index
-// CHECK-DAG:   %[[C2:.+]] = arith.constant 2 : index
-// CHECK-DAG:   %[[DIM_0:.+]] = memref.dim %[[SRC]], %[[C0]]
-// CHECK-DAG:   %[[DIM_1:.+]] = memref.dim %[[SRC]], %[[C1]]
-// CHECK-DAG:   %[[DIM_2:.+]] = memref.dim %[[SRC]], %[[C2]]
-// CHECK:       %[[DIM_0_STRIDE:.+]] = arith.muli %[[DIM_2]], %[[DIM_1]]
-// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
-// CHECK-SAME:  , shape : [%[[DIM_0]], %[[DIM_1]], %[[DIM_2]]], strides : [%[[DIM_0_STRIDE]], %[[DIM_2]], 1]
-// CHECK-SAME:    memref<?x?x?xf32> -> !xegpu.tensor_desc<8x16xf32
-// CHECK:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8x16xf32>
+  gpu.return
+}
 
-// -----
+// STORE-ND-LABEL: @store_dynamic_source(
+// STORE-ND-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
+// STORE-ND-SAME:  %[[SRC:.+]]: memref<?x?x?xf32>,
+// STORE-ND-SAME:  %[[OFFSET:.+]]: index
+// STORE-ND-DAG:   %[[C0:.+]] = arith.constant 0 : index
+// STORE-ND-DAG:   %[[C1:.+]] = arith.constant 1 : index
+// STORE-ND-DAG:   %[[C2:.+]] = arith.constant 2 : index
+// STORE-ND-DAG:   %[[DIM_0:.+]] = memref.dim %[[SRC]], %[[C0]]
+// STORE-ND-DAG:   %[[DIM_1:.+]] = memref.dim %[[SRC]], %[[C1]]
+// STORE-ND-DAG:   %[[DIM_2:.+]] = memref.dim %[[SRC]], %[[C2]]
+// STORE-ND:       %[[DIM_0_STRIDE:.+]] = arith.muli %[[DIM_2]], %[[DIM_1]]
+// STORE-ND:       %[[DESC:.+]] = xegpu.create_nd_tdesc %[[SRC]][%[[OFFSET]], %[[OFFSET]], %[[OFFSET]]]
+// STORE-ND-SAME:  , shape : [%[[DIM_0]], %[[DIM_1]], %[[DIM_2]]], strides : [%[[DIM_0_STRIDE]], %[[DIM_2]], 1]
+// STORE-ND-SAME:    memref<?x?x?xf32> -> !xegpu.tensor_desc<8x16xf32
+// STORE-ND:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8x16xf32>
+
+// STORE-SCATTER-LABEL: @store_dynamic_source(
+// STORE-SCATTER-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
+// STORE-SCATTER-SAME:  %[[SRC:.+]]: memref<?x?x?xf32>,
+// STORE-SCATTER-DAG:   %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>
+// STORE-SCATTER-DAG:   memref.extract_strided_metadata %[[SRC]] : memref<?x?x?xf32> -> memref<f32>, index, index, index, index, index, index, index
+// STORE-SCATTER-COUNT2: %[[STEP:.+]] = vector.step
+// STORE-SCATTER-COUNT2: vector.shape_cast {{.*}}
+// STORE-SCATTER-COUNT2: vector.broadcast {{.*}} : vector<8x16xindex>
+// STORE-SCATTER-DAG:   %[[BCAST2:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>
+// STORE-SCATTER-DAG:   %[[IDX:.+]] = arith.addi %[[BCAST2]], {{.*}} : vector<8x16xindex>
+// STORE-SCATTER-DAG:   %[[COLLAPSE:.+]] = memref.collapse_shape %[[SRC]] {{\[}}[0, 1, 2]{{\]}} : memref<?x?x?xf32> into memref<?xf32>
+// STORE-SCATTER:       xegpu.store %[[VEC]], %[[COLLAPSE]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : vector<8x16xf32>, memref<?xf32>, vector<8x16xindex>, vector<8x16xi1>
+}
 
-func.func @store_out_of_bounds(%vec: vector<8x16xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @store_out_of_bounds(%vec: vector<8x16xf32>,
     %source: memref<7x64xf32>, %offset: index) {
   vector.transfer_write %vec, %source[%offset, %offset]
     {in_bounds = [false, true]}
     : vector<8x16xf32>, memref<7x64xf32>
-  return
+  gpu.return
 }
 
-// CHECK-LABEL:   @store_out_of_bounds(
-// CHECK-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
-// CHECK-SAME:  %[[SRC:.+]]: memref<7x64xf32>,
-// CHECK-SAME:  %[[OFFSET:.+]]: index
-// CHECK:       %[[DESC:.+]] = xegpu.create_nd_tdesc
-// CHECK-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]]]
-// CHECK-SAME:    memref<7x64xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8x16xf32>
+// STORE-ND-LABEL:   @store_out_of_bounds(
+// STORE-ND-SAME:  %[[VEC:.+]]: vector<8x16xf32>,
+// STORE-ND-SAME:  %[[SRC:.+]]: memref<7x64xf32>,
+// STORE-ND-SAME:  %[[OFFSET:.+]]: index
+// STORE-ND:       %[[DESC:.+]] = xegpu.create_nd_tdesc
+// STORE-ND-SAME:    %[[SRC]][%[[OFFSET]], %[[OFFSET]]]
+// STORE-ND-SAME:    memref<7x64xf32> -> !xegpu.tensor_desc<8x16xf32>
+// STORE-ND:       xegpu.store_nd %[[VEC]], %[[DESC]] : vector<8x16xf32>
+
+// STORE-SCATTER-LABEL:  @store_out_of_bounds(
+// STORE-SCATTER:   vector.transfer_write
+}
 
 // -----
-
-func.func @no_store_transposed(%vec: vector<8x16xf32>,
+gpu.module @xevm_module {
+gpu.func @no_store_transposed(%vec: vector<8x16xf32>,
     %source: memref<32x64xf32>, %offset: index) {
   vector.transfer_write %vec, %source[%offset, %offset]
     {permutation_map = affine_map<(d0, d1) -> (d1, d0)>,
     in_bounds = [true, true]}
     : vector<8x16xf32>, memref<32x64xf32>
-  return
+  gpu.return
 }
 
-// CHECK-LABEL: @no_store_transposed(
-// CHECK:       vector.transfer_write
+// STORE-ND-LABEL: @no_store_transposed(
+// STORE-ND:       vector.transfer_write
+
+// STORE-SCATTER-LABEL:  @no_store_transposed(
+// STORE-SCATTER-SAME:   %[[VEC:.+]]: vector<8x16xf32>,
+// STORE-SCATTER-SAME:   %[[SRC:.+]]: memref<32x64xf32>,
+// STORE-SCATTER-SAME:   %[[OFFSET:.+]]: index
+// STORE-SCATTER:        %[[CST:.+]] = arith.constant dense<true> : vector<8x16xi1>
+// STORE-SCATTER-COUNT2: %[[STEP:.+]] = vector.step
+// STORE-SCATTER-COUNT2: vector.shape_cast {{.*}}
+// STORE-SCATTER-COUNT2: vector.broadcast {{.*}} : vector<8x16xindex>
+// STORE-SCATTER-DAG:    %[[BCAST2:.+]] = vector.broadcast {{.*}} : index to vector<8x16xindex>
+// STORE-SCATTER-DAG:    %[[IDX:.+]] = arith.addi %[[BCAST2]], {{.*}} : vector<8x16xindex>
+// STORE-SCATTER-DAG:    %[[COLLAPSE:.+]] = memref.collapse_shape %[[SRC]] {{\[}}[0, 1]{{\]}} : memref<32x64xf32> into memref<2048xf32>
+// STORE-SCATTER:        xegpu.store %[[VEC]], %[[COLLAPSE]]{{\[}}%[[IDX]]{{\]}}, %[[CST]] : vector<8x16xf32>, memref<2048xf32>, vector<8x16xindex>, vector<8x16xi1>
+}
 
 // -----
+gpu.module @xevm_module {
+gpu.func @store_high_dim_vector(%vec: vector<8x16x32xf32>,
+    %source: memref<16x32x64xf32>, %offset: index) {
+  vector.transfer_write %vec, %source[%offset, %offset, %offset]
+    {in_bounds = [true, true, true]}
+    : vector<8x16x32xf32>, memref<16x32x64xf32>
+  gpu.return
+}
+
+// STORE-ND-LABEL: @store_high_dim_vector(
+// STORE-ND:       vector.transfer_write
+
+// STORE-SCATTER-LABEL:  @store_high_dim_vector(
+// STORE-SCATTER-SAME:   %[[VEC:.+]]: vector<8x16x32xf32>,
+// STORE-SCATTER-SAME:   %[[SRC:.+]]: memref<16x32x64xf32>
+// STORE-SCATTER:        %[[CST:.+]] = arith.constant dense<true> : vector<8x16x32xi1>
+// STORE-SCATTER:        %[[CST_0:.+]] = arith.constant dense<64> : vector<16xindex>
+// STORE-SCATTER:        %[[CST_1:.+]] = arith.constant dense<2048> : vector<8xindex>
+// STORE-SCATTER:        %[[C2048:.+]] = arith.constant 2048 : index
+// STORE-SCATTER:        %[[C64:.+]] = arith.constant 64 : index
+// STORE-SCATTER-COUNT3: vector.step
+// STORE-SCATTER-COUNT3: vector.shape_cast
+// STORE-SCATTER-COUNT3: vector.broadcast {{.*}} : vector<8x16x32xindex>
+// STORE-SCATTER-COUNT2: arith.addi {{.*}} : vector<8x16x32xindex>
+// STORE-SCATTER:        %[[BCASTOFF:.+]] = vector.broadcast {{.*}} : index to vector<8x16x32xindex>
+// STORE-SCATTER:        %[[IDX:.+]] = arith.addi %[[BCASTOFF]], {{.*}} : vector<8x16x32xindex>
+// STORE-SCATTER:        %[[COLLAPSE:.+]] = memref.collapse_shape %[[SRC]] {{\[}}[0, 1, 2]{{\]}} : memref<16x32x64xf32> into memref<32768xf32>
+// STORE-SCATTER:        xegpu.store %[[VEC]], %[[COLLAPSE]][%[[IDX]]], %[[CST]] : vector<8x16x32xf32>, memref<32768xf32>, vector<8x16x32xindex>, vector<8x16x32xi1> 
+}
 
-func.func @no_store_masked(%vec: vector<4xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @no_store_masked(%vec: vector<4xf32>,
     %source: memref<4xf32>, %offset: index) {
   %mask = arith.constant dense<[0, 1, 0, 1]> : vector<4xi1>
   vector.transfer_write %vec, %source[%offset], %mask
     {in_bounds = [true]}
     : vector<4xf32>, memref<4xf32>
-  return
+  gpu.return
 }
 
-// CHECK-LABEL: @no_store_masked(
-// CHECK:       vector.transfer_write
+// STORE-ND-LABEL: @no_store_masked(
+// STORE-ND:       vector.transfer_write
 
-// -----
+// STORE-SCATTER-LABEL:  @no_store_masked(
+// STORE-SCATTER:        vector.transfer_write
+}
 
-func.func @no_store_tensor(%vec: vector<8x16xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @no_store_tensor(%vec: vector<8x16xf32>,
     %source: tensor<32x64xf32>, %offset: index) -> tensor<32x64xf32> {
   %0 = vector.transfer_write %vec, %source[%offset, %offset]
     {in_bounds = [true, true]}
     : vector<8x16xf32>, tensor<32x64xf32>
-  return %0 : tensor<32x64xf32>
+  gpu.return %0 : tensor<32x64xf32>
 }
 
-// CHECK-LABEL: @no_store_tensor(
-// CHECK:       vector.transfer_write
-
-// -----
+// STORE-ND-LABEL: @no_store_tensor(
+// STORE-ND:       vector.transfer_write
 
-func.func @no_store_high_dim_vector(%vec: vector<8x16x32xf32>,
-    %source: memref<16x32x64xf32>, %offset: index) {
-  vector.transfer_write %vec, %source[%offset, %offset, %offset]
-    {in_bounds = [true, true, true]}
-    : vector<8x16x32xf32>, memref<16x32x64xf32>
-  return
+// STORE-SCATTER-LABEL:  @no_store_tensor(
+// STORE-SCATTER:        vector.transfer_write
 }
 
-// CHECK-LABEL: @no_store_high_dim_vector(
-// CHECK:       vector.transfer_write
-
 // -----
-
-func.func @no_store_non_unit_inner_stride(%vec: vector<8xf32>,
+gpu.module @xevm_module {
+gpu.func @no_store_non_unit_inner_stride(%vec: vector<8xf32>,
     %source: memref<32xf32, strided<[?], offset: ?>>, %offset: index) {
   vector.transfer_write %vec, %source[%offset]
     {in_bounds = [true]}
     : vector<8xf32>, memref<32xf32, strided<[?], offset: ?>>
-  return
+  gpu.return
 }
 
-// CHECK-LABEL: @no_store_non_unit_inner_stride(
-// CHECK:       vector.transfer_write
+// STORE-ND-LABEL: @no_store_non_unit_inner_stride(
+// STORE-ND:       vector.transfer_write
 
-// -----
+// STORE-SCATTER-LABEL:  @no_store_non_unit_inner_stride(
+// STORE-SCATTER:        vector.transfer_write
+}
 
-func.func @no_store_unsupported_map(%vec: vector<8x16xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @no_store_unsupported_map(%vec: vector<8x16xf32>,
     %source: memref<16x32x64xf32>, %offset: index) {
   vector.transfer_write %vec, %source[%offset, %offset, %offset]
     {permutation_map = affine_map<(d0, d1, d2) -> (d0, d2)>,
     in_bounds = [true, true]}
     : vector<8x16xf32>, memref<16x32x64xf32>
-  return
+  gpu.return
 }
 
-// CHECK-LABEL: @no_store_unsupported_map(
-// CHECK:       vector.transfer_write
+// STORE-ND-LABEL: @no_store_unsupported_map(
+// STORE-ND:       vector.transfer_write
 
-// -----
+// STORE-SCATTER-LABEL:  @no_store_unsupported_map(
+// STORE-SCATTER:        vector.transfer_write
+}
 
-func.func @no_store_out_of_bounds_1D_vector(%vec: vector<8xf32>,
+// -----
+gpu.module @xevm_module {
+gpu.func @no_store_out_of_bounds_1D_vector(%vec: vector<8xf32>,
     %source: memref<8x16x32xf32>, %offset: index) {
   vector.transfer_write %vec, %source[%offset, %offset, %offset]
     {in_bounds = [false]}
     : vector<8xf32>, memref<8x16x32xf32>
-  return
+  gpu.return
 }
 
-// CHECK-LABEL: @no_store_out_of_bounds_1D_vector(
-// CHECK:       vector.transfer_write
+// STORE-ND-LABEL: @no_store_out_of_bounds_1D_vector(
+// STORE-ND:       vector.transfer_write
+
+// STORE-SCATTER-LABEL:  @no_store_out_of_bounds_1D_vector(
+// STORE-SCATTER:        vector.transfer_write
+}
\ No newline at end of file
diff --git a/mlir/test/Dialect/AMDGPU/amdgpu-fold-memrefs.mlir b/mlir/test/Dialect/AMDGPU/amdgpu-fold-memrefs.mlir
index 57afa127c9da8..8ca3dd60414df 100644
--- a/mlir/test/Dialect/AMDGPU/amdgpu-fold-memrefs.mlir
+++ b/mlir/test/Dialect/AMDGPU/amdgpu-fold-memrefs.mlir
@@ -54,18 +54,20 @@ func.func @subview_folding_offset(%offset_i: index, %offset_j: index) {
 // CHECK: func @test_expand_shape
 // CHECK-SAME: %[[ARG0:.*]]: index, %[[ARG1:.*]]: index
 func.func @test_expand_shape(%offset_i: index, %offset_j: index) {
-  // CHECK: %[[LOCAL:.*]] = memref.alloc() : memref<64x64xf16, 3>
+  // CHECK: %[[LOCAL:.*]] = memref.alloc() : memref<4096xf16, 3>
   // CHECK: %[[MEM:.*]] = memref.alloc() : memref<8192xf16>
   // CHECK: %[[C0:.*]] = arith.constant 0 : index
-  // CHECK: %[[IDX:.*]] = affine.linearize_index [%[[ARG0]], %[[ARG1]]] by (64, 128) : index
-  // CHECK: amdgpu.gather_to_lds %[[MEM]][%[[IDX]]], %[[LOCAL]][%[[C0]], %[[C0]]]
-  // CHECK-SAME: vector<8xf16>, memref<8192xf16>, memref<64x64xf16, 3>
+  // CHECK: %[[IDXM:.*]] = affine.linearize_index [%[[ARG0]], %[[ARG1]]] by (64, 128) : index
+  // CHECK: %[[IDXL:.*]] = affine.linearize_index [%[[C0]], %[[C0]]] by (64, 64) : index
+  // CHECK: amdgpu.gather_to_lds %[[MEM]][%[[IDXM]]], %[[LOCAL]][%[[IDXL]]]
+  // CHECK-SAME: vector<8xf16>, memref<8192xf16>, memref<4096xf16, 3>
 
-  %alloc = memref.alloc() : memref<64x64xf16, #gpu_lds_addrspace>
+  %alloc = memref.alloc() : memref<4096xf16, #gpu_lds_addrspace>
   %mem = memref.alloc() : memref<8192xf16>
-  %expand = memref.expand_shape %mem [[0, 1]] output_shape [64, 128] : memref<8192xf16> into memref<64x128xf16>
+  %expand_mem = memref.expand_shape %mem [[0, 1]] output_shape [64, 128] : memref<8192xf16> into memref<64x128xf16>
+  %expand_alloc = memref.expand_shape %alloc [[0, 1]] output_shape [64, 64] : memref<4096xf16, #gpu_lds_addrspace> into memref<64x64xf16, #gpu_lds_addrspace>
   %c0 = arith.constant 0 : index
-  amdgpu.gather_to_lds %expand[%offset_i, %offset_j], %alloc[%c0, %c0]
+  amdgpu.gather_to_lds %expand_mem[%offset_i, %offset_j], %expand_alloc[%c0, %c0]
     : vector<8xf16>, memref<64x128xf16>, memref<64x64xf16, #gpu_lds_addrspace>
   func.return
 }
@@ -80,15 +82,82 @@ func.func @test_collapse_shape(%offset_i: index, %offset_j: index) {
   // CHECK: %[[LOCAL:.*]] = memref.alloc() : memref<64x64xf16, 3>
   // CHECK: %[[MEM:.*]] = memref.alloc() : memref<64x128xf16>
   // CHECK: %[[C0:.*]] = arith.constant 0 : index
-  // CHECK: %[[INDICES:.*]]:2 = affine.delinearize_index %[[ARG0]] into (64, 128) : index, index
-  // CHECK: amdgpu.gather_to_lds %[[MEM]][%[[INDICES]]#0, %[[INDICES]]#1], %[[LOCAL]][%[[C0]], %[[C0]]]
+  // CHECK: %[[INDICES_MEM:.*]]:2 = affine.delinearize_index %[[ARG0]] into (64, 128) : index, index
+  // CHECK: %[[INDICES_LDS:.*]]:2 = affine.delinearize_index %[[ARG1]] into (64, 64) : index, index
+  // CHECK: amdgpu.gather_to_lds %[[MEM]][%[[INDICES_MEM]]#0, %[[INDICES_MEM]]#1], %[[LOCAL]][%[[INDICES_LDS]]#0, %[[INDICES_LDS]]#1]
   // CHECK-SAME: vector<8xf16>, memref<64x128xf16>, memref<64x64xf16, 3>
 
   %alloc = memref.alloc() : memref<64x64xf16, #gpu_lds_addrspace>
+  %collapse_alloc = memref.collapse_shape %alloc [[0, 1]] : memref<64x64xf16, #gpu_lds_addrspace> into memref<4096xf16, #gpu_lds_addrspace>
   %mem = memref.alloc() : memref<64x128xf16>
-  %collapse = memref.collapse_shape %mem [[0, 1]] : memref<64x128xf16> into memref<8192xf16>
+  %collapse_mem = memref.collapse_shape %mem [[0, 1]] : memref<64x128xf16> into memref<8192xf16>
   %c0 = arith.constant 0 : index
-  amdgpu.gather_to_lds %collapse[%offset_i], %alloc[%c0, %c0]
+  amdgpu.gather_to_lds %collapse_mem[%offset_i], %collapse_alloc[%offset_j]
+    : vector<8xf16>, memref<8192xf16>, memref<4096xf16, #gpu_lds_addrspace>
+  func.return
+}
+
+
+// -----
+
+#gpu_lds_addrspace = 3
+
+
+// CHECK: func @test_expand_shape_src_raw_buffer
+// CHECK-SAME: %[[ARG0:.*]]: memref<8192xf16, #amdgpu.address_space<fat_raw_buffer>>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index
+func.func @test_expand_shape_src_raw_buffer(%mem : memref<8192xf16, #amdgpu.address_space<fat_raw_buffer>>, %offset_i: index, %offset_j: index) {
+  // CHECK: %[[LOCAL:.*]] = memref.alloc() : memref<4096xf16, 3>
+  // CHECK: %[[C0:.*]] = arith.constant 0 : index
+  // CHECK: %[[IDXM:.*]] = affine.linearize_index [%[[ARG1]], %[[ARG2]]] by (64, 128) : index
+  // CHECK: amdgpu.gather_to_lds %[[ARG0]][%[[IDXM]]], %[[LOCAL]][%[[C0]]]
+  // CHECK-SAME: vector<8xf16>, memref<8192xf16, #amdgpu.address_space<fat_raw_buffer>>, memref<4096xf16, 3>
+
+  %alloc = memref.alloc() : memref<4096xf16, #gpu_lds_addrspace>
+  %expand_mem = memref.expand_shape %mem [[0, 1]] output_shape [64, 128] : memref<8192xf16, #amdgpu.address_space<fat_raw_buffer>> into memref<64x128xf16, #amdgpu.address_space<fat_raw_buffer>>
+
+  %c0 = arith.constant 0 : index
+  amdgpu.gather_to_lds %expand_mem[%offset_i, %offset_j], %alloc[%c0]
+    : vector<8xf16>, memref<64x128xf16, #amdgpu.address_space<fat_raw_buffer>>, memref<4096xf16, #gpu_lds_addrspace>
+  func.return
+}
+
+// -----
+
+#gpu_lds_addrspace = 3
+
+// CHECK: func @test_expand_shape_dst_only
+// CHECK-SAME: %[[ARG0:.*]]: index, %[[ARG1:.*]]: index
+func.func @test_expand_shape_dst_only(%offset_i: index, %offset_j: index) {
+  // CHECK: %[[LOCAL:.*]] = memref.alloc() : memref<4096xf16, 3>
+  // CHECK: %[[MEM:.*]] = memref.alloc() : memref<8192xf16>
+  // CHECK: %[[C0:.*]] = arith.constant 0 : index
+  // CHECK: %[[IDX_LDS:.*]] = affine.linearize_index [%[[ARG1]], %[[C0]]] by (64, 64) : index
+  // CHECK: amdgpu.gather_to_lds %[[MEM]][%[[ARG0]]], %[[LOCAL]][%[[IDX_LDS]]]
+  // CHECK-SAME: vector<8xf16>, memref<8192xf16>, memref<4096xf16, 3>
+
+  %alloc = memref.alloc() : memref<4096xf16, #gpu_lds_addrspace>
+  %mem = memref.alloc() : memref<8192xf16>
+  %expand_alloc = memref.expand_shape %alloc [[0, 1]] output_shape [64, 64] : memref<4096xf16, #gpu_lds_addrspace> into memref<64x64xf16, #gpu_lds_addrspace>
+
+  %c0 = arith.constant 0 : index
+  amdgpu.gather_to_lds %mem[%offset_i], %expand_alloc[%offset_j, %c0]
     : vector<8xf16>, memref<8192xf16>, memref<64x64xf16, #gpu_lds_addrspace>
   func.return
 }
+
+// -----
+
+#gpu_lds_addrspace = 3
+
+// CHECK: func @test_nop
+// CHECK-SAME: %[[ARG0:.*]]: memref<8192xf16, #amdgpu.address_space<fat_raw_buffer>>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index
+func.func @test_nop(%mem : memref<8192xf16, #amdgpu.address_space<fat_raw_buffer>>, %offset_i: index, %offset_j: index) {
+  // CHECK: %[[LOCAL:.*]] = memref.alloc() : memref<4096xf16, 3>
+  // CHECK: amdgpu.gather_to_lds %[[ARG0]][%[[ARG1]]], %[[LOCAL]][%[[ARG2]]]
+  // CHECK-SAME: vector<8xf16>, memref<8192xf16, #amdgpu.address_space<fat_raw_buffer>>, memref<4096xf16, 3>
+
+  %alloc = memref.alloc() : memref<4096xf16, #gpu_lds_addrspace>
+  amdgpu.gather_to_lds %mem[%offset_i], %alloc[%offset_j]
+    : vector<8xf16>, memref<8192xf16, #amdgpu.address_space<fat_raw_buffer>>, memref<4096xf16, #gpu_lds_addrspace>
+  func.return
+}
diff --git a/mlir/test/Dialect/AMDGPU/invalid.mlir b/mlir/test/Dialect/AMDGPU/invalid.mlir
index 0d2fd245af9e2..66e7dd4014af9 100644
--- a/mlir/test/Dialect/AMDGPU/invalid.mlir
+++ b/mlir/test/Dialect/AMDGPU/invalid.mlir
@@ -230,3 +230,11 @@ func.func @gather_to_lds_non_lds(%idx1 : index, %mem1 : memref<32xf16>, %mem2 :
   amdgpu.gather_to_lds %mem1[%idx1], %mem2[%idx1] : vector<2xf16>, memref<32xf16>, memref<32xf16>
   func.return
 }
+
+// -----
+
+func.func @gather_to_lds_non_lds(%idx1 : index, %mem1 : memref<32xf16>, %mem2 : memref<32xf16, strided<[?]>, #gpu.address_space<workgroup>>) {
+  // expected-error@+1 {{'amdgpu.gather_to_lds' op destination type inner most dim must be contiguous}}
+  amdgpu.gather_to_lds %mem1[%idx1], %mem2[%idx1] : vector<2xf16>, memref<32xf16>, memref<32xf16, strided<[?]>, #gpu.address_space<workgroup>>
+  func.return
+}
diff --git a/mlir/test/Dialect/AMDGPU/ops.mlir b/mlir/test/Dialect/AMDGPU/ops.mlir
index fe78b5365745a..369e0fff538e1 100644
--- a/mlir/test/Dialect/AMDGPU/ops.mlir
+++ b/mlir/test/Dialect/AMDGPU/ops.mlir
@@ -524,6 +524,20 @@ func.func @swizzle_bitmode(%arg0 : f32) -> f32 {
   func.return %0 : f32
 }
 
+// CHECK-LABEL: func @permlane16_swap
+func.func @permlane16_swap(%arg0 : f32) -> f32 {
+  // CHECK: amdgpu.permlane_swap
+  %0 = amdgpu.permlane_swap %arg0 16 : f32
+  func.return %0 : f32
+}
+
+// CHECK-LABEL: func @permlane32_swap
+func.func @permlane32_swap(%arg0 : f32) -> f32 {
+  // CHECK: amdgpu.permlane_swap
+  %0 = amdgpu.permlane_swap %arg0 32 : f32
+  func.return %0 : f32
+}
+
 // CHECK-LABEL: func @scaled_mfma
 func.func @scaled_mfma(%arg0 : f8E8M0FNU, %arg1 : vector<32xf6E2M3FN>, %arg2 : vector<16xf32>) -> vector<16xf32> {
   // CHECK: amdgpu.scaled_mfma
@@ -539,13 +553,15 @@ func.func @transpose_load(%idx1 : index, %idx2 : index, %mem : memref<128x32xf16
 }
 
 // CHECK-LABEL: func @gather_to_lds
-func.func @gather_to_lds(%idx1 : index, %idx2 : index, %mem1 : memref<32xf16>, %mem2 : memref<32x32xf16>, %smem1 : memref<32xf16, #gpu.address_space<workgroup>>, %smem2 : memref<32x32xf16, #gpu.address_space<workgroup>>) {
+func.func @gather_to_lds(%idx1 : index, %idx2 : index, %mem1 : memref<32xf16>, %mem2 : memref<32x32xf16>, %smem1 : memref<32xf16, #gpu.address_space<workgroup>>, %smem2 : memref<32x32xf16, #gpu.address_space<workgroup>>, %smem3 : memref<?x?xf16, strided<[?, 1]>, #gpu.address_space<workgroup>>) {
   // CHECK: amdgpu.gather_to_lds %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}, %{{.*}}]
   // CHECK: amdgpu.gather_to_lds %{{.*}}[%{{.*}}, %{{.*}}], %{{.*}}[%{{.*}}]
   // CHECK: amdgpu.gather_to_lds %{{.*}}[%{{.*}}],          %{{.*}}[%{{.*}}, %{{.*}}]
+  // CHECK: amdgpu.gather_to_lds %{{.*}}[%{{.*}}],          %{{.*}}[%{{.*}}, %{{.*}}]
   amdgpu.gather_to_lds %mem2[%idx1, %idx2], %smem2[%idx1, %idx2] : vector<2xf16>, memref<32x32xf16>, memref<32x32xf16, #gpu.address_space<workgroup>>
   amdgpu.gather_to_lds %mem2[%idx1, %idx2], %smem1[%idx1]        : vector<2xf16>, memref<32x32xf16>, memref<32xf16,    #gpu.address_space<workgroup>>
   amdgpu.gather_to_lds %mem1[%idx1],        %smem2[%idx1, %idx2] : vector<2xf16>, memref<32xf16>,    memref<32x32xf16, #gpu.address_space<workgroup>>
+  amdgpu.gather_to_lds %mem1[%idx1],        %smem3[%idx1, %idx2] : vector<2xf16>, memref<32xf16>,   memref<?x?xf16, strided<[?, 1]>, #gpu.address_space<workgroup>>
   func.return
 }
 
diff --git a/mlir/test/Dialect/Affine/loop-permute.mlir b/mlir/test/Dialect/Affine/loop-permute.mlir
index 118165b2fb2a2..e38aeb543fceb 100644
--- a/mlir/test/Dialect/Affine/loop-permute.mlir
+++ b/mlir/test/Dialect/Affine/loop-permute.mlir
@@ -4,6 +4,7 @@
 // RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=0,2,1" | FileCheck %s --check-prefix=CHECK-021
 // RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=2,0,1" | FileCheck %s --check-prefix=CHECK-201
 // RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=2,1,0" | FileCheck %s --check-prefix=CHECK-210
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-permutation="permutation-map=2,1,0 check-validity=1" | FileCheck %s --check-prefix=CHECK-210-VALID
 
 // CHECK-120-LABEL: func @permute
 func.func @permute(%U0 : index, %U1 : index, %U2 : index) {
@@ -45,3 +46,34 @@ func.func @permute(%U0 : index, %U1 : index, %U2 : index) {
 
 // CHECK-201:      "foo"(%arg5, %arg3)
 // CHECK-201-NEXT: "bar"(%arg4)
+
+// -----
+
+// Tests that the permutation validation check utility conservatively returns false when the
+// for loop has an iter_arg.
+
+// CHECK-210-VALID-LABEL: func @check_validity_with_iter_args
+// CHECK-210-VALID-SAME:    %[[ARG0:.*]]: index, %[[ARG1:.*]]: index, %[[ARG2:.*]]: index
+func.func @check_validity_with_iter_args(%U0 : index, %U1 : index, %U2 : index) {
+  %buf = memref.alloc() : memref<100x100xf32>
+  %cst = arith.constant 1.0 : f32
+  %c10 = arith.constant 10 : index
+  %c20 = arith.constant 20 : index
+
+  // Check that the loops are not permuted.
+  // CHECK-210-VALID:       affine.for %{{.*}} = 0 to %[[ARG0]] {
+  // CHECK-210-VALID-NEXT:    affine.for %{{.*}} = 0 to %[[ARG1]] {
+  // CHECK-210-VALID-NEXT:      affine.for %{{.*}} = 0 to %[[ARG2]] iter_args(
+  affine.for %arg0 = 0 to %U0 {
+    affine.for %arg1 = 0 to %U1 {
+      %res = affine.for %arg2 = 0 to %U2 iter_args(%iter1 = %cst) -> (f32) {
+        %val = affine.load %buf[%arg0 + 10, %arg1 + 20] : memref<100x100xf32>
+        %newVal = arith.addf %val, %cst : f32
+        affine.store %newVal, %buf[%arg0 + 10, %arg1 + 20] : memref<100x100xf32>
+        %newVal2 = arith.addf %newVal, %iter1 : f32
+        affine.yield %iter1 : f32
+      }
+    }
+  }
+  return
+}
diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir
index 3d5a46d13e59d..ca3de3a2d7703 100644
--- a/mlir/test/Dialect/Arith/canonicalize.mlir
+++ b/mlir/test/Dialect/Arith/canonicalize.mlir
@@ -654,7 +654,7 @@ func.func @signExtendConstant() -> i16 {
 //       CHECK:   return %[[cres]]
 func.func @signExtendConstantSplat() -> vector<4xi16> {
   %c-2 = arith.constant -2 : i8
-  %splat = vector.splat %c-2 : vector<4xi8>
+  %splat = vector.broadcast %c-2 : i8 to vector<4xi8>
   %ext = arith.extsi %splat : vector<4xi8> to vector<4xi16>
   return %ext : vector<4xi16>
 }
@@ -682,7 +682,7 @@ func.func @unsignedExtendConstant() -> i16 {
 //       CHECK:   return %[[cres]]
 func.func @unsignedExtendConstantSplat() -> vector<4xi16> {
   %c2 = arith.constant 2 : i8
-  %splat = vector.splat %c2 : vector<4xi8>
+  %splat = vector.broadcast %c2 : i8 to vector<4xi8>
   %ext = arith.extui %splat : vector<4xi8> to vector<4xi16>
   return %ext : vector<4xi16>
 }
@@ -866,7 +866,7 @@ func.func @truncExtsiVector(%arg0: vector<2xi32>) -> vector<2xi16> {
 //       CHECK:   return %[[cres]]
 func.func @truncConstantSplat() -> vector<4xi8> {
   %c-2 = arith.constant -2 : i16
-  %splat = vector.splat %c-2 : vector<4xi16>
+  %splat = vector.broadcast %c-2 : i16 to vector<4xi16>
   %trunc = arith.trunci %splat : vector<4xi16> to vector<4xi8>
   return %trunc : vector<4xi8>
 }
@@ -2334,7 +2334,7 @@ func.func @constant_FPtoUI_splat() -> vector<4xi32> {
   // CHECK: %[[C0:.+]] = arith.constant dense<2> : vector<4xi32>
   // CHECK: return %[[C0]]
   %c0 = arith.constant 2.0 : f32
-  %splat = vector.splat %c0 : vector<4xf32>
+  %splat = vector.broadcast %c0 : f32 to vector<4xf32>
   %res = arith.fptoui %splat : vector<4xf32> to vector<4xi32>
   return %res : vector<4xi32>
 }
@@ -2374,7 +2374,7 @@ func.func @constant_FPtoSI_splat() -> vector<4xi32> {
   // CHECK: %[[C0:.+]] = arith.constant dense<-2> : vector<4xi32>
   // CHECK: return %[[C0]]
   %c0 = arith.constant -2.0 : f32
-  %splat = vector.splat %c0 : vector<4xf32>
+  %splat = vector.broadcast %c0 : f32 to vector<4xf32>
   %res = arith.fptosi %splat : vector<4xf32> to vector<4xi32>
   return %res : vector<4xi32>
 }
@@ -2413,7 +2413,7 @@ func.func @constant_SItoFP_splat() -> vector<4xf32> {
   // CHECK: %[[C0:.+]] = arith.constant dense<2.000000e+00> : vector<4xf32>
   // CHECK: return %[[C0]]
   %c0 = arith.constant 2 : i32
-  %splat = vector.splat %c0 : vector<4xi32>
+  %splat = vector.broadcast %c0 : i32 to vector<4xi32>
   %res = arith.sitofp %splat : vector<4xi32> to vector<4xf32>
   return %res : vector<4xf32>
 }
@@ -2442,7 +2442,7 @@ func.func @constant_UItoFP_splat() -> vector<4xf32> {
   // CHECK: %[[C0:.+]] = arith.constant dense<2.000000e+00> : vector<4xf32>
   // CHECK: return %[[C0]]
   %c0 = arith.constant 2 : i32
-  %splat = vector.splat %c0 : vector<4xi32>
+  %splat = vector.broadcast %c0 : i32 to vector<4xi32>
   %res = arith.uitofp %splat : vector<4xi32> to vector<4xf32>
   return %res : vector<4xf32>
 }
@@ -3363,3 +3363,18 @@ func.func @bf16_fma(%arg0: vector<32x32x32xbf16>, %arg1: vector<32x32x32xbf16>,
     }
   }
 #-}
+
+// CHECK-LABEL: func @unreachable()
+// CHECK-NEXT: return
+// CHECK-NOT: arith
+func.func @unreachable() {
+  return
+^unreachable:
+  %c1_i64 = arith.constant 1 : i64
+  // This self referencing operation is legal in an unreachable block.
+  // Many patterns are unsafe with respect to this kind of situation,
+  // check that we don't infinite loop here.
+  %add = arith.addi %add, %c1_i64 : i64
+  cf.br ^unreachable
+}
+
diff --git a/mlir/test/Dialect/Arith/int-range-interface.mlir b/mlir/test/Dialect/Arith/int-range-interface.mlir
index 2128d36f1a28e..130782ba9f525 100644
--- a/mlir/test/Dialect/Arith/int-range-interface.mlir
+++ b/mlir/test/Dialect/Arith/int-range-interface.mlir
@@ -224,6 +224,15 @@ func.func @ceil_divui(%arg0 : index) -> i1 {
     func.return %7 : i1
 }
 
+// CHECK-LABEL: func @ceil_divui_by_zero_issue_131273
+// CHECK-NEXT: return
+func.func @ceil_divui_by_zero_issue_131273() {
+    %0 = test.with_bounds {smax = 0 : i32, smin = -1 : i32, umax = 0 : i32, umin = -1 : i32} : i32
+    %c7_i32 = arith.constant 7 : i32
+    %1 = arith.ceildivui %c7_i32, %0 : i32
+    return
+}
+
 // CHECK-LABEL: func @ceil_divsi
 // CHECK: %[[ret:.*]] = arith.cmpi eq
 // CHECK: return %[[ret]]
diff --git a/mlir/test/Dialect/EmitC/attrs.mlir b/mlir/test/Dialect/EmitC/attrs.mlir
index 11251b88ff0c9..5a219c462678e 100644
--- a/mlir/test/Dialect/EmitC/attrs.mlir
+++ b/mlir/test/Dialect/EmitC/attrs.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt %s | FileCheck %s
 // check parser
-// RUN: mlir-opt -verify-diagnostics %s | mlir-opt -verify-diagnostics | FileCheck %s
+// RUN: mlir-opt %s | mlir-opt | FileCheck %s
 
 // CHECK-LABEL: func @opaque_attrs() {
 func.func @opaque_attrs() {
diff --git a/mlir/test/Dialect/EmitC/transforms.mlir b/mlir/test/Dialect/EmitC/form-expressions.mlir
similarity index 98%
rename from mlir/test/Dialect/EmitC/transforms.mlir
rename to mlir/test/Dialect/EmitC/form-expressions.mlir
index a38f396dad953..1ac053bf1215f 100644
--- a/mlir/test/Dialect/EmitC/transforms.mlir
+++ b/mlir/test/Dialect/EmitC/form-expressions.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s --form-expressions --verify-diagnostics --split-input-file | FileCheck %s
+// RUN: mlir-opt %s -form-expressions | FileCheck %s
 
 // CHECK-LABEL: func.func @single_expression(
 // CHECK-SAME:                               %[[VAL_0:.*]]: i32, %[[VAL_1:.*]]: i32, %[[VAL_2:.*]]: i32, %[[VAL_3:.*]]: i32) -> i1 {
@@ -83,7 +83,6 @@ func.func @expression_with_dereference(%arg0: i32, %arg1: i32, %arg2: !emitc.ptr
   return %c : i1
 }
 
-
 // CHECK-LABEL: func.func @expression_with_address_taken(
 // CHECK-SAME:      %[[VAL_0:.*]]: i32, %[[VAL_1:.*]]: i32, %[[VAL_2:.*]]: !emitc.ptr<i32>) -> i1 {
 // CHECK:         %[[VAL_3:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.lvalue<i32>
@@ -121,9 +120,8 @@ func.func @no_nested_expression(%arg0: i32, %arg1: i32) -> i1 {
   return %a : i1
 }
 
-
 // CHECK-LABEL: func.func @single_result_requirement
-//   CHECK-NOT:  emitc.expression
+// CHECK-NOT:  emitc.expression
 
 func.func @single_result_requirement() -> (i32, i32) {
   %0:2 = emitc.call_opaque "foo" () : () -> (i32, i32)
@@ -152,7 +150,6 @@ func.func @single_result_requirement() -> (i32, i32) {
 // CHECK:           return %[[VAL_9]] : i1
 // CHECK:         }
 
-
 func.func @expression_with_load(%arg0: i32, %arg1: !emitc.ptr<i32>) -> i1 {
   %c0 = "emitc.constant"() {value = 0 : i64} : () -> i64
   %0 = "emitc.variable"() <{value = #emitc.opaque<"42">}> : () -> !emitc.lvalue<i32>
diff --git a/mlir/test/Dialect/EmitC/invalid_ops.mlir b/mlir/test/Dialect/EmitC/invalid_ops.mlir
index 3946a36a83c6f..6d7f79ead2974 100644
--- a/mlir/test/Dialect/EmitC/invalid_ops.mlir
+++ b/mlir/test/Dialect/EmitC/invalid_ops.mlir
@@ -676,3 +676,35 @@ func.func @test_verbatim(%arg0 : !emitc.ptr<i32>, %arg1 : i32) {
   emitc.verbatim "{a} " args %arg0, %arg1 : !emitc.ptr<i32>, i32
   return
 }
+
+// -----
+
+// expected-error @+1 {{'emitc.field' op field must be nested within an emitc.class operation}}
+emitc.field @testField : !emitc.array<1xf32>
+
+// -----
+
+// expected-error @+1 {{'emitc.get_field' op  must be nested within an emitc.class operation}}
+%1 = emitc.get_field @testField : !emitc.array<1xf32>
+
+// -----
+
+emitc.func @testMethod() {
+  %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+  // expected-error @+1 {{'emitc.get_field' op  must be nested within an emitc.class operation}}
+  %1 = get_field @testField : !emitc.array<1xf32>
+  %2 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+  return
+}
+
+// -----
+
+emitc.class @testClass {
+  emitc.func @testMethod() {
+    %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+    // expected-error @+1 {{'emitc.get_field' op field '@testField' not found in the class}}
+    %1 = get_field @testField : !emitc.array<1xf32>
+    %2 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    return
+  }
+}
diff --git a/mlir/test/Dialect/EmitC/ops.mlir b/mlir/test/Dialect/EmitC/ops.mlir
index ad40313f95df9..acee0a8d53fe4 100644
--- a/mlir/test/Dialect/EmitC/ops.mlir
+++ b/mlir/test/Dialect/EmitC/ops.mlir
@@ -310,3 +310,15 @@ func.func @switch() {
 
   return 
 }
+
+emitc.class final @finalClass {
+  emitc.field @fieldName0 : !emitc.array<1xf32>
+  emitc.field @fieldName1 : !emitc.array<1xf32>
+  emitc.func @execute() {
+    %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+    %1 = get_field @fieldName0 : !emitc.array<1xf32>
+    %2 = get_field @fieldName1 : !emitc.array<1xf32>
+    %3 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    return
+  }
+}
diff --git a/mlir/test/Dialect/EmitC/types.mlir b/mlir/test/Dialect/EmitC/types.mlir
index d4dd94457f39b..ce1e03a83e5d1 100644
--- a/mlir/test/Dialect/EmitC/types.mlir
+++ b/mlir/test/Dialect/EmitC/types.mlir
@@ -1,6 +1,6 @@
-// RUN: mlir-opt -verify-diagnostics -allow-unregistered-dialect %s | FileCheck %s
-// check parser
-// RUN: mlir-opt -verify-diagnostics -allow-unregistered-dialect %s | mlir-opt -verify-diagnostics --allow-unregistered-dialect | FileCheck %s
+// RUN: mlir-opt %s -allow-unregistered-dialect | FileCheck %s
+// Check parser
+// RUN: mlir-opt %s -allow-unregistered-dialect | mlir-opt -allow-unregistered-dialect | FileCheck %s
 
 // CHECK-LABEL: func @array_types(
 func.func @array_types(
diff --git a/mlir/test/Dialect/EmitC/wrap-func-in-class.mlir b/mlir/test/Dialect/EmitC/wrap-func-in-class.mlir
new file mode 100644
index 0000000000000..809febd0267b1
--- /dev/null
+++ b/mlir/test/Dialect/EmitC/wrap-func-in-class.mlir
@@ -0,0 +1,57 @@
+// RUN: mlir-opt %s -wrap-emitc-func-in-class -split-input-file | FileCheck %s
+
+emitc.func @foo(%arg0 : !emitc.array<1xf32>) {
+  emitc.call_opaque "bar" (%arg0) : (!emitc.array<1xf32>) -> ()
+  emitc.return
+}
+
+// CHECK: module {
+// CHECK:   emitc.class @fooClass {
+// CHECK:     emitc.field @fieldName0 : !emitc.array<1xf32>
+// CHECK:     emitc.func @execute() {
+// CHECK:       %0 = get_field @fieldName0 : !emitc.array<1xf32>
+// CHECK:       call_opaque "bar"(%0) : (!emitc.array<1xf32>) -> ()
+// CHECK:       return
+// CHECK:     }
+// CHECK:   }
+// CHECK: }
+
+// -----
+
+module attributes { } {
+  emitc.func @model(%arg0: !emitc.array<1xf32> {emitc.name_hint = "another_feature"},
+   %arg1: !emitc.array<1xf32> {emitc.name_hint = "some_feature"},
+   %arg2: !emitc.array<1xf32> {emitc.name_hint = "output_0"}) attributes { } {
+    %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+    %1 = subscript %arg1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    %2 = load %1 : <f32>
+    %3 = subscript %arg0[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    %4 = load %3 : <f32>
+    %5 = add %2, %4 : (f32, f32) -> f32
+    %6 = subscript %arg2[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    assign %5 : f32 to %6 : <f32>
+    return
+  }
+}
+
+// CHECK: module {
+// CHECK:   emitc.class @modelClass {
+// CHECK:     emitc.field @fieldName0 : !emitc.array<1xf32> {emitc.name_hint = "another_feature"}
+// CHECK:     emitc.field @fieldName1 : !emitc.array<1xf32>  {emitc.name_hint = "some_feature"}
+// CHECK:     emitc.field @fieldName2 : !emitc.array<1xf32>  {emitc.name_hint = "output_0"}
+// CHECK:     emitc.func @execute() {
+// CHECK:       get_field @fieldName0 : !emitc.array<1xf32>
+// CHECK:       get_field @fieldName1 : !emitc.array<1xf32>
+// CHECK:       get_field @fieldName2 : !emitc.array<1xf32>
+// CHECK:       "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+// CHECK:       subscript {{.*}}[{{.*}}] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+// CHECK:       load {{.*}} : <f32>
+// CHECK:       subscript {{.*}}[{{.*}}] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+// CHECK:       load {{.*}} : <f32>
+// CHECK:       add {{.*}}, {{.*}} : (f32, f32) -> f32
+// CHECK:       subscript {{.*}}[{{.*}}] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+// CHECK:       assign {{.*}} : f32 to {{.*}} : <f32>
+// CHECK:       return
+// CHECK:     }
+// CHECK:   }
+// CHECK: }
diff --git a/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class.mlir b/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class.mlir
deleted file mode 100644
index 029fa78a3f528..0000000000000
--- a/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class.mlir
+++ /dev/null
@@ -1,40 +0,0 @@
-// RUN: mlir-opt --wrap-emitc-func-in-class %s | FileCheck %s
-
-module attributes { } {
-  emitc.func @model(%arg0: !emitc.array<1xf32> {emitc.name_hint = "another_feature"}, 
-   %arg1: !emitc.array<1xf32> {emitc.name_hint = "some_feature"},
-   %arg2: !emitc.array<1xf32> {emitc.name_hint = "output_0"}) attributes { } {
-    %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
-    %1 = subscript %arg1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
-    %2 = load %1 : <f32>
-    %3 = subscript %arg0[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
-    %4 = load %3 : <f32>
-    %5 = add %2, %4 : (f32, f32) -> f32
-    %6 = subscript %arg2[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
-    assign %5 : f32 to %6 : <f32>
-    return
-  }
-}
-
-
-// CHECK: module {
-// CHECK-NEXT:   emitc.class @modelClass {
-// CHECK-NEXT:     emitc.field @fieldName0 : !emitc.array<1xf32> {emitc.name_hint = "another_feature"}
-// CHECK-NEXT:     emitc.field @fieldName1 : !emitc.array<1xf32>  {emitc.name_hint = "some_feature"}
-// CHECK-NEXT:     emitc.field @fieldName2 : !emitc.array<1xf32>  {emitc.name_hint = "output_0"}
-// CHECK-NEXT:     emitc.func @execute() {
-// CHECK-NEXT:       get_field @fieldName0 : !emitc.array<1xf32>
-// CHECK-NEXT:       get_field @fieldName1 : !emitc.array<1xf32>
-// CHECK-NEXT:       get_field @fieldName2 : !emitc.array<1xf32>
-// CHECK-NEXT:       "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
-// CHECK-NEXT:       subscript {{.*}}[{{.*}}] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
-// CHECK-NEXT:       load {{.*}} : <f32>
-// CHECK-NEXT:       subscript {{.*}}[{{.*}}] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
-// CHECK-NEXT:       load {{.*}} : <f32>
-// CHECK-NEXT:       add {{.*}}, {{.*}} : (f32, f32) -> f32
-// CHECK-NEXT:       subscript {{.*}}[{{.*}}] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
-// CHECK-NEXT:       assign {{.*}} : f32 to {{.*}} : <f32>
-// CHECK-NEXT:       return
-// CHECK-NEXT:     }
-// CHECK-NEXT:   }
-// CHECK-NEXT: }
diff --git a/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class_noAttr.mlir b/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class_noAttr.mlir
deleted file mode 100644
index 92ed20c4b14e3..0000000000000
--- a/mlir/test/Dialect/EmitC/wrap_emitc_func_in_class_noAttr.mlir
+++ /dev/null
@@ -1,17 +0,0 @@
-// RUN: mlir-opt --wrap-emitc-func-in-class %s | FileCheck %s
-
-emitc.func @foo(%arg0 : !emitc.array<1xf32>) {
-  emitc.call_opaque "bar" (%arg0) : (!emitc.array<1xf32>) -> ()
-  emitc.return
-}
-
-// CHECK: module {
-// CHECK-NEXT:   emitc.class @fooClass {
-// CHECK-NEXT:     emitc.field @fieldName0 : !emitc.array<1xf32>
-// CHECK-NEXT:     emitc.func @execute() {
-// CHECK-NEXT:       %0 = get_field @fieldName0 : !emitc.array<1xf32>
-// CHECK-NEXT:       call_opaque "bar"(%0) : (!emitc.array<1xf32>) -> ()
-// CHECK-NEXT:       return
-// CHECK-NEXT:     }
-// CHECK-NEXT:   }
-// CHECK-NEXT: }
diff --git a/mlir/test/Dialect/GPU/ops.mlir b/mlir/test/Dialect/GPU/ops.mlir
index ee1fdfa4d02f0..9cc0bf8f41d5a 100644
--- a/mlir/test/Dialect/GPU/ops.mlir
+++ b/mlir/test/Dialect/GPU/ops.mlir
@@ -17,6 +17,18 @@ module attributes {gpu.container_module} {
     return
   }
 
+  // CHECK-LABEL:func @launch_with_module_func_attr(%{{.*}}: index)
+  func.func @launch_with_module_func_attr(%sz : index) {
+    // CHECK: gpu.launch blocks(%{{.*}}, %{{.*}}, %{{.*}}) in (%{{.*}} = %{{.*}}, %{{.*}} = %{{.*}}, %{{.*}} = %{{.*}}) threads(%{{.*}}, %{{.*}}, %{{.*}}) in (%{{.*}} = %{{.*}}, %{{.*}} = %{{.*}}, %{{.*}} = %{{.*}}) module(@test_module) function(@test_kernel_func)
+    gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %sz, %grid_y = %sz, %grid_z = %sz)
+               threads(%tx, %ty, %tz) in (%block_x = %sz, %block_y = %sz, %block_z = %sz)
+               module(@test_module) function(@test_kernel_func) {
+      // CHECK: gpu.terminator
+      gpu.terminator
+    }
+    return
+  }
+
   // CHECK-LABEL:func @args(%{{.*}}: index, %{{.*}}: index, %{{.*}}: f32, %{{.*}}: memref<?xf32, 1>) {
   func.func @args(%blk : index, %thrd : index, %float : f32, %data : memref<?xf32,1>) {
     // CHECK: gpu.launch blocks(%{{.*}}, %{{.*}}, %{{.*}}) in (%{{.*}} = %{{.*}}, %{{.*}} = %{{.*}}, %{{.*}} = %{{.*}}) threads(%{{.*}}, %{{.*}}, %{{.*}}) in (%{{.*}} = %{{.*}}, %{{.*}} = %{{.*}}, %{{.*}} = %{{.*}})
diff --git a/mlir/test/Dialect/GPU/outlining.mlir b/mlir/test/Dialect/GPU/outlining.mlir
index d48fa054432d1..e14521ab9bb5c 100644
--- a/mlir/test/Dialect/GPU/outlining.mlir
+++ b/mlir/test/Dialect/GPU/outlining.mlir
@@ -509,7 +509,7 @@ func.func @launch_cluster() {
 // CHECK-NEXT: = memref.load %[[KERNEL_ARG1]][%[[TID]]] : memref<?xf32, 1>
 
 // -----
-// This test tests the two optional attributes kernelModule and kernelFunc for gpu.launch
+// This test tests the two optional attributes `module` and `function` for gpu.launch
 // CHECK-LABEL: func.func @testKernelAttributes()
 // CHECK: gpu.launch_func  @test_module::@test_kernel_func blocks in (%[[GRID_X:.*]], %[[GRID_Y:.*]], %[[GRID_Z:.*]]) threads in (%[[BLOCK_X:.*]], %[[BLOCK_Y:.*]], %[[BLOCK_Z:.*]])
 // CHECK: gpu.module @test_module
@@ -523,15 +523,16 @@ func.func @testKernelAttributes() {
   %bDimZ = arith.constant 8 : index
 
   gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, %grid_z = %gDimZ)
-             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ) {
+             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ)
+             module(@test_module) function(@test_kernel_func) {
     "some_op"(%bx, %tx) : (index, index) -> ()
     gpu.terminator
-  } {kernelModule = @test_module, kernelFunc = @test_kernel_func}
+  }
   return
 }
 
 // -----
-// This test tests the two optional attributes kernelModule and kernelFunc for gpu.launch, when kernelModule already exists.
+// This test tests the two optional attributes `module` and `function` for gpu.launch, when kernelModule already exists.
 
 // CHECK-LABEL: gpu.module @existing_module
 // CHECK: gpu.func @test_kernel_func()
@@ -556,15 +557,16 @@ func.func @testExistingModule() {
   %bDimZ = arith.constant 8 : index
 
   gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, %grid_z = %gDimZ)
-             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ) {
+             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ)
+             module(@existing_module) function(@test_kernel_func) {
     "some_op"(%bx, %tx) : (index, index) -> ()
     gpu.terminator
-  } {kernelModule = @existing_module, kernelFunc = @test_kernel_func}
+  }
   return
 }
 
 // -----
-// This test tests the optional attribute kernelModule for gpu.launch.
+// This test tests the optional attribute `module` for gpu.launch.
 // CHECK-LABEL: func.func @testKernelModuleOnly()
 // CHECK: gpu.launch_func  @test_module::@testKernelModuleOnly_kernel blocks in (%[[GRID_X:.*]], %[[GRID_Y:.*]], %[[GRID_Z:.*]]) threads in (%[[BLOCK_X:.*]], %[[BLOCK_Y:.*]], %[[BLOCK_Z:.*]])
 // CHECK: gpu.module @test_module
@@ -578,15 +580,16 @@ func.func @testKernelModuleOnly() {
   %bDimZ = arith.constant 8 : index
 
   gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, %grid_z = %gDimZ)
-             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ) {
+             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ)
+             module(@test_module) {
     "some_op"(%bx, %tx) : (index, index) -> ()
     gpu.terminator
-  } {kernelModule = @test_module}
+  }
   return
 }
 
 // -----
-// This test tests the optional attribute kernelFunc for gpu.launch.
+// This test tests the optional attribute `function` for gpu.launch.
 // CHECK-LABEL: func.func @testKernelFuncOnly()
 // CHECK: gpu.launch_func  @test_kernel_func::@test_kernel_func blocks in (%[[GRID_X:.*]], %[[GRID_Y:.*]], %[[GRID_Z:.*]]) threads in (%[[BLOCK_X:.*]], %[[BLOCK_Y:.*]], %[[BLOCK_Z:.*]])
 
@@ -601,15 +604,16 @@ func.func @testKernelFuncOnly() {
   %bDimZ = arith.constant 8 : index
 
   gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %gDimX, %grid_y = %gDimY, %grid_z = %gDimZ)
-             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ) {
+             threads(%tx, %ty, %tz) in (%block_x = %bDimX, %block_y = %bDimY, %block_z = %bDimZ)
+             function(@test_kernel_func) {
     "some_op"(%bx, %tx) : (index, index) -> ()
     gpu.terminator
-  } {kernelFunc = @test_kernel_func}
+  }
   return
 }
 
 // -----
-// This test tests gpu.launch when optional attributes kernelModule and kernelFunc are not specified.
+// This test tests gpu.launch when optional attributes `module` and `function` are not specified.
 // CHECK-LABEL: func.func @testNoAttributes()
 // CHECK: gpu.launch_func  @testNoAttributes_kernel::@testNoAttributes_kernel blocks in (%[[GRID_X:.*]], %[[GRID_Y:.*]], %[[GRID_Z:.*]]) threads in (%[[BLOCK_X:.*]], %[[BLOCK_Y:.*]], %[[BLOCK_Z:.*]])
 
@@ -630,3 +634,29 @@ func.func @testNoAttributes() {
   }
   return
 }
+
+// -----
+
+// This test tests nested `gpu.launch`.
+
+// CHECK-LABEL: func.func @nested_launch(
+//  CHECK-SAME:                          %[[ARG0:.*]]: index) {
+//       CHECK:   gpu.launch_func  @nested_launch_kernel_0::@nested_launch_kernel blocks in (%[[ARG0]], %[[ARG0]], %[[ARG0]]) threads in (%[[ARG0]], %[[ARG0]], %[[ARG0]])  args(%[[ARG0]] : index)
+//       CHECK: gpu.module @nested_launch_kernel
+//       CHECK:   gpu.func @nested_launch_kernel() kernel
+//       CHECK:     "some_op"
+//       CHECK: gpu.module @nested_launch_kernel_0
+//       CHECK:   gpu.func @nested_launch_kernel(%[[VAL_0:.*]]: index) kernel
+//       CHECK:     gpu.launch_func  @nested_launch_kernel::@nested_launch_kernel blocks in (%[[VAL_0]], %[[VAL_0]], %[[VAL_0]]) threads in (%[[VAL_0]], %[[VAL_0]], %[[VAL_0]])
+func.func @nested_launch(%sz : index) {
+  gpu.launch blocks(%bx, %by, %bz) in (%grid_x = %sz, %grid_y = %sz, %grid_z = %sz)
+             threads(%tx, %ty, %tz) in (%block_x = %sz, %block_y = %sz, %block_z = %sz) {
+    gpu.launch blocks(%bx1, %by1, %bz1) in (%grid_x1 = %sz, %grid_y1 = %sz, %grid_z1 = %sz)
+               threads(%tx1, %ty1, %tz1) in (%block_x1 = %sz, %block_y1 = %sz, %block_z1 = %sz) {
+      "some_op"(%bx1, %tx1) : (index, index) -> ()
+      gpu.terminator
+    }
+    gpu.terminator
+  }
+  return
+}
diff --git a/mlir/test/Dialect/GPU/promote-shuffle-amdgpu.mlir b/mlir/test/Dialect/GPU/promote-shuffle-amdgpu.mlir
index 4293b430f71f7..747c997a3b441 100644
--- a/mlir/test/Dialect/GPU/promote-shuffle-amdgpu.mlir
+++ b/mlir/test/Dialect/GPU/promote-shuffle-amdgpu.mlir
@@ -4,7 +4,7 @@ module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
     %func = transform.structured.match ops{["func.func"]} in %module_op : (!transform.any_op) -> !transform.any_op
     transform.apply_patterns to %func {
-      transform.apply_patterns.gpu.gpu_shuffle_to_amdgpu
+      transform.apply_patterns.gpu.gpu_shuffle_to_amdgpu chipset = "gfx950"
     } : !transform.any_op
     transform.yield
   }
@@ -21,3 +21,15 @@ func.func @gpu_shuffle_swizzle(%arg0: i32) -> (i32, i1) {
   %shfl, %pred = gpu.shuffle xor %arg0, %offset, %width : i32
   func.return %shfl, %pred : i32, i1
 }
+
+  // CHECK-LABEL: func @gpu_shuffle_permlane_swap
+  //  CHECK-SAME: (%[[ARG:.*]]: i32)
+func.func @gpu_shuffle_permlane_swap(%arg0: i32) -> (i32, i1) {
+  // CHECK:  %[[TRUE:.*]] = arith.constant true
+  // CHECK:  %[[RES:.*]] = amdgpu.permlane_swap %[[ARG]] 32 : i32
+  // CHECK:  return %[[RES]], %[[TRUE]] : i32, i1
+  %width = arith.constant 64 : i32
+  %offset = arith.constant 32 : i32
+  %shfl, %pred = gpu.shuffle xor %arg0, %offset, %width : i32
+  func.return %shfl, %pred : i32, i1
+}
diff --git a/mlir/test/Dialect/LLVMIR/call-intrin.mlir b/mlir/test/Dialect/LLVMIR/call-intrin.mlir
index b8d845d7ebfe0..bf11e07dc1b6e 100644
--- a/mlir/test/Dialect/LLVMIR/call-intrin.mlir
+++ b/mlir/test/Dialect/LLVMIR/call-intrin.mlir
@@ -27,14 +27,13 @@ llvm.func @round_overloaded() -> f32 {
 
 // CHECK: define void @lifetime_start() {
 // CHECK:   %1 = alloca float, i8 1, align 4
-// CHECK:   call void @llvm.lifetime.start.p0(i64 4, ptr %1)
+// CHECK:   call void @llvm.lifetime.start.p0(ptr %1)
 // CHECK:   ret void
 // CHECK: }
 llvm.func @lifetime_start() {
-  %0 = llvm.mlir.constant(4 : i64) : i64
-  %1 = llvm.mlir.constant(1 : i8) : i8
-  %2 = llvm.alloca %1 x f32 : (i8) -> !llvm.ptr
-  llvm.call_intrinsic "llvm.lifetime.start"(%0, %2) {} : (i64, !llvm.ptr) -> ()
+  %0 = llvm.mlir.constant(1 : i8) : i8
+  %1 = llvm.alloca %0 x f32 : (i8) -> !llvm.ptr
+  llvm.call_intrinsic "llvm.lifetime.start"(%1) {} : (!llvm.ptr) -> ()
   llvm.return
 }
 
diff --git a/mlir/test/Dialect/LLVMIR/inlining.mlir b/mlir/test/Dialect/LLVMIR/inlining.mlir
index 551e0c97912d0..8e292f4c300d3 100644
--- a/mlir/test/Dialect/LLVMIR/inlining.mlir
+++ b/mlir/test/Dialect/LLVMIR/inlining.mlir
@@ -299,7 +299,7 @@ llvm.func @test_inline(%cond0 : i1, %cond1 : i1, %funcArg : f32) -> f32 {
 ^bb1:
   // Make sure the lifetime begin intrinsic has been inserted where the call
   // used to be, even though the alloca has been moved to the entry block.
-  // CHECK-NEXT: llvm.intr.lifetime.start 4, %[[PTR]]
+  // CHECK-NEXT: llvm.intr.lifetime.start %[[PTR]]
   %0 = llvm.call @static_alloca(%cond1) : (i1) -> f32
   // CHECK: llvm.cond_br %{{.+}}, ^[[BB2:.+]], ^[[BB3:.+]]
   llvm.br ^bb3(%0: f32)
@@ -307,9 +307,9 @@ llvm.func @test_inline(%cond0 : i1, %cond1 : i1, %funcArg : f32) -> f32 {
   // return sites of the callee.
   // CHECK: ^[[BB2]]:
   // CHECK-NEXT: llvm.load
-  // CHECK-NEXT: llvm.intr.lifetime.end 4, %[[PTR]]
+  // CHECK-NEXT: llvm.intr.lifetime.end %[[PTR]]
   // CHECK: ^[[BB3]]:
-  // CHECK-NEXT: llvm.intr.lifetime.end 4, %[[PTR]]
+  // CHECK-NEXT: llvm.intr.lifetime.end %[[PTR]]
 ^bb2:
   llvm.br ^bb3(%funcArg: f32)
 ^bb3(%blockArg: f32):
@@ -334,9 +334,9 @@ llvm.func @test_inline(%cond0 : i1) {
   // CHECK: "test.one_region_op"() ({
   "test.one_region_op"() ({
     %0 = llvm.call @static_alloca() : () -> f32
-    // CHECK-NEXT: llvm.intr.lifetime.start 4, %[[ALLOCA]]
+    // CHECK-NEXT: llvm.intr.lifetime.start %[[ALLOCA]]
     // CHECK-NEXT: %[[RES:.+]] = llvm.load %[[ALLOCA]]
-    // CHECK-NEXT: llvm.intr.lifetime.end 4, %[[ALLOCA]]
+    // CHECK-NEXT: llvm.intr.lifetime.end %[[ALLOCA]]
     // CHECK-NEXT: test.region_yield %[[RES]]
     test.region_yield %0 : f32
   }) : () -> ()
@@ -368,9 +368,9 @@ llvm.func @test_inline(%cond0 : i1) {
 llvm.func @alloca_with_lifetime(%cond: i1) -> f32 {
   %0 = llvm.mlir.constant(4 : i32) : i32
   %1 = llvm.alloca %0 x f32 : (i32) -> !llvm.ptr
-  llvm.intr.lifetime.start 4, %1 : !llvm.ptr
+  llvm.intr.lifetime.start %1 : !llvm.ptr
   %2 = llvm.load %1 : !llvm.ptr -> f32
-  llvm.intr.lifetime.end 4, %1 : !llvm.ptr
+  llvm.intr.lifetime.end %1 : !llvm.ptr
   %3 = llvm.fadd %2, %2 : f32
   llvm.return %3 : f32
 }
@@ -385,9 +385,9 @@ llvm.func @test_inline(%cond0 : i1, %cond1 : i1, %funcArg : f32) -> f32 {
 ^bb1:
   // Make sure the original lifetime intrinsic has been preserved, rather than
   // inserting a new one with a larger scope.
-  // CHECK: llvm.intr.lifetime.start 4, %[[PTR]]
+  // CHECK: llvm.intr.lifetime.start %[[PTR]]
   // CHECK-NEXT: llvm.load %[[PTR]]
-  // CHECK-NEXT: llvm.intr.lifetime.end 4, %[[PTR]]
+  // CHECK-NEXT: llvm.intr.lifetime.end %[[PTR]]
   // CHECK: llvm.fadd
   // CHECK-NOT: llvm.intr.lifetime.end
   %0 = llvm.call @alloca_with_lifetime(%cond1) : (i1) -> f32
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index ac1737444fcf0..4394786db5a5d 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -1220,38 +1220,6 @@ llvm.func @gpu_wmma_mma_op_invalid_result(%arg0: vector<2 x f16>, %arg1: vector<
 
 // -----
 
-llvm.func @wmmald_matrix(%arg0: !llvm.ptr) {
-  // expected-error@+1 {{'nvvm.ldmatrix' op expected source pointer in memory space 3}}
-  %l = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr) -> i32
-  llvm.return
-}
-
-// -----
-
-llvm.func @wmmald_matrix(%arg0: !llvm.ptr<3>) {
-  // expected-error@+1 {{'nvvm.ldmatrix' op expected num attribute to be 1, 2 or 4}}
-  %l = nvvm.ldmatrix %arg0 {num = 3 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr<3>) -> i32
-  llvm.return
-}
-
-// -----
-
-llvm.func @wmmald_matrix(%arg0: !llvm.ptr<3>) {
-  // expected-error@+1 {{'nvvm.ldmatrix' op expected destination type is i32}}
-  %l = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr<3>) -> !llvm.struct<(i32)>
-  llvm.return
-}
-
-// -----
-
-llvm.func @wmmald_matrix(%arg0: !llvm.ptr<3>) {
-  // expected-error@+1 {{'nvvm.ldmatrix' op expected destination type is a structure of 4 elements of type i32}}
-  %l = nvvm.ldmatrix %arg0 {num = 4 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
-  llvm.return
-}
-
-// -----
-
 llvm.func @caller() {
   // expected-error @below {{expected function call to produce a value}}
   llvm.call @callee() : () -> ()
@@ -1307,8 +1275,8 @@ func.func @cp_async(%arg0: !llvm.ptr<3>, %arg1: !llvm.ptr<1>) {
 // -----
 
 func.func @mapa(%a: !llvm.ptr, %b : i32) {
-  // expected-error @below {{`res` and `a` should have the same type}}
-  %0 = nvvm.mapa %a, %b: !llvm.ptr -> !llvm.ptr<3>
+  // expected-error @below {{'nvvm.mapa' op failed to verify that Valid address-space check(or mapping) for mapa Op}}
+  %0 = nvvm.mapa %a, %b: !llvm.ptr -> !llvm.ptr<7>
   return
 }
 
diff --git a/mlir/test/Dialect/LLVMIR/mem2reg.mlir b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
index 56634cff87aa9..716a5860a0c07 100644
--- a/mlir/test/Dialect/LLVMIR/mem2reg.mlir
+++ b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
@@ -304,10 +304,9 @@ llvm.func @g()
 // CHECK-NOT: = llvm.alloca
 llvm.func amdgpu_kernelcc @addrspace_discard() {
   %0 = llvm.mlir.constant(1 : i32) : i32
-  %1 = llvm.mlir.constant(2 : i64) : i64
-  %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr<5>
-  %3 = llvm.addrspacecast %2 : !llvm.ptr<5> to !llvm.ptr
-  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
+  %1 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr<5>
+  %2 = llvm.addrspacecast %1 : !llvm.ptr<5> to !llvm.ptr
+  llvm.intr.lifetime.start %2 : !llvm.ptr
   llvm.return
 }
 
@@ -406,9 +405,9 @@ llvm.func @unreachable_jumps_to_merge_point(%arg0: i1) -> i32 {
 llvm.func @ignore_lifetime() {
   %0 = llvm.mlir.constant(1 : i32) : i32
   %1 = llvm.alloca %0 x i32 {alignment = 4 : i64} : (i32) -> !llvm.ptr
-  llvm.intr.lifetime.start 2, %1 : !llvm.ptr
+  llvm.intr.lifetime.start %1 : !llvm.ptr
   llvm.store %0, %1 {alignment = 4 : i64} : i32, !llvm.ptr
-  llvm.intr.lifetime.end 2, %1 : !llvm.ptr
+  llvm.intr.lifetime.end %1 : !llvm.ptr
   llvm.return
 }
 
@@ -437,9 +436,9 @@ llvm.func @ignore_discardable_tree() {
   %5 = llvm.insertvalue %1, %4[1] : !llvm.struct<(i8, i16)>
   %6 = llvm.alloca %0 x !llvm.struct<(i8, i16)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
   %7 = llvm.getelementptr %6[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(i8, i16)>
-  llvm.intr.lifetime.start 2, %7 : !llvm.ptr
+  llvm.intr.lifetime.start %7 : !llvm.ptr
   llvm.store %5, %6 {alignment = 2 : i64} : !llvm.struct<(i8, i16)>, !llvm.ptr
-  llvm.intr.lifetime.end 2, %7 : !llvm.ptr
+  llvm.intr.lifetime.end %7 : !llvm.ptr
   llvm.return
 }
 
@@ -517,8 +516,8 @@ llvm.func @discardable_use_tree() {
   %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
   %3 = llvm.bitcast %2 : !llvm.ptr to !llvm.ptr
   %4 = llvm.bitcast %3 : !llvm.ptr to !llvm.ptr
-  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
-  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.intr.lifetime.start %3 : !llvm.ptr
+  llvm.intr.lifetime.start %4 : !llvm.ptr
   %5 = llvm.intr.invariant.start 2, %3 : !llvm.ptr
   llvm.intr.invariant.end %5, 2, %3 : !llvm.ptr
   llvm.return
@@ -534,8 +533,8 @@ llvm.func @non_discardable_use_tree() {
   %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
   %3 = llvm.bitcast %2 : !llvm.ptr to !llvm.ptr
   %4 = llvm.bitcast %3 : !llvm.ptr to !llvm.ptr
-  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
-  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.intr.lifetime.start %3 : !llvm.ptr
+  llvm.intr.lifetime.start %4 : !llvm.ptr
   llvm.call @use(%4) : (!llvm.ptr) -> i1
   llvm.return
 }
@@ -551,8 +550,8 @@ llvm.func @trivial_get_element_ptr() {
   %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
   %3 = llvm.bitcast %2 : !llvm.ptr to !llvm.ptr
   %4 = llvm.getelementptr %3[0] : (!llvm.ptr) -> !llvm.ptr, i8
-  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
-  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.intr.lifetime.start %3 : !llvm.ptr
+  llvm.intr.lifetime.start %4 : !llvm.ptr
   llvm.return
 }
 
@@ -565,8 +564,8 @@ llvm.func @nontrivial_get_element_ptr() {
   // CHECK: = llvm.alloca
   %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
   %4 = llvm.getelementptr %2[1] : (!llvm.ptr) -> !llvm.ptr, i8
-  llvm.intr.lifetime.start 2, %2 : !llvm.ptr
-  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.intr.lifetime.start %2 : !llvm.ptr
+  llvm.intr.lifetime.start %4 : !llvm.ptr
   llvm.return
 }
 
@@ -580,8 +579,8 @@ llvm.func @dynamic_get_element_ptr() {
   %2 = llvm.alloca %0 x i8 {alignment = 8 : i64} : (i32) -> !llvm.ptr
   %3 = llvm.bitcast %2 : !llvm.ptr to !llvm.ptr
   %4 = llvm.getelementptr %3[%0] : (!llvm.ptr, i32) -> !llvm.ptr, i8
-  llvm.intr.lifetime.start 2, %3 : !llvm.ptr
-  llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  llvm.intr.lifetime.start %3 : !llvm.ptr
+  llvm.intr.lifetime.start %4 : !llvm.ptr
   llvm.return
 }
 
diff --git a/mlir/test/Dialect/LLVMIR/nvvm.mlir b/mlir/test/Dialect/LLVMIR/nvvm.mlir
index c7fa41c98ac92..5821c2eac99dd 100644
--- a/mlir/test/Dialect/LLVMIR/nvvm.mlir
+++ b/mlir/test/Dialect/LLVMIR/nvvm.mlir
@@ -385,17 +385,6 @@ llvm.func @cp_async(%arg0: !llvm.ptr<3>, %arg1: !llvm.ptr<1>) {
   llvm.return
 }
 
-// CHECK-LABEL: llvm.func @ld_matrix
-llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
-  // CHECK: nvvm.ldmatrix %{{.*}} {layout = #nvvm.mma_layout<row>, num = 1 : i32} : (!llvm.ptr<3>) -> i32
-  %l1 = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr<3>) -> i32
-  // CHECK: nvvm.ldmatrix %{{.*}} {layout = #nvvm.mma_layout<row>, num = 2 : i32} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
-  %l2 = nvvm.ldmatrix %arg0 {num = 2 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
-  // CHECK: nvvm.ldmatrix %{{.*}} {layout = #nvvm.mma_layout<row>, num = 4 : i32} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
-  %l4 = nvvm.ldmatrix %arg0 {num = 4 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
-  llvm.return
-}
-
 // CHECK-LABEL: llvm.func @redux_sync
 llvm.func @redux_sync(%value : i32, %offset : i32) -> i32 {
   // CHECK: nvvm.redux.sync  add %{{.*}}
@@ -535,15 +524,15 @@ func.func @wgmma_wait_group_sync_aligned() {
 }
 
 func.func @griddepcontrol_wait() {
-  // CHECK: nvvm.griddepcontrol.wait
-  nvvm.griddepcontrol.wait
+  // CHECK: nvvm.griddepcontrol wait
+  nvvm.griddepcontrol wait
   return
 }
 
 func.func @griddepcontrol_launch_dependents()
 {
-  // CHECK: nvvm.griddepcontrol.launch.dependents
-  nvvm.griddepcontrol.launch.dependents
+  // CHECK: nvvm.griddepcontrol launch_dependents
+  nvvm.griddepcontrol launch_dependents
   return
 }
 
@@ -552,7 +541,7 @@ func.func @mapa(%a: !llvm.ptr, %a_shared: !llvm.ptr<3>, %b : i32) {
   // CHECK:   nvvm.mapa %{{.*}}
   %0 = nvvm.mapa %a, %b: !llvm.ptr -> !llvm.ptr
   // CHECK:   nvvm.mapa %{{.*}}
-  %1 = nvvm.mapa %a_shared, %b: !llvm.ptr<3> -> !llvm.ptr<3>
+  %1 = nvvm.mapa %a_shared, %b: !llvm.ptr<3> -> !llvm.ptr<7>
   return
 }
 
diff --git a/mlir/test/Dialect/LLVMIR/rocdl.mlir b/mlir/test/Dialect/LLVMIR/rocdl.mlir
index a2b2f84606ba0..782ef4e154440 100644
--- a/mlir/test/Dialect/LLVMIR/rocdl.mlir
+++ b/mlir/test/Dialect/LLVMIR/rocdl.mlir
@@ -981,6 +981,13 @@ llvm.func @rocdl.s.wait.expcnt() {
 
 // -----
 
+llvm.func @rocdl.readfirstlane(%src : f32) -> f32 {
+  // CHECK-LABEL: rocdl.readfirstlane
+  // CHECK: rocdl.readfirstlane %{{.*}} : f32
+  %ret = rocdl.readfirstlane %src : f32
+  llvm.return %ret : f32
+}
+
 llvm.func @rocdl.readlane(%src : f32) -> f32 {
   %cst0 = llvm.mlir.constant(0 : i32) : i32
 
@@ -1002,6 +1009,22 @@ llvm.func @rocdl.permlanex16(%src : f32) -> f32 {
 
 // -----
 
+llvm.func @rocdl.permlane16.swap(%src : i32) -> !llvm.struct<(i32, i32)> {
+  // CHECK-LABEL: rocdl.permlane16.swap
+  // CHECK: rocdl.permlane16.swap %{{.*}} %{{.*}}
+  %res = rocdl.permlane16.swap %src, %src, 0, -1  : (i32, i32) -> !llvm.struct<(i32, i32)>
+  llvm.return %res : !llvm.struct<(i32, i32)>
+}
+
+llvm.func @rocdl.permlane32.swap(%src : i32) -> !llvm.struct<(i32, i32)> {
+  // CHECK-LABEL: rocdl.permlane32.swap
+  // CHECK: rocdl.permlane32.swap %{{.*}} %{{.*}}
+  %res = rocdl.permlane32.swap %src, %src, 0, -1  : (i32, i32) -> !llvm.struct<(i32, i32)>
+  llvm.return %res : !llvm.struct<(i32, i32)>
+}
+
+// -----
+
 // expected-error@below {{attribute attached to unexpected op}}
 func.func private @expected_llvm_func() attributes { rocdl.kernel }
 
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index a0273fb1e1bf4..73447978341dc 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -685,10 +685,10 @@ func.func @fastmathFlags(%arg0: f32, %arg1: f32, %arg2: i32, %arg3: vector<2 x f
 // CHECK-LABEL: @lifetime
 // CHECK-SAME: %[[P:.*]]: !llvm.ptr
 llvm.func @lifetime(%p: !llvm.ptr) {
-  // CHECK: llvm.intr.lifetime.start 16, %[[P]]
-  llvm.intr.lifetime.start 16, %p : !llvm.ptr
-  // CHECK: llvm.intr.lifetime.end 16, %[[P]]
-  llvm.intr.lifetime.end 16, %p : !llvm.ptr
+  // CHECK: llvm.intr.lifetime.start %[[P]]
+  llvm.intr.lifetime.start %p : !llvm.ptr
+  // CHECK: llvm.intr.lifetime.end %[[P]]
+  llvm.intr.lifetime.end %p : !llvm.ptr
   llvm.return
 }
 
diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir
index fe1531d988a4f..1674bbd8c796f 100644
--- a/mlir/test/Dialect/LLVMIR/sroa.mlir
+++ b/mlir/test/Dialect/LLVMIR/sroa.mlir
@@ -177,7 +177,7 @@ llvm.func @direct_promotable_use_is_fine() -> i32 {
   // CHECK: %[[RES:.*]] = llvm.load %[[ALLOCA]]
   %3 = llvm.load %2 : !llvm.ptr -> i32
   // This is a direct use of the slot but it can be removed because it implements PromotableOpInterface.
-  llvm.intr.lifetime.start 2, %1 : !llvm.ptr
+  llvm.intr.lifetime.start %1 : !llvm.ptr
   // CHECK: llvm.return %[[RES]] : i32
   llvm.return %3 : i32
 }
diff --git a/mlir/test/Dialect/LLVMIR/target-to-data-layout-invalid.mlir b/mlir/test/Dialect/LLVMIR/target-to-data-layout-invalid.mlir
new file mode 100644
index 0000000000000..c0ff53457d17e
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/target-to-data-layout-invalid.mlir
@@ -0,0 +1,9 @@
+// REQUIRES: target=x86{{.*}}
+// RUN: mlir-opt %s -llvm-target-to-data-layout --split-input-file --verify-diagnostics
+
+// expected-error @+1 {{failed to obtain llvm::DataLayout for #llvm.target}}
+module attributes { dlti.dl_spec = #dlti.dl_spec<index = 32>,
+llvm.target =
+    #llvm.target<triple="x64_86-unknown-linux",
+                 chip="NON-EXISTING CHIP"> } {
+}
diff --git a/mlir/test/Dialect/LLVMIR/target-to-data-layout-no-init.mlir b/mlir/test/Dialect/LLVMIR/target-to-data-layout-no-init.mlir
new file mode 100644
index 0000000000000..2a9e9783eacb1
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/target-to-data-layout-no-init.mlir
@@ -0,0 +1,12 @@
+// REQUIRES: target=x86{{.*}}
+// RUN: mlir-opt %s -llvm-target-to-data-layout="initialize-llvm-targets=false" --split-input-file --verify-diagnostics
+
+// Without initializing the (right) LLVM targets/backends ("initialize-llvm-targets=false"),
+// it is not possible to obtain LLVM's DataLayout for the target.
+
+// expected-error @+1 {{failed to obtain llvm::DataLayout for #llvm.target}}
+module attributes { dlti.dl_spec = #dlti.dl_spec<index = 32>,
+llvm.target =
+    #llvm.target<triple="x64_86-unknown-linux",
+                 chip="skylake"> } {
+}
diff --git a/mlir/test/Dialect/LLVMIR/target-to-data-layout.mlir b/mlir/test/Dialect/LLVMIR/target-to-data-layout.mlir
new file mode 100644
index 0000000000000..45bfd6a465d7c
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/target-to-data-layout.mlir
@@ -0,0 +1,75 @@
+// REQUIRES: target=x86{{.*}}
+// RUN: mlir-opt -llvm-target-to-data-layout -split-input-file %s | FileCheck %s
+
+// CHECK: module attributes
+// CHECK-SAME: dlti.dl_spec = #dlti.dl_spec
+// CHECK-SAME:   "dlti.endianness" = "little"
+// CHECK-SAME: llvm.target = #llvm.target<
+// CHECK-SAME:   triple = "x86_64-unknown-linux"
+// CHECK-SAME:   chip = ""
+// CHECK-NOT:    features =
+
+module attributes { llvm.target = #llvm.target<triple = "x86_64-unknown-linux",
+                                               chip = ""> } {
+}
+
+// -----
+
+// CHECK: module attributes
+// CHECK-SAME: dlti.dl_spec = #dlti.dl_spec
+// CHECK-SAME:   "dlti.endianness" = "little"
+// CHECK-SAME: llvm.target = #llvm.target<
+// CHECK-SAME:   triple = "x86_64-unknown-linux"
+// CHECK-SAME:   chip = ""
+// CHECK-SAME:   features = "+mmx,+sse"
+
+module attributes { llvm.target = #llvm.target<triple = "x86_64-unknown-linux",
+                                               chip = "",
+                                               features = "+mmx,+sse"> } {
+}
+
+// -----
+
+// CHECK: module attributes
+// CHECK-SAME: dlti.dl_spec = #dlti.dl_spec
+// CHECK-SAME:   "dlti.endianness" = "little"
+// CHECK-SAME: llvm.target = #llvm.target<
+// CHECK-SAME:   triple = "x86_64-unknown-linux"
+// CHECK-SAME:   chip = "skylake"
+// CHECK-NOT:    features =
+
+module attributes { llvm.target = #llvm.target<triple = "x86_64-unknown-linux",
+                                               chip = "skylake"> } {
+}
+
+// -----
+
+// CHECK: module attributes
+// CHECK-SAME: dlti.dl_spec = #dlti.dl_spec
+// CHECK-SAME:   "dlti.endianness" = "little"
+// CHECK-SAME: llvm.target = #llvm.target<
+// CHECK-SAME:   triple = "x86_64-unknown-linux"
+// CHECK-SAME:   chip = "skylake"
+// CHECK-SAME:   features = "+mmx,+sse">
+
+module attributes { llvm.target = #llvm.target<triple = "x86_64-unknown-linux",
+                                               chip = "skylake",
+                                               features = "+mmx,+sse"> } {
+}
+
+// -----
+
+// CHECK: module attributes
+// CHECK-SAME: dlti.dl_spec = #dlti.dl_spec
+// CHECK-SAME:   "dlti.endianness" = "little"
+// CHECK-SAME:   index = 32
+// CHECK-SAME: llvm.target = #llvm.target<
+// CHECK-SAME:   triple = "x86_64-unknown-linux"
+// CHECK-SAME:   chip = "skylake"
+// CHECK-SAME:   features = "+mmx,+sse"
+
+module attributes { dlti.dl_spec = #dlti.dl_spec<index = 32>,
+                    llvm.target = #llvm.target<triple = "x86_64-unknown-linux",
+                                               chip = "skylake",
+                                               features = "+mmx,+sse"> } {
+}
diff --git a/mlir/test/Dialect/Linalg/block-pack-matmul-layout.mlir b/mlir/test/Dialect/Linalg/block-pack-matmul-layout.mlir
index 4ba4b09f52163..2f30e8b9d01e7 100644
--- a/mlir/test/Dialect/Linalg/block-pack-matmul-layout.mlir
+++ b/mlir/test/Dialect/Linalg/block-pack-matmul-layout.mlir
@@ -20,20 +20,6 @@ func.func @block_matmul(
   return %0 : tensor<64x64xf32>
 }
 
-func.func @block_matmul_transpose_a(
-    %A: tensor<128x64xf32>, %B: tensor<128x64xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
-  %0 = linalg.matmul_transpose_a ins(%A, %B : tensor<128x64xf32>, tensor<128x64xf32>)
-                                 outs(%C : tensor<64x64xf32>) -> tensor<64x64xf32>
-  return %0 : tensor<64x64xf32>
-}
-
-func.func @block_matmul_transpose_b(
-    %A: tensor<64x128xf32>, %B: tensor<64x128xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
-  %0 = linalg.matmul_transpose_b ins(%A, %B : tensor<64x128xf32>, tensor<64x128xf32>)
-                                 outs(%C : tensor<64x64xf32>) -> tensor<64x64xf32>
-  return %0 : tensor<64x64xf32>
-}
-
 // MMT4D-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
 // MMT4D-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
 // MMT4D-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
@@ -43,18 +29,6 @@ func.func @block_matmul_transpose_b(
 // MMT4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
 // MMT4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
 // MMT4D-COUNT-1: linalg.unpack
-// MMT4D-LABEL: func @block_matmul_transpose_a
-// MMT4D-COUNT-3: linalg.pack
-// MMT4D: linalg.generic
-// MMT4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// MMT4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// MMT4D-COUNT-1: linalg.unpack
-// MMT4D-LABEL: func @block_matmul_transpose_b
-// MMT4D-COUNT-3: linalg.pack
-// MMT4D: linalg.generic
-// MMT4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// MMT4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// MMT4D-COUNT-1: linalg.unpack
 
 // MM4D-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
 // MM4D-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d2, d1, d5, d4)>
@@ -65,18 +39,6 @@ func.func @block_matmul_transpose_b(
 // MM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
 // MM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
 // MM4D-COUNT-1: linalg.unpack
-// MM4D-LABEL: func @block_matmul_transpose_a
-// MM4D-COUNT-3: linalg.pack
-// MM4D: linalg.generic
-// MM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// MM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// MM4D-COUNT-1: linalg.unpack
-// MM4D-LABEL: func @block_matmul_transpose_b
-// MM4D-COUNT-3: linalg.pack
-// MM4D: linalg.generic
-// MM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// MM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// MM4D-COUNT-1: linalg.unpack
 
 // MTM4D-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d2, d0, d5, d3)>
 // MTM4D-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d2, d1, d5, d4)>
@@ -87,15 +49,3 @@ func.func @block_matmul_transpose_b(
 // MTM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
 // MTM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
 // MTM4D-COUNT-1: linalg.unpack
-// MTM4D-LABEL: func @block_matmul_transpose_a
-// MTM4D-COUNT-3: linalg.pack
-// MTM4D: linalg.generic
-// MTM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// MTM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// MTM4D-COUNT-1: linalg.unpack
-// MTM4D-LABEL: func @block_matmul_transpose_b
-// MTM4D-COUNT-3: linalg.pack
-// MTM4D: linalg.generic
-// MTM4D-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// MTM4D-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// MTM4D-COUNT-1: linalg.unpack
diff --git a/mlir/test/Dialect/Linalg/block-pack-matmul.mlir b/mlir/test/Dialect/Linalg/block-pack-matmul.mlir
index aa860dbd581a9..e16af1f6da0e3 100644
--- a/mlir/test/Dialect/Linalg/block-pack-matmul.mlir
+++ b/mlir/test/Dialect/Linalg/block-pack-matmul.mlir
@@ -197,150 +197,6 @@ func.func @block_batch_matmul(
 
 // -----
 
-func.func @block_matmul_transpose_a(
-    %A: tensor<128x64xf32>, %B: tensor<128x64xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
-  %0 = linalg.matmul_transpose_a ins(%A, %B : tensor<128x64xf32>, tensor<128x64xf32>)
-                                 outs(%C : tensor<64x64xf32>) -> tensor<64x64xf32>
-  return %0 : tensor<64x64xf32>
-}
-
-// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
-// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
-// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
-
-// CHECK-LABEL: func @block_matmul_transpose_a(
-// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<128x64xf32>, %[[B:[0-9a-z]+]]: tensor<128x64xf32>, %[[C:[0-9a-z]+]]: tensor<64x64xf32>
-// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<2x2x32x64xf32>
-// CHECK: %[[A_PACKED:.+]] = linalg.pack %[[A]]
-// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [32, 64]
-// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<128x64xf32> -> tensor<2x2x32x64xf32>
-// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<4x2x16x64xf32>
-// CHECK: %[[B_PACKED:.+]] = linalg.pack %[[B]]
-// CHECK-SAME:  outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [16, 64]
-// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<128x64xf32> -> tensor<4x2x16x64xf32>
-// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<2x4x32x16xf32>
-// CHECK: %[[C_PACKED:.+]] = linalg.pack %[[C]]
-// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
-// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<64x64xf32> -> tensor<2x4x32x16xf32>
-// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
-// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<2x2x32x64xf32>, tensor<4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<2x4x32x16xf32>)
-// CHECK: %[[RES_UNPACKED:.+]] = linalg.unpack %[[GEMM_RES_PACKED]]
-// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
-// CHECK-SAME:  into %[[C]] : tensor<2x4x32x16xf32> -> tensor<64x64xf32>
-// CHECK: return %[[RES_UNPACKED]] : tensor<64x64xf32>
-
-// -----
-
-func.func @block_batch_matmul_transpose_a(
-    %A: tensor<512x128x64xf32>, %B: tensor<512x128x64xf32>, %C: tensor<512x64x64xf32>) -> tensor<512x64x64xf32> {
-  %0 = linalg.batch_matmul_transpose_a ins(%A, %B : tensor<512x128x64xf32>, tensor<512x128x64xf32>)
-                                       outs(%C : tensor<512x64x64xf32>) -> tensor<512x64x64xf32>
-  return %0 : tensor<512x64x64xf32>
-}
-
-// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d3, d4, d6)>
-// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d2, d3, d5, d6)>
-// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d2, d4, d5)>
-
-// CHECK-LABEL: func @block_batch_matmul_transpose_a(
-// CHECK-SAME:   %[[A:.+]]: tensor<512x128x64xf32>, %[[B:.+]]: tensor<512x128x64xf32>, %[[C:.+]]: tensor<512x64x64xf32>
-// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<512x2x2x32x64xf32>
-// CHECK: %[[A_PACKED:.+]] = linalg.pack %[[A]]
-// CHECK-SAME:  outer_dims_perm = [0, 2, 1] inner_dims_pos = [2, 1] inner_tiles = [32, 64]
-// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<512x128x64xf32> -> tensor<512x2x2x32x64xf32>
-// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<512x4x2x16x64xf32>
-// CHECK: %[[B_PACKED:.+]] = linalg.pack %[[B]]
-// CHECK-SAME:  outer_dims_perm = [0, 2, 1] inner_dims_pos = [2, 1] inner_tiles = [16, 64]
-// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<512x128x64xf32> -> tensor<512x4x2x16x64xf32>
-// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<512x2x4x32x16xf32>
-// CHECK: %[[C_PACKED:.+]] = linalg.pack %[[C]]
-// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
-// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<512x64x64xf32> -> tensor<512x2x4x32x16xf32>
-// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
-// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// CHECK-SAME:  iterator_types = ["parallel", "parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<512x2x2x32x64xf32>, tensor<512x4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<512x2x4x32x16xf32>)
-// CHECK: %[[RES_UNPACKED:.+]] = linalg.unpack %[[GEMM_RES_PACKED]]
-// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
-// CHECK-SAME:  into %[[C]] : tensor<512x2x4x32x16xf32> -> tensor<512x64x64xf32>
-// CHECK: return %[[RES_UNPACKED]] : tensor<512x64x64xf32>
-
-// -----
-
-func.func @block_matmul_transpose_b(
-    %A: tensor<64x128xf32>, %B: tensor<64x128xf32>, %C: tensor<64x64xf32>) -> tensor<64x64xf32> {
-  %0 = linalg.matmul_transpose_b ins(%A, %B : tensor<64x128xf32>, tensor<64x128xf32>)
-                                 outs(%C : tensor<64x64xf32>) -> tensor<64x64xf32>
-  return %0 : tensor<64x64xf32>
-}
-
-// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d2, d3, d5)>
-// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d1, d2, d4, d5)>
-// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5) -> (d0, d1, d3, d4)>
-
-// CHECK-LABEL: func @block_matmul_transpose_b(
-// CHECK-SAME:    %[[A:[0-9a-z]+]]: tensor<64x128xf32>, %[[B:[0-9a-z]+]]: tensor<64x128xf32>, %[[C:[0-9a-z]+]]: tensor<64x64xf32>
-// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<2x2x32x64xf32>
-// CHECK: %[[A_PACKED:.+]] = linalg.pack %[[A]]
-// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [32, 64]
-// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<64x128xf32> -> tensor<2x2x32x64xf32>
-// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<4x2x16x64xf32>
-// CHECK: %[[B_PACKED:.+]] = linalg.pack %[[B]]
-// CHECK-SAME:  outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [16, 64]
-// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<64x128xf32> -> tensor<4x2x16x64xf32>
-// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<2x4x32x16xf32>
-// CHECK: %[[C_PACKED:.+]] = linalg.pack %[[C]]
-// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
-// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<64x64xf32> -> tensor<2x4x32x16xf32>
-// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
-// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// CHECK-SAME:  iterator_types = ["parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<2x2x32x64xf32>, tensor<4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<2x4x32x16xf32>)
-// CHECK: %[[RES_UNPACKED:.+]] = linalg.unpack %[[GEMM_RES_PACKED]]
-// CHECK-SAME:  inner_dims_pos = [0, 1] inner_tiles = [32, 16]
-// CHECK-SAME:  into %[[C]] : tensor<2x4x32x16xf32> -> tensor<64x64xf32>
-// CHECK: return %[[RES_UNPACKED]] : tensor<64x64xf32>
-
-// -----
-
-func.func @block_batch_matmul_transpose_b(
-    %A: tensor<512x64x128xf32>, %B: tensor<512x64x128xf32>, %C: tensor<512x64x64xf32>) -> tensor<512x64x64xf32> {
-  %0 = linalg.batch_matmul_transpose_b ins(%A, %B : tensor<512x64x128xf32>, tensor<512x64x128xf32>)
-                                       outs(%C : tensor<512x64x64xf32>) -> tensor<512x64x64xf32>
-  return %0 : tensor<512x64x64xf32>
-}
-
-// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d3, d4, d6)>
-// CHECK-DAG: #[[$MAP1:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d2, d3, d5, d6)>
-// CHECK-DAG: #[[$MAP2:.+]] = affine_map<(d0, d1, d2, d3, d4, d5, d6) -> (d0, d1, d2, d4, d5)>
-
-// CHECK-LABEL: func @block_batch_matmul_transpose_b(
-// CHECK-SAME:   %[[A:.+]]: tensor<512x64x128xf32>, %[[B:.+]]: tensor<512x64x128xf32>, %[[C:.+]]: tensor<512x64x64xf32>
-// CHECK: %[[PACK_DST_0:.+]] = tensor.empty() : tensor<512x2x2x32x64xf32>
-// CHECK: %[[A_PACKED:.+]] = linalg.pack %[[A]]
-// CHECK-SAME:  outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [32, 64]
-// CHECK-SAME:  into %[[PACK_DST_0]] : tensor<512x64x128xf32> -> tensor<512x2x2x32x64xf32>
-// CHECK: %[[PACK_DST_1:.+]] = tensor.empty() : tensor<512x4x2x16x64xf32>
-// CHECK: %[[B_PACKED:.+]] = linalg.pack %[[B]]
-// CHECK-SAME:  outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 64]
-// CHECK-SAME:  into %[[PACK_DST_1]] : tensor<512x64x128xf32> -> tensor<512x4x2x16x64xf32>
-// CHECK: %[[PACK_DST_2:.+]] = tensor.empty() : tensor<512x2x4x32x16xf32>
-// CHECK: %[[C_PACKED:.+]] = linalg.pack %[[C]]
-// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
-// CHECK-SAME:  into %[[PACK_DST_2]] : tensor<512x64x64xf32> -> tensor<512x2x4x32x16xf32>
-// CHECK: %[[GEMM_RES_PACKED:.+]] = linalg.generic
-// CHECK-SAME:  indexing_maps = [#[[$MAP]], #[[$MAP1]], #[[$MAP2]]]
-// CHECK-SAME:  iterator_types = ["parallel", "parallel", "parallel", "reduction", "parallel", "parallel", "reduction"]
-// CHECK-SAME:  ins(%[[A_PACKED]], %[[B_PACKED]] : tensor<512x2x2x32x64xf32>, tensor<512x4x2x16x64xf32>) outs(%[[C_PACKED]] : tensor<512x2x4x32x16xf32>)
-// CHECK: %[[RES_UNPACKED:.+]] = linalg.unpack %[[GEMM_RES_PACKED]]
-// CHECK-SAME:  inner_dims_pos = [1, 2] inner_tiles = [32, 16]
-// CHECK-SAME:  into %[[C]] : tensor<512x2x4x32x16xf32> -> tensor<512x64x64xf32>
-// CHECK: return %[[RES_UNPACKED]] : tensor<512x64x64xf32>
-
-// -----
-
 #map = affine_map<(d0, d1, d2) -> (d0, d2)>
 #map1 = affine_map<(d0, d1, d2) -> (d2, d1)>
 #map2 = affine_map<(d0, d1, d2) -> (d0, d1)>
diff --git a/mlir/test/Dialect/Linalg/decompose-unpack.mlir b/mlir/test/Dialect/Linalg/decompose-unpack.mlir
index e173d557c770d..a53dde8d3dd05 100644
--- a/mlir/test/Dialect/Linalg/decompose-unpack.mlir
+++ b/mlir/test/Dialect/Linalg/decompose-unpack.mlir
@@ -203,3 +203,20 @@ func.func @unpack_with_non_trailing_dimensions_in_inner_dims(%arg0: tensor<1x1x1
 // CHECK-SAME:                      outs(%[[EMPTY]] : tensor<1x4xf32>) permutation = [1, 0]
 // CHECK:        %[[INSERT:.+]] = tensor.insert_slice %transposed into %[[DEST]][0, 0, 0] [1, 1, 4] [1, 1, 1] : tensor<1x4xf32> into tensor<1x1x4xf32>
 // CHECK:        return %[[INSERT]]
+
+// -----
+
+/// Note "126", which is a non-unit tile-outer-dim. This is not supported.
+
+func.func @negative_non_unit_tiled_outer_dim(%src: tensor<1x126x1x1x8xf32>, %dest: tensor<1x1x1x1001xf32>) -> tensor<1x1x1x1001xf32> {
+  %unpack = linalg.unpack %src
+    outer_dims_perm = [0, 3, 2, 1]
+    inner_dims_pos = [3]
+    inner_tiles = [8]
+    into %dest : tensor<1x126x1x1x8xf32>
+    -> tensor<1x1x1x1001xf32>
+
+  return %unpack : tensor<1x1x1x1001xf32>
+}
+// CHECK-LABEL: @negative_non_unit_tiled_outer_dim(
+// CHECK: linalg.unpack
diff --git a/mlir/test/Dialect/Linalg/elementwise/named-to-elementwise.mlir b/mlir/test/Dialect/Linalg/elementwise/named-to-elementwise.mlir
new file mode 100644
index 0000000000000..2332b287ace8d
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/elementwise/named-to-elementwise.mlir
@@ -0,0 +1,56 @@
+// RUN: mlir-opt %s -linalg-morph-ops=named-to-category -split-input-file | FileCheck %s
+
+// CHECK: @exp(%[[A:.+]]: tensor<16x8xf32>, %[[B:.+]]: tensor<16x8xf32>) ->  tensor<16x8xf32> {
+// CHECK: {{.*}} = linalg.elementwise
+// CHECK-SAME:       kind=#linalg.elementwise_kind<exp>
+// CHECK-SAME:       ins(%[[A]] : tensor<16x8xf32>)
+// CHECK-SAME:       outs(%[[B]] : tensor<16x8xf32>) -> tensor<16x8xf32>
+//
+func.func @exp(%A : tensor<16x8xf32>, %B : tensor<16x8xf32>) ->  tensor<16x8xf32> {
+  %exp = linalg.exp ins(%A : tensor<16x8xf32>) outs(%B :  tensor<16x8xf32>) -> tensor<16x8xf32>
+  return %exp :  tensor<16x8xf32>
+}
+
+// ----
+
+// CHECK: @add(%[[A:.+]]: tensor<16x8xf32>, %[[B:.+]]: tensor<16x8xf32>, %[[C:.+]]: tensor<16x8xf32>) ->  tensor<16x8xf32> {
+// CHECK: {{.*}} = linalg.elementwise
+// CHECK-SAME:       kind=#linalg.elementwise_kind<add>
+// CHECK-SAME:       ins(%[[A]], %[[B]] : tensor<16x8xf32>, tensor<16x8xf32>)
+// CHECK-SAME:       outs(%[[C]] : tensor<16x8xf32>) -> tensor<16x8xf32>
+//
+func.func @add(%A : tensor<16x8xf32>, %B: tensor<16x8xf32>, %C : tensor<16x8xf32>) ->  tensor<16x8xf32> {
+  %add = linalg.add ins(%A, %B : tensor<16x8xf32>, tensor<16x8xf32>) outs(%C :  tensor<16x8xf32>) -> tensor<16x8xf32>
+  return %add :  tensor<16x8xf32>
+}
+
+// ----
+
+// CHECK: @sub(%[[A:.+]]: tensor<16x8xf32>, %[[B:.+]]: tensor<16x8xf32>, %[[C:.+]]: tensor<16x8xf32>) -> tensor<16x8xf32> {
+// CHECK: {{.*}} = linalg.elementwise
+// CHECK-SAME:       kind=#linalg.elementwise_kind<sub>
+// CHECK-SAME:       ins(%[[A]], %[[B]] : tensor<16x8xf32>, tensor<16x8xf32>)
+// CHECK-SAME:       outs(%[[C]] : tensor<16x8xf32>)
+//
+func.func @sub(%A : tensor<16x8xf32>, %B: tensor<16x8xf32>, %C : tensor<16x8xf32>) -> tensor<16x8xf32> {
+  %sub = linalg.sub ins(%A, %B : tensor<16x8xf32>, tensor<16x8xf32>) outs(%C :  tensor<16x8xf32>) -> tensor<16x8xf32>
+  return %sub : tensor<16x8xf32>
+}
+
+// ----
+
+// CHECK: @ternary_select(%[[A:.+]]: tensor<4x8x16xi1>, %[[B:.+]]: tensor<4x8x16xf32>, %[[C:.+]]: tensor<4x8x16xf32>)
+// CHECK:   %[[E:.+]] =  tensor.empty() : tensor<4x8x16xf32>
+// CHECK: {{.*}} = linalg.elementwise
+// CHECK-SAME:       kind=#linalg.elementwise_kind<select>
+// CHECK-SAME:       ins(%[[A]], %[[B]], %[[C]] : tensor<4x8x16xi1>, tensor<4x8x16xf32>, tensor<4x8x16xf32>)
+// CHECK-SAME:       outs(%[[E]] : tensor<4x8x16xf32>) -> tensor<4x8x16xf32>
+//
+func.func @ternary_select(%A: tensor<4x8x16xi1>, %B: tensor<4x8x16xf32>, %C: tensor<4x8x16xf32>)
+             -> tensor<4x8x16xf32> {
+  %empty = tensor.empty() : tensor<4x8x16xf32>
+  %select = linalg.select
+              ins(%A, %B, %C : tensor<4x8x16xi1>, tensor<4x8x16xf32>, tensor<4x8x16xf32>)
+              outs(%empty: tensor<4x8x16xf32>) -> tensor<4x8x16xf32>
+  return %select : tensor<4x8x16xf32>
+}
diff --git a/mlir/test/Dialect/Linalg/fold-add-into-dest.mlir b/mlir/test/Dialect/Linalg/fold-add-into-dest.mlir
index d8e92e40739dc..e90247d052b4b 100644
--- a/mlir/test/Dialect/Linalg/fold-add-into-dest.mlir
+++ b/mlir/test/Dialect/Linalg/fold-add-into-dest.mlir
@@ -157,36 +157,6 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-!type = tensor<2048x2048xf32>
-func.func @fold_add_on_transposed_matmuls(%arg0: !type, %arg1: !type) -> !type {
-  %0 = arith.constant dense<1.111111e+00> : !type
-  %cst = arith.constant 0.000000e+00 : f32
-  %1 = tensor.empty() : !type
-  %2 = linalg.fill ins(%cst : f32) outs(%1 : !type) -> !type
-  %3 = linalg.matmul_transpose_a ins(%arg0, %0 : !type, !type) outs(%2 : !type) -> !type
-  %4 = linalg.matmul_transpose_b ins(%arg1, %0 : !type, !type) outs(%2 : !type) -> !type
-  %5 = linalg.add ins(%3, %4 : !type, !type) outs(%1 : !type) -> !type
-  return %5 : !type
-}
-
-// CHECK-LABEL: func.func @fold_add_on_transposed_matmuls
-// CHECK: %[[ACC:.+]] = linalg.matmul_transpose_a
-// CHECK-NEXT: %[[RES:.+]] = linalg.matmul_transpose_b ins({{.+}}) outs(%[[ACC]]
-// CHECK-NOT: linalg.add
-// CHECK-NEXT: return %[[RES]]
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    transform.apply_patterns to %func {
-      transform.apply_patterns.linalg.fold_add_into_dest
-    } : !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
 !type = tensor<2048x2048xf32>
 func.func @expect_no_fold_of_add_as_dominated_op_is_not_a_contraction(%arg0: !type, %arg1: !type) -> !type {
   %0 = arith.constant dense<1.111111e+00> : !type
diff --git a/mlir/test/Dialect/Linalg/linalg-morph-category-ops.mlir b/mlir/test/Dialect/Linalg/linalg-morph-category-ops.mlir
new file mode 100644
index 0000000000000..246902389bb9e
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/linalg-morph-category-ops.mlir
@@ -0,0 +1,15 @@
+// Forward path `named -> category -> generic`
+// RUN: mlir-opt %s -linalg-morph-ops=named-to-category | FileCheck %s  --check-prefix=NAMED_TO_CATEGORY
+
+// RUN: mlir-opt %s -linalg-morph-ops=named-to-category |  \
+// RUN:   mlir-opt -linalg-morph-ops=category-to-generic | FileCheck %s  --check-prefix=CATEGORY_TO_GENERIC
+
+func.func @exp(%A : tensor<16x8xf32>, %B : tensor<16x8xf32>) ->  tensor<16x8xf32> {
+  %exp = linalg.exp ins(%A : tensor<16x8xf32>) outs(%B :  tensor<16x8xf32>) -> tensor<16x8xf32>
+  return %exp :  tensor<16x8xf32>
+}
+// NAMED_TO_CATEGORY: linalg.elementwise
+// NAMED_TO_CATEGORY-NOT: linalg.exp
+
+// CATEGORY_TO_GENERIC: linalg.generic
+// CATEGORY_TO_GENERIC-NOT: linalg.elementwise
diff --git a/mlir/test/Dialect/Linalg/linalg-morph-multi-step.mlir b/mlir/test/Dialect/Linalg/linalg-morph-multi-step.mlir
new file mode 100644
index 0000000000000..bdd29b96346e1
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/linalg-morph-multi-step.mlir
@@ -0,0 +1,14 @@
+// RUN: mlir-opt %s -linalg-morph-ops=named-to-generic |  FileCheck %s  --check-prefix=NAMED_TO_GENERIC
+// RUN: mlir-opt %s -linalg-morph-ops=named-to-generic |  mlir-opt -linalg-morph-ops=generic-to-named | \
+// RUN:   FileCheck %s  --check-prefix=ROUND_TRIP
+
+func.func @exp(%A : tensor<16x8xf32>, %B : tensor<16x8xf32>) ->  tensor<16x8xf32> {
+  %exp = linalg.exp ins(%A : tensor<16x8xf32>) outs(%B :  tensor<16x8xf32>) -> tensor<16x8xf32>
+  return %exp :  tensor<16x8xf32>
+}
+
+// NAMED_TO_GENERIC: linalg.generic
+// NAMED_TO_GENERIC-NOT: linalg.exp
+
+// ROUND_TRIP: linalg.exp
+// ROUND_TRIP-NOT: linalg.generic
diff --git a/mlir/test/Dialect/Linalg/named-ops.mlir b/mlir/test/Dialect/Linalg/named-ops.mlir
index 412f40d501154..a93e9799ceb3f 100644
--- a/mlir/test/Dialect/Linalg/named-ops.mlir
+++ b/mlir/test/Dialect/Linalg/named-ops.mlir
@@ -1222,17 +1222,6 @@ func.func @batch_reduce_matmul(%arg0: memref<?x?x?xf32>, %arg1: memref<?x?x?xf32
 
 // -----
 
-// CHECK-LABEL: func @matmul_transpose_a
-//       CHECK:   linalg.matmul_transpose_a
-//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<5x3xf32>, memref<5x7xf32>)
-//  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
-func.func @matmul_transpose_a(%arg0: memref<5x3xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul_transpose_a ins(%arg0, %arg1 : memref<5x3xf32>, memref<5x7xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// -----
-
 // CHECK-LABEL: func @matmul_transpose_a_explicit
 //       CHECK:   linalg.matmul
 //  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<5x3xf32>, memref<5x7xf32>)
@@ -1478,17 +1467,6 @@ func.func @matmul_bcast_b_transpose_a(%arg0: memref<5x3xf32>, %arg1: memref<5xf3
 
 // -----
 
-// CHECK-LABEL: func @matmul_transpose_b
-//       CHECK:   linalg.matmul_transpose_b
-//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<3x5xf32>, memref<7x5xf32>)
-//  CHECK-SAME:     outs(%{{.+}} : memref<3x7xf32>)
-func.func @matmul_transpose_b(%arg0: memref<3x5xf32>, %arg1: memref<7x5xf32>, %arg2: memref<3x7xf32>) {
-  linalg.matmul_transpose_b ins(%arg0, %arg1 : memref<3x5xf32>, memref<7x5xf32>) outs(%arg2: memref<3x7xf32>)
-  return
-}
-
-// -----
-
 // CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d3)>
 // CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
 // CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
@@ -1806,28 +1784,6 @@ func.func @bcast_A_transpose_B(%A: memref<3x5xf32>, %B: memref<2x7x5xf32>, %C: m
 
 // -----
 
-// CHECK-LABEL: func @batchmatmul_transpose_a
-//       CHECK:   linalg.batch_matmul_transpose_a
-//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<2x5x3xf32>, memref<2x5x7xf32>)
-//  CHECK-SAME:     outs(%{{.+}} : memref<2x3x7xf32>)
-func.func @batchmatmul_transpose_a(%arg0: memref<2x5x3xf32>, %arg1: memref<2x5x7xf32>, %arg2: memref<2x3x7xf32>) {
-  linalg.batch_matmul_transpose_a ins(%arg0, %arg1 : memref<2x5x3xf32>, memref<2x5x7xf32>) outs(%arg2: memref<2x3x7xf32>)
-  return
-}
-
-// -----
-
-// CHECK-LABEL: func @batchmatmul_transpose_b
-//       CHECK:   linalg.batch_matmul_transpose_b
-//  CHECK-SAME:     ins(%{{.+}}, %{{.+}} : memref<2x3x5xf32>, memref<2x7x5xf32>)
-//  CHECK-SAME:     outs(%{{.+}} : memref<2x3x7xf32>)
-func.func @batchmatmul_transpose_b(%arg0: memref<2x3x5xf32>, %arg1: memref<2x7x5xf32>, %arg2: memref<2x3x7xf32>) {
-  linalg.batch_matmul_transpose_b ins(%arg0, %arg1 : memref<2x3x5xf32>, memref<2x7x5xf32>) outs(%arg2: memref<2x3x7xf32>)
-  return
-}
-
-// -----
-
 // CHECK: #[[$ATTR_0:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
 // CHECK: #[[$ATTR_1:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
 // CHECK: #[[$ATTR_2:.+]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
diff --git a/mlir/test/Dialect/Linalg/rank-reduce-contraction-ops.mlir b/mlir/test/Dialect/Linalg/rank-reduce-contraction-ops.mlir
index 43bddb075e649..704576de41960 100644
--- a/mlir/test/Dialect/Linalg/rank-reduce-contraction-ops.mlir
+++ b/mlir/test/Dialect/Linalg/rank-reduce-contraction-ops.mlir
@@ -92,38 +92,6 @@ func.func @singleton_batch_vecmat(%arg0 : tensor<1x?xf32>, %arg1 : tensor<1x?x?x
 
 // -----
 
-func.func @singleton_batchmatmul_transpose_a(%arg0: memref<1x5x3xf32>, %arg1: memref<1x5x7xf32>, %arg2: memref<1x3x7xf32>) {
-  // CHECK-LABEL: @singleton_batchmatmul_transpose_a
-  //  CHECK-SAME:     %[[LHS:[a-zA-Z0-9]+]]: memref<1x5x3xf32>
-  //  CHECK-SAME:     %[[RHS:[a-zA-Z0-9]+]]: memref<1x5x7xf32>
-  //  CHECK-SAME:     %[[INIT:[a-zA-Z0-9]+]]: memref<1x3x7xf32>
-  //  CHECK-NEXT:   %[[COLLAPSED_LHS:.*]] = memref.collapse_shape %[[LHS]] {{\[}}[0, 1], [2]]
-  //  CHECK-NEXT:   %[[COLLAPSED_RHS:.*]] = memref.collapse_shape %[[RHS]] {{\[}}[0, 1], [2]]
-  //  CHECK-NEXT:   %[[COLLAPSED_INIT:.*]] = memref.collapse_shape %[[INIT]] {{\[}}[0, 1], [2]]
-  //  CHECK-NEXT:    linalg.matmul_transpose_a ins(%[[COLLAPSED_LHS]], %[[COLLAPSED_RHS]] : memref<5x3xf32>, memref<5x7xf32>) outs(%[[COLLAPSED_INIT]] : memref<3x7xf32>)
-  //  CHECK-NEXT:   return
-  linalg.batch_matmul_transpose_a ins(%arg0, %arg1 : memref<1x5x3xf32>, memref<1x5x7xf32>) outs(%arg2: memref<1x3x7xf32>)
-  return
-}
-
-// -----
-
-func.func @singleton_batchmatmul_transpose_b(%arg0: memref<1x3x5xf32>, %arg1: memref<1x7x5xf32>, %arg2: memref<1x3x7xf32>) {
-  // CHECK-LABEL: @singleton_batchmatmul_transpose_b
-  //  CHECK-SAME:     %[[LHS:[a-zA-Z0-9]+]]: memref<1x3x5xf32>
-  //  CHECK-SAME:     %[[RHS:[a-zA-Z0-9]+]]: memref<1x7x5xf32>
-  //  CHECK-SAME:     %[[INIT:[a-zA-Z0-9]+]]: memref<1x3x7xf32>
-  //  CHECK-NEXT:   %[[COLLAPSED_LHS:.*]] = memref.collapse_shape %[[LHS]] {{\[}}[0, 1], [2]]
-  //  CHECK-NEXT:   %[[COLLAPSED_RHS:.*]] = memref.collapse_shape %[[RHS]] {{\[}}[0, 1], [2]]
-  //  CHECK-NEXT:   %[[COLLAPSED_INIT:.*]] = memref.collapse_shape %[[INIT]] {{\[}}[0, 1], [2]]
-  //  CHECK-NEXT:    linalg.matmul_transpose_b ins(%[[COLLAPSED_LHS]], %[[COLLAPSED_RHS]] : memref<3x5xf32>, memref<7x5xf32>) outs(%[[COLLAPSED_INIT]] : memref<3x7xf32>)
-  //  CHECK-NEXT:   return
-  linalg.batch_matmul_transpose_b ins(%arg0, %arg1 : memref<1x3x5xf32>, memref<1x7x5xf32>) outs(%arg2: memref<1x3x7xf32>)
-  return
-}
-
-// -----
-
 func.func @matmul_to_matvec_tensor(%arg0: tensor<?x?xf32>, %arg1: tensor<?x1xf32>, %arg2: tensor<?x1xf32>) -> tensor<?x1xf32> {
   // CHECK-LABEL: @matmul_to_matvec_tensor
   //  CHECK-SAME:     %[[LHS:[a-zA-Z0-9]+]]: tensor<?x?xf32>
@@ -226,59 +194,6 @@ func.func @matvec_to_dot_tensor(%arg0: tensor<1x?xf32>, %arg1: tensor<?xf32>, %a
 
 // -----
 
-func.func @matmul_transpose_a_to_vecmat(%arg0: tensor<256x1xf32>, %arg1: tensor<256x512xf32>, %arg2: tensor<1x512xf32>) -> tensor<1x512xf32> {
-  // CHECK-LABEL: @matmul_transpose_a_to_vecmat
-  // CHECK: collapse_shape {{.*}} into tensor<256xf32>
-  // CHECK: collapse_shape {{.*}} into tensor<512xf32>
-  // CHECK: linalg.vecmat
-  // CHECK: expand_shape {{.*}} into tensor<1x512xf32>
-    %0 = linalg.matmul_transpose_a ins(%arg0, %arg1: tensor<256x1xf32>, tensor<256x512xf32>) outs(%arg2: tensor<1x512xf32>) -> tensor<1x512xf32>
-    return %0 : tensor<1x512xf32>
-}
-
-// -----
-
-func.func @batch_matmul_transpose_a_to_batch_vecmat(%arg0: tensor<64x256x1xf32>, %arg1: tensor<64x256x512xf32>, %arg2: tensor<64x1x512xf32>) -> tensor<64x1x512xf32> {
-  // CHECK-LABEL: @batch_matmul_transpose_a_to_batch_vecmat
-  // CHECK: collapse_shape {{.*}} into tensor<64x256xf32>
-  // CHECK: collapse_shape {{.*}} into tensor<64x512xf32>
-  // CHECK: linalg.batch_vecmat
-  // CHECK: expand_shape {{.*}} into tensor<64x1x512xf32>
-    %0 = linalg.batch_matmul_transpose_a ins(%arg0, %arg1: tensor<64x256x1xf32>, tensor<64x256x512xf32>) outs(%arg2: tensor<64x1x512xf32>) -> tensor<64x1x512xf32>
-    return %0 : tensor<64x1x512xf32>
-}
-
-// -----
-
-func.func @matmul_transpose_b_to_matvec(%arg0: memref<?x?xf32>, %arg1: memref<1x?xf32>, %arg2: memref<?x1xf32>) {
-  // CHECK-LABEL: @matmul_transpose_b_to_matvec
-  // CHECK: linalg.matvec
-    linalg.matmul_transpose_b ins(%arg0, %arg1: memref<?x?xf32>, memref<1x?xf32>) outs(%arg2: memref<?x1xf32>)
-    return
-}
-
-// -----
-
-func.func @batchmatmul_transpose_b_to_batchmatvec_tensor(%arg0: tensor<64x128x256xf32>, %arg1: tensor<64x1x256xf32>, %arg2: tensor<64x128x1xf32>) -> tensor<64x128x1xf32> {
-  // CHECK: collapse_shape {{.*}} into tensor<64x256xf32>
-  // CHECK: collapse_shape {{.*}} into tensor<64x128xf32>
-  // CHECK: linalg.batch_matvec
-  // CHECK: expand_shape {{.*}} into tensor<64x128x1xf32>
-    %0 = linalg.batch_matmul_transpose_b ins(%arg0, %arg1: tensor<64x128x256xf32>, tensor<64x1x256xf32>) outs(%arg2: tensor<64x128x1xf32>) -> tensor<64x128x1xf32> 
-    return %0 : tensor<64x128x1xf32>
-}
-
-// -----
-
-func.func @batchmatmul_transpose_b_to_to_dot(%arg0: tensor<1x1x?xf32>, %arg1: tensor<1x1x?xf32>, %arg2: tensor<1x1x1xf32>) -> tensor<1x1x1xf32> {
-  // CHECK-LABEL: @batchmatmul_transpose_b_to_to_dot
-  // CHECK: linalg.dot
-    %0 = linalg.batch_matmul_transpose_b ins(%arg0, %arg1: tensor<1x1x?xf32>, tensor<1x1x?xf32>) outs(%arg2: tensor<1x1x1xf32>) -> tensor<1x1x1xf32> 
-    return %0 : tensor<1x1x1xf32>
-}
-
-// -----
-
 func.func @nonsingleton_batch_matmul(%arg0 : tensor<2x?x?xf32>, %arg1 : tensor<2x?x?xf32>, %arg2: tensor<2x?x?xf32>) -> tensor<2x?x?xf32> {
   // CHECK-LABEL: @nonsingleton_batch_matmul
   // CHECK-NOT:   collapse_shape
diff --git a/mlir/test/Dialect/Linalg/roundtrip.mlir b/mlir/test/Dialect/Linalg/roundtrip.mlir
index 4edbc6eda3eae..563013d4083af 100644
--- a/mlir/test/Dialect/Linalg/roundtrip.mlir
+++ b/mlir/test/Dialect/Linalg/roundtrip.mlir
@@ -436,6 +436,34 @@ func.func @reduce(%input: tensor<16x32x64xf32>,
 //  CHECK-SAME:   outs
 //  CHECK-SAME:   dimensions = [1]
 
+
+// -----
+
+
+func.func @reduce_not_short_form_compatible(%input: tensor<16x32x64xf32>,
+                             %init: tensor<16x64xf32>) -> tensor<16x64xf32> {
+  %reduce = linalg.reduce
+    ins(%input:tensor<16x32x64xf32>)
+    outs(%init:tensor<16x64xf32>)
+    dimensions = [1]
+    (%in1: f32, %in2: f32) {
+      %0 = arith.addf %in1, %in2: f32
+      linalg.yield %in1: f32
+    }
+  func.return %reduce : tensor<16x64xf32>
+}
+
+// CHECK-LABEL: func @reduce_not_short_form_compatible
+// CHECK-SAME:  %[[INPUT:.*]]: tensor<16x32x64xf32>
+// CHECK-SAME:  %[[INIT:.*]]: tensor<16x64xf32>
+// CHECK-NOT: linalg.reduce { arith.addf } ins(%[[INPUT]] : tensor<16x32x64xf32>
+// CHECK:     linalg.reduce ins(%[[INPUT]] : tensor<16x32x64xf32>) outs(%[[INIT]] : tensor<16x64xf32>) 
+// CHECK-SAME:  dimensions = [1]
+// CHECK:       (%[[IN1:.*]]: f32, %[[IN2:.*]]: f32) {
+// CHECK-NEXT:    %[[ADD_RESULT:.*]] = arith.addf %[[IN1]], %[[IN2]] : f32
+// CHECK-NEXT:    linalg.yield %[[IN1]] : f32
+// CHECK-NEXT:  }
+
 // -----
 
 func.func @reduce_memref(%input: memref<16x32x64xf32>,
@@ -592,6 +620,27 @@ func.func @map_arith_with_attr(%lhs: tensor<64xf32>, %rhs: tensor<64xf32>,
 
 // -----
 
+func.func @map_not_short_form_compatible(%lhs: tensor<1x32xf32>, %rhs: tensor<1x32xf32>, %init: tensor<1x32xf32>) -> tensor<1x32xf32> {
+  %mapped = linalg.map ins(%lhs, %rhs : tensor<1x32xf32>, tensor<1x32xf32>) outs(%init : tensor<1x32xf32>)
+    (%in_1: f32, %in_2: f32) {
+      %1 = arith.maximumf %in_1, %in_2 : f32
+      linalg.yield %in_1 : f32
+    }
+  func.return %mapped : tensor<1x32xf32>
+}
+
+// CHECK-LABEL: func @map_not_short_form_compatible
+// CHECK-SAME:        %[[LHS:.*]]: tensor<1x32xf32>, %[[RHS:.*]]: tensor<1x32xf32>, %[[INIT:.*]]: tensor<1x32xf32>
+// CHECK-NOT:     linalg.map { arith.maximumf } ins(%[[LHS]] : tensor<1x32xf32>
+// CHECK:         linalg.map ins(%[[LHS]], %[[RHS]] : tensor<1x32xf32>, tensor<1x32xf32>) 
+// CHECK-SAME:               outs(%[[INIT]] : tensor<1x32xf32>)
+// CHECK-NEXT:      (%[[IN1:.*]]: f32, %[[IN2:.*]]: f32) {
+// CHECK-NEXT:        %[[MAX_RESULT:.*]] = arith.maximumf %[[IN1]], %[[IN2]] : f32
+// CHECK-NEXT:        linalg.yield %[[IN1]] : f32
+// CHECK-NEXT:    }
+
+// -----
+
 func.func @reduce_arith_with_attr(%input: tensor<16x32x64xf32>,
                   %init: tensor<16x64xf32>) -> tensor<16x64xf32> {
   %reduce = linalg.reduce
diff --git a/mlir/test/Dialect/Linalg/namedop_conversion.mlir b/mlir/test/Dialect/Linalg/simplify-depthwise-conv.mlir
similarity index 96%
rename from mlir/test/Dialect/Linalg/namedop_conversion.mlir
rename to mlir/test/Dialect/Linalg/simplify-depthwise-conv.mlir
index 4f2f2720037cd..70e68e787242e 100644
--- a/mlir/test/Dialect/Linalg/namedop_conversion.mlir
+++ b/mlir/test/Dialect/Linalg/simplify-depthwise-conv.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -linalg-named-op-conversion -split-input-file | FileCheck %s
+// RUN: mlir-opt %s --simplify-depthwise-conv -split-input-file | FileCheck %s
 
 // CHECK-LABEL: @depthwise_conv
 func.func @depthwise_conv(%arg0: tensor<?x?x?x?xf32>, %arg1: tensor<?x?x?x1xf32>, %arg2: tensor<?x?x?x?x1xf32>) -> tensor<?x?x?x?x1xf32> {
diff --git a/mlir/test/Dialect/Linalg/tile-to-forall.mlir b/mlir/test/Dialect/Linalg/tile-to-forall.mlir
index 778d5bb8b9c84..1b0bade728b44 100644
--- a/mlir/test/Dialect/Linalg/tile-to-forall.mlir
+++ b/mlir/test/Dialect/Linalg/tile-to-forall.mlir
@@ -504,7 +504,7 @@ func.func @matmul_tile_size_dynamic(%A: tensor<?x?xf32>, %B: tensor<?x?xf32>, %C
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.matmul_transpose_b"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %c10 = transform.param.constant 10 : i64 -> !transform.param<i64>
     %c20 = transform.param.constant 20 : i64 -> !transform.param<i64>
     %sz = transform.merge_handles %c10, %c20 : !transform.param<i64>
diff --git a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
index f7418769f79ca..9a3dcf0b485d5 100644
--- a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
@@ -14,11 +14,11 @@ module attributes {transform.with_named_sequence} {
       : (!transform.any_op) -> !transform.any_op
 
     // Tile to 5 then pad to 8
-    %fill_l1, %loops_l1 = transform.structured.tile_using_for %fill tile_sizes [5] 
+    %fill_l1, %loops_l1 = transform.structured.tile_using_for %fill tile_sizes [5]
       : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     %fill_padded, %_ = transform.structured.pad_tiling_interface %fill_l1 to padding_sizes [8] {
-      padding_values=[0.0 : f32, 0.0 : f32]
+      padding_values= [#ub.poison, 0.0 : f32]
     } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
 
     transform.yield
@@ -33,9 +33,9 @@ func.func @pad_lhs(
      -> tensor<24x25xf32>
 {
   //      CHECK: scf.for %{{.*}} -> (tensor<24x25xf32>)
-  //      CHECK:   tensor.pad %{{.*}} 
+  //      CHECK:   tensor.pad %{{.*}}
   //      CHECK:     : tensor<?x12xf32> to tensor<8x12xf32>
-  //      CHECK:   tensor.pad %{{.*}} 
+  //      CHECK:   tensor.pad %{{.*}}
   //      CHECK:     : tensor<?x25xf32> to tensor<8x25xf32>
   //      CHECK:   linalg.matmul ins(%{{.*}}, %{{.*}} : tensor<8x12xf32>, tensor<12x25xf32>) outs(%{{.*}} : tensor<8x25xf32>) -> tensor<8x25xf32>
   //      CHECK:   tensor.extract_slice %{{.*}}[0, 0] [%{{.*}}, 25] [1, 1]
@@ -92,7 +92,7 @@ module {
       %padded, %pad = transform.structured.pad_tiling_interface %0 to padding_sizes [8, 0, 14] {
         padding_values = [0.000000e+00 : f32, 0.000000e+00 : f32, 0.000000e+00 : f32]
       } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
-      transform.yield 
+      transform.yield
     }
   }
 }
@@ -147,7 +147,7 @@ module {
       %padded, %pad = transform.structured.pad_tiling_interface %0 to padding_sizes [8, 0, 14] {
         padding_values = [0.000000e+00 : f32, 0.000000e+00 : f32, 0.000000e+00 : f32]
       } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
-      transform.yield 
+      transform.yield
     }
   }
 }
diff --git a/mlir/test/Dialect/Linalg/transform-op-pad.mlir b/mlir/test/Dialect/Linalg/transform-op-pad.mlir
index f91eb9c30a51a..51bf4a23406d4 100644
--- a/mlir/test/Dialect/Linalg/transform-op-pad.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-pad.mlir
@@ -465,14 +465,14 @@ module attributes {transform.with_named_sequence} {
 // CHECK: %[[RHS:.*]] = tensor.pad
 // CHECK: scf.for
 // CHECK-DAG: tensor.extract_slice %[[LHS]][0, %{{.*}}] [%{{.*}}, 32]
-// CHECK-DAG: tensor.extract_slice %[[RHS]][0, %{{.*}}] [%{{.*}}, 32]
+// CHECK-DAG: tensor.extract_slice %[[RHS]][%{{.*}}, 0] [32, %{{.*}}]
 func.func @dyn_pad_tiling(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>, %arg2: tensor<?x?xf32>) -> tensor<?x?xf32> {
-  %0 = linalg.matmul_transpose_b ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>) outs(%arg2 : tensor<?x?xf32>) -> tensor<?x?xf32>
+  %0 = linalg.matmul ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>) outs(%arg2 : tensor<?x?xf32>) -> tensor<?x?xf32>
   return %0 : tensor<?x?xf32>
 }
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.matmul_transpose_b"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg0 : (!transform.any_op) -> !transform.any_op
     %padded, %pad, %copy = transform.structured.pad %0 pad_to_multiple_of [32] use_prescribed_tensor_shapes {padding_dimensions = [2], padding_values = [0.000000e+00 : f32, 0.000000e+00 : f32, 0.000000e+00 : f32]} : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
     %tiled_linalg_op, %loops = transform.structured.tile_using_for %padded tile_sizes [0, 0, 32] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
     %1 = transform.structured.match ops{["func.func"]} in %arg0 : (!transform.any_op) -> !transform.any_op
diff --git a/mlir/test/Dialect/Linalg/transform-op-specialize-matmul.mlir b/mlir/test/Dialect/Linalg/transform-op-specialize-matmul.mlir
index f64953bceefe1..bd4c65512dfa6 100644
--- a/mlir/test/Dialect/Linalg/transform-op-specialize-matmul.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-specialize-matmul.mlir
@@ -30,66 +30,6 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-#map = affine_map<(d0, d1, d2) -> (d2, d0)>
-#map1 = affine_map<(d0, d1, d2) -> (d2, d1)>
-#map2 = affine_map<(d0, d1, d2) -> (d0, d1)>
-func.func @matmul_transpose_a(%arg0: memref<5x3xf32>, %arg1: memref<5x7xf32>, %arg2: memref<3x7xf32>) {
-  linalg.generic
-     {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "reduction"]}
-     ins(%arg0, %arg1 : memref<5x3xf32>, memref<5x7xf32>) outs(%arg2 : memref<3x7xf32>) {
-      ^bb0(%in: f32, %in_0: f32, %out: f32):
-       %0 = arith.mulf %in, %in_0 : f32
-       %1 = arith.addf %out, %0 : f32
-       linalg.yield %1 : f32
-  }
-  return
-}
-
-// CHECK-LABEL: @matmul_transpose_a
-// CHECK-SAME: %[[ARG0:.+]]: memref<5x3xf32>, %[[ARG1:.+]]: memref<5x7xf32>, %[[ARG2:.+]]: memref<3x7xf32>) {
-// CHECK-NOT: linalg.generic
-// CHECK: linalg.matmul_transpose_a ins(%[[ARG0]], %[[ARG1]] : memref<5x3xf32>, memref<5x7xf32>) outs(%[[ARG2]] : memref<3x7xf32>)
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match interface{LinalgOp} in %arg0 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.structured.specialize %0 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-#map = affine_map<(d0, d1, d2) -> (d0, d2)>
-#map1 = affine_map<(d0, d1, d2) -> (d1, d2)>
-#map2 = affine_map<(d0, d1, d2) -> (d0, d1)>
-func.func @matmul_transpose_b(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32>, %arg2: tensor<?x?xf32>) -> tensor<?x?xf32> {
-  %0 = linalg.generic
-          {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "reduction"]}
-          ins(%arg0, %arg1 : tensor<?x?xf32>, tensor<?x?xf32>) outs(%arg2 : tensor<?x?xf32>) {
-  ^bb0(%in: f32, %in_0: f32, %out: f32):
-    %1 = arith.mulf %in, %in_0 : f32
-    %2 = arith.addf %out, %1 : f32
-    linalg.yield %2 : f32
-  } -> tensor<?x?xf32>
-  return %0 : tensor<?x?xf32>
-}
-
-// CHECK-LABEL: @matmul_transpose_b
-// CHECK-SAME: %[[ARG0:.+]]: tensor<?x?xf32>, %[[ARG1:.+]]: tensor<?x?xf32>, %[[ARG2:.+]]: tensor<?x?xf32>) -> tensor<?x?xf32>
-// CHECK-NOT: linalg.generic
-// CHECK: linalg.matmul_transpose_b ins(%[[ARG0]], %[[ARG1]] : tensor<?x?xf32>, tensor<?x?xf32>) outs(%[[ARG2]] : tensor<?x?xf32>) -> tensor<?x?xf32>
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match interface{LinalgOp} in %arg0 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.structured.specialize %0 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
 #map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
 #map1 = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
 #map2 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
@@ -117,32 +57,3 @@ module attributes {transform.with_named_sequence} {
     transform.yield
   }
 }
-
-// -----
-#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
-#map1 = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>
-#map2 = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
-func.func @batch_matmul_transpose_b(%arg0: tensor<?x?x?xf32>, %arg1: tensor<?x?x?xf32>, %arg2: tensor<?x?x?xf32>) -> tensor<?x?x?xf32> {
-  %0 = linalg.generic
-       {indexing_maps = [#map, #map1, #map2], iterator_types = ["parallel", "parallel", "parallel", "reduction"]}
-       ins(%arg0, %arg1 : tensor<?x?x?xf32>, tensor<?x?x?xf32>) outs(%arg2 : tensor<?x?x?xf32>) {
-  ^bb0(%in: f32, %in_0: f32, %out: f32):
-      %1 = arith.mulf %in, %in_0 : f32
-      %2 = arith.addf %out, %1 : f32
-      linalg.yield %2 : f32
-  } -> tensor<?x?x?xf32>
-  return %0 : tensor<?x?x?xf32>
-}
-
-// CHECK-LABEL: @batch_matmul_transpose_b
-// CHECK-SAME: %[[ARG0:.+]]: tensor<?x?x?xf32>, %[[ARG1:.+]]: tensor<?x?x?xf32>, %[[ARG2:.+]]: tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
-// CHECK-NOT: linalg.generic
-// CHECK: linalg.batch_matmul_transpose_b ins(%[[ARG0]], %[[ARG1]] : tensor<?x?x?xf32>, tensor<?x?x?xf32>) outs(%[[ARG2]] : tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match interface{LinalgOp} in %arg0 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.structured.specialize %0 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
diff --git a/mlir/test/Dialect/Linalg/transpose-matmul.mlir b/mlir/test/Dialect/Linalg/transpose-matmul.mlir
index d2b7e9f7f1992..4ee87fb11d527 100644
--- a/mlir/test/Dialect/Linalg/transpose-matmul.mlir
+++ b/mlir/test/Dialect/Linalg/transpose-matmul.mlir
@@ -1,6 +1,20 @@
 // RUN: mlir-opt -transform-preload-library='transform-library-paths=%p/transpose-matmul-a.mlir' -transform-interpreter -split-input-file %s | FileCheck %s --check-prefixes=CHECK,TRANSPOSE-A
 // RUN: mlir-opt -transform-preload-library='transform-library-paths=%p/transpose-matmul-b.mlir' -transform-interpreter -split-input-file %s | FileCheck %s --check-prefixes=CHECK,TRANSPOSE-B
 
+// TRANSPOSE-A-DAG: #[[$MA:.*]] = affine_map<(d0, d1, d2) -> (d2, d0)>
+// TRANSPOSE-A-DAG: #[[$MB:.*]] = affine_map<(d0, d1, d2) -> (d2, d1)>
+// TRANSPOSE-A-DAG: #[[$MC:.*]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+// TRANSPOSE-A-DAG: #[[$BMA:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d1)>
+// TRANSPOSE-A-DAG: #[[$BMB:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d3, d2)>
+// TRANSPOSE-A-DAG: #[[$BMC:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
+// TRANSPOSE-B-DAG: #[[$MA:.*]] = affine_map<(d0, d1, d2) -> (d0, d2)>
+// TRANSPOSE-B-DAG: #[[$MB:.*]] = affine_map<(d0, d1, d2) -> (d1, d2)>
+// TRANSPOSE-B-DAG: #[[$MC:.*]] = affine_map<(d0, d1, d2) -> (d0, d1)>
+// TRANSPOSE-B-DAG: #[[$BMA:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>
+// TRANSPOSE-B-DAG: #[[$BMB:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>
+// TRANSPOSE-B-DAG: #[[$BMC:.*]] = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2)>
+
 // CHECK-LABEL:   func.func @matmul_static(
 // CHECK-SAME:                             %[[A:.*]]: tensor<16x8xf32>,
 // CHECK-SAME:                             %[[B:.*]]: tensor<8x16xf32>) -> tensor<16x16xf32> {
@@ -9,10 +23,10 @@
 // CHECK:           %[[C_ZERO:.*]] = linalg.fill ins(%[[C0_F32]] : f32) outs(%[[C_INIT]] : tensor<16x16xf32>) -> tensor<16x16xf32>
 // TRANSPOSE-A:     %[[A_TRANSP_INIT:.*]] = tensor.empty() : tensor<8x16xf32>
 // TRANSPOSE-A:     %[[A_TRANSP:.*]] = linalg.transpose ins(%[[A]] : tensor<16x8xf32>) outs(%[[A_TRANSP_INIT]] : tensor<8x16xf32>) permutation = [1, 0]
-// TRANSPOSE-A:     %[[C:.*]] = linalg.matmul_transpose_a ins(%[[A_TRANSP]], %[[B]] : tensor<8x16xf32>, tensor<8x16xf32>) outs(%[[C_ZERO]] : tensor<16x16xf32>) -> tensor<16x16xf32>
+// TRANSPOSE-A:     %[[C:.*]] = linalg.matmul indexing_maps = [#[[$MA]], #[[$MB]], #[[$MC]]] ins(%[[A_TRANSP]], %[[B]] : tensor<8x16xf32>, tensor<8x16xf32>) outs(%[[C_ZERO]] : tensor<16x16xf32>) -> tensor<16x16xf32>
 // TRANSPOSE-B:     %[[B_TRANSP_INIT:.*]] = tensor.empty() : tensor<16x8xf32>
 // TRANSPOSE-B:     %[[B_TRANSP:.*]] = linalg.transpose ins(%[[B]] : tensor<8x16xf32>) outs(%[[B_TRANSP_INIT]] : tensor<16x8xf32>) permutation = [1, 0]
-// TRANSPOSE-B:     %[[C:.*]] = linalg.matmul_transpose_b ins(%[[A]], %[[B_TRANSP]] : tensor<16x8xf32>, tensor<16x8xf32>) outs(%[[C_ZERO]] : tensor<16x16xf32>) -> tensor<16x16xf32>
+// TRANSPOSE-B:     %[[C:.*]] = linalg.matmul indexing_maps = [#[[$MA]], #[[$MB]], #[[$MC]]] ins(%[[A]], %[[B_TRANSP]] : tensor<16x8xf32>, tensor<16x8xf32>) outs(%[[C_ZERO]] : tensor<16x16xf32>) -> tensor<16x16xf32>
 // CHECK:           return %[[C]] : tensor<16x16xf32>
 // CHECK:         }
 func.func @matmul_static(%A: tensor<16x8xf32>, %B: tensor<8x16xf32>) -> (tensor<16x16xf32>) {
@@ -38,11 +52,11 @@ func.func @matmul_static(%A: tensor<16x8xf32>, %B: tensor<8x16xf32>) -> (tensor<
 // TRANSPOSE-A:     %[[A_DIM1:.*]] = tensor.dim %[[A]], %[[C1]] : tensor<?x?xf32>
 // TRANSPOSE-A:     %[[A_TRANSP_INIT:.*]] = tensor.empty(%[[A_DIM1]], %[[A_DIM0]]) : tensor<?x?xf32>
 // TRANSPOSE-A:     %[[A_TRANSP:.*]] = linalg.transpose ins(%[[A]] : tensor<?x?xf32>) outs(%[[A_TRANSP_INIT]] : tensor<?x?xf32>) permutation = [1, 0]
-// TRANSPOSE-A:     %[[C:.*]] = linalg.matmul_transpose_a ins(%[[A_TRANSP]], %[[B]] : tensor<?x?xf32>, tensor<?x?xf32>) outs(%[[C_ZERO]] : tensor<?x?xf32>) -> tensor<?x?xf32>
+// TRANSPOSE-A:     %[[C:.*]] = linalg.matmul indexing_maps = [#[[$MA]], #[[$MB]], #[[$MC]]] ins(%[[A_TRANSP]], %[[B]] : tensor<?x?xf32>, tensor<?x?xf32>) outs(%[[C_ZERO]] : tensor<?x?xf32>) -> tensor<?x?xf32>
 // TRANSPOSE-B:     %[[B_DIM0:.*]] = tensor.dim %[[B]], %[[C0]] : tensor<?x?xf32>
 // TRANSPOSE-B:     %[[B_TRANSP_INIT:.*]] = tensor.empty(%[[B_DIM1]], %[[B_DIM0]]) : tensor<?x?xf32>
 // TRANSPOSE-B:     %[[B_TRANSP:.*]] = linalg.transpose ins(%[[B]] : tensor<?x?xf32>) outs(%[[B_TRANSP_INIT]] : tensor<?x?xf32>) permutation = [1, 0]
-// TRANSPOSE-B:     %[[C:.*]] = linalg.matmul_transpose_b ins(%[[A]], %[[B_TRANSP]] : tensor<?x?xf32>, tensor<?x?xf32>) outs(%[[C_ZERO]] : tensor<?x?xf32>) -> tensor<?x?xf32>
+// TRANSPOSE-B:     %[[C:.*]] = linalg.matmul indexing_maps = [#[[$MA]], #[[$MB]], #[[$MC]]] ins(%[[A]], %[[B_TRANSP]] : tensor<?x?xf32>, tensor<?x?xf32>) outs(%[[C_ZERO]] : tensor<?x?xf32>) -> tensor<?x?xf32>
 // CHECK:           return %[[C]] : tensor<?x?xf32>
 // CHECK:         }
 func.func @matmul_dynamic(%A: tensor<?x?xf32>, %B: tensor<?x?xf32>) -> (tensor<?x?xf32>) {
@@ -69,10 +83,10 @@ func.func @matmul_dynamic(%A: tensor<?x?xf32>, %B: tensor<?x?xf32>) -> (tensor<?
 // CHECK:           %[[C_ZERO:.*]] = linalg.fill ins(%[[C0_F32]] : f32) outs(%[[C_INIT]] : tensor<?x16xf32>) -> tensor<?x16xf32>
 // TRANSPOSE-A:     %[[A_TRANSP_INIT:.*]] = tensor.empty(%[[A_DIM0]]) : tensor<8x?xf32>
 // TRANSPOSE-A:     %[[A_TRANSP:.*]] = linalg.transpose ins(%[[A]] : tensor<?x8xf32>) outs(%[[A_TRANSP_INIT]] : tensor<8x?xf32>) permutation = [1, 0]
-// TRANSPOSE-A:     %[[B0:.*]] = linalg.matmul_transpose_a ins(%[[A_TRANSP]], %[[B]] : tensor<8x?xf32>, tensor<8x16xf32>) outs(%[[C_ZERO]] : tensor<?x16xf32>) -> tensor<?x16xf32>
+// TRANSPOSE-A:     %[[B0:.*]] = linalg.matmul indexing_maps = [#[[$MA]], #[[$MB]], #[[$MC]]] ins(%[[A_TRANSP]], %[[B]] : tensor<8x?xf32>, tensor<8x16xf32>) outs(%[[C_ZERO]] : tensor<?x16xf32>) -> tensor<?x16xf32>
 // TRANSPOSE-B:     %[[B_TRANSP_INIT:.*]] = tensor.empty() : tensor<16x8xf32>
 // TRANSPOSE-B:     %[[B_TRANSP:.*]] = linalg.transpose ins(%[[B]] : tensor<8x16xf32>) outs(%[[B_TRANSP_INIT]] : tensor<16x8xf32>) permutation = [1, 0]
-// TRANSPOSE-B:     %[[B0:.*]] = linalg.matmul_transpose_b ins(%[[A]], %[[B_TRANSP]] : tensor<?x8xf32>, tensor<16x8xf32>) outs(%[[C_ZERO]] : tensor<?x16xf32>) -> tensor<?x16xf32>
+// TRANSPOSE-B:     %[[B0:.*]] = linalg.matmul indexing_maps = [#[[$MA]], #[[$MB]], #[[$MC]]] ins(%[[A]], %[[B_TRANSP]] : tensor<?x8xf32>, tensor<16x8xf32>) outs(%[[C_ZERO]] : tensor<?x16xf32>) -> tensor<?x16xf32>
 // CHECK:           return %[[B0]] : tensor<?x16xf32>
 // CHECK:         }
 func.func @matmul_mixed(%A: tensor<?x8xf32>, %B: tensor<8x16xf32>) -> (tensor<?x16xf32>) {
@@ -96,10 +110,10 @@ func.func @matmul_mixed(%A: tensor<?x8xf32>, %B: tensor<8x16xf32>) -> (tensor<?x
 // CHECK:           %[[C_ZERO:.*]] = linalg.fill ins(%[[C0_F32]] : f32) outs(%[[C_INIT]] : tensor<2x16x16xf32>) -> tensor<2x16x16xf32>
 // TRANSPOSE-A:     %[[A_TRANSP_INIT:.*]] = tensor.empty() : tensor<2x8x16xf32>
 // TRANSPOSE-A:     %[[A_TRANSP:.*]] = linalg.transpose ins(%[[A]] : tensor<2x16x8xf32>) outs(%[[A_TRANSP_INIT]] : tensor<2x8x16xf32>) permutation = [0, 2, 1]
-// TRANSPOSE-A:     %[[C:.*]] = linalg.batch_matmul_transpose_a ins(%[[A_TRANSP]], %[[B]] : tensor<2x8x16xf32>, tensor<2x8x16xf32>) outs(%[[C_ZERO]] : tensor<2x16x16xf32>) -> tensor<2x16x16xf32>
+// TRANSPOSE-A:     %[[C:.*]] = linalg.batch_matmul indexing_maps = [#[[$BMA]], #[[$BMB]], #[[$BMC]]] ins(%[[A_TRANSP]], %[[B]] : tensor<2x8x16xf32>, tensor<2x8x16xf32>) outs(%[[C_ZERO]] : tensor<2x16x16xf32>) -> tensor<2x16x16xf32>
 // TRANSPOSE-B:     %[[B_TRANSP_INIT:.*]] = tensor.empty() : tensor<2x16x8xf32>
 // TRANSPOSE-B:     %[[B_TRANSP:.*]] = linalg.transpose ins(%[[B]] : tensor<2x8x16xf32>) outs(%[[B_TRANSP_INIT]] : tensor<2x16x8xf32>) permutation = [0, 2, 1]
-// TRANSPOSE-B:     %[[C:.*]] = linalg.batch_matmul_transpose_b ins(%[[A]], %[[B_TRANSP]] : tensor<2x16x8xf32>, tensor<2x16x8xf32>) outs(%[[C_ZERO]] : tensor<2x16x16xf32>) -> tensor<2x16x16xf32>
+// TRANSPOSE-B:     %[[C:.*]] = linalg.batch_matmul indexing_maps = [#[[$BMA]], #[[$BMB]], #[[$BMC]]] ins(%[[A]], %[[B_TRANSP]] : tensor<2x16x8xf32>, tensor<2x16x8xf32>) outs(%[[C_ZERO]] : tensor<2x16x16xf32>) -> tensor<2x16x16xf32>
 // CHECK:           return %[[C]] : tensor<2x16x16xf32>
 // CHECK:         }
 func.func @batch_matmul_static(%A: tensor<2x16x8xf32>, %B: tensor<2x8x16xf32>) -> (tensor<2x16x16xf32>) {
@@ -127,12 +141,12 @@ func.func @batch_matmul_static(%A: tensor<2x16x8xf32>, %B: tensor<2x8x16xf32>) -
 // TRANSPOSE-A:     %[[A_DIM2:.*]] = tensor.dim %[[A]], %[[C2]] : tensor<?x?x?xf32>
 // TRANSPOSE-A:     %[[A_TRANSP_INIT:.*]] = tensor.empty(%[[A_DIM0]], %[[A_DIM2]], %[[A_DIM1]]) : tensor<?x?x?xf32>
 // TRANSPOSE-A:     %[[A_TRANSP:.*]] = linalg.transpose ins(%[[A]] : tensor<?x?x?xf32>) outs(%[[A_TRANSP_INIT]] : tensor<?x?x?xf32>) permutation = [0, 2, 1]
-// TRANSPOSE-A:     %[[C:.*]] = linalg.batch_matmul_transpose_a ins(%[[A_TRANSP]], %[[B]] : tensor<?x?x?xf32>, tensor<?x?x?xf32>) outs(%[[C_ZERO]] : tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+// TRANSPOSE-A:     %[[C:.*]] = linalg.batch_matmul indexing_maps = [#[[$BMA]], #[[$BMB]], #[[$BMC]]] ins(%[[A_TRANSP]], %[[B]] : tensor<?x?x?xf32>, tensor<?x?x?xf32>) outs(%[[C_ZERO]] : tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
 // TRANSPOSE-B:     %[[B_DIM0:.*]] = tensor.dim %[[B]], %[[C0]] : tensor<?x?x?xf32>
 // TRANSPOSE-B:     %[[B_DIM1:.*]] = tensor.dim %[[B]], %[[C1]] : tensor<?x?x?xf32>
 // TRANSPOSE-B:     %[[B_TRANSP_INIT:.*]] = tensor.empty(%[[B_DIM0]], %[[B_DIM2]], %[[B_DIM1]]) : tensor<?x?x?xf32>
 // TRANSPOSE-B:     %[[B_TRANSP:.*]] = linalg.transpose ins(%[[B]] : tensor<?x?x?xf32>) outs(%[[B_TRANSP_INIT]] : tensor<?x?x?xf32>) permutation = [0, 2, 1]
-// TRANSPOSE-B:     %[[C:.*]] = linalg.batch_matmul_transpose_b ins(%[[A]], %[[B_TRANSP]] : tensor<?x?x?xf32>, tensor<?x?x?xf32>) outs(%[[C_ZERO]] : tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
+// TRANSPOSE-B:     %[[C:.*]] = linalg.batch_matmul indexing_maps = [#[[$BMA]], #[[$BMB]], #[[$BMC]]] ins(%[[A]], %[[B_TRANSP]] : tensor<?x?x?xf32>, tensor<?x?x?xf32>) outs(%[[C_ZERO]] : tensor<?x?x?xf32>) -> tensor<?x?x?xf32>
 // CHECK:           return %[[C]] : tensor<?x?x?xf32>
 // CHECK:         }
 func.func @batch_matmul_dynamic(%A: tensor<?x?x?xf32>, %B: tensor<?x?x?xf32>) -> (tensor<?x?x?xf32>) {
@@ -161,10 +175,10 @@ func.func @batch_matmul_dynamic(%A: tensor<?x?x?xf32>, %B: tensor<?x?x?xf32>) ->
 // CHECK:           %[[C_ZERO:.*]] = linalg.fill ins(%[[C0_F32]] : f32) outs(%[[C_INIT]] : tensor<2x?x16xf32>) -> tensor<2x?x16xf32>
 // TRANSPOSE-A:     %[[A_TRANSP_INIT:.*]] = tensor.empty(%[[A_DIM1]]) : tensor<2x8x?xf32>
 // TRANSPOSE-A:     %[[A_TRANSP:.*]] = linalg.transpose ins(%[[A]] : tensor<2x?x8xf32>) outs(%[[A_TRANSP_INIT]] : tensor<2x8x?xf32>) permutation = [0, 2, 1]
-// TRANSPOSE-A:     %[[B0:.*]] = linalg.batch_matmul_transpose_a ins(%[[A_TRANSP]], %[[B]] : tensor<2x8x?xf32>, tensor<2x8x16xf32>) outs(%[[C_ZERO]] : tensor<2x?x16xf32>) -> tensor<2x?x16xf32>
+// TRANSPOSE-A:     %[[B0:.*]] = linalg.batch_matmul indexing_maps = [#[[$BMA]], #[[$BMB]], #[[$BMC]]] ins(%[[A_TRANSP]], %[[B]] : tensor<2x8x?xf32>, tensor<2x8x16xf32>) outs(%[[C_ZERO]] : tensor<2x?x16xf32>) -> tensor<2x?x16xf32>
 // TRANSPOSE-B:     %[[B_TRANSP_INIT:.*]] = tensor.empty() : tensor<2x16x8xf32>
 // TRANSPOSE-B:     %[[B_TRANSP:.*]] = linalg.transpose ins(%[[B]] : tensor<2x8x16xf32>) outs(%[[B_TRANSP_INIT]] : tensor<2x16x8xf32>) permutation = [0, 2, 1]
-// TRANSPOSE-B:     %[[B0:.*]] = linalg.batch_matmul_transpose_b ins(%[[A]], %[[B_TRANSP]] : tensor<2x?x8xf32>, tensor<2x16x8xf32>) outs(%[[C_ZERO]] : tensor<2x?x16xf32>) -> tensor<2x?x16xf32>
+// TRANSPOSE-B:     %[[B0:.*]] = linalg.batch_matmul indexing_maps = [#[[$BMA]], #[[$BMB]], #[[$BMC]]] ins(%[[A]], %[[B_TRANSP]] : tensor<2x?x8xf32>, tensor<2x16x8xf32>) outs(%[[C_ZERO]] : tensor<2x?x16xf32>) -> tensor<2x?x16xf32>
 // CHECK:           return %[[B0]] : tensor<2x?x16xf32>
 // CHECK:         }
 func.func @batch_matmul_mixed(%A: tensor<2x?x8xf32>, %B: tensor<2x8x16xf32>) -> (tensor<2x?x16xf32>) {
diff --git a/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir b/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir
index 4eeae4c064519..25cbceb93c297 100644
--- a/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/linalg-ops-with-patterns.mlir
@@ -61,6 +61,83 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// CHECK-LABEL: @float_mixed_precision_matmul
+// CHECK-COUNT-3: vector.transfer_read
+// CHECK-NOT:     arith.extf
+// CHECK:         vector.contract {{.*}} : vector<1584x1584xbf16>, vector<1584x1584xbf16> into vector<1584x1584xf32>
+func.func @float_mixed_precision_matmul(%A: memref<1584x1584xbf16>, %B: memref<1584x1584xbf16>, %C: memref<1584x1584xf32>) {
+  linalg.matmul ins(%A, %B: memref<1584x1584xbf16>, memref<1584x1584xbf16>)
+            outs(%C: memref<1584x1584xf32>)
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1  { fold_type_extensions_into_contract } : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @vectorization_test_2
+func.func @vectorization_test_2(%A: memref<8x16xf32>, %B: memref<16x32xf32>,
+                         %C: memref<8x32xf32>) {
+  //       CHECK: arith.mulf %{{.*}}, %{{.*}} : vector<8x32x16xf32>
+  //       CHECK: vector.multi_reduction <add>, %{{.*}}, {{.*}} [2] : vector<8x32x16xf32> to vector<8x32xf32>
+  linalg.matmul
+    ins(%A, %B: memref<8x16xf32>, memref<16x32xf32>)
+   outs(%C: memref<8x32xf32>)
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1  { disable_multi_reduction_to_contract_patterns } : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @matmul_tensors
+//  CHECK-SAME: (%[[ARG0:.*]]: tensor<8x4xf32>, %[[ARG1:.*]]: tensor<4x12xf32>,
+//  CHECK-SAME:  %[[ARG2:.*]]: tensor<8x12xf32>) -> tensor<8x12xf32>
+func.func @matmul_tensors(
+  %arg0: tensor<8x4xf32>, %arg1: tensor<4x12xf32>, %arg2: tensor<8x12xf32>)
+    -> tensor<8x12xf32> {
+  //   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+  //   CHECK-DAG:   %[[V0:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]]], {{.*}} : tensor<8x4xf32>, vector<8x12x4xf32>
+  //   CHECK-DAG:   %[[V1:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], {{.*}} : tensor<4x12xf32>, vector<8x12x4xf32>
+  //   CHECK-DAG:   %[[V2:.*]] = vector.transfer_read %[[ARG2]][%[[C0]], %[[C0]]], {{.*}} : tensor<8x12xf32>, vector<8x12xf32>
+  //
+  // linalg matmul lowers gets expanded to a 3D reduction, canonicalization later
+  // convert it to a 2D contract.
+  //       CHECK:   %[[MUL:.*]] = arith.mulf %[[V0]], %[[V1]] : vector<8x12x4xf32>
+  //       CHECK:   %[[R:.*]] = vector.multi_reduction <add>, %[[MUL]], %[[V2]] [2] : vector<8x12x4xf32> to vector<8x12xf32>
+  //       CHECK:   %[[W:.*]] = vector.transfer_write %[[R]], %[[ARG2]][%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x12xf32>, tensor<8x12xf32>
+  %0 = linalg.matmul  ins(%arg0, %arg1: tensor<8x4xf32>, tensor<4x12xf32>)
+                     outs(%arg2: tensor<8x12xf32>)
+    -> tensor<8x12xf32>
+  //       CHECK:   return %[[W]] : tensor<8x12xf32>
+  return %0 : tensor<8x12xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1  { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
 // CHECK-LABEL: contraction_batch_matmul
 func.func @contraction_batch_matmul(%A: memref<1584x1584x1584xf32>, %B: memref<1584x1584x1584xf32>, %C: memref<1584x1584x1584xf32>) {
 // CHECK: arith.mulf %{{.*}}, %{{.*}} : vector<1584x1584x1584x1584xf32>
@@ -115,6 +192,265 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// CHECK-LABEL: @float_mixed_precision_matmul_as_contract
+// CHECK-COUNT-3: vector.transfer_read
+// CHECK-NOT:     arith.extf
+// CHECK:         vector.contract {{.*}} : vector<24x12xbf16>, vector<12x25xbf16> into vector<24x25xf32>
+// CHECK:         vector.transfer_write
+func.func @float_mixed_precision_matmul_as_contract(%A: tensor<24x12xbf16>,
+                              %B: tensor<12x25xbf16>,
+                              %C: tensor<24x25xf32>) -> tensor<24x25xf32> {
+  %0 = linalg.contract
+      indexing_maps = [affine_map<(m, n, k) -> (m, k)>,
+                       affine_map<(m, n, k) -> (k, n)>,
+                       affine_map<(m, n, k) -> (m, n)>]
+      ins(%A, %B : tensor<24x12xbf16>, tensor<12x25xbf16>)
+      outs(%C : tensor<24x25xf32>) -> tensor<24x25xf32>
+  func.return %0 : tensor<24x25xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.contract"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 { fold_type_extensions_into_contract } : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @test_vectorize_fill
+func.func @test_vectorize_fill(%A : memref<8x16xf32>, %arg0 : f32) {
+  //       CHECK: %[[V:.*]] = vector.broadcast {{.*}} : f32 to vector<8x16xf32>
+  //       CHECK: vector.transfer_write %[[V]], {{.*}} : vector<8x16xf32>, memref<8x16xf32>
+  linalg.fill ins(%arg0 : f32) outs(%A : memref<8x16xf32>)
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @test_vectorize_fill
+func.func @test_vectorize_fill_0d(%A : memref<f32>, %arg0 : f32) {
+  // CHECK-SAME: (%[[M:.*]]: memref<f32>, %[[val:.*]]: f32)
+  //      CHECK:   %[[VEC:.*]] = vector.broadcast %[[val]] : f32 to vector<f32>
+  //      CHECK:   vector.transfer_write %[[VEC]], %[[M]][] : vector<f32>, memref<f32>
+  linalg.fill ins(%arg0 : f32) outs(%A : memref<f32>)
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @test_vectorize_copy
+func.func @test_vectorize_copy(%A : memref<8x16xf32>, %B : memref<8x16xf32>) {
+  //       CHECK: %[[V:.*]] = vector.transfer_read {{.*}} : memref<8x16xf32>, vector<8x16xf32>
+  //       CHECK: vector.transfer_write %[[V]], {{.*}} : vector<8x16xf32>, memref<8x16xf32>
+  memref.copy %A, %B :  memref<8x16xf32> to memref<8x16xf32>
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["memref.copy"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @test_vectorize_copy_0d
+func.func @test_vectorize_copy_0d(%A : memref<f32>, %B : memref<f32>) {
+  //  CHECK-SAME: (%[[A:.*]]: memref<f32>, %[[B:.*]]: memref<f32>)
+  //       CHECK:   %[[V:.*]] = vector.transfer_read %[[A]][]{{.*}} : memref<f32>, vector<f32>
+  //       CHECK:   %[[val:.*]] = vector.extract %[[V]][] : f32 from vector<f32>
+  //       CHECK:   %[[VV:.*]] = vector.broadcast %[[val]] : f32 to vector<f32>
+  //       CHECK:   vector.transfer_write %[[VV]], %[[B]][] : vector<f32>, memref<f32>
+  memref.copy %A, %B :  memref<f32> to memref<f32>
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["memref.copy"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// CHECK-LABEL: func @test_vectorize_copy_complex
+// CHECK-NOT: vector<
+func.func @test_vectorize_copy_complex(%A : memref<8x16xcomplex<f32>>, %B : memref<8x16xcomplex<f32>>) {
+  memref.copy %A, %B :  memref<8x16xcomplex<f32>> to memref<8x16xcomplex<f32>>
+  return
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["memref.copy"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+// Input identical as the test in vectorization.mlir. Output is different -
+// vector sizes are inferred (rather than user-specified) and hence _no_
+// masking was used.
+
+func.func @test_vectorize_pack(%arg0: tensor<32x8x16xf32>, %arg1: tensor<4x1x32x16x2xf32>) -> tensor<4x1x32x16x2xf32> {
+  %pack = linalg.pack %arg0 outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x8x16xf32> -> tensor<4x1x32x16x2xf32>
+  return %pack : tensor<4x1x32x16x2xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// CHECK-LABEL:   func.func @test_vectorize_pack(
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<32x8x16xf32>,
+// CHECK-SAME:      %[[VAL_1:.*]]: tensor<4x1x32x16x2xf32>) -> tensor<4x1x32x16x2xf32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]]], %[[VAL_2]] {in_bounds = [true, true, true]} : tensor<32x8x16xf32>, vector<32x8x16xf32>
+// CHECK:           %[[VAL_5:.*]] = vector.shape_cast %[[VAL_4]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
+// CHECK:           %[[VAL_6:.*]] = vector.transpose %[[VAL_5]], [1, 3, 0, 4, 2] : vector<32x4x2x1x16xf32> to vector<4x1x32x16x2xf32>
+// CHECK:           %[[VAL_7:.*]] = tensor.empty() : tensor<4x1x32x16x2xf32>
+// CHECK:           %[[VAL_8:.*]] = vector.transfer_write %[[VAL_6]], %[[VAL_7]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]]] {in_bounds = [true, true, true, true, true]} : vector<4x1x32x16x2xf32>, tensor<4x1x32x16x2xf32>
+// CHECK:           return %[[VAL_8]] : tensor<4x1x32x16x2xf32>
+
+// -----
+
+func.func @test_vectorize_padded_pack(%arg0: tensor<32x7x15xf32>, %arg1: tensor<32x4x1x16x2xf32>) -> tensor<32x4x1x16x2xf32> {
+  %pad = arith.constant 0.000000e+00 : f32
+  %pack = linalg.pack %arg0 padding_value(%pad : f32) inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x7x15xf32> -> tensor<32x4x1x16x2xf32>
+  return %pack : tensor<32x4x1x16x2xf32>
+}
+
+// CHECK-LABEL:   func.func @test_vectorize_padded_pack(
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<32x7x15xf32>,
+// CHECK-SAME:      %[[VAL_1:.*]]: tensor<32x4x1x16x2xf32>) -> tensor<32x4x1x16x2xf32> {
+// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
+// CHECK:           %[[VAL_4:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]]], %[[VAL_2]] {in_bounds = [true, false, false]} : tensor<32x7x15xf32>, vector<32x8x16xf32>
+// CHECK:           %[[VAL_5:.*]] = vector.shape_cast %[[VAL_4]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
+// CHECK:           %[[VAL_6:.*]] = vector.transpose %[[VAL_5]], [0, 1, 3, 4, 2] : vector<32x4x2x1x16xf32> to vector<32x4x1x16x2xf32>
+// CHECK:           %[[VAL_7:.*]] = tensor.empty() : tensor<32x4x1x16x2xf32>
+// CHECK:           %[[VAL_8:.*]] = vector.transfer_write %[[VAL_6]], %[[VAL_7]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]]] {in_bounds = [true, true, true, true, true]} : vector<32x4x1x16x2xf32>, tensor<32x4x1x16x2xf32>
+// CHECK:           return %[[VAL_8]] : tensor<32x4x1x16x2xf32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @vectorize_map(%arg0: memref<64xf32>,
+    %arg1: memref<64xf32>, %arg2: memref<64xf32>) {
+  linalg.map ins(%arg0, %arg1 : memref<64xf32>, memref<64xf32>)
+             outs(%arg2 : memref<64xf32>)
+    (%in: f32, %in_0: f32) {
+      %0 = arith.addf %in, %in_0 : f32
+      linalg.yield %0 : f32
+    }
+  return
+}
+// CHECK-LABEL: func @vectorize_map
+// CHECK:         %[[LHS:.*]] = vector.transfer_read
+// CHECK-NEXT:    %[[RHS:.*]] = vector.transfer_read
+// CHECK-NEXT:    arith.addf %[[LHS]], %[[RHS]] : vector<64xf32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.map"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @vectorize_transpose(%arg0: memref<16x32x64xf32>,
+                               %arg1: memref<32x64x16xf32>) {
+  linalg.transpose ins(%arg0 : memref<16x32x64xf32>)
+                   outs(%arg1 : memref<32x64x16xf32>) permutation = [1, 2, 0]
+  return
+}
+// CHECK-LABEL: func @vectorize_transpose
+// CHECK:         vector.transpose
+// CHECK-SAME:      [1, 2, 0] : vector<16x32x64xf32> to vector<32x64x16xf32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.transpose"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @vectorize_reduce(%arg0: memref<16x32x64xf32>,
+                  %arg1: memref<16x64xf32>) {
+  linalg.reduce ins(%arg0 : memref<16x32x64xf32>)
+                outs(%arg1 : memref<16x64xf32>) dimensions = [1]
+    (%in: f32, %init: f32) {
+      %0 = arith.addf %in, %init : f32
+      linalg.yield %0 : f32
+    }
+  return
+}
+// CHECK-LABEL: func @vectorize_reduce
+// CHECK:         vector.multi_reduction <add>
+// CHECK-SAME:    : vector<16x32x64xf32> to vector<16x64xf32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.reduce"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
 #matmul_trait = {
   indexing_maps = [
     affine_map<(m, n, k) -> (m, k)>,
@@ -306,27 +642,6 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-// CHECK-LABEL: func @vectorization_test_2
-func.func @vectorization_test_2(%A: memref<8x16xf32>, %B: memref<16x32xf32>,
-                         %C: memref<8x32xf32>) {
-  //       CHECK: arith.mulf %{{.*}}, %{{.*}} : vector<8x32x16xf32>
-  //       CHECK: vector.multi_reduction <add>, %{{.*}}, {{.*}} [2] : vector<8x32x16xf32> to vector<8x32xf32>
-  linalg.matmul
-    ins(%A, %B: memref<8x16xf32>, memref<16x32xf32>)
-   outs(%C: memref<8x32xf32>)
-  return
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1  { disable_multi_reduction_to_contract_patterns } : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
 
 // CHECK-LABEL: func @test_vectorize_scalar_input
 func.func @test_vectorize_scalar_input(%A : memref<8x16xf32>, %arg0 : f32) {
@@ -345,138 +660,7 @@ func.func @test_vectorize_scalar_input(%A : memref<8x16xf32>, %arg0 : f32) {
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-// CHECK-LABEL: func @test_do_not_vectorize_unsupported_element_types
-func.func @test_do_not_vectorize_unsupported_element_types(%A : memref<8x16xcomplex<f32>>, %arg0 : complex<f32>) {
-  // CHECK-NOT: vector.broadcast
-  // CHECK-NOT: vector.transfer_write
-  linalg.generic {
-    indexing_maps = [affine_map<(m, n) -> ()>, affine_map<(m, n) -> (m, n)>],
-    iterator_types = ["parallel", "parallel"]}
-   ins(%arg0 : complex<f32>)
-  outs(%A: memref<8x16xcomplex<f32>>) {
-    ^bb(%0: complex<f32>, %1: complex<f32>) :
-      linalg.yield %0 : complex<f32>
-  }
-  return
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-#map0 = affine_map<(d0) -> (d0)>
-
-func.func @vectorize_affine_apply(%arg0: tensor<5xf32>, %arg3: index) -> tensor<5xi32> {
-  %0 = tensor.empty() : tensor<5xi32>
-  %1 = linalg.generic {indexing_maps = [#map0, #map0],
-                       iterator_types = ["parallel"]}
-    ins(%arg0 : tensor<5xf32>)
-    outs(%0 : tensor<5xi32>) {
-  ^bb0(%arg1: f32, %arg2: i32):
-    %2 = linalg.index 0 : index
-    %11 = affine.apply affine_map<() -> (123)>()
-    %12 = affine.apply affine_map<(d0, d1) -> (d0 + d1)>(%2, %11)
-    %13 = affine.apply affine_map<(d0)[s0] -> (d0 + s0)>(%12)[%arg3]
-    %14 = affine.apply affine_map<(d0) -> (d0 + 1)>(%13)
-    %15 = affine.apply affine_map<(d0, d1, d2) -> (d0 + d1 + d2)>(%13, %14, %12)
-    %3 = arith.index_cast %15 : index to i32
-    linalg.yield %3 : i32
-  } -> tensor<5xi32>
-  return %1 : tensor<5xi32>
-}
-
-// CHECK-LABEL:  func.func @vectorize_affine_apply
-// CHECK-SAME: %arg0: tensor<5xf32>
-// CHECK-SAME: %[[ARG1:.*]]: index
-// CHECK-DAG: %[[CST:.*]] = arith.constant dense<[123, 124, 125, 126, 127]> : vector<5xindex>
-// CHECK-DAG: %[[CST_0:.*]] = arith.constant dense<1> : vector<5xindex>
-// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
-// CHECK:   %[[EMPTY:.*]] = tensor.empty() : tensor<5xi32>
-// CHECK:   %[[BCAST:.*]] = vector.broadcast %[[ARG1]] : index to vector<5xindex>
-// CHECK:   %[[ADDI_1:.*]] = arith.addi %[[BCAST]], %[[CST]] : vector<5xindex>
-// CHECK:   %[[ADDI_2:.*]] = arith.addi %[[ADDI_1]], %[[CST_0]] : vector<5xindex>
-// CHECK:   %[[ADDI_3:.*]] = arith.addi %[[ADDI_1]], %[[ADDI_2]] : vector<5xindex>
-// CHECK:   %[[ADDI_4:.*]] = arith.addi %[[ADDI_3]], %[[CST]] : vector<5xindex>
-// CHECK:   %[[CAST:.*]] = arith.index_cast %[[ADDI_4]] : vector<5xindex> to vector<5xi32>
-// CHECK:   vector.transfer_write %[[CAST]], %[[EMPTY]][%[[C0:.*]]] {in_bounds = [true]} : vector<5xi32>, tensor<5xi32>
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-     %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-     %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-     %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op) -> !transform.any_op
-     transform.yield
-  }
-}
-
-// -----
-
-// CHECK-LABEL: func @test_vectorize_fill
-func.func @test_vectorize_fill(%A : memref<8x16xf32>, %arg0 : f32) {
-  //       CHECK: %[[V:.*]] = vector.broadcast {{.*}} : f32 to vector<8x16xf32>
-  //       CHECK: vector.transfer_write %[[V]], {{.*}} : vector<8x16xf32>, memref<8x16xf32>
-  linalg.fill ins(%arg0 : f32) outs(%A : memref<8x16xf32>)
-  return
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-// CHECK-LABEL: func @test_vectorize_fill
-func.func @test_vectorize_fill_0d(%A : memref<f32>, %arg0 : f32) {
-  // CHECK-SAME: (%[[M:.*]]: memref<f32>, %[[val:.*]]: f32)
-  //      CHECK:   %[[VEC:.*]] = vector.broadcast %[[val]] : f32 to vector<f32>
-  //      CHECK:   vector.transfer_write %[[VEC]], %[[M]][] : vector<f32>, memref<f32>
-  linalg.fill ins(%arg0 : f32) outs(%A : memref<f32>)
-  return
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.fill"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-// CHECK-LABEL: func @test_vectorize_copy
-func.func @test_vectorize_copy(%A : memref<8x16xf32>, %B : memref<8x16xf32>) {
-  //       CHECK: %[[V:.*]] = vector.transfer_read {{.*}} : memref<8x16xf32>, vector<8x16xf32>
-  //       CHECK: vector.transfer_write %[[V]], {{.*}} : vector<8x16xf32>, memref<8x16xf32>
-  memref.copy %A, %B :  memref<8x16xf32> to memref<8x16xf32>
-  return
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["memref.copy"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
     %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
     transform.yield
@@ -485,20 +669,24 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-// CHECK-LABEL: func @test_vectorize_copy_0d
-func.func @test_vectorize_copy_0d(%A : memref<f32>, %B : memref<f32>) {
-  //  CHECK-SAME: (%[[A:.*]]: memref<f32>, %[[B:.*]]: memref<f32>)
-  //       CHECK:   %[[V:.*]] = vector.transfer_read %[[A]][]{{.*}} : memref<f32>, vector<f32>
-  //       CHECK:   %[[val:.*]] = vector.extract %[[V]][] : f32 from vector<f32>
-  //       CHECK:   %[[VV:.*]] = vector.broadcast %[[val]] : f32 to vector<f32>
-  //       CHECK:   vector.transfer_write %[[VV]], %[[B]][] : vector<f32>, memref<f32>
-  memref.copy %A, %B :  memref<f32> to memref<f32>
+// CHECK-LABEL: func @test_do_not_vectorize_unsupported_element_types
+func.func @test_do_not_vectorize_unsupported_element_types(%A : memref<8x16xcomplex<f32>>, %arg0 : complex<f32>) {
+  // CHECK-NOT: vector.broadcast
+  // CHECK-NOT: vector.transfer_write
+  linalg.generic {
+    indexing_maps = [affine_map<(m, n) -> ()>, affine_map<(m, n) -> (m, n)>],
+    iterator_types = ["parallel", "parallel"]}
+   ins(%arg0 : complex<f32>)
+  outs(%A: memref<8x16xcomplex<f32>>) {
+    ^bb(%0: complex<f32>, %1: complex<f32>) :
+      linalg.yield %0 : complex<f32>
+  }
   return
 }
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["memref.copy"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
     %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
     transform.yield
@@ -507,19 +695,48 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-// CHECK-LABEL: func @test_vectorize_copy_complex
-// CHECK-NOT: vector<
-func.func @test_vectorize_copy_complex(%A : memref<8x16xcomplex<f32>>, %B : memref<8x16xcomplex<f32>>) {
-  memref.copy %A, %B :  memref<8x16xcomplex<f32>> to memref<8x16xcomplex<f32>>
-  return
+#map0 = affine_map<(d0) -> (d0)>
+
+func.func @vectorize_affine_apply(%arg0: tensor<5xf32>, %arg3: index) -> tensor<5xi32> {
+  %0 = tensor.empty() : tensor<5xi32>
+  %1 = linalg.generic {indexing_maps = [#map0, #map0],
+                       iterator_types = ["parallel"]}
+    ins(%arg0 : tensor<5xf32>)
+    outs(%0 : tensor<5xi32>) {
+  ^bb0(%arg1: f32, %arg2: i32):
+    %2 = linalg.index 0 : index
+    %11 = affine.apply affine_map<() -> (123)>()
+    %12 = affine.apply affine_map<(d0, d1) -> (d0 + d1)>(%2, %11)
+    %13 = affine.apply affine_map<(d0)[s0] -> (d0 + s0)>(%12)[%arg3]
+    %14 = affine.apply affine_map<(d0) -> (d0 + 1)>(%13)
+    %15 = affine.apply affine_map<(d0, d1, d2) -> (d0 + d1 + d2)>(%13, %14, %12)
+    %3 = arith.index_cast %15 : index to i32
+    linalg.yield %3 : i32
+  } -> tensor<5xi32>
+  return %1 : tensor<5xi32>
 }
 
+// CHECK-LABEL:  func.func @vectorize_affine_apply
+// CHECK-SAME: %arg0: tensor<5xf32>
+// CHECK-SAME: %[[ARG1:.*]]: index
+// CHECK-DAG: %[[CST:.*]] = arith.constant dense<[123, 124, 125, 126, 127]> : vector<5xindex>
+// CHECK-DAG: %[[CST_0:.*]] = arith.constant dense<1> : vector<5xindex>
+// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+// CHECK:   %[[EMPTY:.*]] = tensor.empty() : tensor<5xi32>
+// CHECK:   %[[BCAST:.*]] = vector.broadcast %[[ARG1]] : index to vector<5xindex>
+// CHECK:   %[[ADDI_1:.*]] = arith.addi %[[BCAST]], %[[CST]] : vector<5xindex>
+// CHECK:   %[[ADDI_2:.*]] = arith.addi %[[ADDI_1]], %[[CST_0]] : vector<5xindex>
+// CHECK:   %[[ADDI_3:.*]] = arith.addi %[[ADDI_1]], %[[ADDI_2]] : vector<5xindex>
+// CHECK:   %[[ADDI_4:.*]] = arith.addi %[[ADDI_3]], %[[CST]] : vector<5xindex>
+// CHECK:   %[[CAST:.*]] = arith.index_cast %[[ADDI_4]] : vector<5xindex> to vector<5xi32>
+// CHECK:   vector.transfer_write %[[CAST]], %[[EMPTY]][%[[C0:.*]]] {in_bounds = [true]} : vector<5xi32>, tensor<5xi32>
+
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["memref.copy"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
-    transform.yield
+     %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+     %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
+     %2 = transform.structured.vectorize_children_and_apply_patterns %1 { vectorize_nd_extract } : (!transform.any_op) -> !transform.any_op
+     transform.yield
   }
 }
 
@@ -855,40 +1072,6 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-// CHECK-LABEL: func @matmul_tensors
-//  CHECK-SAME: (%[[ARG0:.*]]: tensor<8x4xf32>, %[[ARG1:.*]]: tensor<4x12xf32>,
-//  CHECK-SAME:  %[[ARG2:.*]]: tensor<8x12xf32>) -> tensor<8x12xf32>
-func.func @matmul_tensors(
-  %arg0: tensor<8x4xf32>, %arg1: tensor<4x12xf32>, %arg2: tensor<8x12xf32>)
-    -> tensor<8x12xf32> {
-  //   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
-  //   CHECK-DAG:   %[[V0:.*]] = vector.transfer_read %[[ARG0]][%[[C0]], %[[C0]]], {{.*}} : tensor<8x4xf32>, vector<8x12x4xf32>
-  //   CHECK-DAG:   %[[V1:.*]] = vector.transfer_read %[[ARG1]][%[[C0]], %[[C0]]], {{.*}} : tensor<4x12xf32>, vector<8x12x4xf32>
-  //   CHECK-DAG:   %[[V2:.*]] = vector.transfer_read %[[ARG2]][%[[C0]], %[[C0]]], {{.*}} : tensor<8x12xf32>, vector<8x12xf32>
-  //
-  // linalg matmul lowers gets expanded to a 3D reduction, canonicalization later
-  // convert it to a 2D contract.
-  //       CHECK:   %[[MUL:.*]] = arith.mulf %[[V0]], %[[V1]] : vector<8x12x4xf32>
-  //       CHECK:   %[[R:.*]] = vector.multi_reduction <add>, %[[MUL]], %[[V2]] [2] : vector<8x12x4xf32> to vector<8x12xf32>
-  //       CHECK:   %[[W:.*]] = vector.transfer_write %[[R]], %[[ARG2]][%[[C0]], %[[C0]]] {in_bounds = [true, true]} : vector<8x12xf32>, tensor<8x12xf32>
-  %0 = linalg.matmul  ins(%arg0, %arg1: tensor<8x4xf32>, tensor<4x12xf32>)
-                     outs(%arg2: tensor<8x12xf32>)
-    -> tensor<8x12xf32>
-  //       CHECK:   return %[[W]] : tensor<8x12xf32>
-  return %0 : tensor<8x12xf32>
-}
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1  { disable_multi_reduction_to_contract_patterns, disable_transfer_permutation_map_lowering_patterns } : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
 // CHECK-LABEL: func @sum_exp
 func.func @sum_exp(%input: tensor<4x16x8xf32>, %output: tensor<4x16xf32>)
   -> tensor<4x16xf32>
@@ -914,7 +1097,6 @@ func.func @sum_exp(%input: tensor<4x16x8xf32>, %output: tensor<4x16xf32>)
   return %0 : tensor<4x16xf32>
 }
 
-
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
@@ -993,7 +1175,6 @@ func.func @red_maximumf_2d(%arg0: tensor<4x4xf32>) -> tensor<4xf32> {
   return %red : tensor<4xf32>
 }
 
-
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %3 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
@@ -1428,78 +1609,6 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-func.func @vectorize_map(%arg0: memref<64xf32>,
-    %arg1: memref<64xf32>, %arg2: memref<64xf32>) {
-  linalg.map ins(%arg0, %arg1 : memref<64xf32>, memref<64xf32>)
-             outs(%arg2 : memref<64xf32>)
-    (%in: f32, %in_0: f32) {
-      %0 = arith.addf %in, %in_0 : f32
-      linalg.yield %0 : f32
-    }
-  return
-}
-// CHECK-LABEL: func @vectorize_map
-// CHECK:         %[[LHS:.*]] = vector.transfer_read
-// CHECK-NEXT:    %[[RHS:.*]] = vector.transfer_read
-// CHECK-NEXT:    arith.addf %[[LHS]], %[[RHS]] : vector<64xf32>
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.map"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-func.func @vectorize_transpose(%arg0: memref<16x32x64xf32>,
-                               %arg1: memref<32x64x16xf32>) {
-  linalg.transpose ins(%arg0 : memref<16x32x64xf32>)
-                   outs(%arg1 : memref<32x64x16xf32>) permutation = [1, 2, 0]
-  return
-}
-// CHECK-LABEL: func @vectorize_transpose
-// CHECK:         vector.transpose
-// CHECK-SAME:      [1, 2, 0] : vector<16x32x64xf32> to vector<32x64x16xf32>
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.transpose"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
-func.func @vectorize_reduce(%arg0: memref<16x32x64xf32>,
-                  %arg1: memref<16x64xf32>) {
-  linalg.reduce ins(%arg0 : memref<16x32x64xf32>)
-                outs(%arg1 : memref<16x64xf32>) dimensions = [1]
-    (%in: f32, %init: f32) {
-      %0 = arith.addf %in, %init : f32
-      linalg.yield %0 : f32
-    }
-  return
-}
-// CHECK-LABEL: func @vectorize_reduce
-// CHECK:         vector.multi_reduction <add>
-// CHECK-SAME:    : vector<16x32x64xf32> to vector<16x64xf32>
-
-module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.reduce"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
-    transform.yield
-  }
-}
-
-// -----
-
 // This is a regression test. This IR cannot be vectorized, but
 // structured.vectorize_children_and_apply_patterns should nevertheless succeed.
 
@@ -1715,65 +1824,77 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
-// Input identical as the test in vectorization.mlir. Output is different -
-// vector sizes are inferred (rather than user-specified) and hence _no_
-// masking was used.
-
-func.func @test_vectorize_pack(%arg0: tensor<32x8x16xf32>, %arg1: tensor<4x1x32x16x2xf32>) -> tensor<4x1x32x16x2xf32> {
-  %pack = linalg.pack %arg0 outer_dims_perm = [1, 2, 0] inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x8x16xf32> -> tensor<4x1x32x16x2xf32>
-  return %pack : tensor<4x1x32x16x2xf32>
+// CHECK-LABEL: func @float_mixed_precision_matmul_as_generic
+// CHECK-COUNT-3: vector.transfer_read
+// CHECK-NOT:     arith.extf
+// CHECK:         vector.contract {{.*}} : vector<8x16xbf16>, vector<16x32xbf16> into vector<8x32xf32>
+// CHECK:         vector.transfer_write
+func.func @float_mixed_precision_matmul_as_generic(%A: memref<8x16xbf16>, %B: memref<16x32xbf16>,
+                         %C: memref<8x32xf32>) {
+  linalg.generic {
+    indexing_maps = [
+    affine_map<(m, n, k) -> (m, k)>,
+    affine_map<(m, n, k) -> (k, n)>,
+    affine_map<(m, n, k) -> (m, n)>
+    ],
+    iterator_types = ["parallel", "parallel", "reduction"]
+  }
+   ins(%A, %B : memref<8x16xbf16>, memref<16x32xbf16>)
+   outs(%C : memref<8x32xf32>) {
+    ^bb(%in: bf16, %in_0: bf16, %c: f32) :
+      %a = arith.extf %in : bf16 to f32
+      %b = arith.extf %in_0 : bf16 to f32
+      %d = arith.mulf %a, %b: f32
+      %e = arith.addf %c, %d: f32
+      linalg.yield %e : f32
+  }
+  return
 }
 
 module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1  { fold_type_extensions_into_contract } : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
 }
 
-// CHECK-LABEL:   func.func @test_vectorize_pack(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<32x8x16xf32>,
-// CHECK-SAME:      %[[VAL_1:.*]]: tensor<4x1x32x16x2xf32>) -> tensor<4x1x32x16x2xf32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_4:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]]], %[[VAL_2]] {in_bounds = [true, true, true]} : tensor<32x8x16xf32>, vector<32x8x16xf32>
-// CHECK:           %[[VAL_5:.*]] = vector.shape_cast %[[VAL_4]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
-// CHECK:           %[[VAL_6:.*]] = vector.transpose %[[VAL_5]], [1, 3, 0, 4, 2] : vector<32x4x2x1x16xf32> to vector<4x1x32x16x2xf32>
-// CHECK:           %[[VAL_7:.*]] = tensor.empty() : tensor<4x1x32x16x2xf32>
-// CHECK:           %[[VAL_8:.*]] = vector.transfer_write %[[VAL_6]], %[[VAL_7]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]]] {in_bounds = [true, true, true, true, true]} : vector<4x1x32x16x2xf32>, tensor<4x1x32x16x2xf32>
-// CHECK:           return %[[VAL_8]] : tensor<4x1x32x16x2xf32>
-
 // -----
 
-// Input identical as the test in vectorization.mlir. Output is different -
-// vector sizes are inferred (rather than user-specified) and hence _no_
-// masking was used.
-
-func.func @test_vectorize_padded_pack(%arg0: tensor<32x7x15xf32>, %arg1: tensor<32x4x1x16x2xf32>) -> tensor<32x4x1x16x2xf32> {
-  %pad = arith.constant 0.000000e+00 : f32
-  %pack = linalg.pack %arg0 padding_value(%pad : f32) inner_dims_pos = [2, 1] inner_tiles = [16, 2] into %arg1 : tensor<32x7x15xf32> -> tensor<32x4x1x16x2xf32>
-  return %pack : tensor<32x4x1x16x2xf32>
+// CHECK-LABEL: func @integer_mixed_precision_matmul_as_generic
+// CHECK-COUNT-3: vector.transfer_read
+// CHECK-NOT:     arith.extsi
+// CHECK:         vector.contract {{.*}} : vector<8x16xi8>, vector<16x32xi8> into vector<8x32xi32>
+// CHECK:         vector.transfer_write
+func.func @integer_mixed_precision_matmul_as_generic(%A: memref<8x16xi8>, %B: memref<16x32xi8>,
+                         %C: memref<8x32xi32>) {
+  linalg.generic {
+    indexing_maps = [
+    affine_map<(m, n, k) -> (m, k)>,
+    affine_map<(m, n, k) -> (k, n)>,
+    affine_map<(m, n, k) -> (m, n)>
+    ],
+    iterator_types = ["parallel", "parallel", "reduction"]
+  }
+   ins(%A, %B : memref<8x16xi8>, memref<16x32xi8>)
+   outs(%C : memref<8x32xi32>) {
+    ^bb(%in: i8, %in_0: i8, %c: i32) :
+      %a = arith.extsi %in : i8 to i32
+      %b = arith.extsi %in_0 : i8 to i32
+      %d = arith.muli %a, %b: i32
+      %e = arith.addi %c, %d: i32
+      linalg.yield %e : i32
+  }
+  return
 }
 
-// CHECK-LABEL:   func.func @test_vectorize_padded_pack(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<32x7x15xf32>,
-// CHECK-SAME:      %[[VAL_1:.*]]: tensor<32x4x1x16x2xf32>) -> tensor<32x4x1x16x2xf32> {
-// CHECK:           %[[VAL_2:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK:           %[[VAL_3:.*]] = arith.constant 0 : index
-// CHECK:           %[[VAL_4:.*]] = vector.transfer_read %[[VAL_0]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]]], %[[VAL_2]] {in_bounds = [true, false, false]} : tensor<32x7x15xf32>, vector<32x8x16xf32>
-// CHECK:           %[[VAL_5:.*]] = vector.shape_cast %[[VAL_4]] : vector<32x8x16xf32> to vector<32x4x2x1x16xf32>
-// CHECK:           %[[VAL_6:.*]] = vector.transpose %[[VAL_5]], [0, 1, 3, 4, 2] : vector<32x4x2x1x16xf32> to vector<32x4x1x16x2xf32>
-// CHECK:           %[[VAL_7:.*]] = tensor.empty() : tensor<32x4x1x16x2xf32>
-// CHECK:           %[[VAL_8:.*]] = vector.transfer_write %[[VAL_6]], %[[VAL_7]]{{\[}}%[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]], %[[VAL_3]]] {in_bounds = [true, true, true, true, true]} : vector<32x4x1x16x2xf32>, tensor<32x4x1x16x2xf32>
-// CHECK:           return %[[VAL_8]] : tensor<32x4x1x16x2xf32>
-
 module attributes {transform.with_named_sequence} {
-  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
-    %0 = transform.structured.match ops{["linalg.pack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.generic"]} in %arg1 : (!transform.any_op) -> !transform.any_op
     %1 = transform.get_parent_op %0 {isolated_from_above} : (!transform.any_op) -> !transform.any_op
-    %2 = transform.structured.vectorize_children_and_apply_patterns %1 : (!transform.any_op) -> !transform.any_op
+    %2 = transform.structured.vectorize_children_and_apply_patterns %1  { fold_type_extensions_into_contract } : (!transform.any_op) -> !transform.any_op
     transform.yield
   }
 }
+
diff --git a/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir b/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
index d41d86117793b..01eb210a8ff5f 100644
--- a/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
+++ b/mlir/test/Dialect/Linalg/vectorization/linalg-ops.mlir
@@ -880,22 +880,22 @@ func.func @mmt4d_scalable(%A: memref<16x16x8x1xf32>, %B: memref<16x16x?x1xf32>,
 // CHECK-SAME:      %[[A:.*]]: memref<16x16x8x1xf32>,
 // CHECK-SAME:      %[[B:.*]]: memref<16x16x?x1xf32>,
 // CHECK-SAME:      %[[C_IN:.*]]: memref<16x16x8x?xf32>) {
-// CHECK:           %[[VAL_0:.*]] = arith.constant 16 : index
-// CHECK:           %[[VAL_1:.*]] = arith.constant 16 : index
-// CHECK:           %[[VAL_2:.*]] = arith.constant 16 : index
+// CHECK:           %[[C16_M:.*]] = arith.constant 16 : index
+// CHECK:           %[[C16_N:.*]] = arith.constant 16 : index
+// CHECK:           %[[C16_K:.*]] = arith.constant 16 : index
 // CHECK:           %[[C8:.*]] = arith.constant 8 : index
 // CHECK:           %[[C2:.*]] = arith.constant 2 : index
 // CHECK:           %[[DIM_2:.*]] = memref.dim %[[B]], %[[C2]] : memref<16x16x?x1xf32>
-// CHECK:           %[[VAL_6:.*]] = arith.constant 1 : index
+// CHECK:           %[[C1:.*]] = arith.constant 1 : index
 // CHECK:           %[[VEC_A:.*]] = vector.transfer_read %[[A]]{{.*}} : memref<16x16x8x1xf32>, vector<16x16x16x8x[4]x1xf32>
-// CHECK:           %[[MASK_1:.*]] = vector.create_mask %[[VAL_1]], %[[VAL_2]], %[[DIM_2]], %[[VAL_6]] : vector<16x16x[4]x1xi1>
+// CHECK:           %[[MASK_1:.*]] = vector.create_mask %[[C16_N]], %[[C16_K]], %[[DIM_2]], %[[C1]] : vector<16x16x[4]x1xi1>
 // CHECK:           %[[VEC_B:.*]] = vector.mask %[[MASK_1]] { vector.transfer_read %[[B]]{{.*}} : memref<16x16x?x1xf32>, vector<16x16x16x8x[4]x1xf32> } : vector<16x16x[4]x1xi1> -> vector<16x16x16x8x[4]x1xf32>
-// CHECK:           %[[MASK_2:.*]] = vector.create_mask %[[VAL_0]], %[[VAL_1]], %[[C8]], %[[DIM_2]] : vector<16x16x8x[4]xi1>
-// CHECK:           %[[VAL_15:.*]] = vector.mask %[[MASK_2]] { vector.transfer_read %[[C_IN]]{{.*}} : memref<16x16x8x?xf32>, vector<16x16x8x[4]xf32> } : vector<16x16x8x[4]xi1> -> vector<16x16x8x[4]xf32>
-// CHECK:           %[[VAL_16:.*]] = arith.mulf %[[VEC_A]], %[[VEC_B]] : vector<16x16x16x8x[4]x1xf32>
-// CHECK:           %[[MASK_3:.*]] = vector.create_mask %[[VAL_0]], %[[VAL_1]], %[[VAL_2]], %[[C8]], %[[DIM_2]], %[[VAL_6]] : vector<16x16x16x8x[4]x1xi1>
-// CHECK:           %[[VAL_18:.*]] = vector.mask %[[MASK_3]] { vector.multi_reduction <add>, %[[VAL_16]], %[[VAL_15]] [2, 5] : vector<16x16x16x8x[4]x1xf32> to vector<16x16x8x[4]xf32> } : vector<16x16x16x8x[4]x1xi1> -> vector<16x16x8x[4]xf32>
-// CHECK:           vector.mask %[[MASK_2]] { vector.transfer_write %[[VAL_18]], %[[C_IN]]{{.*}} : vector<16x16x8x[4]xf32>, memref<16x16x8x?xf32> } : vector<16x16x8x[4]xi1>
+// CHECK:           %[[MASK_2:.*]] = vector.create_mask %[[C16_M]], %[[C16_N]], %[[C8]], %[[DIM_2]] : vector<16x16x8x[4]xi1>
+// CHECK:           %[[VEC_C:.*]] = vector.mask %[[MASK_2]] { vector.transfer_read %[[C_IN]]{{.*}} : memref<16x16x8x?xf32>, vector<16x16x8x[4]xf32> } : vector<16x16x8x[4]xi1> -> vector<16x16x8x[4]xf32>
+// CHECK:           %[[MUL:.*]] = arith.mulf %[[VEC_A]], %[[VEC_B]] : vector<16x16x16x8x[4]x1xf32>
+// CHECK:           %[[MASK_3:.*]] = vector.create_mask %[[C16_M]], %[[C16_N]], %[[C16_K]], %[[C8]], %[[DIM_2]], %[[C1]] : vector<16x16x16x8x[4]x1xi1>
+// CHECK:           %[[RED:.*]] = vector.mask %[[MASK_3]] { vector.multi_reduction <add>, %[[MUL]], %[[VEC_C]] [2, 5] : vector<16x16x16x8x[4]x1xf32> to vector<16x16x8x[4]xf32> } : vector<16x16x16x8x[4]x1xi1> -> vector<16x16x8x[4]xf32>
+// CHECK:           vector.mask %[[MASK_2]] { vector.transfer_write %[[RED]], %[[C_IN]]{{.*}} : vector<16x16x8x[4]xf32>, memref<16x16x8x?xf32> } : vector<16x16x8x[4]xi1>
 
 
 module attributes {transform.with_named_sequence} {
@@ -920,10 +920,10 @@ func.func @mmt4d_scalable_with_assume(%A: memref<16x16x8x1xf32>, %B: memref<16x1
 // CHECK-NOT:       mask
 // CHECK:           %[[VEC_A:.*]] = vector.transfer_read %[[A]]{{.*}} : memref<16x16x8x1xf32>, vector<16x16x16x8x[4]x1xf32>
 // CHECK:           %[[VEC_B:.*]] = vector.transfer_read %[[B]]{{.*}} : memref<16x16x?x1xf32>, vector<16x16x16x8x[4]x1xf32>
-// CHECK:           %[[VAL_13:.*]] = vector.transfer_read %[[C_IN]]{{.*}} : memref<16x16x8x?xf32>, vector<16x16x8x[4]xf32>
-// CHECK:           %[[VAL_14:.*]] = arith.mulf %[[VEC_A]], %[[VEC_B]] : vector<16x16x16x8x[4]x1xf32>
-// CHECK:           %[[VAL_15:.*]] = vector.multi_reduction <add>, %[[VAL_14]], %[[VAL_13]] [2, 5] : vector<16x16x16x8x[4]x1xf32> to vector<16x16x8x[4]xf32>
-// CHECK:           vector.transfer_write %[[VAL_15]], %[[C_IN]]{{.*}} : vector<16x16x8x[4]xf32>, memref<16x16x8x?xf32>
+// CHECK:           %[[VEC_C:.*]] = vector.transfer_read %[[C_IN]]{{.*}} : memref<16x16x8x?xf32>, vector<16x16x8x[4]xf32>
+// CHECK:           %[[MUL:.*]] = arith.mulf %[[VEC_A]], %[[VEC_B]] : vector<16x16x16x8x[4]x1xf32>
+// CHECK:           %[[RED:.*]] = vector.multi_reduction <add>, %[[MUL]], %[[VEC_C]] [2, 5] : vector<16x16x16x8x[4]x1xf32> to vector<16x16x8x[4]xf32>
+// CHECK:           vector.transfer_write %[[RED]], %[[C_IN]]{{.*}} : vector<16x16x8x[4]xf32>, memref<16x16x8x?xf32>
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
@@ -933,6 +933,100 @@ module attributes {transform.with_named_sequence} {
   }
 }
 
+// -----
+
+///----------------------------------------------------------------------------------------
+/// Tests for linalg.batch_mmt4d
+///----------------------------------------------------------------------------------------
+
+func.func @batch_mmt4d(%A: memref<2x16x16x8x1xf32>, %B: memref<2x16x16x8x1xf32>, %C_in: memref<2x16x16x8x8xf32>) {
+  linalg.batch_mmt4d ins(%A, %B: memref<2x16x16x8x1xf32>, memref<2x16x16x8x1xf32>)
+               outs(%C_in: memref<2x16x16x8x8xf32>)
+  return
+}
+
+// CHECK-LABEL:   func.func @batch_mmt4d(
+// CHECK-SAME:      %[[A:.*]]: memref<2x16x16x8x1xf32>, %[[B:.*]]: memref<2x16x16x8x1xf32>, %[[C:.*]]: memref<2x16x16x8x8xf32>) {
+// CHECK:           %[[VEC_A:.*]] = vector.transfer_read %[[A]]{{.*}} : memref<2x16x16x8x1xf32>, vector<2x16x16x16x8x8x1xf32>
+// CHECK:           %[[VEC_B:.*]] = vector.transfer_read %[[B]]{{.*}} : memref<2x16x16x8x1xf32>, vector<2x16x16x16x8x8x1xf32>
+// CHECK:           %[[VEC_C:.*]] = vector.transfer_read %[[C]]{{.*}} : memref<2x16x16x8x8xf32>, vector<2x16x16x8x8xf32>
+// CHECK:           %[[MUL:.*]] = arith.mulf %[[VEC_A]], %[[VEC_B]] : vector<2x16x16x16x8x8x1xf32>
+// CHECK:           %[[RED:.*]] = vector.multi_reduction <add>, %[[MUL]], %[[VEC_C]] [3, 6] : vector<2x16x16x16x8x8x1xf32> to vector<2x16x16x8x8xf32>
+// CHECK:           vector.transfer_write %[[RED]], %[[C]]{{.*}} : vector<2x16x16x8x8xf32>, memref<2x16x16x8x8xf32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %batch_mmt4d = transform.structured.match ops{["linalg.batch_mmt4d"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %batch_mmt4d : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @batch_mmt4d_scalable(%A: memref<2x16x16x8x1xf32>, %B: memref<2x16x16x?x1xf32>, %C_in: memref<2x16x16x8x?xf32>) {
+  linalg.batch_mmt4d ins(%A, %B: memref<2x16x16x8x1xf32>, memref<2x16x16x?x1xf32>)
+               outs(%C_in: memref<2x16x16x8x?xf32>)
+  return
+}
+// CHECK-LABEL:   func.func @batch_mmt4d_scalable(
+// CHECK-SAME:      %[[A:.*]]: memref<2x16x16x8x1xf32>,
+// CHECK-SAME:      %[[B:.*]]: memref<2x16x16x?x1xf32>,
+// CHECK-SAME:      %[[C_IN:.*]]: memref<2x16x16x8x?xf32>) {
+// CHECK:           %[[C2:.*]] = arith.constant 2 : index
+// CHECK:           %[[C16_M:.*]] = arith.constant 16 : index
+// CHECK:           %[[C16_N:.*]] = arith.constant 16 : index
+// CHECK:           %[[C16_K:.*]] = arith.constant 16 : index
+// CHECK:           %[[C8:.*]] = arith.constant 8 : index
+// CHECK:           %[[C3:.*]] = arith.constant 3 : index
+// CHECK:           %[[DIM_N_IN:.*]] = memref.dim %[[B]], %[[C3]] : memref<2x16x16x?x1xf32>
+// CHECK:           %[[C1:.*]] = arith.constant 1 : index
+// CHECK:           %[[VEC_A:.*]] = vector.transfer_read %[[A]]{{.*}} : memref<2x16x16x8x1xf32>, vector<2x16x16x16x8x[4]x1xf32>
+// CHECK:           %[[MASK_1:.*]] = vector.create_mask %[[C2]], %[[C16_N]], %[[C16_K]], %[[DIM_N_IN]], %[[C1]] : vector<2x16x16x[4]x1xi1>
+// CHECK:           %[[VEC_B:.*]] = vector.mask %[[MASK_1]] { vector.transfer_read %[[B]]{{.*}} : memref<2x16x16x?x1xf32>, vector<2x16x16x16x8x[4]x1xf32> } : vector<2x16x16x[4]x1xi1> -> vector<2x16x16x16x8x[4]x1xf32>
+// CHECK:           %[[MASK_2:.*]] = vector.create_mask %[[C2]], %[[C16_M]], %[[C16_N]], %[[C8]], %[[DIM_N_IN]] : vector<2x16x16x8x[4]xi1>
+// CHECK:           %[[VEC_C:.*]] = vector.mask %[[MASK_2]] { vector.transfer_read %[[C_IN]]{{.*}} : memref<2x16x16x8x?xf32>, vector<2x16x16x8x[4]xf32> } : vector<2x16x16x8x[4]xi1> -> vector<2x16x16x8x[4]xf32>
+// CHECK:           %[[MUL:.*]] = arith.mulf %[[VEC_A]], %[[VEC_B]] : vector<2x16x16x16x8x[4]x1xf32>
+// CHECK:           %[[MASK_3:.*]] = vector.create_mask %[[C2]], %[[C16_M]], %[[C16_N]], %[[C16_K]], %[[C8]], %[[DIM_N_IN]], %[[C1]] : vector<2x16x16x16x8x[4]x1xi1>
+// CHECK:           %[[RED:.*]] = vector.mask %[[MASK_3]] { vector.multi_reduction <add>, %[[MUL]], %[[VEC_C]] [3, 6] : vector<2x16x16x16x8x[4]x1xf32> to vector<2x16x16x8x[4]xf32> } : vector<2x16x16x16x8x[4]x1xi1> -> vector<2x16x16x8x[4]xf32>
+// CHECK:           vector.mask %[[MASK_2]] { vector.transfer_write %[[RED]], %[[C_IN]]{{.*}} : vector<2x16x16x8x[4]xf32>, memref<2x16x16x8x?xf32> } : vector<2x16x16x8x[4]xi1>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %batch_mmt4d = transform.structured.match ops{["linalg.batch_mmt4d"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %batch_mmt4d vector_sizes [2, 16, 16, 16, 8, [4], 1] : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
+func.func @batch_mmt4d_scalable_with_assume(%A: memref<2x16x16x8x1xf32>, %B: memref<2x16x16x?x1xf32>, %C_in: memref<2x16x16x8x?xf32>) {
+  linalg.batch_mmt4d ins(%A, %B: memref<2x16x16x8x1xf32>, memref<2x16x16x?x1xf32>)
+               outs(%C_in: memref<2x16x16x8x?xf32>)
+  return
+}
+// CHECK-LABEL:   func.func @batch_mmt4d_scalable_with_assume(
+// CHECK-SAME:      %[[A:.*]]: memref<2x16x16x8x1xf32>,
+// CHECK-SAME:      %[[B:.*]]: memref<2x16x16x?x1xf32>,
+// CHECK-SAME:      %[[C_IN:.*]]: memref<2x16x16x8x?xf32>) {
+// CHECK-NOT:       mask
+// CHECK:           %[[VEC_A:.*]] = vector.transfer_read %[[A]]{{.*}} : memref<2x16x16x8x1xf32>, vector<2x16x16x16x8x[4]x1xf32>
+// CHECK:           %[[VEC_B:.*]] = vector.transfer_read %[[B]]{{.*}} : memref<2x16x16x?x1xf32>, vector<2x16x16x16x8x[4]x1xf32>
+// CHECK:           %[[VEC_C:.*]] = vector.transfer_read %[[C_IN]]{{.*}} : memref<2x16x16x8x?xf32>, vector<2x16x16x8x[4]xf32>
+// CHECK:           %[[MUL:.*]] = arith.mulf %[[VEC_A]], %[[VEC_B]] : vector<2x16x16x16x8x[4]x1xf32>
+// CHECK:           %[[RED:.*]] = vector.multi_reduction <add>, %[[MUL]], %[[VEC_C]] [3, 6] : vector<2x16x16x16x8x[4]x1xf32> to vector<2x16x16x8x[4]xf32>
+// CHECK:           vector.transfer_write %[[RED]], %[[C_IN]]{{.*}} : vector<2x16x16x8x[4]xf32>, memref<2x16x16x8x?xf32>
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %batch_mmt4d = transform.structured.match ops{["linalg.batch_mmt4d"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.structured.vectorize %batch_mmt4d vector_sizes [2, 16, 16, 16, 8, [4], 1] {assume_dynamic_dims_match_vec_sizes} : !transform.any_op
+    transform.yield
+  }
+}
+
+
 // -----
 
 ///----------------------------------------------------------------------------------------
@@ -940,31 +1034,100 @@ module attributes {transform.with_named_sequence} {
 ///----------------------------------------------------------------------------------------
 
 // CHECK-LABEL: func @test_vectorize_dynamic_shapes_unpack
-// CHECK-SAME:      %[[ARG_0:.*]]: tensor<?x?xf32>,
-// CHECK-SAME:      %[[ARG_1:.*]]: tensor<?x?x16x2xf32>
-func.func @test_vectorize_dynamic_shapes_unpack(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?x16x2xf32>) -> tensor<?x?xf32> {
-// CHECK: %[[C0:.*]] = arith.constant 0
-// CHECK: %[[C01:.*]] = arith.constant 0
-// CHECK: %[[C02:.*]] = arith.constant 0
-// CHECK: %[[DIM_0:.*]] = tensor.dim %[[ARG_1]], %[[C02]] : tensor<?x?x16x2xf32>
-// CHECK: %[[C1:.*]] = arith.constant 1
-// CHECK: %[[DIM6:.*]] = tensor.dim %[[ARG_1]], %[[C1]] : tensor<?x?x16x2xf32>
-// CHECK: %[[CNST16:.*]] = arith.constant 16 : index
-// CHECK: %[[CNST2:.*]] = arith.constant 2 : index
-// CHECK: %[[readMsk0:.*]] = vector.create_mask %[[DIM_0]], %[[DIM6]], %[[CNST16]], %[[CNST2]] : vector<2x1x16x2xi1>
-// CHECK: %[[read0:.*]] = vector.mask %[[readMsk0]] {{.*}} vector.transfer_read %{{.*}} : tensor<?x?x16x2xf32>, vector<2x1x16x2xf32> } : vector<2x1x16x2xi1> -> vector<2x1x16x2xf32>
-// CHECK: %[[trans0:.*]] = vector.transpose %[[read0]], [0, 3, 1, 2] : vector<2x1x16x2xf32> to vector<2x2x1x16xf32>
-// CHECK: %[[sc0:.*]] = vector.shape_cast %[[trans0]] : vector<2x2x1x16xf32> to vector<4x16xf32>
-// CHECK: %[[writeMsk0:.*]] = vector.create_mask {{.*}} : vector<4x16xi1>
-// CHECK: %[[write0:.*]] = vector.mask %[[writeMsk0:.*]] {{.*}} vector.transfer_write %[[sc0]], %[[ARG_0]]
-// CHECK: return %[[write0]]
- %ret = linalg.unpack %arg1 inner_dims_pos = [1, 0] inner_tiles = [16, 2] into %arg0 : tensor<?x?x16x2xf32> -> tensor<?x?xf32>
- return %ret : tensor<?x?xf32>
+// CHECK-SAME:      %[[DEST:.*]]: tensor<?x?xf32>,
+// CHECK-SAME:      %[[SRC:.*]]: tensor<?x?x16x2xf32>
+func.func @test_vectorize_dynamic_shapes_unpack(%dest: tensor<?x?xf32>, %src: tensor<?x?x16x2xf32>) -> tensor<?x?xf32> {
+  // CHECK: %[[C0:.*]] = arith.constant 0 : index
+  // CHECK: %[[C0_1:.*]] = arith.constant 0 : index
+  // CHECK: %[[DIM_0:.*]] = tensor.dim %[[SRC]], %[[C0_1]] : tensor<?x?x16x2xf32>
+  // CHECK: %[[C1:.*]] = arith.constant 1
+  // CHECK: %[[DIM6:.*]] = tensor.dim %[[SRC]], %[[C1]] : tensor<?x?x16x2xf32>
+  // CHECK: %[[CNST16:.*]] = arith.constant 16 : index
+  // CHECK: %[[CNST2:.*]] = arith.constant 2 : index
+  // CHECK: %[[MASK_READ:.*]] = vector.create_mask %[[DIM_0]], %[[DIM6]], %[[CNST16]], %[[CNST2]] : vector<2x1x16x2xi1>
+  // CHECK: %[[READ:.*]] = vector.mask %[[MASK_READ]] {{.*}} vector.transfer_read %{{.*}} : tensor<?x?x16x2xf32>, vector<2x1x16x2xf32> } : vector<2x1x16x2xi1> -> vector<2x1x16x2xf32>
+  // CHECK: %[[TR:.*]] = vector.transpose %[[READ]], [0, 3, 1, 2] : vector<2x1x16x2xf32> to vector<2x2x1x16xf32>
+  // CHECK: %[[SC:.*]] = vector.shape_cast %[[TR]] : vector<2x2x1x16xf32> to vector<4x16xf32>
+  // CHECK: %[[MASK_WRITE:.*]] = vector.create_mask {{.*}} : vector<4x16xi1>
+  // CHECK: %[[WRITE:.*]] = vector.mask %[[MASK_WRITE:.*]] {{.*}} vector.transfer_write %[[SC]], %[[DEST]]
+  // CHECK: return %[[WRITE]]
+  %ret = linalg.unpack %src inner_dims_pos = [1, 0] inner_tiles = [16, 2] into %dest : tensor<?x?x16x2xf32> -> tensor<?x?xf32>
+  return %ret : tensor<?x?xf32>
+}
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+   %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+   transform.structured.vectorize %0 vector_sizes [2, 1, 16, 2] : !transform.any_op
+   transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func @test_vectorize_dynamic_shapes_unpack_scalable_vec
+// CHECK-SAME:      %[[DEST:.*]]: tensor<?x?xf32>,
+// CHECK-SAME:      %[[SRC:.*]]: tensor<?x?x16x2xf32>
+func.func @test_vectorize_dynamic_shapes_unpack_scalable_vec(%dest: tensor<?x?xf32>, %src: tensor<?x?x16x2xf32>) -> tensor<?x?xf32> {
+  // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00
+  // CHECK: %[[C01:.*]] = arith.constant 0
+  // CHECK: %[[C02:.*]] = arith.constant 0
+  // CHECK: %[[DIM4:.*]] = tensor.dim %[[SRC]], %[[C02]] : tensor<?x?x16x2xf32>
+  // CHECK: %[[CNST14:.*]] = arith.constant 1
+  // CHECK: %[[DIM6:.*]] = tensor.dim %[[SRC]], %[[CNST14]] : tensor<?x?x16x2xf32>
+  // CHECK: %[[CNST16:.*]] = arith.constant 16 : index
+  // CHECK: %[[CNST2:.*]] = arith.constant 2 : index
+  // CHECK: %[[MASK_READ:.*]] = vector.create_mask %[[DIM4]], %[[DIM6]], %[[CNST16]], %[[CNST2]] : vector<2x1x[16]x2xi1>
+  // CHECK: %[[READ:.*]] = vector.mask %[[MASK_READ]] {{.*}} vector.transfer_read %{{.*}} : tensor<?x?x16x2xf32>, vector<2x1x[16]x2xf32> } : vector<2x1x[16]x2xi1> -> vector<2x1x[16]x2xf32>
+  // CHECK: %[[TR:.*]] = vector.transpose %[[READ]], [0, 3, 1, 2] : vector<2x1x[16]x2xf32> to vector<2x2x1x[16]xf32>
+  // CHECK: %[[SC:.*]] = vector.shape_cast %[[TR]] : vector<2x2x1x[16]xf32> to vector<4x[16]xf32>
+  // CHECK: %[[MASK_WRITE:.*]] = vector.create_mask {{.*}} : vector<4x[16]xi1>
+  // CHECK: %[[WRITE:.*]] = vector.mask %[[MASK_WRITE:.*]] {{.*}} vector.transfer_write %[[SC]], %[[DEST]]
+  // CHECK: return %[[WRITE]]
+  %ret = linalg.unpack %src inner_dims_pos = [1, 0] inner_tiles = [16, 2] into %dest : tensor<?x?x16x2xf32> -> tensor<?x?xf32>
+  return %ret : tensor<?x?xf32>
+}
+module attributes {transform.with_named_sequence} {
+ transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+   %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+   transform.structured.vectorize %0 vector_sizes [2, 1, [16], 2] : !transform.any_op
+   transform.yield
+ }
+}
+
+// -----
+
+// CHECK-LABEL: func @test_vectorize_dynamic_shapes_unpack_scalable_vec_and_tile_size
+// CHECK-SAME:      %[[DEST:.*]]: tensor<?x?xf32>,
+// CHECK-SAME:      %[[SRC:.*]]: tensor<?x?x?x2xf32>
+func.func @test_vectorize_dynamic_shapes_unpack_scalable_vec_and_tile_size(%dest: tensor<?x?xf32>, %src: tensor<?x?x?x2xf32>) -> tensor<?x?xf32> {
+  // CHECK: %[[CST:.*]] = arith.constant 0.000000e+00
+  // CHECK: %[[C01:.*]] = arith.constant 0
+  // CHECK: %[[C02:.*]] = arith.constant 0
+  // CHECK: %[[DIM4:.*]] = tensor.dim %[[SRC]], %[[C02]] : tensor<?x?x?x2xf32>
+  // CHECK: %[[C1_2:.*]] = arith.constant 1
+  // CHECK: %[[DIM6:.*]] = tensor.dim %[[SRC]], %[[C1_2]] : tensor<?x?x?x2xf32>
+  // CHECK: %[[C2:.*]] = arith.constant 2 : index
+  // CHECK: %[[DIM_2:.*]] = tensor.dim %[[SRC]], %[[C2]] : tensor<?x?x?x2xf32>
+  // CHECK: %[[C2_1:.*]] = arith.constant 2 : index
+  // CHECK: %[[MASK_READ:.*]] = vector.create_mask %[[DIM4]], %[[DIM6]], %[[DIM_2]], %[[C2_1]] : vector<2x1x[16]x2xi1>
+  // CHECK: %[[READ:.*]] = vector.mask %[[MASK_READ]] {{.*}} vector.transfer_read %{{.*}} : tensor<?x?x?x2xf32>, vector<2x1x[16]x2xf32> } : vector<2x1x[16]x2xi1> -> vector<2x1x[16]x2xf32>
+  // CHECK: %[[TR:.*]] = vector.transpose %[[READ]], [0, 3, 1, 2] : vector<2x1x[16]x2xf32> to vector<2x2x1x[16]xf32>
+  // CHECK: %[[SC:.*]] = vector.shape_cast %[[TR]] : vector<2x2x1x[16]xf32> to vector<4x[16]xf32>
+  // CHECK: %[[MASK_WRITE:.*]] = vector.create_mask {{.*}} : vector<4x[16]xi1>
+  // CHECK: %[[WRITE:.*]] = vector.mask %[[MASK_WRITE:.*]] {{.*}} vector.transfer_write %[[SC]], %[[DEST]]
+  // CHECK: return %[[WRITE]]
+
+  %vs = vector.vscale
+  %c16 = arith.constant 16 : index
+  %tile_size = arith.muli %vs, %c16 : index
+
+  %ret = linalg.unpack %src inner_dims_pos = [1, 0] inner_tiles = [%tile_size, 2] into %dest : tensor<?x?x?x2xf32> -> tensor<?x?xf32>
+  return %ret : tensor<?x?xf32>
 }
 module attributes {transform.with_named_sequence} {
  transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
    %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
-   transform.structured.vectorize %0 vector_sizes [4, 16] : !transform.any_op
+   transform.structured.vectorize %0 vector_sizes [2, 1, [16], 2] : !transform.any_op
    transform.yield
  }
 }
@@ -997,7 +1160,7 @@ func.func @test_vectorize_unpack(%source: tensor<8x8x32x16xf32>, %dest: tensor<2
  module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
-   transform.structured.vectorize %0 vector_sizes [512, 128] : !transform.any_op
+   transform.structured.vectorize %0 vector_sizes [16, 8, 32, 16] : !transform.any_op
     transform.yield
   }
 }
@@ -1022,7 +1185,7 @@ func.func @test_vectorize_unpack_no_masks(%source: tensor<8x8x32x16xf32>, %dest:
  module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
-   transform.structured.vectorize %0 vector_sizes [256, 128] : !transform.any_op
+   transform.structured.vectorize %0 vector_sizes [8, 8, 32, 16] : !transform.any_op
     transform.yield
   }
  }
@@ -1047,7 +1210,7 @@ func.func @test_vectorize_unpack_no_masks(%source: tensor<8x8x32x16xf32>, %dest:
  module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.unpack"]} in %arg0 : (!transform.any_op) -> !transform.any_op
-   transform.structured.vectorize %0 vector_sizes [256, 128] : !transform.any_op
+   transform.structured.vectorize %0 vector_sizes [8, 8, 32, 16] : !transform.any_op
     transform.yield
   }
 }
diff --git a/mlir/test/Dialect/NVGPU/invalid.mlir b/mlir/test/Dialect/NVGPU/invalid.mlir
index 2b64fa4a01173..f735e3f8cc623 100644
--- a/mlir/test/Dialect/NVGPU/invalid.mlir
+++ b/mlir/test/Dialect/NVGPU/invalid.mlir
@@ -378,3 +378,14 @@ func.func @check_matrixC_dim(%arg0: vector<4x4xf16>, %arg1: vector<2x2xf16>, %ar
   %d = nvgpu.mma.sync (%arg0, %arg1, %arg2) {mmaShape = [16, 8, 16]} : (vector<4x4xf16>, vector<2x2xf16>, vector<4xf16>) -> vector<2x2xf16>
   return %d : vector<2x2xf16>
 }
+
+// -----
+
+!desc = !nvgpu.tensormap.descriptor<tensor = memref<32x8xi8,3>, swizzle=none, l2promo = none, oob = zero, interleave = none>
+!mbarrier = !nvgpu.mbarrier.group<memorySpace = #gpu.address_space<workgroup>>
+func.func @tma_last_dim_bytes(%desc: !desc, %buffer: memref<32x8xi8,3>, %mbarrier: !mbarrier) {
+  %c0 = arith.constant 0 : index
+  // expected-error @+1 {{the bytes in the last dimension of the tensor map must be a multiple of 16}}
+  nvgpu.tma.async.load %desc[%c0, %c0], %mbarrier[%c0] to %buffer : !desc, !mbarrier -> memref<32x8xi8,3>
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 5088f2dfa7d7a..986c3844d0bb9 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -3017,3 +3017,110 @@ func.func @invalid_allocate_allocator(%arg0 : memref<i32>) -> () {
 
   return
 }
+
+// -----
+func.func @invalid_workdistribute_empty_region() -> () {
+  omp.teams {
+    // expected-error @below {{region cannot be empty}}
+    omp.workdistribute {
+    }
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @invalid_workdistribute_no_terminator() -> () {
+  omp.teams {
+    // expected-error @below {{region must be terminated with omp.terminator}}
+    omp.workdistribute {
+      %c0 = arith.constant 0 : i32
+    }
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @invalid_workdistribute_wrong_terminator() -> () {
+  omp.teams {
+    // expected-error @below {{region must be terminated with omp.terminator}}
+    omp.workdistribute {
+      %c0 = arith.constant 0 : i32
+      func.return
+    }
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @invalid_workdistribute_multiple_terminators() -> () {
+  omp.teams {
+    // expected-error @below {{region must have exactly one terminator}}
+    omp.workdistribute {
+      %cond = arith.constant true
+      cf.cond_br %cond, ^bb1, ^bb2
+    ^bb1:
+      omp.terminator
+    ^bb2:
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @invalid_workdistribute_with_barrier() -> () {
+  omp.teams {
+    // expected-error @below {{explicit barriers are not allowed in workdistribute region}}
+    omp.workdistribute {
+      %c0 = arith.constant 0 : i32
+      omp.barrier
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @invalid_workdistribute_nested_parallel() -> () {
+  omp.teams {
+    // expected-error @below {{nested parallel constructs not allowed in workdistribute}}
+    omp.workdistribute {
+      omp.parallel {
+        omp.terminator
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
+
+// -----
+// Test: nested teams not allowed in workdistribute
+func.func @invalid_workdistribute_nested_teams() -> () {
+  omp.teams {
+    // expected-error @below {{nested teams constructs not allowed in workdistribute}}
+    omp.workdistribute {
+      omp.teams {
+        omp.terminator
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  return
+}
+
+// -----
+func.func @invalid_workdistribute() -> () {
+// expected-error @below {{workdistribute must be nested under teams}}
+  omp.workdistribute {
+    omp.terminator
+  }
+  return
+}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 8c846cde1a3ca..3c2e0a3b7cc15 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -3238,3 +3238,15 @@ func.func @omp_allocate_dir(%arg0 : memref<i32>, %arg1 : memref<i32>) -> () {
   return
 }
 
+// CHECK-LABEL: func.func @omp_workdistribute
+func.func @omp_workdistribute() {
+  // CHECK: omp.teams
+  omp.teams {
+  // CHECK: omp.workdistribute
+  omp.workdistribute {
+    omp.terminator
+  }
+  omp.terminator
+  }
+  return
+}
diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir
index 308cf150aa98e..2752c492cb2be 100644
--- a/mlir/test/Dialect/SCF/canonicalize.mlir
+++ b/mlir/test/Dialect/SCF/canonicalize.mlir
@@ -1479,7 +1479,7 @@ func.func @execute_region_no_inline() {
 // CHECK-NEXT:       scf.execute_region
 // CHECK-NEXT:       %[[VAL:.*]] = "test.val"() : () -> i64
 // CHECK-NEXT:       scf.yield %[[VAL]] : i64
-// CHECK-NEXT:     }
+// CHECK-NOT:      no_inline
 
 // -----
 
@@ -1912,3 +1912,16 @@ func.func @index_switch_fold_no_res() {
 
 // CHECK-LABEL: func.func @index_switch_fold_no_res()
 //  CHECK-NEXT: "test.op"() : () -> ()
+
+// -----
+
+// CHECK-LABEL: func @scf_for_all_step_size_0()
+//       CHECK:   scf.forall (%{{.*}}) = (0) to (1) step (0)
+func.func @scf_for_all_step_size_0()  {
+  %x = arith.constant 0 : index
+  scf.forall (%i, %j) = (0, 4) to (1, 5) step (%x, 8) {
+    vector.print %x : index
+    scf.forall.in_parallel {}
+  }
+  return
+}
diff --git a/mlir/test/Dialect/SCF/ops.mlir b/mlir/test/Dialect/SCF/ops.mlir
index 7f457ef3b6ba0..5930a1df04266 100644
--- a/mlir/test/Dialect/SCF/ops.mlir
+++ b/mlir/test/Dialect/SCF/ops.mlir
@@ -28,14 +28,14 @@ func.func @std_for(%arg0 : index, %arg1 : index, %arg2 : index) {
 
 func.func @std_for_i32(%arg0 : i32, %arg1 : i32, %arg2 : i32) {
   scf.for %i0 = %arg0 to %arg1 step %arg2 : i32 {
-    scf.for %i1 = %arg0 to %arg1 step %arg2 : i32 {
+    scf.for unsigned %i1 = %arg0 to %arg1 step %arg2 : i32 {
     }
   }
   return
 }
 // CHECK-LABEL: func @std_for_i32(
 //  CHECK-NEXT:   scf.for %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} : i32 {
-//  CHECK-NEXT:     scf.for %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} : i32 {
+//  CHECK-NEXT:     scf.for unsigned %{{.*}} = %{{.*}} to %{{.*}} step %{{.*}} : i32 {
 
 func.func @scf_for_i64_iter(%arg1: i64, %arg2: i64) {
   %c1_i64 = arith.constant 1 : i64
diff --git a/mlir/test/Dialect/Tosa/canonicalize.mlir b/mlir/test/Dialect/Tosa/canonicalize.mlir
index 5150ee36e9e5e..930bb9fe96811 100644
--- a/mlir/test/Dialect/Tosa/canonicalize.mlir
+++ b/mlir/test/Dialect/Tosa/canonicalize.mlir
@@ -565,6 +565,33 @@ func.func @mul_zero_broadcast(%arg0: tensor<2x3xf32>) -> (tensor<2x3xf32>, tenso
 
 // -----
 
+// CHECK-LABEL: @mul_zero_dynamic_nofold
+// CHECK-SAME:                    %[[ARG0:.*]]: tensor<?x17xf32>) -> tensor<?x17xf32> {
+// CHECK:           %[[ZERO:.*]] = "tosa.const"() <{values = dense<0.000000e+00> : tensor<1x1xf32>}> : () -> tensor<1x1xf32>
+// CHECK:           %[[SHIFT:.*]] = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+// CHECK:           %[[MUL:.*]] = tosa.mul %[[ARG0]], %[[ZERO]], %[[SHIFT]] : (tensor<?x17xf32>, tensor<1x1xf32>, tensor<1xi8>) -> tensor<?x17xf32>
+// CHECK:           return %[[MUL]]
+func.func @mul_zero_dynamic_nofold(%arg0: tensor<?x17xf32>) -> tensor<?x17xf32> {
+  %0 = "tosa.const"() <{values = dense<0.000000e+00> : tensor<1x1xf32>}> : () -> tensor<1x1xf32>
+  %1 = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %2 = tosa.mul %arg0, %0, %1 : (tensor<?x17xf32>, tensor<1x1xf32>, tensor<1xi8>) -> tensor<?x17xf32>
+  return %2 : tensor<?x17xf32>
+}
+
+// -----
+
+// CHECK-LABEL: @mul_one_dynamic_fold
+// CHECK-SAME:                    %[[ARG0:.*]]: tensor<?x17xf32>) -> tensor<?x17xf32> {
+// CHECK:           return %[[ARG0]]
+func.func @mul_one_dynamic_fold(%arg0: tensor<?x17xf32>) -> tensor<?x17xf32> {
+  %0 = "tosa.const"() <{values = dense<1.000000e+00> : tensor<1x1xf32>}> : () -> tensor<1x1xf32>
+  %1 = "tosa.const"() <{values = dense<0> : tensor<1xi8>}> : () -> tensor<1xi8>
+  %2 = tosa.mul %arg0, %0, %1 : (tensor<?x17xf32>, tensor<1x1xf32>, tensor<1xi8>) -> tensor<?x17xf32>
+  return %2 : tensor<?x17xf32>
+}
+
+// -----
+
 // CHECK-LABEL: @select_same_value
 func.func @select_same_value(%arg0: tensor<2x3xi1>, %arg1: tensor<2x3xi32>) -> tensor<2x3xi32> {
   %0 = tosa.select %arg0, %arg1, %arg1 : (tensor<2x3xi1>, tensor<2x3xi32>, tensor<2x3xi32>) -> tensor<2x3xi32>
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index f86fb387be5b8..c640ddea7507b 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -625,40 +625,40 @@ func.func @insert_extract_transpose_2d(
 // -----
 
 // CHECK-LABEL: insert_extract_chain
-//  CHECK-SAME: %[[V234:[a-zA-Z0-9]*]]: vector<2x3x4xf32>
+//  CHECK-SAME: %[[V334:[a-zA-Z0-9]*]]: vector<3x3x4xf32>
 //  CHECK-SAME: %[[V34:[a-zA-Z0-9]*]]: vector<3x4xf32>
 //  CHECK-SAME: %[[V4:[a-zA-Z0-9]*]]: vector<4xf32>
-func.func @insert_extract_chain(%v234: vector<2x3x4xf32>, %v34: vector<3x4xf32>, %v4: vector<4xf32>)
+func.func @insert_extract_chain(%v334: vector<3x3x4xf32>, %v34: vector<3x4xf32>, %v4: vector<4xf32>)
     -> (vector<4xf32>, vector<4xf32>, vector<3x4xf32>, vector<3x4xf32>) {
   // CHECK-NEXT: %[[A34:.*]] = vector.insert
-  %A34 = vector.insert %v34, %v234[0]: vector<3x4xf32> into vector<2x3x4xf32>
+  %A34 = vector.insert %v34, %v334[0]: vector<3x4xf32> into vector<3x3x4xf32>
   // CHECK-NEXT: %[[B34:.*]] = vector.insert
-  %B34 = vector.insert %v34, %A34[1]: vector<3x4xf32> into vector<2x3x4xf32>
+  %B34 = vector.insert %v34, %A34[1]: vector<3x4xf32> into vector<3x3x4xf32>
   // CHECK-NEXT: %[[A4:.*]] = vector.insert
-  %A4 = vector.insert %v4, %B34[1, 0]: vector<4xf32> into vector<2x3x4xf32>
+  %A4 = vector.insert %v4, %B34[1, 0]: vector<4xf32> into vector<3x3x4xf32>
   // CHECK-NEXT: %[[B4:.*]] = vector.insert
-  %B4 = vector.insert %v4, %A4[1, 1]: vector<4xf32> into vector<2x3x4xf32>
+  %B4 = vector.insert %v4, %A4[1, 1]: vector<4xf32> into vector<3x3x4xf32>
 
   // Case 2.a. [1, 1] == insertpos ([1, 1])
   // Match %A4 insertionpos and fold to its source(i.e. %V4).
-   %r0 = vector.extract %B4[1, 1]: vector<4xf32> from vector<2x3x4xf32>
+   %r0 = vector.extract %B4[1, 1]: vector<4xf32> from vector<3x3x4xf32>
 
   // Case 3.a. insertpos ([1]) is a prefix of [1, 0].
   // Traverse %B34 to its source(i.e. %V34@[*0*]).
   // CHECK-NEXT: %[[R1:.*]] = vector.extract %[[V34]][0]
-   %r1 = vector.extract %B34[1, 0]: vector<4xf32> from vector<2x3x4xf32>
+   %r1 = vector.extract %B34[1, 0]: vector<4xf32> from vector<3x3x4xf32>
 
   // Case 4. [1] is a prefix of insertpos ([1, 1]).
   // Cannot traverse %B4.
   // CHECK-NEXT: %[[R2:.*]] = vector.extract %[[B4]][1]
-   %r2 = vector.extract %B4[1]: vector<3x4xf32> from vector<2x3x4xf32>
+   %r2 = vector.extract %B4[1]: vector<3x4xf32> from vector<3x3x4xf32>
 
   // Case 5. [0] is disjoint from insertpos ([1, 1]).
   // Traverse %B4 to its dest(i.e. %A4@[0]).
   // Traverse %A4 to its dest(i.e. %B34@[0]).
   // Traverse %B34 to its dest(i.e. %A34@[0]).
   // Match %A34 insertionpos and fold to its source(i.e. %V34).
-   %r3 = vector.extract %B4[0]: vector<3x4xf32> from vector<2x3x4xf32>
+   %r3 = vector.extract %B4[0]: vector<3x4xf32> from vector<3x3x4xf32>
 
   // CHECK: return %[[V4]], %[[R1]], %[[R2]], %[[V34]]
   return %r0, %r1, %r2, %r3:
@@ -1057,8 +1057,8 @@ func.func @insert_fold_same_rank(%v: vector<2x2xf32>) -> vector<2x2xf32> {
 
 // CHECK-LABEL: func @insert_no_fold_scalar_to_0d(
 //  CHECK-SAME:     %[[v:.*]]: vector<f32>)
-//       CHECK:   %[[extract:.*]] = vector.insert %{{.*}}, %[[v]] [] : f32 into vector<f32>
-//       CHECK:   return %[[extract]]
+//       CHECK:   %[[cst:.*]] = arith.constant dense<0.000000e+00> : vector<f32>
+//       CHECK:   return %[[cst]]
 func.func @insert_no_fold_scalar_to_0d(%v: vector<f32>) -> vector<f32> {
   %cst = arith.constant 0.000000e+00 : f32
   %0 = vector.insert %cst, %v [] : f32 into vector<f32>
@@ -1168,6 +1168,106 @@ func.func @canonicalize_broadcast_shapecast_both_possible(%arg0: vector<1xf32>)
 
 // -----
 
+// CHECK-LABEL: func @canonicalize_shapecast_broadcast_to_broadcast_prepend_dim
+//   CHECK-NOT:   vector.shape_cast
+//       CHECK:   vector.broadcast {{.+}} : vector<2xf32> to vector<32x2xf32>
+func.func @canonicalize_shapecast_broadcast_to_broadcast_prepend_dim(%arg0 : vector<2xf32>) -> vector<32x2xf32> {
+  %0 = vector.shape_cast %arg0 : vector<2xf32> to vector<1x2xf32>
+  %1 = vector.broadcast %0 : vector<1x2xf32> to vector<32x2xf32>
+  return %1 : vector<32x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @canonicalize_shapecast_broadcast_to_broadcast_prepend_dim2(
+// CHECK-SAME:      %[[ARG0:.*]]: vector<2x1xf32>) -> vector<32x2x1xf32> {
+// CHECK:           %[[VAL_0:.*]] = vector.broadcast %[[ARG0]] : vector<2x1xf32> to vector<32x2x1xf32>
+// CHECK:           return %[[VAL_0]] : vector<32x2x1xf32>
+// CHECK:         }
+func.func @canonicalize_shapecast_broadcast_to_broadcast_prepend_dim2(%arg0 : vector<2x1xf32>) -> vector<32x2x1xf32> {
+  %0 = vector.shape_cast %arg0 : vector<2x1xf32> to vector<1x2x1xf32>
+  %1 = vector.broadcast %0 : vector<1x2x1xf32> to vector<32x2x1xf32>
+  return %1 : vector<32x2x1xf32>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @canonicalize_shapecast_broadcast_to_broadcast_prepend_dim3(
+// CHECK-SAME:      %[[ARG0:.*]]: vector<2x1xf32>) -> vector<32x2x4xf32> {
+// CHECK:           %[[VAL_0:.*]] = vector.broadcast %[[ARG0]] : vector<2x1xf32> to vector<32x2x4xf32>
+// CHECK:           return %[[VAL_0]] : vector<32x2x4xf32>
+// CHECK:         }
+func.func @canonicalize_shapecast_broadcast_to_broadcast_prepend_dim3(%arg0 : vector<2x1xf32>) -> vector<32x2x4xf32> {
+  %0 = vector.shape_cast %arg0 : vector<2x1xf32> to vector<1x2x1xf32>
+  %1 = vector.broadcast %0 : vector<1x2x1xf32> to vector<32x2x4xf32>
+  return %1 : vector<32x2x4xf32>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @canonicalize_shapecast_broadcast_to_broadcast_remove_leading_dim(
+// CHECK-SAME:      %[[ARG0:.*]]: vector<1x2xf32>) -> vector<32x2xf32> {
+// CHECK:           %[[VAL_0:.*]] = vector.broadcast %[[ARG0]] : vector<1x2xf32> to vector<32x2xf32>
+// CHECK:           return %[[VAL_0]] : vector<32x2xf32>
+// CHECK:         }
+func.func @canonicalize_shapecast_broadcast_to_broadcast_remove_leading_dim(%arg0 : vector<1x2xf32>) -> vector<32x2xf32> {
+  %0 = vector.shape_cast %arg0 : vector<1x2xf32> to vector<2xf32>
+  %1 = vector.broadcast %0 : vector<2xf32> to vector<32x2xf32>
+  return %1 : vector<32x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_canonicalize_shapecast_broadcast_invalid_shape
+//       CHECK:   vector.shape_cast {{.+}} : vector<64xf32> to vector<4x16xf32>
+//       CHECK:   vector.broadcast {{.+}} : vector<4x16xf32> to vector<2x4x16xf32>
+func.func @negative_canonicalize_shapecast_broadcast_invalid_shape(%arg0 : vector<64xf32>) -> vector<2x4x16xf32> {
+  %0 = vector.shape_cast %arg0 : vector<64xf32> to vector<4x16xf32>
+  %1 = vector.broadcast %0 : vector<4x16xf32> to vector<2x4x16xf32>
+  return %1 : vector<2x4x16xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @negative_canonicalize_shapecast_broadcast_invalid_broadcasted_dims
+//       CHECK:   vector.shape_cast {{.+}} : vector<2x1xf32> to vector<1x2xf32>
+//       CHECK:   vector.broadcast {{.+}} : vector<1x2xf32> to vector<2x2xf32>
+func.func @negative_canonicalize_shapecast_broadcast_invalid_broadcasted_dims(%arg0 : vector<2x1xf32>) -> vector<2x2xf32> {
+  %0 = vector.shape_cast %arg0 : vector<2x1xf32> to vector<1x2xf32>
+  %1 = vector.broadcast %0 : vector<1x2xf32> to vector<2x2xf32>
+  return %1 : vector<2x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @negative_canonicalize_shapecast_broadcast_to_broadcast_append_dim(
+// CHECK-SAME:      %[[ARG0:.*]]: vector<2xf32>) -> vector<2x4xf32> {
+// CHECK:           %[[VAL_0:.*]] = vector.shape_cast %[[ARG0]] : vector<2xf32> to vector<2x1xf32>
+// CHECK:           %[[VAL_1:.*]] = vector.broadcast %[[VAL_0]] : vector<2x1xf32> to vector<2x4xf32>
+// CHECK:           return %[[VAL_1]] : vector<2x4xf32>
+// CHECK:         }
+func.func @negative_canonicalize_shapecast_broadcast_to_broadcast_append_dim(%arg0 : vector<2xf32>) -> vector<2x4xf32> {
+  %0 = vector.shape_cast %arg0 : vector<2xf32> to vector<2x1xf32>
+  %1 = vector.broadcast %0 : vector<2x1xf32> to vector<2x4xf32>
+  return %1 : vector<2x4xf32>
+}
+
+// -----
+
+// CHECK-LABEL:   func.func @negative_canonicalize_shapecast_broadcast_to_broadcast_remove_trailing_dim(
+// CHECK-SAME:      %[[ARG0:.*]]: vector<2x1xf32>) -> vector<32x2xf32> {
+// CHECK:           %[[VAL_0:.*]] = vector.shape_cast %[[ARG0]] : vector<2x1xf32> to vector<2xf32>
+// CHECK:           %[[VAL_1:.*]] = vector.broadcast %[[VAL_0]] : vector<2xf32> to vector<32x2xf32>
+// CHECK:           return %[[VAL_1]] : vector<32x2xf32>
+// CHECK:         }
+func.func @negative_canonicalize_shapecast_broadcast_to_broadcast_remove_trailing_dim(%arg0 : vector<2x1xf32>) -> vector<32x2xf32> {
+  %0 = vector.shape_cast %arg0 : vector<2x1xf32> to vector<2xf32>
+  %1 = vector.broadcast %0 : vector<2xf32> to vector<32x2xf32>
+  return %1 : vector<32x2xf32>
+}
+
+// -----
+
 // CHECK-LABEL: fold_vector_transfer_masks
 func.func @fold_vector_transfer_masks(%A: memref<?x?xf32>) -> (vector<4x8xf32>, vector<4x[4]xf32>) {
   // CHECK: %[[C0:.+]] = arith.constant 0 : index
@@ -2569,6 +2669,112 @@ func.func @insert_2d_constant() -> (vector<2x3xi32>, vector<2x3xi32>, vector<2x3
 
 // -----
 
+// +---------------------------------------------------------------------------
+// Tests for InsertChainFullyInitialized .
+// +---------------------------------------------------------------------------
+// This pattern should fire when all vector elements are overwritten by vector.insert
+// at static positions, replacing the insert chain with vector.from_elements.
+// CHECK-LABEL: func.func @fully_insert_scalar_to_vector(
+//  CHECK-SAME: %[[DEST:.+]]: vector<2xi64>, %[[SRC1:.+]]: i64, %[[SRC2:.+]]: i64)
+//       CHECK: %[[RES:.+]] = vector.from_elements %[[SRC1]], %[[SRC2]] : vector<2xi64>
+//  CHECK-NEXT: return %[[RES]]
+func.func @fully_insert_scalar_to_vector(%dest : vector<2xi64>, %src1 : i64, %src2 : i64) -> vector<2xi64> {
+  %v1 = vector.insert %src1, %dest[0] : i64 into vector<2xi64>
+  %v2 = vector.insert %src2, %v1[1] : i64 into vector<2xi64>
+  return %v2 : vector<2xi64>
+}
+
+// -----
+
+// Same as the above test, but with vector insertions.
+// CHECK-LABEL: func.func @fully_insert_vector_to_vector(
+//  CHECK-SAME: %[[DEST:.+]]: vector<2x2xi64>, %[[SRC1:.+]]: vector<2xi64>, %[[SRC2:.+]]: vector<2xi64>)
+//       CHECK: %[[ELE1:.+]]:2 = vector.to_elements %[[SRC1]] : vector<2xi64>
+//       CHECK: %[[ELE2:.+]]:2 = vector.to_elements %[[SRC2]] : vector<2xi64>
+//       CHECK: %[[RES:.+]] = vector.from_elements %[[ELE1]]#0, %[[ELE1]]#1, %[[ELE2]]#0, %[[ELE2]]#1 : vector<2x2xi64>
+//  CHECK-NEXT: return %[[RES]]
+func.func @fully_insert_vector_to_vector(%dest : vector<2x2xi64>, %src1 : vector<2xi64>, %src2 : vector<2xi64>) -> vector<2x2xi64> {
+  %v1 = vector.insert %src1, %dest[0] : vector<2xi64> into vector<2x2xi64>
+  %v2 = vector.insert %src2, %v1[1] : vector<2xi64> into vector<2x2xi64>
+  return %v2 : vector<2x2xi64>
+}
+
+// -----
+
+// Test InsertChainFullyInitialized pattern with overlapping insertions.
+// 1. The first op inserts %src2 at [0,0].
+// 2. The second op inserts %src1 at [0,0], [0,1], overwriting %src2 at [0,0].
+// 3. The third op inserts %src1 at [1,0], [1,1].
+// 4. The fourth op inserts %src2 at [1,1], overwriting the existing value at [1,1].
+// CHECK-LABEL: func.func @fully_insert_to_vector_overlap_1(
+//  CHECK-SAME: %[[DEST:.+]]: vector<2x2xi64>, %[[SRC1:.+]]: vector<2xi64>, %[[SRC2:.+]]: i64)
+//       CHECK: %[[ELE1:.+]]:2 = vector.to_elements %[[SRC1]] : vector<2xi64>
+//       CHECK: %[[ELE2:.+]]:2 = vector.to_elements %[[SRC1]] : vector<2xi64>
+//       CHECK: %[[RES:.+]] = vector.from_elements %[[ELE1]]#0, %[[ELE1]]#1, %[[ELE2]]#0, %[[SRC2]] : vector<2x2xi64>
+//  CHECK-NEXT: return %[[RES]]
+func.func @fully_insert_to_vector_overlap_1(%dest : vector<2x2xi64>, %src1 : vector<2xi64>, %src2 : i64) -> vector<2x2xi64> {
+  %v0 = vector.insert %src2, %dest[0, 0] : i64 into vector<2x2xi64>
+  %v1 = vector.insert %src1, %v0[0] : vector<2xi64> into vector<2x2xi64>
+  %v2 = vector.insert %src1, %v1[1] : vector<2xi64> into vector<2x2xi64>
+  %v3 = vector.insert %src2, %v2[1, 1] : i64 into vector<2x2xi64>
+  return %v3 : vector<2x2xi64>
+}
+
+// -----
+
+// Test InsertChainFullyInitialized pattern with overlapping insertions.
+// The vector inserted at last should overwrite the previously inserted scalars.
+// CHECK-LABEL: func.func @fully_insert_to_vector_overlap_2(
+//  CHECK-SAME: %[[DEST:.+]]: vector<2x2xi64>, %[[SRC1:.+]]: i64, %[[SRC2:.+]]: i64, %[[SRC3:.+]]: vector<2xi64>, %[[SRC4:.+]]: vector<2xi64>)
+//       CHECK: %[[ELE1:.+]]:2 = vector.to_elements %[[SRC3]] : vector<2xi64>
+//       CHECK: %[[ELE2:.+]]:2 = vector.to_elements %[[SRC4]] : vector<2xi64>
+//       CHECK: %[[RES:.+]] = vector.from_elements %[[ELE1]]#0, %[[ELE1]]#1, %[[ELE2]]#0, %[[ELE2]]#1 : vector<2x2xi64>
+//  CHECK-NEXT: return %[[RES]]
+func.func @fully_insert_to_vector_overlap_2(%dest : vector<2x2xi64>, %src1 : i64, %src2 : i64, %src3 : vector<2xi64>, %src4 : vector<2xi64>) -> vector<2x2xi64> {
+  %v0 = vector.insert %src1, %dest[0, 0] : i64 into vector<2x2xi64>
+  %v1 = vector.insert %src2, %v0[0, 1] : i64 into vector<2x2xi64>
+  %v2 = vector.insert %src3, %v1[0] : vector<2xi64> into vector<2x2xi64>
+  %v3 = vector.insert %src4, %v2[1] : vector<2xi64> into vector<2x2xi64>
+  return %v3 : vector<2x2xi64>
+}
+
+// -----
+
+// Negative test for InsertChainFullyInitialized pattern when only some elements are overwritten.
+// CHECK-LABEL: func.func @negative_partially_insert_vector_to_vector(
+//  CHECK-SAME: %[[DEST:.+]]: vector<2x2xi64>, %[[SRC1:.+]]: vector<2xi64>, %[[SRC2:.+]]: i64)
+//       CHECK: %[[V0:.+]] = vector.insert %[[SRC1]], %[[DEST]] [0] : vector<2xi64> into vector<2x2xi64>
+//       CHECK: %[[V1:.+]] = vector.insert %[[SRC2]], %[[V0]] [0, 0] : i64 into vector<2x2xi64>
+//       CHECK: return %[[V1]]
+func.func @negative_partially_insert_vector_to_vector(%dest : vector<2x2xi64>, %src1 : vector<2xi64>, %src2 : i64) -> vector<2x2xi64> {
+  %v1 = vector.insert %src1, %dest[0] : vector<2xi64> into vector<2x2xi64>
+  %v2 = vector.insert %src2, %v1[0, 0] : i64 into vector<2x2xi64>
+  return %v2 : vector<2x2xi64>
+}
+
+// -----
+
+// Negative test when intermediate results have more than one user.
+// CHECK-LABEL: func.func @negative_intermediate_insert_multiple_users(
+//  CHECK-SAME: %[[DEST:.+]]: vector<3xi64>, %[[SRC1:.+]]: i64, %[[SRC2:.+]]: i64, %[[SRC3:.+]]: i64, %[[SRC4:.+]]: i64)
+//       CHECK: %[[V0:.+]] = vector.insert %[[SRC1]], %[[DEST]] [0] : i64 into vector<3xi64>
+//       CHECK: %[[V1:.+]] = vector.insert %[[SRC2]], %[[V0]] [1] : i64 into vector<3xi64>
+//       CHECK: %[[V2:.+]] = vector.insert %[[SRC3]], %[[V1]] [2] : i64 into vector<3xi64>
+//       CHECK: %[[V3:.+]] = vector.insert %[[SRC4]], %[[V1]] [2] : i64 into vector<3xi64>
+func.func @negative_intermediate_insert_multiple_users(%dest : vector<3xi64>, %src1 : i64, %src2 : i64, %src3 : i64, %src4 : i64) -> (vector<3xi64>, vector<3xi64>) {
+  %v1 = vector.insert %src1, %dest[0] : i64 into vector<3xi64>
+  %v2 = vector.insert %src2, %v1[1] : i64 into vector<3xi64>
+  %v3_0 = vector.insert %src3, %v2[2] : i64 into vector<3xi64>
+  %v3_1 = vector.insert %src4, %v2[2] : i64 into vector<3xi64>
+  return %v3_0, %v3_1 : vector<3xi64>, vector<3xi64>
+}
+
+// +---------------------------------------------------------------------------
+// End of  Tests For InsertChainFullyInitialized.
+// +---------------------------------------------------------------------------
+
+// -----
+
 // CHECK-LABEL: func.func @insert_2d_splat_constant
 //   CHECK-DAG: %[[ACST:.*]] = arith.constant dense<0> : vector<2x3xi32>
 //   CHECK-DAG: %[[BCST:.*]] = arith.constant dense<{{\[\[99, 0, 0\], \[0, 0, 0\]\]}}> : vector<2x3xi32>
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index c21de562d05e1..6ee70fdd89a85 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -1305,6 +1305,26 @@ func.func @store_memref_index_mismatch(%base : memref<?xf32>, %value : vector<16
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.maskedload
+//===----------------------------------------------------------------------===//
+
+func.func @maskedload_nonpositive_alignment(%base: memref<4xi32>, %mask: vector<32xi1>, %pass: vector<1xi32>, %index: index) {
+  // expected-error@below {{'vector.maskedload' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  %val = vector.maskedload %base[%index], %mask, %pass { alignment = 0 } : memref<4xi32>, vector<32xi1>, vector<1xi32> into vector<1xi32>
+  return
+}
+
+// -----
+
+func.func @maskedload_non_power_of_2_alignment(%base: memref<4xi32>, %mask: vector<32xi1>, %pass: vector<1xi32>, %index: index) {
+  // expected-error@below {{'vector.maskedload' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  %val = vector.maskedload %base[%index], %mask, %pass { alignment = 3 } : memref<4xi32>, vector<32xi1>, vector<1xi32> into vector<1xi32>
+  return
+}
+
+// -----
+
 func.func @maskedload_base_type_mismatch(%base: memref<?xf64>, %mask: vector<16xi1>, %pass: vector<16xf32>) {
   %c0 = arith.constant 0 : index
   // expected-error@+1 {{'vector.maskedload' op base and result element type should match}}
@@ -1336,6 +1356,26 @@ func.func @maskedload_memref_mismatch(%base: memref<?xf32>, %mask: vector<16xi1>
 
 // -----
 
+//===----------------------------------------------------------------------===//
+// vector.maskedstore
+//===----------------------------------------------------------------------===//
+
+func.func @maskedstore_nonpositive_alignment(%base: memref<4xi32>, %mask: vector<32xi1>, %value: vector<1xi32>, %index: index) {
+  // expected-error@below {{'vector.maskedstore' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  vector.maskedstore %base[%index], %mask, %value { alignment = 0 } : memref<4xi32>, vector<32xi1>, vector<1xi32> into vector<1xi32>
+  return
+}
+
+// -----
+
+func.func @maskedstore_non_power_of_2_alignment(%base: memref<4xi32>, %mask: vector<32xi1>, %value: vector<1xi32>, %index: index) {
+  // expected-error@below {{'vector.maskedstore' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  vector.maskedstore %base[%index], %mask, %value { alignment = 3 } : memref<4xi32>, vector<32xi1>, vector<1xi32> into vector<1xi32>
+  return
+}
+
+// -----
+
 func.func @maskedstore_base_type_mismatch(%base: memref<?xf64>, %mask: vector<16xi1>, %value: vector<16xf32>) {
   %c0 = arith.constant 0 : index
   // expected-error@+1 {{'vector.maskedstore' op base and valueToStore element type should match}}
@@ -1430,6 +1470,24 @@ func.func @gather_pass_thru_type_mismatch(%base: memref<?xf32>, %indices: vector
 
 // -----
 
+func.func @gather_nonpositive_alignment(%base: memref<16xf32>, %indices: vector<16xi32>,
+                                %mask: vector<16xi1>, %pass_thru: vector<16xf32>, %c0 : index) {
+  // expected-error@+2 {{'vector.gather' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru
+    { alignment = 0 } : memref<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
+}
+
+// -----
+
+func.func @gather_non_power_of_two_alignment(%base: memref<16xf32>, %indices: vector<16xi32>,
+                                %mask: vector<16xi1>, %pass_thru: vector<16xf32>, %c0 : index) {
+  // expected-error@+2 {{'vector.gather' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru
+    { alignment = 3 } : memref<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
+}
+
+// -----
+
 func.func @scatter_to_vector(%base: vector<16xf32>, %indices: vector<16xi32>,
                              %mask: vector<16xi1>, %pass_thru: vector<16xf32>) {
   %c0 = arith.constant 0 : index
@@ -1491,6 +1549,24 @@ func.func @scatter_dim_mask_mismatch(%base: memref<?xf32>, %indices: vector<16xi
 
 // -----
 
+func.func @scatter_nonpositive_alignment(%base: memref<?xf32>, %indices: vector<16xi32>,
+                                %mask: vector<16xi1>, %value: vector<16xf32>, %c0: index) {
+  // expected-error@+1 {{'vector.scatter' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  vector.scatter %base[%c0][%indices], %mask, %value { alignment = 0 }
+    : memref<?xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32>
+}
+
+// -----
+
+func.func @scatter_non_power_of_2_alignment(%base: memref<?xf32>, %indices: vector<16xi32>,
+                                %mask: vector<16xi1>, %value: vector<16xf32>, %c0: index) {
+  // expected-error@+1 {{'vector.scatter' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  vector.scatter %base[%c0][%indices], %mask, %value { alignment = 3 }
+    : memref<?xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32>
+}
+
+// -----
+
 func.func @expand_base_type_mismatch(%base: memref<?xf64>, %mask: vector<16xi1>, %pass_thru: vector<16xf32>) {
   %c0 = arith.constant 0 : index
   // expected-error@+1 {{'vector.expandload' op base and result element type should match}}
@@ -1531,6 +1607,20 @@ func.func @expand_memref_mismatch(%base: memref<?x?xf32>, %mask: vector<16xi1>,
 
 // -----
 
+func.func @expand_nonpositive_alignment(%base: memref<?xf32>, %mask: vector<16xi1>, %pass_thru: vector<16xf32>, %c0: index) {
+  // expected-error@+1 {{'vector.expandload' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  %0 = vector.expandload %base[%c0], %mask, %pass_thru { alignment = 0 } : memref<?xf32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
+}
+
+// -----
+
+func.func @expand_non_power_of_2_alignment(%base: memref<?xf32>, %mask: vector<16xi1>, %pass_thru: vector<16xf32>, %c0: index) {
+  // expected-error@+1 {{'vector.expandload' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  %0 = vector.expandload %base[%c0], %mask, %pass_thru { alignment = 3 } : memref<?xf32>, vector<16xi1>, vector<16xf32> into vector<16xf32>
+}
+
+// -----
+
 func.func @compress_base_type_mismatch(%base: memref<?xf64>, %mask: vector<16xi1>, %value: vector<16xf32>) {
   %c0 = arith.constant 0 : index
   // expected-error@+1 {{'vector.compressstore' op base and valueToStore element type should match}}
@@ -1563,6 +1653,20 @@ func.func @compress_memref_mismatch(%base: memref<?x?xf32>, %mask: vector<16xi1>
 
 // -----
 
+func.func @compress_nonpositive_alignment(%base: memref<?xf32>, %mask: vector<16xi1>, %value: vector<16xf32>, %c0: index) {
+  // expected-error @below {{'vector.compressstore' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  vector.compressstore %base[%c0], %mask, %value { alignment = 0 } : memref<?xf32>, vector<16xi1>, vector<16xf32>
+}
+
+// -----
+
+func.func @compress_non_power_of_2_alignment(%base: memref<?xf32>, %mask: vector<16xi1>, %value: vector<16xf32>, %c0: index) {
+  // expected-error @below {{'vector.compressstore' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  vector.compressstore %base[%c0], %mask, %value { alignment = 3 } : memref<?xf32>, vector<16xi1>, vector<16xf32>
+}
+
+// -----
+
 func.func @scan_reduction_dim_constraint(%arg0: vector<2x3xi32>, %arg1: vector<3xi32>) -> vector<3xi32> {
   // expected-error@+1 {{'vector.scan' op reduction dimension 5 has to be less than 2}}
   %0:2 = vector.scan <add>, %arg0, %arg1 {inclusive = true, reduction_dim = 5} :
@@ -1912,10 +2016,17 @@ func.func @vector_load(%src : memref<?xi8>) {
 
 // -----
 
-func.func @invalid_load_alignment(%memref: memref<4xi32>) {
-  %c0 = arith.constant 0 : index
+func.func @load_nonpositive_alignment(%memref: memref<4xi32>, %c0: index) {
+  // expected-error @below {{'vector.load' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  %val = vector.load %memref[%c0] { alignment = 0 } : memref<4xi32>, vector<4xi32>
+  return
+}
+
+// -----
+
+func.func @load_non_pow_of_2_alignment(%memref: memref<4xi32>, %c0: index) {
   // expected-error @below {{'vector.load' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
-  %val = vector.load %memref[%c0] { alignment = -1 } : memref<4xi32>, vector<4xi32>
+  %val = vector.load %memref[%c0] { alignment = 3 } : memref<4xi32>, vector<4xi32>
   return
 }
 
@@ -1934,8 +2045,15 @@ func.func @vector_store(%dest : memref<?xi8>, %vec : vector<16x16xi8>) {
 
 // -----
 
-func.func @invalid_store_alignment(%memref: memref<4xi32>, %val: vector<4xi32>) {
-  %c0 = arith.constant 0 : index
+func.func @store_nonpositive_alignment(%memref: memref<4xi32>, %val: vector<4xi32>, %c0: index) {
+  // expected-error @below {{'vector.store' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
+  vector.store %val, %memref[%c0] { alignment = 0 } : memref<4xi32>, vector<4xi32>
+  return
+}
+
+// -----
+
+func.func @store_non_pow_of_2_alignment(%memref: memref<4xi32>, %val: vector<4xi32>, %c0: index) {
   // expected-error @below {{'vector.store' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}}
   vector.store %val, %memref[%c0] { alignment = 3 } : memref<4xi32>, vector<4xi32>
   return
diff --git a/mlir/test/Dialect/Vector/transform-vector.mlir b/mlir/test/Dialect/Vector/transform-vector.mlir
index 4b38db79bff3e..524a4f429211b 100644
--- a/mlir/test/Dialect/Vector/transform-vector.mlir
+++ b/mlir/test/Dialect/Vector/transform-vector.mlir
@@ -121,6 +121,13 @@ func.func @arith_to_outerproduct_trans_rhs_f32(%lhs: vector<16xf32>, %rhs: vecto
   return %mul: vector<8x16xf32>
 }
 
+// See https://github.com/llvm/llvm-project/pull/152957
+// CHECK-LABEL: func.func @negative_non_vector_type
+func.func @negative_non_vector_type(%lhs: f32, %rhs: f32) -> f32 {
+  %mul = arith.mulf %lhs, %rhs : f32
+  return %mul: f32
+}
+
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module_op: !transform.any_op {transform.readonly}) {
     %func = transform.structured.match ops{["func.func"]} in %module_op : (!transform.any_op) -> !transform.any_op
diff --git a/mlir/test/Dialect/Vector/vector-from-elements-lowering.mlir b/mlir/test/Dialect/Vector/vector-from-elements-lowering.mlir
new file mode 100644
index 0000000000000..8fac608ed5692
--- /dev/null
+++ b/mlir/test/Dialect/Vector/vector-from-elements-lowering.mlir
@@ -0,0 +1,45 @@
+// RUN: mlir-opt %s -test-unroll-vector-from-elements | FileCheck %s --check-prefix=CHECK-UNROLL
+
+//===----------------------------------------------------------------------===//
+// Test UnrollFromElements.
+//===----------------------------------------------------------------------===//
+
+// CHECK-UNROLL-LABEL: @unroll_from_elements_2d
+// CHECK-UNROLL-SAME:    (%[[ARG0:.*]]: f32, %[[ARG1:.*]]: f32, %[[ARG2:.*]]: f32, %[[ARG3:.*]]: f32)
+// CHECK-UNROLL-NEXT:    %[[UNDEF_RES:.*]] = ub.poison : vector<2x2xf32>
+// CHECK-UNROLL-NEXT:    %[[VEC_0:.*]] = vector.from_elements %[[ARG0]], %[[ARG1]] : vector<2xf32>
+// CHECK-UNROLL-NEXT:    %[[RES_0:.*]] = vector.insert %[[VEC_0]], %[[UNDEF_RES]] [0] : vector<2xf32> into vector<2x2xf32>
+// CHECK-UNROLL-NEXT:    %[[VEC_1:.*]] = vector.from_elements %[[ARG2]], %[[ARG3]] : vector<2xf32>
+// CHECK-UNROLL-NEXT:    %[[RES_1:.*]] = vector.insert %[[VEC_1]], %[[RES_0]] [1] : vector<2xf32> into vector<2x2xf32>
+// CHECK-UNROLL-NEXT:    return %[[RES_1]] : vector<2x2xf32>
+func.func @unroll_from_elements_2d(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32) -> vector<2x2xf32> {
+  %0 = vector.from_elements %arg0, %arg1, %arg2, %arg3 : vector<2x2xf32>
+  return %0 : vector<2x2xf32>
+}
+
+// CHECK-UNROLL-LABEL: @unroll_from_elements_3d
+// CHECK-UNROLL-SAME:    (%[[ARG0:.*]]: f32, %[[ARG1:.*]]: f32, %[[ARG2:.*]]: f32, %[[ARG3:.*]]: f32)
+// CHECK-UNROLL-NEXT:    %[[UNDEF_RES:.*]] = ub.poison : vector<2x1x2xf32>
+// CHECK-UNROLL-NEXT:    %[[UNDEF_RANK_2:.*]] = ub.poison : vector<1x2xf32>
+// CHECK-UNROLL-NEXT:    %[[VEC_0:.*]] = vector.from_elements %[[ARG0]], %[[ARG1]] : vector<2xf32>
+// CHECK-UNROLL-NEXT:    %[[RANK_2_0:.*]] = vector.insert %[[VEC_0]], %[[UNDEF_RANK_2]] [0] : vector<2xf32> into vector<1x2xf32>
+// CHECK-UNROLL-NEXT:    %[[RES_0:.*]] = vector.insert %[[RANK_2_0]], %[[UNDEF_RES]] [0] : vector<1x2xf32> into vector<2x1x2xf32>
+// CHECK-UNROLL-NEXT:    %[[VEC_1:.*]] = vector.from_elements %[[ARG2]], %[[ARG3]] : vector<2xf32>
+// CHECK-UNROLL-NEXT:    %[[RANK_2_1:.*]] = vector.insert %[[VEC_1]], %[[UNDEF_RANK_2]] [0] : vector<2xf32> into vector<1x2xf32>
+// CHECK-UNROLL-NEXT:    %[[RES_1:.*]] = vector.insert %[[RANK_2_1]], %[[RES_0]] [1] : vector<1x2xf32> into vector<2x1x2xf32>
+// CHECK-UNROLL-NEXT:    return %[[RES_1]] : vector<2x1x2xf32>
+func.func @unroll_from_elements_3d(%arg0: f32, %arg1: f32, %arg2: f32, %arg3: f32) -> vector<2x1x2xf32> {
+  %0 = vector.from_elements %arg0, %arg1, %arg2, %arg3 : vector<2x1x2xf32>
+  return %0 : vector<2x1x2xf32>
+}
+
+// 1-D vector.from_elements should not be unrolled.
+
+// CHECK-UNROLL-LABEL: @negative_unroll_from_elements_1d
+// CHECK-UNROLL-SAME:    (%[[ARG0:.*]]: f32, %[[ARG1:.*]]: f32)
+// CHECK-UNROLL-NEXT:         %[[RES:.*]] = vector.from_elements %[[ARG0]], %[[ARG1]] : vector<2xf32>
+// CHECK-UNROLL-NEXT:    return %[[RES]] : vector<2xf32>
+func.func @negative_unroll_from_elements_1d(%arg0: f32, %arg1: f32) -> vector<2xf32> {
+  %0 = vector.from_elements %arg0, %arg1 : vector<2xf32>
+  return %0 : vector<2xf32>
+}
diff --git a/mlir/test/Dialect/Vector/vector-gather-lowering.mlir b/mlir/test/Dialect/Vector/vector-gather-lowering.mlir
index 5be267c1be984..0e1bad62ce763 100644
--- a/mlir/test/Dialect/Vector/vector-gather-lowering.mlir
+++ b/mlir/test/Dialect/Vector/vector-gather-lowering.mlir
@@ -81,7 +81,7 @@ func.func @gather_memref_1d_i32_index(%base: memref<?xf32>, %v: vector<2xi32>, %
 // CHECK-SAME:      %[[PASS:.*]]: vector<2x[3]xf32>
 // CHECK:         %[[C0:.*]] = arith.constant 0 : index
 // CHECK:         %[[C1:.*]] = arith.constant 1 : index
-// CHECK:         %[[INIT:.*]] = arith.constant dense<0.000000e+00> : vector<2x[3]xf32>
+// CHECK:         %[[INIT:.*]] = ub.poison : vector<2x[3]xf32>
 // CHECK:         %[[IDXVEC0:.*]] = vector.extract %[[IDXVEC]][0] : vector<[3]xindex> from vector<2x[3]xindex>
 // CHECK:         %[[MASK0:.*]] = vector.extract %[[MASK]][0] : vector<[3]xi1> from vector<2x[3]xi1>
 // CHECK:         %[[PASS0:.*]] = vector.extract %[[PASS]][0] : vector<[3]xf32> from vector<2x[3]xf32>
@@ -198,7 +198,7 @@ func.func @gather_memref_non_unit_stride_read_more_than_1_element(%base: memref<
 // CANON-NOT:     scf.if
 // CANON:         tensor.extract
 // CANON:         tensor.extract
-// CANON:         [[FINAL:%.+]] = vector.insert %{{.+}}, %{{.+}} [1] : f32 into vector<2xf32>
+// CANON:         [[FINAL:%.+]] = vector.from_elements %{{.+}}, %{{.+}} : vector<2xf32>
 // CANON-NEXT:    return [[FINAL]] : vector<2xf32>
 func.func @gather_tensor_1d_all_set(%base: tensor<?xf32>, %v: vector<2xindex>, %pass_thru: vector<2xf32>) -> vector<2xf32> {
   %mask = arith.constant dense <true> : vector<2xi1>
diff --git a/mlir/test/Dialect/Vector/vector-sink.mlir b/mlir/test/Dialect/Vector/vector-sink.mlir
index ef881ba05a416..577b06df42929 100644
--- a/mlir/test/Dialect/Vector/vector-sink.mlir
+++ b/mlir/test/Dialect/Vector/vector-sink.mlir
@@ -40,7 +40,7 @@ func.func @broadcast_scalar_with_bcast_scalable(%arg1: index, %arg2: index) -> v
 // CHECK:           %[[BCAST:.*]] = vector.broadcast %[[ADD]] : index to vector<1x4xindex>
 // CHECK:           return %[[BCAST]] : vector<1x4xindex>
 func.func @broadcast_scalar_with_bcast_and_splat(%arg1: index, %arg2: index) -> vector<1x4xindex> {
-  %0 = vector.splat %arg1 : vector<1x4xindex>
+  %0 = vector.broadcast %arg1 : index to vector<1x4xindex>
   %1 = vector.broadcast %arg2 : index to vector<1x4xindex>
   %2 = arith.addi %0, %1 : vector<1x4xindex>
   return %2 : vector<1x4xindex>
@@ -53,7 +53,7 @@ func.func @broadcast_scalar_with_bcast_and_splat(%arg1: index, %arg2: index) ->
 // CHECK:           %[[BCAST:.*]] = vector.broadcast %[[ADD]] : index to vector<1x[4]xindex>
 // CHECK:           return %[[BCAST]] : vector<1x[4]xindex>
 func.func @broadcast_scalar_with_bcast_and_splat_scalable(%arg1: index, %arg2: index) -> vector<1x[4]xindex> {
-  %0 = vector.splat %arg1 : vector<1x[4]xindex>
+  %0 = vector.broadcast %arg1 : index to vector<1x[4]xindex>
   %1 = vector.broadcast %arg2 : index to vector<1x[4]xindex>
   %2 = arith.addi %0, %1 : vector<1x[4]xindex>
   return %2 : vector<1x[4]xindex>
@@ -94,12 +94,12 @@ func.func @broadcast_vector_scalable(%arg1: vector<[4]xf32>, %arg2: vector<[4]xf
 // CHECK-LABEL:   func.func @broadcast_scalar_and_vec(
 // CHECK-SAME:       %[[ARG1:.*]]: index,
 // CHECK-SAME:       %[[ARG2:.*]]: vector<4xindex>) -> vector<1x4xindex> {
-// CHECK:            %[[SPLAT:.*]] = vector.splat %[[ARG1]] : vector<1x4xindex>
+// CHECK:            %[[SPLAT:.*]] = vector.broadcast %[[ARG1]] : index to vector<1x4xindex>
 // CHECK:            %[[BCAST:.*]] = vector.broadcast %[[ARG2]] : vector<4xindex> to vector<1x4xindex>
 // CHECK:            %[[ADD:.*]] = arith.addi %[[SPLAT]], %[[BCAST]] : vector<1x4xindex>
 // CHECK:            return %[[ADD]] : vector<1x4xindex>
 func.func @broadcast_scalar_and_vec(%arg1: index, %arg2: vector<4xindex>) -> vector<1x4xindex> {
-  %0 = vector.splat %arg1 : vector<1x4xindex>
+  %0 = vector.broadcast %arg1 : index to vector<1x4xindex>
   %1 = vector.broadcast %arg2 : vector<4xindex> to vector<1x4xindex>
   %2 = arith.addi %0, %1 : vector<1x4xindex>
   return %2 : vector<1x4xindex>
@@ -108,12 +108,12 @@ func.func @broadcast_scalar_and_vec(%arg1: index, %arg2: vector<4xindex>) -> vec
 // CHECK-LABEL:   func.func @broadcast_scalar_and_vec_scalable(
 // CHECK-SAME:       %[[ARG1:.*]]: index,
 // CHECK-SAME:       %[[ARG2:.*]]: vector<[4]xindex>) -> vector<1x[4]xindex> {
-// CHECK:            %[[SPLAT:.*]] = vector.splat %[[ARG1]] : vector<1x[4]xindex>
+// CHECK:            %[[SPLAT:.*]] = vector.broadcast %[[ARG1]] : index to vector<1x[4]xindex>
 // CHECK:            %[[BCAST:.*]] = vector.broadcast %[[ARG2]] : vector<[4]xindex> to vector<1x[4]xindex>
 // CHECK:            %[[ADD:.*]] = arith.addi %[[SPLAT]], %[[BCAST]] : vector<1x[4]xindex>
 // CHECK:            return %[[ADD]] : vector<1x[4]xindex>
 func.func @broadcast_scalar_and_vec_scalable(%arg1: index, %arg2: vector<[4]xindex>) -> vector<1x[4]xindex> {
-  %0 = vector.splat %arg1 : vector<1x[4]xindex>
+  %0 = vector.broadcast %arg1 : index to vector<1x[4]xindex>
   %1 = vector.broadcast %arg2 : vector<[4]xindex> to vector<1x[4]xindex>
   %2 = arith.addi %0, %1 : vector<1x[4]xindex>
   return %2 : vector<1x[4]xindex>
@@ -787,7 +787,7 @@ func.func @negative_extract_load_scalable(%arg0: memref<?xf32>, %arg1: index) ->
 //  CHECK-SAME:   (%[[ARG0:.*]]: memref<?xf32>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: f32)
 func.func @store_splat(%arg0: memref<?xf32>, %arg1: index, %arg2: f32) {
 // CHECK:   memref.store %[[ARG2]], %[[ARG0]][%[[ARG1]]] : memref<?xf32>
-  %0 = vector.splat %arg2 : vector<1xf32>
+  %0 = vector.broadcast %arg2 : f32 to vector<1xf32>
   vector.store %0, %arg0[%arg1] : memref<?xf32>, vector<1xf32>
   return
 }
@@ -813,9 +813,9 @@ func.func @store_broadcast_1d_to_2d(%arg0: memref<?x?xf32>, %arg1: index, %arg2:
 // CHECK-LABEL: @negative_store_scalable
 //  CHECK-SAME:   (%[[ARG0:.*]]: memref<?xf32>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: f32)
 func.func @negative_store_scalable(%arg0: memref<?xf32>, %arg1: index, %arg2: f32) {
-// CHECK:   %[[RES:.*]] = vector.splat %[[ARG2]] : vector<[1]xf32>
+// CHECK:   %[[RES:.*]] = vector.broadcast %[[ARG2]] : f32 to vector<[1]xf32>
 // CHECK:   vector.store %[[RES]], %[[ARG0]][%[[ARG1]]] : memref<?xf32>, vector<[1]xf32>
-  %0 = vector.splat %arg2 : vector<[1]xf32>
+  %0 = vector.broadcast %arg2 : f32 to vector<[1]xf32>
   vector.store %0, %arg0[%arg1] : memref<?xf32>, vector<[1]xf32>
   return
 }
@@ -823,9 +823,9 @@ func.func @negative_store_scalable(%arg0: memref<?xf32>, %arg1: index, %arg2: f3
 // CHECK-LABEL: @negative_store_memref_of_vec
 //  CHECK-SAME:   (%[[ARG0:.*]]: memref<?xvector<1xf32>>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: f32)
 func.func @negative_store_memref_of_vec(%arg0: memref<?xvector<1xf32>>, %arg1: index, %arg2: f32) {
-// CHECK:   %[[RES:.*]] = vector.splat %[[ARG2]] : vector<1xf32>
+// CHECK:   %[[RES:.*]] = vector.broadcast %[[ARG2]] : f32 to vector<1xf32>
 // CHECK:   vector.store %[[RES]], %[[ARG0]][%[[ARG1]]] : memref<?xvector<1xf32>>, vector<1xf32>
-  %0 = vector.splat %arg2 : vector<1xf32>
+  %0 = vector.broadcast %arg2 : f32 to vector<1xf32>
   vector.store %0, %arg0[%arg1] : memref<?xvector<1xf32>>, vector<1xf32>
   return
 }
@@ -833,9 +833,9 @@ func.func @negative_store_memref_of_vec(%arg0: memref<?xvector<1xf32>>, %arg1: i
 // CHECK-LABEL: @negative_store_more_than_one_element
 //  CHECK-SAME:   (%[[ARG0:.*]]: memref<?xf32>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: f32)
 func.func @negative_store_more_than_one_element(%arg0: memref<?xf32>, %arg1: index, %arg2: f32) {
-// CHECK:   %[[RES:.*]] = vector.splat %[[ARG2]] : vector<4xf32>
+// CHECK:   %[[RES:.*]] = vector.broadcast %[[ARG2]] : f32 to vector<4xf32>
 // CHECK:   vector.store %[[RES]], %[[ARG0]][%[[ARG1]]] : memref<?xf32>, vector<4xf32>
-  %0 = vector.splat %arg2 : vector<4xf32>
+  %0 = vector.broadcast %arg2 : f32 to vector<4xf32>
   vector.store %0, %arg0[%arg1] : memref<?xf32>, vector<4xf32>
   return
 }
@@ -843,10 +843,10 @@ func.func @negative_store_more_than_one_element(%arg0: memref<?xf32>, %arg1: ind
 // CHECK-LABEL: @negative_store_no_single_use
 //  CHECK-SAME:   (%[[ARG0:.*]]: memref<?xf32>, %[[ARG1:.*]]: index, %[[ARG2:.*]]: f32)
 func.func @negative_store_no_single_use(%arg0: memref<?xf32>, %arg1: index, %arg2: f32) -> vector<1xf32> {
-// CHECK:   %[[RES:.*]] = vector.splat %[[ARG2]] : vector<1xf32>
+// CHECK:   %[[RES:.*]] = vector.broadcast %[[ARG2]] : f32 to vector<1xf32>
 // CHECK:   vector.store %[[RES]], %[[ARG0]][%[[ARG1]]] : memref<?xf32>, vector<1xf32>
 // CHECK:   return %[[RES:.*]] : vector<1xf32>
-  %0 = vector.splat %arg2 : vector<1xf32>
+  %0 = vector.broadcast %arg2 : f32 to vector<1xf32>
   vector.store %0, %arg0[%arg1] : memref<?xf32>, vector<1xf32>
   return %0 : vector<1xf32>
 }
diff --git a/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir b/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir
index 1b54d54ffbd9f..45afbffc1be48 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-to-vector-load-store.mlir
@@ -285,19 +285,19 @@ func.func @transfer_read_permutations(%mem_0 : memref<?x?xf32>, %mem_1 : memref<
   %c0 = arith.constant 0 : index
 
 // CHECK: %[[MASK0:.*]] = vector.broadcast %{{.*}} : i1 to vector<14x7xi1>
-  %mask0 = vector.splat %m : vector<14x7xi1>
+  %mask0 = vector.broadcast %m : i1 to vector<14x7xi1>
   %0 = vector.transfer_read %mem_1[%c0, %c0, %c0, %c0], %cst, %mask0 {in_bounds = [true, false, true, true], permutation_map = #map0} : memref<?x?x?x?xf32>, vector<7x14x8x16xf32>
 // CHECK: vector.transfer_read {{.*}} %[[MASK0]] {in_bounds = [false, true, true, true], permutation_map = #[[$MAP0]]} : memref<?x?x?x?xf32>, vector<14x7x8x16xf32>
 // CHECK: vector.transpose %{{.*}}, [1, 0, 2, 3] : vector<14x7x8x16xf32> to vector<7x14x8x16xf32>
 
 // CHECK: %[[MASK1:.*]] = vector.broadcast %{{.*}} : i1 to vector<16x14xi1>
-  %mask1 = vector.splat %m : vector<16x14xi1>
+  %mask1 = vector.broadcast %m : i1 to vector<16x14xi1>
   %1 = vector.transfer_read %mem_1[%c0, %c0, %c0, %c0], %cst, %mask1 {in_bounds = [true, false, true, false], permutation_map = #map1} : memref<?x?x?x?xf32>, vector<7x14x8x16xf32>
 // CHECK: vector.transfer_read {{.*}} %[[MASK1]] {in_bounds = [false, false, true, true], permutation_map = #[[$MAP0]]} : memref<?x?x?x?xf32>, vector<16x14x7x8xf32>
 // CHECK: vector.transpose %{{.*}}, [2, 1, 3, 0] : vector<16x14x7x8xf32> to vector<7x14x8x16xf32>
 
 // CHECK: %[[MASK3:.*]] = vector.broadcast %{{.*}} : i1 to vector<14x7xi1>
-  %mask2 = vector.splat %m : vector<14x7xi1>
+  %mask2 = vector.broadcast %m : i1 to vector<14x7xi1>
   %2 = vector.transfer_read %mem_1[%c0, %c0, %c0, %c0], %cst, %mask2 {in_bounds = [true, false, true, true], permutation_map = #map2} : memref<?x?x?x?xf32>, vector<7x14x8x16xf32>
 // CHECK: vector.transfer_read {{.*}} %[[MASK3]] {in_bounds = [false, true, true], permutation_map = #[[$MAP1]]} : memref<?x?x?x?xf32>, vector<14x16x7xf32>
 // CHECK: vector.broadcast %{{.*}} : vector<14x16x7xf32> to vector<8x14x16x7xf32>
@@ -337,7 +337,7 @@ func.func @transfer_write_permutations_tensor_masked(
   %c0 = arith.constant 0 : index
 
   // CHECK: %[[MASK:.*]] = vector.broadcast %[[M]] : i1 to vector<16x14x7x8xi1>
-  %mask0 = vector.splat %m : vector<16x14x7x8xi1>
+  %mask0 = vector.broadcast %m : i1 to vector<16x14x7x8xi1>
   %res = vector.transfer_write %vec, %dst[%c0, %c0, %c0, %c0], %mask0 {in_bounds = [true, false, false, true], permutation_map = affine_map<(d0, d1, d2, d3) -> (d2, d1, d3, d0)>} : vector<7x14x8x16xf32>, tensor<?x?x?x?xf32>
   // CHECK: %[[NEW_VEC0:.*]] = vector.transpose %{{.*}} [3, 1, 0, 2] : vector<7x14x8x16xf32> to vector<16x14x7x8xf32>
   // CHECK: %[[NEW_RES0:.*]] = vector.transfer_write %[[NEW_VEC0]], %[[DST]][%c0, %c0, %c0, %c0], %[[MASK]] {in_bounds = [true, false, true, false]} : vector<16x14x7x8xf32>, tensor<?x?x?x?xf32>
diff --git a/mlir/test/Dialect/Vector/vector-warp-distribute.mlir b/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
index ae8fce786ee57..4d2c964a6df3c 100644
--- a/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
+++ b/mlir/test/Dialect/Vector/vector-warp-distribute.mlir
@@ -1299,7 +1299,7 @@ func.func @vector_insert_1d_broadcast(%laneid: index, %pos: index) -> (vector<96
 //       CHECK-PROP:     %[[VEC:.*]] = "some_def"
 //       CHECK-PROP:     %[[VAL:.*]] = "another_def"
 //       CHECK-PROP:     gpu.yield %[[VEC]], %[[VAL]]
-//       CHECK-PROP:   vector.insert %[[W]]#1, %[[W]]#0 [] : f32 into vector<f32>
+//       CHECK-PROP:   vector.broadcast %[[W]]#1 : f32 to vector<f32>
 func.func @vector_insert_0d(%laneid: index) -> (vector<f32>) {
   %r = gpu.warp_execute_on_lane_0(%laneid)[32] -> (vector<f32>) {
     %0 = "some_def"() : () -> (vector<f32>)
@@ -1803,3 +1803,24 @@ func.func @warp_propagate_nd_write(%laneid: index, %dest: memref<4x1024xf32>) {
 //       CHECK-DIST-AND-PROP:   %[[IDS:.+]]:2 = affine.delinearize_index %{{.*}} into (4, 8) : index, index
 //       CHECK-DIST-AND-PROP:   %[[INNER_ID:.+]] = affine.apply #map()[%[[IDS]]#1]
 //       CHECK-DIST-AND-PROP:   vector.transfer_write %[[W]], %{{.*}}[%[[IDS]]#0, %[[INNER_ID]]] {{.*}} : vector<1x128xf32>
+
+// -----
+func.func @warp_propagate_duplicated_operands_in_yield(%laneid: index)  {
+  %r:3 = gpu.warp_execute_on_lane_0(%laneid)[32] -> (vector<1xf32>, vector<1xf32>, vector<1xf32>) {
+    %0 = "some_def"() : () -> (vector<32xf32>)
+    %1 = "some_other_def"() : () -> (vector<32xf32>)
+    %2 = math.exp %1 : vector<32xf32>
+    gpu.yield %2, %0, %0 : vector<32xf32>, vector<32xf32>, vector<32xf32>
+  }
+  "some_use"(%r#0) : (vector<1xf32>) -> ()
+  return
+}
+
+// CHECK-PROP-LABEL : func.func @warp_propagate_duplicated_operands_in_yield(
+// CHECK-PROP       :   %[[W:.*]] = gpu.warp_execute_on_lane_0(%{{.*}})[32] -> (vector<1xf32>) {
+// CHECK-PROP       :     %{{.*}} = "some_def"() : () -> vector<32xf32>
+// CHECK-PROP       :     %[[T3:.*]] = "some_other_def"() : () -> vector<32xf32>
+// CHECK-PROP       :     gpu.yield %[[T3]] : vector<32xf32>
+// CHECK-PROP       :   }
+// CHECK-PROP       :   %[T1:.*] = math.exp %[[W]] : vector<1xf32>
+// CHECK-PROP       :   "some_use"(%[[T1]]) : (vector<1xf32>) -> ()
diff --git a/mlir/test/Dialect/WasmSSA/custom_parser/if.mlir b/mlir/test/Dialect/WasmSSA/custom_parser/if.mlir
new file mode 100644
index 0000000000000..01068cbe94fc2
--- /dev/null
+++ b/mlir/test/Dialect/WasmSSA/custom_parser/if.mlir
@@ -0,0 +1,53 @@
+// RUN: mlir-opt %s | FileCheck %s
+
+// CHECK-LABEL:   wasmssa.func nested @func_0(
+// CHECK-SAME:      %[[ARG0:.*]]: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.local_get %[[ARG0]] :  ref to i32
+// CHECK:           wasmssa.if %[[VAL_0]] : {
+// CHECK:             %[[VAL_1:.*]] = wasmssa.const 5.000000e-01 : f32
+// CHECK:             wasmssa.block_return %[[VAL_1]] : f32
+// CHECK:           } "else "{
+// CHECK:             %[[VAL_2:.*]] = wasmssa.const 2.500000e-01 : f32
+// CHECK:             wasmssa.block_return %[[VAL_2]] : f32
+// CHECK:           }> ^bb1
+// CHECK:         ^bb1(%[[VAL_3:.*]]: f32):
+// CHECK:           wasmssa.return %[[VAL_3]] : f32
+wasmssa.func nested @func_0(%arg0 : !wasmssa<local ref to i32>) -> i32 {
+  %cond = wasmssa.local_get %arg0 : ref to i32
+  wasmssa.if %cond : {
+    %c0 = wasmssa.const 0.5 : f32
+    wasmssa.block_return %c0 : f32
+  } else {
+   %c1 = wasmssa.const 0.25 : f32
+   wasmssa.block_return %c1 : f32
+  } >^bb1
+ ^bb1(%retVal: f32):
+  wasmssa.return %retVal : f32
+}
+
+// CHECK-LABEL:   wasmssa.func nested @func_1(
+// CHECK-SAME:      %[[ARG0:.*]]: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.local_get %[[ARG0]] :  ref to i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.local of type i32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.const 0 : i64
+// CHECK:           wasmssa.if %[[VAL_0]] : {
+// CHECK:             %[[VAL_3:.*]] = wasmssa.const 1 : i32
+// CHECK:             wasmssa.local_set %[[VAL_1]] :  ref to i32 to %[[VAL_3]] : i32
+// CHECK:             wasmssa.block_return
+// CHECK:           } > ^bb1
+// CHECK:         ^bb1:
+// CHECK:           %[[VAL_4:.*]] = wasmssa.local_get %[[VAL_1]] :  ref to i32
+// CHECK:           wasmssa.return %[[VAL_4]] : i32
+wasmssa.func nested @func_1(%arg0 : !wasmssa<local ref to i32>) -> i32 {
+  %cond = wasmssa.local_get %arg0 : ref to i32
+  %var = wasmssa.local of type i32
+  %zero = wasmssa.const 0
+  wasmssa.if %cond : {
+    %c1 = wasmssa.const 1 : i32
+    wasmssa.local_set %var : ref to i32 to %c1 : i32
+    wasmssa.block_return
+  } >^bb1
+ ^bb1:
+  %res = wasmssa.local_get %var : ref to i32
+  wasmssa.return %res : i32
+}
diff --git a/mlir/test/Dialect/WasmSSA/custom_parser/memory.mlir b/mlir/test/Dialect/WasmSSA/custom_parser/memory.mlir
new file mode 100644
index 0000000000000..47551dbf18292
--- /dev/null
+++ b/mlir/test/Dialect/WasmSSA/custom_parser/memory.mlir
@@ -0,0 +1,7 @@
+// RUN: mlir-opt %s | FileCheck %s
+
+// CHECK:   wasmssa.memory @mem0 public !wasmssa<limit[0: 65536]>
+wasmssa.memory @mem0 public !wasmssa<limit[0:65536]>
+
+// CHECK:   wasmssa.memory @mem1 nested !wasmssa<limit[512:]>
+wasmssa.memory @mem1 !wasmssa<limit[512:]>
diff --git a/mlir/test/Dialect/WasmSSA/custom_parser/table.mlir b/mlir/test/Dialect/WasmSSA/custom_parser/table.mlir
new file mode 100644
index 0000000000000..5a874f4ac9e08
--- /dev/null
+++ b/mlir/test/Dialect/WasmSSA/custom_parser/table.mlir
@@ -0,0 +1,7 @@
+// RUN: mlir-opt %s | FileCheck %s
+
+// CHECK:   wasmssa.table @tab0 public !wasmssa<tabletype !wasmssa.externref [0: 65536]>
+wasmssa.table @tab0 public !wasmssa<tabletype !wasmssa.externref [0:65536]>
+
+// CHECK:   wasmssa.table @tab1 nested !wasmssa<tabletype !wasmssa.funcref [348:]>
+wasmssa.table @tab1 !wasmssa<tabletype !wasmssa.funcref [348:]>
diff --git a/mlir/test/Dialect/XeGPU/invalid.mlir b/mlir/test/Dialect/XeGPU/invalid.mlir
index dff3ffab39ecf..228ef69d9a478 100644
--- a/mlir/test/Dialect/XeGPU/invalid.mlir
+++ b/mlir/test/Dialect/XeGPU/invalid.mlir
@@ -52,14 +52,14 @@ func.func @create_nd_tdesc_7(%src: memref<128x128xf32>) {
 
 // -----
 func.func @create_nd_tdesc_8(%src: ui64) {
-  // expected-error@+1 {{'xegpu.create_nd_tdesc' op Expecting strides and shape to be present for integer source}}
+  // expected-error@+1 {{'xegpu.create_nd_tdesc' op expecting strides and shape to be present for integer source}}
   %1 = xegpu.create_nd_tdesc %src : ui64-> !xegpu.tensor_desc<128x128xf32>
   return
 }
 
 // -----
 func.func @create_nd_tdesc_9(%src: ui64) {
-  // expected-error@+1 {{expected mixed offsets rank to match mixed sizes rank}}
+  // expected-error@+1 {{expecting strides and shape to be present for integer source}}
   %1 = xegpu.create_nd_tdesc %src[0, 0] : ui64-> !xegpu.tensor_desc<128x128xf32>
   return
 }
@@ -149,7 +149,7 @@ func.func @subgroup_load_nd_offset_2(%src: memref<4x8x16xf16>, %x : index) {
 }
 
 // -----
-func.func @subgroup_load_nd_offset_3(%src: memref<4x8x16xf16>, %x : index) {  
+func.func @subgroup_load_nd_offset_3(%src: memref<4x8x16xf16>, %x : index) {
   %3 = xegpu.create_nd_tdesc %src: memref<4x8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
   %5 = xegpu.load_nd %3[0, 0] : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
     // expected-error@+1 {{Mismatched ranks between offsets and tensor descriptor}}
@@ -387,11 +387,44 @@ func.func @load_gather_vc_3(%src: ui64) {
 // -----
 func.func @prefetch_offset_wi_1(%src: memref<4x4xf32>) {
   %offsets = arith.constant dense<[0]> : vector<1xindex>
-  // expected-error@+1 {{Expecting the source is a 1D memref or pointer}}
+  // expected-error@+1 {{op operand #0 must be TensorDesc describing regions of interested data}}
   xegpu.prefetch %src[%offsets]: memref<4x4xf32>, vector<1xindex>
   return
 }
 
+// -----
+func.func @prefetch_offset_wi_2(%src: memref<16xf32>) {
+  %offsets = arith.constant dense<[0]> : vector<1xindex>
+  %1 = xegpu.create_tdesc %src, %offsets : memref<16xf32>, vector<1xindex>
+          -> !xegpu.tensor_desc<1x3xf32, #xegpu.scatter_tdesc_attr<chunk_size = 3>>
+  // expected-error@+1 {{offsets not allowed}}
+  xegpu.prefetch %1[%offsets]: !xegpu.tensor_desc<1x3xf32, #xegpu.scatter_tdesc_attr<chunk_size = 3>>, vector<1xindex>
+  return
+}
+
+// -----
+func.func @prefetch_offset_wi_3(%src: memref<16xf32>) {
+  // expected-error@+1 {{Expects offsets}}
+  xegpu.prefetch %src: memref<16xf32>
+  return
+}
+
+// -----
+func.func @prefetch_offset_wi_4(%src: memref<16xf32>) {
+  %offsets = arith.constant dense<[0]> : vector<1xindex>
+  // expected-error@+1 {{offset_align_byte only allowed with integer source.}}
+  xegpu.prefetch %src[%offsets] <{offset_align_byte = 4}>: memref<16xf32>, vector<1xindex>
+  return
+}
+
+// -----
+func.func @prefetch_offset_wi_5(%src: i64) {
+  %offsets = arith.constant dense<[0]> : vector<1xindex>
+  // expected-error@+1 {{offset_align_byte is required with integer source.}}
+  xegpu.prefetch %src[%offsets] : i64, vector<1xindex>
+  return
+}
+
 // -----
 func.func @load_gather_offset_sg(%src: memref<?xf16>) {
   %offsets = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
@@ -408,7 +441,7 @@ func.func @load_gather_offset_wi(%src: ui64) {
   %mask = arith.constant dense<1>: vector<1xi1>
   %offsets = arith.constant dense<[0]> : vector<1xindex>
   // expected-error@+1 {{value elements must match chunk size}}
-  %2 = xegpu.load %src[%offsets], %mask <{chunk_size = 2}> : ui64,  vector<1xindex>, vector<1xi1> -> vector<4xf32>
+  %2 = xegpu.load %src[%offsets], %mask <{chunk_size = 2}> : ui64,  vector<1xindex>, vector<1xi1> -> vector<3xf32>
   return
 }
 
@@ -417,7 +450,7 @@ func.func @store_scatter_offset_wi_1(%src: memref<?xf16>) {
   %val = arith.constant dense<2.9>: vector<4xf16>
   %offsets = arith.constant dense<[0]> : vector<1xindex>
   %mask = arith.constant dense<1>: vector<1xi1>
-  // expected-error@+1 {{value elements must match chunk size}}
+  // expected-error@+1 {{Mask should match value except the chunk size dim}}
   xegpu.store %val, %src[%offsets], %mask 
         : vector<4xf16>, memref<?xf16>, vector<1xindex>, vector<1xi1>
   return
@@ -428,18 +461,56 @@ func.func @store_scatter_offset_wi_2(%src: memref<4x4xf16>) {
   %val = arith.constant dense<2.9>: vector<4xf16>
   %offsets = arith.constant dense<[0]> : vector<1xindex>
   %mask = arith.constant dense<1>: vector<1xi1>
-  // expected-error@+1 {{Expecting the dest is a 1D memref or pointer}}
-  xegpu.store %val, %src[%offsets], %mask 
+  // expected-error@+1 {{op operand #1 must be TensorDesc describing regions of interested data}}
+  xegpu.store %val, %src[%offsets], %mask
         : vector<4xf16>, memref<4x4xf16>, vector<1xindex>, vector<1xi1>
   return
 }
 
+// -----
+func.func @store_scatter_offset_wi_3(%src: memref<16xf16>) {
+  %val = arith.constant dense<2.9>: vector<1xf16>
+  %mask = arith.constant dense<1>: vector<1xi1>
+  // expected-error@+1 {{Expects offsets}}
+  xegpu.store %val, %src, %mask
+        : vector<1xf16>, memref<16xf16>, vector<1xi1>
+  return
+}
+
+// -----
+func.func @store_scatter_offset_wi_4(%src: !xegpu.tensor_desc<1x1xf32, #xegpu.scatter_tdesc_attr<>>) {
+  %val = arith.constant dense<2.9>: vector<1xf16>
+  %offsets = arith.constant dense<[0]> : vector<1xindex>
+  %mask = arith.constant dense<1>: vector<1xi1>
+  // expected-error@+1 {{offsets not allowed}}
+  xegpu.store %val, %src[%offsets], %mask
+        : vector<1xf16>, !xegpu.tensor_desc<1x1xf32, #xegpu.scatter_tdesc_attr<>>, vector<1xindex>, vector<1xi1>
+  return
+}
+
+// -----
+func.func @load_gather_offset_wi_4(%src: !xegpu.tensor_desc<1x2xf16, #xegpu.scatter_tdesc_attr<>>) {
+  %mask = arith.constant dense<1>: vector<1xi1>
+  %offsets = arith.constant dense<[0]> : vector<1xindex>
+  // expected-error@+1 {{offsets not allowed}}
+  %2 = xegpu.load %src[%offsets], %mask <{chunk_size = 2}> : !xegpu.tensor_desc<1x2xf16, #xegpu.scatter_tdesc_attr<>>, vector<1xindex>, vector<1xi1> -> vector<2xf16>
+  return
+}
+
+// -----
+func.func @load_gather_offset_wi_3(%src: ui64) {
+  %mask = arith.constant dense<1>: vector<1xi1>
+  // expected-error@+1 {{Expects offsets}}
+  %2 = xegpu.load %src, %mask <{chunk_size = 2}> : ui64, vector<1xi1> -> vector<2xf16>
+  return
+}
+
 // -----
 func.func @load_gather_offset_wi_2(%src: ui64) {
   %mask = arith.constant dense<1>: vector<1xi1>
   %offsets = arith.constant dense<[0]> : vector<1xindex>
   // expected-error@+1 {{value elements must match chunk size}}
-  %2 = xegpu.load %src[%offsets], %mask <{chunk_size = 2}> : ui64,  vector<1xindex>, vector<1xi1> -> vector<4xf16>
+  %2 = xegpu.load %src[%offsets], %mask <{chunk_size = 2}> : ui64,  vector<1xindex>, vector<1xi1> -> vector<3xf16>
   return
 }
 
@@ -447,7 +518,7 @@ func.func @load_gather_offset_wi_2(%src: ui64) {
 func.func @load_gather_offset_wi_1(%src: memref<4x4xf32>) {
   %mask = arith.constant dense<1>: vector<1xi1>
   %offsets = arith.constant dense<[0]> : vector<1xindex>
-  // expected-error@+1 {{Expecting the source is a 1D memref or pointer}}
+  // expected-error@+1 {{op operand #0 must be TensorDesc describing regions of interested data}}
   %2 = xegpu.load %src[%offsets], %mask <{chunk_size = 2}> : memref<4x4xf32>,  vector<1xindex>, vector<1xi1> -> vector<2xf32>
   return
 }
@@ -743,3 +814,108 @@ func.func @tensor_desc_invalid_sg_data(%src: ui64, %offsets: vector<16xindex>) {
         #xegpu.layout<lane_layout = [8, 1], lane_data = [1, 2], order = [0, 1, 2]>>
   return
 }
+
+// -----
+#l = #xegpu.layout<sg_layout = [16, 1, 1], sg_data = [1, 8, 2]>
+// expected-error@+1 {{repeated dim (2) in slice attribute}}
+#s = #xegpu.slice<#l, dims = [2, 2]>
+func.func @slice_attr_repeat_dim() {
+  %offsets = arith.constant {layout_result_0 = #s} dense<0.8> : vector<16x8xindex>
+  return
+}
+
+// -----
+#l = #xegpu.layout<sg_layout = [16, 1, 1], sg_data = [1, 8, 2]>
+// expected-error@+1 {{invalid dim (3) in slice attribute}}
+#s = #xegpu.slice<#l, dims = [3]>
+func.func @slice_attr_repeat_dim() {
+  %offsets = arith.constant {layout_result_0 = #s} dense<0.8> : vector<16x8xindex>
+  return
+}
+
+// -----
+func.func @create_mem_desc_non_slm() {
+  %m = memref.alloca() {alignment = 1024} : memref<2048xi8, 1>
+  // expected-error@+1 {{operand #0 must be statically shaped memref of 8-bit signless integer values for shared memory}}
+  %mem_desc = xegpu.create_mem_desc %m : memref<2048xi8, 1> -> !xegpu.mem_desc<16x64xf16>
+  return
+}
+
+// -----
+func.func @create_mem_desc_mismatch_sizes() {
+  %m = memref.alloca() {alignment = 1024} : memref<2048xi8, 3>
+  // expected-error@+1 {{failed to verify that all of {source, mem_desc} have same size in bits}}
+  %mem_desc = xegpu.create_mem_desc %m : memref<2048xi8, 3> -> !xegpu.mem_desc<16x32xf16>
+  return
+}
+
+// -----
+func.func @load_mem_desc_mismatch_element_type(%arg0: !xegpu.mem_desc<16x64xf16>) {
+  // expected-error@+1 {{failed to verify that all of {mem_desc, res} have same element type}}
+  %data = xegpu.load_matrix %arg0[8, 8]: !xegpu.mem_desc<16x64xf16> -> vector<8x16xf32>
+  return
+}
+
+// -----
+func.func @load_mem_desc_invalid_result_size(%arg0: !xegpu.mem_desc<16x64xf16>) {
+  // expected-error@+1 {{result shape must not exceed mem_desc shape}}
+  %data = xegpu.load_matrix %arg0[8, 8]: !xegpu.mem_desc<16x64xf16> -> vector<32x16xf16>
+  return
+}
+
+// -----
+func.func @load_mem_desc_invalid_rank(%arg0: !xegpu.mem_desc<64xf16>) {
+  // expected-error@+1 {{mem_desc must be 2D}}
+  %data = xegpu.load_matrix %arg0[16]: !xegpu.mem_desc<64xf16> -> vector<16xf16>
+  return
+}
+
+// -----
+func.func @store_mem_desc_mismatch_element_type(%arg0: !xegpu.mem_desc<16x64xf16>, %arg1: vector<16x16xf32>) {
+  // expected-error@+1 {{failed to verify that all of {mem_desc, data} have same element type}}
+  xegpu.store_matrix %arg1, %arg0[8, 8] : vector<16x16xf32>, !xegpu.mem_desc<16x64xf16>
+  return
+}
+
+// -----
+func.func @store_mem_desc_invalid_data_size(%arg0: !xegpu.mem_desc<16x64xf16>, %arg1: vector<32x32xf16>) {
+  // expected-error@+1 {{data shape must not exceed mem_desc shape}}
+  xegpu.store_matrix %arg1, %arg0[8, 8] : vector<32x32xf16>, !xegpu.mem_desc<16x64xf16>
+  return
+}
+
+// -----
+func.func @store_mem_desc_invalid_rank(%arg0: !xegpu.mem_desc<64xf16>, %arg1: vector<32xf16>) {
+  // expected-error@+1 {{mem_desc must be 2D.}}
+  xegpu.store_matrix %arg1, %arg0[32] : vector<32xf16>, !xegpu.mem_desc<64xf16>
+  return
+}
+
+// -----
+func.func @mem_desc_subview_size_mismatch(%arg0: !xegpu.mem_desc<16x64xf16>) {
+  // expected-error@+1 {{result shape must not exceed source shape}}
+  %data = xegpu.mem_desc_subview %arg0[8, 8]: !xegpu.mem_desc<16x64xf16> -> !xegpu.mem_desc<32x16xf16>
+  return
+}
+
+// -----
+func.func @mem_desc_subview_layout_mismatch(%arg0: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride =[1, 16]>>) {
+  // expected-error@+1 {{result must inherit the source strides}}
+  %data = xegpu.mem_desc_subview %arg0[8, 8]: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride =[1, 16]>> -> !xegpu.mem_desc<8x16xf16>
+  return
+}
+
+// -----
+func.func @mem_desc_subview_element_type_mismatch(%arg0: !xegpu.mem_desc<16x64xf16>) {
+  // expected-error@+1 {{failed to verify that all of {src, res} have same element type}}
+  %data = xegpu.mem_desc_subview %arg0[8, 8]: !xegpu.mem_desc<16x64xf16> -> !xegpu.mem_desc<8x16xf32, #xegpu.mem_layout<stride =[64, 1]>>
+  return
+}
+
+// -----
+func.func @mem_desc_subview_rank_mismatch(%arg0: !xegpu.mem_desc<16x64xf16>) {
+  // expected-error@+1 {{result rank must not exceed source rank}}
+  %data = xegpu.mem_desc_subview %arg0[8, 8]: !xegpu.mem_desc<16x64xf16> -> !xegpu.mem_desc<4x8x16xf16>
+  return
+}
+
diff --git a/mlir/test/Dialect/XeGPU/layout.mlir b/mlir/test/Dialect/XeGPU/layout.mlir
index 017dacc8d629a..e4b4e22e5cf97 100644
--- a/mlir/test/Dialect/XeGPU/layout.mlir
+++ b/mlir/test/Dialect/XeGPU/layout.mlir
@@ -50,4 +50,27 @@ gpu.func @convert_layout_wg(%a: vector<32x64xf16>) {
   gpu.return
 }
 
+gpu.func @slice_attr() {
+  //CHECK: arith.constant {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [16, 1, 1], sg_data = [1, 8, 2]>, dims = [2]>} dense<8> : vector<16x8xindex>
+  %cst = arith.constant {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [16, 1, 1], sg_data = [1, 8, 2]>, dims = [2]>} dense<8> : vector<16x8xindex>
+  gpu.return
+}
+
+gpu.func @nested_slice_attr() {
+  //CHECK: arith.constant {layout_result_0 = #xegpu.slice<#xegpu.slice<#xegpu.layout<sg_layout = [16, 1, 1], sg_data = [1, 8, 2]>, dims = [2]>, dims = [1]>} dense<8> : vector<16xindex>
+  %cst = arith.constant {layout_result_0 = #xegpu.slice<#xegpu.slice<#xegpu.layout<sg_layout = [16, 1, 1], sg_data = [1, 8, 2]>, dims = [2]>, dims = [1]>} dense<8> : vector<16xindex>
+  gpu.return
+}
+
+gpu.func @softmax_dim_0(%arg0: vector<256x128xf32>) -> vector<256x128xf32> {
+  %cst = arith.constant dense<0.000000e+00> : vector<128xf32>
+  %0 = math.exp %arg0 {layout_result_0 = #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32]>} : vector<256x128xf32>
+  //CHECK: vector.multi_reduction <add>, {{.*}} {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32]>, dims = [0]>} [0] : vector<256x128xf32> to vector<128xf32>
+  %1 = vector.multi_reduction <add>, %0, %cst {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32]>, dims = [0]>} [0] : vector<256x128xf32> to vector<128xf32>
+  //CHECK: vector.broadcast {{.*}} {layout_result_0 = #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32]>} : vector<128xf32> to vector<256x128xf32>
+  %2 = vector.broadcast %1 {layout_result_0 = #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32]>} : vector<128xf32> to vector<256x128xf32>
+  %3 = arith.divf %0, %2 {layout_result_0 = #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32]>} : vector<256x128xf32>
+  gpu.return %3 : vector<256x128xf32>
+}
+
 }
diff --git a/mlir/test/Dialect/XeGPU/ops.mlir b/mlir/test/Dialect/XeGPU/ops.mlir
index 6be2371d4d7b2..bb379024a34d7 100644
--- a/mlir/test/Dialect/XeGPU/ops.mlir
+++ b/mlir/test/Dialect/XeGPU/ops.mlir
@@ -62,28 +62,28 @@ gpu.func @create_nd_tdesc_7(%src: memref<8x24x32x48x64xf32>) {
 }
 
 
-// CHECK: gpu.func @test_create_nd_tdesc_7(%[[arg0:.*]]: ui64, %[[arg1:.*]]: index, %[[arg2:.*]]: index, %[[arg3:.*]]: index, %[[arg4:.*]]: index, %[[arg5:.*]]: memref<24x32xf32>) 
+// CHECK: gpu.func @test_create_nd_tdesc_7(%[[arg0:.*]]: ui64, %[[arg1:.*]]: index, %[[arg2:.*]]: index, %[[arg3:.*]]: index, %[[arg4:.*]]: index, %[[arg5:.*]]: memref<24x32xf32>)
 gpu.func @test_create_nd_tdesc_7(%src: ui64, %w : index, %h : index, %x : index, %y : index, %src2: memref<24x32xf32>) {
   //CHECK: %[[C:.*]] = arith.constant 1 : index
   %c1 = arith.constant 1 : index
-  
-  // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %[[arg5]][0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32>
+
+  // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %[[arg5]] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32>
   %3 = xegpu.create_nd_tdesc %src2 : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32>
- 
+
   gpu.return
 }
 
-// CHECK: gpu.func @test_create_nd_tdesc_8(%[[arg0:.*]]: ui64, %[[arg1:.*]]: index, %[[arg2:.*]]: index, %[[arg3:.*]]: index, %[[arg4:.*]]: index) 
+// CHECK: gpu.func @test_create_nd_tdesc_8(%[[arg0:.*]]: ui64, %[[arg1:.*]]: index, %[[arg2:.*]]: index, %[[arg3:.*]]: index, %[[arg4:.*]]: index)
 gpu.func @test_create_nd_tdesc_8(%src: ui64, %w : index, %h : index, %x : index, %y : index) {
-  
-  %c1 = arith.constant 1 : index   
-  // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %arg0[0, 0], shape : [%arg2, %arg1], strides : [%arg1, %c1] : ui64 -> !xegpu.tensor_desc<8x16xf32>
+
+  %c1 = arith.constant 1 : index
+  // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %arg0, shape : [%arg2, %arg1], strides : [%arg1, %c1] : ui64 -> !xegpu.tensor_desc<8x16xf32>
   %2 = xegpu.create_nd_tdesc %src, shape : [%h, %w], strides : [%w, %c1]  : ui64 -> !xegpu.tensor_desc<8x16xf32>
- 
+
   gpu.return
 }
 
-// CHECK-LABEL: func @test_create_nd_tdesc_9({{.*}}) 
+// CHECK-LABEL: func @test_create_nd_tdesc_9({{.*}})
 
 gpu.func @test_create_nd_tdesc_9(%src: memref<?x?xf16>, %w : index, %h : index, %x : index, %y : index) {
 
@@ -94,10 +94,10 @@ gpu.func @test_create_nd_tdesc_9(%src: memref<?x?xf16>, %w : index, %h : index,
   gpu.return
 }
 
-// CHECK-LABEL: func @test_create_nd_tdesc_10({{.*}}) 
-gpu.func @test_create_nd_tdesc_10(%src: memref<?x?xf16>, %w : index, %h : index, %x : index, %y : index) {  
+// CHECK-LABEL: func @test_create_nd_tdesc_10({{.*}})
+gpu.func @test_create_nd_tdesc_10(%src: memref<?x?xf16>, %w : index, %h : index, %x : index, %y : index) {
   %c1 = arith.constant 1 : index
-  // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %arg0[0, 0], shape : [%arg2, %arg1], strides : [%arg1, %c1] : memref<?x?xf16> -> !xegpu.tensor_desc<8x16xf16> 
+  // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %arg0, shape : [%arg2, %arg1], strides : [%arg1, %c1] : memref<?x?xf16> -> !xegpu.tensor_desc<8x16xf16>
   %2 = xegpu.create_nd_tdesc %src, shape:[%h, %w], strides:[%w, %c1]  : memref<?x?xf16> -> !xegpu.tensor_desc<8x16xf16>
 
   gpu.return
@@ -123,7 +123,7 @@ gpu.func @prefetch_nd_2(%src: memref<48x64xf16>) {
 
 // CHECK: gpu.func @prefetch_nd_offset_1(%[[arg0:.*]]: memref<48x64xf16>,  %arg1: index, %arg2: index) {
 gpu.func @prefetch_nd_offset_1(%src: memref<48x64xf16>, %x : index, %y : index) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %[[arg0]][0, 0] : memref<48x64xf16> -> !xegpu.tensor_desc<8x16xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %[[arg0]] : memref<48x64xf16> -> !xegpu.tensor_desc<8x16xf16>
   %1 = xegpu.create_nd_tdesc %src : memref<48x64xf16> -> !xegpu.tensor_desc<8x16xf16>
   // CHECK: xegpu.prefetch_nd %[[R0]][%arg1, %arg2] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<8x16xf16>
   xegpu.prefetch_nd %1[%x, %y] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<8x16xf16>
@@ -271,7 +271,7 @@ gpu.func @subgroup_load_nd_8(%src: memref<24x32xf32>) {
 
 // CHECK: func @subgroup_load_nd_offset_1(%[[arg0:.*]]: memref<24x32xf32>, %arg1: index, %arg2: index) {
 gpu.func @subgroup_load_nd_offset_1(%src: memref<24x32xf32>, %x : index, %y : index) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0 : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32>
   %1 = xegpu.create_nd_tdesc %src : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32>
   // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]][%arg1, %arg2] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x8xf32> -> vector<8x16xf32>
   %2 = xegpu.load_nd %1[%x, %y] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x8xf32> -> vector<8x16xf32>
@@ -290,7 +290,7 @@ gpu.func @simt_load_nd_8(%src: memref<24x32xf32>) {
 
 // CHECK: func @simt_load_nd_offset_1(%[[arg0:.*]]: memref<24x32xf32>) {
 gpu.func @simt_load_nd_offset_1(%src: memref<24x32xf32>) {
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0 : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32>
   %1 = xegpu.create_nd_tdesc %src : memref<24x32xf32> -> !xegpu.tensor_desc<16x8xf32>
   // CHECK: %[[R1:.*]] = xegpu.load_nd %[[R0]][0, 0] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x8xf32> -> vector<8xf32>
   %2 = xegpu.load_nd %1[0, 0] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x8xf32> -> vector<8xf32>
@@ -323,7 +323,7 @@ gpu.func @simt_store_nd(%src: memref<24x32xf16>) {
 gpu.func @subgroup_store_nd_2(%dst: memref<24x32xf16>, %x : index) {
   // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<32xf16>
   %1 = arith.constant dense<1.0>: vector<32xf16>
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %[[arg0]][0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %[[arg0]] : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16>
   %2 = xegpu.create_nd_tdesc %dst : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16>
   // CHECK: xegpu.store_nd %[[C]], %[[R0]][%arg1] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<32xf16>, !xegpu.tensor_desc<32xf16>
   xegpu.store_nd %1, %2[%x] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}>: vector<32xf16>, !xegpu.tensor_desc<32xf16>
@@ -356,7 +356,7 @@ gpu.func @simt_store_nd_2(%src: memref<24x32xf16>) {
 gpu.func @simt_store_nd_offset_1(%src: memref<24x32xf16>) {
   // CHECK: %[[C:.*]] = arith.constant dense<1.000000e+00> : vector<2xf16>
   %1 = arith.constant dense<1.0>: vector<2xf16>
-  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16>
+  // CHECK: %[[R0:.*]] = xegpu.create_nd_tdesc %arg0 : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16>
   %2 = xegpu.create_nd_tdesc %src : memref<24x32xf16> -> !xegpu.tensor_desc<32xf16>
   // CHECK: xegpu.store_nd %[[C]], %[[R0]][0] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}> : vector<2xf16>, !xegpu.tensor_desc<32xf16>
   xegpu.store_nd %1, %2[0] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}>: vector<2xf16>, !xegpu.tensor_desc<32xf16>
@@ -508,6 +508,34 @@ gpu.func @simt_load_3(%src: ui64) {
   gpu.return
 }
 
+// CHECK: gpu.func @simt_load_4(%[[arg0:.*]]: memref<256xf16>, %[[arg1:.*]]: vector<1xindex>, %[[arg2:.*]]: vector<1xi1>) {
+gpu.func @simt_load_4(%arg0: memref<256xf16>, %arg1: vector<1xindex>, %arg2: vector<1xi1>) {
+  // CHECK: %0 = xegpu.load %[[arg0]][%[[arg1]]], %[[arg2]] <{chunk_size = 8 : i64}> : memref<256xf16>, vector<1xindex>, vector<1xi1> -> vector<8xf16>
+  %0 = xegpu.load %arg0[%arg1], %arg2 <{chunk_size = 8 : i64}> : memref<256xf16>, vector<1xindex>, vector<1xi1> -> vector<8xf16>
+  gpu.return
+}
+
+// CHECK: gpu.func @simt_load_5(%[[arg0:.*]]: memref<256xf16>, %[[arg1:.*]]: vector<1xindex>, %[[arg2:.*]]: vector<1xi1>) {
+gpu.func @simt_load_5(%arg0: memref<256xf16>, %arg1: vector<1xindex>, %arg2: vector<1xi1>) {
+  // CHECK: %0 = xegpu.load %[[arg0]][%[[arg1]]], %[[arg2]] : memref<256xf16>, vector<1xindex>, vector<1xi1> -> vector<1xf16>
+  %0 = xegpu.load %arg0[%arg1], %arg2 : memref<256xf16>, vector<1xindex>, vector<1xi1> -> vector<1xf16>
+  gpu.return
+}
+
+// CHECK: gpu.func @simt_load_6(%[[arg0:.*]]: memref<256xf16>, %[[arg1:.*]]: index, %[[arg2:.*]]: i1) {
+gpu.func @simt_load_6(%arg0: memref<256xf16>, %arg1: index, %arg2: i1) {
+  // CHECK: %0 = xegpu.load %[[arg0]][%[[arg1]]], %[[arg2]] <{chunk_size = 8 : i64}> : memref<256xf16>, index, i1 -> vector<8xf16>
+  %0 = xegpu.load %arg0[%arg1], %arg2 <{chunk_size = 8 : i64}> : memref<256xf16>, index, i1 -> vector<8xf16>
+  gpu.return
+}
+
+// CHECK: gpu.func @simt_load_7(%[[arg0:.*]]: memref<256xf16>, %[[arg1:.*]]: index, %[[arg2:.*]]: i1) {
+gpu.func @simt_load_7(%arg0: memref<256xf16>, %arg1: index, %arg2: i1) {
+  // CHECK: %0 = xegpu.load %[[arg0]][%[[arg1]]], %[[arg2]] : memref<256xf16>, index, i1 -> f16
+  %0 = xegpu.load %arg0[%arg1], %arg2 : memref<256xf16>, index, i1 -> f16
+  gpu.return
+}
+
 // CHECK: gpu.func @subgroup_load_4(%[[arg0:.*]]: ui64) {
 gpu.func @subgroup_load_4(%src: ui64) {
   //CHECK: %[[cst:.*]] = arith.constant dense<{{.*}}> : vector<2x4xindex>
@@ -621,6 +649,34 @@ gpu.func @simt_store_3(%src: ui64) {
   gpu.return
 }
 
+// CHECK: gpu.func @simt_store_4(%[[arg0:.*]]: vector<8xf16>, %[[arg1:.*]]: memref<256xf16>, %[[arg2:.*]]: vector<1xindex>, %[[arg3:.*]]: vector<1xi1>) {
+gpu.func @simt_store_4(%arg0: vector<8xf16>, %arg1: memref<256xf16>, %arg2: vector<1xindex>, %arg3: vector<1xi1>) {
+  // CHECK: xegpu.store %[[arg0]], %[[arg1]][%[[arg2]]], %[[arg3]] <{chunk_size = 8 : i64}> : vector<8xf16>, memref<256xf16>, vector<1xindex>, vector<1xi1>
+  xegpu.store %arg0, %arg1[%arg2], %arg3 <{chunk_size = 8 : i64}> : vector<8xf16>, memref<256xf16>, vector<1xindex>, vector<1xi1>
+  gpu.return
+}
+
+// CHECK: gpu.func @simt_store_5(%[[arg0:.*]]: vector<8xf16>, %[[arg1:.*]]: memref<256xf16>, %[[arg2:.*]]: index, %[[arg3:.*]]: i1) {
+gpu.func @simt_store_5(%arg0: vector<8xf16>, %arg1: memref<256xf16>, %arg2: index, %arg3: i1) {
+  // CHECK: xegpu.store %[[arg0]], %[[arg1]][%[[arg2]]], %[[arg3]] <{chunk_size = 8 : i64}> : vector<8xf16>, memref<256xf16>, index, i1
+  xegpu.store %arg0, %arg1[%arg2], %arg3 <{chunk_size = 8 : i64}> : vector<8xf16>, memref<256xf16>, index, i1
+  gpu.return
+}
+
+// CHECK: gpu.func @simt_store_6(%[[arg0:.*]]: vector<1xf16>, %[[arg1:.*]]: memref<256xf16>, %[[arg2:.*]]: vector<1xindex>, %[[arg3:.*]]: vector<1xi1>) {
+gpu.func @simt_store_6(%arg0: vector<1xf16>, %arg1: memref<256xf16>, %arg2: vector<1xindex>, %arg3: vector<1xi1>) {
+  // CHECK: xegpu.store %[[arg0]], %[[arg1]][%[[arg2]]], %[[arg3]] : vector<1xf16>, memref<256xf16>, vector<1xindex>, vector<1xi1>
+  xegpu.store %arg0, %arg1[%arg2], %arg3 : vector<1xf16>, memref<256xf16>, vector<1xindex>, vector<1xi1>
+  gpu.return
+}
+
+// CHECK: gpu.func @simt_store_7(%[[arg0:.*]]: f16, %[[arg1:.*]]: memref<256xf16>, %[[arg2:.*]]: index, %[[arg3:.*]]: i1) {
+gpu.func @simt_store_7(%arg0: f16, %arg1: memref<256xf16>, %arg2: index, %arg3: i1) {
+  // CHECK: xegpu.store %[[arg0]], %[[arg1]][%[[arg2]]], %[[arg3]] : f16, memref<256xf16>, index, i1
+  xegpu.store %arg0, %arg1[%arg2], %arg3 : f16, memref<256xf16>, index, i1
+  gpu.return
+}
+
 // CHECK: gpu.func @subgroup_store_4(%[[arg0:.*]]: ui64) {
 gpu.func @subgroup_store_4(%src: ui64) {
   //CHECK: %[[cst:.*]] = arith.constant dense<{{.*}}> : vector<2x4xindex>
@@ -662,8 +718,8 @@ gpu.func @prefetch(%src: ui64) {
 gpu.func @prefetch_offset(%src: ui64) {
   //CHECK: %[[cst:.*]] = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
   %0 = arith.constant dense<[0, 8, 16, 24]> : vector<4xindex>
-  // CHECK: xegpu.prefetch %[[arg0]][%cst] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : ui64, vector<4xindex>
-  xegpu.prefetch %src[%0] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: ui64, vector<4xindex>
+  // CHECK: xegpu.prefetch %[[arg0]][%cst] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, offset_align_byte = 2 : i64}> : ui64, vector<4xindex>
+  xegpu.prefetch %src[%0] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, offset_align_byte = 2}>: ui64, vector<4xindex>
   gpu.return
 }
 
@@ -751,4 +807,72 @@ gpu.func @fence() {
   gpu.return
 }
 
+// CHECK-LABEL: gpu.func @create_mem_desc({{.*}}) {
+gpu.func @create_mem_desc() {
+  //CHECK: [[alloc:%.+]] = memref.alloca() {alignment = 1024 : i64} : memref<2048xi8, 3>
+  //CHECK: [[mdesc:%.+]] = xegpu.create_mem_desc [[alloc]] : memref<2048xi8, 3> -> !xegpu.mem_desc<16x64xf16>
+  %m = memref.alloca() {alignment = 1024} : memref<2048xi8, 3>
+  %mem_desc = xegpu.create_mem_desc %m : memref<2048xi8, 3> -> !xegpu.mem_desc<16x64xf16>
+  gpu.return
+}
+
+// CHECK-LABEL: gpu.func @create_mem_desc_with_stride({{.*}}) {
+gpu.func @create_mem_desc_with_stride() {
+  //CHECK: [[alloc:%.+]] = memref.alloca() {alignment = 1024 : i64} : memref<2048xi8, 3>
+  //CHECK: [[mdesc:%.+]] = xegpu.create_mem_desc [[alloc]] : memref<2048xi8, 3> -> !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>
+  %m = memref.alloca() {alignment = 1024} : memref<2048xi8, 3>
+  %mem_desc = xegpu.create_mem_desc %m : memref<2048xi8, 3> -> !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>
+  gpu.return
+}
+
+// CHECK: gpu.func @load_mem_desc([[ARG0:%.+]]: !xegpu.mem_desc<16x64xf16>)
+gpu.func @load_mem_desc(%arg0: !xegpu.mem_desc<16x64xf16>) {
+  // CHECK: xegpu.load_matrix [[ARG0]][8, 8] : !xegpu.mem_desc<16x64xf16> -> vector<8x16xf16>
+  %data = xegpu.load_matrix %arg0[8, 8]: !xegpu.mem_desc<16x64xf16> -> vector<8x16xf16>
+  gpu.return
+}
+
+// CHECK: gpu.func @load_mem_desc_with_stride(%arg0: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>)
+gpu.func @load_mem_desc_with_stride(%arg0: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>) {
+  // CHECK: xegpu.load_matrix [[ARG0]][8, 8] : !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>> -> vector<8x16xf16>
+  %data = xegpu.load_matrix %arg0[8, 8]: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>> -> vector<8x16xf16>
+  gpu.return
+}
+
+
+// CHECK: gpu.func @store_mem_desc([[ARG0:%.+]]: !xegpu.mem_desc<16x64xf16>, [[ARG1:%.+]]: vector<16x16xf16>)
+gpu.func @store_mem_desc(%arg0: !xegpu.mem_desc<16x64xf16>, %arg1: vector<16x16xf16>) {
+  // CHECK: xegpu.store_matrix [[ARG1]], [[ARG0]][8, 8] : vector<16x16xf16>, !xegpu.mem_desc<16x64xf16>
+  xegpu.store_matrix %arg1, %arg0[8, 8]: vector<16x16xf16>, !xegpu.mem_desc<16x64xf16>
+  gpu.return
+}
+
+// CHECK: gpu.func @store_mem_desc_with_stride([[ARG0:%.+]]: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>, [[ARG1:%.+]]: vector<16x16xf16>)
+gpu.func @store_mem_desc_with_stride(%arg0: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>, %arg1: vector<16x16xf16>) {
+  // CHECK: xegpu.store_matrix [[ARG1]], [[ARG0]][0, 8] : vector<16x16xf16>, !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>
+  xegpu.store_matrix %arg1, %arg0[0, 8]: vector<16x16xf16>, !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>
+  gpu.return
+}
+
+// CHECK: gpu.func @mem_desc_subview([[ARG0:%.+]]: !xegpu.mem_desc<16x64xf16>)
+gpu.func @mem_desc_subview(%arg0: !xegpu.mem_desc<16x64xf16>) {
+  //CHECK: xegpu.mem_desc_subview [[ARG0]][8, 8] : !xegpu.mem_desc<16x64xf16> -> !xegpu.mem_desc<8x16xf16, #xegpu.mem_layout<stride = [64, 1]>>
+  %data = xegpu.mem_desc_subview %arg0[8, 8]: !xegpu.mem_desc<16x64xf16> -> !xegpu.mem_desc<8x16xf16, #xegpu.mem_layout<stride = [64, 1]>>
+  gpu.return
+}
+
+// CHECK: gpu.func @mem_desc_subview_lower_rank([[ARG0:%.+]]: !xegpu.mem_desc<16x64xf16>)
+gpu.func @mem_desc_subview_lower_rank(%arg0: !xegpu.mem_desc<16x64xf16>) {
+  //CHECK: xegpu.mem_desc_subview [[ARG0]][8, 8] : !xegpu.mem_desc<16x64xf16> -> !xegpu.mem_desc<16xf16, #xegpu.mem_layout<stride = [64, 1]>>
+  %data = xegpu.mem_desc_subview %arg0[8, 8]: !xegpu.mem_desc<16x64xf16> -> !xegpu.mem_desc<16xf16, #xegpu.mem_layout<stride = [64, 1]>>
+  gpu.return
+}
+
+// CHECK: gpu.func @mem_desc_subview_with_stride([[ARG0:%.+]]: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>)
+gpu.func @mem_desc_subview_with_stride(%arg0: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>>) {
+  //CHECK: xegpu.mem_desc_subview [[ARG0]][8, 8] : !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>> -> !xegpu.mem_desc<8x16xf16, #xegpu.mem_layout<stride = [1, 16]>>
+  %data = xegpu.mem_desc_subview %arg0[8, 8]: !xegpu.mem_desc<16x64xf16, #xegpu.mem_layout<stride = [1, 16]>> -> !xegpu.mem_desc<8x16xf16, #xegpu.mem_layout<stride = [1, 16]>>
+  gpu.return
+}
+
 }
diff --git a/mlir/test/Dialect/XeGPU/xegpu-attr-interface.mlir b/mlir/test/Dialect/XeGPU/xegpu-attr-interface.mlir
new file mode 100644
index 0000000000000..547c7355e00c6
--- /dev/null
+++ b/mlir/test/Dialect/XeGPU/xegpu-attr-interface.mlir
@@ -0,0 +1,37 @@
+// RUN: mlir-opt --test-xegpu-layout-interface --cse -split-input-file %s | FileCheck %s
+
+//CHECk: #map = affine_map<()[s0] -> (s0 floordiv 8)>
+gpu.module @test {
+  gpu.func @slice_attr() -> vector<128xindex> {
+    //CHECK: [[sgId:%.+]] = gpu.subgroup_id : index
+    //CHECK: [[IDY:%.+]] = affine.apply #map()[[[sgId]]]
+    //CHECK: [[c32:%.+]] = arith.constant 32 : index
+    //CHECK: [[LOCALY:%.+]] = index.mul [[IDY]], [[c32]]
+    //CHECK: [[c0:%.+]] = arith.constant 0 : index
+    //CHECK: [[Y:%.+]] = arith.addi [[LOCALY]], [[c0]] : index
+    //CHECK: [[c128:%.+]] = arith.constant 128 : index
+    //CHECK: [[MODY:%.+]] = index.remu [[Y]], [[c128]]
+    //CHECK: [[BASE:%.+]] = vector.step : vector<32xindex>
+    //CHECK: [[CAST:%.+]] = vector.broadcast [[MODY]] : index to vector<32xindex>
+    //CHECK: [[ADD:%.+]] = arith.addi [[BASE]], [[CAST]] : vector<32xindex>
+    %step = vector.step {layout_result_0 = #xegpu.slice<#xegpu.layout<sg_layout = [4, 8], sg_data = [32, 32]>, dims = [1]>}: vector<128xindex>
+    gpu.return %step : vector<128xindex>
+  }
+
+  gpu.func @nested_slice_attr() -> vector<128xindex> {
+    //CHECK: [[sgId:%.+]] = gpu.subgroup_id : index
+    //CHECK: [[IDY:%.+]] = affine.apply #map()[[[sgId]]]
+    //CHECK: [[c32:%.+]] = arith.constant 32 : index
+    //CHECK: [[LOCALY:%.+]] = index.mul [[IDY]], [[c32]]
+    //CHECK: [[c0:%.+]] = arith.constant 0 : index
+    //CHECK: [[Y:%.+]] = arith.addi [[LOCALY]], [[c0]] : index
+    //CHECK: [[c128:%.+]] = arith.constant 128 : index
+    //CHECK: [[MODY:%.+]] = index.remu [[Y]], [[c128]]
+    //CHECK: [[BASE:%.+]] = vector.step : vector<32xindex>
+    //CHECK: [[CAST:%.+]] = vector.broadcast [[MODY]] : index to vector<32xindex>
+    //CHECK: [[ADD:%.+]] = arith.addi [[BASE]], [[CAST]] : vector<32xindex>
+    %0 = vector.step {layout_result_0 = #xegpu.slice<#xegpu.slice<#xegpu.layout<sg_layout = [4, 8, 1], sg_data = [32, 32, 1]>, dims = [2]>, dims = [1]>} : vector<128xindex>
+    gpu.return %0 : vector<128xindex>
+  }
+
+}
\ No newline at end of file
diff --git a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-rr.mlir b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-rr.mlir
index 628a4857d1253..e5cc65e6bd3d7 100644
--- a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-rr.mlir
+++ b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-rr.mlir
@@ -1,5 +1,8 @@
 // RUN: mlir-opt --xegpu-wg-to-sg-distribute -split-input-file %s | FileCheck %s
 
+#map = affine_map<()[s0] -> (s0 floordiv 4)>
+#map1 = affine_map<()[s0] -> (s0 mod 4)>
+
 gpu.module @test_round_robin_assignment {
   // CHECK-LABEL: create_nd_tdesc
   // CHECK-SAME: %[[ARG_0:.*]]: memref<256x128xf32>
@@ -12,6 +15,30 @@ gpu.module @test_round_robin_assignment {
       gpu.return
     }
 
+  // CHECK-LABEL: create_nd_tdesc_with_shared_data
+  // CHECK-SAME: [[ARG_0:%.*]]: memref<256x128xf32>
+  gpu.func @create_nd_tdesc_with_shared_data(%src: memref<256x128xf32>) {
+    //CHECK: [[sgId:%.+]] = gpu.subgroup_id : index
+    //CHECK: [[IdY:%.+]] = affine.apply #map()[[[sgId]]]
+    //CHECK: [[IdX:%.+]] = affine.apply #map1()[[[sgId]]]
+    //CHECK: [[C16:%.+]] = arith.constant 16 : index
+    //CHECK: [[LY:%.+]] = index.mul [[IdY]], [[C16]]
+    //CHECK: [[C64:%.+]] = arith.constant 64 : index
+    //CHECK: [[LX:%.+]] = index.mul [[IdX]], [[C64]]
+    //CHECK: [[C0:%.+]] = arith.constant 0 : index
+    //CHECK: [[C0_1:%.+]] = arith.constant 0 : index
+    //CHECK: [[ADDY:%.+]] = arith.addi [[LY]], [[C0]] : index
+    //CHECK: [[ADDX:%.+]] = arith.addi [[LX]], [[C0_1]] : index
+    //CHECK: [[C128:%.+]] = arith.constant 128 : index
+    //CHECK: [[offY:%.+]] = index.remu [[ADDY]], [[C128]]
+    //CHECK: [[C64_2:%.+]] = arith.constant 64 : index
+    //CHECK: [[offX:%.+]] = index.remu [[ADDX]], [[C64_2]]
+    //CHECK: xegpu.create_nd_tdesc [[ARG_0]][[[offY]], [[offX]]] : memref<256x128xf32> -> !xegpu.tensor_desc<16x64xf32>
+    %tdesc = xegpu.create_nd_tdesc %src[0, 0] : memref<256x128xf32>
+      -> !xegpu.tensor_desc<128x64xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 64]>>
+    gpu.return
+  }
+
   // CHECK-LABEL: load_nd_tdesc
   // CHECK-SAME: %[[ARG_0:.*]]: memref<256x128xf32>
   gpu.func @load_nd_tdesc(%src: memref<256x128xf32>) {
diff --git a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-unify-ops-rr.mlir b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-unify-ops-rr.mlir
new file mode 100644
index 0000000000000..6ff7a94d678a3
--- /dev/null
+++ b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-unify-ops-rr.mlir
@@ -0,0 +1,85 @@
+// RUN: mlir-opt --xegpu-wg-to-sg-distribute -split-input-file %s | FileCheck %s
+
+gpu.module @test_distribution {
+  // CHECK-LABEL: create_nd_tdesc_no_offset
+  // CHECK-SAME: %[[ARG_0:.*]]: memref<256x128xf32>
+  gpu.func @create_nd_tdesc_no_offset(%src: memref<256x128xf32>) {
+      // CHECK-COUNT-4: xegpu.create_nd_tdesc %[[ARG_0]] : memref<256x128xf32>
+      // CHECK-SAME: -> !xegpu.tensor_desc<16x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+      // CHECK-NOT: xegpu.create_nd_tdesc
+      %tdesc = xegpu.create_nd_tdesc %src: memref<256x128xf32>
+        -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+      gpu.return
+  }
+
+  // CHECK-LABEL: load_nd_tdesc_with_offset
+  gpu.func @load_nd_tdesc_with_offset(%src: memref<256x128xf32>) {
+    // CHECK-COUNT-4: xegpu.load_nd {{%.*}}[{{%.*}}, {{%.*}}]
+    // CHECK-SAME-COUNT-4: : !xegpu.tensor_desc<16x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    // CHECK-SAME-COUNT-4: -> vector<16x16xf32>
+    // CHECK-NOT: xegpu.load_nd
+    %tdesc = xegpu.create_nd_tdesc %src: memref<256x128xf32>
+      -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+    %load =  xegpu.load_nd %tdesc[0, 0]
+      : !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+      -> vector<256x128xf32>
+    gpu.return
+  }
+
+  // CHECK-LABEL: store_nd_with_offset
+  gpu.func @store_nd_with_offset(%src: memref<256x128xf32>) {
+    // CHECK-COUNT-4: xegpu.store_nd %{{.*}}, {{%.*}}[{{%.*}}, {{%.*}}]
+    // CHECK-SAME-COUNT-4: : !xegpu.tensor_desc<16x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    // CHECK-NOT: xegpu.store_nd
+    %tdesc = xegpu.create_nd_tdesc %src: memref<256x128xf32>
+      -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+    %load =  xegpu.load_nd %tdesc[0, 0]
+      : !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+      -> vector<256x128xf32>
+    xegpu.store_nd %load, %tdesc[0, 0]
+      : vector<256x128xf32>, !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+
+  // CHECK-LABEL: prefetch_nd_tdesc_with_offset
+  // CHECK-SAME: %[[ARG_0:.*]]: memref<256x128xf32>
+  gpu.func @prefetch_nd_tdesc_with_offset(%src: memref<256x128xf32>) {
+    // CHECK-COUNT-4: xegpu.prefetch_nd {{%.*}}[{{%.*}}, {{%.*}}]
+    // CHECK-SAME-COUNT-4: !xegpu.tensor_desc<256x128xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    // CHECK-NOT: xegpu.prefetch_nd
+    %tdesc = xegpu.create_nd_tdesc %src : memref<256x128xf32>
+      -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.prefetch_nd %tdesc[0, 0]
+      : !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+
+  // CHECK-LABEL: dpas
+  // CHECK-SAME: (%[[ARG_0:.*]]: memref<256x128xf16>, %[[ARG_1:.*]]: memref<128x256xf16>)
+  gpu.func @dpas(%a: memref<256x128xf16>, %b: memref<128x256xf16>) {
+    // CHECK-COUNT-4: xegpu.create_nd_tdesc %[[ARG_0]] : memref<256x128xf16>
+    // CHECK-SAME-COUNT-4: -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    // CHECK-NOT: xegpu.create_nd_tdesc
+    // CHECK-COUNT-4: xegpu.create_nd_tdesc %[[ARG_1]] : memref<128x256xf16>
+    // CHECK-SAME-COUNT-4: -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [4, 8], lane_data = [1, 1]>>
+    // CHECK-NOT: xegpu.create_nd_tdesc
+    // CHECK-COUNT-16: xegpu.dpas %{{.*}}, %{{.*}}
+    // CHECK-SAME-COUNT-16: {layout = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
+    // CHECK-SAME-COUNT-16: : vector<16x16xf16>, vector<16x16xf16> -> vector<16x16xf32>
+    // CHECK-NOT: xegpu.dpas
+    %tdesc_a = xegpu.create_nd_tdesc %a : memref<256x128xf16>
+      -> !xegpu.tensor_desc<256x128xf16, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+    %load_a =  xegpu.load_nd %tdesc_a[0, 0]
+      : !xegpu.tensor_desc<256x128xf16, #xegpu.layout<sg_layout = [8, 4], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>>
+      -> vector<256x128xf16>
+    %tdesc_b = xegpu.create_nd_tdesc %b : memref<128x256xf16>
+      -> !xegpu.tensor_desc<128x256xf16, #xegpu.layout<sg_layout = [4, 8], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [2, 1]>>
+    %load_b =  xegpu.load_nd %tdesc_b[0, 0]
+      : !xegpu.tensor_desc<128x256xf16, #xegpu.layout<sg_layout = [4, 8], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [2, 1]>>
+      -> vector<128x256xf16>
+    %dpas = xegpu.dpas %load_a, %load_b
+      {layout_result_0 =  #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>}
+      : vector<256x128xf16>, vector<128x256xf16> -> vector<256x256xf32>
+    gpu.return
+  }
+}
diff --git a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-unify-ops.mlir b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-unify-ops.mlir
new file mode 100644
index 0000000000000..32157a7911f62
--- /dev/null
+++ b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg-unify-ops.mlir
@@ -0,0 +1,324 @@
+// RUN: mlir-opt --xegpu-wg-to-sg-distribute -split-input-file %s | FileCheck %s
+
+//CHECK: #map = affine_map<()[s0] -> (s0 floordiv 4)>
+//CHECK: #map1 = affine_map<()[s0] -> (s0 mod 4)>
+gpu.module @test_distribution {
+  // CHECK-LABEL: create_nd_tdesc_no_offset
+  // CHECK-SAME: %[[ARG_0:.*]]: memref<256x128xf32>
+  gpu.func @create_nd_tdesc_no_offset(%src: memref<256x128xf32>) {
+    // CHECK: xegpu.create_nd_tdesc %[[ARG_0]] : memref<256x128xf32>
+    // CHECK-SAME: -> !xegpu.tensor_desc<32x32xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %tdesc = xegpu.create_nd_tdesc %src : memref<256x128xf32>
+        -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+      gpu.return
+  }
+
+  // CHECK-LABEL: create_nd_tdesc_with_ptr
+  // CHECK-SAME: %[[ARG_0:.*]]: ui64
+  gpu.func @create_nd_tdesc_with_ptr(%src: ui64, %w : index, %h : index, %x : index, %y : index) {
+    // CHECK: xegpu.create_nd_tdesc %[[ARG_0]], shape : [{{.*}}, {{.*}}], strides : [{{.*}}, {{.*}}] : ui64
+    // CHECK-SAME: -> !xegpu.tensor_desc<32x32xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %c1 = arith.constant 1 : index
+    %tdesc = xegpu.create_nd_tdesc %src, shape:[%h, %w], strides: [%w, %c1] : ui64
+        -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+      gpu.return
+  }
+
+  // CHECK-LABEL: load_nd_tdesc_with_offset
+  // CHECK-SAME: %[[ARG_0:.*]]: memref<256x128xf32>
+  gpu.func @load_nd_tdesc_with_offset(%src: memref<256x128xf32>) {
+    //CHECK: [[SGID:%.+]] = gpu.subgroup_id : index
+    //CHECK: [[SGIDY:%.+]] = affine.apply #map()[[[SGID]]]
+    //CHECK: [[SGIDX:%.+]] = affine.apply #map1()[[[SGID]]]
+    //CHECK: %[[LOAD:.*]] = xegpu.load_nd {{%.*}}[{{%.*}}, {{%.*}}] : !xegpu.tensor_desc<32x32xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<32x32xf32>
+    %tdesc = xegpu.create_nd_tdesc %src: memref<256x128xf32>
+      -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+    %load =  xegpu.load_nd %tdesc[0, 0]
+      : !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+      -> vector<256x128xf32>
+    gpu.return
+  }
+
+  // CHECK-LABEL: store_nd_with_offsets
+  // CHECK-SAME: %[[ARG_0:.*]]: memref<256x128xf32>
+  gpu.func @store_nd_with_offsets(%src: memref<256x128xf32>) {
+    //CHECK: [[SGID:%.+]] = gpu.subgroup_id : index
+    //CHECK: [[SGIDY:%.+]] = affine.apply #map()[[[SGID]]]
+    //CHECK: [[SGIDX:%.+]] = affine.apply #map1()[[[SGID]]]
+    //CHECK: xegpu.store_nd %{{.*}}, {{%.*}}[{{%.*}}, {{%.*}}]  : vector<32x32xf32>, !xegpu.tensor_desc<32x32xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %tdesc = xegpu.create_nd_tdesc %src: memref<256x128xf32>
+      -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+    %load =  xegpu.load_nd %tdesc[0, 0]
+      : !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+      -> vector<256x128xf32>
+    xegpu.store_nd %load, %tdesc[0, 0]
+      : vector<256x128xf32>, !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+}
+
+  // CHECK-LABEL: prefetch_nd_tdesc_with_offset
+  // CHECK-SAME: %[[ARG_0:.*]]: memref<256x128xf32>
+  gpu.func @prefetch_nd_tdesc_with_offset(%src: memref<256x128xf32>) {
+    //CHECK: [[SGID:%.+]] = gpu.subgroup_id : index
+    //CHECK: [[SGIDY:%.+]] = affine.apply #map()[[[SGID]]]
+    //CHECK: [[SGIDX:%.+]] = affine.apply #map1()[[[SGID]]]
+    //CHECK: xegpu.prefetch_nd %{{.*}}[{{%.*}}, {{%.*}}] : !xegpu.tensor_desc<32x32xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %cst0 = arith.constant 0 : index
+    %tdesc = xegpu.create_nd_tdesc %src : memref<256x128xf32>
+      -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.prefetch_nd %tdesc[%cst0, %cst0]
+      : !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+
+  // CHECK-LABEL: dpas
+  gpu.func @dpas(%a: memref<128x128xf16>, %b: memref<128x128xf16>) {
+    // CHECK: %[[DPAS:.*]] = xegpu.dpas %{{.*}}, %{{.*}} {layout_result_0 =  #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<16x128xf16>, vector<128x16xf16> -> vector<16x16xf32>
+    %tdesc_a = xegpu.create_nd_tdesc %a : memref<128x128xf16>
+      -> !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128], lane_layout = [1, 16], lane_data = [1, 1]>>
+    %load_a =  xegpu.load_nd %tdesc_a[0, 0]
+      : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128], lane_layout = [1, 16], lane_data = [1, 1]>>
+      -> vector<128x128xf16>
+    %tdesc_b = xegpu.create_nd_tdesc %b : memref<128x128xf16>
+      -> !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16], lane_layout = [1, 16], lane_data = [2, 1]>>
+    %load_b =  xegpu.load_nd %tdesc_b[0, 0]
+      : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16], lane_layout = [1, 16], lane_data = [2, 1]>>
+      -> vector<128x128xf16>
+    %dpas = xegpu.dpas %load_a, %load_b
+      {layout_result_0 =  #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>}
+      : vector<128x128xf16>, vector<128x128xf16> -> vector<128x128xf32>
+    gpu.return
+  }
+
+  // CHECK-LABEL: dpas_no_sg_data
+  gpu.func @dpas_no_sg_data(%a: memref<128x128xf16>, %b: memref<128x128xf16>) {
+    // CHECK: %[[DPAS:.*]] = xegpu.dpas %{{.*}}, %{{.*}} {layout_result_0 =  #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1], order = [1, 0]>} : vector<16x16xf16>, vector<16x16xf16> -> vector<16x16xf32>
+    %tdesc_a = xegpu.create_nd_tdesc %a : memref<128x128xf16>
+      -> !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], lane_layout = [1, 16], lane_data = [1, 1],
+      order = [1, 0]>>
+    %load_a =  xegpu.load_nd %tdesc_a[0, 0]
+      : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], lane_layout = [1, 16], lane_data = [1, 1],
+      order = [1, 0]>>
+      -> vector<128x128xf16>
+    %tdesc_b = xegpu.create_nd_tdesc %b : memref<128x128xf16>
+      -> !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], lane_layout = [1, 16], lane_data = [2, 1],
+      order = [1, 0]>>
+    %load_b =  xegpu.load_nd %tdesc_b[0, 0]
+      : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], lane_layout = [1, 16], lane_data = [2, 1],
+      order = [1, 0]>>
+      -> vector<128x128xf16>
+    %dpas = xegpu.dpas %load_a, %load_b
+      {layout_result_0 =  #xegpu.layout<sg_layout = [8, 8], lane_layout = [1, 16], lane_data = [1, 1], order = [1, 0]>}
+      : vector<128x128xf16>, vector<128x128xf16> -> vector<128x128xf32>
+    gpu.return
+  }
+
+  // CHECK-LABEL: dpas_with_no_create_nd_desc
+  gpu.func @dpas_with_no_create_nd_desc(%a: vector<256x128xf32>, %b: vector<128x256xf32>) {
+    // CHECK-NOT: vector<32x32xf32>
+    %dpas = xegpu.dpas %a, %b
+      {layout =  #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16], lane_layout = [1, 16], lane_data = [1, 1]>}
+      : vector<256x128xf32>, vector<128x256xf32> -> vector<256x256xf32>
+    gpu.return
+  }
+
+  // CHECK-LABEL: broadcast_dim1
+  // CHECK-SAME: %[[ARG_0:.*]]: memref<256x1xf32>
+  gpu.func @broadcast_dim1(%src: memref<256x1xf32>) {
+    %tdesc = xegpu.create_nd_tdesc %src : memref<256x1xf32>
+      -> !xegpu.tensor_desc<256x1xf32, #xegpu.layout<sg_layout = [8, 1], sg_data = [32, 1], lane_layout = [8, 1], lane_data = [1, 1]>>
+    %load =  xegpu.load_nd %tdesc[0, 0]
+      : !xegpu.tensor_desc<256x1xf32, #xegpu.layout<sg_layout = [8, 1], sg_data = [32, 1], lane_layout = [8, 1], lane_data = [1, 1]>>
+      -> vector<256x1xf32>
+    // CHECK: vector.broadcast {{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [8, 1], lane_data = [1, 1]>}
+    // CHECK-SAME: : vector<32x1xf32> to vector<32x32xf32>
+    %broadcast = vector.broadcast %load
+      {layout_result_0 = #xegpu.layout<sg_layout = [8, 1], sg_data = [32, 32], lane_layout = [8, 1], lane_data = [1, 1]>}
+      : vector<256x1xf32> to vector<256x32xf32>
+    gpu.return
+  }
+
+  // CHECK-LABEL: broadcast_dim0
+  // CHECK-SAME: %[[ARG_0:.*]]: memref<1x128xf32>
+  gpu.func @broadcast_dim0(%src: memref<1x128xf32>) {
+    %tdesc = xegpu.create_nd_tdesc %src : memref<1x128xf32>
+      -> !xegpu.tensor_desc<1x128xf32, #xegpu.layout<sg_layout = [1, 4], sg_data = [1, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+    %load =  xegpu.load_nd %tdesc[0, 0]
+      : !xegpu.tensor_desc<1x128xf32, #xegpu.layout<sg_layout = [1, 4], sg_data = [1, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+      -> vector<1x128xf32>
+    // CHECK: vector.broadcast {{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
+    // CHECK-SAME: : vector<1x32xf32> to vector<32x32xf32>
+    %broadcast = vector.broadcast %load
+      {layout_result_0 = #xegpu.layout<sg_layout = [1, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>}
+      : vector<1x128xf32> to vector<32x128xf32>
+    gpu.return
+  }
+
+  // CHECK-LABEL: gemm_with_load_store_offset
+  // CHECK-SAME: %[[ARG_0:.*]]: memref<1024x1024xf16>, %[[ARG_1:.*]]: memref<1024x1024xf16>, %[[ARG_2:.*]]: memref<1024x1024xf32>
+  gpu.func @gemm_with_load_store_offset(%arg0: memref<1024x1024xf16>, %arg1: memref<1024x1024xf16>, %arg2: memref<1024x1024xf32>) {
+    //CHECK: [[c0:%.+]] = arith.constant 0 : index
+    //CHECK: [[c128:%.+]] = arith.constant 128 : index
+    //CHECK: [[c1024:%.+]] = arith.constant 1024 : index
+    %c0 = arith.constant 0 : index
+    %c128 = arith.constant 128 : index
+    %c1024 = arith.constant 1024 : index
+    %block_id_x = gpu.block_id  x
+    %block_id_y = gpu.block_id  y
+    %0 = arith.muli %block_id_x, %c128 : index
+    %1 = arith.muli %block_id_y, %c128 : index
+    %2 = xegpu.create_nd_tdesc %arg2 : memref<1024x1024xf32> -> !xegpu.tensor_desc<128x128xf32, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16]>>
+    // CHECK: [[DESC_A:%.+]] = xegpu.create_nd_tdesc %[[ARG_0]] : memref<1024x1024xf16> -> !xegpu.tensor_desc<16x128xf16>
+    // CHECK: [[DESC_B:%.+]] = xegpu.create_nd_tdesc %[[ARG_1]] : memref<1024x1024xf16> -> !xegpu.tensor_desc<128x16xf16>
+    %3 = xegpu.create_nd_tdesc %arg0 : memref<1024x1024xf16> -> !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128]>>
+    %4 = xegpu.create_nd_tdesc %arg1 : memref<1024x1024xf16> -> !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16]>>
+    // load_nd with offset
+    %5 = xegpu.load_nd %2[%0, %1] : !xegpu.tensor_desc<128x128xf32, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16]>> -> vector<128x128xf32>
+    %6 = xegpu.load_nd %3[%0, %c0] : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128]>> -> vector<128x128xf16>
+    %7 = xegpu.load_nd %4[%c0, %1] : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16]>> -> vector<128x128xf16>
+    // scf.for loop
+    //      CHECK: [[scf:%.+]]:3 = scf.for [[arg3:%.+]] = [[c0]] to [[c1024]] step [[c128]]
+    // CHECK-SAME: iter_args([[arg4:%.+]] = {{.*}}, [[arg5:%.+]] = {{.*}}, [[arg6:%.+]] = {{.*}}) ->
+    // CHECK-SAME: (vector<16x128xf16>, vector<128x16xf16>, vector<16x16xf32>)
+    //      CHECK: [[c:%.+]] = xegpu.dpas [[arg4]], [[arg5]], [[arg6]] : vector<16x128xf16>, vector<128x16xf16>, vector<16x16xf32> -> vector<16x16xf32>
+    //      CHECK: [[a:%.+]] = xegpu.load_nd [[DESC_A]][{{%.*}}, {{%.*}}]  : !xegpu.tensor_desc<16x128xf16> -> vector<16x128xf16>
+    //      CHECK: [[b:%.+]] = xegpu.load_nd [[DESC_B]][{{%.*}}, {{%.*}}]  : !xegpu.tensor_desc<128x16xf16> -> vector<128x16xf16>
+    //      CHECK: scf.yield [[a]], [[b]], [[c]] : vector<16x128xf16>, vector<128x16xf16>, vector<16x16xf32>
+    %8:3 = scf.for %arg3 = %c0 to %c1024 step %c128 iter_args(%arg4 = %6, %arg5 = %7, %arg6 = %5)
+        -> (vector<128x128xf16>, vector<128x128xf16>, vector<128x128xf32>) {
+      // load_nd with offset inside loop
+      %9 = xegpu.dpas %arg4, %arg5, %arg6 {layout_result_0 = #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16]>}
+                          : vector<128x128xf16>, vector<128x128xf16>, vector<128x128xf32> -> vector<128x128xf32>
+      %10 = xegpu.load_nd %3[%arg3, %c0] : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 128]>> -> vector<128x128xf16>
+      %11 = xegpu.load_nd %4[%c0, %arg3] : !xegpu.tensor_desc<128x128xf16, #xegpu.layout<sg_layout = [8, 8], sg_data = [128, 16]>> -> vector<128x128xf16>
+      scf.yield %10, %11, %9 : vector<128x128xf16>, vector<128x128xf16>, vector<128x128xf32>
+    }
+    // store_nd with offset
+    xegpu.store_nd %8#2, %2[%0, %1] : vector<128x128xf32>, !xegpu.tensor_desc<128x128xf32, #xegpu.layout<sg_layout = [8, 8], sg_data = [16, 16]>>
+    gpu.return
+  }
+
+  // CHECK-LABEL: @subgroup_id_range
+  gpu.func @subgroup_id_range(%src: memref<256x128xf32>, %src1: memref<128x256xf32>, %src2: memref<128x64xf32>) {
+    %sg_id = gpu.subgroup_id : index
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c2 = arith.constant 2 : index
+    %c31 = arith.constant 31 : index
+    %c3 = arith.constant 3 : index
+    %cond1 = arith.cmpi sge, %sg_id, %c0 : index
+    %cond2 = arith.cmpi slt, %sg_id, %c1 : index
+    %cond = arith.andi %cond1, %cond2 : i1
+    scf.if %cond {
+        // CHECK-NOT: index.sub
+        %tdesc = xegpu.create_nd_tdesc %src : memref<256x128xf32>
+          -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [8, 4], lane_data = [1, 1]>>
+        %load =  xegpu.load_nd %tdesc[0, 0]
+          : !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [8, 4], lane_data = [1, 1]>>
+          -> vector<256x128xf32>
+    } {sg_id_range = #xegpu.range<[0, 32]>}
+    %cond3 = arith.cmpi sge, %sg_id, %c2 : index
+    %cond4 = arith.cmpi slt, %sg_id, %c31 : index
+    %cond5 = arith.andi %cond3, %cond4 : i1
+    scf.if %cond5 {
+      // CHECK: %[[SGID:.*]] = gpu.subgroup_id : index
+      // CHECK: %[[C2:.*]] = arith.constant 2 : index
+      // CHECK: %[[SUB:.*]] = index.sub %{{.*}}, %[[C2]]
+      %tdesc = xegpu.create_nd_tdesc %src2 : memref<128x64xf32>
+        -> !xegpu.tensor_desc<128x64xf32, #xegpu.layout<sg_layout = [4, 4], sg_data = [32, 16], lane_layout = [8, 4], lane_data = [1, 1]>>
+      %load =  xegpu.load_nd %tdesc[0, 0]
+        : !xegpu.tensor_desc<128x64xf32, #xegpu.layout<sg_layout = [4, 4], sg_data = [32, 16], lane_layout = [8, 4], lane_data = [1, 1]>>
+        -> vector<128x64xf32>
+      %exp = math.exp %load {layout_result_0 = #xegpu.layout<sg_layout = [4, 4], sg_data = [32, 16], lane_layout = [8, 4], lane_data = [1, 1]>} : vector<128x64xf32>
+    }{sg_id_range = #xegpu.range<[2, 18]>}
+    gpu.return
+  }
+
+  // CHECK-LABEL: @subgroup_id_range_nested_if
+  gpu.func @subgroup_id_range_nested_if(%src: memref<256x128xf32>, %src1: memref<128x64xf32>) {
+    %sg_id = gpu.subgroup_id : index
+    %c1 = arith.constant 1 : i1
+    %c3 = arith.constant 3 : index
+    %c32 = arith.constant 32 : index
+    %tdesc = xegpu.create_nd_tdesc %src : memref<256x128xf32>
+      -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [8, 4], lane_data = [1, 1]>>
+    %load =  xegpu.load_nd %tdesc[0, 0]
+      : !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [8, 4], lane_data = [1, 1]>>
+      -> vector<256x128xf32>
+    %cond1 = arith.cmpi sge, %sg_id, %c3 : index
+    %cond2 = arith.cmpi slt, %sg_id, %c32 : index
+    %cond = arith.andi %cond1, %cond2 : i1
+    scf.if %c1 {
+      scf.if %cond {
+        // CHECK: %[[SGID:.*]] = gpu.subgroup_id : index
+        // CHECK: %[[C3:.*]] = arith.constant 3 : index
+        // CHECK: %[[SUB:.*]] = index.sub %{{.*}}, %[[C3]]
+        %td = xegpu.create_nd_tdesc %src1 : memref<128x64xf32>
+          -> !xegpu.tensor_desc<128x64xf32, #xegpu.layout<sg_layout = [4, 4], sg_data = [32, 16], lane_layout = [8, 4], lane_data = [1, 1]>>
+        %ld =  xegpu.load_nd %td[0, 0]
+          : !xegpu.tensor_desc<128x64xf32, #xegpu.layout<sg_layout = [4, 4], sg_data = [32, 16], lane_layout = [8, 4], lane_data = [1, 1]>>
+          -> vector<128x64xf32>
+        %exp = math.exp %ld {layout_result_0 = #xegpu.layout<sg_layout = [4, 4], sg_data = [32, 16], lane_layout = [8, 4], lane_data = [1, 1]>} : vector<128x64xf32>
+    }
+  } {sg_id_range = #xegpu.range<[3, 19]>}
+  gpu.return
+  }
+
+  // CHECK-LABEL: distribute_load_matrix
+  // CHECK-SAME: [[arg0:%.+]]: memref<32768xi8, 3>
+  gpu.func @distribute_load_matrix(%arg0: memref<32768xi8, 3>) {
+    //CHECK: [[mdesc:%.+]] = xegpu.create_mem_desc [[arg0]] : memref<32768xi8, 3> -> !xegpu.mem_desc<64x128xf32>
+    //CHECK: [[sgid:%.+]] = gpu.subgroup_id : index
+    //CHECK: [[c2:%.+]] = arith.constant 2 : index
+    //CHECK: [[c4:%.+]] = arith.constant 4 : index
+    //CHECK: [[c4_0:%.+]] = arith.constant 4 : index
+    //CHECK: [[id_y:%.+]] = affine.apply #map()[[[sgid]]]
+    //CHECK: [[id_x:%.+]] = affine.apply #map1()[[[sgid]]]
+    //CHECK: [[c32:%.+]] = arith.constant 32 : index
+    //CHECK: [[l_off_y:%.+]] = index.mul [[id_y]], [[c32]]
+    //CHECK: [[c32_1:%.+]] = arith.constant 32 : index
+    //CHECK: [[l_off_x:%.+]] = index.mul [[id_x]], [[c32_1]]
+    //CHECK: [[c0:%.+]] = arith.constant 0 : index
+    //CHECK: [[c0_1:%.+]] = arith.constant 0 : index
+    //CHECK: [[l_off_y_0:%.+]] = arith.addi [[l_off_y]], [[c0]] : index
+    //CHECK: [[l_off_x_0:%.+]] = arith.addi [[l_off_x]], [[c0_1]] : index
+    //CHECK: [[c64:%.+]] = arith.constant 64 : index
+    //CHECK: [[off_y:%.+]] = index.remu [[l_off_y_0]], [[c64]]
+    //CHECK: [[c128:%.+]] = arith.constant 128 : index
+    //CHECK: [[off_x:%.+]] = index.remu [[l_off_x_0]], [[c128]]
+    //CHECK: xegpu.load_matrix [[mdesc]][[[off_y]], [[off_x]]] <{layout = #xegpu.layout<lane_layout = [2, 8], lane_data = [1, 1]>}>: !xegpu.mem_desc<64x128xf32>, index, index -> vector<32x32xf32>
+    %0 = xegpu.create_mem_desc %arg0 : memref<32768xi8, 3> -> !xegpu.mem_desc<64x128xf32>
+    %1 = xegpu.load_matrix %0[0, 0] <{layout = #xegpu.layout<sg_layout = [2, 4], sg_data = [32, 32], lane_layout = [2, 8], lane_data = [1, 1]>}>: !xegpu.mem_desc<64x128xf32> -> vector<64x128xf32>
+    gpu.return
+  }
+
+  //CHECK-LABEL: distribute_store_matrix
+  //CHECK-SAME: [[arg0:%.+]]: memref<32768xi8, 3>
+  gpu.func @distribute_store_matrix(%arg0 : memref<32768xi8, 3>) {
+    //CHECK: [[cst:%.+]] = arith.constant dense<1.000000e+00> : vector<32x32xf32>
+    //CHECK: [[mdesc:%.+]] = xegpu.create_mem_desc [[arg0]] : memref<32768xi8, 3> -> !xegpu.mem_desc<64x128xf32>
+    //CHECK: [[sgid:%.+]] = gpu.subgroup_id : index
+    //CHECK: [[c2:%.+]] = arith.constant 2 : index
+    //CHECK: [[c4:%.+]] = arith.constant 4 : index
+    //CHECK: [[c4_0:%.+]] = arith.constant 4 : index
+    //CHECK: [[id_y:%.+]] = affine.apply #map()[[[sgid]]]
+    //CHECK: [[id_x:%.+]] = affine.apply #map1()[[[sgid]]]
+    //CHECK: [[c32:%.+]] = arith.constant 32 : index
+    //CHECK: [[l_off_y_0:%.+]] = index.mul [[id_y]], [[c32]]
+    //CHECK: [[c32_1:%.+]] = arith.constant 32 : index
+    //CHECK: [[l_off_x_0:%.+]] = index.mul [[id_x]], [[c32_1]]
+    //CHECK: [[c0:%.+]] = arith.constant 0 : index
+    //CHECK: [[c0_2:%.+]] = arith.constant 0 : index
+    //CHECK: [[l_off_y:%.+]] = arith.addi [[l_off_y_0]], [[c0]] : index
+    //CHECK: [[l_off_x:%.+]] = arith.addi [[l_off_x_0]], [[c0_2]] : index
+    //CHECK: [[c64:%.+]] = arith.constant 64 : index
+    //CHECK: [[off_y:%.+]] = index.remu [[l_off_y]], [[c64]]
+    //CHECK: [[c128:%.+]] = arith.constant 128 : index
+    //CHECK: [[off_x:%.+]] = index.remu [[l_off_x]], [[c128]]
+    //CHECK: xegpu.store_matrix [[cst]], [[mdesc]][[[off_y]], [[off_x]]] : vector<32x32xf32>, !xegpu.mem_desc<64x128xf32>, index, index
+    %cst = arith.constant {layout_result_0 = #xegpu.layout<sg_layout = [2, 4], sg_data = [32, 32]>} dense<1.0> : vector<64x128xf32>
+    %mdesc = xegpu.create_mem_desc %arg0 : memref<32768xi8, 3> -> !xegpu.mem_desc<64x128xf32>
+    xegpu.store_matrix %cst, %mdesc[0, 0] {layout = #xegpu.layout<sg_layout = [2, 4], sg_data = [32, 32]>} : vector<64x128xf32>, !xegpu.mem_desc<64x128xf32>
+    gpu.return
+  }
+}
diff --git a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg.mlir b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg.mlir
index d4b00372bc193..f4a49da71605f 100644
--- a/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg.mlir
+++ b/mlir/test/Dialect/XeGPU/xegpu-wg-to-sg.mlir
@@ -4,34 +4,26 @@
 //CHECK: #map1 = affine_map<()[s0] -> (s0 mod 4)>
 gpu.module @test_1_1_assignment {
   // CHECK-LABEL: create_nd_tdesc
-  // CHECK-SAME: %[[ARG_0:.*]]: memref<256x128xf32>
+  // CHECK-SAME: [[ARG_0:%.*]]: memref<256x128xf32>
   gpu.func @create_nd_tdesc(%src: memref<256x128xf32>) {
-  // CHECK: %[[SGID:.*]] = gpu.subgroup_id
-  // CHECK: %[[C8:.*]] = arith.constant 8 : index
-  // CHECK: %[[C32:.*]] = arith.constant 32 : index
-  // CHECK: %[[C4:.*]] = arith.constant 4 : index
-  // CHECK: %[[C32_0:.*]] = arith.constant 32 : index
-  // CHECK: %[[C4_1:.*]] = arith.constant 4 : index
-  // CHECK: %[[DIV:.*]] = affine.apply #map()[%[[SGID]]]
-  // CHECK: %[[REM:.*]] = affine.apply #map1()[%[[SGID]]]
-  // CHECK: %[[MUL1:.*]] = index.mul %[[DIV]], %[[C32]]
-  // CHECK: %[[MUL2:.*]] = index.mul %[[REM]], %[[C32_0]]
-  // CHECK: %[[C0:.*]] = arith.constant 0 : index
-  // CHECK: %[[C256:.*]] = arith.constant 256 : index
-  // CHECK: %[[MOD:.*]] = index.remu %[[MUL1]], %[[C256]]
-  // CHECK: %[[C0_2:.*]] = arith.constant 0 : index
-  // CHECK: %[[ADD1:.*]] = index.add %[[MOD]], %[[C0_2]]
-  // CHECK: %[[C0_3:.*]] = arith.constant 0 : index
-  // CHECK: %[[C128:.*]] = arith.constant 128 : index
-  // CHECK: %[[MOD1:.*]] = index.remu %[[MUL2]], %[[C128]]
-  // CHECK: %[[C0_4:.*]] = arith.constant 0 : index
-  // CHECK: %[[ADD2:.*]] = index.add %[[MOD1]], %[[C0_4]]
-  // CHECK: %[[TDESC:.*]] = xegpu.create_nd_tdesc %[[ARG_0]][%[[ADD1]], %[[ADD2]]] : memref<256x128xf32>
-  // CHECK-SAME: -> !xegpu.tensor_desc<32x32xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
-  // CHECK: gpu.return
-  %tdesc = xegpu.create_nd_tdesc %src[0, 0] : memref<256x128xf32>
-    -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
-  gpu.return
+    //CHECK: [[SGID:%.+]] = gpu.subgroup_id : index
+    //CHECK: [[SGIDY:%.+]] = affine.apply #map()[[[SGID]]]
+    //CHECK: [[SGIDX:%.+]] = affine.apply #map1()[[[SGID]]]
+    //CHECK: [[C32:%.+]] = arith.constant 32 : index
+    //CHECK: [[LY:%.+]] = index.mul [[SGIDY]], [[C32]]
+    //CHECK: [[LX:%.+]] = index.mul [[SGIDX]], [[C32]]
+    //CHECK: [[C0:%.+]] = arith.constant 0 : index
+    //CHECK: [[C0_1:%.+]] = arith.constant 0 : index
+    //CHECK: [[UY:%.+]] = arith.addi [[LY]], [[C0]] : index
+    //CHECK: [[UX:%.+]] = arith.addi [[LX]], [[C0_1]] : index
+    //CHECK: [[C256:%.+]] = arith.constant 256 : index
+    //CHECK: [[Y:%.+]] = index.remu [[UY]], [[C256]]
+    //CHECK: [[C128:%.+]] = arith.constant 128 : index
+    //CHECK: [[X:%.+]] = index.remu [[UX]], [[C128]]
+    //CHECK: [[TDESC:%.+]] = xegpu.create_nd_tdesc [[ARG_0]][[[Y]], [[X]]] : memref<256x128xf32> -> !xegpu.tensor_desc<32x32xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %tdesc = xegpu.create_nd_tdesc %src[0, 0] : memref<256x128xf32>
+      -> !xegpu.tensor_desc<256x128xf32, #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32], lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
   }
 
   // CHECK-LABEL: load_nd_tdesc
@@ -347,7 +339,7 @@ gpu.func @dpas_no_sg_data(%a: memref<128x128xf16>, %b: memref<128x128xf16>) {
   // CHECK-LABEL: @subgroup_id_range_nested_if
   gpu.func @subgroup_id_range_nested_if(%src: memref<256x128xf32>, %src1: memref<128x64xf32>) {
     %sg_id = gpu.subgroup_id : index
-    %c1 = arith.constant 1 : i1 
+    %c1 = arith.constant 1 : i1
     %c3 = arith.constant 3 : index
     %c32 = arith.constant 32 : index
     %tdesc = xegpu.create_nd_tdesc %src[0, 0] : memref<256x128xf32>
@@ -373,4 +365,11 @@ gpu.func @dpas_no_sg_data(%a: memref<128x128xf16>, %b: memref<128x128xf16>) {
   } {sg_id_range = #xegpu.range<[3, 19]>}
   gpu.return
   }
+
+  // CHECK-LABEL: distribute_constant
+  gpu.func @distribute_constant() {
+    // CHECK: arith.constant dense<1.000000e+00> : vector<32x32xf32>
+    %cst = arith.constant {layout_result_0 = #xegpu.layout<sg_layout = [8, 4], sg_data = [32, 32]>} dense<1.0> : vector<256x128xf32>
+    gpu.return
+  }
 }
diff --git a/mlir/test/Dialect/common_folders.mlir b/mlir/test/Dialect/common_folders.mlir
new file mode 100644
index 0000000000000..92598b4937552
--- /dev/null
+++ b/mlir/test/Dialect/common_folders.mlir
@@ -0,0 +1,22 @@
+// RUN: mlir-opt %s --test-fold-type-converting-op --split-input-file | FileCheck %s
+
+// CHECK-LABEL: @test_fold_unary_op_f32_to_si32(
+func.func @test_fold_unary_op_f32_to_si32() -> tensor<4x2xsi32> {
+  // CHECK-NEXT: %[[POSITIVE_ONE:.*]] = arith.constant dense<1> : tensor<4x2xsi32>
+  // CHECK-NEXT: return %[[POSITIVE_ONE]] : tensor<4x2xsi32>
+  %operand = arith.constant dense<5.1> : tensor<4x2xf32>
+  %sign = test.sign %operand : (tensor<4x2xf32>) -> tensor<4x2xsi32>
+  return %sign : tensor<4x2xsi32>
+}
+
+// -----
+
+// CHECK-LABEL: @test_fold_binary_op_f32_to_i1(
+func.func @test_fold_binary_op_f32_to_i1() -> tensor<8xi1> {
+  // CHECK-NEXT: %[[FALSE:.*]] = arith.constant dense<false> : tensor<8xi1>
+  // CHECK-NEXT: return %[[FALSE]] : tensor<8xi1>
+  %lhs = arith.constant dense<5.1> : tensor<8xf32>
+  %rhs = arith.constant dense<4.2> : tensor<8xf32>
+  %less_than = test.less_than %lhs, %rhs : (tensor<8xf32>, tensor<8xf32>) -> tensor<8xi1>
+  return %less_than : tensor<8xi1>
+}
diff --git a/mlir/test/IR/test-walk-pattern-rewrite-driver.mlir b/mlir/test/IR/test-walk-pattern-rewrite-driver.mlir
index 02f7e60671c9b..c3063416b0360 100644
--- a/mlir/test/IR/test-walk-pattern-rewrite-driver.mlir
+++ b/mlir/test/IR/test-walk-pattern-rewrite-driver.mlir
@@ -40,12 +40,12 @@ func.func @move_before(%cond : i1) {
 }
 
 // Check that the driver handles rewriter.moveAfter. In this case, we expect
-// the moved op to be visited only once since walk uses `make_early_inc_range`.
+// the moved op to be visited twice.
 // CHECK-LABEL: func.func @move_after(
 // CHECK: scf.if
 // CHECK: }
 // CHECK: "test.move_after_parent_op"
-// CHECK: "test.any_attr_of_i32_str"() <{attr = 1 : i32}> : () -> ()
+// CHECK: "test.any_attr_of_i32_str"() <{attr = 2 : i32}> : () -> ()
 // CHECK: return
 func.func @move_after(%cond : i1) {
   scf.if %cond {
@@ -119,3 +119,23 @@ func.func @erase_nested_block() -> i32 {
   }): () -> (i32)
   return %a : i32
 }
+
+
+// CHECK-LABEL: func.func @unreachable_replace_with_new_op
+// CHECK: "test.new_op"
+// CHECK: "test.replace_with_new_op"
+// CHECK-SAME: unreachable
+// CHECK: "test.new_op"
+func.func @unreachable_replace_with_new_op() {
+  "test.br"()[^bb1] : () -> ()
+^bb1:
+  %a = "test.replace_with_new_op"() : () -> (i32)
+  "test.br"()[^end] : () -> () // Test jumping over the unreachable block is visited as well.
+^unreachable:
+  %b = "test.replace_with_new_op"() {test.unreachable} : () -> (i32)
+  return
+^end:
+  %c = "test.replace_with_new_op"() : () -> (i32)
+  return
+}
+
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
index 06a6e2279b6a7..9d043573091eb 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
@@ -9,7 +9,12 @@
 // RUN: FileCheck %s
 
 func.func @matmul_transpose_a(%A : tensor<?x?xf32>, %B : tensor<?x?xf32>, %C : tensor<?x?xf32>) {
-  %res = linalg.matmul_transpose_a ins(%A, %B: tensor<?x?xf32>, tensor<?x?xf32>)
+  %res = linalg.matmul
+            indexing_maps = [
+                    affine_map<(d0, d1, d2) -> (d2, d0)>,
+                    affine_map<(d0, d1, d2) -> (d2, d1)>,
+                    affine_map<(d0, d1, d2) -> (d0, d1)>]
+            ins(%A, %B: tensor<?x?xf32>, tensor<?x?xf32>)
                                    outs(%C: tensor<?x?xf32>) -> tensor<?x?xf32>
   %xf = tensor.cast %res : tensor<?x?xf32> to tensor<*xf32>
   call @printMemrefF32(%xf) : (tensor<*xf32>) -> ()
@@ -56,7 +61,7 @@ func.func @main() {
 
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%module : !transform.any_op {transform.readonly}) {
-    %matmul_transpose_a = transform.structured.match ops{["linalg.matmul_transpose_a"]} in %module
+    %matmul_transpose_a = transform.structured.match ops{["linalg.matmul"]} in %module
       : (!transform.any_op) -> !transform.any_op
 
     // Step 1: Tile for size [4] x [4], which corresponds to SVLs x SVLs, where
diff --git a/mlir/test/Integration/Dialect/MemRef/assume-alignment-runtime-verification.mlir b/mlir/test/Integration/Dialect/MemRef/assume-alignment-runtime-verification.mlir
index 8f74976c59773..25a338df8d790 100644
--- a/mlir/test/Integration/Dialect/MemRef/assume-alignment-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/MemRef/assume-alignment-runtime-verification.mlir
@@ -6,6 +6,15 @@
 // RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -expand-strided-metadata \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func @main() {
   // This buffer is properly aligned. There should be no error.
   // CHECK-NOT: ^ memref is not aligned to 8
diff --git a/mlir/test/Integration/Dialect/MemRef/atomic-rmw-runtime-verification.mlir b/mlir/test/Integration/Dialect/MemRef/atomic-rmw-runtime-verification.mlir
index 26c731c921356..4c6a48d577a6c 100644
--- a/mlir/test/Integration/Dialect/MemRef/atomic-rmw-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/MemRef/atomic-rmw-runtime-verification.mlir
@@ -5,6 +5,14 @@
 // RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func @store_dynamic(%memref: memref<?xf32>, %index: index) {
   %cst = arith.constant 1.0 : f32
   memref.atomic_rmw addf %cst, %memref[%index] : (f32, memref<?xf32>) -> f32
diff --git a/mlir/test/Integration/Dialect/MemRef/cast-runtime-verification.mlir b/mlir/test/Integration/Dialect/MemRef/cast-runtime-verification.mlir
index 8b6308e9c1939..1ac10306395ad 100644
--- a/mlir/test/Integration/Dialect/MemRef/cast-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/MemRef/cast-runtime-verification.mlir
@@ -1,11 +1,20 @@
 // RUN: mlir-opt %s -generate-runtime-verification \
-// RUN:     -test-cf-assert \
 // RUN:     -expand-strided-metadata \
+// RUN:     -test-cf-assert \
 // RUN:     -convert-to-llvm | \
 // RUN: mlir-runner -e main -entry-point-result=void \
 // RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -expand-strided-metadata \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func @cast_to_static_dim(%m: memref<?xf32>) -> memref<10xf32> {
   %0 = memref.cast %m : memref<?xf32> to memref<10xf32>
   return %0 : memref<10xf32>
diff --git a/mlir/test/Integration/Dialect/MemRef/copy-runtime-verification.mlir b/mlir/test/Integration/Dialect/MemRef/copy-runtime-verification.mlir
index 95b9db2832cee..be9417baf93df 100644
--- a/mlir/test/Integration/Dialect/MemRef/copy-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/MemRef/copy-runtime-verification.mlir
@@ -6,6 +6,15 @@
 // RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -expand-strided-metadata \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 // Put memref.copy in a function, otherwise the memref.cast may fold.
 func.func @memcpy_helper(%src: memref<?xf32>, %dest: memref<?xf32>) {
   memref.copy %src, %dest : memref<?xf32> to memref<?xf32>
diff --git a/mlir/test/Integration/Dialect/MemRef/dim-runtime-verification.mlir b/mlir/test/Integration/Dialect/MemRef/dim-runtime-verification.mlir
index 2e3f271743c93..ef4af62459738 100644
--- a/mlir/test/Integration/Dialect/MemRef/dim-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/MemRef/dim-runtime-verification.mlir
@@ -6,6 +6,15 @@
 // RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -expand-strided-metadata \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func @main() {
   %c4 = arith.constant 4 : index
   %alloca = memref.alloca() : memref<1xf32>
diff --git a/mlir/test/Integration/Dialect/MemRef/load-runtime-verification.mlir b/mlir/test/Integration/Dialect/MemRef/load-runtime-verification.mlir
index b87e5bdf0970c..2e42648297875 100644
--- a/mlir/test/Integration/Dialect/MemRef/load-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/MemRef/load-runtime-verification.mlir
@@ -1,12 +1,20 @@
 // RUN: mlir-opt %s -generate-runtime-verification \
-// RUN:     -test-cf-assert \
 // RUN:     -expand-strided-metadata \
-// RUN:     -lower-affine \
+// RUN:     -test-cf-assert \
 // RUN:     -convert-to-llvm | \
 // RUN: mlir-runner -e main -entry-point-result=void \
 // RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -expand-strided-metadata \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func @load(%memref: memref<1xf32>, %index: index) {
     memref.load %memref[%index] :  memref<1xf32>
     return
diff --git a/mlir/test/Integration/Dialect/MemRef/store-runtime-verification.mlir b/mlir/test/Integration/Dialect/MemRef/store-runtime-verification.mlir
index 12253fa3b5e83..dd000c6904bcb 100644
--- a/mlir/test/Integration/Dialect/MemRef/store-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/MemRef/store-runtime-verification.mlir
@@ -5,6 +5,14 @@
 // RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func @store_dynamic(%memref: memref<?xf32>, %index: index) {
   %cst = arith.constant 1.0 : f32
   memref.store %cst, %memref[%index] :  memref<?xf32>
diff --git a/mlir/test/Integration/Dialect/MemRef/subview-runtime-verification.mlir b/mlir/test/Integration/Dialect/MemRef/subview-runtime-verification.mlir
index ec7e4085f2fa5..9fbe5bc60321e 100644
--- a/mlir/test/Integration/Dialect/MemRef/subview-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/MemRef/subview-runtime-verification.mlir
@@ -1,12 +1,22 @@
 // RUN: mlir-opt %s -generate-runtime-verification \
-// RUN:     -test-cf-assert \
 // RUN:     -expand-strided-metadata \
 // RUN:     -lower-affine \
+// RUN:     -test-cf-assert \
 // RUN:     -convert-to-llvm | \
 // RUN: mlir-runner -e main -entry-point-result=void \
 // RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -expand-strided-metadata \
+// RUN:     -lower-affine \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func @subview(%memref: memref<1xf32>, %offset: index) {
     memref.subview %memref[%offset] [1] [1] : 
         memref<1xf32> to 
diff --git a/mlir/test/Integration/Dialect/Tensor/cast-runtime-verification.mlir b/mlir/test/Integration/Dialect/Tensor/cast-runtime-verification.mlir
index e4aab32d4a390..f37a6d6383c48 100644
--- a/mlir/test/Integration/Dialect/Tensor/cast-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/Tensor/cast-runtime-verification.mlir
@@ -8,6 +8,17 @@
 // RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -one-shot-bufferize="bufferize-function-boundaries" \
+// RUN:     -buffer-deallocation-pipeline=private-function-dynamic-ownership \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-scf-to-cf \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func private @cast_to_static_dim(%t: tensor<?xf32>) -> tensor<10xf32> {
   %0 = tensor.cast %t : tensor<?xf32> to tensor<10xf32>
   return %0 : tensor<10xf32>
diff --git a/mlir/test/Integration/Dialect/Tensor/dim-runtime-verification.mlir b/mlir/test/Integration/Dialect/Tensor/dim-runtime-verification.mlir
index c6d8f698b9433..e9e5c040c6488 100644
--- a/mlir/test/Integration/Dialect/Tensor/dim-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/Tensor/dim-runtime-verification.mlir
@@ -1,10 +1,20 @@
 // RUN: mlir-opt %s -generate-runtime-verification \
-// RUN:     -one-shot-bufferize \
-// RUN:     -buffer-deallocation-pipeline \
+// RUN:     -one-shot-bufferize="bufferize-function-boundaries" \
+// RUN:     -buffer-deallocation-pipeline=private-function-dynamic-ownership \
 // RUN:     -test-cf-assert \
 // RUN:     -convert-to-llvm | \
 // RUN: mlir-runner -e main -entry-point-result=void \
-// RUN:     -shared-libs=%mlir_runner_utils 2>&1 | \
+// RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -one-shot-bufferize="bufferize-function-boundaries" \
+// RUN:     -buffer-deallocation-pipeline=private-function-dynamic-ownership \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
 func.func @main() {
diff --git a/mlir/test/Integration/Dialect/Tensor/extract-runtime-verification.mlir b/mlir/test/Integration/Dialect/Tensor/extract-runtime-verification.mlir
index 8e3cab7be704d..73fcec4d7abcd 100644
--- a/mlir/test/Integration/Dialect/Tensor/extract-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/Tensor/extract-runtime-verification.mlir
@@ -8,6 +8,17 @@
 // RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -one-shot-bufferize="bufferize-function-boundaries" \
+// RUN:     -buffer-deallocation-pipeline=private-function-dynamic-ownership \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-scf-to-cf \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func @extract(%tensor: tensor<1xf32>, %index: index) {
     tensor.extract %tensor[%index] :  tensor<1xf32>
     return
diff --git a/mlir/test/Integration/Dialect/Tensor/extract_slice-runtime-verification.mlir b/mlir/test/Integration/Dialect/Tensor/extract_slice-runtime-verification.mlir
index 28f9be0fffe64..341a59e8b8102 100644
--- a/mlir/test/Integration/Dialect/Tensor/extract_slice-runtime-verification.mlir
+++ b/mlir/test/Integration/Dialect/Tensor/extract_slice-runtime-verification.mlir
@@ -8,6 +8,17 @@
 // RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
 // RUN: FileCheck %s
 
+// RUN: mlir-opt %s -generate-runtime-verification \
+// RUN:     -one-shot-bufferize="bufferize-function-boundaries" \
+// RUN:     -buffer-deallocation-pipeline=private-function-dynamic-ownership \
+// RUN:     -test-cf-assert \
+// RUN:     -convert-scf-to-cf \
+// RUN:     -convert-to-llvm="allow-pattern-rollback=0" \
+// RUN:     -reconcile-unrealized-casts | \
+// RUN: mlir-runner -e main -entry-point-result=void \
+// RUN:     -shared-libs=%tlir_runner_utils 2>&1 | \
+// RUN: FileCheck %s
+
 func.func @extract_slice(%tensor: tensor<1xf32>, %offset: index) {
     tensor.extract_slice %tensor[%offset] [1] [1] : tensor<1xf32> to tensor<1xf32>
     return
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir
index 0ee016627440f..219367a41d51a 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f32.mlir
@@ -46,7 +46,7 @@ func.func @test_outerproduct_with_accumulator_4x4xf32() {
   %c0 = arith.constant 0 : index
   %f10 = arith.constant 10.0 : f32
 
-  %acc = vector.splat %f10 : vector<[4]x[4]xf32>
+  %acc = vector.broadcast %f10 : f32 to vector<[4]x[4]xf32>
   %vector_i32 = llvm.intr.stepvector : vector<[4]xi32>
   %vector = arith.sitofp %vector_i32 : vector<[4]xi32> to vector<[4]xf32>
   %tile = vector.outerproduct %vector, %vector, %acc : vector<[4]xf32>, vector<[4]xf32>
@@ -103,7 +103,7 @@ func.func @test_masked_outerproduct_with_accumulator_4x4xf32() {
   %ones = arith.constant dense<1> : vector<[4]xi32>
   %f10 = arith.constant 10.0 : f32
 
-  %acc = vector.splat %f10 : vector<[4]x[4]xf32>
+  %acc = vector.broadcast %f10 : f32 to vector<[4]x[4]xf32>
   %step_vector = llvm.intr.stepvector : vector<[4]xi32>
   %vector_i32 = arith.addi %step_vector, %ones : vector<[4]xi32>
   %vector = arith.sitofp %vector_i32 : vector<[4]xi32> to vector<[4]xf32>
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir
index 8e812108c6055..059f24adbe721 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/outerproduct-f64.mlir
@@ -52,7 +52,7 @@ func.func @test_outerproduct_with_accumulator_2x2xf64() {
   %ones = arith.constant dense<1> : vector<[2]xi32>
   %f10 = arith.constant 10.0 : f64
 
-  %acc = vector.splat %f10 : vector<[2]x[2]xf64>
+  %acc = vector.broadcast %f10 : f64 to vector<[2]x[2]xf64>
   %step_vector = llvm.intr.stepvector : vector<[2]xi32>
   %vector_i32 = arith.addi %step_vector, %ones : vector<[2]xi32>
   %vector = arith.sitofp %vector_i32 : vector<[2]xi32> to vector<[2]xf64>
@@ -108,7 +108,7 @@ func.func @test_masked_outerproduct_with_accumulator_2x2xf64() {
   %ones = arith.constant dense<1> : vector<[2]xi32>
   %f10 = arith.constant 10.0 : f64
 
-  %acc = vector.splat %f10 : vector<[2]x[2]xf64>
+  %acc = vector.broadcast %f10 : f64 to vector<[2]x[2]xf64>
   %step_vector = llvm.intr.stepvector : vector<[2]xi32>
   %vector_i32 = arith.addi %step_vector, %ones : vector<[2]xi32>
   %vector = arith.sitofp %vector_i32 : vector<[2]xi32> to vector<[2]xf64>
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir
index c3bf379cde617..bf6900ca810c2 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/transfer-write-2d.mlir
@@ -10,7 +10,7 @@
 // Vector store.
 func.func @transfer_write_2d(%A : memref<?x?xf32>, %base1: index, %base2: index) {
   %c0 = arith.constant 0.0 : f32
-  %zero = vector.splat %c0 : vector<[4]x[4]xf32>
+  %zero = vector.broadcast %c0 : f32 to vector<[4]x[4]xf32>
   vector.transfer_write %zero, %A[%base1, %base2] {in_bounds=[true, true]} :
     vector<[4]x[4]xf32>, memref<?x?xf32>
   return
@@ -22,7 +22,7 @@ func.func @transfer_write_2d_mask(%A : memref<?x?xf32>, %base1: index, %base2: i
   %c2 = arith.constant 2 : index
   %c3 = arith.constant 3 : index
   %mask = vector.create_mask %c2, %c3 : vector<[4]x[4]xi1>
-  %zero = vector.splat %c0 : vector<[4]x[4]xf32>
+  %zero = vector.broadcast %c0 : f32 to vector<[4]x[4]xf32>
   vector.transfer_write %zero, %A[%base1, %base2], %mask {in_bounds=[true, true]} :
     vector<[4]x[4]xf32>, memref<?x?xf32>
   return
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir
index c9904328763db..192f2911a3ee2 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/contraction.mlir
@@ -106,7 +106,7 @@ func.func @matvec_i32() {
   //    val = (123 * 314) * 4 * vscale
   // so ...
   %vscale = vector.vscale
-  %vscale_v = vector.splat %vscale : vector<3xindex>
+  %vscale_v = vector.broadcast %vscale : index to vector<3xindex>
   %vscale_i32 = arith.index_cast %vscale_v : vector<3xindex> to vector<3xi32>
   %mv1_div = arith.divui %mv1, %vscale_i32 : vector<3xi32>
   // ... val / vscale = 123 * 314 * 4 = 154488
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir
index d3b1fa4bbbc37..2d8180abdfec9 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSVE/scalable-interleave.mlir
@@ -7,8 +7,8 @@
 func.func @entry() {
   %f1 = arith.constant 1.0 : f32
   %f2 = arith.constant 2.0 : f32
-  %v1 = vector.splat %f1 : vector<[4]xf32>
-  %v2 = vector.splat %f2 : vector<[4]xf32>
+  %v1 = vector.broadcast %f1 : f32 to vector<[4]xf32>
+  %v2 = vector.broadcast %f2 : f32 to vector<[4]xf32>
   vector.print %v1 : vector<[4]xf32>
   vector.print %v2 : vector<[4]xf32>
   //
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/interleave.mlir b/mlir/test/Integration/Dialect/Vector/CPU/interleave.mlir
index f812c25c0352f..740c742863919 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/interleave.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/interleave.mlir
@@ -6,8 +6,8 @@
 func.func @entry() {
   %f1 = arith.constant 1.0 : f32
   %f2 = arith.constant 2.0 : f32
-  %v1 = vector.splat %f1 : vector<2x4xf32>
-  %v2 = vector.splat %f2 : vector<2x4xf32>
+  %v1 = vector.broadcast %f1 : f32 to vector<2x4xf32>
+  %v2 = vector.broadcast %f2 : f32 to vector<2x4xf32>
   vector.print %v1 : vector<2x4xf32>
   vector.print %v2 : vector<2x4xf32>
   //
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/outerproduct-f32.mlir b/mlir/test/Integration/Dialect/Vector/CPU/outerproduct-f32.mlir
index f7e2229321c00..e25795ab5f14d 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/outerproduct-f32.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/outerproduct-f32.mlir
@@ -14,9 +14,9 @@
 !vector_type_R = vector<7xf32>
 
 func.func @vector_outerproduct_splat_8x8(%fa: f32, %fb: f32, %fc: f32) -> !vector_type_C {
-  %a = vector.splat %fa: !vector_type_A
-  %b = vector.splat %fb: !vector_type_B
-  %c = vector.splat %fc: !vector_type_C
+  %a = vector.broadcast %fa: f32 to !vector_type_A
+  %b = vector.broadcast %fb: f32 to !vector_type_B
+  %c = vector.broadcast %fc: f32 to !vector_type_C
   %d = vector.outerproduct %a, %b, %c : !vector_type_A, !vector_type_B
   return %d: !vector_type_C
 }
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/outerproduct-i64.mlir b/mlir/test/Integration/Dialect/Vector/CPU/outerproduct-i64.mlir
index a19dfa1ce818e..0675102af2759 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/outerproduct-i64.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/outerproduct-i64.mlir
@@ -14,9 +14,9 @@
 !vector_type_R = vector<7xi64>
 
 func.func @vector_outerproduct_splat_8x8(%ia: i64, %ib: i64, %ic: i64) -> !vector_type_C {
-  %a = vector.splat %ia: !vector_type_A
-  %b = vector.splat %ib: !vector_type_B
-  %c = vector.splat %ic: !vector_type_C
+  %a = vector.broadcast %ia: i64 to !vector_type_A
+  %b = vector.broadcast %ib: i64 to !vector_type_B
+  %c = vector.broadcast %ic: i64 to !vector_type_C
   %d = vector.outerproduct %a, %b, %c : !vector_type_A, !vector_type_B
   return %d: !vector_type_C
 }
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir
index 639eed49e0d20..895b8818de767 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-1d.mlir
@@ -137,7 +137,7 @@ func.func @transfer_read_1d_mask_in_bounds(
 // Non-contiguous, strided store.
 func.func @transfer_write_1d(%A : memref<?x?xf32>, %base1 : index, %base2 : index) {
   %fn1 = arith.constant -1.0 : f32
-  %vf0 = vector.splat %fn1 : vector<7xf32>
+  %vf0 = vector.broadcast %fn1 : f32 to vector<7xf32>
   vector.transfer_write %vf0, %A[%base1, %base2]
     {permutation_map = affine_map<(d0, d1) -> (d0)>}
     : vector<7xf32>, memref<?x?xf32>
@@ -147,7 +147,7 @@ func.func @transfer_write_1d(%A : memref<?x?xf32>, %base1 : index, %base2 : inde
 // Non-contiguous, strided store.
 func.func @transfer_write_1d_mask(%A : memref<?x?xf32>, %base1 : index, %base2 : index) {
   %fn1 = arith.constant -2.0 : f32
-  %vf0 = vector.splat %fn1 : vector<7xf32>
+  %vf0 = vector.broadcast %fn1 : f32 to vector<7xf32>
   %mask = arith.constant dense<[1, 0, 1, 0, 1, 1, 1]> : vector<7xi1>
   vector.transfer_write %vf0, %A[%base1, %base2], %mask
     {permutation_map = affine_map<(d0, d1) -> (d0)>}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir
index 009c1375beaab..80dff9d791f4d 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-2d.mlir
@@ -100,7 +100,7 @@ func.func @transfer_read_2d_broadcast(
 // Vector store.
 func.func @transfer_write_2d(%A : memref<?x?xf32>, %base1: index, %base2: index) {
   %fn1 = arith.constant -1.0 : f32
-  %vf0 = vector.splat %fn1 : vector<1x4xf32>
+  %vf0 = vector.broadcast %fn1 : f32 to vector<1x4xf32>
   vector.transfer_write %vf0, %A[%base1, %base2]
     {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} :
     vector<1x4xf32>, memref<?x?xf32>
@@ -111,7 +111,7 @@ func.func @transfer_write_2d(%A : memref<?x?xf32>, %base1: index, %base2: index)
 func.func @transfer_write_2d_mask(%A : memref<?x?xf32>, %base1: index, %base2: index) {
   %fn1 = arith.constant -2.0 : f32
   %mask = arith.constant dense<[[1, 0, 1, 0]]> : vector<1x4xi1>
-  %vf0 = vector.splat %fn1 : vector<1x4xf32>
+  %vf0 = vector.broadcast %fn1 : f32 to vector<1x4xf32>
   vector.transfer_write %vf0, %A[%base1, %base2], %mask
     {permutation_map = affine_map<(d0, d1) -> (d0, d1)>} :
     vector<1x4xf32>, memref<?x?xf32>
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir
index d41d9c93bc2b3..93e6a12365268 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read-3d.mlir
@@ -62,7 +62,7 @@ func.func @transfer_read_3d_transposed(%A : memref<?x?x?x?xf32>,
 func.func @transfer_write_3d(%A : memref<?x?x?x?xf32>,
                         %o: index, %a: index, %b: index, %c: index) {
   %fn1 = arith.constant -1.0 : f32
-  %vf0 = vector.splat %fn1 : vector<2x9x3xf32>
+  %vf0 = vector.broadcast %fn1 : f32 to vector<2x9x3xf32>
   vector.transfer_write %vf0, %A[%o, %a, %b, %c]
       : vector<2x9x3xf32>, memref<?x?x?x?xf32>
   return
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read.mlir
index d1a2790a7db61..18084e31c0ea0 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-read.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-read.mlir
@@ -45,7 +45,7 @@ func.func @transfer_read_mask_inbounds_4(%A : memref<?xf32>, %base: index) {
 
 func.func @transfer_write_1d(%A : memref<?xf32>, %base: index) {
   %f0 = arith.constant 0.0 : f32
-  %vf0 = vector.splat %f0 : vector<4xf32>
+  %vf0 = vector.broadcast %f0 : f32 to vector<4xf32>
   vector.transfer_write %vf0, %A[%base]
       {permutation_map = affine_map<(d0) -> (d0)>} :
     vector<4xf32>, memref<?xf32>
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/transfer-write.mlir b/mlir/test/Integration/Dialect/Vector/CPU/transfer-write.mlir
index def708103fab1..225173869edb9 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/transfer-write.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/transfer-write.mlir
@@ -5,7 +5,7 @@
 
 func.func @transfer_write16_inbounds_1d(%A : memref<?xf32>, %base: index) {
   %f = arith.constant 16.0 : f32
-  %v = vector.splat %f : vector<16xf32>
+  %v = vector.broadcast %f : f32 to vector<16xf32>
   vector.transfer_write %v, %A[%base]
     {permutation_map = affine_map<(d0) -> (d0)>, in_bounds = [true]}
     : vector<16xf32>, memref<?xf32>
@@ -14,7 +14,7 @@ func.func @transfer_write16_inbounds_1d(%A : memref<?xf32>, %base: index) {
 
 func.func @transfer_write13_1d(%A : memref<?xf32>, %base: index) {
   %f = arith.constant 13.0 : f32
-  %v = vector.splat %f : vector<13xf32>
+  %v = vector.broadcast %f : f32 to vector<13xf32>
   vector.transfer_write %v, %A[%base]
     {permutation_map = affine_map<(d0) -> (d0)>}
     : vector<13xf32>, memref<?xf32>
@@ -23,7 +23,7 @@ func.func @transfer_write13_1d(%A : memref<?xf32>, %base: index) {
 
 func.func @transfer_write17_1d(%A : memref<?xf32>, %base: index) {
   %f = arith.constant 17.0 : f32
-  %v = vector.splat %f : vector<17xf32>
+  %v = vector.broadcast %f : f32 to vector<17xf32>
   vector.transfer_write %v, %A[%base]
     {permutation_map = affine_map<(d0) -> (d0)>}
     : vector<17xf32>, memref<?xf32>
@@ -42,7 +42,7 @@ func.func @transfer_read_1d(%A : memref<?xf32>) -> vector<32xf32> {
 func.func @transfer_write_inbounds_3d(%A : memref<4x4x4xf32>) {
   %c0 = arith.constant 0: index
   %f = arith.constant 0.0 : f32
-  %v0 = vector.splat %f : vector<2x3x4xf32>
+  %v0 = vector.broadcast %f : f32 to vector<2x3x4xf32>
   %f1 = arith.constant 1.0 : f32
   %f2 = arith.constant 2.0 : f32
   %f3 = arith.constant 3.0 : f32
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/lit.local.cfg b/mlir/test/Integration/Dialect/XeVM/GPU/lit.local.cfg
new file mode 100644
index 0000000000000..d0d51c6020588
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/lit.local.cfg
@@ -0,0 +1,4 @@
+if not config.run_xevm_tests:
+    config.unsupported = True
+if not config.enable_levelzero_runner:
+    config.unsupported = True
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_dpas.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_dpas.mlir
new file mode 100644
index 0000000000000..0bd3d3f4cc409
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_dpas.mlir
@@ -0,0 +1,146 @@
+// RUN: mlir-opt %s \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(cse,func.func(gpu-async-region),xevm-attach-target,gpu.module(convert-gpu-to-llvm-spv{use-64bit-index=true},convert-xevm-to-llvm,cse))' \
+// RUN: | mlir-opt -convert-scf-to-cf -convert-cf-to-llvm -convert-vector-to-llvm -convert-arith-to-llvm \
+// RUN: | mlir-opt -gpu-to-llvm -reconcile-unrealized-casts -cse -gpu-module-to-binary \
+// RUN: | mlir-runner \
+// RUN:   --shared-libs=%mlir_levelzero_runtime \
+// RUN:   --shared-libs=%mlir_runner_utils \
+// RUN:   --shared-libs=%mlir_c_runner_utils \
+// RUN:   --entry-point-result=void \
+// RUN: | FileCheck %s
+
+module @gemm attributes {gpu.container_module} {
+  gpu.module @kernel {
+    // - Sets of `matrix_mad` intrinsics can differ based on device's *minimal* supported sub-group size.
+    //   The *minimum supported* sub-group size should be used to call `matrix_mad` intrinsics.
+    // https://registry.khronos.org/OpenCL/extensions/intel/cl_intel_subgroup_matrix_multiply_accumulate.html
+
+    gpu.func @block_dpas(%a: !llvm.ptr<1>, %b: !llvm.ptr<1>, %c: !llvm.ptr<1>) kernel {
+      %base_width_a = arith.constant 32 : i32
+      %base_height_a = arith.constant 8 : i32
+      %base_pitch_a = arith.constant 32 : i32
+      %x = arith.constant 0 : i32
+      %y = arith.constant 0 : i32
+      %loaded_a = xevm.blockload2d %a, %base_width_a, %base_height_a, %base_pitch_a, %x, %y
+          <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=8 : i32, v_blocks=1 : i32,
+            transpose=false, pack_register=false}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi16>
+
+      %base_width_b = arith.constant 32 : i32
+      %base_height_b = arith.constant 16 : i32
+      %base_pitch_b = arith.constant 32 : i32
+      %loaded_b1 = xevm.blockload2d %b, %base_width_b, %base_height_b, %base_pitch_b, %x, %y
+          <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=16 : i32, v_blocks=1 : i32,
+            transpose=false, pack_register=false}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16>
+      %loaded_b_casted = vector.bitcast %loaded_b1 : vector<16xi16> to vector<8xi32>
+
+      %base_width_c = arith.constant 64 : i32
+      %base_height_c = arith.constant 8 : i32
+      %base_pitch_c = arith.constant 64 : i32
+      %loaded_c = xevm.blockload2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y
+          <{elem_size_in_bits=32 : i32, tile_width=16 : i32, tile_height=8 : i32, v_blocks=1 : i32,
+            transpose=false, pack_register=false}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32>
+
+      %loaded_c_casted = vector.bitcast %loaded_c : vector<8xi32> to vector<8xf32>
+      %c_result = xevm.mma %loaded_a, %loaded_b_casted, %loaded_c_casted
+          {shape=<m=8, n=16, k=16>, types=<d=f32, a=f16, b=f16, c=f32>}
+          : (vector<8xi16>, vector<8xi32>, vector<8xf32>) -> vector<8xf32>
+      %c_result_casted = vector.bitcast %c_result : vector<8xf32> to vector<8xi32>
+
+      xevm.blockstore2d %c, %base_width_c, %base_height_c, %base_pitch_c, %x, %y, %c_result_casted
+          <{elem_size_in_bits=32 : i32, tile_width=16 : i32, tile_height=8 : i32}>
+          : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+      gpu.return
+    }
+  }
+
+  func.func @test(%a : memref<8x16xf16>, %b : memref<16x16xf16>, %c : memref<8x16xf32>) -> memref<8x16xf32> attributes {llvm.emit_c_interface} {
+    %c1 = arith.constant 1 : index
+    %c16 = arith.constant 16 : index
+
+    %memref_a = gpu.alloc() : memref<8x16xf16>
+    gpu.memcpy %memref_a, %a : memref<8x16xf16>, memref<8x16xf16>
+    %a_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_a : memref<8x16xf16> -> index
+    %a_ptr_as_i64 = arith.index_cast %a_ptr_as_idx : index to i64
+    %a_ptr = llvm.inttoptr %a_ptr_as_i64 : i64 to !llvm.ptr
+    %a_ptr_casted = llvm.addrspacecast %a_ptr : !llvm.ptr to !llvm.ptr<1>
+
+    %memref_b = gpu.alloc() : memref<16x16xf16>
+    gpu.memcpy %memref_b, %b : memref<16x16xf16>, memref<16x16xf16>
+    %b_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_b : memref<16x16xf16> -> index
+    %b_ptr_as_i64 = arith.index_cast %b_ptr_as_idx : index to i64
+    %b_ptr = llvm.inttoptr %b_ptr_as_i64 : i64 to !llvm.ptr
+    %b_ptr_casted = llvm.addrspacecast %b_ptr : !llvm.ptr to !llvm.ptr<1>
+
+    %memref_c = gpu.alloc() : memref<8x16xf32>
+    gpu.memcpy %memref_c, %c : memref<8x16xf32>, memref<8x16xf32>
+    %c_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_c : memref<8x16xf32> -> index
+    %c_ptr_as_i64 = arith.index_cast %c_ptr_as_idx : index to i64
+    %c_ptr = llvm.inttoptr %c_ptr_as_i64 : i64 to !llvm.ptr
+    %c_ptr_casted = llvm.addrspacecast %c_ptr : !llvm.ptr to !llvm.ptr<1>
+
+    gpu.launch_func @kernel::@block_dpas blocks in (%c1, %c1, %c1) threads in (%c16, %c1, %c1)
+        args(%a_ptr_casted : !llvm.ptr<1>, %b_ptr_casted : !llvm.ptr<1>, %c_ptr_casted : !llvm.ptr<1>)
+    gpu.dealloc %memref_a : memref<8x16xf16>
+    gpu.dealloc %memref_b : memref<16x16xf16>
+    %res = memref.alloc() : memref<8x16xf32>
+    gpu.memcpy %res, %memref_c : memref<8x16xf32>, memref<8x16xf32>
+    gpu.dealloc %memref_c : memref<8x16xf32>
+    return %res : memref<8x16xf32>
+  }
+
+  func.func @main() attributes {llvm.emit_c_interface} {
+    %A = memref.alloc() : memref<8x16xf16>
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c8 = arith.constant 8 : index
+    %c16 = arith.constant 16 : index
+
+    scf.for %i = %c0 to %c8 step %c1 {
+      scf.for %j = %c0 to %c16 step %c1 {
+        %row_idx = arith.index_cast %i : index to i32
+        %row = arith.sitofp %row_idx : i32 to f16
+        memref.store %row, %A[%i, %j] : memref<8x16xf16>
+      }
+    }
+    %B = memref.alloc() : memref<16x16xf16>
+    scf.for %i = %c0 to %c16 step %c1 {
+      scf.for %j = %c0 to %c16 step %c1 {
+        %col_idx = arith.index_cast %j : index to i32
+        %col = arith.sitofp %col_idx : i32 to f16
+        memref.store %col, %B[%i, %j] : memref<16x16xf16>
+      }
+    }
+
+    %C = memref.alloc() : memref<8x16xf32>
+    %c0_f16 = arith.constant 0.0 : f32
+    scf.for %i = %c0 to %c8 step %c1 {
+      scf.for %j = %c0 to %c16 step %c1 {
+        memref.store %c0_f16, %C[%i, %j] : memref<8x16xf32>
+      }
+    }
+
+    %C_res = call @test(%A, %B, %C) : (memref<8x16xf16>, memref<16x16xf16>, memref<8x16xf32>) -> memref<8x16xf32>
+    %C_cast = memref.cast %C_res : memref<8x16xf32> to memref<*xf32>
+    %A_cast = memref.cast %A : memref<8x16xf16> to memref<*xf16>
+    call @printMemrefF32(%C_cast) : (memref<*xf32>) -> ()
+
+    // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+    // CHECK-NEXT: [0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0]
+    // CHECK-NEXT: [0,   16,   32,   48,   64,   80,   96,   112,   128,   144,   160,   176,   192,   208,   224,   240]
+    // CHECK-NEXT: [0,   32,   64,   96,   128,   160,   192,   224,   256,   288,   320,   352,   384,   416,   448,   480]
+    // CHECK-NEXT: [0,   48,   96,   144,   192,   240,   288,   336,   384,   432,   480,   528,   576,   624,   672,   720]
+    // CHECK-NEXT: [0,   64,   128,   192,   256,   320,   384,   448,   512,   576,   640,   704,   768,   832,   896,   960]
+    // CHECK-NEXT: [0,   80,   160,   240,   320,   400,   480,   560,   640,   720,   800,   880,   960,   1040,   1120,   1200]
+    // CHECK-NEXT: [0,   96,   192,   288,   384,   480,   576,   672,   768,   864,   960,   1056,   1152,   1248,   1344,   1440]
+    // CHECK-NEXT: [0,   112,   224,   336,   448,   560,   672,   784,   896,   1008,   1120,   1232,   1344,   1456,   1568,   1680]
+
+    memref.dealloc %A : memref<8x16xf16>
+    memref.dealloc %B : memref<16x16xf16>
+    memref.dealloc %C : memref<8x16xf32>
+    memref.dealloc %C_res : memref<8x16xf32>
+    return
+  }
+  func.func private @printMemrefF16(%ptr : memref<*xf16>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefF32(%ptr : memref<*xf32>) attributes { llvm.emit_c_interface }
+
+}
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_load_store.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_load_store.mlir
new file mode 100644
index 0000000000000..cea05b8709d72
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_load_store.mlir
@@ -0,0 +1,109 @@
+// RUN: mlir-opt %s \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(cse,func.func(gpu-async-region),xevm-attach-target,gpu.module(convert-gpu-to-llvm-spv{use-64bit-index=true},convert-xevm-to-llvm,cse))' \
+// RUN: | mlir-opt -convert-scf-to-cf -convert-cf-to-llvm -convert-vector-to-llvm -convert-arith-to-llvm \
+// RUN: | mlir-opt -gpu-to-llvm -reconcile-unrealized-casts -cse -gpu-module-to-binary \
+// RUN: | mlir-runner \
+// RUN:   --shared-libs=%mlir_levelzero_runtime \
+// RUN:   --shared-libs=%mlir_runner_utils \
+// RUN:   --shared-libs=%mlir_c_runner_utils \
+// RUN:   --entry-point-result=void \
+// RUN: | FileCheck %s
+
+module @gemm attributes {gpu.container_module} {
+
+  gpu.module @kernel {
+    // - `cl_intel_subgroups` block load/store intrinsics operate at the *maximum* sub-group size,
+    //     regardless of the active sub-group size. Make sure `clGetKernelSubGroupInfo` meets your expectations.
+    // - The attribute `intel_reqd_sub_group_size` establishes the maximum sub-group size for a kernel.
+    //
+    // Note: launching 16 threads without explicit `intel_reqd_sub_group_size = 16` may still use
+    //       the default sub-group size of 32.
+    //
+    // https://registry.khronos.org/OpenCL/extensions/intel/cl_intel_required_subgroup_size.html
+    // https://registry.khronos.org/OpenCL/extensions/intel/cl_intel_subgroups.html
+
+    gpu.func @block_load_store(%src: !llvm.ptr<1>, %dst: !llvm.ptr<1>) kernel  {
+      %base_width = arith.constant 64 : i32 // bytewidth of the block
+      %base_height = arith.constant 8 : i32 // number of rows
+      %base_pitch = arith.constant 64 : i32 // bytewidth of the base row
+      %x = arith.constant 0 : i32
+      %y = arith.constant 0 : i32
+      // If `intel_reqd_sub_group_size = 16` is not set, the default (32) is used and this `blockload2d`
+      // would only load 4 elements into vector<8xi32>
+      %loaded = xevm.blockload2d %src, %base_width, %base_height, %base_pitch, %x, %y
+          <{elem_size_in_bits=32 : i32, tile_width=16 : i32, tile_height=8 : i32, v_blocks=1 : i32,
+            transpose=false, pack_register=false}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32>
+      %loaded_f32 = vector.bitcast %loaded : vector<8xi32> to vector<8xf32>
+      %c0 = arith.constant 0 : index
+      %thread_x = gpu.thread_id x
+      %thread_x_i64 = arith.index_cast %thread_x : index to i64
+      %thread_x_i32 = llvm.trunc %thread_x_i64 : i64 to i32
+      %thread_x_f32 = arith.sitofp %thread_x_i32 : i32 to f32
+      %loaded_f32_modified = vector.insert %thread_x_f32, %loaded_f32[%c0] : f32 into vector<8xf32>
+      %loaded_modified = vector.bitcast %loaded_f32_modified : vector<8xf32> to vector<8xi32>
+      xevm.blockstore2d %dst, %base_width, %base_height, %base_pitch, %x, %y, %loaded_modified
+          <{elem_size_in_bits=32 : i32, tile_width=16 : i32, tile_height=8 : i32}>
+          : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+      gpu.return
+    }
+  }
+
+  func.func @test(%src : memref<8x16xf32>) -> memref<8x16xf32> attributes {llvm.emit_c_interface} {
+    %c1 = arith.constant 1 : index
+    %c16 = arith.constant 16 : index // Multiple of the *maximum sub-group size* (see `intel_reqd_sub_group_size`)
+    %memref_src = gpu.alloc() : memref<8x16xf32>
+    gpu.memcpy %memref_src, %src : memref<8x16xf32>, memref<8x16xf32>
+    %src_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_src : memref<8x16xf32> -> index
+    %src_ptr_as_i64 = arith.index_cast %src_ptr_as_idx : index to i64
+    %src_ptr = llvm.inttoptr %src_ptr_as_i64 : i64 to !llvm.ptr
+    %src_ptr_casted = llvm.addrspacecast %src_ptr : !llvm.ptr to !llvm.ptr<1>
+
+    %memref_dst = gpu.alloc() : memref<8x16xf32>
+    %dst_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_dst : memref<8x16xf32> -> index
+    %dst_ptr_as_i64 = arith.index_cast %dst_ptr_as_idx : index to i64
+    %dst_ptr = llvm.inttoptr %dst_ptr_as_i64 : i64 to !llvm.ptr
+    %dst_ptr_casted = llvm.addrspacecast %dst_ptr : !llvm.ptr to !llvm.ptr<1>
+
+    gpu.launch_func @kernel::@block_load_store blocks in (%c1, %c1, %c1) threads in (%c16, %c1, %c1)
+        args(%src_ptr_casted : !llvm.ptr<1>, %dst_ptr_casted : !llvm.ptr<1>)
+    gpu.dealloc %memref_src : memref<8x16xf32>
+    %dst = memref.alloc() : memref<8x16xf32>
+    gpu.memcpy %dst, %memref_dst : memref<8x16xf32>, memref<8x16xf32>
+    gpu.dealloc %memref_dst : memref<8x16xf32>
+    return %dst : memref<8x16xf32>
+  }
+
+  func.func @main() attributes {llvm.emit_c_interface} {
+    %A = memref.alloc() : memref<8x16xf32>
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c8 = arith.constant 8 : index
+    %c16 = arith.constant 16 : index
+    %c11_f32 = arith.constant 11.11 : f32
+    scf.for %i = %c0 to %c8 step %c1 {
+      scf.for %j = %c0 to %c16 step %c1 {
+        memref.store %c11_f32, %A[%i, %j] : memref<8x16xf32>
+      }
+    }
+    %B = call @test(%A) : (memref<8x16xf32>) -> memref<8x16xf32>
+    %B_cast = memref.cast %B : memref<8x16xf32> to memref<*xf32>
+    %A_cast = memref.cast %A : memref<8x16xf32> to memref<*xf32>
+    call @printMemrefF32(%A_cast) : (memref<*xf32>) -> ()
+    call @printMemrefF32(%B_cast) : (memref<*xf32>) -> ()
+
+    // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+    // CHECK-NEXT: [11.11{{.*}}]
+    // CHECK-COUNT-96: 11.11
+    // CHECK-NEXT: [11.11{{.*}}]
+
+    // CHECK-NEXT: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+    // CHECK: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+    // CHECK-COUNT-96: 11.11
+    // CHECK-NEXT: [11.11{{.*}}]
+
+    memref.dealloc %A : memref<8x16xf32>
+    memref.dealloc %B : memref<8x16xf32>
+    return
+  }
+  func.func private @printMemrefF32(%ptr : memref<*xf32>)
+}
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_load_store_pack_register.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_load_store_pack_register.mlir
new file mode 100644
index 0000000000000..cb8ab1cbd32d7
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_load_store_pack_register.mlir
@@ -0,0 +1,131 @@
+// RUN: mlir-opt %s \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(cse,func.func(gpu-async-region),xevm-attach-target,gpu.module(convert-gpu-to-llvm-spv{use-64bit-index=true},convert-xevm-to-llvm,cse))' \
+// RUN: | mlir-opt -convert-scf-to-cf -convert-cf-to-llvm -convert-vector-to-llvm -convert-arith-to-llvm \
+// RUN: | mlir-opt -gpu-to-llvm -reconcile-unrealized-casts -cse -gpu-module-to-binary \
+// RUN: | mlir-runner \
+// RUN:   --shared-libs=%mlir_levelzero_runtime \
+// RUN:   --shared-libs=%mlir_runner_utils \
+// RUN:   --shared-libs=%mlir_c_runner_utils \
+// RUN:   --entry-point-result=void \
+// RUN: | FileCheck %s
+
+module @gemm attributes {gpu.container_module} {
+  gpu.module @kernel {
+    gpu.func @block_load_store(%src: !llvm.ptr<1>, %dst: !llvm.ptr<1>) kernel  {
+      %base_width = arith.constant 32 : i32 // bytewidth of the block
+      %base_height_load = arith.constant 16 : i32 // number of rows
+      %base_pitch = arith.constant 32 : i32 // bytewidth of the base row
+      %x = arith.constant 0 : i32
+      %y = arith.constant 0 : i32
+
+      // Consider the following two loads:
+      // Normal load:
+      %loaded = xevm.blockload2d %src, %base_width, %base_height_load, %base_pitch, %x, %y
+          <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=16 : i32, v_blocks=1 : i32,
+            transpose=false, pack_register=false}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<16xi16>
+      %loaded_f16_flat = vector.bitcast %loaded : vector<16xi16> to vector<16xf16>
+      %loaded_f16 = vector.shape_cast %loaded_f16_flat : vector<16xf16> to vector<8x1x2xf16>
+
+      // Register packed load:
+      %loaded_packed = xevm.blockload2d %src, %base_width, %base_height_load, %base_pitch, %x, %y
+          <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=16 : i32, v_blocks=1 : i32,
+            transpose=false, pack_register=true}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32>
+      %loaded_packed_f16_flat = vector.bitcast %loaded_packed : vector<8xi32> to vector<16xf16>
+      %loaded_packed_f16 = vector.shape_cast %loaded_packed_f16_flat : vector<16xf16> to vector<8x1x2xf16>
+      // Both can be represented the same way in code as vector<16xf16>.
+      // A normal load pads a value to a dword (e.g., 32-bit) when loaded to a register.
+      // Packed load "packs" multiple sub-dword values along the column (↓), allowing a single register
+      // to hold multiple values.
+      //  In SIMT, a work-item reads values along the column (↓), hence a sequence of values loaded by packing
+      // to register is logically equivalent to the sequence of values loaded using a normal load.
+      // The load results of both methods can have the same logical representation, but are expected to
+      // differ in physical layout and register efficiency.
+
+      %thread_x = gpu.thread_id x
+      %thread_x_i64 = arith.index_cast %thread_x : index to i64
+      %thread_x_i32 = llvm.trunc %thread_x_i64 : i64 to i32
+      %thread_x_f16 = arith.sitofp %thread_x_i32 : i32 to f16
+      %loaded_f16_modified = vector.insert %thread_x_f16, %loaded_packed_f16 [0,0,1] : f16 into vector<8x1x2xf16> // Both loaded_packed_f16 and loaded_f16 can be used here
+      // We can only store [1,2,4,8]x[16] shapes for f16, so we have to do 2 stores
+      %loaded_f16_modified_slice_0 = vector.extract_strided_slice %loaded_f16_modified
+          {offsets = [0, 0, 0], sizes = [4, 1, 2], strides = [1, 1, 1]} : vector<8x1x2xf16> to vector<4x1x2xf16>
+      %loaded_f16_modified_slice_0_flat = vector.shape_cast %loaded_f16_modified_slice_0 : vector<4x1x2xf16> to vector<8xf16>
+      %base_height_store = arith.constant 8 : i32 // number of rows
+      %base_width_store = arith.constant 32 : i32 // bytewidth of the block
+      %base_pitch_store = arith.constant 32 : i32 // bytewidth of the base row
+      xevm.blockstore2d %dst, %base_width_store, %base_height_store, %base_pitch_store, %x, %y, %loaded_f16_modified_slice_0_flat
+          <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=8 : i32}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xf16>)
+
+      %loaded_f16_modified_slice_1 = vector.extract_strided_slice %loaded_f16_modified
+          {offsets = [4, 0, 0], sizes = [4, 1, 2], strides = [1, 1, 1]} : vector<8x1x2xf16> to vector<4x1x2xf16>
+      %loaded_f16_modified_slice_1_flat = vector.shape_cast %loaded_f16_modified_slice_1 : vector<4x1x2xf16> to vector<8xf16>
+
+      %second_half_offset = arith.muli %base_pitch_store, %base_height_store : i32
+      %second_half_ptr = llvm.getelementptr %dst[%second_half_offset] : (!llvm.ptr<1>, i32) -> !llvm.ptr<1>, i8
+      xevm.blockstore2d %second_half_ptr, %base_width_store, %base_height_store, %base_pitch_store, %x, %y, %loaded_f16_modified_slice_1_flat
+          <{elem_size_in_bits=16 : i32, tile_width=16 : i32, tile_height=8 : i32}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xf16>)
+      gpu.return
+    }
+  }
+
+
+  func.func @test(%src : memref<16x16xf16>) -> memref<16x16xf16> attributes {llvm.emit_c_interface} {
+    %c1 = arith.constant 1 : index
+    %c16 = arith.constant 16 : index // Multiple of the *maximum sub-group size* (see `intel_reqd_sub_group_size`)
+    %memref_src = gpu.alloc() : memref<16x16xf16>
+    gpu.memcpy %memref_src, %src : memref<16x16xf16>, memref<16x16xf16>
+    %src_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_src : memref<16x16xf16> -> index
+    %src_ptr_as_i64 = arith.index_cast %src_ptr_as_idx : index to i64
+    %src_ptr = llvm.inttoptr %src_ptr_as_i64 : i64 to !llvm.ptr
+    %src_ptr_casted = llvm.addrspacecast %src_ptr : !llvm.ptr to !llvm.ptr<1>
+
+    %memref_dst = gpu.alloc() : memref<16x16xf16>
+    %dst_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_dst : memref<16x16xf16> -> index
+    %dst_ptr_as_i64 = arith.index_cast %dst_ptr_as_idx : index to i64
+    %dst_ptr = llvm.inttoptr %dst_ptr_as_i64 : i64 to !llvm.ptr
+    %dst_ptr_casted = llvm.addrspacecast %dst_ptr : !llvm.ptr to !llvm.ptr<1>
+
+    gpu.launch_func @kernel::@block_load_store blocks in (%c1, %c1, %c1) threads in (%c16, %c1, %c1)
+        args(%src_ptr_casted : !llvm.ptr<1>, %dst_ptr_casted : !llvm.ptr<1>)
+    gpu.dealloc %memref_src : memref<16x16xf16>
+    %dst = memref.alloc() : memref<16x16xf16>
+    gpu.memcpy %dst, %memref_dst : memref<16x16xf16>, memref<16x16xf16>
+    gpu.dealloc %memref_dst : memref<16x16xf16>
+    return %dst : memref<16x16xf16>
+  }
+
+  func.func @main() attributes {llvm.emit_c_interface} {
+    %A = memref.alloc() : memref<16x16xf16>
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c8 = arith.constant 16 : index
+    %c16 = arith.constant 16 : index
+    %c11_f32 = arith.constant 11.1 : f16
+    scf.for %i = %c0 to %c8 step %c1 {
+      scf.for %j = %c0 to %c16 step %c1 {
+        memref.store %c11_f32, %A[%i, %j] : memref<16x16xf16>
+      }
+    }
+    %B = call @test(%A) : (memref<16x16xf16>) -> memref<16x16xf16>
+    %B_cast = memref.cast %B : memref<16x16xf16> to memref<*xf16>
+    %A_cast = memref.cast %A : memref<16x16xf16> to memref<*xf16>
+    call @printMemrefF16(%A_cast) : (memref<*xf16>) -> ()
+    call @printMemrefF16(%B_cast) : (memref<*xf16>) -> ()
+
+    // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+    // CHECK-NEXT: [11.1{{.*}}]
+    // CHECK-COUNT-224: 11.1
+    // CHECK-NEXT: [11.1{{.*}}]
+
+    // CHECK-NEXT: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+    // CHECK-NEXT: [11.1{{.*}}]
+    // CHECK: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+    // CHECK-COUNT-208: 11.1
+    // CHECK-NEXT: [11.1{{.*}}]
+
+    memref.dealloc %A : memref<16x16xf16>
+    memref.dealloc %B : memref<16x16xf16>
+    return
+  }
+  func.func private @printMemrefF16(%ptr : memref<*xf16>) attributes { llvm.emit_c_interface }
+}
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_load_store_transpose.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_load_store_transpose.mlir
new file mode 100644
index 0000000000000..1d164be300246
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_block_load_store_transpose.mlir
@@ -0,0 +1,133 @@
+// RUN: mlir-opt %s \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(cse,func.func(gpu-async-region),xevm-attach-target,gpu.module(convert-gpu-to-llvm-spv{use-64bit-index=true},convert-xevm-to-llvm,cse))' \
+// RUN: | mlir-opt -convert-scf-to-cf -convert-cf-to-llvm -convert-vector-to-llvm -convert-arith-to-llvm \
+// RUN: | mlir-opt -gpu-to-llvm -reconcile-unrealized-casts -cse -gpu-module-to-binary \
+// RUN: | mlir-runner \
+// RUN:   --shared-libs=%mlir_levelzero_runtime \
+// RUN:   --shared-libs=%mlir_runner_utils \
+// RUN:   --shared-libs=%mlir_c_runner_utils \
+// RUN:   --entry-point-result=void \
+// RUN: | FileCheck %s
+
+module @gemm attributes {gpu.container_module} {
+  gpu.module @kernel {
+    gpu.func @block_load_store(%src: !llvm.ptr<1>, %dst: !llvm.ptr<1>) kernel  {
+      %base_width = arith.constant 32 : i32 // bytewidth of the block
+      %base_height = arith.constant 16 : i32 // number of rows
+      %base_pitch = arith.constant 32 : i32 // bytewidth of the base row
+      %x = arith.constant 0 : i32
+      %y = arith.constant 0 : i32
+      // Normally a work-item loads a vertical slice (↓), but with *transpose* a work-item
+      // loads a horizontal slice (→).
+      // The tile dimension we want to slice must be a multiple of the sub-group size:
+      //  e.g., we want to slice rows (→), then we need SG_SIZE % tile_height == 0.
+      %loaded = xevm.blockload2d %src, %base_width, %base_height, %base_pitch, %x, %y
+          <{elem_size_in_bits=32 : i32, tile_width=8 : i32, tile_height=16 : i32, v_blocks=1 : i32,
+            transpose=true, pack_register=false}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32) -> vector<8xi32>
+      %loaded_f32 = vector.bitcast %loaded : vector<8xi32> to vector<8xf32>
+
+      %c0 = arith.constant 0 : i32
+      %thread_x = gpu.thread_id x
+      %thread_x_i64 = arith.index_cast %thread_x : index to i64
+      %thread_x_i32 = llvm.trunc %thread_x_i64 : i64 to i32
+      %thread_x_f32 = arith.sitofp %thread_x_i32 : i32 to f32
+      %loaded_f32_modified = vector.insert %thread_x_f32, %loaded_f32[7] : f32 into vector<8xf32> // Use this to see where threadIds end up stored
+      %loaded_f32_modified_1 = vector.bitcast %loaded_f32_modified : vector<8xf32> to vector<8xi32>
+
+      %base_height_store = arith.constant 8 : i32 // number of rows
+      %base_width_store = arith.constant 64 : i32 // bytewidth of the block
+      %base_pitch_store = arith.constant 64 : i32 // bytewidth of the base row
+      // "Transposed" stores are not available, meaning a work-item can store its vector as a vertical slice (↓).
+      xevm.blockstore2d %dst, %base_width_store, %base_height_store, %base_pitch_store, %x, %y, %loaded
+          <{elem_size_in_bits=32 : i32, tile_width=16 : i32, tile_height=8 : i32}> : (!llvm.ptr<1>, i32, i32, i32, i32, i32, vector<8xi32>)
+      gpu.return
+    }
+  }
+
+
+  func.func @test(%src : memref<16x8xf32>) -> memref<8x16xf32> attributes {llvm.emit_c_interface} {
+    %c1 = arith.constant 1 : index
+    %c16 = arith.constant 16 : index // Multiple of the *maximum sub-group size* (see `intel_reqd_sub_group_size`)
+    %memref_src = gpu.alloc() : memref<16x8xf32>
+    gpu.memcpy %memref_src, %src : memref<16x8xf32>, memref<16x8xf32>
+    %src_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_src : memref<16x8xf32> -> index
+    %src_ptr_as_i64 = arith.index_cast %src_ptr_as_idx : index to i64
+    %src_ptr = llvm.inttoptr %src_ptr_as_i64 : i64 to !llvm.ptr
+    %src_ptr_casted = llvm.addrspacecast %src_ptr : !llvm.ptr to !llvm.ptr<1>
+
+    %memref_dst = gpu.alloc() : memref<8x16xf32>
+    %dst_ptr_as_idx = memref.extract_aligned_pointer_as_index %memref_dst : memref<8x16xf32> -> index
+    %dst_ptr_as_i64 = arith.index_cast %dst_ptr_as_idx : index to i64
+    %dst_ptr = llvm.inttoptr %dst_ptr_as_i64 : i64 to !llvm.ptr
+    %dst_ptr_casted = llvm.addrspacecast %dst_ptr : !llvm.ptr to !llvm.ptr<1>
+
+    gpu.launch_func @kernel::@block_load_store blocks in (%c1, %c1, %c1) threads in (%c16, %c1, %c1)
+        args(%src_ptr_casted : !llvm.ptr<1>, %dst_ptr_casted : !llvm.ptr<1>)
+    gpu.dealloc %memref_src : memref<16x8xf32>
+    %dst = memref.alloc() : memref<8x16xf32>
+    gpu.memcpy %dst, %memref_dst : memref<8x16xf32>, memref<8x16xf32>
+    gpu.dealloc %memref_dst : memref<8x16xf32>
+    return %dst : memref<8x16xf32>
+  }
+
+  func.func @main() attributes {llvm.emit_c_interface} {
+    %A = memref.alloc() : memref<16x8xf32>
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c8 = arith.constant 8 : index
+    %c16 = arith.constant 16 : index
+    %c11_f32 = arith.constant 11.11 : f16
+    scf.for %i = %c0 to %c16 step %c1 {
+      scf.for %j = %c0 to %c8 step %c1 {
+        %c_10_f = arith.constant 10.0 : f32
+        %j_i64 = arith.index_cast %j : index to i64
+        %j_i32 = llvm.trunc %j_i64 : i64 to i32
+        %j_f32 = arith.sitofp %j_i32 : i32 to f32
+        %jj = arith.divf %j_f32, %c_10_f : f32
+
+        %i_i64 = arith.index_cast %i : index to i64
+        %i_i32 = llvm.trunc %i_i64 : i64 to i32
+        %i_f32 = arith.sitofp %i_i32 : i32 to f32
+        %ii = arith.addf %i_f32, %jj : f32
+        memref.store %ii, %A[%i, %j] : memref<16x8xf32>
+      }
+    }
+    %B = call @test(%A) : (memref<16x8xf32>) -> memref<8x16xf32>
+    %A_cast = memref.cast %A : memref<16x8xf32> to memref<*xf32>
+    %B_cast = memref.cast %B : memref<8x16xf32> to memref<*xf32>
+    call @printMemrefF32(%A_cast) : (memref<*xf32>) -> ()
+    // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+    // CHECK-NEXT: [0,   0.1,   0.2,   0.3,   0.4,   0.5,   0.6,   0.7],
+    // CHECK-NEXT: [1,   1.1,   1.2,   1.3,   1.4,   1.5,   1.6,   1.7],
+    // CHECK-NEXT: [2,   2.1,   2.2,   2.3,   2.4,   2.5,   2.6,   2.7],
+    // CHECK-NEXT: [3,   3.1,   3.2,   3.3,   3.4,   3.5,   3.6,   3.7],
+    // CHECK-NEXT: [4,   4.1,   4.2,   4.3,   4.4,   4.5,   4.6,   4.7],
+    // CHECK-NEXT: [5,   5.1,   5.2,   5.3,   5.4,   5.5,   5.6,   5.7],
+    // CHECK-NEXT: [6,   6.1,   6.2,   6.3,   6.4,   6.5,   6.6,   6.7],
+    // CHECK-NEXT: [7,   7.1,   7.2,   7.3,   7.4,   7.5,   7.6,   7.7],
+    // CHECK-NEXT: [8,   8.1,   8.2,   8.3,   8.4,   8.5,   8.6,   8.7],
+    // CHECK-NEXT: [9,   9.1,   9.2,   9.3,   9.4,   9.5,   9.6,   9.7],
+    // CHECK-NEXT: [10,   10.1,   10.2,   10.3,   10.4,   10.5,   10.6,   10.7],
+    // CHECK-NEXT: [11,   11.1,   11.2,   11.3,   11.4,   11.5,   11.6,   11.7],
+    // CHECK-NEXT: [12,   12.1,   12.2,   12.3,   12.4,   12.5,   12.6,   12.7],
+    // CHECK-NEXT: [13,   13.1,   13.2,   13.3,   13.4,   13.5,   13.6,   13.7],
+    // CHECK-NEXT: [14,   14.1,   14.2,   14.3,   14.4,   14.5,   14.6,   14.7],
+    // CHECK-NEXT: [15,   15.1,   15.2,   15.3,   15.4,   15.5,   15.6,   15.7]
+
+    call @printMemrefF32(%B_cast) : (memref<*xf32>) -> ()
+    // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+    // CHECK-NEXT: [0,   1,   2,   3,   4,   5,   6,   7,   8,   9,   10,   11,   12,   13,   14,   15],
+    // CHECK-NEXT: [0.1,   1.1,   2.1,   3.1,   4.1,   5.1,   6.1,   7.1,   8.1,   9.1,   10.1,   11.1,   12.1,   13.1,   14.1,   15.1],
+    // CHECK-NEXT: [0.2,   1.2,   2.2,   3.2,   4.2,   5.2,   6.2,   7.2,   8.2,   9.2,   10.2,   11.2,   12.2,   13.2,   14.2,   15.2],
+    // CHECK-NEXT: [0.3,   1.3,   2.3,   3.3,   4.3,   5.3,   6.3,   7.3,   8.3,   9.3,   10.3,   11.3,   12.3,   13.3,   14.3,   15.3],
+    // CHECK-NEXT: [0.4,   1.4,   2.4,   3.4,   4.4,   5.4,   6.4,   7.4,   8.4,   9.4,   10.4,   11.4,   12.4,   13.4,   14.4,   15.4],
+    // CHECK-NEXT: [0.5,   1.5,   2.5,   3.5,   4.5,   5.5,   6.5,   7.5,   8.5,   9.5,   10.5,   11.5,   12.5,   13.5,   14.5,   15.5],
+    // CHECK-NEXT: [0.6,   1.6,   2.6,   3.6,   4.6,   5.6,   6.6,   7.6,   8.6,   9.6,   10.6,   11.6,   12.6,   13.6,   14.6,   15.6],
+    // CHECK-NEXT: [0.7,   1.7,   2.7,   3.7,   4.7,   5.7,   6.7,   7.7,   8.7,   9.7,   10.7,   11.7,   12.7,   13.7,   14.7,   15.7]
+
+    memref.dealloc %A : memref<16x8xf32>
+    memref.dealloc %B : memref<8x16xf32>
+    return
+  }
+  func.func private @printMemrefF32(%ptr : memref<*xf32>) attributes { llvm.emit_c_interface }
+}
diff --git a/mlir/test/Integration/Dialect/XeVM/GPU/xevm_store_cst.mlir b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_store_cst.mlir
new file mode 100644
index 0000000000000..c5f4cd5d0bda9
--- /dev/null
+++ b/mlir/test/Integration/Dialect/XeVM/GPU/xevm_store_cst.mlir
@@ -0,0 +1,75 @@
+// RUN: mlir-opt %s \
+// RUN: | mlir-opt -pass-pipeline='builtin.module(cse,func.func(gpu-async-region),xevm-attach-target,gpu.module(convert-gpu-to-llvm-spv{use-64bit-index=true},convert-xevm-to-llvm,cse))' \
+// RUN: | mlir-opt -convert-scf-to-cf -convert-cf-to-llvm -convert-vector-to-llvm -convert-arith-to-llvm \
+// RUN: | mlir-opt -gpu-to-llvm -reconcile-unrealized-casts -cse -gpu-module-to-binary \
+// RUN: | mlir-runner \
+// RUN:   --shared-libs=%mlir_levelzero_runtime \
+// RUN:   --shared-libs=%mlir_runner_utils \
+// RUN:   --shared-libs=%mlir_c_runner_utils \
+// RUN:   --entry-point-result=void \
+// RUN: | FileCheck %s
+
+module @gemm attributes {gpu.container_module} {
+
+  gpu.module @kernel {
+    gpu.func @store_constant(%ptr: !llvm.ptr<1>) kernel {
+      %const_val = arith.constant 42.0 : f32
+      %thread_x = gpu.lane_id
+      %thread_x_i64 = arith.index_cast %thread_x : index to i64
+      %ptr_next_1 = llvm.getelementptr %ptr[%thread_x_i64] : (!llvm.ptr<1>, i64) -> !llvm.ptr<1>, i32
+      llvm.store %const_val, %ptr_next_1 : f32, !llvm.ptr<1>
+      gpu.return
+    }
+  }
+  func.func @test(%src : memref<8x16xf32>) -> memref<8x16xf32> attributes {llvm.emit_c_interface} {
+    %c1 = arith.constant 1 : index
+    %c16 = arith.constant 16 : index
+    %memref_0 = gpu.alloc() : memref<8x16xf32>
+    gpu.memcpy %memref_0, %src : memref<8x16xf32>, memref<8x16xf32>
+    %0 = memref.extract_aligned_pointer_as_index %memref_0 : memref<8x16xf32> -> index
+    %1 = arith.index_cast %0 : index to i64
+    %2 = llvm.inttoptr %1 : i64 to !llvm.ptr
+    %src_casted = llvm.addrspacecast %2 : !llvm.ptr to !llvm.ptr<1>
+    gpu.launch_func @kernel::@store_constant blocks in (%c1, %c1, %c1) threads in (%c16, %c1, %c1)
+        args(%src_casted : !llvm.ptr<1>)
+    %dst = memref.alloc() : memref<8x16xf32>
+    gpu.memcpy %dst, %memref_0 : memref<8x16xf32>, memref<8x16xf32>
+    gpu.dealloc %memref_0 : memref<8x16xf32>
+
+    return %dst : memref<8x16xf32>
+  }
+
+  func.func @main() attributes {llvm.emit_c_interface} {
+    %A = memref.alloc() : memref<8x16xf32>
+    %c0 = arith.constant 0 : index
+    %c1 = arith.constant 1 : index
+    %c8 = arith.constant 8 : index
+    %c16 = arith.constant 16 : index
+    %c11_f32 = arith.constant 11.11 : f32
+    scf.for %i = %c0 to %c8 step %c1 {
+      scf.for %j = %c0 to %c16 step %c1 {
+        memref.store %c11_f32, %A[%i, %j] : memref<8x16xf32>
+      }
+    }
+    %B = call @test(%A) : (memref<8x16xf32>) -> memref<8x16xf32>
+    %B_cast = memref.cast %B : memref<8x16xf32> to memref<*xf32>
+    %A_cast = memref.cast %A : memref<8x16xf32> to memref<*xf32>
+    call @printMemrefF32(%A_cast) : (memref<*xf32>) -> ()
+    call @printMemrefF32(%B_cast) : (memref<*xf32>) -> ()
+
+    // CHECK: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+    // CHECK-NEXT: [11.11{{.*}}]
+    // CHECK-COUNT-96: 11.11
+    // CHECK-NEXT: [11.11{{.*}}]
+
+    // CHECK-NEXT: Unranked Memref base@ = 0x{{[0-9a-f]+}}
+    // CHECK-NEXT: [42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42]
+    // CHECK-COUNT-96: 11.11
+    // CHECK-NEXT: [11.11{{.*}}]
+
+    memref.dealloc %A : memref<8x16xf32>
+    memref.dealloc %B : memref<8x16xf32>
+    return
+  }
+  func.func private @printMemrefF32(%ptr : memref<*xf32>)
+}
diff --git a/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir b/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir
index a5653f395a2c4..37564de7442cf 100644
--- a/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir
+++ b/mlir/test/Integration/GPU/CUDA/sm90/gemm_f32_f16_f16_128x128x128.mlir
@@ -57,7 +57,6 @@
 
 func.func private @printMemrefF32(memref<*xf32>)
 
-memref.global "private" @dynamicShmem : memref<0xf16, 3> {alignment = 16 : i64}
 memref.global "private" @accShmem : memref<0xf32, 3> {alignment = 16 : i64}
 
 func.func @main() {
@@ -148,12 +147,11 @@ func.func @main() {
     %c57344 = arith.constant 57344 : index
     %c40960 = arith.constant 40960 : index
 
-    %tidx = gpu.thread_id  x
-    %dynamicMem = memref.get_global @dynamicShmem : memref<0xf16, 3>
-    %lhsShmem = memref.reinterpret_cast %dynamicMem to offset: [0], sizes: [2, 128, 64], strides: [8192, 64, 1] : memref<0xf16, 3> to memref<2x128x64xf16, 3>
-    %rhsShmem2 = memref.reinterpret_cast %dynamicMem to offset: [0], sizes: [4, 64, 128],  strides: [8192,128,1] : memref<0xf16, 3> to memref<4x64x128xf16,3>
-    %rhsShmem = memref.subview %rhsShmem2[2, 0, 0][2, 64, 128][1, 1, 1] : memref<4x64x128xf16,3> to memref<2x64x128xf16, strided<[8192, 128, 1], offset: 16384>, 3>
+    %tidx = gpu.thread_id  x 
     %dynsmem = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+    %lhsShmem = memref.view %dynsmem[%c0][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<2x128x64xf16, #gpu.address_space<workgroup>>
+    %rhsShmem = memref.view %dynsmem[%c32768][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<2x64x128xf16, #gpu.address_space<workgroup>> 
+
     // Step 1. [GPU] Create Async Transactional Barriers (mbarriers)
     %barrier = nvgpu.mbarrier.create -> !barrierType
     %cnd = arith.cmpi eq, %tidx, %c0 : index
@@ -202,11 +200,11 @@ func.func @main() {
       // TMA wait
       %phase_c0 = arith.constant 0 : i1
       nvgpu.mbarrier.try_wait.parity %barrier[%i], %phase_c0, %ticks : !barrierType
-      %lhsSlice = memref.subview %lhsShmem [%i, 0, 0][1, 128, 64][1, 1, 1] : memref<2x128x64xf16, 3> to memref<128x64xf16, strided<[64, 1], offset: ?>, 3>
-      %rhsSlice = memref.subview %rhsShmem [%i, 0, 0][1, 64, 128][1, 1, 1] : memref<2x64x128xf16, strided<[8192, 128, 1], offset: 16384>, 3> to memref<64x128xf16, strided<[128, 1], offset: ?>, 3>
+      %lhsSlice = memref.subview %lhsShmem [%i, 0, 0][1, 128, 64][1, 1, 1] : memref<2x128x64xf16, #gpu.address_space<workgroup>> to memref<128x64xf16, strided<[64, 1], offset: ?>, #gpu.address_space<workgroup>>
+      %rhsSlice = memref.subview %rhsShmem [%i, 0, 0][1, 64, 128][1, 1, 1] : memref<2x64x128xf16, #gpu.address_space<workgroup>> to memref<64x128xf16, strided<[128, 1], offset: ?>, #gpu.address_space<workgroup>>
       // Descriptor WGMMA
-      %dA = nvgpu.warpgroup.generate.descriptor %lhsSlice, %descA : memref<128x64xf16, strided<[64, 1], offset: ?>, 3>, !lhsTensorMap -> !nvgpu.warpgroup.descriptor<tensor=memref<128x64xf16, 3>>
-      %dB = nvgpu.warpgroup.generate.descriptor %rhsSlice, %descB : memref<64x128xf16, strided<[128, 1], offset: ?>, 3>, !rhsTensorMap -> !nvgpu.warpgroup.descriptor<tensor=memref<64x128xf16, 3>>
+      %dA = nvgpu.warpgroup.generate.descriptor %lhsSlice, %descA : memref<128x64xf16, strided<[64, 1], offset: ?>, #gpu.address_space<workgroup>>, !lhsTensorMap -> !nvgpu.warpgroup.descriptor<tensor=memref<128x64xf16, 3>>
+      %dB = nvgpu.warpgroup.generate.descriptor %rhsSlice, %descB : memref<64x128xf16, strided<[128, 1], offset: ?>, #gpu.address_space<workgroup>>, !rhsTensorMap -> !nvgpu.warpgroup.descriptor<tensor=memref<64x128xf16, 3>>
       // Perform WGMMA 128x128x64
       %md  = nvgpu.warpgroup.mma %dA, %dB, %mc {transposeB} : <tensor = memref<128x64xf16,3>>, <tensor = memref<64x128xf16,3>>, <fragmented = vector<128x128xf32>> -> <fragmented = vector<128x128xf32>>
       scf.yield %md : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
@@ -271,7 +269,7 @@ func.func @main() {
   vector.print str "Correct Results :"
   vector.print %correctCount : i32
   vector.print str "Incorrect Results :"
-  vector.print %errorCount : i32
+  vector.print %errorCount : i32 
 
   return
 }
diff --git a/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir b/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir
index 197351f1921e7..db7754c89dcac 100644
--- a/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir
+++ b/mlir/test/Integration/GPU/CUDA/sm90/gemm_pred_f32_f16_f16_128x128x128.mlir
@@ -57,7 +57,6 @@
 
 func.func private @printMemrefF32(memref<*xf32>)
 
-memref.global "private" @dynamicShmem : memref<0xf16, 3> {alignment = 16 : i64}
 memref.global "private" @accShmem : memref<0xf32, 3> {alignment = 16 : i64}
 
 func.func @main() {
@@ -149,11 +148,10 @@ func.func @main() {
     %c40960 = arith.constant 40960 : index
 
     %tidx = gpu.thread_id  x
-    %dynamicMem = memref.get_global @dynamicShmem : memref<0xf16, 3>
-    %lhsShmem = memref.reinterpret_cast %dynamicMem to offset: [0], sizes: [2, 128, 64], strides: [8192, 64, 1] : memref<0xf16, 3> to memref<2x128x64xf16, 3>
-    %rhsShmem2 = memref.reinterpret_cast %dynamicMem to offset: [0], sizes: [4, 64, 128],  strides: [8192,128,1] : memref<0xf16, 3> to memref<4x64x128xf16,3>
-    %rhsShmem = memref.subview %rhsShmem2[2, 0, 0][2, 64, 128][1, 1, 1] : memref<4x64x128xf16,3> to memref<2x64x128xf16, strided<[8192, 128, 1], offset: 16384>, 3>
     %dynsmem = gpu.dynamic_shared_memory : memref<?xi8, #gpu.address_space<workgroup>>
+    %lhsShmem = memref.view %dynsmem[%c0][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<2x128x64xf16, #gpu.address_space<workgroup>>
+    %rhsShmem = memref.view %dynsmem[%c32768][] : memref<?xi8, #gpu.address_space<workgroup>> to memref<2x64x128xf16, #gpu.address_space<workgroup>> 
+
     // Step 1. [GPU] Create Async Transactional Barriers (mbarriers)
     %barrier = nvgpu.mbarrier.create -> !barrierType
     
@@ -210,11 +208,11 @@ func.func @main() {
       // TMA wait
       %phase_c0 = arith.constant 0 : i1
       nvgpu.mbarrier.try_wait.parity %barrier[%i], %phase_c0, %ticks : !barrierType
-      %lhsSlice = memref.subview %lhsShmem [%i, 0, 0][1, 128, 64][1, 1, 1] : memref<2x128x64xf16, 3> to memref<128x64xf16, strided<[64, 1], offset: ?>, 3>
-      %rhsSlice = memref.subview %rhsShmem [%i, 0, 0][1, 64, 128][1, 1, 1] : memref<2x64x128xf16, strided<[8192, 128, 1], offset: 16384>, 3> to memref<64x128xf16, strided<[128, 1], offset: ?>, 3>
+      %lhsSlice = memref.subview %lhsShmem [%i, 0, 0][1, 128, 64][1, 1, 1] : memref<2x128x64xf16, #gpu.address_space<workgroup>> to memref<128x64xf16, strided<[64, 1], offset: ?>, #gpu.address_space<workgroup>>
+      %rhsSlice = memref.subview %rhsShmem [%i, 0, 0][1, 64, 128][1, 1, 1] : memref<2x64x128xf16, #gpu.address_space<workgroup>> to memref<64x128xf16, strided<[128, 1], offset: ?>, #gpu.address_space<workgroup>>
       // Descriptor WGMMA
-      %dA = nvgpu.warpgroup.generate.descriptor %lhsSlice, %descA : memref<128x64xf16, strided<[64, 1], offset: ?>, 3>, !lhsTensorMap -> !nvgpu.warpgroup.descriptor<tensor=memref<128x64xf16, 3>>
-      %dB = nvgpu.warpgroup.generate.descriptor %rhsSlice, %descB : memref<64x128xf16, strided<[128, 1], offset: ?>, 3>, !rhsTensorMap -> !nvgpu.warpgroup.descriptor<tensor=memref<64x128xf16, 3>>
+      %dA = nvgpu.warpgroup.generate.descriptor %lhsSlice, %descA : memref<128x64xf16, strided<[64, 1], offset: ?>, #gpu.address_space<workgroup>>, !lhsTensorMap -> !nvgpu.warpgroup.descriptor<tensor=memref<128x64xf16, 3>>
+      %dB = nvgpu.warpgroup.generate.descriptor %rhsSlice, %descB : memref<64x128xf16, strided<[128, 1], offset: ?>, #gpu.address_space<workgroup>>, !rhsTensorMap -> !nvgpu.warpgroup.descriptor<tensor=memref<64x128xf16, 3>>
       // Perform WGMMA 128x128x64
       %md  = nvgpu.warpgroup.mma %dA, %dB, %mc {transposeB} : <tensor = memref<128x64xf16,3>>, <tensor = memref<64x128xf16,3>>, <fragmented = vector<128x128xf32>> -> <fragmented = vector<128x128xf32>>
       scf.yield %md : !nvgpu.warpgroup.accumulator<fragmented = vector<128x128xf32>>
diff --git a/mlir/test/Integration/GPU/LevelZero/gpu-addf32-to-spirv.mlir b/mlir/test/Integration/GPU/LevelZero/gpu-addf32-to-spirv.mlir
new file mode 100644
index 0000000000000..7e66dee0272f6
--- /dev/null
+++ b/mlir/test/Integration/GPU/LevelZero/gpu-addf32-to-spirv.mlir
@@ -0,0 +1,59 @@
+// RUN: mlir-opt %s -pass-pipeline='builtin.module(spirv-attach-target{ver=v1.0 caps=Addresses,Int64,Kernel},convert-gpu-to-spirv{use-64bit-index=true},gpu.module(spirv.module(spirv-lower-abi-attrs,spirv-update-vce)),func.func(llvm-request-c-wrappers),convert-scf-to-cf,convert-to-llvm,gpu-to-llvm{use-bare-pointers-for-kernels=true},gpu-module-to-binary,expand-strided-metadata,lower-affine,reconcile-unrealized-casts)' \
+// RUN: | mlir-runner \
+// RUN:   --shared-libs=%mlir_levelzero_runtime \
+// RUN:   --shared-libs=%mlir_runner_utils \
+// RUN:   --entry-point-result=void \
+// RUN: | FileCheck %s
+
+module @add attributes {gpu.container_module} {
+  memref.global "private" constant @__constant_2x2x2xf32_0 : memref<2x2x2xf32> = dense<[[[1.1, 2.2], [3.3, 4.4]], [[5.5, 6.6], [7.7, 8.8 ]]]>
+  memref.global "private" constant @__constant_2x2x2xf32 : memref<2x2x2xf32> = dense<[[[1.2, 2.3], [4.5, 5.8]], [[7.2, 8.3], [10.5, 11.8]]]>
+  func.func @main() {
+    %0 = memref.get_global @__constant_2x2x2xf32 : memref<2x2x2xf32>
+    %1 = memref.get_global @__constant_2x2x2xf32_0 : memref<2x2x2xf32>
+    %2 = call @test(%0, %1) : (memref<2x2x2xf32>, memref<2x2x2xf32>) -> memref<2x2x2xf32>
+    %cast = memref.cast %2 : memref<2x2x2xf32> to memref<*xf32>
+    call @printMemrefF32(%cast) : (memref<*xf32>) -> ()
+    return
+  }
+  func.func private @printMemrefF32(memref<*xf32>)
+  func.func @test(%arg0: memref<2x2x2xf32>, %arg1: memref<2x2x2xf32>) -> memref<2x2x2xf32> {
+    %c2 = arith.constant 2 : index
+    %c1 = arith.constant 1 : index
+    %mem = gpu.alloc host_shared () : memref<2x2x2xf32>
+    memref.copy %arg1, %mem : memref<2x2x2xf32> to memref<2x2x2xf32>
+    %memref_0 = gpu.alloc host_shared () : memref<2x2x2xf32>
+    memref.copy %arg0, %memref_0 : memref<2x2x2xf32> to memref<2x2x2xf32>
+    %memref_2 = gpu.alloc host_shared () : memref<2x2x2xf32>
+    %2 = gpu.wait async
+    %3 = gpu.launch_func async [%2] @test_kernel::@test_kernel blocks in (%c2, %c2, %c2) threads in (%c1, %c1, %c1)
+          args(%memref_0 : memref<2x2x2xf32>, %mem : memref<2x2x2xf32>, %memref_2 : memref<2x2x2xf32>)
+    gpu.wait [%3]
+    %alloc = memref.alloc() : memref<2x2x2xf32>
+    memref.copy %memref_2, %alloc : memref<2x2x2xf32> to memref<2x2x2xf32>
+    %4 = gpu.wait async
+    %5 = gpu.dealloc async [%4] %memref_2 : memref<2x2x2xf32>
+    %6 = gpu.dealloc async [%5] %memref_0 : memref<2x2x2xf32>
+    %7 = gpu.dealloc async [%6] %mem : memref<2x2x2xf32>
+    gpu.wait [%7]
+    return %alloc : memref<2x2x2xf32>
+  }
+  gpu.module @test_kernel
+  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.0, [Addresses, Int64, Kernel], []>, api=OpenCL, #spirv.resource_limits<>>} {
+    gpu.func @test_kernel(%arg0: memref<2x2x2xf32>, %arg1: memref<2x2x2xf32>, %arg2: memref<2x2x2xf32>) kernel
+    attributes {gpu.known_block_size = array<i32: 1, 1, 1>, gpu.known_grid_size = array<i32: 2, 2, 2>, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+      %0 = gpu.block_id  x
+      %1 = gpu.block_id  y
+      %2 = gpu.block_id  z
+      %3 = memref.load %arg0[%0, %1, %2] : memref<2x2x2xf32>
+      %4 = memref.load %arg1[%0, %1, %2] : memref<2x2x2xf32>
+      %5 = arith.addf %3, %4 : f32
+      memref.store %5, %arg2[%0, %1, %2] : memref<2x2x2xf32>
+      gpu.return
+    }
+  }
+  // CHECK: [2.3, 4.5]
+  // CHECK: [7.8, 10.2]
+  // CHECK: [12.7, 14.9]
+  // CHECK: [18.2, 20.6]
+}
diff --git a/mlir/test/Integration/GPU/LevelZero/gpu-addi64-to-spirv.mlir b/mlir/test/Integration/GPU/LevelZero/gpu-addi64-to-spirv.mlir
new file mode 100644
index 0000000000000..df8fbe4d86d9c
--- /dev/null
+++ b/mlir/test/Integration/GPU/LevelZero/gpu-addi64-to-spirv.mlir
@@ -0,0 +1,57 @@
+// RUN: mlir-opt %s -pass-pipeline='builtin.module(spirv-attach-target{ver=v1.0 caps=Addresses,Int64,Kernel},convert-gpu-to-spirv{use-64bit-index=true},gpu.module(spirv.module(spirv-lower-abi-attrs,spirv-update-vce)),func.func(llvm-request-c-wrappers),convert-scf-to-cf,convert-to-llvm,gpu-to-llvm{use-bare-pointers-for-kernels=true},gpu-module-to-binary,expand-strided-metadata,lower-affine,reconcile-unrealized-casts)' \
+// RUN: | mlir-runner \
+// RUN:   --shared-libs=%mlir_levelzero_runtime \
+// RUN:   --shared-libs=%mlir_runner_utils \
+// RUN:   --entry-point-result=void \
+// RUN: | FileCheck %s
+
+module @add attributes {gpu.container_module} {
+  memref.global "private" constant @__constant_3x3xi64_0 : memref<3x3xi64> = dense<[[1, 4098, 3], [16777220, 5, 4294967302], [7, 1099511627784, 9]]>
+  memref.global "private" constant @__constant_3x3xi64 : memref<3x3xi64> = dense<[[1, 2, 3], [4, 5, 4102], [16777223, 4294967304, 1099511627785]]>
+  func.func @main() {
+    %0 = memref.get_global @__constant_3x3xi64 : memref<3x3xi64>
+    %1 = memref.get_global @__constant_3x3xi64_0 : memref<3x3xi64>
+    %2 = call @test(%0, %1) : (memref<3x3xi64>, memref<3x3xi64>) -> memref<3x3xi64>
+    %cast = memref.cast %2 : memref<3x3xi64> to memref<*xi64>
+    call @printMemrefI64(%cast) : (memref<*xi64>) -> ()
+    return
+  }
+  func.func private @printMemrefI64(memref<*xi64>)
+  func.func @test(%arg0: memref<3x3xi64>, %arg1: memref<3x3xi64>) -> memref<3x3xi64> {
+    %c3 = arith.constant 3 : index
+    %c1 = arith.constant 1 : index
+    %mem = gpu.alloc host_shared () : memref<3x3xi64>
+    memref.copy %arg1, %mem : memref<3x3xi64> to memref<3x3xi64>
+    %memref_0 = gpu.alloc host_shared () : memref<3x3xi64>
+    memref.copy %arg0, %memref_0 : memref<3x3xi64> to memref<3x3xi64>
+    %memref_2 = gpu.alloc host_shared () : memref<3x3xi64>
+    %2 = gpu.wait async
+    %3 = gpu.launch_func async [%2] @test_kernel::@test_kernel blocks in (%c3, %c3, %c1) threads in (%c1, %c1, %c1)
+          args(%memref_0 : memref<3x3xi64>, %mem : memref<3x3xi64>, %memref_2 : memref<3x3xi64>)
+    gpu.wait [%3]
+    %alloc = memref.alloc() : memref<3x3xi64>
+    memref.copy %memref_2, %alloc : memref<3x3xi64> to memref<3x3xi64>
+    %4 = gpu.wait async
+    %5 = gpu.dealloc async [%4] %memref_2 : memref<3x3xi64>
+    %6 = gpu.dealloc async [%5] %memref_0 : memref<3x3xi64>
+    %7 = gpu.dealloc async [%6] %mem : memref<3x3xi64>
+    gpu.wait [%7]
+    return %alloc : memref<3x3xi64>
+  }
+  gpu.module @test_kernel
+  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.0, [Addresses, Int64, Kernel], []>, api=OpenCL, #spirv.resource_limits<>>} {
+    gpu.func @test_kernel(%arg0: memref<3x3xi64>, %arg1: memref<3x3xi64>, %arg2: memref<3x3xi64>) kernel
+    attributes {gpu.known_block_size = array<i32: 1, 1, 1>, gpu.known_grid_size = array<i32: 3, 3, 1>, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+      %0 = gpu.block_id  x
+      %1 = gpu.block_id  y
+      %2 = memref.load %arg0[%0, %1] : memref<3x3xi64>
+      %3 = memref.load %arg1[%0, %1] : memref<3x3xi64>
+      %4 = arith.addi %2, %3 : i64
+      memref.store %4, %arg2[%0, %1] : memref<3x3xi64>
+      gpu.return
+    }
+  }
+  // CHECK: [2,   4100,   6],
+  // CHECK: [16777224,   10,   4294971404],
+  // CHECK: [16777230,   1103806595088,   1099511627794]
+}
diff --git a/mlir/test/Integration/GPU/LevelZero/gpu-memcpy-addf32-to-spirv.mlir b/mlir/test/Integration/GPU/LevelZero/gpu-memcpy-addf32-to-spirv.mlir
new file mode 100644
index 0000000000000..cd99f2c70dc6e
--- /dev/null
+++ b/mlir/test/Integration/GPU/LevelZero/gpu-memcpy-addf32-to-spirv.mlir
@@ -0,0 +1,56 @@
+// RUN: mlir-opt %s -pass-pipeline='builtin.module(func.func(gpu-async-region),spirv-attach-target{ver=v1.0 caps=Addresses,Int64,Kernel},convert-gpu-to-spirv{use-64bit-index=true},gpu.module(spirv.module(spirv-lower-abi-attrs,spirv-update-vce)),func.func(llvm-request-c-wrappers),convert-scf-to-cf,convert-to-llvm,gpu-to-llvm{use-bare-pointers-for-kernels=true},gpu-module-to-binary,expand-strided-metadata,lower-affine,reconcile-unrealized-casts)' \
+// RUN: | mlir-runner \
+// RUN:   --shared-libs=%mlir_levelzero_runtime \
+// RUN:   --shared-libs=%mlir_runner_utils \
+// RUN:   --entry-point-result=void \
+// RUN: | FileCheck %s
+
+module @add attributes {gpu.container_module} {
+  memref.global "private" constant @__constant_2x2x2xf32_0 : memref<2x2x2xf32> = dense<[[[1.1, 2.2], [3.3, 4.4]], [[5.5, 6.6], [7.7, 8.8 ]]]>
+  memref.global "private" constant @__constant_2x2x2xf32 : memref<2x2x2xf32> = dense<[[[1.2, 2.3], [4.5, 5.8]], [[7.2, 8.3], [10.5, 11.8]]]>
+  func.func @main() {
+    %0 = memref.get_global @__constant_2x2x2xf32 : memref<2x2x2xf32>
+    %1 = memref.get_global @__constant_2x2x2xf32_0 : memref<2x2x2xf32>
+    %2 = call @test(%0, %1) : (memref<2x2x2xf32>, memref<2x2x2xf32>) -> memref<2x2x2xf32>
+    %cast = memref.cast %2 : memref<2x2x2xf32> to memref<*xf32>
+    call @printMemrefF32(%cast) : (memref<*xf32>) -> ()
+    memref.dealloc %2 : memref<2x2x2xf32>
+    return
+  }
+  func.func private @printMemrefF32(memref<*xf32>)
+  func.func @test(%arg0: memref<2x2x2xf32>, %arg1: memref<2x2x2xf32>) -> memref<2x2x2xf32> {
+    %c2 = arith.constant 2 : index
+    %c1 = arith.constant 1 : index
+    %memref = gpu.alloc () : memref<2x2x2xf32>
+    gpu.memcpy %memref, %arg0 : memref<2x2x2xf32>, memref<2x2x2xf32>
+    %memref_0 = gpu.alloc () : memref<2x2x2xf32>
+    gpu.memcpy %memref_0, %arg1 : memref<2x2x2xf32>, memref<2x2x2xf32>
+    %memref_1 = gpu.alloc () : memref<2x2x2xf32>
+    gpu.launch_func @test_kernel::@test_kernel blocks in (%c2, %c2, %c2) threads in (%c1, %c1, %c1)
+      args(%memref : memref<2x2x2xf32>, %memref_0 : memref<2x2x2xf32>, %memref_1 : memref<2x2x2xf32>)
+    %alloc = memref.alloc() : memref<2x2x2xf32>
+    gpu.memcpy %alloc, %memref_1 : memref<2x2x2xf32>, memref<2x2x2xf32>
+    gpu.dealloc %memref_1 : memref<2x2x2xf32>
+    gpu.dealloc %memref_0 : memref<2x2x2xf32>
+    gpu.dealloc %memref : memref<2x2x2xf32>
+    return %alloc : memref<2x2x2xf32>
+  }
+  gpu.module @test_kernel
+  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.0, [Addresses, Int64, Kernel], []>, api=OpenCL, #spirv.resource_limits<>>} {
+    gpu.func @test_kernel(%arg0: memref<2x2x2xf32>, %arg1: memref<2x2x2xf32>, %arg2: memref<2x2x2xf32>) kernel
+    attributes {gpu.known_block_size = array<i32: 1, 1, 1>, gpu.known_grid_size = array<i32: 2, 2, 2>, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+      %0 = gpu.block_id  x
+      %1 = gpu.block_id  y
+      %2 = gpu.block_id  z
+      %3 = memref.load %arg0[%0, %1, %2] : memref<2x2x2xf32>
+      %4 = memref.load %arg1[%0, %1, %2] : memref<2x2x2xf32>
+      %5 = arith.addf %3, %4 : f32
+      memref.store %5, %arg2[%0, %1, %2] : memref<2x2x2xf32>
+      gpu.return
+    }
+  }
+  // CHECK: [2.3, 4.5]
+  // CHECK: [7.8, 10.2]
+  // CHECK: [12.7, 14.9]
+  // CHECK: [18.2, 20.6]
+}
diff --git a/mlir/test/Integration/GPU/LevelZero/gpu-reluf32-to-spirv.mlir b/mlir/test/Integration/GPU/LevelZero/gpu-reluf32-to-spirv.mlir
new file mode 100644
index 0000000000000..8d022ac1cf277
--- /dev/null
+++ b/mlir/test/Integration/GPU/LevelZero/gpu-reluf32-to-spirv.mlir
@@ -0,0 +1,86 @@
+// RUN: mlir-opt %s -pass-pipeline='builtin.module(spirv-attach-target{ver=v1.0 caps=Addresses,Int64,Kernel},convert-gpu-to-spirv{use-64bit-index=true},gpu.module(spirv.module(spirv-lower-abi-attrs,spirv-update-vce)),func.func(llvm-request-c-wrappers),convert-scf-to-cf,convert-to-llvm,gpu-to-llvm{use-bare-pointers-for-kernels=true},gpu-module-to-binary,expand-strided-metadata,lower-affine,reconcile-unrealized-casts)' \
+// RUN: | mlir-runner \
+// RUN:   --shared-libs=%mlir_levelzero_runtime \
+// RUN:   --shared-libs=%mlir_runner_utils \
+// RUN:   --entry-point-result=void \
+// RUN: | FileCheck %s
+
+module @relu attributes {gpu.container_module} {
+  memref.global "private" constant @__constant_4x5xf32 : memref<4x5xf32> = dense<[
+    [-1.000000e-01, -2.000000e-01, -3.000000e-01, 4.000000e-01, 5.000000e-01],
+    [1.000000e-01, -2.000000e-01, 3.000000e-01, -4.000000e-01, 5.000000e-01],
+    [1.000000e-01, 2.000000e-01, 3.000000e-01, -4.000000e-01, -5.000000e-01],
+    [1.000000e-01, 2.000000e-01, 3.000000e-01, 4.000000e-01, 5.000000e-01]
+  ]>
+
+  func.func @main() {
+    %c1 = arith.constant 1 : index
+    %c100 = arith.constant 100 : index
+    %c0 = arith.constant 0 : index
+    %0 = memref.get_global @__constant_4x5xf32 : memref<4x5xf32>
+
+    scf.for %arg0 = %c0 to %c100 step %c1 {
+      %1 = func.call @test(%0) : (memref<4x5xf32>) -> memref<4x5xf32>
+      %cast = memref.cast %1 : memref<4x5xf32> to memref<*xf32>
+      func.call @printMemrefF32(%cast) : (memref<*xf32>) -> ()
+      // CHECK: [0, 0, 0, 0.4, 0.5],
+      // CHECK: [0.1, 0, 0.3, 0, 0.5],
+      // CHECK: [0.1, 0.2, 0.3, 0, 0],
+      // CHECK: [0.1, 0.2, 0.3, 0.4, 0.5]
+    }
+    return
+  }
+
+  func.func private @printMemrefF32(memref<*xf32>)
+  func.func @test(%arg0: memref<4x5xf32>) -> memref<4x5xf32> {
+    %c5 = arith.constant 5 : index
+    %c4 = arith.constant 4 : index
+    %cst = arith.constant 0.000000e+00 : f32
+    %c1 = arith.constant 1 : index
+    %memref = gpu.alloc host_shared () : memref<4x5xf32>
+    memref.copy %arg0, %memref : memref<4x5xf32> to memref<4x5xf32>
+    %memref_0 = gpu.alloc host_shared () : memref<4x5xi1>
+    %2 = gpu.wait async
+    %3 = gpu.launch_func async [%2]  @test_kernel::@test_kernel blocks in (%c4, %c5, %c1) threads in (%c1, %c1, %c1)
+          args(%memref : memref<4x5xf32>, %cst : f32, %memref_0 : memref<4x5xi1>)
+    gpu.wait [%3]
+    %memref_1 = gpu.alloc host_shared () : memref<4x5xf32>
+    %4 = gpu.wait async
+    %5 = gpu.launch_func async [%4]  @test_kernel_0::@test_kernel blocks in (%c4, %c5, %c1) threads in (%c1, %c1, %c1)
+          args(%memref_0 : memref<4x5xi1>, %memref : memref<4x5xf32>, %cst : f32,
+    %memref_1 : memref<4x5xf32>)
+    gpu.wait [%5]
+    %alloc = memref.alloc() : memref<4x5xf32>
+    memref.copy %memref_1, %alloc : memref<4x5xf32> to memref<4x5xf32>
+    %6 = gpu.wait async
+    %7 = gpu.dealloc async [%6] %memref_1 : memref<4x5xf32>
+    %8 = gpu.dealloc async [%7] %memref_0 : memref<4x5xi1>
+    %9 = gpu.dealloc async [%8] %memref : memref<4x5xf32>
+    return %alloc : memref<4x5xf32>
+  }
+  gpu.module @test_kernel
+  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.0, [Addresses, Int64, Int8, Kernel], []>, api=OpenCL, #spirv.resource_limits<>>} {
+    gpu.func @test_kernel(%arg0: memref<4x5xf32>, %arg1: f32, %arg2: memref<4x5xi1>) kernel
+    attributes {gpu.known_block_size = array<i32: 1, 1, 1>, gpu.known_grid_size = array<i32: 4, 5, 1>, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+      %0 = gpu.block_id  x
+      %1 = gpu.block_id  y
+      %2 = memref.load %arg0[%0, %1] : memref<4x5xf32>
+      %3 = arith.cmpf olt, %2, %arg1 : f32
+      memref.store %3, %arg2[%0, %1] : memref<4x5xi1>
+      gpu.return
+    }
+  }
+  gpu.module @test_kernel_0
+  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.0, [Addresses, Int64, Int8, Kernel], []>, api=OpenCL, #spirv.resource_limits<>>} {
+    gpu.func @test_kernel(%arg0: memref<4x5xi1>, %arg1: memref<4x5xf32>, %arg2: f32, %arg3: memref<4x5xf32>) kernel
+    attributes {gpu.known_block_size = array<i32: 1, 1, 1>, gpu.known_grid_size = array<i32: 4, 5, 1>, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
+      %0 = gpu.block_id  x
+      %1 = gpu.block_id  y
+      %2 = memref.load %arg0[%0, %1] : memref<4x5xi1>
+      %3 = memref.load %arg1[%0, %1] : memref<4x5xf32>
+      %4 = arith.select %2, %arg2, %3 : f32
+      memref.store %4, %arg3[%0, %1] : memref<4x5xf32>
+      gpu.return
+    }
+  }
+}
diff --git a/mlir/test/Integration/GPU/LevelZero/lit.local.cfg b/mlir/test/Integration/GPU/LevelZero/lit.local.cfg
new file mode 100644
index 0000000000000..36c7ad5f57c7e
--- /dev/null
+++ b/mlir/test/Integration/GPU/LevelZero/lit.local.cfg
@@ -0,0 +1,2 @@
+if not config.enable_levelzero_runner:
+    config.unsupported = True
diff --git a/mlir/test/Pass/pipeline-options-parsing.mlir b/mlir/test/Pass/pipeline-options-parsing.mlir
index 9385d353faf95..03ac38ea16112 100644
--- a/mlir/test/Pass/pipeline-options-parsing.mlir
+++ b/mlir/test/Pass/pipeline-options-parsing.mlir
@@ -13,6 +13,7 @@
 // RUN: mlir-opt %s -verify-each=false -pass-pipeline='builtin.module(builtin.module(func.func(test-options-pass{list=3}), func.func(test-options-pass{enum=one list=1,2,3,4 string=foo"bar"baz})))' -dump-pass-pipeline 2>&1 | FileCheck --check-prefix=CHECK_6 %s
 // RUN: mlir-opt %s -verify-each=false '-test-options-super-pass-pipeline=super-list={{enum=zero list=1 string=foo},{enum=one list=2 string="bar"},{enum=two list=3 string={baz}}}' -dump-pass-pipeline 2>&1 | FileCheck --check-prefix=CHECK_7 %s
 // RUN: mlir-opt %s -verify-each=false -pass-pipeline='builtin.module(func.func(test-options-super-pass{list={{enum=zero list={1} string=foo },{enum=one list={2} string=bar },{enum=two list={3} string=baz }}}))' -dump-pass-pipeline 2>&1 | FileCheck --check-prefix=CHECK_7 %s
+// RUN: mlir-opt %s -verify-each=false -test-options-super-set-ab-pipeline='foo=true bar=false' -dump-pass-pipeline 2>&1 | FileCheck --check-prefix=CHECK_11 %s
 
 
 // This test checks that lists-of-nested-options like 'option1={...},{....}' can be parsed
@@ -106,3 +107,12 @@
 // CHECK_10-NEXT:     test-options-pass{enum=zero  string= string-list={,}}
 // CHECK_10-NEXT:   )
 // CHECK_10-NEXT: )
+
+// CHECK_11:      builtin.module(
+// CHECK_11-NEXT:   func.func(
+// CHECK_11-NEXT:     test-options-pass-a
+// CHECK_11-NEXT:   )
+// CHECK_11-NEXT:   func.func(
+// CHECK_11-NEXT:     test-options-pass-b
+// CHECK_11-NEXT:   )
+// CHECK_11-NEXT: )
diff --git a/mlir/test/Target/Cpp/class.mlir b/mlir/test/Target/Cpp/class.mlir
new file mode 100644
index 0000000000000..32c6699f3f666
--- /dev/null
+++ b/mlir/test/Target/Cpp/class.mlir
@@ -0,0 +1,78 @@
+// RUN: mlir-translate -mlir-to-cpp %s | FileCheck %s
+
+emitc.class @modelClass {
+  emitc.field @fieldName0 : !emitc.array<1xf32>
+  emitc.field @fieldName1 : !emitc.array<1xf32>
+  emitc.func @execute() {
+    %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+    %1 = get_field @fieldName0 : !emitc.array<1xf32>
+    %2 = get_field @fieldName1 : !emitc.array<1xf32>
+    %3 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    return
+  }
+}
+
+// CHECK-LABEL: class modelClass {
+// CHECK-NEXT:   public:
+// CHECK-NEXT:    float fieldName0[1];
+// CHECK-NEXT:    float fieldName1[1];
+// CHECK-NEXT:    void execute() {
+// CHECK-NEXT:     size_t v1 = 0;
+// CHECK-NEXT:     return;
+// CHECK-NEXT:    }
+// CHECK-NEXT:  };
+
+emitc.class final @finalClass {
+  emitc.field @fieldName0 : !emitc.array<1xf32>
+  emitc.field @fieldName1 : !emitc.array<1xf32>
+  emitc.func @execute() {
+    %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
+    %1 = get_field @fieldName0 : !emitc.array<1xf32>
+    %2 = get_field @fieldName1 : !emitc.array<1xf32>
+    %3 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
+    return
+  }
+}
+
+// CHECK-LABEL: class finalClass final {
+// CHECK-NEXT:   public:
+// CHECK-NEXT:    float fieldName0[1];
+// CHECK-NEXT:    float fieldName1[1];
+// CHECK-NEXT:    void execute() {
+// CHECK-NEXT:     size_t v1 = 0;
+// CHECK-NEXT:     return;
+// CHECK-NEXT:    }
+// CHECK-NEXT:  };
+
+emitc.class @mainClass {
+  emitc.field @fieldName0 : !emitc.array<2xf32> = dense<0.0> {attrs = {emitc.name_hint = "another_feature"}}
+  emitc.func @get_fieldName0() {
+    %0 = emitc.get_field @fieldName0 : !emitc.array<2xf32>
+    return
+  }
+}
+
+// CHECK-LABEL: class mainClass {
+// CHECK-NEXT:   public:
+// CHECK-NEXT:    float fieldName0[2] = {0.0e+00f, 0.0e+00f};
+// CHECK-NEXT:    void get_fieldName0() {
+// CHECK-NEXT:     return;
+// CHECK-NEXT:    }
+// CHECK-NEXT:  };
+
+emitc.class @reflectionClass {
+  emitc.field @reflectionMap : !emitc.opaque<"const std::map<std::string, std::string>"> = #emitc.opaque<"{ { \22another_feature\22, \22fieldName0\22 } }">
+  emitc.func @get_reflectionMap() {
+    %0 = emitc.get_field @reflectionMap : !emitc.opaque<"const std::map<std::string, std::string>">
+    return
+  }
+}
+
+// CHECK-LABEL: class reflectionClass {
+// CHECK-NEXT:   public:
+// CHECK-NEXT:    const std::map<std::string, std::string> reflectionMap = { { "another_feature", "fieldName0" } };
+// CHECK-NEXT:    void get_reflectionMap() {
+// CHECK-NEXT:     return;
+// CHECK-NEXT:    }
+// CHECK-NEXT:  };
+
diff --git a/mlir/test/Target/Cpp/const.mlir b/mlir/test/Target/Cpp/const.mlir
index d3656f830c48c..2a5ff1a0f0177 100644
--- a/mlir/test/Target/Cpp/const.mlir
+++ b/mlir/test/Target/Cpp/const.mlir
@@ -16,6 +16,8 @@ func.func @emitc_constant() {
   %c8 = "emitc.constant"(){value = dense<0> : tensor<i32>} : () -> tensor<i32>
   %c9 = "emitc.constant"(){value = dense<[0, 1]> : tensor<2xindex>} : () -> tensor<2xindex>
   %c10 = "emitc.constant"(){value = dense<[[0.0, 1.0], [2.0, 3.0]]> : tensor<2x2xf32>} : () -> tensor<2x2xf32>
+  %c11 = "emitc.constant"(){value = dense<[0, 1]> : !emitc.array<2xindex>} : () -> !emitc.array<2xindex>
+  %c12 = "emitc.constant"(){value = dense<[0.0, 1.0]> : !emitc.array<2xf32>} : () -> !emitc.array<2xf32>
   return
 }
 // CPP-DEFAULT: void emitc_constant() {
@@ -33,6 +35,8 @@ func.func @emitc_constant() {
 // CPP-DEFAULT-NEXT: Tensor<int32_t> [[V8:[^ ]*]] = {0};
 // CPP-DEFAULT-NEXT: Tensor<size_t, 2> [[V9:[^ ]*]] = {0, 1};
 // CPP-DEFAULT-NEXT: Tensor<float, 2, 2> [[V10:[^ ]*]] = {0.0e+00f, 1.000000000e+00f, 2.000000000e+00f, 3.000000000e+00f};
+// CPP-DEFAULT-NEXT: size_t [[V11:[^ ]*]][2] = {0, 1};
+// CPP-DEFAULT-NEXT: float [[V12:[^ ]*]][2] = {0.0e+00f, 1.000000000e+00f};
 
 // CPP-DECLTOP: void emitc_constant() {
 // CPP-DECLTOP-NEXT: int32_t [[V0:[^ ]*]];
@@ -49,6 +53,8 @@ func.func @emitc_constant() {
 // CPP-DECLTOP-NEXT: Tensor<int32_t> [[V8:[^ ]*]];
 // CPP-DECLTOP-NEXT: Tensor<size_t, 2> [[V9:[^ ]*]];
 // CPP-DECLTOP-NEXT: Tensor<float, 2, 2> [[V10:[^ ]*]];
+// CPP-DECLTOP-NEXT: size_t [[V11:[^ ]*]][2];
+// CPP-DECLTOP-NEXT: float [[V12:[^ ]*]][2];
 // CPP-DECLTOP-NEXT: [[V0]] = INT_MAX;
 // CPP-DECLTOP-NEXT: [[V1]] = 42;
 // CPP-DECLTOP-NEXT: [[V2]] = -1;
@@ -63,3 +69,5 @@ func.func @emitc_constant() {
 // CPP-DECLTOP-NEXT: [[V8]] = {0};
 // CPP-DECLTOP-NEXT: [[V9]] = {0, 1};
 // CPP-DECLTOP-NEXT: [[V10]] = {0.0e+00f, 1.000000000e+00f, 2.000000000e+00f, 3.000000000e+00f};
+// CPP-DECLTOP-NEXT: [[V11]] = {0, 1};
+// CPP-DECLTOP-NEXT: [[V12]] = {0.0e+00f, 1.000000000e+00f};
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic-prefer-unregistered.ll b/mlir/test/Target/LLVMIR/Import/intrinsic-prefer-unregistered.ll
index 797a75cceb3de..18c93191e3ae0 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic-prefer-unregistered.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic-prefer-unregistered.ll
@@ -3,9 +3,9 @@
 ; CHECK-LABEL: llvm.func @lifetime
 define void @lifetime() {
   %a = alloca [16 x i8]
-  ; CHECK: llvm.call_intrinsic "llvm.lifetime.start.p0"({{.*}}, %[[ptr:.*]]) : (i64, !llvm.ptr {llvm.nonnull}) -> ()
-  call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %a)
-  ; CHECK: llvm.call_intrinsic "llvm.lifetime.end.p0"({{.*}}, %[[ptr]]) : (i64, !llvm.ptr {llvm.nonnull}) -> ()
-  call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %a)
+  ; CHECK: llvm.call_intrinsic "llvm.lifetime.start.p0"(%[[ptr:.*]]) : (!llvm.ptr {llvm.nonnull}) -> ()
+  call void @llvm.lifetime.start.p0(ptr nonnull %a)
+  ; CHECK: llvm.call_intrinsic "llvm.lifetime.end.p0"(%[[ptr]]) : (!llvm.ptr {llvm.nonnull}) -> ()
+  call void @llvm.lifetime.end.p0(ptr nonnull %a)
   ret void
 }
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index a419d75be24a6..07d22120153fe 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -545,6 +545,15 @@ define void @masked_expand_compress_intrinsics(ptr %0, <7 x i1> %1, <7 x float>
   ret void
 }
 
+; CHECK-LABEL:  llvm.func @masked_expand_compress_intrinsics_with_alignment
+define void @masked_expand_compress_intrinsics_with_alignment(ptr %0, <7 x i1> %1, <7 x float> %2) {
+  ; CHECK: %[[val1:.+]] = "llvm.intr.masked.expandload"(%{{.*}}, %{{.*}}, %{{.*}}) <{arg_attrs = [{llvm.align = 8 : i64}, {}, {}]}> : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> vector<7xf32>
+  %4 = call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %0, <7 x i1> %1, <7 x float> %2)
+  ; CHECK: "llvm.intr.masked.compressstore"(%[[val1]], %{{.*}}, %{{.*}}) <{arg_attrs = [{}, {llvm.align = 8 : i64}, {}]}> : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+  call void @llvm.masked.compressstore.v7f32(<7 x float> %4, ptr align 8 %0, <7 x i1> %1)
+  ret void
+}
+
 ; CHECK-LABEL:  llvm.func @annotate_intrinsics
 define void @annotate_intrinsics(ptr %var, ptr %ptr, i16 %int, ptr %annotation, ptr %fileName, i32 %line, ptr %args) {
   ; CHECK: "llvm.intr.var.annotation"(%{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}, %{{.*}}) : (!llvm.ptr, !llvm.ptr, !llvm.ptr, i32, !llvm.ptr) -> ()
@@ -876,10 +885,10 @@ define void @stack_restore(ptr %0, ptr addrspace(1) %1) {
 ; CHECK-LABEL: llvm.func @lifetime
 define void @lifetime() {
   %a = alloca [16 x i8]
-  ; CHECK: llvm.intr.lifetime.start 16, %{{.*}} : !llvm.ptr
-  call void @llvm.lifetime.start.p0(i64 16, ptr %a)
-  ; CHECK: llvm.intr.lifetime.end 32, %{{.*}} : !llvm.ptr
-  call void @llvm.lifetime.end.p0(i64 32, ptr %a)
+  ; CHECK: llvm.intr.lifetime.start %{{.*}} : !llvm.ptr
+  call void @llvm.lifetime.start.p0(ptr %a)
+  ; CHECK: llvm.intr.lifetime.end %{{.*}} : !llvm.ptr
+  call void @llvm.lifetime.end.p0(ptr %a)
   ret void
 }
 
@@ -1353,8 +1362,8 @@ declare <8 x i64> @llvm.vp.fptoui.v8i64.v8f64(<8 x double>, <8 x i1>, i32)
 declare <8 x i64> @llvm.vp.fptosi.v8i64.v8f64(<8 x double>, <8 x i1>, i32)
 declare <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0(<8 x ptr>, <8 x i1>, i32)
 declare <8 x ptr> @llvm.vp.inttoptr.v8p0.v8i64(<8 x i64>, <8 x i1>, i32)
-declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare void @llvm.lifetime.start.p0(ptr nocapture)
+declare void @llvm.lifetime.end.p0(ptr nocapture)
 declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture)
 declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr nocapture)
 declare ptr @llvm.launder.invariant.group.p0(ptr nocapture)
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index eb3510ca7317d..c99dde36f5ccb 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -577,6 +577,17 @@ llvm.func @masked_expand_compress_intrinsics(%ptr: !llvm.ptr, %mask: vector<7xi1
   llvm.return
 }
 
+// CHECK-LABEL: @masked_expand_compress_intrinsics_with_alignment
+llvm.func @masked_expand_compress_intrinsics_with_alignment(%ptr: !llvm.ptr, %mask: vector<7xi1>, %passthru: vector<7xf32>) {
+  // CHECK: call <7 x float> @llvm.masked.expandload.v7f32(ptr align 8 %{{.*}}, <7 x i1> %{{.*}}, <7 x float> %{{.*}})
+  %0 = "llvm.intr.masked.expandload"(%ptr, %mask, %passthru) {arg_attrs = [{llvm.align = 8 : i32}, {}, {}]}
+    : (!llvm.ptr, vector<7xi1>, vector<7xf32>) -> (vector<7xf32>)
+  // CHECK: call void @llvm.masked.compressstore.v7f32(<7 x float> %{{.*}}, ptr align 8 %{{.*}}, <7 x i1> %{{.*}})
+  "llvm.intr.masked.compressstore"(%0, %ptr, %mask) {arg_attrs = [{}, {llvm.align = 8 : i32}, {}]}
+    : (vector<7xf32>, !llvm.ptr, vector<7xi1>) -> ()
+  llvm.return
+}
+
 // CHECK-LABEL: @annotate_intrinsics
 llvm.func @annotate_intrinsics(%var: !llvm.ptr, %int: i16, %ptr: !llvm.ptr, %annotation: !llvm.ptr, %fileName: !llvm.ptr, %line: i32, %attr: !llvm.ptr) {
   // CHECK: call void @llvm.var.annotation.p0.p0(ptr %{{.*}}, ptr %{{.*}}, ptr %{{.*}}, i32 %{{.*}}, ptr %{{.*}})
@@ -1104,9 +1115,9 @@ llvm.func @lifetime() {
   %c = llvm.mlir.constant(16 : i64) : i64
   %a = llvm.alloca %c x i8 : (i64) -> !llvm.ptr
   // CHECK: call void @llvm.lifetime.start
-  llvm.intr.lifetime.start 16, %a : !llvm.ptr
+  llvm.intr.lifetime.start %a : !llvm.ptr
   // CHECK: call void @llvm.lifetime.end
-  llvm.intr.lifetime.end 16, %a : !llvm.ptr
+  llvm.intr.lifetime.end %a : !llvm.ptr
   llvm.return
 }
 
@@ -1418,8 +1429,8 @@ llvm.func @experimental_constrained_fpext(%s: f32, %v: vector<4xf32>) {
 // CHECK-DAG: declare <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32>, i64 immarg)
 // CHECK-DAG: declare { <2 x double>, <2 x double> } @llvm.vector.deinterleave2.v4f64(<4 x double>)
 // CHECK-DAG: declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave2.nxv8i32(<vscale x 8 x i32>)
-// CHECK-DAG: declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none))
-// CHECK-DAG: declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none))
+// CHECK-DAG: declare void @llvm.lifetime.start.p0(ptr captures(none))
+// CHECK-DAG: declare void @llvm.lifetime.end.p0(ptr captures(none))
 // CHECK-DAG: declare ptr @llvm.invariant.start.p0(i64 immarg, ptr captures(none))
 // CHECK-DAG: declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr captures(none))
 
diff --git a/mlir/test/Target/LLVMIR/nvvm/tma_prefetch.mlir b/mlir/test/Target/LLVMIR/nvvm/tma_prefetch.mlir
index bfd952636ffbe..536b52b034db8 100644
--- a/mlir/test/Target/LLVMIR/nvvm/tma_prefetch.mlir
+++ b/mlir/test/Target/LLVMIR/nvvm/tma_prefetch.mlir
@@ -1,70 +1,123 @@
 // RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
 
-// CHECK-LABEL: @tma_bulk_prefetch
 llvm.func @tma_bulk_prefetch(%src : !llvm.ptr<1>, %size : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.prefetch.L2(ptr addrspace(1) %{{.*}}, i32 %{{.*}}, i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.prefetch.L2(ptr addrspace(1) %{{.*}}, i32 %{{.*}}, i64 %{{.*}}, i1 true)
+  // CHECK-LABEL: define void @tma_bulk_prefetch(ptr addrspace(1) %0, i32 %1, i64 %2) {
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.prefetch.L2(ptr addrspace(1) %0, i32 %1, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.prefetch.L2(ptr addrspace(1) %0, i32 %1, i64 %2, i1 true)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
   nvvm.cp.async.bulk.prefetch %src, %size : !llvm.ptr<1>
   nvvm.cp.async.bulk.prefetch %src, %size l2_cache_hint = %ch : !llvm.ptr<1>
   llvm.return
 }
 
-// CHECK-LABEL: @tma_prefetch_1d
 llvm.func @tma_prefetch_1d(%tma_desc : !llvm.ptr, %d0 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.1d(ptr %0, i32 %{{.*}}, i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.1d(ptr %0, i32 %{{.*}}, i64 %{{.*}}, i1 true)
+  // CHECK-LABEL: define void @tma_prefetch_1d(ptr %0, i32 %1, i64 %2) {
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.1d(ptr %0, i32 %1, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.1d(ptr %0, i32 %1, i64 %2, i1 true)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
   nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0] : !llvm.ptr
   nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0] l2_cache_hint = %ch : !llvm.ptr
   llvm.return
 }
 
-// CHECK-LABEL: @tma_prefetch_2d
 llvm.func @tma_prefetch_2d(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.2d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.2d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i64 %{{.*}}, i1 true)
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1] : !llvm.ptr
+  // CHECK-LABEL: define void @tma_prefetch_2d(ptr %0, i32 %1, i32 %2, i64 %3) {
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.2d(ptr %0, i32 %1, i32 %2, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.2d(ptr %0, i32 %1, i32 %2, i64 %3, i1 true)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1] {mode = #nvvm.tma_load_mode<tile>} : !llvm.ptr
   nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1] l2_cache_hint = %ch : !llvm.ptr
   llvm.return
 }
 
-// CHECK-LABEL: @tma_prefetch_3d
-llvm.func @tma_prefetch_3d(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %off0 : i16, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.3d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.3d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i64 %{{.*}}, i1 true)
+llvm.func @tma_prefetch_3d(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %off0 : i16, %off1 : i16, %ch : i64) {
+  // CHECK-LABEL: define void @tma_prefetch_3d(ptr %0, i32 %1, i32 %2, i32 %3, i16 %4, i16 %5, i64 %6) {
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.3d(ptr %0, i32 %1, i32 %2, i32 %3, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.3d(ptr %0, i32 %1, i32 %2, i32 %3, i64 %6, i1 true)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.3d(ptr %0, i32 %1, i32 %2, i32 %3, i16 %4, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.3d(ptr %0, i32 %1, i32 %2, i32 %3, i16 %4, i64 %6, i1 true)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.3d(ptr %0, i32 %1, i32 %2, i32 %3, i16 %4, i16 %5, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.3d(ptr %0, i32 %1, i32 %2, i32 %3, i16 %4, i16 %5, i64 %6, i1 true)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.3d(ptr %0, i32 %1, i32 %2, i32 %3, i16 %4, i16 %5, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.3d(ptr %0, i32 %1, i32 %2, i32 %3, i16 %4, i16 %5, i64 %6, i1 true)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
   nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] : !llvm.ptr
   nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] l2_cache_hint = %ch : !llvm.ptr
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.3d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i16 %{{.*}}, i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.3d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i16 %{{.*}}, i64 %{{.*}}, i1 true)
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] im2col[%off0] : !llvm.ptr
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] im2col[%off0] l2_cache_hint = %ch : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] im2col[%off0] {mode = #nvvm.tma_load_mode<im2col>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] im2col[%off0] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col>} : !llvm.ptr
+
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] im2col[%off0, %off1] {mode = #nvvm.tma_load_mode<im2col_w>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] im2col[%off0, %off1] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col_w>} : !llvm.ptr
+
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] im2col[%off0, %off1] {mode = #nvvm.tma_load_mode<im2col_w_128>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] im2col[%off0, %off1] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col_w_128>} : !llvm.ptr
   llvm.return
 }
 
-// CHECK-LABEL: @tma_prefetch_4d
 llvm.func @tma_prefetch_4d(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %d3 : i32, %off0 : i16, %off1 : i16, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.4d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.4d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i64 %{{.*}}, i1 true)
+  // CHECK-LABEL: define void @tma_prefetch_4d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i16 %5, i16 %6, i64 %7) {
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.4d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.4d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i64 %7, i1 true)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.4d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i16 %5, i16 %6, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.4d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i16 %5, i16 %6, i64 %7, i1 true)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.4d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i16 %5, i16 %6, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.4d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i16 %5, i16 %6, i64 %7, i1 true)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.4d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i16 %5, i16 %6, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.4d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i16 %5, i16 %6, i64 %7, i1 true)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
   nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] : !llvm.ptr
   nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] l2_cache_hint = %ch : !llvm.ptr
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.4d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}, i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.4d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}, i64 %{{.*}}, i1 true)
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] im2col[%off0, %off1] : !llvm.ptr
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] im2col[%off0, %off1] l2_cache_hint = %ch : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] im2col[%off0, %off1] {mode = #nvvm.tma_load_mode<im2col>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] im2col[%off0, %off1] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col>} : !llvm.ptr
+
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] im2col[%off0, %off1] {mode = #nvvm.tma_load_mode<im2col_w>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] im2col[%off0, %off1] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col_w>} : !llvm.ptr
+
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] im2col[%off0, %off1] {mode = #nvvm.tma_load_mode<im2col_w_128>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] im2col[%off0, %off1] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col_w_128>} : !llvm.ptr
   llvm.return
 }
 
-// CHECK-LABEL: @tma_prefetch_5d
 llvm.func @tma_prefetch_5d(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %d3 : i32, %d4 : i32, %off0 : i16, %off1 : i16, %off2 : i16, %ch : i64) {
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.5d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.5d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i64 %{{.*}}, i1 true)
+  // CHECK-LABEL: define void @tma_prefetch_5d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9) {
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.5d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.5d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i64 %9, i1 true)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.5d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.5d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i16 %8, i64 %9, i1 true)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.5d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.5d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %9, i1 true)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.5d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.w.128.5d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i16 %6, i16 %7, i64 %9, i1 true)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
   nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] : !llvm.ptr
   nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] l2_cache_hint = %ch : !llvm.ptr
 
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.5d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}, i64 0, i1 false)
-  // CHECK: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.im2col.5d(ptr %0, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i32 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}, i16 %{{.*}}, i64 %{{.*}}, i1 true)
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1, %off2] : !llvm.ptr
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1, %off2] l2_cache_hint = %ch : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1, %off2] {mode = #nvvm.tma_load_mode<im2col>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1, %off2] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col>} : !llvm.ptr
+
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1] {mode = #nvvm.tma_load_mode<im2col_w>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col_w>} : !llvm.ptr
+
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1] {mode = #nvvm.tma_load_mode<im2col_w_128>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col_w_128>} : !llvm.ptr
+  llvm.return
+}
+
+llvm.func @tma_prefetch_gather4_2d(%tma_desc : !llvm.ptr, %x0 : i32, %y1 : i32, %y2 : i32, %y3 : i32, %y4 : i32, %ch : i64) {
+  // CHECK-LABEL: define void @tma_prefetch_gather4_2d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i64 %6) {
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.gather4.2d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i64 0, i1 false)
+  // CHECK-NEXT: call void @llvm.nvvm.cp.async.bulk.tensor.prefetch.tile.gather4.2d(ptr %0, i32 %1, i32 %2, i32 %3, i32 %4, i32 %5, i64 %6, i1 true)
+  // CHECK-NEXT: ret void
+  // CHECK-NEXT: }
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%x0, %y1, %y2, %y3, %y4] {mode = #nvvm.tma_load_mode<tile_gather4>} : !llvm.ptr
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%x0, %y1, %y2, %y3, %y4] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<tile_gather4>} : !llvm.ptr
   llvm.return
 }
diff --git a/mlir/test/Target/LLVMIR/nvvm/tma_prefetch_invalid.mlir b/mlir/test/Target/LLVMIR/nvvm/tma_prefetch_invalid.mlir
new file mode 100644
index 0000000000000..23e47bd594b78
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/nvvm/tma_prefetch_invalid.mlir
@@ -0,0 +1,56 @@
+// RUN: mlir-translate -verify-diagnostics -split-input-file -mlir-to-llvmir %s
+
+llvm.func @tma_prefetch_0d(%tma_desc : !llvm.ptr, %d0 : i32, %ch : i64) {
+  // expected-error @below {{expects coordinates between 1 to 5 dimension}}
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[] : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+llvm.func @tma_prefetch_2d_im2col(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %off0 : i16, %ch : i64) {
+  // expected-error @below {{to use im2col mode, the tensor has to be at least 3-dimensional}}
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1] im2col[%off0] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<im2col>} : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+llvm.func @tma_prefetch_5d_im2col(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %d3 : i32, %d4 : i32, %off0 : i16, %off1 : i16, %off2 : i16, %ch : i64) {
+  // expected-error @below {{im2col offsets expected 3 (provided 2)}}
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1] {mode = #nvvm.tma_load_mode<im2col>} : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+llvm.func @tma_prefetch_3d_im2col_w(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %off0 : i16) {
+  // expected-error @below {{im2col offsets expected 2 (provided 1)}}
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2] im2col[%off0] {mode = #nvvm.tma_load_mode<im2col_w>} : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+llvm.func @tma_prefetch_4d_im2col_w_128(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %d3 : i32, %off0 : i16) {
+  // expected-error @below {{im2col offsets expected 2 (provided 1)}}
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3] im2col[%off0] {mode = #nvvm.tma_load_mode<im2col_w_128>} : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+llvm.func @tma_prefetch_gather4_3d(%tma_desc : !llvm.ptr, %d0 : i32) {
+  // expected-error @below {{Gather4 mode expects 5 coordinates}}
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0] {mode = #nvvm.tma_load_mode<tile_gather4>} : !llvm.ptr
+  llvm.return
+}
+
+// -----
+
+llvm.func @tma_prefetch_gather4_2d(%tma_desc : !llvm.ptr, %x0 : i32, %y1 : i32, %y2 : i32, %y3 : i32, %y4 : i32, %off0 : i16, %ch : i64) {
+  // expected-error @below {{im2col offsets expected 0 (provided 1)}}
+  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%x0, %y1, %y2, %y3, %y4] im2col[%off0] l2_cache_hint = %ch {mode = #nvvm.tma_load_mode<tile_gather4>} : !llvm.ptr
+  llvm.return
+}
+
diff --git a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
index 85478cc160064..db36afd6f22dd 100644
--- a/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir-invalid.mlir
@@ -1,5 +1,24 @@
 // RUN: mlir-translate -verify-diagnostics -split-input-file -mlir-to-llvmir %s
 
+llvm.func @pmevent_no_id() {
+  // expected-error @below {{either `id` or `mask` must be set}}
+  nvvm.pmevent 
+}
+
+// -----
+
+llvm.func @pmevent_bigger15() {
+  // expected-error @below {{`id` must be between 0 and 15}}
+  nvvm.pmevent id  = 141
+}
+
+// -----
+
+llvm.func @pmevent_many_ids() {
+  // expected-error @below {{`id` and `mask` cannot be set at the same time}}
+  nvvm.pmevent id = 1 mask = 1
+}
+
 // -----
 
 llvm.func @kernel_func(%numberOfThreads : i32) {
@@ -37,57 +56,49 @@ llvm.func @kernel_func() attributes {nvvm.kernel, nvvm.maxntid = array<i32: 3, 4
 
 // -----
 
-llvm.func @nvvm_fence_proxy_acquire(%addr : !llvm.ptr, %size : i32) {
-  // expected-error @below {{'nvvm.fence.proxy.acquire' op uni-directional proxies only support generic for from_proxy attribute}}
-  nvvm.fence.proxy.acquire #nvvm.mem_scope<cta> %addr, %size from_proxy=#nvvm.proxy_kind<tensormap> to_proxy=#nvvm.proxy_kind<generic>
+// expected-error @below {{'"nvvm.blocksareclusters"' attribute must be used along with 'nvvm.reqntid' and 'nvvm.cluster_dim'}}
+llvm.func @kernel_func() attributes {nvvm.kernel, nvvm.blocksareclusters,
+                                     nvvm.cluster_dim = array<i32: 3, 5, 7>} {
   llvm.return
 }
 
 // -----
 
-llvm.func @nvvm_fence_proxy_release() {
-  // expected-error @below {{'nvvm.fence.proxy.release' op uni-directional proxies only support generic for from_proxy attribute}}
-  nvvm.fence.proxy.release #nvvm.mem_scope<cta> from_proxy=#nvvm.proxy_kind<tensormap> to_proxy=#nvvm.proxy_kind<generic>
+// expected-error @below {{'"nvvm.blocksareclusters"' attribute must be used along with 'nvvm.reqntid' and 'nvvm.cluster_dim'}}
+llvm.func @kernel_func() attributes {nvvm.kernel, nvvm.blocksareclusters,
+                                     nvvm.reqntid = array<i32: 1, 23, 32>} {
   llvm.return
 }
 
 // -----
 
 llvm.func @nvvm_fence_proxy_acquire(%addr : !llvm.ptr, %size : i32) {
-  // expected-error @below {{'nvvm.fence.proxy.acquire' op uni-directional proxies only support tensormap for to_proxy attribute}}
-  nvvm.fence.proxy.acquire #nvvm.mem_scope<cta> %addr, %size  from_proxy=#nvvm.proxy_kind<generic> to_proxy=#nvvm.proxy_kind<generic>
+  // expected-error @below {{'nvvm.fence.proxy.acquire' op uni-directional proxies only support generic for from_proxy attribute}}
+  nvvm.fence.proxy.acquire #nvvm.mem_scope<cta> %addr, %size from_proxy=#nvvm.proxy_kind<tensormap> to_proxy=#nvvm.proxy_kind<generic>
   llvm.return
 }
 
 // -----
 
 llvm.func @nvvm_fence_proxy_release() {
-  // expected-error @below {{'nvvm.fence.proxy.release' op uni-directional proxies only support tensormap for to_proxy attribute}}
-  nvvm.fence.proxy.release #nvvm.mem_scope<cta> from_proxy=#nvvm.proxy_kind<generic> to_proxy=#nvvm.proxy_kind<generic>
-  llvm.return
-}
-
-// -----
-
-llvm.func @tma_prefetch_0d(%tma_desc : !llvm.ptr, %d0 : i32, %ch : i64) {
-  // expected-error @below {{expects coordinates between 1 to 5 dimension}}
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[] : !llvm.ptr
+  // expected-error @below {{'nvvm.fence.proxy.release' op uni-directional proxies only support generic for from_proxy attribute}}
+  nvvm.fence.proxy.release #nvvm.mem_scope<cta> from_proxy=#nvvm.proxy_kind<tensormap> to_proxy=#nvvm.proxy_kind<generic>
   llvm.return
 }
 
 // -----
 
-llvm.func @tma_prefetch_2d_im2col(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %off0 : i16, %ch : i64) {
-  // expected-error @below {{to use im2col mode, the tensor has to be at least 3-dimensional}}
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1] im2col[%off0] l2_cache_hint = %ch : !llvm.ptr
+llvm.func @nvvm_fence_proxy_acquire(%addr : !llvm.ptr, %size : i32) {
+  // expected-error @below {{'nvvm.fence.proxy.acquire' op uni-directional proxies only support tensormap for to_proxy attribute}}
+  nvvm.fence.proxy.acquire #nvvm.mem_scope<cta> %addr, %size  from_proxy=#nvvm.proxy_kind<generic> to_proxy=#nvvm.proxy_kind<generic>
   llvm.return
 }
 
 // -----
 
-llvm.func @tma_prefetch_5d_im2col(%tma_desc : !llvm.ptr, %d0 : i32, %d1 : i32, %d2 : i32, %d3 : i32, %d4 : i32, %off0 : i16, %off1 : i16, %off2 : i16, %ch : i64) {
-  // expected-error @below {{im2col offsets must be 2 less than number of coordinates}}
-  nvvm.cp.async.bulk.tensor.prefetch %tma_desc, box[%d0, %d1, %d2, %d3, %d4] im2col[%off0, %off1] : !llvm.ptr
+llvm.func @nvvm_fence_proxy_release() {
+  // expected-error @below {{'nvvm.fence.proxy.release' op uni-directional proxies only support tensormap for to_proxy attribute}}
+  nvvm.fence.proxy.release #nvvm.mem_scope<cta> from_proxy=#nvvm.proxy_kind<generic> to_proxy=#nvvm.proxy_kind<generic>
   llvm.return
 }
 
@@ -351,3 +362,136 @@ llvm.func @st_matrix(%arg0: !llvm.ptr<3>, %r1: i32, %r2: i32, %r3: i32, %r4: i32
   nvvm.stmatrix %arg0, %r1 {layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b8>} : !llvm.ptr<3>, i32
   llvm.return
 }
+
+// -----
+
+llvm.func @st_matrix(%arg0: !llvm.ptr<3>, %r1: i32, %r2: i32, %r3: i32, %r4: i32) {
+  // expected-error@+1 {{'nvvm.stmatrix' op expected num attribute to be 1, 2 or 4}}
+  nvvm.stmatrix %arg0, %r1, %r2, %r3 {layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b16>} : !llvm.ptr<3>, i32, i32, i32
+  llvm.return
+}
+
+// -----
+
+llvm.func @st_matrix(%arg0: !llvm.ptr<3>, %r1: i32, %r2: i32, %r3: i32, %r4: i32) {
+  // expected-error@+1 {{'nvvm.stmatrix' op expected shape to be 8x8 or 16x8}}
+  nvvm.stmatrix %arg0, %r1 {layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b16>} : !llvm.ptr<3>, i32
+  llvm.return
+}
+
+// -----
+
+llvm.func @st_matrix(%arg0: !llvm.ptr<3>, %r1: i32, %r2: i32, %r3: i32, %r4: i32) {
+  // expected-error@+1 {{'nvvm.stmatrix' op expected element type to be B16 for 8x8 matrix}}
+  nvvm.stmatrix %arg0, %r1 {layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b8>} : !llvm.ptr<3>, i32
+  llvm.return
+}
+// -----
+
+llvm.func @st_matrix(%arg0: !llvm.ptr<3>, %r1: i32, %r2: i32, %r3: i32, %r4: i32) {
+  // expected-error@+1 {{'nvvm.stmatrix' op expected element type to be B8 for 16x8 matrix}}
+  nvvm.stmatrix %arg0, %r1 {layout = #nvvm.mma_layout<col>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b16>} : !llvm.ptr<3>, i32
+  llvm.return
+}
+
+// -----
+
+llvm.func @st_matrix(%arg0: !llvm.ptr<3>, %r1: i32, %r2: i32, %r3: i32, %r4: i32) {
+  // expected-error@+1 {{'nvvm.stmatrix' op expected layout to be col for 16x8 matrix}}
+  nvvm.stmatrix %arg0, %r1 {layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b8>} : !llvm.ptr<3>, i32
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected num attribute to be 1, 2 or 4 for 8x8 matrix}}
+  %l = nvvm.ldmatrix %arg0 {num = 3 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType  = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> i32
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected destination type is i32}}
+  %l = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType  = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> !llvm.struct<(i32)>
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected destination type is a structure of 4 elements of type i32}}
+  %l = nvvm.ldmatrix %arg0 {num = 4 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType  = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected element type to be b16 for 8x8 matrix}}
+  %l = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType  = #nvvm.ld_st_matrix_elt_type<b8>} : (!llvm.ptr<3>) -> i32
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected num attribute to be 1, 2 or 4 for 8x16 matrix}}
+  %l = nvvm.ldmatrix %arg0 {num = 3 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 16>, eltType  = #nvvm.ld_st_matrix_elt_type<b8x16.b4x16_p64>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32)>
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected layout to be row for 8x16 matrix}}
+  %l = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<col>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 16>, eltType  = #nvvm.ld_st_matrix_elt_type<b8x16.b4x16_p64>} : (!llvm.ptr<3>) -> i32
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected element type to be b8x16.b4x16_p64 or b8x16.b6x16_p32 for 8x16 matrix}}
+  %l = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 16>, eltType  = #nvvm.ld_st_matrix_elt_type<b8>} : (!llvm.ptr<3>) -> i32
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected num attribute to be 1 or 2 for 16x16 matrix}}
+  %l = nvvm.ldmatrix %arg0 {num = 4 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 16>, eltType  = #nvvm.ld_st_matrix_elt_type<b8x16.b4x16_p64>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected layout to be col for 16x16 matrix}}
+  %l = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 16>, eltType  = #nvvm.ld_st_matrix_elt_type<b8x16.b6x16_p32>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
+  llvm.return
+}
+
+// -----
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected element type to be b8, b8x16.b4x16_p64 or b8x16.b6x16_p32 for 16x16 matrix}}
+  %l = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<col>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 16>, eltType  = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> i32
+  llvm.return
+}
+
+llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
+  // expected-error@+1 {{'nvvm.ldmatrix' op expected destination type is a structure of 2 elements of type i32}}
+  %l = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<col>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 16>, eltType  = #nvvm.ld_st_matrix_elt_type<b8>} : (!llvm.ptr<3>) -> i32
+  llvm.return
+}
+
+// -----
+
+llvm.func @nanosleep() {
+  // expected-error@+1 {{integer constant out of range for attribute}}
+  nvvm.nanosleep 100000000000000
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/nvvmir.mlir b/mlir/test/Target/LLVMIR/nvvmir.mlir
index 5c2cfa4683104..fa7dd1daf96ed 100644
--- a/mlir/test/Target/LLVMIR/nvvmir.mlir
+++ b/mlir/test/Target/LLVMIR/nvvmir.mlir
@@ -64,92 +64,94 @@ llvm.func @nvvm_special_regs() -> i32 {
   %30 = nvvm.read.ptx.sreg.clock64 : i64
   // CHECK: call i64 @llvm.nvvm.read.ptx.sreg.globaltimer
   %31 = nvvm.read.ptx.sreg.globaltimer : i64
-  // CHECK: %32 = call range(i32 0, 64) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
-  %32 = nvvm.read.ptx.sreg.tid.x range <i32, 0, 64> : i32
+  // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.globaltimer.lo()
+  %32 = nvvm.read.ptx.sreg.globaltimer.lo : i32
+  // CHECK: %33 = call range(i32 0, 64) i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+  %33 = nvvm.read.ptx.sreg.tid.x range <i32, 0, 64> : i32
   // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.warpid
-  %33 = nvvm.read.ptx.sreg.warpid : i32
+  %34 = nvvm.read.ptx.sreg.warpid : i32
   // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.nwarpid
-  %34 = nvvm.read.ptx.sreg.nwarpid : i32
+  %35 = nvvm.read.ptx.sreg.nwarpid : i32
   // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.smid
-  %35 = nvvm.read.ptx.sreg.smid : i32
+  %36 = nvvm.read.ptx.sreg.smid : i32
   // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.nsmid
-  %36 = nvvm.read.ptx.sreg.nsmid : i32
+  %37 = nvvm.read.ptx.sreg.nsmid : i32
   // CHECK: call i32 @llvm.nvvm.read.ptx.sreg.gridid
-  %37 = nvvm.read.ptx.sreg.gridid : i32
+  %38 = nvvm.read.ptx.sreg.gridid : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg0
-  %38 = nvvm.read.ptx.sreg.envreg0 : i32
+  %39 = nvvm.read.ptx.sreg.envreg0 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg1
-  %39 = nvvm.read.ptx.sreg.envreg1 : i32
+  %40 = nvvm.read.ptx.sreg.envreg1 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg2
-  %40 = nvvm.read.ptx.sreg.envreg2 : i32
+  %41 = nvvm.read.ptx.sreg.envreg2 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg3
-  %41 = nvvm.read.ptx.sreg.envreg3 : i32
+  %42 = nvvm.read.ptx.sreg.envreg3 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg4
-  %42 = nvvm.read.ptx.sreg.envreg4 : i32
+  %43 = nvvm.read.ptx.sreg.envreg4 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg5
-  %43 = nvvm.read.ptx.sreg.envreg5 : i32
+  %44 = nvvm.read.ptx.sreg.envreg5 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg6
-  %44 = nvvm.read.ptx.sreg.envreg6 : i32
+  %45 = nvvm.read.ptx.sreg.envreg6 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg7
-  %45 = nvvm.read.ptx.sreg.envreg7 : i32
+  %46 = nvvm.read.ptx.sreg.envreg7 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg8
-  %46 = nvvm.read.ptx.sreg.envreg8 : i32
+  %47 = nvvm.read.ptx.sreg.envreg8 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg9
-  %47 = nvvm.read.ptx.sreg.envreg9 : i32
+  %48 = nvvm.read.ptx.sreg.envreg9 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg10
-  %48 = nvvm.read.ptx.sreg.envreg10 : i32
+  %49 = nvvm.read.ptx.sreg.envreg10 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg11
-  %49 = nvvm.read.ptx.sreg.envreg11 : i32
+  %50 = nvvm.read.ptx.sreg.envreg11 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg12
-  %50 = nvvm.read.ptx.sreg.envreg12 : i32
+  %51 = nvvm.read.ptx.sreg.envreg12 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg13
-  %51 = nvvm.read.ptx.sreg.envreg13 : i32
+  %52 = nvvm.read.ptx.sreg.envreg13 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg14
-  %52 = nvvm.read.ptx.sreg.envreg14 : i32
+  %53 = nvvm.read.ptx.sreg.envreg14 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg15
-  %53 = nvvm.read.ptx.sreg.envreg15 : i32
+  %54 = nvvm.read.ptx.sreg.envreg15 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg16
-  %54 = nvvm.read.ptx.sreg.envreg16 : i32
+  %55 = nvvm.read.ptx.sreg.envreg16 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg17
-  %55 = nvvm.read.ptx.sreg.envreg17 : i32
+  %56 = nvvm.read.ptx.sreg.envreg17 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg18
-  %56 = nvvm.read.ptx.sreg.envreg18 : i32
+  %57 = nvvm.read.ptx.sreg.envreg18 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg19
-  %57 = nvvm.read.ptx.sreg.envreg19 : i32
+  %58 = nvvm.read.ptx.sreg.envreg19 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg20
-  %58 = nvvm.read.ptx.sreg.envreg20 : i32
+  %59 = nvvm.read.ptx.sreg.envreg20 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg21
-  %59 = nvvm.read.ptx.sreg.envreg21 : i32
+  %60 = nvvm.read.ptx.sreg.envreg21 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg22
-  %60 = nvvm.read.ptx.sreg.envreg22 : i32
+  %61 = nvvm.read.ptx.sreg.envreg22 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg23
-  %61 = nvvm.read.ptx.sreg.envreg23 : i32
+  %62 = nvvm.read.ptx.sreg.envreg23 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg24
-  %62 = nvvm.read.ptx.sreg.envreg24 : i32
+  %63 = nvvm.read.ptx.sreg.envreg24 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg25
-  %63 = nvvm.read.ptx.sreg.envreg25 : i32
+  %64 = nvvm.read.ptx.sreg.envreg25 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg26
-  %64 = nvvm.read.ptx.sreg.envreg26 : i32
+  %65 = nvvm.read.ptx.sreg.envreg26 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg27
-  %65 = nvvm.read.ptx.sreg.envreg27 : i32
+  %66 = nvvm.read.ptx.sreg.envreg27 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg28
-  %66 = nvvm.read.ptx.sreg.envreg28 : i32
+  %67 = nvvm.read.ptx.sreg.envreg28 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg29
-  %67 = nvvm.read.ptx.sreg.envreg29 : i32
+  %68 = nvvm.read.ptx.sreg.envreg29 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg30
-  %68 = nvvm.read.ptx.sreg.envreg30 : i32
+  %69 = nvvm.read.ptx.sreg.envreg30 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.envreg31
-  %69 = nvvm.read.ptx.sreg.envreg31 : i32
+  %70 = nvvm.read.ptx.sreg.envreg31 : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.eq
-  %70 = nvvm.read.ptx.sreg.lanemask.eq : i32
+  %71 = nvvm.read.ptx.sreg.lanemask.eq : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.le
-  %71 = nvvm.read.ptx.sreg.lanemask.le : i32
+  %72 = nvvm.read.ptx.sreg.lanemask.le : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.lt
-  %72 = nvvm.read.ptx.sreg.lanemask.lt : i32
+  %73 = nvvm.read.ptx.sreg.lanemask.lt : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.ge
-  %73 = nvvm.read.ptx.sreg.lanemask.ge : i32
+  %74 = nvvm.read.ptx.sreg.lanemask.ge : i32
   //CHECK: call i32 @llvm.nvvm.read.ptx.sreg.lanemask.gt
-  %74 = nvvm.read.ptx.sreg.lanemask.gt : i32
+  %75 = nvvm.read.ptx.sreg.lanemask.gt : i32
   llvm.return %1 : i32
 }
 
@@ -559,17 +561,47 @@ llvm.func @llvm_nvvm_cp_async_bulk_wait_group() {
 // CHECK-LABEL: @ld_matrix
 llvm.func @ld_matrix(%arg0: !llvm.ptr<3>) {
   // CHECK: call i32 @llvm.nvvm.ldmatrix.sync.aligned.m8n8.x1.b16.p3(ptr addrspace(3) %{{.*}})
-  %l1 = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr<3>) -> i32
+  %l1 = nvvm.ldmatrix %arg0 {num = 1: i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> i32
   // CHECK: call { i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m8n8.x2.b16.p3(ptr addrspace(3) %{{.*}})
-  %l2 = nvvm.ldmatrix %arg0 {num = 2 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
+  %l2 = nvvm.ldmatrix %arg0 {num = 2 : i32, layout = #nvvm.mma_layout<row>, shape =#nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
   // CHECK: call { i32, i32, i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m8n8.x4.b16.p3(ptr addrspace(3) %{{.*}})
-  %l4 = nvvm.ldmatrix %arg0 {num = 4 : i32, layout = #nvvm.mma_layout<row>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
-   // CHECK: call i32 @llvm.nvvm.ldmatrix.sync.aligned.m8n8.x1.trans.b16.p3(ptr addrspace(3) %{{.*}})
-  %l1t = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<col>} : (!llvm.ptr<3>) -> i32
+  %l4 = nvvm.ldmatrix %arg0 {num = 4 : i32, layout = #nvvm.mma_layout<row>, shape =#nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
+
+  // CHECK: call i32 @llvm.nvvm.ldmatrix.sync.aligned.m8n8.x1.trans.b16.p3(ptr addrspace(3) %{{.*}})
+  %l1t = nvvm.ldmatrix %arg0 {num = 1: i32, layout = #nvvm.mma_layout<col>, shape =#nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> i32
   // CHECK: call { i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m8n8.x2.trans.b16.p3(ptr addrspace(3) %{{.*}})
-  %l2t = nvvm.ldmatrix %arg0 {num = 2 : i32, layout = #nvvm.mma_layout<col>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
+  %l2t = nvvm.ldmatrix %arg0 {num = 2 : i32, layout = #nvvm.mma_layout<col>, shape =#nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
   // CHECK: call { i32, i32, i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m8n8.x4.trans.b16.p3(ptr addrspace(3) %{{.*}})
-  %l4t = nvvm.ldmatrix %arg0 {num = 4 : i32, layout = #nvvm.mma_layout<col>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
+  %l4t = nvvm.ldmatrix %arg0 {num = 4 : i32, layout = #nvvm.mma_layout<col>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 8>, eltType = #nvvm.ld_st_matrix_elt_type<b16>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
+
+  // CHECK: call i32 @llvm.nvvm.ldmatrix.sync.aligned.m8n16.x1.b8x16.b6x16_p32.p3(ptr addrspace(3) %{{.*}})
+  %m8n16_b6_l1 = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>, shape =#nvvm.ld_st_matrix_shape<m = 8, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b8x16.b6x16_p32>} : (!llvm.ptr<3>) -> i32
+  // CHECK: call { i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m8n16.x2.b8x16.b6x16_p32.p3(ptr addrspace(3) %{{.*}})
+  %m8n16_b6_l2 = nvvm.ldmatrix %arg0 {num = 2: i32, layout = #nvvm.mma_layout<row>, shape =#nvvm.ld_st_matrix_shape<m = 8, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b8x16.b6x16_p32>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
+  // CHECK: call { i32, i32, i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m8n16.x4.b8x16.b6x16_p32.p3(ptr addrspace(3) %{{.*}})
+  %m8n16_b6_l4 = nvvm.ldmatrix %arg0{num = 4 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 16>,eltType =#nvvm.ld_st_matrix_elt_type<b8x16.b6x16_p32>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
+
+  // CHECK: call i32 @llvm.nvvm.ldmatrix.sync.aligned.m8n16.x1.b8x16.b4x16_p64.p3(ptr addrspace(3) %{{.*}})
+  %m8n16_b4_l1 = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<row>, shape =#nvvm.ld_st_matrix_shape<m = 8, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b8x16.b4x16_p64>} : (!llvm.ptr<3>) -> i32
+  // CHECK: call { i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m8n16.x2.b8x16.b4x16_p64.p3(ptr addrspace(3) %{{.*}})
+  %m8n16_b4_l2 = nvvm.ldmatrix %arg0 {num = 2 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b8x16.b4x16_p64>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
+  // CHECK: call { i32, i32, i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m8n16.x4.b8x16.b4x16_p64.p3(ptr addrspace(3) %{{.*}})
+  %m8n16_b4_l4 = nvvm.ldmatrix %arg0 {num = 4 : i32, layout = #nvvm.mma_layout<row>, shape = #nvvm.ld_st_matrix_shape<m = 8, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b8x16.b4x16_p64>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
+
+  // CHECK: call { i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m16n16.x1.trans.b8.p3(ptr addrspace(3) %{{.*}})
+  %m16n16_l1t = nvvm.ldmatrix %arg0 {num = 1 : i32, layout = #nvvm.mma_layout<col>, shape =#nvvm.ld_st_matrix_shape<m = 16, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b8>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
+  // CHECK: call { i32, i32, i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m16n16.x2.trans.b8.p3(ptr addrspace(3) %{{.*}})
+  %m16n16_l2t = nvvm.ldmatrix %arg0{num = 2 : i32, layout = #nvvm.mma_layout<col>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 16>,eltType =#nvvm.ld_st_matrix_elt_type<b8>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
+
+  // CHECK: call { i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m16n16.x1.trans.b8x16.b6x16_p32.p3(ptr addrspace(3) %{{.*}})
+  %m16n16_b6_l1t = nvvm.ldmatrix %arg0 {num = 1: i32, layout = #nvvm.mma_layout<col>, shape =#nvvm.ld_st_matrix_shape<m = 16, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b8x16.b6x16_p32>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
+  // CHECK: call { i32, i32, i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m16n16.x2.trans.b8x16.b6x16_p32.p3(ptr addrspace(3) %{{.*}})
+  %m16n16_b6_l2t = nvvm.ldmatrix %arg0 {num = 2 : i32, layout = #nvvm.mma_layout<col>, shape =#nvvm.ld_st_matrix_shape<m = 16, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b8x16.b6x16_p32>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
+
+  // CHECK: call { i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m16n16.x1.trans.b8x16.b4x16_p64.p3(ptr addrspace(3) %{{.*}})
+  %m16n16_b4_l1t = nvvm.ldmatrix %arg0 {num = 1: i32, layout = #nvvm.mma_layout<col>, shape =#nvvm.ld_st_matrix_shape<m = 16, n = 16>, eltType = #nvvm.ld_st_matrix_elt_type<b8x16.b4x16_p64>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32)>
+  // CHECK: call { i32, i32, i32, i32 } @llvm.nvvm.ldmatrix.sync.aligned.m16n16.x2.trans.b8x16.b4x16_p64.p3(ptr addrspace(3) %{{.*}})
+  %m16n16_b4_l2t = nvvm.ldmatrix %arg0 {num = 2 : i32, layout = #nvvm.mma_layout<col>, shape = #nvvm.ld_st_matrix_shape<m = 16, n = 16>,eltType =#nvvm.ld_st_matrix_elt_type<b8x16.b4x16_p64>} : (!llvm.ptr<3>) -> !llvm.struct<(i32, i32, i32, i32)>
   llvm.return
 }
 
@@ -662,7 +694,16 @@ llvm.func @kernel_func() attributes {nvvm.kernel, nvvm.maxntid = array<i32: 1, 2
 
 // CHECK: define ptx_kernel void @kernel_func() #[[ATTR0:[0-9]+]]
 // CHECK: attributes #[[ATTR0]] = { "nvvm.maxnreg"="32" "nvvm.maxntid"="1,23,32" "nvvm.minctasm"="16" }
+// -----
 
+llvm.func @kernel_func() attributes {nvvm.kernel, nvvm.blocksareclusters,
+                                     nvvm.reqntid = array<i32: 1, 23, 32>,
+                                     nvvm.cluster_dim = array<i32: 3, 5, 7>} {
+  llvm.return
+}
+
+// CHECK: define ptx_kernel void @kernel_func() #[[ATTR0:[0-9]+]]
+// CHECK: attributes #[[ATTR0]] = { "nvvm.blocksareclusters" "nvvm.cluster_dim"="3,5,7" "nvvm.reqntid"="1,23,32" }
 // -----
 // CHECK: define ptx_kernel void @kernel_func
 // CHECK: !nvvm.annotations =
@@ -766,7 +807,7 @@ llvm.func @nvvm_wgmma_wait_group_aligned() {
 // CHECK-LABEL: @nvvm_griddepcontrol_wait
 llvm.func @nvvm_griddepcontrol_wait() {
   // CHECK: call void @llvm.nvvm.griddepcontrol.wait()
-  nvvm.griddepcontrol.wait
+  nvvm.griddepcontrol wait
   llvm.return
 }
 
@@ -774,7 +815,7 @@ llvm.func @nvvm_griddepcontrol_wait() {
 // CHECK-LABEL: @nvvm_griddepcontrol_launch_dependents
 llvm.func @nvvm_griddepcontrol_launch_dependents() {
   // CHECK: call void @llvm.nvvm.griddepcontrol.launch.dependents()
-  nvvm.griddepcontrol.launch.dependents
+  nvvm.griddepcontrol launch_dependents
   llvm.return
 }
 
@@ -783,8 +824,8 @@ llvm.func @nvvm_griddepcontrol_launch_dependents() {
 llvm.func @nvvm_mapa(%a: !llvm.ptr, %a_shared: !llvm.ptr<3>, %b : i32) {
   // CHECK-LLVM: call ptr @llvm.nvvm.mapa(ptr %{{.*}}, i32 %{{.*}})
   %0 = nvvm.mapa %a, %b: !llvm.ptr -> !llvm.ptr
-  // CHECK-LLVM: call ptr addrspace(3) @llvm.nvvm.mapa.shared.cluster(ptr addrspace(3) %{{.*}}, i32 %{{.*}})
-  %1 = nvvm.mapa %a_shared, %b: !llvm.ptr<3> -> !llvm.ptr<3>
+  // CHECK-LLVM: call ptr addrspace(7) @llvm.nvvm.mapa.shared.cluster(ptr addrspace(3) %{{.*}}, i32 %{{.*}})
+  %1 = nvvm.mapa %a_shared, %b: !llvm.ptr<3> -> !llvm.ptr<7>
   llvm.return
 }
 
@@ -918,3 +959,23 @@ llvm.func @nvvm_dot_accumulate_2way(%a: vector<2xi16>, %b: vector<4xi8>, %c: i32
   %7 = nvvm.dot.accumulate.2way %a <signed>, %b <signed>, %c {b_hi = true}: vector<2xi16>, vector<4xi8>
   llvm.return
 }
+
+// -----
+
+// CHECK-LABEL: @nvvm_pmevent
+llvm.func @nvvm_pmevent() {
+  // CHECK: call void @llvm.nvvm.pm.event.mask(i16 15000)
+  nvvm.pmevent mask = 15000
+  // CHECK: call void @llvm.nvvm.pm.event.mask(i16 4)
+  nvvm.pmevent mask = 4
+  llvm.return
+}
+
+// -----
+
+// CHECK-LABEL: @nanosleep
+llvm.func @nanosleep() {
+  // CHECK: call void @llvm.nvvm.nanosleep(i32 4000)
+  nvvm.nanosleep 4000
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
new file mode 100644
index 0000000000000..1bc97609ccff4
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/ompenmp-target-allocmem-freemem.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt %s -convert-openmp-to-llvm | mlir-translate -mlir-to-llvmir | FileCheck %s
+
+// This file contains MLIR test cases for omp.target_allocmem and omp.target_freemem
+
+// CHECK-LABEL: test_alloc_free_i64
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 8, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_i64() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, i64
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_1d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 64, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_1d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
+
+// CHECK-LABEL: test_alloc_free_vector_2d_f32
+// CHECK: %[[ALLOC:.*]] = call ptr @omp_target_alloc(i64 1024, i32 0)
+// CHECK: %[[PTRTOINT:.*]] = ptrtoint ptr %[[ALLOC]] to i64
+// CHECK: %[[INTTOPTR:.*]] = inttoptr i64 %[[PTRTOINT]] to ptr
+// CHECK: call void @omp_target_free(ptr %[[INTTOPTR]], i32 0)
+// CHECK: ret void
+llvm.func @test_alloc_free_vector_2d_f32() -> () {
+  %device = llvm.mlir.constant(0 : i32) : i32
+  %1 = omp.target_allocmem %device : i32, vector<16x16xf32>
+  omp.target_freemem %device, %1 : i32, i64
+  llvm.return
+}
diff --git a/mlir/test/Target/LLVMIR/omptarget-atomic-capture-control-options.mlir b/mlir/test/Target/LLVMIR/omptarget-atomic-capture-control-options.mlir
new file mode 100644
index 0000000000000..355390719322f
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-atomic-capture-control-options.mlir
@@ -0,0 +1,44 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK: atomicrmw add ptr %loadgep_, i32 1 monotonic, align 4, !amdgpu.no.remote.memory !{{.*}}
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<!llvm.ptr = dense<64> : vector<4xi64>, !llvm.ptr<1> = dense<64> : vector<4xi64>, !llvm.ptr<2> = dense<32> : vector<4xi64>, !llvm.ptr<3> = dense<32> : vector<4xi64>, !llvm.ptr<4> = dense<64> : vector<4xi64>, !llvm.ptr<5> = dense<32> : vector<4xi64>, !llvm.ptr<6> = dense<32> : vector<4xi64>, !llvm.ptr<7> = dense<[160, 256, 256, 32]> : vector<4xi64>, !llvm.ptr<8> = dense<[128, 128, 128, 48]> : vector<4xi64>, !llvm.ptr<9> = dense<[192, 256, 256, 32]> : vector<4xi64>, i64 = dense<64> : vector<2xi64>, i1 = dense<8> : vector<2xi64>, i8 = dense<8> : vector<2xi64>, i16 = dense<16> : vector<2xi64>, i32 = dense<32> : vector<2xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, f128 = dense<128> : vector<2xi64>, "dlti.endianness" = "little", "dlti.legal_int_widths" = array<i32: 32, 64>, "dlti.stack_alignment" = 32 : i64, "dlti.alloca_memory_space" = 5 : ui64, "dlti.global_memory_space" = 1 : ui64>, fir.atomic_fine_grained_memory, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", fir.target_cpu = "generic-hsa", llvm.data_layout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9", llvm.target_triple = "amdgcn-amd-amdhsa", omp.flags = #omp.flags<openmp_device_version = 31>, omp.is_gpu = true, omp.is_target_device = true, omp.requires = #omp<clause_requires none>, omp.target_triples = [], omp.version = #omp.version<version = 31>} {
+  llvm.func @_QQmain() attributes {fir.bindc_name = "TEST", omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to)>, target_cpu = "generic-hsa"} {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "threads"} : (i64) -> !llvm.ptr<5>
+    %2 = llvm.addrspacecast %1 : !llvm.ptr<5> to !llvm.ptr
+    %3 = llvm.mlir.constant(1 : i64) : i64
+    %4 = llvm.alloca %3 x i32 {bindc_name = "capture"} : (i64) -> !llvm.ptr<5>
+    %5 = llvm.addrspacecast %4 : !llvm.ptr<5> to !llvm.ptr
+    %6 = llvm.mlir.constant(1 : i64) : i64
+    %7 = llvm.alloca %6 x i32 {bindc_name = "a"} : (i64) -> !llvm.ptr<5>
+    %8 = llvm.addrspacecast %7 : !llvm.ptr<5> to !llvm.ptr
+    %9 = llvm.mlir.constant(0 : i32) : i32
+    %10 = llvm.mlir.constant(128 : i32) : i32
+    %11 = llvm.mlir.constant(1 : i64) : i64
+    %12 = llvm.mlir.constant(1 : i64) : i64
+    %13 = llvm.mlir.constant(1 : i64) : i64
+    llvm.store %10, %2 : i32, !llvm.ptr
+    llvm.store %9, %8 : i32, !llvm.ptr
+    %14 = omp.map.info var_ptr(%2 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "threads"}
+    %15 = omp.map.info var_ptr(%5 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "capture"}
+    %16 = omp.map.info var_ptr(%8 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "a"}
+    omp.target map_entries(%14 -> %arg0, %15 -> %arg1, %16 -> %arg2 : !llvm.ptr, !llvm.ptr, !llvm.ptr) {
+      %17 = llvm.mlir.constant(1 : i32) : i32
+      %18 = llvm.load %arg0 : !llvm.ptr -> i32
+      omp.parallel num_threads(%18 : i32) {
+        omp.atomic.capture {
+          omp.atomic.read %arg1 = %arg2 : !llvm.ptr, !llvm.ptr, i32
+          omp.atomic.update %arg2 : !llvm.ptr {
+          ^bb0(%arg3: i32):
+            %19 = llvm.add %arg3, %17 : i32
+            omp.yield(%19 : i32)
+          } {atomic_control = #omp.atomic_control<fine_grained_memory = true>}
+        }
+        omp.terminator
+      }
+      omp.terminator
+    }
+    llvm.return
+  }
+}
diff --git a/mlir/test/Target/LLVMIR/omptarget-atomic-update-control-options.mlir b/mlir/test/Target/LLVMIR/omptarget-atomic-update-control-options.mlir
new file mode 100644
index 0000000000000..3b0005bd20798
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-atomic-update-control-options.mlir
@@ -0,0 +1,36 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// CHECK: atomicrmw add ptr %loadgep_, i32 1 monotonic, align 4, !amdgpu.ignore.denormal.mode !{{.*}}, !amdgpu.no.fine.grained.memory !{{.*}}, !amdgpu.no.remote.memory !{{.*}}
+
+module attributes {dlti.dl_spec = #dlti.dl_spec<!llvm.ptr = dense<64> : vector<4xi64>, !llvm.ptr<1> = dense<64> : vector<4xi64>, !llvm.ptr<2> = dense<32> : vector<4xi64>, !llvm.ptr<3> = dense<32> : vector<4xi64>, !llvm.ptr<4> = dense<64> : vector<4xi64>, !llvm.ptr<5> = dense<32> : vector<4xi64>, !llvm.ptr<6> = dense<32> : vector<4xi64>, !llvm.ptr<7> = dense<[160, 256, 256, 32]> : vector<4xi64>, !llvm.ptr<8> = dense<[128, 128, 128, 48]> : vector<4xi64>, !llvm.ptr<9> = dense<[192, 256, 256, 32]> : vector<4xi64>, i64 = dense<64> : vector<2xi64>, i1 = dense<8> : vector<2xi64>, i8 = dense<8> : vector<2xi64>, i16 = dense<16> : vector<2xi64>, i32 = dense<32> : vector<2xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, f128 = dense<128> : vector<2xi64>, "dlti.endianness" = "little", "dlti.legal_int_widths" = array<i32: 32, 64>, "dlti.stack_alignment" = 32 : i64, "dlti.alloca_memory_space" = 5 : ui64, "dlti.global_memory_space" = 1 : ui64>, fir.atomic_ignore_denormal_mode, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", fir.target_cpu = "generic-hsa", llvm.data_layout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-p7:160:256:256:32-p8:128:128:128:48-p9:192:256:256:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-G1-ni:7:8:9", llvm.target_triple = "amdgcn-amd-amdhsa", omp.flags = #omp.flags<openmp_device_version = 31>, omp.is_gpu = true, omp.is_target_device = true, omp.requires = #omp<clause_requires none>, omp.target_triples = [], omp.version = #omp.version<version = 31>} {
+  llvm.func @_QQmain() attributes {fir.bindc_name = "TEST", omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to)>, target_cpu = "generic-hsa"} {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "threads"} : (i64) -> !llvm.ptr<5>
+    %2 = llvm.addrspacecast %1 : !llvm.ptr<5> to !llvm.ptr
+    %3 = llvm.mlir.constant(1 : i64) : i64
+    %4 = llvm.alloca %3 x i32 {bindc_name = "a"} : (i64) -> !llvm.ptr<5>
+    %5 = llvm.addrspacecast %4 : !llvm.ptr<5> to !llvm.ptr
+    %6 = llvm.mlir.constant(0 : i32) : i32
+    %7 = llvm.mlir.constant(128 : i32) : i32
+    %8 = llvm.mlir.constant(1 : i64) : i64
+    %9 = llvm.mlir.constant(1 : i64) : i64
+    llvm.store %7, %2 : i32, !llvm.ptr
+    llvm.store %6, %5 : i32, !llvm.ptr
+    %10 = omp.map.info var_ptr(%2 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "threads"}
+    %11 = omp.map.info var_ptr(%5 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "a"}
+    omp.target map_entries(%10 -> %arg0, %11 -> %arg1 : !llvm.ptr, !llvm.ptr) {
+      %12 = llvm.mlir.constant(1 : i32) : i32
+      %13 = llvm.load %arg0 : !llvm.ptr -> i32
+      omp.parallel num_threads(%13 : i32) {
+        omp.atomic.update %arg1 : !llvm.ptr {
+        ^bb0(%arg2: i32):
+          %14 = llvm.add %arg2, %12 : i32
+          omp.yield(%14 : i32)
+        } {atomic_control = #omp.atomic_control<ignore_denormal_mode = true>}
+        omp.terminator
+      }
+      omp.terminator
+    }
+    llvm.return
+  }
+}
diff --git a/mlir/test/Target/LLVMIR/omptarget-debug-147063.mlir b/mlir/test/Target/LLVMIR/omptarget-debug-147063.mlir
new file mode 100644
index 0000000000000..12d389adbb388
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/omptarget-debug-147063.mlir
@@ -0,0 +1,45 @@
+// RUN: mlir-translate -mlir-to-llvmir %s
+
+module attributes {llvm.target_triple = "x86_64-unknown-linux-gnu", omp.is_gpu = false, omp.is_target_device = false, omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  omp.private {type = private} @_QFFfnEv_private_i32 : i32 loc(#loc1)
+  llvm.func internal @_QFPfn() {
+    %0 = llvm.mlir.constant(1 : i64) : i64 loc(#loc1)
+    %1 = llvm.alloca %0 x i32 {bindc_name = "v"} : (i64) -> !llvm.ptr loc(#loc1)
+    %2 = llvm.mlir.constant(1 : i32) : i32
+    omp.parallel private(@_QFFfnEv_private_i32 %1 -> %arg0 : !llvm.ptr) {
+      llvm.store %2, %arg0 : i32, !llvm.ptr loc(#loc2)
+      %4 = omp.map.info var_ptr(%arg0 : !llvm.ptr, i32) map_clauses(implicit, exit_release_or_enter_alloc) capture(ByCopy) -> !llvm.ptr {name = "v"} loc(#loc2)
+      omp.target map_entries(%4 -> %arg1 : !llvm.ptr) {
+        %5 = llvm.mlir.constant(1 : i32) : i32
+        %6 = llvm.load %arg1 : !llvm.ptr -> i32 loc(#loc3)
+        %7 = llvm.add %6, %5 : i32 loc(#loc3)
+        llvm.store %7, %arg1 : i32, !llvm.ptr loc(#loc3)
+        omp.terminator loc(#loc3)
+      } loc(#loc7)
+      omp.terminator
+    } loc(#loc4)
+    llvm.return
+  } loc(#loc6)
+}
+
+#di_file = #llvm.di_file<"target.f90" in "">
+#di_null_type = #llvm.di_null_type
+#di_compile_unit = #llvm.di_compile_unit<id = distinct[0]<>,
+ sourceLanguage = DW_LANG_Fortran95, file = #di_file, producer = "flang",
+ isOptimized = false, emissionKind = LineTablesOnly>
+#di_subroutine_type = #llvm.di_subroutine_type<
+  callingConvention = DW_CC_program, types = #di_null_type>
+#di_subprogram = #llvm.di_subprogram<id = distinct[1]<>,
+  compileUnit = #di_compile_unit, scope = #di_file, name = "main",
+  file = #di_file, subprogramFlags = "Definition|MainSubprogram",
+  type = #di_subroutine_type>
+#di_subprogram1 = #llvm.di_subprogram<compileUnit = #di_compile_unit,
+  name = "target", file = #di_file, subprogramFlags = "Definition",
+  type = #di_subroutine_type>
+
+#loc1 = loc("test.f90":7:15)
+#loc2 = loc("test.f90":1:7)
+#loc3 = loc("test.f90":3:7)
+#loc4 = loc("test.f90":16:7)
+#loc6 = loc(fused<#di_subprogram>[#loc1])
+#loc7 = loc(fused<#di_subprogram1>[#loc3])
diff --git a/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir b/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir
index 830610f12a5d2..5d2861a5d0f35 100644
--- a/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-parallel-wsloop.mlir
@@ -37,7 +37,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 // CHECK-SAME:  #[[ATTRS1:[0-9]+]]
 // CHECK: call void @__kmpc_for_static_loop_4u(ptr addrspacecast (ptr addrspace(1) @[[GLOB]] to ptr),
 // CHECK-SAME:   ptr @[[LOOP_BODY_FUNC:.*]], ptr %[[LOO_BODY_FUNC_ARG:.*]], i32 10,
-// CHECK-SAME:   i32 %[[THREAD_NUM:.*]], i32 0)
+// CHECK-SAME:   i32 %[[THREAD_NUM:.*]], i8 0)
 
 // CHECK:      define internal void @[[LOOP_BODY_FUNC]](i32 %[[CNT:.*]], ptr %[[LOOP_BODY_ARG_PTR:.*]]) #[[ATTRS2:[0-9]+]] {
 
diff --git a/mlir/test/Target/LLVMIR/omptarget-wsloop-collapsed.mlir b/mlir/test/Target/LLVMIR/omptarget-wsloop-collapsed.mlir
index 0ebcec0e0ec31..b42e387acbb11 100644
--- a/mlir/test/Target/LLVMIR/omptarget-wsloop-collapsed.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-wsloop-collapsed.mlir
@@ -25,7 +25,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 // CHECK: define void @[[FUNC_COLLAPSED_WSLOOP:.*]](ptr %[[ARG0:.*]])
 // CHECK:   call void @__kmpc_for_static_loop_4u(ptr addrspacecast (ptr addrspace(1) @[[GLOB2:[0-9]+]] to ptr),
 // CHECK-SAME: ptr @[[COLLAPSED_WSLOOP_BODY_FN:.*]], ptr %[[STRUCT_ARG:.*]], i32 10000,
-// CHECK-SAME: i32 %[[NUM_THREADS:.*]], i32 0)
+// CHECK-SAME: i32 %[[NUM_THREADS:.*]], i8 0)
 
 // CHECK: define internal void @[[COLLAPSED_WSLOOP_BODY_FN]](i32 %[[LOOP_CNT:.*]], ptr %[[LOOP_BODY_ARG:.*]])
 // CHECK:   %[[TMP0:.*]] = urem i32 %[[LOOP_CNT]], 100
diff --git a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
index a9f913b744489..7be635f46111b 100644
--- a/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-wsloop.mlir
@@ -37,7 +37,7 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 // CHECK:   %[[GEP:.*]] = getelementptr { ptr }, ptr addrspace(5) %[[STRUCTARG]], i32 0, i32 0
 // CHECK:   store ptr %[[ARG0]], ptr addrspace(5) %[[GEP]], align 8
 // CHECK:   %[[NUM_THREADS:.*]] = call i32 @omp_get_num_threads()
-// CHECK:   call void @__kmpc_for_static_loop_4u(ptr addrspacecast (ptr addrspace(1) @[[GLOB1:[0-9]+]] to ptr), ptr @[[LOOP_BODY_FN:.*]], ptr %[[STRUCTARG_ASCAST]], i32 10, i32 %[[NUM_THREADS]], i32 0)
+// CHECK:   call void @__kmpc_for_static_loop_4u(ptr addrspacecast (ptr addrspace(1) @[[GLOB1:[0-9]+]] to ptr), ptr @[[LOOP_BODY_FN:.*]], ptr %[[STRUCTARG_ASCAST]], i32 10, i32 %[[NUM_THREADS]], i32 0, i8 0)
 
 // CHECK: define internal void @[[LOOP_BODY_FN]](i32 %[[LOOP_CNT:.*]], ptr %[[LOOP_BODY_ARG:.*]])
 // CHECK:   %[[GEP2:.*]] = getelementptr { ptr }, ptr %[[LOOP_BODY_ARG]], i32 0, i32 0
@@ -46,6 +46,6 @@ module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry<"dlti.alloca_memo
 // CHECK:   store i32 %[[VAL0:.*]], ptr %[[GEP3]], align 4
 
 // CHECK: define void @[[FUNC_EMPTY_WSLOOP:.*]]()
-// CHECK:   call void @__kmpc_for_static_loop_4u(ptr addrspacecast (ptr addrspace(1) @[[GLOB2:[0-9]+]] to ptr), ptr @[[LOOP_EMPTY_BODY_FN:.*]], ptr null, i32 10, i32 %[[NUM_THREADS:.*]], i32 0)
+// CHECK:   call void @__kmpc_for_static_loop_4u(ptr addrspacecast (ptr addrspace(1) @[[GLOB2:[0-9]+]] to ptr), ptr @[[LOOP_EMPTY_BODY_FN:.*]], ptr null, i32 10, i32 %[[NUM_THREADS:.*]], i32 0, i8 0)
 
 // CHECK: define internal void @[[LOOP_EMPTY_BODY_FN]](i32 %[[LOOP_CNT:.*]])
diff --git a/mlir/test/Target/LLVMIR/rocdl.mlir b/mlir/test/Target/LLVMIR/rocdl.mlir
index 740990a6e589b..a464358250c38 100644
--- a/mlir/test/Target/LLVMIR/rocdl.mlir
+++ b/mlir/test/Target/LLVMIR/rocdl.mlir
@@ -86,12 +86,12 @@ llvm.func @kernel_func_unsafe_fp_atomics()
 }
 
 llvm.func @rocdl.lane_id() -> i32 {
-  // CHECK: [[mbcntlo:%.+]] = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-  // CHECK-NEXT: call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[mbcntlo]])
+  // CHECK: [[mbcntlo:%.+]] = call noundef range(i32 0, 32) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  // CHECK-NEXT: call noundef range(i32 0, 64) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[mbcntlo]])
   %0 = llvm.mlir.constant(-1 : i32) : i32
   %1 = llvm.mlir.constant(0 : i32) : i32
-  %2 = rocdl.mbcnt.lo %0, %1 : (i32, i32) -> i32
-  %3 = rocdl.mbcnt.hi %0, %2 : (i32, i32) -> i32
+  %2 = rocdl.mbcnt.lo %0, %1 {res_attrs = [{llvm.noundef, llvm.range = #llvm.constant_range<i32, 0, 32>}]} : (i32, i32) -> i32
+  %3 = rocdl.mbcnt.hi %0, %2 {res_attrs = [{llvm.noundef, llvm.range = #llvm.constant_range<i32, 0, 64>}]} : (i32, i32) -> i32
   llvm.return %3 : i32
 }
 
@@ -125,6 +125,23 @@ llvm.func @rocdl.ballot64(%pred : i1) -> i64 {
   llvm.return %0 : i64
 }
 
+llvm.func @rocdl.readfirstlane(%src0 : f32, %src1: f64, %src2: i32, %src3: vector<2 x f32>) -> f32 {
+  // CHECK-LABEL: rocdl.readfirstlane
+  // CHECK: call float @llvm.amdgcn.readfirstlane.f32(float %{{.*}})
+  %0 = rocdl.readfirstlane %src0 : f32
+
+  // CHECK: call double @llvm.amdgcn.readfirstlane.f64(double %{{.*}})
+  %1 = rocdl.readfirstlane %src1 : f64
+
+  // CHECK: call i32 @llvm.amdgcn.readfirstlane.i32(i32 %{{.*}})
+  %2 = rocdl.readfirstlane %src2 : i32
+
+  // CHECK: call <2 x float> @llvm.amdgcn.readfirstlane.v2f32(<2 x float> %{{.*}})
+  %3 = rocdl.readfirstlane %src3 : vector<2 x f32>
+
+  llvm.return %0 : f32
+}
+
 llvm.func @rocdl.readlane(%src0 : f32, %src1: f64, %src2: i32, %src3: vector<2 x f32>) -> f32 {
   %idx = llvm.mlir.constant(0 : i32) : i32
 
@@ -924,6 +941,20 @@ llvm.func @rocdl.permlanex16(%src0 : f32, %src1 : i32, %src2 : vector<2 x f32>,
   llvm.return %ret0 : f32
 }
 
+llvm.func @rocdl.permlane16.swap(%src : i32) -> !llvm.struct<(i32, i32)> {
+  // CHECK-LABEL: rocdl.permlane16.swap
+  // CHECK: call { i32, i32 } @llvm.amdgcn.permlane16.swap(i32 %{{.*}}, i32 %{{.*}}, i1 false, i1 true)
+  %ret = rocdl.permlane16.swap %src, %src, 0, -1  : (i32, i32) -> !llvm.struct<(i32, i32)>
+  llvm.return %ret : !llvm.struct<(i32, i32)>
+}
+
+llvm.func @rocdl.permlane32.swap(%src : i32) -> !llvm.struct<(i32, i32)> {
+  // CHECK-LABEL: rocdl.permlane32.swap
+  // CHECK: call { i32, i32 } @llvm.amdgcn.permlane32.swap(i32 %{{.*}}, i32 %{{.*}}, i1 false, i1 true)
+  %ret = rocdl.permlane32.swap %src, %src, 0, -1  : (i32, i32) -> !llvm.struct<(i32, i32)>
+  llvm.return %ret : !llvm.struct<(i32, i32)>
+}
+
 llvm.func @rocdl.wmma.fp8(%arg0 : vector<2 x i32>, %arg1 : vector<8xf32>) -> vector<8xf32> {
   // CHECK: call <8 x float> @llvm.amdgcn.wmma.f32.16x16x16.fp8.fp8.v8f32.v2i32(<2 x i32> %{{.*}}, <2 x i32> %{{.*}}, <8 x float> %{{.*}})
   %r0 = rocdl.wmma.f32.16x16x16.fp8_fp8 %arg0, %arg0, %arg1: (vector<2xi32>, vector<2xi32>, vector<8xf32>) -> vector<8xf32>
diff --git a/mlir/test/Target/SPIRV/arm-tensor-constant.mlir b/mlir/test/Target/SPIRV/arm-tensor-constant.mlir
index 275e586f70634..7fb8af1904388 100644
--- a/mlir/test/Target/SPIRV/arm-tensor-constant.mlir
+++ b/mlir/test/Target/SPIRV/arm-tensor-constant.mlir
@@ -1,17 +1,36 @@
 // RUN: mlir-translate --no-implicit-module --test-spirv-roundtrip %s | FileCheck %s
-// DISABLED: %if spirv-tools %{ mlir-translate --no-implicit-module --serialize-spirv %s | spirv-val %}
-
-// FIXME(#152012): Fix arm tensor constant validation errors and reenable spirv-val tests.
+// RUN: %if spirv-tools %{ mlir-translate --no-implicit-module --serialize-spirv %s | spirv-val %}
 
 spirv.module Logical Vulkan requires #spirv.vce<v1.3,
              [VulkanMemoryModel, Shader, TensorsARM, Linkage], [SPV_KHR_vulkan_memory_model, SPV_ARM_tensors]> {
-  // CHECK-LABEL: @arm_tensor_of_i32
-  spirv.func @arm_tensor_of_i32() -> (!spirv.arm.tensor<2x3xi32>) "None" {
+  // CHECK-LABEL: @rank_1_arm_tensor_of_i32
+  spirv.func @rank_1_arm_tensor_of_i32() -> (!spirv.arm.tensor<3xi32>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<[1, 2, 3]> : !spirv.arm.tensor<3xi32>
+    %0 = spirv.Constant dense<[1, 2, 3]> : !spirv.arm.tensor<3xi32>
+    spirv.ReturnValue %0 : !spirv.arm.tensor<3xi32>
+  }
+
+  // CHECK-LABEL: @rank_2_arm_tensor_of_i32
+  spirv.func @rank_2_arm_tensor_of_i32() -> (!spirv.arm.tensor<2x3xi32>) "None" {
     // CHECK: {{%.*}} = spirv.Constant dense<{{\[}}[1, 2, 3], [4, 5, 6]]> : !spirv.arm.tensor<2x3xi32>
     %0 = spirv.Constant dense<[[1, 2, 3], [4, 5, 6]]> : !spirv.arm.tensor<2x3xi32>
     spirv.ReturnValue %0 : !spirv.arm.tensor<2x3xi32>
   }
 
+  // CHECK-LABEL: @rank_3_arm_tensor_of_i32
+  spirv.func @rank_3_arm_tensor_of_i32() -> (!spirv.arm.tensor<2x2x3xi32>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<{{\[}}{{\[}}[1, 2, 3], [4, 5, 6]], {{\[}}[7, 8, 9], [10, 11, 12]]]> : !spirv.arm.tensor<2x2x3xi32>
+    %0 = spirv.Constant dense<[[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]> : !spirv.arm.tensor<2x2x3xi32>
+    spirv.ReturnValue %0 : !spirv.arm.tensor<2x2x3xi32>
+  }
+
+  // CHECK-LABEL: @rank_4_arm_tensor_of_i32
+  spirv.func @rank_4_arm_tensor_of_i32() -> (!spirv.arm.tensor<2x3x4x5xi32>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<5> : !spirv.arm.tensor<2x3x4x5xi32>
+    %0 = spirv.Constant dense<5> : !spirv.arm.tensor<2x3x4x5xi32>
+    spirv.ReturnValue %0 : !spirv.arm.tensor<2x3x4x5xi32>
+  }
+
   // CHECK-LABEL: @splat_arm_tensor_of_i32
   spirv.func @splat_arm_tensor_of_i32() -> (!spirv.arm.tensor<2x3xi32>) "None" {
     // CHECK: {{%.*}} = spirv.Constant dense<2> : !spirv.arm.tensor<2x3xi32>
@@ -19,13 +38,34 @@ spirv.module Logical Vulkan requires #spirv.vce<v1.3,
     spirv.ReturnValue %0 : !spirv.arm.tensor<2x3xi32>
   }
 
-  // CHECK-LABEL: @arm_tensor_of_f32
-  spirv.func @arm_tensor_of_f32() -> (!spirv.arm.tensor<2x3xf32>) "None" {
+  // CHECK-LABEL: @rank_1_arm_tensor_of_f32
+  spirv.func @rank_1_arm_tensor_of_f32() -> (!spirv.arm.tensor<3xf32>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<[1.000000e+00, 2.000000e+00, 3.000000e+00]> : !spirv.arm.tensor<3xf32>
+    %0 = spirv.Constant dense<[1.0, 2.0, 3.0]> : !spirv.arm.tensor<3xf32>
+    spirv.ReturnValue %0 : !spirv.arm.tensor<3xf32>
+  }
+
+  // CHECK-LABEL: @rank_2_arm_tensor_of_f32
+  spirv.func @rank_2_arm_tensor_of_f32() -> (!spirv.arm.tensor<2x3xf32>) "None" {
     // CHECK: {{%.*}} = spirv.Constant dense<{{\[}}[1.000000e+00, 2.000000e+00, 3.000000e+00], [4.000000e+00, 5.000000e+00, 6.000000e+00]]> : !spirv.arm.tensor<2x3xf32>
-    %0 = spirv.Constant dense<[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]>: !spirv.arm.tensor<2x3xf32>
+    %0 = spirv.Constant dense<[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]> : !spirv.arm.tensor<2x3xf32>
     spirv.ReturnValue %0 : !spirv.arm.tensor<2x3xf32>
   }
 
+  // CHECK-LABEL: @rank_3_arm_tensor_of_f32
+  spirv.func @rank_3_arm_tensor_of_f32() -> (!spirv.arm.tensor<2x2x3xf32>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<{{\[}}{{\[}}[1.000000e+00, 2.000000e+00, 3.000000e+00], [4.000000e+00, 5.000000e+00, 6.000000e+00]], {{\[}}[7.000000e+00, 8.000000e+00, 9.000000e+00], [1.000000e+01, 1.100000e+01, 1.200000e+01]]]> : !spirv.arm.tensor<2x2x3xf32>
+    %0 = spirv.Constant dense<[[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], [[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]]> : !spirv.arm.tensor<2x2x3xf32>
+    spirv.ReturnValue %0 : !spirv.arm.tensor<2x2x3xf32>
+  }
+
+  // CHECK-LABEL: @rank_4_arm_tensor_of_f32
+  spirv.func @rank_4_arm_tensor_of_f32() -> (!spirv.arm.tensor<2x3x4x5xf32>) "None" {
+    // CHECK: {{%.*}} = spirv.Constant dense<5.000000e+00> : !spirv.arm.tensor<2x3x4x5xf32>
+    %0 = spirv.Constant dense<5.0> : !spirv.arm.tensor<2x3x4x5xf32>
+    spirv.ReturnValue %0 : !spirv.arm.tensor<2x3x4x5xf32>
+  }
+
   // CHECK-LABEL: @splat_arm_tensor_of_f32
   spirv.func @splat_arm_tensor_of_f32() -> (!spirv.arm.tensor<2x3xf32>) "None" {
     // CHECK: {{%.*}} = spirv.Constant dense<2.000000e+00> : !spirv.arm.tensor<2x3xf32>
diff --git a/mlir/test/Target/SPIRV/debug-negative.mlir b/mlir/test/Target/SPIRV/debug-negative.mlir
new file mode 100644
index 0000000000000..2c826870b5f02
--- /dev/null
+++ b/mlir/test/Target/SPIRV/debug-negative.mlir
@@ -0,0 +1,5 @@
+// RUN: mlir-translate %s --test-spirv-roundtrip-debug --no-implicit-module --verify-diagnostics
+
+// expected-error@below {{SPV_KHR_non_semantic_info extension not available}}
+spirv.module Logical GLSL450 requires #spirv.vce<v1.3, [Shader], []> attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.3, [Shader], []>, #spirv.resource_limits<>>} {
+}
diff --git a/mlir/test/Target/SPIRV/debug.mlir b/mlir/test/Target/SPIRV/debug.mlir
index 58bf364593fc9..5a7ed19d22c11 100644
--- a/mlir/test/Target/SPIRV/debug.mlir
+++ b/mlir/test/Target/SPIRV/debug.mlir
@@ -1,69 +1,70 @@
 // RUN: mlir-translate -no-implicit-module -test-spirv-roundtrip-debug -mlir-print-debuginfo -mlir-print-local-scope %s | FileCheck %s
+// RUN: %if spirv-tools %{ mlir-translate --no-implicit-module --serialize-spirv %s | spirv-val %}
 
-spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
-  // CHECK: loc({{".*debug.mlir"}}:5:3)
+spirv.module Logical GLSL450 requires #spirv.vce<v1.3, [Shader, GroupNonUniformArithmetic], [SPV_KHR_non_semantic_info, SPV_KHR_storage_buffer_storage_class]> attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.3, [Shader, GroupNonUniformArithmetic], [SPV_KHR_non_semantic_info, SPV_KHR_storage_buffer_storage_class]>, #spirv.resource_limits<>>} {
+  // CHECK: loc({{".*debug.mlir"}}:6:3)
   spirv.GlobalVariable @var0 bind(0, 1) : !spirv.ptr<f32, Input>
   spirv.func @arithmetic(%arg0 : vector<4xf32>, %arg1 : vector<4xf32>) "None" {
-    // CHECK: loc({{".*debug.mlir"}}:8:10)
+    // CHECK: loc({{".*debug.mlir"}}:9:10)
     %0 = spirv.FAdd %arg0, %arg1 : vector<4xf32>
-    // CHECK: loc({{".*debug.mlir"}}:10:10)
+    // CHECK: loc({{".*debug.mlir"}}:11:10)
     %1 = spirv.FNegate %arg0 : vector<4xf32>
     spirv.Return
   }
 
   spirv.func @atomic(%ptr: !spirv.ptr<i32, Workgroup>, %value: i32, %comparator: i32) "None" {
-    // CHECK: loc({{".*debug.mlir"}}:16:10)
+    // CHECK: loc({{".*debug.mlir"}}:17:10)
     %1 = spirv.AtomicAnd <Device> <None> %ptr, %value : !spirv.ptr<i32, Workgroup>
     spirv.Return
   }
 
   spirv.func @bitwiser(%arg0 : i32, %arg1 : i32) "None" {
-    // CHECK: loc({{".*debug.mlir"}}:22:10)
+    // CHECK: loc({{".*debug.mlir"}}:23:10)
     %0 = spirv.BitwiseAnd %arg0, %arg1 : i32
     spirv.Return
   }
 
   spirv.func @convert(%arg0 : f32) "None" {
-    // CHECK: loc({{".*debug.mlir"}}:28:10)
+    // CHECK: loc({{".*debug.mlir"}}:29:10)
     %0 = spirv.ConvertFToU %arg0 : f32 to i32
     spirv.Return
   }
 
   spirv.func @composite(%arg0 : !spirv.struct<(f32, !spirv.struct<(!spirv.array<4xf32>, f32)>)>, %arg1: !spirv.array<4xf32>, %arg2 : f32, %arg3 : f32) "None" {
-    // CHECK: loc({{".*debug.mlir"}}:34:10)
+    // CHECK: loc({{".*debug.mlir"}}:35:10)
     %0 = spirv.CompositeInsert %arg1, %arg0[1 : i32, 0 : i32] : !spirv.array<4xf32> into !spirv.struct<(f32, !spirv.struct<(!spirv.array<4xf32>, f32)>)>
-    // CHECK: loc({{".*debug.mlir"}}:36:10)
+    // CHECK: loc({{".*debug.mlir"}}:37:10)
     %1 = spirv.CompositeConstruct %arg2, %arg3 : (f32, f32) -> vector<2xf32>
     spirv.Return
   }
 
   spirv.func @group_non_uniform(%val: f32) "None" {
-    // CHECK: loc({{".*debug.mlir"}}:42:10)
+    // CHECK: loc({{".*debug.mlir"}}:43:10)
     %0 = spirv.GroupNonUniformFAdd <Workgroup> <Reduce> %val : f32 -> f32
     spirv.Return
   }
 
   spirv.func @local_var() "None" {
     %zero = spirv.Constant 0: i32
-    // CHECK: loc({{".*debug.mlir"}}:49:12)
+    // CHECK: loc({{".*debug.mlir"}}:50:12)
     %var = spirv.Variable init(%zero) : !spirv.ptr<i32, Function>
     spirv.Return
   }
 
   spirv.func @logical(%arg0: i32, %arg1: i32) "None" {
-    // CHECK: loc({{".*debug.mlir"}}:55:10)
+    // CHECK: loc({{".*debug.mlir"}}:56:10)
     %0 = spirv.IEqual %arg0, %arg1 : i32
     spirv.Return
   }
 
   spirv.func @memory_accesses(%arg0 : !spirv.ptr<!spirv.array<4x!spirv.array<4xf32>>, StorageBuffer>, %arg1 : i32, %arg2 : i32) "None" {
-    // CHECK: loc({{".*debug.mlir"}}:61:10)
+    // CHECK: loc({{".*debug.mlir"}}:62:10)
     %2 = spirv.AccessChain %arg0[%arg1, %arg2] : !spirv.ptr<!spirv.array<4x!spirv.array<4xf32>>, StorageBuffer>, i32, i32 -> !spirv.ptr<f32, StorageBuffer>
-    // CHECK: loc({{".*debug.mlir"}}:63:10)
+    // CHECK: loc({{".*debug.mlir"}}:64:10)
     %3 = spirv.Load "StorageBuffer" %2 : f32
-    // CHECK: loc({{.*debug.mlir"}}:65:5)
+    // CHECK: loc({{.*debug.mlir"}}:66:5)
     spirv.Store "StorageBuffer" %2, %3 : f32
-    // CHECK: loc({{".*debug.mlir"}}:67:5)
+    // CHECK: loc({{".*debug.mlir"}}:68:5)
     spirv.Return
   }
 
@@ -73,49 +74,49 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %ivar = spirv.Variable init(%zero) : !spirv.ptr<i32, Function>
     %jvar = spirv.Variable init(%zero) : !spirv.ptr<i32, Function>
     spirv.mlir.loop {
-      // CHECK: loc({{".*debug.mlir"}}:75:5)
+      // CHECK: loc({{".*debug.mlir"}}:76:5)
       spirv.Branch ^header
     ^header:
       %ival0 = spirv.Load "Function" %ivar : i32
       %icmp = spirv.SLessThan %ival0, %count : i32
-      // CHECK: loc({{".*debug.mlir"}}:75:5)
+      // CHECK: loc({{".*debug.mlir"}}:76:5)
       spirv.BranchConditional %icmp, ^body, ^merge
     ^body:
       spirv.Store "Function" %jvar, %zero : i32
       spirv.mlir.loop {
-        // CHECK: loc({{".*debug.mlir"}}:85:7)
+        // CHECK: loc({{".*debug.mlir"}}:86:7)
         spirv.Branch ^header
       ^header:
         %jval0 = spirv.Load "Function" %jvar : i32
         %jcmp = spirv.SLessThan %jval0, %count : i32
-        // CHECK: loc({{".*debug.mlir"}}:85:7)
+        // CHECK: loc({{".*debug.mlir"}}:86:7)
         spirv.BranchConditional %jcmp, ^body, ^merge
       ^body:
-        // CHECK: loc({{".*debug.mlir"}}:95:9)
+        // CHECK: loc({{".*debug.mlir"}}:96:9)
         spirv.Branch ^continue
       ^continue:
         %jval1 = spirv.Load "Function" %jvar : i32
         %add = spirv.IAdd %jval1, %one : i32
         spirv.Store "Function" %jvar, %add : i32
-        // CHECK: loc({{".*debug.mlir"}}:101:9)
+        // CHECK: loc({{".*debug.mlir"}}:102:9)
         spirv.Branch ^header
       ^merge:
-        // CHECK: loc({{".*debug.mlir"}}:85:7)
+        // CHECK: loc({{".*debug.mlir"}}:86:7)
         spirv.mlir.merge
-        // CHECK: loc({{".*debug.mlir"}}:85:7)
+        // CHECK: loc({{".*debug.mlir"}}:86:7)
       }
-      // CHECK: loc({{".*debug.mlir"}}:108:7)
+      // CHECK: loc({{".*debug.mlir"}}:109:7)
       spirv.Branch ^continue
     ^continue:
       %ival1 = spirv.Load "Function" %ivar : i32
       %add = spirv.IAdd %ival1, %one : i32
       spirv.Store "Function" %ivar, %add : i32
-      // CHECK: loc({{".*debug.mlir"}}:114:7)
+      // CHECK: loc({{".*debug.mlir"}}:115:7)
       spirv.Branch ^header
     ^merge:
-      // CHECK: loc({{".*debug.mlir"}}:75:5)
+      // CHECK: loc({{".*debug.mlir"}}:76:5)
       spirv.mlir.merge
-    // CHECK: loc({{".*debug.mlir"}}:75:5)
+    // CHECK: loc({{".*debug.mlir"}}:76:5)
     }
     spirv.Return
   }
@@ -126,21 +127,23 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
     %two = spirv.Constant 2: i32
     %var = spirv.Variable init(%zero) : !spirv.ptr<i32, Function>
     spirv.mlir.selection {
-      // CHECK: loc({{".*debug.mlir"}}:128:5)
+      // CHECK: loc({{".*debug.mlir"}}:129:5)
       spirv.BranchConditional %cond [5, 10], ^then, ^else
     ^then:
       spirv.Store "Function" %var, %one : i32
-      // CHECK: loc({{".*debug.mlir"}}:134:7)
+      // CHECK: loc({{".*debug.mlir"}}:135:7)
       spirv.Branch ^merge
     ^else:
       spirv.Store "Function" %var, %two : i32
-      // CHECK: loc({{".*debug.mlir"}}:138:7)
+      // CHECK: loc({{".*debug.mlir"}}:139:7)
       spirv.Branch ^merge
     ^merge:
-      // CHECK: loc({{".*debug.mlir"}}:128:5)
+      // CHECK: loc({{".*debug.mlir"}}:129:5)
       spirv.mlir.merge
-    // CHECK: loc({{".*debug.mlir"}}:128:5)
+    // CHECK: loc({{".*debug.mlir"}}:129:5)
     }
     spirv.Return
   }
+
+  spirv.EntryPoint "GLCompute" @local_var
 }
diff --git a/mlir/test/Target/SPIRV/decorations.mlir b/mlir/test/Target/SPIRV/decorations.mlir
index ee7ad814ea0cb..90ba690e50b73 100644
--- a/mlir/test/Target/SPIRV/decorations.mlir
+++ b/mlir/test/Target/SPIRV/decorations.mlir
@@ -58,6 +58,20 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
 
 // -----
 
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Tessellation, Linkage], []> {
+  // CHECK: patch
+  spirv.GlobalVariable @var {patch} : !spirv.ptr<vector<4xf32>, Input>
+}
+
+// -----
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], []> {
+  // CHECK: invariant
+  spirv.GlobalVariable @var {invariant} : !spirv.ptr<vector<2xf32>, Output>
+}
+
+// -----
+
 spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], []> {
   // CHECK: linkage_attributes = #spirv.linkage_attributes<linkage_name = "outSideGlobalVar1", linkage_type = <Import>>
   spirv.GlobalVariable @var1 {
diff --git a/mlir/test/Target/SPIRV/mlir-translate.mlir b/mlir/test/Target/SPIRV/mlir-translate.mlir
new file mode 100644
index 0000000000000..cbce351dd35b5
--- /dev/null
+++ b/mlir/test/Target/SPIRV/mlir-translate.mlir
@@ -0,0 +1,29 @@
+// Check that `--spirv-save-validation-files-with-prefix` generates
+// a correct number of files.
+
+// REQUIRES: shell
+// RUN: rm -rf %t
+// RUN: mkdir %t && mlir-translate --serialize-spirv --no-implicit-module \
+// RUN: --split-input-file --spirv-save-validation-files-with-prefix=%t/foo %s \
+// RUN: && ls %t/foo*.spv | wc -l | FileCheck %s
+// RUN: rm -rf %t
+
+// CHECK: 4
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], []> {
+}
+
+// -----
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], []> {
+}
+
+// -----
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], []> {
+}
+
+// -----
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], []> {
+}
diff --git a/mlir/test/Target/SPIRV/module.mlir b/mlir/test/Target/SPIRV/module.mlir
index dcdcab8097e41..d4000dfa53810 100644
--- a/mlir/test/Target/SPIRV/module.mlir
+++ b/mlir/test/Target/SPIRV/module.mlir
@@ -1,21 +1,29 @@
-// RUN: mlir-translate -no-implicit-module -test-spirv-roundtrip -split-input-file %s | FileCheck %s
+// RUN: mlir-translate --no-implicit-module --test-spirv-roundtrip --split-input-file %s | FileCheck %s
+
+// REQUIRES: shell
+// RUN: %if spirv-tools %{ rm -rf %t %}
+// RUN: %if spirv-tools %{ mkdir %t %}
+// RUN: %if spirv-tools %{ mlir-translate --no-implicit-module --serialize-spirv --split-input-file --spirv-save-validation-files-with-prefix=%t/module %s %}
+// RUN: %if spirv-tools %{ ls %t/module*.spv | xargs -I{} spirv-val {} %}
 
 // CHECK:      spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
 // CHECK-NEXT:   spirv.func @foo() "Inline" {
 // CHECK-NEXT:     spirv.Return
 // CHECK-NEXT:   }
+// CHECK-NEXT:   spirv.EntryPoint "Vertex" @foo
 // CHECK-NEXT: }
 
 spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
   spirv.func @foo() -> () "Inline" {
      spirv.Return
   }
+  spirv.EntryPoint "Vertex" @foo
 }
 
 // -----
 
 // CHECK: v1.5
-spirv.module Logical GLSL450 requires #spirv.vce<v1.5, [Shader], []> {
+spirv.module Logical GLSL450 requires #spirv.vce<v1.5, [Shader, Linkage], []> {
 }
 
 // -----
@@ -26,13 +34,13 @@ spirv.module Logical GLSL450 requires #spirv.vce<v1.6, [Shader, Linkage], []> {
 
 // -----
 
-// CHECK: [Shader, Float16]
-spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Float16], []> {
+// CHECK: [Shader, Float16, Linkage]
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Float16, Linkage], []> {
 }
 
 // -----
 
 // CHECK: [SPV_KHR_float_controls, SPV_KHR_subgroup_vote]
-spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], [SPV_KHR_float_controls, SPV_KHR_subgroup_vote]> {
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader, Linkage], [SPV_KHR_float_controls, SPV_KHR_subgroup_vote]> {
 }
 
diff --git a/mlir/test/Target/Wasm/abs.mlir b/mlir/test/Target/Wasm/abs.mlir
new file mode 100644
index 0000000000000..9c45ba78507ad
--- /dev/null
+++ b/mlir/test/Target/Wasm/abs.mlir
@@ -0,0 +1,23 @@
+// RUN: yaml2obj %S/inputs/abs.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "abs_f32") (result f32)
+    f32.const 10
+    f32.abs)
+
+    (func (export "abs_f64") (result f64)
+    f64.const 10
+    f64.abs)
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @abs_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.abs %[[VAL_0]] : f32
+// CHECK:           wasmssa.return %[[VAL_1]] : f32
+
+// CHECK-LABEL:   wasmssa.func @abs_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.abs %[[VAL_0]] : f64
+// CHECK:           wasmssa.return %[[VAL_1]] : f64
diff --git a/mlir/test/Target/Wasm/and.mlir b/mlir/test/Target/Wasm/and.mlir
new file mode 100644
index 0000000000000..4c0fea01b3ae6
--- /dev/null
+++ b/mlir/test/Target/Wasm/and.mlir
@@ -0,0 +1,27 @@
+// RUN: yaml2obj %S/inputs/and.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "and_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.and)
+
+    (func (export "and_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.and)
+)
+*/
+
+// CHECK-LABEL: wasmssa.func @and_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.and %0 %1 : i32
+// CHECK:    wasmssa.return %2 : i32
+
+// CHECK-LABEL: wasmssa.func @and_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.and %0 %1 : i64
+// CHECK:    wasmssa.return %2 : i64
diff --git a/mlir/test/Target/Wasm/bad_wasm_version.yaml b/mlir/test/Target/Wasm/bad_wasm_version.yaml
new file mode 100644
index 0000000000000..f834afbef679d
--- /dev/null
+++ b/mlir/test/Target/Wasm/bad_wasm_version.yaml
@@ -0,0 +1,8 @@
+# RUN: yaml2obj %s -o - | not mlir-translate --import-wasm 2>&1 | FileCheck %s
+
+# CHECK: unsupported Wasm version
+
+--- !WASM
+FileHeader:
+  Version:         0xDEADBEEF
+...
diff --git a/mlir/test/Target/Wasm/clz.mlir b/mlir/test/Target/Wasm/clz.mlir
new file mode 100644
index 0000000000000..3e6641d9a2a96
--- /dev/null
+++ b/mlir/test/Target/Wasm/clz.mlir
@@ -0,0 +1,25 @@
+// RUN: yaml2obj %S/inputs/clz.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "clz_i32") (result i32)
+    i32.const 10
+    i32.clz
+    )
+
+    (func (export "clz_i64") (result i64)
+    i64.const 10
+    i64.clz
+    )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @clz_i32() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.clz %[[VAL_0]] : i32
+// CHECK:           wasmssa.return %[[VAL_1]] : i32
+
+// CHECK-LABEL:   wasmssa.func @clz_i64() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.clz %[[VAL_0]] : i64
+// CHECK:           wasmssa.return %[[VAL_1]] : i64
diff --git a/mlir/test/Target/Wasm/const.mlir b/mlir/test/Target/Wasm/const.mlir
new file mode 100644
index 0000000000000..aa9e76f8502b6
--- /dev/null
+++ b/mlir/test/Target/Wasm/const.mlir
@@ -0,0 +1,37 @@
+// RUN: yaml2obj %S/inputs/const.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+/* Source code used to create this test:
+(module
+    (func(result i32)
+        i32.const 1
+    )
+    (func (result i64)
+        i64.const 3
+    )
+    (func (result f32)
+        f32.const 4.0
+    )
+    (func (result f64)
+        f64.const 9.0
+    )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func nested @func_0() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1 : i32
+// CHECK:           wasmssa.return %[[VAL_0]] : i32
+// CHECK:         }
+
+// CHECK-LABEL:   wasmssa.func nested @func_1() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 3 : i64
+// CHECK:           wasmssa.return %[[VAL_0]] : i64
+// CHECK:         }
+
+// CHECK-LABEL:   wasmssa.func nested @func_2() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 4.000000e+00 : f32
+// CHECK:           wasmssa.return %[[VAL_0]] : f32
+// CHECK:         }
+
+// CHECK-LABEL:   wasmssa.func nested @func_3() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 9.000000e+00 : f64
+// CHECK:           wasmssa.return %[[VAL_0]] : f64
+// CHECK:         }
diff --git a/mlir/test/Target/Wasm/copysign.mlir b/mlir/test/Target/Wasm/copysign.mlir
new file mode 100644
index 0000000000000..33d7a56694c1c
--- /dev/null
+++ b/mlir/test/Target/Wasm/copysign.mlir
@@ -0,0 +1,31 @@
+// RUN: yaml2obj %S/inputs/copysign.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "copysign_f32") (result f32)
+    f32.const 10
+    f32.const 1
+    f32.copysign
+    )
+
+    (func (export "copysign_f64") (result f64)
+    f64.const 10
+    f64.const 1
+    f64.copysign
+    )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @copysign_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 1.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.copysign %[[VAL_0]] %[[VAL_1]] : f32
+// CHECK:           wasmssa.return %[[VAL_2]] : f32
+// CHECK:         }
+
+// CHECK-LABEL:   wasmssa.func @copysign_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 1.000000e+00 : f64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.copysign %[[VAL_0]] %[[VAL_1]] : f64
+// CHECK:           wasmssa.return %[[VAL_2]] : f64
+// CHECK:         }
diff --git a/mlir/test/Target/Wasm/ctz.mlir b/mlir/test/Target/Wasm/ctz.mlir
new file mode 100644
index 0000000000000..6c0806f62303c
--- /dev/null
+++ b/mlir/test/Target/Wasm/ctz.mlir
@@ -0,0 +1,25 @@
+// RUN: yaml2obj %S/inputs/ctz.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "ctz_i32") (result i32)
+    i32.const 10
+    i32.ctz
+    )
+
+    (func (export "ctz_i64") (result i64)
+    i64.const 10
+    i64.ctz
+    )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @ctz_i32() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.ctz %[[VAL_0]] : i32
+// CHECK:           wasmssa.return %[[VAL_1]] : i32
+
+// CHECK-LABEL:   wasmssa.func @ctz_i64() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.ctz %[[VAL_0]] : i64
+// CHECK:           wasmssa.return %[[VAL_1]] : i64
diff --git a/mlir/test/Target/Wasm/div.mlir b/mlir/test/Target/Wasm/div.mlir
new file mode 100644
index 0000000000000..c91f7809a255d
--- /dev/null
+++ b/mlir/test/Target/Wasm/div.mlir
@@ -0,0 +1,127 @@
+// RUN: yaml2obj %S/inputs/div.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to create this test:
+(module
+    (func (export "div_u_i32") (result i32)
+        i32.const 10
+        i32.const 2
+        i32.div_u
+    )
+
+    (func (export "div_u_i32_zero") (result i32)
+        i32.const 10
+        i32.const 0
+        i32.div_u
+    )
+
+    (func (export "div_s_i32") (result i32)
+        i32.const 10
+        i32.const 2
+        i32.div_s
+    )
+
+    (func (export "div_s_i32_zero") (result i32)
+        i32.const 10
+        i32.const 0
+        i32.div_s
+    )
+
+    (func (export "div_u_i64") (result i64)
+        i64.const 10
+        i64.const 2
+        i64.div_u
+    )
+
+    ;; explode
+    (func (export "div_u_i64_zero") (result i64)
+        i64.const 10
+        i64.const 0
+        i64.div_u
+    )
+
+    (func (export "div_s_i64") (result i64)
+        i64.const 10
+        i64.const 2
+        i64.div_s
+    )
+
+    ;; explode
+    (func (export "div_s_i64_zero") (result i64)
+        i64.const 10
+        i64.const 0
+        i64.div_s
+    )
+
+    (func (export "div_f32") (result f32)
+        f32.const 10
+        f32.const 2
+        f32.div
+    )
+
+    (func (export "div_f64") (result f64)
+        f64.const 10
+        f64.const 2
+        f64.div
+    )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @div_u_i32() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 2 : i32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div_ui %[[VAL_0]] %[[VAL_1]] : i32
+// CHECK:           wasmssa.return %[[VAL_2]] : i32
+
+// CHECK-LABEL:   wasmssa.func @div_u_i32_zero() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 0 : i32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div_ui %[[VAL_0]] %[[VAL_1]] : i32
+// CHECK:           wasmssa.return %[[VAL_2]] : i32
+
+// CHECK-LABEL:   wasmssa.func @div_s_i32() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 2 : i32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div_si %[[VAL_0]] %[[VAL_1]] : i32
+// CHECK:           wasmssa.return %[[VAL_2]] : i32
+
+// CHECK-LABEL:   wasmssa.func @div_s_i32_zero() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 0 : i32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div_si %[[VAL_0]] %[[VAL_1]] : i32
+// CHECK:           wasmssa.return %[[VAL_2]] : i32
+
+// CHECK-LABEL:   wasmssa.func @div_u_i64() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 2 : i64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div_ui %[[VAL_0]] %[[VAL_1]] : i64
+// CHECK:           wasmssa.return %[[VAL_2]] : i64
+
+// CHECK-LABEL:   wasmssa.func @div_u_i64_zero() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 0 : i64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div_ui %[[VAL_0]] %[[VAL_1]] : i64
+// CHECK:           wasmssa.return %[[VAL_2]] : i64
+
+// CHECK-LABEL:   wasmssa.func @div_s_i64() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 2 : i64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div_si %[[VAL_0]] %[[VAL_1]] : i64
+// CHECK:           wasmssa.return %[[VAL_2]] : i64
+
+// CHECK-LABEL:   wasmssa.func @div_s_i64_zero() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 0 : i64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div_si %[[VAL_0]] %[[VAL_1]] : i64
+// CHECK:           wasmssa.return %[[VAL_2]] : i64
+
+// CHECK-LABEL:   wasmssa.func @div_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 2.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div %[[VAL_0]] %[[VAL_1]] : f32
+// CHECK:           wasmssa.return %[[VAL_2]] : f32
+
+// CHECK-LABEL:   wasmssa.func @div_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 2.000000e+00 : f64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.div %[[VAL_0]] %[[VAL_1]] : f64
+// CHECK:           wasmssa.return %[[VAL_2]] : f64
diff --git a/mlir/test/Target/Wasm/function_export_out_of_scope.yaml b/mlir/test/Target/Wasm/function_export_out_of_scope.yaml
new file mode 100644
index 0000000000000..b08c2c87abdb3
--- /dev/null
+++ b/mlir/test/Target/Wasm/function_export_out_of_scope.yaml
@@ -0,0 +1,13 @@
+# RUN: yaml2obj %s | not mlir-translate --import-wasm -o - 2>&1 | FileCheck %s
+
+# CHECK: trying to export function 42 which is undefined in this scope
+
+--- !WASM
+FileHeader:
+  Version:         0x00000001
+Sections:
+  - Type:            EXPORT
+    Exports:
+      - Name:            function_export
+        Kind:            FUNCTION
+        Index:           42
diff --git a/mlir/test/Target/Wasm/global.mlir b/mlir/test/Target/Wasm/global.mlir
new file mode 100644
index 0000000000000..e72fe69666999
--- /dev/null
+++ b/mlir/test/Target/Wasm/global.mlir
@@ -0,0 +1,66 @@
+// RUN: yaml2obj %S/inputs/global.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+/* Source code used to create this test:
+(module
+
+;; import a global variable from js
+(global $imported_glob (import "env" "from_js") i32)
+
+;; create a global variable
+(global $normal_glob i32(i32.const 10))
+(global $glob_mut (mut i32) (i32.const 10))
+(global $glob_mut_ext (mut i32) (i32.const 10))
+
+(global $normal_glob_i64 i64(i64.const 11))
+(global $normal_glob_f32 f32(f32.const 12))
+(global $normal_glob_f64 f64(f64.const 13))
+
+(func $main (result i32)
+;; load both global variables onto the stack
+global.get $imported_glob
+global.get $normal_glob
+
+i32.add ;; add up both globals
+
+global.get $glob_mut
+global.get $glob_mut_ext
+i32.add
+i32.add
+)
+)
+*/
+
+// CHECK-LABEL:   wasmssa.import_global "from_js" from "env" as @global_0 nested : i32
+
+// CHECK-LABEL:   wasmssa.func nested @func_0() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.global_get @global_0 : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.global_get @global_1 : i32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.add %[[VAL_0]] %[[VAL_1]] : i32
+// CHECK:           %[[VAL_3:.*]] = wasmssa.global_get @global_2 : i32
+// CHECK:           %[[VAL_4:.*]] = wasmssa.global_get @global_3 : i32
+// CHECK:           %[[VAL_5:.*]] = wasmssa.add %[[VAL_3]] %[[VAL_4]] : i32
+// CHECK:           %[[VAL_6:.*]] = wasmssa.add %[[VAL_2]] %[[VAL_5]] : i32
+// CHECK:           wasmssa.return %[[VAL_6]] : i32
+
+// CHECK-LABEL:   wasmssa.global @global_1 i32 nested : {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           wasmssa.return %[[VAL_0]] : i32
+
+// CHECK-LABEL:   wasmssa.global @global_2 i32 mutable nested : {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           wasmssa.return %[[VAL_0]] : i32
+
+// CHECK-LABEL:   wasmssa.global @global_3 i32 mutable nested : {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           wasmssa.return %[[VAL_0]] : i32
+
+// CHECK-LABEL:   wasmssa.global @global_4 i64 nested : {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 11 : i64
+// CHECK:           wasmssa.return %[[VAL_0]] : i64
+
+// CHECK-LABEL:   wasmssa.global @global_5 f32 nested : {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.200000e+01 : f32
+// CHECK:           wasmssa.return %[[VAL_0]] : f32
+
+// CHECK-LABEL:   wasmssa.global @global_6 f64 nested : {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.300000e+01 : f64
+// CHECK:           wasmssa.return %[[VAL_0]] : f64
diff --git a/mlir/test/Target/Wasm/import.mlir b/mlir/test/Target/Wasm/import.mlir
new file mode 100644
index 0000000000000..541dcf3a2d9eb
--- /dev/null
+++ b/mlir/test/Target/Wasm/import.mlir
@@ -0,0 +1,19 @@
+// RUN: yaml2obj %S/inputs/import.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to create this test:
+(module
+(import "my_module" "foo" (func $foo (param i32)))
+(import "my_module" "bar" (func $bar (param i32)))
+(import "my_module" "table" (table $round 2 funcref))
+(import "my_module" "mem" (memory $mymem 2))
+(import "my_module" "glob" (global $globglob i32))
+(import "my_other_module" "glob_mut" (global $glob_mut (mut i32)))
+)
+*/
+
+// CHECK-LABEL:   wasmssa.import_func "foo" from "my_module" as @func_0 {sym_visibility = "nested", type = (i32) -> ()}
+// CHECK:         wasmssa.import_func "bar" from "my_module" as @func_1 {sym_visibility = "nested", type = (i32) -> ()}
+// CHECK:         wasmssa.import_table "table" from "my_module" as @table_0 {sym_visibility = "nested", type = !wasmssa<tabletype !wasmssa.funcref [2:]>}
+// CHECK:         wasmssa.import_mem "mem" from "my_module" as @mem_0 {limits = !wasmssa<limit[2:]>, sym_visibility = "nested"}
+// CHECK:         wasmssa.import_global "glob" from "my_module" as @global_0 nested : i32
+// CHECK:         wasmssa.import_global "glob_mut" from "my_other_module" as @global_1 mutable nested : i32
diff --git a/mlir/test/Target/Wasm/inputs/abs.yaml.wasm b/mlir/test/Target/Wasm/inputs/abs.yaml.wasm
new file mode 100644
index 0000000000000..1cb6d21b2fdd4
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/abs.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - F64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            abs_f32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            abs_f64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            43000020418B0B
+      - Index:           1
+        Locals:          []
+        Body:            440000000000002440990B
+...
diff --git a/mlir/test/Target/Wasm/inputs/and.yaml.wasm b/mlir/test/Target/Wasm/inputs/and.yaml.wasm
new file mode 100644
index 0000000000000..926445b761080
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/and.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            and_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            and_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A4103710B
+      - Index:           1
+        Locals:          []
+        Body:            420A4203830B
+...
diff --git a/mlir/test/Target/Wasm/inputs/clz.yaml.wasm b/mlir/test/Target/Wasm/inputs/clz.yaml.wasm
new file mode 100644
index 0000000000000..f537bdb359241
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/clz.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            clz_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            clz_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A670B
+      - Index:           1
+        Locals:          []
+        Body:            420A790B
+...
diff --git a/mlir/test/Target/Wasm/inputs/const.yaml.wasm b/mlir/test/Target/Wasm/inputs/const.yaml.wasm
new file mode 100644
index 0000000000000..be8f88e2f3be4
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/const.yaml.wasm
@@ -0,0 +1,39 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+      - Index:           2
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           3
+        ParamTypes:      []
+        ReturnTypes:
+          - F64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1, 2, 3 ]
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            41010B
+      - Index:           1
+        Locals:          []
+        Body:            42030B
+      - Index:           2
+        Locals:          []
+        Body:            43000080400B
+      - Index:           3
+        Locals:          []
+        Body:            4400000000000022400B
+...
diff --git a/mlir/test/Target/Wasm/inputs/copysign.yaml.wasm b/mlir/test/Target/Wasm/inputs/copysign.yaml.wasm
new file mode 100644
index 0000000000000..46c04124ac84a
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/copysign.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - F64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            copysign_f32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            copysign_f64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            4300002041430000803F980B
+      - Index:           1
+        Locals:          []
+        Body:            44000000000000244044000000000000F03FA60B
+...
diff --git a/mlir/test/Target/Wasm/inputs/ctz.yaml.wasm b/mlir/test/Target/Wasm/inputs/ctz.yaml.wasm
new file mode 100644
index 0000000000000..51400852d3dce
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/ctz.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            ctz_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            ctz_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A680B
+      - Index:           1
+        Locals:          []
+        Body:            420A7A0B
+...
diff --git a/mlir/test/Target/Wasm/inputs/div.yaml.wasm b/mlir/test/Target/Wasm/inputs/div.yaml.wasm
new file mode 100644
index 0000000000000..648e10c15cbed
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/div.yaml.wasm
@@ -0,0 +1,89 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+      - Index:           2
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           3
+        ParamTypes:      []
+        ReturnTypes:
+          - F64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 0, 0, 0, 1, 1, 1, 1, 2, 3 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            div_u_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            div_u_i32_zero
+        Kind:            FUNCTION
+        Index:           1
+      - Name:            div_s_i32
+        Kind:            FUNCTION
+        Index:           2
+      - Name:            div_s_i32_zero
+        Kind:            FUNCTION
+        Index:           3
+      - Name:            div_u_i64
+        Kind:            FUNCTION
+        Index:           4
+      - Name:            div_u_i64_zero
+        Kind:            FUNCTION
+        Index:           5
+      - Name:            div_s_i64
+        Kind:            FUNCTION
+        Index:           6
+      - Name:            div_s_i64_zero
+        Kind:            FUNCTION
+        Index:           7
+      - Name:            div_f32
+        Kind:            FUNCTION
+        Index:           8
+      - Name:            div_f64
+        Kind:            FUNCTION
+        Index:           9
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A41026E0B
+      - Index:           1
+        Locals:          []
+        Body:            410A41006E0B
+      - Index:           2
+        Locals:          []
+        Body:            410A41026D0B
+      - Index:           3
+        Locals:          []
+        Body:            410A41006D0B
+      - Index:           4
+        Locals:          []
+        Body:            420A4202800B
+      - Index:           5
+        Locals:          []
+        Body:            420A4200800B
+      - Index:           6
+        Locals:          []
+        Body:            420A42027F0B
+      - Index:           7
+        Locals:          []
+        Body:            420A42007F0B
+      - Index:           8
+        Locals:          []
+        Body:            43000020414300000040950B
+      - Index:           9
+        Locals:          []
+        Body:            440000000000002440440000000000000040A30B
+...
diff --git a/mlir/test/Target/Wasm/inputs/global.yaml.wasm b/mlir/test/Target/Wasm/inputs/global.yaml.wasm
new file mode 100644
index 0000000000000..4bbb434c0c51a
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/global.yaml.wasm
@@ -0,0 +1,63 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+  - Type:            IMPORT
+    Imports:
+      - Module:          env
+        Field:           from_js
+        Kind:            GLOBAL
+        GlobalType:      I32
+        GlobalMutable:   false
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0 ]
+  - Type:            GLOBAL
+    Globals:
+      - Index:           1
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           10
+      - Index:           2
+        Type:            I32
+        Mutable:         true
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           10
+      - Index:           3
+        Type:            I32
+        Mutable:         true
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           10
+      - Index:           4
+        Type:            I64
+        Mutable:         false
+        InitExpr:
+          Opcode:          I64_CONST
+          Value:           11
+      - Index:           5
+        Type:            F32
+        Mutable:         false
+        InitExpr:
+          Opcode:          F32_CONST
+          Value:           1094713344
+      - Index:           6
+        Type:            F64
+        Mutable:         false
+        InitExpr:
+          Opcode:          F64_CONST
+          Value:           4623507967449235456
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            230023016A230223036A6A0B
+...
diff --git a/mlir/test/Target/Wasm/inputs/import.yaml.wasm b/mlir/test/Target/Wasm/inputs/import.yaml.wasm
new file mode 100644
index 0000000000000..7c467ff6fbc67
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/import.yaml.wasm
@@ -0,0 +1,44 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:
+          - I32
+        ReturnTypes:     []
+  - Type:            IMPORT
+    Imports:
+      - Module:          my_module
+        Field:           foo
+        Kind:            FUNCTION
+        SigIndex:        0
+      - Module:          my_module
+        Field:           bar
+        Kind:            FUNCTION
+        SigIndex:        0
+      - Module:          my_module
+        Field:           table
+        Kind:            TABLE
+        Table:
+          Index:           0
+          ElemType:        FUNCREF
+          Limits:
+            Minimum:         0x2
+      - Module:          my_module
+        Field:           mem
+        Kind:            MEMORY
+        Memory:
+          Minimum:         0x2
+      - Module:          my_module
+        Field:           glob
+        Kind:            GLOBAL
+        GlobalType:      I32
+        GlobalMutable:   false
+      - Module:          my_other_module
+        Field:           glob_mut
+        Kind:            GLOBAL
+        GlobalType:      I32
+        GlobalMutable:   true
+...
diff --git a/mlir/test/Target/Wasm/inputs/local.yaml.wasm b/mlir/test/Target/Wasm/inputs/local.yaml.wasm
new file mode 100644
index 0000000000000..3e937f3e7402a
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/local.yaml.wasm
@@ -0,0 +1,37 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           2
+        ParamTypes:
+          - I32
+        ReturnTypes:
+          - I32
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1, 2 ]
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:
+          - Type:            F32
+            Count:           2
+        Body:            43000000412100200043000040412201920B
+      - Index:           1
+        Locals:
+          - Type:            I32
+            Count:           2
+        Body:            410821002000410C22016A0B
+      - Index:           2
+        Locals:          []
+        Body:            4103210020000B
+...
diff --git a/mlir/test/Target/Wasm/inputs/max.yaml.wasm b/mlir/test/Target/Wasm/inputs/max.yaml.wasm
new file mode 100644
index 0000000000000..fc04b019fe34c
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/max.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - F64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            min_f32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            min_f64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            4300002041430000803F970B
+      - Index:           1
+        Locals:          []
+        Body:            44000000000000244044000000000000F03FA50B
+...
diff --git a/mlir/test/Target/Wasm/inputs/memory_min_eq_max.yaml.wasm b/mlir/test/Target/Wasm/inputs/memory_min_eq_max.yaml.wasm
new file mode 100644
index 0000000000000..f3edf5f2d0cc2
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/memory_min_eq_max.yaml.wasm
@@ -0,0 +1,10 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            MEMORY
+    Memories:
+      - Flags:           [ HAS_MAX ]
+        Minimum:         0x0
+        Maximum:         0x0
+...
diff --git a/mlir/test/Target/Wasm/inputs/memory_min_max.yaml.wasm b/mlir/test/Target/Wasm/inputs/memory_min_max.yaml.wasm
new file mode 100644
index 0000000000000..fe70fb686df37
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/memory_min_max.yaml.wasm
@@ -0,0 +1,10 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            MEMORY
+    Memories:
+      - Flags:           [ HAS_MAX ]
+        Minimum:         0x0
+        Maximum:         0x10000
+...
diff --git a/mlir/test/Target/Wasm/inputs/memory_min_no_max.yaml.wasm b/mlir/test/Target/Wasm/inputs/memory_min_no_max.yaml.wasm
new file mode 100644
index 0000000000000..8508ce38251a3
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/memory_min_no_max.yaml.wasm
@@ -0,0 +1,8 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            MEMORY
+    Memories:
+      - Minimum:         0x1
+...
diff --git a/mlir/test/Target/Wasm/inputs/min.yaml.wasm b/mlir/test/Target/Wasm/inputs/min.yaml.wasm
new file mode 100644
index 0000000000000..925a5e96b43c9
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/min.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - F64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            min_f32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            min_f64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            4300002041430000803F960B
+      - Index:           1
+        Locals:          []
+        Body:            44000000000000244044000000000000F03FA40B
+...
diff --git a/mlir/test/Target/Wasm/inputs/neg.yaml.wasm b/mlir/test/Target/Wasm/inputs/neg.yaml.wasm
new file mode 100644
index 0000000000000..8392a2bf9a5fb
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/neg.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - F64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            neg_f32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            neg_f64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            43000020418C0B
+      - Index:           1
+        Locals:          []
+        Body:            4400000000000024409A0B
+...
diff --git a/mlir/test/Target/Wasm/inputs/or.yaml.wasm b/mlir/test/Target/Wasm/inputs/or.yaml.wasm
new file mode 100644
index 0000000000000..9528ce8b3f911
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/or.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            or_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            or_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A4103720B
+      - Index:           1
+        Locals:          []
+        Body:            420A4203840B
+...
diff --git a/mlir/test/Target/Wasm/inputs/popcnt.yaml.wasm b/mlir/test/Target/Wasm/inputs/popcnt.yaml.wasm
new file mode 100644
index 0000000000000..03c57ad0841b9
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/popcnt.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            popcnt_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            popcnt_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A690B
+      - Index:           1
+        Locals:          []
+        Body:            420A7B0B
+...
diff --git a/mlir/test/Target/Wasm/inputs/rem.yaml.wasm b/mlir/test/Target/Wasm/inputs/rem.yaml.wasm
new file mode 100644
index 0000000000000..468a9db6f05ae
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/rem.yaml.wasm
@@ -0,0 +1,45 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1, 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            rem_u_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            rem_u_i64
+        Kind:            FUNCTION
+        Index:           1
+      - Name:            rem_s_i32
+        Kind:            FUNCTION
+        Index:           2
+      - Name:            rem_s_i64
+        Kind:            FUNCTION
+        Index:           3
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A4103700B
+      - Index:           1
+        Locals:          []
+        Body:            420A4203820B
+      - Index:           2
+        Locals:          []
+        Body:            410A41036F0B
+      - Index:           3
+        Locals:          []
+        Body:            420A4203810B
+...
diff --git a/mlir/test/Target/Wasm/inputs/rotl.yaml.wasm b/mlir/test/Target/Wasm/inputs/rotl.yaml.wasm
new file mode 100644
index 0000000000000..87466cb16393f
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/rotl.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            rotl_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            rotl_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A4103770B
+      - Index:           1
+        Locals:          []
+        Body:            420A4203890B
+...
diff --git a/mlir/test/Target/Wasm/inputs/rotr.yaml.wasm b/mlir/test/Target/Wasm/inputs/rotr.yaml.wasm
new file mode 100644
index 0000000000000..805a93fd55a8c
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/rotr.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            rotr_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            rotr_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A4103780B
+      - Index:           1
+        Locals:          []
+        Body:            420A42038A0B
+...
diff --git a/mlir/test/Target/Wasm/inputs/shl.yaml.wasm b/mlir/test/Target/Wasm/inputs/shl.yaml.wasm
new file mode 100644
index 0000000000000..d07605e599483
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/shl.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            shl_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            shl_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A4103740B
+      - Index:           1
+        Locals:          []
+        Body:            420A4203860B
+...
diff --git a/mlir/test/Target/Wasm/inputs/shr_s.yaml.wasm b/mlir/test/Target/Wasm/inputs/shr_s.yaml.wasm
new file mode 100644
index 0000000000000..d5d8013a2f743
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/shr_s.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            shr_s_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            shr_s_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A4103750B
+      - Index:           1
+        Locals:          []
+        Body:            420A4203870B
+...
diff --git a/mlir/test/Target/Wasm/inputs/shr_u.yaml.wasm b/mlir/test/Target/Wasm/inputs/shr_u.yaml.wasm
new file mode 100644
index 0000000000000..cd815144432f7
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/shr_u.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            shr_u_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            shr_u_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A4103760B
+      - Index:           1
+        Locals:          []
+        Body:            420A4203880B
+...
diff --git a/mlir/test/Target/Wasm/inputs/sqrt.yaml.wasm b/mlir/test/Target/Wasm/inputs/sqrt.yaml.wasm
new file mode 100644
index 0000000000000..f8ab84be9255f
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/sqrt.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - F64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            sqrt_f32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            sqrt_f64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            4300002041910B
+      - Index:           1
+        Locals:          []
+        Body:            4400000000000024409F0B
+...
diff --git a/mlir/test/Target/Wasm/inputs/stats.yaml.wasm b/mlir/test/Target/Wasm/inputs/stats.yaml.wasm
new file mode 100644
index 0000000000000..bf577688b3aed
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/stats.yaml.wasm
@@ -0,0 +1,38 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:
+          - I32
+        ReturnTypes:
+          - I32
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0 ]
+  - Type:            TABLE
+    Tables:
+      - Index:           0
+        ElemType:        FUNCREF
+        Limits:
+          Minimum:         0x2
+  - Type:            MEMORY
+    Memories:
+      - Flags:           [ HAS_MAX ]
+        Minimum:         0x0
+        Maximum:         0x10000
+  - Type:            GLOBAL
+    Globals:
+      - Index:           0
+        Type:            I32
+        Mutable:         false
+        InitExpr:
+          Opcode:          I32_CONST
+          Value:           10
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            20000B
+...
diff --git a/mlir/test/Target/Wasm/inputs/sub.yaml.wasm b/mlir/test/Target/Wasm/inputs/sub.yaml.wasm
new file mode 100644
index 0000000000000..95b6bcc1d9021
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/sub.yaml.wasm
@@ -0,0 +1,39 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+      - Index:           2
+        ParamTypes:      []
+        ReturnTypes:
+          - F32
+      - Index:           3
+        ParamTypes:      []
+        ReturnTypes:
+          - F64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1, 2, 3 ]
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410C41326B0B
+      - Index:           1
+        Locals:          []
+        Body:            421442057D0B
+      - Index:           2
+        Locals:          []
+        Body:            430000A0404300006041930B
+      - Index:           3
+        Locals:          []
+        Body:            440000000000003140440000000000000000A10B
+...
diff --git a/mlir/test/Target/Wasm/inputs/table.yaml.wasm b/mlir/test/Target/Wasm/inputs/table.yaml.wasm
new file mode 100644
index 0000000000000..387f41820524f
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/table.yaml.wasm
@@ -0,0 +1,23 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TABLE
+    Tables:
+      - Index:           0
+        ElemType:        FUNCREF
+        Limits:
+          Minimum:         0x2
+      - Index:           1
+        ElemType:        FUNCREF
+        Limits:
+          Flags:           [ HAS_MAX ]
+          Minimum:         0x2
+          Maximum:         0x4
+      - Index:           2
+        ElemType:        EXTERNREF
+        Limits:
+          Flags:           [ HAS_MAX ]
+          Minimum:         0x2
+          Maximum:         0x4
+...
diff --git a/mlir/test/Target/Wasm/inputs/xor.yaml.wasm b/mlir/test/Target/Wasm/inputs/xor.yaml.wasm
new file mode 100644
index 0000000000000..45079c3b4fed3
--- /dev/null
+++ b/mlir/test/Target/Wasm/inputs/xor.yaml.wasm
@@ -0,0 +1,33 @@
+--- !WASM
+FileHeader:
+  Version:         0x1
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:
+          - I32
+      - Index:           1
+        ParamTypes:      []
+        ReturnTypes:
+          - I64
+  - Type:            FUNCTION
+    FunctionTypes:   [ 0, 1 ]
+  - Type:            EXPORT
+    Exports:
+      - Name:            xor_i32
+        Kind:            FUNCTION
+        Index:           0
+      - Name:            xor_i64
+        Kind:            FUNCTION
+        Index:           1
+  - Type:            CODE
+    Functions:
+      - Index:           0
+        Locals:          []
+        Body:            410A4103730B
+      - Index:           1
+        Locals:          []
+        Body:            420A4203850B
+...
diff --git a/mlir/test/Target/Wasm/invalid_function_type_index.yaml b/mlir/test/Target/Wasm/invalid_function_type_index.yaml
new file mode 100644
index 0000000000000..2d2954aa32dda
--- /dev/null
+++ b/mlir/test/Target/Wasm/invalid_function_type_index.yaml
@@ -0,0 +1,16 @@
+# RUN: yaml2obj %s | not mlir-translate --import-wasm -o - 2>&1 | FileCheck %s
+# CHECK: error: invalid type index: 2
+
+--- !WASM
+FileHeader:
+  Version:         0x00000001
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:
+          - I32
+        ReturnTypes:     []
+  - Type:            FUNCTION
+    FunctionTypes:
+      - 2
diff --git a/mlir/test/Target/Wasm/local.mlir b/mlir/test/Target/Wasm/local.mlir
new file mode 100644
index 0000000000000..32f590021b959
--- /dev/null
+++ b/mlir/test/Target/Wasm/local.mlir
@@ -0,0 +1,59 @@
+// RUN: yaml2obj %S/inputs/local.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+/* Source code used to create this test:
+(module
+  (func $local_f32 (result f32)
+    (local $var1 f32)
+    (local $var2 f32)
+    f32.const 8.0
+    local.set $var1
+    local.get $var1
+    f32.const 12.0
+    local.tee $var2
+    f32.add
+  )
+  (func $local_i32 (result i32)
+    (local $var1 i32)
+    (local $var2 i32)
+    i32.const 8
+    local.set $var1
+    local.get $var1
+    i32.const 12
+    local.tee $var2
+    i32.add
+  )
+  (func $local_arg (param $var i32) (result i32)
+    i32.const 3
+    local.set $var
+    local.get $var
+  )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func nested @func_0() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.local of type f32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.local of type f32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.const 8.000000e+00 : f32
+// CHECK:           wasmssa.local_set %[[VAL_0]] :  ref to f32 to %[[VAL_2]] : f32
+// CHECK:           %[[VAL_3:.*]] = wasmssa.local_get %[[VAL_0]] :  ref to f32
+// CHECK:           %[[VAL_4:.*]] = wasmssa.const 1.200000e+01 : f32
+// CHECK:           %[[VAL_5:.*]] = wasmssa.local_tee %[[VAL_1]] :  ref to f32 to %[[VAL_4]] : f32
+// CHECK:           %[[VAL_6:.*]] = wasmssa.add %[[VAL_3]] %[[VAL_5]] : f32
+// CHECK:           wasmssa.return %[[VAL_6]] : f32
+
+// CHECK-LABEL:   wasmssa.func nested @func_1() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.local of type i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.local of type i32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.const 8 : i32
+// CHECK:           wasmssa.local_set %[[VAL_0]] :  ref to i32 to %[[VAL_2]] : i32
+// CHECK:           %[[VAL_3:.*]] = wasmssa.local_get %[[VAL_0]] :  ref to i32
+// CHECK:           %[[VAL_4:.*]] = wasmssa.const 12 : i32
+// CHECK:           %[[VAL_5:.*]] = wasmssa.local_tee %[[VAL_1]] :  ref to i32 to %[[VAL_4]] : i32
+// CHECK:           %[[VAL_6:.*]] = wasmssa.add %[[VAL_3]] %[[VAL_5]] : i32
+// CHECK:           wasmssa.return %[[VAL_6]] : i32
+
+// CHECK-LABEL:   wasmssa.func nested @func_2(
+// CHECK-SAME:      %[[ARG0:.*]]: !wasmssa<local ref to i32>) -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 3 : i32
+// CHECK:           wasmssa.local_set %[[ARG0]] :  ref to i32 to %[[VAL_0]] : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.local_get %[[ARG0]] :  ref to i32
+// CHECK:           wasmssa.return %[[VAL_1]] : i32
diff --git a/mlir/test/Target/Wasm/max.mlir b/mlir/test/Target/Wasm/max.mlir
new file mode 100644
index 0000000000000..4ef2042d923b6
--- /dev/null
+++ b/mlir/test/Target/Wasm/max.mlir
@@ -0,0 +1,30 @@
+// RUN: yaml2obj %S/inputs/max.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "max_f32") (result f32)
+    f32.const 10
+    f32.const 1
+    f32.max
+    )
+
+    (func (export "max_f64") (result f64)
+    f64.const 10
+    f64.const 1
+    f64.max
+    )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @min_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 1.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.max %[[VAL_0]] %[[VAL_1]] : f32
+// CHECK:           wasmssa.return %[[VAL_2]] : f32
+
+
+// CHECK-LABEL:   wasmssa.func @min_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 1.000000e+00 : f64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.max %[[VAL_0]] %[[VAL_1]] : f64
+// CHECK:           wasmssa.return %[[VAL_2]] : f64
diff --git a/mlir/test/Target/Wasm/memory_min_eq_max.mlir b/mlir/test/Target/Wasm/memory_min_eq_max.mlir
new file mode 100644
index 0000000000000..2ba5ab50d51fa
--- /dev/null
+++ b/mlir/test/Target/Wasm/memory_min_eq_max.mlir
@@ -0,0 +1,7 @@
+// RUN: yaml2obj %S/inputs/memory_min_eq_max.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to create this test:
+(module (memory 0 0))
+*/
+
+// CHECK-LABEL:   wasmssa.memory @mem_0 nested !wasmssa<limit[0: 0]>
diff --git a/mlir/test/Target/Wasm/memory_min_max.mlir b/mlir/test/Target/Wasm/memory_min_max.mlir
new file mode 100644
index 0000000000000..ebf64189189f8
--- /dev/null
+++ b/mlir/test/Target/Wasm/memory_min_max.mlir
@@ -0,0 +1,7 @@
+// RUN: yaml2obj %S/inputs/memory_min_max.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to create this test:
+(module (memory 0 65536))
+*/
+
+// CHECK-LABEL:  wasmssa.memory @mem_0 nested !wasmssa<limit[0: 65536]>
diff --git a/mlir/test/Target/Wasm/memory_min_no_max.mlir b/mlir/test/Target/Wasm/memory_min_no_max.mlir
new file mode 100644
index 0000000000000..8d8878618bcc0
--- /dev/null
+++ b/mlir/test/Target/Wasm/memory_min_no_max.mlir
@@ -0,0 +1,7 @@
+// RUN: yaml2obj %S/inputs/memory_min_no_max.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to create this test:
+(module (memory 1))
+*/
+
+// CHECK-LABEL:  wasmssa.memory @mem_0 nested !wasmssa<limit[1:]>
diff --git a/mlir/test/Target/Wasm/min.mlir b/mlir/test/Target/Wasm/min.mlir
new file mode 100644
index 0000000000000..1058c7d32be71
--- /dev/null
+++ b/mlir/test/Target/Wasm/min.mlir
@@ -0,0 +1,29 @@
+// RUN: yaml2obj %S/inputs/min.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "min_f32") (result f32)
+    f32.const 10
+    f32.const 1
+    f32.min
+    )
+
+    (func (export "min_f64") (result f64)
+    f64.const 10
+    f64.const 1
+    f64.min
+    )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @min_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 1.000000e+00 : f32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.min %[[VAL_0]] %[[VAL_1]] : f32
+// CHECK:           wasmssa.return %[[VAL_2]] : f32
+
+// CHECK-LABEL:   wasmssa.func @min_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 1.000000e+00 : f64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.min %[[VAL_0]] %[[VAL_1]] : f64
+// CHECK:           wasmssa.return %[[VAL_2]] : f64
diff --git a/mlir/test/Target/Wasm/missing_header.yaml b/mlir/test/Target/Wasm/missing_header.yaml
new file mode 100644
index 0000000000000..a9f812e0a77f8
--- /dev/null
+++ b/mlir/test/Target/Wasm/missing_header.yaml
@@ -0,0 +1,12 @@
+# RUN: not yaml2obj %s -o - | not mlir-translate --import-wasm 2>&1 | FileCheck %s
+
+# CHECK: source file does not contain valid Wasm header
+
+--- !WASM
+Sections:
+  - Type:            TYPE
+    Signatures:
+      - Index:           0
+        ParamTypes:      []
+        ReturnTypes:     []
+...
diff --git a/mlir/test/Target/Wasm/neg.mlir b/mlir/test/Target/Wasm/neg.mlir
new file mode 100644
index 0000000000000..5811ab50348bd
--- /dev/null
+++ b/mlir/test/Target/Wasm/neg.mlir
@@ -0,0 +1,23 @@
+// RUN: yaml2obj %S/inputs/neg.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "neg_f32") (result f32)
+    f32.const 10
+    f32.neg)
+
+    (func (export "neg_f64") (result f64)
+    f64.const 10
+    f64.neg)
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @neg_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.neg %[[VAL_0]] : f32
+// CHECK:           wasmssa.return %[[VAL_1]] : f32
+
+// CHECK-LABEL:   wasmssa.func @neg_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.neg %[[VAL_0]] : f64
+// CHECK:           wasmssa.return %[[VAL_1]] : f64
diff --git a/mlir/test/Target/Wasm/or.mlir b/mlir/test/Target/Wasm/or.mlir
new file mode 100644
index 0000000000000..521f2bac9fa6a
--- /dev/null
+++ b/mlir/test/Target/Wasm/or.mlir
@@ -0,0 +1,27 @@
+// RUN: yaml2obj %S/inputs/or.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "or_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.or)
+
+    (func (export "or_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.or)
+)
+*/
+
+// CHECK-LABEL: wasmssa.func @or_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.or %0 %1 : i32
+// CHECK:    wasmssa.return %2 : i32
+
+// CHECK-LABEL: wasmssa.func @or_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.or %0 %1 : i64
+// CHECK:    wasmssa.return %2 : i64
diff --git a/mlir/test/Target/Wasm/popcnt.mlir b/mlir/test/Target/Wasm/popcnt.mlir
new file mode 100644
index 0000000000000..235333a085177
--- /dev/null
+++ b/mlir/test/Target/Wasm/popcnt.mlir
@@ -0,0 +1,25 @@
+// RUN: yaml2obj %S/inputs/popcnt.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "popcnt_i32") (result i32)
+    i32.const 10
+    i32.popcnt
+    )
+
+    (func (export "popcnt_i64") (result i64)
+    i64.const 10
+    i64.popcnt
+    )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @popcnt_i32() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.popcnt %[[VAL_0]] : i32
+// CHECK:           wasmssa.return %[[VAL_1]] : i32
+
+// CHECK-LABEL:   wasmssa.func @popcnt_i64() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 10 : i64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.popcnt %[[VAL_0]] : i64
+// CHECK:           wasmssa.return %[[VAL_1]] : i64
diff --git a/mlir/test/Target/Wasm/rem.mlir b/mlir/test/Target/Wasm/rem.mlir
new file mode 100644
index 0000000000000..b19b8d9d45ed7
--- /dev/null
+++ b/mlir/test/Target/Wasm/rem.mlir
@@ -0,0 +1,53 @@
+// RUN: yaml2obj %S/inputs/rem.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "rem_u_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.rem_u)
+
+    (func (export "rem_u_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.rem_u)
+
+    (func (export "rem_s_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.rem_s)
+
+    (func (export "rem_s_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.rem_s)
+)
+*/
+
+// CHECK-LABEL: wasmssa.func @rem_u_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.rem_ui %0 %1 : i32
+// CHECK:    wasmssa.return %2 : i32
+// CHECK:  }
+
+// CHECK-LABEL: wasmssa.func @rem_u_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.rem_ui %0 %1 : i64
+// CHECK:    wasmssa.return %2 : i64
+// CHECK:  }
+
+// CHECK-LABEL:  wasmssa.func @rem_s_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.rem_si %0 %1 : i32
+// CHECK:    wasmssa.return %2 : i32
+// CHECK:  }
+
+// CHECK-LABEL:  wasmssa.func @rem_s_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.rem_si %0 %1 : i64
+// CHECK:    wasmssa.return %2 : i64
+// CHECK:  }
diff --git a/mlir/test/Target/Wasm/rotl.mlir b/mlir/test/Target/Wasm/rotl.mlir
new file mode 100644
index 0000000000000..ec573554f0a8c
--- /dev/null
+++ b/mlir/test/Target/Wasm/rotl.mlir
@@ -0,0 +1,27 @@
+// RUN: yaml2obj %S/inputs/rotl.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "rotl_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.rotl)
+
+    (func (export "rotl_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.rotl)
+)
+*/
+
+// CHECK-LABEL: wasmssa.func @rotl_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.rotl %0 by %1 bits : i32
+// CHECK:    wasmssa.return %2 : i32
+
+// CHECK-LABEL: wasmssa.func @rotl_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.rotl %0 by %1 bits : i64
+// CHECK:    wasmssa.return %2 : i64
diff --git a/mlir/test/Target/Wasm/rotr.mlir b/mlir/test/Target/Wasm/rotr.mlir
new file mode 100644
index 0000000000000..5618b432693ad
--- /dev/null
+++ b/mlir/test/Target/Wasm/rotr.mlir
@@ -0,0 +1,27 @@
+// RUN: yaml2obj %S/inputs/rotr.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "rotr_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.rotr)
+
+    (func (export "rotr_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.rotr)
+)
+*/
+
+// CHECK-LABEL: wasmssa.func @rotr_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.rotr %0 by %1 bits : i32
+// CHECK:    wasmssa.return %2 : i32
+
+// CHECK-LABEL: wasmssa.func @rotr_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.rotr %0 by %1 bits : i64
+// CHECK:    wasmssa.return %2 : i64
diff --git a/mlir/test/Target/Wasm/shl.mlir b/mlir/test/Target/Wasm/shl.mlir
new file mode 100644
index 0000000000000..f2bdd573fab81
--- /dev/null
+++ b/mlir/test/Target/Wasm/shl.mlir
@@ -0,0 +1,27 @@
+// RUN: yaml2obj %S/inputs/shl.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "shl_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.shl)
+
+    (func (export "shl_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.shl)
+)
+*/
+
+// CHECK-LABEL: wasmssa.func @shl_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.shl %0 by %1 bits : i32
+// CHECK:    wasmssa.return %2 : i32
+
+// CHECK-LABEL: wasmssa.func @shl_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.shl %0 by %1 bits : i64
+// CHECK:    wasmssa.return %2 : i64
diff --git a/mlir/test/Target/Wasm/shr_s.mlir b/mlir/test/Target/Wasm/shr_s.mlir
new file mode 100644
index 0000000000000..247d9be28403c
--- /dev/null
+++ b/mlir/test/Target/Wasm/shr_s.mlir
@@ -0,0 +1,27 @@
+// RUN: yaml2obj %S/inputs/shr_s.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "shr_s_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.shr_s)
+
+    (func (export "shr_s_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.shr_s)
+)
+*/
+
+// CHECK-LABEL: wasmssa.func @shr_s_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.shr_s %0 by %1 bits : i32
+// CHECK:    wasmssa.return %2 : i32
+
+// CHECK-LABEL: wasmssa.func @shr_s_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.shr_s %0 by %1 bits : i64
+// CHECK:    wasmssa.return %2 : i64
diff --git a/mlir/test/Target/Wasm/shr_u.mlir b/mlir/test/Target/Wasm/shr_u.mlir
new file mode 100644
index 0000000000000..9a79eed9b1d4a
--- /dev/null
+++ b/mlir/test/Target/Wasm/shr_u.mlir
@@ -0,0 +1,27 @@
+// RUN: yaml2obj %S/inputs/shr_u.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "shr_u_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.shr_u)
+
+    (func (export "shr_u_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.shr_u)
+)
+*/
+
+// CHECK-LABEL: wasmssa.func @shr_u_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.shr_u %0 by %1 bits : i32
+// CHECK:    wasmssa.return %2 : i32
+
+// CHECK-LABEL: wasmssa.func @shr_u_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.shr_u %0 by %1 bits : i64
+// CHECK:    wasmssa.return %2 : i64
diff --git a/mlir/test/Target/Wasm/sqrt.mlir b/mlir/test/Target/Wasm/sqrt.mlir
new file mode 100644
index 0000000000000..77444ad69b16f
--- /dev/null
+++ b/mlir/test/Target/Wasm/sqrt.mlir
@@ -0,0 +1,23 @@
+// RUN: yaml2obj %S/inputs/sqrt.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "sqrt_f32") (result f32)
+    f32.const 10
+    f32.sqrt)
+
+    (func (export "sqrt_f64") (result f64)
+    f64.const 10
+    f64.sqrt)
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func @sqrt_f32() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.sqrt %[[VAL_0]] : f32
+// CHECK:           wasmssa.return %[[VAL_1]] : f32
+
+// CHECK-LABEL:   wasmssa.func @sqrt_f64() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.000000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.sqrt %[[VAL_0]] : f64
+// CHECK:           wasmssa.return %[[VAL_1]] : f64
diff --git a/mlir/test/Target/Wasm/sub.mlir b/mlir/test/Target/Wasm/sub.mlir
new file mode 100644
index 0000000000000..b9c6caf006583
--- /dev/null
+++ b/mlir/test/Target/Wasm/sub.mlir
@@ -0,0 +1,52 @@
+// RUN: yaml2obj %S/inputs/sub.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+/* Source code used to create this test:
+(module
+    (func $sub_i32 (result i32)
+        i32.const 12
+        i32.const 50
+        i32.sub
+    )
+
+    (func $sub_i64 (result i64)
+        i64.const 20
+        i64.const 5
+        i64.sub
+    )
+
+    (func $sub_f32 (result f32)
+        f32.const 5
+        f32.const 14
+        f32.sub
+    )
+
+    (func $sub_f64 (result f64)
+        f64.const 17
+        f64.const 0
+        f64.sub
+    )
+)
+*/
+
+// CHECK-LABEL:   wasmssa.func nested @func_0() -> i32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 12 : i32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 50 : i32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.sub %[[VAL_0]] %[[VAL_1]] : i32
+// CHECK:           wasmssa.return %[[VAL_2]] : i32
+
+// CHECK-LABEL:   wasmssa.func nested @func_1() -> i64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 20 : i64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 5 : i64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.sub %[[VAL_0]] %[[VAL_1]] : i64
+// CHECK:           wasmssa.return %[[VAL_2]] : i64
+
+// CHECK-LABEL:   wasmssa.func nested @func_2() -> f32 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 5.000000e+00 : f32
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 1.400000e+01 : f32
+// CHECK:           %[[VAL_2:.*]] = wasmssa.sub %[[VAL_0]] %[[VAL_1]] : f32
+// CHECK:           wasmssa.return %[[VAL_2]] : f32
+
+// CHECK-LABEL:   wasmssa.func nested @func_3() -> f64 {
+// CHECK:           %[[VAL_0:.*]] = wasmssa.const 1.700000e+01 : f64
+// CHECK:           %[[VAL_1:.*]] = wasmssa.const 0.000000e+00 : f64
+// CHECK:           %[[VAL_2:.*]] = wasmssa.sub %[[VAL_0]] %[[VAL_1]] : f64
+// CHECK:           wasmssa.return %[[VAL_2]] : f64
diff --git a/mlir/test/Target/Wasm/xor.mlir b/mlir/test/Target/Wasm/xor.mlir
new file mode 100644
index 0000000000000..94691de2b3be4
--- /dev/null
+++ b/mlir/test/Target/Wasm/xor.mlir
@@ -0,0 +1,27 @@
+// RUN: yaml2obj %S/inputs/xor.yaml.wasm -o - | mlir-translate --import-wasm | FileCheck %s
+
+/* Source code used to generate this test:
+(module
+    (func (export "xor_i32") (result i32)
+    i32.const 10
+    i32.const 3
+    i32.xor)
+
+    (func (export "xor_i64") (result i64)
+    i64.const 10
+    i64.const 3
+    i64.xor)
+)
+*/
+
+// CHECK-LABEL: wasmssa.func @xor_i32() -> i32 {
+// CHECK:    %0 = wasmssa.const 10 : i32
+// CHECK:    %1 = wasmssa.const 3 : i32
+// CHECK:    %2 = wasmssa.xor %0 %1 : i32
+// CHECK:    wasmssa.return %2 : i32
+
+// CHECK-LABEL: wasmssa.func @xor_i64() -> i64 {
+// CHECK:    %0 = wasmssa.const 10 : i64
+// CHECK:    %1 = wasmssa.const 3 : i64
+// CHECK:    %2 = wasmssa.xor %0 %1 : i64
+// CHECK:    wasmssa.return %2 : i64
diff --git a/mlir/test/Transforms/inlining.mlir b/mlir/test/Transforms/inlining.mlir
index 1ed08878430b5..d8e10aa4212ba 100644
--- a/mlir/test/Transforms/inlining.mlir
+++ b/mlir/test/Transforms/inlining.mlir
@@ -5,14 +5,18 @@
 // RUN: mlir-opt %s -inline='op-pipelines=func.func(canonicalize,cse)' | FileCheck %s --check-prefix INLINE_SIMPLIFY
 
 // Inline a function that takes an argument.
-func.func @func_with_arg(%c : i32) -> i32 {
-  %b = arith.addi %c, %c : i32
-  return %b : i32
+func.func @func_with_arg(%arg0 : i32) -> i32 {
+  %b = arith.addi %arg0, %arg0 : i32
+  %c = builtin.unrealized_conversion_cast %b : i32 to i64
+  %d = builtin.unrealized_conversion_cast %c : i64 to i32
+  return %d : i32
 }
 
 // CHECK-LABEL: func @inline_with_arg
 func.func @inline_with_arg(%arg0 : i32) -> i32 {
   // CHECK-NEXT: arith.addi
+  // CHECK-NEXT: unrealized_conversion_cast
+  // CHECK-NEXT: unrealized_conversion_cast
   // CHECK-NEXT: return
 
   %0 = call @func_with_arg(%arg0) : (i32) -> i32
diff --git a/mlir/test/Transforms/remove-dead-values.mlir b/mlir/test/Transforms/remove-dead-values.mlir
index 9ded6a30d9c95..0f8d757086e87 100644
--- a/mlir/test/Transforms/remove-dead-values.mlir
+++ b/mlir/test/Transforms/remove-dead-values.mlir
@@ -571,3 +571,24 @@ module @return_void_with_unused_argument {
   }
 }
 
+// -----
+
+// CHECK-LABEL: module @dynamically_unreachable
+module @dynamically_unreachable {
+  func.func @dynamically_unreachable() {
+    // This value is used by an operation in a dynamically unreachable block.
+    %zero = arith.constant 0 : i64
+
+    // Dataflow analysis knows from the constant condition that
+    // ^bb1 is unreachable
+    %false = arith.constant false
+    cf.cond_br %false, ^bb1, ^bb4
+  ^bb1:
+    // This unreachable operation should be removed.
+    // CHECK-NOT: arith.cmpi
+    %3 = arith.cmpi eq, %zero, %zero : i64
+    cf.br ^bb1
+  ^bb4:
+    return
+  }
+}
diff --git a/mlir/test/Transforms/test-canonicalize.mlir b/mlir/test/Transforms/test-canonicalize.mlir
index 0fc822b0a23ae..8cad6b98441df 100644
--- a/mlir/test/Transforms/test-canonicalize.mlir
+++ b/mlir/test/Transforms/test-canonicalize.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -pass-pipeline='builtin.module(func.func(canonicalize))' | FileCheck %s
+// RUN: mlir-opt %s -pass-pipeline='builtin.module(func.func(canonicalize))' | FileCheck %s  --check-prefixes=CHECK,RS
 // RUN: mlir-opt %s -pass-pipeline='builtin.module(func.func(canonicalize{region-simplify=disabled}))' | FileCheck %s --check-prefixes=CHECK,NO-RS
 
 // CHECK-LABEL: func @remove_op_with_inner_ops_pattern
@@ -80,12 +80,10 @@ func.func @test_dialect_canonicalizer() -> (i32) {
 
 // Check that the option to control region simplification actually works
 // CHECK-LABEL: test_region_simplify
-func.func @test_region_simplify() {
-  // CHECK-NEXT:   return
-  // NO-RS-NEXT: ^bb1
-  // NO-RS-NEXT:   return
-  // CHECK-NEXT: }
-  return
-^bb1:
-  return
+func.func @test_region_simplify(%input1 : i32, %cond : i1) -> i32 {
+  // RS-NEXT: "test.br"(%arg0)[^bb1] : (i32) -> ()
+  // NO-RS-NEXT: "test.br"(%arg0, %arg0)[^bb1] : (i32, i32) -> ()
+   "test.br"(%input1, %input1)[^bb1] : (i32, i32) -> ()
+^bb1(%used_arg : i32, %unused_arg : i32):
+  return %used_arg : i32
 }
diff --git a/mlir/test/Transforms/test-legalizer-fold-after.mlir b/mlir/test/Transforms/test-legalizer-fold-after.mlir
new file mode 100644
index 0000000000000..7f80252dc9604
--- /dev/null
+++ b/mlir/test/Transforms/test-legalizer-fold-after.mlir
@@ -0,0 +1,9 @@
+// RUN: mlir-opt %s -test-legalize-patterns="test-legalize-folding-mode=after-patterns" | FileCheck %s
+
+// CHECK-LABEL: @fold_legalization
+func.func @fold_legalization() -> i32 {
+  // CHECK-NOT: op_in_place_self_fold
+  // CHECK: 97
+  %1 = "test.op_in_place_self_fold"() : () -> (i32)
+  "test.return"(%1) : (i32) -> ()
+}
diff --git a/mlir/test/Transforms/test-legalizer-fold-before.mlir b/mlir/test/Transforms/test-legalizer-fold-before.mlir
new file mode 100644
index 0000000000000..fe6e29351a5d7
--- /dev/null
+++ b/mlir/test/Transforms/test-legalizer-fold-before.mlir
@@ -0,0 +1,9 @@
+// RUN: mlir-opt %s -test-legalize-patterns="test-legalize-folding-mode=before-patterns" | FileCheck %s
+
+// CHECK-LABEL: @fold_legalization
+func.func @fold_legalization() -> i32 {
+  // CHECK: op_in_place_self_fold
+  // CHECK-SAME: folded
+  %1 = "test.op_in_place_self_fold"() : () -> (i32)
+  "test.return"(%1) : (i32) -> ()
+}
diff --git a/mlir/test/Transforms/test-legalizer-no-fold.mlir b/mlir/test/Transforms/test-legalizer-no-fold.mlir
new file mode 100644
index 0000000000000..720d17f41943d
--- /dev/null
+++ b/mlir/test/Transforms/test-legalizer-no-fold.mlir
@@ -0,0 +1,12 @@
+// RUN: mlir-opt %s -allow-unregistered-dialect -test-legalize-patterns="test-legalize-folding-mode=never" | FileCheck %s
+
+// CHECK-LABEL: @remove_foldable_op(
+func.func @remove_foldable_op(%arg0 : i32) -> (i32) {
+  // Check that op was not folded.
+  // CHECK: "test.op_with_region_fold"
+  %0 = "test.op_with_region_fold"(%arg0) ({
+    "foo.op_with_region_terminator"() : () -> ()
+  }) : (i32) -> (i32)
+  "test.return"(%0) : (i32) -> ()
+}
+
diff --git a/mlir/test/Transforms/test-legalizer.mlir b/mlir/test/Transforms/test-legalizer.mlir
index e4406e60ffead..3fa42ff6b2757 100644
--- a/mlir/test/Transforms/test-legalizer.mlir
+++ b/mlir/test/Transforms/test-legalizer.mlir
@@ -1,9 +1,15 @@
-// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns -verify-diagnostics -profile-actions-to=- %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns="allow-pattern-rollback=1" -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns="allow-pattern-rollback=1" -verify-diagnostics -profile-actions-to=- %s | FileCheck %s --check-prefix=CHECK-PROFILER
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns="allow-pattern-rollback=0" -verify-diagnostics %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -split-input-file -test-legalize-patterns="allow-pattern-rollback=0 build-materializations=0 attach-debug-materialization-kind=1" -verify-diagnostics %s | FileCheck %s --check-prefix=CHECK-KIND
+
+// CHECK-PROFILER: "name": "pass-execution", "cat": "PERF", "ph": "B"
+// CHECK-PROFILER: "name": "apply-conversion", "cat": "PERF", "ph": "B"
+// CHECK-PROFILER: "name": "apply-pattern", "cat": "PERF", "ph": "B"
+// CHECK-PROFILER: "name": "apply-pattern", "cat": "PERF", "ph": "E"
+// CHECK-PROFILER: "name": "apply-conversion", "cat": "PERF", "ph": "E"
+// CHECK-PROFILER: "name": "pass-execution", "cat": "PERF", "ph": "E"
 
-//      CHECK: "name": "pass-execution", "cat": "PERF", "ph": "B"
-//      CHECK: "name": "apply-conversion", "cat": "PERF", "ph": "B"
-//      CHECK: "name": "apply-pattern", "cat": "PERF", "ph": "B"
-//      CHECK: "name": "apply-pattern", "cat": "PERF", "ph": "E"
 // Note: Listener notifications appear after the pattern application because
 // the conversion driver sends all notifications at the end of the conversion
 // in bulk.
@@ -11,8 +17,6 @@
 // CHECK-NEXT: notifyOperationReplaced: test.illegal_op_a
 // CHECK-NEXT: notifyOperationModified: func.return
 // CHECK-NEXT: notifyOperationErased: test.illegal_op_a
-//      CHECK: "name": "apply-conversion", "cat": "PERF", "ph": "E"
-//      CHECK: "name": "pass-execution", "cat": "PERF", "ph": "E"
 // CHECK-LABEL: verifyDirectPattern
 func.func @verifyDirectPattern() -> i32 {
   // CHECK-NEXT:  "test.legal_op_a"() <{status = "Success"}
@@ -29,7 +33,9 @@ func.func @verifyDirectPattern() -> i32 {
 // CHECK-NEXT: notifyOperationErased: test.illegal_op_c
 // CHECK-NEXT: notifyOperationInserted: test.legal_op_a, was unlinked
 // CHECK-NEXT: notifyOperationReplaced: test.illegal_op_e
-// CHECK-NEXT: notifyOperationErased: test.illegal_op_e
+// Note: func.return is modified a second time when running in no-rollback
+//       mode.
+//      CHECK: notifyOperationErased: test.illegal_op_e
 
 // CHECK-LABEL: verifyLargerBenefit
 func.func @verifyLargerBenefit() -> i32 {
@@ -70,7 +76,7 @@ func.func @remap_call_1_to_1(%arg0: i64) {
 // CHECK:      notifyBlockInserted into func.func: was unlinked
 
 // Contents of the old block are moved to the new block.
-// CHECK-NEXT: notifyOperationInserted: test.return, was linked, exact position unknown
+// CHECK-NEXT: notifyOperationInserted: test.return
 
 // The old block is erased.
 // CHECK-NEXT: notifyBlockErased
@@ -185,9 +191,12 @@ func.func @remap_drop_region() {
 // -----
 
 // CHECK-LABEL: func @dropped_input_in_use
+// CHECK-KIND-LABEL: func @dropped_input_in_use
 func.func @dropped_input_in_use(%arg: i16, %arg2: i64) {
-  // CHECK-NEXT: "test.cast"{{.*}} : () -> i16
-  // CHECK-NEXT: "work"{{.*}} : (i16)
+  // CHECK-NEXT: %[[cast:.*]] = "test.cast"() : () -> i16
+  // CHECK-NEXT: "work"(%[[cast]]) : (i16)
+  // CHECK-KIND-NEXT: %[[cast:.*]] = builtin.unrealized_conversion_cast to i16 {__kind__ = "source"}
+  // CHECK-KIND-NEXT: "work"(%[[cast]]) : (i16)
   // expected-remark@+1 {{op 'work' is not legalizable}}
   "work"(%arg) : (i16) -> ()
 }
@@ -409,9 +418,54 @@ func.func @test_remap_block_arg() {
 
 // CHECK-LABEL: func @test_multiple_1_to_n_replacement()
 //       CHECK:   %[[legal_op:.*]]:4 = "test.legal_op"() : () -> (f16, f16, f16, f16)
-//       CHECK:   %[[cast:.*]] = "test.cast"(%[[legal_op]]#0, %[[legal_op]]#1, %[[legal_op]]#2, %[[legal_op]]#3) : (f16, f16, f16, f16) -> f16
-//       CHECK:   "test.valid"(%[[cast]]) : (f16) -> ()
+// Note: There is a bug in the rollback-based conversion driver: it emits a
+// "test.cast" : (f16, f16, f16, f16) -> f16, when it should be emitting
+// three consecutive casts of (f16, f16) -> f16.
+//       CHECK:   "test.valid"(%{{.*}}) : (f16) -> ()
 func.func @test_multiple_1_to_n_replacement() {
   %0 = "test.multiple_1_to_n_replacement"() : () -> (f16)
   "test.invalid"(%0) : (f16) -> ()
 }
+
+// -----
+
+// CHECK-LABEL: func @test_lookup_without_converter
+//       CHECK:   %[[producer:.*]] = "test.valid_producer"() : () -> i16
+//       CHECK:   %[[cast:.*]] = "test.cast"(%[[producer]]) : (i16) -> f64
+//       CHECK:   "test.valid_consumer"(%[[cast]]) : (f64) -> ()
+//       CHECK:   "test.valid_consumer"(%[[producer]]) : (i16) -> ()
+// CHECK-KIND-LABEL: func @test_lookup_without_converter
+//       CHECK-KIND:   %[[producer:.*]] = "test.valid_producer"() : () -> i16
+//       CHECK-KIND:   %[[cast:.*]] = builtin.unrealized_conversion_cast %[[producer]] : i16 to f64 {__kind__ = "target"}
+//       CHECK-KIND:   "test.valid_consumer"(%[[cast]]) : (f64) -> ()
+//       CHECK-KIND:   "test.valid_consumer"(%[[producer]]) : (i16) -> ()
+func.func @test_lookup_without_converter() {
+  %0 = "test.replace_with_valid_producer"() {type = i16} : () -> (i64)
+  "test.replace_with_valid_consumer"(%0) {with_converter} : (i64) -> ()
+  // Make sure that the second "replace_with_valid_consumer" lowering does not
+  // lookup the materialization that was created for the above op.
+  "test.replace_with_valid_consumer"(%0) : (i64) -> ()
+  // expected-remark@+1 {{op 'func.return' is not legalizable}}
+  return
+}
+
+// -----
+// expected-remark@-1 {{applyPartialConversion failed}}
+
+func.func @test_skip_1to1_pattern(%arg0: f32) {
+  // expected-error@+1 {{failed to legalize operation 'test.type_consumer'}}
+  "test.type_consumer"(%arg0) : (f32) -> ()
+  return
+}
+
+// -----
+
+// Demonstrate that the pattern generally works, but only for 1:1 type
+// conversions.
+
+// CHECK-LABEL: @test_working_1to1_pattern(
+func.func @test_working_1to1_pattern(%arg0: f16) {
+  // CHECK-NEXT: "test.return"() : () -> ()
+  "test.type_consumer"(%arg0) : (f16) -> ()
+  "test.return"() : () -> ()
+}
diff --git a/mlir/test/lib/Analysis/DataFlow/TestLivenessAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestLivenessAnalysis.cpp
index 43005e22584c2..8e2f03b644e49 100644
--- a/mlir/test/lib/Analysis/DataFlow/TestLivenessAnalysis.cpp
+++ b/mlir/test/lib/Analysis/DataFlow/TestLivenessAnalysis.cpp
@@ -33,7 +33,6 @@ struct TestLivenessAnalysisPass
 
   void runOnOperation() override {
     auto &livenessAnalysis = getAnalysis<RunLivenessAnalysis>();
-
     Operation *op = getOperation();
 
     raw_ostream &os = llvm::outs();
diff --git a/mlir/test/lib/Dialect/Affine/TestLoopPermutation.cpp b/mlir/test/lib/Dialect/Affine/TestLoopPermutation.cpp
index e708b7de690ec..8bab9a0ef55b8 100644
--- a/mlir/test/lib/Dialect/Affine/TestLoopPermutation.cpp
+++ b/mlir/test/lib/Dialect/Affine/TestLoopPermutation.cpp
@@ -42,6 +42,12 @@ struct TestLoopPermutation
   ListOption<unsigned> permList{*this, "permutation-map",
                                 llvm::cl::desc("Specify the loop permutation"),
                                 llvm::cl::OneOrMore};
+
+  /// Specify whether to check validity of loop permutation.
+  Option<bool> checkValidity{
+      *this, "check-validity",
+      llvm::cl::desc("Check validity of the loop permutation"),
+      llvm::cl::init(false)};
 };
 
 } // namespace
@@ -60,6 +66,9 @@ void TestLoopPermutation::runOnOperation() {
     // Permute if the nest's size is consistent with the specified
     // permutation.
     if (nest.size() >= 2 && nest.size() == permMap.size()) {
+      if (checkValidity.getValue() &&
+          !isValidLoopInterchangePermutation(nest, permMap))
+        continue;
       permuteLoops(nest, permMap);
     }
   }
diff --git a/mlir/test/lib/Dialect/GPU/CMakeLists.txt b/mlir/test/lib/Dialect/GPU/CMakeLists.txt
index 418c884dc03b3..882d5abc2eeb8 100644
--- a/mlir/test/lib/Dialect/GPU/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/GPU/CMakeLists.txt
@@ -30,6 +30,7 @@ set(LIBS
   MLIRVectorDialect
   MLIRVectorToLLVMPass
   MLIRXeVMDialect
+  MLIRXeVMToLLVMIRTranslation
   )
 
 add_mlir_library(MLIRGPUTestPasses
diff --git a/mlir/test/lib/Dialect/LLVM/TestPatterns.cpp b/mlir/test/lib/Dialect/LLVM/TestPatterns.cpp
index ab02866970b1d..69a3d98bc09e4 100644
--- a/mlir/test/lib/Dialect/LLVM/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/LLVM/TestPatterns.cpp
@@ -6,7 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
+#include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
+#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
 #include "mlir/Conversion/LLVMCommon/TypeConverter.h"
+#include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/LLVMIR/LLVMDialect.h"
 #include "mlir/Dialect/LLVMIR/LLVMTypes.h"
 #include "mlir/Pass/Pass.h"
@@ -34,6 +38,10 @@ struct TestLLVMLegalizePatternsPass
     : public PassWrapper<TestLLVMLegalizePatternsPass, OperationPass<>> {
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestLLVMLegalizePatternsPass)
 
+  TestLLVMLegalizePatternsPass() = default;
+  TestLLVMLegalizePatternsPass(const TestLLVMLegalizePatternsPass &other)
+      : PassWrapper(other) {}
+
   StringRef getArgument() const final { return "test-llvm-legalize-patterns"; }
   StringRef getDescription() const final {
     return "Run LLVM dialect legalization patterns";
@@ -45,22 +53,48 @@ struct TestLLVMLegalizePatternsPass
 
   void runOnOperation() override {
     MLIRContext *ctx = &getContext();
+
+    // Set up type converter.
     LLVMTypeConverter converter(ctx);
+    converter.addConversion(
+        [&](IntegerType type, SmallVectorImpl<Type> &result) {
+          if (type.isInteger(17)) {
+            // Convert i17 -> (i18, i18).
+            result.append(2, Builder(ctx).getIntegerType(18));
+            return success();
+          }
+
+          result.push_back(type);
+          return success();
+        });
+
+    // Populate patterns.
     mlir::RewritePatternSet patterns(ctx);
     patterns.add<TestDirectReplacementOp>(ctx, converter);
+    arith::populateArithToLLVMConversionPatterns(converter, patterns);
+    populateFuncToLLVMConversionPatterns(converter, patterns);
+    cf::populateControlFlowToLLVMConversionPatterns(converter, patterns);
 
     // Define the conversion target used for the test.
     ConversionTarget target(*ctx);
     target.addLegalOp(OperationName("test.legal_op", ctx));
+    target.addLegalDialect<LLVM::LLVMDialect>();
+    target.addDynamicallyLegalOp<func::FuncOp>(
+        [&](func::FuncOp funcOp) { return funcOp->hasAttr("is_legal"); });
 
     // Handle a partial conversion.
     DenseSet<Operation *> unlegalizedOps;
     ConversionConfig config;
     config.unlegalizedOps = &unlegalizedOps;
+    config.allowPatternRollback = allowPatternRollback;
     if (failed(applyPartialConversion(getOperation(), target,
                                       std::move(patterns), config)))
       getOperation()->emitError() << "applyPartialConversion failed";
   }
+
+  Option<bool> allowPatternRollback{*this, "allow-pattern-rollback",
+                                    llvm::cl::desc("Allow pattern rollback"),
+                                    llvm::cl::init(true)};
 };
 } // namespace
 
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 2eaad552a7a3a..231400ec9cd29 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -1169,6 +1169,26 @@ def OpP : TEST_Op<"op_p"> {
   let results = (outs I32);
 }
 
+// Test constant-folding a pattern that maps `(F32) -> SI32`.
+def SignOp : TEST_Op<"sign", [SameOperandsAndResultShape]> {
+  let arguments = (ins RankedTensorOf<[F32]>:$operand);
+  let results = (outs RankedTensorOf<[SI32]>:$result);
+
+  let assemblyFormat = [{
+    $operand attr-dict `:` functional-type(operands, results)
+  }];
+}
+
+// Test constant-folding a pattern that maps `(F32, F32) -> I1`.
+def LessThanOp : TEST_Op<"less_than", [SameOperandsAndResultShape]> {
+  let arguments = (ins RankedTensorOf<[F32]>:$lhs, RankedTensorOf<[F32]>:$rhs);
+  let results = (outs RankedTensorOf<[I1]>:$result);
+
+  let assemblyFormat = [{
+    $lhs `,` $rhs attr-dict `:` functional-type(operands, results)
+  }];
+}
+
 // Test same operand name enforces equality condition check.
 def TestEqualArgsPattern : Pat<(OpN $a, $a), (OpO $a)>;
 
@@ -1478,6 +1498,8 @@ def TestOpInPlaceSelfFold : TEST_Op<"op_in_place_self_fold"> {
   let results = (outs I32);
   let hasFolder = 1;
 }
+def : Pat<(TestOpInPlaceSelfFold:$op $_),
+          (TestOpConstant ConstantAttr<I32Attr, "97">)>;
 
 // Test op that simply returns success.
 def TestOpInPlaceFoldSuccess : TEST_Op<"op_in_place_fold_success"> {
@@ -2104,6 +2126,10 @@ def TestInvalidOp : TEST_Op<"invalid", [Terminator]>,
   Arguments<(ins Variadic<AnyType>)>;
 def TestTypeProducerOp : TEST_Op<"type_producer">,
   Results<(outs AnyType)>;
+def TestValidProducerOp : TEST_Op<"valid_producer">,
+  Results<(outs AnyType)>;
+def TestValidConsumerOp : TEST_Op<"valid_consumer">,
+  Arguments<(ins AnyType)>;
 def TestAnotherTypeProducerOp : TEST_Op<"another_type_producer">,
   Results<(outs AnyType)>;
 def TestTypeConsumerOp : TEST_Op<"type_consumer">,
diff --git a/mlir/test/lib/Dialect/Test/TestPatterns.cpp b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
index eda618f5b09c6..b6f16ac1b5c48 100644
--- a/mlir/test/lib/Dialect/Test/TestPatterns.cpp
+++ b/mlir/test/lib/Dialect/Test/TestPatterns.cpp
@@ -10,6 +10,7 @@
 #include "TestOps.h"
 #include "TestTypes.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/CommonFolders.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/Func/Transforms/FuncConversions.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
@@ -202,6 +203,66 @@ struct HoistEligibleOps : public OpRewritePattern<test::OneRegionOp> {
   }
 };
 
+struct FoldSignOpF32ToSI32 : public OpRewritePattern<test::SignOp> {
+  using OpRewritePattern<test::SignOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(test::SignOp op,
+                                PatternRewriter &rewriter) const override {
+    if (op->getNumOperands() != 1 || op->getNumResults() != 1)
+      return failure();
+
+    TypedAttr operandAttr;
+    matchPattern(op->getOperand(0), m_Constant(&operandAttr));
+    if (!operandAttr)
+      return failure();
+
+    TypedAttr res = cast_or_null<TypedAttr>(
+        constFoldUnaryOp<FloatAttr, FloatAttr::ValueType, void, IntegerAttr>(
+            operandAttr, op.getType(), [](APFloat operand) -> APSInt {
+              static const APFloat zero(0.0f);
+              int operandSign = 0;
+              if (operand != zero)
+                operandSign = (operand < zero) ? -1 : +1;
+              return APSInt(APInt(32, operandSign), false);
+            }));
+    if (!res)
+      return failure();
+
+    rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, res);
+    return success();
+  }
+};
+
+struct FoldLessThanOpF32ToI1 : public OpRewritePattern<test::LessThanOp> {
+  using OpRewritePattern<test::LessThanOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(test::LessThanOp op,
+                                PatternRewriter &rewriter) const override {
+    if (op->getNumOperands() != 2 || op->getNumResults() != 1)
+      return failure();
+
+    TypedAttr lhsAttr;
+    TypedAttr rhsAttr;
+    matchPattern(op->getOperand(0), m_Constant(&lhsAttr));
+    matchPattern(op->getOperand(1), m_Constant(&rhsAttr));
+
+    if (!lhsAttr || !rhsAttr)
+      return failure();
+
+    Attribute operandAttrs[2] = {lhsAttr, rhsAttr};
+    TypedAttr res = cast_or_null<TypedAttr>(
+        constFoldBinaryOp<FloatAttr, FloatAttr::ValueType, void, IntegerAttr>(
+            operandAttrs, op.getType(), [](APFloat lhs, APFloat rhs) -> APInt {
+              return APInt(1, lhs < rhs);
+            }));
+    if (!res)
+      return failure();
+
+    rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, res);
+    return success();
+  }
+};
+
 /// This pattern moves "test.move_before_parent_op" before the parent op.
 struct MoveBeforeParentOp : public RewritePattern {
   MoveBeforeParentOp(MLIRContext *context)
@@ -1116,8 +1177,8 @@ struct TestNonRootReplacement : public RewritePattern {
     auto illegalOp = ILLegalOpF::create(rewriter, op->getLoc(), resultType);
     auto legalOp = LegalOpB::create(rewriter, op->getLoc(), resultType);
 
-    rewriter.replaceOp(illegalOp, legalOp);
     rewriter.replaceOp(op, illegalOp);
+    rewriter.replaceOp(illegalOp, legalOp);
     return success();
   }
 };
@@ -1198,6 +1259,47 @@ class TestEraseOp : public ConversionPattern {
   }
 };
 
+/// Pattern that replaces test.replace_with_valid_producer with
+/// test.valid_producer and the specified type.
+class TestReplaceWithValidProducer : public ConversionPattern {
+public:
+  TestReplaceWithValidProducer(MLIRContext *ctx)
+      : ConversionPattern("test.replace_with_valid_producer", 1, ctx) {}
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    auto attr = op->getAttrOfType<TypeAttr>("type");
+    if (!attr)
+      return failure();
+    rewriter.replaceOpWithNewOp<TestValidProducerOp>(op, attr.getValue());
+    return success();
+  }
+};
+
+/// Pattern that replaces test.replace_with_valid_consumer with
+/// test.valid_consumer. Can be used with and without a type converter.
+class TestReplaceWithValidConsumer : public ConversionPattern {
+public:
+  TestReplaceWithValidConsumer(MLIRContext *ctx, const TypeConverter &converter)
+      : ConversionPattern(converter, "test.replace_with_valid_consumer", 1,
+                          ctx) {}
+  TestReplaceWithValidConsumer(MLIRContext *ctx)
+      : ConversionPattern("test.replace_with_valid_consumer", 1, ctx) {}
+
+  LogicalResult
+  matchAndRewrite(Operation *op, ArrayRef<Value> operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    // with_converter present: pattern must have been initialized with a type
+    // converter.
+    // with_converter absent: pattern must have been initialized without a type
+    // converter.
+    if (op->hasAttr("with_converter") != static_cast<bool>(getTypeConverter()))
+      return failure();
+    rewriter.replaceOpWithNewOp<TestValidConsumerOp>(op, operands[0]);
+    return success();
+  }
+};
+
 /// This pattern matches a test.convert_block_args op. It either:
 /// a) Duplicates all block arguments,
 /// b) or: drops all block arguments and replaces each with 2x the first
@@ -1260,6 +1362,7 @@ class TestMultiple1ToNReplacement : public ConversionPattern {
     // Helper function that replaces the given op with a new op of the given
     // name and doubles each result (1 -> 2 replacement of each result).
     auto replaceWithDoubleResults = [&](Operation *op, StringRef name) {
+      rewriter.setInsertionPointAfter(op);
       SmallVector<Type> types;
       for (Type t : op->getResultTypes()) {
         types.push_back(t);
@@ -1283,6 +1386,23 @@ class TestMultiple1ToNReplacement : public ConversionPattern {
   }
 };
 
+/// Pattern that erases 'test.type_consumers' iff the input operand is the
+/// result of a 1:1 type conversion.
+/// Used to test correct skipping of 1:1 patterns in the 1:N case.
+class TestTypeConsumerOpPattern
+    : public OpConversionPattern<TestTypeConsumerOp> {
+public:
+  TestTypeConsumerOpPattern(MLIRContext *ctx, const TypeConverter &converter)
+      : OpConversionPattern<TestTypeConsumerOp>(converter, ctx) {}
+
+  LogicalResult
+  matchAndRewrite(TestTypeConsumerOp op, OpAdaptor operands,
+                  ConversionPatternRewriter &rewriter) const final {
+    rewriter.eraseOp(op);
+    return success();
+  }
+};
+
 /// Test unambiguous overload resolution of replaceOpWithMultiple. This
 /// function is just to trigger compiler errors. It is never executed.
 [[maybe_unused]] void testReplaceOpWithMultipleOverloads(
@@ -1314,6 +1434,7 @@ struct TestTypeConverter : public TypeConverter {
   TestTypeConverter() {
     addConversion(convertType);
     addSourceMaterialization(materializeCast);
+    addTargetMaterialization(materializeCast);
   }
 
   static LogicalResult convertType(Type t, SmallVectorImpl<Type> &results) {
@@ -1389,10 +1510,12 @@ struct TestLegalizePatternDriver
         TestBoundedRecursiveRewrite, TestNestedOpCreationUndoRewrite,
         TestReplaceEraseOp, TestCreateUnregisteredOp, TestUndoMoveOpBefore,
         TestUndoPropertiesModification, TestEraseOp,
+        TestReplaceWithValidProducer, TestReplaceWithValidConsumer,
         TestRepetitive1ToNConsumer>(&getContext());
     patterns.add<TestDropOpSignatureConversion, TestDropAndReplaceInvalidOp,
                  TestPassthroughInvalidOp, TestMultiple1ToNReplacement,
-                 TestBlockArgReplace>(&getContext(), converter);
+                 TestBlockArgReplace, TestReplaceWithValidConsumer,
+                 TestTypeConsumerOpPattern>(&getContext(), converter);
     patterns.add<TestConvertBlockArgs>(converter, &getContext());
     mlir::populateAnyFunctionOpInterfaceTypeConversionPattern(patterns,
                                                               converter);
@@ -1402,7 +1525,8 @@ struct TestLegalizePatternDriver
     ConversionTarget target(getContext());
     target.addLegalOp<ModuleOp>();
     target.addLegalOp<LegalOpA, LegalOpB, LegalOpC, TestCastOp, TestValidOp,
-                      TerminatorOp, OneRegionOp>();
+                      TerminatorOp, TestOpConstant, OneRegionOp,
+                      TestValidProducerOp, TestValidConsumerOp>();
     target.addLegalOp(OperationName("test.legal_op", &getContext()));
     target
         .addIllegalOp<ILLegalOpF, TestRegionBuilderOp, TestOpWithRegionFold>();
@@ -1450,12 +1574,18 @@ struct TestLegalizePatternDriver
     target.addDynamicallyLegalOp<ConvertBlockArgsOp>(
         [](ConvertBlockArgsOp op) { return op.getIsLegal(); });
 
+    // Set up configuration.
+    ConversionConfig config;
+    config.allowPatternRollback = allowPatternRollback;
+    config.foldingMode = foldingMode;
+    config.buildMaterializations = buildMaterializations;
+    config.attachDebugMaterializationKind = attachDebugMaterializationKind;
+    DumpNotifications dumpNotifications;
+    config.listener = &dumpNotifications;
+
     // Handle a partial conversion.
     if (mode == ConversionMode::Partial) {
       DenseSet<Operation *> unlegalizedOps;
-      ConversionConfig config;
-      DumpNotifications dumpNotifications;
-      config.listener = &dumpNotifications;
       config.unlegalizedOps = &unlegalizedOps;
       if (failed(applyPartialConversion(getOperation(), target,
                                         std::move(patterns), config))) {
@@ -1474,9 +1604,6 @@ struct TestLegalizePatternDriver
         return (bool)op->getAttrOfType<UnitAttr>("test.dynamically_legal");
       });
 
-      ConversionConfig config;
-      DumpNotifications dumpNotifications;
-      config.listener = &dumpNotifications;
       if (failed(applyFullConversion(getOperation(), target,
                                      std::move(patterns), config))) {
         getOperation()->emitRemark() << "applyFullConversion failed";
@@ -1489,7 +1616,6 @@ struct TestLegalizePatternDriver
 
     // Analyze the convertible operations.
     DenseSet<Operation *> legalizedOps;
-    ConversionConfig config;
     config.legalizableOps = &legalizedOps;
     if (failed(applyAnalysisConversion(getOperation(), target,
                                        std::move(patterns), config)))
@@ -1510,6 +1636,34 @@ struct TestLegalizePatternDriver
           clEnumValN(ConversionMode::Full, "full", "Perform a full conversion"),
           clEnumValN(ConversionMode::Partial, "partial",
                      "Perform a partial conversion"))};
+
+  Option<DialectConversionFoldingMode> foldingMode{
+      *this, "test-legalize-folding-mode",
+      llvm::cl::desc("The folding mode to use with the test driver"),
+      llvm::cl::init(DialectConversionFoldingMode::BeforePatterns),
+      llvm::cl::values(clEnumValN(DialectConversionFoldingMode::Never, "never",
+                                  "Never attempt to fold"),
+                       clEnumValN(DialectConversionFoldingMode::BeforePatterns,
+                                  "before-patterns",
+                                  "Only attempt to fold not legal operations "
+                                  "before applying patterns"),
+                       clEnumValN(DialectConversionFoldingMode::AfterPatterns,
+                                  "after-patterns",
+                                  "Only attempt to fold not legal operations "
+                                  "after applying patterns"))};
+  Option<bool> allowPatternRollback{*this, "allow-pattern-rollback",
+                                    llvm::cl::desc("Allow pattern rollback"),
+                                    llvm::cl::init(true)};
+  Option<bool> attachDebugMaterializationKind{
+      *this, "attach-debug-materialization-kind",
+      llvm::cl::desc(
+          "Attach materialization kind to unrealized_conversion_cast ops"),
+      llvm::cl::init(false)};
+  Option<bool> buildMaterializations{
+      *this, "build-materializations",
+      llvm::cl::desc(
+          "If set to 'false', leave unrealized_conversion_cast ops in place"),
+      llvm::cl::init(true)};
 };
 } // namespace
 
@@ -2181,6 +2335,24 @@ struct TestSelectiveReplacementPatternDriver
     (void)applyPatternsGreedily(getOperation(), std::move(patterns));
   }
 };
+
+struct TestFoldTypeConvertingOp
+    : public PassWrapper<TestFoldTypeConvertingOp, OperationPass<>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestFoldTypeConvertingOp)
+
+  StringRef getArgument() const final { return "test-fold-type-converting-op"; }
+  StringRef getDescription() const final {
+    return "Test helper functions for folding ops whose input and output types "
+           "differ, e.g. float comparisons of the form `(f32, f32) -> i1`.";
+  }
+  void runOnOperation() override {
+    MLIRContext *context = &getContext();
+    mlir::RewritePatternSet patterns(context);
+    patterns.add<FoldSignOpF32ToSI32, FoldLessThanOpF32ToI1>(context);
+    if (failed(applyPatternsGreedily(getOperation(), std::move(patterns))))
+      signalPassFailure();
+  }
+};
 } // namespace
 
 //===----------------------------------------------------------------------===//
@@ -2211,6 +2383,8 @@ void registerPatternsTestPass() {
 
   PassRegistration<TestMergeBlocksPatternDriver>();
   PassRegistration<TestSelectiveReplacementPatternDriver>();
+
+  PassRegistration<TestFoldTypeConvertingOp>();
 }
 } // namespace test
 } // namespace mlir
diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
index f89c944b5c564..bb1598ee3efe5 100644
--- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
@@ -786,6 +786,28 @@ struct TestVectorGatherLowering
   }
 };
 
+struct TestUnrollVectorFromElements
+    : public PassWrapper<TestUnrollVectorFromElements,
+                         OperationPass<func::FuncOp>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestUnrollVectorFromElements)
+
+  StringRef getArgument() const final {
+    return "test-unroll-vector-from-elements";
+  }
+  StringRef getDescription() const final {
+    return "Test unrolling patterns for from_elements ops";
+  }
+  void getDependentDialects(DialectRegistry &registry) const override {
+    registry.insert<func::FuncDialect, vector::VectorDialect, ub::UBDialect>();
+  }
+
+  void runOnOperation() override {
+    RewritePatternSet patterns(&getContext());
+    populateVectorFromElementsLoweringPatterns(patterns);
+    (void)applyPatternsGreedily(getOperation(), std::move(patterns));
+  }
+};
+
 struct TestFoldArithExtensionIntoVectorContractPatterns
     : public PassWrapper<TestFoldArithExtensionIntoVectorContractPatterns,
                          OperationPass<func::FuncOp>> {
@@ -1059,6 +1081,8 @@ void registerTestVectorLowerings() {
 
   PassRegistration<TestVectorGatherLowering>();
 
+  PassRegistration<TestUnrollVectorFromElements>();
+
   PassRegistration<TestFoldArithExtensionIntoVectorContractPatterns>();
 
   PassRegistration<TestVectorEmulateMaskedLoadStore>();
diff --git a/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp b/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
index c6245b637c2a7..200323c7a4e51 100644
--- a/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
+++ b/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
@@ -7,11 +7,14 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/Index/IR/IndexDialect.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/Dialect/XeGPU/Transforms/Transforms.h"
+#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Pass/PassManager.h"
+#include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
 using namespace mlir;
@@ -79,7 +82,7 @@ struct TestXeGPUUnrollingPatterns
 
             if (auto layout = tdescTy.getLayoutAttr()) {
               auto inst_data = layout.getInstData();
-              if (inst_data && layout.isSgLayout())
+              if (inst_data && layout.isForSubgroup())
                 return SmallVector<int64_t>(inst_data.asArrayRef().begin(),
                                             inst_data.asArrayRef().end());
             }
@@ -147,12 +150,118 @@ struct TestXeGPUUnrollingPatterns
   }
 };
 
+#undef DEBUG_TYPE
+#define DEBUG_TYPE "test-xegpu-layout-interface"
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
+#define LDBG(X) LLVM_DEBUG(DBGS() << X << "\n")
+
+// Test pattern for distributing vector::StepOp from workgroup to subgroup.
+// Validates DistributeLayoutAttr interfaces for offset computation
+// abstraction between LayoutAttr and SliceAttr.
+class TestStepOpPattern : public OpConversionPattern<vector::StepOp> {
+  using OpConversionPattern<vector::StepOp>::OpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(vector::StepOp op, OneToNOpAdaptor adaptor,
+                  ConversionPatternRewriter &rewriter) const override {
+
+    auto layoutName = xegpu::getLayoutName(op->getResult(0));
+    auto sliceAttr = op->getAttrOfType<xegpu::SliceAttr>(layoutName);
+    if (!sliceAttr || sliceAttr.getRank() != 1)
+      return failure();
+
+    std::optional<SmallVector<int64_t>> sgShape = sliceAttr.getSgDataAsInt();
+    if (!sgShape)
+      return failure();
+
+    Location loc = op.getLoc();
+    VectorType type = op.getResult().getType();
+    auto wgShape = type.getShape();
+
+    Value sgId =
+        gpu::SubgroupIdOp::create(rewriter, loc, /*upper_bound=*/nullptr);
+    auto maybeOffsets = sliceAttr.getOffsets(rewriter, loc, sgId, wgShape);
+    if (failed(maybeOffsets))
+      return failure();
+
+    VectorType newTy = type.cloneWith(*sgShape, type.getElementType());
+    Value base = vector::StepOp::create(rewriter, loc, newTy);
+    SmallVector<Value> newOps;
+    for (auto offsets : *maybeOffsets) {
+      Value bcast =
+          vector::BroadcastOp::create(rewriter, loc, newTy, offsets[0]);
+      Value add = arith::AddIOp::create(rewriter, loc, base, bcast);
+      newOps.push_back(add);
+    }
+    rewriter.replaceOpWithMultiple(op, {newOps});
+    return success();
+  }
+};
+
+struct TestXeGPULayoutInterface
+    : public PassWrapper<TestXeGPULayoutInterface,
+                         OperationPass<gpu::GPUModuleOp>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestXeGPULayoutInterface)
+
+  StringRef getArgument() const final { return "test-xegpu-layout-interface"; }
+
+  StringRef getDescription() const final {
+    return "Test the implementation of XeGPU Layout interfaces";
+  }
+
+  void getDependentDialects(::mlir::DialectRegistry &registry) const override {
+    registry.insert<arith::ArithDialect>();
+    registry.insert<memref::MemRefDialect>();
+    registry.insert<xegpu::XeGPUDialect>();
+    registry.insert<vector::VectorDialect>();
+    registry.insert<index::IndexDialect>();
+  }
+
+  TestXeGPULayoutInterface() = default;
+  TestXeGPULayoutInterface(const TestXeGPULayoutInterface &pass)
+      : PassWrapper(pass) {}
+
+  void runOnOperation() override {
+    MLIRContext *ctx = &getContext();
+
+    TypeConverter typeConverter;
+    auto materializeCast = [&](mlir::OpBuilder &builder, mlir::Type type,
+                               mlir::ValueRange inputs,
+                               mlir::Location loc) -> mlir::Value {
+      return UnrealizedConversionCastOp::create(builder, loc, type, inputs)
+          .getResult(0);
+    };
+    typeConverter.addSourceMaterialization(materializeCast);
+    typeConverter.addTargetMaterialization(materializeCast);
+
+    RewritePatternSet patterns(ctx);
+    patterns.add<TestStepOpPattern>(typeConverter, ctx);
+
+    ConversionTarget target(*ctx);
+    auto isLegal = [&](xegpu::SliceAttr layout) -> bool {
+      return !layout || !layout.isForWorkgroup();
+    };
+
+    target.addDynamicallyLegalOp<vector::StepOp>(
+        [&](vector::StepOp op) -> bool {
+          auto layoutName = xegpu::getLayoutName(op->getResult(0));
+          auto sliceAttr = op->getAttrOfType<xegpu::SliceAttr>(layoutName);
+          return isLegal(sliceAttr);
+        });
+
+    target.markUnknownOpDynamicallyLegal([](Operation *op) { return true; });
+
+    (void)applyPartialConversion(getOperation(), target, std::move(patterns));
+  }
+};
+
 } // namespace
 
 namespace mlir {
 namespace test {
 void registerTestXeGPULowerings() {
   PassRegistration<TestXeGPUUnrollingPatterns>();
+  PassRegistration<TestXeGPULayoutInterface>();
 }
 } // namespace test
 } // namespace mlir
diff --git a/mlir/test/lib/Pass/TestPassManager.cpp b/mlir/test/lib/Pass/TestPassManager.cpp
index 25c8e53179803..df2736bd666ca 100644
--- a/mlir/test/lib/Pass/TestPassManager.cpp
+++ b/mlir/test/lib/Pass/TestPassManager.cpp
@@ -133,6 +133,51 @@ struct TestOptionsSuperPass
       llvm::cl::desc("Example list of PassPipelineOptions option")};
 };
 
+struct TestOptionsPassA
+    : public PassWrapper<TestOptionsPassA, OperationPass<func::FuncOp>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestOptionsPassA)
+
+  struct Options : public PassPipelineOptions<Options> {
+    Option<bool> foo{*this, "foo", llvm::cl::desc("Example boolean option")};
+  };
+
+  TestOptionsPassA() = default;
+  TestOptionsPassA(const TestOptionsPassA &) : PassWrapper() {}
+  TestOptionsPassA(const Options &options) { this->options.foo = options.foo; }
+
+  void runOnOperation() final {}
+  StringRef getArgument() const final { return "test-options-pass-a"; }
+  StringRef getDescription() const final {
+    return "Test superset options parsing capabilities - subset A";
+  }
+
+  Options options;
+};
+
+struct TestOptionsPassB
+    : public PassWrapper<TestOptionsPassB, OperationPass<func::FuncOp>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestOptionsPassB)
+
+  struct Options : public PassPipelineOptions<Options> {
+    Option<bool> bar{*this, "bar", llvm::cl::desc("Example boolean option")};
+  };
+
+  TestOptionsPassB() = default;
+  TestOptionsPassB(const TestOptionsPassB &) : PassWrapper() {}
+  TestOptionsPassB(const Options &options) { this->options.bar = options.bar; }
+
+  void runOnOperation() final {}
+  StringRef getArgument() const final { return "test-options-pass-b"; }
+  StringRef getDescription() const final {
+    return "Test superset options parsing capabilities - subset B";
+  }
+
+  Options options;
+};
+
+struct TestPipelineOptionsSuperSetAB : TestOptionsPassA::Options,
+                                       TestOptionsPassB::Options {};
+
 /// A test pass that always aborts to enable testing the crash recovery
 /// mechanism of the pass manager.
 struct TestCrashRecoveryPass
@@ -270,6 +315,9 @@ void registerPassManagerTestPass() {
   PassRegistration<TestOptionsPass>();
   PassRegistration<TestOptionsSuperPass>();
 
+  PassRegistration<TestOptionsPassA>();
+  PassRegistration<TestOptionsPassB>();
+
   PassRegistration<TestModulePass>();
 
   PassRegistration<TestFunctionPass>();
@@ -306,5 +354,16 @@ void registerPassManagerTestPass() {
           [](OpPassManager &pm, const TestOptionsSuperPass::Options &options) {
             pm.addPass(std::make_unique<TestOptionsSuperPass>(options));
           });
+
+  PassPipelineRegistration<TestPipelineOptionsSuperSetAB>
+      registerPipelineOptionsSuperSetABPipeline(
+          "test-options-super-set-ab-pipeline",
+          "Parses options of PassPipelineOptions using pass pipeline "
+          "registration",
+          [](OpPassManager &pm, const TestPipelineOptionsSuperSetAB &options) {
+            // Pass superset AB options to subset options A and B
+            pm.addPass(std::make_unique<TestOptionsPassA>(options));
+            pm.addPass(std::make_unique<TestOptionsPassB>(options));
+          });
 }
 } // namespace mlir
diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py
index feaf5fb852a1d..f99c24d6e299a 100644
--- a/mlir/test/lit.cfg.py
+++ b/mlir/test/lit.cfg.py
@@ -29,6 +29,7 @@
 if lit_shell_env:
     use_lit_shell = lit.util.pythonize_bool(lit_shell_env)
 
+# Set the test format based on shell configuration
 config.test_format = lit.formats.ShTest(execute_external=not use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files.
@@ -181,15 +182,16 @@ def find_real_python_interpreter():
 ]
 
 # Tweak the PATH to include the tools dir.
-llvm_config.with_environment("PATH", config.mlir_tools_dir, append_path=True)
-llvm_config.with_environment("PATH", config.llvm_tools_dir, append_path=True)
+tool_dirs = [config.mlir_tools_dir, config.llvm_tools_dir, config.lit_tools_dir]
+for dirs in tool_dirs:
+    llvm_config.with_environment("PATH", dirs, append_path=True)
 
-tool_dirs = [config.mlir_tools_dir, config.llvm_tools_dir]
 tools = [
     "mlir-tblgen",
     "mlir-translate",
     "mlir-lsp-server",
     "mlir-capi-execution-engine-test",
+    "mlir-capi-global-constructors-test",
     "mlir-capi-ir-test",
     "mlir-capi-irdl-test",
     "mlir-capi-llvm-test",
@@ -224,6 +226,9 @@ def find_real_python_interpreter():
 if config.enable_sycl_runner:
     tools.extend([add_runtime("mlir_sycl_runtime")])
 
+if config.enable_levelzero_runner:
+    tools.extend([add_runtime("mlir_levelzero_runtime")])
+
 if config.enable_spirv_cpu_runner:
     tools.extend([add_runtime("mlir_spirv_cpu_runtime")])
 
@@ -375,3 +380,6 @@ def have_host_jit_feature_support(feature_name):
 
 if config.arm_emulator_executable:
     config.available_features.add("arm-emulator")
+
+if sys.version_info >= (3, 11):
+    config.available_features.add("python-ge-311")
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index b1185e19d86e8..8a742a227847b 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -5,6 +5,7 @@ import sys
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_DIR@")
+config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
 config.spirv_tools_tests = @LLVM_INCLUDE_SPIRV_TOOLS_TESTS@
 config.llvm_shlib_ext = "@SHLIBEXT@"
 config.llvm_shlib_dir = lit_config.substitute(path(r"@SHLIBDIR@"))
@@ -33,7 +34,9 @@ config.run_rocm_tests = @MLIR_ENABLE_ROCM_CONVERSIONS@
 config.enable_rocm_runner = @MLIR_ENABLE_ROCM_RUNNER@
 config.gpu_compilation_format = "@MLIR_GPU_COMPILATION_TEST_FORMAT@"
 config.rocm_test_chipset = "@ROCM_TEST_CHIPSET@"
+config.run_xevm_tests = @MLIR_ENABLE_XEVM_CONVERSIONS@
 config.enable_sycl_runner = @MLIR_ENABLE_SYCL_RUNNER@
+config.enable_levelzero_runner = @MLIR_ENABLE_LEVELZERO_RUNNER@
 config.enable_spirv_cpu_runner = @MLIR_ENABLE_SPIRV_CPU_RUNNER@
 config.enable_vulkan_runner = @MLIR_ENABLE_VULKAN_RUNNER@
 config.enable_bindings_python = @MLIR_ENABLE_BINDINGS_PYTHON@
diff --git a/mlir/test/mlir-tblgen/enums-python-bindings.td b/mlir/test/mlir-tblgen/enums-python-bindings.td
index 1c5567f54a5f4..cd23b6a2effb9 100644
--- a/mlir/test/mlir-tblgen/enums-python-bindings.td
+++ b/mlir/test/mlir-tblgen/enums-python-bindings.td
@@ -62,12 +62,15 @@ def MyEnum64 : I64EnumAttr<"MyEnum64", "An example 64-bit enum", [One64, Two64]>
 // CHECK: def _myenum64(x, context):
 // CHECK:     return _ods_ir.IntegerAttr.get(_ods_ir.IntegerType.get_signless(64, context=context), int(x))
 
+def User : I32BitEnumAttrCaseBit<"User", 0, "user">;
+def Group : I32BitEnumAttrCaseBit<"Group", 1, "group">;
+def Other : I32BitEnumAttrCaseBit<"Other", 2, "other">;
+
 def TestBitEnum
-    : I32BitEnumAttr<"TestBitEnum", "", [
-        I32BitEnumAttrCaseBit<"User", 0, "user">,
-        I32BitEnumAttrCaseBit<"Group", 1, "group">,
-        I32BitEnumAttrCaseBit<"Other", 2, "other">,
-      ]> {
+    : I32BitEnumAttr<
+          "TestBitEnum", "",
+          [User, Group, Other,
+           I32BitEnumAttrCaseGroup<"Any", [User, Group, Other], "any">]> {
   let genSpecializedAttr = 0;
   let separator = " | ";
 }
@@ -79,9 +82,10 @@ def TestBitEnum_Attr : EnumAttr<Test_Dialect, TestBitEnum, "testbitenum">;
 // CHECK:     User = 1
 // CHECK:     Group = 2
 // CHECK:     Other = 4
+// CHECK:     Any = 7
 
 // CHECK:     def __iter__(self):
-// CHECK:         return iter([case for case in type(self) if (self & case) is case])
+// CHECK:         return iter([case for case in type(self) if (self & case) is case and self is not case])
 // CHECK:     def __len__(self):
 // CHECK:         return bin(self).count("1")
 
@@ -94,6 +98,8 @@ def TestBitEnum_Attr : EnumAttr<Test_Dialect, TestBitEnum, "testbitenum">;
 // CHECK:             return "group"
 // CHECK:         if self is TestBitEnum.Other:
 // CHECK:             return "other"
+// CHECK:         if self is TestBitEnum.Any:
+// CHECK:             return "any"
 // CHECK:         raise ValueError("Unknown TestBitEnum enum entry.")
 
 // CHECK: @register_attribute_builder("TestBitEnum")
diff --git a/mlir/test/mlir-translate/emitc_classops.mlir b/mlir/test/mlir-translate/emitc_classops.mlir
deleted file mode 100644
index d880f9b16dfc6..0000000000000
--- a/mlir/test/mlir-translate/emitc_classops.mlir
+++ /dev/null
@@ -1,78 +0,0 @@
-// RUN: mlir-translate --mlir-to-cpp %s | FileCheck %s
-
-emitc.class @modelClass {
-    emitc.field @fieldName0 : !emitc.array<1xf32> 
-    emitc.field @fieldName1 : !emitc.array<1xf32> 
-    emitc.func @execute() {
-        %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
-        %1 = get_field @fieldName0 : !emitc.array<1xf32>
-        %2 = get_field @fieldName1 : !emitc.array<1xf32>
-        %3 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
-        return
-    }
-}
-
-// CHECK-LABEL: class modelClass {
-// CHECK-NEXT: public:
-// CHECK-NEXT:  float fieldName0[1];
-// CHECK-NEXT:  float fieldName1[1];
-// CHECK-NEXT:  void execute() {
-// CHECK-NEXT:    size_t v1 = 0;
-// CHECK-NEXT:    return;
-// CHECK-NEXT:  }
-// CHECK-NEXT: };
-
-emitc.class final @finalClass {
-    emitc.field @fieldName0 : !emitc.array<1xf32> 
-    emitc.field @fieldName1 : !emitc.array<1xf32> 
-    emitc.func @execute() {
-        %0 = "emitc.constant"() <{value = 0 : index}> : () -> !emitc.size_t
-        %1 = get_field @fieldName0 : !emitc.array<1xf32>
-        %2 = get_field @fieldName1 : !emitc.array<1xf32>
-        %3 = subscript %1[%0] : (!emitc.array<1xf32>, !emitc.size_t) -> !emitc.lvalue<f32>
-        return
-    }
-}
-
-// CHECK-LABEL: class finalClass final {
-// CHECK-NEXT: public:
-// CHECK-NEXT:  float fieldName0[1];
-// CHECK-NEXT:  float fieldName1[1];
-// CHECK-NEXT:  void execute() {
-// CHECK-NEXT:    size_t v1 = 0;
-// CHECK-NEXT:    return;
-// CHECK-NEXT:  }
-// CHECK-NEXT: };
-
-emitc.class @mainClass {
-  emitc.field @fieldName0 : !emitc.array<2xf32> = dense<0.0> {attrs = {emitc.name_hint = "another_feature"}}
-  emitc.func @get_fieldName0() {
-    %0 = emitc.get_field @fieldName0 : !emitc.array<2xf32>
-    return 
-  }
-}
-
-// CHECK-LABEL: class mainClass {
-// CHECK-NEXT: public:
-// CHECK-NEXT:  float fieldName0[2] = {0.0e+00f, 0.0e+00f};
-// CHECK-NEXT:  void get_fieldName0() {
-// CHECK-NEXT:    return;
-// CHECK-NEXT:  }
-// CHECK-NEXT: };
-
-emitc.class @reflectionClass {
-  emitc.field @reflectionMap : !emitc.opaque<"const std::map<std::string, std::string>"> = #emitc.opaque<"{ { \22another_feature\22, \22fieldName0\22 } }"> 
-  emitc.func @get_reflectionMap() {
-    %0 = emitc.get_field @reflectionMap : !emitc.opaque<"const std::map<std::string, std::string>">
-    return 
-  }
-}
-
-// CHECK-LABEL: class reflectionClass {
-// CHECK-NEXT: public:
-// CHECK-NEXT:  const std::map<std::string, std::string> reflectionMap = { { "another_feature", "fieldName0" } };
-// CHECK-NEXT:  void get_reflectionMap() {
-// CHECK-NEXT:    return;
-// CHECK-NEXT:  }
-// CHECK-NEXT: };
-
diff --git a/mlir/test/python/dialects/linalg/opdsl/test_core_named_ops.py b/mlir/test/python/dialects/linalg/opdsl/test_core_named_ops.py
index ee76b6d25cae1..bc273bfa06f86 100644
--- a/mlir/test/python/dialects/linalg/opdsl/test_core_named_ops.py
+++ b/mlir/test/python/dialects/linalg/opdsl/test_core_named_ops.py
@@ -1,7 +1,7 @@
 # RUN: %PYTHON -m mlir.dialects.linalg.opdsl.dump_oplib .ops.core_named_ops | FileCheck %s
 
 # Just verify that at least one known op is generated.
-# CHECK: name: matmul
+# CHECK: name: copy
 
 # verify some special cases: negf->NegFOp, powf->PowFOp
 # CHECK cpp_class_name: NegFOp
diff --git a/mlir/test/python/dialects/nvvm.py b/mlir/test/python/dialects/nvvm.py
index 0eef97d95479a..3eb62bef50de9 100644
--- a/mlir/test/python/dialects/nvvm.py
+++ b/mlir/test/python/dialects/nvvm.py
@@ -5,6 +5,8 @@
 from mlir.dialects import nvvm
 from mlir.dialects import llvm
 from mlir.dialects import func
+import mlir.extras.types as T
+from mlir.dialects import arith
 
 
 def constructAndPrintInModule(f):
@@ -25,6 +27,7 @@ def testSmoke():
         "!llvm.struct<(f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32, f32)>"
     )
     shape_attr = Attribute.parse("#nvvm.shape<m = 64, n = 32, k = 16>")
+
     # CHECK-LABEL: func @wgmma_f32_f16_f16(%arg0: i64, %arg1: i64)
     @func.FuncOp.from_py_func(i64, i64)
     def wgmma_f32_f16_f16(desc_a, desc_b):
@@ -48,3 +51,41 @@ def wgmma_f32_f16_f16(desc_a, desc_b):
             layoutA=nvvm.MMALayout.col,
             layoutB=nvvm.MMALayout.col,
         )
+
+
+# CHECK-LABEL: TEST: test_inline_ptx
+# CHECK-LABEL: func.func @my_inline_ptx(
+# CHECK-SAME: %[[arg0:[a-zA-Z0-9_]+]]: f32, %[[arg1:[a-zA-Z0-9_]+]]: f32, %[[arg2:[a-zA-Z0-9_]+]]: i32, %[[arg3:[a-zA-Z0-9_]+]]: i32)
+# CHECK: %[[S0:.+]]:2 = nvvm.inline_ptx
+# CHECK-SAME: ro(%[[arg0]], %[[arg1]] : f32, f32) rw(%[[arg2]], %[[arg3]] : i32, i32) -> f32, f32
+# CHECK: %[[S1:.+]] = arith.addf %[[arg0]], %[[arg1]] : f32
+# CHECK: %[[S2:.+]] = arith.addi %[[arg2]], %[[arg3]] : i32
+# CHECK: %[[S3:.+]] = arith.addf %[[S0]]#0, %[[S0]]#1 : f32
+
+
+@constructAndPrintInModule
+def test_inline_ptx():
+    i32 = T.i32()
+    f32 = T.f32()
+
+    @func.FuncOp.from_py_func(f32, f32, i32, i32)
+    def my_inline_ptx(a, b, c, d):
+        ptx = r"""
+            {
+                .reg .pred p;
+                setp.ge.s32   p,      {$r0}, {$r1};
+                selp.s32      {$r0},  {$r0}, {$r1}, p;
+                selp.s32      {$r1},  {$r0}, {$r1}, p;
+                selp.s32      {$rw0}, {$r0}, {$r1}, p;
+                selp.s32      {$rw1}, {$r0}, {$r1}, p;
+            }
+            """
+        wo0, wo1 = nvvm.inline_ptx(
+            read_only_args=[a, b],
+            read_write_args=[c, d],
+            write_only_args=[f32, f32],
+            ptx_code=ptx,
+        )
+        arith.addf(a, b)
+        arith.addi(c, d)
+        arith.addf(wo0, wo1)
diff --git a/mlir/test/python/dialects/transform_vector_ext.py b/mlir/test/python/dialects/transform_vector_ext.py
index a51f2154d1f7d..5a648fe073315 100644
--- a/mlir/test/python/dialects/transform_vector_ext.py
+++ b/mlir/test/python/dialects/transform_vector_ext.py
@@ -46,6 +46,8 @@ def non_configurable_patterns():
     vector.ApplyLowerOuterProductPatternsOp()
     # CHECK: transform.apply_patterns.vector.lower_gather
     vector.ApplyLowerGatherPatternsOp()
+    # CHECK: transform.apply_patterns.vector.unroll_from_elements
+    vector.ApplyUnrollFromElementsPatternsOp()
     # CHECK: transform.apply_patterns.vector.lower_scan
     vector.ApplyLowerScanPatternsOp()
     # CHECK: transform.apply_patterns.vector.lower_shape_cast
diff --git a/mlir/test/python/global_constructors.py b/mlir/test/python/global_constructors.py
new file mode 100644
index 0000000000000..5020c00344a33
--- /dev/null
+++ b/mlir/test/python/global_constructors.py
@@ -0,0 +1,72 @@
+# UNSUPPORTED: target=aarch64{{.*}}, target=arm64{{.*}}
+# RUN: %PYTHON %s 2>&1 | FileCheck %s
+# REQUIRES: host-supports-jit
+import gc, sys, os, tempfile
+from mlir.ir import *
+from mlir.passmanager import *
+from mlir.execution_engine import *
+from mlir.runtime import *
+
+
+# Log everything to stderr and flush so that we have a unified stream to match
+# errors/info emitted by MLIR to stderr.
+def log(*args):
+    print(*args, file=sys.stderr)
+    sys.stderr.flush()
+
+
+def run(f):
+    log("\nTEST:", f.__name__)
+    f()
+    gc.collect()
+    assert Context._get_live_count() == 0
+
+
+def lowerToLLVM(module):
+    pm = PassManager.parse(
+        "builtin.module(convert-func-to-llvm,reconcile-unrealized-casts)"
+    )
+    pm.run(module.operation)
+    return module
+
+
+# Test JIT callback in global constructor
+# CHECK-LABEL: TEST: testJITCallbackInGlobalCtor
+def testJITCallbackInGlobalCtor():
+    init_cnt = 0
+
+    @ctypes.CFUNCTYPE(None)
+    def initCallback():
+        nonlocal init_cnt
+        init_cnt += 1
+
+    with Context():
+        module = Module.parse(
+            r"""
+llvm.mlir.global_ctors ctors = [@ctor], priorities = [0 : i32], data = [#llvm.zero]
+llvm.func @ctor() {
+  func.call @init_callback() : () -> ()
+  llvm.return
+}
+func.func private @init_callback() attributes { llvm.emit_c_interface }
+        """
+        )
+
+        # Setup execution engine
+        execution_engine = ExecutionEngine(lowerToLLVM(module))
+
+        # Validate initialization hasn't run yet
+        assert init_cnt == 0
+
+        # # Register callback
+        execution_engine.register_runtime("init_callback", initCallback)
+
+        # # Initialize and verify
+        execution_engine.initialize()
+        assert init_cnt == 1
+        # # Second initialization should be no-op
+        execution_engine.initialize()
+        assert init_cnt == 1
+
+
+run(testJITCallbackInGlobalCtor)
diff --git a/mlir/test/python/ir/auto_location.py b/mlir/test/python/ir/auto_location.py
new file mode 100644
index 0000000000000..c2d51083c1379
--- /dev/null
+++ b/mlir/test/python/ir/auto_location.py
@@ -0,0 +1,101 @@
+# RUN: %PYTHON %s | FileCheck %s
+# REQUIRES: python-ge-311
+import gc
+from contextlib import contextmanager
+
+from mlir.ir import *
+from mlir.dialects._ods_common import _cext
+from mlir.dialects import arith, _arith_ops_gen
+
+
+def run(f):
+    print("\nTEST:", f.__name__)
+    f()
+    gc.collect()
+    assert Context._get_live_count() == 0
+
+
+@contextmanager
+def with_infer_location():
+    _cext.globals.set_loc_tracebacks_enabled(True)
+    yield
+    _cext.globals.set_loc_tracebacks_enabled(False)
+
+
+# CHECK-LABEL: TEST: testInferLocations
+@run
+def testInferLocations():
+    with Context() as ctx, with_infer_location():
+        ctx.allow_unregistered_dialects = True
+
+        op = Operation.create("custom.op1")
+        one = arith.constant(IndexType.get(), 1)
+        _cext.globals.register_traceback_file_exclusion(arith.__file__)
+        two = arith.constant(IndexType.get(), 2)
+
+        # fmt: off
+        # CHECK: loc(callsite("testInferLocations"("{{.*}}[[SEP:[/\\]]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":31:13 to :43) at callsite("run"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":13:4 to :7) at "<module>"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":26:1 to :4))))
+        # fmt: on
+        print(op.location)
+
+        # fmt: off
+        # CHECK: loc(callsite("ConstantOp.__init__"("{{.*}}[[SEP]]mlir[[SEP]]dialects[[SEP]]arith.py":65:12 to :76) at callsite("constant"("{{.*}}[[SEP]]mlir[[SEP]]dialects[[SEP]]arith.py":110:40 to :81) at callsite("testInferLocations"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":32:14 to :48) at callsite("run"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":13:4 to :7) at "<module>"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":26:1 to :4))))))
+        # fmt: on
+        print(one.location)
+
+        # fmt: off
+        # CHECK: loc(callsite("testInferLocations"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":34:14 to :48) at callsite("run"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":13:4 to :7) at "<module>"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":26:1 to :4))))
+        # fmt: on
+        print(two.location)
+
+        _cext.globals.register_traceback_file_inclusion(_arith_ops_gen.__file__)
+        three = arith.constant(IndexType.get(), 3)
+        # fmt: off
+        # CHECK: loc(callsite("ConstantOp.__init__"("{{.*}}[[SEP]]mlir[[SEP]]dialects[[SEP]]_arith_ops_gen.py":405:4 to :218) at callsite("testInferLocations"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":52:16 to :50) at callsite("run"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":13:4 to :7) at "<module>"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":26:1 to :4)))))
+        # fmt: on
+        print(three.location)
+
+        def foo():
+            four = arith.constant(IndexType.get(), 4)
+            print(four.location)
+
+        # fmt: off
+        # CHECK: loc(callsite("ConstantOp.__init__"("{{.*}}[[SEP]]mlir[[SEP]]dialects[[SEP]]_arith_ops_gen.py":405:4 to :218) at callsite("testInferLocations.<locals>.foo"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":59:19 to :53) at callsite("testInferLocations"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":65:8 to :13) at callsite("run"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":13:4 to :7) at "<module>"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":26:1 to :4))))))
+        # fmt: on
+        foo()
+
+        _cext.globals.register_traceback_file_exclusion(__file__)
+
+        # fmt: off
+        # CHECK: loc("ConstantOp.__init__"("{{.*}}[[SEP]]mlir[[SEP]]dialects[[SEP]]_arith_ops_gen.py":405:4 to :218))
+        # fmt: on
+        foo()
+
+        def bar1():
+            def bar2():
+                def bar3():
+                    five = arith.constant(IndexType.get(), 5)
+                    print(five.location)
+
+                bar3()
+
+            bar2()
+
+        _cext.globals.register_traceback_file_inclusion(__file__)
+        _cext.globals.register_traceback_file_exclusion(_arith_ops_gen.__file__)
+
+        _cext.globals.set_loc_tracebacks_frame_limit(2)
+        # fmt: off
+        # CHECK: loc(callsite("testInferLocations.<locals>.bar1.<locals>.bar2.<locals>.bar3"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":77:27 to :61) at "testInferLocations.<locals>.bar1.<locals>.bar2"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":80:16 to :22)))
+        # fmt: on
+        bar1()
+
+        _cext.globals.set_loc_tracebacks_frame_limit(1)
+        # fmt: off
+        # CHECK: loc("testInferLocations.<locals>.bar1.<locals>.bar2.<locals>.bar3"("{{.*}}[[SEP]]test[[SEP]]python[[SEP]]ir[[SEP]]auto_location.py":77:27 to :61))
+        # fmt: on
+        bar1()
+
+        _cext.globals.set_loc_tracebacks_frame_limit(0)
+        # CHECK: loc(unknown)
+        bar1()
diff --git a/mlir/test/python/ir/context_managers.py b/mlir/test/python/ir/context_managers.py
index 8091687f7f082..5d9f9ceee97f3 100644
--- a/mlir/test/python/ir/context_managers.py
+++ b/mlir/test/python/ir/context_managers.py
@@ -35,25 +35,14 @@ def testLocationEnterExit():
             # Asserting a different context should clear it.
             with Context() as ctx2:
                 assert Context.current is ctx2
-                try:
-                    _ = Location.current
-                except ValueError:
-                    pass
-                else:
-                    assert False, "Expected exception"
+                assert Location.current is None
 
             # And should restore.
             assert Context.current is ctx1
             assert Location.current is loc1
 
     # All should clear.
-    try:
-        _ = Location.current
-    except ValueError as e:
-        # CHECK: No current Location
-        print(e)
-    else:
-        assert False, "Expected exception"
+    assert Location.current is None
 
 
 run(testLocationEnterExit)
@@ -72,12 +61,7 @@ def testInsertionPointEnterExit():
             assert InsertionPoint.current is ip
             assert Location.current is loc1
         # Location should clear.
-        try:
-            _ = Location.current
-        except ValueError:
-            pass
-        else:
-            assert False, "Expected exception"
+        assert Location.current is None
 
         # Asserting the same Context should preserve.
         with ctx1:
diff --git a/mlir/test/python/ir/operation.py b/mlir/test/python/ir/operation.py
index c6b5dafe792ca..bf16e3f75d60d 100644
--- a/mlir/test/python/ir/operation.py
+++ b/mlir/test/python/ir/operation.py
@@ -1021,6 +1021,8 @@ def testDetachFromParent():
     with Context():
         m1 = Module.parse("func.func private @foo()")
         func = m1.body.operations[0].detach_from_parent()
+        # CHECK: func.attached=False
+        print(f"{func.attached=}")
 
         try:
             func.detach_from_parent()
diff --git a/mlir/tools/mlir-tblgen/EnumPythonBindingGen.cpp b/mlir/tools/mlir-tblgen/EnumPythonBindingGen.cpp
index 8e2d6114e48eb..acc9b61d7121c 100644
--- a/mlir/tools/mlir-tblgen/EnumPythonBindingGen.cpp
+++ b/mlir/tools/mlir-tblgen/EnumPythonBindingGen.cpp
@@ -64,7 +64,7 @@ static void emitEnumClass(EnumInfo enumInfo, raw_ostream &os) {
   if (enumInfo.isBitEnum()) {
     os << formatv("    def __iter__(self):\n"
                   "        return iter([case for case in type(self) if "
-                  "(self & case) is case])\n");
+                  "(self & case) is case and self is not case])\n");
     os << formatv("    def __len__(self):\n"
                   "        return bin(self).count(\"1\")\n");
     os << "\n";
diff --git a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
index d2e38e9d23198..038f56d5a2150 100644
--- a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
@@ -41,6 +41,7 @@ from ._ods_common import (
     segmented_accessor as _ods_segmented_accessor,
 )
 _ods_ir = _ods_cext.ir
+_ods_cext.globals.register_traceback_file_exclusion(__file__)
 
 import builtins
 from typing import Sequence as _Sequence, Union as _Union
diff --git a/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp b/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp
index a6ed5c5b21e79..9ae90a4841f3c 100644
--- a/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp
+++ b/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp
@@ -714,3 +714,14 @@ TEST(IntegerRelationTest, getVarKindRange) {
   }
   EXPECT_THAT(actual, ElementsAre(2, 3, 4));
 }
+
+TEST(IntegerRelationTest, addLocalModulo) {
+  IntegerRelation rel = parseRelationFromSet("(x) : (x >= 0, 100 - x >= 0)", 1);
+  unsigned result = rel.addLocalModulo({1, 0}, 32); // x % 32
+  rel.convertVarKind(VarKind::Local,
+                     result - rel.getVarKindOffset(VarKind::Local),
+                     rel.getNumVarKind(VarKind::Local), VarKind::Range);
+  for (unsigned x = 0; x <= 100; ++x) {
+    EXPECT_TRUE(rel.containsPointNoLocal({x, x % 32}));
+  }
+}
diff --git a/mlir/unittests/ExecutionEngine/Invoke.cpp b/mlir/unittests/ExecutionEngine/Invoke.cpp
index 887db227cfc4b..b9a46c5ce942f 100644
--- a/mlir/unittests/ExecutionEngine/Invoke.cpp
+++ b/mlir/unittests/ExecutionEngine/Invoke.cpp
@@ -205,7 +205,13 @@ TEST(NativeMemRefJit, SKIP_WITHOUT_JIT(BasicMemref)) {
   };
   int64_t shape[] = {k, m};
   int64_t shapeAlloc[] = {k + 1, m + 1};
-  OwningMemRef<float, 2> a(shape, shapeAlloc, init);
+  // Use a large alignment to stress the case where the memref data/basePtr are
+  // disjoint.
+  int alignment = 8192;
+  OwningMemRef<float, 2> a(shape, shapeAlloc, init, alignment);
+  ASSERT_EQ(
+      (void *)(((uintptr_t)a->basePtr + alignment - 1) & ~(alignment - 1)),
+      a->data);
   ASSERT_EQ(a->sizes[0], k);
   ASSERT_EQ(a->sizes[1], m);
   ASSERT_EQ(a->strides[0], m + 1);
@@ -316,4 +322,55 @@ TEST(NativeMemRefJit, MAYBE_JITCallback) {
     ASSERT_EQ(elt, coefficient * count++);
 }
 
+static int initCnt = 0;
+// A helper function that will be called during the JIT's initialization.
+static void initCallback() { initCnt += 1; }
+
+TEST(MLIRExecutionEngine, MAYBE_JITCallbackInGlobalCtor) {
+  auto tmBuilderOrError = llvm::orc::JITTargetMachineBuilder::detectHost();
+  ASSERT_TRUE(!!tmBuilderOrError);
+  if (tmBuilderOrError->getTargetTriple().isAArch64()) {
+    GTEST_SKIP() << "Skipping global ctor initialization test on Aarch64 "
+                    "because of bug #71963";
+    return;
+  }
+  std::string moduleStr = R"mlir(
+  llvm.mlir.global_ctors ctors = [@ctor], priorities = [0 : i32], data = [#llvm.zero]
+  llvm.func @ctor() {
+    func.call @init_callback() : () -> ()
+    llvm.return
+  }
+  func.func private @init_callback() attributes { llvm.emit_c_interface }
+  )mlir";
+
+  DialectRegistry registry;
+  registerAllDialects(registry);
+  registerBuiltinDialectTranslation(registry);
+  registerLLVMDialectTranslation(registry);
+  MLIRContext context(registry);
+  auto module = parseSourceString<ModuleOp>(moduleStr, &context);
+  ASSERT_TRUE(!!module);
+  ASSERT_TRUE(succeeded(lowerToLLVMDialect(*module)));
+  ExecutionEngineOptions jitOptions;
+  auto jitOrError = ExecutionEngine::create(*module, jitOptions);
+  ASSERT_TRUE(!!jitOrError);
+  // validate initialization is not run on construction
+  ASSERT_EQ(initCnt, 0);
+  auto jit = std::move(jitOrError.get());
+  // Define any extra symbols so they're available at initialization.
+  jit->registerSymbols([&](llvm::orc::MangleAndInterner interner) {
+    llvm::orc::SymbolMap symbolMap;
+    symbolMap[interner("_mlir_ciface_init_callback")] = {
+        llvm::orc::ExecutorAddr::fromPtr(initCallback),
+        llvm::JITSymbolFlags::Exported};
+    return symbolMap;
+  });
+  jit->initialize();
+  // validate the side effect of initialization
+  ASSERT_EQ(initCnt, 1);
+  // next initialization should be noop
+  jit->initialize();
+  ASSERT_EQ(initCnt, 1);
+}
+
 #endif // _WIN32
diff --git a/mlir/unittests/IR/CMakeLists.txt b/mlir/unittests/IR/CMakeLists.txt
index a46e64718dab9..75cd2d65ef5a1 100644
--- a/mlir/unittests/IR/CMakeLists.txt
+++ b/mlir/unittests/IR/CMakeLists.txt
@@ -14,6 +14,7 @@ add_mlir_unittest(MLIRIRTests
   MemrefLayoutTest.cpp
   OperationSupportTest.cpp
   PatternMatchTest.cpp
+  RemarkTest.cpp
   ShapedTypeTest.cpp
   SymbolTableTest.cpp
   TypeTest.cpp
@@ -28,3 +29,4 @@ add_mlir_unittest(MLIRIRTests
 target_include_directories(MLIRIRTests PRIVATE "${MLIR_BINARY_DIR}/test/lib/Dialect/Test")
 mlir_target_link_libraries(MLIRIRTests PRIVATE MLIRIR)
 target_link_libraries(MLIRIRTests PRIVATE MLIRTestDialect)
+target_link_libraries(MLIRIRTests PRIVATE MLIRRemarkStreamer)
diff --git a/mlir/unittests/IR/RemarkTest.cpp b/mlir/unittests/IR/RemarkTest.cpp
new file mode 100644
index 0000000000000..65e1e08b83838
--- /dev/null
+++ b/mlir/unittests/IR/RemarkTest.cpp
@@ -0,0 +1,315 @@
+//===- RemarkTest.cpp - Remark unit tests -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/IR/Diagnostics.h"
+#include "mlir/IR/MLIRContext.h"
+#include "mlir/IR/Remarks.h"
+#include "mlir/Remark/RemarkStreamer.h"
+#include "mlir/Support/TypeID.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/LLVMRemarkStreamer.h"
+#include "llvm/Remarks/RemarkFormat.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/LogicalResult.h"
+#include "llvm/Support/YAMLParser.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <optional>
+
+using namespace llvm;
+using namespace mlir;
+using namespace testing;
+namespace {
+
+TEST(Remark, TestOutputOptimizationRemark) {
+  std::string categoryVectorizer("Vectorizer");
+  std::string categoryRegister("Register");
+  std::string categoryUnroll("Unroll");
+  std::string categoryInliner("Inliner");
+  std::string categoryReroller("Reroller");
+  std::string myPassname1("myPass1");
+  SmallString<64> tmpPathStorage;
+  sys::fs::createUniquePath("remarks-%%%%%%.yaml", tmpPathStorage,
+                            /*MakeAbsolute=*/true);
+  std::string yamlFile =
+      std::string(tmpPathStorage.data(), tmpPathStorage.size());
+  ASSERT_FALSE(yamlFile.empty());
+
+  {
+    MLIRContext context;
+    Location loc = UnknownLoc::get(&context);
+
+    context.printOpOnDiagnostic(true);
+    context.printStackTraceOnDiagnostic(true);
+
+    // Setup the remark engine
+    mlir::remark::RemarkCategories cats{/*passed=*/categoryVectorizer,
+                                        /*missed=*/categoryUnroll,
+                                        /*analysis=*/categoryRegister,
+                                        /*failed=*/categoryInliner};
+
+    LogicalResult isEnabled =
+        mlir::remark::enableOptimizationRemarksWithLLVMStreamer(
+            context, yamlFile, llvm::remarks::Format::YAML, cats);
+    ASSERT_TRUE(succeeded(isEnabled)) << "Failed to enable remark engine";
+
+    // PASS: something succeeded
+    remark::passed(loc, remark::RemarkOpts::name("Pass1")
+                            .category(categoryVectorizer)
+                            .subCategory(myPassname1)
+                            .function("bar"))
+        << "vectorized loop" << remark::metric("tripCount", 128);
+
+    // ANALYSIS: neutral insight
+    remark::analysis(
+        loc, remark::RemarkOpts::name("Analysis1").category(categoryRegister))
+        << "Kernel uses 168 registers";
+
+    // MISSED: explain why + suggest a fix
+    remark::missed(loc, remark::RemarkOpts::name("Miss1")
+                            .category(categoryUnroll)
+                            .subCategory(myPassname1))
+        << remark::reason("not profitable at this size")
+        << remark::suggest("increase unroll factor to >=4");
+
+    // FAILURE: action attempted but failed
+    remark::failed(loc, remark::RemarkOpts::name("Failed1")
+                            .category(categoryInliner)
+                            .subCategory(myPassname1))
+        << remark::reason("failed due to unsupported pattern");
+
+    // FAILURE: Won't show up
+    remark::failed(loc, remark::RemarkOpts::name("Failed2")
+                            .category(categoryReroller)
+                            .subCategory(myPassname1))
+        << remark::reason("failed due to rerolling pattern");
+  }
+
+  // Read the file
+  auto bufferOrErr = MemoryBuffer::getFile(yamlFile);
+  ASSERT_TRUE(static_cast<bool>(bufferOrErr)) << "Failed to open remarks file";
+  std::string content = bufferOrErr.get()->getBuffer().str();
+
+  EXPECT_THAT(content, HasSubstr("--- !Passed"));
+  EXPECT_THAT(content, HasSubstr("Name:            Pass1"));
+  EXPECT_THAT(content, HasSubstr("Pass:            'Vectorizer:myPass1'"));
+  EXPECT_THAT(content, HasSubstr("Function:        bar"));
+  EXPECT_THAT(content, HasSubstr("Remark:          vectorized loop"));
+  EXPECT_THAT(content, HasSubstr("tripCount:       '128'"));
+
+  EXPECT_THAT(content, HasSubstr("--- !Analysis"));
+  EXPECT_THAT(content, HasSubstr("Pass:            Register"));
+  EXPECT_THAT(content, HasSubstr("Name:            Analysis1"));
+  EXPECT_THAT(content, HasSubstr("Function:        '<unknown function>'"));
+  EXPECT_THAT(content, HasSubstr("Remark:          Kernel uses 168 registers"));
+
+  EXPECT_THAT(content, HasSubstr("--- !Missed"));
+  EXPECT_THAT(content, HasSubstr("Pass:            'Unroll:myPass1'"));
+  EXPECT_THAT(content, HasSubstr("Name:            Miss1"));
+  EXPECT_THAT(content, HasSubstr("Function:        '<unknown function>'"));
+  EXPECT_THAT(content,
+              HasSubstr("Reason:          not profitable at this size"));
+  EXPECT_THAT(content,
+              HasSubstr("Suggestion:      'increase unroll factor to >=4'"));
+
+  EXPECT_THAT(content, HasSubstr("--- !Failure"));
+  EXPECT_THAT(content, HasSubstr("Pass:            'Inliner:myPass1'"));
+  EXPECT_THAT(content, HasSubstr("Name:            Failed1"));
+  EXPECT_THAT(content, HasSubstr("Function:        '<unknown function>'"));
+  EXPECT_THAT(content,
+              HasSubstr("Reason:          failed due to unsupported pattern"));
+
+  EXPECT_THAT(content, Not(HasSubstr("Failed2")));
+  EXPECT_THAT(content, Not(HasSubstr("Reroller")));
+
+  // Also verify document order to avoid false positives.
+  size_t iPassed = content.find("--- !Passed");
+  size_t iAnalysis = content.find("--- !Analysis");
+  size_t iMissed = content.find("--- !Missed");
+  size_t iFailure = content.find("--- !Failure");
+
+  ASSERT_NE(iPassed, std::string::npos);
+  ASSERT_NE(iAnalysis, std::string::npos);
+  ASSERT_NE(iMissed, std::string::npos);
+  ASSERT_NE(iFailure, std::string::npos);
+
+  EXPECT_LT(iPassed, iAnalysis);
+  EXPECT_LT(iAnalysis, iMissed);
+  EXPECT_LT(iMissed, iFailure);
+}
+
+TEST(Remark, TestNoOutputOptimizationRemark) {
+  const auto *pass1Msg = "My message";
+
+  std::string categoryFailName("myImportantCategory");
+  std::string myPassname1("myPass1");
+  std::string funcName("myFunc");
+  SmallString<64> tmpPathStorage;
+  sys::fs::createUniquePath("remarks-%%%%%%.yaml", tmpPathStorage,
+                            /*MakeAbsolute=*/true);
+  std::string yamlFile =
+      std::string(tmpPathStorage.data(), tmpPathStorage.size());
+  ASSERT_FALSE(yamlFile.empty());
+  std::error_code ec =
+      llvm::sys::fs::remove(yamlFile, /*IgnoreNonExisting=*/true);
+  if (ec) {
+    FAIL() << "Failed to remove file " << yamlFile << ": " << ec.message();
+  }
+  {
+    MLIRContext context;
+    Location loc = UnknownLoc::get(&context);
+    remark::failed(loc, remark::RemarkOpts::name("myfail")
+                            .category(categoryFailName)
+                            .subCategory(myPassname1))
+        << remark::reason(pass1Msg);
+  }
+  // No setup, so no output file should be created
+  // check!
+  bool fileExists = llvm::sys::fs::exists(yamlFile);
+  EXPECT_FALSE(fileExists)
+      << "Expected no YAML file to be created without setupOptimizationRemarks";
+}
+
+TEST(Remark, TestOutputOptimizationRemarkDiagnostic) {
+  std::string categoryVectorizer("Vectorizer");
+  std::string categoryRegister("Register");
+  std::string categoryUnroll("Unroll");
+  std::string myPassname1("myPass1");
+  std::string fName("foo");
+
+  llvm::SmallVector<std::string> seenMsg;
+  {
+    MLIRContext context;
+    Location loc = UnknownLoc::get(&context);
+
+    context.printOpOnDiagnostic(true);
+    context.printStackTraceOnDiagnostic(true);
+
+    // Register a handler that captures the diagnostic.
+    ScopedDiagnosticHandler handler(&context, [&](Diagnostic &diag) {
+      seenMsg.push_back(diag.str());
+      return success();
+    });
+
+    // Setup the remark engine
+    mlir::remark::RemarkCategories cats{/*passed=*/categoryVectorizer,
+                                        /*missed=*/categoryUnroll,
+                                        /*analysis=*/categoryRegister,
+                                        /*failed=*/categoryUnroll};
+
+    LogicalResult isEnabled =
+        remark::enableOptimizationRemarks(context, nullptr, cats, true);
+
+    ASSERT_TRUE(succeeded(isEnabled)) << "Failed to enable remark engine";
+
+    // PASS: something succeeded
+    remark::passed(loc, remark::RemarkOpts::name("pass1")
+                            .category(categoryVectorizer)
+                            .function(fName)
+                            .subCategory(myPassname1))
+        << "vectorized loop" << remark::metric("tripCount", 128);
+
+    // ANALYSIS: neutral insight
+    remark::analysis(loc, remark::RemarkOpts::name("Analysis1")
+                              .category(categoryRegister)
+                              .function(fName))
+        << "Kernel uses 168 registers";
+
+    // MISSED: explain why + suggest a fix
+    int target = 128;
+    int tripBad = 4;
+    int threshold = 256;
+
+    remark::missed(loc, {"", categoryUnroll, "unroller2", ""})
+        << remark::reason("tripCount={0} < threshold={1}", tripBad, threshold);
+
+    remark::missed(loc, {"", categoryUnroll, "", ""})
+        << remark::reason("tripCount={0} < threshold={1}", tripBad, threshold)
+        << remark::suggest("increase unroll to {0}", target);
+
+    // FAILURE: action attempted but failed
+    remark::failed(loc, {"", categoryUnroll, "", ""})
+        << remark::reason("failed due to unsupported pattern");
+  }
+  // clang-format off
+  unsigned long expectedSize = 5;
+  ASSERT_EQ(seenMsg.size(), expectedSize);
+  EXPECT_EQ(seenMsg[0], "[Passed] pass1 | Category:Vectorizer:myPass1 | Function=foo | Remark=\"vectorized loop\", tripCount=128");
+  EXPECT_EQ(seenMsg[1], "[Analysis] Analysis1 | Category:Register | Function=foo | Remark=\"Kernel uses 168 registers\"");
+  EXPECT_EQ(seenMsg[2], "[Missed]  | Category:Unroll:unroller2 | Reason=\"tripCount=4 < threshold=256\"");
+  EXPECT_EQ(seenMsg[3], "[Missed]  | Category:Unroll | Reason=\"tripCount=4 < threshold=256\", Suggestion=\"increase unroll to 128\"");
+  EXPECT_EQ(seenMsg[4], "[Failure]  | Category:Unroll | Reason=\"failed due to unsupported pattern\"");
+  // clang-format on
+}
+
+/// Custom remark streamer that prints remarks to stderr.
+class MyCustomStreamer : public remark::detail::MLIRRemarkStreamerBase {
+public:
+  MyCustomStreamer() = default;
+
+  void streamOptimizationRemark(const remark::detail::Remark &remark) override {
+    llvm::errs() << "Custom remark: ";
+    remark.print(llvm::errs(), true);
+    llvm::errs() << "\n";
+  }
+};
+
+TEST(Remark, TestCustomOptimizationRemarkDiagnostic) {
+  testing::internal::CaptureStderr();
+  const auto *pass1Msg = "My message";
+  const auto *pass2Msg = "My another message";
+  const auto *pass3Msg = "Do not show this message";
+
+  std::string categoryLoopunroll("LoopUnroll");
+  std::string categoryInline("Inliner");
+  std::string myPassname1("myPass1");
+  std::string myPassname2("myPass2");
+  std::string funcName("myFunc");
+
+  std::string seenMsg = "";
+
+  {
+    MLIRContext context;
+    Location loc = UnknownLoc::get(&context);
+
+    // Setup the remark engine
+    mlir::remark::RemarkCategories cats{/*passed=*/categoryLoopunroll,
+                                        /*missed=*/std::nullopt,
+                                        /*analysis=*/std::nullopt,
+                                        /*failed=*/categoryLoopunroll};
+
+    LogicalResult isEnabled = remark::enableOptimizationRemarks(
+        context, std::make_unique<MyCustomStreamer>(), cats, true);
+    ASSERT_TRUE(succeeded(isEnabled)) << "Failed to enable remark engine";
+
+    // Remark 1: pass, category LoopUnroll
+    remark::passed(loc, {"", categoryLoopunroll, myPassname1, ""}) << pass1Msg;
+    // Remark 2: failure, category LoopUnroll
+    remark::failed(loc, {"", categoryLoopunroll, myPassname2, ""})
+        << remark::reason(pass2Msg);
+    // Remark 3: pass, category Inline (should not be printed)
+    remark::passed(loc, {"", categoryInline, myPassname1, ""}) << pass3Msg;
+  }
+
+  llvm::errs().flush();
+  std::string errOut = ::testing::internal::GetCapturedStderr();
+
+  // Expect exactly two "Custom remark:" lines.
+  auto first = errOut.find("Custom remark:");
+  EXPECT_NE(first, std::string::npos);
+  auto second = errOut.find("Custom remark:", first + 1);
+  EXPECT_NE(second, std::string::npos);
+  auto third = errOut.find("Custom remark:", second + 1);
+  EXPECT_EQ(third, std::string::npos);
+
+  // Containment checks for messages.
+  EXPECT_NE(errOut.find(pass1Msg), std::string::npos); // printed
+  EXPECT_NE(errOut.find(pass2Msg), std::string::npos); // printed
+  EXPECT_EQ(errOut.find(pass3Msg), std::string::npos); // filtered out
+}
+} // namespace
diff --git a/mlir/utils/clang-tidy/apply-clang-tidy.sh b/mlir/utils/clang-tidy/apply-clang-tidy.sh
index a3592187361ef..f0973dd73d7e5 100755
--- a/mlir/utils/clang-tidy/apply-clang-tidy.sh
+++ b/mlir/utils/clang-tidy/apply-clang-tidy.sh
@@ -89,7 +89,8 @@ find $SRCS | grep ".cpp$" | sort | while read file ; do
 
     echo "-----------------------------------"
     echo "-- Apply check $check on file $file"
-    echo "$TIMING_TIDY $CLANG_TIDY -p $BUILD_DIR $file --checks="-*,$check" -fix -fix-errors"
+    COMMAND=$(echo "$TIMING_TIDY $CLANG_TIDY -p $BUILD_DIR $file --checks="-*,$check" -fix -fix-errors")
+    echo $COMMAND
     { $TIMING_TIDY $CLANG_TIDY -p $BUILD_DIR $file --checks="-*,$check" -fix -fix-errors ; } 2>&1
     git clang-format -f
     if [[ $(git diff --stat) == '' ]]; then
@@ -101,16 +102,19 @@ find $SRCS | grep ".cpp$" | sort | while read file ; do
     # Clang-tidy sometimes update files in the build directory, erase the .inc file generate by tablegen
     # to force them to be regenerated now.
     find $BUILD_DIR/tools/mlir/ | grep '\.inc' | while read file ; do rm $file ; done
-    ninja -C $BUILD_DIR check-mlir > ${REJECT_DIR}/ninja.${check}.$(basename $file).log 2>&1
+    echo $COMMAND > ${REJECT_DIR}/ninja.${check}.$(basename $file).log
+    ninja -C $BUILD_DIR --quiet check-mlir >> ${REJECT_DIR}/${check}.$(basename $file).ninja.log 2>&1
     if [[ $? != 0 ]] ; then
       echo "check-mlir failed! (see ninja.${check}.${file}.log)"
       [[ ! -z "$REJECT_DIR" ]] && git diff > "${REJECT_DIR}/${check}_$(basename ${file}).reject.diff"
       continue
     fi
+    rm -f ${REJECT_DIR}/ninja.${check}.$(basename $file).log
+
     echo "-----------------------------------"
     echo "-- Success, commit changes for check $check on file $file"
     git clang-format -f
 
-    git commit -a -m "Apply clang-tidy fixes for $check in $(basename $file) (NFC)"
+    git commit -a -m "[MLIR] Apply clang-tidy fixes for $check in $(basename $file) (NFC)"
   done
 done
diff --git a/mlir/utils/tree-sitter-mlir/dialect/linalg.js b/mlir/utils/tree-sitter-mlir/dialect/linalg.js
index ddde92b2f692b..f4658085ce6f3 100644
--- a/mlir/utils/tree-sitter-mlir/dialect/linalg.js
+++ b/mlir/utils/tree-sitter-mlir/dialect/linalg.js
@@ -4,7 +4,6 @@ module.exports = {
   linalg_dialect : $ => prec.right(choice(
                      seq(choice(
                              'linalg.batch_matmul',
-                             'linalg.batch_matmul_transpose_b',
                              'linalg.batch_matvec',
                              'linalg.batch_reduce_matmul', 'linalg.broadcast',
                              'linalg.conv_1d_ncw_fcw', 'linalg.conv_1d_nwc_wcf',
@@ -27,7 +26,6 @@ module.exports = {
                              'linalg.dot', 'linalg.elemwise_binary',
                              'linalg.elemwise_unary', 'linalg.fill',
                              'linalg.fill_rng_2d', 'linalg.matmul',
-                             'linalg.matmul_transpose_b',
                              'linalg.matmul_unsigned', 'linalg.matvec',
                              'linalg.mmt4d', 'linalg.pooling_nchw_max',
                              'linalg.pooling_nchw_sum',
diff --git a/mlir/utils/tree-sitter-mlir/queries/highlights.scm b/mlir/utils/tree-sitter-mlir/queries/highlights.scm
index 4cbea7bbca031..59e280bab414a 100644
--- a/mlir/utils/tree-sitter-mlir/queries/highlights.scm
+++ b/mlir/utils/tree-sitter-mlir/queries/highlights.scm
@@ -213,7 +213,6 @@
   "bufferization.to_tensor"
 
   "linalg.batch_matmul"
-  "linalg.batch_matmul_transpose_b"
   "linalg.batch_matvec"
   "linalg.batch_reduce_matmul"
   "linalg.broadcast"
@@ -244,7 +243,6 @@
   "linalg.fill"
   "linalg.fill_rng_2d"
   "linalg.matmul"
-  "linalg.matmul_transpose_b"
   "linalg.matmul_unsigned"
   "linalg.matvec"
   "linalg.mmt4d"
diff --git a/offload/DeviceRTL/src/Workshare.cpp b/offload/DeviceRTL/src/Workshare.cpp
index a8759307b42bd..59a2cc3f27aca 100644
--- a/offload/DeviceRTL/src/Workshare.cpp
+++ b/offload/DeviceRTL/src/Workshare.cpp
@@ -698,7 +698,7 @@ template <typename Ty> class StaticLoopChunker {
   static void NormalizedLoopNestNoChunk(void (*LoopBody)(Ty, void *), void *Arg,
                                         Ty NumBlocks, Ty BId, Ty NumThreads,
                                         Ty TId, Ty NumIters,
-                                        bool OneIterationPerThread) {
+                                        uint8_t OneIterationPerThread) {
     Ty KernelIteration = NumBlocks * NumThreads;
 
     // Start index in the normalized space.
@@ -729,7 +729,7 @@ template <typename Ty> class StaticLoopChunker {
                                         Ty BlockChunk, Ty NumBlocks, Ty BId,
                                         Ty ThreadChunk, Ty NumThreads, Ty TId,
                                         Ty NumIters,
-                                        bool OneIterationPerThread) {
+                                        uint8_t OneIterationPerThread) {
     Ty KernelIteration = NumBlocks * BlockChunk;
 
     // Start index in the chunked space.
@@ -767,8 +767,18 @@ template <typename Ty> class StaticLoopChunker {
 
 public:
   /// Worksharing `for`-loop.
+  /// \param[in] Loc Description of source location
+  /// \param[in] LoopBody Function which corresponds to loop body
+  /// \param[in] Arg Pointer to struct which contains loop body args
+  /// \param[in] NumIters Number of loop iterations
+  /// \param[in] NumThreads Number of GPU threads
+  /// \param[in] ThreadChunk Size of thread chunk
+  /// \param[in] OneIterationPerThread If true/nonzero, each thread executes
+  /// only one loop iteration or one thread chunk. This avoids an outer loop
+  /// over all loop iterations/chunks.
   static void For(IdentTy *Loc, void (*LoopBody)(Ty, void *), void *Arg,
-                  Ty NumIters, Ty NumThreads, Ty ThreadChunk) {
+                  Ty NumIters, Ty NumThreads, Ty ThreadChunk,
+                  uint8_t OneIterationPerThread) {
     ASSERT(NumIters >= 0, "Bad iteration count");
     ASSERT(ThreadChunk >= 0, "Bad thread count");
 
@@ -790,12 +800,13 @@ template <typename Ty> class StaticLoopChunker {
 
     // If we know we have more threads than iterations we can indicate that to
     // avoid an outer loop.
-    bool OneIterationPerThread = false;
     if (config::getAssumeThreadsOversubscription()) {
-      ASSERT(NumThreads >= NumIters, "Broken assumption");
       OneIterationPerThread = true;
     }
 
+    if (OneIterationPerThread)
+      ASSERT(NumThreads >= NumIters, "Broken assumption");
+
     if (ThreadChunk != 1)
       NormalizedLoopNestChunked(LoopBody, Arg, BlockChunk, NumBlocks, BId,
                                 ThreadChunk, NumThreads, TId, NumIters,
@@ -806,8 +817,17 @@ template <typename Ty> class StaticLoopChunker {
   }
 
   /// Worksharing `distribute`-loop.
+  /// \param[in] Loc Description of source location
+  /// \param[in] LoopBody Function which corresponds to loop body
+  /// \param[in] Arg Pointer to struct which contains loop body args
+  /// \param[in] NumIters Number of loop iterations
+  /// \param[in] BlockChunk Size of block chunk
+  /// \param[in] OneIterationPerThread If true/nonzero, each thread executes
+  /// only one loop iteration or one thread chunk. This avoids an outer loop
+  /// over all loop iterations/chunks.
   static void Distribute(IdentTy *Loc, void (*LoopBody)(Ty, void *), void *Arg,
-                         Ty NumIters, Ty BlockChunk) {
+                         Ty NumIters, Ty BlockChunk,
+                         uint8_t OneIterationPerThread) {
     ASSERT(icv::Level == 0, "Bad distribute");
     ASSERT(icv::ActiveLevel == 0, "Bad distribute");
     ASSERT(state::ParallelRegionFn == nullptr, "Bad distribute");
@@ -831,12 +851,13 @@ template <typename Ty> class StaticLoopChunker {
 
     // If we know we have more blocks than iterations we can indicate that to
     // avoid an outer loop.
-    bool OneIterationPerThread = false;
     if (config::getAssumeTeamsOversubscription()) {
-      ASSERT(NumBlocks >= NumIters, "Broken assumption");
       OneIterationPerThread = true;
     }
 
+    if (OneIterationPerThread)
+      ASSERT(NumBlocks >= NumIters, "Broken assumption");
+
     if (BlockChunk != NumThreads)
       NormalizedLoopNestChunked(LoopBody, Arg, BlockChunk, NumBlocks, BId,
                                 ThreadChunk, NumThreads, TId, NumIters,
@@ -852,9 +873,20 @@ template <typename Ty> class StaticLoopChunker {
   }
 
   /// Worksharing `distribute parallel for`-loop.
+  /// \param[in] Loc Description of source location
+  /// \param[in] LoopBody Function which corresponds to loop body
+  /// \param[in] Arg Pointer to struct which contains loop body args
+  /// \param[in] NumIters Number of loop iterations
+  /// \param[in] NumThreads Number of GPU threads
+  /// \param[in] BlockChunk Size of block chunk
+  /// \param[in] ThreadChunk Size of thread chunk
+  /// \param[in] OneIterationPerThread If true/nonzero, each thread executes
+  /// only one loop iteration or one thread chunk. This avoids an outer loop
+  /// over all loop iterations/chunks.
   static void DistributeFor(IdentTy *Loc, void (*LoopBody)(Ty, void *),
                             void *Arg, Ty NumIters, Ty NumThreads,
-                            Ty BlockChunk, Ty ThreadChunk) {
+                            Ty BlockChunk, Ty ThreadChunk,
+                            uint8_t OneIterationPerThread) {
     ASSERT(icv::Level == 1, "Bad distribute");
     ASSERT(icv::ActiveLevel == 1, "Bad distribute");
     ASSERT(state::ParallelRegionFn == nullptr, "Bad distribute");
@@ -882,13 +914,14 @@ template <typename Ty> class StaticLoopChunker {
 
     // If we know we have more threads (across all blocks) than iterations we
     // can indicate that to avoid an outer loop.
-    bool OneIterationPerThread = false;
     if (config::getAssumeTeamsOversubscription() &
         config::getAssumeThreadsOversubscription()) {
       OneIterationPerThread = true;
-      ASSERT(NumBlocks * NumThreads >= NumIters, "Broken assumption");
     }
 
+    if (OneIterationPerThread)
+      ASSERT(NumBlocks * NumThreads >= NumIters, "Broken assumption");
+
     if (BlockChunk != NumThreads || ThreadChunk != 1)
       NormalizedLoopNestChunked(LoopBody, Arg, BlockChunk, NumBlocks, BId,
                                 ThreadChunk, NumThreads, TId, NumIters,
@@ -907,24 +940,26 @@ template <typename Ty> class StaticLoopChunker {
 
 #define OMP_LOOP_ENTRY(BW, TY)                                                 \
   [[gnu::flatten, clang::always_inline]] void                                  \
-      __kmpc_distribute_for_static_loop##BW(                                   \
-          IdentTy *loc, void (*fn)(TY, void *), void *arg, TY num_iters,       \
-          TY num_threads, TY block_chunk, TY thread_chunk) {                   \
+  __kmpc_distribute_for_static_loop##BW(                                       \
+      IdentTy *loc, void (*fn)(TY, void *), void *arg, TY num_iters,           \
+      TY num_threads, TY block_chunk, TY thread_chunk,                         \
+      uint8_t one_iteration_per_thread) {                                      \
     ompx::StaticLoopChunker<TY>::DistributeFor(                                \
-        loc, fn, arg, num_iters, num_threads, block_chunk, thread_chunk);      \
+        loc, fn, arg, num_iters, num_threads, block_chunk, thread_chunk,       \
+        one_iteration_per_thread);                                             \
   }                                                                            \
   [[gnu::flatten, clang::always_inline]] void                                  \
-      __kmpc_distribute_static_loop##BW(IdentTy *loc, void (*fn)(TY, void *),  \
-                                        void *arg, TY num_iters,               \
-                                        TY block_chunk) {                      \
-    ompx::StaticLoopChunker<TY>::Distribute(loc, fn, arg, num_iters,           \
-                                            block_chunk);                      \
+  __kmpc_distribute_static_loop##BW(IdentTy *loc, void (*fn)(TY, void *),      \
+                                    void *arg, TY num_iters, TY block_chunk,   \
+                                    uint8_t one_iteration_per_thread) {        \
+    ompx::StaticLoopChunker<TY>::Distribute(                                   \
+        loc, fn, arg, num_iters, block_chunk, one_iteration_per_thread);       \
   }                                                                            \
   [[gnu::flatten, clang::always_inline]] void __kmpc_for_static_loop##BW(      \
       IdentTy *loc, void (*fn)(TY, void *), void *arg, TY num_iters,           \
-      TY num_threads, TY thread_chunk) {                                       \
+      TY num_threads, TY thread_chunk, uint8_t one_iteration_per_thread) {     \
     ompx::StaticLoopChunker<TY>::For(loc, fn, arg, num_iters, num_threads,     \
-                                     thread_chunk);                            \
+                                     thread_chunk, one_iteration_per_thread);  \
   }
 
 extern "C" {
diff --git a/offload/cmake/OpenMPTesting.cmake b/offload/cmake/OpenMPTesting.cmake
index 8e955ff399275..ef8cf34ba0c85 100644
--- a/offload/cmake/OpenMPTesting.cmake
+++ b/offload/cmake/OpenMPTesting.cmake
@@ -57,7 +57,7 @@ if (${OPENMP_STANDALONE_BUILD})
   if (MSVC OR XCODE)
     set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --no-progress-bar")
   endif()
-  if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  if ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
     set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests --timeout=1800")
   endif()
   set(OPENMP_LIT_ARGS "${DEFAULT_LIT_ARGS}" CACHE STRING "Options for lit.")
diff --git a/offload/include/OpenMP/InteropAPI.h b/offload/include/OpenMP/InteropAPI.h
index 71c78760a3226..53ac4be2e2e98 100644
--- a/offload/include/OpenMP/InteropAPI.h
+++ b/offload/include/OpenMP/InteropAPI.h
@@ -13,17 +13,69 @@
 
 #include "omp.h"
 
+#include "PerThreadTable.h"
 #include "omptarget.h"
 
 extern "C" {
 
 typedef enum kmp_interop_type_t {
   kmp_interop_type_unknown = -1,
-  kmp_interop_type_platform,
-  kmp_interop_type_device,
-  kmp_interop_type_tasksync,
+  kmp_interop_type_target,
+  kmp_interop_type_targetsync,
 } kmp_interop_type_t;
 
+struct interop_attrs_t {
+  bool inorder : 1;
+  int reserved : 31;
+
+  /// Check if the supported attributes are compatible with the current
+  ///   attributes. Only if an attribute is supported can the value be true,
+  ///   otherwise it needs to be false
+  bool checkSupportedOnly(interop_attrs_t supported) const {
+    return supported.inorder || (!supported.inorder && !inorder);
+  }
+};
+
+struct interop_spec_t {
+  int32_t fr_id;
+  interop_attrs_t attrs; // Common attributes
+  int64_t impl_attrs; // Implementation specific attributes (recognized by each
+                      // plugin)
+};
+
+struct interop_flags_t {
+  bool implicit : 1; // dispatch (true) or interop (false)
+  bool nowait : 1;   // has nowait flag
+  int reserved : 30;
+};
+
+struct interop_ctx_t {
+  uint32_t version; // version of the interface (current is 0)
+  interop_flags_t flags;
+  int gtid;
+};
+
+struct dep_pack_t {
+  int32_t ndeps;
+  int32_t ndeps_noalias;
+  kmp_depend_info_t *deplist;
+  kmp_depend_info_t *noalias_deplist;
+};
+
+struct omp_interop_val_t;
+
+typedef void ompx_interop_cb_t(omp_interop_val_t *interop, void *data);
+
+struct omp_interop_cb_instance_t {
+  ompx_interop_cb_t *cb;
+  void *data;
+
+  omp_interop_cb_instance_t(ompx_interop_cb_t *cb, void *data)
+      : cb(cb), data(data) {}
+
+  void operator()(omp_interop_val_t *interop) { cb(interop, data); }
+};
+
 /// The interop value type, aka. the interop object.
 typedef struct omp_interop_val_t {
   /// Device and interop-type are determined at construction time and fix.
@@ -34,10 +86,98 @@ typedef struct omp_interop_val_t {
   __tgt_device_info device_info;
   const kmp_interop_type_t interop_type;
   const intptr_t device_id;
-  const omp_foreign_runtime_ids_t vendor_id = cuda;
-  const intptr_t backend_type_id = omp_interop_backend_type_cuda_1;
+  omp_vendor_id_t vendor_id = omp_vendor_llvm;
+  tgt_foreign_runtime_id_t fr_id = tgt_fr_none;
+  interop_attrs_t attrs{false, 0}; // Common prefer specification attributes
+  int64_t impl_attrs = 0; // Implementation prefer specification attributes
+
+  // Constants
+  static constexpr int no_owner = -1; // This interop has no current owner
+
+  void *rtl_property = nullptr; // Plugin dependent information
+  // For implicitly created Interop objects (e.g., from a dispatch construct)
+  // who owns the object
+  int owner_gtid = no_owner;
+  // Marks whether the object was requested since the last time it was synced
+  bool clean = true;
+
+  typedef llvm::SmallVector<omp_interop_cb_instance_t> callback_list_t;
+
+  callback_list_t completion_cbs;
+
+  void reset() {
+    owner_gtid = no_owner;
+    markClean();
+    clearCompletionCbs();
+  }
+
+  llvm::Expected<DeviceTy &> getDevice() const;
+
+  bool hasOwner() const { return owner_gtid != no_owner; }
+
+  void setOwner(int gtid) { owner_gtid = gtid; }
+  bool isOwnedBy(int gtid) { return owner_gtid == gtid; }
+  bool isCompatibleWith(int32_t InteropType, const interop_spec_t &Spec);
+  bool isCompatibleWith(int32_t InteropType, const interop_spec_t &Spec,
+                        int64_t DeviceNum, int gtid);
+  void markClean() { clean = true; }
+  void markDirty() { clean = false; }
+  bool isClean() const { return clean; }
+
+  int32_t flush(DeviceTy &Device);
+  int32_t sync_barrier(DeviceTy &Device);
+  int32_t async_barrier(DeviceTy &Device);
+  int32_t release(DeviceTy &Device);
+
+  void addCompletionCb(ompx_interop_cb_t *cb, void *data) {
+    completion_cbs.push_back(omp_interop_cb_instance_t(cb, data));
+  }
+
+  int numCompletionCbs() const { return completion_cbs.size(); }
+  void clearCompletionCbs() { completion_cbs.clear(); }
+
+  void runCompletionCbs() {
+    for (auto &cbInstance : completion_cbs)
+      cbInstance(this);
+    clearCompletionCbs();
+  }
 } omp_interop_val_t;
 
 } // extern "C"
 
+struct InteropTableEntry {
+  using ContainerTy = typename std::vector<omp_interop_val_t *>;
+  using iterator = typename ContainerTy::iterator;
+
+  ContainerTy Interops;
+
+  static constexpr int reservedEntriesPerThread =
+      20; // reserve some entries to avoid reallocation
+
+  void add(omp_interop_val_t *obj) {
+    if (Interops.capacity() == 0)
+      Interops.reserve(reservedEntriesPerThread);
+    Interops.push_back(obj);
+  }
+
+  template <class ClearFuncTy> void clear(ClearFuncTy f) {
+    for (auto &Obj : Interops) {
+      f(Obj);
+    }
+  }
+
+  /// vector interface
+  int size() const { return Interops.size(); }
+  iterator begin() { return Interops.begin(); }
+  iterator end() { return Interops.end(); }
+  iterator erase(iterator it) { return Interops.erase(it); }
+};
+
+struct InteropTblTy
+    : public PerThreadTable<InteropTableEntry, omp_interop_val_t *> {
+  void clear();
+};
+
+void syncImplicitInterops(int gtid, void *event);
+
 #endif // OMPTARGET_OPENMP_INTEROP_API_H
diff --git a/offload/include/OpenMP/Mapping.h b/offload/include/OpenMP/Mapping.h
index b9f5c16582931..93c1e56905ae4 100644
--- a/offload/include/OpenMP/Mapping.h
+++ b/offload/include/OpenMP/Mapping.h
@@ -417,12 +417,42 @@ struct MapperComponentsTy {
 typedef void (*MapperFuncPtrTy)(void *, void *, void *, int64_t, int64_t,
                                 void *);
 
+/// Structure to store information about a single ATTACH map entry.
+struct AttachMapInfo {
+  void *PointerBase;
+  void *PointeeBegin;
+  int64_t PointerSize;
+  int64_t MapType;
+  map_var_info_t Pointername;
+
+  AttachMapInfo(void *PointerBase, void *PointeeBegin, int64_t Size,
+                int64_t Type, map_var_info_t Name)
+      : PointerBase(PointerBase), PointeeBegin(PointeeBegin), PointerSize(Size),
+        MapType(Type), Pointername(Name) {}
+};
+
+/// Structure to track ATTACH entries and new allocations across recursive calls
+/// (for handling mappers) to targetDataBegin for a given construct.
+struct AttachInfoTy {
+  /// ATTACH map entries for deferred processing.
+  llvm::SmallVector<AttachMapInfo> AttachEntries;
+
+  /// Key: host pointer, Value: allocation size.
+  llvm::DenseMap<void *, int64_t> NewAllocations;
+
+  AttachInfoTy() = default;
+
+  // Delete copy constructor and copy assignment operator to prevent copying
+  AttachInfoTy(const AttachInfoTy &) = delete;
+  AttachInfoTy &operator=(const AttachInfoTy &) = delete;
+};
+
 // Function pointer type for targetData* functions (targetDataBegin,
 // targetDataEnd and targetDataUpdate).
 typedef int (*TargetDataFuncPtrTy)(ident_t *, DeviceTy &, int32_t, void **,
                                    void **, int64_t *, int64_t *,
                                    map_var_info_t *, void **, AsyncInfoTy &,
-                                   bool);
+                                   AttachInfoTy *, bool);
 
 void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device,
                                bool toStdOut = false);
@@ -431,20 +461,26 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                     void **ArgsBase, void **Args, int64_t *ArgSizes,
                     int64_t *ArgTypes, map_var_info_t *ArgNames,
                     void **ArgMappers, AsyncInfoTy &AsyncInfo,
+                    AttachInfoTy *AttachInfo = nullptr,
                     bool FromMapper = false);
 
 int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                   void **ArgBases, void **Args, int64_t *ArgSizes,
                   int64_t *ArgTypes, map_var_info_t *ArgNames,
                   void **ArgMappers, AsyncInfoTy &AsyncInfo,
-                  bool FromMapper = false);
+                  AttachInfoTy *AttachInfo = nullptr, bool FromMapper = false);
 
 int targetDataUpdate(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                      void **ArgsBase, void **Args, int64_t *ArgSizes,
                      int64_t *ArgTypes, map_var_info_t *ArgNames,
                      void **ArgMappers, AsyncInfoTy &AsyncInfo,
+                     AttachInfoTy *AttachInfo = nullptr,
                      bool FromMapper = false);
 
+// Process deferred ATTACH map entries collected during targetDataBegin.
+int processAttachEntries(DeviceTy &Device, AttachInfoTy &AttachInfo,
+                         AsyncInfoTy &AsyncInfo);
+
 struct MappingInfoTy {
   MappingInfoTy(DeviceTy &Device) : Device(Device) {}
 
diff --git a/offload/include/OpenMP/omp.h b/offload/include/OpenMP/omp.h
index b44c6aff1b289..49d9f1fa75c20 100644
--- a/offload/include/OpenMP/omp.h
+++ b/offload/include/OpenMP/omp.h
@@ -80,15 +80,18 @@ typedef enum omp_interop_rc {
   omp_irc_other = -6
 } omp_interop_rc_t;
 
-typedef enum omp_interop_fr {
-  omp_ifr_cuda = 1,
-  omp_ifr_cuda_driver = 2,
-  omp_ifr_opencl = 3,
-  omp_ifr_sycl = 4,
-  omp_ifr_hip = 5,
-  omp_ifr_level_zero = 6,
-  omp_ifr_last = 7
-} omp_interop_fr_t;
+/* Foreign runtime values from OpenMP Additional Definitions document v2.1 */
+typedef enum tgt_foreign_runtime_id_t {
+  tgt_fr_none = 0,
+  tgt_fr_cuda = 1,
+  tgt_fr_cuda_driver = 2,
+  tgt_fr_opencl = 3,
+  tgt_fr_sycl = 4,
+  tgt_fr_hip = 5,
+  tgt_fr_level_zero = 6,
+  tgt_fr_hsa = 7,
+  tgt_fr_last = 8
+} tgt_foreign_runtime_id_t;
 
 typedef void *omp_interop_t;
 
@@ -134,19 +137,23 @@ omp_get_interop_type_desc(const omp_interop_t, omp_interop_property_t);
 extern const char *__KAI_KMPC_CONVENTION
 omp_get_interop_rc_desc(const omp_interop_t, omp_interop_rc_t);
 
-typedef enum omp_interop_backend_type_t {
-  // reserve 0
-  omp_interop_backend_type_cuda_1 = 1,
-} omp_interop_backend_type_t;
-
-typedef enum omp_foreign_runtime_ids {
-  cuda = 1,
-  cuda_driver = 2,
-  opencl = 3,
-  sycl = 4,
-  hip = 5,
-  level_zero = 6,
-} omp_foreign_runtime_ids_t;
+/* Vendor defined values from OpenMP Additional Definitions document v2.1*/
+typedef enum omp_vendor_id {
+  omp_vendor_unknown = 0,
+  omp_vendor_amd = 1,
+  omp_vendor_arm = 2,
+  omp_vendor_bsc = 3,
+  omp_vendor_fujitsu = 4,
+  omp_vendor_gnu = 5,
+  omp_vendor_hpe = 6,
+  omp_vendor_ibm = 7,
+  omp_vendor_intel = 8,
+  omp_vendor_llvm = 9,
+  omp_vendor_nec = 10,
+  omp_vendor_nvidia = 11,
+  omp_vendor_ti = 12,
+  omp_vendor_last = 13
+} omp_vendor_id_t;
 
 ///} InteropAPI
 
diff --git a/offload/include/PerThreadTable.h b/offload/include/PerThreadTable.h
new file mode 100644
index 0000000000000..45b196171b4c8
--- /dev/null
+++ b/offload/include/PerThreadTable.h
@@ -0,0 +1,114 @@
+//===-- PerThreadTable.h -- PerThread Storage Structure ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Table indexed with one entry per thread.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PERTHREADTABLE_H
+#define OFFLOAD_PERTHREADTABLE_H
+
+#include <list>
+#include <memory>
+#include <mutex>
+
+// Using an STL container (such as std::vector) indexed by thread ID has
+// too many race conditions issues so we store each thread entry into a
+// thread_local variable.
+// T is the container type used to store the objects, e.g., std::vector,
+// std::set, etc. by each thread. O is the type of the stored objects e.g.,
+// omp_interop_val_t *, ...
+
+template <typename ContainerType, typename ObjectType> struct PerThreadTable {
+  using iterator = typename ContainerType::iterator;
+
+  struct PerThreadData {
+    size_t NElements = 0;
+    std::unique_ptr<ContainerType> ThEntry;
+  };
+
+  std::mutex Mtx;
+  std::list<std::shared_ptr<PerThreadData>> ThreadDataList;
+
+  // define default constructors, disable copy and move constructors
+  PerThreadTable() = default;
+  PerThreadTable(const PerThreadTable &) = delete;
+  PerThreadTable(PerThreadTable &&) = delete;
+  PerThreadTable &operator=(const PerThreadTable &) = delete;
+  PerThreadTable &operator=(PerThreadTable &&) = delete;
+  ~PerThreadTable() {
+    std::lock_guard<std::mutex> Lock(Mtx);
+    ThreadDataList.clear();
+  }
+
+private:
+  PerThreadData &getThreadData() {
+    static thread_local std::shared_ptr<PerThreadData> ThData = nullptr;
+    if (!ThData) {
+      ThData = std::make_shared<PerThreadData>();
+      std::lock_guard<std::mutex> Lock(Mtx);
+      ThreadDataList.push_back(ThData);
+    }
+    return *ThData;
+  }
+
+protected:
+  ContainerType &getThreadEntry() {
+    auto &ThData = getThreadData();
+    if (ThData.ThEntry)
+      return *ThData.ThEntry;
+    ThData.ThEntry = std::make_unique<ContainerType>();
+    return *ThData.ThEntry;
+  }
+
+  size_t &getThreadNElements() {
+    auto &ThData = getThreadData();
+    return ThData.NElements;
+  }
+
+public:
+  void add(ObjectType obj) {
+    auto &Entry = getThreadEntry();
+    auto &NElements = getThreadNElements();
+    NElements++;
+    Entry.add(obj);
+  }
+
+  iterator erase(iterator it) {
+    auto &Entry = getThreadEntry();
+    auto &NElements = getThreadNElements();
+    NElements--;
+    return Entry.erase(it);
+  }
+
+  size_t size() { return getThreadNElements(); }
+
+  // Iterators to traverse objects owned by
+  // the current thread
+  iterator begin() {
+    auto &Entry = getThreadEntry();
+    return Entry.begin();
+  }
+  iterator end() {
+    auto &Entry = getThreadEntry();
+    return Entry.end();
+  }
+
+  template <class F> void clear(F f) {
+    std::lock_guard<std::mutex> Lock(Mtx);
+    for (auto ThData : ThreadDataList) {
+      if (!ThData->ThEntry || ThData->NElements == 0)
+        continue;
+      ThData->ThEntry->clear(f);
+      ThData->NElements = 0;
+    }
+    ThreadDataList.clear();
+  }
+};
+
+#endif
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index ec3adadf0819b..6c6fdebe76dff 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -35,6 +35,8 @@
 #include <mutex>
 #include <string>
 
+#include "OpenMP/InteropAPI.h"
+
 using GenericPluginTy = llvm::omp::target::plugin::GenericPluginTy;
 
 /// Struct for the data required to handle plugins
@@ -88,6 +90,9 @@ struct PluginManager {
   HostPtrToTableMapTy HostPtrToTableMap;
   std::mutex TblMapMtx; ///< For HostPtrToTableMap
 
+  /// Table of cached implicit interop objects
+  InteropTblTy InteropTbl;
+
   // Work around for plugins that call dlopen on shared libraries that call
   // tgt_register_lib during their initialisation. Stash the pointers in a
   // vector until the plugins are all initialised and then register them.
@@ -185,5 +190,6 @@ void initRuntime();
 void deinitRuntime();
 
 extern PluginManager *PM;
-
+extern std::atomic<bool> RTLAlive; // Indicates if the RTL has been initialized
+extern std::atomic<int> RTLOngoingSyncs; // Counts ongoing external syncs
 #endif // OMPTARGET_PLUGIN_MANAGER_H
diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 978b53d5d69b9..8c150b6bfc2d4 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -21,6 +21,7 @@
 
 #include <cstddef>
 #include <cstdint>
+#include <mutex>
 
 extern "C" {
 
@@ -36,6 +37,7 @@ struct __tgt_device_image {
 struct __tgt_device_info {
   void *Context = nullptr;
   void *Device = nullptr;
+  void *Platform = nullptr;
 };
 
 /// This struct is a record of all the host code that may be offloaded to a
@@ -75,6 +77,9 @@ struct __tgt_async_info {
   /// should be freed after finalization.
   llvm::SmallVector<void *, 2> AssociatedAllocations;
 
+  /// Mutex to guard access to AssociatedAllocations and the Queue.
+  std::mutex Mutex;
+
   /// The kernel launch environment used to issue a kernel. Stored here to
   /// ensure it is a valid location while the transfer to the device is
   /// happening.
diff --git a/offload/include/device.h b/offload/include/device.h
index f4b10abbaa3fd..1e85bb1876c83 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -98,6 +98,10 @@ struct DeviceTy {
   int32_t dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
                        int64_t Size, AsyncInfoTy &AsyncInfo);
 
+  // Insert a data fence between previous data operations and the following
+  // operations if necessary for the device.
+  int32_t dataFence(AsyncInfoTy &AsyncInfo);
+
   /// Notify the plugin about a new mapping starting at the host address
   /// \p HstPtr and \p Size bytes.
   int32_t notifyDataMapped(void *HstPtr, int64_t Size);
diff --git a/offload/include/omptarget.h b/offload/include/omptarget.h
index 6971780c7bdb5..8fd722bb15022 100644
--- a/offload/include/omptarget.h
+++ b/offload/include/omptarget.h
@@ -33,9 +33,6 @@
 
 #define OFFLOAD_DEVICE_DEFAULT -1
 
-// Don't format out enums and structs.
-// clang-format off
-
 /// return flags of __tgt_target_XXX public APIs
 enum __tgt_target_return_t : int {
   /// successful offload executed on a target device
@@ -51,39 +48,42 @@ enum __tgt_target_return_t : int {
 /// Data attributes for each data reference used in an OpenMP target region.
 enum tgt_map_type {
   // No flags
-  OMP_TGT_MAPTYPE_NONE            = 0x000,
+  OMP_TGT_MAPTYPE_NONE = 0x000,
   // copy data from host to device
-  OMP_TGT_MAPTYPE_TO              = 0x001,
+  OMP_TGT_MAPTYPE_TO = 0x001,
   // copy data from device to host
-  OMP_TGT_MAPTYPE_FROM            = 0x002,
+  OMP_TGT_MAPTYPE_FROM = 0x002,
   // copy regardless of the reference count
-  OMP_TGT_MAPTYPE_ALWAYS          = 0x004,
+  OMP_TGT_MAPTYPE_ALWAYS = 0x004,
   // force unmapping of data
-  OMP_TGT_MAPTYPE_DELETE          = 0x008,
+  OMP_TGT_MAPTYPE_DELETE = 0x008,
   // map the pointer as well as the pointee
-  OMP_TGT_MAPTYPE_PTR_AND_OBJ     = 0x010,
+  OMP_TGT_MAPTYPE_PTR_AND_OBJ = 0x010,
   // pass device base address to kernel
-  OMP_TGT_MAPTYPE_TARGET_PARAM    = 0x020,
+  OMP_TGT_MAPTYPE_TARGET_PARAM = 0x020,
   // return base device address of mapped data
-  OMP_TGT_MAPTYPE_RETURN_PARAM    = 0x040,
+  OMP_TGT_MAPTYPE_RETURN_PARAM = 0x040,
   // private variable - not mapped
-  OMP_TGT_MAPTYPE_PRIVATE         = 0x080,
+  OMP_TGT_MAPTYPE_PRIVATE = 0x080,
   // copy by value - not mapped
-  OMP_TGT_MAPTYPE_LITERAL         = 0x100,
+  OMP_TGT_MAPTYPE_LITERAL = 0x100,
   // mapping is implicit
-  OMP_TGT_MAPTYPE_IMPLICIT        = 0x200,
+  OMP_TGT_MAPTYPE_IMPLICIT = 0x200,
   // copy data to device
-  OMP_TGT_MAPTYPE_CLOSE           = 0x400,
+  OMP_TGT_MAPTYPE_CLOSE = 0x400,
   // runtime error if not already allocated
-  OMP_TGT_MAPTYPE_PRESENT         = 0x1000,
+  OMP_TGT_MAPTYPE_PRESENT = 0x1000,
   // use a separate reference counter so that the data cannot be unmapped within
   // the structured region
   // This is an OpenMP extension for the sake of OpenACC support.
-  OMP_TGT_MAPTYPE_OMPX_HOLD       = 0x2000,
+  OMP_TGT_MAPTYPE_OMPX_HOLD = 0x2000,
+  // Attach pointer and pointee, after processing all other maps.
+  // Applicable to map-entering directives. Does not change ref-count.
+  OMP_TGT_MAPTYPE_ATTACH = 0x4000,
   // descriptor for non-contiguous target-update
-  OMP_TGT_MAPTYPE_NON_CONTIG      = 0x100000000000,
+  OMP_TGT_MAPTYPE_NON_CONTIG = 0x100000000000,
   // member of struct, member given by [16 MSBs] - 1
-  OMP_TGT_MAPTYPE_MEMBER_OF       = 0xffff000000000000
+  OMP_TGT_MAPTYPE_MEMBER_OF = 0xffff000000000000
 };
 
 /// Flags for offload entries.
@@ -105,9 +105,9 @@ enum TargetAllocTy : int32_t {
   TARGET_ALLOC_DEVICE_NON_BLOCKING,
 };
 
-inline KernelArgsTy CTorDTorKernelArgs = {1,       0,       nullptr,   nullptr,
-	     nullptr, nullptr, nullptr,   nullptr,
-	     0,      {0,0,0},       {1, 0, 0}, {1, 0, 0}, 0};
+inline KernelArgsTy CTorDTorKernelArgs = {
+    1,       0, nullptr,   nullptr,   nullptr,   nullptr, nullptr,
+    nullptr, 0, {0, 0, 0}, {1, 0, 0}, {1, 0, 0}, 0};
 
 struct DeviceTy;
 
diff --git a/offload/liboffload/API/APIDefs.td b/offload/liboffload/API/APIDefs.td
index 640932dcf8464..ea3896fc31035 100644
--- a/offload/liboffload/API/APIDefs.td
+++ b/offload/liboffload/API/APIDefs.td
@@ -31,6 +31,13 @@ class IsHandleType<string Type> {
                 !ne(!find(Type, "_handle_t", !sub(!size(Type), 9)), -1));
 }
 
+// Does the type end with '_cb_t'?
+class IsCallbackType<string Type> {
+  // size("_cb_t") == 5
+  bit ret = !if(!lt(!size(Type), 5), 0,
+                !ne(!find(Type, "_cb_t", !sub(!size(Type), 5)), -1));
+}
+
 // Does the type end with '*'?
 class IsPointerType<string Type> {
   bit ret = !ne(!find(Type, "*", !sub(!size(Type), 1)), -1);
@@ -58,6 +65,7 @@ class Param<string Type, string Name, string Desc, bits<3> Flags = 0> {
   TypeInfo type_info = TypeInfo<"", "">;
   bit IsHandle = IsHandleType<type>.ret;
   bit IsPointer = IsPointerType<type>.ret;
+  bit IsCallback = IsCallbackType<type>.ret;
 }
 
 // A parameter whose range is described by other parameters in the function.
@@ -81,7 +89,7 @@ class ShouldCheckHandle<Param P> {
 }
 
 class ShouldCheckPointer<Param P> {
-  bit ret = !and(P.IsPointer, !eq(!and(PARAM_OPTIONAL, P.flags), 0));
+  bit ret = !and(!or(P.IsPointer, P.IsCallback), !eq(!and(PARAM_OPTIONAL, P.flags), 0));
 }
 
 // For a list of returns that contains a specific return code, find and append
@@ -137,7 +145,6 @@ defvar DefaultReturns = [Return<PREFIX#"_RESULT_SUCCESS">,
                          Return<PREFIX#"_ERRC_DEVICE_LOST">];
 
 class APIObject {
-  string name;
   string desc;
 }
 
@@ -168,6 +175,10 @@ class Enum : APIObject {
   // all Etor values must be TaggedEtor records
   bit is_typed = 0;
 
+  // This refers to whether the enumerator is used to name bits of a bit field,
+  // where consecutive values are bit-shifted rather than incremented.
+  bit is_bit_field = 0;
+
   list<Etor> etors = [];
 }
 
diff --git a/offload/liboffload/API/Common.td b/offload/liboffload/API/Common.td
index 6eaf604c8ebba..ac27d85b6c964 100644
--- a/offload/liboffload/API/Common.td
+++ b/offload/liboffload/API/Common.td
@@ -10,77 +10,64 @@
 //
 //===----------------------------------------------------------------------===//
 
-def : Macro {
-  let name = "OL_VERSION_MAJOR";
+def OL_VERSION_MAJOR : Macro {
   let desc = "Major version of the Offload API";
   let value = "0";
 }
 
-def : Macro {
-  let name = "OL_VERSION_MINOR";
+def OL_VERSION_MINOR : Macro {
   let desc = "Minor version of the Offload API";
   let value = "0";
 }
 
-def : Macro {
-  let name = "OL_VERSION_PATCH";
+def OL_VERSION_PATCH : Macro {
   let desc = "Patch version of the Offload API";
   let value = "1";
 }
 
-def : Macro {
-  let name = "OL_APICALL";
+def OL_APICALL : Macro {
   let desc = "Calling convention for all API functions";
   let condition = "defined(_WIN32)";
   let value = "__cdecl";
   let alt_value = "";
 }
 
-def : Macro {
-  let name = "OL_APIEXPORT";
+def OL_APIEXPORT : Macro {
   let desc = "Microsoft-specific dllexport storage-class attribute";
   let condition = "defined(_WIN32)";
   let value = "__declspec(dllexport)";
   let alt_value = "";
 }
 
-def : Handle {
-  let name = "ol_platform_handle_t";
+def ol_platform_handle_t : Handle {
   let desc = "Handle of a platform instance";
 }
 
-def : Handle {
-  let name = "ol_device_handle_t";
+def ol_device_handle_t : Handle {
   let desc = "Handle of platform's device object";
 }
 
-def : Handle {
-  let name = "ol_context_handle_t";
+def ol_context_handle_t : Handle {
   let desc = "Handle of context object";
 }
 
-def : Handle {
-  let name = "ol_queue_handle_t";
+def ol_queue_handle_t : Handle {
   let desc = "Handle of queue object";
 }
 
-def : Handle {
-  let name = "ol_event_handle_t";
+def ol_event_handle_t : Handle {
   let desc = "Handle of event object";
 }
 
-def : Handle {
-  let name = "ol_program_handle_t";
+def ol_program_handle_t : Handle {
   let desc = "Handle of program object";
 }
 
-def : Handle {
-  let name = "ol_symbol_handle_t";
+def ol_symbol_handle_t : Handle {
   let desc = "Handle of an object in a device's memory for a specific program";
 }
 
-def ErrorCode : Enum {
-  let name = "ol_errc_t";
+def ol_errc_t : Enum {
   let desc = "Defines Return/Error codes";
   let etors =[
     Etor<"SUCCESS", "success">,
@@ -115,8 +102,7 @@ def ErrorCode : Enum {
   ];
 }
 
-def : Struct {
-  let name = "ol_error_struct_t";
+def ol_error_struct_t : Struct {
   let desc = "Details of the error condition returned by an API call";
   let members = [
     StructMember<"ol_errc_t", "Code", "The error code">,
@@ -124,20 +110,17 @@ def : Struct {
   ];
 }
 
-def : Typedef {
-  let name = "ol_result_t";
+def ol_result_t : Typedef {
   let desc = "Result type returned by all entry points.";
-  let value = "const ol_error_struct_t*";
+  let value = "const struct ol_error_struct_t*";
 }
 
-def : Macro {
-  let name = "OL_SUCCESS";
+def OL_SUCCESS : Macro {
   let desc = "Success condition";
   let value = "NULL";
 }
 
-def : Struct {
-  let name = "ol_code_location_t";
+def ol_code_location_t : Struct {
   let desc = "Code location information that can optionally be associated with an API call";
   let members = [
     StructMember<"const char*", "FunctionName", "Function name">,
@@ -147,8 +130,7 @@ def : Struct {
   ];
 }
 
-def : Struct {
-  let name = "ol_dimensions_t";
+def ol_dimensions_t : Struct {
   let desc = "A three element vector";
   let members = [
     StructMember<"uint32_t", "x", "X">,
@@ -157,8 +139,7 @@ def : Struct {
   ];
 }
 
-def : Function {
-  let name = "olInit";
+def olInit : Function {
   let desc = "Perform initialization of the Offload library and plugins";
   let details = [
     "This must be the first API call made by a user of the Offload library",
@@ -168,8 +149,7 @@ def : Function {
   let returns = [];
 }
 
-def : Function {
-  let name = "olShutDown";
+def olShutDown : Function {
   let desc = "Release the resources in use by Offload";
   let details = [
     "This decrements an internal reference count. When this reaches 0, all resources will be released",
diff --git a/offload/liboffload/API/Device.td b/offload/liboffload/API/Device.td
index 857c596124b27..d1baa28687fb2 100644
--- a/offload/liboffload/API/Device.td
+++ b/offload/liboffload/API/Device.td
@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-def : Enum {
-  let name = "ol_device_type_t";
+def ol_device_type_t : Enum {
   let desc = "Supported device types.";
   let etors =[
     Etor<"DEFAULT", "The default device type as preferred by the runtime">,
@@ -22,11 +21,10 @@ def : Enum {
   ];
 }
 
-def DeviceInfo : Enum {
-  let name = "ol_device_info_t";
+def ol_device_info_t : Enum {
   let desc = "Supported device info.";
   let is_typed = 1;
-  let etors =[
+  list<TaggedEtor> basic_etors =[
     TaggedEtor<"TYPE", "ol_device_type_t", "type of the device">,
     TaggedEtor<"PLATFORM", "ol_platform_handle_t", "the platform associated with the device">,
     TaggedEtor<"NAME", "char[]", "Device name">,
@@ -34,11 +32,40 @@ def DeviceInfo : Enum {
     TaggedEtor<"DRIVER_VERSION", "char[]", "Driver version">,
     TaggedEtor<"MAX_WORK_GROUP_SIZE", "uint32_t", "Maximum total work group size in work items">,
     TaggedEtor<"MAX_WORK_GROUP_SIZE_PER_DIMENSION", "ol_dimensions_t", "Maximum work group size in each dimension">,
+    TaggedEtor<"VENDOR_ID", "uint32_t", "A unique vendor device identifier assigned by PCI-SIG">,
+    TaggedEtor<"NUM_COMPUTE_UNITS", "uint32_t", "The number of parallel compute units available to the device">,
+    TaggedEtor<"MAX_CLOCK_FREQUENCY", "uint32_t", "The maximum configured clock frequency of this device in MHz">,
+    TaggedEtor<"MEMORY_CLOCK_RATE", "uint32_t", "Memory clock frequency in MHz">,
+    TaggedEtor<"ADDRESS_BITS", "uint32_t", "Number of bits used to represent an address in device memory">,
+    TaggedEtor<"MAX_MEM_ALLOC_SIZE", "uint64_t", "The maximum size of memory object allocation in bytes">,
+    TaggedEtor<"GLOBAL_MEM_SIZE", "uint64_t", "The size of global device memory in bytes">,
   ];
+  list<TaggedEtor> fp_configs = !foreach(type, ["Single", "Double", "Half"], TaggedEtor<type # "_FP_CONFIG", "ol_device_fp_capability_flags_t", type # " precision floating point capability">);
+  list<TaggedEtor> native_vec_widths = !foreach(type, ["char","short","int","long","float","double","half"], TaggedEtor<"NATIVE_VECTOR_WIDTH_" # type, "uint32_t", "Native vector width for " # type>);
+  let etors = !listconcat(basic_etors, fp_configs, native_vec_widths);
+}
+
+def ol_device_fp_capability_flag_t : Enum {
+  let desc = "Device floating-point capability flags";
+  let is_bit_field = 1;
+  let etors =[
+    Etor<"CORRECTLY_ROUNDED_DIVIDE_SQRT", "Support correctly rounded divide and sqrt">,
+    Etor<"ROUND_TO_NEAREST", "Support round to nearest">,
+    Etor<"ROUND_TO_ZERO", "Support round to zero">,
+    Etor<"ROUND_TO_INF", "Support round to infinity">,
+    Etor<"INF_NAN", "Support INF to NAN">,
+    Etor<"DENORM", "Support denorm">,
+    Etor<"FMA", "Support fused multiply-add">,
+    Etor<"SOFT_FLOAT", "Basic floating point operations implemented in software">,
+  ];
+}
+
+def ol_device_fp_capability_flags_t : Typedef {
+  let desc = "Device floating-point capability flags";
+  let value = "uint32_t";
 }
 
-def : FptrTypedef {
-  let name = "ol_device_iterate_cb_t";
+def ol_device_iterate_cb_t : FptrTypedef {
   let desc = "User-provided function to be used with `olIterateDevices`";
   let params = [
     Param<"ol_device_handle_t", "Device", "the device handle of the current iteration", PARAM_IN>,
@@ -47,8 +74,7 @@ def : FptrTypedef {
   let return = "bool";
 }
 
-def : Function {
-  let name = "olIterateDevices";
+def olIterateDevices : Function {
   let desc = "Iterates over all available devices, calling the callback for each device.";
   let details = [
     "If the user-provided callback returns `false`, the iteration is stopped."
@@ -62,8 +88,7 @@ def : Function {
   ];
 }
 
-def : Function {
-  let name = "olGetDeviceInfo";
+def olGetDeviceInfo : Function {
   let desc = "Queries the given property of the device.";
   let details = [];
   let params = [
@@ -86,8 +111,7 @@ def : Function {
   ];
 }
 
-def : Function {
-  let name = "olGetDeviceInfoSize";
+def olGetDeviceInfoSize : Function {
   let desc = "Returns the storage size of the given device query.";
   let details = [];
   let params = [
diff --git a/offload/liboffload/API/Event.td b/offload/liboffload/API/Event.td
index 9d217ae230384..075bf5bafaa64 100644
--- a/offload/liboffload/API/Event.td
+++ b/offload/liboffload/API/Event.td
@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-def : Function {
-    let name = "olCreateEvent";
+def olCreateEvent : Function {
     let desc = "Enqueue an event to `Queue` and return it.";
     let details = [
       "This event can be used with `olSyncEvent` and `olWaitEvents` and will be complete once all enqueued work prior to the `olCreateEvent` call is complete.",
@@ -23,8 +22,7 @@ def : Function {
     let returns = [];
 }
 
-def : Function {
-    let name = "olDestroyEvent";
+def olDestroyEvent : Function {
     let desc = "Destroy the event and free all underlying resources.";
     let details = [];
     let params = [
@@ -33,8 +31,7 @@ def : Function {
     let returns = [];
 }
 
-def : Function {
-    let name = "olSyncEvent";
+def olSyncEvent : Function {
     let desc = "Block the calling thread until the event is complete.";
     let details = [];
     let params = [
@@ -43,17 +40,16 @@ def : Function {
     let returns = [];
 }
 
-def : Enum {
-  let name = "ol_event_info_t";
+def ol_event_info_t : Enum {
   let desc = "Supported event info.";
   let is_typed = 1;
   let etors = [
-    TaggedEtor<"QUEUE", "ol_queue_handle_t", "The handle of the queue associated with the device.">
+    TaggedEtor<"QUEUE", "ol_queue_handle_t", "The handle of the queue associated with the device.">,
+    TaggedEtor<"IS_COMPLETE", "bool", "True if and only if the event is complete.">,
   ];
 }
 
-def : Function {
-  let name = "olGetEventInfo";
+def olGetEventInfo : Function {
   let desc = "Queries the given property of the event.";
   let details = [
     "`olGetEventInfoSize` can be used to query the storage size "
@@ -77,8 +73,7 @@ def : Function {
   ];
 }
 
-def : Function {
-  let name = "olGetEventInfoSize";
+def olGetEventInfoSize : Function {
   let desc = "Returns the storage size of the given event query.";
   let details = [];
   let params = [
diff --git a/offload/liboffload/API/Kernel.td b/offload/liboffload/API/Kernel.td
index 502fb36467dba..2f5692a19d712 100644
--- a/offload/liboffload/API/Kernel.td
+++ b/offload/liboffload/API/Kernel.td
@@ -6,12 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file contains Offload API definitions related to launching kernels
+// This file contains Offload API definitions related to kernels
 //
 //===----------------------------------------------------------------------===//
 
-def : Struct {
-    let name = "ol_kernel_launch_size_args_t";
+def ol_kernel_launch_size_args_t : Struct {
     let desc = "Size-related arguments for a kernel launch.";
     let members = [
         StructMember<"size_t", "Dimensions", "Number of work dimensions">,
@@ -21,8 +20,7 @@ def : Struct {
     ];
 }
 
-def : Function {
-    let name = "olLaunchKernel";
+def olLaunchKernel : Function {
     let desc = "Enqueue a kernel launch with the specified size and parameters.";
     let details = [
         "If a queue is not specified, kernel execution happens synchronously",
@@ -42,3 +40,20 @@ def : Function {
         Return<"OL_ERRC_SYMBOL_KIND", ["The provided symbol is not a kernel"]>,
     ];
 }
+
+def olCalculateOptimalOccupancy : Function {
+    let desc = "Given dynamic memory size, query the device for a workgroup size that will result in optimal occupancy.";
+    let details = [
+        "For most devices, this will be the largest workgroup size that will result in all work items fitting on the device at once.",
+    ];
+    let params = [
+        Param<"ol_device_handle_t", "Device", "device intended to run the kernel", PARAM_IN>,
+        Param<"ol_symbol_handle_t", "Kernel", "handle of the kernel", PARAM_IN>,
+        Param<"size_t", "SharedMemory", "dynamic shared memory required per work item in bytes", PARAM_IN>,
+        Param<"size_t*", "GroupSize", "optimal block size", PARAM_OUT>
+    ];
+    let returns = [
+        Return<"OL_ERRC_SYMBOL_KIND", ["The provided symbol is not a kernel"]>,
+        Return<"OL_ERRC_UNSUPPORTED", ["The backend cannot provide this information"]>,
+    ];
+}
diff --git a/offload/liboffload/API/Memory.td b/offload/liboffload/API/Memory.td
index 5f7158588bc77..cc98b672a26a9 100644
--- a/offload/liboffload/API/Memory.td
+++ b/offload/liboffload/API/Memory.td
@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-def : Enum {
-  let name = "ol_alloc_type_t";
+def ol_alloc_type_t : Enum {
   let desc = "Represents the type of allocation made with olMemAlloc.";
   let etors = [
     Etor<"HOST", "Host allocation">,
@@ -20,8 +19,7 @@ def : Enum {
   ];
 }
 
-def : Function {
-  let name = "olMemAlloc";
+def olMemAlloc : Function {
   let desc = "Creates a memory allocation on the specified device.";
   let params = [
     Param<"ol_device_handle_t", "Device", "handle of the device to allocate on", PARAM_IN>,
@@ -36,8 +34,7 @@ def : Function {
   ];
 }
 
-def : Function {
-  let name = "olMemFree";
+def olMemFree : Function {
   let desc = "Frees a memory allocation previously made by olMemAlloc.";
   let params = [
     Param<"void*", "Address", "address of the allocation to free", PARAM_IN>,
@@ -45,8 +42,7 @@ def : Function {
   let returns = [];
 }
 
-def : Function {
-    let name = "olMemcpy";
+def olMemcpy : Function {
     let desc = "Enqueue a memcpy operation.";
     let details = [
         "For host pointers, use the host device belonging to the OL_PLATFORM_BACKEND_HOST platform.",
@@ -63,3 +59,22 @@ def : Function {
     ];
     let returns = [];
 }
+
+def olMemFill : Function {
+  let desc = "Fill memory with copies of the given pattern";
+  let details = [
+    "Filling with patterns larger than 4 bytes may be less performant",
+    "The destination pointer and queue must be associated with the same device",
+    "The fill size must be a multiple of the pattern size",
+  ];
+  let params = [
+      Param<"ol_queue_handle_t", "Queue", "handle of the queue", PARAM_IN_OPTIONAL>,
+      Param<"void*", "Ptr", "destination pointer to start filling at", PARAM_IN>,
+      Param<"size_t", "PatternSize", "the size of the pattern in bytes", PARAM_IN>,
+      Param<"const void*", "PatternPtr", "", PARAM_IN>,
+      Param<"size_t", "FillSize", "number of bytes to fill", PARAM_IN>,
+  ];
+  let returns = [
+    Return<"OL_ERRC_INVALID_SIZE", ["`FillSize % PatternSize != 0`"]>
+  ];
+}
diff --git a/offload/liboffload/API/Platform.td b/offload/liboffload/API/Platform.td
index 97c2cc2d05704..906f899076a80 100644
--- a/offload/liboffload/API/Platform.td
+++ b/offload/liboffload/API/Platform.td
@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-def : Enum {
-  let name = "ol_platform_info_t";
+def ol_platform_info_t : Enum {
   let desc = "Supported platform info.";
   let is_typed = 1;
   let etors = [
@@ -22,8 +21,7 @@ def : Enum {
   ];
 }
 
-def : Enum {
-  let name = "ol_platform_backend_t";
+def ol_platform_backend_t : Enum {
   let desc = "Identifies the native backend of the platform.";
   let etors =[
     Etor<"UNKNOWN", "The backend is not recognized">,
@@ -33,8 +31,7 @@ def : Enum {
   ];
 }
 
-def : Function {
-  let name = "olGetPlatformInfo";
+def olGetPlatformInfo : Function {
   let desc = "Queries the given property of the platform.";
   let details = [
     "`olGetPlatformInfoSize` can be used to query the storage size "
@@ -61,8 +58,7 @@ def : Function {
   ];
 }
 
-def : Function {
-  let name = "olGetPlatformInfoSize";
+def olGetPlatformInfoSize : Function {
   let desc = "Returns the storage size of the given platform query.";
   let details = [];
   let params = [
diff --git a/offload/liboffload/API/Program.td b/offload/liboffload/API/Program.td
index 0476fa1f7c27a..1f48f650cab70 100644
--- a/offload/liboffload/API/Program.td
+++ b/offload/liboffload/API/Program.td
@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-def : Function {
-    let name = "olCreateProgram";
+def olCreateProgram : Function {
     let desc = "Create a program for the device from the binary image pointed to by `ProgData`.";
     let details = [
         "The provided `ProgData` will be copied and need not outlive the returned handle",
@@ -25,8 +24,7 @@ def : Function {
     let returns = [];
 }
 
-def : Function {
-    let name = "olDestroyProgram";
+def olDestroyProgram : Function {
     let desc = "Destroy the program and free all underlying resources.";
     let details = [];
     let params = [
diff --git a/offload/liboffload/API/Queue.td b/offload/liboffload/API/Queue.td
index 6e86b2445c73d..ededa9cc92fef 100644
--- a/offload/liboffload/API/Queue.td
+++ b/offload/liboffload/API/Queue.td
@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-def : Function {
-    let name = "olCreateQueue";
+def olCreateQueue : Function {
     let desc = "Create a queue for the given device.";
     let details = [];
     let params = [
@@ -21,18 +20,18 @@ def : Function {
     let returns = [];
 }
 
-def : Function {
-    let name = "olDestroyQueue";
+def olDestroyQueue : Function {
     let desc = "Destroy the queue and free all underlying resources.";
-    let details = [];
+    let details = [
+      "Any work previously enqueued to the queue is still performed and any events generated for this queue remain valid."
+    ];
     let params = [
         Param<"ol_queue_handle_t", "Queue", "handle of the queue", PARAM_IN>
     ];
     let returns = [];
 }
 
-def : Function {
-    let name = "olSyncQueue";
+def olSyncQueue : Function {
     let desc = "Block the calling thread until the enqueued work on a queue is complete.";
     let details = [];
     let params = [
@@ -41,8 +40,7 @@ def : Function {
     let returns = [];
 }
 
-def : Function {
-    let name = "olWaitEvents";
+def olWaitEvents : Function {
     let desc = "Make any future work submitted to this queue wait until the provided events are complete.";
     let details = [
       "All events in `Events` must complete before the queue is unblocked.",
@@ -58,17 +56,16 @@ def : Function {
     ];
 }
 
-def : Enum {
-  let name = "ol_queue_info_t";
+def ol_queue_info_t : Enum {
   let desc = "Supported queue info.";
   let is_typed = 1;
   let etors = [
-    TaggedEtor<"DEVICE", "ol_device_handle_t", "The handle of the device associated with the queue.">
+    TaggedEtor<"DEVICE", "ol_device_handle_t", "The handle of the device associated with the queue.">,
+    TaggedEtor<"EMPTY", "bool", "True if the queue is known to be empty. May be unconditionally false if the device does not support status queries.">,
   ];
 }
 
-def : Function {
-  let name = "olGetQueueInfo";
+def olGetQueueInfo : Function {
   let desc = "Queries the given property of the queue.";
   let details = [
     "`olGetQueueInfoSize` can be used to query the storage size "
@@ -92,8 +89,7 @@ def : Function {
   ];
 }
 
-def : Function {
-  let name = "olGetQueueInfoSize";
+def olGetQueueInfoSize : Function {
   let desc = "Returns the storage size of the given queue query.";
   let details = [];
   let params = [
@@ -105,3 +101,27 @@ def : Function {
     Return<"OL_ERRC_INVALID_QUEUE">
   ];
 }
+
+def ol_host_function_cb_t : FptrTypedef {
+  let desc = "Host function for use by `olLaunchHostFunction`.";
+  let params = [
+    Param<"void *", "UserData", "user specified data passed into `olLaunchHostFunction`.", PARAM_IN>,
+  ];
+  let return = "void";
+}
+
+def olLaunchHostFunction : Function {
+  let desc = "Enqueue a callback function on the host.";
+  let details = [
+    "The provided function will be called from the same process as the one that called `olLaunchHostFunction`.",
+    "The callback will not run until all previous work submitted to the queue has completed.",
+    "The callback must return before any work submitted to the queue after it is started.",
+    "The callback must not call any liboffload API functions or any backend specific functions (such as Cuda or HSA library functions).",
+  ];
+  let params = [
+    Param<"ol_queue_handle_t", "Queue", "handle of the queue", PARAM_IN>,
+    Param<"ol_host_function_cb_t", "Callback", "the callback function to call on the host", PARAM_IN>,
+    Param<"void *", "UserData", "a pointer that will be passed verbatim to the callback function", PARAM_IN_OPTIONAL>,
+  ];
+  let returns = [];
+}
diff --git a/offload/liboffload/API/Symbol.td b/offload/liboffload/API/Symbol.td
index 2e94d703809e7..c57a2e1b83632 100644
--- a/offload/liboffload/API/Symbol.td
+++ b/offload/liboffload/API/Symbol.td
@@ -10,8 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-def : Enum {
-  let name = "ol_symbol_kind_t";
+def ol_symbol_kind_t : Enum {
   let desc = "The kind of a symbol";
   let etors =[
     Etor<"KERNEL", "a kernel object">,
@@ -19,8 +18,7 @@ def : Enum {
   ];
 }
 
-def : Function {
-    let name = "olGetSymbol";
+def olGetSymbol : Function {
     let desc = "Get a symbol (kernel or global variable) identified by `Name` in the given program.";
     let details = [
         "Symbol handles are owned by the program and do not need to be manually destroyed."
@@ -34,8 +32,7 @@ def : Function {
     let returns = [];
 }
 
-def : Enum {
-  let name = "ol_symbol_info_t";
+def ol_symbol_info_t : Enum {
   let desc = "Supported symbol info.";
   let is_typed = 1;
   let etors = [
@@ -45,8 +42,7 @@ def : Enum {
   ];
 }
 
-def : Function {
-  let name = "olGetSymbolInfo";
+def olGetSymbolInfo : Function {
   let desc = "Queries the given property of the symbol.";
   let details = [
     "`olGetSymbolInfoSize` can be used to query the storage size "
@@ -73,8 +69,7 @@ def : Function {
   ];
 }
 
-def : Function {
-  let name = "olGetSymbolInfoSize";
+def olGetSymbolInfoSize : Function {
   let desc = "Returns the storage size of the given symbol query.";
   let details = [];
   let params = [
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 6486b2b6d13a6..9d342e06127a2 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -72,6 +72,8 @@ struct ol_queue_impl_t {
 struct ol_event_impl_t {
   ol_event_impl_t(void *EventInfo, ol_queue_handle_t Queue)
       : EventInfo(EventInfo), Queue(Queue) {}
+  // EventInfo may be null, in which case the event should be considered always
+  // complete
   void *EventInfo;
   ol_queue_handle_t Queue;
 };
@@ -123,6 +125,7 @@ struct OffloadContext {
   bool TracingEnabled = false;
   bool ValidationEnabled = true;
   DenseMap<void *, AllocInfo> AllocInfoMap{};
+  std::mutex AllocInfoMapMutex{};
   SmallVector<ol_platform_impl_t, 4> Platforms{};
   size_t RefCount;
 
@@ -206,7 +209,7 @@ Error initPlugins(OffloadContext &Context) {
 }
 
 Error olInit_impl() {
-  std::lock_guard<std::mutex> Lock{OffloadContextValMutex};
+  std::lock_guard<std::mutex> Lock(OffloadContextValMutex);
 
   if (isOffloadInitialized()) {
     OffloadContext::get().RefCount++;
@@ -224,7 +227,7 @@ Error olInit_impl() {
 }
 
 Error olShutDown_impl() {
-  std::lock_guard<std::mutex> Lock{OffloadContextValMutex};
+  std::lock_guard<std::mutex> Lock(OffloadContextValMutex);
 
   if (--OffloadContext::get().RefCount != 0)
     return Error::success();
@@ -300,10 +303,57 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
   };
 
   // These are not implemented by the plugin interface
-  if (PropName == OL_DEVICE_INFO_PLATFORM)
+  switch (PropName) {
+  case OL_DEVICE_INFO_PLATFORM:
     return Info.write<void *>(Device->Platform);
-  if (PropName == OL_DEVICE_INFO_TYPE)
+
+  case OL_DEVICE_INFO_TYPE:
     return Info.write<ol_device_type_t>(OL_DEVICE_TYPE_GPU);
+
+  case OL_DEVICE_INFO_SINGLE_FP_CONFIG:
+  case OL_DEVICE_INFO_DOUBLE_FP_CONFIG: {
+    ol_device_fp_capability_flags_t flags{0};
+    flags |= OL_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT |
+             OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST |
+             OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO |
+             OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF |
+             OL_DEVICE_FP_CAPABILITY_FLAG_INF_NAN |
+             OL_DEVICE_FP_CAPABILITY_FLAG_DENORM |
+             OL_DEVICE_FP_CAPABILITY_FLAG_FMA;
+    return Info.write(flags);
+  }
+
+  case OL_DEVICE_INFO_HALF_FP_CONFIG:
+    return Info.write<ol_device_fp_capability_flags_t>(0);
+
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE:
+    return Info.write<uint32_t>(1);
+
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF:
+    return Info.write<uint32_t>(0);
+
+  // None of the existing plugins specify a limit on a single allocation,
+  // so return the global memory size instead
+  case OL_DEVICE_INFO_MAX_MEM_ALLOC_SIZE:
+    [[fallthrough]];
+  // AMD doesn't provide the global memory size (trivially) with the device info
+  // struct, so use the plugin interface
+  case OL_DEVICE_INFO_GLOBAL_MEM_SIZE: {
+    uint64_t Mem;
+    if (auto Err = Device->Device->getDeviceMemorySize(Mem))
+      return Err;
+    return Info.write<uint64_t>(Mem);
+  } break;
+
+  default:
+    break;
+  }
+
   if (PropName >= OL_DEVICE_INFO_LAST)
     return createOffloadError(ErrorCode::INVALID_ENUMERATION,
                               "getDeviceInfo enum '%i' is invalid", PropName);
@@ -314,6 +364,7 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
                      "plugin did not provide a response for this information");
   auto Entry = *EntryOpt;
 
+  // Retrieve properties from the plugin interface
   switch (PropName) {
   case OL_DEVICE_INFO_NAME:
   case OL_DEVICE_INFO_VENDOR:
@@ -325,7 +376,12 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
     return Info.writeString(std::get<std::string>(Entry->Value).c_str());
   }
 
-  case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE: {
+  case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE:
+  case OL_DEVICE_INFO_VENDOR_ID:
+  case OL_DEVICE_INFO_NUM_COMPUTE_UNITS:
+  case OL_DEVICE_INFO_ADDRESS_BITS:
+  case OL_DEVICE_INFO_MAX_CLOCK_FREQUENCY:
+  case OL_DEVICE_INFO_MEMORY_CLOCK_RATE: {
     // Uint32 values
     if (!std::holds_alternative<uint64_t>(Entry->Value))
       return makeError(ErrorCode::BACKEND_FAILURE,
@@ -387,9 +443,40 @@ Error olGetDeviceInfoImplDetailHost(ol_device_handle_t Device,
   case OL_DEVICE_INFO_DRIVER_VERSION:
     return Info.writeString(LLVM_VERSION_STRING);
   case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE:
-    return Info.write<uint64_t>(1);
+    return Info.write<uint32_t>(1);
   case OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE_PER_DIMENSION:
     return Info.write<ol_dimensions_t>(ol_dimensions_t{1, 1, 1});
+  case OL_DEVICE_INFO_VENDOR_ID:
+    return Info.write<uint32_t>(0);
+  case OL_DEVICE_INFO_NUM_COMPUTE_UNITS:
+    return Info.write<uint32_t>(1);
+  case OL_DEVICE_INFO_SINGLE_FP_CONFIG:
+  case OL_DEVICE_INFO_DOUBLE_FP_CONFIG:
+    return Info.write<ol_device_fp_capability_flags_t>(
+        OL_DEVICE_FP_CAPABILITY_FLAG_CORRECTLY_ROUNDED_DIVIDE_SQRT |
+        OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_NEAREST |
+        OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_ZERO |
+        OL_DEVICE_FP_CAPABILITY_FLAG_ROUND_TO_INF |
+        OL_DEVICE_FP_CAPABILITY_FLAG_INF_NAN |
+        OL_DEVICE_FP_CAPABILITY_FLAG_DENORM | OL_DEVICE_FP_CAPABILITY_FLAG_FMA);
+  case OL_DEVICE_INFO_HALF_FP_CONFIG:
+    return Info.write<ol_device_fp_capability_flags_t>(0);
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT:
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE:
+    return Info.write<uint32_t>(1);
+  case OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF:
+    return Info.write<uint32_t>(0);
+  case OL_DEVICE_INFO_MAX_CLOCK_FREQUENCY:
+  case OL_DEVICE_INFO_MEMORY_CLOCK_RATE:
+  case OL_DEVICE_INFO_ADDRESS_BITS:
+    return Info.write<uint32_t>(std::numeric_limits<uintptr_t>::digits);
+  case OL_DEVICE_INFO_MAX_MEM_ALLOC_SIZE:
+  case OL_DEVICE_INFO_GLOBAL_MEM_SIZE:
+    return Info.write<uint64_t>(0);
   default:
     return createOffloadError(ErrorCode::INVALID_ENUMERATION,
                               "getDeviceInfo enum '%i' is invalid", PropName);
@@ -447,26 +534,33 @@ Error olMemAlloc_impl(ol_device_handle_t Device, ol_alloc_type_t Type,
     return Alloc.takeError();
 
   *AllocationOut = *Alloc;
-  OffloadContext::get().AllocInfoMap.insert_or_assign(*Alloc,
-                                                      AllocInfo{Device, Type});
+  {
+    std::lock_guard<std::mutex> Lock(OffloadContext::get().AllocInfoMapMutex);
+    OffloadContext::get().AllocInfoMap.insert_or_assign(
+        *Alloc, AllocInfo{Device, Type});
+  }
   return Error::success();
 }
 
 Error olMemFree_impl(void *Address) {
-  if (!OffloadContext::get().AllocInfoMap.contains(Address))
-    return createOffloadError(ErrorCode::INVALID_ARGUMENT,
-                              "address is not a known allocation");
-
-  auto AllocInfo = OffloadContext::get().AllocInfoMap.at(Address);
-  auto Device = AllocInfo.Device;
-  auto Type = AllocInfo.Type;
+  ol_device_handle_t Device;
+  ol_alloc_type_t Type;
+  {
+    std::lock_guard<std::mutex> Lock(OffloadContext::get().AllocInfoMapMutex);
+    if (!OffloadContext::get().AllocInfoMap.contains(Address))
+      return createOffloadError(ErrorCode::INVALID_ARGUMENT,
+                                "address is not a known allocation");
+
+    auto AllocInfo = OffloadContext::get().AllocInfoMap.at(Address);
+    Device = AllocInfo.Device;
+    Type = AllocInfo.Type;
+    OffloadContext::get().AllocInfoMap.erase(Address);
+  }
 
   if (auto Res =
           Device->Device->dataDelete(Address, convertOlToPluginAllocTy(Type)))
     return Res;
 
-  OffloadContext::get().AllocInfoMap.erase(Address);
-
   return Error::success();
 }
 
@@ -485,16 +579,13 @@ Error olSyncQueue_impl(ol_queue_handle_t Queue) {
   // Host plugin doesn't have a queue set so it's not safe to call synchronize
   // on it, but we have nothing to synchronize in that situation anyway.
   if (Queue->AsyncInfo->Queue) {
-    if (auto Err = Queue->Device->Device->synchronize(Queue->AsyncInfo))
+    // We don't need to release the queue and we would like the ability for
+    // other offload threads to submit work concurrently, so pass "false" here
+    // so we don't release the underlying queue object.
+    if (auto Err = Queue->Device->Device->synchronize(Queue->AsyncInfo, false))
       return Err;
   }
 
-  // Recreate the stream resource so the queue can be reused
-  // TODO: Would be easier for the synchronization to (optionally) not release
-  // it to begin with.
-  if (auto Res = Queue->Device->Device->initAsyncInfo(&Queue->AsyncInfo))
-    return Res;
-
   return Error::success();
 }
 
@@ -509,8 +600,8 @@ Error olWaitEvents_impl(ol_queue_handle_t Queue, ol_event_handle_t *Events,
       return Plugin::error(ErrorCode::INVALID_NULL_HANDLE,
                            "olWaitEvents asked to wait on a NULL event");
 
-    // Do nothing if the event is for this queue
-    if (Event->Queue == Queue)
+    // Do nothing if the event is for this queue or the event is always complete
+    if (Event->Queue == Queue || !Event->EventInfo)
       continue;
 
     if (auto Err = Device->waitEvent(Event->EventInfo, Queue->AsyncInfo))
@@ -528,6 +619,12 @@ Error olGetQueueInfoImplDetail(ol_queue_handle_t Queue,
   switch (PropName) {
   case OL_QUEUE_INFO_DEVICE:
     return Info.write<ol_device_handle_t>(Queue->Device);
+  case OL_QUEUE_INFO_EMPTY: {
+    auto Pending = Queue->Device->Device->hasPendingWork(Queue->AsyncInfo);
+    if (auto Err = Pending.takeError())
+      return Err;
+    return Info.write<bool>(!*Pending);
+  }
   default:
     return createOffloadError(ErrorCode::INVALID_ENUMERATION,
                               "olGetQueueInfo enum '%i' is invalid", PropName);
@@ -548,6 +645,10 @@ Error olGetQueueInfoSize_impl(ol_queue_handle_t Queue, ol_queue_info_t PropName,
 }
 
 Error olSyncEvent_impl(ol_event_handle_t Event) {
+  // No event info means that this event was complete on creation
+  if (!Event->EventInfo)
+    return Plugin::success();
+
   if (auto Res = Event->Queue->Device->Device->syncEvent(Event->EventInfo))
     return Res;
 
@@ -555,8 +656,9 @@ Error olSyncEvent_impl(ol_event_handle_t Event) {
 }
 
 Error olDestroyEvent_impl(ol_event_handle_t Event) {
-  if (auto Res = Event->Queue->Device->Device->destroyEvent(Event->EventInfo))
-    return Res;
+  if (Event->EventInfo)
+    if (auto Res = Event->Queue->Device->Device->destroyEvent(Event->EventInfo))
+      return Res;
 
   return olDestroy(Event);
 }
@@ -565,10 +667,22 @@ Error olGetEventInfoImplDetail(ol_event_handle_t Event,
                                ol_event_info_t PropName, size_t PropSize,
                                void *PropValue, size_t *PropSizeRet) {
   InfoWriter Info(PropSize, PropValue, PropSizeRet);
+  auto Queue = Event->Queue;
 
   switch (PropName) {
   case OL_EVENT_INFO_QUEUE:
-    return Info.write<ol_queue_handle_t>(Event->Queue);
+    return Info.write<ol_queue_handle_t>(Queue);
+  case OL_EVENT_INFO_IS_COMPLETE: {
+    // No event info means that this event was complete on creation
+    if (!Event->EventInfo)
+      return Info.write<bool>(true);
+
+    auto Res = Queue->Device->Device->isEventComplete(Event->EventInfo,
+                                                      Queue->AsyncInfo);
+    if (auto Err = Res.takeError())
+      return Err;
+    return Info.write<bool>(*Res);
+  }
   default:
     return createOffloadError(ErrorCode::INVALID_ENUMERATION,
                               "olGetEventInfo enum '%i' is invalid", PropName);
@@ -590,7 +704,16 @@ Error olGetEventInfoSize_impl(ol_event_handle_t Event, ol_event_info_t PropName,
 }
 
 Error olCreateEvent_impl(ol_queue_handle_t Queue, ol_event_handle_t *EventOut) {
+  auto Pending = Queue->Device->Device->hasPendingWork(Queue->AsyncInfo);
+  if (auto Err = Pending.takeError())
+    return Err;
+
   *EventOut = new ol_event_impl_t(nullptr, Queue);
+  if (!*Pending)
+    // Queue is empty, don't record an event and consider the event always
+    // complete
+    return Plugin::success();
+
   if (auto Res = Queue->Device->Device->createEvent(&(*EventOut)->EventInfo))
     return Res;
 
@@ -637,6 +760,12 @@ Error olMemcpy_impl(ol_queue_handle_t Queue, void *DstPtr,
   return Error::success();
 }
 
+Error olMemFill_impl(ol_queue_handle_t Queue, void *Ptr, size_t PatternSize,
+                     const void *PatternPtr, size_t FillSize) {
+  return Queue->Device->Device->dataFill(Ptr, PatternPtr, PatternSize, FillSize,
+                                         Queue->AsyncInfo);
+}
+
 Error olCreateProgram_impl(ol_device_handle_t Device, const void *ProgData,
                            size_t ProgDataSize, ol_program_handle_t *Program) {
   // Make a copy of the program binary in case it is released by the caller.
@@ -677,6 +806,24 @@ Error olDestroyProgram_impl(ol_program_handle_t Program) {
   return olDestroy(Program);
 }
 
+Error olCalculateOptimalOccupancy_impl(ol_device_handle_t Device,
+                                       ol_symbol_handle_t Kernel,
+                                       size_t DynamicMemSize,
+                                       size_t *GroupSize) {
+  if (Kernel->Kind != OL_SYMBOL_KIND_KERNEL)
+    return createOffloadError(ErrorCode::SYMBOL_KIND,
+                              "provided symbol is not a kernel");
+  auto *KernelImpl = std::get<GenericKernelTy *>(Kernel->PluginImpl);
+
+  auto Res = KernelImpl->maxGroupSize(*Device->Device, DynamicMemSize);
+  if (auto Err = Res.takeError())
+    return Err;
+
+  *GroupSize = *Res;
+
+  return Error::success();
+}
+
 Error olLaunchKernel_impl(ol_queue_handle_t Queue, ol_device_handle_t Device,
                           ol_symbol_handle_t Kernel, const void *ArgumentsData,
                           size_t ArgumentsSize,
@@ -725,7 +872,7 @@ Error olGetSymbol_impl(ol_program_handle_t Program, const char *Name,
                        ol_symbol_kind_t Kind, ol_symbol_handle_t *Symbol) {
   auto &Device = Program->Image->getDevice();
 
-  std::lock_guard<std::mutex> Lock{Program->SymbolListMutex};
+  std::lock_guard<std::mutex> Lock(Program->SymbolListMutex);
 
   switch (Kind) {
   case OL_SYMBOL_KIND_KERNEL: {
@@ -746,7 +893,7 @@ Error olGetSymbol_impl(ol_program_handle_t Program, const char *Name,
     return Error::success();
   }
   case OL_SYMBOL_KIND_GLOBAL_VARIABLE: {
-    auto &Global = Program->KernelSymbols[Name];
+    auto &Global = Program->GlobalSymbols[Name];
     if (!Global) {
       GlobalTy GlobalObj{Name};
       if (auto Res =
@@ -814,5 +961,12 @@ Error olGetSymbolInfoSize_impl(ol_symbol_handle_t Symbol,
   return olGetSymbolInfoImplDetail(Symbol, PropName, 0, nullptr, PropSizeRet);
 }
 
+Error olLaunchHostFunction_impl(ol_queue_handle_t Queue,
+                                ol_host_function_cb_t Callback,
+                                void *UserData) {
+  return Queue->Device->Device->enqueueHostCall(Callback, UserData,
+                                                Queue->AsyncInfo);
+}
+
 } // namespace offload
 } // namespace llvm
diff --git a/offload/libomptarget/OffloadRTL.cpp b/offload/libomptarget/OffloadRTL.cpp
index 29b573a27d087..04bd21ec91a49 100644
--- a/offload/libomptarget/OffloadRTL.cpp
+++ b/offload/libomptarget/OffloadRTL.cpp
@@ -22,6 +22,8 @@ extern void llvm::omp::target::ompt::connectLibrary();
 
 static std::mutex PluginMtx;
 static uint32_t RefCount = 0;
+std::atomic<bool> RTLAlive{false};
+std::atomic<int> RTLOngoingSyncs{0};
 
 void initRuntime() {
   std::scoped_lock<decltype(PluginMtx)> Lock(PluginMtx);
@@ -41,6 +43,9 @@ void initRuntime() {
 
     PM->init();
     PM->registerDelayedLibraries();
+
+    // RTL initialization is complete
+    RTLAlive = true;
   }
 }
 
@@ -50,6 +55,13 @@ void deinitRuntime() {
 
   if (RefCount == 1) {
     DP("Deinit offload library!\n");
+    // RTL deinitialization has started
+    RTLAlive = false;
+    while (RTLOngoingSyncs > 0) {
+      DP("Waiting for ongoing syncs to finish, count: %d\n",
+         RTLOngoingSyncs.load());
+      std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    }
     PM->deinit();
     delete PM;
     PM = nullptr;
diff --git a/offload/libomptarget/OpenMP/API.cpp b/offload/libomptarget/OpenMP/API.cpp
index 4576f9bd06121..b0f0573833713 100644
--- a/offload/libomptarget/OpenMP/API.cpp
+++ b/offload/libomptarget/OpenMP/API.cpp
@@ -16,6 +16,7 @@
 #include "rtl.h"
 
 #include "OpenMP/InternalTypes.h"
+#include "OpenMP/InteropAPI.h"
 #include "OpenMP/Mapping.h"
 #include "OpenMP/OMPT/Interface.h"
 #include "OpenMP/omp.h"
@@ -683,3 +684,21 @@ EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {
 
   return TPR.TargetPointer;
 }
+
+// This routine gets called from the Host RTL at sync points (taskwait, barrier,
+// ...) so we can synchronize the necessary objects from the offload side.
+EXTERN void __tgt_target_sync(ident_t *loc_ref, int gtid, void *current_task,
+                              void *event) {
+  if (!RTLAlive)
+    return;
+
+  RTLOngoingSyncs++;
+  if (!RTLAlive) {
+    RTLOngoingSyncs--;
+    return;
+  }
+
+  syncImplicitInterops(gtid, event);
+
+  RTLOngoingSyncs--;
+}
diff --git a/offload/libomptarget/OpenMP/InteropAPI.cpp b/offload/libomptarget/OpenMP/InteropAPI.cpp
index bdbc440c64a2c..eb5425ecbf062 100644
--- a/offload/libomptarget/OpenMP/InteropAPI.cpp
+++ b/offload/libomptarget/OpenMP/InteropAPI.cpp
@@ -10,6 +10,7 @@
 #include "OpenMP/InternalTypes.h"
 #include "OpenMP/omp.h"
 
+#include "OffloadPolicy.h"
 #include "PluginManager.h"
 #include "device.h"
 #include "omptarget.h"
@@ -56,22 +57,22 @@ void getTypeMismatch(omp_interop_property_t Property, int *Err) {
     *Err = getPropertyErrorType(Property);
 }
 
-const char *getVendorIdToStr(const omp_foreign_runtime_ids_t VendorId) {
-  switch (VendorId) {
-  case cuda:
-    return ("cuda");
-  case cuda_driver:
-    return ("cuda_driver");
-  case opencl:
-    return ("opencl");
-  case sycl:
-    return ("sycl");
-  case hip:
-    return ("hip");
-  case level_zero:
-    return ("level_zero");
-  }
-  return ("unknown");
+static const char *VendorStrTbl[] = {
+    "unknown", "amd",   "arm",  "bsc", "fujitsu", "gnu", "hpe",
+    "ibm",     "intel", "llvm", "nec", "nvidia",  "ti"};
+const char *getVendorIdToStr(const omp_vendor_id_t VendorId) {
+  if (VendorId < omp_vendor_unknown || VendorId >= omp_vendor_last)
+    return ("unknown");
+  return VendorStrTbl[VendorId];
+}
+
+static const char *ForeignRuntimeStrTbl[] = {
+    "none", "cuda", "cuda_driver", "opencl",
+    "sycl", "hip",  "level_zero",  "hsa"};
+const char *getForeignRuntimeIdToStr(const tgt_foreign_runtime_id_t FrId) {
+  if (FrId < tgt_fr_none || FrId >= tgt_fr_last)
+    return ("unknown");
+  return ForeignRuntimeStrTbl[FrId];
 }
 
 template <typename PropertyTy>
@@ -83,7 +84,7 @@ intptr_t getProperty<intptr_t>(omp_interop_val_t &InteropVal,
                                omp_interop_property_t Property, int *Err) {
   switch (Property) {
   case omp_ipr_fr_id:
-    return InteropVal.backend_type_id;
+    return InteropVal.fr_id;
   case omp_ipr_vendor:
     return InteropVal.vendor_id;
   case omp_ipr_device_num:
@@ -99,10 +100,8 @@ const char *getProperty<const char *>(omp_interop_val_t &InteropVal,
                                       omp_interop_property_t Property,
                                       int *Err) {
   switch (Property) {
-  case omp_ipr_fr_id:
-    return InteropVal.interop_type == kmp_interop_type_tasksync
-               ? "tasksync"
-               : "device+context";
+  case omp_ipr_fr_name:
+    return getForeignRuntimeIdToStr(InteropVal.fr_id);
   case omp_ipr_vendor_name:
     return getVendorIdToStr(InteropVal.vendor_id);
   default:
@@ -120,6 +119,8 @@ void *getProperty<void *>(omp_interop_val_t &InteropVal,
       return InteropVal.device_info.Device;
     *Err = omp_irc_no_value;
     return const_cast<char *>(InteropVal.err_str);
+  case omp_ipr_platform:
+    return InteropVal.device_info.Platform;
   case omp_ipr_device_context:
     return InteropVal.device_info.Context;
   case omp_ipr_targetsync:
@@ -145,13 +146,13 @@ bool getPropertyCheck(omp_interop_val_t **InteropPtr,
     return false;
   }
   if (Property == omp_ipr_targetsync &&
-      (*InteropPtr)->interop_type != kmp_interop_type_tasksync) {
+      (*InteropPtr)->interop_type != kmp_interop_type_targetsync) {
     if (Err)
       *Err = omp_irc_other;
     return false;
   }
   if ((Property == omp_ipr_device || Property == omp_ipr_device_context) &&
-      (*InteropPtr)->interop_type == kmp_interop_type_tasksync) {
+      (*InteropPtr)->interop_type == kmp_interop_type_targetsync) {
     if (Err)
       *Err = omp_irc_other;
     return false;
@@ -166,7 +167,7 @@ bool getPropertyCheck(omp_interop_val_t **InteropPtr,
                                        omp_interop_property_t property_id,     \
                                        int *err) {                             \
     omp_interop_val_t *interop_val = (omp_interop_val_t *)interop;             \
-    assert((interop_val)->interop_type == kmp_interop_type_tasksync);          \
+    assert((interop_val)->interop_type == kmp_interop_type_targetsync);        \
     if (!getPropertyCheck(&interop_val, property_id, err)) {                   \
       return (RETURN_TYPE)(0);                                                 \
     }                                                                          \
@@ -193,119 +194,278 @@ __OMP_GET_INTEROP_TY3(const char *, type_desc)
 __OMP_GET_INTEROP_TY3(const char *, rc_desc)
 #undef __OMP_GET_INTEROP_TY3
 
-static const char *copyErrorString(llvm::Error &&Err) {
-  // TODO: Use the error string while avoiding leaks.
-  std::string ErrMsg = llvm::toString(std::move(Err));
-  char *UsrMsg = reinterpret_cast<char *>(malloc(ErrMsg.size() + 1));
-  strcpy(UsrMsg, ErrMsg.c_str());
-  return UsrMsg;
-}
-
 extern "C" {
 
-void __tgt_interop_init(ident_t *LocRef, int32_t Gtid,
-                        omp_interop_val_t *&InteropPtr,
-                        kmp_interop_type_t InteropType, int32_t DeviceId,
-                        int32_t Ndeps, kmp_depend_info_t *DepList,
-                        int32_t HaveNowait) {
-  int32_t NdepsNoalias = 0;
-  kmp_depend_info_t *NoaliasDepList = NULL;
-  assert(InteropType != kmp_interop_type_unknown &&
-         "Cannot initialize with unknown interop_type!");
-  if (DeviceId == -1) {
-    DeviceId = omp_get_default_device();
+omp_interop_val_t *__tgt_interop_get(ident_t *LocRef, int32_t InteropType,
+                                     int64_t DeviceNum, int32_t NumPrefers,
+                                     interop_spec_t *Prefers,
+                                     interop_ctx_t *Ctx, dep_pack_t *Deps) {
+
+  DP("Call to %s with device_num %" PRId64 ", interop type %" PRId32
+     ", number of preferred specs %" PRId32 "%s%s\n",
+     __func__, DeviceNum, InteropType, NumPrefers,
+     Ctx->flags.implicit ? " (implicit)" : "",
+     Ctx->flags.nowait ? " (nowait)" : "");
+
+  if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED)
+    return omp_interop_none;
+
+  // Now, try to create an interop with device_num.
+  if (DeviceNum == OFFLOAD_DEVICE_DEFAULT)
+    DeviceNum = omp_get_default_device();
+
+  auto gtid = Ctx->gtid;
+
+  if (InteropType == kmp_interop_type_targetsync) {
+    if (Ctx->flags.nowait)
+      DP("Warning: nowait flag on interop creation not supported yet. "
+         "Ignored\n");
+    if (Deps)
+      __kmpc_omp_wait_deps(LocRef, gtid, Deps->ndeps, Deps->deplist,
+                           Deps->ndeps_noalias, Deps->noalias_deplist);
   }
 
-  if (InteropType == kmp_interop_type_tasksync) {
-    __kmpc_omp_wait_deps(LocRef, Gtid, Ndeps, DepList, NdepsNoalias,
-                         NoaliasDepList);
+  auto DeviceOrErr = PM->getDevice(DeviceNum);
+  if (!DeviceOrErr) {
+    DP("Couldn't find device %" PRId64
+       " while constructing interop object: %s\n",
+       DeviceNum, toString(DeviceOrErr.takeError()).c_str());
+    return omp_interop_none;
+  }
+  auto &Device = *DeviceOrErr;
+  omp_interop_val_t *Interop = omp_interop_none;
+  auto InteropSpec = Device.RTL->select_interop_preference(
+      DeviceNum, InteropType, NumPrefers, Prefers);
+  if (InteropSpec.fr_id == tgt_fr_none) {
+    DP("Interop request not supported by device %" PRId64 "\n", DeviceNum);
+    return omp_interop_none;
+  }
+  DP("Selected interop preference is fr_id=%s%s impl_attrs=%" PRId64 "\n",
+     getForeignRuntimeIdToStr((tgt_foreign_runtime_id_t)InteropSpec.fr_id),
+     InteropSpec.attrs.inorder ? " inorder" : "", InteropSpec.impl_attrs);
+
+  if (Ctx->flags.implicit) {
+    // This is a request for an RTL managed interop object.
+    // Get it from the InteropTbl if possible
+    for (auto iop : PM->InteropTbl) {
+      if (iop->isCompatibleWith(InteropType, InteropSpec, DeviceNum, gtid)) {
+        Interop = iop;
+        Interop->markDirty();
+        DP("Reused interop " DPxMOD " from device number %" PRId64
+           " for gtid %" PRId32 "\n",
+           DPxPTR(Interop), DeviceNum, gtid);
+        return Interop;
+      }
+    }
   }
 
-  InteropPtr = new omp_interop_val_t(DeviceId, InteropType);
+  Interop = Device.RTL->create_interop(DeviceNum, InteropType, &InteropSpec);
+  DP("Created an interop " DPxMOD " from device number %" PRId64 "\n",
+     DPxPTR(Interop), DeviceNum);
+
+  if (Ctx->flags.implicit) {
+    // register the new implicit interop in the RTL
+    Interop->setOwner(gtid);
+    Interop->markDirty();
+    PM->InteropTbl.add(Interop);
+  } else {
+    Interop->setOwner(omp_interop_val_t::no_owner);
+  }
 
-  auto DeviceOrErr = PM->getDevice(DeviceId);
-  if (!DeviceOrErr) {
-    InteropPtr->err_str = copyErrorString(DeviceOrErr.takeError());
-    return;
+  return Interop;
+}
+
+int __tgt_interop_use(ident_t *LocRef, omp_interop_val_t *Interop,
+                      interop_ctx_t *Ctx, dep_pack_t *Deps) {
+  bool Nowait = Ctx->flags.nowait;
+  DP("Call to %s with interop " DPxMOD ", nowait %" PRId32 "\n", __func__,
+     DPxPTR(Interop), Nowait);
+  if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)
+    return OFFLOAD_FAIL;
+
+  if (Interop->interop_type == kmp_interop_type_targetsync) {
+    if (Deps) {
+      if (Nowait) {
+        DP("Warning: nowait flag on interop use with dependences not supported"
+           "yet. Ignored\n");
+        Nowait = false;
+      }
+
+      __kmpc_omp_wait_deps(LocRef, Ctx->gtid, Deps->ndeps, Deps->deplist,
+                           Deps->ndeps_noalias, Deps->noalias_deplist);
+    }
   }
 
-  DeviceTy &Device = *DeviceOrErr;
-  if (!Device.RTL ||
-      Device.RTL->init_device_info(DeviceId, &(InteropPtr)->device_info,
-                                   &(InteropPtr)->err_str)) {
-    delete InteropPtr;
-    InteropPtr = omp_interop_none;
+  auto DeviceOrErr = Interop->getDevice();
+  if (!DeviceOrErr) {
+    REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(Interop),
+           toString(DeviceOrErr.takeError()).c_str());
+    return OFFLOAD_FAIL;
   }
-  if (InteropType == kmp_interop_type_tasksync) {
-    if (!Device.RTL ||
-        Device.RTL->init_async_info(DeviceId, &(InteropPtr)->async_info)) {
-      delete InteropPtr;
-      InteropPtr = omp_interop_none;
+  auto &IOPDevice = *DeviceOrErr;
+
+  if (Interop->async_info && Interop->async_info->Queue) {
+    if (Nowait)
+      Interop->async_barrier(IOPDevice);
+    else {
+      Interop->flush(IOPDevice);
+      Interop->sync_barrier(IOPDevice);
+      Interop->markClean();
     }
   }
+
+  return OFFLOAD_SUCCESS;
 }
 
-void __tgt_interop_use(ident_t *LocRef, int32_t Gtid,
-                       omp_interop_val_t *&InteropPtr, int32_t DeviceId,
-                       int32_t Ndeps, kmp_depend_info_t *DepList,
-                       int32_t HaveNowait) {
-  int32_t NdepsNoalias = 0;
-  kmp_depend_info_t *NoaliasDepList = NULL;
-  assert(InteropPtr && "Cannot use nullptr!");
-  omp_interop_val_t *InteropVal = InteropPtr;
-  if (DeviceId == -1) {
-    DeviceId = omp_get_default_device();
+int __tgt_interop_release(ident_t *LocRef, omp_interop_val_t *Interop,
+                          interop_ctx_t *Ctx, dep_pack_t *Deps) {
+  DP("Call to %s with interop " DPxMOD "\n", __func__, DPxPTR(Interop));
+
+  if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)
+    return OFFLOAD_FAIL;
+
+  if (Interop->interop_type == kmp_interop_type_targetsync) {
+    if (Ctx->flags.nowait)
+      DP("Warning: nowait flag on interop destroy not supported "
+         "yet. Ignored\n");
+    if (Deps) {
+      __kmpc_omp_wait_deps(LocRef, Ctx->gtid, Deps->ndeps, Deps->deplist,
+                           Deps->ndeps_noalias, Deps->noalias_deplist);
+    }
   }
-  assert(InteropVal != omp_interop_none &&
-         "Cannot use uninitialized interop_ptr!");
-  assert((DeviceId == -1 || InteropVal->device_id == DeviceId) &&
-         "Inconsistent device-id usage!");
 
-  auto DeviceOrErr = PM->getDevice(DeviceId);
+  auto DeviceOrErr = Interop->getDevice();
   if (!DeviceOrErr) {
-    InteropPtr->err_str = copyErrorString(DeviceOrErr.takeError());
-    return;
+    REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(Interop),
+           toString(DeviceOrErr.takeError()).c_str());
+    return OFFLOAD_FAIL;
   }
 
-  if (InteropVal->interop_type == kmp_interop_type_tasksync) {
-    __kmpc_omp_wait_deps(LocRef, Gtid, Ndeps, DepList, NdepsNoalias,
-                         NoaliasDepList);
-  }
-  // TODO Flush the queue associated with the interop through the plugin
+  return Interop->release(*DeviceOrErr);
 }
 
-void __tgt_interop_destroy(ident_t *LocRef, int32_t Gtid,
-                           omp_interop_val_t *&InteropPtr, int32_t DeviceId,
-                           int32_t Ndeps, kmp_depend_info_t *DepList,
-                           int32_t HaveNowait) {
-  int32_t NdepsNoalias = 0;
-  kmp_depend_info_t *NoaliasDepList = NULL;
-  assert(InteropPtr && "Cannot use nullptr!");
-  omp_interop_val_t *InteropVal = InteropPtr;
-  if (DeviceId == -1) {
-    DeviceId = omp_get_default_device();
-  }
+EXTERN int ompx_interop_add_completion_callback(omp_interop_val_t *Interop,
+                                                ompx_interop_cb_t *CB,
+                                                void *Data) {
+  DP("Call to %s with interop " DPxMOD ", property callback " DPxMOD
+     "and data " DPxMOD "\n",
+     __func__, DPxPTR(Interop), DPxPTR(CB), DPxPTR(Data));
 
-  if (InteropVal == omp_interop_none)
-    return;
+  if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)
+    return omp_irc_other;
 
-  assert((DeviceId == -1 || InteropVal->device_id == DeviceId) &&
-         "Inconsistent device-id usage!");
-  auto DeviceOrErr = PM->getDevice(DeviceId);
-  if (!DeviceOrErr) {
-    InteropPtr->err_str = copyErrorString(DeviceOrErr.takeError());
-    return;
+  Interop->addCompletionCb(CB, Data);
+
+  return omp_irc_success;
+}
+
+} // extern "C"
+
+llvm::Expected<DeviceTy &> omp_interop_val_t::getDevice() const {
+  return PM->getDevice(device_id);
+}
+
+bool omp_interop_val_t::isCompatibleWith(int32_t InteropType,
+                                         const interop_spec_t &Spec) {
+  if (interop_type != InteropType)
+    return false;
+  if (Spec.fr_id != fr_id)
+    return false;
+  if (Spec.attrs.inorder != attrs.inorder)
+    return false;
+  if (Spec.impl_attrs != impl_attrs)
+    return false;
+
+  return true;
+}
+
+bool omp_interop_val_t::isCompatibleWith(int32_t InteropType,
+                                         const interop_spec_t &Spec,
+                                         int64_t DeviceNum, int GTID) {
+  if (device_id != DeviceNum)
+    return false;
+
+  if (GTID != owner_gtid)
+    return false;
+
+  return isCompatibleWith(InteropType, Spec);
+}
+
+int32_t omp_interop_val_t::flush(DeviceTy &Device) {
+  return Device.RTL->flush_queue(this);
+}
+
+int32_t omp_interop_val_t::sync_barrier(DeviceTy &Device) {
+  if (Device.RTL->sync_barrier(this) != OFFLOAD_SUCCESS) {
+    FATAL_MESSAGE(device_id, "Interop sync barrier failed for %p object\n",
+                  this);
   }
+  DP("Calling completion callbacks for " DPxMOD "\n", DPxPTR(this));
+  runCompletionCbs();
+  return OFFLOAD_SUCCESS;
+}
+
+int32_t omp_interop_val_t::async_barrier(DeviceTy &Device) {
+  return Device.RTL->async_barrier(this);
+}
 
-  if (InteropVal->interop_type == kmp_interop_type_tasksync) {
-    __kmpc_omp_wait_deps(LocRef, Gtid, Ndeps, DepList, NdepsNoalias,
-                         NoaliasDepList);
+int32_t omp_interop_val_t::release(DeviceTy &Device) {
+  if (async_info != nullptr && (!hasOwner() || !isClean())) {
+    flush(Device);
+    sync_barrier(Device);
   }
-  // TODO Flush the queue associated with the interop through the plugin
-  // TODO Signal out dependences
+  return Device.RTL->release_interop(device_id, this);
+}
 
-  delete InteropPtr;
-  InteropPtr = omp_interop_none;
+void syncImplicitInterops(int Gtid, void *Event) {
+  if (PM->InteropTbl.size() == 0)
+    return;
+
+  DP("target_sync: syncing interops for gtid %" PRId32 ", event " DPxMOD "\n",
+     Gtid, DPxPTR(Event));
+
+  for (auto iop : PM->InteropTbl) {
+    if (iop->async_info && iop->async_info->Queue && iop->isOwnedBy(Gtid) &&
+        !iop->isClean()) {
+
+      auto DeviceOrErr = iop->getDevice();
+      if (!DeviceOrErr) {
+        REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(iop),
+               toString(DeviceOrErr.takeError()).c_str());
+        continue;
+      }
+      auto &IOPDevice = *DeviceOrErr;
+
+      iop->flush(IOPDevice);
+      iop->sync_barrier(IOPDevice);
+      iop->markClean();
+
+      // Alternate implementation option in case using barriers is not
+      // efficient enough:
+      //
+      // Instead of using a synchronous barrier, queue an asynchronous
+      // barrier and create a proxy task associated to the event to handle
+      // OpenMP synchronizations.
+      // When the event is completed, fulfill the proxy task to notify the
+      // OpenMP runtime.
+      // event = iop->asyncBarrier();
+      // ptask = createProxyTask();
+      // Events->add(event,ptask);
+    }
+  }
+  // This would be needed for the alternate implementation
+  // processEvents();
 }
 
-} // extern "C"
+void InteropTblTy::clear() {
+  DP("Clearing Interop Table\n");
+  PerThreadTable::clear([](auto &IOP) {
+    auto DeviceOrErr = IOP->getDevice();
+    if (!DeviceOrErr) {
+      REPORT("Failed to get device for interop " DPxMOD ": %s\n", DPxPTR(IOP),
+             toString(DeviceOrErr.takeError()).c_str());
+      return;
+    }
+    IOP->release(*DeviceOrErr);
+  });
+}
diff --git a/offload/libomptarget/PluginManager.cpp b/offload/libomptarget/PluginManager.cpp
index c4d99dfa9f10c..b57a2f815cba6 100644
--- a/offload/libomptarget/PluginManager.cpp
+++ b/offload/libomptarget/PluginManager.cpp
@@ -128,6 +128,13 @@ void PluginManager::initializeAllDevices() {
       initializeDevice(Plugin, DeviceId);
     }
   }
+  // After all plugins are initialized, register atExit cleanup handlers
+  std::atexit([]() {
+    // Interop cleanup should be done before the plugins are deinitialized as
+    // the backend libraries may be already unloaded.
+    if (PM)
+      PM->InteropTbl.clear();
+  });
 }
 
 // Returns a pointer to the binary descriptor, upgrading from a legacy format if
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index f88e30ae9e76b..6585286bf4285 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -191,6 +191,10 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
                                   DstPtr, Size, AsyncInfo);
 }
 
+int32_t DeviceTy::dataFence(AsyncInfoTy &AsyncInfo) {
+  return RTL->data_fence(RTLDeviceID, AsyncInfo);
+}
+
 int32_t DeviceTy::notifyDataMapped(void *HstPtr, int64_t Size) {
   DP("Notifying about new mapping: HstPtr=" DPxMOD ", Size=%" PRId64 "\n",
      DPxPTR(HstPtr), Size);
diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports
index 2406776c1fb5f..8e2db6ba8bba4 100644
--- a/offload/libomptarget/exports
+++ b/offload/libomptarget/exports
@@ -36,6 +36,7 @@ VERS1.0 {
     __kmpc_push_target_tripcount;
     __kmpc_push_target_tripcount_mapper;
     ompx_dump_mapping_tables;
+    ompx_interop_add_completion_callback;
     omp_get_mapped_ptr;
     omp_get_num_devices;
     omp_get_device_num;
@@ -67,9 +68,10 @@ VERS1.0 {
     omp_get_interop_int;
     omp_get_interop_name;
     omp_get_interop_type_desc;
-    __tgt_interop_init;
+    __tgt_interop_get;
     __tgt_interop_use;
-    __tgt_interop_destroy;
+    __tgt_interop_release;
+    __tgt_target_sync;
     __llvmPushCallConfiguration;
     __llvmPopCallConfiguration;
     llvmLaunchKernel;
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index ea354400f2e99..fe18289765906 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -30,6 +30,7 @@
 #include <cstdint>
 #include <cstdio>
 #include <cstdlib>
+#include <memory>
 
 #ifdef OMPT_SUPPORT
 using namespace llvm::omp::target::ompt;
@@ -116,7 +117,7 @@ targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
            TargetDataFuncPtrTy TargetDataFunction, const char *RegionTypeMsg,
            const char *RegionName) {
   assert(PM && "Runtime not initialized");
-  static_assert(std::is_convertible_v<TargetAsyncInfoTy, AsyncInfoTy>,
+  static_assert(std::is_convertible_v<TargetAsyncInfoTy &, AsyncInfoTy &>,
                 "TargetAsyncInfoTy must be convertible to AsyncInfoTy.");
 
   TIMESCOPE_WITH_DETAILS_AND_IDENT("Runtime: Data Copy",
@@ -165,12 +166,24 @@ targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
                                              OMPT_GET_RETURN_ADDRESS);)
 
   int Rc = OFFLOAD_SUCCESS;
+
+  // Only allocate AttachInfo for targetDataBegin
+  std::unique_ptr<AttachInfoTy> AttachInfo;
+  if (TargetDataFunction == targetDataBegin)
+    AttachInfo = std::make_unique<AttachInfoTy>();
+
   Rc = TargetDataFunction(Loc, *DeviceOrErr, ArgNum, ArgsBase, Args, ArgSizes,
                           ArgTypes, ArgNames, ArgMappers, AsyncInfo,
-                          false /*FromMapper=*/);
+                          AttachInfo.get(), /*FromMapper=*/false);
 
-  if (Rc == OFFLOAD_SUCCESS)
-    Rc = AsyncInfo.synchronize();
+  if (Rc == OFFLOAD_SUCCESS) {
+    // Process deferred ATTACH entries BEFORE synchronization
+    if (AttachInfo && !AttachInfo->AttachEntries.empty())
+      Rc = processAttachEntries(*DeviceOrErr, *AttachInfo, AsyncInfo);
+
+    if (Rc == OFFLOAD_SUCCESS)
+      Rc = AsyncInfo.synchronize();
+  }
 
   handleTargetOutcome(Rc == OFFLOAD_SUCCESS, Loc);
 }
@@ -311,7 +324,7 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
                                int32_t ThreadLimit, void *HostPtr,
                                KernelArgsTy *KernelArgs) {
   assert(PM && "Runtime not initialized");
-  static_assert(std::is_convertible_v<TargetAsyncInfoTy, AsyncInfoTy>,
+  static_assert(std::is_convertible_v<TargetAsyncInfoTy &, AsyncInfoTy &>,
                 "Target AsyncInfoTy must be convertible to AsyncInfoTy.");
   DP("Entering target region for device %" PRId64 " with entry point " DPxMOD
      "\n",
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index 5b25d955dd320..32e89cc75efc9 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -293,7 +293,8 @@ void targetUnlockExplicit(void *HostPtr, int DeviceNum, const char *Name) {
 int targetDataMapper(ident_t *Loc, DeviceTy &Device, void *ArgBase, void *Arg,
                      int64_t ArgSize, int64_t ArgType, map_var_info_t ArgNames,
                      void *ArgMapper, AsyncInfoTy &AsyncInfo,
-                     TargetDataFuncPtrTy TargetDataFunction) {
+                     TargetDataFuncPtrTy TargetDataFunction,
+                     AttachInfoTy *AttachInfo = nullptr) {
   DP("Calling the mapper function " DPxMOD "\n", DPxPTR(ArgMapper));
 
   // The mapper function fills up Components.
@@ -324,17 +325,178 @@ int targetDataMapper(ident_t *Loc, DeviceTy &Device, void *ArgBase, void *Arg,
                               MapperArgsBase.data(), MapperArgs.data(),
                               MapperArgSizes.data(), MapperArgTypes.data(),
                               MapperArgNames.data(), /*arg_mappers*/ nullptr,
-                              AsyncInfo, /*FromMapper=*/true);
+                              AsyncInfo, AttachInfo, /*FromMapper=*/true);
 
   return Rc;
 }
 
+/// Utility function to perform a pointer attachment operation.
+///
+/// For something like:
+/// ```cpp
+///  int *p;
+///  ...
+///  #pragma omp target enter data map(to:p[10:10])
+/// ```
+///
+/// for which the attachment operation gets represented using:
+/// ```
+///   &p, &p[10], sizeof(p), ATTACH
+/// ```
+///
+/// (Hst|Tgt)PtrAddr   represents &p
+/// (Hst|Tgt)PteeBase  represents &p[0]
+/// (Hst|Tgt)PteeBegin represents &p[10]
+///
+/// This function first computes the expected TgtPteeBase using:
+///   `<Select>TgtPteeBase = TgtPteeBegin - (HstPteeBegin - HstPteeBase)`
+///
+/// and then attaches TgtPteeBase to TgtPtrAddr.
+///
+/// \p HstPtrSize represents the size of the pointer p. For C/C++, this
+/// should be same as "sizeof(void*)" (say 8).
+///
+/// However, for Fortran, pointers/allocatables, which are also eligible for
+/// "pointer-attachment", may be implemented using descriptors that contain the
+/// address of the pointee in the first 8 bytes, but also contain other
+/// information such as lower-bound/upper-bound etc in their subsequent fields.
+///
+/// For example, for the following:
+/// ```fortran
+///   integer, allocatable :: x(:)
+///   integer, pointer :: p(:)
+///   ...
+///   p => x(10: 19)
+///   ...
+///   !$omp target enter data map(to:p(:))
+/// ```
+///
+/// The map should trigger a pointer-attachment (assuming the pointer-attachment
+/// conditions as noted on processAttachEntries are met) between the descriptor
+/// for p, and its pointee data.
+///
+/// Since only the first 8 bytes of the descriptor contain the address of the
+/// pointee, an attachment operation on device descriptors involves:
+/// * Setting the first 8 bytes of the device descriptor to point the device
+/// address of the pointee.
+/// * Copying the remaining information about bounds/offset etc. from the host
+/// descriptor to the device descriptor.
+///
+/// The function also handles pointer-attachment portion of PTR_AND_OBJ maps,
+/// like:
+/// ```
+///   &p, &p[10], 10 * sizeof(p[10]), PTR_AND_OBJ
+/// ```
+/// by using `sizeof(void*)` as \p HstPtrSize.
+static int performPointerAttachment(DeviceTy &Device, AsyncInfoTy &AsyncInfo,
+                                    void **HstPtrAddr, void *HstPteeBase,
+                                    void *HstPteeBegin, void **TgtPtrAddr,
+                                    void *TgtPteeBegin, int64_t HstPtrSize,
+                                    TargetPointerResultTy &PtrTPR) {
+  assert(PtrTPR.getEntry() &&
+         "Need a valid pointer entry to perform pointer-attachment");
+
+  int64_t VoidPtrSize = sizeof(void *);
+  assert(HstPtrSize >= VoidPtrSize && "PointerSize is too small");
+
+  uint64_t Delta = reinterpret_cast<uint64_t>(HstPteeBegin) -
+                   reinterpret_cast<uint64_t>(HstPteeBase);
+  void *TgtPteeBase = reinterpret_cast<void *>(
+      reinterpret_cast<uint64_t>(TgtPteeBegin) - Delta);
+
+  // Add shadow pointer tracking
+  // TODO: Support shadow-tracking of larger than VoidPtrSize pointers,
+  // to support restoration of Fortran descriptors. Currently, this check
+  // would return false, even if the host Fortran descriptor had been
+  // updated since its previous map, and we should have updated its
+  // device counterpart. e.g.
+  //
+  //   !$omp target enter data map(x(1:100)) !             (1)
+  //   p => x(10: 19)
+  //   !$omp target enter data map(p, p(:)) !              (2)
+  //   p => x(5: 9)
+  //   !$omp target enter data map(attach(always): p(:)) ! (3)
+  //
+  // While PtrAddr(&desc_p) and PteeBase(&p(1)) are same for (2) and (3), the
+  // pointer attachment for (3) needs to update the bounds information
+  // in the descriptor of p on device.
+  if (!PtrTPR.getEntry()->addShadowPointer(
+          ShadowPtrInfoTy{HstPtrAddr, HstPteeBase, TgtPtrAddr, TgtPteeBase})) {
+    DP("Pointer " DPxMOD " is already attached to " DPxMOD "\n",
+       DPxPTR(TgtPtrAddr), DPxPTR(TgtPteeBase));
+    return OFFLOAD_SUCCESS;
+  }
+
+  DP("Update pointer (" DPxMOD ") -> [" DPxMOD "]\n", DPxPTR(TgtPtrAddr),
+     DPxPTR(TgtPteeBase));
+
+  // Lambda to handle submitData result and perform final steps.
+  auto HandleSubmitResult = [&](int SubmitResult) -> int {
+    if (SubmitResult != OFFLOAD_SUCCESS) {
+      REPORT("Failed to update pointer on device.\n");
+      return OFFLOAD_FAIL;
+    }
+
+    if (PtrTPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) !=
+        OFFLOAD_SUCCESS)
+      return OFFLOAD_FAIL;
+
+    return OFFLOAD_SUCCESS;
+  };
+
+  bool IsPtrAFortranDescriptor = HstPtrSize > VoidPtrSize;
+  if (!IsPtrAFortranDescriptor) {
+    // For "regular" pointers, we can use the VoidPtrLocation from AsyncInfo as
+    // the buffer space for the submission.
+    void *&BufferElement = AsyncInfo.getVoidPtrLocation();
+    BufferElement = TgtPteeBase;
+
+    // Submit the updated pointer value to device
+    return HandleSubmitResult(Device.submitData(
+        TgtPtrAddr, &BufferElement, VoidPtrSize, AsyncInfo, PtrTPR.getEntry()));
+  }
+
+  // For larger "pointers" (like Fortran's descriptors), we create a dynamic
+  // buffer, which will be eventually destroyed by AsyncInfo's post-processing
+  // callback.
+  char *DataBuffer = new char[HstPtrSize];
+
+  // For such descriptors, to the first VoidPtrSize bytes, we store the
+  // pointee's device address.
+  std::memcpy(DataBuffer, &TgtPteeBase, sizeof(void *));
+
+  // And to the remaining bytes, we copy the remaining contents of the host
+  // descriptor after the initial VoidPtrSize bytes.
+  uint64_t HstDescriptorFieldsSize = HstPtrSize - VoidPtrSize;
+  void *HstDescriptorFieldsAddr =
+      reinterpret_cast<char *>(HstPtrAddr) + VoidPtrSize;
+  std::memcpy(DataBuffer + VoidPtrSize, HstDescriptorFieldsAddr,
+              HstDescriptorFieldsSize);
+
+  DP("Updating %" PRId64 " bytes of descriptor (" DPxMOD ") (pointer + %" PRId64
+     " additional bytes from host descriptor " DPxMOD ")\n",
+     HstPtrSize, DPxPTR(TgtPtrAddr), HstDescriptorFieldsSize,
+     DPxPTR(HstDescriptorFieldsAddr));
+
+  // Submit the entire buffer to device
+  int SubmitResult = Device.submitData(TgtPtrAddr, DataBuffer, HstPtrSize,
+                                       AsyncInfo, PtrTPR.getEntry());
+
+  AsyncInfo.addPostProcessingFunction([DataBuffer]() -> int {
+    delete[] DataBuffer;
+    return OFFLOAD_SUCCESS;
+  });
+  return HandleSubmitResult(SubmitResult);
+}
+
 /// Internal function to do the mapping and transfer the data to the device
 int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                     void **ArgsBase, void **Args, int64_t *ArgSizes,
                     int64_t *ArgTypes, map_var_info_t *ArgNames,
                     void **ArgMappers, AsyncInfoTy &AsyncInfo,
-                    bool FromMapper) {
+                    AttachInfoTy *AttachInfo, bool FromMapper) {
+  assert(AttachInfo && "AttachInfo must be available for targetDataBegin for "
+                       "handling ATTACH map-types.");
   // process each input.
   for (int32_t I = 0; I < ArgNum; ++I) {
     // Ignore private variables and arrays - there is no mapping for them.
@@ -352,7 +514,7 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
       map_var_info_t ArgName = (!ArgNames) ? nullptr : ArgNames[I];
       int Rc = targetDataMapper(Loc, Device, ArgsBase[I], Args[I], ArgSizes[I],
                                 ArgTypes[I], ArgName, ArgMappers[I], AsyncInfo,
-                                targetDataBegin);
+                                targetDataBegin, AttachInfo);
 
       if (Rc != OFFLOAD_SUCCESS) {
         REPORT("Call to targetDataBegin via targetDataMapper for custom mapper"
@@ -369,6 +531,18 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
     int64_t DataSize = ArgSizes[I];
     map_var_info_t HstPtrName = (!ArgNames) ? nullptr : ArgNames[I];
 
+    // ATTACH map-types are supposed to be handled after all mapping for the
+    // construct is done. Defer their processing.
+    if (ArgTypes[I] & OMP_TGT_MAPTYPE_ATTACH) {
+      AttachInfo->AttachEntries.emplace_back(
+          /*PointerBase=*/HstPtrBase, /*PointeeBegin=*/HstPtrBegin,
+          /*PointerSize=*/DataSize, /*MapType=*/ArgTypes[I],
+          /*PointeeName=*/HstPtrName);
+
+      DP("Deferring ATTACH map-type processing for argument %d\n", I);
+      continue;
+    }
+
     // Adjust for proper alignment if this is a combined entry (for structs).
     // Look at the next argument - if that is MEMBER_OF this one, then this one
     // is a combined entry.
@@ -434,13 +608,18 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                                   : "device failure or illegal mapping");
         return OFFLOAD_FAIL;
       }
+
+      // Track new allocation, for eventual use in attachment decision-making.
+      if (PointerTpr.Flags.IsNewEntry && !IsHostPtr)
+        AttachInfo->NewAllocations[HstPtrBase] = sizeof(void *);
+
       DP("There are %zu bytes allocated at target address " DPxMOD " - is%s new"
          "\n",
          sizeof(void *), DPxPTR(PointerTgtPtrBegin),
          (PointerTpr.Flags.IsNewEntry ? "" : " not"));
       PointerHstPtrBegin = HstPtrBase;
       // modify current entry.
-      HstPtrBase = *(void **)HstPtrBase;
+      HstPtrBase = *reinterpret_cast<void **>(HstPtrBase);
       // No need to update pointee ref count for the first element of the
       // subelement that comes from mapper.
       UpdateRef =
@@ -464,6 +643,11 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                                 : "device failure or illegal mapping");
       return OFFLOAD_FAIL;
     }
+
+    // Track new allocation, for eventual use in attachment decision-making.
+    if (TPR.Flags.IsNewEntry && !IsHostPtr && TgtPtrBegin)
+      AttachInfo->NewAllocations[HstPtrBegin] = DataSize;
+
     DP("There are %" PRId64 " bytes allocated at target address " DPxMOD
        " - is%s new\n",
        DataSize, DPxPTR(TgtPtrBegin), (TPR.Flags.IsNewEntry ? "" : " not"));
@@ -476,30 +660,13 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
     }
 
     if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ && !IsHostPtr) {
-
-      uint64_t Delta = (uint64_t)HstPtrBegin - (uint64_t)HstPtrBase;
-      void *ExpectedTgtPtrBase = (void *)((uint64_t)TgtPtrBegin - Delta);
-
-      if (PointerTpr.getEntry()->addShadowPointer(ShadowPtrInfoTy{
-              (void **)PointerHstPtrBegin, HstPtrBase,
-              (void **)PointerTgtPtrBegin, ExpectedTgtPtrBase})) {
-        DP("Update pointer (" DPxMOD ") -> [" DPxMOD "]\n",
-           DPxPTR(PointerTgtPtrBegin), DPxPTR(TgtPtrBegin));
-
-        void *&TgtPtrBase = AsyncInfo.getVoidPtrLocation();
-        TgtPtrBase = ExpectedTgtPtrBase;
-
-        int Ret =
-            Device.submitData(PointerTgtPtrBegin, &TgtPtrBase, sizeof(void *),
-                              AsyncInfo, PointerTpr.getEntry());
-        if (Ret != OFFLOAD_SUCCESS) {
-          REPORT("Copying data to device failed.\n");
-          return OFFLOAD_FAIL;
-        }
-        if (PointerTpr.getEntry()->addEventIfNecessary(Device, AsyncInfo) !=
-            OFFLOAD_SUCCESS)
-          return OFFLOAD_FAIL;
-      }
+      int Ret = performPointerAttachment(
+          Device, AsyncInfo, reinterpret_cast<void **>(PointerHstPtrBegin),
+          HstPtrBase, HstPtrBegin,
+          reinterpret_cast<void **>(PointerTgtPtrBegin), TgtPtrBegin,
+          sizeof(void *), PointerTpr);
+      if (Ret != OFFLOAD_SUCCESS)
+        return OFFLOAD_FAIL;
     }
 
     // Check if variable can be used on the device:
@@ -515,6 +682,189 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
   return OFFLOAD_SUCCESS;
 }
 
+/// Process deferred ATTACH map entries collected during targetDataBegin.
+///
+/// From OpenMP's perspective, when mapping something that has a base pointer,
+/// such as:
+/// ```cpp
+///   int *p;
+///   #pragma omp enter target data map(to: p[10:20])
+/// ```
+///
+/// a pointer-attachment between p and &p[10] should occur if both p and
+/// p[10] are present on the device after doing all allocations for all maps
+/// on the construct, and one of the following is true:
+///
+/// * The pointer p was newly allocated while handling the construct
+/// * The pointee p[10:20] was newly allocated while handling the construct
+/// * attach(always) map-type modifier was specified (OpenMP 6.1)
+///
+/// That's why we collect all attach entries and new memory allocations during
+/// targetDataBegin, and use that information to make the decision of whether
+/// to perform a pointer-attachment or not here, after maps have been handled.
+///
+/// Additionally, once we decide that a pointer-attachment should be performed,
+/// we need to make sure that it happens after any previously submitted data
+/// transfers have completed, to avoid the possibility of the pending transfers
+/// clobbering the attachment. For example:
+///
+/// ```cpp
+///   int *p = ...;
+///   int **pp = &p;
+///   map(to: pp[0], p[0])
+/// ```
+///
+/// Which would be represented by:
+/// ```
+/// &pp[0], &pp[0], sizeof(pp[0]), TO (1)
+/// &p[0], &p[0], sizeof(p[0]), TO    (2)
+///
+/// &pp, &pp[0], sizeof(pp), ATTACH   (3)
+/// &p, &p[0], sizeof(p), ATTACH      (4)
+/// ```
+///
+/// (4) and (1) are both trying to modify the device memory corresponding to
+/// `&p`. So, if we decide that (4) should do an attachment, we also need to
+/// ensure that (4) happens after (1) is complete.
+///
+/// For this purpose, we insert a data_fence before the first
+/// pointer-attachment, (3), to ensure that all pending transfers finish first.
+int processAttachEntries(DeviceTy &Device, AttachInfoTy &AttachInfo,
+                         AsyncInfoTy &AsyncInfo) {
+  // Report all tracked allocations from both main loop and ATTACH processing
+  if (!AttachInfo.NewAllocations.empty()) {
+    DP("Tracked %u total new allocations:\n",
+       (unsigned)AttachInfo.NewAllocations.size());
+    for (const auto &Alloc : AttachInfo.NewAllocations) {
+      DP("  Host ptr: " DPxMOD ", Size: %" PRId64 " bytes\n",
+         DPxPTR(Alloc.first), Alloc.second);
+    }
+  }
+
+  if (AttachInfo.AttachEntries.empty())
+    return OFFLOAD_SUCCESS;
+
+  DP("Processing %zu deferred ATTACH map entries\n",
+     AttachInfo.AttachEntries.size());
+
+  int Ret = OFFLOAD_SUCCESS;
+  bool IsFirstPointerAttachment = true;
+  for (size_t EntryIdx = 0; EntryIdx < AttachInfo.AttachEntries.size();
+       ++EntryIdx) {
+    const auto &AttachEntry = AttachInfo.AttachEntries[EntryIdx];
+
+    void **HstPtr = reinterpret_cast<void **>(AttachEntry.PointerBase);
+
+    void *HstPteeBase = *HstPtr;
+    void *HstPteeBegin = AttachEntry.PointeeBegin;
+
+    int64_t PtrSize = AttachEntry.PointerSize;
+    int64_t MapType = AttachEntry.MapType;
+
+    DP("Processing ATTACH entry %zu: HstPtr=" DPxMOD ", HstPteeBegin=" DPxMOD
+       ", Size=%" PRId64 ", Type=0x%" PRIx64 "\n",
+       EntryIdx, DPxPTR(HstPtr), DPxPTR(HstPteeBegin), PtrSize, MapType);
+
+    const bool IsAttachAlways = MapType & OMP_TGT_MAPTYPE_ALWAYS;
+
+    // Lambda to check if a pointer was newly allocated
+    auto WasNewlyAllocated = [&](void *Ptr, const char *PtrName) {
+      bool IsNewlyAllocated =
+          llvm::any_of(AttachInfo.NewAllocations, [&](const auto &Alloc) {
+            void *AllocPtr = Alloc.first;
+            int64_t AllocSize = Alloc.second;
+            return Ptr >= AllocPtr &&
+                   Ptr < reinterpret_cast<void *>(
+                             reinterpret_cast<char *>(AllocPtr) + AllocSize);
+          });
+      DP("Attach %s " DPxMOD " was newly allocated: %s\n", PtrName, DPxPTR(Ptr),
+         IsNewlyAllocated ? "yes" : "no");
+      return IsNewlyAllocated;
+    };
+
+    // Only process ATTACH if either the pointee or the pointer was newly
+    // allocated, or the ALWAYS flag is set.
+    if (!IsAttachAlways && !WasNewlyAllocated(HstPteeBegin, "pointee") &&
+        !WasNewlyAllocated(HstPtr, "pointer")) {
+      DP("Skipping ATTACH entry %zu: neither pointer nor pointee was newly "
+         "allocated and no ALWAYS flag\n",
+         EntryIdx);
+      continue;
+    }
+
+    // Lambda to perform target pointer lookup and validation
+    auto LookupTargetPointer =
+        [&](void *Ptr, int64_t Size,
+            const char *PtrType) -> std::optional<TargetPointerResultTy> {
+      // ATTACH map-type does not change ref-count, or do any allocation
+      // We just need to do a lookup for the pointer/pointee.
+      TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+          Ptr, Size, /*UpdateRefCount=*/false,
+          /*UseHoldRefCount=*/false, /*MustContain=*/true);
+
+      DP("Attach %s lookup - IsPresent=%s, IsHostPtr=%s\n", PtrType,
+         TPR.isPresent() ? "yes" : "no",
+         TPR.Flags.IsHostPointer ? "yes" : "no");
+
+      if (!TPR.isPresent()) {
+        DP("Skipping ATTACH entry %zu: %s not present on device\n", EntryIdx,
+           PtrType);
+        return std::nullopt;
+      }
+      if (TPR.Flags.IsHostPointer) {
+        DP("Skipping ATTACH entry %zu: device version of the %s is a host "
+           "pointer.\n",
+           EntryIdx, PtrType);
+        return std::nullopt;
+      }
+
+      return TPR;
+    };
+
+    // Get device version of the pointee (e.g., &p[10]) first, as we can
+    // release its TPR after extracting the pointer value.
+    void *TgtPteeBegin = [&]() -> void * {
+      if (auto PteeTPROpt = LookupTargetPointer(HstPteeBegin, 0, "pointee"))
+        return PteeTPROpt->TargetPointer;
+      return nullptr;
+    }();
+
+    if (!TgtPteeBegin)
+      continue;
+
+    // Get device version of the pointer (e.g., &p) next. We need to keep its
+    // TPR for use in shadow-pointer handling during pointer-attachment.
+    auto PtrTPROpt = LookupTargetPointer(HstPtr, PtrSize, "pointer");
+    if (!PtrTPROpt)
+      continue;
+    TargetPointerResultTy &PtrTPR = *PtrTPROpt;
+    void **TgtPtrBase = reinterpret_cast<void **>(PtrTPR.TargetPointer);
+
+    // Insert a data-fence before the first pointer-attachment.
+    if (IsFirstPointerAttachment) {
+      IsFirstPointerAttachment = false;
+      DP("Inserting a data fence before the first pointer attachment.\n");
+      Ret = Device.dataFence(AsyncInfo);
+      if (Ret != OFFLOAD_SUCCESS) {
+        REPORT("Failed to insert data fence.\n");
+        return OFFLOAD_FAIL;
+      }
+    }
+
+    // Do the pointer-attachment, i.e. update the device pointer to point to
+    // device pointee.
+    Ret = performPointerAttachment(Device, AsyncInfo, HstPtr, HstPteeBase,
+                                   HstPteeBegin, TgtPtrBase, TgtPteeBegin,
+                                   PtrSize, PtrTPR);
+    if (Ret != OFFLOAD_SUCCESS)
+      return OFFLOAD_FAIL;
+
+    DP("ATTACH entry %zu processed successfully\n", EntryIdx);
+  }
+
+  return OFFLOAD_SUCCESS;
+}
+
 namespace {
 /// This structure contains information to deallocate a target pointer, aka.
 /// used to fix up the shadow map and potentially delete the entry from the
@@ -624,7 +974,8 @@ postProcessingTargetDataEnd(DeviceTy *Device,
 int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                   void **ArgBases, void **Args, int64_t *ArgSizes,
                   int64_t *ArgTypes, map_var_info_t *ArgNames,
-                  void **ArgMappers, AsyncInfoTy &AsyncInfo, bool FromMapper) {
+                  void **ArgMappers, AsyncInfoTy &AsyncInfo,
+                  AttachInfoTy *AttachInfo, bool FromMapper) {
   int Ret = OFFLOAD_SUCCESS;
   auto *PostProcessingPtrs = new SmallVector<PostProcessingInfo>();
   // process each input.
@@ -635,6 +986,14 @@ int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
         (ArgTypes[I] & OMP_TGT_MAPTYPE_PRIVATE))
       continue;
 
+    // Ignore ATTACH entries - they should only be honored on map-entering
+    // directives. They may be encountered here while handling the "end" part of
+    // "#pragma omp target".
+    if (ArgTypes[I] & OMP_TGT_MAPTYPE_ATTACH) {
+      DP("Ignoring ATTACH entry %d in targetDataEnd\n", I);
+      continue;
+    }
+
     if (ArgMappers && ArgMappers[I]) {
       // Instead of executing the regular path of targetDataEnd, call the
       // targetDataMapper variant which will call targetDataEnd again
@@ -900,7 +1259,8 @@ static int getNonContigMergedDimension(__tgt_target_non_contig *NonContig,
 int targetDataUpdate(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                      void **ArgsBase, void **Args, int64_t *ArgSizes,
                      int64_t *ArgTypes, map_var_info_t *ArgNames,
-                     void **ArgMappers, AsyncInfoTy &AsyncInfo, bool) {
+                     void **ArgMappers, AsyncInfoTy &AsyncInfo,
+                     AttachInfoTy *AttachInfo, bool FromMapper) {
   // process each input.
   for (int32_t I = 0; I < ArgNum; ++I) {
     if ((ArgTypes[I] & OMP_TGT_MAPTYPE_LITERAL) ||
@@ -1213,13 +1573,27 @@ static int processDataBefore(ident_t *Loc, int64_t DeviceId, void *HostPtr,
   if (!DeviceOrErr)
     FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
 
+  // Create AttachInfo for tracking any ATTACH entries, or new-allocations
+  // when handling the "begin" mapping for a target constructs.
+  AttachInfoTy AttachInfo;
+
   int Ret = targetDataBegin(Loc, *DeviceOrErr, ArgNum, ArgBases, Args, ArgSizes,
-                            ArgTypes, ArgNames, ArgMappers, AsyncInfo);
+                            ArgTypes, ArgNames, ArgMappers, AsyncInfo,
+                            &AttachInfo, false /*FromMapper=*/);
   if (Ret != OFFLOAD_SUCCESS) {
     REPORT("Call to targetDataBegin failed, abort target.\n");
     return OFFLOAD_FAIL;
   }
 
+  // Process collected ATTACH entries
+  if (!AttachInfo.AttachEntries.empty()) {
+    Ret = processAttachEntries(*DeviceOrErr, AttachInfo, AsyncInfo);
+    if (Ret != OFFLOAD_SUCCESS) {
+      REPORT("Failed to process ATTACH entries.\n");
+      return OFFLOAD_FAIL;
+    }
+  }
+
   // List of (first-)private arrays allocated for this target region
   SmallVector<int> TgtArgsPositions(ArgNum, -1);
 
diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.h b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.h
index 61f680bab3a07..ad135f72fff12 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.h
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.h
@@ -70,10 +70,16 @@ typedef enum {
   HSA_ISA_INFO_NAME = 1
 } hsa_isa_info_t;
 
+typedef enum {
+  HSA_MACHINE_MODEL_SMALL = 0,
+  HSA_MACHINE_MODEL_LARGE = 1
+} hsa_machine_model_t;
+
 typedef enum {
   HSA_AGENT_INFO_NAME = 0,
   HSA_AGENT_INFO_VENDOR_NAME = 1,
   HSA_AGENT_INFO_FEATURE = 2,
+  HSA_AGENT_INFO_MACHINE_MODEL = 3,
   HSA_AGENT_INFO_PROFILE = 4,
   HSA_AGENT_INFO_WAVEFRONT_SIZE = 6,
   HSA_AGENT_INFO_WORKGROUP_MAX_DIM = 7,
diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
index 3117763e35896..29cfe78082dbb 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
@@ -67,6 +67,7 @@ typedef enum hsa_amd_agent_info_s {
   HSA_AMD_AGENT_INFO_CACHELINE_SIZE = 0xA001,
   HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT = 0xA002,
   HSA_AMD_AGENT_INFO_MAX_CLOCK_FREQUENCY = 0xA003,
+  HSA_AMD_AGENT_INFO_MEMORY_MAX_FREQUENCY = 0xA008,
   HSA_AMD_AGENT_INFO_PRODUCT_NAME = 0xA009,
   HSA_AMD_AGENT_INFO_MAX_WAVES_PER_CU = 0xA00A,
   HSA_AMD_AGENT_INFO_NUM_SIMDS_PER_CU = 0xA00B,
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index b7bfa89fc9ea6..7ba55715ff58d 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -570,6 +570,16 @@ struct AMDGPUKernelTy : public GenericKernelTy {
                    KernelLaunchParamsTy LaunchParams,
                    AsyncInfoWrapperTy &AsyncInfoWrapper) const override;
 
+  /// Return maximum block size for maximum occupancy
+  ///
+  /// TODO: This needs to be implemented for amdgpu
+  Expected<uint64_t> maxGroupSize(GenericDeviceTy &GenericDevice,
+                                  uint64_t DynamicMemSize) const override {
+    return Plugin::error(
+        ErrorCode::UNSUPPORTED,
+        "occupancy calculations for AMDGPU are not yet implemented");
+  }
+
   /// Print more elaborate kernel launch info for AMDGPU
   Error printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
                                KernelArgsTy &KernelArgs, uint32_t NumThreads[3],
@@ -1063,6 +1073,20 @@ struct AMDGPUStreamTy {
   /// Indicate to spread data transfers across all available SDMAs
   bool UseMultipleSdmaEngines;
 
+  /// Wrapper function for implementing host callbacks
+  static void CallbackWrapper(AMDGPUSignalTy *InputSignal,
+                              AMDGPUSignalTy *OutputSignal,
+                              void (*Callback)(void *), void *UserData) {
+    // The wait call will not error in this context.
+    if (InputSignal)
+      if (auto Err = InputSignal->wait())
+        reportFatalInternalError(std::move(Err));
+
+    Callback(UserData);
+
+    OutputSignal->signal();
+  }
+
   /// Return the current number of asynchronous operations on the stream.
   uint32_t size() const { return NextSlot; }
 
@@ -1495,6 +1519,31 @@ struct AMDGPUStreamTy {
                                    OutputSignal->get());
   }
 
+  Error pushHostCallback(void (*Callback)(void *), void *UserData) {
+    // Retrieve an available signal for the operation's output.
+    AMDGPUSignalTy *OutputSignal = nullptr;
+    if (auto Err = SignalManager.getResource(OutputSignal))
+      return Err;
+    OutputSignal->reset();
+    OutputSignal->increaseUseCount();
+
+    AMDGPUSignalTy *InputSignal;
+    {
+      std::lock_guard<std::mutex> Lock(Mutex);
+
+      // Consume stream slot and compute dependencies.
+      InputSignal = consume(OutputSignal).second;
+    }
+
+    // "Leaking" the thread here is consistent with other work added to the
+    // queue. The input and output signals will remain valid until the output is
+    // signaled.
+    std::thread(CallbackWrapper, InputSignal, OutputSignal, Callback, UserData)
+        .detach();
+
+    return Plugin::success();
+  }
+
   /// Synchronize with the stream. The current thread waits until all operations
   /// are finalized and it performs the pending post actions (i.e., releasing
   /// intermediate buffers).
@@ -1519,6 +1568,9 @@ struct AMDGPUStreamTy {
   /// actions for that and prior events.
   Error synchronizeOn(AMDGPUEventTy &Event);
 
+  /// Return true if the event from this queue is complete
+  Expected<bool> isEventComplete(const AMDGPUEventTy &Event);
+
   /// Query the stream and complete pending post actions if operations finished.
   /// Return whether all the operations completed. This operation does not block
   /// the calling thread.
@@ -1683,6 +1735,18 @@ Error AMDGPUStreamTy::synchronizeOn(AMDGPUEventTy &Event) {
   return completeUntil(Event.RecordedSlot);
 }
 
+Expected<bool> AMDGPUStreamTy::isEventComplete(const AMDGPUEventTy &Event) {
+  std::lock_guard<std::mutex> Lock(Mutex);
+  assert(Event.RecordedStream == this && "event is for a different stream");
+
+  if (Event.RecordedSyncCycle < SyncCycle) {
+    return true;
+  }
+  assert(Event.RecordedSyncCycle == SyncCycle && "event is from the future?");
+
+  return !Slots[Event.RecordedSlot].Signal->load();
+}
+
 struct AMDGPUStreamManagerTy final
     : GenericDeviceResourceManagerTy<AMDGPUResourceRef<AMDGPUStreamTy>> {
   using ResourceRef = AMDGPUResourceRef<AMDGPUStreamTy>;
@@ -2232,16 +2296,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// Get the stream of the asynchronous info structure or get a new one.
   Error getStream(AsyncInfoWrapperTy &AsyncInfoWrapper,
                   AMDGPUStreamTy *&Stream) {
-    // Get the stream (if any) from the async info.
-    Stream = AsyncInfoWrapper.getQueueAs<AMDGPUStreamTy *>();
-    if (!Stream) {
-      // There was no stream; get an idle one.
-      if (auto Err = AMDGPUStreamManager.getResource(Stream))
-        return Err;
-
-      // Modify the async info's stream.
-      AsyncInfoWrapper.setQueueAs<AMDGPUStreamTy *>(Stream);
-    }
+    auto WrapperStream =
+        AsyncInfoWrapper.getOrInitQueue<AMDGPUStreamTy *>(AMDGPUStreamManager);
+    if (!WrapperStream)
+      return WrapperStream.takeError();
+    Stream = *WrapperStream;
     return Plugin::success();
   }
 
@@ -2296,7 +2355,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   }
 
   /// Synchronize current thread with the pending operations on the async info.
-  Error synchronizeImpl(__tgt_async_info &AsyncInfo) override {
+  Error synchronizeImpl(__tgt_async_info &AsyncInfo,
+                        bool ReleaseQueue) override {
     AMDGPUStreamTy *Stream =
         reinterpret_cast<AMDGPUStreamTy *>(AsyncInfo.Queue);
     assert(Stream && "Invalid stream");
@@ -2307,8 +2367,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     // Once the stream is synchronized, return it to stream pool and reset
     // AsyncInfo. This is to make sure the synchronization only works for its
     // own tasks.
-    AsyncInfo.Queue = nullptr;
-    return AMDGPUStreamManager.returnResource(Stream);
+    if (ReleaseQueue) {
+      AsyncInfo.Queue = nullptr;
+      return AMDGPUStreamManager.returnResource(Stream);
+    }
+    return Plugin::success();
   }
 
   /// Query for the completion of the pending operations on the async info.
@@ -2538,6 +2601,38 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
                                           getAgent(), (uint64_t)Size);
   }
 
+  /// Insert a data fence between previous data operations and the following
+  /// operations. This is a no-op for AMDGPU devices as operations inserted into
+  /// a queue are in-order.
+  Error dataFence(__tgt_async_info *Async) override {
+    return Plugin::success();
+  }
+
+  Error dataFillImpl(void *TgtPtr, const void *PatternPtr, int64_t PatternSize,
+                     int64_t Size,
+                     AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+    hsa_status_t Status;
+
+    // We can use hsa_amd_memory_fill for this size, but it's not async so the
+    // queue needs to be synchronized first
+    if (PatternSize == 4) {
+      if (AsyncInfoWrapper.hasQueue())
+        if (auto Err = synchronize(AsyncInfoWrapper))
+          return Err;
+      Status = hsa_amd_memory_fill(TgtPtr,
+                                   *static_cast<const uint32_t *>(PatternPtr),
+                                   Size / PatternSize);
+
+      if (auto Err =
+              Plugin::check(Status, "error in hsa_amd_memory_fill: %s\n"))
+        return Err;
+    } else {
+      // TODO: Implement for AMDGPU. Most likely by doing the fill in pinned
+      // memory and copying to the device in one go.
+      return Plugin::error(ErrorCode::UNSUPPORTED, "Unsupported fill size");
+    }
+  }
+
   /// Initialize the async info for interoperability purposes.
   Error initAsyncInfoImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) override {
     // TODO: Implement this function.
@@ -2554,6 +2649,15 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
 
+  Error enqueueHostCallImpl(void (*Callback)(void *), void *UserData,
+                            AsyncInfoWrapperTy &AsyncInfo) override {
+    AMDGPUStreamTy *Stream = nullptr;
+    if (auto Err = getStream(AsyncInfo, Stream))
+      return Err;
+
+    return Stream->pushHostCallback(Callback, UserData);
+  };
+
   /// Create an event.
   Error createEventImpl(void **EventPtrStorage) override {
     AMDGPUEventTy **Event = reinterpret_cast<AMDGPUEventTy **>(EventPtrStorage);
@@ -2591,6 +2695,24 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Event->wait(*Stream);
   }
 
+  Expected<bool> hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfo) override {
+    auto Stream = AsyncInfo.getQueueAs<AMDGPUStreamTy *>();
+    if (!Stream)
+      return false;
+
+    auto Query = Stream->query();
+    if (Query)
+      return !*Query;
+    return Query.takeError();
+  }
+
+  Expected<bool> isEventCompleteImpl(void *EventPtr,
+                                     AsyncInfoWrapperTy &AsyncInfo) override {
+    AMDGPUEventTy *Event = reinterpret_cast<AMDGPUEventTy *>(EventPtr);
+    auto Stream = AsyncInfo.getQueueAs<AMDGPUStreamTy *>();
+    return Stream && Stream->isEventComplete(*Event);
+  }
+
   /// Synchronize the current thread with the event.
   Error syncEventImpl(void *EventPtr) override {
     AMDGPUEventTy *Event = reinterpret_cast<AMDGPUEventTy *>(EventPtr);
@@ -2632,6 +2754,15 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (Status == HSA_STATUS_SUCCESS)
       Info.add("Vendor Name", TmpChar, "", DeviceInfo::VENDOR);
 
+    Info.add("Vendor ID", uint64_t{4130}, "", DeviceInfo::VENDOR_ID);
+
+    hsa_machine_model_t MachineModel;
+    Status = getDeviceAttrRaw(HSA_AGENT_INFO_MACHINE_MODEL, MachineModel);
+    if (Status == HSA_STATUS_SUCCESS)
+      Info.add("Memory Address Size",
+               uint64_t{MachineModel == HSA_MACHINE_MODEL_SMALL ? 32u : 64u},
+               "bits", DeviceInfo::ADDRESS_BITS);
+
     hsa_device_type_t DevType;
     Status = getDeviceAttrRaw(HSA_AGENT_INFO_DEVICE, DevType);
     if (Status == HSA_STATUS_SUCCESS) {
@@ -2682,11 +2813,17 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     Status = getDeviceAttrRaw(HSA_AMD_AGENT_INFO_MAX_CLOCK_FREQUENCY, TmpUInt);
     if (Status == HSA_STATUS_SUCCESS)
-      Info.add("Max Clock Freq", TmpUInt, "MHz");
+      Info.add("Max Clock Freq", TmpUInt, "MHz",
+               DeviceInfo::MAX_CLOCK_FREQUENCY);
+
+    Status = getDeviceAttrRaw(HSA_AMD_AGENT_INFO_MEMORY_MAX_FREQUENCY, TmpUInt);
+    if (Status == HSA_STATUS_SUCCESS)
+      Info.add("Max Memory Clock Freq", TmpUInt, "MHz",
+               DeviceInfo::MEMORY_CLOCK_RATE);
 
     Status = getDeviceAttrRaw(HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT, TmpUInt);
     if (Status == HSA_STATUS_SUCCESS)
-      Info.add("Compute Units", TmpUInt);
+      Info.add("Compute Units", TmpUInt, "", DeviceInfo::NUM_COMPUTE_UNITS);
 
     Status = getDeviceAttrRaw(HSA_AMD_AGENT_INFO_NUM_SIMDS_PER_CU, TmpUInt);
     if (Status == HSA_STATUS_SUCCESS)
@@ -2768,7 +2905,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
       Status = Pool->getAttrRaw(HSA_AMD_MEMORY_POOL_INFO_SIZE, TmpSt);
       if (Status == HSA_STATUS_SUCCESS)
-        PoolNode.add("Size", TmpSt, "bytes");
+        PoolNode.add(
+            "Size", TmpSt, "bytes",
+            (Pool->isGlobal() && Pool->isCoarseGrained())
+                ? std::optional<DeviceInfo>{DeviceInfo::GLOBAL_MEM_SIZE}
+                : std::nullopt);
 
       Status = Pool->getAttrRaw(HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_ALLOWED,
                                 TmpBool);
@@ -2945,6 +3086,40 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
 
+  bool checkIfCoarseGrainMemoryNearOrAbove64GB() {
+    for (AMDGPUMemoryPoolTy *Pool : AllMemoryPools) {
+      if (!Pool->isGlobal() || !Pool->isCoarseGrained())
+        continue;
+      uint64_t Value;
+      hsa_status_t Status =
+          Pool->getAttrRaw(HSA_AMD_MEMORY_POOL_INFO_SIZE, Value);
+      if (Status != HSA_STATUS_SUCCESS)
+        continue;
+      constexpr uint64_t Almost64Gig = 0xFF0000000;
+      if (Value >= Almost64Gig)
+        return true;
+    }
+    return false; // CoarseGrain pool w/ 64GB or more capacity not found
+  }
+
+  size_t getMemoryManagerSizeThreshold() override {
+    // Targeting high memory capacity GPUs such as
+    // data center GPUs.
+    if (checkIfCoarseGrainMemoryNearOrAbove64GB()) {
+      // Set GenericDeviceTy::MemoryManager's Threshold to 3GiB,
+      // if threshold is not already set by ENV var
+      // LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD.
+      // This MemoryManager is used for omp_target_alloc(), OpenMP
+      // (non-usm) map clause, etc.
+      //
+      // Ideally, this kind of pooling is best performed at
+      // a common level (e.g, user side of HSA) between OpenMP and HIP
+      // but that feature does not exist (yet).
+      return 3ul * 1024 * 1024 * 1024 /* 3 GiB */;
+    }
+    return 0;
+  }
+
   /// Envar for controlling the number of HSA queues per device. High number of
   /// queues may degrade performance.
   UInt32Envar OMPX_NumQueues;
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 8c17a2ee07047..75f87cab6049d 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -21,6 +21,7 @@
 #include <vector>
 
 #include "ExclusiveAccess.h"
+#include "OpenMP/InteropAPI.h"
 #include "Shared/APITypes.h"
 #include "Shared/Debug.h"
 #include "Shared/Environment.h"
@@ -59,6 +60,40 @@ struct GenericPluginTy;
 struct GenericKernelTy;
 struct GenericDeviceTy;
 struct RecordReplayTy;
+template <typename ResourceRef> class GenericDeviceResourceManagerTy;
+
+namespace Plugin {
+/// Create a success error. This is the same as calling Error::success(), but
+/// it is recommended to use this one for consistency with Plugin::error() and
+/// Plugin::check().
+static inline Error success() { return Error::success(); }
+
+/// Create an Offload error.
+template <typename... ArgsTy>
+static Error error(error::ErrorCode Code, const char *ErrFmt, ArgsTy... Args) {
+  return error::createOffloadError(Code, ErrFmt, Args...);
+}
+
+inline Error error(error::ErrorCode Code, const char *S) {
+  return make_error<error::OffloadError>(Code, S);
+}
+
+inline Error error(error::ErrorCode Code, Error &&OtherError,
+                   const char *Context) {
+  return error::createOffloadError(Code, std::move(OtherError), Context);
+}
+
+/// Check the plugin-specific error code and return an error or success
+/// accordingly. In case of an error, create a string error with the error
+/// description. The ErrFmt should follow the format:
+///     "Error in <function name>[<optional info>]: %s"
+/// The last format specifier "%s" is mandatory and will be used to place the
+/// error code's description. Notice this function should be only called from
+/// the plugin-specific code.
+/// TODO: Refactor this, must be defined individually by each plugin.
+template <typename... ArgsTy>
+static Error check(int32_t ErrorCode, const char *ErrFmt, ArgsTy... Args);
+} // namespace Plugin
 
 /// Class that wraps the __tgt_async_info to simply its usage. In case the
 /// object is constructed without a valid __tgt_async_info, the object will use
@@ -93,6 +128,20 @@ struct AsyncInfoWrapperTy {
     AsyncInfoPtr->Queue = Queue;
   }
 
+  /// Get the queue, using the provided resource manager to initialise it if it
+  /// doesn't exist.
+  template <typename Ty, typename RMTy>
+  Expected<Ty>
+  getOrInitQueue(GenericDeviceResourceManagerTy<RMTy> &ResourceManager) {
+    std::lock_guard<std::mutex> Lock(AsyncInfoPtr->Mutex);
+    if (!AsyncInfoPtr->Queue) {
+      if (auto Err = ResourceManager.getResource(
+              *reinterpret_cast<Ty *>(&AsyncInfoPtr->Queue)))
+        return Err;
+    }
+    return getQueueAs<Ty>();
+  }
+
   /// Synchronize with the __tgt_async_info's pending operations if it's the
   /// internal async info. The error associated to the asynchronous operations
   /// issued in this queue must be provided in \p Err. This function will update
@@ -104,6 +153,7 @@ struct AsyncInfoWrapperTy {
   /// Register \p Ptr as an associated allocation that is freed after
   /// finalization.
   void freeAllocationAfterSynchronization(void *Ptr) {
+    std::lock_guard<std::mutex> AllocationGuard(AsyncInfoPtr->Mutex);
     AsyncInfoPtr->AssociatedAllocations.push_back(Ptr);
   }
 
@@ -338,6 +388,9 @@ struct GenericKernelTy {
                            KernelLaunchParamsTy LaunchParams,
                            AsyncInfoWrapperTy &AsyncInfoWrapper) const = 0;
 
+  virtual Expected<uint64_t> maxGroupSize(GenericDeviceTy &GenericDevice,
+                                          uint64_t DynamicMemSize) const = 0;
+
   /// Get the kernel name.
   const char *getName() const { return Name.c_str(); }
 
@@ -793,9 +846,12 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   Error setupRPCServer(GenericPluginTy &Plugin, DeviceImageTy &Image);
 
   /// Synchronize the current thread with the pending operations on the
-  /// __tgt_async_info structure.
-  Error synchronize(__tgt_async_info *AsyncInfo);
-  virtual Error synchronizeImpl(__tgt_async_info &AsyncInfo) = 0;
+  /// __tgt_async_info structure. If ReleaseQueue is false, then the
+  // underlying queue will not be released. In this case, additional
+  // work may be submitted to the queue whilst a synchronize is running.
+  Error synchronize(__tgt_async_info *AsyncInfo, bool ReleaseQueue = true);
+  virtual Error synchronizeImpl(__tgt_async_info &AsyncInfo,
+                                bool ReleaseQueue) = 0;
 
   /// Invokes any global constructors on the device if present and is required
   /// by the target.
@@ -891,6 +947,10 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   virtual Error dataRetrieveImpl(void *HstPtr, const void *TgtPtr, int64_t Size,
                                  AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
 
+  /// Instert a data fence between previous data operations and the following
+  /// operations if necessary for the device
+  virtual Error dataFence(__tgt_async_info *AsyncInfo) = 0;
+
   /// Exchange data between devices (device to device transfer). Calling this
   /// function is only valid if GenericPlugin::isDataExchangable() passing the
   /// two devices returns true.
@@ -900,6 +960,13 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
                                  void *DstPtr, int64_t Size,
                                  AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
 
+  /// Fill data on the device with a pattern from the host
+  Error dataFill(void *TgtPtr, const void *PatternPtr, int64_t PatternSize,
+                 int64_t Size, __tgt_async_info *AsyncInfo);
+  virtual Error dataFillImpl(void *TgtPtr, const void *PatternPtr,
+                             int64_t PatternSize, int64_t Size,
+                             AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
+
   /// Run the kernel associated with \p EntryPtr
   Error launchKernel(void *EntryPtr, void **ArgPtrs, ptrdiff_t *ArgOffsets,
                      KernelArgsTy &KernelArgs, __tgt_async_info *AsyncInfo);
@@ -912,6 +979,12 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   Error initDeviceInfo(__tgt_device_info *DeviceInfo);
   virtual Error initDeviceInfoImpl(__tgt_device_info *DeviceInfo) = 0;
 
+  /// Enqueue a host call to AsyncInfo
+  Error enqueueHostCall(void (*Callback)(void *), void *UserData,
+                        __tgt_async_info *AsyncInfo);
+  virtual Error enqueueHostCallImpl(void (*Callback)(void *), void *UserData,
+                                    AsyncInfoWrapperTy &AsyncInfo) = 0;
+
   /// Create an event.
   Error createEvent(void **EventPtrStorage);
   virtual Error createEventImpl(void **EventPtrStorage) = 0;
@@ -931,6 +1004,11 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   virtual Error waitEventImpl(void *EventPtr,
                               AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
 
+  /// Check if the event enqueued to AsyncInfo is complete
+  Expected<bool> isEventComplete(void *Event, __tgt_async_info *AsyncInfo);
+  virtual Expected<bool>
+  isEventCompleteImpl(void *EventPtr, AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
+
   /// Synchronize the current thread with the event.
   Error syncEvent(void *EventPtr);
   virtual Error syncEventImpl(void *EventPtr) = 0;
@@ -939,6 +1017,14 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   Error printInfo();
   virtual Expected<InfoTreeNode> obtainInfoImpl() = 0;
 
+  /// Return true if the device has work that is either queued or currently
+  /// running
+  ///
+  /// Devices which cannot report this information should always return true
+  Expected<bool> hasPendingWork(__tgt_async_info *AsyncInfo);
+  virtual Expected<bool>
+  hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) = 0;
+
   /// Getters of the grid values.
   uint32_t getWarpSize() const { return GridValues.GV_Warp_Size; }
   uint32_t getThreadLimit() const { return GridValues.GV_Max_WG_Size; }
@@ -1003,6 +1089,21 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   bool useAutoZeroCopy();
   virtual bool useAutoZeroCopyImpl() { return false; }
 
+  virtual Expected<omp_interop_val_t *>
+  createInterop(int32_t InteropType, interop_spec_t &InteropSpec) {
+    return nullptr;
+  }
+
+  virtual Error releaseInterop(omp_interop_val_t *Interop) {
+    return Plugin::success();
+  }
+
+  virtual interop_spec_t selectInteropPreference(int32_t InteropType,
+                                                 int32_t NumPrefers,
+                                                 interop_spec_t *Prefers) {
+    return interop_spec_t{tgt_fr_none, {false, 0}, 0};
+  }
+
   /// Allocate and construct a kernel object.
   virtual Expected<GenericKernelTy &> constructKernel(const char *Name) = 0;
 
@@ -1090,6 +1191,9 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// Pointer to the memory manager or nullptr if not available.
   MemoryManagerTy *MemoryManager;
 
+  /// Per device setting of MemoryManager's Threshold
+  virtual size_t getMemoryManagerSizeThreshold() { return 0; }
+
   /// Environment variables defined by the OpenMP standard.
   Int32Envar OMP_TeamLimit;
   Int32Envar OMP_NumTeams;
@@ -1271,6 +1375,20 @@ struct GenericPluginTy {
   virtual Expected<bool> isELFCompatible(uint32_t DeviceID,
                                          StringRef Image) const = 0;
 
+  virtual Error flushQueueImpl(omp_interop_val_t *Interop) {
+    return Plugin::success();
+  }
+
+  virtual Error syncBarrierImpl(omp_interop_val_t *Interop) {
+    return Plugin::error(error::ErrorCode::UNSUPPORTED,
+                         "sync_barrier not supported");
+  }
+
+  virtual Error asyncBarrierImpl(omp_interop_val_t *Interop) {
+    return Plugin::error(error::ErrorCode::UNSUPPORTED,
+                         "async_barrier not supported");
+  }
+
 protected:
   /// Indicate whether a device id is valid.
   bool isValidDeviceId(int32_t DeviceId) const {
@@ -1355,6 +1473,10 @@ struct GenericPluginTy {
                               int DstDeviceId, void *DstPtr, int64_t Size,
                               __tgt_async_info *AsyncInfo);
 
+  /// Places a fence between previous data movements and following data
+  /// movements if necessary on the device
+  int32_t data_fence(int32_t DeviceId, __tgt_async_info *AsyncInfo);
+
   /// Begin executing a kernel on the given device.
   int32_t launch_kernel(int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs,
                         ptrdiff_t *TgtOffsets, KernelArgsTy *KernelArgs,
@@ -1410,6 +1532,33 @@ struct GenericPluginTy {
   int32_t get_function(__tgt_device_binary Binary, const char *Name,
                        void **KernelPtr);
 
+  /// Return the interop specification that the plugin supports
+  /// It might not be one of the user specified ones.
+  interop_spec_t select_interop_preference(int32_t ID, int32_t InteropType,
+                                           int32_t NumPrefers,
+                                           interop_spec_t *Prefers) {
+    auto &Device = getDevice(ID);
+    return Device.selectInteropPreference(InteropType, NumPrefers, Prefers);
+  }
+
+  /// Create OpenMP interop with the given interop context
+  omp_interop_val_t *create_interop(int32_t ID, int32_t InteropContext,
+                                    interop_spec_t *InteropSpec);
+
+  /// Release OpenMP interop object
+  int32_t release_interop(int32_t ID, omp_interop_val_t *Interop);
+
+  /// Flush the queue associated with the interop object if necessary
+  int32_t flush_queue(omp_interop_val_t *Interop);
+
+  /// Perform a host synchronization with the queue associated with the interop
+  /// object and wait for it to complete.
+  int32_t sync_barrier(omp_interop_val_t *Interop);
+
+  /// Queue an asynchronous barrier in the queue associated with the interop
+  /// object and return immediately.
+  int32_t async_barrier(omp_interop_val_t *Interop);
+
 private:
   /// Indicates if the platform runtime has been fully initialized.
   bool Initialized = false;
@@ -1442,39 +1591,6 @@ struct GenericPluginTy {
   RecordReplayTy *RecordReplay;
 };
 
-namespace Plugin {
-/// Create a success error. This is the same as calling Error::success(), but
-/// it is recommended to use this one for consistency with Plugin::error() and
-/// Plugin::check().
-static inline Error success() { return Error::success(); }
-
-/// Create an Offload error.
-template <typename... ArgsTy>
-static Error error(error::ErrorCode Code, const char *ErrFmt, ArgsTy... Args) {
-  return error::createOffloadError(Code, ErrFmt, Args...);
-}
-
-inline Error error(error::ErrorCode Code, const char *S) {
-  return make_error<error::OffloadError>(Code, S);
-}
-
-inline Error error(error::ErrorCode Code, Error &&OtherError,
-                   const char *Context) {
-  return error::createOffloadError(Code, std::move(OtherError), Context);
-}
-
-/// Check the plugin-specific error code and return an error or success
-/// accordingly. In case of an error, create a string error with the error
-/// description. The ErrFmt should follow the format:
-///     "Error in <function name>[<optional info>]: %s"
-/// The last format specifier "%s" is mandatory and will be used to place the
-/// error code's description. Notice this function should be only called from
-/// the plugin-specific code.
-/// TODO: Refactor this, must be defined individually by each plugin.
-template <typename... ArgsTy>
-static Error check(int32_t ErrorCode, const char *ErrFmt, ArgsTy... Args);
-} // namespace Plugin
-
 /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class
 /// acts as a reference to a device resource, such as a stream, and requires
 /// some basic functions to be implemented. The derived class should define an
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 94a050b559efe..d4b5f914c6672 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -815,8 +815,11 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
 
   // Enable the memory manager if required.
   auto [ThresholdMM, EnableMM] = MemoryManagerTy::getSizeThresholdFromEnv();
-  if (EnableMM)
+  if (EnableMM) {
+    if (ThresholdMM == 0)
+      ThresholdMM = getMemoryManagerSizeThreshold();
     MemoryManager = new MemoryManagerTy(*this, ThresholdMM);
+  }
 
   return Plugin::success();
 }
@@ -1332,18 +1335,25 @@ Error PinnedAllocationMapTy::unlockUnmappedHostBuffer(void *HstPtr) {
   return eraseEntry(*Entry);
 }
 
-Error GenericDeviceTy::synchronize(__tgt_async_info *AsyncInfo) {
-  if (!AsyncInfo || !AsyncInfo->Queue)
-    return Plugin::error(ErrorCode::INVALID_ARGUMENT,
-                         "invalid async info queue");
+Error GenericDeviceTy::synchronize(__tgt_async_info *AsyncInfo,
+                                   bool ReleaseQueue) {
+  SmallVector<void *> AllocsToDelete{};
+  {
+    std::lock_guard<std::mutex> AllocationGuard{AsyncInfo->Mutex};
 
-  if (auto Err = synchronizeImpl(*AsyncInfo))
-    return Err;
+    if (!AsyncInfo || !AsyncInfo->Queue)
+      return Plugin::error(ErrorCode::INVALID_ARGUMENT,
+                           "invalid async info queue");
+
+    if (auto Err = synchronizeImpl(*AsyncInfo, ReleaseQueue))
+      return Err;
+
+    std::swap(AllocsToDelete, AsyncInfo->AssociatedAllocations);
+  }
 
-  for (auto *Ptr : AsyncInfo->AssociatedAllocations)
+  for (auto *Ptr : AllocsToDelete)
     if (auto Err = dataDelete(Ptr, TargetAllocTy::TARGET_ALLOC_DEVICE))
       return Err;
-  AsyncInfo->AssociatedAllocations.clear();
 
   return Plugin::success();
 }
@@ -1530,6 +1540,16 @@ Error GenericDeviceTy::dataExchange(const void *SrcPtr, GenericDeviceTy &DstDev,
   return Err;
 }
 
+Error GenericDeviceTy::dataFill(void *TgtPtr, const void *PatternPtr,
+                                int64_t PatternSize, int64_t Size,
+                                __tgt_async_info *AsyncInfo) {
+  AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo);
+  auto Err =
+      dataFillImpl(TgtPtr, PatternPtr, PatternSize, Size, AsyncInfoWrapper);
+  AsyncInfoWrapper.finalize(Err);
+  return Err;
+}
+
 Error GenericDeviceTy::launchKernel(void *EntryPtr, void **ArgPtrs,
                                     ptrdiff_t *ArgOffsets,
                                     KernelArgsTy &KernelArgs,
@@ -1579,6 +1599,15 @@ Error GenericDeviceTy::initAsyncInfo(__tgt_async_info **AsyncInfoPtr) {
   return Err;
 }
 
+Error GenericDeviceTy::enqueueHostCall(void (*Callback)(void *), void *UserData,
+                                       __tgt_async_info *AsyncInfo) {
+  AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo);
+
+  auto Err = enqueueHostCallImpl(Callback, UserData, AsyncInfoWrapper);
+  AsyncInfoWrapper.finalize(Err);
+  return Err;
+}
+
 Error GenericDeviceTy::initDeviceInfo(__tgt_device_info *DeviceInfo) {
   assert(DeviceInfo && "Invalid device info");
 
@@ -1623,6 +1652,37 @@ Error GenericDeviceTy::waitEvent(void *EventPtr, __tgt_async_info *AsyncInfo) {
   return Err;
 }
 
+Expected<bool> GenericDeviceTy::hasPendingWork(__tgt_async_info *AsyncInfo) {
+  AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo);
+  auto Res = hasPendingWorkImpl(AsyncInfoWrapper);
+  if (auto Err = Res.takeError()) {
+    AsyncInfoWrapper.finalize(Err);
+    return Err;
+  }
+
+  auto Err = Plugin::success();
+  AsyncInfoWrapper.finalize(Err);
+  if (Err)
+    return Err;
+  return Res;
+}
+
+Expected<bool> GenericDeviceTy::isEventComplete(void *Event,
+                                                __tgt_async_info *AsyncInfo) {
+  AsyncInfoWrapperTy AsyncInfoWrapper(*this, AsyncInfo);
+  auto Res = isEventCompleteImpl(Event, AsyncInfoWrapper);
+  if (auto Err = Res.takeError()) {
+    AsyncInfoWrapper.finalize(Err);
+    return Err;
+  }
+
+  auto Err = Plugin::success();
+  AsyncInfoWrapper.finalize(Err);
+  if (Err)
+    return Err;
+  return Res;
+}
+
 Error GenericDeviceTy::syncEvent(void *EventPtr) {
   return syncEventImpl(EventPtr);
 }
@@ -2231,3 +2291,83 @@ int32_t GenericPluginTy::get_function(__tgt_device_binary Binary,
   *KernelPtr = &Kernel;
   return OFFLOAD_SUCCESS;
 }
+
+/// Create OpenMP interop with the given interop context
+omp_interop_val_t *
+GenericPluginTy::create_interop(int32_t ID, int32_t InteropContext,
+                                interop_spec_t *InteropSpec) {
+  assert(InteropSpec && "Interop spec is null");
+  auto &Device = getDevice(ID);
+  auto InteropOrErr = Device.createInterop(InteropContext, *InteropSpec);
+  if (!InteropOrErr) {
+    REPORT("Failure to create interop object for device " DPxMOD ": %s\n",
+           DPxPTR(InteropSpec), toString(InteropOrErr.takeError()).c_str());
+    return nullptr;
+  }
+  return *InteropOrErr;
+}
+
+/// Release OpenMP interop object
+int32_t GenericPluginTy::release_interop(int32_t ID,
+                                         omp_interop_val_t *Interop) {
+  assert(Interop && "Interop is null");
+  assert(Interop->device_id == ID && "Interop does not match device id");
+  auto &Device = getDevice(ID);
+  auto Err = Device.releaseInterop(Interop);
+  if (Err) {
+    REPORT("Failure to release interop object " DPxMOD ": %s\n",
+           DPxPTR(Interop), toString(std::move(Err)).c_str());
+    return OFFLOAD_FAIL;
+  }
+  return OFFLOAD_SUCCESS;
+}
+
+/// Flush the queue associated with the interop object if necessary
+int32_t GenericPluginTy::flush_queue(omp_interop_val_t *Interop) {
+  assert(Interop && "Interop is null");
+  auto Err = flushQueueImpl(Interop);
+  if (Err) {
+    REPORT("Failure to flush interop object " DPxMOD " queue: %s\n",
+           DPxPTR(Interop), toString(std::move(Err)).c_str());
+    return OFFLOAD_FAIL;
+  }
+  return OFFLOAD_SUCCESS;
+}
+
+/// Perform a host synchronization with the queue associated with the interop
+/// object and wait for it to complete.
+int32_t GenericPluginTy::sync_barrier(omp_interop_val_t *Interop) {
+  assert(Interop && "Interop is null");
+  auto Err = syncBarrierImpl(Interop);
+  if (Err) {
+    REPORT("Failure to synchronize interop object " DPxMOD ": %s\n",
+           DPxPTR(Interop), toString(std::move(Err)).c_str());
+    return OFFLOAD_FAIL;
+  }
+  return OFFLOAD_SUCCESS;
+}
+
+/// Queue an asynchronous barrier in the queue associated with the interop
+/// object and return immediately.
+int32_t GenericPluginTy::async_barrier(omp_interop_val_t *Interop) {
+  assert(Interop && "Interop is null");
+  auto Err = asyncBarrierImpl(Interop);
+  if (Err) {
+    REPORT("Failure to queue barrier in interop object " DPxMOD ": %s\n",
+           DPxPTR(Interop), toString(std::move(Err)).c_str());
+    return OFFLOAD_FAIL;
+  }
+  return OFFLOAD_SUCCESS;
+}
+
+int32_t GenericPluginTy::data_fence(int32_t DeviceId,
+                                    __tgt_async_info *AsyncInfo) {
+  auto Err = getDevice(DeviceId).dataFence(AsyncInfo);
+  if (Err) {
+    REPORT("failure to place data fence on device %d: %s\n", DeviceId,
+           toString(std::move(Err)).data());
+    return OFFLOAD_FAIL;
+  }
+
+  return OFFLOAD_SUCCESS;
+}
diff --git a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
index 361a781e8f9b6..f5b2d074a47e7 100644
--- a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
+++ b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.cpp
@@ -53,6 +53,13 @@ DLWRAP(cuMemcpyDtoHAsync, 4)
 DLWRAP(cuMemcpyHtoD, 3)
 DLWRAP(cuMemcpyHtoDAsync, 4)
 
+DLWRAP(cuMemsetD8Async, 4)
+DLWRAP(cuMemsetD16Async, 4)
+DLWRAP(cuMemsetD32Async, 4)
+DLWRAP(cuMemsetD2D8Async, 6)
+DLWRAP(cuMemsetD2D16Async, 6)
+DLWRAP(cuMemsetD2D32Async, 6)
+
 DLWRAP(cuMemFree, 1)
 DLWRAP(cuMemFreeHost, 1)
 DLWRAP(cuMemFreeAsync, 2)
@@ -72,6 +79,7 @@ DLWRAP(cuDevicePrimaryCtxGetState, 3)
 DLWRAP(cuDevicePrimaryCtxSetFlags, 2)
 DLWRAP(cuDevicePrimaryCtxRetain, 2)
 DLWRAP(cuModuleLoadDataEx, 5)
+DLWRAP(cuOccupancyMaxPotentialBlockSize, 6)
 
 DLWRAP(cuDeviceCanAccessPeer, 3)
 DLWRAP(cuCtxEnablePeerAccess, 2)
@@ -82,6 +90,7 @@ DLWRAP(cuCtxSetLimit, 2)
 
 DLWRAP(cuEventCreate, 2)
 DLWRAP(cuEventRecord, 2)
+DLWRAP(cuEventQuery, 1)
 DLWRAP(cuStreamWaitEvent, 3)
 DLWRAP(cuEventSynchronize, 1)
 DLWRAP(cuEventDestroy, 1)
diff --git a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
index b6c022c8e7e8b..dec4e33508c62 100644
--- a/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
+++ b/offload/plugins-nextgen/cuda/dynamic_cuda/cuda.h
@@ -290,6 +290,7 @@ static inline void *CU_LAUNCH_PARAM_BUFFER_POINTER = (void *)0x01;
 static inline void *CU_LAUNCH_PARAM_BUFFER_SIZE = (void *)0x02;
 
 typedef void (*CUstreamCallback)(CUstream, CUresult, void *);
+typedef size_t (*CUoccupancyB2DSize)(int);
 
 CUresult cuCtxGetDevice(CUdevice *);
 CUresult cuDeviceGet(CUdevice *, int);
@@ -321,6 +322,16 @@ CUresult cuMemcpyDtoHAsync(void *, CUdeviceptr, size_t, CUstream);
 CUresult cuMemcpyHtoD(CUdeviceptr, const void *, size_t);
 CUresult cuMemcpyHtoDAsync(CUdeviceptr, const void *, size_t, CUstream);
 
+CUresult cuMemsetD8Async(CUdeviceptr, unsigned int, size_t, CUstream);
+CUresult cuMemsetD16Async(CUdeviceptr, unsigned int, size_t, CUstream);
+CUresult cuMemsetD32Async(CUdeviceptr, unsigned int, size_t, CUstream);
+CUresult cuMemsetD2D8Async(CUdeviceptr, size_t, unsigned int, size_t, size_t,
+                           CUstream);
+CUresult cuMemsetD2D16Async(CUdeviceptr, size_t, unsigned int, size_t, size_t,
+                            CUstream);
+CUresult cuMemsetD2D32Async(CUdeviceptr, size_t, unsigned int, size_t, size_t,
+                            CUstream);
+
 CUresult cuMemFree(CUdeviceptr);
 CUresult cuMemFreeHost(void *);
 CUresult cuMemFreeAsync(CUdeviceptr, CUstream);
@@ -352,6 +363,7 @@ CUresult cuCtxSetLimit(CUlimit, size_t);
 
 CUresult cuEventCreate(CUevent *, unsigned int);
 CUresult cuEventRecord(CUevent, CUstream);
+CUresult cuEventQuery(CUevent);
 CUresult cuStreamWaitEvent(CUstream, CUevent, unsigned int);
 CUresult cuEventSynchronize(CUevent);
 CUresult cuEventDestroy(CUevent);
@@ -372,5 +384,7 @@ CUresult cuMemSetAccess(CUdeviceptr ptr, size_t size,
 CUresult cuMemGetAllocationGranularity(size_t *granularity,
                                        const CUmemAllocationProp *prop,
                                        CUmemAllocationGranularity_flags option);
+CUresult cuOccupancyMaxPotentialBlockSize(int *, int *, CUfunction,
+                                          CUoccupancyB2DSize, size_t, int);
 
 #endif
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index c5f31670079ae..bf335ab20f75b 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -157,6 +157,20 @@ struct CUDAKernelTy : public GenericKernelTy {
                    KernelLaunchParamsTy LaunchParams,
                    AsyncInfoWrapperTy &AsyncInfoWrapper) const override;
 
+  /// Return maximum block size for maximum occupancy
+  Expected<uint64_t> maxGroupSize(GenericDeviceTy &,
+                                  uint64_t DynamicMemSize) const override {
+    int minGridSize;
+    int maxBlockSize;
+    auto Res = cuOccupancyMaxPotentialBlockSize(
+        &minGridSize, &maxBlockSize, Func, NULL, DynamicMemSize, INT_MAX);
+    if (auto Err = Plugin::check(
+            Res, "error in cuOccupancyMaxPotentialBlockSize: %s")) {
+      return Err;
+    }
+    return maxBlockSize;
+  }
+
 private:
   /// The CUDA kernel function to execute.
   CUfunction Func;
@@ -522,16 +536,11 @@ struct CUDADeviceTy : public GenericDeviceTy {
 
   /// Get the stream of the asynchronous info structure or get a new one.
   Error getStream(AsyncInfoWrapperTy &AsyncInfoWrapper, CUstream &Stream) {
-    // Get the stream (if any) from the async info.
-    Stream = AsyncInfoWrapper.getQueueAs<CUstream>();
-    if (!Stream) {
-      // There was no stream; get an idle one.
-      if (auto Err = CUDAStreamManager.getResource(Stream))
-        return Err;
-
-      // Modify the async info's stream.
-      AsyncInfoWrapper.setQueueAs<CUstream>(Stream);
-    }
+    auto WrapperStream =
+        AsyncInfoWrapper.getOrInitQueue<CUstream>(CUDAStreamManager);
+    if (!WrapperStream)
+      return WrapperStream.takeError();
+    Stream = *WrapperStream;
     return Plugin::success();
   }
 
@@ -642,17 +651,20 @@ struct CUDADeviceTy : public GenericDeviceTy {
   }
 
   /// Synchronize current thread with the pending operations on the async info.
-  Error synchronizeImpl(__tgt_async_info &AsyncInfo) override {
+  Error synchronizeImpl(__tgt_async_info &AsyncInfo,
+                        bool ReleaseQueue) override {
     CUstream Stream = reinterpret_cast<CUstream>(AsyncInfo.Queue);
     CUresult Res;
     Res = cuStreamSynchronize(Stream);
 
-    // Once the stream is synchronized, return it to stream pool and reset
-    // AsyncInfo. This is to make sure the synchronization only works for its
-    // own tasks.
-    AsyncInfo.Queue = nullptr;
-    if (auto Err = CUDAStreamManager.returnResource(Stream))
-      return Err;
+    // Once the stream is synchronized and we want to release the queue, return
+    // it to stream pool and reset AsyncInfo. This is to make sure the
+    // synchronization only works for its own tasks.
+    if (ReleaseQueue) {
+      AsyncInfo.Queue = nullptr;
+      if (auto Err = CUDAStreamManager.returnResource(Stream))
+        return Err;
+    }
 
     return Plugin::check(Res, "error in cuStreamSynchronize: %s");
   }
@@ -846,6 +858,64 @@ struct CUDADeviceTy : public GenericDeviceTy {
                          void *DstPtr, int64_t Size,
                          AsyncInfoWrapperTy &AsyncInfoWrapper) override;
 
+  Error dataFillImpl(void *TgtPtr, const void *PatternPtr, int64_t PatternSize,
+                     int64_t Size,
+                     AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+    if (auto Err = setContext())
+      return Err;
+
+    CUstream Stream;
+    if (auto Err = getStream(AsyncInfoWrapper, Stream))
+      return Err;
+
+    CUresult Res;
+    size_t N = Size / PatternSize;
+    if (PatternSize == 1) {
+      Res = cuMemsetD8Async((CUdeviceptr)TgtPtr,
+                            *(static_cast<const uint8_t *>(PatternPtr)), N,
+                            Stream);
+    } else if (PatternSize == 2) {
+      Res = cuMemsetD16Async((CUdeviceptr)TgtPtr,
+                             *(static_cast<const uint16_t *>(PatternPtr)), N,
+                             Stream);
+    } else if (PatternSize == 4) {
+      Res = cuMemsetD32Async((CUdeviceptr)TgtPtr,
+                             *(static_cast<const uint32_t *>(PatternPtr)), N,
+                             Stream);
+    } else {
+      // For larger patterns we can do a series of strided fills to copy the
+      // pattern efficiently
+      int64_t MemsetSize = PatternSize % 4u == 0u   ? 4u
+                           : PatternSize % 2u == 0u ? 2u
+                                                    : 1u;
+
+      int64_t NumberOfSteps = PatternSize / MemsetSize;
+      int64_t Pitch = NumberOfSteps * MemsetSize;
+      int64_t Height = Size / PatternSize;
+
+      for (auto Step = 0u; Step < NumberOfSteps; ++Step) {
+        if (MemsetSize == 4) {
+          Res = cuMemsetD2D32Async(
+              (CUdeviceptr)TgtPtr + Step * MemsetSize, Pitch,
+              *(static_cast<const uint32_t *>(PatternPtr) + Step), 1u, Height,
+              Stream);
+        } else if (MemsetSize == 2) {
+          Res = cuMemsetD2D16Async(
+              (CUdeviceptr)TgtPtr + Step * MemsetSize, Pitch,
+              *(static_cast<const uint16_t *>(PatternPtr) + Step), 1u, Height,
+              Stream);
+        } else {
+          Res = cuMemsetD2D8Async(
+              (CUdeviceptr)TgtPtr + Step * MemsetSize, Pitch,
+              *(static_cast<const uint8_t *>(PatternPtr) + Step), 1u, Height,
+              Stream);
+        }
+      }
+    }
+
+    return Plugin::check(Res, "error in cuMemset: %s");
+  }
+
   /// Initialize the async info for interoperability purposes.
   Error initAsyncInfoImpl(AsyncInfoWrapperTy &AsyncInfoWrapper) override {
     if (auto Err = setContext())
@@ -858,6 +928,13 @@ struct CUDADeviceTy : public GenericDeviceTy {
     return Plugin::success();
   }
 
+  /// Insert a data fence between previous data operations and the following
+  /// operations. This is a no-op for CUDA devices as operations inserted into
+  /// a queue are in-order.
+  Error dataFence(__tgt_async_info *Async) override {
+    return Plugin::success();
+  }
+
   /// Initialize the device info for interoperability purposes.
   Error initDeviceInfoImpl(__tgt_device_info *DeviceInfo) override {
     assert(Context && "Context is null");
@@ -875,6 +952,19 @@ struct CUDADeviceTy : public GenericDeviceTy {
     return Plugin::success();
   }
 
+  Error enqueueHostCallImpl(void (*Callback)(void *), void *UserData,
+                            AsyncInfoWrapperTy &AsyncInfo) override {
+    if (auto Err = setContext())
+      return Err;
+
+    CUstream Stream;
+    if (auto Err = getStream(AsyncInfo, Stream))
+      return Err;
+
+    CUresult Res = cuLaunchHostFunc(Stream, Callback, UserData);
+    return Plugin::check(Res, "error in cuStreamLaunchHostFunc: %s");
+  };
+
   /// Create an event.
   Error createEventImpl(void **EventPtrStorage) override {
     CUevent *Event = reinterpret_cast<CUevent *>(EventPtrStorage);
@@ -916,6 +1006,35 @@ struct CUDADeviceTy : public GenericDeviceTy {
     return Plugin::check(Res, "error in cuStreamWaitEvent: %s");
   }
 
+  Expected<bool> hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfo) override {
+    CUstream Stream;
+    if (auto Err = getStream(AsyncInfo, Stream))
+      return Err;
+
+    CUresult Ret = cuStreamQuery(Stream);
+    if (Ret == CUDA_SUCCESS)
+      return false;
+
+    if (Ret == CUDA_ERROR_NOT_READY)
+      return true;
+
+    return Plugin::check(Ret, "error in cuStreamQuery: %s");
+  }
+
+  Expected<bool> isEventCompleteImpl(void *EventPtr,
+                                     AsyncInfoWrapperTy &) override {
+    CUevent Event = reinterpret_cast<CUevent>(EventPtr);
+
+    CUresult Ret = cuEventQuery(Event);
+    if (Ret == CUDA_SUCCESS)
+      return true;
+
+    if (Ret == CUDA_ERROR_NOT_READY)
+      return false;
+
+    return Plugin::check(Ret, "error in cuEventQuery: %s");
+  }
+
   /// Synchronize the current thread with the event.
   Error syncEventImpl(void *EventPtr) override {
     CUevent Event = reinterpret_cast<CUevent>(EventPtr);
@@ -946,13 +1065,20 @@ struct CUDADeviceTy : public GenericDeviceTy {
 
     Info.add("Vendor Name", "NVIDIA", "", DeviceInfo::VENDOR);
 
+    Info.add("Vendor ID", uint64_t{4318}, "", DeviceInfo::VENDOR_ID);
+
+    Info.add("Memory Address Size", std::numeric_limits<CUdeviceptr>::digits,
+             "bits", DeviceInfo::ADDRESS_BITS);
+
     Res = cuDeviceTotalMem(&TmpSt, Device);
     if (Res == CUDA_SUCCESS)
-      Info.add("Global Memory Size", TmpSt, "bytes");
+      Info.add("Global Memory Size", TmpSt, "bytes",
+               DeviceInfo::GLOBAL_MEM_SIZE);
 
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add("Number of Multiprocessors", TmpInt);
+      Info.add("Number of Multiprocessors", TmpInt, "",
+               DeviceInfo::NUM_COMPUTE_UNITS);
 
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_GPU_OVERLAP, TmpInt);
     if (Res == CUDA_SUCCESS)
@@ -1013,7 +1139,8 @@ struct CUDADeviceTy : public GenericDeviceTy {
 
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_CLOCK_RATE, TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add("Clock Rate", TmpInt, "kHz");
+      Info.add("Clock Rate", TmpInt / 1000, "MHz",
+               DeviceInfo::MAX_CLOCK_FREQUENCY);
 
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_KERNEL_EXEC_TIMEOUT, TmpInt);
     if (Res == CUDA_SUCCESS)
@@ -1050,7 +1177,8 @@ struct CUDADeviceTy : public GenericDeviceTy {
 
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE, TmpInt);
     if (Res == CUDA_SUCCESS)
-      Info.add("Memory Clock Rate", TmpInt, "kHz");
+      Info.add("Memory Clock Rate", TmpInt / 1000, "MHz",
+               DeviceInfo::MEMORY_CLOCK_RATE);
 
     Res = getDeviceAttrRaw(CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH, TmpInt);
     if (Res == CUDA_SUCCESS)
@@ -1314,9 +1442,10 @@ Error CUDAKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   if (MaxDynCGroupMem >= MaxDynCGroupMemLimit) {
     CUresult AttrResult = cuFuncSetAttribute(
         Func, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, MaxDynCGroupMem);
-    return Plugin::check(
-        AttrResult,
-        "Error in cuLaunchKernel while setting the memory limits: %s");
+    if (auto Err = Plugin::check(
+            AttrResult,
+            "Error in cuLaunchKernel while setting the memory limits: %s"))
+      return Err;
     MaxDynCGroupMemLimit = MaxDynCGroupMem;
   }
 
diff --git a/offload/plugins-nextgen/host/src/rtl.cpp b/offload/plugins-nextgen/host/src/rtl.cpp
index d950572265b4c..f440ebaf17fe4 100644
--- a/offload/plugins-nextgen/host/src/rtl.cpp
+++ b/offload/plugins-nextgen/host/src/rtl.cpp
@@ -114,6 +114,14 @@ struct GenELF64KernelTy : public GenericKernelTy {
     return Plugin::success();
   }
 
+  /// Return maximum block size for maximum occupancy
+  Expected<uint64_t> maxGroupSize(GenericDeviceTy &Device,
+                                  uint64_t DynamicMemSize) const override {
+    return Plugin::error(
+        ErrorCode::UNSUPPORTED,
+        "occupancy calculations are not implemented for the host device");
+  }
+
 private:
   /// The kernel function to execute.
   void (*Func)(void);
@@ -295,9 +303,32 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
                          "dataExchangeImpl not supported");
   }
 
+  /// Insert a data fence between previous data operations and the following
+  /// operations. This is a no-op for Host devices as operations inserted into
+  /// a queue are in-order.
+  Error dataFence(__tgt_async_info *Async) override {
+    return Plugin::success();
+  }
+
+  Error dataFillImpl(void *TgtPtr, const void *PatternPtr, int64_t PatternSize,
+                     int64_t Size,
+                     AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+    if (PatternSize == 1) {
+      std::memset(TgtPtr, *static_cast<const char *>(PatternPtr), Size);
+    } else {
+      for (unsigned int Step = 0; Step < Size; Step += PatternSize) {
+        auto *Dst = static_cast<char *>(TgtPtr) + Step;
+        std::memcpy(Dst, PatternPtr, PatternSize);
+      }
+    }
+
+    return Plugin::success();
+  }
+
   /// All functions are already synchronous. No need to do anything on this
   /// synchronization function.
-  Error synchronizeImpl(__tgt_async_info &AsyncInfo) override {
+  Error synchronizeImpl(__tgt_async_info &AsyncInfo,
+                        bool ReleaseQueue) override {
     return Plugin::success();
   }
 
@@ -319,6 +350,12 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
                          "initDeviceInfoImpl not supported");
   }
 
+  Error enqueueHostCallImpl(void (*Callback)(void *), void *UserData,
+                            AsyncInfoWrapperTy &AsyncInfo) override {
+    Callback(UserData);
+    return Plugin::success();
+  };
+
   /// This plugin does not support the event API. Do nothing without failing.
   Error createEventImpl(void **EventPtrStorage) override {
     *EventPtrStorage = nullptr;
@@ -333,6 +370,13 @@ struct GenELF64DeviceTy : public GenericDeviceTy {
                       AsyncInfoWrapperTy &AsyncInfoWrapper) override {
     return Plugin::success();
   }
+  Expected<bool> hasPendingWorkImpl(AsyncInfoWrapperTy &AsyncInfo) override {
+    return true;
+  }
+  Expected<bool> isEventCompleteImpl(void *Event,
+                                     AsyncInfoWrapperTy &AsyncInfo) override {
+    return true;
+  }
   Error syncEventImpl(void *EventPtr) override { return Plugin::success(); }
 
   /// Print information about the device.
diff --git a/offload/test/lit.cfg b/offload/test/lit.cfg
index 800a63bc0ee32..f3e8e9a66685e 100644
--- a/offload/test/lit.cfg
+++ b/offload/test/lit.cfg
@@ -121,6 +121,7 @@ if config.libomptarget_test_pgo:
 # For all other targets, we currently assume it is.
 supports_unified_shared_memory = True
 supports_apu = False
+supports_large_allocation_memory_pool = False
 if config.libomptarget_current_target.startswith('nvptx'):
   try:
     cuda_arch = int(config.cuda_test_arch[:3])
@@ -132,9 +133,11 @@ if config.libomptarget_current_target.startswith('nvptx'):
 elif config.libomptarget_current_target.startswith('amdgcn'):
     # amdgpu_test_arch contains a list of AMD GPUs in the system
     # only check the first one assuming that we will run the test on it.
-    if not (config.amdgpu_test_arch.startswith("gfx90a") or
-            config.amdgpu_test_arch.startswith("gfx942") or
-            config.amdgpu_test_arch.startswith("gfx950")):
+    if (config.amdgpu_test_arch.startswith("gfx90a") or
+        config.amdgpu_test_arch.startswith("gfx942") or
+        config.amdgpu_test_arch.startswith("gfx950")):
+       supports_large_allocation_memory_pool = True
+    else:
        supports_unified_shared_memory = False
     # check if AMD architecture is an APU:
     if ((config.amdgpu_test_arch.startswith("gfx942") and
@@ -144,6 +147,8 @@ if supports_unified_shared_memory:
    config.available_features.add('unified_shared_memory')
 if supports_apu:
    config.available_features.add('apu')
+if supports_large_allocation_memory_pool:
+   config.available_features.add('large_allocation_memory_pool')
 
 # Setup environment to find dynamic library at runtime
 if config.operating_system == 'Windows':
diff --git a/offload/test/mapping/data_member_ref.cpp b/offload/test/mapping/data_member_ref.cpp
index fdb8abcaa6506..7947a62c169f4 100644
--- a/offload/test/mapping/data_member_ref.cpp
+++ b/offload/test/mapping/data_member_ref.cpp
@@ -60,7 +60,8 @@ int main() {
   printf("Host %d %d.\n", Bar.VRef.Data, V.Data);
   // CHECK: Host 123456.
   printf("Host %d.\n", *Baz.VRef.Data);
-#pragma omp target map(*Baz.VRef.Data) map(from : D1, D2)
+#pragma omp target map(Baz.VRef.Data) map(*Baz.VRef.Data) map(V1.Data[0 : 0])  \
+    map(from : D1, D2)
   {
     // CHECK: Device 123456.
     D1 = *Baz.VRef.Data;
diff --git a/offload/test/mapping/declare_mapper_nested_default_mappers.cpp b/offload/test/mapping/declare_mapper_nested_default_mappers.cpp
index c6c5657ae6166..45fd042aedb0e 100644
--- a/offload/test/mapping/declare_mapper_nested_default_mappers.cpp
+++ b/offload/test/mapping/declare_mapper_nested_default_mappers.cpp
@@ -44,8 +44,8 @@ int main() {
 
   int spp00fa = -1, spp00fca = -1, spp00fb_r = -1;
   __intptr_t p = reinterpret_cast<__intptr_t>(&x[0]);
-#pragma omp target map(tofrom: spp[0][0]) firstprivate(p)                           \
-                   map(from: spp00fa, spp00fca, spp00fb_r)
+#pragma omp target map(tofrom : spp[0][0]) map(alloc : spp[0]) firstprivate(p) \
+    map(from : spp00fa, spp00fca, spp00fb_r)
   {
     spp00fa = spp[0][0].f.a;
     spp00fca = spp[0][0].f.c.a;
diff --git a/offload/test/mapping/declare_mapper_nested_mappers.cpp b/offload/test/mapping/declare_mapper_nested_mappers.cpp
index a9e3f05e0f5fd..a59ed6980ec4c 100644
--- a/offload/test/mapping/declare_mapper_nested_mappers.cpp
+++ b/offload/test/mapping/declare_mapper_nested_mappers.cpp
@@ -42,8 +42,8 @@ int main() {
   int spp00fa = -1, spp00fb_r = -1, spp00fg1 = -1, spp00fg_r = -1;
   __intptr_t p = reinterpret_cast<__intptr_t>(&x[0]),
              p1 = reinterpret_cast<__intptr_t>(&y[0]);
-#pragma omp target map(tofrom : spp[0][0]) firstprivate(p, p1)                  \
-                   map(from: spp00fa, spp00fb_r, spp00fg1, spp00fg_r)
+#pragma omp target map(tofrom : spp[0][0]) map(alloc : spp[0])                 \
+    firstprivate(p, p1) map(from : spp00fa, spp00fb_r, spp00fg1, spp00fg_r)
   {
     spp00fa = spp[0][0].f.a;
     spp00fb_r = spp[0][0].f.b == reinterpret_cast<void *>(p) ? 1 : 0;
diff --git a/offload/test/mapping/ptr_and_obj_motion.c b/offload/test/mapping/ptr_and_obj_motion.c
index 8fa2c9865b4ac..a94c07aadc1bc 100644
--- a/offload/test/mapping/ptr_and_obj_motion.c
+++ b/offload/test/mapping/ptr_and_obj_motion.c
@@ -17,7 +17,7 @@ void init(double vertexx[]) {
 }
 
 void change(DV *dvptr) {
-#pragma omp target map(dvptr->dataptr[0 : 100])
+#pragma omp target map(dvptr->dataptr[0 : 100]) map(alloc : dvptr -> dataptr)
   {
     printf("In change: %lf, expected 77.0\n", dvptr->dataptr[77]);
     dvptr->dataptr[77] += 1.0;
diff --git a/offload/test/mapping/target_derefence_array_pointrs.cpp b/offload/test/mapping/target_derefence_array_pointrs.cpp
index a6dd4069a8f58..d213c87443634 100644
--- a/offload/test/mapping/target_derefence_array_pointrs.cpp
+++ b/offload/test/mapping/target_derefence_array_pointrs.cpp
@@ -18,23 +18,24 @@ void foo(int **t1d) {
 
   for (j = 0; j < 3; j++)
     (*t1d)[j] = 0;
-#pragma omp target map(tofrom : (*t1d)[0 : 3])
+#pragma omp target map(tofrom : (*t1d)[0 : 3]) map(alloc : *t1d)
   { (*t1d)[1] = 1; }
   // CHECK: 1
   printf("%d\n", (*t1d)[1]);
-#pragma omp target map(tofrom : (**t2d)[0 : 3])
+#pragma omp target map(tofrom : (**t2d)[0 : 3]) map(alloc : **t2d, *t2d)
   { (**t2d)[1] = 2; }
   // CHECK: 2
   printf("%d\n", (**t2d)[1]);
-#pragma omp target map(tofrom : (***t3d)[0 : 3])
+#pragma omp target map(tofrom : (***t3d)[0 : 3])                               \
+    map(alloc : ***t3d, **t3d, *t3d)
   { (***t3d)[1] = 3; }
   // CHECK: 3
   printf("%d\n", (***t3d)[1]);
-#pragma omp target map(tofrom : (**t1d))
+#pragma omp target map(tofrom : (**t1d)) map(alloc : *t1d)
   { (*t1d)[0] = 4; }
   // CHECK: 4
   printf("%d\n", (*t1d)[0]);
-#pragma omp target map(tofrom : (*(*(t1d + a) + b)))
+#pragma omp target map(tofrom : (*(*(t1d + a) + b))) map(to : *(t1d + a))
   { *(*(t1d + a) + b) = 5; }
   // CHECK: 5
   printf("%d\n", *(*(t1d + a) + b));
@@ -49,7 +50,7 @@ void bar() {
   for (int i = 0; i < 3; i++) {
     (**a)[1] = i;
   }
-#pragma omp target map((**a)[ : 3])
+#pragma omp target map((**a)[ : 3]) map(alloc : **a, *a)
   {
     (**a)[1] = 6;
     // CHECK: 6
@@ -73,7 +74,8 @@ void zoo(int **f, SSA *sa) {
   *(f + sa->i + 1) = t;
   *(sa->sa->i + *(f + sa->i + 1)) = 4;
   printf("%d\n", *(sa->sa->i + *(1 + sa->i + f)));
-#pragma omp target map(sa, *(sa->sa->i + *(1 + sa->i + f)))
+#pragma omp target map(*(sa->sa->i + *(1 + sa->i + f))) map(alloc : sa->sa)    \
+    map(to : sa->i) map(to : sa->sa->i) map(to : *(1 + sa->i + f))
   { *(sa->sa->i + *(1 + sa->i + f)) = 7; }
   // CHECK: 7
   printf("%d\n", *(sa->sa->i + *(1 + sa->i + f)));
@@ -87,13 +89,13 @@ void xoo() {
 
 void yoo(int **x) {
   *x = (int *)malloc(2 * sizeof(int));
-#pragma omp target map(**x)
+#pragma omp target map(**x) map(alloc : *x)
   {
     **x = 8;
     // CHECK: 8
     printf("%d\n", **x);
   }
-#pragma omp target map(*(*x + 1))
+#pragma omp target map(*(*x + 1)) map(alloc : *x)
   {
     *(*x + 1) = 9;
     // CHECK: 9
diff --git a/offload/test/mapping/target_has_device_addr.c b/offload/test/mapping/target_has_device_addr.c
index e8bfff868c7ed..f238832c44054 100644
--- a/offload/test/mapping/target_has_device_addr.c
+++ b/offload/test/mapping/target_has_device_addr.c
@@ -66,8 +66,9 @@ void zoo() {
   short **xpp = &xp[0];
 
   x[1] = 111;
-#pragma omp target data map(tofrom : xpp[1][1]) use_device_addr(xpp[1][1])
-#pragma omp target has_device_addr(xpp[1][1])
+#pragma omp target data map(tofrom : xpp[1][1]) map(xpp[1])                    \
+    use_device_addr(xpp[1])
+#pragma omp target has_device_addr(xpp[1])
   {
     xpp[1][1] = 222;
     // CHECK: 222
diff --git a/offload/test/offloading/fortran/declare-target-automap.f90 b/offload/test/offloading/fortran/declare-target-automap.f90
new file mode 100644
index 0000000000000..b9c2d34c834fa
--- /dev/null
+++ b/offload/test/offloading/fortran/declare-target-automap.f90
@@ -0,0 +1,37 @@
+!Offloading test for AUTOMAP modifier in declare target enter
+! REQUIRES: flang, amdgpu
+
+! RUN: %libomptarget-compile-fortran-run-and-check-generic
+program automap_program
+   use iso_c_binding, only: c_loc
+   use omp_lib, only: omp_get_default_device, omp_target_is_present
+   integer, parameter :: N = 10
+   integer :: i
+   integer, allocatable, target :: automap_array(:)
+   !$omp declare target enter(automap:automap_array)
+
+   ! false since the storage is not present even though the descriptor is present
+   write (*, *) omp_target_is_present(c_loc(automap_array), omp_get_default_device())
+   ! CHECK: 0
+
+   allocate (automap_array(N))
+   ! true since the storage should be allocated and reference count incremented by the allocate
+   write (*, *) omp_target_is_present(c_loc(automap_array), omp_get_default_device())
+   ! CHECK: 1
+
+   ! since storage is present this should not be a runtime error
+   !$omp target teams loop
+   do i = 1, N
+      automap_array(i) = i
+   end do
+
+   !$omp target update from(automap_array)
+   write (*, *) automap_array
+   ! CHECK: 1 2 3 4 5 6 7 8 9 10
+
+   deallocate (automap_array)
+
+   ! automap_array should have it's storage unmapped on device here
+   write (*, *) omp_target_is_present(c_loc(automap_array), omp_get_default_device())
+   ! CHECK: 0
+end program
diff --git a/offload/test/offloading/strided_multiple_update.c b/offload/test/offloading/strided_multiple_update.c
new file mode 100644
index 0000000000000..a3e8d10863aef
--- /dev/null
+++ b/offload/test/offloading/strided_multiple_update.c
@@ -0,0 +1,62 @@
+// This test checks that #pragma omp target update from(data1[0:3:4],
+// data2[0:2:5]) correctly updates disjoint strided sections of multiple arrays
+// from the device to the host.
+
+// RUN: %libomptarget-compile-run-and-check-generic
+#include <omp.h>
+#include <stdio.h>
+
+int main() {
+  int len = 12;
+  double data1[len], data2[len];
+
+// Initial values
+#pragma omp target map(tofrom : data1[0 : len], data2[0 : len])
+  {
+    for (int i = 0; i < len; i++) {
+      data1[i] = i;
+      data2[i] = i * 10;
+    }
+  }
+
+  printf("original host array values:\n");
+  printf("data1: ");
+  for (int i = 0; i < len; i++)
+    printf("%.1f ", data1[i]);
+  printf("\ndata2: ");
+  for (int i = 0; i < len; i++)
+    printf("%.1f ", data2[i]);
+  printf("\n\n");
+
+#pragma omp target data map(to : data1[0 : len], data2[0 : len])
+  {
+// Modify arrays on device
+#pragma omp target
+    {
+      for (int i = 0; i < len; i++)
+        data1[i] += i;
+      for (int i = 0; i < len; i++)
+        data2[i] += 100;
+    }
+
+// data1[0:3:4]  // indices 0,4,8
+// data2[0:2:5]  // indices 0,5
+#pragma omp target update from(data1[0 : 3 : 4], data2[0 : 2 : 5])
+  }
+
+  printf("device array values after update from:\n");
+  printf("data1: ");
+  for (int i = 0; i < len; i++)
+    printf("%.1f ", data1[i]);
+  printf("\ndata2: ");
+  for (int i = 0; i < len; i++)
+    printf("%.1f ", data2[i]);
+  printf("\n\n");
+
+  // CHECK: data1: 0.0 1.0 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 10.0 11.0
+  // CHECK: data2: 0.0 10.0 20.0 30.0 40.0 50.0 60.0 70.0 80.0 90.0 100.0 110.0
+
+  // CHECK: data1: 0.0 1.0 2.0 3.0 8.0 5.0 6.0 7.0 16.0 9.0 10.0 11.0
+  // CHECK: data2: 100.0 10.0 20.0 30.0 40.0 150.0 60.0 70.0 80.0 90.0 100.0
+  // 110.0
+}
diff --git a/offload/test/offloading/strided_partial_update.c b/offload/test/offloading/strided_partial_update.c
new file mode 100644
index 0000000000000..15d477f2b9b78
--- /dev/null
+++ b/offload/test/offloading/strided_partial_update.c
@@ -0,0 +1,63 @@
+// This test checks that #pragma omp target update from(data[0:4:3]) correctly
+// updates every third element (stride 3) from the device to the host, partially
+// across the array
+
+// RUN: %libomptarget-compile-run-and-check-generic
+#include <omp.h>
+#include <stdio.h>
+
+int main() {
+  int len = 11;
+  double data[len];
+
+#pragma omp target map(tofrom : data[0 : len])
+  {
+    for (int i = 0; i < len; i++)
+      data[i] = i;
+  }
+
+  // Initial values
+  printf("original host array values:\n");
+  for (int i = 0; i < len; i++)
+    printf("%f\n", data[i]);
+  printf("\n");
+
+#pragma omp target data map(to : data[0 : len])
+  {
+// Modify arrays on device
+#pragma omp target
+    for (int i = 0; i < len; i++)
+      data[i] += i;
+
+#pragma omp target update from(data[0 : 4 : 3]) // indices 0,3,6,9
+  }
+
+  printf("device array values after update from:\n");
+  for (int i = 0; i < len; i++)
+    printf("%f\n", data[i]);
+  printf("\n");
+
+  // CHECK: 0.000000
+  // CHECK: 1.000000
+  // CHECK: 2.000000
+  // CHECK: 3.000000
+  // CHECK: 4.000000
+  // CHECK: 5.000000
+  // CHECK: 6.000000
+  // CHECK: 7.000000
+  // CHECK: 8.000000
+  // CHECK: 9.000000
+  // CHECK: 10.000000
+
+  // CHECK: 0.000000
+  // CHECK: 1.000000
+  // CHECK: 2.000000
+  // CHECK: 6.000000
+  // CHECK: 4.000000
+  // CHECK: 5.000000
+  // CHECK: 12.000000
+  // CHECK: 7.000000
+  // CHECK: 8.000000
+  // CHECK: 18.000000
+  // CHECK: 10.000000
+}
diff --git a/offload/test/offloading/strided_update.c b/offload/test/offloading/strided_update.c
new file mode 100644
index 0000000000000..fe875b7fd55c9
--- /dev/null
+++ b/offload/test/offloading/strided_update.c
@@ -0,0 +1,54 @@
+// This test checks that "update from" clause in OpenMP is supported when the
+// elements are updated in a non-contiguous manner. This test checks that
+// #pragma omp target update from(data[0:4:2]) correctly updates only every
+// other element (stride 2) from the device to the host
+
+// RUN: %libomptarget-compile-run-and-check-generic
+#include <omp.h>
+#include <stdio.h>
+
+int main() {
+  int len = 8;
+  double data[len];
+#pragma omp target map(tofrom : len, data[0 : len])
+  {
+    for (int i = 0; i < len; i++) {
+      data[i] = i;
+    }
+  }
+  // Initial values
+  printf("original host array values:\n");
+  for (int i = 0; i < len; i++)
+    printf("%f\n", data[i]);
+  printf("\n");
+
+#pragma omp target data map(to : len, data[0 : len])
+  {
+// Modify arrays on device
+#pragma omp target
+    for (int i = 0; i < len; i++) {
+      data[i] += i;
+    }
+
+#pragma omp target update from(data[0 : 4 : 2])
+  }
+  // CHECK: 0.000000
+  // CHECK: 1.000000
+  // CHECK: 4.000000
+  // CHECK: 3.000000
+  // CHECK: 8.000000
+  // CHECK: 5.000000
+  // CHECK: 12.000000
+  // CHECK: 7.000000
+  // CHECK-NOT: 2.000000
+  // CHECK-NOT: 6.000000
+  // CHECK-NOT: 10.000000
+  // CHECK-NOT: 14.000000
+
+  printf("from target array results:\n");
+  for (int i = 0; i < len; i++)
+    printf("%f\n", data[i]);
+  printf("\n");
+
+  return 0;
+}
diff --git a/offload/test/sanitizer/use_after_free_2.c b/offload/test/sanitizer/use_after_free_2.c
index 02aa453d0a975..1c1e09744a750 100644
--- a/offload/test/sanitizer/use_after_free_2.c
+++ b/offload/test/sanitizer/use_after_free_2.c
@@ -10,6 +10,9 @@
 // UNSUPPORTED: s390x-ibm-linux-gnu
 // UNSUPPORTED: s390x-ibm-linux-gnu-LTO
 
+// If offload memory pooling is enabled for a large allocation, reuse error is
+// not detected. UNSUPPORTED: large_allocation_memory_pool
+
 #include <omp.h>
 
 int main() {
diff --git a/offload/test/sanitizer/use_after_free_3.c b/offload/test/sanitizer/use_after_free_3.c
new file mode 100644
index 0000000000000..9d8861433e7e5
--- /dev/null
+++ b/offload/test/sanitizer/use_after_free_3.c
@@ -0,0 +1,37 @@
+// clang-format off
+// RUN: %libomptarget-compileopt-generic
+// RUN: %not --crash env -u LLVM_DISABLE_SYMBOLIZATION OFFLOAD_TRACK_ALLOCATION_TRACES=1 LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD=1024 %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefixes=CHECK
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefixes=CHECK-PASS
+// clang-format on
+
+// If offload memory pooling is enabled for a large allocation, reuse error is
+// not detected. Run the test w/ and w/o ENV var override on memory pooling
+// threshold. REQUIRES: large_allocation_memory_pool
+
+#include <omp.h>
+#include <stdio.h>
+
+int main() {
+  int N = (1 << 30);
+  char *A = (char *)malloc(N);
+  char *P;
+#pragma omp target map(A[ : N]) map(from : P)
+  {
+    P = &A[N / 2];
+    *P = 3;
+  }
+  // clang-format off
+// CHECK: OFFLOAD ERROR: memory access fault by GPU {{.*}} (agent 0x{{.*}}) at virtual address [[PTR:0x[0-9a-z]*]]. Reasons: {{.*}}
+// CHECK: Device pointer [[PTR]] points into prior host-issued allocation:
+// CHECK: Last deallocation:
+// CHECK: Last allocation of size 1073741824
+// clang-format on
+#pragma omp target
+  {
+    *P = 5;
+  }
+
+  // CHECK-PASS: PASS
+  printf("PASS\n");
+  return 0;
+}
diff --git a/offload/tools/offload-tblgen/APIGen.cpp b/offload/tools/offload-tblgen/APIGen.cpp
index 8c61d1f12de7a..1e79c00ae06c5 100644
--- a/offload/tools/offload-tblgen/APIGen.cpp
+++ b/offload/tools/offload-tblgen/APIGen.cpp
@@ -131,7 +131,8 @@ static void ProcessEnum(const EnumRec &Enum, raw_ostream &OS) {
   OS << formatv("/// @brief {0}\n", Enum.getDesc());
   OS << formatv("typedef enum {0} {{\n", Enum.getName());
 
-  uint32_t EtorVal = 0;
+  // Bitfields start from 1, other enums from 0
+  uint32_t EtorVal = Enum.isBitField();
   for (const auto &EnumVal : Enum.getValues()) {
     if (Enum.isTyped()) {
       OS << MakeComment(
@@ -141,7 +142,12 @@ static void ProcessEnum(const EnumRec &Enum, raw_ostream &OS) {
       OS << MakeComment(EnumVal.getDesc());
     }
     OS << formatv(TAB_1 "{0}_{1} = {2},\n", Enum.getEnumValNamePrefix(),
-                  EnumVal.getName(), EtorVal++);
+                  EnumVal.getName(), EtorVal);
+    if (Enum.isBitField()) {
+      EtorVal <<= 1u;
+    } else {
+      ++EtorVal;
+    }
   }
 
   // Add last_element/force uint32 val
@@ -220,31 +226,23 @@ OL_APIEXPORT ol_result_t OL_APICALL {0}WithCodeLoc(
 void EmitOffloadAPI(const RecordKeeper &Records, raw_ostream &OS) {
   OS << GenericHeader;
   OS << FileHeader;
-  // Generate main API definitions
-  for (auto *R : Records.getAllDerivedDefinitions("APIObject")) {
-    if (R->isSubClassOf("Macro")) {
-      ProcessMacro(MacroRec{R}, OS);
-    } else if (R->isSubClassOf("Typedef")) {
-      ProcessTypedef(TypedefRec{R}, OS);
-    } else if (R->isSubClassOf("Handle")) {
-      ProcessHandle(HandleRec{R}, OS);
-    } else if (R->isSubClassOf("Function")) {
-      ProcessFunction(FunctionRec{R}, OS);
-    } else if (R->isSubClassOf("Enum")) {
-      ProcessEnum(EnumRec{R}, OS);
-    } else if (R->isSubClassOf("Struct")) {
-      ProcessStruct(StructRec{R}, OS);
-    } else if (R->isSubClassOf("FptrTypedef")) {
-      ProcessFptrTypedef(FptrTypedefRec{R}, OS);
-    }
-  }
 
-  // Generate auxiliary definitions (func param structs etc)
+  // Generate main API definitions
+  for (auto *R : Records.getAllDerivedDefinitions("Macro"))
+    ProcessMacro(MacroRec{R}, OS);
+  for (auto *R : Records.getAllDerivedDefinitions("Handle"))
+    ProcessHandle(HandleRec{R}, OS);
+  for (auto *R : Records.getAllDerivedDefinitions("Enum"))
+    ProcessEnum(EnumRec{R}, OS);
+  for (auto *R : Records.getAllDerivedDefinitions("Typedef"))
+    ProcessTypedef(TypedefRec{R}, OS);
+  for (auto *R : Records.getAllDerivedDefinitions("FptrTypedef"))
+    ProcessFptrTypedef(FptrTypedefRec{R}, OS);
+  for (auto *R : Records.getAllDerivedDefinitions("Struct"))
+    ProcessStruct(StructRec{R}, OS);
   for (auto *R : Records.getAllDerivedDefinitions("Function")) {
     ProcessFuncParamStruct(FunctionRec{R}, OS);
-  }
-
-  for (auto *R : Records.getAllDerivedDefinitions("Function")) {
+    ProcessFunction(FunctionRec{R}, OS);
     ProcessFuncWithCodeLocVariant(FunctionRec{R}, OS);
   }
 
diff --git a/offload/tools/offload-tblgen/MiscGen.cpp b/offload/tools/offload-tblgen/MiscGen.cpp
index b90e5cfdec8b2..8a8b9caf2348d 100644
--- a/offload/tools/offload-tblgen/MiscGen.cpp
+++ b/offload/tools/offload-tblgen/MiscGen.cpp
@@ -86,7 +86,7 @@ void EmitOffloadErrcodes(const RecordKeeper &Records, raw_ostream &OS) {
 
 )";
 
-  auto ErrorCodeEnum = EnumRec{Records.getDef("ErrorCode")};
+  auto ErrorCodeEnum = EnumRec{Records.getDef("ol_errc_t")};
   uint32_t EtorVal = 0;
   for (const auto &EnumVal : ErrorCodeEnum.getValues()) {
     OS << formatv(TAB_1 "OFFLOAD_ERRC({0}, \"{1}\", {2})\n", EnumVal.getName(),
@@ -107,10 +107,16 @@ void EmitOffloadInfo(const RecordKeeper &Records, raw_ostream &OS) {
 
 )";
 
-  auto ErrorCodeEnum = EnumRec{Records.getDef("DeviceInfo")};
-  uint32_t EtorVal = 0;
-  for (const auto &EnumVal : ErrorCodeEnum.getValues()) {
+  auto Enum = EnumRec{Records.getDef("ol_device_info_t")};
+  // Bitfields start from 1, other enums from 0
+  uint32_t EtorVal = Enum.isBitField();
+  for (const auto &EnumVal : Enum.getValues()) {
     OS << formatv(TAB_1 "OFFLOAD_DEVINFO({0}, \"{1}\", {2})\n",
-                  EnumVal.getName(), EnumVal.getDesc(), EtorVal++);
+                  EnumVal.getName(), EnumVal.getDesc(), EtorVal);
+    if (Enum.isBitField()) {
+      EtorVal <<= 1u;
+    } else {
+      ++EtorVal;
+    }
   }
 }
diff --git a/offload/tools/offload-tblgen/RecordTypes.hpp b/offload/tools/offload-tblgen/RecordTypes.hpp
index 65c0a4ce4a2c7..2abd9e10f0f96 100644
--- a/offload/tools/offload-tblgen/RecordTypes.hpp
+++ b/offload/tools/offload-tblgen/RecordTypes.hpp
@@ -16,25 +16,30 @@ namespace llvm {
 namespace offload {
 namespace tblgen {
 
-class HandleRec {
+class APIObject {
 public:
-  explicit HandleRec(const Record *rec) : rec(rec) {}
-  StringRef getName() const { return rec->getValueAsString("name"); }
+  StringRef getName() const { return rec->getName(); }
   StringRef getDesc() const { return rec->getValueAsString("desc"); }
 
-private:
+protected:
+  APIObject(const Record *rec) : rec(rec) {}
   const Record *rec;
 };
 
-class MacroRec {
+class HandleRec : public APIObject {
 public:
-  explicit MacroRec(const Record *rec) : rec(rec) {
-    auto Name = rec->getValueAsString("name");
+  explicit HandleRec(const Record *rec) : APIObject(rec) {};
+};
+
+class MacroRec : public APIObject {
+public:
+  explicit MacroRec(const Record *rec) : APIObject(rec) {
+    auto Name = rec->getName();
     auto OpenBrace = Name.find_first_of("(");
     nameWithoutArgs = Name.substr(0, OpenBrace);
   }
   StringRef getName() const { return nameWithoutArgs; }
-  StringRef getNameWithArgs() const { return rec->getValueAsString("name"); }
+  StringRef getNameWithArgs() const { return rec->getName(); }
   StringRef getDesc() const { return rec->getValueAsString("desc"); }
 
   std::optional<StringRef> getCondition() const {
@@ -46,19 +51,15 @@ class MacroRec {
   }
 
 private:
-  const Record *rec;
   std::string nameWithoutArgs;
 };
 
-class TypedefRec {
+class TypedefRec : public APIObject {
 public:
-  explicit TypedefRec(const Record *rec) : rec(rec) {}
-  StringRef getName() const { return rec->getValueAsString("name"); }
-  StringRef getDesc() const { return rec->getValueAsString("desc"); }
-  StringRef getValue() const { return rec->getValueAsString("value"); }
+  explicit TypedefRec(const Record *rec) : APIObject(rec) {};
 
-private:
-  const Record *rec;
+public:
+  StringRef getValue() const { return rec->getValueAsString("value"); }
 };
 
 class EnumValueRec {
@@ -74,15 +75,13 @@ class EnumValueRec {
   const Record *rec;
 };
 
-class EnumRec {
+class EnumRec : public APIObject {
 public:
-  explicit EnumRec(const Record *rec) : rec(rec) {
+  explicit EnumRec(const Record *rec) : APIObject(rec) {
     for (const auto *Val : rec->getValueAsListOfDefs("etors")) {
       vals.emplace_back(EnumValueRec{Val});
     }
   }
-  StringRef getName() const { return rec->getValueAsString("name"); }
-  StringRef getDesc() const { return rec->getValueAsString("desc"); }
   const std::vector<EnumValueRec> &getValues() const { return vals; }
 
   std::string getEnumValNamePrefix() const {
@@ -92,8 +91,9 @@ class EnumRec {
 
   bool isTyped() const { return rec->getValueAsBit("is_typed"); }
 
+  bool isBitField() const { return rec->getValueAsBit("is_bit_field"); }
+
 private:
-  const Record *rec;
   std::vector<EnumValueRec> vals;
 };
 
@@ -110,22 +110,19 @@ class StructMemberRec {
   const Record *rec;
 };
 
-class StructRec {
+class StructRec : public APIObject {
 public:
-  explicit StructRec(const Record *rec) : rec(rec) {
+  explicit StructRec(const Record *rec) : APIObject(rec) {
     for (auto *Member : rec->getValueAsListOfDefs("all_members")) {
       members.emplace_back(StructMemberRec(Member));
     }
   }
-  StringRef getName() const { return rec->getValueAsString("name"); }
-  StringRef getDesc() const { return rec->getValueAsString("desc"); }
   std::optional<StringRef> getBaseClass() const {
     return rec->getValueAsOptionalString("base_class");
   }
   const std::vector<StructMemberRec> &getMembers() const { return members; }
 
 private:
-  const Record *rec;
   std::vector<StructMemberRec> members;
 };
 
@@ -205,9 +202,9 @@ class ReturnRec {
   const Record *rec;
 };
 
-class FunctionRec {
+class FunctionRec : public APIObject {
 public:
-  FunctionRec(const Record *rec) : rec(rec) {
+  FunctionRec(const Record *rec) : APIObject(rec) {
     for (auto &Ret : rec->getValueAsListOfDefs("all_returns"))
       rets.emplace_back(Ret);
     for (auto &Param : rec->getValueAsListOfDefs("params"))
@@ -219,11 +216,9 @@ class FunctionRec {
                          llvm::convertToSnakeFromCamelCase(getName()));
   }
 
-  StringRef getName() const { return rec->getValueAsString("name"); }
   StringRef getClass() const { return rec->getValueAsString("api_class"); }
   const std::vector<ReturnRec> &getReturns() const { return rets; }
   const std::vector<ParamRec> &getParams() const { return params; }
-  StringRef getDesc() const { return rec->getValueAsString("desc"); }
   std::vector<StringRef> getDetails() const {
     return rec->getValueAsListOfStrings("details");
   }
@@ -234,25 +229,19 @@ class FunctionRec {
 private:
   std::vector<ReturnRec> rets;
   std::vector<ParamRec> params;
-
-  const Record *rec;
 };
 
-class FptrTypedefRec {
+class FptrTypedefRec : public APIObject {
 public:
-  explicit FptrTypedefRec(const Record *rec) : rec(rec) {
+  explicit FptrTypedefRec(const Record *rec) : APIObject(rec) {
     for (auto &Param : rec->getValueAsListOfDefs("params"))
       params.emplace_back(Param);
   }
-  StringRef getName() const { return rec->getValueAsString("name"); }
-  StringRef getDesc() const { return rec->getValueAsString("desc"); }
   StringRef getReturn() const { return rec->getValueAsString("return"); }
   const std::vector<ParamRec> &getParams() const { return params; }
 
 private:
   std::vector<ParamRec> params;
-
-  const Record *rec;
 };
 
 } // namespace tblgen
diff --git a/offload/unittests/CMakeLists.txt b/offload/unittests/CMakeLists.txt
index 726d3eef59c8f..308849a8364ac 100644
--- a/offload/unittests/CMakeLists.txt
+++ b/offload/unittests/CMakeLists.txt
@@ -18,6 +18,9 @@ endif ()
 set(OFFLOAD_UNITTESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 function(add_offload_test_device_code test_filename test_name)
+  cmake_parse_arguments(
+    "OFFLOAD_TESTS" "WITH_DEVICE_MATH_LIBS" "" "" ${ARGN})
+
   set(SRC_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${test_filename})
   set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
 
@@ -37,13 +40,25 @@ function(add_offload_test_device_code test_filename test_name)
     endif()
 
     if(nvptx_arch AND CUDAToolkit_FOUND)
+      set(nvptx_compile_flags ${OFFLOAD_TESTS_UNPARSED_ARGUMENTS})
+
+      if(OFFLOAD_TESTS_WITH_DEVICE_MATH_LIBS)
+        file(GLOB libdevice_paths "${CUDAToolkit_LIBRARY_ROOT}/nvvm/libdevice/libdevice.*.bc")
+        if(libdevice_paths)
+          list(GET libdevice_paths 0 libdevice_path)
+          list(APPEND nvptx_compile_flags "-Xclang" "-mlink-builtin-bitcode")
+          list(APPEND nvptx_compile_flags "-Xclang" "${libdevice_path}")
+          list(APPEND nvptx_compile_flags "-DCUDA_MATH_FOUND=1")
+        endif()
+      endif()
+
       set(output_file "${CMAKE_CURRENT_BINARY_DIR}/${test_name}.nvptx64.bin")
       add_custom_command(
         OUTPUT ${output_file}
         COMMAND ${CMAKE_CXX_COMPILER}
         -I${OFFLOAD_UNITTESTS_DIR}
         --target=nvptx64-nvidia-cuda -march=${nvptx_arch}
-        -nogpulib --cuda-path=${cuda_path} -flto ${ARGN}
+        -nogpulib --cuda-path=${cuda_path} -flto ${nvptx_compile_flags}
         ${SRC_PATH} -o ${output_file}
         DEPENDS ${SRC_PATH}
       )
@@ -62,13 +77,25 @@ function(add_offload_test_device_code test_filename test_name)
     endif()
 
     if(amdgpu_arch)
+      set(amdgpu_compile_flags ${OFFLOAD_TESTS_UNPARSED_ARGUMENTS})
+
+      if(OFFLOAD_TESTS_WITH_DEVICE_MATH_LIBS)
+        find_package(AMDDeviceLibs QUIET HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/rocm)
+        if(AMDDeviceLibs_FOUND)
+          get_target_property(ocml_path ocml IMPORTED_LOCATION)
+          list(APPEND amdgpu_compile_flags "-Xclang" "-mlink-builtin-bitcode")
+          list(APPEND amdgpu_compile_flags "-Xclang" "${ocml_path}")
+          list(APPEND amdgpu_compile_flags "-DHIP_MATH_FOUND=1")
+        endif()
+      endif()
+
       set(output_file "${CMAKE_CURRENT_BINARY_DIR}/${test_name}.amdgpu.bin")
       add_custom_command(
         OUTPUT ${output_file}
         COMMAND ${CMAKE_CXX_COMPILER}
         -I${OFFLOAD_UNITTESTS_DIR}
         --target=amdgcn-amd-amdhsa -mcpu=${amdgpu_arch}
-        -nogpulib -flto ${ARGN} ${SRC_PATH} -o ${output_file}
+        -nogpulib -flto ${amdgpu_compile_flags} ${SRC_PATH} -o ${output_file}
         DEPENDS ${SRC_PATH}
       )
       add_custom_target(${test_name}.amdgpu DEPENDS ${output_file})
@@ -104,11 +131,6 @@ function(add_conformance_test test_name)
 
   list(TRANSFORM ARGN PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/" OUTPUT_VARIABLE files)
 
-  if(NOT TARGET libc)
-    message(WARNING "Cannot run conformance tests without the LLVM C library")
-    return()
-  endif()
-
   add_executable(${target_name} ${files})
   add_dependencies(${target_name} conformance_device_binaries)
   target_compile_definitions(${target_name}
diff --git a/offload/unittests/Conformance/device_code/CMakeLists.txt b/offload/unittests/Conformance/device_code/CMakeLists.txt
index 789dd167bb9ff..a0c5369f24ae1 100644
--- a/offload/unittests/Conformance/device_code/CMakeLists.txt
+++ b/offload/unittests/Conformance/device_code/CMakeLists.txt
@@ -1,4 +1,10 @@
+add_offload_test_device_code(CUDAMath.cpp cuda-math WITH_DEVICE_MATH_LIBS -O3 -stdlib -fno-builtin)
+add_offload_test_device_code(HIPMath.cpp hip-math WITH_DEVICE_MATH_LIBS -O3 -stdlib -fno-builtin)
 add_offload_test_device_code(LLVMLibm.cpp llvm-libm -O3 -stdlib -fno-builtin)
 
-add_custom_target(conformance_device_binaries DEPENDS llvm-libm.bin)
+add_custom_target(conformance_device_binaries DEPENDS
+  cuda-math.bin
+  hip-math.bin
+  llvm-libm.bin
+)
 set(OFFLOAD_CONFORMANCE_DEVICE_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
diff --git a/offload/unittests/Conformance/device_code/CUDAMath.cpp b/offload/unittests/Conformance/device_code/CUDAMath.cpp
new file mode 100644
index 0000000000000..d80660b2e3c74
--- /dev/null
+++ b/offload/unittests/Conformance/device_code/CUDAMath.cpp
@@ -0,0 +1,300 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the implementation of the device kernels that wrap the
+/// math functions from the cuda-math provider.
+///
+//===----------------------------------------------------------------------===//
+
+#ifdef CUDA_MATH_FOUND
+
+#include "Conformance/device_code/DeviceAPIs.hpp"
+#include "Conformance/device_code/KernelRunner.hpp"
+
+#include <gpuintrin.h>
+#include <stddef.h>
+
+using namespace kernels;
+
+//===----------------------------------------------------------------------===//
+// Helpers
+//===----------------------------------------------------------------------===//
+
+static inline float powfRoundedExponent(float Base, float Exponent) {
+  return __nv_powf(Base, __nv_roundf(Exponent));
+}
+
+static inline double sincosSin(double X) {
+  double SinX, CosX;
+  __nv_sincos(X, &SinX, &CosX);
+  return SinX;
+}
+
+static inline double sincosCos(double X) {
+  double SinX, CosX;
+  __nv_sincos(X, &SinX, &CosX);
+  return CosX;
+}
+
+static inline float sincosfSin(float X) {
+  float SinX, CosX;
+  __nv_sincosf(X, &SinX, &CosX);
+  return SinX;
+}
+
+static inline float sincosfCos(float X) {
+  float SinX, CosX;
+  __nv_sincosf(X, &SinX, &CosX);
+  return CosX;
+}
+
+//===----------------------------------------------------------------------===//
+// Kernels
+//===----------------------------------------------------------------------===//
+
+extern "C" {
+
+__gpu_kernel void acosKernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_acos>(NumElements, Out, X);
+}
+
+__gpu_kernel void acosfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_acosf>(NumElements, Out, X);
+}
+
+__gpu_kernel void acoshfKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_acoshf>(NumElements, Out, X);
+}
+
+__gpu_kernel void asinKernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_asin>(NumElements, Out, X);
+}
+
+__gpu_kernel void asinfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_asinf>(NumElements, Out, X);
+}
+
+__gpu_kernel void asinhfKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_asinhf>(NumElements, Out, X);
+}
+
+__gpu_kernel void atanfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_atanf>(NumElements, Out, X);
+}
+
+__gpu_kernel void atan2fKernel(const float *X, const float *Y, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_atan2f>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void atanhfKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_atanhf>(NumElements, Out, X);
+}
+
+__gpu_kernel void cbrtKernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_cbrt>(NumElements, Out, X);
+}
+
+__gpu_kernel void cbrtfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_cbrtf>(NumElements, Out, X);
+}
+
+__gpu_kernel void cosKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__nv_cos>(NumElements, Out, X);
+}
+
+__gpu_kernel void cosfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_cosf>(NumElements, Out, X);
+}
+
+__gpu_kernel void coshfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_coshf>(NumElements, Out, X);
+}
+
+__gpu_kernel void cospifKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_cospif>(NumElements, Out, X);
+}
+
+__gpu_kernel void erffKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_erff>(NumElements, Out, X);
+}
+
+__gpu_kernel void expKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__nv_exp>(NumElements, Out, X);
+}
+
+__gpu_kernel void expfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_expf>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp10Kernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_exp10>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp10fKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_exp10f>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp2Kernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_exp2>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp2fKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_exp2f>(NumElements, Out, X);
+}
+
+__gpu_kernel void expm1Kernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_expm1>(NumElements, Out, X);
+}
+
+__gpu_kernel void expm1fKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_expm1f>(NumElements, Out, X);
+}
+
+__gpu_kernel void hypotKernel(const double *X, const double *Y, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_hypot>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void hypotfKernel(const float *X, const float *Y, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_hypotf>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void logKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__nv_log>(NumElements, Out, X);
+}
+
+__gpu_kernel void logfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_logf>(NumElements, Out, X);
+}
+
+__gpu_kernel void log10Kernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_log10>(NumElements, Out, X);
+}
+
+__gpu_kernel void log10fKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_log10f>(NumElements, Out, X);
+}
+
+__gpu_kernel void log1pKernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_log1p>(NumElements, Out, X);
+}
+
+__gpu_kernel void log1pfKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_log1pf>(NumElements, Out, X);
+}
+
+__gpu_kernel void log2Kernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_log2>(NumElements, Out, X);
+}
+
+__gpu_kernel void log2fKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_log2f>(NumElements, Out, X);
+}
+
+__gpu_kernel void powfKernel(const float *X, float *Y, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_powf>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void powfRoundedExponentKernel(const float *X, float *Y,
+                                            float *Out,
+                                            size_t NumElements) noexcept {
+  runKernelBody<powfRoundedExponent>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void sinKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__nv_sin>(NumElements, Out, X);
+}
+
+__gpu_kernel void sinfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_sinf>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosSinKernel(const double *X, double *Out,
+                                  size_t NumElements) noexcept {
+  runKernelBody<sincosSin>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosCosKernel(const double *X, double *Out,
+                                  size_t NumElements) noexcept {
+  runKernelBody<sincosCos>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosfSinKernel(const float *X, float *Out,
+                                   size_t NumElements) noexcept {
+  runKernelBody<sincosfSin>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosfCosKernel(const float *X, float *Out,
+                                   size_t NumElements) noexcept {
+  runKernelBody<sincosfCos>(NumElements, Out, X);
+}
+
+__gpu_kernel void sinhfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_sinhf>(NumElements, Out, X);
+}
+
+__gpu_kernel void sinpifKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__nv_sinpif>(NumElements, Out, X);
+}
+
+__gpu_kernel void tanKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__nv_tan>(NumElements, Out, X);
+}
+
+__gpu_kernel void tanfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__nv_tanf>(NumElements, Out, X);
+}
+
+__gpu_kernel void tanhfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__nv_tanhf>(NumElements, Out, X);
+}
+} // extern "C"
+
+#endif // CUDA_MATH_FOUND
diff --git a/offload/unittests/Conformance/device_code/DeviceAPIs.hpp b/offload/unittests/Conformance/device_code/DeviceAPIs.hpp
new file mode 100644
index 0000000000000..894652a8e1af1
--- /dev/null
+++ b/offload/unittests/Conformance/device_code/DeviceAPIs.hpp
@@ -0,0 +1,172 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains platform-specific definitions and forward declarations
+/// for device-side APIs used by the kernels.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef CONFORMANCE_DEVICE_CODE_DEVICEAPIS_HPP
+#define CONFORMANCE_DEVICE_CODE_DEVICEAPIS_HPP
+
+#include <stdint.h>
+
+typedef _Float16 float16;
+
+#ifdef __AMDGPU__
+
+// The ROCm device library uses control globals to alter codegen for the
+// different targets. To avoid needing to link them in manually, we simply
+// define them here.
+extern "C" {
+extern const inline uint8_t __oclc_unsafe_math_opt = 0;
+extern const inline uint8_t __oclc_daz_opt = 0;
+extern const inline uint8_t __oclc_correctly_rounded_sqrt32 = 1;
+extern const inline uint8_t __oclc_finite_only_opt = 0;
+extern const inline uint32_t __oclc_ISA_version = 9000;
+}
+
+// These aliases cause Clang to emit the control constants with ODR linkage.
+// This allows us to link against the symbols without preventing them from being
+// optimized out or causing symbol collisions.
+[[gnu::alias("__oclc_unsafe_math_opt")]] const uint8_t __oclc_unsafe_math_opt__;
+[[gnu::alias("__oclc_daz_opt")]] const uint8_t __oclc_daz_opt__;
+[[gnu::alias("__oclc_correctly_rounded_sqrt32")]] const uint8_t
+    __oclc_correctly_rounded_sqrt32__;
+[[gnu::alias("__oclc_finite_only_opt")]] const uint8_t __oclc_finite_only_opt__;
+[[gnu::alias("__oclc_ISA_version")]] const uint32_t __oclc_ISA_version__;
+
+#endif // __AMDGPU__
+
+#ifdef CUDA_MATH_FOUND
+
+extern "C" {
+
+double __nv_acos(double);
+float __nv_acosf(float);
+float __nv_acoshf(float);
+double __nv_asin(double);
+float __nv_asinf(float);
+float __nv_asinhf(float);
+float __nv_atanf(float);
+float __nv_atan2f(float, float);
+float __nv_atanhf(float);
+double __nv_cbrt(double);
+float __nv_cbrtf(float);
+double __nv_cos(double);
+float __nv_cosf(float);
+float __nv_coshf(float);
+float __nv_cospif(float);
+float __nv_erff(float);
+double __nv_exp(double);
+float __nv_expf(float);
+double __nv_exp10(double);
+float __nv_exp10f(float);
+double __nv_exp2(double);
+float __nv_exp2f(float);
+double __nv_expm1(double);
+float __nv_expm1f(float);
+double __nv_hypot(double, double);
+float __nv_hypotf(float, float);
+double __nv_log(double);
+float __nv_logf(float);
+double __nv_log10(double);
+float __nv_log10f(float);
+double __nv_log1p(double);
+float __nv_log1pf(float);
+double __nv_log2(double);
+float __nv_log2f(float);
+float __nv_powf(float, float);
+float __nv_roundf(float);
+double __nv_sin(double);
+float __nv_sinf(float);
+void __nv_sincos(double, double *, double *);
+void __nv_sincosf(float, float *, float *);
+float __nv_sinhf(float);
+float __nv_sinpif(float);
+double __nv_tan(double);
+float __nv_tanf(float);
+float __nv_tanhf(float);
+} // extern "C"
+
+#endif // CUDA_MATH_FOUND
+
+#ifdef HIP_MATH_FOUND
+
+extern "C" {
+
+double __ocml_acos_f64(double);
+float __ocml_acos_f32(float);
+float16 __ocml_acos_f16(float16);
+float __ocml_acosh_f32(float);
+float16 __ocml_acosh_f16(float16);
+double __ocml_asin_f64(double);
+float __ocml_asin_f32(float);
+float16 __ocml_asin_f16(float16);
+float __ocml_asinh_f32(float);
+float16 __ocml_asinh_f16(float16);
+float __ocml_atan_f32(float);
+float16 __ocml_atan_f16(float16);
+float __ocml_atan2_f32(float, float);
+float __ocml_atanh_f32(float);
+float16 __ocml_atanh_f16(float16);
+double __ocml_cbrt_f64(double);
+float __ocml_cbrt_f32(float);
+double __ocml_cos_f64(double);
+float __ocml_cos_f32(float);
+float16 __ocml_cos_f16(float16);
+float __ocml_cosh_f32(float);
+float16 __ocml_cosh_f16(float16);
+float __ocml_cospi_f32(float);
+float __ocml_erf_f32(float);
+double __ocml_exp_f64(double);
+float __ocml_exp_f32(float);
+float16 __ocml_exp_f16(float16);
+double __ocml_exp10_f64(double);
+float __ocml_exp10_f32(float);
+float16 __ocml_exp10_f16(float16);
+double __ocml_exp2_f64(double);
+float __ocml_exp2_f32(float);
+float16 __ocml_exp2_f16(float16);
+double __ocml_expm1_f64(double);
+float __ocml_expm1_f32(float);
+float16 __ocml_expm1_f16(float16);
+double __ocml_hypot_f64(double, double);
+float __ocml_hypot_f32(float, float);
+double __ocml_log_f64(double);
+float __ocml_log_f32(float);
+float16 __ocml_log_f16(float16);
+double __ocml_log10_f64(double);
+float __ocml_log10_f32(float);
+float16 __ocml_log10_f16(float16);
+double __ocml_log1p_f64(double);
+float __ocml_log1p_f32(float);
+double __ocml_log2_f64(double);
+float __ocml_log2_f32(float);
+float16 __ocml_log2_f16(float16);
+float __ocml_pow_f32(float, float);
+float __ocml_round_f32(float);
+double __ocml_sin_f64(double);
+float __ocml_sin_f32(float);
+float16 __ocml_sin_f16(float16);
+double __ocml_sincos_f64(double, double *);
+float __ocml_sincos_f32(float, float *);
+float __ocml_sinh_f32(float);
+float16 __ocml_sinh_f16(float16);
+float __ocml_sinpi_f32(float);
+double __ocml_tan_f64(double);
+float __ocml_tan_f32(float);
+float16 __ocml_tan_f16(float16);
+float __ocml_tanh_f32(float);
+float16 __ocml_tanh_f16(float16);
+} // extern "C"
+
+#endif // HIP_MATH_FOUND
+
+#endif // CONFORMANCE_DEVICE_CODE_DEVICEAPIS_HPP
diff --git a/offload/unittests/Conformance/device_code/HIPMath.cpp b/offload/unittests/Conformance/device_code/HIPMath.cpp
new file mode 100644
index 0000000000000..7cc0ad5d9142e
--- /dev/null
+++ b/offload/unittests/Conformance/device_code/HIPMath.cpp
@@ -0,0 +1,395 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the implementation of the device kernels that wrap the
+/// math functions from the hip-math provider.
+///
+//===----------------------------------------------------------------------===//
+
+#ifdef HIP_MATH_FOUND
+
+#include "Conformance/device_code/DeviceAPIs.hpp"
+#include "Conformance/device_code/KernelRunner.hpp"
+
+#include <gpuintrin.h>
+#include <stddef.h>
+
+using namespace kernels;
+
+//===----------------------------------------------------------------------===//
+// Helpers
+//===----------------------------------------------------------------------===//
+
+static inline float powfRoundedExponent(float Base, float Exponent) {
+  return __ocml_pow_f32(Base, __ocml_round_f32(Exponent));
+}
+
+static inline double sincosSin(double X) {
+  double CosX;
+  double SinX = __ocml_sincos_f64(X, &CosX);
+  return SinX;
+}
+
+static inline double sincosCos(double X) {
+  double CosX;
+  double SinX = __ocml_sincos_f64(X, &CosX);
+  return CosX;
+}
+
+static inline float sincosfSin(float X) {
+  float CosX;
+  float SinX = __ocml_sincos_f32(X, &CosX);
+  return SinX;
+}
+
+static inline float sincosfCos(float X) {
+  float CosX;
+  float SinX = __ocml_sincos_f32(X, &CosX);
+  return CosX;
+}
+
+//===----------------------------------------------------------------------===//
+// Kernels
+//===----------------------------------------------------------------------===//
+
+extern "C" {
+
+__gpu_kernel void acosKernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_acos_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void acosfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_acos_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void acosf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<__ocml_acos_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void acoshfKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_acosh_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void acoshf16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<__ocml_acosh_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void asinKernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_asin_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void asinfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_asin_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void asinf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<__ocml_asin_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void asinhfKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_asinh_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void asinhf16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<__ocml_asinh_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void atanfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_atan_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void atanf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<__ocml_atan_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void atan2fKernel(const float *X, const float *Y, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_atan2_f32>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void atanhfKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_atanh_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void atanhf16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<__ocml_atanh_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void cbrtKernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_cbrt_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void cbrtfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_cbrt_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void cosKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__ocml_cos_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void cosfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_cos_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void cosf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_cos_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void coshfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_cosh_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void coshf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<__ocml_cosh_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void cospifKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_cospi_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void erffKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_erf_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void expKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__ocml_exp_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void expfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_exp_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void expf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_exp_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp10Kernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_exp10_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp10fKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_exp10_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp10f16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<__ocml_exp10_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp2Kernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_exp2_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp2fKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_exp2_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp2f16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<__ocml_exp2_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void expm1Kernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_expm1_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void expm1fKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_expm1_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void expm1f16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<__ocml_expm1_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void hypotKernel(const double *X, const double *Y, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_hypot_f64>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void hypotfKernel(const float *X, const float *Y, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_hypot_f32>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void logKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__ocml_log_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void logfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_log_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void logf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_log_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void log10Kernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_log10_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void log10fKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_log10_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void log10f16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<__ocml_log10_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void log1pKernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_log1p_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void log1pfKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_log1p_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void log2Kernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_log2_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void log2fKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_log2_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void log2f16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<__ocml_log2_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void powfKernel(const float *X, float *Y, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_pow_f32>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void powfRoundedExponentKernel(const float *X, float *Y,
+                                            float *Out,
+                                            size_t NumElements) noexcept {
+  runKernelBody<powfRoundedExponent>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void sinKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__ocml_sin_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void sinfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_sin_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void sinf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_sin_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosSinKernel(const double *X, double *Out,
+                                  size_t NumElements) noexcept {
+  runKernelBody<sincosSin>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosCosKernel(const double *X, double *Out,
+                                  size_t NumElements) noexcept {
+  runKernelBody<sincosCos>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosfSinKernel(const float *X, float *Out,
+                                   size_t NumElements) noexcept {
+  runKernelBody<sincosfSin>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosfCosKernel(const float *X, float *Out,
+                                   size_t NumElements) noexcept {
+  runKernelBody<sincosfCos>(NumElements, Out, X);
+}
+
+__gpu_kernel void sinhfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_sinh_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void sinhf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<__ocml_sinh_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void sinpifKernel(const float *X, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_sinpi_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void tanKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<__ocml_tan_f64>(NumElements, Out, X);
+}
+
+__gpu_kernel void tanfKernel(const float *X, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<__ocml_tan_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void tanf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<__ocml_tan_f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void tanhfKernel(const float *X, float *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<__ocml_tanh_f32>(NumElements, Out, X);
+}
+
+__gpu_kernel void tanhf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<__ocml_tanh_f16>(NumElements, Out, X);
+}
+} // extern "C"
+
+#endif // HIP_MATH_FOUND
diff --git a/offload/unittests/Conformance/device_code/Common.hpp b/offload/unittests/Conformance/device_code/KernelRunner.hpp
similarity index 70%
rename from offload/unittests/Conformance/device_code/Common.hpp
rename to offload/unittests/Conformance/device_code/KernelRunner.hpp
index bcf3ac617b54c..e64a62fbdf018 100644
--- a/offload/unittests/Conformance/device_code/Common.hpp
+++ b/offload/unittests/Conformance/device_code/KernelRunner.hpp
@@ -7,21 +7,19 @@
 //===----------------------------------------------------------------------===//
 ///
 /// \file
-/// This file contains common utilities for defining device kernel wrappers to
-/// math functions.
+/// This file contains the definition of the runKernelBody, a template helper
+/// that executes the per-thread logic of a math function's kernel wrapper.
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef CONFORMANCE_DEVICE_CODE_COMMON_HPP
-#define CONFORMANCE_DEVICE_CODE_COMMON_HPP
+#ifndef CONFORMANCE_DEVICE_CODE_KERNELRUNNER_HPP
+#define CONFORMANCE_DEVICE_CODE_KERNELRUNNER_HPP
 
 #include <gpuintrin.h>
 #include <stddef.h>
 #include <stdint.h>
 
-namespace common {
-
-typedef _Float16 float16;
+namespace kernels {
 
 template <auto Func, typename OutType, typename... InTypes>
 void runKernelBody(size_t NumElements, OutType *Out, const InTypes *...Ins) {
@@ -32,6 +30,6 @@ void runKernelBody(size_t NumElements, OutType *Out, const InTypes *...Ins) {
     Out[Index] = Func(Ins[Index]...);
   }
 }
-} // namespace common
+} // namespace kernels
 
-#endif // CONFORMANCE_DEVICE_CODE_COMMON_HPP
+#endif // CONFORMANCE_DEVICE_CODE_KERNELRUNNER_HPP
diff --git a/offload/unittests/Conformance/device_code/LLVMLibm.cpp b/offload/unittests/Conformance/device_code/LLVMLibm.cpp
index f137ba3d23752..8673d809fd0a2 100644
--- a/offload/unittests/Conformance/device_code/LLVMLibm.cpp
+++ b/offload/unittests/Conformance/device_code/LLVMLibm.cpp
@@ -12,18 +12,35 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#include "Conformance/device_code/Common.hpp"
+#include "Conformance/device_code/DeviceAPIs.hpp"
+#include "Conformance/device_code/KernelRunner.hpp"
 
 #include <gpuintrin.h>
 #include <math.h>
 #include <stddef.h>
 
-using namespace common;
+using namespace kernels;
 
 //===----------------------------------------------------------------------===//
 // Helpers
 //===----------------------------------------------------------------------===//
 
+static inline float powfRoundedExponent(float Base, float Exponent) {
+  return powf(Base, roundf(Exponent));
+}
+
+static inline double sincosSin(double X) {
+  double SinX, CosX;
+  sincos(X, &SinX, &CosX);
+  return SinX;
+}
+
+static inline double sincosCos(double X) {
+  double SinX, CosX;
+  sincos(X, &SinX, &CosX);
+  return CosX;
+}
+
 static inline float sincosfSin(float X) {
   float SinX, CosX;
   sincosf(X, &SinX, &CosX);
@@ -42,111 +59,302 @@ static inline float sincosfCos(float X) {
 
 extern "C" {
 
+__gpu_kernel void acosKernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<acos>(NumElements, Out, X);
+}
+
 __gpu_kernel void acosfKernel(const float *X, float *Out,
                               size_t NumElements) noexcept {
   runKernelBody<acosf>(NumElements, Out, X);
 }
 
+__gpu_kernel void acosf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<acosf16>(NumElements, Out, X);
+}
+
 __gpu_kernel void acoshfKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<acoshf>(NumElements, Out, X);
 }
 
+__gpu_kernel void acoshf16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<acoshf16>(NumElements, Out, X);
+}
+
+__gpu_kernel void acospif16Kernel(const float16 *X, float16 *Out,
+                                  size_t NumElements) noexcept {
+  runKernelBody<acospif16>(NumElements, Out, X);
+}
+
+__gpu_kernel void asinKernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<asin>(NumElements, Out, X);
+}
+
 __gpu_kernel void asinfKernel(const float *X, float *Out,
                               size_t NumElements) noexcept {
   runKernelBody<asinf>(NumElements, Out, X);
 }
 
+__gpu_kernel void asinf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<asinf16>(NumElements, Out, X);
+}
+
 __gpu_kernel void asinhfKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<asinhf>(NumElements, Out, X);
 }
 
+__gpu_kernel void asinhf16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<asinhf16>(NumElements, Out, X);
+}
+
 __gpu_kernel void atanfKernel(const float *X, float *Out,
                               size_t NumElements) noexcept {
   runKernelBody<atanf>(NumElements, Out, X);
 }
 
+__gpu_kernel void atanf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<atanf16>(NumElements, Out, X);
+}
+
+__gpu_kernel void atan2fKernel(const float *X, const float *Y, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<atan2f>(NumElements, Out, X, Y);
+}
+
 __gpu_kernel void atanhfKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<atanhf>(NumElements, Out, X);
 }
 
+__gpu_kernel void atanhf16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<atanhf16>(NumElements, Out, X);
+}
+
+__gpu_kernel void cbrtKernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<cbrt>(NumElements, Out, X);
+}
+
 __gpu_kernel void cbrtfKernel(const float *X, float *Out,
                               size_t NumElements) noexcept {
   runKernelBody<cbrtf>(NumElements, Out, X);
 }
 
+__gpu_kernel void cosKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<cos>(NumElements, Out, X);
+}
+
 __gpu_kernel void cosfKernel(const float *X, float *Out,
                              size_t NumElements) noexcept {
   runKernelBody<cosf>(NumElements, Out, X);
 }
 
+__gpu_kernel void cosf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<cosf16>(NumElements, Out, X);
+}
+
 __gpu_kernel void coshfKernel(const float *X, float *Out,
                               size_t NumElements) noexcept {
   runKernelBody<coshf>(NumElements, Out, X);
 }
 
+__gpu_kernel void coshf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<coshf16>(NumElements, Out, X);
+}
+
 __gpu_kernel void cospifKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<cospif>(NumElements, Out, X);
 }
 
+__gpu_kernel void cospif16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<cospif16>(NumElements, Out, X);
+}
+
 __gpu_kernel void erffKernel(const float *X, float *Out,
                              size_t NumElements) noexcept {
   runKernelBody<erff>(NumElements, Out, X);
 }
 
+__gpu_kernel void expKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<exp>(NumElements, Out, X);
+}
+
 __gpu_kernel void expfKernel(const float *X, float *Out,
                              size_t NumElements) noexcept {
   runKernelBody<expf>(NumElements, Out, X);
 }
 
+__gpu_kernel void expf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<expf16>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp10Kernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<exp10>(NumElements, Out, X);
+}
+
 __gpu_kernel void exp10fKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<exp10f>(NumElements, Out, X);
 }
 
+__gpu_kernel void exp10f16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<exp10f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void exp2Kernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<exp2>(NumElements, Out, X);
+}
+
 __gpu_kernel void exp2fKernel(const float *X, float *Out,
                               size_t NumElements) noexcept {
   runKernelBody<exp2f>(NumElements, Out, X);
 }
 
+__gpu_kernel void exp2f16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<exp2f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void expm1Kernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<expm1>(NumElements, Out, X);
+}
+
 __gpu_kernel void expm1fKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<expm1f>(NumElements, Out, X);
 }
 
-__gpu_kernel void hypotf16Kernel(const float16 *X, float16 *Y, float16 *Out,
+__gpu_kernel void expm1f16Kernel(const float16 *X, float16 *Out,
                                  size_t NumElements) noexcept {
+  runKernelBody<expm1f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void hypotKernel(const double *X, const double *Y, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<hypot>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void hypotfKernel(const float *X, const float *Y, float *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<hypotf>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void hypotf16Kernel(const float16 *X, const float16 *Y,
+                                 float16 *Out, size_t NumElements) noexcept {
   runKernelBody<hypotf16>(NumElements, Out, X, Y);
 }
 
+__gpu_kernel void logKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<log>(NumElements, Out, X);
+}
+
 __gpu_kernel void logfKernel(const float *X, float *Out,
                              size_t NumElements) noexcept {
   runKernelBody<logf>(NumElements, Out, X);
 }
 
+__gpu_kernel void logf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<logf16>(NumElements, Out, X);
+}
+
+__gpu_kernel void log10Kernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<log10>(NumElements, Out, X);
+}
+
 __gpu_kernel void log10fKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<log10f>(NumElements, Out, X);
 }
 
+__gpu_kernel void log10f16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<log10f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void log1pKernel(const double *X, double *Out,
+                              size_t NumElements) noexcept {
+  runKernelBody<log1p>(NumElements, Out, X);
+}
+
 __gpu_kernel void log1pfKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<log1pf>(NumElements, Out, X);
 }
 
+__gpu_kernel void log2Kernel(const double *X, double *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<log2>(NumElements, Out, X);
+}
+
 __gpu_kernel void log2fKernel(const float *X, float *Out,
                               size_t NumElements) noexcept {
   runKernelBody<log2f>(NumElements, Out, X);
 }
 
+__gpu_kernel void log2f16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<log2f16>(NumElements, Out, X);
+}
+
+__gpu_kernel void powfKernel(const float *X, float *Y, float *Out,
+                             size_t NumElements) noexcept {
+  runKernelBody<powf>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void powfRoundedExponentKernel(const float *X, float *Y,
+                                            float *Out,
+                                            size_t NumElements) noexcept {
+  runKernelBody<powfRoundedExponent>(NumElements, Out, X, Y);
+}
+
+__gpu_kernel void sinKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<sin>(NumElements, Out, X);
+}
+
 __gpu_kernel void sinfKernel(const float *X, float *Out,
                              size_t NumElements) noexcept {
   runKernelBody<sinf>(NumElements, Out, X);
 }
 
+__gpu_kernel void sinf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<sinf16>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosSinKernel(const double *X, double *Out,
+                                  size_t NumElements) noexcept {
+  runKernelBody<sincosSin>(NumElements, Out, X);
+}
+
+__gpu_kernel void sincosCosKernel(const double *X, double *Out,
+                                  size_t NumElements) noexcept {
+  runKernelBody<sincosCos>(NumElements, Out, X);
+}
+
 __gpu_kernel void sincosfSinKernel(const float *X, float *Out,
                                    size_t NumElements) noexcept {
   runKernelBody<sincosfSin>(NumElements, Out, X);
@@ -162,23 +370,53 @@ __gpu_kernel void sinhfKernel(const float *X, float *Out,
   runKernelBody<sinhf>(NumElements, Out, X);
 }
 
+__gpu_kernel void sinhf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<sinhf16>(NumElements, Out, X);
+}
+
 __gpu_kernel void sinpifKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<sinpif>(NumElements, Out, X);
 }
 
+__gpu_kernel void sinpif16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<sinpif16>(NumElements, Out, X);
+}
+
+__gpu_kernel void tanKernel(const double *X, double *Out,
+                            size_t NumElements) noexcept {
+  runKernelBody<tan>(NumElements, Out, X);
+}
+
 __gpu_kernel void tanfKernel(const float *X, float *Out,
                              size_t NumElements) noexcept {
   runKernelBody<tanf>(NumElements, Out, X);
 }
 
+__gpu_kernel void tanf16Kernel(const float16 *X, float16 *Out,
+                               size_t NumElements) noexcept {
+  runKernelBody<tanf16>(NumElements, Out, X);
+}
+
 __gpu_kernel void tanhfKernel(const float *X, float *Out,
                               size_t NumElements) noexcept {
   runKernelBody<tanhf>(NumElements, Out, X);
 }
 
+__gpu_kernel void tanhf16Kernel(const float16 *X, float16 *Out,
+                                size_t NumElements) noexcept {
+  runKernelBody<tanhf16>(NumElements, Out, X);
+}
+
 __gpu_kernel void tanpifKernel(const float *X, float *Out,
                                size_t NumElements) noexcept {
   runKernelBody<tanpif>(NumElements, Out, X);
 }
+
+__gpu_kernel void tanpif16Kernel(const float16 *X, float16 *Out,
+                                 size_t NumElements) noexcept {
+  runKernelBody<tanpif16>(NumElements, Out, X);
+}
 } // extern "C"
diff --git a/offload/unittests/Conformance/include/mathtest/ExhaustiveGenerator.hpp b/offload/unittests/Conformance/include/mathtest/ExhaustiveGenerator.hpp
index 6f7f7a9b665d0..39c6838eecf7e 100644
--- a/offload/unittests/Conformance/include/mathtest/ExhaustiveGenerator.hpp
+++ b/offload/unittests/Conformance/include/mathtest/ExhaustiveGenerator.hpp
@@ -8,8 +8,8 @@
 ///
 /// \file
 /// This file contains the definition of the ExhaustiveGenerator class, a
-/// concrete input generator that exhaustively creates inputs from a given
-/// sequence of ranges.
+/// concrete range-based generator that exhaustively creates inputs from a
+/// given sequence of ranges.
 ///
 //===----------------------------------------------------------------------===//
 
@@ -17,89 +17,62 @@
 #define MATHTEST_EXHAUSTIVEGENERATOR_HPP
 
 #include "mathtest/IndexedRange.hpp"
-#include "mathtest/InputGenerator.hpp"
+#include "mathtest/RangeBasedGenerator.hpp"
 
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/Support/Parallel.h"
-
-#include <algorithm>
 #include <array>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
+#include <optional>
 #include <tuple>
 
 namespace mathtest {
 
 template <typename... InTypes>
 class [[nodiscard]] ExhaustiveGenerator final
-    : public InputGenerator<InTypes...> {
-  static constexpr std::size_t NumInputs = sizeof...(InTypes);
-  static_assert(NumInputs > 0, "The number of inputs must be at least 1");
+    : public RangeBasedGenerator<ExhaustiveGenerator<InTypes...>, InTypes...> {
+
+  friend class RangeBasedGenerator<ExhaustiveGenerator<InTypes...>, InTypes...>;
+
+  using Base = RangeBasedGenerator<ExhaustiveGenerator<InTypes...>, InTypes...>;
+  using IndexArrayType = std::array<uint64_t, Base::NumInputs>;
+
+  using Base::RangesTuple;
+  using Base::Size;
 
 public:
   explicit constexpr ExhaustiveGenerator(
       const IndexedRange<InTypes> &...Ranges) noexcept
-      : RangesTuple(Ranges...) {
-    bool Overflowed = getSizeWithOverflow(Ranges..., Size);
+      : Base(Ranges...) {
+    const auto MaybeSize = getInputSpaceSize(Ranges...);
+
+    assert(MaybeSize.has_value() && "The size is too large");
+    Size = *MaybeSize;
 
-    assert(!Overflowed && "The input space size is too large");
-    assert((Size > 0) && "The input space size must be at least 1");
+    assert((Size > 0) && "The size must be at least 1");
 
     IndexArrayType DimSizes = {};
     std::size_t DimIndex = 0;
     ((DimSizes[DimIndex++] = Ranges.getSize()), ...);
 
-    Strides[NumInputs - 1] = 1;
-    if constexpr (NumInputs > 1)
-      for (int Index = static_cast<int>(NumInputs) - 2; Index >= 0; --Index)
+    Strides[Base::NumInputs - 1] = 1;
+    if constexpr (Base::NumInputs > 1)
+      for (int Index = static_cast<int>(Base::NumInputs) - 2; Index >= 0;
+           --Index)
         Strides[Index] = Strides[Index + 1] * DimSizes[Index + 1];
   }
 
-  void reset() noexcept override { NextFlatIndex = 0; }
-
-  [[nodiscard]] std::size_t
-  fill(llvm::MutableArrayRef<InTypes>... Buffers) noexcept override {
-    const std::array<std::size_t, NumInputs> BufferSizes = {Buffers.size()...};
-    const std::size_t BufferSize = BufferSizes[0];
-    assert((BufferSize != 0) && "Buffer size cannot be zero");
-    assert(std::all_of(BufferSizes.begin(), BufferSizes.end(),
-                       [&](std::size_t Size) { return Size == BufferSize; }) &&
-           "All input buffers must have the same size");
-
-    if (NextFlatIndex >= Size)
-      return 0;
-
-    const auto BatchSize = std::min<uint64_t>(BufferSize, Size - NextFlatIndex);
-    const auto CurrentFlatIndex = NextFlatIndex;
-    NextFlatIndex += BatchSize;
-
-    auto BufferPtrsTuple = std::make_tuple(Buffers.data()...);
-
-    llvm::parallelFor(0, BatchSize, [&](std::size_t Offset) {
-      writeInputs(CurrentFlatIndex, Offset, BufferPtrsTuple);
-    });
-
-    return static_cast<std::size_t>(BatchSize);
-  }
-
 private:
-  using RangesTupleType = std::tuple<IndexedRange<InTypes>...>;
-  using IndexArrayType = std::array<uint64_t, NumInputs>;
-
-  static bool getSizeWithOverflow(const IndexedRange<InTypes> &...Ranges,
-                                  uint64_t &Size) noexcept {
-    Size = 1;
-    bool Overflowed = false;
-
-    auto Multiplier = [&](const uint64_t RangeSize) {
-      if (!Overflowed)
-        Overflowed = __builtin_mul_overflow(Size, RangeSize, &Size);
-    };
+  [[nodiscard]] constexpr IndexArrayType
+  getNDIndex(uint64_t FlatIndex) const noexcept {
+    IndexArrayType NDIndex;
 
-    (Multiplier(Ranges.getSize()), ...);
+    for (std::size_t Index = 0; Index < Base::NumInputs; ++Index) {
+      NDIndex[Index] = FlatIndex / Strides[Index];
+      FlatIndex -= NDIndex[Index] * Strides[Index];
+    }
 
-    return Overflowed;
+    return NDIndex;
   }
 
   template <typename BufferPtrsTupleType>
@@ -109,31 +82,37 @@ class [[nodiscard]] ExhaustiveGenerator final
     writeInputsImpl<0>(NDIndex, Offset, BufferPtrsTuple);
   }
 
-  constexpr IndexArrayType getNDIndex(uint64_t FlatIndex) const noexcept {
-    IndexArrayType NDIndex;
-
-    for (std::size_t Index = 0; Index < NumInputs; ++Index) {
-      NDIndex[Index] = FlatIndex / Strides[Index];
-      FlatIndex -= NDIndex[Index] * Strides[Index];
-    }
-
-    return NDIndex;
-  }
-
   template <std::size_t Index, typename BufferPtrsTupleType>
   void writeInputsImpl(IndexArrayType NDIndex, uint64_t Offset,
                        BufferPtrsTupleType BufferPtrsTuple) const noexcept {
-    if constexpr (Index < NumInputs) {
+    if constexpr (Index < Base::NumInputs) {
       const auto &Range = std::get<Index>(RangesTuple);
       std::get<Index>(BufferPtrsTuple)[Offset] = Range[NDIndex[Index]];
+
       writeInputsImpl<Index + 1>(NDIndex, Offset, BufferPtrsTuple);
     }
   }
 
-  uint64_t Size = 1;
-  RangesTupleType RangesTuple;
+  [[nodiscard]] static constexpr std::optional<uint64_t>
+  getInputSpaceSize(const IndexedRange<InTypes> &...Ranges) noexcept {
+    uint64_t InputSpaceSize = 1;
+    bool Overflowed = false;
+
+    auto Multiplier = [&](const uint64_t RangeSize) {
+      if (!Overflowed)
+        Overflowed =
+            __builtin_mul_overflow(InputSpaceSize, RangeSize, &InputSpaceSize);
+    };
+
+    (Multiplier(Ranges.getSize()), ...);
+
+    if (Overflowed)
+      return std::nullopt;
+
+    return InputSpaceSize;
+  }
+
   IndexArrayType Strides = {};
-  uint64_t NextFlatIndex = 0;
 };
 } // namespace mathtest
 
diff --git a/offload/unittests/Conformance/include/mathtest/RandomGenerator.hpp b/offload/unittests/Conformance/include/mathtest/RandomGenerator.hpp
new file mode 100644
index 0000000000000..436cd05f0a3d3
--- /dev/null
+++ b/offload/unittests/Conformance/include/mathtest/RandomGenerator.hpp
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the definition of the RandomGenerator class, a concrete
+/// range-based generator that randomly creates inputs from a given sequence of
+/// ranges.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef MATHTEST_RANDOMGENERATOR_HPP
+#define MATHTEST_RANDOMGENERATOR_HPP
+
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/RangeBasedGenerator.hpp"
+
+#include <cstddef>
+#include <cstdint>
+#include <tuple>
+
+namespace mathtest {
+
+template <typename... InTypes>
+class [[nodiscard]] RandomGenerator final
+    : public RangeBasedGenerator<RandomGenerator<InTypes...>, InTypes...> {
+
+  friend class RangeBasedGenerator<RandomGenerator<InTypes...>, InTypes...>;
+
+  using Base = RangeBasedGenerator<RandomGenerator<InTypes...>, InTypes...>;
+
+  using Base::RangesTuple;
+  using Base::Size;
+
+public:
+  explicit constexpr RandomGenerator(
+      SeedTy BaseSeed, uint64_t Size,
+      const IndexedRange<InTypes> &...Ranges) noexcept
+      : Base(Size, Ranges...), BaseSeed(BaseSeed) {}
+
+private:
+  [[nodiscard]] static uint64_t getRandomIndex(RandomState &RNG,
+                                               uint64_t RangeSize) noexcept {
+    if (RangeSize == 0)
+      return 0;
+
+    const uint64_t Threshold = (-RangeSize) % RangeSize;
+
+    uint64_t RandomNumber;
+    do {
+      RandomNumber = RNG.next();
+    } while (RandomNumber < Threshold);
+
+    return RandomNumber % RangeSize;
+  }
+
+  template <typename BufferPtrsTupleType>
+  void writeInputs(uint64_t CurrentFlatIndex, uint64_t Offset,
+                   BufferPtrsTupleType BufferPtrsTuple) const noexcept {
+
+    RandomState RNG(SeedTy{BaseSeed.Value ^ (CurrentFlatIndex + Offset)});
+    writeInputsImpl<0>(RNG, Offset, BufferPtrsTuple);
+  }
+
+  template <std::size_t Index, typename BufferPtrsTupleType>
+  void writeInputsImpl(RandomState &RNG, uint64_t Offset,
+                       BufferPtrsTupleType BufferPtrsTuple) const noexcept {
+    if constexpr (Index < Base::NumInputs) {
+      const auto &Range = std::get<Index>(RangesTuple);
+      const auto RandomIndex = getRandomIndex(RNG, Range.getSize());
+      std::get<Index>(BufferPtrsTuple)[Offset] = Range[RandomIndex];
+
+      writeInputsImpl<Index + 1>(RNG, Offset, BufferPtrsTuple);
+    }
+  }
+
+  SeedTy BaseSeed;
+};
+} // namespace mathtest
+
+#endif // MATHTEST_RANDOMGENERATOR_HPP
diff --git a/offload/unittests/Conformance/include/mathtest/RandomState.hpp b/offload/unittests/Conformance/include/mathtest/RandomState.hpp
new file mode 100644
index 0000000000000..322d53175236f
--- /dev/null
+++ b/offload/unittests/Conformance/include/mathtest/RandomState.hpp
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the definition of the RandomState class, a fast and
+/// lightweight pseudo-random number generator.
+///
+/// The implementation is based on the xorshift* generator, seeded using the
+/// SplitMix64 generator for robust initialization. For more details on the
+/// algorithm, see: https://en.wikipedia.org/wiki/Xorshift
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef MATHTEST_RANDOMSTATE_HPP
+#define MATHTEST_RANDOMSTATE_HPP
+
+#include <cstdint>
+
+struct SeedTy {
+  uint64_t Value;
+};
+
+class [[nodiscard]] RandomState {
+  uint64_t State;
+
+  [[nodiscard]] static constexpr uint64_t splitMix64(uint64_t X) noexcept {
+    X += 0x9E3779B97F4A7C15ULL;
+    X = (X ^ (X >> 30)) * 0xBF58476D1CE4E5B9ULL;
+    X = (X ^ (X >> 27)) * 0x94D049BB133111EBULL;
+    X = (X ^ (X >> 31));
+    return X ? X : 0x9E3779B97F4A7C15ULL;
+  }
+
+public:
+  explicit constexpr RandomState(SeedTy Seed) noexcept
+      : State(splitMix64(Seed.Value)) {}
+
+  inline uint64_t next() noexcept {
+    uint64_t X = State;
+    X ^= X >> 12;
+    X ^= X << 25;
+    X ^= X >> 27;
+    State = X;
+    return X * 0x2545F4914F6CDD1DULL;
+  }
+};
+
+#endif // MATHTEST_RANDOMSTATE_HPP
diff --git a/offload/unittests/Conformance/include/mathtest/RangeBasedGenerator.hpp b/offload/unittests/Conformance/include/mathtest/RangeBasedGenerator.hpp
new file mode 100644
index 0000000000000..5e1e1139aba96
--- /dev/null
+++ b/offload/unittests/Conformance/include/mathtest/RangeBasedGenerator.hpp
@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the definition of the RangeBasedGenerator class, a base
+/// class for input generators that operate on a sequence of ranges.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef MATHTEST_RANGEBASEDGENERATOR_HPP
+#define MATHTEST_RANGEBASEDGENERATOR_HPP
+
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/InputGenerator.hpp"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Parallel.h"
+
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <tuple>
+
+namespace mathtest {
+
+template <typename Derived, typename... InTypes>
+class [[nodiscard]] RangeBasedGenerator : public InputGenerator<InTypes...> {
+public:
+  void reset() noexcept override { NextFlatIndex = 0; }
+
+  [[nodiscard]] std::size_t
+  fill(llvm::MutableArrayRef<InTypes>... Buffers) noexcept override {
+    const std::array<std::size_t, NumInputs> BufferSizes = {Buffers.size()...};
+    const std::size_t BufferSize = BufferSizes[0];
+    assert((BufferSize != 0) && "Buffer size cannot be zero");
+    assert(std::all_of(BufferSizes.begin(), BufferSizes.end(),
+                       [&](std::size_t Size) { return Size == BufferSize; }) &&
+           "All input buffers must have the same size");
+
+    if (NextFlatIndex >= Size)
+      return 0;
+
+    const auto BatchSize = std::min<uint64_t>(BufferSize, Size - NextFlatIndex);
+    const auto CurrentFlatIndex = NextFlatIndex;
+    NextFlatIndex += BatchSize;
+
+    auto BufferPtrsTuple = std::make_tuple(Buffers.data()...);
+
+    llvm::parallelFor(0, BatchSize, [&](std::size_t Offset) {
+      static_cast<Derived *>(this)->writeInputs(CurrentFlatIndex, Offset,
+                                                BufferPtrsTuple);
+    });
+
+    return static_cast<std::size_t>(BatchSize);
+  }
+
+protected:
+  using RangesTupleType = std::tuple<IndexedRange<InTypes>...>;
+
+  static constexpr std::size_t NumInputs = sizeof...(InTypes);
+  static_assert(NumInputs > 0, "The number of inputs must be at least 1");
+
+  explicit constexpr RangeBasedGenerator(
+      const IndexedRange<InTypes> &...Ranges) noexcept
+      : RangesTuple(Ranges...) {}
+
+  explicit constexpr RangeBasedGenerator(
+      uint64_t Size, const IndexedRange<InTypes> &...Ranges) noexcept
+      : RangesTuple(Ranges...), Size(Size) {}
+
+  RangesTupleType RangesTuple;
+  uint64_t Size = 0;
+
+private:
+  uint64_t NextFlatIndex = 0;
+};
+} // namespace mathtest
+
+#endif // MATHTEST_RANGEBASEDGENERATOR_HPP
diff --git a/offload/unittests/Conformance/include/mathtest/TestRunner.hpp b/offload/unittests/Conformance/include/mathtest/TestRunner.hpp
index f89d151d0161e..ab17f1d83768a 100644
--- a/offload/unittests/Conformance/include/mathtest/TestRunner.hpp
+++ b/offload/unittests/Conformance/include/mathtest/TestRunner.hpp
@@ -41,11 +41,11 @@ void printPreamble(const TestConfig &Config, size_t Index,
                    size_t Total) noexcept {
   using FunctionConfig = FunctionConfig<Func>;
 
-  llvm::outs() << "[" << (Index + 1) << "/" << Total << "] "
+  llvm::errs() << "[" << (Index + 1) << "/" << Total << "] "
                << "Running conformance test '" << FunctionConfig::Name
                << "' with '" << Config.Provider << "' on '" << Config.Platform
                << "'\n";
-  llvm::outs().flush();
+  llvm::errs().flush();
 }
 
 template <typename T>
diff --git a/offload/unittests/Conformance/tests/AcosTest.cpp b/offload/unittests/Conformance/tests/AcosTest.cpp
new file mode 100644
index 0000000000000..bc0d1d2b72804
--- /dev/null
+++ b/offload/unittests/Conformance/tests/AcosTest.cpp
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the acos function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'acos' function to select the double version
+constexpr auto acosd // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(acos);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<acosd> {
+  static constexpr llvm::StringRef Name = "acos";
+  static constexpr llvm::StringRef KernelName = "acosKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 4;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the acos function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range(/*Begin=*/-1.0,
+                             /*End=*/1.0,
+                             /*Inclusive=*/true);
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<acosd>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Acosf16Test.cpp b/offload/unittests/Conformance/tests/Acosf16Test.cpp
new file mode 100644
index 0000000000000..ce11cc2aa1e83
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Acosf16Test.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the acosf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 acosf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<acosf16> {
+  static constexpr llvm::StringRef Name = "acosf16";
+  static constexpr llvm::StringRef KernelName = "acosf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the acosf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range(/*Begin=*/float16(-1.0),
+                              /*End=*/float16(1.0),
+                              /*Inclusive=*/true);
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<acosf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/AcosfTest.cpp b/offload/unittests/Conformance/tests/AcosfTest.cpp
index e69ee3b7d1fd7..65b2d18d77284 100644
--- a/offload/unittests/Conformance/tests/AcosfTest.cpp
+++ b/offload/unittests/Conformance/tests/AcosfTest.cpp
@@ -40,7 +40,9 @@ int main(int argc, const char **argv) {
 
   using namespace mathtest;
 
-  IndexedRange<float> Range;
+  IndexedRange<float> Range(/*Begin=*/-1.0f,
+                            /*End=*/1.0f,
+                            /*Inclusive=*/true);
   ExhaustiveGenerator<float> Generator(Range);
 
   const auto Configs = cl::getTestConfigs();
diff --git a/offload/unittests/Conformance/tests/Acoshf16Test.cpp b/offload/unittests/Conformance/tests/Acoshf16Test.cpp
new file mode 100644
index 0000000000000..80434477aa437
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Acoshf16Test.cpp
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the acoshf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/Numerics.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 acoshf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<acoshf16> {
+  static constexpr llvm::StringRef Name = "acoshf16";
+  static constexpr llvm::StringRef KernelName = "acoshf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the acoshf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range(/*Begin=*/float16(1.0),
+                              /*End=*/getMaxOrInf<float16>(),
+                              /*Inclusive=*/true);
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<acoshf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Acospif16Test.cpp b/offload/unittests/Conformance/tests/Acospif16Test.cpp
new file mode 100644
index 0000000000000..c5871e27cafc2
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Acospif16Test.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the acospif16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 acospif16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<acospif16> {
+  static constexpr llvm::StringRef Name = "acospif16";
+  static constexpr llvm::StringRef KernelName = "acospif16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the acospif16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range(/*Begin=*/float16(-1.0),
+                              /*End=*/float16(1.0),
+                              /*Inclusive=*/true);
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<acospif16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/AsinTest.cpp b/offload/unittests/Conformance/tests/AsinTest.cpp
new file mode 100644
index 0000000000000..aaaa37af02bca
--- /dev/null
+++ b/offload/unittests/Conformance/tests/AsinTest.cpp
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the asin function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'asin' function to select the double version
+constexpr auto asind // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(asin);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<asind> {
+  static constexpr llvm::StringRef Name = "asin";
+  static constexpr llvm::StringRef KernelName = "asinKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 4;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the asin function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range(/*Begin=*/-1.0,
+                             /*End=*/1.0,
+                             /*Inclusive=*/true);
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<asind>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Asinf16Test.cpp b/offload/unittests/Conformance/tests/Asinf16Test.cpp
new file mode 100644
index 0000000000000..5784d6bfe08e0
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Asinf16Test.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the asinf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 asinf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<asinf16> {
+  static constexpr llvm::StringRef Name = "asinf16";
+  static constexpr llvm::StringRef KernelName = "asinf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the asinf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range(/*Begin=*/float16(-1.0),
+                              /*End=*/float16(1.0),
+                              /*Inclusive=*/true);
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<asinf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/AsinfTest.cpp b/offload/unittests/Conformance/tests/AsinfTest.cpp
index 991f79b111efe..aeee648b5fa0e 100644
--- a/offload/unittests/Conformance/tests/AsinfTest.cpp
+++ b/offload/unittests/Conformance/tests/AsinfTest.cpp
@@ -40,7 +40,9 @@ int main(int argc, const char **argv) {
 
   using namespace mathtest;
 
-  IndexedRange<float> Range;
+  IndexedRange<float> Range(/*Begin=*/-1.0f,
+                            /*End=*/1.0f,
+                            /*Inclusive=*/true);
   ExhaustiveGenerator<float> Generator(Range);
 
   const auto Configs = cl::getTestConfigs();
diff --git a/offload/unittests/Conformance/tests/Asinhf16Test.cpp b/offload/unittests/Conformance/tests/Asinhf16Test.cpp
new file mode 100644
index 0000000000000..0af9bcb06fef0
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Asinhf16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the asinhf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 asinhf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<asinhf16> {
+  static constexpr llvm::StringRef Name = "asinhf16";
+  static constexpr llvm::StringRef KernelName = "asinhf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the asinhf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<asinhf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Atan2fTest.cpp b/offload/unittests/Conformance/tests/Atan2fTest.cpp
new file mode 100644
index 0000000000000..4a46f9a61540a
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Atan2fTest.cpp
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the atan2f function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace mathtest {
+
+template <> struct FunctionConfig<atan2f> {
+  static constexpr llvm::StringRef Name = "atan2f";
+  static constexpr llvm::StringRef KernelName = "atan2fKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 65, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 6;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the atan2f function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<float> RangeX;
+  IndexedRange<float> RangeY;
+  RandomGenerator<float, float> Generator(SeedTy{Seed}, Size, RangeX, RangeY);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<atan2f>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Atanf16Test.cpp b/offload/unittests/Conformance/tests/Atanf16Test.cpp
new file mode 100644
index 0000000000000..3d3fa384e84df
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Atanf16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the atanf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 atanf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<atanf16> {
+  static constexpr llvm::StringRef Name = "atanf16";
+  static constexpr llvm::StringRef KernelName = "atanf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the atanf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<atanf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Atanhf16Test.cpp b/offload/unittests/Conformance/tests/Atanhf16Test.cpp
new file mode 100644
index 0000000000000..86a0f82ce376c
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Atanhf16Test.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the atanhf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 atanhf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<atanhf16> {
+  static constexpr llvm::StringRef Name = "atanhf16";
+  static constexpr llvm::StringRef KernelName = "atanhf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the atanhf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range(/*Begin=*/float16(-1.0),
+                              /*End=*/float16(1.0),
+                              /*Inclusive=*/true);
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<atanhf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/CMakeLists.txt b/offload/unittests/Conformance/tests/CMakeLists.txt
index 5c0c3ba55c062..ad94df8e79788 100644
--- a/offload/unittests/Conformance/tests/CMakeLists.txt
+++ b/offload/unittests/Conformance/tests/CMakeLists.txt
@@ -1,27 +1,74 @@
+if(NOT TARGET libc)
+    message(WARNING "Cannot run conformance tests without the LLVM C library")
+    return()
+endif()
+
+add_conformance_test(acos AcosTest.cpp)
 add_conformance_test(acosf AcosfTest.cpp)
+add_conformance_test(acosf16 Acosf16Test.cpp)
 add_conformance_test(acoshf AcoshfTest.cpp)
+add_conformance_test(acoshf16 Acoshf16Test.cpp)
+add_conformance_test(acospif16 Acospif16Test.cpp)
+add_conformance_test(asin AsinTest.cpp)
 add_conformance_test(asinf AsinfTest.cpp)
+add_conformance_test(asinf16 Asinf16Test.cpp)
 add_conformance_test(asinhf AsinhfTest.cpp)
+add_conformance_test(asinhf16 Asinhf16Test.cpp)
 add_conformance_test(atanf AtanfTest.cpp)
+add_conformance_test(atanf16 Atanf16Test.cpp)
+add_conformance_test(atan2f Atan2fTest.cpp)
 add_conformance_test(atanhf AtanhfTest.cpp)
+add_conformance_test(atanhf16 Atanhf16Test.cpp)
+add_conformance_test(cbrt CbrtTest.cpp)
 add_conformance_test(cbrtf CbrtfTest.cpp)
+add_conformance_test(cos CosTest.cpp)
 add_conformance_test(cosf CosfTest.cpp)
+add_conformance_test(cosf16 Cosf16Test.cpp)
 add_conformance_test(coshf CoshfTest.cpp)
+add_conformance_test(coshf16 Coshf16Test.cpp)
 add_conformance_test(cospif CospifTest.cpp)
+add_conformance_test(cospif16 Cospif16Test.cpp)
 add_conformance_test(erff ErffTest.cpp)
+add_conformance_test(exp ExpTest.cpp)
 add_conformance_test(expf ExpfTest.cpp)
+add_conformance_test(expf16 Expf16Test.cpp)
+add_conformance_test(exp10 Exp10Test.cpp)
 add_conformance_test(exp10f Exp10fTest.cpp)
+add_conformance_test(exp10f16 Exp10f16Test.cpp)
+add_conformance_test(exp2 Exp2Test.cpp)
 add_conformance_test(exp2f Exp2fTest.cpp)
+add_conformance_test(exp2f16 Exp2f16Test.cpp)
+add_conformance_test(expm1 Expm1Test.cpp)
 add_conformance_test(expm1f Expm1fTest.cpp)
+add_conformance_test(expm1f16 Expm1f16Test.cpp)
+add_conformance_test(hypot HypotTest.cpp)
+add_conformance_test(hypotf HypotfTest.cpp)
 add_conformance_test(hypotf16 Hypotf16Test.cpp)
+add_conformance_test(log LogTest.cpp)
 add_conformance_test(logf LogfTest.cpp)
+add_conformance_test(logf16 Logf16Test.cpp)
+add_conformance_test(log10 Log10Test.cpp)
 add_conformance_test(log10f Log10fTest.cpp)
+add_conformance_test(log10f16 Log10f16Test.cpp)
+add_conformance_test(log1p Log1pTest.cpp)
 add_conformance_test(log1pf Log1pfTest.cpp)
+add_conformance_test(log2 Log2Test.cpp)
 add_conformance_test(log2f Log2fTest.cpp)
+add_conformance_test(log2f16 Log2f16Test.cpp)
+add_conformance_test(powf PowfTest.cpp)
+add_conformance_test(sin SinTest.cpp)
 add_conformance_test(sinf SinfTest.cpp)
+add_conformance_test(sinf16 Sinf16Test.cpp)
+add_conformance_test(sincos SincosTest.cpp)
 add_conformance_test(sincosf SincosfTest.cpp)
 add_conformance_test(sinhf SinhfTest.cpp)
+add_conformance_test(sinhf16 Sinhf16Test.cpp)
 add_conformance_test(sinpif SinpifTest.cpp)
+add_conformance_test(sinpif16 Sinpif16Test.cpp)
+add_conformance_test(tan TanTest.cpp)
 add_conformance_test(tanf TanfTest.cpp)
+add_conformance_test(tanf16 Tanf16Test.cpp)
 add_conformance_test(tanhf TanhfTest.cpp)
+add_conformance_test(tanhf16 Tanhf16Test.cpp)
 add_conformance_test(tanpif TanpifTest.cpp)
+add_conformance_test(tanpif16 Tanpif16Test.cpp)
diff --git a/offload/unittests/Conformance/tests/CbrtTest.cpp b/offload/unittests/Conformance/tests/CbrtTest.cpp
new file mode 100644
index 0000000000000..3a6523b66ad83
--- /dev/null
+++ b/offload/unittests/Conformance/tests/CbrtTest.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the cbrt function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'cbrt' function to select the double version
+constexpr auto cbrtd // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(cbrt);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<cbrtd> {
+  static constexpr llvm::StringRef Name = "cbrt";
+  static constexpr llvm::StringRef KernelName = "cbrtKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the cbrt function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range;
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<cbrtd>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/CosTest.cpp b/offload/unittests/Conformance/tests/CosTest.cpp
new file mode 100644
index 0000000000000..e3d3d3da81800
--- /dev/null
+++ b/offload/unittests/Conformance/tests/CosTest.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the cos function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'cos' function to select the double version
+constexpr auto cosd // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(cos);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<cosd> {
+  static constexpr llvm::StringRef Name = "cos";
+  static constexpr llvm::StringRef KernelName = "cosKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 4;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the cos function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range;
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<cosd>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Cosf16Test.cpp b/offload/unittests/Conformance/tests/Cosf16Test.cpp
new file mode 100644
index 0000000000000..680e4b99c549f
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Cosf16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the cosf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 cosf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<cosf16> {
+  static constexpr llvm::StringRef Name = "cosf16";
+  static constexpr llvm::StringRef KernelName = "cosf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the cosf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<cosf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Coshf16Test.cpp b/offload/unittests/Conformance/tests/Coshf16Test.cpp
new file mode 100644
index 0000000000000..1b378b5a9401a
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Coshf16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the coshf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 coshf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<coshf16> {
+  static constexpr llvm::StringRef Name = "coshf16";
+  static constexpr llvm::StringRef KernelName = "coshf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the coshf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<coshf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Cospif16Test.cpp b/offload/unittests/Conformance/tests/Cospif16Test.cpp
new file mode 100644
index 0000000000000..84aa682b48847
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Cospif16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the cospif16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 cospif16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<cospif16> {
+  static constexpr llvm::StringRef Name = "cospif16";
+  static constexpr llvm::StringRef KernelName = "cospif16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the cospif16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<cospif16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Exp10Test.cpp b/offload/unittests/Conformance/tests/Exp10Test.cpp
new file mode 100644
index 0000000000000..05af4780213b1
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Exp10Test.cpp
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the exp10 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'exp10' function to select the double version
+constexpr auto exp10d // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(exp10);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<exp10d> {
+  static constexpr llvm::StringRef Name = "exp10";
+  static constexpr llvm::StringRef KernelName = "exp10Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 3;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the exp10 function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range;
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<exp10d>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Exp10f16Test.cpp b/offload/unittests/Conformance/tests/Exp10f16Test.cpp
new file mode 100644
index 0000000000000..7d61ad0c6aef6
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Exp10f16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the exp10f16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 exp10f16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<exp10f16> {
+  static constexpr llvm::StringRef Name = "exp10f16";
+  static constexpr llvm::StringRef KernelName = "exp10f16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the exp10f16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<exp10f16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Exp2Test.cpp b/offload/unittests/Conformance/tests/Exp2Test.cpp
new file mode 100644
index 0000000000000..bb2fa10a0dfa6
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Exp2Test.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the exp2 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'exp2' function to select the double version
+constexpr auto exp2d // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(exp2);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<exp2d> {
+  static constexpr llvm::StringRef Name = "exp2";
+  static constexpr llvm::StringRef KernelName = "exp2Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 3;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the exp2 function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range;
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<exp2d>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Exp2f16Test.cpp b/offload/unittests/Conformance/tests/Exp2f16Test.cpp
new file mode 100644
index 0000000000000..9ea92564e7389
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Exp2f16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the exp2f16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 exp2f16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<exp2f16> {
+  static constexpr llvm::StringRef Name = "exp2f16";
+  static constexpr llvm::StringRef KernelName = "exp2f16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the exp2f16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<exp2f16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/ExpTest.cpp b/offload/unittests/Conformance/tests/ExpTest.cpp
new file mode 100644
index 0000000000000..9aa52b17905ea
--- /dev/null
+++ b/offload/unittests/Conformance/tests/ExpTest.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the exp function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'exp' function to select the double version
+constexpr auto expd // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(exp);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<expd> {
+  static constexpr llvm::StringRef Name = "exp";
+  static constexpr llvm::StringRef KernelName = "expKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 3;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the exp function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range;
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<expd>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Expf16Test.cpp b/offload/unittests/Conformance/tests/Expf16Test.cpp
new file mode 100644
index 0000000000000..8938815e26a8c
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Expf16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the expf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 expf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<expf16> {
+  static constexpr llvm::StringRef Name = "expf16";
+  static constexpr llvm::StringRef KernelName = "expf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the expf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<expf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Expm1Test.cpp b/offload/unittests/Conformance/tests/Expm1Test.cpp
new file mode 100644
index 0000000000000..a27944bf722f8
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Expm1Test.cpp
@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the expm1 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'expm1' function to select the double version
+constexpr auto expm1d // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(expm1);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<expm1d> {
+  static constexpr llvm::StringRef Name = "expm1";
+  static constexpr llvm::StringRef KernelName = "expm1Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 3;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the expm1 function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range;
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<expm1d>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Expm1f16Test.cpp b/offload/unittests/Conformance/tests/Expm1f16Test.cpp
new file mode 100644
index 0000000000000..447196bb8ea37
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Expm1f16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the expm1f16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 expm1f16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<expm1f16> {
+  static constexpr llvm::StringRef Name = "expm1f16";
+  static constexpr llvm::StringRef KernelName = "expm1f16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the expm1f16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<expm1f16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/HypotTest.cpp b/offload/unittests/Conformance/tests/HypotTest.cpp
new file mode 100644
index 0000000000000..0417ad901d5ee
--- /dev/null
+++ b/offload/unittests/Conformance/tests/HypotTest.cpp
@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the hypot function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'hypot' function to select the double version
+constexpr auto hypotd // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double, double)>(hypot);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<hypotd> {
+  static constexpr llvm::StringRef Name = "hypot";
+  static constexpr llvm::StringRef KernelName = "hypotKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 4;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the hypot function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> RangeX;
+  IndexedRange<double> RangeY;
+  RandomGenerator<double, double> Generator(SeedTy{Seed}, Size, RangeX, RangeY);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<hypotd>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/HypotfTest.cpp b/offload/unittests/Conformance/tests/HypotfTest.cpp
new file mode 100644
index 0000000000000..98a4e906920d5
--- /dev/null
+++ b/offload/unittests/Conformance/tests/HypotfTest.cpp
@@ -0,0 +1,58 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the hypotf function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace mathtest {
+
+template <> struct FunctionConfig<hypotf> {
+  static constexpr llvm::StringRef Name = "hypotf";
+  static constexpr llvm::StringRef KernelName = "hypotfKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 65, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 4;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the hypotf function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<float> RangeX;
+  IndexedRange<float> RangeY;
+  RandomGenerator<float, float> Generator(SeedTy{Seed}, Size, RangeX, RangeY);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<hypotf>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Log10Test.cpp b/offload/unittests/Conformance/tests/Log10Test.cpp
new file mode 100644
index 0000000000000..bf46f11e960bf
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Log10Test.cpp
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the log10 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <limits>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'log10' function to select the double version
+constexpr auto log10d // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(log10);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<log10d> {
+  static constexpr llvm::StringRef Name = "log10";
+  static constexpr llvm::StringRef KernelName = "log10Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 3;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the log10 function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range(/*Begin=*/0.0,
+                             /*End=*/std::numeric_limits<double>::infinity(),
+                             /*Inclusive=*/true);
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<log10d>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Log10f16Test.cpp b/offload/unittests/Conformance/tests/Log10f16Test.cpp
new file mode 100644
index 0000000000000..605e1ae49077e
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Log10f16Test.cpp
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the log10f16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/Numerics.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 log10f16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<log10f16> {
+  static constexpr llvm::StringRef Name = "log10f16";
+  static constexpr llvm::StringRef KernelName = "log10f16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the log10f16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range(/*Begin=*/float16(0.0),
+                              /*End=*/getMaxOrInf<float16>(),
+                              /*Inclusive=*/true);
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<log10f16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Log1pTest.cpp b/offload/unittests/Conformance/tests/Log1pTest.cpp
new file mode 100644
index 0000000000000..023b67e770de0
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Log1pTest.cpp
@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the log1p function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <limits>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'log1p' function to select the double version
+constexpr auto log1pd // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(log1p);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<log1pd> {
+  static constexpr llvm::StringRef Name = "log1p";
+  static constexpr llvm::StringRef KernelName = "log1pKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the log1p function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range(/*Begin=*/-1.0,
+                             /*End=*/std::numeric_limits<double>::infinity(),
+                             /*Inclusive=*/true);
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<log1pd>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Log2Test.cpp b/offload/unittests/Conformance/tests/Log2Test.cpp
new file mode 100644
index 0000000000000..2ae7e5c232928
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Log2Test.cpp
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the log2 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <limits>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'log2' function to select the double version
+constexpr auto log2d // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(log2);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<log2d> {
+  static constexpr llvm::StringRef Name = "log2";
+  static constexpr llvm::StringRef KernelName = "log2Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 3;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the log2 function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range(/*Begin=*/0.0,
+                             /*End=*/std::numeric_limits<double>::infinity(),
+                             /*Inclusive=*/true);
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<log2d>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Log2f16Test.cpp b/offload/unittests/Conformance/tests/Log2f16Test.cpp
new file mode 100644
index 0000000000000..5ce46960774a0
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Log2f16Test.cpp
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the log2f16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/Numerics.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 log2f16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<log2f16> {
+  static constexpr llvm::StringRef Name = "log2f16";
+  static constexpr llvm::StringRef KernelName = "log2f16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the log2f16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range(/*Begin=*/float16(0.0),
+                              /*End=*/getMaxOrInf<float16>(),
+                              /*Inclusive=*/true);
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<log2f16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/LogTest.cpp b/offload/unittests/Conformance/tests/LogTest.cpp
new file mode 100644
index 0000000000000..ae568e2c47404
--- /dev/null
+++ b/offload/unittests/Conformance/tests/LogTest.cpp
@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the log function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <limits>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'log' function to select the double version
+constexpr auto logd // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(log);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<logd> {
+  static constexpr llvm::StringRef Name = "log";
+  static constexpr llvm::StringRef KernelName = "logKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 3;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the log function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range(/*Begin=*/0.0,
+                             /*End=*/std::numeric_limits<double>::infinity(),
+                             /*Inclusive=*/true);
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<logd>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Logf16Test.cpp b/offload/unittests/Conformance/tests/Logf16Test.cpp
new file mode 100644
index 0000000000000..372dccb2ebb9a
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Logf16Test.cpp
@@ -0,0 +1,62 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the logf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/Numerics.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 logf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<logf16> {
+  static constexpr llvm::StringRef Name = "logf16";
+  static constexpr llvm::StringRef KernelName = "logf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the logf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range(/*Begin=*/float16(0.0),
+                              /*End=*/getMaxOrInf<float16>(),
+                              /*Inclusive=*/true);
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<logf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/PowfTest.cpp b/offload/unittests/Conformance/tests/PowfTest.cpp
new file mode 100644
index 0000000000000..246801e390aea
--- /dev/null
+++ b/offload/unittests/Conformance/tests/PowfTest.cpp
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the powf function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+static inline float powfRoundedExponent(float Base, float Exponent) {
+  return powf(Base, roundf(Exponent));
+}
+
+namespace mathtest {
+
+template <> struct FunctionConfig<powf> {
+  static constexpr llvm::StringRef Name = "powf (real exponents)";
+  static constexpr llvm::StringRef KernelName = "powfKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 65, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 16;
+};
+
+template <> struct FunctionConfig<powfRoundedExponent> {
+  static constexpr llvm::StringRef Name = "powf (integer exponents)";
+  static constexpr llvm::StringRef KernelName = "powfRoundedExponentKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 65, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 16;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the powf function");
+
+  using namespace mathtest;
+
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<float> RangeX;
+  IndexedRange<float> RangeY;
+  RandomGenerator<float, float> Generator0(SeedTy{42}, Size, RangeX, RangeY);
+  RandomGenerator<float, float> Generator1(SeedTy{51}, Size, RangeX, RangeY);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool RealExponentsPassed =
+      runTests<powf>(Generator0, Configs, DeviceBinaryDir, IsVerbose);
+  bool IntegerExponentsPassed = runTests<powfRoundedExponent>(
+      Generator1, Configs, DeviceBinaryDir, IsVerbose);
+
+  return (RealExponentsPassed && IntegerExponentsPassed) ? EXIT_SUCCESS
+                                                         : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/SinTest.cpp b/offload/unittests/Conformance/tests/SinTest.cpp
new file mode 100644
index 0000000000000..36897d74c96af
--- /dev/null
+++ b/offload/unittests/Conformance/tests/SinTest.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the sin function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'sin' function to select the double version
+constexpr auto sind // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(sin);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<sind> {
+  static constexpr llvm::StringRef Name = "sin";
+  static constexpr llvm::StringRef KernelName = "sinKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 4;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the sin function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range;
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<sind>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/SincosTest.cpp b/offload/unittests/Conformance/tests/SincosTest.cpp
new file mode 100644
index 0000000000000..a3d1650c54e4b
--- /dev/null
+++ b/offload/unittests/Conformance/tests/SincosTest.cpp
@@ -0,0 +1,80 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the sincos function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+static inline double sincosSin(double X) {
+  double SinX, CosX;
+  sincos(X, &SinX, &CosX);
+  return SinX;
+}
+
+static inline double sincosCos(double X) {
+  double SinX, CosX;
+  sincos(X, &SinX, &CosX);
+  return CosX;
+}
+
+namespace mathtest {
+
+template <> struct FunctionConfig<sincosSin> {
+  static constexpr llvm::StringRef Name = "sincos (sin part)";
+  static constexpr llvm::StringRef KernelName = "sincosSinKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 4;
+};
+
+template <> struct FunctionConfig<sincosCos> {
+  static constexpr llvm::StringRef Name = "sincos (cos part)";
+  static constexpr llvm::StringRef KernelName = "sincosCosKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 4;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the sincos function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range;
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool SinPartPassed =
+      runTests<sincosSin>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+  bool CosPartPassed =
+      runTests<sincosCos>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return (SinPartPassed && CosPartPassed) ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Sinf16Test.cpp b/offload/unittests/Conformance/tests/Sinf16Test.cpp
new file mode 100644
index 0000000000000..4c5fb22262889
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Sinf16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the sinf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 sinf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<sinf16> {
+  static constexpr llvm::StringRef Name = "sinf16";
+  static constexpr llvm::StringRef KernelName = "sinf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the sinf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<sinf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Sinhf16Test.cpp b/offload/unittests/Conformance/tests/Sinhf16Test.cpp
new file mode 100644
index 0000000000000..fe6f7dd4a49c2
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Sinhf16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the sinhf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 sinhf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<sinhf16> {
+  static constexpr llvm::StringRef Name = "sinhf16";
+  static constexpr llvm::StringRef KernelName = "sinhf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the sinhf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<sinhf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Sinpif16Test.cpp b/offload/unittests/Conformance/tests/Sinpif16Test.cpp
new file mode 100644
index 0000000000000..ff9c93c0bb5b8
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Sinpif16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the sinpif16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 sinpif16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<sinpif16> {
+  static constexpr llvm::StringRef Name = "sinpif16";
+  static constexpr llvm::StringRef KernelName = "sinpif16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the sinpif16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<sinpif16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/TanTest.cpp b/offload/unittests/Conformance/tests/TanTest.cpp
new file mode 100644
index 0000000000000..3a9a05874450e
--- /dev/null
+++ b/offload/unittests/Conformance/tests/TanTest.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the tan function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/RandomGenerator.hpp"
+#include "mathtest/RandomState.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+namespace {
+
+// Disambiguate the overloaded 'tan' function to select the double version
+constexpr auto tand // NOLINT(readability-identifier-naming)
+    = static_cast<double (*)(double)>(tan);
+} // namespace
+
+namespace mathtest {
+
+template <> struct FunctionConfig<tand> {
+  static constexpr llvm::StringRef Name = "tan";
+  static constexpr llvm::StringRef KernelName = "tanKernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 68, Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 5;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the tan function");
+
+  using namespace mathtest;
+
+  uint64_t Seed = 42;
+  uint64_t Size = 1ULL << 32;
+  IndexedRange<double> Range;
+  RandomGenerator<double> Generator(SeedTy{Seed}, Size, Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed = runTests<tand>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Tanf16Test.cpp b/offload/unittests/Conformance/tests/Tanf16Test.cpp
new file mode 100644
index 0000000000000..eae9818830a2f
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Tanf16Test.cpp
@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the tanf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 tanf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<tanf16> {
+  static constexpr llvm::StringRef Name = "tanf16";
+  static constexpr llvm::StringRef KernelName = "tanf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  // Note:   The minimum accuracy at the source is 2.5 ULP, but we round it
+  //         down to ensure conformance.
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the tanf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<tanf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Tanhf16Test.cpp b/offload/unittests/Conformance/tests/Tanhf16Test.cpp
new file mode 100644
index 0000000000000..1a11f3da7f09a
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Tanhf16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the tanhf16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 tanhf16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<tanhf16> {
+  static constexpr llvm::StringRef Name = "tanhf16";
+  static constexpr llvm::StringRef KernelName = "tanhf16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(argc, argv,
+                                    "Conformance test of the tanhf16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<tanhf16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/Conformance/tests/Tanpif16Test.cpp b/offload/unittests/Conformance/tests/Tanpif16Test.cpp
new file mode 100644
index 0000000000000..76374807b92f1
--- /dev/null
+++ b/offload/unittests/Conformance/tests/Tanpif16Test.cpp
@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the conformance test of the tanpif16 function.
+///
+//===----------------------------------------------------------------------===//
+
+#include "mathtest/CommandLineExtras.hpp"
+#include "mathtest/ExhaustiveGenerator.hpp"
+#include "mathtest/IndexedRange.hpp"
+#include "mathtest/TestConfig.hpp"
+#include "mathtest/TestRunner.hpp"
+#include "mathtest/TypeExtras.hpp"
+
+#include "llvm/ADT/StringRef.h"
+
+#include <cstdlib>
+#include <math.h>
+
+using namespace mathtest;
+
+extern "C" float16 tanpif16(float16);
+
+namespace mathtest {
+
+template <> struct FunctionConfig<tanpif16> {
+  static constexpr llvm::StringRef Name = "tanpif16";
+  static constexpr llvm::StringRef KernelName = "tanpif16Kernel";
+
+  // Source: The Khronos Group, The OpenCL C Specification v3.0.19, Sec. 7.4,
+  //         Table 69 (Full Profile), Khronos Registry [July 10, 2025].
+  static constexpr uint64_t UlpTolerance = 2;
+};
+} // namespace mathtest
+
+int main(int argc, const char **argv) {
+  llvm::cl::ParseCommandLineOptions(
+      argc, argv, "Conformance test of the tanpif16 function");
+
+  using namespace mathtest;
+
+  IndexedRange<float16> Range;
+  ExhaustiveGenerator<float16> Generator(Range);
+
+  const auto Configs = cl::getTestConfigs();
+  const llvm::StringRef DeviceBinaryDir = DEVICE_BINARY_DIR;
+  const bool IsVerbose = cl::IsVerbose;
+
+  bool Passed =
+      runTests<tanpif16>(Generator, Configs, DeviceBinaryDir, IsVerbose);
+
+  return Passed ? EXIT_SUCCESS : EXIT_FAILURE;
+}
diff --git a/offload/unittests/OffloadAPI/CMakeLists.txt b/offload/unittests/OffloadAPI/CMakeLists.txt
index 8f0267eb39bdf..b2d514423a6ee 100644
--- a/offload/unittests/OffloadAPI/CMakeLists.txt
+++ b/offload/unittests/OffloadAPI/CMakeLists.txt
@@ -20,10 +20,12 @@ add_offload_unittest("init"
 target_compile_definitions("init.unittests" PRIVATE DISABLE_WRAPPER)
 
 add_offload_unittest("kernel"
+    kernel/olCalculateOptimalOccupancy.cpp
     kernel/olLaunchKernel.cpp)
 
 add_offload_unittest("memory"
     memory/olMemAlloc.cpp
+    memory/olMemFill.cpp
     memory/olMemFree.cpp
     memory/olMemcpy.cpp)
 
@@ -41,7 +43,8 @@ add_offload_unittest("queue"
     queue/olDestroyQueue.cpp
     queue/olGetQueueInfo.cpp
     queue/olGetQueueInfoSize.cpp
-    queue/olWaitEvents.cpp)
+    queue/olWaitEvents.cpp
+    queue/olLaunchHostFunction.cpp)
 
 add_offload_unittest("symbol"
     symbol/olGetSymbol.cpp
diff --git a/offload/unittests/OffloadAPI/common/Fixtures.hpp b/offload/unittests/OffloadAPI/common/Fixtures.hpp
index 24ce1a0cf7d7e..fe7198a9c283f 100644
--- a/offload/unittests/OffloadAPI/common/Fixtures.hpp
+++ b/offload/unittests/OffloadAPI/common/Fixtures.hpp
@@ -9,6 +9,7 @@
 #include <OffloadAPI.h>
 #include <OffloadPrint.hpp>
 #include <gtest/gtest.h>
+#include <thread>
 
 #include "Environment.hpp"
 
@@ -25,6 +26,20 @@
   } while (0)
 #endif
 
+#ifndef ASSERT_SUCCESS_OR_UNSUPPORTED
+#define ASSERT_SUCCESS_OR_UNSUPPORTED(ACTUAL)                                  \
+  do {                                                                         \
+    ol_result_t Res = ACTUAL;                                                  \
+    if (Res && Res->Code == OL_ERRC_UNSUPPORTED) {                             \
+      GTEST_SKIP() << #ACTUAL " returned unsupported; skipping test";          \
+      return;                                                                  \
+    } else if (Res && Res->Code != OL_ERRC_SUCCESS) {                          \
+      GTEST_FAIL() << #ACTUAL " returned " << Res->Code << ": "                \
+                   << Res->Details;                                            \
+    }                                                                          \
+  } while (0)
+#endif
+
 // TODO: rework this so the EXPECTED/ACTUAL results are readable
 #ifndef ASSERT_ERROR
 #define ASSERT_ERROR(EXPECTED, ACTUAL)                                         \
@@ -57,6 +72,23 @@ inline std::string SanitizeString(const std::string &Str) {
   return NewStr;
 }
 
+template <typename Fn> inline void threadify(Fn body) {
+  std::vector<std::thread> Threads;
+  for (size_t I = 0; I < 20; I++) {
+    Threads.emplace_back(
+        [&body](size_t I) {
+          std::string ScopeMsg{"Thread #"};
+          ScopeMsg.append(std::to_string(I));
+          SCOPED_TRACE(ScopeMsg);
+          body(I);
+        },
+        I);
+  }
+  for (auto &T : Threads) {
+    T.join();
+  }
+}
+
 struct OffloadTest : ::testing::Test {
   ol_device_handle_t Host = TestEnvironment::getHostDevice();
 };
diff --git a/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp b/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
index 5657320a33a29..212a5d6ddf228 100644
--- a/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
+++ b/offload/unittests/OffloadAPI/device/olGetDeviceInfo.cpp
@@ -13,6 +13,38 @@
 using olGetDeviceInfoTest = OffloadDeviceTest;
 OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olGetDeviceInfoTest);
 
+#define OL_DEVICE_INFO_TEST_SUCCESS_CHECK(TestName, PropType, PropName, Dev,   \
+                                          Expr)                                \
+  TEST_P(olGetDeviceInfoTest, Test##Dev##TestName) {                           \
+    PropType Value;                                                            \
+    ASSERT_SUCCESS(olGetDeviceInfo(Dev, PropName, sizeof(Value), &Value));     \
+    Expr;                                                                      \
+  }
+
+#define OL_DEVICE_INFO_TEST_DEVICE_SUCCESS(TestName, PropType, PropName)       \
+  OL_DEVICE_INFO_TEST_SUCCESS_CHECK(TestName, PropType, PropName, Device, {})
+
+#define OL_DEVICE_INFO_TEST_HOST_SUCCESS(TestName, PropType, PropName)         \
+  OL_DEVICE_INFO_TEST_SUCCESS_CHECK(TestName, PropType, PropName, Host, {})
+
+#define OL_DEVICE_INFO_TEST_SUCCESS(TestName, PropType, PropName)              \
+  OL_DEVICE_INFO_TEST_DEVICE_SUCCESS(TestName, PropType, PropName)             \
+  OL_DEVICE_INFO_TEST_HOST_SUCCESS(TestName, PropType, PropName)
+
+#define OL_DEVICE_INFO_TEST_DEVICE_VALUE_GT(TestName, PropType, PropName,      \
+                                            LowBound)                          \
+  OL_DEVICE_INFO_TEST_SUCCESS_CHECK(TestName, PropType, PropName, Device,      \
+                                    ASSERT_GT(Value, LowBound))
+
+#define OL_DEVICE_INFO_TEST_HOST_VALUE_GT(TestName, PropType, PropName,        \
+                                          LowBound)                            \
+  OL_DEVICE_INFO_TEST_SUCCESS_CHECK(TestName, PropType, PropName, Host,        \
+                                    ASSERT_GT(Value, LowBound))
+
+#define OL_DEVICE_INFO_TEST_VALUE_GT(TestName, PropType, PropName, LowBound)   \
+  OL_DEVICE_INFO_TEST_DEVICE_VALUE_GT(TestName, PropType, PropName, LowBound)  \
+  OL_DEVICE_INFO_TEST_HOST_VALUE_GT(TestName, PropType, PropName, LowBound)
+
 TEST_P(olGetDeviceInfoTest, SuccessType) {
   ol_device_type_t DeviceType;
   ASSERT_SUCCESS(olGetDeviceInfo(Device, OL_DEVICE_INFO_TYPE,
@@ -77,12 +109,8 @@ TEST_P(olGetDeviceInfoTest, SuccessDriverVersion) {
   ASSERT_EQ(std::strlen(DriverVersion.data()), Size - 1);
 }
 
-TEST_P(olGetDeviceInfoTest, SuccessMaxWorkGroupSize) {
-  uint32_t Value;
-  ASSERT_SUCCESS(olGetDeviceInfo(Device, OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE,
-                                 sizeof(Value), &Value));
-  ASSERT_GT(Value, 0u);
-}
+OL_DEVICE_INFO_TEST_VALUE_GT(MaxWorkGroupSize, uint32_t,
+                             OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE, 0);
 
 TEST_P(olGetDeviceInfoTest, SuccessMaxWorkGroupSizePerDimension) {
   ol_dimensions_t Value{0, 0, 0};
@@ -94,6 +122,46 @@ TEST_P(olGetDeviceInfoTest, SuccessMaxWorkGroupSizePerDimension) {
   ASSERT_GT(Value.z, 0u);
 }
 
+OL_DEVICE_INFO_TEST_DEVICE_VALUE_GT(VendorId, uint32_t,
+                                    OL_DEVICE_INFO_VENDOR_ID, 0);
+OL_DEVICE_INFO_TEST_HOST_SUCCESS(VendorId, uint32_t, OL_DEVICE_INFO_VENDOR_ID);
+OL_DEVICE_INFO_TEST_VALUE_GT(NumComputeUnits, uint32_t,
+                             OL_DEVICE_INFO_NUM_COMPUTE_UNITS, 0);
+OL_DEVICE_INFO_TEST_VALUE_GT(SingleFPConfig, ol_device_fp_capability_flags_t,
+                             OL_DEVICE_INFO_SINGLE_FP_CONFIG, 0);
+OL_DEVICE_INFO_TEST_SUCCESS(HalfFPConfig, ol_device_fp_capability_flags_t,
+                            OL_DEVICE_INFO_HALF_FP_CONFIG);
+OL_DEVICE_INFO_TEST_VALUE_GT(DoubleFPConfig, ol_device_fp_capability_flags_t,
+                             OL_DEVICE_INFO_DOUBLE_FP_CONFIG, 0);
+OL_DEVICE_INFO_TEST_VALUE_GT(NativeVectorWidthChar, uint32_t,
+                             OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR, 0);
+OL_DEVICE_INFO_TEST_VALUE_GT(NativeVectorWidthShort, uint32_t,
+                             OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT, 0);
+OL_DEVICE_INFO_TEST_VALUE_GT(NativeVectorWidthInt, uint32_t,
+                             OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT, 0);
+OL_DEVICE_INFO_TEST_VALUE_GT(NativeVectorWidthLong, uint32_t,
+                             OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG, 0);
+OL_DEVICE_INFO_TEST_VALUE_GT(NativeVectorWidthFloat, uint32_t,
+                             OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT, 0);
+OL_DEVICE_INFO_TEST_VALUE_GT(NativeVectorWidthDouble, uint32_t,
+                             OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE, 0);
+OL_DEVICE_INFO_TEST_SUCCESS(NativeVectorWidthHalf, uint32_t,
+                            OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF);
+OL_DEVICE_INFO_TEST_VALUE_GT(MaxClockFrequency, uint32_t,
+                             OL_DEVICE_INFO_MAX_CLOCK_FREQUENCY, 0);
+OL_DEVICE_INFO_TEST_VALUE_GT(MemoryClockRate, uint32_t,
+                             OL_DEVICE_INFO_MEMORY_CLOCK_RATE, 0);
+OL_DEVICE_INFO_TEST_VALUE_GT(AddressBits, uint32_t, OL_DEVICE_INFO_ADDRESS_BITS,
+                             0);
+OL_DEVICE_INFO_TEST_DEVICE_VALUE_GT(MaxMemAllocSize, uint64_t,
+                                    OL_DEVICE_INFO_MAX_MEM_ALLOC_SIZE, 0);
+OL_DEVICE_INFO_TEST_HOST_SUCCESS(MaxMemAllocSize, uint64_t,
+                                 OL_DEVICE_INFO_MAX_MEM_ALLOC_SIZE);
+OL_DEVICE_INFO_TEST_DEVICE_VALUE_GT(GlobalMemSize, uint64_t,
+                                    OL_DEVICE_INFO_GLOBAL_MEM_SIZE, 0);
+OL_DEVICE_INFO_TEST_HOST_SUCCESS(GlobalMemSize, uint64_t,
+                                 OL_DEVICE_INFO_GLOBAL_MEM_SIZE);
+
 TEST_P(olGetDeviceInfoTest, InvalidNullHandleDevice) {
   ol_device_type_t DeviceType;
   ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
diff --git a/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp b/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
index 4e29978fc20f0..a28089d918e03 100644
--- a/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
+++ b/offload/unittests/OffloadAPI/device/olGetDeviceInfoSize.cpp
@@ -13,43 +13,60 @@
 using olGetDeviceInfoSizeTest = OffloadDeviceTest;
 OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olGetDeviceInfoSizeTest);
 
-TEST_P(olGetDeviceInfoSizeTest, SuccessType) {
-  size_t Size = 0;
-  ASSERT_SUCCESS(olGetDeviceInfoSize(Device, OL_DEVICE_INFO_TYPE, &Size));
-  ASSERT_EQ(Size, sizeof(ol_device_type_t));
-}
-
-TEST_P(olGetDeviceInfoSizeTest, SuccessPlatform) {
-  size_t Size = 0;
-  ASSERT_SUCCESS(olGetDeviceInfoSize(Device, OL_DEVICE_INFO_PLATFORM, &Size));
-  ASSERT_EQ(Size, sizeof(ol_platform_handle_t));
-}
-
-TEST_P(olGetDeviceInfoSizeTest, SuccessName) {
-  size_t Size = 0;
-  ASSERT_SUCCESS(olGetDeviceInfoSize(Device, OL_DEVICE_INFO_NAME, &Size));
-  ASSERT_NE(Size, 0ul);
-}
+#define OL_DEVICE_INFO_SIZE_TEST(TestName, PropName, Expr)                     \
+  TEST_P(olGetDeviceInfoSizeTest, Success##TestName) {                         \
+    size_t Size = 0;                                                           \
+    ASSERT_SUCCESS(olGetDeviceInfoSize(Device, PropName, &Size));              \
+    Expr;                                                                      \
+  }
 
-TEST_P(olGetDeviceInfoSizeTest, SuccessVendor) {
-  size_t Size = 0;
-  ASSERT_SUCCESS(olGetDeviceInfoSize(Device, OL_DEVICE_INFO_VENDOR, &Size));
-  ASSERT_NE(Size, 0ul);
-}
+#define OL_DEVICE_INFO_SIZE_TEST_EQ(TestName, PropType, PropName)              \
+  OL_DEVICE_INFO_SIZE_TEST(TestName, PropName,                                 \
+                           ASSERT_EQ(Size, sizeof(PropType)));
 
-TEST_P(olGetDeviceInfoSizeTest, SuccessDriverVersion) {
-  size_t Size = 0;
-  ASSERT_SUCCESS(
-      olGetDeviceInfoSize(Device, OL_DEVICE_INFO_DRIVER_VERSION, &Size));
-  ASSERT_NE(Size, 0ul);
-}
+#define OL_DEVICE_INFO_SIZE_TEST_NONZERO(TestName, PropName)                   \
+  OL_DEVICE_INFO_SIZE_TEST(TestName, PropName, ASSERT_NE(Size, 0ul));
 
-TEST_P(olGetDeviceInfoSizeTest, SuccessMaxWorkGroupSize) {
-  size_t Size = 0;
-  ASSERT_SUCCESS(
-      olGetDeviceInfoSize(Device, OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE, &Size));
-  ASSERT_EQ(Size, sizeof(uint32_t));
-}
+OL_DEVICE_INFO_SIZE_TEST_EQ(Type, ol_device_type_t, OL_DEVICE_INFO_TYPE);
+OL_DEVICE_INFO_SIZE_TEST_EQ(Platform, ol_platform_handle_t,
+                            OL_DEVICE_INFO_PLATFORM);
+OL_DEVICE_INFO_SIZE_TEST_NONZERO(Name, OL_DEVICE_INFO_NAME);
+OL_DEVICE_INFO_SIZE_TEST_NONZERO(Vendor, OL_DEVICE_INFO_VENDOR);
+OL_DEVICE_INFO_SIZE_TEST_NONZERO(DriverVersion, OL_DEVICE_INFO_DRIVER_VERSION);
+OL_DEVICE_INFO_SIZE_TEST_EQ(MaxWorkGroupSize, uint32_t,
+                            OL_DEVICE_INFO_MAX_WORK_GROUP_SIZE);
+OL_DEVICE_INFO_SIZE_TEST_EQ(VendorId, uint32_t, OL_DEVICE_INFO_VENDOR_ID);
+OL_DEVICE_INFO_SIZE_TEST_EQ(NumComputeUnits, uint32_t,
+                            OL_DEVICE_INFO_NUM_COMPUTE_UNITS);
+OL_DEVICE_INFO_SIZE_TEST_EQ(SingleFPConfig, ol_device_fp_capability_flags_t,
+                            OL_DEVICE_INFO_SINGLE_FP_CONFIG);
+OL_DEVICE_INFO_SIZE_TEST_EQ(HalfFPConfig, ol_device_fp_capability_flags_t,
+                            OL_DEVICE_INFO_HALF_FP_CONFIG);
+OL_DEVICE_INFO_SIZE_TEST_EQ(DoubleFPConfig, ol_device_fp_capability_flags_t,
+                            OL_DEVICE_INFO_DOUBLE_FP_CONFIG);
+OL_DEVICE_INFO_SIZE_TEST_EQ(NativeVectorWidthChar, uint32_t,
+                            OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_CHAR);
+OL_DEVICE_INFO_SIZE_TEST_EQ(NativeVectorWidthShort, uint32_t,
+                            OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_SHORT);
+OL_DEVICE_INFO_SIZE_TEST_EQ(NativeVectorWidthInt, uint32_t,
+                            OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_INT);
+OL_DEVICE_INFO_SIZE_TEST_EQ(NativeVectorWidthLong, uint32_t,
+                            OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_LONG);
+OL_DEVICE_INFO_SIZE_TEST_EQ(NativeVectorWidthFloat, uint32_t,
+                            OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_FLOAT);
+OL_DEVICE_INFO_SIZE_TEST_EQ(NativeVectorWidthDouble, uint32_t,
+                            OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_DOUBLE);
+OL_DEVICE_INFO_SIZE_TEST_EQ(NativeVectorWidthHalf, uint32_t,
+                            OL_DEVICE_INFO_NATIVE_VECTOR_WIDTH_HALF);
+OL_DEVICE_INFO_SIZE_TEST_EQ(MaxClockFrequency, uint32_t,
+                            OL_DEVICE_INFO_MAX_CLOCK_FREQUENCY);
+OL_DEVICE_INFO_SIZE_TEST_EQ(MemoryClockRate, uint32_t,
+                            OL_DEVICE_INFO_MEMORY_CLOCK_RATE);
+OL_DEVICE_INFO_SIZE_TEST_EQ(AddressBits, uint32_t, OL_DEVICE_INFO_ADDRESS_BITS);
+OL_DEVICE_INFO_SIZE_TEST_EQ(MaxMemAllocSize, uint64_t,
+                            OL_DEVICE_INFO_MAX_MEM_ALLOC_SIZE);
+OL_DEVICE_INFO_SIZE_TEST_EQ(GlobalMemSize, uint64_t,
+                            OL_DEVICE_INFO_GLOBAL_MEM_SIZE);
 
 TEST_P(olGetDeviceInfoSizeTest, SuccessMaxWorkGroupSizePerDimension) {
   size_t Size = 0;
diff --git a/offload/unittests/OffloadAPI/event/olGetEventInfo.cpp b/offload/unittests/OffloadAPI/event/olGetEventInfo.cpp
index 908d2dcb6df5d..b86d15f045ebc 100644
--- a/offload/unittests/OffloadAPI/event/olGetEventInfo.cpp
+++ b/offload/unittests/OffloadAPI/event/olGetEventInfo.cpp
@@ -13,13 +13,22 @@
 using olGetEventInfoTest = OffloadEventTest;
 OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olGetEventInfoTest);
 
-TEST_P(olGetEventInfoTest, SuccessDevice) {
+TEST_P(olGetEventInfoTest, SuccessQueue) {
   ol_queue_handle_t RetrievedQueue;
   ASSERT_SUCCESS(olGetEventInfo(Event, OL_EVENT_INFO_QUEUE,
                                 sizeof(ol_queue_handle_t), &RetrievedQueue));
   ASSERT_EQ(Queue, RetrievedQueue);
 }
 
+TEST_P(olGetEventInfoTest, SuccessIsComplete) {
+  bool Complete = false;
+  while (!Complete) {
+    ASSERT_SUCCESS(olGetEventInfo(Event, OL_EVENT_INFO_IS_COMPLETE,
+                                  sizeof(Complete), &Complete));
+  }
+  ASSERT_EQ(Complete, true);
+}
+
 TEST_P(olGetEventInfoTest, InvalidNullHandle) {
   ol_queue_handle_t RetrievedQueue;
   ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
diff --git a/offload/unittests/OffloadAPI/event/olGetEventInfoSize.cpp b/offload/unittests/OffloadAPI/event/olGetEventInfoSize.cpp
index d7dee58e35e8d..36f36c3a187f2 100644
--- a/offload/unittests/OffloadAPI/event/olGetEventInfoSize.cpp
+++ b/offload/unittests/OffloadAPI/event/olGetEventInfoSize.cpp
@@ -19,6 +19,12 @@ TEST_P(olGetEventInfoSizeTest, SuccessQueue) {
   ASSERT_EQ(Size, sizeof(ol_queue_handle_t));
 }
 
+TEST_P(olGetEventInfoSizeTest, SuccessIsComplete) {
+  size_t Size = 0;
+  ASSERT_SUCCESS(olGetEventInfoSize(Event, OL_EVENT_INFO_IS_COMPLETE, &Size));
+  ASSERT_EQ(Size, sizeof(bool));
+}
+
 TEST_P(olGetEventInfoSizeTest, InvalidNullHandle) {
   size_t Size = 0;
   ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
diff --git a/offload/unittests/OffloadAPI/kernel/olCalculateOptimalOccupancy.cpp b/offload/unittests/OffloadAPI/kernel/olCalculateOptimalOccupancy.cpp
new file mode 100644
index 0000000000000..17fa383cac3f4
--- /dev/null
+++ b/offload/unittests/OffloadAPI/kernel/olCalculateOptimalOccupancy.cpp
@@ -0,0 +1,45 @@
+//===------- Offload API tests - olCalculateOptimalOccupancy --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+using olCalculateOptimalOccupancyTest = OffloadKernelTest;
+OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olCalculateOptimalOccupancyTest);
+
+TEST_P(olCalculateOptimalOccupancyTest, Success) {
+  size_t Size{0};
+  ASSERT_SUCCESS_OR_UNSUPPORTED(
+      olCalculateOptimalOccupancy(Device, Kernel, 0, &Size));
+  ASSERT_GT(Size, 0u);
+}
+
+TEST_P(olCalculateOptimalOccupancyTest, SuccessMem) {
+  size_t Size{0};
+  ASSERT_SUCCESS_OR_UNSUPPORTED(
+      olCalculateOptimalOccupancy(Device, Kernel, 1024, &Size));
+  ASSERT_GT(Size, 0u);
+}
+
+TEST_P(olCalculateOptimalOccupancyTest, NullKernel) {
+  size_t Size;
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
+               olCalculateOptimalOccupancy(Device, nullptr, 0, &Size));
+}
+
+TEST_P(olCalculateOptimalOccupancyTest, NullDevice) {
+  size_t Size;
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
+               olCalculateOptimalOccupancy(nullptr, Kernel, 0, &Size));
+}
+
+TEST_P(olCalculateOptimalOccupancyTest, NullOutput) {
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER,
+               olCalculateOptimalOccupancy(Device, Kernel, 0, nullptr));
+}
diff --git a/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp b/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
index 758f6071b030f..1dac8c50271b5 100644
--- a/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
+++ b/offload/unittests/OffloadAPI/kernel/olLaunchKernel.cpp
@@ -104,6 +104,29 @@ TEST_P(olLaunchKernelFooTest, Success) {
   ASSERT_SUCCESS(olMemFree(Mem));
 }
 
+TEST_P(olLaunchKernelFooTest, SuccessThreaded) {
+  threadify([&](size_t) {
+    void *Mem;
+    ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                              LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));
+    struct {
+      void *Mem;
+    } Args{Mem};
+
+    ASSERT_SUCCESS(olLaunchKernel(Queue, Device, Kernel, &Args, sizeof(Args),
+                                  &LaunchArgs));
+
+    ASSERT_SUCCESS(olSyncQueue(Queue));
+
+    uint32_t *Data = (uint32_t *)Mem;
+    for (uint32_t i = 0; i < 64; i++) {
+      ASSERT_EQ(Data[i], i);
+    }
+
+    ASSERT_SUCCESS(olMemFree(Mem));
+  });
+}
+
 TEST_P(olLaunchKernelNoArgsTest, Success) {
   ASSERT_SUCCESS(
       olLaunchKernel(Queue, Device, Kernel, nullptr, 0, &LaunchArgs));
diff --git a/offload/unittests/OffloadAPI/memory/olMemFill.cpp b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
new file mode 100644
index 0000000000000..1b0bafa202080
--- /dev/null
+++ b/offload/unittests/OffloadAPI/memory/olMemFill.cpp
@@ -0,0 +1,134 @@
+//===------- Offload API tests - olMemFill --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+using olMemFillTest = OffloadQueueTest;
+OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olMemFillTest);
+
+TEST_P(olMemFillTest, Success8) {
+  constexpr size_t Size = 1024;
+  void *Alloc;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+  uint8_t Pattern = 0x42;
+  ASSERT_SUCCESS(olMemFill(Queue, Alloc, sizeof(Pattern), &Pattern, Size));
+
+  olSyncQueue(Queue);
+
+  size_t N = Size / sizeof(Pattern);
+  for (size_t i = 0; i < N; i++) {
+    uint8_t *AllocPtr = reinterpret_cast<uint8_t *>(Alloc);
+    ASSERT_EQ(AllocPtr[i], Pattern);
+  }
+
+  olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, Success16) {
+  constexpr size_t Size = 1024;
+  void *Alloc;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+  uint16_t Pattern = 0x4242;
+  ASSERT_SUCCESS(olMemFill(Queue, Alloc, sizeof(Pattern), &Pattern, Size));
+
+  olSyncQueue(Queue);
+
+  size_t N = Size / sizeof(Pattern);
+  for (size_t i = 0; i < N; i++) {
+    uint16_t *AllocPtr = reinterpret_cast<uint16_t *>(Alloc);
+    ASSERT_EQ(AllocPtr[i], Pattern);
+  }
+
+  olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, Success32) {
+  constexpr size_t Size = 1024;
+  void *Alloc;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+  uint32_t Pattern = 0xDEADBEEF;
+  ASSERT_SUCCESS(olMemFill(Queue, Alloc, sizeof(Pattern), &Pattern, Size));
+
+  olSyncQueue(Queue);
+
+  size_t N = Size / sizeof(Pattern);
+  for (size_t i = 0; i < N; i++) {
+    uint32_t *AllocPtr = reinterpret_cast<uint32_t *>(Alloc);
+    ASSERT_EQ(AllocPtr[i], Pattern);
+  }
+
+  olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, SuccessLarge) {
+  constexpr size_t Size = 1024;
+  void *Alloc;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+  struct PatternT {
+    uint64_t A;
+    uint64_t B;
+  } Pattern{UINT64_MAX, UINT64_MAX};
+
+  ASSERT_SUCCESS(olMemFill(Queue, Alloc, sizeof(Pattern), &Pattern, Size));
+
+  olSyncQueue(Queue);
+
+  size_t N = Size / sizeof(Pattern);
+  for (size_t i = 0; i < N; i++) {
+    PatternT *AllocPtr = reinterpret_cast<PatternT *>(Alloc);
+    ASSERT_EQ(AllocPtr[i].A, UINT64_MAX);
+    ASSERT_EQ(AllocPtr[i].B, UINT64_MAX);
+  }
+
+  olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, SuccessLargeByteAligned) {
+  constexpr size_t Size = 17 * 64;
+  void *Alloc;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+  struct __attribute__((packed)) PatternT {
+    uint64_t A;
+    uint64_t B;
+    uint8_t C;
+  } Pattern{UINT64_MAX, UINT64_MAX, 255};
+
+  ASSERT_SUCCESS(olMemFill(Queue, Alloc, sizeof(Pattern), &Pattern, Size));
+
+  olSyncQueue(Queue);
+
+  size_t N = Size / sizeof(Pattern);
+  for (size_t i = 0; i < N; i++) {
+    PatternT *AllocPtr = reinterpret_cast<PatternT *>(Alloc);
+    ASSERT_EQ(AllocPtr[i].A, UINT64_MAX);
+    ASSERT_EQ(AllocPtr[i].B, UINT64_MAX);
+    ASSERT_EQ(AllocPtr[i].C, 255);
+  }
+
+  olMemFree(Alloc);
+}
+
+TEST_P(olMemFillTest, InvalidPatternSize) {
+  constexpr size_t Size = 1025;
+  void *Alloc;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED, Size, &Alloc));
+
+  uint16_t Pattern = 0x4242;
+  ASSERT_ERROR(OL_ERRC_INVALID_SIZE,
+               olMemFill(Queue, Alloc, sizeof(Pattern), &Pattern, Size));
+
+  olSyncQueue(Queue);
+  olMemFree(Alloc);
+}
diff --git a/offload/unittests/OffloadAPI/queue/olGetQueueInfo.cpp b/offload/unittests/OffloadAPI/queue/olGetQueueInfo.cpp
index f4fb752d7ab40..2dccd33005563 100644
--- a/offload/unittests/OffloadAPI/queue/olGetQueueInfo.cpp
+++ b/offload/unittests/OffloadAPI/queue/olGetQueueInfo.cpp
@@ -20,6 +20,12 @@ TEST_P(olGetQueueInfoTest, SuccessDevice) {
   ASSERT_EQ(Device, RetrievedDevice);
 }
 
+TEST_P(olGetQueueInfoTest, SuccessEmpty) {
+  bool Empty;
+  ASSERT_SUCCESS(
+      olGetQueueInfo(Queue, OL_QUEUE_INFO_EMPTY, sizeof(Empty), &Empty));
+}
+
 TEST_P(olGetQueueInfoTest, InvalidNullHandle) {
   ol_device_handle_t RetrievedDevice;
   ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
diff --git a/offload/unittests/OffloadAPI/queue/olGetQueueInfoSize.cpp b/offload/unittests/OffloadAPI/queue/olGetQueueInfoSize.cpp
index f1ad9fa890d71..735dad6a29384 100644
--- a/offload/unittests/OffloadAPI/queue/olGetQueueInfoSize.cpp
+++ b/offload/unittests/OffloadAPI/queue/olGetQueueInfoSize.cpp
@@ -19,6 +19,12 @@ TEST_P(olGetQueueInfoSizeTest, SuccessDevice) {
   ASSERT_EQ(Size, sizeof(ol_device_handle_t));
 }
 
+TEST_P(olGetQueueInfoSizeTest, SuccessEmpty) {
+  size_t Size = 0;
+  ASSERT_SUCCESS(olGetQueueInfoSize(Queue, OL_QUEUE_INFO_EMPTY, &Size));
+  ASSERT_EQ(Size, sizeof(bool));
+}
+
 TEST_P(olGetQueueInfoSizeTest, InvalidNullHandle) {
   size_t Size = 0;
   ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
diff --git a/offload/unittests/OffloadAPI/queue/olLaunchHostFunction.cpp b/offload/unittests/OffloadAPI/queue/olLaunchHostFunction.cpp
new file mode 100644
index 0000000000000..aa86750f6adf9
--- /dev/null
+++ b/offload/unittests/OffloadAPI/queue/olLaunchHostFunction.cpp
@@ -0,0 +1,107 @@
+//===------- Offload API tests - olLaunchHostFunction ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+#include <thread>
+
+struct olLaunchHostFunctionTest : OffloadQueueTest {};
+OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchHostFunctionTest);
+
+struct olLaunchHostFunctionKernelTest : OffloadKernelTest {};
+OFFLOAD_TESTS_INSTANTIATE_DEVICE_FIXTURE(olLaunchHostFunctionKernelTest);
+
+TEST_P(olLaunchHostFunctionTest, Success) {
+  ASSERT_SUCCESS(olLaunchHostFunction(Queue, [](void *) {}, nullptr));
+}
+
+TEST_P(olLaunchHostFunctionTest, SuccessSequence) {
+  uint32_t Buff[16] = {1, 1};
+
+  for (auto BuffPtr = &Buff[2]; BuffPtr != &Buff[16]; BuffPtr++) {
+    ASSERT_SUCCESS(olLaunchHostFunction(
+        Queue,
+        [](void *BuffPtr) {
+          uint32_t *AsU32 = reinterpret_cast<uint32_t *>(BuffPtr);
+          AsU32[0] = AsU32[-1] + AsU32[-2];
+        },
+        BuffPtr));
+  }
+
+  ASSERT_SUCCESS(olSyncQueue(Queue));
+
+  for (uint32_t i = 2; i < 16; i++) {
+    ASSERT_EQ(Buff[i], Buff[i - 1] + Buff[i - 2]);
+  }
+}
+
+TEST_P(olLaunchHostFunctionKernelTest, SuccessBlocking) {
+  // Verify that a host kernel can block execution - A host task is created that
+  // only resolves when Block is set to false.
+  ol_kernel_launch_size_args_t LaunchArgs;
+  LaunchArgs.Dimensions = 1;
+  LaunchArgs.GroupSize = {64, 1, 1};
+  LaunchArgs.NumGroups = {1, 1, 1};
+  LaunchArgs.DynSharedMemory = 0;
+
+  ol_queue_handle_t Queue;
+  ASSERT_SUCCESS(olCreateQueue(Device, &Queue));
+
+  void *Mem;
+  ASSERT_SUCCESS(olMemAlloc(Device, OL_ALLOC_TYPE_MANAGED,
+                            LaunchArgs.GroupSize.x * sizeof(uint32_t), &Mem));
+
+  uint32_t *Data = (uint32_t *)Mem;
+  for (uint32_t i = 0; i < 64; i++) {
+    Data[i] = 0;
+  }
+
+  volatile bool Block = true;
+  ASSERT_SUCCESS(olLaunchHostFunction(
+      Queue,
+      [](void *Ptr) {
+        volatile bool *Block =
+            reinterpret_cast<volatile bool *>(reinterpret_cast<bool *>(Ptr));
+
+        while (*Block)
+          std::this_thread::yield();
+      },
+      const_cast<bool *>(&Block)));
+
+  struct {
+    void *Mem;
+  } Args{Mem};
+  ASSERT_SUCCESS(
+      olLaunchKernel(Queue, Device, Kernel, &Args, sizeof(Args), &LaunchArgs));
+
+  std::this_thread::sleep_for(std::chrono::milliseconds(500));
+  for (uint32_t i = 0; i < 64; i++) {
+    ASSERT_EQ(Data[i], 0);
+  }
+
+  Block = false;
+  ASSERT_SUCCESS(olSyncQueue(Queue));
+
+  for (uint32_t i = 0; i < 64; i++) {
+    ASSERT_EQ(Data[i], i);
+  }
+
+  ASSERT_SUCCESS(olDestroyQueue(Queue));
+  ASSERT_SUCCESS(olMemFree(Mem));
+}
+
+TEST_P(olLaunchHostFunctionTest, InvalidNullCallback) {
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_POINTER,
+               olLaunchHostFunction(Queue, nullptr, nullptr));
+}
+
+TEST_P(olLaunchHostFunctionTest, InvalidNullQueue) {
+  ASSERT_ERROR(OL_ERRC_INVALID_NULL_HANDLE,
+               olLaunchHostFunction(nullptr, [](void *) {}, nullptr));
+}
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index edd79df7fb767..f3de4bc4ee87b 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -117,7 +117,7 @@ set(ENABLE_LIBOMPTARGET ON)
 # there is no point in trying to compile libomptarget on other OSes.
 # 32-bit systems are not supported either.
 if (APPLE OR WIN32 OR WASM OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
-    OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+    OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(ENABLE_LIBOMPTARGET OFF)
 endif()
 
diff --git a/openmp/README.rst b/openmp/README.rst
index 2dfc8630858b8..c34d3e8a40d7d 100644
--- a/openmp/README.rst
+++ b/openmp/README.rst
@@ -369,6 +369,7 @@ There are following check-* make targets for tests.
 
 - ``check-ompt`` (ompt tests under runtime/test/ompt)
 - ``check-ompt-multiplex`` (ompt multiplex tests under tools/multiplex/tests)
+- ``check-ompt-omptest`` (ompt omptest tests under tools/omptest/tests)
 - ``check-libarcher`` (libarcher tests under tools/archer/tests)
 - ``check-libomp`` (libomp tests under runtime/test. This includes check-ompt tests too)
 - ``check-libomptarget-*`` (libomptarget tests for specific target under libomptarget/test)
diff --git a/openmp/cmake/OpenMPTesting.cmake b/openmp/cmake/OpenMPTesting.cmake
index 14cc5c67d84c2..85240aede728d 100644
--- a/openmp/cmake/OpenMPTesting.cmake
+++ b/openmp/cmake/OpenMPTesting.cmake
@@ -57,7 +57,7 @@ if (${OPENMP_STANDALONE_BUILD})
   if (MSVC OR XCODE)
     set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --no-progress-bar")
   endif()
-  if (${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+  if ("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
     set(DEFAULT_LIT_ARGS "${DEFAULT_LIT_ARGS} --time-tests --timeout=3000")
   endif()
   set(OPENMP_LIT_ARGS "${DEFAULT_LIT_ARGS}" CACHE STRING "Options for lit.")
diff --git a/openmp/libompd/gdb-plugin/ompd/__init__.py b/openmp/libompd/gdb-plugin/ompd/__init__.py
index b9f572dce8dce..ab30e65392ea2 100644
--- a/openmp/libompd/gdb-plugin/ompd/__init__.py
+++ b/openmp/libompd/gdb-plugin/ompd/__init__.py
@@ -4,7 +4,7 @@
 
 if __name__ == "__main__":
     try:
-        sys.path.append(os.path.dirname(__file__))
+        sys.path.insert(0, os.path.dirname(__file__))
 
         import ompd
 
diff --git a/openmp/libompd/gdb-plugin/ompdModule.c b/openmp/libompd/gdb-plugin/ompdModule.c
index df602c54aba37..c6020de30671b 100644
--- a/openmp/libompd/gdb-plugin/ompdModule.c
+++ b/openmp/libompd/gdb-plugin/ompdModule.c
@@ -943,7 +943,8 @@ static PyObject *call_ompd_get_enclosing_parallel_handle(PyObject *self,
 
   if (retVal != ompd_rc_ok) {
     _printf("An error occurred when calling "
-            "ompd_get_enclosing_parallel_handle! Error code: %d",
+            "ompd_get_enclosing_parallel_handle!"
+            "Error code: %d",
             retVal);
     return Py_BuildValue("l", retVal);
   }
@@ -967,7 +968,7 @@ static PyObject *call_ompd_get_task_parallel_handle(PyObject *self,
 
   if (retVal != ompd_rc_ok) {
     _printf("An error occurred when calling ompd_get_task_parallel_handle! "
-            "Error code: %d");
+            "Error code: %d", retVal);
     return Py_BuildValue("l", retVal);
   }
   return PyCapsule_New(taskParallelHandle, "ParallelHandle",
@@ -1180,9 +1181,10 @@ static PyObject *call_ompd_get_icv_from_scope(PyObject *self, PyObject *args) {
 
   if (retVal != ompd_rc_ok) {
     if (retVal != ompd_rc_incomplete) {
-      _printf("An error occurred when calling ompd_get_icv_from_scope(%i, %i): "
-              "Error code: %d",
-              scope, icvId, retVal);
+      _printf(
+          "An error occurred when calling ompd_get_icv_from_scope(%i, %" PRIu64
+          "): Error code: %d",
+          scope, icvId, retVal);
     }
     return Py_None;
   }
diff --git a/openmp/runtime/CMakeLists.txt b/openmp/runtime/CMakeLists.txt
index b6c4759ec03f7..93eb14f10a50a 100644
--- a/openmp/runtime/CMakeLists.txt
+++ b/openmp/runtime/CMakeLists.txt
@@ -136,7 +136,7 @@ set(LIBOMP_ASMFLAGS "" CACHE STRING
   "Appended user specified assembler flags.")
 set(LIBOMP_LDFLAGS "" CACHE STRING
   "Appended user specified linker flags.")
-if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(LIBOMP_LIBFLAGS "-lperfstat" CACHE STRING
     "Appended user specified linked libs flags. (e.g., -lm)")
   if("${LIBOMP_ARCH}" STREQUAL "ppc")
diff --git a/openmp/runtime/cmake/LibompHandleFlags.cmake b/openmp/runtime/cmake/LibompHandleFlags.cmake
index 61fc31d0fc7fd..c36a88fb862ae 100644
--- a/openmp/runtime/cmake/LibompHandleFlags.cmake
+++ b/openmp/runtime/cmake/LibompHandleFlags.cmake
@@ -140,14 +140,14 @@ function(libomp_get_libflags libflags)
   if(LIBOMP_HAVE_SHM_OPEN_WITH_LRT)
     libomp_append(libflags_local -lrt)
   endif()
-  if(${CMAKE_SYSTEM_NAME} MATCHES "DragonFly|FreeBSD|OpenBSD")
+  if("${CMAKE_SYSTEM_NAME}" MATCHES "DragonFly|FreeBSD|OpenBSD")
     libomp_append(libflags_local "-Wl,--no-as-needed" LIBOMP_HAVE_AS_NEEDED_FLAG)
     libomp_append(libflags_local "-lm")
     libomp_append(libflags_local "-Wl,--as-needed" LIBOMP_HAVE_AS_NEEDED_FLAG)
-    if (${CMAKE_SYSTEM_NAME} STREQUAL "DragonFly")
+    if ("${CMAKE_SYSTEM_NAME}" STREQUAL "DragonFly")
       libomp_append(libflags_local "-lkvm")
     endif()
-  elseif(${CMAKE_SYSTEM_NAME} MATCHES "Linux|NetBSD|SunOS")
+  elseif("${CMAKE_SYSTEM_NAME}" MATCHES "Linux|NetBSD|SunOS")
     libomp_append(libflags_local -lm)
   endif()
   set(libflags_local ${libflags_local} ${LIBOMP_LIBFLAGS})
diff --git a/openmp/runtime/cmake/config-ix.cmake b/openmp/runtime/cmake/config-ix.cmake
index 144c657be67e8..30b69d46a8bd1 100644
--- a/openmp/runtime/cmake/config-ix.cmake
+++ b/openmp/runtime/cmake/config-ix.cmake
@@ -321,7 +321,7 @@ else()
       (LIBOMP_ARCH STREQUAL sparcv9))
      AND # OS supported?
      ((WIN32 AND LIBOMP_HAVE_PSAPI) OR APPLE OR
-      (NOT (WIN32 OR ${CMAKE_SYSTEM_NAME} MATCHES "AIX") AND LIBOMP_HAVE_WEAK_ATTRIBUTE)))
+      (NOT (WIN32 OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX") AND LIBOMP_HAVE_WEAK_ATTRIBUTE)))
     set(LIBOMP_HAVE_OMPT_SUPPORT TRUE)
   else()
     set(LIBOMP_HAVE_OMPT_SUPPORT FALSE)
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 569061c6494b8..08e1753b93636 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -121,7 +121,7 @@ else()
     # Unix specific files
     libomp_append(LIBOMP_CXXFILES z_Linux_util.cpp)
     libomp_append(LIBOMP_CXXFILES kmp_gsupport.cpp)
-    if(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+    if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
       libomp_append(LIBOMP_GNUASMFILES z_AIX_asm.S) # AIX assembly file
     else()
       libomp_append(LIBOMP_GNUASMFILES z_Linux_asm.S) # Unix assembly file
@@ -218,7 +218,7 @@ if(OPENMP_MSVC_NAME_SCHEME)
     LINK_FLAGS "${LIBOMP_CONFIGURED_LDFLAGS}"
     LINKER_LANGUAGE ${LIBOMP_LINKER_LANGUAGE}
  )
-elseif(${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+elseif("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(LIBOMP_SHARED_OUTPUT_NAME "omp" CACHE STRING "Output name for the shared libomp runtime library.")
   set_target_properties(omp PROPERTIES
     OUTPUT_NAME "${LIBOMP_SHARED_OUTPUT_NAME}"
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 818edf9060ad1..83afc0e83f231 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -4621,6 +4621,13 @@ static inline int __kmp_adjust_gtid_for_hidden_helpers(int gtid) {
   return adjusted_gtid;
 }
 
+#if ENABLE_LIBOMPTARGET
+// Pointers to callbacks registered by the offload library to be notified of
+// task progress.
+extern void (*kmp_target_sync_cb)(ident_t *loc_ref, int gtid,
+                                  void *current_task, void *event);
+#endif // ENABLE_LIBOMPTARGET
+
 // Support for error directive
 typedef enum kmp_severity_t {
   severity_warning = 1,
diff --git a/openmp/runtime/src/kmp_barrier.cpp b/openmp/runtime/src/kmp_barrier.cpp
index 88a5cbb69ba87..4d7989d1ce5eb 100644
--- a/openmp/runtime/src/kmp_barrier.cpp
+++ b/openmp/runtime/src/kmp_barrier.cpp
@@ -1853,6 +1853,14 @@ static int __kmp_barrier_template(enum barrier_type bt, int gtid, int is_split,
   }
 #endif
 
+#if ENABLE_LIBOMPTARGET
+  // Give an opportunity to the offload runtime to make progress and create
+  // proxy tasks if necessary
+  if (UNLIKELY(kmp_target_sync_cb != NULL))
+    (*kmp_target_sync_cb)(
+        NULL, gtid, KMP_TASKDATA_TO_TASK(this_thr->th.th_current_task), NULL);
+#endif
+
   if (!team->t.t_serialized) {
 #if USE_ITT_BUILD
     // This value will be used in itt notify events below.
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 39b7834d358af..48e29c9f9fe45 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -93,6 +93,9 @@ static void __kmp_partition_places(kmp_team_t *team,
                                    int update_master_only = 0);
 #endif
 static void __kmp_do_serial_initialize(void);
+#if ENABLE_LIBOMPTARGET
+static void __kmp_target_init(void);
+#endif // ENABLE_LIBOMPTARGET
 void __kmp_fork_barrier(int gtid, int tid);
 void __kmp_join_barrier(int gtid);
 void __kmp_setup_icv_copy(kmp_team_t *team, int new_nproc,
@@ -7129,6 +7132,9 @@ static void __kmp_do_serial_initialize(void) {
 #if KMP_MIC_SUPPORTED
   __kmp_check_mic_type();
 #endif
+#if ENABLE_LIBOMPTARGET
+  __kmp_target_init();
+#endif /* ENABLE_LIBOMPTARGET */
 
 // Some global variable initialization moved here from kmp_env_initialize()
 #ifdef KMP_DEBUG
@@ -9355,6 +9361,15 @@ void __kmp_set_nesting_mode_threads() {
     set__max_active_levels(thread, __kmp_nesting_mode_nlevels);
 }
 
+#if ENABLE_LIBOMPTARGET
+void (*kmp_target_sync_cb)(ident_t *loc_ref, int gtid, void *current_task,
+                           void *event) = NULL;
+void __kmp_target_init() {
+  // Look for hooks in the libomptarget library
+  *(void **)(&kmp_target_sync_cb) = KMP_DLSYM("__tgt_target_sync");
+}
+#endif // ENABLE_LIBOMPTARGET
+
 // Empty symbols to export (see exports_so.txt) when feature is disabled
 extern "C" {
 #if !KMP_STATS_ENABLED
diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp
index 31342c8c6203d..b9d615f43b570 100644
--- a/openmp/runtime/src/kmp_settings.cpp
+++ b/openmp/runtime/src/kmp_settings.cpp
@@ -3278,9 +3278,9 @@ static void __kmp_stg_print_places(kmp_str_buf_t *buffer, char const *name,
         int eff = __kmp_affinity.core_attr_gran.core_eff;
         if (ct != KMP_HW_CORE_TYPE_UNKNOWN) {
           const char *ct_name = __kmp_hw_get_core_type_keyword(ct);
-          __kmp_str_buf_print(buffer, ":%s", name, ct_name);
+          __kmp_str_buf_print(buffer, ":%s", ct_name);
         } else if (eff >= 0 && eff < KMP_HW_MAX_NUM_CORE_EFFS) {
-          __kmp_str_buf_print(buffer, ":eff%d", name, eff);
+          __kmp_str_buf_print(buffer, ":eff%d", eff);
         }
       }
 
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index e4d92a78fd6b9..37836fb457537 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -1149,6 +1149,13 @@ void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,
 // thread:  thread data structure corresponding to implicit task
 void __kmp_finish_implicit_task(kmp_info_t *thread) {
   kmp_taskdata_t *task = thread->th.th_current_task;
+#if ENABLE_LIBOMPTARGET
+  // Give an opportunity to the offload runtime to synchronize any unfinished
+  // target async regions before finishing the implicit task
+  if (UNLIKELY(kmp_target_sync_cb != NULL))
+    (*kmp_target_sync_cb)(NULL, thread->th.th_info.ds.ds_gtid,
+                          KMP_TASKDATA_TO_TASK(task), NULL);
+#endif // ENABLE_LIBOMPTARGET
   if (task->td_dephash) {
     int children;
     task->td_flags.complete = 1;
@@ -2020,6 +2027,14 @@ static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
     }
 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
 
+#if ENABLE_LIBOMPTARGET
+    // Give an opportunity to the offload runtime to make progress and create
+    // any necessary proxy tasks
+    if (UNLIKELY(kmp_target_sync_cb))
+      (*kmp_target_sync_cb)(loc_ref, gtid, KMP_TASKDATA_TO_TASK(taskdata),
+                            NULL);
+#endif // ENABLE_LIBOMPTARGET
+
 // Debugger: The taskwait is active. Store location and thread encountered the
 // taskwait.
 #if USE_ITT_BUILD
@@ -2719,6 +2734,13 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
     }
 #endif
 
+#if ENABLE_LIBOMPTARGET
+    // Give an opportunity to the offload runtime to make progress and create
+    // any necessary proxy tasks
+    if (UNLIKELY(kmp_target_sync_cb))
+      (*kmp_target_sync_cb)(loc, gtid, KMP_TASKDATA_TO_TASK(taskdata), NULL);
+#endif // ENABLE_LIBOMPTARGET
+
     if (!taskdata->td_flags.team_serial ||
         (thread->th.th_task_team != NULL &&
          (thread->th.th_task_team->tt.tt_found_proxy_tasks ||
@@ -3162,6 +3184,13 @@ static inline int __kmp_execute_tasks_template(
   while (1) { // Outer loop keeps trying to find tasks in case of single thread
     // getting tasks from target constructs
     while (1) { // Inner loop to find a task and execute it
+#if ENABLE_LIBOMPTARGET
+      // Give an opportunity to the offload runtime to make progress
+      if (UNLIKELY(kmp_target_sync_cb))
+        (*kmp_target_sync_cb)(NULL, gtid, KMP_TASKDATA_TO_TASK(current_task),
+                              NULL);
+#endif // ENABLE_LIBOMPTARGET
+
       task = NULL;
       if (task_team->tt.tt_num_task_pri) { // get priority task first
         task = __kmp_get_priority_task(gtid, task_team, is_constrained);
diff --git a/openmp/runtime/test/ompt/callback.h b/openmp/runtime/test/ompt/callback.h
index 0837cc5dfefcd..cd8acb57ee2f7 100644
--- a/openmp/runtime/test/ompt/callback.h
+++ b/openmp/runtime/test/ompt/callback.h
@@ -39,15 +39,11 @@ static const char *ompt_task_status_t_values[] = {
     "ompt_task_switch", // 7
     "ompt_taskwait_complete" // 8
 };
-static const char* ompt_cancel_flag_t_values[] = {
-  "ompt_cancel_parallel",
-  "ompt_cancel_sections",
-  "ompt_cancel_loop",
-  "ompt_cancel_taskgroup",
-  "ompt_cancel_activated",
-  "ompt_cancel_detected",
-  "ompt_cancel_discarded_task"
-};
+static const char *ompt_cancel_flag_t_values[] = {
+    "ompt_cancel_parallel",      "ompt_cancel_sections",
+    "ompt_cancel_loop",          "ompt_cancel_taskgroup",
+    "ompt_cancel_activated",     "ompt_cancel_detected",
+    "ompt_cancel_discarded_task"};
 
 static const char *ompt_work_t_values[] = {"undefined",
                                            "ompt_work_loop",
@@ -88,9 +84,32 @@ static const char *ompt_dependence_type_t_values[36] = {
     "ompt_dependence_type_source", // 5
     "ompt_dependence_type_sink", // 6
     "ompt_dependence_type_inoutset", // 7
-    "", "", "", "", "", "", // 8-13
-    "", "", "", "", "", "", "", "", "", "", // 14-23
-    "", "", "", "", "", "", "", "", "", "", // 24-33
+    "",
+    "",
+    "",
+    "",
+    "",
+    "", // 8-13
+    "",
+    "",
+    "",
+    "",
+    "",
+    "",
+    "",
+    "",
+    "",
+    "", // 14-23
+    "",
+    "",
+    "",
+    "",
+    "",
+    "",
+    "",
+    "",
+    "",
+    "", // 24-33
     "ompt_dependence_type_out_all_memory", // 34
     "ompt_dependence_type_inout_all_memory" // 35
 };
@@ -165,8 +184,7 @@ void assert_frame_flags(int enterf, int exitf) {
   assert(exitf == (ompt_frame_application | ompt_frame_cfa) ||
          exitf == (ompt_frame_runtime | ompt_frame_cfa));
 }
-static void print_ids(int level)
-{
+static void print_ids(int level) {
   int task_type, thread_num;
   ompt_frame_t *frame;
   ompt_data_t *task_parallel_data;
@@ -176,8 +194,8 @@ static void print_ids(int level)
   char buffer[2048];
   format_task_type(task_type, buffer);
   if (frame) {
-    printf("%" PRIu64 ": task level %d: parallel_id=%" PRIu64
-           ", task_id=%" PRIu64 ", exit_frame=%p, reenter_frame=%p, "
+    printf("%" PRIu64 ": task level %d: parallel_id=%" PRIx64
+           ", task_id=%" PRIx64 ", exit_frame=%p, reenter_frame=%p, "
            "task_type=%s=%d, thread_num=%d\n",
            ompt_get_thread_data()->value, level,
            exists_task ? task_parallel_data->value : 0,
@@ -193,18 +211,21 @@ static void print_ids(int level)
   printf("%" PRIu64 ": __builtin_frame_address(%d)=%p\n",                      \
          ompt_get_thread_data()->value, level, get_frame_address(level))
 
-// clang (version 5.0 and above) adds an intermediate function call with debug flag (-g)
+// clang (version 5.0 and above) adds an intermediate function call with debug
+// flag (-g)
 #if defined(TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN)
-  #if defined(DEBUG) && defined(__clang__) && __clang_major__ >= 5
-    #define print_frame_from_outlined_fn(level) print_frame(level+1)
-  #else
-    #define print_frame_from_outlined_fn(level) print_frame(level)
-  #endif
+#if defined(DEBUG) && defined(__clang__) && __clang_major__ >= 5
+#define print_frame_from_outlined_fn(level) print_frame(level + 1)
+#else
+#define print_frame_from_outlined_fn(level) print_frame(level)
+#endif
 
-  #if defined(__clang__) && __clang_major__ >= 5
-    #warning "Clang 5.0 and later add an additional wrapper for outlined functions when compiling with debug information."
-    #warning "Please define -DDEBUG iff you manually pass in -g to make the tests succeed!"
-  #endif
+#if defined(__clang__) && __clang_major__ >= 5
+#warning                                                                       \
+    "Clang 5.0 and later add an additional wrapper for outlined functions when compiling with debug information."
+#warning                                                                       \
+    "Please define -DDEBUG iff you manually pass in -g to make the tests succeed!"
+#endif
 #endif
 
 // This macro helps to define a label at the current position that can be used
@@ -218,36 +239,39 @@ static void print_ids(int level)
 //
 // (The empty block between "#pragma omp ..." and the __asm__ statement is a
 // workaround for a bug in the Intel Compiler.)
-#define define_ompt_label(id) \
-  {} \
-  __asm__("nop"); \
-ompt_label_##id:
+#define define_ompt_label(id)                                                  \
+  {                                                                            \
+  }                                                                            \
+  __asm__("nop");                                                              \
+  ompt_label_##id:
 
 // This macro helps to get the address of a label that is inserted by the above
 // macro define_ompt_label(). The address is obtained with a GNU extension
 // (&&label) that has been tested with gcc, clang and icc.
-#define get_ompt_label_address(id) (&& ompt_label_##id)
+#define get_ompt_label_address(id) (&&ompt_label_##id)
 
 // This macro prints the exact address that a previously called runtime function
 // returns to.
-#define print_current_address(id) \
-  define_ompt_label(id) \
-  print_possible_return_addresses(get_ompt_label_address(id))
+#define print_current_address(id)                                              \
+  define_ompt_label(id)                                                        \
+      print_possible_return_addresses(get_ompt_label_address(id))
 
 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
 // On X86 the NOP instruction is 1 byte long. In addition, the compiler inserts
 // a MOV instruction for non-void runtime functions which is 3 bytes long.
-#define print_possible_return_addresses(addr) \
-  printf("%" PRIu64 ": current_address=%p or %p for non-void functions\n", \
-         ompt_get_thread_data()->value, ((char *)addr) - 1, ((char *)addr) - 4)
+#define print_possible_return_addresses(addr)                                  \
+  printf("%" PRIu64 ": current_address=%p or %p for non-void functions\n",     \
+         ompt_get_thread_data()->value, ((char *)addr) - 1,                    \
+         ((char *)addr) - 4)
 #elif KMP_ARCH_PPC64
 // On Power the NOP instruction is 4 bytes long. In addition, the compiler
 // inserts a second NOP instruction (another 4 bytes). For non-void runtime
 // functions Clang inserts a STW instruction (but only if compiling under
 // -fno-PIC which will be the default with Clang 8.0, another 4 bytes).
-#define print_possible_return_addresses(addr) \
-  printf("%" PRIu64 ": current_address=%p or %p\n", ompt_get_thread_data()->value, \
-         ((char *)addr) - 8, ((char *)addr) - 12)
+#define print_possible_return_addresses(addr)                                  \
+  printf("%" PRIu64 ": current_address=%p or %p\n",                            \
+         ompt_get_thread_data()->value, ((char *)addr) - 8,                    \
+         ((char *)addr) - 12)
 #elif KMP_ARCH_AARCH64 || KMP_ARCH_AARCH64_32
 // On AArch64 the NOP instruction is 4 bytes long, can be followed by inserted
 // store instruction (another 4 bytes long).
@@ -266,18 +290,20 @@ ompt_label_##id:
 // accounts for another 4 bytes. Finally, an additional J instruction may
 // appear (adding 4 more bytes) when the C.NOP is referenced elsewhere (ie.
 // another branch).
-#define print_possible_return_addresses(addr) \
-  printf("%" PRIu64 ": current_address=%p or %p\n", \
-         ompt_get_thread_data()->value, ((char *)addr) - 6, ((char *)addr) - 10)
+#define print_possible_return_addresses(addr)                                  \
+  printf("%" PRIu64 ": current_address=%p or %p\n",                            \
+         ompt_get_thread_data()->value, ((char *)addr) - 6,                    \
+         ((char *)addr) - 10)
 #else
 // On RV64G the NOP instruction is 4 byte long. In addition, the compiler
 // inserts a J instruction (targeting the successor basic block), which
 // accounts for another 4 bytes. Finally, an additional J instruction may
 // appear (adding 4 more bytes) when the NOP is referenced elsewhere (ie.
 // another branch).
-#define print_possible_return_addresses(addr) \
-  printf("%" PRIu64 ": current_address=%p or %p\n", \
-         ompt_get_thread_data()->value, ((char *)addr) - 8, ((char *)addr) - 12)
+#define print_possible_return_addresses(addr)                                  \
+  printf("%" PRIu64 ": current_address=%p or %p\n",                            \
+         ompt_get_thread_data()->value, ((char *)addr) - 8,                    \
+         ((char *)addr) - 12)
 #endif
 #elif KMP_ARCH_LOONGARCH64
 // On LoongArch64 the NOP instruction is 4 bytes long, can be followed by
@@ -323,7 +349,6 @@ ompt_label_##id:
 #error Unsupported target architecture, cannot determine address offset!
 #endif
 
-
 // This macro performs a somewhat similar job to print_current_address(), except
 // that it discards a certain number of nibbles from the address and only prints
 // the most significant bits / nibbles. This can be used for cases where the
@@ -332,9 +357,8 @@ ompt_label_##id:
 // To account for overflows (ie the most significant bits / nibbles have just
 // changed as we are a few bytes above the relevant power of two) the addresses
 // of the "current" and of the "previous block" are printed.
-#define print_fuzzy_address(id) \
-  define_ompt_label(id) \
-  print_fuzzy_address_blocks(get_ompt_label_address(id))
+#define print_fuzzy_address(id)                                                \
+  define_ompt_label(id) print_fuzzy_address_blocks(get_ompt_label_address(id))
 
 // If you change this define you need to adapt all capture patterns in the tests
 // to include or discard the new number of nibbles!
@@ -359,193 +383,169 @@ ompt_label_##id:
 #define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
 
 #ifndef USE_PRIVATE_TOOL
-static void
-on_ompt_callback_mutex_acquire(
-  ompt_mutex_t kind,
-  unsigned int hint,
-  unsigned int impl,
-  ompt_wait_id_t wait_id,
-  const void *codeptr_ra)
-{
-  switch(kind)
-  {
-    case ompt_mutex_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_wait_lock: wait_id=%" PRIu64 ", hint=%" PRIu32
-             ", impl=%" PRIu32 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
-      break;
-    case ompt_mutex_test_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_wait_test_lock: wait_id=%" PRIu64 ", hint=%" PRIu32
-             ", impl=%" PRIu32 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
-      break;
-    case ompt_mutex_nest_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_wait_nest_lock: wait_id=%" PRIu64 ", hint=%" PRIu32
-             ", impl=%" PRIu32 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
-      break;
-    case ompt_mutex_test_nest_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_wait_test_nest_lock: wait_id=%" PRIu64
-             ", hint=%" PRIu32 ", impl=%" PRIu32 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
-      break;
-    case ompt_mutex_critical:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_wait_critical: wait_id=%" PRIu64 ", hint=%" PRIu32
-             ", impl=%" PRIu32 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
-      break;
-    case ompt_mutex_atomic:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_wait_atomic: wait_id=%" PRIu64 ", hint=%" PRIu32
-             ", impl=%" PRIu32 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
-      break;
-    case ompt_mutex_ordered:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_wait_ordered: wait_id=%" PRIu64 ", hint=%" PRIu32
-             ", impl=%" PRIu32 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
-      break;
-    default:
-      break;
+static void on_ompt_callback_mutex_acquire(ompt_mutex_t kind, unsigned int hint,
+                                           unsigned int impl,
+                                           ompt_wait_id_t wait_id,
+                                           const void *codeptr_ra) {
+  switch (kind) {
+  case ompt_mutex_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_wait_lock: wait_id=%" PRIu64
+           ", hint=%" PRIu32 ", impl=%" PRIu32 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
+    break;
+  case ompt_mutex_test_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_wait_test_lock: wait_id=%" PRIu64 ", hint=%" PRIu32
+           ", impl=%" PRIu32 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
+    break;
+  case ompt_mutex_nest_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_wait_nest_lock: wait_id=%" PRIu64 ", hint=%" PRIu32
+           ", impl=%" PRIu32 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
+    break;
+  case ompt_mutex_test_nest_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_wait_test_nest_lock: wait_id=%" PRIu64 ", hint=%" PRIu32
+           ", impl=%" PRIu32 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
+    break;
+  case ompt_mutex_critical:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_wait_critical: wait_id=%" PRIu64 ", hint=%" PRIu32
+           ", impl=%" PRIu32 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
+    break;
+  case ompt_mutex_atomic:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_wait_atomic: wait_id=%" PRIu64 ", hint=%" PRIu32
+           ", impl=%" PRIu32 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
+    break;
+  case ompt_mutex_ordered:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_wait_ordered: wait_id=%" PRIu64 ", hint=%" PRIu32
+           ", impl=%" PRIu32 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
+    break;
+  default:
+    break;
   }
 }
 
-static void
-on_ompt_callback_mutex_acquired(
-  ompt_mutex_t kind,
-  ompt_wait_id_t wait_id,
-  const void *codeptr_ra)
-{
-  switch(kind)
-  {
-    case ompt_mutex_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_acquired_lock: wait_id=%" PRIu64 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_test_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_acquired_test_lock: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_nest_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_acquired_nest_lock_first: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_test_nest_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_acquired_test_nest_lock_first: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_critical:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_acquired_critical: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_atomic:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_acquired_atomic: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_ordered:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_acquired_ordered: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    default:
-      break;
+static void on_ompt_callback_mutex_acquired(ompt_mutex_t kind,
+                                            ompt_wait_id_t wait_id,
+                                            const void *codeptr_ra) {
+  switch (kind) {
+  case ompt_mutex_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_acquired_lock: wait_id=%" PRIu64 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_test_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_acquired_test_lock: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_nest_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_acquired_nest_lock_first: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_test_nest_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_acquired_test_nest_lock_first: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_critical:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_acquired_critical: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_atomic:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_acquired_atomic: wait_id=%" PRIu64 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_ordered:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_acquired_ordered: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  default:
+    break;
   }
 }
 
-static void
-on_ompt_callback_mutex_released(
-  ompt_mutex_t kind,
-  ompt_wait_id_t wait_id,
-  const void *codeptr_ra)
-{
-  switch(kind)
-  {
-    case ompt_mutex_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_release_lock: wait_id=%" PRIu64 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_nest_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_release_nest_lock_last: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_critical:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_release_critical: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_atomic:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_release_atomic: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_ordered:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_release_ordered: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    default:
-      break;
+static void on_ompt_callback_mutex_released(ompt_mutex_t kind,
+                                            ompt_wait_id_t wait_id,
+                                            const void *codeptr_ra) {
+  switch (kind) {
+  case ompt_mutex_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_release_lock: wait_id=%" PRIu64 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_nest_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_release_nest_lock_last: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_critical:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_release_critical: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_atomic:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_release_atomic: wait_id=%" PRIu64 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_ordered:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_release_ordered: wait_id=%" PRIu64 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  default:
+    break;
   }
 }
 
-static void
-on_ompt_callback_nest_lock(
-    ompt_scope_endpoint_t endpoint,
-    ompt_wait_id_t wait_id,
-    const void *codeptr_ra)
-{
-  switch(endpoint)
-  {
-    case ompt_scope_begin:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_acquired_nest_lock_next: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_scope_end:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_release_nest_lock_prev: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_scope_beginend:
-      printf("ompt_scope_beginend should never be passed to %s\n", __func__);
-      exit(-1);
+static void on_ompt_callback_nest_lock(ompt_scope_endpoint_t endpoint,
+                                       ompt_wait_id_t wait_id,
+                                       const void *codeptr_ra) {
+  switch (endpoint) {
+  case ompt_scope_begin:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_acquired_nest_lock_next: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_scope_end:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_release_nest_lock_prev: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_scope_beginend:
+    printf("ompt_scope_beginend should never be passed to %s\n", __func__);
+    exit(-1);
   }
 }
 
-static void
-on_ompt_callback_sync_region(
-  ompt_sync_region_t kind,
-  ompt_scope_endpoint_t endpoint,
-  ompt_data_t *parallel_data,
-  ompt_data_t *task_data,
-  const void *codeptr_ra)
-{
+static void on_ompt_callback_sync_region(ompt_sync_region_t kind,
+                                         ompt_scope_endpoint_t endpoint,
+                                         ompt_data_t *parallel_data,
+                                         ompt_data_t *task_data,
+                                         const void *codeptr_ra) {
   if (endpoint == ompt_scope_beginend) {
     printf("ompt_scope_beginend should never be passed to %s\n", __func__);
     exit(-1);
@@ -557,8 +557,8 @@ on_ompt_callback_sync_region(
   }
   uint64_t parallel_data_value = parallel_data ? parallel_data->value : 0;
   const char *begin_or_end = (endpoint == ompt_scope_begin) ? "begin" : "end";
-  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_%s_%s: parallel_id=%" PRIu64
-         ", task_id=%" PRIu64 ", codeptr_ra=%p\n",
+  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_%s_%s: parallel_id=%" PRIx64
+         ", task_id=%" PRIx64 ", codeptr_ra=%p\n",
          ompt_get_thread_data()->value, ompt_sync_region_t_values[kind],
          begin_or_end, parallel_data_value, task_data->value, codeptr_ra);
   switch (kind) {
@@ -575,14 +575,11 @@ on_ompt_callback_sync_region(
   }
 }
 
-static void
-on_ompt_callback_sync_region_wait(
-  ompt_sync_region_t kind,
-  ompt_scope_endpoint_t endpoint,
-  ompt_data_t *parallel_data,
-  ompt_data_t *task_data,
-  const void *codeptr_ra)
-{
+static void on_ompt_callback_sync_region_wait(ompt_sync_region_t kind,
+                                              ompt_scope_endpoint_t endpoint,
+                                              ompt_data_t *parallel_data,
+                                              ompt_data_t *task_data,
+                                              const void *codeptr_ra) {
   if (endpoint == ompt_scope_beginend) {
     printf("ompt_scope_beginend should never be passed to %s\n", __func__);
     exit(-1);
@@ -595,7 +592,7 @@ on_ompt_callback_sync_region_wait(
   uint64_t parallel_data_value = parallel_data ? parallel_data->value : 0;
   const char *begin_or_end = (endpoint == ompt_scope_begin) ? "begin" : "end";
   printf("%" PRIu64 ":" _TOOL_PREFIX
-         " ompt_event_wait_%s_%s: parallel_id=%" PRIu64 ", task_id=%" PRIu64
+         " ompt_event_wait_%s_%s: parallel_id=%" PRIx64 ", task_id=%" PRIx64
          ", codeptr_ra=%p\n",
          ompt_get_thread_data()->value, ompt_sync_region_t_values[kind],
          begin_or_end, parallel_data_value, task_data->value, codeptr_ra);
@@ -609,16 +606,16 @@ static void on_ompt_callback_reduction(ompt_sync_region_t kind,
   switch (endpoint) {
   case ompt_scope_begin:
     printf("%" PRIu64 ":" _TOOL_PREFIX
-           " ompt_event_reduction_begin: parallel_id=%" PRIu64
-           ", task_id=%" PRIu64 ", codeptr_ra=%p\n",
+           " ompt_event_reduction_begin: parallel_id=%" PRIx64
+           ", task_id=%" PRIx64 ", codeptr_ra=%p\n",
            ompt_get_thread_data()->value,
            (parallel_data) ? parallel_data->value : 0, task_data->value,
            codeptr_ra);
     break;
   case ompt_scope_end:
     printf("%" PRIu64 ":" _TOOL_PREFIX
-           " ompt_event_reduction_end: parallel_id=%" PRIu64
-           ", task_id=%" PRIu64 ", codeptr_ra=%p\n",
+           " ompt_event_reduction_end: parallel_id=%" PRIx64
+           ", task_id=%" PRIx64 ", codeptr_ra=%p\n",
            ompt_get_thread_data()->value,
            (parallel_data) ? parallel_data->value : 0, task_data->value,
            codeptr_ra);
@@ -629,184 +626,156 @@ static void on_ompt_callback_reduction(ompt_sync_region_t kind,
   }
 }
 
-static void
-on_ompt_callback_flush(
-    ompt_data_t *thread_data,
-    const void *codeptr_ra)
-{
+static void on_ompt_callback_flush(ompt_data_t *thread_data,
+                                   const void *codeptr_ra) {
   printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_flush: codeptr_ra=%p\n",
          thread_data->value, codeptr_ra);
 }
 
-static void
-on_ompt_callback_cancel(
-    ompt_data_t *task_data,
-    int flags,
-    const void *codeptr_ra)
-{
-  const char* first_flag_value;
-  const char* second_flag_value;
-  if(flags & ompt_cancel_parallel)
+static void on_ompt_callback_cancel(ompt_data_t *task_data, int flags,
+                                    const void *codeptr_ra) {
+  const char *first_flag_value;
+  const char *second_flag_value;
+  if (flags & ompt_cancel_parallel)
     first_flag_value = ompt_cancel_flag_t_values[0];
-  else if(flags & ompt_cancel_sections)
+  else if (flags & ompt_cancel_sections)
     first_flag_value = ompt_cancel_flag_t_values[1];
-  else if(flags & ompt_cancel_loop)
+  else if (flags & ompt_cancel_loop)
     first_flag_value = ompt_cancel_flag_t_values[2];
-  else if(flags & ompt_cancel_taskgroup)
+  else if (flags & ompt_cancel_taskgroup)
     first_flag_value = ompt_cancel_flag_t_values[3];
 
-  if(flags & ompt_cancel_activated)
+  if (flags & ompt_cancel_activated)
     second_flag_value = ompt_cancel_flag_t_values[4];
-  else if(flags & ompt_cancel_detected)
+  else if (flags & ompt_cancel_detected)
     second_flag_value = ompt_cancel_flag_t_values[5];
-  else if(flags & ompt_cancel_discarded_task)
+  else if (flags & ompt_cancel_discarded_task)
     second_flag_value = ompt_cancel_flag_t_values[6];
 
-  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_cancel: task_data=%" PRIu64
+  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_cancel: task_id=%" PRIx64
          ", flags=%s|%s=%" PRIu32 ", codeptr_ra=%p\n",
          ompt_get_thread_data()->value, task_data->value, first_flag_value,
          second_flag_value, flags, codeptr_ra);
 }
 
-static void
-on_ompt_callback_implicit_task(
-    ompt_scope_endpoint_t endpoint,
-    ompt_data_t *parallel_data,
-    ompt_data_t *task_data,
-    unsigned int team_size,
-    unsigned int thread_num,
-    int flags)
-{
-  switch(endpoint)
-  {
-    case ompt_scope_begin:
-      if(task_data->ptr)
-        printf("%s\n", "0: task_data initially not null");
-      task_data->value = ompt_get_unique_id();
-
-      //there is no parallel_begin callback for implicit parallel region
-      //thus it is initialized in initial task
-      if(flags & ompt_task_initial)
-      {
-        char buffer[2048];
-
-        format_task_type(flags, buffer);
-        // Only check initial task not created by teams construct
-        if (team_size == 1 && thread_num == 1 && parallel_data->ptr)
-          printf("%s\n", "0: parallel_data initially not null");
-        parallel_data->value = ompt_get_unique_id();
-        printf("%" PRIu64 ":" _TOOL_PREFIX
-               " ompt_event_initial_task_begin: parallel_id=%" PRIu64
-               ", task_id=%" PRIu64 ", actual_parallelism=%" PRIu32
-               ", index=%" PRIu32 ", flags=%" PRIu32 "\n",
-               ompt_get_thread_data()->value, parallel_data->value,
-               task_data->value, team_size, thread_num, flags);
-      } else {
-        printf("%" PRIu64 ":" _TOOL_PREFIX
-               " ompt_event_implicit_task_begin: parallel_id=%" PRIu64
-               ", task_id=%" PRIu64 ", team_size=%" PRIu32
-               ", thread_num=%" PRIu32 "\n",
-               ompt_get_thread_data()->value, parallel_data->value,
-               task_data->value, team_size, thread_num);
-      }
+static void on_ompt_callback_implicit_task(ompt_scope_endpoint_t endpoint,
+                                           ompt_data_t *parallel_data,
+                                           ompt_data_t *task_data,
+                                           unsigned int team_size,
+                                           unsigned int thread_num, int flags) {
+  switch (endpoint) {
+  case ompt_scope_begin:
+    if (task_data->ptr)
+      printf("%s\n", "0: task_data initially not null");
+    task_data->value = ompt_get_unique_id();
+
+    // there is no parallel_begin callback for implicit parallel region
+    // thus it is initialized in initial task
+    if (flags & ompt_task_initial) {
+      char buffer[2048];
+
+      format_task_type(flags, buffer);
+      // Only check initial task not created by teams construct
+      if (team_size == 1 && thread_num == 1 && parallel_data->ptr)
+        printf("%s\n", "0: parallel_data initially not null");
+      parallel_data->value = ompt_get_unique_id();
+      printf("%" PRIu64 ":" _TOOL_PREFIX
+             " ompt_event_initial_task_begin: parallel_id=%" PRIx64
+             ", task_id=%" PRIx64 ", actual_parallelism=%" PRIu32
+             ", index=%" PRIu32 ", flags=%" PRIu32 "\n",
+             ompt_get_thread_data()->value, parallel_data->value,
+             task_data->value, team_size, thread_num, flags);
+    } else {
+      printf("%" PRIu64 ":" _TOOL_PREFIX
+             " ompt_event_implicit_task_begin: parallel_id=%" PRIx64
+             ", task_id=%" PRIx64 ", team_size=%" PRIu32 ", thread_num=%" PRIu32
+             "\n",
+             ompt_get_thread_data()->value, parallel_data->value,
+             task_data->value, team_size, thread_num);
+    }
 
-      break;
-    case ompt_scope_end:
-      if(flags & ompt_task_initial){
-        printf("%" PRIu64 ":" _TOOL_PREFIX
-               " ompt_event_initial_task_end: parallel_id=%" PRIu64
-               ", task_id=%" PRIu64 ", actual_parallelism=%" PRIu32
-               ", index=%" PRIu32 "\n",
-               ompt_get_thread_data()->value,
-               (parallel_data) ? parallel_data->value : 0, task_data->value,
-               team_size, thread_num);
-      } else {
-        printf("%" PRIu64 ":" _TOOL_PREFIX
-               " ompt_event_implicit_task_end: parallel_id=%" PRIu64
-               ", task_id=%" PRIu64 ", team_size=%" PRIu32
-               ", thread_num=%" PRIu32 "\n",
-               ompt_get_thread_data()->value,
-               (parallel_data) ? parallel_data->value : 0, task_data->value,
-               team_size, thread_num);
-      }
-      break;
-    case ompt_scope_beginend:
-      printf("ompt_scope_beginend should never be passed to %s\n", __func__);
-      exit(-1);
+    break;
+  case ompt_scope_end:
+    if (flags & ompt_task_initial) {
+      printf("%" PRIu64 ":" _TOOL_PREFIX
+             " ompt_event_initial_task_end: parallel_id=%" PRIx64
+             ", task_id=%" PRIx64 ", actual_parallelism=%" PRIu32
+             ", index=%" PRIu32 "\n",
+             ompt_get_thread_data()->value,
+             (parallel_data) ? parallel_data->value : 0, task_data->value,
+             team_size, thread_num);
+    } else {
+      printf("%" PRIu64 ":" _TOOL_PREFIX
+             " ompt_event_implicit_task_end: parallel_id=%" PRIx64
+             ", task_id=%" PRIx64 ", team_size=%" PRIu32 ", thread_num=%" PRIu32
+             "\n",
+             ompt_get_thread_data()->value,
+             (parallel_data) ? parallel_data->value : 0, task_data->value,
+             team_size, thread_num);
+    }
+    break;
+  case ompt_scope_beginend:
+    printf("ompt_scope_beginend should never be passed to %s\n", __func__);
+    exit(-1);
   }
 }
 
-static void
-on_ompt_callback_lock_init(
-  ompt_mutex_t kind,
-  unsigned int hint,
-  unsigned int impl,
-  ompt_wait_id_t wait_id,
-  const void *codeptr_ra)
-{
-  switch(kind)
-  {
-    case ompt_mutex_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_init_lock: wait_id=%" PRIu64 ", hint=%" PRIu32
-             ", impl=%" PRIu32 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
-      break;
-    case ompt_mutex_nest_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_init_nest_lock: wait_id=%" PRIu64 ", hint=%" PRIu32
-             ", impl=%" PRIu32 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
-      break;
-    default:
-      break;
+static void on_ompt_callback_lock_init(ompt_mutex_t kind, unsigned int hint,
+                                       unsigned int impl,
+                                       ompt_wait_id_t wait_id,
+                                       const void *codeptr_ra) {
+  switch (kind) {
+  case ompt_mutex_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_init_lock: wait_id=%" PRIu64
+           ", hint=%" PRIu32 ", impl=%" PRIu32 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
+    break;
+  case ompt_mutex_nest_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_init_nest_lock: wait_id=%" PRIu64 ", hint=%" PRIu32
+           ", impl=%" PRIu32 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, hint, impl, codeptr_ra);
+    break;
+  default:
+    break;
   }
 }
 
-static void
-on_ompt_callback_lock_destroy(
-  ompt_mutex_t kind,
-  ompt_wait_id_t wait_id,
-  const void *codeptr_ra)
-{
-  switch(kind)
-  {
-    case ompt_mutex_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_destroy_lock: wait_id=%" PRIu64 ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    case ompt_mutex_nest_lock:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_destroy_nest_lock: wait_id=%" PRIu64
-             ", codeptr_ra=%p \n",
-             ompt_get_thread_data()->value, wait_id, codeptr_ra);
-      break;
-    default:
-      break;
+static void on_ompt_callback_lock_destroy(ompt_mutex_t kind,
+                                          ompt_wait_id_t wait_id,
+                                          const void *codeptr_ra) {
+  switch (kind) {
+  case ompt_mutex_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_destroy_lock: wait_id=%" PRIu64 ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  case ompt_mutex_nest_lock:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_destroy_nest_lock: wait_id=%" PRIu64
+           ", codeptr_ra=%p \n",
+           ompt_get_thread_data()->value, wait_id, codeptr_ra);
+    break;
+  default:
+    break;
   }
 }
 
-static void
-on_ompt_callback_work(
-  ompt_work_t wstype,
-  ompt_scope_endpoint_t endpoint,
-  ompt_data_t *parallel_data,
-  ompt_data_t *task_data,
-  uint64_t count,
-  const void *codeptr_ra)
-{
-  switch(endpoint)
-  {
+static void on_ompt_callback_work(ompt_work_t wstype,
+                                  ompt_scope_endpoint_t endpoint,
+                                  ompt_data_t *parallel_data,
+                                  ompt_data_t *task_data, uint64_t count,
+                                  const void *codeptr_ra) {
+  switch (endpoint) {
   case ompt_scope_begin:
-    printf("%" PRIu64 ":" _TOOL_PREFIX " %s_begin: parallel_id=%" PRIu64
-           ", task_id=%" PRIu64 ", codeptr_ra=%p, count=%" PRIu64 "\n",
+    printf("%" PRIu64 ":" _TOOL_PREFIX " %s_begin: parallel_id=%" PRIx64
+           ", task_id=%" PRIx64 ", codeptr_ra=%p, count=%" PRIu64 "\n",
            ompt_get_thread_data()->value, ompt_work_events_t_values[wstype],
            parallel_data->value, task_data->value, codeptr_ra, count);
     break;
   case ompt_scope_end:
-    printf("%" PRIu64 ":" _TOOL_PREFIX " %s_end: parallel_id=%" PRIu64
-           ", task_id=%" PRIu64 ", codeptr_ra=%p, count=%" PRIu64 "\n",
+    printf("%" PRIu64 ":" _TOOL_PREFIX " %s_end: parallel_id=%" PRIx64
+           ", task_id=%" PRIx64 ", codeptr_ra=%p, count=%" PRIu64 "\n",
            ompt_get_thread_data()->value, ompt_work_events_t_values[wstype],
            parallel_data->value, task_data->value, codeptr_ra, count);
     break;
@@ -816,11 +785,10 @@ on_ompt_callback_work(
   }
 }
 
-static void on_ompt_callback_dispatch(
-    ompt_data_t *parallel_data,
-    ompt_data_t *task_data,
-    ompt_dispatch_t kind,
-    ompt_data_t instance) {
+static void on_ompt_callback_dispatch(ompt_data_t *parallel_data,
+                                      ompt_data_t *task_data,
+                                      ompt_dispatch_t kind,
+                                      ompt_data_t instance) {
   char *event_name = NULL;
   void *codeptr_ra = NULL;
   ompt_dispatch_chunk_t *dispatch_chunk = NULL;
@@ -844,10 +812,10 @@ static void on_ompt_callback_dispatch(
   default:
     event_name = "ompt_ws_loop_iteration_begin";
   }
-  printf("%" PRIu64 ":" _TOOL_PREFIX
-         " %s: parallel_id=%" PRIu64 ", task_id=%" PRIu64
-         ", codeptr_ra=%p, chunk_start=%" PRIu64 ", chunk_iterations=%" PRIu64
-         "\n", ompt_get_thread_data()->value, event_name, parallel_data->value,
+  printf("%" PRIu64 ":" _TOOL_PREFIX " %s: parallel_id=%" PRIx64
+         ", task_id=%" PRIx64 ", codeptr_ra=%p, chunk_start=%" PRIu64
+         ", chunk_iterations=%" PRIu64 "\n",
+         ompt_get_thread_data()->value, event_name, parallel_data->value,
          task_data->value, codeptr_ra,
          dispatch_chunk ? dispatch_chunk->start : 0,
          dispatch_chunk ? dispatch_chunk->iterations : 0);
@@ -857,25 +825,24 @@ static void on_ompt_callback_masked(ompt_scope_endpoint_t endpoint,
                                     ompt_data_t *parallel_data,
                                     ompt_data_t *task_data,
                                     const void *codeptr_ra) {
-  switch(endpoint)
-  {
-    case ompt_scope_begin:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_masked_begin: parallel_id=%" PRIu64
-             ", task_id=%" PRIu64 ", codeptr_ra=%p\n",
-             ompt_get_thread_data()->value, parallel_data->value,
-             task_data->value, codeptr_ra);
-      break;
-    case ompt_scope_end:
-      printf("%" PRIu64 ":" _TOOL_PREFIX
-             " ompt_event_masked_end: parallel_id=%" PRIu64 ", task_id=%" PRIu64
-             ", codeptr_ra=%p\n",
-             ompt_get_thread_data()->value, parallel_data->value,
-             task_data->value, codeptr_ra);
-      break;
-    case ompt_scope_beginend:
-      printf("ompt_scope_beginend should never be passed to %s\n", __func__);
-      exit(-1);
+  switch (endpoint) {
+  case ompt_scope_begin:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_masked_begin: parallel_id=%" PRIx64 ", task_id=%" PRIx64
+           ", codeptr_ra=%p\n",
+           ompt_get_thread_data()->value, parallel_data->value,
+           task_data->value, codeptr_ra);
+    break;
+  case ompt_scope_end:
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " ompt_event_masked_end: parallel_id=%" PRIx64 ", task_id=%" PRIx64
+           ", codeptr_ra=%p\n",
+           ompt_get_thread_data()->value, parallel_data->value,
+           task_data->value, codeptr_ra);
+    break;
+  case ompt_scope_beginend:
+    printf("ompt_scope_beginend should never be passed to %s\n", __func__);
+    exit(-1);
   }
 }
 
@@ -883,16 +850,16 @@ static void on_ompt_callback_parallel_begin(
     ompt_data_t *encountering_task_data,
     const ompt_frame_t *encountering_task_frame, ompt_data_t *parallel_data,
     uint32_t requested_team_size, int flag, const void *codeptr_ra) {
-  if(parallel_data->ptr)
+  if (parallel_data->ptr)
     printf("0: parallel_data initially not null\n");
   parallel_data->value = ompt_get_unique_id();
   int invoker = flag & 0xF;
   const char *event = (flag & ompt_parallel_team) ? "parallel" : "teams";
   const char *size = (flag & ompt_parallel_team) ? "team_size" : "num_teams";
   printf("%" PRIu64 ":" _TOOL_PREFIX
-         " ompt_event_%s_begin: parent_task_id=%" PRIu64
+         " ompt_event_%s_begin: parent_task_id=%" PRIx64
          ", parent_task_frame.exit=%p, parent_task_frame.reenter=%p, "
-         "parallel_id=%" PRIu64 ", requested_%s=%" PRIu32
+         "parallel_id=%" PRIx64 ", requested_%s=%" PRIu32
          ", codeptr_ra=%p, invoker=%d\n",
          ompt_get_thread_data()->value, event, encountering_task_data->value,
          encountering_task_frame->exit_frame.ptr,
@@ -905,22 +872,18 @@ static void on_ompt_callback_parallel_end(ompt_data_t *parallel_data,
                                           int flag, const void *codeptr_ra) {
   int invoker = flag & 0xF;
   const char *event = (flag & ompt_parallel_team) ? "parallel" : "teams";
-  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_%s_end: parallel_id=%" PRIu64
-         ", task_id=%" PRIu64 ", invoker=%d, codeptr_ra=%p\n",
+  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_%s_end: parallel_id=%" PRIx64
+         ", task_id=%" PRIx64 ", invoker=%d, codeptr_ra=%p\n",
          ompt_get_thread_data()->value, event, parallel_data->value,
          encountering_task_data->value, invoker, codeptr_ra);
 }
 
 static void
-on_ompt_callback_task_create(
-    ompt_data_t *encountering_task_data,
-    const ompt_frame_t *encountering_task_frame,
-    ompt_data_t* new_task_data,
-    int type,
-    int has_dependences,
-    const void *codeptr_ra)
-{
-  if(new_task_data->ptr)
+on_ompt_callback_task_create(ompt_data_t *encountering_task_data,
+                             const ompt_frame_t *encountering_task_frame,
+                             ompt_data_t *new_task_data, int type,
+                             int has_dependences, const void *codeptr_ra) {
+  if (new_task_data->ptr)
     printf("0: new_task_data initially not null\n");
   new_task_data->value = ompt_get_unique_id();
   char buffer[2048];
@@ -929,9 +892,9 @@ on_ompt_callback_task_create(
 
   printf(
       "%" PRIu64 ":" _TOOL_PREFIX
-      " ompt_event_task_create: parent_task_id=%" PRIu64
+      " ompt_event_task_create: parent_task_id=%" PRIx64
       ", parent_task_frame.exit=%p, parent_task_frame.reenter=%p, "
-      "new_task_id=%" PRIu64
+      "new_task_id=%" PRIx64
       ", codeptr_ra=%p, task_type=%s=%d, has_dependences=%s\n",
       ompt_get_thread_data()->value,
       encountering_task_data ? encountering_task_data->value : 0,
@@ -941,32 +904,27 @@ on_ompt_callback_task_create(
       has_dependences ? "yes" : "no");
 }
 
-static void
-on_ompt_callback_task_schedule(
-    ompt_data_t *first_task_data,
-    ompt_task_status_t prior_task_status,
-    ompt_data_t *second_task_data)
-{
+static void on_ompt_callback_task_schedule(ompt_data_t *first_task_data,
+                                           ompt_task_status_t prior_task_status,
+                                           ompt_data_t *second_task_data) {
   printf("%" PRIu64 ":" _TOOL_PREFIX
-         " ompt_event_task_schedule: first_task_id=%" PRIu64
-         ", second_task_id=%" PRIu64 ", prior_task_status=%s=%d\n",
+         " ompt_event_task_schedule: first_task_id=%" PRIx64
+         ", second_task_id=%" PRIx64 ", prior_task_status=%s=%d\n",
          ompt_get_thread_data()->value, first_task_data->value,
          (second_task_data ? second_task_data->value : -1),
          ompt_task_status_t_values[prior_task_status], prior_task_status);
   if (prior_task_status == ompt_task_complete ||
       prior_task_status == ompt_task_late_fulfill ||
       prior_task_status == ompt_taskwait_complete) {
-    printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_task_end: task_id=%" PRIu64
-           "\n", ompt_get_thread_data()->value, first_task_data->value);
+    printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_task_end: task_id=%" PRIx64
+           "\n",
+           ompt_get_thread_data()->value, first_task_data->value);
   }
 }
 
-static void
-on_ompt_callback_dependences(
-  ompt_data_t *task_data,
-  const ompt_dependence_t *deps,
-  int ndeps)
-{
+static void on_ompt_callback_dependences(ompt_data_t *task_data,
+                                         const ompt_dependence_t *deps,
+                                         int ndeps) {
   char buffer[2048];
   char *progress = buffer;
   int i;
@@ -983,29 +941,23 @@ on_ompt_callback_dependences(
   }
   if (ndeps > 0)
     progress[-2] = 0;
-  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_dependences: task_id=%" PRIu64
+  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_dependences: task_id=%" PRIx64
          ", deps=[%s], ndeps=%d\n",
          ompt_get_thread_data()->value, task_data->value, buffer, ndeps);
 }
 
-static void
-on_ompt_callback_task_dependence(
-  ompt_data_t *first_task_data,
-  ompt_data_t *second_task_data)
-{
+static void on_ompt_callback_task_dependence(ompt_data_t *first_task_data,
+                                             ompt_data_t *second_task_data) {
   printf("%" PRIu64 ":" _TOOL_PREFIX
-         " ompt_event_task_dependence_pair: first_task_id=%" PRIu64
-         ", second_task_id=%" PRIu64 "\n",
+         " ompt_event_task_dependence_pair: first_task_id=%" PRIx64
+         ", second_task_id=%" PRIx64 "\n",
          ompt_get_thread_data()->value, first_task_data->value,
          second_task_data->value);
 }
 
-static void
-on_ompt_callback_thread_begin(
-  ompt_thread_t thread_type,
-  ompt_data_t *thread_data)
-{
-  if(thread_data->ptr)
+static void on_ompt_callback_thread_begin(ompt_thread_t thread_type,
+                                          ompt_data_t *thread_data) {
+  if (thread_data->ptr)
     printf("%s\n", "0: thread_data initially not null");
   thread_data->value = ompt_get_unique_id();
   printf("%" PRIu64 ":" _TOOL_PREFIX
@@ -1014,26 +966,18 @@ on_ompt_callback_thread_begin(
          thread_type, thread_data->value);
 }
 
-static void
-on_ompt_callback_thread_end(
-  ompt_data_t *thread_data)
-{
-  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_thread_end: thread_id=%" PRIu64
-         "\n",
+static void on_ompt_callback_thread_end(ompt_data_t *thread_data) {
+  printf("%" PRIu64 ":" _TOOL_PREFIX
+         " ompt_event_thread_end: thread_id=%" PRIu64 "\n",
          ompt_get_thread_data()->value, thread_data->value);
 }
 
-static int
-on_ompt_callback_control_tool(
-  uint64_t command,
-  uint64_t modifier,
-  void *arg,
-  const void *codeptr_ra)
-{
-  ompt_frame_t* omptTaskFrame;
-  ompt_get_task_info(0, NULL, (ompt_data_t**) NULL, &omptTaskFrame, NULL, NULL);
-  printf("%" PRIu64 ":" _TOOL_PREFIX " ompt_event_control_tool: command=%" PRIu64
-         ", modifier=%" PRIu64
+static int on_ompt_callback_control_tool(uint64_t command, uint64_t modifier,
+                                         void *arg, const void *codeptr_ra) {
+  ompt_frame_t *omptTaskFrame;
+  ompt_get_task_info(0, NULL, (ompt_data_t **)NULL, &omptTaskFrame, NULL, NULL);
+  printf("%" PRIu64 ":" _TOOL_PREFIX
+         " ompt_event_control_tool: command=%" PRIu64 ", modifier=%" PRIu64
          ", arg=%p, codeptr_ra=%p, current_task_frame.exit=%p, "
          "current_task_frame.reenter=%p \n",
          ompt_get_thread_data()->value, command, modifier, arg, codeptr_ra,
@@ -1046,7 +990,7 @@ on_ompt_callback_control_tool(
   ompt_data_t *task_data;
   while (ompt_get_task_info(task_level, NULL, (ompt_data_t **)&task_data, NULL,
                             NULL, NULL)) {
-    printf("%" PRIu64 ":" _TOOL_PREFIX " task level %d: task_id=%" PRIu64 "\n",
+    printf("%" PRIu64 ":" _TOOL_PREFIX " task level %d: task_id=%" PRIx64 "\n",
            ompt_get_thread_data()->value, task_level, task_data->value);
     task_level++;
   }
@@ -1056,13 +1000,13 @@ on_ompt_callback_control_tool(
   ompt_data_t *parallel_data;
   while (ompt_get_parallel_info(parallel_level, (ompt_data_t **)&parallel_data,
                                 NULL)) {
-    printf("%" PRIu64 ":" _TOOL_PREFIX " parallel level %d: parallel_id=%" PRIu64
-           "\n",
+    printf("%" PRIu64 ":" _TOOL_PREFIX
+           " parallel level %d: parallel_id=%" PRIx64 "\n",
            ompt_get_thread_data()->value, parallel_level, parallel_data->value);
     parallel_level++;
   }
 #endif
-  return 0; //success
+  return 0; // success
 }
 
 static void on_ompt_callback_error(ompt_severity_t severity,
@@ -1074,44 +1018,49 @@ static void on_ompt_callback_error(ompt_severity_t severity,
          codeptr_ra);
 }
 
-int ompt_initialize(
-  ompt_function_lookup_t lookup,
-  int initial_device_num,
-  ompt_data_t *tool_data)
-{
-  ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
-  ompt_get_callback = (ompt_get_callback_t) lookup("ompt_get_callback");
-  ompt_get_state = (ompt_get_state_t) lookup("ompt_get_state");
-  ompt_get_task_info = (ompt_get_task_info_t) lookup("ompt_get_task_info");
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+  ompt_get_callback = (ompt_get_callback_t)lookup("ompt_get_callback");
+  ompt_get_state = (ompt_get_state_t)lookup("ompt_get_state");
+  ompt_get_task_info = (ompt_get_task_info_t)lookup("ompt_get_task_info");
   ompt_get_task_memory = (ompt_get_task_memory_t)lookup("ompt_get_task_memory");
-  ompt_get_thread_data = (ompt_get_thread_data_t) lookup("ompt_get_thread_data");
-  ompt_get_parallel_info = (ompt_get_parallel_info_t) lookup("ompt_get_parallel_info");
-  ompt_get_unique_id = (ompt_get_unique_id_t) lookup("ompt_get_unique_id");
+  ompt_get_thread_data = (ompt_get_thread_data_t)lookup("ompt_get_thread_data");
+  ompt_get_parallel_info =
+      (ompt_get_parallel_info_t)lookup("ompt_get_parallel_info");
+  ompt_get_unique_id = (ompt_get_unique_id_t)lookup("ompt_get_unique_id");
   ompt_finalize_tool = (ompt_finalize_tool_t)lookup("ompt_finalize_tool");
 
   ompt_get_unique_id();
 
-  ompt_get_num_procs = (ompt_get_num_procs_t) lookup("ompt_get_num_procs");
-  ompt_get_num_places = (ompt_get_num_places_t) lookup("ompt_get_num_places");
-  ompt_get_place_proc_ids = (ompt_get_place_proc_ids_t) lookup("ompt_get_place_proc_ids");
-  ompt_get_place_num = (ompt_get_place_num_t) lookup("ompt_get_place_num");
-  ompt_get_partition_place_nums = (ompt_get_partition_place_nums_t) lookup("ompt_get_partition_place_nums");
-  ompt_get_proc_id = (ompt_get_proc_id_t) lookup("ompt_get_proc_id");
-  ompt_enumerate_states = (ompt_enumerate_states_t) lookup("ompt_enumerate_states");
-  ompt_enumerate_mutex_impls = (ompt_enumerate_mutex_impls_t) lookup("ompt_enumerate_mutex_impls");
+  ompt_get_num_procs = (ompt_get_num_procs_t)lookup("ompt_get_num_procs");
+  ompt_get_num_places = (ompt_get_num_places_t)lookup("ompt_get_num_places");
+  ompt_get_place_proc_ids =
+      (ompt_get_place_proc_ids_t)lookup("ompt_get_place_proc_ids");
+  ompt_get_place_num = (ompt_get_place_num_t)lookup("ompt_get_place_num");
+  ompt_get_partition_place_nums =
+      (ompt_get_partition_place_nums_t)lookup("ompt_get_partition_place_nums");
+  ompt_get_proc_id = (ompt_get_proc_id_t)lookup("ompt_get_proc_id");
+  ompt_enumerate_states =
+      (ompt_enumerate_states_t)lookup("ompt_enumerate_states");
+  ompt_enumerate_mutex_impls =
+      (ompt_enumerate_mutex_impls_t)lookup("ompt_enumerate_mutex_impls");
 
   register_ompt_callback(ompt_callback_mutex_acquire);
   register_ompt_callback_t(ompt_callback_mutex_acquired, ompt_callback_mutex_t);
   register_ompt_callback_t(ompt_callback_mutex_released, ompt_callback_mutex_t);
   register_ompt_callback(ompt_callback_nest_lock);
   register_ompt_callback(ompt_callback_sync_region);
-  register_ompt_callback_t(ompt_callback_sync_region_wait, ompt_callback_sync_region_t);
-  register_ompt_callback_t(ompt_callback_reduction, ompt_callback_sync_region_t);
+  register_ompt_callback_t(ompt_callback_sync_region_wait,
+                           ompt_callback_sync_region_t);
+  register_ompt_callback_t(ompt_callback_reduction,
+                           ompt_callback_sync_region_t);
   register_ompt_callback(ompt_callback_control_tool);
   register_ompt_callback(ompt_callback_flush);
   register_ompt_callback(ompt_callback_cancel);
   register_ompt_callback(ompt_callback_implicit_task);
-  register_ompt_callback_t(ompt_callback_lock_init, ompt_callback_mutex_acquire_t);
+  register_ompt_callback_t(ompt_callback_lock_init,
+                           ompt_callback_mutex_acquire_t);
   register_ompt_callback_t(ompt_callback_lock_destroy, ompt_callback_mutex_t);
   register_ompt_callback(ompt_callback_work);
   register_ompt_callback(ompt_callback_dispatch);
@@ -1125,23 +1074,21 @@ int ompt_initialize(
   register_ompt_callback(ompt_callback_thread_begin);
   register_ompt_callback(ompt_callback_thread_end);
   register_ompt_callback(ompt_callback_error);
-  printf("0: NULL_POINTER=%p\n", (void*)NULL);
-  return 1; //success
+  printf("0: NULL_POINTER=%p\n", (void *)NULL);
+  return 1; // success
 }
 
-void ompt_finalize(ompt_data_t *tool_data)
-{
+void ompt_finalize(ompt_data_t *tool_data) {
   printf("0: ompt_event_runtime_shutdown\n");
 }
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-ompt_start_tool_result_t* ompt_start_tool(
-  unsigned int omp_version,
-  const char *runtime_version)
-{
-  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
   return &ompt_start_tool_result;
 }
 #ifdef __cplusplus
diff --git a/openmp/runtime/test/ompt/cancel/cancel_parallel.c b/openmp/runtime/test/ompt/cancel/cancel_parallel.c
index 209b190c046b5..99437d3e99cd1 100644
--- a/openmp/runtime/test/ompt/cancel/cancel_parallel.c
+++ b/openmp/runtime/test/ompt/cancel/cancel_parallel.c
@@ -1,40 +1,46 @@
+// clang-format off
 // RUN: %libomp-compile && env OMP_CANCELLATION=true %libomp-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // Current GOMP interface implementation does not support cancellation
 // XFAIL: gcc
+// clang-format on
 
 #include "callback.h"
 #include "omp.h"
 
 int main() {
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
     if (omp_get_thread_num() == 0) {
       print_fuzzy_address_blocks(get_ompt_label_address(1));
-      #pragma omp cancel parallel
+#pragma omp cancel parallel
       define_ompt_label(1);
-      // We cannot print at this location because the parallel region is cancelled!
+      // We cannot print at this location because the parallel region is
+      // cancelled!
     } else {
       delay(100);
       print_fuzzy_address_blocks(get_ompt_label_address(2));
-      #pragma omp cancellation point parallel
+#pragma omp cancellation point parallel
       define_ompt_label(2);
-      // We cannot print at this location because the parallel region is cancelled!
+      // We cannot print at this location because the parallel region is
+      // cancelled!
     }
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_cancel'
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, actual_parallelism=1, index=1, flags=1 
-  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_cancel: task_data=[[TASK_ID:[0-9]+]], flags=ompt_cancel_parallel|ompt_cancel_activated=17, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, actual_parallelism=1, index=1, flags=1 
+  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_cancel: task_id=[[TASK_ID:[0-f]+]], flags=ompt_cancel_parallel|ompt_cancel_activated=17, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK-DAG: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
   // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin
-  // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_cancel: task_data=[[TASK_ID:[0-9]+]], flags=ompt_cancel_parallel|ompt_cancel_detected=33, codeptr_ra=[[OTHER_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_cancel: task_id=[[TASK_ID:[0-f]+]], flags=ompt_cancel_parallel|ompt_cancel_detected=33, codeptr_ra=[[OTHER_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK-DAG: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[OTHER_RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/cancel/cancel_taskgroup.c b/openmp/runtime/test/ompt/cancel/cancel_taskgroup.c
index 01ab2d18e5c9c..dac7381f73ccb 100644
--- a/openmp/runtime/test/ompt/cancel/cancel_taskgroup.c
+++ b/openmp/runtime/test/ompt/cancel/cancel_taskgroup.c
@@ -1,63 +1,65 @@
+// clang-format off
 // RUN:  %libomp-compile && env OMP_CANCELLATION=true %libomp-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: clang-3, clang-4.0.0
 // Current GOMP interface implementation does not support cancellation; icc 16 has a bug
 // XFAIL: gcc, icc-16
+// clang-format on
 
 #include "callback.h"
-#include <unistd.h>  
+#include <unistd.h>
 #include <stdio.h>
 
-int main()
-{
-  int condition=0;
-  #pragma omp parallel num_threads(2)
+int main() {
+  int condition = 0;
+#pragma omp parallel num_threads(2)
   {}
 
   print_frame(0);
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp master
+#pragma omp master
     {
-      #pragma omp taskgroup
+#pragma omp taskgroup
       {
-        #pragma omp task shared(condition)
+#pragma omp task shared(condition)
         {
           printf("start execute task 1\n");
           OMPT_SIGNAL(condition);
-          OMPT_WAIT(condition,2);
-          #pragma omp cancellation point taskgroup
+          OMPT_WAIT(condition, 2);
+#pragma omp cancellation point taskgroup
           printf("end execute task 1\n");
         }
-        #pragma omp task shared(condition)
+#pragma omp task shared(condition)
         {
           printf("start execute task 2\n");
           OMPT_SIGNAL(condition);
-          OMPT_WAIT(condition,2);
-          #pragma omp cancellation point taskgroup
+          OMPT_WAIT(condition, 2);
+#pragma omp cancellation point taskgroup
           printf("end execute task 2\n");
         }
-      #pragma omp task shared(condition)
+#pragma omp task shared(condition)
         {
           printf("start execute task 3\n");
           OMPT_SIGNAL(condition);
-          OMPT_WAIT(condition,2);
-          #pragma omp cancellation point taskgroup
+          OMPT_WAIT(condition, 2);
+#pragma omp cancellation point taskgroup
           printf("end execute task 3\n");
         }
-      #pragma omp task if(0) shared(condition)
+#pragma omp task if (0) shared(condition)
         {
           printf("start execute task 4\n");
-          OMPT_WAIT(condition,1);
-          #pragma omp cancel taskgroup
+          OMPT_WAIT(condition, 1);
+#pragma omp cancel taskgroup
           printf("end execute task 4\n");
         }
         OMPT_SIGNAL(condition);
       }
     }
-    #pragma omp barrier
+#pragma omp barrier
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_masked'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
@@ -67,24 +69,25 @@ int main()
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_masked_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
-  // CHECK-SAME: task_id=[[PARENT_TASK_ID:[0-9]+]],
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
+  // CHECK-SAME: task_id=[[PARENT_TASK_ID:[0-f]+]],
   // CHECK-SAME: codeptr_ra={{(0x)?[0-f]*}}
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]*}}, parent_task_frame.reenter={{(0x)?[0-f]*}}, new_task_id=[[FIRST_TASK_ID:[0-9]+]], codeptr_ra={{(0x)?[0-f]*}}, task_type=ompt_task_explicit=4, has_dependences=no
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]*}}, parent_task_frame.reenter={{(0x)?[0-f]*}}, new_task_id=[[SECOND_TASK_ID:[0-9]+]], codeptr_ra={{(0x)?[0-f]*}}, task_type=ompt_task_explicit=4, has_dependences=no
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]*}}, parent_task_frame.reenter={{(0x)?[0-f]*}}, new_task_id=[[THIRD_TASK_ID:[0-9]+]], codeptr_ra={{(0x)?[0-f]*}}, task_type=ompt_task_explicit=4, has_dependences=no
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]*}}, parent_task_frame.reenter={{(0x)?[0-f]*}}, new_task_id=[[FIRST_TASK_ID:[0-f]+]], codeptr_ra={{(0x)?[0-f]*}}, task_type=ompt_task_explicit=4, has_dependences=no
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]*}}, parent_task_frame.reenter={{(0x)?[0-f]*}}, new_task_id=[[SECOND_TASK_ID:[0-f]+]], codeptr_ra={{(0x)?[0-f]*}}, task_type=ompt_task_explicit=4, has_dependences=no
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]*}}, parent_task_frame.reenter={{(0x)?[0-f]*}}, new_task_id=[[THIRD_TASK_ID:[0-f]+]], codeptr_ra={{(0x)?[0-f]*}}, task_type=ompt_task_explicit=4, has_dependences=no
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]*}}, parent_task_frame.reenter={{(0x)?[0-f]*}}, new_task_id=[[CANCEL_TASK_ID:[0-9]+]], codeptr_ra={{(0x)?[0-f]*}}, task_type=ompt_task_explicit|ompt_task_undeferred=134217732, has_dependences=no
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[PARENT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]*}}, parent_task_frame.reenter={{(0x)?[0-f]*}}, new_task_id=[[CANCEL_TASK_ID:[0-f]+]], codeptr_ra={{(0x)?[0-f]*}}, task_type=ompt_task_explicit|ompt_task_undeferred=134217732, has_dependences=no
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[PARENT_TASK_ID]], second_task_id=[[CANCEL_TASK_ID]], prior_task_status=ompt_task_switch=7
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_cancel: task_data=[[CANCEL_TASK_ID]], flags=ompt_cancel_taskgroup|ompt_cancel_activated=24, codeptr_ra={{(0x)?[0-f]*}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_cancel: task_id=[[CANCEL_TASK_ID]], flags=ompt_cancel_taskgroup|ompt_cancel_activated=24, codeptr_ra={{(0x)?[0-f]*}}
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[CANCEL_TASK_ID]], second_task_id=[[PARENT_TASK_ID]], prior_task_status=ompt_task_cancel=3
 
-  // CHECK-DAG: {{^}}{{[0-9]+}}: ompt_event_cancel: task_data={{[0-9]+}}, flags=ompt_cancel_taskgroup|ompt_cancel_discarded_task=72, codeptr_ra=[[NULL]]
-  // CHECK-DAG: {{^}}{{[0-9]+}}: ompt_event_cancel: task_data={{[0-9]+}}, flags=ompt_cancel_taskgroup|ompt_cancel_discarded_task=72, codeptr_ra=[[NULL]]
+  // CHECK-DAG: {{^}}{{[0-9]+}}: ompt_event_cancel: task_id={{[0-f]+}}, flags=ompt_cancel_taskgroup|ompt_cancel_discarded_task=72, codeptr_ra=[[NULL]]
+  // CHECK-DAG: {{^}}{{[0-9]+}}: ompt_event_cancel: task_id={{[0-f]+}}, flags=ompt_cancel_taskgroup|ompt_cancel_discarded_task=72, codeptr_ra=[[NULL]]
   
   // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_thread_begin: thread_type=ompt_thread_worker=2, thread_id=[[THREAD_ID]]
-  // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_cancel: task_data={{[0-9]+}}, flags=ompt_cancel_taskgroup|ompt_cancel_detected=40, codeptr_ra={{(0x)?[0-f]*}}
+  // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_cancel: task_id={{[0-f]+}}, flags=ompt_cancel_taskgroup|ompt_cancel_detected=40, codeptr_ra={{(0x)?[0-f]*}}
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/cancel/cancel_worksharing.c b/openmp/runtime/test/ompt/cancel/cancel_worksharing.c
index 7f0913200dd62..7371384b85d17 100644
--- a/openmp/runtime/test/ompt/cancel/cancel_worksharing.c
+++ b/openmp/runtime/test/ompt/cancel/cancel_worksharing.c
@@ -1,67 +1,65 @@
+// clang-format off
 // RUN: %libomp-compile && env OMP_CANCELLATION=true %libomp-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // Current GOMP interface implementation does not support cancellation; icc 16 does not distinguish between sections and loops
 // XFAIL: icc-16
+// clang-format on
 
 #include "callback.h"
 #include <unistd.h>
 
-int main()
-{
-  int condition=0;
-  #pragma omp parallel num_threads(2)
+int main() {
+  int condition = 0;
+#pragma omp parallel num_threads(2)
   {
     int x = 0;
     int i;
-    #pragma omp for
-    for(i = 0; i < 2; i++)
-    {
-      if(i == 0)
-      {
+#pragma omp for
+    for (i = 0; i < 2; i++) {
+      if (i == 0) {
         x++;
         OMPT_SIGNAL(condition);
-        #pragma omp cancel for
-      }
-      else
-      {
+#pragma omp cancel for
+      } else {
         x++;
-        OMPT_WAIT(condition,1);
+        OMPT_WAIT(condition, 1);
         delay(10000);
-        #pragma omp cancellation point for
+#pragma omp cancellation point for
       }
     }
   }
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp sections
+#pragma omp sections
     {
-      #pragma omp section
+#pragma omp section
       {
         OMPT_SIGNAL(condition);
-        #pragma omp cancel sections
+#pragma omp cancel sections
       }
-      #pragma omp section
+#pragma omp section
       {
-        OMPT_WAIT(condition,2);
+        OMPT_WAIT(condition, 2);
         delay(10000);
-        #pragma omp cancellation point sections
+#pragma omp cancellation point sections
       }
     }
   }
 
-
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_cancel'
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, actual_parallelism=1, index=1, flags=1
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, actual_parallelism=1, index=1, flags=1
  
   // cancel for and sections
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_cancel: task_data=[[TASK_ID:[0-9]+]], flags=ompt_cancel_loop|ompt_cancel_activated=20, codeptr_ra={{(0x)?[0-f]*}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_cancel: task_data=[[TASK_ID:[0-9]+]], flags=ompt_cancel_sections|ompt_cancel_{{activated=18|detected=34}}, codeptr_ra={{(0x)?[0-f]*}}
-  // CHECK: {{^}}[[OTHER_THREAD_ID:[0-9]+]]: ompt_event_cancel: task_data=[[TASK_ID:[0-9]+]], flags=ompt_cancel_loop|ompt_cancel_detected=36, codeptr_ra={{(0x)?[0-f]*}}
-  // CHECK: {{^}}[[OTHER_THREAD_ID:[0-9]+]]: ompt_event_cancel: task_data=[[TASK_ID:[0-9]+]], flags=ompt_cancel_sections|ompt_cancel_{{activated=18|detected=34}}, codeptr_ra={{(0x)?[0-f]*}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_cancel: task_id=[[TASK_ID:[0-f]+]], flags=ompt_cancel_loop|ompt_cancel_activated=20, codeptr_ra={{(0x)?[0-f]*}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_cancel: task_id=[[TASK_ID:[0-f]+]], flags=ompt_cancel_sections|ompt_cancel_{{activated=18|detected=34}}, codeptr_ra={{(0x)?[0-f]*}}
+  // CHECK: {{^}}[[OTHER_THREAD_ID:[0-9]+]]: ompt_event_cancel: task_id=[[TASK_ID:[0-f]+]], flags=ompt_cancel_loop|ompt_cancel_detected=36, codeptr_ra={{(0x)?[0-f]*}}
+  // CHECK: {{^}}[[OTHER_THREAD_ID:[0-9]+]]: ompt_event_cancel: task_id=[[TASK_ID:[0-f]+]], flags=ompt_cancel_sections|ompt_cancel_{{activated=18|detected=34}}, codeptr_ra={{(0x)?[0-f]*}}
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/loadtool/tool_available.c b/openmp/runtime/test/ompt/loadtool/tool_available.c
index a08581b9a1521..5759de769fc0f 100644
--- a/openmp/runtime/test/ompt/loadtool/tool_available.c
+++ b/openmp/runtime/test/ompt/loadtool/tool_available.c
@@ -1,3 +1,4 @@
+// clang-format off
 // The OpenMP standard defines 3 ways of providing ompt_start_tool:
 
 // RUN: mkdir -p %t.tool_dir
@@ -74,6 +75,7 @@
 
 
 // REQUIRES: ompt
+// clang-format on
 
 /*
  *  This file contains code for an OMPT shared library tool to be
@@ -82,6 +84,7 @@
  *  -DCODE enables the code for the executable during compilation
  */
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback
 
@@ -99,21 +102,23 @@
 // TOOLLIB-SAME: [[PARENTPATH]]/tool.so... Success.
 // TOOLLIB-NEXT: Tool was started and is using the OMPT interface.
 // TOOLLIB-NEXT: ----- END LOGGING OF TOOL REGISTRATION -----
+// clang-format on
 
 #ifdef CODE
 #include "omp.h"
 
-int main()
-{
-  #pragma omp parallel num_threads(2)
+int main() {
+#pragma omp parallel num_threads(2)
   {
   }
 
+  // clang-format off
   // CHECK-NOT: ----- START LOGGING OF TOOL REGISTRATION -----
   // CHECK-NOT: ----- END LOGGING OF TOOL REGISTRATION -----
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
   // CHECK: {{^}}0: ompt_event_runtime_shutdown
+  // clang-format on
 
   return 0;
 }
@@ -127,20 +132,18 @@ int main()
 
 int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
                     ompt_data_t *tool_data) {
-  printf("0: NULL_POINTER=%p\n", (void*)NULL);
-  return 1; //success
+  printf("0: NULL_POINTER=%p\n", (void *)NULL);
+  return 1; // success
 }
 
-void ompt_finalize(ompt_data_t* tool_data)
-{
+void ompt_finalize(ompt_data_t *tool_data) {
   printf("0: ompt_event_runtime_shutdown\n");
 }
 
-ompt_start_tool_result_t* ompt_start_tool(
-  unsigned int omp_version,
-  const char *runtime_version)
-{
-  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
   return &ompt_start_tool_result;
 }
 #endif /* TOOL */
diff --git a/openmp/runtime/test/ompt/loadtool/tool_available_search.c b/openmp/runtime/test/ompt/loadtool/tool_available_search.c
index 73e2dea1b0a71..a5dfea8996cab 100644
--- a/openmp/runtime/test/ompt/loadtool/tool_available_search.c
+++ b/openmp/runtime/test/ompt/loadtool/tool_available_search.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: mkdir -p %t.tool_dir
 // RUN: %clang %flags -shared -fPIC %s -o %t.tool_dir/first_tool.so
 // RUN: %clang %flags -DTOOL -DSECOND_TOOL -shared -fPIC %s -o %t.tool_dir/second_tool.so
@@ -8,6 +9,7 @@
 
 // REQUIRES: ompt
 // XFAIL: darwin
+// clang-format on
 
 /*
  *  This file contains code for three OMPT shared library tool to be
@@ -19,6 +21,7 @@
  *  -DCODE enables the code for the executable during compilation
  */
 
+// clang-format off
 // CHECK: ----- START LOGGING OF TOOL REGISTRATION -----
 // CHECK-NEXT: Search for OMP tool in current address space... Failed.
 // CHECK-NEXT: Searching tool libraries...
@@ -53,25 +56,23 @@
 // CHECK-DAG: {{^}}0: ompt_event_thread_begin
 // CHECK-DAG: {{^}}0: control_tool()=-1
 // CHECK: {{^}}0: Tool finalized
-
+// clang-format on
 
 #ifdef CODE
 #include "stdio.h"
 #include "omp.h"
 #include "omp-tools.h"
 
-int main()
-{
-  #pragma omp parallel num_threads(2)
+int main() {
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp master
+#pragma omp master
     {
       int result = omp_control_tool(omp_control_tool_start, 0, NULL);
       printf("0: control_tool()=%d\n", result);
     }
   }
 
-
   return 0;
 }
 
@@ -84,10 +85,8 @@ int main()
 
 #ifdef SECOND_TOOL
 // The second tool has an implementation of ompt_start_tool that returns NULL
-ompt_start_tool_result_t* ompt_start_tool(
-  unsigned int omp_version,
-  const char *runtime_version)
-{
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
   printf("0: Do not initialize tool\n");
   return NULL;
 }
@@ -95,33 +94,28 @@ ompt_start_tool_result_t* ompt_start_tool(
 // The third tool has an implementation of ompt_start_tool that returns a
 // pointer to a valid instance of ompt_start_tool_result_t
 
-static void
-on_ompt_callback_thread_begin(
-  ompt_thread_t thread_type,
-  ompt_data_t *thread_data)
-{
+static void on_ompt_callback_thread_begin(ompt_thread_t thread_type,
+                                          ompt_data_t *thread_data) {
   printf("0: ompt_event_thread_begin\n");
 }
 
 int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
                     ompt_data_t *tool_data) {
-  ompt_set_callback_t ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
-  ompt_set_callback(ompt_callback_thread_begin, (ompt_callback_t)on_ompt_callback_thread_begin);
+  ompt_set_callback_t ompt_set_callback =
+      (ompt_set_callback_t)lookup("ompt_set_callback");
+  ompt_set_callback(ompt_callback_thread_begin,
+                    (ompt_callback_t)on_ompt_callback_thread_begin);
   printf("0: Tool initialized\n");
   return 1;
 }
 
-void ompt_finalize(ompt_data_t *tool_data)
-{
-  printf("0: Tool finalized\n");
-}
+void ompt_finalize(ompt_data_t *tool_data) { printf("0: Tool finalized\n"); }
 
-ompt_start_tool_result_t* ompt_start_tool(
-  unsigned int omp_version,
-  const char *runtime_version)
-{
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
   printf("0: Do initialize tool\n");
-  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
   return &ompt_start_tool_result;
 }
 #endif
diff --git a/openmp/runtime/test/ompt/loadtool/tool_not_available.c b/openmp/runtime/test/ompt/loadtool/tool_not_available.c
index df56d31cb2b54..e1c0d4170f7fa 100644
--- a/openmp/runtime/test/ompt/loadtool/tool_not_available.c
+++ b/openmp/runtime/test/ompt/loadtool/tool_not_available.c
@@ -1,3 +1,4 @@
+// clang-format off
 // The OpenMP standard defines 3 ways of providing ompt_start_tool:
 
 // RUN: mkdir -p %t.tool_dir
@@ -44,10 +45,11 @@
 // RUN:    FileCheck %s -DPARENTPATH=%t.tool_dir --check-prefixes CHECK,TOOLLIB
 
 // REQUIRES: ompt
+// clang-format on
 
 /*
- *  This file contains code for an OMPT shared library tool to be 
- *  loaded and the code for the OpenMP executable. 
+ *  This file contains code for an OMPT shared library tool to be
+ *  loaded and the code for the OpenMP executable.
  *  -DTOOL enables the code for the tool during compilation
  *  -DCODE enables the code for the executable during compilation
  */
@@ -57,18 +59,17 @@
 #include "omp.h"
 #include "omp-tools.h"
 
-int main()
-{
-  #pragma omp parallel num_threads(2)
+int main() {
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp master
+#pragma omp master
     {
       int result = omp_control_tool(omp_control_tool_start, 0, NULL);
       printf("0: control_tool()=%d\n", result);
     }
   }
 
-
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback
 
@@ -98,7 +99,7 @@ int main()
   // TOOLLIB-NEXT: ----- END LOGGING OF TOOL REGISTRATION -----
 
   // CHECK: {{^}}0: control_tool()=-2
-  
+  // clang-format on
 
   return 0;
 }
@@ -110,10 +111,8 @@ int main()
 #include <omp-tools.h>
 #include "stdio.h"
 
-ompt_start_tool_result_t* ompt_start_tool(
-  unsigned int omp_version,
-  const char *runtime_version)
-{
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
   printf("0: Do not initialize tool\n");
   return NULL;
 }
diff --git a/openmp/runtime/test/ompt/misc/api_calls_from_other_thread.cpp b/openmp/runtime/test/ompt/misc/api_calls_from_other_thread.cpp
index e2ef1fcdcc77e..34e63ff0cfe41 100644
--- a/openmp/runtime/test/ompt/misc/api_calls_from_other_thread.cpp
+++ b/openmp/runtime/test/ompt/misc/api_calls_from_other_thread.cpp
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-cxx-compile-and-run | FileCheck %s
 // REQUIRES: ompt, linux
+// clang-format on
 
 #include <thread>
 #include "callback.h"
@@ -58,6 +60,7 @@ int main() {
   std::thread t1(f);
   t1.join();
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
@@ -87,6 +90,7 @@ int main() {
   // CHECK: {{^}}[[MASTER_ID]]: ompt_get_parallel_info()=0
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_get_task_info()=0
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/misc/api_calls_misc.c b/openmp/runtime/test/ompt/misc/api_calls_misc.c
index cf6db927694c3..e4470a5c0fa0e 100644
--- a/openmp/runtime/test/ompt/misc/api_calls_misc.c
+++ b/openmp/runtime/test/ompt/misc/api_calls_misc.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile && %libomp-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
@@ -52,6 +54,7 @@ int main() {
     }
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
@@ -67,6 +70,7 @@ int main() {
 
   // CHECK-NOT: {{^}}[[THREAD_ID]]: impl_name is NULL
   // CHECK-NOT: {{^}}[[THREAD_ID]]: mutex_impls enumeration did not end
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/misc/api_calls_places.c b/openmp/runtime/test/ompt/misc/api_calls_places.c
index 3385c9c62abf8..4e135210bbd5e 100644
--- a/openmp/runtime/test/ompt/misc/api_calls_places.c
+++ b/openmp/runtime/test/ompt/misc/api_calls_places.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile && env OMP_PLACES=cores %libomp-run | FileCheck %s
 // REQUIRES: ompt, linux
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 #define __USE_GNU
@@ -58,6 +60,7 @@ int main() {
            ompt_get_thread_data()->value, ompt_get_num_procs());
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
@@ -83,6 +86,7 @@ int main() {
 
   // CHECK: {{^}}[[MASTER_ID]]: omp_get_num_procs()=[[NUM_PROCS:[-]?[0-9]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_get_num_procs()=[[NUM_PROCS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/misc/api_calls_without_ompt.c b/openmp/runtime/test/ompt/misc/api_calls_without_ompt.c
index f76446ef7dc5a..8bb2a7e68b762 100644
--- a/openmp/runtime/test/ompt/misc/api_calls_without_ompt.c
+++ b/openmp/runtime/test/ompt/misc/api_calls_without_ompt.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #define _BSD_SOURCE
 #define _DEFAULT_SOURCE
@@ -76,6 +78,7 @@ int main() {
   printf("%" PRIu64 ": ompt_get_task_info()=%d\n", tvalue,
          ompt_get_task_info(0, NULL, NULL, NULL, NULL, NULL));
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
@@ -105,6 +108,7 @@ int main() {
   // CHECK: {{^}}[[MASTER_ID]]: ompt_get_parallel_info()=0
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_get_task_info()=0
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/misc/control_tool.c b/openmp/runtime/test/ompt/misc/control_tool.c
index ed164206ba146..4b671e9eeab4d 100644
--- a/openmp/runtime/test/ompt/misc/control_tool.c
+++ b/openmp/runtime/test/ompt/misc/control_tool.c
@@ -1,13 +1,14 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #define TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
-  #pragma omp parallel num_threads(1)
+int main() {
+#pragma omp parallel num_threads(1)
   {
     print_frame_from_outlined_fn(1);
     print_frame(0);
@@ -15,6 +16,7 @@ int main()
     print_current_address(0);
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_control_tool'
 
@@ -24,6 +26,7 @@ int main()
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)=[[REENTER_FRAME:(0x)?[0-f]*]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_control_tool: command=3, modifier=1, arg=[[NULL]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]*]], current_task_frame.exit=[[EXIT_FRAME]], current_task_frame.reenter={{(0x)?[0-f]*}}
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/misc/control_tool_no_ompt_support.c b/openmp/runtime/test/ompt/misc/control_tool_no_ompt_support.c
index bcfe3cab38c1e..276d7c4c77f1a 100644
--- a/openmp/runtime/test/ompt/misc/control_tool_no_ompt_support.c
+++ b/openmp/runtime/test/ompt/misc/control_tool_no_ompt_support.c
@@ -1,10 +1,11 @@
+// clang-format off
 // RUN: %libomp-compile-and-run
+// clang-format on
 
 #include <omp.h>
 
-int main()
-{
-  #pragma omp parallel num_threads(1)
+int main() {
+#pragma omp parallel num_threads(1)
   {
     omp_control_tool(omp_control_tool_flush, 1, NULL);
   }
diff --git a/openmp/runtime/test/ompt/misc/finalize_tool.c b/openmp/runtime/test/ompt/misc/finalize_tool.c
index 54709aa5820a6..27062ade7099b 100644
--- a/openmp/runtime/test/ompt/misc/finalize_tool.c
+++ b/openmp/runtime/test/ompt/misc/finalize_tool.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include "omp_testsuite.h"
 
@@ -13,6 +15,7 @@ int main() {
   return get_exit_value();
 }
 
+// clang-format off
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_thread_begin:
 // CHECK-SAME: thread_type=ompt_thread_initial=1
@@ -26,3 +29,4 @@ int main() {
 // CHECK: 0: ompt_event_runtime_shutdown
 
 // CHECK: {{^}}After ompt_finalize_tool
+// clang-format on
diff --git a/openmp/runtime/test/ompt/misc/interoperability.cpp b/openmp/runtime/test/ompt/misc/interoperability.cpp
index 94e65e3f6f2d6..d353a2477f8a1 100644
--- a/openmp/runtime/test/ompt/misc/interoperability.cpp
+++ b/openmp/runtime/test/ompt/misc/interoperability.cpp
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-cxx-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include <iostream>
 #include <thread>
@@ -43,6 +45,7 @@ int main() {
   t2.join();
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_schedule'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_begin'
@@ -57,7 +60,7 @@ int main() {
 // CHECK-SAME: thread_type=ompt_thread_initial=1, thread_id=[[MASTER_ID_1]]
 
 
-// CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-9]+}}
+// CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-f]+}}
 // CHECK-SAME: task_id=[[PARENT_TASK_ID_1:[0-9]+]], actual_parallelism=1,
 // CHECK-SAME: index=1, flags=1
 
@@ -73,7 +76,7 @@ int main() {
 // CHECK-SAME: invoker={{[0-9]+}}
 
 // CHECK: {{^}}[[MASTER_ID_1]]: ompt_event_initial_task_end:
-// CHECK-SAME: parallel_id={{[0-9]+}}, task_id=[[PARENT_TASK_ID_1]],
+// CHECK-SAME: parallel_id={{[0-f]+}}, task_id=[[PARENT_TASK_ID_1]],
 // CHECK-SAME: actual_parallelism=0, index=1
 
 // CHECK: {{^}}[[MASTER_ID_1]]: ompt_event_thread_end:
@@ -83,7 +86,7 @@ int main() {
 // CHECK: {{^}}[[MASTER_ID_2:[0-9]+]]: ompt_event_thread_begin:
 // CHECK-SAME: thread_type=ompt_thread_initial=1, thread_id=[[MASTER_ID_2]]
 
-// CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-9]+}}
+// CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-f]+}}
 // CHECK-SAME: task_id=[[PARENT_TASK_ID_2:[0-9]+]], actual_parallelism=1,
 // CHECK-SAME: index=1, flags=1
 
@@ -100,7 +103,7 @@ int main() {
 // CHECK-SAME: invoker={{[0-9]+}}
 
 // CHECK: {{^}}[[MASTER_ID_2]]: ompt_event_initial_task_end:
-// CHECK-SAME: parallel_id={{[0-9]+}}, task_id=[[PARENT_TASK_ID_2]],
+// CHECK-SAME: parallel_id={{[0-f]+}}, task_id=[[PARENT_TASK_ID_2]],
 // CHECK-SAME: actual_parallelism=0, index=1
 
 // CHECK: {{^}}[[MASTER_ID_2]]: ompt_event_thread_end:
@@ -120,3 +123,4 @@ int main() {
 
 // CHECK: {{^}}[[THREAD_ID_2]]: ompt_event_thread_end:
 // CHECK-SAME: thread_id=[[THREAD_ID_2]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/misc/lock_double_destroy.cpp b/openmp/runtime/test/ompt/misc/lock_double_destroy.cpp
index 4d57f069a4b19..1a26f98196459 100644
--- a/openmp/runtime/test/ompt/misc/lock_double_destroy.cpp
+++ b/openmp/runtime/test/ompt/misc/lock_double_destroy.cpp
@@ -1,8 +1,10 @@
+// clang-format off
 // RUN: %libomp-cxx-compile-and-run | FileCheck %s
 
 // disabled until fixed, see: https://github.com/llvm/llvm-project/pull/145625#issuecomment-3007625680
 // remove "needs-fix", after fixing the issue in the runtime
 // REQUIRES: ompt, needs-fix
+// clang-format on
 #include "callback.h"
 #include "omp_testsuite.h"
 
@@ -26,6 +28,7 @@ int main() {
   return get_exit_value();
 }
 
+// clang-format off
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_thread_begin:
 // CHECK-SAME: thread_type=ompt_thread_initial=1
@@ -41,3 +44,4 @@ int main() {
 // CHECK: 0: ompt_event_runtime_shutdown
 
 // CHECK: {{^}}After ompt_finalize_tool
+// clang-format on
diff --git a/openmp/runtime/test/ompt/misc/pause_stop_tool.c b/openmp/runtime/test/ompt/misc/pause_stop_tool.c
index c939137932445..59448509c40e5 100644
--- a/openmp/runtime/test/ompt/misc/pause_stop_tool.c
+++ b/openmp/runtime/test/ompt/misc/pause_stop_tool.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 int main() {
@@ -23,6 +25,7 @@ int main() {
   printf("x = %d\n", x);
   return 0;
 
+  // clang-format off
   // Check if
   // -- omp_pause_resource/resource_all returns expected code
   // -- OMPT interface is shut down as expected
@@ -41,4 +44,5 @@ int main() {
   // CHECK: omp_pause_resource_all succeeded
 
   // CHECK-NOT: ompt_event
+  // clang-format on
 }
diff --git a/openmp/runtime/test/ompt/misc/runtime_error.c b/openmp/runtime/test/ompt/misc/runtime_error.c
index 2c3d7ac9c8556..90f05f7d97f14 100644
--- a/openmp/runtime/test/ompt/misc/runtime_error.c
+++ b/openmp/runtime/test/ompt/misc/runtime_error.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run 2>&1 | sort | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include <string.h>
 #include <stdio.h>
@@ -21,6 +23,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // CHECK: {{^}}0: Message length=[[LENGTH:[0-9]+]]
 // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
 
@@ -33,3 +36,4 @@ int main() {
 
 // Message from runtime
 // CHECK: {{^}}OMP: Warning{{.*}}User message goes here
+// clang-format on
diff --git a/openmp/runtime/test/ompt/misc/threads.c b/openmp/runtime/test/ompt/misc/threads.c
index 4a0fc6f466bef..25f3347c19af6 100644
--- a/openmp/runtime/test/ompt/misc/threads.c
+++ b/openmp/runtime/test/ompt/misc/threads.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
@@ -11,6 +13,7 @@ int main() {
     x++;
   }
 
+  // clang-format off
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_thread_begin:
@@ -29,6 +32,7 @@ int main() {
   // CHECK-SAME: thread_type=ompt_thread_worker=2, thread_id=[[WORKER_ID3]]
   // CHECK: {{^}}[[WORKER_ID3]]: ompt_event_thread_end:
   // CHECK-SAME: thread_id=[[WORKER_ID3]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/misc/threads_nested.c b/openmp/runtime/test/ompt/misc/threads_nested.c
index 0d38dcf882d6a..b100098d8fc80 100644
--- a/openmp/runtime/test/ompt/misc/threads_nested.c
+++ b/openmp/runtime/test/ompt/misc/threads_nested.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
@@ -17,6 +19,7 @@ int main() {
     }
   }
 
+  // clang-format off
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_thread_begin:
@@ -35,6 +38,7 @@ int main() {
   // CHECK-SAME: thread_type=ompt_thread_worker=2, thread_id=[[WORKER_ID3]]
   // CHECK: {{^}}[[WORKER_ID3]]: ompt_event_thread_end:
   // CHECK-SAME: thread_id=[[WORKER_ID3]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/misc/unset_callback.c b/openmp/runtime/test/ompt/misc/unset_callback.c
index 53669fcfdd678..6a8ec5aef81f4 100644
--- a/openmp/runtime/test/ompt/misc/unset_callback.c
+++ b/openmp/runtime/test/ompt/misc/unset_callback.c
@@ -1,15 +1,17 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include "omp_testsuite.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   go_parallel_nthreads(1);
   ompt_set_callback(ompt_callback_parallel_begin, NULL);
   go_parallel_nthreads(1);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_idle'
 
@@ -19,6 +21,7 @@ int main()
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_parallel_end:
   // CHECK-NOT: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin:
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_parallel_end:
+  // clang-format on
 
   return get_exit_value();
 }
diff --git a/openmp/runtime/test/ompt/ompt-signal.h b/openmp/runtime/test/ompt/ompt-signal.h
index b5c28cf5dee0f..f3eaa55968aa5 100644
--- a/openmp/runtime/test/ompt/ompt-signal.h
+++ b/openmp/runtime/test/ompt/ompt-signal.h
@@ -6,26 +6,25 @@
 #define delay(t) usleep(t);
 #endif
 
-// These functions are used to provide a signal-wait mechanism to enforce expected scheduling for the test cases.
-// Conditional variable (s) needs to be shared! Initialize to 0
+// These functions are used to provide a signal-wait mechanism to enforce
+// expected scheduling for the test cases. Conditional variable (s) needs to be
+// shared! Initialize to 0
 
 #define OMPT_SIGNAL(s) ompt_signal(&s)
-//inline 
-void ompt_signal(int* s) 
-{                
-  #pragma omp atomic
+// inline
+void ompt_signal(int *s) {
+#pragma omp atomic
   (*s)++;
 }
-                
-#define OMPT_WAIT(s,v) ompt_wait(&s,v)
+
+#define OMPT_WAIT(s, v) ompt_wait(&s, v)
 // wait for s >= v
-//inline 
-void ompt_wait(int *s, int v)
-{
-  int wait=0;
-  do{
+// inline
+void ompt_wait(int *s, int v) {
+  int wait = 0;
+  do {
     delay(10);
-    #pragma omp atomic read
-	  wait = (*s);
-  }while(wait<v);
+#pragma omp atomic read
+    wait = (*s);
+  } while (wait < v);
 }
diff --git a/openmp/runtime/test/ompt/parallel/dynamic_enough_threads.c b/openmp/runtime/test/ompt/parallel/dynamic_enough_threads.c
index 975561aab5472..48ec65536af3c 100644
--- a/openmp/runtime/test/ompt/parallel/dynamic_enough_threads.c
+++ b/openmp/runtime/test/ompt/parallel/dynamic_enough_threads.c
@@ -1,18 +1,20 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 
-int main()
-{
+int main() {
   omp_set_dynamic(1);
 
-  #pragma omp parallel num_threads(4)
+#pragma omp parallel num_threads(4)
   {
     print_ids(0);
     print_ids(1);
   }
   print_fuzzy_address(1);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_thread_begin'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_thread_end'
@@ -29,15 +31,16 @@ int main()
   // CHECK-NOT: 0: task_data initially not null
   // CHECK-NOT: 0: thread_data initially not null
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]], team_size={{[1-4]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]], team_size={{[1-4]}}
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/parallel/dynamic_not_enough_threads.c b/openmp/runtime/test/ompt/parallel/dynamic_not_enough_threads.c
index ce5040d6b8ea4..0bf1026db827f 100644
--- a/openmp/runtime/test/ompt/parallel/dynamic_not_enough_threads.c
+++ b/openmp/runtime/test/ompt/parallel/dynamic_not_enough_threads.c
@@ -1,18 +1,20 @@
+// clang-format off
 // RUN: %libomp-compile && env OMP_THREAD_LIMIT=2 %libomp-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 
-int main()
-{
+int main() {
   omp_set_dynamic(1);
 
-  #pragma omp parallel num_threads(4)
+#pragma omp parallel num_threads(4)
   {
     print_ids(0);
     print_ids(1);
   }
   print_fuzzy_address(1);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_thread_begin'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_thread_end'
@@ -29,15 +31,16 @@ int main()
   // CHECK-NOT: 0: task_data initially not null
   // CHECK-NOT: 0: thread_data initially not null
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]], team_size={{[1-4]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]], team_size={{[1-4]}}
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/parallel/max_active_levels_serialized.c b/openmp/runtime/test/ompt/parallel/max_active_levels_serialized.c
index d896d544f80cc..3c62d01e966b2 100644
--- a/openmp/runtime/test/ompt/parallel/max_active_levels_serialized.c
+++ b/openmp/runtime/test/ompt/parallel/max_active_levels_serialized.c
@@ -1,20 +1,21 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=THREADS %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   omp_set_nested(1);
   omp_set_max_active_levels(1);
 
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
     print_ids(0);
     print_ids(1);
-    #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
     {
       print_ids(0);
       print_ids(1);
@@ -24,6 +25,7 @@ int main()
   }
   print_fuzzy_address(2);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_begin'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_end'
@@ -39,35 +41,36 @@ int main()
   // CHECK-NOT: 0: task_data initially not null
   // CHECK-NOT: 0: thread_data initially not null
 
-  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=2, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=2, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
   
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=2, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=2, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[NESTED_RETURN_ADDRESS]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=2, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=2, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[NESTED_RETURN_ADDRESS]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/parallel/nested.c b/openmp/runtime/test/ompt/parallel/nested.c
index d8f3317193695..0e02577da048c 100644
--- a/openmp/runtime/test/ompt/parallel/nested.c
+++ b/openmp/runtime/test/ompt/parallel/nested.c
@@ -1,29 +1,30 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=THREADS %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #define TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN
 #include "callback.h"
 #include <omp.h>
 #include <unistd.h>
 
-int main()
-{
-  int condition=0;
+int main() {
+  int condition = 0;
   omp_set_nested(1);
   print_frame(0);
 
-  #pragma omp parallel num_threads(4)
+#pragma omp parallel num_threads(4)
   {
     print_frame_from_outlined_fn(1);
     print_ids(0);
     print_ids(1);
     print_frame(0);
 
-    //get all implicit task events before starting nested:
-    #pragma omp barrier
-    
-    #pragma omp parallel num_threads(4)
+// get all implicit task events before starting nested:
+#pragma omp barrier
+
+#pragma omp parallel num_threads(4)
     {
       print_frame_from_outlined_fn(1);
       print_ids(0);
@@ -31,8 +32,8 @@ int main()
       print_ids(2);
       print_frame(0);
       OMPT_SIGNAL(condition);
-      OMPT_WAIT(condition,16);
-      #pragma omp barrier
+      OMPT_WAIT(condition, 16);
+#pragma omp barrier
       print_fuzzy_address(1);
       print_ids(0);
     }
@@ -58,20 +59,20 @@ int main()
   // CHECK-NOT: 0: task_data initially not null
   // CHECK-NOT: 0: thread_data initially not null
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
-  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
-  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
+  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // Note that we cannot ensure that the worker threads have already called barrier_end and implicit_task_end before parallel_end!
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
@@ -80,16 +81,16 @@ int main()
 
   // THREADS: {{^}}0: NULL_POINTER=[[NULL:.*$]]
   // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: __builtin_frame_address(0)=[[MAIN_REENTER:(0x)?[0-f]+]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
   // nested parallel masters
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: __builtin_frame_address({{.}})=[[EXIT:(0x)?[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
-  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
+  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
   // THREADS: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)=[[REENTER:(0x)?[0-f]+]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: __builtin_frame_address({{.}})=[[NESTED_EXIT:(0x)?[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_EXIT]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame={{(0x)?[0-f]+}}
@@ -105,8 +106,8 @@ int main()
   // implicit barrier
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[NULL]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]], codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]], codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]], codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
@@ -114,185 +115,185 @@ int main()
   // implicit barrier
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[NULL]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]], codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]], codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]], codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // nested parallel worker threads
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/parallel/nested_lwt.c b/openmp/runtime/test/ompt/parallel/nested_lwt.c
index 8a36955ba3037..d2b7c732df5f8 100644
--- a/openmp/runtime/test/ompt/parallel/nested_lwt.c
+++ b/openmp/runtime/test/ompt/parallel/nested_lwt.c
@@ -1,37 +1,38 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=THREADS %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 #include <unistd.h>
 
-int main()
-{
+int main() {
   omp_set_nested(1);
   int condition = 0;
 
-  #pragma omp parallel num_threads(4)
+#pragma omp parallel num_threads(4)
   {
     print_ids(0);
     print_ids(1);
-    //get all implicit task events before starting nested:
-    #pragma omp barrier
-    #pragma omp parallel num_threads(1)
+// get all implicit task events before starting nested:
+#pragma omp barrier
+#pragma omp parallel num_threads(1)
     {
       print_ids(0);
       print_ids(1);
       print_ids(2);
-      //get all implicit task events before starting nested:
-      #pragma omp barrier
-      #pragma omp parallel num_threads(4)
+// get all implicit task events before starting nested:
+#pragma omp barrier
+#pragma omp parallel num_threads(4)
       {
         print_ids(0);
         print_ids(1);
         print_ids(2);
         print_ids(3);
-      OMPT_SIGNAL(condition);
-      OMPT_WAIT(condition,16);
+        OMPT_SIGNAL(condition);
+        OMPT_WAIT(condition, 16);
       }
       print_fuzzy_address(1);
     }
@@ -54,281 +55,281 @@ int main()
   // CHECK-NOT: 0: parallel_data initially not null
   // CHECK-NOT: 0: task_data initially not null
   // CHECK-NOT: 0: thread_data initially not null
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
-  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
-  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
+  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // Note that we cannot ensure that the worker threads have already called barrier_end and implicit_task_end before parallel_end!
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
 
 
   // THREADS: 0: NULL_POINTER=[[NULL:.*$]]
-  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter=[[TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter=[[TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
   // nested parallel masters
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
-  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
+  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
 
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
 
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 2: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[NESTED_NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[NESTED_NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[NESTED_NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit=[[NESTED_NESTED_TASK_FRAME_EXIT]], parent_task_frame.reenter=[[NESTED_NESTED_TASK_FRAME_ENTER:(0x)?[0-f]+]], parallel_id=[[NESTED_NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_NESTED_TASK_FRAME_EXIT:(0x)?[0-f]+]], reenter_frame=[[NULL]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame=[[NESTED_NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NESTED_TASK_FRAME_EXIT]], reenter_frame=[[NESTED_TASK_FRAME_ENTER]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[NESTED_NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // nested parallel worker threads
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // can't reliably tell which parallel region is the parent...
-  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
-  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
+  // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}
   // THREADS: {{^}}[[THREAD_ID]]: task level 3: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[TASK_FRAME_ENTER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/parallel/nested_lwt_thread_num.c b/openmp/runtime/test/ompt/parallel/nested_lwt_thread_num.c
index 63d9005263713..2b852c6e6cd44 100644
--- a/openmp/runtime/test/ompt/parallel/nested_lwt_thread_num.c
+++ b/openmp/runtime/test/ompt/parallel/nested_lwt_thread_num.c
@@ -1,30 +1,31 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-
 __attribute__ ((noinline)) // workaround for bug in icc
 void print_task_info_at(int ancestor_level, int id)
 {
 #pragma omp critical
   {
     int task_type;
-    char buffer[2048];
-    ompt_data_t *parallel_data;
-    ompt_data_t *task_data;
-    int thread_num;
-    ompt_get_task_info(ancestor_level, &task_type, &task_data, NULL,
-                       &parallel_data, &thread_num);
-    format_task_type(task_type, buffer);
-    printf("%" PRIu64 ": ancestor_level=%d id=%d task_type=%s=%d "
-                      "parallel_id=%" PRIu64 " task_id=%" PRIu64
-                      " thread_num=%d\n",
-        ompt_get_thread_data()->value, ancestor_level, id, buffer,
-        task_type, parallel_data->value, task_data->value, thread_num);
-  }
-};
+char buffer[2048];
+ompt_data_t *parallel_data;
+ompt_data_t *task_data;
+int thread_num;
+ompt_get_task_info(ancestor_level, &task_type, &task_data, NULL, &parallel_data,
+                   &thread_num);
+format_task_type(task_type, buffer);
+printf("%" PRIu64 ": ancestor_level=%d id=%d task_type=%s=%d "
+       "parallel_id=%" PRIx64 " task_id=%" PRIx64 " thread_num=%d\n",
+       ompt_get_thread_data()->value, ancestor_level, id, buffer, task_type,
+       parallel_data->value, task_data->value, thread_num);
+}
+}
+;
 
 __attribute__ ((noinline)) // workaround for bug in icc
 void print_innermost_task_info(int id)
@@ -32,9 +33,7 @@ void print_innermost_task_info(int id)
   print_task_info_at(0, id);
 }
 
-
-int main()
-{
+int main() {
 
 #pragma omp parallel num_threads(2)
   {
@@ -63,14 +62,12 @@ int main()
           print_task_info_at(0, 3);
           print_task_info_at(1, 2);
           print_task_info_at(2, 1);
-
         }
-
       }
     }
   }
 
-
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
@@ -128,6 +125,7 @@ int main()
   // CHECK: {{^}}[[WORKER_ID]]: ancestor_level=2 id=1
   // CHECK-SAME: parallel_id=[[PARALLEL_ID_1]] task_id=[[TASK_ID_2]]
   // CHECK-SAME: thread_num=1
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/parallel/nested_serialized.c b/openmp/runtime/test/ompt/parallel/nested_serialized.c
index d601a8bc6ce5f..c5d52baee73d4 100644
--- a/openmp/runtime/test/ompt/parallel/nested_serialized.c
+++ b/openmp/runtime/test/ompt/parallel/nested_serialized.c
@@ -1,19 +1,20 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=THREADS %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   omp_set_nested(0);
 
-  #pragma omp parallel num_threads(4)
+#pragma omp parallel num_threads(4)
   {
     print_ids(0);
     print_ids(1);
-    #pragma omp parallel num_threads(4)
+#pragma omp parallel num_threads(4)
     {
       print_ids(0);
       print_ids(1);
@@ -39,91 +40,91 @@ int main()
   // CHECK-NOT: 0: task_data initially not null
   // CHECK-NOT: 0: thread_data initially not null
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
-  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
-  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
+  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // Note that we cannot ensure that the worker threads have already called barrier_end and implicit_task_end before parallel_end!
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
 
 
   // THREADS: 0: NULL_POINTER=[[NULL:.*$]]
-  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[NESTED_RETURN_ADDRESS]]
   // THREADS-NOT: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/parallel/nested_serialized_task_frames.c b/openmp/runtime/test/ompt/parallel/nested_serialized_task_frames.c
index b6e3b972a6db0..f5ada415c9335 100644
--- a/openmp/runtime/test/ompt/parallel/nested_serialized_task_frames.c
+++ b/openmp/runtime/test/ompt/parallel/nested_serialized_task_frames.c
@@ -1,11 +1,12 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
 #pragma omp parallel num_threads(1)
   {
     // region 0
@@ -27,13 +28,14 @@ int main()
     }
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
 
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id=[[INITIAL_PARALLEL_ID:[0-9]+]], task_id=[[INITIAL_TASK_ID:[0-9]+]], actual_parallelism=1, index=1, flags=1
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id=[[INITIAL_PARALLEL_ID:[0-f]+]], task_id=[[INITIAL_TASK_ID:[0-f]+]], actual_parallelism=1, index=1, flags=1
 
   // region 0
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin
@@ -76,6 +78,7 @@ int main()
   // CHECK-SAME: exit_frame=[[NULL]]
   // CHECK-SAME: reenter_frame=[[INITIAL_TASK_FRAME_ENTER]]
   // CHECK-SAME: task_type=ompt_task_initial
+  // clang-format on
 
   return 0;
 }
\ No newline at end of file
diff --git a/openmp/runtime/test/ompt/parallel/nested_thread_num.c b/openmp/runtime/test/ompt/parallel/nested_thread_num.c
index e4e9f6ad70767..2dd5139a4de68 100644
--- a/openmp/runtime/test/ompt/parallel/nested_thread_num.c
+++ b/openmp/runtime/test/ompt/parallel/nested_thread_num.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=THREADS %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #define TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN
 #include "callback.h"
 #include <omp.h>
@@ -42,6 +44,7 @@ int main() {
 
   return 0;
 }
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback
 
@@ -53,10 +56,10 @@ int main() {
 // CHECK-NOT: 0: thread_data initially not null
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
-// CHECK-SAME: parent_task_id=[[PARENT_TASK_ID:[0-9]+]],
+// CHECK-SAME: parent_task_id=[[PARENT_TASK_ID:[0-f]+]],
 // CHECK-SAME: parent_task_frame.exit=[[NULL]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
 // CHECK-SAME: requested_team_size=2,
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
 // CHECK-SAME: invoker=[[PARALLEL_INVOKER:[0-9]+]]
@@ -77,17 +80,17 @@ int main() {
 // THREADS: {{^}}0: NULL_POINTER=[[NULL:.*$]]
 // THREADS: __builtin_frame_address(0)=[[MAIN_REENTER:(0x)?[0-f]+]]
 // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
-// THREADS-SAME: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], 
+// THREADS-SAME: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], 
 // THREADS-SAME: parent_task_frame.exit=[[NULL]],
 // THREADS-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// THREADS-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=2,
+// THREADS-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=2,
 // THREADS-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
 // THREADS-SAME: invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
 // nested parallel masters
 // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
 // THREADS-SAME: parallel_id=[[PARALLEL_ID]], 
-// THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]],
+// THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]],
 // THREADS-SAME: team_size=2, thread_num=0
 
 // THREADS: __builtin_frame_address({{.}})=[[EXIT:(0x)?[0-f]+]]
@@ -98,7 +101,7 @@ int main() {
 // THREADS-SAME: thread_num=0
 
 // THREADS: {{^}}[[MASTER_ID]]: task level 1:
-// THREADS-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], 
+// THREADS-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], 
 // THREADS-SAME: task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], 
 // THREADS-SAME: reenter_frame={{(0x)?[0-f]+}}
 
@@ -108,14 +111,14 @@ int main() {
 // THREADS-SAME: parent_task_id=[[IMPLICIT_TASK_ID]], 
 // THREADS-SAME: parent_task_frame.exit=[[EXIT]],
 // THREADS-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], 
+// THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], 
 // THREADS-SAME: requested_team_size=2,
 // THREADS-SAME: codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
 // THREADS-SAME: invoker=[[PARALLEL_INVOKER]]
 
 // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
 // THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID]],
-// THREADS-SAME: task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]], team_size=2, 
+// THREADS-SAME: task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]], team_size=2, 
 // THREADS-SAME: thread_num=0
 
 // THREADS: __builtin_frame_address({{.}})=[[NESTED_EXIT:(0x)?[0-f]+]]
@@ -173,11 +176,11 @@ int main() {
 // THREADS-SAME: exit_frame=[[NULL]], reenter_frame=[[NULL]]
 
 // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]],
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]],
 // THREADS-SAME: codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}}
 
 // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end:
 // THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID]], 
@@ -203,11 +206,11 @@ int main() {
 // THREADS-SAME: reenter_frame=[[NULL]]
 
 // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]],
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]],
 // THREADS-SAME: codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
 
 // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end:
 // THREADS-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]],
@@ -220,7 +223,7 @@ int main() {
 
 // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin:
 // THREADS-SAME: parallel_id=[[PARALLEL_ID]], 
-// THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]], team_size=2, 
+// THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]], team_size=2, 
 // THREADS-SAME: thread_num=[[OUTER_THREADNUM:[0-9]+]]
 
 // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
@@ -235,13 +238,13 @@ int main() {
 // THREADS-SAME: parent_task_id=[[IMPLICIT_TASK_ID]], 
 // THREADS-SAME: parent_task_frame.exit={{(0x)?[0-f]+}},
 // THREADS-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=2,
+// THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=2,
 // THREADS-SAME: codeptr_ra=[[NESTED_RETURN_ADDRESS]]{{[0-f][0-f]}},
 // THREADS-SAME: invoker=[[PARALLEL_INVOKER]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin:
 // THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID]],
-// THREADS-SAME: task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]], team_size=2,
+// THREADS-SAME: task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]], team_size=2,
 // THREADS-SAME: thread_num=[[INNER_THREADNUM:[0-9]+]]
 
 // THREADS: {{^}}[[THREAD_ID]]: task level 0:
@@ -264,10 +267,10 @@ int main() {
 // THREADS-SAME: task_id=[[NESTED_IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[NESTED_IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_parallel_end:
 // THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID]], 
@@ -279,16 +282,16 @@ int main() {
 // THREADS-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
 // nested parallel worker threads
 
 // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin:
-// THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]],
-// THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+// THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]],
+// THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
 // THREADS-SAME: thread_num=[[THREADNUM:[0-9]+]]
 
 // THREADS: {{^}}[[THREAD_ID]]: task level 0:
@@ -298,8 +301,8 @@ int main() {
 
 // can't reliably tell which parallel region is the parent...
 
-// THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}},
-// THREADS-SAME: task_id={{[0-9]+}}
+// THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}},
+// THREADS-SAME: task_id={{[0-f]+}}
 // THREADS-SAME: thread_num={{[01]}}
 
 // THREADS: {{^}}[[THREAD_ID]]: task level 2:
@@ -314,16 +317,16 @@ int main() {
 // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
 // other nested parallel worker threads
 
 // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin:
-// THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]],
-// THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+// THREADS-SAME: parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]],
+// THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
 // THREADS-SAME: thread_num=[[THREADNUM:[0-9]+]]
 
 // THREADS: {{^}}[[THREAD_ID]]: task level 0:
@@ -333,8 +336,8 @@ int main() {
 
 // can't reliably tell which parallel region is the parent...
 
-// THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-9]+}},
-// THREADS-SAME: task_id={{[0-9]+}}
+// THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id={{[0-f]+}},
+// THREADS-SAME: task_id={{[0-f]+}}
 // THREADS-SAME: thread_num={{[01]}}
 
 // THREADS: {{^}}[[THREAD_ID]]: task level 2:
@@ -349,8 +352,8 @@ int main() {
 // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
 // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end:
-// THREADS-SAME: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-
+// THREADS-SAME: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/parallel/nested_threadnum.c b/openmp/runtime/test/ompt/parallel/nested_threadnum.c
index a24853007d32e..c7afb0ffc29a5 100644
--- a/openmp/runtime/test/ompt/parallel/nested_threadnum.c
+++ b/openmp/runtime/test/ompt/parallel/nested_threadnum.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include <omp.h>
 #include "callback.h"
 
@@ -12,10 +14,11 @@ int main() {
     { print_frame(0); }
   }
 
+  // clang-format off
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]]
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
   // CHECK-SAME: thread_num=[[OUTER_THREAD_NUM1:[0-9]+]]
@@ -57,6 +60,7 @@ int main() {
   // CHECK-SAME: thread_num=[[INNER_THREAD_NUM4:[0-9]+]]
   // CHECK: {{^}}[[WORKER_ID3]]: ompt_event_implicit_task_end
   // CHECK-SAME: thread_num=[[INNER_THREAD_NUM4]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/parallel/no_thread_num_clause.c b/openmp/runtime/test/ompt/parallel/no_thread_num_clause.c
index dc5ebe1270915..4aa118d87ef23 100644
--- a/openmp/runtime/test/ompt/parallel/no_thread_num_clause.c
+++ b/openmp/runtime/test/ompt/parallel/no_thread_num_clause.c
@@ -1,12 +1,13 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=THREADS %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 
-int main()
-{
+int main() {
   omp_set_num_threads(4);
-  #pragma omp parallel
+#pragma omp parallel
   {
     print_ids(0);
     print_ids(1);
@@ -31,20 +32,20 @@ int main()
   // CHECK-NOT: 0: task_data initially not null
   // CHECK-NOT: 0: thread_data initially not null
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
-  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
-  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
+  // CHECK-DAG: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // Note that we cannot ensure that the worker threads have already called barrier_end and implicit_task_end before parallel_end!
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-DAG: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK-DAG: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
@@ -52,46 +53,46 @@ int main()
 
   // THREADS: 0: NULL_POINTER=[[NULL:.*$]]
   // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_thread_begin: thread_type=ompt_thread_initial=1, thread_id=[[MASTER_ID]]
-  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, actual_parallelism=1, index=1, flags=1 
+  // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, actual_parallelism=1, index=1, flags=1 
 
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker={{[0-9]+}}
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker={{[0-9]+}}
 
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
   // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_thread_begin: thread_type=ompt_thread_worker=2, thread_id=[[THREAD_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_thread_begin: thread_type=ompt_thread_worker=2, thread_id=[[THREAD_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
   // THREADS: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_thread_begin: thread_type=ompt_thread_worker=2, thread_id=[[THREAD_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/parallel/normal.c b/openmp/runtime/test/ompt/parallel/normal.c
index 6842dc25eb53c..25510691941b6 100644
--- a/openmp/runtime/test/ompt/parallel/normal.c
+++ b/openmp/runtime/test/ompt/parallel/normal.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile-and-run | %sort-threads \
 // RUN:                         | FileCheck --check-prefix=THREADS %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 
 int main() {
@@ -12,6 +14,7 @@ int main() {
   }
   print_fuzzy_address(1);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback
 
@@ -46,19 +49,19 @@ int main() {
   // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_thread_begin
   // THREADS-SAME: thread_type=ompt_thread_initial=1, thread_id=[[MASTER_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin
-  // THREADS-SAME: parent_task_id=[[PARENT_TASK_ID:[0-9]+]]
+  // THREADS-SAME: parent_task_id=[[PARENT_TASK_ID:[0-f]+]]
   // THREADS-SAME: parent_task_frame.exit=[[NULL]]
   // THREADS-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}}
-  // THREADS-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4
+  // THREADS-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4
   // THREADS-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
 
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]]
-  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 1
-  // THREADS-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]]
+  // THREADS-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]]
   // THREADS-SAME: task_id=[[PARENT_TASK_ID]]
   // THREADS-NOT: ompt_event_implicit_task_end
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin
@@ -78,7 +81,7 @@ int main() {
   // THREADS-SAME: thread_type=ompt_thread_worker=2, thread_id=[[THREAD_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]]
-  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1
@@ -97,7 +100,7 @@ int main() {
   // THREADS-SAME: thread_type=ompt_thread_worker=2, thread_id=[[THREAD_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]]
-  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1
@@ -116,7 +119,7 @@ int main() {
   // THREADS-SAME: thread_type=ompt_thread_worker=2, thread_id=[[THREAD_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]]
-  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1
@@ -130,6 +133,7 @@ int main() {
   // THREADS-SAME: parallel_id=0, task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS-SAME: parallel_id=0, task_id=[[IMPLICIT_TASK_ID]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/parallel/not_enough_threads.c b/openmp/runtime/test/ompt/parallel/not_enough_threads.c
index efa3bf72532d5..fc697e2e6c61c 100644
--- a/openmp/runtime/test/ompt/parallel/not_enough_threads.c
+++ b/openmp/runtime/test/ompt/parallel/not_enough_threads.c
@@ -1,8 +1,10 @@
+// clang-format off
 // RUN: %libomp-compile && env OMP_THREAD_LIMIT=2 %libomp-run | FileCheck %s
 // RUN: %libomp-compile && env OMP_THREAD_LIMIT=2 %libomp-run | %sort-threads \
 // RUN:     | FileCheck --check-prefix=THREADS %s
 
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 
@@ -14,6 +16,7 @@ int main() {
   }
   print_fuzzy_address(1);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback
 
@@ -40,19 +43,19 @@ int main() {
   // THREADS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_thread_begin
   // THREADS-SAME: thread_type=ompt_thread_initial=1, thread_id=[[MASTER_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin
-  // THREADS-SAME: parent_task_id=[[PARENT_TASK_ID:[0-9]+]]
+  // THREADS-SAME: parent_task_id=[[PARENT_TASK_ID:[0-f]+]]
   // THREADS-SAME: parent_task_frame.exit=[[NULL]]
   // THREADS-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}}
-  // THREADS-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4
+  // THREADS-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4
   // THREADS-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
 
   // THREADS: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]]
-  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 0
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[MASTER_ID]]: task level 1
-  // THREADS-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]]
+  // THREADS-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]]
   // THREADS-SAME: task_id=[[PARENT_TASK_ID]]
 
   // THREADS-NOT: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end
@@ -71,7 +74,7 @@ int main() {
   // THREADS-SAME: thread_type=ompt_thread_worker=2, thread_id=[[THREAD_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_begin
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]]
-  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // THREADS-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 0
   // THREADS-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: task level 1
@@ -85,6 +88,7 @@ int main() {
   // THREADS-SAME: parallel_id=0, task_id=[[IMPLICIT_TASK_ID]]
   // THREADS: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // THREADS-SAME: parallel_id=0, task_id=[[IMPLICIT_TASK_ID]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/parallel/parallel_if0.c b/openmp/runtime/test/ompt/parallel/parallel_if0.c
index 5a0e71622f9c0..780c005a61ec2 100644
--- a/openmp/runtime/test/ompt/parallel/parallel_if0.c
+++ b/openmp/runtime/test/ompt/parallel/parallel_if0.c
@@ -1,27 +1,28 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 
-int main()
-{
-//  print_frame(0);
-  #pragma omp parallel if(0)
+int main() {
+  //  print_frame(0);
+#pragma omp parallel if (0)
   {
-//    print_frame(1);
+    //    print_frame(1);
     print_ids(0);
     print_ids(1);
-//    print_frame(0);
-    #pragma omp parallel if(0)
+    //    print_frame(0);
+#pragma omp parallel if (0)
     {
-//      print_frame(1);
+      //      print_frame(1);
       print_ids(0);
       print_ids(1);
       print_ids(2);
-//      print_frame(0);
-      #pragma omp task
+      //      print_frame(0);
+#pragma omp task
       {
-//        print_frame(1);
+        //        print_frame(1);
         print_ids(0);
         print_ids(1);
         print_ids(2);
@@ -32,6 +33,7 @@ int main()
   }
   print_fuzzy_address(2);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_begin'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_end'
@@ -45,18 +47,18 @@ int main()
   // CHECK-NOT: 0: task_data initially not null
   // CHECK-NOT: 0: thread_data initially not null
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=1, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=1, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=1, codeptr_ra=[[NESTED_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame={{(0x)?[0-f]+}}
   // CHECK: {{^}}[[MASTER_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[EXPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[EXPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[NESTED_IMPLICIT_TASK_ID]], second_task_id=[[EXPLICIT_TASK_ID]], prior_task_status=ompt_task_switch=7
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[EXPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame={{(0x)?[0-f]+}}
@@ -71,6 +73,7 @@ int main()
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id=0, task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]]
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/parallel/region_in_expl_task_task_frames.c b/openmp/runtime/test/ompt/parallel/region_in_expl_task_task_frames.c
index 6359adf6017fb..30fad3a495fd9 100644
--- a/openmp/runtime/test/ompt/parallel/region_in_expl_task_task_frames.c
+++ b/openmp/runtime/test/ompt/parallel/region_in_expl_task_task_frames.c
@@ -1,16 +1,17 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
 #pragma omp parallel num_threads(2)
   {
     if (omp_get_thread_num() == 0) {
       // region 0
-#pragma omp task if(0)
+#pragma omp task if (0)
       {
         // explicit task immediately executed by the initial master thread
 #pragma omp parallel num_threads(2)
@@ -32,13 +33,14 @@ int main()
     }
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
 
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id=[[INITIAL_PARALLEL_ID:[0-9]+]], task_id=[[INITIAL_TASK_ID:[0-9]+]], actual_parallelism=1, index=1, flags=1
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id=[[INITIAL_PARALLEL_ID:[0-f]+]], task_id=[[INITIAL_TASK_ID:[0-f]+]], actual_parallelism=1, index=1, flags=1
 
   // region 0
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin
@@ -82,6 +84,7 @@ int main()
   // CHECK-SAME: exit_frame=[[NULL]]
   // CHECK-SAME: reenter_frame=[[INITIAL_TASK_FRAME_ENTER]]
   // CHECK-SAME: task_type=ompt_task_initial
+  // clang-format on
 
   return 0;
 }
\ No newline at end of file
diff --git a/openmp/runtime/test/ompt/parallel/repeated_calls.c b/openmp/runtime/test/ompt/parallel/repeated_calls.c
index 155ead4ba0604..af8718170242b 100644
--- a/openmp/runtime/test/ompt/parallel/repeated_calls.c
+++ b/openmp/runtime/test/ompt/parallel/repeated_calls.c
@@ -1,11 +1,12 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #define USE_PRIVATE_TOOL 1
 #include "callback.h"
 
-__attribute__((noinline))
-int foo(int x) {
+__attribute__((noinline)) int foo(int x) {
 #pragma omp parallel num_threads(2)
   {
 #pragma omp atomic
@@ -14,8 +15,7 @@ int foo(int x) {
   return x;
 }
 
-__attribute__((noinline))
-int bar(int x) {
+__attribute__((noinline)) int bar(int x) {
 #pragma omp parallel num_threads(2)
   {
 #pragma omp critical
@@ -31,6 +31,7 @@ int main() {
   y = foo(y);
   return 0;
 
+  // clang-format off
   // CHECK-NOT: {{^}}0: Could not register callback
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
@@ -44,12 +45,11 @@ int main() {
   // Second call to foo
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin
   // CHECK-SAME: {{.*}}codeptr_ra=[[RETURN_ADDRESS]]
-
+  // clang-format on
 }
 
-static void on_ompt_callback_thread_begin(
-    ompt_thread_t thread_type,
-    ompt_data_t *thread_data) {
+static void on_ompt_callback_thread_begin(ompt_thread_t thread_type,
+                                          ompt_data_t *thread_data) {
   if (thread_data->ptr)
     printf("%s\n", "0: thread_data initially not null");
   thread_data->value = ompt_get_unique_id();
diff --git a/openmp/runtime/test/ompt/parallel/serialized.c b/openmp/runtime/test/ompt/parallel/serialized.c
index 9774c6c574de1..51324b44c8d82 100644
--- a/openmp/runtime/test/ompt/parallel/serialized.c
+++ b/openmp/runtime/test/ompt/parallel/serialized.c
@@ -1,27 +1,28 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 
-int main()
-{
-//  print_frame(0);
-  #pragma omp parallel num_threads(1)
+int main() {
+  //  print_frame(0);
+#pragma omp parallel num_threads(1)
   {
-//    print_frame(1);
+    //    print_frame(1);
     print_ids(0);
     print_ids(1);
-//    print_frame(0);
-    #pragma omp parallel num_threads(1)
+    //    print_frame(0);
+#pragma omp parallel num_threads(1)
     {
-//      print_frame(1);
+      //      print_frame(1);
       print_ids(0);
       print_ids(1);
       print_ids(2);
-//      print_frame(0);
-      #pragma omp task
+      //      print_frame(0);
+#pragma omp task
       {
-//        print_frame(1);
+        //        print_frame(1);
         print_ids(0);
         print_ids(1);
         print_ids(2);
@@ -32,6 +33,7 @@ int main()
   }
   print_fuzzy_address(2);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_begin'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_end'
@@ -45,18 +47,18 @@ int main()
   // CHECK-NOT: 0: task_data initially not null
   // CHECK-NOT: 0: thread_data initially not null
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=1, codeptr_ra=[[OUTER_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=1, codeptr_ra=[[OUTER_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-9]+]], requested_team_size=1, codeptr_ra=[[INNER_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[NESTED_PARALLEL_ID:[0-f]+]], requested_team_size=1, codeptr_ra=[[INNER_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker=[[PARALLEL_INVOKER]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame={{(0x)?[0-f]+}}
   // CHECK: {{^}}[[MASTER_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[EXPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[NESTED_IMPLICIT_TASK_ID]], parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[EXPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[NESTED_IMPLICIT_TASK_ID]], second_task_id=[[EXPLICIT_TASK_ID]], prior_task_status=ompt_task_switch=7
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[EXPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame=[[NULL]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[NESTED_PARALLEL_ID]], task_id=[[NESTED_IMPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}, reenter_frame={{(0x)?[0-f]+}}
@@ -72,6 +74,7 @@ int main()
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id=0, task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], invoker=[[PARALLEL_INVOKER]], codeptr_ra=[[OUTER_RETURN_ADDRESS]]{{[0-f][0-f]}}
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[OUTER_RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/barrier/explicit.c b/openmp/runtime/test/ompt/synchronization/barrier/explicit.c
index d3d153979dc4c..134f46e6f3340 100644
--- a/openmp/runtime/test/ompt/synchronization/barrier/explicit.c
+++ b/openmp/runtime/test/ompt/synchronization/barrier/explicit.c
@@ -1,22 +1,23 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   int x = 0;
 
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp atomic
+#pragma omp atomic
     x++;
 
-    #pragma omp barrier
+#pragma omp barrier
     print_current_address();
 
-    #pragma omp atomic
+#pragma omp atomic
     x++;
   }
 
@@ -28,31 +29,31 @@ int main()
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // master thread explicit barrier
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_explicit_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_explicit_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_explicit_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_explicit_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_explicit_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_explicit_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_explicit_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_explicit_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
 
   // master thread implicit barrier at parallel end
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
 
 
   // worker thread explicit barrier
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_explicit_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_explicit_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_explicit_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_explicit_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_explicit_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_explicit_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_explicit_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_explicit_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[THREAD_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
 
   // worker thread implicit barrier at parallel end
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/synchronization/barrier/for_loop.c b/openmp/runtime/test/ompt/synchronization/barrier/for_loop.c
index 31a1f22d4fa93..61d90a2d99ab3 100644
--- a/openmp/runtime/test/ompt/synchronization/barrier/for_loop.c
+++ b/openmp/runtime/test/ompt/synchronization/barrier/for_loop.c
@@ -1,20 +1,20 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
-  int y[] = {0,1,2,3};
+int main() {
+  int y[] = {0, 1, 2, 3};
 
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
-    //implicit barrier at end of for loop
+    // implicit barrier at end of for loop
     int i;
-    #pragma omp for
-    for (i = 0; i < 4; i++)
-    {
+#pragma omp for
+    for (i = 0; i < 4; i++) {
       y[i]++;
     }
     print_current_address();
@@ -28,28 +28,28 @@ int main()
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // master thread implicit barrier at loop end
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
 
   // master thread implicit barrier at parallel end
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
 
   // worker thread implicit barrier at loop end
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
 
   // worker thread implicit barrier after parallel
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/synchronization/barrier/for_simd.c b/openmp/runtime/test/ompt/synchronization/barrier/for_simd.c
index cf280c27a6058..bbe9d29732747 100644
--- a/openmp/runtime/test/ompt/synchronization/barrier/for_simd.c
+++ b/openmp/runtime/test/ompt/synchronization/barrier/for_simd.c
@@ -1,18 +1,18 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // XFAIL: gcc-4
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
-  int y[] = {0,1,2,3};
+int main() {
+  int y[] = {0, 1, 2, 3};
 
   int i;
-  #pragma omp for simd
-  for (i = 0; i < 4; i++)
-  {
+#pragma omp for simd
+  for (i = 0; i < 4; i++) {
     y[i]++;
   }
 
@@ -24,10 +24,10 @@ int main()
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // master thread implicit barrier at simd loop end
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/synchronization/barrier/implicit_task_data.c b/openmp/runtime/test/ompt/synchronization/barrier/implicit_task_data.c
index 10629a350e66a..ff44d33933a46 100644
--- a/openmp/runtime/test/ompt/synchronization/barrier/implicit_task_data.c
+++ b/openmp/runtime/test/ompt/synchronization/barrier/implicit_task_data.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 
@@ -6,6 +7,7 @@
 // Therefore, callback implementations different from the ones in callback.h are necessary.
 // This is a test for an issue reported in 
 // https://github.com/OpenMPToolsInterface/LLVM-openmp/issues/39
+// clang-format on
 
 #define _BSD_SOURCE
 #include <stdio.h>
@@ -14,21 +16,16 @@
 #include <omp.h>
 #include <omp-tools.h>
 
-static const char* ompt_thread_t_values[] = {
-  NULL,
-  "ompt_thread_initial",
-  "ompt_thread_worker",
-  "ompt_thread_other"
-};
+static const char *ompt_thread_t_values[] = {
+    NULL, "ompt_thread_initial", "ompt_thread_worker", "ompt_thread_other"};
 
 static ompt_get_unique_id_t ompt_get_unique_id;
 static ompt_get_thread_data_t ompt_get_thread_data;
 
-int main()
-{
-  #pragma omp parallel num_threads(4)
+int main() {
+#pragma omp parallel num_threads(4)
   {
-    #pragma omp master
+#pragma omp master
     {
       sleep(1);
     }
@@ -42,14 +39,14 @@ int main()
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // master thread implicit barrier at parallel end
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=0, task_id=[[TASK_ID:[0-9]+]], codeptr_ra={{(0x)?[0-f]*}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=0, task_id=[[TASK_ID:[0-f]+]], codeptr_ra={{(0x)?[0-f]*}}
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id=0, task_id=[[TASK_ID]], codeptr_ra={{(0x)?[0-f]*}}
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id=0, task_id=[[TASK_ID]], codeptr_ra={{(0x)?[0-f]*}}
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id=0, task_id=[[TASK_ID]], codeptr_ra={{(0x)?[0-f]*}}
 
 
   // worker thread implicit barrier at parallel end
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=0, task_id=[[TASK_ID:[0-9]+]], codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=0, task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[NULL]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id=0, task_id=[[TASK_ID]], codeptr_ra=[[NULL]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id=0, task_id=[[TASK_ID]], codeptr_ra=[[NULL]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id=0, task_id=[[TASK_ID]], codeptr_ra=[[NULL]]
@@ -58,25 +55,23 @@ int main()
   return 0;
 }
 
-static void
-on_ompt_callback_thread_begin(
-  ompt_thread_t thread_type,
-  ompt_data_t *thread_data)
-{
-  if(thread_data->ptr)
+static void on_ompt_callback_thread_begin(ompt_thread_t thread_type,
+                                          ompt_data_t *thread_data) {
+  if (thread_data->ptr)
     printf("%s\n", "0: thread_data initially not null");
   thread_data->value = ompt_get_unique_id();
-  printf("%" PRIu64 ": ompt_event_thread_begin: thread_type=%s=%d, thread_id=%" PRIu64 "\n", ompt_get_thread_data()->value, ompt_thread_t_values[thread_type], thread_type, thread_data->value);
+  printf("%" PRIu64
+         ": ompt_event_thread_begin: thread_type=%s=%d, thread_id=%" PRIu64
+         "\n",
+         ompt_get_thread_data()->value, ompt_thread_t_values[thread_type],
+         thread_type, thread_data->value);
 }
 
-static void
-on_ompt_callback_sync_region(
-  ompt_sync_region_t kind,
-  ompt_scope_endpoint_t endpoint,
-  ompt_data_t *parallel_data,
-  ompt_data_t *task_data,
-  const void *codeptr_ra)
-{
+static void on_ompt_callback_sync_region(ompt_sync_region_t kind,
+                                         ompt_scope_endpoint_t endpoint,
+                                         ompt_data_t *parallel_data,
+                                         ompt_data_t *task_data,
+                                         const void *codeptr_ra) {
   // We only expect implicit parallel barrier in this code.
   if (kind != ompt_sync_region_barrier_implicit_parallel) {
     printf("unexpected ompt_sync_region_t passed to %s\n", __func__);
@@ -99,14 +94,11 @@ on_ompt_callback_sync_region(
          codeptr_ra);
 }
 
-static void
-on_ompt_callback_sync_region_wait(
-  ompt_sync_region_t kind,
-  ompt_scope_endpoint_t endpoint,
-  ompt_data_t *parallel_data,
-  ompt_data_t *task_data,
-  const void *codeptr_ra)
-{
+static void on_ompt_callback_sync_region_wait(ompt_sync_region_t kind,
+                                              ompt_scope_endpoint_t endpoint,
+                                              ompt_data_t *parallel_data,
+                                              ompt_data_t *task_data,
+                                              const void *codeptr_ra) {
   if (kind != ompt_sync_region_barrier_implicit_parallel) {
     printf("unexpected ompt_sync_region_t passed to %s\n", __func__);
     exit(-1);
@@ -127,38 +119,36 @@ on_ompt_callback_sync_region_wait(
          codeptr_ra);
 }
 
-#define register_ompt_callback_t(name, type)                       \
-do{                                                           \
-  type f_##name = &on_##name;                                 \
-  if (ompt_set_callback(name, (ompt_callback_t)f_##name) ==   \
-      ompt_set_never)                                         \
-    printf("0: Could not register callback '" #name "'\n");   \
-}while(0)
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
 
 #define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
 
 int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
                     ompt_data_t *tool_data) {
   ompt_set_callback_t ompt_set_callback;
-  ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
-  ompt_get_unique_id = (ompt_get_unique_id_t) lookup("ompt_get_unique_id");
-  ompt_get_thread_data = (ompt_get_thread_data_t) lookup("ompt_get_thread_data");
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+  ompt_get_unique_id = (ompt_get_unique_id_t)lookup("ompt_get_unique_id");
+  ompt_get_thread_data = (ompt_get_thread_data_t)lookup("ompt_get_thread_data");
   register_ompt_callback(ompt_callback_sync_region);
-  register_ompt_callback_t(ompt_callback_sync_region_wait, ompt_callback_sync_region_t);
+  register_ompt_callback_t(ompt_callback_sync_region_wait,
+                           ompt_callback_sync_region_t);
   register_ompt_callback(ompt_callback_thread_begin);
-  printf("0: NULL_POINTER=%p\n", (void*)NULL);
-  return 1; //success
+  printf("0: NULL_POINTER=%p\n", (void *)NULL);
+  return 1; // success
 }
 
-void ompt_finalize(ompt_data_t *tool_data)
-{
+void ompt_finalize(ompt_data_t *tool_data) {
   printf("0: ompt_event_runtime_shutdown\n");
 }
 
-ompt_start_tool_result_t* ompt_start_tool(
-  unsigned int omp_version,
-  const char *runtime_version)
-{
-  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
   return &ompt_start_tool_result;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/barrier/parallel_region.c b/openmp/runtime/test/ompt/synchronization/barrier/parallel_region.c
index 3a08678fc8aec..dcbf08f165702 100644
--- a/openmp/runtime/test/ompt/synchronization/barrier/parallel_region.c
+++ b/openmp/runtime/test/ompt/synchronization/barrier/parallel_region.c
@@ -1,16 +1,17 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   int x = 0;
 
-  //implicit barrier at end of a parallel region
-  #pragma omp parallel num_threads(2)
+// implicit barrier at end of a parallel region
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp atomic
+#pragma omp atomic
     x++;
   }
   print_fuzzy_address();
@@ -23,18 +24,18 @@ int main()
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // master thread implicit barrier at parallel end
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
 
   // worker thread implicit barrier at parallel end
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/synchronization/barrier/sections.c b/openmp/runtime/test/ompt/synchronization/barrier/sections.c
index e7372c8d38144..11b85874a080b 100644
--- a/openmp/runtime/test/ompt/synchronization/barrier/sections.c
+++ b/openmp/runtime/test/ompt/synchronization/barrier/sections.c
@@ -1,27 +1,28 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads  | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   int x = 0;
 
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
-    //implicit barrier after sections with nowait but with lastprivates
-    //implicit barrier at end of sections
-    #pragma omp sections
+// implicit barrier after sections with nowait but with lastprivates
+// implicit barrier at end of sections
+#pragma omp sections
     {
-      #pragma omp section 
+#pragma omp section
       {
-        #pragma omp atomic
+#pragma omp atomic
         x++;
       }
-      
-      #pragma omp section 
+
+#pragma omp section
       {
-        #pragma omp atomic
+#pragma omp atomic
         x++;
       }
     }
@@ -35,29 +36,29 @@ int main()
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // master thread implicit barrier at sections end
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
 
   // master thread implicit barrier at parallel end
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
 
 
   // worker thread implicit barrier at sections end
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
 
   // worker thread implicit barrier at parallel end
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/synchronization/barrier/single.c b/openmp/runtime/test/ompt/synchronization/barrier/single.c
index 4c74c092587f8..c22903a345300 100644
--- a/openmp/runtime/test/ompt/synchronization/barrier/single.c
+++ b/openmp/runtime/test/ompt/synchronization/barrier/single.c
@@ -1,23 +1,24 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   int x = 0;
 
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
-    //implicit barrier at end of single
-    #pragma omp single
+// implicit barrier at end of single
+#pragma omp single
     {
       x++;
     }
     print_fuzzy_address();
-    //critical section to avoid merge of two barriers into one
-    #pragma omp critical
+// critical section to avoid merge of two barriers into one
+#pragma omp critical
     {
       x++;
     }
@@ -31,31 +32,31 @@ int main()
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // master thread implicit barrier at single end
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
   // master thread implicit barrier at parallel end
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
 
 
   // worker thread implicit barrier at single end
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_workshare_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]{{[0-f][0-f]}}
   // CHECK: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
   // worker thread implicit barrier at parallel end
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_wait_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[NULL]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/synchronization/critical.c b/openmp/runtime/test/ompt/synchronization/critical.c
index 6f8f93746a877..866f61614a910 100644
--- a/openmp/runtime/test/ompt/synchronization/critical.c
+++ b/openmp/runtime/test/ompt/synchronization/critical.c
@@ -1,19 +1,20 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
-  #pragma omp critical
+int main() {
+#pragma omp critical
   {
     print_current_address(1);
     print_ids(0);
   }
   print_current_address(2);
 
-
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquire'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquired'
@@ -27,6 +28,7 @@ int main()
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_release_critical: wait_id=[[WAIT_ID]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/flush.c b/openmp/runtime/test/ompt/synchronization/flush.c
index b7436385a4f24..78dff0c9cb9c3 100644
--- a/openmp/runtime/test/ompt/synchronization/flush.c
+++ b/openmp/runtime/test/ompt/synchronization/flush.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // GCC generates code that does not call the runtime for the flush construct
 // XFAIL: gcc
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -17,6 +19,7 @@ int main() {
 
   return 0;
 }
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_flush'
 
@@ -28,3 +31,4 @@ int main() {
 // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_flush:
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
 // CHECK: {{^}}[[THREAD_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/synchronization/lock.c b/openmp/runtime/test/ompt/synchronization/lock.c
index 7bd5ac963ca3e..4596507be6367 100644
--- a/openmp/runtime/test/ompt/synchronization/lock.c
+++ b/openmp/runtime/test/ompt/synchronization/lock.c
@@ -1,16 +1,18 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
-  //need to use an OpenMP construct so that OMPT will be initialized
-  #pragma omp parallel num_threads(1)
-    print_ids(0);
+int main() {
+// need to use an OpenMP construct so that OMPT will be initialized
+#pragma omp parallel num_threads(1)
+  print_ids(0);
 
   omp_lock_t lock;
-  printf("%" PRIu64 ": &lock: %" PRIu64 "\n", ompt_get_thread_data()->value, (ompt_wait_id_t)(uintptr_t) &lock);
+  printf("%" PRIu64 ": &lock: %" PRIu64 "\n", ompt_get_thread_data()->value,
+         (ompt_wait_id_t)(uintptr_t)&lock);
   omp_init_lock(&lock);
   print_fuzzy_address(1);
   omp_set_lock(&lock);
@@ -20,7 +22,7 @@ int main()
   omp_destroy_lock(&lock);
   print_fuzzy_address(4);
 
-
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquire'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquired'
@@ -39,6 +41,7 @@ int main()
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_destroy_lock: wait_id=[[WAIT_ID]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
- 
+  // clang-format on
+
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/masked.c b/openmp/runtime/test/ompt/synchronization/masked.c
index 7e5af36ff4254..8d466b7f6fd35 100644
--- a/openmp/runtime/test/ompt/synchronization/masked.c
+++ b/openmp/runtime/test/ompt/synchronization/masked.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // GCC generates code that does not call the runtime for the master construct
 // XFAIL: gcc
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -23,16 +25,18 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_masked'
 
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_masked_begin:
-// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]], task_id=[[TASK_ID:[0-9]+]],
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]], task_id=[[TASK_ID:[0-f]+]],
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
 // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_masked_end:
 // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]],
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS_END:(0x)?[0-f]+]]
 // CHECK: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS_END]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/synchronization/master.c b/openmp/runtime/test/ompt/synchronization/master.c
index e2625937cbe82..f7983257da12d 100644
--- a/openmp/runtime/test/ompt/synchronization/master.c
+++ b/openmp/runtime/test/ompt/synchronization/master.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // GCC generates code that does not call the runtime for the master construct
 // XFAIL: gcc
+// clang-format on
 
 #define USE_PRIVATE_TOOL 1
 #include "callback.h"
@@ -76,6 +78,7 @@ ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
   return &ompt_start_tool_result;
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_master'
 
@@ -87,3 +90,4 @@ ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_master_end:
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS_END:(0x)?[0-f]+]]
 // CHECK: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS_END]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/synchronization/nest_lock.c b/openmp/runtime/test/ompt/synchronization/nest_lock.c
index 680ec6b565742..1c5076683bb32 100644
--- a/openmp/runtime/test/ompt/synchronization/nest_lock.c
+++ b/openmp/runtime/test/ompt/synchronization/nest_lock.c
@@ -1,13 +1,14 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
-  //need to use an OpenMP construct so that OMPT will be initialized
-  #pragma omp parallel num_threads(1)
-    print_ids(0);
+int main() {
+// need to use an OpenMP construct so that OMPT will be initialized
+#pragma omp parallel num_threads(1)
+  print_ids(0);
 
   omp_nest_lock_t nest_lock;
   printf("%" PRIu64 ": &nest_lock: %" PRIu64 "\n",
@@ -25,7 +26,7 @@ int main()
   omp_destroy_nest_lock(&nest_lock);
   print_fuzzy_address(6);
 
-
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquire'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquired'
@@ -48,6 +49,7 @@ int main()
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_destroy_nest_lock: wait_id=[[WAIT_ID]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/ordered.c b/openmp/runtime/test/ompt/synchronization/ordered.c
index ce1ab2e930924..38d4ef1269ace 100644
--- a/openmp/runtime/test/ompt/synchronization/ordered.c
+++ b/openmp/runtime/test/ompt/synchronization/ordered.c
@@ -1,19 +1,20 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
-  #pragma omp ordered
+int main() {
+#pragma omp ordered
   {
     print_current_address(1);
     print_ids(0);
   }
   print_current_address(2);
-  
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquire'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquired'
@@ -27,6 +28,7 @@ int main()
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_release_ordered: wait_id=[[WAIT_ID]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
-  
+  // clang-format on
+
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/ordered_dependences.c b/openmp/runtime/test/ompt/synchronization/ordered_dependences.c
index 0eb64543d2a64..7a8556d3b3c80 100644
--- a/openmp/runtime/test/ompt/synchronization/ordered_dependences.c
+++ b/openmp/runtime/test/ompt/synchronization/ordered_dependences.c
@@ -1,6 +1,8 @@
+// clang-format off
 // RUN: %libomp-c99-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
@@ -21,10 +23,11 @@ int main() {
 
   return 0;
 }
+// clang-format off
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
 // CHECK: {{^}}[[MASTER:[0-9]+]]: ompt_event_loop_static_begin:
-// CHECK-SAME: parallel_id={{[0-9]+}}, task_id=[[ITASK:[0-9]+]],
+// CHECK-SAME: parallel_id={{[0-f]+}}, task_id=[[ITASK:[0-9]+]],
 
 // CHECK: {{^}}[[MASTER]]: ompt_event_dependences: task_id=[[ITASK]],
 // CHECK-SAME: deps=[(0, ompt_dependence_type_source), (0,
@@ -39,7 +42,7 @@ int main() {
 // CHECK-SAME: ompt_dependence_type_source)], ndeps=2
 
 // CHECK: {{^}}[[WORKER:[0-9]+]]: ompt_event_loop_static_begin:
-// CHECK-SAME: parallel_id={{[0-9]+}}, task_id=[[ITASK:[0-9]+]],
+// CHECK-SAME: parallel_id={{[0-f]+}}, task_id=[[ITASK:[0-9]+]],
 
 // CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[ITASK]],
 // CHECK-SAME: deps=[(0, ompt_dependence_type_sink), (0,
@@ -58,3 +61,4 @@ int main() {
 // CHECK: {{^}}[[WORKER]]: ompt_event_dependences: task_id=[[ITASK]],
 // CHECK-SAME: deps=[(1, ompt_dependence_type_source), (1,
 // CHECK-SAME: ompt_dependence_type_source)], ndeps=2
+// clang-format on
diff --git a/openmp/runtime/test/ompt/synchronization/reduction/empty_reduce.c b/openmp/runtime/test/ompt/synchronization/reduction/empty_reduce.c
index ca032984d50e3..663fe4e08089b 100644
--- a/openmp/runtime/test/ompt/synchronization/reduction/empty_reduce.c
+++ b/openmp/runtime/test/ompt/synchronization/reduction/empty_reduce.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile -DNOWAIT && %libomp-run | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
@@ -20,12 +22,13 @@ int main() {
     sum += i;
   }
 
+  // clang-format off
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]]
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-9]+]]
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-f]+]]
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_reduction_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]],
@@ -33,6 +36,7 @@ int main() {
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_reduction_end:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]],
   // CHECK-SAME: task_id=[[TASK_ID]], codeptr_ra=
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/reduction/tree_reduce.c b/openmp/runtime/test/ompt/synchronization/reduction/tree_reduce.c
index 847abc109b2b8..ddaade4c51682 100644
--- a/openmp/runtime/test/ompt/synchronization/reduction/tree_reduce.c
+++ b/openmp/runtime/test/ompt/synchronization/reduction/tree_reduce.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // RUN: %libomp-compile -DNOWAIT && %libomp-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
@@ -21,32 +23,33 @@ int main() {
     a = b = sum += i;
   }
 
-
   printf("%i\n", sum);
+  // clang-format off
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]]
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-9]+]]
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-f]+]]
 
   // order and distribution to threads not determined
   // CHECK: {{^}}{{[0-f]+}}: ompt_event_reduction_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}
   // CHECK: {{^}}{{[0-f]+}}: ompt_event_reduction_end:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}
   // CHECK: {{^}}{{[0-f]+}}: ompt_event_reduction_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}
   // CHECK: {{^}}{{[0-f]+}}: ompt_event_reduction_end:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}
   // CHECK: {{^}}{{[0-f]+}}: ompt_event_reduction_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}
   // CHECK: {{^}}{{[0-f]+}}: ompt_event_reduction_end:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}
   // CHECK: {{^}}{{[0-f]+}}: ompt_event_reduction_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}
   // CHECK: {{^}}{{[0-f]+}}: ompt_event_reduction_end:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/taskgroup.c b/openmp/runtime/test/ompt/synchronization/taskgroup.c
index bd41cc07c490e..0fcf537120dc2 100644
--- a/openmp/runtime/test/ompt/synchronization/taskgroup.c
+++ b/openmp/runtime/test/ompt/synchronization/taskgroup.c
@@ -1,25 +1,26 @@
+// clang-format off
 // RUN:  %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 
 #include "callback.h"
-#include <unistd.h>  
+#include <unistd.h>
 #include <stdio.h>
 
-int main()
-{
-  int condition=0;
-  int x=0;
-  #pragma omp parallel num_threads(2)
+int main() {
+  int condition = 0;
+  int x = 0;
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp master
+#pragma omp master
     {
-      #pragma omp taskgroup
+#pragma omp taskgroup
       {
         print_current_address(1);
-        #pragma omp task
+#pragma omp task
         {
-          #pragma omp atomic
+#pragma omp atomic
           x++;
         }
       }
@@ -27,7 +28,7 @@ int main()
     }
   }
 
-
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_schedule'
@@ -37,12 +38,13 @@ int main()
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_taskgroup_begin: parallel_id=[[PARALLEL_ID:[0-9]+]], task_id=[[TASK_ID:[0-9]+]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_taskgroup_begin: parallel_id=[[PARALLEL_ID:[0-f]+]], task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_taskgroup_begin: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_taskgroup_end: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]], codeptr_ra=[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_taskgroup_end: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]], codeptr_ra=[[RETURN_ADDRESS]]
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/taskwait.c b/openmp/runtime/test/ompt/synchronization/taskwait.c
index 9056b7dd8656a..769c1a85e3e9c 100644
--- a/openmp/runtime/test/ompt/synchronization/taskwait.c
+++ b/openmp/runtime/test/ompt/synchronization/taskwait.c
@@ -1,36 +1,39 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   int x = 0;
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp master
+#pragma omp master
     {
-      #pragma omp task
+#pragma omp task
       {
         x++;
       }
-      #pragma omp taskwait
+#pragma omp taskwait
       print_current_address(1);
     }
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_sync_region'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_sync_region_wait'
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_taskwait_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_taskwait_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_taskwait_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]
-  // CHECK-NEXT: {{^}}[[MASTER_ID]]: ompt_event_taskwait_end: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_taskwait_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_taskwait_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_taskwait_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]
+  // CHECK-NEXT: {{^}}[[MASTER_ID]]: ompt_event_taskwait_end: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, codeptr_ra=[[RETURN_ADDRESS]]
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: current_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/test_lock.c b/openmp/runtime/test/ompt/synchronization/test_lock.c
index 16e4d06245d32..6230ffd68f80a 100644
--- a/openmp/runtime/test/ompt/synchronization/test_lock.c
+++ b/openmp/runtime/test/ompt/synchronization/test_lock.c
@@ -1,11 +1,12 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   omp_lock_t lock;
   omp_init_lock(&lock);
   print_fuzzy_address(1);
@@ -25,6 +26,7 @@ int main()
   omp_destroy_lock(&lock);
   print_fuzzy_address(7);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquire'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquired'
@@ -49,6 +51,7 @@ int main()
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_destroy_lock: wait_id=[[WAIT_ID]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK-NEXT: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/test_nest_lock.c b/openmp/runtime/test/ompt/synchronization/test_nest_lock.c
index fcf41785459db..67f8a733f5453 100644
--- a/openmp/runtime/test/ompt/synchronization/test_nest_lock.c
+++ b/openmp/runtime/test/ompt/synchronization/test_nest_lock.c
@@ -1,11 +1,12 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   omp_nest_lock_t nest_lock;
   omp_init_nest_lock(&nest_lock);
 
@@ -19,6 +20,7 @@ int main()
 
   omp_destroy_nest_lock(&nest_lock);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquire'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquired'
@@ -36,7 +38,8 @@ int main()
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_wait_test_nest_lock: wait_id=[[WAIT_ID]], hint=0, impl={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_acquired_nest_lock_next: wait_id=[[WAIT_ID]], codeptr_ra={{(0x)?[0-f]+}}
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_release_nest_lock_prev: wait_id=[[WAIT_ID]], codeptr_ra={{(0x)?[0-f]+}}  
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_release_nest_lock_last: wait_id=[[WAIT_ID]], codeptr_ra={{(0x)?[0-f]+}}  
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_release_nest_lock_last: wait_id=[[WAIT_ID]], codeptr_ra={{(0x)?[0-f]+}}
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/synchronization/test_nest_lock_parallel.c b/openmp/runtime/test/ompt/synchronization/test_nest_lock_parallel.c
index 37456b00ced51..2592a1e5106be 100644
--- a/openmp/runtime/test/ompt/synchronization/test_nest_lock_parallel.c
+++ b/openmp/runtime/test/ompt/synchronization/test_nest_lock_parallel.c
@@ -1,26 +1,27 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   omp_nest_lock_t nest_lock;
   omp_init_nest_lock(&nest_lock);
 
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp master
+#pragma omp master
     {
       omp_set_nest_lock(&nest_lock);
       print_fuzzy_address(1);
     }
-    #pragma omp barrier
-    omp_test_nest_lock(&nest_lock); //should fail for non-master
+#pragma omp barrier
+    omp_test_nest_lock(&nest_lock); // should fail for non-master
     print_fuzzy_address(2);
-    #pragma omp barrier
-    #pragma omp master
+#pragma omp barrier
+#pragma omp master
     {
       omp_unset_nest_lock(&nest_lock);
       print_fuzzy_address(3);
@@ -31,6 +32,7 @@ int main()
 
   omp_destroy_nest_lock(&nest_lock);
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquire'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_mutex_acquired'
@@ -55,6 +57,7 @@ int main()
   // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_wait_test_nest_lock: wait_id=[[WAIT_ID]], hint=0, impl={{[0-9]+}}, codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK-NOT: {{^}}[[THREAD_ID]]: ompt_event_acquired_test_nest_lock_next: wait_id=[[WAIT_ID]]
   // CHECK-NEXT: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/tasks/dependences.c b/openmp/runtime/test/ompt/tasks/dependences.c
index f6f3fcf7d7dc9..4f916a8edb5d4 100644
--- a/openmp/runtime/test/ompt/tasks/dependences.c
+++ b/openmp/runtime/test/ompt/tasks/dependences.c
@@ -1,6 +1,8 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -9,7 +11,7 @@
 
 int main() {
   int x = 0;
-  int condition=0;
+  int condition = 0;
 #pragma omp parallel num_threads(2)
   {
 #pragma omp master
@@ -20,7 +22,7 @@ int main() {
 #pragma omp task depend(out : x) shared(condition)
       {
         x++;
-        OMPT_WAIT(condition,1);
+        OMPT_WAIT(condition, 1);
       }
       print_fuzzy_address(1);
       print_ids(0);
@@ -37,6 +39,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dependences'
@@ -48,8 +51,8 @@ int main() {
 // CHECK-NOT: 0: new_task_data initially not null
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
 
 // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
 // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT:(0x)?[0-f]+]],
@@ -57,7 +60,7 @@ int main() {
 
 // CHECK: {{^}}[[MASTER_ID]]: address of x: [[ADDRX:(0x)?[0-f]+]]
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[EXIT]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
 // CHECK-SAME: new_task_id=[[FIRST_TASK:[0-f]+]],
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
@@ -73,7 +76,7 @@ int main() {
 // CHECK-SAME: reenter_frame=[[NULL]]
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[EXIT]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
 // CHECK-SAME: new_task_id=[[SECOND_TASK:[0-f]+]], codeptr_ra={{(0x)?[0-f]+}},
 // CHECK-SAME: task_type=ompt_task_explicit=4, has_dependences=yes
@@ -88,3 +91,4 @@ int main() {
 // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
 // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]],
 // CHECK-SAME: reenter_frame=[[NULL]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/tasks/dependences_mutexinoutset.c b/openmp/runtime/test/ompt/tasks/dependences_mutexinoutset.c
index 56066cd39c6ac..62af7ae9f522c 100644
--- a/openmp/runtime/test/ompt/tasks/dependences_mutexinoutset.c
+++ b/openmp/runtime/test/ompt/tasks/dependences_mutexinoutset.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 
@@ -9,6 +10,7 @@
 
 // clang 9 introduced codegen for mutexinoutset
 // UNSUPPORTED: clang-4, clang-5, clang-6, clang-7, clang-8
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -51,6 +53,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dependences'
@@ -62,8 +65,8 @@ int main() {
 // CHECK-NOT: 0: new_task_data initially not null
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
 
 // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
 // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT:(0x)?[0-f]+]],
@@ -71,7 +74,7 @@ int main() {
 
 // CHECK: {{^}}[[MASTER_ID]]: address of x: [[ADDRX:(0x)?[0-f]+]]
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[EXIT]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
 // CHECK-SAME: new_task_id=[[FIRST_TASK:[0-f]+]],
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
@@ -87,7 +90,7 @@ int main() {
 // CHECK-SAME: reenter_frame=[[NULL]]
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[EXIT]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
 // CHECK-SAME: new_task_id=[[SECOND_TASK:[0-f]+]],
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
@@ -103,7 +106,7 @@ int main() {
 // CHECK-SAME: reenter_frame=[[NULL]]
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[EXIT]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
 // CHECK-SAME: new_task_id=[[THIRD_TASK:[0-f]+]], codeptr_ra={{(0x)?[0-f]+}},
 // CHECK-SAME: task_type=ompt_task_explicit=4, has_dependences=yes
@@ -115,3 +118,4 @@ int main() {
 // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
 // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]],
 // CHECK-SAME: reenter_frame=[[NULL]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/tasks/explicit_task.c b/openmp/runtime/test/ompt/tasks/explicit_task.c
index 89c308f6a4558..230aafb1cfb51 100644
--- a/openmp/runtime/test/ompt/tasks/explicit_task.c
+++ b/openmp/runtime/test/ompt/tasks/explicit_task.c
@@ -1,25 +1,26 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #define TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN
 #include "callback.h"
-#include <omp.h> 
+#include <omp.h>
 
-int main()
-{
-  int condition=0;
+int main() {
+  int condition = 0;
   omp_set_nested(0);
   print_frame(0);
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
     print_frame_from_outlined_fn(1);
     print_ids(0);
     print_ids(1);
     print_frame(0);
-    #pragma omp master
+#pragma omp master
     {
       print_ids(0);
-      #pragma omp task shared(condition)
+#pragma omp task shared(condition)
       {
         OMPT_SIGNAL(condition);
         print_frame(1);
@@ -28,10 +29,10 @@ int main()
         print_ids(2);
       }
       print_fuzzy_address(1);
-      OMPT_WAIT(condition,1);
+      OMPT_WAIT(condition, 1);
       print_ids(0);
     }
-    #pragma omp barrier
+#pragma omp barrier
     print_ids(0);
   }
 
@@ -53,16 +54,16 @@ int main()
   // CHECK-NOT: 0: new_task_data initially not null
   
   // CHECK--doesnotwork: {{^}}[[MASTER_ID:[0-9]+]]: __builtin_frame_address(0)=[[MAIN_REENTER:(0x)?[0-f]+]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=2, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=2, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
   // nested parallel masters
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address({{.}})=[[EXIT:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)=[[REENTER:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
   // <- ompt_event_task_create would be expected here
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[TASK_ID:[0-9]+]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
   // explicit barrier after master
@@ -72,10 +73,10 @@ int main()
   // implicit barrier parallel
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: __builtin_frame_address({{.}})=[[EXIT:(0x)?[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
   // CHECK: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
@@ -93,8 +94,8 @@ int main()
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_explicit_end: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/tasks/explicit_task_thread_num.c b/openmp/runtime/test/ompt/tasks/explicit_task_thread_num.c
index 6f3c106476c00..a508a336d838c 100644
--- a/openmp/runtime/test/ompt/tasks/explicit_task_thread_num.c
+++ b/openmp/runtime/test/ompt/tasks/explicit_task_thread_num.c
@@ -1,33 +1,33 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-
 __attribute__ ((noinline)) // workaround for bug in icc
 void print_task_info_at(int ancestor_level, int id)
 {
 #pragma omp critical
   {
     int task_type;
-    char buffer[2048];
-    ompt_data_t *parallel_data;
-    ompt_data_t *task_data;
-    int thread_num;
-    ompt_get_task_info(ancestor_level, &task_type, &task_data, NULL,
-                       &parallel_data, &thread_num);
-    format_task_type(task_type, buffer);
-    printf("%" PRIu64 ": ancestor_level=%d id=%d task_type=%s=%d "
-                      "parallel_id=%" PRIu64 " task_id=%" PRIu64
-                      " thread_num=%d\n",
-        ompt_get_thread_data()->value, ancestor_level, id, buffer,
-        task_type, parallel_data->value, task_data->value, thread_num);
-  }
-};
+char buffer[2048];
+ompt_data_t *parallel_data;
+ompt_data_t *task_data;
+int thread_num;
+ompt_get_task_info(ancestor_level, &task_type, &task_data, NULL, &parallel_data,
+                   &thread_num);
+format_task_type(task_type, buffer);
+printf("%" PRIu64 ": ancestor_level=%d id=%d task_type=%s=%d "
+       "parallel_id=%" PRIu64 " task_id=%" PRIu64 " thread_num=%d\n",
+       ompt_get_thread_data()->value, ancestor_level, id, buffer, task_type,
+       parallel_data->value, task_data->value, thread_num);
+}
+}
+;
 
-int main()
-{
+int main() {
 
 #pragma omp parallel num_threads(2)
   {
@@ -36,7 +36,7 @@ int main()
       // To assert that task is executed by the worker thread,
       // if(0) is used in order to ensure that the task is immediately
       // executed after its creation.
-#pragma omp task if(0)
+#pragma omp task if (0)
       {
         // thread_num should be equal to 1 for both explicit and implicit task
         print_task_info_at(0, 1);
@@ -45,6 +45,7 @@ int main()
     }
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_event_parallel_begin'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
@@ -62,6 +63,7 @@ int main()
   // CHECK-SAME: thread_num=1
   // CHECK: {{^}}[[WID]]: ancestor_level=1 id=0 task_type=ompt_task_implicit
   // CHECK-SAME: thread_num=1
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/tasks/kmp_task_depend_all.c b/openmp/runtime/test/ompt/tasks/kmp_task_depend_all.c
index a18fe5a726e77..78a8d9455e1c1 100644
--- a/openmp/runtime/test/ompt/tasks/kmp_task_depend_all.c
+++ b/openmp/runtime/test/ompt/tasks/kmp_task_depend_all.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 
@@ -66,6 +67,7 @@
 // CHECK: ompt_event_dependences:
 // CHECK-SAME: ompt_dependence_type_in
 // CHECK-SAME: ndeps=1
+// clang-format on
 
 #include "callback.h"
 #include <stdio.h>
diff --git a/openmp/runtime/test/ompt/tasks/omp_task_depend_all.c b/openmp/runtime/test/ompt/tasks/omp_task_depend_all.c
index eff6ea5444b51..19ae662a2ec62 100644
--- a/openmp/runtime/test/ompt/tasks/omp_task_depend_all.c
+++ b/openmp/runtime/test/ompt/tasks/omp_task_depend_all.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 
@@ -65,6 +66,7 @@
 // CHECK: ompt_event_dependences:
 // CHECK-SAME: ompt_dependence_type_in
 // CHECK-SAME: ndeps=1
+// clang-format on
 
 #include "callback.h"
 #include <stdio.h>
diff --git a/openmp/runtime/test/ompt/tasks/serialized.c b/openmp/runtime/test/ompt/tasks/serialized.c
index ad7342d3fdd7d..1939ac4cc60e3 100644
--- a/openmp/runtime/test/ompt/tasks/serialized.c
+++ b/openmp/runtime/test/ompt/tasks/serialized.c
@@ -1,6 +1,8 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #define TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN
 #include "callback.h"
 #include <omp.h>
@@ -23,14 +25,16 @@ int main() {
 #pragma omp task if (t)
       {
         if (creator_frame == get_frame_address(0)) {
-          printf("Assume this code was inlined which the compiler is allowed to do:\n");
+          printf("Assume this code was inlined which the compiler is allowed "
+                 "to do:\n");
           print_frame(0);
         } else if (creator_frame == get_frame_address(1)) {
           printf("Assume this code was called from the application:\n");
           print_frame(1);
         } else {
           // The exit frame must be our parent!
-          printf("Assume this code was not inlined, exit frame must be our parent:\n");
+          printf("Assume this code was not inlined, exit frame must be our "
+                 "parent:\n");
           print_frame_from_outlined_fn(1);
         }
         print_ids(0);
@@ -43,6 +47,7 @@ int main() {
     print_ids(0);
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback
 
@@ -51,22 +56,22 @@ int main() {
   // make sure initial data pointers are null
   // CHECK-NOT: 0: new_task_data initially not null
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-9]+}}
-  // CHECK-SAME: task_id={{[0-9]+}}, actual_parallelism=1, index=1, flags=1 
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-f]+}}
+  // CHECK-SAME: task_id={{[0-f]+}}, actual_parallelism=1, index=1, flags=1 
 
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)
   // CHECK-SAME: =[[MAIN_REENTER:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin
-  // CHECK-SAME: parent_task_id=[[PARENT_TASK_ID:[0-9]+]]
+  // CHECK-SAME: parent_task_id=[[PARENT_TASK_ID:[0-f]+]]
   // CHECK-SAME: parent_task_frame.exit=[[NULL]]
   // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}}
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=2
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=2
   // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}, invoker={{[0-9]+}}
 
   // nested parallel masters
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
-  // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address
   // CHECK-SAME: =[[EXIT:(0x)?[0-f]+]]
 
@@ -75,7 +80,7 @@ int main() {
   // CHECK-SAME: exit_frame=[[EXIT]], reenter_frame=[[NULL]]
 
   // CHECK: {{^}}[[MASTER_ID]]: task level 1
-  // CHECK-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]]
+  // CHECK-SAME: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]]
   // CHECK-SAME: task_id=[[PARENT_TASK_ID]],
   // CHECK-SAME: exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
 
@@ -84,7 +89,7 @@ int main() {
   // CHECK-SAME: parent_task_id=[[IMPLICIT_TASK_ID]]
   // CHECK-SAME: parent_task_frame.exit=[[EXIT]]
   // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}}
-  // CHECK-SAME: new_task_id=[[TASK_ID:[0-9]+]]
+  // CHECK-SAME: new_task_id=[[TASK_ID:[0-f]+]]
   // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule:
@@ -127,7 +132,7 @@ int main() {
 
   // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
-  // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: __builtin_frame_address
   // CHECK-SAME: =[[EXIT:(0x)?[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: task level 0
@@ -150,6 +155,7 @@ int main() {
 
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end
   // CHECK-SAME: parallel_id=0, task_id=[[IMPLICIT_TASK_ID]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/tasks/task_early_fulfill.c b/openmp/runtime/test/ompt/tasks/task_early_fulfill.c
index 983e4c73dfdb5..1877f180a7cd5 100644
--- a/openmp/runtime/test/ompt/tasks/task_early_fulfill.c
+++ b/openmp/runtime/test/ompt/tasks/task_early_fulfill.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-compile && env OMP_NUM_THREADS='3' \
 // RUN:    %libomp-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
@@ -10,6 +11,7 @@
 // UNSUPPORTED: clang-10, clang-9, clang-8, clang-7
 // icc compiler does not support detach clause.
 // UNSUPPORTED: icc
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -26,6 +28,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_schedule'
@@ -39,21 +42,21 @@ int main() {
 // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
-// CHECK-SAME: parent_task_id=[[PARENT_TASK_ID:[0-9]+]],
+// CHECK-SAME: parent_task_id=[[PARENT_TASK_ID:[0-f]+]],
 // CHECK-SAME: parent_task_frame.exit=[[NULL]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
 // CHECK-SAME: requested_team_size=3,
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[PARALLEL_ID]],
-// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
 // CHECK-SAME: parent_task_id=[[IMPLICIT_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}},
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: new_task_id=[[TASK_ID:[0-9]+]],
+// CHECK-SAME: new_task_id=[[TASK_ID:[0-f]+]],
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_task_schedule:
 // CHECK-SAME: first_task_id=[[IMPLICIT_TASK_ID]],
@@ -62,10 +65,11 @@ int main() {
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_task_schedule:
 // CHECK-SAME: first_task_id=[[TASK_ID]],
-// CHECK-SAME: second_task_id=18446744073709551615,
+// CHECK-SAME: second_task_id=ffffffffffffffff,
 // CHECK-SAME: prior_task_status=ompt_task_early_fulfill=5
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_task_schedule:
 // CHECK-SAME: first_task_id=[[TASK_ID]],
 // CHECK-SAME: second_task_id=[[IMPLICIT_TASK_ID]],
 // CHECK-SAME: prior_task_status=ompt_task_complete=1
+// clang-format on
diff --git a/openmp/runtime/test/ompt/tasks/task_if0-depend.c b/openmp/runtime/test/ompt/tasks/task_if0-depend.c
index f98695b3113ff..fca97fd7273e3 100644
--- a/openmp/runtime/test/ompt/tasks/task_if0-depend.c
+++ b/openmp/runtime/test/ompt/tasks/task_if0-depend.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -25,6 +27,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dependences'
@@ -36,8 +39,8 @@ int main() {
 // CHECK-NOT: 0: new_task_data initially not null
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
 
 // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
 // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT:(0x)?[0-f]+]],
@@ -46,7 +49,7 @@ int main() {
 // CHECK: {{^}}[[MASTER_ID]]: address of x: [[ADDRX:(0x)?[0-f]+]]
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[EXIT]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
 // CHECK-SAME: new_task_id=[[FIRST_TASK:[0-f]+]],
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
@@ -59,7 +62,7 @@ int main() {
 // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[EXIT]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
 // CHECK-SAME: new_task_id=[[SECOND_TASK:[0-f]+]],
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
@@ -73,3 +76,4 @@ int main() {
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_end: task_id=[[SECOND_TASK]]
 
 // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/tasks/task_in_joinbarrier.c b/openmp/runtime/test/ompt/tasks/task_in_joinbarrier.c
index 114bda81d6d80..1114ff699aa80 100644
--- a/openmp/runtime/test/ompt/tasks/task_in_joinbarrier.c
+++ b/openmp/runtime/test/ompt/tasks/task_in_joinbarrier.c
@@ -1,24 +1,25 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #define TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN
 #include "callback.h"
-#include <omp.h> 
+#include <omp.h>
 
-int main()
-{
-  int condition=0;
+int main() {
+  int condition = 0;
   omp_set_nested(0);
   print_frame(0);
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
     print_frame_from_outlined_fn(1);
     print_ids(0);
     print_ids(1);
     print_frame(0);
-    #pragma omp master
+#pragma omp master
     {
       print_ids(0);
-      #pragma omp task shared(condition)
+#pragma omp task shared(condition)
       {
         OMPT_SIGNAL(condition);
         print_frame(1);
@@ -26,7 +27,7 @@ int main()
         print_ids(1);
         print_ids(2);
       }
-      OMPT_WAIT(condition,1);
+      OMPT_WAIT(condition, 1);
       print_ids(0);
     }
     print_ids(0);
@@ -50,24 +51,24 @@ int main()
   // CHECK-NOT: 0: new_task_data initially not null
 
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: __builtin_frame_address(0)=[[MAIN_REENTER:(0x)?[0-f]+]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=2, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=2, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
   // nested parallel masters
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address({{.}})=[[EXIT:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)=[[REENTER:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
   // <- ompt_event_task_create would be expected here
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[TASK_ID:[0-9]+]], codeptr_ra=[[TASK_FUNCTION:(0x)?[0-f]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[TASK_FUNCTION:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
   // implicit barrier parallel
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: __builtin_frame_address({{.}})=[[EXIT:(0x)?[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
   // CHECK: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
@@ -82,8 +83,8 @@ int main()
   // CHECK: {{^}}[[THREAD_ID]]: task level 2: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_task_schedule: first_task_id=[[TASK_ID]], second_task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_task_end: task_id=[[TASK_ID]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/tasks/task_late_fulfill.c b/openmp/runtime/test/ompt/tasks/task_late_fulfill.c
index 8de18f9576c46..df731671b9864 100644
--- a/openmp/runtime/test/ompt/tasks/task_late_fulfill.c
+++ b/openmp/runtime/test/ompt/tasks/task_late_fulfill.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-compile && env OMP_NUM_THREADS='3' \
 // RUN:    %libomp-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
@@ -10,6 +11,7 @@
 // UNSUPPORTED: clang-10, clang-9, clang-8, clang-7
 // icc compiler does not support detach clause.
 // UNSUPPORTED: icc
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -34,6 +36,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_schedule'
@@ -47,15 +50,15 @@ int main() {
 // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
-// CHECK-SAME: parent_task_id=[[PARENT_TASK_ID:[0-9]+]],
+// CHECK-SAME: parent_task_id=[[PARENT_TASK_ID:[0-f]+]],
 // CHECK-SAME: parent_task_frame.exit=[[NULL]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
 // CHECK-SAME: requested_team_size=3,
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[PARALLEL_ID]],
-// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
 
 // The following is to match the taskwait task created in __kmpc_omp_wait_deps
 // this should go away, once codegen for "detached if(0)" is fixed
@@ -68,7 +71,7 @@ int main() {
 // CHECK-SAME: parent_task_id=[[IMPLICIT_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}},
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: new_task_id=[[TASK_ID:[0-9]+]],
+// CHECK-SAME: new_task_id=[[TASK_ID:[0-f]+]],
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_task_schedule:
 // CHECK-SAME: first_task_id=[[IMPLICIT_TASK_ID]],
@@ -82,5 +85,6 @@ int main() {
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_task_schedule:
 // CHECK-SAME: first_task_id=[[TASK_ID]],
-// CHECK-SAME: second_task_id=18446744073709551615,
+// CHECK-SAME: second_task_id=ffffffffffffffff,
 // CHECK-SAME: prior_task_status=ompt_task_late_fulfill=6
+// clang-format on
diff --git a/openmp/runtime/test/ompt/tasks/task_memory.c b/openmp/runtime/test/ompt/tasks/task_memory.c
index ba78f6691f85a..0c612ffdb4768 100644
--- a/openmp/runtime/test/ompt/tasks/task_memory.c
+++ b/openmp/runtime/test/ompt/tasks/task_memory.c
@@ -1,6 +1,8 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 #define USE_PRIVATE_TOOL 1
 #include "callback.h"
 #include <omp.h>
@@ -93,9 +95,10 @@ ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
   return &ompt_start_tool_result;
 }
 
+// clang-format off
 // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
 
-// CHECK: ompt_event_implicit_task_begin: task_id=[[TASK_ID:[0-9]+]]
+// CHECK: ompt_event_implicit_task_begin: task_id=[[TASK_ID:[0-f]+]]
 // CHECK-SAME: memory_addr=[[NULL]], memory_size=0, result=0
 
 // CHECK: ompt_event_task_create: task_id=[[TASK_ID_0:[0-9]+]]
@@ -109,3 +112,4 @@ ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
 
 // CHECK: ompt_event_implicit_task_end: task_id=[[TASK_ID]]
 // CHECK-SAME: memory_addr=[[NULL]], memory_size=0, result=0
+// clang-format on
diff --git a/openmp/runtime/test/ompt/tasks/task_types.c b/openmp/runtime/test/ompt/tasks/task_types.c
index c8121de6635ab..37b1b9758edf2 100644
--- a/openmp/runtime/test/ompt/tasks/task_types.c
+++ b/openmp/runtime/test/ompt/tasks/task_types.c
@@ -1,11 +1,13 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #include "callback.h"
 #include <omp.h>
 #include <math.h>
 
 int main() {
-  //initialize the OpenMP runtime
+  // initialize the OpenMP runtime
   omp_get_num_threads();
 
   // initial task
@@ -85,35 +87,36 @@ int main() {
     // TODO: merged task
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-9]+}}
-  // CHECK-SAME: task_id=[[INITIAL_TASK_ID:[0-9]+]], actual_parallelism=1, index=1, flags=1 
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-f]+}}
+  // CHECK-SAME: task_id=[[INITIAL_TASK_ID:[0-f]+]], actual_parallelism=1, index=1, flags=1 
 
   // CHECK-NOT: 0: parallel_data initially not null
 
   // initial task
-  // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id={{[0-9]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id={{[0-f]+}}
   // CHECK-SAME: task_id=[[INITIAL_TASK_ID]], exit_frame=[[NULL]]
   // CHECK-SAME: reenter_frame=[[NULL]]
   // CHECK-SAME: task_type=ompt_task_initial=1, thread_num=0
 
   // implicit task
-  // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id={{[0-9]+}}
-  // CHECK-SAME: task_id={{[0-9]+}}, exit_frame={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id={{[0-f]+}}
+  // CHECK-SAME: task_id={{[0-f]+}}, exit_frame={{(0x)?[0-f]+}}
   // CHECK-SAME: reenter_frame=[[NULL]]
   // CHECK-SAME: task_type=ompt_task_implicit|ompt_task_undeferred=134217730
   // CHECK-SAME: thread_num=0
 
   // explicit task
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}
   // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}}
   // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}}
-  // CHECK-SAME: new_task_id=[[EXPLICIT_TASK_ID:[0-9]+]]
+  // CHECK-SAME: new_task_id=[[EXPLICIT_TASK_ID:[0-f]+]]
   // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}
   // CHECK-SAME: task_type=ompt_task_explicit=4
   // CHECK-SAME: has_dependences=no
@@ -121,16 +124,16 @@ int main() {
   // CHECK: [[THREAD_ID_1:[0-9]+]]: ompt_event_task_schedule:
   // CHECK-SAME: second_task_id=[[EXPLICIT_TASK_ID]]
 
-  // CHECK: [[THREAD_ID_1]]: task level 0: parallel_id=[[PARALLEL_ID:[0-9]+]]
+  // CHECK: [[THREAD_ID_1]]: task level 0: parallel_id=[[PARALLEL_ID:[0-f]+]]
   // CHECK-SAME: task_id=[[EXPLICIT_TASK_ID]], exit_frame={{(0x)?[0-f]+}}
   // CHECK-SAME: reenter_frame=[[NULL]], task_type=ompt_task_explicit=4
   // CHECK-SAME: thread_num={{[01]}}
 
   // explicit task with undeferred
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}
   // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}}
   // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}}
-  // CHECK-SAME: new_task_id=[[EXPLICIT_UNDEFERRED_TASK_ID:[0-9]+]]
+  // CHECK-SAME: new_task_id=[[EXPLICIT_UNDEFERRED_TASK_ID:[0-f]+]]
   // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}
   // CHECK-SAME: task_type=ompt_task_explicit|ompt_task_undeferred=134217732
   // CHECK-SAME: has_dependences=no
@@ -145,10 +148,10 @@ int main() {
   // CHECK-SAME: thread_num={{[01]}}
 
   // explicit task with untied
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}
   // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}}
   // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}}
-  // CHECK-SAME: new_task_id=[[EXPLICIT_UNTIED_TASK_ID:[0-9]+]]
+  // CHECK-SAME: new_task_id=[[EXPLICIT_UNTIED_TASK_ID:[0-f]+]]
   // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}
   // CHECK-SAME: task_type=ompt_task_explicit|ompt_task_untied=268435460
   // CHECK-SAME: has_dependences=no
@@ -179,10 +182,10 @@ int main() {
   // CHECK-SAME: thread_num={{[01]}}
 
   // explicit task with final
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}
   // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}}
   // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}}
-  // CHECK-SAME: new_task_id=[[EXPLICIT_FINAL_TASK_ID:[0-9]+]]
+  // CHECK-SAME: new_task_id=[[EXPLICIT_FINAL_TASK_ID:[0-f]+]]
   // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}
   // CHECK-SAME: task_type=ompt_task_explicit|ompt_task_final=536870916
   // CHECK-SAME: has_dependences=no
@@ -197,10 +200,10 @@ int main() {
   // CHECK-SAME: thread_num={{[01]}}
 
   // nested explicit task with final and undeferred
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}
   // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}}
   // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}}
-  // CHECK-SAME: new_task_id=[[NESTED_FINAL_UNDEFERRED_TASK_ID:[0-9]+]]
+  // CHECK-SAME: new_task_id=[[NESTED_FINAL_UNDEFERRED_TASK_ID:[0-f]+]]
   // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}
   // CHECK-SAME: task_type=ompt_task_explicit|ompt_task_undeferred
   // CHECK-SAME:|ompt_task_final=671088644
@@ -215,6 +218,7 @@ int main() {
   // CHECK-SAME: task_type=ompt_task_explicit|ompt_task_undeferred
   // CHECK-SAME:|ompt_task_final=671088644
   // CHECK-SAME: thread_num={{[01]}}
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/tasks/task_types_serialized.c b/openmp/runtime/test/ompt/tasks/task_types_serialized.c
index b7b4d4d0e80f3..fa927f2fe72b4 100644
--- a/openmp/runtime/test/ompt/tasks/task_types_serialized.c
+++ b/openmp/runtime/test/ompt/tasks/task_types_serialized.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -7,80 +9,81 @@
 __attribute__ ((noinline)) // workaround for bug in icc
 void print_task_type(int id)
 {
-  #pragma omp critical
+#pragma omp critical
   {
     int task_type;
-    char buffer[2048];
-    ompt_get_task_info(0, &task_type, NULL, NULL, NULL, NULL);
-    format_task_type(task_type, buffer);
-    printf("%" PRIu64 ": id=%d task_type=%s=%d\n", ompt_get_thread_data()->value, id, buffer, task_type);
-  }
-};
+char buffer[2048];
+ompt_get_task_info(0, &task_type, NULL, NULL, NULL, NULL);
+format_task_type(task_type, buffer);
+printf("%" PRIu64 ": id=%d task_type=%s=%d\n", ompt_get_thread_data()->value,
+       id, buffer, task_type);
+}
+}
+;
 
-int main()
-{
-  //initial task
+int main() {
+  // initial task
   print_task_type(0);
 
   int x;
-  //implicit task
-  #pragma omp parallel num_threads(1)
+// implicit task
+#pragma omp parallel num_threads(1)
   {
     print_task_type(1);
     x++;
   }
 
-  #pragma omp parallel num_threads(1)
-  #pragma omp master
+#pragma omp parallel num_threads(1)
+#pragma omp master
   {
-    //explicit task
-    #pragma omp task
+// explicit task
+#pragma omp task
     {
       print_task_type(2);
       x++;
     }
 
-    //explicit task with undeferred
-    #pragma omp task if(0)
+// explicit task with undeferred
+#pragma omp task if (0)
     {
       print_task_type(3);
       x++;
     }
 
-    //explicit task with untied
-    #pragma omp task untied
+// explicit task with untied
+#pragma omp task untied
     {
       print_task_type(4);
       x++;
     }
 
-    //explicit task with final
-    #pragma omp task final(1)
+// explicit task with final
+#pragma omp task final(1)
     {
       print_task_type(5);
       x++;
-      //nested explicit task with final and undeferred
-      #pragma omp task
+// nested explicit task with final and undeferred
+#pragma omp task
       {
         print_task_type(6);
         x++;
       }
     }
 
-/*
-    //TODO:not working
-    //explicit task with mergeable
-    #pragma omp task mergeable
-    {
-      print_task_type(7);
-      x++;
-    }
-*/
-
-    //TODO: merged task
+    /*
+        //TODO:not working
+        //explicit task with mergeable
+        #pragma omp task mergeable
+        {
+          print_task_type(7);
+          x++;
+        }
+    */
+
+    // TODO: merged task
   }
 
-
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
@@ -88,27 +91,28 @@ int main()
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
   
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-9]+}}, task_id={{[0-9]+}}, actual_parallelism=1, index=1, flags=1
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id={{[0-f]+}}, task_id={{[0-f]+}}, actual_parallelism=1, index=1, flags=1
   // CHECK: {{^}}[[MASTER_ID]]: id=0 task_type=ompt_task_initial=1
   // CHECK: {{^}}[[MASTER_ID]]: id=1 task_type=ompt_task_implicit|ompt_task_undeferred=134217730
 
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred=134217732, has_dependences=no
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred=134217732, has_dependences=no
   // CHECK: {{^[0-9]+}}: id=2 task_type=ompt_task_explicit|ompt_task_undeferred=134217732
 
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred=134217732, has_dependences=no
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred=134217732, has_dependences=no
   // CHECK: {{^[0-9]+}}: id=3 task_type=ompt_task_explicit|ompt_task_undeferred=134217732
 
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred|ompt_task_untied=402653188, has_dependences=no
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred|ompt_task_untied=402653188, has_dependences=no
   // CHECK: {{^[0-9]+}}: id=4 task_type=ompt_task_explicit|ompt_task_undeferred|ompt_task_untied=402653188
 
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred|ompt_task_final=671088644, has_dependences=no
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred|ompt_task_final=671088644, has_dependences=no
   // CHECK: {{^[0-9]+}}: id=5 task_type=ompt_task_explicit|ompt_task_undeferred|ompt_task_final=671088644
 
-  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred|ompt_task_final=671088644, has_dependences=no
+  // CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred|ompt_task_final=671088644, has_dependences=no
   // CHECK: {{^[0-9]+}}: id=6 task_type=ompt_task_explicit|ompt_task_undeferred|ompt_task_final=671088644
 
-  // ___CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-9]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred=134217732, has_dependences=no
+  // ___CHECK: {{^[0-9]+}}: ompt_event_task_create: parent_task_id={{[0-f]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit|ompt_task_undeferred=134217732, has_dependences=no
   // ___CHECK: {{^[0-9]+}}: id=7 task_type=ompt_task_explicit|ompt_task_undeferred=134217732
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/tasks/taskloop.c b/openmp/runtime/test/ompt/tasks/taskloop.c
index 9b1e552857d63..00bc85a6aa9f5 100644
--- a/openmp/runtime/test/ompt/tasks/taskloop.c
+++ b/openmp/runtime/test/ompt/tasks/taskloop.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // RUN: %libomp-compile-and-run | FileCheck --check-prefix=TASKS %s
 // REQUIRES: ompt
@@ -6,6 +7,7 @@
 // UNSUPPORTED: gcc-4, gcc-5, icc-16
 // GCC 6 has support for taskloops, but at least 6.3.0 is crashing on this test
 // UNSUPPORTED: gcc-6
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -23,11 +25,12 @@ int main() {
       x++;
     }
   }
+  // clang-format off
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
-  // CHECK-SAME: parent_task_id={{[0-9]+}}
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]]
+  // CHECK-SAME: parent_task_id={{[0-f]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]]
   // CHECK-SAME: requested_team_size=2
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
@@ -64,17 +67,18 @@ int main() {
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
 
-  // TASKS: ompt_event_initial_task_begin:{{.*}} task_id={{[0-9]+}}
+  // TASKS: ompt_event_initial_task_begin:{{.*}} task_id={{[0-f]+}}
   // TASKS: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_taskloop_begin:
   // TASKS: ompt_event_task_create:{{.*}} new_task_id=[[TASK_ID1:[0-9]+]]
   // TASKS-SAME: task_type=ompt_task_explicit
   // TASKS-DAG: ompt_event_task_create:{{.*}} new_task_id=[[TASK_ID2:[0-9]+]]
   // Schedule events:
-  // TASKS-DAG: {{^.*}}first_task_id={{[0-9]+}}, second_task_id=[[TASK_ID1]]
-  // TASKS-DAG: {{^.*}}first_task_id=[[TASK_ID1]], second_task_id={{[0-9]+}}
-  // TASKS-DAG: {{^.*}}first_task_id={{[0-9]+}}, second_task_id=[[TASK_ID2]]
-  // TASKS-DAG: {{^.*}}first_task_id=[[TASK_ID2]], second_task_id={{[0-9]+}}
+  // TASKS-DAG: {{^.*}}first_task_id={{[0-f]+}}, second_task_id=[[TASK_ID1]]
+  // TASKS-DAG: {{^.*}}first_task_id=[[TASK_ID1]], second_task_id={{[0-f]+}}
+  // TASKS-DAG: {{^.*}}first_task_id={{[0-f]+}}, second_task_id=[[TASK_ID2]]
+  // TASKS-DAG: {{^.*}}first_task_id=[[TASK_ID2]], second_task_id={{[0-f]+}}
   // TASKS-NOT: ompt_event_task_schedule
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/tasks/taskloop_dispatch.c b/openmp/runtime/test/ompt/tasks/taskloop_dispatch.c
index 92650caa788e1..b69ff5f45f778 100644
--- a/openmp/runtime/test/ompt/tasks/taskloop_dispatch.c
+++ b/openmp/runtime/test/ompt/tasks/taskloop_dispatch.c
@@ -1,6 +1,8 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gnu, intel-16.0
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -19,11 +21,12 @@ int main() {
       delay(1000);
     }
   }
+  // clang-format off
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin:
-  // CHECK-SAME: parent_task_id={{[0-9]+}}
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]]
+  // CHECK-SAME: parent_task_id={{[0-f]+}}
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]]
   // CHECK-SAME: requested_team_size=2
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]]
@@ -48,6 +51,7 @@ int main() {
   // CHECK-DAG: {{.*}}: ompt_event_taskloop_chunk_begin:{{.*}}task_id=[[TASK_ID1]]{{.*}}chunk_iterations=4
   // CHECK-DAG: {{.*}}: ompt_event_taskloop_chunk_begin:{{.*}}task_id=[[TASK_ID2]]{{.*}}chunk_iterations=4
   // CHECK-DAG: {{.*}}: ompt_event_taskloop_chunk_begin:{{.*}}task_id=[[TASK_ID3]]{{.*}}chunk_iterations=4
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/tasks/taskwait-depend.c b/openmp/runtime/test/ompt/tasks/taskwait-depend.c
index 284e18824705f..ecadf49be63ab 100644
--- a/openmp/runtime/test/ompt/tasks/taskwait-depend.c
+++ b/openmp/runtime/test/ompt/tasks/taskwait-depend.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 
@@ -9,6 +10,7 @@
 
 // support for taskwait with depend clause introduced in clang-14
 // UNSUPPORTED: clang-5, clang-6, clang-6, clang-8, clang-9, clang-10, clang-11, clang-12, clang-13
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
@@ -25,7 +27,7 @@ int main() {
 #pragma omp task depend(out : x)
       { x++; }
       print_fuzzy_address(1);
-      #pragma omp taskwait depend(in: x)
+#pragma omp taskwait depend(in : x)
       print_fuzzy_address(2);
     }
   }
@@ -33,6 +35,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dependences'
@@ -44,8 +47,8 @@ int main() {
 // CHECK-NOT: 0: new_task_data initially not null
 
 // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
+// CHECK-SAME: task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
 
 // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]],
 // CHECK-SAME: task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT:(0x)?[0-f]+]],
@@ -54,7 +57,7 @@ int main() {
 // CHECK: {{^}}[[MASTER_ID]]: address of x: [[ADDRX:(0x)?[0-f]+]]
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[EXIT]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
 // CHECK-SAME: new_task_id=[[FIRST_TASK:[0-f]+]],
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
@@ -67,7 +70,7 @@ int main() {
 // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
 
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create:
-// CHECK-SAME: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[EXIT]],
+// CHECK-SAME: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[EXIT]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
 // CHECK-SAME: new_task_id=[[SECOND_TASK:[0-f]+]],
 // CHECK-SAME: codeptr_ra=[[RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}},
@@ -81,3 +84,4 @@ int main() {
 // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_end: task_id=[[SECOND_TASK]]
 
 // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[RETURN_ADDRESS]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/tasks/taskyield.c b/openmp/runtime/test/ompt/tasks/taskyield.c
index 40bd6bfe65965..3fc4fcaa8c7ca 100644
--- a/openmp/runtime/test/ompt/tasks/taskyield.c
+++ b/openmp/runtime/test/ompt/tasks/taskyield.c
@@ -1,37 +1,38 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // Current GOMP interface implements taskyield as stub
 // XFAIL: gcc
+// clang-format on
 
 #include "callback.h"
-#include <omp.h>   
+#include <omp.h>
 #include <unistd.h>
 
-int main()
-{
-  int condition=0, x=0;
-  #pragma omp parallel num_threads(2)
+int main() {
+  int condition = 0, x = 0;
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp master
+#pragma omp master
     {
-        #pragma omp task shared(condition)
-        {
-          OMPT_SIGNAL(condition);
-          OMPT_WAIT(condition,2);
-        }
-        OMPT_WAIT(condition,1);
-        #pragma omp task shared(x)
-        {
-          x++;
-        }
-        printf("%" PRIu64 ": before yield\n", ompt_get_thread_data()->value);
-        #pragma omp taskyield
-        printf("%" PRIu64 ": after yield\n", ompt_get_thread_data()->value);
+#pragma omp task shared(condition)
+      {
         OMPT_SIGNAL(condition);
+        OMPT_WAIT(condition, 2);
+      }
+      OMPT_WAIT(condition, 1);
+#pragma omp task shared(x)
+      {
+        x++;
+      }
+      printf("%" PRIu64 ": before yield\n", ompt_get_thread_data()->value);
+#pragma omp taskyield
+      printf("%" PRIu64 ": after yield\n", ompt_get_thread_data()->value);
+      OMPT_SIGNAL(condition);
     }
   }
 
-
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_schedule'
@@ -43,20 +44,17 @@ int main()
   // make sure initial data pointers are null
   // CHECK-NOT: 0: new_task_data initially not null
   
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID:[0-9]+]], team_size={{[0-9]+}}, thread_num={{[0-9]+}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID:[0-f]+]], team_size={{[0-9]+}}, thread_num={{[0-9]+}}
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id={{[0-9]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[WORKER_TASK:[0-9]+]], codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit=4, has_dependences=no
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id={{[0-9]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[MAIN_TASK:[0-9]+]], codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit=4, has_dependences=no
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id={{[0-f]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[WORKER_TASK:[0-9]+]], codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit=4, has_dependences=no
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id={{[0-f]+}}, parent_task_frame.exit={{(0x)?[0-f]+}}, parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[MAIN_TASK:[0-9]+]], codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit=4, has_dependences=no
 
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[IMPLICIT_TASK_ID]], second_task_id=[[MAIN_TASK]], prior_task_status=ompt_task_yield=2
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_schedule: first_task_id=[[MAIN_TASK]], second_task_id=[[IMPLICIT_TASK_ID]], prior_task_status=ompt_task_complete=1
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_task_schedule: first_task_id={{[0-9]+}}, second_task_id=[[WORKER_TASK]], prior_task_status=ompt_task_switch=7
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_task_schedule: first_task_id=[[WORKER_TASK]], second_task_id={{[0-9]+}}, prior_task_status=ompt_task_complete=1
-
-
-
-
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_task_schedule: first_task_id={{[0-f]+}}, second_task_id=[[WORKER_TASK]], prior_task_status=ompt_task_switch=7
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_task_schedule: first_task_id=[[WORKER_TASK]], second_task_id={{[0-f]+}}, prior_task_status=ompt_task_complete=1
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/tasks/untied_task.c b/openmp/runtime/test/ompt/tasks/untied_task.c
index 2cf5c3a00ad7d..08683a50edf68 100644
--- a/openmp/runtime/test/ompt/tasks/untied_task.c
+++ b/openmp/runtime/test/ompt/tasks/untied_task.c
@@ -1,31 +1,32 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 #define TEST_NEED_PRINT_FRAME_FROM_OUTLINED_FN
 #include "callback.h"
-#include <omp.h> 
+#include <omp.h>
 
-int main()
-{
-  int condition=0;
+int main() {
+  int condition = 0;
   omp_set_nested(0);
   print_frame(0);
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
     print_frame_from_outlined_fn(1);
     print_ids(0);
     print_ids(1);
     print_frame(0);
-    #pragma omp master
+#pragma omp master
     {
       print_ids(0);
-      #pragma omp task untied shared(condition)
+#pragma omp task untied shared(condition)
       {
         OMPT_SIGNAL(condition);
         print_frame(1);
         print_ids(0);
         print_ids(1);
         print_ids(2);
-        #pragma omp task if(0)
+#pragma omp task if (0)
         {
           print_ids(0);
           print_ids(1);
@@ -35,10 +36,10 @@ int main()
         print_ids(1);
         print_ids(2);
       }
-      OMPT_WAIT(condition,1);
+      OMPT_WAIT(condition, 1);
       print_ids(0);
     }
-    #pragma omp barrier
+#pragma omp barrier
     print_ids(0);
   }
 
@@ -60,16 +61,16 @@ int main()
   // CHECK-NOT: 0: new_task_data initially not null
   
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: __builtin_frame_address(0)=[[MAIN_REENTER:(0x)?[0-f]+]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=2, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=2, codeptr_ra={{(0x)?[0-f]+}}, invoker=[[PARALLEL_INVOKER:[0-9]+]]
   // nested parallel masters
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address({{.}})=[[EXIT:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-9]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[MASTER_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID:[0-f]+]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
   // CHECK: {{^}}[[MASTER_ID]]: __builtin_frame_address(0)=[[REENTER:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
   // <- ompt_event_task_create would be expected here
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[TASK_ID:[0-9]+]], codeptr_ra=[[TASK_FUNCTION:(0x)?[0-f]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_task_create: parent_task_id=[[IMPLICIT_TASK_ID]], parent_task_frame.exit=[[EXIT]], parent_task_frame.reenter={{(0x)?[0-f]+}}, new_task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[TASK_FUNCTION:(0x)?[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
   // explicit barrier after master
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_explicit_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
@@ -78,10 +79,10 @@ int main()
   // implicit barrier parallel
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: __builtin_frame_address({{.}})=[[EXIT:(0x)?[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[EXIT]], reenter_frame=[[NULL]]
   // CHECK: {{^}}[[THREAD_ID]]: task level 1: parallel_id=[[IMPLICIT_PARALLEL_ID]], task_id=[[PARENT_TASK_ID]], exit_frame=[[NULL]], reenter_frame={{(0x)?[0-f]+}}
@@ -99,8 +100,8 @@ int main()
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_explicit_end: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[THREAD_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], exit_frame=[[NULL]], reenter_frame=[[NULL]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_barrier_implicit_parallel_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/teams/distribute_dispatch.c b/openmp/runtime/test/ompt/teams/distribute_dispatch.c
index 1dcede84f28ec..179e65d9881f6 100644
--- a/openmp/runtime/test/ompt/teams/distribute_dispatch.c
+++ b/openmp/runtime/test/ompt/teams/distribute_dispatch.c
@@ -1,9 +1,11 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 
 /// GCC lowering of distribute results in calls to 
 /// omp_get_num_teams/omp_get_team_num rather than region calls
 // UNSUPPORTED: gcc
+// clang-format on
 #include "callback.h"
 
 #define WORK_SIZE 64
@@ -12,11 +14,13 @@ int main() {
   int i;
 #pragma omp teams num_teams(4) thread_limit(1)
 #pragma omp distribute dist_schedule(static, WORK_SIZE / 4)
-  for (i = 0; i < WORK_SIZE; i++) {}
+  for (i = 0; i < WORK_SIZE; i++) {
+  }
 
   return 0;
 }
 
+// clang-format off
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_work'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dispatch'
 
@@ -49,3 +53,4 @@ int main() {
 // CHECK: {{^}}[[THREAD_ID3]]: ompt_event_distribute_chunk_begin:
 // CHECK-SAME: parallel_id=[[PARALLEL_ID3]], task_id=[[TASK_ID3]]
 // CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations=16
+// clang-format on
diff --git a/openmp/runtime/test/ompt/teams/parallel_team.c b/openmp/runtime/test/ompt/teams/parallel_team.c
index 255d03b4c5c61..f48a61ac5cfaf 100644
--- a/openmp/runtime/test/ompt/teams/parallel_team.c
+++ b/openmp/runtime/test/ompt/teams/parallel_team.c
@@ -1,8 +1,10 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt, multicpu
 // UNSUPPORTED: gcc
 // Compilation fails for icc
 // XFAIL: icc
+// clang-format on
 #include "callback.h"
 
 int main() {
@@ -12,6 +14,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
 // CHECK-NOT: 0: parallel_data initially not null
@@ -19,7 +22,7 @@ int main() {
 // CHECK-NOT: 0: thread_data initially not null
 
 // CHECK: {{^}}[[MASTER:[0-9]+]]: ompt_event_initial_task_begin:
-// CHECK-SAME: task_id=[[INIT_TASK:[0-9]+]], {{.*}}, index=1
+// CHECK-SAME: task_id=[[INIT_TASK:[0-f]+]], {{.*}}, index=1
 
 // CHECK: {{^}}[[MASTER]]: ompt_event_teams_begin:
 // CHECK-SAME: parent_task_id=[[INIT_TASK]]
@@ -31,33 +34,33 @@ int main() {
 //
 // initial task in the teams construct
 // CHECK: {{^}}[[MASTER]]: ompt_event_initial_task_begin:
-// CHECK-SAME: task_id=[[INIT_TASK_0:[0-9]+]], actual_parallelism=1, index=0
+// CHECK-SAME: task_id=[[INIT_TASK_0:[0-f]+]], actual_parallelism=1, index=0
 
 // parallel region forked by runtime
 // CHECK: {{^}}[[MASTER]]: ompt_event_parallel_begin:
 // CHECK-SAME: {{.*}} parent_task_id=[[INIT_TASK_0]]
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_0:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_0:[0-f]+]]
 // CHECK: {{^}}[[MASTER]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[IMPL_TASK_0:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[IMPL_TASK_0:[0-f]+]]
 
 // user parallel region
 // CHECK: {{^}}[[MASTER]]: ompt_event_parallel_begin:
 // CHECK-SAME: {{.*}} parent_task_id=[[IMPL_TASK_0]]
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_00:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_00:[0-f]+]]
 // CHECK-SAME: {{.*}} requested_team_size=2
 // CHECK: {{^}}[[MASTER]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_00]], task_id=[[IMPL_TASK_00:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_00]], task_id=[[IMPL_TASK_00:[0-f]+]]
 // CHECK-SAME: {{.*}} team_size=2, thread_num=0
 //
 // barrier event is here
 //
 // CHECK: {{^}}[[MASTER]]: ompt_event_implicit_task_end:
-// CHECK-SAME: {{.*}} parallel_id={{[0-9]+}}, task_id=[[IMPL_TASK_00]]
+// CHECK-SAME: {{.*}} parallel_id={{[0-f]+}}, task_id=[[IMPL_TASK_00]]
 // CHECK: {{^}}[[MASTER]]: ompt_event_parallel_end:
 // CHECK-SAME: {{.*}} parallel_id=[[PAR_00]], task_id=[[IMPL_TASK_0]]
 
 // CHECK: {{^}}[[MASTER]]: ompt_event_implicit_task_end:
-// CHECK-SAME: {{.*}} parallel_id={{[0-9]+}}, task_id=[[IMPL_TASK_0]]
+// CHECK-SAME: {{.*}} parallel_id={{[0-f]+}}, task_id=[[IMPL_TASK_0]]
 // CHECK: {{^}}[[MASTER]]: ompt_event_parallel_end:
 // CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[INIT_TASK_0]]
 
@@ -74,10 +77,11 @@ int main() {
 // team 0/thread 1
 //
 // CHECK: {{^}}[[WORKER:[0-9]+]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_00]], task_id=[[IMPL_TASK_01:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_00]], task_id=[[IMPL_TASK_01:[0-f]+]]
 // CHECK-SAME: {{.*}} team_size=2, thread_num=1
 //
 // barrier event is here
 //
 // CHECK: {{^}}[[WORKER]]: ompt_event_implicit_task_end:
-// CHECK-SAME: {{.*}} parallel_id={{[0-9]+}}, task_id=[[IMPL_TASK_01]]
+// CHECK-SAME: {{.*}} parallel_id={{[0-f]+}}, task_id=[[IMPL_TASK_01]]
+// clang-format on
diff --git a/openmp/runtime/test/ompt/teams/serial_teams.c b/openmp/runtime/test/ompt/teams/serial_teams.c
index d436facbbe5ac..cbf60e8f672b3 100644
--- a/openmp/runtime/test/ompt/teams/serial_teams.c
+++ b/openmp/runtime/test/ompt/teams/serial_teams.c
@@ -1,8 +1,10 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt, multicpu
 // UNSUPPORTED: gcc
 // Compilation fails for icc
 // XFAIL: icc
+// clang-format on
 #include "callback.h"
 
 int main() {
@@ -12,6 +14,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
 // CHECK-NOT: 0: parallel_data initially not null
@@ -19,7 +22,7 @@ int main() {
 // CHECK-NOT: 0: thread_data initially not null
 
 // CHECK: {{^}}[[MASTER_0:[0-9]+]]: ompt_event_initial_task_begin:
-// CHECK-SAME: task_id=[[INIT_TASK:[0-9]+]], {{.*}}, index=1
+// CHECK-SAME: task_id=[[INIT_TASK:[0-f]+]], {{.*}}, index=1
 
 // CHECK: {{^}}[[MASTER_0]]: ompt_event_teams_begin:
 // CHECK-SAME: parent_task_id=[[INIT_TASK]]
@@ -31,24 +34,24 @@ int main() {
 //
 // initial task in the teams construct
 // CHECK: {{^}}[[MASTER_0]]: ompt_event_initial_task_begin:
-// CHECK-SAME: task_id=[[INIT_TASK_0:[0-9]+]], actual_parallelism=2, index=0
+// CHECK-SAME: task_id=[[INIT_TASK_0:[0-f]+]], actual_parallelism=2, index=0
 
 // parallel region forked by runtime
 // CHECK: {{^}}[[MASTER_0]]: ompt_event_parallel_begin:
 // CHECK-SAME: {{.*}} parent_task_id=[[INIT_TASK_0]]
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_0:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_0:[0-f]+]]
 // CHECK: {{^}}[[MASTER_0]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[IMPL_TASK_0:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[IMPL_TASK_0:[0-f]+]]
 
 // user parallel region
 // CHECK: {{^}}[[MASTER_0]]: ompt_event_parallel_begin:
 // CHECK-SAME: {{.*}} parent_task_id=[[IMPL_TASK_0]]
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_00:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_00:[0-f]+]]
 // CHECK: {{^}}[[MASTER_0]]: ompt_event_parallel_end:
 // CHECK-SAME: {{.*}} parallel_id=[[PAR_00]], task_id=[[IMPL_TASK_0]]
 
 // CHECK: {{^}}[[MASTER_0]]: ompt_event_implicit_task_end:
-// CHECK-SAME: {{.*}} parallel_id={{[0-9]+}}, task_id=[[IMPL_TASK_0]]
+// CHECK-SAME: {{.*}} parallel_id={{[0-f]+}}, task_id=[[IMPL_TASK_0]]
 // CHECK: {{^}}[[MASTER_0]]: ompt_event_parallel_end:
 // CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[INIT_TASK_0]]
 
@@ -66,26 +69,27 @@ int main() {
 //
 // initial task in the teams construct
 // CHECK: {{^}}[[MASTER_1:[0-9]+]]: ompt_event_initial_task_begin:
-// CHECK-SAME: task_id=[[INIT_TASK_1:[0-9]+]], actual_parallelism=2, index=1
+// CHECK-SAME: task_id=[[INIT_TASK_1:[0-f]+]], actual_parallelism=2, index=1
 
 // parallel region forked by runtime
 // CHECK: {{^}}[[MASTER_1]]: ompt_event_parallel_begin:
 // CHECK-SAME: {{.*}} parent_task_id=[[INIT_TASK_1]]
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_ID_1:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_ID_1:[0-f]+]]
 // CHECK: {{^}}[[MASTER_1]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_ID_1]], task_id=[[IMPL_TASK_1:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_ID_1]], task_id=[[IMPL_TASK_1:[0-f]+]]
 
 // user parallel region
 // CHECK: {{^}}[[MASTER_1]]: ompt_event_parallel_begin:
 // CHECK-SAME: {{.*}} parent_task_id=[[IMPL_TASK_1]]
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_ID_11:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_ID_11:[0-f]+]]
 // CHECK: {{^}}[[MASTER_1]]: ompt_event_parallel_end:
 // CHECK-SAME: {{.*}} parallel_id=[[PAR_ID_11]], task_id=[[IMPL_TASK_1]]
 
 // CHECK: {{^}}[[MASTER_1]]: ompt_event_implicit_task_end:
-// CHECK-SAME: {{.*}} parallel_id={{[0-9]+}}, task_id=[[IMPL_TASK_1]]
+// CHECK-SAME: {{.*}} parallel_id={{[0-f]+}}, task_id=[[IMPL_TASK_1]]
 // CHECK: {{^}}[[MASTER_1]]: ompt_event_parallel_end:
 // CHECK-SAME: {{.*}} parallel_id=[[PAR_ID_1]], task_id=[[INIT_TASK_1]]
 
 // CHECK: {{^}}[[MASTER_1]]: ompt_event_initial_task_end:
 // CHECK-SAME: task_id=[[INIT_TASK_1]], actual_parallelism=0, index=1
+// clang-format on
diff --git a/openmp/runtime/test/ompt/teams/serialized.c b/openmp/runtime/test/ompt/teams/serialized.c
index 4aeb406045dfd..e984fe3d5a127 100644
--- a/openmp/runtime/test/ompt/teams/serialized.c
+++ b/openmp/runtime/test/ompt/teams/serialized.c
@@ -1,8 +1,10 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc
 // Compilation fails for icc
 // XFAIL: icc
+// clang-format on
 
 #include "callback.h"
 
@@ -13,6 +15,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
 // CHECK-NOT: 0: parallel_data initially not null
@@ -20,7 +23,7 @@ int main() {
 // CHECK-NOT: 0: thread_data initially not null
 
 // CHECK: {{^}}[[MASTER:[0-9]+]]: ompt_event_initial_task_begin:
-// CHECK-SAME: task_id=[[INIT_TASK:[0-9]+]], {{.*}}, index=1
+// CHECK-SAME: task_id=[[INIT_TASK:[0-f]+]], {{.*}}, index=1
 
 // CHECK: {{^}}[[MASTER]]: ompt_event_teams_begin:
 // CHECK-SAME: parent_task_id=[[INIT_TASK]]
@@ -29,25 +32,25 @@ int main() {
 
 // initial task in the teams construct starts
 // CHECK: {{^}}[[MASTER]]: ompt_event_initial_task_begin:
-// CHECK-SAME: task_id=[[INIT_TASK_0:[0-9]+]], actual_parallelism=1, index=0
+// CHECK-SAME: task_id=[[INIT_TASK_0:[0-f]+]], actual_parallelism=1, index=0
 
 // parallel region forked by runtime
 // CHECK: {{^}}[[MASTER]]: ompt_event_parallel_begin:
 // CHECK-SAME: {{.*}} parent_task_id=[[INIT_TASK_0]]
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_0:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_0:[0-f]+]]
 // CHECK: {{^}}[[MASTER]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[IMPL_TASK_0:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[IMPL_TASK_0:[0-f]+]]
 
 // user parallel region
 // CHECK: {{^}}[[MASTER]]: ompt_event_parallel_begin:
 // CHECK-SAME: {{.*}} parent_task_id=[[IMPL_TASK_0]]
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_00:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_00:[0-f]+]]
 // CHECK-SAME: {{.*}} requested_team_size=1
 // CHECK: {{^}}[[MASTER]]: ompt_event_implicit_task_begin:
-// CHECK-SAME: {{.*}} parallel_id=[[PAR_00]], task_id=[[IMPL_TASK_00:[0-9]+]]
+// CHECK-SAME: {{.*}} parallel_id=[[PAR_00]], task_id=[[IMPL_TASK_00:[0-f]+]]
 // CHECK-SAME: {{.*}} team_size=1, thread_num=0
 // CHECK: {{^}}[[MASTER]]: ompt_event_implicit_task_end:
-// CHECK-SAME: {{.*}} parallel_id={{[0-9]+}}, task_id=[[IMPL_TASK_00]]
+// CHECK-SAME: {{.*}} parallel_id={{[0-f]+}}, task_id=[[IMPL_TASK_00]]
 // CHECK: {{^}}[[MASTER]]: ompt_event_parallel_end:
 // CHECK-SAME: {{.*}} parallel_id=[[PAR_00]], task_id=[[IMPL_TASK_0]]
 
@@ -63,3 +66,4 @@ int main() {
 
 // CHECK: {{^}}[[MASTER]]: ompt_event_initial_task_end:
 // CHECK-SAME: task_id=[[INIT_TASK]], {{.*}}, index=1
+// clang-format on
diff --git a/openmp/runtime/test/ompt/teams/team.c b/openmp/runtime/test/ompt/teams/team.c
index 55b57e79f5913..81f1e11f4f6be 100644
--- a/openmp/runtime/test/ompt/teams/team.c
+++ b/openmp/runtime/test/ompt/teams/team.c
@@ -1,8 +1,10 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: gcc
 // Compilation fails for icc
 // XFAIL: icc
+// clang-format on
 #include "callback.h"
 
 int main() {
@@ -11,6 +13,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
 // CHECK-NOT: 0: parallel_data initially not null
@@ -36,7 +39,7 @@ int main() {
 // CHECK: {{^}}[[MASTER]]: ompt_event_implicit_task_begin:
 // CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[IMPL_TASK_0:[0-9]+]]
 // CHECK: {{^}}[[MASTER]]: ompt_event_implicit_task_end:
-// CHECK-SAME: {{.*}} parallel_id={{[0-9]+}}, task_id=[[IMPL_TASK_0]]
+// CHECK-SAME: {{.*}} parallel_id={{[0-f]+}}, task_id=[[IMPL_TASK_0]]
 // CHECK: {{^}}[[MASTER]]: ompt_event_parallel_end:
 // CHECK-SAME: {{.*}} parallel_id=[[PAR_0]], task_id=[[INIT_TASK_0]]
 
@@ -49,3 +52,4 @@ int main() {
 
 // CHECK: {{^}}[[MASTER]]: ompt_event_initial_task_end:
 // CHECK-SAME: task_id=[[INIT_TASK]], {{.*}}, index=1
+// clang-format on
diff --git a/openmp/runtime/test/ompt/worksharing/for/auto.c b/openmp/runtime/test/ompt/worksharing/for/auto.c
index 3ce66d511ab3c..eda7211a62821 100644
--- a/openmp/runtime/test/ompt/worksharing/for/auto.c
+++ b/openmp/runtime/test/ompt/worksharing/for/auto.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base.h
 // REQUIRES: ompt
 // GCC doesn't call runtime for auto = static schedule
 // XFAIL: gcc
+// clang-format on
 
 #define SCHEDULE auto
 // The runtime uses guided schedule for auto,
diff --git a/openmp/runtime/test/ompt/worksharing/for/auto_serialized.c b/openmp/runtime/test/ompt/worksharing/for/auto_serialized.c
index 400fcc8f3a78f..3c6c3faf79158 100644
--- a/openmp/runtime/test/ompt/worksharing/for/auto_serialized.c
+++ b/openmp/runtime/test/ompt/worksharing/for/auto_serialized.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_serialized.h
 // REQUIRES: ompt
 // GCC doesn't call runtime for auto = static schedule
 // XFAIL: gcc
+// clang-format on
 
 #define SCHEDULE auto
 // The runtime uses static schedule for serialized loop,
diff --git a/openmp/runtime/test/ompt/worksharing/for/auto_split.c b/openmp/runtime/test/ompt/worksharing/for/auto_split.c
index c7b07d7407bad..09a619a01438e 100644
--- a/openmp/runtime/test/ompt/worksharing/for/auto_split.c
+++ b/openmp/runtime/test/ompt/worksharing/for/auto_split.c
@@ -1,8 +1,10 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_split.h
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=CHECK-LOOP %S/base_split.h
 // REQUIRES: ompt
 // GCC doesn't call runtime for auto = static schedule
 // XFAIL: gcc
+// clang-format on
 
 #define SCHEDULE auto
 // The runtime uses guided schedule for auto,
diff --git a/openmp/runtime/test/ompt/worksharing/for/base.h b/openmp/runtime/test/ompt/worksharing/for/base.h
index f29a34981cf63..0bc7bd9e763c6 100644
--- a/openmp/runtime/test/ompt/worksharing/for/base.h
+++ b/openmp/runtime/test/ompt/worksharing/for/base.h
@@ -8,8 +8,7 @@
 #define SCHED_OUTPUT STR(SCHEDULE)
 #endif
 
-int main()
-{
+int main() {
   unsigned int i;
   printf("0: Schedule: " SCHED_OUTPUT "\n");
 
@@ -26,27 +25,27 @@ int main()
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
   // CHECK: 0: Schedule: [[SCHED:[a-z]+]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker={{[0-9]+}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker={{[0-9]+}}
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], codeptr_ra=
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], codeptr_ra=
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], codeptr_ra=
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], codeptr_ra=
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-9]+}}, task_id=[[IMPLICIT_TASK_ID]]
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_implicit_task_end: parallel_id={{[0-f]+}}, task_id=[[IMPLICIT_TASK_ID]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/worksharing/for/base_serialized.h b/openmp/runtime/test/ompt/worksharing/for/base_serialized.h
index ee9970580f860..6af1d55b27ac8 100644
--- a/openmp/runtime/test/ompt/worksharing/for/base_serialized.h
+++ b/openmp/runtime/test/ompt/worksharing/for/base_serialized.h
@@ -8,8 +8,7 @@
 #define SCHED_OUTPUT STR(SCHEDULE)
 #endif
 
-int main()
-{
+int main() {
   unsigned int i;
   printf("0: Schedule: " SCHED_OUTPUT "\n");
 
@@ -27,9 +26,9 @@ int main()
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
   // CHECK: 0: Schedule: [[SCHED:[a-z]+]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-9]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=1, codeptr_ra={{(0x)?[0-f]+}}, invoker={{[0-9]+}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id=[[PARENT_TASK_ID:[0-f]+]], parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=1, codeptr_ra={{(0x)?[0-f]+}}, invoker={{[0-9]+}}
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID:[0-9]+]], task_id=[[IMPLICIT_TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID:[0-f]+]], task_id=[[IMPLICIT_TASK_ID:[0-f]+]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]], codeptr_ra={{(0x)?[0-f]+}}
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id=[[IMPLICIT_TASK_ID]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_end: parallel_id={{[PARALLEL_ID,0]}}, task_id=[[IMPLICIT_TASK_ID]]
diff --git a/openmp/runtime/test/ompt/worksharing/for/base_split.h b/openmp/runtime/test/ompt/worksharing/for/base_split.h
index 39d1f45f689f1..cb26b2736595c 100644
--- a/openmp/runtime/test/ompt/worksharing/for/base_split.h
+++ b/openmp/runtime/test/ompt/worksharing/for/base_split.h
@@ -8,21 +8,21 @@
 #define SCHED_OUTPUT STR(SCHEDULE)
 #endif
 
-/* With the combined parallel-for construct (base.h), the return-addresses are hard to compare.
-   With the separate parallel and for-nowait construct, the addresses become more predictable,
-   but the begin of the for-loop still generates additional code, so the offset of loop-begin 
-   to the label is >4 Byte.
+/* With the combined parallel-for construct (base.h), the return-addresses are
+   hard to compare. With the separate parallel and for-nowait construct, the
+   addresses become more predictable, but the begin of the for-loop still
+   generates additional code, so the offset of loop-begin to the label is >4
+   Byte.
 */
 
-int main()
-{
+int main() {
   unsigned int i;
   printf("0: Schedule: " SCHED_OUTPUT "\n");
 
 #pragma omp parallel num_threads(4)
   {
     print_current_address(0);
-    #pragma omp for schedule(SCHEDULE) nowait
+#pragma omp for schedule(SCHEDULE) nowait
     for (i = 0; i < 64; i++) {
       print_fuzzy_address(1);
     }
@@ -40,33 +40,33 @@ int main()
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
   // CHECK: 0: Schedule: [[SCHED:[a-z]+]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra=[[PARALLEL_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker={{[0-9]+}}
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra=[[PARALLEL_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}, invoker={{[0-9]+}}
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra=[[LOOP_BEGIN_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra=[[LOOP_END_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[LOOP_BEGIN_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[LOOP_END_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[LOOP_END_RETURN_ADDRESS]]
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, invoker={{[0-9]+}}, codeptr_ra=[[PARALLEL_RETURN_ADDRESS]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_parallel_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, invoker={{[0-9]+}}, codeptr_ra=[[PARALLEL_RETURN_ADDRESS]]
   // CHECK: {{^}}[[MASTER_ID]]: fuzzy_address={{.*}}[[PARALLEL_RETURN_ADDRESS]]
   
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra=[[LOOP_END_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[LOOP_END_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[LOOP_END_RETURN_ADDRESS]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra=[[LOOP_END_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[LOOP_END_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[LOOP_END_RETURN_ADDRESS]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra={{(0x)?[0-f]+}}
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra=[[LOOP_END_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra={{(0x)?[0-f]+}}
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_[[SCHED]]_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[LOOP_END_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK: {{^}}[[THREAD_ID]]: fuzzy_address={{.*}}[[LOOP_END_RETURN_ADDRESS]]
 
 
   // CHECK-LOOP: 0: NULL_POINTER=[[NULL:.*$]]
   // CHECK-LOOP: 0: Schedule: [[SCHED:[a-z]+]]
   // CHECK-LOOP: 0: ompt_event_runtime_shutdown
-  // CHECK-LOOP: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id={{[0-9]+}}, parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-9]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker={{[0-9]+}}
-  // CHECK-LOOP: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra=[[LOOP_BEGIN_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
+  // CHECK-LOOP: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin: parent_task_id={{[0-f]+}}, parent_task_frame.exit=[[NULL]], parent_task_frame.reenter={{(0x)?[0-f]+}}, parallel_id=[[PARALLEL_ID:[0-f]+]], requested_team_size=4, codeptr_ra={{(0x)?[0-f]+}}, invoker={{[0-9]+}}
+  // CHECK-LOOP: {{^}}[[MASTER_ID]]: ompt_event_loop_[[SCHED]]_begin: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[LOOP_BEGIN_RETURN_ADDRESS:(0x)?[0-f]+]]{{[0-f][0-f]}}
   // CHECK-LOOP: {{^}}{{[0-9]+}}: fuzzy_address={{.*}}[[LOOP_BEGIN_RETURN_ADDRESS]]
   // CHECK-LOOP: {{^}}{{[0-9]+}}: fuzzy_address={{.*}}[[LOOP_BEGIN_RETURN_ADDRESS]]
   // CHECK-LOOP: {{^}}{{[0-9]+}}: fuzzy_address={{.*}}[[LOOP_BEGIN_RETURN_ADDRESS]]
diff --git a/openmp/runtime/test/ompt/worksharing/for/dynamic.c b/openmp/runtime/test/ompt/worksharing/for/dynamic.c
index ca5ae10ac5d5d..cd328cc505965 100644
--- a/openmp/runtime/test/ompt/worksharing/for/dynamic.c
+++ b/openmp/runtime/test/ompt/worksharing/for/dynamic.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base.h
 // REQUIRES: ompt
+// clang-format on
 
 #define SCHEDULE dynamic
 #include "base.h"
diff --git a/openmp/runtime/test/ompt/worksharing/for/dynamic_serialized.c b/openmp/runtime/test/ompt/worksharing/for/dynamic_serialized.c
index 0f80929eb0912..67857829d2e6e 100644
--- a/openmp/runtime/test/ompt/worksharing/for/dynamic_serialized.c
+++ b/openmp/runtime/test/ompt/worksharing/for/dynamic_serialized.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_serialized.h
 // REQUIRES: ompt
+// clang-format on
 
 #define SCHEDULE dynamic
 #include "base_serialized.h"
diff --git a/openmp/runtime/test/ompt/worksharing/for/dynamic_split.c b/openmp/runtime/test/ompt/worksharing/for/dynamic_split.c
index cf14971129609..95e05483cd747 100644
--- a/openmp/runtime/test/ompt/worksharing/for/dynamic_split.c
+++ b/openmp/runtime/test/ompt/worksharing/for/dynamic_split.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_split.h
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=CHECK-LOOP %S/base_split.h
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 
 #define SCHEDULE dynamic
 #include "base_split.h"
diff --git a/openmp/runtime/test/ompt/worksharing/for/guided.c b/openmp/runtime/test/ompt/worksharing/for/guided.c
index 01bff4edf2d44..6a85a1a3e5607 100644
--- a/openmp/runtime/test/ompt/worksharing/for/guided.c
+++ b/openmp/runtime/test/ompt/worksharing/for/guided.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base.h
 // REQUIRES: ompt
+// clang-format on
 
 #define SCHEDULE guided
 #include "base.h"
diff --git a/openmp/runtime/test/ompt/worksharing/for/guided_serialized.c b/openmp/runtime/test/ompt/worksharing/for/guided_serialized.c
index a1e239e081d88..fa64475f043c2 100644
--- a/openmp/runtime/test/ompt/worksharing/for/guided_serialized.c
+++ b/openmp/runtime/test/ompt/worksharing/for/guided_serialized.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_serialized.h
 // REQUIRES: ompt
+// clang-format on
 
 #define SCHEDULE guided
 // The runtime uses static schedule for serialized loop,
diff --git a/openmp/runtime/test/ompt/worksharing/for/guided_split.c b/openmp/runtime/test/ompt/worksharing/for/guided_split.c
index 7d560c2b3bddf..1a7549d28d259 100644
--- a/openmp/runtime/test/ompt/worksharing/for/guided_split.c
+++ b/openmp/runtime/test/ompt/worksharing/for/guided_split.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_split.h
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=CHECK-LOOP %S/base_split.h
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 
 #define SCHEDULE guided
 #include "base_split.h"
diff --git a/openmp/runtime/test/ompt/worksharing/for/loop_dispatch.c b/openmp/runtime/test/ompt/worksharing/for/loop_dispatch.c
index 4f10a67bcabb4..2c015b61b9730 100644
--- a/openmp/runtime/test/ompt/worksharing/for/loop_dispatch.c
+++ b/openmp/runtime/test/ompt/worksharing/for/loop_dispatch.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // XFAIL: gcc
 // GCC doesn't call runtime for static schedule
+// clang-format on
 
 #include "callback.h"
 
@@ -16,7 +18,8 @@ int main() {
     int wait_id = 0;
     int team_size = omp_get_num_threads();
 #pragma omp for schedule(static, WORK_SIZE / 4)
-    for (i = 0; i < WORK_SIZE; i++) {}
+    for (i = 0; i < WORK_SIZE; i++) {
+    }
 
 #pragma omp for schedule(dynamic)
     for (i = 0; i < WORK_SIZE; i++) {
@@ -42,6 +45,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_parallel_begin'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
 // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_work'
@@ -49,7 +53,7 @@ int main() {
 
 // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 // CHECK: {{^}}[[THREAD_ID0:[0-9]+]]: ompt_event_parallel_begin:
-// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]]
+// CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]]
 
 // Each thread should have at least one ws-loop-chunk-begin event for each
 // for loop.
@@ -125,3 +129,4 @@ int main() {
 // CHECK: {{^}}[[THREAD_ID3]]: ompt_event_ws_loop_chunk_begin:
 // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID3]]
 // CHECK-SAME: chunk_start={{[0-9]+}}, chunk_iterations={{[1-9][0-9]*}}
+// clang-format on
diff --git a/openmp/runtime/test/ompt/worksharing/for/runtime.c b/openmp/runtime/test/ompt/worksharing/for/runtime.c
index 8c71c8b336246..9a8b66b6b594e 100644
--- a/openmp/runtime/test/ompt/worksharing/for/runtime.c
+++ b/openmp/runtime/test/ompt/worksharing/for/runtime.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base.h
 // REQUIRES: ompt
+// clang-format on
 
 #define SCHEDULE runtime
 // Without any schedule specified, the runtime uses static schedule,
diff --git a/openmp/runtime/test/ompt/worksharing/for/runtime_serialized.c b/openmp/runtime/test/ompt/worksharing/for/runtime_serialized.c
index 38e3a3ca7521c..23a0425013b8f 100644
--- a/openmp/runtime/test/ompt/worksharing/for/runtime_serialized.c
+++ b/openmp/runtime/test/ompt/worksharing/for/runtime_serialized.c
@@ -1,5 +1,7 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_serialized.h
 // REQUIRES: ompt
+// clang-format on
 
 #define SCHEDULE runtime
 // Without any schedule specified, the runtime uses static schedule,
diff --git a/openmp/runtime/test/ompt/worksharing/for/runtime_split.c b/openmp/runtime/test/ompt/worksharing/for/runtime_split.c
index 5c70908862d7f..63f3fee1b8b28 100644
--- a/openmp/runtime/test/ompt/worksharing/for/runtime_split.c
+++ b/openmp/runtime/test/ompt/worksharing/for/runtime_split.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_split.h
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=CHECK-LOOP %S/base_split.h
 // REQUIRES: ompt
 // UNSUPPORTED: gcc-4, gcc-5, gcc-6, gcc-7
+// clang-format on
 
 #define SCHEDULE runtime
 // Without any schedule specified, the runtime uses static schedule,
diff --git a/openmp/runtime/test/ompt/worksharing/for/static.c b/openmp/runtime/test/ompt/worksharing/for/static.c
index 4d99059854d58..502397e2449cd 100644
--- a/openmp/runtime/test/ompt/worksharing/for/static.c
+++ b/openmp/runtime/test/ompt/worksharing/for/static.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base.h
 // REQUIRES: ompt
 // GCC doesn't call runtime for static schedule
 // XFAIL: gcc
+// clang-format on
 
 #define SCHEDULE static
 #include "base.h"
diff --git a/openmp/runtime/test/ompt/worksharing/for/static_serialized.c b/openmp/runtime/test/ompt/worksharing/for/static_serialized.c
index 4860d49e3d6f5..a59260c3469bb 100644
--- a/openmp/runtime/test/ompt/worksharing/for/static_serialized.c
+++ b/openmp/runtime/test/ompt/worksharing/for/static_serialized.c
@@ -1,7 +1,9 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_serialized.h
 // REQUIRES: ompt
 // GCC doesn't call runtime for static schedule
 // XFAIL: gcc
+// clang-format on
 
 #define SCHEDULE static
 #include "base_serialized.h"
diff --git a/openmp/runtime/test/ompt/worksharing/for/static_split.c b/openmp/runtime/test/ompt/worksharing/for/static_split.c
index d8c88dd015f29..3607b9a5c9108 100644
--- a/openmp/runtime/test/ompt/worksharing/for/static_split.c
+++ b/openmp/runtime/test/ompt/worksharing/for/static_split.c
@@ -1,8 +1,10 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %S/base_split.h
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck --check-prefix=CHECK-LOOP %S/base_split.h
 // REQUIRES: ompt
 // GCC doesn't call runtime for static schedule
 // XFAIL: gcc
+// clang-format on
 
 #define SCHEDULE static
 #include "base_split.h"
diff --git a/openmp/runtime/test/ompt/worksharing/sections.c b/openmp/runtime/test/ompt/worksharing/sections.c
index eabc6de0da24b..5cf927def6905 100644
--- a/openmp/runtime/test/ompt/worksharing/sections.c
+++ b/openmp/runtime/test/ompt/worksharing/sections.c
@@ -1,21 +1,22 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // Some compilers generate code that does not distinguish between sections and loops
 // XFAIL: gcc, clang-3, clang-4, clang-5, icc-16, icc-17
 // UNSUPPORTED: icc-18
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
-  #pragma omp parallel sections num_threads(2)
+int main() {
+#pragma omp parallel sections num_threads(2)
   {
-    #pragma omp section
+#pragma omp section
     {
       printf("%lu: section 1\n", ompt_get_thread_data()->value);
     }
-    #pragma omp section
+#pragma omp section
     {
       printf("%lu: section 2\n", ompt_get_thread_data()->value);
     }
@@ -27,11 +28,11 @@ int main()
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_sections_begin: parallel_id=[[PARALLEL_ID:[0-9]+]], task_id=[[TASK_ID:[0-9]+]], codeptr_ra=[[SECT_BEGIN:(0x)?[0-f]+]], count=2
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_sections_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra=[[SECT_END:(0x)?[0-f]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_sections_begin: parallel_id=[[PARALLEL_ID:[0-f]+]], task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[SECT_BEGIN:(0x)?[0-f]+]], count=2
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_sections_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[SECT_END:(0x)?[0-f]+]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_sections_begin: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-9]+]], codeptr_ra=[[SECT_BEGIN]], count=2
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_sections_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}}, codeptr_ra=[[SECT_END]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_sections_begin: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[SECT_BEGIN]], count=2
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_sections_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[SECT_END]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/worksharing/sections_dispatch.c b/openmp/runtime/test/ompt/worksharing/sections_dispatch.c
index 92a22369e2e73..ec3bb5bd7552d 100644
--- a/openmp/runtime/test/ompt/worksharing/sections_dispatch.c
+++ b/openmp/runtime/test/ompt/worksharing/sections_dispatch.c
@@ -1,26 +1,28 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // UNSUPPORTED: intel-15.0, intel-16.0, intel-17.0, intel-18.0
 // GCC generates code that does not distinguish between sections and loops
 // UNSUPPORTED: gcc
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
-  #pragma omp parallel sections num_threads(2)
+int main() {
+#pragma omp parallel sections num_threads(2)
   {
-    #pragma omp section
+#pragma omp section
     {
       printf("%lu: section 1\n", ompt_get_thread_data()->value);
     }
-    #pragma omp section
+#pragma omp section
     {
       printf("%lu: section 2\n", ompt_get_thread_data()->value);
     }
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_work'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_dispatch'
@@ -28,25 +30,26 @@ int main()
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_sections_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]],
-  // CHECK-SAME: task_id=[[TASK_ID:[0-9]+]],
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
+  // CHECK-SAME: task_id=[[TASK_ID:[0-f]+]],
   // CHECK-SAME: codeptr_ra=[[SECT_BEGIN:(0x)?[0-f]+]], count=2
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_section_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]]
   // CHECK-SAME: codeptr_ra=[[SECT_BEGIN]]
   // CHECK: {{^}}[[MASTER_ID]]: ompt_event_sections_end:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}},
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}},
   // CHECK-SAME: codeptr_ra=[[SECT_END:(0x)?[0-f]+]]
 
   // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_sections_begin:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-9]+]],
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-f]+]],
   // CHECK-SAME: codeptr_ra=[[SECT_BEGIN]], count=2
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_section_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]]
   // CHECK-SAME: codeptr_ra=[[SECT_BEGIN]]
   // CHECK: {{^}}[[THREAD_ID]]: ompt_event_sections_end:
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-9]+}},
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}},
   // CHECK-SAME: codeptr_ra=[[SECT_END]]
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/runtime/test/ompt/worksharing/single.c b/openmp/runtime/test/ompt/worksharing/single.c
index d129195b49a65..4f24155e9a144 100644
--- a/openmp/runtime/test/ompt/worksharing/single.c
+++ b/openmp/runtime/test/ompt/worksharing/single.c
@@ -1,17 +1,18 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
 // GCC generates code that does not call the runtime for the single construct
 // XFAIL: gcc
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
   int x = 0;
-  #pragma omp parallel num_threads(2)
+#pragma omp parallel num_threads(2)
   {
-    #pragma omp single
+#pragma omp single
     {
       printf("%" PRIu64 ": in single\n", ompt_get_thread_data()->value);
       x++;
@@ -26,11 +27,11 @@ int main()
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
-  // CHECK-DAG: {{^}}[[THREAD_ID_1:[0-9]+]]: ompt_event_single_in_block_begin: parallel_id=[[PARALLEL_ID:[0-9]+]], task_id=[[TASK_ID:[0-9]+]], codeptr_ra={{(0x)?[0-f]+}}, count=1
+  // CHECK-DAG: {{^}}[[THREAD_ID_1:[0-9]+]]: ompt_event_single_in_block_begin: parallel_id=[[PARALLEL_ID:[0-f]+]], task_id=[[TASK_ID:[0-f]+]], codeptr_ra={{(0x)?[0-f]+}}, count=1
   // CHECK-DAG: {{^}}[[THREAD_ID_1]]: in single
   // CHECK-DAG: {{^}}[[THREAD_ID_1]]: ompt_event_single_in_block_end: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]], codeptr_ra={{(0x)?[0-f]+}}, count=1
 
-  // CHECK-DAG: {{^}}[[THREAD_ID_2:[0-9]+]]: ompt_event_single_others_begin: parallel_id=[[PARALLEL_ID:[0-9]+]], task_id=[[TASK_ID:[0-9]+]], codeptr_ra={{(0x)?[0-f]+}}, count=1
+  // CHECK-DAG: {{^}}[[THREAD_ID_2:[0-9]+]]: ompt_event_single_others_begin: parallel_id=[[PARALLEL_ID:[0-f]+]], task_id=[[TASK_ID:[0-f]+]], codeptr_ra={{(0x)?[0-f]+}}, count=1
   // CHECK-DAG: {{^}}[[THREAD_ID_2]]: ompt_event_single_others_end: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]], codeptr_ra={{(0x)?[0-f]+}}, count=1
   // clang-format on
 
diff --git a/openmp/runtime/test/ompt/worksharing/taskinfo/sections_serialized.c b/openmp/runtime/test/ompt/worksharing/taskinfo/sections_serialized.c
index 8a4c14deb8b58..8591d07d50bcf 100644
--- a/openmp/runtime/test/ompt/worksharing/taskinfo/sections_serialized.c
+++ b/openmp/runtime/test/ompt/worksharing/taskinfo/sections_serialized.c
@@ -1,11 +1,12 @@
+// clang-format off
 // RUN: %libomp-compile-and-run | %sort-threads | FileCheck %s
 // REQUIRES: ompt
+// clang-format on
 
 #include "callback.h"
 #include <omp.h>
 
-int main()
-{
+int main() {
 #pragma omp parallel sections num_threads(1)
   {
 #pragma omp section
@@ -17,20 +18,21 @@ int main()
     }
   }
 
+  // clang-format off
   // Check if libomp supports the callbacks for this test.
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_task_create'
   // CHECK-NOT: {{^}}0: Could not register callback 'ompt_callback_implicit_task'
 
 
   // CHECK: {{^}}0: NULL_POINTER=[[NULL:.*$]]
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id=[[INITIAL_PARALLEL_ID:[0-9]+]], task_id=[[INITIAL_TASK_ID:[0-9]+]], actual_parallelism=1, index=1, flags=1
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_initial_task_begin: parallel_id=[[INITIAL_PARALLEL_ID:[0-f]+]], task_id=[[INITIAL_TASK_ID:[0-f]+]], actual_parallelism=1, index=1, flags=1
 
   // region 0
   // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_parallel_begin
   // CHECK-SAME: parent_task_frame.exit=[[NULL]], parent_task_frame.reenter=[[INITIAL_TASK_FRAME_ENTER:(0x)?[0-f]+]],
-  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-9]+]]
+  // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]]
 
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-9]+]]
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_implicit_task_begin: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-f]+]]
 
   // information about implicit task (exit frame should be set, while enter should be NULL)
   // CHECK: {{^}}[[MASTER_ID]]: task level 0: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]]
@@ -43,6 +45,7 @@ int main()
   // CHECK-SAME: exit_frame=[[NULL]]
   // CHECK-SAME: reenter_frame=[[INITIAL_TASK_FRAME_ENTER]]
   // CHECK-SAME: task_type=ompt_task_initial
+  // clang-format on
 
   return 0;
 }
diff --git a/openmp/tools/archer/CMakeLists.txt b/openmp/tools/archer/CMakeLists.txt
index 980eb2b0b675f..a1c768c449d04 100644
--- a/openmp/tools/archer/CMakeLists.txt
+++ b/openmp/tools/archer/CMakeLists.txt
@@ -15,6 +15,21 @@ if(LIBOMP_OMPT_SUPPORT AND LIBOMP_ARCHER_SUPPORT)
   target_link_libraries(archer ${CMAKE_DL_LIBS})
   add_library(archer_static STATIC ompt-tsan.cpp)
 
+  if(TARGET cxx-headers)
+    add_dependencies(archer cxx-headers)
+    add_dependencies(archer_static cxx-headers)
+  endif()
+
+  if(TARGET cxx_shared)
+    add_dependencies(archer cxx_shared)
+    add_dependencies(archer_static cxx_shared)
+  endif()
+
+  if(TARGET cxxabi_shared)
+    add_dependencies(archer cxxabi_shared)
+    add_dependencies(archer_static cxxabi_shared)
+  endif()
+
   install(TARGETS archer archer_static
     LIBRARY DESTINATION ${OPENMP_INSTALL_LIBDIR}
     ARCHIVE DESTINATION ${OPENMP_INSTALL_LIBDIR})
diff --git a/openmp/tools/multiplex/tests/custom_data_storage/custom_data_storage.c b/openmp/tools/multiplex/tests/custom_data_storage/custom_data_storage.c
index 9ce2d58147830..ee6fc40e8d2e3 100644
--- a/openmp/tools/multiplex/tests/custom_data_storage/custom_data_storage.c
+++ b/openmp/tools/multiplex/tests/custom_data_storage/custom_data_storage.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-tool -DFIRST_TOOL -o %t.first.tool.so %s && \
 // RUN: %libomp-tool -DSECOND_TOOL -o %t.second.tool.so %s && \
 // RUN: %libomp-compile && \
@@ -8,6 +9,7 @@
 // For GCC we don't get an event for master,
 // see runtime/test/ompt/sycnchronization/master.c
 // UNSUPPORTED: gcc
+// clang-format on
 
 #if defined(FIRST_TOOL)
 #include "first-tool.h"
@@ -36,6 +38,7 @@ int main() {
   }
   return 0;
 }
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback
 
@@ -49,19 +52,19 @@ int main() {
 // CHECK-SAME: thread_id=[[_1ST_MSTR_TID]]
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_initial_task_begin:
-// CHECK-SAME: parallel_id=[[_FIRST_INIT_PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: task_id=[[_FIRST_INITIAL_TASK_ID:[0-9]+]], actual_parallelism=1,
+// CHECK-SAME: parallel_id=[[_FIRST_INIT_PARALLEL_ID:[0-f]+]],
+// CHECK-SAME: task_id=[[_FIRST_INITIAL_TASK_ID:[0-f]+]], actual_parallelism=1,
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_parallel_begin:
 // CHECK-SAME: parent_task_id=[[_FIRST_INITIAL_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit=[[NULL]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID:[0-9]+]], requested_team_size=2,
+// CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID:[0-f]+]], requested_team_size=2,
 // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}, invoker
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[_FIRST_MASTER_IMPLICIT_TASK_ID:[0-9]+]], team_size=2,
+// CHECK-SAME: task_id=[[_FIRST_MASTER_IMPLICIT_TASK_ID:[0-f]+]], team_size=2,
 // CHECK-SAME: thread_num=0
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_masked_begin:
@@ -73,7 +76,7 @@ int main() {
 // CHECK-SAME: parent_task_id=[[_FIRST_MASTER_IMPLICIT_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}},
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: new_task_id=[[_FIRST_EXPLICIT_TASK_ID:[0-9]+]],
+// CHECK-SAME: new_task_id=[[_FIRST_EXPLICIT_TASK_ID:[0-f]+]],
 // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit=4,
 // CHECK-SAME: has_dependences=no
 
@@ -154,20 +157,20 @@ int main() {
 // CHECK-SAME: thread_id=[[_2ND_MSTR_TID]]
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_initial_task_begin:
-// CHECK-SAME: parallel_id=[[SECOND_INIT_PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: task_id=[[SECOND_INITIAL_TASK_ID:[0-9]+]], actual_parallelism=1,
+// CHECK-SAME: parallel_id=[[SECOND_INIT_PARALLEL_ID:[0-f]+]],
+// CHECK-SAME: task_id=[[SECOND_INITIAL_TASK_ID:[0-f]+]], actual_parallelism=1,
 // CHECK-SAME: index=1, flags=1
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_parallel_begin:
 // CHECK-SAME: parent_task_id=[[SECOND_INITIAL_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit=[[NULL]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID:[0-9]+]], requested_team_size=2,
+// CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID:[0-f]+]], requested_team_size=2,
 // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}, invoker
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[SECOND_MASTER_IMPLICIT_TASK_ID:[0-9]+]], team_size=2,
+// CHECK-SAME: task_id=[[SECOND_MASTER_IMPLICIT_TASK_ID:[0-f]+]], team_size=2,
 // CHECK-SAME: thread_num=0
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_masked_begin:
@@ -179,7 +182,7 @@ int main() {
 // CHECK-SAME: parent_task_id=[[SECOND_MASTER_IMPLICIT_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}},
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: new_task_id=[[SECOND_EXPLICIT_TASK_ID:[0-9]+]],
+// CHECK-SAME: new_task_id=[[SECOND_EXPLICIT_TASK_ID:[0-f]+]],
 // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit=4,
 // CHECK-SAME: has_dependences=no
 
@@ -262,7 +265,7 @@ int main() {
 
 // CHECK: {{^}}[[_1ST_WRKR_TID]]: _first_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[_FIRST_WORKER_IMPLICIT_TASK_ID:[0-9]+]], team_size=2,
+// CHECK-SAME: task_id=[[_FIRST_WORKER_IMPLICIT_TASK_ID:[0-f]+]], team_size=2,
 // CHECK-SAME: thread_num=1
 
 // CHECK: {{^}}[[_1ST_WRKR_TID]]: _first_tool:
@@ -299,7 +302,7 @@ int main() {
 
 // CHECK: {{^}}[[_2ND_WRKR_TID]]: second_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[SECOND_WORKER_IMPLICIT_TASK_ID:[0-9]+]], team_size=2,
+// CHECK-SAME: task_id=[[SECOND_WORKER_IMPLICIT_TASK_ID:[0-f]+]], team_size=2,
 // CHECK-SAME: thread_num=1
 
 // CHECK: {{^}}[[_2ND_WRKR_TID]]: second_tool:
@@ -329,5 +332,6 @@ int main() {
 
 // CHECK: {{^}}[[_2ND_WRKR_TID]]: second_tool: ompt_event_thread_end:
 // CHECK-SAME: thread_id=[[_2ND_WRKR_TID]]
+// clang-format on
 
 #endif /* APP */
diff --git a/openmp/tools/multiplex/tests/custom_data_storage/first-tool.h b/openmp/tools/multiplex/tests/custom_data_storage/first-tool.h
index 91cb34e233abc..dfa6140891702 100644
--- a/openmp/tools/multiplex/tests/custom_data_storage/first-tool.h
+++ b/openmp/tools/multiplex/tests/custom_data_storage/first-tool.h
@@ -216,7 +216,7 @@ static int on_cds_ompt_callback_control_tool(uint64_t command,
   while (ompt_get_task_info(task_level, NULL, (ompt_data_t **)&task_data, NULL,
                             NULL, NULL)) {
     task_data = get_own_ompt_data(task_data);
-    printf("%" PRIu64 ": _first_tool: task level %d: task_id=%" PRIu64 "\n",
+    printf("%" PRIu64 ": _first_tool: task level %d: task_id=%" PRIx64 "\n",
            ompt_get_thread_data()->value, task_level, task_data->value);
     task_level++;
   }
@@ -227,7 +227,7 @@ static int on_cds_ompt_callback_control_tool(uint64_t command,
   while (ompt_get_parallel_info(parallel_level, (ompt_data_t **)&parallel_data,
                                 NULL)) {
     parallel_data = get_own_ompt_data(parallel_data);
-    printf("%" PRIu64 ": _first_tool: parallel level %d: parallel_id=%" PRIu64
+    printf("%" PRIu64 ": _first_tool: parallel level %d: parallel_id=%" PRIx64
            "\n",
            ompt_get_thread_data()->value, parallel_level, parallel_data->value);
     parallel_level++;
@@ -240,7 +240,7 @@ ompt_data_t *ompt_get_own_thread_data() {
   return get_own_ompt_data(ompt_cds_get_thread_data());
 }
 
-#define register_ompt_callback2_t(name, type)                                       \
+#define register_ompt_callback2_t(name, type)                                  \
   do {                                                                         \
     type f_##name = &on_cds_##name;                                            \
     if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
@@ -261,12 +261,13 @@ int ompt_cds_initialize(ompt_function_lookup_t lookup, int initial_device_num,
   register_ompt_callback(ompt_callback_nest_lock);
   register_ompt_callback2(ompt_callback_sync_region);
   register_ompt_callback2_t(ompt_callback_sync_region_wait,
-                       ompt_callback_sync_region_t);
+                            ompt_callback_sync_region_t);
   register_ompt_callback2(ompt_callback_control_tool);
   register_ompt_callback2(ompt_callback_flush);
   register_ompt_callback2(ompt_callback_cancel);
   register_ompt_callback2(ompt_callback_implicit_task);
-  register_ompt_callback_t(ompt_callback_lock_init, ompt_callback_mutex_acquire_t);
+  register_ompt_callback_t(ompt_callback_lock_init,
+                           ompt_callback_mutex_acquire_t);
   register_ompt_callback_t(ompt_callback_lock_destroy, ompt_callback_mutex_t);
   register_ompt_callback2(ompt_callback_work);
   register_ompt_callback2(ompt_callback_master);
diff --git a/openmp/tools/multiplex/tests/print/print.c b/openmp/tools/multiplex/tests/print/print.c
index 019788eab9d14..d99680bcd7b02 100644
--- a/openmp/tools/multiplex/tests/print/print.c
+++ b/openmp/tools/multiplex/tests/print/print.c
@@ -1,3 +1,4 @@
+// clang-format off
 // RUN: %libomp-tool -DFIRST_TOOL -o %t.first.tool.so %s && \
 // RUN: %libomp-tool -DSECOND_TOOL -o %t.second.tool.so %s && \
 // RUN: %libomp-compile && \
@@ -8,6 +9,7 @@
 // For GCC we don't get an event for master,
 // see runtime/test/ompt/sycnchronization/master.c
 // UNSUPPORTED: gcc
+// clang-format on
 
 #if defined(FIRST_TOOL)
 #include "first-tool.h"
@@ -37,6 +39,7 @@ int main() {
   return 0;
 }
 
+// clang-format off
 // Check if libomp supports the callbacks for this test.
 // CHECK-NOT: {{^}}0: Could not register callback
 
@@ -49,20 +52,20 @@ int main() {
 // CHECK-SAME: thread_type=ompt_thread_initial=1, thread_id=[[_1ST_MSTR_TID]]
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_initial_task_begin:
-// CHECK-SAME: parallel_id=[[_FIRST_INIT_PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: task_id=[[_FIRST_INITIAL_TASK_ID:[0-9]+]],
+// CHECK-SAME: parallel_id=[[_FIRST_INIT_PARALLEL_ID:[0-f]+]],
+// CHECK-SAME: task_id=[[_FIRST_INITIAL_TASK_ID:[0-f]+]],
 // CHECK-SAME: actual_parallelism=1, index=1, flags=1
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_parallel_begin:
 // CHECK-SAME: parent_task_id=[[_FIRST_INITIAL_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit=[[NULL]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID:[0-9]+]],
+// CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID:[0-f]+]],
 // CHECK-SAME: requested_team_size=2, codeptr_ra={{(0x)?[0-f]+}}, invoker
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[_FIRST_MASTER_IMPLICIT_TASK_ID:[0-9]+]], team_size=2,
+// CHECK-SAME: task_id=[[_FIRST_MASTER_IMPLICIT_TASK_ID:[0-f]+]], team_size=2,
 // CHECK-SAME: thread_num=0
 
 // CHECK: {{^}}[[_1ST_MSTR_TID]]: _first_tool: ompt_event_masked_begin:
@@ -74,7 +77,7 @@ int main() {
 // CHECK-SAME: parent_task_id=[[_FIRST_MASTER_IMPLICIT_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}},
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: new_task_id=[[_FIRST_EXPLICIT_TASK_ID:[0-9]+]],
+// CHECK-SAME: new_task_id=[[_FIRST_EXPLICIT_TASK_ID:[0-f]+]],
 // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit=4,
 // CHECK-SAME: has_dependences=no
 
@@ -151,20 +154,20 @@ int main() {
 // CHECK-SAME: thread_type=ompt_thread_initial=1, thread_id=[[_2ND_MSTR_TID]]
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_initial_task_begin:
-// CHECK-SAME: parallel_id=[[SECOND_INIT_PARALLEL_ID:[0-9]+]],
-// CHECK-SAME: task_id=[[SECOND_INITIAL_TASK_ID:[0-9]+]],
+// CHECK-SAME: parallel_id=[[SECOND_INIT_PARALLEL_ID:[0-f]+]],
+// CHECK-SAME: task_id=[[SECOND_INITIAL_TASK_ID:[0-f]+]],
 // CHECK-SAME: actual_parallelism=1, index=1, flags=1
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_parallel_begin:
 // CHECK-SAME: parent_task_id=[[SECOND_INITIAL_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit=[[NULL]],
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID:[0-9]+]],
+// CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID:[0-f]+]],
 // CHECK-SAME: requested_team_size=2, codeptr_ra={{(0x)?[0-f]+}}, invoker
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[SECOND_MASTER_IMPLICIT_TASK_ID:[0-9]+]], team_size=2,
+// CHECK-SAME: task_id=[[SECOND_MASTER_IMPLICIT_TASK_ID:[0-f]+]], team_size=2,
 // CHECK-SAME: thread_num=0
 
 // CHECK: {{^}}[[_2ND_MSTR_TID]]: second_tool: ompt_event_masked_begin:
@@ -176,7 +179,7 @@ int main() {
 // CHECK-SAME: parent_task_id=[[SECOND_MASTER_IMPLICIT_TASK_ID]],
 // CHECK-SAME: parent_task_frame.exit={{(0x)?[0-f]+}},
 // CHECK-SAME: parent_task_frame.reenter={{(0x)?[0-f]+}},
-// CHECK-SAME: new_task_id=[[SECOND_EXPLICIT_TASK_ID:[0-9]+]],
+// CHECK-SAME: new_task_id=[[SECOND_EXPLICIT_TASK_ID:[0-f]+]],
 // CHECK-SAME: codeptr_ra={{(0x)?[0-f]+}}, task_type=ompt_task_explicit=4,
 // CHECK-SAME: has_dependences=no
 
@@ -255,7 +258,7 @@ int main() {
 
 // CHECK: {{^}}[[_1ST_WRKR_TID]]: _first_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[_FIRST_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[_FIRST_WORKER_IMPLICIT_TASK_ID:[0-9]+]], team_size=2,
+// CHECK-SAME: task_id=[[_FIRST_WORKER_IMPLICIT_TASK_ID:[0-f]+]], team_size=2,
 // CHECK-SAME: thread_num=1
 
 // CHECK: {{^}}[[_1ST_WRKR_TID]]: _first_tool:
@@ -291,7 +294,7 @@ int main() {
 
 // CHECK: {{^}}[[_2ND_WRKR_TID]]: second_tool: ompt_event_implicit_task_begin:
 // CHECK-SAME: parallel_id=[[SECOND_PARALLEL_ID]],
-// CHECK-SAME: task_id=[[SECOND_WORKER_IMPLICIT_TASK_ID:[0-9]+]],
+// CHECK-SAME: task_id=[[SECOND_WORKER_IMPLICIT_TASK_ID:[0-f]+]],
 // CHECK-SAME: team_size=2, thread_num=1
 
 // CHECK: {{^}}[[_2ND_WRKR_TID]]: second_tool:
@@ -320,5 +323,6 @@ int main() {
 
 // CHECK: {{^}}[[_2ND_WRKR_TID]]: second_tool: ompt_event_thread_end:
 // CHECK-SAME: thread_id=[[_2ND_WRKR_TID]]
+// clang-format on
 
 #endif /* APP */
diff --git a/openmp/tools/omptest/CMakeLists.txt b/openmp/tools/omptest/CMakeLists.txt
new file mode 100644
index 0000000000000..10c0cd98ccbc2
--- /dev/null
+++ b/openmp/tools/omptest/CMakeLists.txt
@@ -0,0 +1,149 @@
+##===----------------------------------------------------------------------===##
+#
+# Build OMPT unit testing library: ompTest
+#
+##===----------------------------------------------------------------------===##
+
+cmake_minimum_required(VERSION 3.20)
+project(omptest LANGUAGES CXX)
+
+option(LIBOMPTEST_BUILD_STANDALONE
+       "Build ompTest 'standalone', i.e. w/o GoogleTest."
+       ${OPENMP_STANDALONE_BUILD})
+option(LIBOMPTEST_BUILD_UNITTESTS
+       "Build ompTest's unit tests, requires GoogleTest." OFF)
+
+# Exit early if OMPT support or LLVM-tests were disabled by the user.
+if((NOT ${LIBOMP_OMPT_SUPPORT}) OR (NOT ${LLVM_INCLUDE_TESTS}))
+  return()
+endif()
+
+include(CMakePackageConfigHelpers)
+
+set(OMPTEST_HEADERS
+  ./include/AssertMacros.h
+  ./include/InternalEvent.h
+  ./include/InternalEventCommon.h
+  ./include/Logging.h
+  ./include/OmptAliases.h
+  ./include/OmptAsserter.h
+  ./include/OmptAssertEvent.h
+  ./include/OmptCallbackHandler.h
+  ./include/OmptTester.h
+  ./include/OmptTesterGlobals.h
+)
+
+add_library(omptest
+  SHARED
+
+  ${OMPTEST_HEADERS}
+  ./src/InternalEvent.cpp
+  ./src/InternalEventOperators.cpp
+  ./src/Logging.cpp
+  ./src/OmptAsserter.cpp
+  ./src/OmptAssertEvent.cpp
+  ./src/OmptCallbackHandler.cpp
+  ./src/OmptTester.cpp
+)
+
+# Target: ompTest library
+# On (implicit) request of GoogleTest, link against the one provided with LLVM.
+if ((NOT LIBOMPTEST_BUILD_STANDALONE) OR LIBOMPTEST_BUILD_UNITTESTS)
+  # Check if standalone build was requested together with unittests
+  if (LIBOMPTEST_BUILD_STANDALONE)
+    # Emit warning: this build actually depends on LLVM's GoogleTest
+    message(WARNING "LIBOMPTEST_BUILD_STANDALONE and LIBOMPTEST_BUILD_UNITTESTS"
+                    " requested simultaneously.\n"
+                    "Linking against LLVM's GoogleTest library archives.\n"
+                    "Disable LIBOMPTEST_BUILD_UNITTESTS to perform an actual"
+                    " standalone build.")
+    # Explicitly disable LIBOMPTEST_BUILD_STANDALONE
+    set(LIBOMPTEST_BUILD_STANDALONE OFF)
+  endif()
+
+  # Add dependency llvm_gtest; emits error if unavailable.
+  add_dependencies(omptest llvm_gtest)
+
+  # Link llvm_gtest as whole-archive to expose required symbols
+  set(GTEST_LINK_CMD "-Wl,--whole-archive" llvm_gtest
+                     "-Wl,--no-whole-archive" LLVMSupport)
+
+  # Add GoogleTest-based header
+  target_sources(omptest PRIVATE ./include/OmptTesterGoogleTest.h)
+
+  # Add LLVM-provided GoogleTest include directories.
+  target_include_directories(omptest PRIVATE
+    ${LLVM_THIRD_PARTY_DIR}/unittest/googletest/include)
+
+  # TODO: Re-visit ABI breaking checks, disable for now.
+  target_compile_definitions(omptest PUBLIC
+    -DLLVM_DISABLE_ABI_BREAKING_CHECKS_ENFORCING)
+
+  # Link against gtest and gtest_main
+  target_link_libraries(omptest PRIVATE ${GTEST_LINK_CMD})
+else()
+  # Add 'standalone' compile definitions
+  target_compile_definitions(omptest PRIVATE
+    -DOPENMP_LIBOMPTEST_BUILD_STANDALONE)
+
+  # Add 'standalone' source files
+  target_sources(omptest PRIVATE
+    ./include/OmptTesterStandalone.h
+    ./src/OmptTesterStandalone.cpp)
+endif()
+
+if(TARGET cxx-headers)
+  add_dependencies(omptest cxx-headers)
+endif()
+
+if(TARGET cxx_shared)
+  add_dependencies(omptest cxx_shared)
+endif()
+
+if(TARGET cxxabi_shared)
+  add_dependencies(omptest cxxabi_shared)
+endif()
+
+# Add common include directories.
+target_include_directories(omptest PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:$<INSTALL_PREFIX>/${LIBOMP_HEADERS_INSTALL_PATH}/omptest>
+)
+
+target_compile_features(omptest PRIVATE cxx_std_17)
+
+# Create and install package configuration files.
+configure_package_config_file(
+  ${CMAKE_CURRENT_SOURCE_DIR}/cmake/omptest-config.cmake.in
+  ${CMAKE_CURRENT_BINARY_DIR}/cmake/omptest-config.cmake
+  INSTALL_DESTINATION "${OPENMP_INSTALL_LIBDIR}/cmake/openmp/omptest"
+)
+
+install(FILES ${omptest_BINARY_DIR}/cmake/omptest-config.cmake
+        DESTINATION "${OPENMP_INSTALL_LIBDIR}/cmake/openmp/omptest")
+
+# Install libomptest header files: Copy header-files from include dir
+install(DIRECTORY ./include/
+        DESTINATION "${LIBOMP_HEADERS_INSTALL_PATH}/omptest"
+        FILES_MATCHING PATTERN "*.h")
+
+# Install library and export targets.
+# Note: find_package(omptest) may require setting of PATH_SUFFIXES
+#       Example: "lib/cmake/openmp/omptest", this is due to the install location
+install(TARGETS omptest
+        EXPORT OPENMPomptest
+        LIBRARY COMPONENT omptest
+        DESTINATION "${OPENMP_INSTALL_LIBDIR}"
+        INCLUDES DESTINATION "${LIBOMP_HEADERS_INSTALL_PATH}/omptest")
+
+# Allow to link omptest by using: target_link_libraries( ... omptest::omptest)
+# Additionally, it automatically propagates the include directory.
+install(EXPORT OPENMPomptest
+        DESTINATION "${OPENMP_INSTALL_LIBDIR}/cmake/openmp/omptest"
+        NAMESPACE omptest::
+        FILE omptest-targets.cmake)
+
+# Discover unit tests (added to check-openmp)
+if(LIBOMPTEST_BUILD_UNITTESTS)
+  add_subdirectory(test)
+endif()
diff --git a/openmp/tools/omptest/README.md b/openmp/tools/omptest/README.md
new file mode 100644
index 0000000000000..4ea35ede0419f
--- /dev/null
+++ b/openmp/tools/omptest/README.md
@@ -0,0 +1,279 @@
+
+README for the OpenMP Tooling Interface Testing Library (ompTest)
+=================================================================
+
+# Introduction
+OpenMP Tooling Interface Testing Library (ompTest)
+ompTest is a unit testing framework for testing OpenMP implementations.
+It offers a simple-to-use framework that allows a tester to check for OMPT
+events in addition to regular unit testing code, supported by linking against
+GoogleTest by default. It also facilitates writing concise tests while bridging
+the semantic gap between the unit under test and the OMPT-event testing.
+
+# Testing macros
+
+Corresponding macro definitions are located in:  `./include/AssertMacros.h`
+
+## OMPT_GENERATE_EVENTS(NumberOfCopies, EventMacro)
+`TODO`
+
+## OMPT_ASSERT_SET_EVENT(Name, Group, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SET(EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SET_GROUPED(Group, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SET_NAMED(Name, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SET_EVENT_NOT(Name, Group, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SET_NOT(EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SET_GROUPED_NOT(Group, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SET_NAMED_NOT(Name, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_EVENT(Name, Group, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE(EventTy, ...)
+This macro checks for the occurrence of the provided event, which also
+entails the exact sequence of events. When only using this assertion macro one
+has to provide every single event in the exact order of occurrence.
+
+## OMPT_ASSERT_SEQUENCE_GROUPED(Group, EventTy, ...)
+This macro acts like `OMPT_ASSERT_SEQUENCE` with the addition of grouping.
+
+## OMPT_ASSERT_SEQUENCE_NAMED(Name, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_EVENT_NOT(Name, Group, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_NOT(EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_GROUPED_NOT(Group, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_NAMED_NOT(Name, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_SUSPEND()
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_ONLY(EventTy, ...)
+This macro acts like `OMPT_ASSERT_SEQUENCE`, while actually being preceded
+-AND- succeeded by commands to suspend sequenced assertion until the next match.
+As a result, one may omit all other "unnecessary" events from the sequence.
+
+## OMPT_ASSERT_SEQUENCE_GROUPED_ONLY(Group, EventTy, ...)
+This macro acts like `OMPT_ASSERT_SEQUENCE_ONLY`, plus grouping.
+
+## OMPT_ASSERT_SEQUENCE_NAMED_ONLY(Name, EventTy, ...)
+`TODO`
+
+## OMPT_ASSERTER_MODE_STRICT(Asserter)
+`TODO`
+
+## OMPT_ASSERTER_MODE_RELAXED(Asserter)
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_MODE_STRICT()
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_MODE_RELAXED()
+`TODO`
+
+## OMPT_ASSERT_SET_MODE_STRICT()
+`TODO`
+
+## OMPT_ASSERT_SET_MODE_RELAXED()
+`TODO`
+
+## OMPT_ASSERTER_DISABLE(Asserter)
+`TODO`
+
+## OMPT_ASSERTER_ENABLE(Asserter)
+`TODO`
+
+## OMPT_ASSERT_SET_DISABLE()
+`TODO`
+
+## OMPT_ASSERT_SET_ENABLE()
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_DISABLE()
+`TODO`
+
+## OMPT_ASSERT_SEQUENCE_ENABLE()
+`TODO`
+
+## OMPT_REPORT_EVENT_DISABLE()
+`TODO`
+
+## OMPT_REPORT_EVENT_ENABLE()
+`TODO`
+
+## OMPT_ASSERTER_PERMIT_EVENT(Asserter, EventTy)
+`TODO`
+
+## OMPT_ASSERTER_SUPPRESS_EVENT(Asserter, EventTy)
+`TODO`
+
+## OMPT_PERMIT_EVENT(EventTy)
+`TODO`
+
+## OMPT_SUPPRESS_EVENT(EventTy)
+`TODO`
+
+## OMPT_ASSERTER_LOG_LEVEL(Asserter, LogLevel)
+`TODO`
+
+## OMPT_ASSERTER_LOG_FORMATTED(Asserter, FormatLog)
+`TODO`
+
+## OMPT_ASSERT_SYNC_POINT(SyncPointName)
+`TODO`
+
+### Grouping Asserts
+
+This allows to generate and verify data during runtime of a test.
+Currently, we only use target region information which manifests into groups.
+This allows to correlate multiple events to a certain target region without
+manual interaction just by specifying a groupname for these events.
+
+When a target region is encountered and we are about to enter it, we gather the
+`target_id` (non-EMI) -OR- `target_data->value` (EMI). This value is stored
+along the groupname for future reference. Upon target region end, the
+corresponding group is erased. (Note: The groupname is available again.)
+
+Other asserted callbacks which may occur within target regions query their
+groupname: retrieving and comparing the value of the group against the observed
+event's value.
+
+### Suspending Sequenced Asserts
+
+When a sequence of events is not of interest while testing, these additional
+events may be ignored by suspending the assertion until the next match. This
+can be done by using `OMPT_ASSERT_SEQUENCE_SUSPEND` manually or the `_ONLY`
+macro variants, like `OMPT_ASSERT_GROUPED_SEQUENCE_ONLY`.
+
+The former adds a special event to the queue of expected events and signal
+that any non-matching event should be ignored rather than failing the test.
+`_ONLY` macros embed their corresponding macro between two calls to
+`OMPT_ASSERT_SEQUENCE_SUSPEND`. As a consequence, we enter passive assertion
+until a match occurs, then enter passive assertion again. This enables us to
+"only" assert a certain, single event in arbitrary circumstances.
+
+### Asserter Modes
+`TODO`
+
+## Aliases (shorthands)
+To allow for easier writing of tests and enhanced readability, the following set
+of aliases is introduced. The left hand side represents the original value,
+while the right hand side depicts the shorthand version.
+
+| Type                      | Enum Value                                  | Shorthand                 |
+|---------------------------|---------------------------------------------|---------------------------|
+| **ompt_scope_endpoint_t** |                                             |                           |
+|                           | ompt_scope_begin                            | BEGIN                     |
+|                           | ompt_scope_end                              | END                       |
+|                           | ompt_scope_beginend                         | BEGINEND                  |
+| **ompt_target_t**         |                                             |                           |
+|                           | ompt_target                                 | TARGET                    |
+|                           | ompt_target_enter_data                      | ENTER_DATA                |
+|                           | ompt_target_exit_data                       | EXIT_DATA                 |
+|                           | ompt_target_update                          | UPDATE                    |
+|                           | ompt_target_nowait                          | TARGET_NOWAIT             |
+|                           | ompt_target_enter_data_nowait               | ENTER_DATA_NOWAIT         |
+|                           | ompt_target_exit_data_nowait                | EXIT_DATA_NOWAIT          |
+|                           | ompt_target_update_nowait                   | UPDATE_NOWAIT             |
+| **ompt_target_data_op_t** |                                             |                           |
+|                           | ompt_target_data_alloc                      | ALLOC                     |
+|                           | ompt_target_data_transfer_to_device         | H2D                       |
+|                           | ompt_target_data_transfer_from_device       | D2H                       |
+|                           | ompt_target_data_delete                     | DELETE                    |
+|                           | ompt_target_data_associate                  | ASSOCIATE                 |
+|                           | ompt_target_data_disassociate               | DISASSOCIATE              |
+|                           | ompt_target_data_alloc_async                | ALLOC_ASYNC               |
+|                           | ompt_target_data_transfer_to_device_async   | H2D_ASYNC                 |
+|                           | ompt_target_data_transfer_from_device_async | D2H_ASYNC                 |
+|                           | ompt_target_data_delete_async               | DELETE_ASYNC              |
+| **ompt_callbacks_t**      |                                             |                           |
+|                           | ompt_callback_target                        | CB_TARGET                 |
+|                           | ompt_callback_target_data_op                | CB_DATAOP                 |
+|                           | ompt_callback_target_submit                 | CB_KERNEL                 |
+| **ompt_work_t**           |                                             |                           |
+|                           | ompt_work_loop                              | WORK_LOOP                 |
+|                           | ompt_work_sections                          | WORK_SECT                 |
+|                           | ompt_work_single_executor                   | WORK_EXEC                 |
+|                           | ompt_work_single_other                      | WORK_SINGLE               |
+|                           | ompt_work_workshare                         | WORK_SHARE                |
+|                           | ompt_work_distribute                        | WORK_DIST                 |
+|                           | ompt_work_taskloop                          | WORK_TASK                 |
+|                           | ompt_work_scope                             | WORK_SCOPE                |
+|                           | ompt_work_loop_static                       | WORK_LOOP_STA             |
+|                           | ompt_work_loop_dynamic                      | WORK_LOOP_DYN             |
+|                           | ompt_work_loop_guided                       | WORK_LOOP_GUI             |
+|                           | ompt_work_loop_other                        | WORK_LOOP_OTH             |
+| **ompt_sync_region_t**    |                                             |                           |
+|                           | ompt_sync_region_barrier                    | SR_BARRIER                |
+|                           | ompt_sync_region_barrier_implicit           | SR_BARRIER_IMPL           |
+|                           | ompt_sync_region_barrier_explicit           | SR_BARRIER_EXPL           |
+|                           | ompt_sync_region_barrier_implementation     | SR_BARRIER_IMPLEMENTATION |
+|                           | ompt_sync_region_taskwait                   | SR_TASKWAIT               |
+|                           | ompt_sync_region_taskgroup                  | SR_TASKGROUP              |
+|                           | ompt_sync_region_reduction                  | SR_REDUCTION              |
+|                           | ompt_sync_region_barrier_implicit_workshare | SR_BARRIER_IMPL_WORKSHARE |
+|                           | ompt_sync_region_barrier_implicit_parallel  | SR_BARRIER_IMPL_PARALLEL  |
+|                           | ompt_sync_region_barrier_teams              | SR_BARRIER_TEAMS          |
+
+
+Limitations
+===========
+Currently, there are some peculiarities which have to be kept in mind when using
+this library:
+
+## Callbacks
+  * It is not possible to e.g. test non-EMI -AND- EMI callbacks within the same
+    test file. Reason: all testsuites share the initialization and therefore the
+    registered callbacks.
+  * It is not possible to check for device initialization and/or load callbacks
+    more than once per test file. The first testcase being run, triggers these
+    callbacks and is therefore the only testcase that is able to check for them.
+    This is because, after that, the device remains initialized.
+  * It is not possible to check for device finalization callbacks, as libomptest
+    is un-loaded before this callback occurs. Same holds true for the final
+    ThreadEnd event(s).
+
+Miscellaneous
+=============
+
+## Default values
+
+To allow for easier writing of tests, many OMPT events may be created using less
+parameters than actually requested by the spec -- by using default values. These
+defaults are currently set to the corresponding data type's minimum as follows,
+for example integers use: `std::numeric_limits<int>::min()`.
+
+When an expected / user-specified event has certain values set to the
+corresponding default, these values are ignored. That is, when compared to an
+observed event, this property is considered as 'equal' regardless of their
+actual equality relation.
+
+References
+==========
+[0]: ompTest – Unit Testing with OMPT
+     https://doi.org/10.1109/SCW63240.2024.00031
+
+[1]: OMPTBench – OpenMP Tool Interface Conformance Testing
+     https://doi.org/10.1109/SCW63240.2024.00036
diff --git a/openmp/tools/omptest/cmake/omptest-config.cmake.in b/openmp/tools/omptest/cmake/omptest-config.cmake.in
new file mode 100644
index 0000000000000..2ab416efab1f3
--- /dev/null
+++ b/openmp/tools/omptest/cmake/omptest-config.cmake.in
@@ -0,0 +1,32 @@
+################################################################################
+##
+## omptest cmake configuration file.
+## Enable support for find_package(omptest).
+##
+################################################################################
+
+# Initialize package.
+@PACKAGE_INIT@
+
+# Include exported targets.
+include("${CMAKE_CURRENT_LIST_DIR}/omptest-targets.cmake")
+
+# Provide information, if ompTest has been built 'standalone'.
+set(LIBOMPTEST_BUILD_STANDALONE "@LIBOMPTEST_BUILD_STANDALONE@")
+
+# Note: The following variables are meant as a fallback.
+#       But eventually we may remove them in favor of linking via namespace.
+
+# Set LLVM installation root directory by using exported package information.
+set(LLVM_ROOT "${PACKAGE_PREFIX_DIR}")
+
+# Provide header and library paths.
+set(LIBOMP_HEADERS_INSTALL_PATH "${LLVM_ROOT}/@LIBOMP_HEADERS_INSTALL_PATH@")
+set(LIBOMP_LIBRARY_INSTALL_PATH "${LLVM_ROOT}/@OPENMP_INSTALL_LIBDIR@")
+set(omptest_INCLUDE_DIR         "${LIBOMP_HEADERS_INSTALL_PATH}/omptest")
+set(omptest_LIBRARY_DIR         "${LIBOMP_LIBRARY_INSTALL_PATH}")
+
+# Provide compiler default values.
+set(LLVM_BIN_INSTALL_DIR "${LLVM_ROOT}/bin")
+set(omptest_C_COMPILER   "${LLVM_BIN_INSTALL_DIR}/clang")
+set(omptest_CXX_COMPILER "${LLVM_BIN_INSTALL_DIR}/clang++")
diff --git a/openmp/tools/omptest/include/AssertMacros.h b/openmp/tools/omptest/include/AssertMacros.h
new file mode 100644
index 0000000000000..4e99ebdf98ad1
--- /dev/null
+++ b/openmp/tools/omptest/include/AssertMacros.h
@@ -0,0 +1,138 @@
+//===- AssertMacros.h - Macro aliases for ease-of-use -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provides macros to be used in unit tests for OMPT events.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_ASSERTMACROS_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_ASSERTMACROS_H
+
+#define OMPTEST_EXCLUDED_EVENT omptest::ObserveState::Never
+#define OMPTEST_REQUIRED_EVENT omptest::ObserveState::Always
+
+/// ASSERT MACROS TO BE USED BY THE USER
+
+#define OMPT_GENERATE_EVENTS(NumberOfCopies, EventMacro)                       \
+  for (size_t i = 0; i < NumberOfCopies; ++i) {                                \
+    EventMacro                                                                 \
+  }
+
+// Handle a minimum unordered set of events
+// Required events
+#define OMPT_ASSERT_SET_EVENT(Name, Group, EventTy, ...)                       \
+  SetAsserter->insert(OmptAssertEvent::EventTy(                                \
+      Name, Group, OMPTEST_REQUIRED_EVENT, __VA_ARGS__));
+#define OMPT_ASSERT_SET(EventTy, ...)                                          \
+  OMPT_ASSERT_SET_EVENT("", "", EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SET_GROUPED(Group, EventTy, ...)                           \
+  OMPT_ASSERT_SET_EVENT("", Group, EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SET_NAMED(Name, EventTy, ...)                              \
+  OMPT_ASSERT_SET_EVENT(Name, "", EventTy, __VA_ARGS__)
+// Excluded ("NOT") events
+#define OMPT_ASSERT_SET_EVENT_NOT(Name, Group, EventTy, ...)                   \
+  SetAsserter->insert(OmptAssertEvent::EventTy(                                \
+      Name, Group, OMPTEST_EXCLUDED_EVENT, __VA_ARGS__));
+#define OMPT_ASSERT_SET_NOT(EventTy, ...)                                      \
+  OMPT_ASSERT_SET_EVENT_NOT("", "", EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SET_GROUPED_NOT(Group, EventTy, ...)                       \
+  OMPT_ASSERT_SET_EVENT_NOT("", Group, EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SET_NAMED_NOT(Name, EventTy, ...)                          \
+  OMPT_ASSERT_SET_EVENT_NOT(Name, "", EventTy, __VA_ARGS__)
+
+// Handle an exact sequence of events
+// Required events
+#define OMPT_ASSERT_SEQUENCE_EVENT(Name, Group, EventTy, ...)                  \
+  SequenceAsserter->insert(OmptAssertEvent::EventTy(                           \
+      Name, Group, OMPTEST_REQUIRED_EVENT, __VA_ARGS__));
+#define OMPT_ASSERT_SEQUENCE(EventTy, ...)                                     \
+  OMPT_ASSERT_SEQUENCE_EVENT("", "", EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SEQUENCE_GROUPED(Group, EventTy, ...)                      \
+  OMPT_ASSERT_SEQUENCE_EVENT("", Group, EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SEQUENCE_NAMED(Name, EventTy, ...)                         \
+  OMPT_ASSERT_SEQUENCE_EVENT(Name, "", EventTy, __VA_ARGS__)
+// Excluded ("NOT") events
+#define OMPT_ASSERT_SEQUENCE_EVENT_NOT(Name, Group, EventTy, ...)              \
+  SequenceAsserter->insert(OmptAssertEvent::EventTy(                           \
+      Name, Group, OMPTEST_EXCLUDED_EVENT, __VA_ARGS__));
+#define OMPT_ASSERT_SEQUENCE_NOT(EventTy, ...)                                 \
+  OMPT_ASSERT_SEQUENCE_EVENT_NOT("", "", EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SEQUENCE_GROUPED_NOT(Group, EventTy, ...)                  \
+  OMPT_ASSERT_SEQUENCE_EVENT_NOT("", Group, EventTy, __VA_ARGS__)
+#define OMPT_ASSERT_SEQUENCE_NAMED_NOT(Name, EventTy, ...)                     \
+  OMPT_ASSERT_SEQUENCE_EVENT_NOT(Name, "", EventTy, __VA_ARGS__)
+// Special command: suspend active assertion
+// The created event is not correlated to any observed event
+#define OMPT_ASSERT_SEQUENCE_SUSPEND()                                         \
+  SequenceAsserter->insert(                                                    \
+      OmptAssertEvent::AssertionSuspend("", "", OMPTEST_EXCLUDED_EVENT));
+#define OMPT_ASSERT_SEQUENCE_ONLY(EventTy, ...)                                \
+  OMPT_ASSERT_SEQUENCE_SUSPEND()                                               \
+  OMPT_ASSERT_SEQUENCE_EVENT("", "", EventTy, __VA_ARGS__)                     \
+  OMPT_ASSERT_SEQUENCE_SUSPEND()
+#define OMPT_ASSERT_SEQUENCE_GROUPED_ONLY(Group, EventTy, ...)                 \
+  OMPT_ASSERT_SEQUENCE_SUSPEND()                                               \
+  OMPT_ASSERT_SEQUENCE_EVENT("", Group, EventTy, __VA_ARGS__)                  \
+  OMPT_ASSERT_SEQUENCE_SUSPEND()
+#define OMPT_ASSERT_SEQUENCE_NAMED_ONLY(Name, EventTy, ...)                    \
+  OMPT_ASSERT_SEQUENCE_SUSPEND()                                               \
+  OMPT_ASSERT_SEQUENCE_EVENT(Name, "", EventTy, __VA_ARGS__)                   \
+  OMPT_ASSERT_SEQUENCE_SUSPEND()
+
+#define OMPT_ASSERTER_MODE_STRICT(Asserter)                                    \
+  Asserter->setOperationMode(AssertMode::Strict);
+#define OMPT_ASSERTER_MODE_RELAXED(Asserter)                                   \
+  Asserter->setOperationMode(AssertMode::Relaxed);
+#define OMPT_ASSERT_SEQUENCE_MODE_STRICT()                                     \
+  OMPT_ASSERTER_MODE_STRICT(SequenceAsserter)
+#define OMPT_ASSERT_SEQUENCE_MODE_RELAXED()                                    \
+  OMPT_ASSERTER_MODE_RELAXED(SequenceAsserter)
+#define OMPT_ASSERT_SET_MODE_STRICT() OMPT_ASSERTER_MODE_STRICT(SetAsserter)
+#define OMPT_ASSERT_SET_MODE_RELAXED() OMPT_ASSERTER_MODE_RELAXED(SetAsserter)
+
+// Enable / disable asserters entirely
+#define OMPT_ASSERTER_DISABLE(Asserter) Asserter->setActive(false);
+#define OMPT_ASSERTER_ENABLE(Asserter) Asserter->setActive(true);
+#define OMPT_ASSERT_SET_DISABLE() OMPT_ASSERTER_DISABLE(SetAsserter)
+#define OMPT_ASSERT_SET_ENABLE() OMPT_ASSERTER_ENABLE(SetAsserter)
+#define OMPT_ASSERT_SEQUENCE_DISABLE() OMPT_ASSERTER_DISABLE(SequenceAsserter)
+#define OMPT_ASSERT_SEQUENCE_ENABLE() OMPT_ASSERTER_ENABLE(SequenceAsserter)
+#define OMPT_REPORT_EVENT_DISABLE() OMPT_ASSERTER_DISABLE(EventReporter)
+#define OMPT_REPORT_EVENT_ENABLE() OMPT_ASSERTER_ENABLE(EventReporter)
+
+// Enable / disable certain event types for asserters
+#define OMPT_ASSERTER_PERMIT_EVENT(Asserter, EventTy)                          \
+  Asserter->permitEvent(EventTy);
+#define OMPT_ASSERTER_SUPPRESS_EVENT(Asserter, EventTy)                        \
+  Asserter->suppressEvent(EventTy);
+#define OMPT_PERMIT_EVENT(EventTy)                                             \
+  OMPT_ASSERTER_PERMIT_EVENT(SetAsserter, EventTy);                            \
+  OMPT_ASSERTER_PERMIT_EVENT(EventReporter, EventTy);                          \
+  OMPT_ASSERTER_PERMIT_EVENT(SequenceAsserter, EventTy);
+#define OMPT_SUPPRESS_EVENT(EventTy)                                           \
+  OMPT_ASSERTER_SUPPRESS_EVENT(SetAsserter, EventTy);                          \
+  OMPT_ASSERTER_SUPPRESS_EVENT(EventReporter, EventTy);                        \
+  OMPT_ASSERTER_SUPPRESS_EVENT(SequenceAsserter, EventTy);
+
+// Set logging level for asserters
+// Note: Logger is a singleton, hence this will affect all asserter instances
+#define OMPT_ASSERTER_LOG_LEVEL(Asserter, LogLevel)                            \
+  Asserter->getLog()->setLoggingLevel(LogLevel);
+
+// Set log formatting (esp. coloring) for asserters
+// Note: Logger is a singleton, hence this will affect all asserter instances
+#define OMPT_ASSERTER_LOG_FORMATTED(Asserter, FormatLog)                       \
+  Asserter->getLog()->setFormatOutput(FormatLog);
+
+// SyncPoint handling
+#define OMPT_ASSERT_SYNC_POINT(SyncPointName)                                  \
+  flush_traced_devices();                                                      \
+  OmptCallbackHandler::get().handleAssertionSyncPoint(SyncPointName);
+
+#endif
diff --git a/openmp/tools/omptest/include/InternalEvent.h b/openmp/tools/omptest/include/InternalEvent.h
new file mode 100644
index 0000000000000..1348c48f72005
--- /dev/null
+++ b/openmp/tools/omptest/include/InternalEvent.h
@@ -0,0 +1,436 @@
+//===- InternalEvent.h - Internal event representation ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Declares internal event representations along the default CTOR definition.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_INTERNALEVENT_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_INTERNALEVENT_H
+
+#include "InternalEventCommon.h"
+
+#include <cstring>
+#include <limits>
+#include <omp-tools.h>
+
+#define expectedDefault(TypeName) std::numeric_limits<TypeName>::min()
+
+namespace omptest {
+
+namespace util {
+
+/// String manipulation helper function. Takes up to 8 bytes of data and returns
+/// their hexadecimal representation as string. The data can be expanded to the
+/// given size in bytes and will by default be prefixed with '0x'.
+std::string makeHexString(uint64_t Data, bool IsPointer = true,
+                          size_t DataBytes = 0, bool ShowHexBase = true);
+
+} // namespace util
+
+namespace internal {
+struct AssertionSyncPoint : public EventBase<AssertionSyncPoint> {
+  std::string toString() const override;
+  AssertionSyncPoint(const std::string &Name) : Name(Name) {}
+  const std::string Name;
+};
+
+struct AssertionSuspend : public EventBase<AssertionSuspend> {
+  AssertionSuspend() = default;
+};
+
+struct ThreadBegin : public EventBase<ThreadBegin> {
+  std::string toString() const override;
+  ThreadBegin(ompt_thread_t ThreadType) : ThreadType(ThreadType) {}
+  ompt_thread_t ThreadType;
+};
+
+struct ThreadEnd : public EventBase<ThreadEnd> {
+  std::string toString() const override;
+  ThreadEnd() = default;
+};
+
+struct ParallelBegin : public EventBase<ParallelBegin> {
+  std::string toString() const override;
+  ParallelBegin(int NumThreads) : NumThreads(NumThreads) {}
+  unsigned int NumThreads;
+};
+
+struct ParallelEnd : public EventBase<ParallelEnd> {
+  std::string toString() const override;
+  ParallelEnd(ompt_data_t *ParallelData, ompt_data_t *EncounteringTaskData,
+              int Flags, const void *CodeptrRA)
+      : ParallelData(ParallelData), EncounteringTaskData(EncounteringTaskData),
+        Flags(Flags), CodeptrRA(CodeptrRA) {}
+  ompt_data_t *ParallelData;
+  ompt_data_t *EncounteringTaskData;
+  int Flags;
+  const void *CodeptrRA;
+};
+
+struct Work : public EventBase<Work> {
+  std::string toString() const override;
+  Work(ompt_work_t WorkType, ompt_scope_endpoint_t Endpoint,
+       ompt_data_t *ParallelData, ompt_data_t *TaskData, uint64_t Count,
+       const void *CodeptrRA)
+      : WorkType(WorkType), Endpoint(Endpoint), ParallelData(ParallelData),
+        TaskData(TaskData), Count(Count), CodeptrRA(CodeptrRA) {}
+  ompt_work_t WorkType;
+  ompt_scope_endpoint_t Endpoint;
+  ompt_data_t *ParallelData;
+  ompt_data_t *TaskData;
+  uint64_t Count;
+  const void *CodeptrRA;
+};
+
+struct Dispatch : public EventBase<Dispatch> {
+  std::string toString() const override;
+  Dispatch(ompt_data_t *ParallelData, ompt_data_t *TaskData,
+           ompt_dispatch_t Kind, ompt_data_t Instance)
+      : ParallelData(ParallelData), TaskData(TaskData), Kind(Kind),
+        Instance(Instance) {}
+  ompt_data_t *ParallelData;
+  ompt_data_t *TaskData;
+  ompt_dispatch_t Kind;
+  ompt_data_t Instance;
+};
+
+struct TaskCreate : public EventBase<TaskCreate> {
+  std::string toString() const override;
+  TaskCreate(ompt_data_t *EncounteringTaskData,
+             const ompt_frame_t *EncounteringTaskFrame,
+             ompt_data_t *NewTaskData, int Flags, int HasDependences,
+             const void *CodeptrRA)
+      : EncounteringTaskData(EncounteringTaskData),
+        EncounteringTaskFrame(EncounteringTaskFrame), NewTaskData(NewTaskData),
+        Flags(Flags), HasDependences(HasDependences), CodeptrRA(CodeptrRA) {}
+  ompt_data_t *EncounteringTaskData;
+  const ompt_frame_t *EncounteringTaskFrame;
+  ompt_data_t *NewTaskData;
+  int Flags;
+  int HasDependences;
+  const void *CodeptrRA;
+};
+
+struct Dependences : public EventBase<Dependences> {
+  Dependences() = default;
+};
+
+struct TaskDependence : public EventBase<TaskDependence> {
+  TaskDependence() = default;
+};
+
+struct TaskSchedule : public EventBase<TaskSchedule> {
+  TaskSchedule() = default;
+};
+
+struct ImplicitTask : public EventBase<ImplicitTask> {
+  std::string toString() const override;
+  ImplicitTask(ompt_scope_endpoint_t Endpoint, ompt_data_t *ParallelData,
+               ompt_data_t *TaskData, unsigned int ActualParallelism,
+               unsigned int Index, int Flags)
+      : Endpoint(Endpoint), ParallelData(ParallelData), TaskData(TaskData),
+        ActualParallelism(ActualParallelism), Index(Index), Flags(Flags) {}
+  ompt_scope_endpoint_t Endpoint;
+  ompt_data_t *ParallelData;
+  ompt_data_t *TaskData;
+  unsigned int ActualParallelism;
+  unsigned int Index;
+  int Flags;
+};
+
+struct Masked : public EventBase<Masked> {
+  Masked() = default;
+};
+
+struct SyncRegion : public EventBase<SyncRegion> {
+  std::string toString() const override;
+  SyncRegion(ompt_sync_region_t Kind, ompt_scope_endpoint_t Endpoint,
+             ompt_data_t *ParallelData, ompt_data_t *TaskData,
+             const void *CodeptrRA)
+      : Kind(Kind), Endpoint(Endpoint), ParallelData(ParallelData),
+        TaskData(TaskData), CodeptrRA(CodeptrRA) {}
+  ompt_sync_region_t Kind;
+  ompt_scope_endpoint_t Endpoint;
+  ompt_data_t *ParallelData;
+  ompt_data_t *TaskData;
+  const void *CodeptrRA;
+};
+
+struct MutexAcquire : public EventBase<MutexAcquire> {
+  MutexAcquire() = default;
+};
+
+struct Mutex : public EventBase<Mutex> {
+  Mutex() = default;
+};
+
+struct NestLock : public EventBase<NestLock> {
+  NestLock() = default;
+};
+
+struct Flush : public EventBase<Flush> {
+  Flush() = default;
+};
+
+struct Cancel : public EventBase<Cancel> {
+  Cancel() = default;
+};
+
+struct Target : public EventBase<Target> {
+  std::string toString() const override;
+  Target(ompt_target_t Kind, ompt_scope_endpoint_t Endpoint, int DeviceNum,
+         ompt_data_t *TaskData, ompt_id_t TargetId, const void *CodeptrRA)
+      : Kind(Kind), Endpoint(Endpoint), DeviceNum(DeviceNum),
+        TaskData(TaskData), TargetId(TargetId), CodeptrRA(CodeptrRA) {}
+  ompt_target_t Kind;
+  ompt_scope_endpoint_t Endpoint;
+  int DeviceNum;
+  ompt_data_t *TaskData;
+  ompt_id_t TargetId;
+  const void *CodeptrRA;
+};
+
+struct TargetEmi : public EventBase<TargetEmi> {
+  std::string toString() const override;
+  TargetEmi(ompt_target_t Kind, ompt_scope_endpoint_t Endpoint, int DeviceNum,
+            ompt_data_t *TaskData, ompt_data_t *TargetTaskData,
+            ompt_data_t *TargetData, const void *CodeptrRA)
+      : Kind(Kind), Endpoint(Endpoint), DeviceNum(DeviceNum),
+        TaskData(TaskData), TargetTaskData(TargetTaskData),
+        TargetData(TargetData), CodeptrRA(CodeptrRA) {}
+  ompt_target_t Kind;
+  ompt_scope_endpoint_t Endpoint;
+  int DeviceNum;
+  ompt_data_t *TaskData;
+  ompt_data_t *TargetTaskData;
+  ompt_data_t *TargetData;
+  const void *CodeptrRA;
+};
+
+struct TargetDataOp : public EventBase<TargetDataOp> {
+  std::string toString() const override;
+  TargetDataOp(ompt_id_t TargetId, ompt_id_t HostOpId,
+               ompt_target_data_op_t OpType, void *SrcAddr, int SrcDeviceNum,
+               void *DstAddr, int DstDeviceNum, size_t Bytes,
+               const void *CodeptrRA)
+      : TargetId(TargetId), HostOpId(HostOpId), OpType(OpType),
+        SrcAddr(SrcAddr), SrcDeviceNum(SrcDeviceNum), DstAddr(DstAddr),
+        DstDeviceNum(DstDeviceNum), Bytes(Bytes), CodeptrRA(CodeptrRA) {}
+  ompt_id_t TargetId;
+  ompt_id_t HostOpId;
+  ompt_target_data_op_t OpType;
+  void *SrcAddr;
+  int SrcDeviceNum;
+  void *DstAddr;
+  int DstDeviceNum;
+  size_t Bytes;
+  const void *CodeptrRA;
+};
+
+struct TargetDataOpEmi : public EventBase<TargetDataOpEmi> {
+  std::string toString() const override;
+  TargetDataOpEmi(ompt_scope_endpoint_t Endpoint, ompt_data_t *TargetTaskData,
+                  ompt_data_t *TargetData, ompt_id_t *HostOpId,
+                  ompt_target_data_op_t OpType, void *SrcAddr, int SrcDeviceNum,
+                  void *DstAddr, int DstDeviceNum, size_t Bytes,
+                  const void *CodeptrRA)
+      : Endpoint(Endpoint), TargetTaskData(TargetTaskData),
+        TargetData(TargetData), HostOpId(HostOpId), OpType(OpType),
+        SrcAddr(SrcAddr), SrcDeviceNum(SrcDeviceNum), DstAddr(DstAddr),
+        DstDeviceNum(DstDeviceNum), Bytes(Bytes), CodeptrRA(CodeptrRA) {}
+  ompt_scope_endpoint_t Endpoint;
+  ompt_data_t *TargetTaskData;
+  ompt_data_t *TargetData;
+  ompt_id_t *HostOpId;
+  ompt_target_data_op_t OpType;
+  void *SrcAddr;
+  int SrcDeviceNum;
+  void *DstAddr;
+  int DstDeviceNum;
+  size_t Bytes;
+  const void *CodeptrRA;
+};
+
+struct TargetSubmit : public EventBase<TargetSubmit> {
+  std::string toString() const override;
+  TargetSubmit(ompt_id_t TargetId, ompt_id_t HostOpId,
+               unsigned int RequestedNumTeams)
+      : TargetId(TargetId), HostOpId(HostOpId),
+        RequestedNumTeams(RequestedNumTeams) {}
+  ompt_id_t TargetId;
+  ompt_id_t HostOpId;
+  unsigned int RequestedNumTeams;
+};
+
+struct TargetSubmitEmi : public EventBase<TargetSubmitEmi> {
+  std::string toString() const override;
+  TargetSubmitEmi(ompt_scope_endpoint_t Endpoint, ompt_data_t *TargetData,
+                  ompt_id_t *HostOpId, unsigned int RequestedNumTeams)
+      : Endpoint(Endpoint), TargetData(TargetData), HostOpId(HostOpId),
+        RequestedNumTeams(RequestedNumTeams) {}
+  ompt_scope_endpoint_t Endpoint;
+  ompt_data_t *TargetData;
+  ompt_id_t *HostOpId;
+  unsigned int RequestedNumTeams;
+};
+
+struct ControlTool : public EventBase<ControlTool> {
+  ControlTool() = default;
+};
+
+struct DeviceInitialize : public EventBase<DeviceInitialize> {
+  std::string toString() const override;
+  DeviceInitialize(int DeviceNum, const char *Type, ompt_device_t *Device,
+                   ompt_function_lookup_t LookupFn, const char *DocStr)
+      : DeviceNum(DeviceNum), Type(Type), Device(Device), LookupFn(LookupFn),
+        DocStr(DocStr) {}
+  int DeviceNum;
+  const char *Type;
+  ompt_device_t *Device;
+  ompt_function_lookup_t LookupFn;
+  const char *DocStr;
+};
+
+struct DeviceFinalize : public EventBase<DeviceFinalize> {
+  std::string toString() const override;
+  DeviceFinalize(int DeviceNum) : DeviceNum(DeviceNum) {}
+  int DeviceNum;
+};
+
+struct DeviceLoad : public EventBase<DeviceLoad> {
+  std::string toString() const override;
+  DeviceLoad(int DeviceNum, const char *Filename, int64_t OffsetInFile,
+             void *VmaInFile, size_t Bytes, void *HostAddr, void *DeviceAddr,
+             uint64_t ModuleId)
+      : DeviceNum(DeviceNum), Filename(Filename), OffsetInFile(OffsetInFile),
+        VmaInFile(VmaInFile), Bytes(Bytes), HostAddr(HostAddr),
+        DeviceAddr(DeviceAddr), ModuleId(ModuleId) {}
+  int DeviceNum;
+  const char *Filename;
+  int64_t OffsetInFile;
+  void *VmaInFile;
+  size_t Bytes;
+  void *HostAddr;
+  void *DeviceAddr;
+  uint64_t ModuleId;
+};
+
+struct DeviceUnload : public EventBase<DeviceUnload> {
+  DeviceUnload() = default;
+};
+
+struct BufferRequest : public EventBase<BufferRequest> {
+  std::string toString() const override;
+  BufferRequest(int DeviceNum, ompt_buffer_t **Buffer, size_t *Bytes)
+      : DeviceNum(DeviceNum), Buffer(Buffer), Bytes(Bytes) {}
+  int DeviceNum;
+  ompt_buffer_t **Buffer;
+  size_t *Bytes;
+};
+
+struct BufferComplete : public EventBase<BufferComplete> {
+  std::string toString() const override;
+  BufferComplete(int DeviceNum, ompt_buffer_t *Buffer, size_t Bytes,
+                 ompt_buffer_cursor_t Begin, int BufferOwned)
+      : DeviceNum(DeviceNum), Buffer(Buffer), Bytes(Bytes), Begin(Begin),
+        BufferOwned(BufferOwned) {}
+  int DeviceNum;
+  ompt_buffer_t *Buffer;
+  size_t Bytes;
+  ompt_buffer_cursor_t Begin;
+  int BufferOwned;
+};
+
+struct BufferRecord : public EventBase<BufferRecord> {
+  std::string toString() const override;
+  BufferRecord(ompt_record_ompt_t *RecordPtr) : RecordPtr(RecordPtr) {
+    if (RecordPtr != nullptr)
+      Record = *RecordPtr;
+    else
+      memset(&Record, 0, sizeof(ompt_record_ompt_t));
+  }
+  ompt_record_ompt_t Record;
+  ompt_record_ompt_t *RecordPtr;
+};
+
+struct BufferRecordDeallocation : public EventBase<BufferRecordDeallocation> {
+  std::string toString() const override;
+  BufferRecordDeallocation(ompt_buffer_t *Buffer) : Buffer(Buffer) {}
+  ompt_buffer_t *Buffer;
+};
+
+// Add specialized event equality operators here.
+// Note: Placement of these forward declarations is important as they need to
+// take precedence over the following default equality operator definition.
+bool operator==(const ParallelBegin &, const ParallelBegin &);
+bool operator==(const Work &, const Work &);
+bool operator==(const ImplicitTask &, const ImplicitTask &);
+bool operator==(const SyncRegion &, const SyncRegion &);
+bool operator==(const Target &, const Target &);
+bool operator==(const TargetEmi &, const TargetEmi &);
+bool operator==(const TargetDataOp &, const TargetDataOp &);
+bool operator==(const TargetDataOpEmi &, const TargetDataOpEmi &);
+bool operator==(const TargetSubmit &, const TargetSubmit &);
+bool operator==(const TargetSubmitEmi &, const TargetSubmitEmi &);
+bool operator==(const DeviceInitialize &, const DeviceInitialize &);
+bool operator==(const DeviceFinalize &, const DeviceFinalize &);
+bool operator==(const DeviceLoad &, const DeviceLoad &);
+bool operator==(const BufferRequest &, const BufferRequest &);
+bool operator==(const BufferComplete &, const BufferComplete &);
+bool operator==(const BufferRecord &, const BufferRecord &);
+
+/// Default (fallback) event equality operator definition.
+template <typename Event> bool operator==(const Event &, const Event &) {
+  return true;
+}
+
+// clang-format off
+event_type_trait(AssertionSyncPoint)
+event_type_trait(AssertionSuspend)
+event_type_trait(ThreadBegin)
+event_type_trait(ThreadEnd)
+event_type_trait(ParallelBegin)
+event_type_trait(ParallelEnd)
+event_type_trait(Work)
+event_type_trait(Dispatch)
+event_type_trait(TaskCreate)
+event_type_trait(Dependences)
+event_type_trait(TaskDependence)
+event_type_trait(TaskSchedule)
+event_type_trait(ImplicitTask)
+event_type_trait(Masked)
+event_type_trait(SyncRegion)
+event_type_trait(MutexAcquire)
+event_type_trait(Mutex)
+event_type_trait(NestLock)
+event_type_trait(Flush)
+event_type_trait(Cancel)
+event_type_trait(Target)
+event_type_trait(TargetEmi)
+event_type_trait(TargetDataOp)
+event_type_trait(TargetDataOpEmi)
+event_type_trait(TargetSubmit)
+event_type_trait(TargetSubmitEmi)
+event_type_trait(ControlTool)
+event_type_trait(DeviceInitialize)
+event_type_trait(DeviceFinalize)
+event_type_trait(DeviceLoad)
+event_type_trait(DeviceUnload)
+event_type_trait(BufferRequest)
+event_type_trait(BufferComplete)
+event_type_trait(BufferRecord)
+event_type_trait(BufferRecordDeallocation)
+// clang-format on
+
+} // namespace internal
+
+} // namespace omptest
+
+#endif
diff --git a/openmp/tools/omptest/include/InternalEventCommon.h b/openmp/tools/omptest/include/InternalEventCommon.h
new file mode 100644
index 0000000000000..8b2b7dbc3fedd
--- /dev/null
+++ b/openmp/tools/omptest/include/InternalEventCommon.h
@@ -0,0 +1,130 @@
+//===- InternalEventCommon.h - Common internal event basics -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provides event types, and class/operator declaration macros.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_INTERNALEVENTCOMMON_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_INTERNALEVENTCOMMON_H
+
+#include "omp-tools.h"
+
+#include <cassert>
+#include <string>
+
+namespace omptest {
+
+namespace internal {
+/// Enum values are used for comparison of observed and asserted events
+/// List is based on OpenMP 5.2 specification, table 19.2 (page 447)
+enum class EventTy {
+  None,                     // not part of OpenMP spec, used for implementation
+  AssertionSyncPoint,       // not part of OpenMP spec, used for implementation
+  AssertionSuspend,         // not part of OpenMP spec, used for implementation
+  BufferRecord,             // not part of OpenMP spec, used for implementation
+  BufferRecordDeallocation, // not part of OpenMP spec, used for implementation
+  ThreadBegin,
+  ThreadEnd,
+  ParallelBegin,
+  ParallelEnd,
+  Work,
+  Dispatch,
+  TaskCreate,     // TODO: Implement
+  Dependences,    // TODO: Implement
+  TaskDependence, // TODO: Implement
+  TaskSchedule,   // TODO: Implement
+  ImplicitTask,   // TODO: Implement
+  Masked,         // TODO: Implement
+  SyncRegion,
+  MutexAcquire, // TODO: Implement
+  Mutex,        // TODO: Implement
+  NestLock,     // TODO: Implement
+  Flush,        // TODO: Implement
+  Cancel,       // TODO: Implement
+  DeviceInitialize,
+  DeviceFinalize,
+  DeviceLoad,
+  DeviceUnload,
+  BufferRequest,
+  BufferComplete,
+  TargetDataOp,
+  TargetDataOpEmi,
+  Target,
+  TargetEmi,
+  TargetSubmit,
+  TargetSubmitEmi,
+  ControlTool
+};
+
+/// Base event class
+/// Offers default CTOR, DTOR and CTOR which assigns the actual event type.
+struct InternalEvent {
+  InternalEvent() : Type(EventTy::None) {}
+  InternalEvent(EventTy T) : Type(T) {}
+  virtual ~InternalEvent() = default;
+
+  virtual bool equals(const InternalEvent *o) const {
+    assert(false && "Base class implementation");
+    return false;
+  };
+
+  virtual std::string toString() const {
+    std::string S{"InternalEvent: Type="};
+    S.append(std::to_string((uint32_t)Type));
+    return S;
+  }
+
+  /// Identifying event type
+  EventTy Type;
+};
+
+/// Specialize EventType member for each derived internal event type.
+/// Effectively selecting an event type as initialization value.
+template <typename EventType> struct EventTypeOf;
+
+/// Actual definition macro for EventTypeOf.
+#define event_type_trait(EvTy)                                                 \
+  template <> struct EventTypeOf<EvTy> {                                       \
+    static constexpr EventTy Value = EventTy::EvTy;                            \
+  };
+
+/// CRTP (Curiously Recurring Template Pattern) intermediate class
+/// Adding a new event type can be achieved by inheriting from an EventBase
+/// template instantiation of the new class' name, like this:
+/// struct NewEventType : public EventBase<NewEventType>
+template <typename Derived> class EventBase : public InternalEvent {
+public:
+  static constexpr EventTy EventType = EventTypeOf<Derived>::Value;
+  EventBase() : InternalEvent(EventType) {}
+  virtual ~EventBase() = default;
+
+  /// Equals method to cast and dispatch to the specific class operator==
+  virtual bool equals(const InternalEvent *o) const override {
+    // Note: When the if-condition evaluates to true, the event types are
+    // trivially identical. Otherwise, a cast to the Derived pointer would have
+    // been impossible.
+    if (const auto Other = dynamic_cast<const Derived *>(o))
+      return operator==(*static_cast<const Derived *>(this), *Other);
+    return false;
+  }
+
+  /// Basic toString method, which may be overridden with own implementations.
+  virtual std::string toString() const override {
+    std::string S{"EventBase: Type="};
+    S.append(std::to_string((uint32_t)Type));
+    return S;
+  }
+};
+
+} // namespace internal
+
+} // namespace omptest
+
+#endif
diff --git a/openmp/tools/omptest/include/Logging.h b/openmp/tools/omptest/include/Logging.h
new file mode 100644
index 0000000000000..a8a1333795859
--- /dev/null
+++ b/openmp/tools/omptest/include/Logging.h
@@ -0,0 +1,155 @@
+//===- Logging.h - General logging class ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Provides ompTest-tailored logging, with log-levels and formatting/coloring.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_LOGGING_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_LOGGING_H
+
+#include "OmptAssertEvent.h"
+
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <set>
+#include <sstream>
+#include <string>
+
+namespace omptest {
+namespace logging {
+
+enum class Level : uint32_t {
+  // Levels (Note: DEBUG may already be reserved)
+  Diagnostic = 10,
+  Info = 20,
+  Warning = 30,
+  Error = 40,
+  Critical = 50,
+
+  // Types used for formatting options
+  Default,
+  ExpectedEvent,
+  ObservedEvent,
+  OffendingEvent,
+
+  // Suppress all prints
+  Silent = 0xFFFFFFFF
+};
+
+enum class FormatOption : uint32_t {
+  // General options
+  // Note: Bold is actually "BRIGHT" -- But it will be perceived as 'bold' font
+  //       It is implicitly switching colors to the 'Light' variant
+  //       Thus, it has -NO EFFECT- when already using a Light* color
+  None = 0,
+  Bold = 1,
+  Dim = 2,
+  Underlined = 4,
+  Blink = 5,
+  Inverted = 7,
+  Hidden = 8,
+  // Foreground colors
+  ColorDefault = 39,
+  ColorBlack = 30,
+  ColorRed = 31,
+  ColorGreen = 32,
+  ColorYellow = 33,
+  ColorBlue = 34,
+  ColorMagenta = 35,
+  ColorCyan = 36,
+  ColorLightGray = 37,
+  ColorDarkGray = 90,
+  ColorLightRed = 91,
+  ColorLightGreen = 92,
+  ColorLightYellow = 93,
+  ColorLightBlue = 94,
+  ColorLightMagenta = 95,
+  ColorLightCyan = 96,
+  ColorWhite = 97,
+  // Background colors
+  ColorBackgroundDefault = 49,
+  ColorBackgroundBlack = 40,
+  ColorBackgroundRed = 41,
+  ColorBackgroundGreen = 42,
+  ColorBackgroundYellow = 43,
+  ColorBackgroundBlue = 44,
+  ColorBackgroundMagenta = 45,
+  ColorBackgroundCyan = 46,
+  ColorBackgroundLightGray = 47,
+  ColorBackgroundDarkGray = 100,
+  ColorBackgroundLightRed = 101,
+  ColorBackgroundLightGreen = 102,
+  ColorBackgroundLightYellow = 103,
+  ColorBackgroundLightBlue = 104,
+  ColorBackgroundLightMagenta = 105,
+  ColorBackgroundLightCyan = 106,
+  ColorBackgroundWhite = 107
+};
+
+/// Returns a string representation of the given logging level.
+const char *to_string(Level LogLevel);
+
+/// Returns the format options as escaped sequence, for the given logging level
+std::string getFormatSequence(Level LogLevel = Level::Default);
+
+/// Format the given message with the provided option(s) and return it.
+/// Here formatting is only concerning control sequences using <Esc> character
+/// which can be obtained using '\e' (on console), '\033' or '\x1B'.
+std::string format(const std::string &Message, FormatOption Option);
+std::string format(const std::string &Message, std::set<FormatOption> Options);
+
+class Logger {
+public:
+  Logger(Level LogLevel = Level::Warning, std::ostream &OutStream = std::cerr,
+         bool FormatOutput = true);
+  ~Logger();
+
+  /// Log the given message to the output.
+  void log(const std::string &Message, Level LogLevel) const;
+
+  /// Log a single event mismatch.
+  void logEventMismatch(const std::string &Message,
+                        const omptest::OmptAssertEvent &OffendingEvent,
+                        Level LogLevel = Level::Error) const;
+
+  /// Log an event-pair mismatch.
+  void logEventMismatch(const std::string &Message,
+                        const omptest::OmptAssertEvent &ExpectedEvent,
+                        const omptest::OmptAssertEvent &ObservedEvent,
+                        Level LogLevel = Level::Error) const;
+
+  /// Set if output is being formatted (e.g. colored).
+  void setFormatOutput(bool Enabled);
+
+  /// Return the current (minimum) Logging Level.
+  Level getLoggingLevel() const;
+
+  /// Set the (minimum) Logging Level.
+  void setLoggingLevel(Level LogLevel);
+
+private:
+  /// The minimum logging level that is considered by the logger instance.
+  Level LoggingLevel;
+
+  /// The output stream used by the logger instance.
+  std::ostream &OutStream;
+
+  /// Determine if log messages are formatted using control sequences.
+  bool FormatOutput;
+
+  /// Mutex to ensure serialized logging
+  mutable std::mutex LogMutex;
+};
+
+} // namespace logging
+} // namespace omptest
+
+#endif
diff --git a/openmp/tools/omptest/include/OmptAliases.h b/openmp/tools/omptest/include/OmptAliases.h
new file mode 100644
index 0000000000000..500be5ef9f749
--- /dev/null
+++ b/openmp/tools/omptest/include/OmptAliases.h
@@ -0,0 +1,85 @@
+//===- OmptAliases.h - Shorthand aliases for OMPT enum values ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Defines shorthand aliases for OMPT enum values, providing improved
+/// ease-of-use and readability.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTALIASES_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTALIASES_H
+
+#include <omp-tools.h>
+
+/// Aliases for enum: ompt_scope_endpoint_t
+constexpr ompt_scope_endpoint_t BEGIN = ompt_scope_begin;
+constexpr ompt_scope_endpoint_t END = ompt_scope_end;
+constexpr ompt_scope_endpoint_t BEGINEND = ompt_scope_beginend;
+
+/// Aliases for enum: ompt_target_t
+constexpr ompt_target_t TARGET = ompt_target;
+constexpr ompt_target_t ENTER_DATA = ompt_target_enter_data;
+constexpr ompt_target_t EXIT_DATA = ompt_target_exit_data;
+constexpr ompt_target_t UPDATE = ompt_target_update;
+constexpr ompt_target_t TARGET_NOWAIT = ompt_target_nowait;
+constexpr ompt_target_t ENTER_DATA_NOWAIT = ompt_target_enter_data_nowait;
+constexpr ompt_target_t EXIT_DATA_NOWAIT = ompt_target_exit_data_nowait;
+constexpr ompt_target_t UPDATE_NOWAIT = ompt_target_update_nowait;
+
+/// Aliases for enum: ompt_target_data_op_t
+constexpr ompt_target_data_op_t ALLOC = ompt_target_data_alloc;
+constexpr ompt_target_data_op_t H2D = ompt_target_data_transfer_to_device;
+constexpr ompt_target_data_op_t D2H = ompt_target_data_transfer_from_device;
+constexpr ompt_target_data_op_t DELETE = ompt_target_data_delete;
+constexpr ompt_target_data_op_t ASSOCIATE = ompt_target_data_associate;
+constexpr ompt_target_data_op_t DISASSOCIATE = ompt_target_data_disassociate;
+constexpr ompt_target_data_op_t ALLOC_ASYNC = ompt_target_data_alloc_async;
+constexpr ompt_target_data_op_t H2D_ASYNC =
+    ompt_target_data_transfer_to_device_async;
+constexpr ompt_target_data_op_t D2H_ASYNC =
+    ompt_target_data_transfer_from_device_async;
+constexpr ompt_target_data_op_t DELETE_ASYNC = ompt_target_data_delete_async;
+
+/// Aliases for enum: ompt_callbacks_t (partial)
+constexpr ompt_callbacks_t CB_TARGET = ompt_callback_target;
+constexpr ompt_callbacks_t CB_DATAOP = ompt_callback_target_data_op;
+constexpr ompt_callbacks_t CB_KERNEL = ompt_callback_target_submit;
+
+/// Aliases for enum: ompt_work_t
+constexpr ompt_work_t WORK_LOOP = ompt_work_loop;
+constexpr ompt_work_t WORK_SECT = ompt_work_sections;
+constexpr ompt_work_t WORK_EXEC = ompt_work_single_executor;
+constexpr ompt_work_t WORK_SINGLE = ompt_work_single_other;
+constexpr ompt_work_t WORK_SHARE = ompt_work_workshare;
+constexpr ompt_work_t WORK_DIST = ompt_work_distribute;
+constexpr ompt_work_t WORK_TASK = ompt_work_taskloop;
+constexpr ompt_work_t WORK_SCOPE = ompt_work_scope;
+constexpr ompt_work_t WORK_LOOP_STA = ompt_work_loop_static;
+constexpr ompt_work_t WORK_LOOP_DYN = ompt_work_loop_dynamic;
+constexpr ompt_work_t WORK_LOOP_GUI = ompt_work_loop_guided;
+constexpr ompt_work_t WORK_LOOP_OTH = ompt_work_loop_other;
+
+/// Aliases for enum: ompt_sync_region_t
+constexpr ompt_sync_region_t SR_BARRIER = ompt_sync_region_barrier;
+constexpr ompt_sync_region_t SR_BARRIER_IMPL =
+    ompt_sync_region_barrier_implicit;
+constexpr ompt_sync_region_t SR_BARRIER_EXPL =
+    ompt_sync_region_barrier_explicit;
+constexpr ompt_sync_region_t SR_BARRIER_IMPLEMENTATION =
+    ompt_sync_region_barrier_implementation;
+constexpr ompt_sync_region_t SR_TASKWAIT = ompt_sync_region_taskwait;
+constexpr ompt_sync_region_t SR_TASKGROUP = ompt_sync_region_taskgroup;
+constexpr ompt_sync_region_t SR_REDUCTION = ompt_sync_region_reduction;
+constexpr ompt_sync_region_t SR_BARRIER_IMPL_WORKSHARE =
+    ompt_sync_region_barrier_implicit_workshare;
+constexpr ompt_sync_region_t SR_BARRIER_IMPL_PARALLEL =
+    ompt_sync_region_barrier_implicit_parallel;
+constexpr ompt_sync_region_t SR_BARRIER_TEAMS = ompt_sync_region_barrier_teams;
+
+#endif
diff --git a/openmp/tools/omptest/include/OmptAssertEvent.h b/openmp/tools/omptest/include/OmptAssertEvent.h
new file mode 100644
index 0000000000000..3165e2837d22e
--- /dev/null
+++ b/openmp/tools/omptest/include/OmptAssertEvent.h
@@ -0,0 +1,377 @@
+//===- OmptAssertEvent.h - Assertion event declarations ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Contains assertion event constructors, for generally all observable events.
+/// This includes user-generated events, like synchronization.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTASSERTEVENT_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTASSERTEVENT_H
+
+#include "InternalEvent.h"
+#include "omp-tools.h"
+
+#include <cassert>
+#include <limits>
+#include <memory>
+#include <string>
+
+namespace omptest {
+
+enum class ObserveState { Generated, Always, Never };
+
+/// Helper function, returning an ObserveState string representation
+const char *to_string(ObserveState State);
+
+/// Assertion event struct, provides statically callable CTORs.
+struct OmptAssertEvent {
+  static OmptAssertEvent AssertionSyncPoint(const std::string &Name,
+                                            const std::string &Group,
+                                            const ObserveState &Expected,
+                                            const std::string &SyncPointName);
+
+  static OmptAssertEvent AssertionSuspend(const std::string &Name,
+                                          const std::string &Group,
+                                          const ObserveState &Expected);
+
+  static OmptAssertEvent ThreadBegin(const std::string &Name,
+                                     const std::string &Group,
+                                     const ObserveState &Expected,
+                                     ompt_thread_t ThreadType);
+
+  static OmptAssertEvent ThreadEnd(const std::string &Name,
+                                   const std::string &Group,
+                                   const ObserveState &Expected);
+
+  static OmptAssertEvent ParallelBegin(const std::string &Name,
+                                       const std::string &Group,
+                                       const ObserveState &Expected,
+                                       int NumThreads);
+
+  static OmptAssertEvent ParallelEnd(
+      const std::string &Name, const std::string &Group,
+      const ObserveState &Expected,
+      ompt_data_t *ParallelData = expectedDefault(ompt_data_t *),
+      ompt_data_t *EncounteringTaskData = expectedDefault(ompt_data_t *),
+      int Flags = expectedDefault(int),
+      const void *CodeptrRA = expectedDefault(const void *));
+
+  static OmptAssertEvent
+  Work(const std::string &Name, const std::string &Group,
+       const ObserveState &Expected, ompt_work_t WorkType,
+       ompt_scope_endpoint_t Endpoint,
+       ompt_data_t *ParallelData = expectedDefault(ompt_data_t *),
+       ompt_data_t *TaskData = expectedDefault(ompt_data_t *),
+       uint64_t Count = expectedDefault(uint64_t),
+       const void *CodeptrRA = expectedDefault(const void *));
+
+  static OmptAssertEvent
+  Dispatch(const std::string &Name, const std::string &Group,
+           const ObserveState &Expected,
+           ompt_data_t *ParallelData = expectedDefault(ompt_data_t *),
+           ompt_data_t *TaskData = expectedDefault(ompt_data_t *),
+           ompt_dispatch_t Kind = expectedDefault(ompt_dispatch_t),
+           ompt_data_t Instance = expectedDefault(ompt_data_t));
+
+  static OmptAssertEvent
+  TaskCreate(const std::string &Name, const std::string &Group,
+             const ObserveState &Expected,
+             ompt_data_t *EncounteringTaskData = expectedDefault(ompt_data_t *),
+             const ompt_frame_t *EncounteringTaskFrame =
+                 expectedDefault(ompt_frame_t *),
+             ompt_data_t *NewTaskData = expectedDefault(ompt_data_t *),
+             int Flags = expectedDefault(int),
+             int HasDependences = expectedDefault(int),
+             const void *CodeptrRA = expectedDefault(const void *));
+
+  static OmptAssertEvent TaskSchedule(const std::string &Name,
+                                      const std::string &Group,
+                                      const ObserveState &Expected);
+
+  static OmptAssertEvent
+  ImplicitTask(const std::string &Name, const std::string &Group,
+               const ObserveState &Expected, ompt_scope_endpoint_t Endpoint,
+               ompt_data_t *ParallelData = expectedDefault(ompt_data_t *),
+               ompt_data_t *TaskData = expectedDefault(ompt_data_t *),
+               unsigned int ActualParallelism = expectedDefault(unsigned int),
+               unsigned int Index = expectedDefault(unsigned int),
+               int Flags = expectedDefault(int));
+
+  static OmptAssertEvent
+  SyncRegion(const std::string &Name, const std::string &Group,
+             const ObserveState &Expected, ompt_sync_region_t Kind,
+             ompt_scope_endpoint_t Endpoint,
+             ompt_data_t *ParallelData = expectedDefault(ompt_data_t *),
+             ompt_data_t *TaskData = expectedDefault(ompt_data_t *),
+             const void *CodeptrRA = expectedDefault(const void *));
+
+  static OmptAssertEvent
+  Target(const std::string &Name, const std::string &Group,
+         const ObserveState &Expected, ompt_target_t Kind,
+         ompt_scope_endpoint_t Endpoint, int DeviceNum = expectedDefault(int),
+         ompt_data_t *TaskData = expectedDefault(ompt_data_t *),
+         ompt_id_t TargetId = expectedDefault(ompt_id_t),
+         const void *CodeptrRA = expectedDefault(void *));
+
+  static OmptAssertEvent
+  TargetEmi(const std::string &Name, const std::string &Group,
+            const ObserveState &Expected, ompt_target_t Kind,
+            ompt_scope_endpoint_t Endpoint,
+            int DeviceNum = expectedDefault(int),
+            ompt_data_t *TaskData = expectedDefault(ompt_data_t *),
+            ompt_data_t *TargetTaskData = expectedDefault(ompt_data_t *),
+            ompt_data_t *TargetData = expectedDefault(ompt_data_t *),
+            const void *CodeptrRA = expectedDefault(void *));
+
+  static OmptAssertEvent
+  TargetDataOp(const std::string &Name, const std::string &Group,
+               const ObserveState &Expected, ompt_id_t TargetId,
+               ompt_id_t HostOpId, ompt_target_data_op_t OpType, void *SrcAddr,
+               int SrcDeviceNum, void *DstAddr, int DstDeviceNum, size_t Bytes,
+               const void *CodeptrRA);
+
+  static OmptAssertEvent
+  TargetDataOp(const std::string &Name, const std::string &Group,
+               const ObserveState &Expected, ompt_target_data_op_t OpType,
+               size_t Bytes = expectedDefault(size_t),
+               void *SrcAddr = expectedDefault(void *),
+               void *DstAddr = expectedDefault(void *),
+               int SrcDeviceNum = expectedDefault(int),
+               int DstDeviceNum = expectedDefault(int),
+               ompt_id_t TargetId = expectedDefault(ompt_id_t),
+               ompt_id_t HostOpId = expectedDefault(ompt_id_t),
+               const void *CodeptrRA = expectedDefault(void *));
+
+  static OmptAssertEvent
+  TargetDataOpEmi(const std::string &Name, const std::string &Group,
+                  const ObserveState &Expected, ompt_scope_endpoint_t Endpoint,
+                  ompt_data_t *TargetTaskData, ompt_data_t *TargetData,
+                  ompt_id_t *HostOpId, ompt_target_data_op_t OpType,
+                  void *SrcAddr, int SrcDeviceNum, void *DstAddr,
+                  int DstDeviceNum, size_t Bytes, const void *CodeptrRA);
+
+  static OmptAssertEvent
+  TargetDataOpEmi(const std::string &Name, const std::string &Group,
+                  const ObserveState &Expected, ompt_target_data_op_t OpType,
+                  ompt_scope_endpoint_t Endpoint,
+                  size_t Bytes = expectedDefault(size_t),
+                  void *SrcAddr = expectedDefault(void *),
+                  void *DstAddr = expectedDefault(void *),
+                  int SrcDeviceNum = expectedDefault(int),
+                  int DstDeviceNum = expectedDefault(int),
+                  ompt_data_t *TargetTaskData = expectedDefault(ompt_data_t *),
+                  ompt_data_t *TargetData = expectedDefault(ompt_data_t *),
+                  ompt_id_t *HostOpId = expectedDefault(ompt_id_t *),
+                  const void *CodeptrRA = expectedDefault(void *));
+
+  static OmptAssertEvent TargetSubmit(const std::string &Name,
+                                      const std::string &Group,
+                                      const ObserveState &Expected,
+                                      ompt_id_t TargetId, ompt_id_t HostOpId,
+                                      unsigned int RequestedNumTeams);
+
+  static OmptAssertEvent
+  TargetSubmit(const std::string &Name, const std::string &Group,
+               const ObserveState &Expected, unsigned int RequestedNumTeams,
+               ompt_id_t TargetId = expectedDefault(ompt_id_t),
+               ompt_id_t HostOpId = expectedDefault(ompt_id_t));
+
+  static OmptAssertEvent
+  TargetSubmitEmi(const std::string &Name, const std::string &Group,
+                  const ObserveState &Expected, ompt_scope_endpoint_t Endpoint,
+                  ompt_data_t *TargetData, ompt_id_t *HostOpId,
+                  unsigned int RequestedNumTeams);
+
+  static OmptAssertEvent
+  TargetSubmitEmi(const std::string &Name, const std::string &Group,
+                  const ObserveState &Expected, unsigned int RequestedNumTeams,
+                  ompt_scope_endpoint_t Endpoint,
+                  ompt_data_t *TargetData = expectedDefault(ompt_data_t *),
+                  ompt_id_t *HostOpId = expectedDefault(ompt_id_t *));
+
+  static OmptAssertEvent ControlTool(const std::string &Name,
+                                     const std::string &Group,
+                                     const ObserveState &Expected);
+
+  static OmptAssertEvent DeviceInitialize(
+      const std::string &Name, const std::string &Group,
+      const ObserveState &Expected, int DeviceNum,
+      const char *Type = expectedDefault(const char *),
+      ompt_device_t *Device = expectedDefault(ompt_device_t *),
+      ompt_function_lookup_t LookupFn = expectedDefault(ompt_function_lookup_t),
+      const char *DocumentationStr = expectedDefault(const char *));
+
+  static OmptAssertEvent DeviceFinalize(const std::string &Name,
+                                        const std::string &Group,
+                                        const ObserveState &Expected,
+                                        int DeviceNum);
+
+  static OmptAssertEvent
+  DeviceLoad(const std::string &Name, const std::string &Group,
+             const ObserveState &Expected, int DeviceNum,
+             const char *Filename = expectedDefault(const char *),
+             int64_t OffsetInFile = expectedDefault(int64_t),
+             void *VmaInFile = expectedDefault(void *),
+             size_t Bytes = expectedDefault(size_t),
+             void *HostAddr = expectedDefault(void *),
+             void *DeviceAddr = expectedDefault(void *),
+             uint64_t ModuleId = expectedDefault(int64_t));
+
+  static OmptAssertEvent DeviceUnload(const std::string &Name,
+                                      const std::string &Group,
+                                      const ObserveState &Expected);
+
+  static OmptAssertEvent BufferRequest(const std::string &Name,
+                                       const std::string &Group,
+                                       const ObserveState &Expected,
+                                       int DeviceNum, ompt_buffer_t **Buffer,
+                                       size_t *Bytes);
+
+  static OmptAssertEvent
+  BufferComplete(const std::string &Name, const std::string &Group,
+                 const ObserveState &Expected, int DeviceNum,
+                 ompt_buffer_t *Buffer, size_t Bytes,
+                 ompt_buffer_cursor_t Begin, int BufferOwned);
+
+  static OmptAssertEvent BufferRecord(const std::string &Name,
+                                      const std::string &Group,
+                                      const ObserveState &Expected,
+                                      ompt_record_ompt_t *Record);
+
+  /// Handle type = ompt_record_target_t
+  static OmptAssertEvent
+  BufferRecord(const std::string &Name, const std::string &Group,
+               const ObserveState &Expected, ompt_callbacks_t Type,
+               ompt_target_t Kind, ompt_scope_endpoint_t Endpoint,
+               int DeviceNum = expectedDefault(int),
+               ompt_id_t TaskId = expectedDefault(ompt_id_t),
+               ompt_id_t TargetId = expectedDefault(ompt_id_t),
+               const void *CodeptrRA = expectedDefault(void *));
+
+  /// Handle type = ompt_callback_target_data_op
+  static OmptAssertEvent
+  BufferRecord(const std::string &Name, const std::string &Group,
+               const ObserveState &Expected, ompt_callbacks_t Type,
+               ompt_target_data_op_t OpType, size_t Bytes,
+               std::pair<ompt_device_time_t, ompt_device_time_t> Timeframe,
+               void *SrcAddr = expectedDefault(void *),
+               void *DstAddr = expectedDefault(void *),
+               int SrcDeviceNum = expectedDefault(int),
+               int DstDeviceNum = expectedDefault(int),
+               ompt_id_t TargetId = expectedDefault(ompt_id_t),
+               ompt_id_t HostOpId = expectedDefault(ompt_id_t),
+               const void *CodeptrRA = expectedDefault(void *));
+
+  /// Handle type = ompt_callback_target_data_op
+  static OmptAssertEvent BufferRecord(
+      const std::string &Name, const std::string &Group,
+      const ObserveState &Expected, ompt_callbacks_t Type,
+      ompt_target_data_op_t OpType, size_t Bytes = expectedDefault(size_t),
+      ompt_device_time_t MinimumTimeDelta = expectedDefault(ompt_device_time_t),
+      void *SrcAddr = expectedDefault(void *),
+      void *DstAddr = expectedDefault(void *),
+      int SrcDeviceNum = expectedDefault(int),
+      int DstDeviceNum = expectedDefault(int),
+      ompt_id_t TargetId = expectedDefault(ompt_id_t),
+      ompt_id_t HostOpId = expectedDefault(ompt_id_t),
+      const void *CodeptrRA = expectedDefault(void *));
+
+  /// Handle type = ompt_callback_target_submit
+  static OmptAssertEvent
+  BufferRecord(const std::string &Name, const std::string &Group,
+               const ObserveState &Expected, ompt_callbacks_t Type,
+               std::pair<ompt_device_time_t, ompt_device_time_t> Timeframe,
+               unsigned int RequestedNumTeams = expectedDefault(unsigned int),
+               unsigned int GrantedNumTeams = expectedDefault(unsigned int),
+               ompt_id_t TargetId = expectedDefault(ompt_id_t),
+               ompt_id_t HostOpId = expectedDefault(ompt_id_t));
+
+  /// Handle type = ompt_callback_target_submit
+  /// Note: This will also act as the simplest default CTOR
+  static OmptAssertEvent BufferRecord(
+      const std::string &Name, const std::string &Group,
+      const ObserveState &Expected, ompt_callbacks_t Type,
+      ompt_device_time_t MinimumTimeDelta = expectedDefault(ompt_device_time_t),
+      unsigned int RequestedNumTeams = expectedDefault(unsigned int),
+      unsigned int GrantedNumTeams = expectedDefault(unsigned int),
+      ompt_id_t TargetId = expectedDefault(ompt_id_t),
+      ompt_id_t HostOpId = expectedDefault(ompt_id_t));
+
+  static OmptAssertEvent BufferRecordDeallocation(const std::string &Name,
+                                                  const std::string &Group,
+                                                  const ObserveState &Expected,
+                                                  ompt_buffer_t *Buffer);
+
+  /// Allow move construction (due to std::unique_ptr)
+  OmptAssertEvent(OmptAssertEvent &&o) = default;
+  OmptAssertEvent &operator=(OmptAssertEvent &&o) = default;
+
+  /// Get the event's name
+  std::string getEventName() const;
+
+  /// Get the event's group name
+  std::string getEventGroup() const;
+
+  /// Get the event's expected observation state
+  ObserveState getEventExpectedState() const;
+
+  /// Return the actual event type enum value
+  internal::EventTy getEventType() const;
+
+  /// Get a pointer to the internal event
+  internal::InternalEvent *getEvent() const;
+
+  /// Make events comparable
+  friend bool operator==(const OmptAssertEvent &A, const OmptAssertEvent &B);
+
+  /// Returns the string representation of the event
+  std::string toString(bool PrefixEventName = false) const;
+
+private:
+  OmptAssertEvent(const std::string &Name, const std::string &Group,
+                  const ObserveState &Expected, internal::InternalEvent *IE);
+  OmptAssertEvent(const OmptAssertEvent &o) = delete;
+
+  /// Determine the event name. Either it is provided directly or determined
+  /// from the calling function's name.
+  static std::string getName(const std::string &Name,
+                             const char *Caller = __builtin_FUNCTION()) {
+    std::string EName = Name;
+    if (EName.empty())
+      EName.append(Caller).append(" (auto generated)");
+
+    return EName;
+  }
+
+  /// Determine the event name. Either it is provided directly or "default".
+  static std::string getGroup(const std::string &Group) {
+    if (Group.empty())
+      return "default";
+
+    return Group;
+  }
+
+  std::string Name;
+  std::string Group;
+  ObserveState ExpectedState;
+  std::unique_ptr<internal::InternalEvent> TheEvent;
+};
+
+/// POD type, which holds the target region id, corresponding to an event group.
+struct AssertEventGroup {
+  AssertEventGroup(uint64_t TargetRegion) : TargetRegion(TargetRegion) {}
+  uint64_t TargetRegion;
+};
+
+bool operator==(const OmptAssertEvent &A, const OmptAssertEvent &B);
+
+} // namespace omptest
+
+#endif
diff --git a/openmp/tools/omptest/include/OmptAsserter.h b/openmp/tools/omptest/include/OmptAsserter.h
new file mode 100644
index 0000000000000..23d6dfeb7cd40
--- /dev/null
+++ b/openmp/tools/omptest/include/OmptAsserter.h
@@ -0,0 +1,291 @@
+//===- OmptAsserter.h - Asserter-related classes, enums, etc. ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Contains all asserter-related class declarations and important enums.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTASSERTER_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTASSERTER_H
+
+#include "Logging.h"
+#include "OmptAssertEvent.h"
+
+#include <cassert>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <set>
+#include <vector>
+
+namespace omptest {
+
+// Forward declaration.
+class OmptEventGroupInterface;
+
+enum class AssertMode { Strict, Relaxed };
+enum class AssertState { Pass, Fail };
+
+/// General base class for the subscriber/notification pattern in
+/// OmptCallbackHandler. Derived classes need to implement the notify method.
+class OmptListener {
+public:
+  virtual ~OmptListener() = default;
+
+  /// Called for each registered OMPT event of the OmptCallbackHandler
+  virtual void notify(omptest::OmptAssertEvent &&AE) = 0;
+
+  /// Control whether this asserter should be considered 'active'.
+  void setActive(bool Enabled);
+
+  /// Check if this asserter is considered 'active'.
+  bool isActive();
+
+  /// Check if the given event type is from the set of suppressed event types.
+  bool isSuppressedEventType(omptest::internal::EventTy EvTy);
+
+  /// Remove the given event type to the set of suppressed events.
+  void permitEvent(omptest::internal::EventTy EvTy);
+
+  /// Add the given event type to the set of suppressed events.
+  void suppressEvent(omptest::internal::EventTy EvTy);
+
+private:
+  bool Active{true};
+
+  // Add event types to the set of suppressed events by default.
+  std::set<omptest::internal::EventTy> SuppressedEvents{
+      omptest::internal::EventTy::ThreadBegin,
+      omptest::internal::EventTy::ThreadEnd,
+      omptest::internal::EventTy::ParallelBegin,
+      omptest::internal::EventTy::ParallelEnd,
+      omptest::internal::EventTy::Work,
+      omptest::internal::EventTy::Dispatch,
+      omptest::internal::EventTy::TaskCreate,
+      omptest::internal::EventTy::Dependences,
+      omptest::internal::EventTy::TaskDependence,
+      omptest::internal::EventTy::TaskSchedule,
+      omptest::internal::EventTy::ImplicitTask,
+      omptest::internal::EventTy::Masked,
+      omptest::internal::EventTy::SyncRegion,
+      omptest::internal::EventTy::MutexAcquire,
+      omptest::internal::EventTy::Mutex,
+      omptest::internal::EventTy::NestLock,
+      omptest::internal::EventTy::Flush,
+      omptest::internal::EventTy::Cancel};
+};
+
+/// Base class for asserting on OMPT events
+class OmptAsserter : public OmptListener {
+public:
+  OmptAsserter();
+  virtual ~OmptAsserter() = default;
+
+  /// Add an event to the asserter's internal data structure.
+  virtual void insert(omptest::OmptAssertEvent &&AE);
+
+  /// Called from the CallbackHandler with a corresponding AssertEvent to which
+  /// callback was handled.
+  void notify(omptest::OmptAssertEvent &&AE) override;
+
+  /// Implemented in subclasses to implement what should actually be done with
+  /// the notification.
+  virtual void notifyImpl(omptest::OmptAssertEvent &&AE) = 0;
+
+  /// Get the number of currently remaining events, with: ObserveState::Always.
+  virtual size_t getRemainingEventCount() = 0;
+
+  /// Get the total number of received, effective notifications.
+  int getNotificationCount() { return NumNotifications; }
+
+  /// Get the total number of successful assertion checks.
+  int getSuccessfulAssertionCount() { return NumSuccessfulAsserts; }
+
+  /// Get the asserter's current operationmode: e.g.: Strict or Relaxed.
+  AssertMode getOperationMode() { return OperationMode; }
+
+  /// Return the asserter's current state.
+  omptest::AssertState getState() { return State; }
+
+  /// Determine and return the asserter's state.
+  virtual omptest::AssertState checkState();
+
+  /// Accessor for the event group interface.
+  std::shared_ptr<OmptEventGroupInterface> getEventGroups() const {
+    return EventGroups;
+  }
+
+  /// Accessor for the event group interface.
+  std::shared_ptr<logging::Logger> getLog() const { return Log; }
+
+  /// Check the observed events' group association. If the event indicates the
+  /// begin/end of an OpenMP target region, we will create/deprecate the
+  /// expected event's group. Return true if the expected event group exists
+  /// (and is active), otherwise: false. Note: BufferRecords may also match with
+  /// deprecated groups as they may be delivered asynchronously.
+  bool verifyEventGroups(const omptest::OmptAssertEvent &ExpectedEvent,
+                         const omptest::OmptAssertEvent &ObservedEvent);
+
+  /// Set the asserter's mode of operation w.r.t. assertion.
+  void setOperationMode(AssertMode Mode);
+
+protected:
+  /// The asserter's current state.
+  omptest::AssertState State{omptest::AssertState::Pass};
+
+  /// Mutex to avoid data races w.r.t. event notifications and/or insertions.
+  std::mutex AssertMutex;
+
+  /// Pointer to the OmptEventGroupInterface.
+  std::shared_ptr<OmptEventGroupInterface> EventGroups{nullptr};
+
+  /// Pointer to the logging instance.
+  std::shared_ptr<logging::Logger> Log{nullptr};
+
+  /// Operation mode during assertion / notification.
+  AssertMode OperationMode{AssertMode::Strict};
+
+  /// The total number of effective notifications. For example, if specific
+  /// notifications are to be ignored, they will not count towards this total.
+  int NumNotifications{0};
+
+  /// The number of successful assertion checks.
+  int NumSuccessfulAsserts{0};
+
+private:
+  /// Mutex for creating/accessing the singleton members
+  static std::mutex StaticMemberAccessMutex;
+
+  /// Static member to manage the singleton event group interface instance
+  static std::weak_ptr<OmptEventGroupInterface> EventGroupInterfaceInstance;
+
+  /// Static member to manage the singleton logging instance
+  static std::weak_ptr<logging::Logger> LoggingInstance;
+};
+
+/// Class that can assert in a sequenced fashion, i.e., events have to occur in
+/// the order they were registered
+class OmptSequencedAsserter : public OmptAsserter {
+public:
+  OmptSequencedAsserter() : OmptAsserter(), NextEvent(0) {}
+
+  /// Add the event to the in-sequence set of events that the asserter should
+  /// check for.
+  void insert(omptest::OmptAssertEvent &&AE) override;
+
+  /// Implements the asserter's actual logic
+  virtual void notifyImpl(omptest::OmptAssertEvent &&AE) override;
+
+  size_t getRemainingEventCount() override;
+
+  omptest::AssertState checkState() override;
+
+  bool AssertionSuspended{false};
+
+protected:
+  /// Notification helper function, implementing SyncPoint logic. Returns true
+  /// in case of consumed event, indicating early exit of notification.
+  bool consumeSyncPoint(const omptest::OmptAssertEvent &AE);
+
+  /// Notification helper function, implementing excess event notification
+  /// logic. Returns true when no more events were expected, indicating early
+  /// exit of notification.
+  bool checkExcessNotify(const omptest::OmptAssertEvent &AE);
+
+  /// Notification helper function, implementing Suspend logic. Returns true
+  /// in case of consumed event, indicating early exit of notification.
+  bool consumeSuspend();
+
+  /// Notification helper function, implementing regular event notification
+  /// logic. Returns true when a matching event was encountered, indicating
+  /// early exit of notification.
+  bool consumeRegularEvent(const omptest::OmptAssertEvent &AE);
+
+public:
+  /// Index of the next, expected event.
+  size_t NextEvent{0};
+  std::vector<omptest::OmptAssertEvent> Events{};
+};
+
+/// Class that asserts with set semantics, i.e., unordered
+struct OmptEventAsserter : public OmptAsserter {
+  OmptEventAsserter() : OmptAsserter(), NumEvents(0), Events() {}
+
+  /// Add the event to the set of events that the asserter should check for.
+  void insert(omptest::OmptAssertEvent &&AE) override;
+
+  /// Implements the asserter's logic
+  virtual void notifyImpl(omptest::OmptAssertEvent &&AE) override;
+
+  size_t getRemainingEventCount() override;
+
+  omptest::AssertState checkState() override;
+
+  size_t NumEvents{0};
+
+  /// For now use vector (but do set semantics)
+  // TODO std::unordered_set?
+  std::vector<omptest::OmptAssertEvent> Events{};
+};
+
+/// Class that reports the occurred events
+class OmptEventReporter : public OmptListener {
+public:
+  OmptEventReporter(std::ostream &OutStream = std::cout)
+      : OutStream(OutStream) {}
+
+  /// Called from the CallbackHandler with a corresponding AssertEvent to which
+  /// callback was handled.
+  void notify(omptest::OmptAssertEvent &&AE) override;
+
+private:
+  std::ostream &OutStream;
+};
+
+/// This class provides the members and methods to manage event groups and
+/// SyncPoints in conjunction with asserters. Most importantly it maintains a
+/// coherent view of active and past events or SyncPoints.
+class OmptEventGroupInterface {
+public:
+  OmptEventGroupInterface() = default;
+  ~OmptEventGroupInterface() = default;
+
+  /// Non-copyable and non-movable
+  OmptEventGroupInterface(const OmptEventGroupInterface &) = delete;
+  OmptEventGroupInterface &operator=(const OmptEventGroupInterface &) = delete;
+  OmptEventGroupInterface(OmptEventGroupInterface &&) = delete;
+  OmptEventGroupInterface &operator=(OmptEventGroupInterface &&) = delete;
+
+  /// Add given group to the set of active event groups. Effectively connecting
+  /// the given groupname (expected) with a target region id (observed).
+  bool addActiveEventGroup(const std::string &GroupName,
+                           omptest::AssertEventGroup Group);
+
+  /// Move given group from the set of active event groups to the set of
+  /// previously active event groups.
+  bool deprecateActiveEventGroup(const std::string &GroupName);
+
+  /// Check if given group is currently part of the active event groups.
+  bool checkActiveEventGroups(const std::string &GroupName,
+                              omptest::AssertEventGroup Group);
+
+  /// Check if given group is currently part of the deprecated event groups.
+  bool checkDeprecatedEventGroups(const std::string &GroupName,
+                                  omptest::AssertEventGroup Group);
+
+private:
+  mutable std::mutex GroupMutex;
+  std::map<std::string, omptest::AssertEventGroup> ActiveEventGroups{};
+  std::map<std::string, omptest::AssertEventGroup> DeprecatedEventGroups{};
+  std::set<std::string> EncounteredSyncPoints{};
+};
+
+} // namespace omptest
+
+#endif
diff --git a/openmp/tools/omptest/include/OmptCallbackHandler.h b/openmp/tools/omptest/include/OmptCallbackHandler.h
new file mode 100644
index 0000000000000..40076c386107e
--- /dev/null
+++ b/openmp/tools/omptest/include/OmptCallbackHandler.h
@@ -0,0 +1,165 @@
+//===- OmptCallbackHandler.h - Callback reception and handling --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file provides the OMPT callback handling declarations.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTCALLBACKHANDLER_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTCALLBACKHANDLER_H
+
+#include "OmptAssertEvent.h"
+#include "OmptAsserter.h"
+
+#include "omp-tools.h"
+
+#include <vector>
+
+namespace omptest {
+
+/// Handler class to do whatever is needed to be done when a callback is invoked
+/// by the OMP runtime
+/// Supports a RecordAndReplay mechanism in which all OMPT events are recorded
+/// and then replayed. This is so that a test can assert on, e.g., a device
+/// initialize event, even though this would occur before a unit test is
+/// actually executed.
+class OmptCallbackHandler {
+public:
+  ~OmptCallbackHandler() = default;
+
+  /// Singleton handler
+  static OmptCallbackHandler &get();
+
+  /// Subscribe a listener to be notified for OMPT events
+  void subscribe(OmptListener *Listener);
+
+  /// Remove all subscribers
+  void clearSubscribers();
+
+  /// When the record and replay mechanism is enabled this replays all OMPT
+  /// events
+  void replay();
+
+  /// Special asserter callback which checks that upon encountering the
+  /// synchronization point, all expected events have been processed. That is:
+  /// there are currently no remaining expected events for any asserter.
+  void handleAssertionSyncPoint(const std::string &SyncPointName);
+
+  void handleThreadBegin(ompt_thread_t ThreadType, ompt_data_t *ThreadData);
+
+  void handleThreadEnd(ompt_data_t *ThreadData);
+
+  void handleTaskCreate(ompt_data_t *EncounteringTaskData,
+                        const ompt_frame_t *EncounteringTaskFrame,
+                        ompt_data_t *NewTaskData, int Flags, int HasDependences,
+                        const void *CodeptrRA);
+
+  void handleTaskSchedule(ompt_data_t *PriorTaskData,
+                          ompt_task_status_t PriorTaskStatus,
+                          ompt_data_t *NextTaskData);
+
+  void handleImplicitTask(ompt_scope_endpoint_t Endpoint,
+                          ompt_data_t *ParallelData, ompt_data_t *TaskData,
+                          unsigned int ActualParallelism, unsigned int Index,
+                          int Flags);
+
+  void handleParallelBegin(ompt_data_t *EncounteringTaskData,
+                           const ompt_frame_t *EncounteringTaskFrame,
+                           ompt_data_t *ParallelData,
+                           unsigned int RequestedParallelism, int Flags,
+                           const void *CodeptrRA);
+
+  void handleParallelEnd(ompt_data_t *ParallelData,
+                         ompt_data_t *EncounteringTaskData, int Flags,
+                         const void *CodeptrRA);
+
+  void handleDeviceInitialize(int DeviceNum, const char *Type,
+                              ompt_device_t *Device,
+                              ompt_function_lookup_t LookupFn,
+                              const char *DocumentationStr);
+
+  void handleDeviceFinalize(int DeviceNum);
+
+  void handleTarget(ompt_target_t Kind, ompt_scope_endpoint_t Endpoint,
+                    int DeviceNum, ompt_data_t *TaskData, ompt_id_t TargetId,
+                    const void *CodeptrRA);
+
+  void handleTargetEmi(ompt_target_t Kind, ompt_scope_endpoint_t Endpoint,
+                       int DeviceNum, ompt_data_t *TaskData,
+                       ompt_data_t *TargetTaskData, ompt_data_t *TargetData,
+                       const void *CodeptrRA);
+
+  void handleTargetSubmit(ompt_id_t TargetId, ompt_id_t HostOpId,
+                          unsigned int RequestedNumTeams);
+
+  void handleTargetSubmitEmi(ompt_scope_endpoint_t Endpoint,
+                             ompt_data_t *TargetData, ompt_id_t *HostOpId,
+                             unsigned int RequestedNumTeams);
+
+  void handleTargetDataOp(ompt_id_t TargetId, ompt_id_t HostOpId,
+                          ompt_target_data_op_t OpType, void *SrcAddr,
+                          int SrcDeviceNum, void *DstAddr, int DstDeviceNum,
+                          size_t Bytes, const void *CodeptrRA);
+
+  void handleTargetDataOpEmi(ompt_scope_endpoint_t Endpoint,
+                             ompt_data_t *TargetTaskData,
+                             ompt_data_t *TargetData, ompt_id_t *HostOpId,
+                             ompt_target_data_op_t OpType, void *SrcAddr,
+                             int SrcDeviceNum, void *DstAddr, int DstDeviceNum,
+                             size_t Bytes, const void *CodeptrRA);
+
+  void handleDeviceLoad(int DeviceNum, const char *Filename,
+                        int64_t OffsetInFile, void *VmaInFile, size_t Bytes,
+                        void *HostAddr, void *DeviceAddr, uint64_t ModuleId);
+
+  void handleDeviceUnload(int DeviceNum, uint64_t ModuleId);
+
+  void handleBufferRequest(int DeviceNum, ompt_buffer_t **Buffer,
+                           size_t *Bytes);
+
+  void handleBufferComplete(int DeviceNum, ompt_buffer_t *Buffer, size_t Bytes,
+                            ompt_buffer_cursor_t Begin, int BufferOwned);
+
+  void handleBufferRecord(ompt_record_ompt_t *Record);
+
+  void handleBufferRecordDeallocation(ompt_buffer_t *Buffer);
+
+  /// Not needed for a conforming minimal OMPT implementation
+  void handleWork(ompt_work_t WorkType, ompt_scope_endpoint_t Endpoint,
+                  ompt_data_t *ParallelData, ompt_data_t *TaskData,
+                  uint64_t Count, const void *CodeptrRA);
+
+  void handleDispatch(ompt_data_t *ParallelData, ompt_data_t *TaskData,
+                      ompt_dispatch_t Kind, ompt_data_t Instance);
+
+  void handleSyncRegion(ompt_sync_region_t Kind, ompt_scope_endpoint_t Endpoint,
+                        ompt_data_t *ParallelData, ompt_data_t *TaskData,
+                        const void *CodeptrRA);
+
+private:
+  /// Wrapper around emplace_back for potential additional logging / checking or
+  /// so
+  void recordEvent(OmptAssertEvent &&Event);
+
+  /// Listeners to be notified
+  std::vector<OmptListener *> Subscribers;
+
+  /// Toggle if OMPT events should notify subscribers immediately or not
+  bool RecordAndReplay{false};
+
+  /// Recorded events in Record and Replay mode
+  std::vector<OmptAssertEvent> RecordedEvents;
+};
+
+} // namespace omptest
+
+// Pointer to global callback handler
+extern omptest::OmptCallbackHandler *Handler;
+
+#endif
diff --git a/openmp/tools/omptest/include/OmptTester.h b/openmp/tools/omptest/include/OmptTester.h
new file mode 100644
index 0000000000000..155e61d5f7482
--- /dev/null
+++ b/openmp/tools/omptest/include/OmptTester.h
@@ -0,0 +1,60 @@
+//===- OmptTester.h - Main header for ompTest usage -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file represents the main header file for usage of the ompTest library.
+/// Depending on the build either 'standalone' or GoogleTest headers are
+/// included and corresponding main-function macros are defined.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTTESTER_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTTESTER_H
+
+#include "AssertMacros.h"
+#include "Logging.h"
+#include "OmptAliases.h"
+#include "OmptAssertEvent.h"
+#include "OmptAsserter.h"
+#include "OmptCallbackHandler.h"
+
+#include <cassert>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <thread>
+#include <unordered_set>
+#include <vector>
+
+// Standalone header section
+#ifdef OPENMP_LIBOMPTEST_BUILD_STANDALONE
+
+#include "OmptTesterStandalone.h"
+
+// Define standalone main function (place once at the bottom of a testsuite)
+#define OMPTEST_TESTSUITE_MAIN()                                               \
+  int main(int argc, char **argv) {                                            \
+    Runner R;                                                                  \
+    return R.run();                                                            \
+  }
+
+// GoogleTest header section
+#else
+
+#include "OmptTesterGoogleTest.h"
+
+// Define GoogleTest main function (place once at the bottom of a testsuite)
+#define OMPTEST_TESTSUITE_MAIN()                                               \
+  int main(int argc, char **argv) {                                            \
+    testing::InitGoogleTest(&argc, argv);                                      \
+    return RUN_ALL_TESTS();                                                    \
+  }
+
+#endif
+
+#endif
diff --git a/openmp/tools/omptest/include/OmptTesterGlobals.h b/openmp/tools/omptest/include/OmptTesterGlobals.h
new file mode 100644
index 0000000000000..62f443aed80e0
--- /dev/null
+++ b/openmp/tools/omptest/include/OmptTesterGlobals.h
@@ -0,0 +1,36 @@
+//===- OmptTesterGlobals.h - Global function declarations -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Contains global function declarations, esp. for OMPT symbols.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTTESTERGLOBALS_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTTESTERGLOBALS_H
+
+#include <omp-tools.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version);
+int start_trace(ompt_device_t *Device);
+int flush_trace(ompt_device_t *Device);
+// Function which calls flush_trace(ompt_device_t *) on all traced devices.
+int flush_traced_devices();
+int stop_trace(ompt_device_t *Device);
+// Function which calls stop_trace(ompt_device_t *) on all traced devices.
+int stop_trace_devices();
+void libomptest_global_eventreporter_set_active(bool State);
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/openmp/tools/omptest/include/OmptTesterGoogleTest.h b/openmp/tools/omptest/include/OmptTesterGoogleTest.h
new file mode 100644
index 0000000000000..80e752c849fe0
--- /dev/null
+++ b/openmp/tools/omptest/include/OmptTesterGoogleTest.h
@@ -0,0 +1,86 @@
+//===- OmptTesterGoogleTest.h - GoogleTest header variant -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file represents the GoogleTest-based header variant, defining the
+/// actual test classes and their behavior.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTTESTERGOOGLETEST_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTTESTERGOOGLETEST_H
+
+#include "AssertMacros.h"
+#include "OmptAliases.h"
+#include "OmptAssertEvent.h"
+#include "OmptAsserter.h"
+#include "OmptCallbackHandler.h"
+#include "OmptTesterGlobals.h"
+
+// This will allow us to override the "TEST" macro of gtest
+#define GTEST_DONT_DEFINE_TEST 1
+#include "gtest/gtest.h"
+
+namespace testing {
+class GTEST_API_ OmptTestCase : public testing::Test,
+                                public omptest::OmptEventGroupInterface {
+public:
+  std::unique_ptr<omptest::OmptSequencedAsserter> SequenceAsserter =
+      std::make_unique<omptest::OmptSequencedAsserter>();
+  std::unique_ptr<omptest::OmptEventAsserter> SetAsserter =
+      std::make_unique<omptest::OmptEventAsserter>();
+  std::unique_ptr<omptest::OmptEventReporter> EventReporter =
+      std::make_unique<omptest::OmptEventReporter>();
+
+protected:
+  void SetUp() override {
+    omptest::OmptCallbackHandler::get().subscribe(SequenceAsserter.get());
+    omptest::OmptCallbackHandler::get().subscribe(SetAsserter.get());
+    omptest::OmptCallbackHandler::get().subscribe(EventReporter.get());
+  }
+
+  void TearDown() override {
+    // Actively flush potential in-flight trace records
+    flush_traced_devices();
+
+    // Remove subscribers to not be notified of events after test execution.
+    omptest::OmptCallbackHandler::get().clearSubscribers();
+
+    // This common testcase must not encounter any failures.
+    if (SequenceAsserter->checkState() == omptest::AssertState::Fail ||
+        SetAsserter->checkState() == omptest::AssertState::Fail)
+      ADD_FAILURE();
+  }
+};
+
+class GTEST_API_ OmptTestCaseXFail : public testing::OmptTestCase {
+protected:
+  void TearDown() override {
+    // Actively flush potential in-flight trace records
+    flush_traced_devices();
+
+    // Remove subscribers to not be notified of events after test execution.
+    omptest::OmptCallbackHandler::get().clearSubscribers();
+
+    // This eXpectedly failing testcase has to encounter at least one failure.
+    if (SequenceAsserter->checkState() == omptest::AssertState::Pass &&
+        SetAsserter->checkState() == omptest::AssertState::Pass)
+      ADD_FAILURE();
+  }
+};
+} // namespace testing
+
+#define TEST(test_suite_name, test_name)                                       \
+  GTEST_TEST_(test_suite_name, test_name, ::testing::OmptTestCase,             \
+              ::testing::internal::GetTypeId<::testing::OmptTestCase>())
+
+#define TEST_XFAIL(test_suite_name, test_name)                                 \
+  GTEST_TEST_(test_suite_name, test_name, ::testing::OmptTestCaseXFail,        \
+              ::testing::internal::GetTypeId<::testing::OmptTestCaseXFail>())
+
+#endif // include guard
diff --git a/openmp/tools/omptest/include/OmptTesterStandalone.h b/openmp/tools/omptest/include/OmptTesterStandalone.h
new file mode 100644
index 0000000000000..aca72f7db375f
--- /dev/null
+++ b/openmp/tools/omptest/include/OmptTesterStandalone.h
@@ -0,0 +1,126 @@
+//===- OmptTesterStandalone.h - Standalone header variant -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file represents the 'standalone' header variant, defining the actual
+/// test classes and their behavior (it does not have external dependencies).
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTTESTERSTANDALONE_H
+#define OPENMP_TOOLS_OMPTEST_INCLUDE_OMPTTESTERSTANDALONE_H
+
+#include "OmptAssertEvent.h"
+#include "OmptAsserter.h"
+#include "OmptTesterGlobals.h"
+
+#include <utility>
+#include <vector>
+
+// Forward declarations.
+namespace omptest {
+struct OmptEventAsserter;
+class OmptEventReporter;
+class OmptSequencedAsserter;
+} // namespace omptest
+
+struct Error {
+  operator bool() { return Fail; }
+  bool Fail;
+};
+
+/// A pretty crude test case abstraction
+struct TestCase {
+  TestCase(const std::string &name)
+      : IsDisabled(name.rfind("DISABLED_", 0) == 0), Name(name) {}
+  TestCase(const std::string &name, const omptest::AssertState &expected)
+      : IsDisabled(name.rfind("DISABLED_", 0) == 0), Name(name),
+        ExpectedState(expected) {}
+  virtual ~TestCase() = default;
+  Error exec();
+  virtual void execImpl() { assert(false && "Allocating base class"); }
+
+  bool IsDisabled{false};
+  std::string Name;
+  omptest::AssertState ExpectedState{omptest::AssertState::Pass};
+  omptest::AssertState ResultState{omptest::AssertState::Pass};
+
+  std::unique_ptr<omptest::OmptSequencedAsserter> SequenceAsserter =
+      std::make_unique<omptest::OmptSequencedAsserter>();
+  std::unique_ptr<omptest::OmptEventAsserter> SetAsserter =
+      std::make_unique<omptest::OmptEventAsserter>();
+  std::unique_ptr<omptest::OmptEventReporter> EventReporter =
+      std::make_unique<omptest::OmptEventReporter>();
+};
+/// A pretty crude test suite abstraction
+struct TestSuite {
+  using TestCaseVec = std::vector<std::unique_ptr<TestCase>>;
+  TestSuite() = default;
+  TestSuite(const std::string &TSName) : Name(TSName) {}
+  TestSuite(const TestSuite &O) = delete;
+  TestSuite(TestSuite &&O);
+  void setup();
+  void teardown();
+  TestCaseVec::iterator begin();
+  TestCaseVec::iterator end();
+  std::string Name;
+  TestCaseVec TestCases;
+};
+/// Static class used to register all test cases and provide them to the driver
+class TestRegistrar {
+public:
+  static TestRegistrar &get();
+  static std::vector<TestSuite> getTestSuites();
+  static void addCaseToSuite(TestCase *TC, const std::string &TSName);
+
+private:
+  TestRegistrar() = default;
+  TestRegistrar(const TestRegistrar &o) = delete;
+  TestRegistrar operator=(const TestRegistrar &o) = delete;
+  // Keep tests in order 'of appearance', i.e. top -> bottom.
+  // This effectively mimicks the (observed) behavior of GoogleTest.
+  // Avoid maps as they do not have corresponding order guarantees.
+  static std::vector<std::pair<std::string, TestSuite>> Tests;
+};
+/// Hack to register test cases
+struct Registerer {
+  Registerer(TestCase *TC, const std::string SuiteName);
+};
+/// Eventually executes all test suites and cases, should contain logic to skip
+/// stuff if needed
+struct Runner {
+  Runner() : TestSuites(TestRegistrar::get().getTestSuites()) {}
+  int run();
+  void reportError(const Error &Err);
+  void abortOrKeepGoing();
+  // Print an execution summary of all testsuites and their corresponding
+  // testcases.
+  void printSummary();
+  std::vector<TestSuite> TestSuites;
+};
+
+/// MACROS TO DEFINE A TESTSUITE + TESTCASE (like GoogleTest does)
+#define XQUOTE(str) QUOTE(str)
+#define QUOTE(str) #str
+
+#define TEST_TEMPLATE(SuiteName, CaseName, ExpectedState)                      \
+  struct SuiteName##_##CaseName : public TestCase {                            \
+    SuiteName##_##CaseName()                                                   \
+        : TestCase(XQUOTE(CaseName), omptest::AssertState::ExpectedState) {}   \
+    virtual void execImpl() override;                                          \
+  };                                                                           \
+  static Registerer R_##SuiteName##CaseName(new SuiteName##_##CaseName(),      \
+                                            #SuiteName);                       \
+  void SuiteName##_##CaseName::execImpl()
+
+#define TEST(SuiteName, CaseName)                                              \
+  TEST_TEMPLATE(SuiteName, CaseName, /*ExpectedState=*/Pass)
+#define TEST_XFAIL(SuiteName, CaseName)                                        \
+  TEST_TEMPLATE(SuiteName, CaseName, /*ExpectedState=*/Fail)
+
+#endif
diff --git a/openmp/tools/omptest/src/InternalEvent.cpp b/openmp/tools/omptest/src/InternalEvent.cpp
new file mode 100644
index 0000000000000..35673e06e479f
--- /dev/null
+++ b/openmp/tools/omptest/src/InternalEvent.cpp
@@ -0,0 +1,367 @@
+//===- InternalEvent.cpp - Internal event implementation --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implements internal event representation methods and helper functions.
+///
+//===----------------------------------------------------------------------===//
+
+#include "InternalEvent.h"
+
+#include <iomanip>
+#include <sstream>
+
+using namespace omptest;
+using namespace util;
+
+std::string util::makeHexString(uint64_t Data, bool IsPointer, size_t MinBytes,
+                                bool ShowHexBase) {
+  if (Data == 0 && IsPointer)
+    return "(nil)";
+
+  thread_local std::ostringstream os;
+  // Clear the content of the stream
+  os.str(std::string());
+
+  // Manually prefixing "0x" will make the use of std::setfill more easy
+  if (ShowHexBase)
+    os << "0x";
+
+  // Default to 32bit (8 hex digits) width, if exceeding 64bit or zero value
+  size_t NumDigits = (MinBytes > 0 && MinBytes < 9) ? (MinBytes << 1) : 8;
+
+  if (MinBytes > 0)
+    os << std::setfill('0') << std::setw(NumDigits);
+
+  os << std::hex << Data;
+  return os.str();
+}
+
+std::string internal::AssertionSyncPoint::toString() const {
+  std::string S{"Assertion SyncPoint: '"};
+  S.append(Name).append(1, '\'');
+  return S;
+}
+
+std::string internal::ThreadBegin::toString() const {
+  std::string S{"OMPT Callback ThreadBegin: "};
+  S.append("ThreadType=").append(std::to_string(ThreadType));
+  return S;
+}
+
+std::string internal::ThreadEnd::toString() const {
+  std::string S{"OMPT Callback ThreadEnd"};
+  return S;
+}
+
+std::string internal::ParallelBegin::toString() const {
+  std::string S{"OMPT Callback ParallelBegin: "};
+  S.append("NumThreads=").append(std::to_string(NumThreads));
+  return S;
+}
+
+std::string internal::ParallelEnd::toString() const {
+  // TODO: Should we expose more detailed info here?
+  std::string S{"OMPT Callback ParallelEnd"};
+  return S;
+}
+
+std::string internal::Work::toString() const {
+  std::string S{"OMPT Callback Work: "};
+  S.append("work_type=").append(std::to_string(WorkType));
+  S.append(" endpoint=").append(std::to_string(Endpoint));
+  S.append(" parallel_data=").append(makeHexString((uint64_t)ParallelData));
+  S.append(" task_data=").append(makeHexString((uint64_t)TaskData));
+  S.append(" count=").append(std::to_string(Count));
+  S.append(" codeptr=").append(makeHexString((uint64_t)CodeptrRA));
+  return S;
+}
+
+std::string internal::Dispatch::toString() const {
+  std::string S{"OMPT Callback Dispatch: "};
+  S.append("parallel_data=").append(makeHexString((uint64_t)ParallelData));
+  S.append(" task_data=").append(makeHexString((uint64_t)TaskData));
+  S.append(" kind=").append(std::to_string(Kind));
+  // TODO Check what to print for instance in all different cases
+  if (Kind == ompt_dispatch_iteration) {
+    S.append(" instance=[it=")
+        .append(std::to_string(Instance.value))
+        .append(1, ']');
+  } else if (Kind == ompt_dispatch_section) {
+    S.append(" instance=[ptr=")
+        .append(makeHexString((uint64_t)Instance.ptr))
+        .append(1, ']');
+  } else if ((Kind == ompt_dispatch_ws_loop_chunk ||
+              Kind == ompt_dispatch_taskloop_chunk ||
+              Kind == ompt_dispatch_distribute_chunk) &&
+             Instance.ptr != nullptr) {
+    auto Chunk = static_cast<ompt_dispatch_chunk_t *>(Instance.ptr);
+    S.append(" instance=[chunk=(start=")
+        .append(std::to_string(Chunk->start))
+        .append(", iterations=")
+        .append(std::to_string(Chunk->iterations))
+        .append(")]");
+  }
+  return S;
+}
+
+std::string internal::TaskCreate::toString() const {
+  std::string S{"OMPT Callback TaskCreate: "};
+  S.append("encountering_task_data=")
+      .append(makeHexString((uint64_t)EncounteringTaskData));
+  S.append(" encountering_task_frame=")
+      .append(makeHexString((uint64_t)EncounteringTaskFrame));
+  S.append(" new_task_data=").append(makeHexString((uint64_t)NewTaskData));
+  S.append(" flags=").append(std::to_string(Flags));
+  S.append(" has_dependences=").append(std::to_string(HasDependences));
+  S.append(" codeptr=").append(makeHexString((uint64_t)CodeptrRA));
+  return S;
+}
+
+std::string internal::ImplicitTask::toString() const {
+  std::string S{"OMPT Callback ImplicitTask: "};
+  S.append("endpoint=").append(std::to_string(Endpoint));
+  S.append(" parallel_data=").append(makeHexString((uint64_t)ParallelData));
+  S.append(" task_data=").append(makeHexString((uint64_t)TaskData));
+  S.append(" actual_parallelism=").append(std::to_string(ActualParallelism));
+  S.append(" index=").append(std::to_string(Index));
+  S.append(" flags=").append(std::to_string(Flags));
+  return S;
+}
+
+std::string internal::SyncRegion::toString() const {
+  std::string S{"OMPT Callback SyncRegion: "};
+  S.append("kind=").append(std::to_string(Kind));
+  S.append(" endpoint=").append(std::to_string(Endpoint));
+  S.append(" parallel_data=").append(makeHexString((uint64_t)ParallelData));
+  S.append(" task_data=").append(makeHexString((uint64_t)TaskData));
+  S.append(" codeptr=").append(makeHexString((uint64_t)CodeptrRA));
+  return S;
+}
+
+std::string internal::Target::toString() const {
+  // TODO Should we canonicalize the string prefix (use "OMPT ..." everywhere)?
+  std::string S{"Callback Target: target_id="};
+  S.append(std::to_string(TargetId));
+  S.append(" kind=").append(std::to_string(Kind));
+  S.append(" endpoint=").append(std::to_string(Endpoint));
+  S.append(" device_num=").append(std::to_string(DeviceNum));
+  S.append(" code=").append(makeHexString((uint64_t)CodeptrRA));
+  return S;
+}
+
+std::string internal::TargetEmi::toString() const {
+  // TODO Should we canonicalize the string prefix (use "OMPT ..." everywhere)?
+  std::string S{"Callback Target EMI: kind="};
+  S.append(std::to_string(Kind));
+  S.append(" endpoint=").append(std::to_string(Endpoint));
+  S.append(" device_num=").append(std::to_string(DeviceNum));
+  S.append(" task_data=").append(makeHexString((uint64_t)TaskData));
+  S.append(" (")
+      .append(makeHexString((uint64_t)(TaskData) ? TaskData->value : 0,
+                            /*IsPointer=*/false))
+      .append(1, ')');
+  S.append(" target_task_data=")
+      .append(makeHexString((uint64_t)TargetTaskData));
+  S.append(" (")
+      .append(
+          makeHexString((uint64_t)(TargetTaskData) ? TargetTaskData->value : 0,
+                        /*IsPointer=*/false))
+      .append(1, ')');
+  S.append(" target_data=").append(makeHexString((uint64_t)TargetData));
+  S.append(" (")
+      .append(makeHexString((uint64_t)(TargetData) ? TargetData->value : 0,
+                            /*IsPointer=*/false))
+      .append(1, ')');
+  S.append(" code=").append(makeHexString((uint64_t)CodeptrRA));
+  return S;
+}
+
+std::string internal::TargetDataOp::toString() const {
+  std::string S{"  Callback DataOp: target_id="};
+  S.append(std::to_string(TargetId));
+  S.append(" host_op_id=").append(std::to_string(HostOpId));
+  S.append(" optype=").append(std::to_string(OpType));
+  S.append(" src=").append(makeHexString((uint64_t)SrcAddr));
+  S.append(" src_device_num=").append(std::to_string(SrcDeviceNum));
+  S.append(" dest=").append(makeHexString((uint64_t)DstAddr));
+  S.append(" dest_device_num=").append(std::to_string(DstDeviceNum));
+  S.append(" bytes=").append(std::to_string(Bytes));
+  S.append(" code=").append(makeHexString((uint64_t)CodeptrRA));
+  return S;
+}
+
+std::string internal::TargetDataOpEmi::toString() const {
+  std::string S{"  Callback DataOp EMI: endpoint="};
+  S.append(std::to_string(Endpoint));
+  S.append(" optype=").append(std::to_string(OpType));
+  S.append(" target_task_data=")
+      .append(makeHexString((uint64_t)TargetTaskData));
+  S.append(" (")
+      .append(
+          makeHexString((uint64_t)(TargetTaskData) ? TargetTaskData->value : 0,
+                        /*IsPointer=*/false))
+      .append(1, ')');
+  S.append(" target_data=").append(makeHexString((uint64_t)TargetData));
+  S.append(" (")
+      .append(makeHexString((uint64_t)(TargetData) ? TargetData->value : 0,
+                            /*IsPointer=*/false))
+      .append(1, ')');
+  S.append(" host_op_id=").append(makeHexString((uint64_t)HostOpId));
+  S.append(" (")
+      .append(makeHexString((uint64_t)(HostOpId) ? (*HostOpId) : 0,
+                            /*IsPointer=*/false))
+      .append(1, ')');
+  S.append(" src=").append(makeHexString((uint64_t)SrcAddr));
+  S.append(" src_device_num=").append(std::to_string(SrcDeviceNum));
+  S.append(" dest=").append(makeHexString((uint64_t)DstAddr));
+  S.append(" dest_device_num=").append(std::to_string(DstDeviceNum));
+  S.append(" bytes=").append(std::to_string(Bytes));
+  S.append(" code=").append(makeHexString((uint64_t)CodeptrRA));
+  return S;
+}
+
+std::string internal::TargetSubmit::toString() const {
+  std::string S{"  Callback Submit: target_id="};
+  S.append(std::to_string(TargetId));
+  S.append(" host_op_id=").append(std::to_string(HostOpId));
+  S.append(" req_num_teams=").append(std::to_string(RequestedNumTeams));
+  return S;
+}
+
+std::string internal::TargetSubmitEmi::toString() const {
+  std::string S{"  Callback Submit EMI: endpoint="};
+  S.append(std::to_string(Endpoint));
+  S.append(" req_num_teams=").append(std::to_string(RequestedNumTeams));
+  S.append(" target_data=").append(makeHexString((uint64_t)TargetData));
+  S.append(" (")
+      .append(makeHexString((uint64_t)(TargetData) ? TargetData->value : 0,
+                            /*IsPointer=*/false))
+      .append(1, ')');
+  S.append(" host_op_id=").append(makeHexString((uint64_t)HostOpId));
+  S.append(" (")
+      .append(makeHexString((uint64_t)(HostOpId) ? (*HostOpId) : 0,
+                            /*IsPointer=*/false))
+      .append(1, ')');
+  return S;
+}
+
+std::string internal::DeviceInitialize::toString() const {
+  std::string S{"Callback Init: device_num="};
+  S.append(std::to_string(DeviceNum));
+  S.append(" type=").append((Type) ? Type : "(null)");
+  S.append(" device=").append(makeHexString((uint64_t)Device));
+  S.append(" lookup=").append(makeHexString((uint64_t)LookupFn));
+  S.append(" doc=").append(makeHexString((uint64_t)DocStr));
+  return S;
+}
+
+std::string internal::DeviceFinalize::toString() const {
+  std::string S{"Callback Fini: device_num="};
+  S.append(std::to_string(DeviceNum));
+  return S;
+}
+
+std::string internal::DeviceLoad::toString() const {
+  std::string S{"Callback Load: device_num:"};
+  S.append(std::to_string(DeviceNum));
+  S.append(" module_id:").append(std::to_string(ModuleId));
+  S.append(" filename:").append((Filename == nullptr) ? "(null)" : Filename);
+  S.append(" host_addr:").append(makeHexString((uint64_t)HostAddr));
+  S.append(" device_addr:").append(makeHexString((uint64_t)DeviceAddr));
+  S.append(" bytes:").append(std::to_string(Bytes));
+  return S;
+}
+
+std::string internal::BufferRequest::toString() const {
+  std::string S{"Allocated "};
+  S.append(std::to_string((Bytes != nullptr) ? *Bytes : 0))
+      .append(" bytes at ");
+  S.append(makeHexString((Buffer != nullptr) ? (uint64_t)*Buffer : 0));
+  S.append(" in buffer request callback");
+  return S;
+}
+
+std::string internal::BufferComplete::toString() const {
+  std::string S{"Executing buffer complete callback: "};
+  S.append(std::to_string(DeviceNum)).append(1, ' ');
+  S.append(makeHexString((uint64_t)Buffer)).append(1, ' ');
+  S.append(std::to_string(Bytes)).append(1, ' ');
+  S.append(makeHexString((uint64_t)Begin)).append(1, ' ');
+  S.append(std::to_string(BufferOwned));
+  return S;
+}
+
+std::string internal::BufferRecord::toString() const {
+  std::string S{""};
+  std::string T{""};
+  S.append("rec=").append(makeHexString((uint64_t)RecordPtr));
+  S.append(" type=").append(std::to_string(Record.type));
+
+  T.append("time=").append(std::to_string(Record.time));
+  T.append(" thread_id=").append(std::to_string(Record.thread_id));
+  T.append(" target_id=").append(std::to_string(Record.target_id));
+
+  switch (Record.type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    // Handle Target Record
+    ompt_record_target_t TR = Record.record.target;
+    S.append(" (Target task) ").append(T);
+    S.append(" kind=").append(std::to_string(TR.kind));
+    S.append(" endpoint=").append(std::to_string(TR.endpoint));
+    S.append(" device=").append(std::to_string(TR.device_num));
+    S.append(" task_id=").append(std::to_string(TR.task_id));
+    S.append(" codeptr=").append(makeHexString((uint64_t)TR.codeptr_ra));
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    // Handle Target DataOp Record
+    ompt_record_target_data_op_t TDR = Record.record.target_data_op;
+    S.append(" (Target data op) ").append(T);
+    S.append(" host_op_id=").append(std::to_string(TDR.host_op_id));
+    S.append(" optype=").append(std::to_string(TDR.optype));
+    S.append(" src_addr=").append(makeHexString((uint64_t)TDR.src_addr));
+    S.append(" src_device=").append(std::to_string(TDR.src_device_num));
+    S.append(" dest_addr=").append(makeHexString((uint64_t)TDR.dest_addr));
+    S.append(" dest_device=").append(std::to_string(TDR.dest_device_num));
+    S.append(" bytes=").append(std::to_string(TDR.bytes));
+    S.append(" end_time=").append(std::to_string(TDR.end_time));
+    S.append(" duration=").append(std::to_string(TDR.end_time - Record.time));
+    S.append(" ns codeptr=").append(makeHexString((uint64_t)TDR.codeptr_ra));
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    // Handle Target Kernel Record
+    ompt_record_target_kernel_t TKR = Record.record.target_kernel;
+    S.append(" (Target kernel) ").append(T);
+    S.append(" host_op_id=").append(std::to_string(TKR.host_op_id));
+    S.append(" requested_num_teams=")
+        .append(std::to_string(TKR.requested_num_teams));
+    S.append(" granted_num_teams=")
+        .append(std::to_string(TKR.granted_num_teams));
+    S.append(" end_time=").append(std::to_string(TKR.end_time));
+    S.append(" duration=").append(std::to_string(TKR.end_time - Record.time));
+    S.append(" ns");
+    break;
+  }
+  default:
+    S.append(" (unsupported record type)");
+    break;
+  }
+
+  return S;
+}
+
+std::string internal::BufferRecordDeallocation::toString() const {
+  std::string S{"Deallocated "};
+  S.append(makeHexString((uint64_t)Buffer));
+  return S;
+}
diff --git a/openmp/tools/omptest/src/InternalEventOperators.cpp b/openmp/tools/omptest/src/InternalEventOperators.cpp
new file mode 100644
index 0000000000000..1ec33d574381f
--- /dev/null
+++ b/openmp/tools/omptest/src/InternalEventOperators.cpp
@@ -0,0 +1,281 @@
+//===- InternalEventOperators.cpp - Operator implementations ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Defines the internal event operators, like comparators.
+///
+//===----------------------------------------------------------------------===//
+
+#include "InternalEvent.h"
+
+namespace omptest {
+
+namespace internal {
+
+bool operator==(const ParallelBegin &Expected, const ParallelBegin &Observed) {
+  return Expected.NumThreads == Observed.NumThreads;
+}
+
+bool operator==(const Work &Expected, const Work &Observed) {
+  bool isSameWorkType = (Expected.WorkType == Observed.WorkType);
+  bool isSameEndpoint = (Expected.Endpoint == Observed.Endpoint);
+  bool isSameParallelData =
+      (Expected.ParallelData == std::numeric_limits<ompt_data_t *>::min()) ||
+      (Expected.ParallelData == Observed.ParallelData);
+  bool isSameTaskData =
+      (Expected.TaskData == std::numeric_limits<ompt_data_t *>::min()) ||
+      (Expected.TaskData == Observed.TaskData);
+  bool isSameCount = (Expected.Count == std::numeric_limits<uint64_t>::min()) ||
+                     (Expected.Count == Observed.Count);
+  return isSameWorkType && isSameEndpoint && isSameParallelData &&
+         isSameTaskData && isSameCount;
+}
+
+bool operator==(const ImplicitTask &Expected, const ImplicitTask &Observed) {
+  bool isSameEndpoint = (Expected.Endpoint == Observed.Endpoint);
+  bool isSameActualParallelism =
+      (Expected.ActualParallelism ==
+       std::numeric_limits<unsigned int>::min()) ||
+      (Expected.ActualParallelism == Observed.ActualParallelism);
+  bool isSameIndex =
+      (Expected.Index == std::numeric_limits<unsigned int>::min()) ||
+      (Expected.Index == Observed.Index);
+  return isSameEndpoint && isSameActualParallelism && isSameIndex;
+}
+
+bool operator==(const SyncRegion &Expected, const SyncRegion &Observed) {
+  bool isSameKind = (Expected.Kind == Observed.Kind);
+  bool isSameEndpoint = (Expected.Endpoint == Observed.Endpoint);
+  bool isSameParallelData =
+      (Expected.ParallelData == std::numeric_limits<ompt_data_t *>::min()) ||
+      (Expected.ParallelData == Observed.ParallelData);
+  bool isSameTaskData =
+      (Expected.TaskData == std::numeric_limits<ompt_data_t *>::min()) ||
+      (Expected.TaskData == Observed.TaskData);
+  return isSameKind && isSameEndpoint && isSameParallelData && isSameTaskData;
+}
+
+bool operator==(const Target &Expected, const Target &Observed) {
+  bool isSameKind = (Expected.Kind == Observed.Kind);
+  bool isSameEndpoint = (Expected.Endpoint == Observed.Endpoint);
+  bool isSameDeviceNum =
+      (Expected.DeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.DeviceNum == Observed.DeviceNum);
+  return isSameKind && isSameEndpoint && isSameDeviceNum;
+}
+
+bool operator==(const TargetEmi &Expected, const TargetEmi &Observed) {
+  bool isSameKind = (Expected.Kind == Observed.Kind);
+  bool isSameEndpoint = (Expected.Endpoint == Observed.Endpoint);
+  bool isSameDeviceNum =
+      (Expected.DeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.DeviceNum == Observed.DeviceNum);
+  return isSameKind && isSameEndpoint && isSameDeviceNum;
+}
+
+bool operator==(const TargetDataOp &Expected, const TargetDataOp &Observed) {
+  bool isSameOpType = (Expected.OpType == Observed.OpType);
+  bool isSameSize = (Expected.Bytes == std::numeric_limits<size_t>::min()) ||
+                    (Expected.Bytes == Observed.Bytes);
+  bool isSameSrcAddr =
+      (Expected.SrcAddr == std::numeric_limits<void *>::min()) ||
+      (Expected.SrcAddr == Observed.SrcAddr);
+  bool isSameDstAddr =
+      (Expected.DstAddr == std::numeric_limits<void *>::min()) ||
+      (Expected.DstAddr == Observed.DstAddr);
+  bool isSameSrcDeviceNum =
+      (Expected.SrcDeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.SrcDeviceNum == Observed.SrcDeviceNum);
+  bool isSameDstDeviceNum =
+      (Expected.DstDeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.DstDeviceNum == Observed.DstDeviceNum);
+  return isSameOpType && isSameSize && isSameSrcAddr && isSameDstAddr &&
+         isSameSrcDeviceNum && isSameDstDeviceNum;
+}
+
+bool operator==(const TargetDataOpEmi &Expected,
+                const TargetDataOpEmi &Observed) {
+  bool isSameOpType = (Expected.OpType == Observed.OpType);
+  bool isSameEndpoint = (Expected.Endpoint == Observed.Endpoint);
+  bool isSameSize = (Expected.Bytes == std::numeric_limits<size_t>::min()) ||
+                    (Expected.Bytes == Observed.Bytes);
+  bool isSameSrcAddr =
+      (Expected.SrcAddr == std::numeric_limits<void *>::min()) ||
+      (Expected.SrcAddr == Observed.SrcAddr);
+  bool isSameDstAddr =
+      (Expected.DstAddr == std::numeric_limits<void *>::min()) ||
+      (Expected.DstAddr == Observed.DstAddr);
+  bool isSameSrcDeviceNum =
+      (Expected.SrcDeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.SrcDeviceNum == Observed.SrcDeviceNum);
+  bool isSameDstDeviceNum =
+      (Expected.DstDeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.DstDeviceNum == Observed.DstDeviceNum);
+  return isSameOpType && isSameEndpoint && isSameSize && isSameSrcAddr &&
+         isSameDstAddr && isSameSrcDeviceNum && isSameDstDeviceNum;
+}
+
+bool operator==(const TargetSubmit &Expected, const TargetSubmit &Observed) {
+  bool isSameReqNumTeams =
+      (Expected.RequestedNumTeams == Observed.RequestedNumTeams);
+  return isSameReqNumTeams;
+}
+
+bool operator==(const TargetSubmitEmi &Expected,
+                const TargetSubmitEmi &Observed) {
+  bool isSameReqNumTeams =
+      (Expected.RequestedNumTeams == Observed.RequestedNumTeams);
+  bool isSameEndpoint = (Expected.Endpoint == Observed.Endpoint);
+  return isSameReqNumTeams && isSameEndpoint;
+}
+
+bool operator==(const DeviceInitialize &Expected,
+                const DeviceInitialize &Observed) {
+  bool isSameDeviceNum = (Expected.DeviceNum == Observed.DeviceNum);
+  bool isSameType =
+      (Expected.Type == std::numeric_limits<const char *>::min()) ||
+      ((Expected.Type == Observed.Type) ||
+       (strcmp(Expected.Type, Observed.Type) == 0));
+  bool isSameDevice =
+      (Expected.Device == std::numeric_limits<ompt_device_t *>::min()) ||
+      (Expected.Device == Observed.Device);
+  return isSameDeviceNum && isSameType && isSameDevice;
+}
+
+bool operator==(const DeviceFinalize &Expected,
+                const DeviceFinalize &Observed) {
+  bool isSameDeviceNum =
+      (Expected.DeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.DeviceNum == Observed.DeviceNum);
+  return isSameDeviceNum;
+}
+
+bool operator==(const DeviceLoad &Expected, const DeviceLoad &Observed) {
+  bool isSameDeviceNum =
+      (Expected.DeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.DeviceNum == Observed.DeviceNum);
+  bool isSameSize = (Expected.Bytes == std::numeric_limits<size_t>::min()) ||
+                    (Expected.Bytes == Observed.Bytes);
+  return isSameDeviceNum && isSameSize;
+}
+
+bool operator==(const BufferRequest &Expected, const BufferRequest &Observed) {
+  bool isSameDeviceNum =
+      (Expected.DeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.DeviceNum == Observed.DeviceNum);
+  bool isSameSize = (Expected.Bytes == std::numeric_limits<size_t *>::min()) ||
+                    (Expected.Bytes == Observed.Bytes);
+  return isSameDeviceNum && isSameSize;
+}
+
+bool operator==(const BufferComplete &Expected,
+                const BufferComplete &Observed) {
+  bool isSameDeviceNum =
+      (Expected.DeviceNum == std::numeric_limits<int>::min()) ||
+      (Expected.DeviceNum == Observed.DeviceNum);
+  bool isSameSize = (Expected.Bytes == std::numeric_limits<size_t>::min()) ||
+                    (Expected.Bytes == Observed.Bytes);
+  return isSameDeviceNum && isSameSize;
+}
+
+bool operator==(const BufferRecord &Expected, const BufferRecord &Observed) {
+  bool isSameType = (Expected.Record.type == Observed.Record.type);
+  bool isSameTargetId =
+      (Expected.Record.target_id == std::numeric_limits<ompt_id_t>::min()) ||
+      (Expected.Record.target_id == Observed.Record.target_id);
+  if (!(isSameType && isSameTargetId))
+    return false;
+  bool isEqual = true;
+  ompt_device_time_t ObservedDurationNs =
+      Observed.Record.record.target_data_op.end_time - Observed.Record.time;
+  switch (Expected.Record.type) {
+  case ompt_callback_target:
+    isEqual &= (Expected.Record.record.target.kind ==
+                std::numeric_limits<ompt_target_t>::min()) ||
+               (Expected.Record.record.target.kind ==
+                Observed.Record.record.target.kind);
+    isEqual &= (Expected.Record.record.target.endpoint ==
+                std::numeric_limits<ompt_scope_endpoint_t>::min()) ||
+               (Expected.Record.record.target.endpoint ==
+                Observed.Record.record.target.endpoint);
+    isEqual &= (Expected.Record.record.target.device_num ==
+                std::numeric_limits<int>::min()) ||
+               (Expected.Record.record.target.device_num ==
+                Observed.Record.record.target.device_num);
+    break;
+  case ompt_callback_target_data_op:
+    isEqual &= (Expected.Record.record.target_data_op.optype ==
+                std::numeric_limits<ompt_target_data_op_t>::min()) ||
+               (Expected.Record.record.target_data_op.optype ==
+                Observed.Record.record.target_data_op.optype);
+    isEqual &= (Expected.Record.record.target_data_op.bytes ==
+                std::numeric_limits<size_t>::min()) ||
+               (Expected.Record.record.target_data_op.bytes ==
+                Observed.Record.record.target_data_op.bytes);
+    isEqual &= (Expected.Record.record.target_data_op.src_addr ==
+                std::numeric_limits<void *>::min()) ||
+               (Expected.Record.record.target_data_op.src_addr ==
+                Observed.Record.record.target_data_op.src_addr);
+    isEqual &= (Expected.Record.record.target_data_op.dest_addr ==
+                std::numeric_limits<void *>::min()) ||
+               (Expected.Record.record.target_data_op.dest_addr ==
+                Observed.Record.record.target_data_op.dest_addr);
+    isEqual &= (Expected.Record.record.target_data_op.src_device_num ==
+                std::numeric_limits<int>::min()) ||
+               (Expected.Record.record.target_data_op.src_device_num ==
+                Observed.Record.record.target_data_op.src_device_num);
+    isEqual &= (Expected.Record.record.target_data_op.dest_device_num ==
+                std::numeric_limits<int>::min()) ||
+               (Expected.Record.record.target_data_op.dest_device_num ==
+                Observed.Record.record.target_data_op.dest_device_num);
+    isEqual &= (Expected.Record.record.target_data_op.host_op_id ==
+                std::numeric_limits<ompt_id_t>::min()) ||
+               (Expected.Record.record.target_data_op.host_op_id ==
+                Observed.Record.record.target_data_op.host_op_id);
+    isEqual &= (Expected.Record.record.target_data_op.codeptr_ra ==
+                std::numeric_limits<void *>::min()) ||
+               (Expected.Record.record.target_data_op.codeptr_ra ==
+                Observed.Record.record.target_data_op.codeptr_ra);
+    if (Expected.Record.record.target_data_op.end_time !=
+        std::numeric_limits<ompt_device_time_t>::min()) {
+      isEqual &=
+          ObservedDurationNs <= Expected.Record.record.target_data_op.end_time;
+    }
+    isEqual &= ObservedDurationNs >= Expected.Record.time;
+    break;
+  case ompt_callback_target_submit:
+    ObservedDurationNs =
+        Observed.Record.record.target_kernel.end_time - Observed.Record.time;
+    isEqual &= (Expected.Record.record.target_kernel.requested_num_teams ==
+                std::numeric_limits<unsigned int>::min()) ||
+               (Expected.Record.record.target_kernel.requested_num_teams ==
+                Observed.Record.record.target_kernel.requested_num_teams);
+    isEqual &= (Expected.Record.record.target_kernel.granted_num_teams ==
+                std::numeric_limits<unsigned int>::min()) ||
+               (Expected.Record.record.target_kernel.granted_num_teams ==
+                Observed.Record.record.target_kernel.granted_num_teams);
+    isEqual &= (Expected.Record.record.target_kernel.host_op_id ==
+                std::numeric_limits<ompt_id_t>::min()) ||
+               (Expected.Record.record.target_kernel.host_op_id ==
+                Observed.Record.record.target_kernel.host_op_id);
+    if (Expected.Record.record.target_kernel.end_time !=
+        std::numeric_limits<ompt_device_time_t>::min()) {
+      isEqual &=
+          ObservedDurationNs <= Expected.Record.record.target_kernel.end_time;
+    }
+    isEqual &= ObservedDurationNs >= Expected.Record.time;
+    break;
+  default:
+    assert(false && "Encountered invalid record type");
+  }
+  return isEqual;
+}
+
+} // namespace internal
+
+} // namespace omptest
diff --git a/openmp/tools/omptest/src/Logging.cpp b/openmp/tools/omptest/src/Logging.cpp
new file mode 100644
index 0000000000000..c609b6ee83aec
--- /dev/null
+++ b/openmp/tools/omptest/src/Logging.cpp
@@ -0,0 +1,181 @@
+//===- Logging.cpp - General logging class implementation -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implements ompTest-tailored logging.
+///
+//===----------------------------------------------------------------------===//
+
+#include "Logging.h"
+
+using namespace omptest;
+using namespace logging;
+
+Logger::Logger(Level LogLevel, std::ostream &OutStream, bool FormatOutput)
+    : LoggingLevel(LogLevel), OutStream(OutStream), FormatOutput(FormatOutput) {
+  // Flush any buffered output
+  OutStream << std::flush;
+}
+
+Logger::~Logger() {
+  // Flush any buffered output
+  OutStream << std::flush;
+}
+
+std::map<Level, std::set<FormatOption>> AggregatedFormatOptions{
+    {Level::Diagnostic, {FormatOption::ColorLightBlue}},
+    {Level::Info, {FormatOption::ColorLightGray}},
+    {Level::Warning, {FormatOption::ColorLightYellow}},
+    {Level::Error, {FormatOption::ColorRed}},
+    {Level::Critical, {FormatOption::ColorLightRed}},
+    {Level::Default, {FormatOption::None}},
+    {Level::ExpectedEvent, {FormatOption::Bold, FormatOption::ColorCyan}},
+    {Level::ObservedEvent, {FormatOption::ColorCyan}},
+    {Level::OffendingEvent, {FormatOption::ColorYellow}}};
+
+const char *logging::to_string(Level LogLevel) {
+  switch (LogLevel) {
+  case Level::Diagnostic:
+    return "Diagnostic";
+  case Level::Info:
+    return "Info";
+  case Level::Warning:
+    return "Warning";
+  case Level::Error:
+    return "Error";
+  case Level::Critical:
+    return "Critical";
+  case Level::Silent:
+    return "Silent";
+  default:
+    assert(false && "Requested string representation for unknown LogLevel");
+    return "UNKNOWN";
+  }
+}
+
+std::string logging::getFormatSequence(Level LogLevel) {
+  auto Options = AggregatedFormatOptions[LogLevel];
+  std::stringstream SS{"\033["};
+  SS << "\033[";
+  if (!Options.empty()) {
+    for (auto &Option : AggregatedFormatOptions[LogLevel])
+      SS << int(Option) << ';';
+    SS.seekp(-1, SS.cur);
+    SS << 'm';
+  } else {
+    // Fallback to None / reset formatting
+    SS << "0m";
+  }
+  return SS.str();
+}
+
+std::string logging::format(const std::string &Message, FormatOption Option) {
+  std::stringstream SS{"\033["};
+  SS << "\033[";
+  SS << int(Option) << 'm' << Message << "\033[0m";
+  return SS.str();
+}
+
+std::string logging::format(const std::string &Message,
+                            std::set<FormatOption> Options) {
+  std::stringstream SS{"\033["};
+  SS << "\033[";
+  for (auto &Option : Options)
+    SS << int(Option) << ';';
+  SS.seekp(-1, SS.cur);
+  SS << 'm' << Message << "\033[0m";
+  return SS.str();
+}
+
+void Logger::log(const std::string &Message, Level LogLevel) const {
+  // Serialize logging
+  std::lock_guard<std::mutex> Lock(LogMutex);
+
+  if (LoggingLevel > LogLevel)
+    return;
+
+  if (FormatOutput) {
+    OutStream << getFormatSequence(LogLevel) << '[' << to_string(LogLevel)
+              << "] " << Message << getFormatSequence() << std::endl;
+  } else {
+    OutStream << '[' << to_string(LogLevel) << "] " << Message << std::endl;
+  }
+}
+
+void Logger::logEventMismatch(const std::string &Message,
+                              const omptest::OmptAssertEvent &OffendingEvent,
+                              Level LogLevel) const {
+  // Serialize logging
+  std::lock_guard<std::mutex> Lock(LogMutex);
+  if (LoggingLevel > LogLevel)
+    return;
+
+  if (FormatOutput) {
+    OutStream << getFormatSequence(LogLevel) << '[' << to_string(LogLevel)
+              << "] " << getFormatSequence()
+              << format(Message, AggregatedFormatOptions[LogLevel])
+              << "\n\tOffending event name='"
+              << format(OffendingEvent.getEventName(),
+                        AggregatedFormatOptions[Level::OffendingEvent])
+              << "'\n\tOffending='"
+              << format(OffendingEvent.toString(),
+                        AggregatedFormatOptions[Level::OffendingEvent])
+              << '\'' << std::endl;
+  } else {
+    OutStream << '[' << to_string(LogLevel) << "] " << Message
+              << "\n\tOffending event name='" << OffendingEvent.getEventName()
+              << "'\n\tOffending='" << OffendingEvent.toString() << '\''
+              << std::endl;
+  }
+}
+
+void Logger::logEventMismatch(const std::string &Message,
+                              const omptest::OmptAssertEvent &ExpectedEvent,
+                              const omptest::OmptAssertEvent &ObservedEvent,
+                              Level LogLevel) const {
+  // Serialize logging
+  std::lock_guard<std::mutex> Lock(LogMutex);
+  if (LoggingLevel > LogLevel)
+    return;
+
+  if (FormatOutput) {
+    OutStream << getFormatSequence(LogLevel) << '[' << to_string(LogLevel)
+              << "] " << Message << getFormatSequence()
+              << "\n\tExpected event name='"
+              << format(ExpectedEvent.getEventName(),
+                        AggregatedFormatOptions[Level::ExpectedEvent])
+              << "' observe='"
+              << format(to_string(ExpectedEvent.getEventExpectedState()),
+                        AggregatedFormatOptions[Level::ExpectedEvent])
+              << "'\n\tObserved event name='"
+              << format(ObservedEvent.getEventName(),
+                        AggregatedFormatOptions[Level::ObservedEvent])
+              << "'\n\tExpected='"
+              << format(ExpectedEvent.toString(),
+                        AggregatedFormatOptions[Level::ExpectedEvent])
+              << "'\n\tObserved='"
+              << format(ObservedEvent.toString(),
+                        AggregatedFormatOptions[Level::ObservedEvent])
+              << '\'' << std::endl;
+  } else {
+    OutStream << '[' << to_string(LogLevel) << "] " << Message
+              << "\n\tExpected event name='" << ExpectedEvent.getEventName()
+              << "' observe='"
+              << to_string(ExpectedEvent.getEventExpectedState())
+              << "'\n\tObserved event name='" << ObservedEvent.getEventName()
+              << "'\n\tExpected='" << ExpectedEvent.toString()
+              << "'\n\tObserved='" << ObservedEvent.toString() << '\''
+              << std::endl;
+  }
+}
+
+void Logger::setFormatOutput(bool Enabled) { FormatOutput = Enabled; }
+
+Level Logger::getLoggingLevel() const { return LoggingLevel; }
+
+void Logger::setLoggingLevel(Level LogLevel) { LoggingLevel = LogLevel; }
diff --git a/openmp/tools/omptest/src/OmptAssertEvent.cpp b/openmp/tools/omptest/src/OmptAssertEvent.cpp
new file mode 100644
index 0000000000000..bbf2d7cd4a10f
--- /dev/null
+++ b/openmp/tools/omptest/src/OmptAssertEvent.cpp
@@ -0,0 +1,587 @@
+//===- OmptAssertEvent.cpp - Assertion event implementations ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implements assertion event CTORs, for generally all observable events.
+///
+//===----------------------------------------------------------------------===//
+
+#include "OmptAssertEvent.h"
+#include <omp-tools.h>
+
+using namespace omptest;
+
+const char *omptest::to_string(ObserveState State) {
+  switch (State) {
+  case ObserveState::Generated:
+    return "Generated";
+  case ObserveState::Always:
+    return "Always";
+  case ObserveState::Never:
+    return "Never";
+  default:
+    assert(false && "Requested string representation for unknown ObserveState");
+    return "UNKNOWN";
+  }
+}
+
+OmptAssertEvent::OmptAssertEvent(const std::string &Name,
+                                 const std::string &Group,
+                                 const ObserveState &Expected,
+                                 internal::InternalEvent *IE)
+    : Name(Name), Group(Group), ExpectedState(Expected), TheEvent(IE) {}
+
+OmptAssertEvent OmptAssertEvent::AssertionSyncPoint(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, const std::string &SyncPointName) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::AssertionSyncPoint(SyncPointName));
+}
+
+OmptAssertEvent
+OmptAssertEvent::AssertionSuspend(const std::string &Name,
+                                  const std::string &Group,
+                                  const ObserveState &Expected) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::AssertionSuspend());
+}
+
+OmptAssertEvent OmptAssertEvent::ThreadBegin(const std::string &Name,
+                                             const std::string &Group,
+                                             const ObserveState &Expected,
+                                             ompt_thread_t ThreadType) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::ThreadBegin(ThreadType));
+}
+
+OmptAssertEvent OmptAssertEvent::ThreadEnd(const std::string &Name,
+                                           const std::string &Group,
+                                           const ObserveState &Expected) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected, new internal::ThreadEnd());
+}
+
+OmptAssertEvent OmptAssertEvent::ParallelBegin(const std::string &Name,
+                                               const std::string &Group,
+                                               const ObserveState &Expected,
+                                               int NumThreads) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::ParallelBegin(NumThreads));
+}
+
+OmptAssertEvent OmptAssertEvent::ParallelEnd(const std::string &Name,
+                                             const std::string &Group,
+                                             const ObserveState &Expected,
+                                             ompt_data_t *ParallelData,
+                                             ompt_data_t *EncounteringTaskData,
+                                             int Flags, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::ParallelEnd(ParallelData,
+                                                   EncounteringTaskData, Flags,
+                                                   CodeptrRA));
+}
+
+OmptAssertEvent
+OmptAssertEvent::Work(const std::string &Name, const std::string &Group,
+                      const ObserveState &Expected, ompt_work_t WorkType,
+                      ompt_scope_endpoint_t Endpoint, ompt_data_t *ParallelData,
+                      ompt_data_t *TaskData, uint64_t Count,
+                      const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::Work(WorkType, Endpoint, ParallelData,
+                                            TaskData, Count, CodeptrRA));
+}
+
+OmptAssertEvent
+OmptAssertEvent::Dispatch(const std::string &Name, const std::string &Group,
+                          const ObserveState &Expected,
+                          ompt_data_t *ParallelData, ompt_data_t *TaskData,
+                          ompt_dispatch_t Kind, ompt_data_t Instance) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(
+      EName, EGroup, Expected,
+      new internal::Dispatch(ParallelData, TaskData, Kind, Instance));
+}
+
+OmptAssertEvent OmptAssertEvent::TaskCreate(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_data_t *EncounteringTaskData,
+    const ompt_frame_t *EncounteringTaskFrame, ompt_data_t *NewTaskData,
+    int Flags, int HasDependences, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(
+      EName, EGroup, Expected,
+      new internal::TaskCreate(EncounteringTaskData, EncounteringTaskFrame,
+                               NewTaskData, Flags, HasDependences, CodeptrRA));
+}
+
+OmptAssertEvent OmptAssertEvent::TaskSchedule(const std::string &Name,
+                                              const std::string &Group,
+                                              const ObserveState &Expected) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected, new internal::TaskSchedule());
+}
+
+OmptAssertEvent OmptAssertEvent::ImplicitTask(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_scope_endpoint_t Endpoint,
+    ompt_data_t *ParallelData, ompt_data_t *TaskData,
+    unsigned int ActualParallelism, unsigned int Index, int Flags) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::ImplicitTask(Endpoint, ParallelData,
+                                                    TaskData, ActualParallelism,
+                                                    Index, Flags));
+}
+
+OmptAssertEvent OmptAssertEvent::SyncRegion(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_sync_region_t Kind,
+    ompt_scope_endpoint_t Endpoint, ompt_data_t *ParallelData,
+    ompt_data_t *TaskData, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::SyncRegion(Kind, Endpoint, ParallelData,
+                                                  TaskData, CodeptrRA));
+}
+
+OmptAssertEvent
+OmptAssertEvent::Target(const std::string &Name, const std::string &Group,
+                        const ObserveState &Expected, ompt_target_t Kind,
+                        ompt_scope_endpoint_t Endpoint, int DeviceNum,
+                        ompt_data_t *TaskData, ompt_id_t TargetId,
+                        const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::Target(Kind, Endpoint, DeviceNum,
+                                              TaskData, TargetId, CodeptrRA));
+}
+
+OmptAssertEvent
+OmptAssertEvent::TargetEmi(const std::string &Name, const std::string &Group,
+                           const ObserveState &Expected, ompt_target_t Kind,
+                           ompt_scope_endpoint_t Endpoint, int DeviceNum,
+                           ompt_data_t *TaskData, ompt_data_t *TargetTaskData,
+                           ompt_data_t *TargetData, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::TargetEmi(Kind, Endpoint, DeviceNum,
+                                                 TaskData, TargetTaskData,
+                                                 TargetData, CodeptrRA));
+}
+
+OmptAssertEvent OmptAssertEvent::TargetDataOp(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_id_t TargetId, ompt_id_t HostOpId,
+    ompt_target_data_op_t OpType, void *SrcAddr, int SrcDeviceNum,
+    void *DstAddr, int DstDeviceNum, size_t Bytes, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::TargetDataOp(
+                             TargetId, HostOpId, OpType, SrcAddr, SrcDeviceNum,
+                             DstAddr, DstDeviceNum, Bytes, CodeptrRA));
+}
+
+OmptAssertEvent OmptAssertEvent::TargetDataOp(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_target_data_op_t OpType, size_t Bytes,
+    void *SrcAddr, void *DstAddr, int SrcDeviceNum, int DstDeviceNum,
+    ompt_id_t TargetId, ompt_id_t HostOpId, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::TargetDataOp(
+                             TargetId, HostOpId, OpType, SrcAddr, SrcDeviceNum,
+                             DstAddr, DstDeviceNum, Bytes, CodeptrRA));
+}
+
+OmptAssertEvent OmptAssertEvent::TargetDataOpEmi(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_scope_endpoint_t Endpoint,
+    ompt_data_t *TargetTaskData, ompt_data_t *TargetData, ompt_id_t *HostOpId,
+    ompt_target_data_op_t OpType, void *SrcAddr, int SrcDeviceNum,
+    void *DstAddr, int DstDeviceNum, size_t Bytes, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(
+      EName, EGroup, Expected,
+      new internal::TargetDataOpEmi(Endpoint, TargetTaskData, TargetData,
+                                    HostOpId, OpType, SrcAddr, SrcDeviceNum,
+                                    DstAddr, DstDeviceNum, Bytes, CodeptrRA));
+}
+
+OmptAssertEvent OmptAssertEvent::TargetDataOpEmi(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_target_data_op_t OpType,
+    ompt_scope_endpoint_t Endpoint, size_t Bytes, void *SrcAddr, void *DstAddr,
+    int SrcDeviceNum, int DstDeviceNum, ompt_data_t *TargetTaskData,
+    ompt_data_t *TargetData, ompt_id_t *HostOpId, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(
+      EName, EGroup, Expected,
+      new internal::TargetDataOpEmi(Endpoint, TargetTaskData, TargetData,
+                                    HostOpId, OpType, SrcAddr, SrcDeviceNum,
+                                    DstAddr, DstDeviceNum, Bytes, CodeptrRA));
+}
+
+OmptAssertEvent OmptAssertEvent::TargetSubmit(const std::string &Name,
+                                              const std::string &Group,
+                                              const ObserveState &Expected,
+                                              ompt_id_t TargetId,
+                                              ompt_id_t HostOpId,
+                                              unsigned int RequestedNumTeams) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(
+      EName, EGroup, Expected,
+      new internal::TargetSubmit(TargetId, HostOpId, RequestedNumTeams));
+}
+
+OmptAssertEvent OmptAssertEvent::TargetSubmit(const std::string &Name,
+                                              const std::string &Group,
+                                              const ObserveState &Expected,
+                                              unsigned int RequestedNumTeams,
+                                              ompt_id_t TargetId,
+                                              ompt_id_t HostOpId) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(
+      EName, EGroup, Expected,
+      new internal::TargetSubmit(TargetId, HostOpId, RequestedNumTeams));
+}
+
+OmptAssertEvent OmptAssertEvent::TargetSubmitEmi(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_scope_endpoint_t Endpoint,
+    ompt_data_t *TargetData, ompt_id_t *HostOpId,
+    unsigned int RequestedNumTeams) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::TargetSubmitEmi(Endpoint, TargetData,
+                                                       HostOpId,
+                                                       RequestedNumTeams));
+}
+
+OmptAssertEvent OmptAssertEvent::TargetSubmitEmi(const std::string &Name,
+                                                 const std::string &Group,
+                                                 const ObserveState &Expected,
+                                                 unsigned int RequestedNumTeams,
+                                                 ompt_scope_endpoint_t Endpoint,
+                                                 ompt_data_t *TargetData,
+                                                 ompt_id_t *HostOpId) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::TargetSubmitEmi(Endpoint, TargetData,
+                                                       HostOpId,
+                                                       RequestedNumTeams));
+}
+
+OmptAssertEvent OmptAssertEvent::ControlTool(const std::string &Name,
+                                             const std::string &Group,
+                                             const ObserveState &Expected) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected, new internal::ControlTool());
+}
+
+OmptAssertEvent OmptAssertEvent::DeviceInitialize(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, int DeviceNum, const char *Type,
+    ompt_device_t *Device, ompt_function_lookup_t LookupFn,
+    const char *DocumentationStr) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::DeviceInitialize(DeviceNum, Type, Device,
+                                                        LookupFn,
+                                                        DocumentationStr));
+}
+
+OmptAssertEvent OmptAssertEvent::DeviceFinalize(const std::string &Name,
+                                                const std::string &Group,
+                                                const ObserveState &Expected,
+                                                int DeviceNum) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::DeviceFinalize(DeviceNum));
+}
+
+OmptAssertEvent
+OmptAssertEvent::DeviceLoad(const std::string &Name, const std::string &Group,
+                            const ObserveState &Expected, int DeviceNum,
+                            const char *Filename, int64_t OffsetInFile,
+                            void *VmaInFile, size_t Bytes, void *HostAddr,
+                            void *DeviceAddr, uint64_t ModuleId) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(
+      EName, EGroup, Expected,
+      new internal::DeviceLoad(DeviceNum, Filename, OffsetInFile, VmaInFile,
+                               Bytes, HostAddr, DeviceAddr, ModuleId));
+}
+
+OmptAssertEvent OmptAssertEvent::DeviceUnload(const std::string &Name,
+                                              const std::string &Group,
+                                              const ObserveState &Expected) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected, new internal::DeviceUnload());
+}
+
+OmptAssertEvent OmptAssertEvent::BufferRequest(const std::string &Name,
+                                               const std::string &Group,
+                                               const ObserveState &Expected,
+                                               int DeviceNum,
+                                               ompt_buffer_t **Buffer,
+                                               size_t *Bytes) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::BufferRequest(DeviceNum, Buffer, Bytes));
+}
+
+OmptAssertEvent OmptAssertEvent::BufferComplete(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, int DeviceNum, ompt_buffer_t *Buffer,
+    size_t Bytes, ompt_buffer_cursor_t Begin, int BufferOwned) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::BufferComplete(DeviceNum, Buffer, Bytes,
+                                                      Begin, BufferOwned));
+}
+
+OmptAssertEvent OmptAssertEvent::BufferRecord(const std::string &Name,
+                                              const std::string &Group,
+                                              const ObserveState &Expected,
+                                              ompt_record_ompt_t *Record) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::BufferRecord(Record));
+}
+
+OmptAssertEvent OmptAssertEvent::BufferRecord(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_callbacks_t Type, ompt_target_t Kind,
+    ompt_scope_endpoint_t Endpoint, int DeviceNum, ompt_id_t TaskId,
+    ompt_id_t TargetId, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+
+  if (Type != ompt_callback_target)
+    assert(false && "CTOR only suited for type: 'ompt_callback_target'");
+
+  ompt_record_target_t Subrecord{Kind,   Endpoint, DeviceNum,
+                                 TaskId, TargetId, CodeptrRA};
+
+  ompt_record_ompt_t *RecordPtr =
+      (ompt_record_ompt_t *)malloc(sizeof(ompt_record_ompt_t));
+  memset(RecordPtr, 0, sizeof(ompt_record_ompt_t));
+  RecordPtr->type = Type;
+  RecordPtr->time = expectedDefault(ompt_device_time_t);
+  RecordPtr->thread_id = expectedDefault(ompt_id_t);
+  RecordPtr->target_id = TargetId;
+  RecordPtr->record.target = Subrecord;
+
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::BufferRecord(RecordPtr));
+}
+
+OmptAssertEvent OmptAssertEvent::BufferRecord(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_callbacks_t Type,
+    ompt_target_data_op_t OpType, size_t Bytes,
+    std::pair<ompt_device_time_t, ompt_device_time_t> Timeframe, void *SrcAddr,
+    void *DstAddr, int SrcDeviceNum, int DstDeviceNum, ompt_id_t TargetId,
+    ompt_id_t HostOpId, const void *CodeptrRA) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+
+  if (Type != ompt_callback_target_data_op)
+    assert(false &&
+           "CTOR only suited for type: 'ompt_callback_target_data_op'");
+
+  ompt_record_target_data_op_t Subrecord{
+      HostOpId,     OpType, SrcAddr,          SrcDeviceNum, DstAddr,
+      DstDeviceNum, Bytes,  Timeframe.second, CodeptrRA};
+
+  ompt_record_ompt_t *RecordPtr =
+      (ompt_record_ompt_t *)malloc(sizeof(ompt_record_ompt_t));
+  memset(RecordPtr, 0, sizeof(ompt_record_ompt_t));
+  RecordPtr->type = Type;
+  RecordPtr->time = Timeframe.first;
+  RecordPtr->thread_id = expectedDefault(ompt_id_t);
+  RecordPtr->target_id = TargetId;
+  RecordPtr->record.target_data_op = Subrecord;
+
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::BufferRecord(RecordPtr));
+}
+
+OmptAssertEvent OmptAssertEvent::BufferRecord(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_callbacks_t Type,
+    ompt_target_data_op_t OpType, size_t Bytes,
+    ompt_device_time_t MinimumTimeDelta, void *SrcAddr, void *DstAddr,
+    int SrcDeviceNum, int DstDeviceNum, ompt_id_t TargetId, ompt_id_t HostOpId,
+    const void *CodeptrRA) {
+  return BufferRecord(Name, Group, Expected, Type, OpType, Bytes,
+                      {MinimumTimeDelta, expectedDefault(ompt_device_time_t)},
+                      SrcAddr, DstAddr, SrcDeviceNum, DstDeviceNum, TargetId,
+                      HostOpId, CodeptrRA);
+}
+
+OmptAssertEvent OmptAssertEvent::BufferRecord(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_callbacks_t Type,
+    std::pair<ompt_device_time_t, ompt_device_time_t> Timeframe,
+    unsigned int RequestedNumTeams, unsigned int GrantedNumTeams,
+    ompt_id_t TargetId, ompt_id_t HostOpId) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+
+  bool isDefault = (Timeframe.first == expectedDefault(ompt_device_time_t));
+  isDefault &= (Timeframe.second == expectedDefault(ompt_device_time_t));
+  isDefault &= (RequestedNumTeams == expectedDefault(unsigned int));
+  isDefault &= (GrantedNumTeams == expectedDefault(unsigned int));
+  isDefault &= (TargetId == expectedDefault(ompt_id_t));
+  isDefault &= (HostOpId == expectedDefault(ompt_id_t));
+
+  ompt_record_ompt_t *RecordPtr =
+      (ompt_record_ompt_t *)malloc(sizeof(ompt_record_ompt_t));
+  memset(RecordPtr, 0, sizeof(ompt_record_ompt_t));
+  RecordPtr->type = Type;
+
+  // This handles the simplest occurrence of a device tracing record
+  // We can only check for Type -- since all other properties are set to default
+  if (isDefault) {
+    RecordPtr->time = expectedDefault(ompt_device_time_t);
+    RecordPtr->thread_id = expectedDefault(ompt_id_t);
+    RecordPtr->target_id = expectedDefault(ompt_id_t);
+    if (Type == ompt_callback_target) {
+      ompt_record_target_t Subrecord{expectedDefault(ompt_target_t),
+                                     expectedDefault(ompt_scope_endpoint_t),
+                                     expectedDefault(int),
+                                     expectedDefault(ompt_id_t),
+                                     expectedDefault(ompt_id_t),
+                                     expectedDefault(void *)};
+      RecordPtr->record.target = Subrecord;
+    }
+
+    if (Type == ompt_callback_target_data_op) {
+      ompt_record_target_data_op_t Subrecord{
+          expectedDefault(ompt_id_t), expectedDefault(ompt_target_data_op_t),
+          expectedDefault(void *),    expectedDefault(int),
+          expectedDefault(void *),    expectedDefault(int),
+          expectedDefault(size_t),    expectedDefault(ompt_device_time_t),
+          expectedDefault(void *)};
+      RecordPtr->record.target_data_op = Subrecord;
+    }
+
+    if (Type == ompt_callback_target_submit) {
+      ompt_record_target_kernel_t Subrecord{
+          expectedDefault(ompt_id_t), expectedDefault(unsigned int),
+          expectedDefault(unsigned int), expectedDefault(ompt_device_time_t)};
+      RecordPtr->record.target_kernel = Subrecord;
+    }
+
+    return OmptAssertEvent(EName, EGroup, Expected,
+                           new internal::BufferRecord(RecordPtr));
+  }
+
+  if (Type != ompt_callback_target_submit)
+    assert(false && "CTOR only suited for type: 'ompt_callback_target_submit'");
+
+  ompt_record_target_kernel_t Subrecord{HostOpId, RequestedNumTeams,
+                                        GrantedNumTeams, Timeframe.second};
+
+  RecordPtr->time = Timeframe.first;
+  RecordPtr->thread_id = expectedDefault(ompt_id_t);
+  RecordPtr->target_id = TargetId;
+  RecordPtr->record.target_kernel = Subrecord;
+
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::BufferRecord(RecordPtr));
+}
+
+OmptAssertEvent OmptAssertEvent::BufferRecord(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_callbacks_t Type,
+    ompt_device_time_t MinimumTimeDelta, unsigned int RequestedNumTeams,
+    unsigned int GrantedNumTeams, ompt_id_t TargetId, ompt_id_t HostOpId) {
+  return BufferRecord(Name, Group, Expected, Type,
+                      {MinimumTimeDelta, expectedDefault(ompt_device_time_t)},
+                      RequestedNumTeams, GrantedNumTeams, TargetId, HostOpId);
+}
+
+OmptAssertEvent OmptAssertEvent::BufferRecordDeallocation(
+    const std::string &Name, const std::string &Group,
+    const ObserveState &Expected, ompt_buffer_t *Buffer) {
+  auto EName = getName(Name);
+  auto EGroup = getGroup(Group);
+  return OmptAssertEvent(EName, EGroup, Expected,
+                         new internal::BufferRecordDeallocation(Buffer));
+}
+
+std::string OmptAssertEvent::getEventName() const { return Name; }
+
+std::string OmptAssertEvent::getEventGroup() const { return Group; }
+
+ObserveState OmptAssertEvent::getEventExpectedState() const {
+  return ExpectedState;
+}
+
+internal::EventTy OmptAssertEvent::getEventType() const {
+  return TheEvent->Type;
+}
+
+internal::InternalEvent *OmptAssertEvent::getEvent() const {
+  return TheEvent.get();
+}
+
+std::string OmptAssertEvent::toString(bool PrefixEventName) const {
+  std::string S;
+  if (PrefixEventName)
+    S.append(getEventName()).append(": ");
+  S.append((TheEvent == nullptr) ? "OmptAssertEvent" : TheEvent->toString());
+  return S;
+}
+
+bool omptest::operator==(const OmptAssertEvent &A, const OmptAssertEvent &B) {
+  assert(A.TheEvent.get() != nullptr && "A is valid");
+  assert(B.TheEvent.get() != nullptr && "B is valid");
+
+  return A.TheEvent->Type == B.TheEvent->Type &&
+         A.TheEvent->equals(B.TheEvent.get());
+}
diff --git a/openmp/tools/omptest/src/OmptAsserter.cpp b/openmp/tools/omptest/src/OmptAsserter.cpp
new file mode 100644
index 0000000000000..342ff7edae95e
--- /dev/null
+++ b/openmp/tools/omptest/src/OmptAsserter.cpp
@@ -0,0 +1,484 @@
+//===- OmptAsserter.cpp - Asserter-related implementations ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Implements all asserter-related class methods, like: notifications, handling
+/// of groups or determination of the testcase state.
+///
+//===----------------------------------------------------------------------===//
+
+#include "OmptAsserter.h"
+#include "Logging.h"
+
+#include <algorithm>
+
+using namespace omptest;
+using namespace internal;
+
+// Initialize static members
+std::mutex OmptAsserter::StaticMemberAccessMutex;
+std::weak_ptr<OmptEventGroupInterface>
+    OmptAsserter::EventGroupInterfaceInstance;
+std::weak_ptr<logging::Logger> OmptAsserter::LoggingInstance;
+
+OmptAsserter::OmptAsserter() {
+  // Protect static members access
+  std::lock_guard<std::mutex> Lock(StaticMemberAccessMutex);
+
+  // Upgrade OmptEventGroupInterface weak_ptr to shared_ptr
+  {
+    EventGroups = EventGroupInterfaceInstance.lock();
+    if (!EventGroups) {
+      // Coordinator doesn't exist or was previously destroyed, create a new
+      // one.
+      EventGroups = std::make_shared<OmptEventGroupInterface>();
+      // Store a weak reference to it
+      EventGroupInterfaceInstance = EventGroups;
+    }
+    // EventGroups is now a valid shared_ptr, either to a new or existing
+    // instance.
+  }
+
+  // Upgrade logging::Logger weak_ptr to shared_ptr
+  {
+    Log = LoggingInstance.lock();
+    if (!Log) {
+      // Coordinator doesn't exist or was previously destroyed, create a new
+      // one.
+      Log = std::make_shared<logging::Logger>();
+      // Store a weak reference to it
+      LoggingInstance = Log;
+    }
+    // Log is now a valid shared_ptr, either to a new or existing instance.
+  }
+}
+
+void OmptListener::setActive(bool Enabled) { Active = Enabled; }
+
+bool OmptListener::isActive() { return Active; }
+
+bool OmptListener::isSuppressedEventType(EventTy EvTy) {
+  return SuppressedEvents.find(EvTy) != SuppressedEvents.end();
+}
+
+void OmptListener::permitEvent(EventTy EvTy) { SuppressedEvents.erase(EvTy); }
+
+void OmptListener::suppressEvent(EventTy EvTy) {
+  SuppressedEvents.insert(EvTy);
+}
+
+void OmptAsserter::insert(OmptAssertEvent &&AE) {
+  assert(false && "Base class 'insert' has undefined semantics.");
+}
+
+void OmptAsserter::notify(OmptAssertEvent &&AE) {
+  // Ignore notifications while inactive
+  if (!isActive() || isSuppressedEventType(AE.getEventType()))
+    return;
+
+  this->notifyImpl(std::move(AE));
+}
+
+AssertState OmptAsserter::checkState() { return State; }
+
+bool OmptAsserter::verifyEventGroups(const OmptAssertEvent &ExpectedEvent,
+                                     const OmptAssertEvent &ObservedEvent) {
+  assert(ExpectedEvent.getEventType() == ObservedEvent.getEventType() &&
+         "Type mismatch: Expected != Observed event type");
+  assert(EventGroups && "Missing EventGroups interface");
+
+  // Ignore all events within "default" group
+  auto GroupName = ExpectedEvent.getEventGroup();
+
+  if (GroupName == "default")
+    return true;
+
+  // Get a pointer to the observed internal event
+  auto Event = ObservedEvent.getEvent();
+
+  switch (Event->Type) {
+  case EventTy::Target:
+    if (auto E = static_cast<const internal::Target *>(Event)) {
+      if (E->Endpoint == ompt_scope_begin) {
+        // Add new group since we entered a Target Region
+        EventGroups->addActiveEventGroup(GroupName,
+                                         AssertEventGroup{E->TargetId});
+      } else if (E->Endpoint == ompt_scope_end) {
+        // Deprecate group since we return from a Target Region
+        EventGroups->deprecateActiveEventGroup(GroupName);
+      }
+      return true;
+    }
+    return false;
+  case EventTy::TargetEmi:
+    if (auto E = static_cast<const internal::TargetEmi *>(Event)) {
+      if (E->Endpoint == ompt_scope_begin) {
+        // Add new group since we entered a Target Region
+        EventGroups->addActiveEventGroup(
+            GroupName, AssertEventGroup{E->TargetData->value});
+      } else if (E->Endpoint == ompt_scope_end) {
+        // Deprecate group since we return from a Target Region
+        EventGroups->deprecateActiveEventGroup(GroupName);
+      }
+      return true;
+    }
+    return false;
+  case EventTy::TargetDataOp:
+    if (auto E = static_cast<const internal::TargetDataOp *>(Event))
+      return EventGroups->checkActiveEventGroups(GroupName,
+                                                 AssertEventGroup{E->TargetId});
+
+    return false;
+  case EventTy::TargetDataOpEmi:
+    if (auto E = static_cast<const internal::TargetDataOpEmi *>(Event))
+      return EventGroups->checkActiveEventGroups(
+          GroupName, AssertEventGroup{E->TargetData->value});
+
+    return false;
+  case EventTy::TargetSubmit:
+    if (auto E = static_cast<const internal::TargetSubmit *>(Event))
+      return EventGroups->checkActiveEventGroups(GroupName,
+                                                 AssertEventGroup{E->TargetId});
+
+    return false;
+  case EventTy::TargetSubmitEmi:
+    if (auto E = static_cast<const internal::TargetSubmitEmi *>(Event))
+      return EventGroups->checkActiveEventGroups(
+          GroupName, AssertEventGroup{E->TargetData->value});
+
+    return false;
+  case EventTy::BufferRecord:
+    // BufferRecords are delivered asynchronously: also check deprecated groups.
+    if (auto E = static_cast<const internal::BufferRecord *>(Event))
+      return (EventGroups->checkActiveEventGroups(
+                  GroupName, AssertEventGroup{E->Record.target_id}) ||
+              EventGroups->checkDeprecatedEventGroups(
+                  GroupName, AssertEventGroup{E->Record.target_id}));
+    return false;
+  // Some event types do not need any handling
+  case EventTy::ThreadBegin:
+  case EventTy::ThreadEnd:
+  case EventTy::ParallelBegin:
+  case EventTy::ParallelEnd:
+  case EventTy::Work:
+  case EventTy::Dispatch:
+  case EventTy::TaskCreate:
+  case EventTy::Dependences:
+  case EventTy::TaskDependence:
+  case EventTy::TaskSchedule:
+  case EventTy::ImplicitTask:
+  case EventTy::Masked:
+  case EventTy::SyncRegion:
+  case EventTy::MutexAcquire:
+  case EventTy::Mutex:
+  case EventTy::NestLock:
+  case EventTy::Flush:
+  case EventTy::Cancel:
+  case EventTy::DeviceInitialize:
+  case EventTy::DeviceFinalize:
+  case EventTy::DeviceLoad:
+  case EventTy::DeviceUnload:
+  case EventTy::BufferRequest:
+  case EventTy::BufferComplete:
+  case EventTy::BufferRecordDeallocation:
+    return true;
+  // Some event types must not be encountered
+  case EventTy::None:
+  case EventTy::AssertionSyncPoint:
+  case EventTy::AssertionSuspend:
+  default:
+    Log->log("Observed invalid event type: " + Event->toString(),
+             logging::Level::Critical);
+    __builtin_unreachable();
+  }
+
+  return true;
+}
+
+void OmptAsserter::setOperationMode(AssertMode Mode) { OperationMode = Mode; }
+
+void OmptSequencedAsserter::insert(OmptAssertEvent &&AE) {
+  std::lock_guard<std::mutex> Lock(AssertMutex);
+  Events.emplace_back(std::move(AE));
+}
+
+void OmptSequencedAsserter::notifyImpl(OmptAssertEvent &&AE) {
+  std::lock_guard<std::mutex> Lock(AssertMutex);
+  // Ignore notifications while inactive, or for suppressed events
+  if (Events.empty() || !isActive() || isSuppressedEventType(AE.getEventType()))
+    return;
+
+  ++NumNotifications;
+
+  // Note: Order of these checks has semantic meaning.
+  // (1) Synchronization points should fail if there are remaining events,
+  // otherwise pass. (2) Regular notification while no further events are
+  // expected: fail. (3) Assertion suspension relies on a next expected event
+  // being available. (4) All other cases are considered 'regular' and match the
+  // next expected against the observed event. (5+6) Depending on the state /
+  // mode we signal failure if no other check has done already, or signaled pass
+  // by early-exit.
+  if (consumeSyncPoint(AE) ||               // Handle observed SyncPoint event
+      checkExcessNotify(AE) ||              // Check for remaining expected
+      consumeSuspend() ||                   // Handle requested suspend
+      consumeRegularEvent(AE) ||            // Handle regular event
+      AssertionSuspended ||                 // Ignore fail, if suspended
+      OperationMode == AssertMode::Relaxed) // Ignore fail, if Relaxed op-mode
+    return;
+
+  Log->logEventMismatch("[OmptSequencedAsserter] The events are not equal",
+                        Events[NextEvent], AE);
+  State = AssertState::Fail;
+}
+
+bool OmptSequencedAsserter::consumeSyncPoint(
+    const omptest::OmptAssertEvent &AE) {
+  if (AE.getEventType() == EventTy::AssertionSyncPoint) {
+    auto NumRemainingEvents = getRemainingEventCount();
+    // Upon encountering a SyncPoint, all events should have been processed
+    if (NumRemainingEvents == 0)
+      return true;
+
+    Log->logEventMismatch(
+        "[OmptSequencedAsserter] Encountered SyncPoint while still awaiting " +
+            std::to_string(NumRemainingEvents) + " events. Asserted " +
+            std::to_string(NumSuccessfulAsserts) + "/" +
+            std::to_string(Events.size()) + " events successfully.",
+        AE);
+    State = AssertState::Fail;
+    return true;
+  }
+
+  // Nothing to process: continue.
+  return false;
+}
+
+bool OmptSequencedAsserter::checkExcessNotify(
+    const omptest::OmptAssertEvent &AE) {
+  if (NextEvent >= Events.size()) {
+    // If we are not expecting any more events and passively asserting: return
+    if (AssertionSuspended)
+      return true;
+
+    Log->logEventMismatch(
+        "[OmptSequencedAsserter] Too many events to check (" +
+            std::to_string(NumNotifications) + "). Asserted " +
+            std::to_string(NumSuccessfulAsserts) + "/" +
+            std::to_string(Events.size()) + " events successfully.",
+        AE);
+    State = AssertState::Fail;
+    return true;
+  }
+
+  // Remaining expected events present: continue.
+  return false;
+}
+
+bool OmptSequencedAsserter::consumeSuspend() {
+  // On AssertionSuspend -- enter 'passive' assertion.
+  // Since we may encounter multiple, successive AssertionSuspend events, loop
+  // until we hit the next non-AssertionSuspend event.
+  while (Events[NextEvent].getEventType() == EventTy::AssertionSuspend) {
+    AssertionSuspended = true;
+    // We just hit the very last event: indicate early exit.
+    if (++NextEvent >= Events.size())
+      return true;
+  }
+
+  // Continue with remaining notification logic.
+  return false;
+}
+
+bool OmptSequencedAsserter::consumeRegularEvent(
+    const omptest::OmptAssertEvent &AE) {
+  // If we are actively asserting, increment the event counter.
+  // Otherwise: If passively asserting, we will keep waiting for a match.
+  auto &E = Events[NextEvent];
+  if (E == AE && verifyEventGroups(E, AE)) {
+    if (E.getEventExpectedState() == ObserveState::Always) {
+      ++NumSuccessfulAsserts;
+    } else if (E.getEventExpectedState() == ObserveState::Never) {
+      Log->logEventMismatch(
+          "[OmptSequencedAsserter] Encountered forbidden event", E, AE);
+      State = AssertState::Fail;
+    }
+
+    // Return to active assertion
+    if (AssertionSuspended)
+      AssertionSuspended = false;
+
+    // Match found, increment index and indicate early exit (success).
+    ++NextEvent;
+    return true;
+  }
+
+  // Continue with remaining notification logic.
+  return false;
+}
+
+size_t OmptSequencedAsserter::getRemainingEventCount() {
+  return std::count_if(Events.begin(), Events.end(),
+                       [](const omptest::OmptAssertEvent &E) {
+                         return E.getEventExpectedState() ==
+                                ObserveState::Always;
+                       }) -
+         NumSuccessfulAsserts;
+}
+
+AssertState OmptSequencedAsserter::checkState() {
+  // This is called after the testcase executed.
+  // Once reached the number of successful notifications should be equal to the
+  // number of expected events. However, there may still be excluded as well as
+  // special asserter events remaining in the sequence.
+  for (size_t i = NextEvent; i < Events.size(); ++i) {
+    auto &E = Events[i];
+    if (E.getEventExpectedState() == ObserveState::Always) {
+      State = AssertState::Fail;
+      Log->logEventMismatch("[OmptSequencedAsserter] Expected event was not "
+                            "encountered (Remaining events: " +
+                                std::to_string(getRemainingEventCount()) + ")",
+                            E);
+      break;
+    }
+  }
+
+  return State;
+}
+
+void OmptEventAsserter::insert(OmptAssertEvent &&AE) {
+  std::lock_guard<std::mutex> Lock(AssertMutex);
+  Events.emplace_back(std::move(AE));
+}
+
+void OmptEventAsserter::notifyImpl(OmptAssertEvent &&AE) {
+  std::lock_guard<std::mutex> Lock(AssertMutex);
+  if (Events.empty() || !isActive() || isSuppressedEventType(AE.getEventType()))
+    return;
+
+  if (NumEvents == 0)
+    NumEvents = Events.size();
+
+  ++NumNotifications;
+
+  if (AE.getEventType() == EventTy::AssertionSyncPoint) {
+    auto NumRemainingEvents = getRemainingEventCount();
+    // Upon encountering a SyncPoint, all events should have been processed
+    if (NumRemainingEvents == 0)
+      return;
+
+    Log->logEventMismatch(
+        "[OmptEventAsserter] Encountered SyncPoint while still awaiting " +
+            std::to_string(NumRemainingEvents) + " events. Asserted " +
+            std::to_string(NumSuccessfulAsserts) + " events successfully.",
+        AE);
+    State = AssertState::Fail;
+    return;
+  }
+
+  for (size_t i = 0; i < Events.size(); ++i) {
+    auto &E = Events[i];
+    if (E == AE && verifyEventGroups(E, AE)) {
+      if (E.getEventExpectedState() == ObserveState::Always) {
+        Events.erase(Events.begin() + i);
+        ++NumSuccessfulAsserts;
+      } else if (E.getEventExpectedState() == ObserveState::Never) {
+        Log->logEventMismatch("[OmptEventAsserter] Encountered forbidden event",
+                              E, AE);
+        State = AssertState::Fail;
+      }
+      return;
+    }
+  }
+
+  if (OperationMode == AssertMode::Strict) {
+    Log->logEventMismatch("[OmptEventAsserter] Too many events to check (" +
+                              std::to_string(NumNotifications) +
+                              "). Asserted " +
+                              std::to_string(NumSuccessfulAsserts) +
+                              " events successfully. (Remaining events: " +
+                              std::to_string(getRemainingEventCount()) + ")",
+                          AE);
+    State = AssertState::Fail;
+    return;
+  }
+}
+
+size_t OmptEventAsserter::getRemainingEventCount() {
+  return std::count_if(
+      Events.begin(), Events.end(), [](const omptest::OmptAssertEvent &E) {
+        return E.getEventExpectedState() == ObserveState::Always;
+      });
+}
+
+AssertState OmptEventAsserter::checkState() {
+  // This is called after the testcase executed.
+  // Once reached no more expected events should be in the queue
+  for (const auto &E : Events) {
+    // Check if any of the remaining events were expected to be observed
+    if (E.getEventExpectedState() == ObserveState::Always) {
+      State = AssertState::Fail;
+      Log->logEventMismatch("[OmptEventAsserter] Expected event was not "
+                            "encountered (Remaining events: " +
+                                std::to_string(getRemainingEventCount()) + ")",
+                            E);
+      break;
+    }
+  }
+
+  return State;
+}
+
+void OmptEventReporter::notify(OmptAssertEvent &&AE) {
+  if (!isActive() || isSuppressedEventType(AE.getEventType()))
+    return;
+
+  // Prepare notification, containing the newline to avoid stream interleaving.
+  auto Notification{AE.toString()};
+  Notification.push_back('\n');
+  OutStream << Notification;
+}
+
+bool OmptEventGroupInterface::addActiveEventGroup(
+    const std::string &GroupName, omptest::AssertEventGroup Group) {
+  std::lock_guard<std::mutex> Lock(GroupMutex);
+  auto EventGroup = ActiveEventGroups.find(GroupName);
+  if (EventGroup != ActiveEventGroups.end() &&
+      EventGroup->second.TargetRegion == Group.TargetRegion)
+    return false;
+  ActiveEventGroups.emplace(GroupName, Group);
+  return true;
+}
+
+bool OmptEventGroupInterface::deprecateActiveEventGroup(
+    const std::string &GroupName) {
+  std::lock_guard<std::mutex> Lock(GroupMutex);
+  auto EventGroup = ActiveEventGroups.find(GroupName);
+  auto DeprecatedEventGroup = DeprecatedEventGroups.find(GroupName);
+  if (EventGroup == ActiveEventGroups.end() &&
+      DeprecatedEventGroup != DeprecatedEventGroups.end())
+    return false;
+  DeprecatedEventGroups.emplace(GroupName, EventGroup->second);
+  ActiveEventGroups.erase(GroupName);
+  return true;
+}
+
+bool OmptEventGroupInterface::checkActiveEventGroups(
+    const std::string &GroupName, omptest::AssertEventGroup Group) {
+  std::lock_guard<std::mutex> Lock(GroupMutex);
+  auto EventGroup = ActiveEventGroups.find(GroupName);
+  return (EventGroup != ActiveEventGroups.end() &&
+          EventGroup->second.TargetRegion == Group.TargetRegion);
+}
+
+bool OmptEventGroupInterface::checkDeprecatedEventGroups(
+    const std::string &GroupName, omptest::AssertEventGroup Group) {
+  std::lock_guard<std::mutex> Lock(GroupMutex);
+  auto EventGroup = DeprecatedEventGroups.find(GroupName);
+  return (EventGroup != DeprecatedEventGroups.end() &&
+          EventGroup->second.TargetRegion == Group.TargetRegion);
+}
diff --git a/openmp/tools/omptest/src/OmptCallbackHandler.cpp b/openmp/tools/omptest/src/OmptCallbackHandler.cpp
new file mode 100644
index 0000000000000..3330adf09cc79
--- /dev/null
+++ b/openmp/tools/omptest/src/OmptCallbackHandler.cpp
@@ -0,0 +1,445 @@
+//===- OmptCallbackHandler.cpp - OMPT Callback handling impl. ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the OMPT callback handling implementations.
+///
+//===----------------------------------------------------------------------===//
+
+#include "OmptCallbackHandler.h"
+
+using namespace omptest;
+
+OmptCallbackHandler *Handler = nullptr;
+
+OmptCallbackHandler &OmptCallbackHandler::get() {
+  if (Handler == nullptr)
+    Handler = new OmptCallbackHandler();
+
+  return *Handler;
+}
+
+void OmptCallbackHandler::subscribe(OmptListener *Listener) {
+  Subscribers.push_back(Listener);
+}
+
+void OmptCallbackHandler::clearSubscribers() {
+  replay();
+
+  Subscribers.clear();
+}
+
+void OmptCallbackHandler::replay() {
+  if (!RecordAndReplay)
+    return;
+
+  for (auto &E : RecordedEvents)
+    for (const auto &S : Subscribers)
+      S->notify(std::move(E));
+}
+
+void OmptCallbackHandler::handleThreadBegin(ompt_thread_t ThreadType,
+                                            ompt_data_t *ThreadData) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::ThreadBegin(
+        "Thread Begin", "", ObserveState::Generated, ThreadType));
+    return;
+  }
+
+  // Initial thread event likely to preceed assertion registration, so skip
+  if (ThreadType == ompt_thread_initial)
+    return;
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::ThreadBegin(
+        "Thread Begin", "", ObserveState::Generated, ThreadType));
+}
+
+void OmptCallbackHandler::handleThreadEnd(ompt_data_t *ThreadData) {
+  if (RecordAndReplay) {
+    recordEvent(
+        OmptAssertEvent::ThreadEnd("Thread End", "", ObserveState::Generated));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(
+        OmptAssertEvent::ThreadEnd("Thread End", "", ObserveState::Generated));
+}
+
+void OmptCallbackHandler::handleTaskCreate(
+    ompt_data_t *EncounteringTaskData,
+    const ompt_frame_t *EncounteringTaskFrame, ompt_data_t *NewTaskData,
+    int Flags, int HasDependences, const void *CodeptrRA) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::TaskCreate(
+        "Task Create", "", ObserveState::Generated, EncounteringTaskData,
+        EncounteringTaskFrame, NewTaskData, Flags, HasDependences, CodeptrRA));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::TaskCreate(
+        "Task Create", "", ObserveState::Generated, EncounteringTaskData,
+        EncounteringTaskFrame, NewTaskData, Flags, HasDependences, CodeptrRA));
+}
+
+void OmptCallbackHandler::handleTaskSchedule(ompt_data_t *PriorTaskData,
+                                             ompt_task_status_t PriorTaskStatus,
+                                             ompt_data_t *NextTaskData) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::TaskSchedule("Task Schedule", "",
+                                              ObserveState::Generated));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::TaskSchedule("Task Schedule", "",
+                                            ObserveState::Generated));
+}
+
+void OmptCallbackHandler::handleImplicitTask(ompt_scope_endpoint_t Endpoint,
+                                             ompt_data_t *ParallelData,
+                                             ompt_data_t *TaskData,
+                                             unsigned int ActualParallelism,
+                                             unsigned int Index, int Flags) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::ImplicitTask(
+        "Implicit Task", "", ObserveState::Generated, Endpoint, ParallelData,
+        TaskData, ActualParallelism, Index, Flags));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::ImplicitTask(
+        "Implicit Task", "", ObserveState::Generated, Endpoint, ParallelData,
+        TaskData, ActualParallelism, Index, Flags));
+}
+
+void OmptCallbackHandler::handleParallelBegin(
+    ompt_data_t *EncounteringTaskData,
+    const ompt_frame_t *EncounteringTaskFrame, ompt_data_t *ParallelData,
+    unsigned int RequestedParallelism, int Flags, const void *CodeptrRA) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::ParallelBegin(
+        "Parallel Begin", "", ObserveState::Generated, RequestedParallelism));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::ParallelBegin(
+        "Parallel Begin", "", ObserveState::Generated, RequestedParallelism));
+}
+
+void OmptCallbackHandler::handleParallelEnd(ompt_data_t *ParallelData,
+                                            ompt_data_t *EncounteringTaskData,
+                                            int Flags, const void *CodeptrRA) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::ParallelEnd("Parallel End", "",
+                                             ObserveState::Generated));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::ParallelEnd("Parallel End", "",
+                                           ObserveState::Generated));
+}
+
+void OmptCallbackHandler::handleDeviceInitialize(
+    int DeviceNum, const char *Type, ompt_device_t *Device,
+    ompt_function_lookup_t LookupFn, const char *DocumentationStr) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::DeviceInitialize(
+        "Device Inititalize", "", ObserveState::Generated, DeviceNum, Type,
+        Device, LookupFn, DocumentationStr));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::DeviceInitialize(
+        "Device Inititalize", "", ObserveState::Generated, DeviceNum, Type,
+        Device, LookupFn, DocumentationStr));
+}
+
+void OmptCallbackHandler::handleDeviceFinalize(int DeviceNum) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::DeviceFinalize(
+        "Device Finalize", "", ObserveState::Generated, DeviceNum));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::DeviceFinalize(
+        "Device Finalize", "", ObserveState::Generated, DeviceNum));
+}
+
+void OmptCallbackHandler::handleTarget(ompt_target_t Kind,
+                                       ompt_scope_endpoint_t Endpoint,
+                                       int DeviceNum, ompt_data_t *TaskData,
+                                       ompt_id_t TargetId,
+                                       const void *CodeptrRA) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::Target("Target", "", ObserveState::Generated,
+                                        Kind, Endpoint, DeviceNum, TaskData,
+                                        TargetId, CodeptrRA));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::Target("Target", "", ObserveState::Generated,
+                                      Kind, Endpoint, DeviceNum, TaskData,
+                                      TargetId, CodeptrRA));
+}
+
+void OmptCallbackHandler::handleTargetEmi(ompt_target_t Kind,
+                                          ompt_scope_endpoint_t Endpoint,
+                                          int DeviceNum, ompt_data_t *TaskData,
+                                          ompt_data_t *TargetTaskData,
+                                          ompt_data_t *TargetData,
+                                          const void *CodeptrRA) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::TargetEmi(
+        "Target EMI", "", ObserveState::Generated, Kind, Endpoint, DeviceNum,
+        TaskData, TargetTaskData, TargetData, CodeptrRA));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::TargetEmi(
+        "Target EMI", "", ObserveState::Generated, Kind, Endpoint, DeviceNum,
+        TaskData, TargetTaskData, TargetData, CodeptrRA));
+}
+
+void OmptCallbackHandler::handleTargetSubmit(ompt_id_t TargetId,
+                                             ompt_id_t HostOpId,
+                                             unsigned int RequestedNumTeams) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::TargetSubmit("Target Submit", "",
+                                              ObserveState::Generated, TargetId,
+                                              HostOpId, RequestedNumTeams));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::TargetSubmit("Target Submit", "",
+                                            ObserveState::Generated, TargetId,
+                                            HostOpId, RequestedNumTeams));
+}
+
+void OmptCallbackHandler::handleTargetSubmitEmi(
+    ompt_scope_endpoint_t Endpoint, ompt_data_t *TargetData,
+    ompt_id_t *HostOpId, unsigned int RequestedNumTeams) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::TargetSubmitEmi(
+        "Target Submit EMI", "", ObserveState::Generated, Endpoint, TargetData,
+        HostOpId, RequestedNumTeams));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::TargetSubmitEmi(
+        "Target Submit EMI", "", ObserveState::Generated, Endpoint, TargetData,
+        HostOpId, RequestedNumTeams));
+}
+
+void OmptCallbackHandler::handleTargetDataOp(
+    ompt_id_t TargetId, ompt_id_t HostOpId, ompt_target_data_op_t OpType,
+    void *SrcAddr, int SrcDeviceNum, void *DstAddr, int DstDeviceNum,
+    size_t Bytes, const void *CodeptrRA) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::TargetDataOp(
+        "Target Data Op", "", ObserveState::Generated, TargetId, HostOpId,
+        OpType, SrcAddr, SrcDeviceNum, DstAddr, DstDeviceNum, Bytes,
+        CodeptrRA));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::TargetDataOp(
+        "Target Data Op", "", ObserveState::Generated, TargetId, HostOpId,
+        OpType, SrcAddr, SrcDeviceNum, DstAddr, DstDeviceNum, Bytes,
+        CodeptrRA));
+}
+
+void OmptCallbackHandler::handleTargetDataOpEmi(
+    ompt_scope_endpoint_t Endpoint, ompt_data_t *TargetTaskData,
+    ompt_data_t *TargetData, ompt_id_t *HostOpId, ompt_target_data_op_t OpType,
+    void *SrcAddr, int SrcDeviceNum, void *DstAddr, int DstDeviceNum,
+    size_t Bytes, const void *CodeptrRA) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::TargetDataOpEmi(
+        "Target Data Op EMI", "", ObserveState::Generated, Endpoint,
+        TargetTaskData, TargetData, HostOpId, OpType, SrcAddr, SrcDeviceNum,
+        DstAddr, DstDeviceNum, Bytes, CodeptrRA));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::TargetDataOpEmi(
+        "Target Data Op EMI", "", ObserveState::Generated, Endpoint,
+        TargetTaskData, TargetData, HostOpId, OpType, SrcAddr, SrcDeviceNum,
+        DstAddr, DstDeviceNum, Bytes, CodeptrRA));
+}
+
+void OmptCallbackHandler::handleDeviceLoad(int DeviceNum, const char *Filename,
+                                           int64_t OffsetInFile,
+                                           void *VmaInFile, size_t Bytes,
+                                           void *HostAddr, void *DeviceAddr,
+                                           uint64_t ModuleId) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::DeviceLoad(
+        "Device Load", "", ObserveState::Generated, DeviceNum, Filename,
+        OffsetInFile, VmaInFile, Bytes, HostAddr, DeviceAddr, ModuleId));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::DeviceLoad(
+        "Device Load", "", ObserveState::Generated, DeviceNum, Filename,
+        OffsetInFile, VmaInFile, Bytes, HostAddr, DeviceAddr, ModuleId));
+}
+
+void OmptCallbackHandler::handleDeviceUnload(int DeviceNum, uint64_t ModuleId) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::DeviceUnload("Device Unload", "",
+                                              ObserveState::Generated));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::DeviceUnload("Device Unload", "",
+                                            ObserveState::Generated));
+}
+
+void OmptCallbackHandler::handleBufferRequest(int DeviceNum,
+                                              ompt_buffer_t **Buffer,
+                                              size_t *Bytes) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::BufferRequest("Buffer Request", "",
+                                               ObserveState::Generated,
+                                               DeviceNum, Buffer, Bytes));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::BufferRequest("Buffer Request", "",
+                                             ObserveState::Generated, DeviceNum,
+                                             Buffer, Bytes));
+}
+
+void OmptCallbackHandler::handleBufferComplete(int DeviceNum,
+                                               ompt_buffer_t *Buffer,
+                                               size_t Bytes,
+                                               ompt_buffer_cursor_t Begin,
+                                               int BufferOwned) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::BufferComplete(
+        "Buffer Complete", "", ObserveState::Generated, DeviceNum, Buffer,
+        Bytes, Begin, BufferOwned));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::BufferComplete(
+        "Buffer Complete", "", ObserveState::Generated, DeviceNum, Buffer,
+        Bytes, Begin, BufferOwned));
+}
+
+void OmptCallbackHandler::handleBufferRecord(ompt_record_ompt_t *Record) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::BufferRecord("Buffer Record", "",
+                                              ObserveState::Generated, Record));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::BufferRecord("Buffer Record", "",
+                                            ObserveState::Generated, Record));
+}
+
+void OmptCallbackHandler::handleBufferRecordDeallocation(
+    ompt_buffer_t *Buffer) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::BufferRecordDeallocation(
+        "Buffer Deallocation", "", ObserveState::Generated, Buffer));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::BufferRecordDeallocation(
+        "Buffer Deallocation", "", ObserveState::Generated, Buffer));
+}
+
+void OmptCallbackHandler::handleWork(ompt_work_t WorkType,
+                                     ompt_scope_endpoint_t Endpoint,
+                                     ompt_data_t *ParallelData,
+                                     ompt_data_t *TaskData, uint64_t Count,
+                                     const void *CodeptrRA) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::Work("Work", "", ObserveState::Generated,
+                                      WorkType, Endpoint, ParallelData,
+                                      TaskData, Count, CodeptrRA));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::Work("Work", "", ObserveState::Generated,
+                                    WorkType, Endpoint, ParallelData, TaskData,
+                                    Count, CodeptrRA));
+}
+
+void OmptCallbackHandler::handleSyncRegion(ompt_sync_region_t Kind,
+                                           ompt_scope_endpoint_t Endpoint,
+                                           ompt_data_t *ParallelData,
+                                           ompt_data_t *TaskData,
+                                           const void *CodeptrRA) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::SyncRegion(
+        "SyncRegion", "", ObserveState::Generated, Kind, Endpoint, ParallelData,
+        TaskData, CodeptrRA));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::SyncRegion(
+        "SyncRegion", "", ObserveState::Generated, Kind, Endpoint, ParallelData,
+        TaskData, CodeptrRA));
+}
+
+void OmptCallbackHandler::handleDispatch(ompt_data_t *ParallelData,
+                                         ompt_data_t *TaskData,
+                                         ompt_dispatch_t Kind,
+                                         ompt_data_t Instance) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::Dispatch("Dispatch", "",
+                                          ObserveState::Generated, ParallelData,
+                                          TaskData, Kind, Instance));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::Dispatch("Dispatch", "", ObserveState::Generated,
+                                        ParallelData, TaskData, Kind,
+                                        Instance));
+}
+
+void OmptCallbackHandler::handleAssertionSyncPoint(
+    const std::string &SyncPointName) {
+  if (RecordAndReplay) {
+    recordEvent(OmptAssertEvent::AssertionSyncPoint(
+        "Assertion SyncPoint", "", ObserveState::Generated, SyncPointName));
+    return;
+  }
+
+  for (const auto &S : Subscribers)
+    S->notify(OmptAssertEvent::AssertionSyncPoint(
+        "Assertion SyncPoint", "", ObserveState::Generated, SyncPointName));
+}
+
+void OmptCallbackHandler::recordEvent(OmptAssertEvent &&Event) {
+  RecordedEvents.emplace_back(std::forward<OmptAssertEvent>(Event));
+}
diff --git a/openmp/tools/omptest/src/OmptTester.cpp b/openmp/tools/omptest/src/OmptTester.cpp
new file mode 100644
index 0000000000000..afa96ac1d4b23
--- /dev/null
+++ b/openmp/tools/omptest/src/OmptTester.cpp
@@ -0,0 +1,504 @@
+//===- OmptTester.cpp - ompTest OMPT tool implementation --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file represents the core implementation file for the ompTest library.
+/// It provides the actual OMPT tool implementation: registers callbacks, etc.
+/// OMPT callbacks are passed to their corresponding handler, which in turn
+/// notifies all registered asserters.
+///
+//===----------------------------------------------------------------------===//
+
+#include "OmptTester.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+
+using namespace omptest;
+
+// Callback handler, which receives and relays OMPT callbacks
+extern OmptCallbackHandler *Handler;
+
+// EventListener, which actually prints the OMPT events
+static OmptEventReporter *EventReporter;
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+#ifdef OPENMP_LIBOMPTEST_BUILD_STANDALONE
+std::vector<std::pair<std::string, TestSuite>> TestRegistrar::Tests;
+#endif
+
+static std::atomic<ompt_id_t> NextOpId{0x8000000000000001};
+static bool UseEMICallbacks = false;
+static bool UseTracing = false;
+static bool RunAsTestSuite = false;
+static bool ColoredLog = false;
+
+// OMPT entry point handles
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+
+// OMPT device side tracing: Currently traced devices
+typedef std::unordered_set<ompt_device_t *> OmptDeviceSetTy;
+typedef std::unique_ptr<OmptDeviceSetTy> OmptDeviceSetPtrTy;
+static OmptDeviceSetPtrTy TracedDevices;
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  OmptCallbackHandler::get().handleBufferRequest(device_num, buffer, bytes);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  OmptCallbackHandler::get().handleBufferComplete(device_num, buffer, bytes,
+                                                  begin, buffer_owned);
+
+  int Status = 1;
+  ompt_buffer_cursor_t CurrentPos = begin;
+  while (Status) {
+    ompt_record_ompt_t *Record = ompt_get_record_ompt(buffer, CurrentPos);
+    if (ompt_get_record_type_fn(buffer, CurrentPos) != ompt_record_ompt) {
+      printf("Warning: received non-ompt type buffer object\n");
+    }
+    // TODO: Sometimes it may happen that the retrieved record may be null?!
+    // Only handle non-null records
+    if (Record != nullptr)
+      OmptCallbackHandler::get().handleBufferRecord(Record);
+    Status = ompt_advance_buffer_cursor(/*device=*/NULL, buffer, bytes,
+                                        CurrentPos, &CurrentPos);
+  }
+  if (buffer_owned) {
+    OmptCallbackHandler::get().handleBufferRecordDeallocation(buffer);
+    free(buffer);
+  }
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  if (UseEMICallbacks) {
+    ompt_set_trace_ompt(Device, /*enable=*/1,
+                        /*etype=*/ompt_callback_target_emi);
+    ompt_set_trace_ompt(Device, /*enable=*/1,
+                        /*etype=*/ompt_callback_target_data_op_emi);
+    ompt_set_trace_ompt(Device, /*enable=*/1,
+                        /*etype=*/ompt_callback_target_submit_emi);
+  } else {
+    ompt_set_trace_ompt(Device, /*enable=*/1, /*etype=*/ompt_callback_target);
+    ompt_set_trace_ompt(Device, /*enable=*/1,
+                        /*etype=*/ompt_callback_target_data_op);
+    ompt_set_trace_ompt(Device, /*enable=*/1,
+                        /*etype=*/ompt_callback_target_submit);
+  }
+
+  return ompt_set_always;
+}
+
+/////// HOST-RELATED //////
+
+static void on_ompt_callback_thread_begin(ompt_thread_t thread_type,
+                                          ompt_data_t *thread_data) {
+  OmptCallbackHandler::get().handleThreadBegin(thread_type, thread_data);
+}
+
+static void on_ompt_callback_thread_end(ompt_data_t *thread_data) {
+  OmptCallbackHandler::get().handleThreadEnd(thread_data);
+}
+
+static void on_ompt_callback_parallel_begin(
+    ompt_data_t *encountering_task_data,
+    const ompt_frame_t *encountering_task_frame, ompt_data_t *parallel_data,
+    unsigned int requested_parallelism, int flags, const void *codeptr_ra) {
+  OmptCallbackHandler::get().handleParallelBegin(
+      encountering_task_data, encountering_task_frame, parallel_data,
+      requested_parallelism, flags, codeptr_ra);
+}
+
+static void on_ompt_callback_parallel_end(ompt_data_t *parallel_data,
+                                          ompt_data_t *encountering_task_data,
+                                          int flags, const void *codeptr_ra) {
+  OmptCallbackHandler::get().handleParallelEnd(
+      parallel_data, encountering_task_data, flags, codeptr_ra);
+}
+
+static void
+on_ompt_callback_task_create(ompt_data_t *encountering_task_data,
+                             const ompt_frame_t *encountering_task_frame,
+                             ompt_data_t *new_task_data, int flags,
+                             int has_dependences, const void *codeptr_ra) {
+  OmptCallbackHandler::get().handleTaskCreate(
+      encountering_task_data, encountering_task_frame, new_task_data, flags,
+      has_dependences, codeptr_ra);
+}
+
+static void on_ompt_callback_task_schedule(ompt_data_t *prior_task_data,
+                                           ompt_task_status_t prior_task_status,
+                                           ompt_data_t *next_task_data) {
+  OmptCallbackHandler::get().handleTaskSchedule(
+      prior_task_data, prior_task_status, next_task_data);
+}
+
+static void on_ompt_callback_implicit_task(ompt_scope_endpoint_t endpoint,
+                                           ompt_data_t *parallel_data,
+                                           ompt_data_t *task_data,
+                                           unsigned int actual_parallelism,
+                                           unsigned int index, int flags) {
+  OmptCallbackHandler::get().handleImplicitTask(
+      endpoint, parallel_data, task_data, actual_parallelism, index, flags);
+}
+
+// Callbacks as of Table 19.4, which are not considered required for a minimal
+// conforming OMPT implementation.
+static void on_ompt_callback_work(ompt_work_t work_type,
+                                  ompt_scope_endpoint_t endpoint,
+                                  ompt_data_t *parallel_data,
+                                  ompt_data_t *task_data, uint64_t count,
+                                  const void *codeptr_ra) {
+  OmptCallbackHandler::get().handleWork(work_type, endpoint, parallel_data,
+                                        task_data, count, codeptr_ra);
+}
+
+static void on_ompt_callback_dispatch(ompt_data_t *parallel_data,
+                                      ompt_data_t *task_data,
+                                      ompt_dispatch_t kind,
+                                      ompt_data_t instance) {
+  OmptCallbackHandler::get().handleDispatch(parallel_data, task_data, kind,
+                                            instance);
+}
+
+static void on_ompt_callback_sync_region(ompt_sync_region_t kind,
+                                         ompt_scope_endpoint_t endpoint,
+                                         ompt_data_t *parallel_data,
+                                         ompt_data_t *task_data,
+                                         const void *codeptr_ra) {
+  OmptCallbackHandler::get().handleSyncRegion(kind, endpoint, parallel_data,
+                                              task_data, codeptr_ra);
+}
+
+/////// DEVICE-RELATED //////
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  OmptCallbackHandler::get().handleDeviceInitialize(device_num, type, device,
+                                                    lookup, documentation);
+  if (!UseTracing)
+    return;
+
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("Warning: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    TracedDevices = std::make_unique<OmptDeviceSetTy>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this is a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  OmptCallbackHandler::get().handleDeviceFinalize(device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  OmptCallbackHandler::get().handleDeviceLoad(
+      device_num, filename, offset_in_file, vma_in_file, bytes, host_addr,
+      device_addr, module_id);
+}
+
+static void on_ompt_callback_device_unload(int device_num, uint64_t module_id) {
+  OmptCallbackHandler::get().handleDeviceUnload(device_num, module_id);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  OmptCallbackHandler::get().handleTargetDataOp(
+      target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+      dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  OmptCallbackHandler::get().handleTarget(kind, endpoint, device_num, task_data,
+                                          target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  OmptCallbackHandler::get().handleTargetSubmit(target_id, host_op_id,
+                                                requested_num_teams);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  assert(codeptr_ra != 0 && "Unexpected null codeptr");
+  // Both src and dest must not be null
+  // However, for omp_target_alloc only the END call holds a value for one of
+  // the two entries
+  if (optype != ompt_target_data_alloc)
+    assert((src_addr != 0 || dest_addr != 0) && "Both src and dest addr null");
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = NextOpId.fetch_add(1, std::memory_order_relaxed);
+  OmptCallbackHandler::get().handleTargetDataOpEmi(
+      endpoint, target_task_data, target_data, host_op_id, optype, src_addr,
+      src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  assert(codeptr_ra != 0 && "Unexpected null codeptr");
+  if (endpoint == ompt_scope_begin)
+    target_data->value = NextOpId.fetch_add(1, std::memory_order_relaxed);
+  OmptCallbackHandler::get().handleTargetEmi(kind, endpoint, device_num,
+                                             task_data, target_task_data,
+                                             target_data, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  OmptCallbackHandler::get().handleTargetSubmitEmi(
+      endpoint, target_data, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_map(ompt_id_t target_id,
+                                        unsigned int nitems, void **host_addr,
+                                        void **device_addr, size_t *bytes,
+                                        unsigned int *mapping_flags,
+                                        const void *codeptr_ra) {
+  assert(0 && "Target map callback is unimplemented");
+}
+
+static void on_ompt_callback_target_map_emi(ompt_data_t *target_data,
+                                            unsigned int nitems,
+                                            void **host_addr,
+                                            void **device_addr, size_t *bytes,
+                                            unsigned int *mapping_flags,
+                                            const void *codeptr_ra) {
+  assert(0 && "Target map emi callback is unimplemented");
+}
+
+/// Load the value of a given boolean environmental variable.
+bool getBoolEnvironmentVariable(const char *VariableName) {
+  if (VariableName == nullptr)
+    return false;
+  if (const char *EnvValue = std::getenv(VariableName)) {
+    std::string S{EnvValue};
+    for (auto &C : S)
+      C = (char)std::tolower(C);
+    if (S == "1" || S == "on" || S == "true" || S == "yes")
+      return true;
+  }
+  return false;
+}
+
+/// Called by the OMP runtime to initialize the OMPT
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback_t ompt_set_callback = nullptr;
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+  if (!ompt_set_callback)
+    return 0; // failure
+
+  UseEMICallbacks = getBoolEnvironmentVariable("OMPTEST_USE_OMPT_EMI");
+  UseTracing = getBoolEnvironmentVariable("OMPTEST_USE_OMPT_TRACING");
+  RunAsTestSuite = getBoolEnvironmentVariable("OMPTEST_RUN_AS_TESTSUITE");
+  ColoredLog = getBoolEnvironmentVariable("OMPTEST_LOG_COLORED");
+
+  register_ompt_callback(ompt_callback_thread_begin);
+  register_ompt_callback(ompt_callback_thread_end);
+  register_ompt_callback(ompt_callback_parallel_begin);
+  register_ompt_callback(ompt_callback_parallel_end);
+  register_ompt_callback(ompt_callback_work);
+  // register_ompt_callback(ompt_callback_dispatch);
+  register_ompt_callback(ompt_callback_task_create);
+  // register_ompt_callback(ompt_callback_dependences);
+  // register_ompt_callback(ompt_callback_task_dependence);
+  register_ompt_callback(ompt_callback_task_schedule);
+  register_ompt_callback(ompt_callback_implicit_task);
+  // register_ompt_callback(ompt_callback_masked);
+  register_ompt_callback(ompt_callback_sync_region);
+  // register_ompt_callback(ompt_callback_mutex_acquire);
+  // register_ompt_callback(ompt_callback_mutex);
+  // register_ompt_callback(ompt_callback_nestLock);
+  // register_ompt_callback(ompt_callback_flush);
+  // register_ompt_callback(ompt_callback_cancel);
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_device_unload);
+
+  if (UseEMICallbacks) {
+    register_ompt_callback(ompt_callback_target_emi);
+    register_ompt_callback(ompt_callback_target_submit_emi);
+    register_ompt_callback(ompt_callback_target_data_op_emi);
+    register_ompt_callback(ompt_callback_target_map_emi);
+  } else {
+    register_ompt_callback(ompt_callback_target);
+    register_ompt_callback(ompt_callback_target_submit);
+    register_ompt_callback(ompt_callback_target_data_op);
+    register_ompt_callback(ompt_callback_target_map);
+  }
+
+  // Construct & subscribe the reporter, so it gets notified of events
+  EventReporter = new OmptEventReporter();
+  OmptCallbackHandler::get().subscribe(EventReporter);
+
+  if (RunAsTestSuite)
+    EventReporter->setActive(false);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {
+  assert(Handler && "Callback handler should be present at this point");
+  assert(EventReporter && "EventReporter should be present at this point");
+  delete Handler;
+  delete EventReporter;
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/// Called from the OMP Runtime to start / initialize the tool
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {
+      &ompt_initialize, &ompt_finalize, {0}};
+  return &ompt_start_tool_result;
+}
+
+int start_trace(ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // Start tracing this device (add to set)
+  assert(TracedDevices->find(Device) == TracedDevices->end() &&
+         "Device already present in the map");
+  TracedDevices->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+int flush_traced_devices() {
+  if (!ompt_flush_trace || TracedDevices == nullptr)
+    return 0;
+
+  size_t NumFlushedDevices = 0;
+  for (auto Device : *TracedDevices)
+    if (ompt_flush_trace(Device) == 1)
+      ++NumFlushedDevices;
+
+  // Provide time to process triggered assert events
+  std::this_thread::sleep_for(std::chrono::milliseconds(1));
+
+  return (NumFlushedDevices == TracedDevices->size());
+}
+
+int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+
+  // Stop tracing this device (erase from set)
+  assert(TracedDevices->find(Device) != TracedDevices->end() &&
+         "Device not present in the map");
+  TracedDevices->erase(Device);
+
+  return ompt_stop_trace(Device);
+}
+
+// This is primarily used to stop unwanted prints from happening.
+void libomptest_global_eventreporter_set_active(bool State) {
+  assert(EventReporter && "EventReporter should be present at this point");
+  EventReporter->setActive(State);
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/openmp/tools/omptest/src/OmptTesterStandalone.cpp b/openmp/tools/omptest/src/OmptTesterStandalone.cpp
new file mode 100644
index 0000000000000..6f13939c92941
--- /dev/null
+++ b/openmp/tools/omptest/src/OmptTesterStandalone.cpp
@@ -0,0 +1,158 @@
+//===- OmptTesterStandalone.cpp - Standalone unit testing impl. -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file represents the 'standalone' ompTest unit testing core
+/// implementation, defining the general test suite and test case execution.
+///
+//===----------------------------------------------------------------------===//
+
+#include "OmptTesterStandalone.h"
+#include "OmptCallbackHandler.h"
+
+#include <algorithm>
+#include <cassert>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+using namespace omptest;
+
+Error TestCase::exec() {
+  Error E;
+  E.Fail = false;
+
+  if (IsDisabled)
+    return E;
+
+  OmptCallbackHandler::get().subscribe(SequenceAsserter.get());
+  OmptCallbackHandler::get().subscribe(SetAsserter.get());
+  OmptCallbackHandler::get().subscribe(EventReporter.get());
+
+  execImpl();
+
+  // Actively flush potential in-flight trace records
+  flush_traced_devices();
+
+  // We remove subscribers to not be notified of events after our test case
+  // finished.
+  OmptCallbackHandler::get().clearSubscribers();
+  omptest::AssertState SequenceResultState = SequenceAsserter->checkState();
+  omptest::AssertState SetResultState = SetAsserter->checkState();
+  bool AnyFail = SequenceResultState == omptest::AssertState::Fail ||
+                 SetResultState == omptest::AssertState::Fail;
+  bool AllPass = SequenceResultState == omptest::AssertState::Pass &&
+                 SetResultState == omptest::AssertState::Pass;
+  if (ExpectedState == omptest::AssertState::Pass && AnyFail)
+    E.Fail = true;
+  else if (ExpectedState == omptest::AssertState::Fail && AllPass)
+    E.Fail = true;
+  if (AnyFail)
+    ResultState = omptest::AssertState::Fail;
+  return E;
+}
+
+TestSuite::TestSuite(TestSuite &&O) {
+  Name = O.Name;
+  TestCases.swap(O.TestCases);
+}
+
+void TestSuite::setup() {}
+
+void TestSuite::teardown() {}
+
+TestSuite::TestCaseVec::iterator TestSuite::begin() {
+  return TestCases.begin();
+}
+
+TestSuite::TestCaseVec::iterator TestSuite::end() { return TestCases.end(); }
+
+TestRegistrar &TestRegistrar::get() {
+  static TestRegistrar TR;
+  return TR;
+}
+
+std::vector<TestSuite> TestRegistrar::getTestSuites() {
+  std::vector<TestSuite> TSs;
+  for (auto &[k, v] : Tests)
+    TSs.emplace_back(std::move(v));
+  return TSs;
+}
+
+void TestRegistrar::addCaseToSuite(TestCase *TC, const std::string &TSName) {
+  // Search the test suites for a matching name
+  auto It = std::find_if(Tests.begin(), Tests.end(),
+                         [&](const auto &P) { return P.first == TSName; });
+
+  if (It != Tests.end()) {
+    // Test suite exists: add the test case
+    It->second.TestCases.emplace_back(TC);
+  } else {
+    // Test suite does not exist: construct it and add the test case
+    TestSuite TS(TSName);
+    TS.TestCases.emplace_back(TC);
+    // Move and emplace the suite
+    Tests.emplace_back(TSName, std::move(TS));
+  }
+}
+
+Registerer::Registerer(TestCase *TC, const std::string SuiteName) {
+  std::cout << "Adding " << TC->Name << " to " << SuiteName << std::endl;
+  TestRegistrar::get().addCaseToSuite(TC, SuiteName);
+}
+
+int Runner::run() {
+  int ErrorCount = 0;
+  for (auto &TS : TestSuites) {
+    std::cout << "\n======\nExecuting for " << TS.Name << std::endl;
+    TS.setup();
+    for (auto &TC : TS) {
+      std::cout << "\nExecuting " << TC->Name << std::endl;
+      if (Error Err = TC->exec()) {
+        reportError(Err);
+        abortOrKeepGoing();
+        ++ErrorCount;
+      }
+    }
+    TS.teardown();
+  }
+  printSummary();
+  return ErrorCount;
+}
+
+void Runner::reportError(const Error &Err) {}
+
+void Runner::abortOrKeepGoing() {}
+
+void Runner::printSummary() {
+  std::cout << "\n====== SUMMARY\n";
+  for (auto &TS : TestSuites) {
+    std::cout << "  - " << TS.Name;
+    for (auto &TC : TS) {
+      std::string Result;
+      if (TC->IsDisabled) {
+        Result = "-#-#-";
+      } else if (TC->ResultState == TC->ExpectedState) {
+        if (TC->ResultState == omptest::AssertState::Pass)
+          Result = "PASS";
+        else if (TC->ResultState == omptest::AssertState::Fail)
+          Result = "XFAIL";
+      } else {
+        if (TC->ResultState == omptest::AssertState::Fail)
+          Result = "FAIL";
+        else if (TC->ResultState == omptest::AssertState::Pass)
+          Result = "UPASS";
+      }
+      std::cout << "\n      " << std::setw(5) << Result << " : " << TC->Name;
+    }
+    std::cout << std::endl;
+  }
+}
diff --git a/openmp/tools/omptest/test/CMakeLists.txt b/openmp/tools/omptest/test/CMakeLists.txt
new file mode 100644
index 0000000000000..427893313cc67
--- /dev/null
+++ b/openmp/tools/omptest/test/CMakeLists.txt
@@ -0,0 +1,28 @@
+##===----------------------------------------------------------------------===##
+#
+# Add ompTest unit tests to check-openmp.
+#
+##===----------------------------------------------------------------------===##
+
+# Target: ompTest library unit tests
+file(GLOB UNITTEST_SOURCES "unittests/*.cpp")
+add_executable(omptest-unittests ${UNITTEST_SOURCES})
+
+# Add local and LLVM-provided GoogleTest include directories.
+target_include_directories(omptest-unittests PRIVATE
+  ../include
+  ${LLVM_THIRD_PARTY_DIR}/unittest/googletest/include)
+
+target_link_libraries(omptest-unittests PRIVATE omptest)
+
+set_target_properties(omptest-unittests PROPERTIES
+  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+# Add ompTest unit tests to check-openmp
+add_openmp_testsuite(check-ompt-omptest "Running OMPT ompTest unit tests"
+  ${CMAKE_CURRENT_BINARY_DIR} DEPENDS omptest-unittests)
+
+# Configure the lit.site.cfg.in file
+set(AUTO_GEN_COMMENT "## Autogenerated by OPENMP_TOOLS_OMPTEST_TEST "
+                     "configuration.\n# Do not edit!")
+configure_file(lit.site.cfg.in lit.site.cfg @ONLY)
diff --git a/openmp/tools/omptest/test/lit.cfg b/openmp/tools/omptest/test/lit.cfg
new file mode 100644
index 0000000000000..69c401aed83b8
--- /dev/null
+++ b/openmp/tools/omptest/test/lit.cfg
@@ -0,0 +1,26 @@
+# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
+# Configuration file for the 'lit' test runner.
+
+import os
+import lit.formats
+
+# Tell pylint that we know config and lit_config exist somewhere.
+if 'PYLINT_IMPORT' in os.environ:
+    config = object()
+    lit_config = object()
+
+# name: The name of this test suite.
+config.name = 'OMPT ompTest'
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = config.test_obj_root
+
+# test_exec_root: The root object directory where output is placed
+config.test_exec_root = config.test_obj_root
+
+# test format, match (omptest-)unittests
+# Matched binaries (GoogleTests) are executed
+config.test_format = lit.formats.GoogleTest(".", "unittests")
diff --git a/openmp/tools/omptest/test/lit.site.cfg.in b/openmp/tools/omptest/test/lit.site.cfg.in
new file mode 100644
index 0000000000000..4fa8c7e349681
--- /dev/null
+++ b/openmp/tools/omptest/test/lit.site.cfg.in
@@ -0,0 +1,9 @@
+@AUTO_GEN_COMMENT@
+
+config.test_obj_root = "@CMAKE_CURRENT_BINARY_DIR@"
+
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg")
diff --git a/openmp/tools/omptest/test/unittests/asserter-seq-test.cpp b/openmp/tools/omptest/test/unittests/asserter-seq-test.cpp
new file mode 100644
index 0000000000000..32a223c0da832
--- /dev/null
+++ b/openmp/tools/omptest/test/unittests/asserter-seq-test.cpp
@@ -0,0 +1,358 @@
+#include "OmptAliases.h"
+#include "OmptAsserter.h"
+#include <omp-tools.h>
+#include <sstream>
+
+#include "gtest/gtest.h"
+
+using namespace omptest;
+using OAE = omptest::OmptAssertEvent;
+using OS = omptest::ObserveState;
+
+/// SequencedAsserter test-fixture class to avoid code duplication among tests.
+class OmptSequencedAsserterTest : public testing::Test {
+protected:
+  OmptSequencedAsserterTest() {
+    // Construct default sequenced asserter
+    SeqAsserter = std::make_unique<omptest::OmptSequencedAsserter>();
+
+    // Silence all potential log prints
+    SeqAsserter->getLog()->setLoggingLevel(logging::Level::Critical);
+  }
+
+  std::unique_ptr<omptest::OmptSequencedAsserter> SeqAsserter;
+};
+
+TEST_F(OmptSequencedAsserterTest, DefaultState) {
+  // Assertion should neither start as 'deactivated' nor 'suspended'
+  ASSERT_EQ(SeqAsserter->isActive(), true);
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, false);
+
+  // Assertion should begin with event ID zero
+  ASSERT_EQ(SeqAsserter->NextEvent, 0);
+
+  // Assertion should begin without previous notifications or assertions
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+
+  // There should be no expected events
+  ASSERT_EQ(SeqAsserter->Events.empty(), true);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 0);
+
+  // Default mode should be Strict
+  ASSERT_NE(SeqAsserter->getOperationMode(), AssertMode::Relaxed);
+  ASSERT_EQ(SeqAsserter->getOperationMode(), AssertMode::Strict);
+
+  // Default state should be passing
+  ASSERT_NE(SeqAsserter->getState(), AssertState::Fail);
+  ASSERT_EQ(SeqAsserter->getState(), AssertState::Pass);
+  ASSERT_NE(SeqAsserter->checkState(), AssertState::Fail);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, IgnoreNotificationsWhenEmpty) {
+  // ParallelBegin events are suppressed by default
+  auto SuppressedEvent = OAE::ParallelBegin(
+      /*Name=*/"ParBegin", /*Group=*/"", /*Expected=*/OS::Always,
+      /*NumThreads=*/3);
+
+  // DeviceFinalize events are not ignored by default
+  auto IgnoredEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/7);
+
+  // Situation: There is nothing to assert.
+  // Result: All notifications are ignored.
+  // Hence, check that the perceived count of notifications remains unchanged
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+
+  SeqAsserter->notify(std::move(SuppressedEvent));
+
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  SeqAsserter->notify(std::move(IgnoredEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, IgnoreNotificationsWhileDeactivated) {
+  auto ExpectedEvent = OAE::DeviceUnload(
+      /*Name=*/"DevUnload", /*Group=*/"", /*Expected=*/OS::Always);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+
+  // Deactivate asserter, effectively ignoring notifications
+  SeqAsserter->setActive(false);
+  ASSERT_EQ(SeqAsserter->isActive(), false);
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+
+  // DeviceFinalize events are not ignored by default
+  auto IgnoredEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/7);
+  SeqAsserter->notify(std::move(IgnoredEvent));
+
+  // Assertion was deactivated: No change
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+
+  SeqAsserter->setActive(true);
+  ASSERT_EQ(SeqAsserter->isActive(), true);
+
+  auto ObservedEvent = OAE::DeviceUnload(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always);
+  SeqAsserter->notify(std::move(ObservedEvent));
+
+  // Assertion was activated, one notification expected
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEvent) {
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 0);
+  auto ExpectedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/7);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // Sanity check: Notifications should not be triggered
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  // Adding an expected event must change the event count but not the state
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->getState(), AssertState::Pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEventIgnoreSuppressed) {
+  auto ExpectedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/7);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // ParallelBegin events are suppressed by default
+  auto SuppressedEvent = OAE::ParallelBegin(
+      /*Name=*/"ParBegin", /*Group=*/"", /*Expected=*/OS::Always,
+      /*NumThreads=*/3);
+  // Situation: There is one expected event and ParallelBegins are suppressed.
+  // Notification count remains unchanged for suppressed events
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  SeqAsserter->notify(std::move(SuppressedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->getState(), AssertState::Pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEventObservePass) {
+  auto ExpectedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/7);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // DeviceFinalize events are not ignored by default
+  auto ObservedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/7);
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEventObserveFail) {
+  auto ExpectedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/7);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // DeviceFinalize events are not ignored by default
+  // Provide wrong DeviceNum
+  auto ObservedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/23);
+
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  // Observed and expected event do not match: Fail
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Fail);
+}
+
+TEST_F(OmptSequencedAsserterTest, AddEventObserveDifferentType) {
+  auto ExpectedEvent = OAE::DeviceUnload(
+      /*Name=*/"DevUnload", /*Group=*/"", /*Expected=*/OS::Always);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  // DeviceFinalize events are not ignored by default
+  auto ObservedEvent = OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/7);
+
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  // Observed and expected event do not match: Fail
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Fail);
+}
+
+TEST_F(OmptSequencedAsserterTest, CheckTargetGroupNoEffect) {
+  // Situation: Groups are designed to be used as an indicator -WITHIN- target
+  // regions. Hence, comparing two target regions w.r.t. their groups has no
+  // effect on pass or fail.
+
+  auto ExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+
+  // Deactivate asserter, effectively ignoring notifications
+  SeqAsserter->setActive(false);
+  ASSERT_EQ(SeqAsserter->isActive(), false);
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+
+  // Target events are not ignored by default
+  auto ObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+  SeqAsserter->notify(std::move(ObservedEvent));
+
+  // Assertion was deactivated: No change
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+
+  // Re-activate asserter
+  SeqAsserter->setActive(true);
+  ASSERT_EQ(SeqAsserter->isActive(), true);
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+
+  // Actually observe a target event from "AnotherGroup"
+  auto AnotherObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"AnotherGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+  SeqAsserter->notify(std::move(AnotherObservedEvent));
+
+  // Observed all expected events; groups of target regions do not affect pass
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 0);
+
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Pass);
+}
+
+TEST_F(OmptSequencedAsserterTest, CheckSyncPoint) {
+  auto ExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+
+  // Target events are not ignored by default
+  auto ObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+
+  SeqAsserter->notify(OAE::AssertionSyncPoint(
+      /*Name=*/"", /*Group=*/"", /*Expected=*/OS::Always,
+      /*SyncPointName=*/"SyncPoint 1"));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 2);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+
+  // All events processed: SyncPoint "passes"
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Pass);
+
+  auto AnotherExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 0);
+  SeqAsserter->insert(std::move(AnotherExpectedEvent));
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+
+  // Remaining events present: SyncPoint "fails"
+  SeqAsserter->notify(OAE::AssertionSyncPoint(
+      /*Name=*/"", /*Group=*/"", /*Expected=*/OS::Always,
+      /*SyncPointName=*/"SyncPoint 2"));
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Fail);
+}
+
+TEST_F(OmptSequencedAsserterTest, CheckExcessNotify) {
+  auto ExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+  ASSERT_EQ(SeqAsserter->getRemainingEventCount(), 1);
+
+  // Target events are not ignored by default
+  auto ObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+  SeqAsserter->notify(std::move(ObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+
+  // All events processed: pass
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Pass);
+
+  // Target events are not ignored by default
+  auto AnotherObservedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN, /*DeviceNum=*/7,
+      /*TaskData=*/nullptr, /*TargetId=*/23, /*CodeptrRA=*/nullptr);
+
+  // No more events expected: notify "fails"
+  SeqAsserter->notify(std::move(AnotherObservedEvent));
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 2);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Fail);
+}
+
+TEST_F(OmptSequencedAsserterTest, CheckSuspend) {
+  SeqAsserter->insert(OAE::AssertionSuspend(
+      /*Name=*/"", /*Group=*/"", /*Expected=*/OS::Never));
+  ASSERT_EQ(SeqAsserter->Events.empty(), false);
+
+  // Being notified while the next expected event is a "suspend" should change
+  // the asserter's state
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 0);
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, false);
+  SeqAsserter->notify(OAE::DeviceFinalize(
+      /*Name=*/"DevFini", /*Group=*/"", /*Expected=*/OS::Always,
+      /*DeviceNum=*/7));
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, true);
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 1);
+
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 0);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Pass);
+
+  auto ExpectedEvent = OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr);
+  SeqAsserter->insert(std::move(ExpectedEvent));
+
+  // Being notified with an observed event, which matches the next expected
+  // event, resumes assertion (suspended = false)
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, true);
+  SeqAsserter->notify(OAE::Target(
+      /*Name=*/"Target", /*Group=*/"MyTargetGroup", /*Expected=*/OS::Always,
+      /*Kind=*/TARGET, /*Endpoint=*/BEGIN,
+      /*DeviceNum=*/7, /*TaskData=*/nullptr, /*TargetId=*/23,
+      /*CodeptrRA=*/nullptr));
+  ASSERT_EQ(SeqAsserter->AssertionSuspended, false);
+
+  ASSERT_EQ(SeqAsserter->getNotificationCount(), 2);
+  ASSERT_EQ(SeqAsserter->getSuccessfulAssertionCount(), 1);
+  ASSERT_EQ(SeqAsserter->checkState(), AssertState::Pass);
+}
diff --git a/openmp/tools/omptest/test/unittests/internal-event-test.cpp b/openmp/tools/omptest/test/unittests/internal-event-test.cpp
new file mode 100644
index 0000000000000..fcd8956203c9d
--- /dev/null
+++ b/openmp/tools/omptest/test/unittests/internal-event-test.cpp
@@ -0,0 +1,530 @@
+#include "InternalEvent.h"
+#include <omp-tools.h>
+#include <sstream>
+
+#include "gtest/gtest.h"
+
+using namespace omptest;
+
+TEST(InternalEvent_toString, AssertionSyncPoint) {
+  internal::AssertionSyncPoint SP{/*Name=*/"Test Sync Point"};
+
+  EXPECT_EQ(SP.toString(), "Assertion SyncPoint: 'Test Sync Point'");
+}
+
+TEST(InternalEvent_toString, ThreadBegin) {
+  internal::ThreadBegin TB{/*ThreadType=*/ompt_thread_t::ompt_thread_initial};
+
+  EXPECT_EQ(TB.toString(), "OMPT Callback ThreadBegin: ThreadType=1");
+}
+
+TEST(InternalEvent_toString, ThreadEnd) {
+  internal::ThreadEnd TE{};
+
+  EXPECT_EQ(TE.toString(), "OMPT Callback ThreadEnd");
+}
+
+TEST(InternalEvent_toString, ParallelBegin) {
+  internal::ParallelBegin PB{/*NumThreads=*/31};
+
+  EXPECT_EQ(PB.toString(), "OMPT Callback ParallelBegin: NumThreads=31");
+}
+
+TEST(InternalEvent_toString, ParallelEnd) {
+  internal::ParallelEnd PE{/*ParallelData=*/(ompt_data_t *)0x11,
+                           /*EncounteringTaskData=*/(ompt_data_t *)0x22,
+                           /*Flags=*/31,
+                           /*CodeptrRA=*/(const void *)0x33};
+
+  EXPECT_EQ(PE.toString(), "OMPT Callback ParallelEnd");
+}
+
+TEST(InternalEvent_toString, Work) {
+  internal::Work WK{/*WorkType=*/ompt_work_t::ompt_work_loop_dynamic,
+                    /*Endpoint=*/ompt_scope_endpoint_t::ompt_scope_beginend,
+                    /*ParallelData=*/(ompt_data_t *)0x11,
+                    /*TaskData=*/(ompt_data_t *)0x22,
+                    /*Count=*/31,
+                    /*CodeptrRA=*/(const void *)0x33};
+
+  EXPECT_EQ(WK.toString(),
+            "OMPT Callback Work: work_type=11 endpoint=3 parallel_data=0x11 "
+            "task_data=0x22 count=31 codeptr=0x33");
+}
+
+TEST(InternalEvent_toString, Dispatch_iteration) {
+  ompt_data_t DI{.value = 31};
+  internal::Dispatch D{/*ParallelData=*/(ompt_data_t *)0x11,
+                       /*TaskData=*/(ompt_data_t *)0x22,
+                       /*Kind=*/ompt_dispatch_t::ompt_dispatch_iteration,
+                       /*Instance=*/DI};
+
+  EXPECT_EQ(D.toString(), "OMPT Callback Dispatch: parallel_data=0x11 "
+                          "task_data=0x22 kind=1 instance=[it=31]");
+}
+
+TEST(InternalEvent_toString, Dispatch_section) {
+  ompt_data_t DI{.ptr = (void *)0x33};
+  internal::Dispatch D{/*ParallelData=*/(ompt_data_t *)0x11,
+                       /*TaskData=*/(ompt_data_t *)0x22,
+                       /*Kind=*/ompt_dispatch_t::ompt_dispatch_section,
+                       /*Instance=*/DI};
+
+  EXPECT_EQ(D.toString(), "OMPT Callback Dispatch: parallel_data=0x11 "
+                          "task_data=0x22 kind=2 instance=[ptr=0x33]");
+}
+
+TEST(InternalEvent_toString, Dispatch_chunks) {
+  ompt_dispatch_chunk_t DC{.start = 7, .iterations = 31};
+  ompt_data_t DI{.ptr = (void *)&DC};
+
+  internal::Dispatch DLoop{
+      /*ParallelData=*/(ompt_data_t *)0x11,
+      /*TaskData=*/(ompt_data_t *)0x22,
+      /*Kind=*/ompt_dispatch_t::ompt_dispatch_ws_loop_chunk,
+      /*Instance=*/DI};
+
+  internal::Dispatch DTask{
+      /*ParallelData=*/(ompt_data_t *)0x11,
+      /*TaskData=*/(ompt_data_t *)0x22,
+      /*Kind=*/ompt_dispatch_t::ompt_dispatch_taskloop_chunk,
+      /*Instance=*/DI};
+
+  internal::Dispatch DDist{
+      /*ParallelData=*/(ompt_data_t *)0x11,
+      /*TaskData=*/(ompt_data_t *)0x22,
+      /*Kind=*/ompt_dispatch_t::ompt_dispatch_distribute_chunk,
+      /*Instance=*/DI};
+
+  ompt_data_t DINull{.ptr = nullptr};
+  internal::Dispatch DDistNull{
+      /*ParallelData=*/(ompt_data_t *)0x11,
+      /*TaskData=*/(ompt_data_t *)0x22,
+      /*Kind=*/ompt_dispatch_t::ompt_dispatch_distribute_chunk,
+      /*Instance=*/DINull};
+
+  EXPECT_EQ(DLoop.toString(),
+            "OMPT Callback Dispatch: parallel_data=0x11 "
+            "task_data=0x22 kind=3 instance=[chunk=(start=7, iterations=31)]");
+
+  EXPECT_EQ(DTask.toString(),
+            "OMPT Callback Dispatch: parallel_data=0x11 "
+            "task_data=0x22 kind=4 instance=[chunk=(start=7, iterations=31)]");
+
+  EXPECT_EQ(DDist.toString(),
+            "OMPT Callback Dispatch: parallel_data=0x11 "
+            "task_data=0x22 kind=5 instance=[chunk=(start=7, iterations=31)]");
+
+  EXPECT_EQ(DDistNull.toString(), "OMPT Callback Dispatch: parallel_data=0x11 "
+                                  "task_data=0x22 kind=5");
+}
+
+TEST(InternalEvent_toString, TaskCreate) {
+  internal::TaskCreate TC{/*EncounteringTaskData=*/(ompt_data_t *)0x11,
+                          /*EncounteringTaskFrame=*/(const ompt_frame_t *)0x22,
+                          /*NewTaskData=*/(ompt_data_t *)0x33,
+                          /*Flags=*/7,
+                          /*HasDependences=*/31,
+                          /*CodeptrRA=*/(const void *)0x44};
+
+  EXPECT_EQ(TC.toString(),
+            "OMPT Callback TaskCreate: encountering_task_data=0x11 "
+            "encountering_task_frame=0x22 new_task_data=0x33 flags=7 "
+            "has_dependences=31 codeptr=0x44");
+}
+
+TEST(InternalEvent_toString, ImplicitTask) {
+  internal::ImplicitTask IT{
+      /*Endpoint=*/ompt_scope_endpoint_t::ompt_scope_begin,
+      /*ParallelData=*/(ompt_data_t *)0x11,
+      /*TaskData=*/(ompt_data_t *)0x22,
+      /*ActualParallelism=*/7,
+      /*Index=*/31,
+      /*Flags=*/127};
+
+  EXPECT_EQ(IT.toString(),
+            "OMPT Callback ImplicitTask: endpoint=1 parallel_data=0x11 "
+            "task_data=0x22 actual_parallelism=7 index=31 flags=127");
+}
+
+TEST(InternalEvent_toString, SyncRegion) {
+  internal::SyncRegion SR{
+      /*Kind=*/ompt_sync_region_t::ompt_sync_region_taskwait,
+      /*Endpoint=*/ompt_scope_endpoint_t::ompt_scope_end,
+      /*ParallelData=*/(ompt_data_t *)0x11,
+      /*TaskData=*/(ompt_data_t *)0x22,
+      /*CodeptrRA=*/(const void *)0x33};
+
+  EXPECT_EQ(SR.toString(), "OMPT Callback SyncRegion: kind=5 endpoint=2 "
+                           "parallel_data=0x11 task_data=0x22 codeptr=0x33");
+}
+
+TEST(InternalEvent_toString, Target) {
+  internal::Target T{/*Kind=*/ompt_target_t::ompt_target_enter_data_nowait,
+                     /*Endpoint=*/ompt_scope_endpoint_t::ompt_scope_end,
+                     /*DeviceNum=*/7,
+                     /*TaskData=*/(ompt_data_t *)0x11,
+                     /*TargetId=*/(ompt_id_t)31,
+                     /*CodeptrRA=*/(const void *)0x22};
+
+  EXPECT_EQ(T.toString(), "Callback Target: target_id=31 kind=10 "
+                          "endpoint=2 device_num=7 code=0x22");
+}
+
+TEST(InternalEvent_toString, TargetEmi) {
+  ompt_data_t TaskData{.value = 31};
+  ompt_data_t TargetTaskData{.value = 127};
+  ompt_data_t TargetData{.value = 8191};
+
+  internal::TargetEmi T{/*Kind=*/ompt_target_t::ompt_target_update,
+                        /*Endpoint=*/ompt_scope_endpoint_t::ompt_scope_begin,
+                        /*DeviceNum=*/7,
+                        /*TaskData=*/(ompt_data_t *)&TaskData,
+                        /*TargetTaskData=*/(ompt_data_t *)&TargetTaskData,
+                        /*TargetData=*/(ompt_data_t *)&TargetData,
+                        /*CodeptrRA=*/(const void *)0x11};
+
+  internal::TargetEmi TDataNull{
+      /*Kind=*/ompt_target_t::ompt_target_update,
+      /*Endpoint=*/ompt_scope_endpoint_t::ompt_scope_begin,
+      /*DeviceNum=*/7,
+      /*TaskData=*/(ompt_data_t *)&TaskData,
+      /*TargetTaskData=*/(ompt_data_t *)nullptr,
+      /*TargetData=*/(ompt_data_t *)&TargetData,
+      /*CodeptrRA=*/(const void *)0x11};
+
+  std::ostringstream StreamT1;
+  std::ostringstream StreamT2;
+  std::string CallBackPrefix{
+      "Callback Target EMI: kind=4 endpoint=1 device_num=7"};
+  StreamT1 << CallBackPrefix << std::showbase << std::hex;
+  StreamT1 << " task_data=" << &TaskData << " (0x1f)";
+  StreamT1 << " target_task_data=" << &TargetTaskData << " (0x7f)";
+  StreamT1 << " target_data=" << &TargetData << " (0x1fff)";
+  StreamT1 << " code=0x11";
+
+  StreamT2 << CallBackPrefix << std::showbase << std::hex;
+  StreamT2 << " task_data=" << &TaskData << " (0x1f)";
+  StreamT2 << " target_task_data=(nil) (0x0)";
+  StreamT2 << " target_data=" << &TargetData << " (0x1fff)";
+  StreamT2 << " code=0x11";
+
+  EXPECT_EQ(T.toString(), StreamT1.str());
+  EXPECT_EQ(TDataNull.toString(), StreamT2.str());
+}
+
+TEST(InternalEvent_toString, TargetDataOp) {
+  internal::TargetDataOp TDO{
+      /*TargetId=*/7,
+      /*HostOpId=*/31,
+      /*OpType=*/ompt_target_data_op_t::ompt_target_data_associate,
+      /*SrcAddr=*/(void *)0x11,
+      /*SrcDeviceNum=*/127,
+      /*DstAddr=*/(void *)0x22,
+      /*DstDeviceNum=*/8191,
+      /*Bytes=*/4096,
+      /*CodeptrRA=*/(const void *)0x33};
+
+  EXPECT_EQ(
+      TDO.toString(),
+      "  Callback DataOp: target_id=7 host_op_id=31 optype=5 src=0x11 "
+      "src_device_num=127 dest=0x22 dest_device_num=8191 bytes=4096 code=0x33");
+}
+
+TEST(InternalEvent_toString, TargetDataOpEmi) {
+  ompt_data_t TargetTaskData{.value = 31};
+  ompt_data_t TargetData{.value = 127};
+  ompt_id_t HostOpId = 8191;
+
+  internal::TargetDataOpEmi TDO{
+      /*Endpoint=*/ompt_scope_endpoint_t::ompt_scope_begin,
+      /*TargetTaskData=*/(ompt_data_t *)&TargetTaskData,
+      /*TargetData=*/(ompt_data_t *)&TargetData,
+      /*HostOpId=*/(ompt_id_t *)&HostOpId,
+      /*OpType=*/ompt_target_data_op_t::ompt_target_data_disassociate,
+      /*SrcAddr=*/(void *)0x11,
+      /*SrcDeviceNum=*/1,
+      /*DstAddr=*/(void *)0x22,
+      /*DstDeviceNum=*/2,
+      /*Bytes=*/4096,
+      /*CodeptrRA=*/(const void *)0x33};
+
+  // Set HostOpId=nullptr
+  internal::TargetDataOpEmi TDO_HostOpIdNull{
+      /*Endpoint=*/ompt_scope_endpoint_t::ompt_scope_begin,
+      /*TargetTaskData=*/(ompt_data_t *)&TargetTaskData,
+      /*TargetData=*/(ompt_data_t *)&TargetData,
+      /*HostOpId=*/(ompt_id_t *)nullptr,
+      /*OpType=*/ompt_target_data_op_t::ompt_target_data_disassociate,
+      /*SrcAddr=*/(void *)0x11,
+      /*SrcDeviceNum=*/1,
+      /*DstAddr=*/(void *)0x22,
+      /*DstDeviceNum=*/2,
+      /*Bytes=*/4096,
+      /*CodeptrRA=*/(const void *)0x33};
+
+  std::ostringstream StreamTDO1;
+  std::ostringstream StreamTDO2;
+  std::string CallBackPrefix{"  Callback DataOp EMI: endpoint=1 optype=6"};
+  std::string CallBackSuffix{
+      " src=0x11 src_device_num=1 dest=0x22 dest_device_num=2 "
+      "bytes=4096 code=0x33"};
+  StreamTDO1 << CallBackPrefix << std::showbase << std::hex;
+  StreamTDO1 << " target_task_data=" << &TargetTaskData << " (0x1f)";
+  StreamTDO1 << " target_data=" << &TargetData << " (0x7f)";
+  StreamTDO1 << " host_op_id=" << &HostOpId << " (0x1fff)";
+  StreamTDO1 << CallBackSuffix;
+
+  StreamTDO2 << CallBackPrefix << std::showbase << std::hex;
+  StreamTDO2 << " target_task_data=" << &TargetTaskData << " (0x1f)";
+  StreamTDO2 << " target_data=" << &TargetData << " (0x7f)";
+  StreamTDO2 << " host_op_id=(nil) (0x0)";
+  StreamTDO2 << CallBackSuffix;
+
+  EXPECT_EQ(TDO.toString(), StreamTDO1.str());
+  EXPECT_EQ(TDO_HostOpIdNull.toString(), StreamTDO2.str());
+}
+
+TEST(InternalEvent_toString, TargetSubmit) {
+  internal::TargetSubmit TS{/*TargetId=*/7,
+                            /*HostOpId=*/31,
+                            /*RequestedNumTeams=*/127};
+
+  EXPECT_EQ(TS.toString(),
+            "  Callback Submit: target_id=7 host_op_id=31 req_num_teams=127");
+}
+
+TEST(InternalEvent_toString, TargetSubmitEmi) {
+  ompt_data_t TargetData{.value = 127};
+  ompt_id_t HostOpId = 8191;
+  internal::TargetSubmitEmi TS{
+      /*Endpoint=*/ompt_scope_endpoint_t::ompt_scope_begin,
+      /*TargetData=*/(ompt_data_t *)&TargetData,
+      /*HostOpId=*/(ompt_id_t *)&HostOpId,
+      /*RequestedNumTeams=*/7};
+
+  std::ostringstream StreamTS;
+  std::string CallBackPrefix{
+      "  Callback Submit EMI: endpoint=1 req_num_teams=7"};
+  StreamTS << CallBackPrefix << std::showbase << std::hex;
+  StreamTS << " target_data=" << &TargetData << " (0x7f)";
+  StreamTS << " host_op_id=" << &HostOpId << " (0x1fff)";
+
+  EXPECT_EQ(TS.toString(), StreamTS.str());
+}
+
+TEST(InternalEvent_toString, DeviceInitialize) {
+  const char *Type = "DeviceType";
+  const char *DocStr = "DocumentationString";
+
+  internal::DeviceInitialize DI{/*DeviceNum=*/7,
+                                /*Type=*/Type,
+                                /*Device=*/(ompt_device_t *)0x11,
+                                /*LookupFn=*/(ompt_function_lookup_t)0x22,
+                                /*DocStr=*/DocStr};
+
+  internal::DeviceInitialize DINull{/*DeviceNum=*/0,
+                                    /*Type=*/nullptr,
+                                    /*Device=*/nullptr,
+                                    /*LookupFn=*/(ompt_function_lookup_t)0x0,
+                                    /*DocStr=*/nullptr};
+
+  std::ostringstream StreamDI;
+  std::string CallBackPrefix{"Callback Init: device_num=7 type=DeviceType "
+                             "device=0x11 lookup=0x22 doc="};
+  StreamDI << CallBackPrefix << std::showbase << std::hex;
+  StreamDI << (uint64_t)DocStr;
+  EXPECT_EQ(DI.toString(), StreamDI.str());
+
+  // TODO This looks inconsistent: (null) vs. (nil)
+  EXPECT_EQ(DINull.toString(), "Callback Init: device_num=0 type=(null) "
+                               "device=(nil) lookup=(nil) doc=(nil)");
+}
+
+TEST(InternalEvent_toString, DeviceFinalize) {
+  internal::DeviceFinalize DF{/*DeviceNum=*/7};
+
+  EXPECT_EQ(DF.toString(), "Callback Fini: device_num=7");
+}
+
+TEST(InternalEvent_toString, DeviceLoad) {
+  const char *Filename = "FilenameToLoad";
+
+  internal::DeviceLoad DL{/*DeviceNum=*/7,
+                          /*Filename=*/Filename,
+                          /*OffsetInFile=*/31,
+                          /*VmaInFile=*/(void *)0x11,
+                          /*Bytes=*/127,
+                          /*HostAddr=*/(void *)0x22,
+                          /*DeviceAddr=*/(void *)0x33,
+                          /*ModuleId=*/8191};
+
+  internal::DeviceLoad DLNull{/*DeviceNum=*/0,
+                              /*Filename=*/nullptr,
+                              /*OffsetInFile=*/0,
+                              /*VmaInFile=*/nullptr,
+                              /*Bytes=*/0,
+                              /*HostAddr=*/nullptr,
+                              /*DeviceAddr=*/nullptr,
+                              /*ModuleId=*/0};
+
+  EXPECT_EQ(
+      DL.toString(),
+      "Callback Load: device_num:7 module_id:8191 "
+      "filename:FilenameToLoad host_addr:0x22 device_addr:0x33 bytes:127");
+
+  // TODO This looks inconsistent: (null) vs. (nil) and ':' instead of '='
+  EXPECT_EQ(DLNull.toString(),
+            "Callback Load: device_num:0 module_id:0 filename:(null) "
+            "host_addr:(nil) device_addr:(nil) bytes:0");
+}
+
+TEST(InternalEvent_toString, BufferRequest) {
+  size_t Bytes = 7;
+  ompt_buffer_t *Buffer = (void *)0x11;
+
+  internal::BufferRequest BR{/*DeviceNum=*/31,
+                             /*Buffer=*/&Buffer,
+                             /*Bytes=*/&Bytes};
+
+  internal::BufferRequest BRNull{/*DeviceNum=*/127,
+                                 /*Buffer=*/nullptr,
+                                 /*Bytes=*/nullptr};
+
+  EXPECT_EQ(BR.toString(),
+            "Allocated 7 bytes at 0x11 in buffer request callback");
+  EXPECT_EQ(BRNull.toString(),
+            "Allocated 0 bytes at (nil) in buffer request callback");
+}
+
+TEST(InternalEvent_toString, BufferComplete) {
+  ompt_buffer_t *Buffer = (void *)0x11;
+
+  internal::BufferComplete BC{/*DeviceNum=*/7,
+                              /*Buffer=*/Buffer,
+                              /*Bytes=*/127,
+                              /*Begin=*/8191,
+                              /*BufferOwned=*/1};
+
+  internal::BufferComplete BCNull{/*DeviceNum=*/0,
+                                  /*Buffer=*/nullptr,
+                                  /*Bytes=*/0,
+                                  /*Begin=*/0,
+                                  /*BufferOwned=*/0};
+
+  EXPECT_EQ(BC.toString(),
+            "Executing buffer complete callback: 7 0x11 127 0x1fff 1");
+  EXPECT_EQ(BCNull.toString(),
+            "Executing buffer complete callback: 0 (nil) 0 (nil) 0");
+}
+
+TEST(InternalEvent_toString, BufferRecordInvalid) {
+  ompt_record_ompt_t InvalidRecord{
+      /*type=*/ompt_callbacks_t::ompt_callback_parallel_begin,
+      /*time=*/7,
+      /*thread_id=*/31,
+      /*target_id=*/127,
+      /*record=*/{.parallel_begin = {}}};
+
+  internal::BufferRecord BRNull{/*RecordPtr=*/nullptr};
+  internal::BufferRecord BRInvalid{/*RecordPtr=*/&InvalidRecord};
+
+  std::ostringstream StreamBRInvalid;
+  StreamBRInvalid << "rec=" << std::showbase << std::hex << &InvalidRecord;
+  StreamBRInvalid << " type=3 (unsupported record type)";
+
+  EXPECT_EQ(BRNull.toString(), "rec=(nil) type=0 (unsupported record type)");
+  EXPECT_EQ(BRInvalid.toString(), StreamBRInvalid.str());
+}
+
+TEST(InternalEvent_toString, BufferRecordTarget) {
+  ompt_record_target_t SubRecordTarget{
+      /*kind=*/ompt_target_t::ompt_target_update,
+      /*endpoint=*/ompt_scope_endpoint_t::ompt_scope_begin,
+      /*device_num=*/2,
+      /*task_id=*/127,
+      /*target_id=*/31,
+      /*codeptr_ra=*/(const void *)0x11};
+
+  ompt_record_ompt_t TargetRecord{
+      /*type=*/ompt_callbacks_t::ompt_callback_target,
+      /*time=*/7,
+      /*thread_id=*/29,
+      /*target_id=*/31,
+      /*record*/ {.target = SubRecordTarget}};
+
+  internal::BufferRecord BR{/*RecordPtr=*/&TargetRecord};
+
+  std::ostringstream StreamBR;
+  StreamBR << "rec=" << std::showbase << std::hex << &TargetRecord;
+  StreamBR << " type=8 (Target task) time=7 thread_id=29 target_id=31 kind=4";
+  StreamBR << " endpoint=1 device=2 task_id=127 codeptr=0x11";
+
+  EXPECT_EQ(BR.toString(), StreamBR.str());
+}
+
+TEST(InternalEvent_toString, BufferRecordDataOp) {
+  ompt_record_target_data_op_t SubRecordTargetDataOp{
+      /*host_op_id=*/7,
+      /*optype=*/ompt_target_data_op_t::ompt_target_data_alloc_async,
+      /*src_addr=*/(void *)0x11,
+      /*src_device_num=*/1,
+      /*dest_addr=*/(void *)0x22,
+      /*dest_device_num=*/2,
+      /*bytes=*/127,
+      /*end_time=*/128,
+      /*codeptr_ra=*/(const void *)0x33,
+  };
+
+  ompt_record_ompt_t DataOpRecord{
+      /*type=*/ompt_callbacks_t::ompt_callback_target_data_op_emi,
+      /*time=*/8,
+      /*thread_id=*/3,
+      /*target_id=*/5,
+      /*record=*/{.target_data_op = SubRecordTargetDataOp}};
+
+  internal::BufferRecord BR{/*RecordPtr=*/&DataOpRecord};
+
+  std::ostringstream StreamBR;
+  StreamBR << "rec=" << std::showbase << std::hex << &DataOpRecord;
+  StreamBR << " type=34 (Target data op) time=8 thread_id=3 target_id=5";
+  StreamBR << " host_op_id=7 optype=17 src_addr=0x11 src_device=1";
+  StreamBR << " dest_addr=0x22 dest_device=2 bytes=127 end_time=128";
+  StreamBR << " duration=120 ns codeptr=0x33";
+
+  EXPECT_EQ(BR.toString(), StreamBR.str());
+}
+
+TEST(InternalEvent_toString, BufferRecordKernel) {
+  ompt_record_target_kernel_t SubRecordTargetKernel{
+      /*host_op_id=*/11,
+      /*requested_num_teams=*/127,
+      /*granted_num_teams=*/63,
+      /*end_time=*/8191,
+  };
+
+  ompt_record_ompt_t KernelRecord{
+      /*type=*/ompt_callbacks_t::ompt_callback_target_submit_emi,
+      /*time=*/9,
+      /*thread_id=*/19,
+      /*target_id=*/33,
+      /*record=*/{.target_kernel = SubRecordTargetKernel}};
+
+  internal::BufferRecord BR{/*RecordPtr=*/&KernelRecord};
+
+  std::ostringstream StreamBR;
+  StreamBR << "rec=" << std::showbase << std::hex << &KernelRecord;
+  StreamBR << " type=35 (Target kernel) time=9 thread_id=19 target_id=33";
+  StreamBR << " host_op_id=11 requested_num_teams=127 granted_num_teams=63";
+  StreamBR << " end_time=8191 duration=8182 ns";
+
+  EXPECT_EQ(BR.toString(), StreamBR.str());
+}
+
+TEST(InternalEvent_toString, BufferRecordDeallocation) {
+  internal::BufferRecordDeallocation BRD{/*Buffer=*/(ompt_record_ompt_t *)0x11};
+  internal::BufferRecordDeallocation BRDNull{/*Buffer=*/nullptr};
+
+  EXPECT_EQ(BRD.toString(), "Deallocated 0x11");
+  EXPECT_EQ(BRDNull.toString(), "Deallocated (nil)");
+}
diff --git a/openmp/tools/omptest/test/unittests/internal-util-test.cpp b/openmp/tools/omptest/test/unittests/internal-util-test.cpp
new file mode 100644
index 0000000000000..1369e4bba3aa0
--- /dev/null
+++ b/openmp/tools/omptest/test/unittests/internal-util-test.cpp
@@ -0,0 +1,90 @@
+#include "InternalEvent.h"
+#include <omp-tools.h>
+
+#include "gtest/gtest.h"
+
+using namespace omptest;
+using namespace util;
+
+TEST(InternalUtility, ExpectedDefault_Integer) {
+  // int: -2147483648 (decimal) = 0x80000000 (hexadecimal)
+  EXPECT_EQ(expectedDefault(int), 0x80000000);
+  EXPECT_EQ(expectedDefault(int), (0x1 << 31));
+  // int64_t: -9223372036854775808 (decimal) = 0x8000000000000000 (hexadecimal)
+  EXPECT_EQ(expectedDefault(int64_t), 0x8000000000000000);
+  EXPECT_EQ(expectedDefault(int64_t), (0x1L << 63));
+}
+
+TEST(InternalUtility, ExpectedDefault_Zero) {
+  // Expectedly zero
+  EXPECT_EQ(expectedDefault(size_t), 0);
+  EXPECT_EQ(expectedDefault(unsigned int), 0);
+  EXPECT_EQ(expectedDefault(ompt_id_t), 0);
+  EXPECT_EQ(expectedDefault(ompt_dispatch_t), 0);
+  EXPECT_EQ(expectedDefault(ompt_device_time_t), 0);
+}
+
+TEST(InternalUtility, ExpectedDefault_Nullpointer) {
+  // Expectedly nullptr
+  EXPECT_EQ(expectedDefault(const char *), nullptr);
+  EXPECT_EQ(expectedDefault(const void *), nullptr);
+  EXPECT_EQ(expectedDefault(int *), nullptr);
+  EXPECT_EQ(expectedDefault(void *), nullptr);
+  EXPECT_EQ(expectedDefault(ompt_data_t *), nullptr);
+  EXPECT_EQ(expectedDefault(ompt_device_t *), nullptr);
+  EXPECT_EQ(expectedDefault(ompt_frame_t *), nullptr);
+  EXPECT_EQ(expectedDefault(ompt_function_lookup_t), nullptr);
+  EXPECT_EQ(expectedDefault(ompt_id_t *), nullptr);
+}
+
+TEST(InternalUtility, MakeHexString_PointerValues) {
+  // IsPointer should only affect zero value
+  EXPECT_EQ(makeHexString(0, /*IsPointer=*/true), "(nil)");
+  EXPECT_EQ(makeHexString(0, /*IsPointer=*/false), "0x0");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true), "0xff");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/false), "0xff");
+}
+
+TEST(InternalUtility, MakeHexString_MinimumBytes) {
+  // Return a minimum length, based on the (minimum) requested bytes
+  EXPECT_EQ(makeHexString(15, /*IsPointer=*/true, /*MinBytes=*/0), "0xf");
+  EXPECT_EQ(makeHexString(15, /*IsPointer=*/true, /*MinBytes=*/1), "0x0f");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/0), "0xff");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/1), "0xff");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/2), "0x00ff");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/3), "0x0000ff");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/4),
+            "0x000000ff");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/5),
+            "0x00000000ff");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/6),
+            "0x0000000000ff");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/7),
+            "0x000000000000ff");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/8),
+            "0x00000000000000ff");
+
+  // Default to four bytes, if request exceeds eight byte range
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true, /*MinBytes=*/9),
+            "0x000000ff");
+
+  // Disregard requested minimum byte width, if actual value exceeds it
+  EXPECT_EQ(makeHexString(1024, /*IsPointer=*/true, /*MinBytes=*/1), "0x400");
+}
+
+TEST(InternalUtility, MakeHexString_HexBase) {
+  // Cut off "0x" when requested
+  EXPECT_EQ(makeHexString(0, /*IsPointer=*/true, /*MinBytes=*/0,
+                          /*ShowHexBase=*/false),
+            "(nil)");
+  EXPECT_EQ(makeHexString(0, /*IsPointer=*/false, /*MinBytes=*/0,
+                          /*ShowHexBase=*/false),
+            "0");
+  EXPECT_EQ(makeHexString(0, /*IsPointer=*/false, /*MinBytes=*/1,
+                          /*ShowHexBase=*/false),
+            "00");
+  EXPECT_EQ(makeHexString(255, /*IsPointer=*/true,
+                          /*MinBytes=*/2,
+                          /*ShowHexBase=*/false),
+            "00ff");
+}
diff --git a/openmp/tools/omptest/test/unittests/main-test.cpp b/openmp/tools/omptest/test/unittests/main-test.cpp
new file mode 100644
index 0000000000000..093ac36280099
--- /dev/null
+++ b/openmp/tools/omptest/test/unittests/main-test.cpp
@@ -0,0 +1,141 @@
+#include "OmptAssertEvent.h"
+#include "OmptAsserter.h"
+#include "OmptTester.h"
+#include <omp-tools.h>
+
+#include "gtest/gtest.h"
+
+using OS = omptest::ObserveState;
+using OAE = omptest::OmptAssertEvent;
+
+TEST(CompareOperatorTests, ThreadBeginIdentity) {
+  auto TBInitial =
+      OAE::ThreadBegin("dflt", "", OS::Always, ompt_thread_initial);
+  auto TBWorker = OAE::ThreadBegin("dflt", "", OS::Always, ompt_thread_worker);
+  auto TBOther = OAE::ThreadBegin("dflt", "", OS::Always, ompt_thread_other);
+  auto TBUnknown =
+      OAE::ThreadBegin("dflt", "", OS::Always, ompt_thread_unknown);
+
+  ASSERT_EQ(TBInitial, TBInitial);
+  ASSERT_EQ(TBWorker, TBWorker);
+  ASSERT_EQ(TBOther, TBOther);
+  ASSERT_EQ(TBUnknown, TBUnknown);
+}
+
+TEST(CompareOperatorTests, ThreadEndIdentity) {
+  auto TE = OAE::ThreadEnd("dflt", "", OS::Always);
+
+  ASSERT_EQ(TE, TE);
+}
+
+TEST(CompareOperatorTests, ParallelBeginIdentity) {
+  auto PBNumT = OAE::ParallelBegin("thrdenable", "", OS::Always, 3);
+
+  ASSERT_EQ(PBNumT, PBNumT);
+}
+
+TEST(CompareOperatorTests, ParallelEndIdentity) {
+  auto PEDflt = OAE::ParallelEnd("dflt", "", OS::Always);
+  // TODO: Add cases with parallel data set, task data set, flags
+
+  ASSERT_EQ(PEDflt, PEDflt);
+}
+
+TEST(CompareOperatorTests, WorkIdentity) {
+  auto WDLoopBgn =
+      OAE::Work("loopbgn", "", OS::Always, ompt_work_loop, ompt_scope_begin);
+  auto WDLoopEnd =
+      OAE::Work("loopend", "", OS::Always, ompt_work_loop, ompt_scope_end);
+
+  ASSERT_EQ(WDLoopBgn, WDLoopBgn);
+  ASSERT_EQ(WDLoopEnd, WDLoopEnd);
+
+  auto WDSectionsBgn = OAE::Work("sectionsbgn", "", OS::Always,
+                                 ompt_work_sections, ompt_scope_begin);
+  auto WDSectionsEnd = OAE::Work("sectionsend", "", OS::Always,
+                                 ompt_work_sections, ompt_scope_end);
+
+  // TODO: singleexecutor, single_other, workshare, distribute, taskloop, scope,
+  // loop_static, loop_dynamic, loop_guided, loop_other
+
+  ASSERT_EQ(WDSectionsBgn, WDSectionsBgn);
+  ASSERT_EQ(WDSectionsEnd, WDSectionsEnd);
+}
+
+TEST(CompareOperatorTests, DispatchIdentity) {
+  auto DIDflt = OAE::Dispatch("dflt", "", OS::Always);
+
+  ASSERT_EQ(DIDflt, DIDflt);
+}
+
+TEST(CompareOperatorTests, TaskCreateIdentity) {
+  auto TCDflt = OAE::TaskCreate("dflt", "", OS::Always);
+
+  ASSERT_EQ(TCDflt, TCDflt);
+}
+
+TEST(CompareOperatorTests, TaskScheduleIdentity) {
+  auto TS = OAE::TaskSchedule("dflt", "", OS::Always);
+
+  ASSERT_EQ(TS, TS);
+}
+
+TEST(CompareOperatorTests, ImplicitTaskIdentity) {
+  auto ITDfltBgn =
+      OAE::ImplicitTask("dfltbgn", "", OS::Always, ompt_scope_begin);
+  auto ITDfltEnd = OAE::ImplicitTask("dfltend", "", OS::Always, ompt_scope_end);
+
+  ASSERT_EQ(ITDfltBgn, ITDfltBgn);
+  ASSERT_EQ(ITDfltEnd, ITDfltEnd);
+}
+
+TEST(CompareOperatorTests, SyncRegionIdentity) {
+  auto SRDfltBgn =
+      OAE::SyncRegion("srdfltbgn", "", OS::Always,
+                      ompt_sync_region_barrier_explicit, ompt_scope_begin);
+  auto SRDfltEnd =
+      OAE::SyncRegion("srdfltend", "", OS::Always,
+                      ompt_sync_region_barrier_explicit, ompt_scope_end);
+
+  ASSERT_EQ(SRDfltBgn, SRDfltBgn);
+  ASSERT_EQ(SRDfltEnd, SRDfltEnd);
+}
+
+TEST(CompareOperatorTests, TargetIdentity) {
+  auto TargetDfltBgn =
+      OAE::Target("dfltbgn", "", OS::Always, ompt_target, ompt_scope_begin);
+  auto TargetDfltEnd =
+      OAE::Target("dfltend", "", OS::Always, ompt_target, ompt_scope_end);
+
+  ASSERT_EQ(TargetDfltBgn, TargetDfltBgn);
+  ASSERT_EQ(TargetDfltEnd, TargetDfltEnd);
+
+  auto TargetDevBgn = OAE::Target("tgtdevbgn", "", OS::Always, ompt_target,
+                                  ompt_scope_begin, 1);
+  auto TargetDevEnd =
+      OAE::Target("tgtdevend", "", OS::Always, ompt_target, ompt_scope_end, 1);
+
+  ASSERT_EQ(TargetDevBgn, TargetDevBgn);
+  ASSERT_EQ(TargetDevEnd, TargetDevEnd);
+}
+
+TEST(CompareOperatorTests, BufferRecordIdentity) {
+  // Default, no time limit or anything
+  auto BRDflt =
+      OAE::BufferRecord("dflt", "", OS::Always, ompt_callback_target_submit);
+
+  // Minimum time set, no max time
+  auto BRMinSet = OAE::BufferRecord("minset", "", OS::Always,
+                                    ompt_callback_target_submit, 10);
+
+  // Minimum time and maximum time set
+  auto BRMinMaxSet = OAE::BufferRecord("minmaxset", "", OS::Always,
+                                       ompt_callback_target_submit, {10, 100});
+
+  ASSERT_EQ(BRDflt, BRDflt);
+  ASSERT_EQ(BRMinSet, BRMinSet);
+  ASSERT_EQ(BRMinMaxSet, BRMinMaxSet);
+}
+
+// Add main definition
+OMPTEST_TESTSUITE_MAIN()
diff --git a/orc-rt/CMakeLists.txt b/orc-rt/CMakeLists.txt
new file mode 100644
index 0000000000000..77448185311da
--- /dev/null
+++ b/orc-rt/CMakeLists.txt
@@ -0,0 +1,58 @@
+# CMake build for ORC-RT.
+
+#===============================================================================
+# Setup Project
+#===============================================================================
+
+cmake_minimum_required(VERSION 3.20.0)
+
+set(LLVM_COMMON_CMAKE_UTILS ${CMAKE_CURRENT_SOURCE_DIR}/../cmake)
+include(${LLVM_COMMON_CMAKE_UTILS}/Modules/CMakePolicy.cmake
+  NO_POLICY_SCOPE)
+
+project(OrcRT LANGUAGES C CXX ASM)
+
+list(INSERT CMAKE_MODULE_PATH 0
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules"
+  "${CMAKE_CURRENT_SOURCE_DIR}/../runtimes/cmake/Modules"
+  "${LLVM_COMMON_CMAKE_UTILS}"
+  "${LLVM_COMMON_CMAKE_UTILS}/Modules"
+  )
+
+include(GNUInstallDirs)
+
+#===============================================================================
+# Setup CMake Options
+#===============================================================================
+
+option(ORC_RT_INCLUDE_DOCS "Build the ORC-RT documentation." ON)
+option(ORC_RT_ENABLE_ASSERTIONS "Enable assertions independent of build mode." ON)
+option(ORC_RT_ENABLE_PEDANTIC "Compile with pedantic enabled." ON)
+option(ORC_RT_ENABLE_WERROR "Fail and stop if a warning is triggered." OFF)
+option(ORC_RT_INCLUDE_TESTS "Build ORC-RT tests." ${LLVM_INCLUDE_TESTS})
+
+set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to conform to")
+set(CMAKE_CXX_STANDARD_REQUIRED YES)
+set(CMAKE_CXX_EXTENSIONS NO)
+set(CMAKE_FOLDER "orc-rt")
+
+set(ORC_RT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(ORC_RT_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
+#===============================================================================
+# Setup Source Code
+#===============================================================================
+
+if (ORC_RT_INCLUDE_DOCS)
+    add_subdirectory(docs)
+endif()
+
+add_subdirectory(include)
+add_subdirectory(lib)
+add_subdirectory(tools)
+
+if(ORC_RT_INCLUDE_TESTS)
+  add_subdirectory(test)
+  add_subdirectory(unittests)
+endif()
diff --git a/orc-rt/LICENSE.TXT b/orc-rt/LICENSE.TXT
new file mode 100644
index 0000000000000..53bb2e7fbc764
--- /dev/null
+++ b/orc-rt/LICENSE.TXT
@@ -0,0 +1,234 @@
+==============================================================================
+The LLVM Project is under the Apache License v2.0 with LLVM Exceptions:
+==============================================================================
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+    1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+    2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+    3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+    4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+    5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+    6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+    7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+    8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+    9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+    END OF TERMS AND CONDITIONS
+
+    APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+    Copyright [yyyy] [name of copyright owner]
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+
+---- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+
+==============================================================================
+Software from third parties included in the LLVM Project:
+==============================================================================
+The LLVM Project contains third party software which is under different license
+terms. All such code will be identified clearly using at least one of two
+mechanisms:
+1) It will be in a separate directory tree with its own `LICENSE.txt` or
+   `LICENSE` file at the top containing the specific license and restrictions
+   which apply to that software, or
+2) It will contain specific license and restriction terms at the top of every
+   file.
diff --git a/orc-rt/cmake/OrcRTTesting.cmake b/orc-rt/cmake/OrcRTTesting.cmake
new file mode 100644
index 0000000000000..b26a02eede25e
--- /dev/null
+++ b/orc-rt/cmake/OrcRTTesting.cmake
@@ -0,0 +1,36 @@
+# Keep track if we have all dependencies.
+set(ORC_RT_LLVM_TOOLS_AVAILABLE TRUE)
+
+if (NOT DEFINED ORC_RT_LLVM_TOOLS_DIR AND DEFINED LLVM_BINARY_DIR)
+  cmake_path(APPEND ORC_RT_LLVM_TOOLS_DIR "${LLVM_BINARY_DIR}" "bin")
+endif()
+
+if (TARGET utils/llvm-lit/all)
+   list(APPEND ORC_RT_TEST_DEPS utils/llvm-lit/all)
+endif()
+
+# Add dependence on FileCheck.
+if (TARGET FileCheck)
+  list(APPEND ORC_RT_TEST_DEPS FileCheck)
+endif()
+
+find_program(ORC_RT_FILECHECK_EXECUTABLE
+  NAMES FileCheck
+  PATHS ${ORC_RT_LLVM_TOOLS_DIR})
+if (NOT ORC_RT_FILECHECK_EXECUTABLE)
+  message(STATUS "Cannot find FileCheck. Please put it in your PATH, set ORC_RT_FILECHECK_EXECUTABLE to its full path, or point ORC_RT_LLVM_TOOLS_DIR to its directory.")
+  set(ORC_RT_LLVM_TOOLS_AVAILABLE FALSE)
+endif()
+
+# Add dependence on not.
+if (TARGET not)
+  list(APPEND ORC_RT_TEST_DEPS not)
+endif()
+
+find_program(ORC_RT_NOT_EXECUTABLE
+  NAMES not
+  PATHS ${ORC_RT_LLVM_TOOLS_DIR})
+if (NOT ORC_RT_NOT_EXECUTABLE)
+  message(STATUS "Cannot find 'not'. Please put it in your PATH, set ORC_RT_NOT_EXECUTABLE to its full path, or point ORC_RT_LLVM_TOOLS_DIR to its directory.")
+  set(ORC_RT_LLVM_TOOLS_AVAILABLE FALSE)
+endif()
diff --git a/orc-rt/docs/Building-orc-rt.md b/orc-rt/docs/Building-orc-rt.md
new file mode 100644
index 0000000000000..69dbe7f3f3141
--- /dev/null
+++ b/orc-rt/docs/Building-orc-rt.md
@@ -0,0 +1,74 @@
+# Building orc-rt
+
+## Getting Started
+
+The basic steps needed to build orc-rt are:
+
+* Checkout llvm-project:
+
+   * ``cd where-you-want-llvm-to-live``
+   * ``git clone https://github.com/llvm/llvm-project.git``
+
+* Configure and build orc-rt:
+
+   CMake is the only supported configuration system.
+
+   Clang is the preferred compiler when building and using orc-rt.
+
+   * ``cd where you want to build llvm``
+   * ``mkdir build``
+   * ``cd build``
+   * ``cmake -G <generator> -DLLVM_ENABLE_RUNTIMES=orc-rt [options] <llvm-monorepo>/runtimes``
+
+   For more information about configuring orc-rt see :ref:`CMake Options`.
+
+   * ``make orc-rt`` --- will build orc-rt.
+   * ``make check-orc-rt`` --- will run the test suite.
+
+   Shared and static libraries for orc-rt should now be present in
+   llvm/build/lib.
+
+* **Optional**: Install orc-rt
+
+   Remember to use the CMake option ``CMAKE_INSTALL_PREFIX`` to select a safe
+   place to install orc-rt.
+
+   * ``make install-orc-rt`` --- Will install the libraries and the headers
+
+## CMake Options
+
+Here are some of the CMake variables that are used often, along with a
+brief explanation and LLVM-specific notes. For full documentation, check the
+CMake docs or execute ``cmake --help-variable VARIABLE_NAME``.
+
+**CMAKE_BUILD_TYPE**:STRING
+  Sets the build type for ``make`` based generators. Possible values are
+  Release, Debug, RelWithDebInfo and MinSizeRel. On systems like Visual Studio
+  the user sets the build type with the IDE settings.
+
+**CMAKE_INSTALL_PREFIX**:PATH
+  Path where LLVM will be installed if "make install" is invoked or the
+  "INSTALL" target is built.
+
+**CMAKE_CXX_COMPILER**:STRING
+  The C++ compiler to use when building and testing orc-rt.
+
+## orc-rt specific options
+
+* option:: ORC_RT_ENABLE_ASSERTIONS:BOOL
+
+  **Default**: ``ON``
+
+  Toggle assertions independent of the build mode.
+
+* option:: ORC_RT_ENABLE_PEDANTIC:BOOL
+
+  **Default**: ``ON``
+
+  Compile with -Wpedantic.
+
+* option:: ORC_RT_ENABLE_WERROR:BOOL
+
+  **Default**: ``ON``
+
+  Compile with -Werror
diff --git a/orc-rt/docs/CMakeLists.txt b/orc-rt/docs/CMakeLists.txt
new file mode 100644
index 0000000000000..8709000c6254e
--- /dev/null
+++ b/orc-rt/docs/CMakeLists.txt
@@ -0,0 +1,8 @@
+if (LLVM_ENABLE_SPHINX)
+  include(AddSphinxTarget)
+  if (SPHINX_FOUND)
+    if (${SPHINX_OUTPUT_HTML})
+      add_sphinx_target(html orc-rt)
+    endif()
+  endif()
+endif()
diff --git a/orc-rt/docs/README.txt b/orc-rt/docs/README.txt
new file mode 100644
index 0000000000000..da202c5fa8391
--- /dev/null
+++ b/orc-rt/docs/README.txt
@@ -0,0 +1,13 @@
+ORC-RT Documentation
+====================
+
+The ORC-RT documentation is written using the Sphinx documentation generator. It is
+currently tested with Sphinx 5.3.0.
+
+To build the documents into html configure ORC-RT with the following cmake options:
+
+  * -DLLVM_ENABLE_SPHINX=ON
+  * -DORC_RT_INCLUDE_DOCS=ON
+
+After configuring ORC-RT with these options the make rule `docs-orc-rt-html`
+should be available.
diff --git a/orc-rt/docs/conf.py b/orc-rt/docs/conf.py
new file mode 100644
index 0000000000000..b9f197d85b847
--- /dev/null
+++ b/orc-rt/docs/conf.py
@@ -0,0 +1,253 @@
+# -*- coding: utf-8 -*-
+# ORC-RT documentation build configuration file.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+from datetime import date
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+# sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+# needs_sphinx = '1.0'
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [
+    "sphinx.ext.todo",
+    "sphinx.ext.mathjax",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.autodoc",
+]
+
+# When building man pages, we do not use the markdown pages,
+# So, we can continue without the myst_parser dependencies.
+# Doing so reduces dependencies of some packaged llvm distributions.
+try:
+    import myst_parser
+
+    extensions.append("myst_parser")
+except ImportError:
+    if not tags.has("builder-man"):
+        raise
+
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+myst_heading_anchors = 6
+
+import sphinx
+
+# The encoding of source files.
+# source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = "index"
+
+# General information about the project.
+project = "ORC-RT"
+copyright = "2025-%d, The ORC-RT Team" % date.today().year
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents. These are currently set to zero because we don't use them.
+# Should somebody consider in the future to change them, they need to be updated
+# everytime a new release comes out.
+#
+# The short version.
+# version = '0'
+# The full version, including alpha/beta/rc tags.
+# release = '0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+# language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ["_build", "analyzer", "FIR/*"]
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "friendly"
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = "haiku"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+# html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+html_title = "ORC Runtime"
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+# html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+# html_static_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = "%b %d, %Y"
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_domain_indices = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "orc-rt-doc"
+
+# If true, the reST sources are included in the HTML build as
+# _sources/name. The default is True.
+html_copy_source = False
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+    ("Overview", "ORC-RT.tex", "orc-rt Documentation", "The orc-rt Team", "manual"),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+# latex_appendices = []
+
+# If false, no module index is generated.
+# latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = []
+
+# If true, show URL addresses after external links.
+# man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    ("Overview"),
+]
+
+# Documents to append as an appendix to all manuals.
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+# texinfo_show_urls = 'footnote'
diff --git a/orc-rt/docs/index.md b/orc-rt/docs/index.md
new file mode 100644
index 0000000000000..56f9bb3829c17
--- /dev/null
+++ b/orc-rt/docs/index.md
@@ -0,0 +1,54 @@
+# LLVM ORC Runtime
+
+## Overview
+
+The ORC runtime provides executor-side support code for the LLVM ORC APIs.
+
+```{eval-rst}
+.. toctree::
+   :titlesonly:
+
+   Building-orc-rt
+```
+
+### Current Status
+
+The ORC Runtime is a new, experimental project. It is being actively developed,
+and neither the ABI nor API are stable. LLVM ORC API clients should be careful
+to use an ORC Runtime from the same build as their LLVM ORC libraries.
+
+### Platform and Compiler Support
+
+* TODO
+
+The following minimum compiler versions are strongly recommended.
+
+* Clang 16 and above
+
+Anything older *may* work.
+
+### Notes and Known Issues
+
+* TODO
+
+## Getting Involved
+
+First please review our
+[Developer's Policy](https://llvm.org/docs/DeveloperPolicy.html) and
+[Getting started with LLVM](https://llvm.org/docs/GettingStarted.html).
+
+**Bug Reports**
+
+If you think you've found a bug in the ORC Runtime, please report it using
+the [LLVM bug tracker](https://github.com/llvm/llvm-project/labels/orc-rt/).
+Please use the tag "orc-rt" for new threads.
+
+**Patches**
+
+If you want to contribute a patch to th ORC runtime, please start by reading
+the LLVM
+[documentation about contributing](https://www.llvm.org/docs/Contributing.html).
+
+**Discussion and Questions**
+
+* TODO
diff --git a/orc-rt/include/CMakeLists.txt b/orc-rt/include/CMakeLists.txt
new file mode 100644
index 0000000000000..61834f765fc8c
--- /dev/null
+++ b/orc-rt/include/CMakeLists.txt
@@ -0,0 +1,32 @@
+set(ORC_RT_HEADERS
+    orc-rt-c/ExternC.h
+    orc-rt-c/WrapperFunctionResult.h
+    orc-rt-c/orc-rt.h
+    orc-rt/BitmaskEnum.h
+    orc-rt/Compiler.h
+    orc-rt/Error.h
+    orc-rt/ExecutorAddress.h
+    orc-rt/IntervalMap.h
+    orc-rt/IntervalSet.h
+    orc-rt/Math.h
+    orc-rt/RTTI.h
+    orc-rt/WrapperFunctionResult.h
+    orc-rt/SimplePackedSerialization.h
+    orc-rt/bit.h
+    orc-rt/move_only_function.h
+    orc-rt/span.h
+)
+
+# TODO: Switch to filesets when we move to cmake-3.23.
+add_library(orc-rt-headers INTERFACE)
+target_include_directories(orc-rt-headers INTERFACE
+    $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include>
+    $<INSTALL_INTERFACE:include>
+)
+set_property(TARGET orc-rt-headers
+    PROPERTY PUBLIC_HEADER ${ORC_RT_HEADERS}
+)
+install(TARGETS orc-rt-headers
+    DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/
+    COMPONENT OrcRT_Development
+)
diff --git a/orc-rt/include/orc-rt-c/ExternC.h b/orc-rt/include/orc-rt-c/ExternC.h
new file mode 100644
index 0000000000000..dd839037adf1e
--- /dev/null
+++ b/orc-rt/include/orc-rt-c/ExternC.h
@@ -0,0 +1,41 @@
+/*===- ExternC.h - C API for the ORC runtime ----------------------*- C -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* This file defines the C API for the ORC runtime                            *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef ORC_RT_C_EXTERNC_H
+#define ORC_RT_C_EXTERNC_H
+
+/* Helper to suppress strict prototype warnings. */
+#ifdef __clang__
+#define ORC_RT_C_STRICT_PROTOTYPES_BEGIN                                       \
+  _Pragma("clang diagnostic push")                                             \
+      _Pragma("clang diagnostic error \"-Wstrict-prototypes\"")
+#define ORC_RT_C_STRICT_PROTOTYPES_END _Pragma("clang diagnostic pop")
+#else
+#define ORC_RT_C_STRICT_PROTOTYPES_BEGIN
+#define ORC_RT_C_STRICT_PROTOTYPES_END
+#endif
+
+/* Helper to wrap C code for C++ */
+#ifdef __cplusplus
+#define ORC_RT_C_EXTERN_C_BEGIN                                                \
+  extern "C" {                                                                 \
+  ORC_RT_C_STRICT_PROTOTYPES_BEGIN
+#define ORC_RT_C_EXTERN_C_END                                                  \
+  ORC_RT_C_STRICT_PROTOTYPES_END                                               \
+  }
+#else
+#define ORC_RT_C_EXTERN_C_BEGIN ORC_RT_C_STRICT_PROTOTYPES_BEGIN
+#define ORC_RT_C_EXTERN_C_END ORC_RT_C_STRICT_PROTOTYPES_END
+#endif
+
+#endif /* ORC_RT_C_EXTERNC_H */
diff --git a/orc-rt/include/orc-rt-c/WrapperFunctionResult.h b/orc-rt/include/orc-rt-c/WrapperFunctionResult.h
new file mode 100644
index 0000000000000..d8dec28680f1b
--- /dev/null
+++ b/orc-rt/include/orc-rt-c/WrapperFunctionResult.h
@@ -0,0 +1,178 @@
+/*===----- WrapperFunctionResult.h - blob-of-bytes container -----*- C -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+|*===----------------------------------------------------------------------===*|
+|*                                                                            *|
+|* Defines orc_rt_WrapperFunctionResult and related APIs.                     *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef ORC_RT_C_WRAPPERFUNCTIONRESULT_H
+#define ORC_RT_C_WRAPPERFUNCTIONRESULT_H
+
+#include "orc-rt-c/ExternC.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+ORC_RT_C_EXTERN_C_BEGIN
+
+typedef union {
+  char *ValuePtr;
+  char Value[sizeof(char *)];
+} orc_rt_WrapperFunctionResultDataUnion;
+
+/**
+ * orc_rt_WrapperFunctionResult is a kind of C-SmallVector with an
+ * out-of-band error state.
+ *
+ * If Size == 0 and Data.ValuePtr is non-zero then the value is in the
+ * 'out-of-band error' state, and Data.ValuePtr points at a malloc-allocated,
+ * null-terminated string error message.
+ *
+ * If Size <= sizeof(orc_rt_WrapperFunctionResultData) then the value is in
+ * the 'small' state and the content is held in the first Size bytes of
+ * Data.Value.
+ *
+ * If Size > sizeof(orc_rt_WrapperFunctionResultData) then the value is in the
+ * 'large' state and the content is held in the first Size bytes of the
+ * memory pointed to by Data.ValuePtr. This memory must have been allocated by
+ * malloc, and will be freed with free when this value is destroyed.
+ */
+typedef struct {
+  orc_rt_WrapperFunctionResultDataUnion Data;
+  size_t Size;
+} orc_rt_WrapperFunctionResult;
+
+/**
+ * Zero-initialize an orc_rt_WrapperFunctionResult.
+ */
+static inline void
+orc_rt_WrapperFunctionResultInit(orc_rt_WrapperFunctionResult *R) {
+  R->Size = 0;
+  R->Data.ValuePtr = 0;
+}
+
+/**
+ * Create an orc_rt_WrapperFunctionResult with an uninitialized buffer of
+ * size Size. The buffer is returned via the DataPtr argument.
+ */
+static inline orc_rt_WrapperFunctionResult
+orc_rt_WrapperFunctionResultAllocate(size_t Size) {
+  orc_rt_WrapperFunctionResult R;
+  R.Size = Size;
+  // If Size is 0 ValuePtr must be 0 or it is considered an out-of-band error.
+  R.Data.ValuePtr = 0;
+  if (Size > sizeof(R.Data.Value))
+    R.Data.ValuePtr = (char *)malloc(Size);
+  return R;
+}
+
+/**
+ * Create an orc_rt_WrapperFunctionResult from the given data range.
+ */
+static inline orc_rt_WrapperFunctionResult
+orc_rt_CreateWrapperFunctionResultFromRange(const char *Data, size_t Size) {
+  orc_rt_WrapperFunctionResult R;
+  R.Size = Size;
+  if (R.Size > sizeof(R.Data.Value)) {
+    char *Tmp = (char *)malloc(Size);
+    memcpy(Tmp, Data, Size);
+    R.Data.ValuePtr = Tmp;
+  } else
+    memcpy(R.Data.Value, Data, Size);
+  return R;
+}
+
+/**
+ * Create an orc_rt_WrapperFunctionResult by copying the given string,
+ * including the null-terminator.
+ *
+ * This function copies the input string. The client is responsible for freeing
+ * the ErrMsg arg.
+ */
+static inline orc_rt_WrapperFunctionResult
+orc_rt_CreateWrapperFunctionResultFromString(const char *Source) {
+  return orc_rt_CreateWrapperFunctionResultFromRange(Source,
+                                                     strlen(Source) + 1);
+}
+
+/**
+ * Create an orc_rt_WrapperFunctionResult representing an out-of-band
+ * error.
+ *
+ * This function copies the input string. The client is responsible for freeing
+ * the ErrMsg arg.
+ */
+static inline orc_rt_WrapperFunctionResult
+orc_rt_CreateWrapperFunctionResultFromOutOfBandError(const char *ErrMsg) {
+  orc_rt_WrapperFunctionResult R;
+  R.Size = 0;
+  char *Tmp = (char *)malloc(strlen(ErrMsg) + 1);
+  strcpy(Tmp, ErrMsg);
+  R.Data.ValuePtr = Tmp;
+  return R;
+}
+
+/**
+ * This should be called to destroy orc_rt_WrapperFunctionResult values
+ * regardless of their state.
+ */
+static inline void
+orc_rt_DisposeWrapperFunctionResult(orc_rt_WrapperFunctionResult *R) {
+  if (R->Size > sizeof(R->Data.Value) || (R->Size == 0 && R->Data.ValuePtr))
+    free(R->Data.ValuePtr);
+}
+
+/**
+ * Get a pointer to the data contained in the given
+ * orc_rt_WrapperFunctionResult.
+ */
+static inline char *
+orc_rt_WrapperFunctionResultData(orc_rt_WrapperFunctionResult *R) {
+  assert((R->Size != 0 || R->Data.ValuePtr == NULL) &&
+         "Cannot get data for out-of-band error value");
+  return R->Size > sizeof(R->Data.Value) ? R->Data.ValuePtr : R->Data.Value;
+}
+
+/**
+ * Safely get the size of the given orc_rt_WrapperFunctionResult.
+ *
+ * Asserts that we're not trying to access the size of an error value.
+ */
+static inline size_t
+orc_rt_WrapperFunctionResultSize(const orc_rt_WrapperFunctionResult *R) {
+  assert((R->Size != 0 || R->Data.ValuePtr == NULL) &&
+         "Cannot get size for out-of-band error value");
+  return R->Size;
+}
+
+/**
+ * Returns 1 if this value is equivalent to a value just initialized by
+ * orc_rt_WrapperFunctionResultInit, 0 otherwise.
+ */
+static inline size_t
+orc_rt_WrapperFunctionResultEmpty(const orc_rt_WrapperFunctionResult *R) {
+  return R->Size == 0 && R->Data.ValuePtr == 0;
+}
+
+/**
+ * Returns a pointer to the out-of-band error string for this
+ * orc_rt_WrapperFunctionResult, or null if there is no error.
+ *
+ * The orc_rt_WrapperFunctionResult retains ownership of the error
+ * string, so it should be copied if the caller wishes to preserve it.
+ */
+static inline const char *orc_rt_WrapperFunctionResultGetOutOfBandError(
+    const orc_rt_WrapperFunctionResult *R) {
+  return R->Size == 0 ? R->Data.ValuePtr : 0;
+}
+
+ORC_RT_C_EXTERN_C_END
+
+#endif /* ORC_RT_WRAPPERFUNCTIONRESULT_H */
diff --git a/libcxx/test/libcxx/utilities/template.bitset/includes.pass.cpp b/orc-rt/include/orc-rt-c/orc-rt.h
similarity index 50%
rename from libcxx/test/libcxx/utilities/template.bitset/includes.pass.cpp
rename to orc-rt/include/orc-rt-c/orc-rt.h
index ae8464ac946a9..31ef74f6c1e9b 100644
--- a/libcxx/test/libcxx/utilities/template.bitset/includes.pass.cpp
+++ b/orc-rt/include/orc-rt-c/orc-rt.h
@@ -1,33 +1,32 @@
-//===----------------------------------------------------------------------===//
+//===- orc-rt-c/orc-rt.h - Placeholder header for orc-rt ----------*- C -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Placeholder header for initial orc-rt checkin.
+//
+//===----------------------------------------------------------------------===//
 
-// test that <bitset> includes <string>, <stdexcept> and <iosfwd>
-
-// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+#ifndef ORC_RT_C_ORC_RT_H
+#define ORC_RT_C_ORC_RT_H
 
-#include <bitset>
+#ifdef __cplusplus
+extern "C" {
+#endif
 
-#include "test_macros.h"
+/// \addtogroup orc_rt_c_api orc-rt C APIs
+/// @{
 
-#ifndef _LIBCPP_STRING
-#error <string> has not been included
-#endif
+void orc_rt(void);
 
-#ifndef _LIBCPP_STDEXCEPT
-#error <stdexcept> has not been included
-#endif
+/// @}
+///
 
-#ifndef _LIBCPP_IOSFWD
-#error <iosfwd> has not been included
+#ifdef __cplusplus
+} // extern "C"
 #endif
 
-int main(int, char**)
-{
-
-  return 0;
-}
+#endif // ORC_RT_C_ORC_RT_H
diff --git a/orc-rt/include/orc-rt/BitmaskEnum.h b/orc-rt/include/orc-rt/BitmaskEnum.h
new file mode 100644
index 0000000000000..f3aff3287a2ff
--- /dev/null
+++ b/orc-rt/include/orc-rt/BitmaskEnum.h
@@ -0,0 +1,161 @@
+//===---- BitmaskEnum.h - Enable bitmask operations on enums ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Provides utilities for easily adding bitmask operation support to enums.
+//
+// This code was derived from LLVM's include/llvm/ADT/BitmaskEnum.h header, and
+// adapted for the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_BITMASKENUM_H
+#define ORC_RT_BITMASKENUM_H
+
+#include "Math.h"
+
+#include <cassert>
+#include <type_traits>
+
+namespace orc_rt {
+
+/// ORC_RT_MARK_AS_BITMASK_ENUM lets you opt in an individual enum type so you
+/// can perform bitwise operations on it without putting static_cast everywhere.
+///
+/// \code
+///   enum MyEnum {
+///     E1 = 1, E2 = 2, E3 = 4, E4 = 8,
+///     ORC_RT_MARK_AS_BITMASK_ENUM(/* LargestValue = */ E4)
+///   };
+///
+///   void Foo() {
+///     MyEnum A = (E1 | E2) & E3 ^ ~E4; // Look, ma: No static_cast!
+///   }
+/// \endcode
+///
+/// Normally when you do a bitwise operation on an enum value, you get back an
+/// instance of the underlying type (e.g. int).  But using this macro, bitwise
+/// ops on your enum will return you back instances of the enum.  This is
+/// particularly useful for enums which represent a combination of flags.
+///
+/// The parameter to ORC_RT_MARK_AS_BITMASK_ENUM should be the largest
+/// individual value in your enum.
+///
+/// All of the enum's values must be non-negative.
+#define ORC_RT_MARK_AS_BITMASK_ENUM(LargestValue)                              \
+  ORC_RT_BITMASK_LARGEST_ENUMERATOR = LargestValue
+
+/// ORC_RT_DECLARE_ENUM_AS_BITMASK can be used to declare an enum type as a bit
+/// set, so that bitwise operation on such enum does not require static_cast.
+///
+/// \code
+///   enum MyEnum { E1 = 1, E2 = 2, E3 = 4, E4 = 8 };
+///   ORC_RT_DECLARE_ENUM_AS_BITMASK(MyEnum, E4);
+///
+///   void Foo() {
+///     MyEnum A = (E1 | E2) & E3 ^ ~E4; // No static_cast
+///   }
+/// \endcode
+///
+/// The second parameter to ORC_RT_DECLARE_ENUM_AS_BITMASK specifies the largest
+/// bit value of the enum type.
+///
+/// ORC_RT_DECLARE_ENUM_AS_BITMASK should be used in __orc_rt namespace.
+///
+/// This a non-intrusive alternative for ORC_RT_MARK_AS_BITMASK_ENUM. It allows
+/// declaring more than one non-scoped enumerations as bitmask types in the same
+/// scope. Otherwise it provides the same functionality as
+/// ORC_RT_MARK_AS_BITMASK_ENUM.
+#define ORC_RT_DECLARE_ENUM_AS_BITMASK(Enum, LargestValue)                     \
+  template <> struct is_bitmask_enum<Enum> : std::true_type {};                \
+  template <> struct largest_bitmask_enum_bit<Enum> {                          \
+    static constexpr std::underlying_type_t<Enum> value = LargestValue;        \
+  }
+
+/// Traits class to determine whether an enum has been declared as a bitwise
+/// enum via ORC_RT_DECLARE_ENUM_AS_BITMASK.
+template <typename E, typename Enable = void>
+struct is_bitmask_enum : std::false_type {};
+
+template <typename E>
+struct is_bitmask_enum<
+    E, std::enable_if_t<sizeof(E::ORC_RT_BITMASK_LARGEST_ENUMERATOR) >= 0>>
+    : std::true_type {};
+
+template <typename E>
+inline constexpr bool is_bitmask_enum_v = is_bitmask_enum<E>::value;
+
+/// Traits class to deermine bitmask enum largest bit.
+template <typename E, typename Enable = void> struct largest_bitmask_enum_bit;
+
+template <typename E>
+struct largest_bitmask_enum_bit<
+    E, std::enable_if_t<sizeof(E::ORC_RT_BITMASK_LARGEST_ENUMERATOR) >= 0>> {
+  using UnderlyingTy = std::underlying_type_t<E>;
+  static constexpr UnderlyingTy value =
+      static_cast<UnderlyingTy>(E::ORC_RT_BITMASK_LARGEST_ENUMERATOR);
+};
+
+template <typename E>
+constexpr std::underlying_type_t<E> bitmask_enum_mask() noexcept {
+  return nextPowerOf2(largest_bitmask_enum_bit<E>::value) - 1;
+}
+
+template <typename E>
+constexpr std::underlying_type_t<E> bitmask_enum_to_underlying(E Val) noexcept {
+  auto U = static_cast<std::underlying_type_t<E>>(Val);
+  assert(U >= 0 && "Negative enum values are not allowed");
+  assert(U <= bitmask_enum_mask<E>() &&
+         "Enum value too large (or langest val too small");
+  return U;
+}
+
+template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>>
+constexpr E operator~(E Val) noexcept {
+  return static_cast<E>(~bitmask_enum_to_underlying(Val) &
+                        bitmask_enum_mask<E>());
+}
+
+template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>>
+constexpr E operator|(E LHS, E RHS) noexcept {
+  return static_cast<E>(bitmask_enum_to_underlying(LHS) |
+                        bitmask_enum_to_underlying(RHS));
+}
+
+template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>>
+constexpr E operator&(E LHS, E RHS) noexcept {
+  return static_cast<E>(bitmask_enum_to_underlying(LHS) &
+                        bitmask_enum_to_underlying(RHS));
+}
+
+template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>>
+constexpr E operator^(E LHS, E RHS) noexcept {
+  return static_cast<E>(bitmask_enum_to_underlying(LHS) ^
+                        bitmask_enum_to_underlying(RHS));
+}
+
+template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>>
+constexpr E &operator|=(E &LHS, E RHS) noexcept {
+  LHS = LHS | RHS;
+  return LHS;
+}
+
+template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>>
+constexpr E &operator&=(E &LHS, E RHS) noexcept {
+  LHS = LHS & RHS;
+  return LHS;
+}
+
+template <typename E, typename = std::enable_if_t<is_bitmask_enum_v<E>>>
+constexpr E &operator^=(E &LHS, E RHS) noexcept {
+  LHS = LHS ^ RHS;
+  return LHS;
+}
+
+} // namespace orc_rt
+
+#endif // ORC_RT_BITMASKENUM_H
diff --git a/orc-rt/include/orc-rt/Compiler.h b/orc-rt/include/orc-rt/Compiler.h
new file mode 100644
index 0000000000000..e0c0c0989a1c7
--- /dev/null
+++ b/orc-rt/include/orc-rt/Compiler.h
@@ -0,0 +1,74 @@
+//===--------- Compiler.h - Compiler abstraction support --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+// Most functionality in this file was swiped from llvm/Support/Compiler.h.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_COMPILER_H
+#define ORC_RT_COMPILER_H
+
+#if defined(_WIN32)
+#define ORC_RT_INTERFACE extern "C"
+#define ORC_RT_HIDDEN
+#define ORC_RT_IMPORT extern "C" __declspec(dllimport)
+#else
+#define ORC_RT_INTERFACE extern "C" __attribute__((visibility("default")))
+#define ORC_RT_HIDDEN __attribute__((visibility("hidden")))
+#define ORC_RT_IMPORT extern "C"
+#endif
+
+#ifndef __has_builtin
+#define __has_builtin(x) 0
+#endif
+
+// Only use __has_cpp_attribute in C++ mode. GCC defines __has_cpp_attribute in
+// C mode, but the :: in __has_cpp_attribute(scoped::attribute) is invalid.
+#ifndef ORC_RT_HAS_CPP_ATTRIBUTE
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
+#define ORC_RT_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
+#else
+#define ORC_RT_HAS_CPP_ATTRIBUTE(x) 0
+#endif
+#endif
+
+// Use the 'nodiscard' attribute in C++17 or newer mode.
+#if defined(__cplusplus) && __cplusplus > 201402L &&                           \
+    ORC_RT_HAS_CPP_ATTRIBUTE(nodiscard)
+#define ORC_RT_NODISCARD [[nodiscard]]
+#elif ORC_RT_HAS_CPP_ATTRIBUTE(clang::warn_unused_result)
+#define ORC_RT_NODISCARD [[clang::warn_unused_result]]
+// Clang in C++14 mode claims that it has the 'nodiscard' attribute, but also
+// warns in the pedantic mode that 'nodiscard' is a C++17 extension (PR33518).
+// Use the 'nodiscard' attribute in C++14 mode only with GCC.
+// TODO: remove this workaround when PR33518 is resolved.
+#elif defined(__GNUC__) && ORC_RT_HAS_CPP_ATTRIBUTE(nodiscard)
+#define ORC_RT_NODISCARD [[nodiscard]]
+#else
+#define ORC_RT_NODISCARD
+#endif
+
+#if __has_builtin(__builtin_expect)
+#define ORC_RT_LIKELY(EXPR) __builtin_expect((bool)(EXPR), true)
+#define ORC_RT_UNLIKELY(EXPR) __builtin_expect((bool)(EXPR), false)
+#else
+#define ORC_RT_LIKELY(EXPR) (EXPR)
+#define ORC_RT_UNLIKELY(EXPR) (EXPR)
+#endif
+
+#if defined(__APPLE__)
+#define ORC_RT_WEAK_IMPORT __attribute__((weak_import))
+#elif defined(_WIN32)
+#define ORC_RT_WEAK_IMPORT
+#else
+#define ORC_RT_WEAK_IMPORT __attribute__((weak))
+#endif
+
+#endif // ORC_RT_COMPILER_H
diff --git a/orc-rt/include/orc-rt/Error.h b/orc-rt/include/orc-rt/Error.h
new file mode 100644
index 0000000000000..fe0754bd1103f
--- /dev/null
+++ b/orc-rt/include/orc-rt/Error.h
@@ -0,0 +1,539 @@
+//===-------- Error.h - Enforced error checking for ORC RT ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_ERROR_H
+#define ORC_RT_ERROR_H
+
+#include "orc-rt/Compiler.h"
+#include "orc-rt/RTTI.h"
+
+#include <cassert>
+#include <cstdlib>
+#include <memory>
+#include <string>
+#include <type_traits>
+
+namespace orc_rt {
+
+/// Base class for all errors.
+class ErrorInfoBase : public RTTIExtends<ErrorInfoBase, RTTIRoot> {
+public:
+  virtual std::string toString() const = 0;
+};
+
+/// Represents an environmental error.
+class ORC_RT_NODISCARD Error {
+
+  template <typename T> friend class Expected;
+
+  friend Error make_error(std::unique_ptr<ErrorInfoBase> Payload);
+
+  template <typename... HandlerTs>
+  friend Error handleErrors(Error E, HandlerTs &&...Hs);
+
+public:
+  /// Destroy this error. Aborts if error was not checked, or was checked but
+  /// not handled.
+  ~Error() { assertIsChecked(); }
+
+  Error(const Error &) = delete;
+  Error &operator=(const Error &) = delete;
+
+  /// Move-construct an error. The newly constructed error is considered
+  /// unchecked, even if the source error had been checked. The original error
+  /// becomes a checked success value.
+  Error(Error &&Other) {
+    setChecked(true);
+    *this = std::move(Other);
+  }
+
+  /// Move-assign an error value. The current error must represent success, you
+  /// you cannot overwrite an unhandled error. The current error is then
+  /// considered unchecked. The source error becomes a checked success value,
+  /// regardless of its original state.
+  Error &operator=(Error &&Other) {
+    // Don't allow overwriting of unchecked values.
+    assertIsChecked();
+    setPtr(Other.getPtr());
+
+    // This Error is unchecked, even if the source error was checked.
+    setChecked(false);
+
+    // Null out Other's payload and set its checked bit.
+    Other.setPtr(nullptr);
+    Other.setChecked(true);
+
+    return *this;
+  }
+
+  /// Create a success value.
+  static Error success() { return Error(); }
+
+  /// Error values convert to true for failure values, false otherwise.
+  explicit operator bool() {
+    setChecked(getPtr() == nullptr);
+    return getPtr() != nullptr;
+  }
+
+  /// Return true if this Error contains a failure value of the given type.
+  template <typename ErrT> bool isA() const {
+    return getPtr() && getPtr()->isA<ErrT>();
+  }
+
+private:
+  Error() = default;
+
+  Error(std::unique_ptr<ErrorInfoBase> ErrInfo) {
+    auto RawErrPtr = reinterpret_cast<uintptr_t>(ErrInfo.release());
+    assert((RawErrPtr & 0x1) == 0 && "ErrorInfo is insufficiently aligned");
+    ErrPtr = RawErrPtr | 0x1;
+  }
+
+  void assertIsChecked() {
+    if (ORC_RT_UNLIKELY(!isChecked() || getPtr())) {
+      fprintf(stderr, "Error must be checked prior to destruction.\n");
+      abort(); // Some sort of JIT program abort?
+    }
+  }
+
+  template <typename ErrT = ErrorInfoBase> ErrT *getPtr() const {
+    return reinterpret_cast<ErrT *>(ErrPtr & ~uintptr_t(1));
+  }
+
+  void setPtr(ErrorInfoBase *Ptr) {
+    ErrPtr = (reinterpret_cast<uintptr_t>(Ptr) & ~uintptr_t(1)) | (ErrPtr & 1);
+  }
+
+  bool isChecked() const { return ErrPtr & 0x1; }
+
+  void setChecked(bool Checked) { ErrPtr = (ErrPtr & ~uintptr_t(1)) | Checked; }
+
+  template <typename ErrT = ErrorInfoBase> std::unique_ptr<ErrT> takePayload() {
+    static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value,
+                  "ErrT is not an ErrorInfoBase subclass");
+    std::unique_ptr<ErrT> Tmp(getPtr<ErrT>());
+    setPtr(nullptr);
+    setChecked(true);
+    return Tmp;
+  }
+
+  uintptr_t ErrPtr = 0;
+};
+
+/// Create an Error from an ErrorInfoBase.
+inline Error make_error(std::unique_ptr<ErrorInfoBase> Payload) {
+  return Error(std::move(Payload));
+}
+
+/// Construct an error of ErrT with the given arguments.
+template <typename ErrT, typename... ArgTs> Error make_error(ArgTs &&...Args) {
+  static_assert(std::is_base_of<ErrorInfoBase, ErrT>::value,
+                "ErrT is not an ErrorInfoBase subclass");
+  return make_error(std::make_unique<ErrT>(std::forward<ArgTs>(Args)...));
+}
+
+/// Traits class for selecting and applying error handlers.
+template <typename HandlerT>
+struct ErrorHandlerTraits
+    : public ErrorHandlerTraits<
+          decltype(&std::remove_reference_t<HandlerT>::operator())> {};
+
+// Specialization functions of the form 'Error (const ErrT&)'.
+template <typename ErrT> struct ErrorHandlerTraits<Error (&)(ErrT &)> {
+  static bool appliesTo(const ErrorInfoBase &E) {
+    return E.template isA<ErrT>();
+  }
+
+  template <typename HandlerT>
+  static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
+    assert(appliesTo(*E) && "Applying incorrect handler");
+    return H(static_cast<ErrT &>(*E));
+  }
+};
+
+// Specialization functions of the form 'void (const ErrT&)'.
+template <typename ErrT> struct ErrorHandlerTraits<void (&)(ErrT &)> {
+  static bool appliesTo(const ErrorInfoBase &E) {
+    return E.template isA<ErrT>();
+  }
+
+  template <typename HandlerT>
+  static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
+    assert(appliesTo(*E) && "Applying incorrect handler");
+    H(static_cast<ErrT &>(*E));
+    return Error::success();
+  }
+};
+
+/// Specialization for functions of the form 'Error (std::unique_ptr<ErrT>)'.
+template <typename ErrT>
+struct ErrorHandlerTraits<Error (&)(std::unique_ptr<ErrT>)> {
+  static bool appliesTo(const ErrorInfoBase &E) {
+    return E.template isA<ErrT>();
+  }
+
+  template <typename HandlerT>
+  static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
+    assert(appliesTo(*E) && "Applying incorrect handler");
+    std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release()));
+    return H(std::move(SubE));
+  }
+};
+
+/// Specialization for functions of the form 'void (std::unique_ptr<ErrT>)'.
+template <typename ErrT>
+struct ErrorHandlerTraits<void (&)(std::unique_ptr<ErrT>)> {
+  static bool appliesTo(const ErrorInfoBase &E) {
+    return E.template isA<ErrT>();
+  }
+
+  template <typename HandlerT>
+  static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
+    assert(appliesTo(*E) && "Applying incorrect handler");
+    std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release()));
+    H(std::move(SubE));
+    return Error::success();
+  }
+};
+
+// Specialization for member functions of the form 'RetT (const ErrT&)'.
+template <typename C, typename RetT, typename ErrT>
+class ErrorHandlerTraits<RetT (C::*)(ErrT &)>
+    : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
+
+// Specialization for member functions of the form 'RetT (const ErrT&) const'.
+template <typename C, typename RetT, typename ErrT>
+class ErrorHandlerTraits<RetT (C::*)(ErrT &) const>
+    : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
+
+// Specialization for member functions of the form 'RetT (const ErrT&)'.
+template <typename C, typename RetT, typename ErrT>
+class ErrorHandlerTraits<RetT (C::*)(const ErrT &)>
+    : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
+
+// Specialization for member functions of the form 'RetT (const ErrT&) const'.
+template <typename C, typename RetT, typename ErrT>
+class ErrorHandlerTraits<RetT (C::*)(const ErrT &) const>
+    : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
+
+/// Specialization for member functions of the form
+/// 'RetT (std::unique_ptr<ErrT>)'.
+template <typename C, typename RetT, typename ErrT>
+class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>)>
+    : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {};
+
+/// Specialization for member functions of the form
+/// 'RetT (std::unique_ptr<ErrT>) const'.
+template <typename C, typename RetT, typename ErrT>
+class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>) const>
+    : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {};
+
+inline Error handleErrorsImpl(std::unique_ptr<ErrorInfoBase> Payload) {
+  return make_error(std::move(Payload));
+}
+
+template <typename HandlerT, typename... HandlerTs>
+Error handleErrorsImpl(std::unique_ptr<ErrorInfoBase> Payload,
+                       HandlerT &&Handler, HandlerTs &&...Handlers) {
+  if (ErrorHandlerTraits<HandlerT>::appliesTo(*Payload))
+    return ErrorHandlerTraits<HandlerT>::apply(std::forward<HandlerT>(Handler),
+                                               std::move(Payload));
+  return handleErrorsImpl(std::move(Payload),
+                          std::forward<HandlerTs>(Handlers)...);
+}
+
+/// Pass the ErrorInfo(s) contained in E to their respective handlers. Any
+/// unhandled errors (or Errors returned by handlers) are re-concatenated and
+/// returned.
+/// Because this function returns an error, its result must also be checked
+/// or returned. If you intend to handle all errors use handleAllErrors
+/// (which returns void, and will abort() on unhandled errors) instead.
+template <typename... HandlerTs>
+Error handleErrors(Error E, HandlerTs &&...Hs) {
+  if (!E)
+    return Error::success();
+  return handleErrorsImpl(E.takePayload(), std::forward<HandlerTs>(Hs)...);
+}
+
+/// Behaves the same as handleErrors, except that by contract all errors
+/// *must* be handled by the given handlers (i.e. there must be no remaining
+/// errors after running the handlers, or llvm_unreachable is called).
+template <typename... HandlerTs>
+void handleAllErrors(Error E, HandlerTs &&...Handlers) {
+  cantFail(handleErrors(std::move(E), std::forward<HandlerTs>(Handlers)...));
+}
+
+/// Helper for Errors used as out-parameters.
+/// Sets the 'checked' flag on construction, resets it on destruction.
+class ErrorAsOutParameter {
+public:
+  ErrorAsOutParameter(Error *Err) : Err(Err) {
+    // Raise the checked bit if Err is success.
+    if (Err)
+      (void)!!*Err;
+  }
+
+  ~ErrorAsOutParameter() {
+    // Clear the checked bit.
+    if (Err && !*Err)
+      *Err = Error::success();
+  }
+
+private:
+  Error *Err;
+};
+
+template <typename T> class ORC_RT_NODISCARD Expected {
+
+  template <class OtherT> friend class Expected;
+
+  static constexpr bool IsRef = std::is_reference<T>::value;
+  using wrap = std::reference_wrapper<std::remove_reference_t<T>>;
+  using error_type = std::unique_ptr<ErrorInfoBase>;
+  using storage_type = std::conditional_t<IsRef, wrap, T>;
+  using value_type = T;
+
+  using reference = std::remove_reference_t<T> &;
+  using const_reference = const std::remove_reference_t<T> &;
+  using pointer = std::remove_reference_t<T> *;
+  using const_pointer = const std::remove_reference_t<T> *;
+
+public:
+  /// Create an Expected from a failure value.
+  Expected(Error Err) : HasError(true), Unchecked(true) {
+    assert(Err && "Cannot create Expected<T> from Error success value");
+    new (getErrorStorage()) error_type(Err.takePayload());
+  }
+
+  /// Create an Expected from a T value.
+  template <typename OtherT>
+  Expected(OtherT &&Val,
+           std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr)
+      : HasError(false), Unchecked(true) {
+    new (getStorage()) storage_type(std::forward<OtherT>(Val));
+  }
+
+  /// Move-construct an Expected<T> from an Expected<OtherT>.
+  Expected(Expected &&Other) { moveConstruct(std::move(Other)); }
+
+  /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT
+  /// must be convertible to T.
+  template <class OtherT>
+  Expected(
+      Expected<OtherT> &&Other,
+      std::enable_if_t<std::is_convertible<OtherT, T>::value> * = nullptr) {
+    moveConstruct(std::move(Other));
+  }
+
+  /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT
+  /// isn't convertible to T.
+  template <class OtherT>
+  explicit Expected(
+      Expected<OtherT> &&Other,
+      std::enable_if_t<!std::is_convertible<OtherT, T>::value> * = nullptr) {
+    moveConstruct(std::move(Other));
+  }
+
+  /// Move-assign from another Expected<T>.
+  Expected &operator=(Expected &&Other) {
+    moveAssign(std::move(Other));
+    return *this;
+  }
+
+  /// Destroy an Expected<T>.
+  ~Expected() {
+    assertIsChecked();
+    if (!HasError)
+      getStorage()->~storage_type();
+    else
+      getErrorStorage()->~error_type();
+  }
+
+  /// Returns true if this Expected value is in a success state (holding a T),
+  /// and false if this Expected value is in a failure state.
+  explicit operator bool() {
+    Unchecked = HasError;
+    return !HasError;
+  }
+
+  /// Returns true if this Expected value holds an Error of type error_type.
+  template <typename ErrT> bool isFailureOfType() const {
+    return HasError && (*getErrorStorage())->template isFailureOfType<ErrT>();
+  }
+
+  /// Take ownership of the stored error.
+  ///
+  /// If this Expected value is in a success state (holding a T) then this
+  /// method is a no-op and returns Error::success.
+  ///
+  /// If thsi Expected value is in a failure state (holding an Error) then this
+  /// method returns the contained error and leaves this Expected in an
+  /// 'empty' state from which it may be safely destructed but not otherwise
+  /// accessed.
+  Error takeError() {
+    Unchecked = false;
+    return HasError ? Error(std::move(*getErrorStorage())) : Error::success();
+  }
+
+  /// Returns a pointer to the stored T value.
+  pointer operator->() {
+    assertIsChecked();
+    return toPointer(getStorage());
+  }
+
+  /// Returns a pointer to the stored T value.
+  const_pointer operator->() const {
+    assertIsChecked();
+    return toPointer(getStorage());
+  }
+
+  /// Returns a reference to the stored T value.
+  reference operator*() {
+    assertIsChecked();
+    return *getStorage();
+  }
+
+  /// Returns a reference to the stored T value.
+  const_reference operator*() const {
+    assertIsChecked();
+    return *getStorage();
+  }
+
+private:
+  template <class T1>
+  static bool compareThisIfSameType(const T1 &a, const T1 &b) {
+    return &a == &b;
+  }
+
+  template <class T1, class T2>
+  static bool compareThisIfSameType(const T1 &a, const T2 &b) {
+    return false;
+  }
+
+  template <class OtherT> void moveConstruct(Expected<OtherT> &&Other) {
+    HasError = Other.HasError;
+    Unchecked = true;
+    Other.Unchecked = false;
+
+    if (!HasError)
+      new (getStorage()) storage_type(std::move(*Other.getStorage()));
+    else
+      new (getErrorStorage()) error_type(std::move(*Other.getErrorStorage()));
+  }
+
+  template <class OtherT> void moveAssign(Expected<OtherT> &&Other) {
+    assertIsChecked();
+
+    if (compareThisIfSameType(*this, Other))
+      return;
+
+    this->~Expected();
+    new (this) Expected(std::move(Other));
+  }
+
+  pointer toPointer(pointer Val) { return Val; }
+
+  const_pointer toPointer(const_pointer Val) const { return Val; }
+
+  pointer toPointer(wrap *Val) { return &Val->get(); }
+
+  const_pointer toPointer(const wrap *Val) const { return &Val->get(); }
+
+  storage_type *getStorage() {
+    assert(!HasError && "Cannot get value when an error exists!");
+    return reinterpret_cast<storage_type *>(&TStorage);
+  }
+
+  const storage_type *getStorage() const {
+    assert(!HasError && "Cannot get value when an error exists!");
+    return reinterpret_cast<const storage_type *>(&TStorage);
+  }
+
+  error_type *getErrorStorage() {
+    assert(HasError && "Cannot get error when a value exists!");
+    return reinterpret_cast<error_type *>(&ErrorStorage);
+  }
+
+  const error_type *getErrorStorage() const {
+    assert(HasError && "Cannot get error when a value exists!");
+    return reinterpret_cast<const error_type *>(&ErrorStorage);
+  }
+
+  void assertIsChecked() {
+    if (ORC_RT_UNLIKELY(Unchecked)) {
+      fprintf(stderr,
+              "Expected<T> must be checked before access or destruction.\n");
+      abort();
+    }
+  }
+
+  union {
+    alignas(storage_type) char TStorage[sizeof(storage_type)];
+    alignas(error_type) char ErrorStorage[sizeof(error_type)];
+  };
+
+  bool HasError : 1;
+  bool Unchecked : 1;
+};
+
+/// Consume an error without doing anything.
+inline void consumeError(Error Err) {
+  handleAllErrors(std::move(Err), [](const ErrorInfoBase &) {});
+}
+
+/// Consumes success values. It is a programmatic error to call this function
+/// on a failure value.
+inline void cantFail(Error Err) {
+#ifndef NDEBUG
+  // TODO: Log unhandled error.
+  if (Err)
+    abort();
+#else
+  Err.operator bool(); // Reset checked flag.
+#endif
+}
+
+/// Auto-unwrap an Expected<T> value in the success state. It is a programmatic
+/// error to call this function on a failure value.
+template <typename T> T cantFail(Expected<T> E) {
+  assert(E && "cantFail called on failure value");
+  consumeError(E.takeError());
+  return std::move(*E);
+}
+
+/// Auto-unwrap an Expected<T> value in the success state. It is a programmatic
+/// error to call this function on a failure value.
+template <typename T> T &cantFail(Expected<T &> E) {
+  assert(E && "cantFail called on failure value");
+  consumeError(E.takeError());
+  return *E;
+}
+
+/// Convert the given error to a string. The error value is consumed in the
+/// process.
+inline std::string toString(Error Err) {
+  assert(Err && "Cannot convert success value to string");
+  std::string ErrMsg;
+  handleAllErrors(std::move(Err),
+                  [&](const ErrorInfoBase &EIB) { ErrMsg = EIB.toString(); });
+  return ErrMsg;
+}
+
+/// Simple string error type.
+class StringError : public RTTIExtends<StringError, ErrorInfoBase> {
+public:
+  StringError(std::string ErrMsg) : ErrMsg(std::move(ErrMsg)) {}
+  std::string toString() const override { return ErrMsg; }
+
+private:
+  std::string ErrMsg;
+};
+
+} // namespace orc_rt
+
+#endif // ORC_RT_ERROR_H
diff --git a/orc-rt/include/orc-rt/ExecutorAddress.h b/orc-rt/include/orc-rt/ExecutorAddress.h
new file mode 100644
index 0000000000000..cc7bbf53ac51a
--- /dev/null
+++ b/orc-rt/include/orc-rt/ExecutorAddress.h
@@ -0,0 +1,230 @@
+//===------ ExecutorAddress.h - Executing process address -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Utilites for representing addresses and address ranges in the executing
+// program that can be shared with an ORC controller.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_EXECUTORADDRESS_H
+#define ORC_RT_EXECUTORADDRESS_H
+
+#include "span.h"
+
+#include <cassert>
+#include <cstdint>
+#include <functional>
+#include <type_traits>
+
+namespace orc_rt {
+
+using ExecutorAddrDiff = uint64_t;
+
+/// Represents an address in the executor process.
+class ExecutorAddr {
+public:
+  /// Return pointer unmodified.
+  template <typename T> struct rawPtr {
+    T *operator()(T *p) const { return p; }
+  };
+
+  /// Default wrap function to use on this host.
+  template <typename T> using defaultWrap = rawPtr<T>;
+
+  /// Default unwrap function to use on this host.
+  template <typename T> using defaultUnwrap = rawPtr<T>;
+
+  /// Merges a tag into the raw address value:
+  ///   P' = P | (TagValue << TagOffset).
+  class Tag {
+  public:
+    constexpr Tag(uintptr_t TagValue, uintptr_t TagOffset)
+        : TagMask(TagValue << TagOffset) {}
+
+    template <typename T> constexpr T *operator()(T *P) {
+      return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(P) | TagMask);
+    }
+
+  private:
+    uintptr_t TagMask;
+  };
+
+  /// Strips a tag of the given length from the given offset within the pointer:
+  /// P' = P & ~(((1 << TagLen) -1) << TagOffset)
+  class Untag {
+  public:
+    constexpr Untag(uintptr_t TagLen, uintptr_t TagOffset)
+        : UntagMask(~(((uintptr_t(1) << TagLen) - 1) << TagOffset)) {}
+
+    template <typename T> constexpr T *operator()(T *P) {
+      return reinterpret_cast<T *>(reinterpret_cast<uintptr_t>(P) & UntagMask);
+    }
+
+  private:
+    uintptr_t UntagMask;
+  };
+
+  constexpr ExecutorAddr() noexcept = default;
+  explicit constexpr ExecutorAddr(uint64_t Addr) noexcept : Addr(Addr) {}
+
+  /// Create an ExecutorAddr from the given pointer.
+  template <typename T, typename UnwrapFn = defaultUnwrap<T>>
+  static constexpr ExecutorAddr fromPtr(T *Ptr,
+                                        UnwrapFn &&Unwrap = UnwrapFn()) {
+    return ExecutorAddr(
+        static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Unwrap(Ptr))));
+  }
+
+  /// Cast this ExecutorAddr to a pointer of the given type.
+  template <typename T, typename WrapFn = defaultWrap<std::remove_pointer_t<T>>>
+  constexpr std::enable_if_t<std::is_pointer<T>::value, T>
+  toPtr(WrapFn &&Wrap = WrapFn()) const {
+    uintptr_t IntPtr = static_cast<uintptr_t>(Addr);
+    assert(IntPtr == Addr && "ExecutorAddr value out of range for uintptr_t");
+    return Wrap(reinterpret_cast<T>(IntPtr));
+  }
+
+  /// Cast this ExecutorAddr to a pointer of the given function type.
+  template <typename T, typename WrapFn = defaultWrap<T>>
+  constexpr std::enable_if_t<std::is_function<T>::value, T *>
+  toPtr(WrapFn &&Wrap = WrapFn()) const {
+    uintptr_t IntPtr = static_cast<uintptr_t>(Addr);
+    assert(IntPtr == Addr && "ExecutorAddr value out of range for uintptr_t");
+    return Wrap(reinterpret_cast<T *>(IntPtr));
+  }
+
+  constexpr uint64_t getValue() const noexcept { return Addr; }
+  constexpr void setValue(uint64_t Addr) noexcept { this->Addr = Addr; }
+  constexpr bool isNull() const noexcept { return Addr == 0; }
+
+  constexpr explicit operator bool() const noexcept { return Addr != 0; }
+
+  friend constexpr bool operator==(const ExecutorAddr &LHS,
+                                   const ExecutorAddr &RHS) noexcept {
+    return LHS.Addr == RHS.Addr;
+  }
+
+  friend constexpr bool operator!=(const ExecutorAddr &LHS,
+                                   const ExecutorAddr &RHS) noexcept {
+    return LHS.Addr != RHS.Addr;
+  }
+
+  friend constexpr bool operator<(const ExecutorAddr &LHS,
+                                  const ExecutorAddr &RHS) noexcept {
+    return LHS.Addr < RHS.Addr;
+  }
+
+  friend constexpr bool operator<=(const ExecutorAddr &LHS,
+                                   const ExecutorAddr &RHS) noexcept {
+    return LHS.Addr <= RHS.Addr;
+  }
+
+  friend constexpr bool operator>(const ExecutorAddr &LHS,
+                                  const ExecutorAddr &RHS) noexcept {
+    return LHS.Addr > RHS.Addr;
+  }
+
+  friend constexpr bool operator>=(const ExecutorAddr &LHS,
+                                   const ExecutorAddr &RHS) noexcept {
+    return LHS.Addr >= RHS.Addr;
+  }
+
+  constexpr ExecutorAddr &operator++() noexcept {
+    ++Addr;
+    return *this;
+  }
+  constexpr ExecutorAddr &operator--() noexcept {
+    --Addr;
+    return *this;
+  }
+  constexpr ExecutorAddr operator++(int) noexcept {
+    return ExecutorAddr(Addr++);
+  }
+  constexpr ExecutorAddr operator--(int) noexcept {
+    return ExecutorAddr(Addr++);
+  }
+
+  constexpr ExecutorAddr &operator+=(const ExecutorAddrDiff Delta) noexcept {
+    Addr += Delta;
+    return *this;
+  }
+
+  constexpr ExecutorAddr &operator-=(const ExecutorAddrDiff Delta) noexcept {
+    Addr -= Delta;
+    return *this;
+  }
+
+private:
+  uint64_t Addr = 0;
+};
+
+/// Subtracting two addresses yields an offset.
+inline constexpr ExecutorAddrDiff operator-(const ExecutorAddr &LHS,
+                                            const ExecutorAddr &RHS) noexcept {
+  return ExecutorAddrDiff(LHS.getValue() - RHS.getValue());
+}
+
+/// Adding an offset and an address yields an address.
+inline constexpr ExecutorAddr operator+(const ExecutorAddr &LHS,
+                                        const ExecutorAddrDiff &RHS) noexcept {
+  return ExecutorAddr(LHS.getValue() + RHS);
+}
+
+/// Adding an address and an offset yields an address.
+inline constexpr ExecutorAddr operator+(const ExecutorAddrDiff &LHS,
+                                        const ExecutorAddr &RHS) noexcept {
+  return ExecutorAddr(LHS + RHS.getValue());
+}
+
+/// Represents an address range in the exceutor process.
+struct ExecutorAddrRange {
+  constexpr ExecutorAddrRange() noexcept = default;
+  constexpr ExecutorAddrRange(ExecutorAddr Start, ExecutorAddr End) noexcept
+      : Start(Start), End(End) {}
+  constexpr ExecutorAddrRange(ExecutorAddr Start,
+                              ExecutorAddrDiff Size) noexcept
+      : Start(Start), End(Start + Size) {}
+
+  constexpr bool empty() const noexcept { return Start == End; }
+  constexpr ExecutorAddrDiff size() const noexcept { return End - Start; }
+
+  friend constexpr bool operator==(const ExecutorAddrRange &LHS,
+                                   const ExecutorAddrRange &RHS) noexcept {
+    return LHS.Start == RHS.Start && LHS.End == RHS.End;
+  }
+  friend constexpr bool operator!=(const ExecutorAddrRange &LHS,
+                                   const ExecutorAddrRange &RHS) noexcept {
+    return !(LHS == RHS);
+  }
+  constexpr bool contains(ExecutorAddr Addr) const noexcept {
+    return Start <= Addr && Addr < End;
+  }
+  constexpr bool overlaps(const ExecutorAddrRange &Other) const noexcept {
+    return !(Other.End <= Start || End <= Other.Start);
+  }
+
+  template <typename T> constexpr span<T> toSpan() const noexcept {
+    assert(size() % sizeof(T) == 0 &&
+           "AddressRange is not a multiple of sizeof(T)");
+    return span<T>(Start.toPtr<T *>(), size() / sizeof(T));
+  }
+
+  ExecutorAddr Start;
+  ExecutorAddr End;
+};
+
+} // namespace orc_rt
+
+// Make ExecutorAddr hashable.
+template <> struct std::hash<orc_rt::ExecutorAddr> {
+  constexpr size_t operator()(const orc_rt::ExecutorAddr &A) const noexcept {
+    return std::hash<uint64_t>()(A.getValue());
+  }
+};
+
+#endif // ORC_RT_EXECUTORADDRESS_H
diff --git a/orc-rt/include/orc-rt/IntervalMap.h b/orc-rt/include/orc-rt/IntervalMap.h
new file mode 100644
index 0000000000000..ca16ed7cd534a
--- /dev/null
+++ b/orc-rt/include/orc-rt/IntervalMap.h
@@ -0,0 +1,167 @@
+//===---------- IntervalMap.h - A sorted interval map -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements a coalescing interval map.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_INTERVALMAP_H
+#define ORC_RT_INTERVALMAP_H
+
+#include <cassert>
+#include <map>
+
+namespace orc_rt {
+
+enum class IntervalCoalescing { Enabled, Disabled };
+
+/// Maps intervals to keys with optional coalescing.
+///
+/// NOTE: The interface is kept mostly compatible with LLVM's IntervalMap
+///       collection to make it easy to swap over in the future if we choose
+///       to.
+template <typename KeyT, typename ValT> class IntervalMapBase {
+private:
+  using KeyPairT = std::pair<KeyT, KeyT>;
+
+  struct Compare {
+    using is_transparent = std::true_type;
+    bool operator()(const KeyPairT &LHS, const KeyPairT &RHS) const {
+      return LHS < RHS;
+    }
+    bool operator()(const KeyPairT &LHS, const KeyT &RHS) const {
+      return LHS.first < RHS;
+    }
+    bool operator()(const KeyT &LHS, const KeyPairT &RHS) const {
+      return LHS < RHS.first;
+    }
+  };
+
+  using ImplMap = std::map<KeyPairT, ValT, Compare>;
+
+public:
+  using iterator = typename ImplMap::iterator;
+  using const_iterator = typename ImplMap::const_iterator;
+  using size_type = typename ImplMap::size_type;
+
+  bool empty() const { return Impl.empty(); }
+
+  void clear() { Impl.clear(); }
+
+  iterator begin() { return Impl.begin(); }
+  iterator end() { return Impl.end(); }
+
+  const_iterator begin() const { return Impl.begin(); }
+  const_iterator end() const { return Impl.end(); }
+
+  iterator find(KeyT K) {
+    // Early out if the key is clearly outside the range.
+    if (empty() || K < begin()->first.first ||
+        K >= std::prev(end())->first.second)
+      return end();
+
+    auto I = Impl.upper_bound(K);
+    assert(I != begin() && "Should have hit early out above");
+    I = std::prev(I);
+    if (K < I->first.second)
+      return I;
+    return end();
+  }
+
+  const_iterator find(KeyT K) const {
+    return const_cast<IntervalMapBase<KeyT, ValT> *>(this)->find(K);
+  }
+
+  ValT lookup(KeyT K, ValT NotFound = ValT()) const {
+    auto I = find(K);
+    if (I == end())
+      return NotFound;
+    return I->second;
+  }
+
+  // Erase [KS, KE), which must be entirely containing within one existing
+  // range in the map. Removal is allowed to split the range.
+  void erase(KeyT KS, KeyT KE) {
+    if (empty())
+      return;
+
+    auto J = Impl.upper_bound(KS);
+
+    // Check previous range. Bail out if range to remove is entirely after
+    // it.
+    auto I = std::prev(J);
+    if (KS >= I->first.second)
+      return;
+
+    // Assert that range is wholly contained.
+    assert(KE <= I->first.second);
+
+    auto Tmp = std::move(*I);
+    Impl.erase(I);
+
+    // Split-right -- introduce right-split range.
+    if (KE < Tmp.first.second) {
+      Impl.insert(
+          J, std::make_pair(std::make_pair(KE, Tmp.first.second), Tmp.second));
+      J = std::prev(J);
+    }
+
+    // Split-left -- introduce left-split range.
+    if (KS > Tmp.first.first)
+      Impl.insert(
+          J, std::make_pair(std::make_pair(Tmp.first.first, KS), Tmp.second));
+  }
+
+protected:
+  ImplMap Impl;
+};
+
+template <typename KeyT, typename ValT, IntervalCoalescing Coalescing>
+class IntervalMap;
+
+template <typename KeyT, typename ValT>
+class IntervalMap<KeyT, ValT, IntervalCoalescing::Enabled>
+    : public IntervalMapBase<KeyT, ValT> {
+public:
+  // Coalescing insert. Requires that ValTs be equality-comparable.
+  void insert(KeyT KS, KeyT KE, ValT V) {
+    auto J = this->Impl.upper_bound(KS);
+
+    // Coalesce-right if possible. Either way, J points at our insertion
+    // point.
+    if (J != this->end() && KE == J->first.first && J->second == V) {
+      KE = J->first.second;
+      auto Tmp = J++;
+      this->Impl.erase(Tmp);
+    }
+
+    // Coalesce-left if possible.
+    if (J != this->begin()) {
+      auto I = std::prev(J);
+      if (I->first.second == KS && I->second == V) {
+        KS = I->first.first;
+        this->Impl.erase(I);
+      }
+    }
+    this->Impl.insert(J, std::make_pair(std::make_pair(KS, KE), std::move(V)));
+  }
+};
+
+template <typename KeyT, typename ValT>
+class IntervalMap<KeyT, ValT, IntervalCoalescing::Disabled>
+    : public IntervalMapBase<KeyT, ValT> {
+public:
+  // Non-coalescing insert. Does not require ValT to be equality-comparable.
+  void insert(KeyT KS, KeyT KE, ValT V) {
+    this->Impl.insert(std::make_pair(std::make_pair(KS, KE), std::move(V)));
+  }
+};
+
+} // End namespace orc_rt
+
+#endif // ORC_RT_INTERVALMAP_H
diff --git a/orc-rt/include/orc-rt/IntervalSet.h b/orc-rt/include/orc-rt/IntervalSet.h
new file mode 100644
index 0000000000000..3a204cff114f0
--- /dev/null
+++ b/orc-rt/include/orc-rt/IntervalSet.h
@@ -0,0 +1,93 @@
+//===---------- IntervalSet.h - A sorted interval set -----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements a coalescing interval set.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_INTERVALSET_H
+#define ORC_RT_INTERVALSET_H
+
+#include "IntervalMap.h"
+
+#include <variant> // For std::monostate.
+
+namespace orc_rt {
+
+/// Implements a coalescing interval set.
+///
+/// Adjacent intervals are coalesced.
+///
+/// NOTE: The interface is kept mostly compatible with LLVM's IntervalMap
+///       collection to make it easy to swap over in the future if we choose
+///       to.
+template <typename KeyT, IntervalCoalescing Coalescing> class IntervalSet {
+private:
+  using ImplMap = IntervalMap<KeyT, std::monostate, Coalescing>;
+
+public:
+  using value_type = std::pair<KeyT, KeyT>;
+
+  class const_iterator {
+    friend class IntervalSet;
+
+  public:
+    using difference_type = typename ImplMap::iterator::difference_type;
+    using value_type = IntervalSet::value_type;
+    using pointer = const value_type *;
+    using reference = const value_type &;
+    using iterator_category = std::input_iterator_tag;
+
+    const_iterator() = default;
+    const value_type &operator*() const { return I->first; }
+    const value_type *operator->() const { return &I->first; }
+    const_iterator &operator++() {
+      ++I;
+      return *this;
+    }
+    const_iterator operator++(int) {
+      auto Tmp = I;
+      ++I;
+      return Tmp;
+    }
+    friend bool operator==(const const_iterator &LHS,
+                           const const_iterator &RHS) {
+      return LHS.I == RHS.I;
+    }
+    friend bool operator!=(const const_iterator &LHS,
+                           const const_iterator &RHS) {
+      return LHS.I != RHS.I;
+    }
+
+  private:
+    const_iterator(typename ImplMap::const_iterator I) : I(std::move(I)) {}
+    typename ImplMap::const_iterator I;
+  };
+
+  bool empty() const { return Map.empty(); }
+
+  void clear() { Map.clear(); }
+
+  const_iterator begin() const { return const_iterator(Map.begin()); }
+  const_iterator end() const { return const_iterator(Map.end()); }
+
+  const_iterator find(KeyT K) const { return const_iterator(Map.find(K)); }
+
+  void insert(KeyT KS, KeyT KE) {
+    Map.insert(std::move(KS), std::move(KE), std::monostate());
+  }
+
+  void erase(KeyT KS, KeyT KE) { Map.erase(KS, KE); }
+
+private:
+  ImplMap Map;
+};
+
+} // End namespace orc_rt
+
+#endif // ORC_RT_INTERVALSET_H
diff --git a/orc-rt/include/orc-rt/Math.h b/orc-rt/include/orc-rt/Math.h
new file mode 100644
index 0000000000000..7d5dfbe7e0763
--- /dev/null
+++ b/orc-rt/include/orc-rt/Math.h
@@ -0,0 +1,35 @@
+//===--------- Math.h - Math helpers for the ORC runtime --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Math helper functions for the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_MATH_H
+#define ORC_RT_MATH_H
+
+#include <cstdint>
+#include <limits>
+
+namespace orc_rt {
+
+/// Test whether the given value is a power of 2.
+template <typename T> constexpr bool isPowerOf2(T Val) noexcept {
+  return Val != 0 && (Val & (Val - 1)) == 0;
+}
+
+/// Calculates the next power of 2.
+template <typename T> constexpr T nextPowerOf2(T Val) noexcept {
+  for (std::size_t I = 1; I < std::numeric_limits<T>::digits; I <<= 1)
+    Val |= (Val >> I);
+  return Val + 1;
+}
+
+} // namespace orc_rt
+
+#endif // ORC_RT_MATH_H
diff --git a/orc-rt/include/orc-rt/RTTI.h b/orc-rt/include/orc-rt/RTTI.h
new file mode 100644
index 0000000000000..734205200cc6e
--- /dev/null
+++ b/orc-rt/include/orc-rt/RTTI.h
@@ -0,0 +1,138 @@
+//===------------- RTTI.h - RTTI support for ORC RT -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// \file
+//
+// Provides an extensible RTTI mechanism, that can be used regardless of whether
+// the runtime is built with -frtti or not. This is predominantly used to
+// support error handling.
+//
+// The RTTIRoot class defines methods for comparing type ids. Implementations
+// of these methods can be injected into new classes using the RTTIExtends
+// class template.
+//
+// E.g.
+//
+//   @code{.cpp}
+//   class MyBaseClass : public RTTIExtends<MyBaseClass, RTTIRoot> {
+//   public:
+//     virtual void foo() = 0;
+//   };
+//
+//   class MyDerivedClass1 : public RTTIExtends<MyDerivedClass1, MyBaseClass> {
+//   public:
+//     void foo() override {}
+//   };
+//
+//   class MyDerivedClass2 : public RTTIExtends<MyDerivedClass2, MyBaseClass> {
+//   public:
+//     void foo() override {}
+//   };
+//
+//   void fn() {
+//     std::unique_ptr<MyBaseClass> B = std::make_unique<MyDerivedClass1>();
+//     outs() << isa<MyBaseClass>(B) << "\n"; // Outputs "1".
+//     outs() << isa<MyDerivedClass1>(B) << "\n"; // Outputs "1".
+//     outs() << isa<MyDerivedClass2>(B) << "\n"; // Outputs "0'.
+//   }
+//
+//   @endcode
+//
+// Note:
+//   This header was adapted from llvm/Support/ExtensibleRTTI.h, however the
+// data structures are not shared and the code need not be kept in sync.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_RTTI_H
+#define ORC_RT_RTTI_H
+
+namespace orc_rt {
+
+template <typename ThisT, typename ParentT> class RTTIExtends;
+
+/// Base class for the extensible RTTI hierarchy.
+///
+/// This class defines virtual methods, dynamicClassID and isA, that enable
+/// type comparisons.
+class RTTIRoot {
+public:
+  virtual ~RTTIRoot() = default;
+
+  /// Returns the class ID for this type.
+  static const void *classID() noexcept { return &ID; }
+
+  /// Returns the class ID for the dynamic type of this RTTIRoot instance.
+  virtual const void *dynamicClassID() const noexcept = 0;
+
+  /// Returns true if this class's ID matches the given class ID.
+  virtual bool isA(const void *const ClassID) const noexcept {
+    return ClassID == classID();
+  }
+
+  /// Check whether this instance is a subclass of QueryT.
+  template <typename QueryT> bool isA() const noexcept {
+    return isA(QueryT::classID());
+  }
+
+  static bool classof(const RTTIRoot *R) noexcept { return R->isA<RTTIRoot>(); }
+
+private:
+  virtual void anchor();
+
+  static char ID;
+};
+
+/// Inheritance utility for extensible RTTI.
+///
+/// Supports single inheritance only: A class can only have one
+/// ExtensibleRTTI-parent (i.e. a parent for which the isa<> test will work),
+/// though it can have many non-ExtensibleRTTI parents.
+///
+/// RTTIExtents uses CRTP so the first template argument to RTTIExtends is the
+/// newly introduced type, and the *second* argument is the parent class.
+///
+/// class MyType : public RTTIExtends<MyType, RTTIRoot> {
+///   ...
+/// };
+///
+/// class MyDerivedType : public RTTIExtends<MyDerivedType, MyType> {
+///   ...
+/// };
+///
+template <typename ThisT, typename ParentT> class RTTIExtends : public ParentT {
+public:
+  // Inherit constructors and isA methods from ParentT.
+  using ParentT::isA;
+  using ParentT::ParentT;
+
+  static char ID;
+
+  static const void *classID() noexcept { return &ThisT::ID; }
+
+  const void *dynamicClassID() const noexcept override { return &ThisT::ID; }
+
+  bool isA(const void *const ClassID) const noexcept override {
+    return ClassID == classID() || ParentT::isA(ClassID);
+  }
+
+  static bool classof(const RTTIRoot *R) { return R->isA<ThisT>(); }
+};
+
+template <typename ThisT, typename ParentT>
+char RTTIExtends<ThisT, ParentT>::ID = 0;
+
+/// Returns true if the given value is an instance of the template type
+/// parameter.
+template <typename To, typename From> bool isa(const From &Value) noexcept {
+  return To::classof(&Value);
+}
+
+} // namespace orc_rt
+
+#endif // ORC_RT_RTTI_H
diff --git a/orc-rt/include/orc-rt/SimplePackedSerialization.h b/orc-rt/include/orc-rt/SimplePackedSerialization.h
new file mode 100644
index 0000000000000..aaba64cc26ffe
--- /dev/null
+++ b/orc-rt/include/orc-rt/SimplePackedSerialization.h
@@ -0,0 +1,684 @@
+//===---- SimplePackedSerialization.h - simple serialization ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime support library.
+//
+// The behavior of the utilities in this header must be synchronized with the
+// behavior of the utilities in
+// llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h.
+//
+// The Simple Packed Serialization (SPS) utilities are used to generate
+// argument and return buffers for wrapper functions using the following
+// serialization scheme:
+//
+// Primitives:
+//   bool, char, int8_t, uint8_t -- Two's complement 8-bit (0=false, 1=true)
+//   int16_t, uint16_t           -- Two's complement 16-bit little endian
+//   int32_t, uint32_t           -- Two's complement 32-bit little endian
+//   int64_t, int64_t            -- Two's complement 64-bit little endian
+//
+// Sequence<T>:
+//   Serialized as the sequence length (as a uint64_t) followed by the
+//   serialization of each of the elements without padding.
+//
+// Tuple<T1, ..., TN>:
+//   Serialized as each of the element types from T1 to TN without padding.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_SIMPLEPACKEDSERIALIZATION_H
+#define ORC_RT_SIMPLEPACKEDSERIALIZATION_H
+
+#include "orc-rt/Error.h"
+#include "orc-rt/bit.h"
+#include "orc-rt/span.h"
+
+#include <optional>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <type_traits>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace orc_rt {
+
+/// Output char buffer with overflow check.
+class SPSOutputBuffer {
+public:
+  SPSOutputBuffer(char *Buffer, size_t Remaining)
+      : Buffer(Buffer), Remaining(Remaining) {}
+  bool write(const char *Data, size_t Size) {
+    if (Size > Remaining)
+      return false;
+    memcpy(Buffer, Data, Size);
+    Buffer += Size;
+    Remaining -= Size;
+    return true;
+  }
+
+private:
+  char *Buffer = nullptr;
+  size_t Remaining = 0;
+};
+
+/// Input char buffer with underflow check.
+class SPSInputBuffer {
+public:
+  SPSInputBuffer() = default;
+  SPSInputBuffer(const char *Buffer, size_t Remaining)
+      : Buffer(Buffer), Remaining(Remaining) {}
+  bool read(char *Data, size_t Size) {
+    if (Size > Remaining)
+      return false;
+    memcpy(Data, Buffer, Size);
+    Buffer += Size;
+    Remaining -= Size;
+    return true;
+  }
+
+  const char *data() const { return Buffer; }
+  bool skip(size_t Size) {
+    if (Size > Remaining)
+      return false;
+    Buffer += Size;
+    Remaining -= Size;
+    return true;
+  }
+
+private:
+  const char *Buffer = nullptr;
+  size_t Remaining = 0;
+};
+
+/// Specialize to describe how to serialize/deserialize to/from the given
+/// concrete type.
+template <typename SPSTagT, typename ConcreteT, typename _ = void>
+class SPSSerializationTraits;
+
+/// A utility class for serializing to a blob from a variadic list.
+template <typename... ArgTs> class SPSArgList;
+
+// Empty list specialization for SPSArgList.
+template <> class SPSArgList<> {
+public:
+  static size_t size() { return 0; }
+
+  static bool serialize(SPSOutputBuffer &OB) { return true; }
+  static bool deserialize(SPSInputBuffer &IB) { return true; }
+};
+
+// Non-empty list specialization for SPSArgList.
+template <typename SPSTagT, typename... SPSTagTs>
+class SPSArgList<SPSTagT, SPSTagTs...> {
+public:
+  template <typename ArgT, typename... ArgTs>
+  static size_t size(const ArgT &Arg, const ArgTs &...Args) {
+    return SPSSerializationTraits<SPSTagT, ArgT>::size(Arg) +
+           SPSArgList<SPSTagTs...>::size(Args...);
+  }
+
+  template <typename ArgT, typename... ArgTs>
+  static bool serialize(SPSOutputBuffer &OB, const ArgT &Arg,
+                        const ArgTs &...Args) {
+    return SPSSerializationTraits<SPSTagT, ArgT>::serialize(OB, Arg) &&
+           SPSArgList<SPSTagTs...>::serialize(OB, Args...);
+  }
+
+  template <typename ArgT, typename... ArgTs>
+  static bool deserialize(SPSInputBuffer &IB, ArgT &Arg, ArgTs &...Args) {
+    return SPSSerializationTraits<SPSTagT, ArgT>::deserialize(IB, Arg) &&
+           SPSArgList<SPSTagTs...>::deserialize(IB, Args...);
+  }
+};
+
+/// SPS serialization for integral types, bool, and char.
+template <typename SPSTagT>
+class SPSSerializationTraits<
+    SPSTagT, SPSTagT,
+    std::enable_if_t<std::is_same<SPSTagT, bool>::value ||
+                     std::is_same<SPSTagT, char>::value ||
+                     std::is_same<SPSTagT, int8_t>::value ||
+                     std::is_same<SPSTagT, int16_t>::value ||
+                     std::is_same<SPSTagT, int32_t>::value ||
+                     std::is_same<SPSTagT, int64_t>::value ||
+                     std::is_same<SPSTagT, uint8_t>::value ||
+                     std::is_same<SPSTagT, uint16_t>::value ||
+                     std::is_same<SPSTagT, uint32_t>::value ||
+                     std::is_same<SPSTagT, uint64_t>::value>> {
+public:
+  static size_t size(const SPSTagT &Value) { return sizeof(SPSTagT); }
+
+  static bool serialize(SPSOutputBuffer &OB, const SPSTagT &Value) {
+    SPSTagT Tmp = (endian::native == endian::big) ? byteswap(Value) : Value;
+    return OB.write(reinterpret_cast<const char *>(&Tmp), sizeof(Tmp));
+  }
+
+  static bool deserialize(SPSInputBuffer &IB, SPSTagT &Value) {
+    SPSTagT Tmp;
+    if (!IB.read(reinterpret_cast<char *>(&Tmp), sizeof(Tmp)))
+      return false;
+    Value = (endian::native == endian::big) ? byteswap(Tmp) : Tmp;
+    return true;
+  }
+};
+
+/// Any empty placeholder suitable as a substitute for void when deserializing
+class SPSEmpty {};
+
+/// Represents an address in the executor.
+class SPSExecutorAddr {};
+
+/// SPS tag type for tuples.
+///
+/// A blob tuple should be serialized by serializing each of the elements in
+/// sequence.
+template <typename... SPSTagTs> class SPSTuple {
+public:
+  /// Convenience typedef of the corresponding arg list.
+  typedef SPSArgList<SPSTagTs...> AsArgList;
+};
+
+/// SPS tag type for optionals.
+///
+/// SPSOptionals should be serialized as a bool with true indicating that an
+/// SPSTagT value is present, and false indicating that there is no value.
+/// If the boolean is true then the serialized SPSTagT will follow immediately
+/// after it.
+template <typename SPSTagT> class SPSOptional {};
+
+/// SPS tag type for sequences.
+///
+/// SPSSequences should be serialized as a uint64_t sequence length,
+/// followed by the serialization of each of the elements.
+template <typename SPSElementTagT> class SPSSequence;
+
+/// SPS tag type for strings, which are equivalent to sequences of chars.
+using SPSString = SPSSequence<char>;
+
+/// SPS tag type for maps.
+///
+/// SPS maps are just sequences of (Key, Value) tuples.
+template <typename SPSTagT1, typename SPSTagT2>
+using SPSMap = SPSSequence<SPSTuple<SPSTagT1, SPSTagT2>>;
+
+/// Serialization for SPSEmpty type.
+template <> class SPSSerializationTraits<SPSEmpty, SPSEmpty> {
+public:
+  static size_t size(const SPSEmpty &EP) { return 0; }
+  static bool serialize(SPSOutputBuffer &OB, const SPSEmpty &BE) {
+    return true;
+  }
+  static bool deserialize(SPSInputBuffer &IB, SPSEmpty &BE) { return true; }
+};
+
+/// Specialize this to implement 'trivial' sequence serialization for
+/// a concrete sequence type.
+///
+/// Trivial sequence serialization uses the sequence's 'size' member to get the
+/// length of the sequence, and uses a range-based for loop to iterate over the
+/// elements.
+///
+/// Specializing this template class means that you do not need to provide a
+/// specialization of SPSSerializationTraits for your type.
+template <typename SPSElementTagT, typename ConcreteSequenceT>
+class TrivialSPSSequenceSerialization {
+public:
+  static constexpr bool available = false;
+};
+
+/// Specialize this to implement 'trivial' sequence deserialization for
+/// a concrete sequence type.
+///
+/// Trivial deserialization calls a static 'reserve(SequenceT&)' method on your
+/// specialization (you must implement this) to reserve space, and then calls
+/// a static 'append(SequenceT&, ElementT&) method to append each of the
+/// deserialized elements.
+///
+/// Specializing this template class means that you do not need to provide a
+/// specialization of SPSSerializationTraits for your type.
+template <typename SPSElementTagT, typename ConcreteSequenceT>
+class TrivialSPSSequenceDeserialization {
+public:
+  static constexpr bool available = false;
+};
+
+/// Trivial std::string -> SPSSequence<char> serialization.
+template <> class TrivialSPSSequenceSerialization<char, std::string> {
+public:
+  static constexpr bool available = true;
+};
+
+/// Trivial SPSSequence<char> -> std::string deserialization.
+template <> class TrivialSPSSequenceDeserialization<char, std::string> {
+public:
+  static constexpr bool available = true;
+
+  using element_type = char;
+
+  static void reserve(std::string &S, uint64_t Size) { S.reserve(Size); }
+  static bool append(std::string &S, char C) {
+    S.push_back(C);
+    return true;
+  }
+};
+
+/// Trivial std::vector<T> -> SPSSequence<SPSElementTagT> serialization.
+template <typename SPSElementTagT, typename T>
+class TrivialSPSSequenceSerialization<SPSElementTagT, std::vector<T>> {
+public:
+  static constexpr bool available = true;
+};
+
+/// Trivial span<T> -> SPSSequence<SPSElementTagT> serialization.
+template <typename SPSElementTagT, typename T>
+class TrivialSPSSequenceSerialization<SPSElementTagT, span<T>> {
+public:
+  static constexpr bool available = true;
+};
+
+/// Trivial SPSSequence<SPSElementTagT> -> std::vector<T> deserialization.
+template <typename SPSElementTagT, typename T>
+class TrivialSPSSequenceDeserialization<SPSElementTagT, std::vector<T>> {
+public:
+  static constexpr bool available = true;
+
+  using element_type = typename std::vector<T>::value_type;
+
+  static void reserve(std::vector<T> &V, uint64_t Size) { V.reserve(Size); }
+  static bool append(std::vector<T> &V, T E) {
+    V.push_back(std::move(E));
+    return true;
+  }
+};
+
+/// Trivial std::unordered_map<K, V> -> SPSSequence<SPSTuple<SPSKey, SPSValue>>
+/// serialization.
+template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V>
+class TrivialSPSSequenceSerialization<SPSTuple<SPSKeyTagT, SPSValueTagT>,
+                                      std::unordered_map<K, V>> {
+public:
+  static constexpr bool available = true;
+};
+
+/// Trivial SPSSequence<SPSTuple<SPSKey, SPSValue>> -> std::unordered_map<K, V>
+/// deserialization.
+template <typename SPSKeyTagT, typename SPSValueTagT, typename K, typename V>
+class TrivialSPSSequenceDeserialization<SPSTuple<SPSKeyTagT, SPSValueTagT>,
+                                        std::unordered_map<K, V>> {
+public:
+  static constexpr bool available = true;
+
+  using element_type = std::pair<K, V>;
+
+  static void reserve(std::unordered_map<K, V> &M, uint64_t Size) {
+    M.reserve(Size);
+  }
+  static bool append(std::unordered_map<K, V> &M, element_type E) {
+    return M.insert(std::move(E)).second;
+  }
+};
+
+/// 'Trivial' sequence serialization: Sequence is serialized as a uint64_t size
+/// followed by a for-earch loop over the elements of the sequence to serialize
+/// each of them.
+template <typename SPSElementTagT, typename SequenceT>
+class SPSSerializationTraits<SPSSequence<SPSElementTagT>, SequenceT,
+                             std::enable_if_t<TrivialSPSSequenceSerialization<
+                                 SPSElementTagT, SequenceT>::available>> {
+public:
+  static size_t size(const SequenceT &S) {
+    size_t Size = SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size()));
+    for (const auto &E : S)
+      Size += SPSArgList<SPSElementTagT>::size(E);
+    return Size;
+  }
+
+  static bool serialize(SPSOutputBuffer &OB, const SequenceT &S) {
+    if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size())))
+      return false;
+    for (const auto &E : S)
+      if (!SPSArgList<SPSElementTagT>::serialize(OB, E))
+        return false;
+    return true;
+  }
+
+  static bool deserialize(SPSInputBuffer &IB, SequenceT &S) {
+    using TBSD = TrivialSPSSequenceDeserialization<SPSElementTagT, SequenceT>;
+    uint64_t Size;
+    if (!SPSArgList<uint64_t>::deserialize(IB, Size))
+      return false;
+    TBSD::reserve(S, Size);
+    for (size_t I = 0; I != Size; ++I) {
+      typename TBSD::element_type E;
+      if (!SPSArgList<SPSElementTagT>::deserialize(IB, E))
+        return false;
+      if (!TBSD::append(S, std::move(E)))
+        return false;
+    }
+    return true;
+  }
+};
+
+/// Trivial serialization / deserialization for span<char>
+template <> class SPSSerializationTraits<SPSSequence<char>, span<const char>> {
+public:
+  static size_t size(const span<const char> &S) {
+    return SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())) +
+           S.size();
+  }
+  static bool serialize(SPSOutputBuffer &OB, const span<const char> &S) {
+    if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size())))
+      return false;
+    return OB.write(S.data(), S.size());
+  }
+  static bool deserialize(SPSInputBuffer &IB, span<const char> &S) {
+    uint64_t Size;
+    if (!SPSArgList<uint64_t>::deserialize(IB, Size))
+      return false;
+    S = span<const char>(IB.data(), Size);
+    return IB.skip(Size);
+  }
+};
+
+/// SPSTuple serialization for std::tuple.
+template <typename... SPSTagTs, typename... Ts>
+class SPSSerializationTraits<SPSTuple<SPSTagTs...>, std::tuple<Ts...>> {
+private:
+  using TupleArgList = typename SPSTuple<SPSTagTs...>::AsArgList;
+  using ArgIndices = std::make_index_sequence<sizeof...(Ts)>;
+
+  template <std::size_t... I>
+  static size_t size(const std::tuple<Ts...> &T, std::index_sequence<I...>) {
+    return TupleArgList::size(std::get<I>(T)...);
+  }
+
+  template <std::size_t... I>
+  static bool serialize(SPSOutputBuffer &OB, const std::tuple<Ts...> &T,
+                        std::index_sequence<I...>) {
+    return TupleArgList::serialize(OB, std::get<I>(T)...);
+  }
+
+  template <std::size_t... I>
+  static bool deserialize(SPSInputBuffer &IB, std::tuple<Ts...> &T,
+                          std::index_sequence<I...>) {
+    return TupleArgList::deserialize(IB, std::get<I>(T)...);
+  }
+
+public:
+  static size_t size(const std::tuple<Ts...> &T) {
+    return size(T, ArgIndices{});
+  }
+
+  static bool serialize(SPSOutputBuffer &OB, const std::tuple<Ts...> &T) {
+    return serialize(OB, T, ArgIndices{});
+  }
+
+  static bool deserialize(SPSInputBuffer &IB, std::tuple<Ts...> &T) {
+    return deserialize(IB, T, ArgIndices{});
+  }
+};
+
+/// SPSTuple serialization for std::pair.
+template <typename SPSTagT1, typename SPSTagT2, typename T1, typename T2>
+class SPSSerializationTraits<SPSTuple<SPSTagT1, SPSTagT2>, std::pair<T1, T2>> {
+public:
+  static size_t size(const std::pair<T1, T2> &P) {
+    return SPSArgList<SPSTagT1>::size(P.first) +
+           SPSArgList<SPSTagT2>::size(P.second);
+  }
+
+  static bool serialize(SPSOutputBuffer &OB, const std::pair<T1, T2> &P) {
+    return SPSArgList<SPSTagT1>::serialize(OB, P.first) &&
+           SPSArgList<SPSTagT2>::serialize(OB, P.second);
+  }
+
+  static bool deserialize(SPSInputBuffer &IB, std::pair<T1, T2> &P) {
+    return SPSArgList<SPSTagT1>::deserialize(IB, P.first) &&
+           SPSArgList<SPSTagT2>::deserialize(IB, P.second);
+  }
+};
+
+/// SPSOptional serialization for std::optional.
+template <typename SPSTagT, typename T>
+class SPSSerializationTraits<SPSOptional<SPSTagT>, std::optional<T>> {
+public:
+  static size_t size(const std::optional<T> &Value) {
+    size_t Size = SPSArgList<bool>::size(!!Value);
+    if (Value)
+      Size += SPSArgList<SPSTagT>::size(*Value);
+    return Size;
+  }
+
+  static bool serialize(SPSOutputBuffer &OB, const std::optional<T> &Value) {
+    if (!SPSArgList<bool>::serialize(OB, !!Value))
+      return false;
+    if (Value)
+      return SPSArgList<SPSTagT>::serialize(OB, *Value);
+    return true;
+  }
+
+  static bool deserialize(SPSInputBuffer &IB, std::optional<T> &Value) {
+    bool HasValue;
+    if (!SPSArgList<bool>::deserialize(IB, HasValue))
+      return false;
+    if (HasValue) {
+      Value = T();
+      return SPSArgList<SPSTagT>::deserialize(IB, *Value);
+    } else
+      Value = std::optional<T>();
+    return true;
+  }
+};
+
+/// Serialization for string_views.
+///
+/// Serialization is as for regular strings. Deserialization points directly
+/// into the blob.
+template <> class SPSSerializationTraits<SPSString, std::string_view> {
+public:
+  static size_t size(const std::string_view &S) {
+    return SPSArgList<uint64_t>::size(static_cast<uint64_t>(S.size())) +
+           S.size();
+  }
+
+  static bool serialize(SPSOutputBuffer &OB, const std::string_view &S) {
+    if (!SPSArgList<uint64_t>::serialize(OB, static_cast<uint64_t>(S.size())))
+      return false;
+    return OB.write(S.data(), S.size());
+  }
+
+  static bool deserialize(SPSInputBuffer &IB, std::string_view &S) {
+    const char *Data = nullptr;
+    uint64_t Size;
+    if (!SPSArgList<uint64_t>::deserialize(IB, Size))
+      return false;
+    if (Size > std::numeric_limits<size_t>::max())
+      return false;
+    Data = IB.data();
+    if (!IB.skip(Size))
+      return false;
+    S = {Data, static_cast<size_t>(Size)};
+    return true;
+  }
+};
+
+/// SPS tag type for errors.
+class SPSError;
+
+/// SPS tag type for expecteds, which are either a T or a string representing
+/// an error.
+template <typename SPSTagT> class SPSExpected;
+
+namespace detail {
+
+/// Helper type for serializing Errors.
+///
+/// llvm::Errors are move-only, and not inspectable except by consuming them.
+/// This makes them unsuitable for direct serialization via
+/// SPSSerializationTraits, which needs to inspect values twice (once to
+/// determine the amount of space to reserve, and then again to serialize).
+///
+/// The SPSSerializableError type is a helper that can be
+/// constructed from an llvm::Error, but inspected more than once.
+struct SPSSerializableError {
+  bool HasError = false;
+  std::string ErrMsg;
+};
+
+/// Helper type for serializing Expected<T>s.
+///
+/// See SPSSerializableError for more details.
+///
+// FIXME: Use std::variant for storage once we have c++17.
+template <typename T> struct SPSSerializableExpected {
+  bool HasValue = false;
+  T Value{};
+  std::string ErrMsg;
+};
+
+inline SPSSerializableError toSPSSerializable(Error Err) {
+  if (Err)
+    return {true, toString(std::move(Err))};
+  return {false, {}};
+}
+
+inline Error fromSPSSerializable(SPSSerializableError BSE) {
+  if (BSE.HasError)
+    return make_error<StringError>(BSE.ErrMsg);
+  return Error::success();
+}
+
+template <typename T>
+SPSSerializableExpected<T> toSPSSerializable(Expected<T> E) {
+  if (E)
+    return {true, std::move(*E), {}};
+  else
+    return {false, {}, toString(E.takeError())};
+}
+
+template <typename T>
+Expected<T> fromSPSSerializable(SPSSerializableExpected<T> BSE) {
+  if (BSE.HasValue)
+    return std::move(BSE.Value);
+  else
+    return make_error<StringError>(BSE.ErrMsg);
+}
+
+} // namespace detail
+
+/// Serialize to a SPSError from a detail::SPSSerializableError.
+template <>
+class SPSSerializationTraits<SPSError, detail::SPSSerializableError> {
+public:
+  static size_t size(const detail::SPSSerializableError &BSE) {
+    size_t Size = SPSArgList<bool>::size(BSE.HasError);
+    if (BSE.HasError)
+      Size += SPSArgList<SPSString>::size(BSE.ErrMsg);
+    return Size;
+  }
+
+  static bool serialize(SPSOutputBuffer &OB,
+                        const detail::SPSSerializableError &BSE) {
+    if (!SPSArgList<bool>::serialize(OB, BSE.HasError))
+      return false;
+    if (BSE.HasError)
+      if (!SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg))
+        return false;
+    return true;
+  }
+
+  static bool deserialize(SPSInputBuffer &IB,
+                          detail::SPSSerializableError &BSE) {
+    if (!SPSArgList<bool>::deserialize(IB, BSE.HasError))
+      return false;
+
+    if (!BSE.HasError)
+      return true;
+
+    return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg);
+  }
+};
+
+/// Serialize to a SPSExpected<SPSTagT> from a
+/// detail::SPSSerializableExpected<T>.
+template <typename SPSTagT, typename T>
+class SPSSerializationTraits<SPSExpected<SPSTagT>,
+                             detail::SPSSerializableExpected<T>> {
+public:
+  static size_t size(const detail::SPSSerializableExpected<T> &BSE) {
+    size_t Size = SPSArgList<bool>::size(BSE.HasValue);
+    if (BSE.HasValue)
+      Size += SPSArgList<SPSTagT>::size(BSE.Value);
+    else
+      Size += SPSArgList<SPSString>::size(BSE.ErrMsg);
+    return Size;
+  }
+
+  static bool serialize(SPSOutputBuffer &OB,
+                        const detail::SPSSerializableExpected<T> &BSE) {
+    if (!SPSArgList<bool>::serialize(OB, BSE.HasValue))
+      return false;
+
+    if (BSE.HasValue)
+      return SPSArgList<SPSTagT>::serialize(OB, BSE.Value);
+
+    return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg);
+  }
+
+  static bool deserialize(SPSInputBuffer &IB,
+                          detail::SPSSerializableExpected<T> &BSE) {
+    if (!SPSArgList<bool>::deserialize(IB, BSE.HasValue))
+      return false;
+
+    if (BSE.HasValue)
+      return SPSArgList<SPSTagT>::deserialize(IB, BSE.Value);
+
+    return SPSArgList<SPSString>::deserialize(IB, BSE.ErrMsg);
+  }
+};
+
+/// Serialize to a SPSExpected<SPSTagT> from a detail::SPSSerializableError.
+template <typename SPSTagT>
+class SPSSerializationTraits<SPSExpected<SPSTagT>,
+                             detail::SPSSerializableError> {
+public:
+  static size_t size(const detail::SPSSerializableError &BSE) {
+    assert(BSE.HasError && "Cannot serialize expected from a success value");
+    return SPSArgList<bool>::size(false) +
+           SPSArgList<SPSString>::size(BSE.ErrMsg);
+  }
+
+  static bool serialize(SPSOutputBuffer &OB,
+                        const detail::SPSSerializableError &BSE) {
+    assert(BSE.HasError && "Cannot serialize expected from a success value");
+    if (!SPSArgList<bool>::serialize(OB, false))
+      return false;
+    return SPSArgList<SPSString>::serialize(OB, BSE.ErrMsg);
+  }
+};
+
+/// Serialize to a SPSExpected<SPSTagT> from a T.
+template <typename SPSTagT, typename T>
+class SPSSerializationTraits<SPSExpected<SPSTagT>, T> {
+public:
+  static size_t size(const T &Value) {
+    return SPSArgList<bool>::size(true) + SPSArgList<SPSTagT>::size(Value);
+  }
+
+  static bool serialize(SPSOutputBuffer &OB, const T &Value) {
+    if (!SPSArgList<bool>::serialize(OB, true))
+      return false;
+    return SPSArgList<SPSTagT>::serialize(Value);
+  }
+};
+
+} // namespace orc_rt
+
+#endif // ORC_RT_SIMPLEPACKEDSERIALIZATION_H
diff --git a/orc-rt/include/orc-rt/WrapperFunctionResult.h b/orc-rt/include/orc-rt/WrapperFunctionResult.h
new file mode 100644
index 0000000000000..d3bc132070e86
--- /dev/null
+++ b/orc-rt/include/orc-rt/WrapperFunctionResult.h
@@ -0,0 +1,106 @@
+//===---- WrapperFunctionResult.h -- blob-of-bytes container ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines WrapperFunctionResult and related APIs.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_WRAPPERFUNCTIONRESULT_H
+#define ORC_RT_WRAPPERFUNCTIONRESULT_H
+
+#include "orc-rt-c/WrapperFunctionResult.h"
+
+#include <utility>
+
+namespace orc_rt {
+
+/// A C++ convenience wrapper for orc_rt_WrapperFunctionResult. Auto-disposes
+/// the contained result on destruction.
+class WrapperFunctionResult {
+public:
+  /// Create a default WrapperFunctionResult.
+  WrapperFunctionResult() { orc_rt_WrapperFunctionResultInit(&R); }
+
+  /// Create a WrapperFunctionResult from a WrapperFunctionResult. This
+  /// instance takes ownership of the result object and will automatically
+  /// call dispose on the result upon destruction.
+  WrapperFunctionResult(orc_rt_WrapperFunctionResult R) : R(R) {}
+
+  WrapperFunctionResult(const WrapperFunctionResult &) = delete;
+  WrapperFunctionResult &operator=(const WrapperFunctionResult &) = delete;
+
+  WrapperFunctionResult(WrapperFunctionResult &&Other) {
+    orc_rt_WrapperFunctionResultInit(&R);
+    std::swap(R, Other.R);
+  }
+
+  WrapperFunctionResult &operator=(WrapperFunctionResult &&Other) {
+    orc_rt_WrapperFunctionResult Tmp;
+    orc_rt_WrapperFunctionResultInit(&Tmp);
+    std::swap(Tmp, Other.R);
+    std::swap(R, Tmp);
+    return *this;
+  }
+
+  ~WrapperFunctionResult() { orc_rt_DisposeWrapperFunctionResult(&R); }
+
+  /// Relinquish ownership of and return the
+  /// orc_rt_WrapperFunctionResult.
+  orc_rt_WrapperFunctionResult release() {
+    orc_rt_WrapperFunctionResult Tmp;
+    orc_rt_WrapperFunctionResultInit(&Tmp);
+    std::swap(R, Tmp);
+    return Tmp;
+  }
+
+  /// Get a pointer to the data contained in this instance.
+  char *data() { return orc_rt_WrapperFunctionResultData(&R); }
+
+  /// Returns the size of the data contained in this instance.
+  size_t size() const { return orc_rt_WrapperFunctionResultSize(&R); }
+
+  /// Returns true if this value is equivalent to a default-constructed
+  /// WrapperFunctionResult.
+  bool empty() const { return orc_rt_WrapperFunctionResultEmpty(&R); }
+
+  /// Create a WrapperFunctionResult with the given size and return a pointer
+  /// to the underlying memory.
+  static WrapperFunctionResult allocate(size_t Size) {
+    WrapperFunctionResult R;
+    R.R = orc_rt_WrapperFunctionResultAllocate(Size);
+    return R;
+  }
+
+  /// Copy from the given char range.
+  static WrapperFunctionResult copyFrom(const char *Source, size_t Size) {
+    return orc_rt_CreateWrapperFunctionResultFromRange(Source, Size);
+  }
+
+  /// Copy from the given null-terminated string (includes the null-terminator).
+  static WrapperFunctionResult copyFrom(const char *Source) {
+    return orc_rt_CreateWrapperFunctionResultFromString(Source);
+  }
+
+  /// Create an out-of-band error by copying the given string.
+  static WrapperFunctionResult createOutOfBandError(const char *Msg) {
+    return orc_rt_CreateWrapperFunctionResultFromOutOfBandError(Msg);
+  }
+
+  /// If this value is an out-of-band error then this returns the error message,
+  /// otherwise returns nullptr.
+  const char *getOutOfBandError() const {
+    return orc_rt_WrapperFunctionResultGetOutOfBandError(&R);
+  }
+
+private:
+  orc_rt_WrapperFunctionResult R;
+};
+
+} // namespace orc_rt
+
+#endif // ORC_RT_WRAPPERFUNCTIONRESULT_H
diff --git a/orc-rt/include/orc-rt/bit.h b/orc-rt/include/orc-rt/bit.h
new file mode 100644
index 0000000000000..97dbee2f3442b
--- /dev/null
+++ b/orc-rt/include/orc-rt/bit.h
@@ -0,0 +1,110 @@
+//===-------- bit.h - Substitute for future STL bit APIs -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Substitutes for STL <bit> APIs that aren't available to the ORC runtime yet.
+//
+// TODO: Replace all uses once the respective APIs are available.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_BIT_H
+#define ORC_RT_BIT_H
+
+#include <cstddef>
+#include <cstdint>
+#include <type_traits>
+#if defined(_MSC_VER) && !defined(_DEBUG)
+#include <stdlib.h>
+#endif
+
+#if defined(__linux__) || defined(__GNU__) || defined(__HAIKU__) ||            \
+    defined(__Fuchsia__) || defined(__EMSCRIPTEN__)
+#include <endian.h>
+#elif defined(_AIX)
+#include <sys/machine.h>
+#elif defined(__sun)
+/* Solaris provides _BIG_ENDIAN/_LITTLE_ENDIAN selector in sys/types.h */
+#include <sys/types.h>
+#define BIG_ENDIAN 4321
+#define LITTLE_ENDIAN 1234
+#if defined(_BIG_ENDIAN)
+#define BYTE_ORDER BIG_ENDIAN
+#else
+#define BYTE_ORDER LITTLE_ENDIAN
+#endif
+#elif defined(__MVS__)
+#define BIG_ENDIAN 4321
+#define LITTLE_ENDIAN 1234
+#define BYTE_ORDER BIG_ENDIAN
+#else
+#if !defined(BYTE_ORDER) && !defined(_WIN32)
+#include <machine/endian.h>
+#endif
+#endif
+
+namespace orc_rt {
+
+enum class endian {
+  big,
+  little,
+#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && BYTE_ORDER == BIG_ENDIAN
+  native = big
+#else
+  native = little
+#endif
+};
+
+template <typename T, typename = std::enable_if_t<std::is_integral_v<T>>>
+[[nodiscard]] constexpr T byteswap(T V) noexcept {
+  // Implementation taken from llvm/include/ADT/bit.h.
+  if constexpr (sizeof(T) == 1) {
+    return V;
+  } else if constexpr (sizeof(T) == 2) {
+    uint16_t UV = V;
+#if defined(_MSC_VER) && !defined(_DEBUG)
+    // The DLL version of the runtime lacks these functions (bug!?), but in a
+    // release build they're replaced with BSWAP instructions anyway.
+    return _byteswap_ushort(UV);
+#else
+    uint16_t Hi = UV << 8;
+    uint16_t Lo = UV >> 8;
+    return Hi | Lo;
+#endif
+  } else if constexpr (sizeof(T) == 4) {
+    uint32_t UV = V;
+#if __has_builtin(__builtin_bswap32)
+    return __builtin_bswap32(UV);
+#elif defined(_MSC_VER) && !defined(_DEBUG)
+    return _byteswap_ulong(UV);
+#else
+    uint32_t Byte0 = UV & 0x000000FF;
+    uint32_t Byte1 = UV & 0x0000FF00;
+    uint32_t Byte2 = UV & 0x00FF0000;
+    uint32_t Byte3 = UV & 0xFF000000;
+    return (Byte0 << 24) | (Byte1 << 8) | (Byte2 >> 8) | (Byte3 >> 24);
+#endif
+  } else if constexpr (sizeof(T) == 8) {
+    uint64_t UV = V;
+#if __has_builtin(__builtin_bswap64)
+    return __builtin_bswap64(UV);
+#elif defined(_MSC_VER) && !defined(_DEBUG)
+    return _byteswap_uint64(UV);
+#else
+    uint64_t Hi = llvm::byteswap<uint32_t>(UV);
+    uint32_t Lo = llvm::byteswap<uint32_t>(UV >> 32);
+    return (Hi << 32) | Lo;
+#endif
+  } else {
+    static_assert(!sizeof(T *), "Don't know how to handle the given type.");
+    return 0;
+  }
+}
+
+} // namespace orc_rt
+
+#endif // ORC_RT_BIT_H
diff --git a/orc-rt/include/orc-rt/move_only_function.h b/orc-rt/include/orc-rt/move_only_function.h
new file mode 100644
index 0000000000000..c87ce50d4daed
--- /dev/null
+++ b/orc-rt/include/orc-rt/move_only_function.h
@@ -0,0 +1,84 @@
+//===-- move_only_function.h - moveable, type-erasing function --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// A substitute for std::move_only_function that can be used until the ORC
+/// runtime is allowed to assume c++-23.
+///
+/// TODO: Replace all uses with std::move_only_function once we can assume
+///       c++-23.
+///
+/// TODO: Re-implement using techniques from LLVM's unique_function
+///       (llvm/include/llvm/ADT/FunctionExtras.h), which uses some extra
+///       inline storage to avoid heap allocations for small objects. This
+///       would require first porting some other LLVM utilities like
+///       PointerIntPair, PointerUnion, and PointerLikeTypeTraits. (These are
+///       likely to be independently useful in the orc runtime, so porting will
+///       have additional benefits).
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_MOVE_ONLY_FUNCTION_H
+#define ORC_RT_MOVE_ONLY_FUNCTION_H
+
+#include <memory>
+
+namespace orc_rt {
+
+namespace move_only_function_detail {
+
+template <typename RetT, typename... ArgTs> class Callable {
+public:
+  virtual ~Callable() = default;
+  virtual RetT call(ArgTs &&...Args) = 0;
+};
+
+template <typename CallableT, typename RetT, typename... ArgTs>
+class CallableImpl : public Callable<RetT, ArgTs...> {
+public:
+  CallableImpl(CallableT &&Callable) : Callable(std::move(Callable)) {}
+  RetT call(ArgTs &&...Args) override {
+    return Callable(std::forward<ArgTs>(Args)...);
+  }
+
+private:
+  CallableT Callable;
+};
+
+} // namespace move_only_function_detail
+
+template <typename FnT> class move_only_function;
+
+template <typename RetT, typename... ArgTs>
+class move_only_function<RetT(ArgTs...)> {
+public:
+  move_only_function() = default;
+  move_only_function(std::nullptr_t) {}
+  move_only_function(move_only_function &&) = default;
+  move_only_function(const move_only_function &) = delete;
+  move_only_function &operator=(move_only_function &&) = default;
+  move_only_function &operator=(const move_only_function &) = delete;
+
+  template <typename CallableT>
+  move_only_function(CallableT &&Callable)
+      : C(std::make_unique<
+            move_only_function_detail::CallableImpl<CallableT, RetT, ArgTs...>>(
+            std::forward<CallableT>(Callable))) {}
+
+  RetT operator()(ArgTs... Params) {
+    return C->call(std::forward<ArgTs>(Params)...);
+  }
+
+  explicit operator bool() const { return !!C; }
+
+private:
+  std::unique_ptr<move_only_function_detail::Callable<RetT, ArgTs...>> C;
+};
+
+} // namespace orc_rt
+
+#endif // ORC_RT_MOVE_ONLY_FUNCTION_H
diff --git a/orc-rt/include/orc-rt/span.h b/orc-rt/include/orc-rt/span.h
new file mode 100644
index 0000000000000..054d02af66b9c
--- /dev/null
+++ b/orc-rt/include/orc-rt/span.h
@@ -0,0 +1,65 @@
+//===---------- span.h - Substitute for C++20 std::span ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A substitute for std::span that can be used until the ORC runtime is allowed
+// to assume c++-20.
+//
+// TODO: Replace all uses with std::span once we can assume c++20.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_SPAN_H
+#define ORC_RT_SPAN_H
+
+#include <cstddef>
+#include <limits>
+#include <type_traits>
+
+namespace orc_rt {
+
+constexpr std::size_t dynamic_extent = std::numeric_limits<std::size_t>::max();
+
+/// A substitute for std::span (and llvm::ArrayRef).
+/// FIXME: Remove in favor of std::span once we can use c++20.
+template <typename T, std::size_t Extent = dynamic_extent> class span {
+public:
+  typedef T element_type;
+  typedef std::remove_cv<T> value_type;
+  typedef std::size_t size_type;
+  typedef std::ptrdiff_t difference_type;
+  typedef T *pointer;
+  typedef const T *const_pointer;
+  typedef T &reference;
+  typedef const T &const_reference;
+
+  typedef pointer iterator;
+
+  static constexpr std::size_t extent = Extent;
+
+  constexpr span() noexcept = default;
+  constexpr span(T *first, size_type count) noexcept
+      : Data(first), Size(count) {}
+
+  template <std::size_t N>
+  constexpr span(T (&arr)[N]) noexcept : Data(&arr[0]), Size(N) {}
+
+  constexpr iterator begin() const noexcept { return Data; }
+  constexpr iterator end() const noexcept { return Data + Size; }
+  constexpr pointer data() const noexcept { return Data; }
+  constexpr reference operator[](size_type idx) const { return Data[idx]; }
+  constexpr size_type size() const noexcept { return Size; }
+  constexpr bool empty() const noexcept { return Size == 0; }
+
+private:
+  T *Data = nullptr;
+  size_type Size = 0;
+};
+
+} // namespace orc_rt
+
+#endif // ORC_RT_SPAN_H
diff --git a/orc-rt/lib/CMakeLists.txt b/orc-rt/lib/CMakeLists.txt
new file mode 100644
index 0000000000000..993f2b8b52eb7
--- /dev/null
+++ b/orc-rt/lib/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(executor)
diff --git a/orc-rt/lib/executor/CMakeLists.txt b/orc-rt/lib/executor/CMakeLists.txt
new file mode 100644
index 0000000000000..331954c2c556f
--- /dev/null
+++ b/orc-rt/lib/executor/CMakeLists.txt
@@ -0,0 +1,14 @@
+set(files
+  orc-rt-executor.cpp
+  RTTI.cpp
+  )
+
+add_library(orc-rt-executor STATIC ${files})
+target_link_libraries(orc-rt-executor
+  PUBLIC orc-rt-headers
+  )
+install(TARGETS orc-rt-executor
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+          COMPONENT OrcRT_Development
+  PUBLIC_HEADER DESTINATION include COMPONENT OrcRT_Development
+)
diff --git a/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc b/orc-rt/lib/executor/RTTI.cpp
similarity index 56%
rename from libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc
rename to orc-rt/lib/executor/RTTI.cpp
index 05a53393170f6..f4e4bc83b6c91 100644
--- a/libclc/clc/include/clc/shared/binary_decl_with_scalar_second_arg.inc
+++ b/orc-rt/lib/executor/RTTI.cpp
@@ -1,15 +1,20 @@
-//===----------------------------------------------------------------------===//
+//===- RTTI.cpp -----------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Contains the implementation of APIs in the orc-rt/RTTI.h header.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/RTTI.h"
+
+namespace orc_rt {
 
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               __CLC_GENTYPE y);
+char RTTIRoot::ID = 0;
+void RTTIRoot::anchor() {}
 
-#ifndef __CLC_SCALAR
-_CLC_OVERLOAD _CLC_DECL __CLC_GENTYPE FUNCTION(__CLC_GENTYPE x,
-                                               __CLC_SCALAR_GENTYPE y);
-#endif
+} // namespace orc_rt
diff --git a/orc-rt/lib/executor/orc-rt-executor.cpp b/orc-rt/lib/executor/orc-rt-executor.cpp
new file mode 100644
index 0000000000000..d70488d11d3fa
--- /dev/null
+++ b/orc-rt/lib/executor/orc-rt-executor.cpp
@@ -0,0 +1,15 @@
+//===- orc-rt-executor.cpp - Placeholder implementation for orc-rt --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Placeholder implementation file for initial orc-rt checkin.
+//
+//===----------------------------------------------------------------------===//
+
+#include <stdio.h>
+
+extern "C" void orc_rt(void) { printf("hello, world!\n"); }
diff --git a/orc-rt/test/CMakeLists.txt b/orc-rt/test/CMakeLists.txt
new file mode 100644
index 0000000000000..211009f5c1dd5
--- /dev/null
+++ b/orc-rt/test/CMakeLists.txt
@@ -0,0 +1,41 @@
+include(OrcRTTesting)
+
+if (ORC_RT_LLVM_TOOLS_AVAILABLE)
+  configure_lit_site_cfg(
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+    ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
+    MAIN_CONFIG
+    ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
+  )
+
+  list(APPEND ORC_RT_TEST_DEPS
+    orc-executor
+  )
+
+  add_custom_target(orc-rt-test-depends DEPENDS ${ORC_RT_TEST_DEPS})
+  set_target_properties(orc-rt-test-depends PROPERTIES FOLDER "orc-rt/tests")
+
+  add_lit_testsuite(check-orc-rt "Running the ORC-RT regression tests"
+    ${CMAKE_CURRENT_BINARY_DIR}
+    DEPENDS ${ORC_RT_TEST_DEPS}
+    )
+else()
+  message(WARNING "ORC-RT testing tools missing (see cmake log for details). ORC-RT regression tests disabled.")
+endif()
+
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/unit/lit.site.cfg.py
+  MAIN_CONFIG
+  ${CMAKE_CURRENT_SOURCE_DIR}/unit/lit.cfg.py
+  )
+
+# add_lit_testsuites(ORC-RT ${CMAKE_CURRENT_SOURCE_DIR}
+#   DEPENDS ${ORC_RT_TEST_DEPS}
+# )
+
+add_lit_testsuite(check-orc-rt-unit "Running orc-rt unittest suites"
+  ${CMAKE_CURRENT_BINARY_DIR}/unit
+  EXCLUDE_FROM_CHECK_ALL
+  DEPENDS OrcRTUnitTests)
+
diff --git a/orc-rt/test/init.test b/orc-rt/test/init.test
new file mode 100644
index 0000000000000..eaa666ef75d79
--- /dev/null
+++ b/orc-rt/test/init.test
@@ -0,0 +1 @@
+RUN: orc-executor %s %s
diff --git a/orc-rt/test/lit.cfg.py b/orc-rt/test/lit.cfg.py
new file mode 100644
index 0000000000000..759d96e613978
--- /dev/null
+++ b/orc-rt/test/lit.cfg.py
@@ -0,0 +1,24 @@
+# -*- Python -*-
+
+import os
+
+import lit.formats
+import lit.util
+
+from lit.llvm import llvm_config
+from lit.llvm.subst import ToolSubst
+
+config.name = "ORC-RT"
+config.test_format = lit.formats.ShTest()
+config.test_source_root = os.path.dirname(__file__)
+config.test_exec_root = os.path.join(config.orc_rt_obj_root, "test")
+config.suffixes = [
+    ".test"
+]
+
+llvm_config.with_environment(
+    "PATH",
+    os.path.join(config.orc_rt_obj_root, "tools", "orc-executor"),
+    append_path=True)
+
+llvm_config.use_default_substitutions()
diff --git a/orc-rt/test/lit.site.cfg.py.in b/orc-rt/test/lit.site.cfg.py.in
new file mode 100644
index 0000000000000..2c6ce81821d35
--- /dev/null
+++ b/orc-rt/test/lit.site.cfg.py.in
@@ -0,0 +1,16 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+import sys
+
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.orc_rt_obj_root = "@ORC_RT_BINARY_DIR@"
+config.host_triple = "@LLVM_HOST_TRIPLE@"
+config.target_triple = "@LLVM_TARGET_TRIPLE@"
+config.llvm_tools_dir = "@ORC_RT_LLVM_TOOLS_DIR@"
+
+
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+# Let the main config do the real work.
+lit_config.load_config(config, "@ORC_RT_SOURCE_DIR@/test/lit.cfg.py")
diff --git a/orc-rt/test/unit/lit.cfg.py b/orc-rt/test/unit/lit.cfg.py
new file mode 100644
index 0000000000000..af4b8d5982915
--- /dev/null
+++ b/orc-rt/test/unit/lit.cfg.py
@@ -0,0 +1,15 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+
+import lit.formats
+import lit.util
+
+from lit.llvm import llvm_config
+
+config.name = "ORC-RT-Unit"
+config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, "Tests")
+config.test_exec_root = os.path.join(config.orc_rt_obj_root, "unittests")
+config.test_source_root = config.test_exec_root
diff --git a/orc-rt/test/unit/lit.site.cfg.py.in b/orc-rt/test/unit/lit.site.cfg.py.in
new file mode 100644
index 0000000000000..d44bfc08795bc
--- /dev/null
+++ b/orc-rt/test/unit/lit.site.cfg.py.in
@@ -0,0 +1,17 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+import sys
+
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.llvm_build_mode = lit_config.substitute("@LLVM_BUILD_MODE@")
+config.orc_rt_obj_root = "@ORC_RT_BINARY_DIR@"
+config.host_triple = "@LLVM_HOST_TRIPLE@"
+config.target_triple = "@LLVM_TARGET_TRIPLE@"
+config.llvm_tools_dir = "@ORC_RT_LLVM_TOOLS_DIR@"
+
+
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+# Let the main config do the real work.
+lit_config.load_config(config, "@ORC_RT_SOURCE_DIR@/test/unit/lit.cfg.py")
diff --git a/orc-rt/tools/CMakeLists.txt b/orc-rt/tools/CMakeLists.txt
new file mode 100644
index 0000000000000..fc78ea4131606
--- /dev/null
+++ b/orc-rt/tools/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(orc-executor)
diff --git a/orc-rt/tools/orc-executor/CMakeLists.txt b/orc-rt/tools/orc-executor/CMakeLists.txt
new file mode 100644
index 0000000000000..b4a2292f58173
--- /dev/null
+++ b/orc-rt/tools/orc-executor/CMakeLists.txt
@@ -0,0 +1,2 @@
+add_executable(orc-executor orc-executor.cpp)
+target_link_libraries(orc-executor PRIVATE orc-rt-executor)
diff --git a/libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_for_tested_elsewhere.pass.cpp b/orc-rt/tools/orc-executor/orc-executor.cpp
similarity index 54%
rename from libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_for_tested_elsewhere.pass.cpp
rename to orc-rt/tools/orc-executor/orc-executor.cpp
index 2be9cb4187fa2..92bdd5bf5d53f 100644
--- a/libcxx/test/std/thread/thread.threads/thread.thread.this/sleep_for_tested_elsewhere.pass.cpp
+++ b/orc-rt/tools/orc-executor/orc-executor.cpp
@@ -1,22 +1,18 @@
-//===----------------------------------------------------------------------===//
+//===- orc-executor.cpp - Placeholder implementation for orc-rt ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// Placeholder implementation file for initial orc-rt checkin.
+//
+//===----------------------------------------------------------------------===//
 
-// <thread>
-
-// template <class Rep, class Period>
-//   void sleep_for(const chrono::duration<Rep, Period>& rel_time);
-
-// The std::this_thread::sleep_for test requires POSIX specific headers and
-// is therefore non-standard. For this reason the test lives under the 'libcxx'
-// subdirectory.
-
-int main(int, char**)
-{
+#include "orc-rt-c/orc-rt.h"
 
+int main(int argc, char *argv[]) {
+  orc_rt();
   return 0;
 }
diff --git a/orc-rt/unittests/BitmaskEnumTest.cpp b/orc-rt/unittests/BitmaskEnumTest.cpp
new file mode 100644
index 0000000000000..f3e791de06c2d
--- /dev/null
+++ b/orc-rt/unittests/BitmaskEnumTest.cpp
@@ -0,0 +1,143 @@
+//===- BitmaskEnumTest.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Test bitmask-enum support APIs.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/BitmaskEnum.h"
+#include "gtest/gtest.h"
+
+#include <sstream>
+#include <string>
+
+using namespace orc_rt;
+
+namespace {
+
+enum Flags { F0 = 0, F1 = 1, F2 = 2, F3 = 4, F4 = 8 };
+
+} // namespace
+
+namespace orc_rt {
+ORC_RT_DECLARE_ENUM_AS_BITMASK(Flags, F4);
+} // namespace orc_rt
+
+static_assert(is_bitmask_enum<Flags>::value != 0);
+static_assert(largest_bitmask_enum_bit<Flags>::value == Flags::F4);
+
+namespace {
+
+static_assert(is_bitmask_enum<Flags>::value != 0);
+static_assert(largest_bitmask_enum_bit<Flags>::value == Flags::F4);
+
+TEST(BitmaskEnumTest, BitwiseOr) {
+  Flags f = F1 | F2;
+  EXPECT_EQ(3, f);
+
+  f = f | F3;
+  EXPECT_EQ(7, f);
+}
+
+TEST(BitmaskEnumTest, BitwiseOrEquals) {
+  Flags f = F1;
+  f |= F3;
+  EXPECT_EQ(5, f);
+
+  // |= should return a reference to the LHS.
+  f = F2;
+  (f |= F3) = F1;
+  EXPECT_EQ(F1, f);
+}
+
+TEST(BitmaskEnumTest, BitwiseAnd) {
+  Flags f = static_cast<Flags>(3) & F2;
+  EXPECT_EQ(F2, f);
+
+  f = (f | F3) & (F1 | F2 | F3);
+  EXPECT_EQ(6, f);
+}
+
+TEST(BitmaskEnumTest, BitwiseAndEquals) {
+  Flags f = F1 | F2 | F3;
+  f &= F1 | F2;
+  EXPECT_EQ(3, f);
+
+  // &= should return a reference to the LHS.
+  (f &= F1) = F3;
+  EXPECT_EQ(F3, f);
+}
+
+TEST(BitmaskEnumTest, BitwiseXor) {
+  Flags f = (F1 | F2) ^ (F2 | F3);
+  EXPECT_EQ(5, f);
+
+  f = f ^ F1;
+  EXPECT_EQ(4, f);
+}
+
+TEST(BitmaskEnumTest, BitwiseXorEquals) {
+  Flags f = (F1 | F2);
+  f ^= (F2 | F4);
+  EXPECT_EQ(9, f);
+
+  // ^= should return a reference to the LHS.
+  (f ^= F4) = F3;
+  EXPECT_EQ(F3, f);
+}
+
+TEST(BitmaskEnumTest, ConstantExpression) {
+  constexpr Flags f1 = ~F1;
+  constexpr Flags f2 = F1 | F2;
+  constexpr Flags f3 = F1 & F2;
+  constexpr Flags f4 = F1 ^ F2;
+  EXPECT_EQ(f1, ~F1);
+  EXPECT_EQ(f2, F1 | F2);
+  EXPECT_EQ(f3, F1 & F2);
+  EXPECT_EQ(f4, F1 ^ F2);
+}
+
+TEST(BitmaskEnumTest, BitwiseNot) {
+  Flags f = ~F1;
+  EXPECT_EQ(14, f); // Largest value for f is 15.
+  EXPECT_EQ(15, ~F0);
+}
+
+enum class FlagsClass {
+  F0 = 0,
+  F1 = 1,
+  F2 = 2,
+  F3 = 4,
+  ORC_RT_MARK_AS_BITMASK_ENUM(F3)
+};
+
+TEST(BitmaskEnumTest, ScopedEnum) {
+  FlagsClass f = (FlagsClass::F1 & ~FlagsClass::F0) | FlagsClass::F2;
+  f |= FlagsClass::F3;
+  EXPECT_EQ(7, static_cast<int>(f));
+}
+
+struct Container {
+  enum Flags {
+    F0 = 0,
+    F1 = 1,
+    F2 = 2,
+    F3 = 4,
+    ORC_RT_MARK_AS_BITMASK_ENUM(F3)
+  };
+
+  static Flags getFlags() {
+    Flags f = F0 | F1;
+    f |= F2;
+    return f;
+  }
+};
+
+TEST(BitmaskEnumTest, EnumInStruct) { EXPECT_EQ(3, Container::getFlags()); }
+
+} // namespace
diff --git a/orc-rt/unittests/CMakeLists.txt b/orc-rt/unittests/CMakeLists.txt
new file mode 100644
index 0000000000000..0fd0d09d10aa5
--- /dev/null
+++ b/orc-rt/unittests/CMakeLists.txt
@@ -0,0 +1,29 @@
+add_custom_target(OrcRTUnitTests)
+set_target_properties(OrcRTUnitTests PROPERTIES FOLDER "orc-rt/Tests")
+
+if (NOT TARGET llvm_gtest)
+  message(WARNING "orc-rt unittests disabled due to GTest being unavailable; "
+                  "Try LLVM_INSTALL_GTEST=ON for the LLVM build")
+  return ()
+endif ()
+
+function(add_orc_rt_unittest test_dirname)
+  add_unittest(OrcRTUnitTests ${test_dirname} ${ARGN})
+endfunction()
+
+add_orc_rt_unittest(CoreTests
+  BitmaskEnumTest.cpp
+  ErrorTest.cpp
+  ExecutorAddressTest.cpp
+  IntervalMapTest.cpp
+  IntervalSetTest.cpp
+  MathTest.cpp
+  RTTITest.cpp
+  SimplePackedSerializationTest.cpp
+  WrapperFunctionResultTest.cpp
+  bit-test.cpp
+  move_only_function-test.cpp
+  span-test.cpp
+  DISABLE_LLVM_LINK_LLVM_DYLIB
+  )
+target_link_libraries(CoreTests PRIVATE orc-rt-executor)
diff --git a/orc-rt/unittests/ErrorTest.cpp b/orc-rt/unittests/ErrorTest.cpp
new file mode 100644
index 0000000000000..47c67b8a1d551
--- /dev/null
+++ b/orc-rt/unittests/ErrorTest.cpp
@@ -0,0 +1,408 @@
+//===- ErrorTest.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+// Note:
+//   This unit test was adapted from
+//   llvm/unittests/Support/ErrorTest.cpp
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/Error.h"
+#include "gtest/gtest.h"
+
+using namespace orc_rt;
+
+namespace {
+
+class CustomError : public RTTIExtends<CustomError, ErrorInfoBase> {
+public:
+  CustomError(int Info) : Info(Info) {}
+  std::string toString() const override {
+    return "CustomError (" + std::to_string(Info) + ")";
+  }
+  int getInfo() const { return Info; }
+
+protected:
+  int Info;
+};
+
+class CustomSubError : public RTTIExtends<CustomSubError, CustomError> {
+public:
+  CustomSubError(int Info, std::string ExtraInfo)
+      : RTTIExtends<CustomSubError, CustomError>(Info),
+        ExtraInfo(std::move(ExtraInfo)) {}
+
+  std::string toString() const override {
+    return "CustomSubError (" + std::to_string(Info) + ", " + ExtraInfo + ")";
+  }
+  const std::string &getExtraInfo() const { return ExtraInfo; }
+
+protected:
+  std::string ExtraInfo;
+};
+
+static Error handleCustomError(const CustomError &CE) {
+  return Error::success();
+}
+
+static void handleCustomErrorVoid(const CustomError &CE) {}
+
+static Error handleCustomErrorUP(std::unique_ptr<CustomError> CE) {
+  return Error::success();
+}
+
+static void handleCustomErrorUPVoid(std::unique_ptr<CustomError> CE) {}
+
+} // end anonymous namespace
+
+// Test that a checked success value doesn't cause any issues.
+TEST(Error, CheckedSuccess) {
+  Error E = Error::success();
+  EXPECT_FALSE(E) << "Unexpected error while testing Error 'Success'";
+}
+
+// Check that a consumed success value doesn't cause any issues.
+TEST(Error, ConsumeSuccess) { consumeError(Error::success()); }
+
+TEST(Error, ConsumeError) {
+  Error E = make_error<CustomError>(42);
+  if (E) {
+    consumeError(std::move(E));
+  } else
+    ADD_FAILURE() << "Error failure value should convert to true";
+}
+
+// Test that unchecked success values cause an abort.
+TEST(Error, UncheckedSuccess) {
+  EXPECT_DEATH(
+      { Error E = Error::success(); },
+      "Error must be checked prior to destruction")
+      << "Unchecked Error Succes value did not cause abort()";
+}
+
+// Test that a checked but unhandled error causes an abort.
+TEST(Error, CheckedButUnhandledError) {
+  auto DropUnhandledError = []() {
+    Error E = make_error<CustomError>(42);
+    (void)!E;
+  };
+  EXPECT_DEATH(DropUnhandledError(),
+               "Error must be checked prior to destruction")
+      << "Unhandled Error failure value did not cause an abort()";
+}
+
+// Check that we can handle a custom error.
+TEST(Error, HandleCustomError) {
+  int CaughtErrorInfo = 0;
+  handleAllErrors(make_error<CustomError>(42), [&](const CustomError &CE) {
+    CaughtErrorInfo = CE.getInfo();
+  });
+
+  EXPECT_EQ(CaughtErrorInfo, 42) << "Wrong result from CustomError handler";
+}
+
+// Check that handler type deduction also works for handlers
+// of the following types:
+// void (const Err&)
+// Error (const Err&) mutable
+// void (const Err&) mutable
+// Error (Err&)
+// void (Err&)
+// Error (Err&) mutable
+// void (Err&) mutable
+// Error (unique_ptr<Err>)
+// void (unique_ptr<Err>)
+// Error (unique_ptr<Err>) mutable
+// void (unique_ptr<Err>) mutable
+TEST(Error, HandlerTypeDeduction) {
+
+  handleAllErrors(make_error<CustomError>(42), [](const CustomError &CE) {});
+
+  handleAllErrors(
+      make_error<CustomError>(42),
+      [](const CustomError &CE) mutable -> Error { return Error::success(); });
+
+  handleAllErrors(make_error<CustomError>(42),
+                  [](const CustomError &CE) mutable {});
+
+  handleAllErrors(make_error<CustomError>(42),
+                  [](CustomError &CE) -> Error { return Error::success(); });
+
+  handleAllErrors(make_error<CustomError>(42), [](CustomError &CE) {});
+
+  handleAllErrors(
+      make_error<CustomError>(42),
+      [](CustomError &CE) mutable -> Error { return Error::success(); });
+
+  handleAllErrors(make_error<CustomError>(42), [](CustomError &CE) mutable {});
+
+  handleAllErrors(make_error<CustomError>(42),
+                  [](std::unique_ptr<CustomError> CE) -> Error {
+                    return Error::success();
+                  });
+
+  handleAllErrors(make_error<CustomError>(42),
+                  [](std::unique_ptr<CustomError> CE) {});
+
+  handleAllErrors(make_error<CustomError>(42),
+                  [](std::unique_ptr<CustomError> CE) mutable -> Error {
+                    return Error::success();
+                  });
+
+  handleAllErrors(make_error<CustomError>(42),
+                  [](std::unique_ptr<CustomError> CE) mutable {});
+
+  // Check that named handlers of type 'Error (const Err&)' work.
+  handleAllErrors(make_error<CustomError>(42), handleCustomError);
+
+  // Check that named handlers of type 'void (const Err&)' work.
+  handleAllErrors(make_error<CustomError>(42), handleCustomErrorVoid);
+
+  // Check that named handlers of type 'Error (std::unique_ptr<Err>)' work.
+  handleAllErrors(make_error<CustomError>(42), handleCustomErrorUP);
+
+  // Check that named handlers of type 'Error (std::unique_ptr<Err>)' work.
+  handleAllErrors(make_error<CustomError>(42), handleCustomErrorUPVoid);
+}
+
+// Test that we can handle errors with custom base classes.
+TEST(Error, HandleCustomErrorWithCustomBaseClass) {
+  int CaughtErrorInfo = 0;
+  std::string CaughtErrorExtraInfo;
+  handleAllErrors(make_error<CustomSubError>(42, "foo"),
+                  [&](const CustomSubError &SE) {
+                    CaughtErrorInfo = SE.getInfo();
+                    CaughtErrorExtraInfo = SE.getExtraInfo();
+                  });
+
+  EXPECT_EQ(CaughtErrorInfo, 42) << "Wrong result from CustomSubError handler";
+  EXPECT_EQ(CaughtErrorExtraInfo, "foo")
+      << "Wrong result from CustomSubError handler";
+}
+
+// Check that we trigger only the first handler that applies.
+TEST(Error, FirstHandlerOnly) {
+  int DummyInfo = 0;
+  int CaughtErrorInfo = 0;
+  std::string CaughtErrorExtraInfo;
+
+  handleAllErrors(
+      make_error<CustomSubError>(42, "foo"),
+      [&](const CustomSubError &SE) {
+        CaughtErrorInfo = SE.getInfo();
+        CaughtErrorExtraInfo = SE.getExtraInfo();
+      },
+      [&](const CustomError &CE) { DummyInfo = CE.getInfo(); });
+
+  EXPECT_EQ(CaughtErrorInfo, 42) << "Activated the wrong Error handler(s)";
+  EXPECT_EQ(CaughtErrorExtraInfo, "foo")
+      << "Activated the wrong Error handler(s)";
+  EXPECT_EQ(DummyInfo, 0) << "Activated the wrong Error handler(s)";
+}
+
+// Check that general handlers shadow specific ones.
+TEST(Error, HandlerShadowing) {
+  int CaughtErrorInfo = 0;
+  int DummyInfo = 0;
+  std::string DummyExtraInfo;
+
+  handleAllErrors(
+      make_error<CustomSubError>(42, "foo"),
+      [&](const CustomError &CE) { CaughtErrorInfo = CE.getInfo(); },
+      [&](const CustomSubError &SE) {
+        DummyInfo = SE.getInfo();
+        DummyExtraInfo = SE.getExtraInfo();
+      });
+
+  EXPECT_EQ(CaughtErrorInfo, 42)
+      << "General Error handler did not shadow specific handler";
+  EXPECT_EQ(DummyInfo, 0)
+      << "General Error handler did not shadow specific handler";
+  EXPECT_EQ(DummyExtraInfo, "")
+      << "General Error handler did not shadow specific handler";
+}
+
+// ErrorAsOutParameter tester.
+static void errAsOutParamHelper(Error &Err) {
+  ErrorAsOutParameter ErrAsOutParam(&Err);
+  // Verify that checked flag is raised - assignment should not crash.
+  Err = Error::success();
+  // Raise the checked bit manually - caller should still have to test the
+  // error.
+  (void)!!Err;
+}
+
+// Test that ErrorAsOutParameter sets the checked flag on construction.
+TEST(Error, ErrorAsOutParameterChecked) {
+  Error E = Error::success();
+  errAsOutParamHelper(E);
+  (void)!!E;
+}
+
+// Test that ErrorAsOutParameter clears the checked flag on destruction.
+TEST(Error, ErrorAsOutParameterUnchecked) {
+  EXPECT_DEATH(
+      {
+        Error E = Error::success();
+        errAsOutParamHelper(E);
+      },
+      "Error must be checked prior to destruction")
+      << "ErrorAsOutParameter did not clear the checked flag on destruction.";
+}
+
+// Check 'Error::isA<T>' method handling.
+TEST(Error, IsAHandling) {
+  // Check 'isA' handling.
+  Error E = make_error<CustomError>(42);
+  Error F = make_error<CustomSubError>(42, "foo");
+  Error G = Error::success();
+
+  EXPECT_TRUE(E.isA<CustomError>());
+  EXPECT_FALSE(E.isA<CustomSubError>());
+  EXPECT_TRUE(F.isA<CustomError>());
+  EXPECT_TRUE(F.isA<CustomSubError>());
+  EXPECT_FALSE(G.isA<CustomError>());
+
+  consumeError(std::move(E));
+  consumeError(std::move(F));
+  consumeError(std::move(G));
+}
+
+TEST(Error, StringError) {
+  auto E = make_error<StringError>("foo");
+  if (E.isA<StringError>())
+    EXPECT_EQ(toString(std::move(E)), "foo") << "Unexpected StringError value";
+  else
+    ADD_FAILURE() << "Expected StringError value";
+}
+
+// Test Checked Expected<T> in success mode.
+TEST(Error, CheckedExpectedInSuccessMode) {
+  Expected<int> A = 7;
+  EXPECT_TRUE(!!A) << "Expected with non-error value doesn't convert to 'true'";
+  // Access is safe in second test, since we checked the error in the first.
+  EXPECT_EQ(*A, 7) << "Incorrect Expected non-error value";
+}
+
+// Test Expected with reference type.
+TEST(Error, ExpectedWithReferenceType) {
+  int A = 7;
+  Expected<int &> B = A;
+  // 'Check' B.
+  (void)!!B;
+  int &C = *B;
+  EXPECT_EQ(&A, &C) << "Expected failed to propagate reference";
+}
+
+// Test Unchecked Expected<T> in success mode.
+// We expect this to blow up the same way Error would.
+// Test runs in debug mode only.
+TEST(Error, UncheckedExpectedInSuccessModeDestruction) {
+  EXPECT_DEATH(
+      { Expected<int> A = 7; },
+      "Expected<T> must be checked before access or destruction.")
+      << "Unchecekd Expected<T> success value did not cause an abort().";
+}
+
+// Test Unchecked Expected<T> in success mode.
+// We expect this to blow up the same way Error would.
+// Test runs in debug mode only.
+TEST(Error, UncheckedExpectedInSuccessModeAccess) {
+  EXPECT_DEATH(
+      {
+        Expected<int> A = 7;
+        *A;
+      },
+      "Expected<T> must be checked before access or destruction.")
+      << "Unchecekd Expected<T> success value did not cause an abort().";
+}
+
+// Test Unchecked Expected<T> in success mode.
+// We expect this to blow up the same way Error would.
+// Test runs in debug mode only.
+TEST(Error, UncheckedExpectedInSuccessModeAssignment) {
+  EXPECT_DEATH(
+      {
+        Expected<int> A = 7;
+        A = 7;
+      },
+      "Expected<T> must be checked before access or destruction.")
+      << "Unchecekd Expected<T> success value did not cause an abort().";
+}
+
+// Test Expected<T> in failure mode.
+TEST(Error, ExpectedInFailureMode) {
+  Expected<int> A = make_error<CustomError>(42);
+  EXPECT_FALSE(!!A) << "Expected with error value doesn't convert to 'false'";
+  Error E = A.takeError();
+  EXPECT_TRUE(E.isA<CustomError>()) << "Incorrect Expected error value";
+  consumeError(std::move(E));
+}
+
+// Check that an Expected instance with an error value doesn't allow access to
+// operator*.
+// Test runs in debug mode only.
+TEST(Error, AccessExpectedInFailureMode) {
+  Expected<int> A = make_error<CustomError>(42);
+  EXPECT_DEATH(*A, "Expected<T> must be checked before access or destruction.")
+      << "Incorrect Expected error value";
+  consumeError(A.takeError());
+}
+
+// Check that an Expected instance with an error triggers an abort if
+// unhandled.
+// Test runs in debug mode only.
+TEST(Error, UnhandledExpectedInFailureMode) {
+  EXPECT_DEATH(
+      { Expected<int> A = make_error<CustomError>(42); },
+      "Expected<T> must be checked before access or destruction.")
+      << "Unchecked Expected<T> failure value did not cause an abort()";
+}
+
+// Test covariance of Expected.
+TEST(Error, ExpectedCovariance) {
+  class B {};
+  class D : public B {};
+
+  Expected<B *> A1(Expected<D *>(nullptr));
+  // Check A1 by converting to bool before assigning to it.
+  (void)!!A1;
+  A1 = Expected<D *>(nullptr);
+  // Check A1 again before destruction.
+  (void)!!A1;
+
+  Expected<std::unique_ptr<B>> A2(Expected<std::unique_ptr<D>>(nullptr));
+  // Check A2 by converting to bool before assigning to it.
+  (void)!!A2;
+  A2 = Expected<std::unique_ptr<D>>(nullptr);
+  // Check A2 again before destruction.
+  (void)!!A2;
+}
+
+// Test that the ExitOnError utility works as expected.
+TEST(Error, CantFailSuccess) {
+  cantFail(Error::success());
+
+  int X = cantFail(Expected<int>(42));
+  EXPECT_EQ(X, 42) << "Expected value modified by cantFail";
+
+  int Dummy = 42;
+  int &Y = cantFail(Expected<int &>(Dummy));
+  EXPECT_EQ(&Dummy, &Y) << "Reference mangled by cantFail";
+}
+
+// Test that cantFail results in a crash if you pass it a failure value.
+TEST(Error, CantFailDeath) {
+  EXPECT_DEATH(cantFail(make_error<StringError>("foo")), "")
+      << "cantFail(Error) did not cause an abort for failure value";
+
+  EXPECT_DEATH(cantFail(Expected<int>(make_error<StringError>("foo"))), "")
+      << "cantFail(Expected<int>) did not cause an abort for failure value";
+}
diff --git a/orc-rt/unittests/ExecutorAddressTest.cpp b/orc-rt/unittests/ExecutorAddressTest.cpp
new file mode 100644
index 0000000000000..98074a7617de4
--- /dev/null
+++ b/orc-rt/unittests/ExecutorAddressTest.cpp
@@ -0,0 +1,111 @@
+//===- executorAddressTest.cpp --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Test ExecutorAddress.h APIs.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/ExecutorAddress.h"
+#include "gtest/gtest.h"
+
+using namespace orc_rt;
+
+TEST(ExecutorAddrTest, DefaultAndNull) {
+  // Check that default constructed values and isNull behave as expected.
+
+  ExecutorAddr Default;
+  ExecutorAddr Null(0);
+  ExecutorAddr NonNull(1);
+
+  EXPECT_TRUE(Null.isNull());
+  EXPECT_EQ(Default, Null);
+
+  EXPECT_FALSE(NonNull.isNull());
+  EXPECT_NE(Default, NonNull);
+}
+
+TEST(ExecutorAddrTest, Ordering) {
+  // Check that ordering operations.
+  ExecutorAddr A1(1), A2(2);
+
+  EXPECT_LE(A1, A1);
+  EXPECT_LT(A1, A2);
+  EXPECT_GT(A2, A1);
+  EXPECT_GE(A2, A2);
+}
+
+TEST(ExecutorAddrTest, PtrConversion) {
+  // Test toPtr / fromPtr round-tripping.
+  int X = 0;
+  auto XAddr = ExecutorAddr::fromPtr(&X);
+  int *XPtr = XAddr.toPtr<int *>();
+
+  EXPECT_EQ(XPtr, &X);
+}
+
+static void F() {}
+
+TEST(ExecutorAddrTest, PtrConversionWithFunctionType) {
+  // Test that function types (as opposed to function pointer types) can be
+  // used with toPtr.
+  auto FAddr = ExecutorAddr::fromPtr(F);
+  void (*FPtr)() = FAddr.toPtr<void()>();
+
+  EXPECT_EQ(FPtr, &F);
+}
+
+TEST(ExecutorAddrTest, WrappingAndUnwrapping) {
+  constexpr uintptr_t RawAddr = 0x123456;
+  int *RawPtr = (int *)RawAddr;
+
+  constexpr uintptr_t TagOffset = 8 * (sizeof(uintptr_t) - 1);
+  uintptr_t TagVal = 0xA5;
+  uintptr_t TagBits = TagVal << TagOffset;
+  void *TaggedPtr = (void *)((uintptr_t)RawPtr | TagBits);
+
+  ExecutorAddr EA =
+      ExecutorAddr::fromPtr(TaggedPtr, ExecutorAddr::Untag(8, TagOffset));
+
+  EXPECT_EQ(EA.getValue(), RawAddr);
+
+  void *ReconstitutedTaggedPtr =
+      EA.toPtr<void *>(ExecutorAddr::Tag(TagVal, TagOffset));
+
+  EXPECT_EQ(TaggedPtr, ReconstitutedTaggedPtr);
+}
+
+TEST(ExecutorAddrTest, AddrRanges) {
+  ExecutorAddr A0(0), A1(1), A2(2), A3(3);
+  ExecutorAddrRange R0(A0, A1), R1(A1, A2), R2(A2, A3), R3(A0, A2), R4(A1, A3);
+  //     012
+  // R0: #      -- Before R1
+  // R1:  #     --
+  // R2:   #    -- After R1
+  // R3: ##     -- Overlaps R1 start
+  // R4:  ##    -- Overlaps R1 end
+
+  EXPECT_EQ(R1, ExecutorAddrRange(A1, A2));
+  EXPECT_EQ(R1, ExecutorAddrRange(A1, ExecutorAddrDiff(1)));
+  EXPECT_NE(R1, R2);
+
+  EXPECT_TRUE(R1.contains(A1));
+  EXPECT_FALSE(R1.contains(A0));
+  EXPECT_FALSE(R1.contains(A2));
+
+  EXPECT_FALSE(R1.overlaps(R0));
+  EXPECT_FALSE(R1.overlaps(R2));
+  EXPECT_TRUE(R1.overlaps(R3));
+  EXPECT_TRUE(R1.overlaps(R4));
+}
+
+TEST(ExecutorAddrTest, Hashable) {
+  uint64_t RawAddr = 0x1234567890ABCDEF;
+  ExecutorAddr Addr(RawAddr);
+
+  EXPECT_EQ(std::hash<uint64_t>()(RawAddr), std::hash<ExecutorAddr>()(Addr));
+}
diff --git a/orc-rt/unittests/IntervalMapTest.cpp b/orc-rt/unittests/IntervalMapTest.cpp
new file mode 100644
index 0000000000000..4b40c8ca9a912
--- /dev/null
+++ b/orc-rt/unittests/IntervalMapTest.cpp
@@ -0,0 +1,204 @@
+//===- IntervalMapTest.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/IntervalMap.h"
+#include "gtest/gtest.h"
+
+using namespace orc_rt;
+
+TEST(IntervalMapTest, DefaultConstructed) {
+  // Check that a default-constructed IntervalMap behaves as expected.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+  EXPECT_TRUE(M.empty());
+  EXPECT_TRUE(M.begin() == M.end());
+  EXPECT_TRUE(M.find(0) == M.end());
+}
+
+TEST(IntervalMapTest, InsertSingleElement) {
+  // Check that a map with a single element inserted behaves as expected.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+  M.insert(7, 8, 42);
+
+  EXPECT_FALSE(M.empty());
+  EXPECT_EQ(std::next(M.begin()), M.end());
+  EXPECT_EQ(M.find(7), M.begin());
+  EXPECT_EQ(M.find(8), M.end());
+  EXPECT_EQ(M.lookup(7), 42U);
+  EXPECT_EQ(M.lookup(8), 0U); // 8 not present, so should return unsigned().
+}
+
+TEST(IntervalMapTest, InsertCoalesceWithPrevious) {
+  // Check that insertions coalesce with previous ranges that share the same
+  // value. Also check that they _don't_ coalesce if the values are different.
+
+  // Check that insertion coalesces with previous range when values are equal.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M1;
+
+  M1.insert(7, 8, 42);
+  M1.insert(8, 9, 42);
+
+  EXPECT_FALSE(M1.empty());
+  EXPECT_EQ(std::next(M1.begin()), M1.end()); // Should see just one range.
+  EXPECT_EQ(M1.find(7), M1.find(8)); // 7 and 8 should point to same range.
+  EXPECT_EQ(M1.lookup(7), 42U);      // Value should be preserved.
+
+  // Check that insertion does not coalesce with previous range when values are
+  // not equal.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M2;
+
+  M2.insert(7, 8, 42);
+  M2.insert(8, 9, 7);
+
+  EXPECT_FALSE(M2.empty());
+  EXPECT_EQ(std::next(std::next(M2.begin())), M2.end()); // Expect two ranges.
+  EXPECT_NE(M2.find(7), M2.find(8)); // 7 and 8 should be different ranges.
+  EXPECT_EQ(M2.lookup(7), 42U); // Keys 7 and 8 should map to different values.
+  EXPECT_EQ(M2.lookup(8), 7U);
+}
+
+TEST(IntervalMapTest, InsertCoalesceWithFollowing) {
+  // Check that insertions coalesce with following ranges that share the same
+  // value. Also check that they _don't_ coalesce if the values are different.
+
+  // Check that insertion coalesces with following range when values are equal.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M1;
+
+  M1.insert(8, 9, 42);
+  M1.insert(7, 8, 42);
+
+  EXPECT_FALSE(M1.empty());
+  EXPECT_EQ(std::next(M1.begin()), M1.end()); // Should see just one range.
+  EXPECT_EQ(M1.find(7), M1.find(8)); // 7 and 8 should point to same range.
+  EXPECT_EQ(M1.lookup(7), 42U);      // Value should be preserved.
+
+  // Check that insertion does not coalesce with previous range when values are
+  // not equal.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M2;
+
+  M2.insert(8, 9, 42);
+  M2.insert(7, 8, 7);
+
+  EXPECT_FALSE(M2.empty());
+  EXPECT_EQ(std::next(std::next(M2.begin())), M2.end()); // Expect two ranges.
+  EXPECT_EQ(M2.lookup(7), 7U); // Keys 7 and 8 should map to different values.
+  EXPECT_EQ(M2.lookup(8), 42U);
+}
+
+TEST(IntervalMapTest, InsertCoalesceBoth) {
+  // Check that insertions coalesce with ranges on both sides where posssible.
+  // Also check that they _don't_ coalesce if the values are different.
+
+  // Check that insertion coalesces with both previous and following ranges
+  // when values are equal.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M1;
+
+  M1.insert(7, 8, 42);
+  M1.insert(9, 10, 42);
+
+  // Check no coalescing yet.
+  EXPECT_NE(M1.find(7), M1.find(9));
+
+  // Insert a 3rd range to trigger coalescing on both sides.
+  M1.insert(8, 9, 42);
+
+  EXPECT_FALSE(M1.empty());
+  EXPECT_EQ(std::next(M1.begin()), M1.end()); // Should see just one range.
+  EXPECT_EQ(M1.find(7), M1.find(8)); // 7, 8, and 9 should point to same range.
+  EXPECT_EQ(M1.find(8), M1.find(9));
+  EXPECT_EQ(M1.lookup(7), 42U); // Value should be preserved.
+
+  // Check that insertion does not coalesce with previous range when values are
+  // not equal.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M2;
+
+  M2.insert(7, 8, 42);
+  M2.insert(8, 9, 7);
+  M2.insert(9, 10, 42);
+
+  EXPECT_FALSE(M2.empty());
+  // Expect three ranges.
+  EXPECT_EQ(std::next(std::next(std::next(M2.begin()))), M2.end());
+  EXPECT_NE(M2.find(7), M2.find(8)); // All keys should map to different ranges.
+  EXPECT_NE(M2.find(8), M2.find(9));
+  EXPECT_EQ(M2.lookup(7), 42U); // Key 7, 8, and 9 should map to different vals.
+  EXPECT_EQ(M2.lookup(8), 7U);
+  EXPECT_EQ(M2.lookup(9), 42U);
+}
+
+TEST(IntervalMapTest, EraseSingleElement) {
+  // Check that we can insert and then remove a single range.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+  M.insert(7, 10, 42);
+  EXPECT_FALSE(M.empty());
+  M.erase(7, 10);
+  EXPECT_TRUE(M.empty());
+}
+
+TEST(IntervalMapTest, EraseSplittingLeft) {
+  // Check that removal of a trailing subrange succeeds, but leaves the
+  // residual range in-place.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+  M.insert(7, 10, 42);
+  EXPECT_FALSE(M.empty());
+  M.erase(9, 10);
+  EXPECT_EQ(std::next(M.begin()), M.end());
+  EXPECT_EQ(M.begin()->first.first, 7U);
+  EXPECT_EQ(M.begin()->first.second, 9U);
+}
+
+TEST(IntervalMapTest, EraseSplittingRight) {
+  // Check that removal of a leading subrange succeeds, but leaves the
+  // residual range in-place.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+  M.insert(7, 10, 42);
+  EXPECT_FALSE(M.empty());
+  M.erase(7, 8);
+  EXPECT_EQ(std::next(M.begin()), M.end());
+  EXPECT_EQ(M.begin()->first.first, 8U);
+  EXPECT_EQ(M.begin()->first.second, 10U);
+}
+
+TEST(IntervalMapTest, EraseSplittingBoth) {
+  // Check that removal of an interior subrange leaves both the leading and
+  // trailing residual subranges in-place.
+  IntervalMap<unsigned, unsigned, IntervalCoalescing::Enabled> M;
+
+  M.insert(7, 10, 42);
+  EXPECT_FALSE(M.empty());
+  M.erase(8, 9);
+  EXPECT_EQ(std::next(std::next(M.begin())), M.end());
+  EXPECT_EQ(M.begin()->first.first, 7U);
+  EXPECT_EQ(M.begin()->first.second, 8U);
+  EXPECT_EQ(std::next(M.begin())->first.first, 9U);
+  EXPECT_EQ(std::next(M.begin())->first.second, 10U);
+}
+
+TEST(IntervalMapTest, NonCoalescingMapPermitsNonComparableKeys) {
+  // Test that values that can't be equality-compared are still usable when
+  // coalescing is disabled and behave as expected.
+
+  struct S {}; // Struct with no equality comparison.
+
+  IntervalMap<unsigned, S, IntervalCoalescing::Disabled> M;
+
+  M.insert(7, 8, S());
+
+  EXPECT_FALSE(M.empty());
+  EXPECT_EQ(std::next(M.begin()), M.end());
+  EXPECT_EQ(M.find(7), M.begin());
+  EXPECT_EQ(M.find(8), M.end());
+}
diff --git a/orc-rt/unittests/IntervalSetTest.cpp b/orc-rt/unittests/IntervalSetTest.cpp
new file mode 100644
index 0000000000000..fb9babe8850de
--- /dev/null
+++ b/orc-rt/unittests/IntervalSetTest.cpp
@@ -0,0 +1,121 @@
+//===- IntervalSetTest.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/IntervalSet.h"
+#include "gtest/gtest.h"
+
+using namespace orc_rt;
+
+TEST(IntervalSetTest, DefaultConstructed) {
+  // Check that a default-constructed IntervalSet behaves as expected.
+  IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+  EXPECT_TRUE(S.empty());
+  EXPECT_TRUE(S.begin() == S.end());
+  EXPECT_TRUE(S.find(0) == S.end());
+}
+
+TEST(IntervalSetTest, InsertSingleElement) {
+  // Check that a set with a single element inserted behaves as expected.
+  IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+  S.insert(7, 8);
+
+  EXPECT_FALSE(S.empty());
+  EXPECT_EQ(std::next(S.begin()), S.end());
+  EXPECT_EQ(S.find(7), S.begin());
+  EXPECT_EQ(S.find(8), S.end());
+}
+
+TEST(IntervalSetTest, InsertCoalesceWithPrevious) {
+  // Check that insertions coalesce with previous ranges.
+  IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+  S.insert(7, 8);
+  S.insert(8, 9);
+
+  EXPECT_FALSE(S.empty());
+  EXPECT_EQ(std::next(S.begin()), S.end()); // Should see just one range.
+  EXPECT_EQ(S.find(7), S.find(8)); // 7 and 8 should point to same range.
+}
+
+TEST(IntervalSetTest, InsertCoalesceWithFollowing) {
+  // Check that insertions coalesce with following ranges.
+  IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+  S.insert(8, 9);
+  S.insert(7, 8);
+
+  EXPECT_FALSE(S.empty());
+  EXPECT_EQ(std::next(S.begin()), S.end()); // Should see just one range.
+  EXPECT_EQ(S.find(7), S.find(8)); // 7 and 8 should point to same range.
+}
+
+TEST(IntervalSetTest, InsertCoalesceBoth) {
+  // Check that insertions coalesce with ranges on both sides.
+  IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+  S.insert(7, 8);
+  S.insert(9, 10);
+
+  // Check no coalescing yet.
+  EXPECT_NE(S.find(7), S.find(9));
+
+  // Insert a 3rd range to trigger coalescing on both sides.
+  S.insert(8, 9);
+
+  EXPECT_FALSE(S.empty());
+  EXPECT_EQ(std::next(S.begin()), S.end()); // Should see just one range.
+  EXPECT_EQ(S.find(7), S.find(8)); // 7, 8, and 9 should point to same range.
+  EXPECT_EQ(S.find(8), S.find(9));
+}
+
+TEST(IntervalSetTest, EraseSplittingLeft) {
+  // Check that removal of a trailing subrange succeeds, but leaves the
+  // residual range in-place.
+  IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+  S.insert(7, 10);
+  EXPECT_FALSE(S.empty());
+  S.erase(9, 10);
+  EXPECT_EQ(std::next(S.begin()), S.end());
+  EXPECT_EQ(S.begin()->first, 7U);
+  EXPECT_EQ(S.begin()->second, 9U);
+}
+
+TEST(IntervalSetTest, EraseSplittingRight) {
+  // Check that removal of a leading subrange succeeds, but leaves the
+  // residual range in-place.
+  IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+  S.insert(7, 10);
+  EXPECT_FALSE(S.empty());
+  S.erase(7, 8);
+  EXPECT_EQ(std::next(S.begin()), S.end());
+  EXPECT_EQ(S.begin()->first, 8U);
+  EXPECT_EQ(S.begin()->second, 10U);
+}
+
+TEST(IntervalSetTest, EraseSplittingBoth) {
+  // Check that removal of an interior subrange leaves both the leading and
+  // trailing residual subranges in-place.
+  IntervalSet<unsigned, IntervalCoalescing::Enabled> S;
+
+  S.insert(7, 10);
+  EXPECT_FALSE(S.empty());
+  S.erase(8, 9);
+  EXPECT_EQ(std::next(std::next(S.begin())), S.end());
+  EXPECT_EQ(S.begin()->first, 7U);
+  EXPECT_EQ(S.begin()->second, 8U);
+  EXPECT_EQ(std::next(S.begin())->first, 9U);
+  EXPECT_EQ(std::next(S.begin())->second, 10U);
+}
diff --git a/orc-rt/unittests/MathTest.cpp b/orc-rt/unittests/MathTest.cpp
new file mode 100644
index 0000000000000..f64aacd092477
--- /dev/null
+++ b/orc-rt/unittests/MathTest.cpp
@@ -0,0 +1,78 @@
+//===- MathTest.cpp -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tests for orc-rt's Math.h APIs.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/Math.h"
+#include "gtest/gtest.h"
+
+using namespace orc_rt;
+
+TEST(MathTest, isPowerOf2) {
+  // Test [0..16]
+  EXPECT_FALSE(isPowerOf2(0x00));
+  EXPECT_TRUE(isPowerOf2(0x01));
+  EXPECT_TRUE(isPowerOf2(0x02));
+  EXPECT_FALSE(isPowerOf2(0x03));
+  EXPECT_TRUE(isPowerOf2(0x04));
+  EXPECT_FALSE(isPowerOf2(0x05));
+  EXPECT_FALSE(isPowerOf2(0x06));
+  EXPECT_FALSE(isPowerOf2(0x07));
+  EXPECT_TRUE(isPowerOf2(0x08));
+  EXPECT_FALSE(isPowerOf2(0x09));
+  EXPECT_FALSE(isPowerOf2(0x0A));
+  EXPECT_FALSE(isPowerOf2(0x0B));
+  EXPECT_FALSE(isPowerOf2(0x0C));
+  EXPECT_FALSE(isPowerOf2(0x0D));
+  EXPECT_FALSE(isPowerOf2(0x0E));
+  EXPECT_FALSE(isPowerOf2(0x0F));
+  EXPECT_TRUE(isPowerOf2(0x10));
+
+  // Test some higher powers of two and their adjacent values.
+  EXPECT_FALSE(isPowerOf2(0x1F));
+  EXPECT_TRUE(isPowerOf2(0x20));
+  EXPECT_FALSE(isPowerOf2(0x21));
+
+  EXPECT_FALSE(isPowerOf2(0x3F));
+  EXPECT_TRUE(isPowerOf2(0x40));
+  EXPECT_FALSE(isPowerOf2(0x41));
+
+  EXPECT_FALSE(isPowerOf2(0x7F));
+  EXPECT_TRUE(isPowerOf2(0x80));
+  EXPECT_FALSE(isPowerOf2(0x81));
+
+  // Test larger values.
+  EXPECT_FALSE(isPowerOf2(0x3fffffff));
+  EXPECT_TRUE(isPowerOf2(0x40000000));
+  EXPECT_FALSE(isPowerOf2(0x40000001));
+
+  // Test negatives.
+  EXPECT_FALSE(isPowerOf2(-1));
+}
+
+TEST(MathTest, nextPowerOf2) {
+  EXPECT_EQ(nextPowerOf2(0x00), (1 << 0));
+  EXPECT_EQ(nextPowerOf2(0x01), (1 << 1));
+  EXPECT_EQ(nextPowerOf2(0x02), (1 << 2));
+  EXPECT_EQ(nextPowerOf2(0x03), (1 << 2));
+  EXPECT_EQ(nextPowerOf2(0x04), (1 << 3));
+  EXPECT_EQ(nextPowerOf2(0x05), (1 << 3));
+  EXPECT_EQ(nextPowerOf2(0x06), (1 << 3));
+  EXPECT_EQ(nextPowerOf2(0x07), (1 << 3));
+  EXPECT_EQ(nextPowerOf2(0x08), (1 << 4));
+  EXPECT_EQ(nextPowerOf2(0x09), (1 << 4));
+  EXPECT_EQ(nextPowerOf2(0x0a), (1 << 4));
+  EXPECT_EQ(nextPowerOf2(0x0b), (1 << 4));
+  EXPECT_EQ(nextPowerOf2(0x0c), (1 << 4));
+  EXPECT_EQ(nextPowerOf2(0x0d), (1 << 4));
+  EXPECT_EQ(nextPowerOf2(0x0e), (1 << 4));
+  EXPECT_EQ(nextPowerOf2(0x0f), (1 << 4));
+  EXPECT_EQ(nextPowerOf2(0x10), (1 << 5));
+}
diff --git a/orc-rt/unittests/RTTITest.cpp b/orc-rt/unittests/RTTITest.cpp
new file mode 100644
index 0000000000000..445efee887268
--- /dev/null
+++ b/orc-rt/unittests/RTTITest.cpp
@@ -0,0 +1,52 @@
+//===-- rtti_test.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Note:
+//   This unit test was adapted from
+//   llvm/unittests/Support/ExtensibleRTTITest.cpp
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/RTTI.h"
+#include "gtest/gtest.h"
+
+using namespace orc_rt;
+
+namespace {
+
+class MyBase : public RTTIExtends<MyBase, RTTIRoot> {};
+
+class MyDerivedA : public RTTIExtends<MyDerivedA, MyBase> {};
+
+class MyDerivedB : public RTTIExtends<MyDerivedB, MyBase> {};
+
+} // end anonymous namespace
+
+TEST(ExtensibleRTTITest, BaseCheck) {
+  MyBase MB;
+  MyDerivedA MDA;
+  MyDerivedB MDB;
+
+  // Check MB properties.
+  EXPECT_TRUE(isa<RTTIRoot>(MB));
+  EXPECT_TRUE(isa<MyBase>(MB));
+  EXPECT_FALSE(isa<MyDerivedA>(MB));
+  EXPECT_FALSE(isa<MyDerivedB>(MB));
+
+  // Check MDA properties.
+  EXPECT_TRUE(isa<RTTIRoot>(MDA));
+  EXPECT_TRUE(isa<MyBase>(MDA));
+  EXPECT_TRUE(isa<MyDerivedA>(MDA));
+  EXPECT_FALSE(isa<MyDerivedB>(MDA));
+
+  // Check MDB properties.
+  EXPECT_TRUE(isa<RTTIRoot>(MDB));
+  EXPECT_TRUE(isa<MyBase>(MDB));
+  EXPECT_FALSE(isa<MyDerivedA>(MDB));
+  EXPECT_TRUE(isa<MyDerivedB>(MDB));
+}
diff --git a/orc-rt/unittests/SimplePackedSerializationTest.cpp b/orc-rt/unittests/SimplePackedSerializationTest.cpp
new file mode 100644
index 0000000000000..6c58503f0797a
--- /dev/null
+++ b/orc-rt/unittests/SimplePackedSerializationTest.cpp
@@ -0,0 +1,184 @@
+//===- SimplePackedSerializationTest.cpp ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tests for SimplePackedSerialization infrastructure.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/SimplePackedSerialization.h"
+#include "SimplePackedSerializationTestUtils.h"
+#include "gtest/gtest.h"
+
+using namespace orc_rt;
+
+TEST(SimplePackedSerializationTest, SPSOutputBuffer) {
+  constexpr unsigned NumBytes = 8;
+  char Buffer[NumBytes];
+  char Zero = 0;
+  SPSOutputBuffer OB(Buffer, NumBytes);
+
+  // Expect that we can write NumBytes of content.
+  for (unsigned I = 0; I != NumBytes; ++I) {
+    char C = I;
+    EXPECT_TRUE(OB.write(&C, 1));
+  }
+
+  // Expect an error when we attempt to write an extra byte.
+  EXPECT_FALSE(OB.write(&Zero, 1));
+
+  // Check that the buffer contains the expected content.
+  for (unsigned I = 0; I != NumBytes; ++I)
+    EXPECT_EQ(Buffer[I], (char)I);
+}
+
+TEST(SimplePackedSerializationTest, SPSInputBuffer) {
+  char Buffer[] = {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07};
+  SPSInputBuffer IB(Buffer, sizeof(Buffer));
+
+  char C;
+  for (unsigned I = 0; I != sizeof(Buffer); ++I) {
+    EXPECT_TRUE(IB.read(&C, 1));
+    EXPECT_EQ(C, (char)I);
+  }
+
+  EXPECT_FALSE(IB.read(&C, 1));
+}
+
+template <typename T> static void testFixedIntegralTypeSerialization() {
+  blobSerializationRoundTrip<T, T>(0);
+  blobSerializationRoundTrip<T, T>(static_cast<T>(1));
+  if (std::is_signed<T>::value) {
+    blobSerializationRoundTrip<T, T>(static_cast<T>(-1));
+    blobSerializationRoundTrip<T, T>(std::numeric_limits<T>::min());
+  }
+  blobSerializationRoundTrip<T, T>(std::numeric_limits<T>::max());
+}
+
+TEST(SimplePackedSerializationTest, BoolSerialization) {
+  blobSerializationRoundTrip<bool, bool>(true);
+  blobSerializationRoundTrip<bool, bool>(false);
+}
+
+TEST(SimplePackedSerializationTest, CharSerialization) {
+  blobSerializationRoundTrip<char, char>((char)0x00);
+  blobSerializationRoundTrip<char, char>((char)0xAA);
+  blobSerializationRoundTrip<char, char>((char)0xFF);
+}
+
+TEST(SimplePackedSerializationTest, Int8Serialization) {
+  testFixedIntegralTypeSerialization<int8_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt8Serialization) {
+  testFixedIntegralTypeSerialization<uint8_t>();
+}
+
+TEST(SimplePackedSerializationTest, Int16Serialization) {
+  testFixedIntegralTypeSerialization<int16_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt16Serialization) {
+  testFixedIntegralTypeSerialization<uint16_t>();
+}
+
+TEST(SimplePackedSerializationTest, Int32Serialization) {
+  testFixedIntegralTypeSerialization<int32_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt32Serialization) {
+  testFixedIntegralTypeSerialization<uint32_t>();
+}
+
+TEST(SimplePackedSerializationTest, Int64Serialization) {
+  testFixedIntegralTypeSerialization<int64_t>();
+}
+
+TEST(SimplePackedSerializationTest, UInt64Serialization) {
+  testFixedIntegralTypeSerialization<uint64_t>();
+}
+
+TEST(SimplePackedSerializationTest, SequenceSerialization) {
+  std::vector<int32_t> V({1, 2, -47, 139});
+  blobSerializationRoundTrip<SPSSequence<int32_t>, std::vector<int32_t>>(V);
+}
+
+TEST(SimplePackedSerializationTest, StringViewCharSequenceSerialization) {
+  const char *HW = "Hello, world!";
+  blobSerializationRoundTrip<SPSString, std::string_view>(std::string_view(HW));
+}
+
+TEST(SimplePackedSerializationTest, SpanSerialization) {
+  const char Data[] = {3, 2, 1, 0, 1, 2, 3}; // Span should handle nulls.
+  span<const char> OutS(Data, sizeof(Data));
+
+  size_t Size = SPSArgList<SPSSequence<char>>::size(OutS);
+  auto Buffer = std::make_unique<char[]>(Size);
+  SPSOutputBuffer OB(Buffer.get(), Size);
+
+  EXPECT_TRUE(SPSArgList<SPSSequence<char>>::serialize(OB, OutS));
+
+  SPSInputBuffer IB(Buffer.get(), Size);
+
+  span<const char> InS;
+
+  EXPECT_TRUE(SPSArgList<SPSSequence<char>>::deserialize(IB, InS));
+
+  // Check that the serialized and deserialized values match.
+  EXPECT_EQ(InS.size(), OutS.size());
+  EXPECT_EQ(memcmp(OutS.data(), InS.data(), InS.size()), 0);
+
+  // Check that the span points directly to the input buffer.
+  EXPECT_EQ(InS.data(), Buffer.get() + sizeof(uint64_t));
+}
+
+TEST(SimplePackedSerializationTest, StdTupleSerialization) {
+  std::tuple<int32_t, std::string, bool> P(42, "foo", true);
+  blobSerializationRoundTrip<SPSTuple<int32_t, SPSString, bool>>(P);
+}
+
+TEST(SimplePackedSerializationTest, StdPairSerialization) {
+  std::pair<int32_t, std::string> P(42, "foo");
+  blobSerializationRoundTrip<SPSTuple<int32_t, SPSString>,
+                             std::pair<int32_t, std::string>>(P);
+}
+
+TEST(SimplePackedSerializationTest, StdOptionalNoValueSerialization) {
+  std::optional<int64_t> NoValue;
+  blobSerializationRoundTrip<SPSOptional<int64_t>>(NoValue);
+}
+
+TEST(SimplePackedSerializationTest, StdOptionalValueSerialization) {
+  std::optional<int64_t> Value(42);
+  blobSerializationRoundTrip<SPSOptional<int64_t>>(Value);
+}
+
+TEST(SimplePackedSerializationTest, ArgListSerialization) {
+  using BAL = SPSArgList<bool, int32_t, SPSString>;
+
+  bool Arg1 = true;
+  int32_t Arg2 = 42;
+  std::string Arg3 = "foo";
+
+  size_t Size = BAL::size(Arg1, Arg2, Arg3);
+  auto Buffer = std::make_unique<char[]>(Size);
+  SPSOutputBuffer OB(Buffer.get(), Size);
+
+  EXPECT_TRUE(BAL::serialize(OB, Arg1, Arg2, Arg3));
+
+  SPSInputBuffer IB(Buffer.get(), Size);
+
+  bool ArgOut1;
+  int32_t ArgOut2;
+  std::string ArgOut3;
+
+  EXPECT_TRUE(BAL::deserialize(IB, ArgOut1, ArgOut2, ArgOut3));
+
+  EXPECT_EQ(Arg1, ArgOut1);
+  EXPECT_EQ(Arg2, ArgOut2);
+  EXPECT_EQ(Arg3, ArgOut3);
+}
diff --git a/orc-rt/unittests/SimplePackedSerializationTestUtils.h b/orc-rt/unittests/SimplePackedSerializationTestUtils.h
new file mode 100644
index 0000000000000..5468045f5fbe7
--- /dev/null
+++ b/orc-rt/unittests/SimplePackedSerializationTestUtils.h
@@ -0,0 +1,34 @@
+//===- SimplePackedSerializationTestUtils.h -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef ORC_RT_UNITTEST_SIMPLEPACKEDSERIALIZATIONTESTUTILS_H
+#define ORC_RT_UNITTEST_SIMPLEPACKEDSERIALIZATIONTESTUTILS_H
+
+#include "orc-rt/SimplePackedSerialization.h"
+#include "gtest/gtest.h"
+
+template <typename SPSTagT, typename T>
+static void blobSerializationRoundTrip(const T &Value) {
+  using BST = orc_rt::SPSSerializationTraits<SPSTagT, T>;
+
+  size_t Size = BST::size(Value);
+  auto Buffer = std::make_unique<char[]>(Size);
+  orc_rt::SPSOutputBuffer OB(Buffer.get(), Size);
+
+  EXPECT_TRUE(BST::serialize(OB, Value));
+
+  orc_rt::SPSInputBuffer IB(Buffer.get(), Size);
+
+  T DSValue;
+  EXPECT_TRUE(BST::deserialize(IB, DSValue));
+
+  EXPECT_EQ(Value, DSValue)
+      << "Incorrect value after serialization/deserialization round-trip";
+}
+
+#endif // ORC_RT_UNITTEST_SIMPLEPACKEDSERIALIZATIONTESTUTILS_H
diff --git a/orc-rt/unittests/WrapperFunctionResultTest.cpp b/orc-rt/unittests/WrapperFunctionResultTest.cpp
new file mode 100644
index 0000000000000..9ecc513ba50fa
--- /dev/null
+++ b/orc-rt/unittests/WrapperFunctionResultTest.cpp
@@ -0,0 +1,60 @@
+//===-- wrapper_function_utils_test.cpp -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of the ORC runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt-c/WrapperFunctionResult.h"
+#include "orc-rt/WrapperFunctionResult.h"
+#include "gtest/gtest.h"
+
+using namespace orc_rt;
+
+namespace {
+constexpr const char *TestString = "test string";
+} // end anonymous namespace
+
+TEST(WrapperFunctionUtilsTest, DefaultWrapperFunctionResult) {
+  WrapperFunctionResult R;
+  EXPECT_TRUE(R.empty());
+  EXPECT_EQ(R.size(), 0U);
+  EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromCStruct) {
+  orc_rt_WrapperFunctionResult CR =
+      orc_rt_CreateWrapperFunctionResultFromString(TestString);
+  WrapperFunctionResult R(CR);
+  EXPECT_EQ(R.size(), strlen(TestString) + 1);
+  EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+  EXPECT_FALSE(R.empty());
+  EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromRange) {
+  auto R = WrapperFunctionResult::copyFrom(TestString, strlen(TestString) + 1);
+  EXPECT_EQ(R.size(), strlen(TestString) + 1);
+  EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+  EXPECT_FALSE(R.empty());
+  EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromCString) {
+  auto R = WrapperFunctionResult::copyFrom(TestString);
+  EXPECT_EQ(R.size(), strlen(TestString) + 1);
+  EXPECT_TRUE(strcmp(R.data(), TestString) == 0);
+  EXPECT_FALSE(R.empty());
+  EXPECT_EQ(R.getOutOfBandError(), nullptr);
+}
+
+TEST(WrapperFunctionUtilsTest, WrapperFunctionResultFromOutOfBandError) {
+  auto R = WrapperFunctionResult::createOutOfBandError(TestString);
+  EXPECT_FALSE(R.empty());
+  EXPECT_TRUE(strcmp(R.getOutOfBandError(), TestString) == 0);
+}
diff --git a/orc-rt/unittests/bit-test.cpp b/orc-rt/unittests/bit-test.cpp
new file mode 100644
index 0000000000000..43662e0e641b8
--- /dev/null
+++ b/orc-rt/unittests/bit-test.cpp
@@ -0,0 +1,79 @@
+//===- BitTest.cpp --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tests for orc-rt's bit.h APIs.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/bit.h"
+#include "gtest/gtest.h"
+
+#include <cstdint>
+
+#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && BYTE_ORDER == BIG_ENDIAN
+#define IS_BIG_ENDIAN
+#elif defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) &&                         \
+    BYTE_ORDER == LITTLE_ENDIAN
+#define IS_LITTLE_ENDIAN
+#endif
+
+using namespace orc_rt;
+
+TEST(BitTest, endian) {
+#if defined(IS_BIG_ENDIAN)
+  EXPECT_EQ(endian::native, endian::big);
+#elif defined(IS_LITTLE_ENDIAN)
+  EXPECT_EQ(endian::native, endian::little);
+#else
+  ADD_FAILURE() << "BYTE_ORDER is neither BIG_ENDIAN nor LITTLE_ENDIAN.";
+#endif
+}
+
+TEST(MathTest, byte_swap_32) {
+  unsigned char Seq[] = {0x01, 0x23, 0x45, 0x67};
+  uint32_t X = 0;
+  memcpy(&X, Seq, sizeof(X));
+  auto Y = byteswap(X);
+
+  static_assert(sizeof(Seq) == sizeof(X),
+                "sizeof(char[4]) != sizeof(uint32_t) ?");
+  static_assert(std::is_same_v<decltype(X), decltype(Y)>,
+                "byte_swap return type doesn't match input");
+
+#if defined(IS_BIG_ENDIAN)
+  EXPECT_EQ(X, uint32_t(0x01234567));
+  EXPECT_EQ(Y, uint32_t(0x67452301));
+#elif defined(IS_LITTLE_ENDIAN)
+  EXPECT_EQ(X, uint32_t(0x67452301));
+  EXPECT_EQ(Y, uint32_t(0x01234567));
+#else
+  ADD_FAILURE() << "BYTE_ORDER is neither BIG_ENDIAN nor LITTLE_ENDIAN.";
+#endif
+}
+
+TEST(MathTest, byte_swap_64) {
+  unsigned char Seq[] = {0x01, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF};
+  uint64_t X = 0;
+  memcpy(&X, Seq, sizeof(X));
+  auto Y = byteswap(X);
+
+  static_assert(sizeof(Seq) == sizeof(X),
+                "sizeof(char[8]) != sizeof(uint64_t) ?");
+  static_assert(std::is_same_v<decltype(X), decltype(Y)>,
+                "byte_swap return type doesn't match input");
+
+#if defined(IS_BIG_ENDIAN)
+  EXPECT_EQ(X, uint64_t(0x0123456789ABCDEF));
+  EXPECT_EQ(Y, uint64_t(0xEFCDAB8967452301));
+#elif defined(IS_LITTLE_ENDIAN)
+  EXPECT_EQ(X, uint64_t(0xEFCDAB8967452301));
+  EXPECT_EQ(Y, uint64_t(0x0123456789ABCDEF));
+#else
+  ADD_FAILURE() << "BYTE_ORDER is neither BIG_ENDIAN nor LITTLE_ENDIAN.";
+#endif
+}
diff --git a/orc-rt/unittests/move_only_function-test.cpp b/orc-rt/unittests/move_only_function-test.cpp
new file mode 100644
index 0000000000000..95194edc69f6d
--- /dev/null
+++ b/orc-rt/unittests/move_only_function-test.cpp
@@ -0,0 +1,171 @@
+//===- move_only_function-test.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/move_only_function.h"
+#include "gtest/gtest.h"
+
+using namespace orc_rt;
+
+TEST(MoveOnlyFunctionTest, Basic) {
+  move_only_function<int(int, int)> Sum = [](int A, int B) { return A + B; };
+  EXPECT_EQ(Sum(1, 2), 3);
+
+  move_only_function<int(int, int)> Sum2 = std::move(Sum);
+  EXPECT_EQ(Sum2(1, 2), 3);
+
+  move_only_function<int(int, int)> Sum3 = [](int A, int B) { return A + B; };
+  Sum2 = std::move(Sum3);
+  EXPECT_EQ(Sum2(1, 2), 3);
+
+  Sum2 = move_only_function<int(int, int)>([](int A, int B) { return A + B; });
+  EXPECT_EQ(Sum2(1, 2), 3);
+
+  // Explicit self-move test.
+  *&Sum2 = std::move(Sum2);
+  EXPECT_EQ(Sum2(1, 2), 3);
+
+  Sum2 = move_only_function<int(int, int)>();
+  EXPECT_FALSE(Sum2);
+
+  // Make sure we can forward through l-value reference parameters.
+  move_only_function<void(int &)> Inc = [](int &X) { ++X; };
+  int X = 42;
+  Inc(X);
+  EXPECT_EQ(X, 43);
+
+  // Make sure we can forward through r-value reference parameters with
+  // move-only types.
+  move_only_function<int(std::unique_ptr<int> &&)> ReadAndDeallocByRef =
+      [](std::unique_ptr<int> &&Ptr) {
+        int V = *Ptr;
+        Ptr.reset();
+        return V;
+      };
+  std::unique_ptr<int> Ptr{new int(13)};
+  EXPECT_EQ(ReadAndDeallocByRef(std::move(Ptr)), 13);
+  EXPECT_FALSE((bool)Ptr);
+
+  // Make sure we can pass a move-only temporary as opposed to a local variable.
+  EXPECT_EQ(ReadAndDeallocByRef(std::unique_ptr<int>(new int(42))), 42);
+
+  // Make sure we can pass a move-only type by-value.
+  move_only_function<int(std::unique_ptr<int>)> ReadAndDeallocByVal =
+      [](std::unique_ptr<int> Ptr) {
+        int V = *Ptr;
+        Ptr.reset();
+        return V;
+      };
+  Ptr.reset(new int(13));
+  EXPECT_EQ(ReadAndDeallocByVal(std::move(Ptr)), 13);
+  EXPECT_FALSE((bool)Ptr);
+
+  EXPECT_EQ(ReadAndDeallocByVal(std::unique_ptr<int>(new int(42))), 42);
+}
+
+TEST(MoveOnlyFunctionTest, Captures) {
+  long A = 1, B = 2, C = 3, D = 4, E = 5;
+
+  move_only_function<long()> Tmp;
+
+  move_only_function<long()> C1 = [A]() { return A; };
+  EXPECT_EQ(C1(), 1);
+  Tmp = std::move(C1);
+  EXPECT_EQ(Tmp(), 1);
+
+  move_only_function<long()> C2 = [A, B]() { return A + B; };
+  EXPECT_EQ(C2(), 3);
+  Tmp = std::move(C2);
+  EXPECT_EQ(Tmp(), 3);
+
+  move_only_function<long()> C3 = [A, B, C]() { return A + B + C; };
+  EXPECT_EQ(C3(), 6);
+  Tmp = std::move(C3);
+  EXPECT_EQ(Tmp(), 6);
+
+  move_only_function<long()> C4 = [A, B, C, D]() { return A + B + C + D; };
+  EXPECT_EQ(C4(), 10);
+  Tmp = std::move(C4);
+  EXPECT_EQ(Tmp(), 10);
+
+  move_only_function<long()> C5 = [A, B, C, D, E]() {
+    return A + B + C + D + E;
+  };
+  EXPECT_EQ(C5(), 15);
+  Tmp = std::move(C5);
+  EXPECT_EQ(Tmp(), 15);
+}
+
+TEST(MoveOnlyFunctionTest, MoveOnly) {
+  struct SmallCallable {
+    std::unique_ptr<int> A = std::make_unique<int>(1);
+    int operator()(int B) { return *A + B; }
+  };
+
+  move_only_function<int(int)> Small = SmallCallable();
+  EXPECT_EQ(Small(2), 3);
+  move_only_function<int(int)> Small2 = std::move(Small);
+  EXPECT_EQ(Small2(2), 3);
+}
+
+TEST(MoveOnlyFunctionTest, CountForwardingCopies) {
+  struct CopyCounter {
+    int &CopyCount;
+
+    CopyCounter(int &CopyCount) : CopyCount(CopyCount) {}
+    CopyCounter(const CopyCounter &Arg) : CopyCount(Arg.CopyCount) {
+      ++CopyCount;
+    }
+  };
+
+  move_only_function<void(CopyCounter)> ByValF = [](CopyCounter) {};
+  int CopyCount = 0;
+  ByValF(CopyCounter(CopyCount));
+  EXPECT_EQ(1, CopyCount);
+
+  CopyCount = 0;
+  {
+    CopyCounter Counter{CopyCount};
+    ByValF(Counter);
+  }
+  EXPECT_EQ(2, CopyCount);
+
+  // Check that we don't generate a copy at all when we can bind a reference all
+  // the way down, even if that reference could *in theory* allow copies.
+  move_only_function<void(const CopyCounter &)> ByRefF =
+      [](const CopyCounter &) {};
+  CopyCount = 0;
+  ByRefF(CopyCounter(CopyCount));
+  EXPECT_EQ(0, CopyCount);
+
+  CopyCount = 0;
+  {
+    CopyCounter Counter{CopyCount};
+    ByRefF(Counter);
+  }
+  EXPECT_EQ(0, CopyCount);
+
+  // If we use a reference, we can make a stronger guarantee that *no* copy
+  // occurs.
+  struct Uncopyable {
+    Uncopyable() = default;
+    Uncopyable(const Uncopyable &) = delete;
+  };
+  move_only_function<void(const Uncopyable &)> UncopyableF =
+      [](const Uncopyable &) {};
+  UncopyableF(Uncopyable());
+  Uncopyable X;
+  UncopyableF(X);
+}
+
+TEST(MoveOnlyFunctionTest, BooleanConversion) {
+  move_only_function<void()> D;
+  EXPECT_FALSE(D);
+
+  move_only_function<void()> F = []() {};
+  EXPECT_TRUE(F);
+}
diff --git a/orc-rt/unittests/span-test.cpp b/orc-rt/unittests/span-test.cpp
new file mode 100644
index 0000000000000..fdf62f5630ac0
--- /dev/null
+++ b/orc-rt/unittests/span-test.cpp
@@ -0,0 +1,50 @@
+//===- span_test.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Tests orc-rt's span implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "orc-rt/span.h"
+#include "gtest/gtest.h"
+
+#include <sstream>
+#include <string>
+
+using namespace orc_rt;
+
+TEST(SpanTest, SpanDefaultConstruction) {
+  span<int> S;
+  EXPECT_TRUE(S.empty()) << "Default constructed span not empty";
+  EXPECT_EQ(S.size(), 0U) << "Default constructed span size not zero";
+  EXPECT_EQ(S.begin(), S.end()) << "Default constructed span begin != end";
+}
+
+TEST(SpanTest, SpanConstructFromFixedArray) {
+  int A[] = {1, 2, 3, 4, 5};
+  span<int> S(A);
+  EXPECT_FALSE(S.empty()) << "Span should be non-empty";
+  EXPECT_EQ(S.size(), 5U) << "Span has unexpected size";
+  EXPECT_EQ(std::distance(S.begin(), S.end()), 5U)
+      << "Unexpected iterator range size";
+  EXPECT_EQ(S.data(), &A[0]) << "Span data has unexpected value";
+  for (unsigned I = 0; I != S.size(); ++I)
+    EXPECT_EQ(S[I], A[I]) << "Unexpected span element value";
+}
+
+TEST(SpanTest, SpanConstructFromIteratorAndSize) {
+  int A[] = {1, 2, 3, 4, 5};
+  span<int> S(&A[0], 5);
+  EXPECT_FALSE(S.empty()) << "Span should be non-empty";
+  EXPECT_EQ(S.size(), 5U) << "Span has unexpected size";
+  EXPECT_EQ(std::distance(S.begin(), S.end()), 5U)
+      << "Unexpected iterator range size";
+  EXPECT_EQ(S.data(), &A[0]) << "Span data has unexpected value";
+  for (unsigned I = 0; I != S.size(); ++I)
+    EXPECT_EQ(S[I], A[I]) << "Unexpected span element value";
+}
diff --git a/polly/lib/Analysis/ScopBuilder.cpp b/polly/lib/Analysis/ScopBuilder.cpp
index c0babb85f5c46..67a4c43455809 100644
--- a/polly/lib/Analysis/ScopBuilder.cpp
+++ b/polly/lib/Analysis/ScopBuilder.cpp
@@ -689,7 +689,7 @@ isl::set ScopBuilder::getPredecessorDomainConstraints(BasicBlock *BB,
 
   // Set of regions of which the entry block domain has been propagated to BB.
   // all predecessors inside any of the regions can be skipped.
-  SmallSet<Region *, 8> PropagatedRegions;
+  SmallPtrSet<Region *, 8> PropagatedRegions;
 
   for (auto *PredBB : predecessors(BB)) {
     // Skip backedges.
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index bfb4341b172cc..45a6517737b95 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -36,7 +36,7 @@ list(INSERT CMAKE_MODULE_PATH 0
 # We order libraries to mirror roughly how they are layered, except that compiler-rt can depend
 # on libc++, so we put it after.
 set(LLVM_DEFAULT_RUNTIMES "libc;libunwind;libcxxabi;pstl;libcxx;compiler-rt;libclc;openmp;offload")
-set(LLVM_SUPPORTED_RUNTIMES "${LLVM_DEFAULT_RUNTIMES};llvm-libgcc;flang-rt;libsycl")
+set(LLVM_SUPPORTED_RUNTIMES "${LLVM_DEFAULT_RUNTIMES};llvm-libgcc;flang-rt;libsycl;orc-rt")
 set(LLVM_ENABLE_RUNTIMES "" CACHE STRING
   "Semicolon-separated list of runtimes to build, or \"all\" (${LLVM_DEFAULT_RUNTIMES}). Supported runtimes are ${LLVM_SUPPORTED_RUNTIMES}.")
 if(LLVM_ENABLE_RUNTIMES STREQUAL "all" )
@@ -44,7 +44,6 @@ if(LLVM_ENABLE_RUNTIMES STREQUAL "all" )
 endif()
 include(SortSubset)
 sort_subset("${LLVM_SUPPORTED_RUNTIMES}" "${LLVM_ENABLE_RUNTIMES}" LLVM_ENABLE_RUNTIMES)
-
 foreach(proj ${LLVM_ENABLE_RUNTIMES})
   set(proj_dir "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}")
   if(IS_DIRECTORY ${proj_dir} AND EXISTS ${proj_dir}/CMakeLists.txt)
@@ -160,13 +159,26 @@ endif()
 # Check for -nostdlib++ first; if there's no C++ standard library yet,
 # all check_cxx_compiler_flag commands will fail until we add -nostdlib++
 # (or -nodefaultlibs).
-llvm_check_compiler_linker_flag(CXX "-nostdlib++" CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
-  set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
-endif()
-check_cxx_compiler_flag(-nostdinc++ CXX_SUPPORTS_NOSTDINCXX_FLAG)
-if (CXX_SUPPORTS_NOSTDINCXX_FLAG)
-  set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdinc++")
+#
+# CMAKE_REQUIRED_FLAGS is used both for C and C++ compilation. This breaks
+# compiling with GCC (and we should not need to force -nostd* for GCC anyway),
+# so apply it only with Clang. However, since there are cases when appending
+# the flag actually breaks the build, still perform the checks rather than
+# appending it unconditionally.
+#
+# TODO: find a better solution. See the discussion on:
+# https://github.com/llvm/llvm-project/issues/90332
+# https://github.com/llvm/llvm-project/pull/108357
+
+if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+  llvm_check_compiler_linker_flag(CXX "-nostdlib++" CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+  if (CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
+  endif()
+  check_cxx_compiler_flag(-nostdinc++ CXX_SUPPORTS_NOSTDINCXX_FLAG)
+  if (CXX_SUPPORTS_NOSTDINCXX_FLAG)
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdinc++")
+  endif()
 endif()
 
 # Avoid checking whether the compiler is working.
@@ -254,7 +266,6 @@ foreach(entry ${runtimes})
   # will be included under here.
   set(HAVE_${canon_name} ON)
 endforeach()
-
 if(LLVM_INCLUDE_TESTS)
   # If built with the runtimes build (rooted at runtimes/CMakeLists.txt), we
   # won't have llvm-lit. If built with the bootstrapping build (rooted at
diff --git a/third-party/benchmark/src/CMakeLists.txt b/third-party/benchmark/src/CMakeLists.txt
index 943594b70bcd0..0357dcce3f831 100644
--- a/third-party/benchmark/src/CMakeLists.txt
+++ b/third-party/benchmark/src/CMakeLists.txt
@@ -57,12 +57,12 @@ endif(HAVE_LIB_RT)
 
 
 # We need extra libraries on Windows
-if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "Windows")
   target_link_libraries(benchmark PRIVATE shlwapi)
 endif()
 
 # We need extra libraries on Solaris
-if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+if("${CMAKE_SYSTEM_NAME}" MATCHES "SunOS")
   target_link_libraries(benchmark PRIVATE kstat)
 endif()
 
diff --git a/third-party/unittest/CMakeLists.txt b/third-party/unittest/CMakeLists.txt
index b9f3593320d77..3fa885a16ea1e 100644
--- a/third-party/unittest/CMakeLists.txt
+++ b/third-party/unittest/CMakeLists.txt
@@ -16,7 +16,7 @@ if(WIN32)
 endif()
 
 # Google Test requires headers which need _ALL_SOURCE to build on AIX
-if (UNIX AND ${CMAKE_SYSTEM_NAME} MATCHES "AIX")
+if (UNIX AND "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   remove_definitions("-D_XOPEN_SOURCE=700")
   add_definitions("-D_ALL_SOURCE")
 endif()
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 64f86c4b15083..7076c104860da 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1529,6 +1529,7 @@ cc_library(
         ":basic",
         ":config",
         ":driver_options_inc_gen",
+        ":lex",
         ":parse",
         ":static_analyzer_checkers_gen",
         "//llvm:BinaryFormat",
diff --git a/utils/bazel/llvm-project-overlay/clang/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/unittests/BUILD.bazel
index cd66c1fa7a6ab..518f723bab8e8 100644
--- a/utils/bazel/llvm-project-overlay/clang/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/unittests/BUILD.bazel
@@ -682,12 +682,13 @@ cc_test(
     args = [
         # TODO: some tests fail with: "JIT session error: Symbols not found:
         # [ _ZnwmPv26__clang_Interpreter_NewTag, __clang_Interpreter_SetValueWithAlloc ]
-        "--gtest_filter=-InterpreterTest.*",
+        "--gtest_filter=-InterpreterTest.*:InterpreterTestBase.*",
     ],
     deps = [
         "//clang:ast",
         "//clang:basic",
         "//clang:codegen",
+        "//clang:config",
         "//clang:frontend",
         "//clang:interpreter",
         "//clang:lex",
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index b46f334512979..bfda5385f012b 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1109,6 +1109,7 @@ libc_support_library(
     deps = [
         ":__support_cpp_bit",
         ":__support_cpp_type_traits",
+        ":__support_fputil_basic_operations",
         ":__support_fputil_cast",
         ":__support_fputil_comparison_operations",
         ":__support_fputil_dyadic_float",
@@ -1229,6 +1230,7 @@ libc_support_library(
         ":__support_cpp_bit",
         ":__support_cpp_type_traits",
         ":__support_fputil_basic_operations",
+        ":__support_fputil_cast",
         ":__support_fputil_fenv_impl",
         ":__support_fputil_fp_bits",
         ":__support_fputil_rounding_mode",
@@ -1952,92 +1954,6 @@ libc_support_library(
     ],
 )
 
-libc_support_library(
-    name = "range_reduction",
-    hdrs = [
-        "src/math/generic/range_reduction.h",
-        "src/math/generic/range_reduction_fma.h",
-    ],
-    deps = [
-        ":__support_common",
-        ":__support_fputil_fma",
-        ":__support_fputil_fp_bits",
-        ":__support_fputil_multiply_add",
-        ":__support_fputil_nearest_integer",
-    ],
-)
-
-libc_support_library(
-    name = "range_reduction_double",
-    hdrs = [
-        "src/math/generic/range_reduction_double_common.h",
-        "src/math/generic/range_reduction_double_fma.h",
-        "src/math/generic/range_reduction_double_nofma.h",
-    ],
-    deps = [
-        ":__support_common",
-        ":__support_fputil_double_double",
-        ":__support_fputil_dyadic_float",
-        ":__support_fputil_fp_bits",
-        ":__support_fputil_multiply_add",
-        ":__support_fputil_nearest_integer",
-        ":__support_fputil_rounding_mode",
-        ":__support_integer_literals",
-    ],
-)
-
-libc_support_library(
-    name = "sincos_eval",
-    hdrs = [
-        "src/math/generic/sincos_eval.h",
-    ],
-    deps = [
-        ":__support_common",
-        ":__support_fputil_double_double",
-        ":__support_fputil_dyadic_float",
-        ":__support_fputil_fp_bits",
-        ":__support_fputil_multiply_add",
-        ":__support_fputil_polyeval",
-        ":__support_integer_literals",
-    ],
-)
-
-libc_support_library(
-    name = "sincosf_utils",
-    hdrs = ["src/math/generic/sincosf_utils.h"],
-    deps = [
-        ":__support_fputil_fp_bits",
-        ":__support_fputil_polyeval",
-        ":range_reduction",
-    ],
-)
-
-libc_support_library(
-    name = "sincosf16_utils",
-    hdrs = ["src/math/generic/sincosf16_utils.h"],
-    deps = [
-        ":__support_common",
-        ":__support_fputil_nearest_integer",
-        ":__support_fputil_polyeval",
-    ],
-)
-
-libc_support_library(
-    name = "explogxf",
-    hdrs = ["src/math/generic/explogxf.h"],
-    deps = [
-        ":__support_fputil_fenv_impl",
-        ":__support_fputil_fma",
-        ":__support_fputil_multiply_add",
-        ":__support_fputil_nearest_integer",
-        ":__support_macros_properties_cpu_features",
-        ":__support_math_acoshf_utils",
-        ":__support_math_exp10f_utils",
-        ":__support_math_exp_utils",
-        ":common_constants",
-    ],
-)
-
 libc_support_library(
     name = "log_range_reduction",
     hdrs = ["src/math/generic/log_range_reduction.h"],
@@ -2060,20 +1976,8 @@ libc_support_library(
         ":__support_fputil_polyeval",
         ":__support_fputil_rounding_mode",
         ":__support_macros_optimization",
+        ":__support_math_exp10f_utils",
         ":common_constants",
-        ":explogxf",
-    ],
-)
-
-libc_support_library(
-    name = "expxf16",
-    hdrs = ["src/math/generic/expxf16.h"],
-    deps = [
-        ":__support_fputil_cast",
-        ":__support_fputil_fp_bits",
-        ":__support_fputil_nearest_integer",
-        ":__support_math_exp10_float16_constants",
-        ":__support_math_expf16_utils",
     ],
 )
 
@@ -2205,7 +2109,6 @@ libc_support_library(
     name = "__support_math_asin",
     hdrs = ["src/__support/math/asin.h"],
     deps = [
-        ":__support_math_asin_utils",
         ":__support_fputil_double_double",
         ":__support_fputil_dyadic_float",
         ":__support_fputil_fenv_impl",
@@ -2215,6 +2118,7 @@ libc_support_library(
         ":__support_fputil_sqrt",
         ":__support_macros_optimization",
         ":__support_macros_properties_cpu_features",
+        ":__support_math_asin_utils",
     ],
 )
 
@@ -2222,13 +2126,13 @@ libc_support_library(
     name = "__support_math_asinhf",
     hdrs = ["src/__support/math/asinhf.h"],
     deps = [
-        ":__support_math_acoshf_utils",
         ":__support_fputil_fp_bits",
-        ":__support_fputil_polyeval",
         ":__support_fputil_multiply_add",
+        ":__support_fputil_polyeval",
         ":__support_fputil_sqrt",
         ":__support_macros_config",
         ":__support_macros_optimization",
+        ":__support_math_acoshf_utils",
     ],
 )
 
@@ -2236,17 +2140,17 @@ libc_support_library(
     name = "__support_math_asinhf16",
     hdrs = ["src/__support/math/asinhf16.h"],
     deps = [
-        ":__support_math_acoshf_utils",
-        ":__support_fputil_fenv_impl",
-        ":__support_fputil_fp_bits",
-        ":__support_fputil_polyeval",
         ":__support_fputil_cast",
         ":__support_fputil_except_value_utils",
+        ":__support_fputil_fenv_impl",
+        ":__support_fputil_fp_bits",
         ":__support_fputil_multiply_add",
+        ":__support_fputil_polyeval",
         ":__support_fputil_rounding_mode",
         ":__support_fputil_sqrt",
         ":__support_macros_config",
         ":__support_macros_optimization",
+        ":__support_math_acoshf_utils",
     ],
 )
 
@@ -2288,12 +2192,12 @@ libc_support_library(
     name = "__support_math_atan2f",
     hdrs = ["src/__support/math/atan2f.h"],
     deps = [
+        ":__support_fputil_double_double",
         ":__support_fputil_fenv_impl",
         ":__support_fputil_fp_bits",
-        ":__support_fputil_polyeval",
-        ":__support_fputil_double_double",
         ":__support_fputil_multiply_add",
         ":__support_fputil_nearest_integer",
+        ":__support_fputil_polyeval",
         ":__support_macros_config",
         ":__support_macros_optimization",
         ":__support_math_inv_trigf_utils",
@@ -2304,13 +2208,13 @@ libc_support_library(
     name = "__support_math_atan2f128",
     hdrs = ["src/__support/math/atan2f128.h"],
     deps = [
-        ":__support_math_atan_utils",
-        ":__support_fputil_fp_bits",
         ":__support_fputil_dyadic_float",
+        ":__support_fputil_fp_bits",
         ":__support_fputil_nearest_integer",
         ":__support_integer_literals",
         ":__support_macros_config",
         ":__support_macros_optimization",
+        ":__support_math_atan_utils",
         ":__support_uint128",
     ],
 )
@@ -2337,8 +2241,8 @@ libc_support_library(
         ":__support_fputil_except_value_utils",
         ":__support_fputil_fenv_impl",
         ":__support_fputil_fp_bits",
-        ":__support_fputil_polyeval",
         ":__support_fputil_multiply_add",
+        ":__support_fputil_polyeval",
         ":__support_fputil_sqrt",
         ":__support_macros_optimization",
     ],
@@ -2348,15 +2252,15 @@ libc_support_library(
     name = "__support_math_asinf",
     hdrs = ["src/__support/math/asinf.h"],
     deps = [
-        ":__support_math_inv_trigf_utils",
+        ":__support_fputil_except_value_utils",
         ":__support_fputil_fenv_impl",
         ":__support_fputil_fp_bits",
-        ":__support_fputil_except_value_utils",
         ":__support_fputil_multiply_add",
         ":__support_fputil_sqrt",
         ":__support_macros_config",
         ":__support_macros_optimization",
         ":__support_macros_properties_cpu_features",
+        ":__support_math_inv_trigf_utils",
     ],
 )
 
@@ -2366,8 +2270,8 @@ libc_support_library(
     deps = [
         ":__support_fputil_fenv_impl",
         ":__support_fputil_fp_bits",
-        ":__support_fputil_polyeval",
         ":__support_fputil_multiply_add",
+        ":__support_fputil_polyeval",
         ":__support_fputil_sqrt",
         ":__support_macros_optimization",
     ],
@@ -2377,11 +2281,11 @@ libc_support_library(
     name = "__support_math_atanhf",
     hdrs = ["src/__support/math/atanhf.h"],
     deps = [
-        ":__support_math_acoshf_utils",
         ":__support_fputil_fenv_impl",
         ":__support_fputil_fp_bits",
         ":__support_macros_config",
         ":__support_macros_optimization",
+        ":__support_math_acoshf_utils",
     ],
 )
 
@@ -2389,12 +2293,12 @@ libc_support_library(
     name = "__support_math_atanhf16",
     hdrs = ["src/__support/math/atanhf16.h"],
     deps = [
-        ":__support_fputil_fenv_impl",
-        ":__support_fputil_fp_bits",
-        ":__support_fputil_polyeval",
         ":__support_fputil_cast",
         ":__support_fputil_except_value_utils",
+        ":__support_fputil_fenv_impl",
+        ":__support_fputil_fp_bits",
         ":__support_fputil_multiply_add",
+        ":__support_fputil_polyeval",
         ":__support_macros_config",
         ":__support_macros_optimization",
     ],
@@ -2405,13 +2309,117 @@ libc_support_library(
     hdrs = ["src/__support/math/cbrt.h"],
     deps = [
         ":__support_fputil_double_double",
-        ":__support_fputil_polyeval",
+        ":__support_fputil_dyadic_float",
         ":__support_fputil_fenv_impl",
+        ":__support_fputil_polyeval",
+        ":__support_integer_literals",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_cbrtf",
+    hdrs = ["src/__support/math/cbrtf.h"],
+    deps = [
+        ":__support_fputil_double_double",
         ":__support_fputil_dyadic_float",
+        ":__support_fputil_fenv_impl",
+        ":__support_fputil_polyeval",
         ":__support_integer_literals",
     ],
 )
 
+libc_support_library(
+    name = "__support_math_cos",
+    hdrs = ["src/__support/math/cos.h"],
+    deps = [
+        ":__support_fputil_except_value_utils",
+        ":__support_fputil_multiply_add",
+        ":__support_macros_optimization",
+        ":__support_macros_properties_cpu_features",
+        ":__support_range_reduction_double",
+        ":__support_sincos_eval",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_cosf",
+    hdrs = ["src/__support/math/cosf.h"],
+    deps = [
+        ":__support_fputil_except_value_utils",
+        ":__support_macros_optimization",
+        ":__support_macros_properties_cpu_features",
+        ":__support_sincosf_utils",
+        ":errno",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_cosf16",
+    hdrs = ["src/__support/math/cosf16.h"],
+    deps = [
+        ":__support_fputil_cast",
+        ":__support_fputil_except_value_utils",
+        ":__support_fputil_fenv_impl",
+        ":__support_fputil_multiply_add",
+        ":__support_macros_optimization",
+        ":__support_math_sincosf16_utils",
+        ":errno",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_coshf",
+    hdrs = ["src/__support/math/coshf.h"],
+    deps = [
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_multiply_add",
+        ":__support_fputil_rounding_mode",
+        ":__support_macros_optimization",
+        ":__support_math_sinhfcoshf_utils",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_coshf16",
+    hdrs = ["src/__support/math/coshf16.h"],
+    deps = [
+        ":__support_fputil_except_value_utils",
+        ":__support_fputil_fenv_impl",
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_rounding_mode",
+        ":__support_macros_config",
+        ":__support_macros_optimization",
+        ":__support_math_expxf16_utils",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_cospif",
+    hdrs = ["src/__support/math/cospif.h"],
+    deps = [
+        ":__support_fputil_fma",
+        ":__support_fputil_multiply_add",
+        ":__support_fputil_nearest_integer",
+        ":__support_fputil_polyeval",
+        ":__support_fputil_rounding_mode",
+        ":__support_macros_optimization",
+        ":__support_sincosf_utils",
+        ":common_constants",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_cospif16",
+    hdrs = ["src/__support/math/cospif16.h"],
+    deps = [
+        ":__support_fputil_cast",
+        ":__support_fputil_fenv_impl",
+        ":__support_fputil_multiply_add",
+        ":__support_macros_optimization",
+        ":__support_math_sincosf16_utils",
+    ],
+)
+
 libc_support_library(
     name = "__support_math_erff",
     hdrs = ["src/__support/math/erff.h"],
@@ -2483,6 +2491,18 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_math_expxf16_utils",
+    hdrs = ["src/__support/math/expxf16_utils.h"],
+    deps = [
+        ":__support_fputil_cast",
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_nearest_integer",
+        ":__support_math_exp10_float16_constants",
+        ":__support_math_expf16_utils",
+    ],
+)
+
 libc_support_library(
     name = "__support_math_frexpf128",
     hdrs = ["src/__support/math/frexpf128.h"],
@@ -2666,6 +2686,85 @@ libc_support_library(
     ],
 )
 
+libc_support_library(
+    name = "__support_range_reduction_double",
+    hdrs = [
+        "src/__support/math/range_reduction_double_common.h",
+        "src/__support/math/range_reduction_double_fma.h",
+        "src/__support/math/range_reduction_double_nofma.h",
+    ],
+    deps = [
+        ":__support_common",
+        ":__support_fputil_double_double",
+        ":__support_fputil_dyadic_float",
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_multiply_add",
+        ":__support_fputil_nearest_integer",
+        ":__support_fputil_rounding_mode",
+        ":__support_integer_literals",
+    ],
+)
+
+libc_support_library(
+    name = "__support_range_reduction",
+    hdrs = [
+        "src/__support/math/range_reduction.h",
+        "src/__support/math/range_reduction_fma.h",
+    ],
+    deps = [
+        ":__support_common",
+        ":__support_fputil_fma",
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_multiply_add",
+        ":__support_fputil_nearest_integer",
+    ],
+)
+
+libc_support_library(
+    name = "__support_sincos_eval",
+    hdrs = [
+        "src/__support/math/sincos_eval.h",
+    ],
+    deps = [
+        ":__support_common",
+        ":__support_fputil_double_double",
+        ":__support_fputil_dyadic_float",
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_multiply_add",
+        ":__support_fputil_polyeval",
+        ":__support_integer_literals",
+    ],
+)
+
+libc_support_library(
+    name = "__support_sincosf_utils",
+    hdrs = ["src/__support/math/sincosf_utils.h"],
+    deps = [
+        ":__support_fputil_fp_bits",
+        ":__support_fputil_polyeval",
+        ":__support_range_reduction",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_sincosf16_utils",
+    hdrs = ["src/__support/math/sincosf16_utils.h"],
+    deps = [
+        ":__support_common",
+        ":__support_fputil_nearest_integer",
+        ":__support_fputil_polyeval",
+    ],
+)
+
+libc_support_library(
+    name = "__support_math_sinhfcoshf_utils",
+    hdrs = ["src/__support/math/sinhfcoshf_utils.h"],
+    deps = [
+        ":__support_fputil_multiply_add",
+        ":__support_math_exp10f_utils",
+    ],
+)
+
 ############################### complex targets ################################
 
 libc_function(
@@ -2937,7 +3036,6 @@ libc_math_function(
     name = "acoshf",
     additional_deps = [
         ":__support_math_acoshf",
-        ":explogxf",
     ],
 )
 
@@ -2995,21 +3093,21 @@ libc_math_function(
 libc_math_function(
     name = "atanf",
     additional_deps = [
-        ":__support_math_atanf"
+        ":__support_math_atanf",
     ],
 )
 
 libc_math_function(
     name = "atanf16",
     additional_deps = [
-        ":__support_math_atanf16"
+        ":__support_math_atanf16",
     ],
 )
 
 libc_math_function(
     name = "atan",
     additional_deps = [
-        ":__support_math_atan"
+        ":__support_math_atan",
     ],
 )
 
@@ -3030,7 +3128,7 @@ libc_math_function(
 libc_math_function(
     name = "atan2",
     additional_deps = [
-       ":__support_math_atan2",
+        ":__support_math_atan2",
     ],
 )
 
@@ -3088,7 +3186,7 @@ libc_math_function(
 libc_math_function(
     name = "cbrtf",
     additional_deps = [
-        ":__support_fputil_polyeval",
+        ":__support_math_cbrtf",
     ],
 )
 
@@ -3115,76 +3213,49 @@ libc_math_function(name = "copysignf16")
 libc_math_function(
     name = "cos",
     additional_deps = [
-        ":__support_fputil_multiply_add",
-        ":__support_macros_optimization",
-        ":__support_macros_properties_cpu_features",
-        ":range_reduction_double",
-        ":sincos_eval",
+        ":__support_math_cos",
     ],
 )
 
 libc_math_function(
     name = "cosf",
     additional_deps = [
-        ":__support_fputil_fma",
-        ":__support_macros_optimization",
-        ":__support_macros_properties_cpu_features",
-        ":sincosf_utils",
-        ":errno",
+        ":__support_math_cosf",
     ],
 )
 
 libc_math_function(
     name = "cosf16",
     additional_deps = [
-        ":__support_fputil_multiply_add",
-        ":__support_macros_optimization",
-        ":sincosf16_utils",
+        ":__support_math_cosf16",
     ],
 )
 
 libc_math_function(
     name = "coshf",
     additional_deps = [
-        ":__support_fputil_fma",
-        ":__support_fputil_multiply_add",
-        ":__support_fputil_nearest_integer",
-        ":__support_fputil_polyeval",
-        ":__support_fputil_rounding_mode",
-        ":__support_macros_optimization",
-        ":common_constants",
-        ":explogxf",
+        ":__support_math_coshf",
     ],
 )
 
 libc_math_function(
     name = "coshf16",
     additional_deps = [
-        ":expxf16",
+        ":__support_math_coshf16",
     ],
 )
 
 libc_math_function(
     name = "cospif",
     additional_deps = [
-        ":__support_fputil_fma",
-        ":__support_fputil_multiply_add",
-        ":__support_fputil_nearest_integer",
-        ":__support_fputil_polyeval",
-        ":__support_fputil_rounding_mode",
-        ":__support_macros_optimization",
-        ":common_constants",
-        ":explogxf",
-        ":sincosf_utils",
+        ":__support_math_cospif",
     ],
 )
 
 libc_math_function(
     name = "cospif16",
     additional_deps = [
-        ":__support_fputil_multiply_add",
-        ":__support_macros_optimization",
-        ":sincosf16_utils",
+        ":__support_math_cospif16",
     ],
 )
 
@@ -3259,7 +3330,7 @@ libc_math_function(
     name = "expf16",
     additional_deps = [
         ":__support_math_expf16",
-        ":expxf16",
+        ":__support_math_expxf16_utils",
     ],
 )
 
@@ -3297,7 +3368,7 @@ libc_math_function(
 libc_math_function(
     name = "exp10m1f",
     additional_deps = [
-        ":explogxf",
+        ":__support_math_exp10f_utils",
     ],
 )
 
@@ -3313,8 +3384,8 @@ libc_math_function(
         ":__support_fputil_triple_double",
         ":__support_integer_literals",
         ":__support_macros_optimization",
+        ":__support_math_exp_utils",
         ":common_constants",
-        ":explogxf",
     ],
 )
 
@@ -3328,7 +3399,7 @@ libc_math_function(
 libc_math_function(
     name = "exp2f16",
     additional_deps = [
-        ":expxf16",
+        ":__support_math_expxf16_utils",
     ],
 )
 
@@ -3336,14 +3407,14 @@ libc_math_function(
     name = "exp2m1f",
     additional_deps = [
         ":__support_fputil_polyeval",
-        ":explogxf",
+        ":__support_math_exp10f_utils",
     ],
 )
 
 libc_math_function(
     name = "exp2m1f16",
     additional_deps = [
-        ":expxf16",
+        ":__support_math_expxf16_utils",
     ],
 )
 
@@ -3353,14 +3424,12 @@ libc_math_function(
         ":__support_fputil_double_double",
         ":__support_fputil_dyadic_float",
         ":__support_fputil_multiply_add",
-        ":__support_fputil_nearest_integer",
         ":__support_fputil_polyeval",
         ":__support_fputil_rounding_mode",
         ":__support_fputil_triple_double",
         ":__support_integer_literals",
         ":__support_macros_optimization",
         ":common_constants",
-        ":explogxf",
     ],
 )
 
@@ -3381,7 +3450,7 @@ libc_math_function(
 libc_math_function(
     name = "expm1f16",
     additional_deps = [
-        ":expxf16",
+        ":__support_math_expxf16_utils",
     ],
 )
 
@@ -3887,7 +3956,7 @@ libc_math_function(
 libc_math_function(
     name = "logf16",
     additional_deps = [
-        ":expxf16",
+        ":__support_math_expxf16_utils",
     ],
 )
 
@@ -3922,7 +3991,7 @@ libc_math_function(
 libc_math_function(
     name = "log10f16",
     additional_deps = [
-        ":expxf16",
+        ":__support_math_expxf16_utils",
     ],
 )
 
@@ -3983,7 +4052,7 @@ libc_math_function(
 libc_math_function(
     name = "log2f16",
     additional_deps = [
-        ":expxf16",
+        ":__support_math_expxf16_utils",
     ],
 )
 
@@ -4138,7 +4207,6 @@ libc_math_function(
         ":__support_macros_optimization",
         ":__support_math_exp10f",
         ":common_constants",
-        ":explogxf",
         ":exp2f_impl",
     ],
 )
@@ -4239,8 +4307,8 @@ libc_math_function(
         ":__support_fputil_multiply_add",
         ":__support_macros_optimization",
         ":__support_macros_properties_cpu_features",
-        ":range_reduction_double",
-        ":sincos_eval",
+        ":__support_range_reduction_double",
+        ":__support_sincos_eval",
     ],
 )
 
@@ -4252,8 +4320,8 @@ libc_math_function(
         ":__support_fputil_rounding_mode",
         ":__support_macros_optimization",
         ":__support_macros_properties_cpu_features",
-        ":range_reduction",
-        ":sincosf_utils",
+        ":__support_range_reduction",
+        ":__support_sincosf_utils",
     ],
 )
 
@@ -4262,7 +4330,7 @@ libc_math_function(
     additional_deps = [
         ":__support_fputil_nearest_integer",
         ":__support_fputil_polyeval",
-        ":sincosf16_utils",
+        ":__support_math_sincosf16_utils",
     ],
 )
 
@@ -4272,8 +4340,8 @@ libc_math_function(
         ":__support_fputil_multiply_add",
         ":__support_macros_optimization",
         ":__support_macros_properties_cpu_features",
-        ":range_reduction_double",
-        ":sincos_eval",
+        ":__support_range_reduction_double",
+        ":__support_sincos_eval",
     ],
 )
 
@@ -4285,7 +4353,7 @@ libc_math_function(
         ":__support_fputil_rounding_mode",
         ":__support_macros_optimization",
         ":__support_macros_properties_cpu_features",
-        ":sincosf_utils",
+        ":__support_sincosf_utils",
     ],
 )
 
@@ -4298,22 +4366,22 @@ libc_math_function(
         ":__support_fputil_polyeval",
         ":__support_fputil_rounding_mode",
         ":__support_macros_optimization",
+        ":__support_math_sinhfcoshf_utils",
         ":common_constants",
-        ":explogxf",
     ],
 )
 
 libc_math_function(
     name = "sinhf16",
     additional_deps = [
-        ":expxf16",
+        ":__support_math_expxf16_utils",
     ],
 )
 
 libc_math_function(
     name = "sinpif",
     additional_deps = [
-        ":sincosf_utils",
+        ":__support_sincosf_utils",
     ],
 )
 
@@ -4322,7 +4390,7 @@ libc_math_function(
     additional_deps = [
         ":__support_fputil_nearest_integer",
         ":__support_fputil_polyeval",
-        ":sincosf16_utils",
+        ":__support_math_sincosf16_utils",
     ],
 )
 
@@ -4367,8 +4435,8 @@ libc_math_function(
         ":__support_fputil_multiply_add",
         ":__support_macros_optimization",
         ":__support_macros_properties_cpu_features",
-        ":range_reduction_double",
-        ":sincos_eval",
+        ":__support_range_reduction_double",
+        ":__support_sincos_eval",
     ],
 )
 
@@ -4381,8 +4449,8 @@ libc_math_function(
         ":__support_fputil_polyeval",
         ":__support_macros_optimization",
         ":__support_macros_properties_cpu_features",
-        ":range_reduction",
-        ":sincosf_utils",
+        ":__support_range_reduction",
+        ":__support_sincosf_utils",
     ],
 )
 
@@ -4391,7 +4459,7 @@ libc_math_function(
     additional_deps = [
         ":__support_fputil_nearest_integer",
         ":__support_fputil_polyeval",
-        ":sincosf16_utils",
+        ":__support_math_sincosf16_utils",
     ],
 )
 
@@ -4405,22 +4473,22 @@ libc_math_function(
         ":__support_fputil_rounding_mode",
         ":__support_macros_optimization",
         ":__support_macros_properties_cpu_features",
+        ":__support_math_exp10f_utils",
         ":common_constants",
-        ":explogxf",
     ],
 )
 
 libc_math_function(
     name = "tanhf16",
     additional_deps = [
-        ":expxf16",
+        ":__support_math_expxf16_utils",
     ],
 )
 
 libc_math_function(
     name = "tanpif",
     additional_deps = [
-        ":sincosf_utils",
+        ":__support_sincosf_utils",
         ":hdr_fenv_macros",
         ":__support_macros_config",
         ":__support_macros_optimization",
@@ -4431,7 +4499,7 @@ libc_math_function(
 libc_math_function(
     name = "tanpif16",
     additional_deps = [
-        ":sincosf16_utils",
+        ":__support_math_sincosf16_utils",
         ":hdr_errno_macros",
         ":hdr_fenv_macros",
         ":__support_fputil_cast",
@@ -4864,6 +4932,7 @@ libc_support_library(
         "src/string/memory_utils/aarch64/inline_memcpy.h",
         "src/string/memory_utils/aarch64/inline_memmove.h",
         "src/string/memory_utils/aarch64/inline_memset.h",
+        "src/string/memory_utils/aarch64/inline_strlen.h",
         "src/string/memory_utils/arm/common.h",
         "src/string/memory_utils/arm/inline_memcpy.h",
         "src/string/memory_utils/arm/inline_memset.h",
@@ -4888,6 +4957,7 @@ libc_support_library(
         "src/string/memory_utils/x86_64/inline_memcpy.h",
         "src/string/memory_utils/x86_64/inline_memmove.h",
         "src/string/memory_utils/x86_64/inline_memset.h",
+        "src/string/memory_utils/x86_64/inline_strlen.h",
     ],
     deps = [
         ":__support_common",
@@ -4912,6 +4982,7 @@ libc_support_library(
         ":__support_macros_optimization",
         ":hdr_limits_macros",
         ":llvm_libc_types_size_t",
+        ":string_memory_utils",
         ":types_size_t",
     ],
 )
@@ -5088,6 +5159,19 @@ libc_function(
     ],
 )
 
+libc_function(
+    name = "strsep",
+    srcs = ["src/string/strsep.cpp"],
+    hdrs = ["src/string/strsep.h"],
+    deps = [
+        ":__support_common",
+        ":__support_macros_null_check",
+        ":hdr_signal_macros",
+        ":string_memory_utils",
+        ":string_utils",
+    ],
+)
+
 libc_function(
     name = "strstr",
     srcs = ["src/string/strstr.cpp"],
diff --git a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
index b49e7c3fad954..179fc83e67298 100644
--- a/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
+++ b/utils/bazel/llvm-project-overlay/libc/libc_configure_options.bzl
@@ -49,4 +49,7 @@ LIBC_CONFIGURE_OPTIONS = [
 
     # Documentation in libc/docs/configure.rst
     "LIBC_THREAD_MODE=LIBC_THREAD_MODE_PLATFORM",
+
+    # Documentation in libc/src/__support/libc_errno.h
+    "LIBC_ERRNO_MODE=LIBC_ERRNO_MODE_SYSTEM_INLINE",
 ]
diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index 156515309d8ca..24baaf1983a08 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -22,11 +22,11 @@ libc_test_library(
         "//libc:__support_macros_properties_types",
         "//libc:__support_osutil_io",
         "//libc:__support_uint128",
-        "//libc:hdr_stdint_proxy",
         "//libc:func_aligned_alloc",
         "//libc:func_free",
         "//libc:func_malloc",
         "//libc:func_realloc",
+        "//libc:hdr_stdint_proxy",
     ],
 )
 
@@ -65,12 +65,12 @@ libc_test_library(
         "//libc:__support_macros_properties_types",
         "//libc:__support_stringutil",
         "//libc:__support_uint128",
-        "//libc:hdr_stdint_proxy",
         "//libc:errno",
         "//libc:func_aligned_alloc",
         "//libc:func_free",
         "//libc:func_malloc",
         "//libc:func_realloc",
+        "//libc:hdr_stdint_proxy",
         "//libc:llvm_libc_macros_stdfix_macros",
         "//llvm:Support",
     ],
@@ -107,6 +107,7 @@ libc_test_library(
     ],
     deps = [
         ":LibcUnitTest",
+        ":errno_test_helpers",
         ":string_utils",
         ":test_logger",
         "//libc:__support_cpp_array",
@@ -119,6 +120,7 @@ libc_test_library(
         "//libc:__support_fputil_fp_bits",
         "//libc:__support_fputil_fpbits_str",
         "//libc:__support_fputil_rounding_mode",
+        "//libc:__support_libc_errno",
         "//libc:__support_macros_config",
         "//libc:__support_macros_properties_architectures",
         "//libc:hdr_fenv_macros",
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel
index 6d9d1d6a33255..087c3a62472e6 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/math/smoke/BUILD.bazel
@@ -1174,26 +1174,31 @@ math_test(
 math_test(
     name = "nextafter",
     hdrs = ["NextAfterTest.h"],
+    deps = ["//libc:__support_sign"],
 )
 
 math_test(
     name = "nextafterf",
     hdrs = ["NextAfterTest.h"],
+    deps = ["//libc:__support_sign"],
 )
 
 math_test(
     name = "nextafterl",
     hdrs = ["NextAfterTest.h"],
+    deps = ["//libc:__support_sign"],
 )
 
 math_test(
     name = "nextafterf128",
     hdrs = ["NextAfterTest.h"],
+    deps = ["//libc:__support_sign"],
 )
 
 math_test(
     name = "nextafterf16",
     hdrs = ["NextAfterTest.h"],
+    deps = ["//libc:__support_sign"],
 )
 
 math_test(
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
index 5f50b10516fb5..d90992417a721 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
@@ -59,6 +59,15 @@ libc_test(
     ],
 )
 
+libc_test(
+    name = "strsep_test",
+    srcs = ["strsep_test.cpp"],
+    deps = [
+        "//libc:hdr_signal_macros",
+        "//libc:strsep",
+    ],
+)
+
 libc_test(
     name = "strstr_test",
     srcs = ["strstr_test.cpp"],
diff --git a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
index ff93eb51e4872..dc934a4422105 100644
--- a/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/utils/MPFRWrapper/BUILD.bazel
@@ -77,6 +77,7 @@ libc_test_library(
         "//libc:__support_cpp_string_view",
         "//libc:__support_cpp_stringstream",
         "//libc:__support_cpp_type_traits",
+        "//libc:__support_fputil_bfloat16",
         "//libc:__support_fputil_cast",
         "//libc:__support_fputil_fp_bits",
         "//libc:__support_fputil_fpbits_str",
diff --git a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
index fca964088b1a6..56d44af02e5cc 100644
--- a/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/BUILD.bazel
@@ -6,7 +6,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects")
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 load("@build_bazel_apple_support//rules:apple_genrule.bzl", "apple_genrule")
-load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_import", "cc_library", "objc_library")
 load("@rules_python//python:defs.bzl", "py_binary")
 load("//:vars.bzl", "LLVM_VERSION_MAJOR", "LLVM_VERSION_MINOR", "LLVM_VERSION_PATCH", "LLVM_VERSION_SUFFIX", "PACKAGE_VERSION")
 load("//lldb/source/Plugins:plugin_config.bzl", "DEFAULT_PLUGINS", "DEFAULT_SCRIPT_PLUGINS", "OBJCPP_COPTS")
diff --git a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
index 8c06c4ef6de88..c0f5a79dc53d4 100644
--- a/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lldb/source/Plugins/BUILD.bazel
@@ -2,7 +2,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
-load("@rules_cc//cc:defs.bzl", "cc_library")
+load("@rules_cc//cc:defs.bzl", "cc_library", "objc_library")
 load("//mlir:tblgen.bzl", "gentbl_cc_library")
 load(":plugin_config.bzl", "DEFAULT_PLUGINS", "DEFAULT_SCRIPT_PLUGINS", "OBJCPP_COPTS")
 
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 00fba3b081706..2c7902c161fc8 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -4731,6 +4731,25 @@ cc_binary(
     ],
 )
 
+cc_binary(
+    name = "llvm-offload-wrapper",
+    srcs = glob([
+        "tools/llvm-offload-wrapper/*.cpp",
+    ]),
+    copts = llvm_copts,
+    stamp = 0,
+    deps = [
+        ":BitWriter",
+        ":Core",
+        ":FrontendOffloading",
+        ":Object",
+        ":Option",
+        ":Support",
+        ":TargetParser",
+        ":intrinsic_enums_gen",
+    ],
+)
+
 cc_binary(
     name = "llvm-pdbutil",
     srcs = glob([
diff --git a/utils/bazel/llvm-project-overlay/llvm/config.bzl b/utils/bazel/llvm-project-overlay/llvm/config.bzl
index 2309175d04a9b..3e9c032d5b8ce 100644
--- a/utils/bazel/llvm-project-overlay/llvm/config.bzl
+++ b/utils/bazel/llvm-project-overlay/llvm/config.bzl
@@ -113,7 +113,6 @@ llvm_config_defines = os_defines + builtin_thread_pointer + select({
     "LLVM_VERSION_PATCH={}".format(LLVM_VERSION_PATCH),
     r'LLVM_VERSION_STRING=\"{}\"'.format(PACKAGE_VERSION),
     # Set globally in HandleLLVMOptions.cmake
-    "EXPERIMENTAL_KEY_INSTRUCTIONS",
     # These shouldn't be needed by the C++11 standard, but are for some
     # platforms (e.g. glibc < 2.18. See
     # https://sourceware.org/bugzilla/show_bug.cgi?id=15366). These are also
diff --git a/utils/bazel/llvm-project-overlay/llvm/driver.bzl b/utils/bazel/llvm-project-overlay/llvm/driver.bzl
index 39093db5f73b0..331e3f7e0db1f 100644
--- a/utils/bazel/llvm-project-overlay/llvm/driver.bzl
+++ b/utils/bazel/llvm-project-overlay/llvm/driver.bzl
@@ -6,7 +6,7 @@
 
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
-load("@rules_cc//cc:defs.bzl", "cc_binary")
+load("@rules_cc//cc:defs.bzl", "CcInfo", "cc_binary")
 
 # Mapping from every tool to the cc_library that implements the tool's entrypoint.
 _TOOLS = {
diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
index dc8aa406fb2f6..3fa62bf708514 100644
--- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel
@@ -2,7 +2,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test")
+load("@rules_cc//cc:defs.bzl", "cc_library", "cc_shared_library", "cc_test")
 load("//mlir:tblgen.bzl", "gentbl_cc_library")
 
 package(
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 76893003255bb..c2e2764d50aff 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -8,6 +8,7 @@
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
+load("//llvm:targets.bzl", "llvm_targets")
 load(
     ":build_defs.bzl",
     "cc_headers_only",
@@ -397,6 +398,8 @@ cc_library(
         ":SymbolInterfacesIncGen",
         ":TensorEncodingIncGen",
         ":config",
+        "//llvm:Core",
+        "//llvm:Remarks",
         "//llvm:Support",
         "//llvm:config",
     ],
@@ -484,6 +487,24 @@ mlir_c_api_cc_library(
     ],
 )
 
+cc_library(
+    name = "RemarkStreamer",
+    srcs = glob([
+        "lib/Remark/*.cpp",
+    ]),
+    hdrs = glob([
+        "include/mlir/Remark/*.h",
+    ]),
+    includes = ["include"],
+    deps = [
+        ":IR",
+        "//llvm:BitstreamReader",
+        "//llvm:Core",
+        "//llvm:Remarks",
+        "//llvm:Support",
+    ],
+)
+
 mlir_c_api_cc_library(
     name = "CAPIInterfaces",
     srcs = [
@@ -3541,14 +3562,13 @@ cc_library(
         ":InferTypeOpInterface",
         ":Support",
         ":WasmSSAIncGen",
+        ":WasmSSAInterfacesIncGen",
         ":WasmSSAOpsIncGen",
         ":WasmSSATypesIncGen",
-        ":WasmSSAInterfacesIncGen",
         "//llvm:Support",
     ],
 )
 
-
 ##---------------------------------------------------------------------------##
 # XeGPU dialect.
 ##---------------------------------------------------------------------------##
@@ -3632,23 +3652,39 @@ gentbl_cc_library(
     deps = [":XeGPUAttrTdFiles"],
 )
 
+gentbl_cc_library(
+    name = "XeGPUAttrInterfaceIncGen",
+    tbl_outs = {
+        "include/mlir/Dialect/XeGPU/IR/XeGPUAttrInterface.h.inc": ["-gen-attr-interface-decls"],
+        "include/mlir/Dialect/XeGPU/IR/XeGPUAttrInterface.cpp.inc": ["-gen-attr-interface-defs"],
+    },
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td",
+    deps = [":XeGPUAttrTdFiles"],
+)
+
 cc_library(
     name = "XeGPUDialect",
     srcs = glob(["lib/Dialect/XeGPU/IR/*.cpp"]),
     hdrs = glob(["include/mlir/Dialect/XeGPU/IR/*.h"]),
     includes = ["include"],
     deps = [
+        ":AffineUtils",
         ":ArithDialect",
         ":ArithUtils",
         ":BytecodeOpInterface",
         ":DialectUtils",
+        ":GPUDialect",
         ":IR",
+        ":IndexDialect",
         ":ShapedOpInterfaces",
         ":SideEffectInterfaces",
         ":VectorDialect",
         ":ViewLikeInterface",
+        ":XeGPUAttrInterfaceIncGen",
         ":XeGPUEnumsIncGen",
         ":XeGPUIncGen",
+        ":XeVMDialect",
         "//llvm:Support",
     ],
 )
@@ -3709,11 +3745,15 @@ cc_library(
     includes = ["include"],
     deps = [
         ":DialectUtils",
+        ":GPUDialect",
         ":IR",
+        ":IndexDialect",
+        ":LLVMDialect",
         ":LoopLikeInterface",
         ":SCFTransforms",
         ":TransformUtils",
         ":XeGPUDialect",
+        ":XeVMDialect",
         "//llvm:Support",
     ],
 )
@@ -4120,6 +4160,7 @@ cc_library(
         ":TosaToTensor",
         ":UBToLLVM",
         ":UBToSPIRV",
+        ":VectorToAMX",
         ":VectorToArmSME",
         ":VectorToGPU",
         ":VectorToLLVM",
@@ -4900,6 +4941,7 @@ cc_library(
         ":MemRefDialect",
         ":Support",
         ":TensorDialect",
+        ":UBDialect",
         ":VectorDialect",
         "//llvm:Support",
     ],
@@ -5495,6 +5537,7 @@ cc_library(
         ":VCIXToLLVMIRTranslation",
         ":VectorDialect",
         ":XeVMDialect",
+        ":XeVMTarget",
         ":config",
         "//llvm:Core",
         "//llvm:MC",
@@ -5561,6 +5604,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":AMDGPUDialect",
+        ":AMDGPUUtils",
         ":AffineDialect",
         ":ArithDialect",
         ":DialectUtils",
@@ -5696,6 +5740,7 @@ cc_library(
         ":NVGPUDialect",
         ":NVVMDialect",
         ":TransformUtils",
+        ":VectorTransforms",
     ],
 )
 
@@ -7118,6 +7163,26 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "WasmTranslationRegistration",
+    srcs = [
+        "lib/Target/Wasm/TranslateFromWasm.cpp",
+        "lib/Target/Wasm/TranslateRegistration.cpp",
+    ],
+    hdrs = [
+        "include/mlir/Target/Wasm/WasmBinaryEncoding.h",
+        "include/mlir/Target/Wasm/WasmImporter.h",
+    ],
+    includes = ["include"],
+    deps = [
+        ":IR",
+        ":Support",
+        ":TranslateLib",
+        ":WasmSSADialect",
+        "//llvm:Support",
+    ],
+)
+
 td_library(
     name = "TensorOpsTdFiles",
     srcs = [
@@ -7854,6 +7919,7 @@ cc_library(
         ":Pass",
         ":Rewrite",
         ":TransformUtils",
+        "//llvm:Support",
     ],
 )
 
@@ -7937,6 +8003,7 @@ cc_library(
         ":Transforms",
         ":VectorDialect",
         ":XeGPUDialect",
+        ":XeGPUUtils",
         "//llvm:Support",
     ],
 )
@@ -8023,6 +8090,7 @@ cc_library(
         ":MemRefDialect",
         ":Pass",
         ":TransformUtils",
+        "//llvm:Support",
     ],
 )
 
@@ -8835,7 +8903,6 @@ cc_library(
     srcs = [
         "lib/Target/LLVMIR/AttrKindDetail.h",
         "lib/Target/LLVMIR/DataLayoutImporter.cpp",
-        "lib/Target/LLVMIR/DataLayoutImporter.h",
         "lib/Target/LLVMIR/DebugImporter.cpp",
         "lib/Target/LLVMIR/DebugImporter.h",
         "lib/Target/LLVMIR/LLVMImportInterface.cpp",
@@ -8845,6 +8912,7 @@ cc_library(
         "lib/Target/LLVMIR/TypeFromLLVM.cpp",
     ],
     hdrs = [
+        "include/mlir/Target/LLVMIR/DataLayoutImporter.h",
         "include/mlir/Target/LLVMIR/Import.h",
         "include/mlir/Target/LLVMIR/LLVMImportInterface.h",
         "include/mlir/Target/LLVMIR/ModuleImport.h",
@@ -8868,6 +8936,43 @@ cc_library(
     ],
 )
 
+gentbl_cc_library(
+    name = "TargetLLVMIRTransformsIncGen",
+    tbl_outs = {
+        "include/mlir/Target/LLVMIR/Transforms/Passes.h.inc": [
+            "-gen-pass-decls",
+            "-name=TargetLLVMIRTransforms",
+        ],
+    },
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Target/LLVMIR/Transforms/Passes.td",
+    deps = [":PassBaseTdFiles"],
+)
+
+cc_library(
+    name = "TargetLLVMIRTransforms",
+    srcs = glob([
+        "lib/Target/LLVMIR/Transforms/*.cpp",
+    ]),
+    hdrs = glob([
+        "include/mlir/Target/LLVMIR/Transforms/*.h",
+    ]),
+    includes = ["include"],
+    deps = [
+        ":DLTIDialect",
+        ":FromLLVMIRTranslation",
+        ":LLVMDialect",
+        ":Pass",
+        ":TargetLLVMIRTransformsIncGen",
+        "//llvm:AllTargetsAsmParsers",
+        "//llvm:AllTargetsCodeGens",
+        "//llvm:MC",
+        "//llvm:Support",
+        "//llvm:Target",
+        "//llvm:TargetParser",
+    ],
+)
+
 cc_library(
     name = "FromLLVMIRTranslationRegistration",
     srcs = [
@@ -8981,6 +9086,7 @@ cc_library(
         ":TargetCpp",
         ":TargetSMTLIB",
         ":ToLLVMIRTranslationRegistration",
+        ":WasmTranslationRegistration",
     ],
 )
 
@@ -9089,6 +9195,7 @@ cc_library(
         ":X86VectorDialect",
         ":XeGPUDialect",
         ":XeVMDialect",
+        ":XeVMTarget",
     ],
 )
 
@@ -9124,6 +9231,7 @@ cc_library(
         ":ShardTransforms",
         ":SparseTensorPipelines",
         ":SparseTensorTransforms",
+        ":TargetLLVMIRTransforms",
         ":TensorTransforms",
         ":TosaDialect",
         ":TransformDialectTransforms",
@@ -9183,6 +9291,7 @@ cc_library(
         ":VectorToLLVM",
         ":VectorTransformOps",
         ":XeVMToLLVM",
+        ":XeVMToLLVMIRTranslation",
     ],
 )
 
@@ -10683,6 +10792,7 @@ cc_library(
         ":AsmParser",
         ":BufferizationDialect",
         ":BufferizationTransforms",
+        ":CommonFolders",
         ":DialectUtils",
         ":FuncDialect",
         ":FunctionInterfaces",
@@ -10706,6 +10816,7 @@ cc_library(
         ":TransformDialectInterfaces",
         ":TransformDialectUtils",
         ":TransformUtils",
+        ":UBDialect",
         ":VectorDialect",
         ":VectorTransforms",
         "//llvm:Support",
@@ -10730,7 +10841,10 @@ gentbl_cc_library(
     },
     tblgen = ":mlir-tblgen",
     td_file = "include/mlir/Dialect/Linalg/Passes.td",
-    deps = [":PassBaseTdFiles"],
+    deps = [
+        ":OpBaseTdFiles",
+        ":PassBaseTdFiles",
+    ],
 )
 
 cc_library(
@@ -10818,6 +10932,7 @@ cc_library(
         ":TensorUtils",
         ":TilingInterface",
         ":TransformUtils",
+        ":UBDialect",
         ":ValueBoundsOpInterface",
         ":VectorDialect",
         ":VectorToSCF",
@@ -11116,6 +11231,32 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "VectorToAMX",
+    srcs = glob([
+        "lib/Conversion/VectorToAMX/*.cpp",
+    ]),
+    hdrs = glob([
+        "include/mlir/Conversion/VectorToAMX/*.h",
+    ]),
+    includes = ["include"],
+    deps = [
+        ":AMXDialect",
+        ":AffineDialect",
+        ":AffineUtils",
+        ":ArithDialect",
+        ":ConversionPassIncGen",
+        ":DialectUtils",
+        ":IR",
+        ":LinalgInterfaces",
+        ":MemRefDialect",
+        ":Pass",
+        ":SCFDialect",
+        ":TransformUtils",
+        ":VectorDialect",
+    ],
+)
+
 cc_library(
     name = "VectorToGPU",
     srcs = glob([
@@ -13774,6 +13915,34 @@ gentbl_cc_library(
     deps = [":XeVMTdFiles"],
 )
 
+cc_library(
+    name = "XeVMTarget",
+    srcs = ["lib/Target/LLVM/XeVM/Target.cpp"],
+    hdrs = glob(["include/mlir/Target/LLVM/XeVM/*.h"]),
+    includes = ["include"],
+    deps = [
+        ":ExecutionEngineUtils",
+        ":GPUDialect",
+        ":GPUToLLVMIRTranslation",
+        ":IR",
+        ":LLVMToLLVMIRTranslation",
+        ":Support",
+        ":TargetLLVM",
+        ":ToLLVMIRTranslation",
+        ":XeVMDialect",
+        ":XeVMToLLVMIRTranslation",
+        "//llvm:BitWriter",
+        "//llvm:Core",
+        "//llvm:Object",
+        "//llvm:Support",
+        "//llvm:Target",
+        "//llvm:config",
+    ] + ([
+        "//llvm:SPIRVCodeGen",
+        "//llvm:SPIRVUtilsAndDesc",
+    ] if "SPIRV" in llvm_targets else []),
+)
+
 cc_library(
     name = "XeVMToLLVM",
     srcs = glob([
diff --git a/utils/bazel/llvm-project-overlay/mlir/build_defs.bzl b/utils/bazel/llvm-project-overlay/mlir/build_defs.bzl
index b3fb4fcc32338..5a3a776258b06 100644
--- a/utils/bazel/llvm-project-overlay/mlir/build_defs.bzl
+++ b/utils/bazel/llvm-project-overlay/mlir/build_defs.bzl
@@ -4,7 +4,7 @@
 
 """Rules and macros for MLIR"""
 
-load("@rules_cc//cc:defs.bzl", "cc_library")
+load("@rules_cc//cc:defs.bzl", "CcInfo", "cc_library")
 
 def if_cuda_available(if_true, if_false = []):
     return select({
diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/transform-opt/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/transform-opt/BUILD.bazel
index 22670bddb6d33..6f6a3f5310467 100644
--- a/utils/bazel/llvm-project-overlay/mlir/examples/transform-opt/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/examples/transform-opt/BUILD.bazel
@@ -1,5 +1,7 @@
 # MLIR transform dialect standalone example.
 
+load("@rules_cc//cc:defs.bzl", "cc_binary")
+
 package(default_visibility = ["//visibility:public"])
 
 cc_binary(
diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch2/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch2/BUILD.bazel
index a6f120cdca3f1..d1610fe367aec 100644
--- a/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch2/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch2/BUILD.bazel
@@ -1,6 +1,7 @@
 # Description:
 #   MLIR Tutorial
 
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
 load("//mlir:tblgen.bzl", "gentbl_cc_library", "td_library")
 
 licenses(["notice"])
diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch3/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch3/BUILD.bazel
index cba4339c2f83f..0f7b9abfaa8d1 100644
--- a/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch3/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch3/BUILD.bazel
@@ -1,6 +1,7 @@
 # Description:
 #   MLIR Tutorial
 
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
 load("//mlir:tblgen.bzl", "gentbl_cc_library", "td_library")
 
 licenses(["notice"])
diff --git a/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch4/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch4/BUILD.bazel
index 81c0161be118e..39e4ddd2b8d3c 100644
--- a/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch4/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/examples/transform/Ch4/BUILD.bazel
@@ -1,6 +1,7 @@
 # Description:
 #   MLIR Tutorial
 
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library")
 load("//mlir:tblgen.bzl", "gentbl_cc_library", "td_library")
 
 licenses(["notice"])
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 72398ce13d085..6cfd8f4a6d675 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -43,9 +43,11 @@ expand_template(
         "@MLIR_ENABLE_ROCM_CONVERSIONS@": "0",
         "@MLIR_ENABLE_ROCM_RUNNER@": "0",
         "@MLIR_ENABLE_SYCL_RUNNER@": "0",
+        "@MLIR_ENABLE_LEVELZERO_RUNNER@": "0",
         "@MLIR_ENABLE_SPIRV_CPU_RUNNER@": "0",
         "@MLIR_ENABLE_VULKAN_RUNNER@": "0",
         "@MLIR_ENABLE_BINDINGS_PYTHON@": "0",
+        "@MLIR_ENABLE_XEVM_CONVERSIONS@": "0",
         "@MLIR_RUN_AMX_TESTS@": "0",
         "@MLIR_RUN_ARM_SVE_TESTS@": "0",
         "@MLIR_RUN_ARM_SME_TESTS@": "0",
@@ -381,6 +383,7 @@ cc_library(
         "//mlir:BufferizationInterfaces",
         "//mlir:BytecodeOpInterface",
         "//mlir:CallOpInterfaces",
+        "//mlir:CommonFolders",
         "//mlir:ControlFlowInterfaces",
         "//mlir:CopyOpInterface",
         "//mlir:DLTIDialect",
@@ -914,10 +917,10 @@ cc_library(
         "//mlir:DialectUtils",
         "//mlir:FuncDialect",
         "//mlir:IR",
-        "//mlir:ShardDialect",
-        "//mlir:ShardTransforms",
         "//mlir:Pass",
         "//mlir:SPIRVDialect",
+        "//mlir:ShardDialect",
+        "//mlir:ShardTransforms",
         "//mlir:Support",
         "//mlir:TransformUtils",
         "//mlir:Transforms",
@@ -1202,11 +1205,13 @@ cc_library(
     deps = [
         "//mlir:GPUDialect",
         "//mlir:IR",
+        "//mlir:IndexDialect",
         "//mlir:MemRefDialect",
         "//mlir:Pass",
         "//mlir:TransformUtils",
         "//mlir:VectorTransforms",
         "//mlir:XeGPUDialect",
         "//mlir:XeGPUTransforms",
+        "//mlir:XeGPUUtils",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/Target/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/Target/BUILD.bazel
index 7fab1ea1dd75f..b5a8dbd2885b7 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/Target/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/Target/BUILD.bazel
@@ -10,10 +10,11 @@ package(default_visibility = ["//visibility:public"])
         srcs = [src],
         data = [
             "//llvm:split-file",
+            "//llvm:yaml2obj",
             "//mlir:mlir-opt",
             "//mlir:mlir-translate",
             "//mlir/test:lit_data",
-        ],
+        ] + glob(["Wasm/inputs/*"]),
     )
     for src in glob([
         "**/*.mlir",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/mlir-tblgen/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/mlir-tblgen/BUILD.bazel
index 5ebdc33dfdd1d..2559a1888b429 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/mlir-tblgen/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/mlir-tblgen/BUILD.bazel
@@ -33,6 +33,7 @@ package(default_visibility = ["//visibility:public"])
             "//mlir:include/mlir/IR/BytecodeBase.td",
             "//mlir:include/mlir/IR/OpBase.td",
             "//mlir:include/mlir/Interfaces/CallInterfaces.td",
+            "//mlir:include/mlir/Interfaces/DataLayoutInterfaces.td",
             "//mlir:include/mlir/Interfaces/InferTypeOpInterface.td",
             "//mlir:include/mlir/Interfaces/SideEffectInterfaces.td",
             "//mlir:include/mlir/Pass/PassBase.td",
diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
index 2cf7df7457cc0..dfb5be5f85027 100644
--- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel
@@ -37,6 +37,8 @@ cc_test(
         "IR/*.cpp",
     ]),
     deps = [
+        "//llvm:Core",
+        "//llvm:Remarks",
         "//llvm:Support",
         "//mlir:BytecodeReader",
         "//mlir:CallOpInterfaces",
@@ -44,6 +46,7 @@ cc_test(
         "//mlir:IR",
         "//mlir:MemRefDialect",
         "//mlir:Parser",
+        "//mlir:RemarkStreamer",
         "//mlir:Support",
         "//mlir/test:TestDialect",
         "//third-party/unittest:gmock",
diff --git a/utils/bazel/third_party_build/zlib-ng.BUILD b/utils/bazel/third_party_build/zlib-ng.BUILD
index 055261acb0f69..2a107208a4817 100644
--- a/utils/bazel/third_party_build/zlib-ng.BUILD
+++ b/utils/bazel/third_party_build/zlib-ng.BUILD
@@ -2,7 +2,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
-load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 
 package(
     default_visibility = ["//visibility:public"],
@@ -20,12 +20,12 @@ config_setting(
     flag_values = {":llvm_enable_zlib": "true"},
 )
 
-genrule(
+copy_file(
     # The input template is identical to the CMake output.
     name = "zconf_gen",
-    srcs = ["zconf.h.in"],
-    outs = ["zconf.h"],
-    cmd = "cp $(SRCS) $(OUTS)",
+    src = "zconf.h.in",
+    out = "zconf.h",
+    allow_symlink = True,
 )
 
 cc_library(